diff --git a/MAINTAINERS b/MAINTAINERS
index 300dbd1b30d7..5772e1db5621 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -123,8 +123,6 @@ usr.sbin/zic		edwin   Heads-up appreciated, since this code is
 				maintained by a third party source.
 lib/libc/stdtime	edwin   Heads-up appreciated, since parts of this code
 				is maintained by a third party source.
-sysinstall	randi	Please contact about any major changes so that
-			they can be co-ordinated.
 sbin/routed	bms	Pre-commit review; notify vendor at rhyolite.com
 
 Following are the entries from the Makefiles, and a few other sources.
diff --git a/Makefile.inc1 b/Makefile.inc1
index 7228f859ce25..07e43018852d 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1018,7 +1018,8 @@ _yacc=		usr.bin/yacc
 _awk=		usr.bin/awk
 .endif
 
-.if ${BOOTSTRAPPING} < 700018
+.if ${MK_BSNMP} != "no" && \
+    (${BOOTSTRAPPING} < 700018 || !exists(/usr/sbin/gensnmptree))
 _gensnmptree=	usr.sbin/bsnmpd/gensnmptree
 .endif
 
@@ -1030,7 +1031,9 @@ _crunchgen=	usr.sbin/crunch/crunchgen
 .if ${MK_CLANG} != "no"
 _clang_tblgen= \
 	lib/clang/libllvmsupport \
-	usr.bin/clang/tblgen
+	lib/clang/libllvmtablegen \
+	usr.bin/clang/tblgen \
+	usr.bin/clang/clang-tblgen
 .endif
 
 .if ${MK_CDDL} != "no" && \
@@ -1106,7 +1109,6 @@ build-tools:
     ${_aicasm} \
     usr.bin/awk \
     lib/libmagic \
-    usr.sbin/sysinstall \
     usr.bin/mkesdb_static \
     usr.bin/mkcsmapper_static
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,build-tools)"; \
@@ -1218,7 +1220,8 @@ _startup_libs+=	lib/libc
 
 gnu/lib/libgcc__L: lib/libc__L
 
-_prebuild_libs=	${_kerberos5_lib_libasn1} ${_kerberos5_lib_libheimntlm} \
+_prebuild_libs=	${_kerberos5_lib_libasn1} ${_kerberos5_lib_libhdb} \
+		${_kerberos5_lib_libheimntlm} \
 		${_kerberos5_lib_libhx509} ${_kerberos5_lib_libkrb5} \
 		${_kerberos5_lib_libroken} \
 		lib/libbz2 lib/libcom_err lib/libcrypt \
@@ -1268,6 +1271,19 @@ secure/lib/libssh__L: lib/libgssapi__L kerberos5/lib/libkrb5__L \
 _secure_lib=	secure/lib
 .endif
 
+.if ${MK_KERBEROS} != "no"
+kerberos5/lib/libasn1__L: lib/libcom_err__L kerberos5/lib/libroken__L
+kerberos5/lib/libhdb__L: kerberos5/lib/libasn1__L lib/libcom_err__L \
+    kerberos5/lib/libkrb5__L kerberos5/lib/libroken__L
+kerberos5/lib/libheimntlm__L: secure/lib/libcrypto__L kerberos5/lib/libkrb5__L
+kerberos5/lib/libhx509__L: kerberos5/lib/libasn1__L lib/libcom_err__L \
+    secure/lib/libcrypto__L kerberos5/lib/libroken__L
+kerberos5/lib/libkrb5__L: kerberos5/lib/libasn1__L lib/libcom_err__L \
+    lib/libcrypt__L secure/lib/libcrypto__L kerberos5/lib/libhx509__L \
+    kerberos5/lib/libroken__L
+kerberos5/lib/libroken__L: lib/libcrypt__L
+.endif
+
 .if ${MK_GSSAPI} != "no"
 _lib_libgssapi=	lib/libgssapi
 .endif
@@ -1279,6 +1295,7 @@ _lib_libipx=	lib/libipx
 .if ${MK_KERBEROS} != "no"
 _kerberos5_lib=	kerberos5/lib
 _kerberos5_lib_libasn1= kerberos5/lib/libasn1
+_kerberos5_lib_libhdb= kerberos5/lib/libhdb
 _kerberos5_lib_libkrb5= kerberos5/lib/libkrb5
 _kerberos5_lib_libhx509= kerberos5/lib/libhx509
 _kerberos5_lib_libroken= kerberos5/lib/libroken
diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index b0670f38fe52..6ca9d8423810 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -38,6 +38,19 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20110930: sysinstall removed
+OLD_FILES+=usr/sbin/sysinstall
+OLD_FILES+=usr/share/man/man8/sysinstall.8.gz
+OLD_FILES+=usr/lib/libftpio.a
+OLD_FILES+=usr/lib/libftpio.so
+OLD_LIBS+=usr/lib/libftpio.so.8
+.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
+OLD_FILES+=usr/lib32/libftpio.a
+OLD_FILES+=usr/lib32/libftpio.so
+OLD_LIBS+=usr/lib32/libftpio.so.8
+.endif
+OLD_FILES+=usr/include/ftpio.h
+OLD_FILES+=usr/share/man/man3/ftpio.3.gz
 # 20110915: rename congestion control manpages
 OLD_FILES+=usr/share/man/man4/cc.4.gz
 OLD_FILES+=usr/share/man/man9/cc.9.gz
@@ -54,7 +67,7 @@ OLD_LIBS+=usr/lib/libdwarf.so.2
 OLD_LIBS+=usr/lib/libopie.so.6
 OLD_LIBS+=usr/lib/librtld_db.so.1
 OLD_LIBS+=usr/lib/libtacplus.so.4
-.if ${TARGET_ARCH} == "amd64"
+.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_LIBS+=usr/lib32/libcam.so.5
 OLD_LIBS+=usr/lib32/libpcap.so.7
 OLD_LIBS+=usr/lib32/libufs.so.5
@@ -92,7 +105,7 @@ OLD_FILES+=usr/lib/libpkg.a
 OLD_FILES+=usr/lib/libpkg.so
 OLD_LIBS+=usr/lib/libpkg.so.0
 OLD_FILES+=usr/lib/libpkg_p.a
-.if ${TARGET_ARCH} == "amd64"
+.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_FILES+=usr/lib32/libpkg.a
 OLD_FILES+=usr/lib32/libpkg.so
 OLD_LIBS+=usr/lib32/libpkg.so.0
@@ -100,7 +113,7 @@ OLD_FILES+=usr/lib32/libpkg_p.a
 .endif
 # 20110517: libsbuf version bump
 OLD_LIBS+=lib/libsbuf.so.5
-.if ${TARGET_ARCH} == "amd64"
+.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_LIBS+=usr/lib32/libsbuf.so.5
 .endif
 # 20110502: new clang import which bumps version from 2.9 to 3.0
@@ -131,7 +144,7 @@ OLD_FILES+=usr/lib/libobjc_p.a
 OLD_FILES+=usr/libexec/cc1obj
 OLD_LIBS+=usr/lib/libobjc.so.4
 OLD_DIRS+=usr/include/objc
-.if ${TARGET_ARCH} == "amd64"
+.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_FILES+=usr/lib32/libobjc.a
 OLD_FILES+=usr/lib32/libobjc.so
 OLD_FILES+=usr/lib32/libobjc_p.a
@@ -258,7 +271,7 @@ OLD_FILES+=usr/include/machine/intr.h
 .endif
 # 20100514: library version bump for versioned symbols for liblzma
 OLD_LIBS+=usr/lib/liblzma.so.0
-.if ${TARGET_ARCH} == "amd64"
+.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_LIBS+=usr/lib32/liblzma.so.0
 .endif
 # 20100511: move GCC-specific headers to /usr/include/gcc
@@ -2580,7 +2593,7 @@ OLD_FILES+=usr/lib/libpam_ssh.a
 OLD_FILES+=usr/lib/libpam_ssh_p.a
 OLD_FILES+=usr/bin/help
 OLD_FILES+=usr/bin/sccs
-.if ${TARGET_ARCH} != "amd64" && ${TARGET_ARCH} != "arm" && ${TARGET_ARCH} != "i386" && ${TARGET_ARCH} != "powerpc"
+.if ${TARGET_ARCH} != "amd64" && ${TARGET} != "arm" && ${TARGET_ARCH} != "i386" && ${TARGET} != "powerpc"
 OLD_FILES+=usr/bin/gdbserver
 .endif
 OLD_FILES+=usr/bin/ssh-keysign
diff --git a/UPDATING b/UPDATING
index dd3513166506..f4c029c43684 100644
--- a/UPDATING
+++ b/UPDATING
@@ -9,8 +9,8 @@ handbook.
 Items affecting the ports and packages system can be found in
 /usr/ports/UPDATING.  Please read that file before running portupgrade.
 
-NOTE TO PEOPLE WHO THINK THAT FreeBSD 9.x IS SLOW:
-	FreeBSD 9.x has many debugging features turned on, in both the kernel
+NOTE TO PEOPLE WHO THINK THAT FreeBSD 10.x IS SLOW:
+	FreeBSD 10.x has many debugging features turned on, in both the kernel
 	and userland.  These features attempt to detect incorrect use of
 	system primitives, and encourage loud failure through extra sanity
 	checking and fail stop semantics.  They also substantially impact
@@ -22,6 +22,13 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 9.x IS SLOW:
 	machines to maximize performance.  (To disable malloc debugging, run
 	ln -s aj /etc/malloc.conf.)
 
+20110930:
+	sysinstall has been removed
+
+20110923:
+	The stable/9 branch created in subversion.  This corresponds to the
+	RELENG_9 branch in CVS.
+
 20110913:
 	This commit modifies vfs_register() so that it uses a hash
 	calculation to set vfc_typenum, which is enabled by default.
@@ -455,7 +462,7 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 9.x IS SLOW:
 	There's one kernel module for each firmware. Adding "device iwnfw"
 	to the kernel configuration file means including all three firmware
 	images inside the kernel. If you want to include just the one for
-	your wireless card, use the the devices iwn4965fw, iwn5000fw or
+	your wireless card, use the devices iwn4965fw, iwn5000fw or
 	iwn5150fw.
 
 20090926:
diff --git a/bin/df/df.1 b/bin/df/df.1
index 8ed6fc6d15cf..a08aef6a19e9 100644
--- a/bin/df/df.1
+++ b/bin/df/df.1
@@ -29,7 +29,7 @@
 .\"     @(#)df.1	8.3 (Berkeley) 5/8/95
 .\" $FreeBSD$
 .\"
-.Dd November 23, 2008
+.Dd October 18, 2011
 .Dt DF 1
 .Os
 .Sh NAME
@@ -63,6 +63,7 @@ The following options are available:
 Show all mount points, including those that were mounted with the
 .Dv MNT_IGNORE
 flag.
+This is implied for file systems specified on the command line.
 .It Fl b
 Use 512-byte blocks rather than the default.
 Note that
diff --git a/bin/df/df.c b/bin/df/df.c
index 73f6acd75dfb..f865b8f1eeee 100644
--- a/bin/df/df.c
+++ b/bin/df/df.c
@@ -107,7 +107,7 @@ main(int argc, char *argv[])
 	const char *fstype;
 	char *mntpath, *mntpt;
 	const char **vfslist;
-	size_t i, mntsize;
+	int i, mntsize;
 	int ch, rv;
 
 	fstype = "ufs";
@@ -187,30 +187,21 @@ main(int argc, char *argv[])
 	argc -= optind;
 	argv += optind;
 
-	mntsize = getmntinfo(&mntbuf, MNT_NOWAIT);
-	bzero(&maxwidths, sizeof(maxwidths));
-	for (i = 0; i < mntsize; i++)
-		update_maxwidths(&maxwidths, &mntbuf[i]);
-
 	rv = 0;
 	if (!*argv) {
+		/* everything (modulo -t) */
+		mntsize = getmntinfo(&mntbuf, MNT_NOWAIT);
 		mntsize = regetmntinfo(&mntbuf, mntsize, vfslist);
-		bzero(&maxwidths, sizeof(maxwidths));
-		for (i = 0; i < mntsize; i++) {
-			if (cflag)
-				addstat(&totalbuf, &mntbuf[i]);
-			update_maxwidths(&maxwidths, &mntbuf[i]);
-		}
-		if (cflag)
-			update_maxwidths(&maxwidths, &totalbuf);
-		for (i = 0; i < mntsize; i++)
-			if (aflag || (mntbuf[i].f_flags & MNT_IGNORE) == 0)
-				prtstat(&mntbuf[i], &maxwidths);
-		if (cflag)
-			prtstat(&totalbuf, &maxwidths);
-		exit(rv);
+	} else {
+		/* just the filesystems specified on the command line */
+		mntbuf = malloc(argc * sizeof(*mntbuf));
+		if (mntbuf == 0)
+			err(1, "malloc()");
+		mntsize = 0;
+		/* continued in for loop below */
 	}
 
+	/* iterate through specified filesystems */
 	for (; *argv; argv++) {
 		if (stat(*argv, &stbuf) < 0) {
 			if ((mntpt = getmntpt(*argv)) == 0) {
@@ -279,14 +270,24 @@ main(int argc, char *argv[])
 			continue;
 		}
 
-		if (argc == 1) {
-			bzero(&maxwidths, sizeof(maxwidths));
-			update_maxwidths(&maxwidths, &statfsbuf);
-		}
-		prtstat(&statfsbuf, &maxwidths);
-		if (cflag)
-			addstat(&totalbuf, &statfsbuf);
+		/* the user asked for it, so ignore the ignore flag */
+		statfsbuf.f_flags &= ~MNT_IGNORE;
+
+		/* add to list */
+		mntbuf[mntsize++] = statfsbuf;
 	}
+
+	bzero(&maxwidths, sizeof(maxwidths));
+	for (i = 0; i < mntsize; i++) {
+		if (aflag || (mntbuf[i].f_flags & MNT_IGNORE) == 0) {
+			update_maxwidths(&maxwidths, &mntbuf[i]);
+			if (cflag)
+				addstat(&totalbuf, &mntbuf[i]);
+		}
+	}
+	for (i = 0; i < mntsize; i++)
+		if (aflag || (mntbuf[i].f_flags & MNT_IGNORE) == 0)
+			prtstat(&mntbuf[i], &maxwidths);
 	if (cflag)
 		prtstat(&totalbuf, &maxwidths);
 	return (rv);
diff --git a/bin/ls/ls.1 b/bin/ls/ls.1
index 3b23df45b88e..cc5ff482187b 100644
--- a/bin/ls/ls.1
+++ b/bin/ls/ls.1
@@ -32,7 +32,7 @@
 .\"     @(#)ls.1	8.7 (Berkeley) 7/29/94
 .\" $FreeBSD$
 .\"
-.Dd April 4, 2008
+.Dd September 28, 2011
 .Dt LS 1
 .Os
 .Sh NAME
@@ -357,8 +357,7 @@ option is given,
 the numeric ID's are displayed.
 .Pp
 If the file is a character special or block special file,
-the major and minor device numbers for the file are displayed
-in the size field.
+the device number for the file is displayed in the size field.
 If the file is a symbolic link the pathname of the
 linked-to file is preceded by
 .Dq Li -> .
diff --git a/bin/ls/ls.c b/bin/ls/ls.c
index e482e2252f04..b6373e20c1f5 100644
--- a/bin/ls/ls.c
+++ b/bin/ls/ls.c
@@ -44,7 +44,7 @@ static char sccsid[] = "@(#)ls.c	8.5 (Berkeley) 4/2/94";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/types.h>
+#include <sys/param.h>
 #include <sys/stat.h>
 #include <sys/ioctl.h>
 #include <sys/mac.h>
@@ -414,8 +414,8 @@ main(int argc, char *argv[])
 		fts_options |= FTS_WHITEOUT;
 #endif
 
-	/* If -l or -s, figure out block size. */
-	if (f_longform || f_size) {
+	/* If -i, -l or -s, figure out block size. */
+	if (f_inode || f_longform || f_size) {
 		if (f_kblocks)
 			blocksize = 2;
 		else {
@@ -563,7 +563,7 @@ display(const FTSENT *p, FTSENT *list, int options)
 	long maxblock;
 	u_long btotal, labelstrlen, maxinode, maxlen, maxnlink;
 	u_long maxlabelstr;
-	u_int devstrlen;
+	u_int sizelen;
 	int maxflags;
 	gid_t maxgroup;
 	uid_t maxuser;
@@ -572,7 +572,6 @@ display(const FTSENT *p, FTSENT *list, int options)
 	int entries, needstats;
 	const char *user, *group;
 	char *flags, *labelstr = NULL;
-	char buf[STRBUF_SIZEOF(u_quad_t) + 1];
 	char ngroup[STRBUF_SIZEOF(uid_t) + 1];
 	char nuser[STRBUF_SIZEOF(gid_t) + 1];
 
@@ -656,7 +655,8 @@ display(const FTSENT *p, FTSENT *list, int options)
 		MAKENINES(maxsize);
 		free(jinitmax);
 	}
-	devstrlen = 0;
+	d.s_size = 0;
+	sizelen = 0;
 	flags = NULL;
 	for (cur = list, entries = 0; cur; cur = cur->fts_link) {
 		if (cur->fts_info == FTS_ERR || cur->fts_info == FTS_NS) {
@@ -796,14 +796,12 @@ display(const FTSENT *p, FTSENT *list, int options)
 				np->group = &np->data[ulen + 1];
 				(void)strcpy(np->group, group);
 
-				if ((S_ISCHR(sp->st_mode) ||
-				    S_ISBLK(sp->st_mode)) &&
-				    devstrlen < DEVSTR_HEX_LEN) {
-					if (minor(sp->st_rdev) > 255 ||
-					    minor(sp->st_rdev) < 0)
-						devstrlen = DEVSTR_HEX_LEN;
-					else
-						devstrlen = DEVSTR_LEN;
+				if (S_ISCHR(sp->st_mode) ||
+				    S_ISBLK(sp->st_mode)) {
+					sizelen = snprintf(NULL, 0,
+					    "%#jx", (uintmax_t)sp->st_rdev);
+					if (d.s_size < sizelen)
+						d.s_size = sizelen;
 				}
 
 				if (f_flags) {
@@ -837,23 +835,16 @@ display(const FTSENT *p, FTSENT *list, int options)
 	d.maxlen = maxlen;
 	if (needstats) {
 		d.btotal = btotal;
-		(void)snprintf(buf, sizeof(buf), "%lu", maxblock);
-		d.s_block = strlen(buf);
+		d.s_block = snprintf(NULL, 0, "%lu", howmany(maxblock, blocksize));
 		d.s_flags = maxflags;
 		d.s_label = maxlabelstr;
 		d.s_group = maxgroup;
-		(void)snprintf(buf, sizeof(buf), "%lu", maxinode);
-		d.s_inode = strlen(buf);
-		(void)snprintf(buf, sizeof(buf), "%lu", maxnlink);
-		d.s_nlink = strlen(buf);
-		if (f_humanval)
-			d.s_size = HUMANVALSTR_LEN;
-		else {
-			(void)snprintf(buf, sizeof(buf), "%ju", maxsize);
-			d.s_size = strlen(buf);
-		}
-		if (d.s_size < devstrlen)
-			d.s_size = devstrlen;
+		d.s_inode = snprintf(NULL, 0, "%lu", maxinode);
+		d.s_nlink = snprintf(NULL, 0, "%lu", maxnlink);
+		sizelen = f_humanval ? HUMANVALSTR_LEN :
+		    snprintf(NULL, 0, "%ju", maxsize);
+		if (d.s_size < sizelen)
+			d.s_size = sizelen;
 		d.s_user = maxuser;
 	}
 	printfcn(&d);
diff --git a/bin/ls/ls.h b/bin/ls/ls.h
index a74abf0c9e5b..ee2a7a5004bb 100644
--- a/bin/ls/ls.h
+++ b/bin/ls/ls.h
@@ -36,8 +36,6 @@
 #define NO_PRINT	1
 
 #define HUMANVALSTR_LEN	5
-#define DEVSTR_LEN	8
-#define DEVSTR_HEX_LEN	15
 
 extern long blocksize;		/* block size units */
 
diff --git a/bin/ls/print.c b/bin/ls/print.c
index 3f2033c404db..a788042144bb 100644
--- a/bin/ls/print.c
+++ b/bin/ls/print.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <langinfo.h>
 #include <libutil.h>
 #include <stdio.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
@@ -351,16 +352,8 @@ printaname(const FTSENT *p, u_long inodefield, u_long sizefield)
 static void
 printdev(size_t width, dev_t dev)
 {
-	char buf[DEVSTR_HEX_LEN + 1];
 
-	if (minor(dev) > 255 || minor(dev) < 0)
-		(void)snprintf(buf, sizeof(buf), "%3d, 0x%08x",
-		    major(dev), (u_int)minor(dev));
-	else
-		(void)snprintf(buf, sizeof(buf), "%3d, %3d",
-		    major(dev), minor(dev));
-
-	(void)printf("%*s ", (u_int)width, buf);
+	(void)printf("%#*jx ", (u_int)width, (uintmax_t)dev);
 }
 
 static void
diff --git a/bin/mv/mv.c b/bin/mv/mv.c
index 1f98f94657f4..67108f7203fb 100644
--- a/bin/mv/mv.c
+++ b/bin/mv/mv.c
@@ -260,40 +260,34 @@ static int
 fastcopy(const char *from, const char *to, struct stat *sbp)
 {
 	struct timeval tval[2];
-	static u_int blen;
-	static char *bp;
+	static u_int blen = MAXPHYS;
+	static char *bp = NULL;
 	mode_t oldmode;
 	int nread, from_fd, to_fd;
 
 	if ((from_fd = open(from, O_RDONLY, 0)) < 0) {
-		warn("%s", from);
+		warn("fastcopy: open() failed (from): %s", from);
 		return (1);
 	}
-	if (blen < sbp->st_blksize) {
-		if (bp != NULL)
-			free(bp);
-		if ((bp = malloc((size_t)sbp->st_blksize)) == NULL) {
-			blen = 0;
-			warnx("malloc failed");
-			return (1);
-		}
-		blen = sbp->st_blksize;
+	if (bp == NULL && (bp = malloc((size_t)blen)) == NULL) {
+		warnx("malloc(%u) failed", blen);
+		return (1);
 	}
 	while ((to_fd =
 	    open(to, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0)) < 0) {
 		if (errno == EEXIST && unlink(to) == 0)
 			continue;
-		warn("%s", to);
+		warn("fastcopy: open() failed (to): %s", to);
 		(void)close(from_fd);
 		return (1);
 	}
 	while ((nread = read(from_fd, bp, (size_t)blen)) > 0)
 		if (write(to_fd, bp, (size_t)nread) != nread) {
-			warn("%s", to);
+			warn("fastcopy: write() failed: %s", to);
 			goto err;
 		}
 	if (nread < 0) {
-		warn("%s", from);
+		warn("fastcopy: read() failed: %s", from);
 err:		if (unlink(to))
 			warn("%s: remove", to);
 		(void)close(from_fd);
diff --git a/bin/ps/extern.h b/bin/ps/extern.h
index 7b5522dc1508..930b85b0bcaa 100644
--- a/bin/ps/extern.h
+++ b/bin/ps/extern.h
@@ -39,63 +39,52 @@ extern fixpt_t ccpu;
 extern int cflag, eval, fscale, nlistread, rawcpu;
 extern unsigned long mempages;
 extern time_t now;
-extern int showthreads, sumrusage, termwidth, totwidth;
+extern int showthreads, sumrusage, termwidth;
 extern STAILQ_HEAD(velisthead, varent) varlist;
 
 __BEGIN_DECLS
-void	 arguments(KINFO *, VARENT *);
-void	 command(KINFO *, VARENT *);
-void	 cputime(KINFO *, VARENT *);
+char	 *arguments(KINFO *, VARENT *);
+char	 *command(KINFO *, VARENT *);
+char	 *cputime(KINFO *, VARENT *);
 int	 donlist(void);
-void	 elapsed(KINFO *, VARENT *);
-void	 elapseds(KINFO *, VARENT *);
-void	 emulname(KINFO *, VARENT *);
+char	 *elapsed(KINFO *, VARENT *);
+char	 *elapseds(KINFO *, VARENT *);
+char	 *emulname(KINFO *, VARENT *);
 VARENT	*find_varentry(VAR *);
 const	 char *fmt_argv(char **, char *, size_t);
 double	 getpcpu(const KINFO *);
-void	 kvar(KINFO *, VARENT *);
-void	 label(KINFO *, VARENT *);
-void	 loginclass(KINFO *, VARENT *);
-void	 logname(KINFO *, VARENT *);
-void	 longtname(KINFO *, VARENT *);
-void	 lstarted(KINFO *, VARENT *);
-void	 maxrss(KINFO *, VARENT *);
-void	 lockname(KINFO *, VARENT *);
-void	 mwchan(KINFO *, VARENT *);
-void	 nwchan(KINFO *, VARENT *);
-void	 pagein(KINFO *, VARENT *);
+char	 *kvar(KINFO *, VARENT *);
+char	 *label(KINFO *, VARENT *);
+char	 *loginclass(KINFO *, VARENT *);
+char	 *logname(KINFO *, VARENT *);
+char	 *longtname(KINFO *, VARENT *);
+char	 *lstarted(KINFO *, VARENT *);
+char	 *maxrss(KINFO *, VARENT *);
+char	 *lockname(KINFO *, VARENT *);
+char	 *mwchan(KINFO *, VARENT *);
+char	 *nwchan(KINFO *, VARENT *);
+char	 *pagein(KINFO *, VARENT *);
 void	 parsefmt(const char *, int);
-void	 pcpu(KINFO *, VARENT *);
-void	 pmem(KINFO *, VARENT *);
-void	 pri(KINFO *, VARENT *);
+char	 *pcpu(KINFO *, VARENT *);
+char	 *pmem(KINFO *, VARENT *);
+char	 *pri(KINFO *, VARENT *);
 void	 printheader(void);
-void	 priorityr(KINFO *, VARENT *);
-void	 egroupname(KINFO *, VARENT *);
-void	 rgroupname(KINFO *, VARENT *);
-void	 runame(KINFO *, VARENT *);
-void	 rvar(KINFO *, VARENT *);
-int	 s_comm(KINFO *);
-int	 s_cputime(KINFO *);
-int	 s_label(KINFO *);
-int	 s_loginclass(KINFO *);
-int	 s_logname(KINFO *);
-int	 s_egroupname(KINFO *);
-int	 s_rgroupname(KINFO *);
-int	 s_runame(KINFO *);
-int	 s_systime(KINFO *);
-int	 s_uname(KINFO *);
-int	 s_usertime(KINFO *);
+char	 *priorityr(KINFO *, VARENT *);
+char	 *egroupname(KINFO *, VARENT *);
+char	 *rgroupname(KINFO *, VARENT *);
+char	 *runame(KINFO *, VARENT *);
+char	 *rvar(KINFO *, VARENT *);
 void	 showkey(void);
-void	 started(KINFO *, VARENT *);
-void	 state(KINFO *, VARENT *);
-void	 systime(KINFO *, VARENT *);
-void	 tdev(KINFO *, VARENT *);
-void	 tdnam(KINFO *, VARENT *);
-void	 tname(KINFO *, VARENT *);
-void	 ucomm(KINFO *, VARENT *);
-void	 uname(KINFO *, VARENT *);
-void	 upr(KINFO *, VARENT *);
-void	 usertime(KINFO *, VARENT *);
-void	 vsize(KINFO *, VARENT *);
-void	 wchan(KINFO *, VARENT *);
+char	 *started(KINFO *, VARENT *);
+char	 *state(KINFO *, VARENT *);
+char	 *systime(KINFO *, VARENT *);
+char	 *tdev(KINFO *, VARENT *);
+char	 *tdnam(KINFO *, VARENT *);
+char	 *tname(KINFO *, VARENT *);
+char	 *ucomm(KINFO *, VARENT *);
+char	 *uname(KINFO *, VARENT *);
+char	 *upr(KINFO *, VARENT *);
+char	 *usertime(KINFO *, VARENT *);
+char	 *vsize(KINFO *, VARENT *);
+char	 *wchan(KINFO *, VARENT *);
 __END_DECLS
diff --git a/bin/ps/keyword.c b/bin/ps/keyword.c
index 7dbaa4fe74cf..17ca0d87c512 100644
--- a/bin/ps/keyword.c
+++ b/bin/ps/keyword.c
@@ -57,178 +57,116 @@ static int  vcmp(const void *, const void *);
 #define	KOFF(x)	offsetof(struct kinfo_proc, x)
 #define	ROFF(x)	offsetof(struct rusage, x)
 
-#define	EMULLEN	13		/* enough for "FreeBSD ELF32" */
 #define	LWPFMT	"d"
-#define	LWPLEN	6
 #define	NLWPFMT	"d"
-#define	NLWPLEN	4
 #define	UIDFMT	"u"
-#define	UIDLEN	5
 #define	PIDFMT	"d"
-#define	PIDLEN	5
-#define USERLEN (MAXLOGNAME - 1)
 
 /* PLEASE KEEP THE TABLE BELOW SORTED ALPHABETICALLY!!! */
 static VAR var[] = {
-	{"%cpu", "%CPU", NULL, 0, pcpu, NULL, 5, 0, CHAR, NULL, 0},
-	{"%mem", "%MEM", NULL, 0, pmem, NULL, 4, 0, CHAR, NULL, 0},
-	{"acflag", "ACFLG", NULL, 0, kvar, NULL, 3, KOFF(ki_acflag), USHORT,
-		"x", 0},
-	{"acflg", "", "acflag", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"args", "COMMAND", NULL, COMM|LJUST|USER, arguments, NULL, 16, 0,
+	{"%cpu", "%CPU", NULL, 0, pcpu, 0, CHAR, NULL, 0},
+	{"%mem", "%MEM", NULL, 0, pmem, 0, CHAR, NULL, 0},
+	{"acflag", "ACFLG", NULL, 0, kvar, KOFF(ki_acflag), USHORT, "x", 0},
+	{"acflg", "", "acflag", 0, NULL, 0, CHAR, NULL, 0},
+	{"args", "COMMAND", NULL, COMM|LJUST|USER, arguments, 0,
 		CHAR, NULL, 0},
-	{"blocked", "", "sigmask", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"caught", "", "sigcatch", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"class", "CLASS", NULL, LJUST, loginclass, s_loginclass,
-		MAXLOGNAME-1, 0, CHAR, NULL, 0},
-	{"comm", "COMMAND", NULL, LJUST, ucomm, s_comm,
-		COMMLEN + TDNAMLEN + 1, 0, CHAR, NULL, 0},
-	{"command", "COMMAND", NULL, COMM|LJUST|USER, command, NULL, 16, 0,
+	{"blocked", "", "sigmask", 0, NULL, 0, CHAR, NULL, 0},
+	{"caught", "", "sigcatch", 0, NULL, 0, CHAR, NULL, 0},
+	{"class", "CLASS", NULL, LJUST, loginclass, 0, CHAR, NULL, 0},
+	{"comm", "COMMAND", NULL, LJUST, ucomm, 0, CHAR, NULL, 0},
+	{"command", "COMMAND", NULL, COMM|LJUST|USER, command, 0,
 		CHAR, NULL, 0},
-	{"cpu", "CPU", NULL, 0, kvar, NULL, 3, KOFF(ki_estcpu), UINT, "d",
-		0},
-	{"cputime", "", "time", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"egid", "", "gid", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"egroup", "", "group", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"emul", "EMUL", NULL, LJUST, emulname, NULL, EMULLEN, 0, CHAR,
-		NULL, 0},
-	{"etime", "ELAPSED", NULL, USER, elapsed, NULL, 12, 0, CHAR, NULL, 0},
-	{"etimes", "ELAPSED", NULL, USER, elapseds, NULL, 12, 0, CHAR, NULL, 0},
-	{"euid", "", "uid", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"f", "F", NULL, 0, kvar, NULL, 8, KOFF(ki_flag), INT, "x", 0},
-	{"flags", "", "f", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"gid", "GID", NULL, 0, kvar, NULL, UIDLEN, KOFF(ki_groups),
-		UINT, UIDFMT, 0},
-	{"group", "GROUP", NULL, LJUST, egroupname, s_egroupname,
-	    USERLEN, 0, CHAR, NULL, 0},
-	{"ignored", "", "sigignore", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"inblk", "INBLK", NULL, USER, rvar, NULL, 4, ROFF(ru_inblock), LONG,
-		"ld", 0},
-	{"inblock", "", "inblk", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"jid", "JID", NULL, 0, kvar, NULL, 6, KOFF(ki_jid), INT, "d", 0},
-	{"jobc", "JOBC", NULL, 0, kvar, NULL, 4, KOFF(ki_jobc), SHORT, "d",
-		0},
-	{"ktrace", "KTRACE", NULL, 0, kvar, NULL, 8, KOFF(ki_traceflag), INT,
-		"x", 0},
-	{"label", "LABEL", NULL, LJUST, label, s_label, SHRT_MAX, 0, CHAR,
-	    NULL, 0},
-	{"lim", "LIM", NULL, 0, maxrss, NULL, 5, 0, CHAR, NULL, 0},
-	{"lockname", "LOCK", NULL, LJUST, lockname, NULL, 6, 0, CHAR, NULL,
-		0},
-	{"login", "LOGIN", NULL, LJUST, logname, s_logname, MAXLOGNAME-1,
-		0, CHAR, NULL, 0},
-	{"logname", "", "login", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"lstart", "STARTED", NULL, LJUST|USER, lstarted, NULL, 28, 0, CHAR,
-		NULL, 0},
-	{"lwp", "LWP", NULL, 0, kvar, NULL, LWPLEN, KOFF(ki_tid), UINT,
-		LWPFMT, 0},
-	{"majflt", "MAJFLT", NULL, USER, rvar, NULL, 4, ROFF(ru_majflt),
-		LONG, "ld", 0},
-	{"minflt", "MINFLT", NULL, USER, rvar, NULL, 4, ROFF(ru_minflt),
-		LONG, "ld", 0},
-	{"msgrcv", "MSGRCV", NULL, USER, rvar, NULL, 4, ROFF(ru_msgrcv),
-		LONG, "ld", 0},
-	{"msgsnd", "MSGSND", NULL, USER, rvar, NULL, 4, ROFF(ru_msgsnd),
-		LONG, "ld", 0},
-	{"mwchan", "MWCHAN", NULL, LJUST, mwchan, NULL, 6, 0, CHAR, NULL, 0},
-	{"ni", "", "nice", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"nice", "NI", NULL, 0, kvar, NULL, 2, KOFF(ki_nice), CHAR, "d",
-		0},
-	{"nivcsw", "NIVCSW", NULL, USER, rvar, NULL, 5, ROFF(ru_nivcsw),
-		LONG, "ld", 0},
-	{"nlwp", "NLWP", NULL, 0, kvar, NULL, NLWPLEN, KOFF(ki_numthreads),
-		UINT, NLWPFMT, 0},
-	{"nsignals", "", "nsigs", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"nsigs", "NSIGS", NULL, USER, rvar, NULL, 4, ROFF(ru_nsignals),
-		LONG, "ld", 0},
-	{"nswap", "NSWAP", NULL, USER, rvar, NULL, 4, ROFF(ru_nswap),
-		LONG, "ld", 0},
-	{"nvcsw", "NVCSW", NULL, USER, rvar, NULL, 5, ROFF(ru_nvcsw),
-		LONG, "ld", 0},
-	{"nwchan", "NWCHAN", NULL, LJUST, nwchan, NULL, sizeof(void *) * 2, 0,
-		CHAR, NULL, 0},
-	{"oublk", "OUBLK", NULL, USER, rvar, NULL, 4, ROFF(ru_oublock),
-		LONG, "ld", 0},
-	{"oublock", "", "oublk", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"paddr", "PADDR", NULL, 0, kvar, NULL, sizeof(void *) * 2,
-		KOFF(ki_paddr), KPTR, "lx", 0},
-	{"pagein", "PAGEIN", NULL, USER, pagein, NULL, 6, 0, CHAR, NULL, 0},
-	{"pcpu", "", "%cpu", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"pending", "", "sig", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"pgid", "PGID", NULL, 0, kvar, NULL, PIDLEN, KOFF(ki_pgid), UINT,
-		PIDFMT, 0},
-	{"pid", "PID", NULL, 0, kvar, NULL, PIDLEN, KOFF(ki_pid), UINT,
-		PIDFMT, 0},
-	{"pmem", "", "%mem", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"ppid", "PPID", NULL, 0, kvar, NULL, PIDLEN, KOFF(ki_ppid), UINT,
-		PIDFMT, 0},
-	{"pri", "PRI", NULL, 0, pri, NULL, 3, 0, CHAR, NULL, 0},
-	{"re", "RE", NULL, INF127, kvar, NULL, 3, KOFF(ki_swtime), UINT, "d",
-		0},
-	{"rgid", "RGID", NULL, 0, kvar, NULL, UIDLEN, KOFF(ki_rgid),
-		UINT, UIDFMT, 0},
-	{"rgroup", "RGROUP", NULL, LJUST, rgroupname, s_rgroupname,
-		USERLEN, 0, CHAR, NULL, 0},
-	{"rss", "RSS", NULL, 0, kvar, NULL, 6, KOFF(ki_rssize), PGTOK, "ld", 0},
-	{"rtprio", "RTPRIO", NULL, 0, priorityr, NULL, 7, KOFF(ki_pri), CHAR,
-		NULL, 0},
-	{"ruid", "RUID", NULL, 0, kvar, NULL, UIDLEN, KOFF(ki_ruid),
-		UINT, UIDFMT, 0},
-	{"ruser", "RUSER", NULL, LJUST, runame, s_runame, USERLEN,
-		0, CHAR, NULL, 0},
-	{"sid", "SID", NULL, 0, kvar, NULL, PIDLEN, KOFF(ki_sid), UINT,
-		PIDFMT, 0},
-	{"sig", "PENDING", NULL, 0, kvar, NULL, 8, KOFF(ki_siglist), INT,
-		"x", 0},
-	{"sigcatch", "CAUGHT", NULL, 0, kvar, NULL, 8, KOFF(ki_sigcatch),
+	{"cpu", "CPU", NULL, 0, kvar, KOFF(ki_estcpu), UINT, "d", 0},
+	{"cputime", "", "time", 0, NULL, 0, CHAR, NULL, 0},
+	{"egid", "", "gid", 0, NULL, 0, CHAR, NULL, 0},
+	{"egroup", "", "group", 0, NULL, 0, CHAR, NULL, 0},
+	{"emul", "EMUL", NULL, LJUST, emulname, 0, CHAR, NULL, 0},
+	{"etime", "ELAPSED", NULL, USER, elapsed, 0, CHAR, NULL, 0},
+	{"etimes", "ELAPSED", NULL, USER, elapseds, 0, CHAR, NULL, 0},
+	{"euid", "", "uid", 0, NULL, 0, CHAR, NULL, 0},
+	{"f", "F", NULL, 0, kvar, KOFF(ki_flag), INT, "x", 0},
+	{"flags", "", "f", 0, NULL, 0, CHAR, NULL, 0},
+	{"gid", "GID", NULL, 0, kvar, KOFF(ki_groups), UINT, UIDFMT, 0},
+	{"group", "GROUP", NULL, LJUST, egroupname, 0, CHAR, NULL, 0},
+	{"ignored", "", "sigignore", 0, NULL, 0, CHAR, NULL, 0},
+	{"inblk", "INBLK", NULL, USER, rvar, ROFF(ru_inblock), LONG, "ld", 0},
+	{"inblock", "", "inblk", 0, NULL, 0, CHAR, NULL, 0},
+	{"jid", "JID", NULL, 0, kvar, KOFF(ki_jid), INT, "d", 0},
+	{"jobc", "JOBC", NULL, 0, kvar, KOFF(ki_jobc), SHORT, "d", 0},
+	{"ktrace", "KTRACE", NULL, 0, kvar, KOFF(ki_traceflag), INT, "x", 0},
+	{"label", "LABEL", NULL, LJUST, label, 0, CHAR, NULL, 0},
+	{"lim", "LIM", NULL, 0, maxrss, 0, CHAR, NULL, 0},
+	{"lockname", "LOCK", NULL, LJUST, lockname, 0, CHAR, NULL, 0},
+	{"login", "LOGIN", NULL, LJUST, logname, 0, CHAR, NULL, 0},
+	{"logname", "", "login", 0, NULL, 0, CHAR, NULL, 0},
+	{"lstart", "STARTED", NULL, LJUST|USER, lstarted, 0, CHAR, NULL, 0},
+	{"lwp", "LWP", NULL, 0, kvar, KOFF(ki_tid), UINT, LWPFMT, 0},
+	{"majflt", "MAJFLT", NULL, USER, rvar, ROFF(ru_majflt), LONG, "ld", 0},
+	{"minflt", "MINFLT", NULL, USER, rvar, ROFF(ru_minflt), LONG, "ld", 0},
+	{"msgrcv", "MSGRCV", NULL, USER, rvar, ROFF(ru_msgrcv), LONG, "ld", 0},
+	{"msgsnd", "MSGSND", NULL, USER, rvar, ROFF(ru_msgsnd), LONG, "ld", 0},
+	{"mwchan", "MWCHAN", NULL, LJUST, mwchan, 0, CHAR, NULL, 0},
+	{"ni", "", "nice", 0, NULL, 0, CHAR, NULL, 0},
+	{"nice", "NI", NULL, 0, kvar, KOFF(ki_nice), CHAR, "d", 0},
+	{"nivcsw", "NIVCSW", NULL, USER, rvar, ROFF(ru_nivcsw), LONG, "ld", 0},
+	{"nlwp", "NLWP", NULL, 0, kvar, KOFF(ki_numthreads), UINT, NLWPFMT, 0},
+	{"nsignals", "", "nsigs", 0, NULL, 0, CHAR, NULL, 0},
+	{"nsigs", "NSIGS", NULL, USER, rvar, ROFF(ru_nsignals), LONG, "ld", 0},
+	{"nswap", "NSWAP", NULL, USER, rvar, ROFF(ru_nswap), LONG, "ld", 0},
+	{"nvcsw", "NVCSW", NULL, USER, rvar, ROFF(ru_nvcsw), LONG, "ld", 0},
+	{"nwchan", "NWCHAN", NULL, LJUST, nwchan, 0, CHAR, NULL, 0},
+	{"oublk", "OUBLK", NULL, USER, rvar, ROFF(ru_oublock), LONG, "ld", 0},
+	{"oublock", "", "oublk", 0, NULL, 0, CHAR, NULL, 0},
+	{"paddr", "PADDR", NULL, 0, kvar, KOFF(ki_paddr), KPTR, "lx", 0},
+	{"pagein", "PAGEIN", NULL, USER, pagein, 0, CHAR, NULL, 0},
+	{"pcpu", "", "%cpu", 0, NULL, 0, CHAR, NULL, 0},
+	{"pending", "", "sig", 0, NULL, 0, CHAR, NULL, 0},
+	{"pgid", "PGID", NULL, 0, kvar, KOFF(ki_pgid), UINT, PIDFMT, 0},
+	{"pid", "PID", NULL, 0, kvar, KOFF(ki_pid), UINT, PIDFMT, 0},
+	{"pmem", "", "%mem", 0, NULL, 0, CHAR, NULL, 0},
+	{"ppid", "PPID", NULL, 0, kvar, KOFF(ki_ppid), UINT, PIDFMT, 0},
+	{"pri", "PRI", NULL, 0, pri, 0, CHAR, NULL, 0},
+	{"re", "RE", NULL, INF127, kvar, KOFF(ki_swtime), UINT, "d", 0},
+	{"rgid", "RGID", NULL, 0, kvar, KOFF(ki_rgid), UINT, UIDFMT, 0},
+	{"rgroup", "RGROUP", NULL, LJUST, rgroupname, 0, CHAR, NULL, 0},
+	{"rss", "RSS", NULL, 0, kvar, KOFF(ki_rssize), PGTOK, "ld", 0},
+	{"rtprio", "RTPRIO", NULL, 0, priorityr, KOFF(ki_pri), CHAR, NULL, 0},
+	{"ruid", "RUID", NULL, 0, kvar, KOFF(ki_ruid), UINT, UIDFMT, 0},
+	{"ruser", "RUSER", NULL, LJUST, runame, 0, CHAR, NULL, 0},
+	{"sid", "SID", NULL, 0, kvar, KOFF(ki_sid), UINT, PIDFMT, 0},
+	{"sig", "PENDING", NULL, 0, kvar, KOFF(ki_siglist), INT, "x", 0},
+	{"sigcatch", "CAUGHT", NULL, 0, kvar, KOFF(ki_sigcatch), UINT, "x", 0},
+	{"sigignore", "IGNORED", NULL, 0, kvar, KOFF(ki_sigignore),
 		UINT, "x", 0},
-	{"sigignore", "IGNORED", NULL, 0, kvar, NULL, 8, KOFF(ki_sigignore),
-		UINT, "x", 0},
-	{"sigmask", "BLOCKED", NULL, 0, kvar, NULL, 8, KOFF(ki_sigmask),
-		UINT, "x", 0},
-	{"sl", "SL", NULL, INF127, kvar, NULL, 3, KOFF(ki_slptime), UINT, "d",
-		0},
-	{"start", "STARTED", NULL, LJUST|USER, started, NULL, 7, 0, CHAR, NULL,
-		0},
-	{"stat", "", "state", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"state", "STAT", NULL, 0, state, NULL, 4, 0, CHAR, NULL, 0},
-	{"svgid", "SVGID", NULL, 0, kvar, NULL, UIDLEN, KOFF(ki_svgid),
-		UINT, UIDFMT, 0},
-	{"svuid", "SVUID", NULL, 0, kvar, NULL, UIDLEN, KOFF(ki_svuid),
-		UINT, UIDFMT, 0},
-	{"systime", "SYSTIME", NULL, USER, systime, s_systime, 15, 0, CHAR,
-		NULL, 0},
-	{"tdaddr", "TDADDR", NULL, 0, kvar, NULL, sizeof(void *) * 2,
-		KOFF(ki_tdaddr), KPTR, "lx", 0},
-	{"tdev", "TDEV", NULL, 0, tdev, NULL, 5, 0, CHAR, NULL, 0},
-	{"tdnam", "TDNAM", NULL, LJUST, tdnam, NULL, COMMLEN, 0, CHAR, NULL, 0},
-	{"time", "TIME", NULL, USER, cputime, s_cputime, 15, 0, CHAR,
-		NULL, 0},
-	{"tpgid", "TPGID", NULL, 0, kvar, NULL, 4, KOFF(ki_tpgid), UINT,
-		PIDFMT, 0},
-	{"tsid", "TSID", NULL, 0, kvar, NULL, PIDLEN, KOFF(ki_tsid), UINT,
-		PIDFMT, 0},
-	{"tsiz", "TSIZ", NULL, 0, kvar, NULL, 4, KOFF(ki_tsize), PGTOK, "ld", 0},
-	{"tt", "TT ", NULL, 0, tname, NULL, 4, 0, CHAR, NULL, 0},
-	{"tty", "TTY", NULL, LJUST, longtname, NULL, 8, 0, CHAR, NULL, 0},
-	{"ucomm", "UCOMM", NULL, LJUST, ucomm, s_comm,
-		COMMLEN + TDNAMLEN + 1, 0, CHAR, NULL, 0},
-	{"uid", "UID", NULL, 0, kvar, NULL, UIDLEN, KOFF(ki_uid), UINT,
-		UIDFMT, 0},
-	{"upr", "UPR", NULL, 0, upr, NULL, 3, 0, CHAR, NULL, 0},
-	{"uprocp", "UPROCP", NULL, 0, kvar, NULL, sizeof(void *) * 2,
-		KOFF(ki_paddr), KPTR, "lx", 0},
-	{"user", "USER", NULL, LJUST, uname, s_uname, USERLEN, 0, CHAR,
-		NULL, 0},
-	{"usertime", "USERTIME", NULL, USER, usertime, s_usertime, 15, 0,
-		CHAR, NULL, 0},
-	{"usrpri", "", "upr", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"vsize", "", "vsz", 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
-	{"vsz", "VSZ", NULL, 0, vsize, NULL, 6, 0, CHAR, NULL, 0},
-	{"wchan", "WCHAN", NULL, LJUST, wchan, NULL, 6, 0, CHAR, NULL, 0},
-	{"xstat", "XSTAT", NULL, 0, kvar, NULL, 4, KOFF(ki_xstat), USHORT,
-		"x", 0},
-	{"", NULL, NULL, 0, NULL, NULL, 0, 0, CHAR, NULL, 0},
+	{"sigmask", "BLOCKED", NULL, 0, kvar, KOFF(ki_sigmask), UINT, "x", 0},
+	{"sl", "SL", NULL, INF127, kvar, KOFF(ki_slptime), UINT, "d", 0},
+	{"start", "STARTED", NULL, LJUST|USER, started, 0, CHAR, NULL, 0},
+	{"stat", "", "state", 0, NULL, 0, CHAR, NULL, 0},
+	{"state", "STAT", NULL, LJUST, state, 0, CHAR, NULL, 0},
+	{"svgid", "SVGID", NULL, 0, kvar, KOFF(ki_svgid), UINT, UIDFMT, 0},
+	{"svuid", "SVUID", NULL, 0, kvar, KOFF(ki_svuid), UINT, UIDFMT, 0},
+	{"systime", "SYSTIME", NULL, USER, systime, 0, CHAR, NULL, 0},
+	{"tdaddr", "TDADDR", NULL, 0, kvar, KOFF(ki_tdaddr), KPTR, "lx", 0},
+	{"tdev", "TDEV", NULL, 0, tdev, 0, CHAR, NULL, 0},
+	{"tdnam", "TDNAM", NULL, LJUST, tdnam, 0, CHAR, NULL, 0},
+	{"time", "TIME", NULL, USER, cputime, 0, CHAR, NULL, 0},
+	{"tpgid", "TPGID", NULL, 0, kvar, KOFF(ki_tpgid), UINT, PIDFMT, 0},
+	{"tsid", "TSID", NULL, 0, kvar, KOFF(ki_tsid), UINT, PIDFMT, 0},
+	{"tsiz", "TSIZ", NULL, 0, kvar, KOFF(ki_tsize), PGTOK, "ld", 0},
+	{"tt", "TT ", NULL, 0, tname, 0, CHAR, NULL, 0},
+	{"tty", "TTY", NULL, LJUST, longtname, 0, CHAR, NULL, 0},
+	{"ucomm", "UCOMM", NULL, LJUST, ucomm, 0, CHAR, NULL, 0},
+	{"uid", "UID", NULL, 0, kvar, KOFF(ki_uid), UINT, UIDFMT, 0},
+	{"upr", "UPR", NULL, 0, upr, 0, CHAR, NULL, 0},
+	{"uprocp", "UPROCP", NULL, 0, kvar, KOFF(ki_paddr), KPTR, "lx", 0},
+	{"user", "USER", NULL, LJUST, uname, 0, CHAR, NULL, 0},
+	{"usertime", "USERTIME", NULL, USER, usertime, 0, CHAR, NULL, 0},
+	{"usrpri", "", "upr", 0, NULL, 0, CHAR, NULL, 0},
+	{"vsize", "", "vsz", 0, NULL, 0, CHAR, NULL, 0},
+	{"vsz", "VSZ", NULL, 0, vsize, 0, CHAR, NULL, 0},
+	{"wchan", "WCHAN", NULL, LJUST, wchan, 0, CHAR, NULL, 0},
+	{"xstat", "XSTAT", NULL, 0, kvar, KOFF(ki_xstat), USHORT, "x", 0},
+	{"", NULL, NULL, 0, NULL, 0, CHAR, NULL, 0},
 };
 
 void
diff --git a/bin/ps/print.c b/bin/ps/print.c
index 7d1d1902b9b3..45c7823056ac 100644
--- a/bin/ps/print.c
+++ b/bin/ps/print.c
@@ -64,6 +64,9 @@ __FBSDID("$FreeBSD$");
 
 #include "ps.h"
 
+#define	COMMAND_WIDTH	16
+#define	ARGUMENTS_WIDTH	16
+
 #define	ps_pgtok(a)	(((a) * getpagesize()) / 1024)
 
 void
@@ -93,53 +96,42 @@ printheader(void)
 	(void)putchar('\n');
 }
 
-void
+char *
 arguments(KINFO *k, VARENT *ve)
 {
 	VAR *v;
-	int left;
-	char *cp, *vis_args;
+	char *vis_args;
 
 	v = ve->var;
 	if ((vis_args = malloc(strlen(k->ki_args) * 4 + 1)) == NULL)
 		errx(1, "malloc failed");
 	strvis(vis_args, k->ki_args, VIS_TAB | VIS_NL | VIS_NOSLASH);
-	if (STAILQ_NEXT(ve, next_ve) == NULL) {
-		/* last field */
-		if (termwidth == UNLIMITED) {
-			(void)printf("%s", vis_args);
-		} else {
-			left = termwidth - (totwidth - v->width);
-			if (left < 1) /* already wrapped, just use std width */
-				left = v->width;
-			for (cp = vis_args; --left >= 0 && *cp != '\0';)
-				(void)putchar(*cp++);
-		}
-	} else {
-		(void)printf("%-*.*s", v->width, v->width, vis_args);
-	}
-	free(vis_args);
+
+	if (STAILQ_NEXT(ve, next_ve) != NULL && strlen(vis_args) > ARGUMENTS_WIDTH)
+		vis_args[ARGUMENTS_WIDTH] = '\0';
+
+	return (vis_args);
 }
 
-void
+char *
 command(KINFO *k, VARENT *ve)
 {
 	VAR *v;
-	int left;
-	char *cp, *vis_env, *vis_args;
+	char *vis_args, *vis_env, *str;
 
 	v = ve->var;
 	if (cflag) {
 		/* If it is the last field, then don't pad */
 		if (STAILQ_NEXT(ve, next_ve) == NULL) {
-			if (k->ki_d.prefix)
-				(void)printf("%s", k->ki_d.prefix);
-			(void)printf("%s", k->ki_p->ki_comm);
-			if (showthreads && k->ki_p->ki_numthreads > 1)
-				(void)printf("/%s", k->ki_p->ki_tdname);
+			asprintf(&str, "%s%s%s%s",
+			    k->ki_d.prefix ? k->ki_d.prefix : "",
+			    k->ki_p->ki_comm,
+			    (showthreads && k->ki_p->ki_numthreads > 1) ? "/" : "",
+			    (showthreads && k->ki_p->ki_numthreads > 1) ? k->ki_p->ki_tdname : "");
 		} else
-			(void)printf("%-*s", v->width, k->ki_p->ki_comm);
-		return;
+			str = strdup(k->ki_p->ki_comm);
+
+		return (str);
 	}
 	if ((vis_args = malloc(strlen(k->ki_args) * 4 + 1)) == NULL)
 		errx(1, "malloc failed");
@@ -157,89 +149,84 @@ command(KINFO *k, VARENT *ve)
 		} else
 			vis_env = NULL;
 
-		if (termwidth == UNLIMITED) {
-			if (k->ki_d.prefix)
-				(void)printf("%s", k->ki_d.prefix);
-			if (vis_env)
-				(void)printf("%s ", vis_env);
-			(void)printf("%s", vis_args);
-		} else {
-			left = termwidth - (totwidth - v->width);
-			if (left < 1) /* already wrapped, just use std width */
-				left = v->width;
-			if ((cp = k->ki_d.prefix) != NULL)
-				while (--left >= 0 && *cp)
-					(void)putchar(*cp++);
-			if ((cp = vis_env) != NULL) {
-				while (--left >= 0 && *cp)
-					(void)putchar(*cp++);
-				if (--left >= 0)
-					putchar(' ');
-			}
-			for (cp = vis_args; --left >= 0 && *cp != '\0';)
-				(void)putchar(*cp++);
-		}
+		asprintf(&str, "%s%s%s%s",
+		    k->ki_d.prefix ? k->ki_d.prefix : "",
+		    vis_env ? vis_env : "",
+		    vis_env ? " " : "",
+		    vis_args);
+
 		if (vis_env != NULL)
 			free(vis_env);
-	} else
+		free(vis_args);
+	} else {
 		/* ki_d.prefix & ki_env aren't shown for interim fields */
-		(void)printf("%-*.*s", v->width, v->width, vis_args);
-	free(vis_args);
+		str = vis_args;
+
+		if (strlen(str) > COMMAND_WIDTH)
+			str[COMMAND_WIDTH] = '\0';
+	}
+
+	return (str);
 }
 
-void
+char *
 ucomm(KINFO *k, VARENT *ve)
 {
-	char tmpbuff[COMMLEN + TDNAMLEN + 2];
 	VAR *v;
+	char *str;
 
 	v = ve->var;
 	if (STAILQ_NEXT(ve, next_ve) == NULL) {	/* last field, don't pad */
-		if (k->ki_d.prefix)
-			(void)printf("%s", k->ki_d.prefix);
-		(void)printf("%s", k->ki_p->ki_comm);
-		if (showthreads && k->ki_p->ki_numthreads > 1)
-			printf("/%s", k->ki_p->ki_tdname);
+		asprintf(&str, "%s%s%s%s",
+		    k->ki_d.prefix ? k->ki_d.prefix : "",
+		    k->ki_p->ki_comm,
+		    (showthreads && k->ki_p->ki_numthreads > 1) ? "/" : "",
+		    (showthreads && k->ki_p->ki_numthreads > 1) ? k->ki_p->ki_tdname : "");
 	} else {
-		bzero(tmpbuff, sizeof(tmpbuff));
 		if (showthreads && k->ki_p->ki_numthreads > 1)
-			sprintf(tmpbuff, "%s/%s", k->ki_p->ki_comm,
-			    k->ki_p->ki_tdname);
+			asprintf(&str, "%s/%s", k->ki_p->ki_comm, k->ki_p->ki_tdname);
 		else
-			sprintf(tmpbuff, "%s", k->ki_p->ki_comm);
-		(void)printf("%-*s", v->width, tmpbuff);
+			str = strdup(k->ki_p->ki_comm);
 	}
+	return (str);
 }
 
-void
+char *
 tdnam(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
 	if (showthreads && k->ki_p->ki_numthreads > 1)
-		(void)printf("%-*s", v->width, k->ki_p->ki_tdname);
+		str = strdup(k->ki_p->ki_tdname);
 	else
-		(void)printf("%-*s", v->width, "      ");
+		str = strdup("      ");
+
+	return (str);
 }
 
-void
+char *
 logname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
-	char *s;
 
 	v = ve->var;
-	(void)printf("%-*s", v->width, (s = k->ki_p->ki_login, *s) ? s : "-");
+	if (*k->ki_p->ki_login == '\0')
+		return (NULL);
+	return (strdup(k->ki_p->ki_login));
 }
 
-void
+char *
 state(KINFO *k, VARENT *ve)
 {
 	int flag, tdflags;
-	char *cp;
+	char *cp, *buf;
 	VAR *v;
-	char buf[16];
+
+	buf = malloc(16);
+	if (buf == NULL)
+		errx(1, "malloc failed");
 
 	v = ve->var;
 	flag = k->ki_p->ki_flag;
@@ -301,278 +288,270 @@ state(KINFO *k, VARENT *ve)
 	if (flag & P_JAILED)
 		*cp++ = 'J';
 	*cp = '\0';
-	(void)printf("%-*s", v->width, buf);
+	return (buf);
 }
 
 #define	scalepri(x)	((x) - PZERO)
 
-void
+char *
 pri(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	(void)printf("%*d", v->width, scalepri(k->ki_p->ki_pri.pri_level));
+	asprintf(&str, "%d", scalepri(k->ki_p->ki_pri.pri_level));
+	return (str);
 }
 
-void
+char *
 upr(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	(void)printf("%*d", v->width, scalepri(k->ki_p->ki_pri.pri_user));
+	asprintf(&str, "%d", scalepri(k->ki_p->ki_pri.pri_user));
+	return (str);
 }
 #undef scalepri
 
-void
+char *
 uname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	(void)printf("%-*s", v->width, user_from_uid(k->ki_p->ki_uid, 0));
+	return (strdup(user_from_uid(k->ki_p->ki_uid, 0)));
 }
 
-int
-s_uname(KINFO *k)
-{
-	return (strlen(user_from_uid(k->ki_p->ki_uid, 0)));
-}
-
-void
+char *
 egroupname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	(void)printf("%-*s", v->width,
-	    group_from_gid(k->ki_p->ki_groups[0], 0));
+	return (strdup(group_from_gid(k->ki_p->ki_groups[0], 0)));
 }
 
-int
-s_egroupname(KINFO *k)
-{
-	return (strlen(group_from_gid(k->ki_p->ki_groups[0], 0)));
-}
-
-void
+char *
 rgroupname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	(void)printf("%-*s", v->width, group_from_gid(k->ki_p->ki_rgid, 0));
+	return (strdup(group_from_gid(k->ki_p->ki_rgid, 0)));
 }
 
-int
-s_rgroupname(KINFO *k)
-{
-	return (strlen(group_from_gid(k->ki_p->ki_rgid, 0)));
-}
-
-void
+char *
 runame(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	(void)printf("%-*s", v->width, user_from_uid(k->ki_p->ki_ruid, 0));
+	return (strdup(user_from_uid(k->ki_p->ki_ruid, 0)));
 }
 
-int
-s_runame(KINFO *k)
-{
-	return (strlen(user_from_uid(k->ki_p->ki_ruid, 0)));
-}
-
-
-void
+char *
 tdev(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	dev_t dev;
-	char buff[16];
+	char *str;
 
 	v = ve->var;
 	dev = k->ki_p->ki_tdev;
 	if (dev == NODEV)
-		(void)printf("%*s", v->width, "??");
-	else {
-		(void)snprintf(buff, sizeof(buff),
-		    "%d/%d", major(dev), minor(dev));
-		(void)printf("%*s", v->width, buff);
-	}
+		str = strdup("??");
+	else
+		asprintf(&str, "%#jx", (uintmax_t)dev);
+
+	return (str);
 }
 
-void
+char *
 tname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	dev_t dev;
-	char *ttname;
+	char *ttname, *str;
 
 	v = ve->var;
 	dev = k->ki_p->ki_tdev;
 	if (dev == NODEV || (ttname = devname(dev, S_IFCHR)) == NULL)
-		(void)printf("%*s ", v->width - 1, "??");
+		str = strdup("?? ");
 	else {
 		if (strncmp(ttname, "tty", 3) == 0 ||
 		    strncmp(ttname, "cua", 3) == 0)
 			ttname += 3;
 		if (strncmp(ttname, "pts/", 4) == 0)
 			ttname += 4;
-		(void)printf("%*.*s%c", v->width - 1, v->width - 1, ttname,
+		asprintf(&str, "%s%c", ttname,
 		    k->ki_p->ki_kiflag & KI_CTTY ? ' ' : '-');
 	}
+
+	return (str);
 }
 
-void
+char *
 longtname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	dev_t dev;
-	char *ttname;
+	const char *ttname;
 
 	v = ve->var;
 	dev = k->ki_p->ki_tdev;
 	if (dev == NODEV || (ttname = devname(dev, S_IFCHR)) == NULL)
-		(void)printf("%-*s", v->width, "??");
-	else
-		(void)printf("%-*s", v->width, ttname);
+		ttname = "??";
+
+	return (strdup(ttname));
 }
 
-void
+char *
 started(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	time_t then;
 	struct tm *tp;
 	static int use_ampm = -1;
-	char buf[100];
+	size_t buflen = 100;
+	char *buf;
+
+	buf = malloc(buflen);
+	if (buf == NULL)
+		errx(1, "malloc failed");
 
 	v = ve->var;
-	if (!k->ki_valid) {
-		(void)printf("%-*s", v->width, "-");
-		return;
-	}
+	if (!k->ki_valid)
+		return (NULL);
 	if (use_ampm < 0)
 		use_ampm = (*nl_langinfo(T_FMT_AMPM) != '\0');
 	then = k->ki_p->ki_start.tv_sec;
 	tp = localtime(&then);
 	if (now - k->ki_p->ki_start.tv_sec < 24 * 3600) {
-		(void)strftime(buf, sizeof(buf),
+		(void)strftime(buf, buflen,
 		    use_ampm ? "%l:%M%p" : "%k:%M  ", tp);
 	} else if (now - k->ki_p->ki_start.tv_sec < 7 * 86400) {
-		(void)strftime(buf, sizeof(buf),
+		(void)strftime(buf, buflen,
 		    use_ampm ? "%a%I%p" : "%a%H  ", tp);
 	} else
-		(void)strftime(buf, sizeof(buf), "%e%b%y", tp);
-	(void)printf("%-*s", v->width, buf);
+		(void)strftime(buf, buflen, "%e%b%y", tp);
+	return (buf);
 }
 
-void
+char *
 lstarted(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	time_t then;
-	char buf[100];
+	char *buf;
+	size_t buflen = 100;
+
+	buf = malloc(buflen);
+	if (buf == NULL)
+		errx(1, "malloc failed");
 
 	v = ve->var;
-	if (!k->ki_valid) {
-		(void)printf("%-*s", v->width, "-");
-		return;
-	}
+	if (!k->ki_valid)
+		return (NULL);
 	then = k->ki_p->ki_start.tv_sec;
-	(void)strftime(buf, sizeof(buf), "%c", localtime(&then));
-	(void)printf("%-*s", v->width, buf);
+	(void)strftime(buf, buflen, "%c", localtime(&then));
+	return (buf);
 }
 
-void
+char *
 lockname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
 	if (k->ki_p->ki_kiflag & KI_LOCKBLOCK) {
 		if (k->ki_p->ki_lockname[0] != 0)
-			(void)printf("%-*.*s", v->width, v->width,
-			    k->ki_p->ki_lockname);
+			str = strdup(k->ki_p->ki_lockname);
 		else
-			(void)printf("%-*s", v->width, "???");
+			str = strdup("???");
 	} else
-		(void)printf("%-*s", v->width, "-");
+		str = NULL;
+
+	return (str);
 }
 
-void
+char *
 wchan(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
 	if (k->ki_p->ki_wchan) {
 		if (k->ki_p->ki_wmesg[0] != 0)
-			(void)printf("%-*.*s", v->width, v->width,
-			    k->ki_p->ki_wmesg);
+			str = strdup(k->ki_p->ki_wmesg);
 		else
-			(void)printf("%-*lx", v->width,
-			    (long)k->ki_p->ki_wchan);
+			asprintf(&str, "%lx", (long)k->ki_p->ki_wchan);
 	} else
-		(void)printf("%-*s", v->width, "-");
+		str = NULL;
+
+	return (str);
 }
 
-void
+char *
 nwchan(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	if (k->ki_p->ki_wchan) {
-		(void)printf("%0*lx", v->width,
-		    (long)k->ki_p->ki_wchan);
-	} else
-		(void)printf("%-*s", v->width, "-");
+	if (k->ki_p->ki_wchan)
+		asprintf(&str, "%0lx", (long)k->ki_p->ki_wchan);
+	else
+		str = NULL;
+
+	return (str);
 }
 
-void
+char *
 mwchan(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
 	if (k->ki_p->ki_wchan) {
 		if (k->ki_p->ki_wmesg[0] != 0)
-			(void)printf("%-*.*s", v->width, v->width,
-			    k->ki_p->ki_wmesg);
+			str = strdup(k->ki_p->ki_wmesg);
 		else
-			(void)printf("%-*lx", v->width,
-			    (long)k->ki_p->ki_wchan);
+                        asprintf(&str, "%lx", (long)k->ki_p->ki_wchan);
 	} else if (k->ki_p->ki_kiflag & KI_LOCKBLOCK) {
 		if (k->ki_p->ki_lockname[0]) {
-			(void)printf("%-*.*s", v->width, v->width,
-			    k->ki_p->ki_lockname);
+			str = strdup(k->ki_p->ki_lockname);
 		} else
-			(void)printf("%-*s", v->width, "???");
+			str = strdup("???");
 	} else
-		(void)printf("%-*s", v->width, "-");
+		str = NULL;
+
+	return (str);
 }
 
-void
+char *
 vsize(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	(void)printf("%*lu", v->width, (u_long)(k->ki_p->ki_size / 1024));
+	asprintf(&str, "%lu", (u_long)(k->ki_p->ki_size / 1024));
+	return (str);
 }
 
-static void
+static char *
 printtime(KINFO *k, VARENT *ve, long secs, long psecs)
 /* psecs is "parts" of a second. first micro, then centi */
 {
 	VAR *v;
-	char obuff[128];
 	static char decimal_point;
+	char *str;
 
 	if (decimal_point == '\0')
 		decimal_point = localeconv()->decimal_point[0];
@@ -586,21 +565,12 @@ printtime(KINFO *k, VARENT *ve, long secs, long psecs)
 		secs += psecs / 100;
 		psecs = psecs % 100;
 	}
-	(void)snprintf(obuff, sizeof(obuff), "%ld:%02ld%c%02ld",
+	asprintf(&str, "%ld:%02ld%c%02ld",
 	    secs / 60, secs % 60, decimal_point, psecs);
-	(void)printf("%*s", v->width, obuff);
+	return (str);
 }
 
-static int
-sizetime(long secs)
-{
-
-	if (secs < 60)
-		return (7);
-	return (log10(secs / 60) + 7);
-}
-
-void
+char *
 cputime(KINFO *k, VARENT *ve)
 {
 	long secs, psecs;
@@ -616,10 +586,10 @@ cputime(KINFO *k, VARENT *ve)
 		secs += k->ki_p->ki_childtime.tv_sec;
 		psecs += k->ki_p->ki_childtime.tv_usec;
 	}
-	printtime(k, ve, secs, psecs);
+	return (printtime(k, ve, secs, psecs));
 }
 
-void
+char *
 systime(KINFO *k, VARENT *ve)
 {
 	long secs, psecs;
@@ -630,10 +600,10 @@ systime(KINFO *k, VARENT *ve)
 		secs += k->ki_p->ki_childstime.tv_sec;
 		psecs += k->ki_p->ki_childstime.tv_usec;
 	}
-	printtime(k, ve, secs, psecs);
+	return (printtime(k, ve, secs, psecs));
 }
 
-void
+char *
 usertime(KINFO *k, VARENT *ve)
 {
 	long secs, psecs;
@@ -644,22 +614,20 @@ usertime(KINFO *k, VARENT *ve)
 		secs += k->ki_p->ki_childutime.tv_sec;
 		psecs += k->ki_p->ki_childutime.tv_usec;
 	}
-	printtime(k, ve, secs, psecs);
+	return (printtime(k, ve, secs, psecs));
 }
 
-void
+char *
 elapsed(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	time_t val;
 	int days, hours, mins, secs;
-	char obuff[128];
+	char *str;
 
 	v = ve->var;
-	if (!k->ki_valid) {
-		(void)printf("%-*s", v->width, "-");
-		return;
-	}
+	if (!k->ki_valid)
+		return (NULL);
 	val = now - k->ki_p->ki_start.tv_sec;
 	days = val / (24 * 60 * 60);
 	val %= 24 * 60 * 60;
@@ -668,29 +636,28 @@ elapsed(KINFO *k, VARENT *ve)
 	mins = val / 60;
 	secs = val % 60;
 	if (days != 0)
-		(void)snprintf(obuff, sizeof(obuff), "%3d-%02d:%02d:%02d",
-		    days, hours, mins, secs);
+		asprintf(&str, "%3d-%02d:%02d:%02d", days, hours, mins, secs);
 	else if (hours != 0)
-		(void)snprintf(obuff, sizeof(obuff), "%02d:%02d:%02d",
-		    hours, mins, secs);
+		asprintf(&str, "%02d:%02d:%02d", hours, mins, secs);
 	else
-		(void)snprintf(obuff, sizeof(obuff), "%02d:%02d", mins, secs);
-	(void)printf("%*s", v->width, obuff);
+		asprintf(&str, "%02d:%02d", mins, secs);
+
+	return (str);
 }
 
-void
+char *
 elapseds(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	time_t val;
+	char *str;
 
 	v = ve->var;
-	if (!k->ki_valid) {
-		(void)printf("%-*s", v->width, "-");
-		return;
-	}
+	if (!k->ki_valid)
+		return (NULL);
 	val = now - k->ki_p->ki_start.tv_sec;
-	(void)printf("%*jd", v->width, (intmax_t)val);
+	asprintf(&str, "%jd", (intmax_t)val);
+	return (str);
 }
 
 double
@@ -714,13 +681,15 @@ getpcpu(const KINFO *k)
 		(1.0 - exp(k->ki_p->ki_swtime * log(fxtofl(ccpu)))));
 }
 
-void
+char *
 pcpu(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	(void)printf("%*.1f", v->width, getpcpu(k));
+	asprintf(&str, "%.1f", getpcpu(k));
+	return (str);
 }
 
 static double
@@ -742,42 +711,45 @@ getpmem(KINFO *k)
 	return (100.0 * fracmem);
 }
 
-void
+char *
 pmem(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	(void)printf("%*.1f", v->width, getpmem(k));
+	asprintf(&str, "%.1f", getpmem(k));
+	return (str);
 }
 
-void
+char *
 pagein(KINFO *k, VARENT *ve)
 {
 	VAR *v;
+	char *str;
 
 	v = ve->var;
-	(void)printf("%*ld", v->width,
-	    k->ki_valid ? k->ki_p->ki_rusage.ru_majflt : 0);
+	asprintf(&str, "%ld", k->ki_valid ? k->ki_p->ki_rusage.ru_majflt : 0);
+	return (str);
 }
 
 /* ARGSUSED */
-void
+char *
 maxrss(KINFO *k __unused, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
 	/* XXX not yet */
-	(void)printf("%*s", v->width, "-");
+	return (NULL);
 }
 
-void
+char *
 priorityr(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 	struct priority *lpri;
-	char str[8];
+	char *str;
 	unsigned class, level;
 
 	v = ve->var;
@@ -786,112 +758,111 @@ priorityr(KINFO *k, VARENT *ve)
 	level = lpri->pri_level;
 	switch (class) {
 	case PRI_ITHD:
-		snprintf(str, sizeof(str), "intr:%u", level);
+		asprintf(&str, "intr:%u", level);
 		break;
 	case PRI_REALTIME:
-		snprintf(str, sizeof(str), "real:%u", level);
+		asprintf(&str, "real:%u", level);
 		break;
 	case PRI_TIMESHARE:
-		strncpy(str, "normal", sizeof(str));
+		asprintf(&str, "normal");
 		break;
 	case PRI_IDLE:
-		snprintf(str, sizeof(str), "idle:%u", level);
+		asprintf(&str, "idle:%u", level);
 		break;
 	default:
-		snprintf(str, sizeof(str), "%u:%u", class, level);
+		asprintf(&str, "%u:%u", class, level);
 		break;
 	}
-	str[sizeof(str) - 1] = '\0';
-	(void)printf("%*s", v->width, str);
+	return (str);
 }
 
 /*
  * Generic output routines.  Print fields from various prototype
  * structures.
  */
-static void
+static char *
 printval(void *bp, VAR *v)
 {
 	static char ofmt[32] = "%";
 	const char *fcp;
-	char *cp;
+	char *cp, *str;
 
 	cp = ofmt + 1;
 	fcp = v->fmt;
-	if (v->flag & LJUST)
-		*cp++ = '-';
-	*cp++ = '*';
 	while ((*cp++ = *fcp++));
 
 #define	CHKINF127(n)	(((n) > 127) && (v->flag & INF127) ? 127 : (n))
 
 	switch (v->type) {
 	case CHAR:
-		(void)printf(ofmt, v->width, *(char *)bp);
+		(void)asprintf(&str, ofmt, *(char *)bp);
 		break;
 	case UCHAR:
-		(void)printf(ofmt, v->width, *(u_char *)bp);
+		(void)asprintf(&str, ofmt, *(u_char *)bp);
 		break;
 	case SHORT:
-		(void)printf(ofmt, v->width, *(short *)bp);
+		(void)asprintf(&str, ofmt, *(short *)bp);
 		break;
 	case USHORT:
-		(void)printf(ofmt, v->width, *(u_short *)bp);
+		(void)asprintf(&str, ofmt, *(u_short *)bp);
 		break;
 	case INT:
-		(void)printf(ofmt, v->width, *(int *)bp);
+		(void)asprintf(&str, ofmt, *(int *)bp);
 		break;
 	case UINT:
-		(void)printf(ofmt, v->width, CHKINF127(*(u_int *)bp));
+		(void)asprintf(&str, ofmt, CHKINF127(*(u_int *)bp));
 		break;
 	case LONG:
-		(void)printf(ofmt, v->width, *(long *)bp);
+		(void)asprintf(&str, ofmt, *(long *)bp);
 		break;
 	case ULONG:
-		(void)printf(ofmt, v->width, *(u_long *)bp);
+		(void)asprintf(&str, ofmt, *(u_long *)bp);
 		break;
 	case KPTR:
-		(void)printf(ofmt, v->width, *(u_long *)bp);
+		(void)asprintf(&str, ofmt, *(u_long *)bp);
 		break;
 	case PGTOK:
-		(void)printf(ofmt, v->width, ps_pgtok(*(u_long *)bp));
+		(void)asprintf(&str, ofmt, ps_pgtok(*(u_long *)bp));
 		break;
 	default:
 		errx(1, "unknown type %d", v->type);
 	}
+
+	return (str);
 }
 
-void
+char *
 kvar(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	printval((char *)((char *)k->ki_p + v->off), v);
+	return (printval((char *)((char *)k->ki_p + v->off), v));
 }
 
-void
+char *
 rvar(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	if (k->ki_valid)
-		printval((char *)((char *)(&k->ki_p->ki_rusage) + v->off), v);
-	else
-		(void)printf("%*s", v->width, "-");
+	if (!k->ki_valid)
+		return (NULL);
+	return (printval((char *)((char *)(&k->ki_p->ki_rusage) + v->off), v));
 }
 
-void
+char *
 emulname(KINFO *k, VARENT *ve)
 {
 	VAR *v;
 
 	v = ve->var;
-	printf("%-*s", v->width, *k->ki_p->ki_emul ? k->ki_p->ki_emul : "-");
+	if (k->ki_p->ki_emul == NULL)
+		return (NULL);
+	return (strdup(k->ki_p->ki_emul));
 }
 
-void
+char *
 label(KINFO *k, VARENT *ve)
 {
 	char *string;
@@ -912,15 +883,10 @@ label(KINFO *k, VARENT *ve)
 	}
 	mac_free(proclabel);
 out:
-	if (string != NULL) {
-		(void)printf("%-*s", v->width, string);
-		free(string);
-	} else
-		(void)printf("%-*s", v->width, "  -");
-	return;
+	return (string);
 }
 
-void
+char *
 loginclass(KINFO *k, VARENT *ve)
 {
 	VAR *v;
@@ -933,103 +899,10 @@ loginclass(KINFO *k, VARENT *ve)
 	 * and limits don't apply to system processes.
 	 */
 	if (k->ki_p->ki_flag & P_SYSTEM) {
-		(void)printf("%-*s", v->width, "-");
-		return;
+		return (strdup("-"));
 	}
-	s = k->ki_p->ki_loginclass;
-	(void)printf("%-*s", v->width, *s ? s : "-");
-}
-
-int
-s_comm(KINFO *k)
-{
-	char tmpbuff[COMMLEN + TDNAMLEN + 2];
-
-	bzero(tmpbuff, sizeof(tmpbuff));
-	if (showthreads && k->ki_p->ki_numthreads > 1)
-		sprintf(tmpbuff, "%s/%s", k->ki_p->ki_comm,
-		    k->ki_p->ki_tdname);
-	else
-		sprintf(tmpbuff, "%s", k->ki_p->ki_comm);
-	return (strlen(tmpbuff));
-}
-
-int
-s_cputime(KINFO *k)
-{
-	long secs;
-
-	secs = k->ki_p->ki_runtime / 1000000;
-	if (sumrusage)
-		secs += k->ki_p->ki_childtime.tv_sec;
-	return (sizetime(secs));
-}
-
-int
-s_label(KINFO *k)
-{
-	char *string = NULL;
-	mac_t proclabel;
-	int error, size = 0;
-
-	if (mac_prepare_process_label(&proclabel) == -1) {
-		warn("mac_prepare_process_label");
-		return (0);
-	}
-	error = mac_get_pid(k->ki_p->ki_pid, proclabel);
-	if (error == 0 && mac_to_text(proclabel, &string) == 0) {
-		size = strlen(string);
-		free(string);
-	}
-	mac_free(proclabel);
-	return (size);
-}
-
-int
-s_loginclass(KINFO *k)
-{
-	char *s;
-
-	if (k->ki_p->ki_flag & P_SYSTEM)
-		return (1);
-
 	s = k->ki_p->ki_loginclass;
 	if (s == NULL)
-		return (1);
-
-	return (strlen(s));
-}
-
-int
-s_logname(KINFO *k)
-{
-	char *s;
-
-	s = k->ki_p->ki_login;
-	if (s == NULL)
-		return (1);
-
-	return (strlen(s));
-}
-
-int
-s_systime(KINFO *k)
-{
-	long secs;
-
-	secs = k->ki_p->ki_rusage.ru_stime.tv_sec;
-	if (sumrusage)
-		secs += k->ki_p->ki_childstime.tv_sec;
-	return (sizetime(secs));
-}
-
-int
-s_usertime(KINFO *k)
-{
-	long secs;
-
-	secs = k->ki_p->ki_rusage.ru_utime.tv_sec;
-	if (sumrusage)
-		secs += k->ki_p->ki_childutime.tv_sec;
-	return (sizetime(secs));
+		return (NULL);
+	return (strdup(s));
 }
diff --git a/bin/ps/ps.1 b/bin/ps/ps.1
index a66ed5d850d7..2c04c21ae072 100644
--- a/bin/ps/ps.1
+++ b/bin/ps/ps.1
@@ -29,7 +29,7 @@
 .\"     @(#)ps.1	8.3 (Berkeley) 4/18/94
 .\" $FreeBSD$
 .\"
-.Dd July 1, 2011
+.Dd October 1, 2011
 .Dt PS 1
 .Os
 .Sh NAME
@@ -54,6 +54,11 @@ utility
 displays a header line, followed by lines containing information about
 all of your
 processes that have controlling terminals.
+If the
+.Fl x
+options is specified,
+.Nm
+will also display processes that do not have controlling terminals.
 .Pp
 A different set of processes can be selected for display by using any
 combination of the
@@ -90,8 +95,8 @@ and
 .Fl o
 options).
 The default output format includes, for each process, the process' ID,
-controlling terminal, CPU time (including both user and system time),
-state, and associated command.
+controlling terminal, state, CPU time (including both user and system time)
+and associated command.
 .Pp
 The process file system (see
 .Xr procfs 5 )
@@ -103,13 +108,9 @@ The options are as follows:
 .Bl -tag -width indent
 .It Fl a
 Display information about other users' processes as well as your own.
-This will skip any processes which do not have a controlling terminal,
-unless the
-.Fl x
-option is also specified.
-This can be disabled by setting the
+If the
 .Va security.bsd.see_other_uids
-sysctl to zero.
+sysctl is set to zero, this option is honored only if the UID of the user is 0.
 .It Fl c
 Change the
 .Dq command
@@ -216,6 +217,9 @@ with the standard input.
 .It Fl t
 Display information about processes attached to the specified terminal
 devices.
+Full pathnames, as well as abbreviations (see explanation of the
+.Cm tt
+keyword) can be specified.
 .It Fl U
 Display the processes belonging to the specified usernames.
 .It Fl u
@@ -427,12 +431,15 @@ The process is being traced or debugged.
 An abbreviation for the pathname of the controlling terminal, if any.
 The abbreviation consists of the three letters following
 .Pa /dev/tty ,
-or, for the console,
-.Dq Li con .
+or, for pseudo-terminals, the corresponding entry in
+.Pa /dev/pts .
 This is followed by a
 .Ql -
 if the process can no longer reach that
 controlling terminal (i.e., it has been revoked).
+The full pathname of the controlling terminal is available via the
+.Cm tty
+keyword.
 .It Cm wchan
 The event (an address in the system) on which a process waits.
 When printed numerically, the initial part of the address is
diff --git a/bin/ps/ps.c b/bin/ps/ps.c
index fb2b752b2b9b..eb2d6301dc7b 100644
--- a/bin/ps/ps.c
+++ b/bin/ps/ps.c
@@ -99,14 +99,12 @@ time_t	 now;			/* Current time(3) value */
 int	 rawcpu;		/* -C */
 int	 sumrusage;		/* -S */
 int	 termwidth;		/* Width of the screen (0 == infinity). */
-int	 totwidth;		/* Calculated-width of requested variables. */
 int	 showthreads;		/* will threads be shown? */
 
 struct velisthead varlist = STAILQ_HEAD_INITIALIZER(varlist);
 
 static int	 forceuread = DEF_UREAD; /* Do extra work to get u-area. */
 static kvm_t	*kd;
-static KINFO	*kinfo;
 static int	 needcomm;	/* -o "command" */
 static int	 needenv;	/* -e */
 static int	 needuser;	/* -o "user" */
@@ -139,7 +137,7 @@ static int	 addelem_tty(struct listinfo *, const char *);
 static int	 addelem_uid(struct listinfo *, const char *);
 static void	 add_list(struct listinfo *, const char *);
 static void	 descendant_sort(KINFO *, int);
-static void	 dynsizevars(KINFO *);
+static void	 format_output(KINFO *);
 static void	*expand_list(struct listinfo *);
 static const char *
 		 fmt(char **(*)(kvm_t *, const struct kinfo_proc *, int),
@@ -172,12 +170,13 @@ main(int argc, char *argv[])
 	struct listinfo gidlist, pgrplist, pidlist;
 	struct listinfo ruidlist, sesslist, ttylist, uidlist;
 	struct kinfo_proc *kp;
-	KINFO *next_KINFO;
+	KINFO *kinfo = NULL, *next_KINFO;
+	KINFO_STR *ks;
 	struct varent *vent;
 	struct winsize ws;
-	const char *nlistf, *memf;
+	const char *nlistf, *memf, *fmtstr, *str;
 	char *cols;
-	int all, ch, elem, flag, _fmt, i, lineno;
+	int all, ch, elem, flag, _fmt, i, lineno, linelen, left;
 	int descendancy, nentries, nkept, nselectors;
 	int prtheader, wflag, what, xkeep, xkeep_implied;
 	char errbuf[_POSIX2_LINE_MAX];
@@ -588,19 +587,16 @@ main(int argc, char *argv[])
 				    kp->ki_dsize + kp->ki_ssize;
 			if (needuser)
 				saveuser(next_KINFO);
-			dynsizevars(next_KINFO);
 			nkept++;
 		}
 	}
 
 	sizevars();
 
-	/*
-	 * print header
-	 */
-	printheader();
-	if (nkept == 0)
+	if (nkept == 0) {
+		printheader();
 		exit(1);
+	}
 
 	/*
 	 * sort proc list
@@ -613,14 +609,59 @@ main(int argc, char *argv[])
 	if (descendancy)
 		descendant_sort(kinfo, nkept);
 
+
 	/*
-	 * For each process, call each variable output function.
+	 * Prepare formatted output.
+	 */
+	for (i = 0; i < nkept; i++)
+		format_output(&kinfo[i]);
+
+	/*
+	 * Print header.
+	 */
+	printheader();
+
+	/*
+	 * Output formatted lines.
 	 */
 	for (i = lineno = 0; i < nkept; i++) {
+		linelen = 0;
 		STAILQ_FOREACH(vent, &varlist, next_ve) {
-			(vent->var->oproc)(&kinfo[i], vent);
-			if (STAILQ_NEXT(vent, next_ve) != NULL)
+	        	if (vent->var->flag & LJUST)
+				fmtstr = "%-*s";
+			else
+				fmtstr = "%*s";
+
+			ks = STAILQ_FIRST(&kinfo[i].ki_ks);
+			STAILQ_REMOVE_HEAD(&kinfo[i].ki_ks, ks_next);
+			/* Truncate rightmost column if neccessary.  */
+			if (STAILQ_NEXT(vent, next_ve) == NULL &&
+			   termwidth != UNLIMITED && ks->ks_str != NULL) {
+				left = termwidth - linelen;
+				if (left > 0 && left < (int)strlen(ks->ks_str))
+					ks->ks_str[left] = '\0';
+			}
+			str = ks->ks_str;
+			if (str == NULL)
+				str = "-";
+			/* No padding for the last column, if it's LJUST. */
+			if (STAILQ_NEXT(vent, next_ve) == NULL &&
+			    vent->var->flag & LJUST)
+				linelen += printf(fmtstr, 0, str);
+			else
+				linelen += printf(fmtstr, vent->var->width, str);
+
+			if (ks->ks_str != NULL) {
+				free(ks->ks_str);
+				ks->ks_str = NULL;
+			}
+			free(ks);
+			ks = NULL;
+
+			if (STAILQ_NEXT(vent, next_ve) != NULL) {
 				(void)putchar(' ');
+				linelen++;
+			}
 		}
 		(void)putchar('\n');
 		if (prtheader && lineno++ == prtheader - 4) {
@@ -1078,10 +1119,6 @@ scanvars(void)
 
 	STAILQ_FOREACH(vent, &varlist, next_ve) {
 		v = vent->var;
-		if (v->sproc != NULL) {
-			v->dwidth = v->width;
-			v->width = 0;
-		}
 		if (v->flag & USER)
 			needuser = 1;
 		if (v->flag & COMM)
@@ -1090,21 +1127,29 @@ scanvars(void)
 }
 
 static void
-dynsizevars(KINFO *ki)
+format_output(KINFO *ki)
 {
 	struct varent *vent;
 	VAR *v;
-	int i;
+	KINFO_STR *ks;
+	char *str;
+	int len;
 
+	STAILQ_INIT(&ki->ki_ks);
 	STAILQ_FOREACH(vent, &varlist, next_ve) {
 		v = vent->var;
-		if (v->sproc == NULL)
-			continue;
-		i = (v->sproc)(ki);
-		if (v->width < i)
-			v->width = i;
-		if (v->width > v->dwidth)
-			v->width = v->dwidth;
+		str = (v->oproc)(ki, vent);
+		ks = malloc(sizeof(*ks));
+		if (ks == NULL)
+			errx(1, "malloc failed");
+		ks->ks_str = str;
+		STAILQ_INSERT_TAIL(&ki->ki_ks, ks, ks_next);
+		if (str != NULL) {
+			len = strlen(str);
+		} else
+			len = 1; /* "-" */
+		if (v->width < len)
+			v->width = len;
 	}
 }
 
@@ -1120,9 +1165,7 @@ sizevars(void)
 		i = strlen(vent->header);
 		if (v->width < i)
 			v->width = i;
-		totwidth += v->width + 1;	/* +1 for space */
 	}
-	totwidth--;
 }
 
 static const char *
diff --git a/bin/ps/ps.h b/bin/ps/ps.h
index b1e9d9b7c1ed..86aaeb0fe652 100644
--- a/bin/ps/ps.h
+++ b/bin/ps/ps.h
@@ -35,6 +35,11 @@
 #define	UNLIMITED	0	/* unlimited terminal width */
 enum type { CHAR, UCHAR, SHORT, USHORT, INT, UINT, LONG, ULONG, KPTR, PGTOK };
 
+typedef struct kinfo_str {
+	STAILQ_ENTRY(kinfo_str) ks_next;
+	char *ks_str;	/* formatted string */
+} KINFO_STR;
+
 typedef struct kinfo {
 	struct kinfo_proc *ki_p;	/* kinfo_proc structure */
 	char *ki_args;		/* exec args */
@@ -46,6 +51,7 @@ typedef struct kinfo {
 		int level;	/* used in decendant_sort() */
 		char *prefix;	/* calculated in decendant_sort() */
 	} ki_d;
+	STAILQ_HEAD(, kinfo_str) ki_ks;
 } KINFO;
 
 /* Variables. */
@@ -65,10 +71,7 @@ typedef struct var {
 #define	INF127	0x10		/* values >127 displayed as 127 */
 	u_int	flag;
 				/* output routine */
-	void	(*oproc)(struct kinfo *, struct varent *);
-				/* sizing routine */
-	int	(*sproc)(struct kinfo *);
-	short	width;		/* printing width */
+	char 	*(*oproc)(struct kinfo *, struct varent *);
 	/*
 	 * The following (optional) elements are hooks for passing information
 	 * to the generic output routine pvar (which prints simple elements
@@ -77,10 +80,8 @@ typedef struct var {
 	size_t	off;		/* offset in structure */
 	enum	type type;	/* type of element */
 	const char *fmt;	/* printf format */
-	short	dwidth;		/* dynamic printing width */
-	/*
-	 * glue to link selected fields together
-	 */
+
+	short	width;		/* calculated width */
 } VAR;
 
 #include "extern.h"
diff --git a/cddl/contrib/opensolaris/cmd/zdb/zdb.c b/cddl/contrib/opensolaris/cmd/zdb/zdb.c
index c6e219df9e1d..bf710009d539 100644
--- a/cddl/contrib/opensolaris/cmd/zdb/zdb.c
+++ b/cddl/contrib/opensolaris/cmd/zdb/zdb.c
@@ -2190,8 +2190,10 @@ dump_block_stats(spa_t *spa)
 	 */
 	(void) bpobj_iterate_nofree(&spa->spa_deferred_bpobj,
 	    count_block_cb, &zcb, NULL);
-	(void) bpobj_iterate_nofree(&spa->spa_dsl_pool->dp_free_bpobj,
-	    count_block_cb, &zcb, NULL);
+	if (spa_version(spa) >= SPA_VERSION_DEADLISTS) {
+		(void) bpobj_iterate_nofree(&spa->spa_dsl_pool->dp_free_bpobj,
+		    count_block_cb, &zcb, NULL);
+	}
 
 	if (dump_opt['c'] > 1)
 		flags |= TRAVERSE_PREFETCH_DATA;
diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs.8 b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
index e30b4b285bd8..03deef269280 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs.8
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
@@ -74,6 +74,11 @@ zfs \- configures ZFS file systems
 \fBzfs\fR \fBrename\fR \fB-r\fR \fIsnapshot\fR \fIsnapshot\fR
 .fi
 
+.LP
+.nf
+\fBzfs\fR \fBrename\fR \fB-u\fR [\fB-p\fR] \fIfilesystem\fR \fIfilesystem\fR
+.fi
+
 .LP
 .nf
 \fBzfs\fR \fBlist\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR][\fB-H\fR][\fB-o\fR \fIproperty\fR[,...]] [\fB-t\fR \fItype\fR[,...]]
@@ -1479,6 +1484,10 @@ The snapshot that was cloned, and any snapshots previous to this snapshot, are n
 .na
 \fB\fBzfs rename\fR [\fB-p\fR] \fIfilesystem\fR|\fIvolume\fR \fIfilesystem\fR|\fIvolume\fR\fR
 .ad
+.br
+.na
+\fB\fBzfs rename\fR \fB-u\fR [\fB-p\fR] \fIfilesystem\fR \fIfilesystem\fR\fR
+.ad
 .sp .6
 .RS 4n
 Renames the given dataset. The new target can be located anywhere in the \fBZFS\fR hierarchy, with the exception of snapshots. Snapshots can only be renamed within the parent file system or volume. When renaming a snapshot, the parent file system of the snapshot does not need to be specified as part of the second argument. Renamed file systems can inherit new mount points, in which case they are unmounted and remounted at the new mount point.
@@ -1493,6 +1502,17 @@ Renames the given dataset. The new target can be located anywhere in the \fBZFS\
 Creates all the nonexistent parent datasets. Datasets created in this manner are automatically mounted according to the \fBmountpoint\fR property inherited from their parent.
 .RE
 
+.sp
+.ne 2
+.mk
+.na
+\fB\fB-u\fR\fR
+.ad
+.sp .6
+.RS 4n
+Do not remount file systems during rename. If a file system's \fBmountpoint\fR property is set to \fBlegacy\fR or \fBnone\fR, file system is not unmounted even if this option is not given.
+.RE
+
 .RE
 
 .sp
diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
index bc5a66292d70..ff8984759857 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
@@ -22,6 +22,8 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #include <assert.h>
@@ -253,7 +255,8 @@ get_usage(zfs_help_t idx)
 		return (gettext("\trename <filesystem|volume|snapshot> "
 		    "<filesystem|volume|snapshot>\n"
 		    "\trename -p <filesystem|volume> <filesystem|volume>\n"
-		    "\trename -r <snapshot> <snapshot>"));
+		    "\trename -r <snapshot> <snapshot>\n"
+		    "\trename -u [-p] <filesystem> <filesystem>"));
 	case HELP_ROLLBACK:
 		return (gettext("\trollback [-rRf] <snapshot>\n"));
 	case HELP_SEND:
@@ -2851,6 +2854,7 @@ zfs_do_list(int argc, char **argv)
  * zfs rename <fs | snap | vol> <fs | snap | vol>
  * zfs rename -p <fs | vol> <fs | vol>
  * zfs rename -r <snap> <snap>
+ * zfs rename -u [-p] <fs> <fs>
  *
  * Renames the given dataset to another of the same type.
  *
@@ -2861,19 +2865,21 @@ static int
 zfs_do_rename(int argc, char **argv)
 {
 	zfs_handle_t *zhp;
-	int c;
-	int ret;
-	boolean_t recurse = B_FALSE;
+	renameflags_t flags = { 0 };
+	int c, ret, types;
 	boolean_t parents = B_FALSE;
 
 	/* check options */
-	while ((c = getopt(argc, argv, "pr")) != -1) {
+	while ((c = getopt(argc, argv, "pru")) != -1) {
 		switch (c) {
 		case 'p':
 			parents = B_TRUE;
 			break;
 		case 'r':
-			recurse = B_TRUE;
+			flags.recurse = B_TRUE;
+			break;
+		case 'u':
+			flags.nounmount = B_TRUE;
 			break;
 		case '?':
 		default:
@@ -2902,20 +2908,32 @@ zfs_do_rename(int argc, char **argv)
 		usage(B_FALSE);
 	}
 
-	if (recurse && parents) {
+	if (flags.recurse && parents) {
 		(void) fprintf(stderr, gettext("-p and -r options are mutually "
 		    "exclusive\n"));
 		usage(B_FALSE);
 	}
 
-	if (recurse && strchr(argv[0], '@') == 0) {
+	if (flags.recurse && strchr(argv[0], '@') == 0) {
 		(void) fprintf(stderr, gettext("source dataset for recursive "
 		    "rename must be a snapshot\n"));
 		usage(B_FALSE);
 	}
 
-	if ((zhp = zfs_open(g_zfs, argv[0], parents ? ZFS_TYPE_FILESYSTEM |
-	    ZFS_TYPE_VOLUME : ZFS_TYPE_DATASET)) == NULL)
+	if (flags.nounmount && parents) {
+		(void) fprintf(stderr, gettext("-u and -r options are mutually "
+		    "exclusive\n"));
+		usage(B_FALSE);
+	}
+
+	if (flags.nounmount)
+		types = ZFS_TYPE_FILESYSTEM;
+	else if (parents)
+		types = ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME;
+	else
+		types = ZFS_TYPE_DATASET;
+
+	if ((zhp = zfs_open(g_zfs, argv[0], types)) == NULL)
 		return (1);
 
 	/* If we were asked and the name looks good, try to create ancestors. */
@@ -2925,7 +2943,7 @@ zfs_do_rename(int argc, char **argv)
 		return (1);
 	}
 
-	ret = (zfs_rename(zhp, argv[1], recurse) != 0);
+	ret = (zfs_rename(zhp, argv[1], flags) != 0);
 
 	zfs_close(zhp);
 	return (ret);
diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
index 45465b8e761d..849f0e0b4fc2 100644
--- a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
+++ b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
@@ -3377,7 +3377,7 @@ print_scan_status(pool_scan_stat_t *ps)
 	double fraction_done;
 	char processed_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
 
-	(void) printf(gettext(" scan: "));
+	(void) printf(gettext("  scan: "));
 
 	/* If there's never been a scan, there's not much to say. */
 	if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
@@ -3457,7 +3457,7 @@ print_scan_status(pool_scan_stat_t *ps)
 	/*
 	 * do not print estimated time if hours_left is more than 30 days
 	 */
-	(void) printf(gettext("    %s scanned out of %s at %s/s"),
+	(void) printf(gettext("        %s scanned out of %s at %s/s"),
 	    examined_buf, total_buf, rate_buf);
 	if (hours_left < (30 * 24)) {
 		(void) printf(gettext(", %lluh%um to go\n"),
@@ -3468,10 +3468,10 @@ print_scan_status(pool_scan_stat_t *ps)
 	}
 
 	if (ps->pss_func == POOL_SCAN_RESILVER) {
-		(void) printf(gettext("    %s resilvered, %.2f%% done\n"),
+		(void) printf(gettext("        %s resilvered, %.2f%% done\n"),
 		    processed_buf, 100 * fraction_done);
 	} else if (ps->pss_func == POOL_SCAN_SCRUB) {
-		(void) printf(gettext("    %s repaired, %.2f%% done\n"),
+		(void) printf(gettext("        %s repaired, %.2f%% done\n"),
 		    processed_buf, 100 * fraction_done);
 	}
 }
diff --git a/cddl/contrib/opensolaris/head/thread.h b/cddl/contrib/opensolaris/head/thread.h
index 818f0781117a..d813a25dd4bc 100644
--- a/cddl/contrib/opensolaris/head/thread.h
+++ b/cddl/contrib/opensolaris/head/thread.h
@@ -76,6 +76,7 @@ static __inline int
 thr_create(void *stack_base, size_t stack_size, void *(*start_func) (void*),
     void *arg, long flags, thread_t *new_thread_ID)
 {
+	pthread_t dummy;
 	int ret;
 
 	assert(stack_base == NULL);
@@ -85,9 +86,12 @@ thr_create(void *stack_base, size_t stack_size, void *(*start_func) (void*),
 	pthread_attr_t attr;
 	pthread_attr_init(&attr);
 
-	if(flags & THR_DETACHED)
+	if (flags & THR_DETACHED)
 		pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
+	if (new_thread_ID == NULL)
+		new_thread_ID = &dummy;
+
 	/* This function ignores the THR_BOUND flag, since NPTL doesn't seem to support PTHREAD_SCOPE_PROCESS */
 
 	ret = pthread_create(new_thread_ID, &attr, start_func, arg);
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
index 1077958b88c0..6b25879ed5fe 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
@@ -22,6 +22,8 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2010 Nexenta Systems, Inc. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #ifndef	_LIBZFS_H
@@ -518,7 +520,16 @@ extern int zfs_destroy_snaps(zfs_handle_t *, char *, boolean_t);
 extern int zfs_clone(zfs_handle_t *, const char *, nvlist_t *);
 extern int zfs_snapshot(libzfs_handle_t *, const char *, boolean_t, nvlist_t *);
 extern int zfs_rollback(zfs_handle_t *, zfs_handle_t *, boolean_t);
-extern int zfs_rename(zfs_handle_t *, const char *, boolean_t);
+
+typedef struct renameflags {
+	/* recursive rename */
+	int recurse : 1;
+
+	/* don't unmount file systems */
+	int nounmount : 1;
+} renameflags_t;
+
+extern int zfs_rename(zfs_handle_t *, const char *, renameflags_t flags);
 
 typedef struct sendflags {
 	/* print informational messages (ie, -v was specified) */
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_changelist.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_changelist.c
index 4328d38a2c36..a899965ca0f7 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_changelist.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_changelist.c
@@ -24,6 +24,9 @@
  * Use is subject to license terms.
  *
  * Portions Copyright 2007 Ramprakash Jelari
+ *
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #include <libintl.h>
@@ -122,6 +125,8 @@ changelist_prefix(prop_changelist_t *clp)
 			 */
 			switch (clp->cl_prop) {
 			case ZFS_PROP_MOUNTPOINT:
+				if (clp->cl_gflags & CL_GATHER_DONT_UNMOUNT)
+					break;
 				if (zfs_unmount(cn->cn_handle, NULL,
 				    clp->cl_mflags) != 0) {
 					ret = -1;
@@ -168,8 +173,10 @@ changelist_postfix(prop_changelist_t *clp)
 	if ((cn = uu_list_last(clp->cl_list)) == NULL)
 		return (0);
 
-	if (clp->cl_prop == ZFS_PROP_MOUNTPOINT)
+	if (clp->cl_prop == ZFS_PROP_MOUNTPOINT &&
+	    !(clp->cl_gflags & CL_GATHER_DONT_UNMOUNT)) {
 		remove_mountpoint(cn->cn_handle);
+	}
 
 	/*
 	 * It is possible that the changelist_prefix() used libshare
@@ -224,7 +231,8 @@ changelist_postfix(prop_changelist_t *clp)
 		    shareopts, sizeof (shareopts), NULL, NULL, 0,
 		    B_FALSE) == 0) && (strcmp(shareopts, "off") != 0));
 
-		mounted = zfs_is_mounted(cn->cn_handle, NULL);
+		mounted = (clp->cl_gflags & CL_GATHER_DONT_UNMOUNT) ||
+		    zfs_is_mounted(cn->cn_handle, NULL);
 
 		if (!mounted && (cn->cn_mounted ||
 		    ((sharenfs || sharesmb || clp->cl_waslegacy) &&
@@ -467,7 +475,6 @@ change_one(zfs_handle_t *zhp, void *data)
 			 * This is necessary when the original mountpoint
 			 * is legacy or none.
 			 */
-			ASSERT(!clp->cl_alldependents);
 			verify(uu_list_insert_before(clp->cl_list,
 			    uu_list_first(clp->cl_list), cn) == 0);
 		}
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
index 381f5edb1c5c..a65b0793cab3 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
@@ -23,6 +23,8 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2010 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #include <ctype.h>
@@ -3480,7 +3482,7 @@ zfs_iter_dependents(zfs_handle_t *zhp, boolean_t allowrecursion,
  * Renames the given dataset.
  */
 int
-zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
+zfs_rename(zfs_handle_t *zhp, const char *target, renameflags_t flags)
 {
 	int ret;
 	zfs_cmd_t zc = { 0 };
@@ -3489,6 +3491,7 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 	zfs_handle_t *zhrp = NULL;
 	char *parentname = NULL;
 	char parent[ZFS_MAXNAMELEN];
+	char property[ZFS_MAXPROPLEN];
 	libzfs_handle_t *hdl = zhp->zfs_hdl;
 	char errbuf[1024];
 
@@ -3535,7 +3538,7 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 		if (!zfs_validate_name(hdl, target, zhp->zfs_type, B_TRUE))
 			return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));
 	} else {
-		if (recursive) {
+		if (flags.recurse) {
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "recursive rename must be a snapshot"));
 			return (zfs_error(hdl, EZFS_BADTYPE, errbuf));
@@ -3576,7 +3579,20 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 		return (zfs_error(hdl, EZFS_ZONED, errbuf));
 	}
 
-	if (recursive) {
+	/*
+	 * Avoid unmounting file systems with mountpoint property set to
+	 * 'legacy' or 'none' even if -u option is not given.
+	 */
+	if (zhp->zfs_type == ZFS_TYPE_FILESYSTEM &&
+	    !flags.recurse && !flags.nounmount &&
+	    zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, property,
+	    sizeof (property), NULL, NULL, 0, B_FALSE) == 0 &&
+	    (strcmp(property, "legacy") == 0 ||
+	     strcmp(property, "none") == 0)) {
+		flags.nounmount = B_TRUE;
+	}
+
+	if (flags.recurse) {
 
 		parentname = zfs_strdup(zhp->zfs_hdl, zhp->zfs_name);
 		if (parentname == NULL) {
@@ -3592,8 +3608,10 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 		}
 
 	} else {
-		if ((cl = changelist_gather(zhp, ZFS_PROP_NAME, 0, 0)) == NULL)
+		if ((cl = changelist_gather(zhp, ZFS_PROP_NAME,
+		    flags.nounmount ? CL_GATHER_DONT_UNMOUNT : 0, 0)) == NULL) {
 			return (-1);
+		}
 
 		if (changelist_haszonedchild(cl)) {
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
@@ -3615,7 +3633,9 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
 	(void) strlcpy(zc.zc_value, target, sizeof (zc.zc_value));
 
-	zc.zc_cookie = recursive;
+	zc.zc_cookie = flags.recurse ? 1 : 0;
+	if (flags.nounmount)
+		zc.zc_cookie |= 2;
 
 	if ((ret = zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_RENAME, &zc)) != 0) {
 		/*
@@ -3625,7 +3645,7 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 		(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
 		    "cannot rename '%s'"), zc.zc_name);
 
-		if (recursive && errno == EEXIST) {
+		if (flags.recurse && errno == EEXIST) {
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "a child dataset already has a snapshot "
 			    "with the new name"));
@@ -3638,10 +3658,10 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive)
 		 * On failure, we still want to remount any filesystems that
 		 * were previously mounted, so we don't alter the system state.
 		 */
-		if (!recursive)
+		if (!flags.recurse)
 			(void) changelist_postfix(cl);
 	} else {
-		if (!recursive) {
+		if (!flags.recurse) {
 			changelist_rename(cl, zfs_get_name(zhp), target);
 			ret = changelist_postfix(cl);
 		}
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_impl.h b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_impl.h
index 9d1ecb72f411..9cd1ed38c924 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_impl.h
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_impl.h
@@ -21,6 +21,8 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #ifndef	_LIBFS_IMPL_H
@@ -158,7 +160,11 @@ int zprop_expand_list(libzfs_handle_t *hdl, zprop_list_t **plp,
  * on each change node regardless of whether or not it is currently
  * mounted.
  */
-#define	CL_GATHER_MOUNT_ALWAYS	1
+#define	CL_GATHER_MOUNT_ALWAYS	0x01
+/*
+ * Use this changelist_gather() flag to prevent unmounting of file systems.
+ */
+#define	CL_GATHER_DONT_UNMOUNT	0x02
 
 typedef struct prop_changelist prop_changelist_t;
 
diff --git a/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c b/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c
index 8db5d11c1327..c407bba09ccd 100644
--- a/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c
+++ b/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c
@@ -87,7 +87,7 @@ again:	if ((t = tq->tq_freelist) != NULL && tq->tq_nalloc >= tq->tq_minalloc) {
 		}
 		mutex_exit(&tq->tq_lock);
 
-		t = kmem_alloc(sizeof (task_t), tqflags);
+		t = kmem_alloc(sizeof (task_t), tqflags & KM_SLEEP);
 
 		mutex_enter(&tq->tq_lock);
 		if (t != NULL)
diff --git a/contrib/gcc/BASE-VER b/contrib/gcc/BASE-VER
index af8c8ec7c134..fae6e3d04b2c 100644
--- a/contrib/gcc/BASE-VER
+++ b/contrib/gcc/BASE-VER
@@ -1 +1 @@
-4.2.2
+4.2.1
diff --git a/contrib/gcc/DEV-PHASE b/contrib/gcc/DEV-PHASE
index 373fbc60bb98..893adcd31e96 100644
--- a/contrib/gcc/DEV-PHASE
+++ b/contrib/gcc/DEV-PHASE
@@ -1 +1 @@
-prerelease
+patched
diff --git a/contrib/gcc/config/i386/pmm_malloc.h b/contrib/gcc/config/i386/pmm_malloc.h
index 744ac6d41cc1..0d39d8db1aef 100644
--- a/contrib/gcc/config/i386/pmm_malloc.h
+++ b/contrib/gcc/config/i386/pmm_malloc.h
@@ -34,7 +34,7 @@
 #ifndef __cplusplus
 extern int posix_memalign (void **, size_t, size_t);
 #else
-extern "C" int posix_memalign (void **, size_t, size_t) throw ();
+extern "C" int posix_memalign (void **, size_t, size_t);
 #endif
 
 static __inline void *
diff --git a/contrib/llvm/LICENSE.TXT b/contrib/llvm/LICENSE.TXT
new file mode 100644
index 000000000000..1b1047ca37db
--- /dev/null
+++ b/contrib/llvm/LICENSE.TXT
@@ -0,0 +1,69 @@
+==============================================================================
+LLVM Release License
+==============================================================================
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2003-2011 University of Illinois at Urbana-Champaign.
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+
+==============================================================================
+Copyrights and Licenses for Third Party Software Distributed with LLVM:
+==============================================================================
+The LLVM software contains code written by third parties.  Such software will
+have its own individual LICENSE.TXT file in the directory in which it appears.
+This file will describe the copyrights, license, and restrictions which apply
+to that code.
+
+The disclaimer of warranty in the University of Illinois Open Source License
+applies to all code in the LLVM Distribution, and nothing in any of the
+other licenses gives permission to use the names of the LLVM Team or the
+University of Illinois to endorse or promote products derived from this
+Software.
+
+The following pieces of software have additional or alternate copyrights,
+licenses, and/or restrictions:
+
+Program             Directory
+-------             ---------
+Autoconf            llvm/autoconf
+                    llvm/projects/ModuleMaker/autoconf
+                    llvm/projects/sample/autoconf
+CellSPU backend     llvm/lib/Target/CellSPU/README.txt
+Google Test         llvm/utils/unittest/googletest
+OpenBSD regex       llvm/lib/Support/{reg*, COPYRIGHT.regex}
diff --git a/contrib/llvm/include/llvm-c/Core.h b/contrib/llvm/include/llvm-c/Core.h
index a4456dd13e4b..d23b91c4e0de 100644
--- a/contrib/llvm/include/llvm-c/Core.h
+++ b/contrib/llvm/include/llvm-c/Core.h
@@ -115,7 +115,10 @@ typedef enum {
     LLVMNoImplicitFloatAttribute = 1<<23,
     LLVMNakedAttribute      = 1<<24,
     LLVMInlineHintAttribute = 1<<25,
-    LLVMStackAlignment = 7<<26
+    LLVMStackAlignment = 7<<26,
+    LLVMReturnsTwice = 1 << 29,
+    LLVMUWTable = 1 << 30,
+    LLVMNonLazyBind = 1 << 31
 } LLVMAttribute;
 
 typedef enum {
@@ -125,7 +128,7 @@ typedef enum {
   LLVMSwitch         = 3,
   LLVMIndirectBr     = 4,
   LLVMInvoke         = 5,
-  LLVMUnwind         = 6,
+  /* removed 6 due to API changes */
   LLVMUnreachable    = 7,
 
   /* Standard Binary Operators */
@@ -176,14 +179,26 @@ typedef enum {
   LLVMPHI            = 44,
   LLVMCall           = 45,
   LLVMSelect         = 46,
-  /* UserOp1 */
-  /* UserOp2 */
+  LLVMUserOp1        = 47,
+  LLVMUserOp2        = 48,
   LLVMVAArg          = 49,
   LLVMExtractElement = 50,
   LLVMInsertElement  = 51,
   LLVMShuffleVector  = 52,
   LLVMExtractValue   = 53,
-  LLVMInsertValue    = 54
+  LLVMInsertValue    = 54,
+
+  /* Atomic operators */
+  LLVMFence          = 55,
+  LLVMAtomicCmpXchg  = 56,
+  LLVMAtomicRMW      = 57,
+
+  /* Exception Handling Operators */
+  LLVMResume         = 58,
+  LLVMLandingPad     = 59,
+  LLVMUnwind         = 60
+
+
 } LLVMOpcode;
 
 typedef enum {
@@ -274,6 +289,11 @@ typedef enum {
   LLVMRealPredicateTrue   /**< Always true (always folded) */
 } LLVMRealPredicate;
 
+typedef enum {
+  LLVMLandingPadCatch,    /**< A catch clause   */
+  LLVMLandingPadFilter    /**< A filter clause  */
+} LLVMLandingPadClauseTy;
+
 void LLVMInitializeCore(LLVMPassRegistryRef R);
 
 
@@ -340,6 +360,7 @@ LLVMContextRef LLVMGetModuleContext(LLVMModuleRef M);
 
 /** See llvm::LLVMTypeKind::getTypeID. */
 LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty);
+LLVMBool LLVMTypeIsSized(LLVMTypeRef Ty);
 
 /** See llvm::LLVMType::getContext. */
 LLVMContextRef LLVMGetTypeContext(LLVMTypeRef Ty);
@@ -388,6 +409,7 @@ LLVMTypeRef LLVMStructTypeInContext(LLVMContextRef C, LLVMTypeRef *ElementTypes,
 LLVMTypeRef LLVMStructType(LLVMTypeRef *ElementTypes, unsigned ElementCount,
                            LLVMBool Packed);
 LLVMTypeRef LLVMStructCreateNamed(LLVMContextRef C, const char *Name);
+const char *LLVMGetStructName(LLVMTypeRef Ty);
 void LLVMStructSetBody(LLVMTypeRef StructTy, LLVMTypeRef *ElementTypes,
                        unsigned ElementCount, LLVMBool Packed);
 
@@ -427,8 +449,11 @@ LLVMTypeRef LLVMX86MMXType(void);
   macro(Argument)                           \
   macro(BasicBlock)                         \
   macro(InlineAsm)                          \
+  macro(MDNode)                             \
+  macro(MDString)                           \
   macro(User)                               \
     macro(Constant)                         \
+      macro(BlockAddress)                   \
       macro(ConstantAggregateZero)          \
       macro(ConstantArray)                  \
       macro(ConstantExpr)                   \
@@ -448,29 +473,32 @@ LLVMTypeRef LLVMX86MMXType(void);
         macro(IntrinsicInst)                \
           macro(DbgInfoIntrinsic)           \
             macro(DbgDeclareInst)           \
+          macro(EHExceptionInst)            \
           macro(EHSelectorInst)             \
           macro(MemIntrinsic)               \
             macro(MemCpyInst)               \
             macro(MemMoveInst)              \
             macro(MemSetInst)               \
       macro(CmpInst)                        \
-      macro(FCmpInst)                       \
-      macro(ICmpInst)                       \
+        macro(FCmpInst)                     \
+        macro(ICmpInst)                     \
       macro(ExtractElementInst)             \
       macro(GetElementPtrInst)              \
       macro(InsertElementInst)              \
       macro(InsertValueInst)                \
+      macro(LandingPadInst)                 \
       macro(PHINode)                        \
       macro(SelectInst)                     \
       macro(ShuffleVectorInst)              \
       macro(StoreInst)                      \
       macro(TerminatorInst)                 \
         macro(BranchInst)                   \
+        macro(IndirectBrInst)               \
         macro(InvokeInst)                   \
         macro(ReturnInst)                   \
         macro(SwitchInst)                   \
         macro(UnreachableInst)              \
-        macro(UnwindInst)                   \
+        macro(ResumeInst)                   \
     macro(UnaryInstruction)                 \
       macro(AllocaInst)                     \
       macro(CastInst)                       \
@@ -533,6 +561,11 @@ LLVMValueRef LLVMMDString(const char *Str, unsigned SLen);
 LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals,
                                  unsigned Count);
 LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count);
+const char  *LLVMGetMDString(LLVMValueRef V, unsigned* Len);
+int LLVMGetMDNodeNumOperands(LLVMValueRef V);
+LLVMValueRef *LLVMGetMDNodeOperand(LLVMValueRef V, unsigned i);
+unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char* name);
+void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char* name, LLVMValueRef *Dest);
 
 /* Operations on scalar constants */
 LLVMValueRef LLVMConstInt(LLVMTypeRef IntTy, unsigned long long N,
@@ -728,6 +761,7 @@ LLVMValueRef LLVMBasicBlockAsValue(LLVMBasicBlockRef BB);
 LLVMBool LLVMValueIsBasicBlock(LLVMValueRef Val);
 LLVMBasicBlockRef LLVMValueAsBasicBlock(LLVMValueRef Val);
 LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB);
+LLVMValueRef LLVMGetBasicBlockTerminator(LLVMBasicBlockRef BB);
 unsigned LLVMCountBasicBlocks(LLVMValueRef Fn);
 void LLVMGetBasicBlocks(LLVMValueRef Fn, LLVMBasicBlockRef *BasicBlocks);
 LLVMBasicBlockRef LLVMGetFirstBasicBlock(LLVMValueRef Fn);
@@ -747,16 +781,21 @@ LLVMBasicBlockRef LLVMAppendBasicBlock(LLVMValueRef Fn, const char *Name);
 LLVMBasicBlockRef LLVMInsertBasicBlock(LLVMBasicBlockRef InsertBeforeBB,
                                        const char *Name);
 void LLVMDeleteBasicBlock(LLVMBasicBlockRef BB);
+void LLVMRemoveBasicBlockFromParent(LLVMBasicBlockRef BB);
 
 void LLVMMoveBasicBlockBefore(LLVMBasicBlockRef BB, LLVMBasicBlockRef MovePos);
 void LLVMMoveBasicBlockAfter(LLVMBasicBlockRef BB, LLVMBasicBlockRef MovePos);
 
-/* Operations on instructions */
-LLVMBasicBlockRef LLVMGetInstructionParent(LLVMValueRef Inst);
 LLVMValueRef LLVMGetFirstInstruction(LLVMBasicBlockRef BB);
 LLVMValueRef LLVMGetLastInstruction(LLVMBasicBlockRef BB);
+
+/* Operations on instructions */
+LLVMBasicBlockRef LLVMGetInstructionParent(LLVMValueRef Inst);
 LLVMValueRef LLVMGetNextInstruction(LLVMValueRef Inst);
 LLVMValueRef LLVMGetPreviousInstruction(LLVMValueRef Inst);
+void LLVMInstructionEraseFromParent(LLVMValueRef Inst);
+LLVMOpcode   LLVMGetInstructionOpcode(LLVMValueRef Inst);
+LLVMIntPredicate LLVMGetICmpPredicate(LLVMValueRef Inst);
 
 /* Operations on call sites */
 void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC);
@@ -771,6 +810,9 @@ void LLVMSetInstrParamAlignment(LLVMValueRef Instr, unsigned index,
 LLVMBool LLVMIsTailCall(LLVMValueRef CallInst);
 void LLVMSetTailCall(LLVMValueRef CallInst, LLVMBool IsTailCall);
 
+/* Operations on switch instructions (only) */
+LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef SwitchInstr);
+
 /* Operations on phi nodes */
 void LLVMAddIncoming(LLVMValueRef PhiNode, LLVMValueRef *IncomingValues,
                      LLVMBasicBlockRef *IncomingBlocks, unsigned Count);
@@ -818,7 +860,10 @@ LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef, LLVMValueRef Fn,
                              LLVMValueRef *Args, unsigned NumArgs,
                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
                              const char *Name);
-LLVMValueRef LLVMBuildUnwind(LLVMBuilderRef);
+LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                 LLVMValueRef PersFn, unsigned NumClauses,
+                                 const char *Name);
+LLVMValueRef LLVMBuildResume(LLVMBuilderRef B, LLVMValueRef Exn);
 LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef);
 
 /* Add a case to the switch instruction */
@@ -828,6 +873,12 @@ void LLVMAddCase(LLVMValueRef Switch, LLVMValueRef OnVal,
 /* Add a destination to the indirectbr instruction */
 void LLVMAddDestination(LLVMValueRef IndirectBr, LLVMBasicBlockRef Dest);
 
+/* Add a catch or filter clause to the landingpad instruction */
+void LLVMAddClause(LLVMValueRef LandingPad, LLVMValueRef ClauseVal);
+
+/* Set the 'cleanup' flag in the landingpad instruction */
+void LLVMSetCleanup(LLVMValueRef LandingPad, LLVMBool Val);
+
 /* Arithmetic */
 LLVMValueRef LLVMBuildAdd(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
                           const char *Name);
@@ -1136,7 +1187,7 @@ namespace llvm {
     return reinterpret_cast<Type**>(Tys);
   }
   
-  inline LLVMTypeRef *wrap(const Type **Tys) {
+  inline LLVMTypeRef *wrap(Type **Tys) {
     return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
   }
   
diff --git a/contrib/llvm/include/llvm-c/Disassembler.h b/contrib/llvm/include/llvm-c/Disassembler.h
index 3a3eb235e983..bf2f2767cd34 100644
--- a/contrib/llvm/include/llvm-c/Disassembler.h
+++ b/contrib/llvm/include/llvm-c/Disassembler.h
@@ -66,7 +66,7 @@ typedef int (*LLVMOpInfoCallback)(void *DisInfo, uint64_t PC,
  */
 struct LLVMOpInfoSymbol1 {
   uint64_t Present;  /* 1 if this symbol is present */
-  char *Name;        /* symbol name if not NULL */
+  const char *Name;  /* symbol name if not NULL */
   uint64_t Value;    /* symbol value if name is NULL */
 };
 
@@ -93,11 +93,35 @@ struct LLVMOpInfo1 {
  * disassembler for things like adding a comment for a PC plus a constant
  * offset load instruction to use a symbol name instead of a load address value.
  * It is passed the block information is saved when the disassembler context is
- * created and a value of a symbol to look up.  If no symbol is found NULL is
- * returned.
+ * created and the ReferenceValue to look up as a symbol.  If no symbol is found
+ * for the ReferenceValue NULL is returned.  The ReferenceType of the
+ * instruction is passed indirectly as is the PC of the instruction in
+ * ReferencePC.  If the output reference can be determined its type is returned
+ * indirectly in ReferenceType along with ReferenceName if any, or that is set
+ * to NULL.
  */
 typedef const char *(*LLVMSymbolLookupCallback)(void *DisInfo,
-                                                uint64_t SymbolValue);
+                                                uint64_t ReferenceValue,
+						uint64_t *ReferenceType,
+						uint64_t ReferencePC,
+						const char **ReferenceName);
+/**
+ * The reference types on input and output.
+ */
+/* No input reference type or no output reference type. */
+#define LLVMDisassembler_ReferenceType_InOut_None 0
+
+/* The input reference is from a branch instruction. */
+#define LLVMDisassembler_ReferenceType_In_Branch 1
+/* The input reference is from a PC relative load instruction. */
+#define LLVMDisassembler_ReferenceType_In_PCrel_Load 2
+
+/* The output reference is to as symbol stub. */
+#define LLVMDisassembler_ReferenceType_Out_SymbolStub 1
+/* The output reference is to a symbol address in a literal pool. */
+#define LLVMDisassembler_ReferenceType_Out_LitPool_SymAddr 2
+/* The output reference is to a cstring address in a literal pool. */
+#define LLVMDisassembler_ReferenceType_Out_LitPool_CstrAddr 3
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/contrib/llvm/include/llvm-c/Object.h b/contrib/llvm/include/llvm-c/Object.h
index 6e72b5946644..7b1cf717f777 100644
--- a/contrib/llvm/include/llvm-c/Object.h
+++ b/contrib/llvm/include/llvm-c/Object.h
@@ -59,14 +59,14 @@ namespace llvm {
       return reinterpret_cast<LLVMObjectFileRef>(const_cast<ObjectFile*>(OF));
     }
 
-    inline ObjectFile::section_iterator *unwrap(LLVMSectionIteratorRef SI) {
-      return reinterpret_cast<ObjectFile::section_iterator*>(SI);
+    inline section_iterator *unwrap(LLVMSectionIteratorRef SI) {
+      return reinterpret_cast<section_iterator*>(SI);
     }
 
     inline LLVMSectionIteratorRef
-    wrap(const ObjectFile::section_iterator *SI) {
+    wrap(const section_iterator *SI) {
       return reinterpret_cast<LLVMSectionIteratorRef>
-        (const_cast<ObjectFile::section_iterator*>(SI));
+        (const_cast<section_iterator*>(SI));
     }
   }
 }
diff --git a/contrib/llvm/include/llvm-c/Target.h b/contrib/llvm/include/llvm-c/Target.h
index d21644011ad4..7afaef15c419 100644
--- a/contrib/llvm/include/llvm-c/Target.h
+++ b/contrib/llvm/include/llvm-c/Target.h
@@ -29,6 +29,7 @@ extern "C" {
 enum LLVMByteOrdering { LLVMBigEndian, LLVMLittleEndian };
 
 typedef struct LLVMOpaqueTargetData *LLVMTargetDataRef;
+typedef struct LLVMOpaqueTargetLibraryInfotData *LLVMTargetLibraryInfoRef;
 typedef struct LLVMStructLayout *LLVMStructLayoutRef;
 
 /* Declare all of the target-initialization functions that are available. */
@@ -42,7 +43,7 @@ typedef struct LLVMStructLayout *LLVMStructLayoutRef;
 #undef LLVM_TARGET  /* Explicit undef to make SWIG happier */
 
 #define LLVM_TARGET(TargetName) \
-  void LLVMInitialize##TargetName##MCAsmInfo(void);
+  void LLVMInitialize##TargetName##TargetMC(void);
 #include "llvm/Config/Targets.def"
 #undef LLVM_TARGET  /* Explicit undef to make SWIG happier */
   
@@ -72,7 +73,7 @@ static inline LLVMBool LLVMInitializeNativeTarget(void) {
 #ifdef LLVM_NATIVE_TARGET
   LLVM_NATIVE_TARGETINFO();
   LLVM_NATIVE_TARGET();
-  LLVM_NATIVE_MCASMINFO();
+  LLVM_NATIVE_TARGETMC();
   return 0;
 #else
   return 1;
@@ -90,6 +91,11 @@ LLVMTargetDataRef LLVMCreateTargetData(const char *StringRep);
     See the method llvm::PassManagerBase::add. */
 void LLVMAddTargetData(LLVMTargetDataRef, LLVMPassManagerRef);
 
+/** Adds target library information to a pass manager. This does not take
+    ownership of the target library info.
+    See the method llvm::PassManagerBase::add. */
+void LLVMAddTargetLibraryInfo(LLVMTargetLibraryInfoRef, LLVMPassManagerRef);
+
 /** Converts target data to a target layout string. The string must be disposed
     with LLVMDisposeMessage.
     See the constructor llvm::TargetData::TargetData. */
@@ -157,6 +163,7 @@ void LLVMDisposeTargetData(LLVMTargetDataRef);
 
 namespace llvm {
   class TargetData;
+  class TargetLibraryInfo;
 
   inline TargetData *unwrap(LLVMTargetDataRef P) {
     return reinterpret_cast<TargetData*>(P);
@@ -165,6 +172,15 @@ namespace llvm {
   inline LLVMTargetDataRef wrap(const TargetData *P) {
     return reinterpret_cast<LLVMTargetDataRef>(const_cast<TargetData*>(P));
   }
+
+  inline TargetLibraryInfo *unwrap(LLVMTargetLibraryInfoRef P) {
+    return reinterpret_cast<TargetLibraryInfo*>(P);
+  }
+
+  inline LLVMTargetLibraryInfoRef wrap(const TargetLibraryInfo *P) {
+    TargetLibraryInfo *X = const_cast<TargetLibraryInfo*>(P);
+    return reinterpret_cast<LLVMTargetLibraryInfoRef>(X);
+  }
 }
 
 #endif /* defined(__cplusplus) */
diff --git a/contrib/llvm/include/llvm-c/Transforms/IPO.h b/contrib/llvm/include/llvm-c/Transforms/IPO.h
index 89b129869c7b..710bebe598be 100644
--- a/contrib/llvm/include/llvm-c/Transforms/IPO.h
+++ b/contrib/llvm/include/llvm-c/Transforms/IPO.h
@@ -36,6 +36,9 @@ void LLVMAddFunctionAttrsPass(LLVMPassManagerRef PM);
 /** See llvm::createFunctionInliningPass function. */
 void LLVMAddFunctionInliningPass(LLVMPassManagerRef PM);
 
+/** See llvm::createAlwaysInlinerPass function. */
+void LLVMAddAlwaysInlinerPass(LLVMPassManagerRef PM);
+
 /** See llvm::createGlobalDCEPass function. */
 void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM);
 
@@ -45,9 +48,6 @@ void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM);
 /** See llvm::createIPConstantPropagationPass function. */
 void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM);
 
-/** See llvm::createLowerSetJmpPass function. */
-void LLVMAddLowerSetJmpPass(LLVMPassManagerRef PM);
-
 /** See llvm::createPruneEHPass function. */
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM);
 
@@ -57,9 +57,6 @@ void LLVMAddIPSCCPPass(LLVMPassManagerRef PM);
 /** See llvm::createInternalizePass function. */
 void LLVMAddInternalizePass(LLVMPassManagerRef, unsigned AllButMain);
 
-// FIXME: Remove in LLVM 3.0.
-void LLVMAddRaiseAllocationsPass(LLVMPassManagerRef PM);
-
 /** See llvm::createStripDeadPrototypesPass function. */
 void LLVMAddStripDeadPrototypesPass(LLVMPassManagerRef PM);
 
diff --git a/contrib/llvm/include/llvm-c/Transforms/PassManagerBuilder.h b/contrib/llvm/include/llvm-c/Transforms/PassManagerBuilder.h
new file mode 100644
index 000000000000..fa722c95874d
--- /dev/null
+++ b/contrib/llvm/include/llvm-c/Transforms/PassManagerBuilder.h
@@ -0,0 +1,90 @@
+/*===-- llvm-c/Transform/PassManagerBuilder.h - PMB C Interface ---*- C -*-===*\
+|*                                                                            *|
+|*                     The LLVM Compiler Infrastructure                       *|
+|*                                                                            *|
+|* This file is distributed under the University of Illinois Open Source      *|
+|* License. See LICENSE.TXT for details.                                      *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This header declares the C interface to the PassManagerBuilder class.      *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_C_PASSMANAGERBUILDER
+#define LLVM_C_PASSMANAGERBUILDER
+
+#include "llvm-c/Core.h"
+
+typedef struct LLVMOpaquePassManagerBuilder *LLVMPassManagerBuilderRef;
+
+#ifdef __cplusplus
+#include "llvm/Transforms/IPO/PassManagerBuilder.h"
+extern "C" {
+#endif
+
+/** See llvm::PassManagerBuilder. */
+LLVMPassManagerBuilderRef LLVMPassManagerBuilderCreate(void);
+void LLVMPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB);
+
+/** See llvm::PassManagerBuilder::OptLevel. */
+void
+LLVMPassManagerBuilderSetOptLevel(LLVMPassManagerBuilderRef PMB,
+                                  unsigned OptLevel);
+
+/** See llvm::PassManagerBuilder::SizeLevel. */
+void
+LLVMPassManagerBuilderSetSizeLevel(LLVMPassManagerBuilderRef PMB,
+                                   unsigned SizeLevel);
+
+/** See llvm::PassManagerBuilder::DisableUnitAtATime. */
+void
+LLVMPassManagerBuilderSetDisableUnitAtATime(LLVMPassManagerBuilderRef PMB,
+                                            LLVMBool Value);
+
+/** See llvm::PassManagerBuilder::DisableUnrollLoops. */
+void
+LLVMPassManagerBuilderSetDisableUnrollLoops(LLVMPassManagerBuilderRef PMB,
+                                            LLVMBool Value);
+
+/** See llvm::PassManagerBuilder::DisableSimplifyLibCalls */
+void
+LLVMPassManagerBuilderSetDisableSimplifyLibCalls(LLVMPassManagerBuilderRef PMB,
+                                                 LLVMBool Value);
+
+/** See llvm::PassManagerBuilder::Inliner. */
+void
+LLVMPassManagerBuilderUseInlinerWithThreshold(LLVMPassManagerBuilderRef PMB,
+                                              unsigned Threshold);
+
+/** See llvm::PassManagerBuilder::populateFunctionPassManager. */
+void
+LLVMPassManagerBuilderPopulateFunctionPassManager(LLVMPassManagerBuilderRef PMB,
+                                                  LLVMPassManagerRef PM);
+
+/** See llvm::PassManagerBuilder::populateModulePassManager. */
+void
+LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB,
+                                                LLVMPassManagerRef PM);
+
+/** See llvm::PassManagerBuilder::populateLTOPassManager. */
+void LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB,
+                                                  LLVMPassManagerRef PM,
+                                                  bool Internalize,
+                                                  bool RunInliner);
+
+#ifdef __cplusplus
+}
+
+namespace llvm {
+  inline PassManagerBuilder *unwrap(LLVMPassManagerBuilderRef P) {
+    return reinterpret_cast<PassManagerBuilder*>(P);
+  }
+
+  inline LLVMPassManagerBuilderRef wrap(PassManagerBuilder *P) {
+    return reinterpret_cast<LLVMPassManagerBuilderRef>(P);
+  }
+}
+#endif
+
+#endif
diff --git a/contrib/llvm/include/llvm-c/Transforms/Scalar.h b/contrib/llvm/include/llvm-c/Transforms/Scalar.h
index cf8d71f5d007..6015ef90eed2 100644
--- a/contrib/llvm/include/llvm-c/Transforms/Scalar.h
+++ b/contrib/llvm/include/llvm-c/Transforms/Scalar.h
@@ -107,6 +107,9 @@ void LLVMAddCorrelatedValuePropagationPass(LLVMPassManagerRef PM);
 /** See llvm::createEarlyCSEPass function */
 void LLVMAddEarlyCSEPass(LLVMPassManagerRef PM);
 
+/** See llvm::createLowerExpectIntrinsicPass function */
+void LLVMAddLowerExpectIntrinsicPass(LLVMPassManagerRef PM);
+
 /** See llvm::createTypeBasedAliasAnalysisPass function */
 void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM);
 
diff --git a/contrib/llvm/include/llvm/ADT/APInt.h b/contrib/llvm/include/llvm/ADT/APInt.h
index e68e579cdfc6..707e0dbb6b91 100644
--- a/contrib/llvm/include/llvm/ADT/APInt.h
+++ b/contrib/llvm/include/llvm/ADT/APInt.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_APINT_H
 #define LLVM_APINT_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
 #include <climits>
@@ -160,7 +161,7 @@ class APInt {
   /// not assume that the string is well-formed and (2) grows the
   /// result to hold the input.
   ///
-  /// @param radix 2, 8, 10, or 16
+  /// @param radix 2, 8, 10, 16, or 36
   /// @brief Convert a char array into an APInt
   void fromString(unsigned numBits, StringRef str, uint8_t radix);
 
@@ -176,6 +177,9 @@ class APInt {
   /// out-of-line slow case for inline constructor
   void initSlowCase(unsigned numBits, uint64_t val, bool isSigned);
 
+  /// shared code between two array constructors
+  void initFromArray(ArrayRef<uint64_t> array);
+
   /// out-of-line slow case for inline copy constructor
   void initSlowCase(const APInt& that);
 
@@ -230,19 +234,26 @@ class APInt {
     clearUnusedBits();
   }
 
-  /// Note that numWords can be smaller or larger than the corresponding bit
-  /// width but any extraneous bits will be dropped.
+  /// Note that bigVal.size() can be smaller or larger than the corresponding
+  /// bit width but any extraneous bits will be dropped.
   /// @param numBits the bit width of the constructed APInt
-  /// @param numWords the number of words in bigVal
   /// @param bigVal a sequence of words to form the initial value of the APInt
   /// @brief Construct an APInt of numBits width, initialized as bigVal[].
+  APInt(unsigned numBits, ArrayRef<uint64_t> bigVal);
+  /// Equivalent to APInt(numBits, ArrayRef<uint64_t>(bigVal, numWords)), but
+  /// deprecated because this constructor is prone to ambiguity with the
+  /// APInt(unsigned, uint64_t, bool) constructor.
+  ///
+  /// If this overload is ever deleted, care should be taken to prevent calls
+  /// from being incorrectly captured by the APInt(unsigned, uint64_t, bool)
+  /// constructor.
   APInt(unsigned numBits, unsigned numWords, const uint64_t bigVal[]);
 
   /// This constructor interprets the string \arg str in the given radix. The
   /// interpretation stops when the first character that is not suitable for the
   /// radix is encountered, or the end of the string. Acceptable radix values
-  /// are 2, 8, 10 and 16. It is an error for the value implied by the string to
-  /// require more bits than numBits.
+  /// are 2, 8, 10, 16, and 36. It is an error for the value implied by the 
+  /// string to require more bits than numBits.
   ///
   /// @param numBits the bit width of the constructed APInt
   /// @param str the string to be interpreted
@@ -342,7 +353,8 @@ class APInt {
 
     if (isSingleWord())
       return isUIntN(N, VAL);
-    return APInt(N, getNumWords(), pVal).zext(getBitWidth()) == (*this);
+    return APInt(N, makeArrayRef(pVal, getNumWords())).zext(getBitWidth())
+      == (*this);
   }
 
   /// @brief Check if this APInt has an N-bits signed integer value.
@@ -1245,13 +1257,13 @@ class APInt {
                 bool formatAsCLiteral = false) const;
 
   /// Considers the APInt to be unsigned and converts it into a string in the
-  /// radix given. The radix can be 2, 8, 10 or 16.
+  /// radix given. The radix can be 2, 8, 10 16, or 36.
   void toStringUnsigned(SmallVectorImpl<char> &Str, unsigned Radix = 10) const {
     toString(Str, Radix, false, false);
   }
 
   /// Considers the APInt to be signed and converts it into a string in the
-  /// radix given. The radix can be 2, 8, 10 or 16.
+  /// radix given. The radix can be 2, 8, 10, 16, or 36.
   void toStringSigned(SmallVectorImpl<char> &Str, unsigned Radix = 10) const {
     toString(Str, Radix, true, false);
   }
diff --git a/contrib/llvm/include/llvm/ADT/ArrayRef.h b/contrib/llvm/include/llvm/ADT/ArrayRef.h
index 6db866e8c70d..33a8c651b23a 100644
--- a/contrib/llvm/include/llvm/ADT/ArrayRef.h
+++ b/contrib/llvm/include/llvm/ADT/ArrayRef.h
@@ -147,7 +147,53 @@ namespace llvm {
     
     /// @}
   };
-  
+
+  /// @name ArrayRef Convenience constructors
+  /// @{
+
+  /// Construct an ArrayRef from a single element.
+  template<typename T>
+  ArrayRef<T> makeArrayRef(const T &OneElt) {
+    return OneElt;
+  }
+
+  /// Construct an ArrayRef from a pointer and length.
+  template<typename T>
+  ArrayRef<T> makeArrayRef(const T *data, size_t length) {
+    return ArrayRef<T>(data, length);
+  }
+
+  /// Construct an ArrayRef from a range.
+  template<typename T>
+  ArrayRef<T> makeArrayRef(const T *begin, const T *end) {
+    return ArrayRef<T>(begin, end);
+  }
+
+  /// Construct an ArrayRef from a SmallVector.
+  template <typename T>
+  ArrayRef<T> makeArrayRef(const SmallVectorImpl<T> &Vec) {
+    return Vec;
+  }
+
+  /// Construct an ArrayRef from a SmallVector.
+  template <typename T, unsigned N>
+  ArrayRef<T> makeArrayRef(const SmallVector<T, N> &Vec) {
+    return Vec;
+  }
+
+  /// Construct an ArrayRef from a std::vector.
+  template<typename T>
+  ArrayRef<T> makeArrayRef(const std::vector<T> &Vec) {
+    return Vec;
+  }
+
+  /// Construct an ArrayRef from a C array.
+  template<typename T, size_t N>
+  ArrayRef<T> makeArrayRef(const T (&Arr)[N]) {
+    return ArrayRef<T>(Arr);
+  }
+
+  /// @}
   /// @name ArrayRef Comparison Operators
   /// @{
 
diff --git a/contrib/llvm/include/llvm/ADT/DenseMap.h b/contrib/llvm/include/llvm/ADT/DenseMap.h
index 0f1cfebc3672..e70cacf3ca5b 100644
--- a/contrib/llvm/include/llvm/ADT/DenseMap.h
+++ b/contrib/llvm/include/llvm/ADT/DenseMap.h
@@ -540,6 +540,12 @@ class DenseMapIterator {
       ++Ptr;
   }
 };
+  
+template<typename KeyT, typename ValueT, typename KeyInfoT, typename ValueInfoT>
+static inline size_t
+capacity_in_bytes(const DenseMap<KeyT, ValueT, KeyInfoT, ValueInfoT> &X) {
+  return X.getMemorySize();
+}
 
 } // end namespace llvm
 
diff --git a/contrib/llvm/include/llvm/ADT/DenseMapInfo.h b/contrib/llvm/include/llvm/ADT/DenseMapInfo.h
index 744b6f4aef30..df4084e6f411 100644
--- a/contrib/llvm/include/llvm/ADT/DenseMapInfo.h
+++ b/contrib/llvm/include/llvm/ADT/DenseMapInfo.h
@@ -51,7 +51,7 @@ struct DenseMapInfo<T*> {
 template<> struct DenseMapInfo<char> {
   static inline char getEmptyKey() { return ~0; }
   static inline char getTombstoneKey() { return ~0 - 1; }
-  static unsigned getHashValue(const char& Val) { return Val * 37; }
+  static unsigned getHashValue(const char& Val) { return Val * 37U; }
   static bool isEqual(const char &LHS, const char &RHS) {
     return LHS == RHS;
   }
@@ -61,7 +61,7 @@ template<> struct DenseMapInfo<char> {
 template<> struct DenseMapInfo<unsigned> {
   static inline unsigned getEmptyKey() { return ~0; }
   static inline unsigned getTombstoneKey() { return ~0U - 1; }
-  static unsigned getHashValue(const unsigned& Val) { return Val * 37; }
+  static unsigned getHashValue(const unsigned& Val) { return Val * 37U; }
   static bool isEqual(const unsigned& LHS, const unsigned& RHS) {
     return LHS == RHS;
   }
@@ -96,7 +96,7 @@ template<> struct DenseMapInfo<unsigned long long> {
 template<> struct DenseMapInfo<int> {
   static inline int getEmptyKey() { return 0x7fffffff; }
   static inline int getTombstoneKey() { return -0x7fffffff - 1; }
-  static unsigned getHashValue(const int& Val) { return (unsigned)(Val * 37); }
+  static unsigned getHashValue(const int& Val) { return (unsigned)(Val * 37U); }
   static bool isEqual(const int& LHS, const int& RHS) {
     return LHS == RHS;
   }
@@ -109,7 +109,7 @@ template<> struct DenseMapInfo<long> {
   }
   static inline long getTombstoneKey() { return getEmptyKey() - 1L; }
   static unsigned getHashValue(const long& Val) {
-    return (unsigned)(Val * 37L);
+    return (unsigned)(Val * 37UL);
   }
   static bool isEqual(const long& LHS, const long& RHS) {
     return LHS == RHS;
@@ -121,7 +121,7 @@ template<> struct DenseMapInfo<long long> {
   static inline long long getEmptyKey() { return 0x7fffffffffffffffLL; }
   static inline long long getTombstoneKey() { return -0x7fffffffffffffffLL-1; }
   static unsigned getHashValue(const long long& Val) {
-    return (unsigned)(Val * 37LL);
+    return (unsigned)(Val * 37ULL);
   }
   static bool isEqual(const long long& LHS,
                       const long long& RHS) {
@@ -142,7 +142,7 @@ struct DenseMapInfo<std::pair<T, U> > {
   }
   static inline Pair getTombstoneKey() {
     return std::make_pair(FirstInfo::getTombstoneKey(),
-                            SecondInfo::getEmptyKey());
+                          SecondInfo::getTombstoneKey());
   }
   static unsigned getHashValue(const Pair& PairVal) {
     uint64_t key = (uint64_t)FirstInfo::getHashValue(PairVal.first) << 32
@@ -158,7 +158,7 @@ struct DenseMapInfo<std::pair<T, U> > {
     return (unsigned)key;
   }
   static bool isEqual(const Pair &LHS, const Pair &RHS) {
-    return FirstInfo::isEqual(LHS.first, RHS.first) && 
+    return FirstInfo::isEqual(LHS.first, RHS.first) &&
            SecondInfo::isEqual(LHS.second, RHS.second);
   }
 };
diff --git a/contrib/llvm/include/llvm/ADT/DenseSet.h b/contrib/llvm/include/llvm/ADT/DenseSet.h
index 67321f539848..8ab9a33200c3 100644
--- a/contrib/llvm/include/llvm/ADT/DenseSet.h
+++ b/contrib/llvm/include/llvm/ADT/DenseSet.h
@@ -28,7 +28,7 @@ class DenseSet {
   MapTy TheMap;
 public:
   DenseSet(const DenseSet &Other) : TheMap(Other.TheMap) {}
-  explicit DenseSet(unsigned NumInitBuckets = 64) : TheMap(NumInitBuckets) {}
+  explicit DenseSet(unsigned NumInitBuckets = 0) : TheMap(NumInitBuckets) {}
 
   bool empty() const { return TheMap.empty(); }
   unsigned size() const { return TheMap.size(); }
diff --git a/contrib/llvm/include/llvm/ADT/ImmutableMap.h b/contrib/llvm/include/llvm/ADT/ImmutableMap.h
index d6cce7ccfa05..8346ffabff76 100644
--- a/contrib/llvm/include/llvm/ADT/ImmutableMap.h
+++ b/contrib/llvm/include/llvm/ADT/ImmutableMap.h
@@ -117,6 +117,10 @@ class ImmutableMap {
       return ImmutableMap(Canonicalize ? F.getCanonicalTree(T): T);
     }
 
+    typename TreeTy::Factory *getTreeFactory() const {
+      return const_cast<typename TreeTy::Factory *>(&F);
+    }
+
   private:
     Factory(const Factory& RHS); // DO NOT IMPLEMENT
     void operator=(const Factory& RHS); // DO NOT IMPLEMENT
@@ -256,6 +260,159 @@ class ImmutableMap {
   }
 };
 
+// NOTE: This will possibly become the new implementation of ImmutableMap some day.
+template <typename KeyT, typename ValT,
+typename ValInfo = ImutKeyValueInfo<KeyT,ValT> >
+class ImmutableMapRef {
+public:
+  typedef typename ValInfo::value_type      value_type;
+  typedef typename ValInfo::value_type_ref  value_type_ref;
+  typedef typename ValInfo::key_type        key_type;
+  typedef typename ValInfo::key_type_ref    key_type_ref;
+  typedef typename ValInfo::data_type       data_type;
+  typedef typename ValInfo::data_type_ref   data_type_ref;
+  typedef ImutAVLTree<ValInfo>              TreeTy;
+  typedef typename TreeTy::Factory          FactoryTy;
+  
+protected:
+  TreeTy *Root;
+  FactoryTy *Factory;
+  
+public:
+  /// Constructs a map from a pointer to a tree root.  In general one
+  /// should use a Factory object to create maps instead of directly
+  /// invoking the constructor, but there are cases where make this
+  /// constructor public is useful.
+  explicit ImmutableMapRef(const TreeTy* R, FactoryTy *F) 
+    : Root(const_cast<TreeTy*>(R)),
+      Factory(F) {
+    if (Root) { Root->retain(); }
+  }
+  
+  ImmutableMapRef(const ImmutableMapRef &X)
+    : Root(X.Root),
+      Factory(X.Factory) {
+    if (Root) { Root->retain(); }
+  }
+
+  ImmutableMapRef &operator=(const ImmutableMapRef &X) {
+    if (Root != X.Root) {
+      if (X.Root)
+        X.Root->retain();
+      
+      if (Root)
+        Root->release();
+      
+      Root = X.Root;
+      Factory = X.Factory;
+    }
+    return *this;
+  }
+
+  ~ImmutableMapRef() {
+    if (Root)
+      Root->release();
+  }
+  
+  static inline ImmutableMapRef getEmptyMap(FactoryTy *F) {
+    return ImmutableMapRef(0, F);
+  }
+
+  ImmutableMapRef add(key_type_ref K, data_type_ref D) {
+    TreeTy *NewT = Factory->add(Root, std::pair<key_type, data_type>(K, D));
+    return ImmutableMapRef(NewT, Factory);
+  }
+
+  ImmutableMapRef remove(key_type_ref K) {
+    TreeTy *NewT = Factory->remove(Root, K);
+    return ImmutableMapRef(NewT, Factory);
+  }
+  
+  bool contains(key_type_ref K) const {
+    return Root ? Root->contains(K) : false;
+  }
+  
+  ImmutableMap<KeyT, ValT> asImmutableMap() const {
+    return ImmutableMap<KeyT, ValT>(Factory->getCanonicalTree(Root));
+  }
+  
+  bool operator==(const ImmutableMapRef &RHS) const {
+    return Root && RHS.Root ? Root->isEqual(*RHS.Root) : Root == RHS.Root;
+  }
+  
+  bool operator!=(const ImmutableMapRef &RHS) const {
+    return Root && RHS.Root ? Root->isNotEqual(*RHS.Root) : Root != RHS.Root;
+  }
+    
+  bool isEmpty() const { return !Root; }
+    
+  //===--------------------------------------------------===//
+  // For testing.
+  //===--------------------------------------------------===//
+  
+  void verify() const { if (Root) Root->verify(); }
+  
+  //===--------------------------------------------------===//
+  // Iterators.
+  //===--------------------------------------------------===//
+  
+  class iterator {
+    typename TreeTy::iterator itr;
+    
+    iterator() {}
+    iterator(TreeTy* t) : itr(t) {}
+    friend class ImmutableMapRef;
+    
+  public:
+    value_type_ref operator*() const { return itr->getValue(); }
+    value_type*    operator->() const { return &itr->getValue(); }
+    
+    key_type_ref getKey() const { return itr->getValue().first; }
+    data_type_ref getData() const { return itr->getValue().second; }
+    
+    
+    iterator& operator++() { ++itr; return *this; }
+    iterator  operator++(int) { iterator tmp(*this); ++itr; return tmp; }
+    iterator& operator--() { --itr; return *this; }
+    iterator  operator--(int) { iterator tmp(*this); --itr; return tmp; }
+    bool operator==(const iterator& RHS) const { return RHS.itr == itr; }
+    bool operator!=(const iterator& RHS) const { return RHS.itr != itr; }
+  };
+  
+  iterator begin() const { return iterator(Root); }
+  iterator end() const { return iterator(); }
+  
+  data_type* lookup(key_type_ref K) const {
+    if (Root) {
+      TreeTy* T = Root->find(K);
+      if (T) return &T->getValue().second;
+    }
+    
+    return 0;
+  }
+  
+  /// getMaxElement - Returns the <key,value> pair in the ImmutableMap for
+  ///  which key is the highest in the ordering of keys in the map.  This
+  ///  method returns NULL if the map is empty.
+  value_type* getMaxElement() const {
+    return Root ? &(Root->getMaxElement()->getValue()) : 0;
+  }
+  
+  //===--------------------------------------------------===//
+  // Utility methods.
+  //===--------------------------------------------------===//
+  
+  unsigned getHeight() const { return Root ? Root->getHeight() : 0; }
+  
+  static inline void Profile(FoldingSetNodeID& ID, const ImmutableMapRef &M) {
+    ID.AddPointer(M.Root);
+  }
+  
+  inline void Profile(FoldingSetNodeID& ID) const {
+    return Profile(ID, *this);
+  }
+};
+  
 } // end namespace llvm
 
 #endif
diff --git a/contrib/llvm/include/llvm/ADT/ImmutableSet.h b/contrib/llvm/include/llvm/ADT/ImmutableSet.h
index 3ca910ce944f..d597a7c9be72 100644
--- a/contrib/llvm/include/llvm/ADT/ImmutableSet.h
+++ b/contrib/llvm/include/llvm/ADT/ImmutableSet.h
@@ -997,6 +997,10 @@ class ImmutableSet {
 
     BumpPtrAllocator& getAllocator() { return F.getAllocator(); }
 
+    typename TreeTy::Factory *getTreeFactory() const {
+      return const_cast<typename TreeTy::Factory *>(&F);
+    }
+    
   private:
     Factory(const Factory& RHS); // DO NOT IMPLEMENT
     void operator=(const Factory& RHS); // DO NOT IMPLEMENT
@@ -1021,6 +1025,10 @@ class ImmutableSet {
     if (Root) { Root->retain(); }
     return Root;
   }
+  
+  TreeTy *getRootWithoutRetain() const {
+    return Root;
+  }
 
   /// isEmpty - Return true if the set contains no elements.
   bool isEmpty() const { return !Root; }
@@ -1078,6 +1086,132 @@ class ImmutableSet {
 
   void validateTree() const { if (Root) Root->validateTree(); }
 };
+  
+// NOTE: This may some day replace the current ImmutableSet.
+template <typename ValT, typename ValInfo = ImutContainerInfo<ValT> >
+class ImmutableSetRef {
+public:
+  typedef typename ValInfo::value_type      value_type;
+  typedef typename ValInfo::value_type_ref  value_type_ref;
+  typedef ImutAVLTree<ValInfo> TreeTy;
+  typedef typename TreeTy::Factory          FactoryTy;
+  
+private:
+  TreeTy *Root;
+  FactoryTy *Factory;
+  
+public:
+  /// Constructs a set from a pointer to a tree root.  In general one
+  /// should use a Factory object to create sets instead of directly
+  /// invoking the constructor, but there are cases where make this
+  /// constructor public is useful.
+  explicit ImmutableSetRef(TreeTy* R, FactoryTy *F)
+    : Root(R),
+      Factory(F) {
+    if (Root) { Root->retain(); }
+  }
+  ImmutableSetRef(const ImmutableSetRef &X)
+    : Root(X.Root),
+      Factory(X.Factory) {
+    if (Root) { Root->retain(); }
+  }
+  ImmutableSetRef &operator=(const ImmutableSetRef &X) {
+    if (Root != X.Root) {
+      if (X.Root) { X.Root->retain(); }
+      if (Root) { Root->release(); }
+      Root = X.Root;
+      Factory = X.Factory;
+    }
+    return *this;
+  }
+  ~ImmutableSetRef() {
+    if (Root) { Root->release(); }
+  }
+  
+  static inline ImmutableSetRef getEmptySet(FactoryTy *F) {
+    return ImmutableSetRef(0, F);
+  }
+  
+  ImmutableSetRef add(value_type_ref V) {
+    return ImmutableSetRef(Factory->add(Root, V), Factory);
+  }
+  
+  ImmutableSetRef remove(value_type_ref V) {
+    return ImmutableSetRef(Factory->remove(Root, V), Factory);
+  }
+    
+  /// Returns true if the set contains the specified value.
+  bool contains(value_type_ref V) const {
+    return Root ? Root->contains(V) : false;
+  }
+  
+  ImmutableSet<ValT> asImmutableSet(bool canonicalize = true) const {
+    return ImmutableSet<ValT>(canonicalize ?
+                              Factory->getCanonicalTree(Root) : Root);
+  }
+  
+  TreeTy *getRootWithoutRetain() const {
+    return Root;
+  }
+  
+  bool operator==(const ImmutableSetRef &RHS) const {
+    return Root && RHS.Root ? Root->isEqual(*RHS.Root) : Root == RHS.Root;
+  }
+  
+  bool operator!=(const ImmutableSetRef &RHS) const {
+    return Root && RHS.Root ? Root->isNotEqual(*RHS.Root) : Root != RHS.Root;
+  }
+
+  /// isEmpty - Return true if the set contains no elements.
+  bool isEmpty() const { return !Root; }
+  
+  /// isSingleton - Return true if the set contains exactly one element.
+  ///   This method runs in constant time.
+  bool isSingleton() const { return getHeight() == 1; }
+
+  //===--------------------------------------------------===//
+  // Iterators.
+  //===--------------------------------------------------===//
+  
+  class iterator {
+    typename TreeTy::iterator itr;
+    iterator(TreeTy* t) : itr(t) {}
+    friend class ImmutableSetRef<ValT,ValInfo>;
+  public:
+    iterator() {}
+    inline value_type_ref operator*() const { return itr->getValue(); }
+    inline iterator& operator++() { ++itr; return *this; }
+    inline iterator  operator++(int) { iterator tmp(*this); ++itr; return tmp; }
+    inline iterator& operator--() { --itr; return *this; }
+    inline iterator  operator--(int) { iterator tmp(*this); --itr; return tmp; }
+    inline bool operator==(const iterator& RHS) const { return RHS.itr == itr; }
+    inline bool operator!=(const iterator& RHS) const { return RHS.itr != itr; }
+    inline value_type *operator->() const { return &(operator*()); }
+  };
+  
+  iterator begin() const { return iterator(Root); }
+  iterator end() const { return iterator(); }
+  
+  //===--------------------------------------------------===//
+  // Utility methods.
+  //===--------------------------------------------------===//
+  
+  unsigned getHeight() const { return Root ? Root->getHeight() : 0; }
+  
+  static inline void Profile(FoldingSetNodeID& ID, const ImmutableSetRef& S) {
+    ID.AddPointer(S.Root);
+  }
+  
+  inline void Profile(FoldingSetNodeID& ID) const {
+    return Profile(ID,*this);
+  }
+  
+  //===--------------------------------------------------===//
+  // For testing.
+  //===--------------------------------------------------===//
+  
+  void validateTree() const { if (Root) Root->validateTree(); }
+};
 
 } // end namespace llvm
 
diff --git a/contrib/llvm/include/llvm/ADT/IntervalMap.h b/contrib/llvm/include/llvm/ADT/IntervalMap.h
index f28ebf3b9a5f..1230e8f5fb43 100644
--- a/contrib/llvm/include/llvm/ADT/IntervalMap.h
+++ b/contrib/llvm/include/llvm/ADT/IntervalMap.h
@@ -1335,6 +1335,9 @@ class IntervalMap<KeyT, ValT, N, Traits>::const_iterator :
   /// valid - Return true if the current position is valid, false for end().
   bool valid() const { return path.valid(); }
 
+  /// atBegin - Return true if the current position is the first map entry.
+  bool atBegin() const { return path.atBegin(); }
+
   /// start - Return the beginning of the current interval.
   const KeyT &start() const { return unsafeStart(); }
 
diff --git a/contrib/llvm/include/llvm/ADT/PointerUnion.h b/contrib/llvm/include/llvm/ADT/PointerUnion.h
index 13b98cef07ab..487096a17105 100644
--- a/contrib/llvm/include/llvm/ADT/PointerUnion.h
+++ b/contrib/llvm/include/llvm/ADT/PointerUnion.h
@@ -108,7 +108,11 @@ namespace llvm {
     
     /// isNull - Return true if the pointer held in the union is null,
     /// regardless of which type it is.
-    bool isNull() const { return Val.getPointer() == 0; }
+    bool isNull() const {
+      // Convert from the void* to one of the pointer types, to make sure that
+      // we recursively strip off low bits if we have a nested PointerUnion.
+      return !PointerLikeTypeTraits<PT1>::getFromVoidPointer(Val.getPointer());
+    }
     operator bool() const { return !isNull(); }
 
     /// is<T>() return true if the Union currently holds the type matching T.
diff --git a/contrib/llvm/include/llvm/ADT/PostOrderIterator.h b/contrib/llvm/include/llvm/ADT/PostOrderIterator.h
index e3b499488d0c..63a2b5219f73 100644
--- a/contrib/llvm/include/llvm/ADT/PostOrderIterator.h
+++ b/contrib/llvm/include/llvm/ADT/PostOrderIterator.h
@@ -29,6 +29,14 @@ class po_iterator_storage {
   SetType Visited;
 };
 
+/// DFSetTraits - Allow the SetType used to record depth-first search results to
+/// optionally record node postorder.
+template<class SetType>
+struct DFSetTraits {
+  static void finishPostorder(
+    typename SetType::iterator::value_type, SetType &) {}
+};
+
 template<class SetType>
 class po_iterator_storage<SetType, true> {
 public:
@@ -109,6 +117,8 @@ class po_iterator : public std::iterator<std::forward_iterator_tag,
   inline NodeType *operator->() const { return operator*(); }
 
   inline _Self& operator++() {   // Preincrement
+    DFSetTraits<SetType>::finishPostorder(VisitStack.back().first,
+                                          this->Visited);
     VisitStack.pop_back();
     if (!VisitStack.empty())
       traverseChild();
diff --git a/contrib/llvm/include/llvm/ADT/SCCIterator.h b/contrib/llvm/include/llvm/ADT/SCCIterator.h
index 3e93cfe914fa..48436c667474 100644
--- a/contrib/llvm/include/llvm/ADT/SCCIterator.h
+++ b/contrib/llvm/include/llvm/ADT/SCCIterator.h
@@ -1,4 +1,4 @@
-//===-- Support/SCCIterator.h - Strongly Connected Comp. Iter. --*- C++ -*-===//
+//===---- ADT/SCCIterator.h - Strongly Connected Comp. Iter. ----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -87,7 +87,7 @@ class scc_iterator
         DFSVisitOne(childN);
         continue;
       }
-      
+
       unsigned childNum = nodeVisitNumbers[childN];
       if (MinVisitNumStack.back() > childNum)
         MinVisitNumStack.back() = childNum;
@@ -114,7 +114,7 @@ class scc_iterator
 
       if (minVisitNum != nodeVisitNumbers[visitingN])
         continue;
-      
+
       // A full SCC is on the SCCNodeStack!  It includes all nodes below
       // visitingN on the stack.  Copy those nodes to CurrentSCC,
       // reset their minVisit values, and return (this suspends
@@ -139,7 +139,7 @@ class scc_iterator
 
   // Provide static "constructors"...
   static inline _Self begin(const GraphT &G){return _Self(GT::getEntryNode(G));}
-  static inline _Self end  (const GraphT &G) { return _Self(); }
+  static inline _Self end  (const GraphT &) { return _Self(); }
 
   // Direct loop termination test: I.isAtEnd() is more efficient than I == end()
   inline bool isAtEnd() const {
@@ -183,7 +183,7 @@ class scc_iterator
         return true;
     return false;
   }
-                           
+
   /// ReplaceNode - This informs the scc_iterator that the specified Old node
   /// has been deleted, and New is to be used in its place.
   void ReplaceNode(NodeType *Old, NodeType *New) {
diff --git a/contrib/llvm/include/llvm/ADT/STLExtras.h b/contrib/llvm/include/llvm/ADT/STLExtras.h
index 0b0346be2cc5..5da906dc8cf0 100644
--- a/contrib/llvm/include/llvm/ADT/STLExtras.h
+++ b/contrib/llvm/include/llvm/ADT/STLExtras.h
@@ -186,25 +186,21 @@ inline ItTy prior(ItTy it)
 //   // do stuff
 // else
 //   // do other stuff
+template <typename T1, typename T2>
+struct tier {
+  typedef T1 &first_type;
+  typedef T2 &second_type;
 
-namespace
-{
-  template <typename T1, typename T2>
-  struct tier {
-    typedef T1 &first_type;
-    typedef T2 &second_type;
+  first_type first;
+  second_type second;
 
-    first_type first;
-    second_type second;
-
-    tier(first_type f, second_type s) : first(f), second(s) { }
-    tier& operator=(const std::pair<T1, T2>& p) {
-      first = p.first;
-      second = p.second;
-      return *this;
-    }
-  };
-}
+  tier(first_type f, second_type s) : first(f), second(s) { }
+  tier& operator=(const std::pair<T1, T2>& p) {
+    first = p.first;
+    second = p.second;
+    return *this;
+  }
+};
 
 template <typename T1, typename T2>
 inline tier<T1, T2> tie(T1& f, T2& s) {
diff --git a/contrib/llvm/include/llvm/ADT/SmallVector.h b/contrib/llvm/include/llvm/ADT/SmallVector.h
index 5f0a55bec46e..1c42f29771b3 100644
--- a/contrib/llvm/include/llvm/ADT/SmallVector.h
+++ b/contrib/llvm/include/llvm/ADT/SmallVector.h
@@ -78,21 +78,21 @@ class SmallVectorBase {
     return BeginX == static_cast<const void*>(&FirstEl);
   }
 
-  /// size_in_bytes - This returns size()*sizeof(T).
-  size_t size_in_bytes() const {
-    return size_t((char*)EndX - (char*)BeginX);
-  }
-
-  /// capacity_in_bytes - This returns capacity()*sizeof(T).
-  size_t capacity_in_bytes() const {
-    return size_t((char*)CapacityX - (char*)BeginX);
-  }
-
   /// grow_pod - This is an implementation of the grow() method which only works
   /// on POD-like data types and is out of line to reduce code duplication.
   void grow_pod(size_t MinSizeInBytes, size_t TSize);
 
 public:
+  /// size_in_bytes - This returns size()*sizeof(T).
+  size_t size_in_bytes() const {
+    return size_t((char*)EndX - (char*)BeginX);
+  }
+  
+  /// capacity_in_bytes - This returns capacity()*sizeof(T).
+  size_t capacity_in_bytes() const {
+    return size_t((char*)CapacityX - (char*)BeginX);
+  }
+
   bool empty() const { return BeginX == EndX; }
 };
 
@@ -738,6 +738,11 @@ class SmallVector<T,0> : public SmallVectorImpl<T> {
 
 };
 
+template<typename T, unsigned N>
+static inline size_t capacity_in_bytes(const SmallVector<T, N> &X) {
+  return X.capacity_in_bytes();
+}
+
 } // End llvm namespace
 
 namespace std {
diff --git a/contrib/llvm/include/llvm/ADT/Statistic.h b/contrib/llvm/include/llvm/ADT/Statistic.h
index fda99c6edbc3..b8a1a2f5c4e8 100644
--- a/contrib/llvm/include/llvm/ADT/Statistic.h
+++ b/contrib/llvm/include/llvm/ADT/Statistic.h
@@ -54,7 +54,7 @@ class Statistic {
     Value = Val;
     return init();
   }
-  
+
   const Statistic &operator++() {
     // FIXME: This function and all those that follow carefully use an
     // atomic operation to update the value safely in the presence of
@@ -63,41 +63,43 @@ class Statistic {
     sys::AtomicIncrement(&Value);
     return init();
   }
-  
+
   unsigned operator++(int) {
     init();
     unsigned OldValue = Value;
     sys::AtomicIncrement(&Value);
     return OldValue;
   }
-  
+
   const Statistic &operator--() {
     sys::AtomicDecrement(&Value);
     return init();
   }
-  
+
   unsigned operator--(int) {
     init();
     unsigned OldValue = Value;
     sys::AtomicDecrement(&Value);
     return OldValue;
   }
-  
+
   const Statistic &operator+=(const unsigned &V) {
+    if (!V) return *this;
     sys::AtomicAdd(&Value, V);
     return init();
   }
-  
+
   const Statistic &operator-=(const unsigned &V) {
+    if (!V) return *this;
     sys::AtomicAdd(&Value, -V);
     return init();
   }
-  
+
   const Statistic &operator*=(const unsigned &V) {
     sys::AtomicMul(&Value, V);
     return init();
   }
-  
+
   const Statistic &operator/=(const unsigned &V) {
     sys::AtomicDiv(&Value, V);
     return init();
diff --git a/contrib/llvm/include/llvm/ADT/StringExtras.h b/contrib/llvm/include/llvm/ADT/StringExtras.h
index 5f5c04187ada..d01d3e1d6b10 100644
--- a/contrib/llvm/include/llvm/ADT/StringExtras.h
+++ b/contrib/llvm/include/llvm/ADT/StringExtras.h
@@ -16,6 +16,7 @@
 
 #include "llvm/Support/DataTypes.h"
 #include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/StringRef.h"
 #include <cctype>
 #include <cstdio>
diff --git a/contrib/llvm/include/llvm/ADT/TinyPtrVector.h b/contrib/llvm/include/llvm/ADT/TinyPtrVector.h
new file mode 100644
index 000000000000..ee86d8bdf70f
--- /dev/null
+++ b/contrib/llvm/include/llvm/ADT/TinyPtrVector.h
@@ -0,0 +1,133 @@
+//===- llvm/ADT/TinyPtrVector.h - 'Normally tiny' vectors -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_TINYPTRVECTOR_H
+#define LLVM_ADT_TINYPTRVECTOR_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/PointerUnion.h"
+
+namespace llvm {
+  
+/// TinyPtrVector - This class is specialized for cases where there are
+/// normally 0 or 1 element in a vector, but is general enough to go beyond that
+/// when required.
+///
+/// NOTE: This container doesn't allow you to store a null pointer into it.
+///
+template <typename EltTy>
+class TinyPtrVector {
+public:
+  typedef llvm::SmallVector<EltTy, 4> VecTy;
+  llvm::PointerUnion<EltTy, VecTy*> Val;
+  
+  TinyPtrVector() {}
+  TinyPtrVector(const TinyPtrVector &RHS) : Val(RHS.Val) {
+    if (VecTy *V = Val.template dyn_cast<VecTy*>())
+      Val = new VecTy(*V);
+  }
+  ~TinyPtrVector() {
+    if (VecTy *V = Val.template dyn_cast<VecTy*>())
+      delete V;
+  }
+  
+  bool empty() const {
+    // This vector can be empty if it contains no element, or if it
+    // contains a pointer to an empty vector.
+    if (Val.isNull()) return true;
+    if (VecTy *Vec = Val.template dyn_cast<VecTy*>())
+      return Vec->empty();
+    return false;
+  }
+  
+  unsigned size() const {
+    if (empty())
+      return 0;
+    if (Val.template is<EltTy>())
+      return 1;
+    return Val.template get<VecTy*>()->size();
+  }
+  
+  typedef const EltTy *iterator;
+  iterator begin() const {
+    if (empty())
+      return 0;
+    
+    if (Val.template is<EltTy>())
+      return Val.template getAddrOf<EltTy>();
+    
+    return Val.template get<VecTy *>()->begin();
+
+  }
+  iterator end() const {
+    if (empty())
+      return 0;
+    
+    if (Val.template is<EltTy>())
+      return begin() + 1;
+    
+    return Val.template get<VecTy *>()->end();
+  }
+
+  
+  EltTy operator[](unsigned i) const {
+    assert(!Val.isNull() && "can't index into an empty vector");
+    if (EltTy V = Val.template dyn_cast<EltTy>()) {
+      assert(i == 0 && "tinyvector index out of range");
+      return V;
+    }
+    
+    assert(i < Val.template get<VecTy*>()->size() && 
+           "tinyvector index out of range");
+    return (*Val.template get<VecTy*>())[i];
+  }
+  
+  EltTy front() const {
+    assert(!empty() && "vector empty");
+    if (EltTy V = Val.template dyn_cast<EltTy>())
+      return V;
+    return Val.template get<VecTy*>()->front();
+  }
+  
+  void push_back(EltTy NewVal) {
+    assert(NewVal != 0 && "Can't add a null value");
+    
+    // If we have nothing, add something.
+    if (Val.isNull()) {
+      Val = NewVal;
+      return;
+    }
+    
+    // If we have a single value, convert to a vector.
+    if (EltTy V = Val.template dyn_cast<EltTy>()) {
+      Val = new VecTy();
+      Val.template get<VecTy*>()->push_back(V);
+    }
+    
+    // Add the new value, we know we have a vector.
+    Val.template get<VecTy*>()->push_back(NewVal);
+  }
+  
+  void clear() {
+    // If we have a single value, convert to empty.
+    if (Val.template is<EltTy>()) {
+      Val = (EltTy)0;
+    } else if (VecTy *Vec = Val.template dyn_cast<VecTy*>()) {
+      // If we have a vector form, just clear it.
+      Vec->clear();
+    }
+    // Otherwise, we're already empty.
+  }
+  
+private:
+  void operator=(const TinyPtrVector&); // NOT IMPLEMENTED YET.
+};
+} // end namespace llvm
+
+#endif
diff --git a/contrib/llvm/include/llvm/ADT/Triple.h b/contrib/llvm/include/llvm/ADT/Triple.h
index fd23608a7591..3503c0f22145 100644
--- a/contrib/llvm/include/llvm/ADT/Triple.h
+++ b/contrib/llvm/include/llvm/ADT/Triple.h
@@ -10,8 +10,7 @@
 #ifndef LLVM_ADT_TRIPLE_H
 #define LLVM_ADT_TRIPLE_H
 
-#include "llvm/ADT/StringRef.h"
-#include <string>
+#include "llvm/ADT/Twine.h"
 
 // Some system headers or GCC predefined macros conflict with identifiers in
 // this file.  Undefine them here.
@@ -19,8 +18,6 @@
 #undef sparc
 
 namespace llvm {
-class StringRef;
-class Twine;
 
 /// Triple - Helper class for working with target triples.
 ///
@@ -52,6 +49,8 @@ class Triple {
     cellspu, // CellSPU: spu, cellspu
     mips,    // MIPS: mips, mipsallegrex
     mipsel,  // MIPSEL: mipsel, mipsallegrexel, psp
+    mips64,  // MIPS64: mips64
+    mips64el,// MIPS64EL: mips64el
     msp430,  // MSP430: msp430
     ppc,     // PPC: powerpc
     ppc64,   // PPC64: powerpc64, ppu
@@ -66,6 +65,8 @@ class Triple {
     mblaze,  // MBlaze: mblaze
     ptx32,   // PTX: ptx (32-bit)
     ptx64,   // PTX: ptx (64-bit)
+    le32,    // le32: generic little-endian 32-bit CPU (PNaCl / Emscripten)
+    amdil,   // amdil: amd IL
 
     InvalidArch
   };
@@ -85,6 +86,7 @@ class Triple {
     DragonFly,
     FreeBSD,
     IOS,
+    KFreeBSD,
     Linux,
     Lv2,        // PS3
     MacOSX,
@@ -96,7 +98,8 @@ class Triple {
     Win32,
     Haiku,
     Minix,
-    RTEMS
+    RTEMS,
+    NativeClient
   };
   enum EnvironmentType {
     UnknownEnvironment,
@@ -134,24 +137,16 @@ class Triple {
   /// @{
 
   Triple() : Data(), Arch(InvalidArch) {}
-  explicit Triple(StringRef Str) : Data(Str), Arch(InvalidArch) {}
-  explicit Triple(StringRef ArchStr, StringRef VendorStr, StringRef OSStr)
-    : Data(ArchStr), Arch(InvalidArch) {
-    Data += '-';
-    Data += VendorStr;
-    Data += '-';
-    Data += OSStr;
+  explicit Triple(const Twine &Str) : Data(Str.str()), Arch(InvalidArch) {}
+  Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr)
+    : Data((ArchStr + Twine('-') + VendorStr + Twine('-') + OSStr).str()),
+      Arch(InvalidArch) {
   }
 
-  explicit Triple(StringRef ArchStr, StringRef VendorStr, StringRef OSStr,
-    StringRef EnvironmentStr)
-    : Data(ArchStr), Arch(InvalidArch) {
-    Data += '-';
-    Data += VendorStr;
-    Data += '-';
-    Data += OSStr;
-    Data += '-';
-    Data += EnvironmentStr;
+  Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr,
+         const Twine &EnvironmentStr)
+    : Data((ArchStr + Twine('-') + VendorStr + Twine('-') + OSStr + Twine('-') +
+            EnvironmentStr).str()), Arch(InvalidArch) {
   }
 
   /// @}
diff --git a/contrib/llvm/include/llvm/ADT/Twine.h b/contrib/llvm/include/llvm/ADT/Twine.h
index ab8d3653e33f..3a60cab77935 100644
--- a/contrib/llvm/include/llvm/ADT/Twine.h
+++ b/contrib/llvm/include/llvm/ADT/Twine.h
@@ -99,6 +99,9 @@ namespace llvm {
       /// A pointer to a StringRef instance.
       StringRefKind,
 
+      /// A char value reinterpreted as a pointer, to render as a character.
+      CharKind,
+
       /// An unsigned int value reinterpreted as a pointer, to render as an
       /// unsigned decimal integer.
       DecUIKind,
@@ -126,13 +129,31 @@ namespace llvm {
       UHexKind
     };
 
+    union Child
+    {
+      const Twine *twine;
+      const char *cString;
+      const std::string *stdString;
+      const StringRef *stringRef;
+      char character;
+      unsigned int decUI;
+      int decI;
+      const unsigned long *decUL;
+      const long *decL;
+      const unsigned long long *decULL;
+      const long long *decLL;
+      const uint64_t *uHex;
+    };
+
   private:
     /// LHS - The prefix in the concatenation, which may be uninitialized for
     /// Null or Empty kinds.
-    const void *LHS;
+    Child LHS;
     /// RHS - The suffix in the concatenation, which may be uninitialized for
     /// Null or Empty kinds.
-    const void *RHS;
+    Child RHS;
+    // enums stored as unsigned chars to save on space while some compilers
+    // don't support specifying the backing type for an enum
     /// LHSKind - The NodeKind of the left hand side, \see getLHSKind().
     unsigned char LHSKind;
     /// RHSKind - The NodeKind of the left hand side, \see getLHSKind().
@@ -147,13 +168,15 @@ namespace llvm {
 
     /// Construct a binary twine.
     explicit Twine(const Twine &_LHS, const Twine &_RHS)
-      : LHS(&_LHS), RHS(&_RHS), LHSKind(TwineKind), RHSKind(TwineKind) {
+      : LHSKind(TwineKind), RHSKind(TwineKind) {
+      LHS.twine = &_LHS;
+      RHS.twine = &_RHS;
       assert(isValid() && "Invalid twine!");
     }
 
     /// Construct a twine from explicit values.
-    explicit Twine(const void *_LHS, NodeKind _LHSKind,
-                   const void *_RHS, NodeKind _RHSKind)
+    explicit Twine(Child _LHS, NodeKind _LHSKind,
+                   Child _RHS, NodeKind _RHSKind)
       : LHS(_LHS), RHS(_RHS), LHSKind(_LHSKind), RHSKind(_RHSKind) {
       assert(isValid() && "Invalid twine!");
     }
@@ -200,10 +223,10 @@ namespace llvm {
 
       // A twine child should always be binary.
       if (getLHSKind() == TwineKind &&
-          !static_cast<const Twine*>(LHS)->isBinary())
+          !LHS.twine->isBinary())
         return false;
       if (getRHSKind() == TwineKind &&
-          !static_cast<const Twine*>(RHS)->isBinary())
+          !RHS.twine->isBinary())
         return false;
 
       return true;
@@ -216,10 +239,10 @@ namespace llvm {
     NodeKind getRHSKind() const { return (NodeKind) RHSKind; }
 
     /// printOneChild - Print one child from a twine.
-    void printOneChild(raw_ostream &OS, const void *Ptr, NodeKind Kind) const;
+    void printOneChild(raw_ostream &OS, Child Ptr, NodeKind Kind) const;
 
     /// printOneChildRepr - Print the representation of one child from a twine.
-    void printOneChildRepr(raw_ostream &OS, const void *Ptr,
+    void printOneChildRepr(raw_ostream &OS, Child Ptr,
                            NodeKind Kind) const;
 
   public:
@@ -239,7 +262,7 @@ namespace llvm {
     /*implicit*/ Twine(const char *Str)
       : RHSKind(EmptyKind) {
       if (Str[0] != '\0') {
-        LHS = Str;
+        LHS.cString = Str;
         LHSKind = CStringKind;
       } else
         LHSKind = EmptyKind;
@@ -249,44 +272,70 @@ namespace llvm {
 
     /// Construct from an std::string.
     /*implicit*/ Twine(const std::string &Str)
-      : LHS(&Str), LHSKind(StdStringKind), RHSKind(EmptyKind) {
+      : LHSKind(StdStringKind), RHSKind(EmptyKind) {
+      LHS.stdString = &Str;
       assert(isValid() && "Invalid twine!");
     }
 
     /// Construct from a StringRef.
     /*implicit*/ Twine(const StringRef &Str)
-      : LHS(&Str), LHSKind(StringRefKind), RHSKind(EmptyKind) {
+      : LHSKind(StringRefKind), RHSKind(EmptyKind) {
+      LHS.stringRef = &Str;
       assert(isValid() && "Invalid twine!");
     }
 
+    /// Construct from a char.
+    explicit Twine(char Val)
+      : LHSKind(CharKind), RHSKind(EmptyKind) {
+      LHS.character = Val;
+    }
+
+    /// Construct from a signed char.
+    explicit Twine(signed char Val)
+      : LHSKind(CharKind), RHSKind(EmptyKind) {
+      LHS.character = static_cast<char>(Val);
+    }
+
+    /// Construct from an unsigned char.
+    explicit Twine(unsigned char Val)
+      : LHSKind(CharKind), RHSKind(EmptyKind) {
+      LHS.character = static_cast<char>(Val);
+    }
+
     /// Construct a twine to print \arg Val as an unsigned decimal integer.
     explicit Twine(unsigned Val)
-      : LHS((void*)(intptr_t)Val), LHSKind(DecUIKind), RHSKind(EmptyKind) {
+      : LHSKind(DecUIKind), RHSKind(EmptyKind) {
+      LHS.decUI = Val;
     }
 
     /// Construct a twine to print \arg Val as a signed decimal integer.
     explicit Twine(int Val)
-      : LHS((void*)(intptr_t)Val), LHSKind(DecIKind), RHSKind(EmptyKind) {
+      : LHSKind(DecIKind), RHSKind(EmptyKind) {
+      LHS.decI = Val;
     }
 
     /// Construct a twine to print \arg Val as an unsigned decimal integer.
     explicit Twine(const unsigned long &Val)
-      : LHS(&Val), LHSKind(DecULKind), RHSKind(EmptyKind) {
+      : LHSKind(DecULKind), RHSKind(EmptyKind) {
+      LHS.decUL = &Val;
     }
 
     /// Construct a twine to print \arg Val as a signed decimal integer.
     explicit Twine(const long &Val)
-      : LHS(&Val), LHSKind(DecLKind), RHSKind(EmptyKind) {
+      : LHSKind(DecLKind), RHSKind(EmptyKind) {
+      LHS.decL = &Val;
     }
 
     /// Construct a twine to print \arg Val as an unsigned decimal integer.
     explicit Twine(const unsigned long long &Val)
-      : LHS(&Val), LHSKind(DecULLKind), RHSKind(EmptyKind) {
+      : LHSKind(DecULLKind), RHSKind(EmptyKind) {
+      LHS.decULL = &Val;
     }
 
     /// Construct a twine to print \arg Val as a signed decimal integer.
     explicit Twine(const long long &Val)
-      : LHS(&Val), LHSKind(DecLLKind), RHSKind(EmptyKind) {
+      : LHSKind(DecLLKind), RHSKind(EmptyKind) {
+      LHS.decLL = &Val;
     }
 
     // FIXME: Unfortunately, to make sure this is as efficient as possible we
@@ -296,13 +345,17 @@ namespace llvm {
 
     /// Construct as the concatenation of a C string and a StringRef.
     /*implicit*/ Twine(const char *_LHS, const StringRef &_RHS)
-      : LHS(_LHS), RHS(&_RHS), LHSKind(CStringKind), RHSKind(StringRefKind) {
+      : LHSKind(CStringKind), RHSKind(StringRefKind) {
+      LHS.cString = _LHS;
+      RHS.stringRef = &_RHS;
       assert(isValid() && "Invalid twine!");
     }
 
     /// Construct as the concatenation of a StringRef and a C string.
     /*implicit*/ Twine(const StringRef &_LHS, const char *_RHS)
-      : LHS(&_LHS), RHS(_RHS), LHSKind(StringRefKind), RHSKind(CStringKind) {
+      : LHSKind(StringRefKind), RHSKind(CStringKind) {
+      LHS.stringRef = &_LHS;
+      RHS.cString = _RHS;
       assert(isValid() && "Invalid twine!");
     }
 
@@ -318,7 +371,10 @@ namespace llvm {
 
     // Construct a twine to print \arg Val as an unsigned hexadecimal integer.
     static Twine utohexstr(const uint64_t &Val) {
-      return Twine(&Val, UHexKind, 0, EmptyKind);
+      Child LHS, RHS;
+      LHS.uHex = &Val;
+      RHS.twine = 0;
+      return Twine(LHS, UHexKind, RHS, EmptyKind);
     }
 
     /// @}
@@ -371,9 +427,9 @@ namespace llvm {
       switch (getLHSKind()) {
       default: assert(0 && "Out of sync with isSingleStringRef");
       case EmptyKind:      return StringRef();
-      case CStringKind:    return StringRef((const char*)LHS);
-      case StdStringKind:  return StringRef(*(const std::string*)LHS);
-      case StringRefKind:  return *(const StringRef*)LHS;
+      case CStringKind:    return StringRef(LHS.cString);
+      case StdStringKind:  return StringRef(*LHS.stdString);
+      case StringRefKind:  return *LHS.stringRef;
       }
     }
 
@@ -422,7 +478,9 @@ namespace llvm {
 
     // Otherwise we need to create a new node, taking care to fold in unary
     // twines.
-    const void *NewLHS = this, *NewRHS = &Suffix;
+    Child NewLHS, NewRHS;
+    NewLHS.twine = this;
+    NewRHS.twine = &Suffix;
     NodeKind NewLHSKind = TwineKind, NewRHSKind = TwineKind;
     if (isUnary()) {
       NewLHS = LHS;
diff --git a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h
index 5d8edd1bf4cd..d71ba208a129 100644
--- a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -88,7 +88,7 @@ class AliasAnalysis {
   /// getTypeStoreSize - Return the TargetData store size for the given type,
   /// if known, or a conservative value otherwise.
   ///
-  uint64_t getTypeStoreSize(const Type *Ty);
+  uint64_t getTypeStoreSize(Type *Ty);
 
   //===--------------------------------------------------------------------===//
   /// Alias Queries...
@@ -136,6 +136,8 @@ class AliasAnalysis {
   Location getLocation(const LoadInst *LI);
   Location getLocation(const StoreInst *SI);
   Location getLocation(const VAArgInst *VI);
+  Location getLocation(const AtomicCmpXchgInst *CXI);
+  Location getLocation(const AtomicRMWInst *RMWI);
   static Location getLocationForSource(const MemTransferInst *MTI);
   static Location getLocationForDest(const MemIntrinsic *MI);
 
@@ -341,6 +343,11 @@ class AliasAnalysis {
     case Instruction::VAArg:  return getModRefInfo((const VAArgInst*)I, Loc);
     case Instruction::Load:   return getModRefInfo((const LoadInst*)I,  Loc);
     case Instruction::Store:  return getModRefInfo((const StoreInst*)I, Loc);
+    case Instruction::Fence:  return getModRefInfo((const FenceInst*)I, Loc);
+    case Instruction::AtomicCmpXchg:
+      return getModRefInfo((const AtomicCmpXchgInst*)I, Loc);
+    case Instruction::AtomicRMW:
+      return getModRefInfo((const AtomicRMWInst*)I, Loc);
     case Instruction::Call:   return getModRefInfo((const CallInst*)I,  Loc);
     case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc);
     default:                  return NoModRef;
@@ -406,6 +413,39 @@ class AliasAnalysis {
     return getModRefInfo(S, Location(P, Size));
   }
 
+  /// getModRefInfo (for fences) - Return whether information about whether
+  /// a particular store modifies or reads the specified memory location.
+  ModRefResult getModRefInfo(const FenceInst *S, const Location &Loc) {
+    // Conservatively correct.  (We could possibly be a bit smarter if
+    // Loc is a alloca that doesn't escape.)
+    return ModRef;
+  }
+
+  /// getModRefInfo (for fences) - A convenience wrapper.
+  ModRefResult getModRefInfo(const FenceInst *S, const Value *P, uint64_t Size){
+    return getModRefInfo(S, Location(P, Size));
+  }
+
+  /// getModRefInfo (for cmpxchges) - Return whether information about whether
+  /// a particular cmpxchg modifies or reads the specified memory location.
+  ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc);
+
+  /// getModRefInfo (for cmpxchges) - A convenience wrapper.
+  ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX,
+                             const Value *P, unsigned Size) {
+    return getModRefInfo(CX, Location(P, Size));
+  }
+
+  /// getModRefInfo (for atomicrmws) - Return whether information about whether
+  /// a particular atomicrmw modifies or reads the specified memory location.
+  ModRefResult getModRefInfo(const AtomicRMWInst *RMW, const Location &Loc);
+
+  /// getModRefInfo (for atomicrmws) - A convenience wrapper.
+  ModRefResult getModRefInfo(const AtomicRMWInst *RMW,
+                             const Value *P, unsigned Size) {
+    return getModRefInfo(RMW, Location(P, Size));
+  }
+
   /// getModRefInfo (for va_args) - Return whether information about whether
   /// a particular va_arg modifies or reads the specified memory location.
   ModRefResult getModRefInfo(const VAArgInst* I, const Location &Loc);
diff --git a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h
index 03149c662e83..c4ebe4015bf8 100644
--- a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -111,8 +111,8 @@ class AliasSet : public ilist_node<AliasSet> {
   AliasSet *Forward;             // Forwarding pointer.
   AliasSet *Next, *Prev;         // Doubly linked list of AliasSets.
 
-  // All calls & invokes in this alias set.
-  std::vector<AssertingVH<Instruction> > CallSites;
+  // All instructions without a specific address in this alias set.
+  std::vector<AssertingVH<Instruction> > UnknownInsts;
 
   // RefCount - Number of nodes pointing to this AliasSet plus the number of
   // AliasSets forwarding to it.
@@ -147,9 +147,9 @@ class AliasSet : public ilist_node<AliasSet> {
       removeFromTracker(AST);
   }
 
-  CallSite getCallSite(unsigned i) const {
-    assert(i < CallSites.size());
-    return CallSite(CallSites[i]);
+  Instruction *getUnknownInst(unsigned i) const {
+    assert(i < UnknownInsts.size());
+    return UnknownInsts[i];
   }
   
 public:
@@ -253,12 +253,12 @@ class AliasSet : public ilist_node<AliasSet> {
   void addPointer(AliasSetTracker &AST, PointerRec &Entry, uint64_t Size,
                   const MDNode *TBAAInfo,
                   bool KnownMustAlias = false);
-  void addCallSite(CallSite CS, AliasAnalysis &AA);
-  void removeCallSite(CallSite CS) {
-    for (size_t i = 0, e = CallSites.size(); i != e; ++i)
-      if (CallSites[i] == CS.getInstruction()) {
-        CallSites[i] = CallSites.back();
-        CallSites.pop_back();
+  void addUnknownInst(Instruction *I, AliasAnalysis &AA);
+  void removeUnknownInst(Instruction *I) {
+    for (size_t i = 0, e = UnknownInsts.size(); i != e; ++i)
+      if (UnknownInsts[i] == I) {
+        UnknownInsts[i] = UnknownInsts.back();
+        UnknownInsts.pop_back();
         --i; --e;  // Revisit the moved entry.
       }
   }
@@ -269,7 +269,7 @@ class AliasSet : public ilist_node<AliasSet> {
   ///
   bool aliasesPointer(const Value *Ptr, uint64_t Size, const MDNode *TBAAInfo,
                       AliasAnalysis &AA) const;
-  bool aliasesCallSite(CallSite CS, AliasAnalysis &AA) const;
+  bool aliasesUnknownInst(Instruction *Inst, AliasAnalysis &AA) const;
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const AliasSet &AS) {
@@ -326,12 +326,10 @@ class AliasSetTracker {
   bool add(LoadInst *LI);
   bool add(StoreInst *SI);
   bool add(VAArgInst *VAAI);
-  bool add(CallSite CS);          // Call/Invoke instructions
-  bool add(CallInst *CI)   { return add(CallSite(CI)); }
-  bool add(InvokeInst *II) { return add(CallSite(II)); }
   bool add(Instruction *I);       // Dispatch to one of the other add methods...
   void add(BasicBlock &BB);       // Add all instructions in basic block
   void add(const AliasSetTracker &AST); // Add alias relations from another AST
+  bool addUnknown(Instruction *I);
 
   /// remove methods - These methods are used to remove all entries that might
   /// be aliased by the specified instruction.  These methods return true if any
@@ -341,11 +339,9 @@ class AliasSetTracker {
   bool remove(LoadInst *LI);
   bool remove(StoreInst *SI);
   bool remove(VAArgInst *VAAI);
-  bool remove(CallSite CS);
-  bool remove(CallInst *CI)   { return remove(CallSite(CI)); }
-  bool remove(InvokeInst *II) { return remove(CallSite(II)); }
   bool remove(Instruction *I);
   void remove(AliasSet &AS);
+  bool removeUnknown(Instruction *I);
   
   void clear();
 
@@ -429,7 +425,7 @@ class AliasSetTracker {
   AliasSet *findAliasSetForPointer(const Value *Ptr, uint64_t Size,
                                    const MDNode *TBAAInfo);
 
-  AliasSet *findAliasSetForCallSite(CallSite CS);
+  AliasSet *findAliasSetForUnknownInst(Instruction *Inst);
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const AliasSetTracker &AST) {
diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequencyImpl.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyImpl.h
index 6580fd1e4a9c..0fb2bd7db50e 100644
--- a/contrib/llvm/include/llvm/Analysis/BlockFrequencyImpl.h
+++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyImpl.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/Support/BlockFrequency.h"
 #include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -29,8 +30,8 @@
 namespace llvm {
 
 
-class BlockFrequency;
-class MachineBlockFrequency;
+class BlockFrequencyInfo;
+class MachineBlockFrequencyInfo;
 
 /// BlockFrequencyImpl implements block frequency algorithm for IR and
 /// Machine Instructions. Algorithm starts with value 1024 (START_FREQ)
@@ -40,7 +41,7 @@ class MachineBlockFrequency;
 template<class BlockT, class FunctionT, class BlockProbInfoT>
 class BlockFrequencyImpl {
 
-  DenseMap<BlockT *, uint32_t> Freqs;
+  DenseMap<BlockT *, BlockFrequency> Freqs;
 
   BlockProbInfoT *BPI;
 
@@ -48,7 +49,7 @@ class BlockFrequencyImpl {
 
   typedef GraphTraits< Inverse<BlockT *> > GT;
 
-  static const uint32_t START_FREQ = 1024;
+  const uint32_t EntryFreq;
 
   std::string getBlockName(BasicBlock *BB) const {
     return BB->getNameStr();
@@ -64,26 +65,21 @@ class BlockFrequencyImpl {
     return ss.str();
   }
 
-  void setBlockFreq(BlockT *BB, uint32_t Freq) {
+  void setBlockFreq(BlockT *BB, BlockFrequency Freq) {
     Freqs[BB] = Freq;
     DEBUG(dbgs() << "Frequency(" << getBlockName(BB) << ") = " << Freq << "\n");
   }
 
   /// getEdgeFreq - Return edge frequency based on SRC frequency and Src -> Dst
   /// edge probability.
-  uint32_t getEdgeFreq(BlockT *Src, BlockT *Dst) const {
+  BlockFrequency getEdgeFreq(BlockT *Src, BlockT *Dst) const {
     BranchProbability Prob = BPI->getEdgeProbability(Src, Dst);
-    uint64_t N = Prob.getNumerator();
-    uint64_t D = Prob.getDenominator();
-    uint64_t Res = (N * getBlockFreq(Src)) / D;
-
-    assert(Res <= UINT32_MAX);
-    return (uint32_t) Res;
+    return getBlockFreq(Src) * Prob;
   }
 
   /// incBlockFreq - Increase BB block frequency by FREQ.
   ///
-  void incBlockFreq(BlockT *BB, uint32_t Freq) {
+  void incBlockFreq(BlockT *BB, BlockFrequency Freq) {
     Freqs[BB] += Freq;
     DEBUG(dbgs() << "Frequency(" << getBlockName(BB) << ") += " << Freq
                  << " --> " << Freqs[BB] << "\n");
@@ -95,13 +91,13 @@ class BlockFrequencyImpl {
     uint64_t N = Prob.getNumerator();
     assert(N && "Illegal division by zero!");
     uint64_t D = Prob.getDenominator();
-    uint64_t Freq = (Freqs[BB] * D) / N;
+    uint64_t Freq = (Freqs[BB].getFrequency() * D) / N;
 
     // Should we assert it?
     if (Freq > UINT32_MAX)
       Freq = UINT32_MAX;
 
-    Freqs[BB] = (uint32_t) Freq;
+    Freqs[BB] = BlockFrequency(Freq);
     DEBUG(dbgs() << "Frequency(" << getBlockName(BB) << ") /= (" << Prob
                  << ") --> " << Freqs[BB] << "\n");
   }
@@ -136,15 +132,6 @@ class BlockFrequencyImpl {
   }
 
 
-  /// Return a probability of getting to the DST block through SRC->DST edge.
-  ///
-  BranchProbability getBackEdgeProbability(BlockT *Src, BlockT *Dst) const {
-    uint32_t N = getEdgeFreq(Src, Dst);
-    uint32_t D = getBlockFreq(Dst);
-
-    return BranchProbability(N, D);
-  }
-
   /// isReachable - Returns if BB block is reachable from the entry.
   ///
   bool isReachable(BlockT *BB) {
@@ -160,7 +147,7 @@ class BlockFrequencyImpl {
     unsigned a = RPO[Src];
     unsigned b = RPO[Dst];
 
-    return a > b;
+    return a >= b;
   }
 
   /// getSingleBlockPred - return single BB block predecessor or NULL if
@@ -189,7 +176,7 @@ class BlockFrequencyImpl {
     setBlockFreq(BB, 0);
 
     if (BB == LoopHead) {
-      setBlockFreq(BB, START_FREQ);
+      setBlockFreq(BB, EntryFreq);
       return;
     }
 
@@ -224,10 +211,10 @@ class BlockFrequencyImpl {
     if (!isLoopHead)
       return;
 
-    assert(START_FREQ >= CycleProb[BB]);
+    assert(EntryFreq >= CycleProb[BB]);
     uint32_t CProb = CycleProb[BB];
-    uint32_t Numerator = START_FREQ - CProb ? START_FREQ - CProb : 1;
-    divBlockFreq(BB, BranchProbability(Numerator, START_FREQ));
+    uint32_t Numerator = EntryFreq - CProb ? EntryFreq - CProb : 1;
+    divBlockFreq(BB, BranchProbability(Numerator, EntryFreq));
   }
 
   /// doLoop - Propagate block frequency down throught the loop.
@@ -237,11 +224,13 @@ class BlockFrequencyImpl {
 
     SmallPtrSet<BlockT *, 8> BlocksInLoop;
 
-    for (rpot_iterator I = rpot_at(Head), E = rpot_end(); I != E; ++I) {
+    for (rpot_iterator I = rpot_at(Head), E = rpot_at(Tail); ; ++I) {
       BlockT *BB = *I;
       doBlock(BB, Head, BlocksInLoop);
 
       BlocksInLoop.insert(BB);
+      if (I == E)
+        break;
     }
 
     // Compute loop's cyclic probability using backedges probabilities.
@@ -252,19 +241,23 @@ class BlockFrequencyImpl {
       BlockT *Pred = *PI;
       assert(Pred);
       if (isReachable(Pred) && isBackedge(Pred, Head)) {
-        BranchProbability Prob = getBackEdgeProbability(Pred, Head);
-        uint64_t N = Prob.getNumerator();
-        uint64_t D = Prob.getDenominator();
-        uint64_t Res = (N * START_FREQ) / D;
+        uint64_t N = getEdgeFreq(Pred, Head).getFrequency();
+        uint64_t D = getBlockFreq(Head).getFrequency();
+        assert(N <= EntryFreq && "Backedge frequency must be <= EntryFreq!");
+        uint64_t Res = (N * EntryFreq) / D;
 
         assert(Res <= UINT32_MAX);
         CycleProb[Head] += (uint32_t) Res;
+        DEBUG(dbgs() << "  CycleProb[" << getBlockName(Head) << "] += " << Res
+                     << " --> " << CycleProb[Head] << "\n");
       }
     }
   }
 
-  friend class BlockFrequency;
-  friend class MachineBlockFrequency;
+  friend class BlockFrequencyInfo;
+  friend class MachineBlockFrequencyInfo;
+
+  BlockFrequencyImpl() : EntryFreq(BlockFrequency::getEntryFrequency()) { }
 
   void doFunction(FunctionT *fn, BlockProbInfoT *bpi) {
     Fn = fn;
@@ -314,13 +307,12 @@ class BlockFrequencyImpl {
   }
 
 public:
-  /// getBlockFreq - Return block frequency. Never return 0, value must be
-  /// positive.
-  uint32_t getBlockFreq(BlockT *BB) const {
-    typename DenseMap<BlockT *, uint32_t>::const_iterator I = Freqs.find(BB);
+  /// getBlockFreq - Return block frequency. Return 0 if we don't have it.
+  BlockFrequency getBlockFreq(BlockT *BB) const {
+    typename DenseMap<BlockT *, BlockFrequency>::const_iterator I = Freqs.find(BB);
     if (I != Freqs.end())
-      return I->second ? I->second : 1;
-    return 1;
+      return I->second;
+    return 0;
   }
 
   void print(raw_ostream &OS) const {
diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequency.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
similarity index 58%
rename from contrib/llvm/include/llvm/Analysis/BlockFrequency.h
rename to contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
index c4b1e087a0b1..9e698a9f4bb1 100644
--- a/contrib/llvm/include/llvm/Analysis/BlockFrequency.h
+++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -1,4 +1,4 @@
-//========-------- BlockFrequency.h - Block Frequency Analysis -------========//
+//========-------- BlockFrequencyInfo.h - Block Frequency Analysis -------========//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -11,10 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_ANALYSIS_BLOCKFREQUENCY_H
-#define LLVM_ANALYSIS_BLOCKFREQUENCY_H
+#ifndef LLVM_ANALYSIS_BLOCKFREQUENCYINFO_H
+#define LLVM_ANALYSIS_BLOCKFREQUENCYINFO_H
 
 #include "llvm/Pass.h"
+#include "llvm/Support/BlockFrequency.h"
 #include <climits>
 
 namespace llvm {
@@ -23,29 +24,30 @@ class BranchProbabilityInfo;
 template<class BlockT, class FunctionT, class BranchProbInfoT>
 class BlockFrequencyImpl;
 
-/// BlockFrequency pass uses BlockFrequencyImpl implementation to estimate
+/// BlockFrequencyInfo pass uses BlockFrequencyImpl implementation to estimate
 /// IR basic block frequencies.
-class BlockFrequency : public FunctionPass {
+class BlockFrequencyInfo : public FunctionPass {
 
   BlockFrequencyImpl<BasicBlock, Function, BranchProbabilityInfo> *BFI;
 
 public:
   static char ID;
 
-  BlockFrequency();
+  BlockFrequencyInfo();
 
-  ~BlockFrequency();
+  ~BlockFrequencyInfo();
 
   void getAnalysisUsage(AnalysisUsage &AU) const;
 
   bool runOnFunction(Function &F);
+  void print(raw_ostream &O, const Module *M) const;
 
-  /// getblockFreq - Return block frequency. Never return 0, value must be
-  /// positive. Please note that initial frequency is equal to 1024. It means
+  /// getblockFreq - Return block frequency. Return 0 if we don't have the
+  /// information. Please note that initial frequency is equal to 1024. It means
   /// that we should not rely on the value itself, but only on the comparison to
-  /// the other block frequencies. We do this to avoid using of the floating
-  /// points.
-  uint32_t getBlockFreq(BasicBlock *BB);
+  /// the other block frequencies. We do this to avoid using of floating points.
+  ///
+  BlockFrequency getBlockFreq(BasicBlock *BB) const;
 };
 
 }
diff --git a/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index 02ead98321ff..a2c12ab9e824 100644
--- a/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -33,12 +33,12 @@ class BranchProbabilityInfo : public FunctionPass {
   // weight to just "inherit" the non-zero weight of an adjacent successor.
   static const uint32_t DEFAULT_WEIGHT = 16;
 
-  typedef std::pair<BasicBlock *, BasicBlock *> Edge;
+  typedef std::pair<const BasicBlock *, const BasicBlock *> Edge;
 
   DenseMap<Edge, uint32_t> Weights;
 
   // Get sum of the block successors' weights.
-  uint32_t getSumForBlock(BasicBlock *BB) const;
+  uint32_t getSumForBlock(const BasicBlock *BB) const;
 
 public:
   static char ID;
@@ -53,13 +53,14 @@ class BranchProbabilityInfo : public FunctionPass {
 
   // Returned value is between 1 and UINT32_MAX. Look at
   // BranchProbabilityInfo.cpp for details.
-  uint32_t getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const;
+  uint32_t getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const;
 
   // Look at BranchProbabilityInfo.cpp for details. Use it with caution!
-  void setEdgeWeight(BasicBlock *Src, BasicBlock *Dst, uint32_t Weight);
+  void setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst,
+                     uint32_t Weight);
 
   // A 'Hot' edge is an edge which probability is >= 80%.
-  bool isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const;
+  bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const;
 
   // Return a hot successor for the block BB or null if there isn't one.
   BasicBlock *getHotSucc(BasicBlock *BB) const;
@@ -67,7 +68,8 @@ class BranchProbabilityInfo : public FunctionPass {
   // Return a probability as a fraction between 0 (0% probability) and
   // 1 (100% probability), however the value is never equal to 0, and can be 1
   // only iff SRC block has only one successor.
-  BranchProbability getEdgeProbability(BasicBlock *Src, BasicBlock *Dst) const;
+  BranchProbability getEdgeProbability(const BasicBlock *Src,
+                                       const BasicBlock *Dst) const;
 
   // Print value between 0 (0% probability) and 1 (100% probability),
   // however the value is never equal to 0, and can be 1 only iff SRC block
diff --git a/contrib/llvm/include/llvm/Analysis/CodeMetrics.h b/contrib/llvm/include/llvm/Analysis/CodeMetrics.h
index 75edfbbed2e3..d96dd82b3591 100644
--- a/contrib/llvm/include/llvm/Analysis/CodeMetrics.h
+++ b/contrib/llvm/include/llvm/Analysis/CodeMetrics.h
@@ -18,6 +18,9 @@
 #include "llvm/ADT/DenseMap.h"
 
 namespace llvm {
+
+  class TargetData;
+
   // CodeMetrics - Calculate size and a few similar metrics for a set of
   // basic blocks.
   struct CodeMetrics {
@@ -46,7 +49,7 @@ namespace llvm {
 
     /// NumCalls - Keep track of the number of calls to 'big' functions.
     unsigned NumCalls;
-    
+
     /// NumInlineCandidates - Keep track of the number of calls to internal
     /// functions with only a single caller.  These are likely targets for
     /// future inlining, likely exposed by interleaved devirtualization.
@@ -61,24 +64,24 @@ namespace llvm {
     unsigned NumRets;
 
     CodeMetrics() : callsSetJmp(false), isRecursive(false),
-                    containsIndirectBr(false), usesDynamicAlloca(false), 
+                    containsIndirectBr(false), usesDynamicAlloca(false),
                     NumInsts(0), NumBlocks(0), NumCalls(0),
-                    NumInlineCandidates(0), NumVectorInsts(0), 
+                    NumInlineCandidates(0), NumVectorInsts(0),
                     NumRets(0) {}
 
     /// analyzeBasicBlock - Add information about the specified basic block
     /// to the current structure.
-    void analyzeBasicBlock(const BasicBlock *BB);
+    void analyzeBasicBlock(const BasicBlock *BB, const TargetData *TD = 0);
 
     /// analyzeFunction - Add information about the specified function
     /// to the current structure.
-    void analyzeFunction(Function *F);
-    
+    void analyzeFunction(Function *F, const TargetData *TD = 0);
+
     /// CountCodeReductionForConstant - Figure out an approximation for how
     /// many instructions will be constant folded if the specified value is
     /// constant.
     unsigned CountCodeReductionForConstant(Value *V);
-   
+
     /// CountBonusForConstant - Figure out an approximation for how much
     /// per-call performance boost we can expect if the specified value is
     /// constant.
diff --git a/contrib/llvm/include/llvm/Analysis/ConstantFolding.h b/contrib/llvm/include/llvm/Analysis/ConstantFolding.h
index f6b1f5ab9915..05018fa1617a 100644
--- a/contrib/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/contrib/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -27,6 +27,8 @@ namespace llvm {
   class TargetData;
   class Function;
   class Type;
+  template<typename T>
+  class ArrayRef;
 
 /// ConstantFoldInstruction - Try to constant fold the specified instruction.
 /// If successful, the constant result is returned, if not, null is returned.
@@ -47,8 +49,8 @@ Constant *ConstantFoldConstantExpression(const ConstantExpr *CE,
 /// fold instructions like loads and stores, which have no constant expression 
 /// form.
 ///
-Constant *ConstantFoldInstOperands(unsigned Opcode, const Type *DestTy,
-                                   Constant *const *Ops, unsigned NumOps,
+Constant *ConstantFoldInstOperands(unsigned Opcode, Type *DestTy,
+                                   ArrayRef<Constant *> Ops,
                                    const TargetData *TD = 0);
 
 /// ConstantFoldCompareInstOperands - Attempt to constant fold a compare
@@ -59,6 +61,12 @@ Constant *ConstantFoldCompareInstOperands(unsigned Predicate,
                                           Constant *LHS, Constant *RHS,
                                           const TargetData *TD = 0);
 
+/// ConstantFoldInsertValueInstruction - Attempt to constant fold an insertvalue
+/// instruction with the specified operands and indices.  The constant result is
+/// returned if successful; if not, null is returned.
+Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
+                                             ArrayRef<unsigned> Idxs);
+
 /// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
 /// produce if it is constant and determinable.  If this is not determinable,
 /// return null.
@@ -76,7 +84,7 @@ bool canConstantFoldCallTo(const Function *F);
 /// ConstantFoldCall - Attempt to constant fold a call to the specified function
 /// with the specified arguments, returning null if unsuccessful.
 Constant *
-ConstantFoldCall(Function *F, Constant *const *Operands, unsigned NumOperands);
+ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands);
 }
 
 #endif
diff --git a/contrib/llvm/include/llvm/Analysis/DIBuilder.h b/contrib/llvm/include/llvm/Analysis/DIBuilder.h
index a706cc8f35db..ee24226c748f 100644
--- a/contrib/llvm/include/llvm/Analysis/DIBuilder.h
+++ b/contrib/llvm/include/llvm/Analysis/DIBuilder.h
@@ -37,6 +37,7 @@ namespace llvm {
   class DINameSpace;
   class DIVariable;
   class DISubrange;
+  class DILexicalBlockFile;
   class DILexicalBlock;
   class DISubprogram;
   class DITemplateTypeParameter;
@@ -48,9 +49,19 @@ namespace llvm {
     LLVMContext & VMContext;
     MDNode *TheCU;
 
+    MDNode *TempEnumTypes;
+    MDNode *TempRetainTypes;
+    MDNode *TempSubprograms;
+    MDNode *TempGVs;
+
     Function *DeclareFn;     // llvm.dbg.declare
     Function *ValueFn;       // llvm.dbg.value
 
+    SmallVector<Value *, 4> AllEnumTypes;
+    SmallVector<Value *, 4> AllRetainTypes;
+    SmallVector<Value *, 4> AllSubprograms;
+    SmallVector<Value *, 4> AllGVs;
+
     DIBuilder(const DIBuilder &);       // DO NOT IMPLEMENT
     void operator=(const DIBuilder &);  // DO NOT IMPLEMENT
 
@@ -59,6 +70,9 @@ namespace llvm {
     const MDNode *getCU() { return TheCU; }
     enum ComplexAddrKind { OpPlus=1, OpDeref };
 
+    /// finalize - Construct any deferred debug info descriptors.
+    void finalize();
+
     /// createCompileUnit - A CompileUnit provides an anchor for all debugging
     /// information generated during this instance of compilation.
     /// @param Lang     Source programming language, eg. dwarf::DW_LANG_C99
@@ -84,6 +98,9 @@ namespace llvm {
     /// createEnumerator - Create a single enumerator value.
     DIEnumerator createEnumerator(StringRef Name, uint64_t Val);
 
+    /// createNullPtrType - Create C++0x nullptr type.
+    DIType createNullPtrType(StringRef Name);
+
     /// createBasicType - Create debugging information entry for a basic 
     /// type.
     /// @param Name        Type name.
@@ -447,6 +464,14 @@ namespace llvm {
                                 DIFile File, unsigned LineNo);
 
 
+    /// createLexicalBlockFile - This creates a descriptor for a lexical
+    /// block with a new file attached. This merely extends the existing
+    /// lexical block as it crosses a file.
+    /// @param Scope       Lexical block.
+    /// @param File        Source file.
+    DILexicalBlockFile createLexicalBlockFile(DIDescriptor Scope,
+					      DIFile File);
+    
     /// createLexicalBlock - This creates a descriptor for a lexical block
     /// with the specified parent context.
     /// @param Scope       Parent lexical scope.
diff --git a/contrib/llvm/include/llvm/Analysis/DebugInfo.h b/contrib/llvm/include/llvm/Analysis/DebugInfo.h
index fbee5a6311e3..9a53c4dadba0 100644
--- a/contrib/llvm/include/llvm/Analysis/DebugInfo.h
+++ b/contrib/llvm/include/llvm/Analysis/DebugInfo.h
@@ -40,6 +40,7 @@ namespace llvm {
   class DIFile;
   class DISubprogram;
   class DILexicalBlock;
+  class DILexicalBlockFile;
   class DIVariable;
   class DIType;
 
@@ -84,6 +85,7 @@ namespace llvm {
     explicit DIDescriptor(const MDNode *N) : DbgNode(N) {}
     explicit DIDescriptor(const DIFile F);
     explicit DIDescriptor(const DISubprogram F);
+    explicit DIDescriptor(const DILexicalBlockFile F);
     explicit DIDescriptor(const DILexicalBlock F);
     explicit DIDescriptor(const DIVariable F);
     explicit DIDescriptor(const DIType F);
@@ -117,6 +119,7 @@ namespace llvm {
     bool isFile() const;
     bool isCompileUnit() const;
     bool isNameSpace() const;
+    bool isLexicalBlockFile() const;
     bool isLexicalBlock() const;
     bool isSubrange() const;
     bool isEnumerator() const;
@@ -182,6 +185,11 @@ namespace llvm {
     StringRef getFlags() const       { return getStringField(8);   }
     unsigned getRunTimeVersion() const { return getUnsignedField(9); }
 
+    DIArray getEnumTypes() const;
+    DIArray getRetainedTypes() const;
+    DIArray getSubprograms() const;
+    DIArray getGlobalVariables() const;
+
     /// Verify - Verify that a compile unit is well formed.
     bool Verify() const;
 
@@ -201,7 +209,10 @@ namespace llvm {
     }
     StringRef getFilename() const  { return getStringField(1);   }
     StringRef getDirectory() const { return getStringField(2);   }
-    DICompileUnit getCompileUnit() const{ return getFieldAs<DICompileUnit>(3); }
+    DICompileUnit getCompileUnit() const{ 
+      assert (getVersion() <= LLVMDebugVersion10  && "Invalid CompileUnit!");
+      return getFieldAs<DICompileUnit>(3); 
+    }
   };
 
   /// DIEnumerator - A wrapper for an enumerator (e.g. X and Y in 'enum {X,Y}').
@@ -237,6 +248,7 @@ namespace llvm {
     DIScope getContext() const          { return getFieldAs<DIScope>(1); }
     StringRef getName() const           { return getStringField(2);     }
     DICompileUnit getCompileUnit() const{ 
+      assert (getVersion() <= LLVMDebugVersion10 && "Invalid getCompileUnit!");
      if (getVersion() == llvm::LLVMDebugVersion7)
        return getFieldAs<DICompileUnit>(3);
      
@@ -291,6 +303,9 @@ namespace llvm {
       return getFieldAs<DIFile>(3).getFilename();
     }
 
+    /// isUnsignedDIType - Return true if type encoding is unsigned.
+    bool isUnsignedDIType();
+
     /// replaceAllUsesWith - Replace all uses of debug info referenced by
     /// this descriptor.
     void replaceAllUsesWith(DIDescriptor &D);
@@ -447,6 +462,7 @@ namespace llvm {
     StringRef getDisplayName() const  { return getStringField(4); }
     StringRef getLinkageName() const  { return getStringField(5); }
     DICompileUnit getCompileUnit() const{ 
+      assert (getVersion() <= LLVMDebugVersion10 && "Invalid getCompileUnit!");
       if (getVersion() == llvm::LLVMDebugVersion7)
         return getFieldAs<DICompileUnit>(6);
 
@@ -545,6 +561,8 @@ namespace llvm {
     DISubprogram getFunctionDeclaration() const {
       return getFieldAs<DISubprogram>(18);
     }
+    MDNode *getVariablesNodes() const;
+    DIArray getVariables() const;
   };
 
   /// DIGlobalVariable - This is a wrapper for a global variable.
@@ -557,12 +575,24 @@ namespace llvm {
     StringRef getDisplayName() const  { return getStringField(4); }
     StringRef getLinkageName() const  { return getStringField(5); }
     DICompileUnit getCompileUnit() const{ 
+      assert (getVersion() <= LLVMDebugVersion10 && "Invalid getCompileUnit!");
       if (getVersion() == llvm::LLVMDebugVersion7)
         return getFieldAs<DICompileUnit>(6);
 
       DIFile F = getFieldAs<DIFile>(6); 
       return F.getCompileUnit();
     }
+    StringRef getFilename() const {
+      if (getVersion() <= llvm::LLVMDebugVersion10)
+        return getContext().getFilename();
+      return getFieldAs<DIFile>(6).getFilename();
+    } 
+    StringRef getDirectory() const {
+      if (getVersion() <= llvm::LLVMDebugVersion10)
+        return getContext().getDirectory();
+      return getFieldAs<DIFile>(6).getDirectory();
+
+    } 
 
     unsigned getLineNumber() const      { return getUnsignedField(7); }
     DIType getType() const              { return getFieldAs<DIType>(8); }
@@ -592,6 +622,7 @@ namespace llvm {
     DIScope getContext() const          { return getFieldAs<DIScope>(1); }
     StringRef getName() const           { return getStringField(2);     }
     DICompileUnit getCompileUnit() const{ 
+      assert (getVersion() <= LLVMDebugVersion10 && "Invalid getCompileUnit!");
       if (getVersion() == llvm::LLVMDebugVersion7)
         return getFieldAs<DICompileUnit>(3);
 
@@ -614,6 +645,8 @@ namespace llvm {
       return (getUnsignedField(6) & FlagArtificial) != 0;
     }
 
+    /// getInlinedAt - If this variable is inlined then return inline location.
+    MDNode *getInlinedAt() const;
 
     /// Verify - Verify that a variable descriptor is well formed.
     bool Verify() const;
@@ -628,7 +661,9 @@ namespace llvm {
     uint64_t getAddrElement(unsigned Idx) const {
       if (getVersion() <= llvm::LLVMDebugVersion8)
         return getUInt64Field(Idx+6);
-      return getUInt64Field(Idx+7);
+      if (getVersion() == llvm::LLVMDebugVersion9)
+        return getUInt64Field(Idx+7);
+      return getUInt64Field(Idx+8);
     }
 
     /// isBlockByrefVariable - Return true if the variable was declared as
@@ -644,6 +679,8 @@ namespace llvm {
     /// print - print variable.
     void print(raw_ostream &OS) const;
 
+    void printExtendedName(raw_ostream &OS) const;
+
     /// dump - print variable to dbgs() with a newline.
     void dump() const;
   };
@@ -665,6 +702,26 @@ namespace llvm {
     }
   };
 
+  /// DILexicalBlockFile - This is a wrapper for a lexical block with
+  /// a filename change.
+  class DILexicalBlockFile : public DIScope {
+  public:
+    explicit DILexicalBlockFile(const MDNode *N = 0) : DIScope(N) {}
+    DIScope getContext() const { return getScope().getContext(); }
+    unsigned getLineNumber() const { return getScope().getLineNumber(); }
+    unsigned getColumnNumber() const { return getScope().getColumnNumber(); }
+    StringRef getDirectory() const {
+      StringRef dir = getFieldAs<DIFile>(2).getDirectory();
+      return !dir.empty() ? dir : getContext().getDirectory();
+    }
+    StringRef getFilename() const {
+      StringRef filename = getFieldAs<DIFile>(2).getFilename();
+      assert(!filename.empty() && "Why'd you create this then?");
+      return filename;
+    }
+    DILexicalBlock getScope() const { return getFieldAs<DILexicalBlock>(1); }
+  };
+
   /// DINameSpace - A wrapper for a C++ style name space.
   class DINameSpace : public DIScope { 
   public:
@@ -678,6 +735,7 @@ namespace llvm {
       return getFieldAs<DIFile>(3).getFilename();
     }
     DICompileUnit getCompileUnit() const{ 
+      assert (getVersion() <= LLVMDebugVersion10 && "Invalid getCompileUnit!");
       if (getVersion() == llvm::LLVMDebugVersion7)
         return getFieldAs<DICompileUnit>(3);
 
@@ -708,13 +766,27 @@ namespace llvm {
   /// getDICompositeType - Find underlying composite type.
   DICompositeType getDICompositeType(DIType T);
 
+  /// isSubprogramContext - Return true if Context is either a subprogram
+  /// or another context nested inside a subprogram.
+  bool isSubprogramContext(const MDNode *Context);
+
   /// getOrInsertFnSpecificMDNode - Return a NameMDNode that is suitable
   /// to hold function specific information.
-  NamedMDNode *getOrInsertFnSpecificMDNode(Module &M, StringRef Name);
+  NamedMDNode *getOrInsertFnSpecificMDNode(Module &M, DISubprogram SP);
 
   /// getFnSpecificMDNode - Return a NameMDNode, if available, that is 
   /// suitable to hold function specific information.
-  NamedMDNode *getFnSpecificMDNode(const Module &M, StringRef Name);
+  NamedMDNode *getFnSpecificMDNode(const Module &M, DISubprogram SP);
+
+  /// createInlinedVariable - Create a new inlined variable based on current
+  /// variable.
+  /// @param DV            Current Variable.
+  /// @param InlinedScope  Location at current variable is inlined.
+  DIVariable createInlinedVariable(MDNode *DV, MDNode *InlinedScope,
+                                   LLVMContext &VMContext);
+
+  /// cleanseInlinedVariable - Remove inlined scope from the variable.
+  DIVariable cleanseInlinedVariable(MDNode *DV, LLVMContext &VMContext);
 
   class DebugInfoFinder {
   public:
diff --git a/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h b/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h
index 3e5da572b284..b22cb8813513 100644
--- a/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h
+++ b/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h
@@ -23,7 +23,7 @@ class Type;
 class Value;
 
 class FindUsedTypes : public ModulePass {
-  SetVector<const Type *> UsedTypes;
+  SetVector<Type *> UsedTypes;
 public:
   static char ID; // Pass identification, replacement for typeid
   FindUsedTypes() : ModulePass(ID) {
@@ -33,7 +33,7 @@ class FindUsedTypes : public ModulePass {
   /// getTypes - After the pass has been run, return the set containing all of
   /// the types used in the module.
   ///
-  const SetVector<const Type *> &getTypes() const { return UsedTypes; }
+  const SetVector<Type *> &getTypes() const { return UsedTypes; }
 
   /// Print the types found in the module.  If the optional Module parameter is
   /// passed in, then the types are printed symbolically if possible, using the
@@ -45,7 +45,7 @@ class FindUsedTypes : public ModulePass {
   /// IncorporateType - Incorporate one type and all of its subtypes into the
   /// collection of used types.
   ///
-  void IncorporateType(const Type *Ty);
+  void IncorporateType(Type *Ty);
 
   /// IncorporateValue - Incorporate all of the types used by this value.
   ///
diff --git a/contrib/llvm/include/llvm/Analysis/IVUsers.h b/contrib/llvm/include/llvm/Analysis/IVUsers.h
index e56d24d583df..2fb607cc5c37 100644
--- a/contrib/llvm/include/llvm/Analysis/IVUsers.h
+++ b/contrib/llvm/include/llvm/Analysis/IVUsers.h
@@ -140,6 +140,8 @@ class IVUsers : public LoopPass {
   static char ID; // Pass ID, replacement for typeid
   IVUsers();
 
+  Loop *getLoop() const { return L; }
+
   /// AddUsersIfInteresting - Inspect the specified Instruction.  If it is a
   /// reducible SCEV, recursively add its users to the IVUsesByStride set and
   /// return true.  Otherwise, return false.
diff --git a/contrib/llvm/include/llvm/Analysis/InlineCost.h b/contrib/llvm/include/llvm/Analysis/InlineCost.h
index a0cce515e9e2..36a16e68df07 100644
--- a/contrib/llvm/include/llvm/Analysis/InlineCost.h
+++ b/contrib/llvm/include/llvm/Analysis/InlineCost.h
@@ -29,6 +29,7 @@ namespace llvm {
   class CallSite;
   template<class PtrType, unsigned SmallSize>
   class SmallPtrSet;
+  class TargetData;
 
   namespace InlineConstants {
     // Various magic constants used to adjust heuristics.
@@ -113,7 +114,7 @@ namespace llvm {
 
       /// analyzeFunction - Add information about the specified function
       /// to the current structure.
-      void analyzeFunction(Function *F);
+      void analyzeFunction(Function *F, const TargetData *TD);
 
       /// NeverInline - Returns true if the function should never be
       /// inlined into any caller.
@@ -124,11 +125,17 @@ namespace llvm {
     // the ValueMap will update itself when this happens.
     ValueMap<const Function *, FunctionInfo> CachedFunctionInfo;
 
+    // TargetData if available, or null.
+    const TargetData *TD;
+
     int CountBonusForConstant(Value *V, Constant *C = NULL);
     int ConstantFunctionBonus(CallSite CS, Constant *C);
     int getInlineSize(CallSite CS, Function *Callee);
     int getInlineBonuses(CallSite CS, Function *Callee);
   public:
+    InlineCostAnalyzer(): TD(0) {}
+
+    void setTargetData(const TargetData *TData) { TD = TData; }
 
     /// getInlineCost - The heuristic used to determine if we should inline the
     /// function call or not.
diff --git a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h
index bc6e55f5490a..c1d87d3f7712 100644
--- a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -24,6 +24,8 @@ namespace llvm {
   class Instruction;
   class Value;
   class TargetData;
+  template<typename T>
+  class ArrayRef;
 
   /// SimplifyAddInst - Given operands for an Add, see if we can
   /// fold the result.  If not, this returns null.
@@ -121,9 +123,16 @@ namespace llvm {
 
   /// SimplifyGEPInst - Given operands for an GetElementPtrInst, see if we can
   /// fold the result.  If not, this returns null.
-  Value *SimplifyGEPInst(Value * const *Ops, unsigned NumOps,
+  Value *SimplifyGEPInst(ArrayRef<Value *> Ops,
                          const TargetData *TD = 0, const DominatorTree *DT = 0);
 
+  /// SimplifyInsertValueInst - Given operands for an InsertValueInst, see if we
+  /// can fold the result.  If not, this returns null.
+  Value *SimplifyInsertValueInst(Value *Agg, Value *Val,
+                                 ArrayRef<unsigned> Idxs,
+                                 const TargetData *TD = 0,
+                                 const DominatorTree *DT = 0);
+
   //=== Helper functions for higher up the class hierarchy.
 
 
diff --git a/contrib/llvm/include/llvm/Analysis/LoopInfo.h b/contrib/llvm/include/llvm/Analysis/LoopInfo.h
index 392bdad5ab02..12cb6c5cc480 100644
--- a/contrib/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/contrib/llvm/include/llvm/Analysis/LoopInfo.h
@@ -33,6 +33,7 @@
 
 #include "llvm/Pass.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/SmallVector.h"
@@ -105,7 +106,7 @@ class LoopBase {
     if (L == 0)    return false;
     return contains(L->getParentLoop());
   }
-    
+
   /// contains - Return true if the specified basic block is in this loop.
   ///
   bool contains(const BlockT *BB) const {
@@ -134,6 +135,11 @@ class LoopBase {
   block_iterator block_begin() const { return Blocks.begin(); }
   block_iterator block_end() const { return Blocks.end(); }
 
+  /// getNumBlocks - Get the number of blocks in this loop in constant time.
+  unsigned getNumBlocks() const {
+    return Blocks.size();
+  }
+
   /// isLoopExiting - True if terminator in the block can branch to another
   /// block that is outside of the current loop.
   ///
@@ -479,12 +485,13 @@ class LoopBase {
   }
 
   /// verifyLoop - Verify loop structure of this loop and all nested loops.
-  void verifyLoopNest() const {
+  void verifyLoopNest(DenseSet<const LoopT*> *Loops) const {
+    Loops->insert(static_cast<const LoopT *>(this));
     // Verify this loop.
     verifyLoop();
     // Verify the subloops.
     for (iterator I = begin(), E = end(); I != E; ++I)
-      (*I)->verifyLoopNest();
+      (*I)->verifyLoopNest(Loops);
   }
 
   void print(raw_ostream &OS, unsigned Depth = 0) const {
@@ -527,7 +534,7 @@ class Loop : public LoopBase<BasicBlock, Loop> {
   bool isLoopInvariant(Value *V) const;
 
   /// hasLoopInvariantOperands - Return true if all the operands of the
-  /// specified instruction are loop invariant. 
+  /// specified instruction are loop invariant.
   bool hasLoopInvariantOperands(Instruction *I) const;
 
   /// makeLoopInvariant - If the given value is an instruction inside of the
@@ -607,7 +614,7 @@ class Loop : public LoopBase<BasicBlock, Loop> {
   /// has a predecessor that is outside the loop.
   bool hasDedicatedExits() const;
 
-  /// getUniqueExitBlocks - Return all unique successor blocks of this loop. 
+  /// getUniqueExitBlocks - Return all unique successor blocks of this loop.
   /// These are the blocks _outside of the current loop_ which are branched to.
   /// This assumes that loop exits are in canonical form.
   ///
@@ -618,7 +625,7 @@ class Loop : public LoopBase<BasicBlock, Loop> {
   BasicBlock *getUniqueExitBlock() const;
 
   void dump() const;
-  
+
 private:
   friend class LoopInfoBase<BasicBlock, Loop>;
   explicit Loop(BasicBlock *BB) : LoopBase<BasicBlock, Loop>(BB) {}
@@ -635,13 +642,14 @@ class LoopInfoBase {
   DenseMap<BlockT *, LoopT *> BBMap;
   std::vector<LoopT *> TopLevelLoops;
   friend class LoopBase<BlockT, LoopT>;
+  friend class LoopInfo;
 
   void operator=(const LoopInfoBase &); // do not implement
   LoopInfoBase(const LoopInfo &);       // do not implement
 public:
   LoopInfoBase() { }
   ~LoopInfoBase() { releaseMemory(); }
-  
+
   void releaseMemory() {
     for (typename std::vector<LoopT *>::iterator I =
          TopLevelLoops.begin(), E = TopLevelLoops.end(); I != E; ++I)
@@ -650,7 +658,7 @@ class LoopInfoBase {
     BBMap.clear();                           // Reset internal state of analysis
     TopLevelLoops.clear();
   }
-  
+
   /// iterator/begin/end - The interface to the top-level loops in the current
   /// function.
   ///
@@ -658,7 +666,7 @@ class LoopInfoBase {
   iterator begin() const { return TopLevelLoops.begin(); }
   iterator end() const { return TopLevelLoops.end(); }
   bool empty() const { return TopLevelLoops.empty(); }
-  
+
   /// getLoopFor - Return the inner most loop that BB lives in.  If a basic
   /// block is in no loop (for example the entry node), null is returned.
   ///
@@ -667,13 +675,13 @@ class LoopInfoBase {
       BBMap.find(const_cast<BlockT*>(BB));
     return I != BBMap.end() ? I->second : 0;
   }
-  
+
   /// operator[] - same as getLoopFor...
   ///
   const LoopT *operator[](const BlockT *BB) const {
     return getLoopFor(BB);
   }
-  
+
   /// getLoopDepth - Return the loop nesting level of the specified block.  A
   /// depth of 0 means the block is not inside any loop.
   ///
@@ -687,7 +695,7 @@ class LoopInfoBase {
     const LoopT *L = getLoopFor(BB);
     return L && L->getHeader() == BB;
   }
-  
+
   /// removeLoop - This removes the specified top-level loop from this loop info
   /// object.  The loop is not deleted, as it will presumably be inserted into
   /// another loop.
@@ -698,16 +706,20 @@ class LoopInfoBase {
     TopLevelLoops.erase(TopLevelLoops.begin() + (I-begin()));
     return L;
   }
-  
+
   /// changeLoopFor - Change the top-level loop that contains BB to the
   /// specified loop.  This should be used by transformations that restructure
   /// the loop hierarchy tree.
   void changeLoopFor(BlockT *BB, LoopT *L) {
-    LoopT *&OldLoop = BBMap[BB];
-    assert(OldLoop && "Block not in a loop yet!");
-    OldLoop = L;
+    if (!L) {
+      typename DenseMap<BlockT *, LoopT *>::iterator I = BBMap.find(BB);
+      if (I != BBMap.end())
+        BBMap.erase(I);
+      return;
+    }
+    BBMap[BB] = L;
   }
-  
+
   /// changeTopLevelLoop - Replace the specified loop in the top-level loops
   /// list with the indicated loop.
   void changeTopLevelLoop(LoopT *OldLoop,
@@ -719,14 +731,14 @@ class LoopInfoBase {
     assert(NewLoop->ParentLoop == 0 && OldLoop->ParentLoop == 0 &&
            "Loops already embedded into a subloop!");
   }
-  
+
   /// addTopLevelLoop - This adds the specified loop to the collection of
   /// top-level loops.
   void addTopLevelLoop(LoopT *New) {
     assert(New->getParentLoop() == 0 && "Loop already in subloop!");
     TopLevelLoops.push_back(New);
   }
-  
+
   /// removeBlock - This method completely removes BB from all data structures,
   /// including all of the Loop objects it is nested in and our mapping from
   /// BasicBlocks to loops.
@@ -739,16 +751,16 @@ class LoopInfoBase {
       BBMap.erase(I);
     }
   }
-  
+
   // Internals
-  
+
   static bool isNotAlreadyContainedIn(const LoopT *SubLoop,
                                       const LoopT *ParentLoop) {
     if (SubLoop == 0) return true;
     if (SubLoop == ParentLoop) return false;
     return isNotAlreadyContainedIn(SubLoop->getParentLoop(), ParentLoop);
   }
-  
+
   void Calculate(DominatorTreeBase<BlockT> &DT) {
     BlockT *RootNode = DT.getRootNode()->getBlock();
 
@@ -757,7 +769,7 @@ class LoopInfoBase {
       if (LoopT *L = ConsiderForLoop(*NI, DT))
         TopLevelLoops.push_back(L);
   }
-  
+
   LoopT *ConsiderForLoop(BlockT *BB, DominatorTreeBase<BlockT> &DT) {
     if (BBMap.find(BB) != BBMap.end()) return 0;// Haven't processed this node?
 
@@ -812,9 +824,9 @@ class LoopInfoBase {
 
         // Normal case, add the block to our loop...
         L->Blocks.push_back(X);
-        
+
         typedef GraphTraits<Inverse<BlockT*> > InvBlockTraits;
-        
+
         // Add all of the predecessors of X to the end of the work stack...
         TodoStack.insert(TodoStack.end(), InvBlockTraits::child_begin(X),
                          InvBlockTraits::child_end(X));
@@ -878,7 +890,7 @@ class LoopInfoBase {
 
     return L;
   }
-  
+
   /// MoveSiblingLoopInto - This method moves the NewChild loop to live inside
   /// of the NewParent Loop, instead of being a sibling of it.
   void MoveSiblingLoopInto(LoopT *NewChild,
@@ -897,7 +909,7 @@ class LoopInfoBase {
 
     InsertLoopInto(NewChild, NewParent);
   }
-  
+
   /// InsertLoopInto - This inserts loop L into the specified parent loop.  If
   /// the parent loop contains a loop which should contain L, the loop gets
   /// inserted into L instead.
@@ -918,9 +930,9 @@ class LoopInfoBase {
     Parent->SubLoops.push_back(L);
     L->ParentLoop = Parent;
   }
-  
+
   // Debugging
-  
+
   void print(raw_ostream &OS) const {
     for (unsigned i = 0; i < TopLevelLoops.size(); ++i)
       TopLevelLoops[i]->print(OS);
@@ -990,7 +1002,7 @@ class LoopInfo : public FunctionPass {
   virtual void releaseMemory() { LI.releaseMemory(); }
 
   virtual void print(raw_ostream &O, const Module* M = 0) const;
-  
+
   virtual void getAnalysisUsage(AnalysisUsage &AU) const;
 
   /// removeLoop - This removes the specified top-level loop from this loop info
@@ -1024,6 +1036,12 @@ class LoopInfo : public FunctionPass {
     LI.removeBlock(BB);
   }
 
+  /// updateUnloop - Update LoopInfo after removing the last backedge from a
+  /// loop--now the "unloop". This updates the loop forest and parent loops for
+  /// each block so that Unloop is no longer referenced, but the caller must
+  /// actually delete the Unloop object.
+  void updateUnloop(Loop *Unloop);
+
   /// replacementPreservesLCSSAForm - Returns true if replacing From with To
   /// everywhere is guaranteed to preserve LCSSA form.
   bool replacementPreservesLCSSAForm(Instruction *From, Value *To) {
diff --git a/contrib/llvm/include/llvm/Analysis/LoopIterator.h b/contrib/llvm/include/llvm/Analysis/LoopIterator.h
new file mode 100644
index 000000000000..269ac8074054
--- /dev/null
+++ b/contrib/llvm/include/llvm/Analysis/LoopIterator.h
@@ -0,0 +1,186 @@
+//===--------- LoopIterator.h - Iterate over loop blocks --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This file defines iterators to visit the basic blocks within a loop.
+//
+// These iterators currently visit blocks within subloops as well.
+// Unfortunately we have no efficient way of summarizing loop exits which would
+// allow skipping subloops during traversal.
+//
+// If you want to visit all blocks in a loop and don't need an ordered traveral,
+// use Loop::block_begin() instead.
+//
+// This is intentionally designed to work with ill-formed loops in which the
+// backedge has been deleted. The only prerequisite is that all blocks
+// contained within the loop according to the most recent LoopInfo analysis are
+// reachable from the loop header.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_LOOP_ITERATOR_H
+#define LLVM_ANALYSIS_LOOP_ITERATOR_H
+
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/Analysis/LoopInfo.h"
+
+namespace llvm {
+
+class LoopBlocksTraversal;
+
+/// Store the result of a depth first search within basic blocks contained by a
+/// single loop.
+///
+/// TODO: This could be generalized for any CFG region, or the entire CFG.
+class LoopBlocksDFS {
+public:
+  /// Postorder list iterators.
+  typedef std::vector<BasicBlock*>::const_iterator POIterator;
+  typedef std::vector<BasicBlock*>::const_reverse_iterator RPOIterator;
+
+  friend class LoopBlocksTraversal;
+
+private:
+  Loop *L;
+
+  /// Map each block to its postorder number. A block is only mapped after it is
+  /// preorder visited by DFS. It's postorder number is initially zero and set
+  /// to nonzero after it is finished by postorder traversal.
+  DenseMap<BasicBlock*, unsigned> PostNumbers;
+  std::vector<BasicBlock*> PostBlocks;
+
+public:
+  LoopBlocksDFS(Loop *Container) :
+    L(Container), PostNumbers(NextPowerOf2(Container->getNumBlocks())) {
+    PostBlocks.reserve(Container->getNumBlocks());
+  }
+
+  Loop *getLoop() const { return L; }
+
+  /// Traverse the loop blocks and store the DFS result.
+  void perform(LoopInfo *LI);
+
+  /// Return true if postorder numbers are assigned to all loop blocks.
+  bool isComplete() const { return PostBlocks.size() == L->getNumBlocks(); }
+
+  /// Iterate over the cached postorder blocks.
+  POIterator beginPostorder() const {
+    assert(isComplete() && "bad loop DFS");
+    return PostBlocks.begin();
+  }
+  POIterator endPostorder() const { return PostBlocks.end(); }
+
+  /// Reverse iterate over the cached postorder blocks.
+  RPOIterator beginRPO() const {
+    assert(isComplete() && "bad loop DFS");
+    return PostBlocks.rbegin();
+  }
+  RPOIterator endRPO() const { return PostBlocks.rend(); }
+
+  /// Return true if this block has been preorder visited.
+  bool hasPreorder(BasicBlock *BB) const { return PostNumbers.count(BB); }
+
+  /// Return true if this block has a postorder number.
+  bool hasPostorder(BasicBlock *BB) const {
+    DenseMap<BasicBlock*, unsigned>::const_iterator I = PostNumbers.find(BB);
+    return I != PostNumbers.end() && I->second;
+  }
+
+  /// Get a block's postorder number.
+  unsigned getPostorder(BasicBlock *BB) const {
+    DenseMap<BasicBlock*, unsigned>::const_iterator I = PostNumbers.find(BB);
+    assert(I != PostNumbers.end() && "block not visited by DFS");
+    assert(I->second && "block not finished by DFS");
+    return I->second;
+  }
+
+  /// Get a block's reverse postorder number.
+  unsigned getRPO(BasicBlock *BB) const {
+    return 1 + PostBlocks.size() - getPostorder(BB);
+  }
+
+  void clear() {
+    PostNumbers.clear();
+    PostBlocks.clear();
+  }
+};
+
+/// Traverse the blocks in a loop using a depth-first search.
+class LoopBlocksTraversal {
+public:
+  /// Graph traversal iterator.
+  typedef po_iterator<BasicBlock*, LoopBlocksTraversal, true> POTIterator;
+
+private:
+  LoopBlocksDFS &DFS;
+  LoopInfo *LI;
+
+public:
+  LoopBlocksTraversal(LoopBlocksDFS &Storage, LoopInfo *LInfo) :
+    DFS(Storage), LI(LInfo) {}
+
+  /// Postorder traversal over the graph. This only needs to be done once.
+  /// po_iterator "automatically" calls back to visitPreorder and
+  /// finishPostorder to record the DFS result.
+  POTIterator begin() {
+    assert(DFS.PostBlocks.empty() && "Need clear DFS result before traversing");
+    assert(DFS.L->getNumBlocks() && "po_iterator cannot handle an empty graph");
+    return po_ext_begin(DFS.L->getHeader(), *this);
+  }
+  POTIterator end() {
+    // po_ext_end interface requires a basic block, but ignores its value.
+    return po_ext_end(DFS.L->getHeader(), *this);
+  }
+
+  /// Called by po_iterator upon reaching a block via a CFG edge. If this block
+  /// is contained in the loop and has not been visited, then mark it preorder
+  /// visited and return true.
+  ///
+  /// TODO: If anyone is interested, we could record preorder numbers here.
+  bool visitPreorder(BasicBlock *BB) {
+    if (!DFS.L->contains(LI->getLoopFor(BB)))
+      return false;
+
+    return DFS.PostNumbers.insert(std::make_pair(BB, 0)).second;
+  }
+
+  /// Called by po_iterator each time it advances, indicating a block's
+  /// postorder.
+  void finishPostorder(BasicBlock *BB) {
+    assert(DFS.PostNumbers.count(BB) && "Loop DFS skipped preorder");
+    DFS.PostBlocks.push_back(BB);
+    DFS.PostNumbers[BB] = DFS.PostBlocks.size();
+  }
+
+  //===----------------------------------------------------------------------
+  // Implement part of the std::set interface for the purpose of driving the
+  // generic po_iterator.
+
+  /// Return true if the block is outside the loop or has already been visited.
+  /// Sorry if this is counterintuitive.
+  bool count(BasicBlock *BB) const {
+    return !DFS.L->contains(LI->getLoopFor(BB)) || DFS.PostNumbers.count(BB);
+  }
+
+  /// If this block is contained in the loop and has not been visited, return
+  /// true and assign a preorder number. This is a proxy for visitPreorder
+  /// called by POIterator.
+  bool insert(BasicBlock *BB) {
+    return visitPreorder(BB);
+  }
+};
+
+/// Specialize DFSetTraits to record postorder numbers.
+template<> struct DFSetTraits<LoopBlocksTraversal> {
+  static void finishPostorder(BasicBlock *BB, LoopBlocksTraversal& LBT) {
+    LBT.finishPostorder(BB);
+  }
+};
+
+} // End namespace llvm
+
+#endif
diff --git a/contrib/llvm/include/llvm/Analysis/LoopPass.h b/contrib/llvm/include/llvm/Analysis/LoopPass.h
index 1603d2ea7a4f..e6ed9bccee31 100644
--- a/contrib/llvm/include/llvm/Analysis/LoopPass.h
+++ b/contrib/llvm/include/llvm/Analysis/LoopPass.h
@@ -84,7 +84,7 @@ class LoopPass : public Pass {
 class LPPassManager : public FunctionPass, public PMDataManager {
 public:
   static char ID;
-  explicit LPPassManager(int Depth);
+  explicit LPPassManager();
 
   /// run - Execute all of the passes scheduled for execution.  Keep track of
   /// whether any of the passes modifies the module, and if so, return true.
diff --git a/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h b/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h
index 22493f6f8b9e..865d236f6f3a 100644
--- a/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -51,14 +51,14 @@ const CallInst *isArrayMalloc(const Value *I, const TargetData *TD);
 ///   0: PointerType is the malloc calls' return type.
 ///   1: PointerType is the bitcast's result type.
 ///  >1: Unique PointerType cannot be determined, return NULL.
-const PointerType *getMallocType(const CallInst *CI);
+PointerType *getMallocType(const CallInst *CI);
 
 /// getMallocAllocatedType - Returns the Type allocated by malloc call.
 /// The Type depends on the number of bitcast uses of the malloc call:
 ///   0: PointerType is the malloc calls' return type.
 ///   1: PointerType is the bitcast's result type.
 ///  >1: Unique PointerType cannot be determined, return NULL.
-const Type *getMallocAllocatedType(const CallInst *CI);
+Type *getMallocAllocatedType(const CallInst *CI);
 
 /// getMallocArraySize - Returns the array size of a malloc call.  If the 
 /// argument passed to malloc is a multiple of the size of the malloced type,
diff --git a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index 34860e712fbb..e18d937f6916 100644
--- a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -52,9 +52,6 @@ namespace llvm {
       ///   1. Loads are clobbered by may-alias stores.
       ///   2. Loads are considered clobbered by partially-aliased loads.  The
       ///      client may choose to analyze deeper into these cases.
-      ///
-      /// A dependence query on the first instruction of the entry block will
-      /// return a clobber(self) result.
       Clobber,
 
       /// Def - This is a dependence on the specified instruction which
@@ -76,11 +73,27 @@ namespace llvm {
       ///      operands to the calls are the same.
       Def,
       
+      /// Other - This marker indicates that the query has no known dependency
+      /// in the specified block.  More detailed state info is encoded in the
+      /// upper part of the pair (i.e. the Instruction*)
+      Other
+    };
+    /// If DepType is "Other", the upper part of the pair
+    /// (i.e. the Instruction* part) is instead used to encode more detailed
+    /// type information as follows
+    enum OtherType {
       /// NonLocal - This marker indicates that the query has no dependency in
       /// the specified block.  To find out more, the client should query other
       /// predecessor blocks.
-      NonLocal
+      NonLocal = 0x4,
+      /// NonFuncLocal - This marker indicates that the query has no
+      /// dependency in the specified function.
+      NonFuncLocal = 0x8,
+      /// Unknown - This marker indicates that the query dependency
+      /// is unknown.
+      Unknown = 0xc
     };
+
     typedef PointerIntPair<Instruction*, 2, DepType> PairTy;
     PairTy Value;
     explicit MemDepResult(PairTy V) : Value(V) {}
@@ -98,19 +111,21 @@ namespace llvm {
       return MemDepResult(PairTy(Inst, Clobber));
     }
     static MemDepResult getNonLocal() {
-      return MemDepResult(PairTy(0, NonLocal));
+      return MemDepResult(
+        PairTy(reinterpret_cast<Instruction*>(NonLocal), Other));
+    }
+    static MemDepResult getNonFuncLocal() {
+      return MemDepResult(
+        PairTy(reinterpret_cast<Instruction*>(NonFuncLocal), Other));
     }
     static MemDepResult getUnknown() {
-      return MemDepResult(PairTy(0, Clobber));
+      return MemDepResult(
+        PairTy(reinterpret_cast<Instruction*>(Unknown), Other));
     }
 
     /// isClobber - Return true if this MemDepResult represents a query that is
     /// a instruction clobber dependency.
-    bool isClobber() const { return Value.getInt() == Clobber && getInst(); }
-
-    /// isUnknown - Return true if this MemDepResult represents a query which
-    /// cannot and/or will not be computed.
-    bool isUnknown() const { return Value.getInt() == Clobber && !getInst(); }
+    bool isClobber() const { return Value.getInt() == Clobber; }
 
     /// isDef - Return true if this MemDepResult represents a query that is
     /// a instruction definition dependency.
@@ -119,11 +134,31 @@ namespace llvm {
     /// isNonLocal - Return true if this MemDepResult represents a query that
     /// is transparent to the start of the block, but where a non-local hasn't
     /// been done.
-    bool isNonLocal() const { return Value.getInt() == NonLocal; }
+    bool isNonLocal() const {
+      return Value.getInt() == Other
+        && Value.getPointer() == reinterpret_cast<Instruction*>(NonLocal);
+    }
+
+    /// isNonFuncLocal - Return true if this MemDepResult represents a query
+    /// that is transparent to the start of the function.
+    bool isNonFuncLocal() const {
+      return Value.getInt() == Other
+        && Value.getPointer() == reinterpret_cast<Instruction*>(NonFuncLocal);
+    }
     
+    /// isUnknown - Return true if this MemDepResult represents a query which
+    /// cannot and/or will not be computed.
+    bool isUnknown() const {
+      return Value.getInt() == Other
+        && Value.getPointer() == reinterpret_cast<Instruction*>(Unknown);
+    }
+
     /// getInst() - If this is a normal dependency, return the instruction that
     /// is depended on.  Otherwise, return null.
-    Instruction *getInst() const { return Value.getPointer(); }
+    Instruction *getInst() const {
+      if (Value.getInt() == Other) return NULL;
+      return Value.getPointer();
+    }
     
     bool operator==(const MemDepResult &M) const { return Value == M.Value; }
     bool operator!=(const MemDepResult &M) const { return Value != M.Value; }
diff --git a/contrib/llvm/include/llvm/Analysis/RegionPass.h b/contrib/llvm/include/llvm/Analysis/RegionPass.h
index 1a93859bab9c..68f12012bcd1 100644
--- a/contrib/llvm/include/llvm/Analysis/RegionPass.h
+++ b/contrib/llvm/include/llvm/Analysis/RegionPass.h
@@ -88,7 +88,7 @@ class RGPassManager : public FunctionPass, public PMDataManager {
 
 public:
   static char ID;
-  explicit RGPassManager(int Depth);
+  explicit RGPassManager();
 
   /// @brief Execute all of the passes scheduled for execution.
   ///
diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h
index 554524a127a8..10d933e68f5b 100644
--- a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -103,7 +103,7 @@ namespace llvm {
 
     /// getType - Return the LLVM type of this SCEV expression.
     ///
-    const Type *getType() const;
+    Type *getType() const;
 
     /// isZero - Return true if the expression is a constant zero.
     ///
@@ -241,31 +241,94 @@ namespace llvm {
     ///
     ValueExprMapType ValueExprMap;
 
+    /// ExitLimit - Information about the number of loop iterations for
+    /// which a loop exit's branch condition evaluates to the not-taken path.
+    /// This is a temporary pair of exact and max expressions that are
+    /// eventually summarized in ExitNotTakenInfo and BackedgeTakenInfo.
+    struct ExitLimit {
+      const SCEV *Exact;
+      const SCEV *Max;
+
+      /*implicit*/ ExitLimit(const SCEV *E) : Exact(E), Max(E) {}
+
+      ExitLimit(const SCEV *E, const SCEV *M) : Exact(E), Max(M) {}
+
+      /// hasAnyInfo - Test whether this ExitLimit contains any computed
+      /// information, or whether it's all SCEVCouldNotCompute values.
+      bool hasAnyInfo() const {
+        return !isa<SCEVCouldNotCompute>(Exact) ||
+          !isa<SCEVCouldNotCompute>(Max);
+      }
+    };
+
+    /// ExitNotTakenInfo - Information about the number of times a particular
+    /// loop exit may be reached before exiting the loop.
+    struct ExitNotTakenInfo {
+      AssertingVH<BasicBlock> ExitingBlock;
+      const SCEV *ExactNotTaken;
+      PointerIntPair<ExitNotTakenInfo*, 1> NextExit;
+
+      ExitNotTakenInfo() : ExitingBlock(0), ExactNotTaken(0) {}
+
+      /// isCompleteList - Return true if all loop exits are computable.
+      bool isCompleteList() const {
+        return NextExit.getInt() == 0;
+      }
+
+      void setIncomplete() { NextExit.setInt(1); }
+
+      /// getNextExit - Return a pointer to the next exit's not-taken info.
+      ExitNotTakenInfo *getNextExit() const {
+        return NextExit.getPointer();
+      }
+
+      void setNextExit(ExitNotTakenInfo *ENT) { NextExit.setPointer(ENT); }
+    };
+
     /// BackedgeTakenInfo - Information about the backedge-taken count
     /// of a loop. This currently includes an exact count and a maximum count.
     ///
-    struct BackedgeTakenInfo {
-      /// Exact - An expression indicating the exact backedge-taken count of
-      /// the loop if it is known, or a SCEVCouldNotCompute otherwise.
-      const SCEV *Exact;
+    class BackedgeTakenInfo {
+      /// ExitNotTaken - A list of computable exits and their not-taken counts.
+      /// Loops almost never have more than one computable exit.
+      ExitNotTakenInfo ExitNotTaken;
 
       /// Max - An expression indicating the least maximum backedge-taken
       /// count of the loop that is known, or a SCEVCouldNotCompute.
       const SCEV *Max;
 
-      /*implicit*/ BackedgeTakenInfo(const SCEV *exact) :
-        Exact(exact), Max(exact) {}
+    public:
+      BackedgeTakenInfo() : Max(0) {}
 
-      BackedgeTakenInfo(const SCEV *exact, const SCEV *max) :
-        Exact(exact), Max(max) {}
+      /// Initialize BackedgeTakenInfo from a list of exact exit counts.
+      BackedgeTakenInfo(
+        SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts,
+        bool Complete, const SCEV *MaxCount);
 
       /// hasAnyInfo - Test whether this BackedgeTakenInfo contains any
       /// computed information, or whether it's all SCEVCouldNotCompute
       /// values.
       bool hasAnyInfo() const {
-        return !isa<SCEVCouldNotCompute>(Exact) ||
-               !isa<SCEVCouldNotCompute>(Max);
+        return ExitNotTaken.ExitingBlock || !isa<SCEVCouldNotCompute>(Max);
       }
+
+      /// getExact - Return an expression indicating the exact backedge-taken
+      /// count of the loop if it is known, or SCEVCouldNotCompute
+      /// otherwise. This is the number of times the loop header can be
+      /// guaranteed to execute, minus one.
+      const SCEV *getExact(ScalarEvolution *SE) const;
+
+      /// getExact - Return the number of times this loop exit may fall through
+      /// to the back edge, or SCEVCouldNotCompute. The loop is guaranteed not
+      /// to exit via this block before this number of iterations, but may exit
+      /// via another block.
+      const SCEV *getExact(BasicBlock *ExitingBlock, ScalarEvolution *SE) const;
+
+      /// getMax - Get the max backedge taken count for the loop.
+      const SCEV *getMax(ScalarEvolution *SE) const;
+
+      /// clear - Invalidate this result and free associated memory.
+      void clear();
     };
 
     /// BackedgeTakenCounts - Cache the backedge-taken count of the loops for
@@ -365,64 +428,59 @@ namespace llvm {
     /// loop will iterate.
     BackedgeTakenInfo ComputeBackedgeTakenCount(const Loop *L);
 
-    /// ComputeBackedgeTakenCountFromExit - Compute the number of times the
-    /// backedge of the specified loop will execute if it exits via the
-    /// specified block.
-    BackedgeTakenInfo ComputeBackedgeTakenCountFromExit(const Loop *L,
-                                                      BasicBlock *ExitingBlock);
+    /// ComputeExitLimit - Compute the number of times the backedge of the
+    /// specified loop will execute if it exits via the specified block.
+    ExitLimit ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock);
 
-    /// ComputeBackedgeTakenCountFromExitCond - Compute the number of times the
-    /// backedge of the specified loop will execute if its exit condition
-    /// were a conditional branch of ExitCond, TBB, and FBB.
-    BackedgeTakenInfo
-      ComputeBackedgeTakenCountFromExitCond(const Loop *L,
-                                            Value *ExitCond,
-                                            BasicBlock *TBB,
-                                            BasicBlock *FBB);
+    /// ComputeExitLimitFromCond - Compute the number of times the backedge of
+    /// the specified loop will execute if its exit condition were a conditional
+    /// branch of ExitCond, TBB, and FBB.
+    ExitLimit ComputeExitLimitFromCond(const Loop *L,
+                                       Value *ExitCond,
+                                       BasicBlock *TBB,
+                                       BasicBlock *FBB);
 
-    /// ComputeBackedgeTakenCountFromExitCondICmp - Compute the number of
-    /// times the backedge of the specified loop will execute if its exit
-    /// condition were a conditional branch of the ICmpInst ExitCond, TBB,
-    /// and FBB.
-    BackedgeTakenInfo
-      ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L,
-                                                ICmpInst *ExitCond,
-                                                BasicBlock *TBB,
-                                                BasicBlock *FBB);
+    /// ComputeExitLimitFromICmp - Compute the number of times the backedge of
+    /// the specified loop will execute if its exit condition were a conditional
+    /// branch of the ICmpInst ExitCond, TBB, and FBB.
+    ExitLimit ComputeExitLimitFromICmp(const Loop *L,
+                                       ICmpInst *ExitCond,
+                                       BasicBlock *TBB,
+                                       BasicBlock *FBB);
 
-    /// ComputeLoadConstantCompareBackedgeTakenCount - Given an exit condition
+    /// ComputeLoadConstantCompareExitLimit - Given an exit condition
     /// of 'icmp op load X, cst', try to see if we can compute the
     /// backedge-taken count.
-    BackedgeTakenInfo
-      ComputeLoadConstantCompareBackedgeTakenCount(LoadInst *LI,
-                                                   Constant *RHS,
-                                                   const Loop *L,
-                                                   ICmpInst::Predicate p);
+    ExitLimit ComputeLoadConstantCompareExitLimit(LoadInst *LI,
+                                                  Constant *RHS,
+                                                  const Loop *L,
+                                                  ICmpInst::Predicate p);
 
-    /// ComputeBackedgeTakenCountExhaustively - If the loop is known to execute
-    /// a constant number of times (the condition evolves only from constants),
+    /// ComputeExitCountExhaustively - If the loop is known to execute a
+    /// constant number of times (the condition evolves only from constants),
     /// try to evaluate a few iterations of the loop until we get the exit
     /// condition gets a value of ExitWhen (true or false).  If we cannot
-    /// evaluate the backedge-taken count of the loop, return CouldNotCompute.
-    const SCEV *ComputeBackedgeTakenCountExhaustively(const Loop *L,
-                                                      Value *Cond,
-                                                      bool ExitWhen);
+    /// evaluate the exit count of the loop, return CouldNotCompute.
+    const SCEV *ComputeExitCountExhaustively(const Loop *L,
+                                             Value *Cond,
+                                             bool ExitWhen);
 
-    /// HowFarToZero - Return the number of times a backedge comparing the
-    /// specified value to zero will execute.  If not computable, return
+    /// HowFarToZero - Return the number of times an exit condition comparing
+    /// the specified value to zero will execute.  If not computable, return
     /// CouldNotCompute.
-    BackedgeTakenInfo HowFarToZero(const SCEV *V, const Loop *L);
+    ExitLimit HowFarToZero(const SCEV *V, const Loop *L);
 
-    /// HowFarToNonZero - Return the number of times a backedge checking the
-    /// specified value for nonzero will execute.  If not computable, return
+    /// HowFarToNonZero - Return the number of times an exit condition checking
+    /// the specified value for nonzero will execute.  If not computable, return
     /// CouldNotCompute.
-    BackedgeTakenInfo HowFarToNonZero(const SCEV *V, const Loop *L);
+    ExitLimit HowFarToNonZero(const SCEV *V, const Loop *L);
 
-    /// HowManyLessThans - Return the number of times a backedge containing the
-    /// specified less-than comparison will execute.  If not computable, return
-    /// CouldNotCompute. isSigned specifies whether the less-than is signed.
-    BackedgeTakenInfo HowManyLessThans(const SCEV *LHS, const SCEV *RHS,
-                                       const Loop *L, bool isSigned);
+    /// HowManyLessThans - Return the number of times an exit condition
+    /// containing the specified less-than comparison will execute.  If not
+    /// computable, return CouldNotCompute. isSigned specifies whether the
+    /// less-than is signed.
+    ExitLimit HowManyLessThans(const SCEV *LHS, const SCEV *RHS,
+                               const Loop *L, bool isSigned);
 
     /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB
     /// (which may not be an immediate predecessor) which has exactly one
@@ -450,7 +508,8 @@ namespace llvm {
     /// FoundLHS, and FoundRHS is true.
     bool isImpliedCondOperandsHelper(ICmpInst::Predicate Pred,
                                      const SCEV *LHS, const SCEV *RHS,
-                                     const SCEV *FoundLHS, const SCEV *FoundRHS);
+                                     const SCEV *FoundLHS,
+                                     const SCEV *FoundRHS);
 
     /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
     /// in the header of its containing loop, we know the loop executes a
@@ -479,17 +538,17 @@ namespace llvm {
     /// the SCEV framework. This primarily includes integer types, and it
     /// can optionally include pointer types if the ScalarEvolution class
     /// has access to target-specific information.
-    bool isSCEVable(const Type *Ty) const;
+    bool isSCEVable(Type *Ty) const;
 
     /// getTypeSizeInBits - Return the size in bits of the specified type,
     /// for which isSCEVable must return true.
-    uint64_t getTypeSizeInBits(const Type *Ty) const;
+    uint64_t getTypeSizeInBits(Type *Ty) const;
 
     /// getEffectiveSCEVType - Return a type with the same bitwidth as
     /// the given type and which represents how SCEV will treat the given
     /// type, for which isSCEVable must return true. For pointer types,
     /// this is the pointer-sized integer type.
-    const Type *getEffectiveSCEVType(const Type *Ty) const;
+    Type *getEffectiveSCEVType(Type *Ty) const;
 
     /// getSCEV - Return a SCEV expression for the full generality of the
     /// specified expression.
@@ -497,11 +556,11 @@ namespace llvm {
 
     const SCEV *getConstant(ConstantInt *V);
     const SCEV *getConstant(const APInt& Val);
-    const SCEV *getConstant(const Type *Ty, uint64_t V, bool isSigned = false);
-    const SCEV *getTruncateExpr(const SCEV *Op, const Type *Ty);
-    const SCEV *getZeroExtendExpr(const SCEV *Op, const Type *Ty);
-    const SCEV *getSignExtendExpr(const SCEV *Op, const Type *Ty);
-    const SCEV *getAnyExtendExpr(const SCEV *Op, const Type *Ty);
+    const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false);
+    const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty);
+    const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty);
+    const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty);
+    const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty);
     const SCEV *getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
                            SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap);
     const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS,
@@ -529,6 +588,14 @@ namespace llvm {
       Ops.push_back(RHS);
       return getMulExpr(Ops, Flags);
     }
+    const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2,
+                           SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap) {
+      SmallVector<const SCEV *, 3> Ops;
+      Ops.push_back(Op0);
+      Ops.push_back(Op1);
+      Ops.push_back(Op2);
+      return getMulExpr(Ops, Flags);
+    }
     const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS);
     const SCEV *getAddRecExpr(const SCEV *Start, const SCEV *Step,
                               const Loop *L, SCEV::NoWrapFlags Flags);
@@ -550,19 +617,19 @@ namespace llvm {
 
     /// getSizeOfExpr - Return an expression for sizeof on the given type.
     ///
-    const SCEV *getSizeOfExpr(const Type *AllocTy);
+    const SCEV *getSizeOfExpr(Type *AllocTy);
 
     /// getAlignOfExpr - Return an expression for alignof on the given type.
     ///
-    const SCEV *getAlignOfExpr(const Type *AllocTy);
+    const SCEV *getAlignOfExpr(Type *AllocTy);
 
     /// getOffsetOfExpr - Return an expression for offsetof on the given field.
     ///
-    const SCEV *getOffsetOfExpr(const StructType *STy, unsigned FieldNo);
+    const SCEV *getOffsetOfExpr(StructType *STy, unsigned FieldNo);
 
     /// getOffsetOfExpr - Return an expression for offsetof on the given field.
     ///
-    const SCEV *getOffsetOfExpr(const Type *CTy, Constant *FieldNo);
+    const SCEV *getOffsetOfExpr(Type *CTy, Constant *FieldNo);
 
     /// getNegativeSCEV - Return the SCEV object corresponding to -V.
     ///
@@ -579,33 +646,33 @@ namespace llvm {
     /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion
     /// of the input value to the specified type.  If the type must be
     /// extended, it is zero extended.
-    const SCEV *getTruncateOrZeroExtend(const SCEV *V, const Type *Ty);
+    const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty);
 
     /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion
     /// of the input value to the specified type.  If the type must be
     /// extended, it is sign extended.
-    const SCEV *getTruncateOrSignExtend(const SCEV *V, const Type *Ty);
+    const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty);
 
     /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of
     /// the input value to the specified type.  If the type must be extended,
     /// it is zero extended.  The conversion must not be narrowing.
-    const SCEV *getNoopOrZeroExtend(const SCEV *V, const Type *Ty);
+    const SCEV *getNoopOrZeroExtend(const SCEV *V, Type *Ty);
 
     /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of
     /// the input value to the specified type.  If the type must be extended,
     /// it is sign extended.  The conversion must not be narrowing.
-    const SCEV *getNoopOrSignExtend(const SCEV *V, const Type *Ty);
+    const SCEV *getNoopOrSignExtend(const SCEV *V, Type *Ty);
 
     /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of
     /// the input value to the specified type. If the type must be extended,
     /// it is extended with unspecified bits. The conversion must not be
     /// narrowing.
-    const SCEV *getNoopOrAnyExtend(const SCEV *V, const Type *Ty);
+    const SCEV *getNoopOrAnyExtend(const SCEV *V, Type *Ty);
 
     /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the
     /// input value to the specified type.  The conversion must not be
     /// widening.
-    const SCEV *getTruncateOrNoop(const SCEV *V, const Type *Ty);
+    const SCEV *getTruncateOrNoop(const SCEV *V, Type *Ty);
 
     /// getUMaxFromMismatchedTypes - Promote the operands to the wider of
     /// the types using zero-extension, and then perform a umax operation
@@ -653,6 +720,23 @@ namespace llvm {
     bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred,
                                      const SCEV *LHS, const SCEV *RHS);
 
+    /// getSmallConstantTripCount - Returns the maximum trip count of this loop
+    /// as a normal unsigned value, if possible. Returns 0 if the trip count is
+    /// unknown or not constant.
+    unsigned getSmallConstantTripCount(Loop *L, BasicBlock *ExitBlock);
+
+    /// getSmallConstantTripMultiple - Returns the largest constant divisor of
+    /// the trip count of this loop as a normal unsigned value, if
+    /// possible. This means that the actual trip count is always a multiple of
+    /// the returned value (don't forget the trip count could very well be zero
+    /// as well!).
+    unsigned getSmallConstantTripMultiple(Loop *L, BasicBlock *ExitBlock);
+
+    // getExitCount - Get the expression for the number of loop iterations for
+    // which this loop is guaranteed not to exit via ExitingBlock. Otherwise
+    // return SCEVCouldNotCompute.
+    const SCEV *getExitCount(Loop *L, BasicBlock *ExitingBlock);
+
     /// getBackedgeTakenCount - If the specified loop has a predictable
     /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute
     /// object. The backedge-taken count is the number of times the loop header
diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h
index a8c03b2219e1..a4ad1451d412 100644
--- a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h
+++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h
@@ -64,16 +64,34 @@ namespace llvm {
     /// in a more literal form.
     bool CanonicalMode;
 
+    /// When invoked from LSR, the expander is in "strength reduction" mode. The
+    /// only difference is that phi's are only reused if they are already in
+    /// "expanded" form.
+    bool LSRMode;
+
     typedef IRBuilder<true, TargetFolder> BuilderType;
     BuilderType Builder;
 
+#ifndef NDEBUG
+    const char *DebugType;
+#endif
+
     friend struct SCEVVisitor<SCEVExpander, Value*>;
 
   public:
     /// SCEVExpander - Construct a SCEVExpander in "canonical" mode.
     explicit SCEVExpander(ScalarEvolution &se, const char *name)
       : SE(se), IVName(name), IVIncInsertLoop(0), IVIncInsertPos(0),
-        CanonicalMode(true), Builder(se.getContext(), TargetFolder(se.TD)) {}
+        CanonicalMode(true), LSRMode(false),
+        Builder(se.getContext(), TargetFolder(se.TD)) {
+#ifndef NDEBUG
+      DebugType = "";
+#endif
+    }
+
+#ifndef NDEBUG
+    void setDebugType(const char* s) { DebugType = s; }
+#endif
 
     /// clear - Erase the contents of the InsertedExpressions map so that users
     /// trying to expand the same expression into multiple BasicBlocks or
@@ -88,13 +106,21 @@ namespace llvm {
     /// canonical induction variable of the specified type for the specified
     /// loop (inserting one if there is none).  A canonical induction variable
     /// starts at zero and steps by one on each iteration.
-    PHINode *getOrInsertCanonicalInductionVariable(const Loop *L,
-                                                   const Type *Ty);
+    PHINode *getOrInsertCanonicalInductionVariable(const Loop *L, Type *Ty);
+
+    /// hoistStep - Utility for hoisting an IV increment.
+    static bool hoistStep(Instruction *IncV, Instruction *InsertPos,
+                          const DominatorTree *DT);
+
+    /// replaceCongruentIVs - replace congruent phis with their most canonical
+    /// representative. Return the number of phis eliminated.
+    unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT,
+                                 SmallVectorImpl<WeakVH> &DeadInsts);
 
     /// expandCodeFor - Insert code to directly compute the specified SCEV
     /// expression into the program.  The inserted code is inserted into the
     /// specified block.
-    Value *expandCodeFor(const SCEV *SH, const Type *Ty, Instruction *I);
+    Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I);
 
     /// setIVIncInsertPos - Set the current IV increment loop and position.
     void setIVIncInsertPos(const Loop *L, Instruction *Pos) {
@@ -127,13 +153,14 @@ namespace llvm {
     /// is useful for late optimization passes.
     void disableCanonicalMode() { CanonicalMode = false; }
 
+    void enableLSRMode() { LSRMode = true; }
+
     /// clearInsertPoint - Clear the current insertion point. This is useful
     /// if the instruction that had been serving as the insertion point may
     /// have been deleted.
     void clearInsertPoint() {
       Builder.ClearInsertionPoint();
     }
-
   private:
     LLVMContext &getContext() const { return SE.getContext(); }
 
@@ -145,20 +172,20 @@ namespace llvm {
     /// reusing an existing cast if a suitable one exists, moving an existing
     /// cast if a suitable one exists but isn't in the right place, or
     /// or creating a new one.
-    Value *ReuseOrCreateCast(Value *V, const Type *Ty,
+    Value *ReuseOrCreateCast(Value *V, Type *Ty,
                              Instruction::CastOps Op,
                              BasicBlock::iterator IP);
 
     /// InsertNoopCastOfTo - Insert a cast of V to the specified type,
     /// which must be possible with a noop cast, doing what we can to
     /// share the casts.
-    Value *InsertNoopCastOfTo(Value *V, const Type *Ty);
+    Value *InsertNoopCastOfTo(Value *V, Type *Ty);
 
     /// expandAddToGEP - Expand a SCEVAddExpr with a pointer type into a GEP
     /// instead of using ptrtoint+arithmetic+inttoptr.
     Value *expandAddToGEP(const SCEV *const *op_begin,
                           const SCEV *const *op_end,
-                          const PointerType *PTy, const Type *Ty, Value *V);
+                          PointerType *PTy, Type *Ty, Value *V);
 
     Value *expand(const SCEV *S);
 
@@ -166,7 +193,7 @@ namespace llvm {
     /// expression into the program.  The inserted code is inserted into the
     /// SCEVExpander's current insertion point. If a type is specified, the
     /// result will be expanded to have that type, with a cast if necessary.
-    Value *expandCodeFor(const SCEV *SH, const Type *Ty = 0);
+    Value *expandCodeFor(const SCEV *SH, Type *Ty = 0);
 
     /// isInsertedInstruction - Return true if the specified instruction was
     /// inserted by the code rewriter.  If so, the client should not modify the
@@ -208,11 +235,15 @@ namespace llvm {
 
     void restoreInsertPoint(BasicBlock *BB, BasicBlock::iterator I);
 
+    bool isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
+
+    bool isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
+
     Value *expandAddRecExprLiterally(const SCEVAddRecExpr *);
     PHINode *getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
                                        const Loop *L,
-                                       const Type *ExpandTy,
-                                       const Type *IntTy);
+                                       Type *ExpandTy,
+                                       Type *IntTy);
   };
 }
 
diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 856d92c97c08..b6f0ae54cfa0 100644
--- a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -42,7 +42,7 @@ namespace llvm {
   public:
     ConstantInt *getValue() const { return V; }
 
-    const Type *getType() const { return V->getType(); }
+    Type *getType() const { return V->getType(); }
 
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
     static inline bool classof(const SCEVConstant *S) { return true; }
@@ -57,14 +57,14 @@ namespace llvm {
   class SCEVCastExpr : public SCEV {
   protected:
     const SCEV *Op;
-    const Type *Ty;
+    Type *Ty;
 
     SCEVCastExpr(const FoldingSetNodeIDRef ID,
-                 unsigned SCEVTy, const SCEV *op, const Type *ty);
+                 unsigned SCEVTy, const SCEV *op, Type *ty);
 
   public:
     const SCEV *getOperand() const { return Op; }
-    const Type *getType() const { return Ty; }
+    Type *getType() const { return Ty; }
 
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
     static inline bool classof(const SCEVCastExpr *S) { return true; }
@@ -83,7 +83,7 @@ namespace llvm {
     friend class ScalarEvolution;
 
     SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
-                     const SCEV *op, const Type *ty);
+                     const SCEV *op, Type *ty);
 
   public:
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -101,7 +101,7 @@ namespace llvm {
     friend class ScalarEvolution;
 
     SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
-                       const SCEV *op, const Type *ty);
+                       const SCEV *op, Type *ty);
 
   public:
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -119,7 +119,7 @@ namespace llvm {
     friend class ScalarEvolution;
 
     SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
-                       const SCEV *op, const Type *ty);
+                       const SCEV *op, Type *ty);
 
   public:
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -158,7 +158,7 @@ namespace llvm {
     op_iterator op_begin() const { return Operands; }
     op_iterator op_end() const { return Operands + NumOperands; }
 
-    const Type *getType() const { return getOperand(0)->getType(); }
+    Type *getType() const { return getOperand(0)->getType(); }
 
     NoWrapFlags getNoWrapFlags(NoWrapFlags Mask = NoWrapMask) const {
       return (NoWrapFlags)(SubclassData & Mask);
@@ -214,7 +214,7 @@ namespace llvm {
     }
 
   public:
-    const Type *getType() const {
+    Type *getType() const {
       // Use the type of the last operand, which is likely to be a pointer
       // type, if there is one. This doesn't usually matter, but it can help
       // reduce casts when the expressions are expanded.
@@ -263,7 +263,7 @@ namespace llvm {
     const SCEV *getLHS() const { return LHS; }
     const SCEV *getRHS() const { return RHS; }
 
-    const Type *getType() const {
+    Type *getType() const {
       // In most cases the types of LHS and RHS will be the same, but in some
       // crazy cases one or the other may be a pointer. ScalarEvolution doesn't
       // depend on the type for correctness, but handling types carefully can
@@ -441,11 +441,11 @@ namespace llvm {
     /// folded with other operations into something unrecognizable. This
     /// is mainly only useful for pretty-printing and other situations
     /// where it isn't absolutely required for these to succeed.
-    bool isSizeOf(const Type *&AllocTy) const;
-    bool isAlignOf(const Type *&AllocTy) const;
-    bool isOffsetOf(const Type *&STy, Constant *&FieldNo) const;
+    bool isSizeOf(Type *&AllocTy) const;
+    bool isAlignOf(Type *&AllocTy) const;
+    bool isOffsetOf(Type *&STy, Constant *&FieldNo) const;
 
-    const Type *getType() const { return getValPtr()->getType(); }
+    Type *getType() const { return getValPtr()->getType(); }
 
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
     static inline bool classof(const SCEVUnknown *S) { return true; }
diff --git a/contrib/llvm/include/llvm/Argument.h b/contrib/llvm/include/llvm/Argument.h
index ff8637881eb7..cd7488266231 100644
--- a/contrib/llvm/include/llvm/Argument.h
+++ b/contrib/llvm/include/llvm/Argument.h
@@ -39,7 +39,7 @@ class Argument : public Value, public ilist_node<Argument> {
   /// Argument ctor - If Function argument is specified, this argument is
   /// inserted at the end of the argument list for the function.
   ///
-  explicit Argument(const Type *Ty, const Twine &Name = "", Function *F = 0);
+  explicit Argument(Type *Ty, const Twine &Name = "", Function *F = 0);
 
   inline const Function *getParent() const { return Parent; }
   inline       Function *getParent()       { return Parent; }
diff --git a/contrib/llvm/include/llvm/Attributes.h b/contrib/llvm/include/llvm/Attributes.h
index 233eab8bf179..2d7b33b29bcf 100644
--- a/contrib/llvm/include/llvm/Attributes.h
+++ b/contrib/llvm/include/llvm/Attributes.h
@@ -65,8 +65,7 @@ const Attributes StackAlignment  = 7<<26; ///< Alignment of stack for
                                           ///of alignment with +1 bias
                                           ///0 means unaligned (different from
                                           ///alignstack(1))
-const Attributes Hotpatch    = 1<<29;     ///< Function should have special
-                                          ///'hotpatch' sequence in prologue
+const Attributes ReturnsTwice    = 1<<29; ///< Function can return twice
 const Attributes UWTable     = 1<<30;     ///< Function must be in a unwind
                                           ///table
 const Attributes NonLazyBind = 1U<<31;    ///< Function is called early and/or
@@ -93,7 +92,7 @@ const Attributes ParameterOnly = ByVal | Nest | StructRet | NoCapture;
 const Attributes FunctionOnly = NoReturn | NoUnwind | ReadNone | ReadOnly |
   NoInline | AlwaysInline | OptimizeForSize | StackProtect | StackProtectReq |
   NoRedZone | NoImplicitFloat | Naked | InlineHint | StackAlignment |
-  Hotpatch | UWTable | NonLazyBind;
+  UWTable | NonLazyBind | ReturnsTwice;
 
 /// @brief Parameter attributes that do not apply to vararg call arguments.
 const Attributes VarArgsIncompatible = StructRet;
@@ -107,7 +106,7 @@ const Attributes MutuallyIncompatible[4] = {
 };
 
 /// @brief Which attributes cannot be applied to a type.
-Attributes typeIncompatible(const Type *Ty);
+Attributes typeIncompatible(Type *Ty);
 
 /// This turns an int alignment (a power of 2, normally) into the
 /// form used internally in Attributes.
diff --git a/contrib/llvm/include/llvm/AutoUpgrade.h b/contrib/llvm/include/llvm/AutoUpgrade.h
index 5ce20b69e2fb..8ca3548f533e 100644
--- a/contrib/llvm/include/llvm/AutoUpgrade.h
+++ b/contrib/llvm/include/llvm/AutoUpgrade.h
@@ -43,6 +43,10 @@ namespace llvm {
   /// This function checks debug info intrinsics. If an intrinsic is invalid
   /// then this function simply removes the intrinsic. 
   void CheckDebugInfoIntrinsics(Module *M);
+  
+  /// This function upgrades the old pre-3.0 exception handling system to the
+  /// new one. N.B. This will be removed in 3.1.
+  void UpgradeExceptionHandling(Module *M);
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/include/llvm/BasicBlock.h b/contrib/llvm/include/llvm/BasicBlock.h
index 3b953c06a81b..1cd8dc55ab58 100644
--- a/contrib/llvm/include/llvm/BasicBlock.h
+++ b/contrib/llvm/include/llvm/BasicBlock.h
@@ -22,6 +22,7 @@
 
 namespace llvm {
 
+class LandingPadInst;
 class TerminatorInst;
 class LLVMContext;
 class BlockAddress;
@@ -144,6 +145,14 @@ class BasicBlock : public Value, // Basic blocks are data objects also
     return const_cast<BasicBlock*>(this)->getFirstNonPHIOrDbgOrLifetime();
   }
 
+  /// getFirstInsertionPt - Returns an iterator to the first instruction in this
+  /// block that is suitable for inserting a non-PHI instruction. In particular,
+  /// it skips all PHIs and LandingPad instructions.
+  iterator getFirstInsertionPt();
+  const_iterator getFirstInsertionPt() const {
+    return const_cast<BasicBlock*>(this)->getFirstInsertionPt();
+  }
+
   /// removeFromParent - This method unlinks 'this' from the containing
   /// function, but does not delete it.
   ///
@@ -258,6 +267,14 @@ class BasicBlock : public Value, // Basic blocks are data objects also
   /// to refer to basic block New instead of to us.
   void replaceSuccessorsPhiUsesWith(BasicBlock *New);
 
+  /// isLandingPad - Return true if this basic block is a landing pad. I.e.,
+  /// it's the destination of the 'unwind' edge of an invoke instruction.
+  bool isLandingPad() const;
+
+  /// getLandingPadInst() - Return the landingpad instruction associated with
+  /// the landing pad.
+  LandingPadInst *getLandingPadInst();
+
 private:
   /// AdjustBlockAddressRefCount - BasicBlock stores the number of BlockAddress
   /// objects using it.  This is almost always 0, sometimes one, possibly but
diff --git a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index df68bd5ddd39..4b0dcc36232f 100644
--- a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -205,6 +205,23 @@ namespace bitc {
     BINOP_XOR  = 12
   };
 
+  /// These are values used in the bitcode files to encode AtomicRMW operations.
+  /// The values of these enums have no fixed relation to the LLVM IR enum
+  /// values.  Changing these will break compatibility with old files.
+  enum RMWOperations {
+    RMW_XCHG = 0,
+    RMW_ADD = 1,
+    RMW_SUB = 2,
+    RMW_AND = 3,
+    RMW_NAND = 4,
+    RMW_OR = 5,
+    RMW_XOR = 6,
+    RMW_MAX = 7,
+    RMW_MIN = 8,
+    RMW_UMAX = 9,
+    RMW_UMIN = 10
+  };
+
   /// OverflowingBinaryOperatorOptionalFlags - Flags for serializing
   /// OverflowingBinaryOperator's SubclassOptionalData contents.
   enum OverflowingBinaryOperatorOptionalFlags {
@@ -218,6 +235,23 @@ namespace bitc {
     PEO_EXACT = 0
   };
 
+  /// Encoded AtomicOrdering values.
+  enum AtomicOrderingCodes {
+    ORDERING_NOTATOMIC = 0,
+    ORDERING_UNORDERED = 1,
+    ORDERING_MONOTONIC = 2,
+    ORDERING_ACQUIRE = 3,
+    ORDERING_RELEASE = 4,
+    ORDERING_ACQREL = 5,
+    ORDERING_SEQCST = 6
+  };
+
+  /// Encoded SynchronizationScope values.
+  enum AtomicSynchScopeCodes {
+    SYNCHSCOPE_SINGLETHREAD = 0,
+    SYNCHSCOPE_CROSSTHREAD = 1
+  };
+
   // The function body block (FUNCTION_BLOCK_ID) describes function bodies.  It
   // can contain a constant block (CONSTANTS_BLOCK_ID).
   enum FunctionCodes {
@@ -266,7 +300,19 @@ namespace bitc {
 
     FUNC_CODE_INST_CALL        = 34, // CALL:       [attr, fnty, fnid, args...]
 
-    FUNC_CODE_DEBUG_LOC        = 35  // DEBUG_LOC:  [Line,Col,ScopeVal, IAVal]
+    FUNC_CODE_DEBUG_LOC        = 35, // DEBUG_LOC:  [Line,Col,ScopeVal, IAVal]
+    FUNC_CODE_INST_FENCE       = 36, // FENCE: [ordering, synchscope]
+    FUNC_CODE_INST_CMPXCHG     = 37, // CMPXCHG: [ptrty,ptr,cmp,new, align, vol,
+                                     //           ordering, synchscope]
+    FUNC_CODE_INST_ATOMICRMW   = 38, // ATOMICRMW: [ptrty,ptr,val, operation,
+                                     //             align, vol,
+                                     //             ordering, synchscope]
+    FUNC_CODE_INST_RESUME      = 39, // RESUME:     [opval]
+    FUNC_CODE_INST_LANDINGPAD  = 40, // LANDINGPAD: [ty,val,val,num,id0,val0...]
+    FUNC_CODE_INST_LOADATOMIC  = 41, // LOAD: [opty, op, align, vol,
+                                     //        ordering, synchscope]
+    FUNC_CODE_INST_STOREATOMIC = 42  // STORE: [ptrty,ptr,val, align, vol
+                                     //         ordering, synchscope]
   };
 } // End bitc namespace
 } // End llvm namespace
diff --git a/contrib/llvm/include/llvm/CodeGen/Analysis.h b/contrib/llvm/include/llvm/CodeGen/Analysis.h
index f8a70293c155..d8e64071a1d9 100644
--- a/contrib/llvm/include/llvm/CodeGen/Analysis.h
+++ b/contrib/llvm/include/llvm/CodeGen/Analysis.h
@@ -33,12 +33,12 @@ class SelectionDAG;
 /// of insertvalue or extractvalue indices that identify a member, return
 /// the linearized index of the start of the member.
 ///
-unsigned ComputeLinearIndex(const Type *Ty,
+unsigned ComputeLinearIndex(Type *Ty,
                             const unsigned *Indices,
                             const unsigned *IndicesEnd,
                             unsigned CurIndex = 0);
 
-inline unsigned ComputeLinearIndex(const Type *Ty,
+inline unsigned ComputeLinearIndex(Type *Ty,
                                    ArrayRef<unsigned> Indices,
                                    unsigned CurIndex = 0) {
   return ComputeLinearIndex(Ty, Indices.begin(), Indices.end(), CurIndex);
@@ -51,7 +51,7 @@ inline unsigned ComputeLinearIndex(const Type *Ty,
 /// If Offsets is non-null, it points to a vector to be filled in
 /// with the in-memory offsets of each of the individual values.
 ///
-void ComputeValueVTs(const TargetLowering &TLI, const Type *Ty,
+void ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
                      SmallVectorImpl<EVT> &ValueVTs,
                      SmallVectorImpl<uint64_t> *Offsets = 0,
                      uint64_t StartingOffset = 0);
diff --git a/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h b/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h
index 60edcc584559..2f76a6cc5583 100644
--- a/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h
+++ b/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h
@@ -49,11 +49,6 @@ namespace llvm {
                    const MachineLoopInfo &loops) :
       MF(mf), LIS(lis), Loops(loops) {}
 
-    /// CalculateRegClass - recompute the register class for reg from its uses.
-    /// Since the register class can affect the allocation hint, this function
-    /// should be called before CalculateWeightAndHint if both are called.
-    void CalculateRegClass(unsigned reg);
-
     /// CalculateWeightAndHint - (re)compute li's spill weight and allocation
     /// hint.
     void CalculateWeightAndHint(LiveInterval &li);
diff --git a/contrib/llvm/include/llvm/CodeGen/FastISel.h b/contrib/llvm/include/llvm/CodeGen/FastISel.h
index 962a4e263538..18202d93b460 100644
--- a/contrib/llvm/include/llvm/CodeGen/FastISel.h
+++ b/contrib/llvm/include/llvm/CodeGen/FastISel.h
@@ -54,8 +54,18 @@ class FastISel {
   const TargetInstrInfo &TII;
   const TargetLowering &TLI;
   const TargetRegisterInfo &TRI;
+
+  /// The position of the last instruction for materializing constants
+  /// for use in the current block. It resets to EmitStartPt when it
+  /// makes sense (for example, it's usually profitable to avoid function
+  /// calls between the definition and the use)
   MachineInstr *LastLocalValue;
 
+  /// The top most instruction in the current block that is allowed for
+  /// emitting local variables. LastLocalValue resets to EmitStartPt when
+  /// it makes sense (for example, on function calls)
+  MachineInstr *EmitStartPt;
+
 public:
   /// getLastLocalValue - Return the position of the last instruction
   /// emitted for materializing constants for use in the current block.
@@ -63,7 +73,10 @@ class FastISel {
 
   /// setLastLocalValue - Update the position of the last instruction
   /// emitted for materializing constants for use in the current block.
-  void setLastLocalValue(MachineInstr *I) { LastLocalValue = I; }
+  void setLastLocalValue(MachineInstr *I) {
+    EmitStartPt = I;
+    LastLocalValue = I;
+  }
 
   /// startNewBlock - Set the current block to which generated machine
   /// instructions will be appended, and clear the local CSE map.
@@ -358,6 +371,11 @@ class FastISel {
   /// be materialized with new instructions.
   unsigned materializeRegForValue(const Value *V, MVT VT);
 
+  /// flushLocalValueMap - clears LocalValueMap and moves the area for the
+  /// new local variables to the beginning of the block. It helps to avoid
+  /// spilling cached variables across heavy instructions like calls.
+  void flushLocalValueMap();
+
   /// hasTrivialKill - Test whether the given value has exactly one use.
   bool hasTrivialKill(const Value *V) const;
 };
diff --git a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
index 84bbf480473a..09dac8547a62 100644
--- a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -19,6 +19,7 @@
 #include "llvm/Instructions.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallVector.h"
 #ifndef NDEBUG
@@ -139,7 +140,7 @@ class FunctionLoweringInfo {
 
   unsigned CreateReg(EVT VT);
   
-  unsigned CreateRegs(const Type *Ty);
+  unsigned CreateRegs(Type *Ty);
   
   unsigned InitializeRegForValue(const Value *V) {
     unsigned &R = ValueMap[V];
@@ -198,12 +199,12 @@ class FunctionLoweringInfo {
     LiveOutRegInfo[Reg].IsValid = false;
   }
 
-  /// setByValArgumentFrameIndex - Record frame index for the byval
+  /// setArgumentFrameIndex - Record frame index for the byval
   /// argument.
-  void setByValArgumentFrameIndex(const Argument *A, int FI);
+  void setArgumentFrameIndex(const Argument *A, int FI);
   
-  /// getByValArgumentFrameIndex - Get frame index for the byval argument.
-  int getByValArgumentFrameIndex(const Argument *A);
+  /// getArgumentFrameIndex - Get frame index for the byval argument.
+  int getArgumentFrameIndex(const Argument *A);
 
 private:
   /// LiveOutRegInfo - Information about live out vregs.
@@ -220,6 +221,11 @@ void AddCatchInfo(const CallInst &I,
 void CopyCatchInfo(const BasicBlock *SuccBB, const BasicBlock *LPad,
                    MachineModuleInfo *MMI, FunctionLoweringInfo &FLI);
 
+/// AddLandingPadInfo - Extract the exception handling information from the
+/// landingpad instruction and add them to the specified machine module info.
+void AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI,
+                       MachineBasicBlock *MBB);
+
 } // end namespace llvm
 
 #endif
diff --git a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 459cecda21e0..184e96dc4766 100644
--- a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -95,7 +95,7 @@ namespace ISD {
     // execution to HANDLER. Many platform-related details also :)
     EH_RETURN,
 
-    // OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer)
+    // RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer)
     // This corresponds to the eh.sjlj.setjmp intrinsic.
     // It takes an input chain and a pointer to the jump buffer as inputs
     // and returns an outchain.
@@ -323,6 +323,12 @@ namespace ISD {
     // i1 then the high bits must conform to getBooleanContents.
     SELECT,
 
+    // Select with a vector condition (op #0) and two vector operands (ops #1
+    // and #2), returning a vector result.  All vectors have the same length.
+    // Much like the scalar select and setcc, each bit in the condition selects
+    // whether the corresponding result element is taken from op #1 or op #2.
+    VSELECT,
+
     // Select with condition operator - This selects between a true value and
     // a false value (ops #2 and #3) based on the boolean result of comparing
     // the lhs and rhs (ops #0 and #1) of a conditional expression with the
@@ -333,16 +339,10 @@ namespace ISD {
     // true.  If the result value type is not i1 then the high bits conform
     // to getBooleanContents.  The operands to this are the left and right
     // operands to compare (ops #0, and #1) and the condition code to compare
-    // them with (op #2) as a CondCodeSDNode.
+    // them with (op #2) as a CondCodeSDNode. If the operands are vector types
+    // then the result type must also be a vector type.
     SETCC,
 
-    // RESULT = VSETCC(LHS, RHS, COND) operator - This evaluates to a vector of
-    // integer elements with all bits of the result elements set to true if the
-    // comparison is true or all cleared if the comparison is false.  The
-    // operands to this are the left and right operands to compare (LHS/RHS) and
-    // the condition code to compare them with (COND) as a CondCodeSDNode.
-    VSETCC,
-
     // SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded
     // integer shift operations, just like ADD/SUB_PARTS.  The operation
     // ordering is:
@@ -566,14 +566,19 @@ namespace ISD {
     // HANDLENODE node - Used as a handle for various purposes.
     HANDLENODE,
 
-    // TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
-    // It takes as input a token chain, the pointer to the trampoline,
-    // the pointer to the nested function, the pointer to pass for the
-    // 'nest' parameter, a SRCVALUE for the trampoline and another for
-    // the nested function (allowing targets to access the original
-    // Function*).  It produces the result of the intrinsic and a token
-    // chain as output.
-    TRAMPOLINE,
+    // INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.  It
+    // takes as input a token chain, the pointer to the trampoline, the pointer
+    // to the nested function, the pointer to pass for the 'nest' parameter, a
+    // SRCVALUE for the trampoline and another for the nested function (allowing
+    // targets to access the original Function*).  It produces a token chain as
+    // output.
+    INIT_TRAMPOLINE,
+
+    // ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
+    // It takes a pointer to the trampoline and produces a (possibly) new
+    // pointer to the same trampoline with platform-specific adjustments
+    // applied.  The pointer it returns points to an executable block of code.
+    ADJUST_TRAMPOLINE,
 
     // TRAP - Trapping instruction
     TRAP,
@@ -592,22 +597,27 @@ namespace ISD {
     // and produces an output chain.
     MEMBARRIER,
 
+    // OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope)
+    // This corresponds to the fence instruction. It takes an input chain, and
+    // two integer constants: an AtomicOrdering and a SynchronizationScope.
+    ATOMIC_FENCE,
+
+    // Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr)
+    // This corresponds to "load atomic" instruction.
+    ATOMIC_LOAD,
+
+    // OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)
+    // This corresponds to "store atomic" instruction.
+    ATOMIC_STORE,
+
     // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
-    // this corresponds to the atomic.lcs intrinsic.
-    // cmp is compared to *ptr, and if equal, swap is stored in *ptr.
-    // the return is always the original value in *ptr
+    // This corresponds to the cmpxchg instruction.
     ATOMIC_CMP_SWAP,
 
     // Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
-    // this corresponds to the atomic.swap intrinsic.
-    // amt is stored to *ptr atomically.
-    // the return is always the original value in *ptr
-    ATOMIC_SWAP,
-
     // Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
-    // this corresponds to the atomic.load.[OpName] intrinsic.
-    // op(*ptr, amt) is stored to *ptr atomically.
-    // the return is always the original value in *ptr
+    // These correspond to the atomicrmw instruction.
+    ATOMIC_SWAP,
     ATOMIC_LOAD_ADD,
     ATOMIC_LOAD_SUB,
     ATOMIC_LOAD_AND,
diff --git a/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h b/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h
new file mode 100644
index 000000000000..0271c5d85222
--- /dev/null
+++ b/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h
@@ -0,0 +1,248 @@
+//===- LexicalScopes.cpp - Collecting lexical scope info -*- C++ -*--------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements LexicalScopes analysis.
+//
+// This pass collects lexical scope information and maps machine instructions
+// to respective lexical scopes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_LEXICALSCOPES_H
+#define LLVM_CODEGEN_LEXICALSCOPES_H
+
+#include "llvm/Metadata.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/DebugLoc.h"
+#include "llvm/Support/ValueHandle.h"
+#include <utility>
+namespace llvm {
+
+class MachineInstr;
+class MachineBasicBlock;
+class MachineFunction;
+class LexicalScope;
+
+//===----------------------------------------------------------------------===//
+/// InsnRange - This is used to track range of instructions with identical
+/// lexical scope.
+///
+typedef std::pair<const MachineInstr *, const MachineInstr *> InsnRange;
+
+//===----------------------------------------------------------------------===//
+/// LexicalScopes -  This class provides interface to collect and use lexical
+/// scoping information from machine instruction.
+///
+class LexicalScopes {
+public:
+  LexicalScopes() : MF(NULL),  CurrentFnLexicalScope(NULL) { }
+  virtual ~LexicalScopes();
+
+  /// initialize - Scan machine function and constuct lexical scope nest.
+  virtual void initialize(const MachineFunction &);
+
+  /// releaseMemory - release memory.
+  virtual void releaseMemory();
+  
+  /// empty - Return true if there is any lexical scope information available.
+  bool empty() { return CurrentFnLexicalScope == NULL; }
+
+  /// isCurrentFunctionScope - Return true if given lexical scope represents 
+  /// current function.
+  bool isCurrentFunctionScope(const LexicalScope *LS) { 
+    return LS == CurrentFnLexicalScope;
+  }
+
+  /// getCurrentFunctionScope - Return lexical scope for the current function.
+  LexicalScope *getCurrentFunctionScope() const { return CurrentFnLexicalScope;}
+
+  /// getMachineBasicBlocks - Populate given set using machine basic blocks
+  /// which have machine instructions that belong to lexical scope identified by
+  /// DebugLoc.
+  void getMachineBasicBlocks(DebugLoc DL,
+                             SmallPtrSet<const MachineBasicBlock*, 4> &MBBs);
+
+  /// dominates - Return true if DebugLoc's lexical scope dominates at least one
+  /// machine instruction's lexical scope in a given machine basic block.
+  bool dominates(DebugLoc DL, MachineBasicBlock *MBB);
+
+  /// findLexicalScope - Find lexical scope, either regular or inlined, for the
+  /// given DebugLoc. Return NULL if not found.
+  LexicalScope *findLexicalScope(DebugLoc DL);
+
+  /// getAbstractScopesList - Return a reference to list of abstract scopes.
+  ArrayRef<LexicalScope *> getAbstractScopesList() const {
+    return AbstractScopesList;
+  }
+
+  /// findAbstractScope - Find an abstract scope or return NULL.
+  LexicalScope *findAbstractScope(const MDNode *N) {
+    return AbstractScopeMap.lookup(N);
+  }
+
+  /// findInlinedScope - Find an inlined scope for the given DebugLoc or return
+  /// NULL.
+  LexicalScope *findInlinedScope(DebugLoc DL) {
+    return InlinedLexicalScopeMap.lookup(DL);
+  }
+
+  /// findLexicalScope - Find regular lexical scope or return NULL.
+  LexicalScope *findLexicalScope(const MDNode *N) {
+    return LexicalScopeMap.lookup(N);
+  }
+
+  /// dump - Print data structures to dbgs().
+  void dump();
+
+private:
+
+  /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If
+  /// not available then create new lexical scope.
+  LexicalScope *getOrCreateLexicalScope(DebugLoc DL);
+
+  /// getOrCreateRegularScope - Find or create a regular lexical scope.
+  LexicalScope *getOrCreateRegularScope(MDNode *Scope);
+
+  /// getOrCreateInlinedScope - Find or create an inlined lexical scope.
+  LexicalScope *getOrCreateInlinedScope(MDNode *Scope, MDNode *InlinedAt);
+
+  /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
+  LexicalScope *getOrCreateAbstractScope(const MDNode *N);
+
+  /// extractLexicalScopes - Extract instruction ranges for each lexical scopes
+  /// for the given machine function.
+  void extractLexicalScopes(SmallVectorImpl<InsnRange> &MIRanges,
+                            DenseMap<const MachineInstr *, LexicalScope *> &M);
+  void constructScopeNest(LexicalScope *Scope);
+  void assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges,
+                             DenseMap<const MachineInstr *, LexicalScope *> &M);
+
+private:
+  const MachineFunction *MF;
+
+  /// LexicalScopeMap - Tracks the scopes in the current function.  Owns the
+  /// contained LexicalScope*s.
+  DenseMap<const MDNode *, LexicalScope *> LexicalScopeMap;
+
+  /// InlinedLexicalScopeMap - Tracks inlined function scopes in current function.
+  DenseMap<DebugLoc, LexicalScope *> InlinedLexicalScopeMap;
+
+  /// AbstractScopeMap - These scopes are  not included LexicalScopeMap.  
+  /// AbstractScopes owns its LexicalScope*s.
+  DenseMap<const MDNode *, LexicalScope *> AbstractScopeMap;
+
+  /// AbstractScopesList - Tracks abstract scopes constructed while processing
+  /// a function. 
+  SmallVector<LexicalScope *, 4>AbstractScopesList;
+
+  /// CurrentFnLexicalScope - Top level scope for the current function.
+  ///
+  LexicalScope *CurrentFnLexicalScope;
+};
+
+//===----------------------------------------------------------------------===//
+/// LexicalScope - This class is used to track scope information.
+///
+class LexicalScope {
+
+public:
+  LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A)
+    : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A),
+      LastInsn(0), FirstInsn(0), DFSIn(0), DFSOut(0), IndentLevel(0) {
+    if (Parent)
+      Parent->addChild(this);
+  }
+
+  virtual ~LexicalScope() {}
+
+  // Accessors.
+  LexicalScope *getParent() const               { return Parent; }
+  const MDNode *getDesc() const                 { return Desc; }
+  const MDNode *getInlinedAt() const            { return InlinedAtLocation; }
+  const MDNode *getScopeNode() const            { return Desc; }
+  bool isAbstractScope() const                  { return AbstractScope; }
+  SmallVector<LexicalScope *, 4> &getChildren() { return Children; }
+  SmallVector<InsnRange, 4> &getRanges()        { return Ranges; }
+
+  /// addChild - Add a child scope.
+  void addChild(LexicalScope *S) { Children.push_back(S); }
+
+  /// openInsnRange - This scope covers instruction range starting from MI.
+  void openInsnRange(const MachineInstr *MI) {
+    if (!FirstInsn)
+      FirstInsn = MI;
+
+    if (Parent)
+      Parent->openInsnRange(MI);
+  }
+
+  /// extendInsnRange - Extend the current instruction range covered by
+  /// this scope.
+  void extendInsnRange(const MachineInstr *MI) {
+    assert (FirstInsn && "MI Range is not open!");
+    LastInsn = MI;
+    if (Parent)
+      Parent->extendInsnRange(MI);
+  }
+
+  /// closeInsnRange - Create a range based on FirstInsn and LastInsn collected
+  /// until now. This is used when a new scope is encountered while walking
+  /// machine instructions.
+  void closeInsnRange(LexicalScope *NewScope = NULL) {
+    assert (LastInsn && "Last insn missing!");
+    Ranges.push_back(InsnRange(FirstInsn, LastInsn));
+    FirstInsn = NULL;
+    LastInsn = NULL;
+    // If Parent dominates NewScope then do not close Parent's instruction
+    // range.
+    if (Parent && (!NewScope || !Parent->dominates(NewScope)))
+      Parent->closeInsnRange(NewScope);
+  }
+
+  /// dominates - Return true if current scope dominsates given lexical scope.
+  bool dominates(const LexicalScope *S) const {
+    if (S == this)
+      return true;
+    if (DFSIn < S->getDFSIn() && DFSOut > S->getDFSOut())
+      return true;
+    return false;
+  }
+
+  // Depth First Search support to walk and manipulate LexicalScope hierarchy.
+  unsigned getDFSOut() const            { return DFSOut; }
+  void setDFSOut(unsigned O)            { DFSOut = O; }
+  unsigned getDFSIn() const             { return DFSIn; }
+  void setDFSIn(unsigned I)             { DFSIn = I; }
+
+  /// dump - print lexical scope.
+  void dump() const;
+
+private:
+  LexicalScope *Parent;                          // Parent to this scope.
+  AssertingVH<const MDNode> Desc;                // Debug info descriptor.
+  AssertingVH<const MDNode> InlinedAtLocation;   // Location at which this 
+                                                 // scope is inlined.
+  bool AbstractScope;                            // Abstract Scope
+  SmallVector<LexicalScope *, 4> Children;       // Scopes defined in scope.  
+                                                 // Contents not owned.
+  SmallVector<InsnRange, 4> Ranges;
+
+  const MachineInstr *LastInsn;       // Last instruction of this scope.
+  const MachineInstr *FirstInsn;      // First instruction of this scope.
+  unsigned DFSIn, DFSOut;             // In & Out Depth use to determine
+                                      // scope nesting.
+  mutable unsigned IndentLevel;       // Private state for dump()
+};
+
+} // end llvm namespace
+
+#endif
diff --git a/contrib/llvm/include/llvm/CodeGen/LiveInterval.h b/contrib/llvm/include/llvm/CodeGen/LiveInterval.h
index 5fd4d3df2666..2288c1a98b2d 100644
--- a/contrib/llvm/include/llvm/CodeGen/LiveInterval.h
+++ b/contrib/llvm/include/llvm/CodeGen/LiveInterval.h
@@ -100,6 +100,7 @@ namespace llvm {
     bool isDefByCopy() const { return copy != 0; }
 
     /// Returns true if one or more kills are PHI nodes.
+    /// Obsolete, do not use!
     bool hasPHIKill() const { return flags & HAS_PHI_KILL; }
     /// Set the PHI kill flag on this value.
     void setHasPHIKill(bool hasKill) {
@@ -313,7 +314,6 @@ namespace llvm {
 
     /// RenumberValues - Renumber all values in order of appearance and remove
     /// unused values.
-    /// Recalculate phi-kill flags in case any phi-def values were removed.
     void RenumberValues(LiveIntervals &lis);
 
     /// isOnlyLROfValNo - Return true if the specified live range is the only
@@ -411,6 +411,14 @@ namespace llvm {
       return I == end() ? 0 : I->valno;
     }
 
+    /// getVNInfoBefore - Return the VNInfo that is live up to but not
+    /// necessarilly including Idx, or NULL. Use this to find the reaching def
+    /// used by an instruction at this SlotIndex position.
+    VNInfo *getVNInfoBefore(SlotIndex Idx) const {
+      const_iterator I = FindLiveRangeContaining(Idx.getPrevSlot());
+      return I == end() ? 0 : I->valno;
+    }
+
     /// FindLiveRangeContaining - Return an iterator to the live range that
     /// contains the specified index, or end() if there is none.
     iterator FindLiveRangeContaining(SlotIndex Idx) {
@@ -452,10 +460,10 @@ namespace llvm {
       addRangeFrom(LR, ranges.begin());
     }
 
-    /// extendInBlock - If this interval is live before UseIdx in the basic
-    /// block that starts at StartIdx, extend it to be live at UseIdx and return
-    /// the value. If there is no live range before UseIdx, return NULL.
-    VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex UseIdx);
+    /// extendInBlock - If this interval is live before Kill in the basic block
+    /// that starts at StartIdx, extend it to be live up to Kill, and return
+    /// the value. If there is no live range before Kill, return NULL.
+    VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Kill);
 
     /// join - Join two live intervals (this, and other) together.  This applies
     /// mappings to the value numbers in the LHS/RHS intervals as specified.  If
diff --git a/contrib/llvm/include/llvm/CodeGen/LiveStackAnalysis.h b/contrib/llvm/include/llvm/CodeGen/LiveStackAnalysis.h
index 8a8dcaf5728f..86c4d7c11067 100644
--- a/contrib/llvm/include/llvm/CodeGen/LiveStackAnalysis.h
+++ b/contrib/llvm/include/llvm/CodeGen/LiveStackAnalysis.h
@@ -25,6 +25,8 @@
 namespace llvm {
 
   class LiveStacks : public MachineFunctionPass {
+    const TargetRegisterInfo *TRI;
+
     /// Special pool allocator for VNInfo's (LiveInterval val#).
     ///
     VNInfo::Allocator VNInfoAllocator;
diff --git a/contrib/llvm/include/llvm/CodeGen/LiveVariables.h b/contrib/llvm/include/llvm/CodeGen/LiveVariables.h
index f9b81b1ea7d6..7ba901fc28a4 100644
--- a/contrib/llvm/include/llvm/CodeGen/LiveVariables.h
+++ b/contrib/llvm/include/llvm/CodeGen/LiveVariables.h
@@ -231,6 +231,7 @@ class LiveVariables : public MachineFunctionPass {
     }
 
     assert(Removed && "Register is not used by this instruction!");
+    (void)Removed;
     return true;
   }
 
@@ -265,6 +266,7 @@ class LiveVariables : public MachineFunctionPass {
       }
     }
     assert(Removed && "Register is not defined by this instruction!");
+    (void)Removed;
     return true;
   }
   
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index 397e59ef18b9..5a20e952b9cc 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -232,7 +232,7 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
 
   /// setIsLandingPad - Indicates the block is a landing pad.  That is
   /// this basic block is entered via an exception handler.
-  void setIsLandingPad() { IsLandingPad = true; }
+  void setIsLandingPad(bool V = true) { IsLandingPad = V; }
 
   /// getLandingPadSuccessor - If this block has a successor that is a landing
   /// pad, return it. Otherwise return NULL.
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequency.h b/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
similarity index 60%
rename from contrib/llvm/include/llvm/CodeGen/MachineBlockFrequency.h
rename to contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index 25bf1f08dc64..416d40bf3098 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequency.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -1,4 +1,4 @@
-//====----- MachineBlockFrequency.h - MachineBlock Frequency Analysis ----====//
+//====----- MachineBlockFrequencyInfo.h - MachineBlock Frequency Analysis ----====//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -11,10 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CODEGEN_MACHINEBLOCKFREQUENCY_H
-#define LLVM_CODEGEN_MACHINEBLOCKFREQUENCY_H
+#ifndef LLVM_CODEGEN_MACHINEBLOCKFREQUENCYINFO_H
+#define LLVM_CODEGEN_MACHINEBLOCKFREQUENCYINFO_H
 
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Support/BlockFrequency.h"
 #include <climits>
 
 namespace llvm {
@@ -23,29 +24,29 @@ class MachineBranchProbabilityInfo;
 template<class BlockT, class FunctionT, class BranchProbInfoT>
 class BlockFrequencyImpl;
 
-/// MachineBlockFrequency pass uses BlockFrequencyImpl implementation to estimate
+/// MachineBlockFrequencyInfo pass uses BlockFrequencyImpl implementation to estimate
 /// machine basic block frequencies.
-class MachineBlockFrequency : public MachineFunctionPass {
+class MachineBlockFrequencyInfo : public MachineFunctionPass {
 
   BlockFrequencyImpl<MachineBasicBlock, MachineFunction, MachineBranchProbabilityInfo> *MBFI;
 
 public:
   static char ID;
 
-  MachineBlockFrequency();
+  MachineBlockFrequencyInfo();
 
-  ~MachineBlockFrequency();
+  ~MachineBlockFrequencyInfo();
 
   void getAnalysisUsage(AnalysisUsage &AU) const;
 
   bool runOnMachineFunction(MachineFunction &F);
 
-  /// getblockFreq - Return block frequency. Never return 0, value must be
-  /// positive. Please note that initial frequency is equal to 1024. It means
+  /// getblockFreq - Return block frequency. Return 0 if we don't have the
+  /// information. Please note that initial frequency is equal to 1024. It means
   /// that we should not rely on the value itself, but only on the comparison to
-  /// the other block frequencies. We do this to avoid using of the floating
-  /// points.
-  uint32_t getBlockFreq(MachineBasicBlock *MBB);
+  /// the other block frequencies. We do this to avoid using of floating points.
+  ///
+  BlockFrequency getBlockFreq(MachineBasicBlock *MBB) const;
 };
 
 }
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineConstantPool.h b/contrib/llvm/include/llvm/CodeGen/MachineConstantPool.h
index beb16a2824d7..29f4f443bf7b 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineConstantPool.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineConstantPool.h
@@ -34,15 +34,15 @@ class raw_ostream;
 /// Abstract base class for all machine specific constantpool value subclasses.
 ///
 class MachineConstantPoolValue {
-  const Type *Ty;
+  Type *Ty;
 
 public:
-  explicit MachineConstantPoolValue(const Type *ty) : Ty(ty) {}
+  explicit MachineConstantPoolValue(Type *ty) : Ty(ty) {}
   virtual ~MachineConstantPoolValue() {}
 
   /// getType - get type of this MachineConstantPoolValue.
   ///
-  const Type *getType() const { return Ty; }
+  Type *getType() const { return Ty; }
 
   
   /// getRelocationInfo - This method classifies the entry according to
@@ -54,7 +54,7 @@ class MachineConstantPoolValue {
   virtual int getExistingMachineCPValue(MachineConstantPool *CP,
                                         unsigned Alignment) = 0;
 
-  virtual void AddSelectionDAGCSEId(FoldingSetNodeID &ID) = 0;
+  virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID) = 0;
 
   /// print - Implement operator<<
   virtual void print(raw_ostream &O) const = 0;
@@ -104,7 +104,7 @@ class MachineConstantPoolEntry {
     return Alignment & ~(1 << (sizeof(unsigned)*CHAR_BIT-1));
   }
 
-  const Type *getType() const;
+  Type *getType() const;
   
   /// getRelocationInfo - This method classifies the entry according to
   /// whether or not it may generate a relocation entry.  This must be
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h
index 4ea6aa3396a9..b347ca8e680a 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h
@@ -174,6 +174,10 @@ class MachineFrameInfo {
   /// StackProtectorIdx - The frame index for the stack protector.
   int StackProtectorIdx;
 
+  /// FunctionContextIdx - The frame index for the function context. Used for
+  /// SjLj exceptions.
+  int FunctionContextIdx;
+
   /// MaxCallFrameSize - This contains the size of the largest call frame if the
   /// target uses frame setup/destroy pseudo instructions (as defined in the
   /// TargetFrameInfo class).  This information is important for frame pointer
@@ -220,6 +224,7 @@ class MachineFrameInfo {
     AdjustsStack = false;
     HasCalls = false;
     StackProtectorIdx = -1;
+    FunctionContextIdx = -1;
     MaxCallFrameSize = 0;
     CSIValid = false;
     LocalFrameSize = 0;
@@ -244,6 +249,11 @@ class MachineFrameInfo {
   int getStackProtectorIndex() const { return StackProtectorIdx; }
   void setStackProtectorIndex(int I) { StackProtectorIdx = I; }
 
+  /// getFunctionContextIndex/setFunctionContextIndex - Return the index for the
+  /// function context object. This object is used for SjLj exceptions.
+  int getFunctionContextIndex() const { return FunctionContextIdx; }
+  void setFunctionContextIndex(int I) { FunctionContextIdx = I; }
+
   /// isFrameAddressTaken - This method may be called any time after instruction
   /// selection is complete to determine if there is a call to
   /// \@llvm.frameaddress in this function.
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h
index 5b3d3ea62a45..cae38f34709d 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -32,6 +32,7 @@ namespace llvm {
 template <typename T> class SmallVectorImpl;
 class AliasAnalysis;
 class TargetInstrInfo;
+class TargetRegisterClass;
 class TargetRegisterInfo;
 class MachineFunction;
 class MachineMemOperand;
@@ -58,8 +59,6 @@ class MachineInstr : public ilist_node<MachineInstr> {
   };
 private:
   const MCInstrDesc *MCID;              // Instruction descriptor.
-  uint16_t NumImplicitOps;              // Number of implicit operands (which
-                                        // are determined at construction time).
 
   uint8_t Flags;                        // Various bits of additional
                                         // information about machine
@@ -78,9 +77,6 @@ class MachineInstr : public ilist_node<MachineInstr> {
   MachineBasicBlock *Parent;            // Pointer to the owning basic block.
   DebugLoc debugLoc;                    // Source line information.
 
-  // OperandComplete - Return true if it's illegal to add a new operand
-  bool OperandsComplete() const;
-
   MachineInstr(const MachineInstr&);   // DO NOT IMPLEMENT
   void operator=(const MachineInstr&); // DO NOT IMPLEMENT
 
@@ -393,6 +389,30 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// none is found.
   int findFirstPredOperandIdx() const;
 
+  /// findInlineAsmFlagIdx() - Find the index of the flag word operand that
+  /// corresponds to operand OpIdx on an inline asm instruction.  Returns -1 if
+  /// getOperand(OpIdx) does not belong to an inline asm operand group.
+  ///
+  /// If GroupNo is not NULL, it will receive the number of the operand group
+  /// containing OpIdx.
+  ///
+  /// The flag operand is an immediate that can be decoded with methods like
+  /// InlineAsm::hasRegClassConstraint().
+  ///
+  int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = 0) const;
+
+  /// getRegClassConstraint - Compute the static register class constraint for
+  /// operand OpIdx.  For normal instructions, this is derived from the
+  /// MCInstrDesc.  For inline assembly it is derived from the flag words.
+  ///
+  /// Returns NULL if the static register classs constraint cannot be
+  /// determined.
+  ///
+  const TargetRegisterClass*
+  getRegClassConstraint(unsigned OpIdx,
+                        const TargetInstrInfo *TII,
+                        const TargetRegisterInfo *TRI) const;
+
   /// isRegTiedToUseOperand - Given the index of a register def operand,
   /// check if the register def is tied to a source operand, due to either
   /// two-address elimination or inline assembly constraints. Returns the
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h
index fa185c46d5e3..2bf7f1788f8a 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h
@@ -34,7 +34,7 @@
 #include "llvm/Pass.h"
 #include "llvm/GlobalValue.h"
 #include "llvm/Metadata.h"
-#include "llvm/CodeGen/MachineLocation.h"
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/DebugLoc.h"
@@ -107,36 +107,42 @@ class MachineModuleInfo : public ImmutablePass {
   /// want.
   MachineModuleInfoImpl *ObjFileMMI;
 
-  // FrameMoves - List of moves done by a function's prolog.  Used to construct
-  // frame maps by debug and exception handling consumers.
+  /// FrameMoves - List of moves done by a function's prolog.  Used to construct
+  /// frame maps by debug and exception handling consumers.
   std::vector<MachineMove> FrameMoves;
 
-  // LandingPads - List of LandingPadInfo describing the landing pad information
-  // in the current function.
+  /// CompactUnwindEncoding - If the target supports it, this is the compact
+  /// unwind encoding. It replaces a function's CIE and FDE.
+  uint32_t CompactUnwindEncoding;
+
+  /// LandingPads - List of LandingPadInfo describing the landing pad
+  /// information in the current function.
   std::vector<LandingPadInfo> LandingPads;
 
-  // Map of invoke call site index values to associated begin EH_LABEL for
-  // the current function.
+  /// LPadToCallSiteMap - Map a landing pad's EH symbol to the call site
+  /// indexes.
+  DenseMap<MCSymbol*, SmallVector<unsigned, 4> > LPadToCallSiteMap;
+
+  /// CallSiteMap - Map of invoke call site index values to associated begin
+  /// EH_LABEL for the current function.
   DenseMap<MCSymbol*, unsigned> CallSiteMap;
 
-  // The current call site index being processed, if any. 0 if none.
+  /// CurCallSite - The current call site index being processed, if any. 0 if
+  /// none.
   unsigned CurCallSite;
 
-  // TypeInfos - List of C++ TypeInfo used in the current function.
-  //
+  /// TypeInfos - List of C++ TypeInfo used in the current function.
   std::vector<const GlobalVariable *> TypeInfos;
 
-  // FilterIds - List of typeids encoding filters used in the current function.
-  //
+  /// FilterIds - List of typeids encoding filters used in the current function.
   std::vector<unsigned> FilterIds;
 
-  // FilterEnds - List of the indices in FilterIds corresponding to filter
-  // terminators.
-  //
+  /// FilterEnds - List of the indices in FilterIds corresponding to filter
+  /// terminators.
   std::vector<unsigned> FilterEnds;
 
-  // Personalities - Vector of all personality functions ever seen. Used to emit
-  // common EH frames.
+  /// Personalities - Vector of all personality functions ever seen. Used to
+  /// emit common EH frames.
   std::vector<const Function *> Personalities;
 
   /// UsedFunctions - The functions in the @llvm.used list in a more easily
@@ -144,7 +150,6 @@ class MachineModuleInfo : public ImmutablePass {
   /// llvm.compiler.used.
   SmallPtrSet<const Function *, 32> UsedFunctions;
 
-
   /// AddrLabelSymbols - This map keeps track of which symbol is being used for
   /// the specified basic block's address of label.
   MMIAddrLabelMap *AddrLabelSymbols;
@@ -156,8 +161,9 @@ class MachineModuleInfo : public ImmutablePass {
   /// in this module.
   bool DbgInfoAvailable;
 
-  /// True if this module calls VarArg function with floating point arguments.
-  /// This is used to emit an undefined reference to fltused on Windows targets.
+  /// CallsExternalVAFunctionWithFloatingPointArguments - True if this module
+  /// calls VarArg function with floating point arguments.  This is used to emit
+  /// an undefined reference to fltused on Windows targets.
   bool CallsExternalVAFunctionWithFloatingPointArguments;
 
 public:
@@ -170,7 +176,8 @@ class MachineModuleInfo : public ImmutablePass {
 
   MachineModuleInfo();  // DUMMY CONSTRUCTOR, DO NOT CALL.
   // Real constructor.
-  MachineModuleInfo(const MCAsmInfo &MAI, const TargetAsmInfo *TAI);
+  MachineModuleInfo(const MCAsmInfo &MAI, const MCRegisterInfo &MRI,
+                    const MCObjectFileInfo *MOFI);
   ~MachineModuleInfo();
 
   bool doInitialization();
@@ -229,6 +236,15 @@ class MachineModuleInfo : public ImmutablePass {
   /// handling comsumers.
   std::vector<MachineMove> &getFrameMoves() { return FrameMoves; }
 
+  /// getCompactUnwindEncoding - Returns the compact unwind encoding for a
+  /// function if the target supports the encoding. This encoding replaces a
+  /// function's CIE and FDE.
+  uint32_t getCompactUnwindEncoding() const { return CompactUnwindEncoding; }
+
+  /// setCompactUnwindEncoding - Set the compact unwind encoding for a function
+  /// if the target supports the encoding.
+  void setCompactUnwindEncoding(uint32_t Enc) { CompactUnwindEncoding = Enc; }
+
   /// getAddrLabelSymbol - Return the symbol to be used for the specified basic
   /// block when its address is taken.  This cannot be its normal LBB label
   /// because the block may be accessed outside its containing function.
@@ -286,12 +302,12 @@ class MachineModuleInfo : public ImmutablePass {
   /// addCatchTypeInfo - Provide the catch typeinfo for a landing pad.
   ///
   void addCatchTypeInfo(MachineBasicBlock *LandingPad,
-                        std::vector<const GlobalVariable *> &TyInfo);
+                        ArrayRef<const GlobalVariable *> TyInfo);
 
   /// addFilterTypeInfo - Provide the filter typeinfo for a landing pad.
   ///
   void addFilterTypeInfo(MachineBasicBlock *LandingPad,
-                         std::vector<const GlobalVariable *> &TyInfo);
+                         ArrayRef<const GlobalVariable *> TyInfo);
 
   /// addCleanup - Add a cleanup action for a landing pad.
   ///
@@ -315,18 +331,42 @@ class MachineModuleInfo : public ImmutablePass {
     return LandingPads;
   }
 
-  /// setCallSiteBeginLabel - Map the begin label for a call site
+  /// setCallSiteLandingPad - Map the landing pad's EH symbol to the call
+  /// site indexes.
+  void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef<unsigned> Sites);
+
+  /// getCallSiteLandingPad - Get the call site indexes for a landing pad EH
+  /// symbol.
+  SmallVectorImpl<unsigned> &getCallSiteLandingPad(MCSymbol *Sym) {
+    assert(hasCallSiteLandingPad(Sym) &&
+           "missing call site number for landing pad!");
+    return LPadToCallSiteMap[Sym];
+  }
+
+  /// hasCallSiteLandingPad - Return true if the landing pad Eh symbol has an
+  /// associated call site.
+  bool hasCallSiteLandingPad(MCSymbol *Sym) {
+    return !LPadToCallSiteMap[Sym].empty();
+  }
+
+  /// setCallSiteBeginLabel - Map the begin label for a call site.
   void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site) {
     CallSiteMap[BeginLabel] = Site;
   }
 
-  /// getCallSiteBeginLabel - Get the call site number for a begin label
+  /// getCallSiteBeginLabel - Get the call site number for a begin label.
   unsigned getCallSiteBeginLabel(MCSymbol *BeginLabel) {
-    assert(CallSiteMap.count(BeginLabel) &&
+    assert(hasCallSiteBeginLabel(BeginLabel) &&
            "Missing call site number for EH_LABEL!");
     return CallSiteMap[BeginLabel];
   }
 
+  /// hasCallSiteBeginLabel - Return true if the begin label has a call site
+  /// number associated with it.
+  bool hasCallSiteBeginLabel(MCSymbol *BeginLabel) {
+    return CallSiteMap[BeginLabel] != 0;
+  }
+
   /// setCurrentCallSite - Set the call site currently being processed.
   void setCurrentCallSite(unsigned Site) { CurCallSite = Site; }
 
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineOperand.h b/contrib/llvm/include/llvm/CodeGen/MachineOperand.h
index fdef574be932..5440a636a4a5 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineOperand.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineOperand.h
@@ -83,8 +83,23 @@ class MachineOperand {
   /// This is only valid on definitions of registers.
   bool IsDead : 1;
 
-  /// IsUndef - True if this is a register def / use of "undef", i.e. register
-  /// defined by an IMPLICIT_DEF. This is only valid on registers.
+  /// IsUndef - True if this register operand reads an "undef" value, i.e. the
+  /// read value doesn't matter.  This flag can be set on both use and def
+  /// operands.  On a sub-register def operand, it refers to the part of the
+  /// register that isn't written.  On a full-register def operand, it is a
+  /// noop.  See readsReg().
+  ///
+  /// This is only valid on registers.
+  ///
+  /// Note that an instruction may have multiple <undef> operands referring to
+  /// the same register.  In that case, the instruction may depend on those
+  /// operands reading the same dont-care value.  For example:
+  ///
+  ///   %vreg1<def> = XOR %vreg2<undef>, %vreg2<undef>
+  ///
+  /// Any register can be used for %vreg2, and its value doesn't matter, but
+  /// the two operands must be the same register.
+  ///
   bool IsUndef : 1;
 
   /// IsEarlyClobber - True if this MO_Register 'def' operand is written to
@@ -253,6 +268,15 @@ class MachineOperand {
     return IsDebug;
   }
 
+  /// readsReg - Returns true if this operand reads the previous value of its
+  /// register.  A use operand with the <undef> flag set doesn't read its
+  /// register.  A sub-register def implicitly reads the other parts of the
+  /// register being redefined unless the <undef> flag is set.
+  bool readsReg() const {
+    assert(isReg() && "Wrong MachineOperand accessor");
+    return !isUndef() && (isUse() || getSubReg());
+  }
+
   /// getNextOperandForReg - Return the next MachineOperand in the function that
   /// uses or defines this register.
   MachineOperand *getNextOperandForReg() const {
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index 10797263650d..3866b2650d0b 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -25,6 +25,12 @@ namespace llvm {
 /// registers, including vreg register classes, use/def chains for registers,
 /// etc.
 class MachineRegisterInfo {
+  const TargetRegisterInfo *const TRI;
+
+  /// IsSSA - True when the machine function is in SSA form and virtual
+  /// registers have a single def.
+  bool IsSSA;
+
   /// VRegInfo - Information we keep for each virtual register.
   ///
   /// Each element in this list contains the register class of the vreg and the
@@ -65,7 +71,23 @@ class MachineRegisterInfo {
 public:
   explicit MachineRegisterInfo(const TargetRegisterInfo &TRI);
   ~MachineRegisterInfo();
-  
+
+  //===--------------------------------------------------------------------===//
+  // Function State
+  //===--------------------------------------------------------------------===//
+
+  // isSSA - Returns true when the machine function is in SSA form. Early
+  // passes require the machine function to be in SSA form where every virtual
+  // register has a single defining instruction.
+  //
+  // The TwoAddressInstructionPass and PHIElimination passes take the machine
+  // function out of SSA form when they introduce multiple defs per virtual
+  // register.
+  bool isSSA() const { return IsSSA; }
+
+  // leaveSSA - Indicates that the machine function is no longer in SSA form.
+  void leaveSSA() { IsSSA = false; }
+
   //===--------------------------------------------------------------------===//
   // Register Info
   //===--------------------------------------------------------------------===//
@@ -195,12 +217,25 @@ class MachineRegisterInfo {
   void setRegClass(unsigned Reg, const TargetRegisterClass *RC);
 
   /// constrainRegClass - Constrain the register class of the specified virtual
-  /// register to be a common subclass of RC and the current register class.
-  /// Return the new register class, or NULL if no such class exists.
+  /// register to be a common subclass of RC and the current register class,
+  /// but only if the new class has at least MinNumRegs registers.  Return the
+  /// new register class, or NULL if no such class exists.
   /// This should only be used when the constraint is known to be trivial, like
   /// GR32 -> GR32_NOSP. Beware of increasing register pressure.
+  ///
   const TargetRegisterClass *constrainRegClass(unsigned Reg,
-                                               const TargetRegisterClass *RC);
+                                               const TargetRegisterClass *RC,
+                                               unsigned MinNumRegs = 0);
+
+  /// recomputeRegClass - Try to find a legal super-class of Reg's register
+  /// class that still satisfies the constraints from the instructions using
+  /// Reg.  Returns true if Reg was upgraded.
+  ///
+  /// This method can be used after constraints have been removed from a
+  /// virtual register, for example after removing instructions or splitting
+  /// the live range.
+  ///
+  bool recomputeRegClass(unsigned Reg, const TargetMachine&);
 
   /// createVirtualRegister - Create and return a new virtual register in the
   /// function with the specified register class.
diff --git a/contrib/llvm/include/llvm/CodeGen/Passes.h b/contrib/llvm/include/llvm/CodeGen/Passes.h
index e7928cb65bd3..7a03ce905d89 100644
--- a/contrib/llvm/include/llvm/CodeGen/Passes.h
+++ b/contrib/llvm/include/llvm/CodeGen/Passes.h
@@ -24,7 +24,7 @@ namespace llvm {
   class MachineFunctionPass;
   class PassInfo;
   class TargetLowering;
-  class RegisterCoalescer;
+  class TargetRegisterClass;
   class raw_ostream;
 
   /// createUnreachableBlockEliminationPass - The LLVM code generator does not
@@ -81,6 +81,9 @@ namespace llvm {
   /// register allocators.
   extern char &TwoAddressInstructionPassID;
 
+  /// RegisteCoalescer pass - This pass merges live ranges to eliminate copies.
+  extern char &RegisterCoalescerPassID;
+
   /// SpillPlacement analysis. Suggest optimal placement of spill code between
   /// basic blocks.
   ///
@@ -125,21 +128,15 @@ namespace llvm {
   ///
   FunctionPass *createDefaultPBQPRegisterAllocator();
 
-  /// RegisterCoalescer Pass - Coalesce all copies possible.  Can run
-  /// independently of the register allocator.
-  ///
-  RegisterCoalescer *createRegisterCoalescer();
-
   /// PrologEpilogCodeInserter Pass - This pass inserts prolog and epilog code,
   /// and eliminates abstract frame references.
   ///
   FunctionPass *createPrologEpilogCodeInserter();
 
-  /// LowerSubregs Pass - This pass lowers subregs to register-register copies
-  /// which yields suboptimal, but correct code if the register allocator
-  /// cannot coalesce all subreg operations during allocation.
+  /// ExpandPostRAPseudos Pass - This pass expands pseudo instructions after
+  /// register allocation.
   ///
-  FunctionPass *createLowerSubregsPass();
+  FunctionPass *createExpandPostRAPseudosPass();
 
   /// createPostRAScheduler - This pass performs post register allocation
   /// scheduling.
@@ -229,6 +226,14 @@ namespace llvm {
   ///
   FunctionPass *createExpandISelPseudosPass();
 
+  /// createExecutionDependencyFixPass - This pass fixes execution time
+  /// problems with dependent instructions, such as switching execution
+  /// domains to match.
+  ///
+  /// The pass will examine instructions using and defining registers in RC.
+  ///
+  FunctionPass *createExecutionDependencyFixPass(const TargetRegisterClass *RC);
+
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h
index 3ec5adabe6f2..132983c504e7 100644
--- a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -450,6 +450,10 @@ class SelectionDAG {
   SDValue getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, SDValue N2,
                            const int *MaskElts);
 
+  /// getAnyExtOrTrunc - Convert Op, which must be of integer type, to the
+  /// integer type VT, by either any-extending or truncating it.
+  SDValue getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT);
+
   /// getSExtOrTrunc - Convert Op, which must be of integer type, to the
   /// integer type VT, by either sign-extending or truncating it.
   SDValue getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT);
@@ -560,17 +564,13 @@ class SelectionDAG {
   ///
   SDValue getSetCC(DebugLoc DL, EVT VT, SDValue LHS, SDValue RHS,
                    ISD::CondCode Cond) {
+    assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&
+      "Cannot compare scalars to vectors");
+    assert(LHS.getValueType().isVector() == VT.isVector() &&
+      "Cannot compare scalars to vectors");
     return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
   }
 
-  /// getVSetCC - Helper function to make it easier to build VSetCC's nodes
-  /// if you just have an ISD::CondCode instead of an SDValue.
-  ///
-  SDValue getVSetCC(DebugLoc DL, EVT VT, SDValue LHS, SDValue RHS,
-                    ISD::CondCode Cond) {
-    return getNode(ISD::VSETCC, DL, VT, LHS, RHS, getCondCode(Cond));
-  }
-
   /// getSelectCC - Helper function to make it easier to build SelectCC's if you
   /// just have an ISD::CondCode instead of an SDValue.
   ///
@@ -589,19 +589,37 @@ class SelectionDAG {
   /// takes 3 operands
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Cmp, SDValue Swp,
-                    MachinePointerInfo PtrInfo, unsigned Alignment=0);
+                    MachinePointerInfo PtrInfo, unsigned Alignment,
+                    AtomicOrdering Ordering,
+                    SynchronizationScope SynchScope);
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Cmp, SDValue Swp,
-                    MachineMemOperand *MMO);
+                    MachineMemOperand *MMO,
+                    AtomicOrdering Ordering,
+                    SynchronizationScope SynchScope);
 
-  /// getAtomic - Gets a node for an atomic op, produces result and chain and
-  /// takes 2 operands.
+  /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
+  /// and chain and takes 2 operands.
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, const Value* PtrVal,
-                    unsigned Alignment = 0);
+                    unsigned Alignment, AtomicOrdering Ordering,
+                    SynchronizationScope SynchScope);
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
-                    SDValue Ptr, SDValue Val,
-                    MachineMemOperand *MMO);
+                    SDValue Ptr, SDValue Val, MachineMemOperand *MMO,
+                    AtomicOrdering Ordering,
+                    SynchronizationScope SynchScope);
+
+  /// getAtomic - Gets a node for an atomic op, produces result and chain and
+  /// takes 1 operand.
+  SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,
+                    SDValue Chain, SDValue Ptr, const Value* PtrVal,
+                    unsigned Alignment,
+                    AtomicOrdering Ordering,
+                    SynchronizationScope SynchScope);
+  SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,
+                    SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
+                    AtomicOrdering Ordering,
+                    SynchronizationScope SynchScope);
 
   /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
   /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index a5c4201ef69a..6c7be69b4d2f 100644
--- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -20,6 +20,7 @@
 #define LLVM_CODEGEN_SELECTIONDAGNODES_H
 
 #include "llvm/Constants.h"
+#include "llvm/Instructions.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/ilist_node.h"
@@ -917,6 +918,13 @@ class MemSDNode : public SDNode {
   bool isVolatile() const { return (SubclassData >> 5) & 1; }
   bool isNonTemporal() const { return (SubclassData >> 6) & 1; }
 
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering((SubclassData >> 7) & 15);
+  }
+  SynchronizationScope getSynchScope() const {
+    return SynchronizationScope((SubclassData >> 11) & 1);
+  }
+
   /// Returns the SrcValue and offset that describes the location of the access
   const Value *getSrcValue() const { return MMO->getValue(); }
   int64_t getSrcValueOffset() const { return MMO->getOffset(); }
@@ -968,6 +976,8 @@ class MemSDNode : public SDNode {
            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD         ||
+           N->getOpcode() == ISD::ATOMIC_STORE        ||
            N->isTargetMemoryOpcode();
   }
 };
@@ -977,6 +987,23 @@ class MemSDNode : public SDNode {
 class AtomicSDNode : public MemSDNode {
   SDUse Ops[4];
 
+  void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) {
+    // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp.
+    assert((Ordering & 15) == Ordering &&
+           "Ordering may not require more than 4 bits!");
+    assert((SynchScope & 1) == SynchScope &&
+           "SynchScope may not require more than 1 bit!");
+    SubclassData |= Ordering << 7;
+    SubclassData |= SynchScope << 11;
+    assert(getOrdering() == Ordering && "Ordering encoding error!");
+    assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
+
+    assert((readMem() || getOrdering() <= Monotonic) &&
+           "Acquire/Release MachineMemOperand must be a load!");
+    assert((writeMem() || getOrdering() <= Monotonic) &&
+           "Acquire/Release MachineMemOperand must be a store!");
+  }
+
 public:
   // Opc:   opcode for atomic
   // VTL:    value type list
@@ -988,20 +1015,28 @@ class AtomicSDNode : public MemSDNode {
   // Align:  alignment of memory
   AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
                SDValue Chain, SDValue Ptr,
-               SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
+               SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
+               AtomicOrdering Ordering, SynchronizationScope SynchScope)
     : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
-    assert(readMem() && "Atomic MachineMemOperand is not a load!");
-    assert(writeMem() && "Atomic MachineMemOperand is not a store!");
+    InitAtomic(Ordering, SynchScope);
     InitOperands(Ops, Chain, Ptr, Cmp, Swp);
   }
   AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
                SDValue Chain, SDValue Ptr,
-               SDValue Val, MachineMemOperand *MMO)
+               SDValue Val, MachineMemOperand *MMO,
+               AtomicOrdering Ordering, SynchronizationScope SynchScope)
     : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
-    assert(readMem() && "Atomic MachineMemOperand is not a load!");
-    assert(writeMem() && "Atomic MachineMemOperand is not a store!");
+    InitAtomic(Ordering, SynchScope);
     InitOperands(Ops, Chain, Ptr, Val);
   }
+  AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
+               SDValue Chain, SDValue Ptr,
+               MachineMemOperand *MMO,
+               AtomicOrdering Ordering, SynchronizationScope SynchScope)
+    : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
+    InitAtomic(Ordering, SynchScope);
+    InitOperands(Ops, Chain, Ptr);
+  }
 
   const SDValue &getBasePtr() const { return getOperand(1); }
   const SDValue &getVal() const { return getOperand(2); }
@@ -1025,7 +1060,9 @@ class AtomicSDNode : public MemSDNode {
            N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX;
+           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD         ||
+           N->getOpcode() == ISD::ATOMIC_STORE;
   }
 };
 
@@ -1291,7 +1328,7 @@ class ConstantPoolSDNode : public SDNode {
   unsigned getAlignment() const { return Alignment; }
   unsigned char getTargetFlags() const { return TargetFlags; }
 
-  const Type *getType() const;
+  Type *getType() const;
 
   static bool classof(const ConstantPoolSDNode *) { return true; }
   static bool classof(const SDNode *N) {
diff --git a/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h b/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h
index 6eb31802120e..2d98864dc9ed 100644
--- a/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h
+++ b/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements SlotIndex and related classes. The purpuse of SlotIndex
+// This file implements SlotIndex and related classes. The purpose of SlotIndex
 // is to describe a position at which a register can become live, or cease to
 // be live.
 //
diff --git a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 711280e98c91..ca40ccf85378 100644
--- a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -33,42 +33,13 @@ namespace llvm {
 
 
 class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
-protected:
-  /// TLSDataSection - Section directive for Thread Local data.
-  ///
-  const MCSection *TLSDataSection;        // Defaults to ".tdata".
-
-  /// TLSBSSSection - Section directive for Thread Local uninitialized data.
-  /// Null if this target doesn't support a BSS section.
-  ///
-  const MCSection *TLSBSSSection;         // Defaults to ".tbss".
-
-  const MCSection *DataRelSection;
-  const MCSection *DataRelLocalSection;
-  const MCSection *DataRelROSection;
-  const MCSection *DataRelROLocalSection;
-
-  const MCSection *MergeableConst4Section;
-  const MCSection *MergeableConst8Section;
-  const MCSection *MergeableConst16Section;
 public:
-  TargetLoweringObjectFileELF();
-  ~TargetLoweringObjectFileELF() {}
-
-  virtual void Initialize(MCContext &Ctx, const TargetMachine &TM);
-
-  virtual const MCSection *getEHFrameSection() const;
-  virtual const MCSection *getWin64EHFuncTableSection(StringRef) const {
-    return NULL;
-  }
-  virtual const MCSection *getWin64EHTableSection(StringRef) const{return NULL;}
+  virtual ~TargetLoweringObjectFileELF() {}
 
   virtual void emitPersonalityValue(MCStreamer &Streamer,
                                     const TargetMachine &TM,
                                     const MCSymbol *Sym) const;
 
-  const MCSection *getDataRelSection() const { return DataRelSection; }
-
   /// getSectionForConstant - Given a constant with the SectionKind, return a
   /// section that it should be placed in.
   virtual const MCSection *getSectionForConstant(SectionKind Kind) const;
@@ -99,48 +70,8 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
 
 
 class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
-  /// TLSDataSection - Section for thread local data.
-  ///
-  const MCSection *TLSDataSection;        // Defaults to ".tdata".
-
-  /// TLSBSSSection - Section for thread local uninitialized data.
-  ///
-  const MCSection *TLSBSSSection;         // Defaults to ".tbss".
-  
-  /// TLSTLVSection - Section for thread local structure information.
-  /// Contains the source code name of the variable, visibility and a pointer
-  /// to the initial value (.tdata or .tbss).
-  const MCSection *TLSTLVSection;         // Defaults to ".tlv".
-  
-  /// TLSThreadInitSection - Section for thread local data initialization
-  /// functions.
-  const MCSection *TLSThreadInitSection;  // Defaults to ".thread_init_func".
-  
-  const MCSection *CStringSection;
-  const MCSection *UStringSection;
-  const MCSection *TextCoalSection;
-  const MCSection *ConstTextCoalSection;
-  const MCSection *ConstDataSection;
-  const MCSection *DataCoalSection;
-  const MCSection *DataCommonSection;
-  const MCSection *DataBSSSection;
-  const MCSection *FourByteConstantSection;
-  const MCSection *EightByteConstantSection;
-  const MCSection *SixteenByteConstantSection;
-
-  const MCSection *LazySymbolPointerSection;
-  const MCSection *NonLazySymbolPointerSection;
 public:
-  TargetLoweringObjectFileMachO();
-  ~TargetLoweringObjectFileMachO() {}
-
-  virtual void Initialize(MCContext &Ctx, const TargetMachine &TM);
-
-  virtual const MCSection *getEHFrameSection() const;
-  virtual const MCSection *getWin64EHFuncTableSection(StringRef) const {
-    return NULL;
-  }
-  virtual const MCSection *getWin64EHTableSection(StringRef) const{return NULL;}
+  virtual ~TargetLoweringObjectFileMachO() {}
 
   virtual const MCSection *
   SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
@@ -158,30 +89,6 @@ class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
   virtual bool shouldEmitUsedDirectiveFor(const GlobalValue *GV,
                                           Mangler *) const;
 
-  /// getTextCoalSection - Return the "__TEXT,__textcoal_nt" section we put weak
-  /// text symbols into.
-  const MCSection *getTextCoalSection() const {
-    return TextCoalSection;
-  }
-
-  /// getConstTextCoalSection - Return the "__TEXT,__const_coal" section
-  /// we put weak read-only symbols into.
-  const MCSection *getConstTextCoalSection() const {
-    return ConstTextCoalSection;
-  }
-
-  /// getLazySymbolPointerSection - Return the section corresponding to
-  /// the .lazy_symbol_pointer directive.
-  const MCSection *getLazySymbolPointerSection() const {
-    return LazySymbolPointerSection;
-  }
-
-  /// getNonLazySymbolPointerSection - Return the section corresponding to
-  /// the .non_lazy_symbol_pointer directive.
-  const MCSection *getNonLazySymbolPointerSection() const {
-    return NonLazySymbolPointerSection;
-  }
-
   /// getExprForDwarfGlobalReference - The mach-o version of this method
   /// defaults to returning a stub reference.
   virtual const MCExpr *
@@ -193,30 +100,13 @@ class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
   virtual MCSymbol *
   getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
                           MachineModuleInfo *MMI) const;
-
-  virtual unsigned getPersonalityEncoding() const;
-  virtual unsigned getLSDAEncoding() const;
-  virtual unsigned getFDEEncoding(bool CFI) const;
-  virtual unsigned getTTypeEncoding() const;
 };
 
 
 
 class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile {
-  const MCSection *DrectveSection;
-  const MCSection *PDataSection;
-  const MCSection *XDataSection;
 public:
-  TargetLoweringObjectFileCOFF();
-  ~TargetLoweringObjectFileCOFF() {}
-
-  virtual void Initialize(MCContext &Ctx, const TargetMachine &TM);
-
-  virtual const MCSection *getEHFrameSection() const;
-  virtual const MCSection *getWin64EHFuncTableSection(StringRef) const;
-  virtual const MCSection *getWin64EHTableSection(StringRef) const;
-
-  virtual const MCSection *getDrectveSection() const { return DrectveSection; }
+  virtual ~TargetLoweringObjectFileCOFF() {}
 
   virtual const MCSection *
   getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
diff --git a/contrib/llvm/include/llvm/CodeGen/ValueTypes.h b/contrib/llvm/include/llvm/CodeGen/ValueTypes.h
index 424721bf6490..cae0bcb165c1 100644
--- a/contrib/llvm/include/llvm/CodeGen/ValueTypes.h
+++ b/contrib/llvm/include/llvm/CodeGen/ValueTypes.h
@@ -144,14 +144,14 @@ namespace llvm {
     /// isFloatingPoint - Return true if this is a FP, or a vector FP type.
     bool isFloatingPoint() const {
       return ((SimpleTy >= MVT::f32 && SimpleTy <= MVT::ppcf128) ||
-        (SimpleTy >= MVT::v2f32 && SimpleTy <= MVT::v4f64));
+	      (SimpleTy >= MVT::v2f32 && SimpleTy <= MVT::v4f64));
     }
 
     /// isInteger - Return true if this is an integer, or a vector integer type.
     bool isInteger() const {
       return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
                SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
-               (SimpleTy >= MVT::v2i8 && SimpleTy <= MVT::v8i64));
+	      (SimpleTy >= MVT::v2i8 && SimpleTy <= MVT::v8i64));
     }
 
     /// isVector - Return true if this is a vector value type.
@@ -380,7 +380,7 @@ namespace llvm {
   struct EVT {
   private:
     MVT V;
-    const Type *LLVMTy;
+    Type *LLVMTy;
 
   public:
     EVT() : V((MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE)),
@@ -438,6 +438,21 @@ namespace llvm {
       return MVT::INVALID_SIMPLE_VALUE_TYPE;
     }
 
+    /// changeVectorElementTypeToInteger - Return a vector with the same number
+    /// of elements as this vector, but with the element type converted to an
+    /// integer type with the same bitwidth.
+    EVT changeVectorElementTypeToInteger() const {
+      if (!isSimple())
+        return changeExtendedVectorElementTypeToInteger();
+      MVT EltTy = getSimpleVT().getVectorElementType();
+      unsigned BitWidth = EltTy.getSizeInBits();
+      MVT IntTy = MVT::getIntegerVT(BitWidth);
+      MVT VecTy = MVT::getVectorVT(IntTy, getVectorNumElements());
+      assert(VecTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
+             "Simple vector VT not representable by simple integer vector VT!");
+      return VecTy;
+    }
+
     /// isSimple - Test if the given EVT is simple (as opposed to being
     /// extended).
     bool isSimple() const {
@@ -645,12 +660,12 @@ namespace llvm {
     /// getTypeForEVT - This method returns an LLVM type corresponding to the
     /// specified EVT.  For integer types, this returns an unsigned type.  Note
     /// that this will abort for types that cannot be represented.
-    const Type *getTypeForEVT(LLVMContext &Context) const;
+    Type *getTypeForEVT(LLVMContext &Context) const;
 
     /// getEVT - Return the value type corresponding to the specified type.
     /// This returns all pointers as iPTR.  If HandleUnknown is true, unknown
     /// types are returned as Other, otherwise they are invalid.
-    static EVT getEVT(const Type *Ty, bool HandleUnknown = false);
+    static EVT getEVT(Type *Ty, bool HandleUnknown = false);
 
     intptr_t getRawBits() {
       if (isSimple())
@@ -674,6 +689,7 @@ namespace llvm {
     // Methods for handling the Extended-type case in functions above.
     // These are all out-of-line to prevent users of this header file
     // from having a dependency on Type.h.
+    EVT changeExtendedVectorElementTypeToInteger() const;
     static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth);
     static EVT getExtendedVectorVT(LLVMContext &C, EVT VT,
                                    unsigned NumElements);
diff --git a/contrib/llvm/include/llvm/CompilerDriver/Action.h b/contrib/llvm/include/llvm/CompilerDriver/Action.h
deleted file mode 100644
index f2b79655f60f..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/Action.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===--- Action.h - The LLVM Compiler Driver --------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Action - encapsulates a single shell command.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_ACTION_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_ACTION_H
-
-#include <string>
-#include <vector>
-
-namespace llvmc {
-
-  typedef std::vector<std::string> StrVector;
-
-  /// Action - A class that encapsulates a single shell command.
-  class Action {
-    /// Command_ - The actual command (for example, 'ls').
-    std::string Command_;
-    /// Args_ - Command arguments. Stdout redirection ("> file") is allowed.
-    std::vector<std::string> Args_;
-    /// StopCompilation_ - Should we stop compilation after executing
-    /// this action?
-    bool StopCompilation_;
-    /// OutFile_ - The output file name.
-    std::string OutFile_;
-
-  public:
-    void Construct (const std::string& C, const StrVector& A,
-                    bool S, const std::string& O) {
-      Command_ = C;
-      Args_ = A;
-      StopCompilation_ = S;
-      OutFile_ = O;
-    }
-    bool IsConstructed () { return (Command_.size() != 0);}
-
-    /// Execute - Executes the command. Returns -1 on error.
-    int Execute () const;
-    bool StopCompilation () const { return StopCompilation_; }
-    const std::string& OutFile() { return OutFile_; }
-  };
-
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_ACTION_H
diff --git a/contrib/llvm/include/llvm/CompilerDriver/AutoGenerated.h b/contrib/llvm/include/llvm/CompilerDriver/AutoGenerated.h
deleted file mode 100644
index 7b926c622c90..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/AutoGenerated.h
+++ /dev/null
@@ -1,40 +0,0 @@
-//===--- AutoGenerated.h - The LLVM Compiler Driver -------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Interface to the autogenerated driver code.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_AUTOGENERATED_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_AUTOGENERATED_H
-
-namespace llvmc {
-  class LanguageMap;
-  class CompilationGraph;
-
-  namespace autogenerated {
-
-    int PreprocessOptions();
-    int PopulateLanguageMap(LanguageMap& langMap);
-    int PopulateCompilationGraph(CompilationGraph& graph);
-
-    inline int RunInitialization (LanguageMap& M, CompilationGraph& G) {
-      if (int ret = PreprocessOptions())
-        return ret;
-      if (int ret = PopulateLanguageMap(M))
-        return ret;
-      if (int ret = PopulateCompilationGraph(G))
-        return ret;
-
-      return 0;
-    }
-  }
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_AUTOGENERATED_H
diff --git a/contrib/llvm/include/llvm/CompilerDriver/BuiltinOptions.h b/contrib/llvm/include/llvm/CompilerDriver/BuiltinOptions.h
deleted file mode 100644
index 7b9c15c52f7f..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/BuiltinOptions.h
+++ /dev/null
@@ -1,39 +0,0 @@
-//===--- BuiltinOptions.h - The LLVM Compiler Driver ------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Declarations of all global command-line option variables.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_BUILTIN_OPTIONS_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_BUILTIN_OPTIONS_H
-
-#include "llvm/Support/CommandLine.h"
-
-#include <string>
-
-namespace llvmc {
-
-namespace SaveTempsEnum { enum Values { Cwd, Obj, Unset }; }
-
-extern llvm::cl::list<std::string> InputFilenames;
-extern llvm::cl::opt<std::string> OutputFilename;
-extern llvm::cl::opt<std::string> TempDirname;
-extern llvm::cl::list<std::string> Languages;
-extern llvm::cl::opt<bool> DryRun;
-extern llvm::cl::opt<bool> Time;
-extern llvm::cl::opt<bool> VerboseMode;
-extern llvm::cl::opt<bool> CheckGraph;
-extern llvm::cl::opt<bool> ViewGraph;
-extern llvm::cl::opt<bool> WriteGraph;
-extern llvm::cl::opt<SaveTempsEnum::Values> SaveTemps;
-
-} // End namespace llvmc.
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_BUILTIN_OPTIONS_H
diff --git a/contrib/llvm/include/llvm/CompilerDriver/Common.td b/contrib/llvm/include/llvm/CompilerDriver/Common.td
deleted file mode 100644
index 6ba30aaa6406..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/Common.td
+++ /dev/null
@@ -1,127 +0,0 @@
-//===- Common.td - Common definitions for LLVMC2  ----------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains common definitions used in llvmc tool description files.
-//
-//===----------------------------------------------------------------------===//
-
-class Tool<list<dag> l> {
-      list<dag> properties = l;
-}
-
-// Possible Tool properties.
-
-def in_language;
-def out_language;
-def output_suffix;
-def command;
-def out_file_option;
-def in_file_option;
-def join;
-def sink;
-def works_on_empty;
-def actions;
-
-// Possible option types.
-
-def alias_option;
-def switch_option;
-def switch_list_option;
-def parameter_option;
-def parameter_list_option;
-def prefix_option;
-def prefix_list_option;
-
-// Possible option properties.
-
-def help;
-def hidden;
-def init;
-def multi_val;
-def one_or_more;
-def zero_or_more;
-def optional;
-def really_hidden;
-def required;
-def comma_separated;
-def forward_not_split;
-
-// The 'case' construct.
-def case;
-
-// Boolean constants.
-class Bool<bit val> {
-      bit Value = val;
-}
-def true : Bool<1>;
-def false : Bool<0>;
-
-// Boolean operators.
-def and;
-def or;
-def not;
-
-// Primitive tests.
-def switch_on;
-def parameter_equals;
-def element_in_list;
-def input_languages_contain;
-def empty;
-def not_empty;
-def default;
-def single_input_file;
-def multiple_input_files;
-def any_switch_on;
-def any_not_empty;
-def any_empty;
-
-// Possible actions.
-
-def append_cmd;
-def forward;
-def forward_as;
-def forward_value;
-def forward_transformed_value;
-def stop_compilation;
-def no_out_file;
-def unpack_values;
-def warning;
-def error;
-def set_option;
-def unset_option;
-
-// Increase the edge weight.
-def inc_weight;
-
-// Option list - a single place to specify options.
-class OptionList<list<dag> l> {
-      list<dag> options = l;
-}
-
-// Option preprocessor - actions taken during plugin loading.
-class OptionPreprocessor<dag d> {
-      dag preprocessor = d;
-}
-
-// Map from suffixes to language names
-
-def lang_to_suffixes;
-
-class LanguageMap<list<dag> l> {
-      list<dag> map = l;
-}
-
-// Compilation graph
-
-def edge;
-def optional_edge;
-
-class CompilationGraph<list<dag> l> {
-      list<dag> edges = l;
-}
diff --git a/contrib/llvm/include/llvm/CompilerDriver/CompilationGraph.h b/contrib/llvm/include/llvm/CompilerDriver/CompilationGraph.h
deleted file mode 100644
index 951aff6f938d..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/CompilationGraph.h
+++ /dev/null
@@ -1,330 +0,0 @@
-//===--- CompilationGraph.h - The LLVM Compiler Driver ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Compilation graph - definition.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_COMPILATION_GRAPH_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_COMPILATION_GRAPH_H
-
-#include "llvm/CompilerDriver/Tool.h"
-
-#include "llvm/ADT/GraphTraits.h"
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringSet.h"
-#include "llvm/Support/Path.h"
-
-#include <cassert>
-#include <string>
-
-namespace llvmc {
-
-  class CompilationGraph;
-  typedef llvm::StringSet<> InputLanguagesSet;
-
-  /// LanguageMap - Maps from extensions to language names.
-  class LanguageMap : public llvm::StringMap<std::string> {
-  public:
-
-    /// GetLanguage -  Find the language name corresponding to a given file.
-    const std::string* GetLanguage(const llvm::sys::Path&) const;
-  };
-
-  /// Edge - Represents an edge of the compilation graph.
-  class Edge : public llvm::RefCountedBaseVPTR {
-  public:
-    Edge(const std::string& T) : ToolName_(T) {}
-    virtual ~Edge() {}
-
-    const std::string& ToolName() const { return ToolName_; }
-    virtual int Weight(const InputLanguagesSet& InLangs) const = 0;
-  private:
-    std::string ToolName_;
-  };
-
-  /// SimpleEdge - An edge that has no properties.
-  class SimpleEdge : public Edge {
-  public:
-    SimpleEdge(const std::string& T) : Edge(T) {}
-    int Weight(const InputLanguagesSet&) const { return 1; }
-  };
-
-  /// Node - A node (vertex) of the compilation graph.
-  struct Node {
-    // A Node holds a list of the outward edges.
-    typedef llvm::SmallVector<llvm::IntrusiveRefCntPtr<Edge>, 3> container_type;
-    typedef container_type::iterator iterator;
-    typedef container_type::const_iterator const_iterator;
-
-    Node() : OwningGraph(0), InEdges(0) {}
-    Node(CompilationGraph* G) : OwningGraph(G), InEdges(0) {}
-    Node(CompilationGraph* G, Tool* T) :
-      OwningGraph(G), ToolPtr(T), InEdges(0) {}
-
-    bool HasChildren() const { return !OutEdges.empty(); }
-    const std::string Name() const
-    { return ToolPtr ? ToolPtr->Name() : "root"; }
-
-    // Iteration.
-    iterator EdgesBegin() { return OutEdges.begin(); }
-    const_iterator EdgesBegin() const { return OutEdges.begin(); }
-    iterator EdgesEnd() { return OutEdges.end(); }
-    const_iterator EdgesEnd() const { return OutEdges.end(); }
-
-    /// AddEdge - Add an outward edge. Takes ownership of the provided
-    /// Edge object.
-    void AddEdge(Edge* E);
-
-    // Inward edge counter. Used to implement topological sort.
-    void IncrInEdges() { ++InEdges; }
-    void DecrInEdges() { --InEdges; }
-    bool HasNoInEdges() const { return InEdges == 0; }
-
-    // Needed to implement NodeChildIterator/GraphTraits
-    CompilationGraph* OwningGraph;
-    // The corresponding Tool.
-    // WARNING: ToolPtr can be NULL (for the root node).
-    llvm::IntrusiveRefCntPtr<Tool> ToolPtr;
-    // Links to children.
-    container_type OutEdges;
-    // Inward edge counter. Updated in
-    // CompilationGraph::insertEdge(). Used for topological sorting.
-    unsigned InEdges;
-  };
-
-  class NodesIterator;
-
-  /// CompilationGraph - The compilation graph itself.
-  class CompilationGraph {
-    /// nodes_map_type - The main data structure.
-    typedef llvm::StringMap<Node> nodes_map_type;
-    /// tools_vector_type, tools_map_type - Data structures used to
-    /// map from language names to tools. (We can have several tools
-    /// associated with each language name, hence the need for a
-    /// vector.)
-    typedef
-    llvm::SmallVector<llvm::IntrusiveRefCntPtr<Edge>, 3> tools_vector_type;
-    typedef llvm::StringMap<tools_vector_type> tools_map_type;
-
-    /// ToolsMap - Map from language names to lists of tool names.
-    tools_map_type ToolsMap;
-    /// NodesMap - Map from tool names to Tool objects.
-    nodes_map_type NodesMap;
-
-  public:
-
-    typedef nodes_map_type::iterator nodes_iterator;
-    typedef nodes_map_type::const_iterator const_nodes_iterator;
-
-    CompilationGraph();
-
-    /// insertNode - Insert a new node into the graph. Takes
-    /// ownership of the object.
-    void insertNode(Tool* T);
-
-    /// insertEdge - Insert a new edge into the graph. Takes ownership
-    /// of the Edge object. Returns non-zero value on error.
-    int insertEdge(const std::string& A, Edge* E);
-
-    /// Build - Build target(s) from the input file set. Command-line options
-    /// are passed implicitly as global variables. Returns non-zero value on
-    /// error (usually the failed program's exit code).
-    int Build(llvm::sys::Path const& TempDir, const LanguageMap& LangMap);
-
-    /// Check - Check the compilation graph for common errors like cycles,
-    /// input/output language mismatch and multiple default edges. Prints error
-    /// messages and in case it finds any errors.
-    int Check();
-
-    /// getNode - Return a reference to the node corresponding to the given tool
-    /// name. Returns 0 on error.
-    Node* getNode(const std::string& ToolName);
-    const Node* getNode(const std::string& ToolName) const;
-
-    /// viewGraph - This function is meant for use from the debugger. You can
-    /// just say 'call G->viewGraph()' and a ghostview window should pop up from
-    /// the program, displaying the compilation graph. This depends on there
-    /// being a 'dot' and 'gv' program in your path.
-    void viewGraph();
-
-    /// writeGraph - Write Graphviz .dot source file to the current direcotry.
-    int writeGraph(const std::string& OutputFilename);
-
-    // GraphTraits support.
-    friend NodesIterator GraphBegin(CompilationGraph*);
-    friend NodesIterator GraphEnd(CompilationGraph*);
-
-  private:
-    // Helper functions.
-
-    /// getToolsVector - Return a reference to the list of tool names
-    /// corresponding to the given language name. Returns 0 on error.
-    const tools_vector_type* getToolsVector(const std::string& LangName) const;
-
-    /// PassThroughGraph - Pass the input file through the toolchain starting at
-    /// StartNode.
-    int PassThroughGraph (const llvm::sys::Path& In, const Node* StartNode,
-                          const InputLanguagesSet& InLangs,
-                          const llvm::sys::Path& TempDir,
-                          const LanguageMap& LangMap) const;
-
-    /// FindToolChain - Find head of the toolchain corresponding to
-    /// the given file.
-    const Node* FindToolChain(const llvm::sys::Path& In,
-                              const std::string* ForceLanguage,
-                              InputLanguagesSet& InLangs,
-                              const LanguageMap& LangMap) const;
-
-    /// BuildInitial - Traverse the initial parts of the toolchains. Returns
-    /// non-zero value on error.
-    int BuildInitial(InputLanguagesSet& InLangs,
-                     const llvm::sys::Path& TempDir,
-                     const LanguageMap& LangMap);
-
-    /// TopologicalSort - Sort the nodes in topological order. Returns non-zero
-    /// value on error.
-    int TopologicalSort(std::vector<const Node*>& Out);
-    /// TopologicalSortFilterJoinNodes - Call TopologicalSort and filter the
-    /// resulting list to include only Join nodes. Returns non-zero value on
-    /// error.
-    int TopologicalSortFilterJoinNodes(std::vector<const Node*>& Out);
-
-    // Functions used to implement Check().
-
-    /// CheckLanguageNames - Check that output/input language names match for
-    /// all nodes. Returns non-zero value on error (number of errors
-    /// encountered).
-    int CheckLanguageNames() const;
-    /// CheckMultipleDefaultEdges - check that there are no multiple default
-    /// default edges. Returns non-zero value on error (number of errors
-    /// encountered).
-    int CheckMultipleDefaultEdges() const;
-    /// CheckCycles - Check that there are no cycles in the graph. Returns
-    /// non-zero value on error (number of errors encountered).
-    int CheckCycles();
-
-  };
-
-  // GraphTraits support code.
-
-  /// NodesIterator - Auxiliary class needed to implement GraphTraits
-  /// support. Can be generalised to something like value_iterator
-  /// for map-like containers.
-  class NodesIterator : public CompilationGraph::nodes_iterator {
-    typedef CompilationGraph::nodes_iterator super;
-    typedef NodesIterator ThisType;
-    typedef Node* pointer;
-    typedef Node& reference;
-
-  public:
-    NodesIterator(super I) : super(I) {}
-
-    inline reference operator*() const {
-      return super::operator->()->second;
-    }
-    inline pointer operator->() const {
-      return &super::operator->()->second;
-    }
-  };
-
-  inline NodesIterator GraphBegin(CompilationGraph* G) {
-    return NodesIterator(G->NodesMap.begin());
-  }
-
-  inline NodesIterator GraphEnd(CompilationGraph* G) {
-    return NodesIterator(G->NodesMap.end());
-  }
-
-
-  /// NodeChildIterator - Another auxiliary class needed by GraphTraits.
-  class NodeChildIterator : public
-               std::iterator<std::bidirectional_iterator_tag, Node, ptrdiff_t> {
-    typedef NodeChildIterator ThisType;
-    typedef Node::container_type::iterator iterator;
-
-    CompilationGraph* OwningGraph;
-    iterator EdgeIter;
-  public:
-    typedef Node* pointer;
-    typedef Node& reference;
-
-    NodeChildIterator(Node* N, iterator I) :
-      OwningGraph(N->OwningGraph), EdgeIter(I) {}
-
-    const ThisType& operator=(const ThisType& I) {
-      assert(OwningGraph == I.OwningGraph);
-      EdgeIter = I.EdgeIter;
-      return *this;
-    }
-
-    inline bool operator==(const ThisType& I) const {
-      assert(OwningGraph == I.OwningGraph);
-      return EdgeIter == I.EdgeIter;
-    }
-    inline bool operator!=(const ThisType& I) const {
-      return !this->operator==(I);
-    }
-
-    inline pointer operator*() const {
-      return OwningGraph->getNode((*EdgeIter)->ToolName());
-    }
-    inline pointer operator->() const {
-      return this->operator*();
-    }
-
-    ThisType& operator++() { ++EdgeIter; return *this; } // Preincrement
-    ThisType operator++(int) { // Postincrement
-      ThisType tmp = *this;
-      ++*this;
-      return tmp;
-    }
-
-    inline ThisType& operator--() { --EdgeIter; return *this; }  // Predecrement
-    inline ThisType operator--(int) { // Postdecrement
-      ThisType tmp = *this;
-      --*this;
-      return tmp;
-    }
-
-  };
-}
-
-namespace llvm {
-  template <>
-  struct GraphTraits<llvmc::CompilationGraph*> {
-    typedef llvmc::CompilationGraph GraphType;
-    typedef llvmc::Node NodeType;
-    typedef llvmc::NodeChildIterator ChildIteratorType;
-
-    static NodeType* getEntryNode(GraphType* G) {
-      return G->getNode("root");
-    }
-
-    static ChildIteratorType child_begin(NodeType* N) {
-      return ChildIteratorType(N, N->OutEdges.begin());
-    }
-    static ChildIteratorType child_end(NodeType* N) {
-      return ChildIteratorType(N, N->OutEdges.end());
-    }
-
-    typedef llvmc::NodesIterator nodes_iterator;
-    static nodes_iterator nodes_begin(GraphType *G) {
-      return GraphBegin(G);
-    }
-    static nodes_iterator nodes_end(GraphType *G) {
-      return GraphEnd(G);
-    }
-  };
-
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_COMPILATION_GRAPH_H
diff --git a/contrib/llvm/include/llvm/CompilerDriver/Error.h b/contrib/llvm/include/llvm/CompilerDriver/Error.h
deleted file mode 100644
index 013094e5dd79..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/Error.h
+++ /dev/null
@@ -1,29 +0,0 @@
-//===--- Error.h - The LLVM Compiler Driver ---------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Error handling.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_ERROR_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_ERROR_H
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace llvmc {
-
-  inline void PrintError(llvm::StringRef Err) {
-    extern const char* ProgramName;
-    llvm::errs() << ProgramName << ": " << Err << '\n';
-  }
-
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_ERROR_H
diff --git a/contrib/llvm/include/llvm/CompilerDriver/Main.h b/contrib/llvm/include/llvm/CompilerDriver/Main.h
deleted file mode 100644
index d136a5d2fa19..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/Main.h
+++ /dev/null
@@ -1,21 +0,0 @@
-//===--- Main.h - The LLVM Compiler Driver ----------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Entry point for the driver executable.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_MAIN_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_MAIN_H
-
-namespace llvmc {
-  int Main(int argc, char** argv);
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_MAIN_H
diff --git a/contrib/llvm/include/llvm/CompilerDriver/Main.inc b/contrib/llvm/include/llvm/CompilerDriver/Main.inc
deleted file mode 100644
index 41640437de89..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/Main.inc
+++ /dev/null
@@ -1,23 +0,0 @@
-//===--- Main.inc - The LLVM Compiler Driver --------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Default main() for the driver executable.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_MAIN_INC
-#define LLVM_INCLUDE_COMPILER_DRIVER_MAIN_INC
-
-#include "llvm/CompilerDriver/Main.h"
-
-int main(int argc, char** argv) {
-  return llvmc::Main(argc, argv);
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_MAIN_INC
diff --git a/contrib/llvm/include/llvm/CompilerDriver/Tool.h b/contrib/llvm/include/llvm/CompilerDriver/Tool.h
deleted file mode 100644
index 18a2b767923e..000000000000
--- a/contrib/llvm/include/llvm/CompilerDriver/Tool.h
+++ /dev/null
@@ -1,100 +0,0 @@
-//===--- Tool.h - The LLVM Compiler Driver ----------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Tool abstract base class - an interface to tool descriptions.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INCLUDE_COMPILER_DRIVER_TOOL_H
-#define LLVM_INCLUDE_COMPILER_DRIVER_TOOL_H
-
-#include "llvm/CompilerDriver/Action.h"
-
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/StringSet.h"
-#include "llvm/Support/Path.h"
-
-#include <string>
-#include <vector>
-#include <utility>
-
-namespace llvmc {
-
-  class LanguageMap;
-  typedef std::vector<std::pair<unsigned, std::string> > ArgsVector;
-  typedef std::vector<llvm::sys::Path> PathVector;
-  typedef std::vector<std::string> StrVector;
-  typedef llvm::StringSet<> InputLanguagesSet;
-
-  /// Tool - Represents a single tool.
-  class Tool : public llvm::RefCountedBaseVPTR {
-  public:
-
-    virtual ~Tool() {}
-
-    /// GenerateAction - Generate an Action given particular command-line
-    /// options. Returns non-zero value on error.
-    virtual int GenerateAction (Action& Out,
-                                const PathVector& inFiles,
-                                const bool HasChildren,
-                                const llvm::sys::Path& TempDir,
-                                const InputLanguagesSet& InLangs,
-                                const LanguageMap& LangMap) const = 0;
-
-    /// GenerateAction - Generate an Action given particular command-line
-    /// options. Returns non-zero value on error.
-    virtual int GenerateAction (Action& Out,
-                                const llvm::sys::Path& inFile,
-                                const bool HasChildren,
-                                const llvm::sys::Path& TempDir,
-                                const InputLanguagesSet& InLangs,
-                                const LanguageMap& LangMap) const = 0;
-
-    virtual const char*  Name() const = 0;
-    virtual const char** InputLanguages() const = 0;
-    virtual const char** OutputLanguages() const = 0;
-
-    virtual bool IsJoin() const = 0;
-    virtual bool WorksOnEmpty() const = 0;
-
-  protected:
-    /// OutFileName - Generate the output file name.
-    llvm::sys::Path OutFilename(const llvm::sys::Path& In,
-                                const llvm::sys::Path& TempDir,
-                                bool StopCompilation,
-                                const char* OutputSuffix) const;
-
-    StrVector SortArgs(ArgsVector& Args) const;
-  };
-
-  /// JoinTool - A Tool that has an associated input file list.
-  class JoinTool : public Tool {
-  public:
-    void AddToJoinList(const llvm::sys::Path& P) { JoinList_.push_back(P); }
-    void ClearJoinList() { JoinList_.clear(); }
-    bool JoinListEmpty() const { return JoinList_.empty(); }
-
-    int GenerateAction(Action& Out,
-                       const bool HasChildren,
-                       const llvm::sys::Path& TempDir,
-                       const InputLanguagesSet& InLangs,
-                       const LanguageMap& LangMap) const {
-      return GenerateAction(Out, JoinList_, HasChildren, TempDir, InLangs,
-                            LangMap);
-    }
-    // We shouldn't shadow base class's version of GenerateAction.
-    using Tool::GenerateAction;
-
-  private:
-    PathVector JoinList_;
-  };
-
-}
-
-#endif // LLVM_INCLUDE_COMPILER_DRIVER_TOOL_H
diff --git a/contrib/llvm/include/llvm/Constant.h b/contrib/llvm/include/llvm/Constant.h
index 5e351c4ec504..ecc1fe70cc5d 100644
--- a/contrib/llvm/include/llvm/Constant.h
+++ b/contrib/llvm/include/llvm/Constant.h
@@ -43,7 +43,7 @@ class Constant : public User {
   Constant(const Constant &);           // Do not implement
   
 protected:
-  Constant(const Type *ty, ValueTy vty, Use *Ops, unsigned NumOps)
+  Constant(Type *ty, ValueTy vty, Use *Ops, unsigned NumOps)
     : User(ty, vty, Ops, NumOps) {}
 
   void destroyConstantImpl();
@@ -52,6 +52,10 @@ class Constant : public User {
   /// getNullValue.
   bool isNullValue() const;
 
+  /// isAllOnesValue - Return true if this is the value that would be returned by
+  /// getAllOnesValue.
+  bool isAllOnesValue() const;
+
   /// isNegativeZeroValue - Return true if the value is what would be returned 
   /// by getZeroValueForNegation.
   bool isNegativeZeroValue() const;
@@ -128,16 +132,16 @@ class Constant : public User {
     assert(0 && "Constants that do not have operands cannot be using 'From'!");
   }
   
-  static Constant *getNullValue(const Type* Ty);
+  static Constant *getNullValue(Type* Ty);
   
   /// @returns the value for an integer constant of the given type that has all
   /// its bits set to true.
   /// @brief Get the all ones value
-  static Constant *getAllOnesValue(const Type* Ty);
+  static Constant *getAllOnesValue(Type* Ty);
 
   /// getIntegerValue - Return the value for an integer or pointer constant,
   /// or a vector thereof, with the given scalar value.
-  static Constant *getIntegerValue(const Type* Ty, const APInt &V);
+  static Constant *getIntegerValue(Type* Ty, const APInt &V);
   
   /// removeDeadConstantUsers - If there are any dead constant users dangling
   /// off of this constant, remove them.  This method is useful for clients
diff --git a/contrib/llvm/include/llvm/Constants.h b/contrib/llvm/include/llvm/Constants.h
index 01fca291843a..6545a3fedb92 100644
--- a/contrib/llvm/include/llvm/Constants.h
+++ b/contrib/llvm/include/llvm/Constants.h
@@ -47,7 +47,7 @@ struct ConvertConstantType;
 class ConstantInt : public Constant {
   void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
   ConstantInt(const ConstantInt &);      // DO NOT IMPLEMENT
-  ConstantInt(const IntegerType *Ty, const APInt& V);
+  ConstantInt(IntegerType *Ty, const APInt& V);
   APInt Val;
 protected:
   // allocate space for exactly zero operands
@@ -57,12 +57,12 @@ class ConstantInt : public Constant {
 public:
   static ConstantInt *getTrue(LLVMContext &Context);
   static ConstantInt *getFalse(LLVMContext &Context);
-  static Constant *getTrue(const Type *Ty);
-  static Constant *getFalse(const Type *Ty);
+  static Constant *getTrue(Type *Ty);
+  static Constant *getFalse(Type *Ty);
   
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantInt for the given value.
-  static Constant *get(const Type *Ty, uint64_t V, bool isSigned = false);
+  static Constant *get(Type *Ty, uint64_t V, bool isSigned = false);
                               
   /// Return a ConstantInt with the specified integer value for the specified
   /// type. If the type is wider than 64 bits, the value will be zero-extended
@@ -70,7 +70,7 @@ class ConstantInt : public Constant {
   /// be interpreted as a 64-bit signed integer and sign-extended to fit
   /// the type.
   /// @brief Get a ConstantInt for a specific value.
-  static ConstantInt *get(const IntegerType *Ty, uint64_t V,
+  static ConstantInt *get(IntegerType *Ty, uint64_t V,
                           bool isSigned = false);
 
   /// Return a ConstantInt with the specified value for the specified type. The
@@ -78,8 +78,8 @@ class ConstantInt : public Constant {
   /// either getSExtValue() or getZExtValue() will yield a correctly sized and
   /// signed value for the type Ty.
   /// @brief Get a ConstantInt for a specific signed value.
-  static ConstantInt *getSigned(const IntegerType *Ty, int64_t V);
-  static Constant *getSigned(const Type *Ty, int64_t V);
+  static ConstantInt *getSigned(IntegerType *Ty, int64_t V);
+  static Constant *getSigned(Type *Ty, int64_t V);
   
   /// Return a ConstantInt with the specified value and an implied Type. The
   /// type is the integer type that corresponds to the bit width of the value.
@@ -87,12 +87,12 @@ class ConstantInt : public Constant {
 
   /// Return a ConstantInt constructed from the string strStart with the given
   /// radix. 
-  static ConstantInt *get(const IntegerType *Ty, StringRef Str,
+  static ConstantInt *get(IntegerType *Ty, StringRef Str,
                           uint8_t radix);
   
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantInt for the given value.
-  static Constant *get(const Type* Ty, const APInt& V);
+  static Constant *get(Type* Ty, const APInt& V);
   
   /// Return the constant as an APInt value reference. This allows clients to
   /// obtain a copy of the value, with all its precision in tact.
@@ -133,8 +133,8 @@ class ConstantInt : public Constant {
   /// getType - Specialize the getType() method to always return an IntegerType,
   /// which reduces the amount of casting needed in parts of the compiler.
   ///
-  inline const IntegerType *getType() const {
-    return reinterpret_cast<const IntegerType*>(Value::getType());
+  inline IntegerType *getType() const {
+    return reinterpret_cast<IntegerType*>(Value::getType());
   }
 
   /// This static method returns true if the type Ty is big enough to 
@@ -146,8 +146,8 @@ class ConstantInt : public Constant {
   /// to the appropriate unsigned type before calling the method.
   /// @returns true if V is a valid value for type Ty
   /// @brief Determine if the value is in range for the given type.
-  static bool isValueValidForType(const Type *Ty, uint64_t V);
-  static bool isValueValidForType(const Type *Ty, int64_t V);
+  static bool isValueValidForType(Type *Ty, uint64_t V);
+  static bool isValueValidForType(Type *Ty, int64_t V);
 
   bool isNegative() const { return Val.isNegative(); }
 
@@ -170,7 +170,7 @@ class ConstantInt : public Constant {
   /// to true.
   /// @returns true iff this constant's bits are all set to true.
   /// @brief Determine if the value is all ones.
-  bool isAllOnesValue() const { 
+  bool isMinusOne() const { 
     return Val.isAllOnesValue();
   }
 
@@ -203,7 +203,7 @@ class ConstantInt : public Constant {
   /// value.
   /// @returns true iff this constant is greater or equal to the given number.
   /// @brief Determine if the value is greater or equal to the given number.
-  bool uge(uint64_t Num) {
+  bool uge(uint64_t Num) const {
     return Val.getActiveBits() > 64 || Val.getZExtValue() >= Num;
   }
 
@@ -233,7 +233,7 @@ class ConstantFP : public Constant {
   ConstantFP(const ConstantFP &);      // DO NOT IMPLEMENT
   friend class LLVMContextImpl;
 protected:
-  ConstantFP(const Type *Ty, const APFloat& V);
+  ConstantFP(Type *Ty, const APFloat& V);
 protected:
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
@@ -243,20 +243,20 @@ class ConstantFP : public Constant {
   /// Floating point negation must be implemented with f(x) = -0.0 - x. This
   /// method returns the negative zero constant for floating point or vector
   /// floating point types; for all other types, it returns the null value.
-  static Constant *getZeroValueForNegation(const Type *Ty);
+  static Constant *getZeroValueForNegation(Type *Ty);
   
   /// get() - This returns a ConstantFP, or a vector containing a splat of a
   /// ConstantFP, for the specified value in the specified type.  This should
   /// only be used for simple constant values like 2.0/1.0 etc, that are
   /// known-valid both as host double and as the target format.
-  static Constant *get(const Type* Ty, double V);
-  static Constant *get(const Type* Ty, StringRef Str);
+  static Constant *get(Type* Ty, double V);
+  static Constant *get(Type* Ty, StringRef Str);
   static ConstantFP *get(LLVMContext &Context, const APFloat &V);
-  static ConstantFP *getNegativeZero(const Type* Ty);
-  static ConstantFP *getInfinity(const Type *Ty, bool Negative = false);
+  static ConstantFP *getNegativeZero(Type* Ty);
+  static ConstantFP *getInfinity(Type *Ty, bool Negative = false);
   
   /// isValueValidForType - return true if Ty is big enough to represent V.
-  static bool isValueValidForType(const Type *Ty, const APFloat &V);
+  static bool isValueValidForType(Type *Ty, const APFloat &V);
   inline const APFloat &getValueAPF() const { return Val; }
 
   /// isZero - Return true if the value is positive or negative zero.
@@ -300,7 +300,7 @@ class ConstantAggregateZero : public Constant {
   void *operator new(size_t, unsigned);                      // DO NOT IMPLEMENT
   ConstantAggregateZero(const ConstantAggregateZero &);      // DO NOT IMPLEMENT
 protected:
-  explicit ConstantAggregateZero(const Type *ty)
+  explicit ConstantAggregateZero(Type *ty)
     : Constant(ty, ConstantAggregateZeroVal, 0, 0) {}
 protected:
   // allocate space for exactly zero operands
@@ -308,7 +308,7 @@ class ConstantAggregateZero : public Constant {
     return User::operator new(s, 0);
   }
 public:
-  static ConstantAggregateZero* get(const Type *Ty);
+  static ConstantAggregateZero* get(Type *Ty);
   
   virtual void destroyConstant();
 
@@ -329,10 +329,10 @@ class ConstantArray : public Constant {
                                     std::vector<Constant*> >;
   ConstantArray(const ConstantArray &);      // DO NOT IMPLEMENT
 protected:
-  ConstantArray(const ArrayType *T, const std::vector<Constant*> &Val);
+  ConstantArray(ArrayType *T, ArrayRef<Constant *> Val);
 public:
   // ConstantArray accessors
-  static Constant *get(const ArrayType *T, ArrayRef<Constant*> V);
+  static Constant *get(ArrayType *T, ArrayRef<Constant*> V);
                              
   /// This method constructs a ConstantArray and initializes it with a text
   /// string. The default behavior (AddNull==true) causes a null terminator to
@@ -349,8 +349,8 @@ class ConstantArray : public Constant {
   /// getType - Specialize the getType() method to always return an ArrayType,
   /// which reduces the amount of casting needed in parts of the compiler.
   ///
-  inline const ArrayType *getType() const {
-    return reinterpret_cast<const ArrayType*>(Value::getType());
+  inline ArrayType *getType() const {
+    return reinterpret_cast<ArrayType*>(Value::getType());
   }
 
   /// isString - This method returns true if the array is an array of i8 and
@@ -390,7 +390,7 @@ struct OperandTraits<ConstantArray> :
   public VariadicOperandTraits<ConstantArray> {
 };
 
-DEFINE_TRANSPARENT_CASTED_OPERAND_ACCESSORS(ConstantArray, Constant)
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantArray, Constant)
 
 //===----------------------------------------------------------------------===//
 // ConstantStruct - Constant Struct Declarations
@@ -400,11 +400,11 @@ class ConstantStruct : public Constant {
                                     std::vector<Constant*> >;
   ConstantStruct(const ConstantStruct &);      // DO NOT IMPLEMENT
 protected:
-  ConstantStruct(const StructType *T, const std::vector<Constant*> &Val);
+  ConstantStruct(StructType *T, ArrayRef<Constant *> Val);
 public:
   // ConstantStruct accessors
-  static Constant *get(const StructType *T, ArrayRef<Constant*> V);
-  static Constant *get(const StructType *T, ...) END_WITH_NULL;
+  static Constant *get(StructType *T, ArrayRef<Constant*> V);
+  static Constant *get(StructType *T, ...) END_WITH_NULL;
 
   /// getAnon - Return an anonymous struct that has the specified
   /// elements.  If the struct is possibly empty, then you must specify a
@@ -431,8 +431,8 @@ class ConstantStruct : public Constant {
 
   /// getType() specialization - Reduce amount of casting...
   ///
-  inline const StructType *getType() const {
-    return reinterpret_cast<const StructType*>(Value::getType());
+  inline StructType *getType() const {
+    return reinterpret_cast<StructType*>(Value::getType());
   }
 
   virtual void destroyConstant();
@@ -450,7 +450,7 @@ struct OperandTraits<ConstantStruct> :
   public VariadicOperandTraits<ConstantStruct> {
 };
 
-DEFINE_TRANSPARENT_CASTED_OPERAND_ACCESSORS(ConstantStruct, Constant)
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantStruct, Constant)
 
 
 //===----------------------------------------------------------------------===//
@@ -461,7 +461,7 @@ class ConstantVector : public Constant {
                                     std::vector<Constant*> >;
   ConstantVector(const ConstantVector &);      // DO NOT IMPLEMENT
 protected:
-  ConstantVector(const VectorType *T, const std::vector<Constant*> &Val);
+  ConstantVector(VectorType *T, ArrayRef<Constant *> Val);
 public:
   // ConstantVector accessors
   static Constant *get(ArrayRef<Constant*> V);
@@ -472,8 +472,8 @@ class ConstantVector : public Constant {
   /// getType - Specialize the getType() method to always return a VectorType,
   /// which reduces the amount of casting needed in parts of the compiler.
   ///
-  inline const VectorType *getType() const {
-    return reinterpret_cast<const VectorType*>(Value::getType());
+  inline VectorType *getType() const {
+    return reinterpret_cast<VectorType*>(Value::getType());
   }
   
   /// This function will return true iff every element in this vector constant
@@ -501,7 +501,7 @@ struct OperandTraits<ConstantVector> :
   public VariadicOperandTraits<ConstantVector> {
 };
 
-DEFINE_TRANSPARENT_CASTED_OPERAND_ACCESSORS(ConstantVector, Constant)
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantVector, Constant)
 
 //===----------------------------------------------------------------------===//
 /// ConstantPointerNull - a constant pointer value that points to null
@@ -511,8 +511,8 @@ class ConstantPointerNull : public Constant {
   void *operator new(size_t, unsigned);                  // DO NOT IMPLEMENT
   ConstantPointerNull(const ConstantPointerNull &);      // DO NOT IMPLEMENT
 protected:
-  explicit ConstantPointerNull(const PointerType *T)
-    : Constant(reinterpret_cast<const Type*>(T),
+  explicit ConstantPointerNull(PointerType *T)
+    : Constant(reinterpret_cast<Type*>(T),
                Value::ConstantPointerNullVal, 0, 0) {}
 
 protected:
@@ -522,15 +522,15 @@ class ConstantPointerNull : public Constant {
   }
 public:
   /// get() - Static factory methods - Return objects of the specified value
-  static ConstantPointerNull *get(const PointerType *T);
+  static ConstantPointerNull *get(PointerType *T);
 
   virtual void destroyConstant();
 
   /// getType - Specialize the getType() method to always return an PointerType,
   /// which reduces the amount of casting needed in parts of the compiler.
   ///
-  inline const PointerType *getType() const {
-    return reinterpret_cast<const PointerType*>(Value::getType());
+  inline PointerType *getType() const {
+    return reinterpret_cast<PointerType*>(Value::getType());
   }
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -575,7 +575,7 @@ struct OperandTraits<BlockAddress> :
   public FixedNumOperandTraits<BlockAddress, 2> {
 };
 
-DEFINE_TRANSPARENT_CASTED_OPERAND_ACCESSORS(BlockAddress, Value)
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value)
   
 
 //===----------------------------------------------------------------------===//
@@ -591,7 +591,7 @@ class ConstantExpr : public Constant {
   friend struct ConvertConstantType<ConstantExpr, Type>;
 
 protected:
-  ConstantExpr(const Type *ty, unsigned Opcode, Use *Ops, unsigned NumOps)
+  ConstantExpr(Type *ty, unsigned Opcode, Use *Ops, unsigned NumOps)
     : Constant(ty, ConstantExprVal, Ops, NumOps) {
     // Operation type (an Instruction opcode) is stored as the SubclassData.
     setValueSubclassData(Opcode);
@@ -605,23 +605,23 @@ class ConstantExpr : public Constant {
 
   /// getAlignOf constant expr - computes the alignment of a type in a target
   /// independent way (Note: the return type is an i64).
-  static Constant *getAlignOf(const Type *Ty);
+  static Constant *getAlignOf(Type *Ty);
   
   /// getSizeOf constant expr - computes the (alloc) size of a type (in
   /// address-units, not bits) in a target independent way (Note: the return
   /// type is an i64).
   ///
-  static Constant *getSizeOf(const Type *Ty);
+  static Constant *getSizeOf(Type *Ty);
 
   /// getOffsetOf constant expr - computes the offset of a struct field in a 
   /// target independent way (Note: the return type is an i64).
   ///
-  static Constant *getOffsetOf(const StructType *STy, unsigned FieldNo);
+  static Constant *getOffsetOf(StructType *STy, unsigned FieldNo);
 
   /// getOffsetOf constant expr - This is a generalized form of getOffsetOf,
   /// which supports any aggregate type, and any Constant index.
   ///
-  static Constant *getOffsetOf(const Type *Ty, Constant *FieldNo);
+  static Constant *getOffsetOf(Type *Ty, Constant *FieldNo);
   
   static Constant *getNeg(Constant *C, bool HasNUW = false, bool HasNSW =false);
   static Constant *getFNeg(Constant *C);
@@ -648,18 +648,18 @@ class ConstantExpr : public Constant {
                           bool HasNUW = false, bool HasNSW = false);
   static Constant *getLShr(Constant *C1, Constant *C2, bool isExact = false);
   static Constant *getAShr(Constant *C1, Constant *C2, bool isExact = false);
-  static Constant *getTrunc   (Constant *C, const Type *Ty);
-  static Constant *getSExt    (Constant *C, const Type *Ty);
-  static Constant *getZExt    (Constant *C, const Type *Ty);
-  static Constant *getFPTrunc (Constant *C, const Type *Ty);
-  static Constant *getFPExtend(Constant *C, const Type *Ty);
-  static Constant *getUIToFP  (Constant *C, const Type *Ty);
-  static Constant *getSIToFP  (Constant *C, const Type *Ty);
-  static Constant *getFPToUI  (Constant *C, const Type *Ty);
-  static Constant *getFPToSI  (Constant *C, const Type *Ty);
-  static Constant *getPtrToInt(Constant *C, const Type *Ty);
-  static Constant *getIntToPtr(Constant *C, const Type *Ty);
-  static Constant *getBitCast (Constant *C, const Type *Ty);
+  static Constant *getTrunc   (Constant *C, Type *Ty);
+  static Constant *getSExt    (Constant *C, Type *Ty);
+  static Constant *getZExt    (Constant *C, Type *Ty);
+  static Constant *getFPTrunc (Constant *C, Type *Ty);
+  static Constant *getFPExtend(Constant *C, Type *Ty);
+  static Constant *getUIToFP  (Constant *C, Type *Ty);
+  static Constant *getSIToFP  (Constant *C, Type *Ty);
+  static Constant *getFPToUI  (Constant *C, Type *Ty);
+  static Constant *getFPToSI  (Constant *C, Type *Ty);
+  static Constant *getPtrToInt(Constant *C, Type *Ty);
+  static Constant *getIntToPtr(Constant *C, Type *Ty);
+  static Constant *getBitCast (Constant *C, Type *Ty);
 
   static Constant *getNSWNeg(Constant *C) { return getNeg(C, false, true); }
   static Constant *getNUWNeg(Constant *C) { return getNeg(C, true, false); }
@@ -708,44 +708,44 @@ class ConstantExpr : public Constant {
   static Constant *getCast(
     unsigned ops,  ///< The opcode for the conversion
     Constant *C,   ///< The constant to be converted
-    const Type *Ty ///< The type to which the constant is converted
+    Type *Ty ///< The type to which the constant is converted
   );
 
   // @brief Create a ZExt or BitCast cast constant expression
   static Constant *getZExtOrBitCast(
     Constant *C,   ///< The constant to zext or bitcast
-    const Type *Ty ///< The type to zext or bitcast C to
+    Type *Ty ///< The type to zext or bitcast C to
   );
 
   // @brief Create a SExt or BitCast cast constant expression 
   static Constant *getSExtOrBitCast(
     Constant *C,   ///< The constant to sext or bitcast
-    const Type *Ty ///< The type to sext or bitcast C to
+    Type *Ty ///< The type to sext or bitcast C to
   );
 
   // @brief Create a Trunc or BitCast cast constant expression
   static Constant *getTruncOrBitCast(
     Constant *C,   ///< The constant to trunc or bitcast
-    const Type *Ty ///< The type to trunc or bitcast C to
+    Type *Ty ///< The type to trunc or bitcast C to
   );
 
   /// @brief Create a BitCast or a PtrToInt cast constant expression
   static Constant *getPointerCast(
     Constant *C,   ///< The pointer value to be casted (operand 0)
-    const Type *Ty ///< The type to which cast should be made
+    Type *Ty ///< The type to which cast should be made
   );
 
   /// @brief Create a ZExt, Bitcast or Trunc for integer -> integer casts
   static Constant *getIntegerCast(
     Constant *C,    ///< The integer constant to be casted 
-    const Type *Ty, ///< The integer type to cast to
+    Type *Ty, ///< The integer type to cast to
     bool isSigned   ///< Whether C should be treated as signed or not
   );
 
   /// @brief Create a FPExt, Bitcast or FPTrunc for fp -> fp casts
   static Constant *getFPCast(
     Constant *C,    ///< The integer constant to be casted 
-    const Type *Ty ///< The integer type to cast to
+    Type *Ty ///< The integer type to cast to
   );
 
   /// @brief Return true if this is a convert constant expression
@@ -788,25 +788,40 @@ class ConstantExpr : public Constant {
   /// all elements must be Constant's.
   ///
   static Constant *getGetElementPtr(Constant *C,
-                                    Constant *const *IdxList, unsigned NumIdx,
+                                    ArrayRef<Constant *> IdxList,
                                     bool InBounds = false) {
-    return getGetElementPtr(C, (Value**)IdxList, NumIdx, InBounds);
+    return getGetElementPtr(C, makeArrayRef((Value * const *)IdxList.data(),
+                                            IdxList.size()),
+                            InBounds);
   }
   static Constant *getGetElementPtr(Constant *C,
-                                    Value *const *IdxList, unsigned NumIdx,
+                                    Constant *Idx,
+                                    bool InBounds = false) {
+    // This form of the function only exists to avoid ambiguous overload
+    // warnings about whether to convert Idx to ArrayRef<Constant *> or
+    // ArrayRef<Value *>.
+    return getGetElementPtr(C, cast<Value>(Idx), InBounds);
+  }
+  static Constant *getGetElementPtr(Constant *C,
+                                    ArrayRef<Value *> IdxList,
                                     bool InBounds = false);
 
   /// Create an "inbounds" getelementptr. See the documentation for the
   /// "inbounds" flag in LangRef.html for details.
   static Constant *getInBoundsGetElementPtr(Constant *C,
-                                            Constant *const *IdxList,
-                                            unsigned NumIdx) {
-    return getGetElementPtr(C, IdxList, NumIdx, true);
+                                            ArrayRef<Constant *> IdxList) {
+    return getGetElementPtr(C, IdxList, true);
   }
   static Constant *getInBoundsGetElementPtr(Constant *C,
-                                            Value* const *IdxList,
-                                            unsigned NumIdx) {
-    return getGetElementPtr(C, IdxList, NumIdx, true);
+                                            Constant *Idx) {
+    // This form of the function only exists to avoid ambiguous overload
+    // warnings about whether to convert Idx to ArrayRef<Constant *> or
+    // ArrayRef<Value *>.
+    return getGetElementPtr(C, Idx, true);
+  }
+  static Constant *getInBoundsGetElementPtr(Constant *C,
+                                            ArrayRef<Value *> IdxList) {
+    return getGetElementPtr(C, IdxList, true);
   }
 
   static Constant *getExtractElement(Constant *Vec, Constant *Idx);
@@ -845,7 +860,7 @@ class ConstantExpr : public Constant {
   /// operands replaced with the specified values and with the specified result
   /// type.  The specified array must have the same number of operands as our
   /// current one.
-  Constant *getWithOperands(ArrayRef<Constant*> Ops, const Type *Ty) const;
+  Constant *getWithOperands(ArrayRef<Constant*> Ops, Type *Ty) const;
 
   virtual void destroyConstant();
   virtual void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U);
@@ -869,7 +884,7 @@ struct OperandTraits<ConstantExpr> :
   public VariadicOperandTraits<ConstantExpr, 1> {
 };
 
-DEFINE_TRANSPARENT_CASTED_OPERAND_ACCESSORS(ConstantExpr, Constant)
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantExpr, Constant)
 
 //===----------------------------------------------------------------------===//
 /// UndefValue - 'undef' values are things that do not have specified contents.
@@ -886,7 +901,7 @@ class UndefValue : public Constant {
   void *operator new(size_t, unsigned); // DO NOT IMPLEMENT
   UndefValue(const UndefValue &);      // DO NOT IMPLEMENT
 protected:
-  explicit UndefValue(const Type *T) : Constant(T, UndefValueVal, 0, 0) {}
+  explicit UndefValue(Type *T) : Constant(T, UndefValueVal, 0, 0) {}
 protected:
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
@@ -896,7 +911,7 @@ class UndefValue : public Constant {
   /// get() - Static factory methods - Return an 'undef' object of the specified
   /// type.
   ///
-  static UndefValue *get(const Type *T);
+  static UndefValue *get(Type *T);
 
   virtual void destroyConstant();
 
diff --git a/contrib/llvm/include/llvm/DebugInfo/DIContext.h b/contrib/llvm/include/llvm/DebugInfo/DIContext.h
new file mode 100644
index 000000000000..64f80c506504
--- /dev/null
+++ b/contrib/llvm/include/llvm/DebugInfo/DIContext.h
@@ -0,0 +1,68 @@
+//===-- DIContext.h ---------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines DIContext, an abstract data structure that holds
+// debug information data.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DICONTEXT_H
+#define LLVM_DEBUGINFO_DICONTEXT_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/DataTypes.h"
+#include <cstring>
+
+namespace llvm {
+
+class raw_ostream;
+
+/// DILineInfo - a format-neutral container for source line information.
+class DILineInfo {
+  const char *FileName;
+  uint32_t Line;
+  uint32_t Column;
+public:
+  DILineInfo() : FileName("<invalid>"), Line(0), Column(0) {}
+  DILineInfo(const char *fileName, uint32_t line, uint32_t column)
+    : FileName(fileName), Line(line), Column(column) {}
+
+  const char *getFileName() const { return FileName; }
+  uint32_t getLine() const { return Line; }
+  uint32_t getColumn() const { return Column; }
+
+  bool operator==(const DILineInfo &RHS) const {
+    return Line == RHS.Line && Column == RHS.Column &&
+           std::strcmp(FileName, RHS.FileName) == 0;
+  }
+  bool operator!=(const DILineInfo &RHS) const {
+    return !(*this == RHS);
+  }
+};
+
+class DIContext {
+public:
+  virtual ~DIContext();
+
+  /// getDWARFContext - get a context for binary DWARF data.
+  static DIContext *getDWARFContext(bool isLittleEndian,
+                                    StringRef infoSection,
+                                    StringRef abbrevSection,
+                                    StringRef aRangeSection = StringRef(),
+                                    StringRef lineSection = StringRef(),
+                                    StringRef stringSection = StringRef());
+
+  virtual void dump(raw_ostream &OS) = 0;
+
+  virtual DILineInfo getLineInfoForAddress(uint64_t address) = 0;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/include/llvm/DerivedTypes.h b/contrib/llvm/include/llvm/DerivedTypes.h
index acb28deada10..445c3deb7ce2 100644
--- a/contrib/llvm/include/llvm/DerivedTypes.h
+++ b/contrib/llvm/include/llvm/DerivedTypes.h
@@ -96,26 +96,26 @@ class IntegerType : public Type {
 class FunctionType : public Type {
   FunctionType(const FunctionType &);                   // Do not implement
   const FunctionType &operator=(const FunctionType &);  // Do not implement
-  FunctionType(const Type *Result, ArrayRef<Type*> Params, bool IsVarArgs);
+  FunctionType(Type *Result, ArrayRef<Type*> Params, bool IsVarArgs);
 
 public:
   /// FunctionType::get - This static method is the primary way of constructing
   /// a FunctionType.
   ///
-  static FunctionType *get(const Type *Result,
+  static FunctionType *get(Type *Result,
                            ArrayRef<Type*> Params, bool isVarArg);
 
   /// FunctionType::get - Create a FunctionType taking no parameters.
   ///
-  static FunctionType *get(const Type *Result, bool isVarArg);
+  static FunctionType *get(Type *Result, bool isVarArg);
   
   /// isValidReturnType - Return true if the specified type is valid as a return
   /// type.
-  static bool isValidReturnType(const Type *RetTy);
+  static bool isValidReturnType(Type *RetTy);
 
   /// isValidArgumentType - Return true if the specified type is valid as an
   /// argument type.
-  static bool isValidArgumentType(const Type *ArgTy);
+  static bool isValidArgumentType(Type *ArgTy);
 
   bool isVarArg() const { return getSubclassData(); }
   Type *getReturnType() const { return ContainedTys[0]; }
@@ -150,8 +150,8 @@ class CompositeType : public Type {
   /// getTypeAtIndex - Given an index value into the type, return the type of
   /// the element.
   ///
-  Type *getTypeAtIndex(const Value *V) const;
-  Type *getTypeAtIndex(unsigned Idx) const;
+  Type *getTypeAtIndex(const Value *V);
+  Type *getTypeAtIndex(unsigned Idx);
   bool indexValid(const Value *V) const;
   bool indexValid(unsigned Idx) const;
 
@@ -166,10 +166,25 @@ class CompositeType : public Type {
 };
 
 
-/// StructType - Class to represent struct types, both normal and packed.
-/// Besides being optionally packed, structs can be either "anonymous" or may
-/// have an identity.  Anonymous structs are uniqued by structural equivalence,
-/// but types are each unique when created, and optionally have a name.
+/// StructType - Class to represent struct types.  There are two different kinds
+/// of struct types: Literal structs and Identified structs.
+///
+/// Literal struct types (e.g. { i32, i32 }) are uniqued structurally, and must
+/// always have a body when created.  You can get one of these by using one of
+/// the StructType::get() forms.
+///  
+/// Identified structs (e.g. %foo or %42) may optionally have a name and are not
+/// uniqued.  The names for identified structs are managed at the LLVMContext
+/// level, so there can only be a single identified struct with a given name in
+/// a particular LLVMContext.  Identified structs may also optionally be opaque
+/// (have no body specified).  You get one of these by using one of the
+/// StructType::create() forms.
+///
+/// Independent of what kind of struct you have, the body of a struct type are
+/// laid out in memory consequtively with the elements directly one after the
+/// other (if the struct is packed) or (if not packed) with padding between the
+/// elements as defined by TargetData (which is required to match what the code
+/// generator for a target expects).
 ///
 class StructType : public CompositeType {
   StructType(const StructType &);                   // Do not implement
@@ -180,13 +195,13 @@ class StructType : public CompositeType {
     // This is the contents of the SubClassData field.
     SCDB_HasBody = 1,
     SCDB_Packed = 2,
-    SCDB_IsAnonymous = 4
+    SCDB_IsLiteral = 4
   };
   
   /// SymbolTableEntry - For a named struct that actually has a name, this is a
   /// pointer to the symbol table entry (maintained by LLVMContext) for the
-  /// struct.  This is null if the type is an anonymous struct or if it is
-  /// a named type that has an empty name.
+  /// struct.  This is null if the type is an literal struct or if it is
+  /// a identified type that has an empty name.
   /// 
   void *SymbolTableEntry;
 public:
@@ -194,20 +209,23 @@ class StructType : public CompositeType {
     delete [] ContainedTys; // Delete the body.
   }
 
-  /// StructType::createNamed - This creates a named struct with no body
-  /// specified.  If the name is empty, it creates an unnamed struct, which has
-  /// a unique identity but no actual name.
-  static StructType *createNamed(LLVMContext &Context, StringRef Name);
+  /// StructType::create - This creates an identified struct.
+  static StructType *create(LLVMContext &Context, StringRef Name);
+  static StructType *create(LLVMContext &Context);
   
-  static StructType *createNamed(StringRef Name, ArrayRef<Type*> Elements,
-                                 bool isPacked = false);
-  static StructType *createNamed(LLVMContext &Context, StringRef Name,
-                                 ArrayRef<Type*> Elements,
-                                 bool isPacked = false);
-  static StructType *createNamed(StringRef Name, Type *elt1, ...) END_WITH_NULL;
+  static StructType *create(ArrayRef<Type*> Elements,
+                            StringRef Name,
+                            bool isPacked = false);
+  static StructType *create(ArrayRef<Type*> Elements);
+  static StructType *create(LLVMContext &Context,
+                            ArrayRef<Type*> Elements,
+                            StringRef Name,
+                            bool isPacked = false);
+  static StructType *create(LLVMContext &Context, ArrayRef<Type*> Elements);
+  static StructType *create(StringRef Name, Type *elt1, ...) END_WITH_NULL;
 
   /// StructType::get - This static method is the primary way to create a
-  /// StructType.
+  /// literal StructType.
   static StructType *get(LLVMContext &Context, ArrayRef<Type*> Elements,
                          bool isPacked = false);
 
@@ -223,9 +241,9 @@ class StructType : public CompositeType {
 
   bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; }
   
-  /// isAnonymous - Return true if this type is uniqued by structural
-  /// equivalence, false if it has an identity.
-  bool isAnonymous() const {return (getSubclassData() & SCDB_IsAnonymous) != 0;}
+  /// isLiteral - Return true if this type is uniqued by structural
+  /// equivalence, false if it is a struct definition.
+  bool isLiteral() const { return (getSubclassData() & SCDB_IsLiteral) != 0; }
   
   /// isOpaque - Return true if this is a type with an identity that has no body
   /// specified yet.  These prints as 'opaque' in .ll files.
@@ -236,21 +254,21 @@ class StructType : public CompositeType {
   
   /// getName - Return the name for this struct type if it has an identity.
   /// This may return an empty string for an unnamed struct type.  Do not call
-  /// this on an anonymous type.
+  /// this on an literal type.
   StringRef getName() const;
   
   /// setName - Change the name of this type to the specified name, or to a name
-  /// with a suffix if there is a collision.  Do not call this on an anonymous
+  /// with a suffix if there is a collision.  Do not call this on an literal
   /// type.
   void setName(StringRef Name);
 
-  /// setBody - Specify a body for an opaque type.
+  /// setBody - Specify a body for an opaque identified type.
   void setBody(ArrayRef<Type*> Elements, bool isPacked = false);
   void setBody(Type *elt1, ...) END_WITH_NULL;
   
   /// isValidElementType - Return true if the specified type is valid as a
   /// element type.
-  static bool isValidElementType(const Type *ElemTy);
+  static bool isValidElementType(Type *ElemTy);
   
 
   // Iterator access to the elements.
@@ -260,7 +278,7 @@ class StructType : public CompositeType {
 
   /// isLayoutIdentical - Return true if this is layout identical to the
   /// specified struct.
-  bool isLayoutIdentical(const StructType *Other) const;  
+  bool isLayoutIdentical(StructType *Other) const;  
   
   // Random access to the elements
   unsigned getNumElements() const { return NumContainedTys; }
@@ -321,11 +339,11 @@ class ArrayType : public SequentialType {
   /// ArrayType::get - This static method is the primary way to construct an
   /// ArrayType
   ///
-  static ArrayType *get(const Type *ElementType, uint64_t NumElements);
+  static ArrayType *get(Type *ElementType, uint64_t NumElements);
 
   /// isValidElementType - Return true if the specified type is valid as a
   /// element type.
-  static bool isValidElementType(const Type *ElemTy);
+  static bool isValidElementType(Type *ElemTy);
 
   uint64_t getNumElements() const { return NumElements; }
 
@@ -348,13 +366,13 @@ class VectorType : public SequentialType {
   /// VectorType::get - This static method is the primary way to construct an
   /// VectorType.
   ///
-  static VectorType *get(const Type *ElementType, unsigned NumElements);
+  static VectorType *get(Type *ElementType, unsigned NumElements);
 
   /// VectorType::getInteger - This static method gets a VectorType with the
   /// same number of elements as the input type, and the element type is an
   /// integer type of the same width as the input element type.
   ///
-  static VectorType *getInteger(const VectorType *VTy) {
+  static VectorType *getInteger(VectorType *VTy) {
     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
     Type *EltTy = IntegerType::get(VTy->getContext(), EltBits);
     return VectorType::get(EltTy, VTy->getNumElements());
@@ -364,7 +382,7 @@ class VectorType : public SequentialType {
   /// getInteger except that the element types are twice as wide as the
   /// elements in the input type.
   ///
-  static VectorType *getExtendedElementVectorType(const VectorType *VTy) {
+  static VectorType *getExtendedElementVectorType(VectorType *VTy) {
     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
     Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2);
     return VectorType::get(EltTy, VTy->getNumElements());
@@ -374,7 +392,7 @@ class VectorType : public SequentialType {
   /// getInteger except that the element types are half as wide as the
   /// elements in the input type.
   ///
-  static VectorType *getTruncatedElementVectorType(const VectorType *VTy) {
+  static VectorType *getTruncatedElementVectorType(VectorType *VTy) {
     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
     assert((EltBits & 1) == 0 &&
            "Cannot truncate vector element with odd bit-width");
@@ -384,7 +402,7 @@ class VectorType : public SequentialType {
 
   /// isValidElementType - Return true if the specified type is valid as a
   /// element type.
-  static bool isValidElementType(const Type *ElemTy);
+  static bool isValidElementType(Type *ElemTy);
 
   /// @brief Return the number of elements in the Vector type.
   unsigned getNumElements() const { return NumElements; }
@@ -411,17 +429,17 @@ class PointerType : public SequentialType {
 public:
   /// PointerType::get - This constructs a pointer to an object of the specified
   /// type in a numbered address space.
-  static PointerType *get(const Type *ElementType, unsigned AddressSpace);
+  static PointerType *get(Type *ElementType, unsigned AddressSpace);
 
   /// PointerType::getUnqual - This constructs a pointer to an object of the
   /// specified type in the generic address space (address space zero).
-  static PointerType *getUnqual(const Type *ElementType) {
+  static PointerType *getUnqual(Type *ElementType) {
     return PointerType::get(ElementType, 0);
   }
 
   /// isValidElementType - Return true if the specified type is valid as a
   /// element type.
-  static bool isValidElementType(const Type *ElemTy);
+  static bool isValidElementType(Type *ElemTy);
 
   /// @brief Return the address space of the Pointer type.
   inline unsigned getAddressSpace() const { return getSubclassData(); }
diff --git a/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h b/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h
index 88b21cd4d253..cf85671eb414 100644
--- a/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h
+++ b/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h
@@ -18,6 +18,7 @@
 #include <vector>
 #include <map>
 #include <string>
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/ValueMap.h"
@@ -119,9 +120,7 @@ class ExecutionEngine {
   /// optimize for the case where there is only one module.
   SmallVector<Module*, 1> Modules;
 
-  void setTargetData(const TargetData *td) {
-    TD = td;
-  }
+  void setTargetData(const TargetData *td) { TD = td; }
 
   /// getMemoryforGV - Allocate memory for a global variable.
   virtual char *getMemoryForGV(const GlobalVariable *GV);
@@ -143,8 +142,7 @@ class ExecutionEngine {
     CodeGenOpt::Level OptLevel,
     bool GVsWithCode,
     TargetMachine *TM);
-  static ExecutionEngine *(*InterpCtor)(Module *M,
-                                        std::string *ErrorStr);
+  static ExecutionEngine *(*InterpCtor)(Module *M, std::string *ErrorStr);
 
   /// LazyFunctionCreator - If an unknown function is needed, this function
   /// pointer is invoked to create it.  If this returns null, the JIT will
@@ -186,7 +184,7 @@ class ExecutionEngine {
                                  bool ForceInterpreter = false,
                                  std::string *ErrorStr = 0,
                                  CodeGenOpt::Level OptLevel =
-                                   CodeGenOpt::Default,
+                                 CodeGenOpt::Default,
                                  bool GVsWithCode = true);
 
   /// createJIT - This is the factory method for creating a JIT for the current
@@ -199,10 +197,11 @@ class ExecutionEngine {
                                     std::string *ErrorStr = 0,
                                     JITMemoryManager *JMM = 0,
                                     CodeGenOpt::Level OptLevel =
-                                      CodeGenOpt::Default,
+                                    CodeGenOpt::Default,
                                     bool GVsWithCode = true,
+                                    Reloc::Model RM = Reloc::Default,
                                     CodeModel::Model CMM =
-                                      CodeModel::Default);
+                                    CodeModel::JITDefault);
 
   /// addModule - Add a Module to the list of modules that we can JIT from.
   /// Note that this takes ownership of the Module: when the ExecutionEngine is
@@ -314,7 +313,7 @@ class ExecutionEngine {
   /// GenericValue *.  It is not a pointer to a GenericValue containing the
   /// address at which to store Val.
   void StoreValueToMemory(const GenericValue &Val, GenericValue *Ptr,
-                          const Type *Ty);
+                          Type *Ty);
 
   void InitializeMemory(const Constant *Init, void *Addr);
 
@@ -440,7 +439,7 @@ class ExecutionEngine {
 
   GenericValue getConstantValue(const Constant *C);
   void LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr,
-                           const Type *Ty);
+                           Type *Ty);
 };
 
 namespace EngineKind {
@@ -463,6 +462,7 @@ class EngineBuilder {
   CodeGenOpt::Level OptLevel;
   JITMemoryManager *JMM;
   bool AllocateGVsWithCode;
+  Reloc::Model RelocModel;
   CodeModel::Model CMModel;
   std::string MArch;
   std::string MCPU;
@@ -476,7 +476,8 @@ class EngineBuilder {
     OptLevel = CodeGenOpt::Default;
     JMM = NULL;
     AllocateGVsWithCode = false;
-    CMModel = CodeModel::Default;
+    RelocModel = Reloc::Default;
+    CMModel = CodeModel::JITDefault;
     UseMCJIT = false;
   }
 
@@ -517,8 +518,16 @@ class EngineBuilder {
     return *this;
   }
 
+  /// setRelocationModel - Set the relocation model that the ExecutionEngine
+  /// target is using. Defaults to target specific default "Reloc::Default".
+  EngineBuilder &setRelocationModel(Reloc::Model RM) {
+    RelocModel = RM;
+    return *this;
+  }
+
   /// setCodeModel - Set the CodeModel that the ExecutionEngine target
-  /// data is using. Defaults to target specific default "CodeModel::Default".
+  /// data is using. Defaults to target specific default
+  /// "CodeModel::JITDefault".
   EngineBuilder &setCodeModel(CodeModel::Model M) {
     CMModel = M;
     return *this;
@@ -569,6 +578,8 @@ class EngineBuilder {
                                      StringRef MArch,
                                      StringRef MCPU,
                                      const SmallVectorImpl<std::string>& MAttrs,
+                                     Reloc::Model RM,
+                                     CodeModel::Model CM,
                                      std::string *Err);
 
   ExecutionEngine *create();
diff --git a/contrib/llvm/include/llvm/Function.h b/contrib/llvm/include/llvm/Function.h
index 0aa5b2a9fa4c..678651bbf1f8 100644
--- a/contrib/llvm/include/llvm/Function.h
+++ b/contrib/llvm/include/llvm/Function.h
@@ -117,11 +117,11 @@ class Function : public GlobalValue,
   /// function is automatically inserted into the end of the function list for
   /// the module.
   ///
-  Function(const FunctionType *Ty, LinkageTypes Linkage,
+  Function(FunctionType *Ty, LinkageTypes Linkage,
            const Twine &N = "", Module *M = 0);
 
 public:
-  static Function *Create(const FunctionType *Ty, LinkageTypes Linkage,
+  static Function *Create(FunctionType *Ty, LinkageTypes Linkage,
                           const Twine &N = "", Module *M = 0) {
     return new(0) Function(Ty, Linkage, N, M);
   }
diff --git a/contrib/llvm/include/llvm/GlobalAlias.h b/contrib/llvm/include/llvm/GlobalAlias.h
index c3d3c38bd344..164d976588d6 100644
--- a/contrib/llvm/include/llvm/GlobalAlias.h
+++ b/contrib/llvm/include/llvm/GlobalAlias.h
@@ -23,7 +23,6 @@
 namespace llvm {
 
 class Module;
-class Constant;
 template<typename ValueSubClass, typename ItemParentClass>
   class SymbolTableListTraits;
 
@@ -41,11 +40,11 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
   }
   /// GlobalAlias ctor - If a parent module is specified, the alias is
   /// automatically inserted into the end of the specified module's alias list.
-  GlobalAlias(const Type *Ty, LinkageTypes Linkage, const Twine &Name = "",
+  GlobalAlias(Type *Ty, LinkageTypes Linkage, const Twine &Name = "",
               Constant* Aliasee = 0, Module *Parent = 0);
 
   /// Provide fast operand accessors
-  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant);
 
   /// removeFromParent - This method unlinks 'this' from the containing module,
   /// but does not delete it.
@@ -60,10 +59,10 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
   /// set/getAliasee - These methods retrive and set alias target.
   void setAliasee(Constant *GV);
   const Constant *getAliasee() const {
-    return cast_or_null<Constant>(getOperand(0));
+    return getOperand(0);
   }
   Constant *getAliasee() {
-    return cast_or_null<Constant>(getOperand(0));
+    return getOperand(0);
   }
   /// getAliasedGlobal() - Aliasee can be either global or bitcast of
   /// global. This method retrives the global for both aliasee flavours.
@@ -88,7 +87,7 @@ struct OperandTraits<GlobalAlias> :
   public FixedNumOperandTraits<GlobalAlias, 1> {
 };
 
-DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GlobalAlias, Value)
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GlobalAlias, Constant)
 
 } // End llvm namespace
 
diff --git a/contrib/llvm/include/llvm/GlobalValue.h b/contrib/llvm/include/llvm/GlobalValue.h
index d0f0888a227a..63dc4ab6bae0 100644
--- a/contrib/llvm/include/llvm/GlobalValue.h
+++ b/contrib/llvm/include/llvm/GlobalValue.h
@@ -57,7 +57,7 @@ class GlobalValue : public Constant {
   };
 
 protected:
-  GlobalValue(const Type *ty, ValueTy vty, Use *Ops, unsigned NumOps,
+  GlobalValue(Type *ty, ValueTy vty, Use *Ops, unsigned NumOps,
               LinkageTypes linkage, const Twine &Name)
     : Constant(ty, vty, Ops, NumOps), Parent(0),
       Linkage(linkage), Visibility(DefaultVisibility), Alignment(0),
diff --git a/contrib/llvm/include/llvm/GlobalVariable.h b/contrib/llvm/include/llvm/GlobalVariable.h
index bbc09c177e20..034ade1fb031 100644
--- a/contrib/llvm/include/llvm/GlobalVariable.h
+++ b/contrib/llvm/include/llvm/GlobalVariable.h
@@ -50,12 +50,12 @@ class GlobalVariable : public GlobalValue, public ilist_node<GlobalVariable> {
   }
   /// GlobalVariable ctor - If a parent module is specified, the global is
   /// automatically inserted into the end of the specified modules global list.
-  GlobalVariable(const Type *Ty, bool isConstant, LinkageTypes Linkage,
+  GlobalVariable(Type *Ty, bool isConstant, LinkageTypes Linkage,
                  Constant *Initializer = 0, const Twine &Name = "",
                  bool ThreadLocal = false, unsigned AddressSpace = 0);
   /// GlobalVariable ctor - This creates a global and inserts it before the
   /// specified other global.
-  GlobalVariable(Module &M, const Type *Ty, bool isConstant,
+  GlobalVariable(Module &M, Type *Ty, bool isConstant,
                  LinkageTypes Linkage, Constant *Initializer,
                  const Twine &Name,
                  GlobalVariable *InsertBefore = 0, bool ThreadLocal = false,
diff --git a/contrib/llvm/include/llvm/InitializePasses.h b/contrib/llvm/include/llvm/InitializePasses.h
index 4caf8f1b46a7..c91fbf8de812 100644
--- a/contrib/llvm/include/llvm/InitializePasses.h
+++ b/contrib/llvm/include/llvm/InitializePasses.h
@@ -65,7 +65,7 @@ void initializeArgPromotionPass(PassRegistry&);
 void initializeBasicAliasAnalysisPass(PassRegistry&);
 void initializeBasicCallGraphPass(PassRegistry&);
 void initializeBlockExtractorPassPass(PassRegistry&);
-void initializeBlockFrequencyPass(PassRegistry&);
+void initializeBlockFrequencyInfoPass(PassRegistry&);
 void initializeBlockPlacementPass(PassRegistry&);
 void initializeBranchProbabilityInfoPass(PassRegistry&);
 void initializeBreakCriticalEdgesPass(PassRegistry&);
@@ -143,9 +143,8 @@ void initializeLowerAtomicPass(PassRegistry&);
 void initializeLowerExpectIntrinsicPass(PassRegistry&);
 void initializeLowerIntrinsicsPass(PassRegistry&);
 void initializeLowerInvokePass(PassRegistry&);
-void initializeLowerSetJmpPass(PassRegistry&);
 void initializeLowerSwitchPass(PassRegistry&);
-void initializeMachineBlockFrequencyPass(PassRegistry&);
+void initializeMachineBlockFrequencyInfoPass(PassRegistry&);
 void initializeMachineBranchProbabilityInfoPass(PassRegistry&);
 void initializeMachineCSEPass(PassRegistry&);
 void initializeMachineDominatorTreePass(PassRegistry&);
@@ -220,7 +219,6 @@ void initializeStripNonDebugSymbolsPass(PassRegistry&);
 void initializeStripSymbolsPass(PassRegistry&);
 void initializeStrongPHIEliminationPass(PassRegistry&);
 void initializeTailCallElimPass(PassRegistry&);
-void initializeTailDupPass(PassRegistry&);
 void initializeTargetDataPass(PassRegistry&);
 void initializeTargetLibraryInfoPass(PassRegistry&);
 void initializeTwoAddressInstructionPassPass(PassRegistry&);
diff --git a/contrib/llvm/include/llvm/InlineAsm.h b/contrib/llvm/include/llvm/InlineAsm.h
index a98aff178cc4..de5ce4ecafc7 100644
--- a/contrib/llvm/include/llvm/InlineAsm.h
+++ b/contrib/llvm/include/llvm/InlineAsm.h
@@ -43,7 +43,7 @@ class InlineAsm : public Value {
   bool HasSideEffects;
   bool IsAlignStack;
   
-  InlineAsm(const PointerType *Ty, const std::string &AsmString,
+  InlineAsm(PointerType *Ty, const std::string &AsmString,
             const std::string &Constraints, bool hasSideEffects,
             bool isAlignStack);
   virtual ~InlineAsm();
@@ -55,7 +55,7 @@ class InlineAsm : public Value {
 
   /// InlineAsm::get - Return the specified uniqued inline asm string.
   ///
-  static InlineAsm *get(const FunctionType *Ty, StringRef AsmString,
+  static InlineAsm *get(FunctionType *Ty, StringRef AsmString,
                         StringRef Constraints, bool hasSideEffects,
                         bool isAlignStack = false);
   
@@ -79,7 +79,7 @@ class InlineAsm : public Value {
   /// the specified constraint string is legal for the type.  This returns true
   /// if legal, false if not.
   ///
-  static bool Verify(const FunctionType *Ty, StringRef Constraints);
+  static bool Verify(FunctionType *Ty, StringRef Constraints);
 
   // Constraint String Parsing 
   enum ConstraintPrefix {
@@ -219,6 +219,7 @@ class InlineAsm : public Value {
   
   static unsigned getFlagWord(unsigned Kind, unsigned NumOps) {
     assert(((NumOps << 3) & ~0xffff) == 0 && "Too many inline asm operands!");
+    assert(Kind >= Kind_RegUse && Kind <= Kind_Mem && "Invalid Kind");
     return Kind | (NumOps << 3);
   }
   
@@ -227,9 +228,24 @@ class InlineAsm : public Value {
   /// to a previous output operand.
   static unsigned getFlagWordForMatchingOp(unsigned InputFlag,
                                            unsigned MatchedOperandNo) {
+    assert(MatchedOperandNo <= 0x7fff && "Too big matched operand");
+    assert((InputFlag & ~0xffff) == 0 && "High bits already contain data");
     return InputFlag | Flag_MatchingOperand | (MatchedOperandNo << 16);
   }
 
+  /// getFlagWordForRegClass - Augment an existing flag word returned by
+  /// getFlagWord with the required register class for the following register
+  /// operands.
+  /// A tied use operand cannot have a register class, use the register class
+  /// from the def operand instead.
+  static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC) {
+    // Store RC + 1, reserve the value 0 to mean 'no register class'.
+    ++RC;
+    assert(RC <= 0x7fff && "Too large register class ID");
+    assert((InputFlag & ~0xffff) == 0 && "High bits already contain data");
+    return InputFlag | (RC << 16);
+  }
+
   static unsigned getKind(unsigned Flags) {
     return Flags & 7;
   }
@@ -259,6 +275,19 @@ class InlineAsm : public Value {
     return true;
   }
 
+  /// hasRegClassConstraint - Returns true if the flag contains a register
+  /// class constraint.  Sets RC to the register class ID.
+  static bool hasRegClassConstraint(unsigned Flag, unsigned &RC) {
+    if (Flag & Flag_MatchingOperand)
+      return false;
+    unsigned High = Flag >> 16;
+    // getFlagWordForRegClass() uses 0 to mean no register class, and otherwise
+    // stores RC + 1.
+    if (!High)
+      return false;
+    RC = High - 1;
+    return true;
+  }
 
 };
 
diff --git a/contrib/llvm/include/llvm/InstrTypes.h b/contrib/llvm/include/llvm/InstrTypes.h
index cc9ec3ac76e1..a1492f3c141a 100644
--- a/contrib/llvm/include/llvm/InstrTypes.h
+++ b/contrib/llvm/include/llvm/InstrTypes.h
@@ -34,12 +34,12 @@ class LLVMContext;
 ///
 class TerminatorInst : public Instruction {
 protected:
-  TerminatorInst(const Type *Ty, Instruction::TermOps iType,
+  TerminatorInst(Type *Ty, Instruction::TermOps iType,
                  Use *Ops, unsigned NumOps,
                  Instruction *InsertBefore = 0)
     : Instruction(Ty, iType, Ops, NumOps, InsertBefore) {}
 
-  TerminatorInst(const Type *Ty, Instruction::TermOps iType,
+  TerminatorInst(Type *Ty, Instruction::TermOps iType,
                  Use *Ops, unsigned NumOps, BasicBlock *InsertAtEnd)
     : Instruction(Ty, iType, Ops, NumOps, InsertAtEnd) {}
 
@@ -91,12 +91,12 @@ class UnaryInstruction : public Instruction {
   void *operator new(size_t, unsigned);      // Do not implement
 
 protected:
-  UnaryInstruction(const Type *Ty, unsigned iType, Value *V,
+  UnaryInstruction(Type *Ty, unsigned iType, Value *V,
                    Instruction *IB = 0)
     : Instruction(Ty, iType, &Op<0>(), 1, IB) {
     Op<0>() = V;
   }
-  UnaryInstruction(const Type *Ty, unsigned iType, Value *V, BasicBlock *IAE)
+  UnaryInstruction(Type *Ty, unsigned iType, Value *V, BasicBlock *IAE)
     : Instruction(Ty, iType, &Op<0>(), 1, IAE) {
     Op<0>() = V;
   }
@@ -141,9 +141,9 @@ class BinaryOperator : public Instruction {
   void *operator new(size_t, unsigned); // Do not implement
 protected:
   void init(BinaryOps iType);
-  BinaryOperator(BinaryOps iType, Value *S1, Value *S2, const Type *Ty,
+  BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
                  const Twine &Name, Instruction *InsertBefore);
-  BinaryOperator(BinaryOps iType, Value *S1, Value *S2, const Type *Ty,
+  BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
                  const Twine &Name, BasicBlock *InsertAtEnd);
   virtual BinaryOperator *clone_impl() const;
 public:
@@ -390,13 +390,13 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryOperator, Value)
 class CastInst : public UnaryInstruction {
 protected:
   /// @brief Constructor with insert-before-instruction semantics for subclasses
-  CastInst(const Type *Ty, unsigned iType, Value *S,
+  CastInst(Type *Ty, unsigned iType, Value *S,
            const Twine &NameStr = "", Instruction *InsertBefore = 0)
     : UnaryInstruction(Ty, iType, S, InsertBefore) {
     setName(NameStr);
   }
   /// @brief Constructor with insert-at-end-of-block semantics for subclasses
-  CastInst(const Type *Ty, unsigned iType, Value *S,
+  CastInst(Type *Ty, unsigned iType, Value *S,
            const Twine &NameStr, BasicBlock *InsertAtEnd)
     : UnaryInstruction(Ty, iType, S, InsertAtEnd) {
     setName(NameStr);
@@ -411,7 +411,7 @@ class CastInst : public UnaryInstruction {
   static CastInst *Create(
     Instruction::CastOps,    ///< The opcode of the cast instruction
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which cast should be made
+    Type *Ty,          ///< The type to which cast should be made
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
   );
@@ -424,7 +424,7 @@ class CastInst : public UnaryInstruction {
   static CastInst *Create(
     Instruction::CastOps,    ///< The opcode for the cast instruction
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which operand is casted
+    Type *Ty,          ///< The type to which operand is casted
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
@@ -432,7 +432,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a ZExt or BitCast cast instruction
   static CastInst *CreateZExtOrBitCast(
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which cast should be made
+    Type *Ty,          ///< The type to which cast should be made
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
   );
@@ -440,7 +440,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a ZExt or BitCast cast instruction
   static CastInst *CreateZExtOrBitCast(
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which operand is casted
+    Type *Ty,          ///< The type to which operand is casted
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
@@ -448,7 +448,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a SExt or BitCast cast instruction
   static CastInst *CreateSExtOrBitCast(
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which cast should be made
+    Type *Ty,          ///< The type to which cast should be made
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
   );
@@ -456,7 +456,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a SExt or BitCast cast instruction
   static CastInst *CreateSExtOrBitCast(
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which operand is casted
+    Type *Ty,          ///< The type to which operand is casted
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
@@ -464,7 +464,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a BitCast or a PtrToInt cast instruction
   static CastInst *CreatePointerCast(
     Value *S,                ///< The pointer value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which operand is casted
+    Type *Ty,          ///< The type to which operand is casted
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
@@ -472,7 +472,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a BitCast or a PtrToInt cast instruction
   static CastInst *CreatePointerCast(
     Value *S,                ///< The pointer value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which cast should be made
+    Type *Ty,          ///< The type to which cast should be made
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
   );
@@ -480,7 +480,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a ZExt, BitCast, or Trunc for int -> int casts.
   static CastInst *CreateIntegerCast(
     Value *S,                ///< The pointer value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which cast should be made
+    Type *Ty,          ///< The type to which cast should be made
     bool isSigned,           ///< Whether to regard S as signed or not
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
@@ -489,7 +489,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a ZExt, BitCast, or Trunc for int -> int casts.
   static CastInst *CreateIntegerCast(
     Value *S,                ///< The integer value to be casted (operand 0)
-    const Type *Ty,          ///< The integer type to which operand is casted
+    Type *Ty,          ///< The integer type to which operand is casted
     bool isSigned,           ///< Whether to regard S as signed or not
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
@@ -498,7 +498,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
   static CastInst *CreateFPCast(
     Value *S,                ///< The floating point value to be casted
-    const Type *Ty,          ///< The floating point type to cast to
+    Type *Ty,          ///< The floating point type to cast to
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
   );
@@ -506,7 +506,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
   static CastInst *CreateFPCast(
     Value *S,                ///< The floating point value to be casted
-    const Type *Ty,          ///< The floating point type to cast to
+    Type *Ty,          ///< The floating point type to cast to
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
@@ -514,7 +514,7 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a Trunc or BitCast cast instruction
   static CastInst *CreateTruncOrBitCast(
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which cast should be made
+    Type *Ty,          ///< The type to which cast should be made
     const Twine &Name = "", ///< Name for the instruction
     Instruction *InsertBefore = 0 ///< Place to insert the instruction
   );
@@ -522,15 +522,15 @@ class CastInst : public UnaryInstruction {
   /// @brief Create a Trunc or BitCast cast instruction
   static CastInst *CreateTruncOrBitCast(
     Value *S,                ///< The value to be casted (operand 0)
-    const Type *Ty,          ///< The type to which operand is casted
+    Type *Ty,          ///< The type to which operand is casted
     const Twine &Name, ///< The name for the instruction
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
 
   /// @brief Check whether it is valid to call getCastOpcode for these types.
   static bool isCastable(
-    const Type *SrcTy, ///< The Type from which the value should be cast.
-    const Type *DestTy ///< The Type to which the value should be cast.
+    Type *SrcTy, ///< The Type from which the value should be cast.
+    Type *DestTy ///< The Type to which the value should be cast.
   );
 
   /// Returns the opcode necessary to cast Val into Ty using usual casting
@@ -539,7 +539,7 @@ class CastInst : public UnaryInstruction {
   static Instruction::CastOps getCastOpcode(
     const Value *Val, ///< The value to cast
     bool SrcIsSigned, ///< Whether to treat the source as signed
-    const Type *Ty,   ///< The Type to which the value should be casted
+    Type *Ty,   ///< The Type to which the value should be casted
     bool DstIsSigned  ///< Whether to treate the dest. as signed
   );
 
@@ -568,14 +568,14 @@ class CastInst : public UnaryInstruction {
   /// @brief Determine if the described cast is a no-op cast.
   static bool isNoopCast(
     Instruction::CastOps Opcode,  ///< Opcode of cast
-    const Type *SrcTy,   ///< SrcTy of cast
-    const Type *DstTy,   ///< DstTy of cast
-    const Type *IntPtrTy ///< Integer type corresponding to Ptr types, or null
+    Type *SrcTy,   ///< SrcTy of cast
+    Type *DstTy,   ///< DstTy of cast
+    Type *IntPtrTy ///< Integer type corresponding to Ptr types, or null
   );
 
   /// @brief Determine if this cast is a no-op cast.
   bool isNoopCast(
-    const Type *IntPtrTy ///< Integer type corresponding to pointer
+    Type *IntPtrTy ///< Integer type corresponding to pointer
   ) const;
 
   /// Determine how a pair of casts can be eliminated, if they can be at all.
@@ -587,10 +587,10 @@ class CastInst : public UnaryInstruction {
   static unsigned isEliminableCastPair(
     Instruction::CastOps firstOpcode,  ///< Opcode of first cast
     Instruction::CastOps secondOpcode, ///< Opcode of second cast
-    const Type *SrcTy, ///< SrcTy of 1st cast
-    const Type *MidTy, ///< DstTy of 1st cast & SrcTy of 2nd cast
-    const Type *DstTy, ///< DstTy of 2nd cast
-    const Type *IntPtrTy ///< Integer type corresponding to Ptr types, or null
+    Type *SrcTy, ///< SrcTy of 1st cast
+    Type *MidTy, ///< DstTy of 1st cast & SrcTy of 2nd cast
+    Type *DstTy, ///< DstTy of 2nd cast
+    Type *IntPtrTy ///< Integer type corresponding to Ptr types, or null
   );
 
   /// @brief Return the opcode of this CastInst
@@ -599,15 +599,15 @@ class CastInst : public UnaryInstruction {
   }
 
   /// @brief Return the source type, as a convenience
-  const Type* getSrcTy() const { return getOperand(0)->getType(); }
+  Type* getSrcTy() const { return getOperand(0)->getType(); }
   /// @brief Return the destination type, as a convenience
-  const Type* getDestTy() const { return getType(); }
+  Type* getDestTy() const { return getType(); }
 
   /// This method can be used to determine if a cast from S to DstTy using
   /// Opcode op is valid or not.
   /// @returns true iff the proposed cast is valid.
   /// @brief Determine if a cast is valid without creating one.
-  static bool castIsValid(Instruction::CastOps op, Value *S, const Type *DstTy);
+  static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy);
 
   /// @brief Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const CastInst *) { return true; }
@@ -629,11 +629,11 @@ class CmpInst : public Instruction {
   void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
   CmpInst(); // do not implement
 protected:
-  CmpInst(const Type *ty, Instruction::OtherOps op, unsigned short pred,
+  CmpInst(Type *ty, Instruction::OtherOps op, unsigned short pred,
           Value *LHS, Value *RHS, const Twine &Name = "",
           Instruction *InsertBefore = 0);
 
-  CmpInst(const Type *ty, Instruction::OtherOps op, unsigned short pred,
+  CmpInst(Type *ty, Instruction::OtherOps op, unsigned short pred,
           Value *LHS, Value *RHS, const Twine &Name,
           BasicBlock *InsertAtEnd);
 
@@ -825,8 +825,8 @@ class CmpInst : public Instruction {
   }
   
   /// @brief Create a result type for fcmp/icmp
-  static const Type* makeCmpResultType(const Type* opnd_type) {
-    if (const VectorType* vt = dyn_cast<const VectorType>(opnd_type)) {
+  static Type* makeCmpResultType(Type* opnd_type) {
+    if (VectorType* vt = dyn_cast<VectorType>(opnd_type)) {
       return VectorType::get(Type::getInt1Ty(opnd_type->getContext()),
                              vt->getNumElements());
     }
diff --git a/contrib/llvm/include/llvm/Instruction.def b/contrib/llvm/include/llvm/Instruction.def
index 205f30313e76..d36e4be1d912 100644
--- a/contrib/llvm/include/llvm/Instruction.def
+++ b/contrib/llvm/include/llvm/Instruction.def
@@ -100,76 +100,80 @@ HANDLE_TERM_INST  ( 3, Switch     , SwitchInst)
 HANDLE_TERM_INST  ( 4, IndirectBr , IndirectBrInst)
 HANDLE_TERM_INST  ( 5, Invoke     , InvokeInst)
 HANDLE_TERM_INST  ( 6, Unwind     , UnwindInst)
-HANDLE_TERM_INST  ( 7, Unreachable, UnreachableInst)
-  LAST_TERM_INST  ( 7)
+HANDLE_TERM_INST  ( 7, Resume     , ResumeInst)
+HANDLE_TERM_INST  ( 8, Unreachable, UnreachableInst)
+  LAST_TERM_INST  ( 8)
 
 // Standard binary operators...
- FIRST_BINARY_INST( 8)
-HANDLE_BINARY_INST( 8, Add  , BinaryOperator)
-HANDLE_BINARY_INST( 9, FAdd  , BinaryOperator)
-HANDLE_BINARY_INST(10, Sub  , BinaryOperator)
-HANDLE_BINARY_INST(11, FSub  , BinaryOperator)
-HANDLE_BINARY_INST(12, Mul  , BinaryOperator)
-HANDLE_BINARY_INST(13, FMul  , BinaryOperator)
-HANDLE_BINARY_INST(14, UDiv , BinaryOperator)
-HANDLE_BINARY_INST(15, SDiv , BinaryOperator)
-HANDLE_BINARY_INST(16, FDiv , BinaryOperator)
-HANDLE_BINARY_INST(17, URem , BinaryOperator)
-HANDLE_BINARY_INST(18, SRem , BinaryOperator)
-HANDLE_BINARY_INST(19, FRem , BinaryOperator)
+ FIRST_BINARY_INST( 9)
+HANDLE_BINARY_INST( 9, Add  , BinaryOperator)
+HANDLE_BINARY_INST(10, FAdd , BinaryOperator)
+HANDLE_BINARY_INST(11, Sub  , BinaryOperator)
+HANDLE_BINARY_INST(12, FSub , BinaryOperator)
+HANDLE_BINARY_INST(13, Mul  , BinaryOperator)
+HANDLE_BINARY_INST(14, FMul , BinaryOperator)
+HANDLE_BINARY_INST(15, UDiv , BinaryOperator)
+HANDLE_BINARY_INST(16, SDiv , BinaryOperator)
+HANDLE_BINARY_INST(17, FDiv , BinaryOperator)
+HANDLE_BINARY_INST(18, URem , BinaryOperator)
+HANDLE_BINARY_INST(19, SRem , BinaryOperator)
+HANDLE_BINARY_INST(20, FRem , BinaryOperator)
 
 // Logical operators (integer operands)
-HANDLE_BINARY_INST(20, Shl  , BinaryOperator) // Shift left  (logical)
-HANDLE_BINARY_INST(21, LShr , BinaryOperator) // Shift right (logical)
-HANDLE_BINARY_INST(22, AShr , BinaryOperator) // Shift right (arithmetic)
-HANDLE_BINARY_INST(23, And  , BinaryOperator)
-HANDLE_BINARY_INST(24, Or   , BinaryOperator)
-HANDLE_BINARY_INST(25, Xor  , BinaryOperator)
-  LAST_BINARY_INST(25)
+HANDLE_BINARY_INST(21, Shl  , BinaryOperator) // Shift left  (logical)
+HANDLE_BINARY_INST(22, LShr , BinaryOperator) // Shift right (logical)
+HANDLE_BINARY_INST(23, AShr , BinaryOperator) // Shift right (arithmetic)
+HANDLE_BINARY_INST(24, And  , BinaryOperator)
+HANDLE_BINARY_INST(25, Or   , BinaryOperator)
+HANDLE_BINARY_INST(26, Xor  , BinaryOperator)
+  LAST_BINARY_INST(26)
 
 // Memory operators...
- FIRST_MEMORY_INST(26)
-HANDLE_MEMORY_INST(26, Alloca, AllocaInst)  // Stack management
-HANDLE_MEMORY_INST(27, Load  , LoadInst  )  // Memory manipulation instrs
-HANDLE_MEMORY_INST(28, Store , StoreInst )
-HANDLE_MEMORY_INST(29, GetElementPtr, GetElementPtrInst)
-  LAST_MEMORY_INST(29)
+ FIRST_MEMORY_INST(27)
+HANDLE_MEMORY_INST(27, Alloca, AllocaInst)  // Stack management
+HANDLE_MEMORY_INST(28, Load  , LoadInst  )  // Memory manipulation instrs
+HANDLE_MEMORY_INST(29, Store , StoreInst )
+HANDLE_MEMORY_INST(30, GetElementPtr, GetElementPtrInst)
+HANDLE_MEMORY_INST(31, Fence , FenceInst )
+HANDLE_MEMORY_INST(32, AtomicCmpXchg , AtomicCmpXchgInst )
+HANDLE_MEMORY_INST(33, AtomicRMW , AtomicRMWInst )
+  LAST_MEMORY_INST(33)
 
 // Cast operators ...
 // NOTE: The order matters here because CastInst::isEliminableCastPair 
 // NOTE: (see Instructions.cpp) encodes a table based on this ordering.
- FIRST_CAST_INST(30)
-HANDLE_CAST_INST(30, Trunc   , TruncInst   )  // Truncate integers
-HANDLE_CAST_INST(31, ZExt    , ZExtInst    )  // Zero extend integers
-HANDLE_CAST_INST(32, SExt    , SExtInst    )  // Sign extend integers
-HANDLE_CAST_INST(33, FPToUI  , FPToUIInst  )  // floating point -> UInt
-HANDLE_CAST_INST(34, FPToSI  , FPToSIInst  )  // floating point -> SInt
-HANDLE_CAST_INST(35, UIToFP  , UIToFPInst  )  // UInt -> floating point
-HANDLE_CAST_INST(36, SIToFP  , SIToFPInst  )  // SInt -> floating point
-HANDLE_CAST_INST(37, FPTrunc , FPTruncInst )  // Truncate floating point
-HANDLE_CAST_INST(38, FPExt   , FPExtInst   )  // Extend floating point
-HANDLE_CAST_INST(39, PtrToInt, PtrToIntInst)  // Pointer -> Integer
-HANDLE_CAST_INST(40, IntToPtr, IntToPtrInst)  // Integer -> Pointer
-HANDLE_CAST_INST(41, BitCast , BitCastInst )  // Type cast
-  LAST_CAST_INST(41)
+ FIRST_CAST_INST(34)
+HANDLE_CAST_INST(34, Trunc   , TruncInst   )  // Truncate integers
+HANDLE_CAST_INST(35, ZExt    , ZExtInst    )  // Zero extend integers
+HANDLE_CAST_INST(36, SExt    , SExtInst    )  // Sign extend integers
+HANDLE_CAST_INST(37, FPToUI  , FPToUIInst  )  // floating point -> UInt
+HANDLE_CAST_INST(38, FPToSI  , FPToSIInst  )  // floating point -> SInt
+HANDLE_CAST_INST(39, UIToFP  , UIToFPInst  )  // UInt -> floating point
+HANDLE_CAST_INST(40, SIToFP  , SIToFPInst  )  // SInt -> floating point
+HANDLE_CAST_INST(41, FPTrunc , FPTruncInst )  // Truncate floating point
+HANDLE_CAST_INST(42, FPExt   , FPExtInst   )  // Extend floating point
+HANDLE_CAST_INST(43, PtrToInt, PtrToIntInst)  // Pointer -> Integer
+HANDLE_CAST_INST(44, IntToPtr, IntToPtrInst)  // Integer -> Pointer
+HANDLE_CAST_INST(45, BitCast , BitCastInst )  // Type cast
+  LAST_CAST_INST(45)
 
 // Other operators...
- FIRST_OTHER_INST(42)
-HANDLE_OTHER_INST(42, ICmp   , ICmpInst   )  // Integer comparison instruction
-HANDLE_OTHER_INST(43, FCmp   , FCmpInst   )  // Floating point comparison instr.
-HANDLE_OTHER_INST(44, PHI    , PHINode    )  // PHI node instruction
-HANDLE_OTHER_INST(45, Call   , CallInst   )  // Call a function
-HANDLE_OTHER_INST(46, Select , SelectInst )  // select instruction
-HANDLE_OTHER_INST(47, UserOp1, Instruction)  // May be used internally in a pass
-HANDLE_OTHER_INST(48, UserOp2, Instruction)  // Internal to passes only
-HANDLE_OTHER_INST(49, VAArg  , VAArgInst  )  // vaarg instruction
-HANDLE_OTHER_INST(50, ExtractElement, ExtractElementInst)// extract from vector
-HANDLE_OTHER_INST(51, InsertElement, InsertElementInst)  // insert into vector
-HANDLE_OTHER_INST(52, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
-HANDLE_OTHER_INST(53, ExtractValue, ExtractValueInst)// extract from aggregate
-HANDLE_OTHER_INST(54, InsertValue, InsertValueInst)  // insert into aggregate
-
-  LAST_OTHER_INST(54)
+ FIRST_OTHER_INST(46)
+HANDLE_OTHER_INST(46, ICmp   , ICmpInst   )  // Integer comparison instruction
+HANDLE_OTHER_INST(47, FCmp   , FCmpInst   )  // Floating point comparison instr.
+HANDLE_OTHER_INST(48, PHI    , PHINode    )  // PHI node instruction
+HANDLE_OTHER_INST(49, Call   , CallInst   )  // Call a function
+HANDLE_OTHER_INST(50, Select , SelectInst )  // select instruction
+HANDLE_OTHER_INST(51, UserOp1, Instruction)  // May be used internally in a pass
+HANDLE_OTHER_INST(52, UserOp2, Instruction)  // Internal to passes only
+HANDLE_OTHER_INST(53, VAArg  , VAArgInst  )  // vaarg instruction
+HANDLE_OTHER_INST(54, ExtractElement, ExtractElementInst)// extract from vector
+HANDLE_OTHER_INST(55, InsertElement, InsertElementInst)  // insert into vector
+HANDLE_OTHER_INST(56, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
+HANDLE_OTHER_INST(57, ExtractValue, ExtractValueInst)// extract from aggregate
+HANDLE_OTHER_INST(58, InsertValue, InsertValueInst)  // insert into aggregate
+HANDLE_OTHER_INST(59, LandingPad, LandingPadInst)  // Landing pad instruction.
+  LAST_OTHER_INST(59)
 
 #undef  FIRST_TERM_INST
 #undef HANDLE_TERM_INST
diff --git a/contrib/llvm/include/llvm/Instruction.h b/contrib/llvm/include/llvm/Instruction.h
index 89bb9fdf423d..934e890151f0 100644
--- a/contrib/llvm/include/llvm/Instruction.h
+++ b/contrib/llvm/include/llvm/Instruction.h
@@ -223,6 +223,13 @@ class Instruction : public User, public ilist_node<Instruction> {
   ///
   bool mayReadFromMemory() const;
 
+  /// mayReadOrWriteMemory - Return true if this instruction may read or
+  /// write memory.
+  ///
+  bool mayReadOrWriteMemory() const {
+    return mayReadFromMemory() || mayWriteToMemory();
+  }
+
   /// mayThrow - Return true if this instruction may throw an exception.
   ///
   bool mayThrow() const;
@@ -365,9 +372,9 @@ class Instruction : public User, public ilist_node<Instruction> {
     return getSubclassDataFromValue() & ~HasMetadataBit;
   }
   
-  Instruction(const Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
+  Instruction(Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
               Instruction *InsertBefore = 0);
-  Instruction(const Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
+  Instruction(Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
               BasicBlock *InsertAtEnd);
   virtual Instruction *clone_impl() const = 0;
   
diff --git a/contrib/llvm/include/llvm/Instructions.h b/contrib/llvm/include/llvm/Instructions.h
index 0bc9a3b6436e..3faab35bf687 100644
--- a/contrib/llvm/include/llvm/Instructions.h
+++ b/contrib/llvm/include/llvm/Instructions.h
@@ -22,6 +22,7 @@
 #include "llvm/CallingConv.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <iterator>
 
 namespace llvm {
@@ -31,6 +32,22 @@ class ConstantRange;
 class APInt;
 class LLVMContext;
 
+enum AtomicOrdering {
+  NotAtomic = 0,
+  Unordered = 1,
+  Monotonic = 2,
+  // Consume = 3,  // Not specified yet.
+  Acquire = 4,
+  Release = 5,
+  AcquireRelease = 6,
+  SequentiallyConsistent = 7
+};
+
+enum SynchronizationScope {
+  SingleThread = 0,
+  CrossThread = 1
+};
+
 //===----------------------------------------------------------------------===//
 //                                AllocaInst Class
 //===----------------------------------------------------------------------===//
@@ -41,17 +58,17 @@ class AllocaInst : public UnaryInstruction {
 protected:
   virtual AllocaInst *clone_impl() const;
 public:
-  explicit AllocaInst(const Type *Ty, Value *ArraySize = 0,
+  explicit AllocaInst(Type *Ty, Value *ArraySize = 0,
                       const Twine &Name = "", Instruction *InsertBefore = 0);
-  AllocaInst(const Type *Ty, Value *ArraySize,
+  AllocaInst(Type *Ty, Value *ArraySize,
              const Twine &Name, BasicBlock *InsertAtEnd);
 
-  AllocaInst(const Type *Ty, const Twine &Name, Instruction *InsertBefore = 0);
-  AllocaInst(const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd);
+  AllocaInst(Type *Ty, const Twine &Name, Instruction *InsertBefore = 0);
+  AllocaInst(Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd);
 
-  AllocaInst(const Type *Ty, Value *ArraySize, unsigned Align,
+  AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
              const Twine &Name = "", Instruction *InsertBefore = 0);
-  AllocaInst(const Type *Ty, Value *ArraySize, unsigned Align,
+  AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
              const Twine &Name, BasicBlock *InsertAtEnd);
 
   // Out of line virtual method, so the vtable, etc. has a home.
@@ -70,8 +87,8 @@ class AllocaInst : public UnaryInstruction {
 
   /// getType - Overload to return most specific pointer type
   ///
-  const PointerType *getType() const {
-    return reinterpret_cast<const PointerType*>(Instruction::getType());
+  PointerType *getType() const {
+    return reinterpret_cast<PointerType*>(Instruction::getType());
   }
 
   /// getAllocatedType - Return the type that is being allocated by the
@@ -125,12 +142,20 @@ class LoadInst : public UnaryInstruction {
   LoadInst(Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile = false,
            Instruction *InsertBefore = 0);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
-           unsigned Align, Instruction *InsertBefore = 0);
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
            BasicBlock *InsertAtEnd);
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+           unsigned Align, Instruction *InsertBefore = 0);
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, BasicBlock *InsertAtEnd);
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+           unsigned Align, AtomicOrdering Order,
+           SynchronizationScope SynchScope = CrossThread,
+           Instruction *InsertBefore = 0);
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+           unsigned Align, AtomicOrdering Order,
+           SynchronizationScope SynchScope,
+           BasicBlock *InsertAtEnd);
 
   LoadInst(Value *Ptr, const char *NameStr, Instruction *InsertBefore);
   LoadInst(Value *Ptr, const char *NameStr, BasicBlock *InsertAtEnd);
@@ -154,11 +179,47 @@ class LoadInst : public UnaryInstruction {
   /// getAlignment - Return the alignment of the access that is being performed
   ///
   unsigned getAlignment() const {
-    return (1 << (getSubclassDataFromInstruction() >> 1)) >> 1;
+    return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1;
   }
 
   void setAlignment(unsigned Align);
 
+  /// Returns the ordering effect of this fence.
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
+  }
+
+  /// Set the ordering constraint on this load. May not be Release or
+  /// AcquireRelease.
+  void setOrdering(AtomicOrdering Ordering) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
+                               (Ordering << 7));
+  }
+
+  SynchronizationScope getSynchScope() const {
+    return SynchronizationScope((getSubclassDataFromInstruction() >> 6) & 1);
+  }
+
+  /// Specify whether this load is ordered with respect to all
+  /// concurrently executing threads, or only with respect to signal handlers
+  /// executing in the same thread.
+  void setSynchScope(SynchronizationScope xthread) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(1 << 6)) |
+                               (xthread << 6));
+  }
+
+  bool isAtomic() const { return getOrdering() != NotAtomic; }
+  void setAtomic(AtomicOrdering Ordering,
+                 SynchronizationScope SynchScope = CrossThread) {
+    setOrdering(Ordering);
+    setSynchScope(SynchScope);
+  }
+
+  bool isSimple() const { return !isAtomic() && !isVolatile(); }
+  bool isUnordered() const {
+    return getOrdering() <= Unordered && !isVolatile();
+  }
+
   Value *getPointerOperand() { return getOperand(0); }
   const Value *getPointerOperand() const { return getOperand(0); }
   static unsigned getPointerOperandIndex() { return 0U; }
@@ -205,19 +266,27 @@ class StoreInst : public Instruction {
   StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
   StoreInst(Value *Val, Value *Ptr, bool isVolatile = false,
             Instruction *InsertBefore = 0);
-  StoreInst(Value *Val, Value *Ptr, bool isVolatile,
-            unsigned Align, Instruction *InsertBefore = 0);
   StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
+  StoreInst(Value *Val, Value *Ptr, bool isVolatile,
+            unsigned Align, Instruction *InsertBefore = 0);
   StoreInst(Value *Val, Value *Ptr, bool isVolatile,
             unsigned Align, BasicBlock *InsertAtEnd);
+  StoreInst(Value *Val, Value *Ptr, bool isVolatile,
+            unsigned Align, AtomicOrdering Order,
+            SynchronizationScope SynchScope = CrossThread,
+            Instruction *InsertBefore = 0);
+  StoreInst(Value *Val, Value *Ptr, bool isVolatile,
+            unsigned Align, AtomicOrdering Order,
+            SynchronizationScope SynchScope,
+            BasicBlock *InsertAtEnd);
+          
 
-
-  /// isVolatile - Return true if this is a load from a volatile memory
+  /// isVolatile - Return true if this is a store to a volatile memory
   /// location.
   ///
   bool isVolatile() const { return getSubclassDataFromInstruction() & 1; }
 
-  /// setVolatile - Specify whether this is a volatile load or not.
+  /// setVolatile - Specify whether this is a volatile store or not.
   ///
   void setVolatile(bool V) {
     setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
@@ -230,11 +299,47 @@ class StoreInst : public Instruction {
   /// getAlignment - Return the alignment of the access that is being performed
   ///
   unsigned getAlignment() const {
-    return (1 << (getSubclassDataFromInstruction() >> 1)) >> 1;
+    return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1;
   }
 
   void setAlignment(unsigned Align);
 
+  /// Returns the ordering effect of this store.
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
+  }
+
+  /// Set the ordering constraint on this store.  May not be Acquire or
+  /// AcquireRelease.
+  void setOrdering(AtomicOrdering Ordering) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
+                               (Ordering << 7));
+  }
+
+  SynchronizationScope getSynchScope() const {
+    return SynchronizationScope((getSubclassDataFromInstruction() >> 6) & 1);
+  }
+
+  /// Specify whether this store instruction is ordered with respect to all
+  /// concurrently executing threads, or only with respect to signal handlers
+  /// executing in the same thread.
+  void setSynchScope(SynchronizationScope xthread) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(1 << 6)) |
+                               (xthread << 6));
+  }
+
+  bool isAtomic() const { return getOrdering() != NotAtomic; }
+  void setAtomic(AtomicOrdering Ordering,
+                 SynchronizationScope SynchScope = CrossThread) {
+    setOrdering(Ordering);
+    setSynchScope(SynchScope);
+  }
+
+  bool isSimple() const { return !isAtomic() && !isVolatile(); }
+  bool isUnordered() const {
+    return getOrdering() <= Unordered && !isVolatile();
+  }
+
   Value *getValueOperand() { return getOperand(0); }
   const Value *getValueOperand() const { return getOperand(0); }
 
@@ -268,6 +373,325 @@ struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)
 
+//===----------------------------------------------------------------------===//
+//                                FenceInst Class
+//===----------------------------------------------------------------------===//
+
+/// FenceInst - an instruction for ordering other memory operations
+///
+class FenceInst : public Instruction {
+  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void Init(AtomicOrdering Ordering, SynchronizationScope SynchScope);
+protected:
+  virtual FenceInst *clone_impl() const;
+public:
+  // allocate space for exactly zero operands
+  void *operator new(size_t s) {
+    return User::operator new(s, 0);
+  }
+
+  // Ordering may only be Acquire, Release, AcquireRelease, or
+  // SequentiallyConsistent.
+  FenceInst(LLVMContext &C, AtomicOrdering Ordering,
+            SynchronizationScope SynchScope = CrossThread,
+            Instruction *InsertBefore = 0);
+  FenceInst(LLVMContext &C, AtomicOrdering Ordering,
+            SynchronizationScope SynchScope,
+            BasicBlock *InsertAtEnd);
+
+  /// Returns the ordering effect of this fence.
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering(getSubclassDataFromInstruction() >> 1);
+  }
+
+  /// Set the ordering constraint on this fence.  May only be Acquire, Release,
+  /// AcquireRelease, or SequentiallyConsistent.
+  void setOrdering(AtomicOrdering Ordering) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & 1) |
+                               (Ordering << 1));
+  }
+
+  SynchronizationScope getSynchScope() const {
+    return SynchronizationScope(getSubclassDataFromInstruction() & 1);
+  }
+
+  /// Specify whether this fence orders other operations with respect to all
+  /// concurrently executing threads, or only with respect to signal handlers
+  /// executing in the same thread.
+  void setSynchScope(SynchronizationScope xthread) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
+                               xthread);
+  }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const FenceInst *) { return true; }
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::Fence;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+private:
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+//===----------------------------------------------------------------------===//
+//                                AtomicCmpXchgInst Class
+//===----------------------------------------------------------------------===//
+
+/// AtomicCmpXchgInst - an instruction that atomically checks whether a
+/// specified value is in a memory location, and, if it is, stores a new value
+/// there.  Returns the value that was loaded.
+///
+class AtomicCmpXchgInst : public Instruction {
+  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void Init(Value *Ptr, Value *Cmp, Value *NewVal,
+            AtomicOrdering Ordering, SynchronizationScope SynchScope);
+protected:
+  virtual AtomicCmpXchgInst *clone_impl() const;
+public:
+  // allocate space for exactly three operands
+  void *operator new(size_t s) {
+    return User::operator new(s, 3);
+  }
+  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                    AtomicOrdering Ordering, SynchronizationScope SynchScope,
+                    Instruction *InsertBefore = 0);
+  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                    AtomicOrdering Ordering, SynchronizationScope SynchScope,
+                    BasicBlock *InsertAtEnd);
+
+  /// isVolatile - Return true if this is a cmpxchg from a volatile memory
+  /// location.
+  ///
+  bool isVolatile() const {
+    return getSubclassDataFromInstruction() & 1;
+  }
+
+  /// setVolatile - Specify whether this is a volatile cmpxchg.
+  ///
+  void setVolatile(bool V) {
+     setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
+                                (unsigned)V);
+  }
+
+  /// Transparently provide more efficient getOperand methods.
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// Set the ordering constraint on this cmpxchg.
+  void setOrdering(AtomicOrdering Ordering) {
+    assert(Ordering != NotAtomic &&
+           "CmpXchg instructions can only be atomic.");
+    setInstructionSubclassData((getSubclassDataFromInstruction() & 3) |
+                               (Ordering << 2));
+  }
+
+  /// Specify whether this cmpxchg is atomic and orders other operations with
+  /// respect to all concurrently executing threads, or only with respect to
+  /// signal handlers executing in the same thread.
+  void setSynchScope(SynchronizationScope SynchScope) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~2) |
+                               (SynchScope << 1));
+  }
+
+  /// Returns the ordering constraint on this cmpxchg.
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering(getSubclassDataFromInstruction() >> 2);
+  }
+
+  /// Returns whether this cmpxchg is atomic between threads or only within a
+  /// single thread.
+  SynchronizationScope getSynchScope() const {
+    return SynchronizationScope((getSubclassDataFromInstruction() & 2) >> 1);
+  }
+
+  Value *getPointerOperand() { return getOperand(0); }
+  const Value *getPointerOperand() const { return getOperand(0); }
+  static unsigned getPointerOperandIndex() { return 0U; }
+
+  Value *getCompareOperand() { return getOperand(1); }
+  const Value *getCompareOperand() const { return getOperand(1); }
+  
+  Value *getNewValOperand() { return getOperand(2); }
+  const Value *getNewValOperand() const { return getOperand(2); }
+  
+  unsigned getPointerAddressSpace() const {
+    return cast<PointerType>(getPointerOperand()->getType())->getAddressSpace();
+  }
+  
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const AtomicCmpXchgInst *) { return true; }
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::AtomicCmpXchg;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+private:
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+template <>
+struct OperandTraits<AtomicCmpXchgInst> :
+    public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
+};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)
+
+//===----------------------------------------------------------------------===//
+//                                AtomicRMWInst Class
+//===----------------------------------------------------------------------===//
+
+/// AtomicRMWInst - an instruction that atomically reads a memory location,
+/// combines it with another value, and then stores the result back.  Returns
+/// the old value.
+///
+class AtomicRMWInst : public Instruction {
+  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+protected:
+  virtual AtomicRMWInst *clone_impl() const;
+public:
+  /// This enumeration lists the possible modifications atomicrmw can make.  In
+  /// the descriptions, 'p' is the pointer to the instruction's memory location,
+  /// 'old' is the initial value of *p, and 'v' is the other value passed to the
+  /// instruction.  These instructions always return 'old'.
+  enum BinOp {
+    /// *p = v
+    Xchg,
+    /// *p = old + v
+    Add,
+    /// *p = old - v
+    Sub,
+    /// *p = old & v
+    And,
+    /// *p = ~old & v
+    Nand,
+    /// *p = old | v
+    Or,
+    /// *p = old ^ v
+    Xor,
+    /// *p = old >signed v ? old : v
+    Max,
+    /// *p = old <signed v ? old : v
+    Min,
+    /// *p = old >unsigned v ? old : v
+    UMax,
+    /// *p = old <unsigned v ? old : v
+    UMin,
+
+    FIRST_BINOP = Xchg,
+    LAST_BINOP = UMin,
+    BAD_BINOP
+  };
+
+  // allocate space for exactly two operands
+  void *operator new(size_t s) {
+    return User::operator new(s, 2);
+  }
+  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+                AtomicOrdering Ordering, SynchronizationScope SynchScope,
+                Instruction *InsertBefore = 0);
+  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+                AtomicOrdering Ordering, SynchronizationScope SynchScope,
+                BasicBlock *InsertAtEnd);
+
+  BinOp getOperation() const {
+    return static_cast<BinOp>(getSubclassDataFromInstruction() >> 5);
+  }
+
+  void setOperation(BinOp Operation) {
+    unsigned short SubclassData = getSubclassDataFromInstruction();
+    setInstructionSubclassData((SubclassData & 31) |
+                               (Operation << 5));
+  }
+
+  /// isVolatile - Return true if this is a RMW on a volatile memory location.
+  ///
+  bool isVolatile() const {
+    return getSubclassDataFromInstruction() & 1;
+  }
+
+  /// setVolatile - Specify whether this is a volatile RMW or not.
+  ///
+  void setVolatile(bool V) {
+     setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
+                                (unsigned)V);
+  }
+
+  /// Transparently provide more efficient getOperand methods.
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// Set the ordering constraint on this RMW.
+  void setOrdering(AtomicOrdering Ordering) {
+    assert(Ordering != NotAtomic &&
+           "atomicrmw instructions can only be atomic.");
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 2)) |
+                               (Ordering << 2));
+  }
+
+  /// Specify whether this RMW orders other operations with respect to all
+  /// concurrently executing threads, or only with respect to signal handlers
+  /// executing in the same thread.
+  void setSynchScope(SynchronizationScope SynchScope) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~2) |
+                               (SynchScope << 1));
+  }
+
+  /// Returns the ordering constraint on this RMW.
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
+  }
+
+  /// Returns whether this RMW is atomic between threads or only within a
+  /// single thread.
+  SynchronizationScope getSynchScope() const {
+    return SynchronizationScope((getSubclassDataFromInstruction() & 2) >> 1);
+  }
+
+  Value *getPointerOperand() { return getOperand(0); }
+  const Value *getPointerOperand() const { return getOperand(0); }
+  static unsigned getPointerOperandIndex() { return 0U; }
+
+  Value *getValOperand() { return getOperand(1); }
+  const Value *getValOperand() const { return getOperand(1); }
+
+  unsigned getPointerAddressSpace() const {
+    return cast<PointerType>(getPointerOperand()->getType())->getAddressSpace();
+  }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const AtomicRMWInst *) { return true; }
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::AtomicRMW;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+private:
+  void Init(BinOp Operation, Value *Ptr, Value *Val,
+            AtomicOrdering Ordering, SynchronizationScope SynchScope);
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+template <>
+struct OperandTraits<AtomicRMWInst>
+    : public FixedNumOperandTraits<AtomicRMWInst,2> {
+};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)
+
 //===----------------------------------------------------------------------===//
 //                             GetElementPtrInst Class
 //===----------------------------------------------------------------------===//
@@ -275,7 +699,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)
 // checkGEPType - Simple wrapper function to give a better assertion failure
 // message on bad indexes for a gep instruction.
 //
-static inline const Type *checkGEPType(const Type *Ty) {
+static inline Type *checkGEPType(Type *Ty) {
   assert(Ty && "Invalid GetElementPtrInst indices for type!");
   return Ty;
 }
@@ -285,149 +709,51 @@ static inline const Type *checkGEPType(const Type *Ty) {
 ///
 class GetElementPtrInst : public Instruction {
   GetElementPtrInst(const GetElementPtrInst &GEPI);
-  void init(Value *Ptr, Value* const *Idx, unsigned NumIdx,
-            const Twine &NameStr);
-  void init(Value *Ptr, Value *Idx, const Twine &NameStr);
-
-  template<typename RandomAccessIterator>
-  void init(Value *Ptr,
-            RandomAccessIterator IdxBegin,
-            RandomAccessIterator IdxEnd,
-            const Twine &NameStr,
-            // This argument ensures that we have an iterator we can
-            // do arithmetic on in constant time
-            std::random_access_iterator_tag) {
-    unsigned NumIdx = static_cast<unsigned>(std::distance(IdxBegin, IdxEnd));
-
-    if (NumIdx > 0) {
-      // This requires that the iterator points to contiguous memory.
-      init(Ptr, &*IdxBegin, NumIdx, NameStr); // FIXME: for the general case
-                                     // we have to build an array here
-    }
-    else {
-      init(Ptr, 0, NumIdx, NameStr);
-    }
-  }
-
-  /// getIndexedType - Returns the type of the element that would be loaded with
-  /// a load instruction with the specified parameters.
-  ///
-  /// Null is returned if the indices are invalid for the specified
-  /// pointer type.
-  ///
-  template<typename RandomAccessIterator>
-  static Type *getIndexedType(const Type *Ptr,
-                              RandomAccessIterator IdxBegin,
-                              RandomAccessIterator IdxEnd,
-                              // This argument ensures that we
-                              // have an iterator we can do
-                              // arithmetic on in constant time
-                              std::random_access_iterator_tag) {
-    unsigned NumIdx = static_cast<unsigned>(std::distance(IdxBegin, IdxEnd));
-
-    if (NumIdx > 0)
-      // This requires that the iterator points to contiguous memory.
-      return getIndexedType(Ptr, &*IdxBegin, NumIdx);
-    else
-      return getIndexedType(Ptr, (Value *const*)0, NumIdx);
-  }
+  void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
 
   /// Constructors - Create a getelementptr instruction with a base pointer an
   /// list of indices. The first ctor can optionally insert before an existing
   /// instruction, the second appends the new instruction to the specified
   /// BasicBlock.
-  template<typename RandomAccessIterator>
-  inline GetElementPtrInst(Value *Ptr, RandomAccessIterator IdxBegin,
-                           RandomAccessIterator IdxEnd,
-                           unsigned Values,
-                           const Twine &NameStr,
+  inline GetElementPtrInst(Value *Ptr, ArrayRef<Value *> IdxList,
+                           unsigned Values, const Twine &NameStr,
                            Instruction *InsertBefore);
-  template<typename RandomAccessIterator>
-  inline GetElementPtrInst(Value *Ptr,
-                           RandomAccessIterator IdxBegin,
-                           RandomAccessIterator IdxEnd,
-                           unsigned Values,
-                           const Twine &NameStr, BasicBlock *InsertAtEnd);
-
-  /// Constructors - These two constructors are convenience methods because one
-  /// and two index getelementptr instructions are so common.
-  GetElementPtrInst(Value *Ptr, Value *Idx, const Twine &NameStr = "",
-                    Instruction *InsertBefore = 0);
-  GetElementPtrInst(Value *Ptr, Value *Idx,
-                    const Twine &NameStr, BasicBlock *InsertAtEnd);
+  inline GetElementPtrInst(Value *Ptr, ArrayRef<Value *> IdxList,
+                           unsigned Values, const Twine &NameStr,
+                           BasicBlock *InsertAtEnd);
 protected:
   virtual GetElementPtrInst *clone_impl() const;
 public:
-  template<typename RandomAccessIterator>
-  static GetElementPtrInst *Create(Value *Ptr, RandomAccessIterator IdxBegin,
-                                   RandomAccessIterator IdxEnd,
+  static GetElementPtrInst *Create(Value *Ptr, ArrayRef<Value *> IdxList,
                                    const Twine &NameStr = "",
                                    Instruction *InsertBefore = 0) {
-    typename std::iterator_traits<RandomAccessIterator>::difference_type
-      Values = 1 + std::distance(IdxBegin, IdxEnd);
+    unsigned Values = 1 + unsigned(IdxList.size());
     return new(Values)
-      GetElementPtrInst(Ptr, IdxBegin, IdxEnd, Values, NameStr, InsertBefore);
+      GetElementPtrInst(Ptr, IdxList, Values, NameStr, InsertBefore);
   }
-  template<typename RandomAccessIterator>
-  static GetElementPtrInst *Create(Value *Ptr,
-                                   RandomAccessIterator IdxBegin,
-                                   RandomAccessIterator IdxEnd,
+  static GetElementPtrInst *Create(Value *Ptr, ArrayRef<Value *> IdxList,
                                    const Twine &NameStr,
                                    BasicBlock *InsertAtEnd) {
-    typename std::iterator_traits<RandomAccessIterator>::difference_type
-      Values = 1 + std::distance(IdxBegin, IdxEnd);
+    unsigned Values = 1 + unsigned(IdxList.size());
     return new(Values)
-      GetElementPtrInst(Ptr, IdxBegin, IdxEnd, Values, NameStr, InsertAtEnd);
-  }
-
-  /// Constructors - These two creators are convenience methods because one
-  /// index getelementptr instructions are so common.
-  static GetElementPtrInst *Create(Value *Ptr, Value *Idx,
-                                   const Twine &NameStr = "",
-                                   Instruction *InsertBefore = 0) {
-    return new(2) GetElementPtrInst(Ptr, Idx, NameStr, InsertBefore);
-  }
-  static GetElementPtrInst *Create(Value *Ptr, Value *Idx,
-                                   const Twine &NameStr,
-                                   BasicBlock *InsertAtEnd) {
-    return new(2) GetElementPtrInst(Ptr, Idx, NameStr, InsertAtEnd);
+      GetElementPtrInst(Ptr, IdxList, Values, NameStr, InsertAtEnd);
   }
 
   /// Create an "inbounds" getelementptr. See the documentation for the
   /// "inbounds" flag in LangRef.html for details.
-  template<typename RandomAccessIterator>
   static GetElementPtrInst *CreateInBounds(Value *Ptr,
-                                           RandomAccessIterator IdxBegin,
-                                           RandomAccessIterator IdxEnd,
+                                           ArrayRef<Value *> IdxList,
                                            const Twine &NameStr = "",
                                            Instruction *InsertBefore = 0) {
-    GetElementPtrInst *GEP = Create(Ptr, IdxBegin, IdxEnd,
-                                    NameStr, InsertBefore);
+    GetElementPtrInst *GEP = Create(Ptr, IdxList, NameStr, InsertBefore);
     GEP->setIsInBounds(true);
     return GEP;
   }
-  template<typename RandomAccessIterator>
   static GetElementPtrInst *CreateInBounds(Value *Ptr,
-                                           RandomAccessIterator IdxBegin,
-                                           RandomAccessIterator IdxEnd,
+                                           ArrayRef<Value *> IdxList,
                                            const Twine &NameStr,
                                            BasicBlock *InsertAtEnd) {
-    GetElementPtrInst *GEP = Create(Ptr, IdxBegin, IdxEnd,
-                                    NameStr, InsertAtEnd);
-    GEP->setIsInBounds(true);
-    return GEP;
-  }
-  static GetElementPtrInst *CreateInBounds(Value *Ptr, Value *Idx,
-                                           const Twine &NameStr = "",
-                                           Instruction *InsertBefore = 0) {
-    GetElementPtrInst *GEP = Create(Ptr, Idx, NameStr, InsertBefore);
-    GEP->setIsInBounds(true);
-    return GEP;
-  }
-  static GetElementPtrInst *CreateInBounds(Value *Ptr, Value *Idx,
-                                           const Twine &NameStr,
-                                           BasicBlock *InsertAtEnd) {
-    GetElementPtrInst *GEP = Create(Ptr, Idx, NameStr, InsertAtEnd);
+    GetElementPtrInst *GEP = Create(Ptr, IdxList, NameStr, InsertAtEnd);
     GEP->setIsInBounds(true);
     return GEP;
   }
@@ -436,8 +762,8 @@ class GetElementPtrInst : public Instruction {
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
   // getType - Overload to return most specific pointer type...
-  const PointerType *getType() const {
-    return reinterpret_cast<const PointerType*>(Instruction::getType());
+  PointerType *getType() const {
+    return reinterpret_cast<PointerType*>(Instruction::getType());
   }
 
   /// getIndexedType - Returns the type of the element that would be loaded with
@@ -446,23 +772,9 @@ class GetElementPtrInst : public Instruction {
   /// Null is returned if the indices are invalid for the specified
   /// pointer type.
   ///
-  template<typename RandomAccessIterator>
-  static Type *getIndexedType(const Type *Ptr, RandomAccessIterator IdxBegin,
-                              RandomAccessIterator IdxEnd) {
-    return getIndexedType(Ptr, IdxBegin, IdxEnd,
-                          typename std::iterator_traits<RandomAccessIterator>::
-                          iterator_category());
-  }
-
-  // FIXME: Use ArrayRef
-  static Type *getIndexedType(const Type *Ptr,
-                              Value* const *Idx, unsigned NumIdx);
-  static Type *getIndexedType(const Type *Ptr,
-                              Constant* const *Idx, unsigned NumIdx);
-
-  static Type *getIndexedType(const Type *Ptr,
-                              uint64_t const *Idx, unsigned NumIdx);
-  static Type *getIndexedType(const Type *Ptr, Value *Idx);
+  static Type *getIndexedType(Type *Ptr, ArrayRef<Value *> IdxList);
+  static Type *getIndexedType(Type *Ptr, ArrayRef<Constant *> IdxList);
+  static Type *getIndexedType(Type *Ptr, ArrayRef<uint64_t> IdxList);
 
   inline op_iterator       idx_begin()       { return op_begin()+1; }
   inline const_op_iterator idx_begin() const { return op_begin()+1; }
@@ -485,8 +797,8 @@ class GetElementPtrInst : public Instruction {
 
   /// getPointerOperandType - Method to return the pointer operand as a
   /// PointerType.
-  const PointerType *getPointerOperandType() const {
-    return reinterpret_cast<const PointerType*>(getPointerOperand()->getType());
+  PointerType *getPointerOperandType() const {
+    return reinterpret_cast<PointerType*>(getPointerOperand()->getType());
   }
 
 
@@ -530,43 +842,33 @@ struct OperandTraits<GetElementPtrInst> :
   public VariadicOperandTraits<GetElementPtrInst, 1> {
 };
 
-template<typename RandomAccessIterator>
 GetElementPtrInst::GetElementPtrInst(Value *Ptr,
-                                     RandomAccessIterator IdxBegin,
-                                     RandomAccessIterator IdxEnd,
+                                     ArrayRef<Value *> IdxList,
                                      unsigned Values,
                                      const Twine &NameStr,
                                      Instruction *InsertBefore)
   : Instruction(PointerType::get(checkGEPType(
-                                   getIndexedType(Ptr->getType(),
-                                                  IdxBegin, IdxEnd)),
+                                   getIndexedType(Ptr->getType(), IdxList)),
                                  cast<PointerType>(Ptr->getType())
                                    ->getAddressSpace()),
                 GetElementPtr,
                 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
                 Values, InsertBefore) {
-  init(Ptr, IdxBegin, IdxEnd, NameStr,
-       typename std::iterator_traits<RandomAccessIterator>
-       ::iterator_category());
+  init(Ptr, IdxList, NameStr);
 }
-template<typename RandomAccessIterator>
 GetElementPtrInst::GetElementPtrInst(Value *Ptr,
-                                     RandomAccessIterator IdxBegin,
-                                     RandomAccessIterator IdxEnd,
+                                     ArrayRef<Value *> IdxList,
                                      unsigned Values,
                                      const Twine &NameStr,
                                      BasicBlock *InsertAtEnd)
   : Instruction(PointerType::get(checkGEPType(
-                                   getIndexedType(Ptr->getType(),
-                                                  IdxBegin, IdxEnd)),
+                                   getIndexedType(Ptr->getType(), IdxList)),
                                  cast<PointerType>(Ptr->getType())
                                    ->getAddressSpace()),
                 GetElementPtr,
                 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
                 Values, InsertAtEnd) {
-  init(Ptr, IdxBegin, IdxEnd, NameStr,
-       typename std::iterator_traits<RandomAccessIterator>
-       ::iterator_category());
+  init(Ptr, IdxList, NameStr);
 }
 
 
@@ -893,12 +1195,12 @@ class CallInst : public Instruction {
   /// 2. Call malloc with that argument.
   /// 3. Bitcast the result of the malloc call to the specified type.
   static Instruction *CreateMalloc(Instruction *InsertBefore,
-                                   const Type *IntPtrTy, const Type *AllocTy,
+                                   Type *IntPtrTy, Type *AllocTy,
                                    Value *AllocSize, Value *ArraySize = 0,
                                    Function* MallocF = 0,
                                    const Twine &Name = "");
   static Instruction *CreateMalloc(BasicBlock *InsertAtEnd,
-                                   const Type *IntPtrTy, const Type *AllocTy,
+                                   Type *IntPtrTy, Type *AllocTy,
                                    Value *AllocSize, Value *ArraySize = 0,
                                    Function* MallocF = 0,
                                    const Twine &Name = "");
@@ -965,6 +1267,15 @@ class CallInst : public Instruction {
     else removeAttribute(~0, Attribute::NoInline);
   }
 
+  /// @brief Return true if the call can return twice
+  bool canReturnTwice() const {
+    return paramHasAttr(~0, Attribute::ReturnsTwice);
+  }
+  void setCanReturnTwice(bool Value = true) {
+    if (Value) addAttribute(~0, Attribute::ReturnsTwice);
+    else removeAttribute(~0, Attribute::ReturnsTwice);
+  }
+
   /// @brief Determine if the call does not access memory.
   bool doesNotAccessMemory() const {
     return paramHasAttr(~0, Attribute::ReadNone);
@@ -1165,12 +1476,12 @@ class VAArgInst : public UnaryInstruction {
   virtual VAArgInst *clone_impl() const;
 
 public:
-  VAArgInst(Value *List, const Type *Ty, const Twine &NameStr = "",
+  VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
              Instruction *InsertBefore = 0)
     : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
     setName(NameStr);
   }
-  VAArgInst(Value *List, const Type *Ty, const Twine &NameStr,
+  VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
             BasicBlock *InsertAtEnd)
     : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
     setName(NameStr);
@@ -1226,8 +1537,8 @@ class ExtractElementInst : public Instruction {
   const Value *getVectorOperand() const { return Op<0>(); }
   const Value *getIndexOperand() const { return Op<1>(); }
 
-  const VectorType *getVectorOperandType() const {
-    return reinterpret_cast<const VectorType*>(getVectorOperand()->getType());
+  VectorType *getVectorOperandType() const {
+    return reinterpret_cast<VectorType*>(getVectorOperand()->getType());
   }
 
 
@@ -1286,8 +1597,8 @@ class InsertElementInst : public Instruction {
 
   /// getType - Overload to return most specific vector type.
   ///
-  const VectorType *getType() const {
-    return reinterpret_cast<const VectorType*>(Instruction::getType());
+  VectorType *getType() const {
+    return reinterpret_cast<VectorType*>(Instruction::getType());
   }
 
   /// Transparently provide more efficient getOperand methods.
@@ -1339,8 +1650,8 @@ class ShuffleVectorInst : public Instruction {
 
   /// getType - Overload to return most specific vector type.
   ///
-  const VectorType *getType() const {
-    return reinterpret_cast<const VectorType*>(Instruction::getType());
+  VectorType *getType() const {
+    return reinterpret_cast<VectorType*>(Instruction::getType());
   }
 
   /// Transparently provide more efficient getOperand methods.
@@ -1419,7 +1730,7 @@ class ExtractValueInst : public UnaryInstruction {
   /// with an extractvalue instruction with the specified parameters.
   ///
   /// Null is returned if the indices are invalid for the specified type.
-  static Type *getIndexedType(const Type *Agg, ArrayRef<unsigned> Idxs);
+  static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
 
   typedef const unsigned* idx_iterator;
   inline idx_iterator idx_begin() const { return Indices.begin(); }
@@ -1625,7 +1936,7 @@ class PHINode : public Instruction {
   void *operator new(size_t s) {
     return User::operator new(s, 0);
   }
-  explicit PHINode(const Type *Ty, unsigned NumReservedValues,
+  explicit PHINode(Type *Ty, unsigned NumReservedValues,
                    const Twine &NameStr = "", Instruction *InsertBefore = 0)
     : Instruction(Ty, Instruction::PHI, 0, 0, InsertBefore),
       ReservedSpace(NumReservedValues) {
@@ -1633,7 +1944,7 @@ class PHINode : public Instruction {
     OperandList = allocHungoffUses(ReservedSpace);
   }
 
-  PHINode(const Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
+  PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
           BasicBlock *InsertAtEnd)
     : Instruction(Ty, Instruction::PHI, 0, 0, InsertAtEnd),
       ReservedSpace(NumReservedValues) {
@@ -1650,12 +1961,12 @@ class PHINode : public Instruction {
 public:
   /// Constructors - NumReservedValues is a hint for the number of incoming
   /// edges that this phi node will have (use 0 if you really have no idea).
-  static PHINode *Create(const Type *Ty, unsigned NumReservedValues,
+  static PHINode *Create(Type *Ty, unsigned NumReservedValues,
                          const Twine &NameStr = "",
                          Instruction *InsertBefore = 0) {
     return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
   }
-  static PHINode *Create(const Type *Ty, unsigned NumReservedValues, 
+  static PHINode *Create(Type *Ty, unsigned NumReservedValues, 
                          const Twine &NameStr, BasicBlock *InsertAtEnd) {
     return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
   }
@@ -1804,6 +2115,111 @@ struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)
 
+//===----------------------------------------------------------------------===//
+//                           LandingPadInst Class
+//===----------------------------------------------------------------------===//
+
+//===---------------------------------------------------------------------------
+/// LandingPadInst - The landingpad instruction holds all of the information
+/// necessary to generate correct exception handling. The landingpad instruction
+/// cannot be moved from the top of a landing pad block, which itself is
+/// accessible only from the 'unwind' edge of an invoke. This uses the
+/// SubclassData field in Value to store whether or not the landingpad is a
+/// cleanup.
+///
+class LandingPadInst : public Instruction {
+  /// ReservedSpace - The number of operands actually allocated.  NumOperands is
+  /// the number actually in use.
+  unsigned ReservedSpace;
+  LandingPadInst(const LandingPadInst &LP);
+public:
+  enum ClauseType { Catch, Filter };
+private:
+  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  // Allocate space for exactly zero operands.
+  void *operator new(size_t s) {
+    return User::operator new(s, 0);
+  }
+  void growOperands(unsigned Size);
+  void init(Value *PersFn, unsigned NumReservedValues, const Twine &NameStr);
+
+  explicit LandingPadInst(Type *RetTy, Value *PersonalityFn,
+                          unsigned NumReservedValues, const Twine &NameStr,
+                          Instruction *InsertBefore);
+  explicit LandingPadInst(Type *RetTy, Value *PersonalityFn,
+                          unsigned NumReservedValues, const Twine &NameStr,
+                          BasicBlock *InsertAtEnd);
+protected:
+  virtual LandingPadInst *clone_impl() const;
+public:
+  /// Constructors - NumReservedClauses is a hint for the number of incoming
+  /// clauses that this landingpad will have (use 0 if you really have no idea).
+  static LandingPadInst *Create(Type *RetTy, Value *PersonalityFn,
+                                unsigned NumReservedClauses,
+                                const Twine &NameStr = "",
+                                Instruction *InsertBefore = 0);
+  static LandingPadInst *Create(Type *RetTy, Value *PersonalityFn,
+                                unsigned NumReservedClauses,
+                                const Twine &NameStr, BasicBlock *InsertAtEnd);
+  ~LandingPadInst();
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// getPersonalityFn - Get the personality function associated with this
+  /// landing pad.
+  Value *getPersonalityFn() const { return getOperand(0); }
+
+  /// isCleanup - Return 'true' if this landingpad instruction is a
+  /// cleanup. I.e., it should be run when unwinding even if its landing pad
+  /// doesn't catch the exception.
+  bool isCleanup() const { return getSubclassDataFromInstruction() & 1; }
+
+  /// setCleanup - Indicate that this landingpad instruction is a cleanup.
+  void setCleanup(bool V) {
+    setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
+                               (V ? 1 : 0));
+  }
+
+  /// addClause - Add a catch or filter clause to the landing pad.
+  void addClause(Value *ClauseVal);
+
+  /// getClause - Get the value of the clause at index Idx. Use isCatch/isFilter
+  /// to determine what type of clause this is.
+  Value *getClause(unsigned Idx) const { return OperandList[Idx + 1]; }
+
+  /// isCatch - Return 'true' if the clause and index Idx is a catch clause.
+  bool isCatch(unsigned Idx) const {
+    return !isa<ArrayType>(OperandList[Idx + 1]->getType());
+  }
+
+  /// isFilter - Return 'true' if the clause and index Idx is a filter clause.
+  bool isFilter(unsigned Idx) const {
+    return isa<ArrayType>(OperandList[Idx + 1]->getType());
+  }
+
+  /// getNumClauses - Get the number of clauses for this landing pad.
+  unsigned getNumClauses() const { return getNumOperands() - 1; }
+
+  /// reserveClauses - Grow the size of the operand list to accomodate the new
+  /// number of clauses.
+  void reserveClauses(unsigned Size) { growOperands(Size); }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const LandingPadInst *) { return true; }
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::LandingPad;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+};
+
+template <>
+struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<2> {
+};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)
 
 //===----------------------------------------------------------------------===//
 //                               ReturnInst Class
@@ -1951,6 +2367,13 @@ class BranchInst : public TerminatorInst {
     *(&Op<-1>() - idx) = (Value*)NewSucc;
   }
 
+  /// \brief Swap the successors of this branch instruction.
+  ///
+  /// Swaps the successors of the branch instruction. This also swaps any
+  /// branch weight metadata associated with the instruction so that it
+  /// continues to map correctly to each operand.
+  void swapSuccessors();
+
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const BranchInst *) { return true; }
   static inline bool classof(const Instruction *I) {
@@ -2102,6 +2525,13 @@ class SwitchInst : public TerminatorInst {
     return reinterpret_cast<ConstantInt*>(getOperand(idx*2));
   }
 
+  // setSuccessorValue - Updates the value associated with the specified
+  // successor.
+  void setSuccessorValue(unsigned idx, ConstantInt* SuccessorValue) {
+    assert(idx < getNumSuccessors() && "Successor # out of range!");
+    setOperand(idx*2, reinterpret_cast<Value*>(SuccessorValue));
+  }
+
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const SwitchInst *) { return true; }
   static inline bool classof(const Instruction *I) {
@@ -2393,6 +2823,10 @@ class InvokeInst : public TerminatorInst {
     Op<-1>() = reinterpret_cast<Value*>(B);
   }
 
+  /// getLandingPadInst - Get the landingpad instruction from the landing pad
+  /// block (the unwind destination).
+  LandingPadInst *getLandingPadInst() const;
+
   BasicBlock *getSuccessor(unsigned i) const {
     assert(i < 2 && "Successor # out of range for invoke!");
     return i == 0 ? getNormalDest() : getUnwindDest();
@@ -2491,6 +2925,57 @@ class UnwindInst : public TerminatorInst {
   virtual void setSuccessorV(unsigned idx, BasicBlock *B);
 };
 
+//===----------------------------------------------------------------------===//
+//                              ResumeInst Class
+//===----------------------------------------------------------------------===//
+
+//===---------------------------------------------------------------------------
+/// ResumeInst - Resume the propagation of an exception.
+///
+class ResumeInst : public TerminatorInst {
+  ResumeInst(const ResumeInst &RI);
+
+  explicit ResumeInst(Value *Exn, Instruction *InsertBefore=0);
+  ResumeInst(Value *Exn, BasicBlock *InsertAtEnd);
+protected:
+  virtual ResumeInst *clone_impl() const;
+public:
+  static ResumeInst *Create(Value *Exn, Instruction *InsertBefore = 0) {
+    return new(1) ResumeInst(Exn, InsertBefore);
+  }
+  static ResumeInst *Create(Value *Exn, BasicBlock *InsertAtEnd) {
+    return new(1) ResumeInst(Exn, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// Convenience accessor.
+  Value *getValue() const { return Op<0>(); }
+
+  unsigned getNumSuccessors() const { return 0; }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const ResumeInst *) { return true; }
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::Resume;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+private:
+  virtual BasicBlock *getSuccessorV(unsigned idx) const;
+  virtual unsigned getNumSuccessorsV() const;
+  virtual void setSuccessorV(unsigned idx, BasicBlock *B);
+};
+
+template <>
+struct OperandTraits<ResumeInst> :
+    public FixedNumOperandTraits<ResumeInst, 1> {
+};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)
+
 //===----------------------------------------------------------------------===//
 //                           UnreachableInst Class
 //===----------------------------------------------------------------------===//
@@ -2543,7 +3028,7 @@ class TruncInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   TruncInst(
     Value *S,                     ///< The value to be truncated
-    const Type *Ty,               ///< The (smaller) type to truncate to
+    Type *Ty,               ///< The (smaller) type to truncate to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2551,7 +3036,7 @@ class TruncInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   TruncInst(
     Value *S,                     ///< The value to be truncated
-    const Type *Ty,               ///< The (smaller) type to truncate to
+    Type *Ty,               ///< The (smaller) type to truncate to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2580,7 +3065,7 @@ class ZExtInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   ZExtInst(
     Value *S,                     ///< The value to be zero extended
-    const Type *Ty,               ///< The type to zero extend to
+    Type *Ty,               ///< The type to zero extend to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2588,7 +3073,7 @@ class ZExtInst : public CastInst {
   /// @brief Constructor with insert-at-end semantics.
   ZExtInst(
     Value *S,                     ///< The value to be zero extended
-    const Type *Ty,               ///< The type to zero extend to
+    Type *Ty,               ///< The type to zero extend to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2617,7 +3102,7 @@ class SExtInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   SExtInst(
     Value *S,                     ///< The value to be sign extended
-    const Type *Ty,               ///< The type to sign extend to
+    Type *Ty,               ///< The type to sign extend to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2625,7 +3110,7 @@ class SExtInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   SExtInst(
     Value *S,                     ///< The value to be sign extended
-    const Type *Ty,               ///< The type to sign extend to
+    Type *Ty,               ///< The type to sign extend to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2654,7 +3139,7 @@ class FPTruncInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   FPTruncInst(
     Value *S,                     ///< The value to be truncated
-    const Type *Ty,               ///< The type to truncate to
+    Type *Ty,               ///< The type to truncate to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2662,7 +3147,7 @@ class FPTruncInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   FPTruncInst(
     Value *S,                     ///< The value to be truncated
-    const Type *Ty,               ///< The type to truncate to
+    Type *Ty,               ///< The type to truncate to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2691,7 +3176,7 @@ class FPExtInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   FPExtInst(
     Value *S,                     ///< The value to be extended
-    const Type *Ty,               ///< The type to extend to
+    Type *Ty,               ///< The type to extend to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2699,7 +3184,7 @@ class FPExtInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   FPExtInst(
     Value *S,                     ///< The value to be extended
-    const Type *Ty,               ///< The type to extend to
+    Type *Ty,               ///< The type to extend to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2728,7 +3213,7 @@ class UIToFPInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   UIToFPInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2736,7 +3221,7 @@ class UIToFPInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   UIToFPInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2765,7 +3250,7 @@ class SIToFPInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   SIToFPInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2773,7 +3258,7 @@ class SIToFPInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   SIToFPInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2802,7 +3287,7 @@ class FPToUIInst  : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   FPToUIInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2810,7 +3295,7 @@ class FPToUIInst  : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   FPToUIInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< Where to insert the new instruction
   );
@@ -2839,7 +3324,7 @@ class FPToSIInst  : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   FPToSIInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2847,7 +3332,7 @@ class FPToSIInst  : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   FPToSIInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2872,7 +3357,7 @@ class IntToPtrInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   IntToPtrInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2880,7 +3365,7 @@ class IntToPtrInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   IntToPtrInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2912,7 +3397,7 @@ class PtrToIntInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   PtrToIntInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2920,7 +3405,7 @@ class PtrToIntInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   PtrToIntInst(
     Value *S,                     ///< The value to be converted
-    const Type *Ty,               ///< The type to convert to
+    Type *Ty,               ///< The type to convert to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
@@ -2949,7 +3434,7 @@ class BitCastInst : public CastInst {
   /// @brief Constructor with insert-before-instruction semantics
   BitCastInst(
     Value *S,                     ///< The value to be casted
-    const Type *Ty,               ///< The type to casted to
+    Type *Ty,               ///< The type to casted to
     const Twine &NameStr = "",    ///< A name for the new instruction
     Instruction *InsertBefore = 0 ///< Where to insert the new instruction
   );
@@ -2957,7 +3442,7 @@ class BitCastInst : public CastInst {
   /// @brief Constructor with insert-at-end-of-block semantics
   BitCastInst(
     Value *S,                     ///< The value to be casted
-    const Type *Ty,               ///< The type to casted to
+    Type *Ty,               ///< The type to casted to
     const Twine &NameStr,         ///< A name for the new instruction
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
diff --git a/contrib/llvm/include/llvm/IntrinsicInst.h b/contrib/llvm/include/llvm/IntrinsicInst.h
index 24e5fe7845fe..42862011ac7a 100644
--- a/contrib/llvm/include/llvm/IntrinsicInst.h
+++ b/contrib/llvm/include/llvm/IntrinsicInst.h
@@ -170,7 +170,7 @@ namespace llvm {
       setArgOperand(4, V);
     }
 
-    const Type *getAlignmentType() const {
+    Type *getAlignmentType() const {
       return getArgOperand(3)->getType();
     }
 
diff --git a/contrib/llvm/include/llvm/Intrinsics.h b/contrib/llvm/include/llvm/Intrinsics.h
index 46361ca0c2f8..370382560337 100644
--- a/contrib/llvm/include/llvm/Intrinsics.h
+++ b/contrib/llvm/include/llvm/Intrinsics.h
@@ -49,7 +49,7 @@ namespace Intrinsic {
   
   /// Intrinsic::getType(ID) - Return the function type for an intrinsic.
   ///
-  const FunctionType *getType(LLVMContext &Context, ID id,
+  FunctionType *getType(LLVMContext &Context, ID id,
                               ArrayRef<Type*> Tys = ArrayRef<Type*>());
 
   /// Intrinsic::isOverloaded(ID) - Returns true if the intrinsic can be
diff --git a/contrib/llvm/include/llvm/Intrinsics.td b/contrib/llvm/include/llvm/Intrinsics.td
index 947cf1be7d40..d70f9153fd8a 100644
--- a/contrib/llvm/include/llvm/Intrinsics.td
+++ b/contrib/llvm/include/llvm/Intrinsics.td
@@ -309,7 +309,9 @@ def int_eh_selector  : Intrinsic<[llvm_i32_ty],
                                  [llvm_ptr_ty, llvm_ptr_ty, llvm_vararg_ty]>;
 def int_eh_resume    : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [Throws]>;
 
-def int_eh_typeid_for : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>;
+// The result of eh.typeid.for depends on the enclosing function, but inside a
+// given function it is 'const' and may be CSE'd etc.
+def int_eh_typeid_for : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>;
 
 def int_eh_return_i32 : Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty]>;
 def int_eh_return_i64 : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty]>;
@@ -320,12 +322,13 @@ def int_eh_unwind_init: Intrinsic<[]>,
 def int_eh_dwarf_cfa  : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty]>;
 
 let Properties = [IntrNoMem] in {
-  def int_eh_sjlj_lsda    : Intrinsic<[llvm_ptr_ty]>;
-  def int_eh_sjlj_callsite: Intrinsic<[], [llvm_i32_ty]>;
+  def int_eh_sjlj_lsda             : Intrinsic<[llvm_ptr_ty]>;
+  def int_eh_sjlj_callsite         : Intrinsic<[], [llvm_i32_ty]>;
 }
-def int_eh_sjlj_dispatch_setup : Intrinsic<[], [llvm_i32_ty]>;
-def int_eh_sjlj_setjmp  : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>;
-def int_eh_sjlj_longjmp : Intrinsic<[], [llvm_ptr_ty]>;
+def int_eh_sjlj_functioncontext : Intrinsic<[], [llvm_ptr_ty]>;
+def int_eh_sjlj_dispatch_setup  : Intrinsic<[], [llvm_i32_ty]>;
+def int_eh_sjlj_setjmp          : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>;
+def int_eh_sjlj_longjmp         : Intrinsic<[], [llvm_ptr_ty]>;
 
 //===---------------- Generic Variable Attribute Intrinsics----------------===//
 //
@@ -344,10 +347,14 @@ def int_annotation : Intrinsic<[llvm_anyint_ty],
 
 //===------------------------ Trampoline Intrinsics -----------------------===//
 //
-def int_init_trampoline : Intrinsic<[llvm_ptr_ty],
+def int_init_trampoline : Intrinsic<[],
                                     [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
-                                    [IntrReadWriteArgMem]>,
-                          GCCBuiltin<"__builtin_init_trampoline">;
+                                    [IntrReadWriteArgMem, NoCapture<0>]>,
+                                   GCCBuiltin<"__builtin_init_trampoline">;
+
+def int_adjust_trampoline : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty],
+                                      [IntrReadArgMem]>,
+                                     GCCBuiltin<"__builtin_adjust_trampoline">;
 
 //===------------------------ Overflow Intrinsics -------------------------===//
 //
@@ -374,74 +381,6 @@ def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem]>;
 
-//===------------------------- Atomic Intrinsics --------------------------===//
-//
-def int_memory_barrier : Intrinsic<[],
-                                   [llvm_i1_ty, llvm_i1_ty,
-                                    llvm_i1_ty, llvm_i1_ty, llvm_i1_ty], []>,
-                                    GCCBuiltin<"__builtin_llvm_memory_barrier">;
-
-def int_atomic_cmp_swap : Intrinsic<[llvm_anyint_ty],
-                                    [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                     LLVMMatchType<0>, LLVMMatchType<0>],
-                                    [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_val_compare_and_swap">;
-def int_atomic_load_add : Intrinsic<[llvm_anyint_ty],
-                                    [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                     LLVMMatchType<0>],
-                                    [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_add">;
-def int_atomic_swap     : Intrinsic<[llvm_anyint_ty],
-                                    [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                     LLVMMatchType<0>],
-                                    [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_lock_test_and_set">;
-def int_atomic_load_sub : Intrinsic<[llvm_anyint_ty],
-                                    [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                     LLVMMatchType<0>],
-                                    [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_sub">;
-def int_atomic_load_and : Intrinsic<[llvm_anyint_ty],
-                                    [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                     LLVMMatchType<0>],
-                                    [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_and">;
-def int_atomic_load_or   : Intrinsic<[llvm_anyint_ty],
-                                     [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                      LLVMMatchType<0>],
-                                     [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_or">;
-def int_atomic_load_xor : Intrinsic<[llvm_anyint_ty],
-                                    [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                     LLVMMatchType<0>],
-                                    [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_xor">;
-def int_atomic_load_nand : Intrinsic<[llvm_anyint_ty],
-                                     [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                      LLVMMatchType<0>],
-                                     [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_nand">;
-def int_atomic_load_min  : Intrinsic<[llvm_anyint_ty],
-                                     [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                      LLVMMatchType<0>],
-                                     [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_min">;
-def int_atomic_load_max  : Intrinsic<[llvm_anyint_ty],
-                                     [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                      LLVMMatchType<0>],
-                                     [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_max">;
-def int_atomic_load_umin : Intrinsic<[llvm_anyint_ty],
-                                     [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                      LLVMMatchType<0>],
-                                     [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_umin">;
-def int_atomic_load_umax : Intrinsic<[llvm_anyint_ty],
-                                     [LLVMAnyPointerType<LLVMMatchType<0>>,
-                                      LLVMMatchType<0>],
-                                     [IntrReadWriteArgMem, NoCapture<0>]>,
-                           GCCBuiltin<"__sync_fetch_and_umax">;
-
 //===------------------------- Memory Use Markers -------------------------===//
 //
 def int_lifetime_start  : Intrinsic<[],
diff --git a/contrib/llvm/include/llvm/IntrinsicsXCore.td b/contrib/llvm/include/llvm/IntrinsicsXCore.td
index a062fc4fc7e8..a4813135da8d 100644
--- a/contrib/llvm/include/llvm/IntrinsicsXCore.td
+++ b/contrib/llvm/include/llvm/IntrinsicsXCore.td
@@ -1,6 +1,9 @@
 //==- IntrinsicsXCore.td - XCore intrinsics                 -*- tablegen -*-==//
-// 
-// Copyright (C) 2008 XMOS
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,9 +20,15 @@ let TargetPrefix = "xcore" in {  // All intrinsics start with "llvm.xcore.".
   def int_xcore_crc32 : Intrinsic<[llvm_i32_ty],
                                   [llvm_i32_ty,llvm_i32_ty,llvm_i32_ty],
                                   [IntrNoMem]>;
+  def int_xcore_sext : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+                                 [IntrNoMem]>;
+  def int_xcore_zext : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+                                 [IntrNoMem]>;
   def int_xcore_getid : Intrinsic<[llvm_i32_ty],[],[IntrNoMem]>;
   def int_xcore_getps : Intrinsic<[llvm_i32_ty],[llvm_i32_ty]>;
   def int_xcore_setps : Intrinsic<[],[llvm_i32_ty, llvm_i32_ty]>;
+  def int_xcore_geted : Intrinsic<[llvm_i32_ty],[]>;
+  def int_xcore_getet : Intrinsic<[llvm_i32_ty],[]>;
   def int_xcore_setsr : Intrinsic<[],[llvm_i32_ty]>;
   def int_xcore_clrsr : Intrinsic<[],[llvm_i32_ty]>;
 
@@ -40,6 +49,10 @@ let TargetPrefix = "xcore" in {  // All intrinsics start with "llvm.xcore.".
                                   [NoCapture<0>]>;
   def int_xcore_chkct : Intrinsic<[],[llvm_anyptr_ty, llvm_i32_ty],
                                   [NoCapture<0>]>;
+  def int_xcore_testct : Intrinsic<[llvm_i32_ty],[llvm_anyptr_ty],
+                                   [NoCapture<0>]>;
+  def int_xcore_testwct : Intrinsic<[llvm_i32_ty],[llvm_anyptr_ty],
+                                    [NoCapture<0>]>;
   def int_xcore_setd : Intrinsic<[],[llvm_anyptr_ty, llvm_i32_ty],
                                   [NoCapture<0>]>;
   def int_xcore_setc : Intrinsic<[],[llvm_anyptr_ty, llvm_i32_ty],
@@ -58,6 +71,8 @@ let TargetPrefix = "xcore" in {  // All intrinsics start with "llvm.xcore.".
                                   [NoCapture<0>]>;
   def int_xcore_setv : Intrinsic<[],[llvm_anyptr_ty, llvm_ptr_ty],
                                  [NoCapture<0>]>;
+  def int_xcore_setev : Intrinsic<[],[llvm_anyptr_ty, llvm_ptr_ty],
+                                  [NoCapture<0>]>;
   def int_xcore_eeu : Intrinsic<[],[llvm_anyptr_ty], [NoCapture<0>]>;
   def int_xcore_setclk : Intrinsic<[],[llvm_anyptr_ty, llvm_anyptr_ty],
                                    [NoCapture<0>, NoCapture<1>]>;
@@ -65,6 +80,10 @@ let TargetPrefix = "xcore" in {  // All intrinsics start with "llvm.xcore.".
                                    [NoCapture<0>, NoCapture<1>]>;
   def int_xcore_setpsc : Intrinsic<[],[llvm_anyptr_ty, llvm_i32_ty],
                                    [NoCapture<0>]>;
+  def int_xcore_peek : Intrinsic<[llvm_i32_ty],[llvm_anyptr_ty],
+                                 [NoCapture<0>]>;
+  def int_xcore_endin : Intrinsic<[llvm_i32_ty],[llvm_anyptr_ty],
+                                 [NoCapture<0>]>;
 
   // Intrinsics for events.
   def int_xcore_waitevent : Intrinsic<[llvm_ptr_ty],[], [IntrReadMem]>;
diff --git a/contrib/llvm/include/llvm/LinkAllPasses.h b/contrib/llvm/include/llvm/LinkAllPasses.h
index 8467d1149033..f690d045d172 100644
--- a/contrib/llvm/include/llvm/LinkAllPasses.h
+++ b/contrib/llvm/include/llvm/LinkAllPasses.h
@@ -93,7 +93,6 @@ namespace {
       (void) llvm::createLoopRotatePass();
       (void) llvm::createLowerExpectIntrinsicPass();
       (void) llvm::createLowerInvokePass();
-      (void) llvm::createLowerSetJmpPass();
       (void) llvm::createLowerSwitchPass();
       (void) llvm::createNoAAPass();
       (void) llvm::createNoProfileInfoPass();
@@ -128,7 +127,6 @@ namespace {
       (void) llvm::createStripDeadDebugInfoPass();
       (void) llvm::createStripDeadPrototypesPass();
       (void) llvm::createTailCallEliminationPass();
-      (void) llvm::createTailDuplicationPass();
       (void) llvm::createJumpThreadingPass();
       (void) llvm::createUnifyFunctionExitNodesPass();
       (void) llvm::createInstCountPass();
@@ -157,7 +155,7 @@ namespace {
       (void)new llvm::FindUsedTypes();
       (void)new llvm::ScalarEvolution();
       ((llvm::Function*)0)->viewCFGOnly();
-      llvm::RGPassManager RGM(0);
+      llvm::RGPassManager RGM;
       ((llvm::RegionPass*)0)->runOnRegion((llvm::Region*)0, RGM);
       llvm::AliasSetTracker X(*(llvm::AliasAnalysis*)0);
       X.add((llvm::Value*)0, 0, 0);  // for -print-alias-sets
diff --git a/contrib/llvm/include/llvm/Linker.h b/contrib/llvm/include/llvm/Linker.h
index b402a6090e2c..88908fbd72a7 100644
--- a/contrib/llvm/include/llvm/Linker.h
+++ b/contrib/llvm/include/llvm/Linker.h
@@ -57,7 +57,12 @@ class Linker {
       QuietWarnings = 2, ///< Don't print warnings to stderr.
       QuietErrors   = 4  ///< Don't print errors to stderr.
     };
-
+  
+    enum LinkerMode {
+      DestroySource = 0, // Allow source module to be destroyed.
+      PreserveSource = 1 // Preserve the source module.
+    };
+  
   /// @}
   /// @name Constructors
   /// @{
@@ -245,7 +250,7 @@ class Linker {
       Module* Src,              ///< Module linked into \p Dest
       std::string* ErrorMsg = 0 /// Error/diagnostic string
     ) { 
-      return LinkModules(Composite, Src, ErrorMsg ); 
+      return LinkModules(Composite, Src, Linker::DestroySource, ErrorMsg ); 
     }
 
     /// This is the heart of the linker. This method will take unconditional
@@ -259,7 +264,8 @@ class Linker {
     /// error.
     /// @returns True if an error occurs, false otherwise.
     /// @brief Generically link two modules together.
-    static bool LinkModules(Module* Dest, Module* Src, std::string* ErrorMsg);
+    static bool LinkModules(Module* Dest, Module* Src, unsigned Mode,
+                            std::string* ErrorMsg);
 
     /// This function looks through the Linker's LibPaths to find a library with
     /// the name \p Filename. If the library cannot be found, the returned path
diff --git a/contrib/llvm/include/llvm/MC/EDInstInfo.h b/contrib/llvm/include/llvm/MC/EDInstInfo.h
index 83d9e780feb7..0b9d3f63f677 100644
--- a/contrib/llvm/include/llvm/MC/EDInstInfo.h
+++ b/contrib/llvm/include/llvm/MC/EDInstInfo.h
@@ -21,7 +21,7 @@ struct EDInstInfo {
   uint8_t       numOperands;
   uint8_t       operandTypes[EDIS_MAX_OPERANDS];
   uint8_t       operandFlags[EDIS_MAX_OPERANDS];
-  const char    operandOrders[EDIS_MAX_SYNTAXES][EDIS_MAX_OPERANDS];
+  const signed char operandOrders[EDIS_MAX_SYNTAXES][EDIS_MAX_OPERANDS];
 };
   
 } // namespace llvm
diff --git a/contrib/llvm/include/llvm/Target/TargetAsmBackend.h b/contrib/llvm/include/llvm/MC/MCAsmBackend.h
similarity index 90%
rename from contrib/llvm/include/llvm/Target/TargetAsmBackend.h
rename to contrib/llvm/include/llvm/MC/MCAsmBackend.h
index 2111f6b7a950..4a0cf37a6eb2 100644
--- a/contrib/llvm/include/llvm/Target/TargetAsmBackend.h
+++ b/contrib/llvm/include/llvm/MC/MCAsmBackend.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetAsmBackend.h - Target Asm Backend -----*- C++ -*-===//
+//===-- llvm/MC/MCAsmBack.h - MC Asm Backend --------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TARGET_TARGETASMBACKEND_H
-#define LLVM_TARGET_TARGETASMBACKEND_H
+#ifndef LLVM_MC_MCASMBACKEND_H
+#define LLVM_MC_MCASMBACKEND_H
 
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCFixup.h"
@@ -25,17 +25,17 @@ template<typename T>
 class SmallVectorImpl;
 class raw_ostream;
 
-/// TargetAsmBackend - Generic interface to target specific assembler backends.
-class TargetAsmBackend {
-  TargetAsmBackend(const TargetAsmBackend &);   // DO NOT IMPLEMENT
-  void operator=(const TargetAsmBackend &);  // DO NOT IMPLEMENT
+/// MCAsmBackend - Generic interface to target specific assembler backends.
+class MCAsmBackend {
+  MCAsmBackend(const MCAsmBackend &);   // DO NOT IMPLEMENT
+  void operator=(const MCAsmBackend &);  // DO NOT IMPLEMENT
 protected: // Can only create subclasses.
-  TargetAsmBackend();
+  MCAsmBackend();
 
   unsigned HasReliableSymbolDifference : 1;
 
 public:
-  virtual ~TargetAsmBackend();
+  virtual ~MCAsmBackend();
 
   /// createObjectWriter - Create a new MCObjectWriter instance for use by the
   /// assembler backend to emit the final object file.
diff --git a/contrib/llvm/include/llvm/MC/MCAsmInfo.h b/contrib/llvm/include/llvm/MC/MCAsmInfo.h
index 41c171761e78..c3c296e23dc1 100644
--- a/contrib/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/contrib/llvm/include/llvm/MC/MCAsmInfo.h
@@ -16,8 +16,10 @@
 #ifndef LLVM_TARGET_ASM_INFO_H
 #define LLVM_TARGET_ASM_INFO_H
 
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/MC/MCDirectives.h"
 #include <cassert>
+#include <vector>
 
 namespace llvm {
   class MCExpr;
@@ -30,6 +32,14 @@ namespace llvm {
     enum ExceptionsType { None, DwarfCFI, SjLj, ARM, Win64 };
   }
 
+  namespace LCOMM {
+    enum LCOMMType { None, NoAlignment, ByteAlignment };
+  }
+
+  namespace Structors {
+    enum OutputOrder { None, PriorityOrder, ReversePriorityOrder };
+  }
+
   /// MCAsmInfo - This class is intended to be used as a base class for asm
   /// properties and features specific to the target.
   class MCAsmInfo {
@@ -62,6 +72,11 @@ namespace llvm {
     /// the macho-specific .tbss directive for emitting thread local BSS Symbols
     bool HasMachoTBSSDirective;                 // Default is false.
 
+    /// StructorOutputOrder - Whether the static ctor/dtor list should be output
+    /// in no particular order, in order of increasing priority or the reverse:
+    /// in order of decreasing priority (the default).
+    Structors::OutputOrder StructorOutputOrder; // Default is reverse order.
+
     /// HasStaticCtorDtorReferenceInStaticMode - True if the compiler should
     /// emit a ".reference .constructors_used" or ".reference .destructors_used"
     /// directive after the a static ctor/dtor list.  This directive is only
@@ -115,6 +130,13 @@ namespace llvm {
     const char *InlineAsmStart;              // Defaults to "#APP\n"
     const char *InlineAsmEnd;                // Defaults to "#NO_APP\n"
 
+    /// Code16Directive, Code32Directive, Code64Directive - These are assembly
+    /// directives that tells the assembler to interpret the following
+    /// instructions differently.
+    const char *Code16Directive;             // Defaults to ".code16"
+    const char *Code32Directive;             // Defaults to ".code32"
+    const char *Code64Directive;             // Defaults to ".code64"
+
     /// AssemblerDialect - Which dialect of an assembler variant to use.
     unsigned AssemblerDialect;               // Defaults to 0
 
@@ -155,6 +177,18 @@ namespace llvm {
     const char *Data32bitsDirective;         // Defaults to "\t.long\t"
     const char *Data64bitsDirective;         // Defaults to "\t.quad\t"
 
+    /// [Data|Code]Begin - These magic labels are used to marked a region as
+    /// data or code, and are used to provide additional information for
+    /// correct disassembly on targets that like to mix data and code within
+    /// a segment.  These labels will be implicitly suffixed by the streamer
+    /// to give them unique names.
+    const char *DataBegin;                   // Defaults to "$d."
+    const char *CodeBegin;                   // Defaults to "$a."
+    const char *JT8Begin;                    // Defaults to "$a."
+    const char *JT16Begin;                   // Defaults to "$a."
+    const char *JT32Begin;                   // Defaults to "$a."
+    bool SupportsDataRegions;
+
     /// GPRel32Directive - if non-null, a directive that is used to emit a word
     /// which should be relocated as a 32-bit GP-relative offset, e.g. .gpword
     /// on Mips or .gprel32 on Alpha.
@@ -220,9 +254,9 @@ namespace llvm {
     /// .long a - b
     bool HasAggressiveSymbolFolding;           // Defaults to true.
 
-    /// HasLCOMMDirective - This is true if the target supports the .lcomm
-    /// directive.
-    bool HasLCOMMDirective;                  // Defaults to false.
+    /// LCOMMDirectiveType - Describes if the target supports the .lcomm
+    /// directive and whether it has an alignment parameter.
+    LCOMM::LCOMMType LCOMMDirectiveType;     // Defaults to LCOMM::None.
 
     /// COMMDirectiveAlignmentIsInBytes - True is COMMDirective's optional
     /// alignment is to be specified in bytes instead of log2(n).
@@ -304,6 +338,10 @@ namespace llvm {
 
     const char *const *AsmTransCBE;          // Defaults to empty
 
+    //===--- Prologue State ----------------------------------------------===//
+
+    std::vector<MachineMove> InitialFrameState;
+
   public:
     explicit MCAsmInfo();
     virtual ~MCAsmInfo();
@@ -345,6 +383,14 @@ namespace llvm {
     }
     const char *getGPRel32Directive() const { return GPRel32Directive; }
 
+    /// [Code|Data]Begin label name accessors.
+    const char *getCodeBeginLabelName() const { return CodeBegin; }
+    const char *getDataBeginLabelName() const { return DataBegin; }
+    const char *getJumpTable8BeginLabelName() const { return JT8Begin; }
+    const char *getJumpTable16BeginLabelName() const { return JT16Begin; }
+    const char *getJumpTable32BeginLabelName() const { return JT32Begin; }
+    bool getSupportsDataRegions() const { return SupportsDataRegions; }
+
     /// getNonexecutableStackSection - Targets can implement this method to
     /// specify a section to switch to if the translation unit doesn't have any
     /// trampolines that require an executable stack.
@@ -378,6 +424,9 @@ namespace llvm {
     //
     bool hasMachoZeroFillDirective() const { return HasMachoZeroFillDirective; }
     bool hasMachoTBSSDirective() const { return HasMachoTBSSDirective; }
+    Structors::OutputOrder getStructorOutputOrder() const {
+      return StructorOutputOrder;
+    }
     bool hasStaticCtorDtorReferenceInStaticMode() const {
       return HasStaticCtorDtorReferenceInStaticMode;
     }
@@ -417,6 +466,15 @@ namespace llvm {
     const char *getInlineAsmEnd() const {
       return InlineAsmEnd;
     }
+    const char *getCode16Directive() const {
+      return Code16Directive;
+    }
+    const char *getCode32Directive() const {
+      return Code32Directive;
+    }
+    const char *getCode64Directive() const {
+      return Code64Directive;
+    }
     unsigned getAssemblerDialect() const {
       return AssemblerDialect;
     }
@@ -457,7 +515,9 @@ namespace llvm {
     bool hasAggressiveSymbolFolding() const {
       return HasAggressiveSymbolFolding;
     }
-    bool hasLCOMMDirective() const { return HasLCOMMDirective; }
+    LCOMM::LCOMMType getLCOMMDirectiveType() const {
+      return LCOMMDirectiveType;
+    }
     bool hasDotTypeDotSizeDirective() const {return HasDotTypeDotSizeDirective;}
     bool getCOMMDirectiveAlignmentIsInBytes() const {
       return COMMDirectiveAlignmentIsInBytes;
@@ -512,6 +572,14 @@ namespace llvm {
     const char *const *getAsmCBE() const {
       return AsmTransCBE;
     }
+
+    void addInitialFrameState(MCSymbol *label, const MachineLocation &D,
+                              const MachineLocation &S) {
+      InitialFrameState.push_back(MachineMove(label, D, S));
+    }
+    const std::vector<MachineMove> &getInitialFrameState() const {
+      return InitialFrameState;
+    }
   };
 }
 
diff --git a/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h b/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h
index c85aa3da9572..1f6c49938c9c 100644
--- a/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h
+++ b/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h
@@ -18,11 +18,6 @@
 #include "llvm/MC/MCAsmInfo.h"
 
 namespace llvm {
-  class GlobalValue;
-  class GlobalVariable;
-  class Type;
-  class Mangler;
-
   struct MCAsmInfoDarwin : public MCAsmInfo {
     explicit MCAsmInfoDarwin();
   };
diff --git a/contrib/llvm/include/llvm/MC/MCAssembler.h b/contrib/llvm/include/llvm/MC/MCAssembler.h
index fc919669e82d..b8f8cc4cec90 100644
--- a/contrib/llvm/include/llvm/MC/MCAssembler.h
+++ b/contrib/llvm/include/llvm/MC/MCAssembler.h
@@ -10,14 +10,14 @@
 #ifndef LLVM_MC_MCASSEMBLER_H
 #define LLVM_MC_MCASSEMBLER_H
 
+#include "llvm/MC/MCFixup.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCInst.h"
 #include "llvm/Support/DataTypes.h"
 #include <vector> // FIXME: Shouldn't be needed.
 
@@ -36,7 +36,7 @@ class MCSectionData;
 class MCSymbol;
 class MCSymbolData;
 class MCValue;
-class TargetAsmBackend;
+class MCAsmBackend;
 
 class MCFragment : public ilist_node<MCFragment> {
   friend class MCAsmLayout;
@@ -660,7 +660,7 @@ class MCAssembler {
 
   MCContext &Context;
 
-  TargetAsmBackend &Backend;
+  MCAsmBackend &Backend;
 
   MCCodeEmitter &Emitter;
 
@@ -780,14 +780,14 @@ class MCAssembler {
   // concrete and require clients to pass in a target like object. The other
   // option is to make this abstract, and have targets provide concrete
   // implementations as we do with AsmParser.
-  MCAssembler(MCContext &Context_, TargetAsmBackend &Backend_,
+  MCAssembler(MCContext &Context_, MCAsmBackend &Backend_,
               MCCodeEmitter &Emitter_, MCObjectWriter &Writer_,
               raw_ostream &OS);
   ~MCAssembler();
 
   MCContext &getContext() const { return Context; }
 
-  TargetAsmBackend &getBackend() const { return Backend; }
+  MCAsmBackend &getBackend() const { return Backend; }
 
   MCCodeEmitter &getEmitter() const { return Emitter; }
 
diff --git a/contrib/llvm/include/llvm/MC/MCAtom.h b/contrib/llvm/include/llvm/MC/MCAtom.h
new file mode 100644
index 000000000000..682cf7cd76c6
--- /dev/null
+++ b/contrib/llvm/include/llvm/MC/MCAtom.h
@@ -0,0 +1,68 @@
+//===-- llvm/MC/MCAtom.h - MCAtom class ---------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the MCAtom class, which is used to
+// represent a contiguous region in a decoded object that is uniformly data or
+// instructions;
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCATOM_H
+#define LLVM_MC_MCATOM_H
+
+#include "llvm/MC/MCInst.h"
+#include "llvm/Support/DataTypes.h"
+#include <vector>
+
+namespace llvm {
+
+class MCModule;
+
+/// MCData - An entry in a data MCAtom.
+// NOTE: This may change to a more complex type in the future.
+typedef uint8_t MCData;
+
+/// MCAtom - Represents a contiguous range of either instructions (a TextAtom)
+/// or data (a DataAtom).  Address ranges are expressed as _closed_ intervals.
+class MCAtom {
+  friend class MCModule;
+  typedef enum { TextAtom, DataAtom } AtomType;
+
+  AtomType Type;
+  MCModule *Parent;
+  uint64_t Begin, End;
+
+  std::vector<std::pair<uint64_t, MCInst> > Text;
+  std::vector<MCData> Data;
+
+  // Private constructor - only callable by MCModule
+  MCAtom(AtomType T, MCModule *P, uint64_t B, uint64_t E)
+    : Type(T), Parent(P), Begin(B), End(E) { }
+
+public:
+  bool isTextAtom() { return Type == TextAtom; }
+  bool isDataAtom() { return Type == DataAtom; }
+
+  void addInst(const MCInst &I, uint64_t Address, unsigned Size);
+  void addData(const MCData &D);
+
+  /// split - Splits the atom in two at a given address, which must align with
+  /// and instruction boundary if this is a TextAtom.  Returns the newly created
+  /// atom representing the high part of the split.
+  MCAtom *split(uint64_t SplitPt);
+
+  /// truncate - Truncates an atom so that TruncPt is the last byte address
+  /// contained in the atom.
+  void truncate(uint64_t TruncPt);
+};
+
+}
+
+#endif
+
diff --git a/contrib/llvm/include/llvm/MC/MCCodeGenInfo.h b/contrib/llvm/include/llvm/MC/MCCodeGenInfo.h
new file mode 100644
index 000000000000..1c54c47e2d95
--- /dev/null
+++ b/contrib/llvm/include/llvm/MC/MCCodeGenInfo.h
@@ -0,0 +1,41 @@
+//===-- llvm/MC/MCCodeGenInfo.h - Target CodeGen Info -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file tracks information about the target which can affect codegen,
+// asm parsing, and asm printing. For example, relocation model.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCCODEGENINFO_H
+#define LLVM_MC_MCCODEGENINFO_H
+
+#include "llvm/Support/CodeGen.h"
+
+namespace llvm {
+
+  class MCCodeGenInfo {
+    /// RelocationModel - Relocation model: statcic, pic, etc.
+    ///
+    Reloc::Model RelocationModel;
+
+    /// CMModel - Code model.
+    ///
+    CodeModel::Model CMModel;
+
+  public:
+    void InitMCCodeGenInfo(Reloc::Model RM = Reloc::Default,
+                           CodeModel::Model CM = CodeModel::Default);
+
+    Reloc::Model getRelocationModel() const { return RelocationModel; }
+
+    CodeModel::Model getCodeModel() const { return CMModel; }
+  };
+} // namespace llvm
+
+#endif
diff --git a/contrib/llvm/include/llvm/MC/MCContext.h b/contrib/llvm/include/llvm/MC/MCContext.h
index 43a9ce6cfa4d..a49a35c8d5ba 100644
--- a/contrib/llvm/include/llvm/MC/MCContext.h
+++ b/contrib/llvm/include/llvm/MC/MCContext.h
@@ -26,10 +26,11 @@ namespace llvm {
   class MCLabel;
   class MCDwarfFile;
   class MCDwarfLoc;
+  class MCObjectFileInfo;
+  class MCRegisterInfo;
   class MCLineSection;
   class StringRef;
   class Twine;
-  class TargetAsmInfo;
   class MCSectionMachO;
   class MCSectionELF;
 
@@ -46,7 +47,11 @@ namespace llvm {
     /// The MCAsmInfo for this target.
     const MCAsmInfo &MAI;
 
-    const TargetAsmInfo *TAI;
+    /// The MCRegisterInfo for this target.
+    const MCRegisterInfo &MRI;
+
+    /// The MCObjectFileInfo for this target.
+    const MCObjectFileInfo *MOFI;
 
     /// Allocator - Allocator object used for creating machine code objects.
     ///
@@ -110,12 +115,15 @@ namespace llvm {
     MCSymbol *CreateSymbol(StringRef Name);
 
   public:
-    explicit MCContext(const MCAsmInfo &MAI, const TargetAsmInfo *TAI);
+    explicit MCContext(const MCAsmInfo &MAI, const MCRegisterInfo &MRI,
+                       const MCObjectFileInfo *MOFI);
     ~MCContext();
 
     const MCAsmInfo &getAsmInfo() const { return MAI; }
 
-    const TargetAsmInfo &getTargetAsmInfo() const { return *TAI; }
+    const MCRegisterInfo &getRegisterInfo() const { return MRI; }
+
+    const MCObjectFileInfo *getObjectFileInfo() const { return MOFI; }
 
     void setAllowTemporaryLabels(bool Value) { AllowTemporaryLabels = Value; }
 
diff --git a/contrib/llvm/include/llvm/MC/MCDirectives.h b/contrib/llvm/include/llvm/MC/MCDirectives.h
index 1df55dc252e3..9180d1b369fe 100644
--- a/contrib/llvm/include/llvm/MC/MCDirectives.h
+++ b/contrib/llvm/include/llvm/MC/MCDirectives.h
@@ -47,8 +47,9 @@ enum MCSymbolAttr {
 enum MCAssemblerFlag {
   MCAF_SyntaxUnified,         ///< .syntax (ARM/ELF)
   MCAF_SubsectionsViaSymbols, ///< .subsections_via_symbols (MachO)
-  MCAF_Code16,                ///< .code 16
-  MCAF_Code32                 ///< .code 32
+  MCAF_Code16,                ///< .code16 (X86) / .code 16 (ARM)
+  MCAF_Code32,                ///< .code32 (X86) / .code 32 (ARM)
+  MCAF_Code64                 ///< .code64 (X86)
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/include/llvm/MC/MCDisassembler.h b/contrib/llvm/include/llvm/MC/MCDisassembler.h
index ce8759a882eb..454277d6852c 100644
--- a/contrib/llvm/include/llvm/MC/MCDisassembler.h
+++ b/contrib/llvm/include/llvm/MC/MCDisassembler.h
@@ -15,6 +15,7 @@
 namespace llvm {
   
 class MCInst;
+class MCSubtargetInfo;
 class MemoryObject;
 class raw_ostream;
 class MCContext;
@@ -25,8 +26,38 @@ struct EDInstInfo;
 ///   and provides an array of assembly instructions.
 class MCDisassembler {
 public:
+  /// Ternary decode status. Most backends will just use Fail and
+  /// Success, however some have a concept of an instruction with
+  /// understandable semantics but which is architecturally
+  /// incorrect. An example of this is ARM UNPREDICTABLE instructions
+  /// which are disassemblable but cause undefined behaviour.
+  ///
+  /// Because it makes sense to disassemble these instructions, there
+  /// is a "soft fail" failure mode that indicates the MCInst& is
+  /// valid but architecturally incorrect.
+  ///
+  /// The enum numbers are deliberately chosen such that reduction
+  /// from Success->SoftFail ->Fail can be done with a simple
+  /// bitwise-AND:
+  ///
+  ///   LEFT & TOP =  | Success       Unpredictable   Fail
+  ///   --------------+-----------------------------------
+  ///   Success       | Success       Unpredictable   Fail
+  ///   Unpredictable | Unpredictable Unpredictable   Fail
+  ///   Fail          | Fail          Fail            Fail
+  ///
+  /// An easy way of encoding this is as 0b11, 0b01, 0b00 for
+  /// Success, SoftFail, Fail respectively.
+  enum DecodeStatus {
+    Fail = 0,
+    SoftFail = 1,
+    Success = 3
+  };
+
   /// Constructor     - Performs initial setup for the disassembler.
-  MCDisassembler() : GetOpInfo(0), DisInfo(0), Ctx(0) {}
+  MCDisassembler(const MCSubtargetInfo &STI) : GetOpInfo(0), SymbolLookUp(0),
+                                               DisInfo(0), Ctx(0),
+                                               STI(STI), CommentStream(0) {}
   
   virtual ~MCDisassembler();
   
@@ -41,12 +72,17 @@ class MCDisassembler {
   /// @param address  - The address, in the memory space of region, of the first
   ///                   byte of the instruction.
   /// @param vStream  - The stream to print warnings and diagnostic messages on.
-  /// @return         - True if the instruction is valid; false otherwise.
-  virtual bool          getInstruction(MCInst& instr,
+  /// @param cStream  - The stream to print comments and annotations on.
+  /// @return         - MCDisassembler::Success if the instruction is valid,
+  ///                   MCDisassembler::SoftFail if the instruction was 
+  ///                                            disassemblable but invalid,
+  ///                   MCDisassembler::Fail if the instruction was invalid.
+  virtual DecodeStatus  getInstruction(MCInst& instr,
                                        uint64_t& size,
                                        const MemoryObject &region,
                                        uint64_t address,
-                                       raw_ostream &vStream) const = 0;
+                                       raw_ostream &vStream,
+                                       raw_ostream &cStream) const = 0;
 
   /// getEDInfo - Returns the enhanced instruction information corresponding to
   ///   the disassembler.
@@ -62,23 +98,37 @@ class MCDisassembler {
   //
   // The function to get the symbolic information for operands.
   LLVMOpInfoCallback GetOpInfo;
+  // The function to lookup a symbol name.
+  LLVMSymbolLookupCallback SymbolLookUp;
   // The pointer to the block of symbolic information for above call back.
   void *DisInfo;
   // The assembly context for creating symbols and MCExprs in place of
   // immediate operands when there is symbolic information.
   MCContext *Ctx;
+protected:
+  // Subtarget information, for instruction decoding predicates if required.
+  const MCSubtargetInfo &STI;
 
 public:
   void setupForSymbolicDisassembly(LLVMOpInfoCallback getOpInfo,
+                                   LLVMSymbolLookupCallback symbolLookUp,
                                    void *disInfo,
                                    MCContext *ctx) {
     GetOpInfo = getOpInfo;
+    SymbolLookUp = symbolLookUp;
     DisInfo = disInfo;
     Ctx = ctx;
   }
   LLVMOpInfoCallback getLLVMOpInfoCallback() const { return GetOpInfo; }
+  LLVMSymbolLookupCallback getLLVMSymbolLookupCallback() const {
+    return SymbolLookUp;
+  }
   void *getDisInfoBlock() const { return DisInfo; }
   MCContext *getMCContext() const { return Ctx; }
+
+  // Marked mutable because we cache it inside the disassembler, rather than
+  // having to pass it around as an argument through all the autogenerated code.
+  mutable raw_ostream *CommentStream;
 };
 
 } // namespace llvm
diff --git a/contrib/llvm/include/llvm/MC/MCDwarf.h b/contrib/llvm/include/llvm/MC/MCDwarf.h
index 90c372856749..431e3c4da86a 100644
--- a/contrib/llvm/include/llvm/MC/MCDwarf.h
+++ b/contrib/llvm/include/llvm/MC/MCDwarf.h
@@ -16,15 +16,13 @@
 #define LLVM_MC_MCDWARF_H
 
 #include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/MachineLocation.h" // FIXME
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Dwarf.h"
 #include <vector>
 
 namespace llvm {
-  class TargetAsmInfo;
-  class MachineMove;
   class MCContext;
   class MCExpr;
   class MCSection;
@@ -265,7 +263,7 @@ namespace llvm {
   struct MCDwarfFrameInfo {
     MCDwarfFrameInfo() : Begin(0), End(0), Personality(0), Lsda(0),
                          Function(0), Instructions(), PersonalityEncoding(),
-                         LsdaEncoding(0) {}
+                         LsdaEncoding(0), CompactUnwindEncoding(0) {}
     MCSymbol *Begin;
     MCSymbol *End;
     const MCSymbol *Personality;
@@ -274,6 +272,7 @@ namespace llvm {
     std::vector<MCCFIInstruction> Instructions;
     unsigned PersonalityEncoding;
     unsigned LsdaEncoding;
+    uint32_t CompactUnwindEncoding;
   };
 
   class MCDwarfFrameEmitter {
diff --git a/contrib/llvm/include/llvm/MC/MCInst.h b/contrib/llvm/include/llvm/MC/MCInst.h
index d6ef7b4c33c1..d38476477495 100644
--- a/contrib/llvm/include/llvm/MC/MCInst.h
+++ b/contrib/llvm/include/llvm/MC/MCInst.h
@@ -144,6 +144,16 @@ class MCInst {
     Operands.push_back(Op);
   }
 
+  void clear() { Operands.clear(); }
+  size_t size() { return Operands.size(); }
+
+  typedef SmallVector<MCOperand, 8>::iterator iterator;
+  iterator begin() { return Operands.begin(); }
+  iterator end()   { return Operands.end();   }
+  iterator insert(iterator I, const MCOperand &Op) {
+    return Operands.insert(I, Op);
+  }
+
   void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
   void dump() const;
 
diff --git a/contrib/llvm/include/llvm/MC/MCInstPrinter.h b/contrib/llvm/include/llvm/MC/MCInstPrinter.h
index 39002dabca14..01ad2d3f8088 100644
--- a/contrib/llvm/include/llvm/MC/MCInstPrinter.h
+++ b/contrib/llvm/include/llvm/MC/MCInstPrinter.h
@@ -28,6 +28,9 @@ class MCInstPrinter {
 
   /// The current set of available features.
   unsigned AvailableFeatures;
+
+  /// Utility function for printing annotations.
+  void printAnnotation(raw_ostream &OS, StringRef Annot);
 public:
   MCInstPrinter(const MCAsmInfo &mai)
     : CommentStream(0), MAI(mai), AvailableFeatures(0) {}
@@ -39,7 +42,8 @@ class MCInstPrinter {
 
   /// printInst - Print the specified MCInst to the specified raw_ostream.
   ///
-  virtual void printInst(const MCInst *MI, raw_ostream &OS) = 0;
+  virtual void printInst(const MCInst *MI, raw_ostream &OS,
+                         StringRef Annot) = 0;
 
   /// getOpcodeName - Return the name of the specified opcode enum (e.g.
   /// "MOV32ri") or empty if we can't resolve it.
diff --git a/contrib/llvm/include/llvm/MC/MCInstrAnalysis.h b/contrib/llvm/include/llvm/MC/MCInstrAnalysis.h
new file mode 100644
index 000000000000..8f3c499b1c73
--- /dev/null
+++ b/contrib/llvm/include/llvm/MC/MCInstrAnalysis.h
@@ -0,0 +1,61 @@
+//===-- llvm/MC/MCInstrAnalysis.h - InstrDesc target hooks ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the MCInstrAnalysis class which the MCTargetDescs can
+// derive from to give additional information to MC.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCInstrInfo.h"
+
+namespace llvm {
+
+class MCInstrAnalysis {
+protected:
+  friend class Target;
+  const MCInstrInfo *Info;
+
+public:
+  MCInstrAnalysis(const MCInstrInfo *Info) : Info(Info) {}
+
+  virtual ~MCInstrAnalysis() {}
+
+  virtual bool isBranch(const MCInst &Inst) const {
+    return Info->get(Inst.getOpcode()).isBranch();
+  }
+
+  virtual bool isConditionalBranch(const MCInst &Inst) const {
+    return Info->get(Inst.getOpcode()).isBranch();
+  }
+
+  virtual bool isUnconditionalBranch(const MCInst &Inst) const {
+    return Info->get(Inst.getOpcode()).isUnconditionalBranch();
+  }
+
+  virtual bool isIndirectBranch(const MCInst &Inst) const {
+    return Info->get(Inst.getOpcode()).isIndirectBranch();
+  }
+
+  virtual bool isCall(const MCInst &Inst) const {
+    return Info->get(Inst.getOpcode()).isCall();
+  }
+
+  virtual bool isReturn(const MCInst &Inst) const {
+    return Info->get(Inst.getOpcode()).isReturn();
+  }
+
+  /// evaluateBranch - Given a branch instruction try to get the address the
+  /// branch targets. Otherwise return -1.
+  virtual uint64_t
+  evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size) const;
+};
+
+}
diff --git a/contrib/llvm/include/llvm/MC/MCInstrDesc.h b/contrib/llvm/include/llvm/MC/MCInstrDesc.h
index 49969143cd74..aafa800c1ac8 100644
--- a/contrib/llvm/include/llvm/MC/MCInstrDesc.h
+++ b/contrib/llvm/include/llvm/MC/MCInstrDesc.h
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This file defines the MCOperandInfo and MCInstrDesc classes, which
-// are used to describe target instructions and their operands. 
+// are used to describe target instructions and their operands.
 //
 //===----------------------------------------------------------------------===//
 
@@ -22,14 +22,14 @@ namespace llvm {
 //===----------------------------------------------------------------------===//
 // Machine Operand Flags and Description
 //===----------------------------------------------------------------------===//
-  
+
 namespace MCOI {
   // Operand constraints
   enum OperandConstraint {
     TIED_TO = 0,    // Must be allocated the same register as.
     EARLY_CLOBBER   // Operand is an early clobber register operand
   };
-  
+
   /// OperandFlags - These are flags set on operands, but should be considered
   /// private, all access should go through the MCOperandInfo accessors.
   /// See the accessors for a description of what these are.
@@ -54,15 +54,15 @@ namespace MCOI {
 ///
 class MCOperandInfo {
 public:
-  /// RegClass - This specifies the register class enumeration of the operand 
+  /// RegClass - This specifies the register class enumeration of the operand
   /// if the operand is a register.  If isLookupPtrRegClass is set, then this is
   /// an index that is passed to TargetRegisterInfo::getPointerRegClass(x) to
   /// get a dynamic register class.
   short RegClass;
-  
+
   /// Flags - These are flags from the MCOI::OperandFlags enum.
   unsigned short Flags;
-  
+
   /// Lower 16 bits are used to specify which constraints are set. The higher 16
   /// bits are used to specify the value of constraints (4 bits each).
   unsigned Constraints;
@@ -70,21 +70,21 @@ class MCOperandInfo {
   /// OperandType - Information about the type of the operand.
   MCOI::OperandType OperandType;
   /// Currently no other information.
-  
+
   /// isLookupPtrRegClass - Set if this operand is a pointer value and it
   /// requires a callback to look up its register class.
-  bool isLookupPtrRegClass() const { return Flags&(1 <<MCOI::LookupPtrRegClass);}
-  
+  bool isLookupPtrRegClass() const {return Flags&(1 <<MCOI::LookupPtrRegClass);}
+
   /// isPredicate - Set if this is one of the operands that made up of
   /// the predicate operand that controls an isPredicable() instruction.
   bool isPredicate() const { return Flags & (1 << MCOI::Predicate); }
-  
+
   /// isOptionalDef - Set if this operand is a optional def.
   ///
   bool isOptionalDef() const { return Flags & (1 << MCOI::OptionalDef); }
 };
 
-  
+
 //===----------------------------------------------------------------------===//
 // Machine Instruction Flags and Description
 //===----------------------------------------------------------------------===//
@@ -97,6 +97,7 @@ namespace MCID {
   enum {
     Variadic = 0,
     HasOptionalDef,
+    Pseudo,
     Return,
     Call,
     Barrier,
@@ -116,6 +117,7 @@ namespace MCID {
     Commutable,
     ConvertibleTo3Addr,
     UsesCustomInserter,
+    HasPostISelHook,
     Rematerializable,
     CheapAsAMove,
     ExtraSrcRegAllocReq,
@@ -158,13 +160,13 @@ class MCInstrDesc {
   unsigned getOpcode() const {
     return Opcode;
   }
-  
+
   /// getName - Return the name of the record in the .td file for this
   /// instruction, for example "ADD8ri".
   const char *getName() const {
     return Name;
   }
-  
+
   /// getNumOperands - Return the number of declared MachineOperands for this
   /// MachineInstruction.  Note that variadic (isVariadic() returns true)
   /// instructions may have additional operands at the end of the list, and note
@@ -173,15 +175,15 @@ class MCInstrDesc {
   unsigned getNumOperands() const {
     return NumOperands;
   }
-  
+
   /// getNumDefs - Return the number of MachineOperands that are register
-  /// definitions.  Register definitions always occur at the start of the 
+  /// definitions.  Register definitions always occur at the start of the
   /// machine operand list.  This is the number of "outs" in the .td file,
   /// and does not include implicit defs.
   unsigned getNumDefs() const {
     return NumDefs;
   }
-  
+
   /// isVariadic - Return true if this instruction can have a variable number of
   /// operands.  In this case, the variable operands will be after the normal
   /// operands but before the implicit definitions and uses (if any are
@@ -189,13 +191,13 @@ class MCInstrDesc {
   bool isVariadic() const {
     return Flags & (1 << MCID::Variadic);
   }
-  
+
   /// hasOptionalDef - Set if this instruction has an optional definition, e.g.
   /// ARM instructions which can set condition code if 's' bit is set.
   bool hasOptionalDef() const {
     return Flags & (1 << MCID::HasOptionalDef);
   }
-  
+
   /// getImplicitUses - Return a list of registers that are potentially
   /// read by any instance of this machine instruction.  For example, on X86,
   /// the "adc" instruction adds two register operands and adds the carry bit in
@@ -208,7 +210,7 @@ class MCInstrDesc {
   const unsigned *getImplicitUses() const {
     return ImplicitUses;
   }
-      
+
   /// getNumImplicitUses - Return the number of implicit uses this instruction
   /// has.
   unsigned getNumImplicitUses() const {
@@ -231,7 +233,7 @@ class MCInstrDesc {
   const unsigned *getImplicitDefs() const {
     return ImplicitDefs;
   }
-  
+
   /// getNumImplicitDefs - Return the number of implicit defs this instruction
   /// has.
   unsigned getNumImplicitDefs() const {
@@ -240,7 +242,7 @@ class MCInstrDesc {
     for (; ImplicitDefs[i]; ++i) /*empty*/;
     return i;
   }
-  
+
   /// hasImplicitUseOfPhysReg - Return true if this instruction implicitly
   /// uses the specified physical register.
   bool hasImplicitUseOfPhysReg(unsigned Reg) const {
@@ -249,7 +251,7 @@ class MCInstrDesc {
         if (*ImpUses == Reg) return true;
     return false;
   }
-  
+
   /// hasImplicitDefOfPhysReg - Return true if this instruction implicitly
   /// defines the specified physical register.
   bool hasImplicitDefOfPhysReg(unsigned Reg) const {
@@ -267,28 +269,47 @@ class MCInstrDesc {
   unsigned getSchedClass() const {
     return SchedClass;
   }
-  
+
   /// getSize - Return the number of bytes in the encoding of this instruction,
   /// or zero if the encoding size cannot be known from the opcode.
   unsigned getSize() const {
     return Size;
   }
 
+  /// isPseudo - Return true if this is a pseudo instruction that doesn't
+  /// correspond to a real machine instruction.
+  ///
+  bool isPseudo() const {
+    return Flags & (1 << MCID::Pseudo);
+  }
+
   bool isReturn() const {
     return Flags & (1 << MCID::Return);
   }
-  
+
   bool isCall() const {
     return Flags & (1 << MCID::Call);
   }
-  
+
   /// isBarrier - Returns true if the specified instruction stops control flow
   /// from executing the instruction immediately following it.  Examples include
   /// unconditional branches and return instructions.
   bool isBarrier() const {
     return Flags & (1 << MCID::Barrier);
   }
-  
+
+  /// findFirstPredOperandIdx() - Find the index of the first operand in the
+  /// operand list that is used to represent the predicate. It returns -1 if
+  /// none is found.
+  int findFirstPredOperandIdx() const {
+    if (isPredicable()) {
+      for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
+        if (OpInfo[i].isPredicate())
+          return i;
+    }
+    return -1;
+  }
+
   /// isTerminator - Returns true if this instruction part of the terminator for
   /// a basic block.  Typically this is things like return and branch
   /// instructions.
@@ -298,7 +319,7 @@ class MCInstrDesc {
   bool isTerminator() const {
     return Flags & (1 << MCID::Terminator);
   }
-  
+
   /// isBranch - Returns true if this is a conditional, unconditional, or
   /// indirect branch.  Predicates below can be used to discriminate between
   /// these cases, and the TargetInstrInfo::AnalyzeBranch method can be used to
@@ -320,7 +341,7 @@ class MCInstrDesc {
   bool isConditionalBranch() const {
     return isBranch() & !isBarrier() & !isIndirectBranch();
   }
-  
+
   /// isUnconditionalBranch - Return true if this is a branch which always
   /// transfers control flow to some other block.  The
   /// TargetInstrInfo::AnalyzeBranch method can be used to get more information
@@ -328,7 +349,7 @@ class MCInstrDesc {
   bool isUnconditionalBranch() const {
     return isBranch() & isBarrier() & !isIndirectBranch();
   }
-  
+
   // isPredicable - Return true if this instruction has a predicate operand that
   // controls execution.  It may be set to 'always', or may be set to other
   /// values.   There are various methods in TargetInstrInfo that can be used to
@@ -336,14 +357,14 @@ class MCInstrDesc {
   bool isPredicable() const {
     return Flags & (1 << MCID::Predicable);
   }
-  
+
   /// isCompare - Return true if this instruction is a comparison.
   bool isCompare() const {
     return Flags & (1 << MCID::Compare);
   }
-  
+
   /// isMoveImmediate - Return true if this instruction is a move immediate
-  /// (including conditional moves) instruction. 
+  /// (including conditional moves) instruction.
   bool isMoveImmediate() const {
     return Flags & (1 << MCID::MoveImm);
   }
@@ -353,20 +374,20 @@ class MCInstrDesc {
   bool isBitcast() const {
     return Flags & (1 << MCID::Bitcast);
   }
-  
+
   /// isNotDuplicable - Return true if this instruction cannot be safely
   /// duplicated.  For example, if the instruction has a unique labels attached
   /// to it, duplicating it would cause multiple definition errors.
   bool isNotDuplicable() const {
     return Flags & (1 << MCID::NotDuplicable);
   }
-  
+
   /// hasDelaySlot - Returns true if the specified instruction has a delay slot
   /// which must be filled by the code generator.
   bool hasDelaySlot() const {
     return Flags & (1 << MCID::DelaySlot);
   }
-  
+
   /// canFoldAsLoad - Return true for instructions that can be folded as
   /// memory operands in other instructions. The most common use for this
   /// is instructions that are simple loads from memory that don't modify
@@ -378,7 +399,7 @@ class MCInstrDesc {
   bool canFoldAsLoad() const {
     return Flags & (1 << MCID::FoldableAsLoad);
   }
-  
+
   //===--------------------------------------------------------------------===//
   // Side Effect Analysis
   //===--------------------------------------------------------------------===//
@@ -389,8 +410,8 @@ class MCInstrDesc {
   bool mayLoad() const {
     return Flags & (1 << MCID::MayLoad);
   }
-  
-  
+
+
   /// mayStore - Return true if this instruction could possibly modify memory.
   /// Instructions with this flag set are not necessarily simple store
   /// instructions, they may store a modified value based on their operands, or
@@ -398,7 +419,7 @@ class MCInstrDesc {
   bool mayStore() const {
     return Flags & (1 << MCID::MayStore);
   }
-  
+
   /// hasUnmodeledSideEffects - Return true if this instruction has side
   /// effects that are not modeled by other flags.  This does not return true
   /// for instructions whose effects are captured by:
@@ -415,14 +436,14 @@ class MCInstrDesc {
   bool hasUnmodeledSideEffects() const {
     return Flags & (1 << MCID::UnmodeledSideEffects);
   }
-  
+
   //===--------------------------------------------------------------------===//
   // Flags that indicate whether an instruction can be modified by a method.
   //===--------------------------------------------------------------------===//
-  
+
   /// isCommutable - Return true if this may be a 2- or 3-address
   /// instruction (of the form "X = op Y, Z, ..."), which produces the same
-  /// result if Y and Z are exchanged.  If this flag is set, then the 
+  /// result if Y and Z are exchanged.  If this flag is set, then the
   /// TargetInstrInfo::commuteInstruction method may be used to hack on the
   /// instruction.
   ///
@@ -433,7 +454,7 @@ class MCInstrDesc {
   bool isCommutable() const {
     return Flags & (1 << MCID::Commutable);
   }
-  
+
   /// isConvertibleTo3Addr - Return true if this is a 2-address instruction
   /// which can be changed into a 3-address instruction if needed.  Doing this
   /// transformation can be profitable in the register allocator, because it
@@ -451,11 +472,11 @@ class MCInstrDesc {
   bool isConvertibleTo3Addr() const {
     return Flags & (1 << MCID::ConvertibleTo3Addr);
   }
-  
+
   /// usesCustomInsertionHook - Return true if this instruction requires
   /// custom insertion support when the DAG scheduler is inserting it into a
   /// machine basic block.  If this is true for the instruction, it basically
-  /// means that it is a pseudo instruction used at SelectionDAG time that is 
+  /// means that it is a pseudo instruction used at SelectionDAG time that is
   /// expanded out into magic code by the target when MachineInstrs are formed.
   ///
   /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method
@@ -463,7 +484,15 @@ class MCInstrDesc {
   bool usesCustomInsertionHook() const {
     return Flags & (1 << MCID::UsesCustomInserter);
   }
-  
+
+  /// hasPostISelHook - Return true if this instruction requires *adjustment*
+  /// after instruction selection by calling a target hook. For example, this
+  /// can be used to fill in ARM 's' optional operand depending on whether
+  /// the conditional flag register is used.
+  bool hasPostISelHook() const {
+    return Flags & (1 << MCID::HasPostISelHook);
+  }
+
   /// isRematerializable - Returns true if this instruction is a candidate for
   /// remat.  This flag is deprecated, please don't use it anymore.  If this
   /// flag is set, the isReallyTriviallyReMaterializable() method is called to
diff --git a/contrib/llvm/include/llvm/MC/MCModule.h b/contrib/llvm/include/llvm/MC/MCModule.h
new file mode 100644
index 000000000000..755fa025fbc7
--- /dev/null
+++ b/contrib/llvm/include/llvm/MC/MCModule.h
@@ -0,0 +1,58 @@
+//===-- llvm/MC/MCModule.h - MCModule class ---------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the MCModule class, which is used to
+// represent a complete, disassembled object file or executable.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCMODULE_H
+#define LLVM_MC_MCMODULE_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/IntervalMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/DataTypes.h"
+
+namespace llvm {
+
+class MCAtom;
+
+/// MCModule - This class represent a completely disassembled object file or
+/// executable.  It comprises a list of MCAtom's, and a branch target table.
+/// Each atom represents a contiguous range of either instructions or data.
+class MCModule {
+  /// AtomAllocationTracker - An MCModule owns its component MCAtom's, so it
+  /// must track them in order to ensure they are properly freed as atoms are
+  /// merged or otherwise manipulated.
+  SmallPtrSet<MCAtom*, 8> AtomAllocationTracker;
+
+  /// OffsetMap - Efficiently maps offset ranges to MCAtom's.
+  IntervalMap<uint64_t, MCAtom*> OffsetMap;
+
+  /// BranchTargetMap - Maps offsets that are determined to be branches and
+  /// can be statically resolved to their target offsets.
+  DenseMap<uint64_t, MCAtom*> BranchTargetMap;
+
+  friend class MCAtom;
+
+  /// remap - Update the interval mapping for an MCAtom.
+  void remap(MCAtom *Atom, uint64_t NewBegin, uint64_t NewEnd);
+
+public:
+  MCModule(IntervalMap<uint64_t, MCAtom*>::Allocator &A) : OffsetMap(A) { }
+
+  /// createAtom - Creates a new MCAtom covering the specified offset range.
+  MCAtom *createAtom(MCAtom::AtomType Type, uint64_t Begin, uint64_t End);
+};
+
+}
+
+#endif
+
diff --git a/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h b/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h
new file mode 100644
index 000000000000..060d5085d0c3
--- /dev/null
+++ b/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -0,0 +1,294 @@
+//===-- llvm/MC/MCObjectFileInfo.h - Object File Info -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes common object file formats.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCBJECTFILEINFO_H
+#define LLVM_MC_MCBJECTFILEINFO_H
+
+#include "llvm/MC/MCCodeGenInfo.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/MC/SectionKind.h"
+
+namespace llvm {
+class MCContext;
+class MCSection;
+class Triple;
+  
+class MCObjectFileInfo {  
+protected:
+  /// CommDirectiveSupportsAlignment - True if .comm supports alignment.  This
+  /// is a hack for as long as we support 10.4 Tiger, whose assembler doesn't
+  /// support alignment on comm.
+  bool CommDirectiveSupportsAlignment;
+  
+  /// SupportsWeakEmptyEHFrame - True if target object file supports a
+  /// weak_definition of constant 0 for an omitted EH frame.
+  bool SupportsWeakOmittedEHFrame;
+  
+  /// IsFunctionEHFrameSymbolPrivate - This flag is set to true if the
+  /// "EH_frame" symbol for EH information should be an assembler temporary (aka
+  /// private linkage, aka an L or .L label) or false if it should be a normal
+  /// non-.globl label.  This defaults to true.
+  bool IsFunctionEHFrameSymbolPrivate;
+
+  /// PersonalityEncoding, LSDAEncoding, FDEEncoding, TTypeEncoding - Some
+  /// encoding values for EH.
+  unsigned PersonalityEncoding;
+  unsigned LSDAEncoding;
+  unsigned FDEEncoding;
+  unsigned FDECFIEncoding;
+  unsigned TTypeEncoding;
+
+  /// TextSection - Section directive for standard text.
+  ///
+  const MCSection *TextSection;
+  
+  /// DataSection - Section directive for standard data.
+  ///
+  const MCSection *DataSection;
+  
+  /// BSSSection - Section that is default initialized to zero.
+  const MCSection *BSSSection;
+  
+  /// ReadOnlySection - Section that is readonly and can contain arbitrary
+  /// initialized data.  Targets are not required to have a readonly section.
+  /// If they don't, various bits of code will fall back to using the data
+  /// section for constants.
+  const MCSection *ReadOnlySection;
+  
+  /// StaticCtorSection - This section contains the static constructor pointer
+  /// list.
+  const MCSection *StaticCtorSection;
+
+  /// StaticDtorSection - This section contains the static destructor pointer
+  /// list.
+  const MCSection *StaticDtorSection;
+  
+  /// LSDASection - If exception handling is supported by the target, this is
+  /// the section the Language Specific Data Area information is emitted to.
+  const MCSection *LSDASection;
+
+  /// CompactUnwindSection - If exception handling is supported by the target
+  /// and the target can support a compact representation of the CIE and FDE,
+  /// this is the section to emit them into.
+  const MCSection *CompactUnwindSection;
+
+  // Dwarf sections for debug info.  If a target supports debug info, these must
+  // be set.
+  const MCSection *DwarfAbbrevSection;
+  const MCSection *DwarfInfoSection;
+  const MCSection *DwarfLineSection;
+  const MCSection *DwarfFrameSection;
+  const MCSection *DwarfPubNamesSection;
+  const MCSection *DwarfPubTypesSection;
+  const MCSection *DwarfDebugInlineSection;
+  const MCSection *DwarfStrSection;
+  const MCSection *DwarfLocSection;
+  const MCSection *DwarfARangesSection;
+  const MCSection *DwarfRangesSection;
+  const MCSection *DwarfMacroInfoSection;
+
+  // Extra TLS Variable Data section.  If the target needs to put additional
+  // information for a TLS variable, it'll go here.
+  const MCSection *TLSExtraDataSection;
+  
+  /// TLSDataSection - Section directive for Thread Local data.
+  /// ELF and MachO only.
+  const MCSection *TLSDataSection;        // Defaults to ".tdata".
+
+  /// TLSBSSSection - Section directive for Thread Local uninitialized data.
+  /// Null if this target doesn't support a BSS section.
+  /// ELF and MachO only.
+  const MCSection *TLSBSSSection;         // Defaults to ".tbss".
+
+
+  /// EHFrameSection - EH frame section. It is initialized on demand so it
+  /// can be overwritten (with uniquing).
+  const MCSection *EHFrameSection;
+
+  /// ELF specific sections.
+  ///
+  const MCSection *DataRelSection;
+  const MCSection *DataRelLocalSection;
+  const MCSection *DataRelROSection;
+  const MCSection *DataRelROLocalSection;
+  const MCSection *MergeableConst4Section;
+  const MCSection *MergeableConst8Section;
+  const MCSection *MergeableConst16Section;
+
+  /// MachO specific sections.
+  ///
+
+  /// TLSTLVSection - Section for thread local structure information.
+  /// Contains the source code name of the variable, visibility and a pointer
+  /// to the initial value (.tdata or .tbss).
+  const MCSection *TLSTLVSection;         // Defaults to ".tlv".
+  
+  /// TLSThreadInitSection - Section for thread local data initialization
+  /// functions.
+  const MCSection *TLSThreadInitSection;  // Defaults to ".thread_init_func".
+  
+  const MCSection *CStringSection;
+  const MCSection *UStringSection;
+  const MCSection *TextCoalSection;
+  const MCSection *ConstTextCoalSection;
+  const MCSection *ConstDataSection;
+  const MCSection *DataCoalSection;
+  const MCSection *DataCommonSection;
+  const MCSection *DataBSSSection;
+  const MCSection *FourByteConstantSection;
+  const MCSection *EightByteConstantSection;
+  const MCSection *SixteenByteConstantSection;
+  const MCSection *LazySymbolPointerSection;
+  const MCSection *NonLazySymbolPointerSection;
+
+  /// COFF specific sections.
+  ///
+  const MCSection *DrectveSection;
+  const MCSection *PDataSection;
+  const MCSection *XDataSection;
+  
+public:
+  void InitMCObjectFileInfo(StringRef TT, Reloc::Model RM, CodeModel::Model CM,
+                            MCContext &ctx);
+  
+  bool isFunctionEHFrameSymbolPrivate() const {
+    return IsFunctionEHFrameSymbolPrivate;
+  }
+  bool getSupportsWeakOmittedEHFrame() const {
+    return SupportsWeakOmittedEHFrame;
+  }
+  bool getCommDirectiveSupportsAlignment() const {
+    return CommDirectiveSupportsAlignment;
+  }
+
+  unsigned getPersonalityEncoding() const { return PersonalityEncoding; }
+  unsigned getLSDAEncoding() const { return LSDAEncoding; }
+  unsigned getFDEEncoding(bool CFI) const {
+    return CFI ? FDECFIEncoding : FDEEncoding;
+  }
+  unsigned getTTypeEncoding() const { return TTypeEncoding; }
+
+  const MCSection *getTextSection() const { return TextSection; }
+  const MCSection *getDataSection() const { return DataSection; }
+  const MCSection *getBSSSection() const { return BSSSection; }
+  const MCSection *getStaticCtorSection() const { return StaticCtorSection; }
+  const MCSection *getStaticDtorSection() const { return StaticDtorSection; }
+  const MCSection *getLSDASection() const { return LSDASection; }
+  const MCSection *getCompactUnwindSection() const{
+    return CompactUnwindSection;
+  }
+  const MCSection *getDwarfAbbrevSection() const { return DwarfAbbrevSection; }
+  const MCSection *getDwarfInfoSection() const { return DwarfInfoSection; }
+  const MCSection *getDwarfLineSection() const { return DwarfLineSection; }
+  const MCSection *getDwarfFrameSection() const { return DwarfFrameSection; }
+  const MCSection *getDwarfPubNamesSection() const{return DwarfPubNamesSection;}
+  const MCSection *getDwarfPubTypesSection() const{return DwarfPubTypesSection;}
+  const MCSection *getDwarfDebugInlineSection() const {
+    return DwarfDebugInlineSection;
+  }
+  const MCSection *getDwarfStrSection() const { return DwarfStrSection; }
+  const MCSection *getDwarfLocSection() const { return DwarfLocSection; }
+  const MCSection *getDwarfARangesSection() const { return DwarfARangesSection;}
+  const MCSection *getDwarfRangesSection() const { return DwarfRangesSection; }
+  const MCSection *getDwarfMacroInfoSection() const {
+    return DwarfMacroInfoSection;
+  }
+  const MCSection *getTLSExtraDataSection() const {
+    return TLSExtraDataSection;
+  }
+  const MCSection *getTLSDataSection() const { return TLSDataSection; }
+  const MCSection *getTLSBSSSection() const { return TLSBSSSection; }
+
+  /// ELF specific sections.
+  ///
+  const MCSection *getDataRelSection() const { return DataRelSection; }
+  const MCSection *getDataRelLocalSection() const {
+    return DataRelLocalSection;
+  }
+  const MCSection *getDataRelROSection() const { return DataRelROSection; }
+  const MCSection *getDataRelROLocalSection() const {
+    return DataRelROLocalSection;
+  }
+  const MCSection *getMergeableConst4Section() const {
+    return MergeableConst4Section;
+  }
+  const MCSection *getMergeableConst8Section() const {
+    return MergeableConst8Section;
+  }
+  const MCSection *getMergeableConst16Section() const {
+    return MergeableConst16Section;
+  }
+
+  /// MachO specific sections.
+  ///
+  const MCSection *getTLSTLVSection() const { return TLSTLVSection; }
+  const MCSection *getTLSThreadInitSection() const {
+    return TLSThreadInitSection;
+  }
+  const MCSection *getCStringSection() const { return CStringSection; }
+  const MCSection *getUStringSection() const { return UStringSection; }
+  const MCSection *getTextCoalSection() const { return TextCoalSection; }
+  const MCSection *getConstTextCoalSection() const {
+    return ConstTextCoalSection;
+  }
+  const MCSection *getConstDataSection() const { return ConstDataSection; }
+  const MCSection *getDataCoalSection() const { return DataCoalSection; }
+  const MCSection *getDataCommonSection() const { return DataCommonSection; }
+  const MCSection *getDataBSSSection() const { return DataBSSSection; }
+  const MCSection *getFourByteConstantSection() const {
+    return FourByteConstantSection;
+  }
+  const MCSection *getEightByteConstantSection() const {
+    return EightByteConstantSection;
+  }
+  const MCSection *getSixteenByteConstantSection() const {
+    return SixteenByteConstantSection;
+  }
+  const MCSection *getLazySymbolPointerSection() const {
+    return LazySymbolPointerSection;
+  }
+  const MCSection *getNonLazySymbolPointerSection() const {
+    return NonLazySymbolPointerSection;
+  }
+
+  /// COFF specific sections.
+  ///
+  const MCSection *getDrectveSection() const { return DrectveSection; }
+  const MCSection *getPDataSection() const { return PDataSection; }
+  const MCSection *getXDataSection() const { return XDataSection; }
+
+  const MCSection *getEHFrameSection() {
+    if (!EHFrameSection)
+      InitEHFrameSection();
+    return EHFrameSection;
+  }
+
+private:
+  enum Environment { IsMachO, IsELF, IsCOFF };
+  Environment Env;
+  Reloc::Model RelocM;
+  CodeModel::Model CMModel;
+  MCContext *Ctx;
+
+  void InitMachOMCObjectFileInfo(Triple T);
+  void InitELFMCObjectFileInfo(Triple T);
+  void InitCOFFMCObjectFileInfo(Triple T);
+
+  /// InitEHFrameSection - Initialize EHFrameSection on demand.
+  ///
+  void InitEHFrameSection();
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/contrib/llvm/include/llvm/MC/MCObjectStreamer.h b/contrib/llvm/include/llvm/MC/MCObjectStreamer.h
index a89933b230ef..f897e64f4456 100644
--- a/contrib/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/contrib/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -19,7 +19,7 @@ class MCSectionData;
 class MCExpr;
 class MCFragment;
 class MCDataFragment;
-class TargetAsmBackend;
+class MCAsmBackend;
 class raw_ostream;
 
 /// \brief Streaming object file generation interface.
@@ -36,9 +36,9 @@ class MCObjectStreamer : public MCStreamer {
   virtual void EmitInstToData(const MCInst &Inst) = 0;
 
 protected:
-  MCObjectStreamer(MCContext &Context, TargetAsmBackend &TAB,
+  MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB,
                    raw_ostream &_OS, MCCodeEmitter *_Emitter);
-  MCObjectStreamer(MCContext &Context, TargetAsmBackend &TAB,
+  MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB,
                    raw_ostream &_OS, MCCodeEmitter *_Emitter,
                    MCAssembler *_Assembler);
   ~MCObjectStreamer();
diff --git a/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h b/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h
index ab78799fdcd2..dcecfb6aa012 100644
--- a/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h
+++ b/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h
@@ -32,6 +32,7 @@ class AsmLexer : public MCAsmLexer {
 
   const char *CurPtr;
   const MemoryBuffer *CurBuf;
+  bool isAtStartOfLine;
 
   void operator=(const AsmLexer&); // DO NOT IMPLEMENT
   AsmLexer(const AsmLexer&);       // DO NOT IMPLEMENT
@@ -47,6 +48,7 @@ class AsmLexer : public MCAsmLexer {
   void setBuffer(const MemoryBuffer *buf, const char *ptr = NULL);
 
   virtual StringRef LexUntilEndOfStatement();
+  StringRef LexUntilEndOfLine();
 
   bool isAtStartOfComment(char Char);
   bool isAtStatementSeparator(const char *Ptr);
diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h b/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h
index 47c580f85b8c..9bbb75581c25 100644
--- a/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h
+++ b/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h
@@ -17,7 +17,6 @@
 namespace llvm {
 class MCAsmLexer;
 class MCInst;
-class Target;
 
 /// AsmToken - Target independent representation for an assembler token.
 class AsmToken {
@@ -36,7 +35,7 @@ class AsmToken {
     // Real values.
     Real,
 
-    // Register values (stored in IntVal).  Only used by TargetAsmLexer.
+    // Register values (stored in IntVal).  Only used by MCTargetAsmLexer.
     Register,
 
     // No-value.
diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h
index 73766930dfe1..6ff175349e43 100644
--- a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h
@@ -20,11 +20,10 @@ class MCAsmParserExtension;
 class MCContext;
 class MCExpr;
 class MCStreamer;
+class MCTargetAsmParser;
 class SMLoc;
 class SourceMgr;
 class StringRef;
-class Target;
-class TargetAsmParser;
 class Twine;
 
 /// MCAsmParser - Generic assembler parser interface, for use by target specific
@@ -37,7 +36,7 @@ class MCAsmParser {
   MCAsmParser(const MCAsmParser &);   // DO NOT IMPLEMENT
   void operator=(const MCAsmParser &);  // DO NOT IMPLEMENT
 
-  TargetAsmParser *TargetParser;
+  MCTargetAsmParser *TargetParser;
 
   unsigned ShowParsedOperands : 1;
 
@@ -60,8 +59,8 @@ class MCAsmParser {
   /// getStreamer - Return the output streamer for the assembler.
   virtual MCStreamer &getStreamer() = 0;
 
-  TargetAsmParser &getTargetParser() const { return *TargetParser; }
-  void setTargetParser(TargetAsmParser &P);
+  MCTargetAsmParser &getTargetParser() const { return *TargetParser; }
+  void setTargetParser(MCTargetAsmParser &P);
 
   bool getShowParsedOperands() const { return ShowParsedOperands; }
   void setShowParsedOperands(bool Value) { ShowParsedOperands = Value; }
@@ -131,7 +130,7 @@ class MCAsmParser {
 };
 
 /// \brief Create an MCAsmParser instance.
-MCAsmParser *createMCAsmParser(const Target &, SourceMgr &, MCContext &,
+MCAsmParser *createMCAsmParser(SourceMgr &, MCContext &,
                                MCStreamer &, const MCAsmInfo &);
 
 } // End llvm namespace
diff --git a/contrib/llvm/include/llvm/MC/MCRegisterInfo.h b/contrib/llvm/include/llvm/MC/MCRegisterInfo.h
index caf98bb89f6b..ada5ae80af0c 100644
--- a/contrib/llvm/include/llvm/MC/MCRegisterInfo.h
+++ b/contrib/llvm/include/llvm/MC/MCRegisterInfo.h
@@ -16,10 +16,94 @@
 #ifndef LLVM_MC_MCREGISTERINFO_H
 #define LLVM_MC_MCREGISTERINFO_H
 
+#include "llvm/ADT/DenseMap.h"
 #include <cassert>
 
 namespace llvm {
 
+/// MCRegisterClass - Base class of TargetRegisterClass.
+class MCRegisterClass {
+public:
+  typedef const unsigned* iterator;
+  typedef const unsigned* const_iterator;
+private:
+  unsigned ID;
+  const char *Name;
+  const unsigned RegSize, Alignment; // Size & Alignment of register in bytes
+  const int CopyCost;
+  const bool Allocatable;
+  const iterator RegsBegin, RegsEnd;
+  const unsigned char *const RegSet;
+  const unsigned RegSetSize;
+public:
+  MCRegisterClass(unsigned id, const char *name,
+                  unsigned RS, unsigned Al, int CC, bool Allocable,
+                  iterator RB, iterator RE, const unsigned char *Bits,
+                  unsigned NumBytes)
+    : ID(id), Name(name), RegSize(RS), Alignment(Al), CopyCost(CC),
+      Allocatable(Allocable), RegsBegin(RB), RegsEnd(RE), RegSet(Bits),
+      RegSetSize(NumBytes) {
+    for (iterator i = RegsBegin; i != RegsEnd; ++i)
+       assert(contains(*i) && "Bit field corrupted.");
+  }
+
+  /// getID() - Return the register class ID number.
+  ///
+  unsigned getID() const { return ID; }
+
+  /// getName() - Return the register class name for debugging.
+  ///
+  const char *getName() const { return Name; }
+
+  /// begin/end - Return all of the registers in this class.
+  ///
+  iterator       begin() const { return RegsBegin; }
+  iterator         end() const { return RegsEnd; }
+
+  /// getNumRegs - Return the number of registers in this class.
+  ///
+  unsigned getNumRegs() const { return (unsigned)(RegsEnd-RegsBegin); }
+
+  /// getRegister - Return the specified register in the class.
+  ///
+  unsigned getRegister(unsigned i) const {
+    assert(i < getNumRegs() && "Register number out of range!");
+    return RegsBegin[i];
+  }
+
+  /// contains - Return true if the specified register is included in this
+  /// register class.  This does not include virtual registers.
+  bool contains(unsigned Reg) const {
+    unsigned InByte = Reg % 8;
+    unsigned Byte = Reg / 8;
+    if (Byte >= RegSetSize)
+      return false;
+    return (RegSet[Byte] & (1 << InByte)) != 0;
+  }
+
+  /// contains - Return true if both registers are in this class.
+  bool contains(unsigned Reg1, unsigned Reg2) const {
+    return contains(Reg1) && contains(Reg2);
+  }
+
+  /// getSize - Return the size of the register in bytes, which is also the size
+  /// of a stack slot allocated to hold a spilled copy of this register.
+  unsigned getSize() const { return RegSize; }
+
+  /// getAlignment - Return the minimum required alignment for a register of
+  /// this class.
+  unsigned getAlignment() const { return Alignment; }
+
+  /// getCopyCost - Return the cost of copying a value between two registers in
+  /// this class. A negative number means the register class is very expensive
+  /// to copy e.g. status flag register classes.
+  int getCopyCost() const { return CopyCost; }
+
+  /// isAllocatable - Return true if this register class may be used to create
+  /// virtual registers.
+  bool isAllocatable() const { return Allocatable; }
+};
+
 /// MCRegisterDesc - This record contains all of the information known about
 /// a particular register.  The Overlaps field contains a pointer to a zero
 /// terminated array of registers that this register aliases, starting with
@@ -50,18 +134,67 @@ struct MCRegisterDesc {
 /// virtual methods.
 ///
 class MCRegisterInfo {
+public:
+  typedef const MCRegisterClass *regclass_iterator;
 private:
-  const MCRegisterDesc *Desc;             // Pointer to the descriptor array
-  unsigned NumRegs;                       // Number of entries in the array
+  const MCRegisterDesc *Desc;                 // Pointer to the descriptor array
+  unsigned NumRegs;                           // Number of entries in the array
+  unsigned RAReg;                             // Return address register
+  const MCRegisterClass *Classes;             // Pointer to the regclass array
+  unsigned NumClasses;                        // Number of entries in the array
+  DenseMap<unsigned, int> L2DwarfRegs;        // LLVM to Dwarf regs mapping
+  DenseMap<unsigned, int> EHL2DwarfRegs;      // LLVM to Dwarf regs mapping EH
+  DenseMap<unsigned, unsigned> Dwarf2LRegs;   // Dwarf to LLVM regs mapping
+  DenseMap<unsigned, unsigned> EHDwarf2LRegs; // Dwarf to LLVM regs mapping EH
+  DenseMap<unsigned, int> L2SEHRegs;          // LLVM to SEH regs mapping
 
 public:
   /// InitMCRegisterInfo - Initialize MCRegisterInfo, called by TableGen
   /// auto-generated routines. *DO NOT USE*.
-  void InitMCRegisterInfo(const MCRegisterDesc *D, unsigned NR) {
+  void InitMCRegisterInfo(const MCRegisterDesc *D, unsigned NR, unsigned RA,
+                          const MCRegisterClass *C, unsigned NC) {
     Desc = D;
     NumRegs = NR;
+    RAReg = RA;
+    Classes = C;
+    NumClasses = NC;
+  }
+
+  /// mapLLVMRegToDwarfReg - Used to initialize LLVM register to Dwarf
+  /// register number mapping. Called by TableGen auto-generated routines.
+  /// *DO NOT USE*.
+  void mapLLVMRegToDwarfReg(unsigned LLVMReg, int DwarfReg, bool isEH) {
+    if (isEH)
+      EHL2DwarfRegs[LLVMReg] = DwarfReg;
+    else
+      L2DwarfRegs[LLVMReg] = DwarfReg;
   }
     
+  /// mapDwarfRegToLLVMReg - Used to initialize Dwarf register to LLVM
+  /// register number mapping. Called by TableGen auto-generated routines.
+  /// *DO NOT USE*.
+  void mapDwarfRegToLLVMReg(unsigned DwarfReg, unsigned LLVMReg, bool isEH) {
+    if (isEH)
+      EHDwarf2LRegs[DwarfReg] = LLVMReg;
+    else
+      Dwarf2LRegs[DwarfReg] = LLVMReg;
+  }
+     
+  /// mapLLVMRegToSEHReg - Used to initialize LLVM register to SEH register
+  /// number mapping. By default the SEH register number is just the same
+  /// as the LLVM register number.
+  /// FIXME: TableGen these numbers. Currently this requires target specific
+  /// initialization code.
+  void mapLLVMRegToSEHReg(unsigned LLVMReg, int SEHReg) {
+    L2SEHRegs[LLVMReg] = SEHReg;
+  }
+
+  /// getRARegister - This method should return the register where the return
+  /// address can be found.
+  unsigned getRARegister() const {
+    return RAReg;
+  }
+
   const MCRegisterDesc &operator[](unsigned RegNo) const {
     assert(RegNo < NumRegs &&
            "Attempting to access record for invalid register number!");
@@ -122,6 +255,51 @@ class MCRegisterInfo {
   unsigned getNumRegs() const {
     return NumRegs;
   }
+
+  /// getDwarfRegNum - Map a target register to an equivalent dwarf register
+  /// number.  Returns -1 if there is no equivalent value.  The second
+  /// parameter allows targets to use different numberings for EH info and
+  /// debugging info.
+  int getDwarfRegNum(unsigned RegNum, bool isEH) const {
+    const DenseMap<unsigned, int> &M = isEH ? EHL2DwarfRegs : L2DwarfRegs;
+    const DenseMap<unsigned, int>::const_iterator I = M.find(RegNum);
+    if (I == M.end()) return -1;
+    return I->second;
+  }
+
+  /// getLLVMRegNum - Map a dwarf register back to a target register.
+  ///
+  int getLLVMRegNum(unsigned RegNum, bool isEH) const {
+    const DenseMap<unsigned, unsigned> &M = isEH ? EHDwarf2LRegs : Dwarf2LRegs;
+    const DenseMap<unsigned, unsigned>::const_iterator I = M.find(RegNum);
+    if (I == M.end()) {
+      assert(0 && "Invalid RegNum");
+      return -1;
+    }
+    return I->second;
+  }
+
+  /// getSEHRegNum - Map a target register to an equivalent SEH register
+  /// number.  Returns LLVM register number if there is no equivalent value.
+  int getSEHRegNum(unsigned RegNum) const {
+    const DenseMap<unsigned, int>::const_iterator I = L2SEHRegs.find(RegNum);
+    if (I == L2SEHRegs.end()) return (int)RegNum;
+    return I->second;
+  }
+
+  regclass_iterator regclass_begin() const { return Classes; }
+  regclass_iterator regclass_end() const { return Classes+NumClasses; }
+
+  unsigned getNumRegClasses() const {
+    return (unsigned)(regclass_end()-regclass_begin());
+  }
+
+  /// getRegClass - Returns the register class associated with the enumeration
+  /// value.  See class MCOperandInfo.
+  const MCRegisterClass getRegClass(unsigned i) const {
+    assert(i < getNumRegClasses() && "Register Class ID out of range");
+    return Classes[i];
+  }
 };
  
 } // End llvm namespace
diff --git a/contrib/llvm/include/llvm/MC/MCStreamer.h b/contrib/llvm/include/llvm/MC/MCStreamer.h
index 7bdba5fa50b5..451efbff6e3a 100644
--- a/contrib/llvm/include/llvm/MC/MCStreamer.h
+++ b/contrib/llvm/include/llvm/MC/MCStreamer.h
@@ -14,13 +14,15 @@
 #ifndef LLVM_MC_MCSTREAMER_H
 #define LLVM_MC_MCSTREAMER_H
 
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCWin64EH.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
 
 namespace llvm {
+  class MCAsmBackend;
   class MCAsmInfo;
   class MCCodeEmitter;
   class MCContext;
@@ -30,7 +32,6 @@ namespace llvm {
   class MCSection;
   class MCSymbol;
   class StringRef;
-  class TargetAsmBackend;
   class TargetLoweringObjectFile;
   class Twine;
   class raw_ostream;
@@ -63,14 +64,29 @@ namespace llvm {
     void setCurrentW64UnwindInfo(MCWin64EHUnwindInfo *Frame);
     void EnsureValidW64UnwindInfo();
 
-    const MCSymbol* LastNonPrivate;
+    MCSymbol* LastSymbol;
 
     /// SectionStack - This is stack of current and previous section
     /// values saved by PushSection.
     SmallVector<std::pair<const MCSection *,
                 const MCSection *>, 4> SectionStack;
 
+    unsigned UniqueCodeBeginSuffix;
+    unsigned UniqueDataBeginSuffix;
+
   protected:
+    /// Indicator of whether the previous data-or-code indicator was for
+    /// code or not.  Used to determine when we need to emit a new indicator.
+    enum DataType {
+      Data,
+      Code,
+      JumpTable8,
+      JumpTable16,
+      JumpTable32
+    };
+    DataType RegionIndicator;
+
+
     MCStreamer(MCContext &Ctx);
 
     const MCExpr *BuildSymbolDiff(MCContext &Context, const MCSymbol *A,
@@ -96,6 +112,10 @@ namespace llvm {
       return FrameInfos[i];
     }
 
+    ArrayRef<MCDwarfFrameInfo> getFrameInfos() {
+      return FrameInfos;
+    }
+
     unsigned getNumW64UnwindInfos() {
       return W64UnwindInfos.size();
     }
@@ -219,6 +239,41 @@ namespace llvm {
     /// used in an assignment.
     virtual void EmitLabel(MCSymbol *Symbol);
 
+    /// EmitDataRegion - Emit a label that marks the beginning of a data
+    /// region.
+    /// On ELF targets, this corresponds to an assembler statement such as:
+    ///   $d.1:
+    virtual void EmitDataRegion();
+
+    /// EmitJumpTable8Region - Emit a label that marks the beginning of a
+    /// jump table composed of 8-bit offsets.
+    /// On ELF targets, this corresponds to an assembler statement such as:
+    ///   $d.1:
+    virtual void EmitJumpTable8Region();
+
+    /// EmitJumpTable16Region - Emit a label that marks the beginning of a
+    /// jump table composed of 16-bit offsets.
+    /// On ELF targets, this corresponds to an assembler statement such as:
+    ///   $d.1:
+    virtual void EmitJumpTable16Region();
+
+    /// EmitJumpTable32Region - Emit a label that marks the beginning of a
+    /// jump table composed of 32-bit offsets.
+    /// On ELF targets, this corresponds to an assembler statement such as:
+    ///   $d.1:
+    virtual void EmitJumpTable32Region();
+
+    /// EmitCodeRegion - Emit a label that marks the beginning of a code
+    /// region.
+    /// On ELF targets, this corresponds to an assembler statement such as:
+    ///   $a.1:
+    virtual void EmitCodeRegion();
+
+    /// ForceCodeRegion - Forcibly sets the current region mode to code.  Used
+    /// at function entry points.
+    void ForceCodeRegion() { RegionIndicator = Code; }
+
+
     virtual void EmitEHSymAttributes(const MCSymbol *Symbol,
                                      MCSymbol *EHSymbol);
 
@@ -299,7 +354,9 @@ namespace llvm {
     ///
     /// @param Symbol - The common symbol to emit.
     /// @param Size - The size of the common symbol.
-    virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) = 0;
+    /// @param ByteAlignment - The alignment of the common symbol in bytes.
+    virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                       unsigned ByteAlignment) = 0;
 
     /// EmitZerofill - Emit the zerofill section and an optional symbol.
     ///
@@ -470,6 +527,7 @@ namespace llvm {
     void EmitDwarfSetLineAddr(int64_t LineDelta, const MCSymbol *Label,
                               int PointerSize);
 
+    virtual void EmitCompactUnwindEncoding(uint32_t CompactUnwindEncoding);
     virtual void EmitCFISections(bool EH, bool Debug);
     virtual void EmitCFIStartProc();
     virtual void EmitCFIEndProc();
@@ -557,14 +615,14 @@ namespace llvm {
                                 bool useCFI,
                                 MCInstPrinter *InstPrint = 0,
                                 MCCodeEmitter *CE = 0,
-                                TargetAsmBackend *TAB = 0,
+                                MCAsmBackend *TAB = 0,
                                 bool ShowInst = false);
 
   /// createMachOStreamer - Create a machine code streamer which will generate
   /// Mach-O format object files.
   ///
   /// Takes ownership of \arg TAB and \arg CE.
-  MCStreamer *createMachOStreamer(MCContext &Ctx, TargetAsmBackend &TAB,
+  MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB,
                                   raw_ostream &OS, MCCodeEmitter *CE,
                                   bool RelaxAll = false);
 
@@ -573,13 +631,13 @@ namespace llvm {
   ///
   /// Takes ownership of \arg TAB and \arg CE.
   MCStreamer *createWinCOFFStreamer(MCContext &Ctx,
-                                    TargetAsmBackend &TAB,
+                                    MCAsmBackend &TAB,
                                     MCCodeEmitter &CE, raw_ostream &OS,
                                     bool RelaxAll = false);
 
   /// createELFStreamer - Create a machine code streamer which will generate
   /// ELF format object files.
-  MCStreamer *createELFStreamer(MCContext &Ctx, TargetAsmBackend &TAB,
+  MCStreamer *createELFStreamer(MCContext &Ctx, MCAsmBackend &TAB,
 				raw_ostream &OS, MCCodeEmitter *CE,
 				bool RelaxAll, bool NoExecStack);
 
@@ -593,7 +651,7 @@ namespace llvm {
   /// "pure" MC object files, for use with MC-JIT and testing tools.
   ///
   /// Takes ownership of \arg TAB and \arg CE.
-  MCStreamer *createPureStreamer(MCContext &Ctx, TargetAsmBackend &TAB,
+  MCStreamer *createPureStreamer(MCContext &Ctx, MCAsmBackend &TAB,
                                  raw_ostream &OS, MCCodeEmitter *CE);
 
 } // end namespace llvm
diff --git a/contrib/llvm/include/llvm/Target/TargetAsmLexer.h b/contrib/llvm/include/llvm/MC/MCTargetAsmLexer.h
similarity index 81%
rename from contrib/llvm/include/llvm/Target/TargetAsmLexer.h
rename to contrib/llvm/include/llvm/MC/MCTargetAsmLexer.h
index 9fcf449a86cd..acb3d4d6144c 100644
--- a/contrib/llvm/include/llvm/Target/TargetAsmLexer.h
+++ b/contrib/llvm/include/llvm/MC/MCTargetAsmLexer.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetAsmLexer.h - Target Assembly Lexer ----*- C++ -*-===//
+//===-- llvm/MC/MCTargetAsmLexer.h - Target Assembly Lexer ------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,16 +7,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TARGET_TARGETASMLEXER_H
-#define LLVM_TARGET_TARGETASMLEXER_H
+#ifndef LLVM_MC_MCTARGETASMLEXER_H
+#define LLVM_MC_MCTARGETASMLEXER_H
 
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 
 namespace llvm {
 class Target;
   
-/// TargetAsmLexer - Generic interface to target specific assembly lexers.
-class TargetAsmLexer {
+/// MCTargetAsmLexer - Generic interface to target specific assembly lexers.
+class MCTargetAsmLexer {
   /// The current token
   AsmToken CurTok;
   
@@ -24,10 +24,10 @@ class TargetAsmLexer {
   SMLoc ErrLoc;
   std::string Err;
   
-  TargetAsmLexer(const TargetAsmLexer &);   // DO NOT IMPLEMENT
-  void operator=(const TargetAsmLexer &);  // DO NOT IMPLEMENT
+  MCTargetAsmLexer(const MCTargetAsmLexer &);   // DO NOT IMPLEMENT
+  void operator=(const MCTargetAsmLexer &);  // DO NOT IMPLEMENT
 protected: // Can only create subclasses.
-  TargetAsmLexer(const Target &);
+  MCTargetAsmLexer(const Target &);
   
   virtual AsmToken LexToken() = 0;
   
@@ -41,7 +41,7 @@ class TargetAsmLexer {
   MCAsmLexer *Lexer;
   
 public:
-  virtual ~TargetAsmLexer();
+  virtual ~MCTargetAsmLexer();
   
   const Target &getTarget() const { return TheTarget; }
   
diff --git a/contrib/llvm/include/llvm/Target/TargetAsmParser.h b/contrib/llvm/include/llvm/MC/MCTargetAsmParser.h
similarity index 75%
rename from contrib/llvm/include/llvm/Target/TargetAsmParser.h
rename to contrib/llvm/include/llvm/MC/MCTargetAsmParser.h
index df84231eea19..4e3fd0d3a9ec 100644
--- a/contrib/llvm/include/llvm/Target/TargetAsmParser.h
+++ b/contrib/llvm/include/llvm/MC/MCTargetAsmParser.h
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetAsmParser.h - Target Assembly Parser --*- C++ -*-===//
+//===-- llvm/MC/MCTargetAsmParser.h - Target Assembly Parser ----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TARGET_TARGETPARSER_H
-#define LLVM_TARGET_TARGETPARSER_H
+#ifndef LLVM_MC_TARGETPARSER_H
+#define LLVM_MC_TARGETPARSER_H
 
 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
 
@@ -18,20 +18,32 @@ class StringRef;
 class SMLoc;
 class AsmToken;
 class MCParsedAsmOperand;
+class MCInst;
 template <typename T> class SmallVectorImpl;
 
-/// TargetAsmParser - Generic interface to target specific assembly parsers.
-class TargetAsmParser : public MCAsmParserExtension {
-  TargetAsmParser(const TargetAsmParser &);   // DO NOT IMPLEMENT
-  void operator=(const TargetAsmParser &);  // DO NOT IMPLEMENT
+/// MCTargetAsmParser - Generic interface to target specific assembly parsers.
+class MCTargetAsmParser : public MCAsmParserExtension {
+public:
+  enum MatchResultTy {
+    Match_ConversionFail,
+    Match_InvalidOperand,
+    Match_MissingFeature,
+    Match_MnemonicFail,
+    Match_Success,
+    FIRST_TARGET_MATCH_RESULT_TY
+  };
+
+private:
+  MCTargetAsmParser(const MCTargetAsmParser &);   // DO NOT IMPLEMENT
+  void operator=(const MCTargetAsmParser &);  // DO NOT IMPLEMENT
 protected: // Can only create subclasses.
-  TargetAsmParser();
- 
+  MCTargetAsmParser();
+
   /// AvailableFeatures - The current set of available features.
   unsigned AvailableFeatures;
 
 public:
-  virtual ~TargetAsmParser();
+  virtual ~MCTargetAsmParser();
 
   unsigned getAvailableFeatures() const { return AvailableFeatures; }
   void setAvailableFeatures(unsigned Value) { AvailableFeatures = Value; }
@@ -66,18 +78,24 @@ class TargetAsmParser : public MCAsmParserExtension {
   ///
   /// \param DirectiveID - the identifier token of the directive.
   virtual bool ParseDirective(AsmToken DirectiveID) = 0;
-  
+
   /// MatchAndEmitInstruction - Recognize a series of operands of a parsed
   /// instruction as an actual MCInst and emit it to the specified MCStreamer.
   /// This returns false on success and returns true on failure to match.
   ///
   /// On failure, the target parser is responsible for emitting a diagnostic
   /// explaining the match failure.
-  virtual bool 
+  virtual bool
   MatchAndEmitInstruction(SMLoc IDLoc,
                           SmallVectorImpl<MCParsedAsmOperand*> &Operands,
                           MCStreamer &Out) = 0;
-  
+
+  /// checkTargetMatchPredicate - Validate the instruction match against
+  /// any complex target predicates not expressible via match classes.
+  virtual unsigned checkTargetMatchPredicate(MCInst &Inst) {
+    return Match_Success;
+  }
+
 };
 
 } // End llvm namespace
diff --git a/contrib/llvm/include/llvm/MC/MCValue.h b/contrib/llvm/include/llvm/MC/MCValue.h
index df8dbd930bf7..8352ed183f09 100644
--- a/contrib/llvm/include/llvm/MC/MCValue.h
+++ b/contrib/llvm/include/llvm/MC/MCValue.h
@@ -46,16 +46,6 @@ class MCValue {
   /// isAbsolute - Is this an absolute (as opposed to relocatable) value.
   bool isAbsolute() const { return !SymA && !SymB; }
 
-  /// getAssociatedSection - For relocatable values, return the section the
-  /// value is associated with.
-  ///
-  /// @result - The value's associated section, or null for external or constant
-  /// values.
-  //
-  // FIXME: Switch to a tagged section, so this can return the tagged section
-  // value.
-  const MCSection *getAssociatedSection() const;
-
   /// print - Print the value to the stream \arg OS.
   void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
 
diff --git a/contrib/llvm/include/llvm/CodeGen/MachineLocation.h b/contrib/llvm/include/llvm/MC/MachineLocation.h
similarity index 91%
rename from contrib/llvm/include/llvm/CodeGen/MachineLocation.h
rename to contrib/llvm/include/llvm/MC/MachineLocation.h
index 21951b6680b6..8ddfdbcece49 100644
--- a/contrib/llvm/include/llvm/CodeGen/MachineLocation.h
+++ b/contrib/llvm/include/llvm/MC/MachineLocation.h
@@ -1,4 +1,4 @@
-//===-- llvm/CodeGen/MachineLocation.h --------------------------*- C++ -*-===//
+//===-- llvm/MC/MachineLocation.h -------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -18,8 +18,8 @@
 //===----------------------------------------------------------------------===//
 
 
-#ifndef LLVM_CODEGEN_MACHINELOCATION_H
-#define LLVM_CODEGEN_MACHINELOCATION_H
+#ifndef LLVM_MC_MACHINELOCATION_H
+#define LLVM_MC_MACHINELOCATION_H
 
 namespace llvm {
   class MCSymbol;
@@ -36,11 +36,11 @@ class MachineLocation {
     VirtualFP = ~0U
   };
   MachineLocation()
-  : IsRegister(false), Register(0), Offset(0) {}
+    : IsRegister(false), Register(0), Offset(0) {}
   explicit MachineLocation(unsigned R)
-  : IsRegister(true), Register(R), Offset(0) {}
+    : IsRegister(true), Register(R), Offset(0) {}
   MachineLocation(unsigned R, int O)
-  : IsRegister(false), Register(R), Offset(O) {}
+    : IsRegister(false), Register(R), Offset(O) {}
 
   bool operator==(const MachineLocation &Other) const {
       return IsRegister == Other.IsRegister && Register == Other.Register &&
diff --git a/contrib/llvm/include/llvm/Module.h b/contrib/llvm/include/llvm/Module.h
index 47d23f36c13c..8ce5ec4f1d14 100644
--- a/contrib/llvm/include/llvm/Module.h
+++ b/contrib/llvm/include/llvm/Module.h
@@ -50,17 +50,35 @@ template<> struct ilist_traits<Function>
 private:
   mutable ilist_node<Function> Sentinel;
 };
+
 template<> struct ilist_traits<GlobalVariable>
   : public SymbolTableListTraits<GlobalVariable, Module> {
   // createSentinel is used to create a node that marks the end of the list.
-  static GlobalVariable *createSentinel();
-  static void destroySentinel(GlobalVariable *GV) { delete GV; }
+  GlobalVariable *createSentinel() const {
+    return static_cast<GlobalVariable*>(&Sentinel);
+  }
+  static void destroySentinel(GlobalVariable*) {}
+
+  GlobalVariable *provideInitialHead() const { return createSentinel(); }
+  GlobalVariable *ensureHead(GlobalVariable*) const { return createSentinel(); }
+  static void noteHead(GlobalVariable*, GlobalVariable*) {}
+private:
+  mutable ilist_node<GlobalVariable> Sentinel;
 };
+
 template<> struct ilist_traits<GlobalAlias>
   : public SymbolTableListTraits<GlobalAlias, Module> {
   // createSentinel is used to create a node that marks the end of the list.
-  static GlobalAlias *createSentinel();
-  static void destroySentinel(GlobalAlias *GA) { delete GA; }
+  GlobalAlias *createSentinel() const {
+    return static_cast<GlobalAlias*>(&Sentinel);
+  }
+  static void destroySentinel(GlobalAlias*) {}
+
+  GlobalAlias *provideInitialHead() const { return createSentinel(); }
+  GlobalAlias *ensureHead(GlobalAlias*) const { return createSentinel(); }
+  static void noteHead(GlobalAlias*, GlobalAlias*) {}
+private:
+  mutable ilist_node<GlobalAlias> Sentinel;
 };
 
 template<> struct ilist_traits<NamedMDNode>
@@ -272,10 +290,10 @@ class Module {
   ///      the existing function.
   ///   4. Finally, the function exists but has the wrong prototype: return the
   ///      function with a constantexpr cast to the right prototype.
-  Constant *getOrInsertFunction(StringRef Name, const FunctionType *T,
+  Constant *getOrInsertFunction(StringRef Name, FunctionType *T,
                                 AttrListPtr AttributeList);
 
-  Constant *getOrInsertFunction(StringRef Name, const FunctionType *T);
+  Constant *getOrInsertFunction(StringRef Name, FunctionType *T);
 
   /// getOrInsertFunction - Look up the specified function in the module symbol
   /// table.  If it does not exist, add a prototype for the function and return
@@ -286,14 +304,14 @@ class Module {
   /// clients to use.
   Constant *getOrInsertFunction(StringRef Name,
                                 AttrListPtr AttributeList,
-                                const Type *RetTy, ...)  END_WITH_NULL;
+                                Type *RetTy, ...)  END_WITH_NULL;
 
   /// getOrInsertFunction - Same as above, but without the attributes.
-  Constant *getOrInsertFunction(StringRef Name, const Type *RetTy, ...)
+  Constant *getOrInsertFunction(StringRef Name, Type *RetTy, ...)
     END_WITH_NULL;
 
   Constant *getOrInsertTargetIntrinsic(StringRef Name,
-                                       const FunctionType *Ty,
+                                       FunctionType *Ty,
                                        AttrListPtr AttributeList);
 
   /// getFunction - Look up the specified function in the module symbol table.
@@ -325,7 +343,7 @@ class Module {
   ///      with a constantexpr cast to the right type.
   ///   3. Finally, if the existing global is the correct declaration, return
   ///      the existing global.
-  Constant *getOrInsertGlobal(StringRef Name, const Type *Ty);
+  Constant *getOrInsertGlobal(StringRef Name, Type *Ty);
 
 /// @}
 /// @name Global Alias Accessors
diff --git a/contrib/llvm/include/llvm/Object/Archive.h b/contrib/llvm/include/llvm/Object/Archive.h
new file mode 100644
index 000000000000..4f081206c5bc
--- /dev/null
+++ b/contrib/llvm/include/llvm/Object/Archive.h
@@ -0,0 +1,90 @@
+//===- Archive.h - ar archive file format -----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the ar archive file format class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_ARCHIVE_H
+#define LLVM_OBJECT_ARCHIVE_H
+
+#include "llvm/Object/Binary.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/DataTypes.h"
+
+namespace llvm {
+namespace object {
+
+class Archive : public Binary {
+public:
+  class Child {
+    const Archive *Parent;
+    StringRef Data;
+
+  public:
+    Child(const Archive *p, StringRef d) : Parent(p), Data(d) {}
+
+    bool operator ==(const Child &other) const {
+      return (Parent == other.Parent) && (Data.begin() == other.Data.begin());
+    }
+
+    Child getNext() const;
+    error_code getName(StringRef &Result) const;
+    int getLastModified() const;
+    int getUID() const;
+    int getGID() const;
+    int getAccessMode() const;
+    ///! Return the size of the archive member without the header or padding.
+    uint64_t getSize() const;
+
+    MemoryBuffer *getBuffer() const;
+    error_code getAsBinary(OwningPtr<Binary> &Result) const;
+  };
+
+  class child_iterator {
+    Child child;
+  public:
+    child_iterator(const Child &c) : child(c) {}
+    const Child* operator->() const {
+      return &child;
+    }
+
+    bool operator==(const child_iterator &other) const {
+      return child == other.child;
+    }
+
+    bool operator!=(const child_iterator &other) const {
+      return !(*this == other);
+    }
+
+    child_iterator& operator++() {  // Preincrement
+      child = child.getNext();
+      return *this;
+    }
+  };
+
+  Archive(MemoryBuffer *source, error_code &ec);
+
+  child_iterator begin_children() const;
+  child_iterator end_children() const;
+
+  // Cast methods.
+  static inline bool classof(Archive const *v) { return true; }
+  static inline bool classof(Binary const *v) {
+    return v->getType() == Binary::isArchive;
+  }
+
+private:
+  child_iterator StringTable;
+};
+
+}
+}
+
+#endif
diff --git a/contrib/llvm/include/llvm/Object/COFF.h b/contrib/llvm/include/llvm/Object/COFF.h
index 121f9e850451..067bcd471ae9 100644
--- a/contrib/llvm/include/llvm/Object/COFF.h
+++ b/contrib/llvm/include/llvm/Object/COFF.h
@@ -67,6 +67,12 @@ struct coff_section {
   support::ulittle32_t Characteristics;
 };
 
+struct coff_relocation {
+  support::ulittle32_t VirtualAddress;
+  support::ulittle32_t SymbolTableIndex;
+  support::ulittle16_t Type;
+};
+
 class COFFObjectFile : public ObjectFile {
 private:
   const coff_file_header *Header;
@@ -78,26 +84,52 @@ class COFFObjectFile : public ObjectFile {
         error_code        getSection(int32_t index,
                                      const coff_section *&Res) const;
         error_code        getString(uint32_t offset, StringRef &Res) const;
+        error_code        getSymbol(uint32_t index,
+                                    const coff_symbol *&Res) const;
 
   const coff_symbol      *toSymb(DataRefImpl Symb) const;
   const coff_section     *toSec(DataRefImpl Sec) const;
+  const coff_relocation  *toRel(DataRefImpl Rel) const;
 
 protected:
   virtual error_code getSymbolNext(DataRefImpl Symb, SymbolRef &Res) const;
   virtual error_code getSymbolName(DataRefImpl Symb, StringRef &Res) const;
+  virtual error_code getSymbolOffset(DataRefImpl Symb, uint64_t &Res) const;
   virtual error_code getSymbolAddress(DataRefImpl Symb, uint64_t &Res) const;
   virtual error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const;
   virtual error_code getSymbolNMTypeChar(DataRefImpl Symb, char &Res) const;
   virtual error_code isSymbolInternal(DataRefImpl Symb, bool &Res) const;
+  virtual error_code isSymbolGlobal(DataRefImpl Symb, bool &Res) const;
+  virtual error_code getSymbolType(DataRefImpl Symb, SymbolRef::SymbolType &Res) const;
 
   virtual error_code getSectionNext(DataRefImpl Sec, SectionRef &Res) const;
   virtual error_code getSectionName(DataRefImpl Sec, StringRef &Res) const;
   virtual error_code getSectionAddress(DataRefImpl Sec, uint64_t &Res) const;
   virtual error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const;
   virtual error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const;
+  virtual error_code getSectionAlignment(DataRefImpl Sec, uint64_t &Res) const;
   virtual error_code isSectionText(DataRefImpl Sec, bool &Res) const;
+  virtual error_code isSectionData(DataRefImpl Sec, bool &Res) const;
+  virtual error_code isSectionBSS(DataRefImpl Sec, bool &Res) const;
   virtual error_code sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb,
                                            bool &Result) const;
+  virtual relocation_iterator getSectionRelBegin(DataRefImpl Sec) const;
+  virtual relocation_iterator getSectionRelEnd(DataRefImpl Sec) const;
+
+  virtual error_code getRelocationNext(DataRefImpl Rel,
+                                       RelocationRef &Res) const;
+  virtual error_code getRelocationAddress(DataRefImpl Rel,
+                                          uint64_t &Res) const;
+  virtual error_code getRelocationSymbol(DataRefImpl Rel,
+                                         SymbolRef &Res) const;
+  virtual error_code getRelocationType(DataRefImpl Rel,
+                                       uint32_t &Res) const;
+  virtual error_code getRelocationTypeName(DataRefImpl Rel,
+                                           SmallVectorImpl<char> &Result) const;
+  virtual error_code getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                 int64_t &Res) const;
+  virtual error_code getRelocationValueString(DataRefImpl Rel,
+                                           SmallVectorImpl<char> &Result) const;
 
 public:
   COFFObjectFile(MemoryBuffer *Object, error_code &ec);
diff --git a/contrib/llvm/include/llvm/Object/MachO.h b/contrib/llvm/include/llvm/Object/MachO.h
new file mode 100644
index 000000000000..f5e7461a488a
--- /dev/null
+++ b/contrib/llvm/include/llvm/Object/MachO.h
@@ -0,0 +1,106 @@
+//===- MachO.h - MachO object file implementation ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the MachOObjectFile class, which binds the MachOObject
+// class to the generic ObjectFile wrapper.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_MACHO_H
+#define LLVM_OBJECT_MACHO_H
+
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/MachOObject.h"
+#include "llvm/Support/MachO.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+namespace object {
+
+typedef MachOObject::LoadCommandInfo LoadCommandInfo;
+
+class MachOObjectFile : public ObjectFile {
+public:
+  MachOObjectFile(MemoryBuffer *Object, MachOObject *MOO, error_code &ec);
+
+  virtual symbol_iterator begin_symbols() const;
+  virtual symbol_iterator end_symbols() const;
+  virtual section_iterator begin_sections() const;
+  virtual section_iterator end_sections() const;
+
+  virtual uint8_t getBytesInAddress() const;
+  virtual StringRef getFileFormatName() const;
+  virtual unsigned getArch() const;
+
+protected:
+  virtual error_code getSymbolNext(DataRefImpl Symb, SymbolRef &Res) const;
+  virtual error_code getSymbolName(DataRefImpl Symb, StringRef &Res) const;
+  virtual error_code getSymbolOffset(DataRefImpl Symb, uint64_t &Res) const;
+  virtual error_code getSymbolAddress(DataRefImpl Symb, uint64_t &Res) const;
+  virtual error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const;
+  virtual error_code getSymbolNMTypeChar(DataRefImpl Symb, char &Res) const;
+  virtual error_code isSymbolInternal(DataRefImpl Symb, bool &Res) const;
+  virtual error_code isSymbolGlobal(DataRefImpl Symb, bool &Res) const;
+  virtual error_code getSymbolType(DataRefImpl Symb, SymbolRef::SymbolType &Res) const;
+
+  virtual error_code getSectionNext(DataRefImpl Sec, SectionRef &Res) const;
+  virtual error_code getSectionName(DataRefImpl Sec, StringRef &Res) const;
+  virtual error_code getSectionAddress(DataRefImpl Sec, uint64_t &Res) const;
+  virtual error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const;
+  virtual error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const;
+  virtual error_code getSectionAlignment(DataRefImpl Sec, uint64_t &Res) const;
+  virtual error_code isSectionText(DataRefImpl Sec, bool &Res) const;
+  virtual error_code isSectionData(DataRefImpl Sec, bool &Res) const;
+  virtual error_code isSectionBSS(DataRefImpl Sec, bool &Res) const;
+  virtual error_code sectionContainsSymbol(DataRefImpl DRI, DataRefImpl S,
+                                           bool &Result) const;
+  virtual relocation_iterator getSectionRelBegin(DataRefImpl Sec) const;
+  virtual relocation_iterator getSectionRelEnd(DataRefImpl Sec) const;
+
+  virtual error_code getRelocationNext(DataRefImpl Rel,
+                                       RelocationRef &Res) const;
+  virtual error_code getRelocationAddress(DataRefImpl Rel,
+                                          uint64_t &Res) const;
+  virtual error_code getRelocationSymbol(DataRefImpl Rel,
+                                         SymbolRef &Res) const;
+  virtual error_code getRelocationType(DataRefImpl Rel,
+                                       uint32_t &Res) const;
+  virtual error_code getRelocationTypeName(DataRefImpl Rel,
+                                           SmallVectorImpl<char> &Result) const;
+  virtual error_code getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                 int64_t &Res) const;
+  virtual error_code getRelocationValueString(DataRefImpl Rel,
+                                           SmallVectorImpl<char> &Result) const;
+
+private:
+  MachOObject *MachOObj;
+  mutable uint32_t RegisteredStringTable;
+  typedef SmallVector<DataRefImpl, 1> SectionList;
+  SectionList Sections;
+
+
+  void moveToNextSection(DataRefImpl &DRI) const;
+  void getSymbolTableEntry(DataRefImpl DRI,
+                           InMemoryStruct<macho::SymbolTableEntry> &Res) const;
+  void getSymbol64TableEntry(DataRefImpl DRI,
+                          InMemoryStruct<macho::Symbol64TableEntry> &Res) const;
+  void moveToNextSymbol(DataRefImpl &DRI) const;
+  void getSection(DataRefImpl DRI, InMemoryStruct<macho::Section> &Res) const;
+  void getSection64(DataRefImpl DRI,
+                    InMemoryStruct<macho::Section64> &Res) const;
+  void getRelocation(DataRefImpl Rel,
+                     InMemoryStruct<macho::RelocationEntry> &Res) const;
+  std::size_t getSectionIndex(DataRefImpl Sec) const;
+};
+
+}
+}
+
+#endif
+
diff --git a/contrib/llvm/include/llvm/Object/MachOFormat.h b/contrib/llvm/include/llvm/Object/MachOFormat.h
index 31cd523ea219..089cde92a0a3 100644
--- a/contrib/llvm/include/llvm/Object/MachOFormat.h
+++ b/contrib/llvm/include/llvm/Object/MachOFormat.h
@@ -137,7 +137,10 @@ namespace macho {
     LCT_Symtab = 0x2,
     LCT_Dysymtab = 0xb,
     LCT_Segment64 = 0x19,
-    LCT_UUID = 0x1b
+    LCT_UUID = 0x1b,
+    LCT_CodeSignature = 0x1d,
+    LCT_SegmentSplitInfo = 0x1e,
+    LCT_FunctionStarts = 0x26
   };
 
   /// \brief Load command structure.
@@ -218,6 +221,13 @@ namespace macho {
     uint32_t NumLocalRelocationTableEntries;
   };
 
+  struct LinkeditDataLoadCommand {
+    uint32_t Type;
+    uint32_t Size;
+    uint32_t DataOffset;
+    uint32_t DataSize;
+  };
+
   /// @}
   /// @name Section Data
   /// @{
diff --git a/contrib/llvm/include/llvm/Object/MachOObject.h b/contrib/llvm/include/llvm/Object/MachOObject.h
index 19a399e62fe3..51be847858a1 100644
--- a/contrib/llvm/include/llvm/Object/MachOObject.h
+++ b/contrib/llvm/include/llvm/Object/MachOObject.h
@@ -150,6 +150,9 @@ class MachOObject {
   void ReadDysymtabLoadCommand(
     const LoadCommandInfo &LCI,
     InMemoryStruct<macho::DysymtabLoadCommand> &Res) const;
+  void ReadLinkeditDataLoadCommand(
+    const LoadCommandInfo &LCI,
+    InMemoryStruct<macho::LinkeditDataLoadCommand> &Res) const;
   void ReadIndirectSymbolTableEntry(
     const macho::DysymtabLoadCommand &DLC,
     unsigned Index,
@@ -171,6 +174,7 @@ class MachOObject {
   void ReadSymbol64TableEntry(
     uint64_t SymbolTableOffset, unsigned Index,
     InMemoryStruct<macho::Symbol64TableEntry> &Res) const;
+  void ReadULEB128s(uint64_t Index, SmallVectorImpl<uint64_t> &Out) const;
 
   /// @}
   
diff --git a/contrib/llvm/include/llvm/Object/ObjectFile.h b/contrib/llvm/include/llvm/Object/ObjectFile.h
index 98ac0672796f..83854a0d6c28 100644
--- a/contrib/llvm/include/llvm/Object/ObjectFile.h
+++ b/contrib/llvm/include/llvm/Object/ObjectFile.h
@@ -27,34 +27,57 @@ namespace object {
 class ObjectFile;
 
 union DataRefImpl {
+  struct {
+    // ELF needs this for relocations. This entire union should probably be a
+    // char[max(8, sizeof(uintptr_t))] and require the impl to cast.
+    uint16_t a, b;
+    uint32_t c;
+  } w;
   struct {
     uint32_t a, b;
   } d;
   uintptr_t p;
 };
 
+template<class content_type>
+class content_iterator {
+  content_type Current;
+public:
+  content_iterator(content_type symb)
+    : Current(symb) {}
+
+  const content_type* operator->() const {
+    return &Current;
+  }
+
+  const content_type &operator*() const {
+    return Current;
+  }
+
+  bool operator==(const content_iterator &other) const {
+    return Current == other.Current;
+  }
+
+  bool operator!=(const content_iterator &other) const {
+    return !(*this == other);
+  }
+
+  content_iterator& increment(error_code &err) {
+    content_type next;
+    if (error_code ec = Current.getNext(next))
+      err = ec;
+    else
+      Current = next;
+    return *this;
+  }
+};
+
 static bool operator ==(const DataRefImpl &a, const DataRefImpl &b) {
   // Check bitwise identical. This is the only legal way to compare a union w/o
   // knowing which member is in use.
   return std::memcmp(&a, &b, sizeof(DataRefImpl)) == 0;
 }
 
-class RelocationRef {
-  DataRefImpl RelocationPimpl;
-  const ObjectFile *OwningObject;
-
-public:
-  RelocationRef() : OwningObject(NULL) {
-    std::memset(&RelocationPimpl, 0, sizeof(RelocationPimpl));
-  }
-
-  RelocationRef(DataRefImpl RelocationP, const ObjectFile *Owner);
-
-  bool operator==(const RelocationRef &Other) const;
-
-  error_code getNext(RelocationRef &Result);
-};
-
 /// SymbolRef - This is a value type class that represents a single symbol in
 /// the list of symbols in the object file.
 class SymbolRef {
@@ -67,6 +90,13 @@ class SymbolRef {
     std::memset(&SymbolPimpl, 0, sizeof(SymbolPimpl));
   }
 
+  enum SymbolType {
+    ST_Function,
+    ST_Data,
+    ST_External,    // Defined in another object file
+    ST_Other
+  };
+
   SymbolRef(DataRefImpl SymbolP, const ObjectFile *Owner);
 
   bool operator==(const SymbolRef &Other) const;
@@ -75,7 +105,9 @@ class SymbolRef {
 
   error_code getName(StringRef &Result) const;
   error_code getAddress(uint64_t &Result) const;
+  error_code getOffset(uint64_t &Result) const;
   error_code getSize(uint64_t &Result) const;
+  error_code getSymbolType(SymbolRef::SymbolType &Result) const;
 
   /// Returns the ascii char that should be displayed in a symbol table dump via
   /// nm for this symbol.
@@ -84,7 +116,49 @@ class SymbolRef {
   /// Returns true for symbols that are internal to the object file format such
   /// as section symbols.
   error_code isInternal(bool &Result) const;
+
+  /// Returns true for symbols that can be used in another objects,
+  /// such as library functions
+  error_code isGlobal(bool &Result) const;
+
+  DataRefImpl getRawDataRefImpl() const;
 };
+typedef content_iterator<SymbolRef> symbol_iterator;
+
+/// RelocationRef - This is a value type class that represents a single
+/// relocation in the list of relocations in the object file.
+class RelocationRef {
+  DataRefImpl RelocationPimpl;
+  const ObjectFile *OwningObject;
+
+public:
+  RelocationRef() : OwningObject(NULL) {
+    std::memset(&RelocationPimpl, 0, sizeof(RelocationPimpl));
+  }
+
+  RelocationRef(DataRefImpl RelocationP, const ObjectFile *Owner);
+
+  bool operator==(const RelocationRef &Other) const;
+
+  error_code getNext(RelocationRef &Result) const;
+
+  error_code getAddress(uint64_t &Result) const;
+  error_code getSymbol(SymbolRef &Result) const;
+  error_code getType(uint32_t &Result) const;
+
+  /// @brief Get a string that represents the type of this relocation.
+  ///
+  /// This is for display purposes only.
+  error_code getTypeName(SmallVectorImpl<char> &Result) const;
+  error_code getAdditionalInfo(int64_t &Result) const;
+
+  /// @brief Get a string that represents the calculation of the value of this
+  ///        relocation.
+  ///
+  /// This is for display purposes only.
+  error_code getValueString(SmallVectorImpl<char> &Result) const;
+};
+typedef content_iterator<RelocationRef> relocation_iterator;
 
 /// SectionRef - This is a value type class that represents a single section in
 /// the list of sections in the object file.
@@ -109,11 +183,20 @@ class SectionRef {
   error_code getSize(uint64_t &Result) const;
   error_code getContents(StringRef &Result) const;
 
+  /// @brief Get the alignment of this section as the actual value (not log 2).
+  error_code getAlignment(uint64_t &Result) const;
+
   // FIXME: Move to the normalization layer when it's created.
   error_code isText(bool &Result) const;
+  error_code isData(bool &Result) const;
+  error_code isBSS(bool &Result) const;
 
   error_code containsSymbol(SymbolRef S, bool &Result) const;
+
+  relocation_iterator begin_relocations() const;
+  relocation_iterator end_relocations() const;
 };
+typedef content_iterator<SectionRef> section_iterator;
 
 const uint64_t UnknownAddressOrSize = ~0ULL;
 
@@ -144,9 +227,12 @@ class ObjectFile : public Binary {
   virtual error_code getSymbolNext(DataRefImpl Symb, SymbolRef &Res) const = 0;
   virtual error_code getSymbolName(DataRefImpl Symb, StringRef &Res) const = 0;
   virtual error_code getSymbolAddress(DataRefImpl Symb, uint64_t &Res) const =0;
+  virtual error_code getSymbolOffset(DataRefImpl Symb, uint64_t &Res) const =0;
   virtual error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const = 0;
   virtual error_code getSymbolNMTypeChar(DataRefImpl Symb, char &Res) const = 0;
   virtual error_code isSymbolInternal(DataRefImpl Symb, bool &Res) const = 0;
+  virtual error_code isSymbolGlobal(DataRefImpl Symb, bool &Res) const = 0;
+  virtual error_code getSymbolType(DataRefImpl Symb, SymbolRef::SymbolType &Res) const = 0;
 
   // Same as above for SectionRef.
   friend class SectionRef;
@@ -155,47 +241,34 @@ class ObjectFile : public Binary {
   virtual error_code getSectionAddress(DataRefImpl Sec, uint64_t &Res) const =0;
   virtual error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const = 0;
   virtual error_code getSectionContents(DataRefImpl Sec, StringRef &Res)const=0;
+  virtual error_code getSectionAlignment(DataRefImpl Sec, uint64_t &Res)const=0;
   virtual error_code isSectionText(DataRefImpl Sec, bool &Res) const = 0;
+  virtual error_code isSectionData(DataRefImpl Sec, bool &Res) const = 0;
+  virtual error_code isSectionBSS(DataRefImpl Sec, bool &Res) const = 0;
   virtual error_code sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb,
                                            bool &Result) const = 0;
+  virtual relocation_iterator getSectionRelBegin(DataRefImpl Sec) const = 0;
+  virtual relocation_iterator getSectionRelEnd(DataRefImpl Sec) const = 0;
 
 
+  // Same as above for RelocationRef.
+  friend class RelocationRef;
+  virtual error_code getRelocationNext(DataRefImpl Rel,
+                                       RelocationRef &Res) const = 0;
+  virtual error_code getRelocationAddress(DataRefImpl Rel,
+                                          uint64_t &Res) const =0;
+  virtual error_code getRelocationSymbol(DataRefImpl Rel,
+                                         SymbolRef &Res) const = 0;
+  virtual error_code getRelocationType(DataRefImpl Rel,
+                                       uint32_t &Res) const = 0;
+  virtual error_code getRelocationTypeName(DataRefImpl Rel,
+                                       SmallVectorImpl<char> &Result) const = 0;
+  virtual error_code getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                 int64_t &Res) const = 0;
+  virtual error_code getRelocationValueString(DataRefImpl Rel,
+                                       SmallVectorImpl<char> &Result) const = 0;
+
 public:
-  template<class content_type>
-  class content_iterator {
-    content_type Current;
-  public:
-    content_iterator(content_type symb)
-      : Current(symb) {}
-
-    const content_type* operator->() const {
-      return &Current;
-    }
-
-    const content_type &operator*() const {
-      return Current;
-    }
-
-    bool operator==(const content_iterator &other) const {
-      return Current == other.Current;
-    }
-
-    bool operator!=(const content_iterator &other) const {
-      return !(*this == other);
-    }
-
-    content_iterator& increment(error_code &err) {
-      content_type next;
-      if (error_code ec = Current.getNext(next))
-        err = ec;
-      else
-        Current = next;
-      return *this;
-    }
-  };
-
-  typedef content_iterator<SymbolRef> symbol_iterator;
-  typedef content_iterator<SectionRef> section_iterator;
 
   virtual symbol_iterator begin_symbols() const = 0;
   virtual symbol_iterator end_symbols() const = 0;
@@ -250,6 +323,10 @@ inline error_code SymbolRef::getAddress(uint64_t &Result) const {
   return OwningObject->getSymbolAddress(SymbolPimpl, Result);
 }
 
+inline error_code SymbolRef::getOffset(uint64_t &Result) const {
+  return OwningObject->getSymbolOffset(SymbolPimpl, Result);
+}
+
 inline error_code SymbolRef::getSize(uint64_t &Result) const {
   return OwningObject->getSymbolSize(SymbolPimpl, Result);
 }
@@ -262,6 +339,18 @@ inline error_code SymbolRef::isInternal(bool &Result) const {
   return OwningObject->isSymbolInternal(SymbolPimpl, Result);
 }
 
+inline error_code SymbolRef::isGlobal(bool &Result) const {
+  return OwningObject->isSymbolGlobal(SymbolPimpl, Result);
+}
+
+inline error_code SymbolRef::getSymbolType(SymbolRef::SymbolType &Result) const {
+  return OwningObject->getSymbolType(SymbolPimpl, Result);
+}
+
+inline DataRefImpl SymbolRef::getRawDataRefImpl() const {
+  return SymbolPimpl;
+}
+
 
 /// SectionRef
 inline SectionRef::SectionRef(DataRefImpl SectionP,
@@ -293,15 +382,76 @@ inline error_code SectionRef::getContents(StringRef &Result) const {
   return OwningObject->getSectionContents(SectionPimpl, Result);
 }
 
+inline error_code SectionRef::getAlignment(uint64_t &Result) const {
+  return OwningObject->getSectionAlignment(SectionPimpl, Result);
+}
+
 inline error_code SectionRef::isText(bool &Result) const {
   return OwningObject->isSectionText(SectionPimpl, Result);
 }
 
+inline error_code SectionRef::isData(bool &Result) const {
+  return OwningObject->isSectionData(SectionPimpl, Result);
+}
+
+inline error_code SectionRef::isBSS(bool &Result) const {
+  return OwningObject->isSectionBSS(SectionPimpl, Result);
+}
+
 inline error_code SectionRef::containsSymbol(SymbolRef S, bool &Result) const {
   return OwningObject->sectionContainsSymbol(SectionPimpl, S.SymbolPimpl,
                                              Result);
 }
 
+inline relocation_iterator SectionRef::begin_relocations() const {
+  return OwningObject->getSectionRelBegin(SectionPimpl);
+}
+
+inline relocation_iterator SectionRef::end_relocations() const {
+  return OwningObject->getSectionRelEnd(SectionPimpl);
+}
+
+
+/// RelocationRef
+inline RelocationRef::RelocationRef(DataRefImpl RelocationP,
+                              const ObjectFile *Owner)
+  : RelocationPimpl(RelocationP)
+  , OwningObject(Owner) {}
+
+inline bool RelocationRef::operator==(const RelocationRef &Other) const {
+  return RelocationPimpl == Other.RelocationPimpl;
+}
+
+inline error_code RelocationRef::getNext(RelocationRef &Result) const {
+  return OwningObject->getRelocationNext(RelocationPimpl, Result);
+}
+
+inline error_code RelocationRef::getAddress(uint64_t &Result) const {
+  return OwningObject->getRelocationAddress(RelocationPimpl, Result);
+}
+
+inline error_code RelocationRef::getSymbol(SymbolRef &Result) const {
+  return OwningObject->getRelocationSymbol(RelocationPimpl, Result);
+}
+
+inline error_code RelocationRef::getType(uint32_t &Result) const {
+  return OwningObject->getRelocationType(RelocationPimpl, Result);
+}
+
+inline error_code RelocationRef::getTypeName(SmallVectorImpl<char> &Result)
+  const {
+  return OwningObject->getRelocationTypeName(RelocationPimpl, Result);
+}
+
+inline error_code RelocationRef::getAdditionalInfo(int64_t &Result) const {
+  return OwningObject->getRelocationAdditionalInfo(RelocationPimpl, Result);
+}
+
+inline error_code RelocationRef::getValueString(SmallVectorImpl<char> &Result)
+  const {
+  return OwningObject->getRelocationValueString(RelocationPimpl, Result);
+}
+
 } // end namespace object
 } // end namespace llvm
 
diff --git a/contrib/llvm/include/llvm/OperandTraits.h b/contrib/llvm/include/llvm/OperandTraits.h
index f0df5fa9bde8..3d8dc329b39f 100644
--- a/contrib/llvm/include/llvm/OperandTraits.h
+++ b/contrib/llvm/include/llvm/OperandTraits.h
@@ -136,45 +136,8 @@ CLASS::const_op_iterator CLASS::op_end() const { \
 VALUECLASS *CLASS::getOperand(unsigned i_nocapture) const { \
   assert(i_nocapture < OperandTraits<CLASS>::operands(this) \
          && "getOperand() out of range!"); \
-  return static_cast<VALUECLASS*>( \
-    OperandTraits<CLASS>::op_begin(const_cast<CLASS*>(this))[i_nocapture]); \
-} \
-void CLASS::setOperand(unsigned i_nocapture, VALUECLASS *Val_nocapture) { \
-  assert(i_nocapture < OperandTraits<CLASS>::operands(this) \
-         && "setOperand() out of range!"); \
-  OperandTraits<CLASS>::op_begin(this)[i_nocapture] = Val_nocapture; \
-} \
-unsigned CLASS::getNumOperands() const { \
-  return OperandTraits<CLASS>::operands(this);  \
-} \
-template <int Idx_nocapture> Use &CLASS::Op() { \
-  return this->OpFrom<Idx_nocapture>(this); \
-} \
-template <int Idx_nocapture> const Use &CLASS::Op() const { \
-  return this->OpFrom<Idx_nocapture>(this); \
-}
-
-
-/// Macro for generating out-of-class operand accessor
-/// definitions with casted result
-#define DEFINE_TRANSPARENT_CASTED_OPERAND_ACCESSORS(CLASS, VALUECLASS) \
-CLASS::op_iterator CLASS::op_begin() { \
-  return OperandTraits<CLASS>::op_begin(this); \
-} \
-CLASS::const_op_iterator CLASS::op_begin() const { \
-  return OperandTraits<CLASS>::op_begin(const_cast<CLASS*>(this)); \
-} \
-CLASS::op_iterator CLASS::op_end() { \
-  return OperandTraits<CLASS>::op_end(this); \
-} \
-CLASS::const_op_iterator CLASS::op_end() const { \
-  return OperandTraits<CLASS>::op_end(const_cast<CLASS*>(this)); \
-} \
-VALUECLASS *CLASS::getOperand(unsigned i_nocapture) const { \
-  assert(i_nocapture < OperandTraits<CLASS>::operands(this) \
-         && "getOperand() out of range!"); \
-  return cast<VALUECLASS>( \
-    OperandTraits<CLASS>::op_begin(const_cast<CLASS*>(this))[i_nocapture]); \
+  return cast_or_null<VALUECLASS>( \
+    OperandTraits<CLASS>::op_begin(const_cast<CLASS*>(this))[i_nocapture].get()); \
 } \
 void CLASS::setOperand(unsigned i_nocapture, VALUECLASS *Val_nocapture) { \
   assert(i_nocapture < OperandTraits<CLASS>::operands(this) \
diff --git a/contrib/llvm/include/llvm/Operator.h b/contrib/llvm/include/llvm/Operator.h
index e9aa4997f285..48a5796383b4 100644
--- a/contrib/llvm/include/llvm/Operator.h
+++ b/contrib/llvm/include/llvm/Operator.h
@@ -261,8 +261,8 @@ class GEPOperator
 
   /// getPointerOperandType - Method to return the pointer operand as a
   /// PointerType.
-  const PointerType *getPointerOperandType() const {
-    return reinterpret_cast<const PointerType*>(getPointerOperand()->getType());
+  PointerType *getPointerOperandType() const {
+    return reinterpret_cast<PointerType*>(getPointerOperand()->getType());
   }
 
   unsigned getNumIndices() const {  // Note: always non-negative
diff --git a/contrib/llvm/include/llvm/PassManagers.h b/contrib/llvm/include/llvm/PassManagers.h
index c4f409ef525c..c05347da7934 100644
--- a/contrib/llvm/include/llvm/PassManagers.h
+++ b/contrib/llvm/include/llvm/PassManagers.h
@@ -263,7 +263,7 @@ class PMTopLevelManager {
 class PMDataManager {
 public:
 
-  explicit PMDataManager(int Depth) : TPM(NULL), Depth(Depth) {
+  explicit PMDataManager() : TPM(NULL), Depth(0) {
     initializeAnalysisInfo();
   }
 
@@ -333,6 +333,7 @@ class PMDataManager {
   void setTopLevelManager(PMTopLevelManager *T) { TPM = T; }
 
   unsigned getDepth() const { return Depth; }
+  void setDepth(unsigned newDepth) { Depth = newDepth; }
 
   // Print routines used by debug-pass
   void dumpLastUses(Pass *P, unsigned Offset) const;
@@ -408,8 +409,8 @@ class PMDataManager {
 class FPPassManager : public ModulePass, public PMDataManager {
 public:
   static char ID;
-  explicit FPPassManager(int Depth) 
-  : ModulePass(ID), PMDataManager(Depth) { }
+  explicit FPPassManager() 
+  : ModulePass(ID), PMDataManager() { }
   
   /// run - Execute all of the passes scheduled for execution.  Keep track of
   /// whether any of the passes modifies the module, and if so, return true.
diff --git a/contrib/llvm/include/llvm/Support/BlockFrequency.h b/contrib/llvm/include/llvm/Support/BlockFrequency.h
new file mode 100644
index 000000000000..554b7845696d
--- /dev/null
+++ b/contrib/llvm/include/llvm/Support/BlockFrequency.h
@@ -0,0 +1,63 @@
+//===-------- BlockFrequency.h - Block Frequency Wrapper --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements Block Frequency class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_BLOCKFREQUENCY_H
+#define LLVM_SUPPORT_BLOCKFREQUENCY_H
+
+namespace llvm {
+
+class raw_ostream;
+class BranchProbability;
+
+// This class represents Block Frequency as a 64-bit value.
+class BlockFrequency {
+
+  uint64_t Frequency;
+  static const int64_t ENTRY_FREQ = 1024;
+
+public:
+  BlockFrequency(uint64_t Freq = 0) : Frequency(Freq) { }
+
+  static uint64_t getEntryFrequency() { return ENTRY_FREQ; }
+  uint64_t getFrequency() const { return Frequency; }
+
+  BlockFrequency &operator*=(const BranchProbability &Prob);
+  const BlockFrequency operator*(const BranchProbability &Prob) const;
+
+  BlockFrequency &operator+=(const BlockFrequency &Freq);
+  const BlockFrequency operator+(const BlockFrequency &Freq) const;
+
+  bool operator<(const BlockFrequency &RHS) const {
+    return Frequency < RHS.Frequency;
+  }
+
+  bool operator<=(const BlockFrequency &RHS) const {
+    return Frequency <= RHS.Frequency;
+  }
+
+  bool operator>(const BlockFrequency &RHS) const {
+    return Frequency > RHS.Frequency;
+  }
+
+  bool operator>=(const BlockFrequency &RHS) const {
+    return Frequency >= RHS.Frequency;
+  }
+
+  void print(raw_ostream &OS) const;
+};
+
+raw_ostream &operator<<(raw_ostream &OS, const BlockFrequency &Freq);
+
+}
+
+#endif
diff --git a/contrib/llvm/include/llvm/Support/BranchProbability.h b/contrib/llvm/include/llvm/Support/BranchProbability.h
index 2e81490ebf66..05c24d4fcfcb 100644
--- a/contrib/llvm/include/llvm/Support/BranchProbability.h
+++ b/contrib/llvm/include/llvm/Support/BranchProbability.h
@@ -1,4 +1,4 @@
-//===- BranchProbability.h - Branch Probability Analysis --------*- C++ -*-===//
+//===- BranchProbability.h - Branch Probability Wrapper ---------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -34,13 +34,13 @@ class BranchProbability {
 
   uint32_t getNumerator() const { return N; }
   uint32_t getDenominator() const { return D; }
-  
+
   // Return (1 - Probability).
   BranchProbability getCompl() {
     return BranchProbability(D - N, D);
   }
 
-  raw_ostream &print(raw_ostream &OS) const;
+  void print(raw_ostream &OS) const;
 
   void dump() const;
 };
diff --git a/contrib/llvm/include/llvm/Support/CallSite.h b/contrib/llvm/include/llvm/Support/CallSite.h
index 8a998a8cd0d1..04b8c4e69c52 100644
--- a/contrib/llvm/include/llvm/Support/CallSite.h
+++ b/contrib/llvm/include/llvm/Support/CallSite.h
@@ -147,7 +147,7 @@ class CallSiteBase {
   
   /// getType - Return the type of the instruction that generated this call site
   ///
-  const Type *getType() const { return (*this)->getType(); }
+  Type *getType() const { return (*this)->getType(); }
 
   /// getCaller - Return the caller function for this call site
   ///
diff --git a/contrib/llvm/include/llvm/Support/Capacity.h b/contrib/llvm/include/llvm/Support/Capacity.h
new file mode 100644
index 000000000000..d8cda43b3576
--- /dev/null
+++ b/contrib/llvm/include/llvm/Support/Capacity.h
@@ -0,0 +1,30 @@
+//===--- Capacity.h - Generic computation of ADT memory use -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the capacity function that computes the amount of
+// memory used by an ADT.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_CAPACITY_H
+#define LLVM_SUPPORT_CAPACITY_H
+
+namespace llvm {
+
+template <typename T>
+static inline size_t capacity_in_bytes(const T &x) {
+  // This default definition of capacity should work for things like std::vector
+  // and friends.  More specialized versions will work for others.
+  return x.capacity() * sizeof(typename T::value_type);
+}
+
+} // end namespace llvm
+
+#endif
+
diff --git a/contrib/llvm/include/llvm/Support/CodeGen.h b/contrib/llvm/include/llvm/Support/CodeGen.h
new file mode 100644
index 000000000000..41351dc73f3b
--- /dev/null
+++ b/contrib/llvm/include/llvm/Support/CodeGen.h
@@ -0,0 +1,32 @@
+//===-- llvm/Support/CodeGen.h - CodeGen Concepts ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file define some types which define code generation concepts. For
+// example, relocation model.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_CODEGEN_H
+#define LLVM_SUPPORT_CODEGEN_H
+
+namespace llvm {
+
+  // Relocation model types.
+  namespace Reloc {
+    enum Model { Default, Static, PIC_, DynamicNoPIC };
+  }
+
+  // Code model types.
+  namespace CodeModel {
+    enum Model { Default, JITDefault, Small, Kernel, Medium, Large };
+  }
+
+}  // end llvm namespace
+
+#endif
diff --git a/contrib/llvm/include/llvm/Support/CommandLine.h b/contrib/llvm/include/llvm/Support/CommandLine.h
index d6098711a07a..c6b62a8df9a4 100644
--- a/contrib/llvm/include/llvm/Support/CommandLine.h
+++ b/contrib/llvm/include/llvm/Support/CommandLine.h
@@ -59,6 +59,15 @@ void ParseEnvironmentOptions(const char *progName, const char *envvar,
 ///                     CommandLine utilities to print their own version string.
 void SetVersionPrinter(void (*func)());
 
+///===---------------------------------------------------------------------===//
+/// AddExtraVersionPrinter - Add an extra printer to use in addition to the
+///                          default one. This can be called multiple times,
+///                          and each time it adds a new function to the list
+///                          which will be called after the basic LLVM version
+///                          printing is complete. Each can then add additional
+///                          information specific to the tool.
+void AddExtraVersionPrinter(void (*func)());
+
 
 // PrintOptionValues - Print option values.
 // With -print-options print the difference between option values and defaults.
@@ -796,6 +805,28 @@ class parser<unsigned> : public basic_parser<unsigned> {
 
 EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<unsigned>);
 
+//--------------------------------------------------
+// parser<unsigned long long>
+//
+template<>
+class parser<unsigned long long> : public basic_parser<unsigned long long> {
+public:
+  // parse - Return true on error.
+  bool parse(Option &O, StringRef ArgName, StringRef Arg,
+             unsigned long long &Val);
+
+  // getValueName - Overload in subclass to provide a better default value.
+  virtual const char *getValueName() const { return "uint"; }
+
+  void printOptionDiff(const Option &O, unsigned long long V, OptVal Default,
+                       size_t GlobalWidth) const;
+
+  // An out-of-line virtual method to provide a 'home' for this class.
+  virtual void anchor();
+};
+
+EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<unsigned long long>);
+
 //--------------------------------------------------
 // parser<double>
 //
diff --git a/contrib/llvm/include/llvm/Support/ConstantFolder.h b/contrib/llvm/include/llvm/Support/ConstantFolder.h
index 733023566a6f..93aa3436d273 100644
--- a/contrib/llvm/include/llvm/Support/ConstantFolder.h
+++ b/contrib/llvm/include/llvm/Support/ConstantFolder.h
@@ -118,22 +118,34 @@ class ConstantFolder {
   // Memory Instructions
   //===--------------------------------------------------------------------===//
 
-  Constant *CreateGetElementPtr(Constant *C, Constant* const *IdxList,
-                                unsigned NumIdx) const {
-    return ConstantExpr::getGetElementPtr(C, IdxList, NumIdx);
+  Constant *CreateGetElementPtr(Constant *C,
+                                ArrayRef<Constant *> IdxList) const {
+    return ConstantExpr::getGetElementPtr(C, IdxList);
   }
-  Constant *CreateGetElementPtr(Constant *C, Value* const *IdxList,
-                                unsigned NumIdx) const {
-    return ConstantExpr::getGetElementPtr(C, IdxList, NumIdx);
+  Constant *CreateGetElementPtr(Constant *C, Constant *Idx) const {
+    // This form of the function only exists to avoid ambiguous overload
+    // warnings about whether to convert Idx to ArrayRef<Constant *> or
+    // ArrayRef<Value *>.
+    return ConstantExpr::getGetElementPtr(C, Idx);
+  }
+  Constant *CreateGetElementPtr(Constant *C,
+                                ArrayRef<Value *> IdxList) const {
+    return ConstantExpr::getGetElementPtr(C, IdxList);
   }
 
-  Constant *CreateInBoundsGetElementPtr(Constant *C, Constant* const *IdxList,
-                                        unsigned NumIdx) const {
-    return ConstantExpr::getInBoundsGetElementPtr(C, IdxList, NumIdx);
+  Constant *CreateInBoundsGetElementPtr(Constant *C,
+                                        ArrayRef<Constant *> IdxList) const {
+    return ConstantExpr::getInBoundsGetElementPtr(C, IdxList);
   }
-  Constant *CreateInBoundsGetElementPtr(Constant *C, Value* const *IdxList,
-                                        unsigned NumIdx) const {
-    return ConstantExpr::getInBoundsGetElementPtr(C, IdxList, NumIdx);
+  Constant *CreateInBoundsGetElementPtr(Constant *C, Constant *Idx) const {
+    // This form of the function only exists to avoid ambiguous overload
+    // warnings about whether to convert Idx to ArrayRef<Constant *> or
+    // ArrayRef<Value *>.
+    return ConstantExpr::getInBoundsGetElementPtr(C, Idx);
+  }
+  Constant *CreateInBoundsGetElementPtr(Constant *C,
+                                        ArrayRef<Value *> IdxList) const {
+    return ConstantExpr::getInBoundsGetElementPtr(C, IdxList);
   }
 
   //===--------------------------------------------------------------------===//
@@ -141,37 +153,37 @@ class ConstantFolder {
   //===--------------------------------------------------------------------===//
 
   Constant *CreateCast(Instruction::CastOps Op, Constant *C,
-                       const Type *DestTy) const {
+                       Type *DestTy) const {
     return ConstantExpr::getCast(Op, C, DestTy);
   }
-  Constant *CreatePointerCast(Constant *C, const Type *DestTy) const {
+  Constant *CreatePointerCast(Constant *C, Type *DestTy) const {
     return ConstantExpr::getPointerCast(C, DestTy);
   }
-  Constant *CreateIntCast(Constant *C, const Type *DestTy,
+  Constant *CreateIntCast(Constant *C, Type *DestTy,
                           bool isSigned) const {
     return ConstantExpr::getIntegerCast(C, DestTy, isSigned);
   }
-  Constant *CreateFPCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateFPCast(Constant *C, Type *DestTy) const {
     return ConstantExpr::getFPCast(C, DestTy);
   }
 
-  Constant *CreateBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateBitCast(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::BitCast, C, DestTy);
   }
-  Constant *CreateIntToPtr(Constant *C, const Type *DestTy) const {
+  Constant *CreateIntToPtr(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::IntToPtr, C, DestTy);
   }
-  Constant *CreatePtrToInt(Constant *C, const Type *DestTy) const {
+  Constant *CreatePtrToInt(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::PtrToInt, C, DestTy);
   }
-  Constant *CreateZExtOrBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateZExtOrBitCast(Constant *C, Type *DestTy) const {
     return ConstantExpr::getZExtOrBitCast(C, DestTy);
   }
-  Constant *CreateSExtOrBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateSExtOrBitCast(Constant *C, Type *DestTy) const {
     return ConstantExpr::getSExtOrBitCast(C, DestTy);
   }
 
-  Constant *CreateTruncOrBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateTruncOrBitCast(Constant *C, Type *DestTy) const {
     return ConstantExpr::getTruncOrBitCast(C, DestTy);
   }
 
diff --git a/contrib/llvm/include/llvm/Support/DataExtractor.h b/contrib/llvm/include/llvm/Support/DataExtractor.h
new file mode 100644
index 000000000000..506ec96930d9
--- /dev/null
+++ b/contrib/llvm/include/llvm/Support/DataExtractor.h
@@ -0,0 +1,352 @@
+//===-- DataExtractor.h -----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_DATAEXTRACTOR_H
+#define LLVM_SUPPORT_DATAEXTRACTOR_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/DataTypes.h"
+
+namespace llvm {
+class DataExtractor {
+  StringRef Data;
+  uint8_t IsLittleEndian;
+  uint8_t PointerSize;
+public:
+  /// Construct with a buffer that is owned by the caller.
+  ///
+  /// This constructor allows us to use data that is owned by the
+  /// caller. The data must stay around as long as this object is
+  /// valid.
+  DataExtractor(StringRef Data, bool IsLittleEndian, uint8_t PointerSize)
+    : Data(Data), IsLittleEndian(IsLittleEndian), PointerSize(PointerSize) {}
+
+  /// getData - Get the data pointed to by this extractor.
+  StringRef getData() const { return Data; }
+  /// isLittleEndian - Get the endianess for this extractor.
+  bool isLittleEndian() const { return IsLittleEndian; }
+  /// getAddressSize - Get the address size for this extractor.
+  uint8_t getAddressSize() const { return PointerSize; }
+
+  /// Extract a C string from \a *offset_ptr.
+  ///
+  /// Returns a pointer to a C String from the data at the offset
+  /// pointed to by \a offset_ptr. A variable length NULL terminated C
+  /// string will be extracted and the \a offset_ptr will be
+  /// updated with the offset of the byte that follows the NULL
+  /// terminator byte.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     A pointer to the C string value in the data. If the offset
+  ///     pointed to by \a offset_ptr is out of bounds, or if the
+  ///     offset plus the length of the C string is out of bounds,
+  ///     NULL will be returned.
+  const char *getCStr(uint32_t *offset_ptr) const;
+
+  /// Extract an unsigned integer of size \a byte_size from \a
+  /// *offset_ptr.
+  ///
+  /// Extract a single unsigned integer value and update the offset
+  /// pointed to by \a offset_ptr. The size of the extracted integer
+  /// is specified by the \a byte_size argument. \a byte_size should
+  /// have a value greater than or equal to one and less than or equal
+  /// to eight since the return value is 64 bits wide. Any
+  /// \a byte_size values less than 1 or greater than 8 will result in
+  /// nothing being extracted, and zero being returned.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @param[in] byte_size
+  ///     The size in byte of the integer to extract.
+  ///
+  /// @return
+  ///     The unsigned integer value that was extracted, or zero on
+  ///     failure.
+  uint64_t getUnsigned(uint32_t *offset_ptr, uint32_t byte_size) const;
+
+  /// Extract an signed integer of size \a byte_size from \a *offset_ptr.
+  ///
+  /// Extract a single signed integer value (sign extending if required)
+  /// and update the offset pointed to by \a offset_ptr. The size of
+  /// the extracted integer is specified by the \a byte_size argument.
+  /// \a byte_size should have a value greater than or equal to one
+  /// and less than or equal to eight since the return value is 64
+  /// bits wide. Any \a byte_size values less than 1 or greater than
+  /// 8 will result in nothing being extracted, and zero being returned.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @param[in] byte_size
+  ///     The size in byte of the integer to extract.
+  ///
+  /// @return
+  ///     The sign extended signed integer value that was extracted,
+  ///     or zero on failure.
+  int64_t getSigned(uint32_t *offset_ptr, uint32_t size) const;
+
+  //------------------------------------------------------------------
+  /// Extract an pointer from \a *offset_ptr.
+  ///
+  /// Extract a single pointer from the data and update the offset
+  /// pointed to by \a offset_ptr. The size of the extracted pointer
+  /// comes from the \a m_addr_size member variable and should be
+  /// set correctly prior to extracting any pointer values.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted pointer value as a 64 integer.
+  uint64_t getAddress(uint32_t *offset_ptr) const {
+    return getUnsigned(offset_ptr, PointerSize);
+  }
+
+  /// Extract a uint8_t value from \a *offset_ptr.
+  ///
+  /// Extract a single uint8_t from the binary data at the offset
+  /// pointed to by \a offset_ptr, and advance the offset on success.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted uint8_t value.
+  uint8_t getU8(uint32_t *offset_ptr) const;
+
+  /// Extract \a count uint8_t values from \a *offset_ptr.
+  ///
+  /// Extract \a count uint8_t values from the binary data at the
+  /// offset pointed to by \a offset_ptr, and advance the offset on
+  /// success. The extracted values are copied into \a dst.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @param[out] dst
+  ///     A buffer to copy \a count uint8_t values into. \a dst must
+  ///     be large enough to hold all requested data.
+  ///
+  /// @param[in] count
+  ///     The number of uint8_t values to extract.
+  ///
+  /// @return
+  ///     \a dst if all values were properly extracted and copied,
+  ///     NULL otherise.
+  uint8_t *getU8(uint32_t *offset_ptr, uint8_t *dst, uint32_t count) const;
+
+  //------------------------------------------------------------------
+  /// Extract a uint16_t value from \a *offset_ptr.
+  ///
+  /// Extract a single uint16_t from the binary data at the offset
+  /// pointed to by \a offset_ptr, and update the offset on success.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted uint16_t value.
+  //------------------------------------------------------------------
+  uint16_t getU16(uint32_t *offset_ptr) const;
+
+  /// Extract \a count uint16_t values from \a *offset_ptr.
+  ///
+  /// Extract \a count uint16_t values from the binary data at the
+  /// offset pointed to by \a offset_ptr, and advance the offset on
+  /// success. The extracted values are copied into \a dst.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @param[out] dst
+  ///     A buffer to copy \a count uint16_t values into. \a dst must
+  ///     be large enough to hold all requested data.
+  ///
+  /// @param[in] count
+  ///     The number of uint16_t values to extract.
+  ///
+  /// @return
+  ///     \a dst if all values were properly extracted and copied,
+  ///     NULL otherise.
+  uint16_t *getU16(uint32_t *offset_ptr, uint16_t *dst, uint32_t count) const;
+
+  /// Extract a uint32_t value from \a *offset_ptr.
+  ///
+  /// Extract a single uint32_t from the binary data at the offset
+  /// pointed to by \a offset_ptr, and update the offset on success.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted uint32_t value.
+  uint32_t getU32(uint32_t *offset_ptr) const;
+
+  /// Extract \a count uint32_t values from \a *offset_ptr.
+  ///
+  /// Extract \a count uint32_t values from the binary data at the
+  /// offset pointed to by \a offset_ptr, and advance the offset on
+  /// success. The extracted values are copied into \a dst.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @param[out] dst
+  ///     A buffer to copy \a count uint32_t values into. \a dst must
+  ///     be large enough to hold all requested data.
+  ///
+  /// @param[in] count
+  ///     The number of uint32_t values to extract.
+  ///
+  /// @return
+  ///     \a dst if all values were properly extracted and copied,
+  ///     NULL otherise.
+  uint32_t *getU32(uint32_t *offset_ptr, uint32_t *dst, uint32_t count) const;
+
+  /// Extract a uint64_t value from \a *offset_ptr.
+  ///
+  /// Extract a single uint64_t from the binary data at the offset
+  /// pointed to by \a offset_ptr, and update the offset on success.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted uint64_t value.
+  uint64_t getU64(uint32_t *offset_ptr) const;
+
+  /// Extract \a count uint64_t values from \a *offset_ptr.
+  ///
+  /// Extract \a count uint64_t values from the binary data at the
+  /// offset pointed to by \a offset_ptr, and advance the offset on
+  /// success. The extracted values are copied into \a dst.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @param[out] dst
+  ///     A buffer to copy \a count uint64_t values into. \a dst must
+  ///     be large enough to hold all requested data.
+  ///
+  /// @param[in] count
+  ///     The number of uint64_t values to extract.
+  ///
+  /// @return
+  ///     \a dst if all values were properly extracted and copied,
+  ///     NULL otherise.
+  uint64_t *getU64(uint32_t *offset_ptr, uint64_t *dst, uint32_t count) const;
+
+  /// Extract a signed LEB128 value from \a *offset_ptr.
+  ///
+  /// Extracts an signed LEB128 number from this object's data
+  /// starting at the offset pointed to by \a offset_ptr. The offset
+  /// pointed to by \a offset_ptr will be updated with the offset of
+  /// the byte following the last extracted byte.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted signed integer value.
+  int64_t getSLEB128(uint32_t *offset_ptr) const;
+
+  /// Extract a unsigned LEB128 value from \a *offset_ptr.
+  ///
+  /// Extracts an unsigned LEB128 number from this object's data
+  /// starting at the offset pointed to by \a offset_ptr. The offset
+  /// pointed to by \a offset_ptr will be updated with the offset of
+  /// the byte following the last extracted byte.
+  ///
+  /// @param[in,out] offset_ptr
+  ///     A pointer to an offset within the data that will be advanced
+  ///     by the appropriate number of bytes if the value is extracted
+  ///     correctly. If the offset is out of bounds or there are not
+  ///     enough bytes to extract this value, the offset will be left
+  ///     unmodified.
+  ///
+  /// @return
+  ///     The extracted unsigned integer value.
+  uint64_t getULEB128(uint32_t *offset_ptr) const;
+
+  /// Test the validity of \a offset.
+  ///
+  /// @return
+  ///     \b true if \a offset is a valid offset into the data in this
+  ///     object, \b false otherwise.
+  bool isValidOffset(uint32_t offset) const { return Data.size() > offset; }
+
+  /// Test the availability of \a length bytes of data from \a offset.
+  ///
+  /// @return
+  ///     \b true if \a offset is a valid offset and there are \a
+  ///     length bytes available at that offset, \b false otherwise.
+  bool isValidOffsetForDataOfSize(uint32_t offset, uint32_t length) const {
+    return offset + length >= offset && isValidOffset(offset + length - 1);
+  }
+};
+
+} // namespace llvm
+
+#endif
diff --git a/contrib/llvm/include/llvm/Support/DataTypes.h.in b/contrib/llvm/include/llvm/Support/DataTypes.h.in
index 5965e8c0b2a9..425805a1669b 100644
--- a/contrib/llvm/include/llvm/Support/DataTypes.h.in
+++ b/contrib/llvm/include/llvm/Support/DataTypes.h.in
@@ -1,4 +1,4 @@
-/*===-- include/System/DataTypes.h - Define fixed size types -----*- C -*-===*\
+/*===-- include/Support/DataTypes.h - Define fixed size types -----*- C -*-===*\
 |*                                                                            *|
 |*                     The LLVM Compiler Infrastructure                       *|
 |*                                                                            *|
@@ -21,6 +21,8 @@
 
 /* Please leave this file C-compatible. */
 
+/* Please keep this file in sync with DataTypes.h.cmake */
+
 #ifndef SUPPORT_DATATYPES_H
 #define SUPPORT_DATATYPES_H
 
@@ -36,17 +38,19 @@
 #include <math.h>
 #endif
 
+#ifndef _MSC_VER
+
 /* Note that this header's correct operation depends on __STDC_LIMIT_MACROS
    being defined.  We would define it here, but in order to prevent Bad Things
    happening when system headers or C++ STL headers include stdint.h before we
    define it here, we define it on the g++ command line (in Makefile.rules). */
 #if !defined(__STDC_LIMIT_MACROS)
-# error "Must #define __STDC_LIMIT_MACROS before #including System/DataTypes.h"
+# error "Must #define __STDC_LIMIT_MACROS before #including Support/DataTypes.h"
 #endif
 
 #if !defined(__STDC_CONSTANT_MACROS)
 # error "Must #define __STDC_CONSTANT_MACROS before " \
-        "#including System/DataTypes.h"
+        "#including Support/DataTypes.h"
 #endif
 
 /* Note that <inttypes.h> includes <stdint.h>, if this is a C99 system. */
@@ -87,6 +91,88 @@ typedef u_int64_t uint64_t;
 #define UINT32_MAX 4294967295U
 #endif
 
+#else /* _MSC_VER */
+/* Visual C++ doesn't provide standard integer headers, but it does provide
+   built-in data types. */
+#include <stdlib.h>
+#include <stddef.h>
+#include <sys/types.h>
+#ifdef __cplusplus
+#include <cmath>
+#else
+#include <math.h>
+#endif
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+typedef signed int int32_t;
+typedef unsigned int uint32_t;
+typedef short int16_t;
+typedef unsigned short uint16_t;
+typedef signed char int8_t;
+typedef unsigned char uint8_t;
+typedef signed int ssize_t;
+#ifndef INT8_MAX
+# define INT8_MAX 127
+#endif
+#ifndef INT8_MIN
+# define INT8_MIN -128
+#endif
+#ifndef UINT8_MAX
+# define UINT8_MAX 255
+#endif
+#ifndef INT16_MAX
+# define INT16_MAX 32767
+#endif
+#ifndef INT16_MIN
+# define INT16_MIN -32768
+#endif
+#ifndef UINT16_MAX
+# define UINT16_MAX 65535
+#endif
+#ifndef INT32_MAX
+# define INT32_MAX 2147483647
+#endif
+#ifndef INT32_MIN
+/* MSC treats -2147483648 as -(2147483648U). */
+# define INT32_MIN (-INT32_MAX - 1)
+#endif
+#ifndef UINT32_MAX
+# define UINT32_MAX 4294967295U
+#endif
+/* Certain compatibility updates to VC++ introduce the `cstdint'
+ * header, which defines the INT*_C macros. On default installs they
+ * are absent. */
+#ifndef INT8_C
+# define INT8_C(C)   C##i8
+#endif
+#ifndef UINT8_C
+# define UINT8_C(C)  C##ui8
+#endif
+#ifndef INT16_C
+# define INT16_C(C)  C##i16
+#endif
+#ifndef UINT16_C
+# define UINT16_C(C) C##ui16
+#endif
+#ifndef INT32_C
+# define INT32_C(C)  C##i32
+#endif
+#ifndef UINT32_C
+# define UINT32_C(C) C##ui32
+#endif
+#ifndef INT64_C
+# define INT64_C(C)  C##i64
+#endif
+#ifndef UINT64_C
+# define UINT64_C(C) C##ui64
+#endif
+
+#ifndef PRIx64
+# define PRIx64 "I64x"
+#endif
+
+#endif /* _MSC_VER */
+
 /* Set defaults for constants which we cannot find. */
 #if !defined(INT64_MAX)
 # define INT64_MAX 9223372036854775807LL
diff --git a/contrib/llvm/include/llvm/Support/Dwarf.h b/contrib/llvm/include/llvm/Support/Dwarf.h
index 70bac0c9fc86..30f91874db20 100644
--- a/contrib/llvm/include/llvm/Support/Dwarf.h
+++ b/contrib/llvm/include/llvm/Support/Dwarf.h
@@ -22,8 +22,10 @@ namespace llvm {
 // Debug info constants.
 
 enum {
-  LLVMDebugVersion = (9 << 16),         // Current version of debug information.
-  LLVMDebugVersion8 = (8 << 16),         // Cconstant for version 8.
+  LLVMDebugVersion = (11 << 16),        // Current version of debug information.
+  LLVMDebugVersion10 = (10 << 16),      // Constant for version 10.
+  LLVMDebugVersion9 = (9 << 16),        // Constant for version 9.
+  LLVMDebugVersion8 = (8 << 16),        // Constant for version 8.
   LLVMDebugVersion7 = (7 << 16),        // Constant for version 7.
   LLVMDebugVersion6 = (6 << 16),        // Constant for version 6.
   LLVMDebugVersion5 = (5 << 16),        // Constant for version 5.
@@ -117,7 +119,16 @@ enum dwarf_constants {
   DW_TAG_imported_unit = 0x3d,
   DW_TAG_condition = 0x3f,
   DW_TAG_shared_type = 0x40,
-  DW_TAG_rvalue_reference_type = 0x41,
+  DW_TAG_type_unit = 0x41,
+  DW_TAG_rvalue_reference_type = 0x42,
+  DW_TAG_template_alias = 0x43,
+  DW_TAG_MIPS_loop = 0x4081,
+  DW_TAG_format_label = 0x4101,
+  DW_TAG_function_template = 0x4102,
+  DW_TAG_class_template = 0x4103,
+  DW_TAG_GNU_template_template_param = 0x4106,
+  DW_TAG_GNU_template_parameter_pack = 0x4107,
+  DW_TAG_GNU_formal_parameter_pack = 0x4108,
   DW_TAG_lo_user = 0x4080,
   DW_TAG_hi_user = 0xffff,
 
@@ -212,14 +223,36 @@ enum dwarf_constants {
   DW_AT_elemental = 0x66,
   DW_AT_pure = 0x67,
   DW_AT_recursive = 0x68,
+  DW_AT_signature = 0x69,
+  DW_AT_main_subprogram = 0x6a,
+  DW_AT_data_bit_offset = 0x6b,
+  DW_AT_const_expr = 0x6c,
+  DW_AT_enum_class = 0x6d,
+  DW_AT_linkage_name = 0x6e,
+  DW_AT_MIPS_loop_begin = 0x2002,
+  DW_AT_MIPS_tail_loop_begin = 0x2003,
+  DW_AT_MIPS_epilog_begin = 0x2004,
+  DW_AT_MIPS_loop_unroll_factor = 0x2005,
+  DW_AT_MIPS_software_pipeline_depth = 0x2006,
   DW_AT_MIPS_linkage_name = 0x2007,
-  DW_AT_sf_names   = 0x2101,
+  DW_AT_MIPS_stride = 0x2008,
+  DW_AT_MIPS_abstract_name = 0x2009,
+  DW_AT_MIPS_clone_origin = 0x200a,
+  DW_AT_MIPS_has_inlines = 0x200b,
+  DW_AT_MIPS_stride_byte = 0x200c,
+  DW_AT_MIPS_stride_elem = 0x200d,
+  DW_AT_MIPS_ptr_dopetype = 0x200e,
+  DW_AT_MIPS_allocatable_dopetype = 0x200f,
+  DW_AT_MIPS_assumed_shape_dopetype = 0x2010,
+  DW_AT_sf_names = 0x2101,
   DW_AT_src_info = 0x2102,
   DW_AT_mac_info = 0x2103,
   DW_AT_src_coords = 0x2104,
   DW_AT_body_begin = 0x2105,
   DW_AT_body_end = 0x2106,
   DW_AT_GNU_vector = 0x2107,
+  DW_AT_GNU_template_name = 0x2110,
+  DW_AT_MIPS_assumed_size = 0x2011,
   DW_AT_lo_user = 0x2000,
   DW_AT_hi_user = 0x3fff,
 
@@ -259,6 +292,10 @@ enum dwarf_constants {
   DW_FORM_ref8 = 0x14,
   DW_FORM_ref_udata = 0x15,
   DW_FORM_indirect = 0x16,
+  DW_FORM_sec_offset = 0x17,
+  DW_FORM_exprloc = 0x18,
+  DW_FORM_flag_present = 0x19,
+  DW_FORM_ref_sig8 = 0x20,
 
   // Operation encodings
   DW_OP_addr = 0x03,
@@ -413,6 +450,8 @@ enum dwarf_constants {
   DW_OP_form_tls_address = 0x9b,
   DW_OP_call_frame_cfa = 0x9c,
   DW_OP_bit_piece = 0x9d,
+  DW_OP_implicit_value = 0x9e,
+  DW_OP_stack_value = 0x9f,
   DW_OP_lo_user = 0xe0,
   DW_OP_hi_user = 0xff,
 
@@ -432,6 +471,7 @@ enum dwarf_constants {
   DW_ATE_signed_fixed = 0x0d,
   DW_ATE_unsigned_fixed = 0x0e,
   DW_ATE_decimal_float = 0x0f,
+  DW_ATE_UTF = 0x10,
   DW_ATE_lo_user = 0x80,
   DW_ATE_hi_user = 0xff,
 
@@ -484,6 +524,7 @@ enum dwarf_constants {
   DW_LANG_ObjC_plus_plus = 0x0011,
   DW_LANG_UPC = 0x0012,
   DW_LANG_D = 0x0013,
+  DW_LANG_Python = 0x0014,
   DW_LANG_lo_user = 0x8000,
   DW_LANG_hi_user = 0xffff,
 
@@ -533,6 +574,7 @@ enum dwarf_constants {
   DW_LNE_end_sequence = 0x01,
   DW_LNE_set_address = 0x02,
   DW_LNE_define_file = 0x03,
+  DW_LNE_set_discriminator = 0x04,
   DW_LNE_lo_user = 0x80,
   DW_LNE_hi_user = 0xff,
 
@@ -571,6 +613,9 @@ enum dwarf_constants {
   DW_CFA_val_offset = 0x14,
   DW_CFA_val_offset_sf = 0x15,
   DW_CFA_val_expression = 0x16,
+  DW_CFA_MIPS_advance_loc8 = 0x1d,
+  DW_CFA_GNU_window_save = 0x2d,
+  DW_CFA_GNU_args_size = 0x2e,
   DW_CFA_lo_user = 0x1c,
   DW_CFA_hi_user = 0x3f,
 
diff --git a/contrib/llvm/include/llvm/Support/DynamicLibrary.h b/contrib/llvm/include/llvm/Support/DynamicLibrary.h
index e6d9ff57ae83..288936bc0b9b 100644
--- a/contrib/llvm/include/llvm/Support/DynamicLibrary.h
+++ b/contrib/llvm/include/llvm/Support/DynamicLibrary.h
@@ -28,36 +28,62 @@ namespace sys {
   /// It also allows for symbols to be defined which don't live in any library,
   /// but rather the main program itself, useful on Windows where the main
   /// executable cannot be searched.
+  ///
+  /// Note: there is currently no interface for temporarily loading a library,
+  /// or for unloading libraries when the LLVM library is unloaded.
   class DynamicLibrary {
-    DynamicLibrary(); // DO NOT IMPLEMENT
+    // Placeholder whose address represents an invalid library.
+    // We use this instead of NULL or a pointer-int pair because the OS library
+    // might define 0 or 1 to be "special" handles, such as "search all".
+    static char Invalid;
+
+    // Opaque data used to interface with OS-specific dynamic library handling.
+    void *Data;
+
+    explicit DynamicLibrary(void *data = &Invalid) : Data(data) {}
   public:
-    /// This function allows a library to be loaded without instantiating a
-    /// DynamicLibrary object. Consequently, it is marked as being permanent
-    /// and will only be unloaded when the program terminates.  This returns
-    /// false on success or returns true and fills in *ErrMsg on failure.
+    /// Returns true if the object refers to a valid library.
+    bool isValid() { return Data != &Invalid; }
+
+    /// Searches through the library for the symbol \p symbolName. If it is
+    /// found, the address of that symbol is returned. If not, NULL is returned.
+    /// Note that NULL will also be returned if the library failed to load.
+    /// Use isValid() to distinguish these cases if it is important.
+    /// Note that this will \e not search symbols explicitly registered by
+    /// AddSymbol().
+    void *getAddressOfSymbol(const char *symbolName);
+
+    /// This function permanently loads the dynamic library at the given path.
+    /// The library will only be unloaded when the program terminates.
+    /// This returns a valid DynamicLibrary instance on success and an invalid
+    /// instance on failure (see isValid()). \p *errMsg will only be modified
+    /// if the library fails to load.
+    ///
+    /// It is safe to call this function multiple times for the same library.
     /// @brief Open a dynamic library permanently.
+    static DynamicLibrary getPermanentLibrary(const char *filename,
+                                              std::string *errMsg = 0);
+
+    /// This function permanently loads the dynamic library at the given path.
+    /// Use this instead of getPermanentLibrary() when you won't need to get
+    /// symbols from the library itself.
     ///
-    /// NOTE: This function is not thread safe.
-    ///
-    static bool LoadLibraryPermanently(const char *filename,
-                                       std::string *ErrMsg = 0);
+    /// It is safe to call this function multiple times for the same library.
+    static bool LoadLibraryPermanently(const char *Filename,
+                                       std::string *ErrMsg = 0) {
+      return !getPermanentLibrary(Filename, ErrMsg).isValid();
+    }
 
     /// This function will search through all previously loaded dynamic
-    /// libraries for the symbol \p symbolName. If it is found, the addressof
+    /// libraries for the symbol \p symbolName. If it is found, the address of
     /// that symbol is returned. If not, null is returned. Note that this will
-    /// search permanently loaded libraries (LoadLibraryPermanently) as well
-    /// as ephemerally loaded libraries (constructors).
+    /// search permanently loaded libraries (getPermanentLibrary()) as well
+    /// as explicitly registered symbols (AddSymbol()).
     /// @throws std::string on error.
     /// @brief Search through libraries for address of a symbol
-    ///
-    /// NOTE: This function is not thread safe.
-    ///
     static void *SearchForAddressOfSymbol(const char *symbolName);
 
     /// @brief Convenience function for C++ophiles.
-    ///
-    /// NOTE: This function is not thread safe.
-    ///
     static void *SearchForAddressOfSymbol(const std::string &symbolName) {
       return SearchForAddressOfSymbol(symbolName.c_str());
     }
@@ -66,18 +92,7 @@ namespace sys {
     /// value \p symbolValue.  These symbols are searched before any
     /// libraries.
     /// @brief Add searchable symbol/value pair.
-    ///
-    /// NOTE: This function is not thread safe.
-    ///
-    static void AddSymbol(const char *symbolName, void *symbolValue);
-
-    /// @brief Convenience function for C++ophiles.
-    ///
-    /// NOTE: This function is not thread safe.
-    ///
-    static void AddSymbol(const std::string &symbolName, void *symbolValue) {
-      AddSymbol(symbolName.c_str(), symbolValue);
-    }
+    static void AddSymbol(StringRef symbolName, void *symbolValue);
   };
 
 } // End sys namespace
diff --git a/contrib/llvm/include/llvm/Support/ELF.h b/contrib/llvm/include/llvm/Support/ELF.h
index be48112a908b..c5b85e2e6a12 100644
--- a/contrib/llvm/include/llvm/Support/ELF.h
+++ b/contrib/llvm/include/llvm/Support/ELF.h
@@ -28,20 +28,18 @@ namespace llvm {
 namespace ELF {
 
 typedef uint32_t Elf32_Addr; // Program address
-typedef uint16_t Elf32_Half;
 typedef uint32_t Elf32_Off;  // File offset
-typedef int32_t  Elf32_Sword;
+typedef uint16_t Elf32_Half;
 typedef uint32_t Elf32_Word;
+typedef int32_t  Elf32_Sword;
 
 typedef uint64_t Elf64_Addr;
 typedef uint64_t Elf64_Off;
-typedef int32_t  Elf64_Shalf;
-typedef int32_t  Elf64_Sword;
+typedef uint16_t Elf64_Half;
 typedef uint32_t Elf64_Word;
-typedef int64_t  Elf64_Sxword;
+typedef int32_t  Elf64_Sword;
 typedef uint64_t Elf64_Xword;
-typedef uint32_t Elf64_Half;
-typedef uint16_t Elf64_Quarter;
+typedef int64_t  Elf64_Sxword;
 
 // Object file magic string.
 static const char ElfMagic[] = { 0x7f, 'E', 'L', 'F', '\0' };
@@ -87,19 +85,19 @@ struct Elf32_Ehdr {
 // types (see above).
 struct Elf64_Ehdr {
   unsigned char e_ident[EI_NIDENT];
-  Elf64_Quarter e_type;
-  Elf64_Quarter e_machine;
-  Elf64_Half    e_version;
+  Elf64_Half    e_type;
+  Elf64_Half    e_machine;
+  Elf64_Word    e_version;
   Elf64_Addr    e_entry;
   Elf64_Off     e_phoff;
   Elf64_Off     e_shoff;
-  Elf64_Half    e_flags;
-  Elf64_Quarter e_ehsize;
-  Elf64_Quarter e_phentsize;
-  Elf64_Quarter e_phnum;
-  Elf64_Quarter e_shentsize;
-  Elf64_Quarter e_shnum;
-  Elf64_Quarter e_shstrndx;
+  Elf64_Word    e_flags;
+  Elf64_Half    e_ehsize;
+  Elf64_Half    e_phentsize;
+  Elf64_Half    e_phnum;
+  Elf64_Half    e_shentsize;
+  Elf64_Half    e_shnum;
+  Elf64_Half    e_shstrndx;
   bool checkMagic() const {
     return (memcmp(e_ident, ElfMagic, strlen(ElfMagic))) == 0;
   }
@@ -126,22 +124,170 @@ enum {
 
 // Machine architectures
 enum {
-  EM_NONE = 0,      // No machine
-  EM_M32 = 1,       // AT&T WE 32100
-  EM_SPARC = 2,     // SPARC
-  EM_386 = 3,       // Intel 386
-  EM_68K = 4,       // Motorola 68000
-  EM_88K = 5,       // Motorola 88000
-  EM_486 = 6,       // Intel 486 (deprecated)
-  EM_860 = 7,       // Intel 80860
-  EM_MIPS = 8,      // MIPS R3000
-  EM_PPC = 20,      // PowerPC
-  EM_PPC64 = 21,    // PowerPC64
-  EM_ARM = 40,      // ARM
-  EM_ALPHA = 41,    // DEC Alpha
-  EM_SPARCV9 = 43,  // SPARC V9
-  EM_X86_64 = 62,   // AMD64
-  EM_MBLAZE = 47787 // Xilinx MicroBlaze
+  EM_NONE          = 0, // No machine
+  EM_M32           = 1, // AT&T WE 32100
+  EM_SPARC         = 2, // SPARC
+  EM_386           = 3, // Intel 386
+  EM_68K           = 4, // Motorola 68000
+  EM_88K           = 5, // Motorola 88000
+  EM_486           = 6, // Intel 486 (deprecated)
+  EM_860           = 7, // Intel 80860
+  EM_MIPS          = 8, // MIPS R3000
+  EM_S370          = 9, // IBM System/370
+  EM_MIPS_RS3_LE   = 10, // MIPS RS3000 Little-endian
+  EM_PARISC        = 15, // Hewlett-Packard PA-RISC
+  EM_VPP500        = 17, // Fujitsu VPP500
+  EM_SPARC32PLUS   = 18, // Enhanced instruction set SPARC
+  EM_960           = 19, // Intel 80960
+  EM_PPC           = 20, // PowerPC
+  EM_PPC64         = 21, // PowerPC64
+  EM_S390          = 22, // IBM System/390
+  EM_SPU           = 23, // IBM SPU/SPC
+  EM_V800          = 36, // NEC V800
+  EM_FR20          = 37, // Fujitsu FR20
+  EM_RH32          = 38, // TRW RH-32
+  EM_RCE           = 39, // Motorola RCE
+  EM_ARM           = 40, // ARM
+  EM_ALPHA         = 41, // DEC Alpha
+  EM_SH            = 42, // Hitachi SH
+  EM_SPARCV9       = 43, // SPARC V9
+  EM_TRICORE       = 44, // Siemens TriCore
+  EM_ARC           = 45, // Argonaut RISC Core
+  EM_H8_300        = 46, // Hitachi H8/300
+  EM_H8_300H       = 47, // Hitachi H8/300H
+  EM_H8S           = 48, // Hitachi H8S
+  EM_H8_500        = 49, // Hitachi H8/500
+  EM_IA_64         = 50, // Intel IA-64 processor architecture
+  EM_MIPS_X        = 51, // Stanford MIPS-X
+  EM_COLDFIRE      = 52, // Motorola ColdFire
+  EM_68HC12        = 53, // Motorola M68HC12
+  EM_MMA           = 54, // Fujitsu MMA Multimedia Accelerator
+  EM_PCP           = 55, // Siemens PCP
+  EM_NCPU          = 56, // Sony nCPU embedded RISC processor
+  EM_NDR1          = 57, // Denso NDR1 microprocessor
+  EM_STARCORE      = 58, // Motorola Star*Core processor
+  EM_ME16          = 59, // Toyota ME16 processor
+  EM_ST100         = 60, // STMicroelectronics ST100 processor
+  EM_TINYJ         = 61, // Advanced Logic Corp. TinyJ embedded processor family
+  EM_X86_64        = 62, // AMD x86-64 architecture
+  EM_PDSP          = 63, // Sony DSP Processor
+  EM_PDP10         = 64, // Digital Equipment Corp. PDP-10
+  EM_PDP11         = 65, // Digital Equipment Corp. PDP-11
+  EM_FX66          = 66, // Siemens FX66 microcontroller
+  EM_ST9PLUS       = 67, // STMicroelectronics ST9+ 8/16 bit microcontroller
+  EM_ST7           = 68, // STMicroelectronics ST7 8-bit microcontroller
+  EM_68HC16        = 69, // Motorola MC68HC16 Microcontroller
+  EM_68HC11        = 70, // Motorola MC68HC11 Microcontroller
+  EM_68HC08        = 71, // Motorola MC68HC08 Microcontroller
+  EM_68HC05        = 72, // Motorola MC68HC05 Microcontroller
+  EM_SVX           = 73, // Silicon Graphics SVx
+  EM_ST19          = 74, // STMicroelectronics ST19 8-bit microcontroller
+  EM_VAX           = 75, // Digital VAX
+  EM_CRIS          = 76, // Axis Communications 32-bit embedded processor
+  EM_JAVELIN       = 77, // Infineon Technologies 32-bit embedded processor
+  EM_FIREPATH      = 78, // Element 14 64-bit DSP Processor
+  EM_ZSP           = 79, // LSI Logic 16-bit DSP Processor
+  EM_MMIX          = 80, // Donald Knuth's educational 64-bit processor
+  EM_HUANY         = 81, // Harvard University machine-independent object files
+  EM_PRISM         = 82, // SiTera Prism
+  EM_AVR           = 83, // Atmel AVR 8-bit microcontroller
+  EM_FR30          = 84, // Fujitsu FR30
+  EM_D10V          = 85, // Mitsubishi D10V
+  EM_D30V          = 86, // Mitsubishi D30V
+  EM_V850          = 87, // NEC v850
+  EM_M32R          = 88, // Mitsubishi M32R
+  EM_MN10300       = 89, // Matsushita MN10300
+  EM_MN10200       = 90, // Matsushita MN10200
+  EM_PJ            = 91, // picoJava
+  EM_OPENRISC      = 92, // OpenRISC 32-bit embedded processor
+  EM_ARC_COMPACT   = 93, // ARC International ARCompact processor (old
+                         // spelling/synonym: EM_ARC_A5)
+  EM_XTENSA        = 94, // Tensilica Xtensa Architecture
+  EM_VIDEOCORE     = 95, // Alphamosaic VideoCore processor
+  EM_TMM_GPP       = 96, // Thompson Multimedia General Purpose Processor
+  EM_NS32K         = 97, // National Semiconductor 32000 series
+  EM_TPC           = 98, // Tenor Network TPC processor
+  EM_SNP1K         = 99, // Trebia SNP 1000 processor
+  EM_ST200         = 100, // STMicroelectronics (www.st.com) ST200
+  EM_IP2K          = 101, // Ubicom IP2xxx microcontroller family
+  EM_MAX           = 102, // MAX Processor
+  EM_CR            = 103, // National Semiconductor CompactRISC microprocessor
+  EM_F2MC16        = 104, // Fujitsu F2MC16
+  EM_MSP430        = 105, // Texas Instruments embedded microcontroller msp430
+  EM_BLACKFIN      = 106, // Analog Devices Blackfin (DSP) processor
+  EM_SE_C33        = 107, // S1C33 Family of Seiko Epson processors
+  EM_SEP           = 108, // Sharp embedded microprocessor
+  EM_ARCA          = 109, // Arca RISC Microprocessor
+  EM_UNICORE       = 110, // Microprocessor series from PKU-Unity Ltd. and MPRC
+                          // of Peking University
+  EM_EXCESS        = 111, // eXcess: 16/32/64-bit configurable embedded CPU
+  EM_DXP           = 112, // Icera Semiconductor Inc. Deep Execution Processor
+  EM_ALTERA_NIOS2  = 113, // Altera Nios II soft-core processor
+  EM_CRX           = 114, // National Semiconductor CompactRISC CRX
+  EM_XGATE         = 115, // Motorola XGATE embedded processor
+  EM_C166          = 116, // Infineon C16x/XC16x processor
+  EM_M16C          = 117, // Renesas M16C series microprocessors
+  EM_DSPIC30F      = 118, // Microchip Technology dsPIC30F Digital Signal
+                          // Controller
+  EM_CE            = 119, // Freescale Communication Engine RISC core
+  EM_M32C          = 120, // Renesas M32C series microprocessors
+  EM_TSK3000       = 131, // Altium TSK3000 core
+  EM_RS08          = 132, // Freescale RS08 embedded processor
+  EM_SHARC         = 133, // Analog Devices SHARC family of 32-bit DSP
+                          // processors
+  EM_ECOG2         = 134, // Cyan Technology eCOG2 microprocessor
+  EM_SCORE7        = 135, // Sunplus S+core7 RISC processor
+  EM_DSP24         = 136, // New Japan Radio (NJR) 24-bit DSP Processor
+  EM_VIDEOCORE3    = 137, // Broadcom VideoCore III processor
+  EM_LATTICEMICO32 = 138, // RISC processor for Lattice FPGA architecture
+  EM_SE_C17        = 139, // Seiko Epson C17 family
+  EM_TI_C6000      = 140, // The Texas Instruments TMS320C6000 DSP family
+  EM_TI_C2000      = 141, // The Texas Instruments TMS320C2000 DSP family
+  EM_TI_C5500      = 142, // The Texas Instruments TMS320C55x DSP family
+  EM_MMDSP_PLUS    = 160, // STMicroelectronics 64bit VLIW Data Signal Processor
+  EM_CYPRESS_M8C   = 161, // Cypress M8C microprocessor
+  EM_R32C          = 162, // Renesas R32C series microprocessors
+  EM_TRIMEDIA      = 163, // NXP Semiconductors TriMedia architecture family
+  EM_QDSP6         = 164, // QUALCOMM DSP6 Processor
+  EM_8051          = 165, // Intel 8051 and variants
+  EM_STXP7X        = 166, // STMicroelectronics STxP7x family of configurable
+                          // and extensible RISC processors
+  EM_NDS32         = 167, // Andes Technology compact code size embedded RISC
+                          // processor family
+  EM_ECOG1         = 168, // Cyan Technology eCOG1X family
+  EM_ECOG1X        = 168, // Cyan Technology eCOG1X family
+  EM_MAXQ30        = 169, // Dallas Semiconductor MAXQ30 Core Micro-controllers
+  EM_XIMO16        = 170, // New Japan Radio (NJR) 16-bit DSP Processor
+  EM_MANIK         = 171, // M2000 Reconfigurable RISC Microprocessor
+  EM_CRAYNV2       = 172, // Cray Inc. NV2 vector architecture
+  EM_RX            = 173, // Renesas RX family
+  EM_METAG         = 174, // Imagination Technologies META processor
+                          // architecture
+  EM_MCST_ELBRUS   = 175, // MCST Elbrus general purpose hardware architecture
+  EM_ECOG16        = 176, // Cyan Technology eCOG16 family
+  EM_CR16          = 177, // National Semiconductor CompactRISC CR16 16-bit
+                          // microprocessor
+  EM_ETPU          = 178, // Freescale Extended Time Processing Unit
+  EM_SLE9X         = 179, // Infineon Technologies SLE9X core
+  EM_L10M          = 180, // Intel L10M
+  EM_K10M          = 181, // Intel K10M
+  EM_AVR32         = 185, // Atmel Corporation 32-bit microprocessor family
+  EM_STM8          = 186, // STMicroeletronics STM8 8-bit microcontroller
+  EM_TILE64        = 187, // Tilera TILE64 multicore architecture family
+  EM_TILEPRO       = 188, // Tilera TILEPro multicore architecture family
+  EM_MICROBLAZE    = 189, // Xilinx MicroBlaze 32-bit RISC soft processor core
+  EM_CUDA          = 190, // NVIDIA CUDA architecture
+  EM_TILEGX        = 191, // Tilera TILE-Gx multicore architecture family
+  EM_CLOUDSHIELD   = 192, // CloudShield architecture family
+  EM_COREA_1ST     = 193, // KIPO-KAIST Core-A 1st generation processor family
+  EM_COREA_2ND     = 194, // KIPO-KAIST Core-A 2nd generation processor family
+  EM_ARC_COMPACT2  = 195, // Synopsys ARCompact V2
+  EM_OPEN8         = 196, // Open8 8-bit RISC soft processor core
+  EM_RL78          = 197, // Renesas RL78 family
+  EM_VIDEOCORE5    = 198, // Broadcom VideoCore V processor
+  EM_78KOR         = 199, // Renesas 78KOR family
+  EM_56800EX       = 200, // Freescale 56800EX Digital Signal Controller (DSC)
+  EM_MBLAZE        = 47787 // Xilinx MicroBlaze
 };
 
 // Object file classes.
@@ -211,6 +357,11 @@ enum {
   R_X86_64_PC64       = 24,
   R_X86_64_GOTOFF64   = 25,
   R_X86_64_GOTPC32    = 26,
+  R_X86_64_GOT64      = 27,
+  R_X86_64_GOTPCREL64 = 28,
+  R_X86_64_GOTPC64    = 29,
+  R_X86_64_GOTPLT64   = 30,
+  R_X86_64_PLTOFF64   = 31,
   R_X86_64_SIZE32     = 32,
   R_X86_64_SIZE64     = 33,
   R_X86_64_GOTPC32_TLSDESC = 34,
@@ -290,6 +441,23 @@ enum {
   R_MICROBLAZE_COPY           = 21
 };
 
+enum {
+  R_PPC_NONE                  = 0,      /* No relocation. */
+  R_PPC_ADDR32                = 1,
+  R_PPC_ADDR24                = 2,
+  R_PPC_ADDR16                = 3,
+  R_PPC_ADDR16_LO             = 4,
+  R_PPC_ADDR16_HI             = 5,
+  R_PPC_ADDR16_HA             = 6,
+  R_PPC_ADDR14                = 7,
+  R_PPC_ADDR14_BRTAKEN        = 8,
+  R_PPC_ADDR14_BRNTAKEN       = 9,
+  R_PPC_REL24                 = 10,
+  R_PPC_REL14                 = 11,
+  R_PPC_REL14_BRTAKEN         = 12,
+  R_PPC_REL14_BRNTAKEN        = 13,
+  R_PPC_REL32                 = 26
+};
 
 // ARM Specific e_flags
 enum { EF_ARM_EABIMASK = 0xFF000000U };
@@ -431,7 +599,61 @@ enum {
   R_ARM_THM_TLS_DESCSEQ32     = 0x82
 };
 
-
+// ELF Relocation types for Mips
+enum {
+  R_MIPS_NONE              =  0,
+  R_MIPS_16                =  1,
+  R_MIPS_32                =  2,
+  R_MIPS_REL32             =  3,
+  R_MIPS_26                =  4,
+  R_MIPS_HI16              =  5,
+  R_MIPS_LO16              =  6,
+  R_MIPS_GPREL16           =  7,
+  R_MIPS_LITERAL           =  8,
+  R_MIPS_GOT16             =  9,
+  R_MIPS_PC16              = 10,
+  R_MIPS_CALL16            = 11,
+  R_MIPS_GPREL32           = 12,
+  R_MIPS_SHIFT5            = 16,
+  R_MIPS_SHIFT6            = 17,
+  R_MIPS_64                = 18,
+  R_MIPS_GOT_DISP          = 19,
+  R_MIPS_GOT_PAGE          = 20,
+  R_MIPS_GOT_OFST          = 21,
+  R_MIPS_GOT_HI16          = 22,
+  R_MIPS_GOT_LO16          = 23,
+  R_MIPS_SUB               = 24,
+  R_MIPS_INSERT_A          = 25,
+  R_MIPS_INSERT_B          = 26,
+  R_MIPS_DELETE            = 27,
+  R_MIPS_HIGHER            = 28,
+  R_MIPS_HIGHEST           = 29,
+  R_MIPS_CALL_HI16         = 30,
+  R_MIPS_CALL_LO16         = 31,
+  R_MIPS_SCN_DISP          = 32,
+  R_MIPS_REL16             = 33,
+  R_MIPS_ADD_IMMEDIATE     = 34,
+  R_MIPS_PJUMP             = 35,
+  R_MIPS_RELGOT            = 36,
+  R_MIPS_JALR              = 37,
+  R_MIPS_TLS_DTPMOD32      = 38,
+  R_MIPS_TLS_DTPREL32      = 39,
+  R_MIPS_TLS_DTPMOD64      = 40,
+  R_MIPS_TLS_DTPREL64      = 41,
+  R_MIPS_TLS_GD            = 42,
+  R_MIPS_TLS_LDM           = 43,
+  R_MIPS_TLS_DTPREL_HI16   = 44,
+  R_MIPS_TLS_DTPREL_LO16   = 45,
+  R_MIPS_TLS_GOTTPREL      = 46,
+  R_MIPS_TLS_TPREL32       = 47,
+  R_MIPS_TLS_TPREL64       = 48,
+  R_MIPS_TLS_TPREL_HI16    = 49,
+  R_MIPS_TLS_TPREL_LO16    = 50,
+  R_MIPS_GLOB_DAT          = 51,
+  R_MIPS_COPY              = 126,
+  R_MIPS_JUMP_SLOT         = 127,
+  R_MIPS_NUM               = 218
+};
 
 // Section header.
 struct Elf32_Shdr {
@@ -449,14 +671,14 @@ struct Elf32_Shdr {
 
 // Section header for ELF64 - same fields as ELF32, different types.
 struct Elf64_Shdr {
-  Elf64_Half  sh_name;
-  Elf64_Half  sh_type;
+  Elf64_Word  sh_name;
+  Elf64_Word  sh_type;
   Elf64_Xword sh_flags;
   Elf64_Addr  sh_addr;
   Elf64_Off   sh_offset;
   Elf64_Xword sh_size;
-  Elf64_Half  sh_link;
-  Elf64_Half  sh_info;
+  Elf64_Word  sh_link;
+  Elf64_Word  sh_info;
   Elf64_Xword sh_addralign;
   Elf64_Xword sh_entsize;
 };
@@ -467,6 +689,8 @@ enum {
   SHN_LORESERVE = 0xff00, // Lowest reserved index
   SHN_LOPROC    = 0xff00, // Lowest processor-specific index
   SHN_HIPROC    = 0xff1f, // Highest processor-specific index
+  SHN_LOOS      = 0xff20, // Lowest operating system-specific index
+  SHN_HIOS      = 0xff3f, // Highest operating system-specific index
   SHN_ABS       = 0xfff1, // Symbol has absolute value; does not need relocation
   SHN_COMMON    = 0xfff2, // FORTRAN COMMON or C external global variables
   SHN_XINDEX    = 0xffff, // Mark that the index is >= SHN_LORESERVE
@@ -557,8 +781,19 @@ enum {
   /// set to the start of the section by the boot code.
   XCORE_SHF_DP_SECTION = 0x1000U,
 
+  SHF_MASKOS   = 0x0ff00000,
+
   // Bits indicating processor-specific flags.
-  SHF_MASKPROC = 0xf0000000
+  SHF_MASKPROC = 0xf0000000,
+
+  // If an object file section does not have this flag set, then it may not hold
+  // more than 2GB and can be freely referred to in objects using smaller code
+  // models. Otherwise, only objects using larger code models can refer to them.
+  // For example, a medium code model object can refer to data in a section that
+  // sets this flag besides being able to refer to data in a section that does
+  // not set it; likewise, a small code model object can refer only to code in a
+  // section that does not set this flag.
+  SHF_X86_64_LARGE = 0x10000000
 };
 
 // Section Group Flags
@@ -619,6 +854,8 @@ enum {
   STB_LOCAL = 0,   // Local symbol, not visible outside obj file containing def
   STB_GLOBAL = 1,  // Global symbol, visible to all object files being combined
   STB_WEAK = 2,    // Weak symbol, like global but lower-precedence
+  STB_LOOS   = 10, // Lowest operating system-specific binding type
+  STB_HIOS   = 12, // Highest operating system-specific binding type
   STB_LOPROC = 13, // Lowest processor-specific binding type
   STB_HIPROC = 15  // Highest processor-specific binding type
 };
@@ -632,6 +869,8 @@ enum {
   STT_FILE    = 4,   // Local, absolute symbol that refers to a file
   STT_COMMON  = 5,   // An uninitialized common block
   STT_TLS     = 6,   // Thread local data object
+  STT_LOOS    = 7,   // Lowest operating system-specific symbol type
+  STT_HIOS    = 8,   // Highest operating system-specific symbol type
   STT_LOPROC  = 13,  // Lowest processor-specific symbol type
   STT_HIPROC  = 15   // Highest processor-specific symbol type
 };
@@ -746,6 +985,16 @@ enum {
   PT_NOTE    = 4, // Auxiliary information.
   PT_SHLIB   = 5, // Reserved.
   PT_PHDR    = 6, // The program header table itself.
+  PT_TLS     = 7, // The thread-local storage template.
+  PT_LOOS    = 0x60000000, // Lowest operating system-specific pt entry type.
+
+  // x86-64 program header types.
+  // These all contain stack unwind tables.
+  PT_GNU_EH_FRAME  = 0x6474e550,
+  PT_SUNW_EH_FRAME = 0x6474e550,
+  PT_SUNW_UNWIND   = 0x6464e550,
+
+  PT_HIOS    = 0x6fffffff, // Highest operating system-specific pt entry type.
   PT_LOPROC  = 0x70000000, // Lowest processor-specific program hdr entry type.
   PT_HIPROC  = 0x7fffffff  // Highest processor-specific program hdr entry type.
 };
@@ -755,7 +1004,8 @@ enum {
   PF_X        = 1,         // Execute
   PF_W        = 2,         // Write
   PF_R        = 4,         // Read
-  PF_MASKPROC = 0xf0000000 // Unspecified
+  PF_MASKOS   = 0x0ff00000,// Bits for operating system-specific semantics.
+  PF_MASKPROC = 0xf0000000 // Bits for processor-specific semantics.
 };
 
 // Dynamic table entry for ELF32.
@@ -811,12 +1061,29 @@ enum {
   DT_FINI_ARRAY   = 26,       // Pointer to array of termination functions.
   DT_INIT_ARRAYSZ = 27,       // Size of DT_INIT_ARRAY.
   DT_FINI_ARRAYSZ = 28,       // Size of DT_FINI_ARRAY.
+  DT_RUNPATH      = 29,       // String table offset of lib search path.
+  DT_FLAGS        = 30,       // Flags.
+  DT_ENCODING     = 32,       // Values from here to DT_LOOS follow the rules
+                              // for the interpretation of the d_un union.
+
+  DT_PREINIT_ARRAY = 32,      // Pointer to array of preinit functions.
+  DT_PREINIT_ARRAYSZ = 33,    // Size of the DT_PREINIT_ARRAY array.
+
   DT_LOOS         = 0x60000000, // Start of environment specific tags.
   DT_HIOS         = 0x6FFFFFFF, // End of environment specific tags.
   DT_LOPROC       = 0x70000000, // Start of processor specific tags.
   DT_HIPROC       = 0x7FFFFFFF  // End of processor specific tags.
 };
 
+// DT_FLAGS values.
+enum {
+  DF_ORIGIN     = 0x01, // The object may reference $ORIGIN.
+  DF_SYMBOLIC   = 0x02, // Search the shared lib before searching the exe.
+  DF_TEXTREL    = 0x04, // Relocations may modify a non-writable segment.
+  DF_BIND_NOW   = 0x08, // Process all relocations on load.
+  DF_STATIC_TLS = 0x10  // Reject attempts to load dynamically.
+};
+
 } // end namespace ELF
 
 } // end namespace llvm
diff --git a/contrib/llvm/include/llvm/Support/FileSystem.h b/contrib/llvm/include/llvm/Support/FileSystem.h
index 4f013f89e86c..a868e5f9f70b 100644
--- a/contrib/llvm/include/llvm/Support/FileSystem.h
+++ b/contrib/llvm/include/llvm/Support/FileSystem.h
@@ -201,30 +201,6 @@ error_code rename(const Twine &from, const Twine &to);
 ///          platform specific error_code.
 error_code resize_file(const Twine &path, uint64_t size);
 
-/// @brief Make file readable.
-///
-/// @param path Input path.
-/// @param value If true, make readable, else, make unreadable.
-/// @results errc::success if readability has been successfully set, otherwise a
-///          platform specific error_code.
-error_code set_read(const Twine &path, bool value);
-
-/// @brief Make file writeable.
-///
-/// @param path Input path.
-/// @param value If true, make writeable, else, make unwriteable.
-/// @results errc::success if writeability has been successfully set, otherwise
-///          a platform specific error_code.
-error_code set_write(const Twine &path, bool value);
-
-/// @brief Make file executable.
-///
-/// @param path Input path.
-/// @param value If true, make executable, else, make unexecutable.
-/// @results errc::success if executability has been successfully set, otherwise
-///          a platform specific error_code.
-error_code set_execute(const Twine &path, bool value);
-
 /// @}
 /// @name Physical Observers
 /// @{
@@ -245,6 +221,13 @@ bool exists(file_status status);
 ///          platform specific error_code.
 error_code exists(const Twine &path, bool &result);
 
+/// @brief Simpler version of exists for clients that don't need to
+///        differentiate between an error and false.
+inline bool exists(const Twine &path) {
+  bool result;
+  return !exists(path, result) && result;
+}
+
 /// @brief Do file_status's represent the same thing?
 ///
 /// @param A Input file_status.
@@ -289,15 +272,6 @@ bool is_directory(file_status status);
 ///          platform specific error_code.
 error_code is_directory(const Twine &path, bool &result);
 
-/// @brief Is path an empty file?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is a an empty file, false if it is not.
-///               Undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code is_empty(const Twine &path, bool &result);
-
 /// @brief Does status represent a regular file?
 ///
 /// @param status A file_status previously returned from status.
@@ -346,40 +320,6 @@ bool is_symlink(file_status status);
 ///          platform specific error_code.
 error_code is_symlink(const Twine &path, bool &result);
 
-/// @brief Get last write time without changing it.
-///
-/// @param path Input path.
-/// @param result Set to the last write time (UNIX time) of \a path if it
-///               exists.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code last_write_time(const Twine &path, std::time_t &result);
-
-/// @brief Set last write time.
-///
-/// @param path Input path.
-/// @param value Time to set (UNIX time) \a path's last write time to.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code set_last_write_time(const Twine &path, std::time_t value);
-
-/// @brief Read a symlink's value.
-///
-/// @param path Input path.
-/// @param result Set to the value of the symbolic link \a path.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code read_symlink(const Twine &path, SmallVectorImpl<char> &result);
-
-/// @brief Get disk space usage information.
-///
-/// @param path Input path.
-/// @param result Set to the capacity, free, and available space on the device
-///               \a path is on.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code disk_space(const Twine &path, space_info &result);
-
 /// @brief Get file status as if by POSIX stat().
 ///
 /// @param path Input path.
@@ -402,16 +342,6 @@ bool status_known(file_status s);
 ///          platform specific error_code.
 error_code status_known(const Twine &path, bool &result);
 
-/// @brief Get file status as if by POSIX lstat().
-///
-/// Does not resolve symlinks.
-///
-/// @param path Input path.
-/// @param result Set to the file status.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code symlink_status(const Twine &path, file_status &result);
-
 /// @brief Generate a unique path and open it as a file.
 ///
 /// Generates a unique path suitable for a temporary file and then opens it as a
@@ -427,10 +357,13 @@ error_code symlink_status(const Twine &path, file_status &result);
 /// @param model Name to base unique path off of.
 /// @param result_fs Set to the opened file's file descriptor.
 /// @param result_path Set to the opened file's absolute path.
+/// @param makeAbsolute If true and @model is not an absolute path, a temp
+///        directory will be prepended.
 /// @results errc::success if result_{fd,path} have been successfully set,
 ///          otherwise a platform specific error_code.
 error_code unique_file(const Twine &model, int &result_fd,
-                             SmallVectorImpl<char> &result_path);
+                             SmallVectorImpl<char> &result_path,
+                             bool makeAbsolute = true);
 
 /// @brief Canonicalize path.
 ///
@@ -472,60 +405,6 @@ error_code get_magic(const Twine &path, uint32_t len,
 ///          platform specific error_code.
 error_code identify_magic(const Twine &path, LLVMFileType &result);
 
-/// @brief Is file bitcode?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is a bitcode file, false if it is not,
-///               undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code is_bitcode(const Twine &path, bool &result);
-
-/// @brief Is file a dynamic library?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is a dynamic library, false if it is
-///               not, undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code is_dynamic_library(const Twine &path, bool &result);
-
-/// @brief Is an object file?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is an object file, false if it is not,
-///               undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code is_object_file(const Twine &path, bool &result);
-
-/// @brief Can file be read?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is readable, false it it is not,
-///               undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code can_read(const Twine &path, bool &result);
-
-/// @brief Can file be written?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is writeable, false it it is not,
-///               undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code can_write(const Twine &path, bool &result);
-
-/// @brief Can file be executed?
-///
-/// @param path Input path.
-/// @param result Set to true if \a path is executable, false it it is not,
-///               undefined otherwise.
-/// @results errc::success if result has been successfully set, otherwise a
-///          platform specific error_code.
-error_code can_execute(const Twine &path, bool &result);
-
 /// @brief Get library paths the system linker uses.
 ///
 /// @param result Set to the list of system library paths.
@@ -569,35 +448,28 @@ error_code GetMainExecutable(const char *argv0, void *MainAddr,
 /// @{
 
 /// directory_entry - A single entry in a directory. Caches the status either
-/// from the result of the iteration syscall, or the first time status or
-/// symlink_status is called.
+/// from the result of the iteration syscall, or the first time status is
+/// called.
 class directory_entry {
   std::string Path;
   mutable file_status Status;
-  mutable file_status SymlinkStatus;
 
 public:
-  explicit directory_entry(const Twine &path, file_status st = file_status(),
-                                       file_status symlink_st = file_status())
+  explicit directory_entry(const Twine &path, file_status st = file_status())
     : Path(path.str())
-    , Status(st)
-    , SymlinkStatus(symlink_st) {}
+    , Status(st) {}
 
   directory_entry() {}
 
-  void assign(const Twine &path, file_status st = file_status(),
-                          file_status symlink_st = file_status()) {
+  void assign(const Twine &path, file_status st = file_status()) {
     Path = path.str();
     Status = st;
-    SymlinkStatus = symlink_st;
   }
 
-  void replace_filename(const Twine &filename, file_status st = file_status(),
-                              file_status symlink_st = file_status());
+  void replace_filename(const Twine &filename, file_status st = file_status());
 
   const std::string &path() const { return Path; }
   error_code status(file_status &result) const;
-  error_code symlink_status(file_status &result) const;
 
   bool operator==(const directory_entry& rhs) const { return Path == rhs.Path; }
   bool operator!=(const directory_entry& rhs) const { return !(*this == rhs); }
diff --git a/contrib/llvm/include/llvm/Support/Format.h b/contrib/llvm/include/llvm/Support/Format.h
index f64e3db7d650..59812d98f589 100644
--- a/contrib/llvm/include/llvm/Support/Format.h
+++ b/contrib/llvm/include/llvm/Support/Format.h
@@ -126,6 +126,50 @@ class format_object3 : public format_object_base {
   }
 };
 
+/// format_object4 - This is a templated helper class used by the format
+/// function that captures the object to be formated and the format string. When
+/// actually printed, this synthesizes the string into a temporary buffer
+/// provided and returns whether or not it is big enough.
+template <typename T1, typename T2, typename T3, typename T4>
+class format_object4 : public format_object_base {
+  T1 Val1;
+  T2 Val2;
+  T3 Val3;
+  T4 Val4;
+public:
+  format_object4(const char *fmt, const T1 &val1, const T2 &val2,
+                 const T3 &val3, const T4 &val4)
+    : format_object_base(fmt), Val1(val1), Val2(val2), Val3(val3), Val4(val4) {
+  }
+
+  virtual int snprint(char *Buffer, unsigned BufferSize) const {
+    return snprintf(Buffer, BufferSize, Fmt, Val1, Val2, Val3, Val4);
+  }
+};
+
+/// format_object5 - This is a templated helper class used by the format
+/// function that captures the object to be formated and the format string. When
+/// actually printed, this synthesizes the string into a temporary buffer
+/// provided and returns whether or not it is big enough.
+template <typename T1, typename T2, typename T3, typename T4, typename T5>
+class format_object5 : public format_object_base {
+  T1 Val1;
+  T2 Val2;
+  T3 Val3;
+  T4 Val4;
+  T5 Val5;
+public:
+  format_object5(const char *fmt, const T1 &val1, const T2 &val2,
+                 const T3 &val3, const T4 &val4, const T5 &val5)
+    : format_object_base(fmt), Val1(val1), Val2(val2), Val3(val3), Val4(val4),
+      Val5(val5) {
+  }
+
+  virtual int snprint(char *Buffer, unsigned BufferSize) const {
+    return snprintf(Buffer, BufferSize, Fmt, Val1, Val2, Val3, Val4, Val5);
+  }
+};
+
 /// format - This is a helper function that is used to produce formatted output.
 /// This is typically used like:  OS << format("%0.4f", myfloat) << '\n';
 template <typename T>
@@ -149,6 +193,24 @@ template <typename T1, typename T2, typename T3>
   return format_object3<T1, T2, T3>(Fmt, Val1, Val2, Val3);
 }
 
+/// format - This is a helper function that is used to produce formatted output.
+/// This is typically used like:  OS << format("%0.4f", myfloat) << '\n';
+template <typename T1, typename T2, typename T3, typename T4>
+inline format_object4<T1, T2, T3, T4> format(const char *Fmt, const T1 &Val1,
+                                             const T2 &Val2, const T3 &Val3,
+                                             const T4 &Val4) {
+  return format_object4<T1, T2, T3, T4>(Fmt, Val1, Val2, Val3, Val4);
+}
+
+/// format - This is a helper function that is used to produce formatted output.
+/// This is typically used like:  OS << format("%0.4f", myfloat) << '\n';
+template <typename T1, typename T2, typename T3, typename T4, typename T5>
+inline format_object5<T1, T2, T3, T4, T5> format(const char *Fmt,const T1 &Val1,
+                                             const T2 &Val2, const T3 &Val3,
+                                             const T4 &Val4, const T5 &Val5) {
+  return format_object5<T1, T2, T3, T4, T5>(Fmt, Val1, Val2, Val3, Val4, Val5);
+}
+
 } // end namespace llvm
 
 #endif
diff --git a/contrib/llvm/include/llvm/Support/GCOV.h b/contrib/llvm/include/llvm/Support/GCOV.h
new file mode 100644
index 000000000000..49cd87fc7b44
--- /dev/null
+++ b/contrib/llvm/include/llvm/Support/GCOV.h
@@ -0,0 +1,224 @@
+//===-- llvm/Support/GCOV.h - LLVM coverage tool ----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This header provides the interface to read and write coverage files that 
+// use 'gcov' format.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_GCOV_H
+#define LLVM_GCOV_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+
+class GCOVFunction;
+class GCOVBlock;
+class GCOVLines;
+class FileInfo;
+
+enum GCOVFormat {
+  InvalidGCOV,
+  GCNO_402,
+  GCNO_404,
+  GCDA_402,
+  GCDA_404
+};
+
+/// GCOVBuffer - A wrapper around MemoryBuffer to provide GCOV specific
+/// read operations.
+class GCOVBuffer {
+public:
+  GCOVBuffer(MemoryBuffer *B) : Buffer(B), Cursor(0) {}
+  
+  /// readGCOVFormat - Read GCOV signature at the beginning of buffer.
+  enum GCOVFormat readGCOVFormat() {
+    StringRef Magic = Buffer->getBuffer().slice(0, 12);
+    Cursor = 12;
+    if (Magic == "oncg*404MVLL")
+      return GCNO_404;
+    else if (Magic == "oncg*204MVLL")
+      return GCNO_402;
+    else if (Magic == "adcg*404MVLL")
+      return GCDA_404;
+    else if (Magic == "adcg*204MVLL")
+      return GCDA_402;
+    
+    Cursor = 0;
+    return InvalidGCOV;
+  }
+
+  /// readFunctionTag - If cursor points to a function tag then increment the
+  /// cursor and return true otherwise return false.
+  bool readFunctionTag() {
+    StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor+4);
+    if (Tag.empty() || 
+	Tag[0] != '\0' || Tag[1] != '\0' ||
+	Tag[2] != '\0' || Tag[3] != '\1') {
+      return false;
+    }
+    Cursor += 4;
+    return true;
+  }
+
+  /// readBlockTag - If cursor points to a block tag then increment the
+  /// cursor and return true otherwise return false.
+  bool readBlockTag() {
+    StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor+4);
+    if (Tag.empty() || 
+	Tag[0] != '\0' || Tag[1] != '\0' ||
+	Tag[2] != '\x41' || Tag[3] != '\x01') {
+      return false;
+    }
+    Cursor += 4;
+    return true;
+  }
+
+  /// readEdgeTag - If cursor points to an edge tag then increment the
+  /// cursor and return true otherwise return false.
+  bool readEdgeTag() {
+    StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor+4);
+    if (Tag.empty() || 
+	Tag[0] != '\0' || Tag[1] != '\0' ||
+	Tag[2] != '\x43' || Tag[3] != '\x01') {
+      return false;
+    }
+    Cursor += 4;
+    return true;
+  }
+
+  /// readLineTag - If cursor points to a line tag then increment the
+  /// cursor and return true otherwise return false.
+  bool readLineTag() {
+    StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor+4);
+    if (Tag.empty() || 
+	Tag[0] != '\0' || Tag[1] != '\0' ||
+	Tag[2] != '\x45' || Tag[3] != '\x01') {
+      return false;
+    }
+    Cursor += 4;
+    return true;
+  }
+
+  /// readArcTag - If cursor points to an gcda arc tag then increment the
+  /// cursor and return true otherwise return false.
+  bool readArcTag() {
+    StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor+4);
+    if (Tag.empty() || 
+	Tag[0] != '\0' || Tag[1] != '\0' ||
+	Tag[2] != '\xa1' || Tag[3] != '\1') {
+      return false;
+    }
+    Cursor += 4;
+    return true;
+  }
+
+  uint32_t readInt() {
+    uint32_t Result;
+    StringRef Str = Buffer->getBuffer().slice(Cursor, Cursor+4);
+    assert (Str.empty() == false && "Unexpected memory buffer end!");
+    Cursor += 4;
+    Result = *(uint32_t *)(Str.data());
+    return Result;
+  }
+
+  uint64_t readInt64() {
+    uint64_t Lo = readInt();
+    uint64_t Hi = readInt();
+    uint64_t Result = Lo | (Hi << 32);
+    return Result;
+  }
+
+  StringRef readString() {
+    uint32_t Len = readInt() * 4;
+    StringRef Str = Buffer->getBuffer().slice(Cursor, Cursor+Len);
+    Cursor += Len;
+    return Str;
+  }
+
+  uint64_t getCursor() const { return Cursor; }
+private:
+  MemoryBuffer *Buffer;
+  uint64_t Cursor;
+};
+
+/// GCOVFile - Collects coverage information for one pair of coverage file
+/// (.gcno and .gcda).
+class GCOVFile {
+public:
+  GCOVFile() {}
+  ~GCOVFile();
+  bool read(GCOVBuffer &Buffer);
+  void dump();
+  void collectLineCounts(FileInfo &FI);
+private:
+  SmallVector<GCOVFunction *, 16> Functions;
+};
+
+/// GCOVFunction - Collects function information.
+class GCOVFunction {
+public:
+  GCOVFunction() : Ident(0), LineNumber(0) {}
+  ~GCOVFunction();
+  bool read(GCOVBuffer &Buffer, GCOVFormat Format);
+  void dump();
+  void collectLineCounts(FileInfo &FI);
+private:
+  uint32_t Ident;
+  uint32_t LineNumber;
+  StringRef Name;
+  StringRef Filename;
+  SmallVector<GCOVBlock *, 16> Blocks;
+};
+
+/// GCOVBlock - Collects block information.
+class GCOVBlock {
+public:
+  GCOVBlock(uint32_t N) : Number(N), Counter(0) {}
+  ~GCOVBlock();
+  void addEdge(uint32_t N) { Edges.push_back(N); }
+  void addLine(StringRef Filename, uint32_t LineNo);
+  void addCount(uint64_t N) { Counter = N; }
+  void dump();
+  void collectLineCounts(FileInfo &FI);
+private:
+  uint32_t Number;
+  uint64_t Counter;
+  SmallVector<uint32_t, 16> Edges;
+  StringMap<GCOVLines *> Lines;
+};
+
+/// GCOVLines - A wrapper around a vector of int to keep track of line nos.
+class GCOVLines {
+public:
+  ~GCOVLines() { Lines.clear(); }
+  void add(uint32_t N) { Lines.push_back(N); }
+  void collectLineCounts(FileInfo &FI, StringRef Filename, uint32_t Count);
+  void dump();
+
+private:
+  SmallVector<uint32_t, 4> Lines;
+};
+
+typedef SmallVector<uint32_t, 16> LineCounts;
+class FileInfo {
+public:
+  void addLineCount(StringRef Filename, uint32_t Line, uint32_t Count);
+  void print();
+private:
+  StringMap<LineCounts> LineInfo;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/include/llvm/Support/GetElementPtrTypeIterator.h b/contrib/llvm/include/llvm/Support/GetElementPtrTypeIterator.h
index e5e7fc74095d..ef92c95ee7e0 100644
--- a/contrib/llvm/include/llvm/Support/GetElementPtrTypeIterator.h
+++ b/contrib/llvm/include/llvm/Support/GetElementPtrTypeIterator.h
@@ -21,16 +21,16 @@
 namespace llvm {
   template<typename ItTy = User::const_op_iterator>
   class generic_gep_type_iterator
-    : public std::iterator<std::forward_iterator_tag, const Type *, ptrdiff_t> {
+    : public std::iterator<std::forward_iterator_tag, Type *, ptrdiff_t> {
     typedef std::iterator<std::forward_iterator_tag,
-                          const Type *, ptrdiff_t> super;
+                          Type *, ptrdiff_t> super;
 
     ItTy OpIt;
-    const Type *CurTy;
+    Type *CurTy;
     generic_gep_type_iterator() {}
   public:
 
-    static generic_gep_type_iterator begin(const Type *Ty, ItTy It) {
+    static generic_gep_type_iterator begin(Type *Ty, ItTy It) {
       generic_gep_type_iterator I;
       I.CurTy = Ty;
       I.OpIt = It;
@@ -50,23 +50,23 @@ namespace llvm {
       return !operator==(x);
     }
 
-    const Type *operator*() const {
+    Type *operator*() const {
       return CurTy;
     }
 
-    const Type *getIndexedType() const {
-      const CompositeType *CT = cast<CompositeType>(CurTy);
+    Type *getIndexedType() const {
+      CompositeType *CT = cast<CompositeType>(CurTy);
       return CT->getTypeAtIndex(getOperand());
     }
 
     // This is a non-standard operator->.  It allows you to call methods on the
     // current type directly.
-    const Type *operator->() const { return operator*(); }
+    Type *operator->() const { return operator*(); }
 
     Value *getOperand() const { return *OpIt; }
 
     generic_gep_type_iterator& operator++() {   // Preincrement
-      if (const CompositeType *CT = dyn_cast<CompositeType>(CurTy)) {
+      if (CompositeType *CT = dyn_cast<CompositeType>(CurTy)) {
         CurTy = CT->getTypeAtIndex(getOperand());
       } else {
         CurTy = 0;
@@ -97,16 +97,16 @@ namespace llvm {
     return gep_type_iterator::end(GEP.op_end());
   }
 
-  template<typename ItTy>
-  inline generic_gep_type_iterator<ItTy>
-  gep_type_begin(const Type *Op0, ItTy I, ItTy E) {
-    return generic_gep_type_iterator<ItTy>::begin(Op0, I);
+  template<typename T>
+  inline generic_gep_type_iterator<const T *>
+  gep_type_begin(Type *Op0, ArrayRef<T> A) {
+    return generic_gep_type_iterator<const T *>::begin(Op0, A.begin());
   }
 
-  template<typename ItTy>
-  inline generic_gep_type_iterator<ItTy>
-  gep_type_end(const Type *Op0, ItTy I, ItTy E) {
-    return generic_gep_type_iterator<ItTy>::end(E);
+  template<typename T>
+  inline generic_gep_type_iterator<const T *>
+  gep_type_end(Type *Op0, ArrayRef<T> A) {
+    return generic_gep_type_iterator<const T *>::end(A.end());
   }
 } // end namespace llvm
 
diff --git a/contrib/llvm/include/llvm/Support/IRBuilder.h b/contrib/llvm/include/llvm/Support/IRBuilder.h
index 91cd78e7f634..782800173f4b 100644
--- a/contrib/llvm/include/llvm/Support/IRBuilder.h
+++ b/contrib/llvm/include/llvm/Support/IRBuilder.h
@@ -475,8 +475,8 @@ class IRBuilder : public IRBuilderBase, public Inserter {
                   Name);
   }
 
-  UnwindInst *CreateUnwind() {
-    return Insert(new UnwindInst(Context));
+  ResumeInst *CreateResume(Value *Exn) {
+    return Insert(ResumeInst::Create(Exn));
   }
 
   UnreachableInst *CreateUnreachable() {
@@ -744,7 +744,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
   // Instruction creation methods: Memory Instructions
   //===--------------------------------------------------------------------===//
 
-  AllocaInst *CreateAlloca(const Type *Ty, Value *ArraySize = 0,
+  AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = 0,
                            const Twine &Name = "") {
     return Insert(new AllocaInst(Ty, ArraySize), Name);
   }
@@ -762,71 +762,74 @@ class IRBuilder : public IRBuilderBase, public Inserter {
   StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
     return Insert(new StoreInst(Val, Ptr, isVolatile));
   }
-  template<typename RandomAccessIterator>
-  Value *CreateGEP(Value *Ptr,
-                   RandomAccessIterator IdxBegin,
-                   RandomAccessIterator IdxEnd,
+  FenceInst *CreateFence(AtomicOrdering Ordering,
+                         SynchronizationScope SynchScope = CrossThread) {
+    return Insert(new FenceInst(Context, Ordering, SynchScope));
+  }
+  AtomicCmpXchgInst *CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
+                                         AtomicOrdering Ordering,
+                               SynchronizationScope SynchScope = CrossThread) {
+    return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, SynchScope));
+  }
+  AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
+                                 AtomicOrdering Ordering,
+                               SynchronizationScope SynchScope = CrossThread) {
+    return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SynchScope));
+  }
+  Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
                    const Twine &Name = "") {
     if (Constant *PC = dyn_cast<Constant>(Ptr)) {
       // Every index must be constant.
-      RandomAccessIterator i;
-      for (i = IdxBegin; i < IdxEnd; ++i)
-        if (!isa<Constant>(*i))
+      size_t i, e;
+      for (i = 0, e = IdxList.size(); i != e; ++i)
+        if (!isa<Constant>(IdxList[i]))
           break;
-      if (i == IdxEnd)
-        return Insert(Folder.CreateGetElementPtr(PC, &IdxBegin[0],
-                                                 IdxEnd - IdxBegin),
-                      Name);
+      if (i == e)
+        return Insert(Folder.CreateGetElementPtr(PC, IdxList), Name);
     }
-    return Insert(GetElementPtrInst::Create(Ptr, IdxBegin, IdxEnd), Name);
+    return Insert(GetElementPtrInst::Create(Ptr, IdxList), Name);
   }
-  template<typename RandomAccessIterator>
-  Value *CreateInBoundsGEP(Value *Ptr, RandomAccessIterator IdxBegin,
-                           RandomAccessIterator IdxEnd,
+  Value *CreateInBoundsGEP(Value *Ptr, ArrayRef<Value *> IdxList,
                            const Twine &Name = "") {
     if (Constant *PC = dyn_cast<Constant>(Ptr)) {
       // Every index must be constant.
-      RandomAccessIterator i;
-      for (i = IdxBegin; i < IdxEnd; ++i)
-        if (!isa<Constant>(*i))
+      size_t i, e;
+      for (i = 0, e = IdxList.size(); i != e; ++i)
+        if (!isa<Constant>(IdxList[i]))
           break;
-      if (i == IdxEnd)
-        return Insert(Folder.CreateInBoundsGetElementPtr(PC,
-                                                         &IdxBegin[0],
-                                                         IdxEnd - IdxBegin),
-                      Name);
+      if (i == e)
+        return Insert(Folder.CreateInBoundsGetElementPtr(PC, IdxList), Name);
     }
-    return Insert(GetElementPtrInst::CreateInBounds(Ptr, IdxBegin, IdxEnd),
-                  Name);
+    return Insert(GetElementPtrInst::CreateInBounds(Ptr, IdxList), Name);
   }
   Value *CreateGEP(Value *Ptr, Value *Idx, const Twine &Name = "") {
     if (Constant *PC = dyn_cast<Constant>(Ptr))
       if (Constant *IC = dyn_cast<Constant>(Idx))
-        return Insert(Folder.CreateGetElementPtr(PC, &IC, 1), Name);
+        return Insert(Folder.CreateGetElementPtr(PC, IC), Name);
     return Insert(GetElementPtrInst::Create(Ptr, Idx), Name);
   }
   Value *CreateInBoundsGEP(Value *Ptr, Value *Idx, const Twine &Name = "") {
     if (Constant *PC = dyn_cast<Constant>(Ptr))
       if (Constant *IC = dyn_cast<Constant>(Idx))
-        return Insert(Folder.CreateInBoundsGetElementPtr(PC, &IC, 1), Name);
+        return Insert(Folder.CreateInBoundsGetElementPtr(PC, IC), Name);
     return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idx), Name);
   }
   Value *CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name = "") {
     Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateGetElementPtr(PC, &Idx, 1), Name);
+      return Insert(Folder.CreateGetElementPtr(PC, Idx), Name);
 
-    return Insert(GetElementPtrInst::Create(Ptr, &Idx, &Idx+1), Name);
+    return Insert(GetElementPtrInst::Create(Ptr, Idx), Name);
   }
   Value *CreateConstInBoundsGEP1_32(Value *Ptr, unsigned Idx0,
                                     const Twine &Name = "") {
     Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateInBoundsGetElementPtr(PC, &Idx, 1), Name);
+      return Insert(Folder.CreateInBoundsGetElementPtr(PC, Idx), Name);
 
-    return Insert(GetElementPtrInst::CreateInBounds(Ptr, &Idx, &Idx+1), Name);
+    return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idx), Name);
   }
   Value *CreateConstGEP2_32(Value *Ptr, unsigned Idx0, unsigned Idx1,
                     const Twine &Name = "") {
@@ -836,9 +839,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     };
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateGetElementPtr(PC, Idxs, 2), Name);
+      return Insert(Folder.CreateGetElementPtr(PC, Idxs), Name);
 
-    return Insert(GetElementPtrInst::Create(Ptr, Idxs, Idxs+2), Name);
+    return Insert(GetElementPtrInst::Create(Ptr, Idxs), Name);
   }
   Value *CreateConstInBoundsGEP2_32(Value *Ptr, unsigned Idx0, unsigned Idx1,
                                     const Twine &Name = "") {
@@ -848,26 +851,26 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     };
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateInBoundsGetElementPtr(PC, Idxs, 2), Name);
+      return Insert(Folder.CreateInBoundsGetElementPtr(PC, Idxs), Name);
 
-    return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idxs, Idxs+2), Name);
+    return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idxs), Name);
   }
   Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") {
     Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateGetElementPtr(PC, &Idx, 1), Name);
+      return Insert(Folder.CreateGetElementPtr(PC, Idx), Name);
 
-    return Insert(GetElementPtrInst::Create(Ptr, &Idx, &Idx+1), Name);
+    return Insert(GetElementPtrInst::Create(Ptr, Idx), Name);
   }
   Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0,
                                     const Twine &Name = "") {
     Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateInBoundsGetElementPtr(PC, &Idx, 1), Name);
+      return Insert(Folder.CreateInBoundsGetElementPtr(PC, Idx), Name);
 
-    return Insert(GetElementPtrInst::CreateInBounds(Ptr, &Idx, &Idx+1), Name);
+    return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idx), Name);
   }
   Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
                     const Twine &Name = "") {
@@ -877,9 +880,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     };
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateGetElementPtr(PC, Idxs, 2), Name);
+      return Insert(Folder.CreateGetElementPtr(PC, Idxs), Name);
 
-    return Insert(GetElementPtrInst::Create(Ptr, Idxs, Idxs+2), Name);
+    return Insert(GetElementPtrInst::Create(Ptr, Idxs), Name);
   }
   Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
                                     const Twine &Name = "") {
@@ -889,9 +892,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     };
 
     if (Constant *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateInBoundsGetElementPtr(PC, Idxs, 2), Name);
+      return Insert(Folder.CreateInBoundsGetElementPtr(PC, Idxs), Name);
 
-    return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idxs, Idxs+2), Name);
+    return Insert(GetElementPtrInst::CreateInBounds(Ptr, Idxs), Name);
   }
   Value *CreateStructGEP(Value *Ptr, unsigned Idx, const Twine &Name = "") {
     return CreateConstInBoundsGEP2_32(Ptr, 0, Idx, Name);
@@ -903,54 +906,54 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     Value *gv = CreateGlobalString(Str, Name);
     Value *zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
     Value *Args[] = { zero, zero };
-    return CreateInBoundsGEP(gv, Args, Args+2, Name);
+    return CreateInBoundsGEP(gv, Args, Name);
   }
 
   //===--------------------------------------------------------------------===//
   // Instruction creation methods: Cast/Conversion Operators
   //===--------------------------------------------------------------------===//
 
-  Value *CreateTrunc(Value *V, const Type *DestTy, const Twine &Name = "") {
+  Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
     return CreateCast(Instruction::Trunc, V, DestTy, Name);
   }
-  Value *CreateZExt(Value *V, const Type *DestTy, const Twine &Name = "") {
+  Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") {
     return CreateCast(Instruction::ZExt, V, DestTy, Name);
   }
-  Value *CreateSExt(Value *V, const Type *DestTy, const Twine &Name = "") {
+  Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") {
     return CreateCast(Instruction::SExt, V, DestTy, Name);
   }
-  Value *CreateFPToUI(Value *V, const Type *DestTy, const Twine &Name = ""){
+  Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = ""){
     return CreateCast(Instruction::FPToUI, V, DestTy, Name);
   }
-  Value *CreateFPToSI(Value *V, const Type *DestTy, const Twine &Name = ""){
+  Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = ""){
     return CreateCast(Instruction::FPToSI, V, DestTy, Name);
   }
-  Value *CreateUIToFP(Value *V, const Type *DestTy, const Twine &Name = ""){
+  Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
     return CreateCast(Instruction::UIToFP, V, DestTy, Name);
   }
-  Value *CreateSIToFP(Value *V, const Type *DestTy, const Twine &Name = ""){
+  Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
     return CreateCast(Instruction::SIToFP, V, DestTy, Name);
   }
-  Value *CreateFPTrunc(Value *V, const Type *DestTy,
+  Value *CreateFPTrunc(Value *V, Type *DestTy,
                        const Twine &Name = "") {
     return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
   }
-  Value *CreateFPExt(Value *V, const Type *DestTy, const Twine &Name = "") {
+  Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
     return CreateCast(Instruction::FPExt, V, DestTy, Name);
   }
-  Value *CreatePtrToInt(Value *V, const Type *DestTy,
+  Value *CreatePtrToInt(Value *V, Type *DestTy,
                         const Twine &Name = "") {
     return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
   }
-  Value *CreateIntToPtr(Value *V, const Type *DestTy,
+  Value *CreateIntToPtr(Value *V, Type *DestTy,
                         const Twine &Name = "") {
     return CreateCast(Instruction::IntToPtr, V, DestTy, Name);
   }
-  Value *CreateBitCast(Value *V, const Type *DestTy,
+  Value *CreateBitCast(Value *V, Type *DestTy,
                        const Twine &Name = "") {
     return CreateCast(Instruction::BitCast, V, DestTy, Name);
   }
-  Value *CreateZExtOrBitCast(Value *V, const Type *DestTy,
+  Value *CreateZExtOrBitCast(Value *V, Type *DestTy,
                              const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
@@ -958,7 +961,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
       return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name);
     return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name);
   }
-  Value *CreateSExtOrBitCast(Value *V, const Type *DestTy,
+  Value *CreateSExtOrBitCast(Value *V, Type *DestTy,
                              const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
@@ -966,7 +969,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
       return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name);
     return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name);
   }
-  Value *CreateTruncOrBitCast(Value *V, const Type *DestTy,
+  Value *CreateTruncOrBitCast(Value *V, Type *DestTy,
                               const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
@@ -974,7 +977,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
       return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name);
     return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name);
   }
-  Value *CreateCast(Instruction::CastOps Op, Value *V, const Type *DestTy,
+  Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
                     const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
@@ -982,7 +985,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
       return Insert(Folder.CreateCast(Op, VC, DestTy), Name);
     return Insert(CastInst::Create(Op, V, DestTy), Name);
   }
-  Value *CreatePointerCast(Value *V, const Type *DestTy,
+  Value *CreatePointerCast(Value *V, Type *DestTy,
                            const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
@@ -990,7 +993,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
       return Insert(Folder.CreatePointerCast(VC, DestTy), Name);
     return Insert(CastInst::CreatePointerCast(V, DestTy), Name);
   }
-  Value *CreateIntCast(Value *V, const Type *DestTy, bool isSigned,
+  Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned,
                        const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
@@ -1001,9 +1004,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
 private:
   // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a compile time
   // error, instead of converting the string to bool for the isSigned parameter.
-  Value *CreateIntCast(Value *, const Type *, const char *); // DO NOT IMPLEMENT
+  Value *CreateIntCast(Value *, Type *, const char *); // DO NOT IMPLEMENT
 public:
-  Value *CreateFPCast(Value *V, const Type *DestTy, const Twine &Name = "") {
+  Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
     if (Constant *VC = dyn_cast<Constant>(V))
@@ -1108,7 +1111,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
   // Instruction creation methods: Other Instructions
   //===--------------------------------------------------------------------===//
 
-  PHINode *CreatePHI(const Type *Ty, unsigned NumReservedValues,
+  PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
                      const Twine &Name = "") {
     return Insert(PHINode::Create(Ty, NumReservedValues), Name);
   }
@@ -1142,7 +1145,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
 
   CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args,
                        const Twine &Name = "") {
-    return Insert(CallInst::Create(Callee, Args, Name));
+    return Insert(CallInst::Create(Callee, Args), Name);
   }
 
   Value *CreateSelect(Value *C, Value *True, Value *False,
@@ -1154,7 +1157,7 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(SelectInst::Create(C, True, False), Name);
   }
 
-  VAArgInst *CreateVAArg(Value *List, const Type *Ty, const Twine &Name = "") {
+  VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
     return Insert(new VAArgInst(List, Ty), Name);
   }
 
@@ -1201,6 +1204,11 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
   }
 
+  LandingPadInst *CreateLandingPad(Type *Ty, Value *PersFn, unsigned NumClauses,
+                                   const Twine &Name = "") {
+    return Insert(LandingPadInst::Create(Ty, PersFn, NumClauses, Name));
+  }
+
   //===--------------------------------------------------------------------===//
   // Utility creation methods
   //===--------------------------------------------------------------------===//
diff --git a/contrib/llvm/include/llvm/Support/InstVisitor.h b/contrib/llvm/include/llvm/Support/InstVisitor.h
index b2e5d58b7c34..a661c4fac68d 100644
--- a/contrib/llvm/include/llvm/Support/InstVisitor.h
+++ b/contrib/llvm/include/llvm/Support/InstVisitor.h
@@ -163,12 +163,16 @@ class InstVisitor {
   RetTy visitIndirectBrInst(IndirectBrInst &I)      { DELEGATE(TerminatorInst);}
   RetTy visitInvokeInst(InvokeInst &I)              { DELEGATE(TerminatorInst);}
   RetTy visitUnwindInst(UnwindInst &I)              { DELEGATE(TerminatorInst);}
+  RetTy visitResumeInst(ResumeInst &I)              { DELEGATE(TerminatorInst);}
   RetTy visitUnreachableInst(UnreachableInst &I)    { DELEGATE(TerminatorInst);}
   RetTy visitICmpInst(ICmpInst &I)                  { DELEGATE(CmpInst);}
   RetTy visitFCmpInst(FCmpInst &I)                  { DELEGATE(CmpInst);}
   RetTy visitAllocaInst(AllocaInst &I)              { DELEGATE(Instruction); }
   RetTy visitLoadInst(LoadInst     &I)              { DELEGATE(Instruction); }
   RetTy visitStoreInst(StoreInst   &I)              { DELEGATE(Instruction); }
+  RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I){ DELEGATE(Instruction); }
+  RetTy visitAtomicRMWInst(AtomicRMWInst &I)        { DELEGATE(Instruction); }
+  RetTy visitFenceInst(FenceInst   &I)              { DELEGATE(Instruction); }
   RetTy visitGetElementPtrInst(GetElementPtrInst &I){ DELEGATE(Instruction); }
   RetTy visitPHINode(PHINode       &I)              { DELEGATE(Instruction); }
   RetTy visitTruncInst(TruncInst &I)                { DELEGATE(CastInst); }
@@ -191,6 +195,7 @@ class InstVisitor {
   RetTy visitShuffleVectorInst(ShuffleVectorInst &I) { DELEGATE(Instruction); }
   RetTy visitExtractValueInst(ExtractValueInst &I)  { DELEGATE(Instruction);}
   RetTy visitInsertValueInst(InsertValueInst &I)    { DELEGATE(Instruction); }
+  RetTy visitLandingPadInst(LandingPadInst &I)      { DELEGATE(Instruction); }
 
   // Next level propagators: If the user does not overload a specific
   // instruction type, they can overload one of these to get the whole class
diff --git a/contrib/llvm/include/llvm/Support/MachO.h b/contrib/llvm/include/llvm/Support/MachO.h
index 6841a0f1fc15..5b6858613ff6 100644
--- a/contrib/llvm/include/llvm/Support/MachO.h
+++ b/contrib/llvm/include/llvm/Support/MachO.h
@@ -110,6 +110,10 @@ namespace llvm {
       LoadCommandDynamicLinkerInfo        = 0x00000022u, // LC_DYLD_INFO
       LoadCommandDynamicLinkerInfoOnly    = 0x80000022u, // LC_DYLD_INFO_ONLY
       LoadCommandDylibLoadUpward          = 0x80000023u, // LC_LOAD_UPWARD_DYLIB
+      LoadCommandVersionMinMacOSX         = 0x00000024u, // LC_VERSION_MIN_MACOSX
+      LoadCommandVersionMinIPhoneOS       = 0x00000025u, // LC_VERSION_MIN_IPHONEOS
+      LoadCommandFunctionStarts           = 0x00000026u, // LC_FUNCTION_STARTS
+      LoadCommandDyldEnvironment          = 0x00000027u, // LC_DYLD_ENVIRONMENT
 
       // Constant bits for the "flags" field in llvm::MachO::segment_command
       SegmentCommandFlagBitHighVM             = 0x1u, // SG_HIGHVM
@@ -569,6 +573,13 @@ namespace llvm {
       uint32_t cryptid;
     };
 
+    struct version_min_command {
+      uint32_t cmd;
+      uint32_t cmdsize;
+      uint32_t version;
+      uint32_t reserved;
+    };
+
     struct dyld_info_command {
       uint32_t cmd;
       uint32_t cmdsize;
diff --git a/contrib/llvm/include/llvm/Support/MemoryBuffer.h b/contrib/llvm/include/llvm/Support/MemoryBuffer.h
index 5e55bd981f0b..06816de9716a 100644
--- a/contrib/llvm/include/llvm/Support/MemoryBuffer.h
+++ b/contrib/llvm/include/llvm/Support/MemoryBuffer.h
@@ -75,9 +75,9 @@ class MemoryBuffer {
   /// return a MemoryBuffer.
   static error_code getOpenFile(int FD, const char *Filename,
                                 OwningPtr<MemoryBuffer> &result,
-                                size_t FileSize = -1,
-                                size_t MapSize = -1,
-                                off_t Offset = 0,
+                                uint64_t FileSize = -1,
+                                uint64_t MapSize = -1,
+                                int64_t Offset = 0,
                                 bool RequiresNullTerminator = true);
 
   /// getMemBuffer - Open the specified memory range as a MemoryBuffer.  Note
diff --git a/contrib/llvm/include/llvm/Support/NoFolder.h b/contrib/llvm/include/llvm/Support/NoFolder.h
index 94359a5328ef..75c1a79265e2 100644
--- a/contrib/llvm/include/llvm/Support/NoFolder.h
+++ b/contrib/llvm/include/llvm/Support/NoFolder.h
@@ -177,22 +177,22 @@ class NoFolder {
   // Memory Instructions
   //===--------------------------------------------------------------------===//
 
-  Constant *CreateGetElementPtr(Constant *C, Constant* const *IdxList,
-                                unsigned NumIdx) const {
-    return ConstantExpr::getGetElementPtr(C, IdxList, NumIdx);
+  Constant *CreateGetElementPtr(Constant *C,
+                                ArrayRef<Constant *> IdxList) const {
+    return ConstantExpr::getGetElementPtr(C, IdxList);
   }
-  Instruction *CreateGetElementPtr(Constant *C, Value* const *IdxList,
-                                   unsigned NumIdx) const {
-    return GetElementPtrInst::Create(C, IdxList, IdxList+NumIdx);
+  Instruction *CreateGetElementPtr(Constant *C,
+                                   ArrayRef<Value *> IdxList) const {
+    return GetElementPtrInst::Create(C, IdxList);
   }
 
-  Constant *CreateInBoundsGetElementPtr(Constant *C, Constant* const *IdxList,
-                                        unsigned NumIdx) const {
-    return ConstantExpr::getInBoundsGetElementPtr(C, IdxList, NumIdx);
+  Constant *CreateInBoundsGetElementPtr(Constant *C,
+                                        ArrayRef<Constant *> IdxList) const {
+    return ConstantExpr::getInBoundsGetElementPtr(C, IdxList);
   }
-  Instruction *CreateInBoundsGetElementPtr(Constant *C, Value* const *IdxList,
-                                           unsigned NumIdx) const {
-    return GetElementPtrInst::CreateInBounds(C, IdxList, IdxList+NumIdx);
+  Instruction *CreateInBoundsGetElementPtr(Constant *C,
+                                           ArrayRef<Value *> IdxList) const {
+    return GetElementPtrInst::CreateInBounds(C, IdxList);
   }
 
   //===--------------------------------------------------------------------===//
@@ -200,37 +200,37 @@ class NoFolder {
   //===--------------------------------------------------------------------===//
 
   Instruction *CreateCast(Instruction::CastOps Op, Constant *C,
-                    const Type *DestTy) const {
+                    Type *DestTy) const {
     return CastInst::Create(Op, C, DestTy);
   }
-  Instruction *CreatePointerCast(Constant *C, const Type *DestTy) const {
+  Instruction *CreatePointerCast(Constant *C, Type *DestTy) const {
     return CastInst::CreatePointerCast(C, DestTy);
   }
-  Instruction *CreateIntCast(Constant *C, const Type *DestTy,
+  Instruction *CreateIntCast(Constant *C, Type *DestTy,
                        bool isSigned) const {
     return CastInst::CreateIntegerCast(C, DestTy, isSigned);
   }
-  Instruction *CreateFPCast(Constant *C, const Type *DestTy) const {
+  Instruction *CreateFPCast(Constant *C, Type *DestTy) const {
     return CastInst::CreateFPCast(C, DestTy);
   }
 
-  Instruction *CreateBitCast(Constant *C, const Type *DestTy) const {
+  Instruction *CreateBitCast(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::BitCast, C, DestTy);
   }
-  Instruction *CreateIntToPtr(Constant *C, const Type *DestTy) const {
+  Instruction *CreateIntToPtr(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::IntToPtr, C, DestTy);
   }
-  Instruction *CreatePtrToInt(Constant *C, const Type *DestTy) const {
+  Instruction *CreatePtrToInt(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::PtrToInt, C, DestTy);
   }
-  Instruction *CreateZExtOrBitCast(Constant *C, const Type *DestTy) const {
+  Instruction *CreateZExtOrBitCast(Constant *C, Type *DestTy) const {
     return CastInst::CreateZExtOrBitCast(C, DestTy);
   }
-  Instruction *CreateSExtOrBitCast(Constant *C, const Type *DestTy) const {
+  Instruction *CreateSExtOrBitCast(Constant *C, Type *DestTy) const {
     return CastInst::CreateSExtOrBitCast(C, DestTy);
   }
 
-  Instruction *CreateTruncOrBitCast(Constant *C, const Type *DestTy) const {
+  Instruction *CreateTruncOrBitCast(Constant *C, Type *DestTy) const {
     return CastInst::CreateTruncOrBitCast(C, DestTy);
   }
 
diff --git a/contrib/llvm/include/llvm/Support/PassManagerBuilder.h b/contrib/llvm/include/llvm/Support/PassManagerBuilder.h
deleted file mode 100644
index b0cec6e81b10..000000000000
--- a/contrib/llvm/include/llvm/Support/PassManagerBuilder.h
+++ /dev/null
@@ -1,331 +0,0 @@
-//===-- llvm/Support/PassManagerBuilder.h - Build Standard Pass -*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the PassManagerBuilder class, which is used to set up a
-// "standard" optimization sequence suitable for languages like C and C++.
-//
-// These are implemented as inline functions so that we do not have to worry
-// about link issues.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_PASSMANAGERBUILDER_H
-#define LLVM_SUPPORT_PASSMANAGERBUILDER_H
-
-#include "llvm/PassManager.h"
-#include "llvm/DefaultPasses.h"
-#include "llvm/Analysis/Passes.h"
-#include "llvm/Analysis/Verifier.h"
-#include "llvm/Target/TargetLibraryInfo.h"
-#include "llvm/Transforms/Scalar.h"
-#include "llvm/Transforms/IPO.h"
-
-namespace llvm {
-  
-/// PassManagerBuilder - This class is used to set up a standard optimization
-/// sequence for languages like C and C++, allowing some APIs to customize the
-/// pass sequence in various ways. A simple example of using it would be:
-///
-///  PassManagerBuilder Builder;
-///  Builder.OptLevel = 2;
-///  Builder.populateFunctionPassManager(FPM);
-///  Builder.populateModulePassManager(MPM);
-///
-/// In addition to setting up the basic passes, PassManagerBuilder allows
-/// frontends to vend a plugin API, where plugins are allowed to add extensions
-/// to the default pass manager.  They do this by specifying where in the pass
-/// pipeline they want to be added, along with a callback function that adds
-/// the pass(es).  For example, a plugin that wanted to add a loop optimization
-/// could do something like this:
-///
-/// static void addMyLoopPass(const PMBuilder &Builder, PassManagerBase &PM) {
-///   if (Builder.getOptLevel() > 2 && Builder.getOptSizeLevel() == 0)
-///     PM.add(createMyAwesomePass());
-/// }
-///   ...
-///   Builder.addExtension(PassManagerBuilder::EP_LoopOptimizerEnd,
-///                        addMyLoopPass);
-///   ...
-class PassManagerBuilder {
-public:
-  
-  /// Extensions are passed the builder itself (so they can see how it is
-  /// configured) as well as the pass manager to add stuff to.
-  typedef void (*ExtensionFn)(const PassManagerBuilder &Builder,
-                              PassManagerBase &PM);
-  enum ExtensionPointTy {
-    /// EP_EarlyAsPossible - This extension point allows adding passes before
-    /// any other transformations, allowing them to see the code as it is coming
-    /// out of the frontend.
-    EP_EarlyAsPossible,
-    
-    /// EP_LoopOptimizerEnd - This extension point allows adding loop passes to
-    /// the end of the loop optimizer.
-    EP_LoopOptimizerEnd,
-
-    /// EP_ScalarOptimizerLate - This extension point allows adding optimization
-    /// passes after most of the main optimizations, but before the last
-    /// cleanup-ish optimizations.
-    EP_ScalarOptimizerLate
-  };
-  
-  /// The Optimization Level - Specify the basic optimization level.
-  ///    0 = -O0, 1 = -O1, 2 = -O2, 3 = -O3
-  unsigned OptLevel;
-  
-  /// SizeLevel - How much we're optimizing for size.
-  ///    0 = none, 1 = -Os, 2 = -Oz
-  unsigned SizeLevel;
-  
-  /// LibraryInfo - Specifies information about the runtime library for the
-  /// optimizer.  If this is non-null, it is added to both the function and
-  /// per-module pass pipeline.
-  TargetLibraryInfo *LibraryInfo;
-  
-  /// Inliner - Specifies the inliner to use.  If this is non-null, it is
-  /// added to the per-module passes.
-  Pass *Inliner;
-  
-  bool DisableSimplifyLibCalls;
-  bool DisableUnitAtATime;
-  bool DisableUnrollLoops;
-  
-private:
-  /// ExtensionList - This is list of all of the extensions that are registered.
-  std::vector<std::pair<ExtensionPointTy, ExtensionFn> > Extensions;
-  
-public:
-  PassManagerBuilder() {
-    OptLevel = 2;
-    SizeLevel = 0;
-    LibraryInfo = 0;
-    Inliner = 0;
-    DisableSimplifyLibCalls = false;
-    DisableUnitAtATime = false;
-    DisableUnrollLoops = false;
-  }
-  
-  ~PassManagerBuilder() {
-    delete LibraryInfo;
-    delete Inliner;
-  }
-  
-  void addExtension(ExtensionPointTy Ty, ExtensionFn Fn) {
-    Extensions.push_back(std::make_pair(Ty, Fn));
-  }
-  
-private:
-  void addExtensionsToPM(ExtensionPointTy ETy, PassManagerBase &PM) const {
-    for (unsigned i = 0, e = Extensions.size(); i != e; ++i)
-      if (Extensions[i].first == ETy)
-        Extensions[i].second(*this, PM);
-  }
-  
-  void addInitialAliasAnalysisPasses(PassManagerBase &PM) const {
-    // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
-    // BasicAliasAnalysis wins if they disagree. This is intended to help
-    // support "obvious" type-punning idioms.
-    PM.add(createTypeBasedAliasAnalysisPass());
-    PM.add(createBasicAliasAnalysisPass());
-  }
-public:
-  
-  /// populateFunctionPassManager - This fills in the function pass manager,
-  /// which is expected to be run on each function immediately as it is
-  /// generated.  The idea is to reduce the size of the IR in memory.
-  void populateFunctionPassManager(FunctionPassManager &FPM) {
-    addExtensionsToPM(EP_EarlyAsPossible, FPM);
-    
-    // Add LibraryInfo if we have some.
-    if (LibraryInfo) FPM.add(new TargetLibraryInfo(*LibraryInfo));
-
-    if (OptLevel == 0) return;
-
-    addInitialAliasAnalysisPasses(FPM);
-    
-    FPM.add(createCFGSimplificationPass());
-    FPM.add(createScalarReplAggregatesPass());
-    FPM.add(createEarlyCSEPass());
-    FPM.add(createLowerExpectIntrinsicPass());
-  }
-  
-  /// populateModulePassManager - This sets up the primary pass manager.
-  void populateModulePassManager(PassManagerBase &MPM) {
-    // If all optimizations are disabled, just run the always-inline pass.
-    if (OptLevel == 0) {
-      if (Inliner) {
-        MPM.add(Inliner);
-        Inliner = 0;
-      }
-      return;
-    }
-      
-    // Add LibraryInfo if we have some.
-    if (LibraryInfo) MPM.add(new TargetLibraryInfo(*LibraryInfo));
-
-    addInitialAliasAnalysisPasses(MPM);
-    
-    if (!DisableUnitAtATime) {
-      MPM.add(createGlobalOptimizerPass());     // Optimize out global vars
-      
-      MPM.add(createIPSCCPPass());              // IP SCCP
-      MPM.add(createDeadArgEliminationPass());  // Dead argument elimination
-      
-      MPM.add(createInstructionCombiningPass());// Clean up after IPCP & DAE
-      MPM.add(createCFGSimplificationPass());   // Clean up after IPCP & DAE
-    }
-    
-    // Start of CallGraph SCC passes.
-    if (!DisableUnitAtATime)
-      MPM.add(createPruneEHPass());             // Remove dead EH info
-    if (Inliner) {
-      MPM.add(Inliner);
-      Inliner = 0;
-    }
-    if (!DisableUnitAtATime)
-      MPM.add(createFunctionAttrsPass());       // Set readonly/readnone attrs
-    if (OptLevel > 2)
-      MPM.add(createArgumentPromotionPass());   // Scalarize uninlined fn args
-    
-    // Start of function pass.
-    // Break up aggregate allocas, using SSAUpdater.
-    MPM.add(createScalarReplAggregatesPass(-1, false));
-    MPM.add(createEarlyCSEPass());              // Catch trivial redundancies
-    if (!DisableSimplifyLibCalls)
-      MPM.add(createSimplifyLibCallsPass());    // Library Call Optimizations
-    MPM.add(createJumpThreadingPass());         // Thread jumps.
-    MPM.add(createCorrelatedValuePropagationPass()); // Propagate conditionals
-    MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
-    MPM.add(createInstructionCombiningPass());  // Combine silly seq's
-    
-    MPM.add(createTailCallEliminationPass());   // Eliminate tail calls
-    MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
-    MPM.add(createReassociatePass());           // Reassociate expressions
-    MPM.add(createLoopRotatePass());            // Rotate Loop
-    MPM.add(createLICMPass());                  // Hoist loop invariants
-    MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3));
-    MPM.add(createInstructionCombiningPass());  
-    MPM.add(createIndVarSimplifyPass());        // Canonicalize indvars
-    MPM.add(createLoopIdiomPass());             // Recognize idioms like memset.
-    MPM.add(createLoopDeletionPass());          // Delete dead loops
-    if (!DisableUnrollLoops)
-      MPM.add(createLoopUnrollPass());          // Unroll small loops
-    addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
-
-    if (OptLevel > 1)
-      MPM.add(createGVNPass());                 // Remove redundancies
-    MPM.add(createMemCpyOptPass());             // Remove memcpy / form memset
-    MPM.add(createSCCPPass());                  // Constant prop with SCCP
-    
-    // Run instcombine after redundancy elimination to exploit opportunities
-    // opened up by them.
-    MPM.add(createInstructionCombiningPass());
-    MPM.add(createJumpThreadingPass());         // Thread jumps
-    MPM.add(createCorrelatedValuePropagationPass());
-    MPM.add(createDeadStoreEliminationPass());  // Delete dead stores
-
-    addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
-
-    MPM.add(createAggressiveDCEPass());         // Delete dead instructions
-    MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
-    MPM.add(createInstructionCombiningPass());  // Clean up after everything.
-    
-    if (!DisableUnitAtATime) {
-      // FIXME: We shouldn't bother with this anymore.
-      MPM.add(createStripDeadPrototypesPass()); // Get rid of dead prototypes
-      
-      // GlobalOpt already deletes dead functions and globals, at -O3 try a
-      // late pass of GlobalDCE.  It is capable of deleting dead cycles.
-      if (OptLevel > 2)
-        MPM.add(createGlobalDCEPass());         // Remove dead fns and globals.
-      
-      if (OptLevel > 1)
-        MPM.add(createConstantMergePass());     // Merge dup global constants
-    }
-  }
-  
-  void populateLTOPassManager(PassManagerBase &PM, bool Internalize,
-                              bool RunInliner) {
-    // Provide AliasAnalysis services for optimizations.
-    addInitialAliasAnalysisPasses(PM);
-    
-    // Now that composite has been compiled, scan through the module, looking
-    // for a main function.  If main is defined, mark all other functions
-    // internal.
-    if (Internalize)
-      PM.add(createInternalizePass(true));
-    
-    // Propagate constants at call sites into the functions they call.  This
-    // opens opportunities for globalopt (and inlining) by substituting function
-    // pointers passed as arguments to direct uses of functions.  
-    PM.add(createIPSCCPPass());
-    
-    // Now that we internalized some globals, see if we can hack on them!
-    PM.add(createGlobalOptimizerPass());
-    
-    // Linking modules together can lead to duplicated global constants, only
-    // keep one copy of each constant.
-    PM.add(createConstantMergePass());
-    
-    // Remove unused arguments from functions.
-    PM.add(createDeadArgEliminationPass());
-    
-    // Reduce the code after globalopt and ipsccp.  Both can open up significant
-    // simplification opportunities, and both can propagate functions through
-    // function pointers.  When this happens, we often have to resolve varargs
-    // calls, etc, so let instcombine do this.
-    PM.add(createInstructionCombiningPass());
-
-    // Inline small functions
-    if (RunInliner)
-      PM.add(createFunctionInliningPass());
-    
-    PM.add(createPruneEHPass());   // Remove dead EH info.
-    
-    // Optimize globals again if we ran the inliner.
-    if (RunInliner)
-      PM.add(createGlobalOptimizerPass());
-    PM.add(createGlobalDCEPass()); // Remove dead functions.
-    
-    // If we didn't decide to inline a function, check to see if we can
-    // transform it to pass arguments by value instead of by reference.
-    PM.add(createArgumentPromotionPass());
-    
-    // The IPO passes may leave cruft around.  Clean up after them.
-    PM.add(createInstructionCombiningPass());
-    PM.add(createJumpThreadingPass());
-    // Break up allocas
-    PM.add(createScalarReplAggregatesPass());
-    
-    // Run a few AA driven optimizations here and now, to cleanup the code.
-    PM.add(createFunctionAttrsPass()); // Add nocapture.
-    PM.add(createGlobalsModRefPass()); // IP alias analysis.
-    
-    PM.add(createLICMPass());      // Hoist loop invariants.
-    PM.add(createGVNPass());       // Remove redundancies.
-    PM.add(createMemCpyOptPass()); // Remove dead memcpys.
-    // Nuke dead stores.
-    PM.add(createDeadStoreEliminationPass());
-    
-    // Cleanup and simplify the code after the scalar optimizations.
-    PM.add(createInstructionCombiningPass());
-    
-    PM.add(createJumpThreadingPass());
-    
-    // Delete basic blocks, which optimization passes may have killed.
-    PM.add(createCFGSimplificationPass());
-   
-    // Now that we have optimized the program, discard unreachable functions.
-    PM.add(createGlobalDCEPass());
-  }
-};
-
-  
-} // end namespace llvm
-#endif
diff --git a/contrib/llvm/include/llvm/Support/PathV1.h b/contrib/llvm/include/llvm/Support/PathV1.h
index 024bb39cedc2..45165ded619c 100644
--- a/contrib/llvm/include/llvm/Support/PathV1.h
+++ b/contrib/llvm/include/llvm/Support/PathV1.h
@@ -733,6 +733,7 @@ namespace sys {
     Mach_O_DynamicLinker_FileType,     ///< The Mach-O dynamic linker
     Mach_O_Bundle_FileType,            ///< Mach-O Bundle file
     Mach_O_DynamicallyLinkedSharedLibStub_FileType, ///< Mach-O Shared lib stub
+    Mach_O_DSYMCompanion_FileType,     ///< Mach-O dSYM companion file
     COFF_FileType                      ///< COFF object file or lib
   };
 
diff --git a/contrib/llvm/include/llvm/Support/PathV2.h b/contrib/llvm/include/llvm/Support/PathV2.h
index 251563398fb4..6d38c9571558 100644
--- a/contrib/llvm/include/llvm/Support/PathV2.h
+++ b/contrib/llvm/include/llvm/Support/PathV2.h
@@ -187,7 +187,7 @@ const StringRef root_name(StringRef path);
 /// @result The root directory of \a path if it has one, otherwise
 ///               "".
 const StringRef root_directory(StringRef path);
-
+  
 /// @brief Get root path.
 ///
 /// Equivalent to root_name + root_directory.
@@ -264,6 +264,17 @@ const StringRef extension(StringRef path);
 /// @result true if \a value is a path separator character on the host OS
 bool is_separator(char value);
 
+/// @brief Get the typical temporary directory for the system, e.g., 
+/// "/var/tmp" or "C:/TEMP"
+///
+/// @param erasedOnReboot Whether to favor a path that is erased on reboot
+/// rather than one that potentially persists longer. This parameter will be
+/// ignored if the user or system has set the typical environment variable
+/// (e.g., TEMP on Windows, TMPDIR on *nix) to specify a temporary directory.
+///
+/// @param Result Holds the resulting path name.
+void system_temp_directory(bool erasedOnReboot, SmallVectorImpl<char> &result);
+
 /// @brief Has root name?
 ///
 /// root_name != ""
diff --git a/contrib/llvm/include/llvm/Support/Process.h b/contrib/llvm/include/llvm/Support/Process.h
index 33799229ff35..27ef2670093a 100644
--- a/contrib/llvm/include/llvm/Support/Process.h
+++ b/contrib/llvm/include/llvm/Support/Process.h
@@ -138,6 +138,9 @@ namespace sys {
 
       /// Resets the terminals colors, or returns an escape sequence to do so.
       static const char *ResetColor();
+
+      /// Change the program working directory to that given by \arg Path.
+      static void SetWorkingDirectory(std::string Path);
     /// @}
   };
 }
diff --git a/contrib/llvm/include/llvm/Support/SMLoc.h b/contrib/llvm/include/llvm/Support/SMLoc.h
index 967bf1432c64..02db32794b6d 100644
--- a/contrib/llvm/include/llvm/Support/SMLoc.h
+++ b/contrib/llvm/include/llvm/Support/SMLoc.h
@@ -18,19 +18,19 @@
 namespace llvm {
 
 // SMLoc - Represents a location in source code.
-class SMLoc {    
+class SMLoc {
   const char *Ptr;
 public:
   SMLoc() : Ptr(0) {}
   SMLoc(const SMLoc &RHS) : Ptr(RHS.Ptr) {}
-    
+
   bool isValid() const { return Ptr != 0; }
-    
+
   bool operator==(const SMLoc &RHS) const { return RHS.Ptr == Ptr; }
   bool operator!=(const SMLoc &RHS) const { return RHS.Ptr != Ptr; }
-    
+
   const char *getPointer() const { return Ptr; }
-    
+
   static SMLoc getFromPointer(const char *Ptr) {
     SMLoc L;
     L.Ptr = Ptr;
diff --git a/contrib/llvm/include/llvm/Support/SourceMgr.h b/contrib/llvm/include/llvm/Support/SourceMgr.h
index 030db8f4e3fd..deb8cafa06d2 100644
--- a/contrib/llvm/include/llvm/Support/SourceMgr.h
+++ b/contrib/llvm/include/llvm/Support/SourceMgr.h
@@ -78,6 +78,9 @@ class SourceMgr {
     DiagContext = Ctx;
   }
 
+  DiagHandlerTy getDiagHandler() const { return DiagHandler; }
+  void *getDiagContext() const { return DiagContext; }
+
   const SrcBuffer &getBufferInfo(unsigned i) const {
     assert(i < Buffers.size() && "Invalid Buffer ID!");
     return Buffers[i];
@@ -138,8 +141,12 @@ class SourceMgr {
                           const Twine &Msg, const char *Type,
                           bool ShowLine = true) const;
 
-
-private:
+  /// PrintIncludeStack - Prints the names of included files and the line of the
+  /// file they were included from.  A diagnostic handler can use this before
+  /// printing its custom formatted message.
+  ///
+  /// @param IncludeLoc - The line of the include.
+  /// @param OS the raw_ostream to print on.
   void PrintIncludeStack(SMLoc IncludeLoc, raw_ostream &OS) const;
 };
 
diff --git a/contrib/llvm/include/llvm/Support/TargetFolder.h b/contrib/llvm/include/llvm/Support/TargetFolder.h
index 3233a98da9c1..c65faa66219e 100644
--- a/contrib/llvm/include/llvm/Support/TargetFolder.h
+++ b/contrib/llvm/include/llvm/Support/TargetFolder.h
@@ -130,22 +130,34 @@ class TargetFolder {
   // Memory Instructions
   //===--------------------------------------------------------------------===//
 
-  Constant *CreateGetElementPtr(Constant *C, Constant* const *IdxList,
-                                unsigned NumIdx) const {
-    return Fold(ConstantExpr::getGetElementPtr(C, IdxList, NumIdx));
+  Constant *CreateGetElementPtr(Constant *C,
+                                ArrayRef<Constant *> IdxList) const {
+    return Fold(ConstantExpr::getGetElementPtr(C, IdxList));
   }
-  Constant *CreateGetElementPtr(Constant *C, Value* const *IdxList,
-                                unsigned NumIdx) const {
-    return Fold(ConstantExpr::getGetElementPtr(C, IdxList, NumIdx));
+  Constant *CreateGetElementPtr(Constant *C, Constant *Idx) const {
+    // This form of the function only exists to avoid ambiguous overload
+    // warnings about whether to convert Idx to ArrayRef<Constant *> or
+    // ArrayRef<Value *>.
+    return Fold(ConstantExpr::getGetElementPtr(C, Idx));
+  }
+  Constant *CreateGetElementPtr(Constant *C,
+                                ArrayRef<Value *> IdxList) const {
+    return Fold(ConstantExpr::getGetElementPtr(C, IdxList));
   }
 
-  Constant *CreateInBoundsGetElementPtr(Constant *C, Constant* const *IdxList,
-                                        unsigned NumIdx) const {
-    return Fold(ConstantExpr::getInBoundsGetElementPtr(C, IdxList, NumIdx));
+  Constant *CreateInBoundsGetElementPtr(Constant *C,
+                                        ArrayRef<Constant *> IdxList) const {
+    return Fold(ConstantExpr::getInBoundsGetElementPtr(C, IdxList));
   }
-  Constant *CreateInBoundsGetElementPtr(Constant *C, Value* const *IdxList,
-                                        unsigned NumIdx) const {
-    return Fold(ConstantExpr::getInBoundsGetElementPtr(C, IdxList, NumIdx));
+  Constant *CreateInBoundsGetElementPtr(Constant *C, Constant *Idx) const {
+    // This form of the function only exists to avoid ambiguous overload
+    // warnings about whether to convert Idx to ArrayRef<Constant *> or
+    // ArrayRef<Value *>.
+    return Fold(ConstantExpr::getInBoundsGetElementPtr(C, Idx));
+  }
+  Constant *CreateInBoundsGetElementPtr(Constant *C,
+                                        ArrayRef<Value *> IdxList) const {
+    return Fold(ConstantExpr::getInBoundsGetElementPtr(C, IdxList));
   }
 
   //===--------------------------------------------------------------------===//
@@ -153,40 +165,40 @@ class TargetFolder {
   //===--------------------------------------------------------------------===//
 
   Constant *CreateCast(Instruction::CastOps Op, Constant *C,
-                       const Type *DestTy) const {
+                       Type *DestTy) const {
     if (C->getType() == DestTy)
       return C; // avoid calling Fold
     return Fold(ConstantExpr::getCast(Op, C, DestTy));
   }
-  Constant *CreateIntCast(Constant *C, const Type *DestTy,
+  Constant *CreateIntCast(Constant *C, Type *DestTy,
                           bool isSigned) const {
     if (C->getType() == DestTy)
       return C; // avoid calling Fold
     return Fold(ConstantExpr::getIntegerCast(C, DestTy, isSigned));
   }
-  Constant *CreatePointerCast(Constant *C, const Type *DestTy) const {
+  Constant *CreatePointerCast(Constant *C, Type *DestTy) const {
     return ConstantExpr::getPointerCast(C, DestTy);
   }
-  Constant *CreateBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateBitCast(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::BitCast, C, DestTy);
   }
-  Constant *CreateIntToPtr(Constant *C, const Type *DestTy) const {
+  Constant *CreateIntToPtr(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::IntToPtr, C, DestTy);
   }
-  Constant *CreatePtrToInt(Constant *C, const Type *DestTy) const {
+  Constant *CreatePtrToInt(Constant *C, Type *DestTy) const {
     return CreateCast(Instruction::PtrToInt, C, DestTy);
   }
-  Constant *CreateZExtOrBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateZExtOrBitCast(Constant *C, Type *DestTy) const {
     if (C->getType() == DestTy)
       return C; // avoid calling Fold
     return Fold(ConstantExpr::getZExtOrBitCast(C, DestTy));
   }
-  Constant *CreateSExtOrBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateSExtOrBitCast(Constant *C, Type *DestTy) const {
     if (C->getType() == DestTy)
       return C; // avoid calling Fold
     return Fold(ConstantExpr::getSExtOrBitCast(C, DestTy));
   }
-  Constant *CreateTruncOrBitCast(Constant *C, const Type *DestTy) const {
+  Constant *CreateTruncOrBitCast(Constant *C, Type *DestTy) const {
     if (C->getType() == DestTy)
       return C; // avoid calling Fold
     return Fold(ConstantExpr::getTruncOrBitCast(C, DestTy));
diff --git a/contrib/llvm/include/llvm/Target/TargetRegistry.h b/contrib/llvm/include/llvm/Support/TargetRegistry.h
similarity index 66%
rename from contrib/llvm/include/llvm/Target/TargetRegistry.h
rename to contrib/llvm/include/llvm/Support/TargetRegistry.h
index 7e0ce19f8f84..45f249d7ed0e 100644
--- a/contrib/llvm/include/llvm/Target/TargetRegistry.h
+++ b/contrib/llvm/include/llvm/Support/TargetRegistry.h
@@ -1,4 +1,4 @@
-//===-- Target/TargetRegistry.h - Target Registration -----------*- C++ -*-===//
+//===-- Support/TargetRegistry.h - Target Registration ----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -16,9 +16,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TARGET_TARGETREGISTRY_H
-#define LLVM_TARGET_TARGETREGISTRY_H
+#ifndef LLVM_SUPPORT_TARGETREGISTRY_H
+#define LLVM_SUPPORT_TARGETREGISTRY_H
 
+#include "llvm/Support/CodeGen.h"
 #include "llvm/ADT/Triple.h"
 #include <string>
 #include <cassert>
@@ -27,19 +28,21 @@ namespace llvm {
   class AsmPrinter;
   class Module;
   class MCAssembler;
+  class MCAsmBackend;
   class MCAsmInfo;
   class MCAsmParser;
   class MCCodeEmitter;
+  class MCCodeGenInfo;
   class MCContext;
   class MCDisassembler;
+  class MCInstrAnalysis;
   class MCInstPrinter;
   class MCInstrInfo;
   class MCRegisterInfo;
   class MCStreamer;
   class MCSubtargetInfo;
-  class TargetAsmBackend;
-  class TargetAsmLexer;
-  class TargetAsmParser;
+  class MCTargetAsmLexer;
+  class MCTargetAsmParser;
   class TargetMachine;
   class raw_ostream;
   class formatted_raw_ostream;
@@ -49,7 +52,7 @@ namespace llvm {
                                 bool useLoc, bool useCFI,
                                 MCInstPrinter *InstPrint,
                                 MCCodeEmitter *CE,
-                                TargetAsmBackend *TAB,
+                                MCAsmBackend *TAB,
                                 bool ShowInst);
 
   /// Target - Wrapper for Target specific information.
@@ -68,38 +71,46 @@ namespace llvm {
 
     typedef MCAsmInfo *(*MCAsmInfoCtorFnTy)(const Target &T,
                                             StringRef TT);
+    typedef MCCodeGenInfo *(*MCCodeGenInfoCtorFnTy)(StringRef TT,
+                                                    Reloc::Model RM,
+                                                    CodeModel::Model CM);
     typedef MCInstrInfo *(*MCInstrInfoCtorFnTy)(void);
-    typedef MCRegisterInfo *(*MCRegInfoCtorFnTy)(void);
+    typedef MCInstrAnalysis *(*MCInstrAnalysisCtorFnTy)(const MCInstrInfo*Info);
+    typedef MCRegisterInfo *(*MCRegInfoCtorFnTy)(StringRef TT);
     typedef MCSubtargetInfo *(*MCSubtargetInfoCtorFnTy)(StringRef TT,
                                                         StringRef CPU,
                                                         StringRef Features);
     typedef TargetMachine *(*TargetMachineCtorTy)(const Target &T,
-                                                  const std::string &TT,
-                                                  const std::string &CPU,
-                                                  const std::string &Features);
+                                                  StringRef TT,
+                                                  StringRef CPU,
+                                                  StringRef Features,
+                                                  Reloc::Model RM,
+                                                  CodeModel::Model CM);
     typedef AsmPrinter *(*AsmPrinterCtorTy)(TargetMachine &TM,
                                             MCStreamer &Streamer);
-    typedef TargetAsmBackend *(*AsmBackendCtorTy)(const Target &T,
-                                                  const std::string &TT);
-    typedef TargetAsmLexer *(*AsmLexerCtorTy)(const Target &T,
-                                              const MCAsmInfo &MAI);
-    typedef TargetAsmParser *(*AsmParserCtorTy)(MCSubtargetInfo &STI,
-                                                MCAsmParser &P);
-    typedef MCDisassembler *(*MCDisassemblerCtorTy)(const Target &T);
+    typedef MCAsmBackend *(*MCAsmBackendCtorTy)(const Target &T, StringRef TT);
+    typedef MCTargetAsmLexer *(*MCAsmLexerCtorTy)(const Target &T,
+                                                  const MCRegisterInfo &MRI,
+                                                  const MCAsmInfo &MAI);
+    typedef MCTargetAsmParser *(*MCAsmParserCtorTy)(MCSubtargetInfo &STI,
+                                                    MCAsmParser &P);
+    typedef MCDisassembler *(*MCDisassemblerCtorTy)(const Target &T,
+                                                    const MCSubtargetInfo &STI);
     typedef MCInstPrinter *(*MCInstPrinterCtorTy)(const Target &T,
                                                   unsigned SyntaxVariant,
-                                                  const MCAsmInfo &MAI);
-    typedef MCCodeEmitter *(*CodeEmitterCtorTy)(const MCInstrInfo &II,
-                                                const MCSubtargetInfo &STI,
-                                                MCContext &Ctx);
-    typedef MCStreamer *(*ObjectStreamerCtorTy)(const Target &T,
-                                                const std::string &TT,
-                                                MCContext &Ctx,
-                                                TargetAsmBackend &TAB,
-                                                raw_ostream &_OS,
-                                                MCCodeEmitter *_Emitter,
-                                                bool RelaxAll,
-                                                bool NoExecStack);
+                                                  const MCAsmInfo &MAI,
+                                                  const MCSubtargetInfo &STI);
+    typedef MCCodeEmitter *(*MCCodeEmitterCtorTy)(const MCInstrInfo &II,
+                                                  const MCSubtargetInfo &STI,
+                                                  MCContext &Ctx);
+    typedef MCStreamer *(*MCObjectStreamerCtorTy)(const Target &T,
+                                                  StringRef TT,
+                                                  MCContext &Ctx,
+                                                  MCAsmBackend &TAB,
+                                                  raw_ostream &_OS,
+                                                  MCCodeEmitter *_Emitter,
+                                                  bool RelaxAll,
+                                                  bool NoExecStack);
     typedef MCStreamer *(*AsmStreamerCtorTy)(MCContext &Ctx,
                                              formatted_raw_ostream &OS,
                                              bool isVerboseAsm,
@@ -107,7 +118,7 @@ namespace llvm {
                                              bool useCFI,
                                              MCInstPrinter *InstPrint,
                                              MCCodeEmitter *CE,
-                                             TargetAsmBackend *TAB,
+                                             MCAsmBackend *TAB,
                                              bool ShowInst);
 
   private:
@@ -132,10 +143,18 @@ namespace llvm {
     /// registered.
     MCAsmInfoCtorFnTy MCAsmInfoCtorFn;
 
+    /// MCCodeGenInfoCtorFn - Constructor function for this target's MCCodeGenInfo,
+    /// if registered.
+    MCCodeGenInfoCtorFnTy MCCodeGenInfoCtorFn;
+
     /// MCInstrInfoCtorFn - Constructor function for this target's MCInstrInfo,
     /// if registered.
     MCInstrInfoCtorFnTy MCInstrInfoCtorFn;
 
+    /// MCInstrAnalysisCtorFn - Constructor function for this target's
+    /// MCInstrAnalysis, if registered.
+    MCInstrAnalysisCtorFnTy MCInstrAnalysisCtorFn;
+
     /// MCRegInfoCtorFn - Constructor function for this target's MCRegisterInfo,
     /// if registered.
     MCRegInfoCtorFnTy MCRegInfoCtorFn;
@@ -148,17 +167,17 @@ namespace llvm {
     /// TargetMachine, if registered.
     TargetMachineCtorTy TargetMachineCtorFn;
 
-    /// AsmBackendCtorFn - Construction function for this target's
-    /// TargetAsmBackend, if registered.
-    AsmBackendCtorTy AsmBackendCtorFn;
+    /// MCAsmBackendCtorFn - Construction function for this target's
+    /// MCAsmBackend, if registered.
+    MCAsmBackendCtorTy MCAsmBackendCtorFn;
 
-    /// AsmLexerCtorFn - Construction function for this target's TargetAsmLexer,
-    /// if registered.
-    AsmLexerCtorTy AsmLexerCtorFn;
+    /// MCAsmLexerCtorFn - Construction function for this target's
+    /// MCTargetAsmLexer, if registered.
+    MCAsmLexerCtorTy MCAsmLexerCtorFn;
 
-    /// AsmParserCtorFn - Construction function for this target's
-    /// TargetAsmParser, if registered.
-    AsmParserCtorTy AsmParserCtorFn;
+    /// MCAsmParserCtorFn - Construction function for this target's
+    /// MCTargetAsmParser, if registered.
+    MCAsmParserCtorTy MCAsmParserCtorFn;
 
     /// AsmPrinterCtorFn - Construction function for this target's AsmPrinter,
     /// if registered.
@@ -172,13 +191,13 @@ namespace llvm {
     /// MCInstPrinter, if registered.
     MCInstPrinterCtorTy MCInstPrinterCtorFn;
 
-    /// CodeEmitterCtorFn - Construction function for this target's CodeEmitter,
-    /// if registered.
-    CodeEmitterCtorTy CodeEmitterCtorFn;
+    /// MCCodeEmitterCtorFn - Construction function for this target's
+    /// CodeEmitter, if registered.
+    MCCodeEmitterCtorTy MCCodeEmitterCtorFn;
 
-    /// ObjectStreamerCtorFn - Construction function for this target's
-    /// ObjectStreamer, if registered.
-    ObjectStreamerCtorTy ObjectStreamerCtorFn;
+    /// MCObjectStreamerCtorFn - Construction function for this target's
+    /// MCObjectStreamer, if registered.
+    MCObjectStreamerCtorTy MCObjectStreamerCtorFn;
 
     /// AsmStreamerCtorFn - Construction function for this target's
     /// AsmStreamer, if registered (default = llvm::createAsmStreamer).
@@ -209,14 +228,14 @@ namespace llvm {
     /// hasTargetMachine - Check if this target supports code generation.
     bool hasTargetMachine() const { return TargetMachineCtorFn != 0; }
 
-    /// hasAsmBackend - Check if this target supports .o generation.
-    bool hasAsmBackend() const { return AsmBackendCtorFn != 0; }
+    /// hasMCAsmBackend - Check if this target supports .o generation.
+    bool hasMCAsmBackend() const { return MCAsmBackendCtorFn != 0; }
 
-    /// hasAsmLexer - Check if this target supports .s lexing.
-    bool hasAsmLexer() const { return AsmLexerCtorFn != 0; }
+    /// hasMCAsmLexer - Check if this target supports .s lexing.
+    bool hasMCAsmLexer() const { return MCAsmLexerCtorFn != 0; }
 
     /// hasAsmParser - Check if this target supports .s parsing.
-    bool hasAsmParser() const { return AsmParserCtorFn != 0; }
+    bool hasMCAsmParser() const { return MCAsmParserCtorFn != 0; }
 
     /// hasAsmPrinter - Check if this target supports .s printing.
     bool hasAsmPrinter() const { return AsmPrinterCtorFn != 0; }
@@ -227,11 +246,11 @@ namespace llvm {
     /// hasMCInstPrinter - Check if this target has an instruction printer.
     bool hasMCInstPrinter() const { return MCInstPrinterCtorFn != 0; }
 
-    /// hasCodeEmitter - Check if this target supports instruction encoding.
-    bool hasCodeEmitter() const { return CodeEmitterCtorFn != 0; }
+    /// hasMCCodeEmitter - Check if this target supports instruction encoding.
+    bool hasMCCodeEmitter() const { return MCCodeEmitterCtorFn != 0; }
 
-    /// hasObjectStreamer - Check if this target supports streaming to files.
-    bool hasObjectStreamer() const { return ObjectStreamerCtorFn != 0; }
+    /// hasMCObjectStreamer - Check if this target supports streaming to files.
+    bool hasMCObjectStreamer() const { return MCObjectStreamerCtorFn != 0; }
 
     /// hasAsmStreamer - Check if this target supports streaming to files.
     bool hasAsmStreamer() const { return AsmStreamerCtorFn != 0; }
@@ -253,6 +272,15 @@ namespace llvm {
       return MCAsmInfoCtorFn(*this, Triple);
     }
 
+    /// createMCCodeGenInfo - Create a MCCodeGenInfo implementation.
+    ///
+    MCCodeGenInfo *createMCCodeGenInfo(StringRef Triple, Reloc::Model RM,
+                                       CodeModel::Model CM) const {
+      if (!MCCodeGenInfoCtorFn)
+        return 0;
+      return MCCodeGenInfoCtorFn(Triple, RM, CM);
+    }
+
     /// createMCInstrInfo - Create a MCInstrInfo implementation.
     ///
     MCInstrInfo *createMCInstrInfo() const {
@@ -261,12 +289,20 @@ namespace llvm {
       return MCInstrInfoCtorFn();
     }
 
+    /// createMCInstrAnalysis - Create a MCInstrAnalysis implementation.
+    ///
+    MCInstrAnalysis *createMCInstrAnalysis(const MCInstrInfo *Info) const {
+      if (!MCInstrAnalysisCtorFn)
+        return 0;
+      return MCInstrAnalysisCtorFn(Info);
+    }
+
     /// createMCRegInfo - Create a MCRegisterInfo implementation.
     ///
-    MCRegisterInfo *createMCRegInfo() const {
+    MCRegisterInfo *createMCRegInfo(StringRef Triple) const {
       if (!MCRegInfoCtorFn)
         return 0;
-      return MCRegInfoCtorFn();
+      return MCRegInfoCtorFn(Triple);
     }
 
     /// createMCSubtargetInfo - Create a MCSubtargetInfo implementation.
@@ -292,41 +328,43 @@ namespace llvm {
     /// feature set; it should always be provided. Generally this should be
     /// either the target triple from the module, or the target triple of the
     /// host if that does not exist.
-    TargetMachine *createTargetMachine(const std::string &Triple,
-                                       const std::string &CPU,
-                                       const std::string &Features) const {
+    TargetMachine *createTargetMachine(StringRef Triple, StringRef CPU,
+                               StringRef Features,
+                               Reloc::Model RM = Reloc::Default,
+                               CodeModel::Model CM = CodeModel::Default) const {
       if (!TargetMachineCtorFn)
         return 0;
-      return TargetMachineCtorFn(*this, Triple, CPU, Features);
+      return TargetMachineCtorFn(*this, Triple, CPU, Features, RM, CM);
     }
 
-    /// createAsmBackend - Create a target specific assembly parser.
+    /// createMCAsmBackend - Create a target specific assembly parser.
     ///
     /// \arg Triple - The target triple string.
     /// \arg Backend - The target independent assembler object.
-    TargetAsmBackend *createAsmBackend(const std::string &Triple) const {
-      if (!AsmBackendCtorFn)
+    MCAsmBackend *createMCAsmBackend(StringRef Triple) const {
+      if (!MCAsmBackendCtorFn)
         return 0;
-      return AsmBackendCtorFn(*this, Triple);
+      return MCAsmBackendCtorFn(*this, Triple);
     }
 
-    /// createAsmLexer - Create a target specific assembly lexer.
+    /// createMCAsmLexer - Create a target specific assembly lexer.
     ///
-    TargetAsmLexer *createAsmLexer(const MCAsmInfo &MAI) const {
-      if (!AsmLexerCtorFn)
+    MCTargetAsmLexer *createMCAsmLexer(const MCRegisterInfo &MRI,
+                                       const MCAsmInfo &MAI) const {
+      if (!MCAsmLexerCtorFn)
         return 0;
-      return AsmLexerCtorFn(*this, MAI);
+      return MCAsmLexerCtorFn(*this, MRI, MAI);
     }
 
-    /// createAsmParser - Create a target specific assembly parser.
+    /// createMCAsmParser - Create a target specific assembly parser.
     ///
     /// \arg Parser - The target independent parser implementation to use for
     /// parsing and lexing.
-    TargetAsmParser *createAsmParser(MCSubtargetInfo &STI,
-                                     MCAsmParser &Parser) const {
-      if (!AsmParserCtorFn)
+    MCTargetAsmParser *createMCAsmParser(MCSubtargetInfo &STI,
+                                         MCAsmParser &Parser) const {
+      if (!MCAsmParserCtorFn)
         return 0;
-      return AsmParserCtorFn(STI, Parser);
+      return MCAsmParserCtorFn(STI, Parser);
     }
 
     /// createAsmPrinter - Create a target specific assembly printer pass.  This
@@ -337,30 +375,31 @@ namespace llvm {
       return AsmPrinterCtorFn(TM, Streamer);
     }
 
-    MCDisassembler *createMCDisassembler() const {
+    MCDisassembler *createMCDisassembler(const MCSubtargetInfo &STI) const {
       if (!MCDisassemblerCtorFn)
         return 0;
-      return MCDisassemblerCtorFn(*this);
+      return MCDisassemblerCtorFn(*this, STI);
     }
 
     MCInstPrinter *createMCInstPrinter(unsigned SyntaxVariant,
-                                       const MCAsmInfo &MAI) const {
+                                       const MCAsmInfo &MAI,
+                                       const MCSubtargetInfo &STI) const {
       if (!MCInstPrinterCtorFn)
         return 0;
-      return MCInstPrinterCtorFn(*this, SyntaxVariant, MAI);
+      return MCInstPrinterCtorFn(*this, SyntaxVariant, MAI, STI);
     }
 
 
-    /// createCodeEmitter - Create a target specific code emitter.
-    MCCodeEmitter *createCodeEmitter(const MCInstrInfo &II,
-                                     const MCSubtargetInfo &STI,
-                                     MCContext &Ctx) const {
-      if (!CodeEmitterCtorFn)
+    /// createMCCodeEmitter - Create a target specific code emitter.
+    MCCodeEmitter *createMCCodeEmitter(const MCInstrInfo &II,
+                                       const MCSubtargetInfo &STI,
+                                       MCContext &Ctx) const {
+      if (!MCCodeEmitterCtorFn)
         return 0;
-      return CodeEmitterCtorFn(II, STI, Ctx);
+      return MCCodeEmitterCtorFn(II, STI, Ctx);
     }
 
-    /// createObjectStreamer - Create a target specific MCStreamer.
+    /// createMCObjectStreamer - Create a target specific MCStreamer.
     ///
     /// \arg TT - The target triple.
     /// \arg Ctx - The target context.
@@ -369,16 +408,16 @@ namespace llvm {
     /// \arg _Emitter - The target independent assembler object.Takes ownership.
     /// \arg RelaxAll - Relax all fixups?
     /// \arg NoExecStack - Mark file as not needing a executable stack.
-    MCStreamer *createObjectStreamer(const std::string &TT, MCContext &Ctx,
-                                     TargetAsmBackend &TAB,
-                                     raw_ostream &_OS,
-                                     MCCodeEmitter *_Emitter,
-                                     bool RelaxAll,
-                                     bool NoExecStack) const {
-      if (!ObjectStreamerCtorFn)
+    MCStreamer *createMCObjectStreamer(StringRef TT, MCContext &Ctx,
+                                       MCAsmBackend &TAB,
+                                       raw_ostream &_OS,
+                                       MCCodeEmitter *_Emitter,
+                                       bool RelaxAll,
+                                       bool NoExecStack) const {
+      if (!MCObjectStreamerCtorFn)
         return 0;
-      return ObjectStreamerCtorFn(*this, TT, Ctx, TAB, _OS, _Emitter, RelaxAll,
-                                  NoExecStack);
+      return MCObjectStreamerCtorFn(*this, TT, Ctx, TAB, _OS, _Emitter,
+                                    RelaxAll, NoExecStack);
     }
 
     /// createAsmStreamer - Create a target specific MCStreamer.
@@ -389,7 +428,7 @@ namespace llvm {
                                   bool useCFI,
                                   MCInstPrinter *InstPrint,
                                   MCCodeEmitter *CE,
-                                  TargetAsmBackend *TAB,
+                                  MCAsmBackend *TAB,
                                   bool ShowInst) const {
       // AsmStreamerCtorFn is default to llvm::createAsmStreamer
       return AsmStreamerCtorFn(Ctx, OS, isVerboseAsm, useLoc, useCFI,
@@ -438,6 +477,10 @@ namespace llvm {
       }
     };
 
+    /// printRegisteredTargetsForVersion - Print the registered targets
+    /// appropriately for inclusion in a tool's version output.
+    static void printRegisteredTargetsForVersion();
+
     /// @name Registry Access
     /// @{
 
@@ -500,6 +543,22 @@ namespace llvm {
         T.MCAsmInfoCtorFn = Fn;
     }
 
+    /// RegisterMCCodeGenInfo - Register a MCCodeGenInfo implementation for the
+    /// given target.
+    ///
+    /// Clients are responsible for ensuring that registration doesn't occur
+    /// while another thread is attempting to access the registry. Typically
+    /// this is done by initializing all targets at program startup.
+    ///
+    /// @param T - The target being registered.
+    /// @param Fn - A function to construct a MCCodeGenInfo for the target.
+    static void RegisterMCCodeGenInfo(Target &T,
+                                     Target::MCCodeGenInfoCtorFnTy Fn) {
+      // Ignore duplicate registration.
+      if (!T.MCCodeGenInfoCtorFn)
+        T.MCCodeGenInfoCtorFn = Fn;
+    }
+
     /// RegisterMCInstrInfo - Register a MCInstrInfo implementation for the
     /// given target.
     ///
@@ -515,6 +574,15 @@ namespace llvm {
         T.MCInstrInfoCtorFn = Fn;
     }
 
+    /// RegisterMCInstrAnalysis - Register a MCInstrAnalysis implementation for
+    /// the given target.
+    static void RegisterMCInstrAnalysis(Target &T,
+                                        Target::MCInstrAnalysisCtorFnTy Fn) {
+      // Ignore duplicate registration.
+      if (!T.MCInstrAnalysisCtorFn)
+        T.MCInstrAnalysisCtorFn = Fn;
+    }
+
     /// RegisterMCRegInfo - Register a MCRegisterInfo implementation for the
     /// given target.
     ///
@@ -562,7 +630,7 @@ namespace llvm {
         T.TargetMachineCtorFn = Fn;
     }
 
-    /// RegisterAsmBackend - Register a TargetAsmBackend implementation for the
+    /// RegisterMCAsmBackend - Register a MCAsmBackend implementation for the
     /// given target.
     ///
     /// Clients are responsible for ensuring that registration doesn't occur
@@ -571,12 +639,12 @@ namespace llvm {
     ///
     /// @param T - The target being registered.
     /// @param Fn - A function to construct an AsmBackend for the target.
-    static void RegisterAsmBackend(Target &T, Target::AsmBackendCtorTy Fn) {
-      if (!T.AsmBackendCtorFn)
-        T.AsmBackendCtorFn = Fn;
+    static void RegisterMCAsmBackend(Target &T, Target::MCAsmBackendCtorTy Fn) {
+      if (!T.MCAsmBackendCtorFn)
+        T.MCAsmBackendCtorFn = Fn;
     }
 
-    /// RegisterAsmLexer - Register a TargetAsmLexer implementation for the
+    /// RegisterMCAsmLexer - Register a MCTargetAsmLexer implementation for the
     /// given target.
     ///
     /// Clients are responsible for ensuring that registration doesn't occur
@@ -584,24 +652,24 @@ namespace llvm {
     /// this is done by initializing all targets at program startup.
     ///
     /// @param T - The target being registered.
-    /// @param Fn - A function to construct an AsmLexer for the target.
-    static void RegisterAsmLexer(Target &T, Target::AsmLexerCtorTy Fn) {
-      if (!T.AsmLexerCtorFn)
-        T.AsmLexerCtorFn = Fn;
+    /// @param Fn - A function to construct an MCAsmLexer for the target.
+    static void RegisterMCAsmLexer(Target &T, Target::MCAsmLexerCtorTy Fn) {
+      if (!T.MCAsmLexerCtorFn)
+        T.MCAsmLexerCtorFn = Fn;
     }
 
-    /// RegisterAsmParser - Register a TargetAsmParser implementation for the
-    /// given target.
+    /// RegisterMCAsmParser - Register a MCTargetAsmParser implementation for
+    /// the given target.
     ///
     /// Clients are responsible for ensuring that registration doesn't occur
     /// while another thread is attempting to access the registry. Typically
     /// this is done by initializing all targets at program startup.
     ///
     /// @param T - The target being registered.
-    /// @param Fn - A function to construct an AsmParser for the target.
-    static void RegisterAsmParser(Target &T, Target::AsmParserCtorTy Fn) {
-      if (!T.AsmParserCtorFn)
-        T.AsmParserCtorFn = Fn;
+    /// @param Fn - A function to construct an MCTargetAsmParser for the target.
+    static void RegisterMCAsmParser(Target &T, Target::MCAsmParserCtorTy Fn) {
+      if (!T.MCAsmParserCtorFn)
+        T.MCAsmParserCtorFn = Fn;
     }
 
     /// RegisterAsmPrinter - Register an AsmPrinter implementation for the given
@@ -649,7 +717,7 @@ namespace llvm {
         T.MCInstPrinterCtorFn = Fn;
     }
 
-    /// RegisterCodeEmitter - Register a MCCodeEmitter implementation for the
+    /// RegisterMCCodeEmitter - Register a MCCodeEmitter implementation for the
     /// given target.
     ///
     /// Clients are responsible for ensuring that registration doesn't occur
@@ -658,13 +726,14 @@ namespace llvm {
     ///
     /// @param T - The target being registered.
     /// @param Fn - A function to construct an MCCodeEmitter for the target.
-    static void RegisterCodeEmitter(Target &T, Target::CodeEmitterCtorTy Fn) {
-      if (!T.CodeEmitterCtorFn)
-        T.CodeEmitterCtorFn = Fn;
+    static void RegisterMCCodeEmitter(Target &T,
+                                      Target::MCCodeEmitterCtorTy Fn) {
+      if (!T.MCCodeEmitterCtorFn)
+        T.MCCodeEmitterCtorFn = Fn;
     }
 
-    /// RegisterObjectStreamer - Register a object code MCStreamer implementation
-    /// for the given target.
+    /// RegisterMCObjectStreamer - Register a object code MCStreamer
+    /// implementation for the given target.
     ///
     /// Clients are responsible for ensuring that registration doesn't occur
     /// while another thread is attempting to access the registry. Typically
@@ -672,9 +741,10 @@ namespace llvm {
     ///
     /// @param T - The target being registered.
     /// @param Fn - A function to construct an MCStreamer for the target.
-    static void RegisterObjectStreamer(Target &T, Target::ObjectStreamerCtorTy Fn) {
-      if (!T.ObjectStreamerCtorFn)
-        T.ObjectStreamerCtorFn = Fn;
+    static void RegisterMCObjectStreamer(Target &T,
+                                         Target::MCObjectStreamerCtorTy Fn) {
+      if (!T.MCObjectStreamerCtorFn)
+        T.MCObjectStreamerCtorFn = Fn;
     }
 
     /// RegisterAsmStreamer - Register an assembly MCStreamer implementation
@@ -756,6 +826,40 @@ namespace llvm {
     }
   };
 
+  /// RegisterMCCodeGenInfo - Helper template for registering a target codegen info
+  /// implementation.  This invokes the static "Create" method on the class
+  /// to actually do the construction.  Usage:
+  ///
+  /// extern "C" void LLVMInitializeFooTarget() {
+  ///   extern Target TheFooTarget;
+  ///   RegisterMCCodeGenInfo<FooMCCodeGenInfo> X(TheFooTarget);
+  /// }
+  template<class MCCodeGenInfoImpl>
+  struct RegisterMCCodeGenInfo {
+    RegisterMCCodeGenInfo(Target &T) {
+      TargetRegistry::RegisterMCCodeGenInfo(T, &Allocator);
+    }
+  private:
+    static MCCodeGenInfo *Allocator(StringRef TT,
+                                    Reloc::Model RM, CodeModel::Model CM) {
+      return new MCCodeGenInfoImpl();
+    }
+  };
+
+  /// RegisterMCCodeGenInfoFn - Helper template for registering a target codegen
+  /// info implementation.  This invokes the specified function to do the
+  /// construction.  Usage:
+  ///
+  /// extern "C" void LLVMInitializeFooTarget() {
+  ///   extern Target TheFooTarget;
+  ///   RegisterMCCodeGenInfoFn X(TheFooTarget, TheFunction);
+  /// }
+  struct RegisterMCCodeGenInfoFn {
+    RegisterMCCodeGenInfoFn(Target &T, Target::MCCodeGenInfoCtorFnTy Fn) {
+      TargetRegistry::RegisterMCCodeGenInfo(T, Fn);
+    }
+  };
+
   /// RegisterMCInstrInfo - Helper template for registering a target instruction
   /// info implementation.  This invokes the static "Create" method on the class
   /// to actually do the construction.  Usage:
@@ -789,6 +893,39 @@ namespace llvm {
     }
   };
 
+  /// RegisterMCInstrAnalysis - Helper template for registering a target
+  /// instruction analyzer implementation.  This invokes the static "Create"
+  /// method on the class to actually do the construction.  Usage:
+  ///
+  /// extern "C" void LLVMInitializeFooTarget() {
+  ///   extern Target TheFooTarget;
+  ///   RegisterMCInstrAnalysis<FooMCInstrAnalysis> X(TheFooTarget);
+  /// }
+  template<class MCInstrAnalysisImpl>
+  struct RegisterMCInstrAnalysis {
+    RegisterMCInstrAnalysis(Target &T) {
+      TargetRegistry::RegisterMCInstrAnalysis(T, &Allocator);
+    }
+  private:
+    static MCInstrAnalysis *Allocator(const MCInstrInfo *Info) {
+      return new MCInstrAnalysisImpl(Info);
+    }
+  };
+
+  /// RegisterMCInstrAnalysisFn - Helper template for registering a target
+  /// instruction analyzer implementation.  This invokes the specified function
+  /// to do the construction.  Usage:
+  ///
+  /// extern "C" void LLVMInitializeFooTarget() {
+  ///   extern Target TheFooTarget;
+  ///   RegisterMCInstrAnalysisFn X(TheFooTarget, TheFunction);
+  /// }
+  struct RegisterMCInstrAnalysisFn {
+    RegisterMCInstrAnalysisFn(Target &T, Target::MCInstrAnalysisCtorFnTy Fn) {
+      TargetRegistry::RegisterMCInstrAnalysis(T, Fn);
+    }
+  };
+
   /// RegisterMCRegInfo - Helper template for registering a target register info
   /// implementation.  This invokes the static "Create" method on the class to
   /// actually do the construction.  Usage:
@@ -803,7 +940,7 @@ namespace llvm {
       TargetRegistry::RegisterMCRegInfo(T, &Allocator);
     }
   private:
-    static MCRegisterInfo *Allocator() {
+    static MCRegisterInfo *Allocator(StringRef TT) {
       return new MCRegisterInfoImpl();
     }
   };
@@ -871,70 +1008,72 @@ namespace llvm {
     }
 
   private:
-    static TargetMachine *Allocator(const Target &T, const std::string &TT,
-                                    const std::string &CPU,
-                                    const std::string &FS) {
-      return new TargetMachineImpl(T, TT, CPU, FS);
+    static TargetMachine *Allocator(const Target &T, StringRef TT,
+                                    StringRef CPU, StringRef FS,
+                                    Reloc::Model RM,
+                                    CodeModel::Model CM) {
+      return new TargetMachineImpl(T, TT, CPU, FS, RM, CM);
     }
   };
 
-  /// RegisterAsmBackend - Helper template for registering a target specific
+  /// RegisterMCAsmBackend - Helper template for registering a target specific
   /// assembler backend. Usage:
   ///
-  /// extern "C" void LLVMInitializeFooAsmBackend() {
+  /// extern "C" void LLVMInitializeFooMCAsmBackend() {
   ///   extern Target TheFooTarget;
-  ///   RegisterAsmBackend<FooAsmLexer> X(TheFooTarget);
+  ///   RegisterMCAsmBackend<FooAsmLexer> X(TheFooTarget);
   /// }
-  template<class AsmBackendImpl>
-  struct RegisterAsmBackend {
-    RegisterAsmBackend(Target &T) {
-      TargetRegistry::RegisterAsmBackend(T, &Allocator);
+  template<class MCAsmBackendImpl>
+  struct RegisterMCAsmBackend {
+    RegisterMCAsmBackend(Target &T) {
+      TargetRegistry::RegisterMCAsmBackend(T, &Allocator);
     }
 
   private:
-    static TargetAsmBackend *Allocator(const Target &T,
-                                       const std::string &Triple) {
-      return new AsmBackendImpl(T, Triple);
+    static MCAsmBackend *Allocator(const Target &T, StringRef Triple) {
+      return new MCAsmBackendImpl(T, Triple);
     }
   };
 
-  /// RegisterAsmLexer - Helper template for registering a target specific
+  /// RegisterMCAsmLexer - Helper template for registering a target specific
   /// assembly lexer, for use in the target machine initialization
   /// function. Usage:
   ///
-  /// extern "C" void LLVMInitializeFooAsmLexer() {
+  /// extern "C" void LLVMInitializeFooMCAsmLexer() {
   ///   extern Target TheFooTarget;
-  ///   RegisterAsmLexer<FooAsmLexer> X(TheFooTarget);
+  ///   RegisterMCAsmLexer<FooMCAsmLexer> X(TheFooTarget);
   /// }
-  template<class AsmLexerImpl>
-  struct RegisterAsmLexer {
-    RegisterAsmLexer(Target &T) {
-      TargetRegistry::RegisterAsmLexer(T, &Allocator);
+  template<class MCAsmLexerImpl>
+  struct RegisterMCAsmLexer {
+    RegisterMCAsmLexer(Target &T) {
+      TargetRegistry::RegisterMCAsmLexer(T, &Allocator);
     }
 
   private:
-    static TargetAsmLexer *Allocator(const Target &T, const MCAsmInfo &MAI) {
-      return new AsmLexerImpl(T, MAI);
+    static MCTargetAsmLexer *Allocator(const Target &T,
+                                       const MCRegisterInfo &MRI,
+                                       const MCAsmInfo &MAI) {
+      return new MCAsmLexerImpl(T, MRI, MAI);
     }
   };
 
-  /// RegisterAsmParser - Helper template for registering a target specific
+  /// RegisterMCAsmParser - Helper template for registering a target specific
   /// assembly parser, for use in the target machine initialization
   /// function. Usage:
   ///
-  /// extern "C" void LLVMInitializeFooAsmParser() {
+  /// extern "C" void LLVMInitializeFooMCAsmParser() {
   ///   extern Target TheFooTarget;
-  ///   RegisterAsmParser<FooAsmParser> X(TheFooTarget);
+  ///   RegisterMCAsmParser<FooAsmParser> X(TheFooTarget);
   /// }
-  template<class AsmParserImpl>
-  struct RegisterAsmParser {
-    RegisterAsmParser(Target &T) {
-      TargetRegistry::RegisterAsmParser(T, &Allocator);
+  template<class MCAsmParserImpl>
+  struct RegisterMCAsmParser {
+    RegisterMCAsmParser(Target &T) {
+      TargetRegistry::RegisterMCAsmParser(T, &Allocator);
     }
 
   private:
-    static TargetAsmParser *Allocator(MCSubtargetInfo &STI, MCAsmParser &P) {
-      return new AsmParserImpl(STI, P);
+    static MCTargetAsmParser *Allocator(MCSubtargetInfo &STI, MCAsmParser &P) {
+      return new MCAsmParserImpl(STI, P);
     }
   };
 
@@ -958,25 +1097,25 @@ namespace llvm {
     }
   };
 
-  /// RegisterCodeEmitter - Helper template for registering a target specific
+  /// RegisterMCCodeEmitter - Helper template for registering a target specific
   /// machine code emitter, for use in the target initialization
   /// function. Usage:
   ///
-  /// extern "C" void LLVMInitializeFooCodeEmitter() {
+  /// extern "C" void LLVMInitializeFooMCCodeEmitter() {
   ///   extern Target TheFooTarget;
-  ///   RegisterCodeEmitter<FooCodeEmitter> X(TheFooTarget);
+  ///   RegisterMCCodeEmitter<FooCodeEmitter> X(TheFooTarget);
   /// }
-  template<class CodeEmitterImpl>
-  struct RegisterCodeEmitter {
-    RegisterCodeEmitter(Target &T) {
-      TargetRegistry::RegisterCodeEmitter(T, &Allocator);
+  template<class MCCodeEmitterImpl>
+  struct RegisterMCCodeEmitter {
+    RegisterMCCodeEmitter(Target &T) {
+      TargetRegistry::RegisterMCCodeEmitter(T, &Allocator);
     }
 
   private:
     static MCCodeEmitter *Allocator(const MCInstrInfo &II,
                                     const MCSubtargetInfo &STI,
                                     MCContext &Ctx) {
-      return new CodeEmitterImpl();
+      return new MCCodeEmitterImpl();
     }
   };
 
diff --git a/contrib/llvm/include/llvm/Target/TargetSelect.h b/contrib/llvm/include/llvm/Support/TargetSelect.h
similarity index 75%
rename from contrib/llvm/include/llvm/Target/TargetSelect.h
rename to contrib/llvm/include/llvm/Support/TargetSelect.h
index 272ee09464f9..83ff68caaeac 100644
--- a/contrib/llvm/include/llvm/Target/TargetSelect.h
+++ b/contrib/llvm/include/llvm/Support/TargetSelect.h
@@ -13,8 +13,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TARGET_TARGETSELECT_H
-#define LLVM_TARGET_TARGETSELECT_H
+#ifndef LLVM_SUPPORT_TARGETSELECT_H
+#define LLVM_SUPPORT_TARGETSELECT_H
 
 #include "llvm/Config/llvm-config.h"
 
@@ -26,18 +26,10 @@ extern "C" {
 #define LLVM_TARGET(TargetName) void LLVMInitialize##TargetName##Target();
 #include "llvm/Config/Targets.def"
   
-#define LLVM_TARGET(TargetName) \
-  void LLVMInitialize##TargetName##MCAsmInfo();
+  // Declare all of the target-MC-initialization functions that are available.
+#define LLVM_TARGET(TargetName) void LLVMInitialize##TargetName##TargetMC();
 #include "llvm/Config/Targets.def"
-
-#define LLVM_TARGET(TargetName) \
-  void LLVMInitialize##TargetName##MCInstrInfo();
-#include "llvm/Config/Targets.def"
-
-#define LLVM_TARGET(TargetName) \
-  void LLVMInitialize##TargetName##MCSubtargetInfo();
-#include "llvm/Config/Targets.def"
-
+  
   // Declare all of the available assembly printer initialization functions.
 #define LLVM_ASM_PRINTER(TargetName) void LLVMInitialize##TargetName##AsmPrinter();
 #include "llvm/Config/AsmPrinters.def"
@@ -76,35 +68,13 @@ namespace llvm {
 #include "llvm/Config/Targets.def"
   }
   
-  /// InitializeAllMCAsmInfos - The main program should call this function
-  /// if it wants access to all available assembly infos for targets that
-  /// LLVM is configured to support, to make them available via the
-  /// TargetRegistry.
+  /// InitializeAllTargetMCs - The main program should call this function if it
+  /// wants access to all available target MC that LLVM is configured to
+  /// support, to make them available via the TargetRegistry.
   ///
   /// It is legal for a client to make multiple calls to this function.
-  inline void InitializeAllMCAsmInfos() {
-#define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##MCAsmInfo();
-#include "llvm/Config/Targets.def"
-  }
-  
-  /// InitializeAllMCInstrInfos - The main program should call this function
-  /// if it wants access to all available instruction infos for targets that
-  /// LLVM is configured to support, to make them available via the
-  /// TargetRegistry.
-  ///
-  /// It is legal for a client to make multiple calls to this function.
-  inline void InitializeAllMCInstrInfos() {
-#define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##MCInstrInfo();
-#include "llvm/Config/Targets.def"
-  }
-  
-  /// InitializeAllMCSubtargetInfos - The main program should call this function
-  /// if it wants access to all available subtarget infos for targets that LLVM
-  /// is configured to support, to make them available via the TargetRegistry.
-  ///
-  /// It is legal for a client to make multiple calls to this function.
-  inline void InitializeAllMCSubtargetInfos() {
-#define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##MCSubtargetInfo();
+  inline void InitializeAllTargetMCs() {
+#define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##TargetMC();
 #include "llvm/Config/Targets.def"
   }
   
@@ -148,7 +118,7 @@ namespace llvm {
 #ifdef LLVM_NATIVE_TARGET
     LLVM_NATIVE_TARGETINFO();
     LLVM_NATIVE_TARGET();
-    LLVM_NATIVE_MCASMINFO();
+    LLVM_NATIVE_TARGETMC();
     return false;
 #else
     return true;
diff --git a/contrib/llvm/include/llvm/Support/TypeBuilder.h b/contrib/llvm/include/llvm/Support/TypeBuilder.h
index 18007789736a..c75606917c1c 100644
--- a/contrib/llvm/include/llvm/Support/TypeBuilder.h
+++ b/contrib/llvm/include/llvm/Support/TypeBuilder.h
@@ -18,7 +18,6 @@
 #include "llvm/DerivedTypes.h"
 #include "llvm/LLVMContext.h"
 #include <limits.h>
-#include <vector>
 
 namespace llvm {
 
@@ -254,9 +253,9 @@ template<typename R, bool cross> class TypeBuilder<R(), cross> {
 template<typename R, typename A1, bool cross> class TypeBuilder<R(A1), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(1);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                              params, false);
   }
@@ -265,10 +264,10 @@ template<typename R, typename A1, typename A2, bool cross>
 class TypeBuilder<R(A1, A2), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(2);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                              params, false);
   }
@@ -277,11 +276,11 @@ template<typename R, typename A1, typename A2, typename A3, bool cross>
 class TypeBuilder<R(A1, A2, A3), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(3);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
-    params.push_back(TypeBuilder<A3, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+      TypeBuilder<A3, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                              params, false);
   }
@@ -292,12 +291,12 @@ template<typename R, typename A1, typename A2, typename A3, typename A4,
 class TypeBuilder<R(A1, A2, A3, A4), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(4);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
-    params.push_back(TypeBuilder<A3, cross>::get(Context));
-    params.push_back(TypeBuilder<A4, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+      TypeBuilder<A3, cross>::get(Context),
+      TypeBuilder<A4, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                              params, false);
   }
@@ -308,13 +307,13 @@ template<typename R, typename A1, typename A2, typename A3, typename A4,
 class TypeBuilder<R(A1, A2, A3, A4, A5), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(5);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
-    params.push_back(TypeBuilder<A3, cross>::get(Context));
-    params.push_back(TypeBuilder<A4, cross>::get(Context));
-    params.push_back(TypeBuilder<A5, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+      TypeBuilder<A3, cross>::get(Context),
+      TypeBuilder<A4, cross>::get(Context),
+      TypeBuilder<A5, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                              params, false);
   }
@@ -330,9 +329,9 @@ template<typename R, typename A1, bool cross>
 class TypeBuilder<R(A1, ...), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(1);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context), params, true);
   }
 };
@@ -340,10 +339,10 @@ template<typename R, typename A1, typename A2, bool cross>
 class TypeBuilder<R(A1, A2, ...), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(2);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                                    params, true);
   }
@@ -352,11 +351,11 @@ template<typename R, typename A1, typename A2, typename A3, bool cross>
 class TypeBuilder<R(A1, A2, A3, ...), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(3);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
-    params.push_back(TypeBuilder<A3, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+      TypeBuilder<A3, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                                    params, true);
   }
@@ -367,12 +366,12 @@ template<typename R, typename A1, typename A2, typename A3, typename A4,
 class TypeBuilder<R(A1, A2, A3, A4, ...), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(4);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
-    params.push_back(TypeBuilder<A3, cross>::get(Context));
-    params.push_back(TypeBuilder<A4, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+      TypeBuilder<A3, cross>::get(Context),
+      TypeBuilder<A4, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                              params, true);
   }
@@ -383,13 +382,13 @@ template<typename R, typename A1, typename A2, typename A3, typename A4,
 class TypeBuilder<R(A1, A2, A3, A4, A5, ...), cross> {
 public:
   static FunctionType *get(LLVMContext &Context) {
-    std::vector<Type*> params;
-    params.reserve(5);
-    params.push_back(TypeBuilder<A1, cross>::get(Context));
-    params.push_back(TypeBuilder<A2, cross>::get(Context));
-    params.push_back(TypeBuilder<A3, cross>::get(Context));
-    params.push_back(TypeBuilder<A4, cross>::get(Context));
-    params.push_back(TypeBuilder<A5, cross>::get(Context));
+    Type *params[] = {
+      TypeBuilder<A1, cross>::get(Context),
+      TypeBuilder<A2, cross>::get(Context),
+      TypeBuilder<A3, cross>::get(Context),
+      TypeBuilder<A4, cross>::get(Context),
+      TypeBuilder<A5, cross>::get(Context),
+    };
     return FunctionType::get(TypeBuilder<R, cross>::get(Context),
                                    params, true);
   }
diff --git a/contrib/llvm/utils/TableGen/Error.h b/contrib/llvm/include/llvm/TableGen/Error.h
similarity index 89%
rename from contrib/llvm/utils/TableGen/Error.h
rename to contrib/llvm/include/llvm/TableGen/Error.h
index b3a01461940d..c01b32b1c2d2 100644
--- a/contrib/llvm/utils/TableGen/Error.h
+++ b/contrib/llvm/include/llvm/TableGen/Error.h
@@ -1,4 +1,4 @@
-//===- Error.h - tblgen error handling helper routines ----------*- C++ -*-===//
+//===- llvm/TableGen/Error.h - tblgen error handling helpers ----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef ERROR_H
-#define ERROR_H
+#ifndef LLVM_TABLEGEN_ERROR_H
+#define LLVM_TABLEGEN_ERROR_H
 
 #include "llvm/Support/SourceMgr.h"
 
diff --git a/contrib/llvm/include/llvm/TableGen/Main.h b/contrib/llvm/include/llvm/TableGen/Main.h
new file mode 100644
index 000000000000..deaef4a9908a
--- /dev/null
+++ b/contrib/llvm/include/llvm/TableGen/Main.h
@@ -0,0 +1,26 @@
+//===- llvm/TableGen/Main.h - tblgen entry point ----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the common entry point for tblgen tools.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TABLEGEN_MAIN_H
+#define LLVM_TABLEGEN_MAIN_H
+
+namespace llvm {
+
+class TableGenAction;
+
+/// Run the table generator, performing the specified Action on parsed records.
+int TableGenMain(char *argv0, TableGenAction &Action);
+
+}
+
+#endif
diff --git a/contrib/llvm/utils/TableGen/Record.h b/contrib/llvm/include/llvm/TableGen/Record.h
similarity index 80%
rename from contrib/llvm/utils/TableGen/Record.h
rename to contrib/llvm/include/llvm/TableGen/Record.h
index 2f4080bbfd9c..afce76099867 100644
--- a/contrib/llvm/utils/TableGen/Record.h
+++ b/contrib/llvm/include/llvm/TableGen/Record.h
@@ -1,4 +1,4 @@
-//===- Record.h - Classes to represent Table Records ------------*- C++ -*-===//
+//===- llvm/TableGen/Record.h - Classes for Table Records -------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,9 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef RECORD_H
-#define RECORD_H
+#ifndef LLVM_TABLEGEN_RECORD_H
+#define LLVM_TABLEGEN_RECORD_H
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/raw_ostream.h"
@@ -63,7 +66,10 @@ class RecordKeeper;
 //  Type Classes
 //===----------------------------------------------------------------------===//
 
-struct RecTy {
+class RecTy {
+  ListRecTy *ListTy;
+public:
+  RecTy() : ListTy(0) {}
   virtual ~RecTy() {}
 
   virtual std::string getAsString() const = 0;
@@ -74,6 +80,9 @@ struct RecTy {
   /// converted to the specified type.
   virtual bool typeIsConvertibleTo(const RecTy *RHS) const = 0;
 
+  /// getListTy - Returns the type representing list<this>.
+  ListRecTy *getListTy();
+
 public:   // These methods should only be called from subclasses of Init
   virtual Init *convertValue( UnsetInit *UI) { return 0; }
   virtual Init *convertValue(   BitInit *BI) { return 0; }
@@ -124,7 +133,11 @@ inline raw_ostream &operator<<(raw_ostream &OS, const RecTy &Ty) {
 /// BitRecTy - 'bit' - Represent a single bit
 ///
 class BitRecTy : public RecTy {
+  static BitRecTy Shared;
+  BitRecTy() {}
 public:
+  static BitRecTy *get() { return &Shared; }
+
   virtual Init *convertValue( UnsetInit *UI) { return (Init*)UI; }
   virtual Init *convertValue(   BitInit *BI) { return (Init*)BI; }
   virtual Init *convertValue(  BitsInit *BI);
@@ -164,8 +177,9 @@ class BitRecTy : public RecTy {
 ///
 class BitsRecTy : public RecTy {
   unsigned Size;
-public:
   explicit BitsRecTy(unsigned Sz) : Size(Sz) {}
+public:
+  static BitsRecTy *get(unsigned Sz);
 
   unsigned getNumBits() const { return Size; }
 
@@ -208,7 +222,11 @@ class BitsRecTy : public RecTy {
 /// IntRecTy - 'int' - Represent an integer value of no particular size
 ///
 class IntRecTy : public RecTy {
+  static IntRecTy Shared;
+  IntRecTy() {}
 public:
+  static IntRecTy *get() { return &Shared; }
+
   virtual Init *convertValue( UnsetInit *UI) { return (Init*)UI; }
   virtual Init *convertValue(   BitInit *BI);
   virtual Init *convertValue(  BitsInit *BI);
@@ -246,7 +264,11 @@ class IntRecTy : public RecTy {
 /// StringRecTy - 'string' - Represent an string value
 ///
 class StringRecTy : public RecTy {
+  static StringRecTy Shared;
+  StringRecTy() {}
 public:
+  static StringRecTy *get() { return &Shared; }
+
   virtual Init *convertValue( UnsetInit *UI) { return (Init*)UI; }
   virtual Init *convertValue(   BitInit *BI) { return 0; }
   virtual Init *convertValue(  BitsInit *BI) { return 0; }
@@ -288,9 +310,10 @@ class StringRecTy : public RecTy {
 ///
 class ListRecTy : public RecTy {
   RecTy *Ty;
-public:
   explicit ListRecTy(RecTy *T) : Ty(T) {}
-
+  friend ListRecTy *RecTy::getListTy();
+public:
+  static ListRecTy *get(RecTy *T) { return T->getListTy(); }
   RecTy *getElementType() const { return Ty; }
 
   virtual Init *convertValue( UnsetInit *UI) { return (Init*)UI; }
@@ -331,7 +354,11 @@ class ListRecTy : public RecTy {
 /// CodeRecTy - 'code' - Represent an code fragment, function or method.
 ///
 class CodeRecTy : public RecTy {
+  static CodeRecTy Shared;
+  CodeRecTy() {}
 public:
+  static CodeRecTy *get() { return &Shared; }
+
   virtual Init *convertValue( UnsetInit *UI) { return (Init*)UI; }
   virtual Init *convertValue(   BitInit *BI) { return 0; }
   virtual Init *convertValue(  BitsInit *BI) { return 0; }
@@ -367,7 +394,11 @@ class CodeRecTy : public RecTy {
 /// DagRecTy - 'dag' - Represent a dag fragment
 ///
 class DagRecTy : public RecTy {
+  static DagRecTy Shared;
+  DagRecTy() {}
 public:
+  static DagRecTy *get() { return &Shared; }
+
   virtual Init *convertValue( UnsetInit *UI) { return (Init*)UI; }
   virtual Init *convertValue(   BitInit *BI) { return 0; }
   virtual Init *convertValue(  BitsInit *BI) { return 0; }
@@ -407,8 +438,10 @@ class DagRecTy : public RecTy {
 ///
 class RecordRecTy : public RecTy {
   Record *Rec;
-public:
   explicit RecordRecTy(Record *R) : Rec(R) {}
+  friend class Record;
+public:
+  static RecordRecTy *get(Record *R);
 
   Record *getRecord() const { return Rec; }
 
@@ -454,6 +487,12 @@ RecTy *resolveTypes(RecTy *T1, RecTy *T2);
 //===----------------------------------------------------------------------===//
 
 class Init {
+  Init(const Init &);  // Do not define.
+  Init &operator=(const Init &);  // Do not define.
+
+protected:
+  Init(void) {}
+
 public:
   virtual ~Init() {}
 
@@ -466,6 +505,11 @@ class Init {
 
   /// getAsString - Convert this value to a string form.
   virtual std::string getAsString() const = 0;
+  /// getAsUnquotedString - Convert this value to a string form,
+  /// without adding quote markers.  This primaruly affects
+  /// StringInits where we will not surround the string value with
+  /// quotes.
+  virtual std::string getAsUnquotedString() const { return getAsString(); }  
 
   /// dump - Debugging method that may be called through a debugger, just
   /// invokes print on stderr.
@@ -475,14 +519,15 @@ class Init {
   /// function that should be overridden to call the appropriate
   /// RecTy::convertValue method.
   ///
-  virtual Init *convertInitializerTo(RecTy *Ty) = 0;
+  virtual Init *convertInitializerTo(RecTy *Ty) const = 0;
 
   /// convertInitializerBitRange - This method is used to implement the bitrange
   /// selection operator.  Given an initializer, it selects the specified bits
   /// out, returning them as a new init of bits type.  If it is not legal to use
   /// the bit subscript operator on this initializer, return null.
   ///
-  virtual Init *convertInitializerBitRange(const std::vector<unsigned> &Bits) {
+  virtual Init *
+  convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
     return 0;
   }
 
@@ -491,7 +536,8 @@ class Init {
   /// elements, returning them as a new init of list type.  If it is not legal
   /// to take a slice of this, return null.
   ///
-  virtual Init *convertInitListSlice(const std::vector<unsigned> &Elements) {
+  virtual Init *
+  convertInitListSlice(const std::vector<unsigned> &Elements) const {
     return 0;
   }
 
@@ -515,8 +561,8 @@ class Init {
   /// If a value is set for the variable later, this method will be called on
   /// users of the value to allow the value to propagate out.
   ///
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV) {
-    return this;
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const {
+    return const_cast<Init *>(this);
   }
 };
 
@@ -529,13 +575,20 @@ inline raw_ostream &operator<<(raw_ostream &OS, const Init &I) {
 ///
 class TypedInit : public Init {
   RecTy *Ty;
-public:
+
+  TypedInit(const TypedInit &Other);  // Do not define.
+  TypedInit &operator=(const TypedInit &Other);  // Do not define.
+
+protected:
   explicit TypedInit(RecTy *T) : Ty(T) {}
 
+public:
   RecTy *getType() const { return Ty; }
 
-  virtual Init *convertInitializerBitRange(const std::vector<unsigned> &Bits);
-  virtual Init *convertInitListSlice(const std::vector<unsigned> &Elements);
+  virtual Init *
+  convertInitializerBitRange(const std::vector<unsigned> &Bits) const;
+  virtual Init *
+  convertInitListSlice(const std::vector<unsigned> &Elements) const;
 
   /// getFieldType - This method is used to implement the FieldInit class.
   /// Implementors of this method should return the type of the named field if
@@ -548,22 +601,28 @@ class TypedInit : public Init {
   /// simply return the resolved value, otherwise we return null.
   ///
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit) = 0;
+                                    unsigned Bit) const = 0;
 
   /// resolveListElementReference - This method is used to implement
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
   /// now, we return the resolved value, otherwise we return null.
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt) = 0;
+                                            unsigned Elt) const = 0;
 };
 
 
 /// UnsetInit - ? - Represents an uninitialized value
 ///
 class UnsetInit : public Init {
+  UnsetInit() : Init() {}
+  UnsetInit(const UnsetInit &);  // Do not define.
+  UnsetInit &operator=(const UnsetInit &Other);  // Do not define.
+
 public:
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  static UnsetInit *get();
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<UnsetInit *>(this));
   }
 
   virtual bool isComplete() const { return false; }
@@ -575,13 +634,18 @@ class UnsetInit : public Init {
 ///
 class BitInit : public Init {
   bool Value;
-public:
+
   explicit BitInit(bool V) : Value(V) {}
+  BitInit(const BitInit &Other);  // Do not define.
+  BitInit &operator=(BitInit &Other);  // Do not define.
+
+public:
+  static BitInit *get(bool V);
 
   bool getValue() const { return Value; }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<BitInit *>(this));
   }
 
   virtual std::string getAsString() const { return Value ? "1" : "0"; }
@@ -590,10 +654,18 @@ class BitInit : public Init {
 /// BitsInit - { a, b, c } - Represents an initializer for a BitsRecTy value.
 /// It contains a vector of bits, whose size is determined by the type.
 ///
-class BitsInit : public Init {
+class BitsInit : public Init, public FoldingSetNode {
   std::vector<Init*> Bits;
+
+  BitsInit(ArrayRef<Init *> Range) : Bits(Range.begin(), Range.end()) {}
+
+  BitsInit(const BitsInit &Other);  // Do not define.
+  BitsInit &operator=(const BitsInit &Other);  // Do not define.
+
 public:
-  explicit BitsInit(unsigned Size) : Bits(Size) {}
+  static BitsInit *get(ArrayRef<Init *> Range);
+
+  void Profile(FoldingSetNodeID &ID) const;
 
   unsigned getNumBits() const { return Bits.size(); }
 
@@ -601,16 +673,12 @@ class BitsInit : public Init {
     assert(Bit < Bits.size() && "Bit index out of range!");
     return Bits[Bit];
   }
-  void setBit(unsigned Bit, Init *V) {
-    assert(Bit < Bits.size() && "Bit index out of range!");
-    assert(Bits[Bit] == 0 && "Bit already set!");
-    Bits[Bit] = V;
-  }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<BitsInit *>(this));
   }
-  virtual Init *convertInitializerBitRange(const std::vector<unsigned> &Bits);
+  virtual Init *
+  convertInitializerBitRange(const std::vector<unsigned> &Bits) const;
 
   virtual bool isComplete() const {
     for (unsigned i = 0; i != getNumBits(); ++i)
@@ -624,7 +692,7 @@ class BitsInit : public Init {
   }
   virtual std::string getAsString() const;
 
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 };
 
 
@@ -632,15 +700,22 @@ class BitsInit : public Init {
 ///
 class IntInit : public TypedInit {
   int64_t Value;
+
+  explicit IntInit(int64_t V) : TypedInit(IntRecTy::get()), Value(V) {}
+
+  IntInit(const IntInit &Other);  // Do not define.
+  IntInit &operator=(const IntInit &Other);  // Do note define.
+
 public:
-  explicit IntInit(int64_t V) : TypedInit(new IntRecTy), Value(V) {}
+  static IntInit *get(int64_t V);
 
   int64_t getValue() const { return Value; }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<IntInit *>(this));
   }
-  virtual Init *convertInitializerBitRange(const std::vector<unsigned> &Bits);
+  virtual Init *
+  convertInitializerBitRange(const std::vector<unsigned> &Bits) const;
 
   virtual std::string getAsString() const;
 
@@ -649,7 +724,7 @@ class IntInit : public TypedInit {
   /// simply return the resolved value, otherwise we return null.
   ///
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit) {
+                                    unsigned Bit) const {
     assert(0 && "Illegal bit reference off int");
     return 0;
   }
@@ -658,7 +733,7 @@ class IntInit : public TypedInit {
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
   /// now, we return the resolved value, otherwise we return null.
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt) {
+                                            unsigned Elt) const {
     assert(0 && "Illegal element reference off int");
     return 0;
   }
@@ -669,24 +744,31 @@ class IntInit : public TypedInit {
 ///
 class StringInit : public TypedInit {
   std::string Value;
-public:
+
   explicit StringInit(const std::string &V)
-    : TypedInit(new StringRecTy), Value(V) {}
+    : TypedInit(StringRecTy::get()), Value(V) {}
+
+  StringInit(const StringInit &Other);  // Do not define.
+  StringInit &operator=(const StringInit &Other);  // Do not define.
+
+public:
+  static StringInit *get(const std::string &V);
 
   const std::string &getValue() const { return Value; }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<StringInit *>(this));
   }
 
   virtual std::string getAsString() const { return "\"" + Value + "\""; }
+  virtual std::string getAsUnquotedString() const { return Value; }
 
   /// resolveBitReference - This method is used to implement
   /// VarBitInit::resolveReferences.  If the bit is able to be resolved, we
   /// simply return the resolved value, otherwise we return null.
   ///
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit) {
+                                    unsigned Bit) const {
     assert(0 && "Illegal bit reference off string");
     return 0;
   }
@@ -695,7 +777,7 @@ class StringInit : public TypedInit {
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
   /// now, we return the resolved value, otherwise we return null.
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt) {
+                                            unsigned Elt) const {
     assert(0 && "Illegal element reference off string");
     return 0;
   }
@@ -705,13 +787,19 @@ class StringInit : public TypedInit {
 ///
 class CodeInit : public Init {
   std::string Value;
-public:
+
   explicit CodeInit(const std::string &V) : Value(V) {}
 
+  CodeInit(const CodeInit &Other);  // Do not define.
+  CodeInit &operator=(const CodeInit &Other);  // Do not define.
+
+public:
+  static CodeInit *get(const std::string &V);
+
   const std::string &getValue() const { return Value; }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<CodeInit *>(this));
   }
 
   virtual std::string getAsString() const { return "[{" + Value + "}]"; }
@@ -719,18 +807,22 @@ class CodeInit : public Init {
 
 /// ListInit - [AL, AH, CL] - Represent a list of defs
 ///
-class ListInit : public TypedInit {
+class ListInit : public TypedInit, public FoldingSetNode {
   std::vector<Init*> Values;
 public:
-  typedef std::vector<Init*>::iterator       iterator;
   typedef std::vector<Init*>::const_iterator const_iterator;
 
-  explicit ListInit(std::vector<Init*> &Vs, RecTy *EltTy)
-    : TypedInit(new ListRecTy(EltTy)) {
-    Values.swap(Vs);
-  }
-  explicit ListInit(iterator Start, iterator End, RecTy *EltTy)
-      : TypedInit(new ListRecTy(EltTy)), Values(Start, End) {}
+private:
+  explicit ListInit(ArrayRef<Init *> Range, RecTy *EltTy)
+      : TypedInit(ListRecTy::get(EltTy)), Values(Range.begin(), Range.end()) {}
+
+  ListInit(const ListInit &Other);  // Do not define.
+  ListInit &operator=(const ListInit &Other);  // Do not define.
+
+public:
+  static ListInit *get(ArrayRef<Init *> Range, RecTy *EltTy);
+
+  void Profile(FoldingSetNodeID &ID) const;
 
   unsigned getSize() const { return Values.size(); }
   Init *getElement(unsigned i) const {
@@ -740,10 +832,10 @@ class ListInit : public TypedInit {
 
   Record *getElementAsRecord(unsigned i) const;
 
-  Init *convertInitListSlice(const std::vector<unsigned> &Elements);
+  Init *convertInitListSlice(const std::vector<unsigned> &Elements) const;
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<ListInit *>(this));
   }
 
   /// resolveReferences - This method is used by classes that refer to other
@@ -751,13 +843,13 @@ class ListInit : public TypedInit {
   /// If a value is set for the variable later, this method will be called on
   /// users of the value to allow the value to propagate out.
   ///
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const;
 
-  inline iterator       begin()       { return Values.begin(); }
+  ArrayRef<Init*> getValues() const { return Values; }
+
   inline const_iterator begin() const { return Values.begin(); }
-  inline iterator       end  ()       { return Values.end();   }
   inline const_iterator end  () const { return Values.end();   }
 
   inline size_t         size () const { return Values.size();  }
@@ -768,7 +860,7 @@ class ListInit : public TypedInit {
   /// simply return the resolved value, otherwise we return null.
   ///
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit) {
+                                    unsigned Bit) const {
     assert(0 && "Illegal bit reference off list");
     return 0;
   }
@@ -777,34 +869,38 @@ class ListInit : public TypedInit {
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
   /// now, we return the resolved value, otherwise we return null.
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt);
+                                            unsigned Elt) const;
 };
 
 
 /// OpInit - Base class for operators
 ///
 class OpInit : public TypedInit {
-public:
-  OpInit(RecTy *Type) : TypedInit(Type) {}
+  OpInit(const OpInit &Other);  // Do not define.
+  OpInit &operator=(OpInit &Other);  // Do not define.
 
+protected:
+  explicit OpInit(RecTy *Type) : TypedInit(Type) {}
+
+public:
   // Clone - Clone this operator, replacing arguments with the new list
-  virtual OpInit *clone(std::vector<Init *> &Operands) = 0;
+  virtual OpInit *clone(std::vector<Init *> &Operands) const = 0;
 
   virtual int getNumOperands() const = 0;
-  virtual Init *getOperand(int i) = 0;
+  virtual Init *getOperand(int i) const = 0;
 
   // Fold - If possible, fold this to a simpler init.  Return this if not
   // possible to fold.
-  virtual Init *Fold(Record *CurRec, MultiClass *CurMultiClass) = 0;
+  virtual Init *Fold(Record *CurRec, MultiClass *CurMultiClass) const = 0;
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<OpInit *>(this));
   }
 
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit);
+                                    unsigned Bit) const;
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt);
+                                            unsigned Elt) const;
 };
 
 
@@ -816,20 +912,25 @@ class UnOpInit : public OpInit {
 private:
   UnaryOp Opc;
   Init *LHS;
+
+  UnOpInit(UnaryOp opc, Init *lhs, RecTy *Type)
+      : OpInit(Type), Opc(opc), LHS(lhs) {}
+
+  UnOpInit(const UnOpInit &Other);  // Do not define.
+  UnOpInit &operator=(const UnOpInit &Other);  // Do not define.
+
 public:
-  UnOpInit(UnaryOp opc, Init *lhs, RecTy *Type) :
-      OpInit(Type), Opc(opc), LHS(lhs) {
-  }
+  static UnOpInit *get(UnaryOp opc, Init *lhs, RecTy *Type);
 
   // Clone - Clone this operator, replacing arguments with the new list
-  virtual OpInit *clone(std::vector<Init *> &Operands) {
+  virtual OpInit *clone(std::vector<Init *> &Operands) const {
     assert(Operands.size() == 1 &&
            "Wrong number of operands for unary operation");
-    return new UnOpInit(getOpcode(), *Operands.begin(), getType());
+    return UnOpInit::get(getOpcode(), *Operands.begin(), getType());
   }
 
   int getNumOperands() const { return 1; }
-  Init *getOperand(int i) {
+  Init *getOperand(int i) const {
     assert(i == 0 && "Invalid operand id for unary operator");
     return getOperand();
   }
@@ -839,9 +940,9 @@ class UnOpInit : public OpInit {
 
   // Fold - If possible, fold this to a simpler init.  Return this if not
   // possible to fold.
-  Init *Fold(Record *CurRec, MultiClass *CurMultiClass);
+  Init *Fold(Record *CurRec, MultiClass *CurMultiClass) const;
 
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const;
 };
@@ -854,20 +955,26 @@ class BinOpInit : public OpInit {
 private:
   BinaryOp Opc;
   Init *LHS, *RHS;
-public:
+
   BinOpInit(BinaryOp opc, Init *lhs, Init *rhs, RecTy *Type) :
-      OpInit(Type), Opc(opc), LHS(lhs), RHS(rhs) {
-  }
+      OpInit(Type), Opc(opc), LHS(lhs), RHS(rhs) {}
+
+  BinOpInit(const BinOpInit &Other);  // Do not define.
+  BinOpInit &operator=(const BinOpInit &Other);  // Do not define.
+
+public:
+  static BinOpInit *get(BinaryOp opc, Init *lhs, Init *rhs,
+                        RecTy *Type);
 
   // Clone - Clone this operator, replacing arguments with the new list
-  virtual OpInit *clone(std::vector<Init *> &Operands) {
+  virtual OpInit *clone(std::vector<Init *> &Operands) const {
     assert(Operands.size() == 2 &&
            "Wrong number of operands for binary operation");
-    return new BinOpInit(getOpcode(), Operands[0], Operands[1], getType());
+    return BinOpInit::get(getOpcode(), Operands[0], Operands[1], getType());
   }
 
   int getNumOperands() const { return 2; }
-  Init *getOperand(int i) {
+  Init *getOperand(int i) const {
     assert((i == 0 || i == 1) && "Invalid operand id for binary operator");
     if (i == 0) {
       return getLHS();
@@ -882,9 +989,9 @@ class BinOpInit : public OpInit {
 
   // Fold - If possible, fold this to a simpler init.  Return this if not
   // possible to fold.
-  Init *Fold(Record *CurRec, MultiClass *CurMultiClass);
+  Init *Fold(Record *CurRec, MultiClass *CurMultiClass) const;
 
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const;
 };
@@ -897,21 +1004,29 @@ class TernOpInit : public OpInit {
 private:
   TernaryOp Opc;
   Init *LHS, *MHS, *RHS;
+
+  TernOpInit(TernaryOp opc, Init *lhs, Init *mhs, Init *rhs,
+             RecTy *Type) :
+      OpInit(Type), Opc(opc), LHS(lhs), MHS(mhs), RHS(rhs) {}
+
+  TernOpInit(const TernOpInit &Other);  // Do not define.
+  TernOpInit &operator=(const TernOpInit &Other);  // Do not define.
+
 public:
-  TernOpInit(TernaryOp opc, Init *lhs, Init *mhs, Init *rhs, RecTy *Type) :
-      OpInit(Type), Opc(opc), LHS(lhs), MHS(mhs), RHS(rhs) {
-  }
+  static TernOpInit *get(TernaryOp opc, Init *lhs,
+                         Init *mhs, Init *rhs,
+                         RecTy *Type);
 
   // Clone - Clone this operator, replacing arguments with the new list
-  virtual OpInit *clone(std::vector<Init *> &Operands) {
+  virtual OpInit *clone(std::vector<Init *> &Operands) const {
     assert(Operands.size() == 3 &&
            "Wrong number of operands for ternary operation");
-    return new TernOpInit(getOpcode(), Operands[0], Operands[1], Operands[2],
-                          getType());
+    return TernOpInit::get(getOpcode(), Operands[0], Operands[1], Operands[2],
+                           getType());
   }
 
   int getNumOperands() const { return 3; }
-  Init *getOperand(int i) {
+  Init *getOperand(int i) const {
     assert((i == 0 || i == 1 || i == 2) &&
            "Invalid operand id for ternary operator");
     if (i == 0) {
@@ -930,11 +1045,11 @@ class TernOpInit : public OpInit {
 
   // Fold - If possible, fold this to a simpler init.  Return this if not
   // possible to fold.
-  Init *Fold(Record *CurRec, MultiClass *CurMultiClass);
+  Init *Fold(Record *CurRec, MultiClass *CurMultiClass) const;
 
   virtual bool isComplete() const { return false; }
 
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const;
 };
@@ -944,20 +1059,27 @@ class TernOpInit : public OpInit {
 ///
 class VarInit : public TypedInit {
   std::string VarName;
-public:
-  explicit VarInit(const std::string &VN, RecTy *T)
-    : TypedInit(T), VarName(VN) {}
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  explicit VarInit(const std::string &VN, RecTy *T)
+      : TypedInit(T), VarName(VN) {}
+
+  VarInit(const VarInit &Other);  // Do not define.
+  VarInit &operator=(const VarInit &Other);  // Do not define.
+
+public:
+  static VarInit *get(const std::string &VN, RecTy *T);
+  static VarInit *get(Init *VN, RecTy *T);
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<VarInit *>(this));
   }
 
   const std::string &getName() const { return VarName; }
 
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit);
+                                    unsigned Bit) const;
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt);
+                                            unsigned Elt) const;
 
   virtual RecTy *getFieldType(const std::string &FieldName) const;
   virtual Init *getFieldInit(Record &R, const RecordVal *RV,
@@ -968,7 +1090,7 @@ class VarInit : public TypedInit {
   /// If a value is set for the variable later, this method will be called on
   /// users of the value to allow the value to propagate out.
   ///
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const { return VarName; }
 };
@@ -979,22 +1101,28 @@ class VarInit : public TypedInit {
 class VarBitInit : public Init {
   TypedInit *TI;
   unsigned Bit;
-public:
+
   VarBitInit(TypedInit *T, unsigned B) : TI(T), Bit(B) {
     assert(T->getType() && dynamic_cast<BitsRecTy*>(T->getType()) &&
            ((BitsRecTy*)T->getType())->getNumBits() > B &&
            "Illegal VarBitInit expression!");
   }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  VarBitInit(const VarBitInit &Other);  // Do not define.
+  VarBitInit &operator=(const VarBitInit &Other);  // Do not define.
+
+public:
+  static VarBitInit *get(TypedInit *T, unsigned B);
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<VarBitInit *>(this));
   }
 
   TypedInit *getVariable() const { return TI; }
   unsigned getBitNum() const { return Bit; }
 
   virtual std::string getAsString() const;
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 };
 
 /// VarListElementInit - List[4] - Represent access to one element of a var or
@@ -1002,43 +1130,59 @@ class VarBitInit : public Init {
 class VarListElementInit : public TypedInit {
   TypedInit *TI;
   unsigned Element;
-public:
+
   VarListElementInit(TypedInit *T, unsigned E)
-    : TypedInit(dynamic_cast<ListRecTy*>(T->getType())->getElementType()),
-                TI(T), Element(E) {
+      : TypedInit(dynamic_cast<ListRecTy*>(T->getType())->getElementType()),
+          TI(T), Element(E) {
     assert(T->getType() && dynamic_cast<ListRecTy*>(T->getType()) &&
            "Illegal VarBitInit expression!");
   }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  VarListElementInit(const VarListElementInit &Other);  // Do not define.
+  VarListElementInit &operator=(const VarListElementInit &Other);  // Do
+                                                                   // not
+                                                                   // define.
+
+public:
+  static VarListElementInit *get(TypedInit *T, unsigned E);
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<VarListElementInit *>(this));
   }
 
   TypedInit *getVariable() const { return TI; }
   unsigned getElementNum() const { return Element; }
 
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit);
+                                    unsigned Bit) const;
 
   /// resolveListElementReference - This method is used to implement
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
   /// now, we return the resolved value, otherwise we return null.
-  virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt);
+  virtual Init *resolveListElementReference(Record &R,
+                                            const RecordVal *RV,
+                                            unsigned Elt) const;
 
   virtual std::string getAsString() const;
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 };
 
 /// DefInit - AL - Represent a reference to a 'def' in the description
 ///
 class DefInit : public TypedInit {
   Record *Def;
-public:
-  explicit DefInit(Record *D) : TypedInit(new RecordRecTy(D)), Def(D) {}
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  DefInit(Record *D, RecordRecTy *T) : TypedInit(T), Def(D) {}
+  friend class Record;
+
+  DefInit(const DefInit &Other);  // Do not define.
+  DefInit &operator=(const DefInit &Other);  // Do not define.
+
+public:
+  static DefInit *get(Record*);
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<DefInit *>(this));
   }
 
   Record *getDef() const { return Def; }
@@ -1056,7 +1200,7 @@ class DefInit : public TypedInit {
   /// simply return the resolved value, otherwise we return null.
   ///
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit) {
+                                    unsigned Bit) const {
     assert(0 && "Illegal bit reference off def");
     return 0;
   }
@@ -1065,7 +1209,7 @@ class DefInit : public TypedInit {
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
   /// now, we return the resolved value, otherwise we return null.
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt) {
+                                            unsigned Elt) const {
     assert(0 && "Illegal element reference off def");
     return 0;
   }
@@ -1077,22 +1221,30 @@ class DefInit : public TypedInit {
 class FieldInit : public TypedInit {
   Init *Rec;                // Record we are referring to
   std::string FieldName;    // Field we are accessing
-public:
+
   FieldInit(Init *R, const std::string &FN)
-    : TypedInit(R->getFieldType(FN)), Rec(R), FieldName(FN) {
+      : TypedInit(R->getFieldType(FN)), Rec(R), FieldName(FN) {
     assert(getType() && "FieldInit with non-record type!");
   }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  FieldInit(const FieldInit &Other);  // Do not define.
+  FieldInit &operator=(const FieldInit &Other);  // Do not define.
+
+public:
+  static FieldInit *get(Init *R, const std::string &FN);
+  static FieldInit *get(Init *R, const Init *FN);
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<FieldInit *>(this));
   }
 
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit);
-  virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt);
+                                    unsigned Bit) const;
+  virtual Init *resolveListElementReference(Record &R,
+                                            const RecordVal *RV,
+                                            unsigned Elt) const;
 
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const {
     return Rec->getAsString() + "." + FieldName;
@@ -1103,29 +1255,34 @@ class FieldInit : public TypedInit {
 /// to have at least one value then a (possibly empty) list of arguments.  Each
 /// argument can have a name associated with it.
 ///
-class DagInit : public TypedInit {
+class DagInit : public TypedInit, public FoldingSetNode {
   Init *Val;
   std::string ValName;
   std::vector<Init*> Args;
   std::vector<std::string> ArgNames;
-public:
-  DagInit(Init *V, std::string VN,
-          const std::vector<std::pair<Init*, std::string> > &args)
-    : TypedInit(new DagRecTy), Val(V), ValName(VN) {
-    Args.reserve(args.size());
-    ArgNames.reserve(args.size());
-    for (unsigned i = 0, e = args.size(); i != e; ++i) {
-      Args.push_back(args[i].first);
-      ArgNames.push_back(args[i].second);
-    }
-  }
-  DagInit(Init *V, std::string VN, const std::vector<Init*> &args,
-          const std::vector<std::string> &argNames)
-    : TypedInit(new DagRecTy), Val(V), ValName(VN), Args(args),
-      ArgNames(argNames) { }
 
-  virtual Init *convertInitializerTo(RecTy *Ty) {
-    return Ty->convertValue(this);
+  DagInit(Init *V, const std::string &VN,
+          ArrayRef<Init *> ArgRange,
+          ArrayRef<std::string> NameRange)
+      : TypedInit(DagRecTy::get()), Val(V), ValName(VN),
+          Args(ArgRange.begin(), ArgRange.end()),
+          ArgNames(NameRange.begin(), NameRange.end()) {}
+
+  DagInit(const DagInit &Other);  // Do not define.
+  DagInit &operator=(const DagInit &Other);  // Do not define.
+
+public:
+  static DagInit *get(Init *V, const std::string &VN,
+                      ArrayRef<Init *> ArgRange,
+                      ArrayRef<std::string> NameRange);
+  static DagInit *get(Init *V, const std::string &VN,
+                      const std::vector<
+                        std::pair<Init*, std::string> > &args);
+
+  void Profile(FoldingSetNodeID &ID) const;
+
+  virtual Init *convertInitializerTo(RecTy *Ty) const {
+    return Ty->convertValue(const_cast<DagInit *>(this));
   }
 
   Init *getOperator() const { return Val; }
@@ -1142,44 +1299,33 @@ class DagInit : public TypedInit {
     return ArgNames[Num];
   }
 
-  void setArg(unsigned Num, Init *I) {
-    assert(Num < Args.size() && "Arg number out of range!");
-    Args[Num] = I;
-  }
-
-  virtual Init *resolveReferences(Record &R, const RecordVal *RV);
+  virtual Init *resolveReferences(Record &R, const RecordVal *RV) const;
 
   virtual std::string getAsString() const;
 
-  typedef std::vector<Init*>::iterator             arg_iterator;
   typedef std::vector<Init*>::const_iterator       const_arg_iterator;
-  typedef std::vector<std::string>::iterator       name_iterator;
   typedef std::vector<std::string>::const_iterator const_name_iterator;
 
-  inline arg_iterator        arg_begin()       { return Args.begin(); }
   inline const_arg_iterator  arg_begin() const { return Args.begin(); }
-  inline arg_iterator        arg_end  ()       { return Args.end();   }
   inline const_arg_iterator  arg_end  () const { return Args.end();   }
 
   inline size_t              arg_size () const { return Args.size();  }
   inline bool                arg_empty() const { return Args.empty(); }
 
-  inline name_iterator       name_begin()       { return ArgNames.begin(); }
   inline const_name_iterator name_begin() const { return ArgNames.begin(); }
-  inline name_iterator       name_end  ()       { return ArgNames.end();   }
   inline const_name_iterator name_end  () const { return ArgNames.end();   }
 
   inline size_t              name_size () const { return ArgNames.size();  }
   inline bool                name_empty() const { return ArgNames.empty(); }
 
   virtual Init *resolveBitReference(Record &R, const RecordVal *RV,
-                                    unsigned Bit) {
+                                    unsigned Bit) const {
     assert(0 && "Illegal bit reference off dag");
     return 0;
   }
 
   virtual Init *resolveListElementReference(Record &R, const RecordVal *RV,
-                                            unsigned Elt) {
+                                            unsigned Elt) const {
     assert(0 && "Illegal element reference off dag");
     return 0;
   }
@@ -1190,14 +1336,15 @@ class DagInit : public TypedInit {
 //===----------------------------------------------------------------------===//
 
 class RecordVal {
-  std::string Name;
+  Init *Name;
   RecTy *Ty;
   unsigned Prefix;
   Init *Value;
 public:
+  RecordVal(Init *N, RecTy *T, unsigned P);
   RecordVal(const std::string &N, RecTy *T, unsigned P);
 
-  const std::string &getName() const { return Name; }
+  const std::string &getName() const;
 
   unsigned getPrefix() const { return Prefix; }
   RecTy *getType() const { return Ty; }
@@ -1226,7 +1373,7 @@ class Record {
 
   // Unique record ID.
   unsigned ID;
-  std::string Name;
+  Init *Name;
   SMLoc Loc;
   std::vector<std::string> TemplateArgs;
   std::vector<RecordVal> Values;
@@ -1235,11 +1382,15 @@ class Record {
   // Tracks Record instances. Not owned by Record.
   RecordKeeper &TrackedRecords;
 
+  DefInit *TheInit;
+
+  void checkName();
+
 public:
 
   // Constructs a record.
   explicit Record(const std::string &N, SMLoc loc, RecordKeeper &records) :
-    ID(LastID++), Name(N), Loc(loc), TrackedRecords(records) {}
+    ID(LastID++), Name(StringInit::get(N)), Loc(loc), TrackedRecords(records), TheInit(0) {}
   ~Record() {}
 
 
@@ -1248,11 +1399,15 @@ class Record {
 
   unsigned getID() const { return ID; }
 
-  const std::string &getName() const { return Name; }
+  const std::string &getName() const;
+  void setName(Init *Name);               // Also updates RecordKeeper.
   void setName(const std::string &Name);  // Also updates RecordKeeper.
 
   SMLoc getLoc() const { return Loc; }
 
+  /// get the corresponding DefInit.
+  DefInit *getDefInit();
+
   const std::vector<std::string> &getTemplateArgs() const {
     return TemplateArgs;
   }
diff --git a/contrib/llvm/include/llvm/TableGen/TableGenAction.h b/contrib/llvm/include/llvm/TableGen/TableGenAction.h
new file mode 100644
index 000000000000..9f1c23c5b457
--- /dev/null
+++ b/contrib/llvm/include/llvm/TableGen/TableGenAction.h
@@ -0,0 +1,34 @@
+//===- llvm/TableGen/TableGenAction.h - defines TableGenAction --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the TableGenAction base class to be derived from by
+// tblgen tools.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TABLEGEN_TABLEGENACTION_H
+#define LLVM_TABLEGEN_TABLEGENACTION_H
+
+namespace llvm {
+
+class raw_ostream;
+class RecordKeeper;
+
+class TableGenAction {
+public:
+  virtual ~TableGenAction() {}
+
+  /// Perform the action using Records, and write output to OS.
+  /// @returns true on error, false otherwise
+  virtual bool operator()(raw_ostream &OS, RecordKeeper &Records) = 0;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/utils/TableGen/TableGenBackend.h b/contrib/llvm/include/llvm/TableGen/TableGenBackend.h
similarity index 88%
rename from contrib/llvm/utils/TableGen/TableGenBackend.h
rename to contrib/llvm/include/llvm/TableGen/TableGenBackend.h
index 9c2b948b0dfc..853f92e406fb 100644
--- a/contrib/llvm/utils/TableGen/TableGenBackend.h
+++ b/contrib/llvm/include/llvm/TableGen/TableGenBackend.h
@@ -1,4 +1,4 @@
-//===- TableGenBackend.h - Base class for TableGen Backends -----*- C++ -*-===//
+//===- llvm/TableGen/TableGenBackend.h - Backend base class -----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef TABLEGENBACKEND_H
-#define TABLEGENBACKEND_H
+#ifndef LLVM_TABLEGEN_TABLEGENBACKEND_H
+#define LLVM_TABLEGEN_TABLEGENBACKEND_H
 
 #include "llvm/Support/raw_ostream.h"
 #include <string>
diff --git a/contrib/llvm/include/llvm/Target/Target.td b/contrib/llvm/include/llvm/Target/Target.td
index 018ccbd72846..aa9a4f5af18c 100644
--- a/contrib/llvm/include/llvm/Target/Target.td
+++ b/contrib/llvm/include/llvm/Target/Target.td
@@ -297,6 +297,10 @@ class Instruction {
   // from the opcode.
   int Size = 0;
 
+  // DecoderNamespace - The "namespace" in which this instruction exists, on
+  // targets like ARM which multiple ISA namespaces exist.
+  string DecoderNamespace = "";
+
   // Code size, for instruction selection.
   // FIXME: What does this actually mean?
   int CodeSize = 0;
@@ -324,6 +328,7 @@ class Instruction {
   bit isPredicable = 0;     // Is this instruction predicable?
   bit hasDelaySlot = 0;     // Does this instruction have an delay slot?
   bit usesCustomInserter = 0; // Pseudo instr needing special help.
+  bit hasPostISelHook = 0;  // To be *adjusted* after isel by target hook.
   bit hasCtrlDep   = 0;     // Does this instruction r/w ctrl-flow chains?
   bit isNotDuplicable = 0;  // Is it unsafe to duplicate this instruction?
   bit isAsCheapAsAMove = 0; // As cheap (or cheaper) than a move instruction.
@@ -581,7 +586,7 @@ class InstrInfo {
 // Standard Pseudo Instructions.
 // This list must match TargetOpcodes.h and CodeGenTarget.cpp.
 // Only these instructions are allowed in the TargetOpcode namespace.
-let isCodeGenOnly = 1, Namespace = "TargetOpcode" in {
+let isCodeGenOnly = 1, isPseudo = 1, Namespace = "TargetOpcode" in {
 def PHI : Instruction {
   let OutOperandList = (outs);
   let InOperandList = (ins variable_ops);
diff --git a/contrib/llvm/include/llvm/Target/TargetAsmInfo.h b/contrib/llvm/include/llvm/Target/TargetAsmInfo.h
deleted file mode 100644
index 5a526dcebc9d..000000000000
--- a/contrib/llvm/include/llvm/Target/TargetAsmInfo.h
+++ /dev/null
@@ -1,103 +0,0 @@
-//===-- llvm/Target/TargetAsmInfo.h -----------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Interface to provide the information necessary for producing assembly files.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TARGET_TARGETASMINFO_H
-#define LLVM_TARGET_TARGETASMINFO_H
-
-#include "llvm/CodeGen/MachineLocation.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetFrameLowering.h"
-#include "llvm/Target/TargetRegisterInfo.h"
-
-namespace llvm {
-  template <typename T> class ArrayRef;
-  class MCSection;
-  class MCContext;
-  class MachineFunction;
-  class TargetMachine;
-  class TargetLoweringObjectFile;
-
-class TargetAsmInfo {
-  std::vector<MachineMove> InitialFrameState;
-  const TargetRegisterInfo *TRI;
-  const TargetFrameLowering *TFI;
-  const TargetLoweringObjectFile *TLOF;
-
-public:
-  explicit TargetAsmInfo(const TargetMachine &TM);
-
-  const MCSection *getDwarfLineSection() const {
-    return TLOF->getDwarfLineSection();
-  }
-
-  const MCSection *getEHFrameSection() const {
-    return TLOF->getEHFrameSection();
-  }
-
-  const MCSection *getCompactUnwindSection() const {
-    return TLOF->getCompactUnwindSection();
-  }
-
-  const MCSection *getDwarfFrameSection() const {
-    return TLOF->getDwarfFrameSection();
-  }
-
-  const MCSection *getWin64EHFuncTableSection(StringRef Suffix) const {
-    return TLOF->getWin64EHFuncTableSection(Suffix);
-  }
-
-  const MCSection *getWin64EHTableSection(StringRef Suffix) const {
-    return TLOF->getWin64EHTableSection(Suffix);
-  }
-
-  unsigned getFDEEncoding(bool CFI) const {
-    return TLOF->getFDEEncoding(CFI);
-  }
-
-  bool isFunctionEHFrameSymbolPrivate() const {
-    return TLOF->isFunctionEHFrameSymbolPrivate();
-  }
-
-  int getCompactUnwindEncoding(ArrayRef<MCCFIInstruction> Instrs,
-                               int DataAlignmentFactor,
-                               bool IsEH) const {
-    return TFI->getCompactUnwindEncoding(Instrs, DataAlignmentFactor, IsEH);
-  }
-
-  const unsigned *getCalleeSavedRegs(MachineFunction *MF = 0) const {
-    return TRI->getCalleeSavedRegs(MF);
-  }
-
-  unsigned getDwarfRARegNum(bool isEH) const {
-    return TRI->getDwarfRegNum(TRI->getRARegister(), isEH);
-  }
-
-  const std::vector<MachineMove> &getInitialFrameState() const {
-    return InitialFrameState;
-  }
-
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const {
-    return TRI->getDwarfRegNum(RegNum, isEH);
-  }
-
-  int getLLVMRegNum(unsigned DwarfRegNum, bool isEH) const {
-    return TRI->getLLVMRegNum(DwarfRegNum, isEH);
-  }
-
-  int getSEHRegNum(unsigned RegNum) const {
-    return TRI->getSEHRegNum(RegNum);
-  }
-};
-
-}
-#endif
diff --git a/contrib/llvm/include/llvm/Target/TargetData.h b/contrib/llvm/include/llvm/Target/TargetData.h
index c28081000d71..26fd1870ac39 100644
--- a/contrib/llvm/include/llvm/Target/TargetData.h
+++ b/contrib/llvm/include/llvm/Target/TargetData.h
@@ -33,6 +33,8 @@ class StructType;
 class StructLayout;
 class GlobalVariable;
 class LLVMContext;
+template<typename T>
+class ArrayRef;
 
 /// Enum used to categorize the alignment types stored by TargetAlignElem
 enum AlignTypeEnum {
@@ -42,6 +44,7 @@ enum AlignTypeEnum {
   AGGREGATE_ALIGN = 'a',             ///< Aggregate alignment
   STACK_ALIGN = 's'                  ///< Stack objects alignment
 };
+  
 /// Target alignment element.
 ///
 /// Stores the alignment data associated with a given alignment type (pointer,
@@ -62,12 +65,19 @@ struct TargetAlignElem {
   bool operator==(const TargetAlignElem &rhs) const;
 };
 
+/// TargetData - This class holds a parsed version of the target data layout
+/// string in a module and provides methods for querying it.  The target data
+/// layout string is specified *by the target* - a frontend generating LLVM IR
+/// is required to generate the right target data for the target being codegen'd
+/// to.  If some measure of portability is desired, an empty string may be
+/// specified in the module.
 class TargetData : public ImmutablePass {
 private:
   bool          LittleEndian;          ///< Defaults to false
   unsigned      PointerMemSize;        ///< Pointer size in bytes
   unsigned      PointerABIAlign;       ///< Pointer ABI alignment
   unsigned      PointerPrefAlign;      ///< Pointer preferred alignment
+  unsigned      StackNaturalAlign;     ///< Stack natural alignment
 
   SmallVector<unsigned char, 8> LegalIntWidths; ///< Legal Integers.
   
@@ -90,9 +100,9 @@ class TargetData : public ImmutablePass {
   void setAlignment(AlignTypeEnum align_type, unsigned abi_align,
                     unsigned pref_align, uint32_t bit_width);
   unsigned getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width,
-                            bool ABIAlign, const Type *Ty) const;
+                            bool ABIAlign, Type *Ty) const;
   //! Internal helper method that returns requested alignment for type.
-  unsigned getAlignment(const Type *Ty, bool abi_or_pref) const;
+  unsigned getAlignment(Type *Ty, bool abi_or_pref) const;
 
   /// Valid alignment predicate.
   ///
@@ -161,6 +171,11 @@ class TargetData : public ImmutablePass {
     return !isLegalInteger(Width);
   }
 
+  /// Returns true if the given alignment exceeds the natural stack alignment.
+  bool exceedsNaturalStackAlignment(unsigned Align) const {
+    return (StackNaturalAlign != 0) && (Align > StackNaturalAlign);
+  }
+
   /// fitsInLegalInteger - This function returns true if the specified type fits
   /// in a native integer type supported by the CPU.  For example, if the CPU
   /// only supports i32 as a native integer type, then i27 fits in a legal
@@ -200,19 +215,19 @@ class TargetData : public ImmutablePass {
 
   /// getTypeSizeInBits - Return the number of bits necessary to hold the
   /// specified type.  For example, returns 36 for i36 and 80 for x86_fp80.
-  uint64_t getTypeSizeInBits(const Type* Ty) const;
+  uint64_t getTypeSizeInBits(Type* Ty) const;
 
   /// getTypeStoreSize - Return the maximum number of bytes that may be
   /// overwritten by storing the specified type.  For example, returns 5
   /// for i36 and 10 for x86_fp80.
-  uint64_t getTypeStoreSize(const Type *Ty) const {
+  uint64_t getTypeStoreSize(Type *Ty) const {
     return (getTypeSizeInBits(Ty)+7)/8;
   }
 
   /// getTypeStoreSizeInBits - Return the maximum number of bits that may be
   /// overwritten by storing the specified type; always a multiple of 8.  For
   /// example, returns 40 for i36 and 80 for x86_fp80.
-  uint64_t getTypeStoreSizeInBits(const Type *Ty) const {
+  uint64_t getTypeStoreSizeInBits(Type *Ty) const {
     return 8*getTypeStoreSize(Ty);
   }
 
@@ -220,7 +235,7 @@ class TargetData : public ImmutablePass {
   /// of the specified type, including alignment padding.  This is the amount
   /// that alloca reserves for this type.  For example, returns 12 or 16 for
   /// x86_fp80, depending on alignment.
-  uint64_t getTypeAllocSize(const Type* Ty) const {
+  uint64_t getTypeAllocSize(Type* Ty) const {
     // Round up to the next alignment boundary.
     return RoundUpAlignment(getTypeStoreSize(Ty), getABITypeAlignment(Ty));
   }
@@ -229,13 +244,13 @@ class TargetData : public ImmutablePass {
   /// objects of the specified type, including alignment padding; always a
   /// multiple of 8.  This is the amount that alloca reserves for this type.
   /// For example, returns 96 or 128 for x86_fp80, depending on alignment.
-  uint64_t getTypeAllocSizeInBits(const Type* Ty) const {
+  uint64_t getTypeAllocSizeInBits(Type* Ty) const {
     return 8*getTypeAllocSize(Ty);
   }
 
   /// getABITypeAlignment - Return the minimum ABI-required alignment for the
   /// specified type.
-  unsigned getABITypeAlignment(const Type *Ty) const;
+  unsigned getABITypeAlignment(Type *Ty) const;
   
   /// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for
   /// an integer type of the specified bitwidth.
@@ -244,17 +259,17 @@ class TargetData : public ImmutablePass {
 
   /// getCallFrameTypeAlignment - Return the minimum ABI-required alignment
   /// for the specified type when it is part of a call frame.
-  unsigned getCallFrameTypeAlignment(const Type *Ty) const;
+  unsigned getCallFrameTypeAlignment(Type *Ty) const;
 
 
   /// getPrefTypeAlignment - Return the preferred stack/global alignment for
   /// the specified type.  This is always at least as good as the ABI alignment.
-  unsigned getPrefTypeAlignment(const Type *Ty) const;
+  unsigned getPrefTypeAlignment(Type *Ty) const;
 
   /// getPreferredTypeAlignmentShift - Return the preferred alignment for the
   /// specified type, returned as log2 of the value (a shift amount).
   ///
-  unsigned getPreferredTypeAlignmentShift(const Type *Ty) const;
+  unsigned getPreferredTypeAlignmentShift(Type *Ty) const;
 
   /// getIntPtrType - Return an unsigned integer type that is the same size or
   /// greater to the host pointer size.
@@ -264,13 +279,12 @@ class TargetData : public ImmutablePass {
   /// getIndexedOffset - return the offset from the beginning of the type for
   /// the specified indices.  This is used to implement getelementptr.
   ///
-  uint64_t getIndexedOffset(const Type *Ty,
-                            Value* const* Indices, unsigned NumIndices) const;
+  uint64_t getIndexedOffset(Type *Ty, ArrayRef<Value *> Indices) const;
 
   /// getStructLayout - Return a StructLayout object, indicating the alignment
   /// of the struct, its size, and the offsets of its fields.  Note that this
   /// information is lazily cached.
-  const StructLayout *getStructLayout(const StructType *Ty) const;
+  const StructLayout *getStructLayout(StructType *Ty) const;
 
   /// getPreferredAlignment - Return the preferred alignment of the specified
   /// global.  This includes an explicitly requested alignment (if the global
@@ -333,7 +347,7 @@ class StructLayout {
 
 private:
   friend class TargetData;   // Only TargetData can create this class
-  StructLayout(const StructType *ST, const TargetData &TD);
+  StructLayout(StructType *ST, const TargetData &TD);
 };
 
 } // End llvm namespace
diff --git a/contrib/llvm/include/llvm/Target/TargetFrameLowering.h b/contrib/llvm/include/llvm/Target/TargetFrameLowering.h
index e3d77cf7625d..4c759b2ccb9f 100644
--- a/contrib/llvm/include/llvm/Target/TargetFrameLowering.h
+++ b/contrib/llvm/include/llvm/Target/TargetFrameLowering.h
@@ -114,6 +114,10 @@ class TargetFrameLowering {
   virtual void emitEpilogue(MachineFunction &MF,
                             MachineBasicBlock &MBB) const = 0;
 
+  /// Adjust the prologue to have the function use segmented stacks. This works
+  /// by adding a check even before the "normal" function prologue.
+  virtual void adjustForSegmentedStacks(MachineFunction &MF) const { }
+
   /// spillCalleeSavedRegisters - Issues instruction(s) to spill all callee
   /// saved registers and returns true if it isn't possible / profitable to do
   /// so by issuing a series of store instructions via
@@ -161,11 +165,6 @@ class TargetFrameLowering {
     return hasReservedCallFrame(MF) || hasFP(MF);
   }
 
-  /// getInitialFrameState - Returns a list of machine moves that are assumed
-  /// on entry to all functions.  Note that LabelID is ignored (assumed to be
-  /// the beginning of the function.)
-  virtual void getInitialFrameState(std::vector<MachineMove> &Moves) const;
-
   /// getFrameIndexOffset - Returns the displacement from the frame register to
   /// the stack frame of the specified index.
   virtual int getFrameIndexOffset(const MachineFunction &MF, int FI) const;
@@ -191,14 +190,6 @@ class TargetFrameLowering {
   ///
   virtual void processFunctionBeforeFrameFinalized(MachineFunction &MF) const {
   }
-
-  /// getCompactUnwindEncoding - Get the compact unwind encoding for the
-  /// function. Return 0 if the compact unwind isn't available.
-  virtual uint32_t getCompactUnwindEncoding(ArrayRef<MCCFIInstruction> Instrs,
-                                            int DataAlignmentFactor,
-                                            bool IsEH) const {
-    return 0;
-  }
 };
 
 } // End llvm namespace
diff --git a/contrib/llvm/include/llvm/Target/TargetInstrInfo.h b/contrib/llvm/include/llvm/Target/TargetInstrInfo.h
index f6635667b5d9..07f614d61d93 100644
--- a/contrib/llvm/include/llvm/Target/TargetInstrInfo.h
+++ b/contrib/llvm/include/llvm/Target/TargetInstrInfo.h
@@ -49,7 +49,7 @@ class TargetInstrInfo : public MCInstrInfo {
     : CallFrameSetupOpcode(CFSetupOpcode),
       CallFrameDestroyOpcode(CFDestroyOpcode) {
   }
-    
+
   virtual ~TargetInstrInfo();
 
   /// getRegClass - Givem a machine instruction descriptor, returns the register
@@ -386,6 +386,16 @@ class TargetInstrInfo : public MCInstrInfo {
   assert(0 && "Target didn't implement TargetInstrInfo::loadRegFromStackSlot!");
   }
 
+  /// expandPostRAPseudo - This function is called for all pseudo instructions
+  /// that remain after register allocation. Many pseudo instructions are
+  /// created to help register allocation. This is the place to convert them
+  /// into real instructions. The target can edit MI in place, or it can insert
+  /// new instructions and erase MI. The function should return true if
+  /// anything was changed.
+  virtual bool expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
+    return false;
+  }
+
   /// emitFrameIndexDebugValue - Emit a target-dependent form of
   /// DBG_VALUE encoding the address of a frame index.  Addresses would
   /// normally be lowered the same way as other addresses on the target,
@@ -671,6 +681,43 @@ class TargetInstrInfo : public MCInstrInfo {
   bool hasLowDefLatency(const InstrItineraryData *ItinData,
                         const MachineInstr *DefMI, unsigned DefIdx) const;
 
+  /// verifyInstruction - Perform target specific instruction verification.
+  virtual
+  bool verifyInstruction(const MachineInstr *MI, StringRef &ErrInfo) const {
+    return true;
+  }
+
+  /// getExecutionDomain - Return the current execution domain and bit mask of
+  /// possible domains for instruction.
+  ///
+  /// Some micro-architectures have multiple execution domains, and multiple
+  /// opcodes that perform the same operation in different domains.  For
+  /// example, the x86 architecture provides the por, orps, and orpd
+  /// instructions that all do the same thing.  There is a latency penalty if a
+  /// register is written in one domain and read in another.
+  ///
+  /// This function returns a pair (domain, mask) containing the execution
+  /// domain of MI, and a bit mask of possible domains.  The setExecutionDomain
+  /// function can be used to change the opcode to one of the domains in the
+  /// bit mask.  Instructions whose execution domain can't be changed should
+  /// return a 0 mask.
+  ///
+  /// The execution domain numbers don't have any special meaning except domain
+  /// 0 is used for instructions that are not associated with any interesting
+  /// execution domain.
+  ///
+  virtual std::pair<uint16_t, uint16_t>
+  getExecutionDomain(const MachineInstr *MI) const {
+    return std::make_pair(0, 0);
+  }
+
+  /// setExecutionDomain - Change the opcode of MI to execute in Domain.
+  ///
+  /// The bit (1 << Domain) must be set in the mask returned from
+  /// getExecutionDomain(MI).
+  ///
+  virtual void setExecutionDomain(MachineInstr *MI, unsigned Domain) const {}
+
 private:
   int CallFrameSetupOpcode, CallFrameDestroyOpcode;
 };
@@ -693,6 +740,12 @@ class TargetInstrInfoImpl : public TargetInstrInfo {
                                      unsigned &SrcOpIdx2) const;
   virtual bool canFoldMemoryOperand(const MachineInstr *MI,
                                     const SmallVectorImpl<unsigned> &Ops) const;
+  virtual bool hasLoadFromStackSlot(const MachineInstr *MI,
+                                    const MachineMemOperand *&MMO,
+                                    int &FrameIndex) const;
+  virtual bool hasStoreToStackSlot(const MachineInstr *MI,
+                                   const MachineMemOperand *&MMO,
+                                   int &FrameIndex) const;
   virtual bool PredicateInstruction(MachineInstr *MI,
                             const SmallVectorImpl<MachineOperand> &Pred) const;
   virtual void reMaterialize(MachineBasicBlock &MBB,
diff --git a/contrib/llvm/include/llvm/Target/TargetIntrinsicInfo.h b/contrib/llvm/include/llvm/Target/TargetIntrinsicInfo.h
index ad8ac925e930..c44b9230c0d8 100644
--- a/contrib/llvm/include/llvm/Target/TargetIntrinsicInfo.h
+++ b/contrib/llvm/include/llvm/Target/TargetIntrinsicInfo.h
@@ -39,7 +39,7 @@ class TargetIntrinsicInfo {
   /// intrinsic, Tys should point to an array of numTys pointers to Type,
   /// and must provide exactly one type for each overloaded type in the
   /// intrinsic.
-  virtual std::string getName(unsigned IID, const Type **Tys = 0,
+  virtual std::string getName(unsigned IID, Type **Tys = 0,
                               unsigned numTys = 0) const = 0;
 
   /// Look up target intrinsic by name. Return intrinsic ID or 0 for unknown
@@ -55,7 +55,7 @@ class TargetIntrinsicInfo {
   /// Create or insert an LLVM Function declaration for an intrinsic,
   /// and return it. The Tys and numTys are for intrinsics with overloaded
   /// types. See above for more information.
-  virtual Function *getDeclaration(Module *M, unsigned ID, const Type **Tys = 0,
+  virtual Function *getDeclaration(Module *M, unsigned ID, Type **Tys = 0,
                                    unsigned numTys = 0) const = 0;
 };
 
diff --git a/contrib/llvm/include/llvm/Target/TargetLowering.h b/contrib/llvm/include/llvm/Target/TargetLowering.h
index 533c3ac8784f..013e70a05c03 100644
--- a/contrib/llvm/include/llvm/Target/TargetLowering.h
+++ b/contrib/llvm/include/llvm/Target/TargetLowering.h
@@ -113,6 +113,22 @@ class TargetLowering {
     ZeroOrNegativeOneBooleanContent // All bits equal to bit 0.
   };
 
+  static ISD::NodeType getExtendForContent(BooleanContent Content) {
+    switch (Content) {
+    default:
+      assert(false && "Unknown BooleanContent!");
+    case UndefinedBooleanContent:
+      // Extend by adding rubbish bits.
+      return ISD::ANY_EXTEND;
+    case ZeroOrOneBooleanContent:
+      // Extend by adding zero bits.
+      return ISD::ZERO_EXTEND;
+    case ZeroOrNegativeOneBooleanContent:
+      // Extend by copying the sign bit.
+      return ISD::SIGN_EXTEND;
+    }
+  }
+
   /// NOTE: The constructor takes ownership of TLOF.
   explicit TargetLowering(const TargetMachine &TM,
                           const TargetLoweringObjectFile *TLOF);
@@ -148,8 +164,7 @@ class TargetLowering {
   /// the condition operand of SELECT and BRCOND nodes.  In the case of
   /// BRCOND the argument passed is MVT::Other since there are no other
   /// operands to get a type hint from.
-  virtual
-  MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+  virtual EVT getSetCCResultType(EVT VT) const;
 
   /// getCmpLibcallReturnType - Return the ValueType for comparison
   /// libcalls. Comparions libcalls include floating point comparion calls,
@@ -162,7 +177,13 @@ class TargetLowering {
   /// "Boolean values" are special true/false values produced by nodes like
   /// SETCC and consumed (as the condition) by nodes like SELECT and BRCOND.
   /// Not to be confused with general values promoted from i1.
-  BooleanContent getBooleanContents() const { return BooleanContents;}
+  /// Some cpus distinguish between vectors of boolean and scalars; the isVec
+  /// parameter selects between the two kinds.  For example on X86 a scalar
+  /// boolean should be zero extended from i1, while the elements of a vector
+  /// of booleans should be sign extended from i1.
+  BooleanContent getBooleanContents(bool isVec) const {
+    return isVec ? BooleanVectorContents : BooleanContents;
+  }
 
   /// getSchedulingPreference - Return target scheduling preference.
   Sched::Preference getSchedulingPreference() const {
@@ -172,7 +193,7 @@ class TargetLowering {
   /// getSchedulingPreference - Some scheduler, e.g. hybrid, can switch to
   /// different scheduling heuristics for different nodes. This function returns
   /// the preference (or none) for the given node.
-  virtual Sched::Preference getSchedulingPreference(SDNode *N) const {
+  virtual Sched::Preference getSchedulingPreference(SDNode *) const {
     return Sched::None;
   }
 
@@ -265,9 +286,9 @@ class TargetLowering {
     assert(!VT.isVector());
     while (true) {
       switch (getTypeAction(Context, VT)) {
-      case Legal:
+      case TypeLegal:
         return VT;
-      case Expand:
+      case TypeExpandInteger:
         VT = getTypeToTransformTo(Context, VT);
         break;
       default:
@@ -307,15 +328,15 @@ class TargetLowering {
     bool         writeMem;    // writes memory?
   };
 
-  virtual bool getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                  const CallInst &I, unsigned Intrinsic) const {
+  virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+                                  unsigned /*Intrinsic*/) const {
     return false;
   }
 
   /// isFPImmLegal - Returns true if the target can instruction select the
   /// specified FP immediate natively. If false, the legalizer will materialize
   /// the FP immediate as a load from a constant pool.
-  virtual bool isFPImmLegal(const APFloat &Imm, EVT VT) const {
+  virtual bool isFPImmLegal(const APFloat &/*Imm*/, EVT /*VT*/) const {
     return false;
   }
 
@@ -323,8 +344,8 @@ class TargetLowering {
   /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
   /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
   /// are assumed to be legal.
-  virtual bool isShuffleMaskLegal(const SmallVectorImpl<int> &Mask,
-                                  EVT VT) const {
+  virtual bool isShuffleMaskLegal(const SmallVectorImpl<int> &/*Mask*/,
+                                  EVT /*VT*/) const {
     return true;
   }
 
@@ -337,8 +358,8 @@ class TargetLowering {
   /// used by Targets can use this to indicate if there is a suitable
   /// VECTOR_SHUFFLE that can be used to replace a VAND with a constant
   /// pool entry.
-  virtual bool isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
-                                      EVT VT) const {
+  virtual bool isVectorClearMaskLegal(const SmallVectorImpl<int> &/*Mask*/,
+                                      EVT /*VT*/) const {
     return false;
   }
 
@@ -383,9 +404,7 @@ class TargetLowering {
   /// isLoadExtLegal - Return true if the specified load with extension is legal
   /// on this target.
   bool isLoadExtLegal(unsigned ExtType, EVT VT) const {
-    return VT.isSimple() &&
-      (getLoadExtAction(ExtType, VT) == Legal ||
-       getLoadExtAction(ExtType, VT) == Custom);
+    return VT.isSimple() && getLoadExtAction(ExtType, VT) == Legal;
   }
 
   /// getTruncStoreAction - Return how this store with truncation should be
@@ -404,8 +423,7 @@ class TargetLowering {
   /// legal on this target.
   bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const {
     return isTypeLegal(ValVT) && MemVT.isSimple() &&
-      (getTruncStoreAction(ValVT, MemVT) == Legal ||
-       getTruncStoreAction(ValVT, MemVT) == Custom);
+           getTruncStoreAction(ValVT, MemVT) == Legal;
   }
 
   /// getIndexedLoadAction - Return how the indexed load should be treated:
@@ -501,7 +519,7 @@ class TargetLowering {
   /// This is fixed by the LLVM operations except for the pointer size.  If
   /// AllowUnknown is true, this will return MVT::Other for types with no EVT
   /// counterpart (e.g. structs), otherwise it will assert.
-  EVT getValueType(const Type *Ty, bool AllowUnknown = false) const {
+  EVT getValueType(Type *Ty, bool AllowUnknown = false) const {
     EVT VT = EVT::getEVT(Ty, AllowUnknown);
     return VT == MVT::iPTR ? PointerTy : VT;
   }
@@ -509,7 +527,7 @@ class TargetLowering {
   /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
   /// function arguments in the caller parameter area.  This is the actual
   /// alignment, not its logarithm.
-  virtual unsigned getByValTypeAlignment(const Type *Ty) const;
+  virtual unsigned getByValTypeAlignment(Type *Ty) const;
 
   /// getRegisterType - Return the type of registers that this ValueType will
   /// eventually require.
@@ -569,7 +587,7 @@ class TargetLowering {
   /// ShouldShrinkFPConstant - If true, then instruction selection should
   /// seek to shrink the FP constant of the specified type to a smaller type
   /// in order to save space and / or reduce runtime.
-  virtual bool ShouldShrinkFPConstant(EVT VT) const { return true; }
+  virtual bool ShouldShrinkFPConstant(EVT) const { return true; }
 
   /// hasTargetDAGCombine - If true, the target has custom DAG combine
   /// transformations that it can perform for the specified node.
@@ -611,7 +629,7 @@ class TargetLowering {
   /// use helps to ensure that such replacements don't generate code that causes
   /// an alignment error  (trap) on the target machine.
   /// @brief Determine if the target supports unaligned memory accesses.
-  virtual bool allowsUnalignedMemoryAccesses(EVT VT) const {
+  virtual bool allowsUnalignedMemoryAccesses(EVT) const {
     return false;
   }
 
@@ -634,10 +652,11 @@ class TargetLowering {
   /// constant so it does not need to be loaded.
   /// It returns EVT::Other if the type should be determined using generic
   /// target-independent logic.
-  virtual EVT getOptimalMemOpType(uint64_t Size,
-                                  unsigned DstAlign, unsigned SrcAlign,
-                                  bool NonScalarIntSafe, bool MemcpyStrSrc,
-                                  MachineFunction &MF) const {
+  virtual EVT getOptimalMemOpType(uint64_t /*Size*/,
+                                  unsigned /*DstAlign*/, unsigned /*SrcAlign*/,
+                                  bool /*NonScalarIntSafe*/,
+                                  bool /*MemcpyStrSrc*/,
+                                  MachineFunction &/*MF*/) const {
     return MVT::Other;
   }
 
@@ -717,23 +736,30 @@ class TargetLowering {
     return ShouldFoldAtomicFences;
   }
 
+  /// getInsertFencesFor - return whether the DAG builder should automatically
+  /// insert fences and reduce ordering for atomics.
+  ///
+  bool getInsertFencesForAtomic() const {
+    return InsertFencesForAtomic;
+  }
+
   /// getPreIndexedAddressParts - returns true by value, base pointer and
   /// offset pointer and addressing mode by reference if the node's address
   /// can be legally represented as pre-indexed load / store address.
-  virtual bool getPreIndexedAddressParts(SDNode *N, SDValue &Base,
-                                         SDValue &Offset,
-                                         ISD::MemIndexedMode &AM,
-                                         SelectionDAG &DAG) const {
+  virtual bool getPreIndexedAddressParts(SDNode * /*N*/, SDValue &/*Base*/,
+                                         SDValue &/*Offset*/,
+                                         ISD::MemIndexedMode &/*AM*/,
+                                         SelectionDAG &/*DAG*/) const {
     return false;
   }
 
   /// getPostIndexedAddressParts - returns true by value, base pointer and
   /// offset pointer and addressing mode by reference if this node can be
   /// combined with a load / store to form a post-indexed load / store.
-  virtual bool getPostIndexedAddressParts(SDNode *N, SDNode *Op,
-                                          SDValue &Base, SDValue &Offset,
-                                          ISD::MemIndexedMode &AM,
-                                          SelectionDAG &DAG) const {
+  virtual bool getPostIndexedAddressParts(SDNode * /*N*/, SDNode * /*Op*/,
+                                          SDValue &/*Base*/, SDValue &/*Offset*/,
+                                          ISD::MemIndexedMode &/*AM*/,
+                                          SelectionDAG &/*DAG*/) const {
     return false;
   }
 
@@ -743,9 +769,9 @@ class TargetLowering {
   virtual unsigned getJumpTableEncoding() const;
 
   virtual const MCExpr *
-  LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
-                            const MachineBasicBlock *MBB, unsigned uid,
-                            MCContext &Ctx) const {
+  LowerCustomJumpTableEntry(const MachineJumpTableInfo * /*MJTI*/,
+                            const MachineBasicBlock * /*MBB*/, unsigned /*uid*/,
+                            MCContext &/*Ctx*/) const {
     assert(0 && "Need to implement this hook if target has custom JTIs");
     return 0;
   }
@@ -771,7 +797,8 @@ class TargetLowering {
   /// protector cookies at a fixed offset in some non-standard address
   /// space, and populates the address space and offset as
   /// appropriate.
-  virtual bool getStackCookieLocation(unsigned &AddressSpace, unsigned &Offset) const {
+  virtual bool getStackCookieLocation(unsigned &/*AddressSpace*/,
+                                      unsigned &/*Offset*/) const {
     return false;
   }
 
@@ -906,7 +933,7 @@ class TargetLowering {
   /// the specified value type and it is 'desirable' to use the type for the
   /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
   /// instruction encodings are longer and some i16 instructions are slow.
-  virtual bool isTypeDesirableForOp(unsigned Opc, EVT VT) const {
+  virtual bool isTypeDesirableForOp(unsigned /*Opc*/, EVT VT) const {
     // By default, assume all legal types are desirable.
     return isTypeLegal(VT);
   }
@@ -914,14 +941,15 @@ class TargetLowering {
   /// isDesirableToPromoteOp - Return true if it is profitable for dag combiner
   /// to transform a floating point op of specified opcode to a equivalent op of
   /// an integer type. e.g. f32 load -> i32 load can be profitable on ARM.
-  virtual bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const {
+  virtual bool isDesirableToTransformToIntegerOp(unsigned /*Opc*/,
+                                                 EVT /*VT*/) const {
     return false;
   }
 
   /// IsDesirableToPromoteOp - This method query the target whether it is
   /// beneficial for dag combiner to promote the specified node. If true, it
   /// should return the desired promotion type by reference.
-  virtual bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
+  virtual bool IsDesirableToPromoteOp(SDValue /*Op*/, EVT &/*PVT*/) const {
     return false;
   }
 
@@ -934,6 +962,12 @@ class TargetLowering {
   /// setBooleanContents - Specify how the target extends the result of a
   /// boolean value from i1 to a wider type.  See getBooleanContents.
   void setBooleanContents(BooleanContent Ty) { BooleanContents = Ty; }
+  /// setBooleanVectorContents - Specify how the target extends the result
+  /// of a vector boolean value from a vector of i1 to a wider type.  See
+  /// getBooleanContents.
+  void setBooleanVectorContents(BooleanContent Ty) {
+    BooleanVectorContents = Ty;
+  }
 
   /// setSchedulingPreference - Specify the target scheduling preference.
   void setSchedulingPreference(Sched::Preference Pref) {
@@ -1137,6 +1171,13 @@ class TargetLowering {
     ShouldFoldAtomicFences = fold;
   }
 
+  /// setInsertFencesForAtomic - Set if the the DAG builder should
+  /// automatically insert fences and reduce the order of atomic memory
+  /// operations to Monotonic.
+  void setInsertFencesForAtomic(bool fence) {
+    InsertFencesForAtomic = fence;
+  }
+
 public:
   //===--------------------------------------------------------------------===//
   // Lowering methods - These methods must be implemented by targets so that
@@ -1150,11 +1191,11 @@ class TargetLowering {
   /// chain value.
   ///
   virtual SDValue
-    LowerFormalArguments(SDValue Chain,
-                         CallingConv::ID CallConv, bool isVarArg,
-                         const SmallVectorImpl<ISD::InputArg> &Ins,
-                         DebugLoc dl, SelectionDAG &DAG,
-                         SmallVectorImpl<SDValue> &InVals) const {
+    LowerFormalArguments(SDValue /*Chain*/, CallingConv::ID /*CallConv*/,
+                         bool /*isVarArg*/,
+                         const SmallVectorImpl<ISD::InputArg> &/*Ins*/,
+                         DebugLoc /*dl*/, SelectionDAG &/*DAG*/,
+                         SmallVectorImpl<SDValue> &/*InVals*/) const {
     assert(0 && "Not Implemented");
     return SDValue();    // this is here to silence compiler errors
   }
@@ -1166,7 +1207,7 @@ class TargetLowering {
   /// lowering.
   struct ArgListEntry {
     SDValue Node;
-    const Type* Ty;
+    Type* Ty;
     bool isSExt  : 1;
     bool isZExt  : 1;
     bool isInReg : 1;
@@ -1180,7 +1221,7 @@ class TargetLowering {
   };
   typedef std::vector<ArgListEntry> ArgListTy;
   std::pair<SDValue, SDValue>
-  LowerCallTo(SDValue Chain, const Type *RetTy, bool RetSExt, bool RetZExt,
+  LowerCallTo(SDValue Chain, Type *RetTy, bool RetSExt, bool RetZExt,
               bool isVarArg, bool isInreg, unsigned NumFixedArgs,
               CallingConv::ID CallConv, bool isTailCall,
               bool isReturnValueUsed, SDValue Callee, ArgListTy &Args,
@@ -1193,13 +1234,14 @@ class TargetLowering {
   /// InVals array with legal-type return values from the call, and return
   /// the resulting token chain value.
   virtual SDValue
-    LowerCall(SDValue Chain, SDValue Callee,
-              CallingConv::ID CallConv, bool isVarArg, bool &isTailCall,
-              const SmallVectorImpl<ISD::OutputArg> &Outs,
-              const SmallVectorImpl<SDValue> &OutVals,
-              const SmallVectorImpl<ISD::InputArg> &Ins,
-              DebugLoc dl, SelectionDAG &DAG,
-              SmallVectorImpl<SDValue> &InVals) const {
+    LowerCall(SDValue /*Chain*/, SDValue /*Callee*/,
+              CallingConv::ID /*CallConv*/, bool /*isVarArg*/,
+              bool &/*isTailCall*/,
+              const SmallVectorImpl<ISD::OutputArg> &/*Outs*/,
+              const SmallVectorImpl<SDValue> &/*OutVals*/,
+              const SmallVectorImpl<ISD::InputArg> &/*Ins*/,
+              DebugLoc /*dl*/, SelectionDAG &/*DAG*/,
+              SmallVectorImpl<SDValue> &/*InVals*/) const {
     assert(0 && "Not Implemented");
     return SDValue();    // this is here to silence compiler errors
   }
@@ -1211,10 +1253,10 @@ class TargetLowering {
   /// return values described by the Outs array can fit into the return
   /// registers.  If false is returned, an sret-demotion is performed.
   ///
-  virtual bool CanLowerReturn(CallingConv::ID CallConv,
-			      MachineFunction &MF, bool isVarArg,
-               const SmallVectorImpl<ISD::OutputArg> &Outs,
-               LLVMContext &Context) const
+  virtual bool CanLowerReturn(CallingConv::ID /*CallConv*/,
+			      MachineFunction &/*MF*/, bool /*isVarArg*/,
+               const SmallVectorImpl<ISD::OutputArg> &/*Outs*/,
+               LLVMContext &/*Context*/) const
   {
     // Return true by default to get preexisting behavior.
     return true;
@@ -1226,10 +1268,11 @@ class TargetLowering {
   /// value.
   ///
   virtual SDValue
-    LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
-                const SmallVectorImpl<ISD::OutputArg> &Outs,
-                const SmallVectorImpl<SDValue> &OutVals,
-                DebugLoc dl, SelectionDAG &DAG) const {
+    LowerReturn(SDValue /*Chain*/, CallingConv::ID /*CallConv*/,
+                bool /*isVarArg*/,
+                const SmallVectorImpl<ISD::OutputArg> &/*Outs*/,
+                const SmallVectorImpl<SDValue> &/*OutVals*/,
+                DebugLoc /*dl*/, SelectionDAG &/*DAG*/) const {
     assert(0 && "Not Implemented");
     return SDValue();    // this is here to silence compiler errors
   }
@@ -1237,7 +1280,7 @@ class TargetLowering {
   /// isUsedByReturnOnly - Return true if result of the specified node is used
   /// by a return node only. This is used to determine whether it is possible
   /// to codegen a libcall as tail call at legalization time.
-  virtual bool isUsedByReturnOnly(SDNode *N) const {
+  virtual bool isUsedByReturnOnly(SDNode *) const {
     return false;
   }
 
@@ -1245,7 +1288,7 @@ class TargetLowering {
   /// call instruction as a tail call. This is used by optimization passes to
   /// determine if it's profitable to duplicate return instructions to enable
   /// tailcall optimization.
-  virtual bool mayBeEmittedAsTailCall(CallInst *CI) const {
+  virtual bool mayBeEmittedAsTailCall(CallInst *) const {
     return false;
   }
 
@@ -1256,7 +1299,7 @@ class TargetLowering {
   /// necessary for non-C calling conventions. The frontend should handle this
   /// and include all of the necessary information.
   virtual EVT getTypeForExtArgOrReturn(LLVMContext &Context, EVT VT,
-                                       ISD::NodeType ExtendKind) const {
+                                       ISD::NodeType /*ExtendKind*/) const {
     EVT MinVT = getRegisterType(Context, MVT::i32);
     return VT.bitsLT(MinVT) ? MinVT : VT;
   }
@@ -1293,8 +1336,9 @@ class TargetLowering {
   ///
   /// If the target has no operations that require custom lowering, it need not
   /// implement this.  The default implementation aborts.
-  virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
-                                  SelectionDAG &DAG) const {
+  virtual void ReplaceNodeResults(SDNode * /*N*/,
+                                  SmallVectorImpl<SDValue> &/*Results*/,
+                                  SelectionDAG &/*DAG*/) const {
     assert(0 && "ReplaceNodeResults not implemented for this target!");
   }
 
@@ -1304,7 +1348,7 @@ class TargetLowering {
 
   /// createFastISel - This method returns a target specific FastISel object,
   /// or null if the target does not support "fast" ISel.
-  virtual FastISel *createFastISel(FunctionLoweringInfo &funcInfo) const {
+  virtual FastISel *createFastISel(FunctionLoweringInfo &) const {
     return 0;
   }
 
@@ -1316,7 +1360,7 @@ class TargetLowering {
   /// call to be explicit llvm code if it wants to.  This is useful for
   /// turning simple inline asms into LLVM intrinsics, which gives the
   /// compiler more information about the behavior of the code.
-  virtual bool ExpandInlineAsm(CallInst *CI) const {
+  virtual bool ExpandInlineAsm(CallInst *) const {
     return false;
   }
 
@@ -1460,6 +1504,13 @@ class TargetLowering {
   virtual MachineBasicBlock *
     EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const;
 
+  /// AdjustInstrPostInstrSelection - This method should be implemented by
+  /// targets that mark instructions with the 'hasPostISelHook' flag. These
+  /// instructions must be adjusted after instruction selection by target hooks.
+  /// e.g. To fill in optional defs for ARM 's' setting instructions.
+  virtual void
+  AdjustInstrPostInstrSelection(MachineInstr *MI, SDNode *Node) const;
+
   //===--------------------------------------------------------------------===//
   // Addressing mode description hooks (used by LSR etc).
   //
@@ -1485,16 +1536,32 @@ class TargetLowering {
   /// The type may be VoidTy, in which case only return true if the addressing
   /// mode is legal for a load/store of any legal type.
   /// TODO: Handle pre/postinc as well.
-  virtual bool isLegalAddressingMode(const AddrMode &AM, const Type *Ty) const;
+  virtual bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const;
+
+  /// isLegalICmpImmediate - Return true if the specified immediate is legal
+  /// icmp immediate, that is the target has icmp instructions which can compare
+  /// a register against the immediate without having to materialize the
+  /// immediate into a register.
+  virtual bool isLegalICmpImmediate(int64_t) const {
+    return true;
+  }
+
+  /// isLegalAddImmediate - Return true if the specified immediate is legal
+  /// add immediate, that is the target has add instructions which can add
+  /// a register with the immediate without having to materialize the
+  /// immediate into a register.
+  virtual bool isLegalAddImmediate(int64_t) const {
+    return true;
+  }
 
   /// isTruncateFree - Return true if it's free to truncate a value of
   /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
   /// register EAX to i16 by referencing its sub-register AX.
-  virtual bool isTruncateFree(const Type *Ty1, const Type *Ty2) const {
+  virtual bool isTruncateFree(Type * /*Ty1*/, Type * /*Ty2*/) const {
     return false;
   }
 
-  virtual bool isTruncateFree(EVT VT1, EVT VT2) const {
+  virtual bool isTruncateFree(EVT /*VT1*/, EVT /*VT2*/) const {
     return false;
   }
 
@@ -1506,37 +1573,21 @@ class TargetLowering {
   /// does not necessarily apply to truncate instructions. e.g. on x86-64,
   /// all instructions that define 32-bit values implicit zero-extend the
   /// result out to 64 bits.
-  virtual bool isZExtFree(const Type *Ty1, const Type *Ty2) const {
+  virtual bool isZExtFree(Type * /*Ty1*/, Type * /*Ty2*/) const {
     return false;
   }
 
-  virtual bool isZExtFree(EVT VT1, EVT VT2) const {
+  virtual bool isZExtFree(EVT /*VT1*/, EVT /*VT2*/) const {
     return false;
   }
 
   /// isNarrowingProfitable - Return true if it's profitable to narrow
   /// operations of type VT1 to VT2. e.g. on x86, it's profitable to narrow
   /// from i32 to i8 but not from i32 to i16.
-  virtual bool isNarrowingProfitable(EVT VT1, EVT VT2) const {
+  virtual bool isNarrowingProfitable(EVT /*VT1*/, EVT /*VT2*/) const {
     return false;
   }
 
-  /// isLegalICmpImmediate - Return true if the specified immediate is legal
-  /// icmp immediate, that is the target has icmp instructions which can compare
-  /// a register against the immediate without having to materialize the
-  /// immediate into a register.
-  virtual bool isLegalICmpImmediate(int64_t Imm) const {
-    return true;
-  }
-
-  /// isLegalAddImmediate - Return true if the specified immediate is legal
-  /// add immediate, that is the target has add instructions which can add
-  /// a register with the immediate without having to materialize the
-  /// immediate into a register.
-  virtual bool isLegalAddImmediate(int64_t Imm) const {
-    return true;
-  }
-
   //===--------------------------------------------------------------------===//
   // Div utility functions
   //
@@ -1639,6 +1690,10 @@ class TargetLowering {
   /// BooleanContents - Information about the contents of the high-bits in
   /// boolean values held in a type wider than i1.  See getBooleanContents.
   BooleanContent BooleanContents;
+  /// BooleanVectorContents - Information about the contents of the high-bits
+  /// in boolean vector values when the element type is wider than i1.  See
+  /// getBooleanContents.
+  BooleanContent BooleanVectorContents;
 
   /// SchedPreferenceInfo - The target scheduling preference: shortest possible
   /// total cycles or lowest register usage.
@@ -1676,6 +1731,11 @@ class TargetLowering {
   /// combiner.
   bool ShouldFoldAtomicFences;
 
+  /// InsertFencesForAtomic - Whether the DAG builder should automatically
+  /// insert fences and reduce ordering for atomics.  (This will be set for
+  /// for most architectures with weak memory ordering.)
+  bool InsertFencesForAtomic;
+
   /// StackPointerRegisterToSaveRestore - If set to a physical register, this
   /// specifies the register that llvm.savestack/llvm.restorestack should save
   /// and restore.
@@ -1963,7 +2023,7 @@ class TargetLowering {
 /// GetReturnInfo - Given an LLVM IR type and return type attributes,
 /// compute the return value EVTs and flags, and optionally also
 /// the offsets, if the return value is being lowered to memory.
-void GetReturnInfo(const Type* ReturnType, Attributes attr,
+void GetReturnInfo(Type* ReturnType, Attributes attr,
                    SmallVectorImpl<ISD::OutputArg> &Outs,
                    const TargetLowering &TLI,
                    SmallVectorImpl<uint64_t> *Offsets = 0);
diff --git a/contrib/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/contrib/llvm/include/llvm/Target/TargetLoweringObjectFile.h
index 2e1d6b97e733..7d06cec0a4e1 100644
--- a/contrib/llvm/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/contrib/llvm/include/llvm/Target/TargetLoweringObjectFile.h
@@ -16,6 +16,7 @@
 #define LLVM_TARGET_TARGETLOWERINGOBJECTFILE_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/SectionKind.h"
 
 namespace llvm {
@@ -31,137 +32,27 @@ namespace llvm {
   class GlobalValue;
   class TargetMachine;
   
-class TargetLoweringObjectFile {
+class TargetLoweringObjectFile : public MCObjectFileInfo {
   MCContext *Ctx;
   
   TargetLoweringObjectFile(const TargetLoweringObjectFile&); // DO NOT IMPLEMENT
   void operator=(const TargetLoweringObjectFile&);           // DO NOT IMPLEMENT
-protected:
   
-  TargetLoweringObjectFile();
-  
-  /// TextSection - Section directive for standard text.
-  ///
-  const MCSection *TextSection;
-  
-  /// DataSection - Section directive for standard data.
-  ///
-  const MCSection *DataSection;
-  
-  /// BSSSection - Section that is default initialized to zero.
-  const MCSection *BSSSection;
-  
-  /// ReadOnlySection - Section that is readonly and can contain arbitrary
-  /// initialized data.  Targets are not required to have a readonly section.
-  /// If they don't, various bits of code will fall back to using the data
-  /// section for constants.
-  const MCSection *ReadOnlySection;
-  
-  /// StaticCtorSection - This section contains the static constructor pointer
-  /// list.
-  const MCSection *StaticCtorSection;
-
-  /// StaticDtorSection - This section contains the static destructor pointer
-  /// list.
-  const MCSection *StaticDtorSection;
-  
-  /// LSDASection - If exception handling is supported by the target, this is
-  /// the section the Language Specific Data Area information is emitted to.
-  const MCSection *LSDASection;
-
-  /// CompactUnwindSection - If exception handling is supported by the target
-  /// and the target can support a compact representation of the CIE and FDE,
-  /// this is the section to emit them into.
-  const MCSection *CompactUnwindSection;
-  
-  // Dwarf sections for debug info.  If a target supports debug info, these must
-  // be set.
-  const MCSection *DwarfAbbrevSection;
-  const MCSection *DwarfInfoSection;
-  const MCSection *DwarfLineSection;
-  const MCSection *DwarfFrameSection;
-  const MCSection *DwarfPubNamesSection;
-  const MCSection *DwarfPubTypesSection;
-  const MCSection *DwarfDebugInlineSection;
-  const MCSection *DwarfStrSection;
-  const MCSection *DwarfLocSection;
-  const MCSection *DwarfARangesSection;
-  const MCSection *DwarfRangesSection;
-  const MCSection *DwarfMacroInfoSection;
-  
-  // Extra TLS Variable Data section.  If the target needs to put additional
-  // information for a TLS variable, it'll go here.
-  const MCSection *TLSExtraDataSection;
-  
-  /// CommDirectiveSupportsAlignment - True if .comm supports alignment.  This
-  /// is a hack for as long as we support 10.4 Tiger, whose assembler doesn't
-  /// support alignment on comm.
-  bool CommDirectiveSupportsAlignment;
-  
-  /// SupportsWeakEmptyEHFrame - True if target object file supports a
-  /// weak_definition of constant 0 for an omitted EH frame.
-  bool SupportsWeakOmittedEHFrame;
-  
-  /// IsFunctionEHFrameSymbolPrivate - This flag is set to true if the
-  /// "EH_frame" symbol for EH information should be an assembler temporary (aka
-  /// private linkage, aka an L or .L label) or false if it should be a normal
-  /// non-.globl label.  This defaults to true.
-  bool IsFunctionEHFrameSymbolPrivate;
-
 public:
   MCContext &getContext() const { return *Ctx; }
+
+  TargetLoweringObjectFile() : MCObjectFileInfo(), Ctx(0) {}
   
   virtual ~TargetLoweringObjectFile();
   
   /// Initialize - this method must be called before any actual lowering is
   /// done.  This specifies the current context for codegen, and gives the
   /// lowering implementations a chance to set up their default sections.
-  virtual void Initialize(MCContext &ctx, const TargetMachine &TM) {
-    Ctx = &ctx;
-  }
+  virtual void Initialize(MCContext &ctx, const TargetMachine &TM);
   
-  bool isFunctionEHFrameSymbolPrivate() const {
-    return IsFunctionEHFrameSymbolPrivate;
-  }
-  bool getSupportsWeakOmittedEHFrame() const {
-    return SupportsWeakOmittedEHFrame;
-  }
-  bool getCommDirectiveSupportsAlignment() const {
-    return CommDirectiveSupportsAlignment;
-  }
-
-  const MCSection *getTextSection() const { return TextSection; }
-  const MCSection *getDataSection() const { return DataSection; }
-  const MCSection *getBSSSection() const { return BSSSection; }
-  const MCSection *getStaticCtorSection() const { return StaticCtorSection; }
-  const MCSection *getStaticDtorSection() const { return StaticDtorSection; }
-  const MCSection *getLSDASection() const { return LSDASection; }
-  const MCSection *getCompactUnwindSection() const{return CompactUnwindSection;}
-  virtual const MCSection *getEHFrameSection() const = 0;
   virtual void emitPersonalityValue(MCStreamer &Streamer,
                                     const TargetMachine &TM,
                                     const MCSymbol *Sym) const;
-  const MCSection *getDwarfAbbrevSection() const { return DwarfAbbrevSection; }
-  const MCSection *getDwarfInfoSection() const { return DwarfInfoSection; }
-  const MCSection *getDwarfLineSection() const { return DwarfLineSection; }
-  const MCSection *getDwarfFrameSection() const { return DwarfFrameSection; }
-  const MCSection *getDwarfPubNamesSection() const{return DwarfPubNamesSection;}
-  const MCSection *getDwarfPubTypesSection() const{return DwarfPubTypesSection;}
-  const MCSection *getDwarfDebugInlineSection() const {
-    return DwarfDebugInlineSection;
-  }
-  const MCSection *getDwarfStrSection() const { return DwarfStrSection; }
-  const MCSection *getDwarfLocSection() const { return DwarfLocSection; }
-  const MCSection *getDwarfARangesSection() const { return DwarfARangesSection;}
-  const MCSection *getDwarfRangesSection() const { return DwarfRangesSection; }
-  const MCSection *getDwarfMacroInfoSection() const {
-    return DwarfMacroInfoSection;
-  }
-  const MCSection *getTLSExtraDataSection() const {
-    return TLSExtraDataSection;
-  }
-  virtual const MCSection *getWin64EHFuncTableSection(StringRef suffix)const=0;
-  virtual const MCSection *getWin64EHTableSection(StringRef suffix) const = 0;
   
   /// shouldEmitUsedDirectiveFor - This hook allows targets to selectively
   /// decide not to emit the UsedDirective for some symbols in llvm.used.
@@ -231,11 +122,6 @@ class TargetLoweringObjectFile {
   getExprForDwarfReference(const MCSymbol *Sym, unsigned Encoding,
                            MCStreamer &Streamer) const;
   
-  virtual unsigned getPersonalityEncoding() const;
-  virtual unsigned getLSDAEncoding() const;
-  virtual unsigned getFDEEncoding(bool CFI) const;
-  virtual unsigned getTTypeEncoding() const;
-
 protected:
   virtual const MCSection *
   SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
diff --git a/contrib/llvm/include/llvm/Target/TargetMachine.h b/contrib/llvm/include/llvm/Target/TargetMachine.h
index ac41a58ccda2..8a8d14229055 100644
--- a/contrib/llvm/include/llvm/Target/TargetMachine.h
+++ b/contrib/llvm/include/llvm/Target/TargetMachine.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TARGET_TARGETMACHINE_H
 #define LLVM_TARGET_TARGETMACHINE_H
 
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
 #include <string>
@@ -23,6 +24,7 @@ namespace llvm {
 class InstrItineraryData;
 class JITCodeEmitter;
 class MCAsmInfo;
+class MCCodeGenInfo;
 class MCContext;
 class Pass;
 class PassManager;
@@ -41,27 +43,6 @@ class TargetSubtargetInfo;
 class formatted_raw_ostream;
 class raw_ostream;
 
-// Relocation model types.
-namespace Reloc {
-  enum Model {
-    Default,
-    Static,
-    PIC_,         // Cannot be named PIC due to collision with -DPIC
-    DynamicNoPIC
-  };
-}
-
-// Code model types.
-namespace CodeModel {
-  enum Model {
-    Default,
-    Small,
-    Kernel,
-    Medium,
-    Large
-  };
-}
-
 // Code generation optimization level.
 namespace CodeGenOpt {
   enum Level {
@@ -108,6 +89,9 @@ class TargetMachine {
   std::string TargetCPU;
   std::string TargetFS;
 
+  /// CodeGenInfo - Low level target information such as relocation model.
+  const MCCodeGenInfo *CodeGenInfo;
+
   /// AsmInfo - Contains target specific asm information.
   ///
   const MCAsmInfo *AsmInfo;
@@ -214,19 +198,11 @@ class TargetMachine {
 
   /// getRelocationModel - Returns the code generation relocation model. The
   /// choices are static, PIC, and dynamic-no-pic, and target default.
-  static Reloc::Model getRelocationModel();
-
-  /// setRelocationModel - Sets the code generation relocation model.
-  ///
-  static void setRelocationModel(Reloc::Model Model);
+  Reloc::Model getRelocationModel() const;
 
   /// getCodeModel - Returns the code model. The choices are small, kernel,
   /// medium, large, and target default.
-  static CodeModel::Model getCodeModel();
-
-  /// setCodeModel - Sets the code model.
-  ///
-  static void setCodeModel(CodeModel::Model Model);
+  CodeModel::Model getCodeModel() const;
 
   /// getAsmVerbosityDefault - Returns the default value of asm verbosity.
   ///
@@ -309,7 +285,8 @@ class TargetMachine {
 class LLVMTargetMachine : public TargetMachine {
 protected: // Can only create subclasses.
   LLVMTargetMachine(const Target &T, StringRef TargetTriple,
-                    StringRef CPU, StringRef FS);
+                    StringRef CPU, StringRef FS,
+                    Reloc::Model RM, CodeModel::Model CM);
 
 private:
   /// addCommonCodeGenPasses - Add standard LLVM codegen passes used for
@@ -318,9 +295,6 @@ class LLVMTargetMachine : public TargetMachine {
   bool addCommonCodeGenPasses(PassManagerBase &, CodeGenOpt::Level,
                               bool DisableVerify, MCContext *&OutCtx);
 
-  virtual void setCodeModelForJIT();
-  virtual void setCodeModelForStatic();
-
 public:
   /// addPassesToEmitFile - Add passes to the specified pass manager to get the
   /// specified file emitted.  Typically this will involve several steps of code
diff --git a/contrib/llvm/include/llvm/Target/TargetOptions.h b/contrib/llvm/include/llvm/Target/TargetOptions.h
index 55d50d977dc4..e07e8c1cea08 100644
--- a/contrib/llvm/include/llvm/Target/TargetOptions.h
+++ b/contrib/llvm/include/llvm/Target/TargetOptions.h
@@ -158,6 +158,8 @@ namespace llvm {
   /// instead of an ISD::TRAP node.
   extern StringRef getTrapFunctionName();
 
+  extern bool EnableSegmentedStacks;
+
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/include/llvm/Target/TargetRegisterInfo.h b/contrib/llvm/include/llvm/Target/TargetRegisterInfo.h
index 8d827f117bad..682aa50736db 100644
--- a/contrib/llvm/include/llvm/Target/TargetRegisterInfo.h
+++ b/contrib/llvm/include/llvm/Target/TargetRegisterInfo.h
@@ -20,7 +20,6 @@
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseSet.h"
 #include <cassert>
 #include <functional>
 
@@ -36,76 +35,75 @@ class TargetRegisterClass {
 public:
   typedef const unsigned* iterator;
   typedef const unsigned* const_iterator;
-
   typedef const EVT* vt_iterator;
   typedef const TargetRegisterClass* const * sc_iterator;
 private:
-  unsigned ID;
-  const char *Name;
+  const MCRegisterClass *MC;
   const vt_iterator VTs;
-  const sc_iterator SubClasses;
+  const unsigned *SubClassMask;
   const sc_iterator SuperClasses;
-  const sc_iterator SubRegClasses;
   const sc_iterator SuperRegClasses;
-  const unsigned RegSize, Alignment;    // Size & Alignment of register in bytes
-  const int CopyCost;
-  const bool Allocatable;
-  const iterator RegsBegin, RegsEnd;
-  DenseSet<unsigned> RegSet;
 public:
-  TargetRegisterClass(unsigned id,
-                      const char *name,
-                      const EVT *vts,
-                      const TargetRegisterClass * const *subcs,
+  TargetRegisterClass(const MCRegisterClass *MC, const EVT *vts,
+                      const unsigned *subcm,
                       const TargetRegisterClass * const *supcs,
-                      const TargetRegisterClass * const *subregcs,
-                      const TargetRegisterClass * const *superregcs,
-                      unsigned RS, unsigned Al, int CC, bool Allocable,
-                      iterator RB, iterator RE)
-    : ID(id), Name(name), VTs(vts), SubClasses(subcs), SuperClasses(supcs),
-    SubRegClasses(subregcs), SuperRegClasses(superregcs),
-    RegSize(RS), Alignment(Al), CopyCost(CC), Allocatable(Allocable),
-    RegsBegin(RB), RegsEnd(RE) {
-      for (iterator I = RegsBegin, E = RegsEnd; I != E; ++I)
-        RegSet.insert(*I);
-    }
+                      const TargetRegisterClass * const *superregcs)
+    : MC(MC), VTs(vts), SubClassMask(subcm), SuperClasses(supcs),
+      SuperRegClasses(superregcs) {}
+
   virtual ~TargetRegisterClass() {}     // Allow subclasses
 
   /// getID() - Return the register class ID number.
   ///
-  unsigned getID() const { return ID; }
+  unsigned getID() const { return MC->getID(); }
 
   /// getName() - Return the register class name for debugging.
   ///
-  const char *getName() const { return Name; }
+  const char *getName() const { return MC->getName(); }
 
   /// begin/end - Return all of the registers in this class.
   ///
-  iterator       begin() const { return RegsBegin; }
-  iterator         end() const { return RegsEnd; }
+  iterator       begin() const { return MC->begin(); }
+  iterator         end() const { return MC->end(); }
 
   /// getNumRegs - Return the number of registers in this class.
   ///
-  unsigned getNumRegs() const { return (unsigned)(RegsEnd-RegsBegin); }
+  unsigned getNumRegs() const { return MC->getNumRegs(); }
 
   /// getRegister - Return the specified register in the class.
   ///
   unsigned getRegister(unsigned i) const {
-    assert(i < getNumRegs() && "Register number out of range!");
-    return RegsBegin[i];
+    return MC->getRegister(i);
   }
 
   /// contains - Return true if the specified register is included in this
   /// register class.  This does not include virtual registers.
   bool contains(unsigned Reg) const {
-    return RegSet.count(Reg);
+    return MC->contains(Reg);
   }
 
   /// contains - Return true if both registers are in this class.
   bool contains(unsigned Reg1, unsigned Reg2) const {
-    return contains(Reg1) && contains(Reg2);
+    return MC->contains(Reg1, Reg2);
   }
 
+  /// getSize - Return the size of the register in bytes, which is also the size
+  /// of a stack slot allocated to hold a spilled copy of this register.
+  unsigned getSize() const { return MC->getSize(); }
+
+  /// getAlignment - Return the minimum required alignment for a register of
+  /// this class.
+  unsigned getAlignment() const { return MC->getAlignment(); }
+
+  /// getCopyCost - Return the cost of copying a value between two registers in
+  /// this class. A negative number means the register class is very expensive
+  /// to copy e.g. status flag register classes.
+  int getCopyCost() const { return MC->getCopyCost(); }
+
+  /// isAllocatable - Return true if this register class may be used to create
+  /// virtual registers.
+  bool isAllocatable() const { return MC->isAllocatable(); }
+
   /// hasType - return true if this TargetRegisterClass has the ValueType vt.
   ///
   bool hasType(EVT vt) const {
@@ -127,25 +125,6 @@ class TargetRegisterClass {
     return I;
   }
 
-  /// subregclasses_begin / subregclasses_end - Loop over all of
-  /// the subreg register classes of this register class.
-  sc_iterator subregclasses_begin() const {
-    return SubRegClasses;
-  }
-
-  sc_iterator subregclasses_end() const {
-    sc_iterator I = SubRegClasses;
-    while (*I != NULL) ++I;
-    return I;
-  }
-
-  /// getSubRegisterRegClass - Return the register class of subregisters with
-  /// index SubIdx, or NULL if no such class exists.
-  const TargetRegisterClass* getSubRegisterRegClass(unsigned SubIdx) const {
-    assert(SubIdx>0 && "Invalid subregister index");
-    return SubRegClasses[SubIdx-1];
-  }
-
   /// superregclasses_begin / superregclasses_end - Loop over all of
   /// the superreg register classes of this register class.
   sc_iterator superregclasses_begin() const {
@@ -159,59 +138,44 @@ class TargetRegisterClass {
   }
 
   /// hasSubClass - return true if the specified TargetRegisterClass
-  /// is a proper subset of this TargetRegisterClass.
-  bool hasSubClass(const TargetRegisterClass *cs) const {
-    for (int i = 0; SubClasses[i] != NULL; ++i)
-      if (SubClasses[i] == cs)
-        return true;
-    return false;
+  /// is a proper sub-class of this TargetRegisterClass.
+  bool hasSubClass(const TargetRegisterClass *RC) const {
+    return RC != this && hasSubClassEq(RC);
   }
 
-  /// hasSubClassEq - Returns true if RC is a subclass of or equal to this
+  /// hasSubClassEq - Returns true if RC is a sub-class of or equal to this
   /// class.
   bool hasSubClassEq(const TargetRegisterClass *RC) const {
-    return RC == this || hasSubClass(RC);
-  }
-
-  /// subclasses_begin / subclasses_end - Loop over all of the classes
-  /// that are proper subsets of this register class.
-  sc_iterator subclasses_begin() const {
-    return SubClasses;
-  }
-
-  sc_iterator subclasses_end() const {
-    sc_iterator I = SubClasses;
-    while (*I != NULL) ++I;
-    return I;
+    unsigned ID = RC->getID();
+    return (SubClassMask[ID / 32] >> (ID % 32)) & 1;
   }
 
   /// hasSuperClass - return true if the specified TargetRegisterClass is a
-  /// proper superset of this TargetRegisterClass.
-  bool hasSuperClass(const TargetRegisterClass *cs) const {
-    for (int i = 0; SuperClasses[i] != NULL; ++i)
-      if (SuperClasses[i] == cs)
-        return true;
-    return false;
+  /// proper super-class of this TargetRegisterClass.
+  bool hasSuperClass(const TargetRegisterClass *RC) const {
+    return RC->hasSubClass(this);
   }
 
-  /// hasSuperClassEq - Returns true if RC is a superclass of or equal to this
+  /// hasSuperClassEq - Returns true if RC is a super-class of or equal to this
   /// class.
   bool hasSuperClassEq(const TargetRegisterClass *RC) const {
-    return RC == this || hasSuperClass(RC);
+    return RC->hasSubClassEq(this);
   }
 
-  /// superclasses_begin / superclasses_end - Loop over all of the classes
-  /// that are proper supersets of this register class.
-  sc_iterator superclasses_begin() const {
+  /// getSubClassMask - Returns a bit vector of subclasses, including this one.
+  /// The vector is indexed by class IDs, see hasSubClassEq() above for how to
+  /// use it.
+  const unsigned *getSubClassMask() const {
+    return SubClassMask;
+  }
+
+  /// getSuperClasses - Returns a NULL terminated list of super-classes.  The
+  /// classes are ordered by ID which is also a topological ordering from large
+  /// to small classes.  The list does NOT include the current class.
+  sc_iterator getSuperClasses() const {
     return SuperClasses;
   }
 
-  sc_iterator superclasses_end() const {
-    sc_iterator I = SuperClasses;
-    while (*I != NULL) ++I;
-    return I;
-  }
-
   /// isASubClass - return true if this TargetRegisterClass is a subset
   /// class of at least one other TargetRegisterClass.
   bool isASubClass() const {
@@ -234,25 +198,8 @@ class TargetRegisterClass {
   ///
   virtual
   ArrayRef<unsigned> getRawAllocationOrder(const MachineFunction &MF) const {
-    return ArrayRef<unsigned>(begin(), getNumRegs());
+    return makeArrayRef(begin(), getNumRegs());
   }
-
-  /// getSize - Return the size of the register in bytes, which is also the size
-  /// of a stack slot allocated to hold a spilled copy of this register.
-  unsigned getSize() const { return RegSize; }
-
-  /// getAlignment - Return the minimum required alignment for a register of
-  /// this class.
-  unsigned getAlignment() const { return Alignment; }
-
-  /// getCopyCost - Return the cost of copying a value between two registers in
-  /// this class. A negative number means the register class is very expensive
-  /// to copy e.g. status flag register classes.
-  int getCopyCost() const { return CopyCost; }
-
-  /// isAllocatable - Return true if this register class may be used to create
-  /// virtual registers.
-  bool isAllocatable() const { return Allocatable; }
 };
 
 /// TargetRegisterInfoDesc - Extra information, not in MCRegisterDesc, about
@@ -461,6 +408,20 @@ class TargetRegisterInfo : public MCRegisterInfo {
     return 0;
   }
 
+  /// getSubClassWithSubReg - Returns the largest legal sub-class of RC that
+  /// supports the sub-register index Idx.
+  /// If no such sub-class exists, return NULL.
+  /// If all registers in RC already have an Idx sub-register, return RC.
+  ///
+  /// TableGen generates a version of this function that is good enough in most
+  /// cases.  Targets can override if they have constraints that TableGen
+  /// doesn't understand.  For example, the x86 sub_8bit sub-register index is
+  /// supported by the full GR32 register class in 64-bit mode, but only by the
+  /// GR32_ABCD regiister class in 32-bit mode.
+  ///
+  virtual const TargetRegisterClass *
+  getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const =0;
+
   /// composeSubRegIndices - Return the subregister index you get from composing
   /// two subregister indices.
   ///
@@ -498,6 +459,12 @@ class TargetRegisterInfo : public MCRegisterInfo {
     return RegClassBegin[i];
   }
 
+  /// getCommonSubClass - find the largest common subclass of A and B. Return
+  /// NULL if there is no common subclass.
+  const TargetRegisterClass *
+  getCommonSubClass(const TargetRegisterClass *A,
+                    const TargetRegisterClass *B) const;
+
   /// getPointerRegClass - Returns a TargetRegisterClass used for pointer
   /// values.  If a target supports multiple different pointer register classes,
   /// kind specifies which one is indicated.
@@ -699,28 +666,10 @@ class TargetRegisterInfo : public MCRegisterInfo {
   //===--------------------------------------------------------------------===//
   /// Debug information queries.
 
-  /// getDwarfRegNum - Map a target register to an equivalent dwarf register
-  /// number.  Returns -1 if there is no equivalent value.  The second
-  /// parameter allows targets to use different numberings for EH info and
-  /// debugging info.
-  virtual int getDwarfRegNum(unsigned RegNum, bool isEH) const = 0;
-
-  virtual int getLLVMRegNum(unsigned RegNum, bool isEH) const = 0;
-
   /// getFrameRegister - This method should return the register used as a base
   /// for values allocated in the current stack frame.
   virtual unsigned getFrameRegister(const MachineFunction &MF) const = 0;
 
-  /// getRARegister - This method should return the register where the return
-  /// address can be found.
-  virtual unsigned getRARegister() const = 0;
-
-  /// getSEHRegNum - Map a target register to an equivalent SEH register
-  /// number.  Returns -1 if there is no equivalent value.
-  virtual int getSEHRegNum(unsigned i) const {
-    return i;
-  }
-
   /// getCompactUnwindRegNum - This function maps the register to the number for
   /// compact unwind encoding. Return -1 if the register isn't valid.
   virtual int getCompactUnwindRegNum(unsigned, bool) const {
@@ -736,11 +685,6 @@ struct VirtReg2IndexFunctor : public std::unary_function<unsigned, unsigned> {
   }
 };
 
-/// getCommonSubClass - find the largest common subclass of A and B. Return NULL
-/// if there is no common subclass.
-const TargetRegisterClass *getCommonSubClass(const TargetRegisterClass *A,
-                                             const TargetRegisterClass *B);
-
 /// PrintReg - Helper class for printing registers on a raw_ostream.
 /// Prints virtual and physical registers with or without a TRI instance.
 ///
diff --git a/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td b/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td
index 9d1ef2c13729..612635ea746d 100644
--- a/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -149,6 +149,10 @@ def SDTSelect : SDTypeProfile<1, 3, [       // select
   SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>
 ]>;
 
+def SDTVSelect : SDTypeProfile<1, 3, [       // vselect
+  SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>
+]>;
+
 def SDTSelectCC : SDTypeProfile<1, 5, [     // select_cc
   SDTCisSameAs<1, 2>, SDTCisSameAs<3, 4>, SDTCisSameAs<0, 3>,
   SDTCisVT<5, OtherVT>
@@ -205,12 +209,21 @@ def SDTMemBarrier : SDTypeProfile<0, 5, [   // memory barier
   SDTCisSameAs<0,1>,  SDTCisSameAs<0,2>,  SDTCisSameAs<0,3>, SDTCisSameAs<0,4>,
   SDTCisInt<0>
 ]>;
+def SDTAtomicFence : SDTypeProfile<0, 2, [
+  SDTCisSameAs<0,1>, SDTCisPtrTy<0>
+]>;
 def SDTAtomic3 : SDTypeProfile<1, 3, [
   SDTCisSameAs<0,2>,  SDTCisSameAs<0,3>, SDTCisInt<0>, SDTCisPtrTy<1>
 ]>;
 def SDTAtomic2 : SDTypeProfile<1, 2, [
   SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1>
 ]>;
+def SDTAtomicStore : SDTypeProfile<0, 2, [
+  SDTCisPtrTy<0>, SDTCisInt<1>
+]>;
+def SDTAtomicLoad : SDTypeProfile<1, 1, [
+  SDTCisInt<0>, SDTCisPtrTy<1>
+]>;
 
 def SDTConvertOp : SDTypeProfile<1, 5, [ //cvtss, su, us, uu, ff, fs, fu, sf, su
   SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>, SDTCisPtrTy<4>, SDTCisPtrTy<5>
@@ -381,8 +394,8 @@ def f32_to_f16 : SDNode<"ISD::FP32_TO_FP16", SDTFPToIntOp>;
 
 def setcc      : SDNode<"ISD::SETCC"      , SDTSetCC>;
 def select     : SDNode<"ISD::SELECT"     , SDTSelect>;
+def vselect    : SDNode<"ISD::VSELECT"    , SDTVSelect>;
 def selectcc   : SDNode<"ISD::SELECT_CC"  , SDTSelectCC>;
-def vsetcc     : SDNode<"ISD::VSETCC"     , SDTSetCC>;
 
 def brcond     : SDNode<"ISD::BRCOND"     , SDTBrcond, [SDNPHasChain]>;
 def brind      : SDNode<"ISD::BRIND"      , SDTBrind,  [SDNPHasChain]>;
@@ -397,6 +410,9 @@ def prefetch   : SDNode<"ISD::PREFETCH"   , SDTPrefetch,
 def membarrier : SDNode<"ISD::MEMBARRIER" , SDTMemBarrier,
                         [SDNPHasChain, SDNPSideEffect]>;
 
+def atomic_fence : SDNode<"ISD::ATOMIC_FENCE" , SDTAtomicFence,
+                          [SDNPHasChain, SDNPSideEffect]>;
+
 def atomic_cmp_swap : SDNode<"ISD::ATOMIC_CMP_SWAP" , SDTAtomic3,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 def atomic_load_add : SDNode<"ISD::ATOMIC_LOAD_ADD" , SDTAtomic2,
@@ -421,6 +437,10 @@ def atomic_load_umin : SDNode<"ISD::ATOMIC_LOAD_UMIN", SDTAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_load      : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,
+                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_store     : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,
+                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 
 // Do not use ld, st directly. Use load, extload, sextload, zextload, store,
 // and truncst (see below).
@@ -838,6 +858,28 @@ defm atomic_load_min  : binary_atomic_op<atomic_load_min>;
 defm atomic_load_max  : binary_atomic_op<atomic_load_max>;
 defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;
 defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;
+defm atomic_store     : binary_atomic_op<atomic_store>;
+
+def atomic_load_8 :
+  PatFrag<(ops node:$ptr),
+          (atomic_load node:$ptr), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;
+}]>;
+def atomic_load_16 :
+  PatFrag<(ops node:$ptr),
+          (atomic_load node:$ptr), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
+def atomic_load_32 :
+  PatFrag<(ops node:$ptr),
+          (atomic_load node:$ptr), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;
+}]>;
+def atomic_load_64 :
+  PatFrag<(ops node:$ptr),
+          (atomic_load node:$ptr), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;
+}]>;
 
 //===----------------------------------------------------------------------===//
 // Selection DAG CONVERT_RNDSAT patterns
diff --git a/contrib/llvm/include/llvm/Transforms/IPO.h b/contrib/llvm/include/llvm/Transforms/IPO.h
index f025e180678b..f9d7f9e6b98a 100644
--- a/contrib/llvm/include/llvm/Transforms/IPO.h
+++ b/contrib/llvm/include/llvm/Transforms/IPO.h
@@ -49,13 +49,6 @@ ModulePass *createStripDebugDeclarePass();
 // These pass removes unused symbols' debug info.
 ModulePass *createStripDeadDebugInfoPass();
 
-//===----------------------------------------------------------------------===//
-/// createLowerSetJmpPass - This function lowers the setjmp/longjmp intrinsics
-/// to invoke/unwind instructions.  This should really be part of the C/C++
-/// front-end, but it's so much easier to write transformations in LLVM proper.
-///
-ModulePass *createLowerSetJmpPass();
-
 //===----------------------------------------------------------------------===//
 /// createConstantMergePass - This function returns a new pass that merges
 /// duplicate global constants together into a single constant that is shared.
@@ -81,7 +74,7 @@ ModulePass *createGlobalDCEPass();
 
 
 //===----------------------------------------------------------------------===//
-/// createGVExtractionPass - If deleteFn is true, this pass deletes as
+/// createGVExtractionPass - If deleteFn is true, this pass deletes
 /// the specified global values. Otherwise, it deletes as much of the module as
 /// possible, except for the global values specified.
 ///
diff --git a/contrib/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h b/contrib/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h
new file mode 100644
index 000000000000..cc74e7fefe16
--- /dev/null
+++ b/contrib/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h
@@ -0,0 +1,133 @@
+// llvm/Transforms/IPO/PassManagerBuilder.h - Build Standard Pass -*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PassManagerBuilder class, which is used to set up a
+// "standard" optimization sequence suitable for languages like C and C++.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_PASSMANAGERBUILDER_H
+#define LLVM_SUPPORT_PASSMANAGERBUILDER_H
+
+#include <vector>
+
+namespace llvm {
+  class TargetLibraryInfo;
+  class PassManagerBase;
+  class Pass;
+  class FunctionPassManager;
+
+/// PassManagerBuilder - This class is used to set up a standard optimization
+/// sequence for languages like C and C++, allowing some APIs to customize the
+/// pass sequence in various ways. A simple example of using it would be:
+///
+///  PassManagerBuilder Builder;
+///  Builder.OptLevel = 2;
+///  Builder.populateFunctionPassManager(FPM);
+///  Builder.populateModulePassManager(MPM);
+///
+/// In addition to setting up the basic passes, PassManagerBuilder allows
+/// frontends to vend a plugin API, where plugins are allowed to add extensions
+/// to the default pass manager.  They do this by specifying where in the pass
+/// pipeline they want to be added, along with a callback function that adds
+/// the pass(es).  For example, a plugin that wanted to add a loop optimization
+/// could do something like this:
+///
+/// static void addMyLoopPass(const PMBuilder &Builder, PassManagerBase &PM) {
+///   if (Builder.getOptLevel() > 2 && Builder.getOptSizeLevel() == 0)
+///     PM.add(createMyAwesomePass());
+/// }
+///   ...
+///   Builder.addExtension(PassManagerBuilder::EP_LoopOptimizerEnd,
+///                        addMyLoopPass);
+///   ...
+class PassManagerBuilder {
+public:
+
+  /// Extensions are passed the builder itself (so they can see how it is
+  /// configured) as well as the pass manager to add stuff to.
+  typedef void (*ExtensionFn)(const PassManagerBuilder &Builder,
+                              PassManagerBase &PM);
+  enum ExtensionPointTy {
+    /// EP_EarlyAsPossible - This extension point allows adding passes before
+    /// any other transformations, allowing them to see the code as it is coming
+    /// out of the frontend.
+    EP_EarlyAsPossible,
+
+    /// EP_LoopOptimizerEnd - This extension point allows adding loop passes to
+    /// the end of the loop optimizer.
+    EP_LoopOptimizerEnd,
+
+    /// EP_ScalarOptimizerLate - This extension point allows adding optimization
+    /// passes after most of the main optimizations, but before the last
+    /// cleanup-ish optimizations.
+    EP_ScalarOptimizerLate
+  };
+
+  /// The Optimization Level - Specify the basic optimization level.
+  ///    0 = -O0, 1 = -O1, 2 = -O2, 3 = -O3
+  unsigned OptLevel;
+
+  /// SizeLevel - How much we're optimizing for size.
+  ///    0 = none, 1 = -Os, 2 = -Oz
+  unsigned SizeLevel;
+
+  /// LibraryInfo - Specifies information about the runtime library for the
+  /// optimizer.  If this is non-null, it is added to both the function and
+  /// per-module pass pipeline.
+  TargetLibraryInfo *LibraryInfo;
+
+  /// Inliner - Specifies the inliner to use.  If this is non-null, it is
+  /// added to the per-module passes.
+  Pass *Inliner;
+
+  bool DisableSimplifyLibCalls;
+  bool DisableUnitAtATime;
+  bool DisableUnrollLoops;
+
+private:
+  /// ExtensionList - This is list of all of the extensions that are registered.
+  std::vector<std::pair<ExtensionPointTy, ExtensionFn> > Extensions;
+
+public:
+  PassManagerBuilder();
+  ~PassManagerBuilder();
+  /// Adds an extension that will be used by all PassManagerBuilder instances.
+  /// This is intended to be used by plugins, to register a set of
+  /// optimisations to run automatically.
+  static void addGlobalExtension(ExtensionPointTy Ty, ExtensionFn Fn);
+  void addExtension(ExtensionPointTy Ty, ExtensionFn Fn);
+
+private:
+  void addExtensionsToPM(ExtensionPointTy ETy, PassManagerBase &PM) const;
+  void addInitialAliasAnalysisPasses(PassManagerBase &PM) const;
+public:
+
+  /// populateFunctionPassManager - This fills in the function pass manager,
+  /// which is expected to be run on each function immediately as it is
+  /// generated.  The idea is to reduce the size of the IR in memory.
+  void populateFunctionPassManager(FunctionPassManager &FPM);
+
+  /// populateModulePassManager - This sets up the primary pass manager.
+  void populateModulePassManager(PassManagerBase &MPM);
+  void populateLTOPassManager(PassManagerBase &PM, bool Internalize,
+                              bool RunInliner);
+};
+/// Registers a function for adding a standard set of passes.  This should be
+/// used by optimizer plugins to allow all front ends to transparently use
+/// them.  Create a static instance of this class in your plugin, providing a
+/// private function that the PassManagerBuilder can use to add your passes.
+struct RegisterStandardPasses {
+  RegisterStandardPasses(PassManagerBuilder::ExtensionPointTy Ty,
+                         PassManagerBuilder::ExtensionFn Fn) {
+    PassManagerBuilder::addGlobalExtension(Ty, Fn);
+  }
+};
+} // end namespace llvm
+#endif
diff --git a/contrib/llvm/include/llvm/Transforms/Scalar.h b/contrib/llvm/include/llvm/Transforms/Scalar.h
index 2187d4eb7651..b1536f906d8c 100644
--- a/contrib/llvm/include/llvm/Transforms/Scalar.h
+++ b/contrib/llvm/include/llvm/Transforms/Scalar.h
@@ -174,13 +174,6 @@ extern char &DemoteRegisterToMemoryID;
 //
 FunctionPass *createReassociatePass();
 
-//===----------------------------------------------------------------------===//
-//
-// TailDuplication - Eliminate unconditional branches through controlled code
-// duplication, creating simpler CFG structures.
-//
-FunctionPass *createTailDuplicationPass();
-
 //===----------------------------------------------------------------------===//
 //
 // JumpThreading - Thread control through mult-pred/multi-succ blocks where some
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/AddrModeMatcher.h b/contrib/llvm/include/llvm/Transforms/Utils/AddrModeMatcher.h
index 0678eccb5d69..90485eb4c69c 100644
--- a/contrib/llvm/include/llvm/Transforms/Utils/AddrModeMatcher.h
+++ b/contrib/llvm/include/llvm/Transforms/Utils/AddrModeMatcher.h
@@ -58,7 +58,7 @@ class AddressingModeMatcher {
 
   /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
   /// the memory instruction that we're computing this address for.
-  const Type *AccessTy;
+  Type *AccessTy;
   Instruction *MemoryInst;
   
   /// AddrMode - This is the addressing mode that we're building up.  This is
@@ -71,7 +71,7 @@ class AddressingModeMatcher {
   bool IgnoreProfitability;
   
   AddressingModeMatcher(SmallVectorImpl<Instruction*> &AMI,
-                        const TargetLowering &T, const Type *AT,
+                        const TargetLowering &T, Type *AT,
                         Instruction *MI, ExtAddrMode &AM)
     : AddrModeInsts(AMI), TLI(T), AccessTy(AT), MemoryInst(MI), AddrMode(AM) {
     IgnoreProfitability = false;
@@ -81,7 +81,7 @@ class AddressingModeMatcher {
   /// Match - Find the maximal addressing mode that a load/store of V can fold,
   /// give an access type of AccessTy.  This returns a list of involved
   /// instructions in AddrModeInsts.
-  static ExtAddrMode Match(Value *V, const Type *AccessTy,
+  static ExtAddrMode Match(Value *V, Type *AccessTy,
                            Instruction *MemoryInst,
                            SmallVectorImpl<Instruction*> &AddrModeInsts,
                            const TargetLowering &TLI) {
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index 90eabef12fa7..6fcd160e64e4 100644
--- a/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -31,8 +31,8 @@ class ReturnInst;
 /// DeleteDeadBlock - Delete the specified block, which must have no
 /// predecessors.
 void DeleteDeadBlock(BasicBlock *BB);
-  
-  
+
+
 /// FoldSingleEntryPHINodes - We know that BB has one predecessor.  If there are
 /// any single-entry PHI nodes in it, fold them away.  This handles the case
 /// when all entries to the PHI nodes in a block are guaranteed equal, such as
@@ -75,7 +75,7 @@ void ReplaceInstWithInst(Instruction *From, Instruction *To);
 /// The output is added to Result, as pairs of <from,to> edge info.
 void FindFunctionBackedges(const Function &F,
       SmallVectorImpl<std::pair<const BasicBlock*,const BasicBlock*> > &Result);
-  
+
 
 /// GetSuccessorNumber - Search for the specified successor of basic block BB
 /// and return its position in the terminator instruction's list of
@@ -97,10 +97,10 @@ bool isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum,
 /// was split, null otherwise.
 ///
 /// If MergeIdenticalEdges is true (not the default), *all* edges from TI to the
-/// specified successor will be merged into the same critical edge block.  
-/// This is most commonly interesting with switch instructions, which may 
+/// specified successor will be merged into the same critical edge block.
+/// This is most commonly interesting with switch instructions, which may
 /// have many edges to any one destination.  This ensures that all edges to that
-/// dest go to one block instead of each going to a different block, but isn't 
+/// dest go to one block instead of each going to a different block, but isn't
 /// the standard definition of a "critical edge".
 ///
 /// It is invalid to call this function on a critical edge that starts at an
@@ -109,7 +109,8 @@ bool isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum,
 /// to.
 ///
 BasicBlock *SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
-                              Pass *P = 0, bool MergeIdenticalEdges = false);
+                              Pass *P = 0, bool MergeIdenticalEdges = false,
+                              bool DontDeleteUselessPHIs = false);
 
 inline BasicBlock *SplitCriticalEdge(BasicBlock *BB, succ_iterator SI,
                                      Pass *P = 0) {
@@ -136,19 +137,21 @@ inline bool SplitCriticalEdge(BasicBlock *Succ, pred_iterator PI, Pass *P = 0) {
 /// described above.
 inline BasicBlock *SplitCriticalEdge(BasicBlock *Src, BasicBlock *Dst,
                                      Pass *P = 0,
-                                     bool MergeIdenticalEdges = false) {
+                                     bool MergeIdenticalEdges = false,
+                                     bool DontDeleteUselessPHIs = false) {
   TerminatorInst *TI = Src->getTerminator();
   unsigned i = 0;
   while (1) {
     assert(i != TI->getNumSuccessors() && "Edge doesn't exist!");
     if (TI->getSuccessor(i) == Dst)
-      return SplitCriticalEdge(TI, i, P, MergeIdenticalEdges);
+      return SplitCriticalEdge(TI, i, P, MergeIdenticalEdges,
+                               DontDeleteUselessPHIs);
     ++i;
   }
 }
 
-/// SplitEdge -  Split the edge connecting specified block. Pass P must 
-/// not be NULL. 
+/// SplitEdge -  Split the edge connecting specified block. Pass P must
+/// not be NULL.
 BasicBlock *SplitEdge(BasicBlock *From, BasicBlock *To, Pass *P);
 
 /// SplitBlock - Split the specified block at the specified instruction - every
@@ -157,7 +160,7 @@ BasicBlock *SplitEdge(BasicBlock *From, BasicBlock *To, Pass *P);
 /// the loop info is updated.
 ///
 BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt, Pass *P);
- 
+
 /// SplitBlockPredecessors - This method transforms BB by introducing a new
 /// basic block into the function, and moving some of the predecessors of BB to
 /// be predecessors of the new block.  The new predecessors are indicated by the
@@ -174,6 +177,23 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, BasicBlock *const *Preds,
                                    unsigned NumPreds, const char *Suffix,
                                    Pass *P = 0);
 
+/// SplitLandingPadPredecessors - This method transforms the landing pad,
+/// OrigBB, by introducing two new basic blocks into the function. One of those
+/// new basic blocks gets the predecessors listed in Preds. The other basic
+/// block gets the remaining predecessors of OrigBB. The landingpad instruction
+/// OrigBB is clone into both of the new basic blocks. The new blocks are given
+/// the suffixes 'Suffix1' and 'Suffix2', and are returned in the NewBBs vector.
+///
+/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree,
+/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses. In particular,
+/// it does not preserve LoopSimplify (because it's complicated to handle the
+/// case where one of the edges being split is an exit of a loop with other
+/// exits).
+///
+void SplitLandingPadPredecessors(BasicBlock *OrigBB,ArrayRef<BasicBlock*> Preds,
+                                 const char *Suffix, const char *Suffix2,
+                                 Pass *P, SmallVectorImpl<BasicBlock*> &NewBBs);
+
 /// FoldReturnIntoUncondBranch - This method duplicates the specified return
 /// instruction into a predecessor which ends in an unconditional branch. If
 /// the return instruction returns a value defined by a PHI, propagate the
@@ -182,7 +202,7 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, BasicBlock *const *Preds,
 ReturnInst *FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
                                        BasicBlock *Pred);
 
-/// GetFirstDebugLocInBasicBlock - Return first valid DebugLoc entry in a 
+/// GetFirstDebugLocInBasicBlock - Return first valid DebugLoc entry in a
 /// given basic block.
 DebugLoc GetFirstDebugLocInBasicBlock(const BasicBlock *BB);
 
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/FunctionUtils.h b/contrib/llvm/include/llvm/Transforms/Utils/FunctionUtils.h
index 785b08f82917..8d71e43aa921 100644
--- a/contrib/llvm/include/llvm/Transforms/Utils/FunctionUtils.h
+++ b/contrib/llvm/include/llvm/Transforms/Utils/FunctionUtils.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TRANSFORMS_UTILS_FUNCTION_H
 #define LLVM_TRANSFORMS_UTILS_FUNCTION_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include <vector>
 
 namespace llvm {
@@ -22,20 +23,23 @@ namespace llvm {
   class Function;
   class Loop;
 
-  /// ExtractCodeRegion - rip out a sequence of basic blocks into a new function
+  /// ExtractCodeRegion - Rip out a sequence of basic blocks into a new
+  /// function.
   ///
   Function* ExtractCodeRegion(DominatorTree& DT,
-                              const std::vector<BasicBlock*> &code,
+                              ArrayRef<BasicBlock*> code,
                               bool AggregateArgs = false);
 
-  /// ExtractLoop - rip out a natural loop into a new function
+  /// ExtractLoop - Rip out a natural loop into a new function.
   ///
   Function* ExtractLoop(DominatorTree& DT, Loop *L,
                         bool AggregateArgs = false);
 
-  /// ExtractBasicBlock - rip out a basic block into a new function
+  /// ExtractBasicBlock - Rip out a basic block (and the associated landing pad)
+  /// into a new function.
   ///
-  Function* ExtractBasicBlock(BasicBlock *BB, bool AggregateArgs = false);
+  Function* ExtractBasicBlock(ArrayRef<BasicBlock*> BBs,
+                              bool AggregateArgs = false);
 }
 
 #endif
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
index 063d41398958..064e5501a455 100644
--- a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -39,7 +39,7 @@ class SSAUpdater {
   void *AV;
 
   /// ProtoType holds the type of the values being rewritten.
-  const Type *ProtoType;
+  Type *ProtoType;
 
   // PHI nodes are given a name based on ProtoName.
   std::string ProtoName;
@@ -56,7 +56,7 @@ class SSAUpdater {
 
   /// Initialize - Reset this object to get ready for a new set of SSA
   /// updates with type 'Ty'.  PHI nodes get a name based on 'Name'.
-  void Initialize(const Type *Ty, StringRef Name);
+  void Initialize(Type *Ty, StringRef Name);
 
   /// AddAvailableValue - Indicate that a rewritten value is available at the
   /// end of the specified block with the specified value.
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
new file mode 100644
index 000000000000..524cf5ad9793
--- /dev/null
+++ b/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
@@ -0,0 +1,58 @@
+//===-- llvm/Transforms/Utils/SimplifyIndVar.h - Indvar Utils ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines in interface for induction variable simplification. It does
+// not define any actual pass or policy, but provides a single function to
+// simplify a loop's induction variables based on ScalarEvolution.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_SIMPLIFYINDVAR_H
+#define LLVM_TRANSFORMS_UTILS_SIMPLIFYINDVAR_H
+
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+
+extern cl::opt<bool> DisableIVRewrite;
+
+class Loop;
+class LoopInfo;
+class DominatorTree;
+class ScalarEvolution;
+class LPPassManager;
+class IVUsers;
+
+/// Interface for visiting interesting IV users that are recognized but not
+/// simplified by this utility.
+class IVVisitor {
+public:
+  virtual ~IVVisitor() {}
+  virtual void visitCast(CastInst *Cast) = 0;
+};
+
+/// simplifyUsersOfIV - Simplify instructions that use this induction variable
+/// by using ScalarEvolution to analyze the IV's recurrence.
+bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, LPPassManager *LPM,
+                       SmallVectorImpl<WeakVH> &Dead, IVVisitor *V = NULL);
+
+/// SimplifyLoopIVs - Simplify users of induction variables within this
+/// loop. This does not actually change or add IVs.
+bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, LPPassManager *LPM,
+                     SmallVectorImpl<WeakVH> &Dead);
+
+/// simplifyIVUsers - Simplify instructions recorded by the IVUsers pass.
+/// This is a legacy implementation to reproduce the behavior of the
+/// IndVarSimplify pass prior to DisableIVRewrite.
+bool simplifyIVUsers(IVUsers *IU, ScalarEvolution *SE, LPPassManager *LPM,
+                     SmallVectorImpl<WeakVH> &Dead);
+
+} // namespace llvm
+
+#endif
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 3d5ee1a62b8a..7212a8c76069 100644
--- a/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -22,7 +22,8 @@ class Loop;
 class LoopInfo;
 class LPPassManager;
 
-bool UnrollLoop(Loop *L, unsigned Count, LoopInfo* LI, LPPassManager* LPM);
+bool UnrollLoop(Loop *L, unsigned Count, unsigned TripCount,
+                unsigned TripMultiple, LoopInfo* LI, LPPassManager* LPM);
 
 }
 
diff --git a/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h
index 2194373c45b3..03846567dbcc 100644
--- a/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h
+++ b/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h
@@ -66,12 +66,12 @@ namespace llvm {
   inline MDNode *MapValue(const MDNode *V, ValueToValueMapTy &VM,
                           RemapFlags Flags = RF_None,
                           ValueMapTypeRemapper *TypeMapper = 0) {
-    return (MDNode*)MapValue((const Value*)V, VM, Flags, TypeMapper);
+    return cast<MDNode>(MapValue((const Value*)V, VM, Flags, TypeMapper));
   }
   inline Constant *MapValue(const Constant *V, ValueToValueMapTy &VM,
                             RemapFlags Flags = RF_None,
                             ValueMapTypeRemapper *TypeMapper = 0) {
-    return (Constant*)MapValue((const Value*)V, VM, Flags, TypeMapper);
+    return cast<Constant>(MapValue((const Value*)V, VM, Flags, TypeMapper));
   }
   
 
diff --git a/contrib/llvm/include/llvm/Type.h b/contrib/llvm/include/llvm/Type.h
index e4ff3e1c8d1b..43b7dc578886 100644
--- a/contrib/llvm/include/llvm/Type.h
+++ b/contrib/llvm/include/llvm/Type.h
@@ -193,7 +193,7 @@ class Type {
   /// are valid for types of the same size only where no re-interpretation of 
   /// the bits is done.
   /// @brief Determine if this type could be losslessly bitcast to Ty
-  bool canLosslesslyBitCastTo(const Type *Ty) const;
+  bool canLosslesslyBitCastTo(Type *Ty) const;
 
   /// isEmptyTy - Return true if this type is empty, that is, it has no
   /// elements or all its elements are empty.
@@ -262,7 +262,7 @@ class Type {
   /// getScalarSizeInBits - If this is a vector type, return the
   /// getPrimitiveSizeInBits value for the element type. Otherwise return the
   /// getPrimitiveSizeInBits value for this type.
-  unsigned getScalarSizeInBits() const;
+  unsigned getScalarSizeInBits();
 
   /// getFPMantissaWidth - Return the width of the mantissa of this type.  This
   /// is only valid on floating point types.  If the FP type does not
@@ -271,7 +271,7 @@ class Type {
 
   /// getScalarType - If this is a vector type, return the element type,
   /// otherwise return 'this'.
-  const Type *getScalarType() const;
+  Type *getScalarType();
 
   //===--------------------------------------------------------------------===//
   // Type Iteration support.
@@ -342,7 +342,7 @@ class Type {
 
   /// getPointerTo - Return a pointer to the current type.  This is equivalent
   /// to PointerType::get(Foo, AddrSpace).
-  PointerType *getPointerTo(unsigned AddrSpace = 0) const;
+  PointerType *getPointerTo(unsigned AddrSpace = 0);
 
 private:
   /// isSizedDerivedType - Derived types like structures and arrays are sized
@@ -352,7 +352,7 @@ class Type {
 };
 
 // Printing of types.
-static inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) {
+static inline raw_ostream &operator<<(raw_ostream &OS, Type &T) {
   T.print(OS);
   return OS;
 }
@@ -387,7 +387,7 @@ template <> struct GraphTraits<const Type*> {
   typedef const Type NodeType;
   typedef Type::subtype_iterator ChildIteratorType;
 
-  static inline NodeType *getEntryNode(const Type *T) { return T; }
+  static inline NodeType *getEntryNode(NodeType *T) { return T; }
   static inline ChildIteratorType child_begin(NodeType *N) {
     return N->subtype_begin();
   }
diff --git a/contrib/llvm/include/llvm/User.h b/contrib/llvm/include/llvm/User.h
index 3f9c28e7b381..62bc9f034618 100644
--- a/contrib/llvm/include/llvm/User.h
+++ b/contrib/llvm/include/llvm/User.h
@@ -47,7 +47,7 @@ class User : public Value {
   unsigned NumOperands;
 
   void *operator new(size_t s, unsigned Us);
-  User(const Type *ty, unsigned vty, Use *OpList, unsigned NumOps)
+  User(Type *ty, unsigned vty, Use *OpList, unsigned NumOps)
     : Value(ty, vty), OperandList(OpList), NumOperands(NumOps) {}
   Use *allocHungoffUses(unsigned) const;
   void dropHungoffUses() {
diff --git a/contrib/llvm/include/llvm/Value.h b/contrib/llvm/include/llvm/Value.h
index 08fa1c90348b..a71e2fdefd72 100644
--- a/contrib/llvm/include/llvm/Value.h
+++ b/contrib/llvm/include/llvm/Value.h
@@ -91,7 +91,7 @@ class Value {
   /// printing behavior.
   virtual void printCustom(raw_ostream &O) const;
 
-  Value(const Type *Ty, unsigned scid);
+  Value(Type *Ty, unsigned scid);
 public:
   virtual ~Value();
 
@@ -183,7 +183,7 @@ class Value {
   bool isUsedInBasicBlock(const BasicBlock *BB) const;
 
   /// getNumUses - This method computes the number of uses of this Value.  This
-  /// is a linear time operation.  Use hasOneUse, hasNUses, or hasMoreThanNUses
+  /// is a linear time operation.  Use hasOneUse, hasNUses, or hasNUsesOrMore
   /// to check for specific values.
   unsigned getNumUses() const;
 
diff --git a/contrib/llvm/lib/Analysis/AliasAnalysis.cpp b/contrib/llvm/lib/Analysis/AliasAnalysis.cpp
index c189a0042928..bd132c05c327 100644
--- a/contrib/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/contrib/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -237,6 +237,19 @@ AliasAnalysis::Location AliasAnalysis::getLocation(const VAArgInst *VI) {
                   VI->getMetadata(LLVMContext::MD_tbaa));
 }
 
+AliasAnalysis::Location
+AliasAnalysis::getLocation(const AtomicCmpXchgInst *CXI) {
+  return Location(CXI->getPointerOperand(),
+                  getTypeStoreSize(CXI->getCompareOperand()->getType()),
+                  CXI->getMetadata(LLVMContext::MD_tbaa));
+}
+
+AliasAnalysis::Location
+AliasAnalysis::getLocation(const AtomicRMWInst *RMWI) {
+  return Location(RMWI->getPointerOperand(),
+                  getTypeStoreSize(RMWI->getValOperand()->getType()),
+                  RMWI->getMetadata(LLVMContext::MD_tbaa));
+}
 
 AliasAnalysis::Location 
 AliasAnalysis::getLocationForSource(const MemTransferInst *MTI) {
@@ -268,8 +281,8 @@ AliasAnalysis::getLocationForDest(const MemIntrinsic *MTI) {
 
 AliasAnalysis::ModRefResult
 AliasAnalysis::getModRefInfo(const LoadInst *L, const Location &Loc) {
-  // Be conservative in the face of volatile.
-  if (L->isVolatile())
+  // Be conservative in the face of volatile/atomic.
+  if (!L->isUnordered())
     return ModRef;
 
   // If the load address doesn't alias the given address, it doesn't read
@@ -283,8 +296,8 @@ AliasAnalysis::getModRefInfo(const LoadInst *L, const Location &Loc) {
 
 AliasAnalysis::ModRefResult
 AliasAnalysis::getModRefInfo(const StoreInst *S, const Location &Loc) {
-  // Be conservative in the face of volatile.
-  if (S->isVolatile())
+  // Be conservative in the face of volatile/atomic.
+  if (!S->isUnordered())
     return ModRef;
 
   // If the store address cannot alias the pointer in question, then the
@@ -317,6 +330,33 @@ AliasAnalysis::getModRefInfo(const VAArgInst *V, const Location &Loc) {
   return ModRef;
 }
 
+AliasAnalysis::ModRefResult
+AliasAnalysis::getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc) {
+  // Acquire/Release cmpxchg has properties that matter for arbitrary addresses.
+  if (CX->getOrdering() > Monotonic)
+    return ModRef;
+
+  // If the cmpxchg address does not alias the location, it does not access it.
+  if (!alias(getLocation(CX), Loc))
+    return NoModRef;
+
+  return ModRef;
+}
+
+AliasAnalysis::ModRefResult
+AliasAnalysis::getModRefInfo(const AtomicRMWInst *RMW, const Location &Loc) {
+  // Acquire/Release atomicrmw has properties that matter for arbitrary addresses.
+  if (RMW->getOrdering() > Monotonic)
+    return ModRef;
+
+  // If the atomicrmw address does not alias the location, it does not access it.
+  if (!alias(getLocation(RMW), Loc))
+    return NoModRef;
+
+  return ModRef;
+}
+
+
 // AliasAnalysis destructor: DO NOT move this to the header file for
 // AliasAnalysis or else clients of the AliasAnalysis class may not depend on
 // the AliasAnalysis.o file in the current .a file, causing alias analysis
@@ -341,7 +381,7 @@ void AliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
 /// getTypeStoreSize - Return the TargetData store size for the given type,
 /// if known, or a conservative value otherwise.
 ///
-uint64_t AliasAnalysis::getTypeStoreSize(const Type *Ty) {
+uint64_t AliasAnalysis::getTypeStoreSize(Type *Ty) {
   return TD ? TD->getTypeStoreSize(Ty) : UnknownSize;
 }
 
diff --git a/contrib/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp b/contrib/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp
index 1afc1b71d93e..37271b94a201 100644
--- a/contrib/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp
+++ b/contrib/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp
@@ -171,12 +171,12 @@ bool AAEval::runOnFunction(Function &F) {
   for (SetVector<Value *>::iterator I1 = Pointers.begin(), E = Pointers.end();
        I1 != E; ++I1) {
     uint64_t I1Size = AliasAnalysis::UnknownSize;
-    const Type *I1ElTy = cast<PointerType>((*I1)->getType())->getElementType();
+    Type *I1ElTy = cast<PointerType>((*I1)->getType())->getElementType();
     if (I1ElTy->isSized()) I1Size = AA.getTypeStoreSize(I1ElTy);
 
     for (SetVector<Value *>::iterator I2 = Pointers.begin(); I2 != I1; ++I2) {
       uint64_t I2Size = AliasAnalysis::UnknownSize;
-      const Type *I2ElTy =cast<PointerType>((*I2)->getType())->getElementType();
+      Type *I2ElTy =cast<PointerType>((*I2)->getType())->getElementType();
       if (I2ElTy->isSized()) I2Size = AA.getTypeStoreSize(I2ElTy);
 
       switch (AA.alias(*I1, I1Size, *I2, I2Size)) {
@@ -207,7 +207,7 @@ bool AAEval::runOnFunction(Function &F) {
     for (SetVector<Value *>::iterator V = Pointers.begin(), Ve = Pointers.end();
          V != Ve; ++V) {
       uint64_t Size = AliasAnalysis::UnknownSize;
-      const Type *ElTy = cast<PointerType>((*V)->getType())->getElementType();
+      Type *ElTy = cast<PointerType>((*V)->getType())->getElementType();
       if (ElTy->isSized()) Size = AA.getTypeStoreSize(ElTy);
 
       switch (AA.getModRefInfo(*C, *V, Size)) {
diff --git a/contrib/llvm/lib/Analysis/AliasSetTracker.cpp b/contrib/llvm/lib/Analysis/AliasSetTracker.cpp
index 2ed694941212..3fcd3b55de57 100644
--- a/contrib/llvm/lib/Analysis/AliasSetTracker.cpp
+++ b/contrib/llvm/lib/Analysis/AliasSetTracker.cpp
@@ -56,12 +56,12 @@ void AliasSet::mergeSetIn(AliasSet &AS, AliasSetTracker &AST) {
       AliasTy = MayAlias;
   }
 
-  if (CallSites.empty()) {            // Merge call sites...
-    if (!AS.CallSites.empty())
-      std::swap(CallSites, AS.CallSites);
-  } else if (!AS.CallSites.empty()) {
-    CallSites.insert(CallSites.end(), AS.CallSites.begin(), AS.CallSites.end());
-    AS.CallSites.clear();
+  if (UnknownInsts.empty()) {            // Merge call sites...
+    if (!AS.UnknownInsts.empty())
+      std::swap(UnknownInsts, AS.UnknownInsts);
+  } else if (!AS.UnknownInsts.empty()) {
+    UnknownInsts.insert(UnknownInsts.end(), AS.UnknownInsts.begin(), AS.UnknownInsts.end());
+    AS.UnknownInsts.clear();
   }
 
   AS.Forward = this;  // Forward across AS now...
@@ -123,13 +123,10 @@ void AliasSet::addPointer(AliasSetTracker &AST, PointerRec &Entry,
   addRef();               // Entry points to alias set.
 }
 
-void AliasSet::addCallSite(CallSite CS, AliasAnalysis &AA) {
-  CallSites.push_back(CS.getInstruction());
+void AliasSet::addUnknownInst(Instruction *I, AliasAnalysis &AA) {
+  UnknownInsts.push_back(I);
 
-  AliasAnalysis::ModRefBehavior Behavior = AA.getModRefBehavior(CS);
-  if (Behavior == AliasAnalysis::DoesNotAccessMemory)
-    return;
-  if (AliasAnalysis::onlyReadsMemory(Behavior)) {
+  if (!I->mayWriteToMemory()) {
     AliasTy = MayAlias;
     AccessTy |= Refs;
     return;
@@ -147,7 +144,7 @@ bool AliasSet::aliasesPointer(const Value *Ptr, uint64_t Size,
                               const MDNode *TBAAInfo,
                               AliasAnalysis &AA) const {
   if (AliasTy == MustAlias) {
-    assert(CallSites.empty() && "Illegal must alias set!");
+    assert(UnknownInsts.empty() && "Illegal must alias set!");
 
     // If this is a set of MustAliases, only check to see if the pointer aliases
     // SOME value in the set.
@@ -167,10 +164,10 @@ bool AliasSet::aliasesPointer(const Value *Ptr, uint64_t Size,
                                          I.getTBAAInfo())))
       return true;
 
-  // Check the call sites list and invoke list...
-  if (!CallSites.empty()) {
-    for (unsigned i = 0, e = CallSites.size(); i != e; ++i)
-      if (AA.getModRefInfo(CallSites[i],
+  // Check the unknown instructions...
+  if (!UnknownInsts.empty()) {
+    for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i)
+      if (AA.getModRefInfo(UnknownInsts[i],
                            AliasAnalysis::Location(Ptr, Size, TBAAInfo)) !=
             AliasAnalysis::NoModRef)
         return true;
@@ -179,18 +176,20 @@ bool AliasSet::aliasesPointer(const Value *Ptr, uint64_t Size,
   return false;
 }
 
-bool AliasSet::aliasesCallSite(CallSite CS, AliasAnalysis &AA) const {
-  if (AA.doesNotAccessMemory(CS))
+bool AliasSet::aliasesUnknownInst(Instruction *Inst, AliasAnalysis &AA) const {
+  if (!Inst->mayReadOrWriteMemory())
     return false;
 
-  for (unsigned i = 0, e = CallSites.size(); i != e; ++i) {
-    if (AA.getModRefInfo(getCallSite(i), CS) != AliasAnalysis::NoModRef ||
-        AA.getModRefInfo(CS, getCallSite(i)) != AliasAnalysis::NoModRef)
+  for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i) {
+    CallSite C1 = getUnknownInst(i), C2 = Inst;
+    if (!C1 || !C2 ||
+        AA.getModRefInfo(C1, C2) != AliasAnalysis::NoModRef ||
+        AA.getModRefInfo(C2, C1) != AliasAnalysis::NoModRef)
       return true;
   }
 
   for (iterator I = begin(), E = end(); I != E; ++I)
-    if (AA.getModRefInfo(CS, I.getPointer(), I.getSize()) !=
+    if (AA.getModRefInfo(Inst, I.getPointer(), I.getSize()) !=
            AliasAnalysis::NoModRef)
       return true;
 
@@ -244,10 +243,10 @@ bool AliasSetTracker::containsPointer(Value *Ptr, uint64_t Size,
 
 
 
-AliasSet *AliasSetTracker::findAliasSetForCallSite(CallSite CS) {
+AliasSet *AliasSetTracker::findAliasSetForUnknownInst(Instruction *Inst) {
   AliasSet *FoundSet = 0;
   for (iterator I = begin(), E = end(); I != E; ++I) {
-    if (I->Forward || !I->aliasesCallSite(CS, AA))
+    if (I->Forward || !I->aliasesUnknownInst(Inst, AA))
       continue;
     
     if (FoundSet == 0)        // If this is the first alias set ptr can go into.
@@ -296,22 +295,28 @@ bool AliasSetTracker::add(Value *Ptr, uint64_t Size, const MDNode *TBAAInfo) {
 
 
 bool AliasSetTracker::add(LoadInst *LI) {
+  if (LI->getOrdering() > Monotonic) return addUnknown(LI);
+  AliasSet::AccessType ATy = AliasSet::Refs;
+  if (!LI->isUnordered()) ATy = AliasSet::ModRef;
   bool NewPtr;
   AliasSet &AS = addPointer(LI->getOperand(0),
                             AA.getTypeStoreSize(LI->getType()),
                             LI->getMetadata(LLVMContext::MD_tbaa),
-                            AliasSet::Refs, NewPtr);
+                            ATy, NewPtr);
   if (LI->isVolatile()) AS.setVolatile();
   return NewPtr;
 }
 
 bool AliasSetTracker::add(StoreInst *SI) {
+  if (SI->getOrdering() > Monotonic) return addUnknown(SI);
+  AliasSet::AccessType ATy = AliasSet::Mods;
+  if (!SI->isUnordered()) ATy = AliasSet::ModRef;
   bool NewPtr;
   Value *Val = SI->getOperand(0);
   AliasSet &AS = addPointer(SI->getOperand(1),
                             AA.getTypeStoreSize(Val->getType()),
                             SI->getMetadata(LLVMContext::MD_tbaa),
-                            AliasSet::Mods, NewPtr);
+                            ATy, NewPtr);
   if (SI->isVolatile()) AS.setVolatile();
   return NewPtr;
 }
@@ -325,20 +330,20 @@ bool AliasSetTracker::add(VAArgInst *VAAI) {
 }
 
 
-bool AliasSetTracker::add(CallSite CS) {
-  if (isa<DbgInfoIntrinsic>(CS.getInstruction())) 
+bool AliasSetTracker::addUnknown(Instruction *Inst) {
+  if (isa<DbgInfoIntrinsic>(Inst)) 
     return true; // Ignore DbgInfo Intrinsics.
-  if (AA.doesNotAccessMemory(CS))
+  if (!Inst->mayReadOrWriteMemory())
     return true; // doesn't alias anything
 
-  AliasSet *AS = findAliasSetForCallSite(CS);
+  AliasSet *AS = findAliasSetForUnknownInst(Inst);
   if (AS) {
-    AS->addCallSite(CS, AA);
+    AS->addUnknownInst(Inst, AA);
     return false;
   }
   AliasSets.push_back(new AliasSet());
   AS = &AliasSets.back();
-  AS->addCallSite(CS, AA);
+  AS->addUnknownInst(Inst, AA);
   return true;
 }
 
@@ -348,13 +353,9 @@ bool AliasSetTracker::add(Instruction *I) {
     return add(LI);
   if (StoreInst *SI = dyn_cast<StoreInst>(I))
     return add(SI);
-  if (CallInst *CI = dyn_cast<CallInst>(I))
-    return add(CI);
-  if (InvokeInst *II = dyn_cast<InvokeInst>(I))
-    return add(II);
   if (VAArgInst *VAAI = dyn_cast<VAArgInst>(I))
     return add(VAAI);
-  return true;
+  return addUnknown(I);
 }
 
 void AliasSetTracker::add(BasicBlock &BB) {
@@ -375,8 +376,8 @@ void AliasSetTracker::add(const AliasSetTracker &AST) {
     AliasSet &AS = const_cast<AliasSet&>(*I);
 
     // If there are any call sites in the alias set, add them to this AST.
-    for (unsigned i = 0, e = AS.CallSites.size(); i != e; ++i)
-      add(AS.CallSites[i]);
+    for (unsigned i = 0, e = AS.UnknownInsts.size(); i != e; ++i)
+      add(AS.UnknownInsts[i]);
 
     // Loop over all of the pointers in this alias set.
     bool X;
@@ -393,7 +394,7 @@ void AliasSetTracker::add(const AliasSetTracker &AST) {
 /// tracker.
 void AliasSetTracker::remove(AliasSet &AS) {
   // Drop all call sites.
-  AS.CallSites.clear();
+  AS.UnknownInsts.clear();
   
   // Clear the alias set.
   unsigned NumRefs = 0;
@@ -453,11 +454,11 @@ bool AliasSetTracker::remove(VAArgInst *VAAI) {
   return true;
 }
 
-bool AliasSetTracker::remove(CallSite CS) {
-  if (AA.doesNotAccessMemory(CS))
+bool AliasSetTracker::removeUnknown(Instruction *I) {
+  if (!I->mayReadOrWriteMemory())
     return false; // doesn't alias anything
 
-  AliasSet *AS = findAliasSetForCallSite(CS);
+  AliasSet *AS = findAliasSetForUnknownInst(I);
   if (!AS) return false;
   remove(*AS);
   return true;
@@ -469,11 +470,9 @@ bool AliasSetTracker::remove(Instruction *I) {
     return remove(LI);
   if (StoreInst *SI = dyn_cast<StoreInst>(I))
     return remove(SI);
-  if (CallInst *CI = dyn_cast<CallInst>(I))
-    return remove(CI);
   if (VAArgInst *VAAI = dyn_cast<VAArgInst>(I))
     return remove(VAAI);
-  return true;
+  return removeUnknown(I);
 }
 
 
@@ -488,13 +487,13 @@ void AliasSetTracker::deleteValue(Value *PtrVal) {
 
   // If this is a call instruction, remove the callsite from the appropriate
   // AliasSet (if present).
-  if (CallSite CS = PtrVal) {
-    if (!AA.doesNotAccessMemory(CS)) {
+  if (Instruction *Inst = dyn_cast<Instruction>(PtrVal)) {
+    if (Inst->mayReadOrWriteMemory()) {
       // Scan all the alias sets to see if this call site is contained.
       for (iterator I = begin(), E = end(); I != E; ++I) {
         if (I->Forward) continue;
         
-        I->removeCallSite(CS);
+        I->removeUnknownInst(Inst);
       }
     }
   }
@@ -571,11 +570,11 @@ void AliasSet::print(raw_ostream &OS) const {
       OS << ", " << I.getSize() << ")";
     }
   }
-  if (!CallSites.empty()) {
-    OS << "\n    " << CallSites.size() << " Call Sites: ";
-    for (unsigned i = 0, e = CallSites.size(); i != e; ++i) {
+  if (!UnknownInsts.empty()) {
+    OS << "\n    " << UnknownInsts.size() << " Unknown instructions: ";
+    for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i) {
       if (i) OS << ", ";
-      WriteAsOperand(OS, CallSites[i]);
+      WriteAsOperand(OS, UnknownInsts[i]);
     }
   }
   OS << "\n";
diff --git a/contrib/llvm/lib/Analysis/Analysis.cpp b/contrib/llvm/lib/Analysis/Analysis.cpp
index 71e0a832696c..0ba6af93b511 100644
--- a/contrib/llvm/lib/Analysis/Analysis.cpp
+++ b/contrib/llvm/lib/Analysis/Analysis.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm-c/Analysis.h"
+#include "llvm-c/Initialization.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Analysis/Verifier.h"
 #include <cstring>
@@ -23,7 +24,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeAliasSetPrinterPass(Registry);
   initializeNoAAPass(Registry);
   initializeBasicAliasAnalysisPass(Registry);
-  initializeBlockFrequencyPass(Registry);
+  initializeBlockFrequencyInfoPass(Registry);
   initializeBranchProbabilityInfoPass(Registry);
   initializeCFGViewerPass(Registry);
   initializeCFGPrinterPass(Registry);
diff --git a/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 8330ea7c7036..af400ba7e70e 100644
--- a/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -100,7 +101,7 @@ static bool isEscapeSource(const Value *V) {
 /// getObjectSize - Return the size of the object specified by V, or
 /// UnknownSize if unknown.
 static uint64_t getObjectSize(const Value *V, const TargetData &TD) {
-  const Type *AccessTy;
+  Type *AccessTy;
   if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) {
     if (!GV->hasDefinitiveInitializer())
       return AliasAnalysis::UnknownSize;
@@ -317,7 +318,7 @@ DecomposeGEPExpression(const Value *V, int64_t &BaseOffs,
          E = GEPOp->op_end(); I != E; ++I) {
       Value *Index = *I;
       // Compute the (potentially symbolic) offset in bytes for this index.
-      if (const StructType *STy = dyn_cast<StructType>(*GTI++)) {
+      if (StructType *STy = dyn_cast<StructType>(*GTI++)) {
         // For a struct, add the member offset.
         unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
         if (FieldNo == 0) continue;
@@ -374,7 +375,8 @@ DecomposeGEPExpression(const Value *V, int64_t &BaseOffs,
       }
       
       if (Scale) {
-        VariableGEPIndex Entry = {Index, Extension, Scale};
+        VariableGEPIndex Entry = {Index, Extension,
+                                  static_cast<int64_t>(Scale)};
         VarIndices.push_back(Entry);
       }
     }
@@ -467,6 +469,7 @@ namespace {
 
     virtual void getAnalysisUsage(AnalysisUsage &AU) const {
       AU.addRequired<AliasAnalysis>();
+      AU.addRequired<TargetLibraryInfo>();
     }
 
     virtual AliasResult alias(const Location &LocA,
@@ -549,9 +552,14 @@ namespace {
 
 // Register this pass...
 char BasicAliasAnalysis::ID = 0;
-INITIALIZE_AG_PASS(BasicAliasAnalysis, AliasAnalysis, "basicaa",
+INITIALIZE_AG_PASS_BEGIN(BasicAliasAnalysis, AliasAnalysis, "basicaa",
                    "Basic Alias Analysis (stateless AA impl)",
                    false, true, false)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo)
+INITIALIZE_AG_PASS_END(BasicAliasAnalysis, AliasAnalysis, "basicaa",
+                   "Basic Alias Analysis (stateless AA impl)",
+                   false, true, false)
+
 
 ImmutablePass *llvm::createBasicAliasAnalysisPass() {
   return new BasicAliasAnalysis();
@@ -706,7 +714,7 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
       // is impossible to alias the pointer we're checking.  If not, we have to
       // assume that the call could touch the pointer, even though it doesn't
       // escape.
-      if (!isNoAlias(Location(cast<Value>(CI)), Loc)) {
+      if (!isNoAlias(Location(*CI), Location(Object))) {
         PassedAsArg = true;
         break;
       }
@@ -716,6 +724,7 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
       return NoModRef;
   }
 
+  const TargetLibraryInfo &TLI = getAnalysis<TargetLibraryInfo>();
   ModRefResult Min = ModRef;
 
   // Finally, handle specific knowledge of intrinsics.
@@ -754,26 +763,6 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
       // We know that memset doesn't load anything.
       Min = Mod;
       break;
-    case Intrinsic::atomic_cmp_swap:
-    case Intrinsic::atomic_swap:
-    case Intrinsic::atomic_load_add:
-    case Intrinsic::atomic_load_sub:
-    case Intrinsic::atomic_load_and:
-    case Intrinsic::atomic_load_nand:
-    case Intrinsic::atomic_load_or:
-    case Intrinsic::atomic_load_xor:
-    case Intrinsic::atomic_load_max:
-    case Intrinsic::atomic_load_min:
-    case Intrinsic::atomic_load_umax:
-    case Intrinsic::atomic_load_umin:
-      if (TD) {
-        Value *Op1 = II->getArgOperand(0);
-        uint64_t Op1Size = TD->getTypeStoreSize(Op1->getType());
-        MDNode *Tag = II->getMetadata(LLVMContext::MD_tbaa);
-        if (isNoAlias(Location(Op1, Op1Size, Tag), Loc))
-          return NoModRef;
-      }
-      break;
     case Intrinsic::lifetime_start:
     case Intrinsic::lifetime_end:
     case Intrinsic::invariant_start: {
@@ -818,6 +807,39 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
     }
     }
 
+  // We can bound the aliasing properties of memset_pattern16 just as we can
+  // for memcpy/memset.  This is particularly important because the 
+  // LoopIdiomRecognizer likes to turn loops into calls to memset_pattern16
+  // whenever possible.
+  else if (TLI.has(LibFunc::memset_pattern16) &&
+           CS.getCalledFunction() &&
+           CS.getCalledFunction()->getName() == "memset_pattern16") {
+    const Function *MS = CS.getCalledFunction();
+    FunctionType *MemsetType = MS->getFunctionType();
+    if (!MemsetType->isVarArg() && MemsetType->getNumParams() == 3 &&
+        isa<PointerType>(MemsetType->getParamType(0)) &&
+        isa<PointerType>(MemsetType->getParamType(1)) &&
+        isa<IntegerType>(MemsetType->getParamType(2))) {
+      uint64_t Len = UnknownSize;
+      if (const ConstantInt *LenCI = dyn_cast<ConstantInt>(CS.getArgument(2)))
+        Len = LenCI->getZExtValue();
+      const Value *Dest = CS.getArgument(0);
+      const Value *Src = CS.getArgument(1);
+      // If it can't overlap the source dest, then it doesn't modref the loc.
+      if (isNoAlias(Location(Dest, Len), Loc)) {
+        // Always reads 16 bytes of the source.
+        if (isNoAlias(Location(Src, 16), Loc))
+          return NoModRef;
+        // If it can't overlap the dest, then worst case it reads the loc.
+        Min = Ref;
+      // Always reads 16 bytes of the source.
+      } else if (isNoAlias(Location(Src, 16), Loc)) {
+        // If it can't overlap the source, then worst case it mutates the loc.
+        Min = Mod;
+      }
+    }
+  }
+
   // The AliasAnalysis base class has some smarts, lets use them.
   return ModRefResult(AliasAnalysis::getModRefInfo(CS, Loc) & Min);
 }
@@ -913,43 +935,43 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
   if (GEP1BaseOffset == 0 && GEP1VariableIndices.empty())
     return MustAlias;
 
-  // If there is a difference between the pointers, but the difference is
-  // less than the size of the associated memory object, then we know
-  // that the objects are partially overlapping.
+  // If there is a constant difference between the pointers, but the difference
+  // is less than the size of the associated memory object, then we know
+  // that the objects are partially overlapping.  If the difference is
+  // greater, we know they do not overlap.
   if (GEP1BaseOffset != 0 && GEP1VariableIndices.empty()) {
-    if (GEP1BaseOffset >= 0 ?
-        (V2Size != UnknownSize && (uint64_t)GEP1BaseOffset < V2Size) :
-        (V1Size != UnknownSize && -(uint64_t)GEP1BaseOffset < V1Size &&
-         GEP1BaseOffset != INT64_MIN))
-      return PartialAlias;
+    if (GEP1BaseOffset >= 0) {
+      if (V2Size != UnknownSize) {
+        if ((uint64_t)GEP1BaseOffset < V2Size)
+          return PartialAlias;
+        return NoAlias;
+      }
+    } else {
+      if (V1Size != UnknownSize) {
+        if (-(uint64_t)GEP1BaseOffset < V1Size)
+          return PartialAlias;
+        return NoAlias;
+      }
+    }
   }
 
-  // If we have a known constant offset, see if this offset is larger than the
-  // access size being queried.  If so, and if no variable indices can remove
-  // pieces of this constant, then we know we have a no-alias.  For example,
-  //   &A[100] != &A.
-  
-  // In order to handle cases like &A[100][i] where i is an out of range
-  // subscript, we have to ignore all constant offset pieces that are a multiple
-  // of a scaled index.  Do this by removing constant offsets that are a
-  // multiple of any of our variable indices.  This allows us to transform
-  // things like &A[i][1] because i has a stride of (e.g.) 8 bytes but the 1
-  // provides an offset of 4 bytes (assuming a <= 4 byte access).
-  for (unsigned i = 0, e = GEP1VariableIndices.size();
-       i != e && GEP1BaseOffset;++i)
-    if (int64_t RemovedOffset = GEP1BaseOffset/GEP1VariableIndices[i].Scale)
-      GEP1BaseOffset -= RemovedOffset*GEP1VariableIndices[i].Scale;
-  
-  // If our known offset is bigger than the access size, we know we don't have
-  // an alias.
-  if (GEP1BaseOffset) {
-    if (GEP1BaseOffset >= 0 ?
-        (V2Size != UnknownSize && (uint64_t)GEP1BaseOffset >= V2Size) :
-        (V1Size != UnknownSize && -(uint64_t)GEP1BaseOffset >= V1Size &&
-         GEP1BaseOffset != INT64_MIN))
+  // Try to distinguish something like &A[i][1] against &A[42][0].
+  // Grab the least significant bit set in any of the scales.
+  if (!GEP1VariableIndices.empty()) {
+    uint64_t Modulo = 0;
+    for (unsigned i = 0, e = GEP1VariableIndices.size(); i != e; ++i)
+      Modulo |= (uint64_t)GEP1VariableIndices[i].Scale;
+    Modulo = Modulo ^ (Modulo & (Modulo - 1));
+
+    // We can compute the difference between the two addresses
+    // mod Modulo. Check whether that difference guarantees that the
+    // two locations do not alias.
+    uint64_t ModOffset = (uint64_t)GEP1BaseOffset & (Modulo - 1);
+    if (V1Size != UnknownSize && V2Size != UnknownSize &&
+        ModOffset >= V2Size && V1Size <= Modulo - ModOffset)
       return NoAlias;
   }
-  
+
   // Statically, we can see that the base objects are the same, but the
   // pointers have dynamic offsets which we can't resolve. And none of our
   // little tricks above worked.
diff --git a/contrib/llvm/lib/Analysis/BlockFrequency.cpp b/contrib/llvm/lib/Analysis/BlockFrequency.cpp
deleted file mode 100644
index 4b86d1db1f04..000000000000
--- a/contrib/llvm/lib/Analysis/BlockFrequency.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//=======-------- BlockFrequency.cpp - Block Frequency Analysis -------=======//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Loops should be simplified before this analysis.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/InitializePasses.h"
-#include "llvm/Analysis/BlockFrequencyImpl.h"
-#include "llvm/Analysis/BlockFrequency.h"
-#include "llvm/Analysis/LoopInfo.h"
-#include "llvm/Analysis/Passes.h"
-#include "llvm/Analysis/BranchProbabilityInfo.h"
-
-using namespace llvm;
-
-INITIALIZE_PASS_BEGIN(BlockFrequency, "block-freq", "Block Frequency Analysis",
-                      true, true)
-INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfo)
-INITIALIZE_PASS_END(BlockFrequency, "block-freq", "Block Frequency Analysis",
-                    true, true)
-
-char BlockFrequency::ID = 0;
-
-
-BlockFrequency::BlockFrequency() : FunctionPass(ID) {
-  initializeBlockFrequencyPass(*PassRegistry::getPassRegistry());
-  BFI = new BlockFrequencyImpl<BasicBlock, Function, BranchProbabilityInfo>();
-}
-
-BlockFrequency::~BlockFrequency() {
-  delete BFI;
-}
-
-void BlockFrequency::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<BranchProbabilityInfo>();
-  AU.setPreservesAll();
-}
-
-bool BlockFrequency::runOnFunction(Function &F) {
-  BranchProbabilityInfo &BPI = getAnalysis<BranchProbabilityInfo>();
-  BFI->doFunction(&F, &BPI);
-  return false;
-}
-
-/// getblockFreq - Return block frequency. Never return 0, value must be
-/// positive. Please note that initial frequency is equal to 1024. It means that
-/// we should not rely on the value itself, but only on the comparison to the
-/// other block frequencies. We do this to avoid using of floating points.
-///
-uint32_t BlockFrequency::getBlockFreq(BasicBlock *BB) {
-  return BFI->getBlockFreq(BB);
-}
diff --git a/contrib/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/contrib/llvm/lib/Analysis/BlockFrequencyInfo.cpp
new file mode 100644
index 000000000000..d16665fa55cf
--- /dev/null
+++ b/contrib/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -0,0 +1,63 @@
+//=======-------- BlockFrequencyInfo.cpp - Block Frequency Analysis -------=======//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Loops should be simplified before this analysis.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/InitializePasses.h"
+#include "llvm/Analysis/BlockFrequencyImpl.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/Passes.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+
+using namespace llvm;
+
+INITIALIZE_PASS_BEGIN(BlockFrequencyInfo, "block-freq", "Block Frequency Analysis",
+                      true, true)
+INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfo)
+INITIALIZE_PASS_END(BlockFrequencyInfo, "block-freq", "Block Frequency Analysis",
+                    true, true)
+
+char BlockFrequencyInfo::ID = 0;
+
+
+BlockFrequencyInfo::BlockFrequencyInfo() : FunctionPass(ID) {
+  initializeBlockFrequencyInfoPass(*PassRegistry::getPassRegistry());
+  BFI = new BlockFrequencyImpl<BasicBlock, Function, BranchProbabilityInfo>();
+}
+
+BlockFrequencyInfo::~BlockFrequencyInfo() {
+  delete BFI;
+}
+
+void BlockFrequencyInfo::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<BranchProbabilityInfo>();
+  AU.setPreservesAll();
+}
+
+bool BlockFrequencyInfo::runOnFunction(Function &F) {
+  BranchProbabilityInfo &BPI = getAnalysis<BranchProbabilityInfo>();
+  BFI->doFunction(&F, &BPI);
+  return false;
+}
+
+void BlockFrequencyInfo::print(raw_ostream &O, const Module *) const {
+  if (BFI) BFI->print(O);
+}
+
+/// getblockFreq - Return block frequency. Return 0 if we don't have the
+/// information. Please note that initial frequency is equal to 1024. It means
+/// that we should not rely on the value itself, but only on the comparison to
+/// the other block frequencies. We do this to avoid using of floating points.
+///
+BlockFrequency BlockFrequencyInfo::getBlockFreq(BasicBlock *BB) const {
+  return BFI->getBlockFreq(BB);
+}
diff --git a/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index e39cd221b5a7..bde3b76708fa 100644
--- a/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -11,7 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Constants.h"
 #include "llvm/Instructions.h"
+#include "llvm/LLVMContext.h"
+#include "llvm/Metadata.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Support/Debug.h"
@@ -33,7 +36,7 @@ namespace {
 // private methods are hidden in the .cpp file.
 class BranchProbabilityAnalysis {
 
-  typedef std::pair<BasicBlock *, BasicBlock *> Edge;
+  typedef std::pair<const BasicBlock *, const BasicBlock *> Edge;
 
   DenseMap<Edge, uint32_t> *Weights;
 
@@ -52,7 +55,7 @@ class BranchProbabilityAnalysis {
   //          V
   //         BB1<-+
   //          |   |
-  //          |   | (Weight = 128)
+  //          |   | (Weight = 124)
   //          V   |
   //         BB2--+
   //          |
@@ -60,12 +63,21 @@ class BranchProbabilityAnalysis {
   //          V
   //         BB3
   //
-  // Probability of the edge BB2->BB1 = 128 / (128 + 4) = 0.9696..
-  // Probability of the edge BB2->BB3 = 4 / (128 + 4) = 0.0303..
+  // Probability of the edge BB2->BB1 = 124 / (124 + 4) = 0.96875
+  // Probability of the edge BB2->BB3 = 4 / (124 + 4) = 0.03125
 
-  static const uint32_t LBH_TAKEN_WEIGHT = 128;
+  static const uint32_t LBH_TAKEN_WEIGHT = 124;
   static const uint32_t LBH_NONTAKEN_WEIGHT = 4;
 
+  static const uint32_t RH_TAKEN_WEIGHT = 24;
+  static const uint32_t RH_NONTAKEN_WEIGHT = 8;
+
+  static const uint32_t PH_TAKEN_WEIGHT = 20;
+  static const uint32_t PH_NONTAKEN_WEIGHT = 12;
+
+  static const uint32_t ZH_TAKEN_WEIGHT = 20;
+  static const uint32_t ZH_NONTAKEN_WEIGHT = 12;
+
   // Standard weight value. Used when none of the heuristics set weight for
   // the edge.
   static const uint32_t NORMAL_WEIGHT = 16;
@@ -100,29 +112,6 @@ class BranchProbabilityAnalysis {
     return false;
   }
 
-  // Multiply Edge Weight by two.
-  void incEdgeWeight(BasicBlock *Src, BasicBlock *Dst) {
-    uint32_t Weight = BP->getEdgeWeight(Src, Dst);
-    uint32_t MaxWeight = getMaxWeightFor(Src);
-
-    if (Weight * 2 > MaxWeight)
-      BP->setEdgeWeight(Src, Dst, MaxWeight);
-    else
-      BP->setEdgeWeight(Src, Dst, Weight * 2);
-  }
-
-  // Divide Edge Weight by two.
-  void decEdgeWeight(BasicBlock *Src, BasicBlock *Dst) {
-    uint32_t Weight = BP->getEdgeWeight(Src, Dst);
-
-    assert(Weight > 0);
-    if (Weight / 2 < MIN_WEIGHT)
-      BP->setEdgeWeight(Src, Dst, MIN_WEIGHT);
-    else
-      BP->setEdgeWeight(Src, Dst, Weight / 2);
-  }
-
-
   uint32_t getMaxWeightFor(BasicBlock *BB) const {
     return UINT32_MAX / BB->getTerminator()->getNumSuccessors();
   }
@@ -133,49 +122,119 @@ class BranchProbabilityAnalysis {
     : Weights(W), BP(BP), LI(LI) {
   }
 
+  // Metadata Weights
+  bool calcMetadataWeights(BasicBlock *BB);
+
   // Return Heuristics
-  void calcReturnHeuristics(BasicBlock *BB);
+  bool calcReturnHeuristics(BasicBlock *BB);
 
   // Pointer Heuristics
-  void calcPointerHeuristics(BasicBlock *BB);
+  bool calcPointerHeuristics(BasicBlock *BB);
 
   // Loop Branch Heuristics
-  void calcLoopBranchHeuristics(BasicBlock *BB);
+  bool calcLoopBranchHeuristics(BasicBlock *BB);
+
+  // Zero Heurestics
+  bool calcZeroHeuristics(BasicBlock *BB);
 
   bool runOnFunction(Function &F);
 };
 } // end anonymous namespace
 
+// Propagate existing explicit probabilities from either profile data or
+// 'expect' intrinsic processing.
+bool BranchProbabilityAnalysis::calcMetadataWeights(BasicBlock *BB) {
+  TerminatorInst *TI = BB->getTerminator();
+  if (TI->getNumSuccessors() == 1)
+    return false;
+  if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
+    return false;
+
+  MDNode *WeightsNode = TI->getMetadata(LLVMContext::MD_prof);
+  if (!WeightsNode)
+    return false;
+
+  // Ensure there are weights for all of the successors. Note that the first
+  // operand to the metadata node is a name, not a weight.
+  if (WeightsNode->getNumOperands() != TI->getNumSuccessors() + 1)
+    return false;
+
+  // Build up the final weights that will be used in a temporary buffer, but
+  // don't add them until all weihts are present. Each weight value is clamped
+  // to [1, getMaxWeightFor(BB)].
+  uint32_t WeightLimit = getMaxWeightFor(BB);
+  SmallVector<uint32_t, 2> Weights;
+  Weights.reserve(TI->getNumSuccessors());
+  for (unsigned i = 1, e = WeightsNode->getNumOperands(); i != e; ++i) {
+    ConstantInt *Weight = dyn_cast<ConstantInt>(WeightsNode->getOperand(i));
+    if (!Weight)
+      return false;
+    Weights.push_back(
+      std::max<uint32_t>(1, Weight->getLimitedValue(WeightLimit)));
+  }
+  assert(Weights.size() == TI->getNumSuccessors() && "Checked above");
+  for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
+    BP->setEdgeWeight(BB, TI->getSuccessor(i), Weights[i]);
+
+  return true;
+}
+
 // Calculate Edge Weights using "Return Heuristics". Predict a successor which
 // leads directly to Return Instruction will not be taken.
-void BranchProbabilityAnalysis::calcReturnHeuristics(BasicBlock *BB){
+bool BranchProbabilityAnalysis::calcReturnHeuristics(BasicBlock *BB){
   if (BB->getTerminator()->getNumSuccessors() == 1)
-    return;
+    return false;
+
+  SmallPtrSet<BasicBlock *, 4> ReturningEdges;
+  SmallPtrSet<BasicBlock *, 4> StayEdges;
 
   for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
     BasicBlock *Succ = *I;
-    if (isReturningBlock(Succ)) {
-      decEdgeWeight(BB, Succ);
+    if (isReturningBlock(Succ))
+      ReturningEdges.insert(Succ);
+    else
+      StayEdges.insert(Succ);
+  }
+
+  if (uint32_t numStayEdges = StayEdges.size()) {
+    uint32_t stayWeight = RH_TAKEN_WEIGHT / numStayEdges;
+    if (stayWeight < NORMAL_WEIGHT)
+      stayWeight = NORMAL_WEIGHT;
+
+    for (SmallPtrSet<BasicBlock *, 4>::iterator I = StayEdges.begin(),
+         E = StayEdges.end(); I != E; ++I)
+      BP->setEdgeWeight(BB, *I, stayWeight);
+  }
+
+  if (uint32_t numRetEdges = ReturningEdges.size()) {
+    uint32_t retWeight = RH_NONTAKEN_WEIGHT / numRetEdges;
+    if (retWeight < MIN_WEIGHT)
+      retWeight = MIN_WEIGHT;
+    for (SmallPtrSet<BasicBlock *, 4>::iterator I = ReturningEdges.begin(),
+         E = ReturningEdges.end(); I != E; ++I) {
+      BP->setEdgeWeight(BB, *I, retWeight);
     }
   }
+
+  return ReturningEdges.size() > 0;
 }
 
 // Calculate Edge Weights using "Pointer Heuristics". Predict a comparsion
 // between two pointer or pointer and NULL will fail.
-void BranchProbabilityAnalysis::calcPointerHeuristics(BasicBlock *BB) {
+bool BranchProbabilityAnalysis::calcPointerHeuristics(BasicBlock *BB) {
   BranchInst * BI = dyn_cast<BranchInst>(BB->getTerminator());
   if (!BI || !BI->isConditional())
-    return;
+    return false;
 
   Value *Cond = BI->getCondition();
   ICmpInst *CI = dyn_cast<ICmpInst>(Cond);
   if (!CI || !CI->isEquality())
-    return;
+    return false;
 
   Value *LHS = CI->getOperand(0);
 
   if (!LHS->getType()->isPointerTy())
-    return;
+    return false;
 
   assert(CI->getOperand(1)->getType()->isPointerTy());
 
@@ -190,29 +249,35 @@ void BranchProbabilityAnalysis::calcPointerHeuristics(BasicBlock *BB) {
   if (!isProb)
     std::swap(Taken, NonTaken);
 
-  incEdgeWeight(BB, Taken);
-  decEdgeWeight(BB, NonTaken);
+  BP->setEdgeWeight(BB, Taken, PH_TAKEN_WEIGHT);
+  BP->setEdgeWeight(BB, NonTaken, PH_NONTAKEN_WEIGHT);
+  return true;
 }
 
 // Calculate Edge Weights using "Loop Branch Heuristics". Predict backedges
 // as taken, exiting edges as not-taken.
-void BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) {
+bool BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) {
   uint32_t numSuccs = BB->getTerminator()->getNumSuccessors();
 
   Loop *L = LI->getLoopFor(BB);
   if (!L)
-    return;
+    return false;
 
-  SmallVector<BasicBlock *, 8> BackEdges;
-  SmallVector<BasicBlock *, 8> ExitingEdges;
+  SmallPtrSet<BasicBlock *, 8> BackEdges;
+  SmallPtrSet<BasicBlock *, 8> ExitingEdges;
+  SmallPtrSet<BasicBlock *, 8> InEdges; // Edges from header to the loop.
+
+  bool isHeader = BB == L->getHeader();
 
   for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
     BasicBlock *Succ = *I;
     Loop *SuccL = LI->getLoopFor(Succ);
     if (SuccL != L)
-      ExitingEdges.push_back(Succ);
+      ExitingEdges.insert(Succ);
     else if (Succ == L->getHeader())
-      BackEdges.push_back(Succ);
+      BackEdges.insert(Succ);
+    else if (isHeader)
+      InEdges.insert(Succ);
   }
 
   if (uint32_t numBackEdges = BackEdges.size()) {
@@ -220,39 +285,121 @@ void BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) {
     if (backWeight < NORMAL_WEIGHT)
       backWeight = NORMAL_WEIGHT;
 
-    for (SmallVector<BasicBlock *, 8>::iterator EI = BackEdges.begin(),
+    for (SmallPtrSet<BasicBlock *, 8>::iterator EI = BackEdges.begin(),
          EE = BackEdges.end(); EI != EE; ++EI) {
       BasicBlock *Back = *EI;
       BP->setEdgeWeight(BB, Back, backWeight);
     }
   }
 
+  if (uint32_t numInEdges = InEdges.size()) {
+    uint32_t inWeight = LBH_TAKEN_WEIGHT / numInEdges;
+    if (inWeight < NORMAL_WEIGHT)
+      inWeight = NORMAL_WEIGHT;
+
+    for (SmallPtrSet<BasicBlock *, 8>::iterator EI = InEdges.begin(),
+         EE = InEdges.end(); EI != EE; ++EI) {
+      BasicBlock *Back = *EI;
+      BP->setEdgeWeight(BB, Back, inWeight);
+    }
+  }
+
   uint32_t numExitingEdges = ExitingEdges.size();
   if (uint32_t numNonExitingEdges = numSuccs - numExitingEdges) {
     uint32_t exitWeight = LBH_NONTAKEN_WEIGHT / numNonExitingEdges;
     if (exitWeight < MIN_WEIGHT)
       exitWeight = MIN_WEIGHT;
 
-    for (SmallVector<BasicBlock *, 8>::iterator EI = ExitingEdges.begin(),
+    for (SmallPtrSet<BasicBlock *, 8>::iterator EI = ExitingEdges.begin(),
          EE = ExitingEdges.end(); EI != EE; ++EI) {
       BasicBlock *Exiting = *EI;
       BP->setEdgeWeight(BB, Exiting, exitWeight);
     }
   }
+
+  return true;
 }
 
+bool BranchProbabilityAnalysis::calcZeroHeuristics(BasicBlock *BB) {
+  BranchInst * BI = dyn_cast<BranchInst>(BB->getTerminator());
+  if (!BI || !BI->isConditional())
+    return false;
+
+  Value *Cond = BI->getCondition();
+  ICmpInst *CI = dyn_cast<ICmpInst>(Cond);
+  if (!CI)
+    return false;
+
+  Value *RHS = CI->getOperand(1);
+  ConstantInt *CV = dyn_cast<ConstantInt>(RHS);
+  if (!CV)
+    return false;
+
+  bool isProb;
+  if (CV->isZero()) {
+    switch (CI->getPredicate()) {
+    case CmpInst::ICMP_EQ:
+      // X == 0   ->  Unlikely
+      isProb = false;
+      break;
+    case CmpInst::ICMP_NE:
+      // X != 0   ->  Likely
+      isProb = true;
+      break;
+    case CmpInst::ICMP_SLT:
+      // X < 0   ->  Unlikely
+      isProb = false;
+      break;
+    case CmpInst::ICMP_SGT:
+      // X > 0   ->  Likely
+      isProb = true;
+      break;
+    default:
+      return false;
+    }
+  } else if (CV->isOne() && CI->getPredicate() == CmpInst::ICMP_SLT) {
+    // InstCombine canonicalizes X <= 0 into X < 1.
+    // X <= 0   ->  Unlikely
+    isProb = false;
+  } else if (CV->isAllOnesValue() && CI->getPredicate() == CmpInst::ICMP_SGT) {
+    // InstCombine canonicalizes X >= 0 into X > -1.
+    // X >= 0   ->  Likely
+    isProb = true;
+  } else {
+    return false;
+  }
+
+  BasicBlock *Taken = BI->getSuccessor(0);
+  BasicBlock *NonTaken = BI->getSuccessor(1);
+
+  if (!isProb)
+    std::swap(Taken, NonTaken);
+
+  BP->setEdgeWeight(BB, Taken, ZH_TAKEN_WEIGHT);
+  BP->setEdgeWeight(BB, NonTaken, ZH_NONTAKEN_WEIGHT);
+
+  return true;
+}
+
+
 bool BranchProbabilityAnalysis::runOnFunction(Function &F) {
 
   for (Function::iterator I = F.begin(), E = F.end(); I != E; ) {
     BasicBlock *BB = I++;
 
-    // Only LBH uses setEdgeWeight method.
-    calcLoopBranchHeuristics(BB);
+    if (calcMetadataWeights(BB))
+      continue;
 
-    // PH and RH use only incEdgeWeight and decEwdgeWeight methods to
-    // not efface LBH results.
-    calcPointerHeuristics(BB);
-    calcReturnHeuristics(BB);
+    if (calcLoopBranchHeuristics(BB))
+      continue;
+
+    if (calcReturnHeuristics(BB))
+      continue;
+
+    if (calcPointerHeuristics(BB))
+      continue;
+
+    calcZeroHeuristics(BB);
   }
 
   return false;
@@ -269,11 +416,11 @@ bool BranchProbabilityInfo::runOnFunction(Function &F) {
   return BPA.runOnFunction(F);
 }
 
-uint32_t BranchProbabilityInfo::getSumForBlock(BasicBlock *BB) const {
+uint32_t BranchProbabilityInfo::getSumForBlock(const BasicBlock *BB) const {
   uint32_t Sum = 0;
 
-  for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
-    BasicBlock *Succ = *I;
+  for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
+    const BasicBlock *Succ = *I;
     uint32_t Weight = getEdgeWeight(BB, Succ);
     uint32_t PrevSum = Sum;
 
@@ -284,7 +431,8 @@ uint32_t BranchProbabilityInfo::getSumForBlock(BasicBlock *BB) const {
   return Sum;
 }
 
-bool BranchProbabilityInfo::isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const {
+bool BranchProbabilityInfo::
+isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const {
   // Hot probability is at least 4/5 = 80%
   uint32_t Weight = getEdgeWeight(Src, Dst);
   uint32_t Sum = getSumForBlock(Src);
@@ -321,8 +469,8 @@ BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const {
 }
 
 // Return edge's weight. If can't find it, return DEFAULT_WEIGHT value.
-uint32_t
-BranchProbabilityInfo::getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const {
+uint32_t BranchProbabilityInfo::
+getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
   Edge E(Src, Dst);
   DenseMap<Edge, uint32_t>::const_iterator I = Weights.find(E);
 
@@ -332,8 +480,8 @@ BranchProbabilityInfo::getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const {
   return DEFAULT_WEIGHT;
 }
 
-void BranchProbabilityInfo::setEdgeWeight(BasicBlock *Src, BasicBlock *Dst,
-                                     uint32_t Weight) {
+void BranchProbabilityInfo::
+setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst, uint32_t Weight) {
   Weights[std::make_pair(Src, Dst)] = Weight;
   DEBUG(dbgs() << "set edge " << Src->getNameStr() << " -> "
                << Dst->getNameStr() << " weight to " << Weight
@@ -342,7 +490,7 @@ void BranchProbabilityInfo::setEdgeWeight(BasicBlock *Src, BasicBlock *Dst,
 
 
 BranchProbability BranchProbabilityInfo::
-getEdgeProbability(BasicBlock *Src, BasicBlock *Dst) const {
+getEdgeProbability(const BasicBlock *Src, const BasicBlock *Dst) const {
 
   uint32_t N = getEdgeWeight(Src, Dst);
   uint32_t D = getSumForBlock(Src);
diff --git a/contrib/llvm/lib/Analysis/ConstantFolding.cpp b/contrib/llvm/lib/Analysis/ConstantFolding.cpp
index 7fca17eb69f6..df79849c3cf4 100644
--- a/contrib/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/contrib/llvm/lib/Analysis/ConstantFolding.cpp
@@ -43,11 +43,16 @@ using namespace llvm;
 /// FoldBitCast - Constant fold bitcast, symbolically evaluating it with 
 /// TargetData.  This always returns a non-null constant, but it may be a
 /// ConstantExpr if unfoldable.
-static Constant *FoldBitCast(Constant *C, const Type *DestTy,
+static Constant *FoldBitCast(Constant *C, Type *DestTy,
                              const TargetData &TD) {
-  
-  // This only handles casts to vectors currently.
-  const VectorType *DestVTy = dyn_cast<VectorType>(DestTy);
+  // Catch the obvious splat cases.
+  if (C->isNullValue() && !DestTy->isX86_MMXTy())
+    return Constant::getNullValue(DestTy);
+  if (C->isAllOnesValue() && !DestTy->isX86_MMXTy())
+    return Constant::getAllOnesValue(DestTy);
+
+  // The code below only handles casts to vectors currently.
+  VectorType *DestVTy = dyn_cast<VectorType>(DestTy);
   if (DestVTy == 0)
     return ConstantExpr::getBitCast(C, DestTy);
   
@@ -69,8 +74,8 @@ static Constant *FoldBitCast(Constant *C, const Type *DestTy,
   if (NumDstElt == NumSrcElt)
     return ConstantExpr::getBitCast(C, DestTy);
   
-  const Type *SrcEltTy = CV->getType()->getElementType();
-  const Type *DstEltTy = DestVTy->getElementType();
+  Type *SrcEltTy = CV->getType()->getElementType();
+  Type *DstEltTy = DestVTy->getElementType();
   
   // Otherwise, we're changing the number of elements in a vector, which 
   // requires endianness information to do the right thing.  For example,
@@ -85,7 +90,7 @@ static Constant *FoldBitCast(Constant *C, const Type *DestTy,
   if (DstEltTy->isFloatingPointTy()) {
     // Fold to an vector of integers with same size as our FP type.
     unsigned FPWidth = DstEltTy->getPrimitiveSizeInBits();
-    const Type *DestIVTy =
+    Type *DestIVTy =
       VectorType::get(IntegerType::get(C->getContext(), FPWidth), NumDstElt);
     // Recursively handle this integer conversion, if possible.
     C = FoldBitCast(C, DestIVTy, TD);
@@ -99,7 +104,7 @@ static Constant *FoldBitCast(Constant *C, const Type *DestTy,
   // it to integer first.
   if (SrcEltTy->isFloatingPointTy()) {
     unsigned FPWidth = SrcEltTy->getPrimitiveSizeInBits();
-    const Type *SrcIVTy =
+    Type *SrcIVTy =
       VectorType::get(IntegerType::get(C->getContext(), FPWidth), NumSrcElt);
     // Ask VMCore to do the conversion now that #elts line up.
     C = ConstantExpr::getBitCast(C, SrcIVTy);
@@ -212,11 +217,11 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
       if (!CI) return false;  // Index isn't a simple constant?
       if (CI->isZero()) continue;  // Not adding anything.
       
-      if (const StructType *ST = dyn_cast<StructType>(*GTI)) {
+      if (StructType *ST = dyn_cast<StructType>(*GTI)) {
         // N = N + Offset
         Offset += TD.getStructLayout(ST)->getElementOffset(CI->getZExtValue());
       } else {
-        const SequentialType *SQT = cast<SequentialType>(*GTI);
+        SequentialType *SQT = cast<SequentialType>(*GTI);
         Offset += TD.getTypeAllocSize(SQT->getElementType())*CI->getSExtValue();
       }
     }
@@ -354,8 +359,8 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset,
 
 static Constant *FoldReinterpretLoadFromConstPtr(Constant *C,
                                                  const TargetData &TD) {
-  const Type *LoadTy = cast<PointerType>(C->getType())->getElementType();
-  const IntegerType *IntType = dyn_cast<IntegerType>(LoadTy);
+  Type *LoadTy = cast<PointerType>(C->getType())->getElementType();
+  IntegerType *IntType = dyn_cast<IntegerType>(LoadTy);
   
   // If this isn't an integer load we can't fold it directly.
   if (!IntType) {
@@ -363,7 +368,7 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C,
     // and then bitcast the result.  This can be useful for union cases.  Note
     // that address spaces don't matter here since we're not going to result in
     // an actual new load.
-    const Type *MapTy;
+    Type *MapTy;
     if (LoadTy->isFloatTy())
       MapTy = Type::getInt32PtrTy(C->getContext());
     else if (LoadTy->isDoubleTy())
@@ -443,7 +448,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
   std::string Str;
   if (TD && GetConstantStringInfo(CE, Str) && !Str.empty()) {
     unsigned StrLen = Str.length();
-    const Type *Ty = cast<PointerType>(CE->getType())->getElementType();
+    Type *Ty = cast<PointerType>(CE->getType())->getElementType();
     unsigned NumBits = Ty->getPrimitiveSizeInBits();
     // Replace load with immediate integer if the result is an integer or fp
     // value.
@@ -478,7 +483,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
   if (GlobalVariable *GV =
         dyn_cast<GlobalVariable>(GetUnderlyingObject(CE, TD))) {
     if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
-      const Type *ResTy = cast<PointerType>(C->getType())->getElementType();
+      Type *ResTy = cast<PointerType>(C->getType())->getElementType();
       if (GV->getInitializer()->isNullValue())
         return Constant::getNullValue(ResTy);
       if (isa<UndefValue>(GV->getInitializer()))
@@ -536,19 +541,18 @@ static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0,
 /// CastGEPIndices - If array indices are not pointer-sized integers,
 /// explicitly cast them so that they aren't implicitly casted by the
 /// getelementptr.
-static Constant *CastGEPIndices(Constant *const *Ops, unsigned NumOps,
-                                const Type *ResultTy,
+static Constant *CastGEPIndices(ArrayRef<Constant *> Ops,
+                                Type *ResultTy,
                                 const TargetData *TD) {
   if (!TD) return 0;
-  const Type *IntPtrTy = TD->getIntPtrType(ResultTy->getContext());
+  Type *IntPtrTy = TD->getIntPtrType(ResultTy->getContext());
 
   bool Any = false;
   SmallVector<Constant*, 32> NewIdxs;
-  for (unsigned i = 1; i != NumOps; ++i) {
+  for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
     if ((i == 1 ||
          !isa<StructType>(GetElementPtrInst::getIndexedType(Ops[0]->getType(),
-                                        reinterpret_cast<Value *const *>(Ops+1),
-                                                            i-1))) &&
+                                                        Ops.slice(1, i-1)))) &&
         Ops[i]->getType() != IntPtrTy) {
       Any = true;
       NewIdxs.push_back(ConstantExpr::getCast(CastInst::getCastOpcode(Ops[i],
@@ -562,7 +566,7 @@ static Constant *CastGEPIndices(Constant *const *Ops, unsigned NumOps,
   if (!Any) return 0;
 
   Constant *C =
-    ConstantExpr::getGetElementPtr(Ops[0], &NewIdxs[0], NewIdxs.size());
+    ConstantExpr::getGetElementPtr(Ops[0], NewIdxs);
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
     if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
       C = Folded;
@@ -571,23 +575,23 @@ static Constant *CastGEPIndices(Constant *const *Ops, unsigned NumOps,
 
 /// SymbolicallyEvaluateGEP - If we can symbolically evaluate the specified GEP
 /// constant expression, do so.
-static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
-                                         const Type *ResultTy,
+static Constant *SymbolicallyEvaluateGEP(ArrayRef<Constant *> Ops,
+                                         Type *ResultTy,
                                          const TargetData *TD) {
   Constant *Ptr = Ops[0];
   if (!TD || !cast<PointerType>(Ptr->getType())->getElementType()->isSized())
     return 0;
   
-  const Type *IntPtrTy = TD->getIntPtrType(Ptr->getContext());
+  Type *IntPtrTy = TD->getIntPtrType(Ptr->getContext());
 
   // If this is a constant expr gep that is effectively computing an
   // "offsetof", fold it into 'cast int Size to T*' instead of 'gep 0, 0, 12'
-  for (unsigned i = 1; i != NumOps; ++i)
+  for (unsigned i = 1, e = Ops.size(); i != e; ++i)
     if (!isa<ConstantInt>(Ops[i])) {
       
       // If this is "gep i8* Ptr, (sub 0, V)", fold this as:
       // "inttoptr (sub (ptrtoint Ptr), V)"
-      if (NumOps == 2 &&
+      if (Ops.size() == 2 &&
           cast<PointerType>(ResultTy)->getElementType()->isIntegerTy(8)) {
         ConstantExpr *CE = dyn_cast<ConstantExpr>(Ops[1]);
         assert((CE == 0 || CE->getType() == IntPtrTy) &&
@@ -606,9 +610,10 @@ static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
     }
   
   unsigned BitWidth = TD->getTypeSizeInBits(IntPtrTy);
-  APInt Offset = APInt(BitWidth,
-                       TD->getIndexedOffset(Ptr->getType(),
-                                            (Value**)Ops+1, NumOps-1));
+  APInt Offset =
+    APInt(BitWidth, TD->getIndexedOffset(Ptr->getType(),
+                                         makeArrayRef((Value **)Ops.data() + 1,
+                                                      Ops.size() - 1)));
   Ptr = cast<Constant>(Ptr->stripPointerCasts());
 
   // If this is a GEP of a GEP, fold it all into a single GEP.
@@ -627,9 +632,7 @@ static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
 
     Ptr = cast<Constant>(GEP->getOperand(0));
     Offset += APInt(BitWidth,
-                    TD->getIndexedOffset(Ptr->getType(),
-                                         (Value**)NestedOps.data(),
-                                         NestedOps.size()));
+                    TD->getIndexedOffset(Ptr->getType(), NestedOps));
     Ptr = cast<Constant>(Ptr->stripPointerCasts());
   }
 
@@ -649,10 +652,10 @@ static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
   // we eliminate over-indexing of the notional static type array bounds.
   // This makes it easy to determine if the getelementptr is "inbounds".
   // Also, this helps GlobalOpt do SROA on GlobalVariables.
-  const Type *Ty = Ptr->getType();
+  Type *Ty = Ptr->getType();
   SmallVector<Constant*, 32> NewIdxs;
   do {
-    if (const SequentialType *ATy = dyn_cast<SequentialType>(Ty)) {
+    if (SequentialType *ATy = dyn_cast<SequentialType>(Ty)) {
       if (ATy->isPointerTy()) {
         // The only pointer indexing we'll do is on the first index of the GEP.
         if (!NewIdxs.empty())
@@ -665,7 +668,7 @@ static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
         
       // Determine which element of the array the offset points into.
       APInt ElemSize(BitWidth, TD->getTypeAllocSize(ATy->getElementType()));
-      const IntegerType *IntPtrTy = TD->getIntPtrType(Ty->getContext());
+      IntegerType *IntPtrTy = TD->getIntPtrType(Ty->getContext());
       if (ElemSize == 0)
         // The element size is 0. This may be [0 x Ty]*, so just use a zero
         // index for this level and proceed to the next level to see if it can
@@ -679,7 +682,7 @@ static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
         NewIdxs.push_back(ConstantInt::get(IntPtrTy, NewIdx));
       }
       Ty = ATy->getElementType();
-    } else if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+    } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
       // Determine which field of the struct the offset points into. The
       // getZExtValue is at least as safe as the StructLayout API because we
       // know the offset is within the struct at this point.
@@ -703,7 +706,7 @@ static Constant *SymbolicallyEvaluateGEP(Constant *const *Ops, unsigned NumOps,
 
   // Create a GEP.
   Constant *C =
-    ConstantExpr::getGetElementPtr(Ptr, &NewIdxs[0], NewIdxs.size());
+    ConstantExpr::getGetElementPtr(Ptr, NewIdxs);
   assert(cast<PointerType>(C->getType())->getElementType() == Ty &&
          "Computed GetElementPtr has unexpected type!");
 
@@ -778,8 +781,7 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I, const TargetData *TD) {
                                     cast<Constant>(EVI->getAggregateOperand()),
                                     EVI->getIndices());
 
-  return ConstantFoldInstOperands(I->getOpcode(), I->getType(),
-                                  Ops.data(), Ops.size(), TD);
+  return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Ops, TD);
 }
 
 /// ConstantFoldConstantExpression - Attempt to fold the constant expression
@@ -800,8 +802,7 @@ Constant *llvm::ConstantFoldConstantExpression(const ConstantExpr *CE,
   if (CE->isCompare())
     return ConstantFoldCompareInstOperands(CE->getPredicate(), Ops[0], Ops[1],
                                            TD);
-  return ConstantFoldInstOperands(CE->getOpcode(), CE->getType(),
-                                  Ops.data(), Ops.size(), TD);
+  return ConstantFoldInstOperands(CE->getOpcode(), CE->getType(), Ops, TD);
 }
 
 /// ConstantFoldInstOperands - Attempt to constant fold an instruction with the
@@ -814,8 +815,8 @@ Constant *llvm::ConstantFoldConstantExpression(const ConstantExpr *CE,
 /// information, due to only being passed an opcode and operands. Constant
 /// folding using this function strips this information.
 ///
-Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, const Type *DestTy, 
-                                         Constant* const* Ops, unsigned NumOps,
+Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, 
+                                         ArrayRef<Constant *> Ops,
                                          const TargetData *TD) {
   // Handle easy binops first.
   if (Instruction::isBinaryOp(Opcode)) {
@@ -831,9 +832,9 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, const Type *DestTy,
   case Instruction::ICmp:
   case Instruction::FCmp: assert(0 && "Invalid for compares");
   case Instruction::Call:
-    if (Function *F = dyn_cast<Function>(Ops[NumOps - 1]))
+    if (Function *F = dyn_cast<Function>(Ops.back()))
       if (canConstantFoldCallTo(F))
-        return ConstantFoldCall(F, Ops, NumOps - 1);
+        return ConstantFoldCall(F, Ops.slice(0, Ops.size() - 1));
     return 0;
   case Instruction::PtrToInt:
     // If the input is a inttoptr, eliminate the pair.  This requires knowing
@@ -887,12 +888,12 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, const Type *DestTy,
   case Instruction::ShuffleVector:
     return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
   case Instruction::GetElementPtr:
-    if (Constant *C = CastGEPIndices(Ops, NumOps, DestTy, TD))
+    if (Constant *C = CastGEPIndices(Ops, DestTy, TD))
       return C;
-    if (Constant *C = SymbolicallyEvaluateGEP(Ops, NumOps, DestTy, TD))
+    if (Constant *C = SymbolicallyEvaluateGEP(Ops, DestTy, TD))
       return C;
     
-    return ConstantExpr::getGetElementPtr(Ops[0], Ops+1, NumOps-1);
+    return ConstantExpr::getGetElementPtr(Ops[0], Ops.slice(1));
   }
 }
 
@@ -912,7 +913,7 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
   // around to know if bit truncation is happening.
   if (ConstantExpr *CE0 = dyn_cast<ConstantExpr>(Ops0)) {
     if (TD && Ops1->isNullValue()) {
-      const Type *IntPtrTy = TD->getIntPtrType(CE0->getContext());
+      Type *IntPtrTy = TD->getIntPtrType(CE0->getContext());
       if (CE0->getOpcode() == Instruction::IntToPtr) {
         // Convert the integer value to the right size to ensure we get the
         // proper extension or truncation.
@@ -934,7 +935,7 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
     
     if (ConstantExpr *CE1 = dyn_cast<ConstantExpr>(Ops1)) {
       if (TD && CE0->getOpcode() == CE1->getOpcode()) {
-        const Type *IntPtrTy = TD->getIntPtrType(CE0->getContext());
+        Type *IntPtrTy = TD->getIntPtrType(CE0->getContext());
 
         if (CE0->getOpcode() == Instruction::IntToPtr) {
           // Convert the integer value to the right size to ensure we get the
@@ -967,7 +968,7 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
       unsigned OpC = 
         Predicate == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
       Constant *Ops[] = { LHS, RHS };
-      return ConstantFoldInstOperands(OpC, LHS->getType(), Ops, 2, TD);
+      return ConstantFoldInstOperands(OpC, LHS->getType(), Ops, TD);
     }
   }
   
@@ -987,7 +988,7 @@ Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C,
   // addressing...
   gep_type_iterator I = gep_type_begin(CE), E = gep_type_end(CE);
   for (++I; I != E; ++I)
-    if (const StructType *STy = dyn_cast<StructType>(*I)) {
+    if (StructType *STy = dyn_cast<StructType>(*I)) {
       ConstantInt *CU = cast<ConstantInt>(I.getOperand());
       assert(CU->getZExtValue() < STy->getNumElements() &&
              "Struct index out of range!");
@@ -1002,7 +1003,7 @@ Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C,
         return 0;
       }
     } else if (ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand())) {
-      if (const ArrayType *ATy = dyn_cast<ArrayType>(*I)) {
+      if (ArrayType *ATy = dyn_cast<ArrayType>(*I)) {
         if (CI->getZExtValue() >= ATy->getNumElements())
          return 0;
         if (ConstantArray *CA = dyn_cast<ConstantArray>(C))
@@ -1013,7 +1014,7 @@ Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C,
           C = UndefValue::get(ATy->getElementType());
         else
           return 0;
-      } else if (const VectorType *VTy = dyn_cast<VectorType>(*I)) {
+      } else if (VectorType *VTy = dyn_cast<VectorType>(*I)) {
         if (CI->getZExtValue() >= VTy->getNumElements())
           return 0;
         if (ConstantVector *CP = dyn_cast<ConstantVector>(C))
@@ -1101,7 +1102,7 @@ llvm::canConstantFoldCallTo(const Function *F) {
 }
 
 static Constant *ConstantFoldFP(double (*NativeFP)(double), double V, 
-                                const Type *Ty) {
+                                Type *Ty) {
   sys::llvm_fenv_clearexcept();
   V = NativeFP(V);
   if (sys::llvm_fenv_testexcept()) {
@@ -1118,7 +1119,7 @@ static Constant *ConstantFoldFP(double (*NativeFP)(double), double V,
 }
 
 static Constant *ConstantFoldBinaryFP(double (*NativeFP)(double, double),
-                                      double V, double W, const Type *Ty) {
+                                      double V, double W, Type *Ty) {
   sys::llvm_fenv_clearexcept();
   V = NativeFP(V, W);
   if (sys::llvm_fenv_testexcept()) {
@@ -1143,7 +1144,7 @@ static Constant *ConstantFoldBinaryFP(double (*NativeFP)(double, double),
 /// performed, otherwise returns the Constant value resulting from the
 /// conversion.
 static Constant *ConstantFoldConvertToInt(ConstantFP *Op, bool roundTowardZero,
-                                          const Type *Ty) {
+                                          Type *Ty) {
   assert(Op && "Called with NULL operand");
   APFloat Val(Op->getValueAPF());
 
@@ -1167,13 +1168,12 @@ static Constant *ConstantFoldConvertToInt(ConstantFP *Op, bool roundTowardZero,
 /// ConstantFoldCall - Attempt to constant fold a call to the specified function
 /// with the specified arguments, returning null if unsuccessful.
 Constant *
-llvm::ConstantFoldCall(Function *F, 
-                       Constant *const *Operands, unsigned NumOperands) {
+llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands) {
   if (!F->hasName()) return 0;
   StringRef Name = F->getName();
 
-  const Type *Ty = F->getReturnType();
-  if (NumOperands == 1) {
+  Type *Ty = F->getReturnType();
+  if (Operands.size() == 1) {
     if (ConstantFP *Op = dyn_cast<ConstantFP>(Operands[0])) {
       if (F->getIntrinsicID() == Intrinsic::convert_to_fp16) {
         APFloat Val(Op->getValueAPF());
@@ -1327,7 +1327,7 @@ llvm::ConstantFoldCall(Function *F,
     return 0;
   }
 
-  if (NumOperands == 2) {
+  if (Operands.size() == 2) {
     if (ConstantFP *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
       if (!Ty->isFloatTy() && !Ty->isDoubleTy())
         return 0;
diff --git a/contrib/llvm/lib/Analysis/DIBuilder.cpp b/contrib/llvm/lib/Analysis/DIBuilder.cpp
index ac5eeeb4706a..bfa429d54120 100644
--- a/contrib/llvm/lib/Analysis/DIBuilder.cpp
+++ b/contrib/llvm/lib/Analysis/DIBuilder.cpp
@@ -29,14 +29,74 @@ static Constant *GetTagConstant(LLVMContext &VMContext, unsigned Tag) {
 }
 
 DIBuilder::DIBuilder(Module &m)
-  : M(m), VMContext(M.getContext()), TheCU(0), DeclareFn(0), ValueFn(0) {}
+  : M(m), VMContext(M.getContext()), TheCU(0), TempEnumTypes(0),
+    TempRetainTypes(0), TempSubprograms(0), TempGVs(0), DeclareFn(0),
+    ValueFn(0)
+{}
+
+/// finalize - Construct any deferred debug info descriptors.
+void DIBuilder::finalize() {
+  DIArray Enums = getOrCreateArray(AllEnumTypes);
+  DIType(TempEnumTypes).replaceAllUsesWith(Enums);
+
+  DIArray RetainTypes = getOrCreateArray(AllRetainTypes);
+  DIType(TempRetainTypes).replaceAllUsesWith(RetainTypes);
+
+  DIArray SPs = getOrCreateArray(AllSubprograms);
+  DIType(TempSubprograms).replaceAllUsesWith(SPs);
+  for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) {
+    DISubprogram SP(SPs.getElement(i));
+    if (NamedMDNode *NMD = getFnSpecificMDNode(M, SP)) {
+      SmallVector<Value *, 4> Variables;
+      for (unsigned ii = 0, ee = NMD->getNumOperands(); ii != ee; ++ii)
+        Variables.push_back(NMD->getOperand(ii));
+      if (MDNode *Temp = SP.getVariablesNodes()) {
+        DIArray AV = getOrCreateArray(Variables);
+        DIType(Temp).replaceAllUsesWith(AV);
+      }
+      NMD->eraseFromParent();
+    }
+  }
+
+  DIArray GVs = getOrCreateArray(AllGVs);
+  DIType(TempGVs).replaceAllUsesWith(GVs);
+}
+
+/// getNonCompileUnitScope - If N is compile unit return NULL otherwise return
+/// N.
+static MDNode *getNonCompileUnitScope(MDNode *N) {
+  if (DIDescriptor(N).isCompileUnit())
+    return NULL;
+  return N;
+}
 
 /// createCompileUnit - A CompileUnit provides an anchor for all debugging
 /// information generated during this instance of compilation.
-void DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename, 
-                                  StringRef Directory, StringRef Producer, 
-                                  bool isOptimized, StringRef Flags, 
+void DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename,
+                                  StringRef Directory, StringRef Producer,
+                                  bool isOptimized, StringRef Flags,
                                   unsigned RunTimeVer) {
+  assert (Lang <= dwarf::DW_LANG_D && Lang >= dwarf::DW_LANG_C89
+	  && "Invalid Language tag");
+  assert (!Filename.empty() 
+	  && "Unable to create compile unit without filename");
+  Value *TElts[] = { GetTagConstant(VMContext, DW_TAG_base_type) };
+  TempEnumTypes = MDNode::getTemporary(VMContext, TElts);
+  Value *THElts[] = { TempEnumTypes };
+  MDNode *EnumHolder = MDNode::get(VMContext, THElts);
+
+  TempRetainTypes = MDNode::getTemporary(VMContext, TElts);
+  Value *TRElts[] = { TempRetainTypes };
+  MDNode *RetainHolder = MDNode::get(VMContext, TRElts);
+
+  TempSubprograms = MDNode::getTemporary(VMContext, TElts);
+  Value *TSElts[] = { TempSubprograms };
+  MDNode *SPHolder = MDNode::get(VMContext, TSElts);
+
+  TempGVs = MDNode::getTemporary(VMContext, TElts);
+  Value *TVElts[] = { TempGVs };
+  MDNode *GVHolder = MDNode::get(VMContext, TVElts);
+
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_compile_unit),
     llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
@@ -48,7 +108,11 @@ void DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename,
     ConstantInt::get(Type::getInt1Ty(VMContext), true), // isMain
     ConstantInt::get(Type::getInt1Ty(VMContext), isOptimized),
     MDString::get(VMContext, Flags),
-    ConstantInt::get(Type::getInt32Ty(VMContext), RunTimeVer)
+    ConstantInt::get(Type::getInt32Ty(VMContext), RunTimeVer),
+    EnumHolder,
+    RetainHolder,
+    SPHolder,
+    GVHolder
   };
   TheCU = DICompileUnit(MDNode::get(VMContext, Elts));
 
@@ -61,17 +125,19 @@ void DIBuilder::createCompileUnit(unsigned Lang, StringRef Filename,
 /// for a file.
 DIFile DIBuilder::createFile(StringRef Filename, StringRef Directory) {
   assert(TheCU && "Unable to create DW_TAG_file_type without CompileUnit");
+  assert(!Filename.empty() && "Unable to create file without name");
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_file_type),
     MDString::get(VMContext, Filename),
     MDString::get(VMContext, Directory),
-    TheCU
+    NULL // TheCU
   };
   return DIFile(MDNode::get(VMContext, Elts));
 }
 
 /// createEnumerator - Create a single enumerator value.
 DIEnumerator DIBuilder::createEnumerator(StringRef Name, uint64_t Val) {
+  assert(!Name.empty() && "Unable to create enumerator without name");
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_enumerator),
     MDString::get(VMContext, Name),
@@ -80,16 +146,37 @@ DIEnumerator DIBuilder::createEnumerator(StringRef Name, uint64_t Val) {
   return DIEnumerator(MDNode::get(VMContext, Elts));
 }
 
-/// createBasicType - Create debugging information entry for a basic 
+/// createNullPtrType - Create C++0x nullptr type.
+DIType DIBuilder::createNullPtrType(StringRef Name) {
+  assert(!Name.empty() && "Unable to create type without name");
+  // nullptr is encoded in DIBasicType format. Line number, filename,
+  // ,size, alignment, offset and flags are always empty here.
+  Value *Elts[] = {
+    GetTagConstant(VMContext, dwarf::DW_TAG_unspecified_type),
+    NULL, //TheCU,
+    MDString::get(VMContext, Name),
+    NULL, // Filename
+    ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Line
+    ConstantInt::get(Type::getInt64Ty(VMContext), 0), // Size
+    ConstantInt::get(Type::getInt64Ty(VMContext), 0), // Align
+    ConstantInt::get(Type::getInt64Ty(VMContext), 0), // Offset
+    ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Flags;
+    ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Encoding
+  };
+  return DIType(MDNode::get(VMContext, Elts));
+}
+
+/// createBasicType - Create debugging information entry for a basic
 /// type, e.g 'char'.
-DIType DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits, 
+DIType DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits,
                                   uint64_t AlignInBits,
                                   unsigned Encoding) {
+  assert(!Name.empty() && "Unable to create type without name");
   // Basic types are encoded in DIBasicType format. Line number, filename,
   // offset and flags are always empty here.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_base_type),
-    TheCU,
+    NULL, //TheCU,
     MDString::get(VMContext, Name),
     NULL, // Filename
     ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Line
@@ -108,7 +195,7 @@ DIType DIBuilder::createQualifiedType(unsigned Tag, DIType FromTy) {
   // Qualified types are encoded in DIDerivedType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, Tag),
-    TheCU,
+    NULL, //TheCU,
     MDString::get(VMContext, StringRef()), // Empty name.
     NULL, // Filename
     ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Line
@@ -127,7 +214,7 @@ DIType DIBuilder::createPointerType(DIType PointeeTy, uint64_t SizeInBits,
   // Pointer types are encoded in DIDerivedType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_pointer_type),
-    TheCU,
+    NULL, //TheCU,
     MDString::get(VMContext, Name),
     NULL, // Filename
     ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Line
@@ -142,10 +229,11 @@ DIType DIBuilder::createPointerType(DIType PointeeTy, uint64_t SizeInBits,
 
 /// createReferenceType - Create debugging information entry for a reference.
 DIType DIBuilder::createReferenceType(DIType RTy) {
+  assert(RTy.Verify() && "Unable to create reference type");
   // References are encoded in DIDerivedType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_reference_type),
-    TheCU,
+    NULL, // TheCU,
     NULL, // Name
     NULL, // Filename
     ConstantInt::get(Type::getInt32Ty(VMContext), 0), // Line
@@ -165,7 +253,7 @@ DIType DIBuilder::createTypedef(DIType Ty, StringRef Name, DIFile File,
   assert(Ty.Verify() && "Invalid typedef type!");
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_typedef),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNo),
@@ -199,9 +287,10 @@ DIType DIBuilder::createFriend(DIType Ty, DIType FriendTy) {
 }
 
 /// createInheritance - Create debugging information entry to establish
-/// inheritnace relationship between two types.
-DIType DIBuilder::createInheritance(DIType Ty, DIType BaseTy, 
+/// inheritance relationship between two types.
+DIType DIBuilder::createInheritance(DIType Ty, DIType BaseTy,
                                     uint64_t BaseOffset, unsigned Flags) {
+  assert(Ty.Verify() && "Unable to create inheritance");
   // TAG_inheritance is encoded in DIDerivedType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_inheritance),
@@ -219,15 +308,15 @@ DIType DIBuilder::createInheritance(DIType Ty, DIType BaseTy,
 }
 
 /// createMemberType - Create debugging information entry for a member.
-DIType DIBuilder::createMemberType(DIDescriptor Scope, StringRef Name, 
-                                   DIFile File, unsigned LineNumber, 
+DIType DIBuilder::createMemberType(DIDescriptor Scope, StringRef Name,
+                                   DIFile File, unsigned LineNumber,
                                    uint64_t SizeInBits, uint64_t AlignInBits,
-                                   uint64_t OffsetInBits, unsigned Flags, 
+                                   uint64_t OffsetInBits, unsigned Flags,
                                    DIType Ty) {
   // TAG_member is encoded in DIDerivedType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_member),
-    Scope,
+    getNonCompileUnitScope(Scope),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNumber),
@@ -242,17 +331,17 @@ DIType DIBuilder::createMemberType(DIDescriptor Scope, StringRef Name,
 
 /// createObjCIVar - Create debugging information entry for Objective-C
 /// instance variable.
-DIType DIBuilder::createObjCIVar(StringRef Name, 
-                                 DIFile File, unsigned LineNumber, 
+DIType DIBuilder::createObjCIVar(StringRef Name,
+                                 DIFile File, unsigned LineNumber,
                                  uint64_t SizeInBits, uint64_t AlignInBits,
-                                 uint64_t OffsetInBits, unsigned Flags, 
+                                 uint64_t OffsetInBits, unsigned Flags,
                                  DIType Ty, StringRef PropertyName,
                                  StringRef GetterName, StringRef SetterName,
                                  unsigned PropertyAttributes) {
   // TAG_member is encoded in DIDerivedType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_member),
-    File, // Or TheCU ? Ty ?
+    getNonCompileUnitScope(File),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNumber),
@@ -270,8 +359,8 @@ DIType DIBuilder::createObjCIVar(StringRef Name,
 }
 
 /// createClassType - Create debugging information entry for a class.
-DIType DIBuilder::createClassType(DIDescriptor Context, StringRef Name, 
-                                  DIFile File, unsigned LineNumber, 
+DIType DIBuilder::createClassType(DIDescriptor Context, StringRef Name,
+                                  DIFile File, unsigned LineNumber,
                                   uint64_t SizeInBits, uint64_t AlignInBits,
                                   uint64_t OffsetInBits, unsigned Flags,
                                   DIType DerivedFrom, DIArray Elements,
@@ -279,7 +368,7 @@ DIType DIBuilder::createClassType(DIDescriptor Context, StringRef Name,
  // TAG_class_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_class_type),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNumber),
@@ -298,13 +387,13 @@ DIType DIBuilder::createClassType(DIDescriptor Context, StringRef Name,
 
 /// createTemplateTypeParameter - Create debugging information for template
 /// type parameter.
-DITemplateTypeParameter 
+DITemplateTypeParameter
 DIBuilder::createTemplateTypeParameter(DIDescriptor Context, StringRef Name,
                                        DIType Ty, MDNode *File, unsigned LineNo,
                                        unsigned ColumnNo) {
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_template_type_parameter),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     Ty,
     File,
@@ -316,14 +405,14 @@ DIBuilder::createTemplateTypeParameter(DIDescriptor Context, StringRef Name,
 
 /// createTemplateValueParameter - Create debugging information for template
 /// value parameter.
-DITemplateValueParameter 
+DITemplateValueParameter
 DIBuilder::createTemplateValueParameter(DIDescriptor Context, StringRef Name,
                                         DIType Ty, uint64_t Val,
                                         MDNode *File, unsigned LineNo,
                                         unsigned ColumnNo) {
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_template_value_parameter),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     Ty,
     ConstantInt::get(Type::getInt64Ty(VMContext), Val),
@@ -335,15 +424,15 @@ DIBuilder::createTemplateValueParameter(DIDescriptor Context, StringRef Name,
 }
 
 /// createStructType - Create debugging information entry for a struct.
-DIType DIBuilder::createStructType(DIDescriptor Context, StringRef Name, 
-                                   DIFile File, unsigned LineNumber, 
+DIType DIBuilder::createStructType(DIDescriptor Context, StringRef Name,
+                                   DIFile File, unsigned LineNumber,
                                    uint64_t SizeInBits, uint64_t AlignInBits,
-                                   unsigned Flags, DIArray Elements, 
+                                   unsigned Flags, DIArray Elements,
                                    unsigned RunTimeLang) {
  // TAG_structure_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_structure_type),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNumber),
@@ -360,7 +449,7 @@ DIType DIBuilder::createStructType(DIDescriptor Context, StringRef Name,
 }
 
 /// createUnionType - Create debugging information entry for an union.
-DIType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name, 
+DIType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name,
                                   DIFile File,
                                   unsigned LineNumber, uint64_t SizeInBits,
                                   uint64_t AlignInBits, unsigned Flags,
@@ -368,7 +457,7 @@ DIType DIBuilder::createUnionType(DIDescriptor Scope, StringRef Name,
   // TAG_union_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_union_type),
-    Scope,
+    getNonCompileUnitScope(Scope),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNumber),
@@ -389,9 +478,9 @@ DIType DIBuilder::createSubroutineType(DIFile File, DIArray ParameterTypes) {
   // TAG_subroutine_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_subroutine_type),
-    File,
+    llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
     MDString::get(VMContext, ""),
-    File,
+    llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
     ConstantInt::get(Type::getInt32Ty(VMContext), 0),
     ConstantInt::get(Type::getInt64Ty(VMContext), 0),
     ConstantInt::get(Type::getInt64Ty(VMContext), 0),
@@ -405,16 +494,17 @@ DIType DIBuilder::createSubroutineType(DIFile File, DIArray ParameterTypes) {
   return DIType(MDNode::get(VMContext, Elts));
 }
 
-/// createEnumerationType - Create debugging information entry for an 
+/// createEnumerationType - Create debugging information entry for an
 /// enumeration.
-DIType DIBuilder::createEnumerationType(DIDescriptor Scope, StringRef Name, 
-                                        DIFile File, unsigned LineNumber, 
-                                        uint64_t SizeInBits, 
-                                        uint64_t AlignInBits, DIArray Elements) {
+DIType DIBuilder::createEnumerationType(DIDescriptor Scope, StringRef Name,
+                                        DIFile File, unsigned LineNumber,
+                                        uint64_t SizeInBits,
+                                        uint64_t AlignInBits,
+					DIArray Elements) {
   // TAG_enumeration_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_enumeration_type),
-    Scope,
+    getNonCompileUnitScope(Scope),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNumber),
@@ -428,20 +518,19 @@ DIType DIBuilder::createEnumerationType(DIDescriptor Scope, StringRef Name,
     llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
   };
   MDNode *Node = MDNode::get(VMContext, Elts);
-  NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.enum");
-  NMD->addOperand(Node);
+  AllEnumTypes.push_back(Node);
   return DIType(Node);
 }
 
 /// createArrayType - Create debugging information entry for an array.
-DIType DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, 
+DIType DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits,
                                   DIType Ty, DIArray Subscripts) {
   // TAG_array_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_array_type),
-    TheCU,
+    NULL, //TheCU,
     MDString::get(VMContext, ""),
-    TheCU,
+    NULL, //TheCU,
     ConstantInt::get(Type::getInt32Ty(VMContext), 0),
     ConstantInt::get(Type::getInt64Ty(VMContext), Size),
     ConstantInt::get(Type::getInt64Ty(VMContext), AlignInBits),
@@ -456,14 +545,14 @@ DIType DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits,
 }
 
 /// createVectorType - Create debugging information entry for a vector.
-DIType DIBuilder::createVectorType(uint64_t Size, uint64_t AlignInBits, 
+DIType DIBuilder::createVectorType(uint64_t Size, uint64_t AlignInBits,
                                    DIType Ty, DIArray Subscripts) {
   // TAG_vector_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_vector_type),
-    TheCU,
+    NULL, //TheCU,
     MDString::get(VMContext, ""),
-    TheCU,
+    NULL, //TheCU,
     ConstantInt::get(Type::getInt32Ty(VMContext), 0),
     ConstantInt::get(Type::getInt64Ty(VMContext), Size),
     ConstantInt::get(Type::getInt64Ty(VMContext), AlignInBits),
@@ -501,18 +590,17 @@ DIType DIBuilder::createArtificialType(DIType Ty) {
   return DIType(MDNode::get(VMContext, Elts));
 }
 
-/// retainType - Retain DIType in a module even if it is not referenced 
+/// retainType - Retain DIType in a module even if it is not referenced
 /// through debug info anchors.
 void DIBuilder::retainType(DIType T) {
-  NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.ty");
-  NMD->addOperand(T);
+  AllRetainTypes.push_back(T);
 }
 
 /// createUnspecifiedParameter - Create unspeicified type descriptor
 /// for the subroutine type.
 DIDescriptor DIBuilder::createUnspecifiedParameter() {
-  Value *Elts[] = { 
-    GetTagConstant(VMContext, dwarf::DW_TAG_unspecified_parameters) 
+  Value *Elts[] = {
+    GetTagConstant(VMContext, dwarf::DW_TAG_unspecified_parameters)
   };
   return DIDescriptor(MDNode::get(VMContext, Elts));
 }
@@ -532,7 +620,7 @@ DIType DIBuilder::createTemporaryType(DIFile F) {
   // use here as long as DIType accepts it.
   Value *Elts[] = {
     GetTagConstant(VMContext, DW_TAG_base_type),
-    F.getCompileUnit(),
+    TheCU,
     NULL,
     F
   };
@@ -563,12 +651,12 @@ DISubrange DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Hi) {
 
 /// createGlobalVariable - Create a new descriptor for the specified global.
 DIGlobalVariable DIBuilder::
-createGlobalVariable(StringRef Name, DIFile F, unsigned LineNumber, 
+createGlobalVariable(StringRef Name, DIFile F, unsigned LineNumber,
                      DIType Ty, bool isLocalToUnit, llvm::Value *Val) {
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_variable),
     llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
-    TheCU,
+    NULL, // TheCU,
     MDString::get(VMContext, Name),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, Name),
@@ -580,22 +668,20 @@ createGlobalVariable(StringRef Name, DIFile F, unsigned LineNumber,
     Val
   };
   MDNode *Node = MDNode::get(VMContext, Elts);
-  // Create a named metadata so that we do not lose this mdnode.
-  NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.gv");
-  NMD->addOperand(Node);
+  AllGVs.push_back(Node);
   return DIGlobalVariable(Node);
 }
 
 /// createStaticVariable - Create a new descriptor for the specified static
 /// variable.
 DIGlobalVariable DIBuilder::
-createStaticVariable(DIDescriptor Context, StringRef Name, 
-                     StringRef LinkageName, DIFile F, unsigned LineNumber, 
+createStaticVariable(DIDescriptor Context, StringRef Name,
+                     StringRef LinkageName, DIFile F, unsigned LineNumber,
                      DIType Ty, bool isLocalToUnit, llvm::Value *Val) {
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_variable),
     llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, LinkageName),
@@ -607,26 +693,25 @@ createStaticVariable(DIDescriptor Context, StringRef Name,
     Val
   };
   MDNode *Node = MDNode::get(VMContext, Elts);
-  // Create a named metadata so that we do not lose this mdnode.
-  NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.gv");
-  NMD->addOperand(Node);
+  AllGVs.push_back(Node);
   return DIGlobalVariable(Node);
 }
 
 /// createVariable - Create a new descriptor for the specified variable.
 DIVariable DIBuilder::createLocalVariable(unsigned Tag, DIDescriptor Scope,
                                           StringRef Name, DIFile File,
-                                          unsigned LineNo, DIType Ty, 
+                                          unsigned LineNo, DIType Ty,
                                           bool AlwaysPreserve, unsigned Flags,
                                           unsigned ArgNo) {
   Value *Elts[] = {
     GetTagConstant(VMContext, Tag),
-    Scope,
+    getNonCompileUnitScope(Scope),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), (LineNo | (ArgNo << 24))),
     Ty,
-    ConstantInt::get(Type::getInt32Ty(VMContext), Flags)
+    ConstantInt::get(Type::getInt32Ty(VMContext), Flags),
+    Constant::getNullValue(Type::getInt32Ty(VMContext)),
   };
   MDNode *Node = MDNode::get(VMContext, Elts);
   if (AlwaysPreserve) {
@@ -634,13 +719,7 @@ DIVariable DIBuilder::createLocalVariable(unsigned Tag, DIDescriptor Scope,
     // to preserve variable info in such situation then stash it in a
     // named mdnode.
     DISubprogram Fn(getDISubprogram(Scope));
-    StringRef FName = "fn";
-    if (Fn.getFunction())
-      FName = Fn.getFunction()->getName();
-    char One = '\1';
-    if (FName.startswith(StringRef(&One, 1)))
-      FName = FName.substr(1);
-    NamedMDNode *FnLocals = getOrInsertFnSpecificMDNode(M, FName);
+    NamedMDNode *FnLocals = getOrInsertFnSpecificMDNode(M, Fn);
     FnLocals->addOperand(Node);
   }
   return DIVariable(Node);
@@ -655,12 +734,14 @@ DIVariable DIBuilder::createComplexVariable(unsigned Tag, DIDescriptor Scope,
                                             unsigned ArgNo) {
   SmallVector<Value *, 15> Elts;
   Elts.push_back(GetTagConstant(VMContext, Tag));
-  Elts.push_back(Scope);
+  Elts.push_back(getNonCompileUnitScope(Scope)),
   Elts.push_back(MDString::get(VMContext, Name));
   Elts.push_back(F);
-  Elts.push_back(ConstantInt::get(Type::getInt32Ty(VMContext), (LineNo | (ArgNo << 24))));
+  Elts.push_back(ConstantInt::get(Type::getInt32Ty(VMContext),
+				  (LineNo | (ArgNo << 24))));
   Elts.push_back(Ty);
   Elts.push_back(llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)));
+  Elts.push_back(llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)));
   Elts.append(Addr.begin(), Addr.end());
 
   return DIVariable(MDNode::get(VMContext, Elts));
@@ -677,10 +758,15 @@ DISubprogram DIBuilder::createFunction(DIDescriptor Context,
                                        Function *Fn,
                                        MDNode *TParams,
                                        MDNode *Decl) {
+  Value *TElts[] = { GetTagConstant(VMContext, DW_TAG_base_type) };
+  MDNode *Temp = MDNode::getTemporary(VMContext, TElts);
+  Value *TVElts[] = { Temp };
+  MDNode *THolder = MDNode::get(VMContext, TVElts);
+
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_subprogram),
     llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, LinkageName),
@@ -696,13 +782,13 @@ DISubprogram DIBuilder::createFunction(DIDescriptor Context,
     ConstantInt::get(Type::getInt1Ty(VMContext), isOptimized),
     Fn,
     TParams,
-    Decl
+    Decl,
+    THolder
   };
   MDNode *Node = MDNode::get(VMContext, Elts);
 
   // Create a named metadata so that we do not lose this mdnode.
-  NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.sp");
-  NMD->addOperand(Node);
+  AllSubprograms.push_back(Node);
   return DISubprogram(Node);
 }
 
@@ -720,10 +806,15 @@ DISubprogram DIBuilder::createMethod(DIDescriptor Context,
                                      bool isOptimized,
                                      Function *Fn,
                                      MDNode *TParam) {
+  Value *TElts[] = { GetTagConstant(VMContext, DW_TAG_base_type) };
+  MDNode *Temp = MDNode::getTemporary(VMContext, TElts);
+  Value *TVElts[] = { Temp };
+  MDNode *THolder = MDNode::get(VMContext, TVElts);
+
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_subprogram),
     llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
-    Context,
+    getNonCompileUnitScope(Context),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, Name),
     MDString::get(VMContext, LinkageName),
@@ -739,12 +830,10 @@ DISubprogram DIBuilder::createMethod(DIDescriptor Context,
     ConstantInt::get(Type::getInt1Ty(VMContext), isOptimized),
     Fn,
     TParam,
+    llvm::Constant::getNullValue(Type::getInt32Ty(VMContext)),
+    THolder
   };
   MDNode *Node = MDNode::get(VMContext, Elts);
-
-  // Create a named metadata so that we do not lose this mdnode.
-  NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.sp");
-  NMD->addOperand(Node);
   return DISubprogram(Node);
 }
 
@@ -754,7 +843,7 @@ DINameSpace DIBuilder::createNameSpace(DIDescriptor Scope, StringRef Name,
                                        DIFile File, unsigned LineNo) {
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_namespace),
-    Scope,
+    getNonCompileUnitScope(Scope),
     MDString::get(VMContext, Name),
     File,
     ConstantInt::get(Type::getInt32Ty(VMContext), LineNo)
@@ -762,13 +851,25 @@ DINameSpace DIBuilder::createNameSpace(DIDescriptor Scope, StringRef Name,
   return DINameSpace(MDNode::get(VMContext, Elts));
 }
 
+/// createLexicalBlockFile - This creates a new MDNode that encapsulates
+/// an existing scope with a new filename.
+DILexicalBlockFile DIBuilder::createLexicalBlockFile(DIDescriptor Scope,
+						     DIFile File) {
+  Value *Elts[] = {
+    GetTagConstant(VMContext, dwarf::DW_TAG_lexical_block),
+    Scope,
+    File
+  };
+  return DILexicalBlockFile(MDNode::get(VMContext, Elts));
+}
+
 DILexicalBlock DIBuilder::createLexicalBlock(DIDescriptor Scope, DIFile File,
                                              unsigned Line, unsigned Col) {
   // Defeat MDNode uniqing for lexical blocks by using unique id.
   static unsigned int unique_id = 0;
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_lexical_block),
-    Scope,
+    getNonCompileUnitScope(Scope),
     ConstantInt::get(Type::getInt32Ty(VMContext), Line),
     ConstantInt::get(Type::getInt32Ty(VMContext), Col),
     File,
@@ -836,4 +937,3 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset,
                     VarInfo };
   return CallInst::Create(ValueFn, Args, "", InsertAtEnd);
 }
-
diff --git a/contrib/llvm/lib/Analysis/DbgInfoPrinter.cpp b/contrib/llvm/lib/Analysis/DbgInfoPrinter.cpp
index b23c3514d0bd..cd832abeba51 100644
--- a/contrib/llvm/lib/Analysis/DbgInfoPrinter.cpp
+++ b/contrib/llvm/lib/Analysis/DbgInfoPrinter.cpp
@@ -171,7 +171,7 @@ static bool getLocationInfo(const Value *V, std::string &DisplayName,
 
 void PrintDbgInfo::printVariableDeclaration(const Value *V) {
   std::string DisplayName, File, Directory, Type;
-  unsigned LineNo;
+  unsigned LineNo = 0;
 
   if (!getLocationInfo(V, DisplayName, Type, LineNo, File, Directory))
     return;
diff --git a/contrib/llvm/lib/Analysis/DebugInfo.cpp b/contrib/llvm/lib/Analysis/DebugInfo.cpp
index b42e946f2ffa..44457d3c3de9 100644
--- a/contrib/llvm/lib/Analysis/DebugInfo.cpp
+++ b/contrib/llvm/lib/Analysis/DebugInfo.cpp
@@ -39,6 +39,9 @@ DIDescriptor::DIDescriptor(const DIFile F) : DbgNode(F.DbgNode) {
 DIDescriptor::DIDescriptor(const DISubprogram F) : DbgNode(F.DbgNode) {
 }
 
+DIDescriptor::DIDescriptor(const DILexicalBlockFile F) : DbgNode(F.DbgNode) {
+}
+
 DIDescriptor::DIDescriptor(const DILexicalBlock F) : DbgNode(F.DbgNode) {
 }
 
@@ -111,9 +114,17 @@ Function *DIDescriptor::getFunctionField(unsigned Elt) const {
 unsigned DIVariable::getNumAddrElements() const {
   if (getVersion() <= llvm::LLVMDebugVersion8)
     return DbgNode->getNumOperands()-6;
-  return DbgNode->getNumOperands()-7;
+  if (getVersion() == llvm::LLVMDebugVersion9)
+    return DbgNode->getNumOperands()-7;
+  return DbgNode->getNumOperands()-8;
 }
 
+/// getInlinedAt - If this variable is inlined then return inline location.
+MDNode *DIVariable::getInlinedAt() const {
+  if (getVersion() <= llvm::LLVMDebugVersion9)
+    return NULL;
+  return dyn_cast_or_null<MDNode>(DbgNode->getOperand(7));
+}
 
 //===----------------------------------------------------------------------===//
 // Predicates
@@ -122,7 +133,14 @@ unsigned DIVariable::getNumAddrElements() const {
 /// isBasicType - Return true if the specified tag is legal for
 /// DIBasicType.
 bool DIDescriptor::isBasicType() const {
-  return DbgNode && getTag() == dwarf::DW_TAG_base_type;
+  if (!DbgNode) return false;
+  switch (getTag()) {
+  case dwarf::DW_TAG_base_type:
+  case dwarf::DW_TAG_unspecified_type:
+    return true;
+  default:
+    return false;
+  }
 }
 
 /// isDerivedType - Return true if the specified tag is legal for DIDerivedType.
@@ -248,9 +266,17 @@ bool DIDescriptor::isNameSpace() const {
   return DbgNode && getTag() == dwarf::DW_TAG_namespace;
 }
 
+/// isLexicalBlockFile - Return true if the specified descriptor is a
+/// lexical block with an extra file.
+bool DIDescriptor::isLexicalBlockFile() const {
+  return DbgNode && getTag() == dwarf::DW_TAG_lexical_block &&
+    (DbgNode->getNumOperands() == 3);
+}
+
 /// isLexicalBlock - Return true if the specified tag is DW_TAG_lexical_block.
 bool DIDescriptor::isLexicalBlock() const {
-  return DbgNode && getTag() == dwarf::DW_TAG_lexical_block;
+  return DbgNode && getTag() == dwarf::DW_TAG_lexical_block &&
+    (DbgNode->getNumOperands() > 3);
 }
 
 /// isSubrange - Return true if the specified tag is DW_TAG_subrange_type.
@@ -320,6 +346,22 @@ void DIType::replaceAllUsesWith(MDNode *D) {
   }
 }
 
+/// isUnsignedDIType - Return true if type encoding is unsigned.
+bool DIType::isUnsignedDIType() {
+  DIDerivedType DTy(DbgNode);
+  if (DTy.Verify())
+    return DTy.getTypeDerivedFrom().isUnsignedDIType();
+
+  DIBasicType BTy(DbgNode);
+  if (BTy.Verify()) {
+    unsigned Encoding = BTy.getEncoding();
+    if (Encoding == dwarf::DW_ATE_unsigned ||
+        Encoding == dwarf::DW_ATE_unsigned_char)
+      return true;
+  }
+  return false;
+}
+
 /// Verify - Verify that a compile unit is well formed.
 bool DICompileUnit::Verify() const {
   if (!DbgNode)
@@ -335,7 +377,7 @@ bool DICompileUnit::Verify() const {
 bool DIType::Verify() const {
   if (!DbgNode)
     return false;
-  if (!getContext().Verify())
+  if (getContext() && !getContext().Verify())
     return false;
   unsigned Tag = getTag();
   if (!isBasicType() && Tag != dwarf::DW_TAG_const_type &&
@@ -343,6 +385,7 @@ bool DIType::Verify() const {
       Tag != dwarf::DW_TAG_reference_type && Tag != dwarf::DW_TAG_restrict_type 
       && Tag != dwarf::DW_TAG_vector_type && Tag != dwarf::DW_TAG_array_type
       && Tag != dwarf::DW_TAG_enumeration_type 
+      && Tag != dwarf::DW_TAG_subroutine_type
       && getFilename().empty())
     return false;
   return true;
@@ -362,12 +405,9 @@ bool DIDerivedType::Verify() const {
 bool DICompositeType::Verify() const {
   if (!DbgNode)
     return false;
-  if (!getContext().Verify())
+  if (getContext() && !getContext().Verify())
     return false;
 
-  DICompileUnit CU = getCompileUnit();
-  if (!CU.Verify())
-    return false;
   return true;
 }
 
@@ -376,11 +416,7 @@ bool DISubprogram::Verify() const {
   if (!DbgNode)
     return false;
 
-  if (!getContext().Verify())
-    return false;
-
-  DICompileUnit CU = getCompileUnit();
-  if (!CU.Verify())
+  if (getContext() && !getContext().Verify())
     return false;
 
   DICompositeType Ty = getType();
@@ -397,11 +433,7 @@ bool DIGlobalVariable::Verify() const {
   if (getDisplayName().empty())
     return false;
 
-  if (!getContext().Verify())
-    return false;
-
-  DICompileUnit CU = getCompileUnit();
-  if (!CU.Verify())
+  if (getContext() && !getContext().Verify())
     return false;
 
   DIType Ty = getType();
@@ -419,10 +451,7 @@ bool DIVariable::Verify() const {
   if (!DbgNode)
     return false;
 
-  if (!getContext().Verify())
-    return false;
-
-  if (!getCompileUnit().Verify())
+  if (getContext() && !getContext().Verify())
     return false;
 
   DIType Ty = getType();
@@ -446,8 +475,6 @@ bool DINameSpace::Verify() const {
     return false;
   if (getName().empty())
     return false;
-  if (!getCompileUnit().Verify())
-    return false;
   return true;
 }
 
@@ -504,9 +531,28 @@ unsigned DISubprogram::isOptimized() const {
   return 0;
 }
 
+MDNode *DISubprogram::getVariablesNodes() const {
+  if (!DbgNode || DbgNode->getNumOperands() <= 19)
+    return NULL;
+  if (MDNode *Temp = dyn_cast_or_null<MDNode>(DbgNode->getOperand(19)))
+    return dyn_cast_or_null<MDNode>(Temp->getOperand(0));
+  return NULL;
+}
+
+DIArray DISubprogram::getVariables() const {
+  if (!DbgNode || DbgNode->getNumOperands() <= 19)
+    return DIArray();
+  if (MDNode *T = dyn_cast_or_null<MDNode>(DbgNode->getOperand(19)))
+    if (MDNode *A = dyn_cast_or_null<MDNode>(T->getOperand(0)))
+      return DIArray(A);
+  return DIArray();
+}
+
 StringRef DIScope::getFilename() const {
   if (!DbgNode)
     return StringRef();
+  if (isLexicalBlockFile())
+    return DILexicalBlockFile(DbgNode).getFilename();
   if (isLexicalBlock())
     return DILexicalBlock(DbgNode).getFilename();
   if (isSubprogram())
@@ -526,6 +572,8 @@ StringRef DIScope::getFilename() const {
 StringRef DIScope::getDirectory() const {
   if (!DbgNode)
     return StringRef();
+  if (isLexicalBlockFile())
+    return DILexicalBlockFile(DbgNode).getDirectory();
   if (isLexicalBlock())
     return DILexicalBlock(DbgNode).getDirectory();
   if (isSubprogram())
@@ -542,6 +590,47 @@ StringRef DIScope::getDirectory() const {
   return StringRef();
 }
 
+DIArray DICompileUnit::getEnumTypes() const {
+  if (!DbgNode || DbgNode->getNumOperands() < 14)
+    return DIArray();
+
+  if (MDNode *N = dyn_cast_or_null<MDNode>(DbgNode->getOperand(10)))
+    if (MDNode *A = dyn_cast_or_null<MDNode>(N->getOperand(0)))
+      return DIArray(A);
+  return DIArray();
+}
+
+DIArray DICompileUnit::getRetainedTypes() const {
+  if (!DbgNode || DbgNode->getNumOperands() < 14)
+    return DIArray();
+
+  if (MDNode *N = dyn_cast_or_null<MDNode>(DbgNode->getOperand(11)))
+    if (MDNode *A = dyn_cast_or_null<MDNode>(N->getOperand(0)))
+      return DIArray(A);
+  return DIArray();
+}
+
+DIArray DICompileUnit::getSubprograms() const {
+  if (!DbgNode || DbgNode->getNumOperands() < 14)
+    return DIArray();
+
+  if (MDNode *N = dyn_cast_or_null<MDNode>(DbgNode->getOperand(12)))
+    if (MDNode *A = dyn_cast_or_null<MDNode>(N->getOperand(0)))
+      return DIArray(A);
+  return DIArray();
+}
+
+
+DIArray DICompileUnit::getGlobalVariables() const {
+  if (!DbgNode || DbgNode->getNumOperands() < 14)
+    return DIArray();
+
+  if (MDNode *N = dyn_cast_or_null<MDNode>(DbgNode->getOperand(13)))
+    if (MDNode *A = dyn_cast_or_null<MDNode>(N->getOperand(0)))
+      return DIArray(A);
+  return DIArray();
+}
+
 //===----------------------------------------------------------------------===//
 // DIDescriptor: dump routines for all descriptors.
 //===----------------------------------------------------------------------===//
@@ -573,7 +662,6 @@ void DIType::print(raw_ostream &OS) const {
   OS << " [" << dwarf::TagString(Tag) << "] ";
 
   // TODO : Print context
-  getCompileUnit().print(OS);
   OS << " ["
          << "line " << getLineNumber() << ", "
          << getSizeInBits() << " bits, "
@@ -629,7 +717,6 @@ void DISubprogram::print(raw_ostream &OS) const {
   OS << " [" << dwarf::TagString(Tag) << "] ";
 
   // TODO : Print context
-  getCompileUnit().print(OS);
   OS << " [" << getLineNumber() << "] ";
 
   if (isLocalToUnit())
@@ -652,7 +739,6 @@ void DIGlobalVariable::print(raw_ostream &OS) const {
   OS << " [" << dwarf::TagString(Tag) << "] ";
 
   // TODO : Print context
-  getCompileUnit().print(OS);
   OS << " [" << getLineNumber() << "] ";
 
   if (isLocalToUnit())
@@ -666,13 +752,48 @@ void DIGlobalVariable::print(raw_ostream &OS) const {
   OS << "]\n";
 }
 
+static void printDebugLoc(DebugLoc DL, raw_ostream &CommentOS,
+                          const LLVMContext &Ctx) {
+  if (!DL.isUnknown()) {          // Print source line info.
+    DIScope Scope(DL.getScope(Ctx));
+    // Omit the directory, because it's likely to be long and uninteresting.
+    if (Scope.Verify())
+      CommentOS << Scope.getFilename();
+    else
+      CommentOS << "<unknown>";
+    CommentOS << ':' << DL.getLine();
+    if (DL.getCol() != 0)
+      CommentOS << ':' << DL.getCol();
+    DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(DL.getInlinedAt(Ctx));
+    if (!InlinedAtDL.isUnknown()) {
+      CommentOS << " @[ ";
+      printDebugLoc(InlinedAtDL, CommentOS, Ctx);
+      CommentOS << " ]";
+    }
+  }
+}
+
+void DIVariable::printExtendedName(raw_ostream &OS) const {
+  const LLVMContext &Ctx = DbgNode->getContext();
+  StringRef Res = getName();
+  if (!Res.empty())
+    OS << Res << "," << getLineNumber();
+  if (MDNode *InlinedAt = getInlinedAt()) {
+    DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(InlinedAt);
+    if (!InlinedAtDL.isUnknown()) {
+      OS << " @[";
+      printDebugLoc(InlinedAtDL, OS, Ctx);
+      OS << "]";
+    }
+  }
+}
+
 /// print - Print variable.
 void DIVariable::print(raw_ostream &OS) const {
   StringRef Res = getName();
   if (!Res.empty())
     OS << " [" << Res << "] ";
 
-  getCompileUnit().print(OS);
   OS << " [" << getLineNumber() << "] ";
   getType().print(OS);
   OS << "\n";
@@ -744,22 +865,61 @@ static void fixupObjcLikeName(StringRef Str, SmallVectorImpl<char> &Out) {
 
 /// getFnSpecificMDNode - Return a NameMDNode, if available, that is 
 /// suitable to hold function specific information.
-NamedMDNode *llvm::getFnSpecificMDNode(const Module &M, StringRef FuncName) {
+NamedMDNode *llvm::getFnSpecificMDNode(const Module &M, DISubprogram Fn) {
   SmallString<32> Name = StringRef("llvm.dbg.lv.");
-  fixupObjcLikeName(FuncName, Name);
-
+  StringRef FName = "fn";
+  if (Fn.getFunction())
+    FName = Fn.getFunction()->getName();
+  else
+    FName = Fn.getName();
+  char One = '\1';
+  if (FName.startswith(StringRef(&One, 1)))
+    FName = FName.substr(1);
+  fixupObjcLikeName(FName, Name);
   return M.getNamedMetadata(Name.str());
 }
 
 /// getOrInsertFnSpecificMDNode - Return a NameMDNode that is suitable
 /// to hold function specific information.
-NamedMDNode *llvm::getOrInsertFnSpecificMDNode(Module &M, StringRef FuncName) {
+NamedMDNode *llvm::getOrInsertFnSpecificMDNode(Module &M, DISubprogram Fn) {
   SmallString<32> Name = StringRef("llvm.dbg.lv.");
-  fixupObjcLikeName(FuncName, Name);
-
+  StringRef FName = "fn";
+  if (Fn.getFunction())
+    FName = Fn.getFunction()->getName();
+  else
+    FName = Fn.getName();
+  char One = '\1';
+  if (FName.startswith(StringRef(&One, 1)))
+    FName = FName.substr(1);
+  fixupObjcLikeName(FName, Name);
+  
   return M.getOrInsertNamedMetadata(Name.str());
 }
 
+/// createInlinedVariable - Create a new inlined variable based on current
+/// variable.
+/// @param DV            Current Variable.
+/// @param InlinedScope  Location at current variable is inlined.
+DIVariable llvm::createInlinedVariable(MDNode *DV, MDNode *InlinedScope,
+                                       LLVMContext &VMContext) {
+  SmallVector<Value *, 16> Elts;
+  // Insert inlined scope as 7th element.
+  for (unsigned i = 0, e = DV->getNumOperands(); i != e; ++i)
+    i == 7 ? Elts.push_back(InlinedScope) :
+             Elts.push_back(DV->getOperand(i));
+  return DIVariable(MDNode::get(VMContext, Elts));
+}
+
+/// cleanseInlinedVariable - Remove inlined scope from the variable.
+DIVariable llvm::cleanseInlinedVariable(MDNode *DV, LLVMContext &VMContext) {
+  SmallVector<Value *, 16> Elts;
+  // Insert inlined scope as 7th element.
+  for (unsigned i = 0, e = DV->getNumOperands(); i != e; ++i)
+    i == 7 ? 
+      Elts.push_back(llvm::Constant::getNullValue(Type::getInt32Ty(VMContext))):
+      Elts.push_back(DV->getOperand(i));
+  return DIVariable(MDNode::get(VMContext, Elts));
+}
 
 //===----------------------------------------------------------------------===//
 // DebugInfoFinder implementations.
@@ -767,6 +927,10 @@ NamedMDNode *llvm::getOrInsertFnSpecificMDNode(Module &M, StringRef FuncName) {
 
 /// processModule - Process entire module and collect debug info.
 void DebugInfoFinder::processModule(Module &M) {
+  if (NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu"))
+    for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i)
+      addCompileUnit(DICompileUnit(CU_Nodes->getOperand(i)));
+
   for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
     for (Function::iterator FI = (*I).begin(), FE = (*I).end(); FI != FE; ++FI)
       for (BasicBlock::iterator BI = (*FI).begin(), BE = (*FI).end(); BI != BE;
@@ -785,6 +949,10 @@ void DebugInfoFinder::processModule(Module &M) {
           addCompileUnit(DICompileUnit(Scope));
         else if (Scope.isSubprogram())
           processSubprogram(DISubprogram(Scope));
+        else if (Scope.isLexicalBlockFile()) {
+          DILexicalBlockFile DBF = DILexicalBlockFile(Scope);
+          processLexicalBlock(DILexicalBlock(DBF.getScope()));
+        }
         else if (Scope.isLexicalBlock())
           processLexicalBlock(DILexicalBlock(Scope));
 
@@ -796,7 +964,8 @@ void DebugInfoFinder::processModule(Module &M) {
     for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
       DIGlobalVariable DIG(cast<MDNode>(NMD->getOperand(i)));
       if (addGlobalVariable(DIG)) {
-        addCompileUnit(DIG.getCompileUnit());
+        if (DIG.getVersion() <= LLVMDebugVersion10)
+          addCompileUnit(DIG.getCompileUnit());
         processType(DIG.getType());
       }
     }
@@ -817,6 +986,10 @@ void DebugInfoFinder::processLocation(DILocation Loc) {
     processSubprogram(DISubprogram(S));
   else if (S.isLexicalBlock())
     processLexicalBlock(DILexicalBlock(S));
+  else if (S.isLexicalBlockFile()) {
+    DILexicalBlockFile DBF = DILexicalBlockFile(S);
+    processLexicalBlock(DILexicalBlock(DBF.getScope()));
+  }
   processLocation(Loc.getOrigLocation());
 }
 
@@ -824,8 +997,8 @@ void DebugInfoFinder::processLocation(DILocation Loc) {
 void DebugInfoFinder::processType(DIType DT) {
   if (!addType(DT))
     return;
-
-  addCompileUnit(DT.getCompileUnit());
+  if (DT.getVersion() <= LLVMDebugVersion10)
+    addCompileUnit(DT.getCompileUnit());
   if (DT.isCompositeType()) {
     DICompositeType DCT(DT);
     processType(DCT.getTypeDerivedFrom());
@@ -848,6 +1021,10 @@ void DebugInfoFinder::processLexicalBlock(DILexicalBlock LB) {
   DIScope Context = LB.getContext();
   if (Context.isLexicalBlock())
     return processLexicalBlock(DILexicalBlock(Context));
+  else if (Context.isLexicalBlockFile()) {
+    DILexicalBlockFile DBF = DILexicalBlockFile(Context);
+    return processLexicalBlock(DILexicalBlock(DBF.getScope()));
+  }
   else
     return processSubprogram(DISubprogram(Context));
 }
@@ -856,7 +1033,8 @@ void DebugInfoFinder::processLexicalBlock(DILexicalBlock LB) {
 void DebugInfoFinder::processSubprogram(DISubprogram SP) {
   if (!addSubprogram(SP))
     return;
-  addCompileUnit(SP.getCompileUnit());
+  if (SP.getVersion() <= LLVMDebugVersion10)
+    addCompileUnit(SP.getCompileUnit());
   processType(SP.getType());
 }
 
@@ -871,8 +1049,8 @@ void DebugInfoFinder::processDeclare(DbgDeclareInst *DDI) {
 
   if (!NodesSeen.insert(DV))
     return;
-
-  addCompileUnit(DIVariable(N).getCompileUnit());
+  if (DIVariable(N).getVersion() <= LLVMDebugVersion10)
+    addCompileUnit(DIVariable(N).getCompileUnit());
   processType(DIVariable(N).getType());
 }
 
@@ -930,6 +1108,9 @@ DISubprogram llvm::getDISubprogram(const MDNode *Scope) {
   if (D.isSubprogram())
     return DISubprogram(Scope);
 
+  if (D.isLexicalBlockFile())
+    return getDISubprogram(DILexicalBlockFile(Scope).getContext());
+  
   if (D.isLexicalBlock())
     return getDISubprogram(DILexicalBlock(Scope).getContext());
 
@@ -946,3 +1127,17 @@ DICompositeType llvm::getDICompositeType(DIType T) {
 
   return DICompositeType();
 }
+
+/// isSubprogramContext - Return true if Context is either a subprogram
+/// or another context nested inside a subprogram.
+bool llvm::isSubprogramContext(const MDNode *Context) {
+  if (!Context)
+    return false;
+  DIDescriptor D(Context);
+  if (D.isSubprogram())
+    return true;
+  if (D.isType())
+    return isSubprogramContext(DIType(Context).getContext());
+  return false;
+}
+
diff --git a/contrib/llvm/lib/Analysis/IPA/CallGraphSCCPass.cpp b/contrib/llvm/lib/Analysis/IPA/CallGraphSCCPass.cpp
index 659ffab0c6f0..963da752343b 100644
--- a/contrib/llvm/lib/Analysis/IPA/CallGraphSCCPass.cpp
+++ b/contrib/llvm/lib/Analysis/IPA/CallGraphSCCPass.cpp
@@ -44,8 +44,8 @@ namespace {
 class CGPassManager : public ModulePass, public PMDataManager {
 public:
   static char ID;
-  explicit CGPassManager(int Depth) 
-    : ModulePass(ID), PMDataManager(Depth) { }
+  explicit CGPassManager() 
+    : ModulePass(ID), PMDataManager() { }
 
   /// run - Execute all of the passes scheduled for execution.  Keep track of
   /// whether any of the passes modifies the module, and if so, return true.
@@ -350,6 +350,7 @@ bool CGPassManager::RefreshCallGraph(CallGraphSCC &CurSCC,
            dbgs() << "CGSCCPASSMGR: SCC Refresh didn't change call graph.\n";
          }
         );
+  (void)MadeChange;
 
   return DevirtualizedCall;
 }
@@ -542,7 +543,7 @@ void CallGraphSCCPass::assignPassManager(PMStack &PMS,
     PMDataManager *PMD = PMS.top();
 
     // [1] Create new Call Graph Pass Manager
-    CGP = new CGPassManager(PMD->getDepth() + 1);
+    CGP = new CGPassManager();
 
     // [2] Set up new manager's top level manager
     PMTopLevelManager *TPM = PMD->getTopLevelManager();
diff --git a/contrib/llvm/lib/Analysis/IPA/FindUsedTypes.cpp b/contrib/llvm/lib/Analysis/IPA/FindUsedTypes.cpp
index 6535786668bc..e9df3ca01022 100644
--- a/contrib/llvm/lib/Analysis/IPA/FindUsedTypes.cpp
+++ b/contrib/llvm/lib/Analysis/IPA/FindUsedTypes.cpp
@@ -29,7 +29,7 @@ INITIALIZE_PASS(FindUsedTypes, "print-used-types",
 // IncorporateType - Incorporate one type and all of its subtypes into the
 // collection of used types.
 //
-void FindUsedTypes::IncorporateType(const Type *Ty) {
+void FindUsedTypes::IncorporateType(Type *Ty) {
   // If ty doesn't already exist in the used types map, add it now, otherwise
   // return.
   if (!UsedTypes.insert(Ty)) return;  // Already contain Ty.
@@ -94,7 +94,7 @@ bool FindUsedTypes::runOnModule(Module &m) {
 //
 void FindUsedTypes::print(raw_ostream &OS, const Module *M) const {
   OS << "Types in use by this module:\n";
-  for (SetVector<const Type *>::const_iterator I = UsedTypes.begin(),
+  for (SetVector<Type *>::const_iterator I = UsedTypes.begin(),
        E = UsedTypes.end(); I != E; ++I) {
     OS << "   " << **I << '\n';
   }
diff --git a/contrib/llvm/lib/Analysis/IVUsers.cpp b/contrib/llvm/lib/Analysis/IVUsers.cpp
index e5f0a77ab67d..d0ca8920ab9f 100644
--- a/contrib/llvm/lib/Analysis/IVUsers.cpp
+++ b/contrib/llvm/lib/Analysis/IVUsers.cpp
@@ -146,7 +146,8 @@ bool IVUsers::AddUsersIfInteresting(Instruction *I) {
                                    ISE, User, I,
                                    NewUse.PostIncLoops,
                                    *SE, *DT);
-      DEBUG(dbgs() << "   NORMALIZED TO: " << *ISE << '\n');
+      DEBUG(if (SE->getSCEV(I) != ISE)
+              dbgs() << "   NORMALIZED TO: " << *ISE << '\n');
     }
   }
   return true;
diff --git a/contrib/llvm/lib/Analysis/InlineCost.cpp b/contrib/llvm/lib/Analysis/InlineCost.cpp
index efde5984c115..e12e322c2a99 100644
--- a/contrib/llvm/lib/Analysis/InlineCost.cpp
+++ b/contrib/llvm/lib/Analysis/InlineCost.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/CallingConv.h"
 #include "llvm/IntrinsicInst.h"
+#include "llvm/Target/TargetData.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace llvm;
@@ -24,13 +25,13 @@ using namespace llvm;
 /// TODO: Perhaps calls like memcpy, strcpy, etc?
 bool llvm::callIsSmall(const Function *F) {
   if (!F) return false;
-  
+
   if (F->hasLocalLinkage()) return false;
-  
+
   if (!F->hasName()) return false;
-  
+
   StringRef Name = F->getName();
-  
+
   // These will all likely lower to a single selection DAG node.
   if (Name == "copysign" || Name == "copysignf" || Name == "copysignl" ||
       Name == "fabs" || Name == "fabsf" || Name == "fabsl" ||
@@ -38,7 +39,7 @@ bool llvm::callIsSmall(const Function *F) {
       Name == "cos" || Name == "cosf" || Name == "cosl" ||
       Name == "sqrt" || Name == "sqrtf" || Name == "sqrtl" )
     return true;
-  
+
   // These are all likely to be optimized into something smaller.
   if (Name == "pow" || Name == "powf" || Name == "powl" ||
       Name == "exp2" || Name == "exp2l" || Name == "exp2f" ||
@@ -46,13 +47,14 @@ bool llvm::callIsSmall(const Function *F) {
       Name == "round" || Name == "ffs" || Name == "ffsl" ||
       Name == "abs" || Name == "labs" || Name == "llabs")
     return true;
-  
+
   return false;
 }
 
 /// analyzeBasicBlock - Fill in the current structure with information gleaned
 /// from the specified block.
-void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB) {
+void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB,
+                                    const TargetData *TD) {
   ++NumBlocks;
   unsigned NumInstsBeforeThisBB = NumInsts;
   for (BasicBlock::const_iterator II = BB->begin(), E = BB->end();
@@ -67,8 +69,8 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB) {
       ImmutableCallSite CS(cast<Instruction>(II));
 
       if (const Function *F = CS.getCalledFunction()) {
-        // If a function is both internal and has a single use, then it is 
-        // extremely likely to get inlined in the future (it was probably 
+        // If a function is both internal and has a single use, then it is
+        // extremely likely to get inlined in the future (it was probably
         // exposed by an interleaved devirtualization pass).
         if (F->hasInternalLinkage() && F->hasOneUse())
           ++NumInlineCandidates;
@@ -91,20 +93,25 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB) {
           ++NumCalls;
       }
     }
-    
+
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(II)) {
       if (!AI->isStaticAlloca())
         this->usesDynamicAlloca = true;
     }
 
     if (isa<ExtractElementInst>(II) || II->getType()->isVectorTy())
-      ++NumVectorInsts; 
-    
+      ++NumVectorInsts;
+
     if (const CastInst *CI = dyn_cast<CastInst>(II)) {
       // Noop casts, including ptr <-> int,  don't count.
-      if (CI->isLosslessCast() || isa<IntToPtrInst>(CI) || 
+      if (CI->isLosslessCast() || isa<IntToPtrInst>(CI) ||
           isa<PtrToIntInst>(CI))
         continue;
+      // trunc to a native type is free (assuming the target has compare and
+      // shift-right of the same width).
+      if (isa<TruncInst>(CI) && TD &&
+          TD->isLegalInteger(TD->getTypeSizeInBits(CI->getType())))
+        continue;
       // Result of a cmp instruction is often extended (to be used by other
       // cmp instructions, logical or return instructions). These are usually
       // nop on most sane targets.
@@ -119,10 +126,10 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB) {
 
     ++NumInsts;
   }
-  
+
   if (isa<ReturnInst>(BB->getTerminator()))
     ++NumRets;
-  
+
   // We never want to inline functions that contain an indirectbr.  This is
   // incorrect because all the blockaddress's (in static global initializers
   // for example) would be referring to the original function, and this indirect
@@ -217,7 +224,7 @@ unsigned CodeMetrics::CountCodeReductionForAlloca(Value *V) {
 
 /// analyzeFunction - Fill in the current structure with information gleaned
 /// from the specified function.
-void CodeMetrics::analyzeFunction(Function *F) {
+void CodeMetrics::analyzeFunction(Function *F, const TargetData *TD) {
   // If this function contains a call to setjmp or _setjmp, never inline
   // it.  This is a hack because we depend on the user marking their local
   // variables as volatile if they are live across a setjmp call, and they
@@ -227,13 +234,14 @@ void CodeMetrics::analyzeFunction(Function *F) {
 
   // Look at the size of the callee.
   for (Function::const_iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
-    analyzeBasicBlock(&*BB);
+    analyzeBasicBlock(&*BB, TD);
 }
 
 /// analyzeFunction - Fill in the current structure with information gleaned
 /// from the specified function.
-void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F) {
-  Metrics.analyzeFunction(F);
+void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F,
+                                                       const TargetData *TD) {
+  Metrics.analyzeFunction(F, TD);
 
   // A function with exactly one return has it removed during the inlining
   // process (see InlineFunction), so don't count it.
@@ -252,7 +260,7 @@ void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F) {
 /// NeverInline - returns true if the function should never be inlined into
 /// any caller
 bool InlineCostAnalyzer::FunctionInfo::NeverInline() {
-  return (Metrics.callsSetJmp || Metrics.isRecursive || 
+  return (Metrics.callsSetJmp || Metrics.isRecursive ||
           Metrics.containsIndirectBr);
 }
 // getSpecializationBonus - The heuristic used to determine the per-call
@@ -263,19 +271,19 @@ int InlineCostAnalyzer::getSpecializationBonus(Function *Callee,
 {
   if (Callee->mayBeOverridden())
     return 0;
-  
+
   int Bonus = 0;
   // If this function uses the coldcc calling convention, prefer not to
   // specialize it.
   if (Callee->getCallingConv() == CallingConv::Cold)
     Bonus -= InlineConstants::ColdccPenalty;
-  
+
   // Get information about the callee.
   FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
-  
+
   // If we haven't calculated this information yet, do so now.
   if (CalleeFI->Metrics.NumBlocks == 0)
-    CalleeFI->analyzeFunction(Callee);
+    CalleeFI->analyzeFunction(Callee, TD);
 
   unsigned ArgNo = 0;
   unsigned i = 0;
@@ -286,7 +294,7 @@ int InlineCostAnalyzer::getSpecializationBonus(Function *Callee,
       Bonus += CountBonusForConstant(I);
     }
 
-  // Calls usually take a long time, so they make the specialization gain 
+  // Calls usually take a long time, so they make the specialization gain
   // smaller.
   Bonus -= CalleeFI->Metrics.NumCalls * InlineConstants::CallPenalty;
 
@@ -300,13 +308,13 @@ int InlineCostAnalyzer::getSpecializationBonus(Function *Callee,
 // inlining because we decide we don't want to give a bonus for
 // devirtualizing.
 int InlineCostAnalyzer::ConstantFunctionBonus(CallSite CS, Constant *C) {
-  
+
   // This could just be NULL.
   if (!C) return 0;
-  
+
   Function *F = dyn_cast<Function>(C);
   if (!F) return 0;
-  
+
   int Bonus = InlineConstants::IndirectCallBonus + getInlineSize(CS, F);
   return (Bonus > 0) ? 0 : Bonus;
 }
@@ -355,18 +363,18 @@ int InlineCostAnalyzer::CountBonusForConstant(Value *V, Constant *C) {
         Bonus += CountBonusForConstant(&Inst);
     }
   }
-  
+
   return Bonus;
 }
 
 int InlineCostAnalyzer::getInlineSize(CallSite CS, Function *Callee) {
   // Get information about the callee.
   FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
-  
+
   // If we haven't calculated this information yet, do so now.
   if (CalleeFI->Metrics.NumBlocks == 0)
-    CalleeFI->analyzeFunction(Callee);
-  
+    CalleeFI->analyzeFunction(Callee, TD);
+
   // InlineCost - This value measures how good of an inline candidate this call
   // site is to inline.  A lower inline cost make is more likely for the call to
   // be inlined.  This value may go negative.
@@ -392,9 +400,9 @@ int InlineCostAnalyzer::getInlineSize(CallSite CS, Function *Callee) {
     // weights calculated for the callee to determine how much will be folded
     // away with this information.
     else if (isa<Constant>(I))
-      InlineCost -= CalleeFI->ArgumentWeights[ArgNo].ConstantWeight;       
+      InlineCost -= CalleeFI->ArgumentWeights[ArgNo].ConstantWeight;
   }
-  
+
   // Each argument passed in has a cost at both the caller and the callee
   // sides.  Measurements show that each argument costs about the same as an
   // instruction.
@@ -408,28 +416,28 @@ int InlineCostAnalyzer::getInlineSize(CallSite CS, Function *Callee) {
 
   // Look at the size of the callee. Each instruction counts as 5.
   InlineCost += CalleeFI->Metrics.NumInsts*InlineConstants::InstrCost;
-  
+
   return InlineCost;
 }
 
 int InlineCostAnalyzer::getInlineBonuses(CallSite CS, Function *Callee) {
   // Get information about the callee.
   FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
-  
+
   // If we haven't calculated this information yet, do so now.
   if (CalleeFI->Metrics.NumBlocks == 0)
-    CalleeFI->analyzeFunction(Callee);
-    
+    CalleeFI->analyzeFunction(Callee, TD);
+
   bool isDirectCall = CS.getCalledFunction() == Callee;
   Instruction *TheCall = CS.getInstruction();
   int Bonus = 0;
-  
+
   // If there is only one call of the function, and it has internal linkage,
   // make it almost guaranteed to be inlined.
   //
   if (Callee->hasLocalLinkage() && Callee->hasOneUse() && isDirectCall)
     Bonus += InlineConstants::LastCallToStaticBonus;
-  
+
   // If the instruction after the call, or if the normal destination of the
   // invoke is an unreachable instruction, the function is noreturn.  As such,
   // there is little point in inlining this.
@@ -438,12 +446,12 @@ int InlineCostAnalyzer::getInlineBonuses(CallSite CS, Function *Callee) {
       Bonus += InlineConstants::NoreturnPenalty;
   } else if (isa<UnreachableInst>(++BasicBlock::iterator(TheCall)))
     Bonus += InlineConstants::NoreturnPenalty;
-  
+
   // If this function uses the coldcc calling convention, prefer not to inline
   // it.
   if (Callee->getCallingConv() == CallingConv::Cold)
     Bonus += InlineConstants::ColdccPenalty;
-  
+
   // Add to the inline quality for properties that make the call valuable to
   // inline.  This includes factors that indicate that the result of inlining
   // the function will be optimizable.  Currently this just looks at arguments
@@ -455,7 +463,7 @@ int InlineCostAnalyzer::getInlineBonuses(CallSite CS, Function *Callee) {
     // Compute any constant bonus due to inlining we want to give here.
     if (isa<Constant>(I))
       Bonus += CountBonusForConstant(FI, cast<Constant>(I));
-      
+
   return Bonus;
 }
 
@@ -483,10 +491,10 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS,
 
   // Get information about the callee.
   FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
-  
+
   // If we haven't calculated this information yet, do so now.
   if (CalleeFI->Metrics.NumBlocks == 0)
-    CalleeFI->analyzeFunction(Callee);
+    CalleeFI->analyzeFunction(Callee, TD);
 
   // If we should never inline this, return a huge cost.
   if (CalleeFI->NeverInline())
@@ -498,15 +506,15 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS,
   // requires handling setjmp somewhere else, however.
   if (!Callee->isDeclaration() && Callee->hasFnAttr(Attribute::AlwaysInline))
     return InlineCost::getAlways();
-    
+
   if (CalleeFI->Metrics.usesDynamicAlloca) {
     // Get information about the caller.
     FunctionInfo &CallerFI = CachedFunctionInfo[Caller];
 
     // If we haven't calculated this information yet, do so now.
     if (CallerFI.Metrics.NumBlocks == 0) {
-      CallerFI.analyzeFunction(Caller);
-     
+      CallerFI.analyzeFunction(Caller, TD);
+
       // Recompute the CalleeFI pointer, getting Caller could have invalidated
       // it.
       CalleeFI = &CachedFunctionInfo[Callee];
@@ -538,16 +546,16 @@ InlineCost InlineCostAnalyzer::getSpecializationCost(Function *Callee,
   // something else.
   if (Callee->mayBeOverridden())
     return llvm::InlineCost::getNever();
-  
+
   // Get information about the callee.
   FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
-  
+
   // If we haven't calculated this information yet, do so now.
   if (CalleeFI->Metrics.NumBlocks == 0)
-    CalleeFI->analyzeFunction(Callee);
+    CalleeFI->analyzeFunction(Callee, TD);
 
   int Cost = 0;
-  
+
   // Look at the original size of the callee.  Each instruction counts as 5.
   Cost += CalleeFI->Metrics.NumInsts * InlineConstants::InstrCost;
 
@@ -564,13 +572,13 @@ InlineCost InlineCostAnalyzer::getSpecializationCost(Function *Callee,
 // higher threshold to determine if the function call should be inlined.
 float InlineCostAnalyzer::getInlineFudgeFactor(CallSite CS) {
   Function *Callee = CS.getCalledFunction();
-  
+
   // Get information about the callee.
   FunctionInfo &CalleeFI = CachedFunctionInfo[Callee];
-  
+
   // If we haven't calculated this information yet, do so now.
   if (CalleeFI.Metrics.NumBlocks == 0)
-    CalleeFI.analyzeFunction(Callee);
+    CalleeFI.analyzeFunction(Callee, TD);
 
   float Factor = 1.0f;
   // Single BB functions are often written to be inlined.
@@ -604,7 +612,7 @@ InlineCostAnalyzer::growCachedCostInfo(Function *Caller, Function *Callee) {
     --CallerMetrics.NumCalls;
 
   if (Callee == 0) return;
-  
+
   CodeMetrics &CalleeMetrics = CachedFunctionInfo[Callee].Metrics;
 
   // If we don't have metrics for the callee, don't recalculate them just to
@@ -614,7 +622,7 @@ InlineCostAnalyzer::growCachedCostInfo(Function *Caller, Function *Callee) {
     resetCachedCostInfo(Caller);
     return;
   }
-  
+
   // Since CalleeMetrics were already calculated, we know that the CallerMetrics
   // reference isn't invalidated: both were in the DenseMap.
   CallerMetrics.usesDynamicAlloca |= CalleeMetrics.usesDynamicAlloca;
@@ -636,7 +644,7 @@ InlineCostAnalyzer::growCachedCostInfo(Function *Caller, Function *Callee) {
     CallerMetrics.NumInsts -= Callee->arg_size();
   else
     CallerMetrics.NumInsts = 0;
-  
+
   // We are not updating the argument weights. We have already determined that
   // Caller is a fairly large function, so we accept the loss of precision.
 }
diff --git a/contrib/llvm/lib/Analysis/InstructionSimplify.cpp b/contrib/llvm/lib/Analysis/InstructionSimplify.cpp
index 8709f6bf9d26..131cc97d2379 100644
--- a/contrib/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/contrib/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -48,6 +48,26 @@ static Value *SimplifyOrInst(Value *, Value *, const TargetData *,
 static Value *SimplifyXorInst(Value *, Value *, const TargetData *,
                               const DominatorTree *, unsigned);
 
+/// getFalse - For a boolean type, or a vector of boolean type, return false, or
+/// a vector with every element false, as appropriate for the type.
+static Constant *getFalse(Type *Ty) {
+  assert((Ty->isIntegerTy(1) ||
+          (Ty->isVectorTy() &&
+           cast<VectorType>(Ty)->getElementType()->isIntegerTy(1))) &&
+         "Expected i1 type or a vector of i1!");
+  return Constant::getNullValue(Ty);
+}
+
+/// getTrue - For a boolean type, or a vector of boolean type, return true, or
+/// a vector with every element true, as appropriate for the type.
+static Constant *getTrue(Type *Ty) {
+  assert((Ty->isIntegerTy(1) ||
+          (Ty->isVectorTy() &&
+           cast<VectorType>(Ty)->getElementType()->isIntegerTy(1))) &&
+         "Expected i1 type or a vector of i1!");
+  return Constant::getAllOnesValue(Ty);
+}
+
 /// ValueDominatesPHI - Does the given value dominate the specified phi node?
 static bool ValueDominatesPHI(Value *V, PHINode *P, const DominatorTree *DT) {
   Instruction *I = dyn_cast<Instruction>(V);
@@ -526,7 +546,7 @@ static Value *SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
     if (Constant *CRHS = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { CLHS, CRHS };
       return ConstantFoldInstOperands(Instruction::Add, CLHS->getType(),
-                                      Ops, 2, TD);
+                                      Ops, TD);
     }
 
     // Canonicalize the constant to the RHS.
@@ -595,7 +615,7 @@ static Value *SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
     if (Constant *CRHS = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { CLHS, CRHS };
       return ConstantFoldInstOperands(Instruction::Sub, CLHS->getType(),
-                                      Ops, 2, TD);
+                                      Ops, TD);
     }
 
   // X - undef -> undef
@@ -715,7 +735,7 @@ static Value *SimplifyMulInst(Value *Op0, Value *Op1, const TargetData *TD,
     if (Constant *CRHS = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { CLHS, CRHS };
       return ConstantFoldInstOperands(Instruction::Mul, CLHS->getType(),
-                                      Ops, 2, TD);
+                                      Ops, TD);
     }
 
     // Canonicalize the constant to the RHS.
@@ -788,7 +808,7 @@ static Value *SimplifyDiv(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,
   if (Constant *C0 = dyn_cast<Constant>(Op0)) {
     if (Constant *C1 = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { C0, C1 };
-      return ConstantFoldInstOperands(Opcode, C0->getType(), Ops, 2, TD);
+      return ConstantFoldInstOperands(Opcode, C0->getType(), Ops, TD);
     }
   }
 
@@ -909,7 +929,7 @@ static Value *SimplifyRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,
   if (Constant *C0 = dyn_cast<Constant>(Op0)) {
     if (Constant *C1 = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { C0, C1 };
-      return ConstantFoldInstOperands(Opcode, C0->getType(), Ops, 2, TD);
+      return ConstantFoldInstOperands(Opcode, C0->getType(), Ops, TD);
     }
   }
 
@@ -1012,7 +1032,7 @@ static Value *SimplifyShift(unsigned Opcode, Value *Op0, Value *Op1,
   if (Constant *C0 = dyn_cast<Constant>(Op0)) {
     if (Constant *C1 = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { C0, C1 };
-      return ConstantFoldInstOperands(Opcode, C0->getType(), Ops, 2, TD);
+      return ConstantFoldInstOperands(Opcode, C0->getType(), Ops, TD);
     }
   }
 
@@ -1138,7 +1158,7 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const TargetData *TD,
     if (Constant *CRHS = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { CLHS, CRHS };
       return ConstantFoldInstOperands(Instruction::And, CLHS->getType(),
-                                      Ops, 2, TD);
+                                      Ops, TD);
     }
 
     // Canonicalize the constant to the RHS.
@@ -1227,7 +1247,7 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const TargetData *TD,
     if (Constant *CRHS = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { CLHS, CRHS };
       return ConstantFoldInstOperands(Instruction::Or, CLHS->getType(),
-                                      Ops, 2, TD);
+                                      Ops, TD);
     }
 
     // Canonicalize the constant to the RHS.
@@ -1321,7 +1341,7 @@ static Value *SimplifyXorInst(Value *Op0, Value *Op1, const TargetData *TD,
     if (Constant *CRHS = dyn_cast<Constant>(Op1)) {
       Constant *Ops[] = { CLHS, CRHS };
       return ConstantFoldInstOperands(Instruction::Xor, CLHS->getType(),
-                                      Ops, 2, TD);
+                                      Ops, TD);
     }
 
     // Canonicalize the constant to the RHS.
@@ -1372,7 +1392,7 @@ Value *llvm::SimplifyXorInst(Value *Op0, Value *Op1, const TargetData *TD,
   return ::SimplifyXorInst(Op0, Op1, TD, DT, RecursionLimit);
 }
 
-static const Type *GetCompareTy(Value *Op) {
+static Type *GetCompareTy(Value *Op) {
   return CmpInst::makeCmpResultType(Op->getType());
 }
 
@@ -1413,8 +1433,8 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     Pred = CmpInst::getSwappedPredicate(Pred);
   }
 
-  const Type *ITy = GetCompareTy(LHS); // The return type.
-  const Type *OpTy = LHS->getType();   // The operand type.
+  Type *ITy = GetCompareTy(LHS); // The return type.
+  Type *OpTy = LHS->getType();   // The operand type.
 
   // icmp X, X -> true/false
   // X icmp undef -> true/false.  For example, icmp ugt %X, undef -> false
@@ -1478,48 +1498,46 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     default:
       assert(false && "Unknown ICmp predicate!");
     case ICmpInst::ICMP_ULT:
-      // getNullValue also works for vectors, unlike getFalse.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
     case ICmpInst::ICMP_UGE:
-      // getAllOnesValue also works for vectors, unlike getTrue.
-      return ConstantInt::getAllOnesValue(ITy);
+      return getTrue(ITy);
     case ICmpInst::ICMP_EQ:
     case ICmpInst::ICMP_ULE:
       if (isKnownNonZero(LHS, TD))
-        return Constant::getNullValue(ITy);
+        return getFalse(ITy);
       break;
     case ICmpInst::ICMP_NE:
     case ICmpInst::ICMP_UGT:
       if (isKnownNonZero(LHS, TD))
-        return ConstantInt::getAllOnesValue(ITy);
+        return getTrue(ITy);
       break;
     case ICmpInst::ICMP_SLT:
       ComputeSignBit(LHS, LHSKnownNonNegative, LHSKnownNegative, TD);
       if (LHSKnownNegative)
-        return ConstantInt::getAllOnesValue(ITy);
+        return getTrue(ITy);
       if (LHSKnownNonNegative)
-        return Constant::getNullValue(ITy);
+        return getFalse(ITy);
       break;
     case ICmpInst::ICMP_SLE:
       ComputeSignBit(LHS, LHSKnownNonNegative, LHSKnownNegative, TD);
       if (LHSKnownNegative)
-        return ConstantInt::getAllOnesValue(ITy);
+        return getTrue(ITy);
       if (LHSKnownNonNegative && isKnownNonZero(LHS, TD))
-        return Constant::getNullValue(ITy);
+        return getFalse(ITy);
       break;
     case ICmpInst::ICMP_SGE:
       ComputeSignBit(LHS, LHSKnownNonNegative, LHSKnownNegative, TD);
       if (LHSKnownNegative)
-        return Constant::getNullValue(ITy);
+        return getFalse(ITy);
       if (LHSKnownNonNegative)
-        return ConstantInt::getAllOnesValue(ITy);
+        return getTrue(ITy);
       break;
     case ICmpInst::ICMP_SGT:
       ComputeSignBit(LHS, LHSKnownNonNegative, LHSKnownNegative, TD);
       if (LHSKnownNegative)
-        return Constant::getNullValue(ITy);
+        return getFalse(ITy);
       if (LHSKnownNonNegative && isKnownNonZero(LHS, TD))
-        return ConstantInt::getAllOnesValue(ITy);
+        return getTrue(ITy);
       break;
     }
   }
@@ -1593,8 +1611,8 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   if (isa<CastInst>(LHS) && (isa<Constant>(RHS) || isa<CastInst>(RHS))) {
     Instruction *LI = cast<CastInst>(LHS);
     Value *SrcOp = LI->getOperand(0);
-    const Type *SrcTy = SrcOp->getType();
-    const Type *DstTy = LI->getType();
+    Type *SrcTy = SrcOp->getType();
+    Type *DstTy = LI->getType();
 
     // Turn icmp (ptrtoint x), (ptrtoint/constant) into a compare of the input
     // if the integer type is the same size as the pointer type.
@@ -1811,8 +1829,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     case ICmpInst::ICMP_EQ:
     case ICmpInst::ICMP_UGT:
     case ICmpInst::ICMP_UGE:
-      // getNullValue also works for vectors, unlike getFalse.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
     case ICmpInst::ICMP_SLT:
     case ICmpInst::ICMP_SLE:
       ComputeSignBit(LHS, KnownNonNegative, KnownNegative, TD);
@@ -1822,8 +1839,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     case ICmpInst::ICMP_NE:
     case ICmpInst::ICMP_ULT:
     case ICmpInst::ICMP_ULE:
-      // getAllOnesValue also works for vectors, unlike getTrue.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     }
   }
   if (RBO && match(RBO, m_URem(m_Value(), m_Specific(LHS)))) {
@@ -1840,8 +1856,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     case ICmpInst::ICMP_NE:
     case ICmpInst::ICMP_UGT:
     case ICmpInst::ICMP_UGE:
-      // getAllOnesValue also works for vectors, unlike getTrue.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     case ICmpInst::ICMP_SLT:
     case ICmpInst::ICMP_SLE:
       ComputeSignBit(RHS, KnownNonNegative, KnownNegative, TD);
@@ -1851,8 +1866,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     case ICmpInst::ICMP_EQ:
     case ICmpInst::ICMP_ULT:
     case ICmpInst::ICMP_ULE:
-      // getNullValue also works for vectors, unlike getFalse.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
     }
   }
 
@@ -1874,7 +1888,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
         return V;
       break;
     case Instruction::Shl: {
-      bool NUW = LBO->hasNoUnsignedWrap() && LBO->hasNoUnsignedWrap();
+      bool NUW = LBO->hasNoUnsignedWrap() && RBO->hasNoUnsignedWrap();
       bool NSW = LBO->hasNoSignedWrap() && RBO->hasNoSignedWrap();
       if (!NUW && !NSW)
         break;
@@ -1955,10 +1969,10 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     }
     case CmpInst::ICMP_SGE:
       // Always true.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     case CmpInst::ICMP_SLT:
       // Always false.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
     }
   }
 
@@ -2025,10 +2039,10 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     }
     case CmpInst::ICMP_UGE:
       // Always true.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     case CmpInst::ICMP_ULT:
       // Always false.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
     }
   }
 
@@ -2040,40 +2054,40 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     // max(x, ?) pred min(x, ?).
     if (Pred == CmpInst::ICMP_SGE)
       // Always true.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     if (Pred == CmpInst::ICMP_SLT)
       // Always false.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
   } else if (match(LHS, m_SMin(m_Value(A), m_Value(B))) &&
              match(RHS, m_SMax(m_Value(C), m_Value(D))) &&
              (A == C || A == D || B == C || B == D)) {
     // min(x, ?) pred max(x, ?).
     if (Pred == CmpInst::ICMP_SLE)
       // Always true.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     if (Pred == CmpInst::ICMP_SGT)
       // Always false.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
   } else if (match(LHS, m_UMax(m_Value(A), m_Value(B))) &&
              match(RHS, m_UMin(m_Value(C), m_Value(D))) &&
              (A == C || A == D || B == C || B == D)) {
     // max(x, ?) pred min(x, ?).
     if (Pred == CmpInst::ICMP_UGE)
       // Always true.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     if (Pred == CmpInst::ICMP_ULT)
       // Always false.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
   } else if (match(LHS, m_UMin(m_Value(A), m_Value(B))) &&
              match(RHS, m_UMax(m_Value(C), m_Value(D))) &&
              (A == C || A == D || B == C || B == D)) {
     // min(x, ?) pred max(x, ?).
     if (Pred == CmpInst::ICMP_ULE)
       // Always true.
-      return Constant::getAllOnesValue(ITy);
+      return getTrue(ITy);
     if (Pred == CmpInst::ICMP_UGT)
       // Always false.
-      return Constant::getNullValue(ITy);
+      return getFalse(ITy);
   }
 
   // If the comparison is with the result of a select instruction, check whether
@@ -2219,43 +2233,71 @@ Value *llvm::SimplifySelectInst(Value *CondVal, Value *TrueVal, Value *FalseVal,
 
 /// SimplifyGEPInst - Given operands for an GetElementPtrInst, see if we can
 /// fold the result.  If not, this returns null.
-Value *llvm::SimplifyGEPInst(Value *const *Ops, unsigned NumOps,
+Value *llvm::SimplifyGEPInst(ArrayRef<Value *> Ops,
                              const TargetData *TD, const DominatorTree *) {
   // The type of the GEP pointer operand.
-  const PointerType *PtrTy = cast<PointerType>(Ops[0]->getType());
+  PointerType *PtrTy = cast<PointerType>(Ops[0]->getType());
 
   // getelementptr P -> P.
-  if (NumOps == 1)
+  if (Ops.size() == 1)
     return Ops[0];
 
   if (isa<UndefValue>(Ops[0])) {
     // Compute the (pointer) type returned by the GEP instruction.
-    const Type *LastType = GetElementPtrInst::getIndexedType(PtrTy, &Ops[1],
-                                                             NumOps-1);
-    const Type *GEPTy = PointerType::get(LastType, PtrTy->getAddressSpace());
+    Type *LastType = GetElementPtrInst::getIndexedType(PtrTy, Ops.slice(1));
+    Type *GEPTy = PointerType::get(LastType, PtrTy->getAddressSpace());
     return UndefValue::get(GEPTy);
   }
 
-  if (NumOps == 2) {
+  if (Ops.size() == 2) {
     // getelementptr P, 0 -> P.
     if (ConstantInt *C = dyn_cast<ConstantInt>(Ops[1]))
       if (C->isZero())
         return Ops[0];
     // getelementptr P, N -> P if P points to a type of zero size.
     if (TD) {
-      const Type *Ty = PtrTy->getElementType();
+      Type *Ty = PtrTy->getElementType();
       if (Ty->isSized() && TD->getTypeAllocSize(Ty) == 0)
         return Ops[0];
     }
   }
 
   // Check to see if this is constant foldable.
-  for (unsigned i = 0; i != NumOps; ++i)
+  for (unsigned i = 0, e = Ops.size(); i != e; ++i)
     if (!isa<Constant>(Ops[i]))
       return 0;
 
-  return ConstantExpr::getGetElementPtr(cast<Constant>(Ops[0]),
-                                        (Constant *const*)Ops+1, NumOps-1);
+  return ConstantExpr::getGetElementPtr(cast<Constant>(Ops[0]), Ops.slice(1));
+}
+
+/// SimplifyInsertValueInst - Given operands for an InsertValueInst, see if we
+/// can fold the result.  If not, this returns null.
+Value *llvm::SimplifyInsertValueInst(Value *Agg, Value *Val,
+                                     ArrayRef<unsigned> Idxs,
+                                     const TargetData *,
+                                     const DominatorTree *) {
+  if (Constant *CAgg = dyn_cast<Constant>(Agg))
+    if (Constant *CVal = dyn_cast<Constant>(Val))
+      return ConstantFoldInsertValueInstruction(CAgg, CVal, Idxs);
+
+  // insertvalue x, undef, n -> x
+  if (match(Val, m_Undef()))
+    return Agg;
+
+  // insertvalue x, (extractvalue y, n), n
+  if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val))
+    if (EV->getAggregateOperand()->getType() == Agg->getType() &&
+        EV->getIndices() == Idxs) {
+      // insertvalue undef, (extractvalue y, n), n -> y
+      if (match(Agg, m_Undef()))
+        return EV->getAggregateOperand();
+
+      // insertvalue y, (extractvalue y, n), n -> y
+      if (Agg == EV->getAggregateOperand())
+        return Agg;
+    }
+
+  return 0;
 }
 
 /// SimplifyPHINode - See if we can fold the given phi.  If not, returns null.
@@ -2328,7 +2370,7 @@ static Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
     if (Constant *CLHS = dyn_cast<Constant>(LHS))
       if (Constant *CRHS = dyn_cast<Constant>(RHS)) {
         Constant *COps[] = {CLHS, CRHS};
-        return ConstantFoldInstOperands(Opcode, LHS->getType(), COps, 2, TD);
+        return ConstantFoldInstOperands(Opcode, LHS->getType(), COps, TD);
       }
 
     // If the operation is associative, try some generic simplifications.
@@ -2456,7 +2498,14 @@ Value *llvm::SimplifyInstruction(Instruction *I, const TargetData *TD,
     break;
   case Instruction::GetElementPtr: {
     SmallVector<Value*, 8> Ops(I->op_begin(), I->op_end());
-    Result = SimplifyGEPInst(&Ops[0], Ops.size(), TD, DT);
+    Result = SimplifyGEPInst(Ops, TD, DT);
+    break;
+  }
+  case Instruction::InsertValue: {
+    InsertValueInst *IV = cast<InsertValueInst>(I);
+    Result = SimplifyInsertValueInst(IV->getAggregateOperand(),
+                                     IV->getInsertedValueOperand(),
+                                     IV->getIndices(), TD, DT);
     break;
   }
   case Instruction::PHI:
diff --git a/contrib/llvm/lib/Analysis/LazyValueInfo.cpp b/contrib/llvm/lib/Analysis/LazyValueInfo.cpp
index 6e275978276d..f80595c7dbed 100644
--- a/contrib/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/contrib/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -630,7 +630,7 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV,
   if (BB == &BB->getParent()->getEntryBlock()) {
     assert(isa<Argument>(Val) && "Unknown live-in to the entry block");
     if (NotNull) {
-      const PointerType *PTy = cast<PointerType>(Val->getType());
+      PointerType *PTy = cast<PointerType>(Val->getType());
       Result = LVILatticeVal::getNot(ConstantPointerNull::get(PTy));
     } else {
       Result.markOverdefined();
@@ -658,7 +658,7 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV,
       // If we previously determined that this is a pointer that can't be null
       // then return that rather than giving up entirely.
       if (NotNull) {
-        const PointerType *PTy = cast<PointerType>(Val->getType());
+        PointerType *PTy = cast<PointerType>(Val->getType());
         Result = LVILatticeVal::getNot(ConstantPointerNull::get(PTy));
       }
       
@@ -728,7 +728,7 @@ bool LazyValueInfoCache::solveBlockValueConstantRange(LVILatticeVal &BBLV,
   
   ConstantRange LHSRange = LHSVal.getConstantRange();
   ConstantRange RHSRange(1);
-  const IntegerType *ResultTy = cast<IntegerType>(BBI->getType());
+  IntegerType *ResultTy = cast<IntegerType>(BBI->getType());
   if (isa<BinaryOperator>(BBI)) {
     if (ConstantInt *RHS = dyn_cast<ConstantInt>(BBI->getOperand(1))) {
       RHSRange = ConstantRange(RHS->getValue());
diff --git a/contrib/llvm/lib/Analysis/Lint.cpp b/contrib/llvm/lib/Analysis/Lint.cpp
index 89755da85097..38d677d502a7 100644
--- a/contrib/llvm/lib/Analysis/Lint.cpp
+++ b/contrib/llvm/lib/Analysis/Lint.cpp
@@ -71,7 +71,7 @@ namespace {
     void visitCallSite(CallSite CS);
     void visitMemoryReference(Instruction &I, Value *Ptr,
                               uint64_t Size, unsigned Align,
-                              const Type *Ty, unsigned Flags);
+                              Type *Ty, unsigned Flags);
 
     void visitCallInst(CallInst &I);
     void visitInvokeInst(InvokeInst &I);
@@ -201,7 +201,7 @@ void Lint::visitCallSite(CallSite CS) {
             "Undefined behavior: Caller and callee calling convention differ",
             &I);
 
-    const FunctionType *FT = F->getFunctionType();
+    FunctionType *FT = F->getFunctionType();
     unsigned NumActualArgs = unsigned(CS.arg_end()-CS.arg_begin());
 
     Assert1(FT->isVarArg() ?
@@ -240,7 +240,7 @@ void Lint::visitCallSite(CallSite CS) {
 
         // Check that an sret argument points to valid memory.
         if (Formal->hasStructRetAttr() && Actual->getType()->isPointerTy()) {
-          const Type *Ty =
+          Type *Ty =
             cast<PointerType>(Formal->getType())->getElementType();
           visitMemoryReference(I, Actual, AA->getTypeStoreSize(Ty),
                                TD ? TD->getABITypeAlignment(Ty) : 0,
@@ -364,7 +364,7 @@ void Lint::visitReturnInst(ReturnInst &I) {
 // TODO: Check readnone/readonly function attributes.
 void Lint::visitMemoryReference(Instruction &I,
                                 Value *Ptr, uint64_t Size, unsigned Align,
-                                const Type *Ty, unsigned Flags) {
+                                Type *Ty, unsigned Flags) {
   // If no memory is being referenced, it doesn't matter if the pointer
   // is valid.
   if (Size == 0)
diff --git a/contrib/llvm/lib/Analysis/Loads.cpp b/contrib/llvm/lib/Analysis/Loads.cpp
index c5c676b5265f..0e6bcbfae4f6 100644
--- a/contrib/llvm/lib/Analysis/Loads.cpp
+++ b/contrib/llvm/lib/Analysis/Loads.cpp
@@ -63,7 +63,7 @@ static Value *getUnderlyingObjectWithOffset(Value *V, const TargetData *TD,
         return V;
       SmallVector<Value*, 8> Indices(GEP->op_begin() + 1, GEP->op_end());
       ByteOffset += TD->getIndexedOffset(GEP->getPointerOperandType(),
-                                         &Indices[0], Indices.size());
+                                         Indices);
       V = GEP->getPointerOperand();
     } else if (Operator::getOpcode(V) == Instruction::BitCast) {
       V = cast<Operator>(V)->getOperand(0);
@@ -90,7 +90,7 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom,
   if (TD)
     Base = getUnderlyingObjectWithOffset(V, TD, ByteOffset);
 
-  const Type *BaseType = 0;
+  Type *BaseType = 0;
   unsigned BaseAlign = 0;
   if (const AllocaInst *AI = dyn_cast<AllocaInst>(Base)) {
     // An alloca is safe to load from as load as it is suitably aligned.
@@ -114,7 +114,7 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom,
         return true; // Loading directly from an alloca or global is OK.
 
       // Check if the load is within the bounds of the underlying object.
-      const PointerType *AddrTy = cast<PointerType>(V->getType());
+      PointerType *AddrTy = cast<PointerType>(V->getType());
       uint64_t LoadSize = TD->getTypeStoreSize(AddrTy->getElementType());
       if (ByteOffset + LoadSize <= TD->getTypeAllocSize(BaseType) &&
           (Align == 0 || (ByteOffset % Align) == 0))
@@ -169,7 +169,7 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB,
   // If we're using alias analysis to disambiguate get the size of *Ptr.
   uint64_t AccessSize = 0;
   if (AA) {
-    const Type *AccessTy = cast<PointerType>(Ptr->getType())->getElementType();
+    Type *AccessTy = cast<PointerType>(Ptr->getType())->getElementType();
     AccessSize = AA->getTypeStoreSize(AccessTy);
   }
   
@@ -188,12 +188,16 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB,
     
     --ScanFrom;
     // If this is a load of Ptr, the loaded value is available.
+    // (This is true even if the load is volatile or atomic, although
+    // those cases are unlikely.)
     if (LoadInst *LI = dyn_cast<LoadInst>(Inst))
       if (AreEquivalentAddressValues(LI->getOperand(0), Ptr))
         return LI;
     
     if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
       // If this is a store through Ptr, the value is available!
+      // (This is true even if the store is volatile or atomic, although
+      // those cases are unlikely.)
       if (AreEquivalentAddressValues(SI->getOperand(1), Ptr))
         return SI->getOperand(0);
       
diff --git a/contrib/llvm/lib/Analysis/LoopDependenceAnalysis.cpp b/contrib/llvm/lib/Analysis/LoopDependenceAnalysis.cpp
index c1afe8fbd618..3997ac478b52 100644
--- a/contrib/llvm/lib/Analysis/LoopDependenceAnalysis.cpp
+++ b/contrib/llvm/lib/Analysis/LoopDependenceAnalysis.cpp
@@ -76,7 +76,13 @@ static void GetMemRefInstrs(const Loop *L,
 }
 
 static bool IsLoadOrStoreInst(Value *I) {
-  return isa<LoadInst>(I) || isa<StoreInst>(I);
+  // Returns true if the load or store can be analyzed. Atomic and volatile
+  // operations have properties which this analysis does not understand.
+  if (LoadInst *LI = dyn_cast<LoadInst>(I))
+    return LI->isUnordered();
+  else if (StoreInst *SI = dyn_cast<StoreInst>(I))
+    return SI->isUnordered();
+  return false;
 }
 
 static Value *GetPointerOperand(Value *I) {
diff --git a/contrib/llvm/lib/Analysis/LoopInfo.cpp b/contrib/llvm/lib/Analysis/LoopInfo.cpp
index 05831402f409..85aaccaefc37 100644
--- a/contrib/llvm/lib/Analysis/LoopInfo.cpp
+++ b/contrib/llvm/lib/Analysis/LoopInfo.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Constants.h"
 #include "llvm/Instructions.h"
 #include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/LoopIterator.h"
 #include "llvm/Assembly/Writer.h"
 #include "llvm/Support/CFG.h"
 #include "llvm/Support/CommandLine.h"
@@ -55,12 +56,12 @@ bool Loop::isLoopInvariant(Value *V) const {
 }
 
 /// hasLoopInvariantOperands - Return true if all the operands of the
-/// specified instruction are loop invariant. 
+/// specified instruction are loop invariant.
 bool Loop::hasLoopInvariantOperands(Instruction *I) const {
   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
     if (!isLoopInvariant(I->getOperand(i)))
       return false;
-  
+
   return true;
 }
 
@@ -98,6 +99,9 @@ bool Loop::makeLoopInvariant(Instruction *I, bool &Changed,
     return false;
   if (I->mayReadFromMemory())
     return false;
+  // The landingpad instruction is immobile.
+  if (isa<LandingPadInst>(I))
+    return false;
   // Determine the insertion point, unless one was given.
   if (!InsertPt) {
     BasicBlock *Preheader = getLoopPreheader();
@@ -110,7 +114,7 @@ bool Loop::makeLoopInvariant(Instruction *I, bool &Changed,
   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
     if (!makeLoopInvariant(I->getOperand(i), Changed, InsertPt))
       return false;
-  
+
   // Hoist.
   I->moveBefore(InsertPt);
   Changed = true;
@@ -382,6 +386,205 @@ void Loop::dump() const {
   print(dbgs());
 }
 
+//===----------------------------------------------------------------------===//
+// UnloopUpdater implementation
+//
+
+namespace {
+/// Find the new parent loop for all blocks within the "unloop" whose last
+/// backedges has just been removed.
+class UnloopUpdater {
+  Loop *Unloop;
+  LoopInfo *LI;
+
+  LoopBlocksDFS DFS;
+
+  // Map unloop's immediate subloops to their nearest reachable parents. Nested
+  // loops within these subloops will not change parents. However, an immediate
+  // subloop's new parent will be the nearest loop reachable from either its own
+  // exits *or* any of its nested loop's exits.
+  DenseMap<Loop*, Loop*> SubloopParents;
+
+  // Flag the presence of an irreducible backedge whose destination is a block
+  // directly contained by the original unloop.
+  bool FoundIB;
+
+public:
+  UnloopUpdater(Loop *UL, LoopInfo *LInfo) :
+    Unloop(UL), LI(LInfo), DFS(UL), FoundIB(false) {}
+
+  void updateBlockParents();
+
+  void removeBlocksFromAncestors();
+
+  void updateSubloopParents();
+
+protected:
+  Loop *getNearestLoop(BasicBlock *BB, Loop *BBLoop);
+};
+} // end anonymous namespace
+
+/// updateBlockParents - Update the parent loop for all blocks that are directly
+/// contained within the original "unloop".
+void UnloopUpdater::updateBlockParents() {
+  if (Unloop->getNumBlocks()) {
+    // Perform a post order CFG traversal of all blocks within this loop,
+    // propagating the nearest loop from sucessors to predecessors.
+    LoopBlocksTraversal Traversal(DFS, LI);
+    for (LoopBlocksTraversal::POTIterator POI = Traversal.begin(),
+           POE = Traversal.end(); POI != POE; ++POI) {
+
+      Loop *L = LI->getLoopFor(*POI);
+      Loop *NL = getNearestLoop(*POI, L);
+
+      if (NL != L) {
+        // For reducible loops, NL is now an ancestor of Unloop.
+        assert((NL != Unloop && (!NL || NL->contains(Unloop))) &&
+               "uninitialized successor");
+        LI->changeLoopFor(*POI, NL);
+      }
+      else {
+        // Or the current block is part of a subloop, in which case its parent
+        // is unchanged.
+        assert((FoundIB || Unloop->contains(L)) && "uninitialized successor");
+      }
+    }
+  }
+  // Each irreducible loop within the unloop induces a round of iteration using
+  // the DFS result cached by Traversal.
+  bool Changed = FoundIB;
+  for (unsigned NIters = 0; Changed; ++NIters) {
+    assert(NIters < Unloop->getNumBlocks() && "runaway iterative algorithm");
+
+    // Iterate over the postorder list of blocks, propagating the nearest loop
+    // from successors to predecessors as before.
+    Changed = false;
+    for (LoopBlocksDFS::POIterator POI = DFS.beginPostorder(),
+           POE = DFS.endPostorder(); POI != POE; ++POI) {
+
+      Loop *L = LI->getLoopFor(*POI);
+      Loop *NL = getNearestLoop(*POI, L);
+      if (NL != L) {
+        assert(NL != Unloop && (!NL || NL->contains(Unloop)) &&
+               "uninitialized successor");
+        LI->changeLoopFor(*POI, NL);
+        Changed = true;
+      }
+    }
+  }
+}
+
+/// removeBlocksFromAncestors - Remove unloop's blocks from all ancestors below
+/// their new parents.
+void UnloopUpdater::removeBlocksFromAncestors() {
+  // Remove unloop's blocks from all ancestors below their new parents.
+  for (Loop::block_iterator BI = Unloop->block_begin(),
+         BE = Unloop->block_end(); BI != BE; ++BI) {
+    Loop *NewParent = LI->getLoopFor(*BI);
+    // If this block is an immediate subloop, remove all blocks (including
+    // nested subloops) from ancestors below the new parent loop.
+    // Otherwise, if this block is in a nested subloop, skip it.
+    if (SubloopParents.count(NewParent))
+      NewParent = SubloopParents[NewParent];
+    else if (Unloop->contains(NewParent))
+      continue;
+
+    // Remove blocks from former Ancestors except Unloop itself which will be
+    // deleted.
+    for (Loop *OldParent = Unloop->getParentLoop(); OldParent != NewParent;
+         OldParent = OldParent->getParentLoop()) {
+      assert(OldParent && "new loop is not an ancestor of the original");
+      OldParent->removeBlockFromLoop(*BI);
+    }
+  }
+}
+
+/// updateSubloopParents - Update the parent loop for all subloops directly
+/// nested within unloop.
+void UnloopUpdater::updateSubloopParents() {
+  while (!Unloop->empty()) {
+    Loop *Subloop = *llvm::prior(Unloop->end());
+    Unloop->removeChildLoop(llvm::prior(Unloop->end()));
+
+    assert(SubloopParents.count(Subloop) && "DFS failed to visit subloop");
+    if (SubloopParents[Subloop])
+      SubloopParents[Subloop]->addChildLoop(Subloop);
+    else
+      LI->addTopLevelLoop(Subloop);
+  }
+}
+
+/// getNearestLoop - Return the nearest parent loop among this block's
+/// successors. If a successor is a subloop header, consider its parent to be
+/// the nearest parent of the subloop's exits.
+///
+/// For subloop blocks, simply update SubloopParents and return NULL.
+Loop *UnloopUpdater::getNearestLoop(BasicBlock *BB, Loop *BBLoop) {
+
+  // Initially for blocks directly contained by Unloop, NearLoop == Unloop and
+  // is considered uninitialized.
+  Loop *NearLoop = BBLoop;
+
+  Loop *Subloop = 0;
+  if (NearLoop != Unloop && Unloop->contains(NearLoop)) {
+    Subloop = NearLoop;
+    // Find the subloop ancestor that is directly contained within Unloop.
+    while (Subloop->getParentLoop() != Unloop) {
+      Subloop = Subloop->getParentLoop();
+      assert(Subloop && "subloop is not an ancestor of the original loop");
+    }
+    // Get the current nearest parent of the Subloop exits, initially Unloop.
+    if (!SubloopParents.count(Subloop))
+      SubloopParents[Subloop] = Unloop;
+    NearLoop = SubloopParents[Subloop];
+  }
+
+  succ_iterator I = succ_begin(BB), E = succ_end(BB);
+  if (I == E) {
+    assert(!Subloop && "subloop blocks must have a successor");
+    NearLoop = 0; // unloop blocks may now exit the function.
+  }
+  for (; I != E; ++I) {
+    if (*I == BB)
+      continue; // self loops are uninteresting
+
+    Loop *L = LI->getLoopFor(*I);
+    if (L == Unloop) {
+      // This successor has not been processed. This path must lead to an
+      // irreducible backedge.
+      assert((FoundIB || !DFS.hasPostorder(*I)) && "should have seen IB");
+      FoundIB = true;
+    }
+    if (L != Unloop && Unloop->contains(L)) {
+      // Successor is in a subloop.
+      if (Subloop)
+        continue; // Branching within subloops. Ignore it.
+
+      // BB branches from the original into a subloop header.
+      assert(L->getParentLoop() == Unloop && "cannot skip into nested loops");
+
+      // Get the current nearest parent of the Subloop's exits.
+      L = SubloopParents[L];
+      // L could be Unloop if the only exit was an irreducible backedge.
+    }
+    if (L == Unloop) {
+      continue;
+    }
+    // Handle critical edges from Unloop into a sibling loop.
+    if (L && !L->contains(Unloop)) {
+      L = L->getParentLoop();
+    }
+    // Remember the nearest parent loop among successors or subloop exits.
+    if (NearLoop == Unloop || !NearLoop || NearLoop->contains(L))
+      NearLoop = L;
+  }
+  if (Subloop) {
+    SubloopParents[Subloop] = NearLoop;
+    return BBLoop;
+  }
+  return NearLoop;
+}
+
 //===----------------------------------------------------------------------===//
 // LoopInfo implementation
 //
@@ -391,6 +594,68 @@ bool LoopInfo::runOnFunction(Function &) {
   return false;
 }
 
+/// updateUnloop - The last backedge has been removed from a loop--now the
+/// "unloop". Find a new parent for the blocks contained within unloop and
+/// update the loop tree. We don't necessarily have valid dominators at this
+/// point, but LoopInfo is still valid except for the removal of this loop.
+///
+/// Note that Unloop may now be an empty loop. Calling Loop::getHeader without
+/// checking first is illegal.
+void LoopInfo::updateUnloop(Loop *Unloop) {
+
+  // First handle the special case of no parent loop to simplify the algorithm.
+  if (!Unloop->getParentLoop()) {
+    // Since BBLoop had no parent, Unloop blocks are no longer in a loop.
+    for (Loop::block_iterator I = Unloop->block_begin(),
+         E = Unloop->block_end(); I != E; ++I) {
+
+      // Don't reparent blocks in subloops.
+      if (getLoopFor(*I) != Unloop)
+        continue;
+
+      // Blocks no longer have a parent but are still referenced by Unloop until
+      // the Unloop object is deleted.
+      LI.changeLoopFor(*I, 0);
+    }
+
+    // Remove the loop from the top-level LoopInfo object.
+    for (LoopInfo::iterator I = LI.begin();; ++I) {
+      assert(I != LI.end() && "Couldn't find loop");
+      if (*I == Unloop) {
+        LI.removeLoop(I);
+        break;
+      }
+    }
+
+    // Move all of the subloops to the top-level.
+    while (!Unloop->empty())
+      LI.addTopLevelLoop(Unloop->removeChildLoop(llvm::prior(Unloop->end())));
+
+    return;
+  }
+
+  // Update the parent loop for all blocks within the loop. Blocks within
+  // subloops will not change parents.
+  UnloopUpdater Updater(Unloop, this);
+  Updater.updateBlockParents();
+
+  // Remove blocks from former ancestor loops.
+  Updater.removeBlocksFromAncestors();
+
+  // Add direct subloops as children in their new parent loop.
+  Updater.updateSubloopParents();
+
+  // Remove unloop from its parent loop.
+  Loop *ParentLoop = Unloop->getParentLoop();
+  for (Loop::iterator I = ParentLoop->begin();; ++I) {
+    assert(I != ParentLoop->end() && "Couldn't find loop");
+    if (*I == Unloop) {
+      ParentLoop->removeChildLoop(I);
+      break;
+    }
+  }
+}
+
 void LoopInfo::verifyAnalysis() const {
   // LoopInfo is a FunctionPass, but verifying every loop in the function
   // each time verifyAnalysis is called is very expensive. The
@@ -400,12 +665,21 @@ void LoopInfo::verifyAnalysis() const {
 
   if (!VerifyLoopInfo) return;
 
+  DenseSet<const Loop*> Loops;
   for (iterator I = begin(), E = end(); I != E; ++I) {
     assert(!(*I)->getParentLoop() && "Top-level loop has a parent!");
-    (*I)->verifyLoopNest();
+    (*I)->verifyLoopNest(&Loops);
   }
 
-  // TODO: check BBMap consistency.
+  // Verify that blocks are mapped to valid loops.
+  //
+  // FIXME: With an up-to-date DFS (see LoopIterator.h) and DominatorTree, we
+  // could also verify that the blocks are still in the correct loops.
+  for (DenseMap<BasicBlock*, Loop*>::const_iterator I = LI.BBMap.begin(),
+         E = LI.BBMap.end(); I != E; ++I) {
+    assert(Loops.count(I->second) && "orphaned loop");
+    assert(I->second->contains(I->first) && "orphaned block");
+  }
 }
 
 void LoopInfo::getAnalysisUsage(AnalysisUsage &AU) const {
@@ -417,3 +691,15 @@ void LoopInfo::print(raw_ostream &OS, const Module*) const {
   LI.print(OS);
 }
 
+//===----------------------------------------------------------------------===//
+// LoopBlocksDFS implementation
+//
+
+/// Traverse the loop blocks and store the DFS result.
+/// Useful for clients that just want the final DFS result and don't need to
+/// visit blocks during the initial traversal.
+void LoopBlocksDFS::perform(LoopInfo *LI) {
+  LoopBlocksTraversal Traversal(*this, LI);
+  for (LoopBlocksTraversal::POTIterator POI = Traversal.begin(),
+         POE = Traversal.end(); POI != POE; ++POI) ;
+}
diff --git a/contrib/llvm/lib/Analysis/LoopPass.cpp b/contrib/llvm/lib/Analysis/LoopPass.cpp
index 10e3f297f9c5..5ba1f4045d1a 100644
--- a/contrib/llvm/lib/Analysis/LoopPass.cpp
+++ b/contrib/llvm/lib/Analysis/LoopPass.cpp
@@ -59,9 +59,9 @@ char PrintLoopPass::ID = 0;
 static DebugInfoProbeInfo *TheDebugProbe;
 static void createDebugInfoProbe() {
   if (TheDebugProbe) return;
-      
-  // Constructed the first time this is called. This guarantees that the 
-  // object will be constructed, if -enable-debug-info-probe is set, 
+
+  // Constructed the first time this is called. This guarantees that the
+  // object will be constructed, if -enable-debug-info-probe is set,
   // before static globals, thus it will be destroyed before them.
   static ManagedStatic<DebugInfoProbeInfo> DIP;
   TheDebugProbe = &*DIP;
@@ -73,73 +73,29 @@ static void createDebugInfoProbe() {
 
 char LPPassManager::ID = 0;
 
-LPPassManager::LPPassManager(int Depth) 
-  : FunctionPass(ID), PMDataManager(Depth) { 
+LPPassManager::LPPassManager()
+  : FunctionPass(ID), PMDataManager() {
   skipThisLoop = false;
   redoThisLoop = false;
   LI = NULL;
   CurrentLoop = NULL;
 }
 
-/// Delete loop from the loop queue and loop hierarchy (LoopInfo). 
+/// Delete loop from the loop queue and loop hierarchy (LoopInfo).
 void LPPassManager::deleteLoopFromQueue(Loop *L) {
 
-  if (Loop *ParentLoop = L->getParentLoop()) { // Not a top-level loop.
-    // Reparent all of the blocks in this loop.  Since BBLoop had a parent,
-    // they are now all in it.
-    for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); 
-         I != E; ++I)
-      if (LI->getLoopFor(*I) == L)    // Don't change blocks in subloops.
-        LI->changeLoopFor(*I, ParentLoop);
-    
-    // Remove the loop from its parent loop.
-    for (Loop::iterator I = ParentLoop->begin(), E = ParentLoop->end();;
-         ++I) {
-      assert(I != E && "Couldn't find loop");
-      if (*I == L) {
-        ParentLoop->removeChildLoop(I);
-        break;
-      }
-    }
-    
-    // Move all subloops into the parent loop.
-    while (!L->empty())
-      ParentLoop->addChildLoop(L->removeChildLoop(L->end()-1));
-  } else {
-    // Reparent all of the blocks in this loop.  Since BBLoop had no parent,
-    // they no longer in a loop at all.
-    
-    for (unsigned i = 0; i != L->getBlocks().size(); ++i) {
-      // Don't change blocks in subloops.
-      if (LI->getLoopFor(L->getBlocks()[i]) == L) {
-        LI->removeBlock(L->getBlocks()[i]);
-        --i;
-      }
-    }
-
-    // Remove the loop from the top-level LoopInfo object.
-    for (LoopInfo::iterator I = LI->begin(), E = LI->end();; ++I) {
-      assert(I != E && "Couldn't find loop");
-      if (*I == L) {
-        LI->removeLoop(I);
-        break;
-      }
-    }
-
-    // Move all of the subloops to the top-level.
-    while (!L->empty())
-      LI->addTopLevelLoop(L->removeChildLoop(L->end()-1));
-  }
-
-  delete L;
+  LI->updateUnloop(L);
 
   // If L is current loop then skip rest of the passes and let
   // runOnFunction remove L from LQ. Otherwise, remove L from LQ now
   // and continue applying other passes on CurrentLoop.
-  if (CurrentLoop == L) {
+  if (CurrentLoop == L)
     skipThisLoop = true;
+
+  delete L;
+
+  if (skipThisLoop)
     return;
-  }
 
   for (std::deque<Loop *>::iterator I = LQ.begin(),
          E = LQ.end(); I != E; ++I) {
@@ -166,10 +122,10 @@ void LPPassManager::insertLoop(Loop *L, Loop *ParentLoop) {
 
 void LPPassManager::insertLoopIntoQueue(Loop *L) {
   // Insert L into loop queue
-  if (L == CurrentLoop) 
+  if (L == CurrentLoop)
     redoLoop(L);
   else if (!L->getParentLoop())
-    // This is top level loop. 
+    // This is top level loop.
     LQ.push_front(L);
   else {
     // Insert L after the parent loop.
@@ -195,9 +151,9 @@ void LPPassManager::redoLoop(Loop *L) {
 
 /// cloneBasicBlockSimpleAnalysis - Invoke cloneBasicBlockAnalysis hook for
 /// all loop passes.
-void LPPassManager::cloneBasicBlockSimpleAnalysis(BasicBlock *From, 
+void LPPassManager::cloneBasicBlockSimpleAnalysis(BasicBlock *From,
                                                   BasicBlock *To, Loop *L) {
-  for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {  
+  for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
     LoopPass *LP = getContainedPass(Index);
     LP->cloneBasicBlockAnalysis(From, To, L);
   }
@@ -206,13 +162,13 @@ void LPPassManager::cloneBasicBlockSimpleAnalysis(BasicBlock *From,
 /// deleteSimpleAnalysisValue - Invoke deleteAnalysisValue hook for all passes.
 void LPPassManager::deleteSimpleAnalysisValue(Value *V, Loop *L) {
   if (BasicBlock *BB = dyn_cast<BasicBlock>(V)) {
-    for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); BI != BE; 
+    for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); BI != BE;
          ++BI) {
       Instruction &I = *BI;
       deleteSimpleAnalysisValue(&I, L);
     }
   }
-  for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {  
+  for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
     LoopPass *LP = getContainedPass(Index);
     LP->deleteAnalysisValue(V, L);
   }
@@ -228,7 +184,7 @@ static void addLoopIntoQueue(Loop *L, std::deque<Loop *> &LQ) {
 
 /// Pass Manager itself does not invalidate any analysis info.
 void LPPassManager::getAnalysisUsage(AnalysisUsage &Info) const {
-  // LPPassManager needs LoopInfo. In the long term LoopInfo class will 
+  // LPPassManager needs LoopInfo. In the long term LoopInfo class will
   // become part of LPPassManager.
   Info.addRequired<LoopInfo>();
   Info.setPreservesAll();
@@ -255,7 +211,7 @@ bool LPPassManager::runOnFunction(Function &F) {
   for (std::deque<Loop *>::const_iterator I = LQ.begin(), E = LQ.end();
        I != E; ++I) {
     Loop *L = *I;
-    for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {  
+    for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
       LoopPass *P = getContainedPass(Index);
       Changed |= P->doInitialization(L, *this);
     }
@@ -263,13 +219,13 @@ bool LPPassManager::runOnFunction(Function &F) {
 
   // Walk Loops
   while (!LQ.empty()) {
-      
+
     CurrentLoop  = LQ.back();
     skipThisLoop = false;
     redoThisLoop = false;
 
     // Run all passes on the current Loop.
-    for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {  
+    for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
       LoopPass *P = getContainedPass(Index);
       dumpPassInfo(P, EXECUTION_MSG, ON_LOOP_MSG,
                    CurrentLoop->getHeader()->getName());
@@ -319,23 +275,23 @@ bool LPPassManager::runOnFunction(Function &F) {
         // Do not run other passes on this loop.
         break;
     }
-    
+
     // If the loop was deleted, release all the loop passes. This frees up
     // some memory, and avoids trouble with the pass manager trying to call
     // verifyAnalysis on them.
     if (skipThisLoop)
-      for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {  
+      for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
         Pass *P = getContainedPass(Index);
         freePass(P, "<deleted>", ON_LOOP_MSG);
       }
 
     // Pop the loop from queue after running all passes.
     LQ.pop_back();
-    
+
     if (redoThisLoop)
       LQ.push_back(CurrentLoop);
   }
-  
+
   // Finalization
   for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
     LoopPass *P = getContainedPass(Index);
@@ -372,7 +328,7 @@ Pass *LoopPass::createPrinterPass(raw_ostream &O,
 // LPPassManger as expected.
 void LoopPass::preparePassManager(PMStack &PMS) {
 
-  // Find LPPassManager 
+  // Find LPPassManager
   while (!PMS.empty() &&
          PMS.top()->getPassManagerType() > PMT_LoopPassManager)
     PMS.pop();
@@ -381,14 +337,14 @@ void LoopPass::preparePassManager(PMStack &PMS) {
   // by other passes that are managed by LPM then do not insert
   // this pass in current LPM. Use new LPPassManager.
   if (PMS.top()->getPassManagerType() == PMT_LoopPassManager &&
-      !PMS.top()->preserveHigherLevelAnalysis(this)) 
+      !PMS.top()->preserveHigherLevelAnalysis(this))
     PMS.pop();
 }
 
 /// Assign pass manager to manage this pass.
 void LoopPass::assignPassManager(PMStack &PMS,
                                  PassManagerType PreferredType) {
-  // Find LPPassManager 
+  // Find LPPassManager
   while (!PMS.empty() &&
          PMS.top()->getPassManagerType() > PMT_LoopPassManager)
     PMS.pop();
@@ -397,12 +353,12 @@ void LoopPass::assignPassManager(PMStack &PMS,
   if (PMS.top()->getPassManagerType() == PMT_LoopPassManager)
     LPPM = (LPPassManager*)PMS.top();
   else {
-    // Create new Loop Pass Manager if it does not exist. 
+    // Create new Loop Pass Manager if it does not exist.
     assert (!PMS.empty() && "Unable to create Loop Pass Manager");
     PMDataManager *PMD = PMS.top();
 
-    // [1] Create new Call Graph Pass Manager
-    LPPM = new LPPassManager(PMD->getDepth() + 1);
+    // [1] Create new Loop Pass Manager
+    LPPM = new LPPassManager();
     LPPM->populateInheritedAnalysis(PMS);
 
     // [2] Set up new manager's top level manager
diff --git a/contrib/llvm/lib/Analysis/MemDepPrinter.cpp b/contrib/llvm/lib/Analysis/MemDepPrinter.cpp
index 2283db0bc482..fde07ea4f98d 100644
--- a/contrib/llvm/lib/Analysis/MemDepPrinter.cpp
+++ b/contrib/llvm/lib/Analysis/MemDepPrinter.cpp
@@ -25,8 +25,17 @@ namespace {
   struct MemDepPrinter : public FunctionPass {
     const Function *F;
 
-    typedef PointerIntPair<const Instruction *, 1> InstAndClobberFlag;
-    typedef std::pair<InstAndClobberFlag, const BasicBlock *> Dep;
+    enum DepType {
+      Clobber = 0,
+      Def,
+      NonFuncLocal,
+      Unknown
+    };
+
+    static const char* DepTypeStr[];
+
+    typedef PointerIntPair<const Instruction *, 2, DepType> InstTypePair;
+    typedef std::pair<InstTypePair, const BasicBlock *> Dep;
     typedef SmallSetVector<Dep, 4> DepSet;
     typedef DenseMap<const Instruction *, DepSet> DepSetMap;
     DepSetMap Deps;
@@ -50,6 +59,21 @@ namespace {
       Deps.clear();
       F = 0;
     }
+
+  private:
+    static InstTypePair getInstTypePair(MemDepResult dep) {
+      if (dep.isClobber())
+        return InstTypePair(dep.getInst(), Clobber);
+      if (dep.isDef())
+        return InstTypePair(dep.getInst(), Def);
+      if (dep.isNonFuncLocal())
+        return InstTypePair(dep.getInst(), NonFuncLocal);
+      assert(dep.isUnknown() && "unexptected dependence type");
+      return InstTypePair(dep.getInst(), Unknown);
+    }
+    static InstTypePair getInstTypePair(const Instruction* inst, DepType type) {
+      return InstTypePair(inst, type);
+    }
   };
 }
 
@@ -64,6 +88,9 @@ FunctionPass *llvm::createMemDepPrinter() {
   return new MemDepPrinter();
 }
 
+const char* MemDepPrinter::DepTypeStr[]
+  = {"Clobber", "Def", "NonFuncLocal", "Unknown"};
+
 bool MemDepPrinter::runOnFunction(Function &F) {
   this->F = &F;
   AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
@@ -79,10 +106,7 @@ bool MemDepPrinter::runOnFunction(Function &F) {
 
     MemDepResult Res = MDA.getDependency(Inst);
     if (!Res.isNonLocal()) {
-      assert((Res.isUnknown() || Res.isClobber() || Res.isDef()) &&
-              "Local dep should be unknown, def or clobber!");
-      Deps[Inst].insert(std::make_pair(InstAndClobberFlag(Res.getInst(),
-                                                          Res.isClobber()),
+      Deps[Inst].insert(std::make_pair(getInstTypePair(Res),
                                        static_cast<BasicBlock *>(0)));
     } else if (CallSite CS = cast<Value>(Inst)) {
       const MemoryDependenceAnalysis::NonLocalDepInfo &NLDI =
@@ -92,22 +116,26 @@ bool MemDepPrinter::runOnFunction(Function &F) {
       for (MemoryDependenceAnalysis::NonLocalDepInfo::const_iterator
            I = NLDI.begin(), E = NLDI.end(); I != E; ++I) {
         const MemDepResult &Res = I->getResult();
-        assert((Res.isUnknown() || Res.isClobber() || Res.isDef()) &&
-                "Resolved non-local call dep should be unknown, def or "
-                "clobber!");
-        InstDeps.insert(std::make_pair(InstAndClobberFlag(Res.getInst(),
-                                                          Res.isClobber()),
-                                       I->getBB()));
+        InstDeps.insert(std::make_pair(getInstTypePair(Res), I->getBB()));
       }
     } else {
       SmallVector<NonLocalDepResult, 4> NLDI;
       if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
-        // FIXME: Volatile is not handled properly here.
+        if (!LI->isUnordered()) {
+          // FIXME: Handle atomic/volatile loads.
+          Deps[Inst].insert(std::make_pair(getInstTypePair(0, Unknown),
+                                           static_cast<BasicBlock *>(0)));
+          continue;
+        }
         AliasAnalysis::Location Loc = AA.getLocation(LI);
-        MDA.getNonLocalPointerDependency(Loc, !LI->isVolatile(),
-                                         LI->getParent(), NLDI);
+        MDA.getNonLocalPointerDependency(Loc, true, LI->getParent(), NLDI);
       } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
-        // FIXME: Volatile is not handled properly here.
+        if (!LI->isUnordered()) {
+          // FIXME: Handle atomic/volatile stores.
+          Deps[Inst].insert(std::make_pair(getInstTypePair(0, Unknown),
+                                           static_cast<BasicBlock *>(0)));
+          continue;
+        }
         AliasAnalysis::Location Loc = AA.getLocation(SI);
         MDA.getNonLocalPointerDependency(Loc, false, SI->getParent(), NLDI);
       } else if (VAArgInst *VI = dyn_cast<VAArgInst>(Inst)) {
@@ -121,11 +149,7 @@ bool MemDepPrinter::runOnFunction(Function &F) {
       for (SmallVectorImpl<NonLocalDepResult>::const_iterator
            I = NLDI.begin(), E = NLDI.end(); I != E; ++I) {
         const MemDepResult &Res = I->getResult();
-        assert(Res.isClobber() != Res.isDef() &&
-               "Resolved non-local pointer dep should be def or clobber!");
-        InstDeps.insert(std::make_pair(InstAndClobberFlag(Res.getInst(),
-                                                          Res.isClobber()),
-                                       I->getBB()));
+        InstDeps.insert(std::make_pair(getInstTypePair(Res), I->getBB()));
       }
     }
   }
@@ -146,26 +170,18 @@ void MemDepPrinter::print(raw_ostream &OS, const Module *M) const {
     for (DepSet::const_iterator I = InstDeps.begin(), E = InstDeps.end();
          I != E; ++I) {
       const Instruction *DepInst = I->first.getPointer();
-      bool isClobber = I->first.getInt();
+      DepType type = I->first.getInt();
       const BasicBlock *DepBB = I->second;
 
       OS << "    ";
-      if (!DepInst)
-        OS << "Unknown";
-      else if (isClobber)
-        OS << "Clobber";
-      else
-        OS << "    Def";
+      OS << DepTypeStr[type];
       if (DepBB) {
         OS << " in block ";
         WriteAsOperand(OS, DepBB, /*PrintType=*/false, M);
       }
       if (DepInst) {
         OS << " from: ";
-        if (DepInst == Inst)
-          OS << "<unspecified>";
-        else
-          DepInst->print(OS);
+        DepInst->print(OS);
       }
       OS << "\n";
     }
diff --git a/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp b/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp
index 53d430491198..8d451c46f9b0 100644
--- a/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -47,7 +47,7 @@ static bool isMallocCall(const CallInst *CI) {
   // Check malloc prototype.
   // FIXME: workaround for PR5130, this will be obsolete when a nobuiltin 
   // attribute will exist.
-  const FunctionType *FTy = Callee->getFunctionType();
+  FunctionType *FTy = Callee->getFunctionType();
   if (FTy->getNumParams() != 1)
     return false;
   return FTy->getParamType(0)->isIntegerTy(32) ||
@@ -94,12 +94,12 @@ static Value *computeArraySize(const CallInst *CI, const TargetData *TD,
     return NULL;
 
   // The size of the malloc's result type must be known to determine array size.
-  const Type *T = getMallocAllocatedType(CI);
+  Type *T = getMallocAllocatedType(CI);
   if (!T || !T->isSized() || !TD)
     return NULL;
 
   unsigned ElementSize = TD->getTypeAllocSize(T);
-  if (const StructType *ST = dyn_cast<StructType>(T))
+  if (StructType *ST = dyn_cast<StructType>(T))
     ElementSize = TD->getStructLayout(ST)->getSizeInBytes();
 
   // If malloc call's arg can be determined to be a multiple of ElementSize,
@@ -133,10 +133,10 @@ const CallInst *llvm::isArrayMalloc(const Value *I, const TargetData *TD) {
 ///   0: PointerType is the calls' return type.
 ///   1: PointerType is the bitcast's result type.
 ///  >1: Unique PointerType cannot be determined, return NULL.
-const PointerType *llvm::getMallocType(const CallInst *CI) {
+PointerType *llvm::getMallocType(const CallInst *CI) {
   assert(isMalloc(CI) && "getMallocType and not malloc call");
   
-  const PointerType *MallocType = NULL;
+  PointerType *MallocType = NULL;
   unsigned NumOfBitCastUses = 0;
 
   // Determine if CallInst has a bitcast use.
@@ -164,8 +164,8 @@ const PointerType *llvm::getMallocType(const CallInst *CI) {
 ///   0: PointerType is the malloc calls' return type.
 ///   1: PointerType is the bitcast's result type.
 ///  >1: Unique PointerType cannot be determined, return NULL.
-const Type *llvm::getMallocAllocatedType(const CallInst *CI) {
-  const PointerType *PT = getMallocType(CI);
+Type *llvm::getMallocAllocatedType(const CallInst *CI) {
+  PointerType *PT = getMallocType(CI);
   return PT ? PT->getElementType() : NULL;
 }
 
@@ -201,7 +201,7 @@ const CallInst *llvm::isFreeCall(const Value *I) {
   // Check free prototype.
   // FIXME: workaround for PR5130, this will be obsolete when a nobuiltin 
   // attribute will exist.
-  const FunctionType *FTy = Callee->getFunctionType();
+  FunctionType *FTy = Callee->getFunctionType();
   if (!FTy->getReturnType()->isVoidTy())
     return 0;
   if (FTy->getNumParams() != 1)
diff --git a/contrib/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/contrib/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index bba4482f4da5..92967c08dc21 100644
--- a/contrib/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/contrib/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -120,21 +120,27 @@ AliasAnalysis::ModRefResult GetLocation(const Instruction *Inst,
                                         AliasAnalysis::Location &Loc,
                                         AliasAnalysis *AA) {
   if (const LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
-    if (LI->isVolatile()) {
-      Loc = AliasAnalysis::Location();
+    if (LI->isUnordered()) {
+      Loc = AA->getLocation(LI);
+      return AliasAnalysis::Ref;
+    } else if (LI->getOrdering() == Monotonic) {
+      Loc = AA->getLocation(LI);
       return AliasAnalysis::ModRef;
     }
-    Loc = AA->getLocation(LI);
-    return AliasAnalysis::Ref;
+    Loc = AliasAnalysis::Location();
+    return AliasAnalysis::ModRef;
   }
 
   if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
-    if (SI->isVolatile()) {
-      Loc = AliasAnalysis::Location();
+    if (SI->isUnordered()) {
+      Loc = AA->getLocation(SI);
+      return AliasAnalysis::Mod;
+    } else if (SI->getOrdering() == Monotonic) {
+      Loc = AA->getLocation(SI);
       return AliasAnalysis::ModRef;
     }
-    Loc = AA->getLocation(SI);
-    return AliasAnalysis::Mod;
+    Loc = AliasAnalysis::Location();
+    return AliasAnalysis::ModRef;
   }
 
   if (const VAArgInst *V = dyn_cast<VAArgInst>(Inst)) {
@@ -232,7 +238,7 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall,
   // unknown, otherwise it is non-local.
   if (BB != &BB->getParent()->getEntryBlock())
     return MemDepResult::getNonLocal();
-  return MemDepResult::getUnknown();
+  return MemDepResult::getNonFuncLocal();
 }
 
 /// isLoadLoadClobberIfExtendedToFullWidth - Return true if LI is a load that
@@ -270,8 +276,8 @@ unsigned MemoryDependenceAnalysis::
 getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs,
                                 unsigned MemLocSize, const LoadInst *LI,
                                 const TargetData &TD) {
-  // We can only extend non-volatile integer loads.
-  if (!isa<IntegerType>(LI->getType()) || LI->isVolatile()) return 0;
+  // We can only extend simple integer loads.
+  if (!isa<IntegerType>(LI->getType()) || !LI->isSimple()) return 0;
   
   // Get the base of this load.
   int64_t LIOffs = 0;
@@ -369,6 +375,11 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
     // Values depend on loads if the pointers are must aliased.  This means that
     // a load depends on another must aliased load from the same value.
     if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
+      // Atomic loads have complications involved.
+      // FIXME: This is overly conservative.
+      if (!LI->isUnordered())
+        return MemDepResult::getClobber(LI);
+
       AliasAnalysis::Location LoadLoc = AA->getLocation(LI);
       
       // If we found a pointer, check if it could be the same as our pointer.
@@ -382,7 +393,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
           // location is 1 byte at P+1).  If so, return it as a load/load
           // clobber result, allowing the client to decide to widen the load if
           // it wants to.
-          if (const IntegerType *ITy = dyn_cast<IntegerType>(LI->getType()))
+          if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType()))
             if (LI->getAlignment()*8 > ITy->getPrimitiveSizeInBits() &&
                 isLoadLoadClobberIfExtendedToFullWidth(MemLoc, MemLocBase,
                                                        MemLocOffset, LI, TD))
@@ -424,6 +435,11 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
     }
     
     if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
+      // Atomic stores have complications involved.
+      // FIXME: This is overly conservative.
+      if (!SI->isUnordered())
+        return MemDepResult::getClobber(SI);
+
       // If alias analysis can tell that this store is guaranteed to not modify
       // the query pointer, ignore it.  Use getModRefInfo to handle cases where
       // the query pointer points to constant memory etc.
@@ -483,7 +499,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
   // unknown, otherwise it is non-local.
   if (BB != &BB->getParent()->getEntryBlock())
     return MemDepResult::getNonLocal();
-  return MemDepResult::getUnknown();
+  return MemDepResult::getNonFuncLocal();
 }
 
 /// getDependency - Return the instruction on which a memory operation
@@ -516,7 +532,7 @@ MemDepResult MemoryDependenceAnalysis::getDependency(Instruction *QueryInst) {
     if (QueryParent != &QueryParent->getParent()->getEntryBlock())
       LocalCache = MemDepResult::getNonLocal();
     else
-      LocalCache = MemDepResult::getUnknown();
+      LocalCache = MemDepResult::getNonFuncLocal();
   } else {
     AliasAnalysis::Location MemLoc;
     AliasAnalysis::ModRefResult MR = GetLocation(QueryInst, MemLoc, AA);
@@ -672,7 +688,7 @@ MemoryDependenceAnalysis::getNonLocalCallDependency(CallSite QueryCS) {
       // a clobber, otherwise it is unknown.
       Dep = MemDepResult::getNonLocal();
     } else {
-      Dep = MemDepResult::getUnknown();
+      Dep = MemDepResult::getNonFuncLocal();
     }
     
     // If we had a dirty entry for the block, update it.  Otherwise, just add
@@ -790,7 +806,7 @@ GetNonLocalInfoForBlock(const AliasAnalysis::Location &Loc,
   // If the block has a dependency (i.e. it isn't completely transparent to
   // the value), remember the reverse association because we just added it
   // to Cache!
-  if (Dep.isNonLocal() || Dep.isUnknown())
+  if (!Dep.isDef() && !Dep.isClobber())
     return Dep;
   
   // Keep the ReverseNonLocalPtrDeps map up to date so we can efficiently
diff --git a/contrib/llvm/lib/Analysis/PHITransAddr.cpp b/contrib/llvm/lib/Analysis/PHITransAddr.cpp
index 70dcd0df242d..7e22ddc61c09 100644
--- a/contrib/llvm/lib/Analysis/PHITransAddr.cpp
+++ b/contrib/llvm/lib/Analysis/PHITransAddr.cpp
@@ -228,7 +228,7 @@ Value *PHITransAddr::PHITranslateSubExpr(Value *V, BasicBlock *CurBB,
       return GEP;
 
     // Simplify the GEP to handle 'gep x, 0' -> x etc.
-    if (Value *V = SimplifyGEPInst(&GEPOps[0], GEPOps.size(), TD, DT)) {
+    if (Value *V = SimplifyGEPInst(GEPOps, TD, DT)) {
       for (unsigned i = 0, e = GEPOps.size(); i != e; ++i)
         RemoveInstInputs(GEPOps[i], InstInputs);
 
@@ -407,9 +407,9 @@ InsertPHITranslatedSubExpr(Value *InVal, BasicBlock *CurBB,
     }
 
     GetElementPtrInst *Result =
-    GetElementPtrInst::Create(GEPOps[0], GEPOps.begin()+1, GEPOps.end(),
-                              InVal->getName()+".phi.trans.insert",
-                              PredBB->getTerminator());
+      GetElementPtrInst::Create(GEPOps[0], makeArrayRef(GEPOps).slice(1),
+                                InVal->getName()+".phi.trans.insert",
+                                PredBB->getTerminator());
     Result->setIsInBounds(GEP->isInBounds());
     NewInsts.push_back(Result);
     return Result;
diff --git a/contrib/llvm/lib/Analysis/PathNumbering.cpp b/contrib/llvm/lib/Analysis/PathNumbering.cpp
index 7c584daef734..0e3b6e69ce34 100644
--- a/contrib/llvm/lib/Analysis/PathNumbering.cpp
+++ b/contrib/llvm/lib/Analysis/PathNumbering.cpp
@@ -387,7 +387,7 @@ void BallLarusDag::buildNode(BLBlockNodeMap& inDag, BLNodeStack& dfsStack) {
 
     TerminatorInst* terminator = currentNode->getBlock()->getTerminator();
     if(isa<ReturnInst>(terminator) || isa<UnreachableInst>(terminator)
-       || isa<UnwindInst>(terminator))
+       || isa<ResumeInst>(terminator) || isa<UnwindInst>(terminator))
       addEdge(currentNode, getExit(),0);
 
     currentNode->setColor(BallLarusNode::GRAY);
diff --git a/contrib/llvm/lib/Analysis/RegionPass.cpp b/contrib/llvm/lib/Analysis/RegionPass.cpp
index 80eda79d8a42..3a3529baf93e 100644
--- a/contrib/llvm/lib/Analysis/RegionPass.cpp
+++ b/contrib/llvm/lib/Analysis/RegionPass.cpp
@@ -27,8 +27,8 @@ using namespace llvm;
 
 char RGPassManager::ID = 0;
 
-RGPassManager::RGPassManager(int Depth)
-  : FunctionPass(ID), PMDataManager(Depth) {
+RGPassManager::RGPassManager()
+  : FunctionPass(ID), PMDataManager() {
   skipThisRegion = false;
   redoThisRegion = false;
   RI = NULL;
@@ -250,7 +250,7 @@ void RegionPass::assignPassManager(PMStack &PMS,
     PMDataManager *PMD = PMS.top();
 
     // [1] Create new Region Pass Manager
-    RGPM = new RGPassManager(PMD->getDepth() + 1);
+    RGPM = new RGPassManager();
     RGPM->populateInheritedAnalysis(PMS);
 
     // [2] Set up new manager's top level manager
diff --git a/contrib/llvm/lib/Analysis/ScalarEvolution.cpp b/contrib/llvm/lib/Analysis/ScalarEvolution.cpp
index 025718e09feb..e0ac56c65e76 100644
--- a/contrib/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/contrib/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -197,7 +197,7 @@ void SCEV::print(raw_ostream &OS) const {
   }
   case scUnknown: {
     const SCEVUnknown *U = cast<SCEVUnknown>(this);
-    const Type *AllocTy;
+    Type *AllocTy;
     if (U->isSizeOf(AllocTy)) {
       OS << "sizeof(" << *AllocTy << ")";
       return;
@@ -207,7 +207,7 @@ void SCEV::print(raw_ostream &OS) const {
       return;
     }
 
-    const Type *CTy;
+    Type *CTy;
     Constant *FieldNo;
     if (U->isOffsetOf(CTy, FieldNo)) {
       OS << "offsetof(" << *CTy << ", ";
@@ -228,7 +228,7 @@ void SCEV::print(raw_ostream &OS) const {
   llvm_unreachable("Unknown SCEV kind!");
 }
 
-const Type *SCEV::getType() const {
+Type *SCEV::getType() const {
   switch (getSCEVType()) {
   case scConstant:
     return cast<SCEVConstant>(this)->getType();
@@ -297,17 +297,17 @@ const SCEV *ScalarEvolution::getConstant(const APInt& Val) {
 }
 
 const SCEV *
-ScalarEvolution::getConstant(const Type *Ty, uint64_t V, bool isSigned) {
-  const IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
+ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
+  IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
   return getConstant(ConstantInt::get(ITy, V, isSigned));
 }
 
 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID,
-                           unsigned SCEVTy, const SCEV *op, const Type *ty)
+                           unsigned SCEVTy, const SCEV *op, Type *ty)
   : SCEV(ID, SCEVTy), Op(op), Ty(ty) {}
 
 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
-                                   const SCEV *op, const Type *ty)
+                                   const SCEV *op, Type *ty)
   : SCEVCastExpr(ID, scTruncate, op, ty) {
   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
@@ -315,7 +315,7 @@ SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
 }
 
 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
-                                       const SCEV *op, const Type *ty)
+                                       const SCEV *op, Type *ty)
   : SCEVCastExpr(ID, scZeroExtend, op, ty) {
   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
@@ -323,7 +323,7 @@ SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
 }
 
 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
-                                       const SCEV *op, const Type *ty)
+                                       const SCEV *op, Type *ty)
   : SCEVCastExpr(ID, scSignExtend, op, ty) {
   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
@@ -354,7 +354,7 @@ void SCEVUnknown::allUsesReplacedWith(Value *New) {
   setValPtr(New);
 }
 
-bool SCEVUnknown::isSizeOf(const Type *&AllocTy) const {
+bool SCEVUnknown::isSizeOf(Type *&AllocTy) const {
   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
     if (VCE->getOpcode() == Instruction::PtrToInt)
       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
@@ -371,15 +371,15 @@ bool SCEVUnknown::isSizeOf(const Type *&AllocTy) const {
   return false;
 }
 
-bool SCEVUnknown::isAlignOf(const Type *&AllocTy) const {
+bool SCEVUnknown::isAlignOf(Type *&AllocTy) const {
   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
     if (VCE->getOpcode() == Instruction::PtrToInt)
       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
         if (CE->getOpcode() == Instruction::GetElementPtr &&
             CE->getOperand(0)->isNullValue()) {
-          const Type *Ty =
+          Type *Ty =
             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
-          if (const StructType *STy = dyn_cast<StructType>(Ty))
+          if (StructType *STy = dyn_cast<StructType>(Ty))
             if (!STy->isPacked() &&
                 CE->getNumOperands() == 3 &&
                 CE->getOperand(1)->isNullValue()) {
@@ -396,7 +396,7 @@ bool SCEVUnknown::isAlignOf(const Type *&AllocTy) const {
   return false;
 }
 
-bool SCEVUnknown::isOffsetOf(const Type *&CTy, Constant *&FieldNo) const {
+bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const {
   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
     if (VCE->getOpcode() == Instruction::PtrToInt)
       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
@@ -404,7 +404,7 @@ bool SCEVUnknown::isOffsetOf(const Type *&CTy, Constant *&FieldNo) const {
             CE->getNumOperands() == 3 &&
             CE->getOperand(0)->isNullValue() &&
             CE->getOperand(1)->isNullValue()) {
-          const Type *Ty =
+          Type *Ty =
             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
           // Ignore vector types here so that ScalarEvolutionExpander doesn't
           // emit getelementptrs that index into vectors.
@@ -652,7 +652,7 @@ static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
 /// Assume, K > 0.
 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
                                        ScalarEvolution &SE,
-                                       const Type* ResultTy) {
+                                       Type *ResultTy) {
   // Handle the simplest case efficiently.
   if (K == 1)
     return SE.getTruncateOrZeroExtend(It, ResultTy);
@@ -742,7 +742,7 @@ static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
   MultiplyFactor = MultiplyFactor.trunc(W);
 
   // Calculate the product, at width T+W
-  const IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
+  IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
                                                       CalculationBits);
   const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
   for (unsigned i = 1; i != K; ++i) {
@@ -790,7 +790,7 @@ const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
 //===----------------------------------------------------------------------===//
 
 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
-                                             const Type *Ty) {
+                                             Type *Ty) {
   assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
          "This is not a truncating conversion!");
   assert(isSCEVable(Ty) &&
@@ -877,7 +877,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
 }
 
 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
-                                               const Type *Ty) {
+                                               Type *Ty) {
   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
          "This is not an extending conversion!");
   assert(isSCEVable(Ty) &&
@@ -954,7 +954,7 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
         const SCEV *RecastedMaxBECount =
           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
         if (MaxBECount == RecastedMaxBECount) {
-          const Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
+          Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
           // Check whether Start+Step*MaxBECount has no unsigned overflow.
           const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step);
           const SCEV *Add = getAddExpr(Start, ZMul);
@@ -1062,7 +1062,7 @@ static const SCEV *getOverflowLimitForStep(const SCEV *Step,
 // result, the expression "Step + sext(PreIncAR)" is congruent with
 // "sext(PostIncAR)"
 static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR,
-                                            const Type *Ty,
+                                            Type *Ty,
                                             ScalarEvolution *SE) {
   const Loop *L = AR->getLoop();
   const SCEV *Start = AR->getStart();
@@ -1070,14 +1070,26 @@ static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR,
 
   // Check for a simple looking step prior to loop entry.
   const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
-  if (!SA || SA->getNumOperands() != 2 || SA->getOperand(0) != Step)
+  if (!SA)
+    return 0;
+
+  // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
+  // subtraction is expensive. For this purpose, perform a quick and dirty
+  // difference, by checking for Step in the operand list.
+  SmallVector<const SCEV *, 4> DiffOps;
+  for (SCEVAddExpr::op_iterator I = SA->op_begin(), E = SA->op_end();
+       I != E; ++I) {
+    if (*I != Step)
+      DiffOps.push_back(*I);
+  }
+  if (DiffOps.size() == SA->getNumOperands())
     return 0;
 
   // This is a postinc AR. Check for overflow on the preinc recurrence using the
   // same three conditions that getSignExtendedExpr checks.
 
   // 1. NSW flags on the step increment.
-  const SCEV *PreStart = SA->getOperand(1);
+  const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags());
   const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
     SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
 
@@ -1086,7 +1098,7 @@ static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR,
 
   // 2. Direct overflow check on the step operation's expression.
   unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
-  const Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
+  Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
   const SCEV *OperandExtendedStart =
     SE->getAddExpr(SE->getSignExtendExpr(PreStart, WideTy),
                    SE->getSignExtendExpr(Step, WideTy));
@@ -1112,7 +1124,7 @@ static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR,
 
 // Get the normalized sign-extended expression for this AddRec's Start.
 static const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR,
-                                            const Type *Ty,
+                                            Type *Ty,
                                             ScalarEvolution *SE) {
   const SCEV *PreStart = getPreStartForSignExtend(AR, Ty, SE);
   if (!PreStart)
@@ -1123,7 +1135,7 @@ static const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR,
 }
 
 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
-                                               const Type *Ty) {
+                                               Type *Ty) {
   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
          "This is not an extending conversion!");
   assert(isSCEVable(Ty) &&
@@ -1208,7 +1220,7 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
         const SCEV *RecastedMaxBECount =
           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
         if (MaxBECount == RecastedMaxBECount) {
-          const Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
+          Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
           // Check whether Start+Step*MaxBECount has no signed overflow.
           const SCEV *SMul = getMulExpr(CastedMaxBECount, Step);
           const SCEV *Add = getAddExpr(Start, SMul);
@@ -1275,7 +1287,7 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
 /// unspecified bits out to the given type.
 ///
 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
-                                              const Type *Ty) {
+                                              Type *Ty) {
   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
          "This is not an extending conversion!");
   assert(isSCEVable(Ty) &&
@@ -1438,7 +1450,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
   assert(!Ops.empty() && "Cannot get empty add!");
   if (Ops.size() == 1) return Ops[0];
 #ifndef NDEBUG
-  const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
+  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
            "SCEVAddExpr operand types don't match!");
@@ -1488,7 +1500,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
   // Okay, check to see if the same value occurs in the operand list more than
   // once.  If so, merge them together into an multiply expression.  Since we
   // sorted the list, these values are required to be adjacent.
-  const Type *Ty = Ops[0]->getType();
+  Type *Ty = Ops[0]->getType();
   bool FoundMatch = false;
   for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
     if (Ops[i] == Ops[i+1]) {      //  X + Y + Y  -->  X + Y*2
@@ -1515,8 +1527,8 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
   // if the contents of the resulting outer trunc fold to something simple.
   for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) {
     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]);
-    const Type *DstType = Trunc->getType();
-    const Type *SrcType = Trunc->getOperand()->getType();
+    Type *DstType = Trunc->getType();
+    Type *SrcType = Trunc->getOperand()->getType();
     SmallVector<const SCEV *, 8> LargeOps;
     bool Ok = true;
     // Check all the operands to see if they can be represented in the
@@ -1735,7 +1747,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
       // If all of the other operands were loop invariant, we are done.
       if (Ops.size() == 1) return NewRec;
 
-      // Otherwise, add the folded AddRec by the non-liv parts.
+      // Otherwise, add the folded AddRec by the non-invariant parts.
       for (unsigned i = 0;; ++i)
         if (Ops[i] == AddRec) {
           Ops[i] = NewRec;
@@ -1800,6 +1812,38 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
   return S;
 }
 
+static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
+  uint64_t k = i*j;
+  if (j > 1 && k / j != i) Overflow = true;
+  return k;
+}
+
+/// Compute the result of "n choose k", the binomial coefficient.  If an
+/// intermediate computation overflows, Overflow will be set and the return will
+/// be garbage. Overflow is not cleared on absense of overflow.
+static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
+  // We use the multiplicative formula:
+  //     n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
+  // At each iteration, we take the n-th term of the numeral and divide by the
+  // (k-n)th term of the denominator.  This division will always produce an
+  // integral result, and helps reduce the chance of overflow in the
+  // intermediate computations. However, we can still overflow even when the
+  // final result would fit.
+
+  if (n == 0 || n == k) return 1;
+  if (k > n) return 0;
+
+  if (k > n/2)
+    k = n-k;
+
+  uint64_t r = 1;
+  for (uint64_t i = 1; i <= k; ++i) {
+    r = umul_ov(r, n-(i-1), Overflow);
+    r /= i;
+  }
+  return r;
+}
+
 /// getMulExpr - Get a canonical multiply expression, or something simpler if
 /// possible.
 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
@@ -1809,7 +1853,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
   assert(!Ops.empty() && "Cannot get empty mul!");
   if (Ops.size() == 1) return Ops[0];
 #ifndef NDEBUG
-  const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
+  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
            "SCEVMulExpr operand types don't match!");
@@ -1960,7 +2004,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
       // If all of the other operands were loop invariant, we are done.
       if (Ops.size() == 1) return NewRec;
 
-      // Otherwise, multiply the folded AddRec by the non-liv parts.
+      // Otherwise, multiply the folded AddRec by the non-invariant parts.
       for (unsigned i = 0;; ++i)
         if (Ops[i] == AddRec) {
           Ops[i] = NewRec;
@@ -1974,31 +2018,65 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
     // multiplied together.  If so, we can fold them.
     for (unsigned OtherIdx = Idx+1;
          OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
-         ++OtherIdx)
+         ++OtherIdx) {
       if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
-        // F * G, where F = {A,+,B}<L> and G = {C,+,D}<L>  -->
-        // {A*C,+,F*D + G*B + B*D}<L>
+        // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
+        // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
+        //       choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
+        //   ]]],+,...up to x=2n}.
+        // Note that the arguments to choose() are always integers with values
+        // known at compile time, never SCEV objects.
+        //
+        // The implementation avoids pointless extra computations when the two
+        // addrec's are of different length (mathematically, it's equivalent to
+        // an infinite stream of zeros on the right).
+        bool OpsModified = false;
         for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
              ++OtherIdx)
           if (const SCEVAddRecExpr *OtherAddRec =
                 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]))
             if (OtherAddRec->getLoop() == AddRecLoop) {
-              const SCEVAddRecExpr *F = AddRec, *G = OtherAddRec;
-              const SCEV *NewStart = getMulExpr(F->getStart(), G->getStart());
-              const SCEV *B = F->getStepRecurrence(*this);
-              const SCEV *D = G->getStepRecurrence(*this);
-              const SCEV *NewStep = getAddExpr(getMulExpr(F, D),
-                                               getMulExpr(G, B),
-                                               getMulExpr(B, D));
-              const SCEV *NewAddRec = getAddRecExpr(NewStart, NewStep,
-                                                    F->getLoop(),
-                                                    SCEV::FlagAnyWrap);
-              if (Ops.size() == 2) return NewAddRec;
-              Ops[Idx] = AddRec = cast<SCEVAddRecExpr>(NewAddRec);
-              Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
+              bool Overflow = false;
+              Type *Ty = AddRec->getType();
+              bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
+              SmallVector<const SCEV*, 7> AddRecOps;
+              for (int x = 0, xe = AddRec->getNumOperands() +
+                     OtherAddRec->getNumOperands() - 1;
+                   x != xe && !Overflow; ++x) {
+                const SCEV *Term = getConstant(Ty, 0);
+                for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
+                  uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
+                  for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
+                         ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
+                       z < ze && !Overflow; ++z) {
+                    uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
+                    uint64_t Coeff;
+                    if (LargerThan64Bits)
+                      Coeff = umul_ov(Coeff1, Coeff2, Overflow);
+                    else
+                      Coeff = Coeff1*Coeff2;
+                    const SCEV *CoeffTerm = getConstant(Ty, Coeff);
+                    const SCEV *Term1 = AddRec->getOperand(y-z);
+                    const SCEV *Term2 = OtherAddRec->getOperand(z);
+                    Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2));
+                  }
+                }
+                AddRecOps.push_back(Term);
+              }
+              if (!Overflow) {
+                const SCEV *NewAddRec = getAddRecExpr(AddRecOps,
+                                                      AddRec->getLoop(),
+                                                      SCEV::FlagAnyWrap);
+                if (Ops.size() == 2) return NewAddRec;
+                Ops[Idx] = AddRec = cast<SCEVAddRecExpr>(NewAddRec);
+                Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
+                OpsModified = true;
+              }
             }
-        return getMulExpr(Ops);
+        if (OpsModified)
+          return getMulExpr(Ops);
       }
+    }
 
     // Otherwise couldn't fold anything into this recurrence.  Move onto the
     // next one.
@@ -2042,21 +2120,22 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
       // Determine if the division can be folded into the operands of
       // its operands.
       // TODO: Generalize this to non-constants by using known-bits information.
-      const Type *Ty = LHS->getType();
+      Type *Ty = LHS->getType();
       unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros();
       unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
       // For non-power-of-two values, effectively round the value up to the
       // nearest power of two.
       if (!RHSC->getValue()->getValue().isPowerOf2())
         ++MaxShiftAmt;
-      const IntegerType *ExtTy =
+      IntegerType *ExtTy =
         IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
-      // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
       if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
         if (const SCEVConstant *Step =
-              dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this)))
-          if (!Step->getValue()->getValue()
-                .urem(RHSC->getValue()->getValue()) &&
+            dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
+          // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
+          const APInt &StepInt = Step->getValue()->getValue();
+          const APInt &DivInt = RHSC->getValue()->getValue();
+          if (!StepInt.urem(DivInt) &&
               getZeroExtendExpr(AR, ExtTy) ==
               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
                             getZeroExtendExpr(Step, ExtTy),
@@ -2067,6 +2146,22 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
             return getAddRecExpr(Operands, AR->getLoop(),
                                  SCEV::FlagNW);
           }
+          /// Get a canonical UDivExpr for a recurrence.
+          /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
+          // We can currently only fold X%N if X is constant.
+          const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart());
+          if (StartC && !DivInt.urem(StepInt) &&
+              getZeroExtendExpr(AR, ExtTy) ==
+              getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
+                            getZeroExtendExpr(Step, ExtTy),
+                            AR->getLoop(), SCEV::FlagAnyWrap)) {
+            const APInt &StartInt = StartC->getValue()->getValue();
+            const APInt &StartRem = StartInt.urem(StepInt);
+            if (StartRem != 0)
+              LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step,
+                                  AR->getLoop(), SCEV::FlagNW);
+          }
+        }
       // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
       if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
         SmallVector<const SCEV *, 4> Operands;
@@ -2151,7 +2246,7 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
                                const Loop *L, SCEV::NoWrapFlags Flags) {
   if (Operands.size() == 1) return Operands[0];
 #ifndef NDEBUG
-  const Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
+  Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
   for (unsigned i = 1, e = Operands.size(); i != e; ++i)
     assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy &&
            "SCEVAddRecExpr operand types don't match!");
@@ -2269,7 +2364,7 @@ ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
   assert(!Ops.empty() && "Cannot get empty smax!");
   if (Ops.size() == 1) return Ops[0];
 #ifndef NDEBUG
-  const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
+  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
            "SCEVSMaxExpr operand types don't match!");
@@ -2373,7 +2468,7 @@ ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
   assert(!Ops.empty() && "Cannot get empty umax!");
   if (Ops.size() == 1) return Ops[0];
 #ifndef NDEBUG
-  const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
+  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
            "SCEVUMaxExpr operand types don't match!");
@@ -2476,7 +2571,7 @@ const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS,
   return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
 }
 
-const SCEV *ScalarEvolution::getSizeOfExpr(const Type *AllocTy) {
+const SCEV *ScalarEvolution::getSizeOfExpr(Type *AllocTy) {
   // If we have TargetData, we can bypass creating a target-independent
   // constant expression and then folding it back into a ConstantInt.
   // This is just a compile-time optimization.
@@ -2488,20 +2583,20 @@ const SCEV *ScalarEvolution::getSizeOfExpr(const Type *AllocTy) {
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
     if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
       C = Folded;
-  const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
+  Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
   return getTruncateOrZeroExtend(getSCEV(C), Ty);
 }
 
-const SCEV *ScalarEvolution::getAlignOfExpr(const Type *AllocTy) {
+const SCEV *ScalarEvolution::getAlignOfExpr(Type *AllocTy) {
   Constant *C = ConstantExpr::getAlignOf(AllocTy);
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
     if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
       C = Folded;
-  const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
+  Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
   return getTruncateOrZeroExtend(getSCEV(C), Ty);
 }
 
-const SCEV *ScalarEvolution::getOffsetOfExpr(const StructType *STy,
+const SCEV *ScalarEvolution::getOffsetOfExpr(StructType *STy,
                                              unsigned FieldNo) {
   // If we have TargetData, we can bypass creating a target-independent
   // constant expression and then folding it back into a ConstantInt.
@@ -2514,17 +2609,17 @@ const SCEV *ScalarEvolution::getOffsetOfExpr(const StructType *STy,
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
     if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
       C = Folded;
-  const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy));
+  Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy));
   return getTruncateOrZeroExtend(getSCEV(C), Ty);
 }
 
-const SCEV *ScalarEvolution::getOffsetOfExpr(const Type *CTy,
+const SCEV *ScalarEvolution::getOffsetOfExpr(Type *CTy,
                                              Constant *FieldNo) {
   Constant *C = ConstantExpr::getOffsetOf(CTy, FieldNo);
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
     if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
       C = Folded;
-  const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(CTy));
+  Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(CTy));
   return getTruncateOrZeroExtend(getSCEV(C), Ty);
 }
 
@@ -2558,14 +2653,14 @@ const SCEV *ScalarEvolution::getUnknown(Value *V) {
 /// the SCEV framework. This primarily includes integer types, and it
 /// can optionally include pointer types if the ScalarEvolution class
 /// has access to target-specific information.
-bool ScalarEvolution::isSCEVable(const Type *Ty) const {
+bool ScalarEvolution::isSCEVable(Type *Ty) const {
   // Integers and pointers are always SCEVable.
   return Ty->isIntegerTy() || Ty->isPointerTy();
 }
 
 /// getTypeSizeInBits - Return the size in bits of the specified type,
 /// for which isSCEVable must return true.
-uint64_t ScalarEvolution::getTypeSizeInBits(const Type *Ty) const {
+uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
   assert(isSCEVable(Ty) && "Type is not SCEVable!");
 
   // If we have a TargetData, use it!
@@ -2586,7 +2681,7 @@ uint64_t ScalarEvolution::getTypeSizeInBits(const Type *Ty) const {
 /// the given type and which represents how SCEV will treat the given
 /// type, for which isSCEVable must return true. For pointer types,
 /// this is the pointer-sized integer type.
-const Type *ScalarEvolution::getEffectiveSCEVType(const Type *Ty) const {
+Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
   assert(isSCEVable(Ty) && "Type is not SCEVable!");
 
   if (Ty->isIntegerTy())
@@ -2628,7 +2723,7 @@ const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) {
     return getConstant(
                cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
 
-  const Type *Ty = V->getType();
+  Type *Ty = V->getType();
   Ty = getEffectiveSCEVType(Ty);
   return getMulExpr(V,
                   getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))));
@@ -2640,7 +2735,7 @@ const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
     return getConstant(
                 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
 
-  const Type *Ty = V->getType();
+  Type *Ty = V->getType();
   Ty = getEffectiveSCEVType(Ty);
   const SCEV *AllOnes =
                    getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)));
@@ -2664,8 +2759,8 @@ const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
 /// input value to the specified type.  If the type must be extended, it is zero
 /// extended.
 const SCEV *
-ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, const Type *Ty) {
-  const Type *SrcTy = V->getType();
+ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) {
+  Type *SrcTy = V->getType();
   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
          "Cannot truncate or zero extend with non-integer arguments!");
@@ -2681,8 +2776,8 @@ ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, const Type *Ty) {
 /// extended.
 const SCEV *
 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V,
-                                         const Type *Ty) {
-  const Type *SrcTy = V->getType();
+                                         Type *Ty) {
+  Type *SrcTy = V->getType();
   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
          "Cannot truncate or zero extend with non-integer arguments!");
@@ -2697,8 +2792,8 @@ ScalarEvolution::getTruncateOrSignExtend(const SCEV *V,
 /// input value to the specified type.  If the type must be extended, it is zero
 /// extended.  The conversion must not be narrowing.
 const SCEV *
-ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, const Type *Ty) {
-  const Type *SrcTy = V->getType();
+ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
+  Type *SrcTy = V->getType();
   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
          "Cannot noop or zero extend with non-integer arguments!");
@@ -2713,8 +2808,8 @@ ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, const Type *Ty) {
 /// input value to the specified type.  If the type must be extended, it is sign
 /// extended.  The conversion must not be narrowing.
 const SCEV *
-ScalarEvolution::getNoopOrSignExtend(const SCEV *V, const Type *Ty) {
-  const Type *SrcTy = V->getType();
+ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
+  Type *SrcTy = V->getType();
   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
          "Cannot noop or sign extend with non-integer arguments!");
@@ -2730,8 +2825,8 @@ ScalarEvolution::getNoopOrSignExtend(const SCEV *V, const Type *Ty) {
 /// it is extended with unspecified bits. The conversion must not be
 /// narrowing.
 const SCEV *
-ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, const Type *Ty) {
-  const Type *SrcTy = V->getType();
+ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
+  Type *SrcTy = V->getType();
   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
          "Cannot noop or any extend with non-integer arguments!");
@@ -2745,8 +2840,8 @@ ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, const Type *Ty) {
 /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the
 /// input value to the specified type.  The conversion must not be widening.
 const SCEV *
-ScalarEvolution::getTruncateOrNoop(const SCEV *V, const Type *Ty) {
-  const Type *SrcTy = V->getType();
+ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
+  Type *SrcTy = V->getType();
   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
          "Cannot truncate or noop with non-integer arguments!");
@@ -3032,7 +3127,7 @@ const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
   // context.
   bool isInBounds = GEP->isInBounds();
 
-  const Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
+  Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
   Value *Base = GEP->getOperand(0);
   // Don't attempt to analyze GEPs over unsized objects.
   if (!cast<PointerType>(Base->getType())->getElementType()->isSized())
@@ -3044,7 +3139,7 @@ const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
        I != E; ++I) {
     Value *Index = *I;
     // Compute the (potentially symbolic) offset in bytes for this index.
-    if (const StructType *STy = dyn_cast<StructType>(*GTI++)) {
+    if (StructType *STy = dyn_cast<StructType>(*GTI++)) {
       // For a struct, add the member offset.
       unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
       const SCEV *FieldOffset = getOffsetOfExpr(STy, FieldNo);
@@ -3244,7 +3339,7 @@ ScalarEvolution::getUnsignedRange(const SCEV *S) {
 
     // TODO: non-affine addrec
     if (AddRec->isAffine()) {
-      const Type *Ty = AddRec->getType();
+      Type *Ty = AddRec->getType();
       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
@@ -3396,7 +3491,7 @@ ScalarEvolution::getSignedRange(const SCEV *S) {
 
     // TODO: non-affine addrec
     if (AddRec->isAffine()) {
-      const Type *Ty = AddRec->getType();
+      Type *Ty = AddRec->getType();
       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
@@ -3503,7 +3598,13 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
         AddOps.push_back(Op1);
     }
     AddOps.push_back(getSCEV(U->getOperand(0)));
-    return getAddExpr(AddOps);
+    SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
+    OverflowingBinaryOperator *OBO = cast<OverflowingBinaryOperator>(V);
+    if (OBO->hasNoSignedWrap())
+      setFlags(Flags, SCEV::FlagNSW);
+    if (OBO->hasNoUnsignedWrap())
+      setFlags(Flags, SCEV::FlagNUW);
+    return getAddExpr(AddOps, Flags);
   }
   case Instruction::Mul: {
     // See the Add code above.
@@ -3601,9 +3702,9 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
               LCI->getValue() == CI->getValue())
             if (const SCEVZeroExtendExpr *Z =
                   dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) {
-              const Type *UTy = U->getType();
+              Type *UTy = U->getType();
               const SCEV *Z0 = Z->getOperand();
-              const Type *Z0Ty = Z0->getType();
+              Type *Z0Ty = Z0->getType();
               unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
 
               // If C is a low-bits mask, the zero extend is serving to
@@ -3813,6 +3914,70 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
 //                   Iteration Count Computation Code
 //
 
+/// getSmallConstantTripCount - Returns the maximum trip count of this loop as a
+/// normal unsigned value, if possible. Returns 0 if the trip count is unknown
+/// or not constant. Will also return 0 if the maximum trip count is very large
+/// (>= 2^32)
+unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L,
+                                                    BasicBlock *ExitBlock) {
+  const SCEVConstant *ExitCount =
+    dyn_cast<SCEVConstant>(getExitCount(L, ExitBlock));
+  if (!ExitCount)
+    return 0;
+
+  ConstantInt *ExitConst = ExitCount->getValue();
+
+  // Guard against huge trip counts.
+  if (ExitConst->getValue().getActiveBits() > 32)
+    return 0;
+
+  // In case of integer overflow, this returns 0, which is correct.
+  return ((unsigned)ExitConst->getZExtValue()) + 1;
+}
+
+/// getSmallConstantTripMultiple - Returns the largest constant divisor of the
+/// trip count of this loop as a normal unsigned value, if possible. This
+/// means that the actual trip count is always a multiple of the returned
+/// value (don't forget the trip count could very well be zero as well!).
+///
+/// Returns 1 if the trip count is unknown or not guaranteed to be the
+/// multiple of a constant (which is also the case if the trip count is simply
+/// constant, use getSmallConstantTripCount for that case), Will also return 1
+/// if the trip count is very large (>= 2^32).
+unsigned ScalarEvolution::getSmallConstantTripMultiple(Loop *L,
+                                                       BasicBlock *ExitBlock) {
+  const SCEV *ExitCount = getExitCount(L, ExitBlock);
+  if (ExitCount == getCouldNotCompute())
+    return 1;
+
+  // Get the trip count from the BE count by adding 1.
+  const SCEV *TCMul = getAddExpr(ExitCount,
+                                 getConstant(ExitCount->getType(), 1));
+  // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt
+  // to factor simple cases.
+  if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul))
+    TCMul = Mul->getOperand(0);
+
+  const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul);
+  if (!MulC)
+    return 1;
+
+  ConstantInt *Result = MulC->getValue();
+
+  // Guard against huge trip counts.
+  if (!Result || Result->getValue().getActiveBits() > 32)
+    return 1;
+
+  return (unsigned)Result->getZExtValue();
+}
+
+// getExitCount - Get the expression for the number of loop iterations for which
+// this loop is guaranteed not to exit via ExitintBlock. Otherwise return
+// SCEVCouldNotCompute.
+const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) {
+  return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
+}
+
 /// getBackedgeTakenCount - If the specified loop has a predictable
 /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute
 /// object. The backedge-taken count is the number of times the loop header
@@ -3825,14 +3990,14 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
 /// hasLoopInvariantBackedgeTakenCount).
 ///
 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) {
-  return getBackedgeTakenInfo(L).Exact;
+  return getBackedgeTakenInfo(L).getExact(this);
 }
 
 /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except
 /// return the least SCEV value that is known never to be less than the
 /// actual backedge taken count.
 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) {
-  return getBackedgeTakenInfo(L).Max;
+  return getBackedgeTakenInfo(L).getMax(this);
 }
 
 /// PushLoopPHIs - Push PHI nodes in the header of the given loop
@@ -3849,33 +4014,31 @@ PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) {
 
 const ScalarEvolution::BackedgeTakenInfo &
 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
-  // Initially insert a CouldNotCompute for this loop. If the insertion
+  // Initially insert an invalid entry for this loop. If the insertion
   // succeeds, proceed to actually compute a backedge-taken count and
   // update the value. The temporary CouldNotCompute value tells SCEV
   // code elsewhere that it shouldn't attempt to request a new
   // backedge-taken count, which could result in infinite recursion.
   std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
-    BackedgeTakenCounts.insert(std::make_pair(L, getCouldNotCompute()));
+    BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo()));
   if (!Pair.second)
     return Pair.first->second;
 
-  BackedgeTakenInfo Result = getCouldNotCompute();
-  BackedgeTakenInfo Computed = ComputeBackedgeTakenCount(L);
-  if (Computed.Exact != getCouldNotCompute()) {
-    assert(isLoopInvariant(Computed.Exact, L) &&
-           isLoopInvariant(Computed.Max, L) &&
+  // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it
+  // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
+  // must be cleared in this scope.
+  BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L);
+
+  if (Result.getExact(this) != getCouldNotCompute()) {
+    assert(isLoopInvariant(Result.getExact(this), L) &&
+           isLoopInvariant(Result.getMax(this), L) &&
            "Computed backedge-taken count isn't loop invariant for loop!");
     ++NumTripCountsComputed;
-
-    // Update the value in the map.
-    Result = Computed;
-  } else {
-    if (Computed.Max != getCouldNotCompute())
-      // Update the value in the map.
-      Result = Computed;
-    if (isa<PHINode>(L->getHeader()->begin()))
-      // Only count loops that have phi nodes as not being computable.
-      ++NumTripCountsNotComputed;
+  }
+  else if (Result.getMax(this) == getCouldNotCompute() &&
+           isa<PHINode>(L->getHeader()->begin())) {
+    // Only count loops that have phi nodes as not being computable.
+    ++NumTripCountsNotComputed;
   }
 
   // Now that we know more about the trip count for this loop, forget any
@@ -3883,7 +4046,7 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
   // conservative estimates made without the benefit of trip count
   // information. This is similar to the code in forgetLoop, except that
   // it handles SCEVUnknown PHI nodes specially.
-  if (Computed.hasAnyInfo()) {
+  if (Result.hasAnyInfo()) {
     SmallVector<Instruction *, 16> Worklist;
     PushLoopPHIs(L, Worklist);
 
@@ -3928,7 +4091,12 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
 /// compute a trip count, or if the loop is deleted.
 void ScalarEvolution::forgetLoop(const Loop *L) {
   // Drop any stored trip count value.
-  BackedgeTakenCounts.erase(L);
+  DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos =
+    BackedgeTakenCounts.find(L);
+  if (BTCPos != BackedgeTakenCounts.end()) {
+    BTCPos->second.clear();
+    BackedgeTakenCounts.erase(BTCPos);
+  }
 
   // Drop information about expressions based on loop-header PHIs.
   SmallVector<Instruction *, 16> Worklist;
@@ -3984,6 +4152,85 @@ void ScalarEvolution::forgetValue(Value *V) {
   }
 }
 
+/// getExact - Get the exact loop backedge taken count considering all loop
+/// exits. If all exits are computable, this is the minimum computed count.
+const SCEV *
+ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const {
+  // If any exits were not computable, the loop is not computable.
+  if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute();
+
+  // We need at least one computable exit.
+  if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute();
+  assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info");
+
+  const SCEV *BECount = 0;
+  for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
+       ENT != 0; ENT = ENT->getNextExit()) {
+
+    assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV");
+
+    if (!BECount)
+      BECount = ENT->ExactNotTaken;
+    else
+      BECount = SE->getUMinFromMismatchedTypes(BECount, ENT->ExactNotTaken);
+  }
+  assert(BECount && "Invalid not taken count for loop exit");
+  return BECount;
+}
+
+/// getExact - Get the exact not taken count for this loop exit.
+const SCEV *
+ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock,
+                                             ScalarEvolution *SE) const {
+  for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
+       ENT != 0; ENT = ENT->getNextExit()) {
+
+    if (ENT->ExitingBlock == ExitingBlock)
+      return ENT->ExactNotTaken;
+  }
+  return SE->getCouldNotCompute();
+}
+
+/// getMax - Get the max backedge taken count for the loop.
+const SCEV *
+ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const {
+  return Max ? Max : SE->getCouldNotCompute();
+}
+
+/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
+/// computable exit into a persistent ExitNotTakenInfo array.
+ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
+  SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts,
+  bool Complete, const SCEV *MaxCount) : Max(MaxCount) {
+
+  if (!Complete)
+    ExitNotTaken.setIncomplete();
+
+  unsigned NumExits = ExitCounts.size();
+  if (NumExits == 0) return;
+
+  ExitNotTaken.ExitingBlock = ExitCounts[0].first;
+  ExitNotTaken.ExactNotTaken = ExitCounts[0].second;
+  if (NumExits == 1) return;
+
+  // Handle the rare case of multiple computable exits.
+  ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1];
+
+  ExitNotTakenInfo *PrevENT = &ExitNotTaken;
+  for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) {
+    PrevENT->setNextExit(ENT);
+    ENT->ExitingBlock = ExitCounts[i].first;
+    ENT->ExactNotTaken = ExitCounts[i].second;
+  }
+}
+
+/// clear - Invalidate this result and free the ExitNotTakenInfo array.
+void ScalarEvolution::BackedgeTakenInfo::clear() {
+  ExitNotTaken.ExitingBlock = 0;
+  ExitNotTaken.ExactNotTaken = 0;
+  delete[] ExitNotTaken.getNextExit();
+}
+
 /// ComputeBackedgeTakenCount - Compute the number of times the backedge
 /// of the specified loop will execute.
 ScalarEvolution::BackedgeTakenInfo
@@ -3992,38 +4239,31 @@ ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) {
   L->getExitingBlocks(ExitingBlocks);
 
   // Examine all exits and pick the most conservative values.
-  const SCEV *BECount = getCouldNotCompute();
   const SCEV *MaxBECount = getCouldNotCompute();
-  bool CouldNotComputeBECount = false;
+  bool CouldComputeBECount = true;
+  SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts;
   for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
-    BackedgeTakenInfo NewBTI =
-      ComputeBackedgeTakenCountFromExit(L, ExitingBlocks[i]);
-
-    if (NewBTI.Exact == getCouldNotCompute()) {
+    ExitLimit EL = ComputeExitLimit(L, ExitingBlocks[i]);
+    if (EL.Exact == getCouldNotCompute())
       // We couldn't compute an exact value for this exit, so
       // we won't be able to compute an exact value for the loop.
-      CouldNotComputeBECount = true;
-      BECount = getCouldNotCompute();
-    } else if (!CouldNotComputeBECount) {
-      if (BECount == getCouldNotCompute())
-        BECount = NewBTI.Exact;
-      else
-        BECount = getUMinFromMismatchedTypes(BECount, NewBTI.Exact);
-    }
+      CouldComputeBECount = false;
+    else
+      ExitCounts.push_back(std::make_pair(ExitingBlocks[i], EL.Exact));
+
     if (MaxBECount == getCouldNotCompute())
-      MaxBECount = NewBTI.Max;
-    else if (NewBTI.Max != getCouldNotCompute())
-      MaxBECount = getUMinFromMismatchedTypes(MaxBECount, NewBTI.Max);
+      MaxBECount = EL.Max;
+    else if (EL.Max != getCouldNotCompute())
+      MaxBECount = getUMinFromMismatchedTypes(MaxBECount, EL.Max);
   }
 
-  return BackedgeTakenInfo(BECount, MaxBECount);
+  return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount);
 }
 
-/// ComputeBackedgeTakenCountFromExit - Compute the number of times the backedge
-/// of the specified loop will execute if it exits via the specified block.
-ScalarEvolution::BackedgeTakenInfo
-ScalarEvolution::ComputeBackedgeTakenCountFromExit(const Loop *L,
-                                                   BasicBlock *ExitingBlock) {
+/// ComputeExitLimit - Compute the number of times the backedge of the specified
+/// loop will execute if it exits via the specified block.
+ScalarEvolution::ExitLimit
+ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) {
 
   // Okay, we've chosen an exiting block.  See what condition causes us to
   // exit at this block.
@@ -4081,95 +4321,91 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExit(const Loop *L,
   }
 
   // Proceed to the next level to examine the exit condition expression.
-  return ComputeBackedgeTakenCountFromExitCond(L, ExitBr->getCondition(),
-                                               ExitBr->getSuccessor(0),
-                                               ExitBr->getSuccessor(1));
+  return ComputeExitLimitFromCond(L, ExitBr->getCondition(),
+                                  ExitBr->getSuccessor(0),
+                                  ExitBr->getSuccessor(1));
 }
 
-/// ComputeBackedgeTakenCountFromExitCond - Compute the number of times the
+/// ComputeExitLimitFromCond - Compute the number of times the
 /// backedge of the specified loop will execute if its exit condition
 /// were a conditional branch of ExitCond, TBB, and FBB.
-ScalarEvolution::BackedgeTakenInfo
-ScalarEvolution::ComputeBackedgeTakenCountFromExitCond(const Loop *L,
-                                                       Value *ExitCond,
-                                                       BasicBlock *TBB,
-                                                       BasicBlock *FBB) {
+ScalarEvolution::ExitLimit
+ScalarEvolution::ComputeExitLimitFromCond(const Loop *L,
+                                          Value *ExitCond,
+                                          BasicBlock *TBB,
+                                          BasicBlock *FBB) {
   // Check if the controlling expression for this loop is an And or Or.
   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) {
     if (BO->getOpcode() == Instruction::And) {
       // Recurse on the operands of the and.
-      BackedgeTakenInfo BTI0 =
-        ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(0), TBB, FBB);
-      BackedgeTakenInfo BTI1 =
-        ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(1), TBB, FBB);
+      ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB);
+      ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB);
       const SCEV *BECount = getCouldNotCompute();
       const SCEV *MaxBECount = getCouldNotCompute();
       if (L->contains(TBB)) {
         // Both conditions must be true for the loop to continue executing.
         // Choose the less conservative count.
-        if (BTI0.Exact == getCouldNotCompute() ||
-            BTI1.Exact == getCouldNotCompute())
+        if (EL0.Exact == getCouldNotCompute() ||
+            EL1.Exact == getCouldNotCompute())
           BECount = getCouldNotCompute();
         else
-          BECount = getUMinFromMismatchedTypes(BTI0.Exact, BTI1.Exact);
-        if (BTI0.Max == getCouldNotCompute())
-          MaxBECount = BTI1.Max;
-        else if (BTI1.Max == getCouldNotCompute())
-          MaxBECount = BTI0.Max;
+          BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
+        if (EL0.Max == getCouldNotCompute())
+          MaxBECount = EL1.Max;
+        else if (EL1.Max == getCouldNotCompute())
+          MaxBECount = EL0.Max;
         else
-          MaxBECount = getUMinFromMismatchedTypes(BTI0.Max, BTI1.Max);
+          MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
       } else {
         // Both conditions must be true at the same time for the loop to exit.
         // For now, be conservative.
         assert(L->contains(FBB) && "Loop block has no successor in loop!");
-        if (BTI0.Max == BTI1.Max)
-          MaxBECount = BTI0.Max;
-        if (BTI0.Exact == BTI1.Exact)
-          BECount = BTI0.Exact;
+        if (EL0.Max == EL1.Max)
+          MaxBECount = EL0.Max;
+        if (EL0.Exact == EL1.Exact)
+          BECount = EL0.Exact;
       }
 
-      return BackedgeTakenInfo(BECount, MaxBECount);
+      return ExitLimit(BECount, MaxBECount);
     }
     if (BO->getOpcode() == Instruction::Or) {
       // Recurse on the operands of the or.
-      BackedgeTakenInfo BTI0 =
-        ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(0), TBB, FBB);
-      BackedgeTakenInfo BTI1 =
-        ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(1), TBB, FBB);
+      ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB);
+      ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB);
       const SCEV *BECount = getCouldNotCompute();
       const SCEV *MaxBECount = getCouldNotCompute();
       if (L->contains(FBB)) {
         // Both conditions must be false for the loop to continue executing.
         // Choose the less conservative count.
-        if (BTI0.Exact == getCouldNotCompute() ||
-            BTI1.Exact == getCouldNotCompute())
+        if (EL0.Exact == getCouldNotCompute() ||
+            EL1.Exact == getCouldNotCompute())
           BECount = getCouldNotCompute();
         else
-          BECount = getUMinFromMismatchedTypes(BTI0.Exact, BTI1.Exact);
-        if (BTI0.Max == getCouldNotCompute())
-          MaxBECount = BTI1.Max;
-        else if (BTI1.Max == getCouldNotCompute())
-          MaxBECount = BTI0.Max;
+          BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
+        if (EL0.Max == getCouldNotCompute())
+          MaxBECount = EL1.Max;
+        else if (EL1.Max == getCouldNotCompute())
+          MaxBECount = EL0.Max;
         else
-          MaxBECount = getUMinFromMismatchedTypes(BTI0.Max, BTI1.Max);
+          MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
       } else {
         // Both conditions must be false at the same time for the loop to exit.
         // For now, be conservative.
         assert(L->contains(TBB) && "Loop block has no successor in loop!");
-        if (BTI0.Max == BTI1.Max)
-          MaxBECount = BTI0.Max;
-        if (BTI0.Exact == BTI1.Exact)
-          BECount = BTI0.Exact;
+        if (EL0.Max == EL1.Max)
+          MaxBECount = EL0.Max;
+        if (EL0.Exact == EL1.Exact)
+          BECount = EL0.Exact;
       }
 
-      return BackedgeTakenInfo(BECount, MaxBECount);
+      return ExitLimit(BECount, MaxBECount);
     }
   }
 
   // With an icmp, it may be feasible to compute an exact backedge-taken count.
   // Proceed to the next level to examine the icmp.
   if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond))
-    return ComputeBackedgeTakenCountFromExitCondICmp(L, ExitCondICmp, TBB, FBB);
+    return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB);
 
   // Check for a constant condition. These are normally stripped out by
   // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
@@ -4185,17 +4421,17 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExitCond(const Loop *L,
   }
 
   // If it's not an integer or pointer comparison then compute it the hard way.
-  return ComputeBackedgeTakenCountExhaustively(L, ExitCond, !L->contains(TBB));
+  return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
 }
 
-/// ComputeBackedgeTakenCountFromExitCondICmp - Compute the number of times the
+/// ComputeExitLimitFromICmp - Compute the number of times the
 /// backedge of the specified loop will execute if its exit condition
 /// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB.
-ScalarEvolution::BackedgeTakenInfo
-ScalarEvolution::ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L,
-                                                           ICmpInst *ExitCond,
-                                                           BasicBlock *TBB,
-                                                           BasicBlock *FBB) {
+ScalarEvolution::ExitLimit
+ScalarEvolution::ComputeExitLimitFromICmp(const Loop *L,
+                                          ICmpInst *ExitCond,
+                                          BasicBlock *TBB,
+                                          BasicBlock *FBB) {
 
   // If the condition was exit on true, convert the condition to exit on false
   ICmpInst::Predicate Cond;
@@ -4207,8 +4443,8 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L,
   // Handle common loops like: for (X = "string"; *X; ++X)
   if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0)))
     if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) {
-      BackedgeTakenInfo ItCnt =
-        ComputeLoadConstantCompareBackedgeTakenCount(LI, RHS, L, Cond);
+      ExitLimit ItCnt =
+        ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond);
       if (ItCnt.hasAnyInfo())
         return ItCnt;
     }
@@ -4247,36 +4483,36 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L,
   switch (Cond) {
   case ICmpInst::ICMP_NE: {                     // while (X != Y)
     // Convert to: while (X-Y != 0)
-    BackedgeTakenInfo BTI = HowFarToZero(getMinusSCEV(LHS, RHS), L);
-    if (BTI.hasAnyInfo()) return BTI;
+    ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L);
+    if (EL.hasAnyInfo()) return EL;
     break;
   }
   case ICmpInst::ICMP_EQ: {                     // while (X == Y)
     // Convert to: while (X-Y == 0)
-    BackedgeTakenInfo BTI = HowFarToNonZero(getMinusSCEV(LHS, RHS), L);
-    if (BTI.hasAnyInfo()) return BTI;
+    ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L);
+    if (EL.hasAnyInfo()) return EL;
     break;
   }
   case ICmpInst::ICMP_SLT: {
-    BackedgeTakenInfo BTI = HowManyLessThans(LHS, RHS, L, true);
-    if (BTI.hasAnyInfo()) return BTI;
+    ExitLimit EL = HowManyLessThans(LHS, RHS, L, true);
+    if (EL.hasAnyInfo()) return EL;
     break;
   }
   case ICmpInst::ICMP_SGT: {
-    BackedgeTakenInfo BTI = HowManyLessThans(getNotSCEV(LHS),
+    ExitLimit EL = HowManyLessThans(getNotSCEV(LHS),
                                              getNotSCEV(RHS), L, true);
-    if (BTI.hasAnyInfo()) return BTI;
+    if (EL.hasAnyInfo()) return EL;
     break;
   }
   case ICmpInst::ICMP_ULT: {
-    BackedgeTakenInfo BTI = HowManyLessThans(LHS, RHS, L, false);
-    if (BTI.hasAnyInfo()) return BTI;
+    ExitLimit EL = HowManyLessThans(LHS, RHS, L, false);
+    if (EL.hasAnyInfo()) return EL;
     break;
   }
   case ICmpInst::ICMP_UGT: {
-    BackedgeTakenInfo BTI = HowManyLessThans(getNotSCEV(LHS),
+    ExitLimit EL = HowManyLessThans(getNotSCEV(LHS),
                                              getNotSCEV(RHS), L, false);
-    if (BTI.hasAnyInfo()) return BTI;
+    if (EL.hasAnyInfo()) return EL;
     break;
   }
   default:
@@ -4290,8 +4526,7 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L,
 #endif
     break;
   }
-  return
-    ComputeBackedgeTakenCountExhaustively(L, ExitCond, !L->contains(TBB));
+  return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
 }
 
 static ConstantInt *
@@ -4321,10 +4556,10 @@ GetAddressedElementFromGlobal(GlobalVariable *GV,
       if (Idx >= CA->getNumOperands()) return 0;  // Bogus program
       Init = cast<Constant>(CA->getOperand(Idx));
     } else if (isa<ConstantAggregateZero>(Init)) {
-      if (const StructType *STy = dyn_cast<StructType>(Init->getType())) {
+      if (StructType *STy = dyn_cast<StructType>(Init->getType())) {
         assert(Idx < STy->getNumElements() && "Bad struct index!");
         Init = Constant::getNullValue(STy->getElementType(Idx));
-      } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Init->getType())) {
+      } else if (ArrayType *ATy = dyn_cast<ArrayType>(Init->getType())) {
         if (Idx >= ATy->getNumElements()) return 0;  // Bogus program
         Init = Constant::getNullValue(ATy->getElementType());
       } else {
@@ -4338,15 +4573,16 @@ GetAddressedElementFromGlobal(GlobalVariable *GV,
   return Init;
 }
 
-/// ComputeLoadConstantCompareBackedgeTakenCount - Given an exit condition of
+/// ComputeLoadConstantCompareExitLimit - Given an exit condition of
 /// 'icmp op load X, cst', try to see if we can compute the backedge
 /// execution count.
-ScalarEvolution::BackedgeTakenInfo
-ScalarEvolution::ComputeLoadConstantCompareBackedgeTakenCount(
-                                                LoadInst *LI,
-                                                Constant *RHS,
-                                                const Loop *L,
-                                                ICmpInst::Predicate predicate) {
+ScalarEvolution::ExitLimit
+ScalarEvolution::ComputeLoadConstantCompareExitLimit(
+  LoadInst *LI,
+  Constant *RHS,
+  const Loop *L,
+  ICmpInst::Predicate predicate) {
+
   if (LI->isVolatile()) return getCouldNotCompute();
 
   // Check to see if the loaded pointer is a getelementptr of a global.
@@ -4431,69 +4667,117 @@ static bool CanConstantFold(const Instruction *I) {
   return false;
 }
 
+/// Determine whether this instruction can constant evolve within this loop
+/// assuming its operands can all constant evolve.
+static bool canConstantEvolve(Instruction *I, const Loop *L) {
+  // An instruction outside of the loop can't be derived from a loop PHI.
+  if (!L->contains(I)) return false;
+
+  if (isa<PHINode>(I)) {
+    if (L->getHeader() == I->getParent())
+      return true;
+    else
+      // We don't currently keep track of the control flow needed to evaluate
+      // PHIs, so we cannot handle PHIs inside of loops.
+      return false;
+  }
+
+  // If we won't be able to constant fold this expression even if the operands
+  // are constants, bail early.
+  return CanConstantFold(I);
+}
+
+/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
+/// recursing through each instruction operand until reaching a loop header phi.
+static PHINode *
+getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
+                               DenseMap<Instruction *, PHINode *> &PHIMap) {
+
+  // Otherwise, we can evaluate this instruction if all of its operands are
+  // constant or derived from a PHI node themselves.
+  PHINode *PHI = 0;
+  for (Instruction::op_iterator OpI = UseInst->op_begin(),
+         OpE = UseInst->op_end(); OpI != OpE; ++OpI) {
+
+    if (isa<Constant>(*OpI)) continue;
+
+    Instruction *OpInst = dyn_cast<Instruction>(*OpI);
+    if (!OpInst || !canConstantEvolve(OpInst, L)) return 0;
+
+    PHINode *P = dyn_cast<PHINode>(OpInst);
+    if (!P)
+      // If this operand is already visited, reuse the prior result.
+      // We may have P != PHI if this is the deepest point at which the
+      // inconsistent paths meet.
+      P = PHIMap.lookup(OpInst);
+    if (!P) {
+      // Recurse and memoize the results, whether a phi is found or not.
+      // This recursive call invalidates pointers into PHIMap.
+      P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap);
+      PHIMap[OpInst] = P;
+    }
+    if (P == 0) return 0;        // Not evolving from PHI
+    if (PHI && PHI != P) return 0;  // Evolving from multiple different PHIs.
+    PHI = P;
+  }
+  // This is a expression evolving from a constant PHI!
+  return PHI;
+}
+
 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
 /// in the loop that V is derived from.  We allow arbitrary operations along the
 /// way, but the operands of an operation must either be constants or a value
 /// derived from a constant PHI.  If this expression does not fit with these
 /// constraints, return null.
 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
-  // If this is not an instruction, or if this is an instruction outside of the
-  // loop, it can't be derived from a loop PHI.
   Instruction *I = dyn_cast<Instruction>(V);
-  if (I == 0 || !L->contains(I)) return 0;
+  if (I == 0 || !canConstantEvolve(I, L)) return 0;
 
   if (PHINode *PN = dyn_cast<PHINode>(I)) {
-    if (L->getHeader() == I->getParent())
-      return PN;
-    else
-      // We don't currently keep track of the control flow needed to evaluate
-      // PHIs, so we cannot handle PHIs inside of loops.
-      return 0;
+    return PN;
   }
 
-  // If we won't be able to constant fold this expression even if the operands
-  // are constants, return early.
-  if (!CanConstantFold(I)) return 0;
-
-  // Otherwise, we can evaluate this instruction if all of its operands are
-  // constant or derived from a PHI node themselves.
-  PHINode *PHI = 0;
-  for (unsigned Op = 0, e = I->getNumOperands(); Op != e; ++Op)
-    if (!isa<Constant>(I->getOperand(Op))) {
-      PHINode *P = getConstantEvolvingPHI(I->getOperand(Op), L);
-      if (P == 0) return 0;  // Not evolving from PHI
-      if (PHI == 0)
-        PHI = P;
-      else if (PHI != P)
-        return 0;  // Evolving from multiple different PHIs.
-    }
-
-  // This is a expression evolving from a constant PHI!
-  return PHI;
+  // Record non-constant instructions contained by the loop.
+  DenseMap<Instruction *, PHINode *> PHIMap;
+  return getConstantEvolvingPHIOperands(I, L, PHIMap);
 }
 
 /// EvaluateExpression - Given an expression that passes the
 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
 /// in the loop has the value PHIVal.  If we can't fold this expression for some
 /// reason, return null.
-static Constant *EvaluateExpression(Value *V, Constant *PHIVal,
+static Constant *EvaluateExpression(Value *V, const Loop *L,
+                                    DenseMap<Instruction *, Constant *> &Vals,
                                     const TargetData *TD) {
-  if (isa<PHINode>(V)) return PHIVal;
+  // Convenient constant check, but redundant for recursive calls.
   if (Constant *C = dyn_cast<Constant>(V)) return C;
+
   Instruction *I = cast<Instruction>(V);
+  if (Constant *C = Vals.lookup(I)) return C;
+
+  assert(!isa<PHINode>(I) && "loop header phis should be mapped to constant");
+  assert(canConstantEvolve(I, L) && "cannot evaluate expression in this loop");
+  (void)L;
 
   std::vector<Constant*> Operands(I->getNumOperands());
 
   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
-    Operands[i] = EvaluateExpression(I->getOperand(i), PHIVal, TD);
-    if (Operands[i] == 0) return 0;
+    Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
+    if (!Operand) {
+      Operands[i] = dyn_cast<Constant>(I->getOperand(i));
+      if (!Operands[i]) return 0;
+      continue;
+    }
+    Constant *C = EvaluateExpression(Operand, L, Vals, TD);
+    Vals[Operand] = C;
+    if (!C) return 0;
+    Operands[i] = C;
   }
 
   if (const CmpInst *CI = dyn_cast<CmpInst>(I))
     return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
                                            Operands[1], TD);
-  return ConstantFoldInstOperands(I->getOpcode(), I->getType(),
-                                  &Operands[0], Operands.size(), TD);
+  return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, TD);
 }
 
 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
@@ -4514,6 +4798,9 @@ ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
 
   Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
 
+  // FIXME: Nick's fix for PR11034 will seed constants for multiple header phis.
+  DenseMap<Instruction *, Constant *> CurrentIterVals;
+
   // Since the loop is canonicalized, the PHI node must have two entries.  One
   // entry must be a constant (coming in from outside of the loop), and the
   // second must be derived from the same PHI.
@@ -4522,6 +4809,7 @@ ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
     dyn_cast<Constant>(PN->getIncomingValue(!SecondIsBackedge));
   if (StartCST == 0)
     return RetVal = 0;  // Must be a constant.
+  CurrentIterVals[PN] = StartCST;
 
   Value *BEValue = PN->getIncomingValue(SecondIsBackedge);
   if (getConstantEvolvingPHI(BEValue, L) != PN &&
@@ -4534,29 +4822,31 @@ ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
 
   unsigned NumIterations = BEs.getZExtValue(); // must be in range
   unsigned IterationNum = 0;
-  for (Constant *PHIVal = StartCST; ; ++IterationNum) {
+  for (; ; ++IterationNum) {
     if (IterationNum == NumIterations)
-      return RetVal = PHIVal;  // Got exit value!
+      return RetVal = CurrentIterVals[PN];  // Got exit value!
 
     // Compute the value of the PHI node for the next iteration.
-    Constant *NextPHI = EvaluateExpression(BEValue, PHIVal, TD);
-    if (NextPHI == PHIVal)
+    // EvaluateExpression adds non-phi values to the CurrentIterVals map.
+    Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD);
+    if (NextPHI == CurrentIterVals[PN])
       return RetVal = NextPHI;  // Stopped evolving!
     if (NextPHI == 0)
       return 0;        // Couldn't evaluate!
-    PHIVal = NextPHI;
+    DenseMap<Instruction *, Constant *> NextIterVals;
+    NextIterVals[PN] = NextPHI;
+    CurrentIterVals.swap(NextIterVals);
   }
 }
 
-/// ComputeBackedgeTakenCountExhaustively - If the loop is known to execute a
+/// ComputeExitCountExhaustively - If the loop is known to execute a
 /// constant number of times (the condition evolves only from constants),
 /// try to evaluate a few iterations of the loop until we get the exit
 /// condition gets a value of ExitWhen (true or false).  If we cannot
 /// evaluate the trip count of the loop, return getCouldNotCompute().
-const SCEV *
-ScalarEvolution::ComputeBackedgeTakenCountExhaustively(const Loop *L,
-                                                       Value *Cond,
-                                                       bool ExitWhen) {
+const SCEV * ScalarEvolution::ComputeExitCountExhaustively(const Loop *L,
+                                                           Value *Cond,
+                                                           bool ExitWhen) {
   PHINode *PN = getConstantEvolvingPHI(Cond, L);
   if (PN == 0) return getCouldNotCompute();
 
@@ -4583,8 +4873,10 @@ ScalarEvolution::ComputeBackedgeTakenCountExhaustively(const Loop *L,
   unsigned MaxIterations = MaxBruteForceIterations;   // Limit analysis.
   for (Constant *PHIVal = StartCST;
        IterationNum != MaxIterations; ++IterationNum) {
+    DenseMap<Instruction *, Constant *> PHIValMap;
+    PHIValMap[PN] = PHIVal;
     ConstantInt *CondVal =
-      dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, PHIVal, TD));
+      dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, PHIValMap, TD));
 
     // Couldn't symbolically evaluate.
     if (!CondVal) return getCouldNotCompute();
@@ -4595,7 +4887,7 @@ ScalarEvolution::ComputeBackedgeTakenCountExhaustively(const Loop *L,
     }
 
     // Compute the value of the PHI node for the next iteration.
-    Constant *NextPHI = EvaluateExpression(BEValue, PHIVal, TD);
+    Constant *NextPHI = EvaluateExpression(BEValue, L, PHIValMap, TD);
     if (NextPHI == 0 || NextPHI == PHIVal)
       return getCouldNotCompute();// Couldn't evaluate or not making progress...
     PHIVal = NextPHI;
@@ -4703,7 +4995,7 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
                                                 Operands[0], Operands[1], TD);
           else
             C = ConstantFoldInstOperands(I->getOpcode(), I->getType(),
-                                         &Operands[0], Operands.size(), TD);
+                                         Operands, TD);
           if (!C) return V;
           return getSCEV(C);
         }
@@ -4925,7 +5217,7 @@ SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
     // Compute the two solutions for the quadratic formula.
     // The divisions must be performed as signed divisions.
     APInt NegB(-B);
-    APInt TwoA( A << 1 );
+    APInt TwoA(A << 1);
     if (TwoA.isMinValue()) {
       const SCEV *CNC = SE.getCouldNotCompute();
       return std::make_pair(CNC, CNC);
@@ -4940,7 +5232,7 @@ SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
 
     return std::make_pair(SE.getConstant(Solution1),
                           SE.getConstant(Solution2));
-    } // end APIntOps namespace
+  } // end APIntOps namespace
 }
 
 /// HowFarToZero - Return the number of times a backedge comparing the specified
@@ -4950,7 +5242,7 @@ SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
 /// now expressed as a single expression, V = x-y. So the exit test is
 /// effectively V != 0.  We know and take advantage of the fact that this
 /// expression only being used in a comparison by zero context.
-ScalarEvolution::BackedgeTakenInfo
+ScalarEvolution::ExitLimit
 ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) {
   // If the value is a constant
   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
@@ -5034,8 +5326,19 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) {
   // Handle unitary steps, which cannot wraparound.
   // 1*N = -Start; -1*N = Start (mod 2^BW), so:
   //   N = Distance (as unsigned)
-  if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue())
-    return Distance;
+  if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) {
+    ConstantRange CR = getUnsignedRange(Start);
+    const SCEV *MaxBECount;
+    if (!CountDown && CR.getUnsignedMin().isMinValue())
+      // When counting up, the worst starting value is 1, not 0.
+      MaxBECount = CR.getUnsignedMax().isMinValue()
+        ? getConstant(APInt::getMinValue(CR.getBitWidth()))
+        : getConstant(APInt::getMaxValue(CR.getBitWidth()));
+    else
+      MaxBECount = getConstant(CountDown ? CR.getUnsignedMax()
+                                         : -CR.getUnsignedMin());
+    return ExitLimit(Distance, MaxBECount);
+  }
 
   // If the recurrence is known not to wraparound, unsigned divide computes the
   // back edge count. We know that the value will either become zero (and thus
@@ -5062,7 +5365,7 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) {
 /// HowFarToNonZero - Return the number of times a backedge checking the
 /// specified value for nonzero will execute.  If not computable, return
 /// CouldNotCompute
-ScalarEvolution::BackedgeTakenInfo
+ScalarEvolution::ExitLimit
 ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) {
   // Loops that look like: while (X == 0) are very strange indeed.  We don't
   // handle them yet except for the trivial case.  This could be expanded in the
@@ -5741,7 +6044,7 @@ const SCEV *ScalarEvolution::getBECount(const SCEV *Start,
   assert(!isKnownNegative(Step) &&
          "This code doesn't handle negative strides yet!");
 
-  const Type *Ty = Start->getType();
+  Type *Ty = Start->getType();
 
   // When Start == End, we have an exact BECount == 0. Short-circuit this case
   // here because SCEV may not be able to determine that the unsigned division
@@ -5760,7 +6063,7 @@ const SCEV *ScalarEvolution::getBECount(const SCEV *Start,
   if (!NoWrap) {
     // Check Add for unsigned overflow.
     // TODO: More sophisticated things could be done here.
-    const Type *WideTy = IntegerType::get(getContext(),
+    Type *WideTy = IntegerType::get(getContext(),
                                           getTypeSizeInBits(Ty) + 1);
     const SCEV *EDiff = getZeroExtendExpr(Diff, WideTy);
     const SCEV *ERoundUp = getZeroExtendExpr(RoundUp, WideTy);
@@ -5775,7 +6078,7 @@ const SCEV *ScalarEvolution::getBECount(const SCEV *Start,
 /// HowManyLessThans - Return the number of times a backedge containing the
 /// specified less-than comparison will execute.  If not computable, return
 /// CouldNotCompute.
-ScalarEvolution::BackedgeTakenInfo
+ScalarEvolution::ExitLimit
 ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS,
                                   const Loop *L, bool isSigned) {
   // Only handle:  "ADDREC < LoopInvariant".
@@ -5882,7 +6185,7 @@ ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS,
     if (isa<SCEVCouldNotCompute>(MaxBECount))
       MaxBECount = BECount;
 
-    return BackedgeTakenInfo(BECount, MaxBECount);
+    return ExitLimit(BECount, MaxBECount);
   }
 
   return getCouldNotCompute();
@@ -6090,6 +6393,15 @@ void ScalarEvolution::releaseMemory() {
   FirstUnknown = 0;
 
   ValueExprMap.clear();
+
+  // Free any extra memory created for ExitNotTakenInfo in the unlikely event
+  // that a loop had multiple computable exits.
+  for (DenseMap<const Loop*, BackedgeTakenInfo>::iterator I =
+         BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end();
+       I != E; ++I) {
+    I->second.clear();
+  }
+
   BackedgeTakenCounts.clear();
   ConstantEvolutionLoopExitValue.clear();
   ValuesAtScopes.clear();
diff --git a/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp b/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
index befe6d2599d6..47f0f321161b 100644
--- a/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
+++ b/contrib/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
@@ -17,6 +17,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/LLVMContext.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/ADT/STLExtras.h"
 
@@ -26,7 +27,7 @@ using namespace llvm;
 /// reusing an existing cast if a suitable one exists, moving an existing
 /// cast if a suitable one exists but isn't in the right place, or
 /// creating a new one.
-Value *SCEVExpander::ReuseOrCreateCast(Value *V, const Type *Ty,
+Value *SCEVExpander::ReuseOrCreateCast(Value *V, Type *Ty,
                                        Instruction::CastOps Op,
                                        BasicBlock::iterator IP) {
   // Check to see if there is already a cast!
@@ -62,7 +63,7 @@ Value *SCEVExpander::ReuseOrCreateCast(Value *V, const Type *Ty,
 /// InsertNoopCastOfTo - Insert a cast of V to the specified type,
 /// which must be possible with a noop cast, doing what we can to share
 /// the casts.
-Value *SCEVExpander::InsertNoopCastOfTo(Value *V, const Type *Ty) {
+Value *SCEVExpander::InsertNoopCastOfTo(Value *V, Type *Ty) {
   Instruction::CastOps Op = CastInst::getCastOpcode(V, false, Ty, false);
   assert((Op == Instruction::BitCast ||
           Op == Instruction::PtrToInt ||
@@ -103,7 +104,8 @@ Value *SCEVExpander::InsertNoopCastOfTo(Value *V, const Type *Ty) {
     while ((isa<BitCastInst>(IP) &&
             isa<Argument>(cast<BitCastInst>(IP)->getOperand(0)) &&
             cast<BitCastInst>(IP)->getOperand(0) != A) ||
-           isa<DbgInfoIntrinsic>(IP))
+           isa<DbgInfoIntrinsic>(IP) ||
+           isa<LandingPadInst>(IP))
       ++IP;
     return ReuseOrCreateCast(A, Ty, Op, IP);
   }
@@ -113,7 +115,9 @@ Value *SCEVExpander::InsertNoopCastOfTo(Value *V, const Type *Ty) {
   BasicBlock::iterator IP = I; ++IP;
   if (InvokeInst *II = dyn_cast<InvokeInst>(I))
     IP = II->getNormalDest()->begin();
-  while (isa<PHINode>(IP) || isa<DbgInfoIntrinsic>(IP)) ++IP;
+  while (isa<PHINode>(IP) || isa<DbgInfoIntrinsic>(IP) ||
+         isa<LandingPadInst>(IP))
+    ++IP;
   return ReuseOrCreateCast(I, Ty, Op, IP);
 }
 
@@ -160,7 +164,7 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode,
   }
 
   // If we haven't found this binop, insert it.
-  Instruction *BO = cast<Instruction>(Builder.CreateBinOp(Opcode, LHS, RHS, "tmp"));
+  Instruction *BO = cast<Instruction>(Builder.CreateBinOp(Opcode, LHS, RHS));
   BO->setDebugLoc(SaveInsertPt->getDebugLoc());
   rememberInstruction(BO);
 
@@ -277,7 +281,7 @@ static bool FactorOutConstant(const SCEV *&S,
 /// the list.
 ///
 static void SimplifyAddOperands(SmallVectorImpl<const SCEV *> &Ops,
-                                const Type *Ty,
+                                Type *Ty,
                                 ScalarEvolution &SE) {
   unsigned NumAddRecs = 0;
   for (unsigned i = Ops.size(); i > 0 && isa<SCEVAddRecExpr>(Ops[i-1]); --i)
@@ -306,7 +310,7 @@ static void SimplifyAddOperands(SmallVectorImpl<const SCEV *> &Ops,
 /// into GEP indices.
 ///
 static void SplitAddRecs(SmallVectorImpl<const SCEV *> &Ops,
-                         const Type *Ty,
+                         Type *Ty,
                          ScalarEvolution &SE) {
   // Find the addrecs.
   SmallVector<const SCEV *, 8> AddRecs;
@@ -365,10 +369,10 @@ static void SplitAddRecs(SmallVectorImpl<const SCEV *> &Ops,
 ///
 Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
                                     const SCEV *const *op_end,
-                                    const PointerType *PTy,
-                                    const Type *Ty,
+                                    PointerType *PTy,
+                                    Type *Ty,
                                     Value *V) {
-  const Type *ElTy = PTy->getElementType();
+  Type *ElTy = PTy->getElementType();
   SmallVector<Value *, 4> GepIndices;
   SmallVector<const SCEV *, 8> Ops(op_begin, op_end);
   bool AnyNonZeroIndices = false;
@@ -423,7 +427,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
     GepIndices.push_back(Scaled);
 
     // Collect struct field index operands.
-    while (const StructType *STy = dyn_cast<StructType>(ElTy)) {
+    while (StructType *STy = dyn_cast<StructType>(ElTy)) {
       bool FoundFieldNo = false;
       // An empty struct has no fields.
       if (STy->getNumElements() == 0) break;
@@ -451,7 +455,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
         // appropriate struct type.
         for (unsigned i = 0, e = Ops.size(); i != e; ++i)
           if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Ops[i])) {
-            const Type *CTy;
+            Type *CTy;
             Constant *FieldNo;
             if (U->isOffsetOf(CTy, FieldNo) && CTy == STy) {
               GepIndices.push_back(FieldNo);
@@ -474,7 +478,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
       }
     }
 
-    if (const ArrayType *ATy = dyn_cast<ArrayType>(ElTy))
+    if (ArrayType *ATy = dyn_cast<ArrayType>(ElTy))
       ElTy = ATy->getElementType();
     else
       break;
@@ -494,7 +498,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
     // Fold a GEP with constant operands.
     if (Constant *CLHS = dyn_cast<Constant>(V))
       if (Constant *CRHS = dyn_cast<Constant>(Idx))
-        return ConstantExpr::getGetElementPtr(CLHS, &CRHS, 1);
+        return ConstantExpr::getGetElementPtr(CLHS, CRHS);
 
     // Do a quick scan to see if we have this GEP nearby.  If so, reuse it.
     unsigned ScanLimit = 6;
@@ -572,8 +576,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
   if (V->getType() != PTy)
     Casted = InsertNoopCastOfTo(Casted, PTy);
   Value *GEP = Builder.CreateGEP(Casted,
-                                 GepIndices.begin(),
-                                 GepIndices.end(),
+                                 GepIndices,
                                  "scevgep");
   Ops.push_back(SE.getUnknown(GEP));
   rememberInstruction(GEP);
@@ -691,7 +694,7 @@ class LoopCompare {
 }
 
 Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
 
   // Collect all the add operands in a loop, along with their associated loops.
   // Iterate in reverse so that constants are emitted last, all else equal, and
@@ -717,7 +720,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
       // This is the first operand. Just expand it.
       Sum = expand(Op);
       ++I;
-    } else if (const PointerType *PTy = dyn_cast<PointerType>(Sum->getType())) {
+    } else if (PointerType *PTy = dyn_cast<PointerType>(Sum->getType())) {
       // The running sum expression is a pointer. Try to form a getelementptr
       // at this level with that as the base.
       SmallVector<const SCEV *, 4> NewOps;
@@ -731,7 +734,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
         NewOps.push_back(X);
       }
       Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, Sum);
-    } else if (const PointerType *PTy = dyn_cast<PointerType>(Op->getType())) {
+    } else if (PointerType *PTy = dyn_cast<PointerType>(Op->getType())) {
       // The running sum is an integer, and there's a pointer at this level.
       // Try to form a getelementptr. If the running sum is instructions,
       // use a SCEVUnknown to avoid re-analyzing them.
@@ -762,7 +765,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
 }
 
 Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
 
   // Collect all the mul operands in a loop, along with their associated loops.
   // Iterate in reverse so that constants are emitted last, all else equal.
@@ -804,7 +807,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 }
 
 Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
 
   Value *LHS = expandCodeFor(S->getLHS(), Ty);
   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getRHS())) {
@@ -841,81 +844,141 @@ static void ExposePointerBase(const SCEV *&Base, const SCEV *&Rest,
   }
 }
 
+/// Determine if this is a well-behaved chain of instructions leading back to
+/// the PHI. If so, it may be reused by expanded expressions.
+bool SCEVExpander::isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV,
+                                         const Loop *L) {
+  if (IncV->getNumOperands() == 0 || isa<PHINode>(IncV) ||
+      (isa<CastInst>(IncV) && !isa<BitCastInst>(IncV)))
+    return false;
+  // If any of the operands don't dominate the insert position, bail.
+  // Addrec operands are always loop-invariant, so this can only happen
+  // if there are instructions which haven't been hoisted.
+  if (L == IVIncInsertLoop) {
+    for (User::op_iterator OI = IncV->op_begin()+1,
+           OE = IncV->op_end(); OI != OE; ++OI)
+      if (Instruction *OInst = dyn_cast<Instruction>(OI))
+        if (!SE.DT->dominates(OInst, IVIncInsertPos))
+          return false;
+  }
+  // Advance to the next instruction.
+  IncV = dyn_cast<Instruction>(IncV->getOperand(0));
+  if (!IncV)
+    return false;
+
+  if (IncV->mayHaveSideEffects())
+    return false;
+
+  if (IncV != PN)
+    return true;
+
+  return isNormalAddRecExprPHI(PN, IncV, L);
+}
+
+/// Determine if this cyclic phi is in a form that would have been generated by
+/// LSR. We don't care if the phi was actually expanded in this pass, as long
+/// as it is in a low-cost form, for example, no implied multiplication. This
+/// should match any patterns generated by getAddRecExprPHILiterally and
+/// expandAddtoGEP.
+bool SCEVExpander::isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV,
+                                           const Loop *L) {
+  switch (IncV->getOpcode()) {
+  // Check for a simple Add/Sub or GEP of a loop invariant step.
+  case Instruction::Add:
+  case Instruction::Sub:
+    return IncV->getOperand(0) == PN
+      && L->isLoopInvariant(IncV->getOperand(1));
+  case Instruction::BitCast:
+    IncV = dyn_cast<GetElementPtrInst>(IncV->getOperand(0));
+    if (!IncV)
+      return false;
+    // fall-thru to GEP handling
+  case Instruction::GetElementPtr: {
+    // This must be a pointer addition of constants (pretty) or some number of
+    // address-size elements (ugly).
+    for (Instruction::op_iterator I = IncV->op_begin()+1, E = IncV->op_end();
+         I != E; ++I) {
+      if (isa<Constant>(*I))
+        continue;
+      // ugly geps have 2 operands.
+      // i1* is used by the expander to represent an address-size element.
+      if (IncV->getNumOperands() != 2)
+        return false;
+      unsigned AS = cast<PointerType>(IncV->getType())->getAddressSpace();
+      if (IncV->getType() != Type::getInt1PtrTy(SE.getContext(), AS)
+          && IncV->getType() != Type::getInt8PtrTy(SE.getContext(), AS))
+        return false;
+      // Ensure the operands dominate the insertion point. I don't know of a
+      // case when this would not be true, so this is somewhat untested.
+      if (L == IVIncInsertLoop) {
+        for (User::op_iterator OI = IncV->op_begin()+1,
+               OE = IncV->op_end(); OI != OE; ++OI)
+          if (Instruction *OInst = dyn_cast<Instruction>(OI))
+            if (!SE.DT->dominates(OInst, IVIncInsertPos))
+              return false;
+      }
+      break;
+    }
+    IncV = dyn_cast<Instruction>(IncV->getOperand(0));
+    if (IncV && IncV->getOpcode() == Instruction::BitCast)
+      IncV = dyn_cast<Instruction>(IncV->getOperand(0));
+    return IncV == PN;
+  }
+  default:
+    return false;
+  }
+}
+
 /// getAddRecExprPHILiterally - Helper for expandAddRecExprLiterally. Expand
 /// the base addrec, which is the addrec without any non-loop-dominating
 /// values, and return the PHI.
 PHINode *
 SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
                                         const Loop *L,
-                                        const Type *ExpandTy,
-                                        const Type *IntTy) {
+                                        Type *ExpandTy,
+                                        Type *IntTy) {
   assert((!IVIncInsertLoop||IVIncInsertPos) && "Uninitialized insert position");
 
   // Reuse a previously-inserted PHI, if present.
-  for (BasicBlock::iterator I = L->getHeader()->begin();
-       PHINode *PN = dyn_cast<PHINode>(I); ++I)
-    if (SE.isSCEVable(PN->getType()) &&
-        (SE.getEffectiveSCEVType(PN->getType()) ==
-         SE.getEffectiveSCEVType(Normalized->getType())) &&
-        SE.getSCEV(PN) == Normalized)
-      if (BasicBlock *LatchBlock = L->getLoopLatch()) {
-        Instruction *IncV =
-          cast<Instruction>(PN->getIncomingValueForBlock(LatchBlock));
+  BasicBlock *LatchBlock = L->getLoopLatch();
+  if (LatchBlock) {
+    for (BasicBlock::iterator I = L->getHeader()->begin();
+         PHINode *PN = dyn_cast<PHINode>(I); ++I) {
+      if (!SE.isSCEVable(PN->getType()) ||
+          (SE.getEffectiveSCEVType(PN->getType()) !=
+           SE.getEffectiveSCEVType(Normalized->getType())) ||
+          SE.getSCEV(PN) != Normalized)
+        continue;
 
-        // Determine if this is a well-behaved chain of instructions leading
-        // back to the PHI. It probably will be, if we're scanning an inner
-        // loop already visited by LSR for example, but it wouldn't have
-        // to be.
-        do {
-          if (IncV->getNumOperands() == 0 || isa<PHINode>(IncV) ||
-              (isa<CastInst>(IncV) && !isa<BitCastInst>(IncV))) {
-            IncV = 0;
-            break;
-          }
-          // If any of the operands don't dominate the insert position, bail.
-          // Addrec operands are always loop-invariant, so this can only happen
-          // if there are instructions which haven't been hoisted.
-          if (L == IVIncInsertLoop) {
-            for (User::op_iterator OI = IncV->op_begin()+1,
-                   OE = IncV->op_end(); OI != OE; ++OI)
-              if (Instruction *OInst = dyn_cast<Instruction>(OI))
-                if (!SE.DT->dominates(OInst, IVIncInsertPos)) {
-                  IncV = 0;
-                  break;
-                }
-          }
-          if (!IncV)
-            break;
-          // Advance to the next instruction.
-          IncV = dyn_cast<Instruction>(IncV->getOperand(0));
-          if (!IncV)
-            break;
-          if (IncV->mayHaveSideEffects()) {
-            IncV = 0;
-            break;
-          }
-        } while (IncV != PN);
+      Instruction *IncV =
+        cast<Instruction>(PN->getIncomingValueForBlock(LatchBlock));
 
-        if (IncV) {
-          // Ok, the add recurrence looks usable.
-          // Remember this PHI, even in post-inc mode.
-          InsertedValues.insert(PN);
-          // Remember the increment.
-          IncV = cast<Instruction>(PN->getIncomingValueForBlock(LatchBlock));
-          rememberInstruction(IncV);
-          if (L == IVIncInsertLoop)
-            do {
-              if (SE.DT->dominates(IncV, IVIncInsertPos))
-                break;
-              // Make sure the increment is where we want it. But don't move it
-              // down past a potential existing post-inc user.
-              IncV->moveBefore(IVIncInsertPos);
-              IVIncInsertPos = IncV;
-              IncV = cast<Instruction>(IncV->getOperand(0));
-            } while (IncV != PN);
-          return PN;
-        }
+      if (LSRMode) {
+        if (!isExpandedAddRecExprPHI(PN, IncV, L))
+          continue;
       }
+      else {
+        if (!isNormalAddRecExprPHI(PN, IncV, L))
+          continue;
+      }
+      // Ok, the add recurrence looks usable.
+      // Remember this PHI, even in post-inc mode.
+      InsertedValues.insert(PN);
+      // Remember the increment.
+      rememberInstruction(IncV);
+      if (L == IVIncInsertLoop)
+        do {
+          if (SE.DT->dominates(IncV, IVIncInsertPos))
+            break;
+          // Make sure the increment is where we want it. But don't move it
+          // down past a potential existing post-inc user.
+          IncV->moveBefore(IVIncInsertPos);
+          IVIncInsertPos = IncV;
+          IncV = cast<Instruction>(IncV->getOperand(0));
+        } while (IncV != PN);
+      return PN;
+    }
+  }
 
   // Save the original insertion point so we can restore it when we're done.
   BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
@@ -969,7 +1032,7 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
     Value *IncV;
     // If the PHI is a pointer, use a GEP, otherwise use an add or sub.
     if (isPointer) {
-      const PointerType *GEPPtrTy = cast<PointerType>(ExpandTy);
+      PointerType *GEPPtrTy = cast<PointerType>(ExpandTy);
       // If the step isn't constant, don't use an implicitly scaled GEP, because
       // that would require a multiply inside the loop.
       if (!isa<ConstantInt>(StepV))
@@ -978,7 +1041,7 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
       const SCEV *const StepArray[1] = { SE.getSCEV(StepV) };
       IncV = expandAddToGEP(StepArray, StepArray+1, GEPPtrTy, IntTy, PN);
       if (IncV->getType() != PN->getType()) {
-        IncV = Builder.CreateBitCast(IncV, PN->getType(), "tmp");
+        IncV = Builder.CreateBitCast(IncV, PN->getType());
         rememberInstruction(IncV);
       }
     } else {
@@ -1001,8 +1064,8 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
 }
 
 Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
-  const Type *STy = S->getType();
-  const Type *IntTy = SE.getEffectiveSCEVType(STy);
+  Type *STy = S->getType();
+  Type *IntTy = SE.getEffectiveSCEVType(STy);
   const Loop *L = S->getLoop();
 
   // Determine a normalized form of this expression, which is the expression
@@ -1045,7 +1108,7 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 
   // Expand the core addrec. If we need post-loop scaling, force it to
   // expand to an integer type to avoid the need for additional casting.
-  const Type *ExpandTy = PostLoopScale ? IntTy : STy;
+  Type *ExpandTy = PostLoopScale ? IntTy : STy;
   PHINode *PN = getAddRecExprPHILiterally(Normalized, L, ExpandTy, IntTy);
 
   // Accommodate post-inc mode, if necessary.
@@ -1057,6 +1120,14 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
     BasicBlock *LatchBlock = L->getLoopLatch();
     assert(LatchBlock && "PostInc mode requires a unique loop latch!");
     Result = PN->getIncomingValueForBlock(LatchBlock);
+
+    // For an expansion to use the postinc form, the client must call
+    // expandCodeFor with an InsertPoint that is either outside the PostIncLoop
+    // or dominated by IVIncInsertPos.
+    assert((!isa<Instruction>(Result) ||
+            SE.DT->dominates(cast<Instruction>(Result),
+                             Builder.GetInsertPoint())) &&
+           "postinc expansion does not dominate use");
   }
 
   // Re-apply any non-loop-dominating scale.
@@ -1069,7 +1140,7 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 
   // Re-apply any non-loop-dominating offset.
   if (PostLoopOffset) {
-    if (const PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
+    if (PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
       const SCEV *const OffsetArray[1] = { PostLoopOffset };
       Result = expandAddToGEP(OffsetArray, OffsetArray+1, PTy, IntTy, Result);
     } else {
@@ -1086,7 +1157,7 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
   if (!CanonicalMode) return expandAddRecExprLiterally(S);
 
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
   const Loop *L = S->getLoop();
 
   // First check for an existing canonical IV in a suitable type.
@@ -1110,7 +1181,8 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
     BasicBlock::iterator SaveInsertPt = Builder.GetInsertPoint();
     BasicBlock::iterator NewInsertPt =
       llvm::next(BasicBlock::iterator(cast<Instruction>(V)));
-    while (isa<PHINode>(NewInsertPt) || isa<DbgInfoIntrinsic>(NewInsertPt))
+    while (isa<PHINode>(NewInsertPt) || isa<DbgInfoIntrinsic>(NewInsertPt) ||
+           isa<LandingPadInst>(NewInsertPt))
       ++NewInsertPt;
     V = expandCodeFor(SE.getTruncateExpr(SE.getUnknown(V), Ty), 0,
                       NewInsertPt);
@@ -1132,7 +1204,7 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
     // Dig into the expression to find the pointer base for a GEP.
     ExposePointerBase(Base, RestArray[0], SE);
     // If we found a pointer, expand the AddRec with a GEP.
-    if (const PointerType *PTy = dyn_cast<PointerType>(Base->getType())) {
+    if (PointerType *PTy = dyn_cast<PointerType>(Base->getType())) {
       // Make sure the Base isn't something exotic, such as a multiplied
       // or divided pointer value. In those cases, the result type isn't
       // actually a pointer type.
@@ -1216,35 +1288,35 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
 }
 
 Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
   Value *V = expandCodeFor(S->getOperand(),
                            SE.getEffectiveSCEVType(S->getOperand()->getType()));
-  Value *I = Builder.CreateTrunc(V, Ty, "tmp");
+  Value *I = Builder.CreateTrunc(V, Ty);
   rememberInstruction(I);
   return I;
 }
 
 Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
   Value *V = expandCodeFor(S->getOperand(),
                            SE.getEffectiveSCEVType(S->getOperand()->getType()));
-  Value *I = Builder.CreateZExt(V, Ty, "tmp");
+  Value *I = Builder.CreateZExt(V, Ty);
   rememberInstruction(I);
   return I;
 }
 
 Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
-  const Type *Ty = SE.getEffectiveSCEVType(S->getType());
+  Type *Ty = SE.getEffectiveSCEVType(S->getType());
   Value *V = expandCodeFor(S->getOperand(),
                            SE.getEffectiveSCEVType(S->getOperand()->getType()));
-  Value *I = Builder.CreateSExt(V, Ty, "tmp");
+  Value *I = Builder.CreateSExt(V, Ty);
   rememberInstruction(I);
   return I;
 }
 
 Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
   Value *LHS = expand(S->getOperand(S->getNumOperands()-1));
-  const Type *Ty = LHS->getType();
+  Type *Ty = LHS->getType();
   for (int i = S->getNumOperands()-2; i >= 0; --i) {
     // In the case of mixed integer and pointer types, do the
     // rest of the comparisons as integer.
@@ -1253,7 +1325,7 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
-    Value *ICmp = Builder.CreateICmpSGT(LHS, RHS, "tmp");
+    Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
     rememberInstruction(ICmp);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
     rememberInstruction(Sel);
@@ -1268,7 +1340,7 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
 
 Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
   Value *LHS = expand(S->getOperand(S->getNumOperands()-1));
-  const Type *Ty = LHS->getType();
+  Type *Ty = LHS->getType();
   for (int i = S->getNumOperands()-2; i >= 0; --i) {
     // In the case of mixed integer and pointer types, do the
     // rest of the comparisons as integer.
@@ -1277,7 +1349,7 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
-    Value *ICmp = Builder.CreateICmpUGT(LHS, RHS, "tmp");
+    Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
     rememberInstruction(ICmp);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
     rememberInstruction(Sel);
@@ -1290,7 +1362,7 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
   return LHS;
 }
 
-Value *SCEVExpander::expandCodeFor(const SCEV *SH, const Type *Ty,
+Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty,
                                    Instruction *I) {
   BasicBlock::iterator IP = I;
   while (isInsertedInstruction(IP) || isa<DbgInfoIntrinsic>(IP))
@@ -1299,7 +1371,7 @@ Value *SCEVExpander::expandCodeFor(const SCEV *SH, const Type *Ty,
   return expandCodeFor(SH, Ty);
 }
 
-Value *SCEVExpander::expandCodeFor(const SCEV *SH, const Type *Ty) {
+Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) {
   // Expand the code for this SCEV.
   Value *V = expand(SH);
   if (Ty) {
@@ -1325,7 +1397,7 @@ Value *SCEVExpander::expand(const SCEV *S) {
       // after the PHIs (and after any other instructions that we've inserted
       // there) so that it is guaranteed to dominate any user inside the loop.
       if (L && SE.hasComputableLoopEvolution(S, L) && !PostIncLoops.count(L))
-        InsertPt = L->getHeader()->getFirstNonPHI();
+        InsertPt = L->getHeader()->getFirstInsertionPt();
       while (isInsertedInstruction(InsertPt) || isa<DbgInfoIntrinsic>(InsertPt))
         InsertPt = llvm::next(BasicBlock::iterator(InsertPt));
       break;
@@ -1346,8 +1418,12 @@ Value *SCEVExpander::expand(const SCEV *S) {
   Value *V = visit(S);
 
   // Remember the expanded value for this SCEV at this location.
-  if (PostIncLoops.empty())
-    InsertedExpressions[std::make_pair(S, InsertPt)] = V;
+  //
+  // This is independent of PostIncLoops. The mapped value simply materializes
+  // the expression at this insertion point. If the mapped value happened to be
+  // a postinc expansion, it could be reused by a non postinc user, but only if
+  // its insertion point was already at the head of the loop.
+  InsertedExpressions[std::make_pair(S, InsertPt)] = V;
 
   restoreInsertPoint(SaveInsertBB, SaveInsertPt);
   return V;
@@ -1384,7 +1460,7 @@ void SCEVExpander::restoreInsertPoint(BasicBlock *BB, BasicBlock::iterator I) {
 /// starts at zero and steps by one on each iteration.
 PHINode *
 SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
-                                                    const Type *Ty) {
+                                                    Type *Ty) {
   assert(Ty->isIntegerTy() && "Can only insert integer induction variables!");
 
   // Build a SCEV for {0,+,1}<L>.
@@ -1401,3 +1477,102 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
 
   return V;
 }
+
+/// hoistStep - Attempt to hoist an IV increment above a potential use.
+///
+/// To successfully hoist, two criteria must be met:
+/// - IncV operands dominate InsertPos and
+/// - InsertPos dominates IncV
+///
+/// Meeting the second condition means that we don't need to check all of IncV's
+/// existing uses (it's moving up in the domtree).
+///
+/// This does not yet recursively hoist the operands, although that would
+/// not be difficult.
+///
+/// This does not require a SCEVExpander instance and could be replaced by a
+/// general code-insertion helper.
+bool SCEVExpander::hoistStep(Instruction *IncV, Instruction *InsertPos,
+                             const DominatorTree *DT) {
+  if (DT->dominates(IncV, InsertPos))
+    return true;
+
+  if (!DT->dominates(InsertPos->getParent(), IncV->getParent()))
+    return false;
+
+  if (IncV->mayHaveSideEffects())
+    return false;
+
+  // Attempt to hoist IncV
+  for (User::op_iterator OI = IncV->op_begin(), OE = IncV->op_end();
+       OI != OE; ++OI) {
+    Instruction *OInst = dyn_cast<Instruction>(OI);
+    if (OInst && !DT->dominates(OInst, InsertPos))
+      return false;
+  }
+  IncV->moveBefore(InsertPos);
+  return true;
+}
+
+/// replaceCongruentIVs - Check for congruent phis in this loop header and
+/// replace them with their most canonical representative. Return the number of
+/// phis eliminated.
+///
+/// This does not depend on any SCEVExpander state but should be used in
+/// the same context that SCEVExpander is used.
+unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
+                                           SmallVectorImpl<WeakVH> &DeadInsts) {
+  unsigned NumElim = 0;
+  DenseMap<const SCEV *, PHINode *> ExprToIVMap;
+  for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
+    PHINode *Phi = cast<PHINode>(I);
+    if (!SE.isSCEVable(Phi->getType()))
+      continue;
+
+    PHINode *&OrigPhiRef = ExprToIVMap[SE.getSCEV(Phi)];
+    if (!OrigPhiRef) {
+      OrigPhiRef = Phi;
+      continue;
+    }
+
+    // If one phi derives from the other via GEPs, types may differ.
+    // We could consider adding a bitcast here to handle it.
+    if (OrigPhiRef->getType() != Phi->getType())
+      continue;
+
+    if (BasicBlock *LatchBlock = L->getLoopLatch()) {
+      Instruction *OrigInc =
+        cast<Instruction>(OrigPhiRef->getIncomingValueForBlock(LatchBlock));
+      Instruction *IsomorphicInc =
+        cast<Instruction>(Phi->getIncomingValueForBlock(LatchBlock));
+
+      // If this phi is more canonical, swap it with the original.
+      if (!isExpandedAddRecExprPHI(OrigPhiRef, OrigInc, L)
+          && isExpandedAddRecExprPHI(Phi, IsomorphicInc, L)) {
+        std::swap(OrigPhiRef, Phi);
+        std::swap(OrigInc, IsomorphicInc);
+      }
+      // Replacing the congruent phi is sufficient because acyclic redundancy
+      // elimination, CSE/GVN, should handle the rest. However, once SCEV proves
+      // that a phi is congruent, it's often the head of an IV user cycle that
+      // is isomorphic with the original phi. So it's worth eagerly cleaning up
+      // the common case of a single IV increment.
+      if (OrigInc != IsomorphicInc &&
+          OrigInc->getType() == IsomorphicInc->getType() &&
+          SE.getSCEV(OrigInc) == SE.getSCEV(IsomorphicInc) &&
+          hoistStep(OrigInc, IsomorphicInc, DT)) {
+        DEBUG_WITH_TYPE(DebugType, dbgs()
+                        << "INDVARS: Eliminated congruent iv.inc: "
+                        << *IsomorphicInc << '\n');
+        IsomorphicInc->replaceAllUsesWith(OrigInc);
+        DeadInsts.push_back(IsomorphicInc);
+      }
+    }
+    DEBUG_WITH_TYPE(DebugType, dbgs()
+                    << "INDVARS: Eliminated congruent iv: " << *Phi << '\n');
+    ++NumElim;
+    Phi->replaceAllUsesWith(OrigPhiRef);
+    DeadInsts.push_back(Phi);
+  }
+  return NumElim;
+}
diff --git a/contrib/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp b/contrib/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
index 60e630aaab88..c66ecd6e8727 100644
--- a/contrib/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
+++ b/contrib/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
@@ -60,20 +60,40 @@ static bool IVUseShouldUsePostIncValue(Instruction *User, Value *Operand,
   return true;
 }
 
-const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
-                                         const SCEV *S,
-                                         Instruction *User,
-                                         Value *OperandValToReplace,
-                                         PostIncLoopSet &Loops,
-                                         ScalarEvolution &SE,
-                                         DominatorTree &DT) {
-  if (isa<SCEVConstant>(S) || isa<SCEVUnknown>(S))
-    return S;
+namespace {
+
+/// Hold the state used during post-inc expression transformation, including a
+/// map of transformed expressions.
+class PostIncTransform {
+  TransformKind Kind;
+  PostIncLoopSet &Loops;
+  ScalarEvolution &SE;
+  DominatorTree &DT;
+
+  DenseMap<const SCEV*, const SCEV*> Transformed;
+
+public:
+  PostIncTransform(TransformKind kind, PostIncLoopSet &loops,
+                   ScalarEvolution &se, DominatorTree &dt):
+    Kind(kind), Loops(loops), SE(se), DT(dt) {}
+
+  const SCEV *TransformSubExpr(const SCEV *S, Instruction *User,
+                               Value *OperandValToReplace);
+
+protected:
+  const SCEV *TransformImpl(const SCEV *S, Instruction *User,
+                            Value *OperandValToReplace);
+};
+
+} // namespace
+
+/// Implement post-inc transformation for all valid expression types.
+const SCEV *PostIncTransform::
+TransformImpl(const SCEV *S, Instruction *User, Value *OperandValToReplace) {
 
   if (const SCEVCastExpr *X = dyn_cast<SCEVCastExpr>(S)) {
     const SCEV *O = X->getOperand();
-    const SCEV *N = TransformForPostIncUse(Kind, O, User, OperandValToReplace,
-                                           Loops, SE, DT);
+    const SCEV *N = TransformSubExpr(O, User, OperandValToReplace);
     if (O != N)
       switch (S->getSCEVType()) {
       case scZeroExtend: return SE.getZeroExtendExpr(N, S->getType());
@@ -93,9 +113,7 @@ const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
     // Transform each operand.
     for (SCEVNAryExpr::op_iterator I = AR->op_begin(), E = AR->op_end();
          I != E; ++I) {
-      const SCEV *O = *I;
-      const SCEV *N = TransformForPostIncUse(Kind, O, LUser, 0, Loops, SE, DT);
-      Operands.push_back(N);
+      Operands.push_back(TransformSubExpr(*I, LUser, 0));
     }
     // Conservatively use AnyWrap until/unless we need FlagNW.
     const SCEV *Result = SE.getAddRecExpr(Operands, L, SCEV::FlagAnyWrap);
@@ -104,8 +122,8 @@ const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
     case NormalizeAutodetect:
       if (IVUseShouldUsePostIncValue(User, OperandValToReplace, L, &DT)) {
         const SCEV *TransformedStep =
-          TransformForPostIncUse(Kind, AR->getStepRecurrence(SE),
-                                 User, OperandValToReplace, Loops, SE, DT);
+          TransformSubExpr(AR->getStepRecurrence(SE),
+                           User, OperandValToReplace);
         Result = SE.getMinusSCEV(Result, TransformedStep);
         Loops.insert(L);
       }
@@ -114,24 +132,20 @@ const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
       // sometimes fails to canonicalize two equal SCEVs to exactly the same
       // form. It's possibly a pessimization when this happens, but it isn't a
       // correctness problem, so disable this assert for now.
-      assert(S == TransformForPostIncUse(Denormalize, Result,
-                                         User, OperandValToReplace,
-                                         Loops, SE, DT) &&
+      assert(S == TransformSubExpr(Result, User, OperandValToReplace) &&
              "SCEV normalization is not invertible!");
 #endif
       break;
     case Normalize:
       if (Loops.count(L)) {
         const SCEV *TransformedStep =
-          TransformForPostIncUse(Kind, AR->getStepRecurrence(SE),
-                                 User, OperandValToReplace, Loops, SE, DT);
+          TransformSubExpr(AR->getStepRecurrence(SE),
+                           User, OperandValToReplace);
         Result = SE.getMinusSCEV(Result, TransformedStep);
       }
 #if 0
       // See the comment on the assert above.
-      assert(S == TransformForPostIncUse(Denormalize, Result,
-                                         User, OperandValToReplace,
-                                         Loops, SE, DT) &&
+      assert(S == TransformSubExpr(Result, User, OperandValToReplace) &&
              "SCEV normalization is not invertible!");
 #endif
       break;
@@ -150,8 +164,7 @@ const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
     for (SCEVNAryExpr::op_iterator I = X->op_begin(), E = X->op_end();
          I != E; ++I) {
       const SCEV *O = *I;
-      const SCEV *N = TransformForPostIncUse(Kind, O, User, OperandValToReplace,
-                                             Loops, SE, DT);
+      const SCEV *N = TransformSubExpr(O, User, OperandValToReplace);
       Changed |= N != O;
       Operands.push_back(N);
     }
@@ -170,10 +183,8 @@ const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
   if (const SCEVUDivExpr *X = dyn_cast<SCEVUDivExpr>(S)) {
     const SCEV *LO = X->getLHS();
     const SCEV *RO = X->getRHS();
-    const SCEV *LN = TransformForPostIncUse(Kind, LO, User, OperandValToReplace,
-                                            Loops, SE, DT);
-    const SCEV *RN = TransformForPostIncUse(Kind, RO, User, OperandValToReplace,
-                                            Loops, SE, DT);
+    const SCEV *LN = TransformSubExpr(LO, User, OperandValToReplace);
+    const SCEV *RN = TransformSubExpr(RO, User, OperandValToReplace);
     if (LO != LN || RO != RN)
       return SE.getUDivExpr(LN, RN);
     return S;
@@ -182,3 +193,33 @@ const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
   llvm_unreachable("Unexpected SCEV kind!");
   return 0;
 }
+
+/// Manage recursive transformation across an expression DAG. Revisiting
+/// expressions would lead to exponential recursion.
+const SCEV *PostIncTransform::
+TransformSubExpr(const SCEV *S, Instruction *User, Value *OperandValToReplace) {
+
+  if (isa<SCEVConstant>(S) || isa<SCEVUnknown>(S))
+    return S;
+
+  const SCEV *Result = Transformed.lookup(S);
+  if (Result)
+    return Result;
+
+  Result = TransformImpl(S, User, OperandValToReplace);
+  Transformed[S] = Result;
+  return Result;
+}
+
+/// Top level driver for transforming an expression DAG into its requested
+/// post-inc form (either "Normalized" or "Denormalized".
+const SCEV *llvm::TransformForPostIncUse(TransformKind Kind,
+                                         const SCEV *S,
+                                         Instruction *User,
+                                         Value *OperandValToReplace,
+                                         PostIncLoopSet &Loops,
+                                         ScalarEvolution &SE,
+                                         DominatorTree &DT) {
+  PostIncTransform Transform(Kind, Loops, SE, DT);
+  return Transform.TransformSubExpr(S, User, OperandValToReplace);
+}
diff --git a/contrib/llvm/lib/Analysis/ValueTracking.cpp b/contrib/llvm/lib/Analysis/ValueTracking.cpp
index 455c91077dfb..4d94f619fda1 100644
--- a/contrib/llvm/lib/Analysis/ValueTracking.cpp
+++ b/contrib/llvm/lib/Analysis/ValueTracking.cpp
@@ -34,7 +34,7 @@ const unsigned MaxDepth = 6;
 
 /// getBitWidth - Returns the bitwidth of the given scalar or pointer type (if
 /// unknown returns 0).  For vector types, returns the element type's bitwidth.
-static unsigned getBitWidth(const Type *Ty, const TargetData *TD) {
+static unsigned getBitWidth(Type *Ty, const TargetData *TD) {
   if (unsigned BitWidth = Ty->getScalarSizeInBits())
     return BitWidth;
   assert(isa<PointerType>(Ty) && "Expected a pointer type!");
@@ -103,7 +103,7 @@ void llvm::ComputeMaskedBits(Value *V, const APInt &Mask,
   if (GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
     unsigned Align = GV->getAlignment();
     if (Align == 0 && TD && GV->getType()->getElementType()->isSized()) {
-      const Type *ObjectType = GV->getType()->getElementType();
+      Type *ObjectType = GV->getType()->getElementType();
       // If the object is defined in the current Module, we'll be giving
       // it the preferred alignment. Otherwise, we have to assume that it
       // may only have the minimum ABI alignment.
@@ -268,7 +268,7 @@ void llvm::ComputeMaskedBits(Value *V, const APInt &Mask,
     // FALL THROUGH and handle them the same as zext/trunc.
   case Instruction::ZExt:
   case Instruction::Trunc: {
-    const Type *SrcTy = I->getOperand(0)->getType();
+    Type *SrcTy = I->getOperand(0)->getType();
     
     unsigned SrcBitWidth;
     // Note that we handle pointer operands here because of inttoptr/ptrtoint
@@ -291,7 +291,7 @@ void llvm::ComputeMaskedBits(Value *V, const APInt &Mask,
     return;
   }
   case Instruction::BitCast: {
-    const Type *SrcTy = I->getOperand(0)->getType();
+    Type *SrcTy = I->getOperand(0)->getType();
     if ((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
         // TODO: For now, not handling conversions like:
         // (bitcast i64 %x to <2 x i32>)
@@ -559,7 +559,7 @@ void llvm::ComputeMaskedBits(Value *V, const APInt &Mask,
     gep_type_iterator GTI = gep_type_begin(I);
     for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
       Value *Index = I->getOperand(i);
-      if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
         // Handle struct member offset arithmetic.
         if (!TD) return;
         const StructLayout *SL = TD->getStructLayout(STy);
@@ -569,7 +569,7 @@ void llvm::ComputeMaskedBits(Value *V, const APInt &Mask,
                           CountTrailingZeros_64(Offset));
       } else {
         // Handle array index arithmetic.
-        const Type *IndexedTy = GTI.getIndexedType();
+        Type *IndexedTy = GTI.getIndexedType();
         if (!IndexedTy->isSized()) return;
         unsigned GEPOpiBits = Index->getType()->getScalarSizeInBits();
         uint64_t TypeSize = TD ? TD->getTypeAllocSize(IndexedTy) : 1;
@@ -898,7 +898,7 @@ unsigned llvm::ComputeNumSignBits(Value *V, const TargetData *TD,
   assert((TD || V->getType()->isIntOrIntVectorTy()) &&
          "ComputeNumSignBits requires a TargetData object to operate "
          "on non-integer values!");
-  const Type *Ty = V->getType();
+  Type *Ty = V->getType();
   unsigned TyBits = TD ? TD->getTypeSizeInBits(V->getType()->getScalarType()) :
                          Ty->getScalarSizeInBits();
   unsigned Tmp, Tmp2;
@@ -1078,7 +1078,7 @@ bool llvm::ComputeMultiple(Value *V, unsigned Base, Value *&Multiple,
   assert(Depth <= MaxDepth && "Limit Search Depth");
   assert(V->getType()->isIntegerTy() && "Not integer or pointer type!");
 
-  const Type *T = V->getType();
+  Type *T = V->getType();
 
   ConstantInt *CI = dyn_cast<ConstantInt>(V);
 
@@ -1315,11 +1315,11 @@ Value *llvm::isBytewiseValue(Value *V) {
 // indices from Idxs that should be left out when inserting into the resulting
 // struct. To is the result struct built so far, new insertvalue instructions
 // build on that.
-static Value *BuildSubAggregate(Value *From, Value* To, const Type *IndexedType,
+static Value *BuildSubAggregate(Value *From, Value* To, Type *IndexedType,
                                 SmallVector<unsigned, 10> &Idxs,
                                 unsigned IdxSkip,
                                 Instruction *InsertBefore) {
-  const llvm::StructType *STy = llvm::dyn_cast<llvm::StructType>(IndexedType);
+  llvm::StructType *STy = llvm::dyn_cast<llvm::StructType>(IndexedType);
   if (STy) {
     // Save the original To argument so we can modify it
     Value *OrigTo = To;
@@ -1358,8 +1358,7 @@ static Value *BuildSubAggregate(Value *From, Value* To, const Type *IndexedType,
     return NULL;
 
   // Insert the value in the new (sub) aggregrate
-  return llvm::InsertValueInst::Create(To, V,
-                                       ArrayRef<unsigned>(Idxs).slice(IdxSkip),
+  return llvm::InsertValueInst::Create(To, V, makeArrayRef(Idxs).slice(IdxSkip),
                                        "tmp", InsertBefore);
 }
 
@@ -1378,7 +1377,7 @@ static Value *BuildSubAggregate(Value *From, Value* To, const Type *IndexedType,
 static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
                                 Instruction *InsertBefore) {
   assert(InsertBefore && "Must have someplace to insert!");
-  const Type *IndexedType = ExtractValueInst::getIndexedType(From->getType(),
+  Type *IndexedType = ExtractValueInst::getIndexedType(From->getType(),
                                                              idx_range);
   Value *To = UndefValue::get(IndexedType);
   SmallVector<unsigned, 10> Idxs(idx_range.begin(), idx_range.end());
@@ -1404,7 +1403,7 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
          && "Not looking at a struct or array?");
   assert(ExtractValueInst::getIndexedType(V->getType(), idx_range)
          && "Invalid indices for type?");
-  const CompositeType *PTy = cast<CompositeType>(V->getType());
+  CompositeType *PTy = cast<CompositeType>(V->getType());
 
   if (isa<UndefValue>(V))
     return UndefValue::get(ExtractValueInst::getIndexedType(PTy,
@@ -1435,9 +1434,7 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
           // %C = insertvalue {i32, i32 } %A, i32 11, 1
           // which allows the unused 0,0 element from the nested struct to be
           // removed.
-          return BuildSubAggregate(V,
-                                   ArrayRef<unsigned>(idx_range.begin(),
-                                                      req_idx),
+          return BuildSubAggregate(V, makeArrayRef(idx_range.begin(), req_idx),
                                    InsertBefore);
         else
           // We can't handle this without inserting insertvalues
@@ -1455,7 +1452,7 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
     // requested (though possibly only partially). Now we recursively look at
     // the inserted value, passing any remaining indices.
     return FindInsertedValue(I->getInsertedValueOperand(),
-                             ArrayRef<unsigned>(req_idx, idx_range.end()),
+                             makeArrayRef(req_idx, idx_range.end()),
                              InsertBefore);
   } else if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(V)) {
     // If we're extracting a value from an aggregrate that was extracted from
@@ -1506,7 +1503,7 @@ Value *llvm::GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset,
     if (OpC->isZero()) continue;
     
     // Handle a struct and array indices which add their offset to the pointer.
-    if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+    if (StructType *STy = dyn_cast<StructType>(*GTI)) {
       Offset += TD.getStructLayout(STy)->getElementOffset(OpC->getZExtValue());
     } else {
       uint64_t Size = TD.getTypeAllocSize(GTI.getIndexedType());
@@ -1557,8 +1554,8 @@ bool llvm::GetConstantStringInfo(const Value *V, std::string &Str,
       return false;
     
     // Make sure the index-ee is a pointer to array of i8.
-    const PointerType *PT = cast<PointerType>(GEP->getOperand(0)->getType());
-    const ArrayType *AT = dyn_cast<ArrayType>(PT->getElementType());
+    PointerType *PT = cast<PointerType>(GEP->getOperand(0)->getType());
+    ArrayType *AT = dyn_cast<ArrayType>(PT->getElementType());
     if (AT == 0 || !AT->getElementType()->isIntegerTy(8))
       return false;
     
diff --git a/contrib/llvm/lib/AsmParser/LLLexer.cpp b/contrib/llvm/lib/AsmParser/LLLexer.cpp
index 3c63106e8c3b..d0dd98627bab 100644
--- a/contrib/llvm/lib/AsmParser/LLLexer.cpp
+++ b/contrib/llvm/lib/AsmParser/LLLexer.cpp
@@ -506,6 +506,15 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(deplibs);
   KEYWORD(datalayout);
   KEYWORD(volatile);
+  KEYWORD(atomic);
+  KEYWORD(unordered);
+  KEYWORD(monotonic);
+  KEYWORD(acquire);
+  KEYWORD(release);
+  KEYWORD(acq_rel);
+  KEYWORD(seq_cst);
+  KEYWORD(singlethread);
+
   KEYWORD(nuw);
   KEYWORD(nsw);
   KEYWORD(exact);
@@ -549,6 +558,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(readnone);
   KEYWORD(readonly);
   KEYWORD(uwtable);
+  KEYWORD(returns_twice);
 
   KEYWORD(inlinehint);
   KEYWORD(noinline);
@@ -559,7 +569,6 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(noredzone);
   KEYWORD(noimplicitfloat);
   KEYWORD(naked);
-  KEYWORD(hotpatch);
   KEYWORD(nonlazybind);
 
   KEYWORD(type);
@@ -570,8 +579,16 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(oeq); KEYWORD(one); KEYWORD(olt); KEYWORD(ogt); KEYWORD(ole);
   KEYWORD(oge); KEYWORD(ord); KEYWORD(uno); KEYWORD(ueq); KEYWORD(une);
 
+  KEYWORD(xchg); KEYWORD(nand); KEYWORD(max); KEYWORD(min); KEYWORD(umax);
+  KEYWORD(umin);
+
   KEYWORD(x);
   KEYWORD(blockaddress);
+
+  KEYWORD(personality);
+  KEYWORD(cleanup);
+  KEYWORD(catch);
+  KEYWORD(filter);
 #undef KEYWORD
 
   // Keywords for types.
@@ -624,12 +641,16 @@ lltok::Kind LLLexer::LexIdentifier() {
   INSTKEYWORD(switch,      Switch);
   INSTKEYWORD(indirectbr,  IndirectBr);
   INSTKEYWORD(invoke,      Invoke);
+  INSTKEYWORD(resume,      Resume);
   INSTKEYWORD(unwind,      Unwind);
   INSTKEYWORD(unreachable, Unreachable);
 
   INSTKEYWORD(alloca,      Alloca);
   INSTKEYWORD(load,        Load);
   INSTKEYWORD(store,       Store);
+  INSTKEYWORD(cmpxchg,     AtomicCmpXchg);
+  INSTKEYWORD(atomicrmw,   AtomicRMW);
+  INSTKEYWORD(fence,       Fence);
   INSTKEYWORD(getelementptr, GetElementPtr);
 
   INSTKEYWORD(extractelement, ExtractElement);
@@ -637,6 +658,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   INSTKEYWORD(shufflevector,  ShuffleVector);
   INSTKEYWORD(extractvalue,   ExtractValue);
   INSTKEYWORD(insertvalue,    InsertValue);
+  INSTKEYWORD(landingpad,     LandingPad);
 #undef INSTKEYWORD
 
   // Check for [us]0x[0-9A-Fa-f]+ which are Hexadecimal constant generated by
@@ -704,17 +726,17 @@ lltok::Kind LLLexer::Lex0x() {
   case 'K':
     // F80HexFPConstant - x87 long double in hexadecimal format (10 bytes)
     FP80HexToIntPair(TokStart+3, CurPtr, Pair);
-    APFloatVal = APFloat(APInt(80, 2, Pair));
+    APFloatVal = APFloat(APInt(80, Pair));
     return lltok::APFloat;
   case 'L':
     // F128HexFPConstant - IEEE 128-bit in hexadecimal format (16 bytes)
     HexToIntPair(TokStart+3, CurPtr, Pair);
-    APFloatVal = APFloat(APInt(128, 2, Pair), true);
+    APFloatVal = APFloat(APInt(128, Pair), true);
     return lltok::APFloat;
   case 'M':
     // PPC128HexFPConstant - PowerPC 128-bit in hexadecimal format (16 bytes)
     HexToIntPair(TokStart+3, CurPtr, Pair);
-    APFloatVal = APFloat(APInt(128, 2, Pair));
+    APFloatVal = APFloat(APInt(128, Pair));
     return lltok::APFloat;
   }
 }
diff --git a/contrib/llvm/lib/AsmParser/LLParser.cpp b/contrib/llvm/lib/AsmParser/LLParser.cpp
index cfc31f3db8a7..cafaab01afd9 100644
--- a/contrib/llvm/lib/AsmParser/LLParser.cpp
+++ b/contrib/llvm/lib/AsmParser/LLParser.cpp
@@ -26,7 +26,7 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-static std::string getTypeString(const Type *T) {
+static std::string getTypeString(Type *T) {
   std::string Result;
   raw_string_ostream Tmp(Result);
   Tmp << *T;
@@ -120,6 +120,9 @@ bool LLParser::ValidateEndOfModule() {
   for (Module::iterator FI = M->begin(), FE = M->end(); FI != FE; )
     UpgradeCallsToIntrinsic(FI++); // must be post-increment, as we remove
 
+  // Upgrade to new EH scheme. N.B. This will go away in 3.1.
+  UpgradeExceptionHandling(M);
+
   // Check debug info intrinsics.
   CheckDebugInfoIntrinsics(M);
   return false;
@@ -744,9 +747,9 @@ bool LLParser::ParseGlobal(const std::string &Name, LocTy NameLoc,
 /// GetGlobalVal - Get a value with the specified name or ID, creating a
 /// forward reference record if needed.  This can return null if the value
 /// exists but does not have the right type.
-GlobalValue *LLParser::GetGlobalVal(const std::string &Name, const Type *Ty,
+GlobalValue *LLParser::GetGlobalVal(const std::string &Name, Type *Ty,
                                     LocTy Loc) {
-  const PointerType *PTy = dyn_cast<PointerType>(Ty);
+  PointerType *PTy = dyn_cast<PointerType>(Ty);
   if (PTy == 0) {
     Error(Loc, "global variable reference must have pointer type");
     return 0;
@@ -775,7 +778,7 @@ GlobalValue *LLParser::GetGlobalVal(const std::string &Name, const Type *Ty,
 
   // Otherwise, create a new forward reference for this value and remember it.
   GlobalValue *FwdVal;
-  if (const FunctionType *FT = dyn_cast<FunctionType>(PTy->getElementType()))
+  if (FunctionType *FT = dyn_cast<FunctionType>(PTy->getElementType()))
     FwdVal = Function::Create(FT, GlobalValue::ExternalWeakLinkage, Name, M);
   else
     FwdVal = new GlobalVariable(*M, PTy->getElementType(), false,
@@ -785,8 +788,8 @@ GlobalValue *LLParser::GetGlobalVal(const std::string &Name, const Type *Ty,
   return FwdVal;
 }
 
-GlobalValue *LLParser::GetGlobalVal(unsigned ID, const Type *Ty, LocTy Loc) {
-  const PointerType *PTy = dyn_cast<PointerType>(Ty);
+GlobalValue *LLParser::GetGlobalVal(unsigned ID, Type *Ty, LocTy Loc) {
+  PointerType *PTy = dyn_cast<PointerType>(Ty);
   if (PTy == 0) {
     Error(Loc, "global variable reference must have pointer type");
     return 0;
@@ -813,7 +816,7 @@ GlobalValue *LLParser::GetGlobalVal(unsigned ID, const Type *Ty, LocTy Loc) {
 
   // Otherwise, create a new forward reference for this value and remember it.
   GlobalValue *FwdVal;
-  if (const FunctionType *FT = dyn_cast<FunctionType>(PTy->getElementType()))
+  if (FunctionType *FT = dyn_cast<FunctionType>(PTy->getElementType()))
     FwdVal = Function::Create(FT, GlobalValue::ExternalWeakLinkage, "", M);
   else
     FwdVal = new GlobalVariable(*M, PTy->getElementType(), false,
@@ -908,6 +911,7 @@ bool LLParser::ParseOptionalAttrs(unsigned &Attrs, unsigned AttrKind) {
     case lltok::kw_noreturn:        Attrs |= Attribute::NoReturn; break;
     case lltok::kw_nounwind:        Attrs |= Attribute::NoUnwind; break;
     case lltok::kw_uwtable:         Attrs |= Attribute::UWTable; break;
+    case lltok::kw_returns_twice:   Attrs |= Attribute::ReturnsTwice; break;
     case lltok::kw_noinline:        Attrs |= Attribute::NoInline; break;
     case lltok::kw_readnone:        Attrs |= Attribute::ReadNone; break;
     case lltok::kw_readonly:        Attrs |= Attribute::ReadOnly; break;
@@ -919,7 +923,6 @@ bool LLParser::ParseOptionalAttrs(unsigned &Attrs, unsigned AttrKind) {
     case lltok::kw_noredzone:       Attrs |= Attribute::NoRedZone; break;
     case lltok::kw_noimplicitfloat: Attrs |= Attribute::NoImplicitFloat; break;
     case lltok::kw_naked:           Attrs |= Attribute::Naked; break;
-    case lltok::kw_hotpatch:        Attrs |= Attribute::Hotpatch; break;
     case lltok::kw_nonlazybind:     Attrs |= Attribute::NonLazyBind; break;
 
     case lltok::kw_alignstack: {
@@ -1145,6 +1148,32 @@ bool LLParser::ParseOptionalCommaAlign(unsigned &Alignment,
   return false;
 }
 
+/// ParseScopeAndOrdering
+///   if isAtomic: ::= 'singlethread'? AtomicOrdering
+///   else: ::=
+///
+/// This sets Scope and Ordering to the parsed values.
+bool LLParser::ParseScopeAndOrdering(bool isAtomic, SynchronizationScope &Scope,
+                                     AtomicOrdering &Ordering) {
+  if (!isAtomic)
+    return false;
+
+  Scope = CrossThread;
+  if (EatIfPresent(lltok::kw_singlethread))
+    Scope = SingleThread;
+  switch (Lex.getKind()) {
+  default: return TokError("Expected ordering on atomic instruction");
+  case lltok::kw_unordered: Ordering = Unordered; break;
+  case lltok::kw_monotonic: Ordering = Monotonic; break;
+  case lltok::kw_acquire: Ordering = Acquire; break;
+  case lltok::kw_release: Ordering = Release; break;
+  case lltok::kw_acq_rel: Ordering = AcquireRelease; break;
+  case lltok::kw_seq_cst: Ordering = SequentiallyConsistent; break;
+  }
+  Lex.Lex();
+  return false;
+}
+
 /// ParseOptionalStackAlignment
 ///   ::= /* empty */
 ///   ::= 'alignstack' '(' 4 ')'
@@ -1237,7 +1266,7 @@ bool LLParser::ParseType(Type *&Result, bool AllowVoid) {
     // If the type hasn't been defined yet, create a forward definition and
     // remember where that forward def'n was seen (in case it never is defined).
     if (Entry.first == 0) {
-      Entry.first = StructType::createNamed(Context, Lex.getStrVal());
+      Entry.first = StructType::create(Context, Lex.getStrVal());
       Entry.second = Lex.getLoc();
     }
     Result = Entry.first;
@@ -1254,7 +1283,7 @@ bool LLParser::ParseType(Type *&Result, bool AllowVoid) {
     // If the type hasn't been defined yet, create a forward definition and
     // remember where that forward def'n was seen (in case it never is defined).
     if (Entry.first == 0) {
-      Entry.first = StructType::createNamed(Context, "");
+      Entry.first = StructType::create(Context);
       Entry.second = Lex.getLoc();
     }
     Result = Entry.first;
@@ -1476,7 +1505,7 @@ bool LLParser::ParseStructDefinition(SMLoc TypeLoc, StringRef Name,
     
     // If this type number has never been uttered, create it.
     if (Entry.first == 0)
-      Entry.first = StructType::createNamed(Context, Name);
+      Entry.first = StructType::create(Context, Name);
     ResultTy = Entry.first;
     return false;
   }
@@ -1502,7 +1531,7 @@ bool LLParser::ParseStructDefinition(SMLoc TypeLoc, StringRef Name,
   
   // If this type number has never been uttered, create it.
   if (Entry.first == 0)
-    Entry.first = StructType::createNamed(Context, Name);
+    Entry.first = StructType::create(Context, Name);
   
   StructType *STy = cast<StructType>(Entry.first);
  
@@ -1668,7 +1697,7 @@ bool LLParser::PerFunctionState::FinishFunction() {
 /// forward reference record if needed.  This can return null if the value
 /// exists but does not have the right type.
 Value *LLParser::PerFunctionState::GetVal(const std::string &Name,
-                                          const Type *Ty, LocTy Loc) {
+                                          Type *Ty, LocTy Loc) {
   // Look this name up in the normal function symbol table.
   Value *Val = F.getValueSymbolTable().lookup(Name);
 
@@ -1709,7 +1738,7 @@ Value *LLParser::PerFunctionState::GetVal(const std::string &Name,
   return FwdVal;
 }
 
-Value *LLParser::PerFunctionState::GetVal(unsigned ID, const Type *Ty,
+Value *LLParser::PerFunctionState::GetVal(unsigned ID, Type *Ty,
                                           LocTy Loc) {
   // Look this name up in the normal function symbol table.
   Value *Val = ID < NumberedVals.size() ? NumberedVals[ID] : 0;
@@ -2273,16 +2302,11 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
       if (Elts.size() == 0 || !Elts[0]->getType()->isPointerTy())
         return Error(ID.Loc, "getelementptr requires pointer operand");
 
-      if (!GetElementPtrInst::getIndexedType(Elts[0]->getType(),
-                                             (Value**)(Elts.data() + 1),
-                                             Elts.size() - 1))
+      ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end());
+      if (!GetElementPtrInst::getIndexedType(Elts[0]->getType(), Indices))
         return Error(ID.Loc, "invalid indices for getelementptr");
-      ID.ConstantVal = InBounds ?
-        ConstantExpr::getInBoundsGetElementPtr(Elts[0],
-                                               Elts.data() + 1,
-                                               Elts.size() - 1) :
-        ConstantExpr::getGetElementPtr(Elts[0],
-                                       Elts.data() + 1, Elts.size() - 1);
+      ID.ConstantVal = ConstantExpr::getGetElementPtr(Elts[0], Indices,
+                                                      InBounds);
     } else if (Opc == Instruction::Select) {
       if (Elts.size() != 3)
         return Error(ID.Loc, "expected three operands to select");
@@ -2323,7 +2347,7 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
 }
 
 /// ParseGlobalValue - Parse a global value with the specified type.
-bool LLParser::ParseGlobalValue(const Type *Ty, Constant *&C) {
+bool LLParser::ParseGlobalValue(Type *Ty, Constant *&C) {
   C = 0;
   ValID ID;
   Value *V = NULL;
@@ -2410,7 +2434,7 @@ bool LLParser::ParseMetadataValue(ValID &ID, PerFunctionState *PFS) {
 // Function Parsing.
 //===----------------------------------------------------------------------===//
 
-bool LLParser::ConvertValIDToValue(const Type *Ty, ValID &ID, Value *&V,
+bool LLParser::ConvertValIDToValue(Type *Ty, ValID &ID, Value *&V,
                                    PerFunctionState *PFS) {
   if (Ty->isFunctionTy())
     return Error(ID.Loc, "functions are not values, refer to them as pointers");
@@ -2426,8 +2450,8 @@ bool LLParser::ConvertValIDToValue(const Type *Ty, ValID &ID, Value *&V,
     V = PFS->GetVal(ID.StrVal, Ty, ID.Loc);
     return (V == 0);
   case ValID::t_InlineAsm: {
-    const PointerType *PTy = dyn_cast<PointerType>(Ty);
-    const FunctionType *FTy = 
+    PointerType *PTy = dyn_cast<PointerType>(Ty);
+    FunctionType *FTy = 
       PTy ? dyn_cast<FunctionType>(PTy->getElementType()) : 0;
     if (!FTy || !InlineAsm::Verify(FTy, ID.StrVal2))
       return Error(ID.Loc, "invalid type for inline asm constraint string");
@@ -2506,7 +2530,7 @@ bool LLParser::ConvertValIDToValue(const Type *Ty, ValID &ID, Value *&V,
     return false;
   case ValID::t_ConstantStruct:
   case ValID::t_PackedConstantStruct:
-    if (const StructType *ST = dyn_cast<StructType>(Ty)) {
+    if (StructType *ST = dyn_cast<StructType>(Ty)) {
       if (ST->getNumElements() != ID.UIntVal)
         return Error(ID.Loc,
                      "initializer with struct type has wrong # elements");
@@ -2519,15 +2543,15 @@ bool LLParser::ConvertValIDToValue(const Type *Ty, ValID &ID, Value *&V,
           return Error(ID.Loc, "element " + Twine(i) +
                     " of struct initializer doesn't match struct element type");
       
-      V = ConstantStruct::get(ST, ArrayRef<Constant*>(ID.ConstantStructElts,
-                                                      ID.UIntVal));
+      V = ConstantStruct::get(ST, makeArrayRef(ID.ConstantStructElts,
+                                               ID.UIntVal));
     } else
       return Error(ID.Loc, "constant expression type mismatch");
     return false;
   }
 }
 
-bool LLParser::ParseValue(const Type *Ty, Value *&V, PerFunctionState *PFS) {
+bool LLParser::ParseValue(Type *Ty, Value *&V, PerFunctionState *PFS) {
   V = 0;
   ValID ID;
   return ParseValID(ID, PFS) ||
@@ -2671,9 +2695,9 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   if (PAL.paramHasAttr(1, Attribute::StructRet) && !RetType->isVoidTy())
     return Error(RetTypeLoc, "functions with 'sret' argument must return void");
 
-  const FunctionType *FT =
+  FunctionType *FT =
     FunctionType::get(RetType, ParamTypeList, isVarArg);
-  const PointerType *PFT = PointerType::getUnqual(FT);
+  PointerType *PFT = PointerType::getUnqual(FT);
 
   Fn = 0;
   if (!FunctionName.empty()) {
@@ -2864,6 +2888,7 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_switch:      return ParseSwitch(Inst, PFS);
   case lltok::kw_indirectbr:  return ParseIndirectBr(Inst, PFS);
   case lltok::kw_invoke:      return ParseInvoke(Inst, PFS);
+  case lltok::kw_resume:      return ParseResume(Inst, PFS);
   // Binary Operators.
   case lltok::kw_add:
   case lltok::kw_sub:
@@ -2923,13 +2948,18 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_insertelement:  return ParseInsertElement(Inst, PFS);
   case lltok::kw_shufflevector:  return ParseShuffleVector(Inst, PFS);
   case lltok::kw_phi:            return ParsePHI(Inst, PFS);
+  case lltok::kw_landingpad:     return ParseLandingPad(Inst, PFS);
   case lltok::kw_call:           return ParseCall(Inst, PFS, false);
   case lltok::kw_tail:           return ParseCall(Inst, PFS, true);
   // Memory.
   case lltok::kw_alloca:         return ParseAlloc(Inst, PFS);
   case lltok::kw_load:           return ParseLoad(Inst, PFS, false);
   case lltok::kw_store:          return ParseStore(Inst, PFS, false);
+  case lltok::kw_cmpxchg:        return ParseCmpXchg(Inst, PFS);
+  case lltok::kw_atomicrmw:      return ParseAtomicRMW(Inst, PFS);
+  case lltok::kw_fence:          return ParseFence(Inst, PFS);
   case lltok::kw_volatile:
+    // For compatibility; canonical location is after load
     if (EatIfPresent(lltok::kw_load))
       return ParseLoad(Inst, PFS, true);
     else if (EatIfPresent(lltok::kw_store))
@@ -3162,8 +3192,8 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
   // If RetType is a non-function pointer type, then this is the short syntax
   // for the call, which means that RetType is just the return type.  Infer the
   // rest of the function argument types from the arguments that are present.
-  const PointerType *PFTy = 0;
-  const FunctionType *Ty = 0;
+  PointerType *PFTy = 0;
+  FunctionType *Ty = 0;
   if (!(PFTy = dyn_cast<PointerType>(RetType)) ||
       !(Ty = dyn_cast<FunctionType>(PFTy->getElementType()))) {
     // Pull out the types of all of the arguments...
@@ -3194,7 +3224,7 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
   FunctionType::param_iterator I = Ty->param_begin();
   FunctionType::param_iterator E = Ty->param_end();
   for (unsigned i = 0, e = ArgList.size(); i != e; ++i) {
-    const Type *ExpectedTy = 0;
+    Type *ExpectedTy = 0;
     if (I != E) {
       ExpectedTy = *I++;
     } else if (!Ty->isVarArg()) {
@@ -3225,7 +3255,17 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
   return false;
 }
 
+/// ParseResume
+///   ::= 'resume' TypeAndValue
+bool LLParser::ParseResume(Instruction *&Inst, PerFunctionState &PFS) {
+  Value *Exn; LocTy ExnLoc;
+  if (ParseTypeAndValue(Exn, ExnLoc, PFS))
+    return true;
 
+  ResumeInst *RI = ResumeInst::Create(Exn);
+  Inst = RI;
+  return false;
+}
 
 //===----------------------------------------------------------------------===//
 // Binary Operators.
@@ -3473,6 +3513,56 @@ int LLParser::ParsePHI(Instruction *&Inst, PerFunctionState &PFS) {
   return AteExtraComma ? InstExtraComma : InstNormal;
 }
 
+/// ParseLandingPad
+///   ::= 'landingpad' Type 'personality' TypeAndValue 'cleanup'? Clause+
+/// Clause
+///   ::= 'catch' TypeAndValue
+///   ::= 'filter'
+///   ::= 'filter' TypeAndValue ( ',' TypeAndValue )*
+bool LLParser::ParseLandingPad(Instruction *&Inst, PerFunctionState &PFS) {
+  Type *Ty = 0; LocTy TyLoc;
+  Value *PersFn; LocTy PersFnLoc;
+
+  if (ParseType(Ty, TyLoc) ||
+      ParseToken(lltok::kw_personality, "expected 'personality'") ||
+      ParseTypeAndValue(PersFn, PersFnLoc, PFS))
+    return true;
+
+  LandingPadInst *LP = LandingPadInst::Create(Ty, PersFn, 0);
+  LP->setCleanup(EatIfPresent(lltok::kw_cleanup));
+
+  while (Lex.getKind() == lltok::kw_catch || Lex.getKind() == lltok::kw_filter){
+    LandingPadInst::ClauseType CT;
+    if (EatIfPresent(lltok::kw_catch))
+      CT = LandingPadInst::Catch;
+    else if (EatIfPresent(lltok::kw_filter))
+      CT = LandingPadInst::Filter;
+    else
+      return TokError("expected 'catch' or 'filter' clause type");
+
+    Value *V; LocTy VLoc;
+    if (ParseTypeAndValue(V, VLoc, PFS)) {
+      delete LP;
+      return true;
+    }
+
+    // A 'catch' type expects a non-array constant. A filter clause expects an
+    // array constant.
+    if (CT == LandingPadInst::Catch) {
+      if (isa<ArrayType>(V->getType()))
+        Error(VLoc, "'catch' clause has an invalid type");
+    } else {
+      if (!isa<ArrayType>(V->getType()))
+        Error(VLoc, "'filter' clause has an invalid type");
+    }
+
+    LP->addClause(V);
+  }
+
+  Inst = LP;
+  return false;
+}
+
 /// ParseCall
 ///   ::= 'tail'? 'call' OptionalCallingConv OptionalAttrs Type Value
 ///       ParameterList OptionalAttrs
@@ -3498,8 +3588,8 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
   // If RetType is a non-function pointer type, then this is the short syntax
   // for the call, which means that RetType is just the return type.  Infer the
   // rest of the function argument types from the arguments that are present.
-  const PointerType *PFTy = 0;
-  const FunctionType *Ty = 0;
+  PointerType *PFTy = 0;
+  FunctionType *Ty = 0;
   if (!(PFTy = dyn_cast<PointerType>(RetType)) ||
       !(Ty = dyn_cast<FunctionType>(PFTy->getElementType()))) {
     // Pull out the types of all of the arguments...
@@ -3530,7 +3620,7 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
   FunctionType::param_iterator I = Ty->param_begin();
   FunctionType::param_iterator E = Ty->param_end();
   for (unsigned i = 0, e = ArgList.size(); i != e; ++i) {
-    const Type *ExpectedTy = 0;
+    Type *ExpectedTy = 0;
     if (I != E) {
       ExpectedTy = *I++;
     } else if (!Ty->isVarArg()) {
@@ -3596,34 +3686,85 @@ int LLParser::ParseAlloc(Instruction *&Inst, PerFunctionState &PFS) {
 }
 
 /// ParseLoad
-///   ::= 'volatile'? 'load' TypeAndValue (',' OptionalInfo)?
+///   ::= 'load' 'volatile'? TypeAndValue (',' 'align' i32)?
+///   ::= 'load' 'atomic' 'volatile'? TypeAndValue 
+///       'singlethread'? AtomicOrdering (',' 'align' i32)?
+///   Compatibility:
+///   ::= 'volatile' 'load' TypeAndValue (',' 'align' i32)?
 int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS,
                         bool isVolatile) {
   Value *Val; LocTy Loc;
   unsigned Alignment = 0;
   bool AteExtraComma = false;
+  bool isAtomic = false;
+  AtomicOrdering Ordering = NotAtomic;
+  SynchronizationScope Scope = CrossThread;
+
+  if (Lex.getKind() == lltok::kw_atomic) {
+    if (isVolatile)
+      return TokError("mixing atomic with old volatile placement");
+    isAtomic = true;
+    Lex.Lex();
+  }
+
+  if (Lex.getKind() == lltok::kw_volatile) {
+    if (isVolatile)
+      return TokError("duplicate volatile before and after store");
+    isVolatile = true;
+    Lex.Lex();
+  }
+
   if (ParseTypeAndValue(Val, Loc, PFS) ||
+      ParseScopeAndOrdering(isAtomic, Scope, Ordering) ||
       ParseOptionalCommaAlign(Alignment, AteExtraComma))
     return true;
 
   if (!Val->getType()->isPointerTy() ||
       !cast<PointerType>(Val->getType())->getElementType()->isFirstClassType())
     return Error(Loc, "load operand must be a pointer to a first class type");
+  if (isAtomic && !Alignment)
+    return Error(Loc, "atomic load must have explicit non-zero alignment");
+  if (Ordering == Release || Ordering == AcquireRelease)
+    return Error(Loc, "atomic load cannot use Release ordering");
 
-  Inst = new LoadInst(Val, "", isVolatile, Alignment);
+  Inst = new LoadInst(Val, "", isVolatile, Alignment, Ordering, Scope);
   return AteExtraComma ? InstExtraComma : InstNormal;
 }
 
 /// ParseStore
-///   ::= 'volatile'? 'store' TypeAndValue ',' TypeAndValue (',' 'align' i32)?
+
+///   ::= 'store' 'volatile'? TypeAndValue ',' TypeAndValue (',' 'align' i32)?
+///   ::= 'store' 'atomic' 'volatile'? TypeAndValue ',' TypeAndValue
+///       'singlethread'? AtomicOrdering (',' 'align' i32)?
+///   Compatibility:
+///   ::= 'volatile' 'store' TypeAndValue ',' TypeAndValue (',' 'align' i32)?
 int LLParser::ParseStore(Instruction *&Inst, PerFunctionState &PFS,
                          bool isVolatile) {
   Value *Val, *Ptr; LocTy Loc, PtrLoc;
   unsigned Alignment = 0;
   bool AteExtraComma = false;
+  bool isAtomic = false;
+  AtomicOrdering Ordering = NotAtomic;
+  SynchronizationScope Scope = CrossThread;
+
+  if (Lex.getKind() == lltok::kw_atomic) {
+    if (isVolatile)
+      return TokError("mixing atomic with old volatile placement");
+    isAtomic = true;
+    Lex.Lex();
+  }
+
+  if (Lex.getKind() == lltok::kw_volatile) {
+    if (isVolatile)
+      return TokError("duplicate volatile before and after store");
+    isVolatile = true;
+    Lex.Lex();
+  }
+
   if (ParseTypeAndValue(Val, Loc, PFS) ||
       ParseToken(lltok::comma, "expected ',' after store operand") ||
       ParseTypeAndValue(Ptr, PtrLoc, PFS) ||
+      ParseScopeAndOrdering(isAtomic, Scope, Ordering) ||
       ParseOptionalCommaAlign(Alignment, AteExtraComma))
     return true;
 
@@ -3633,11 +3774,131 @@ int LLParser::ParseStore(Instruction *&Inst, PerFunctionState &PFS,
     return Error(Loc, "store operand must be a first class value");
   if (cast<PointerType>(Ptr->getType())->getElementType() != Val->getType())
     return Error(Loc, "stored value and pointer type do not match");
+  if (isAtomic && !Alignment)
+    return Error(Loc, "atomic store must have explicit non-zero alignment");
+  if (Ordering == Acquire || Ordering == AcquireRelease)
+    return Error(Loc, "atomic store cannot use Acquire ordering");
 
-  Inst = new StoreInst(Val, Ptr, isVolatile, Alignment);
+  Inst = new StoreInst(Val, Ptr, isVolatile, Alignment, Ordering, Scope);
   return AteExtraComma ? InstExtraComma : InstNormal;
 }
 
+/// ParseCmpXchg
+///   ::= 'cmpxchg' 'volatile'? TypeAndValue ',' TypeAndValue ',' TypeAndValue
+///       'singlethread'? AtomicOrdering
+int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) {
+  Value *Ptr, *Cmp, *New; LocTy PtrLoc, CmpLoc, NewLoc;
+  bool AteExtraComma = false;
+  AtomicOrdering Ordering = NotAtomic;
+  SynchronizationScope Scope = CrossThread;
+  bool isVolatile = false;
+
+  if (EatIfPresent(lltok::kw_volatile))
+    isVolatile = true;
+
+  if (ParseTypeAndValue(Ptr, PtrLoc, PFS) ||
+      ParseToken(lltok::comma, "expected ',' after cmpxchg address") ||
+      ParseTypeAndValue(Cmp, CmpLoc, PFS) ||
+      ParseToken(lltok::comma, "expected ',' after cmpxchg cmp operand") ||
+      ParseTypeAndValue(New, NewLoc, PFS) ||
+      ParseScopeAndOrdering(true /*Always atomic*/, Scope, Ordering))
+    return true;
+
+  if (Ordering == Unordered)
+    return TokError("cmpxchg cannot be unordered");
+  if (!Ptr->getType()->isPointerTy())
+    return Error(PtrLoc, "cmpxchg operand must be a pointer");
+  if (cast<PointerType>(Ptr->getType())->getElementType() != Cmp->getType())
+    return Error(CmpLoc, "compare value and pointer type do not match");
+  if (cast<PointerType>(Ptr->getType())->getElementType() != New->getType())
+    return Error(NewLoc, "new value and pointer type do not match");
+  if (!New->getType()->isIntegerTy())
+    return Error(NewLoc, "cmpxchg operand must be an integer");
+  unsigned Size = New->getType()->getPrimitiveSizeInBits();
+  if (Size < 8 || (Size & (Size - 1)))
+    return Error(NewLoc, "cmpxchg operand must be power-of-two byte-sized"
+                         " integer");
+
+  AtomicCmpXchgInst *CXI =
+    new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, Scope);
+  CXI->setVolatile(isVolatile);
+  Inst = CXI;
+  return AteExtraComma ? InstExtraComma : InstNormal;
+}
+
+/// ParseAtomicRMW
+///   ::= 'atomicrmw' 'volatile'? BinOp TypeAndValue ',' TypeAndValue
+///       'singlethread'? AtomicOrdering
+int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
+  Value *Ptr, *Val; LocTy PtrLoc, ValLoc;
+  bool AteExtraComma = false;
+  AtomicOrdering Ordering = NotAtomic;
+  SynchronizationScope Scope = CrossThread;
+  bool isVolatile = false;
+  AtomicRMWInst::BinOp Operation;
+
+  if (EatIfPresent(lltok::kw_volatile))
+    isVolatile = true;
+
+  switch (Lex.getKind()) {
+  default: return TokError("expected binary operation in atomicrmw");
+  case lltok::kw_xchg: Operation = AtomicRMWInst::Xchg; break;
+  case lltok::kw_add: Operation = AtomicRMWInst::Add; break;
+  case lltok::kw_sub: Operation = AtomicRMWInst::Sub; break;
+  case lltok::kw_and: Operation = AtomicRMWInst::And; break;
+  case lltok::kw_nand: Operation = AtomicRMWInst::Nand; break;
+  case lltok::kw_or: Operation = AtomicRMWInst::Or; break;
+  case lltok::kw_xor: Operation = AtomicRMWInst::Xor; break;
+  case lltok::kw_max: Operation = AtomicRMWInst::Max; break;
+  case lltok::kw_min: Operation = AtomicRMWInst::Min; break;
+  case lltok::kw_umax: Operation = AtomicRMWInst::UMax; break;
+  case lltok::kw_umin: Operation = AtomicRMWInst::UMin; break;
+  }
+  Lex.Lex();  // Eat the operation.
+
+  if (ParseTypeAndValue(Ptr, PtrLoc, PFS) ||
+      ParseToken(lltok::comma, "expected ',' after atomicrmw address") ||
+      ParseTypeAndValue(Val, ValLoc, PFS) ||
+      ParseScopeAndOrdering(true /*Always atomic*/, Scope, Ordering))
+    return true;
+
+  if (Ordering == Unordered)
+    return TokError("atomicrmw cannot be unordered");
+  if (!Ptr->getType()->isPointerTy())
+    return Error(PtrLoc, "atomicrmw operand must be a pointer");
+  if (cast<PointerType>(Ptr->getType())->getElementType() != Val->getType())
+    return Error(ValLoc, "atomicrmw value and pointer type do not match");
+  if (!Val->getType()->isIntegerTy())
+    return Error(ValLoc, "atomicrmw operand must be an integer");
+  unsigned Size = Val->getType()->getPrimitiveSizeInBits();
+  if (Size < 8 || (Size & (Size - 1)))
+    return Error(ValLoc, "atomicrmw operand must be power-of-two byte-sized"
+                         " integer");
+
+  AtomicRMWInst *RMWI =
+    new AtomicRMWInst(Operation, Ptr, Val, Ordering, Scope);
+  RMWI->setVolatile(isVolatile);
+  Inst = RMWI;
+  return AteExtraComma ? InstExtraComma : InstNormal;
+}
+
+/// ParseFence
+///   ::= 'fence' 'singlethread'? AtomicOrdering
+int LLParser::ParseFence(Instruction *&Inst, PerFunctionState &PFS) {
+  AtomicOrdering Ordering = NotAtomic;
+  SynchronizationScope Scope = CrossThread;
+  if (ParseScopeAndOrdering(true /*Always atomic*/, Scope, Ordering))
+    return true;
+
+  if (Ordering == Unordered)
+    return TokError("fence cannot be unordered");
+  if (Ordering == Monotonic)
+    return TokError("fence cannot be monotonic");
+
+  Inst = new FenceInst(Context, Ordering, Scope);
+  return InstNormal;
+}
+
 /// ParseGetElementPtr
 ///   ::= 'getelementptr' 'inbounds'? TypeAndValue (',' TypeAndValue)*
 int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
@@ -3663,10 +3924,9 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
     Indices.push_back(Val);
   }
 
-  if (!GetElementPtrInst::getIndexedType(Ptr->getType(),
-                                         Indices.begin(), Indices.end()))
+  if (!GetElementPtrInst::getIndexedType(Ptr->getType(), Indices))
     return Error(Loc, "invalid getelementptr indices");
-  Inst = GetElementPtrInst::Create(Ptr, Indices.begin(), Indices.end());
+  Inst = GetElementPtrInst::Create(Ptr, Indices);
   if (InBounds)
     cast<GetElementPtrInst>(Inst)->setIsInBounds(true);
   return AteExtraComma ? InstExtraComma : InstNormal;
diff --git a/contrib/llvm/lib/AsmParser/LLParser.h b/contrib/llvm/lib/AsmParser/LLParser.h
index 963065785061..cbc3c23e8631 100644
--- a/contrib/llvm/lib/AsmParser/LLParser.h
+++ b/contrib/llvm/lib/AsmParser/LLParser.h
@@ -15,6 +15,7 @@
 #define LLVM_ASMPARSER_LLPARSER_H
 
 #include "LLLexer.h"
+#include "llvm/Instructions.h"
 #include "llvm/Module.h"
 #include "llvm/Type.h"
 #include "llvm/ADT/DenseMap.h"
@@ -142,8 +143,8 @@ namespace llvm {
     /// GetGlobalVal - Get a value with the specified name or ID, creating a
     /// forward reference record if needed.  This can return null if the value
     /// exists but does not have the right type.
-    GlobalValue *GetGlobalVal(const std::string &N, const Type *Ty, LocTy Loc);
-    GlobalValue *GetGlobalVal(unsigned ID, const Type *Ty, LocTy Loc);
+    GlobalValue *GetGlobalVal(const std::string &N, Type *Ty, LocTy Loc);
+    GlobalValue *GetGlobalVal(unsigned ID, Type *Ty, LocTy Loc);
 
     // Helper Routines.
     bool ParseToken(lltok::Kind T, const char *ErrMsg);
@@ -178,6 +179,8 @@ namespace llvm {
     bool ParseOptionalVisibility(unsigned &Visibility);
     bool ParseOptionalCallingConv(CallingConv::ID &CC);
     bool ParseOptionalAlignment(unsigned &Alignment);
+    bool ParseScopeAndOrdering(bool isAtomic, SynchronizationScope &Scope,
+                               AtomicOrdering &Ordering);
     bool ParseOptionalStackAlignment(unsigned &Alignment);
     bool ParseOptionalCommaAlign(unsigned &Alignment, bool &AteExtraComma);
     bool ParseIndexList(SmallVectorImpl<unsigned> &Indices,bool &AteExtraComma);
@@ -249,8 +252,8 @@ namespace llvm {
       /// GetVal - Get a value with the specified name or ID, creating a
       /// forward reference record if needed.  This can return null if the value
       /// exists but does not have the right type.
-      Value *GetVal(const std::string &Name, const Type *Ty, LocTy Loc);
-      Value *GetVal(unsigned ID, const Type *Ty, LocTy Loc);
+      Value *GetVal(const std::string &Name, Type *Ty, LocTy Loc);
+      Value *GetVal(unsigned ID, Type *Ty, LocTy Loc);
 
       /// SetInstName - After an instruction is parsed and inserted into its
       /// basic block, this installs its name.
@@ -269,14 +272,14 @@ namespace llvm {
       BasicBlock *DefineBB(const std::string &Name, LocTy Loc);
     };
 
-    bool ConvertValIDToValue(const Type *Ty, ValID &ID, Value *&V,
+    bool ConvertValIDToValue(Type *Ty, ValID &ID, Value *&V,
                              PerFunctionState *PFS);
 
-    bool ParseValue(const Type *Ty, Value *&V, PerFunctionState *PFS);
-    bool ParseValue(const Type *Ty, Value *&V, PerFunctionState &PFS) {
+    bool ParseValue(Type *Ty, Value *&V, PerFunctionState *PFS);
+    bool ParseValue(Type *Ty, Value *&V, PerFunctionState &PFS) {
       return ParseValue(Ty, V, &PFS);
     }
-    bool ParseValue(const Type *Ty, Value *&V, LocTy &Loc,
+    bool ParseValue(Type *Ty, Value *&V, LocTy &Loc,
                     PerFunctionState &PFS) {
       Loc = Lex.getLoc();
       return ParseValue(Ty, V, &PFS);
@@ -310,7 +313,7 @@ namespace llvm {
 
     // Constant Parsing.
     bool ParseValID(ValID &ID, PerFunctionState *PFS = NULL);
-    bool ParseGlobalValue(const Type *Ty, Constant *&V);
+    bool ParseGlobalValue(Type *Ty, Constant *&V);
     bool ParseGlobalTypeAndValue(Constant *&V);
     bool ParseGlobalValueVector(SmallVectorImpl<Constant*> &Elts);
     bool ParseMetadataListValue(ValID &ID, PerFunctionState *PFS);
@@ -344,6 +347,7 @@ namespace llvm {
     bool ParseSwitch(Instruction *&Inst, PerFunctionState &PFS);
     bool ParseIndirectBr(Instruction *&Inst, PerFunctionState &PFS);
     bool ParseInvoke(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseResume(Instruction *&Inst, PerFunctionState &PFS);
 
     bool ParseArithmetic(Instruction *&I, PerFunctionState &PFS, unsigned Opc,
                          unsigned OperandType);
@@ -356,10 +360,14 @@ namespace llvm {
     bool ParseInsertElement(Instruction *&I, PerFunctionState &PFS);
     bool ParseShuffleVector(Instruction *&I, PerFunctionState &PFS);
     int ParsePHI(Instruction *&I, PerFunctionState &PFS);
+    bool ParseLandingPad(Instruction *&I, PerFunctionState &PFS);
     bool ParseCall(Instruction *&I, PerFunctionState &PFS, bool isTail);
     int ParseAlloc(Instruction *&I, PerFunctionState &PFS);
     int ParseLoad(Instruction *&I, PerFunctionState &PFS, bool isVolatile);
     int ParseStore(Instruction *&I, PerFunctionState &PFS, bool isVolatile);
+    int ParseCmpXchg(Instruction *&I, PerFunctionState &PFS);
+    int ParseAtomicRMW(Instruction *&I, PerFunctionState &PFS);
+    int ParseFence(Instruction *&I, PerFunctionState &PFS);
     int ParseGetElementPtr(Instruction *&I, PerFunctionState &PFS);
     int ParseExtractValue(Instruction *&I, PerFunctionState &PFS);
     int ParseInsertValue(Instruction *&I, PerFunctionState &PFS);
diff --git a/contrib/llvm/lib/AsmParser/LLToken.h b/contrib/llvm/lib/AsmParser/LLToken.h
index a5f89fcce0c0..8f167725ed58 100644
--- a/contrib/llvm/lib/AsmParser/LLToken.h
+++ b/contrib/llvm/lib/AsmParser/LLToken.h
@@ -53,6 +53,9 @@ namespace lltok {
     kw_deplibs,
     kw_datalayout,
     kw_volatile,
+    kw_atomic,
+    kw_unordered, kw_monotonic, kw_acquire, kw_release, kw_acq_rel, kw_seq_cst,
+    kw_singlethread,
     kw_nuw,
     kw_nsw,
     kw_exact,
@@ -87,6 +90,7 @@ namespace lltok {
     kw_readnone,
     kw_readonly,
     kw_uwtable,
+    kw_returns_twice,
 
     kw_inlinehint,
     kw_noinline,
@@ -97,7 +101,6 @@ namespace lltok {
     kw_noredzone,
     kw_noimplicitfloat,
     kw_naked,
-    kw_hotpatch,
     kw_nonlazybind,
 
     kw_type,
@@ -107,6 +110,9 @@ namespace lltok {
     kw_uge, kw_oeq, kw_one, kw_olt, kw_ogt, kw_ole, kw_oge, kw_ord, kw_uno,
     kw_ueq, kw_une,
 
+    // atomicrmw operations that aren't also instruction keywords.
+    kw_xchg, kw_nand, kw_max, kw_min, kw_umax, kw_umin,
+
     // Instruction Opcodes (Opcode in UIntVal).
     kw_add,  kw_fadd, kw_sub,  kw_fsub, kw_mul,  kw_fmul,
     kw_udiv, kw_sdiv, kw_fdiv,
@@ -118,10 +124,13 @@ namespace lltok {
     kw_fptoui, kw_fptosi, kw_inttoptr, kw_ptrtoint, kw_bitcast,
     kw_select, kw_va_arg,
 
-    kw_ret, kw_br, kw_switch, kw_indirectbr, kw_invoke, kw_unwind,
+    kw_landingpad, kw_personality, kw_cleanup, kw_catch, kw_filter,
+
+    kw_ret, kw_br, kw_switch, kw_indirectbr, kw_invoke, kw_unwind, kw_resume,
     kw_unreachable,
 
-    kw_alloca, kw_load, kw_store, kw_getelementptr,
+    kw_alloca, kw_load, kw_store, kw_fence, kw_cmpxchg, kw_atomicrmw,
+    kw_getelementptr,
 
     kw_extractelement, kw_insertelement, kw_shufflevector,
     kw_extractvalue, kw_insertvalue, kw_blockaddress,
diff --git a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 24c29941cf16..46565f36af16 100644
--- a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -107,7 +107,7 @@ static int GetDecodedCastOpcode(unsigned Val) {
   case bitc::CAST_BITCAST : return Instruction::BitCast;
   }
 }
-static int GetDecodedBinaryOpcode(unsigned Val, const Type *Ty) {
+static int GetDecodedBinaryOpcode(unsigned Val, Type *Ty) {
   switch (Val) {
   default: return -1;
   case bitc::BINOP_ADD:
@@ -131,6 +131,44 @@ static int GetDecodedBinaryOpcode(unsigned Val, const Type *Ty) {
   }
 }
 
+static AtomicRMWInst::BinOp GetDecodedRMWOperation(unsigned Val) {
+  switch (Val) {
+  default: return AtomicRMWInst::BAD_BINOP;
+  case bitc::RMW_XCHG: return AtomicRMWInst::Xchg;
+  case bitc::RMW_ADD: return AtomicRMWInst::Add;
+  case bitc::RMW_SUB: return AtomicRMWInst::Sub;
+  case bitc::RMW_AND: return AtomicRMWInst::And;
+  case bitc::RMW_NAND: return AtomicRMWInst::Nand;
+  case bitc::RMW_OR: return AtomicRMWInst::Or;
+  case bitc::RMW_XOR: return AtomicRMWInst::Xor;
+  case bitc::RMW_MAX: return AtomicRMWInst::Max;
+  case bitc::RMW_MIN: return AtomicRMWInst::Min;
+  case bitc::RMW_UMAX: return AtomicRMWInst::UMax;
+  case bitc::RMW_UMIN: return AtomicRMWInst::UMin;
+  }
+}
+
+static AtomicOrdering GetDecodedOrdering(unsigned Val) {
+  switch (Val) {
+  case bitc::ORDERING_NOTATOMIC: return NotAtomic;
+  case bitc::ORDERING_UNORDERED: return Unordered;
+  case bitc::ORDERING_MONOTONIC: return Monotonic;
+  case bitc::ORDERING_ACQUIRE: return Acquire;
+  case bitc::ORDERING_RELEASE: return Release;
+  case bitc::ORDERING_ACQREL: return AcquireRelease;
+  default: // Map unknown orderings to sequentially-consistent.
+  case bitc::ORDERING_SEQCST: return SequentiallyConsistent;
+  }
+}
+
+static SynchronizationScope GetDecodedSynchScope(unsigned Val) {
+  switch (Val) {
+  case bitc::SYNCHSCOPE_SINGLETHREAD: return SingleThread;
+  default: // Map unknown scopes to cross-thread.
+  case bitc::SYNCHSCOPE_CROSSTHREAD: return CrossThread;
+  }
+}
+
 namespace llvm {
 namespace {
   /// @brief A class for maintaining the slot number definition
@@ -142,7 +180,7 @@ namespace {
     void *operator new(size_t s) {
       return User::operator new(s, 1);
     }
-    explicit ConstantPlaceHolder(const Type *Ty, LLVMContext& Context)
+    explicit ConstantPlaceHolder(Type *Ty, LLVMContext& Context)
       : ConstantExpr(Ty, Instruction::UserOp1, &Op<0>(), 1) {
       Op<0>() = UndefValue::get(Type::getInt32Ty(Context));
     }
@@ -198,7 +236,7 @@ void BitcodeReaderValueList::AssignValue(Value *V, unsigned Idx) {
 
 
 Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx,
-                                                    const Type *Ty) {
+                                                    Type *Ty) {
   if (Idx >= size())
     resize(Idx + 1);
 
@@ -213,7 +251,7 @@ Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx,
   return C;
 }
 
-Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, const Type *Ty) {
+Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) {
   if (Idx >= size())
     resize(Idx + 1);
 
@@ -362,7 +400,7 @@ Type *BitcodeReader::getTypeByID(unsigned ID) {
 
   // If we have a forward reference, the only possible case is when it is to a
   // named struct.  Just create a placeholder for now.
-  return TypeList[ID] = StructType::createNamed(Context, "");
+  return TypeList[ID] = StructType::create(Context);
 }
 
 /// FIXME: Remove in LLVM 3.1, only used by ParseOldTypeTable.
@@ -630,7 +668,7 @@ bool BitcodeReader::ParseTypeTableBody() {
         Res->setName(TypeName);
         TypeList[NumRecords] = 0;
       } else  // Otherwise, create a new struct.
-        Res = StructType::createNamed(Context, TypeName);
+        Res = StructType::create(Context, TypeName);
       TypeName.clear();
       
       SmallVector<Type*, 8> EltTys;
@@ -659,7 +697,7 @@ bool BitcodeReader::ParseTypeTableBody() {
         Res->setName(TypeName);
         TypeList[NumRecords] = 0;
       } else  // Otherwise, create a new struct with no body.
-        Res = StructType::createNamed(Context, TypeName);
+        Res = StructType::create(Context, TypeName);
       TypeName.clear();
       ResultTy = Res;
       break;
@@ -793,7 +831,7 @@ bool BitcodeReader::ParseOldTypeTable() {
       break;
     case bitc::TYPE_CODE_OPAQUE:    // OPAQUE
       if (NextTypeID < TypeList.size() && TypeList[NextTypeID] == 0)
-        ResultTy = StructType::createNamed(Context, "");
+        ResultTy = StructType::create(Context);
       break;
     case bitc::TYPE_CODE_STRUCT_OLD: {// STRUCT_OLD
       if (NextTypeID >= TypeList.size()) break;
@@ -804,7 +842,7 @@ bool BitcodeReader::ParseOldTypeTable() {
 
       // Set a type.
       if (TypeList[NextTypeID] == 0)
-        TypeList[NextTypeID] = StructType::createNamed(Context, "");
+        TypeList[NextTypeID] = StructType::create(Context);
 
       std::vector<Type*> EltTys;
       for (unsigned i = 1, e = Record.size(); i != e; ++i) {
@@ -923,7 +961,7 @@ bool BitcodeReader::ParseOldTypeSymbolTable() {
 
       // Only apply the type name to a struct type with no name.
       if (StructType *STy = dyn_cast<StructType>(TypeList[TypeID]))
-        if (!STy->isAnonymous() && !STy->hasName())
+        if (!STy->isLiteral() && !STy->hasName())
           STy->setName(TypeName);
       TypeName.clear();
       break;
@@ -1063,7 +1101,7 @@ bool BitcodeReader::ParseMetadata() {
       unsigned Size = Record.size();
       SmallVector<Value*, 8> Elts;
       for (unsigned i = 0; i != Size; i += 2) {
-        const Type *Ty = getTypeByID(Record[i]);
+        Type *Ty = getTypeByID(Record[i]);
         if (!Ty) return Error("Invalid METADATA_NODE record");
         if (Ty->isMetadataTy())
           Elts.push_back(MDValueList.getValueFwdRef(Record[i+1]));
@@ -1163,7 +1201,7 @@ bool BitcodeReader::ParseConstants() {
   SmallVector<uint64_t, 64> Record;
 
   // Read all the records for this value table.
-  const Type *CurTy = Type::getInt32Ty(Context);
+  Type *CurTy = Type::getInt32Ty(Context);
   unsigned NextCstNo = ValueList.size();
   while (1) {
     unsigned Code = Stream.ReadCode();
@@ -1218,7 +1256,7 @@ bool BitcodeReader::ParseConstants() {
         Words[i] = DecodeSignRotatedValue(Record[i]);
       V = ConstantInt::get(Context,
                            APInt(cast<IntegerType>(CurTy)->getBitWidth(),
-                           NumWords, &Words[0]));
+                                 Words));
       break;
     }
     case bitc::CST_CODE_FLOAT: {    // FLOAT: [fpval]
@@ -1233,11 +1271,11 @@ bool BitcodeReader::ParseConstants() {
         uint64_t Rearrange[2];
         Rearrange[0] = (Record[1] & 0xffffLL) | (Record[0] << 16);
         Rearrange[1] = Record[0] >> 48;
-        V = ConstantFP::get(Context, APFloat(APInt(80, 2, Rearrange)));
+        V = ConstantFP::get(Context, APFloat(APInt(80, Rearrange)));
       } else if (CurTy->isFP128Ty())
-        V = ConstantFP::get(Context, APFloat(APInt(128, 2, &Record[0]), true));
+        V = ConstantFP::get(Context, APFloat(APInt(128, Record), true));
       else if (CurTy->isPPC_FP128Ty())
-        V = ConstantFP::get(Context, APFloat(APInt(128, 2, &Record[0])));
+        V = ConstantFP::get(Context, APFloat(APInt(128, Record)));
       else
         V = UndefValue::get(CurTy);
       break;
@@ -1250,18 +1288,18 @@ bool BitcodeReader::ParseConstants() {
       unsigned Size = Record.size();
       std::vector<Constant*> Elts;
 
-      if (const StructType *STy = dyn_cast<StructType>(CurTy)) {
+      if (StructType *STy = dyn_cast<StructType>(CurTy)) {
         for (unsigned i = 0; i != Size; ++i)
           Elts.push_back(ValueList.getConstantFwdRef(Record[i],
                                                      STy->getElementType(i)));
         V = ConstantStruct::get(STy, Elts);
-      } else if (const ArrayType *ATy = dyn_cast<ArrayType>(CurTy)) {
-        const Type *EltTy = ATy->getElementType();
+      } else if (ArrayType *ATy = dyn_cast<ArrayType>(CurTy)) {
+        Type *EltTy = ATy->getElementType();
         for (unsigned i = 0; i != Size; ++i)
           Elts.push_back(ValueList.getConstantFwdRef(Record[i], EltTy));
         V = ConstantArray::get(ATy, Elts);
-      } else if (const VectorType *VTy = dyn_cast<VectorType>(CurTy)) {
-        const Type *EltTy = VTy->getElementType();
+      } else if (VectorType *VTy = dyn_cast<VectorType>(CurTy)) {
+        Type *EltTy = VTy->getElementType();
         for (unsigned i = 0; i != Size; ++i)
           Elts.push_back(ValueList.getConstantFwdRef(Record[i], EltTy));
         V = ConstantVector::get(Elts);
@@ -1274,8 +1312,8 @@ bool BitcodeReader::ParseConstants() {
       if (Record.empty())
         return Error("Invalid CST_AGGREGATE record");
 
-      const ArrayType *ATy = cast<ArrayType>(CurTy);
-      const Type *EltTy = ATy->getElementType();
+      ArrayType *ATy = cast<ArrayType>(CurTy);
+      Type *EltTy = ATy->getElementType();
 
       unsigned Size = Record.size();
       std::vector<Constant*> Elts;
@@ -1288,8 +1326,8 @@ bool BitcodeReader::ParseConstants() {
       if (Record.empty())
         return Error("Invalid CST_AGGREGATE record");
 
-      const ArrayType *ATy = cast<ArrayType>(CurTy);
-      const Type *EltTy = ATy->getElementType();
+      ArrayType *ATy = cast<ArrayType>(CurTy);
+      Type *EltTy = ATy->getElementType();
 
       unsigned Size = Record.size();
       std::vector<Constant*> Elts;
@@ -1335,7 +1373,7 @@ bool BitcodeReader::ParseConstants() {
       if (Opc < 0) {
         V = UndefValue::get(CurTy);  // Unknown cast.
       } else {
-        const Type *OpTy = getTypeByID(Record[1]);
+        Type *OpTy = getTypeByID(Record[1]);
         if (!OpTy) return Error("Invalid CE_CAST record");
         Constant *Op = ValueList.getConstantFwdRef(Record[2], OpTy);
         V = ConstantExpr::getCast(Opc, Op, CurTy);
@@ -1347,16 +1385,14 @@ bool BitcodeReader::ParseConstants() {
       if (Record.size() & 1) return Error("Invalid CE_GEP record");
       SmallVector<Constant*, 16> Elts;
       for (unsigned i = 0, e = Record.size(); i != e; i += 2) {
-        const Type *ElTy = getTypeByID(Record[i]);
+        Type *ElTy = getTypeByID(Record[i]);
         if (!ElTy) return Error("Invalid CE_GEP record");
         Elts.push_back(ValueList.getConstantFwdRef(Record[i+1], ElTy));
       }
-      if (BitCode == bitc::CST_CODE_CE_INBOUNDS_GEP)
-        V = ConstantExpr::getInBoundsGetElementPtr(Elts[0], &Elts[1],
-                                                   Elts.size()-1);
-      else
-        V = ConstantExpr::getGetElementPtr(Elts[0], &Elts[1],
-                                           Elts.size()-1);
+      ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end());
+      V = ConstantExpr::getGetElementPtr(Elts[0], Indices,
+                                         BitCode ==
+                                           bitc::CST_CODE_CE_INBOUNDS_GEP);
       break;
     }
     case bitc::CST_CODE_CE_SELECT:  // CE_SELECT: [opval#, opval#, opval#]
@@ -1368,7 +1404,7 @@ bool BitcodeReader::ParseConstants() {
       break;
     case bitc::CST_CODE_CE_EXTRACTELT: { // CE_EXTRACTELT: [opty, opval, opval]
       if (Record.size() < 3) return Error("Invalid CE_EXTRACTELT record");
-      const VectorType *OpTy =
+      VectorType *OpTy =
         dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
       if (OpTy == 0) return Error("Invalid CE_EXTRACTELT record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
@@ -1377,7 +1413,7 @@ bool BitcodeReader::ParseConstants() {
       break;
     }
     case bitc::CST_CODE_CE_INSERTELT: { // CE_INSERTELT: [opval, opval, opval]
-      const VectorType *OpTy = dyn_cast<VectorType>(CurTy);
+      VectorType *OpTy = dyn_cast<VectorType>(CurTy);
       if (Record.size() < 3 || OpTy == 0)
         return Error("Invalid CE_INSERTELT record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
@@ -1388,26 +1424,26 @@ bool BitcodeReader::ParseConstants() {
       break;
     }
     case bitc::CST_CODE_CE_SHUFFLEVEC: { // CE_SHUFFLEVEC: [opval, opval, opval]
-      const VectorType *OpTy = dyn_cast<VectorType>(CurTy);
+      VectorType *OpTy = dyn_cast<VectorType>(CurTy);
       if (Record.size() < 3 || OpTy == 0)
         return Error("Invalid CE_SHUFFLEVEC record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
-      const Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
+      Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
                                                  OpTy->getNumElements());
       Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy);
       V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
       break;
     }
     case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
-      const VectorType *RTy = dyn_cast<VectorType>(CurTy);
-      const VectorType *OpTy =
+      VectorType *RTy = dyn_cast<VectorType>(CurTy);
+      VectorType *OpTy =
         dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
       if (Record.size() < 4 || RTy == 0 || OpTy == 0)
         return Error("Invalid CE_SHUFVEC_EX record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
-      const Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
+      Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
                                                  RTy->getNumElements());
       Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy);
       V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
@@ -1415,7 +1451,7 @@ bool BitcodeReader::ParseConstants() {
     }
     case bitc::CST_CODE_CE_CMP: {     // CE_CMP: [opty, opval, opval, pred]
       if (Record.size() < 4) return Error("Invalid CE_CMP record");
-      const Type *OpTy = getTypeByID(Record[0]);
+      Type *OpTy = getTypeByID(Record[0]);
       if (OpTy == 0) return Error("Invalid CE_CMP record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
@@ -1442,14 +1478,14 @@ bool BitcodeReader::ParseConstants() {
         AsmStr += (char)Record[2+i];
       for (unsigned i = 0; i != ConstStrSize; ++i)
         ConstrStr += (char)Record[3+AsmStrSize+i];
-      const PointerType *PTy = cast<PointerType>(CurTy);
+      PointerType *PTy = cast<PointerType>(CurTy);
       V = InlineAsm::get(cast<FunctionType>(PTy->getElementType()),
                          AsmStr, ConstrStr, HasSideEffects, IsAlignStack);
       break;
     }
     case bitc::CST_CODE_BLOCKADDRESS:{
       if (Record.size() < 3) return Error("Invalid CE_BLOCKADDRESS record");
-      const Type *FnTy = getTypeByID(Record[0]);
+      Type *FnTy = getTypeByID(Record[0]);
       if (FnTy == 0) return Error("Invalid CE_BLOCKADDRESS record");
       Function *Fn =
         dyn_cast_or_null<Function>(ValueList.getConstantFwdRef(Record[1],FnTy));
@@ -1662,7 +1698,7 @@ bool BitcodeReader::ParseModule() {
     case bitc::MODULE_CODE_GLOBALVAR: {
       if (Record.size() < 6)
         return Error("Invalid MODULE_CODE_GLOBALVAR record");
-      const Type *Ty = getTypeByID(Record[0]);
+      Type *Ty = getTypeByID(Record[0]);
       if (!Ty) return Error("Invalid MODULE_CODE_GLOBALVAR record");
       if (!Ty->isPointerTy())
         return Error("Global not a pointer type!");
@@ -1711,11 +1747,11 @@ bool BitcodeReader::ParseModule() {
     case bitc::MODULE_CODE_FUNCTION: {
       if (Record.size() < 8)
         return Error("Invalid MODULE_CODE_FUNCTION record");
-      const Type *Ty = getTypeByID(Record[0]);
+      Type *Ty = getTypeByID(Record[0]);
       if (!Ty) return Error("Invalid MODULE_CODE_FUNCTION record");
       if (!Ty->isPointerTy())
         return Error("Function not a pointer type!");
-      const FunctionType *FTy =
+      FunctionType *FTy =
         dyn_cast<FunctionType>(cast<PointerType>(Ty)->getElementType());
       if (!FTy)
         return Error("Function not a pointer to function type!");
@@ -1757,7 +1793,7 @@ bool BitcodeReader::ParseModule() {
     case bitc::MODULE_CODE_ALIAS: {
       if (Record.size() < 3)
         return Error("Invalid MODULE_ALIAS record");
-      const Type *Ty = getTypeByID(Record[0]);
+      Type *Ty = getTypeByID(Record[0]);
       if (!Ty) return Error("Invalid MODULE_ALIAS record");
       if (!Ty->isPointerTy())
         return Error("Function not a pointer type!");
@@ -1823,9 +1859,9 @@ bool BitcodeReader::ParseBitcodeInto(Module *M) {
 
     if (Code != bitc::ENTER_SUBBLOCK) {
 
-      // The ranlib in xcode 4 will align archive members by appending newlines to the
-      // end of them. If this file size is a multiple of 4 but not 8, we have to read and
-      // ignore these final 4 bytes :-(
+      // The ranlib in xcode 4 will align archive members by appending newlines
+      // to the end of them. If this file size is a multiple of 4 but not 8, we
+      // have to read and ignore these final 4 bytes :-(
       if (Stream.GetAbbrevIDWidth() == 2 && Code == 2 &&
           Stream.Read(6) == 2 && Stream.Read(24) == 0xa0a0a &&
 	  Stream.AtEndOfStream())
@@ -2160,7 +2196,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
           OpNum+2 != Record.size())
         return Error("Invalid CAST record");
 
-      const Type *ResTy = getTypeByID(Record[OpNum]);
+      Type *ResTy = getTypeByID(Record[OpNum]);
       int Opc = GetDecodedCastOpcode(Record[OpNum+1]);
       if (Opc == -1 || ResTy == 0)
         return Error("Invalid CAST record");
@@ -2183,7 +2219,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
         GEPIdx.push_back(Op);
       }
 
-      I = GetElementPtrInst::Create(BasePtr, GEPIdx.begin(), GEPIdx.end());
+      I = GetElementPtrInst::Create(BasePtr, GEPIdx);
       InstructionList.push_back(I);
       if (BitCode == bitc::FUNC_CODE_INST_INBOUNDS_GEP)
         cast<GetElementPtrInst>(I)->setIsInBounds(true);
@@ -2261,8 +2297,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
         return Error("Invalid SELECT record");
 
       // select condition can be either i1 or [N x i1]
-      if (const VectorType* vector_type =
-          dyn_cast<const VectorType>(Cond->getType())) {
+      if (VectorType* vector_type =
+          dyn_cast<VectorType>(Cond->getType())) {
         // expect <n x i1>
         if (vector_type->getElementType() != Type::getInt1Ty(Context))
           return Error("Invalid SELECT condition type");
@@ -2381,7 +2417,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_SWITCH: { // SWITCH: [opty, op0, op1, ...]
       if (Record.size() < 3 || (Record.size() & 1) == 0)
         return Error("Invalid SWITCH record");
-      const Type *OpTy = getTypeByID(Record[0]);
+      Type *OpTy = getTypeByID(Record[0]);
       Value *Cond = getFnValueByID(Record[1], OpTy);
       BasicBlock *Default = getBasicBlock(Record[2]);
       if (OpTy == 0 || Cond == 0 || Default == 0)
@@ -2405,7 +2441,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_INDIRECTBR: { // INDIRECTBR: [opty, op0, op1, ...]
       if (Record.size() < 2)
         return Error("Invalid INDIRECTBR record");
-      const Type *OpTy = getTypeByID(Record[0]);
+      Type *OpTy = getTypeByID(Record[0]);
       Value *Address = getFnValueByID(Record[1], OpTy);
       if (OpTy == 0 || Address == 0)
         return Error("Invalid INDIRECTBR record");
@@ -2437,8 +2473,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       if (getValueTypePair(Record, OpNum, NextValueNo, Callee))
         return Error("Invalid INVOKE record");
 
-      const PointerType *CalleeTy = dyn_cast<PointerType>(Callee->getType());
-      const FunctionType *FTy = !CalleeTy ? 0 :
+      PointerType *CalleeTy = dyn_cast<PointerType>(Callee->getType());
+      FunctionType *FTy = !CalleeTy ? 0 :
         dyn_cast<FunctionType>(CalleeTy->getElementType());
 
       // Check that the right number of fixed parameters are here.
@@ -2472,6 +2508,15 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       cast<InvokeInst>(I)->setAttributes(PAL);
       break;
     }
+    case bitc::FUNC_CODE_INST_RESUME: { // RESUME: [opval]
+      unsigned Idx = 0;
+      Value *Val = 0;
+      if (getValueTypePair(Record, Idx, NextValueNo, Val))
+        return Error("Invalid RESUME record");
+      I = ResumeInst::Create(Val);
+      InstructionList.push_back(I);
+      break;
+    }
     case bitc::FUNC_CODE_INST_UNWIND: // UNWIND
       I = new UnwindInst(Context);
       InstructionList.push_back(I);
@@ -2483,7 +2528,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...]
       if (Record.size() < 1 || ((Record.size()-1)&1))
         return Error("Invalid PHI record");
-      const Type *Ty = getTypeByID(Record[0]);
+      Type *Ty = getTypeByID(Record[0]);
       if (!Ty) return Error("Invalid PHI record");
 
       PHINode *PN = PHINode::Create(Ty, (Record.size()-1)/2);
@@ -2499,12 +2544,51 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       break;
     }
 
+    case bitc::FUNC_CODE_INST_LANDINGPAD: {
+      // LANDINGPAD: [ty, val, val, num, (id0,val0 ...)?]
+      unsigned Idx = 0;
+      if (Record.size() < 4)
+        return Error("Invalid LANDINGPAD record");
+      Type *Ty = getTypeByID(Record[Idx++]);
+      if (!Ty) return Error("Invalid LANDINGPAD record");
+      Value *PersFn = 0;
+      if (getValueTypePair(Record, Idx, NextValueNo, PersFn))
+        return Error("Invalid LANDINGPAD record");
+
+      bool IsCleanup = !!Record[Idx++];
+      unsigned NumClauses = Record[Idx++];
+      LandingPadInst *LP = LandingPadInst::Create(Ty, PersFn, NumClauses);
+      LP->setCleanup(IsCleanup);
+      for (unsigned J = 0; J != NumClauses; ++J) {
+        LandingPadInst::ClauseType CT =
+          LandingPadInst::ClauseType(Record[Idx++]); (void)CT;
+        Value *Val;
+
+        if (getValueTypePair(Record, Idx, NextValueNo, Val)) {
+          delete LP;
+          return Error("Invalid LANDINGPAD record");
+        }
+
+        assert((CT != LandingPadInst::Catch ||
+                !isa<ArrayType>(Val->getType())) &&
+               "Catch clause has a invalid type!");
+        assert((CT != LandingPadInst::Filter ||
+                isa<ArrayType>(Val->getType())) &&
+               "Filter clause has invalid type!");
+        LP->addClause(Val);
+      }
+
+      I = LP;
+      InstructionList.push_back(I);
+      break;
+    }
+
     case bitc::FUNC_CODE_INST_ALLOCA: { // ALLOCA: [instty, opty, op, align]
       if (Record.size() != 4)
         return Error("Invalid ALLOCA record");
-      const PointerType *Ty =
+      PointerType *Ty =
         dyn_cast_or_null<PointerType>(getTypeByID(Record[0]));
-      const Type *OpTy = getTypeByID(Record[1]);
+      Type *OpTy = getTypeByID(Record[1]);
       Value *Size = getFnValueByID(Record[2], OpTy);
       unsigned Align = Record[3];
       if (!Ty || !Size) return Error("Invalid ALLOCA record");
@@ -2523,6 +2607,28 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     }
+    case bitc::FUNC_CODE_INST_LOADATOMIC: {
+       // LOADATOMIC: [opty, op, align, vol, ordering, synchscope]
+      unsigned OpNum = 0;
+      Value *Op;
+      if (getValueTypePair(Record, OpNum, NextValueNo, Op) ||
+          OpNum+4 != Record.size())
+        return Error("Invalid LOADATOMIC record");
+        
+
+      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+2]);
+      if (Ordering == NotAtomic || Ordering == Release ||
+          Ordering == AcquireRelease)
+        return Error("Invalid LOADATOMIC record");
+      if (Ordering != NotAtomic && Record[OpNum] == 0)
+        return Error("Invalid LOADATOMIC record");
+      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+3]);
+
+      I = new LoadInst(Op, "", Record[OpNum+1], (1 << Record[OpNum]) >> 1,
+                       Ordering, SynchScope);
+      InstructionList.push_back(I);
+      break;
+    }
     case bitc::FUNC_CODE_INST_STORE: { // STORE2:[ptrty, ptr, val, align, vol]
       unsigned OpNum = 0;
       Value *Val, *Ptr;
@@ -2536,6 +2642,83 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     }
+    case bitc::FUNC_CODE_INST_STOREATOMIC: {
+      // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, synchscope]
+      unsigned OpNum = 0;
+      Value *Val, *Ptr;
+      if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
+          getValue(Record, OpNum,
+                    cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
+          OpNum+4 != Record.size())
+        return Error("Invalid STOREATOMIC record");
+
+      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+2]);
+      if (Ordering == NotAtomic || Ordering == Acquire ||
+          Ordering == AcquireRelease)
+        return Error("Invalid STOREATOMIC record");
+      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+3]);
+      if (Ordering != NotAtomic && Record[OpNum] == 0)
+        return Error("Invalid STOREATOMIC record");
+
+      I = new StoreInst(Val, Ptr, Record[OpNum+1], (1 << Record[OpNum]) >> 1,
+                        Ordering, SynchScope);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_CMPXCHG: {
+      // CMPXCHG:[ptrty, ptr, cmp, new, vol, ordering, synchscope]
+      unsigned OpNum = 0;
+      Value *Ptr, *Cmp, *New;
+      if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
+          getValue(Record, OpNum,
+                    cast<PointerType>(Ptr->getType())->getElementType(), Cmp) ||
+          getValue(Record, OpNum,
+                    cast<PointerType>(Ptr->getType())->getElementType(), New) ||
+          OpNum+3 != Record.size())
+        return Error("Invalid CMPXCHG record");
+      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+1]);
+      if (Ordering == NotAtomic || Ordering == Unordered)
+        return Error("Invalid CMPXCHG record");
+      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+2]);
+      I = new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, SynchScope);
+      cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_ATOMICRMW: {
+      // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, synchscope]
+      unsigned OpNum = 0;
+      Value *Ptr, *Val;
+      if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
+          getValue(Record, OpNum,
+                    cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
+          OpNum+4 != Record.size())
+        return Error("Invalid ATOMICRMW record");
+      AtomicRMWInst::BinOp Operation = GetDecodedRMWOperation(Record[OpNum]);
+      if (Operation < AtomicRMWInst::FIRST_BINOP ||
+          Operation > AtomicRMWInst::LAST_BINOP)
+        return Error("Invalid ATOMICRMW record");
+      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+2]);
+      if (Ordering == NotAtomic || Ordering == Unordered)
+        return Error("Invalid ATOMICRMW record");
+      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+3]);
+      I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SynchScope);
+      cast<AtomicRMWInst>(I)->setVolatile(Record[OpNum+1]);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, synchscope]
+      if (2 != Record.size())
+        return Error("Invalid FENCE record");
+      AtomicOrdering Ordering = GetDecodedOrdering(Record[0]);
+      if (Ordering == NotAtomic || Ordering == Unordered ||
+          Ordering == Monotonic)
+        return Error("Invalid FENCE record");
+      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[1]);
+      I = new FenceInst(Context, Ordering, SynchScope);
+      InstructionList.push_back(I);
+      break;
+    }
     case bitc::FUNC_CODE_INST_CALL: {
       // CALL: [paramattrs, cc, fnty, fnid, arg0, arg1...]
       if (Record.size() < 3)
@@ -2549,8 +2732,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       if (getValueTypePair(Record, OpNum, NextValueNo, Callee))
         return Error("Invalid CALL record");
 
-      const PointerType *OpTy = dyn_cast<PointerType>(Callee->getType());
-      const FunctionType *FTy = 0;
+      PointerType *OpTy = dyn_cast<PointerType>(Callee->getType());
+      FunctionType *FTy = 0;
       if (OpTy) FTy = dyn_cast<FunctionType>(OpTy->getElementType());
       if (!FTy || Record.size() < FTy->getNumParams()+OpNum)
         return Error("Invalid CALL record");
@@ -2589,9 +2772,9 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_VAARG: { // VAARG: [valistty, valist, instty]
       if (Record.size() < 3)
         return Error("Invalid VAARG record");
-      const Type *OpTy = getTypeByID(Record[0]);
+      Type *OpTy = getTypeByID(Record[0]);
       Value *Op = getFnValueByID(Record[1], OpTy);
-      const Type *ResTy = getTypeByID(Record[2]);
+      Type *ResTy = getTypeByID(Record[2]);
       if (!OpTy || !Op || !ResTy)
         return Error("Invalid VAARG record");
       I = new VAArgInst(Op, ResTy);
@@ -2756,6 +2939,9 @@ bool BitcodeReader::MaterializeModule(Module *M, std::string *ErrInfo) {
   }
   std::vector<std::pair<Function*, Function*> >().swap(UpgradedIntrinsics);
 
+  // Upgrade to new EH scheme. N.B. This will go away in 3.1.
+  UpgradeExceptionHandling(M);
+
   // Check debug info intrinsics.
   CheckDebugInfoIntrinsics(TheModule);
 
diff --git a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.h b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.h
index 1b3bf1a1854a..6e6118cac0dc 100644
--- a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.h
+++ b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.h
@@ -76,8 +76,8 @@ class BitcodeReaderValueList {
     ValuePtrs.resize(N);
   }
   
-  Constant *getConstantFwdRef(unsigned Idx, const Type *Ty);
-  Value *getValueFwdRef(unsigned Idx, const Type *Ty);
+  Constant *getConstantFwdRef(unsigned Idx, Type *Ty);
+  Value *getValueFwdRef(unsigned Idx, Type *Ty);
   
   void AssignValue(Value *V, unsigned Idx);
   
@@ -212,7 +212,7 @@ class BitcodeReader : public GVMaterializer {
 private:
   Type *getTypeByID(unsigned ID);
   Type *getTypeByIDOrNull(unsigned ID);
-  Value *getFnValueByID(unsigned ID, const Type *Ty) {
+  Value *getFnValueByID(unsigned ID, Type *Ty) {
     if (Ty && Ty->isMetadataTy())
       return MDValueList.getValueFwdRef(ID);
     return ValueList.getValueFwdRef(ID, Ty);
@@ -248,7 +248,7 @@ class BitcodeReader : public GVMaterializer {
     return ResVal == 0;
   }
   bool getValue(SmallVector<uint64_t, 64> &Record, unsigned &Slot,
-                const Type *Ty, Value *&ResVal) {
+                Type *Ty, Value *&ResVal) {
     if (Slot == Record.size()) return true;
     unsigned ValNo = (unsigned)Record[Slot++];
     ResVal = getFnValueByID(ValNo, Ty);
diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 85d67ce62b9f..5b3d96953a0e 100644
--- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -58,7 +58,6 @@ enum {
   FUNCTION_INST_UNREACHABLE_ABBREV
 };
 
-
 static unsigned GetEncodedCastOpcode(unsigned Opcode) {
   switch (Opcode) {
   default: llvm_unreachable("Unknown cast instruction!");
@@ -101,6 +100,44 @@ static unsigned GetEncodedBinaryOpcode(unsigned Opcode) {
   }
 }
 
+static unsigned GetEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
+  switch (Op) {
+  default: llvm_unreachable("Unknown RMW operation!");
+  case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
+  case AtomicRMWInst::Add: return bitc::RMW_ADD;
+  case AtomicRMWInst::Sub: return bitc::RMW_SUB;
+  case AtomicRMWInst::And: return bitc::RMW_AND;
+  case AtomicRMWInst::Nand: return bitc::RMW_NAND;
+  case AtomicRMWInst::Or: return bitc::RMW_OR;
+  case AtomicRMWInst::Xor: return bitc::RMW_XOR;
+  case AtomicRMWInst::Max: return bitc::RMW_MAX;
+  case AtomicRMWInst::Min: return bitc::RMW_MIN;
+  case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
+  case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
+  }
+}
+
+static unsigned GetEncodedOrdering(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default: llvm_unreachable("Unknown atomic ordering");
+  case NotAtomic: return bitc::ORDERING_NOTATOMIC;
+  case Unordered: return bitc::ORDERING_UNORDERED;
+  case Monotonic: return bitc::ORDERING_MONOTONIC;
+  case Acquire: return bitc::ORDERING_ACQUIRE;
+  case Release: return bitc::ORDERING_RELEASE;
+  case AcquireRelease: return bitc::ORDERING_ACQREL;
+  case SequentiallyConsistent: return bitc::ORDERING_SEQCST;
+  }
+}
+
+static unsigned GetEncodedSynchScope(SynchronizationScope SynchScope) {
+  switch (SynchScope) {
+  default: llvm_unreachable("Unknown synchronization scope");
+  case SingleThread: return bitc::SYNCHSCOPE_SINGLETHREAD;
+  case CrossThread: return bitc::SYNCHSCOPE_CROSSTHREAD;
+  }
+}
+
 static void WriteStringRecord(unsigned Code, StringRef Str,
                               unsigned AbbrevToUse, BitstreamWriter &Stream) {
   SmallVector<unsigned, 64> Vals;
@@ -199,7 +236,6 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
                             Log2_32_Ceil(VE.getTypes().size()+1)));
   unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv);
-
   
   // Abbrev for TYPE_CODE_ARRAY.
   Abbv = new BitCodeAbbrev();
@@ -216,7 +252,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
 
   // Loop over all of the types, emitting each in turn.
   for (unsigned i = 0, e = TypeList.size(); i != e; ++i) {
-    const Type *T = TypeList[i];
+    Type *T = TypeList[i];
     int AbbrevToUse = 0;
     unsigned Code = 0;
 
@@ -237,7 +273,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
       TypeVals.push_back(cast<IntegerType>(T)->getBitWidth());
       break;
     case Type::PointerTyID: {
-      const PointerType *PTy = cast<PointerType>(T);
+      PointerType *PTy = cast<PointerType>(T);
       // POINTER: [pointee type, address space]
       Code = bitc::TYPE_CODE_POINTER;
       TypeVals.push_back(VE.getTypeID(PTy->getElementType()));
@@ -247,7 +283,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
       break;
     }
     case Type::FunctionTyID: {
-      const FunctionType *FT = cast<FunctionType>(T);
+      FunctionType *FT = cast<FunctionType>(T);
       // FUNCTION: [isvararg, attrid, retty, paramty x N]
       Code = bitc::TYPE_CODE_FUNCTION;
       TypeVals.push_back(FT->isVarArg());
@@ -259,7 +295,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
       break;
     }
     case Type::StructTyID: {
-      const StructType *ST = cast<StructType>(T);
+      StructType *ST = cast<StructType>(T);
       // STRUCT: [ispacked, eltty x N]
       TypeVals.push_back(ST->isPacked());
       // Output all of the element types.
@@ -267,7 +303,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
            E = ST->element_end(); I != E; ++I)
         TypeVals.push_back(VE.getTypeID(*I));
       
-      if (ST->isAnonymous()) {
+      if (ST->isLiteral()) {
         Code = bitc::TYPE_CODE_STRUCT_ANON;
         AbbrevToUse = StructAnonAbbrev;
       } else {
@@ -286,7 +322,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
       break;
     }
     case Type::ArrayTyID: {
-      const ArrayType *AT = cast<ArrayType>(T);
+      ArrayType *AT = cast<ArrayType>(T);
       // ARRAY: [numelts, eltty]
       Code = bitc::TYPE_CODE_ARRAY;
       TypeVals.push_back(AT->getNumElements());
@@ -295,7 +331,7 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) {
       break;
     }
     case Type::VectorTyID: {
-      const VectorType *VT = cast<VectorType>(T);
+      VectorType *VT = cast<VectorType>(T);
       // VECTOR [numelts, eltty]
       Code = bitc::TYPE_CODE_VECTOR;
       TypeVals.push_back(VT->getNumElements());
@@ -372,14 +408,15 @@ static void WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
        GV != E; ++GV) {
     MaxAlignment = std::max(MaxAlignment, GV->getAlignment());
     MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV->getType()));
-
-    if (!GV->hasSection()) continue;
-    // Give section names unique ID's.
-    unsigned &Entry = SectionMap[GV->getSection()];
-    if (Entry != 0) continue;
-    WriteStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV->getSection(),
-                      0/*TODO*/, Stream);
-    Entry = SectionMap.size();
+    if (GV->hasSection()) {
+      // Give section names unique ID's.
+      unsigned &Entry = SectionMap[GV->getSection()];
+      if (!Entry) {
+        WriteStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV->getSection(),
+                          0/*TODO*/, Stream);
+        Entry = SectionMap.size();
+      }
+    }
   }
   for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) {
     MaxAlignment = std::max(MaxAlignment, F->getAlignment());
@@ -716,7 +753,7 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal,
   SmallVector<uint64_t, 64> Record;
 
   const ValueEnumerator::ValueList &Vals = VE.getValues();
-  const Type *LastTy = 0;
+  Type *LastTy = 0;
   for (unsigned i = FirstVal; i != LastVal; ++i) {
     const Value *V = Vals[i].first;
     // If we need to switch types, do so now.
@@ -781,7 +818,7 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal,
       }
     } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
       Code = bitc::CST_CODE_FLOAT;
-      const Type *Ty = CFP->getType();
+      Type *Ty = CFP->getType();
       if (Ty->isFloatTy() || Ty->isDoubleTy()) {
         Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
       } else if (Ty->isX86_FP80Ty()) {
@@ -1083,8 +1120,8 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::Invoke: {
     const InvokeInst *II = cast<InvokeInst>(&I);
     const Value *Callee(II->getCalledValue());
-    const PointerType *PTy = cast<PointerType>(Callee->getType());
-    const FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+    PointerType *PTy = cast<PointerType>(Callee->getType());
+    FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
     Code = bitc::FUNC_CODE_INST_INVOKE;
 
     Vals.push_back(VE.getAttributeID(II->getAttributes()));
@@ -1105,6 +1142,10 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     }
     break;
   }
+  case Instruction::Resume:
+    Code = bitc::FUNC_CODE_INST_RESUME;
+    PushValueAndType(I.getOperand(0), InstID, Vals, VE);
+    break;
   case Instruction::Unwind:
     Code = bitc::FUNC_CODE_INST_UNWIND;
     break;
@@ -1124,6 +1165,23 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     break;
   }
 
+  case Instruction::LandingPad: {
+    const LandingPadInst &LP = cast<LandingPadInst>(I);
+    Code = bitc::FUNC_CODE_INST_LANDINGPAD;
+    Vals.push_back(VE.getTypeID(LP.getType()));
+    PushValueAndType(LP.getPersonalityFn(), InstID, Vals, VE);
+    Vals.push_back(LP.isCleanup());
+    Vals.push_back(LP.getNumClauses());
+    for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
+      if (LP.isCatch(I))
+        Vals.push_back(LandingPadInst::Catch);
+      else
+        Vals.push_back(LandingPadInst::Filter);
+      PushValueAndType(LP.getClause(I), InstID, Vals, VE);
+    }
+    break;
+  }
+
   case Instruction::Alloca:
     Code = bitc::FUNC_CODE_INST_ALLOCA;
     Vals.push_back(VE.getTypeID(I.getType()));
@@ -1133,24 +1191,66 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     break;
 
   case Instruction::Load:
-    Code = bitc::FUNC_CODE_INST_LOAD;
-    if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE))  // ptr
-      AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
-
+    if (cast<LoadInst>(I).isAtomic()) {
+      Code = bitc::FUNC_CODE_INST_LOADATOMIC;
+      PushValueAndType(I.getOperand(0), InstID, Vals, VE);
+    } else {
+      Code = bitc::FUNC_CODE_INST_LOAD;
+      if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE))  // ptr
+        AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
+    }
     Vals.push_back(Log2_32(cast<LoadInst>(I).getAlignment())+1);
     Vals.push_back(cast<LoadInst>(I).isVolatile());
+    if (cast<LoadInst>(I).isAtomic()) {
+      Vals.push_back(GetEncodedOrdering(cast<LoadInst>(I).getOrdering()));
+      Vals.push_back(GetEncodedSynchScope(cast<LoadInst>(I).getSynchScope()));
+    }
     break;
   case Instruction::Store:
-    Code = bitc::FUNC_CODE_INST_STORE;
+    if (cast<StoreInst>(I).isAtomic())
+      Code = bitc::FUNC_CODE_INST_STOREATOMIC;
+    else
+      Code = bitc::FUNC_CODE_INST_STORE;
     PushValueAndType(I.getOperand(1), InstID, Vals, VE);  // ptrty + ptr
     Vals.push_back(VE.getValueID(I.getOperand(0)));       // val.
     Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1);
     Vals.push_back(cast<StoreInst>(I).isVolatile());
+    if (cast<StoreInst>(I).isAtomic()) {
+      Vals.push_back(GetEncodedOrdering(cast<StoreInst>(I).getOrdering()));
+      Vals.push_back(GetEncodedSynchScope(cast<StoreInst>(I).getSynchScope()));
+    }
+    break;
+  case Instruction::AtomicCmpXchg:
+    Code = bitc::FUNC_CODE_INST_CMPXCHG;
+    PushValueAndType(I.getOperand(0), InstID, Vals, VE);  // ptrty + ptr
+    Vals.push_back(VE.getValueID(I.getOperand(1)));       // cmp.
+    Vals.push_back(VE.getValueID(I.getOperand(2)));       // newval.
+    Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile());
+    Vals.push_back(GetEncodedOrdering(
+                     cast<AtomicCmpXchgInst>(I).getOrdering()));
+    Vals.push_back(GetEncodedSynchScope(
+                     cast<AtomicCmpXchgInst>(I).getSynchScope()));
+    break;
+  case Instruction::AtomicRMW:
+    Code = bitc::FUNC_CODE_INST_ATOMICRMW;
+    PushValueAndType(I.getOperand(0), InstID, Vals, VE);  // ptrty + ptr
+    Vals.push_back(VE.getValueID(I.getOperand(1)));       // val.
+    Vals.push_back(GetEncodedRMWOperation(
+                     cast<AtomicRMWInst>(I).getOperation()));
+    Vals.push_back(cast<AtomicRMWInst>(I).isVolatile());
+    Vals.push_back(GetEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering()));
+    Vals.push_back(GetEncodedSynchScope(
+                     cast<AtomicRMWInst>(I).getSynchScope()));
+    break;
+  case Instruction::Fence:
+    Code = bitc::FUNC_CODE_INST_FENCE;
+    Vals.push_back(GetEncodedOrdering(cast<FenceInst>(I).getOrdering()));
+    Vals.push_back(GetEncodedSynchScope(cast<FenceInst>(I).getSynchScope()));
     break;
   case Instruction::Call: {
     const CallInst &CI = cast<CallInst>(I);
-    const PointerType *PTy = cast<PointerType>(CI.getCalledValue()->getType());
-    const FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+    PointerType *PTy = cast<PointerType>(CI.getCalledValue()->getType());
+    FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
 
     Code = bitc::FUNC_CODE_INST_CALL;
 
diff --git a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
index b68bf92d51b2..9ae9905b9f1d 100644
--- a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -315,7 +315,7 @@ void ValueEnumerator::EnumerateValue(const Value *V) {
 }
 
 
-void ValueEnumerator::EnumerateType(const Type *Ty) {
+void ValueEnumerator::EnumerateType(Type *Ty) {
   unsigned *TypeID = &TypeMap[Ty];
 
   // We've already seen this type.
@@ -325,8 +325,8 @@ void ValueEnumerator::EnumerateType(const Type *Ty) {
   // If it is a non-anonymous struct, mark the type as being visited so that we
   // don't recursively visit it.  This is safe because we allow forward
   // references of these in the bitcode reader.
-  if (const StructType *STy = dyn_cast<StructType>(Ty))
-    if (!STy->isAnonymous())
+  if (StructType *STy = dyn_cast<StructType>(Ty))
+    if (!STy->isLiteral())
       *TypeID = ~0U;
   
   // Enumerate all of the subtypes before we enumerate this type.  This ensures
diff --git a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h
index 6617b60deb26..b6fc920e412b 100644
--- a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h
+++ b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h
@@ -35,12 +35,12 @@ class MDSymbolTable;
 
 class ValueEnumerator {
 public:
-  typedef std::vector<const Type*> TypeList;
+  typedef std::vector<Type*> TypeList;
 
   // For each value, we remember its Value* and occurrence frequency.
   typedef std::vector<std::pair<const Value*, unsigned> > ValueList;
 private:
-  typedef DenseMap<const Type*, unsigned> TypeMapType;
+  typedef DenseMap<Type*, unsigned> TypeMapType;
   TypeMapType TypeMap;
   TypeList Types;
 
@@ -85,7 +85,7 @@ class ValueEnumerator {
 
   unsigned getValueID(const Value *V) const;
 
-  unsigned getTypeID(const Type *T) const {
+  unsigned getTypeID(Type *T) const {
     TypeMapType::const_iterator I = TypeMap.find(T);
     assert(I != TypeMap.end() && "Type not in ValueEnumerator!");
     return I->second-1;
@@ -140,7 +140,7 @@ class ValueEnumerator {
   void EnumerateFunctionLocalMetadata(const MDNode *N);
   void EnumerateNamedMDNode(const NamedMDNode *NMD);
   void EnumerateValue(const Value *V);
-  void EnumerateType(const Type *T);
+  void EnumerateType(Type *T);
   void EnumerateOperandType(const Value *V);
   void EnumerateAttributes(const AttrListPtr &PAL);
   
diff --git a/contrib/llvm/lib/CodeGen/Analysis.cpp b/contrib/llvm/lib/CodeGen/Analysis.cpp
index 125e64196f15..fafc01044d4f 100644
--- a/contrib/llvm/lib/CodeGen/Analysis.cpp
+++ b/contrib/llvm/lib/CodeGen/Analysis.cpp
@@ -31,7 +31,7 @@ using namespace llvm;
 /// of insertvalue or extractvalue indices that identify a member, return
 /// the linearized index of the start of the member.
 ///
-unsigned llvm::ComputeLinearIndex(const Type *Ty,
+unsigned llvm::ComputeLinearIndex(Type *Ty,
                                   const unsigned *Indices,
                                   const unsigned *IndicesEnd,
                                   unsigned CurIndex) {
@@ -40,7 +40,7 @@ unsigned llvm::ComputeLinearIndex(const Type *Ty,
     return CurIndex;
 
   // Given a struct type, recursively traverse the elements.
-  if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+  if (StructType *STy = dyn_cast<StructType>(Ty)) {
     for (StructType::element_iterator EB = STy->element_begin(),
                                       EI = EB,
                                       EE = STy->element_end();
@@ -52,8 +52,8 @@ unsigned llvm::ComputeLinearIndex(const Type *Ty,
     return CurIndex;
   }
   // Given an array type, recursively traverse the elements.
-  else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
-    const Type *EltTy = ATy->getElementType();
+  else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+    Type *EltTy = ATy->getElementType();
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
       if (Indices && *Indices == i)
         return ComputeLinearIndex(EltTy, Indices+1, IndicesEnd, CurIndex);
@@ -72,12 +72,12 @@ unsigned llvm::ComputeLinearIndex(const Type *Ty,
 /// If Offsets is non-null, it points to a vector to be filled in
 /// with the in-memory offsets of each of the individual values.
 ///
-void llvm::ComputeValueVTs(const TargetLowering &TLI, const Type *Ty,
+void llvm::ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
                            SmallVectorImpl<EVT> &ValueVTs,
                            SmallVectorImpl<uint64_t> *Offsets,
                            uint64_t StartingOffset) {
   // Given a struct type, recursively traverse the elements.
-  if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+  if (StructType *STy = dyn_cast<StructType>(Ty)) {
     const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy);
     for (StructType::element_iterator EB = STy->element_begin(),
                                       EI = EB,
@@ -88,8 +88,8 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const Type *Ty,
     return;
   }
   // Given an array type, recursively traverse the elements.
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
-    const Type *EltTy = ATy->getElementType();
+  if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+    Type *EltTy = ATy->getElementType();
     uint64_t EltSize = TLI.getTargetData()->getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
       ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets,
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp
index 5861fa4817f6..3f2387325360 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp
@@ -17,7 +17,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -27,7 +26,6 @@
 #include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetFrameLowering.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Target/TargetRegisterInfo.h"
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 7f314eed3ae6..1999f3608788 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -33,7 +33,6 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetAsmInfo.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
@@ -45,6 +44,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Timer.h"
 using namespace llvm;
 
@@ -290,10 +290,10 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   // Handle common and BSS local symbols (.lcomm).
   if (GVKind.isCommon() || GVKind.isBSSLocal()) {
     if (Size == 0) Size = 1;   // .comm Foo, 0 is undefined, avoid it.
+    unsigned Align = 1 << AlignLog;
 
     // Handle common symbols.
     if (GVKind.isCommon()) {
-      unsigned Align = 1 << AlignLog;
       if (!getObjFileLowering().getCommDirectiveSupportsAlignment())
         Align = 0;
 
@@ -307,17 +307,17 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
       const MCSection *TheSection =
         getObjFileLowering().SectionForGlobal(GV, GVKind, Mang, TM);
       // .zerofill __DATA, __bss, _foo, 400, 5
-      OutStreamer.EmitZerofill(TheSection, GVSym, Size, 1 << AlignLog);
+      OutStreamer.EmitZerofill(TheSection, GVSym, Size, Align);
       return;
     }
 
-    if (MAI->hasLCOMMDirective()) {
+    if (MAI->getLCOMMDirectiveType() != LCOMM::None &&
+        (MAI->getLCOMMDirectiveType() != LCOMM::NoAlignment || Align == 1)) {
       // .lcomm _foo, 42
-      OutStreamer.EmitLocalCommonSymbol(GVSym, Size);
+      OutStreamer.EmitLocalCommonSymbol(GVSym, Size, Align);
       return;
     }
 
-    unsigned Align = 1 << AlignLog;
     if (!getObjFileLowering().getCommDirectiveSupportsAlignment())
       Align = 0;
 
@@ -474,8 +474,10 @@ void AsmPrinter::EmitFunctionHeader() {
 void AsmPrinter::EmitFunctionEntryLabel() {
   // The function label could have already been emitted if two symbols end up
   // conflicting due to asm renaming.  Detect this and emit an error.
-  if (CurrentFnSym->isUndefined())
+  if (CurrentFnSym->isUndefined()) {
+    OutStreamer.ForceCodeRegion();
     return OutStreamer.EmitLabel(CurrentFnSym);
+  }
 
   report_fatal_error("'" + Twine(CurrentFnSym->getName()) +
                      "' label emitted multiple times to assembly file");
@@ -620,6 +622,9 @@ void AsmPrinter::emitPrologLabel(const MachineInstr &MI) {
   if (needsCFIMoves() == CFI_M_None)
     return;
 
+  if (MMI->getCompactUnwindEncoding() != 0)
+    OutStreamer.EmitCompactUnwindEncoding(MMI->getCompactUnwindEncoding());
+
   MachineModuleInfo &MMI = MF->getMMI();
   std::vector<MachineMove> &Moves = MMI.getFrameMoves();
   bool FoundOne = false;
@@ -878,7 +883,7 @@ bool AsmPrinter::doFinalization(Module &M) {
          I != E; ++I) {
       MCSymbol *Name = Mang->getSymbol(I);
 
-      const GlobalValue *GV = cast<GlobalValue>(I->getAliasedGlobal());
+      const GlobalValue *GV = I->getAliasedGlobal();
       MCSymbol *Target = Mang->getSymbol(GV);
 
       if (I->hasExternalLinkage() || !MAI->getWeakRefDirective())
@@ -1009,7 +1014,7 @@ void AsmPrinter::EmitConstantPool() {
       unsigned NewOffset = (Offset + AlignMask) & ~AlignMask;
       OutStreamer.EmitFill(NewOffset - Offset, 0/*fillval*/, 0/*addrspace*/);
 
-      const Type *Ty = CPE.getType();
+      Type *Ty = CPE.getType();
       Offset = NewOffset + TM.getTargetData()->getTypeAllocSize(Ty);
       OutStreamer.EmitLabel(GetCPISymbol(CPI));
 
@@ -1055,6 +1060,15 @@ void AsmPrinter::EmitJumpTableInfo() {
 
   EmitAlignment(Log2_32(MJTI->getEntryAlignment(*TM.getTargetData())));
 
+  // If we know the form of the jump table, go ahead and tag it as such.
+  if (!JTInDiffSection) {
+    if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32) {
+      OutStreamer.EmitJumpTable32Region();
+    } else {
+      OutStreamer.EmitDataRegion();
+    }
+  }
+
   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
 
@@ -1226,22 +1240,53 @@ void AsmPrinter::EmitLLVMUsedList(const Constant *List) {
   }
 }
 
-/// EmitXXStructorList - Emit the ctor or dtor list.  This just prints out the
-/// function pointers, ignoring the init priority.
+typedef std::pair<int, Constant*> Structor;
+
+static bool priority_order(const Structor& lhs, const Structor& rhs) {
+  return lhs.first < rhs.first;
+}
+
+/// EmitXXStructorList - Emit the ctor or dtor list taking into account the init
+/// priority.
 void AsmPrinter::EmitXXStructorList(const Constant *List) {
   // Should be an array of '{ int, void ()* }' structs.  The first value is the
-  // init priority, which we ignore.
+  // init priority.
   if (!isa<ConstantArray>(List)) return;
-  const ConstantArray *InitList = cast<ConstantArray>(List);
-  for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i)
-    if (ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i))){
-      if (CS->getNumOperands() != 2) return;  // Not array of 2-element structs.
 
-      if (CS->getOperand(1)->isNullValue())
-        return;  // Found a null terminator, exit printing.
-      // Emit the function pointer.
-      EmitGlobalConstant(CS->getOperand(1));
-    }
+  // Sanity check the structors list.
+  const ConstantArray *InitList = dyn_cast<ConstantArray>(List);
+  if (!InitList) return; // Not an array!
+  StructType *ETy = dyn_cast<StructType>(InitList->getType()->getElementType());
+  if (!ETy || ETy->getNumElements() != 2) return; // Not an array of pairs!
+  if (!isa<IntegerType>(ETy->getTypeAtIndex(0U)) ||
+      !isa<PointerType>(ETy->getTypeAtIndex(1U))) return; // Not (int, ptr).
+
+  // Gather the structors in a form that's convenient for sorting by priority.
+  SmallVector<Structor, 8> Structors;
+  for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) {
+    ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i));
+    if (!CS) continue; // Malformed.
+    if (CS->getOperand(1)->isNullValue())
+      break;  // Found a null terminator, skip the rest.
+    ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
+    if (!Priority) continue; // Malformed.
+    Structors.push_back(std::make_pair(Priority->getLimitedValue(65535),
+                                       CS->getOperand(1)));
+  }
+
+  // Emit the function pointers in reverse priority order.
+  switch (MAI->getStructorOutputOrder()) {
+  case Structors::None:
+    break;
+  case Structors::PriorityOrder:
+    std::sort(Structors.begin(), Structors.end(), priority_order);
+    break;
+  case Structors::ReversePriorityOrder:
+    std::sort(Structors.rbegin(), Structors.rend(), priority_order);
+    break;
+  }
+  for (unsigned i = 0, e = Structors.size(); i != e; ++i)
+    EmitGlobalConstant(Structors[i].second);
 }
 
 //===--------------------------------------------------------------------===//
@@ -1406,8 +1451,7 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
     // Generate a symbolic expression for the byte address
     const Constant *PtrVal = CE->getOperand(0);
     SmallVector<Value*, 8> IdxVec(CE->op_begin()+1, CE->op_end());
-    int64_t Offset = TD.getIndexedOffset(PtrVal->getType(), &IdxVec[0],
-                                         IdxVec.size());
+    int64_t Offset = TD.getIndexedOffset(PtrVal->getType(), IdxVec);
 
     const MCExpr *Base = LowerConstant(CE->getOperand(0), AP);
     if (Offset == 0)
@@ -1447,7 +1491,7 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
     // Support only foldable casts to/from pointers that can be eliminated by
     // changing the pointer to the appropriately sized integer type.
     Constant *Op = CE->getOperand(0);
-    const Type *Ty = CE->getType();
+    Type *Ty = CE->getType();
 
     const MCExpr *OpExpr = LowerConstant(Op, AP);
 
@@ -1496,12 +1540,67 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
 static void EmitGlobalConstantImpl(const Constant *C, unsigned AddrSpace,
                                    AsmPrinter &AP);
 
+/// isRepeatedByteSequence - Determine whether the given value is
+/// composed of a repeated sequence of identical bytes and return the
+/// byte value.  If it is not a repeated sequence, return -1.
+static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) {
+
+  if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
+    if (CI->getBitWidth() > 64) return -1;
+
+    uint64_t Size = TM.getTargetData()->getTypeAllocSize(V->getType());
+    uint64_t Value = CI->getZExtValue();
+
+    // Make sure the constant is at least 8 bits long and has a power
+    // of 2 bit width.  This guarantees the constant bit width is
+    // always a multiple of 8 bits, avoiding issues with padding out
+    // to Size and other such corner cases.
+    if (CI->getBitWidth() < 8 || !isPowerOf2_64(CI->getBitWidth())) return -1;
+
+    uint8_t Byte = static_cast<uint8_t>(Value);
+
+    for (unsigned i = 1; i < Size; ++i) {
+      Value >>= 8;
+      if (static_cast<uint8_t>(Value) != Byte) return -1;
+    }
+    return Byte;
+  }
+  if (const ConstantArray *CA = dyn_cast<ConstantArray>(V)) {
+    // Make sure all array elements are sequences of the same repeated
+    // byte.
+    if (CA->getNumOperands() == 0) return -1;
+
+    int Byte = isRepeatedByteSequence(CA->getOperand(0), TM);
+    if (Byte == -1) return -1;
+
+    for (unsigned i = 1, e = CA->getNumOperands(); i != e; ++i) {
+      int ThisByte = isRepeatedByteSequence(CA->getOperand(i), TM);
+      if (ThisByte == -1) return -1;
+      if (Byte != ThisByte) return -1;
+    }
+    return Byte;
+  }
+
+  return -1;
+}
+
 static void EmitGlobalConstantArray(const ConstantArray *CA, unsigned AddrSpace,
                                     AsmPrinter &AP) {
   if (AddrSpace != 0 || !CA->isString()) {
-    // Not a string.  Print the values in successive locations
-    for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i)
-      EmitGlobalConstantImpl(CA->getOperand(i), AddrSpace, AP);
+    // Not a string.  Print the values in successive locations.
+
+    // See if we can aggregate some values.  Make sure it can be
+    // represented as a series of bytes of the constant value.
+    int Value = isRepeatedByteSequence(CA, AP.TM);
+
+    if (Value != -1) {
+      uint64_t Bytes = AP.TM.getTargetData()->getTypeAllocSize(CA->getType());
+      AP.OutStreamer.EmitFill(Bytes, Value, AddrSpace);
+    }
+    else {
+      for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i)
+        EmitGlobalConstantImpl(CA->getOperand(i), AddrSpace, AP);
+    }
     return;
   }
 
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index dd5b0e261490..4d6c28118427 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -13,7 +13,7 @@
 
 #define DEBUG_TYPE "asm-printer"
 #include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/CodeGen/MachineLocation.h"
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
index 5ac455e1a1a1..8eda889155a2 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
@@ -23,15 +23,15 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/TargetAsmParser.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
@@ -49,7 +49,7 @@ namespace {
 static void SrcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) {
   SrcMgrDiagInfo *DiagInfo = static_cast<SrcMgrDiagInfo *>(diagInfo);
   assert(DiagInfo && "Diagnostic context not passed down?");
-  
+
   // If the inline asm had metadata associated with it, pull out a location
   // cookie corresponding to which line the error occurred on.
   unsigned LocCookie = 0;
@@ -57,13 +57,13 @@ static void SrcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) {
     unsigned ErrorLine = Diag.getLineNo()-1;
     if (ErrorLine >= LocInfo->getNumOperands())
       ErrorLine = 0;
-    
+
     if (LocInfo->getNumOperands() != 0)
       if (const ConstantInt *CI =
           dyn_cast<ConstantInt>(LocInfo->getOperand(ErrorLine)))
         LocCookie = CI->getZExtValue();
   }
-  
+
   DiagInfo->DiagHandler(Diag, DiagInfo->DiagContext, LocCookie);
 }
 
@@ -109,7 +109,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const {
   // Tell SrcMgr about this buffer, it takes ownership of the buffer.
   SrcMgr.AddNewSourceBuffer(Buffer, SMLoc());
 
-  OwningPtr<MCAsmParser> Parser(createMCAsmParser(TM.getTarget(), SrcMgr,
+  OwningPtr<MCAsmParser> Parser(createMCAsmParser(SrcMgr,
                                                   OutContext, OutStreamer,
                                                   *MAI));
 
@@ -121,7 +121,8 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const {
     STI(TM.getTarget().createMCSubtargetInfo(TM.getTargetTriple(),
                                              TM.getTargetCPU(),
                                              TM.getTargetFeatureString()));
-  OwningPtr<TargetAsmParser> TAP(TM.getTarget().createAsmParser(*STI, *Parser));
+  OwningPtr<MCTargetAsmParser>
+    TAP(TM.getTarget().createMCAsmParser(*STI, *Parser));
   if (!TAP)
     report_fatal_error("Inline asm not supported by this streamer because"
                        " we don't have an asm parser for this target\n");
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
index 21396ca37f06..9c1ce761b0c5 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -69,7 +69,7 @@ void DIEAbbrev::Emit(AsmPrinter *AP) const {
     // Emit attribute type.
     // FIXME: Doing work even in non-asm-verbose runs.
     AP->EmitULEB128(AttrData.getAttribute(),
-                              dwarf::AttributeString(AttrData.getAttribute()));
+                    dwarf::AttributeString(AttrData.getAttribute()));
 
     // Emit form type.
     // FIXME: Doing work even in non-asm-verbose runs.
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
index 91b7d08c6ae2..8ed4f4c43a7c 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
@@ -17,7 +17,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineLocation.h"
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -77,7 +77,8 @@ void DwarfCFIException::EndModule() {
     // This is a temporary hack to keep sections in the same order they
     // were before. This lets us produce bit identical outputs while
     // transitioning to CFI.
-    Asm->OutStreamer.SwitchSection(TLOF.getEHFrameSection());
+    Asm->OutStreamer.SwitchSection(
+               const_cast<TargetLoweringObjectFile&>(TLOF).getEHFrameSection());
   }
 }
 
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 1fe035efde3e..88b752473073 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -16,7 +16,10 @@
 #include "DwarfCompileUnit.h"
 #include "DwarfDebug.h"
 #include "llvm/Constants.h"
+#include "llvm/GlobalVariable.h"
+#include "llvm/Instructions.h"
 #include "llvm/Analysis/DIBuilder.h"
+#include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
@@ -132,8 +135,8 @@ void CompileUnit::addSourceLine(DIE *Die, DIGlobalVariable G) {
   unsigned Line = G.getLineNumber();
   if (Line == 0)
     return;
-  unsigned FileID = DD->GetOrCreateSourceID(G.getContext().getFilename(),
-                                            G.getContext().getDirectory());
+  unsigned FileID = DD->GetOrCreateSourceID(G.getFilename(),
+                                            G.getDirectory());
   assert(FileID && "Invalid file id");
   addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID);
   addUInt(Die, dwarf::DW_AT_decl_line, 0, Line);
@@ -439,27 +442,36 @@ void CompileUnit::addBlockByrefAddress(DbgVariable *&DV, DIE *Die,
   addBlock(Die, Attribute, 0, Block);
 }
 
+/// isTypeSigned - Return true if the type is signed.
+static bool isTypeSigned(DIType Ty, int *SizeInBits) {
+  if (Ty.isDerivedType())
+    return isTypeSigned(DIDerivedType(Ty).getTypeDerivedFrom(), SizeInBits);
+  if (Ty.isBasicType())
+    if (DIBasicType(Ty).getEncoding() == dwarf::DW_ATE_signed
+        || DIBasicType(Ty).getEncoding() == dwarf::DW_ATE_signed_char) {
+      *SizeInBits = Ty.getSizeInBits();
+      return true;
+    }
+  return false;
+}
+
 /// addConstantValue - Add constant value entry in variable DIE.
 bool CompileUnit::addConstantValue(DIE *Die, const MachineOperand &MO,
                                    DIType Ty) {
   assert (MO.isImm() && "Invalid machine operand!");
   DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
-  unsigned form = dwarf::DW_FORM_udata;
-  switch (Ty.getSizeInBits()) {
-    case 8: form = dwarf::DW_FORM_data1; break;
-    case 16: form = dwarf::DW_FORM_data2; break;
-    case 32: form = dwarf::DW_FORM_data4; break;
-    case 64: form = dwarf::DW_FORM_data8; break;
+  int SizeInBits = -1;
+  bool SignedConstant = isTypeSigned(Ty, &SizeInBits);
+  unsigned Form = SignedConstant ? dwarf::DW_FORM_sdata : dwarf::DW_FORM_udata;
+  switch (SizeInBits) {
+    case 8:  Form = dwarf::DW_FORM_data1; break;
+    case 16: Form = dwarf::DW_FORM_data2; break;
+    case 32: Form = dwarf::DW_FORM_data4; break;
+    case 64: Form = dwarf::DW_FORM_data8; break;
     default: break;
   }
-
-  DIBasicType BTy(Ty);
-  if (BTy.Verify() &&
-      (BTy.getEncoding()  == dwarf::DW_ATE_signed 
-       || BTy.getEncoding() == dwarf::DW_ATE_signed_char))
-    addSInt(Block, 0, form, MO.getImm());
-  else
-    addUInt(Block, 0, form, MO.getImm());
+  SignedConstant ? addSInt(Block, 0, Form, MO.getImm()) 
+    : addUInt(Block, 0, Form, MO.getImm());
 
   addBlock(Die, dwarf::DW_AT_const_value, 0, Block);
   return true;
@@ -555,7 +567,7 @@ void CompileUnit::addToContextOwner(DIE *Die, DIDescriptor Context) {
     DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context));
     ContextDIE->addChild(Die);
   } else if (Context.isSubprogram()) {
-    DIE *ContextDIE = DD->createSubprogramDIE(DISubprogram(Context));
+    DIE *ContextDIE = getOrCreateSubprogramDIE(DISubprogram(Context));
     ContextDIE->addChild(Die);
   } else if (DIE *ContextDIE = getDIE(Context))
     ContextDIE->addChild(Die);
@@ -565,7 +577,10 @@ void CompileUnit::addToContextOwner(DIE *Die, DIDescriptor Context) {
 
 /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the
 /// given DIType.
-DIE *CompileUnit::getOrCreateTypeDIE(DIType Ty) {
+DIE *CompileUnit::getOrCreateTypeDIE(const MDNode *TyNode) {
+  DIType Ty(TyNode);
+  if (!Ty.Verify())
+    return NULL;
   DIE *TyDIE = getDIE(Ty);
   if (TyDIE)
     return TyDIE;
@@ -617,7 +632,8 @@ void CompileUnit::addType(DIE *Entity, DIType Ty) {
 void CompileUnit::addGlobalType(DIType Ty) {
   DIDescriptor Context = Ty.getContext();
   if (Ty.isCompositeType() && !Ty.getName().empty() && !Ty.isForwardDecl() 
-      && (Context.isCompileUnit() || Context.isFile() || Context.isNameSpace()))
+      && (!Context || Context.isCompileUnit() || Context.isFile() 
+          || Context.isNameSpace()))
     if (DIEEntry *Entry = getDIEEntry(Ty))
       GlobalTypes[Ty.getName()] = Entry->getEntry();
 }
@@ -642,13 +658,20 @@ void CompileUnit::addPubTypes(DISubprogram SP) {
 void CompileUnit::constructTypeDIE(DIE &Buffer, DIBasicType BTy) {
   // Get core information.
   StringRef Name = BTy.getName();
-  Buffer.setTag(dwarf::DW_TAG_base_type);
-  addUInt(&Buffer, dwarf::DW_AT_encoding,  dwarf::DW_FORM_data1,
-          BTy.getEncoding());
-
   // Add name if not anonymous or intermediate type.
   if (!Name.empty())
     addString(&Buffer, dwarf::DW_AT_name, dwarf::DW_FORM_string, Name);
+
+  if (BTy.getTag() == dwarf::DW_TAG_unspecified_type) {
+    Buffer.setTag(dwarf::DW_TAG_unspecified_type);
+    // Unspecified types has only name, nothing else.
+    return;
+  }
+
+  Buffer.setTag(dwarf::DW_TAG_base_type);
+  addUInt(&Buffer, dwarf::DW_AT_encoding,  dwarf::DW_FORM_data1,
+	  BTy.getEncoding());
+
   uint64_t Size = BTy.getSizeInBits() >> 3;
   addUInt(&Buffer, dwarf::DW_AT_byte_size, 0, Size);
 }
@@ -752,7 +775,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
       DIE *ElemDie = NULL;
       if (Element.isSubprogram()) {
         DISubprogram SP(Element);
-        ElemDie = DD->createSubprogramDIE(DISubprogram(Element));
+        ElemDie = getOrCreateSubprogramDIE(DISubprogram(Element));
         if (SP.isProtected())
           addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag,
                   dwarf::DW_ACCESS_protected);
@@ -880,6 +903,218 @@ DIE *CompileUnit::getOrCreateNameSpace(DINameSpace NS) {
   return NDie;
 }
 
+/// getRealLinkageName - If special LLVM prefix that is used to inform the asm
+/// printer to not emit usual symbol prefix before the symbol name is used then
+/// return linkage name after skipping this special LLVM prefix.
+static StringRef getRealLinkageName(StringRef LinkageName) {
+  char One = '\1';
+  if (LinkageName.startswith(StringRef(&One, 1)))
+    return LinkageName.substr(1);
+  return LinkageName;
+}
+
+/// getOrCreateSubprogramDIE - Create new DIE using SP.
+DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) {
+  DIE *SPDie = getDIE(SP);
+  if (SPDie)
+    return SPDie;
+
+  SPDie = new DIE(dwarf::DW_TAG_subprogram);
+  
+  // DW_TAG_inlined_subroutine may refer to this DIE.
+  insertDIE(SP, SPDie);
+  
+  // Add to context owner.
+  addToContextOwner(SPDie, SP.getContext());
+
+  // Add function template parameters.
+  addTemplateParams(*SPDie, SP.getTemplateParams());
+
+  StringRef LinkageName = SP.getLinkageName();
+  if (!LinkageName.empty())
+    addString(SPDie, dwarf::DW_AT_MIPS_linkage_name, 
+                    dwarf::DW_FORM_string,
+                    getRealLinkageName(LinkageName));
+
+  // If this DIE is going to refer declaration info using AT_specification
+  // then there is no need to add other attributes.
+  if (SP.getFunctionDeclaration().isSubprogram())
+    return SPDie;
+
+  // Constructors and operators for anonymous aggregates do not have names.
+  if (!SP.getName().empty())
+    addString(SPDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, 
+                    SP.getName());
+
+  addSourceLine(SPDie, SP);
+
+  if (SP.isPrototyped()) 
+    addUInt(SPDie, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1);
+
+  // Add Return Type.
+  DICompositeType SPTy = SP.getType();
+  DIArray Args = SPTy.getTypeArray();
+  unsigned SPTag = SPTy.getTag();
+
+  if (Args.getNumElements() == 0 || SPTag != dwarf::DW_TAG_subroutine_type)
+    addType(SPDie, SPTy);
+  else
+    addType(SPDie, DIType(Args.getElement(0)));
+
+  unsigned VK = SP.getVirtuality();
+  if (VK) {
+    addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_flag, VK);
+    DIEBlock *Block = getDIEBlock();
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu);
+    addUInt(Block, 0, dwarf::DW_FORM_udata, SP.getVirtualIndex());
+    addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, 0, Block);
+    ContainingTypeMap.insert(std::make_pair(SPDie,
+                                            SP.getContainingType()));
+  }
+
+  if (!SP.isDefinition()) {
+    addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
+    
+    // Add arguments. Do not add arguments for subprogram definition. They will
+    // be handled while processing variables.
+    DICompositeType SPTy = SP.getType();
+    DIArray Args = SPTy.getTypeArray();
+    unsigned SPTag = SPTy.getTag();
+
+    if (SPTag == dwarf::DW_TAG_subroutine_type)
+      for (unsigned i = 1, N =  Args.getNumElements(); i < N; ++i) {
+        DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter);
+        DIType ATy = DIType(DIType(Args.getElement(i)));
+        addType(Arg, ATy);
+        if (ATy.isArtificial())
+          addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
+        SPDie->addChild(Arg);
+      }
+  }
+
+  if (SP.isArtificial())
+    addUInt(SPDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
+
+  if (!SP.isLocalToUnit())
+    addUInt(SPDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
+
+  if (SP.isOptimized())
+    addUInt(SPDie, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1);
+
+  if (unsigned isa = Asm->getISAEncoding()) {
+    addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa);
+  }
+
+  return SPDie;
+}
+
+// Return const expression if value is a GEP to access merged global
+// constant. e.g.
+// i8* getelementptr ({ i8, i8, i8, i8 }* @_MergedGlobals, i32 0, i32 0)
+static const ConstantExpr *getMergedGlobalExpr(const Value *V) {
+  const ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(V);
+  if (!CE || CE->getNumOperands() != 3 ||
+      CE->getOpcode() != Instruction::GetElementPtr)
+    return NULL;
+
+  // First operand points to a global struct.
+  Value *Ptr = CE->getOperand(0);
+  if (!isa<GlobalValue>(Ptr) ||
+      !isa<StructType>(cast<PointerType>(Ptr->getType())->getElementType()))
+    return NULL;
+
+  // Second operand is zero.
+  const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(CE->getOperand(1));
+  if (!CI || !CI->isZero())
+    return NULL;
+
+  // Third operand is offset.
+  if (!isa<ConstantInt>(CE->getOperand(2)))
+    return NULL;
+
+  return CE;
+}
+
+/// createGlobalVariableDIE - create global variable DIE.
+void CompileUnit::createGlobalVariableDIE(const MDNode *N) {
+  // Check for pre-existence.
+  if (getDIE(N))
+    return;
+
+  DIGlobalVariable GV(N);
+  if (!GV.Verify())
+    return;
+
+  DIE *VariableDIE = new DIE(GV.getTag());
+  // Add to map.
+  insertDIE(N, VariableDIE);
+
+  // Add name.
+  addString(VariableDIE, dwarf::DW_AT_name, dwarf::DW_FORM_string,
+                   GV.getDisplayName());
+  StringRef LinkageName = GV.getLinkageName();
+  bool isGlobalVariable = GV.getGlobal() != NULL;
+  if (!LinkageName.empty() && isGlobalVariable)
+    addString(VariableDIE, dwarf::DW_AT_MIPS_linkage_name, 
+                     dwarf::DW_FORM_string,
+                     getRealLinkageName(LinkageName));
+  // Add type.
+  DIType GTy = GV.getType();
+  addType(VariableDIE, GTy);
+
+  // Add scoping info.
+  if (!GV.isLocalToUnit()) {
+    addUInt(VariableDIE, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
+    // Expose as global. 
+    addGlobal(GV.getName(), VariableDIE);
+  }
+  // Add line number info.
+  addSourceLine(VariableDIE, GV);
+  // Add to context owner.
+  DIDescriptor GVContext = GV.getContext();
+  addToContextOwner(VariableDIE, GVContext);
+  // Add location.
+  if (isGlobalVariable) {
+    DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
+    addLabel(Block, 0, dwarf::DW_FORM_udata,
+             Asm->Mang->getSymbol(GV.getGlobal()));
+    // Do not create specification DIE if context is either compile unit
+    // or a subprogram.
+    if (GVContext && GV.isDefinition() && !GVContext.isCompileUnit() &&
+        !GVContext.isFile() && !isSubprogramContext(GVContext)) {
+      // Create specification DIE.
+      DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable);
+      addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification,
+                  dwarf::DW_FORM_ref4, VariableDIE);
+      addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block);
+      addUInt(VariableDIE, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag,
+                     1);
+      addDie(VariableSpecDIE);
+    } else {
+      addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block);
+    } 
+  } else if (const ConstantInt *CI = 
+             dyn_cast_or_null<ConstantInt>(GV.getConstant()))
+    addConstantValue(VariableDIE, CI, GTy.isUnsignedDIType());
+  else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) {
+    // GV is a merged global.
+    DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
+    Value *Ptr = CE->getOperand(0);
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
+    addLabel(Block, 0, dwarf::DW_FORM_udata,
+                    Asm->Mang->getSymbol(cast<GlobalValue>(Ptr)));
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu);
+    SmallVector<Value*, 3> Idx(CE->op_begin()+1, CE->op_end());
+    addUInt(Block, 0, dwarf::DW_FORM_udata, 
+                   Asm->getTargetData().getIndexedOffset(Ptr->getType(), Idx));
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_plus);
+    addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block);
+  }
+
+  return;
+}
+
 /// constructSubrangeDIE - Construct subrange DIE from DISubrange.
 void CompileUnit::constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy){
   DIE *DW_Subrange = new DIE(dwarf::DW_TAG_subrange_type);
@@ -944,6 +1179,128 @@ DIE *CompileUnit::constructEnumTypeDIE(DIEnumerator ETy) {
   return Enumerator;
 }
 
+/// constructContainingTypeDIEs - Construct DIEs for types that contain
+/// vtables.
+void CompileUnit::constructContainingTypeDIEs() {
+  for (DenseMap<DIE *, const MDNode *>::iterator CI = ContainingTypeMap.begin(),
+         CE = ContainingTypeMap.end(); CI != CE; ++CI) {
+    DIE *SPDie = CI->first;
+    const MDNode *N = CI->second;
+    if (!N) continue;
+    DIE *NDie = getDIE(N);
+    if (!NDie) continue;
+    addDIEEntry(SPDie, dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4, NDie);
+  }
+}
+
+/// constructVariableDIE - Construct a DIE for the given DbgVariable.
+DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) {
+  StringRef Name = DV->getName();
+  if (Name.empty())
+    return NULL;
+
+  // Translate tag to proper Dwarf tag.
+  unsigned Tag = DV->getTag();
+
+  // Define variable debug information entry.
+  DIE *VariableDie = new DIE(Tag);
+  DbgVariable *AbsVar = DV->getAbstractVariable();
+  DIE *AbsDIE = AbsVar ? AbsVar->getDIE() : NULL;
+  if (AbsDIE)
+    addDIEEntry(VariableDie, dwarf::DW_AT_abstract_origin,
+                            dwarf::DW_FORM_ref4, AbsDIE);
+  else {
+    addString(VariableDie, dwarf::DW_AT_name, 
+                          dwarf::DW_FORM_string, Name);
+    addSourceLine(VariableDie, DV->getVariable());
+    addType(VariableDie, DV->getType());
+  }
+
+  if (DV->isArtificial())
+    addUInt(VariableDie, dwarf::DW_AT_artificial,
+                        dwarf::DW_FORM_flag, 1);
+
+  if (isScopeAbstract) {
+    DV->setDIE(VariableDie);
+    return VariableDie;
+  }
+
+  // Add variable address.
+
+  unsigned Offset = DV->getDotDebugLocOffset();
+  if (Offset != ~0U) {
+    addLabel(VariableDie, dwarf::DW_AT_location,
+                         dwarf::DW_FORM_data4,
+                         Asm->GetTempSymbol("debug_loc", Offset));
+    DV->setDIE(VariableDie);
+    return VariableDie;
+  }
+
+  // Check if variable is described by a DBG_VALUE instruction.
+  if (const MachineInstr *DVInsn = DV->getMInsn()) {
+    bool updated = false;
+    if (DVInsn->getNumOperands() == 3) {
+      if (DVInsn->getOperand(0).isReg()) {
+        const MachineOperand RegOp = DVInsn->getOperand(0);
+        const TargetRegisterInfo *TRI = Asm->TM.getRegisterInfo();
+        if (DVInsn->getOperand(1).isImm() &&
+            TRI->getFrameRegister(*Asm->MF) == RegOp.getReg()) {
+          unsigned FrameReg = 0;
+          const TargetFrameLowering *TFI = Asm->TM.getFrameLowering();
+          int Offset = 
+            TFI->getFrameIndexReference(*Asm->MF, 
+                                        DVInsn->getOperand(1).getImm(), 
+                                        FrameReg);
+          MachineLocation Location(FrameReg, Offset);
+          addVariableAddress(DV, VariableDie, Location);
+          
+        } else if (RegOp.getReg())
+          addVariableAddress(DV, VariableDie, 
+                                         MachineLocation(RegOp.getReg()));
+        updated = true;
+      }
+      else if (DVInsn->getOperand(0).isImm())
+        updated = 
+          addConstantValue(VariableDie, DVInsn->getOperand(0),
+                                       DV->getType());
+      else if (DVInsn->getOperand(0).isFPImm())
+        updated =
+          addConstantFPValue(VariableDie, DVInsn->getOperand(0));
+      else if (DVInsn->getOperand(0).isCImm())
+        updated =
+          addConstantValue(VariableDie, 
+                                       DVInsn->getOperand(0).getCImm(),
+                                       DV->getType().isUnsignedDIType());
+    } else {
+      addVariableAddress(DV, VariableDie, 
+                                     Asm->getDebugValueLocation(DVInsn));
+      updated = true;
+    }
+    if (!updated) {
+      // If variableDie is not updated then DBG_VALUE instruction does not
+      // have valid variable info.
+      delete VariableDie;
+      return NULL;
+    }
+    DV->setDIE(VariableDie);
+    return VariableDie;
+  } else {
+    // .. else use frame index.
+    int FI = DV->getFrameIndex();
+    if (FI != ~0) {
+      unsigned FrameReg = 0;
+      const TargetFrameLowering *TFI = Asm->TM.getFrameLowering();
+      int Offset = 
+        TFI->getFrameIndexReference(*Asm->MF, FI, FrameReg);
+      MachineLocation Location(FrameReg, Offset);
+      addVariableAddress(DV, VariableDie, Location);
+    }
+  }
+
+  DV->setDIE(VariableDie);
+  return VariableDie;
+}
+
 /// createMemberDIE - Create new member DIE.
 DIE *CompileUnit::createMemberDIE(DIDerivedType DT) {
   DIE *MemberDie = new DIE(DT.getTag());
@@ -1013,7 +1370,7 @@ DIE *CompileUnit::createMemberDIE(DIDerivedType DT) {
     addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag,
             dwarf::DW_ACCESS_private);
   // Otherwise C++ member and base classes are considered public.
-  else if (DT.getCompileUnit().getLanguage() == dwarf::DW_LANG_C_plus_plus)
+  else 
     addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag,
             dwarf::DW_ACCESS_public);
   if (DT.isVirtual())
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 213c7fc630d3..785926579fa4 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -67,6 +67,11 @@ class CompileUnit {
   /// DIEBlocks - A list of all the DIEBlocks in use.
   std::vector<DIEBlock *> DIEBlocks;
 
+  /// ContainingTypeMap - This map is used to keep track of subprogram DIEs that
+  /// need DW_AT_containing_type attribute. This attribute points to a DIE that
+  /// corresponds to the MDNode mapped with the subprogram DIE.
+  DenseMap<DIE *, const MDNode *> ContainingTypeMap;
+
 public:
   CompileUnit(unsigned I, DIE *D, AsmPrinter *A, DwarfDebug *DW);
   ~CompileUnit();
@@ -226,9 +231,12 @@ class CompileUnit {
   /// getOrCreateNameSpace - Create a DIE for DINameSpace.
   DIE *getOrCreateNameSpace(DINameSpace NS);
 
+  /// getOrCreateSubprogramDIE - Create new DIE using SP.
+  DIE *getOrCreateSubprogramDIE(DISubprogram SP);
+
   /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the
   /// given DIType.
-  DIE *getOrCreateTypeDIE(DIType Ty);
+  DIE *getOrCreateTypeDIE(const MDNode *N);
 
   /// getOrCreateTemplateTypeParameterDIE - Find existing DIE or create new DIE 
   /// for the given DITemplateTypeParameter.
@@ -242,6 +250,9 @@ class CompileUnit {
   /// information entry.
   DIEEntry *createDIEEntry(DIE *Entry);
 
+  /// createGlobalVariableDIE - create global variable DIE.
+  void createGlobalVariableDIE(const MDNode *N);
+
   void addPubTypes(DISubprogram SP);
 
   /// constructTypeDIE - Construct basic type die from DIBasicType.
@@ -266,6 +277,13 @@ class CompileUnit {
   /// constructEnumTypeDIE - Construct enum type DIE from DIEnumerator.
   DIE *constructEnumTypeDIE(DIEnumerator ETy);
 
+  /// constructContainingTypeDIEs - Construct DIEs for types that contain
+  /// vtables.
+  void constructContainingTypeDIEs();
+
+  /// constructVariableDIE - Construct a DIE for the given DbgVariable.
+  DIE *constructVariableDIE(DbgVariable *DV, bool isScopeAbstract);
+
   /// createMemberDIE - Create new member DIE.
   DIE *createMemberDIE(DIDerivedType DT);
 
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 125e1e86b12f..1b7e370fca09 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -24,7 +24,6 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
@@ -45,9 +44,6 @@
 #include "llvm/Support/Path.h"
 using namespace llvm;
 
-static cl::opt<bool> PrintDbgScope("print-dbgscope", cl::Hidden,
-     cl::desc("Print DbgScope information for each machine instruction"));
-
 static cl::opt<bool> DisableDebugInfoPrinting("disable-debug-info-print",
                                               cl::Hidden,
      cl::desc("Disable debug info printing"));
@@ -69,7 +65,7 @@ static const unsigned InitAbbreviationsSetSize = 9; // log2(512)
 
 namespace llvm {
 
-DIType DbgVariable::getType()               const {
+DIType DbgVariable::getType() const {
   DIType Ty = Var.getType();
   // FIXME: isBlockByrefVariable should be reformulated in terms of complex
   // addresses instead.
@@ -120,141 +116,12 @@ DIType DbgVariable::getType()               const {
   return Ty;
 }
 
-//===----------------------------------------------------------------------===//
-/// DbgRange - This is used to track range of instructions with identical
-/// debug info scope.
-///
-typedef std::pair<const MachineInstr *, const MachineInstr *> DbgRange;
-
-//===----------------------------------------------------------------------===//
-/// DbgScope - This class is used to track scope information.
-///
-class DbgScope {
-  DbgScope *Parent;                   // Parent to this scope.
-  DIDescriptor Desc;                  // Debug info descriptor for scope.
-  // Location at which this scope is inlined.
-  AssertingVH<const MDNode> InlinedAtLocation;
-  bool AbstractScope;                 // Abstract Scope
-  const MachineInstr *LastInsn;       // Last instruction of this scope.
-  const MachineInstr *FirstInsn;      // First instruction of this scope.
-  unsigned DFSIn, DFSOut;
-  // Scopes defined in scope.  Contents not owned.
-  SmallVector<DbgScope *, 4> Scopes;
-  // Variables declared in scope.  Contents owned.
-  SmallVector<DbgVariable *, 8> Variables;
-  SmallVector<DbgRange, 4> Ranges;
-  // Private state for dump()
-  mutable unsigned IndentLevel;
-public:
-  DbgScope(DbgScope *P, DIDescriptor D, const MDNode *I = 0)
-    : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(false),
-      LastInsn(0), FirstInsn(0),
-      DFSIn(0), DFSOut(0), IndentLevel(0) {}
-  virtual ~DbgScope();
-
-  // Accessors.
-  DbgScope *getParent()          const { return Parent; }
-  void setParent(DbgScope *P)          { Parent = P; }
-  DIDescriptor getDesc()         const { return Desc; }
-  const MDNode *getInlinedAt()         const { return InlinedAtLocation; }
-  const MDNode *getScopeNode()         const { return Desc; }
-  const SmallVector<DbgScope *, 4> &getScopes() { return Scopes; }
-  const SmallVector<DbgVariable *, 8> &getDbgVariables() { return Variables; }
-  const SmallVector<DbgRange, 4> &getRanges() { return Ranges; }
-
-  /// openInsnRange - This scope covers instruction range starting from MI.
-  void openInsnRange(const MachineInstr *MI) {
-    if (!FirstInsn)
-      FirstInsn = MI;
-
-    if (Parent)
-      Parent->openInsnRange(MI);
-  }
-
-  /// extendInsnRange - Extend the current instruction range covered by
-  /// this scope.
-  void extendInsnRange(const MachineInstr *MI) {
-    assert (FirstInsn && "MI Range is not open!");
-    LastInsn = MI;
-    if (Parent)
-      Parent->extendInsnRange(MI);
-  }
-
-  /// closeInsnRange - Create a range based on FirstInsn and LastInsn collected
-  /// until now. This is used when a new scope is encountered while walking
-  /// machine instructions.
-  void closeInsnRange(DbgScope *NewScope = NULL) {
-    assert (LastInsn && "Last insn missing!");
-    Ranges.push_back(DbgRange(FirstInsn, LastInsn));
-    FirstInsn = NULL;
-    LastInsn = NULL;
-    // If Parent dominates NewScope then do not close Parent's instruction
-    // range.
-    if (Parent && (!NewScope || !Parent->dominates(NewScope)))
-      Parent->closeInsnRange(NewScope);
-  }
-
-  void setAbstractScope() { AbstractScope = true; }
-  bool isAbstractScope() const { return AbstractScope; }
-
-  // Depth First Search support to walk and mainpluate DbgScope hierarchy.
-  unsigned getDFSOut() const { return DFSOut; }
-  void setDFSOut(unsigned O) { DFSOut = O; }
-  unsigned getDFSIn() const  { return DFSIn; }
-  void setDFSIn(unsigned I)  { DFSIn = I; }
-  bool dominates(const DbgScope *S) {
-    if (S == this)
-      return true;
-    if (DFSIn < S->getDFSIn() && DFSOut > S->getDFSOut())
-      return true;
-    return false;
-  }
-
-  /// addScope - Add a scope to the scope.
-  ///
-  void addScope(DbgScope *S) { Scopes.push_back(S); }
-
-  /// addVariable - Add a variable to the scope.
-  ///
-  void addVariable(DbgVariable *V) { Variables.push_back(V); }
-
-#ifndef NDEBUG
-  void dump() const;
-#endif
-};
-
 } // end llvm namespace
 
-#ifndef NDEBUG
-void DbgScope::dump() const {
-  raw_ostream &err = dbgs();
-  err.indent(IndentLevel);
-  err << "DFSIn: " << DFSIn << " DFSOut: " << DFSOut << "\n";
-  const MDNode *N = Desc;
-  N->dump();
-  if (AbstractScope)
-    err << "Abstract Scope\n";
-
-  IndentLevel += 2;
-  if (!Scopes.empty())
-    err << "Children ...\n";
-  for (unsigned i = 0, e = Scopes.size(); i != e; ++i)
-    if (Scopes[i] != this)
-      Scopes[i]->dump();
-
-  IndentLevel -= 2;
-}
-#endif
-
-DbgScope::~DbgScope() {
-  for (unsigned j = 0, M = Variables.size(); j < M; ++j)
-    delete Variables[j];
-}
-
 DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
   : Asm(A), MMI(Asm->MMI), FirstCU(0),
     AbbreviationsSet(InitAbbreviationsSetSize),
-    CurrentFnDbgScope(0), PrevLabel(NULL) {
+    PrevLabel(NULL) {
   NextStringPoolNumber = 0;
 
   DwarfInfoSectionSym = DwarfAbbrevSectionSym = 0;
@@ -311,147 +178,12 @@ static StringRef getRealLinkageName(StringRef LinkageName) {
   return LinkageName;
 }
 
-/// createSubprogramDIE - Create new DIE using SP.
-DIE *DwarfDebug::createSubprogramDIE(DISubprogram SP) {
-  CompileUnit *SPCU = getCompileUnit(SP);
-  DIE *SPDie = SPCU->getDIE(SP);
-  if (SPDie)
-    return SPDie;
-
-  SPDie = new DIE(dwarf::DW_TAG_subprogram);
-  
-  // DW_TAG_inlined_subroutine may refer to this DIE.
-  SPCU->insertDIE(SP, SPDie);
-  
-  // Add to context owner.
-  SPCU->addToContextOwner(SPDie, SP.getContext());
-
-  // Add function template parameters.
-  SPCU->addTemplateParams(*SPDie, SP.getTemplateParams());
-
-  StringRef LinkageName = SP.getLinkageName();
-  if (!LinkageName.empty())
-    SPCU->addString(SPDie, dwarf::DW_AT_MIPS_linkage_name, dwarf::DW_FORM_string,
-                    getRealLinkageName(LinkageName));
-
-  // If this DIE is going to refer declaration info using AT_specification
-  // then there is no need to add other attributes.
-  if (SP.getFunctionDeclaration().isSubprogram())
-    return SPDie;
-
-  // Constructors and operators for anonymous aggregates do not have names.
-  if (!SP.getName().empty())
-    SPCU->addString(SPDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, 
-                    SP.getName());
-
-  SPCU->addSourceLine(SPDie, SP);
-
-  if (SP.isPrototyped()) 
-    SPCU->addUInt(SPDie, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1);
-
-  // Add Return Type.
-  DICompositeType SPTy = SP.getType();
-  DIArray Args = SPTy.getTypeArray();
-  unsigned SPTag = SPTy.getTag();
-
-  if (Args.getNumElements() == 0 || SPTag != dwarf::DW_TAG_subroutine_type)
-    SPCU->addType(SPDie, SPTy);
-  else
-    SPCU->addType(SPDie, DIType(Args.getElement(0)));
-
-  unsigned VK = SP.getVirtuality();
-  if (VK) {
-    SPCU->addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_flag, VK);
-    DIEBlock *Block = SPCU->getDIEBlock();
-    SPCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu);
-    SPCU->addUInt(Block, 0, dwarf::DW_FORM_udata, SP.getVirtualIndex());
-    SPCU->addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, 0, Block);
-    ContainingTypeMap.insert(std::make_pair(SPDie,
-                                            SP.getContainingType()));
-  }
-
-  if (!SP.isDefinition()) {
-    SPCU->addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
-    
-    // Add arguments. Do not add arguments for subprogram definition. They will
-    // be handled while processing variables.
-    DICompositeType SPTy = SP.getType();
-    DIArray Args = SPTy.getTypeArray();
-    unsigned SPTag = SPTy.getTag();
-
-    if (SPTag == dwarf::DW_TAG_subroutine_type)
-      for (unsigned i = 1, N =  Args.getNumElements(); i < N; ++i) {
-        DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter);
-        DIType ATy = DIType(DIType(Args.getElement(i)));
-        SPCU->addType(Arg, ATy);
-        if (ATy.isArtificial())
-          SPCU->addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
-        SPDie->addChild(Arg);
-      }
-  }
-
-  if (SP.isArtificial())
-    SPCU->addUInt(SPDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
-
-  if (!SP.isLocalToUnit())
-    SPCU->addUInt(SPDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
-
-  if (SP.isOptimized())
-    SPCU->addUInt(SPDie, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1);
-
-  if (unsigned isa = Asm->getISAEncoding()) {
-    SPCU->addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa);
-  }
-
-  return SPDie;
-}
-
-DbgScope *DwarfDebug::getOrCreateAbstractScope(const MDNode *N) {
-  assert(N && "Invalid Scope encoding!");
-
-  DbgScope *AScope = AbstractScopes.lookup(N);
-  if (AScope)
-    return AScope;
-
-  DbgScope *Parent = NULL;
-
-  DIDescriptor Scope(N);
-  if (Scope.isLexicalBlock()) {
-    DILexicalBlock DB(N);
-    DIDescriptor ParentDesc = DB.getContext();
-    Parent = getOrCreateAbstractScope(ParentDesc);
-  }
-
-  AScope = new DbgScope(Parent, DIDescriptor(N), NULL);
-
-  if (Parent)
-    Parent->addScope(AScope);
-  AScope->setAbstractScope();
-  AbstractScopes[N] = AScope;
-  if (DIDescriptor(N).isSubprogram())
-    AbstractScopesList.push_back(AScope);
-  return AScope;
-}
-
-/// isSubprogramContext - Return true if Context is either a subprogram
-/// or another context nested inside a subprogram.
-static bool isSubprogramContext(const MDNode *Context) {
-  if (!Context)
-    return false;
-  DIDescriptor D(Context);
-  if (D.isSubprogram())
-    return true;
-  if (D.isType())
-    return isSubprogramContext(DIType(Context).getContext());
-  return false;
-}
-
 /// updateSubprogramScopeDIE - Find DIE for the given subprogram and
 /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes.
 /// If there are global variables in this scope then create and insert
 /// DIEs for these variables.
-DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) {
-  CompileUnit *SPCU = getCompileUnit(SPNode);
+DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU,
+                                          const MDNode *SPNode) {
   DIE *SPDie = SPCU->getDIE(SPNode);
 
   assert(SPDie && "Unable to find subprogram DIE!");
@@ -461,7 +193,7 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) {
   if (SPDecl.isSubprogram())
     // Refer function declaration directly.
     SPCU->addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4,
-                      createSubprogramDIE(SPDecl));
+                      SPCU->getOrCreateSubprogramDIE(SPDecl));
   else {
     // There is not any need to generate specification DIE for a function
     // defined at compile unit level. If a function is defined inside another
@@ -514,25 +246,26 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) {
 
 /// constructLexicalScope - Construct new DW_TAG_lexical_block
 /// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels.
-DIE *DwarfDebug::constructLexicalScopeDIE(DbgScope *Scope) {
+DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU, 
+                                          LexicalScope *Scope) {
 
   DIE *ScopeDIE = new DIE(dwarf::DW_TAG_lexical_block);
   if (Scope->isAbstractScope())
     return ScopeDIE;
 
-  const SmallVector<DbgRange, 4> &Ranges = Scope->getRanges();
+  const SmallVector<InsnRange, 4> &Ranges = Scope->getRanges();
   if (Ranges.empty())
     return 0;
 
-  CompileUnit *TheCU = getCompileUnit(Scope->getScopeNode());
-  SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin();
+  SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin();
   if (Ranges.size() > 1) {
     // .debug_range section has not been laid out yet. Emit offset in
     // .debug_range as a uint, size 4, for now. emitDIE will handle
     // DW_AT_ranges appropriately.
     TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4,
-                   DebugRangeSymbols.size() * Asm->getTargetData().getPointerSize());
-    for (SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(),
+                   DebugRangeSymbols.size() 
+                   * Asm->getTargetData().getPointerSize());
+    for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(),
          RE = Ranges.end(); RI != RE; ++RI) {
       DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first));
       DebugRangeSymbols.push_back(getLabelAfterInsn(RI->second));
@@ -559,22 +292,29 @@ DIE *DwarfDebug::constructLexicalScopeDIE(DbgScope *Scope) {
 /// constructInlinedScopeDIE - This scope represents inlined body of
 /// a function. Construct DIE to represent this concrete inlined copy
 /// of the function.
-DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) {
+DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU,
+                                          LexicalScope *Scope) {
 
-  const SmallVector<DbgRange, 4> &Ranges = Scope->getRanges();
+  const SmallVector<InsnRange, 4> &Ranges = Scope->getRanges();
   assert (Ranges.empty() == false
-          && "DbgScope does not have instruction markers!");
+          && "LexicalScope does not have instruction markers!");
 
-  // FIXME : .debug_inlined section specification does not clearly state how
-  // to emit inlined scope that is split into multiple instruction ranges.
-  // For now, use first instruction range and emit low_pc/high_pc pair and
-  // corresponding .debug_inlined section entry for this pair.
-  SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin();
+  if (!Scope->getScopeNode())
+    return NULL;
+  DIScope DS(Scope->getScopeNode());
+  DISubprogram InlinedSP = getDISubprogram(DS);
+  DIE *OriginDIE = TheCU->getDIE(InlinedSP);
+  if (!OriginDIE) {
+    DEBUG(dbgs() << "Unable to find original DIE for inlined subprogram.");
+    return NULL;
+  }
+
+  SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin();
   const MCSymbol *StartLabel = getLabelBeforeInsn(RI->first);
   const MCSymbol *EndLabel = getLabelAfterInsn(RI->second);
 
   if (StartLabel == 0 || EndLabel == 0) {
-    assert (0 && "Unexpected Start and End  labels for a inlined scope!");
+    assert (0 && "Unexpected Start and End labels for a inlined scope!");
     return 0;
   }
   assert(StartLabel->isDefined() &&
@@ -582,26 +322,38 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) {
   assert(EndLabel->isDefined() &&
          "Invalid end label for an inlined scope!");
 
-  if (!Scope->getScopeNode())
-    return NULL;
-  DIScope DS(Scope->getScopeNode());
-  DISubprogram InlinedSP = getDISubprogram(DS);
-  CompileUnit *TheCU = getCompileUnit(InlinedSP);
-  DIE *OriginDIE = TheCU->getDIE(InlinedSP);
-  if (!OriginDIE) {
-    DEBUG(dbgs() << "Unable to find original DIE for inlined subprogram.");
-    return NULL;
-  }
   DIE *ScopeDIE = new DIE(dwarf::DW_TAG_inlined_subroutine);
   TheCU->addDIEEntry(ScopeDIE, dwarf::DW_AT_abstract_origin,
                      dwarf::DW_FORM_ref4, OriginDIE);
 
-  TheCU->addLabel(ScopeDIE, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, StartLabel);
-  TheCU->addLabel(ScopeDIE, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, EndLabel);
+  if (Ranges.size() > 1) {
+    // .debug_range section has not been laid out yet. Emit offset in
+    // .debug_range as a uint, size 4, for now. emitDIE will handle
+    // DW_AT_ranges appropriately.
+    TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4,
+                   DebugRangeSymbols.size() 
+                   * Asm->getTargetData().getPointerSize());
+    for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(),
+         RE = Ranges.end(); RI != RE; ++RI) {
+      DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first));
+      DebugRangeSymbols.push_back(getLabelAfterInsn(RI->second));
+    }
+    DebugRangeSymbols.push_back(NULL);
+    DebugRangeSymbols.push_back(NULL);
+  } else {
+    TheCU->addLabel(ScopeDIE, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, 
+                    StartLabel);
+    TheCU->addLabel(ScopeDIE, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, 
+                    EndLabel);
+  }
 
   InlinedSubprogramDIEs.insert(OriginDIE);
 
   // Track the start label for this inlined function.
+  //.debug_inlined section specification does not clearly state how
+  // to emit inlined scope that is split into multiple instruction ranges.
+  // For now, use first instruction range and emit low_pc/high_pc pair and
+  // corresponding .debug_inlined section entry for this pair.
   DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator
     I = InlineInfo.find(InlinedSP);
 
@@ -619,200 +371,51 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) {
   return ScopeDIE;
 }
 
-/// isUnsignedDIType - Return true if type encoding is unsigned.
-static bool isUnsignedDIType(DIType Ty) {
-  DIDerivedType DTy(Ty);
-  if (DTy.Verify())
-    return isUnsignedDIType(DTy.getTypeDerivedFrom());
-
-  DIBasicType BTy(Ty);
-  if (BTy.Verify()) {
-    unsigned Encoding = BTy.getEncoding();
-    if (Encoding == dwarf::DW_ATE_unsigned ||
-        Encoding == dwarf::DW_ATE_unsigned_char)
-      return true;
-  }
-  return false;
-}
-
-/// constructVariableDIE - Construct a DIE for the given DbgVariable.
-DIE *DwarfDebug::constructVariableDIE(DbgVariable *DV, DbgScope *Scope) {
-  StringRef Name = DV->getName();
-  if (Name.empty())
-    return NULL;
-
-  // Translate tag to proper Dwarf tag.  The result variable is dropped for
-  // now.
-  unsigned Tag;
-  switch (DV->getTag()) {
-  case dwarf::DW_TAG_return_variable:
-    return NULL;
-  case dwarf::DW_TAG_arg_variable:
-    Tag = dwarf::DW_TAG_formal_parameter;
-    break;
-  case dwarf::DW_TAG_auto_variable:    // fall thru
-  default:
-    Tag = dwarf::DW_TAG_variable;
-    break;
-  }
-
-  // Define variable debug information entry.
-  DIE *VariableDie = new DIE(Tag);
-  CompileUnit *VariableCU = getCompileUnit(DV->getVariable());
-  DIE *AbsDIE = NULL;
-  DenseMap<const DbgVariable *, const DbgVariable *>::iterator
-    V2AVI = VarToAbstractVarMap.find(DV);
-  if (V2AVI != VarToAbstractVarMap.end())
-    AbsDIE = V2AVI->second->getDIE();
-
-  if (AbsDIE)
-    VariableCU->addDIEEntry(VariableDie, dwarf::DW_AT_abstract_origin,
-                       dwarf::DW_FORM_ref4, AbsDIE);
-  else {
-    VariableCU->addString(VariableDie, dwarf::DW_AT_name, dwarf::DW_FORM_string,
-                          Name);
-    VariableCU->addSourceLine(VariableDie, DV->getVariable());
-
-    // Add variable type.
-    VariableCU->addType(VariableDie, DV->getType());
-  }
-
-  if (Tag == dwarf::DW_TAG_formal_parameter && DV->getType().isArtificial())
-    VariableCU->addUInt(VariableDie, dwarf::DW_AT_artificial, 
-                        dwarf::DW_FORM_flag, 1);
-  else if (DIVariable(DV->getVariable()).isArtificial())
-    VariableCU->addUInt(VariableDie, dwarf::DW_AT_artificial, 
-                        dwarf::DW_FORM_flag, 1);
-
-  if (Scope->isAbstractScope()) {
-    DV->setDIE(VariableDie);
-    return VariableDie;
-  }
-
-  // Add variable address.
-
-  unsigned Offset = DV->getDotDebugLocOffset();
-  if (Offset != ~0U) {
-    VariableCU->addLabel(VariableDie, dwarf::DW_AT_location, dwarf::DW_FORM_data4,
-             Asm->GetTempSymbol("debug_loc", Offset));
-    DV->setDIE(VariableDie);
-    UseDotDebugLocEntry.insert(VariableDie);
-    return VariableDie;
-  }
-
-  // Check if variable is described by a  DBG_VALUE instruction.
-  DenseMap<const DbgVariable *, const MachineInstr *>::iterator DVI =
-    DbgVariableToDbgInstMap.find(DV);
-  if (DVI != DbgVariableToDbgInstMap.end()) {
-    const MachineInstr *DVInsn = DVI->second;
-    bool updated = false;
-    // FIXME : Handle getNumOperands != 3
-    if (DVInsn->getNumOperands() == 3) {
-      if (DVInsn->getOperand(0).isReg()) {
-        const MachineOperand RegOp = DVInsn->getOperand(0);
-        const TargetRegisterInfo *TRI = Asm->TM.getRegisterInfo();
-        if (DVInsn->getOperand(1).isImm() &&
-            TRI->getFrameRegister(*Asm->MF) == RegOp.getReg()) {
-          unsigned FrameReg = 0;
-          const TargetFrameLowering *TFI = Asm->TM.getFrameLowering();
-          int Offset = 
-            TFI->getFrameIndexReference(*Asm->MF, 
-                                        DVInsn->getOperand(1).getImm(), 
-                                        FrameReg);
-          MachineLocation Location(FrameReg, Offset);
-          VariableCU->addVariableAddress(DV, VariableDie, Location);
-          
-        } else if (RegOp.getReg())
-          VariableCU->addVariableAddress(DV, VariableDie, 
-                                         MachineLocation(RegOp.getReg()));
-        updated = true;
-      }
-      else if (DVInsn->getOperand(0).isImm())
-        updated = 
-          VariableCU->addConstantValue(VariableDie, DVInsn->getOperand(0),
-                                       DV->getType());
-      else if (DVInsn->getOperand(0).isFPImm())
-        updated =
-          VariableCU->addConstantFPValue(VariableDie, DVInsn->getOperand(0));
-      else if (DVInsn->getOperand(0).isCImm())
-        updated =
-          VariableCU->addConstantValue(VariableDie, 
-                                       DVInsn->getOperand(0).getCImm(),
-                                       isUnsignedDIType(DV->getType()));
-    } else {
-      VariableCU->addVariableAddress(DV, VariableDie, 
-                                     Asm->getDebugValueLocation(DVInsn));
-      updated = true;
-    }
-    if (!updated) {
-      // If variableDie is not updated then DBG_VALUE instruction does not
-      // have valid variable info.
-      delete VariableDie;
-      return NULL;
-    }
-    DV->setDIE(VariableDie);
-    return VariableDie;
-  }
-
-  // .. else use frame index, if available.
-  int FI = 0;
-  if (findVariableFrameIndex(DV, &FI)) {
-    unsigned FrameReg = 0;
-    const TargetFrameLowering *TFI = Asm->TM.getFrameLowering();
-    int Offset = 
-      TFI->getFrameIndexReference(*Asm->MF, FI, FrameReg);
-    MachineLocation Location(FrameReg, Offset);
-    VariableCU->addVariableAddress(DV, VariableDie, Location);
-  }
-
-  DV->setDIE(VariableDie);
-  return VariableDie;
-
-}
-
 /// constructScopeDIE - Construct a DIE for this scope.
-DIE *DwarfDebug::constructScopeDIE(DbgScope *Scope) {
+DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) {
   if (!Scope || !Scope->getScopeNode())
     return NULL;
 
   SmallVector <DIE *, 8> Children;
 
   // Collect arguments for current function.
-  if (Scope == CurrentFnDbgScope)
+  if (LScopes.isCurrentFunctionScope(Scope))
     for (unsigned i = 0, N = CurrentFnArguments.size(); i < N; ++i)
       if (DbgVariable *ArgDV = CurrentFnArguments[i])
-        if (DIE *Arg = constructVariableDIE(ArgDV, Scope))
+        if (DIE *Arg = 
+            TheCU->constructVariableDIE(ArgDV, Scope->isAbstractScope()))
           Children.push_back(Arg);
 
-  // Collect lexical scope childrens first.
-  const SmallVector<DbgVariable *, 8> &Variables = Scope->getDbgVariables();
+  // Collect lexical scope children first.
+  const SmallVector<DbgVariable *, 8> &Variables = ScopeVariables.lookup(Scope);
   for (unsigned i = 0, N = Variables.size(); i < N; ++i)
-    if (DIE *Variable = constructVariableDIE(Variables[i], Scope))
+    if (DIE *Variable = 
+        TheCU->constructVariableDIE(Variables[i], Scope->isAbstractScope()))
       Children.push_back(Variable);
-  const SmallVector<DbgScope *, 4> &Scopes = Scope->getScopes();
+  const SmallVector<LexicalScope *, 4> &Scopes = Scope->getChildren();
   for (unsigned j = 0, M = Scopes.size(); j < M; ++j)
-    if (DIE *Nested = constructScopeDIE(Scopes[j]))
+    if (DIE *Nested = constructScopeDIE(TheCU, Scopes[j]))
       Children.push_back(Nested);
   DIScope DS(Scope->getScopeNode());
   DIE *ScopeDIE = NULL;
   if (Scope->getInlinedAt())
-    ScopeDIE = constructInlinedScopeDIE(Scope);
+    ScopeDIE = constructInlinedScopeDIE(TheCU, Scope);
   else if (DS.isSubprogram()) {
     ProcessedSPNodes.insert(DS);
     if (Scope->isAbstractScope()) {
-      ScopeDIE = getCompileUnit(DS)->getDIE(DS);
+      ScopeDIE = TheCU->getDIE(DS);
       // Note down abstract DIE.
       if (ScopeDIE)
         AbstractSPDies.insert(std::make_pair(DS, ScopeDIE));
     }
     else
-      ScopeDIE = updateSubprogramScopeDIE(DS);
+      ScopeDIE = updateSubprogramScopeDIE(TheCU, DS);
   }
   else {
     // There is no need to emit empty lexical block DIE.
     if (Children.empty())
       return NULL;
-    ScopeDIE = constructLexicalScopeDIE(Scope);
+    ScopeDIE = constructLexicalScopeDIE(TheCU, Scope);
   }
   
   if (!ScopeDIE) return NULL;
@@ -823,7 +426,7 @@ DIE *DwarfDebug::constructScopeDIE(DbgScope *Scope) {
     ScopeDIE->addChild(*I);
 
   if (DS.isSubprogram())
-    getCompileUnit(DS)->addPubTypes(DISubprogram(DS));
+   TheCU->addPubTypes(DISubprogram(DS));
 
  return ScopeDIE;
 }
@@ -862,7 +465,7 @@ unsigned DwarfDebug::GetOrCreateSourceID(StringRef FileName,
 
 /// constructCompileUnit - Create new CompileUnit for the given
 /// metadata node with tag DW_TAG_compile_unit.
-void DwarfDebug::constructCompileUnit(const MDNode *N) {
+CompileUnit *DwarfDebug::constructCompileUnit(const MDNode *N) {
   DICompileUnit DIUnit(N);
   StringRef FN = DIUnit.getFilename();
   StringRef Dir = DIUnit.getDirectory();
@@ -893,7 +496,8 @@ void DwarfDebug::constructCompileUnit(const MDNode *N) {
 
   StringRef Flags = DIUnit.getFlags();
   if (!Flags.empty())
-    NewCU->addString(Die, dwarf::DW_AT_APPLE_flags, dwarf::DW_FORM_string, Flags);
+    NewCU->addString(Die, dwarf::DW_AT_APPLE_flags, dwarf::DW_FORM_string, 
+                     Flags);
   
   unsigned RVer = DIUnit.getRunTimeVersion();
   if (RVer)
@@ -903,159 +507,19 @@ void DwarfDebug::constructCompileUnit(const MDNode *N) {
   if (!FirstCU)
     FirstCU = NewCU;
   CUMap.insert(std::make_pair(N, NewCU));
-}
-
-/// getCompielUnit - Get CompileUnit DIE.
-CompileUnit *DwarfDebug::getCompileUnit(const MDNode *N) const {
-  assert (N && "Invalid DwarfDebug::getCompileUnit argument!");
-  DIDescriptor D(N);
-  const MDNode *CUNode = NULL;
-  if (D.isCompileUnit())
-    CUNode = N;
-  else if (D.isSubprogram())
-    CUNode = DISubprogram(N).getCompileUnit();
-  else if (D.isType())
-    CUNode = DIType(N).getCompileUnit();
-  else if (D.isGlobalVariable())
-    CUNode = DIGlobalVariable(N).getCompileUnit();
-  else if (D.isVariable())
-    CUNode = DIVariable(N).getCompileUnit();
-  else if (D.isNameSpace())
-    CUNode = DINameSpace(N).getCompileUnit();
-  else if (D.isFile())
-    CUNode = DIFile(N).getCompileUnit();
-  else
-    return FirstCU;
-
-  DenseMap<const MDNode *, CompileUnit *>::const_iterator I
-    = CUMap.find(CUNode);
-  if (I == CUMap.end())
-    return FirstCU;
-  return I->second;
-}
-
-// Return const exprssion if value is a GEP to access merged global
-// constant. e.g.
-// i8* getelementptr ({ i8, i8, i8, i8 }* @_MergedGlobals, i32 0, i32 0)
-static const ConstantExpr *getMergedGlobalExpr(const Value *V) {
-  const ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(V);
-  if (!CE || CE->getNumOperands() != 3 ||
-      CE->getOpcode() != Instruction::GetElementPtr)
-    return NULL;
-
-  // First operand points to a global value.
-  if (!isa<GlobalValue>(CE->getOperand(0)))
-    return NULL;
-
-  // Second operand is zero.
-  const ConstantInt *CI = 
-    dyn_cast_or_null<ConstantInt>(CE->getOperand(1));
-  if (!CI || !CI->isZero())
-    return NULL;
-
-  // Third operand is offset.
-  if (!isa<ConstantInt>(CE->getOperand(2)))
-    return NULL;
-
-  return CE;
-}
-
-/// constructGlobalVariableDIE - Construct global variable DIE.
-void DwarfDebug::constructGlobalVariableDIE(const MDNode *N) {
-  DIGlobalVariable GV(N);
-
-  // If debug information is malformed then ignore it.
-  if (GV.Verify() == false)
-    return;
-
-  // Check for pre-existence.
-  CompileUnit *TheCU = getCompileUnit(N);
-  if (TheCU->getDIE(GV))
-    return;
-
-  DIType GTy = GV.getType();
-  DIE *VariableDIE = new DIE(GV.getTag());
-
-  bool isGlobalVariable = GV.getGlobal() != NULL;
-
-  // Add name.
-  TheCU->addString(VariableDIE, dwarf::DW_AT_name, dwarf::DW_FORM_string,
-                   GV.getDisplayName());
-  StringRef LinkageName = GV.getLinkageName();
-  if (!LinkageName.empty() && isGlobalVariable)
-    TheCU->addString(VariableDIE, dwarf::DW_AT_MIPS_linkage_name, 
-                     dwarf::DW_FORM_string,
-                     getRealLinkageName(LinkageName));
-  // Add type.
-  TheCU->addType(VariableDIE, GTy);
-
-  // Add scoping info.
-  if (!GV.isLocalToUnit()) {
-    TheCU->addUInt(VariableDIE, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
-    // Expose as global. 
-    TheCU->addGlobal(GV.getName(), VariableDIE);
-  }
-  // Add line number info.
-  TheCU->addSourceLine(VariableDIE, GV);
-  // Add to map.
-  TheCU->insertDIE(N, VariableDIE);
-  // Add to context owner.
-  DIDescriptor GVContext = GV.getContext();
-  TheCU->addToContextOwner(VariableDIE, GVContext);
-  // Add location.
-  if (isGlobalVariable) {
-    DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
-    TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
-    TheCU->addLabel(Block, 0, dwarf::DW_FORM_udata,
-             Asm->Mang->getSymbol(GV.getGlobal()));
-    // Do not create specification DIE if context is either compile unit
-    // or a subprogram.
-    if (GV.isDefinition() && !GVContext.isCompileUnit() &&
-        !GVContext.isFile() && !isSubprogramContext(GVContext)) {
-      // Create specification DIE.
-      DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable);
-      TheCU->addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification,
-                  dwarf::DW_FORM_ref4, VariableDIE);
-      TheCU->addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block);
-      TheCU->addUInt(VariableDIE, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
-      TheCU->addDie(VariableSpecDIE);
-    } else {
-      TheCU->addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block);
-    } 
-  } else if (const ConstantInt *CI = 
-             dyn_cast_or_null<ConstantInt>(GV.getConstant()))
-    TheCU->addConstantValue(VariableDIE, CI, isUnsignedDIType(GTy));
-  else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) {
-    // GV is a merged global.
-    DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
-    TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
-    TheCU->addLabel(Block, 0, dwarf::DW_FORM_udata,
-                    Asm->Mang->getSymbol(cast<GlobalValue>(CE->getOperand(0))));
-    ConstantInt *CII = cast<ConstantInt>(CE->getOperand(2));
-    TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu);
-    TheCU->addUInt(Block, 0, dwarf::DW_FORM_udata, CII->getZExtValue());
-    TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_plus);
-    TheCU->addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block);
-  }
-
-  return;
+  return NewCU;
 }
 
 /// construct SubprogramDIE - Construct subprogram DIE.
-void DwarfDebug::constructSubprogramDIE(const MDNode *N) {
+void DwarfDebug::constructSubprogramDIE(CompileUnit *TheCU, 
+                                        const MDNode *N) {
   DISubprogram SP(N);
-
-  // Check for pre-existence.
-  CompileUnit *TheCU = getCompileUnit(N);
-  if (TheCU->getDIE(N))
-    return;
-
   if (!SP.isDefinition())
     // This is a method declaration which will be handled while constructing
     // class type.
     return;
 
-  DIE *SubprogramDie = createSubprogramDIE(SP);
+  DIE *SubprogramDie = TheCU->getOrCreateSubprogramDIE(SP);
 
   // Add to map.
   TheCU->insertDIE(N, SubprogramDie);
@@ -1066,71 +530,115 @@ void DwarfDebug::constructSubprogramDIE(const MDNode *N) {
   // Expose as global.
   TheCU->addGlobal(SP.getName(), SubprogramDie);
 
+  SPMap[N] = TheCU;
   return;
 }
 
+/// collectInfoFromNamedMDNodes - Collect debug info from named mdnodes such
+/// as llvm.dbg.enum and llvm.dbg.ty
+void DwarfDebug::collectInfoFromNamedMDNodes(Module *M) {
+  if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.sp"))
+    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
+      const MDNode *N = NMD->getOperand(i);
+      if (CompileUnit *CU = CUMap.lookup(DISubprogram(N).getCompileUnit()))
+        constructSubprogramDIE(CU, N);
+    }
+  
+  if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.gv"))
+    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
+      const MDNode *N = NMD->getOperand(i);
+      if (CompileUnit *CU = CUMap.lookup(DIGlobalVariable(N).getCompileUnit()))
+        CU->createGlobalVariableDIE(N);
+    }
+  
+  if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.enum"))
+    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
+      DIType Ty(NMD->getOperand(i));
+      if (CompileUnit *CU = CUMap.lookup(Ty.getCompileUnit()))
+        CU->getOrCreateTypeDIE(Ty);
+    }
+  
+  if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.ty"))
+    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
+      DIType Ty(NMD->getOperand(i));
+      if (CompileUnit *CU = CUMap.lookup(Ty.getCompileUnit()))
+        CU->getOrCreateTypeDIE(Ty);
+    }
+}
+
+/// collectLegacyDebugInfo - Collect debug info using DebugInfoFinder.
+/// FIXME - Remove this when dragon-egg and llvm-gcc switch to DIBuilder.
+bool DwarfDebug::collectLegacyDebugInfo(Module *M) {
+  DebugInfoFinder DbgFinder;
+  DbgFinder.processModule(*M);
+  
+  bool HasDebugInfo = false;
+  // Scan all the compile-units to see if there are any marked as the main
+  // unit. If not, we do not generate debug info.
+  for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(),
+         E = DbgFinder.compile_unit_end(); I != E; ++I) {
+    if (DICompileUnit(*I).isMain()) {
+      HasDebugInfo = true;
+      break;
+    }
+  }
+  if (!HasDebugInfo) return false;
+  
+  // Create all the compile unit DIEs.
+  for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(),
+         E = DbgFinder.compile_unit_end(); I != E; ++I)
+    constructCompileUnit(*I);
+  
+  // Create DIEs for each global variable.
+  for (DebugInfoFinder::iterator I = DbgFinder.global_variable_begin(),
+         E = DbgFinder.global_variable_end(); I != E; ++I) {
+    const MDNode *N = *I;
+    if (CompileUnit *CU = CUMap.lookup(DIGlobalVariable(N).getCompileUnit()))
+      CU->createGlobalVariableDIE(N);
+  }
+    
+  // Create DIEs for each subprogram.
+  for (DebugInfoFinder::iterator I = DbgFinder.subprogram_begin(),
+         E = DbgFinder.subprogram_end(); I != E; ++I) {
+    const MDNode *N = *I;
+    if (CompileUnit *CU = CUMap.lookup(DISubprogram(N).getCompileUnit()))
+      constructSubprogramDIE(CU, N);
+  }
+
+  return HasDebugInfo;
+}
+
 /// beginModule - Emit all Dwarf sections that should come prior to the
 /// content. Create global DIEs and emit initial debug info sections.
-/// This is inovked by the target AsmPrinter.
+/// This is invoked by the target AsmPrinter.
 void DwarfDebug::beginModule(Module *M) {
   if (DisableDebugInfoPrinting)
     return;
 
-  // If module has named metadata anchors then use them, otherwise scan the module
-  // using debug info finder to collect debug info.
+  // If module has named metadata anchors then use them, otherwise scan the
+  // module using debug info finder to collect debug info.
   NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
   if (CU_Nodes) {
-
-    NamedMDNode *GV_Nodes = M->getNamedMetadata("llvm.dbg.gv");
-    NamedMDNode *SP_Nodes = M->getNamedMetadata("llvm.dbg.sp");
-    if (!GV_Nodes && !SP_Nodes)
-      // If there are not any global variables or any functions then
-      // there is not any debug info in this module.
-      return;
-
-    for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i)
-      constructCompileUnit(CU_Nodes->getOperand(i));
-
-    if (GV_Nodes)
-      for (unsigned i = 0, e = GV_Nodes->getNumOperands(); i != e; ++i)
-        constructGlobalVariableDIE(GV_Nodes->getOperand(i));
-
-    if (SP_Nodes)
-      for (unsigned i = 0, e = SP_Nodes->getNumOperands(); i != e; ++i)
-        constructSubprogramDIE(SP_Nodes->getOperand(i));
-    
-  } else {
-
-    DebugInfoFinder DbgFinder;
-    DbgFinder.processModule(*M);
-    
-    bool HasDebugInfo = false;
-    // Scan all the compile-units to see if there are any marked as the main unit.
-    // if not, we do not generate debug info.
-    for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(),
-           E = DbgFinder.compile_unit_end(); I != E; ++I) {
-      if (DICompileUnit(*I).isMain()) {
-        HasDebugInfo = true;
-        break;
-      }
+    for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
+      DICompileUnit CUNode(CU_Nodes->getOperand(i));
+      CompileUnit *CU = constructCompileUnit(CUNode);
+      DIArray GVs = CUNode.getGlobalVariables();
+      for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i)
+        CU->createGlobalVariableDIE(GVs.getElement(i));
+      DIArray SPs = CUNode.getSubprograms();
+      for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i)
+        constructSubprogramDIE(CU, SPs.getElement(i));
+      DIArray EnumTypes = CUNode.getEnumTypes();
+      for (unsigned i = 0, e = EnumTypes.getNumElements(); i != e; ++i)
+        CU->getOrCreateTypeDIE(EnumTypes.getElement(i));
+      DIArray RetainedTypes = CUNode.getRetainedTypes();
+      for (unsigned i = 0, e = RetainedTypes.getNumElements(); i != e; ++i)
+        CU->getOrCreateTypeDIE(RetainedTypes.getElement(i));
     }
-    if (!HasDebugInfo) return;
-    
-    // Create all the compile unit DIEs.
-    for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(),
-           E = DbgFinder.compile_unit_end(); I != E; ++I)
-      constructCompileUnit(*I);
-    
-    // Create DIEs for each global variable.
-    for (DebugInfoFinder::iterator I = DbgFinder.global_variable_begin(),
-           E = DbgFinder.global_variable_end(); I != E; ++I)
-      constructGlobalVariableDIE(*I);
-    
-    // Create DIEs for each subprogram.
-    for (DebugInfoFinder::iterator I = DbgFinder.subprogram_begin(),
-           E = DbgFinder.subprogram_end(); I != E; ++I)
-      constructSubprogramDIE(*I);
-  }
+  } else if (!collectLegacyDebugInfo(M))
+    return;
+
+  collectInfoFromNamedMDNodes(M);
   
   // Tell MMI that we have debug info.
   MMI->setDebugInfoAvailability(true);
@@ -1138,19 +646,6 @@ void DwarfDebug::beginModule(Module *M) {
   // Emit initial sections.
   EmitSectionLabels();
 
-  //getOrCreateTypeDIE
-  if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.enum"))
-    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
-      DIType Ty(NMD->getOperand(i));
-      getCompileUnit(Ty)->getOrCreateTypeDIE(Ty);
-    }
-
-  if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.ty"))
-    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
-      DIType Ty(NMD->getOperand(i));
-      getCompileUnit(Ty)->getOrCreateTypeDIE(Ty);
-    }
-
   // Prime section data.
   SectionMap.insert(Asm->getObjFileLowering().getTextSection());
 }
@@ -1160,38 +655,38 @@ void DwarfDebug::beginModule(Module *M) {
 void DwarfDebug::endModule() {
   if (!FirstCU) return;
   const Module *M = MMI->getModule();
-  DenseMap<const MDNode *, DbgScope *> DeadFnScopeMap;
-  if (NamedMDNode *AllSPs = M->getNamedMetadata("llvm.dbg.sp")) {
-    for (unsigned SI = 0, SE = AllSPs->getNumOperands(); SI != SE; ++SI) {
-      if (ProcessedSPNodes.count(AllSPs->getOperand(SI)) != 0) continue;
-      DISubprogram SP(AllSPs->getOperand(SI));
-      if (!SP.Verify()) continue;
+  DenseMap<const MDNode *, LexicalScope *> DeadFnScopeMap;
 
-      // Collect info for variables that were optimized out.
-      if (!SP.isDefinition()) continue;
-      StringRef FName = SP.getLinkageName();
-      if (FName.empty())
-        FName = SP.getName();
-      NamedMDNode *NMD = getFnSpecificMDNode(*(MMI->getModule()), FName);
-      if (!NMD) continue;
-      unsigned E = NMD->getNumOperands();
-      if (!E) continue;
-      DbgScope *Scope = new DbgScope(NULL, DIDescriptor(SP), NULL);
-      DeadFnScopeMap[SP] = Scope;
-      for (unsigned I = 0; I != E; ++I) {
-        DIVariable DV(NMD->getOperand(I));
-        if (!DV.Verify()) continue;
-        Scope->addVariable(new DbgVariable(DV));
-      }
+  // Collect info for variables that were optimized out.
+  if (NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu")) {
+    for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
+      DICompileUnit TheCU(CU_Nodes->getOperand(i));
+      DIArray Subprograms = TheCU.getSubprograms();
+      for (unsigned i = 0, e = Subprograms.getNumElements(); i != e; ++i) {
+        DISubprogram SP(Subprograms.getElement(i));
+        if (ProcessedSPNodes.count(SP) != 0) continue;
+        if (!SP.Verify()) continue;
+        if (!SP.isDefinition()) continue;
+        DIArray Variables = SP.getVariables();
+        if (Variables.getNumElements() == 0) continue;
 
-      // Construct subprogram DIE and add variables DIEs.
-      constructSubprogramDIE(SP);
-      DIE *ScopeDIE = getCompileUnit(SP)->getDIE(SP);
-      const SmallVector<DbgVariable *, 8> &Variables = Scope->getDbgVariables();
-      for (unsigned i = 0, N = Variables.size(); i < N; ++i) {
-        DIE *VariableDIE = constructVariableDIE(Variables[i], Scope);
-        if (VariableDIE)
-          ScopeDIE->addChild(VariableDIE);
+        LexicalScope *Scope = 
+          new LexicalScope(NULL, DIDescriptor(SP), NULL, false);
+        DeadFnScopeMap[SP] = Scope;
+        
+        // Construct subprogram DIE and add variables DIEs.
+        CompileUnit *SPCU = CUMap.lookup(TheCU);
+        assert (SPCU && "Unable to find Compile Unit!");
+        constructSubprogramDIE(SPCU, SP);
+        DIE *ScopeDIE = SPCU->getDIE(SP);
+        for (unsigned vi = 0, ve = Variables.getNumElements(); vi != ve; ++vi) {
+          DIVariable DV(Variables.getElement(vi));
+          if (!DV.Verify()) continue;
+          DbgVariable *NewVar = new DbgVariable(DV, NULL);
+          if (DIE *VariableDIE = 
+              SPCU->constructVariableDIE(NewVar, Scope->isAbstractScope()))
+            ScopeDIE->addChild(VariableDIE);
+        }
       }
     }
   }
@@ -1203,15 +698,12 @@ void DwarfDebug::endModule() {
     FirstCU->addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined);
   }
 
-  for (DenseMap<DIE *, const MDNode *>::iterator CI = ContainingTypeMap.begin(),
-         CE = ContainingTypeMap.end(); CI != CE; ++CI) {
-    DIE *SPDie = CI->first;
-    const MDNode *N = dyn_cast_or_null<MDNode>(CI->second);
-    if (!N) continue;
-    DIE *NDie = getCompileUnit(N)->getDIE(N);
-    if (!NDie) continue;
-    getCompileUnit(N)->addDIEEntry(SPDie, dwarf::DW_AT_containing_type, 
-                                   dwarf::DW_FORM_ref4, NDie);
+  // Emit DW_AT_containing_type attribute to connect types with their
+  // vtable holding type.
+  for (DenseMap<const MDNode *, CompileUnit *>::iterator CUI = CUMap.begin(),
+         CUE = CUMap.end(); CUI != CUE; ++CUI) {
+    CompileUnit *TheCU = CUI->second;
+    TheCU->constructContainingTypeDIEs();
   }
 
   // Standard sections final addresses.
@@ -1261,6 +753,7 @@ void DwarfDebug::endModule() {
 
   // clean up.
   DeleteContainerSeconds(DeadFnScopeMap);
+  SPMap.clear();
   for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(),
          E = CUMap.end(); I != E; ++I)
     delete I->second;
@@ -1268,29 +761,30 @@ void DwarfDebug::endModule() {
 }
 
 /// findAbstractVariable - Find abstract variable, if any, associated with Var.
-DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &Var,
+DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &DV,
                                               DebugLoc ScopeLoc) {
-
+  LLVMContext &Ctx = DV->getContext();
+  // More then one inlined variable corresponds to one abstract variable.
+  DIVariable Var = cleanseInlinedVariable(DV, Ctx);
   DbgVariable *AbsDbgVariable = AbstractVariables.lookup(Var);
   if (AbsDbgVariable)
     return AbsDbgVariable;
 
-  LLVMContext &Ctx = Var->getContext();
-  DbgScope *Scope = AbstractScopes.lookup(ScopeLoc.getScope(Ctx));
+  LexicalScope *Scope = LScopes.findAbstractScope(ScopeLoc.getScope(Ctx));
   if (!Scope)
     return NULL;
 
-  AbsDbgVariable = new DbgVariable(Var);
-  Scope->addVariable(AbsDbgVariable);
+  AbsDbgVariable = new DbgVariable(Var, NULL);
+  addScopeVariable(Scope, AbsDbgVariable);
   AbstractVariables[Var] = AbsDbgVariable;
   return AbsDbgVariable;
 }
 
-/// addCurrentFnArgument - If Var is an current function argument that add
-/// it in CurrentFnArguments list.
+/// addCurrentFnArgument - If Var is a current function argument then add
+/// it to CurrentFnArguments list.
 bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF,
-                                      DbgVariable *Var, DbgScope *Scope) {
-  if (Scope != CurrentFnDbgScope) 
+                                      DbgVariable *Var, LexicalScope *Scope) {
+  if (!LScopes.isCurrentFunctionScope(Scope))
     return false;
   DIVariable DV = Var->getVariable();
   if (DV.getTag() != dwarf::DW_TAG_arg_variable)
@@ -1313,7 +807,7 @@ bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF,
 /// collectVariableInfoFromMMITable - Collect variable information from
 /// side table maintained by MMI.
 void
-DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction * MF,
+DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction *MF,
                                    SmallPtrSet<const MDNode *, 16> &Processed) {
   MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo();
   for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(),
@@ -1324,21 +818,19 @@ DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction * MF,
     DIVariable DV(Var);
     const std::pair<unsigned, DebugLoc> &VP = VI->second;
 
-    DbgScope *Scope = findDbgScope(VP.second);
+    LexicalScope *Scope = LScopes.findLexicalScope(VP.second);
 
     // If variable scope is not found then skip this variable.
     if (Scope == 0)
       continue;
 
     DbgVariable *AbsDbgVariable = findAbstractVariable(DV, VP.second);
-    DbgVariable *RegVar = new DbgVariable(DV);
-    recordVariableFrameIndex(RegVar, VP.first);
+    DbgVariable *RegVar = new DbgVariable(DV, AbsDbgVariable);
+    RegVar->setFrameIndex(VP.first);
     if (!addCurrentFnArgument(MF, RegVar, Scope))
-      Scope->addVariable(RegVar);
-    if (AbsDbgVariable) {
-      recordVariableFrameIndex(AbsDbgVariable, VP.first);
-      VarToAbstractVarMap[RegVar] = AbsDbgVariable;
-    }
+      addScopeVariable(Scope, RegVar);
+    if (AbsDbgVariable)
+      AbsDbgVariable->setFrameIndex(VP.first);
   }
 }
 
@@ -1351,7 +843,7 @@ static bool isDbgValueInDefinedReg(const MachineInstr *MI) {
          MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0;
 }
 
-/// getDebugLocEntry - Get .debug_loc entry for the instraction range starting
+/// getDebugLocEntry - Get .debug_loc entry for the instruction range starting
 /// at MI.
 static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm, 
                                          const MCSymbol *FLabel, 
@@ -1379,7 +871,7 @@ static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm,
   return DotDebugLocEntry();
 }
 
-/// collectVariableInfo - Populate DbgScope entries with variables' info.
+/// collectVariableInfo - Find variables for each lexical scope.
 void
 DwarfDebug::collectVariableInfo(const MachineFunction *MF,
                                 SmallPtrSet<const MDNode *, 16> &Processed) {
@@ -1402,30 +894,37 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF,
     const MachineInstr *MInsn = History.front();
 
     DIVariable DV(Var);
-    DbgScope *Scope = NULL;
+    LexicalScope *Scope = NULL;
     if (DV.getTag() == dwarf::DW_TAG_arg_variable &&
         DISubprogram(DV.getContext()).describes(MF->getFunction()))
-      Scope = CurrentFnDbgScope;
-    else
-      Scope = findDbgScope(MInsn->getDebugLoc());
+      Scope = LScopes.getCurrentFunctionScope();
+    else {
+      if (DV.getVersion() <= LLVMDebugVersion9)
+        Scope = LScopes.findLexicalScope(MInsn->getDebugLoc());
+      else {
+        if (MDNode *IA = DV.getInlinedAt())
+          Scope = LScopes.findInlinedScope(DebugLoc::getFromDILocation(IA));
+        else
+          Scope = LScopes.findLexicalScope(cast<MDNode>(DV->getOperand(1)));
+      }
+    }
     // If variable scope is not found then skip this variable.
     if (!Scope)
       continue;
 
     Processed.insert(DV);
     assert(MInsn->isDebugValue() && "History must begin with debug value");
-    DbgVariable *RegVar = new DbgVariable(DV);
+    DbgVariable *AbsVar = findAbstractVariable(DV, MInsn->getDebugLoc());
+    DbgVariable *RegVar = new DbgVariable(DV, AbsVar);
     if (!addCurrentFnArgument(MF, RegVar, Scope))
-      Scope->addVariable(RegVar);
-    if (DbgVariable *AbsVar = findAbstractVariable(DV, MInsn->getDebugLoc())) {
-      DbgVariableToDbgInstMap[AbsVar] = MInsn;
-      VarToAbstractVarMap[RegVar] = AbsVar;
-    }
+      addScopeVariable(Scope, RegVar);
+    if (AbsVar)
+      AbsVar->setMInsn(MInsn);
 
     // Simple ranges that are fully coalesced.
     if (History.size() <= 1 || (History.size() == 2 &&
                                 MInsn->isIdenticalTo(History.back()))) {
-      DbgVariableToDbgInstMap[RegVar] = MInsn;
+      RegVar->setMInsn(MInsn);
       continue;
     }
 
@@ -1471,16 +970,14 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF,
   }
 
   // Collect info for variables that were optimized out.
-  const Function *F = MF->getFunction();
-  if (NamedMDNode *NMD = getFnSpecificMDNode(*(F->getParent()), F->getName())) {
-    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
-      DIVariable DV(cast<MDNode>(NMD->getOperand(i)));
-      if (!DV || !Processed.insert(DV))
-        continue;
-      DbgScope *Scope = DbgScopeMap.lookup(DV.getContext());
-      if (Scope)
-        Scope->addVariable(new DbgVariable(DV));
-    }
+  LexicalScope *FnScope = LScopes.getCurrentFunctionScope();
+  DIArray Variables = DISubprogram(FnScope->getScopeNode()).getVariables();
+  for (unsigned i = 0, e = Variables.getNumElements(); i != e; ++i) {
+    DIVariable DV(Variables.getElement(i));
+    if (!DV || !DV.Verify() || !Processed.insert(DV))
+      continue;
+    if (LexicalScope *Scope = LScopes.findLexicalScope(DV.getContext()))
+      addScopeVariable(Scope, new DbgVariable(DV, NULL));
   }
 }
 
@@ -1561,237 +1058,33 @@ void DwarfDebug::endInstruction(const MachineInstr *MI) {
   I->second = PrevLabel;
 }
 
-/// getOrCreateDbgScope - Create DbgScope for the scope.
-DbgScope *DwarfDebug::getOrCreateDbgScope(DebugLoc DL) {
-  LLVMContext &Ctx = Asm->MF->getFunction()->getContext();
-  MDNode *Scope = NULL;
-  MDNode *InlinedAt = NULL;
-  DL.getScopeAndInlinedAt(Scope, InlinedAt, Ctx);
-
-  if (!InlinedAt) {
-    DbgScope *WScope = DbgScopeMap.lookup(Scope);
-    if (WScope)
-      return WScope;
-    WScope = new DbgScope(NULL, DIDescriptor(Scope), NULL);
-    DbgScopeMap.insert(std::make_pair(Scope, WScope));
-    if (DIDescriptor(Scope).isLexicalBlock()) {
-      DbgScope *Parent =
-        getOrCreateDbgScope(DebugLoc::getFromDILexicalBlock(Scope));
-      WScope->setParent(Parent);
-      Parent->addScope(WScope);
-    } else if (DIDescriptor(Scope).isSubprogram()
-               && DISubprogram(Scope).describes(Asm->MF->getFunction()))
-      CurrentFnDbgScope = WScope;
-
-    return WScope;
-  }
-
-  getOrCreateAbstractScope(Scope);
-  DbgScope *WScope = DbgScopeMap.lookup(InlinedAt);
-  if (WScope)
-    return WScope;
-
-  WScope = new DbgScope(NULL, DIDescriptor(Scope), InlinedAt);
-  DbgScopeMap.insert(std::make_pair(InlinedAt, WScope));
-  InlinedDbgScopeMap[DebugLoc::getFromDILocation(InlinedAt)] = WScope;
-  DbgScope *Parent =
-    getOrCreateDbgScope(DebugLoc::getFromDILocation(InlinedAt));
-  WScope->setParent(Parent);
-  Parent->addScope(WScope);
-  return WScope;
-}
-
-/// calculateDominanceGraph - Calculate dominance graph for DbgScope
-/// hierarchy.
-static void calculateDominanceGraph(DbgScope *Scope) {
-  assert (Scope && "Unable to calculate scop edominance graph!");
-  SmallVector<DbgScope *, 4> WorkStack;
-  WorkStack.push_back(Scope);
-  unsigned Counter = 0;
-  while (!WorkStack.empty()) {
-    DbgScope *WS = WorkStack.back();
-    const SmallVector<DbgScope *, 4> &Children = WS->getScopes();
-    bool visitedChildren = false;
-    for (SmallVector<DbgScope *, 4>::const_iterator SI = Children.begin(),
-           SE = Children.end(); SI != SE; ++SI) {
-      DbgScope *ChildScope = *SI;
-      if (!ChildScope->getDFSOut()) {
-        WorkStack.push_back(ChildScope);
-        visitedChildren = true;
-        ChildScope->setDFSIn(++Counter);
-        break;
-      }
-    }
-    if (!visitedChildren) {
-      WorkStack.pop_back();
-      WS->setDFSOut(++Counter);
-    }
-  }
-}
-
-/// printDbgScopeInfo - Print DbgScope info for each machine instruction.
-static
-void printDbgScopeInfo(const MachineFunction *MF,
-                       DenseMap<const MachineInstr *, DbgScope *> &MI2ScopeMap)
-{
-#ifndef NDEBUG
-  LLVMContext &Ctx = MF->getFunction()->getContext();
-  unsigned PrevDFSIn = 0;
-  for (MachineFunction::const_iterator I = MF->begin(), E = MF->end();
-       I != E; ++I) {
-    for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end();
-         II != IE; ++II) {
-      const MachineInstr *MInsn = II;
-      MDNode *Scope = NULL;
-      MDNode *InlinedAt = NULL;
-
-      // Check if instruction has valid location information.
-      DebugLoc MIDL = MInsn->getDebugLoc();
-      if (!MIDL.isUnknown()) {
-        MIDL.getScopeAndInlinedAt(Scope, InlinedAt, Ctx);
-        dbgs() << " [ ";
-        if (InlinedAt)
-          dbgs() << "*";
-        DenseMap<const MachineInstr *, DbgScope *>::iterator DI =
-          MI2ScopeMap.find(MInsn);
-        if (DI != MI2ScopeMap.end()) {
-          DbgScope *S = DI->second;
-          dbgs() << S->getDFSIn();
-          PrevDFSIn = S->getDFSIn();
-        } else
-          dbgs() << PrevDFSIn;
-      } else
-        dbgs() << " [ x" << PrevDFSIn;
-      dbgs() << " ]";
-      MInsn->dump();
-    }
-    dbgs() << "\n";
-  }
-#endif
-}
-/// extractScopeInformation - Scan machine instructions in this function
-/// and collect DbgScopes. Return true, if at least one scope was found.
-bool DwarfDebug::extractScopeInformation() {
-  // If scope information was extracted using .dbg intrinsics then there is not
-  // any need to extract these information by scanning each instruction.
-  if (!DbgScopeMap.empty())
-    return false;
-
-  // Scan each instruction and create scopes. First build working set of scopes.
-  SmallVector<DbgRange, 4> MIRanges;
-  DenseMap<const MachineInstr *, DbgScope *> MI2ScopeMap;
-  DebugLoc PrevDL;
-  const MachineInstr *RangeBeginMI = NULL;
-  const MachineInstr *PrevMI = NULL;
-  for (MachineFunction::const_iterator I = Asm->MF->begin(), E = Asm->MF->end();
-       I != E; ++I) {
-    for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end();
-         II != IE; ++II) {
-      const MachineInstr *MInsn = II;
-
-      // Check if instruction has valid location information.
-      const DebugLoc MIDL = MInsn->getDebugLoc();
-      if (MIDL.isUnknown()) {
-        PrevMI = MInsn;
-        continue;
-      }
-
-      // If scope has not changed then skip this instruction.
-      if (MIDL == PrevDL) {
-        PrevMI = MInsn;
-        continue;
-      }
-
-      // Ignore DBG_VALUE. It does not contribute any instruction in output.
-      if (MInsn->isDebugValue())
-        continue;
-
-      if (RangeBeginMI) {
-        // If we have alread seen a beginning of a instruction range and
-        // current instruction scope does not match scope of first instruction
-        // in this range then create a new instruction range.
-        DEBUG(dbgs() << "Creating new instruction range :\n");
-        DEBUG(dbgs() << "Begin Range at " << *RangeBeginMI);
-        DEBUG(dbgs() << "End Range at " << *PrevMI);
-        DEBUG(dbgs() << "Next Range starting at " << *MInsn);
-        DEBUG(dbgs() << "------------------------\n");
-        DbgRange R(RangeBeginMI, PrevMI);
-        MI2ScopeMap[RangeBeginMI] = getOrCreateDbgScope(PrevDL);
-        MIRanges.push_back(R);
-      }
-
-      // This is a beginning of a new instruction range.
-      RangeBeginMI = MInsn;
-
-      // Reset previous markers.
-      PrevMI = MInsn;
-      PrevDL = MIDL;
-    }
-  }
-
-  // Create last instruction range.
-  if (RangeBeginMI && PrevMI && !PrevDL.isUnknown()) {
-    DbgRange R(RangeBeginMI, PrevMI);
-    MIRanges.push_back(R);
-    MI2ScopeMap[RangeBeginMI] = getOrCreateDbgScope(PrevDL);
-  }
-
-  if (!CurrentFnDbgScope)
-    return false;
-
-  calculateDominanceGraph(CurrentFnDbgScope);
-  if (PrintDbgScope)
-    printDbgScopeInfo(Asm->MF, MI2ScopeMap);
-
-  // Find ranges of instructions covered by each DbgScope;
-  DbgScope *PrevDbgScope = NULL;
-  for (SmallVector<DbgRange, 4>::const_iterator RI = MIRanges.begin(),
-         RE = MIRanges.end(); RI != RE; ++RI) {
-    const DbgRange &R = *RI;
-    DbgScope *S = MI2ScopeMap.lookup(R.first);
-    assert (S && "Lost DbgScope for a machine instruction!");
-    if (PrevDbgScope && !PrevDbgScope->dominates(S))
-      PrevDbgScope->closeInsnRange(S);
-    S->openInsnRange(R.first);
-    S->extendInsnRange(R.second);
-    PrevDbgScope = S;
-  }
-
-  if (PrevDbgScope)
-    PrevDbgScope->closeInsnRange();
-
-  identifyScopeMarkers();
-
-  return !DbgScopeMap.empty();
-}
-
 /// identifyScopeMarkers() -
-/// Each DbgScope has first instruction and last instruction to mark beginning
-/// and end of a scope respectively. Create an inverse map that list scopes
-/// starts (and ends) with an instruction. One instruction may start (or end)
-/// multiple scopes. Ignore scopes that are not reachable.
+/// Each LexicalScope has first instruction and last instruction to mark
+/// beginning and end of a scope respectively. Create an inverse map that list
+/// scopes starts (and ends) with an instruction. One instruction may start (or
+/// end) multiple scopes. Ignore scopes that are not reachable.
 void DwarfDebug::identifyScopeMarkers() {
-  SmallVector<DbgScope *, 4> WorkList;
-  WorkList.push_back(CurrentFnDbgScope);
+  SmallVector<LexicalScope *, 4> WorkList;
+  WorkList.push_back(LScopes.getCurrentFunctionScope());
   while (!WorkList.empty()) {
-    DbgScope *S = WorkList.pop_back_val();
+    LexicalScope *S = WorkList.pop_back_val();
 
-    const SmallVector<DbgScope *, 4> &Children = S->getScopes();
+    const SmallVector<LexicalScope *, 4> &Children = S->getChildren();
     if (!Children.empty())
-      for (SmallVector<DbgScope *, 4>::const_iterator SI = Children.begin(),
+      for (SmallVector<LexicalScope *, 4>::const_iterator SI = Children.begin(),
              SE = Children.end(); SI != SE; ++SI)
         WorkList.push_back(*SI);
 
     if (S->isAbstractScope())
       continue;
 
-    const SmallVector<DbgRange, 4> &Ranges = S->getRanges();
+    const SmallVector<InsnRange, 4> &Ranges = S->getRanges();
     if (Ranges.empty())
       continue;
-    for (SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(),
+    for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(),
            RE = Ranges.end(); RI != RE; ++RI) {
-      assert(RI->first && "DbgRange does not have first instruction!");
-      assert(RI->second && "DbgRange does not have second instruction!");
+      assert(RI->first && "InsnRange does not have first instruction!");
+      assert(RI->second && "InsnRange does not have second instruction!");
       requestLabelBeforeInsn(RI->first);
       requestLabelAfterInsn(RI->second);
     }
@@ -1819,7 +1112,9 @@ static DebugLoc getFnDebugLoc(DebugLoc DL, const LLVMContext &Ctx) {
 /// emitted immediately after the function entry point.
 void DwarfDebug::beginFunction(const MachineFunction *MF) {
   if (!MMI->hasDebugInfo()) return;
-  if (!extractScopeInformation()) return;
+  LScopes.initialize(*MF);
+  if (LScopes.empty()) return;
+  identifyScopeMarkers();
 
   FunctionBeginSym = Asm->GetTempSymbol("func_begin",
                                         Asm->getFunctionNumber());
@@ -1953,7 +1248,8 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
     const MachineInstr *Prev = History.back();
     if (Prev->isDebugValue() && isDbgValueInDefinedReg(Prev)) {
       const MachineBasicBlock *PrevMBB = Prev->getParent();
-      MachineBasicBlock::const_iterator LastMI = PrevMBB->getLastNonDebugInstr();
+      MachineBasicBlock::const_iterator LastMI = 
+        PrevMBB->getLastNonDebugInstr();
       if (LastMI == PrevMBB->end())
         // Drop DBG_VALUE for empty range.
         History.pop_back();
@@ -1985,110 +1281,73 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
   }
 }
 
+void DwarfDebug::addScopeVariable(LexicalScope *LS, DbgVariable *Var) {
+//  SmallVector<DbgVariable *, 8> &Vars = ScopeVariables.lookup(LS);
+  ScopeVariables[LS].push_back(Var);
+//  Vars.push_back(Var);
+}
+
 /// endFunction - Gather and emit post-function debug information.
 ///
 void DwarfDebug::endFunction(const MachineFunction *MF) {
-  if (!MMI->hasDebugInfo() || DbgScopeMap.empty()) return;
+  if (!MMI->hasDebugInfo() || LScopes.empty()) return;
 
-  if (CurrentFnDbgScope) {
+  // Define end label for subprogram.
+  FunctionEndSym = Asm->GetTempSymbol("func_end",
+                                      Asm->getFunctionNumber());
+  // Assumes in correct section after the entry point.
+  Asm->OutStreamer.EmitLabel(FunctionEndSym);
+  
+  SmallPtrSet<const MDNode *, 16> ProcessedVars;
+  collectVariableInfo(MF, ProcessedVars);
+  
+  LexicalScope *FnScope = LScopes.getCurrentFunctionScope();
+  CompileUnit *TheCU = SPMap.lookup(FnScope->getScopeNode());
+  assert (TheCU && "Unable to find compile unit!");
 
-    // Define end label for subprogram.
-    FunctionEndSym = Asm->GetTempSymbol("func_end",
-                                        Asm->getFunctionNumber());
-    // Assumes in correct section after the entry point.
-    Asm->OutStreamer.EmitLabel(FunctionEndSym);
-
-    SmallPtrSet<const MDNode *, 16> ProcessedVars;
-    collectVariableInfo(MF, ProcessedVars);
-
-    // Construct abstract scopes.
-    for (SmallVector<DbgScope *, 4>::iterator AI = AbstractScopesList.begin(),
-           AE = AbstractScopesList.end(); AI != AE; ++AI) {
-      DISubprogram SP((*AI)->getScopeNode());
-      if (SP.Verify()) {
-        // Collect info for variables that were optimized out.
-        StringRef FName = SP.getLinkageName();
-        if (FName.empty())
-          FName = SP.getName();
-        if (NamedMDNode *NMD = 
-            getFnSpecificMDNode(*(MF->getFunction()->getParent()), FName)) {
-          for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
-          DIVariable DV(cast<MDNode>(NMD->getOperand(i)));
-          if (!DV || !ProcessedVars.insert(DV))
-            continue;
-          DbgScope *Scope = AbstractScopes.lookup(DV.getContext());
-          if (Scope)
-            Scope->addVariable(new DbgVariable(DV));
-          }
-        }
+  // Construct abstract scopes.
+  ArrayRef<LexicalScope *> AList = LScopes.getAbstractScopesList();
+  for (unsigned i = 0, e = AList.size(); i != e; ++i) {
+    LexicalScope *AScope = AList[i];
+    DISubprogram SP(AScope->getScopeNode());
+    if (SP.Verify()) {
+      // Collect info for variables that were optimized out.
+      DIArray Variables = SP.getVariables();
+      for (unsigned i = 0, e = Variables.getNumElements(); i != e; ++i) {
+        DIVariable DV(Variables.getElement(i));
+        if (!DV || !DV.Verify() || !ProcessedVars.insert(DV))
+          continue;
+        if (LexicalScope *Scope = LScopes.findAbstractScope(DV.getContext()))
+          addScopeVariable(Scope, new DbgVariable(DV, NULL));
       }
-      if (ProcessedSPNodes.count((*AI)->getScopeNode()) == 0)
-        constructScopeDIE(*AI);
     }
-
-    DIE *CurFnDIE = constructScopeDIE(CurrentFnDbgScope);
-
-    if (!DisableFramePointerElim(*MF))
-      getCompileUnit(CurrentFnDbgScope->getScopeNode())->addUInt(CurFnDIE, 
-                                                                 dwarf::DW_AT_APPLE_omit_frame_ptr,
-                                                                 dwarf::DW_FORM_flag, 1);
-
-
-    DebugFrames.push_back(FunctionDebugFrameInfo(Asm->getFunctionNumber(),
-                                                 MMI->getFrameMoves()));
+    if (ProcessedSPNodes.count(AScope->getScopeNode()) == 0)
+      constructScopeDIE(TheCU, AScope);
   }
+  
+  DIE *CurFnDIE = constructScopeDIE(TheCU, FnScope);
+  
+  if (!DisableFramePointerElim(*MF))
+    TheCU->addUInt(CurFnDIE, dwarf::DW_AT_APPLE_omit_frame_ptr,
+                   dwarf::DW_FORM_flag, 1);
+
+  DebugFrames.push_back(FunctionDebugFrameInfo(Asm->getFunctionNumber(),
+                                               MMI->getFrameMoves()));
 
   // Clear debug info
-  CurrentFnDbgScope = NULL;
+  for (DenseMap<LexicalScope *, SmallVector<DbgVariable *, 8> >::iterator
+         I = ScopeVariables.begin(), E = ScopeVariables.end(); I != E; ++I)
+    DeleteContainerPointers(I->second);
+  ScopeVariables.clear();
   DeleteContainerPointers(CurrentFnArguments);
-  DbgVariableToFrameIndexMap.clear();
-  VarToAbstractVarMap.clear();
-  DbgVariableToDbgInstMap.clear();
-  InlinedDbgScopeMap.clear();
-  DeleteContainerSeconds(DbgScopeMap);
   UserVariables.clear();
   DbgValues.clear();
-  DeleteContainerSeconds(AbstractScopes);
-  AbstractScopesList.clear();
   AbstractVariables.clear();
   LabelsBeforeInsn.clear();
   LabelsAfterInsn.clear();
   PrevLabel = NULL;
 }
 
-/// recordVariableFrameIndex - Record a variable's index.
-void DwarfDebug::recordVariableFrameIndex(const DbgVariable *V, int Index) {
-  assert (V && "Invalid DbgVariable!");
-  DbgVariableToFrameIndexMap[V] = Index;
-}
-
-/// findVariableFrameIndex - Return true if frame index for the variable
-/// is found. Update FI to hold value of the index.
-bool DwarfDebug::findVariableFrameIndex(const DbgVariable *V, int *FI) {
-  assert (V && "Invalid DbgVariable!");
-  DenseMap<const DbgVariable *, int>::iterator I =
-    DbgVariableToFrameIndexMap.find(V);
-  if (I == DbgVariableToFrameIndexMap.end())
-    return false;
-  *FI = I->second;
-  return true;
-}
-
-/// findDbgScope - Find DbgScope for the debug loc.
-DbgScope *DwarfDebug::findDbgScope(DebugLoc DL) {
-  if (DL.isUnknown())
-    return NULL;
-
-  DbgScope *Scope = NULL;
-  LLVMContext &Ctx = Asm->MF->getFunction()->getContext();
-  if (MDNode *IA = DL.getInlinedAt(Ctx))
-    Scope = InlinedDbgScopeMap.lookup(DebugLoc::getFromDILocation(IA));
-  else
-    Scope = DbgScopeMap.lookup(DL.getScope(Ctx));
-  return Scope;
-}
-
-
 /// recordSourceLine - Register a source line with debug info. Returns the
 /// unique label that was emitted and which provides correspondence to
 /// the source line list.
@@ -2112,6 +1371,10 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S,
       DISubprogram SP(S);
       Fn = SP.getFilename();
       Dir = SP.getDirectory();
+    } else if (Scope.isLexicalBlockFile()) {
+      DILexicalBlockFile DBF(S);
+      Fn = DBF.getFilename();
+      Dir = DBF.getDirectory();
     } else if (Scope.isLexicalBlock()) {
       DILexicalBlock DB(S);
       Fn = DB.getFilename();
@@ -2121,8 +1384,7 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S,
 
     Src = GetOrCreateSourceID(Fn, Dir);
   }
-  Asm->OutStreamer.EmitDwarfLocDirective(Src, Line, Col, Flags,
-                                         0, 0, Fn);
+  Asm->OutStreamer.EmitDwarfLocDirective(Src, Line, Col, Flags, 0, 0, Fn);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2235,7 +1497,7 @@ void DwarfDebug::EmitSectionLabels() {
   EmitSectionSym(Asm, TLOF.getDataSection());
 }
 
-/// emitDIE - Recusively Emits a debug information entry.
+/// emitDIE - Recursively emits a debug information entry.
 ///
 void DwarfDebug::emitDIE(DIE *Die) {
   // Get the abbreviation for this DIE.
@@ -2290,10 +1552,9 @@ void DwarfDebug::emitDIE(DIE *Die) {
       break;
     }
     case dwarf::DW_AT_location: {
-      if (UseDotDebugLocEntry.count(Die) != 0) {
-        DIELabel *L = cast<DIELabel>(Values[i]);
+      if (DIELabel *L = dyn_cast<DIELabel>(Values[i]))
         Asm->EmitLabelDifference(L->getValue(), DwarfDebugLocSectionSym, 4);
-      } else
+      else
         Values[i]->EmitValue(Asm, Form);
       break;
     }
@@ -2464,7 +1725,7 @@ void DwarfDebug::emitDebugPubNames() {
     Asm->OutStreamer.AddComment("End Mark");
     Asm->EmitInt32(0);
     Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_end",
-                                                TheCU->getID()));
+                                                  TheCU->getID()));
   }
 }
 
@@ -2499,7 +1760,7 @@ void DwarfDebug::emitDebugPubTypes() {
     for (StringMap<DIE*>::const_iterator
            GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) {
       const char *Name = GI->getKeyData();
-      DIE * Entity = GI->second;
+      DIE *Entity = GI->second;
 
       if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DIE offset");
       Asm->EmitInt32(Entity->getOffset());
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index b2450064e3d0..35653be5c897 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -15,7 +15,8 @@
 #define CODEGEN_ASMPRINTER_DWARFDEBUG_H__
 
 #include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/CodeGen/MachineLocation.h"
+#include "llvm/CodeGen/LexicalScopes.h"
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/Analysis/DebugInfo.h"
 #include "DIE.h"
 #include "llvm/ADT/DenseMap.h"
@@ -30,7 +31,6 @@ namespace llvm {
 
 class CompileUnit;
 class DbgConcreteScope;
-class DbgScope;
 class DbgVariable;
 class MachineFrameInfo;
 class MachineModuleInfo;
@@ -125,9 +125,14 @@ class DbgVariable {
   DIVariable Var;                    // Variable Descriptor.
   DIE *TheDIE;                       // Variable DIE.
   unsigned DotDebugLocOffset;        // Offset in DotDebugLocEntries.
+  DbgVariable *AbsVar;               // Corresponding Abstract variable, if any.
+  const MachineInstr *MInsn;         // DBG_VALUE instruction of the variable.
+  int FrameIndex;
 public:
   // AbsVar may be NULL.
-  DbgVariable(DIVariable V) : Var(V), TheDIE(0), DotDebugLocOffset(~0U) {}
+  DbgVariable(DIVariable V, DbgVariable *AV) 
+    : Var(V), TheDIE(0), DotDebugLocOffset(~0U), AbsVar(AV), MInsn(0),
+      FrameIndex(~0) {}
 
   // Accessors.
   DIVariable getVariable()           const { return Var; }
@@ -136,7 +141,27 @@ class DbgVariable {
   void setDotDebugLocOffset(unsigned O)    { DotDebugLocOffset = O; }
   unsigned getDotDebugLocOffset()    const { return DotDebugLocOffset; }
   StringRef getName()                const { return Var.getName(); }
-  unsigned getTag()                  const { return Var.getTag(); }
+  DbgVariable *getAbstractVariable() const { return AbsVar; }
+  const MachineInstr *getMInsn()     const { return MInsn; }
+  void setMInsn(const MachineInstr *M)     { MInsn = M; }
+  int getFrameIndex()                const { return FrameIndex; }
+  void setFrameIndex(int FI)               { FrameIndex = FI; }
+  // Translate tag to proper Dwarf tag.  
+  unsigned getTag()                  const { 
+    if (Var.getTag() == dwarf::DW_TAG_arg_variable)
+      return dwarf::DW_TAG_formal_parameter;
+    
+    return dwarf::DW_TAG_variable;
+  }
+  /// isArtificial - Return true if DbgVariable is artificial.
+  bool isArtificial()                const {
+    if (Var.isArtificial())
+      return true;
+    if (Var.getTag() == dwarf::DW_TAG_arg_variable
+        && getType().isArtificial())
+      return true;
+    return false;
+  }
   bool variableHasComplexAddress()   const {
     assert(Var.Verify() && "Invalid complex DbgVariable!");
     return Var.hasComplexAddress();
@@ -167,8 +192,13 @@ class DwarfDebug {
   //
 
   CompileUnit *FirstCU;
+
+  /// Maps MDNode with its corresponding CompileUnit.
   DenseMap <const MDNode *, CompileUnit *> CUMap;
 
+  /// Maps subprogram MDNode with its corresponding CompileUnit.
+  DenseMap <const MDNode *, CompileUnit *> SPMap;
+
   /// AbbreviationsSet - Used to uniquely define abbreviations.
   ///
   FoldingSet<DIEAbbrev> AbbreviationsSet;
@@ -192,63 +222,27 @@ class DwarfDebug {
   ///
   UniqueVector<const MCSection*> SectionMap;
 
-  /// CurrentFnDbgScope - Top level scope for the current function.
-  ///
-  DbgScope *CurrentFnDbgScope;
-  
   /// CurrentFnArguments - List of Arguments (DbgValues) for current function.
   SmallVector<DbgVariable *, 8> CurrentFnArguments;
 
-  /// DbgScopeMap - Tracks the scopes in the current function.  Owns the
-  /// contained DbgScope*s.
-  DenseMap<const MDNode *, DbgScope *> DbgScopeMap;
-
-  /// InlinedDbgScopeMap - Tracks inlined function scopes in current function.
-  DenseMap<DebugLoc, DbgScope *> InlinedDbgScopeMap;
-
-  /// AbstractScopes - Tracks the abstract scopes a module. These scopes are
-  /// not included DbgScopeMap.  AbstractScopes owns its DbgScope*s.
-  DenseMap<const MDNode *, DbgScope *> AbstractScopes;
+  LexicalScopes LScopes;
 
   /// AbstractSPDies - Collection of abstract subprogram DIEs.
   DenseMap<const MDNode *, DIE *> AbstractSPDies;
 
-  /// AbstractScopesList - Tracks abstract scopes constructed while processing
-  /// a function. This list is cleared during endFunction().
-  SmallVector<DbgScope *, 4>AbstractScopesList;
+  /// ScopeVariables - Collection of dbg variables of a scope.
+  DenseMap<LexicalScope *, SmallVector<DbgVariable *, 8> > ScopeVariables;
 
-  /// AbstractVariables - Collection on abstract variables.  Owned by the
-  /// DbgScopes in AbstractScopes.
+  /// AbstractVariables - Collection on abstract variables.
   DenseMap<const MDNode *, DbgVariable *> AbstractVariables;
 
-  /// DbgVariableToFrameIndexMap - Tracks frame index used to find 
-  /// variable's value.
-  DenseMap<const DbgVariable *, int> DbgVariableToFrameIndexMap;
-
-  /// DbgVariableToDbgInstMap - Maps DbgVariable to corresponding DBG_VALUE
-  /// machine instruction.
-  DenseMap<const DbgVariable *, const MachineInstr *> DbgVariableToDbgInstMap;
-
   /// DotDebugLocEntries - Collection of DotDebugLocEntry.
   SmallVector<DotDebugLocEntry, 4> DotDebugLocEntries;
 
-  /// UseDotDebugLocEntry - DW_AT_location attributes for the DIEs in this set
-  /// idetifies corresponding .debug_loc entry offset.
-  SmallPtrSet<const DIE *, 4> UseDotDebugLocEntry;
-
-  /// VarToAbstractVarMap - Maps DbgVariable with corresponding Abstract
-  /// DbgVariable, if any.
-  DenseMap<const DbgVariable *, const DbgVariable *> VarToAbstractVarMap;
-
   /// InliendSubprogramDIEs - Collection of subprgram DIEs that are marked
   /// (at the end of the module) as DW_AT_inline.
   SmallPtrSet<DIE *, 4> InlinedSubprogramDIEs;
 
-  /// ContainingTypeMap - This map is used to keep track of subprogram DIEs that
-  /// need DW_AT_containing_type attribute. This attribute points to a DIE that
-  /// corresponds to the MDNode mapped with the subprogram DIE.
-  DenseMap<DIE *, const MDNode *> ContainingTypeMap;
-
   /// InlineInfo - Keep track of inlined functions and their location.  This
   /// information is used to populate debug_inlined section.
   typedef std::pair<const MCSymbol *, DIE *> InlineInfoLabels;
@@ -316,10 +310,7 @@ class DwarfDebug {
   ///
   void assignAbbrevNumber(DIEAbbrev &Abbrev);
 
-  /// getOrCreateDbgScope - Create DbgScope for the scope.
-  DbgScope *getOrCreateDbgScope(DebugLoc DL);
-
-  DbgScope *getOrCreateAbstractScope(const MDNode *N);
+  void addScopeVariable(LexicalScope *LS, DbgVariable *Var);
 
   /// findAbstractVariable - Find abstract variable associated with Var.
   DbgVariable *findAbstractVariable(DIVariable &Var, DebugLoc Loc);
@@ -328,22 +319,22 @@ class DwarfDebug {
   /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes.
   /// If there are global variables in this scope then create and insert
   /// DIEs for these variables.
-  DIE *updateSubprogramScopeDIE(const MDNode *SPNode);
+  DIE *updateSubprogramScopeDIE(CompileUnit *SPCU, const MDNode *SPNode);
 
   /// constructLexicalScope - Construct new DW_TAG_lexical_block 
   /// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *constructLexicalScopeDIE(DbgScope *Scope);
+  DIE *constructLexicalScopeDIE(CompileUnit *TheCU, LexicalScope *Scope);
 
   /// constructInlinedScopeDIE - This scope represents inlined body of
   /// a function. Construct DIE to represent this concrete inlined copy
   /// of the function.
-  DIE *constructInlinedScopeDIE(DbgScope *Scope);
+  DIE *constructInlinedScopeDIE(CompileUnit *TheCU, LexicalScope *Scope);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
-  DIE *constructVariableDIE(DbgVariable *DV, DbgScope *S);
+  DIE *constructVariableDIE(DbgVariable *DV, LexicalScope *S);
 
   /// constructScopeDIE - Construct a DIE for this scope.
-  DIE *constructScopeDIE(DbgScope *Scope);
+  DIE *constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope);
 
   /// EmitSectionLabels - Emit initial Dwarf sections with a label at
   /// the start of each one.
@@ -424,16 +415,10 @@ class DwarfDebug {
 
   /// constructCompileUnit - Create new CompileUnit for the given 
   /// metadata node with tag DW_TAG_compile_unit.
-  void constructCompileUnit(const MDNode *N);
-
-  /// getCompielUnit - Get CompileUnit DIE.
-  CompileUnit *getCompileUnit(const MDNode *N) const;
-
-  /// constructGlobalVariableDIE - Construct global variable DIE.
-  void constructGlobalVariableDIE(const MDNode *N);
+  CompileUnit *constructCompileUnit(const MDNode *N);
 
   /// construct SubprogramDIE - Construct subprogram DIE.
-  void constructSubprogramDIE(const MDNode *N);
+  void constructSubprogramDIE(CompileUnit *TheCU, const MDNode *N);
 
   /// recordSourceLine - Register a source line with debug info. Returns the
   /// unique label that was emitted and which provides correspondence to
@@ -441,30 +426,16 @@ class DwarfDebug {
   void recordSourceLine(unsigned Line, unsigned Col, const MDNode *Scope,
                         unsigned Flags);
   
-  /// recordVariableFrameIndex - Record a variable's index.
-  void recordVariableFrameIndex(const DbgVariable *V, int Index);
-
-  /// findVariableFrameIndex - Return true if frame index for the variable
-  /// is found. Update FI to hold value of the index.
-  bool findVariableFrameIndex(const DbgVariable *V, int *FI);
-
-  /// findDbgScope - Find DbgScope for the debug loc.
-  DbgScope *findDbgScope(DebugLoc DL);
-
   /// identifyScopeMarkers() - Indentify instructions that are marking
   /// beginning of or end of a scope.
   void identifyScopeMarkers();
 
-  /// extractScopeInformation - Scan machine instructions in this function
-  /// and collect DbgScopes. Return true, if atleast one scope was found.
-  bool extractScopeInformation();
-  
   /// addCurrentFnArgument - If Var is an current function argument that add
   /// it in CurrentFnArguments list.
   bool addCurrentFnArgument(const MachineFunction *MF,
-                            DbgVariable *Var, DbgScope *Scope);
+                            DbgVariable *Var, LexicalScope *Scope);
 
-  /// collectVariableInfo - Populate DbgScope entries with variables' info.
+  /// collectVariableInfo - Populate LexicalScope entries with variables' info.
   void collectVariableInfo(const MachineFunction *,
                            SmallPtrSet<const MDNode *, 16> &ProcessedVars);
   
@@ -496,6 +467,14 @@ class DwarfDebug {
   DwarfDebug(AsmPrinter *A, Module *M);
   ~DwarfDebug();
 
+  /// collectInfoFromNamedMDNodes - Collect debug info from named mdnodes such
+  /// as llvm.dbg.enum and llvm.dbg.ty
+  void collectInfoFromNamedMDNodes(Module *M);
+
+  /// collectLegacyDebugInfo - Collect debug info using DebugInfoFinder.
+  /// FIXME - Remove this when dragon-egg and llvm-gcc switch to DIBuilder.
+  bool collectLegacyDebugInfo(Module *M);
+
   /// beginModule - Emit all Dwarf sections that should come prior to the
   /// content.
   void beginModule(Module *M);
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp
index 1f992faaadb5..18b726b173dc 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp
@@ -17,7 +17,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -527,29 +526,26 @@ void DwarfException::EmitExceptionTable() {
          I = CallSites.begin(), E = CallSites.end(); I != E; ++I, ++idx) {
       const CallSiteEntry &S = *I;
 
+      // Offset of the landing pad, counted in 16-byte bundles relative to the
+      // @LPStart address.
       if (VerboseAsm) {
-        // Emit comments that decode the call site.
         Asm->OutStreamer.AddComment(Twine(">> Call Site ") +
                                     llvm::utostr(idx) + " <<");
         Asm->OutStreamer.AddComment(Twine("  On exception at call site ") +
                                     llvm::utostr(idx));
-
-        if (S.Action == 0)
-          Asm->OutStreamer.AddComment("  Action: cleanup");
-        else
-          Asm->OutStreamer.AddComment(Twine("  Action: ") +
-                                      llvm::utostr((S.Action - 1) / 2 + 1));
-
-        Asm->OutStreamer.AddBlankLine();
       }
-
-      // Offset of the landing pad, counted in 16-byte bundles relative to the
-      // @LPStart address.
       Asm->EmitULEB128(idx);
 
       // Offset of the first associated action record, relative to the start of
       // the action table. This value is biased by 1 (1 indicates the start of
       // the action table), and 0 indicates that there are no actions.
+      if (VerboseAsm) {
+        if (S.Action == 0)
+          Asm->OutStreamer.AddComment("  Action: cleanup");
+        else
+          Asm->OutStreamer.AddComment(Twine("  Action: ") +
+                                      llvm::utostr((S.Action - 1) / 2 + 1));
+      }
       Asm->EmitULEB128(S.Action);
     }
   } else {
@@ -595,46 +591,43 @@ void DwarfException::EmitExceptionTable() {
       if (EndLabel == 0)
         EndLabel = Asm->GetTempSymbol("eh_func_end", Asm->getFunctionNumber());
 
-      if (VerboseAsm) {
-        // Emit comments that decode the call site.
-        Asm->OutStreamer.AddComment(Twine(">> Call Site ") +
-                                    llvm::utostr(++Entry) + " <<");
-        Asm->OutStreamer.AddComment(Twine("  Call between ") +
-                                    BeginLabel->getName() + " and " +
-                                    EndLabel->getName());
-
-        if (!S.PadLabel) {
-          Asm->OutStreamer.AddComment("    has no landing pad");
-        } else {
-          Asm->OutStreamer.AddComment(Twine("    jumps to ") +
-                                      S.PadLabel->getName());
-
-          if (S.Action == 0)
-            Asm->OutStreamer.AddComment("  On action: cleanup");
-          else
-            Asm->OutStreamer.AddComment(Twine("  On action: ") +
-                                        llvm::utostr((S.Action - 1) / 2 + 1));
-        }
-
-        Asm->OutStreamer.AddBlankLine();
-      }
 
       // Offset of the call site relative to the previous call site, counted in
       // number of 16-byte bundles. The first call site is counted relative to
       // the start of the procedure fragment.
+      if (VerboseAsm)
+        Asm->OutStreamer.AddComment(Twine(">> Call Site ") +
+                                    llvm::utostr(++Entry) + " <<");
       Asm->EmitLabelDifference(BeginLabel, EHFuncBeginSym, 4);
+      if (VerboseAsm)
+        Asm->OutStreamer.AddComment(Twine("  Call between ") +
+                                    BeginLabel->getName() + " and " +
+                                    EndLabel->getName());
       Asm->EmitLabelDifference(EndLabel, BeginLabel, 4);
 
       // Offset of the landing pad, counted in 16-byte bundles relative to the
       // @LPStart address.
-      if (!S.PadLabel)
+      if (!S.PadLabel) {
+        if (VerboseAsm)
+          Asm->OutStreamer.AddComment("    has no landing pad");
         Asm->OutStreamer.EmitIntValue(0, 4/*size*/, 0/*addrspace*/);
-      else
+      } else {
+        if (VerboseAsm)
+          Asm->OutStreamer.AddComment(Twine("    jumps to ") +
+                                      S.PadLabel->getName());
         Asm->EmitLabelDifference(S.PadLabel, EHFuncBeginSym, 4);
+      }
 
       // Offset of the first associated action record, relative to the start of
       // the action table. This value is biased by 1 (1 indicates the start of
       // the action table), and 0 indicates that there are no actions.
+      if (VerboseAsm) {
+        if (S.Action == 0)
+          Asm->OutStreamer.AddComment("  On action: cleanup");
+        else
+          Asm->OutStreamer.AddComment(Twine("  On action: ") +
+                                      llvm::utostr((S.Action - 1) / 2 + 1));
+      }
       Asm->EmitULEB128(S.Action);
     }
   }
@@ -649,13 +642,29 @@ void DwarfException::EmitExceptionTable() {
       // Emit comments that decode the action table.
       Asm->OutStreamer.AddComment(Twine(">> Action Record ") +
                                   llvm::utostr(++Entry) + " <<");
-      if (Action.ValueForTypeID >= 0)
+    }
+
+    // Type Filter
+    //
+    //   Used by the runtime to match the type of the thrown exception to the
+    //   type of the catch clauses or the types in the exception specification.
+    if (VerboseAsm) {
+      if (Action.ValueForTypeID > 0)
         Asm->OutStreamer.AddComment(Twine("  Catch TypeInfo ") +
                                     llvm::itostr(Action.ValueForTypeID));
-      else 
+      else if (Action.ValueForTypeID < 0)
         Asm->OutStreamer.AddComment(Twine("  Filter TypeInfo ") +
                                     llvm::itostr(Action.ValueForTypeID));
+      else
+        Asm->OutStreamer.AddComment("  Cleanup");
+    }
+    Asm->EmitSLEB128(Action.ValueForTypeID);
 
+    // Action Record
+    //
+    //   Self-relative signed displacement in bytes of the next action record,
+    //   or 0 if there is no next action record.
+    if (VerboseAsm) {
       if (Action.NextAction == 0) {
         Asm->OutStreamer.AddComment("  No further actions");
       } else {
@@ -663,20 +672,7 @@ void DwarfException::EmitExceptionTable() {
         Asm->OutStreamer.AddComment(Twine("  Continue to action ") +
                                     llvm::utostr(NextAction));
       }
-
-      Asm->OutStreamer.AddBlankLine();
     }
-
-    // Type Filter
-    //
-    //   Used by the runtime to match the type of the thrown exception to the
-    //   type of the catch clauses or the types in the exception specification.
-    Asm->EmitSLEB128(Action.ValueForTypeID);
-
-    // Action Record
-    //
-    //   Self-relative signed displacement in bytes of the next action record,
-    //   or 0 if there is no next action record.
     Asm->EmitSLEB128(Action.NextAction);
   }
 
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp
index c2ad5eba3b3c..b83aa5ae3a1b 100644
--- a/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp
+++ b/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp
@@ -17,7 +17,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
diff --git a/contrib/llvm/lib/CodeGen/BranchFolding.cpp b/contrib/llvm/lib/CodeGen/BranchFolding.cpp
index 99090a8269d4..75288b0934cb 100644
--- a/contrib/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/contrib/llvm/lib/CodeGen/BranchFolding.cpp
@@ -1624,26 +1624,29 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
     if (!TIB->isSafeToMove(TII, 0, DontMoveAcrossStore))
       break;
 
+    // Remove kills from LocalDefsSet, these registers had short live ranges.
+    for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) {
+      MachineOperand &MO = TIB->getOperand(i);
+      if (!MO.isReg() || !MO.isUse() || !MO.isKill())
+        continue;
+      unsigned Reg = MO.getReg();
+      if (!Reg || !LocalDefsSet.count(Reg))
+        continue;
+      for (const unsigned *OR = TRI->getOverlaps(Reg); *OR; ++OR)
+        LocalDefsSet.erase(*OR);
+    }
+
     // Track local defs so we can update liveins.
     for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) {
       MachineOperand &MO = TIB->getOperand(i);
-      if (!MO.isReg())
+      if (!MO.isReg() || !MO.isDef() || MO.isDead())
         continue;
       unsigned Reg = MO.getReg();
       if (!Reg)
         continue;
-      if (MO.isDef()) {
-        if (!MO.isDead()) {
-          LocalDefs.push_back(Reg);
-          LocalDefsSet.insert(Reg);
-          for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR)
-            LocalDefsSet.insert(*SR);
-        }
-      } else if (MO.isKill() && LocalDefsSet.count(Reg)) {
-        LocalDefsSet.erase(Reg);
-        for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR)
-          LocalDefsSet.erase(*SR);
-      }
+      LocalDefs.push_back(Reg);
+      for (const unsigned *OR = TRI->getOverlaps(Reg); *OR; ++OR)
+        LocalDefsSet.insert(*OR);
     }
 
     HasDups = true;;
diff --git a/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp b/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp
index e6b3bbca2068..ea16a253a348 100644
--- a/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -185,35 +185,3 @@ void VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) {
 
   li.weight = normalizeSpillWeight(totalWeight, li.getSize());
 }
-
-void VirtRegAuxInfo::CalculateRegClass(unsigned reg) {
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  const TargetInstrInfo *TII = MF.getTarget().getInstrInfo();
-  const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo();
-  const TargetRegisterClass *OldRC = MRI.getRegClass(reg);
-  const TargetRegisterClass *NewRC = TRI->getLargestLegalSuperClass(OldRC);
-
-  // Stop early if there is no room to grow.
-  if (NewRC == OldRC)
-    return;
-
-  // Accumulate constraints from all uses.
-  for (MachineRegisterInfo::reg_nodbg_iterator I = MRI.reg_nodbg_begin(reg),
-       E = MRI.reg_nodbg_end(); I != E; ++I) {
-    // TRI doesn't have accurate enough information to model this yet.
-    if (I.getOperand().getSubReg())
-      return;
-    // Inline asm instuctions don't remember their constraints.
-    if (I->isInlineAsm())
-      return;
-    const TargetRegisterClass *OpRC =
-      TII->getRegClass(I->getDesc(), I.getOperandNo(), TRI);
-    if (OpRC)
-      NewRC = getCommonSubClass(NewRC, OpRC);
-    if (!NewRC || NewRC == OldRC)
-      return;
-  }
-  DEBUG(dbgs() << "Inflating " << OldRC->getName() << ':' << PrintReg(reg)
-               << " to " << NewRC->getName() <<".\n");
-  MRI.setRegClass(reg, NewRC);
-}
diff --git a/contrib/llvm/lib/CodeGen/CodeGen.cpp b/contrib/llvm/lib/CodeGen/CodeGen.cpp
index 489746cf3c72..424535ba2a1c 100644
--- a/contrib/llvm/lib/CodeGen/CodeGen.cpp
+++ b/contrib/llvm/lib/CodeGen/CodeGen.cpp
@@ -27,6 +27,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeLiveIntervalsPass(Registry);
   initializeLiveStacksPass(Registry);
   initializeLiveVariablesPass(Registry);
+  initializeMachineBlockFrequencyInfoPass(Registry);
   initializeMachineCSEPass(Registry);
   initializeMachineDominatorTreePass(Registry);
   initializeMachineLICMPass(Registry);
diff --git a/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp b/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp
index 03604b0a170f..ed9e409d3e5a 100644
--- a/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp
+++ b/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp
@@ -63,6 +63,8 @@ namespace {
     typedef SmallPtrSet<BasicBlock*, 8> BBSet;
     BBSet LandingPads;
 
+    bool InsertUnwindResumeCalls();
+
     bool NormalizeLandingPads();
     bool LowerUnwindsAndResumes();
     bool MoveExceptionValueCalls();
@@ -658,13 +660,76 @@ Instruction *DwarfEHPrepare::CreateExceptionValueCall(BasicBlock *BB) {
   return CallInst::Create(ExceptionValueIntrinsic, "eh.value.call", Start);
 }
 
+/// InsertUnwindResumeCalls - Convert the ResumeInsts that are still present
+/// into calls to the appropriate _Unwind_Resume function.
+bool DwarfEHPrepare::InsertUnwindResumeCalls() {
+  bool UsesNewEH = false;
+  SmallVector<ResumeInst*, 16> Resumes;
+  for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) {
+    TerminatorInst *TI = I->getTerminator();
+    if (ResumeInst *RI = dyn_cast<ResumeInst>(TI))
+      Resumes.push_back(RI);
+    else if (InvokeInst *II = dyn_cast<InvokeInst>(TI))
+      UsesNewEH = II->getUnwindDest()->isLandingPad();
+  }
+
+  if (Resumes.empty())
+    return UsesNewEH;
+
+  // Find the rewind function if we didn't already.
+  if (!RewindFunction) {
+    LLVMContext &Ctx = Resumes[0]->getContext();
+    FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
+                                          Type::getInt8PtrTy(Ctx), false);
+    const char *RewindName = TLI->getLibcallName(RTLIB::UNWIND_RESUME);
+    RewindFunction = F->getParent()->getOrInsertFunction(RewindName, FTy);
+  }
+
+  // Create the basic block where the _Unwind_Resume call will live.
+  LLVMContext &Ctx = F->getContext();
+  BasicBlock *UnwindBB = BasicBlock::Create(Ctx, "unwind_resume", F);
+  PHINode *PN = PHINode::Create(Type::getInt8PtrTy(Ctx), Resumes.size(),
+                                "exn.obj", UnwindBB);
+
+  // Extract the exception object from the ResumeInst and add it to the PHI node
+  // that feeds the _Unwind_Resume call.
+  BasicBlock *UnwindBBDom = Resumes[0]->getParent();
+  for (SmallVectorImpl<ResumeInst*>::iterator
+         I = Resumes.begin(), E = Resumes.end(); I != E; ++I) {
+    ResumeInst *RI = *I;
+    BranchInst::Create(UnwindBB, RI->getParent());
+    ExtractValueInst *ExnObj = ExtractValueInst::Create(RI->getOperand(0),
+                                                        0, "exn.obj", RI);
+    PN->addIncoming(ExnObj, RI->getParent());
+    UnwindBBDom = DT->findNearestCommonDominator(RI->getParent(), UnwindBBDom);
+    RI->eraseFromParent();
+  }
+
+  // Call the function.
+  CallInst *CI = CallInst::Create(RewindFunction, PN, "", UnwindBB);
+  CI->setCallingConv(TLI->getLibcallCallingConv(RTLIB::UNWIND_RESUME));
+
+  // We never expect _Unwind_Resume to return.
+  new UnreachableInst(Ctx, UnwindBB);
+
+  // Now update DominatorTree analysis information.
+  DT->addNewBlock(UnwindBB, UnwindBBDom);
+  return true;
+}
+
 bool DwarfEHPrepare::runOnFunction(Function &Fn) {
   bool Changed = false;
 
   // Initialize internal state.
-  DT = &getAnalysis<DominatorTree>();
+  DT = &getAnalysis<DominatorTree>(); // FIXME: We won't need this with the new EH.
   F = &Fn;
 
+  if (InsertUnwindResumeCalls()) {
+    // FIXME: The reset of this function can go once the new EH is done.
+    LandingPads.clear();
+    return true;
+  }
+
   // Ensure that only unwind edges end at landing pads (a landing pad is a
   // basic block where an invoke unwind edge ends).
   Changed |= NormalizeLandingPads();
diff --git a/contrib/llvm/lib/CodeGen/ELFCodeEmitter.cpp b/contrib/llvm/lib/CodeGen/ELFCodeEmitter.cpp
index 3fb087c5ea8b..660424c3c141 100644
--- a/contrib/llvm/lib/CodeGen/ELFCodeEmitter.cpp
+++ b/contrib/llvm/lib/CodeGen/ELFCodeEmitter.cpp
@@ -155,7 +155,7 @@ void ELFCodeEmitter::emitConstantPool(MachineConstantPool *MCP) {
     CPSections.push_back(CstPool.SectionIdx);
 
     if (CPE.isMachineConstantPoolEntry())
-      assert("CPE.isMachineConstantPoolEntry not supported yet");
+      assert(0 && "CPE.isMachineConstantPoolEntry not supported yet");
 
     // Emit the constant to constant pool section
     EW.EmitGlobalConstant(CPE.Val.ConstVal, CstPool);
diff --git a/contrib/llvm/lib/CodeGen/ELFCodeEmitter.h b/contrib/llvm/lib/CodeGen/ELFCodeEmitter.h
index 2ec1f6e873d6..8671c674eecf 100644
--- a/contrib/llvm/lib/CodeGen/ELFCodeEmitter.h
+++ b/contrib/llvm/lib/CodeGen/ELFCodeEmitter.h
@@ -58,13 +58,13 @@ namespace llvm {
 
     /// emitLabel - Emits a label
     virtual void emitLabel(MCSymbol *Label) {
-      assert("emitLabel not implemented");
+      assert(0 && "emitLabel not implemented");
     }
 
     /// getLabelAddress - Return the address of the specified LabelID, 
     /// only usable after the LabelID has been emitted.
     virtual uintptr_t getLabelAddress(MCSymbol *Label) const {
-      assert("getLabelAddress not implemented");
+      assert(0 && "getLabelAddress not implemented");
       return 0;
     }
 
diff --git a/contrib/llvm/lib/CodeGen/ELFWriter.cpp b/contrib/llvm/lib/CodeGen/ELFWriter.cpp
index d977651c32f7..f2c218565854 100644
--- a/contrib/llvm/lib/CodeGen/ELFWriter.cpp
+++ b/contrib/llvm/lib/CodeGen/ELFWriter.cpp
@@ -45,12 +45,12 @@
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetAsmInfo.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetELFWriterInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -65,7 +65,8 @@ char ELFWriter::ID = 0;
 
 ELFWriter::ELFWriter(raw_ostream &o, TargetMachine &tm)
   : MachineFunctionPass(ID), O(o), TM(tm),
-    OutContext(*new MCContext(*TM.getMCAsmInfo(), new TargetAsmInfo(tm))),
+    OutContext(*new MCContext(*TM.getMCAsmInfo(), *TM.getRegisterInfo(),
+                              &TM.getTargetLowering()->getObjFileLowering())),
     TLOF(TM.getTargetLowering()->getObjFileLowering()),
     is64Bit(TM.getTargetData()->getPointerSizeInBits() == 64),
     isLittleEndian(TM.getTargetData()->isLittleEndian()),
@@ -482,7 +483,7 @@ void ELFWriter::EmitGlobalConstant(const Constant *CV, ELFSection &GblS) {
       EmitGlobalConstantLargeInt(CI, GblS);
     return;
   } else if (const ConstantVector *CP = dyn_cast<ConstantVector>(CV)) {
-    const VectorType *PTy = CP->getType();
+    VectorType *PTy = CP->getType();
     for (unsigned I = 0, E = PTy->getNumElements(); I < E; ++I)
       EmitGlobalConstant(CP->getOperand(I), GblS);
     return;
@@ -540,8 +541,7 @@ CstExprResTy ELFWriter::ResolveConstantExpr(const Constant *CV) {
   case Instruction::GetElementPtr: {
     const Constant *ptrVal = CE->getOperand(0);
     SmallVector<Value*, 8> idxVec(CE->op_begin()+1, CE->op_end());
-    int64_t Offset = TD->getIndexedOffset(ptrVal->getType(), &idxVec[0],
-                                          idxVec.size());
+    int64_t Offset = TD->getIndexedOffset(ptrVal->getType(), idxVec);
     return std::make_pair(ptrVal, Offset);
   }
   case Instruction::IntToPtr: {
@@ -552,7 +552,7 @@ CstExprResTy ELFWriter::ResolveConstantExpr(const Constant *CV) {
   }
   case Instruction::PtrToInt: {
     Constant *Op = CE->getOperand(0);
-    const Type *Ty = CE->getType();
+    Type *Ty = CE->getType();
 
     // We can emit the pointer value into this slot if the slot is an
     // integer slot greater or equal to the size of the pointer.
diff --git a/contrib/llvm/lib/Target/X86/SSEDomainFix.cpp b/contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp
similarity index 84%
rename from contrib/llvm/lib/Target/X86/SSEDomainFix.cpp
rename to contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp
index 13680c592e01..01dccdb71e4b 100644
--- a/contrib/llvm/lib/Target/X86/SSEDomainFix.cpp
+++ b/contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp
@@ -1,4 +1,4 @@
-//===- SSEDomainFix.cpp - Use proper int/float domain for SSE ---*- C++ -*-===//
+//===- ExecutionDepsFix.cpp - Fix execution dependecy issues ----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,21 +7,25 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contains the SSEDomainFix pass.
+// This file contains the execution dependency fix pass.
 //
-// Some SSE instructions like mov, and, or, xor are available in different
+// Some X86 SSE instructions like mov, and, or, xor are available in different
 // variants for different operand types. These variant instructions are
 // equivalent, but on Nehalem and newer cpus there is extra latency
-// transferring data between integer and floating point domains.
+// transferring data between integer and floating point domains.  ARM cores
+// have similar issues when they are configured with both VFP and NEON
+// pipelines.
 //
 // This pass changes the variant instructions to minimize domain crossings.
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "sse-domain-fix"
-#include "X86InstrInfo.h"
+#define DEBUG_TYPE "execution-fix"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Debug.h"
@@ -97,25 +101,27 @@ struct DomainValue {
 };
 }
 
-static const unsigned NumRegs = 16;
-
 namespace {
-class SSEDomainFixPass : public MachineFunctionPass {
+class ExeDepsFix : public MachineFunctionPass {
   static char ID;
   SpecificBumpPtrAllocator<DomainValue> Allocator;
   SmallVector<DomainValue*,16> Avail;
 
+  const TargetRegisterClass *const RC;
   MachineFunction *MF;
-  const X86InstrInfo *TII;
+  const TargetInstrInfo *TII;
   const TargetRegisterInfo *TRI;
   MachineBasicBlock *MBB;
+  std::vector<int> AliasMap;
+  const unsigned NumRegs;
   DomainValue **LiveRegs;
   typedef DenseMap<MachineBasicBlock*,DomainValue**> LiveOutMap;
   LiveOutMap LiveOuts;
   unsigned Distance;
 
 public:
-  SSEDomainFixPass() : MachineFunctionPass(ID) {}
+  ExeDepsFix(const TargetRegisterClass *rc)
+    : MachineFunctionPass(ID), RC(rc), NumRegs(RC->getNumRegs()) {}
 
   virtual void getAnalysisUsage(AnalysisUsage &AU) const {
     AU.setPreservesAll();
@@ -150,17 +156,16 @@ class SSEDomainFixPass : public MachineFunctionPass {
 };
 }
 
-char SSEDomainFixPass::ID = 0;
+char ExeDepsFix::ID = 0;
 
 /// Translate TRI register number to an index into our smaller tables of
 /// interesting registers. Return -1 for boring registers.
-int SSEDomainFixPass::RegIndex(unsigned reg) {
-  assert(X86::XMM15 == X86::XMM0+NumRegs-1 && "Unexpected sort");
-  reg -= X86::XMM0;
-  return reg < NumRegs ? (int) reg : -1;
+int ExeDepsFix::RegIndex(unsigned Reg) {
+  assert(Reg < AliasMap.size() && "Invalid register");
+  return AliasMap[Reg];
 }
 
-DomainValue *SSEDomainFixPass::Alloc(int domain) {
+DomainValue *ExeDepsFix::Alloc(int domain) {
   DomainValue *dv = Avail.empty() ?
                       new(Allocator.Allocate()) DomainValue :
                       Avail.pop_back_val();
@@ -170,14 +175,14 @@ DomainValue *SSEDomainFixPass::Alloc(int domain) {
   return dv;
 }
 
-void SSEDomainFixPass::Recycle(DomainValue *dv) {
+void ExeDepsFix::Recycle(DomainValue *dv) {
   assert(dv && "Cannot recycle NULL");
   dv->clear();
   Avail.push_back(dv);
 }
 
 /// Set LiveRegs[rx] = dv, updating reference counts.
-void SSEDomainFixPass::SetLiveReg(int rx, DomainValue *dv) {
+void ExeDepsFix::SetLiveReg(int rx, DomainValue *dv) {
   assert(unsigned(rx) < NumRegs && "Invalid index");
   if (!LiveRegs) {
     LiveRegs = new DomainValue*[NumRegs];
@@ -195,7 +200,7 @@ void SSEDomainFixPass::SetLiveReg(int rx, DomainValue *dv) {
 }
 
 // Kill register rx, recycle or collapse any DomainValue.
-void SSEDomainFixPass::Kill(int rx) {
+void ExeDepsFix::Kill(int rx) {
   assert(unsigned(rx) < NumRegs && "Invalid index");
   if (!LiveRegs || !LiveRegs[rx]) return;
 
@@ -208,7 +213,7 @@ void SSEDomainFixPass::Kill(int rx) {
 }
 
 /// Force register rx into domain.
-void SSEDomainFixPass::Force(int rx, unsigned domain) {
+void ExeDepsFix::Force(int rx, unsigned domain) {
   assert(unsigned(rx) < NumRegs && "Invalid index");
   DomainValue *dv;
   if (LiveRegs && (dv = LiveRegs[rx])) {
@@ -217,8 +222,8 @@ void SSEDomainFixPass::Force(int rx, unsigned domain) {
     else if (dv->hasDomain(domain))
       Collapse(dv, domain);
     else {
-      // This is an incompatible open DomainValue. Collapse it to whatever and force
-      // the new value into domain. This costs a domain crossing.
+      // This is an incompatible open DomainValue. Collapse it to whatever and
+      // force the new value into domain. This costs a domain crossing.
       Collapse(dv, dv->getFirstDomain());
       assert(LiveRegs[rx] && "Not live after collapse?");
       LiveRegs[rx]->addDomain(domain);
@@ -231,12 +236,12 @@ void SSEDomainFixPass::Force(int rx, unsigned domain) {
 
 /// Collapse open DomainValue into given domain. If there are multiple
 /// registers using dv, they each get a unique collapsed DomainValue.
-void SSEDomainFixPass::Collapse(DomainValue *dv, unsigned domain) {
+void ExeDepsFix::Collapse(DomainValue *dv, unsigned domain) {
   assert(dv->hasDomain(domain) && "Cannot collapse");
 
   // Collapse all the instructions.
   while (!dv->Instrs.empty())
-    TII->SetSSEDomain(dv->Instrs.pop_back_val(), domain);
+    TII->setExecutionDomain(dv->Instrs.pop_back_val(), domain);
   dv->setSingleDomain(domain);
 
   // If there are multiple users, give them new, unique DomainValues.
@@ -248,7 +253,7 @@ void SSEDomainFixPass::Collapse(DomainValue *dv, unsigned domain) {
 
 /// Merge - All instructions and registers in B are moved to A, and B is
 /// released.
-bool SSEDomainFixPass::Merge(DomainValue *A, DomainValue *B) {
+bool ExeDepsFix::Merge(DomainValue *A, DomainValue *B) {
   assert(!A->isCollapsed() && "Cannot merge into collapsed");
   assert(!B->isCollapsed() && "Cannot merge from collapsed");
   if (A == B)
@@ -266,7 +271,7 @@ bool SSEDomainFixPass::Merge(DomainValue *A, DomainValue *B) {
   return true;
 }
 
-void SSEDomainFixPass::enterBasicBlock() {
+void ExeDepsFix::enterBasicBlock() {
   // Try to coalesce live-out registers from predecessors.
   for (MachineBasicBlock::livein_iterator i = MBB->livein_begin(),
          e = MBB->livein_end(); i != e; ++i) {
@@ -303,7 +308,7 @@ void SSEDomainFixPass::enterBasicBlock() {
 
 // A hard instruction only works in one domain. All input registers will be
 // forced into that domain.
-void SSEDomainFixPass::visitHardInstr(MachineInstr *mi, unsigned domain) {
+void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) {
   // Collapse all uses.
   for (unsigned i = mi->getDesc().getNumDefs(),
                 e = mi->getDesc().getNumOperands(); i != e; ++i) {
@@ -326,7 +331,7 @@ void SSEDomainFixPass::visitHardInstr(MachineInstr *mi, unsigned domain) {
 }
 
 // A soft instruction can be changed to work in other domains given by mask.
-void SSEDomainFixPass::visitSoftInstr(MachineInstr *mi, unsigned mask) {
+void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
   // Bitmask of available domains for this instruction after taking collapsed
   // operands into account.
   unsigned available = mask;
@@ -362,7 +367,7 @@ void SSEDomainFixPass::visitSoftInstr(MachineInstr *mi, unsigned mask) {
   // If the collapsed operands force a single domain, propagate the collapse.
   if (isPowerOf2_32(available)) {
     unsigned domain = CountTrailingZeros_32(available);
-    TII->SetSSEDomain(mi, domain);
+    TII->setExecutionDomain(mi, domain);
     visitHardInstr(mi, domain);
     return;
   }
@@ -431,8 +436,8 @@ void SSEDomainFixPass::visitSoftInstr(MachineInstr *mi, unsigned mask) {
   }
 }
 
-void SSEDomainFixPass::visitGenericInstr(MachineInstr *mi) {
-  // Process explicit defs, kill any XMM registers redefined.
+void ExeDepsFix::visitGenericInstr(MachineInstr *mi) {
+  // Process explicit defs, kill any relevant registers redefined.
   for (unsigned i = 0, e = mi->getDesc().getNumDefs(); i != e; ++i) {
     MachineOperand &mo = mi->getOperand(i);
     if (!mo.isReg()) continue;
@@ -442,25 +447,36 @@ void SSEDomainFixPass::visitGenericInstr(MachineInstr *mi) {
   }
 }
 
-bool SSEDomainFixPass::runOnMachineFunction(MachineFunction &mf) {
+bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) {
   MF = &mf;
-  TII = static_cast<const X86InstrInfo*>(MF->getTarget().getInstrInfo());
+  TII = MF->getTarget().getInstrInfo();
   TRI = MF->getTarget().getRegisterInfo();
   MBB = 0;
   LiveRegs = 0;
   Distance = 0;
-  assert(NumRegs == X86::VR128RegClass.getNumRegs() && "Bad regclass");
+  assert(NumRegs == RC->getNumRegs() && "Bad regclass");
 
-  // If no XMM registers are used in the function, we can skip it completely.
+  // If no relevant registers are used in the function, we can skip it
+  // completely.
   bool anyregs = false;
-  for (TargetRegisterClass::const_iterator I = X86::VR128RegClass.begin(),
-         E = X86::VR128RegClass.end(); I != E; ++I)
+  for (TargetRegisterClass::const_iterator I = RC->begin(), E = RC->end();
+       I != E; ++I)
     if (MF->getRegInfo().isPhysRegUsed(*I)) {
       anyregs = true;
       break;
     }
   if (!anyregs) return false;
 
+  // Initialize the AliasMap on the first use.
+  if (AliasMap.empty()) {
+    // Given a PhysReg, AliasMap[PhysReg] is either the relevant index into RC,
+    // or -1.
+    AliasMap.resize(TRI->getNumRegs(), -1);
+    for (unsigned i = 0, e = RC->getNumRegs(); i != e; ++i)
+      for (const unsigned *AI = TRI->getOverlaps(RC->getRegister(i)); *AI; ++AI)
+        AliasMap[*AI] = i;
+  }
+
   MachineBasicBlock *Entry = MF->begin();
   SmallPtrSet<MachineBasicBlock*, 16> Visited;
   for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*, 16> >
@@ -473,7 +489,7 @@ bool SSEDomainFixPass::runOnMachineFunction(MachineFunction &mf) {
       MachineInstr *mi = I;
       if (mi->isDebugValue()) continue;
       ++Distance;
-      std::pair<uint16_t, uint16_t> domp = TII->GetSSEDomain(mi);
+      std::pair<uint16_t, uint16_t> domp = TII->getExecutionDomain(mi);
       if (domp.first)
         if (domp.second)
           visitSoftInstr(mi, domp.second);
@@ -501,6 +517,7 @@ bool SSEDomainFixPass::runOnMachineFunction(MachineFunction &mf) {
   return false;
 }
 
-FunctionPass *llvm::createSSEDomainFixPass() {
-  return new SSEDomainFixPass();
+FunctionPass *
+llvm::createExecutionDependencyFixPass(const TargetRegisterClass *RC) {
+  return new ExeDepsFix(RC);
 }
diff --git a/contrib/llvm/lib/CodeGen/LowerSubregs.cpp b/contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp
similarity index 69%
rename from contrib/llvm/lib/CodeGen/LowerSubregs.cpp
rename to contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp
index 7871ba9c17e4..e2a14a8dfd97 100644
--- a/contrib/llvm/lib/CodeGen/LowerSubregs.cpp
+++ b/contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp
@@ -1,4 +1,4 @@
-//===-- LowerSubregs.cpp - Subregister Lowering instruction pass ----------===//
+//===-- ExpandPostRAPseudos.cpp - Pseudo instruction expansion pass -------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,14 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines a MachineFunction pass which runs after register
-// allocation that turns subreg insert/extract instructions into register
-// copies, as needed. This ensures correct codegen even if the coalescer
-// isn't able to remove all subreg instructions.
+// This file defines a pass that expands COPY and SUBREG_TO_REG pseudo
+// instructions after register allocation.
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "lowersubregs"
+#define DEBUG_TYPE "postrapseudos"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/Function.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -29,52 +27,51 @@
 using namespace llvm;
 
 namespace {
-  struct LowerSubregsInstructionPass : public MachineFunctionPass {
-  private:
-    const TargetRegisterInfo *TRI;
-    const TargetInstrInfo *TII;
+struct ExpandPostRA : public MachineFunctionPass {
+private:
+  const TargetRegisterInfo *TRI;
+  const TargetInstrInfo *TII;
 
-  public:
-    static char ID; // Pass identification, replacement for typeid
-    LowerSubregsInstructionPass() : MachineFunctionPass(ID) {}
+public:
+  static char ID; // Pass identification, replacement for typeid
+  ExpandPostRA() : MachineFunctionPass(ID) {}
 
-    const char *getPassName() const {
-      return "Subregister lowering instruction pass";
-    }
+  const char *getPassName() const {
+    return "Post-RA pseudo instruction expansion pass";
+  }
 
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
-      AU.setPreservesCFG();
-      AU.addPreservedID(MachineLoopInfoID);
-      AU.addPreservedID(MachineDominatorsID);
-      MachineFunctionPass::getAnalysisUsage(AU);
-    }
+  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+    AU.setPreservesCFG();
+    AU.addPreservedID(MachineLoopInfoID);
+    AU.addPreservedID(MachineDominatorsID);
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
 
-    /// runOnMachineFunction - pass entry point
-    bool runOnMachineFunction(MachineFunction&);
+  /// runOnMachineFunction - pass entry point
+  bool runOnMachineFunction(MachineFunction&);
 
-  private:
-    bool LowerSubregToReg(MachineInstr *MI);
-    bool LowerCopy(MachineInstr *MI);
+private:
+  bool LowerSubregToReg(MachineInstr *MI);
+  bool LowerCopy(MachineInstr *MI);
 
-    void TransferDeadFlag(MachineInstr *MI, unsigned DstReg,
-                          const TargetRegisterInfo *TRI);
-    void TransferImplicitDefs(MachineInstr *MI);
-  };
+  void TransferDeadFlag(MachineInstr *MI, unsigned DstReg,
+                        const TargetRegisterInfo *TRI);
+  void TransferImplicitDefs(MachineInstr *MI);
+};
+} // end anonymous namespace
 
-  char LowerSubregsInstructionPass::ID = 0;
-}
+char ExpandPostRA::ID = 0;
 
-FunctionPass *llvm::createLowerSubregsPass() {
-  return new LowerSubregsInstructionPass();
+FunctionPass *llvm::createExpandPostRAPseudosPass() {
+  return new ExpandPostRA();
 }
 
 /// TransferDeadFlag - MI is a pseudo-instruction with DstReg dead,
 /// and the lowered replacement instructions immediately precede it.
 /// Mark the replacement instructions with the dead flag.
 void
-LowerSubregsInstructionPass::TransferDeadFlag(MachineInstr *MI,
-                                              unsigned DstReg,
-                                              const TargetRegisterInfo *TRI) {
+ExpandPostRA::TransferDeadFlag(MachineInstr *MI, unsigned DstReg,
+                               const TargetRegisterInfo *TRI) {
   for (MachineBasicBlock::iterator MII =
         prior(MachineBasicBlock::iterator(MI)); ; --MII) {
     if (MII->addRegisterDead(DstReg, TRI))
@@ -88,7 +85,7 @@ LowerSubregsInstructionPass::TransferDeadFlag(MachineInstr *MI,
 /// replacement instructions immediately precede it.  Copy any implicit-def
 /// operands from MI to the replacement instruction.
 void
-LowerSubregsInstructionPass::TransferImplicitDefs(MachineInstr *MI) {
+ExpandPostRA::TransferImplicitDefs(MachineInstr *MI) {
   MachineBasicBlock::iterator CopyMI = MI;
   --CopyMI;
 
@@ -100,7 +97,7 @@ LowerSubregsInstructionPass::TransferImplicitDefs(MachineInstr *MI) {
   }
 }
 
-bool LowerSubregsInstructionPass::LowerSubregToReg(MachineInstr *MI) {
+bool ExpandPostRA::LowerSubregToReg(MachineInstr *MI) {
   MachineBasicBlock *MBB = MI->getParent();
   assert((MI->getOperand(0).isReg() && MI->getOperand(0).isDef()) &&
          MI->getOperand(1).isImm() &&
@@ -152,7 +149,7 @@ bool LowerSubregsInstructionPass::LowerSubregToReg(MachineInstr *MI) {
   return true;
 }
 
-bool LowerSubregsInstructionPass::LowerCopy(MachineInstr *MI) {
+bool ExpandPostRA::LowerCopy(MachineInstr *MI) {
   MachineOperand &DstMO = MI->getOperand(0);
   MachineOperand &SrcMO = MI->getOperand(1);
 
@@ -191,9 +188,9 @@ bool LowerSubregsInstructionPass::LowerCopy(MachineInstr *MI) {
 /// runOnMachineFunction - Reduce subregister inserts and extracts to register
 /// copies.
 ///
-bool LowerSubregsInstructionPass::runOnMachineFunction(MachineFunction &MF) {
+bool ExpandPostRA::runOnMachineFunction(MachineFunction &MF) {
   DEBUG(dbgs() << "Machine Function\n"
-               << "********** LOWERING SUBREG INSTRS **********\n"
+               << "********** EXPANDING POST-RA PSEUDO INSTRS **********\n"
                << "********** Function: "
                << MF.getFunction()->getName() << '\n');
   TRI = MF.getTarget().getRegisterInfo();
@@ -205,17 +202,34 @@ bool LowerSubregsInstructionPass::runOnMachineFunction(MachineFunction &MF) {
        mbbi != mbbe; ++mbbi) {
     for (MachineBasicBlock::iterator mi = mbbi->begin(), me = mbbi->end();
          mi != me;) {
-      MachineBasicBlock::iterator nmi = llvm::next(mi);
       MachineInstr *MI = mi;
-      assert(!MI->isInsertSubreg() && "INSERT_SUBREG should no longer appear");
-      assert(MI->getOpcode() != TargetOpcode::EXTRACT_SUBREG &&
-             "EXTRACT_SUBREG should no longer appear");
-      if (MI->isSubregToReg()) {
-        MadeChange |= LowerSubregToReg(MI);
-      } else if (MI->isCopy()) {
-        MadeChange |= LowerCopy(MI);
+      // Advance iterator here because MI may be erased.
+      ++mi;
+
+      // Only expand pseudos.
+      if (!MI->getDesc().isPseudo())
+        continue;
+
+      // Give targets a chance to expand even standard pseudos.
+      if (TII->expandPostRAPseudo(MI)) {
+        MadeChange = true;
+        continue;
+      }
+
+      // Expand standard pseudos.
+      switch (MI->getOpcode()) {
+      case TargetOpcode::SUBREG_TO_REG:
+        MadeChange |= LowerSubregToReg(MI);
+        break;
+      case TargetOpcode::COPY:
+        MadeChange |= LowerCopy(MI);
+        break;
+      case TargetOpcode::DBG_VALUE:
+        continue;
+      case TargetOpcode::INSERT_SUBREG:
+      case TargetOpcode::EXTRACT_SUBREG:
+        llvm_unreachable("Sub-register pseudos should have been eliminated.");
       }
-      mi = nmi;
     }
   }
 
diff --git a/contrib/llvm/lib/CodeGen/IfConversion.cpp b/contrib/llvm/lib/CodeGen/IfConversion.cpp
index 6cb22778caf9..ce7ed293daac 100644
--- a/contrib/llvm/lib/CodeGen/IfConversion.cpp
+++ b/contrib/llvm/lib/CodeGen/IfConversion.cpp
@@ -16,14 +16,13 @@
 #include "llvm/Function.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -154,7 +153,8 @@ namespace {
     const TargetInstrInfo *TII;
     const TargetRegisterInfo *TRI;
     const InstrItineraryData *InstrItins;
-    const MachineLoopInfo *MLI;
+    const MachineBranchProbabilityInfo *MBPI;
+
     bool MadeChange;
     int FnNum;
   public:
@@ -162,9 +162,9 @@ namespace {
     IfConverter() : MachineFunctionPass(ID), FnNum(-1) {
       initializeIfConverterPass(*PassRegistry::getPassRegistry());
     }
-    
+
     virtual void getAnalysisUsage(AnalysisUsage &AU) const {
-      AU.addRequired<MachineLoopInfo>();
+      AU.addRequired<MachineBranchProbabilityInfo>();
       MachineFunctionPass::getAnalysisUsage(AU);
     }
 
@@ -252,7 +252,7 @@ namespace {
 }
 
 INITIALIZE_PASS_BEGIN(IfConverter, "if-converter", "If Converter", false, false)
-INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
 INITIALIZE_PASS_END(IfConverter, "if-converter", "If Converter", false, false)
 
 FunctionPass *llvm::createIfConverterPass() { return new IfConverter(); }
@@ -261,7 +261,7 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
   TLI = MF.getTarget().getTargetLowering();
   TII = MF.getTarget().getInstrInfo();
   TRI = MF.getTarget().getRegisterInfo();
-  MLI = &getAnalysis<MachineLoopInfo>();
+  MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
   InstrItins = MF.getTarget().getInstrItineraryData();
   if (!TII) return false;
 
@@ -790,28 +790,9 @@ IfConverter::BBInfo &IfConverter::AnalyzeBlock(MachineBasicBlock *BB,
   bool TNeedSub = TrueBBI.Predicate.size() > 0;
   bool FNeedSub = FalseBBI.Predicate.size() > 0;
   bool Enqueued = false;
-  
-  // Try to predict the branch, using loop info to guide us.
-  // General heuristics are:
-  //   - backedge -> 90% taken
-  //   - early exit -> 20% taken
-  //   - branch predictor confidence -> 90%
-  BranchProbability Prediction(5, 10);
-  MachineLoop *Loop = MLI->getLoopFor(BB);
-  if (Loop) {
-    if (TrueBBI.BB == Loop->getHeader())
-      Prediction = BranchProbability(9, 10);
-    else if (FalseBBI.BB == Loop->getHeader())
-      Prediction = BranchProbability(1, 10);
 
-    MachineLoop *TrueLoop = MLI->getLoopFor(TrueBBI.BB);
-    MachineLoop *FalseLoop = MLI->getLoopFor(FalseBBI.BB);
-    if (!TrueLoop || TrueLoop->getParentLoop() == Loop)
-      Prediction = BranchProbability(2, 10);
-    else if (!FalseLoop || FalseLoop->getParentLoop() == Loop)
-      Prediction = BranchProbability(8, 10);
-  }
-  
+  BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB);
+
   if (CanRevCond && ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2) &&
       MeetIfcvtSizeLimit(*TrueBBI.BB, (TrueBBI.NonPredSize - (Dups + Dups2) +
                                        TrueBBI.ExtraCost), TrueBBI.ExtraCost2,
diff --git a/contrib/llvm/lib/CodeGen/InlineSpiller.cpp b/contrib/llvm/lib/CodeGen/InlineSpiller.cpp
index 5547f735ba5e..726af4696578 100644
--- a/contrib/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/contrib/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -17,6 +17,7 @@
 #include "LiveRangeEdit.h"
 #include "VirtRegMap.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
 #include "llvm/CodeGen/LiveStackAnalysis.h"
@@ -27,22 +28,26 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
 STATISTIC(NumSpilledRanges,   "Number of spilled live ranges");
-STATISTIC(NumSnippets,        "Number of snippets included in spills");
+STATISTIC(NumSnippets,        "Number of spilled snippets");
 STATISTIC(NumSpills,          "Number of spills inserted");
+STATISTIC(NumSpillsRemoved,   "Number of spills removed");
 STATISTIC(NumReloads,         "Number of reloads inserted");
+STATISTIC(NumReloadsRemoved,  "Number of reloads removed");
 STATISTIC(NumFolded,          "Number of folded stack accesses");
 STATISTIC(NumFoldedLoads,     "Number of folded loads");
 STATISTIC(NumRemats,          "Number of rematerialized defs for spilling");
-STATISTIC(NumOmitReloadSpill, "Number of omitted spills after reloads");
-STATISTIC(NumHoistLocal,      "Number of locally hoisted spills");
-STATISTIC(NumHoistGlobal,     "Number of globally hoisted spills");
-STATISTIC(NumRedundantSpills, "Number of redundant spills identified");
+STATISTIC(NumOmitReloadSpill, "Number of omitted spills of reloads");
+STATISTIC(NumHoists,          "Number of hoisted spills");
+
+static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden,
+                                     cl::desc("Disable inline spill hoisting"));
 
 namespace {
 class InlineSpiller : public Spiller {
@@ -75,26 +80,49 @@ class InlineSpiller : public Spiller {
   // Values that failed to remat at some point.
   SmallPtrSet<VNInfo*, 8> UsedValues;
 
+public:
   // Information about a value that was defined by a copy from a sibling
   // register.
   struct SibValueInfo {
     // True when all reaching defs were reloads: No spill is necessary.
     bool AllDefsAreReloads;
 
+    // True when value is defined by an original PHI not from splitting.
+    bool DefByOrigPHI;
+
+    // True when the COPY defining this value killed its source.
+    bool KillsSource;
+
     // The preferred register to spill.
     unsigned SpillReg;
 
     // The value of SpillReg that should be spilled.
     VNInfo *SpillVNI;
 
+    // The block where SpillVNI should be spilled. Currently, this must be the
+    // block containing SpillVNI->def.
+    MachineBasicBlock *SpillMBB;
+
     // A defining instruction that is not a sibling copy or a reload, or NULL.
     // This can be used as a template for rematerialization.
     MachineInstr *DefMI;
 
+    // List of values that depend on this one.  These values are actually the
+    // same, but live range splitting has placed them in different registers,
+    // or SSA update needed to insert PHI-defs to preserve SSA form.  This is
+    // copies of the current value and phi-kills.  Usually only phi-kills cause
+    // more than one dependent value.
+    TinyPtrVector<VNInfo*> Deps;
+
     SibValueInfo(unsigned Reg, VNInfo *VNI)
-      : AllDefsAreReloads(false), SpillReg(Reg), SpillVNI(VNI), DefMI(0) {}
+      : AllDefsAreReloads(true), DefByOrigPHI(false), KillsSource(false),
+        SpillReg(Reg), SpillVNI(VNI), SpillMBB(0), DefMI(0) {}
+
+    // Returns true when a def has been found.
+    bool hasDef() const { return DefByOrigPHI || DefMI; }
   };
 
+private:
   // Values in RegsToSpill defined by sibling copies.
   typedef DenseMap<VNInfo*, SibValueInfo> SibValueMap;
   SibValueMap SibValues;
@@ -134,6 +162,7 @@ class InlineSpiller : public Spiller {
 
   bool isSibling(unsigned Reg);
   MachineInstr *traceSiblingValue(unsigned, VNInfo*, VNInfo*);
+  void propagateSiblingValue(SibValueMap::iterator, VNInfo *VNI = 0);
   void analyzeSiblingValues();
 
   bool hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI);
@@ -282,6 +311,156 @@ bool InlineSpiller::isSibling(unsigned Reg) {
            VRM.getOriginal(Reg) == Original;
 }
 
+#ifndef NDEBUG
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const InlineSpiller::SibValueInfo &SVI) {
+  OS << "spill " << PrintReg(SVI.SpillReg) << ':'
+     << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def;
+  if (SVI.SpillMBB)
+    OS << " in BB#" << SVI.SpillMBB->getNumber();
+  if (SVI.AllDefsAreReloads)
+    OS << " all-reloads";
+  if (SVI.DefByOrigPHI)
+    OS << " orig-phi";
+  if (SVI.KillsSource)
+    OS << " kill";
+  OS << " deps[";
+  for (unsigned i = 0, e = SVI.Deps.size(); i != e; ++i)
+    OS << ' ' << SVI.Deps[i]->id << '@' << SVI.Deps[i]->def;
+  OS << " ]";
+  if (SVI.DefMI)
+    OS << " def: " << *SVI.DefMI;
+  else
+    OS << '\n';
+  return OS;
+}
+#endif
+
+/// propagateSiblingValue - Propagate the value in SVI to dependents if it is
+/// known.  Otherwise remember the dependency for later.
+///
+/// @param SVI SibValues entry to propagate.
+/// @param VNI Dependent value, or NULL to propagate to all saved dependents.
+void InlineSpiller::propagateSiblingValue(SibValueMap::iterator SVI,
+                                          VNInfo *VNI) {
+  // When VNI is non-NULL, add it to SVI's deps, and only propagate to that.
+  TinyPtrVector<VNInfo*> FirstDeps;
+  if (VNI) {
+    FirstDeps.push_back(VNI);
+    SVI->second.Deps.push_back(VNI);
+  }
+
+  // Has the value been completely determined yet?  If not, defer propagation.
+  if (!SVI->second.hasDef())
+    return;
+
+  // Work list of values to propagate.  It would be nice to use a SetVector
+  // here, but then we would be forced to use a SmallSet.
+  SmallVector<SibValueMap::iterator, 8> WorkList(1, SVI);
+  SmallPtrSet<VNInfo*, 8> WorkSet;
+
+  do {
+    SVI = WorkList.pop_back_val();
+    WorkSet.erase(SVI->first);
+    TinyPtrVector<VNInfo*> *Deps = VNI ? &FirstDeps : &SVI->second.Deps;
+    VNI = 0;
+
+    SibValueInfo &SV = SVI->second;
+    if (!SV.SpillMBB)
+      SV.SpillMBB = LIS.getMBBFromIndex(SV.SpillVNI->def);
+
+    DEBUG(dbgs() << "  prop to " << Deps->size() << ": "
+                 << SVI->first->id << '@' << SVI->first->def << ":\t" << SV);
+
+    assert(SV.hasDef() && "Propagating undefined value");
+
+    // Should this value be propagated as a preferred spill candidate?  We don't
+    // propagate values of registers that are about to spill.
+    bool PropSpill = !DisableHoisting && !isRegToSpill(SV.SpillReg);
+    unsigned SpillDepth = ~0u;
+
+    for (TinyPtrVector<VNInfo*>::iterator DepI = Deps->begin(),
+         DepE = Deps->end(); DepI != DepE; ++DepI) {
+      SibValueMap::iterator DepSVI = SibValues.find(*DepI);
+      assert(DepSVI != SibValues.end() && "Dependent value not in SibValues");
+      SibValueInfo &DepSV = DepSVI->second;
+      if (!DepSV.SpillMBB)
+        DepSV.SpillMBB = LIS.getMBBFromIndex(DepSV.SpillVNI->def);
+
+      bool Changed = false;
+
+      // Propagate defining instruction.
+      if (!DepSV.hasDef()) {
+        Changed = true;
+        DepSV.DefMI = SV.DefMI;
+        DepSV.DefByOrigPHI = SV.DefByOrigPHI;
+      }
+
+      // Propagate AllDefsAreReloads.  For PHI values, this computes an AND of
+      // all predecessors.
+      if (!SV.AllDefsAreReloads && DepSV.AllDefsAreReloads) {
+        Changed = true;
+        DepSV.AllDefsAreReloads = false;
+      }
+
+      // Propagate best spill value.
+      if (PropSpill && SV.SpillVNI != DepSV.SpillVNI) {
+        if (SV.SpillMBB == DepSV.SpillMBB) {
+          // DepSV is in the same block.  Hoist when dominated.
+          if (DepSV.KillsSource && SV.SpillVNI->def < DepSV.SpillVNI->def) {
+            // This is an alternative def earlier in the same MBB.
+            // Hoist the spill as far as possible in SpillMBB. This can ease
+            // register pressure:
+            //
+            //   x = def
+            //   y = use x
+            //   s = copy x
+            //
+            // Hoisting the spill of s to immediately after the def removes the
+            // interference between x and y:
+            //
+            //   x = def
+            //   spill x
+            //   y = use x<kill>
+            //
+            // This hoist only helps when the DepSV copy kills its source.
+            Changed = true;
+            DepSV.SpillReg = SV.SpillReg;
+            DepSV.SpillVNI = SV.SpillVNI;
+            DepSV.SpillMBB = SV.SpillMBB;
+          }
+        } else {
+          // DepSV is in a different block.
+          if (SpillDepth == ~0u)
+            SpillDepth = Loops.getLoopDepth(SV.SpillMBB);
+
+          // Also hoist spills to blocks with smaller loop depth, but make sure
+          // that the new value dominates.  Non-phi dependents are always
+          // dominated, phis need checking.
+          if ((Loops.getLoopDepth(DepSV.SpillMBB) > SpillDepth) &&
+              (!DepSVI->first->isPHIDef() ||
+               MDT.dominates(SV.SpillMBB, DepSV.SpillMBB))) {
+            Changed = true;
+            DepSV.SpillReg = SV.SpillReg;
+            DepSV.SpillVNI = SV.SpillVNI;
+            DepSV.SpillMBB = SV.SpillMBB;
+          }
+        }
+      }
+
+      if (!Changed)
+        continue;
+
+      // Something changed in DepSVI. Propagate to dependents.
+      if (WorkSet.insert(DepSVI->first))
+        WorkList.push_back(DepSVI);
+
+      DEBUG(dbgs() << "  update " << DepSVI->first->id << '@'
+            << DepSVI->first->def << " to:\t" << DepSV);
+    }
+  } while (!WorkList.empty());
+}
+
 /// traceSiblingValue - Trace a value that is about to be spilled back to the
 /// real defining instructions by looking through sibling copies. Always stay
 /// within the range of OrigVNI so the registers are known to carry the same
@@ -294,84 +473,101 @@ bool InlineSpiller::isSibling(unsigned Reg) {
 ///
 MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,
                                                VNInfo *OrigVNI) {
+  // Check if a cached value already exists.
+  SibValueMap::iterator SVI;
+  bool Inserted;
+  tie(SVI, Inserted) =
+    SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI)));
+  if (!Inserted) {
+    DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':'
+                 << UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second);
+    return SVI->second.DefMI;
+  }
+
   DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':'
                << UseVNI->id << '@' << UseVNI->def << '\n');
-  SmallPtrSet<VNInfo*, 8> Visited;
+
+  // List of (Reg, VNI) that have been inserted into SibValues, but need to be
+  // processed.
   SmallVector<std::pair<unsigned, VNInfo*>, 8> WorkList;
   WorkList.push_back(std::make_pair(UseReg, UseVNI));
 
-  // Best spill candidate seen so far. This must dominate UseVNI.
-  SibValueInfo SVI(UseReg, UseVNI);
-  MachineBasicBlock *UseMBB = LIS.getMBBFromIndex(UseVNI->def);
-  MachineBasicBlock *SpillMBB = UseMBB;
-  unsigned SpillDepth = Loops.getLoopDepth(SpillMBB);
-  bool SeenOrigPHI = false; // Original PHI met.
-
   do {
     unsigned Reg;
     VNInfo *VNI;
     tie(Reg, VNI) = WorkList.pop_back_val();
-    if (!Visited.insert(VNI))
-      continue;
+    DEBUG(dbgs() << "  " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def
+                 << ":\t");
 
-    // Is this value a better spill candidate?
-    if (!isRegToSpill(Reg)) {
-      MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def);
-      if (MBB == SpillMBB) {
-        // This is an alternative def earlier in the same MBB.
-        // Hoist the spill as far as possible in SpillMBB. This can ease
-        // register pressure:
-        //
-        //   x = def
-        //   y = use x
-        //   s = copy x
-        //
-        // Hoisting the spill of s to immediately after the def removes the
-        // interference between x and y:
-        //
-        //   x = def
-        //   spill x
-        //   y = use x<kill>
-        //
-        if (VNI->def < SVI.SpillVNI->def) {
-          DEBUG(dbgs() << "  hoist in BB#" << MBB->getNumber() << ": "
-                       << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def
-                       << '\n');
-          SVI.SpillReg = Reg;
-          SVI.SpillVNI = VNI;
-        }
-      } else if (MBB != UseMBB && MDT.dominates(MBB, UseMBB)) {
-        // This is a valid spill location dominating UseVNI.
-        // Prefer to spill at a smaller loop depth.
-        unsigned Depth = Loops.getLoopDepth(MBB);
-        if (Depth < SpillDepth) {
-          DEBUG(dbgs() << "  spill depth " << Depth << ": " << PrintReg(Reg)
-                       << ':' << VNI->id << '@' << VNI->def << '\n');
-          SVI.SpillReg = Reg;
-          SVI.SpillVNI = VNI;
-          SpillMBB = MBB;
-          SpillDepth = Depth;
-        }
-      }
-    }
+    // First check if this value has already been computed.
+    SVI = SibValues.find(VNI);
+    assert(SVI != SibValues.end() && "Missing SibValues entry");
 
     // Trace through PHI-defs created by live range splitting.
     if (VNI->isPHIDef()) {
+      // Stop at original PHIs.  We don't know the value at the predecessors.
       if (VNI->def == OrigVNI->def) {
-        DEBUG(dbgs() << "  orig phi value " << PrintReg(Reg) << ':'
-                     << VNI->id << '@' << VNI->def << '\n');
-        SeenOrigPHI = true;
+        DEBUG(dbgs() << "orig phi value\n");
+        SVI->second.DefByOrigPHI = true;
+        SVI->second.AllDefsAreReloads = false;
+        propagateSiblingValue(SVI);
         continue;
       }
-      // Get values live-out of predecessors.
+
+      // This is a PHI inserted by live range splitting.  We could trace the
+      // live-out value from predecessor blocks, but that search can be very
+      // expensive if there are many predecessors and many more PHIs as
+      // generated by tail-dup when it sees an indirectbr.  Instead, look at
+      // all the non-PHI defs that have the same value as OrigVNI.  They must
+      // jointly dominate VNI->def.  This is not optimal since VNI may actually
+      // be jointly dominated by a smaller subset of defs, so there is a change
+      // we will miss a AllDefsAreReloads optimization.
+
+      // Separate all values dominated by OrigVNI into PHIs and non-PHIs.
+      SmallVector<VNInfo*, 8> PHIs, NonPHIs;
       LiveInterval &LI = LIS.getInterval(Reg);
-      MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def);
-      for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
-             PE = MBB->pred_end(); PI != PE; ++PI) {
-        VNInfo *PVNI = LI.getVNInfoAt(LIS.getMBBEndIdx(*PI).getPrevSlot());
-        if (PVNI)
-          WorkList.push_back(std::make_pair(Reg, PVNI));
+      LiveInterval &OrigLI = LIS.getInterval(Original);
+
+      for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end();
+           VI != VE; ++VI) {
+        VNInfo *VNI2 = *VI;
+        if (VNI2->isUnused())
+          continue;
+        if (!OrigLI.containsOneValue() &&
+            OrigLI.getVNInfoAt(VNI2->def) != OrigVNI)
+          continue;
+        if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def)
+          PHIs.push_back(VNI2);
+        else
+          NonPHIs.push_back(VNI2);
       }
+      DEBUG(dbgs() << "split phi value, checking " << PHIs.size()
+                   << " phi-defs, and " << NonPHIs.size()
+                   << " non-phi/orig defs\n");
+
+      // Create entries for all the PHIs.  Don't add them to the worklist, we
+      // are processing all of them in one go here.
+      for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
+        SibValues.insert(std::make_pair(PHIs[i], SibValueInfo(Reg, PHIs[i])));
+
+      // Add every PHI as a dependent of all the non-PHIs.
+      for (unsigned i = 0, e = NonPHIs.size(); i != e; ++i) {
+        VNInfo *NonPHI = NonPHIs[i];
+        // Known value? Try an insertion.
+        tie(SVI, Inserted) =
+          SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI)));
+        // Add all the PHIs as dependents of NonPHI.
+        for (unsigned pi = 0, pe = PHIs.size(); pi != pe; ++pi)
+          SVI->second.Deps.push_back(PHIs[pi]);
+        // This is the first time we see NonPHI, add it to the worklist.
+        if (Inserted)
+          WorkList.push_back(std::make_pair(Reg, NonPHI));
+        else
+          // Propagate to all inserted PHIs, not just VNI.
+          propagateSiblingValue(SVI);
+      }
+
+      // Next work list item.
       continue;
     }
 
@@ -382,48 +578,49 @@ MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,
     if (unsigned SrcReg = isFullCopyOf(MI, Reg)) {
       if (isSibling(SrcReg)) {
         LiveInterval &SrcLI = LIS.getInterval(SrcReg);
-        VNInfo *SrcVNI = SrcLI.getVNInfoAt(VNI->def.getUseIndex());
-        assert(SrcVNI && "Copy from non-existing value");
-        DEBUG(dbgs() << "  copy of " << PrintReg(SrcReg) << ':'
-                     << SrcVNI->id << '@' << SrcVNI->def << '\n');
-        WorkList.push_back(std::make_pair(SrcReg, SrcVNI));
+        LiveRange *SrcLR = SrcLI.getLiveRangeContaining(VNI->def.getUseIndex());
+        assert(SrcLR && "Copy from non-existing value");
+        // Check if this COPY kills its source.
+        SVI->second.KillsSource = (SrcLR->end == VNI->def);
+        VNInfo *SrcVNI = SrcLR->valno;
+        DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':'
+                     << SrcVNI->id << '@' << SrcVNI->def
+                     << " kill=" << unsigned(SVI->second.KillsSource) << '\n');
+        // Known sibling source value? Try an insertion.
+        tie(SVI, Inserted) = SibValues.insert(std::make_pair(SrcVNI,
+                                                 SibValueInfo(SrcReg, SrcVNI)));
+        // This is the first time we see Src, add it to the worklist.
+        if (Inserted)
+          WorkList.push_back(std::make_pair(SrcReg, SrcVNI));
+        propagateSiblingValue(SVI, VNI);
+        // Next work list item.
         continue;
       }
     }
 
     // Track reachable reloads.
+    SVI->second.DefMI = MI;
+    SVI->second.SpillMBB = MI->getParent();
     int FI;
     if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) {
-      DEBUG(dbgs() << "  reload " << PrintReg(Reg) << ':'
-                   << VNI->id << "@" << VNI->def << '\n');
-      SVI.AllDefsAreReloads = true;
+      DEBUG(dbgs() << "reload\n");
+      propagateSiblingValue(SVI);
+      // Next work list item.
       continue;
     }
 
-    // We have an 'original' def. Don't record trivial cases.
-    if (VNI == UseVNI) {
-      DEBUG(dbgs() << "Not a sibling copy.\n");
-      return MI;
-    }
-
     // Potential remat candidate.
-    DEBUG(dbgs() << "  def " << PrintReg(Reg) << ':'
-                 << VNI->id << '@' << VNI->def << '\t' << *MI);
-    SVI.DefMI = MI;
+    DEBUG(dbgs() << "def " << *MI);
+    SVI->second.AllDefsAreReloads = false;
+    propagateSiblingValue(SVI);
   } while (!WorkList.empty());
 
-  if (SeenOrigPHI || SVI.DefMI)
-    SVI.AllDefsAreReloads = false;
-
-  DEBUG({
-    if (SVI.AllDefsAreReloads)
-      dbgs() << "All defs are reloads.\n";
-    else
-      dbgs() << "Prefer to spill " << PrintReg(SVI.SpillReg) << ':'
-             << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def << '\n';
-  });
-  SibValues.insert(std::make_pair(UseVNI, SVI));
-  return SVI.DefMI;
+  // Look up the value we were looking for.  We already did this lokup at the
+  // top of the function, but SibValues may have been invalidated.
+  SVI = SibValues.find(UseVNI);
+  assert(SVI != SibValues.end() && "Didn't compute requested info");
+  DEBUG(dbgs() << "  traced to:\t" << SVI->second);
+  return SVI->second.DefMI;
 }
 
 /// analyzeSiblingValues - Trace values defined by sibling copies back to
@@ -506,6 +703,7 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) {
 
   // Already spilled everywhere.
   if (SVI.AllDefsAreReloads) {
+    DEBUG(dbgs() << "\tno spill needed: " << SVI);
     ++NumOmitReloadSpill;
     return true;
   }
@@ -531,10 +729,8 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) {
   VRM.addSpillSlotUse(StackSlot, MII);
   DEBUG(dbgs() << "\thoisted: " << SVI.SpillVNI->def << '\t' << *MII);
 
-  if (MBB == CopyMI->getParent())
-    ++NumHoistLocal;
-  else
-    ++NumHoistGlobal;
+  ++NumSpills;
+  ++NumHoists;
   return true;
 }
 
@@ -589,7 +785,8 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) {
         // eliminateDeadDefs won't normally remove stores, so switch opcode.
         MI->setDesc(TII.get(TargetOpcode::KILL));
         DeadDefs.push_back(MI);
-        ++NumRedundantSpills;
+        ++NumSpillsRemoved;
+        --NumSpills;
       }
     }
   } while (!WorkList.empty());
@@ -637,7 +834,7 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
 bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg,
                                      MachineBasicBlock::iterator MI) {
   SlotIndex UseIdx = LIS.getInstructionIndex(MI).getUseIndex();
-  VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx);
+  VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex());
 
   if (!ParentVNI) {
     DEBUG(dbgs() << "\tadding <undef> flags: ");
@@ -787,10 +984,10 @@ void InlineSpiller::reMaterializeAll() {
 /// If MI is a load or store of StackSlot, it can be removed.
 bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) {
   int FI = 0;
-  unsigned InstrReg;
-  if (!(InstrReg = TII.isLoadFromStackSlot(MI, FI)) &&
-      !(InstrReg = TII.isStoreToStackSlot(MI, FI)))
-    return false;
+  unsigned InstrReg = TII.isLoadFromStackSlot(MI, FI);
+  bool IsLoad = InstrReg;
+  if (!IsLoad)
+    InstrReg = TII.isStoreToStackSlot(MI, FI);
 
   // We have a stack access. Is it the right register and slot?
   if (InstrReg != Reg || FI != StackSlot)
@@ -799,6 +996,15 @@ bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) {
   DEBUG(dbgs() << "Coalescing stack access: " << *MI);
   LIS.RemoveMachineInstrFromMaps(MI);
   MI->eraseFromParent();
+
+  if (IsLoad) {
+    ++NumReloadsRemoved;
+    --NumReloads;
+  } else {
+    ++NumSpillsRemoved;
+    --NumSpills;
+  }
+
   return true;
 }
 
@@ -810,6 +1016,7 @@ bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) {
 bool InlineSpiller::foldMemoryOperand(MachineBasicBlock::iterator MI,
                                       const SmallVectorImpl<unsigned> &Ops,
                                       MachineInstr *LoadMI) {
+  bool WasCopy = MI->isCopy();
   // TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied
   // operands.
   SmallVector<unsigned, 8> FoldOps;
@@ -839,7 +1046,12 @@ bool InlineSpiller::foldMemoryOperand(MachineBasicBlock::iterator MI,
     VRM.addSpillSlotUse(StackSlot, FoldMI);
   MI->eraseFromParent();
   DEBUG(dbgs() << "\tfolded: " << *FoldMI);
-  ++NumFolded;
+  if (!WasCopy)
+    ++NumFolded;
+  else if (Ops.front() == 0)
+    ++NumSpills;
+  else
+    ++NumReloads;
   return true;
 }
 
@@ -975,8 +1187,16 @@ void InlineSpiller::spillAroundUses(unsigned Reg) {
     DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI);
 
     // FIXME: Use a second vreg if instruction has no tied ops.
-    if (Writes && hasLiveDef)
+    if (Writes) {
+     if (hasLiveDef)
       insertSpill(NewLI, OldLI, Idx, MI);
+     else {
+       // This instruction defines a dead value.  We don't need to spill it,
+       // but do create a live range for the dead value.
+       VNInfo *VNI = NewLI.getNextValue(Idx, 0, LIS.getVNInfoAllocator());
+       NewLI.addRange(LiveRange(Idx, Idx.getNextSlot(), VNI));
+     }
+    }
 
     DEBUG(dbgs() << "\tinterval: " << NewLI << '\n');
   }
diff --git a/contrib/llvm/lib/CodeGen/InterferenceCache.cpp b/contrib/llvm/lib/CodeGen/InterferenceCache.cpp
index a09bb39f8336..29b47bd67ece 100644
--- a/contrib/llvm/lib/CodeGen/InterferenceCache.cpp
+++ b/contrib/llvm/lib/CodeGen/InterferenceCache.cpp
@@ -18,10 +18,13 @@
 
 using namespace llvm;
 
+// Static member used for null interference cursors.
+InterferenceCache::BlockInterference InterferenceCache::Cursor::NoInterference;
+
 void InterferenceCache::init(MachineFunction *mf,
                              LiveIntervalUnion *liuarray,
                              SlotIndexes *indexes,
-                            const TargetRegisterInfo *tri) {
+                             const TargetRegisterInfo *tri) {
   MF = mf;
   LIUArray = liuarray;
   TRI = tri;
diff --git a/contrib/llvm/lib/CodeGen/InterferenceCache.h b/contrib/llvm/lib/CodeGen/InterferenceCache.h
index 7f0a27a41baa..4df0a9e5c393 100644
--- a/contrib/llvm/lib/CodeGen/InterferenceCache.h
+++ b/contrib/llvm/lib/CodeGen/InterferenceCache.h
@@ -138,6 +138,7 @@ class InterferenceCache {
   class Cursor {
     Entry *CacheEntry;
     BlockInterference *Current;
+    static BlockInterference NoInterference;
 
     void setEntry(Entry *E) {
       Current = 0;
@@ -175,7 +176,7 @@ class InterferenceCache {
 
     /// moveTo - Move cursor to basic block MBBNum.
     void moveToBlock(unsigned MBBNum) {
-      Current = CacheEntry->get(MBBNum);
+      Current = CacheEntry ? CacheEntry->get(MBBNum) : &NoInterference;
     }
 
     /// hasInterference - Return true if the current block has any interference.
diff --git a/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp b/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp
index 611886ff16a1..0f92c2d06bdd 100644
--- a/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp
+++ b/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp
@@ -27,7 +27,7 @@ using namespace llvm;
 template <class ArgIt>
 static void EnsureFunctionExists(Module &M, const char *Name,
                                  ArgIt ArgBegin, ArgIt ArgEnd,
-                                 const Type *RetTy) {
+                                 Type *RetTy) {
   // Insert a correctly-typed definition now.
   std::vector<Type *> ParamTys;
   for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
@@ -64,7 +64,7 @@ static void EnsureFPIntrinsicsExist(Module &M, Function *Fn,
 template <class ArgIt>
 static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
                                  ArgIt ArgBegin, ArgIt ArgEnd,
-                                 const Type *RetTy) {
+                                 Type *RetTy) {
   // If we haven't already looked up this function, check to see if the
   // program already contains a function with this name.
   Module *M = CI->getParent()->getParent()->getParent();
@@ -462,7 +462,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;   // Strip out annotate intrinsic
     
   case Intrinsic::memcpy: {
-    const IntegerType *IntPtr = TD.getIntPtrType(Context);
+    IntegerType *IntPtr = TD.getIntPtrType(Context);
     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
                                         /* isSigned */ false);
     Value *Ops[3];
@@ -473,7 +473,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;
   }
   case Intrinsic::memmove: {
-    const IntegerType *IntPtr = TD.getIntPtrType(Context);
+    IntegerType *IntPtr = TD.getIntPtrType(Context);
     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
                                         /* isSigned */ false);
     Value *Ops[3];
@@ -484,7 +484,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;
   }
   case Intrinsic::memset: {
-    const IntegerType *IntPtr = TD.getIntPtrType(Context);
+    IntegerType *IntPtr = TD.getIntPtrType(Context);
     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
                                         /* isSigned */ false);
     Value *Ops[3];
diff --git a/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index f985af8ba83e..80ecc224810c 100644
--- a/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -27,16 +27,18 @@
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetAsmInfo.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetInstrInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 namespace llvm {
@@ -55,8 +57,12 @@ static cl::opt<bool> DisableCodePlace("disable-code-place", cl::Hidden,
     cl::desc("Disable code placement"));
 static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
     cl::desc("Disable Stack Slot Coloring"));
+static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
+    cl::desc("Disable Machine Dead Code Elimination"));
 static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
     cl::desc("Disable Machine LICM"));
+static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
+    cl::desc("Disable Machine Common Subexpression Elimination"));
 static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
     cl::Hidden,
     cl::desc("Disable Machine LICM"));
@@ -103,20 +109,17 @@ EnableFastISelOption("fast-isel", cl::Hidden,
   cl::desc("Enable the \"fast\" instruction selector"));
 
 LLVMTargetMachine::LLVMTargetMachine(const Target &T, StringRef Triple,
-                                     StringRef CPU, StringRef FS)
+                                     StringRef CPU, StringRef FS,
+                                     Reloc::Model RM, CodeModel::Model CM)
   : TargetMachine(T, Triple, CPU, FS) {
+  CodeGenInfo = T.createMCCodeGenInfo(Triple, RM, CM);
   AsmInfo = T.createMCAsmInfo(Triple);
-}
-
-// Set the default code model for the JIT for a generic target.
-// FIXME: Is small right here? or .is64Bit() ? Large : Small?
-void LLVMTargetMachine::setCodeModelForJIT() {
-  setCodeModel(CodeModel::Small);
-}
-
-// Set the default code model for static compilation for a generic target.
-void LLVMTargetMachine::setCodeModelForStatic() {
-  setCodeModel(CodeModel::Small);
+  // TargetSelect.h moved to a different directory between LLVM 2.9 and 3.0,
+  // and if the old one gets included then MCAsmInfo will be NULL and
+  // we'll crash later.
+  // Provide the user with a useful error message about what's wrong.
+  assert(AsmInfo && "MCAsmInfo not initialized."
+	 "Make sure you include the correct TargetSelect.h!");
 }
 
 bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
@@ -134,21 +137,22 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
     Context->setAllowTemporaryLabels(false);
 
   const MCAsmInfo &MAI = *getMCAsmInfo();
+  const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
   OwningPtr<MCStreamer> AsmStreamer;
 
   switch (FileType) {
   default: return true;
   case CGFT_AssemblyFile: {
     MCInstPrinter *InstPrinter =
-      getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI);
+      getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI, STI);
 
     // Create a code emitter if asked to show the encoding.
     MCCodeEmitter *MCE = 0;
-    TargetAsmBackend *TAB = 0;
+    MCAsmBackend *MAB = 0;
     if (ShowMCEncoding) {
       const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
-      MCE = getTarget().createCodeEmitter(*getInstrInfo(), STI, *Context);
-      TAB = getTarget().createAsmBackend(getTargetTriple());
+      MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), STI, *Context);
+      MAB = getTarget().createMCAsmBackend(getTargetTriple());
     }
 
     MCStreamer *S = getTarget().createAsmStreamer(*Context, Out,
@@ -156,7 +160,7 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
                                                   hasMCUseLoc(),
                                                   hasMCUseCFI(),
                                                   InstPrinter,
-                                                  MCE, TAB,
+                                                  MCE, MAB,
                                                   ShowMCInst);
     AsmStreamer.reset(S);
     break;
@@ -164,17 +168,16 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
   case CGFT_ObjectFile: {
     // Create the code emitter for the target if it exists.  If not, .o file
     // emission fails.
-    const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
-    MCCodeEmitter *MCE = getTarget().createCodeEmitter(*getInstrInfo(), STI,
-                                                       *Context);
-    TargetAsmBackend *TAB = getTarget().createAsmBackend(getTargetTriple());
-    if (MCE == 0 || TAB == 0)
+    MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), STI,
+                                                         *Context);
+    MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple());
+    if (MCE == 0 || MAB == 0)
       return true;
 
-    AsmStreamer.reset(getTarget().createObjectStreamer(getTargetTriple(),
-                                                       *Context, *TAB, Out, MCE,
-                                                       hasMCRelaxAll(),
-                                                       hasMCNoExecStack()));
+    AsmStreamer.reset(getTarget().createMCObjectStreamer(getTargetTriple(),
+                                                         *Context, *MAB, Out,
+                                                         MCE, hasMCRelaxAll(),
+                                                         hasMCNoExecStack()));
     AsmStreamer.get()->InitSections();
     break;
   }
@@ -198,8 +201,6 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
 
   PM.add(Printer);
 
-  // Make sure the code model is set.
-  setCodeModelForStatic();
   PM.add(createGCInfoDeleter());
   return false;
 }
@@ -214,9 +215,6 @@ bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
                                                    JITCodeEmitter &JCE,
                                                    CodeGenOpt::Level OptLevel,
                                                    bool DisableVerify) {
-  // Make sure the code model is set.
-  setCodeModelForJIT();
-
   // Add common CodeGen passes.
   MCContext *Ctx = 0;
   if (addCommonCodeGenPasses(PM, OptLevel, DisableVerify, Ctx))
@@ -248,16 +246,16 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM,
   // Create the code emitter for the target if it exists.  If not, .o file
   // emission fails.
   const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
-  MCCodeEmitter *MCE = getTarget().createCodeEmitter(*getInstrInfo(),STI, *Ctx);
-  TargetAsmBackend *TAB = getTarget().createAsmBackend(getTargetTriple());
-  if (MCE == 0 || TAB == 0)
+  MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(),STI, *Ctx);
+  MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple());
+  if (MCE == 0 || MAB == 0)
     return true;
 
   OwningPtr<MCStreamer> AsmStreamer;
-  AsmStreamer.reset(getTarget().createObjectStreamer(getTargetTriple(), *Ctx,
-                                                     *TAB, Out, MCE,
-                                                     hasMCRelaxAll(),
-                                                     hasMCNoExecStack()));
+  AsmStreamer.reset(getTarget().createMCObjectStreamer(getTargetTriple(), *Ctx,
+                                                       *MAB, Out, MCE,
+                                                       hasMCRelaxAll(),
+                                                       hasMCNoExecStack()));
   AsmStreamer.get()->InitSections();
 
   // Create the AsmPrinter, which takes ownership of AsmStreamer if successful.
@@ -270,9 +268,6 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM,
 
   PM.add(Printer);
 
-  // Make sure the code model is set.
-  setCodeModelForJIT();
-
   return false; // success!
 }
 
@@ -369,8 +364,9 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM,
 
   // Install a MachineModuleInfo class, which is an immutable pass that holds
   // all the per-module stuff we're generating, including MCContext.
-  TargetAsmInfo *TAI = new TargetAsmInfo(*this);
-  MachineModuleInfo *MMI = new MachineModuleInfo(*getMCAsmInfo(), TAI);
+  MachineModuleInfo *MMI = new MachineModuleInfo(*getMCAsmInfo(),
+                                                 *getRegisterInfo(),
+                                     &getTargetLowering()->getObjFileLowering());
   PM.add(MMI);
   OutContext = &MMI->getContext(); // Return the MCContext specifically by-ref.
 
@@ -412,12 +408,14 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM,
     // there is one known exception: lowered code for arguments that are only
     // used by tail calls, where the tail calls reuse the incoming stack
     // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
-    PM.add(createDeadMachineInstructionElimPass());
+    if (!DisableMachineDCE)
+      PM.add(createDeadMachineInstructionElimPass());
     printAndVerify(PM, "After codegen DCE pass");
 
     if (!DisableMachineLICM)
       PM.add(createMachineLICMPass());
-    PM.add(createMachineCSEPass());
+    if (!DisableMachineCSE)
+      PM.add(createMachineCSEPass());
     if (!DisableMachineSink)
       PM.add(createMachineSinkingPass());
     printAndVerify(PM, "After Machine LICM, CSE and Sinking passes");
@@ -452,8 +450,8 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM,
   if (addPostRegAlloc(PM, OptLevel))
     printAndVerify(PM, "After PostRegAlloc passes");
 
-  PM.add(createLowerSubregsPass());
-  printAndVerify(PM, "After LowerSubregs");
+  PM.add(createExpandPostRAPseudosPass());
+  printAndVerify(PM, "After ExpandPostRAPseudos");
 
   // Insert prolog/epilog code.  Eliminate abstract frame index references...
   PM.add(createPrologEpilogCodeInserter());
diff --git a/contrib/llvm/lib/CodeGen/LexicalScopes.cpp b/contrib/llvm/lib/CodeGen/LexicalScopes.cpp
new file mode 100644
index 000000000000..a12e1a36d113
--- /dev/null
+++ b/contrib/llvm/lib/CodeGen/LexicalScopes.cpp
@@ -0,0 +1,335 @@
+//===- LexicalScopes.cpp - Collecting lexical scope info ------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements LexicalScopes analysis.
+//
+// This pass collects lexical scope information and maps machine instructions
+// to respective lexical scopes.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "lexicalscopes"
+#include "llvm/CodeGen/LexicalScopes.h"
+#include "llvm/Function.h"
+#include "llvm/Analysis/DebugInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormattedStream.h"
+using namespace llvm;
+
+LexicalScopes::~LexicalScopes() {
+  releaseMemory();
+}
+
+/// releaseMemory - release memory.
+void LexicalScopes::releaseMemory() {
+  MF = NULL;
+  CurrentFnLexicalScope = NULL;
+  DeleteContainerSeconds(LexicalScopeMap);
+  DeleteContainerSeconds(AbstractScopeMap);
+  InlinedLexicalScopeMap.clear();
+  AbstractScopesList.clear();
+}
+
+/// initialize - Scan machine function and constuct lexical scope nest.
+void LexicalScopes::initialize(const MachineFunction &Fn) {
+  releaseMemory();
+  MF = &Fn;
+  SmallVector<InsnRange, 4> MIRanges;
+  DenseMap<const MachineInstr *, LexicalScope *> MI2ScopeMap;
+  extractLexicalScopes(MIRanges, MI2ScopeMap);
+  if (CurrentFnLexicalScope) {
+    constructScopeNest(CurrentFnLexicalScope);
+    assignInstructionRanges(MIRanges, MI2ScopeMap);
+  }
+}
+
+/// extractLexicalScopes - Extract instruction ranges for each lexical scopes
+/// for the given machine function.
+void LexicalScopes::
+extractLexicalScopes(SmallVectorImpl<InsnRange> &MIRanges,
+                  DenseMap<const MachineInstr *, LexicalScope *> &MI2ScopeMap) {
+
+  // Scan each instruction and create scopes. First build working set of scopes.
+  for (MachineFunction::const_iterator I = MF->begin(), E = MF->end();
+       I != E; ++I) {
+    const MachineInstr *RangeBeginMI = NULL;
+    const MachineInstr *PrevMI = NULL;
+    DebugLoc PrevDL;
+    for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end();
+         II != IE; ++II) {
+      const MachineInstr *MInsn = II;
+
+      // Check if instruction has valid location information.
+      const DebugLoc MIDL = MInsn->getDebugLoc();
+      if (MIDL.isUnknown()) {
+        PrevMI = MInsn;
+        continue;
+      }
+
+      // If scope has not changed then skip this instruction.
+      if (MIDL == PrevDL) {
+        PrevMI = MInsn;
+        continue;
+      }
+
+      // Ignore DBG_VALUE. It does not contribute to any instruction in output.
+      if (MInsn->isDebugValue())
+        continue;
+
+      if (RangeBeginMI) {
+        // If we have already seen a beginning of an instruction range and
+        // current instruction scope does not match scope of first instruction
+        // in this range then create a new instruction range.
+        InsnRange R(RangeBeginMI, PrevMI);
+        MI2ScopeMap[RangeBeginMI] = getOrCreateLexicalScope(PrevDL);
+        MIRanges.push_back(R);
+      }
+
+      // This is a beginning of a new instruction range.
+      RangeBeginMI = MInsn;
+
+      // Reset previous markers.
+      PrevMI = MInsn;
+      PrevDL = MIDL;
+    }
+
+    // Create last instruction range.
+    if (RangeBeginMI && PrevMI && !PrevDL.isUnknown()) {
+      InsnRange R(RangeBeginMI, PrevMI);
+      MIRanges.push_back(R);
+      MI2ScopeMap[RangeBeginMI] = getOrCreateLexicalScope(PrevDL);
+    }
+  }
+}
+
+/// findLexicalScope - Find lexical scope, either regular or inlined, for the
+/// given DebugLoc. Return NULL if not found.
+LexicalScope *LexicalScopes::findLexicalScope(DebugLoc DL) {
+  MDNode *Scope = NULL;
+  MDNode *IA = NULL;
+  DL.getScopeAndInlinedAt(Scope, IA, MF->getFunction()->getContext());
+  if (!Scope) return NULL;
+
+  // The scope that we were created with could have an extra file - which
+  // isn't what we care about in this case.
+  DIDescriptor D = DIDescriptor(Scope);
+  if (D.isLexicalBlockFile())
+    Scope = DILexicalBlockFile(Scope).getScope();
+  
+  if (IA)
+    return InlinedLexicalScopeMap.lookup(DebugLoc::getFromDILocation(IA));
+  return LexicalScopeMap.lookup(Scope);
+}
+
+/// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If
+/// not available then create new lexical scope.
+LexicalScope *LexicalScopes::getOrCreateLexicalScope(DebugLoc DL) {
+  MDNode *Scope = NULL;
+  MDNode *InlinedAt = NULL;
+  DL.getScopeAndInlinedAt(Scope, InlinedAt, MF->getFunction()->getContext());
+
+  if (InlinedAt) {
+    // Create an abstract scope for inlined function.
+    getOrCreateAbstractScope(Scope);
+    // Create an inlined scope for inlined function.
+    return getOrCreateInlinedScope(Scope, InlinedAt);
+  }
+   
+  return getOrCreateRegularScope(Scope);
+}
+
+/// getOrCreateRegularScope - Find or create a regular lexical scope.
+LexicalScope *LexicalScopes::getOrCreateRegularScope(MDNode *Scope) {
+  DIDescriptor D = DIDescriptor(Scope);
+  if (D.isLexicalBlockFile()) {
+    Scope = DILexicalBlockFile(Scope).getScope();
+    D = DIDescriptor(Scope);
+  }
+ 
+  LexicalScope *WScope = LexicalScopeMap.lookup(Scope);
+  if (WScope)
+    return WScope;
+
+  LexicalScope *Parent = NULL;
+  if (D.isLexicalBlock())
+    Parent = getOrCreateLexicalScope(DebugLoc::getFromDILexicalBlock(Scope));
+  WScope = new LexicalScope(Parent, DIDescriptor(Scope), NULL, false);
+  LexicalScopeMap.insert(std::make_pair(Scope, WScope));
+  if (!Parent && DIDescriptor(Scope).isSubprogram()
+      && DISubprogram(Scope).describes(MF->getFunction()))
+    CurrentFnLexicalScope = WScope;
+  
+  return WScope;
+}
+
+/// getOrCreateInlinedScope - Find or create an inlined lexical scope.
+LexicalScope *LexicalScopes::getOrCreateInlinedScope(MDNode *Scope, 
+                                                     MDNode *InlinedAt) {
+  LexicalScope *InlinedScope = LexicalScopeMap.lookup(InlinedAt);
+  if (InlinedScope)
+    return InlinedScope;
+
+  DebugLoc InlinedLoc = DebugLoc::getFromDILocation(InlinedAt);
+  InlinedScope = new LexicalScope(getOrCreateLexicalScope(InlinedLoc),
+                                  DIDescriptor(Scope), InlinedAt, false);
+  InlinedLexicalScopeMap[InlinedLoc] = InlinedScope;
+  LexicalScopeMap[InlinedAt] = InlinedScope;
+  return InlinedScope;
+}
+
+/// getOrCreateAbstractScope - Find or create an abstract lexical scope.
+LexicalScope *LexicalScopes::getOrCreateAbstractScope(const MDNode *N) {
+  assert(N && "Invalid Scope encoding!");
+
+  DIDescriptor Scope(N);
+  if (Scope.isLexicalBlockFile())
+    Scope = DILexicalBlockFile(Scope).getScope();
+  LexicalScope *AScope = AbstractScopeMap.lookup(N);
+  if (AScope)
+    return AScope;
+
+  LexicalScope *Parent = NULL;
+  if (Scope.isLexicalBlock()) {
+    DILexicalBlock DB(N);
+    DIDescriptor ParentDesc = DB.getContext();
+    Parent = getOrCreateAbstractScope(ParentDesc);
+  }
+  AScope = new LexicalScope(Parent, DIDescriptor(N), NULL, true);
+  AbstractScopeMap[N] = AScope;
+  if (DIDescriptor(N).isSubprogram())
+    AbstractScopesList.push_back(AScope);
+  return AScope;
+}
+
+/// constructScopeNest
+void LexicalScopes::constructScopeNest(LexicalScope *Scope) {
+  assert (Scope && "Unable to calculate scop edominance graph!");
+  SmallVector<LexicalScope *, 4> WorkStack;
+  WorkStack.push_back(Scope);
+  unsigned Counter = 0;
+  while (!WorkStack.empty()) {
+    LexicalScope *WS = WorkStack.back();
+    const SmallVector<LexicalScope *, 4> &Children = WS->getChildren();
+    bool visitedChildren = false;
+    for (SmallVector<LexicalScope *, 4>::const_iterator SI = Children.begin(),
+           SE = Children.end(); SI != SE; ++SI) {
+      LexicalScope *ChildScope = *SI;
+      if (!ChildScope->getDFSOut()) {
+        WorkStack.push_back(ChildScope);
+        visitedChildren = true;
+        ChildScope->setDFSIn(++Counter);
+        break;
+      }
+    }
+    if (!visitedChildren) {
+      WorkStack.pop_back();
+      WS->setDFSOut(++Counter);
+    }
+  }
+}
+
+/// assignInstructionRanges - Find ranges of instructions covered by each
+/// lexical scope.
+void LexicalScopes::
+assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges,
+                    DenseMap<const MachineInstr *, LexicalScope *> &MI2ScopeMap)
+{
+  
+  LexicalScope *PrevLexicalScope = NULL;
+  for (SmallVectorImpl<InsnRange>::const_iterator RI = MIRanges.begin(),
+         RE = MIRanges.end(); RI != RE; ++RI) {
+    const InsnRange &R = *RI;
+    LexicalScope *S = MI2ScopeMap.lookup(R.first);
+    assert (S && "Lost LexicalScope for a machine instruction!");
+    if (PrevLexicalScope && !PrevLexicalScope->dominates(S))
+      PrevLexicalScope->closeInsnRange(S);
+    S->openInsnRange(R.first);
+    S->extendInsnRange(R.second);
+    PrevLexicalScope = S;
+  }
+
+  if (PrevLexicalScope)
+    PrevLexicalScope->closeInsnRange();
+}
+
+/// getMachineBasicBlocks - Populate given set using machine basic blocks which
+/// have machine instructions that belong to lexical scope identified by 
+/// DebugLoc.
+void LexicalScopes::
+getMachineBasicBlocks(DebugLoc DL, 
+                      SmallPtrSet<const MachineBasicBlock*, 4> &MBBs) {
+  MBBs.clear();
+  LexicalScope *Scope = getOrCreateLexicalScope(DL);
+  if (!Scope)
+    return;
+  
+  if (Scope == CurrentFnLexicalScope) {
+    for (MachineFunction::const_iterator I = MF->begin(), E = MF->end();
+         I != E; ++I)
+      MBBs.insert(I);
+    return;
+  }
+
+  SmallVector<InsnRange, 4> &InsnRanges = Scope->getRanges();
+  for (SmallVector<InsnRange, 4>::iterator I = InsnRanges.begin(),
+         E = InsnRanges.end(); I != E; ++I) {
+    InsnRange &R = *I;
+    MBBs.insert(R.first->getParent());
+  }
+}
+
+/// dominates - Return true if DebugLoc's lexical scope dominates at least one
+/// machine instruction's lexical scope in a given machine basic block.
+bool LexicalScopes::dominates(DebugLoc DL, MachineBasicBlock *MBB) {
+  LexicalScope *Scope = getOrCreateLexicalScope(DL);
+  if (!Scope)
+    return false;
+
+  // Current function scope covers all basic blocks in the function.
+  if (Scope == CurrentFnLexicalScope && MBB->getParent() == MF)
+    return true;
+
+  bool Result = false;
+  for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+       I != E; ++I) {
+    DebugLoc IDL = I->getDebugLoc();
+    if (IDL.isUnknown())
+      continue;
+    if (LexicalScope *IScope = getOrCreateLexicalScope(IDL))
+      if (Scope->dominates(IScope))
+        return true;
+  }
+  return Result;
+}
+
+/// dump - Print data structures.
+void LexicalScope::dump() const {
+#ifndef NDEBUG
+  raw_ostream &err = dbgs();
+  err.indent(IndentLevel);
+  err << "DFSIn: " << DFSIn << " DFSOut: " << DFSOut << "\n";
+  const MDNode *N = Desc;
+  N->dump();
+  if (AbstractScope)
+    err << "Abstract Scope\n";
+
+  IndentLevel += 2;
+  if (!Children.empty())
+    err << "Children ...\n";
+  for (unsigned i = 0, e = Children.size(); i != e; ++i)
+    if (Children[i] != this)
+      Children[i]->dump();
+
+  IndentLevel -= 2;
+#endif
+}
+
diff --git a/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp b/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 5d38c83b49c2..3dfe4c0e8cfa 100644
--- a/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -25,7 +25,10 @@
 #include "llvm/Constants.h"
 #include "llvm/Metadata.h"
 #include "llvm/Value.h"
+#include "llvm/Analysis/DebugInfo.h"
 #include "llvm/ADT/IntervalMap.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LexicalScopes.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -44,6 +47,7 @@ static cl::opt<bool>
 EnableLDV("live-debug-variables", cl::init(true),
           cl::desc("Enable the live debug variables pass"), cl::Hidden);
 
+STATISTIC(NumInsertedDebugValues, "Number of DBG_VALUEs inserted");
 char LiveDebugVariables::ID = 0;
 
 INITIALIZE_PASS_BEGIN(LiveDebugVariables, "livedebugvars",
@@ -67,6 +71,29 @@ LiveDebugVariables::LiveDebugVariables() : MachineFunctionPass(ID), pImpl(0) {
 /// LocMap - Map of where a user value is live, and its location.
 typedef IntervalMap<SlotIndex, unsigned, 4> LocMap;
 
+namespace {
+/// UserValueScopes - Keeps track of lexical scopes associated with an
+/// user value's source location.
+class UserValueScopes {
+  DebugLoc DL;
+  LexicalScopes &LS;
+  SmallPtrSet<const MachineBasicBlock *, 4> LBlocks;
+
+public:
+  UserValueScopes(DebugLoc D, LexicalScopes &L) : DL(D), LS(L) {}
+
+  /// dominates - Return true if current scope dominates at least one machine
+  /// instruction in a given machine basic block.
+  bool dominates(MachineBasicBlock *MBB) {
+    if (LBlocks.empty())
+      LS.getMachineBasicBlocks(DL, LBlocks);
+    if (LBlocks.count(MBB) != 0 || LS.dominates(DL, MBB))
+      return true;
+    return false;
+  }
+};
+} // end anonymous namespace
+
 /// UserValue - A user value is a part of a debug info user variable.
 ///
 /// A DBG_VALUE instruction notes that (a sub-register of) a virtual register
@@ -179,6 +206,9 @@ class UserValue {
     LocMap::iterator I = locInts.find(Idx);
     if (!I.valid() || I.start() != Idx)
       I.insert(Idx, Idx.getNextSlot(), getLocationNo(LocMO));
+    else
+      // A later DBG_VALUE at the same SlotIndex overrides the old location.
+      I.setValue(getLocationNo(LocMO));
   }
 
   /// extendDef - Extend the current definition as far as possible down the
@@ -195,7 +225,8 @@ class UserValue {
   void extendDef(SlotIndex Idx, unsigned LocNo,
                  LiveInterval *LI, const VNInfo *VNI,
                  SmallVectorImpl<SlotIndex> *Kills,
-                 LiveIntervals &LIS, MachineDominatorTree &MDT);
+                 LiveIntervals &LIS, MachineDominatorTree &MDT,
+		 UserValueScopes &UVS);
 
   /// addDefsFromCopies - The value in LI/LocNo may be copies to other
   /// registers. Determine if any of the copies are available at the kill
@@ -213,7 +244,8 @@ class UserValue {
   /// computeIntervals - Compute the live intervals of all locations after
   /// collecting all their def points.
   void computeIntervals(MachineRegisterInfo &MRI,
-                        LiveIntervals &LIS, MachineDominatorTree &MDT);
+                        LiveIntervals &LIS, MachineDominatorTree &MDT,
+                        UserValueScopes &UVS);
 
   /// renameRegister - Update locations to rewrite OldReg as NewReg:SubIdx.
   void renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx,
@@ -236,6 +268,9 @@ class UserValue {
   /// Only first one needs DebugLoc to identify variable's lexical scope
   /// in source file.
   DebugLoc findDebugLoc();
+
+  /// getDebugLoc - Return DebugLoc of this UserValue.
+  DebugLoc getDebugLoc() { return dl;}
   void print(raw_ostream&, const TargetMachine*);
 };
 } // namespace
@@ -247,6 +282,7 @@ class LDVImpl {
   LocMap::Allocator allocator;
   MachineFunction *MF;
   LiveIntervals *LIS;
+  LexicalScopes LS;
   MachineDominatorTree *MDT;
   const TargetRegisterInfo *TRI;
 
@@ -312,8 +348,10 @@ class LDVImpl {
 } // namespace
 
 void UserValue::print(raw_ostream &OS, const TargetMachine *TM) {
-  if (const MDString *MDS = dyn_cast<MDString>(variable->getOperand(2)))
-    OS << "!\"" << MDS->getString() << "\"\t";
+  DIVariable DV(variable);
+  OS << "!\""; 
+  DV.printExtendedName(OS);
+  OS << "\"\t";
   if (offset)
     OS << '+' << offset;
   for (LocMap::const_iterator I = locInts.begin(); I.valid(); ++I) {
@@ -447,10 +485,10 @@ bool LDVImpl::collectDebugValues(MachineFunction &mf) {
 void UserValue::extendDef(SlotIndex Idx, unsigned LocNo,
                           LiveInterval *LI, const VNInfo *VNI,
                           SmallVectorImpl<SlotIndex> *Kills,
-                          LiveIntervals &LIS, MachineDominatorTree &MDT) {
+                          LiveIntervals &LIS, MachineDominatorTree &MDT,
+			  UserValueScopes &UVS) {
   SmallVector<SlotIndex, 16> Todo;
   Todo.push_back(Idx);
-
   do {
     SlotIndex Start = Todo.pop_back_val();
     MachineBasicBlock *MBB = LIS.getMBBFromIndex(Start);
@@ -497,8 +535,11 @@ void UserValue::extendDef(SlotIndex Idx, unsigned LocNo,
       continue;
     const std::vector<MachineDomTreeNode*> &Children =
       MDT.getNode(MBB)->getChildren();
-    for (unsigned i = 0, e = Children.size(); i != e; ++i)
-      Todo.push_back(LIS.getMBBStartIdx(Children[i]->getBlock()));
+    for (unsigned i = 0, e = Children.size(); i != e; ++i) {
+      MachineBasicBlock *MBB = Children[i]->getBlock();
+      if (UVS.dominates(MBB))
+        Todo.push_back(LIS.getMBBStartIdx(MBB));
+    }
   } while (!Todo.empty());
 }
 
@@ -578,7 +619,8 @@ UserValue::addDefsFromCopies(LiveInterval *LI, unsigned LocNo,
 void
 UserValue::computeIntervals(MachineRegisterInfo &MRI,
                             LiveIntervals &LIS,
-                            MachineDominatorTree &MDT) {
+                            MachineDominatorTree &MDT,
+			    UserValueScopes &UVS) {
   SmallVector<std::pair<SlotIndex, unsigned>, 16> Defs;
 
   // Collect all defs to be extended (Skipping undefs).
@@ -597,10 +639,10 @@ UserValue::computeIntervals(MachineRegisterInfo &MRI,
       LiveInterval *LI = &LIS.getInterval(Loc.getReg());
       const VNInfo *VNI = LI->getVNInfoAt(Idx);
       SmallVector<SlotIndex, 16> Kills;
-      extendDef(Idx, LocNo, LI, VNI, &Kills, LIS, MDT);
+      extendDef(Idx, LocNo, LI, VNI, &Kills, LIS, MDT, UVS);
       addDefsFromCopies(LI, LocNo, Kills, Defs, MRI, LIS);
     } else
-      extendDef(Idx, LocNo, 0, 0, 0, LIS, MDT);
+      extendDef(Idx, LocNo, 0, 0, 0, LIS, MDT, UVS);
   }
 
   // Finally, erase all the undefs.
@@ -613,7 +655,8 @@ UserValue::computeIntervals(MachineRegisterInfo &MRI,
 
 void LDVImpl::computeIntervals() {
   for (unsigned i = 0, e = userValues.size(); i != e; ++i) {
-    userValues[i]->computeIntervals(MF->getRegInfo(), *LIS, *MDT);
+    UserValueScopes UVS(userValues[i]->getDebugLoc(), LS);
+    userValues[i]->computeIntervals(MF->getRegInfo(), *LIS, *MDT, UVS);
     userValues[i]->mapVirtRegs(this);
   }
 }
@@ -624,6 +667,7 @@ bool LDVImpl::runOnMachineFunction(MachineFunction &mf) {
   MDT = &pass.getAnalysis<MachineDominatorTree>();
   TRI = mf.getTarget().getRegisterInfo();
   clear();
+  LS.initialize(mf);
   DEBUG(dbgs() << "********** COMPUTING LIVE DEBUG VARIABLES: "
                << ((Value*)mf.getFunction())->getName()
                << " **********\n");
@@ -631,6 +675,7 @@ bool LDVImpl::runOnMachineFunction(MachineFunction &mf) {
   bool Changed = collectDebugValues(mf);
   computeIntervals();
   DEBUG(print(dbgs()));
+  LS.releaseMemory();
   return Changed;
 }
 
@@ -891,6 +936,7 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex Idx,
                                  const TargetInstrInfo &TII) {
   MachineBasicBlock::iterator I = findInsertLocation(MBB, Idx, LIS);
   MachineOperand &Loc = locations[LocNo];
+  ++NumInsertedDebugValues;
 
   // Frame index locations may require a target callback.
   if (Loc.isFI()) {
@@ -921,7 +967,6 @@ void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS,
 
     DEBUG(dbgs() << " BB#" << MBB->getNumber() << '-' << MBBEnd);
     insertDebugValue(MBB, Start, LocNo, LIS, TII);
-
     // This interval may span multiple basic blocks.
     // Insert a DBG_VALUE into each one.
     while(Stop > MBBEnd) {
diff --git a/contrib/llvm/lib/CodeGen/LiveInterval.cpp b/contrib/llvm/lib/CodeGen/LiveInterval.cpp
index cfade24b8d87..b69945aea98f 100644
--- a/contrib/llvm/lib/CodeGen/LiveInterval.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveInterval.cpp
@@ -148,7 +148,6 @@ void LiveInterval::markValNoForDeletion(VNInfo *ValNo) {
 /// remaining unused values.
 void LiveInterval::RenumberValues(LiveIntervals &lis) {
   SmallPtrSet<VNInfo*, 8> Seen;
-  bool seenPHIDef = false;
   valnos.clear();
   for (const_iterator I = begin(), E = end(); I != E; ++I) {
     VNInfo *VNI = I->valno;
@@ -157,26 +156,6 @@ void LiveInterval::RenumberValues(LiveIntervals &lis) {
     assert(!VNI->isUnused() && "Unused valno used by live range");
     VNI->id = (unsigned)valnos.size();
     valnos.push_back(VNI);
-    VNI->setHasPHIKill(false);
-    if (VNI->isPHIDef())
-      seenPHIDef = true;
-  }
-
-  // Recompute phi kill flags.
-  if (!seenPHIDef)
-    return;
-  for (const_vni_iterator I = vni_begin(), E = vni_end(); I != E; ++I) {
-    VNInfo *VNI = *I;
-    if (!VNI->isPHIDef())
-      continue;
-    const MachineBasicBlock *PHIBB = lis.getMBBFromIndex(VNI->def);
-    assert(PHIBB && "No basic block for phi-def");
-    for (MachineBasicBlock::const_pred_iterator PI = PHIBB->pred_begin(),
-         PE = PHIBB->pred_end(); PI != PE; ++PI) {
-      VNInfo *KVNI = getVNInfoAt(lis.getMBBEndIdx(*PI).getPrevSlot());
-      if (KVNI)
-        KVNI->setHasPHIKill(true);
-    }
   }
 }
 
@@ -294,20 +273,20 @@ LiveInterval::addRangeFrom(LiveRange LR, iterator From) {
   return ranges.insert(it, LR);
 }
 
-/// extendInBlock - If this interval is live before UseIdx in the basic
-/// block that starts at StartIdx, extend it to be live at UseIdx and return
-/// the value. If there is no live range before UseIdx, return NULL.
-VNInfo *LiveInterval::extendInBlock(SlotIndex StartIdx, SlotIndex UseIdx) {
+/// extendInBlock - If this interval is live before Kill in the basic
+/// block that starts at StartIdx, extend it to be live up to Kill and return
+/// the value. If there is no live range before Kill, return NULL.
+VNInfo *LiveInterval::extendInBlock(SlotIndex StartIdx, SlotIndex Kill) {
   if (empty())
     return 0;
-  iterator I = std::upper_bound(begin(), end(), UseIdx);
+  iterator I = std::upper_bound(begin(), end(), Kill.getPrevSlot());
   if (I == begin())
     return 0;
   --I;
   if (I->end <= StartIdx)
     return 0;
-  if (I->end <= UseIdx)
-    extendIntervalEndTo(I, UseIdx.getNextSlot());
+  if (I->end < Kill)
+    extendIntervalEndTo(I, Kill);
   return I->valno;
 }
 
diff --git a/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp b/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
index 9257191f7fc0..b1e202a273d3 100644
--- a/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -304,8 +304,19 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
 
     // Make sure the first definition is not a partial redefinition. Add an
     // <imp-def> of the full register.
-    if (MO.getSubReg())
+    // FIXME: LiveIntervals shouldn't modify the code like this.  Whoever
+    // created the machine instruction should annotate it with <undef> flags
+    // as needed.  Then we can simply assert here.  The REG_SEQUENCE lowering
+    // is the main suspect.
+    if (MO.getSubReg()) {
       mi->addRegisterDefined(interval.reg);
+      // Mark all defs of interval.reg on this instruction as reading <undef>.
+      for (unsigned i = MOIdx, e = mi->getNumOperands(); i != e; ++i) {
+        MachineOperand &MO2 = mi->getOperand(i);
+        if (MO2.isReg() && MO2.getReg() == interval.reg && MO2.getSubReg())
+          MO2.setIsUndef();
+      }
+    }
 
     MachineInstr *CopyMI = NULL;
     if (mi->isCopyLike()) {
@@ -747,6 +758,9 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
   // Find all the values used, including PHI kills.
   SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
 
+  // Blocks that have already been added to WorkList as live-out.
+  SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
+
   // Visit all instructions reading li->reg.
   for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(li->reg);
        MachineInstr *UseMI = I.skipInstruction();) {
@@ -780,8 +794,6 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
     VNInfo *VNI = *I;
     if (VNI->isUnused())
       continue;
-    // We may eliminate PHI values, so recompute PHIKill flags.
-    VNI->setHasPHIKill(false);
     NewLI.addRange(LiveRange(VNI->def, VNI->def.getNextSlot(), VNI));
 
     // A use tied to an early-clobber def ends at the load slot and isn't caught
@@ -804,7 +816,7 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
     SlotIndex BlockStart = getMBBStartIdx(MBB);
 
     // Extend the live range for VNI to be live at Idx.
-    if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) {
+    if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx.getNextSlot())) {
       (void)ExtVNI;
       assert(ExtVNI == VNI && "Unexpected existing value number");
       // Is this a PHIDef we haven't seen before?
@@ -813,13 +825,12 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
       // The PHI is live, make sure the predecessors are live-out.
       for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
            PE = MBB->pred_end(); PI != PE; ++PI) {
+        if (!LiveOut.insert(*PI))
+          continue;
         SlotIndex Stop = getMBBEndIdx(*PI).getPrevSlot();
-        VNInfo *PVNI = li->getVNInfoAt(Stop);
         // A predecessor is not required to have a live-out value for a PHI.
-        if (PVNI) {
-          PVNI->setHasPHIKill(true);
+        if (VNInfo *PVNI = li->getVNInfoAt(Stop))
           WorkList.push_back(std::make_pair(Stop, PVNI));
-        }
       }
       continue;
     }
@@ -831,6 +842,8 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
     // Make sure VNI is live-out from the predecessors.
     for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
          PE = MBB->pred_end(); PI != PE; ++PI) {
+      if (!LiveOut.insert(*PI))
+        continue;
       SlotIndex Stop = getMBBEndIdx(*PI).getPrevSlot();
       assert(li->getVNInfoAt(Stop) == VNI && "Wrong value out of predecessor");
       WorkList.push_back(std::make_pair(Stop, VNI));
diff --git a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp
index 70003e7cc86a..110fe1e62024 100644
--- a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp
@@ -91,25 +91,6 @@ LiveIntervalUnion::print(raw_ostream &OS, const TargetRegisterInfo *TRI) const {
   OS << '\n';
 }
 
-void LiveIntervalUnion::InterferenceResult::print(raw_ostream &OS,
-                                          const TargetRegisterInfo *TRI) const {
-  OS << '[' << start() << ';' << stop() << "):"
-     << PrintReg(interference()->reg, TRI);
-}
-
-void LiveIntervalUnion::Query::print(raw_ostream &OS,
-                                     const TargetRegisterInfo *TRI) {
-  OS << "Interferences with ";
-  LiveUnion->print(OS, TRI);
-  InterferenceResult IR = firstInterference();
-  while (isInterference(IR)) {
-    OS << "  ";
-    IR.print(OS, TRI);
-    OS << '\n';
-    nextInterference(IR);
-  }
-}
-
 #ifndef NDEBUG
 // Verify the live intervals in this union and add them to the visited set.
 void LiveIntervalUnion::verify(LiveVirtRegBitSet& VisitedVRegs) {
@@ -118,114 +99,6 @@ void LiveIntervalUnion::verify(LiveVirtRegBitSet& VisitedVRegs) {
 }
 #endif //!NDEBUG
 
-// Private interface accessed by Query.
-//
-// Find a pair of segments that intersect, one in the live virtual register
-// (LiveInterval), and the other in this LiveIntervalUnion. The caller (Query)
-// is responsible for advancing the LiveIntervalUnion segments to find a
-// "notable" intersection, which requires query-specific logic.
-//
-// This design assumes only a fast mechanism for intersecting a single live
-// virtual register segment with a set of LiveIntervalUnion segments.  This may
-// be ok since most virtual registers have very few segments.  If we had a data
-// structure that optimizd MxN intersection of segments, then we would bypass
-// the loop that advances within the LiveInterval.
-//
-// If no intersection exists, set VirtRegI = VirtRegEnd, and set SI to the first
-// segment whose start point is greater than LiveInterval's end point.
-//
-// Assumes that segments are sorted by start position in both
-// LiveInterval and LiveSegments.
-void LiveIntervalUnion::Query::findIntersection(InterferenceResult &IR) const {
-  // Search until reaching the end of the LiveUnion segments.
-  LiveInterval::iterator VirtRegEnd = VirtReg->end();
-  if (IR.VirtRegI == VirtRegEnd)
-    return;
-  while (IR.LiveUnionI.valid()) {
-    // Slowly advance the live virtual reg iterator until we surpass the next
-    // segment in LiveUnion.
-    //
-    // Note: If this is ever used for coalescing of fixed registers and we have
-    // a live vreg with thousands of segments, then change this code to use
-    // upperBound instead.
-    IR.VirtRegI = VirtReg->advanceTo(IR.VirtRegI, IR.LiveUnionI.start());
-    if (IR.VirtRegI == VirtRegEnd)
-      break; // Retain current (nonoverlapping) LiveUnionI
-
-    // VirtRegI may have advanced far beyond LiveUnionI, catch up.
-    IR.LiveUnionI.advanceTo(IR.VirtRegI->start);
-
-    // Check if no LiveUnionI exists with VirtRegI->Start < LiveUnionI.end
-    if (!IR.LiveUnionI.valid())
-      break;
-    if (IR.LiveUnionI.start() < IR.VirtRegI->end) {
-      assert(overlap(*IR.VirtRegI, IR.LiveUnionI) &&
-             "upperBound postcondition");
-      break;
-    }
-  }
-  if (!IR.LiveUnionI.valid())
-    IR.VirtRegI = VirtRegEnd;
-}
-
-// Find the first intersection, and cache interference info
-// (retain segment iterators into both VirtReg and LiveUnion).
-const LiveIntervalUnion::InterferenceResult &
-LiveIntervalUnion::Query::firstInterference() {
-  if (CheckedFirstInterference)
-    return FirstInterference;
-  CheckedFirstInterference = true;
-  InterferenceResult &IR = FirstInterference;
-  IR.LiveUnionI.setMap(LiveUnion->getMap());
-
-  // Quickly skip interference check for empty sets.
-  if (VirtReg->empty() || LiveUnion->empty()) {
-    IR.VirtRegI = VirtReg->end();
-  } else if (VirtReg->beginIndex() < LiveUnion->startIndex()) {
-    // VirtReg starts first, perform double binary search.
-    IR.VirtRegI = VirtReg->find(LiveUnion->startIndex());
-    if (IR.VirtRegI != VirtReg->end())
-      IR.LiveUnionI.find(IR.VirtRegI->start);
-  } else {
-    // LiveUnion starts first, perform double binary search.
-    IR.LiveUnionI.find(VirtReg->beginIndex());
-    if (IR.LiveUnionI.valid())
-      IR.VirtRegI = VirtReg->find(IR.LiveUnionI.start());
-    else
-      IR.VirtRegI = VirtReg->end();
-  }
-  findIntersection(FirstInterference);
-  assert((IR.VirtRegI == VirtReg->end() || IR.LiveUnionI.valid())
-         && "Uninitialized iterator");
-  return FirstInterference;
-}
-
-// Treat the result as an iterator and advance to the next interfering pair
-// of segments. This is a plain iterator with no filter.
-bool LiveIntervalUnion::Query::nextInterference(InterferenceResult &IR) const {
-  assert(isInterference(IR) && "iteration past end of interferences");
-
-  // Advance either the VirtReg or LiveUnion segment to ensure that we visit all
-  // unique overlapping pairs.
-  if (IR.VirtRegI->end < IR.LiveUnionI.stop()) {
-    if (++IR.VirtRegI == VirtReg->end())
-      return false;
-  }
-  else {
-    if (!(++IR.LiveUnionI).valid()) {
-      IR.VirtRegI = VirtReg->end();
-      return false;
-    }
-  }
-  // Short-circuit findIntersection() if possible.
-  if (overlap(*IR.VirtRegI, IR.LiveUnionI))
-    return true;
-
-  // Find the next intersection.
-  findIntersection(IR);
-  return isInterference(IR);
-}
-
 // Scan the vector of interfering virtual registers in this union. Assume it's
 // quite small.
 bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const {
@@ -234,64 +107,75 @@ bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const {
   return I != InterferingVRegs.end();
 }
 
-// Count the number of virtual registers in this union that interfere with this
+// Collect virtual registers in this union that interfere with this
 // query's live virtual register.
 //
-// The number of times that we either advance IR.VirtRegI or call
-// LiveUnion.upperBound() will be no more than the number of holes in
-// VirtReg. So each invocation of collectInterferingVRegs() takes
-// time proportional to |VirtReg Holes| * time(LiveUnion.upperBound()).
+// The query state is one of:
+//
+// 1. CheckedFirstInterference == false: Iterators are uninitialized.
+// 2. SeenAllInterferences == true: InterferingVRegs complete, iterators unused.
+// 3. Iterators left at the last seen intersection.
 //
-// For comments on how to speed it up, see Query::findIntersection().
 unsigned LiveIntervalUnion::Query::
 collectInterferingVRegs(unsigned MaxInterferingRegs) {
-  InterferenceResult IR = firstInterference();
-  LiveInterval::iterator VirtRegEnd = VirtReg->end();
-  LiveInterval *RecentInterferingVReg = NULL;
-  if (IR.VirtRegI != VirtRegEnd) while (IR.LiveUnionI.valid()) {
-    // Advance the union's iterator to reach an unseen interfering vreg.
-    do {
-      if (IR.LiveUnionI.value() == RecentInterferingVReg)
-        continue;
+  // Fast path return if we already have the desired information.
+  if (SeenAllInterferences || InterferingVRegs.size() >= MaxInterferingRegs)
+    return InterferingVRegs.size();
 
-      if (!isSeenInterference(IR.LiveUnionI.value()))
-        break;
+  // Set up iterators on the first call.
+  if (!CheckedFirstInterference) {
+    CheckedFirstInterference = true;
 
-      // Cache the most recent interfering vreg to bypass isSeenInterference.
-      RecentInterferingVReg = IR.LiveUnionI.value();
-
-    } while ((++IR.LiveUnionI).valid());
-    if (!IR.LiveUnionI.valid())
-      break;
-
-    // Advance the VirtReg iterator until surpassing the next segment in
-    // LiveUnion.
-    IR.VirtRegI = VirtReg->advanceTo(IR.VirtRegI, IR.LiveUnionI.start());
-    if (IR.VirtRegI == VirtRegEnd)
-      break;
-
-    // Check for intersection with the union's segment.
-    if (overlap(*IR.VirtRegI, IR.LiveUnionI)) {
-
-      if (!IR.LiveUnionI.value()->isSpillable())
-        SeenUnspillableVReg = true;
-
-      if (InterferingVRegs.size() == MaxInterferingRegs)
-        // Leave SeenAllInterferences set to false to indicate that at least one
-        // interference exists beyond those we collected.
-        return MaxInterferingRegs;
-
-      InterferingVRegs.push_back(IR.LiveUnionI.value());
-
-      // Cache the most recent interfering vreg to bypass isSeenInterference.
-      RecentInterferingVReg = IR.LiveUnionI.value();
-      ++IR.LiveUnionI;
-
-      continue;
+    // Quickly skip interference check for empty sets.
+    if (VirtReg->empty() || LiveUnion->empty()) {
+      SeenAllInterferences = true;
+      return 0;
     }
-    // VirtRegI may have advanced far beyond LiveUnionI,
-    // do a fast intersection test to "catch up"
-    IR.LiveUnionI.advanceTo(IR.VirtRegI->start);
+
+    // In most cases, the union will start before VirtReg.
+    VirtRegI = VirtReg->begin();
+    LiveUnionI.setMap(LiveUnion->getMap());
+    LiveUnionI.find(VirtRegI->start);
+  }
+
+  LiveInterval::iterator VirtRegEnd = VirtReg->end();
+  LiveInterval *RecentReg = 0;
+  while (LiveUnionI.valid()) {
+    assert(VirtRegI != VirtRegEnd && "Reached end of VirtReg");
+
+    // Check for overlapping interference.
+    while (VirtRegI->start < LiveUnionI.stop() &&
+           VirtRegI->end > LiveUnionI.start()) {
+      // This is an overlap, record the interfering register.
+      LiveInterval *VReg = LiveUnionI.value();
+      if (VReg != RecentReg && !isSeenInterference(VReg)) {
+        RecentReg = VReg;
+        InterferingVRegs.push_back(VReg);
+        if (InterferingVRegs.size() >= MaxInterferingRegs)
+          return InterferingVRegs.size();
+      }
+      // This LiveUnion segment is no longer interesting.
+      if (!(++LiveUnionI).valid()) {
+        SeenAllInterferences = true;
+        return InterferingVRegs.size();
+      }
+    }
+
+    // The iterators are now not overlapping, LiveUnionI has been advanced
+    // beyond VirtRegI.
+    assert(VirtRegI->end <= LiveUnionI.start() && "Expected non-overlap");
+
+    // Advance the iterator that ends first.
+    VirtRegI = VirtReg->advanceTo(VirtRegI, LiveUnionI.start());
+    if (VirtRegI == VirtRegEnd)
+      break;
+
+    // Detect overlap, handle above.
+    if (VirtRegI->start < LiveUnionI.stop())
+      continue;
+
+    // Still not overlapping. Catch up LiveUnionI.
+    LiveUnionI.advanceTo(VirtRegI->start);
   }
   SeenAllInterferences = true;
   return InterferingVRegs.size();
diff --git a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h
index 5e78d5e85029..5d64d285f39a 100644
--- a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h
+++ b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h
@@ -59,7 +59,6 @@ class LiveIntervalUnion {
   // LiveIntervalUnions share an external allocator.
   typedef LiveSegments::Allocator Allocator;
 
-  class InterferenceResult;
   class Query;
 
 private:
@@ -106,62 +105,13 @@ class LiveIntervalUnion {
   void verify(LiveVirtRegBitSet& VisitedVRegs);
 #endif
 
-  /// Cache a single interference test result in the form of two intersecting
-  /// segments. This allows efficiently iterating over the interferences. The
-  /// iteration logic is handled by LiveIntervalUnion::Query which may
-  /// filter interferences depending on the type of query.
-  class InterferenceResult {
-    friend class Query;
-
-    LiveInterval::iterator VirtRegI; // current position in VirtReg
-    SegmentIter LiveUnionI;          // current position in LiveUnion
-
-    // Internal ctor.
-    InterferenceResult(LiveInterval::iterator VRegI, SegmentIter UnionI)
-      : VirtRegI(VRegI), LiveUnionI(UnionI) {}
-
-  public:
-    // Public default ctor.
-    InterferenceResult(): VirtRegI(), LiveUnionI() {}
-
-    /// start - Return the start of the current overlap.
-    SlotIndex start() const {
-      return std::max(VirtRegI->start, LiveUnionI.start());
-    }
-
-    /// stop - Return the end of the current overlap.
-    SlotIndex stop() const {
-      return std::min(VirtRegI->end, LiveUnionI.stop());
-    }
-
-    /// interference - Return the register that is interfering here.
-    LiveInterval *interference() const { return LiveUnionI.value(); }
-
-    // Note: this interface provides raw access to the iterators because the
-    // result has no way to tell if it's valid to dereference them.
-
-    // Access the VirtReg segment.
-    LiveInterval::iterator virtRegPos() const { return VirtRegI; }
-
-    // Access the LiveUnion segment.
-    const SegmentIter &liveUnionPos() const { return LiveUnionI; }
-
-    bool operator==(const InterferenceResult &IR) const {
-      return VirtRegI == IR.VirtRegI && LiveUnionI == IR.LiveUnionI;
-    }
-    bool operator!=(const InterferenceResult &IR) const {
-      return !operator==(IR);
-    }
-
-    void print(raw_ostream &OS, const TargetRegisterInfo *TRI) const;
-  };
-
   /// Query interferences between a single live virtual register and a live
   /// interval union.
   class Query {
     LiveIntervalUnion *LiveUnion;
     LiveInterval *VirtReg;
-    InterferenceResult FirstInterference;
+    LiveInterval::iterator VirtRegI; // current position in VirtReg
+    SegmentIter LiveUnionI;          // current position in LiveUnion
     SmallVector<LiveInterval*,4> InterferingVRegs;
     bool CheckedFirstInterference;
     bool SeenAllInterferences;
@@ -206,26 +156,8 @@ class LiveIntervalUnion {
       return *VirtReg;
     }
 
-    bool isInterference(const InterferenceResult &IR) const {
-      if (IR.VirtRegI != VirtReg->end()) {
-        assert(overlap(*IR.VirtRegI, IR.LiveUnionI) &&
-               "invalid segment iterators");
-        return true;
-      }
-      return false;
-    }
-
     // Does this live virtual register interfere with the union?
-    bool checkInterference() { return isInterference(firstInterference()); }
-
-    // Get the first pair of interfering segments, or a noninterfering result.
-    // This initializes the firstInterference_ cache.
-    const InterferenceResult &firstInterference();
-
-    // Treat the result as an iterator and advance to the next interfering pair
-    // of segments. Visiting each unique interfering pairs means that the same
-    // VirtReg or LiveUnion segment may be visited multiple times.
-    bool nextInterference(InterferenceResult &IR) const;
+    bool checkInterference() { return collectInterferingVRegs(1); }
 
     // Count the virtual registers in this union that interfere with this
     // query's live virtual register, up to maxInterferingRegs.
@@ -249,13 +181,9 @@ class LiveIntervalUnion {
     /// Loop.
     bool checkLoopInterference(MachineLoopRange*);
 
-    void print(raw_ostream &OS, const TargetRegisterInfo *TRI);
   private:
     Query(const Query&);          // DO NOT IMPLEMENT
     void operator=(const Query&); // DO NOT IMPLEMENT
-
-    // Private interface for queries
-    void findIntersection(InterferenceResult &IR) const;
   };
 };
 
diff --git a/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp b/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp
new file mode 100644
index 000000000000..a7d5af5198e5
--- /dev/null
+++ b/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp
@@ -0,0 +1,270 @@
+//===---- LiveRangeCalc.cpp - Calculate live ranges -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of the LiveRangeCalc class.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "regalloc"
+#include "LiveRangeCalc.h"
+#include "llvm/CodeGen/MachineDominators.h"
+
+using namespace llvm;
+
+void LiveRangeCalc::reset(const MachineFunction *MF) {
+  unsigned N = MF->getNumBlockIDs();
+  Seen.clear();
+  Seen.resize(N);
+  LiveOut.resize(N);
+  LiveIn.clear();
+}
+
+
+// Transfer information from the LiveIn vector to the live ranges.
+void LiveRangeCalc::updateLiveIns(VNInfo *OverrideVNI, SlotIndexes *Indexes) {
+  for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(),
+         E = LiveIn.end(); I != E; ++I) {
+    if (!I->DomNode)
+      continue;
+    MachineBasicBlock *MBB = I->DomNode->getBlock();
+
+    VNInfo *VNI = OverrideVNI ? OverrideVNI : I->Value;
+    assert(VNI && "No live-in value found");
+
+    SlotIndex Start, End;
+    tie(Start, End) = Indexes->getMBBRange(MBB);
+
+    if (I->Kill.isValid())
+      I->LI->addRange(LiveRange(Start, I->Kill, VNI));
+    else {
+      I->LI->addRange(LiveRange(Start, End, VNI));
+      // The value is live-through, update LiveOut as well.  Defer the Domtree
+      // lookup until it is needed.
+      assert(Seen.test(MBB->getNumber()));
+      LiveOut[MBB] = LiveOutPair(VNI, (MachineDomTreeNode *)0);
+    }
+  }
+  LiveIn.clear();
+}
+
+
+void LiveRangeCalc::extend(LiveInterval *LI,
+                           SlotIndex Kill,
+                           SlotIndexes *Indexes,
+                           MachineDominatorTree *DomTree,
+                           VNInfo::Allocator *Alloc) {
+  assert(LI && "Missing live range");
+  assert(Kill.isValid() && "Invalid SlotIndex");
+  assert(Indexes && "Missing SlotIndexes");
+  assert(DomTree && "Missing dominator tree");
+
+  MachineBasicBlock *KillMBB = Indexes->getMBBFromIndex(Kill.getPrevSlot());
+  assert(Kill && "No MBB at Kill");
+
+  // Is there a def in the same MBB we can extend?
+  if (LI->extendInBlock(Indexes->getMBBStartIdx(KillMBB), Kill))
+    return;
+
+  // Find the single reaching def, or determine if Kill is jointly dominated by
+  // multiple values, and we may need to create even more phi-defs to preserve
+  // VNInfo SSA form.  Perform a search for all predecessor blocks where we
+  // know the dominating VNInfo.
+  VNInfo *VNI = findReachingDefs(LI, KillMBB, Kill, Indexes, DomTree);
+
+  // When there were multiple different values, we may need new PHIs.
+  if (!VNI)
+    updateSSA(Indexes, DomTree, Alloc);
+
+  updateLiveIns(VNI, Indexes);
+}
+
+
+// This function is called by a client after using the low-level API to add
+// live-out and live-in blocks.  The unique value optimization is not
+// available, SplitEditor::transferValues handles that case directly anyway.
+void LiveRangeCalc::calculateValues(SlotIndexes *Indexes,
+                                    MachineDominatorTree *DomTree,
+                                    VNInfo::Allocator *Alloc) {
+  assert(Indexes && "Missing SlotIndexes");
+  assert(DomTree && "Missing dominator tree");
+  updateSSA(Indexes, DomTree, Alloc);
+  updateLiveIns(0, Indexes);
+}
+
+
+VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI,
+                                        MachineBasicBlock *KillMBB,
+                                        SlotIndex Kill,
+                                        SlotIndexes *Indexes,
+                                        MachineDominatorTree *DomTree) {
+  // Blocks where LI should be live-in.
+  SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB);
+
+  // Remember if we have seen more than one value.
+  bool UniqueVNI = true;
+  VNInfo *TheVNI = 0;
+
+  // Using Seen as a visited set, perform a BFS for all reaching defs.
+  for (unsigned i = 0; i != WorkList.size(); ++i) {
+    MachineBasicBlock *MBB = WorkList[i];
+    assert(!MBB->pred_empty() && "Value live-in to entry block?");
+    for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
+           PE = MBB->pred_end(); PI != PE; ++PI) {
+       MachineBasicBlock *Pred = *PI;
+
+       // Is this a known live-out block?
+       if (Seen.test(Pred->getNumber())) {
+         if (VNInfo *VNI = LiveOut[Pred].first) {
+           if (TheVNI && TheVNI != VNI)
+             UniqueVNI = false;
+           TheVNI = VNI;
+         }
+         continue;
+       }
+
+       SlotIndex Start, End;
+       tie(Start, End) = Indexes->getMBBRange(Pred);
+
+       // First time we see Pred.  Try to determine the live-out value, but set
+       // it as null if Pred is live-through with an unknown value.
+       VNInfo *VNI = LI->extendInBlock(Start, End);
+       setLiveOutValue(Pred, VNI);
+       if (VNI) {
+         if (TheVNI && TheVNI != VNI)
+           UniqueVNI = false;
+         TheVNI = VNI;
+         continue;
+       }
+
+       // No, we need a live-in value for Pred as well
+       if (Pred != KillMBB)
+          WorkList.push_back(Pred);
+       else
+          // Loopback to KillMBB, so value is really live through.
+         Kill = SlotIndex();
+    }
+  }
+
+  // Transfer WorkList to LiveInBlocks in reverse order.
+  // This ordering works best with updateSSA().
+  LiveIn.clear();
+  LiveIn.reserve(WorkList.size());
+  while(!WorkList.empty())
+    addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val()));
+
+  // The kill block may not be live-through.
+  assert(LiveIn.back().DomNode->getBlock() == KillMBB);
+  LiveIn.back().Kill = Kill;
+
+  return UniqueVNI ? TheVNI : 0;
+}
+
+
+// This is essentially the same iterative algorithm that SSAUpdater uses,
+// except we already have a dominator tree, so we don't have to recompute it.
+void LiveRangeCalc::updateSSA(SlotIndexes *Indexes,
+                              MachineDominatorTree *DomTree,
+                              VNInfo::Allocator *Alloc) {
+  assert(Indexes && "Missing SlotIndexes");
+  assert(DomTree && "Missing dominator tree");
+
+  // Interate until convergence.
+  unsigned Changes;
+  do {
+    Changes = 0;
+    // Propagate live-out values down the dominator tree, inserting phi-defs
+    // when necessary.
+    for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(),
+           E = LiveIn.end(); I != E; ++I) {
+      MachineDomTreeNode *Node = I->DomNode;
+      // Skip block if the live-in value has already been determined.
+      if (!Node)
+        continue;
+      MachineBasicBlock *MBB = Node->getBlock();
+      MachineDomTreeNode *IDom = Node->getIDom();
+      LiveOutPair IDomValue;
+
+      // We need a live-in value to a block with no immediate dominator?
+      // This is probably an unreachable block that has survived somehow.
+      bool needPHI = !IDom || !Seen.test(IDom->getBlock()->getNumber());
+
+      // IDom dominates all of our predecessors, but it may not be their
+      // immediate dominator. Check if any of them have live-out values that are
+      // properly dominated by IDom. If so, we need a phi-def here.
+      if (!needPHI) {
+        IDomValue = LiveOut[IDom->getBlock()];
+
+        // Cache the DomTree node that defined the value.
+        if (IDomValue.first && !IDomValue.second)
+          LiveOut[IDom->getBlock()].second = IDomValue.second =
+            DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def));
+
+        for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
+               PE = MBB->pred_end(); PI != PE; ++PI) {
+          LiveOutPair &Value = LiveOut[*PI];
+          if (!Value.first || Value.first == IDomValue.first)
+            continue;
+
+          // Cache the DomTree node that defined the value.
+          if (!Value.second)
+            Value.second =
+              DomTree->getNode(Indexes->getMBBFromIndex(Value.first->def));
+
+          // This predecessor is carrying something other than IDomValue.
+          // It could be because IDomValue hasn't propagated yet, or it could be
+          // because MBB is in the dominance frontier of that value.
+          if (DomTree->dominates(IDom, Value.second)) {
+            needPHI = true;
+            break;
+          }
+        }
+      }
+
+      // The value may be live-through even if Kill is set, as can happen when
+      // we are called from extendRange. In that case LiveOutSeen is true, and
+      // LiveOut indicates a foreign or missing value.
+      LiveOutPair &LOP = LiveOut[MBB];
+
+      // Create a phi-def if required.
+      if (needPHI) {
+        ++Changes;
+        assert(Alloc && "Need VNInfo allocator to create PHI-defs");
+        SlotIndex Start, End;
+        tie(Start, End) = Indexes->getMBBRange(MBB);
+        VNInfo *VNI = I->LI->getNextValue(Start, 0, *Alloc);
+        VNI->setIsPHIDef(true);
+        I->Value = VNI;
+        // This block is done, we know the final value.
+        I->DomNode = 0;
+
+        // Add liveness since updateLiveIns now skips this node.
+        if (I->Kill.isValid())
+          I->LI->addRange(LiveRange(Start, I->Kill, VNI));
+        else {
+          I->LI->addRange(LiveRange(Start, End, VNI));
+          LOP = LiveOutPair(VNI, Node);
+        }
+      } else if (IDomValue.first) {
+        // No phi-def here. Remember incoming value.
+        I->Value = IDomValue.first;
+
+        // If the IDomValue is killed in the block, don't propagate through.
+        if (I->Kill.isValid())
+          continue;
+
+        // Propagate IDomValue if it isn't killed:
+        // MBB is live-out and doesn't define its own value.
+        if (LOP.first == IDomValue.first)
+          continue;
+        ++Changes;
+        LOP = IDomValue;
+      }
+    }
+  } while (Changes);
+}
diff --git a/contrib/llvm/lib/CodeGen/LiveRangeCalc.h b/contrib/llvm/lib/CodeGen/LiveRangeCalc.h
new file mode 100644
index 000000000000..b8c8585846fa
--- /dev/null
+++ b/contrib/llvm/lib/CodeGen/LiveRangeCalc.h
@@ -0,0 +1,226 @@
+//===---- LiveRangeCalc.h - Calculate live ranges ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The LiveRangeCalc class can be used to compute live ranges from scratch.  It
+// caches information about values in the CFG to speed up repeated operations
+// on the same live range.  The cache can be shared by non-overlapping live
+// ranges.  SplitKit uses that when computing the live range of split products.
+//
+// A low-level interface is available to clients that know where a variable is
+// live, but don't know which value it has as every point.  LiveRangeCalc will
+// propagate values down the dominator tree, and even insert PHI-defs where
+// needed.  SplitKit uses this faster interface when possible.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_LIVERANGECALC_H
+#define LLVM_CODEGEN_LIVERANGECALC_H
+
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/CodeGen/LiveInterval.h"
+
+namespace llvm {
+
+/// Forward declarations for MachineDominators.h:
+class MachineDominatorTree;
+template <class NodeT> class DomTreeNodeBase;
+typedef DomTreeNodeBase<MachineBasicBlock> MachineDomTreeNode;
+
+class LiveRangeCalc {
+  /// Seen - Bit vector of active entries in LiveOut, also used as a visited
+  /// set by findReachingDefs.  One entry per basic block, indexed by block
+  /// number.  This is kept as a separate bit vector because it can be cleared
+  /// quickly when switching live ranges.
+  BitVector Seen;
+
+  /// LiveOutPair - A value and the block that defined it.  The domtree node is
+  /// redundant, it can be computed as: MDT[Indexes.getMBBFromIndex(VNI->def)].
+  typedef std::pair<VNInfo*, MachineDomTreeNode*> LiveOutPair;
+
+  /// LiveOutMap - Map basic blocks to the value leaving the block.
+  typedef IndexedMap<LiveOutPair, MBB2NumberFunctor> LiveOutMap;
+
+  /// LiveOut - Map each basic block where a live range is live out to the
+  /// live-out value and its defining block.
+  ///
+  /// For every basic block, MBB, one of these conditions shall be true:
+  ///
+  ///  1. !Seen.count(MBB->getNumber())
+  ///     Blocks without a Seen bit are ignored.
+  ///  2. LiveOut[MBB].second.getNode() == MBB
+  ///     The live-out value is defined in MBB.
+  ///  3. forall P in preds(MBB): LiveOut[P] == LiveOut[MBB]
+  ///     The live-out value passses through MBB. All predecessors must carry
+  ///     the same value.
+  ///
+  /// The domtree node may be null, it can be computed.
+  ///
+  /// The map can be shared by multiple live ranges as long as no two are
+  /// live-out of the same block.
+  LiveOutMap LiveOut;
+
+  /// LiveInBlock - Information about a basic block where a live range is known
+  /// to be live-in, but the value has not yet been determined.
+  struct LiveInBlock {
+    // LI - The live range that is live-in to this block.  The algorithms can
+    // handle multiple non-overlapping live ranges simultaneously.
+    LiveInterval *LI;
+
+    // DomNode - Dominator tree node for the block.
+    // Cleared when the final value has been determined and LI has been updated.
+    MachineDomTreeNode *DomNode;
+
+    // Position in block where the live-in range ends, or SlotIndex() if the
+    // range passes through the block.  When the final value has been
+    // determined, the range from the block start to Kill will be added to LI.
+    SlotIndex Kill;
+
+    // Live-in value filled in by updateSSA once it is known.
+    VNInfo *Value;
+
+    LiveInBlock(LiveInterval *li, MachineDomTreeNode *node, SlotIndex kill)
+      : LI(li), DomNode(node), Kill(kill), Value(0) {}
+  };
+
+  /// LiveIn - Work list of blocks where the live-in value has yet to be
+  /// determined.  This list is typically computed by findReachingDefs() and
+  /// used as a work list by updateSSA().  The low-level interface may also be
+  /// used to add entries directly.
+  SmallVector<LiveInBlock, 16> LiveIn;
+
+  /// findReachingDefs - Assuming that LI is live-in to KillMBB and killed at
+  /// Kill, search for values that can reach KillMBB.  All blocks that need LI
+  /// to be live-in are added to LiveIn.  If a unique reaching def is found,
+  /// its value is returned, if Kill is jointly dominated by multiple values,
+  /// NULL is returned.
+  VNInfo *findReachingDefs(LiveInterval *LI,
+                           MachineBasicBlock *KillMBB,
+                           SlotIndex Kill,
+                           SlotIndexes *Indexes,
+                           MachineDominatorTree *DomTree);
+
+  /// updateSSA - Compute the values that will be live in to all requested
+  /// blocks in LiveIn.  Create PHI-def values as required to preserve SSA form.
+  ///
+  /// Every live-in block must be jointly dominated by the added live-out
+  /// blocks.  No values are read from the live ranges.
+  void updateSSA(SlotIndexes *Indexes,
+                 MachineDominatorTree *DomTree,
+                 VNInfo::Allocator *Alloc);
+
+  /// updateLiveIns - Add liveness as specified in the LiveIn vector, using VNI
+  /// as a wildcard value for LiveIn entries without a value.
+  void updateLiveIns(VNInfo *VNI, SlotIndexes*);
+
+public:
+  //===--------------------------------------------------------------------===//
+  // High-level interface.
+  //===--------------------------------------------------------------------===//
+  //
+  // Calculate live ranges from scratch.
+  //
+
+  /// reset - Prepare caches for a new set of non-overlapping live ranges.  The
+  /// caches must be reset before attempting calculations with a live range
+  /// that may overlap a previously computed live range, and before the first
+  /// live range in a function.  If live ranges are not known to be
+  /// non-overlapping, call reset before each.
+  void reset(const MachineFunction *MF);
+
+  /// calculate - Calculate the live range of a virtual register from its defs
+  /// and uses.  LI must be empty with no values.
+  void calculate(LiveInterval *LI,
+                 MachineRegisterInfo *MRI,
+                 SlotIndexes *Indexes,
+                 VNInfo::Allocator *Alloc);
+
+  //===--------------------------------------------------------------------===//
+  // Mid-level interface.
+  //===--------------------------------------------------------------------===//
+  //
+  // Modify existing live ranges.
+  //
+
+  /// extend - Extend the live range of LI to reach Kill.
+  ///
+  /// The existing values in LI must be live so they jointly dominate Kill.  If
+  /// Kill is not dominated by a single existing value, PHI-defs are inserted
+  /// as required to preserve SSA form.  If Kill is known to be dominated by a
+  /// single existing value, Alloc may be null.
+  void extend(LiveInterval *LI,
+              SlotIndex Kill,
+              SlotIndexes *Indexes,
+              MachineDominatorTree *DomTree,
+              VNInfo::Allocator *Alloc);
+
+  /// extendToUses - Extend the live range of LI to reach all uses.
+  ///
+  /// All uses must be jointly dominated by existing liveness.  PHI-defs are
+  /// inserted as needed to preserve SSA form.
+  void extendToUses(LiveInterval *LI,
+                    MachineRegisterInfo *MRI,
+                    SlotIndexes *Indexes,
+                    MachineDominatorTree *DomTree,
+                    VNInfo::Allocator *Alloc);
+
+  //===--------------------------------------------------------------------===//
+  // Low-level interface.
+  //===--------------------------------------------------------------------===//
+  //
+  // These functions can be used to compute live ranges where the live-in and
+  // live-out blocks are already known, but the SSA value in each block is
+  // unknown.
+  //
+  // After calling reset(), add known live-out values and known live-in blocks.
+  // Then call calculateValues() to compute the actual value that is
+  // live-in to each block, and add liveness to the live ranges.
+  //
+
+  /// setLiveOutValue - Indicate that VNI is live out from MBB.  The
+  /// calculateValues() function will not add liveness for MBB, the caller
+  /// should take care of that.
+  ///
+  /// VNI may be null only if MBB is a live-through block also passed to
+  /// addLiveInBlock().
+  void setLiveOutValue(MachineBasicBlock *MBB, VNInfo *VNI) {
+    Seen.set(MBB->getNumber());
+    LiveOut[MBB] = LiveOutPair(VNI, (MachineDomTreeNode *)0);
+  }
+
+  /// addLiveInBlock - Add a block with an unknown live-in value.  This
+  /// function can only be called once per basic block.  Once the live-in value
+  /// has been determined, calculateValues() will add liveness to LI.
+  ///
+  /// @param LI      The live range that is live-in to the block.
+  /// @param DomNode The domtree node for the block.
+  /// @param Kill    Index in block where LI is killed.  If the value is
+  ///                live-through, set Kill = SLotIndex() and also call
+  ///                setLiveOutValue(MBB, 0).
+  void addLiveInBlock(LiveInterval *LI,
+                      MachineDomTreeNode *DomNode,
+                      SlotIndex Kill = SlotIndex()) {
+    LiveIn.push_back(LiveInBlock(LI, DomNode, Kill));
+  }
+
+  /// calculateValues - Calculate the value that will be live-in to each block
+  /// added with addLiveInBlock.  Add PHI-def values as needed to preserve SSA
+  /// form.  Add liveness to all live-in blocks up to the Kill point, or the
+  /// whole block for live-through blocks.
+  ///
+  /// Every predecessor of a live-in block must have been given a value with
+  /// setLiveOutValue, the value may be null for live-trough blocks.
+  void calculateValues(SlotIndexes *Indexes,
+                       MachineDominatorTree *DomTree,
+                       VNInfo::Allocator *Alloc);
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp b/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp
index b385fb36bbf1..b23f85165360 100644
--- a/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -319,9 +319,12 @@ void LiveRangeEdit::calculateRegClassAndHint(MachineFunction &MF,
                                              LiveIntervals &LIS,
                                              const MachineLoopInfo &Loops) {
   VirtRegAuxInfo VRAI(MF, LIS, Loops);
+  MachineRegisterInfo &MRI = MF.getRegInfo();
   for (iterator I = begin(), E = end(); I != E; ++I) {
     LiveInterval &LI = **I;
-    VRAI.CalculateRegClass(LI.reg);
+    if (MRI.recomputeRegClass(LI.reg, MF.getTarget()))
+      DEBUG(dbgs() << "Inflated " << PrintReg(LI.reg) << " to "
+                   << MRI.getRegClass(LI.reg)->getName() << '\n');
     VRAI.CalculateWeightAndHint(LI);
   }
 }
diff --git a/contrib/llvm/lib/CodeGen/LiveRangeEdit.h b/contrib/llvm/lib/CodeGen/LiveRangeEdit.h
index db6740c1fbcd..9b0a671ea9e5 100644
--- a/contrib/llvm/lib/CodeGen/LiveRangeEdit.h
+++ b/contrib/llvm/lib/CodeGen/LiveRangeEdit.h
@@ -115,7 +115,7 @@ class LiveRangeEdit {
   LiveInterval *get(unsigned idx) const { return newRegs_[idx+firstNew_]; }
 
   ArrayRef<LiveInterval*> regs() const {
-    return ArrayRef<LiveInterval*>(newRegs_).slice(firstNew_);
+    return makeArrayRef(newRegs_).slice(firstNew_);
   }
 
   /// FIXME: Temporary accessors until we can get rid of
diff --git a/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp b/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp
index c75196a47210..939e795b4a38 100644
--- a/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp
@@ -44,7 +44,8 @@ void LiveStacks::releaseMemory() {
   S2RCMap.clear();
 }
 
-bool LiveStacks::runOnMachineFunction(MachineFunction &) {
+bool LiveStacks::runOnMachineFunction(MachineFunction &MF) {
+  TRI = MF.getTarget().getRegisterInfo();
   // FIXME: No analysis is being done right now. We are relying on the
   // register allocators to provide the information.
   return false;
@@ -61,7 +62,7 @@ LiveStacks::getOrCreateInterval(int Slot, const TargetRegisterClass *RC) {
   } else {
     // Use the largest common subclass register class.
     const TargetRegisterClass *OldRC = S2RCMap[Slot];
-    S2RCMap[Slot] = getCommonSubClass(OldRC, RC);
+    S2RCMap[Slot] = TRI->getCommonSubClass(OldRC, RC);
   }
   return I->second;
 }
diff --git a/contrib/llvm/lib/CodeGen/LiveVariables.cpp b/contrib/llvm/lib/CodeGen/LiveVariables.cpp
index 20bad60dedda..2ca90f9f05c0 100644
--- a/contrib/llvm/lib/CodeGen/LiveVariables.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveVariables.cpp
@@ -662,7 +662,7 @@ void LiveVariables::removeVirtualRegistersKilled(MachineInstr *MI) {
       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
         bool removed = getVarInfo(Reg).removeKill(MI);
         assert(removed && "kill not in register's VarInfo?");
-        removed = true;
+        (void)removed;
       }
     }
   }
diff --git a/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp b/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 8f0fb46879ac..4c5fe4c480a6 100644
--- a/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -571,6 +571,11 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) {
       if (i->getOperand(ni+1).getMBB() == this)
         i->getOperand(ni+1).setMBB(NMBB);
 
+  // Inherit live-ins from the successor
+  for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(),
+	 E = Succ->livein_end(); I != E; ++I)
+    NMBB->addLiveIn(*I);
+
   // Update LiveVariables.
   if (LV) {
     // Restore kills of virtual registers that were killed by the terminators.
diff --git a/contrib/llvm/lib/CodeGen/MachineBlockFrequency.cpp b/contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
similarity index 51%
rename from contrib/llvm/lib/CodeGen/MachineBlockFrequency.cpp
rename to contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 893a320a6a63..b92cda961474 100644
--- a/contrib/llvm/lib/CodeGen/MachineBlockFrequency.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -1,4 +1,4 @@
-//====----- MachineBlockFrequency.cpp - Machine Block Frequency Analysis ----====//
+//====----- MachineBlockFrequencyInfo.cpp - Machine Block Frequency Analysis ----====//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -13,47 +13,49 @@
 
 #include "llvm/InitializePasses.h"
 #include "llvm/Analysis/BlockFrequencyImpl.h"
-#include "llvm/CodeGen/MachineBlockFrequency.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
 
 using namespace llvm;
 
-INITIALIZE_PASS_BEGIN(MachineBlockFrequency, "machine-block-freq",
+INITIALIZE_PASS_BEGIN(MachineBlockFrequencyInfo, "machine-block-freq",
                       "Machine Block Frequency Analysis", true, true)
 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
-INITIALIZE_PASS_END(MachineBlockFrequency, "machine-block-freq",
+INITIALIZE_PASS_END(MachineBlockFrequencyInfo, "machine-block-freq",
                     "Machine Block Frequency Analysis", true, true)
 
-char MachineBlockFrequency::ID = 0;
+char MachineBlockFrequencyInfo::ID = 0;
 
 
-MachineBlockFrequency::MachineBlockFrequency() : MachineFunctionPass(ID) {
-  initializeMachineBlockFrequencyPass(*PassRegistry::getPassRegistry());
+MachineBlockFrequencyInfo::MachineBlockFrequencyInfo() : MachineFunctionPass(ID) {
+  initializeMachineBlockFrequencyInfoPass(*PassRegistry::getPassRegistry());
   MBFI = new BlockFrequencyImpl<MachineBasicBlock, MachineFunction,
                                 MachineBranchProbabilityInfo>();
 }
 
-MachineBlockFrequency::~MachineBlockFrequency() {
+MachineBlockFrequencyInfo::~MachineBlockFrequencyInfo() {
   delete MBFI;
 }
 
-void MachineBlockFrequency::getAnalysisUsage(AnalysisUsage &AU) const {
+void MachineBlockFrequencyInfo::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<MachineBranchProbabilityInfo>();
   AU.setPreservesAll();
+  MachineFunctionPass::getAnalysisUsage(AU);
 }
 
-bool MachineBlockFrequency::runOnMachineFunction(MachineFunction &F) {
+bool MachineBlockFrequencyInfo::runOnMachineFunction(MachineFunction &F) {
   MachineBranchProbabilityInfo &MBPI = getAnalysis<MachineBranchProbabilityInfo>();
   MBFI->doFunction(&F, &MBPI);
   return false;
 }
 
-/// getblockFreq - Return block frequency. Never return 0, value must be
-/// positive. Please note that initial frequency is equal to 1024. It means that
-/// we should not rely on the value itself, but only on the comparison to the
-/// other block frequencies. We do this to avoid using of floating points.
+/// getblockFreq - Return block frequency. Return 0 if we don't have the
+/// information. Please note that initial frequency is equal to 1024. It means
+/// that we should not rely on the value itself, but only on the comparison to
+/// the other block frequencies. We do this to avoid using of floating points.
 ///
-uint32_t MachineBlockFrequency::getBlockFreq(MachineBasicBlock *MBB) {
+BlockFrequency MachineBlockFrequencyInfo::
+getBlockFreq(MachineBasicBlock *MBB) const {
   return MBFI->getBlockFreq(MBB);
 }
diff --git a/contrib/llvm/lib/CodeGen/MachineCSE.cpp b/contrib/llvm/lib/CodeGen/MachineCSE.cpp
index 3a60a37af443..7eda8c129dc4 100644
--- a/contrib/llvm/lib/CodeGen/MachineCSE.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineCSE.cpp
@@ -430,13 +430,24 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) {
       unsigned NewReg = CSMI->getOperand(i).getReg();
       if (OldReg == NewReg)
         continue;
+
       assert(TargetRegisterInfo::isVirtualRegister(OldReg) &&
              TargetRegisterInfo::isVirtualRegister(NewReg) &&
              "Do not CSE physical register defs!");
+
       if (!isProfitableToCSE(NewReg, OldReg, CSMI, MI)) {
         DoCSE = false;
         break;
       }
+
+      // Don't perform CSE if the result of the old instruction cannot exist
+      // within the register class of the new instruction.
+      const TargetRegisterClass *OldRC = MRI->getRegClass(OldReg);
+      if (!MRI->constrainRegClass(NewReg, OldRC)) {
+        DoCSE = false;
+        break;
+      }
+
       CSEPairs.push_back(std::make_pair(OldReg, NewReg));
       --NumDefs;
     }
diff --git a/contrib/llvm/lib/CodeGen/MachineFunction.cpp b/contrib/llvm/lib/CodeGen/MachineFunction.cpp
index cd2515652831..20066a067b8f 100644
--- a/contrib/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineFunction.cpp
@@ -619,7 +619,7 @@ void MachineJumpTableInfo::dump() const { print(dbgs()); }
 //  MachineConstantPool implementation
 //===----------------------------------------------------------------------===//
 
-const Type *MachineConstantPoolEntry::getType() const {
+Type *MachineConstantPoolEntry::getType() const {
   if (isMachineConstantPoolEntry())
     return Val.MachineCPVal->getType();
   return Val.ConstVal->getType();
diff --git a/contrib/llvm/lib/CodeGen/MachineInstr.cpp b/contrib/llvm/lib/CodeGen/MachineInstr.cpp
index 143a29b08a1e..a240667f7d6a 100644
--- a/contrib/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineInstr.cpp
@@ -51,7 +51,7 @@ using namespace llvm;
 /// explicitly nulled out.
 void MachineOperand::AddRegOperandToRegInfo(MachineRegisterInfo *RegInfo) {
   assert(isReg() && "Can only add reg operand to use lists");
-  
+
   // If the reginfo pointer is null, just explicitly null out or next/prev
   // pointers, to ensure they are not garbage.
   if (RegInfo == 0) {
@@ -59,23 +59,23 @@ void MachineOperand::AddRegOperandToRegInfo(MachineRegisterInfo *RegInfo) {
     Contents.Reg.Next = 0;
     return;
   }
-  
+
   // Otherwise, add this operand to the head of the registers use/def list.
   MachineOperand **Head = &RegInfo->getRegUseDefListHead(getReg());
-  
+
   // For SSA values, we prefer to keep the definition at the start of the list.
   // we do this by skipping over the definition if it is at the head of the
   // list.
   if (*Head && (*Head)->isDef())
     Head = &(*Head)->Contents.Reg.Next;
-  
+
   Contents.Reg.Next = *Head;
   if (Contents.Reg.Next) {
     assert(getReg() == Contents.Reg.Next->getReg() &&
            "Different regs on the same list!");
     Contents.Reg.Next->Contents.Reg.Prev = &Contents.Reg.Next;
   }
-  
+
   Contents.Reg.Prev = Head;
   *Head = this;
 }
@@ -86,7 +86,7 @@ void MachineOperand::RemoveRegOperandFromRegInfo() {
   assert(isOnRegUseList() && "Reg operand is not on a use list");
   // Unlink this from the doubly linked list of operands.
   MachineOperand *NextOp = Contents.Reg.Next;
-  *Contents.Reg.Prev = NextOp; 
+  *Contents.Reg.Prev = NextOp;
   if (NextOp) {
     assert(NextOp->getReg() == getReg() && "Corrupt reg use/def chain!");
     NextOp->Contents.Reg.Prev = Contents.Reg.Prev;
@@ -97,7 +97,7 @@ void MachineOperand::RemoveRegOperandFromRegInfo() {
 
 void MachineOperand::setReg(unsigned Reg) {
   if (getReg() == Reg) return; // No change.
-  
+
   // Otherwise, we have to change the register.  If this operand is embedded
   // into a machine function, we need to update the old and new register's
   // use/def lists.
@@ -109,7 +109,7 @@ void MachineOperand::setReg(unsigned Reg) {
         AddRegOperandToRegInfo(&MF->getRegInfo());
         return;
       }
-        
+
   // Otherwise, just change the register, no problem.  :)
   SmallContents.RegNo = Reg;
 }
@@ -144,7 +144,7 @@ void MachineOperand::ChangeToImmediate(int64_t ImmVal) {
   if (isReg() && getParent() && getParent()->getParent() &&
       getParent()->getParent()->getParent())
     RemoveRegOperandFromRegInfo();
-  
+
   OpKind = MO_Immediate;
   Contents.ImmVal = ImmVal;
 }
@@ -155,7 +155,7 @@ void MachineOperand::ChangeToImmediate(int64_t ImmVal) {
 void MachineOperand::ChangeToRegister(unsigned Reg, bool isDef, bool isImp,
                                       bool isKill, bool isDead, bool isUndef,
                                       bool isDebug) {
-  // If this operand is already a register operand, use setReg to update the 
+  // If this operand is already a register operand, use setReg to update the
   // register's use/def lists.
   if (isReg()) {
     assert(!isEarlyClobber());
@@ -189,7 +189,7 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
   if (getType() != Other.getType() ||
       getTargetFlags() != Other.getTargetFlags())
     return false;
-  
+
   switch (getType()) {
   default: llvm_unreachable("Unrecognized operand type");
   case MachineOperand::MO_Register:
@@ -322,7 +322,7 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const {
   default:
     llvm_unreachable("Unrecognized operand type");
   }
-  
+
   if (unsigned TF = getTargetFlags())
     OS << "[TF=" << TF << ']';
 }
@@ -408,7 +408,7 @@ uint64_t MachineMemOperand::getAlignment() const {
 raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
   assert((MMO.isLoad() || MMO.isStore()) &&
          "SV has to be a load, store or both.");
-  
+
   if (MMO.isVolatile())
     OS << "Volatile ";
 
@@ -417,7 +417,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
   if (MMO.isStore())
     OS << "ST";
   OS << MMO.getSize();
-  
+
   // Print the address information.
   OS << "[";
   if (!MMO.getValue())
@@ -464,7 +464,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
 /// MachineInstr ctor - This constructor creates a dummy MachineInstr with
 /// MCID NULL and no operands.
 MachineInstr::MachineInstr()
-  : MCID(0), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0),
+  : MCID(0), Flags(0), AsmPrinterFlags(0),
     MemRefs(0), MemRefsEnd(0),
     Parent(0) {
   // Make sure that we get added to a machine basicblock
@@ -484,8 +484,9 @@ void MachineInstr::addImplicitDefUseOperands() {
 /// implicit operands. It reserves space for the number of operands specified by
 /// the MCInstrDesc.
 MachineInstr::MachineInstr(const MCInstrDesc &tid, bool NoImp)
-  : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0),
+  : MCID(&tid), Flags(0), AsmPrinterFlags(0),
     MemRefs(0), MemRefsEnd(0), Parent(0) {
+  unsigned NumImplicitOps = 0;
   if (!NoImp)
     NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
   Operands.reserve(NumImplicitOps + MCID->getNumOperands());
@@ -498,8 +499,9 @@ MachineInstr::MachineInstr(const MCInstrDesc &tid, bool NoImp)
 /// MachineInstr ctor - As above, but with a DebugLoc.
 MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl,
                            bool NoImp)
-  : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0),
+  : MCID(&tid), Flags(0), AsmPrinterFlags(0),
     MemRefs(0), MemRefsEnd(0), Parent(0), debugLoc(dl) {
+  unsigned NumImplicitOps = 0;
   if (!NoImp)
     NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
   Operands.reserve(NumImplicitOps + MCID->getNumOperands());
@@ -510,13 +512,14 @@ MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl,
 }
 
 /// MachineInstr ctor - Work exactly the same as the ctor two above, except
-/// that the MachineInstr is created and added to the end of the specified 
+/// that the MachineInstr is created and added to the end of the specified
 /// basic block.
 MachineInstr::MachineInstr(MachineBasicBlock *MBB, const MCInstrDesc &tid)
-  : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0),
+  : MCID(&tid), Flags(0), AsmPrinterFlags(0),
     MemRefs(0), MemRefsEnd(0), Parent(0) {
   assert(MBB && "Cannot use inserting ctor with null basic block!");
-  NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
+  unsigned NumImplicitOps =
+    MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
   Operands.reserve(NumImplicitOps + MCID->getNumOperands());
   addImplicitDefUseOperands();
   // Make sure that we get added to a machine basicblock
@@ -528,10 +531,11 @@ MachineInstr::MachineInstr(MachineBasicBlock *MBB, const MCInstrDesc &tid)
 ///
 MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl,
                            const MCInstrDesc &tid)
-  : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0),
+  : MCID(&tid), Flags(0), AsmPrinterFlags(0),
     MemRefs(0), MemRefsEnd(0), Parent(0), debugLoc(dl) {
   assert(MBB && "Cannot use inserting ctor with null basic block!");
-  NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
+  unsigned NumImplicitOps =
+    MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
   Operands.reserve(NumImplicitOps + MCID->getNumOperands());
   addImplicitDefUseOperands();
   // Make sure that we get added to a machine basicblock
@@ -542,7 +546,7 @@ MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl,
 /// MachineInstr ctor - Copies MachineInstr arg exactly
 ///
 MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI)
-  : MCID(&MI.getDesc()), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0),
+  : MCID(&MI.getDesc()), Flags(0), AsmPrinterFlags(0),
     MemRefs(MI.MemRefs), MemRefsEnd(MI.MemRefsEnd),
     Parent(0), debugLoc(MI.getDebugLoc()) {
   Operands.reserve(MI.getNumOperands());
@@ -550,7 +554,6 @@ MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI)
   // Add operands
   for (unsigned i = 0; i != MI.getNumOperands(); ++i)
     addOperand(MI.getOperand(i));
-  NumImplicitOps = MI.NumImplicitOps;
 
   // Copy all the flags.
   Flags = MI.Flags;
@@ -605,102 +608,74 @@ void MachineInstr::AddRegOperandsToUseLists(MachineRegisterInfo &RegInfo) {
 /// addOperand - Add the specified operand to the instruction.  If it is an
 /// implicit operand, it is added to the end of the operand list.  If it is
 /// an explicit operand it is added at the end of the explicit operand list
-/// (before the first implicit operand). 
+/// (before the first implicit operand).
 void MachineInstr::addOperand(const MachineOperand &Op) {
+  assert(MCID && "Cannot add operands before providing an instr descriptor");
   bool isImpReg = Op.isReg() && Op.isImplicit();
-  assert((isImpReg || !OperandsComplete()) &&
-         "Trying to add an operand to a machine instr that is already done!");
-
   MachineRegisterInfo *RegInfo = getRegInfo();
 
-  // If we are adding the operand to the end of the list, our job is simpler.
-  // This is true most of the time, so this is a reasonable optimization.
-  if (isImpReg || NumImplicitOps == 0) {
-    // We can only do this optimization if we know that the operand list won't
-    // reallocate.
-    if (Operands.empty() || Operands.size()+1 <= Operands.capacity()) {
-      Operands.push_back(Op);
-    
-      // Set the parent of the operand.
-      Operands.back().ParentMI = this;
-  
-      // If the operand is a register, update the operand's use list.
-      if (Op.isReg()) {
-        Operands.back().AddRegOperandToRegInfo(RegInfo);
-        // If the register operand is flagged as early, mark the operand as such
-        unsigned OpNo = Operands.size() - 1;
-        if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
-          Operands[OpNo].setIsEarlyClobber(true);
-      }
-      return;
+  // If the Operands backing store is reallocated, all register operands must
+  // be removed and re-added to RegInfo.  It is storing pointers to operands.
+  bool Reallocate = RegInfo &&
+    !Operands.empty() && Operands.size() == Operands.capacity();
+
+  // Find the insert location for the new operand.  Implicit registers go at
+  // the end, everything goes before the implicit regs.
+  unsigned OpNo = Operands.size();
+
+  // Remove all the implicit operands from RegInfo if they need to be shifted.
+  // FIXME: Allow mixed explicit and implicit operands on inline asm.
+  // InstrEmitter::EmitSpecialNode() is marking inline asm clobbers as
+  // implicit-defs, but they must not be moved around.  See the FIXME in
+  // InstrEmitter.cpp.
+  if (!isImpReg && !isInlineAsm()) {
+    while (OpNo && Operands[OpNo-1].isReg() && Operands[OpNo-1].isImplicit()) {
+      --OpNo;
+      if (RegInfo)
+        Operands[OpNo].RemoveRegOperandFromRegInfo();
     }
   }
-  
-  // Otherwise, we have to insert a real operand before any implicit ones.
-  unsigned OpNo = Operands.size()-NumImplicitOps;
 
-  // If this instruction isn't embedded into a function, then we don't need to
-  // update any operand lists.
-  if (RegInfo == 0) {
-    // Simple insertion, no reginfo update needed for other register operands.
-    Operands.insert(Operands.begin()+OpNo, Op);
-    Operands[OpNo].ParentMI = this;
+  // OpNo now points as the desired insertion point.  Unless this is a variadic
+  // instruction, only implicit regs are allowed beyond MCID->getNumOperands().
+  assert((isImpReg || MCID->isVariadic() || OpNo < MCID->getNumOperands()) &&
+         "Trying to add an operand to a machine instr that is already done!");
 
-    // Do explicitly set the reginfo for this operand though, to ensure the
-    // next/prev fields are properly nulled out.
-    if (Operands[OpNo].isReg()) {
-      Operands[OpNo].AddRegOperandToRegInfo(0);
-      // If the register operand is flagged as early, mark the operand as such
-      if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
-        Operands[OpNo].setIsEarlyClobber(true);
-    }
+  // All operands from OpNo have been removed from RegInfo.  If the Operands
+  // backing store needs to be reallocated, we also need to remove any other
+  // register operands.
+  if (Reallocate)
+    for (unsigned i = 0; i != OpNo; ++i)
+      if (Operands[i].isReg())
+        Operands[i].RemoveRegOperandFromRegInfo();
 
-  } else if (Operands.size()+1 <= Operands.capacity()) {
-    // Otherwise, we have to remove register operands from their register use
-    // list, add the operand, then add the register operands back to their use
-    // list.  This also must handle the case when the operand list reallocates
-    // to somewhere else.
-  
-    // If insertion of this operand won't cause reallocation of the operand
-    // list, just remove the implicit operands, add the operand, then re-add all
-    // the rest of the operands.
-    for (unsigned i = OpNo, e = Operands.size(); i != e; ++i) {
-      assert(Operands[i].isReg() && "Should only be an implicit reg!");
-      Operands[i].RemoveRegOperandFromRegInfo();
-    }
-    
-    // Add the operand.  If it is a register, add it to the reg list.
-    Operands.insert(Operands.begin()+OpNo, Op);
-    Operands[OpNo].ParentMI = this;
+  // Insert the new operand at OpNo.
+  Operands.insert(Operands.begin() + OpNo, Op);
+  Operands[OpNo].ParentMI = this;
 
-    if (Operands[OpNo].isReg()) {
-      Operands[OpNo].AddRegOperandToRegInfo(RegInfo);
-      // If the register operand is flagged as early, mark the operand as such
-      if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
-        Operands[OpNo].setIsEarlyClobber(true);
-    }
-    
-    // Re-add all the implicit ops.
-    for (unsigned i = OpNo+1, e = Operands.size(); i != e; ++i) {
+  // The Operands backing store has now been reallocated, so we can re-add the
+  // operands before OpNo.
+  if (Reallocate)
+    for (unsigned i = 0; i != OpNo; ++i)
+      if (Operands[i].isReg())
+        Operands[i].AddRegOperandToRegInfo(RegInfo);
+
+  // When adding a register operand, tell RegInfo about it.
+  if (Operands[OpNo].isReg()) {
+    // Add the new operand to RegInfo, even when RegInfo is NULL.
+    // This will initialize the linked list pointers.
+    Operands[OpNo].AddRegOperandToRegInfo(RegInfo);
+    // If the register operand is flagged as early, mark the operand as such.
+    if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
+      Operands[OpNo].setIsEarlyClobber(true);
+  }
+
+  // Re-add all the implicit ops.
+  if (RegInfo) {
+    for (unsigned i = OpNo + 1, e = Operands.size(); i != e; ++i) {
       assert(Operands[i].isReg() && "Should only be an implicit reg!");
       Operands[i].AddRegOperandToRegInfo(RegInfo);
     }
-  } else {
-    // Otherwise, we will be reallocating the operand list.  Remove all reg
-    // operands from their list, then readd them after the operand list is
-    // reallocated.
-    RemoveRegOperandsFromUseLists();
-    
-    Operands.insert(Operands.begin()+OpNo, Op);
-    Operands[OpNo].ParentMI = this;
-  
-    // Re-add all the operands.
-    AddRegOperandsToUseLists(*RegInfo);
-
-      // If the register operand is flagged as early, mark the operand as such
-    if (Operands[OpNo].isReg()
-        && MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
-      Operands[OpNo].setIsEarlyClobber(true);
   }
 }
 
@@ -709,13 +684,13 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
 ///
 void MachineInstr::RemoveOperand(unsigned OpNo) {
   assert(OpNo < Operands.size() && "Invalid operand number");
-  
+
   // Special case removing the last one.
   if (OpNo == Operands.size()-1) {
     // If needed, remove from the reg def/use list.
     if (Operands.back().isReg() && Operands.back().isOnRegUseList())
       Operands.back().RemoveRegOperandFromRegInfo();
-    
+
     Operands.pop_back();
     return;
   }
@@ -730,7 +705,7 @@ void MachineInstr::RemoveOperand(unsigned OpNo) {
         Operands[i].RemoveRegOperandFromRegInfo();
     }
   }
-  
+
   Operands.erase(Operands.begin()+OpNo);
 
   if (RegInfo) {
@@ -827,15 +802,6 @@ void MachineInstr::eraseFromParent() {
 }
 
 
-/// OperandComplete - Return true if it's illegal to add a new operand
-///
-bool MachineInstr::OperandsComplete() const {
-  unsigned short NumOperands = MCID->getNumOperands();
-  if (!MCID->isVariadic() && getNumOperands()-NumImplicitOps >= NumOperands)
-    return true;  // Broken: we have all the operands of this instruction!
-  return false;
-}
-
 /// getNumExplicitOperands - Returns the number of non-implicit operands.
 ///
 unsigned MachineInstr::getNumExplicitOperands() const {
@@ -860,6 +826,67 @@ bool MachineInstr::isStackAligningInlineAsm() const {
   return false;
 }
 
+int MachineInstr::findInlineAsmFlagIdx(unsigned OpIdx,
+                                       unsigned *GroupNo) const {
+  assert(isInlineAsm() && "Expected an inline asm instruction");
+  assert(OpIdx < getNumOperands() && "OpIdx out of range");
+
+  // Ignore queries about the initial operands.
+  if (OpIdx < InlineAsm::MIOp_FirstOperand)
+    return -1;
+
+  unsigned Group = 0;
+  unsigned NumOps;
+  for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); i < e;
+       i += NumOps) {
+    const MachineOperand &FlagMO = getOperand(i);
+    // If we reach the implicit register operands, stop looking.
+    if (!FlagMO.isImm())
+      return -1;
+    NumOps = 1 + InlineAsm::getNumOperandRegisters(FlagMO.getImm());
+    if (i + NumOps > OpIdx) {
+      if (GroupNo)
+        *GroupNo = Group;
+      return i;
+    }
+    ++Group;
+  }
+  return -1;
+}
+
+const TargetRegisterClass*
+MachineInstr::getRegClassConstraint(unsigned OpIdx,
+                                    const TargetInstrInfo *TII,
+                                    const TargetRegisterInfo *TRI) const {
+  // Most opcodes have fixed constraints in their MCInstrDesc.
+  if (!isInlineAsm())
+    return TII->getRegClass(getDesc(), OpIdx, TRI);
+
+  if (!getOperand(OpIdx).isReg())
+    return NULL;
+
+  // For tied uses on inline asm, get the constraint from the def.
+  unsigned DefIdx;
+  if (getOperand(OpIdx).isUse() && isRegTiedToDefOperand(OpIdx, &DefIdx))
+    OpIdx = DefIdx;
+
+  // Inline asm stores register class constraints in the flag word.
+  int FlagIdx = findInlineAsmFlagIdx(OpIdx);
+  if (FlagIdx < 0)
+    return NULL;
+
+  unsigned Flag = getOperand(FlagIdx).getImm();
+  unsigned RCID;
+  if (InlineAsm::hasRegClassConstraint(Flag, RCID))
+    return TRI->getRegClass(RCID);
+
+  // Assume that all registers in a memory operand are pointers.
+  if (InlineAsm::getKind(Flag) == InlineAsm::Kind_Mem)
+    return TRI->getPointerRegClass();
+
+  return NULL;
+}
+
 /// findRegisterUseOperandIdx() - Returns the MachineOperand that is a use of
 /// the specific register or -1 if it is not found. It further tightens
 /// the search criteria to a use that kills the register if isKill is true.
@@ -901,7 +928,8 @@ MachineInstr::readsWritesVirtualRegister(unsigned Reg,
       Ops->push_back(i);
     if (MO.isUse())
       Use |= !MO.isUndef();
-    else if (MO.getSubReg())
+    else if (MO.getSubReg() && !MO.isUndef())
+      // A partial <def,undef> doesn't count as reading the register.
       PartDef = true;
     else
       FullDef = true;
@@ -941,6 +969,10 @@ MachineInstr::findRegisterDefOperandIdx(unsigned Reg, bool isDead, bool Overlap,
 /// operand list that is used to represent the predicate. It returns -1 if
 /// none is found.
 int MachineInstr::findFirstPredOperandIdx() const {
+  // Don't call MCID.findFirstPredOperandIdx() because this variant
+  // is sometimes called on an instruction that's not yet complete, and
+  // so the number of operands is less than the MCID indicates. In
+  // particular, the PTX target does this.
   const MCInstrDesc &MCID = getDesc();
   if (MCID.isPredicable()) {
     for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
@@ -950,7 +982,7 @@ int MachineInstr::findFirstPredOperandIdx() const {
 
   return -1;
 }
-  
+
 /// isRegTiedToUseOperand - Given the index of a register def operand,
 /// check if the register def is tied to a source operand, due to either
 /// two-address elimination or inline assembly constraints. Returns the
@@ -964,23 +996,13 @@ isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx) const {
       return false;
     // Determine the actual operand index that corresponds to this index.
     unsigned DefNo = 0;
-    unsigned DefPart = 0;
-    for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands();
-         i < e; ) {
-      const MachineOperand &FMO = getOperand(i);
-      // After the normal asm operands there may be additional imp-def regs.
-      if (!FMO.isImm())
-        return false;
-      // Skip over this def.
-      unsigned NumOps = InlineAsm::getNumOperandRegisters(FMO.getImm());
-      unsigned PrevDef = i + 1;
-      i = PrevDef + NumOps;
-      if (i > DefOpIdx) {
-        DefPart = DefOpIdx - PrevDef;
-        break;
-      }
-      ++DefNo;
-    }
+    int FlagIdx = findInlineAsmFlagIdx(DefOpIdx, &DefNo);
+    if (FlagIdx < 0)
+      return false;
+
+    // Which part of the group is DefOpIdx?
+    unsigned DefPart = DefOpIdx - (FlagIdx + 1);
+
     for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands();
          i != e; ++i) {
       const MachineOperand &FMO = getOperand(i);
@@ -1024,20 +1046,10 @@ isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx) const {
       return false;
 
     // Find the flag operand corresponding to UseOpIdx
-    unsigned FlagIdx, NumOps=0;
-    for (FlagIdx = InlineAsm::MIOp_FirstOperand;
-         FlagIdx < UseOpIdx; FlagIdx += NumOps+1) {
-      const MachineOperand &UFMO = getOperand(FlagIdx);
-      // After the normal asm operands there may be additional imp-def regs.
-      if (!UFMO.isImm())
-        return false;
-      NumOps = InlineAsm::getNumOperandRegisters(UFMO.getImm());
-      assert(NumOps < getNumOperands() && "Invalid inline asm flag");
-      if (UseOpIdx < FlagIdx+NumOps+1)
-        break;
-    }
-    if (FlagIdx >= UseOpIdx)
+    int FlagIdx = findInlineAsmFlagIdx(UseOpIdx);
+    if (FlagIdx < 0)
       return false;
+
     const MachineOperand &UFMO = getOperand(FlagIdx);
     unsigned DefNo;
     if (InlineAsm::isUseOperandTiedToDef(UFMO.getImm(), DefNo)) {
@@ -1211,7 +1223,7 @@ bool MachineInstr::hasVolatileMemoryRef() const {
   // conservatively assume it wasn't preserved.
   if (memoperands_empty())
     return true;
-  
+
   // Check the memory reference information for volatile references.
   for (mmo_iterator I = memoperands_begin(), E = memoperands_end(); I != E; ++I)
     if ((*I)->isVolatile())
@@ -1318,7 +1330,7 @@ void MachineInstr::dump() const {
   dbgs() << "  " << *this;
 }
 
-static void printDebugLoc(DebugLoc DL, const MachineFunction *MF, 
+static void printDebugLoc(DebugLoc DL, const MachineFunction *MF,
                          raw_ostream &CommentOS) {
   const LLVMContext &Ctx = MF->getFunction()->getContext();
   if (!DL.isUnknown()) {          // Print source line info.
@@ -1380,7 +1392,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const {
   unsigned AsmDescOp = ~0u;
   unsigned AsmOpCount = 0;
 
-  if (isInlineAsm()) {
+  if (isInlineAsm() && e >= InlineAsm::MIOp_FirstOperand) {
     // Print asm string.
     OS << " ";
     getOperand(InlineAsm::MIOp_AsmString).print(OS, TM);
@@ -1451,18 +1463,28 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const {
       OS << '$' << AsmOpCount++;
       unsigned Flag = MO.getImm();
       switch (InlineAsm::getKind(Flag)) {
-      case InlineAsm::Kind_RegUse:             OS << ":[reguse]"; break;
-      case InlineAsm::Kind_RegDef:             OS << ":[regdef]"; break;
-      case InlineAsm::Kind_RegDefEarlyClobber: OS << ":[regdef-ec]"; break;
-      case InlineAsm::Kind_Clobber:            OS << ":[clobber]"; break;
-      case InlineAsm::Kind_Imm:                OS << ":[imm]"; break;
-      case InlineAsm::Kind_Mem:                OS << ":[mem]"; break;
-      default: OS << ":[??" << InlineAsm::getKind(Flag) << ']'; break;
+      case InlineAsm::Kind_RegUse:             OS << ":[reguse"; break;
+      case InlineAsm::Kind_RegDef:             OS << ":[regdef"; break;
+      case InlineAsm::Kind_RegDefEarlyClobber: OS << ":[regdef-ec"; break;
+      case InlineAsm::Kind_Clobber:            OS << ":[clobber"; break;
+      case InlineAsm::Kind_Imm:                OS << ":[imm"; break;
+      case InlineAsm::Kind_Mem:                OS << ":[mem"; break;
+      default: OS << ":[??" << InlineAsm::getKind(Flag); break;
+      }
+
+      unsigned RCID = 0;
+      if (InlineAsm::hasRegClassConstraint(Flag, RCID)) {
+        if (TM)
+          OS << ':' << TM->getRegisterInfo()->getRegClass(RCID)->getName();
+        else
+          OS << ":RC" << RCID;
       }
 
       unsigned TiedTo = 0;
       if (InlineAsm::isUseOperandTiedToDef(Flag, TiedTo))
-        OS << " [tiedto:$" << TiedTo << ']';
+        OS << " tiedto:$" << TiedTo;
+
+      OS << ']';
 
       // Compute the index of the next operand descriptor.
       AsmDescOp += 1 + InlineAsm::getNumOperandRegisters(Flag);
@@ -1516,7 +1538,19 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const {
   }
 
   // Print debug location information.
-  if (!debugLoc.isUnknown() && MF) {
+  if (isDebugValue() && getOperand(e - 1).isMetadata()) {
+    if (!HaveSemi) OS << ";"; HaveSemi = true;
+    DIVariable DV(getOperand(e - 1).getMetadata());
+    OS << " line no:" <<  DV.getLineNumber();
+    if (MDNode *InlinedAt = DV.getInlinedAt()) {
+      DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(InlinedAt);
+      if (!InlinedAtDL.isUnknown()) {
+        OS << " inlined @[ ";
+        printDebugLoc(InlinedAtDL, MF, OS);
+        OS << " ]";
+      }
+    }
+  } else if (!debugLoc.isUnknown() && MF) {
     if (!HaveSemi) OS << ";"; HaveSemi = true;
     OS << " dbg:";
     printDebugLoc(debugLoc, MF, OS);
@@ -1627,7 +1661,7 @@ bool MachineInstr::addRegisterDead(unsigned IncomingReg,
   // new implicit operand if required.
   if (Found || !AddIfNotFound)
     return Found;
-    
+
   addOperand(MachineOperand::CreateReg(IncomingReg,
                                        true  /*IsDef*/,
                                        true  /*IsImp*/,
diff --git a/contrib/llvm/lib/CodeGen/MachineLICM.cpp b/contrib/llvm/lib/CodeGen/MachineLICM.cpp
index 722ceb202439..a1f80d5282e0 100644
--- a/contrib/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineLICM.cpp
@@ -37,10 +37,16 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
+static cl::opt<bool>
+AvoidSpeculation("avoid-speculation",
+                 cl::desc("MachineLICM should avoid speculation"),
+                 cl::init(false), cl::Hidden);
+
 STATISTIC(NumHoisted,
           "Number of machine instructions hoisted out of loops");
 STATISTIC(NumLowRP,
@@ -91,6 +97,17 @@ namespace {
     // For each opcode, keep a list of potential CSE instructions.
     DenseMap<unsigned, std::vector<const MachineInstr*> > CSEMap;
 
+    enum {
+      SpeculateFalse   = 0,
+      SpeculateTrue    = 1,
+      SpeculateUnknown = 2
+    };
+
+    // If a MBB does not dominate loop exiting blocks then it may not safe
+    // to hoist loads from this block.
+    // Tri-state: 0 - false, 1 - true, 2 - unknown
+    unsigned SpeculationState;
+
   public:
     static char ID; // Pass identification, replacement for typeid
     MachineLICM() :
@@ -194,6 +211,10 @@ namespace {
     /// hoist the given loop invariant.
     bool IsProfitableToHoist(MachineInstr &MI);
 
+    /// IsGuaranteedToExecute - Check if this mbb is guaranteed to execute.
+    /// If not then a load from this mbb may not be safe to hoist.
+    bool IsGuaranteedToExecute(MachineBasicBlock *BB);
+
     /// HoistRegion - Walk the specified region of the CFG (defined by all
     /// blocks dominated by the specified block, and that are in the current
     /// loop) in depth first order w.r.t the DominatorTree. This allows us to
@@ -202,6 +223,13 @@ namespace {
     ///
     void HoistRegion(MachineDomTreeNode *N, bool IsHeader = false);
 
+    /// getRegisterClassIDAndCost - For a given MI, register, and the operand
+    /// index, return the ID and cost of its representative register class by
+    /// reference.
+    void getRegisterClassIDAndCost(const MachineInstr *MI,
+                                   unsigned Reg, unsigned OpIdx,
+                                   unsigned &RCId, unsigned &RCCost) const;
+
     /// InitRegPressure - Find all virtual register references that are liveout
     /// of the preheader to initialize the starting "register pressure". Note
     /// this does not count live through (livein but not used) registers.
@@ -229,6 +257,10 @@ namespace {
     bool EliminateCSE(MachineInstr *MI,
            DenseMap<unsigned, std::vector<const MachineInstr*> >::iterator &CI);
 
+    /// MayCSE - Return true if the given instruction will be CSE'd if it's
+    /// hoisted out of the loop.
+    bool MayCSE(MachineInstr *MI);
+
     /// Hoist - When an instruction is found to only use loop invariant operands
     /// that is safe to hoist, this instruction is called to do the dirty work.
     /// It returns true if the instruction is hoisted.
@@ -441,6 +473,12 @@ void MachineLICM::HoistRegionPostRA() {
   const std::vector<MachineBasicBlock*> Blocks = CurLoop->getBlocks();
   for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
     MachineBasicBlock *BB = Blocks[i];
+
+    // If the header of the loop containing this basic block is a landing pad,
+    // then don't try to hoist instructions out of this loop.
+    const MachineLoop *ML = MLI->getLoopFor(BB);
+    if (ML && ML->getHeader()->isLandingPad()) continue;
+
     // Conservatively treat live-in's as an external def.
     // FIXME: That means a reload that're reused in successor block(s) will not
     // be LICM'ed.
@@ -452,6 +490,7 @@ void MachineLICM::HoistRegionPostRA() {
         ++PhysRegDefs[*AS];
     }
 
+    SpeculationState = SpeculateUnknown;
     for (MachineBasicBlock::iterator
            MII = BB->begin(), E = BB->end(); MII != E; ++MII) {
       MachineInstr *MI = &*MII;
@@ -545,6 +584,27 @@ void MachineLICM::HoistPostRA(MachineInstr *MI, unsigned Def) {
   Changed = true;
 }
 
+// IsGuaranteedToExecute - Check if this mbb is guaranteed to execute.
+// If not then a load from this mbb may not be safe to hoist.
+bool MachineLICM::IsGuaranteedToExecute(MachineBasicBlock *BB) {
+  if (SpeculationState != SpeculateUnknown)
+    return SpeculationState == SpeculateFalse;
+    
+  if (BB != CurLoop->getHeader()) {
+    // Check loop exiting blocks.
+    SmallVector<MachineBasicBlock*, 8> CurrentLoopExitingBlocks;
+    CurLoop->getExitingBlocks(CurrentLoopExitingBlocks);
+    for (unsigned i = 0, e = CurrentLoopExitingBlocks.size(); i != e; ++i)
+      if (!DT->dominates(BB, CurrentLoopExitingBlocks[i])) {
+        SpeculationState = SpeculateTrue;
+        return false;
+      }
+  }
+
+  SpeculationState = SpeculateFalse;
+  return true;
+}
+
 /// HoistRegion - Walk the specified region of the CFG (defined by all blocks
 /// dominated by the specified block, and that are in the current loop) in depth
 /// first order w.r.t the DominatorTree. This allows us to visit definitions
@@ -554,6 +614,11 @@ void MachineLICM::HoistRegion(MachineDomTreeNode *N, bool IsHeader) {
   assert(N != 0 && "Null dominator tree node?");
   MachineBasicBlock *BB = N->getBlock();
 
+  // If the header of the loop containing this basic block is a landing pad,
+  // then don't try to hoist instructions out of this loop.
+  const MachineLoop *ML = MLI->getLoopFor(BB);
+  if (ML && ML->getHeader()->isLandingPad()) return;
+
   // If this subregion is not in the top level loop at all, exit.
   if (!CurLoop->contains(BB)) return;
 
@@ -571,6 +636,7 @@ void MachineLICM::HoistRegion(MachineDomTreeNode *N, bool IsHeader) {
   // Remember livein register pressure.
   BackTrace.push_back(RegPressure);
 
+  SpeculationState = SpeculateUnknown;
   for (MachineBasicBlock::iterator
          MII = BB->begin(), E = BB->end(); MII != E; ) {
     MachineBasicBlock::iterator NextMII = MII; ++NextMII;
@@ -596,6 +662,23 @@ static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) {
   return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg());
 }
 
+/// getRegisterClassIDAndCost - For a given MI, register, and the operand
+/// index, return the ID and cost of its representative register class.
+void
+MachineLICM::getRegisterClassIDAndCost(const MachineInstr *MI,
+                                       unsigned Reg, unsigned OpIdx,
+                                       unsigned &RCId, unsigned &RCCost) const {
+  const TargetRegisterClass *RC = MRI->getRegClass(Reg);
+  EVT VT = *RC->vt_begin();
+  if (VT == MVT::untyped) {
+    RCId = RC->getID();
+    RCCost = 1;
+  } else {
+    RCId = TLI->getRepRegClassFor(VT)->getID();
+    RCCost = TLI->getRepRegClassCostFor(VT);
+  }
+}
+                                      
 /// InitRegPressure - Find all virtual register references that are liveout of
 /// the preheader to initialize the starting "register pressure". Note this
 /// does not count live through (livein but not used) registers.
@@ -625,18 +708,17 @@ void MachineLICM::InitRegPressure(MachineBasicBlock *BB) {
         continue;
 
       bool isNew = RegSeen.insert(Reg);
-      const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-      EVT VT = *RC->vt_begin();
-      unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
+      unsigned RCId, RCCost;
+      getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost);
       if (MO.isDef())
-        RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
+        RegPressure[RCId] += RCCost;
       else {
         bool isKill = isOperandKill(MO, MRI);
         if (isNew && !isKill)
           // Haven't seen this, it must be a livein.
-          RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
+          RegPressure[RCId] += RCCost;
         else if (!isNew && isKill)
-          RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT);
+          RegPressure[RCId] -= RCCost;
       }
     }
   }
@@ -661,11 +743,8 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) {
     if (MO.isDef())
       Defs.push_back(Reg);
     else if (!isNew && isOperandKill(MO, MRI)) {
-      const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-      EVT VT = *RC->vt_begin();
-      unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
-      unsigned RCCost = TLI->getRepRegClassCostFor(VT);
-
+      unsigned RCId, RCCost;
+      getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost);
       if (RCCost > RegPressure[RCId])
         RegPressure[RCId] = 0;
       else
@@ -673,13 +752,13 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) {
     }
   }
 
+  unsigned Idx = 0;
   while (!Defs.empty()) {
     unsigned Reg = Defs.pop_back_val();
-    const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-    EVT VT = *RC->vt_begin();
-    unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
-    unsigned RCCost = TLI->getRepRegClassCostFor(VT);
+    unsigned RCId, RCCost;
+    getRegisterClassIDAndCost(MI, Reg, Idx, RCId, RCCost);
     RegPressure[RCId] += RCCost;
+    ++Idx;
   }
 }
 
@@ -691,7 +770,14 @@ bool MachineLICM::IsLICMCandidate(MachineInstr &I) {
   bool DontMoveAcrossStore = true;
   if (!I.isSafeToMove(TII, AA, DontMoveAcrossStore))
     return false;
-  
+
+  // If it is load then check if it is guaranteed to execute by making sure that
+  // it dominates all exiting blocks. If it doesn't, then there is a path out of
+  // the loop which does not execute this load, so we can't hoist it.
+  // Stores and side effects are already checked by isSafeToMove.
+  if (I.getDesc().mayLoad() && !IsGuaranteedToExecute(I.getParent()))
+    return false;
+
   return true;
 }
 
@@ -879,10 +965,8 @@ void MachineLICM::UpdateBackTraceRegPressure(const MachineInstr *MI) {
     if (!TargetRegisterInfo::isVirtualRegister(Reg))
       continue;
 
-    const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-    EVT VT = *RC->vt_begin();
-    unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
-    unsigned RCCost = TLI->getRepRegClassCostFor(VT);
+    unsigned RCId, RCCost;
+    getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost);
     if (MO.isDef()) {
       DenseMap<unsigned, int>::iterator CI = Cost.find(RCId);
       if (CI != Cost.end())
@@ -941,16 +1025,15 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) {
       unsigned Reg = MO.getReg();
       if (!TargetRegisterInfo::isVirtualRegister(Reg))
         continue;
+
+      unsigned RCId, RCCost;
+      getRegisterClassIDAndCost(&MI, Reg, i, RCId, RCCost);
       if (MO.isDef()) {
         if (HasHighOperandLatency(MI, i, Reg)) {
           ++NumHighLatency;
           return true;
         }
 
-        const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-        EVT VT = *RC->vt_begin();
-        unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
-        unsigned RCCost = TLI->getRepRegClassCostFor(VT);
         DenseMap<unsigned, int>::iterator CI = Cost.find(RCId);
         if (CI != Cost.end())
           CI->second += RCCost;
@@ -960,10 +1043,6 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) {
         // Is a virtual register use is a kill, hoisting it out of the loop
         // may actually reduce register pressure or be register pressure
         // neutral.
-        const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-        EVT VT = *RC->vt_begin();
-        unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
-        unsigned RCCost = TLI->getRepRegClassCostFor(VT);
         DenseMap<unsigned, int>::iterator CI = Cost.find(RCId);
         if (CI != Cost.end())
           CI->second -= RCCost;
@@ -979,6 +1058,13 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) {
       return true;
     }
 
+    // Do not "speculate" in high register pressure situation. If an
+    // instruction is not guaranteed to be executed in the loop, it's best to be
+    // conservative.
+    if (AvoidSpeculation &&
+        (!IsGuaranteedToExecute(MI.getParent()) && !MayCSE(&MI)))
+      return false;
+
     // High register pressure situation, only hoist if the instruction is going to
     // be remat'ed.
     if (!TII->isTriviallyReMaterializable(&MI, AA) &&
@@ -1116,6 +1202,20 @@ bool MachineLICM::EliminateCSE(MachineInstr *MI,
   return false;
 }
 
+/// MayCSE - Return true if the given instruction will be CSE'd if it's
+/// hoisted out of the loop.
+bool MachineLICM::MayCSE(MachineInstr *MI) {
+  unsigned Opcode = MI->getOpcode();
+  DenseMap<unsigned, std::vector<const MachineInstr*> >::iterator
+    CI = CSEMap.find(Opcode);
+  // Do not CSE implicit_def so ProcessImplicitDefs can properly propagate
+  // the undef property onto uses.
+  if (CI == CSEMap.end() || MI->isImplicitDef())
+    return false;
+
+  return LookForDuplicate(MI, CI->second) != 0;
+}
+
 /// Hoist - When an instruction is found to use only loop invariant operands
 /// that are safe to hoist, this instruction is called to do the dirty work.
 ///
diff --git a/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp b/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp
index fadc594efcb2..80c4854238af 100644
--- a/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp
@@ -17,9 +17,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/Target/TargetInstrInfo.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetOptions.h"
+#include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/Dwarf.h"
@@ -254,11 +252,12 @@ void MMIAddrLabelMapCallbackPtr::allUsesReplacedWith(Value *V2) {
 //===----------------------------------------------------------------------===//
 
 MachineModuleInfo::MachineModuleInfo(const MCAsmInfo &MAI,
-                                     const TargetAsmInfo *TAI)
-: ImmutablePass(ID), Context(MAI, TAI),
-  ObjFileMMI(0),
-  CurCallSite(0), CallsEHReturn(0), CallsUnwindInit(0), DbgInfoAvailable(false),
-  CallsExternalVAFunctionWithFloatingPointArguments(false) {
+                                     const MCRegisterInfo &MRI,
+                                     const MCObjectFileInfo *MOFI)
+  : ImmutablePass(ID), Context(MAI, MRI, MOFI),
+    ObjFileMMI(0), CompactUnwindEncoding(0), CurCallSite(0), CallsEHReturn(0),
+    CallsUnwindInit(0), DbgInfoAvailable(false),
+    CallsExternalVAFunctionWithFloatingPointArguments(false) {
   initializeMachineModuleInfoPass(*PassRegistry::getPassRegistry());
   // Always emit some info, by default "no personality" info.
   Personalities.push_back(NULL);
@@ -267,7 +266,8 @@ MachineModuleInfo::MachineModuleInfo(const MCAsmInfo &MAI,
 }
 
 MachineModuleInfo::MachineModuleInfo()
-: ImmutablePass(ID), Context(*(MCAsmInfo*)0, NULL) {
+  : ImmutablePass(ID),
+    Context(*(MCAsmInfo*)0, *(MCRegisterInfo*)0, (MCObjectFileInfo*)0) {
   assert(0 && "This MachineModuleInfo constructor should never be called, MMI "
          "should always be explicitly constructed by LLVMTargetMachine");
   abort();
@@ -311,6 +311,7 @@ void MachineModuleInfo::EndFunction() {
   FilterEnds.clear();
   CallsEHReturn = 0;
   CallsUnwindInit = 0;
+  CompactUnwindEncoding = 0;
   VariableDbgInfo.clear();
 }
 
@@ -426,8 +427,9 @@ void MachineModuleInfo::addPersonality(MachineBasicBlock *LandingPad,
 
 /// addCatchTypeInfo - Provide the catch typeinfo for a landing pad.
 ///
-void MachineModuleInfo::addCatchTypeInfo(MachineBasicBlock *LandingPad,
-                                  std::vector<const GlobalVariable *> &TyInfo) {
+void MachineModuleInfo::
+addCatchTypeInfo(MachineBasicBlock *LandingPad,
+                 ArrayRef<const GlobalVariable *> TyInfo) {
   LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
   for (unsigned N = TyInfo.size(); N; --N)
     LP.TypeIds.push_back(getTypeIDFor(TyInfo[N - 1]));
@@ -435,8 +437,9 @@ void MachineModuleInfo::addCatchTypeInfo(MachineBasicBlock *LandingPad,
 
 /// addFilterTypeInfo - Provide the filter typeinfo for a landing pad.
 ///
-void MachineModuleInfo::addFilterTypeInfo(MachineBasicBlock *LandingPad,
-                                  std::vector<const GlobalVariable *> &TyInfo) {
+void MachineModuleInfo::
+addFilterTypeInfo(MachineBasicBlock *LandingPad,
+                  ArrayRef<const GlobalVariable *> TyInfo) {
   LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
   std::vector<unsigned> IdsInFilter(TyInfo.size());
   for (unsigned I = 0, E = TyInfo.size(); I != E; ++I)
@@ -496,6 +499,14 @@ void MachineModuleInfo::TidyLandingPads(DenseMap<MCSymbol*, uintptr_t> *LPMap) {
   }
 }
 
+/// setCallSiteLandingPad - Map the landing pad's EH symbol to the call site
+/// indexes.
+void MachineModuleInfo::setCallSiteLandingPad(MCSymbol *Sym,
+                                              ArrayRef<unsigned> Sites) {
+  for (unsigned I = 0, E = Sites.size(); I != E; ++I)
+    LPadToCallSiteMap[Sym].push_back(Sites[I]);
+}
+
 /// getTypeIDFor - Return the type id for the specified typeinfo.  This is
 /// function wide.
 unsigned MachineModuleInfo::getTypeIDFor(const GlobalVariable *TI) {
diff --git a/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index 4b3e64c25f60..266ebf64a3fc 100644
--- a/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -14,10 +14,11 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/Target/TargetInstrInfo.h"
-#include "llvm/Support/CommandLine.h"
+#include "llvm/Target/TargetMachine.h"
 using namespace llvm;
 
-MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI) {
+MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI)
+  : TRI(&TRI), IsSSA(true) {
   VRegInfo.reserve(256);
   RegAllocHints.reserve(256);
   UsedPhysRegs.resize(TRI.getNumRegs());
@@ -48,18 +49,47 @@ MachineRegisterInfo::setRegClass(unsigned Reg, const TargetRegisterClass *RC) {
 
 const TargetRegisterClass *
 MachineRegisterInfo::constrainRegClass(unsigned Reg,
-                                       const TargetRegisterClass *RC) {
+                                       const TargetRegisterClass *RC,
+                                       unsigned MinNumRegs) {
   const TargetRegisterClass *OldRC = getRegClass(Reg);
   if (OldRC == RC)
     return RC;
-  const TargetRegisterClass *NewRC = getCommonSubClass(OldRC, RC);
-  if (!NewRC)
+  const TargetRegisterClass *NewRC = TRI->getCommonSubClass(OldRC, RC);
+  if (!NewRC || NewRC == OldRC)
+    return NewRC;
+  if (NewRC->getNumRegs() < MinNumRegs)
     return 0;
-  if (NewRC != OldRC)
-    setRegClass(Reg, NewRC);
+  setRegClass(Reg, NewRC);
   return NewRC;
 }
 
+bool
+MachineRegisterInfo::recomputeRegClass(unsigned Reg, const TargetMachine &TM) {
+  const TargetInstrInfo *TII = TM.getInstrInfo();
+  const TargetRegisterClass *OldRC = getRegClass(Reg);
+  const TargetRegisterClass *NewRC = TRI->getLargestLegalSuperClass(OldRC);
+
+  // Stop early if there is no room to grow.
+  if (NewRC == OldRC)
+    return false;
+
+  // Accumulate constraints from all uses.
+  for (reg_nodbg_iterator I = reg_nodbg_begin(Reg), E = reg_nodbg_end(); I != E;
+       ++I) {
+    // TRI doesn't have accurate enough information to model this yet.
+    if (I.getOperand().getSubReg())
+      return false;
+    const TargetRegisterClass *OpRC =
+      I->getRegClassConstraint(I.getOperandNo(), TII, TRI);
+    if (OpRC)
+      NewRC = TRI->getCommonSubClass(NewRC, OpRC);
+    if (!NewRC || NewRC == OldRC)
+      return false;
+  }
+  setRegClass(Reg, NewRC);
+  return true;
+}
+
 /// createVirtualRegister - Create and return a new virtual register in the
 /// function with the specified register class.
 ///
diff --git a/contrib/llvm/lib/CodeGen/MachineSink.cpp b/contrib/llvm/lib/CodeGen/MachineSink.cpp
index 916dff70a41e..29cfb49953b9 100644
--- a/contrib/llvm/lib/CodeGen/MachineSink.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineSink.cpp
@@ -382,6 +382,25 @@ static bool AvoidsSinking(MachineInstr *MI, MachineRegisterInfo *MRI) {
   return MI->isInsertSubreg() || MI->isSubregToReg() || MI->isRegSequence();
 }
 
+/// collectDebgValues - Scan instructions following MI and collect any 
+/// matching DBG_VALUEs.
+static void collectDebugValues(MachineInstr *MI, 
+                               SmallVector<MachineInstr *, 2> & DbgValues) {
+  DbgValues.clear();
+  if (!MI->getOperand(0).isReg())
+    return;
+
+  MachineBasicBlock::iterator DI = MI; ++DI;
+  for (MachineBasicBlock::iterator DE = MI->getParent()->end();
+       DI != DE; ++DI) {
+    if (!DI->isDebugValue())
+      return;
+    if (DI->getOperand(0).isReg() &&
+        DI->getOperand(0).getReg() == MI->getOperand(0).getReg())
+      DbgValues.push_back(DI);
+  }
+}
+
 /// SinkInstruction - Determine whether it is safe to sink the specified machine
 /// instruction out of its current block into a successor.
 bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
@@ -598,10 +617,22 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
   while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
     ++InsertPos;
 
+  // collect matching debug values.
+  SmallVector<MachineInstr *, 2> DbgValuesToSink;
+  collectDebugValues(MI, DbgValuesToSink);
+
   // Move the instruction.
   SuccToSinkTo->splice(InsertPos, ParentBlock, MI,
                        ++MachineBasicBlock::iterator(MI));
 
+  // Move debug values.
+  for (SmallVector<MachineInstr *, 2>::iterator DBI = DbgValuesToSink.begin(),
+         DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) {
+    MachineInstr *DbgMI = *DBI;
+    SuccToSinkTo->splice(InsertPos, ParentBlock,  DbgMI,
+                         ++MachineBasicBlock::iterator(DbgMI));
+  }
+
   // Conservatively, clear any kill flags, since it's possible that they are no
   // longer correct.
   MI->clearKillInfo();
diff --git a/contrib/llvm/lib/CodeGen/MachineVerifier.cpp b/contrib/llvm/lib/CodeGen/MachineVerifier.cpp
index 7a55852a1315..26847d39e7ad 100644
--- a/contrib/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/contrib/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -72,6 +72,8 @@ namespace {
     typedef DenseSet<unsigned> RegSet;
     typedef DenseMap<unsigned, const MachineInstr*> RegMap;
 
+    const MachineInstr *FirstTerminator;
+
     BitVector regsReserved;
     RegSet regsLive;
     RegVector regsDefined, regsDead, regsKilled;
@@ -389,6 +391,8 @@ static bool matchPair(MachineBasicBlock::const_succ_iterator i,
 
 void
 MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
+  FirstTerminator = 0;
+
   // Count the number of landing pad successors.
   SmallPtrSet<MachineBasicBlock*, 4> LandingPadSuccs;
   for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
@@ -570,6 +574,18 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
     }
   }
 
+  // Ensure non-terminators don't follow terminators.
+  if (MCID.isTerminator()) {
+    if (!FirstTerminator)
+      FirstTerminator = MI;
+  } else if (FirstTerminator) {
+    report("Non-terminator instruction after the first terminator", MI);
+    *OS << "First terminator was:\t" << *FirstTerminator;
+  }
+
+  StringRef ErrorInfo;
+  if (!TII->verifyInstruction(MI, ErrorInfo))
+    report(ErrorInfo.data(), MI);
 }
 
 void
@@ -686,6 +702,11 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
       else
         addRegWithSubRegs(regsDefined, Reg);
 
+      // Verify SSA form.
+      if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) &&
+          llvm::next(MRI->def_begin(Reg)) != MRI->def_end())
+        report("Multiple virtual register defs in SSA form", MO, MONum);
+
       // Check LiveInts for a live range, but only for virtual registers.
       if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) &&
           !LiveInts->isNotInMIMap(MI)) {
@@ -714,20 +735,14 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
       unsigned SubIdx = MO->getSubReg();
 
       if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
-        unsigned sr = Reg;
         if (SubIdx) {
-          unsigned s = TRI->getSubReg(Reg, SubIdx);
-          if (!s) {
-            report("Invalid subregister index for physical register",
-                   MO, MONum);
-            return;
-          }
-          sr = s;
+          report("Illegal subregister index for physical register", MO, MONum);
+          return;
         }
         if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) {
-          if (!DRC->contains(sr)) {
+          if (!DRC->contains(Reg)) {
             report("Illegal physical register for instruction", MO, MONum);
-            *OS << TRI->getName(sr) << " is not a "
+            *OS << TRI->getName(Reg) << " is not a "
                 << DRC->getName() << " register.\n";
           }
         }
@@ -735,16 +750,35 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
         // Virtual register.
         const TargetRegisterClass *RC = MRI->getRegClass(Reg);
         if (SubIdx) {
-          const TargetRegisterClass *SRC = RC->getSubRegisterRegClass(SubIdx);
+          const TargetRegisterClass *SRC =
+            TRI->getSubClassWithSubReg(RC, SubIdx);
           if (!SRC) {
             report("Invalid subregister index for virtual register", MO, MONum);
             *OS << "Register class " << RC->getName()
                 << " does not support subreg index " << SubIdx << "\n";
             return;
           }
-          RC = SRC;
+          if (RC != SRC) {
+            report("Invalid register class for subregister index", MO, MONum);
+            *OS << "Register class " << RC->getName()
+                << " does not fully support subreg index " << SubIdx << "\n";
+            return;
+          }
         }
         if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) {
+          if (SubIdx) {
+            const TargetRegisterClass *SuperRC =
+              TRI->getLargestLegalSuperClass(RC);
+            if (!SuperRC) {
+              report("No largest legal super class exists.", MO, MONum);
+              return;
+            }
+            DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx);
+            if (!DRC) {
+              report("No matching super-reg register class.", MO, MONum);
+              return;
+            }
+          }
           if (!RC->hasSuperClassEq(DRC)) {
             report("Illegal virtual register for instruction", MO, MONum);
             *OS << "Expected a " << DRC->getName() << " register, but got a "
@@ -1161,18 +1195,8 @@ void MachineVerifier::verifyLiveIntervals() {
           SlotIndex PEnd = LiveInts->getMBBEndIdx(*PI).getPrevSlot();
           const VNInfo *PVNI = LI.getVNInfoAt(PEnd);
 
-          if (VNI->isPHIDef() && VNI->def == LiveInts->getMBBStartIdx(MFI)) {
-            if (PVNI && !PVNI->hasPHIKill()) {
-              report("Value live out of predecessor doesn't have PHIKill", MF);
-              *OS << "Valno #" << PVNI->id << " live out of BB#"
-                  << (*PI)->getNumber() << '@' << PEnd
-                  << " doesn't have PHIKill, but Valno #" << VNI->id
-                  << " is PHIDef and defined at the beginning of BB#"
-                  << MFI->getNumber() << '@' << LiveInts->getMBBStartIdx(MFI)
-                  << " in " << LI << '\n';
-            }
+          if (VNI->isPHIDef() && VNI->def == LiveInts->getMBBStartIdx(MFI))
             continue;
-          }
 
           if (!PVNI) {
             report("Register not marked live out of predecessor", *PI);
diff --git a/contrib/llvm/lib/CodeGen/PHIElimination.cpp b/contrib/llvm/lib/CodeGen/PHIElimination.cpp
index af65f13bf065..6994aa58fbd5 100644
--- a/contrib/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/contrib/llvm/lib/CodeGen/PHIElimination.cpp
@@ -109,6 +109,9 @@ bool PHIElimination::runOnMachineFunction(MachineFunction &MF) {
 
   bool Changed = false;
 
+  // This pass takes the function out of SSA form.
+  MRI->leaveSSA();
+
   // Split critical edges to help the coalescer
   if (!DisableEdgeSplitting) {
     if (LiveVariables *LV = getAnalysisIfAvailable<LiveVariables>()) {
diff --git a/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index c523e39bc258..bbc7ce2d0a42 100644
--- a/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -295,7 +295,6 @@ bool PeepholeOptimizer::OptimizeBitcastInstr(MachineInstr *MI,
   if (!DefMI || !DefMI->getDesc().isBitcast())
     return false;
 
-  unsigned SrcDef = 0;
   unsigned SrcSrc = 0;
   NumDefs = DefMI->getDesc().getNumDefs();
   NumSrcs = DefMI->getDesc().getNumOperands() - NumDefs;
@@ -308,13 +307,13 @@ bool PeepholeOptimizer::OptimizeBitcastInstr(MachineInstr *MI,
     unsigned Reg = MO.getReg();
     if (!Reg)
       continue;
-    if (MO.isDef())
-      SrcDef = Reg;
-    else if (SrcSrc)
-      // Multiple sources?
-      return false;
-    else
-      SrcSrc = Reg;
+    if (!MO.isDef()) {
+      if (SrcSrc)
+        // Multiple sources?
+        return false;
+      else
+        SrcSrc = Reg;
+    }
   }
 
   if (MRI->getRegClass(SrcSrc) != MRI->getRegClass(Def))
@@ -434,6 +433,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
       if (MCID.isBitcast()) {
         if (OptimizeBitcastInstr(MI, MBB)) {
           // MI is deleted.
+          LocalMIs.erase(MI);
           Changed = true;
           MII = First ? I->begin() : llvm::next(PMII);
           continue;
@@ -441,6 +441,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
       } else if (MCID.isCompare()) {
         if (OptimizeCmpInstr(MI, MBB)) {
           // MI is deleted.
+          LocalMIs.erase(MI);
           Changed = true;
           MII = First ? I->begin() : llvm::next(PMII);
           continue;
diff --git a/contrib/llvm/lib/CodeGen/ProcessImplicitDefs.cpp b/contrib/llvm/lib/CodeGen/ProcessImplicitDefs.cpp
index c04d65637c94..b1d8c9760225 100644
--- a/contrib/llvm/lib/CodeGen/ProcessImplicitDefs.cpp
+++ b/contrib/llvm/lib/CodeGen/ProcessImplicitDefs.cpp
@@ -125,8 +125,14 @@ bool ProcessImplicitDefs::runOnMachineFunction(MachineFunction &fn) {
             LiveVariables::VarInfo& vi = LV->getVarInfo(MO.getReg());
             vi.removeKill(MI);
           }
+          unsigned Reg = MI->getOperand(0).getReg();
           MI->eraseFromParent();
           Changed = true;
+
+          // A REG_SEQUENCE may have been expanded into partial definitions.
+          // If this was the last one, mark Reg as implicitly defined.
+          if (TargetRegisterInfo::isVirtualRegister(Reg) && MRI->def_empty(Reg))
+            ImpDefRegs.insert(Reg);
           continue;
         }
       }
diff --git a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index a901c5fefa3e..32c932552bed 100644
--- a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetInstrInfo.h"
@@ -54,6 +55,8 @@ INITIALIZE_PASS_END(PEI, "prologepilog",
 
 STATISTIC(NumVirtualFrameRegs, "Number of virtual frame regs encountered");
 STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged");
+STATISTIC(NumBytesStackSpace,
+          "Number of bytes used for stack in all functions");
 
 /// createPrologEpilogCodeInserter - This function returns a pass that inserts
 /// prolog and epilog code, and eliminates abstract frame references.
@@ -677,7 +680,9 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) {
   }
 
   // Update frame info to pretend that this is part of the stack...
-  MFI->setStackSize(Offset - LocalAreaOffset);
+  int64_t StackSize = Offset - LocalAreaOffset;
+  MFI->setStackSize(StackSize);
+  NumBytesStackSpace += StackSize;
 }
 
 /// insertPrologEpilogCode - Scan the function for modified callee saved
@@ -696,6 +701,13 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
     if (!I->empty() && I->back().getDesc().isReturn())
       TFI.emitEpilogue(Fn, *I);
   }
+
+  // Emit additional code that is required to support segmented stacks, if
+  // we've been asked for it.  This, when linked with a runtime with support
+  // for segmented stacks (libgcc is one), will result in allocating stack
+  // space in small chunks instead of one large contiguous block.
+  if (EnableSegmentedStacks)
+    TFI.adjustForSegmentedStacks(Fn);
 }
 
 /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical
diff --git a/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp b/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp
index 5ea26adc7644..5496d69fd3df 100644
--- a/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -20,7 +20,6 @@
 #include "RenderMachineFunction.h"
 #include "Spiller.h"
 #include "VirtRegMap.h"
-#include "RegisterCoalescer.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
@@ -160,7 +159,7 @@ void RABasic::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<LiveDebugVariables>();
   if (StrongPHIElim)
     AU.addRequiredID(StrongPHIEliminationID);
-  AU.addRequiredTransitive<RegisterCoalescer>();
+  AU.addRequiredTransitiveID(RegisterCoalescerPassID);
   AU.addRequired<CalculateSpillWeights>();
   AU.addRequired<LiveStacks>();
   AU.addPreserved<LiveStacks>();
@@ -439,6 +438,7 @@ void RegAllocBase::addMBBLiveIns(MachineFunction *MF) {
     LiveIntervalUnion &LiveUnion = PhysReg2LiveUnion[PhysReg];
     if (LiveUnion.empty())
       continue;
+    DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " live-in:");
     MachineFunction::iterator MBB = llvm::next(MF->begin());
     MachineFunction::iterator MFE = MF->end();
     SlotIndex Start, Stop;
@@ -449,6 +449,8 @@ void RegAllocBase::addMBBLiveIns(MachineFunction *MF) {
       if (SI.start() <= Start) {
         if (!MBB->isLiveIn(PhysReg))
           MBB->addLiveIn(PhysReg);
+        DEBUG(dbgs() << "\tBB#" << MBB->getNumber() << ':'
+                     << PrintReg(SI.value()->reg, TRI));
       } else if (SI.start() > Stop)
         MBB = Indexes->getMBBFromIndex(SI.start().getPrevIndex());
       if (++MBB == MFE)
@@ -456,6 +458,7 @@ void RegAllocBase::addMBBLiveIns(MachineFunction *MF) {
       tie(Start, Stop) = Indexes->getMBBRange(MBB);
       SI.advanceTo(Start);
     }
+    DEBUG(dbgs() << '\n');
   }
 }
 
@@ -495,8 +498,9 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg,
       // Found an available register.
       return PhysReg;
     }
+    Queries[interfReg].collectInterferingVRegs(1);
     LiveInterval *interferingVirtReg =
-      Queries[interfReg].firstInterference().liveUnionPos().value();
+      Queries[interfReg].interferingVRegs().front();
 
     // The current VirtReg must either be spillable, or one of its interferences
     // must have less spill weight.
diff --git a/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp b/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp
index e235e87b54f3..f54a2c85d100 100644
--- a/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -22,7 +22,6 @@
 #include "SpillPlacement.h"
 #include "SplitKit.h"
 #include "VirtRegMap.h"
-#include "RegisterCoalescer.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Function.h"
@@ -38,6 +37,7 @@
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/RegAllocRegistry.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -51,6 +51,15 @@ STATISTIC(NumGlobalSplits, "Number of split global live ranges");
 STATISTIC(NumLocalSplits,  "Number of split local live ranges");
 STATISTIC(NumEvicted,      "Number of interferences evicted");
 
+static cl::opt<SplitEditor::ComplementSpillMode>
+SplitSpillMode("split-spill-mode", cl::Hidden,
+  cl::desc("Spill mode for splitting live ranges"),
+  cl::values(clEnumValN(SplitEditor::SM_Partition, "default", "Default"),
+             clEnumValN(SplitEditor::SM_Size,  "size",  "Optimize for size"),
+             clEnumValN(SplitEditor::SM_Speed, "speed", "Optimize for speed"),
+             clEnumValEnd),
+  cl::init(SplitEditor::SM_Partition));
+
 static RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator",
                                        createGreedyRegisterAllocator);
 
@@ -90,12 +99,26 @@ class RAGreedy : public MachineFunctionPass,
   // range splitting algorithm terminates, something that is otherwise hard to
   // ensure.
   enum LiveRangeStage {
-    RS_New,      ///< Never seen before.
-    RS_First,    ///< First time in the queue.
-    RS_Second,   ///< Second time in the queue.
-    RS_Global,   ///< Produced by global splitting.
-    RS_Local,    ///< Produced by local splitting.
-    RS_Spill     ///< Produced by spilling.
+    /// Newly created live range that has never been queued.
+    RS_New,
+
+    /// Only attempt assignment and eviction. Then requeue as RS_Split.
+    RS_Assign,
+
+    /// Attempt live range splitting if assignment is impossible.
+    RS_Split,
+
+    /// Attempt more aggressive live range splitting that is guaranteed to make
+    /// progress.  This is used for split products that may not be making
+    /// progress.
+    RS_Split2,
+
+    /// Live range will be spilled.  No more splitting will be attempted.
+    RS_Spill,
+
+    /// There is nothing more we can do to this live range.  Abort compilation
+    /// if it can't be assigned.
+    RS_Done
   };
 
   static const char *const StageName[];
@@ -157,17 +180,38 @@ class RAGreedy : public MachineFunctionPass,
 
   /// Global live range splitting candidate info.
   struct GlobalSplitCandidate {
+    // Register intended for assignment, or 0.
     unsigned PhysReg;
+
+    // SplitKit interval index for this candidate.
+    unsigned IntvIdx;
+
+    // Interference for PhysReg.
     InterferenceCache::Cursor Intf;
+
+    // Bundles where this candidate should be live.
     BitVector LiveBundles;
     SmallVector<unsigned, 8> ActiveBlocks;
 
     void reset(InterferenceCache &Cache, unsigned Reg) {
       PhysReg = Reg;
+      IntvIdx = 0;
       Intf.setPhysReg(Cache, Reg);
       LiveBundles.clear();
       ActiveBlocks.clear();
     }
+
+    // Set B[i] = C for every live bundle where B[i] was NoCand.
+    unsigned getBundles(SmallVectorImpl<unsigned> &B, unsigned C) {
+      unsigned Count = 0;
+      for (int i = LiveBundles.find_first(); i >= 0;
+           i = LiveBundles.find_next(i))
+        if (B[i] == NoCand) {
+          B[i] = C;
+          Count++;
+        }
+      return Count;
+    }
   };
 
   /// Candidate info for for each PhysReg in AllocationOrder.
@@ -175,6 +219,12 @@ class RAGreedy : public MachineFunctionPass,
   /// class.
   SmallVector<GlobalSplitCandidate, 32> GlobalCand;
 
+  enum { NoCand = ~0u };
+
+  /// Candidate map. Each edge bundle is assigned to a GlobalCand entry, or to
+  /// NoCand which indicates the stack interval.
+  SmallVector<unsigned, 32> BundleCand;
+
 public:
   RAGreedy();
 
@@ -208,8 +258,8 @@ class RAGreedy : public MachineFunctionPass,
   void addThroughConstraints(InterferenceCache::Cursor, ArrayRef<unsigned>);
   void growRegion(GlobalSplitCandidate &Cand);
   float calcGlobalSplitCost(GlobalSplitCandidate&);
-  void splitAroundRegion(LiveInterval&, GlobalSplitCandidate&,
-                         SmallVectorImpl<LiveInterval*>&);
+  bool calcCompactRegion(GlobalSplitCandidate&);
+  void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>);
   void calcGapWeights(unsigned, SmallVectorImpl<float>&);
   bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool);
   bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&);
@@ -222,6 +272,8 @@ class RAGreedy : public MachineFunctionPass,
                     SmallVectorImpl<LiveInterval*>&, unsigned = ~0u);
   unsigned tryRegionSplit(LiveInterval&, AllocationOrder&,
                           SmallVectorImpl<LiveInterval*>&);
+  unsigned tryBlockSplit(LiveInterval&, AllocationOrder&,
+                         SmallVectorImpl<LiveInterval*>&);
   unsigned tryLocalSplit(LiveInterval&, AllocationOrder&,
     SmallVectorImpl<LiveInterval*>&);
   unsigned trySplit(LiveInterval&, AllocationOrder&,
@@ -233,12 +285,12 @@ char RAGreedy::ID = 0;
 
 #ifndef NDEBUG
 const char *const RAGreedy::StageName[] = {
-  "RS_New",
-  "RS_First",
-  "RS_Second",
-  "RS_Global",
-  "RS_Local",
-  "RS_Spill"
+    "RS_New",
+    "RS_Assign",
+    "RS_Split",
+    "RS_Split2",
+    "RS_Spill",
+    "RS_Done"
 };
 #endif
 
@@ -278,7 +330,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<LiveDebugVariables>();
   if (StrongPHIElim)
     AU.addRequiredID(StrongPHIEliminationID);
-  AU.addRequiredTransitive<RegisterCoalescer>();
+  AU.addRequiredTransitiveID(RegisterCoalescerPassID);
   AU.addRequired<CalculateSpillWeights>();
   AU.addRequired<LiveStacks>();
   AU.addPreserved<LiveStacks>();
@@ -325,9 +377,15 @@ void RAGreedy::LRE_WillShrinkVirtReg(unsigned VirtReg) {
 }
 
 void RAGreedy::LRE_DidCloneVirtReg(unsigned New, unsigned Old) {
+  // Cloning a register we haven't even heard about yet?  Just ignore it.
+  if (!ExtraRegInfo.inBounds(Old))
+    return;
+
   // LRE may clone a virtual register because dead code elimination causes it to
-  // be split into connected components. Ensure that the new register gets the
+  // be split into connected components. The new components are much smaller
+  // than the original, so they should get a new chance at being assigned.
   // same stage as the parent.
+  ExtraRegInfo[Old].Stage = RS_Assign;
   ExtraRegInfo.grow(New);
   ExtraRegInfo[New] = ExtraRegInfo[Old];
 }
@@ -350,16 +408,15 @@ void RAGreedy::enqueue(LiveInterval *LI) {
 
   ExtraRegInfo.grow(Reg);
   if (ExtraRegInfo[Reg].Stage == RS_New)
-    ExtraRegInfo[Reg].Stage = RS_First;
+    ExtraRegInfo[Reg].Stage = RS_Assign;
 
-  if (ExtraRegInfo[Reg].Stage == RS_Second)
+  if (ExtraRegInfo[Reg].Stage == RS_Split) {
     // Unsplit ranges that couldn't be allocated immediately are deferred until
-    // everything else has been allocated. Long ranges are allocated last so
-    // they are split against realistic interference.
-    Prio = (1u << 31) - Size;
-  else {
-    // Everything else is allocated in long->short order. Long ranges that don't
-    // fit should be spilled ASAP so they don't create interference.
+    // everything else has been allocated.
+    Prio = Size;
+  } else {
+    // Everything is allocated in long->short order. Long ranges that don't fit
+    // should be spilled (or split) ASAP so they don't create interference.
     Prio = (1u << 31) + Size;
 
     // Boost ranges that have a physical register hint.
@@ -442,7 +499,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
 /// @param BreaksHint True when B is already assigned to its preferred register.
 bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint,
                            LiveInterval &B, bool BreaksHint) {
-  bool CanSplit = getStage(B) <= RS_Second;
+  bool CanSplit = getStage(B) < RS_Spill;
 
   // Be fairly aggressive about following hints as long as the evictee can be
   // split.
@@ -487,7 +544,7 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg,
       if (TargetRegisterInfo::isPhysicalRegister(Intf->reg))
         return false;
       // Never evict spill products. They cannot split or spill.
-      if (getStage(*Intf) == RS_Spill)
+      if (getStage(*Intf) == RS_Done)
         return false;
       // Once a live range becomes small enough, it is urgent that we find a
       // register for it. This is indicated by an infinite spill weight. These
@@ -627,6 +684,7 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf,
     Intf.moveToBlock(BC.Number);
     BC.Entry = BI.LiveIn ? SpillPlacement::PrefReg : SpillPlacement::DontCare;
     BC.Exit = BI.LiveOut ? SpillPlacement::PrefReg : SpillPlacement::DontCare;
+    BC.ChangesValue = BI.FirstDef;
 
     if (!Intf.hasInterference())
       continue;
@@ -638,9 +696,9 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf,
     if (BI.LiveIn) {
       if (Intf.first() <= Indexes->getMBBStartIdx(BC.Number))
         BC.Entry = SpillPlacement::MustSpill, ++Ins;
-      else if (Intf.first() < BI.FirstUse)
+      else if (Intf.first() < BI.FirstInstr)
         BC.Entry = SpillPlacement::PrefSpill, ++Ins;
-      else if (Intf.first() < BI.LastUse)
+      else if (Intf.first() < BI.LastInstr)
         ++Ins;
     }
 
@@ -648,9 +706,9 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf,
     if (BI.LiveOut) {
       if (Intf.last() >= SA->getLastSplitPoint(BC.Number))
         BC.Exit = SpillPlacement::MustSpill, ++Ins;
-      else if (Intf.last() > BI.LastUse)
+      else if (Intf.last() > BI.LastInstr)
         BC.Exit = SpillPlacement::PrefSpill, ++Ins;
-      else if (Intf.last() > BI.FirstUse)
+      else if (Intf.last() > BI.FirstInstr)
         ++Ins;
     }
 
@@ -684,7 +742,7 @@ void RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf,
       assert(T < GroupSize && "Array overflow");
       TBS[T] = Number;
       if (++T == GroupSize) {
-        SpillPlacer->addLinks(ArrayRef<unsigned>(TBS, T));
+        SpillPlacer->addLinks(makeArrayRef(TBS, T));
         T = 0;
       }
       continue;
@@ -714,7 +772,7 @@ void RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf,
 
   ArrayRef<SpillPlacement::BlockConstraint> Array(BCS, B);
   SpillPlacer->addConstraints(Array);
-  SpillPlacer->addLinks(ArrayRef<unsigned>(TBS, T));
+  SpillPlacer->addLinks(makeArrayRef(TBS, T));
 }
 
 void RAGreedy::growRegion(GlobalSplitCandidate &Cand) {
@@ -749,8 +807,16 @@ void RAGreedy::growRegion(GlobalSplitCandidate &Cand) {
     // Any new blocks to add?
     if (ActiveBlocks.size() == AddedTo)
       break;
-    addThroughConstraints(Cand.Intf,
-                          ArrayRef<unsigned>(ActiveBlocks).slice(AddedTo));
+
+    // Compute through constraints from the interference, or assume that all
+    // through blocks prefer spilling when forming compact regions.
+    ArrayRef<unsigned> NewBlocks = makeArrayRef(ActiveBlocks).slice(AddedTo);
+    if (Cand.PhysReg)
+      addThroughConstraints(Cand.Intf, NewBlocks);
+    else
+      // Provide a strong negative bias on through blocks to prevent unwanted
+      // liveness on loop backedges.
+      SpillPlacer->addPrefSpill(NewBlocks, /* Strong= */ true);
     AddedTo = ActiveBlocks.size();
 
     // Perhaps iterating can enable more bundles?
@@ -759,11 +825,55 @@ void RAGreedy::growRegion(GlobalSplitCandidate &Cand) {
   DEBUG(dbgs() << ", v=" << Visited);
 }
 
+/// calcCompactRegion - Compute the set of edge bundles that should be live
+/// when splitting the current live range into compact regions.  Compact
+/// regions can be computed without looking at interference.  They are the
+/// regions formed by removing all the live-through blocks from the live range.
+///
+/// Returns false if the current live range is already compact, or if the
+/// compact regions would form single block regions anyway.
+bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) {
+  // Without any through blocks, the live range is already compact.
+  if (!SA->getNumThroughBlocks())
+    return false;
+
+  // Compact regions don't correspond to any physreg.
+  Cand.reset(IntfCache, 0);
+
+  DEBUG(dbgs() << "Compact region bundles");
+
+  // Use the spill placer to determine the live bundles. GrowRegion pretends
+  // that all the through blocks have interference when PhysReg is unset.
+  SpillPlacer->prepare(Cand.LiveBundles);
+
+  // The static split cost will be zero since Cand.Intf reports no interference.
+  float Cost;
+  if (!addSplitConstraints(Cand.Intf, Cost)) {
+    DEBUG(dbgs() << ", none.\n");
+    return false;
+  }
+
+  growRegion(Cand);
+  SpillPlacer->finish();
+
+  if (!Cand.LiveBundles.any()) {
+    DEBUG(dbgs() << ", none.\n");
+    return false;
+  }
+
+  DEBUG({
+    for (int i = Cand.LiveBundles.find_first(); i>=0;
+         i = Cand.LiveBundles.find_next(i))
+    dbgs() << " EB#" << i;
+    dbgs() << ".\n";
+  });
+  return true;
+}
+
 /// calcSpillCost - Compute how expensive it would be to split the live range in
 /// SA around all use blocks instead of forming bundle regions.
 float RAGreedy::calcSpillCost() {
   float Cost = 0;
-  const LiveInterval &LI = SA->getParent();
   ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
   for (unsigned i = 0; i != UseBlocks.size(); ++i) {
     const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
@@ -772,16 +882,8 @@ float RAGreedy::calcSpillCost() {
     Cost += SpillPlacer->getBlockFrequency(Number);
 
     // Unless the value is redefined in the block.
-    if (BI.LiveIn && BI.LiveOut) {
-      SlotIndex Start, Stop;
-      tie(Start, Stop) = Indexes->getMBBRange(Number);
-      LiveInterval::const_iterator I = LI.find(Start);
-      assert(I != LI.end() && "Expected live-in value");
-      // Is there a different live-out value? If so, we need an extra spill
-      // instruction.
-      if (I->end < Stop)
-        Cost += SpillPlacer->getBlockFrequency(Number);
-    }
+    if (BI.LiveIn && BI.LiveOut && BI.FirstDef)
+      Cost += SpillPlacer->getBlockFrequency(Number);
   }
   return Cost;
 }
@@ -828,81 +930,115 @@ float RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand) {
   return GlobalCost;
 }
 
-/// splitAroundRegion - Split VirtReg around the region determined by
-/// LiveBundles. Make an effort to avoid interference from PhysReg.
+/// splitAroundRegion - Split the current live range around the regions
+/// determined by BundleCand and GlobalCand.
 ///
-/// The 'register' interval is going to contain as many uses as possible while
-/// avoiding interference. The 'stack' interval is the complement constructed by
-/// SplitEditor. It will contain the rest.
+/// Before calling this function, GlobalCand and BundleCand must be initialized
+/// so each bundle is assigned to a valid candidate, or NoCand for the
+/// stack-bound bundles.  The shared SA/SE SplitAnalysis and SplitEditor
+/// objects must be initialized for the current live range, and intervals
+/// created for the used candidates.
 ///
-void RAGreedy::splitAroundRegion(LiveInterval &VirtReg,
-                                 GlobalSplitCandidate &Cand,
-                                 SmallVectorImpl<LiveInterval*> &NewVRegs) {
-  const BitVector &LiveBundles = Cand.LiveBundles;
+/// @param LREdit    The LiveRangeEdit object handling the current split.
+/// @param UsedCands List of used GlobalCand entries. Every BundleCand value
+///                  must appear in this list.
+void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
+                                 ArrayRef<unsigned> UsedCands) {
+  // These are the intervals created for new global ranges. We may create more
+  // intervals for local ranges.
+  const unsigned NumGlobalIntvs = LREdit.size();
+  DEBUG(dbgs() << "splitAroundRegion with " << NumGlobalIntvs << " globals.\n");
+  assert(NumGlobalIntvs && "No global intervals configured");
 
-  DEBUG({
-    dbgs() << "Splitting around region for " << PrintReg(Cand.PhysReg, TRI)
-           << " with bundles";
-    for (int i = LiveBundles.find_first(); i>=0; i = LiveBundles.find_next(i))
-      dbgs() << " EB#" << i;
-    dbgs() << ".\n";
-  });
-
-  InterferenceCache::Cursor &Intf = Cand.Intf;
-  LiveRangeEdit LREdit(VirtReg, NewVRegs, this);
-  SE->reset(LREdit);
-
-  // Create the main cross-block interval.
-  const unsigned MainIntv = SE->openIntv();
+  // Isolate even single instructions when dealing with a proper sub-class.
+  // That guarantees register class inflation for the stack interval because it
+  // is all copies.
+  unsigned Reg = SA->getParent().reg;
+  bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));
 
   // First handle all the blocks with uses.
   ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
   for (unsigned i = 0; i != UseBlocks.size(); ++i) {
     const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
-    bool RegIn  = BI.LiveIn &&
-                  LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)];
-    bool RegOut = BI.LiveOut &&
-                  LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)];
+    unsigned Number = BI.MBB->getNumber();
+    unsigned IntvIn = 0, IntvOut = 0;
+    SlotIndex IntfIn, IntfOut;
+    if (BI.LiveIn) {
+      unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)];
+      if (CandIn != NoCand) {
+        GlobalSplitCandidate &Cand = GlobalCand[CandIn];
+        IntvIn = Cand.IntvIdx;
+        Cand.Intf.moveToBlock(Number);
+        IntfIn = Cand.Intf.first();
+      }
+    }
+    if (BI.LiveOut) {
+      unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)];
+      if (CandOut != NoCand) {
+        GlobalSplitCandidate &Cand = GlobalCand[CandOut];
+        IntvOut = Cand.IntvIdx;
+        Cand.Intf.moveToBlock(Number);
+        IntfOut = Cand.Intf.last();
+      }
+    }
 
     // Create separate intervals for isolated blocks with multiple uses.
-    if (!RegIn && !RegOut) {
+    if (!IntvIn && !IntvOut) {
       DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " isolated.\n");
-      if (!BI.isOneInstr()) {
+      if (SA->shouldSplitSingleBlock(BI, SingleInstrs))
         SE->splitSingleBlock(BI);
-        SE->selectIntv(MainIntv);
-      }
       continue;
     }
 
-    Intf.moveToBlock(BI.MBB->getNumber());
-
-    if (RegIn && RegOut)
-      SE->splitLiveThroughBlock(BI.MBB->getNumber(),
-                                MainIntv, Intf.first(),
-                                MainIntv, Intf.last());
-    else if (RegIn)
-      SE->splitRegInBlock(BI, MainIntv, Intf.first());
+    if (IntvIn && IntvOut)
+      SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut);
+    else if (IntvIn)
+      SE->splitRegInBlock(BI, IntvIn, IntfIn);
     else
-      SE->splitRegOutBlock(BI, MainIntv, Intf.last());
+      SE->splitRegOutBlock(BI, IntvOut, IntfOut);
   }
 
-  // Handle live-through blocks.
-  for (unsigned i = 0, e = Cand.ActiveBlocks.size(); i != e; ++i) {
-    unsigned Number = Cand.ActiveBlocks[i];
-    bool RegIn  = LiveBundles[Bundles->getBundle(Number, 0)];
-    bool RegOut = LiveBundles[Bundles->getBundle(Number, 1)];
-    if (!RegIn && !RegOut)
-      continue;
-    Intf.moveToBlock(Number);
-    SE->splitLiveThroughBlock(Number, RegIn  ? MainIntv : 0, Intf.first(),
-                                      RegOut ? MainIntv : 0, Intf.last());
+  // Handle live-through blocks. The relevant live-through blocks are stored in
+  // the ActiveBlocks list with each candidate. We need to filter out
+  // duplicates.
+  BitVector Todo = SA->getThroughBlocks();
+  for (unsigned c = 0; c != UsedCands.size(); ++c) {
+    ArrayRef<unsigned> Blocks = GlobalCand[UsedCands[c]].ActiveBlocks;
+    for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
+      unsigned Number = Blocks[i];
+      if (!Todo.test(Number))
+        continue;
+      Todo.reset(Number);
+
+      unsigned IntvIn = 0, IntvOut = 0;
+      SlotIndex IntfIn, IntfOut;
+
+      unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)];
+      if (CandIn != NoCand) {
+        GlobalSplitCandidate &Cand = GlobalCand[CandIn];
+        IntvIn = Cand.IntvIdx;
+        Cand.Intf.moveToBlock(Number);
+        IntfIn = Cand.Intf.first();
+      }
+
+      unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)];
+      if (CandOut != NoCand) {
+        GlobalSplitCandidate &Cand = GlobalCand[CandOut];
+        IntvOut = Cand.IntvIdx;
+        Cand.Intf.moveToBlock(Number);
+        IntfOut = Cand.Intf.last();
+      }
+      if (!IntvIn && !IntvOut)
+        continue;
+      SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut);
+    }
   }
 
   ++NumGlobalSplits;
 
   SmallVector<unsigned, 8> IntvMap;
   SE->finish(&IntvMap);
-  DebugVars->splitRegister(VirtReg.reg, LREdit.regs());
+  DebugVars->splitRegister(Reg, LREdit.regs());
 
   ExtraRegInfo.resize(MRI->getNumVirtRegs());
   unsigned OrigBlocks = SA->getNumLiveBlocks();
@@ -922,18 +1058,18 @@ void RAGreedy::splitAroundRegion(LiveInterval &VirtReg,
     // Remainder interval. Don't try splitting again, spill if it doesn't
     // allocate.
     if (IntvMap[i] == 0) {
-      setStage(Reg, RS_Global);
+      setStage(Reg, RS_Spill);
       continue;
     }
 
-    // Main interval. Allow repeated splitting as long as the number of live
+    // Global intervals. Allow repeated splitting as long as the number of live
     // blocks is strictly decreasing.
-    if (IntvMap[i] == MainIntv) {
+    if (IntvMap[i] < NumGlobalIntvs) {
       if (SA->countLiveBlocks(&Reg) >= OrigBlocks) {
         DEBUG(dbgs() << "Main interval covers the same " << OrigBlocks
                      << " blocks as original.\n");
         // Don't allow repeated splitting as a safe guard against looping.
-        setStage(Reg, RS_Global);
+        setStage(Reg, RS_Split2);
       }
       continue;
     }
@@ -948,11 +1084,23 @@ void RAGreedy::splitAroundRegion(LiveInterval &VirtReg,
 
 unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
                                   SmallVectorImpl<LiveInterval*> &NewVRegs) {
-  float BestCost = Hysteresis * calcSpillCost();
-  DEBUG(dbgs() << "Cost of isolating all blocks = " << BestCost << '\n');
-  const unsigned NoCand = ~0u;
-  unsigned BestCand = NoCand;
   unsigned NumCands = 0;
+  unsigned BestCand = NoCand;
+  float BestCost;
+  SmallVector<unsigned, 8> UsedCands;
+
+  // Check if we can split this live range around a compact region.
+  bool HasCompact = calcCompactRegion(GlobalCand.front());
+  if (HasCompact) {
+    // Yes, keep GlobalCand[0] as the compact region candidate.
+    NumCands = 1;
+    BestCost = HUGE_VALF;
+  } else {
+    // No benefit from the compact region, our fallback will be per-block
+    // splitting. Make sure we find a solution that is cheaper than spilling.
+    BestCost = Hysteresis * calcSpillCost();
+    DEBUG(dbgs() << "Cost of isolating all blocks = " << BestCost << '\n');
+  }
 
   Order.rewind();
   while (unsigned PhysReg = Order.next()) {
@@ -962,7 +1110,7 @@ unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
       unsigned WorstCount = ~0u;
       unsigned Worst = 0;
       for (unsigned i = 0; i != NumCands; ++i) {
-        if (i == BestCand)
+        if (i == BestCand || !GlobalCand[i].PhysReg)
           continue;
         unsigned Count = GlobalCand[i].LiveBundles.count();
         if (Count < WorstCount)
@@ -1019,14 +1167,93 @@ unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
     ++NumCands;
   }
 
-  if (BestCand == NoCand)
+  // No solutions found, fall back to single block splitting.
+  if (!HasCompact && BestCand == NoCand)
     return 0;
 
-  splitAroundRegion(VirtReg, GlobalCand[BestCand], NewVRegs);
+  // Prepare split editor.
+  LiveRangeEdit LREdit(VirtReg, NewVRegs, this);
+  SE->reset(LREdit, SplitSpillMode);
+
+  // Assign all edge bundles to the preferred candidate, or NoCand.
+  BundleCand.assign(Bundles->getNumBundles(), NoCand);
+
+  // Assign bundles for the best candidate region.
+  if (BestCand != NoCand) {
+    GlobalSplitCandidate &Cand = GlobalCand[BestCand];
+    if (unsigned B = Cand.getBundles(BundleCand, BestCand)) {
+      UsedCands.push_back(BestCand);
+      Cand.IntvIdx = SE->openIntv();
+      DEBUG(dbgs() << "Split for " << PrintReg(Cand.PhysReg, TRI) << " in "
+                   << B << " bundles, intv " << Cand.IntvIdx << ".\n");
+      (void)B;
+    }
+  }
+
+  // Assign bundles for the compact region.
+  if (HasCompact) {
+    GlobalSplitCandidate &Cand = GlobalCand.front();
+    assert(!Cand.PhysReg && "Compact region has no physreg");
+    if (unsigned B = Cand.getBundles(BundleCand, 0)) {
+      UsedCands.push_back(0);
+      Cand.IntvIdx = SE->openIntv();
+      DEBUG(dbgs() << "Split for compact region in " << B << " bundles, intv "
+                   << Cand.IntvIdx << ".\n");
+      (void)B;
+    }
+  }
+
+  splitAroundRegion(LREdit, UsedCands);
   return 0;
 }
 
 
+//===----------------------------------------------------------------------===//
+//                            Per-Block Splitting
+//===----------------------------------------------------------------------===//
+
+/// tryBlockSplit - Split a global live range around every block with uses. This
+/// creates a lot of local live ranges, that will be split by tryLocalSplit if
+/// they don't allocate.
+unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order,
+                                 SmallVectorImpl<LiveInterval*> &NewVRegs) {
+  assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed");
+  unsigned Reg = VirtReg.reg;
+  bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));
+  LiveRangeEdit LREdit(VirtReg, NewVRegs, this);
+  SE->reset(LREdit, SplitSpillMode);
+  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
+  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
+    const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
+    if (SA->shouldSplitSingleBlock(BI, SingleInstrs))
+      SE->splitSingleBlock(BI);
+  }
+  // No blocks were split.
+  if (LREdit.empty())
+    return 0;
+
+  // We did split for some blocks.
+  SmallVector<unsigned, 8> IntvMap;
+  SE->finish(&IntvMap);
+
+  // Tell LiveDebugVariables about the new ranges.
+  DebugVars->splitRegister(Reg, LREdit.regs());
+
+  ExtraRegInfo.resize(MRI->getNumVirtRegs());
+
+  // Sort out the new intervals created by splitting. The remainder interval
+  // goes straight to spilling, the new local ranges get to stay RS_New.
+  for (unsigned i = 0, e = LREdit.size(); i != e; ++i) {
+    LiveInterval &LI = *LREdit.get(i);
+    if (getStage(LI) == RS_New && IntvMap[i] == 0)
+      setStage(LI, RS_Spill);
+  }
+
+  if (VerifyEnabled)
+    MF->verify(this, "After splitting live range around basic blocks");
+  return 0;
+}
+
 //===----------------------------------------------------------------------===//
 //                             Local Splitting
 //===----------------------------------------------------------------------===//
@@ -1045,8 +1272,10 @@ void RAGreedy::calcGapWeights(unsigned PhysReg,
   const unsigned NumGaps = Uses.size()-1;
 
   // Start and end points for the interference check.
-  SlotIndex StartIdx = BI.LiveIn ? BI.FirstUse.getBaseIndex() : BI.FirstUse;
-  SlotIndex StopIdx = BI.LiveOut ? BI.LastUse.getBoundaryIndex() : BI.LastUse;
+  SlotIndex StartIdx =
+    BI.LiveIn ? BI.FirstInstr.getBaseIndex() : BI.FirstInstr;
+  SlotIndex StopIdx =
+    BI.LiveOut ? BI.LastInstr.getBoundaryIndex() : BI.LastInstr;
 
   GapWeight.assign(NumGaps, 0.0f);
 
@@ -1056,8 +1285,8 @@ void RAGreedy::calcGapWeights(unsigned PhysReg,
            .checkInterference())
       continue;
 
-    // We know that VirtReg is a continuous interval from FirstUse to LastUse,
-    // so we don't need InterferenceQuery.
+    // We know that VirtReg is a continuous interval from FirstInstr to
+    // LastInstr, so we don't need InterferenceQuery.
     //
     // Interference that overlaps an instruction is counted in both gaps
     // surrounding the instruction. The exception is interference before
@@ -1097,8 +1326,8 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
   // while only covering a single block - A phi-def can use undef values from
   // predecessors, and the block could be a single-block loop.
   // We don't bother doing anything clever about such a case, we simply assume
-  // that the interval is continuous from FirstUse to LastUse. We should make
-  // sure that we don't do anything illegal to such an interval, though.
+  // that the interval is continuous from FirstInstr to LastInstr. We should
+  // make sure that we don't do anything illegal to such an interval, though.
 
   const SmallVectorImpl<SlotIndex> &Uses = SA->UseSlots;
   if (Uses.size() <= 2)
@@ -1120,17 +1349,17 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
   //
   // Instead we use these rules:
   //
-  // 1. Allow any split for ranges with getStage() < RS_Local. (Except for the
+  // 1. Allow any split for ranges with getStage() < RS_Split2. (Except for the
   //    noop split, of course).
-  // 2. Require progress be made for ranges with getStage() >= RS_Local. All
+  // 2. Require progress be made for ranges with getStage() == RS_Split2. All
   //    the new ranges must have fewer instructions than before the split.
-  // 3. New ranges with the same number of instructions are marked RS_Local,
+  // 3. New ranges with the same number of instructions are marked RS_Split2,
   //    smaller ranges are marked RS_New.
   //
   // These rules allow a 3 -> 2+3 split once, which we need. They also prevent
   // excessive splitting and infinite loops.
   //
-  bool ProgressRequired = getStage(VirtReg) >= RS_Local;
+  bool ProgressRequired = getStage(VirtReg) >= RS_Split2;
 
   // Best split candidate.
   unsigned BestBefore = NumGaps;
@@ -1249,7 +1478,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
   DebugVars->splitRegister(VirtReg.reg, LREdit.regs());
 
   // If the new range has the same number of instructions as before, mark it as
-  // RS_Local so the next split will be forced to make progress. Otherwise,
+  // RS_Split2 so the next split will be forced to make progress. Otherwise,
   // leave the new intervals as RS_New so they can compete.
   bool LiveBefore = BestBefore != 0 || BI.LiveIn;
   bool LiveAfter = BestAfter != NumGaps || BI.LiveOut;
@@ -1259,7 +1488,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
     assert(!ProgressRequired && "Didn't make progress when it was required.");
     for (unsigned i = 0, e = IntvMap.size(); i != e; ++i)
       if (IntvMap[i] == 1) {
-        setStage(*LREdit.get(i), RS_Local);
+        setStage(*LREdit.get(i), RS_Split2);
         DEBUG(dbgs() << PrintReg(LREdit.get(i)->reg));
       }
     DEBUG(dbgs() << '\n');
@@ -1278,6 +1507,10 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
 /// @return Physreg when VirtReg may be assigned and/or new NewVRegs.
 unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
                             SmallVectorImpl<LiveInterval*>&NewVRegs) {
+  // Ranges must be Split2 or less.
+  if (getStage(VirtReg) >= RS_Spill)
+    return 0;
+
   // Local intervals are handled separately.
   if (LIS->intervalIsInOneMBB(VirtReg)) {
     NamedRegionTimer T("Local Splitting", TimerGroupName, TimePassesIsEnabled);
@@ -1287,11 +1520,6 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
 
   NamedRegionTimer T("Global Splitting", TimerGroupName, TimePassesIsEnabled);
 
-  // Don't iterate global splitting.
-  // Move straight to spilling if this range was produced by a global split.
-  if (getStage(VirtReg) >= RS_Global)
-    return 0;
-
   SA->analyze(&VirtReg);
 
   // FIXME: SplitAnalysis may repair broken live ranges coming from the
@@ -1305,24 +1533,17 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
       return PhysReg;
   }
 
-  // First try to split around a region spanning multiple blocks.
-  unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs);
-  if (PhysReg || !NewVRegs.empty())
-    return PhysReg;
-
-  // Then isolate blocks with multiple uses.
-  SplitAnalysis::BlockPtrSet Blocks;
-  if (SA->getMultiUseBlocks(Blocks)) {
-    LiveRangeEdit LREdit(VirtReg, NewVRegs, this);
-    SE->reset(LREdit);
-    SE->splitSingleBlocks(Blocks);
-    setStage(NewVRegs.begin(), NewVRegs.end(), RS_Global);
-    if (VerifyEnabled)
-      MF->verify(this, "After splitting live range around basic blocks");
+  // First try to split around a region spanning multiple blocks. RS_Split2
+  // ranges already made dubious progress with region splitting, so they go
+  // straight to single block splitting.
+  if (getStage(VirtReg) < RS_Split2) {
+    unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs);
+    if (PhysReg || !NewVRegs.empty())
+      return PhysReg;
   }
 
-  // Don't assign any physregs.
-  return 0;
+  // Then isolate blocks.
+  return tryBlockSplit(VirtReg, Order, NewVRegs);
 }
 
 
@@ -1342,9 +1563,9 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
                << " Cascade " << ExtraRegInfo[VirtReg.reg].Cascade << '\n');
 
   // Try to evict a less worthy live range, but only for ranges from the primary
-  // queue. The RS_Second ranges already failed to do this, and they should not
+  // queue. The RS_Split ranges already failed to do this, and they should not
   // get a second chance until they have been split.
-  if (Stage != RS_Second)
+  if (Stage != RS_Split)
     if (unsigned PhysReg = tryEvict(VirtReg, Order, NewVRegs))
       return PhysReg;
 
@@ -1353,8 +1574,8 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
   // The first time we see a live range, don't try to split or spill.
   // Wait until the second time, when all smaller ranges have been allocated.
   // This gives a better picture of the interference to split around.
-  if (Stage == RS_First) {
-    setStage(VirtReg, RS_Second);
+  if (Stage < RS_Split) {
+    setStage(VirtReg, RS_Split);
     DEBUG(dbgs() << "wait for second round\n");
     NewVRegs.push_back(&VirtReg);
     return 0;
@@ -1362,7 +1583,7 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
 
   // If we couldn't allocate a register from spilling, there is probably some
   // invalid inline assembly. The base class wil report it.
-  if (Stage >= RS_Spill || !VirtReg.isSpillable())
+  if (Stage >= RS_Done || !VirtReg.isSpillable())
     return ~0u;
 
   // Try splitting VirtReg or interferences.
@@ -1374,7 +1595,7 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
   NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled);
   LiveRangeEdit LRE(VirtReg, NewVRegs, this);
   spiller().spill(LRE);
-  setStage(NewVRegs.begin(), NewVRegs.end(), RS_Spill);
+  setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);
 
   if (VerifyEnabled)
     MF->verify(this, "After spilling");
@@ -1408,6 +1629,7 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
   ExtraRegInfo.resize(MRI->getNumVirtRegs());
   NextCascade = 1;
   IntfCache.init(MF, &PhysReg2LiveUnion[0], Indexes, TRI);
+  GlobalCand.resize(32);  // This will grow as needed.
 
   allocatePhysRegs();
   addMBBLiveIns(MF);
@@ -1420,7 +1642,10 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
   }
 
   // Write out new DBG_VALUE instructions.
-  DebugVars->emitDebugValues(VRM);
+  {
+    NamedRegionTimer T("Emit Debug Info", TimerGroupName, TimePassesIsEnabled);
+    DebugVars->emitDebugValues(VRM);
+  }
 
   // The pass output is in VirtRegMap. Release all the transient data.
   releaseMemory();
diff --git a/contrib/llvm/lib/CodeGen/RegAllocLinearScan.cpp b/contrib/llvm/lib/CodeGen/RegAllocLinearScan.cpp
index 0dd3c598c154..ce3fb90b1126 100644
--- a/contrib/llvm/lib/CodeGen/RegAllocLinearScan.cpp
+++ b/contrib/llvm/lib/CodeGen/RegAllocLinearScan.cpp
@@ -18,7 +18,6 @@
 #include "VirtRegRewriter.h"
 #include "RegisterClassInfo.h"
 #include "Spiller.h"
-#include "RegisterCoalescer.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Function.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
@@ -209,7 +208,7 @@ namespace {
         AU.addRequiredID(StrongPHIEliminationID);
       // Make sure PassManager knows which analyses to make available
       // to coalescing and which analyses coalescing invalidates.
-      AU.addRequiredTransitive<RegisterCoalescer>();
+      AU.addRequiredTransitiveID(RegisterCoalescerPassID);
       AU.addRequired<CalculateSpillWeights>();
       AU.addRequiredID(LiveStacksID);
       AU.addPreservedID(LiveStacksID);
diff --git a/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp b/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp
index 72230d4b0c5c..0d2cf2d6184c 100644
--- a/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp
+++ b/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp
@@ -450,7 +450,7 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const {
   au.addPreserved<SlotIndexes>();
   au.addRequired<LiveIntervals>();
   //au.addRequiredID(SplitCriticalEdgesID);
-  au.addRequired<RegisterCoalescer>();
+  au.addRequiredID(RegisterCoalescerPassID);
   if (customPassID)
     au.addRequiredID(*customPassID);
   au.addRequired<CalculateSpillWeights>();
diff --git a/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp b/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp
index 5a77e47bc591..786d279c2b8c 100644
--- a/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp
+++ b/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp
@@ -99,11 +99,16 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const {
   // CSR aliases go after the volatile registers, preserve the target's order.
   std::copy(CSRAlias.begin(), CSRAlias.end(), &RCI.Order[N]);
 
+  // Check if RC is a proper sub-class.
+  if (const TargetRegisterClass *Super = TRI->getLargestLegalSuperClass(RC))
+    if (Super != RC && getNumAllocatableRegs(Super) > RCI.NumRegs)
+      RCI.ProperSubClass = true;
+
   DEBUG({
     dbgs() << "AllocationOrder(" << RC->getName() << ") = [";
     for (unsigned I = 0; I != RCI.NumRegs; ++I)
       dbgs() << ' ' << PrintReg(RCI.Order[I], TRI);
-    dbgs() << " ]\n";
+    dbgs() << (RCI.ProperSubClass ? " ] (sub-class)\n" : " ]\n");
   });
 
   // RCI is now up-to-date.
diff --git a/contrib/llvm/lib/CodeGen/RegisterClassInfo.h b/contrib/llvm/lib/CodeGen/RegisterClassInfo.h
index d21fd67efe8b..2c1407096cd7 100644
--- a/contrib/llvm/lib/CodeGen/RegisterClassInfo.h
+++ b/contrib/llvm/lib/CodeGen/RegisterClassInfo.h
@@ -28,11 +28,12 @@ class RegisterClassInfo {
   struct RCInfo {
     unsigned Tag;
     unsigned NumRegs;
+    bool ProperSubClass;
     OwningArrayPtr<unsigned> Order;
 
-    RCInfo() : Tag(0), NumRegs(0) {}
+    RCInfo() : Tag(0), NumRegs(0), ProperSubClass(false) {}
     operator ArrayRef<unsigned>() const {
-      return ArrayRef<unsigned>(Order.get(), NumRegs);
+      return makeArrayRef(Order.get(), NumRegs);
     }
   };
 
@@ -87,6 +88,16 @@ class RegisterClassInfo {
     return get(RC);
   }
 
+  /// isProperSubClass - Returns true if RC has a legal super-class with more
+  /// allocatable registers.
+  ///
+  /// Register classes like GR32_NOSP are not proper sub-classes because %esp
+  /// is not allocatable.  Similarly, tGPR is not a proper sub-class in Thumb
+  /// mode because the GPR super-class is not legal.
+  bool isProperSubClass(const TargetRegisterClass *RC) const {
+    return get(RC).ProperSubClass;
+  }
+
   /// getLastCalleeSavedAlias - Returns the last callee saved register that
   /// overlaps PhysReg, or 0 if Reg doesn't overlap a CSR.
   unsigned getLastCalleeSavedAlias(unsigned PhysReg) const {
diff --git a/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp b/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp
index b91f92c6aa5a..9b414d6212c7 100644
--- a/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -15,8 +15,9 @@
 
 #define DEBUG_TYPE "regcoalescing"
 #include "RegisterCoalescer.h"
-#include "VirtRegMap.h"
 #include "LiveDebugVariables.h"
+#include "RegisterClassInfo.h"
+#include "VirtRegMap.h"
 
 #include "llvm/Pass.h"
 #include "llvm/Value.h"
@@ -54,6 +55,7 @@ STATISTIC(numExtends  , "Number of copies extended");
 STATISTIC(NumReMats   , "Number of instructions re-materialized");
 STATISTIC(numPeep     , "Number of identity moves eliminated after coalescing");
 STATISTIC(numAborts   , "Number of times interval joining aborted");
+STATISTIC(NumInflated , "Number of register classes inflated");
 
 static cl::opt<bool>
 EnableJoining("join-liveintervals",
@@ -75,6 +77,128 @@ VerifyCoalescing("verify-coalescing",
          cl::desc("Verify machine instrs before and after register coalescing"),
          cl::Hidden);
 
+namespace {
+  class RegisterCoalescer : public MachineFunctionPass {
+    MachineFunction* MF;
+    MachineRegisterInfo* MRI;
+    const TargetMachine* TM;
+    const TargetRegisterInfo* TRI;
+    const TargetInstrInfo* TII;
+    LiveIntervals *LIS;
+    LiveDebugVariables *LDV;
+    const MachineLoopInfo* Loops;
+    AliasAnalysis *AA;
+    RegisterClassInfo RegClassInfo;
+
+    /// JoinedCopies - Keep track of copies eliminated due to coalescing.
+    ///
+    SmallPtrSet<MachineInstr*, 32> JoinedCopies;
+
+    /// ReMatCopies - Keep track of copies eliminated due to remat.
+    ///
+    SmallPtrSet<MachineInstr*, 32> ReMatCopies;
+
+    /// ReMatDefs - Keep track of definition instructions which have
+    /// been remat'ed.
+    SmallPtrSet<MachineInstr*, 8> ReMatDefs;
+
+    /// joinIntervals - join compatible live intervals
+    void joinIntervals();
+
+    /// CopyCoalesceInMBB - Coalesce copies in the specified MBB, putting
+    /// copies that cannot yet be coalesced into the "TryAgain" list.
+    void CopyCoalesceInMBB(MachineBasicBlock *MBB,
+                           std::vector<MachineInstr*> &TryAgain);
+
+    /// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg,
+    /// which are the src/dst of the copy instruction CopyMI.  This returns
+    /// true if the copy was successfully coalesced away. If it is not
+    /// currently possible to coalesce this interval, but it may be possible if
+    /// other things get coalesced, then it returns true by reference in
+    /// 'Again'.
+    bool JoinCopy(MachineInstr *TheCopy, bool &Again);
+
+    /// JoinIntervals - Attempt to join these two intervals.  On failure, this
+    /// returns false.  The output "SrcInt" will not have been modified, so we
+    /// can use this information below to update aliases.
+    bool JoinIntervals(CoalescerPair &CP);
+
+    /// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy. If
+    /// the source value number is defined by a copy from the destination reg
+    /// see if we can merge these two destination reg valno# into a single
+    /// value number, eliminating a copy.
+    bool AdjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI);
+
+    /// HasOtherReachingDefs - Return true if there are definitions of IntB
+    /// other than BValNo val# that can reach uses of AValno val# of IntA.
+    bool HasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB,
+                              VNInfo *AValNo, VNInfo *BValNo);
+
+    /// RemoveCopyByCommutingDef - We found a non-trivially-coalescable copy.
+    /// If the source value number is defined by a commutable instruction and
+    /// its other operand is coalesced to the copy dest register, see if we
+    /// can transform the copy into a noop by commuting the definition.
+    bool RemoveCopyByCommutingDef(const CoalescerPair &CP,MachineInstr *CopyMI);
+
+    /// ReMaterializeTrivialDef - If the source of a copy is defined by a
+    /// trivial computation, replace the copy by rematerialize the definition.
+    /// If PreserveSrcInt is true, make sure SrcInt is valid after the call.
+    bool ReMaterializeTrivialDef(LiveInterval &SrcInt, bool PreserveSrcInt,
+                                 unsigned DstReg, MachineInstr *CopyMI);
+
+    /// shouldJoinPhys - Return true if a physreg copy should be joined.
+    bool shouldJoinPhys(CoalescerPair &CP);
+
+    /// isWinToJoinCrossClass - Return true if it's profitable to coalesce
+    /// two virtual registers from different register classes.
+    bool isWinToJoinCrossClass(unsigned SrcReg,
+                               unsigned DstReg,
+                               const TargetRegisterClass *SrcRC,
+                               const TargetRegisterClass *DstRC,
+                               const TargetRegisterClass *NewRC);
+
+    /// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and
+    /// update the subregister number if it is not zero. If DstReg is a
+    /// physical register and the existing subregister number of the def / use
+    /// being updated is not zero, make sure to set it to the correct physical
+    /// subregister.
+    void UpdateRegDefsUses(const CoalescerPair &CP);
+
+    /// RemoveDeadDef - If a def of a live interval is now determined dead,
+    /// remove the val# it defines. If the live interval becomes empty, remove
+    /// it as well.
+    bool RemoveDeadDef(LiveInterval &li, MachineInstr *DefMI);
+
+    /// RemoveCopyFlag - If DstReg is no longer defined by CopyMI, clear the
+    /// VNInfo copy flag for DstReg and all aliases.
+    void RemoveCopyFlag(unsigned DstReg, const MachineInstr *CopyMI);
+
+    /// markAsJoined - Remember that CopyMI has already been joined.
+    void markAsJoined(MachineInstr *CopyMI);
+
+    /// eliminateUndefCopy - Handle copies of undef values.
+    bool eliminateUndefCopy(MachineInstr *CopyMI, const CoalescerPair &CP);
+
+  public:
+    static char ID; // Class identification, replacement for typeinfo
+    RegisterCoalescer() : MachineFunctionPass(ID) {
+      initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry());
+    }
+
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+
+    virtual void releaseMemory();
+
+    /// runOnMachineFunction - pass entry point
+    virtual bool runOnMachineFunction(MachineFunction&);
+
+    /// print - Implement the dump method.
+    virtual void print(raw_ostream &O, const Module* = 0) const;
+  };
+} /// end anonymous namespace
+
+char &llvm::RegisterCoalescerPassID = RegisterCoalescer::ID;
+
 INITIALIZE_PASS_BEGIN(RegisterCoalescer, "simple-register-coalescing",
                       "Simple Register Coalescing", false, false)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
@@ -116,14 +240,14 @@ static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI,
 }
 
 bool CoalescerPair::setRegisters(const MachineInstr *MI) {
-  srcReg_ = dstReg_ = subIdx_ = 0;
-  newRC_ = 0;
-  flipped_ = crossClass_ = false;
+  SrcReg = DstReg = SubIdx = 0;
+  NewRC = 0;
+  Flipped = CrossClass = false;
 
   unsigned Src, Dst, SrcSub, DstSub;
-  if (!isMoveInstr(tri_, MI, Src, Dst, SrcSub, DstSub))
+  if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub))
     return false;
-  partial_ = SrcSub || DstSub;
+  Partial = SrcSub || DstSub;
 
   // If one register is a physreg, it must be Dst.
   if (TargetRegisterInfo::isPhysicalRegister(Src)) {
@@ -131,7 +255,7 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) {
       return false;
     std::swap(Src, Dst);
     std::swap(SrcSub, DstSub);
-    flipped_ = true;
+    Flipped = true;
   }
 
   const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo();
@@ -139,14 +263,14 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) {
   if (TargetRegisterInfo::isPhysicalRegister(Dst)) {
     // Eliminate DstSub on a physreg.
     if (DstSub) {
-      Dst = tri_.getSubReg(Dst, DstSub);
+      Dst = TRI.getSubReg(Dst, DstSub);
       if (!Dst) return false;
       DstSub = 0;
     }
 
     // Eliminate SrcSub by picking a corresponding Dst superregister.
     if (SrcSub) {
-      Dst = tri_.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src));
+      Dst = TRI.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src));
       if (!Dst) return false;
       SrcSub = 0;
     } else if (!MRI.getRegClass(Src)->contains(Dst)) {
@@ -164,7 +288,7 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) {
         return false;
       const TargetRegisterClass *SrcRC = MRI.getRegClass(Src);
       const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
-      if (!getCommonSubClass(DstRC, SrcRC))
+      if (!TRI.getCommonSubClass(DstRC, SrcRC))
         return false;
       SrcSub = DstSub = 0;
     }
@@ -174,36 +298,36 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) {
       std::swap(Src, Dst);
       DstSub = SrcSub;
       SrcSub = 0;
-      assert(!flipped_ && "Unexpected flip");
-      flipped_ = true;
+      assert(!Flipped && "Unexpected flip");
+      Flipped = true;
     }
 
     // Find the new register class.
     const TargetRegisterClass *SrcRC = MRI.getRegClass(Src);
     const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
     if (DstSub)
-      newRC_ = tri_.getMatchingSuperRegClass(DstRC, SrcRC, DstSub);
+      NewRC = TRI.getMatchingSuperRegClass(DstRC, SrcRC, DstSub);
     else
-      newRC_ = getCommonSubClass(DstRC, SrcRC);
-    if (!newRC_)
+      NewRC = TRI.getCommonSubClass(DstRC, SrcRC);
+    if (!NewRC)
       return false;
-    crossClass_ = newRC_ != DstRC || newRC_ != SrcRC;
+    CrossClass = NewRC != DstRC || NewRC != SrcRC;
   }
   // Check our invariants
   assert(TargetRegisterInfo::isVirtualRegister(Src) && "Src must be virtual");
   assert(!(TargetRegisterInfo::isPhysicalRegister(Dst) && DstSub) &&
          "Cannot have a physical SubIdx");
-  srcReg_ = Src;
-  dstReg_ = Dst;
-  subIdx_ = DstSub;
+  SrcReg = Src;
+  DstReg = Dst;
+  SubIdx = DstSub;
   return true;
 }
 
 bool CoalescerPair::flip() {
-  if (subIdx_ || TargetRegisterInfo::isPhysicalRegister(dstReg_))
+  if (SubIdx || TargetRegisterInfo::isPhysicalRegister(DstReg))
     return false;
-  std::swap(srcReg_, dstReg_);
-  flipped_ = !flipped_;
+  std::swap(SrcReg, DstReg);
+  Flipped = !Flipped;
   return true;
 }
 
@@ -211,36 +335,36 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const {
   if (!MI)
     return false;
   unsigned Src, Dst, SrcSub, DstSub;
-  if (!isMoveInstr(tri_, MI, Src, Dst, SrcSub, DstSub))
+  if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub))
     return false;
 
-  // Find the virtual register that is srcReg_.
-  if (Dst == srcReg_) {
+  // Find the virtual register that is SrcReg.
+  if (Dst == SrcReg) {
     std::swap(Src, Dst);
     std::swap(SrcSub, DstSub);
-  } else if (Src != srcReg_) {
+  } else if (Src != SrcReg) {
     return false;
   }
 
-  // Now check that Dst matches dstReg_.
-  if (TargetRegisterInfo::isPhysicalRegister(dstReg_)) {
+  // Now check that Dst matches DstReg.
+  if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
     if (!TargetRegisterInfo::isPhysicalRegister(Dst))
       return false;
-    assert(!subIdx_ && "Inconsistent CoalescerPair state.");
+    assert(!SubIdx && "Inconsistent CoalescerPair state.");
     // DstSub could be set for a physreg from INSERT_SUBREG.
     if (DstSub)
-      Dst = tri_.getSubReg(Dst, DstSub);
+      Dst = TRI.getSubReg(Dst, DstSub);
     // Full copy of Src.
     if (!SrcSub)
-      return dstReg_ == Dst;
+      return DstReg == Dst;
     // This is a partial register copy. Check that the parts match.
-    return tri_.getSubReg(dstReg_, SrcSub) == Dst;
+    return TRI.getSubReg(DstReg, SrcSub) == Dst;
   } else {
-    // dstReg_ is virtual.
-    if (dstReg_ != Dst)
+    // DstReg is virtual.
+    if (DstReg != Dst)
       return false;
     // Registers match, do the subregisters line up?
-    return compose(tri_, subIdx_, SrcSub) == DstSub;
+    return compose(TRI, SubIdx, SrcSub) == DstSub;
   }
 }
 
@@ -292,14 +416,14 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
                                                     MachineInstr *CopyMI) {
   // Bail if there is no dst interval - can happen when merging physical subreg
   // operations.
-  if (!li_->hasInterval(CP.getDstReg()))
+  if (!LIS->hasInterval(CP.getDstReg()))
     return false;
 
   LiveInterval &IntA =
-    li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
+    LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
   LiveInterval &IntB =
-    li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
-  SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex();
+    LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
+  SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getDefIndex();
 
   // BValNo is a value number in B that is defined by a copy from A.  'B3' in
   // the example above.
@@ -355,7 +479,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
   // Make sure that the end of the live range is inside the same block as
   // CopyMI.
   MachineInstr *ValLREndInst =
-    li_->getInstructionFromIndex(ValLR->end.getPrevSlot());
+    LIS->getInstructionFromIndex(ValLR->end.getPrevSlot());
   if (!ValLREndInst || ValLREndInst->getParent() != CopyMI->getParent())
     return false;
 
@@ -368,11 +492,11 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
   // of its aliases is overlapping the live interval of the virtual register.
   // If so, do not coalesce.
   if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) {
-    for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS)
-      if (li_->hasInterval(*AS) && IntA.overlaps(li_->getInterval(*AS))) {
+    for (const unsigned *AS = TRI->getAliasSet(IntB.reg); *AS; ++AS)
+      if (LIS->hasInterval(*AS) && IntA.overlaps(LIS->getInterval(*AS))) {
         DEBUG({
             dbgs() << "\t\tInterfere with alias ";
-            li_->getInterval(*AS).print(dbgs(), tri_);
+            LIS->getInterval(*AS).print(dbgs(), TRI);
           });
         return false;
       }
@@ -380,7 +504,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
 
   DEBUG({
       dbgs() << "Extending: ";
-      IntB.print(dbgs(), tri_);
+      IntB.print(dbgs(), TRI);
     });
 
   SlotIndex FillerStart = ValLR->end, FillerEnd = BLR->start;
@@ -398,13 +522,13 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
   // If the IntB live range is assigned to a physical register, and if that
   // physreg has sub-registers, update their live intervals as well.
   if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) {
-    for (const unsigned *SR = tri_->getSubRegisters(IntB.reg); *SR; ++SR) {
-      if (!li_->hasInterval(*SR))
+    for (const unsigned *SR = TRI->getSubRegisters(IntB.reg); *SR; ++SR) {
+      if (!LIS->hasInterval(*SR))
         continue;
-      LiveInterval &SRLI = li_->getInterval(*SR);
+      LiveInterval &SRLI = LIS->getInterval(*SR);
       SRLI.addRange(LiveRange(FillerStart, FillerEnd,
                               SRLI.getNextValue(FillerStart, 0,
-                                                li_->getVNInfoAllocator())));
+                                                LIS->getVNInfoAllocator())));
     }
   }
 
@@ -419,7 +543,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
   }
   DEBUG({
       dbgs() << "   result = ";
-      IntB.print(dbgs(), tri_);
+      IntB.print(dbgs(), TRI);
       dbgs() << "\n";
     });
 
@@ -434,7 +558,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP,
   // merge, find the last use and trim the live range. That will also add the
   // isKill marker.
   if (ALR->end == CopyIdx)
-    li_->shrinkToUses(&IntA);
+    LIS->shrinkToUses(&IntA);
 
   ++numExtends;
   return true;
@@ -498,15 +622,15 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
     return false;
 
   // Bail if there is no dst interval.
-  if (!li_->hasInterval(CP.getDstReg()))
+  if (!LIS->hasInterval(CP.getDstReg()))
     return false;
 
-  SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex();
+  SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getDefIndex();
 
   LiveInterval &IntA =
-    li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
+    LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
   LiveInterval &IntB =
-    li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
+    LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
 
   // BValNo is a value number in B that is defined by a copy from A. 'B3' in
   // the example above.
@@ -524,7 +648,7 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
   // the optimization.
   if (AValNo->isPHIDef() || AValNo->isUnused() || AValNo->hasPHIKill())
     return false;
-  MachineInstr *DefMI = li_->getInstructionFromIndex(AValNo->def);
+  MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def);
   if (!DefMI)
     return false;
   const MCInstrDesc &MCID = DefMI->getDesc();
@@ -538,7 +662,7 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
   if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx))
     return false;
   unsigned Op1, Op2, NewDstIdx;
-  if (!tii_->findCommutedOpIndices(DefMI, Op1, Op2))
+  if (!TII->findCommutedOpIndices(DefMI, Op1, Op2))
     return false;
   if (Op1 == UseOpIdx)
     NewDstIdx = Op2;
@@ -560,18 +684,18 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
   // Abort if the aliases of IntB.reg have values that are not simply the
   // clobbers from the superreg.
   if (TargetRegisterInfo::isPhysicalRegister(IntB.reg))
-    for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS)
-      if (li_->hasInterval(*AS) &&
-          HasOtherReachingDefs(IntA, li_->getInterval(*AS), AValNo, 0))
+    for (const unsigned *AS = TRI->getAliasSet(IntB.reg); *AS; ++AS)
+      if (LIS->hasInterval(*AS) &&
+          HasOtherReachingDefs(IntA, LIS->getInterval(*AS), AValNo, 0))
         return false;
 
   // If some of the uses of IntA.reg is already coalesced away, return false.
   // It's not possible to determine whether it's safe to perform the coalescing.
-  for (MachineRegisterInfo::use_nodbg_iterator UI = 
-         mri_->use_nodbg_begin(IntA.reg), 
-       UE = mri_->use_nodbg_end(); UI != UE; ++UI) {
+  for (MachineRegisterInfo::use_nodbg_iterator UI =
+         MRI->use_nodbg_begin(IntA.reg),
+       UE = MRI->use_nodbg_end(); UI != UE; ++UI) {
     MachineInstr *UseMI = &*UI;
-    SlotIndex UseIdx = li_->getInstructionIndex(UseMI);
+    SlotIndex UseIdx = LIS->getInstructionIndex(UseMI);
     LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx);
     if (ULR == IntA.end())
       continue;
@@ -585,15 +709,15 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
   // At this point we have decided that it is legal to do this
   // transformation.  Start by commuting the instruction.
   MachineBasicBlock *MBB = DefMI->getParent();
-  MachineInstr *NewMI = tii_->commuteInstruction(DefMI);
+  MachineInstr *NewMI = TII->commuteInstruction(DefMI);
   if (!NewMI)
     return false;
   if (TargetRegisterInfo::isVirtualRegister(IntA.reg) &&
       TargetRegisterInfo::isVirtualRegister(IntB.reg) &&
-      !mri_->constrainRegClass(IntB.reg, mri_->getRegClass(IntA.reg)))
+      !MRI->constrainRegClass(IntB.reg, MRI->getRegClass(IntA.reg)))
     return false;
   if (NewMI != DefMI) {
-    li_->ReplaceMachineInstrInMaps(DefMI, NewMI);
+    LIS->ReplaceMachineInstrInMaps(DefMI, NewMI);
     MBB->insert(DefMI, NewMI);
     MBB->erase(DefMI);
   }
@@ -610,8 +734,8 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
   //   = B
 
   // Update uses of IntA of the specific Val# with IntB.
-  for (MachineRegisterInfo::use_iterator UI = mri_->use_begin(IntA.reg),
-         UE = mri_->use_end(); UI != UE;) {
+  for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(IntA.reg),
+         UE = MRI->use_end(); UI != UE;) {
     MachineOperand &UseMO = UI.getOperand();
     MachineInstr *UseMI = &*UI;
     ++UI;
@@ -623,12 +747,12 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
       UseMO.setReg(NewReg);
       continue;
     }
-    SlotIndex UseIdx = li_->getInstructionIndex(UseMI).getUseIndex();
+    SlotIndex UseIdx = LIS->getInstructionIndex(UseMI).getUseIndex();
     LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx);
     if (ULR == IntA.end() || ULR->valno != AValNo)
       continue;
     if (TargetRegisterInfo::isPhysicalRegister(NewReg))
-      UseMO.substPhysReg(NewReg, *tri_);
+      UseMO.substPhysReg(NewReg, *TRI);
     else
       UseMO.setReg(NewReg);
     if (UseMI == CopyMI)
@@ -674,27 +798,24 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP,
 bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt,
                                                        bool preserveSrcInt,
                                                        unsigned DstReg,
-                                                       unsigned DstSubIdx,
                                                        MachineInstr *CopyMI) {
-  SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getUseIndex();
+  SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getUseIndex();
   LiveInterval::iterator SrcLR = SrcInt.FindLiveRangeContaining(CopyIdx);
   assert(SrcLR != SrcInt.end() && "Live range not found!");
   VNInfo *ValNo = SrcLR->valno;
-  // If other defs can reach uses of this def, then it's not safe to perform
-  // the optimization.
-  if (ValNo->isPHIDef() || ValNo->isUnused() || ValNo->hasPHIKill())
+  if (ValNo->isPHIDef() || ValNo->isUnused())
     return false;
-  MachineInstr *DefMI = li_->getInstructionFromIndex(ValNo->def);
+  MachineInstr *DefMI = LIS->getInstructionFromIndex(ValNo->def);
   if (!DefMI)
     return false;
   assert(DefMI && "Defining instruction disappeared");
   const MCInstrDesc &MCID = DefMI->getDesc();
   if (!MCID.isAsCheapAsAMove())
     return false;
-  if (!tii_->isTriviallyReMaterializable(DefMI, AA))
+  if (!TII->isTriviallyReMaterializable(DefMI, AA))
     return false;
   bool SawStore = false;
-  if (!DefMI->isSafeToMove(tii_, AA, SawStore))
+  if (!DefMI->isSafeToMove(TII, AA, SawStore))
     return false;
   if (MCID.getNumDefs() != 1)
     return false;
@@ -702,36 +823,20 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt,
     // Make sure the copy destination register class fits the instruction
     // definition register class. The mismatch can happen as a result of earlier
     // extract_subreg, insert_subreg, subreg_to_reg coalescing.
-    const TargetRegisterClass *RC = tii_->getRegClass(MCID, 0, tri_);
+    const TargetRegisterClass *RC = TII->getRegClass(MCID, 0, TRI);
     if (TargetRegisterInfo::isVirtualRegister(DstReg)) {
-      if (mri_->getRegClass(DstReg) != RC)
+      if (MRI->getRegClass(DstReg) != RC)
         return false;
     } else if (!RC->contains(DstReg))
       return false;
   }
 
-  // If destination register has a sub-register index on it, make sure it
-  // matches the instruction register class.
-  if (DstSubIdx) {
-    const MCInstrDesc &MCID = DefMI->getDesc();
-    if (MCID.getNumDefs() != 1)
-      return false;
-    const TargetRegisterClass *DstRC = mri_->getRegClass(DstReg);
-    const TargetRegisterClass *DstSubRC =
-      DstRC->getSubRegisterRegClass(DstSubIdx);
-    const TargetRegisterClass *DefRC = tii_->getRegClass(MCID, 0, tri_);
-    if (DefRC == DstRC)
-      DstSubIdx = 0;
-    else if (DefRC != DstSubRC)
-      return false;
-  }
-
   RemoveCopyFlag(DstReg, CopyMI);
 
   MachineBasicBlock *MBB = CopyMI->getParent();
   MachineBasicBlock::iterator MII =
     llvm::next(MachineBasicBlock::iterator(CopyMI));
-  tii_->reMaterialize(*MBB, MII, DstReg, DstSubIdx, DefMI, *tri_);
+  TII->reMaterialize(*MBB, MII, DstReg, 0, DefMI, *TRI);
   MachineInstr *NewMI = prior(MII);
 
   // CopyMI may have implicit operands, transfer them over to the newly
@@ -746,7 +851,7 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt,
   }
 
   NewMI->copyImplicitOps(CopyMI);
-  li_->ReplaceMachineInstrInMaps(CopyMI, NewMI);
+  LIS->ReplaceMachineInstrInMaps(CopyMI, NewMI);
   CopyMI->eraseFromParent();
   ReMatCopies.insert(CopyMI);
   ReMatDefs.insert(DefMI);
@@ -755,11 +860,54 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt,
 
   // The source interval can become smaller because we removed a use.
   if (preserveSrcInt)
-    li_->shrinkToUses(&SrcInt);
+    LIS->shrinkToUses(&SrcInt);
 
   return true;
 }
 
+/// eliminateUndefCopy - ProcessImpicitDefs may leave some copies of <undef>
+/// values, it only removes local variables. When we have a copy like:
+///
+///   %vreg1 = COPY %vreg2<undef>
+///
+/// We delete the copy and remove the corresponding value number from %vreg1.
+/// Any uses of that value number are marked as <undef>.
+bool RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI,
+                                           const CoalescerPair &CP) {
+  SlotIndex Idx = LIS->getInstructionIndex(CopyMI);
+  LiveInterval *SrcInt = &LIS->getInterval(CP.getSrcReg());
+  if (SrcInt->liveAt(Idx))
+    return false;
+  LiveInterval *DstInt = &LIS->getInterval(CP.getDstReg());
+  if (DstInt->liveAt(Idx))
+    return false;
+
+  // No intervals are live-in to CopyMI - it is undef.
+  if (CP.isFlipped())
+    DstInt = SrcInt;
+  SrcInt = 0;
+
+  VNInfo *DeadVNI = DstInt->getVNInfoAt(Idx.getDefIndex());
+  assert(DeadVNI && "No value defined in DstInt");
+  DstInt->removeValNo(DeadVNI);
+
+  // Find new undef uses.
+  for (MachineRegisterInfo::reg_nodbg_iterator
+         I = MRI->reg_nodbg_begin(DstInt->reg), E = MRI->reg_nodbg_end();
+       I != E; ++I) {
+    MachineOperand &MO = I.getOperand();
+    if (MO.isDef() || MO.isUndef())
+      continue;
+    MachineInstr *MI = MO.getParent();
+    SlotIndex Idx = LIS->getInstructionIndex(MI);
+    if (DstInt->liveAt(Idx))
+      continue;
+    MO.setIsUndef(true);
+    DEBUG(dbgs() << "\tnew undef: " << Idx << '\t' << *MI);
+  }
+  return true;
+}
+
 /// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and
 /// update the subregister number if it is not zero. If DstReg is a
 /// physical register and the existing subregister number of the def / use
@@ -773,22 +921,20 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) {
   unsigned SubIdx = CP.getSubIdx();
 
   // Update LiveDebugVariables.
-  ldv_->renameRegister(SrcReg, DstReg, SubIdx);
+  LDV->renameRegister(SrcReg, DstReg, SubIdx);
 
-  for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(SrcReg);
+  for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(SrcReg);
        MachineInstr *UseMI = I.skipInstruction();) {
     // A PhysReg copy that won't be coalesced can perhaps be rematerialized
     // instead.
     if (DstIsPhys) {
-      if (UseMI->isCopy() &&
-          !UseMI->getOperand(1).getSubReg() &&
-          !UseMI->getOperand(0).getSubReg() &&
+      if (UseMI->isFullCopy() &&
           UseMI->getOperand(1).getReg() == SrcReg &&
           UseMI->getOperand(0).getReg() != SrcReg &&
           UseMI->getOperand(0).getReg() != DstReg &&
           !JoinedCopies.count(UseMI) &&
-          ReMaterializeTrivialDef(li_->getInterval(SrcReg), false,
-                                  UseMI->getOperand(0).getReg(), 0, UseMI))
+          ReMaterializeTrivialDef(LIS->getInterval(SrcReg), false,
+                                  UseMI->getOperand(0).getReg(), UseMI))
         continue;
     }
 
@@ -803,10 +949,18 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) {
       Kills |= MO.isKill();
       Deads |= MO.isDead();
 
+      // Make sure we don't create read-modify-write defs accidentally.  We
+      // assume here that a SrcReg def cannot be joined into a live DstReg.  If
+      // RegisterCoalescer starts tracking partially live registers, we will
+      // need to check the actual LiveInterval to determine if DstReg is live
+      // here.
+      if (SubIdx && !Reads)
+        MO.setIsUndef();
+
       if (DstIsPhys)
-        MO.substPhysReg(DstReg, *tri_);
+        MO.substPhysReg(DstReg, *TRI);
       else
-        MO.substVirtReg(DstReg, SubIdx, *tri_);
+        MO.substVirtReg(DstReg, SubIdx, *TRI);
     }
 
     // This instruction is a copy that will be removed.
@@ -817,19 +971,19 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) {
       // If UseMI was a simple SrcReg def, make sure we didn't turn it into a
       // read-modify-write of DstReg.
       if (Deads)
-        UseMI->addRegisterDead(DstReg, tri_);
+        UseMI->addRegisterDead(DstReg, TRI);
       else if (!Reads && Writes)
-        UseMI->addRegisterDefined(DstReg, tri_);
+        UseMI->addRegisterDefined(DstReg, TRI);
 
       // Kill flags apply to the whole physical register.
       if (DstIsPhys && Kills)
-        UseMI->addRegisterKilled(DstReg, tri_);
+        UseMI->addRegisterKilled(DstReg, TRI);
     }
 
     DEBUG({
         dbgs() << "\t\tupdated: ";
         if (!UseMI->isDebugValue())
-          dbgs() << li_->getInstructionIndex(UseMI) << "\t";
+          dbgs() << LIS->getInstructionIndex(UseMI) << "\t";
         dbgs() << *UseMI;
       });
   }
@@ -838,18 +992,18 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) {
 /// removeIntervalIfEmpty - Check if the live interval of a physical register
 /// is empty, if so remove it and also remove the empty intervals of its
 /// sub-registers. Return true if live interval is removed.
-static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *li_,
-                                  const TargetRegisterInfo *tri_) {
+static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *LIS,
+                                  const TargetRegisterInfo *TRI) {
   if (li.empty()) {
     if (TargetRegisterInfo::isPhysicalRegister(li.reg))
-      for (const unsigned* SR = tri_->getSubRegisters(li.reg); *SR; ++SR) {
-        if (!li_->hasInterval(*SR))
+      for (const unsigned* SR = TRI->getSubRegisters(li.reg); *SR; ++SR) {
+        if (!LIS->hasInterval(*SR))
           continue;
-        LiveInterval &sli = li_->getInterval(*SR);
+        LiveInterval &sli = LIS->getInterval(*SR);
         if (sli.empty())
-          li_->removeInterval(*SR);
+          LIS->removeInterval(*SR);
       }
-    li_->removeInterval(li.reg);
+    LIS->removeInterval(li.reg);
     return true;
   }
   return false;
@@ -859,29 +1013,29 @@ static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *li_,
 /// the val# it defines. If the live interval becomes empty, remove it as well.
 bool RegisterCoalescer::RemoveDeadDef(LiveInterval &li,
                                              MachineInstr *DefMI) {
-  SlotIndex DefIdx = li_->getInstructionIndex(DefMI).getDefIndex();
+  SlotIndex DefIdx = LIS->getInstructionIndex(DefMI).getDefIndex();
   LiveInterval::iterator MLR = li.FindLiveRangeContaining(DefIdx);
   if (DefIdx != MLR->valno->def)
     return false;
   li.removeValNo(MLR->valno);
-  return removeIntervalIfEmpty(li, li_, tri_);
+  return removeIntervalIfEmpty(li, LIS, TRI);
 }
 
 void RegisterCoalescer::RemoveCopyFlag(unsigned DstReg,
                                               const MachineInstr *CopyMI) {
-  SlotIndex DefIdx = li_->getInstructionIndex(CopyMI).getDefIndex();
-  if (li_->hasInterval(DstReg)) {
-    LiveInterval &LI = li_->getInterval(DstReg);
+  SlotIndex DefIdx = LIS->getInstructionIndex(CopyMI).getDefIndex();
+  if (LIS->hasInterval(DstReg)) {
+    LiveInterval &LI = LIS->getInterval(DstReg);
     if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx))
       if (LR->valno->def == DefIdx)
         LR->valno->setCopy(0);
   }
   if (!TargetRegisterInfo::isPhysicalRegister(DstReg))
     return;
-  for (const unsigned* AS = tri_->getAliasSet(DstReg); *AS; ++AS) {
-    if (!li_->hasInterval(*AS))
+  for (const unsigned* AS = TRI->getAliasSet(DstReg); *AS; ++AS) {
+    if (!LIS->hasInterval(*AS))
       continue;
-    LiveInterval &LI = li_->getInterval(*AS);
+    LiveInterval &LI = LIS->getInterval(*AS);
     if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx))
       if (LR->valno->def == DefIdx)
         LR->valno->setCopy(0);
@@ -894,8 +1048,8 @@ void RegisterCoalescer::RemoveCopyFlag(unsigned DstReg,
 /// are not spillable! If the destination interval uses are far away, think
 /// twice about coalescing them!
 bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) {
-  bool Allocatable = li_->isAllocatable(CP.getDstReg());
-  LiveInterval &JoinVInt = li_->getInterval(CP.getSrcReg());
+  bool Allocatable = LIS->isAllocatable(CP.getDstReg());
+  LiveInterval &JoinVInt = LIS->getInterval(CP.getSrcReg());
 
   /// Always join simple intervals that are defined by a single copy from a
   /// reserved register. This doesn't increase register pressure, so it is
@@ -918,8 +1072,8 @@ bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) {
   // Don't join with physregs that have a ridiculous number of live
   // ranges. The data structure performance is really bad when that
   // happens.
-  if (li_->hasInterval(CP.getDstReg()) &&
-      li_->getInterval(CP.getDstReg()).ranges.size() > 1000) {
+  if (LIS->hasInterval(CP.getDstReg()) &&
+      LIS->getInterval(CP.getDstReg()).ranges.size() > 1000) {
     ++numAborts;
     DEBUG(dbgs()
           << "\tPhysical register live interval too complicated, abort!\n");
@@ -929,9 +1083,9 @@ bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) {
   // FIXME: Why are we skipping this test for partial copies?
   //        CodeGen/X86/phys_subreg_coalesce-3.ll needs it.
   if (!CP.isPartial()) {
-    const TargetRegisterClass *RC = mri_->getRegClass(CP.getSrcReg());
+    const TargetRegisterClass *RC = MRI->getRegClass(CP.getSrcReg());
     unsigned Threshold = RegClassInfo.getNumAllocatableRegs(RC) * 2;
-    unsigned Length = li_->getApproximateInstructionCount(JoinVInt);
+    unsigned Length = LIS->getApproximateInstructionCount(JoinVInt);
     if (Length > Threshold) {
       ++numAborts;
       DEBUG(dbgs() << "\tMay tie down a physical register, abort!\n");
@@ -957,12 +1111,12 @@ RegisterCoalescer::isWinToJoinCrossClass(unsigned SrcReg,
       // Early exit if the function is fairly small, coalesce aggressively if
       // that's the case. For really special register classes with 3 or
       // fewer registers, be a bit more careful.
-      (li_->getFuncInstructionCount() / NewRCCount) < 8)
+      (LIS->getFuncInstructionCount() / NewRCCount) < 8)
     return true;
-  LiveInterval &SrcInt = li_->getInterval(SrcReg);
-  LiveInterval &DstInt = li_->getInterval(DstReg);
-  unsigned SrcSize = li_->getApproximateInstructionCount(SrcInt);
-  unsigned DstSize = li_->getApproximateInstructionCount(DstInt);
+  LiveInterval &SrcInt = LIS->getInterval(SrcReg);
+  LiveInterval &DstInt = LIS->getInterval(DstReg);
+  unsigned SrcSize = LIS->getApproximateInstructionCount(SrcInt);
+  unsigned DstSize = LIS->getApproximateInstructionCount(DstInt);
 
   // Coalesce aggressively if the intervals are small compared to the number of
   // registers in the new class. The number 4 is fairly arbitrary, chosen to be
@@ -972,10 +1126,10 @@ RegisterCoalescer::isWinToJoinCrossClass(unsigned SrcReg,
     return true;
 
   // Estimate *register use density*. If it doubles or more, abort.
-  unsigned SrcUses = std::distance(mri_->use_nodbg_begin(SrcReg),
-                                   mri_->use_nodbg_end());
-  unsigned DstUses = std::distance(mri_->use_nodbg_begin(DstReg),
-                                   mri_->use_nodbg_end());
+  unsigned SrcUses = std::distance(MRI->use_nodbg_begin(SrcReg),
+                                   MRI->use_nodbg_end());
+  unsigned DstUses = std::distance(MRI->use_nodbg_begin(DstReg),
+                                   MRI->use_nodbg_end());
   unsigned NewUses = SrcUses + DstUses;
   unsigned NewSize = SrcSize + DstSize;
   if (SrcRC != NewRC && SrcSize > ThresSize) {
@@ -1003,9 +1157,9 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
   if (JoinedCopies.count(CopyMI) || ReMatCopies.count(CopyMI))
     return false; // Already done.
 
-  DEBUG(dbgs() << li_->getInstructionIndex(CopyMI) << '\t' << *CopyMI);
+  DEBUG(dbgs() << LIS->getInstructionIndex(CopyMI) << '\t' << *CopyMI);
 
-  CoalescerPair CP(*tii_, *tri_);
+  CoalescerPair CP(*TII, *TRI);
   if (!CP.setRegisters(CopyMI)) {
     DEBUG(dbgs() << "\tNot coalescable.\n");
     return false;
@@ -1018,8 +1172,15 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
     return false;  // Not coalescable.
   }
 
-  DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), tri_)
-               << " with " << PrintReg(CP.getDstReg(), tri_, CP.getSubIdx())
+  // Eliminate undefs.
+  if (!CP.isPhys() && eliminateUndefCopy(CopyMI, CP)) {
+    markAsJoined(CopyMI);
+    DEBUG(dbgs() << "\tEliminated copy of <undef> value.\n");
+    return false;  // Not coalescable.
+  }
+
+  DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), TRI)
+               << " with " << PrintReg(CP.getDstReg(), TRI, CP.getSubIdx())
                << "\n");
 
   // Enforce policies.
@@ -1028,8 +1189,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
       // Before giving up coalescing, if definition of source is defined by
       // trivial computation, try rematerializing it.
       if (!CP.isFlipped() &&
-          ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true,
-                                  CP.getDstReg(), 0, CopyMI))
+          ReMaterializeTrivialDef(LIS->getInterval(CP.getSrcReg()), true,
+                                  CP.getDstReg(), CopyMI))
         return true;
       return false;
     }
@@ -1042,8 +1203,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
         return false;
       }
       if (!isWinToJoinCrossClass(CP.getSrcReg(), CP.getDstReg(),
-                                 mri_->getRegClass(CP.getSrcReg()),
-                                 mri_->getRegClass(CP.getDstReg()),
+                                 MRI->getRegClass(CP.getSrcReg()),
+                                 MRI->getRegClass(CP.getDstReg()),
                                  CP.getNewRC())) {
         DEBUG(dbgs() << "\tAvoid coalescing to constrained register class.\n");
         Again = true;  // May be possible to coalesce later.
@@ -1052,8 +1213,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
     }
 
     // When possible, let DstReg be the larger interval.
-    if (!CP.getSubIdx() && li_->getInterval(CP.getSrcReg()).ranges.size() >
-                           li_->getInterval(CP.getDstReg()).ranges.size())
+    if (!CP.getSubIdx() && LIS->getInterval(CP.getSrcReg()).ranges.size() >
+                           LIS->getInterval(CP.getDstReg()).ranges.size())
       CP.flip();
   }
 
@@ -1067,8 +1228,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
     // If definition of source is defined by trivial computation, try
     // rematerializing it.
     if (!CP.isFlipped() &&
-        ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true,
-                                CP.getDstReg(), 0, CopyMI))
+        ReMaterializeTrivialDef(LIS->getInterval(CP.getSrcReg()), true,
+                                CP.getDstReg(), CopyMI))
       return true;
 
     // If we can eliminate the copy without merging the live ranges, do so now.
@@ -1091,7 +1252,7 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
   // other. Make sure the resulting register is set to the right register class.
   if (CP.isCrossClass()) {
     ++numCrossRCs;
-    mri_->setRegClass(CP.getDstReg(), CP.getNewRC());
+    MRI->setRegClass(CP.getDstReg(), CP.getNewRC());
   }
 
   // Remember to delete the copy instruction.
@@ -1105,10 +1266,10 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
     SmallVector<MachineBasicBlock*, 16> BlockSeq;
     // JoinIntervals invalidates the VNInfos in SrcInt, but we only need the
     // ranges for this, and they are preserved.
-    LiveInterval &SrcInt = li_->getInterval(CP.getSrcReg());
+    LiveInterval &SrcInt = LIS->getInterval(CP.getSrcReg());
     for (LiveInterval::const_iterator I = SrcInt.begin(), E = SrcInt.end();
          I != E; ++I ) {
-      li_->findLiveInMBBs(I->start, I->end, BlockSeq);
+      LIS->findLiveInMBBs(I->start, I->end, BlockSeq);
       for (unsigned idx = 0, size = BlockSeq.size(); idx != size; ++idx) {
         MachineBasicBlock &block = *BlockSeq[idx];
         if (!block.isLiveIn(CP.getDstReg()))
@@ -1120,15 +1281,15 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) {
 
   // SrcReg is guarateed to be the register whose live interval that is
   // being merged.
-  li_->removeInterval(CP.getSrcReg());
+  LIS->removeInterval(CP.getSrcReg());
 
   // Update regalloc hint.
-  tri_->UpdateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *mf_);
+  TRI->UpdateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *MF);
 
   DEBUG({
-    LiveInterval &DstInt = li_->getInterval(CP.getDstReg());
+    LiveInterval &DstInt = LIS->getInterval(CP.getDstReg());
     dbgs() << "\tJoined. Result = ";
-    DstInt.print(dbgs(), tri_);
+    DstInt.print(dbgs(), TRI);
     dbgs() << "\n";
   });
 
@@ -1197,6 +1358,7 @@ static unsigned ComputeUltimateVN(VNInfo *VNI,
 // which allows us to coalesce A and B.
 // VNI is the definition of B. LR is the life range of A that includes
 // the slot just before B. If we return true, we add "B = X" to DupCopies.
+// This implies that A dominates B.
 static bool RegistersDefinedFromSameValue(LiveIntervals &li,
                                           const TargetRegisterInfo &tri,
                                           CoalescerPair &CP,
@@ -1248,7 +1410,9 @@ static bool RegistersDefinedFromSameValue(LiveIntervals &li,
   // If the copies use two different value numbers of X, we cannot merge
   // A and B.
   LiveInterval &SrcInt = li.getInterval(Src);
-  if (SrcInt.getVNInfoAt(Other->def) != SrcInt.getVNInfoAt(VNI->def))
+  // getVNInfoBefore returns NULL for undef copies. In this case, the
+  // optimization is still safe.
+  if (SrcInt.getVNInfoBefore(Other->def) != SrcInt.getVNInfoBefore(VNI->def))
     return false;
 
   DupCopies.push_back(MI);
@@ -1259,18 +1423,18 @@ static bool RegistersDefinedFromSameValue(LiveIntervals &li,
 /// JoinIntervals - Attempt to join these two intervals.  On failure, this
 /// returns false.
 bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
-  LiveInterval &RHS = li_->getInterval(CP.getSrcReg());
-  DEBUG({ dbgs() << "\t\tRHS = "; RHS.print(dbgs(), tri_); dbgs() << "\n"; });
+  LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
+  DEBUG({ dbgs() << "\t\tRHS = "; RHS.print(dbgs(), TRI); dbgs() << "\n"; });
 
   // If a live interval is a physical register, check for interference with any
   // aliases. The interference check implemented here is a bit more conservative
   // than the full interfeence check below. We allow overlapping live ranges
   // only when one is a copy of the other.
   if (CP.isPhys()) {
-    for (const unsigned *AS = tri_->getAliasSet(CP.getDstReg()); *AS; ++AS){
-      if (!li_->hasInterval(*AS))
+    for (const unsigned *AS = TRI->getAliasSet(CP.getDstReg()); *AS; ++AS){
+      if (!LIS->hasInterval(*AS))
         continue;
-      const LiveInterval &LHS = li_->getInterval(*AS);
+      const LiveInterval &LHS = LIS->getInterval(*AS);
       LiveInterval::const_iterator LI = LHS.begin();
       for (LiveInterval::const_iterator RI = RHS.begin(), RE = RHS.end();
            RI != RE; ++RI) {
@@ -1278,10 +1442,10 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
         // Does LHS have an overlapping live range starting before RI?
         if ((LI != LHS.begin() && LI[-1].end > RI->start) &&
             (RI->start != RI->valno->def ||
-             !CP.isCoalescable(li_->getInstructionFromIndex(RI->start)))) {
+             !CP.isCoalescable(LIS->getInstructionFromIndex(RI->start)))) {
           DEBUG({
             dbgs() << "\t\tInterference from alias: ";
-            LHS.print(dbgs(), tri_);
+            LHS.print(dbgs(), TRI);
             dbgs() << "\n\t\tOverlap at " << RI->start << " and no copy.\n";
           });
           return false;
@@ -1290,10 +1454,10 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
         // Check that LHS ranges beginning in this range are copies.
         for (; LI != LHS.end() && LI->start < RI->end; ++LI) {
           if (LI->start != LI->valno->def ||
-              !CP.isCoalescable(li_->getInstructionFromIndex(LI->start))) {
+              !CP.isCoalescable(LIS->getInstructionFromIndex(LI->start))) {
             DEBUG({
               dbgs() << "\t\tInterference from alias: ";
-              LHS.print(dbgs(), tri_);
+              LHS.print(dbgs(), TRI);
               dbgs() << "\n\t\tDef at " << LI->start << " is not a copy.\n";
             });
             return false;
@@ -1313,8 +1477,8 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
 
   SmallVector<MachineInstr*, 8> DupCopies;
 
-  LiveInterval &LHS = li_->getOrCreateInterval(CP.getDstReg());
-  DEBUG({ dbgs() << "\t\tLHS = "; LHS.print(dbgs(), tri_); dbgs() << "\n"; });
+  LiveInterval &LHS = LIS->getOrCreateInterval(CP.getDstReg());
+  DEBUG({ dbgs() << "\t\tLHS = "; LHS.print(dbgs(), TRI); dbgs() << "\n"; });
 
   // Loop over the value numbers of the LHS, seeing if any are defined from
   // the RHS.
@@ -1337,7 +1501,7 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
     // from the RHS interval, we can use its value #.
     MachineInstr *MI = VNI->getCopy();
     if (!CP.isCoalescable(MI) &&
-        !RegistersDefinedFromSameValue(*li_, *tri_, CP, VNI, lr, DupCopies))
+        !RegistersDefinedFromSameValue(*LIS, *TRI, CP, VNI, lr, DupCopies))
       continue;
 
     LHSValsDefinedFromRHS[VNI] = lr->valno;
@@ -1364,7 +1528,7 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
     // from the LHS interval, we can use its value #.
     MachineInstr *MI = VNI->getCopy();
     if (!CP.isCoalescable(MI) &&
-        !RegistersDefinedFromSameValue(*li_, *tri_, CP, VNI, lr, DupCopies))
+        !RegistersDefinedFromSameValue(*LIS, *TRI, CP, VNI, lr, DupCopies))
         continue;
 
     RHSValsDefinedFromLHS[VNI] = lr->valno;
@@ -1486,7 +1650,7 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
     // and mark the X as coalesced to keep the illusion.
     unsigned Src = MI->getOperand(1).getReg();
     SourceRegisters.push_back(Src);
-    MI->getOperand(0).substVirtReg(Src, 0, *tri_);
+    MI->getOperand(0).substVirtReg(Src, 0, *TRI);
 
     markAsJoined(MI);
   }
@@ -1495,13 +1659,13 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) {
   // that B = X is gone.
   for (SmallVector<unsigned, 8>::iterator I = SourceRegisters.begin(),
          E = SourceRegisters.end(); I != E; ++I) {
-    li_->shrinkToUses(&li_->getInterval(*I));
+    LIS->shrinkToUses(&LIS->getInterval(*I));
   }
 
   // If we get here, we know that we can coalesce the live ranges.  Ask the
   // intervals to coalesce themselves now.
   LHS.join(RHS, &LHSValNoAssignments[0], &RHSValNoAssignments[0], NewVNInfo,
-           mri_);
+           MRI);
   return true;
 }
 
@@ -1552,7 +1716,7 @@ void RegisterCoalescer::CopyCoalesceInMBB(MachineBasicBlock *MBB,
 
     bool SrcIsPhys = TargetRegisterInfo::isPhysicalRegister(SrcReg);
     bool DstIsPhys = TargetRegisterInfo::isPhysicalRegister(DstReg);
-    if (li_->hasInterval(SrcReg) && li_->getInterval(SrcReg).empty())
+    if (LIS->hasInterval(SrcReg) && LIS->getInterval(SrcReg).empty())
       ImpDefCopies.push_back(Inst);
     else if (SrcIsPhys || DstIsPhys)
       PhysCopies.push_back(Inst);
@@ -1590,9 +1754,9 @@ void RegisterCoalescer::joinIntervals() {
   DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n");
 
   std::vector<MachineInstr*> TryAgainList;
-  if (loopInfo->empty()) {
+  if (Loops->empty()) {
     // If there are no loops in the function, join intervals in function order.
-    for (MachineFunction::iterator I = mf_->begin(), E = mf_->end();
+    for (MachineFunction::iterator I = MF->begin(), E = MF->end();
          I != E; ++I)
       CopyCoalesceInMBB(I, TryAgainList);
   } else {
@@ -1603,9 +1767,9 @@ void RegisterCoalescer::joinIntervals() {
     // Join intervals in the function prolog first. We want to join physical
     // registers with virtual registers before the intervals got too long.
     std::vector<std::pair<unsigned, MachineBasicBlock*> > MBBs;
-    for (MachineFunction::iterator I = mf_->begin(), E = mf_->end();I != E;++I){
+    for (MachineFunction::iterator I = MF->begin(), E = MF->end();I != E;++I){
       MachineBasicBlock *MBB = I;
-      MBBs.push_back(std::make_pair(loopInfo->getLoopDepth(MBB), I));
+      MBBs.push_back(std::make_pair(Loops->getLoopDepth(MBB), I));
     }
 
     // Sort by loop depth.
@@ -1644,22 +1808,22 @@ void RegisterCoalescer::releaseMemory() {
 }
 
 bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
-  mf_ = &fn;
-  mri_ = &fn.getRegInfo();
-  tm_ = &fn.getTarget();
-  tri_ = tm_->getRegisterInfo();
-  tii_ = tm_->getInstrInfo();
-  li_ = &getAnalysis<LiveIntervals>();
-  ldv_ = &getAnalysis<LiveDebugVariables>();
+  MF = &fn;
+  MRI = &fn.getRegInfo();
+  TM = &fn.getTarget();
+  TRI = TM->getRegisterInfo();
+  TII = TM->getInstrInfo();
+  LIS = &getAnalysis<LiveIntervals>();
+  LDV = &getAnalysis<LiveDebugVariables>();
   AA = &getAnalysis<AliasAnalysis>();
-  loopInfo = &getAnalysis<MachineLoopInfo>();
+  Loops = &getAnalysis<MachineLoopInfo>();
 
   DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n"
                << "********** Function: "
-               << ((Value*)mf_->getFunction())->getName() << '\n');
+               << ((Value*)MF->getFunction())->getName() << '\n');
 
   if (VerifyCoalescing)
-    mf_->verify(this, "Before register coalescing");
+    MF->verify(this, "Before register coalescing");
 
   RegClassInfo.runOnMachineFunction(fn);
 
@@ -1668,9 +1832,9 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
     joinIntervals();
     DEBUG({
         dbgs() << "********** INTERVALS POST JOINING **********\n";
-        for (LiveIntervals::iterator I = li_->begin(), E = li_->end();
+        for (LiveIntervals::iterator I = LIS->begin(), E = LIS->end();
              I != E; ++I){
-          I->second->print(dbgs(), tri_);
+          I->second->print(dbgs(), TRI);
           dbgs() << "\n";
         }
       });
@@ -1678,8 +1842,8 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
 
   // Perform a final pass over the instructions and compute spill weights
   // and remove identity moves.
-  SmallVector<unsigned, 4> DeadDefs;
-  for (MachineFunction::iterator mbbi = mf_->begin(), mbbe = mf_->end();
+  SmallVector<unsigned, 4> DeadDefs, InflateRegs;
+  for (MachineFunction::iterator mbbi = MF->begin(), mbbe = MF->end();
        mbbi != mbbe; ++mbbi) {
     MachineBasicBlock* mbb = mbbi;
     for (MachineBasicBlock::iterator mii = mbb->begin(), mie = mbb->end();
@@ -1690,6 +1854,16 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
         bool DoDelete = true;
         assert(MI->isCopyLike() && "Unrecognized copy instruction");
         unsigned SrcReg = MI->getOperand(MI->isSubregToReg() ? 2 : 1).getReg();
+        unsigned DstReg = MI->getOperand(0).getReg();
+
+        // Collect candidates for register class inflation.
+        if (TargetRegisterInfo::isVirtualRegister(SrcReg) &&
+            RegClassInfo.isProperSubClass(MRI->getRegClass(SrcReg)))
+          InflateRegs.push_back(SrcReg);
+        if (TargetRegisterInfo::isVirtualRegister(DstReg) &&
+            RegClassInfo.isProperSubClass(MRI->getRegClass(DstReg)))
+          InflateRegs.push_back(DstReg);
+
         if (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
             MI->getNumOperands() > 2)
           // Do not delete extract_subreg, insert_subreg of physical
@@ -1701,8 +1875,8 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
 
         if (MI->allDefsAreDead()) {
           if (TargetRegisterInfo::isVirtualRegister(SrcReg) &&
-              li_->hasInterval(SrcReg))
-            li_->shrinkToUses(&li_->getInterval(SrcReg));
+              LIS->hasInterval(SrcReg))
+            LIS->shrinkToUses(&LIS->getInterval(SrcReg));
           DoDelete = true;
         }
         if (!DoDelete) {
@@ -1711,10 +1885,10 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
             MI->RemoveOperand(3);
             MI->RemoveOperand(1);
           }
-          MI->setDesc(tii_->get(TargetOpcode::KILL));
+          MI->setDesc(TII->get(TargetOpcode::KILL));
           mii = llvm::next(mii);
         } else {
-          li_->RemoveMachineInstrFromMaps(MI);
+          LIS->RemoveMachineInstrFromMaps(MI);
           mii = mbbi->erase(mii);
           ++numPeep;
         }
@@ -1731,12 +1905,16 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
           unsigned Reg = MO.getReg();
           if (!Reg)
             continue;
-          if (TargetRegisterInfo::isVirtualRegister(Reg))
+          if (TargetRegisterInfo::isVirtualRegister(Reg)) {
             DeadDefs.push_back(Reg);
+            // Remat may also enable register class inflation.
+            if (RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)))
+              InflateRegs.push_back(Reg);
+          }
           if (MO.isDead())
             continue;
           if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
-              !mri_->use_nodbg_empty(Reg)) {
+              !MRI->use_nodbg_empty(Reg)) {
             isDead = false;
             break;
           }
@@ -1745,9 +1923,9 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
           while (!DeadDefs.empty()) {
             unsigned DeadDef = DeadDefs.back();
             DeadDefs.pop_back();
-            RemoveDeadDef(li_->getInterval(DeadDef), MI);
+            RemoveDeadDef(LIS->getInterval(DeadDef), MI);
           }
-          li_->RemoveMachineInstrFromMaps(mii);
+          LIS->RemoveMachineInstrFromMaps(mii);
           mii = mbbi->erase(mii);
           continue;
         } else
@@ -1757,14 +1935,14 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
       ++mii;
 
       // Check for now unnecessary kill flags.
-      if (li_->isNotInMIMap(MI)) continue;
-      SlotIndex DefIdx = li_->getInstructionIndex(MI).getDefIndex();
+      if (LIS->isNotInMIMap(MI)) continue;
+      SlotIndex DefIdx = LIS->getInstructionIndex(MI).getDefIndex();
       for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
         MachineOperand &MO = MI->getOperand(i);
         if (!MO.isReg() || !MO.isKill()) continue;
         unsigned reg = MO.getReg();
-        if (!reg || !li_->hasInterval(reg)) continue;
-        if (!li_->getInterval(reg).killedAt(DefIdx)) {
+        if (!reg || !LIS->hasInterval(reg)) continue;
+        if (!LIS->getInterval(reg).killedAt(DefIdx)) {
           MO.setIsKill(false);
           continue;
         }
@@ -1772,26 +1950,40 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
         // remain alive.
         if (!TargetRegisterInfo::isPhysicalRegister(reg))
           continue;
-        for (const unsigned *SR = tri_->getSubRegisters(reg);
+        for (const unsigned *SR = TRI->getSubRegisters(reg);
              unsigned S = *SR; ++SR)
-          if (li_->hasInterval(S) && li_->getInterval(S).liveAt(DefIdx))
-            MI->addRegisterDefined(S, tri_);
+          if (LIS->hasInterval(S) && LIS->getInterval(S).liveAt(DefIdx))
+            MI->addRegisterDefined(S, TRI);
       }
     }
   }
 
+  // After deleting a lot of copies, register classes may be less constrained.
+  // Removing sub-register opreands may alow GR32_ABCD -> GR32 and DPR_VFP2 ->
+  // DPR inflation.
+  array_pod_sort(InflateRegs.begin(), InflateRegs.end());
+  InflateRegs.erase(std::unique(InflateRegs.begin(), InflateRegs.end()),
+                    InflateRegs.end());
+  DEBUG(dbgs() << "Trying to inflate " << InflateRegs.size() << " regs.\n");
+  for (unsigned i = 0, e = InflateRegs.size(); i != e; ++i) {
+    unsigned Reg = InflateRegs[i];
+    if (MRI->reg_nodbg_empty(Reg))
+      continue;
+    if (MRI->recomputeRegClass(Reg, *TM)) {
+      DEBUG(dbgs() << PrintReg(Reg) << " inflated to "
+                   << MRI->getRegClass(Reg)->getName() << '\n');
+      ++NumInflated;
+    }
+  }
+
   DEBUG(dump());
-  DEBUG(ldv_->dump());
+  DEBUG(LDV->dump());
   if (VerifyCoalescing)
-    mf_->verify(this, "After register coalescing");
+    MF->verify(this, "After register coalescing");
   return true;
 }
 
 /// print - Implement the dump method.
 void RegisterCoalescer::print(raw_ostream &O, const Module* m) const {
-   li_->print(O, m);
-}
-
-RegisterCoalescer *llvm::createRegisterCoalescer() {
-  return new RegisterCoalescer();
+   LIS->print(O, m);
 }
diff --git a/contrib/llvm/lib/CodeGen/RegisterCoalescer.h b/contrib/llvm/lib/CodeGen/RegisterCoalescer.h
index 4131d91c00e9..472c48377fef 100644
--- a/contrib/llvm/lib/CodeGen/RegisterCoalescer.h
+++ b/contrib/llvm/lib/CodeGen/RegisterCoalescer.h
@@ -12,198 +12,60 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "RegisterClassInfo.h"
-#include "llvm/Support/IncludeFile.h"
-#include "llvm/CodeGen/LiveInterval.h"
-#include "llvm/ADT/SmallPtrSet.h"
-
 #ifndef LLVM_CODEGEN_REGISTER_COALESCER_H
 #define LLVM_CODEGEN_REGISTER_COALESCER_H
 
 namespace llvm {
 
-  class MachineFunction;
-  class RegallocQuery;
-  class AnalysisUsage;
   class MachineInstr;
   class TargetRegisterInfo;
   class TargetRegisterClass;
   class TargetInstrInfo;
-  class LiveDebugVariables;
-  class VirtRegMap;
-  class MachineLoopInfo;
-
-  class CoalescerPair;
-
-  /// An abstract interface for register coalescers.  Coalescers must
-  /// implement this interface to be part of the coalescer analysis
-  /// group.
-  class RegisterCoalescer : public MachineFunctionPass {
-    MachineFunction* mf_;
-    MachineRegisterInfo* mri_;
-    const TargetMachine* tm_;
-    const TargetRegisterInfo* tri_;
-    const TargetInstrInfo* tii_;
-    LiveIntervals *li_;
-    LiveDebugVariables *ldv_;
-    const MachineLoopInfo* loopInfo;
-    AliasAnalysis *AA;
-    RegisterClassInfo RegClassInfo;
-
-    /// JoinedCopies - Keep track of copies eliminated due to coalescing.
-    ///
-    SmallPtrSet<MachineInstr*, 32> JoinedCopies;
-
-    /// ReMatCopies - Keep track of copies eliminated due to remat.
-    ///
-    SmallPtrSet<MachineInstr*, 32> ReMatCopies;
-
-    /// ReMatDefs - Keep track of definition instructions which have
-    /// been remat'ed.
-    SmallPtrSet<MachineInstr*, 8> ReMatDefs;
-
-    /// joinIntervals - join compatible live intervals
-    void joinIntervals();
-
-    /// CopyCoalesceInMBB - Coalesce copies in the specified MBB, putting
-    /// copies that cannot yet be coalesced into the "TryAgain" list.
-    void CopyCoalesceInMBB(MachineBasicBlock *MBB,
-                           std::vector<MachineInstr*> &TryAgain);
-
-    /// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg,
-    /// which are the src/dst of the copy instruction CopyMI.  This returns true
-    /// if the copy was successfully coalesced away. If it is not currently
-    /// possible to coalesce this interval, but it may be possible if other
-    /// things get coalesced, then it returns true by reference in 'Again'.
-    bool JoinCopy(MachineInstr *TheCopy, bool &Again);
-
-    /// JoinIntervals - Attempt to join these two intervals.  On failure, this
-    /// returns false.  The output "SrcInt" will not have been modified, so we can
-    /// use this information below to update aliases.
-    bool JoinIntervals(CoalescerPair &CP);
-
-    /// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy. If
-    /// the source value number is defined by a copy from the destination reg
-    /// see if we can merge these two destination reg valno# into a single
-    /// value number, eliminating a copy.
-    bool AdjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI);
-
-    /// HasOtherReachingDefs - Return true if there are definitions of IntB
-    /// other than BValNo val# that can reach uses of AValno val# of IntA.
-    bool HasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB,
-                              VNInfo *AValNo, VNInfo *BValNo);
-
-    /// RemoveCopyByCommutingDef - We found a non-trivially-coalescable copy.
-    /// If the source value number is defined by a commutable instruction and
-    /// its other operand is coalesced to the copy dest register, see if we
-    /// can transform the copy into a noop by commuting the definition.
-    bool RemoveCopyByCommutingDef(const CoalescerPair &CP,MachineInstr *CopyMI);
-
-    /// ReMaterializeTrivialDef - If the source of a copy is defined by a trivial
-    /// computation, replace the copy by rematerialize the definition.
-    /// If PreserveSrcInt is true, make sure SrcInt is valid after the call.
-    bool ReMaterializeTrivialDef(LiveInterval &SrcInt, bool PreserveSrcInt,
-                                 unsigned DstReg, unsigned DstSubIdx,
-                                 MachineInstr *CopyMI);
-
-    /// shouldJoinPhys - Return true if a physreg copy should be joined.
-    bool shouldJoinPhys(CoalescerPair &CP);
-
-    /// isWinToJoinCrossClass - Return true if it's profitable to coalesce
-    /// two virtual registers from different register classes.
-    bool isWinToJoinCrossClass(unsigned SrcReg,
-                               unsigned DstReg,
-                               const TargetRegisterClass *SrcRC,
-                               const TargetRegisterClass *DstRC,
-                               const TargetRegisterClass *NewRC);
-
-    /// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and
-    /// update the subregister number if it is not zero. If DstReg is a
-    /// physical register and the existing subregister number of the def / use
-    /// being updated is not zero, make sure to set it to the correct physical
-    /// subregister.
-    void UpdateRegDefsUses(const CoalescerPair &CP);
-
-    /// RemoveDeadDef - If a def of a live interval is now determined dead,
-    /// remove the val# it defines. If the live interval becomes empty, remove
-    /// it as well.
-    bool RemoveDeadDef(LiveInterval &li, MachineInstr *DefMI);
-
-    /// RemoveCopyFlag - If DstReg is no longer defined by CopyMI, clear the
-    /// VNInfo copy flag for DstReg and all aliases.
-    void RemoveCopyFlag(unsigned DstReg, const MachineInstr *CopyMI);
-
-    /// markAsJoined - Remember that CopyMI has already been joined.
-    void markAsJoined(MachineInstr *CopyMI);
-
-  public:
-    static char ID; // Class identification, replacement for typeinfo
-    RegisterCoalescer() : MachineFunctionPass(ID) {
-      initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry());
-    }
-
-    /// Register allocators must call this from their own
-    /// getAnalysisUsage to cover the case where the coalescer is not
-    /// a Pass in the proper sense and isn't managed by PassManager.
-    /// PassManager needs to know which analyses to make available and
-    /// which to invalidate when running the register allocator or any
-    /// pass that might call coalescing.  The long-term solution is to
-    /// allow hierarchies of PassManagers.
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const;
-
-    virtual void releaseMemory();
-
-    /// runOnMachineFunction - pass entry point
-    virtual bool runOnMachineFunction(MachineFunction&);
-
-    /// print - Implement the dump method.
-    virtual void print(raw_ostream &O, const Module* = 0) const;
-  };
 
   /// CoalescerPair - A helper class for register coalescers. When deciding if
   /// two registers can be coalesced, CoalescerPair can determine if a copy
   /// instruction would become an identity copy after coalescing.
   class CoalescerPair {
-    const TargetInstrInfo &tii_;
-    const TargetRegisterInfo &tri_;
+    const TargetInstrInfo &TII;
+    const TargetRegisterInfo &TRI;
 
-    /// dstReg_ - The register that will be left after coalescing. It can be a
+    /// DstReg - The register that will be left after coalescing. It can be a
     /// virtual or physical register.
-    unsigned dstReg_;
+    unsigned DstReg;
 
-    /// srcReg_ - the virtual register that will be coalesced into dstReg.
-    unsigned srcReg_;
+    /// SrcReg - the virtual register that will be coalesced into dstReg.
+    unsigned SrcReg;
 
-    /// subReg_ - The subregister index of srcReg in dstReg_. It is possible the
-    /// coalesce srcReg_ into a subreg of the larger dstReg_ when dstReg_ is a
+    /// subReg_ - The subregister index of srcReg in DstReg. It is possible the
+    /// coalesce SrcReg into a subreg of the larger DstReg when DstReg is a
     /// virtual register.
-    unsigned subIdx_;
+    unsigned SubIdx;
 
-    /// partial_ - True when the original copy was a partial subregister copy.
-    bool partial_;
+    /// Partial - True when the original copy was a partial subregister copy.
+    bool Partial;
 
-    /// crossClass_ - True when both regs are virtual, and newRC is constrained.
-    bool crossClass_;
+    /// CrossClass - True when both regs are virtual, and newRC is constrained.
+    bool CrossClass;
 
-    /// flipped_ - True when DstReg and SrcReg are reversed from the oriignal copy
-    /// instruction.
-    bool flipped_;
+    /// Flipped - True when DstReg and SrcReg are reversed from the oriignal
+    /// copy instruction.
+    bool Flipped;
 
-    /// newRC_ - The register class of the coalesced register, or NULL if dstReg_
+    /// NewRC - The register class of the coalesced register, or NULL if DstReg
     /// is a physreg.
-    const TargetRegisterClass *newRC_;
+    const TargetRegisterClass *NewRC;
 
   public:
     CoalescerPair(const TargetInstrInfo &tii, const TargetRegisterInfo &tri)
-      : tii_(tii), tri_(tri), dstReg_(0), srcReg_(0), subIdx_(0),
-        partial_(false), crossClass_(false), flipped_(false), newRC_(0) {}
+      : TII(tii), TRI(tri), DstReg(0), SrcReg(0), SubIdx(0),
+        Partial(false), CrossClass(false), Flipped(false), NewRC(0) {}
 
     /// setRegisters - set registers to match the copy instruction MI. Return
     /// false if MI is not a coalescable copy instruction.
     bool setRegisters(const MachineInstr*);
 
-    /// flip - Swap srcReg_ and dstReg_. Return false if swapping is impossible
-    /// because dstReg_ is a physical register, or subIdx_ is set.
+    /// flip - Swap SrcReg and DstReg. Return false if swapping is impossible
+    /// because DstReg is a physical register, or SubIdx is set.
     bool flip();
 
     /// isCoalescable - Return true if MI is a copy instruction that will become
@@ -211,32 +73,33 @@ namespace llvm {
     bool isCoalescable(const MachineInstr*) const;
 
     /// isPhys - Return true if DstReg is a physical register.
-    bool isPhys() const { return !newRC_; }
+    bool isPhys() const { return !NewRC; }
 
-    /// isPartial - Return true if the original copy instruction did not copy the
-    /// full register, but was a subreg operation.
-    bool isPartial() const { return partial_; }
+    /// isPartial - Return true if the original copy instruction did not copy
+    /// the full register, but was a subreg operation.
+    bool isPartial() const { return Partial; }
 
-    /// isCrossClass - Return true if DstReg is virtual and NewRC is a smaller register class than DstReg's.
-    bool isCrossClass() const { return crossClass_; }
+    /// isCrossClass - Return true if DstReg is virtual and NewRC is a smaller
+    /// register class than DstReg's.
+    bool isCrossClass() const { return CrossClass; }
 
     /// isFlipped - Return true when getSrcReg is the register being defined by
     /// the original copy instruction.
-    bool isFlipped() const { return flipped_; }
+    bool isFlipped() const { return Flipped; }
 
     /// getDstReg - Return the register (virtual or physical) that will remain
     /// after coalescing.
-    unsigned getDstReg() const { return dstReg_; }
+    unsigned getDstReg() const { return DstReg; }
 
     /// getSrcReg - Return the virtual register that will be coalesced away.
-    unsigned getSrcReg() const { return srcReg_; }
+    unsigned getSrcReg() const { return SrcReg; }
 
     /// getSubIdx - Return the subregister index in DstReg that SrcReg will be
     /// coalesced into, or 0.
-    unsigned getSubIdx() const { return subIdx_; }
+    unsigned getSubIdx() const { return SubIdx; }
 
     /// getNewRC - Return the register class of the coalesced register.
-    const TargetRegisterClass *getNewRC() const { return newRC_; }
+    const TargetRegisterClass *getNewRC() const { return NewRC; }
   };
 } // End llvm namespace
 
diff --git a/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp b/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp
index 9e9a145b0aa4..ca02aa1b8143 100644
--- a/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp
+++ b/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp
@@ -206,6 +206,7 @@ void RegScavenger::forward() {
             break;
           }
         assert(SubUsed && "Using an undefined register!");
+        (void)SubUsed;
       }
       assert((!EarlyClobberRegs.test(Reg) || MI->isRegTiedToDefOperand(i)) &&
              "Using an early clobbered register!");
diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp
index 21375b286c99..1e9b5c89f172 100644
--- a/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp
+++ b/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp
@@ -26,7 +26,7 @@
 using namespace llvm;
 
 #ifndef NDEBUG
-cl::opt<bool> StressSchedOpt(
+static cl::opt<bool> StressSchedOpt(
   "stress-sched", cl::Hidden, cl::init(false),
   cl::desc("Stress test instruction scheduling"));
 #endif
@@ -140,6 +140,7 @@ void SUnit::removePred(const SDep &D) {
           break;
         }
       assert(FoundSucc && "Mismatching preds / succs lists!");
+      (void)FoundSucc;
       Preds.erase(I);
       // Update the bookkeeping.
       if (P.getKind() == SDep::Data) {
diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index 446adfc2b626..34b8ab0b47f2 100644
--- a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -36,7 +36,7 @@ ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf,
                                      const MachineDominatorTree &mdt)
   : ScheduleDAG(mf), MLI(mli), MDT(mdt), MFI(mf.getFrameInfo()),
     InstrItins(mf.getTarget().getInstrItineraryData()),
-    Defs(TRI->getNumRegs()), Uses(TRI->getNumRegs()), 
+    Defs(TRI->getNumRegs()), Uses(TRI->getNumRegs()),
     LoopRegs(MLI, MDT), FirstDbgValue(0) {
   DbgValues.clear();
 }
@@ -134,6 +134,7 @@ static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI,
 }
 
 void ScheduleDAGInstrs::StartBlock(MachineBasicBlock *BB) {
+  LoopRegs.Deps.clear();
   if (MachineLoop *ML = MLI.getLoopFor(BB))
     if (BB == ML->getLoopLatch()) {
       MachineBasicBlock *Header = ML->getHeader();
diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.h b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.h
index 8a4ea855235c..666bdf548c71 100644
--- a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.h
+++ b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.h
@@ -48,7 +48,8 @@ namespace llvm {
     /// VisitLoop - Clear out any previous state and analyze the given loop.
     ///
     void VisitLoop(const MachineLoop *Loop) {
-      Deps.clear();
+      assert(Deps.empty() && "stale loop dependencies");
+
       MachineBasicBlock *Header = Loop->getHeader();
       SmallSet<unsigned, 8> LoopLiveIns;
       for (MachineBasicBlock::livein_iterator LI = Header->livein_begin(),
@@ -109,7 +110,7 @@ namespace llvm {
     /// initialized and destructed for each block.
     std::vector<std::vector<SUnit *> > Defs;
     std::vector<std::vector<SUnit *> > Uses;
- 
+
     /// PendingLoads - Remember where unknown loads are after the most recent
     /// unknown store, as we iterate. As with Defs and Uses, this is here
     /// to minimize construction/destruction.
@@ -127,7 +128,7 @@ namespace llvm {
   protected:
 
     /// DbgValues - Remember instruction that preceeds DBG_VALUE.
-    typedef std::vector<std::pair<MachineInstr *, MachineInstr *> > 
+    typedef std::vector<std::pair<MachineInstr *, MachineInstr *> >
       DbgValueVector;
     DbgValueVector DbgValues;
     MachineInstr *FirstDbgValue;
diff --git a/contrib/llvm/lib/CodeGen/ScoreboardHazardRecognizer.cpp b/contrib/llvm/lib/CodeGen/ScoreboardHazardRecognizer.cpp
index 0e005d35189d..b80c01ed58b9 100644
--- a/contrib/llvm/lib/CodeGen/ScoreboardHazardRecognizer.cpp
+++ b/contrib/llvm/lib/CodeGen/ScoreboardHazardRecognizer.cpp
@@ -213,7 +213,6 @@ void ScoreboardHazardRecognizer::EmitInstruction(SUnit *SU) {
         freeUnits = freeUnit & (freeUnit - 1);
       } while (freeUnits);
 
-      assert(freeUnit && "No function unit available!");
       if (IS->getReservationKind() == InstrStage::Required)
         RequiredScoreboard[cycle + i] |= freeUnit;
       else
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 4f0d2caca22b..7b878688df63 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -216,6 +216,7 @@ namespace {
     SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
     SDValue visitBUILD_VECTOR(SDNode *N);
     SDValue visitCONCAT_VECTORS(SDNode *N);
+    SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
     SDValue visitVECTOR_SHUFFLE(SDNode *N);
     SDValue visitMEMBARRIER(SDNode *N);
 
@@ -1105,6 +1106,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
   case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
   case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
+  case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
   case ISD::MEMBARRIER:         return visitMEMBARRIER(N);
   }
@@ -1526,12 +1528,6 @@ SDValue DAGCombiner::visitADDE(SDNode *N) {
   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
 
-  // If both operands are null we know that carry out will always be false.
-  if (N0C && N0C->isNullValue() && N0 == N1)
-    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), DAG.getNode(ISD::CARRY_FALSE,
-                                                             N->getDebugLoc(),
-                                                             MVT::Glue));
-
   // canonicalize constant to RHS
   if (N0C && !N1C)
     return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(),
@@ -3763,7 +3759,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
   if (VT.isInteger() &&
       (VT0 == MVT::i1 ||
        (VT0.isInteger() &&
-        TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent)) &&
+        TLI.getBooleanContents(false) == TargetLowering::ZeroOrOneBooleanContent)) &&
       N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) {
     SDValue XORNode;
     if (VT == VT0)
@@ -4118,7 +4114,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
         // we know that the element size of the sext'd result matches the
         // element size of the compare operands.
       if (VT.getSizeInBits() == N0VT.getSizeInBits())
-        return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
+        return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
                              N0.getOperand(1),
                              cast<CondCodeSDNode>(N0.getOperand(2))->get());
       // If the desired elements are smaller or larger than the source
@@ -4132,7 +4128,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
           EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
                            N0VT.getVectorNumElements());
         SDValue VsetCC =
-          DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
+          DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
                         N0.getOperand(1),
                         cast<CondCodeSDNode>(N0.getOperand(2))->get());
         return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
@@ -4348,7 +4344,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
         // we know that the element size of the sext'd result matches the
         // element size of the compare operands.
         return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
-                           DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
+                           DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
                                          N0.getOperand(1),
                                  cast<CondCodeSDNode>(N0.getOperand(2))->get()),
                            DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
@@ -4364,7 +4360,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
         EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
                          N0VT.getVectorNumElements());
       SDValue VsetCC =
-        DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
+        DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
                       N0.getOperand(1),
                       cast<CondCodeSDNode>(N0.getOperand(2))->get());
       return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
@@ -4532,7 +4528,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
         // we know that the element size of the sext'd result matches the
         // element size of the compare operands.
       if (VT.getSizeInBits() == N0VT.getSizeInBits())
-        return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
+        return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
                              N0.getOperand(1),
                              cast<CondCodeSDNode>(N0.getOperand(2))->get());
       // If the desired elements are smaller or larger than the source
@@ -4546,7 +4542,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
           EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
                            N0VT.getVectorNumElements());
         SDValue VsetCC =
-          DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
+          DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
                         N0.getOperand(1),
                         cast<CondCodeSDNode>(N0.getOperand(2))->get());
         return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
@@ -6479,7 +6475,7 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
         PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
 
       unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
-      const Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
+      Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
       if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy))
         return SDValue();
 
@@ -6542,7 +6538,7 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
 
     unsigned LDAlign = LD->getAlignment();
     unsigned STAlign = ST->getAlignment();
-    const Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
+    Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
     unsigned ABIAlign = TLI.getTargetData()->getABITypeAlignment(IntVTTy);
     if (LDAlign < ABIAlign || STAlign < ABIAlign)
       return SDValue();
@@ -6776,6 +6772,7 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
   SDValue InVec = N->getOperand(0);
   SDValue InVal = N->getOperand(1);
   SDValue EltNo = N->getOperand(2);
+  DebugLoc dl = N->getDebugLoc();
 
   // If the inserted element is an UNDEF, just use the input vector.
   if (InVal.getOpcode() == ISD::UNDEF)
@@ -6787,32 +6784,40 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
   if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
     return SDValue();
 
-  // If the invec is a BUILD_VECTOR and if EltNo is a constant, build a new
-  // vector with the inserted element.
-  if (InVec.getOpcode() == ISD::BUILD_VECTOR && isa<ConstantSDNode>(EltNo)) {
-    unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
-    SmallVector<SDValue, 8> Ops(InVec.getNode()->op_begin(),
-                                InVec.getNode()->op_end());
-    if (Elt < Ops.size())
-      Ops[Elt] = InVal;
-    return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
-                       VT, &Ops[0], Ops.size());
-  }
-  // If the invec is an UNDEF and if EltNo is a constant, create a new
-  // BUILD_VECTOR with undef elements and the inserted element.
-  if (InVec.getOpcode() == ISD::UNDEF &&
-      isa<ConstantSDNode>(EltNo)) {
-    EVT EltVT = VT.getVectorElementType();
-    unsigned NElts = VT.getVectorNumElements();
-    SmallVector<SDValue, 8> Ops(NElts, DAG.getUNDEF(EltVT));
+  // Check that we know which element is being inserted
+  if (!isa<ConstantSDNode>(EltNo))
+    return SDValue();
+  unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
 
-    unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
-    if (Elt < Ops.size())
-      Ops[Elt] = InVal;
-    return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
-                       VT, &Ops[0], Ops.size());
+  // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
+  // be converted to a BUILD_VECTOR).  Fill in the Ops vector with the
+  // vector elements.
+  SmallVector<SDValue, 8> Ops;
+  if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
+    Ops.append(InVec.getNode()->op_begin(),
+               InVec.getNode()->op_end());
+  } else if (InVec.getOpcode() == ISD::UNDEF) {
+    unsigned NElts = VT.getVectorNumElements();
+    Ops.append(NElts, DAG.getUNDEF(InVal.getValueType()));
+  } else {
+    return SDValue();
   }
-  return SDValue();
+
+  // Insert the element
+  if (Elt < Ops.size()) {
+    // All the operands of BUILD_VECTOR must have the same type;
+    // we enforce that here.
+    EVT OpVT = Ops[0].getValueType();
+    if (InVal.getValueType() != OpVT)
+      InVal = OpVT.bitsGT(InVal.getValueType()) ?
+                DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) :
+                DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal);
+    Ops[Elt] = InVal;
+  }
+
+  // Return the new vector
+  return DAG.getNode(ISD::BUILD_VECTOR, dl,
+                     VT, &Ops[0], Ops.size());
 }
 
 SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
@@ -6896,7 +6901,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
 
     // If Idx was -1 above, Elt is going to be -1, so just return undef.
     if (Elt == -1)
-      return DAG.getUNDEF(LN0->getBasePtr().getValueType());
+      return DAG.getUNDEF(LVT);
 
     unsigned Align = LN0->getAlignment();
     if (NewLoad) {
@@ -7028,6 +7033,36 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
+  EVT NVT = N->getValueType(0);
+  SDValue V = N->getOperand(0);
+
+  if (V->getOpcode() == ISD::INSERT_SUBVECTOR) {
+    // Handle only simple case where vector being inserted and vector
+    // being extracted are of same type, and are half size of larger vectors.
+    EVT BigVT = V->getOperand(0).getValueType();
+    EVT SmallVT = V->getOperand(1).getValueType();
+    if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits())
+      return SDValue();
+
+    // Combine:
+    //    (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx)
+    // Into:
+    //    indicies are equal => V1
+    //    otherwise => (extract_subvec V1, ExtIdx)
+    //
+    SDValue InsIdx = N->getOperand(1);
+    SDValue ExtIdx = V->getOperand(2);
+
+    if (InsIdx == ExtIdx)
+      return V->getOperand(1);
+    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT,
+                       V->getOperand(0), N->getOperand(1));
+  }
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
   EVT VT = N->getValueType(0);
   unsigned NumElts = VT.getVectorNumElements();
@@ -7447,7 +7482,7 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
           const_cast<ConstantFP*>(FV->getConstantFPValue()),
           const_cast<ConstantFP*>(TV->getConstantFPValue())
         };
-        const Type *FPTy = Elts[0]->getType();
+        Type *FPTy = Elts[0]->getType();
         const TargetData &TD = *TLI.getTargetData();
 
         // Create a ConstantArray of the two constants.
@@ -7465,10 +7500,13 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
         SDValue Cond = DAG.getSetCC(DL,
                                     TLI.getSetCCResultType(N0.getValueType()),
                                     N0, N1, CC);
+        AddToWorkList(Cond.getNode());
         SDValue CstOffset = DAG.getNode(ISD::SELECT, DL, Zero.getValueType(),
                                         Cond, One, Zero);
+        AddToWorkList(CstOffset.getNode());
         CPIdx = DAG.getNode(ISD::ADD, DL, TLI.getPointerTy(), CPIdx,
                             CstOffset);
+        AddToWorkList(CPIdx.getNode());
         return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
                            MachinePointerInfo::getConstantPool(), false,
                            false, Alignment);
@@ -7553,7 +7591,8 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
 
   // fold select C, 16, 0 -> shl C, 4
   if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() &&
-      TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent) {
+    TLI.getBooleanContents(N0.getValueType().isVector()) ==
+      TargetLowering::ZeroOrOneBooleanContent) {
 
     // If the caller doesn't want us to simplify this into a zext of a compare,
     // don't do it.
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 54a7d43f46d6..e8f8c73d6883 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -66,17 +66,22 @@ using namespace llvm;
 void FastISel::startNewBlock() {
   LocalValueMap.clear();
 
-  // Start out as null, meaining no local-value instructions have
-  // been emitted.
-  LastLocalValue = 0;
+  EmitStartPt = 0;
 
-  // Advance the last local value past any EH_LABEL instructions.
+  // Advance the emit start point past any EH_LABEL instructions.
   MachineBasicBlock::iterator
     I = FuncInfo.MBB->begin(), E = FuncInfo.MBB->end();
   while (I != E && I->getOpcode() == TargetOpcode::EH_LABEL) {
-    LastLocalValue = I;
+    EmitStartPt = I;
     ++I;
   }
+  LastLocalValue = EmitStartPt;
+}
+
+void FastISel::flushLocalValueMap() {
+  LocalValueMap.clear();
+  LastLocalValue = EmitStartPt;
+  recomputeInsertPt();
 }
 
 bool FastISel::hasTrivialKill(const Value *V) const {
@@ -183,7 +188,7 @@ unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
       (void) Flt.convertToInteger(x, IntBitWidth, /*isSigned=*/true,
                                 APFloat::rmTowardZero, &isExact);
       if (isExact) {
-        APInt IntVal(IntBitWidth, 2, x);
+        APInt IntVal(IntBitWidth, x);
 
         unsigned IntegerReg =
           getRegForValue(ConstantInt::get(V->getContext(), IntVal));
@@ -422,12 +427,12 @@ bool FastISel::SelectGetElementPtr(const User *I) {
 
   bool NIsKill = hasTrivialKill(I->getOperand(0));
 
-  const Type *Ty = I->getOperand(0)->getType();
+  Type *Ty = I->getOperand(0)->getType();
   MVT VT = TLI.getPointerTy();
   for (GetElementPtrInst::const_op_iterator OI = I->op_begin()+1,
        E = I->op_end(); OI != E; ++OI) {
     const Value *Idx = *OI;
-    if (const StructType *StTy = dyn_cast<StructType>(Ty)) {
+    if (StructType *StTy = dyn_cast<StructType>(Ty)) {
       unsigned Field = cast<ConstantInt>(Idx)->getZExtValue();
       if (Field) {
         // N = N + Offset
@@ -489,7 +494,7 @@ bool FastISel::SelectCall(const User *I) {
   const CallInst *Call = cast<CallInst>(I);
 
   // Handle simple inline asms.
-  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getArgOperand(0))) {
+  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
     // Don't attempt to handle constraints.
     if (!IA->getConstraintString().empty())
       return false;
@@ -526,13 +531,10 @@ bool FastISel::SelectCall(const User *I) {
     unsigned Reg = 0;
     unsigned Offset = 0;
     if (const Argument *Arg = dyn_cast<Argument>(Address)) {
-      if (Arg->hasByValAttr()) {
-        // Byval arguments' frame index is recorded during argument lowering.
-        // Use this info directly.
-        Offset = FuncInfo.getByValArgumentFrameIndex(Arg);
-        if (Offset)
-          Reg = TRI.getFrameRegister(*FuncInfo.MF);
-      }
+      // Some arguments' frame index is recorded during argument lowering.
+      Offset = FuncInfo.getArgumentFrameIndex(Arg);
+      if (Offset)
+	Reg = TRI.getFrameRegister(*FuncInfo.MF);
     }
     if (!Reg)
       Reg = getRegForValue(Address);
@@ -645,6 +647,16 @@ bool FastISel::SelectCall(const User *I) {
   }
   }
 
+  // Usually, it does not make sense to initialize a value,
+  // make an unrelated function call and use the value, because
+  // it tends to be spilled on the stack. So, we move the pointer
+  // to the last local value to the beginning of the block, so that
+  // all the values which have already been materialized,
+  // appear after the call. It also makes sense to skip intrinsics
+  // since they tend to be inlined.
+  if (!isa<IntrinsicInst>(F))
+    flushLocalValueMap();
+
   // An arbitrary call. Bail.
   return false;
 }
@@ -839,7 +851,7 @@ FastISel::SelectExtractValue(const User *U) {
     return false;
 
   const Value *Op0 = EVI->getOperand(0);
-  const Type *AggTy = Op0->getType();
+  Type *AggTy = Op0->getType();
 
   // Get the base result register.
   unsigned ResultReg;
@@ -1074,7 +1086,7 @@ unsigned FastISel::FastEmit_ri_(MVT VT, unsigned Opcode,
   if (MaterialReg == 0) {
     // This is a bit ugly/slow, but failing here means falling out of
     // fast-isel, which would be very slow.
-    const IntegerType *ITy = IntegerType::get(FuncInfo.Fn->getContext(),
+    IntegerType *ITy = IntegerType::get(FuncInfo.Fn->getContext(),
                                               VT.getSizeInBits());
     MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
   }
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index d518b5d346a1..b052740a1abe 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -78,7 +78,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf) {
   for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I)
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(I))
       if (const ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) {
-        const Type *Ty = AI->getAllocatedType();
+        Type *Ty = AI->getAllocatedType();
         uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty);
         unsigned Align =
           std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
@@ -216,7 +216,7 @@ unsigned FunctionLoweringInfo::CreateReg(EVT VT) {
 /// In the case that the given value has struct or array type, this function
 /// will assign registers for each member or element.
 ///
-unsigned FunctionLoweringInfo::CreateRegs(const Type *Ty) {
+unsigned FunctionLoweringInfo::CreateRegs(Type *Ty) {
   SmallVector<EVT, 4> ValueVTs;
   ComputeValueVTs(TLI, Ty, ValueVTs);
 
@@ -260,7 +260,7 @@ FunctionLoweringInfo::GetLiveOutRegInfo(unsigned Reg, unsigned BitWidth) {
 /// ComputePHILiveOutRegInfo - Compute LiveOutInfo for a PHI's destination
 /// register based on the LiveOutInfo of its operands.
 void FunctionLoweringInfo::ComputePHILiveOutRegInfo(const PHINode *PN) {
-  const Type *Ty = PN->getType();
+  Type *Ty = PN->getType();
   if (!Ty->isIntegerTy() || Ty->isVectorTy())
     return;
 
@@ -351,20 +351,18 @@ void FunctionLoweringInfo::ComputePHILiveOutRegInfo(const PHINode *PN) {
   }
 }
 
-/// setByValArgumentFrameIndex - Record frame index for the byval
+/// setArgumentFrameIndex - Record frame index for the byval
 /// argument. This overrides previous frame index entry for this argument,
 /// if any.
-void FunctionLoweringInfo::setByValArgumentFrameIndex(const Argument *A,
+void FunctionLoweringInfo::setArgumentFrameIndex(const Argument *A,
                                                       int FI) {
-  assert (A->hasByValAttr() && "Argument does not have byval attribute!");
   ByValArgFrameIndexMap[A] = FI;
 }
 
-/// getByValArgumentFrameIndex - Get frame index for the byval argument.
+/// getArgumentFrameIndex - Get frame index for the byval argument.
 /// If the argument does not have any assigned frame index then 0 is
 /// returned.
-int FunctionLoweringInfo::getByValArgumentFrameIndex(const Argument *A) {
-  assert (A->hasByValAttr() && "Argument does not have byval attribute!");
+int FunctionLoweringInfo::getArgumentFrameIndex(const Argument *A) {
   DenseMap<const Argument *, int>::iterator I =
     ByValArgFrameIndexMap.find(A);
   if (I != ByValArgFrameIndexMap.end())
@@ -454,3 +452,34 @@ void llvm::CopyCatchInfo(const BasicBlock *SuccBB, const BasicBlock *LPad,
       break;
   }
 }
+
+/// AddLandingPadInfo - Extract the exception handling information from the
+/// landingpad instruction and add them to the specified machine module info.
+void llvm::AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI,
+                             MachineBasicBlock *MBB) {
+  MMI.addPersonality(MBB,
+                     cast<Function>(I.getPersonalityFn()->stripPointerCasts()));
+
+  if (I.isCleanup())
+    MMI.addCleanup(MBB);
+
+  // FIXME: New EH - Add the clauses in reverse order. This isn't 100% correct,
+  //        but we need to do it this way because of how the DWARF EH emitter
+  //        processes the clauses.
+  for (unsigned i = I.getNumClauses(); i != 0; --i) {
+    Value *Val = I.getClause(i - 1);
+    if (I.isCatch(i - 1)) {
+      MMI.addCatchTypeInfo(MBB,
+                           dyn_cast<GlobalVariable>(Val->stripPointerCasts()));
+    } else {
+      // Add filters in a list.
+      Constant *CVal = cast<Constant>(Val);
+      SmallVector<const GlobalVariable*, 4> FilterList;
+      for (User::op_iterator
+             II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II)
+        FilterList.push_back(cast<GlobalVariable>((*II)->stripPointerCasts()));
+
+      MMI.addFilterTypeInfo(MBB, FilterList);
+    }
+  }
+}
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index f0f4743298e7..2ff66f8f8715 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -30,6 +30,12 @@
 #include "llvm/Support/MathExtras.h"
 using namespace llvm;
 
+/// MinRCSize - Smallest register class we allow when constraining virtual
+/// registers.  If satisfying all register class constraints would require
+/// using a smaller register class, emit a COPY to a new virtual register
+/// instead.
+const unsigned MinRCSize = 4;
+
 /// CountResults - The results of target nodes have register or immediate
 /// operands first, then an optional chain, and optional glue operands (which do
 /// not go into the resulting MachineInstr).
@@ -87,7 +93,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned,
          UI != E; ++UI) {
       SDNode *User = *UI;
       bool Match = true;
-      if (User->getOpcode() == ISD::CopyToReg && 
+      if (User->getOpcode() == ISD::CopyToReg &&
           User->getOperand(2).getNode() == Node &&
           User->getOperand(2).getResNo() == ResNo) {
         unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
@@ -113,7 +119,8 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned,
             if (!UseRC)
               UseRC = RC;
             else if (RC) {
-              const TargetRegisterClass *ComRC = getCommonSubClass(UseRC, RC);
+              const TargetRegisterClass *ComRC =
+                TRI->getCommonSubClass(UseRC, RC);
               // If multiple uses expect disjoint register classes, we emit
               // copies in AddRegisterOperand.
               if (ComRC)
@@ -139,7 +146,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned,
   } else {
     DstRC = TLI->getRegClassFor(VT);
   }
-    
+
   // If all uses are reading from the src physical register and copying the
   // register is either impossible or very expensive, then don't create a copy.
   if (MatchReg && SrcRC->getCopyCost() < 0) {
@@ -167,7 +174,7 @@ unsigned InstrEmitter::getDstOfOnlyCopyToRegUse(SDNode *Node,
     return 0;
 
   SDNode *User = *Node->use_begin();
-  if (User->getOpcode() == ISD::CopyToReg && 
+  if (User->getOpcode() == ISD::CopyToReg &&
       User->getOperand(2).getNode() == Node &&
       User->getOperand(2).getResNo() == ResNo) {
     unsigned Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
@@ -202,7 +209,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI,
       for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
            UI != E; ++UI) {
         SDNode *User = *UI;
-        if (User->getOpcode() == ISD::CopyToReg && 
+        if (User->getOpcode() == ISD::CopyToReg &&
             User->getOperand(2).getNode() == Node &&
             User->getOperand(2).getResNo() == i) {
           unsigned Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
@@ -280,15 +287,16 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op,
     MCID.OpInfo[IIOpNum].isOptionalDef();
 
   // If the instruction requires a register in a different class, create
-  // a new virtual register and copy the value into it.
+  // a new virtual register and copy the value into it, but first attempt to
+  // shrink VReg's register class within reason.  For example, if VReg == GR32
+  // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
   if (II) {
-    const TargetRegisterClass *SrcRC = MRI->getRegClass(VReg);
     const TargetRegisterClass *DstRC = 0;
     if (IIOpNum < II->getNumOperands())
       DstRC = TII->getRegClass(*II, IIOpNum, TRI);
     assert((DstRC || (MCID.isVariadic() && IIOpNum >= MCID.getNumOperands())) &&
            "Don't have operand info for this instruction!");
-    if (DstRC && !SrcRC->hasSuperClassEq(DstRC)) {
+    if (DstRC && !MRI->constrainRegClass(VReg, DstRC, MinRCSize)) {
       unsigned NewVReg = MRI->createVirtualRegister(DstRC);
       BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
               TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
@@ -326,7 +334,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op,
 
 /// AddOperand - Add the specified operand to the specified machine instr.  II
 /// specifies the instruction information for the node, and IIOpNum is the
-/// operand number (in the II) that we are adding. IIOpNum and II are used for 
+/// operand number (in the II) that we are adding. IIOpNum and II are used for
 /// assertions only.
 void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
                               unsigned IIOpNum,
@@ -356,7 +364,7 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
   } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) {
     int Offset = CP->getOffset();
     unsigned Align = CP->getAlignment();
-    const Type *Type = CP->getType();
+    Type *Type = CP->getType();
     // MachineConstantPool wants an explicit alignment.
     if (Align == 0) {
       Align = TM->getTargetData()->getPrefTypeAlignment(Type);
@@ -365,7 +373,7 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
         Align = TM->getTargetData()->getTypeAllocSize(Type);
       }
     }
-    
+
     unsigned Idx;
     MachineConstantPool *MCP = MF->getConstantPool();
     if (CP->isMachineConstantPoolEntry())
@@ -389,35 +397,44 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
   }
 }
 
-/// getSuperRegisterRegClass - Returns the register class of a superreg A whose
-/// "SubIdx"'th sub-register class is the specified register class and whose
-/// type matches the specified type.
-static const TargetRegisterClass*
-getSuperRegisterRegClass(const TargetRegisterClass *TRC,
-                         unsigned SubIdx, EVT VT) {
-  // Pick the register class of the superegister for this type
-  for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(),
-         E = TRC->superregclasses_end(); I != E; ++I)
-    if ((*I)->hasType(VT) && (*I)->getSubRegisterRegClass(SubIdx) == TRC)
-      return *I;
-  assert(false && "Couldn't find the register class");
-  return 0;
+unsigned InstrEmitter::ConstrainForSubReg(unsigned VReg, unsigned SubIdx,
+                                          EVT VT, DebugLoc DL) {
+  const TargetRegisterClass *VRC = MRI->getRegClass(VReg);
+  const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(VRC, SubIdx);
+
+  // RC is a sub-class of VRC that supports SubIdx.  Try to constrain VReg
+  // within reason.
+  if (RC && RC != VRC)
+    RC = MRI->constrainRegClass(VReg, RC, MinRCSize);
+
+  // VReg has been adjusted.  It can be used with SubIdx operands now.
+  if (RC)
+    return VReg;
+
+  // VReg couldn't be reasonably constrained.  Emit a COPY to a new virtual
+  // register instead.
+  RC = TRI->getSubClassWithSubReg(TLI->getRegClassFor(VT), SubIdx);
+  assert(RC && "No legal register class for VT supports that SubIdx");
+  unsigned NewReg = MRI->createVirtualRegister(RC);
+  BuildMI(*MBB, InsertPos, DL, TII->get(TargetOpcode::COPY), NewReg)
+    .addReg(VReg);
+  return NewReg;
 }
 
 /// EmitSubregNode - Generate machine code for subreg nodes.
 ///
-void InstrEmitter::EmitSubregNode(SDNode *Node, 
+void InstrEmitter::EmitSubregNode(SDNode *Node,
                                   DenseMap<SDValue, unsigned> &VRBaseMap,
                                   bool IsClone, bool IsCloned) {
   unsigned VRBase = 0;
   unsigned Opc = Node->getMachineOpcode();
-  
+
   // If the node is only used by a CopyToReg and the dest reg is a vreg, use
   // the CopyToReg'd destination register instead of creating a new vreg.
   for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
        UI != E; ++UI) {
     SDNode *User = *UI;
-    if (User->getOpcode() == ISD::CopyToReg && 
+    if (User->getOpcode() == ISD::CopyToReg &&
         User->getOperand(2).getNode() == Node) {
       unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
       if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
@@ -426,12 +443,14 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
       }
     }
   }
-  
-  if (Opc == TargetOpcode::EXTRACT_SUBREG) {
-    // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub
-    unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
 
-    // Figure out the register class to create for the destreg.
+  if (Opc == TargetOpcode::EXTRACT_SUBREG) {
+    // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub.  There are no
+    // constraints on the %dst register, COPY can target all legal register
+    // classes.
+    unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
+    const TargetRegisterClass *TRC = TLI->getRegClassFor(Node->getValueType(0));
+
     unsigned VReg = getVR(Node->getOperand(0), VRBaseMap);
     MachineInstr *DefMI = MRI->getVRegDef(VReg);
     unsigned SrcReg, DstReg, DefSubIdx;
@@ -443,62 +462,57 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
       // r1026 = extract_subreg r1025, 4
       // to a copy
       // r1026 = copy r1024
-      const TargetRegisterClass *TRC = MRI->getRegClass(SrcReg);
       VRBase = MRI->createVirtualRegister(TRC);
       BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
               TII->get(TargetOpcode::COPY), VRBase).addReg(SrcReg);
     } else {
-      const TargetRegisterClass *TRC = MRI->getRegClass(VReg);
-      const TargetRegisterClass *SRC = TRC->getSubRegisterRegClass(SubIdx);
-      assert(SRC && "Invalid subregister index in EXTRACT_SUBREG");
+      // VReg may not support a SubIdx sub-register, and we may need to
+      // constrain its register class or issue a COPY to a compatible register
+      // class.
+      VReg = ConstrainForSubReg(VReg, SubIdx,
+                                Node->getOperand(0).getValueType(),
+                                Node->getDebugLoc());
 
-      // Figure out the register class to create for the destreg.
-      // Note that if we're going to directly use an existing register,
-      // it must be precisely the required class, and not a subclass
-      // thereof.
-      if (VRBase == 0 || SRC != MRI->getRegClass(VRBase)) {
-        // Create the reg
-        assert(SRC && "Couldn't find source register class");
-        VRBase = MRI->createVirtualRegister(SRC);
-      }
+      // Create the destreg if it is missing.
+      if (VRBase == 0)
+        VRBase = MRI->createVirtualRegister(TRC);
 
       // Create the extract_subreg machine instruction.
-      MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(),
-                                 TII->get(TargetOpcode::COPY), VRBase);
-
-      // Add source, and subreg index
-      AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap, /*IsDebug=*/false,
-                 IsClone, IsCloned);
-      assert(TargetRegisterInfo::isVirtualRegister(MI->getOperand(1).getReg())&&
-             "Cannot yet extract from physregs");
-      MI->getOperand(1).setSubReg(SubIdx);
-      MBB->insert(InsertPos, MI);
+      BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
+              TII->get(TargetOpcode::COPY), VRBase).addReg(VReg, 0, SubIdx);
     }
   } else if (Opc == TargetOpcode::INSERT_SUBREG ||
              Opc == TargetOpcode::SUBREG_TO_REG) {
     SDValue N0 = Node->getOperand(0);
     SDValue N1 = Node->getOperand(1);
     SDValue N2 = Node->getOperand(2);
-    unsigned SubReg = getVR(N1, VRBaseMap);
     unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue();
-    const TargetRegisterClass *TRC = MRI->getRegClass(SubReg);
-    const TargetRegisterClass *SRC =
-      getSuperRegisterRegClass(TRC, SubIdx, Node->getValueType(0));
 
-    // Figure out the register class to create for the destreg.
-    // Note that if we're going to directly use an existing register,
-    // it must be precisely the required class, and not a subclass
-    // thereof.
-    if (VRBase == 0 || SRC != MRI->getRegClass(VRBase)) {
-      // Create the reg
-      assert(SRC && "Couldn't find source register class");
+    // Figure out the register class to create for the destreg.  It should be
+    // the largest legal register class supporting SubIdx sub-registers.
+    // RegisterCoalescer will constrain it further if it decides to eliminate
+    // the INSERT_SUBREG instruction.
+    //
+    //   %dst = INSERT_SUBREG %src, %sub, SubIdx
+    //
+    // is lowered by TwoAddressInstructionPass to:
+    //
+    //   %dst = COPY %src
+    //   %dst:SubIdx = COPY %sub
+    //
+    // There is no constraint on the %src register class.
+    //
+    const TargetRegisterClass *SRC = TLI->getRegClassFor(Node->getValueType(0));
+    SRC = TRI->getSubClassWithSubReg(SRC, SubIdx);
+    assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
+
+    if (VRBase == 0 || !SRC->hasSubClassEq(MRI->getRegClass(VRBase)))
       VRBase = MRI->createVirtualRegister(SRC);
-    }
 
     // Create the insert_subreg or subreg_to_reg machine instruction.
     MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc));
     MI->addOperand(MachineOperand::CreateReg(VRBase, true));
-    
+
     // If creating a subreg_to_reg, then the first input operand
     // is an implicit value immediate, otherwise it's a register
     if (Opc == TargetOpcode::SUBREG_TO_REG) {
@@ -514,7 +528,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
     MBB->insert(InsertPos, MI);
   } else
     llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
-     
+
   SDValue Op(Node, 0);
   bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
   (void)isNew; // Silence compiler warning.
@@ -643,9 +657,9 @@ void InstrEmitter::
 EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
                 DenseMap<SDValue, unsigned> &VRBaseMap) {
   unsigned Opc = Node->getMachineOpcode();
-  
+
   // Handle subreg insert/extract specially
-  if (Opc == TargetOpcode::EXTRACT_SUBREG || 
+  if (Opc == TargetOpcode::EXTRACT_SUBREG ||
       Opc == TargetOpcode::INSERT_SUBREG ||
       Opc == TargetOpcode::SUBREG_TO_REG) {
     EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
@@ -667,7 +681,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
   if (Opc == TargetOpcode::IMPLICIT_DEF)
     // We want a unique VR for each IMPLICIT_DEF use.
     return;
-  
+
   const MCInstrDesc &II = TII->get(Opc);
   unsigned NumResults = CountResults(Node);
   unsigned NodeOperands = CountOperands(Node);
@@ -712,12 +726,12 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
     // Then mark unused registers as dead.
     MI->setPhysRegsDeadExcept(UsedRegs, *TRI);
   }
-  
+
   // Add result register values for things that are defined by this
   // instruction.
   if (NumResults)
     CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap);
-  
+
   // Emit all of the actual operands of this instruction, adding them to the
   // instruction as appropriate.
   bool HasOptPRefs = II.getNumDefs() > NumResults;
@@ -751,7 +765,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
         MI->addRegisterDead(Reg, TRI);
     }
   }
-  
+
   // If the instruction has implicit defs and the node doesn't, mark the
   // implicit def as dead.  If the node has any glue outputs, we don't do this
   // because we don't know what implicit defs are being used by glued nodes.
@@ -761,6 +775,12 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
            i != e; ++i)
         MI->addRegisterDead(IDList[i-II.getNumDefs()], TRI);
     }
+
+  // Run post-isel target hook to adjust this instruction if needed.
+#ifdef NDEBUG
+  if (II.hasPostISelHook())
+#endif
+    TLI->AdjustInstrPostInstrSelection(MI, Node);
 }
 
 /// EmitSpecialNode - Generate machine code for a target-independent node and
@@ -788,7 +808,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
       SrcReg = R->getReg();
     else
       SrcReg = getVR(SrcVal, VRBaseMap);
-      
+
     unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
     if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
       break;
@@ -808,12 +828,12 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
             TII->get(TargetOpcode::EH_LABEL)).addSym(S);
     break;
   }
-      
+
   case ISD::INLINEASM: {
     unsigned NumOps = Node->getNumOperands();
     if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
       --NumOps;  // Ignore the glue operand.
-      
+
     // Create the inline asm machine instruction.
     MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(),
                                TII->get(TargetOpcode::INLINEASM));
@@ -822,7 +842,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
     SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString);
     const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol();
     MI->addOperand(MachineOperand::CreateES(AsmStr));
-      
+
     // Add the HasSideEffect and isAlignStack bits.
     int64_t ExtraInfo =
       cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))->
@@ -834,10 +854,10 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
       unsigned Flags =
         cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
       unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
-        
+
       MI->addOperand(MachineOperand::CreateImm(Flags));
       ++i;  // Skip the ID value.
-        
+
       switch (InlineAsm::getKind(Flags)) {
       default: llvm_unreachable("Bad flags!");
         case InlineAsm::Kind_RegDef:
@@ -873,13 +893,13 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
         break;
       }
     }
-    
+
     // Get the mdnode from the asm if it exists and add it to the instruction.
     SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode);
     const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD();
     if (MD)
       MI->addOperand(MachineOperand::CreateMetadata(MD));
-    
+
     MBB->insert(InsertPos, MI);
     break;
   }
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
index 19fc0445b166..c081f38be024 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -77,6 +77,12 @@ class InstrEmitter {
                   DenseMap<SDValue, unsigned> &VRBaseMap,
                   bool IsDebug, bool IsClone, bool IsCloned);
 
+  /// ConstrainForSubReg - Try to constrain VReg to a register class that
+  /// supports SubIdx sub-registers.  Emit a copy if that isn't possible.
+  /// Return the virtual register to use.
+  unsigned ConstrainForSubReg(unsigned VReg, unsigned SubIdx,
+                              EVT VT, DebugLoc DL);
+
   /// EmitSubregNode - Generate machine code for subreg nodes.
   ///
   void EmitSubregNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap,
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index d06e2bdce065..63255ae2ebd9 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -53,10 +53,15 @@ class SelectionDAGLegalize {
 
   // Libcall insertion helpers.
 
-  /// LastCALLSEQ - This keeps track of the CALLSEQ_END node that has been
+  /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been
   /// legalized.  We use this to ensure that calls are properly serialized
   /// against each other, including inserted libcalls.
-  SmallVector<SDValue, 8> LastCALLSEQ;
+  SDValue LastCALLSEQ_END;
+
+  /// IsLegalizingCall - This member is used *only* for purposes of providing
+  /// helpful assertions that a libcall isn't created while another call is
+  /// being legalized (which could lead to non-serialized call sequences).
+  bool IsLegalizingCall;
 
   /// LegalizedNodes - For nodes that are of legal width, and that have more
   /// than one use, this map indicates what regularized operand to use.  This
@@ -149,15 +154,6 @@ class SelectionDAGLegalize {
 
   void ExpandNode(SDNode *Node, SmallVectorImpl<SDValue> &Results);
   void PromoteNode(SDNode *Node, SmallVectorImpl<SDValue> &Results);
-
-  SDValue getLastCALLSEQ() { return LastCALLSEQ.back();  }
-  void setLastCALLSEQ(const SDValue s) { LastCALLSEQ.back() = s; }
-  void pushLastCALLSEQ(SDValue s) {
-    LastCALLSEQ.push_back(s);
-  }
-  void popLastCALLSEQ() {
-    LastCALLSEQ.pop_back();
-  }
 };
 }
 
@@ -199,7 +195,8 @@ SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
 }
 
 void SelectionDAGLegalize::LegalizeDAG() {
-  pushLastCALLSEQ(DAG.getEntryNode());
+  LastCALLSEQ_END = DAG.getEntryNode();
+  IsLegalizingCall = false;
 
   // The legalize process is inherently a bottom-up recursive process (users
   // legalize their uses before themselves).  Given infinite stack space, we
@@ -227,15 +224,14 @@ void SelectionDAGLegalize::LegalizeDAG() {
 /// FindCallEndFromCallStart - Given a chained node that is part of a call
 /// sequence, find the CALLSEQ_END node that terminates the call sequence.
 static SDNode *FindCallEndFromCallStart(SDNode *Node, int depth = 0) {
-  int next_depth = depth;
+  // Nested CALLSEQ_START/END constructs aren't yet legal,
+  // but we can DTRT and handle them correctly here.
   if (Node->getOpcode() == ISD::CALLSEQ_START)
-    next_depth = depth + 1;
-  if (Node->getOpcode() == ISD::CALLSEQ_END) {
-    assert(depth > 0 && "negative depth!");
-    if (depth == 1)
+    depth++;
+  else if (Node->getOpcode() == ISD::CALLSEQ_END) {
+    depth--;
+    if (depth == 0)
       return Node;
-    else
-      next_depth = depth - 1;
   }
   if (Node->use_empty())
     return 0;   // No CallSeqEnd
@@ -266,7 +262,7 @@ static SDNode *FindCallEndFromCallStart(SDNode *Node, int depth = 0) {
     SDNode *User = *UI;
     for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
       if (User->getOperand(i) == TheChain)
-        if (SDNode *Result = FindCallEndFromCallStart(User, next_depth))
+        if (SDNode *Result = FindCallEndFromCallStart(User, depth))
           return Result;
   }
   return 0;
@@ -287,7 +283,6 @@ static SDNode *FindCallStartFromCallEnd(SDNode *Node) {
     case ISD::CALLSEQ_START:
       if (!nested)
         return Node;
-      Node = Node->getOperand(0).getNode();
       nested--;
       break;
     case ISD::CALLSEQ_END:
@@ -295,7 +290,7 @@ static SDNode *FindCallStartFromCallEnd(SDNode *Node) {
       break;
     }
   }
-  return (Node->getOpcode() == ISD::CALLSEQ_START) ? Node : 0;
+  return 0;
 }
 
 /// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to
@@ -365,7 +360,7 @@ static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP,
         // smaller type.
         TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) &&
         TLI.ShouldShrinkFPConstant(OrigVT)) {
-      const Type *SType = SVT.getTypeForEVT(*DAG.getContext());
+      Type *SType = SVT.getTypeForEVT(*DAG.getContext());
       LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
       VT = SVT;
       Extend = true;
@@ -819,6 +814,11 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
     break;
   }
+  case ISD::ATOMIC_STORE: {
+    Action = TLI.getOperationAction(Node->getOpcode(),
+                                    Node->getOperand(2).getValueType());
+    break;
+  }
   case ISD::SELECT_CC:
   case ISD::SETCC:
   case ISD::BR_CC: {
@@ -872,7 +872,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     if (Action == TargetLowering::Legal)
       Action = TargetLowering::Expand;
     break;
-  case ISD::TRAMPOLINE:
+  case ISD::INIT_TRAMPOLINE:
+  case ISD::ADJUST_TRAMPOLINE:
   case ISD::FRAMEADDR:
   case ISD::RETURNADDR:
     // These operations lie about being legal: when they claim to be legal,
@@ -912,12 +913,11 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     case ISD::BR_JT:
     case ISD::BR_CC:
     case ISD::BRCOND:
-      assert(LastCALLSEQ.size() == 1 && "branch inside CALLSEQ_BEGIN/END?");
-      // Branches tweak the chain to include LastCALLSEQ
+      // Branches tweak the chain to include LastCALLSEQ_END
       Ops[0] = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ops[0],
-                           getLastCALLSEQ());
+                           LastCALLSEQ_END);
       Ops[0] = LegalizeOp(Ops[0]);
-      setLastCALLSEQ(DAG.getEntryNode());
+      LastCALLSEQ_END = DAG.getEntryNode();
       break;
     case ISD::SHL:
     case ISD::SRL:
@@ -989,6 +989,31 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
 #endif
     assert(0 && "Do not know how to legalize this operator!");
 
+  case ISD::SRA:
+  case ISD::SRL:
+  case ISD::SHL: {
+    // Scalarize vector SRA/SRL/SHL.
+    EVT VT = Node->getValueType(0);
+    assert(VT.isVector() && "Unable to legalize non-vector shift");
+    assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
+    unsigned NumElem = VT.getVectorNumElements();
+
+    SmallVector<SDValue, 8> Scalars;
+    for (unsigned Idx = 0; Idx < NumElem; Idx++) {
+      SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                               VT.getScalarType(),
+                               Node->getOperand(0), DAG.getIntPtrConstant(Idx));
+      SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                               VT.getScalarType(),
+                               Node->getOperand(1), DAG.getIntPtrConstant(Idx));
+      Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
+                                    VT.getScalarType(), Ex, Sh));
+    }
+    Result = DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0),
+                         &Scalars[0], Scalars.size());
+    break;
+  }
+
   case ISD::BUILD_VECTOR:
     switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) {
     default: assert(0 && "This action is not supported yet!");
@@ -1006,7 +1031,6 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     break;
   case ISD::CALLSEQ_START: {
     SDNode *CallEnd = FindCallEndFromCallStart(Node);
-    assert(CallEnd && "didn't find CALLSEQ_END!");
 
     // Recursively Legalize all of the inputs of the call end that do not lead
     // to this call start.  This ensures that any libcalls that need be inserted
@@ -1023,9 +1047,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
 
     // Merge in the last call to ensure that this call starts after the last
     // call ended.
-    if (getLastCALLSEQ().getOpcode() != ISD::EntryToken) {
+    if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) {
       Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
-                         Tmp1, getLastCALLSEQ());
+                         Tmp1, LastCALLSEQ_END);
       Tmp1 = LegalizeOp(Tmp1);
     }
 
@@ -1046,29 +1070,25 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     // sequence have been legalized, legalize the call itself.  During this
     // process, no libcalls can/will be inserted, guaranteeing that no calls
     // can overlap.
+    assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!");
     // Note that we are selecting this call!
-    setLastCALLSEQ(SDValue(CallEnd, 0));
+    LastCALLSEQ_END = SDValue(CallEnd, 0);
+    IsLegalizingCall = true;
 
     // Legalize the call, starting from the CALLSEQ_END.
-    LegalizeOp(getLastCALLSEQ());
+    LegalizeOp(LastCALLSEQ_END);
+    assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!");
     return Result;
   }
   case ISD::CALLSEQ_END:
-    {
-      SDNode *myCALLSEQ_BEGIN = FindCallStartFromCallEnd(Node);
-
-      // If the CALLSEQ_START node hasn't been legalized first, legalize it.
-      // This will cause this node to be legalized as well as handling libcalls
-      // right.
-      if (getLastCALLSEQ().getNode() != Node) {
-        LegalizeOp(SDValue(myCALLSEQ_BEGIN, 0));
-        DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
-        assert(I != LegalizedNodes.end() &&
-               "Legalizing the call start should have legalized this node!");
-        return I->second;
-      }
-
-      pushLastCALLSEQ(SDValue(myCALLSEQ_BEGIN, 0));
+    // If the CALLSEQ_START node hasn't been legalized first, legalize it.  This
+    // will cause this node to be legalized as well as handling libcalls right.
+    if (LastCALLSEQ_END.getNode() != Node) {
+      LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0));
+      DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
+      assert(I != LegalizedNodes.end() &&
+             "Legalizing the call start should have legalized this node!");
+      return I->second;
     }
 
     // Otherwise, the call start has been legalized and everything is going
@@ -1096,8 +1116,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
                          Result.getResNo());
       }
     }
+    assert(IsLegalizingCall && "Call sequence imbalance between start/end?");
     // This finishes up call legalization.
-    popLastCALLSEQ();
+    IsLegalizingCall = false;
 
     // If the CALLSEQ_END node has a flag, remember that we legalized it.
     AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0));
@@ -1124,7 +1145,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
         // If this is an unaligned load and the target doesn't support it,
         // expand it.
         if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) {
-          const Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
+          Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
           unsigned ABIAlignment = TLI.getTargetData()->getABITypeAlignment(Ty);
           if (LD->getAlignment() < ABIAlignment){
             Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()),
@@ -1311,7 +1332,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
           // If this is an unaligned load and the target doesn't support it,
           // expand it.
           if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) {
-            const Type *Ty =
+            Type *Ty =
               LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
             unsigned ABIAlignment =
               TLI.getTargetData()->getABITypeAlignment(Ty);
@@ -1491,7 +1512,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
           // If this is an unaligned store and the target doesn't support it,
           // expand it.
           if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
-            const Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
+            Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
             unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty);
             if (ST->getAlignment() < ABIAlignment)
               Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()),
@@ -1596,7 +1617,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
           // If this is an unaligned store and the target doesn't support it,
           // expand it.
           if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
-            const Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
+            Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
             unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty);
             if (ST->getAlignment() < ABIAlignment)
               Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()),
@@ -1611,82 +1632,101 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
           EVT WideScalarVT = Tmp3.getValueType().getScalarType();
           EVT NarrowScalarVT = StVT.getScalarType();
 
-          // The Store type is illegal, must scalarize the vector store.
-          SmallVector<SDValue, 8> Stores;
-          bool ScalarLegal = TLI.isTypeLegal(WideScalarVT);
-          if (!TLI.isTypeLegal(StVT) && StVT.isVector() && ScalarLegal) {
+          if (StVT.isVector()) {
             unsigned NumElem = StVT.getVectorNumElements();
+            // The type of the data we want to save
+            EVT RegVT = Tmp3.getValueType();
+            EVT RegSclVT = RegVT.getScalarType();
+            // The type of data as saved in memory.
+            EVT MemSclVT = StVT.getScalarType();
 
-            unsigned ScalarSize = StVT.getScalarType().getSizeInBits();
-            // Round odd types to the next pow of two.
-            if (!isPowerOf2_32(ScalarSize))
-              ScalarSize = NextPowerOf2(ScalarSize);
-            // Types smaller than 8 bits are promoted to 8 bits.
-            ScalarSize = std::max<unsigned>(ScalarSize, 8);
-            // Store stride
-            unsigned Stride = ScalarSize/8;
-            assert(isPowerOf2_32(Stride) && "Stride must be a power of two");
+            bool RegScalarLegal = TLI.isTypeLegal(RegSclVT);
+            bool MemScalarLegal = TLI.isTypeLegal(MemSclVT);
 
-            for (unsigned Idx=0; Idx<NumElem; Idx++) {
-              SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                                       WideScalarVT, Tmp3, DAG.getIntPtrConstant(Idx));
+            // We need to expand this store. If the register element type
+            // is legal then we can scalarize the vector and use
+            // truncating stores.
+            if (RegScalarLegal) {
+              // Cast floats into integers
+              unsigned ScalarSize = MemSclVT.getSizeInBits();
+              EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize);
 
+              // Round odd types to the next pow of two.
+              if (!isPowerOf2_32(ScalarSize))
+                ScalarSize = NextPowerOf2(ScalarSize);
 
-              EVT NVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize);
+              // Store Stride in bytes
+              unsigned Stride = ScalarSize/8;
+              // Extract each of the elements from the original vector
+              // and save them into memory individually.
+              SmallVector<SDValue, 8> Stores;
+              for (unsigned Idx = 0; Idx < NumElem; Idx++) {
+                SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                                RegSclVT, Tmp3, DAG.getIntPtrConstant(Idx));
 
-              Ex = DAG.getNode(ISD::TRUNCATE, dl, NVT, Ex);
-              Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
-                                 DAG.getIntPtrConstant(Stride));
-              SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2,
-                                           ST->getPointerInfo().getWithOffset(Idx*Stride),
-                                           isVolatile, isNonTemporal, Alignment);
-              Stores.push_back(Store);
-            }
-            Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
-                                 &Stores[0], Stores.size());
-            break;
-          }
+                Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
+                                   DAG.getIntPtrConstant(Stride));
 
-          // The Store type is illegal, must scalarize the vector store.
-          // However, the scalar type is illegal. Must bitcast the result
-          // and store it in smaller parts.
-          if (!TLI.isTypeLegal(StVT) && StVT.isVector()) {
-            unsigned WideNumElem = StVT.getVectorNumElements();
-            unsigned Stride = NarrowScalarVT.getSizeInBits()/8;
+                // This scalar TruncStore may be illegal, but we lehalize it
+                // later.
+                SDValue Store = DAG.getTruncStore(Tmp1, dl, Ex, Tmp2,
+                      ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT,
+                      isVolatile, isNonTemporal, Alignment);
 
-            unsigned SizeRatio =
-              (WideScalarVT.getSizeInBits() / NarrowScalarVT.getSizeInBits());
-
-            EVT CastValueVT = EVT::getVectorVT(*DAG.getContext(), NarrowScalarVT,
-                                               SizeRatio*WideNumElem);
-
-            // Cast the wide elem vector to wider vec with smaller elem type.
-            // Example <2 x i64> -> <4 x i32>
-            Tmp3 = DAG.getNode(ISD::BITCAST, dl, CastValueVT, Tmp3);
-
-            for (unsigned Idx=0; Idx<WideNumElem*SizeRatio; Idx++) {
-              // Extract elment i
-              SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                                       NarrowScalarVT, Tmp3, DAG.getIntPtrConstant(Idx));
-              // bump pointer.
-              Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
-                                 DAG.getIntPtrConstant(Stride));
-
-              // Store if, this element is:
-              //  - First element on big endian, or
-              //  - Last element on little endian
-              if (( TLI.isBigEndian() && (Idx%SizeRatio == 0)) ||
-                  ((!TLI.isBigEndian() && (Idx%SizeRatio == SizeRatio-1)))) {
-                SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2,
-                                             ST->getPointerInfo().getWithOffset(Idx*Stride),
-                                             isVolatile, isNonTemporal, Alignment);
                 Stores.push_back(Store);
               }
+
+              Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+                                   &Stores[0], Stores.size());
+              break;
             }
-            Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
-                                 &Stores[0], Stores.size());
-            break;
-          }
+
+            // The scalar register type is illegal.
+            // For example saving <2 x i64> -> <2 x i32> on a x86.
+            // In here we bitcast the value into a vector of smaller parts and
+            // save it using smaller scalars.
+            if (!RegScalarLegal && MemScalarLegal) {
+              // Store Stride in bytes
+              unsigned Stride = MemSclVT.getSizeInBits()/8;
+
+              unsigned SizeRatio =
+                (RegSclVT.getSizeInBits() / MemSclVT.getSizeInBits());
+
+              EVT CastValueVT = EVT::getVectorVT(*DAG.getContext(),
+                                                 MemSclVT,
+                                                 SizeRatio * NumElem);
+
+              // Cast the wide elem vector to wider vec with smaller elem type.
+              // Example <2 x i64> -> <4 x i32>
+              Tmp3 = DAG.getNode(ISD::BITCAST, dl, CastValueVT, Tmp3);
+
+              SmallVector<SDValue, 8> Stores;
+              for (unsigned Idx=0; Idx < NumElem * SizeRatio; Idx++) {
+                // Extract the Ith element.
+                SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                               NarrowScalarVT, Tmp3, DAG.getIntPtrConstant(Idx));
+                // Bump pointer.
+                Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
+                                   DAG.getIntPtrConstant(Stride));
+
+                // Store if, this element is:
+                //  - First element on big endian, or
+                //  - Last element on little endian
+                if (( TLI.isBigEndian() && (Idx % SizeRatio == 0)) ||
+                    ((!TLI.isBigEndian() && (Idx % SizeRatio == SizeRatio-1)))) {
+                  SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2,
+                                  ST->getPointerInfo().getWithOffset(Idx*Stride),
+                                           isVolatile, isNonTemporal, Alignment);
+                  Stores.push_back(Store);
+                }
+              }
+              Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+                                   &Stores[0], Stores.size());
+              break;
+            }
+
+            assert(false && "Unable to legalize the vector trunc store!");
+          }// is vector
 
 
           // TRUNCSTORE:i16 i32 -> STORE i16
@@ -1999,7 +2039,7 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
   unsigned SrcSize = SrcOp.getValueType().getSizeInBits();
   unsigned SlotSize = SlotVT.getSizeInBits();
   unsigned DestSize = DestVT.getSizeInBits();
-  const Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
+  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
   unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(DestType);
 
   // Emit a store to the stack slot.  Use a truncstore if the input value is
@@ -2106,7 +2146,7 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
         }
       } else {
         assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
-        const Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
+        Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
         CV.push_back(UndefValue::get(OpNTy));
       }
     }
@@ -2150,6 +2190,7 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
 // and leave the Hi part unset.
 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
                                             bool isSigned) {
+  assert(!IsLegalizingCall && "Cannot overlap legalization of calls!");
   // The input chain to this libcall is the entry node of the function.
   // Legalizing the call will automatically add the previous call to the
   // dependence.
@@ -2159,7 +2200,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
   TargetLowering::ArgListEntry Entry;
   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
     EVT ArgVT = Node->getOperand(i).getValueType();
-    const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
     Entry.isSExt = isSigned;
     Entry.isZExt = !isSigned;
@@ -2169,7 +2210,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
                                          TLI.getPointerTy());
 
   // Splice the libcall in wherever FindInputOutputChains tells us to.
-  const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
+  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
 
   // isTailCall may be true since the callee does not reference caller stack
   // frame. Check if it's in the right position.
@@ -2185,7 +2226,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
     return DAG.getRoot();
 
   // Legalize the call sequence, starting with the chain.  This will advance
-  // the LastCALLSEQ to the legalized version of the CALLSEQ_END node that
+  // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
   // was added by LowerCallTo (guaranteeing proper serialization of calls).
   LegalizeOp(CallInfo.second);
   return CallInfo.first;
@@ -2210,7 +2251,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
                                          TLI.getPointerTy());
 
-  const Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
+  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   std::pair<SDValue,SDValue> CallInfo =
   TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false,
                   false, 0, TLI.getLibcallCallingConv(LC), false,
@@ -2231,13 +2272,14 @@ std::pair<SDValue, SDValue>
 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
                                          SDNode *Node,
                                          bool isSigned) {
+  assert(!IsLegalizingCall && "Cannot overlap legalization of calls!");
   SDValue InChain = Node->getOperand(0);
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
     EVT ArgVT = Node->getOperand(i).getValueType();
-    const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = Node->getOperand(i);
     Entry.Ty = ArgTy;
     Entry.isSExt = isSigned;
@@ -2248,7 +2290,7 @@ SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
                                          TLI.getPointerTy());
 
   // Splice the libcall in wherever FindInputOutputChains tells us to.
-  const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
+  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
   std::pair<SDValue, SDValue> CallInfo =
     TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false,
                     0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false,
@@ -2256,7 +2298,7 @@ SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
                     Callee, Args, DAG, Node->getDebugLoc());
 
   // Legalize the call sequence, starting with the chain.  This will advance
-  // the LastCALLSEQ to the legalized version of the CALLSEQ_END node that
+  // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
   // was added by LowerCallTo (guaranteeing proper serialization of calls).
   LegalizeOp(CallInfo.second);
   return CallInfo;
@@ -2360,13 +2402,13 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
   SDValue InChain = DAG.getEntryNode();
 
   EVT RetVT = Node->getValueType(0);
-  const Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
+  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
     EVT ArgVT = Node->getOperand(i).getValueType();
-    const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
     Entry.isSExt = isSigned;
     Entry.isZExt = !isSigned;
@@ -2397,7 +2439,7 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
   LegalizeOp(CallInfo.second);
 
   // Remainder is loaded back from the stack frame.
-  SDValue Rem = DAG.getLoad(RetVT, dl, getLastCALLSEQ(), FIPtr,
+  SDValue Rem = DAG.getLoad(RetVT, dl, LastCALLSEQ_END, FIPtr,
                             MachinePointerInfo(), false, false, 0);
   Results.push_back(CallInfo.first);
   Results.push_back(Rem);
@@ -2955,8 +2997,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node,
     Results.push_back(DAG.getConstant(0, MVT::i32));
     Results.push_back(Node->getOperand(0));
     break;
+  case ISD::ATOMIC_FENCE:
   case ISD::MEMBARRIER: {
     // If the target didn't lower this, lower it to '__sync_synchronize()' call
+    // FIXME: handle "fence singlethread" more efficiently.
     TargetLowering::ArgListTy Args;
     std::pair<SDValue, SDValue> CallResult =
       TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()),
@@ -2969,6 +3013,32 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node,
     Results.push_back(CallResult.second);
     break;
   }
+  case ISD::ATOMIC_LOAD: {
+    // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
+    SDValue Zero = DAG.getConstant(0, Node->getValueType(0));
+    SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
+                                 cast<AtomicSDNode>(Node)->getMemoryVT(),
+                                 Node->getOperand(0),
+                                 Node->getOperand(1), Zero, Zero,
+                                 cast<AtomicSDNode>(Node)->getMemOperand(),
+                                 cast<AtomicSDNode>(Node)->getOrdering(),
+                                 cast<AtomicSDNode>(Node)->getSynchScope());
+    Results.push_back(Swap.getValue(0));
+    Results.push_back(Swap.getValue(1));
+    break;
+  }
+  case ISD::ATOMIC_STORE: {
+    // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
+    SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
+                                 cast<AtomicSDNode>(Node)->getMemoryVT(),
+                                 Node->getOperand(0),
+                                 Node->getOperand(1), Node->getOperand(2),
+                                 cast<AtomicSDNode>(Node)->getMemOperand(),
+                                 cast<AtomicSDNode>(Node)->getOrdering(),
+                                 cast<AtomicSDNode>(Node)->getSynchScope());
+    Results.push_back(Swap.getValue(1));
+    break;
+  }
   // By default, atomic intrinsics are marked Legal and lowered. Targets
   // which don't support them directly, however, may want libcalls, in which
   // case they mark them Expand, and we get here.
@@ -3727,8 +3797,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node,
 
     LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp2.getValueType()),
                           Tmp2, Tmp3, Tmp4, dl);
-    assert(LastCALLSEQ.size() == 1 && "branch inside CALLSEQ_BEGIN/END?");
-    setLastCALLSEQ(DAG.getEntryNode());
+    LastCALLSEQ_END = DAG.getEntryNode();
 
     assert(!Tmp3.getNode() && "Can't legalize BR_CC with legal condition!");
     Tmp3 = DAG.getConstant(0, Tmp2.getValueType());
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index e6835d87f82c..7c1cc69d6a2f 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -55,6 +55,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
 #endif
     llvm_unreachable("Do not know how to soften the result of this operator!");
 
+    case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
     case ISD::BITCAST:     R = SoftenFloatRes_BITCAST(N); break;
     case ISD::BUILD_PAIR:  R = SoftenFloatRes_BUILD_PAIR(N); break;
     case ISD::ConstantFP:
@@ -107,6 +108,12 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
   return BitConvertToInteger(N->getOperand(0));
 }
 
+SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
+                                                      unsigned ResNo) {
+  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
+  return BitConvertToInteger(Op);
+}
+
 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
   // Convert the inputs to integers, and build a new pair out of them.
   return DAG.getNode(ISD::BUILD_PAIR, N->getDebugLoc(),
@@ -827,11 +834,11 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
 #endif
     llvm_unreachable("Do not know how to expand the result of this operator!");
 
-  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break;
   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
 
+  case ISD::MERGE_VALUES:       ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
   case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
   case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
   case ISD::EXTRACT_ELEMENT:    ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
@@ -879,10 +886,10 @@ void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
   assert(NVT.getSizeInBits() == integerPartWidth &&
          "Do not know how to expand this float constant!");
   APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
-  Lo = DAG.getConstantFP(APFloat(APInt(integerPartWidth, 1,
-                                       &C.getRawData()[1])), NVT);
-  Hi = DAG.getConstantFP(APFloat(APInt(integerPartWidth, 1,
-                                       &C.getRawData()[0])), NVT);
+  Lo = DAG.getConstantFP(APFloat(APInt(integerPartWidth, C.getRawData()[1])),
+                         NVT);
+  Hi = DAG.getConstantFP(APFloat(APInt(integerPartWidth, C.getRawData()[0])),
+                         NVT);
 }
 
 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
@@ -1201,7 +1208,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
   static const uint64_t TwoE32[]  = { 0x41f0000000000000LL, 0 };
   static const uint64_t TwoE64[]  = { 0x43f0000000000000LL, 0 };
   static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
-  const uint64_t *Parts = 0;
+  ArrayRef<uint64_t> Parts;
 
   switch (SrcVT.getSimpleVT().SimpleTy) {
   default:
@@ -1218,7 +1225,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
   }
 
   Lo = DAG.getNode(ISD::FADD, dl, VT, Hi,
-                   DAG.getConstantFP(APFloat(APInt(128, 2, Parts)),
+                   DAG.getConstantFP(APFloat(APInt(128, Parts)),
                                      MVT::ppcf128));
   Lo = DAG.getNode(ISD::SELECT_CC, dl, VT, Src, DAG.getConstant(0, SrcVT),
                    Lo, Hi, DAG.getCondCode(ISD::SETLT));
@@ -1291,8 +1298,7 @@ void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
   GetExpandedFloat(NewLHS, LHSLo, LHSHi);
   GetExpandedFloat(NewRHS, RHSLo, RHSHi);
 
-  EVT VT = NewLHS.getValueType();
-  assert(VT == MVT::ppcf128 && "Unsupported setcc type!");
+  assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
 
   // FIXME:  This generated code sucks.  We want to generate
   //         FCMPU crN, hi1, hi2
@@ -1373,7 +1379,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) {
     assert(N->getOperand(0).getValueType() == MVT::ppcf128 &&
            "Logic only correct for ppcf128!");
     const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
-    APFloat APF = APFloat(APInt(128, 2, TwoE31));
+    APFloat APF = APFloat(APInt(128, TwoE31));
     SDValue Tmp = DAG.getConstantFP(APF, MVT::ppcf128);
     //  X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
     // FIXME: generated code sucks.
@@ -1445,6 +1451,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
                                      ST->getValue().getValueType());
   assert(NVT.isByteSized() && "Expanded type not byte sized!");
   assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
+  (void)NVT;
 
   SDValue Lo, Hi;
   GetExpandedOp(ST->getValue(), Lo, Hi);
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index e7c77dd10cb6..a5c4c2ded4c5 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -48,6 +48,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
     N->dump(&DAG); dbgs() << "\n";
 #endif
     llvm_unreachable("Do not know how to promote this operator!");
+  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
   case ISD::AssertSext:  Res = PromoteIntRes_AssertSext(N); break;
   case ISD::AssertZext:  Res = PromoteIntRes_AssertZext(N); break;
   case ISD::BITCAST:     Res = PromoteIntRes_BITCAST(N); break;
@@ -63,6 +64,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
                          Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
   case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break;
   case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
+  case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
   case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
   case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
   case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
@@ -84,6 +86,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
                          Res = PromoteIntRes_BUILD_VECTOR(N); break;
   case ISD::SCALAR_TO_VECTOR:
                          Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
+  case ISD::CONCAT_VECTORS:
+                         Res = PromoteIntRes_CONCAT_VECTORS(N); break;
 
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
@@ -114,6 +118,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::SMULO:
   case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
 
+  case ISD::ATOMIC_LOAD:
+    Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
+
   case ISD::ATOMIC_LOAD_ADD:
   case ISD::ATOMIC_LOAD_SUB:
   case ISD::ATOMIC_LOAD_AND:
@@ -136,6 +143,12 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
     SetPromotedInteger(SDValue(N, ResNo), Res);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
+                                                     unsigned ResNo) {
+  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
+  return GetPromotedInteger(Op);
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
   // Sign-extend the new bits, and continue the assertion.
   SDValue Op = SExtPromotedInteger(N->getOperand(0));
@@ -150,12 +163,26 @@ SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
                      Op.getValueType(), Op, N->getOperand(1));
 }
 
+SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
+  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
+                              N->getMemoryVT(), ResVT,
+                              N->getChain(), N->getBasePtr(),
+                              N->getMemOperand(), N->getOrdering(),
+                              N->getSynchScope());
+  // Legalized the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+  return Res;
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
                               N->getMemoryVT(),
                               N->getChain(), N->getBasePtr(),
-                              Op2, N->getMemOperand());
+                              Op2, N->getMemOperand(), N->getOrdering(),
+                              N->getSynchScope());
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.
   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -167,7 +194,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) {
   SDValue Op3 = GetPromotedInteger(N->getOperand(3));
   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
                               N->getMemoryVT(), N->getChain(), N->getBasePtr(),
-                              Op2, Op3, N->getMemOperand());
+                              Op2, Op3, N->getMemOperand(), N->getOrdering(),
+                              N->getSynchScope());
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.
   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -457,6 +485,14 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
                      LHS.getValueType(), N->getOperand(0),LHS,RHS);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
+  SDValue Mask = GetPromotedInteger(N->getOperand(0));
+  SDValue LHS = GetPromotedInteger(N->getOperand(1));
+  SDValue RHS = GetPromotedInteger(N->getOperand(2));
+  return DAG.getNode(ISD::VSELECT, N->getDebugLoc(),
+                     LHS.getValueType(), Mask, LHS, RHS);
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
   SDValue LHS = GetPromotedInteger(N->getOperand(2));
   SDValue RHS = GetPromotedInteger(N->getOperand(3));
@@ -467,16 +503,24 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
   EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType());
-  assert(isTypeLegal(SVT) && "Illegal SetCC type!");
+
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+
+  // Only use the result of getSetCCResultType if it is legal,
+  // otherwise just use the promoted result type (NVT).
+  if (!TLI.isTypeLegal(SVT))
+      SVT = NVT;
+
   DebugLoc dl = N->getDebugLoc();
+  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
+         "Vector compare must return a vector result!");
 
   // Get the SETCC result using the canonical SETCC type.
-  SDValue SetCC = DAG.getNode(ISD::SETCC, dl, SVT, N->getOperand(0),
+  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
                               N->getOperand(1), N->getOperand(2));
 
-  // Convert to the expected type.
-  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
+  // Convert to the expected type.
   return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
 }
 
@@ -707,6 +751,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
     llvm_unreachable("Do not know how to promote this operator's operand!");
 
   case ISD::ANY_EXTEND:   Res = PromoteIntOp_ANY_EXTEND(N); break;
+  case ISD::ATOMIC_STORE:
+    Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
+    break;
   case ISD::BITCAST:      Res = PromoteIntOp_BITCAST(N); break;
   case ISD::BR_CC:        Res = PromoteIntOp_BR_CC(N, OpNo); break;
   case ISD::BRCOND:       Res = PromoteIntOp_BRCOND(N, OpNo); break;
@@ -721,6 +768,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::MEMBARRIER:   Res = PromoteIntOp_MEMBARRIER(N); break;
   case ISD::SCALAR_TO_VECTOR:
                           Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
+  case ISD::VSELECT:
   case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
   case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
   case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
@@ -791,6 +839,13 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
   return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
+  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
+  return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(),
+                       N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
+                       N->getOrdering(), N->getSynchScope());
+}
+
 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
   // This should only occur in unusual situations like bitcasting to an
   // x86_fp80, so just turn it into a store+load
@@ -913,14 +968,17 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
-  assert(OpNo == 0 && "Only know how to promote condition");
+  assert(OpNo == 0 && "Only know how to promote the condition!");
+  SDValue Cond = N->getOperand(0);
+  EVT OpTy = N->getOperand(1).getValueType();
 
   // Promote all the way up to the canonical SetCC type.
-  EVT SVT = TLI.getSetCCResultType(N->getOperand(1).getValueType());
-  SDValue Cond = PromoteTargetBoolean(N->getOperand(0), SVT);
+  EVT SVT = TLI.getSetCCResultType(N->getOpcode() == ISD::SELECT ?
+                                   OpTy.getScalarType() : OpTy);
+  Cond = PromoteTargetBoolean(Cond, SVT);
 
-  return SDValue(DAG.UpdateNodeOperands(N, Cond,
-                                N->getOperand(1), N->getOperand(2)), 0);
+  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
+                                        N->getOperand(2)), 0);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
@@ -1024,7 +1082,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
 #endif
     llvm_unreachable("Do not know how to expand the result of this operator!");
 
-  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break;
+  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
@@ -1055,6 +1113,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
   case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
   case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
+  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
 
   case ISD::ATOMIC_LOAD_ADD:
   case ISD::ATOMIC_LOAD_SUB:
@@ -1546,6 +1605,12 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
 }
 
+void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
+                                                 SDValue &Lo, SDValue &Hi) {
+  SDValue Res = DisintegrateMERGE_VALUES(N, ResNo);
+  SplitInteger(Res, Lo, Hi);
+}
+
 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
                                                SDValue &Lo, SDValue &Hi) {
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
@@ -2176,9 +2241,9 @@ void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
                                           SDValue &Lo, SDValue &Hi) {
   EVT VT = N->getValueType(0);
-  const Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
+  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
   EVT PtrVT = TLI.getPointerTy();
-  const Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
+  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
   DebugLoc dl = N->getDebugLoc();
 
   // A divide for UMULO should be faster than a function call.
@@ -2222,7 +2287,7 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
   TargetLowering::ArgListEntry Entry;
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
     EVT ArgVT = N->getOperand(i).getValueType();
-    const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = N->getOperand(i);
     Entry.Ty = ArgTy;
     Entry.isSExt = true;
@@ -2321,6 +2386,20 @@ void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
   }
 }
 
+void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
+                                                SDValue &Lo, SDValue &Hi) {
+  DebugLoc dl = N->getDebugLoc();
+  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
+  SDValue Zero = DAG.getConstant(0, VT);
+  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT,
+                               N->getOperand(0),
+                               N->getOperand(1), Zero, Zero,
+                               cast<AtomicSDNode>(N)->getMemOperand(),
+                               cast<AtomicSDNode>(N)->getOrdering(),
+                               cast<AtomicSDNode>(N)->getSynchScope());
+  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
+  ReplaceValueWith(SDValue(N, 1), Swap.getValue(1));
+}
 
 //===----------------------------------------------------------------------===//
 //  Integer Operand Expansion
@@ -2365,6 +2444,8 @@ bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
   case ISD::RETURNADDR:
   case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
+
+  case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
   }
 
   // If the result is null, the sub-method took care of registering results etc.
@@ -2742,6 +2823,19 @@ SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
   return MakeLibCall(LC, DstVT, &Op, 1, true, dl);
 }
 
+SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
+  DebugLoc dl = N->getDebugLoc();
+  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
+                               cast<AtomicSDNode>(N)->getMemoryVT(),
+                               N->getOperand(0),
+                               N->getOperand(1), N->getOperand(2),
+                               cast<AtomicSDNode>(N)->getMemOperand(),
+                               cast<AtomicSDNode>(N)->getOrdering(),
+                               cast<AtomicSDNode>(N)->getSynchScope());
+  return Swap.getValue(1);
+}
+
+
 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
   SDValue InOp0 = N->getOperand(0);
   EVT InVT = InOp0.getValueType();
@@ -2775,7 +2869,6 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
 
 
 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
-
   ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
   EVT VT = N->getValueType(0);
   DebugLoc dl = N->getDebugLoc();
@@ -2830,6 +2923,46 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
   return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
+  DebugLoc dl = N->getDebugLoc();
+
+  SDValue Op0 = N->getOperand(1);
+  SDValue Op1 = N->getOperand(1);
+  assert(Op0.getValueType() == Op1.getValueType() &&
+         "Invalid input vector types");
+
+  EVT OutVT = N->getValueType(0);
+  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
+  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
+
+  EVT OutElemTy = NOutVT.getVectorElementType();
+
+  unsigned NumElem0 = Op0.getValueType().getVectorNumElements();
+  unsigned NumElem1 = Op1.getValueType().getVectorNumElements();
+  unsigned NumOutElem = NOutVT.getVectorNumElements();
+  assert(NumElem0 + NumElem1 == NumOutElem &&
+         "Invalid number of incoming elements");
+
+  // Take the elements from the first vector.
+  SmallVector<SDValue, 8> Ops(NumOutElem);
+  for (unsigned i = 0; i < NumElem0; ++i) {
+    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                              Op0.getValueType().getScalarType(), Op0,
+                              DAG.getIntPtrConstant(i));
+    Ops[i] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
+  }
+
+  // Take the elements from the second vector
+  for (unsigned i = 0; i < NumElem1; ++i) {
+    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                              Op1.getValueType().getScalarType(), Op1,
+                              DAG.getIntPtrConstant(i));
+    Ops[i + NumElem0] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
+  }
+
+  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size());
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
   EVT OutVT = N->getValueType(0);
   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
@@ -2838,14 +2971,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
   EVT NOutVTElem = NOutVT.getVectorElementType();
 
   DebugLoc dl = N->getDebugLoc();
-
-  SDValue ConvertedVector = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
-                                        N->getOperand(0));
+  SDValue V0 = GetPromotedInteger(N->getOperand(0));
 
   SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
     NOutVTElem, N->getOperand(1));
-  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,NOutVT,
-    ConvertedVector, ConvElem, N->getOperand(2));
+  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
+    V0, ConvElem, N->getOperand(2));
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
@@ -2855,20 +2986,23 @@ SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
     V0->getValueType(0).getScalarType(), V0, V1);
 
-  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
-
+  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
+  // element types. If this is the case then we need to expand the outgoing
+  // value and not truncate it.
+  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
-
   DebugLoc dl = N->getDebugLoc();
+  unsigned NumElems = N->getNumOperands();
 
   EVT RetSclrTy = N->getValueType(0).getVectorElementType();
 
   SmallVector<SDValue, 8> NewOps;
+  NewOps.reserve(NumElems);
 
   // For each incoming vector
-  for (unsigned VecIdx = 0, E = N->getNumOperands(); VecIdx!= E; ++VecIdx) {
+  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
     SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
     EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
     unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index ba658b022053..a4bb577433cc 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -946,6 +946,13 @@ bool DAGTypeLegalizer::CustomWidenLowerNode(SDNode *N, EVT VT) {
   return true;
 }
 
+SDValue DAGTypeLegalizer::DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo) {
+  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
+    if (i != ResNo)
+      ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i)));
+  return SDValue(N, ResNo);
+}
+
 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
 /// which is split into two not necessarily identical pieces.
 void DAGTypeLegalizer::GetSplitDestVTs(EVT InVT, EVT &LoVT, EVT &HiVT) {
@@ -1046,7 +1053,7 @@ SDValue DAGTypeLegalizer::MakeLibCall(RTLIB::Libcall LC, EVT RetVT,
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
                                          TLI.getPointerTy());
 
-  const Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
+  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   std::pair<SDValue,SDValue> CallInfo =
     TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false,
                     false, 0, TLI.getLibcallCallingConv(LC), false,
@@ -1067,7 +1074,7 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC,
   TargetLowering::ArgListEntry Entry;
   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
     EVT ArgVT = Node->getOperand(i).getValueType();
-    const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = Node->getOperand(i);
     Entry.Ty = ArgTy;
     Entry.isSExt = isSigned;
@@ -1078,7 +1085,7 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC,
                                          TLI.getPointerTy());
 
   // Splice the libcall in wherever FindInputOutputChains tells us to.
-  const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
+  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
   std::pair<SDValue, SDValue> CallInfo =
     TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false,
                     0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false,
@@ -1093,24 +1100,8 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC,
 /// type i1, the bits of which conform to getBooleanContents.
 SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT VT) {
   DebugLoc dl = Bool.getDebugLoc();
-  ISD::NodeType ExtendCode;
-  switch (TLI.getBooleanContents()) {
-  default:
-    assert(false && "Unknown BooleanContent!");
-  case TargetLowering::UndefinedBooleanContent:
-    // Extend to VT by adding rubbish bits.
-    ExtendCode = ISD::ANY_EXTEND;
-    break;
-  case TargetLowering::ZeroOrOneBooleanContent:
-    // Extend to VT by adding zero bits.
-    ExtendCode = ISD::ZERO_EXTEND;
-    break;
-  case TargetLowering::ZeroOrNegativeOneBooleanContent: {
-    // Extend to VT by copying the sign bit.
-    ExtendCode = ISD::SIGN_EXTEND;
-    break;
-  }
-  }
+  ISD::NodeType ExtendCode =
+    TargetLowering::getExtendForContent(TLI.getBooleanContents(VT.isVector()));
   return DAG.getNode(ExtendCode, dl, VT, Bool);
 }
 
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 952797dc75b8..abacdac686bc 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -148,15 +148,22 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
   bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
   bool CustomWidenLowerNode(SDNode *N, EVT VT);
+
+  /// DisintegrateMERGE_VALUES - Replace each result of the given MERGE_VALUES
+  /// node with the corresponding input operand, except for the result 'ResNo',
+  /// which is returned.
+  SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
+
   SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index);
   SDValue JoinIntegers(SDValue Lo, SDValue Hi);
   SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
   SDValue MakeLibCall(RTLIB::Libcall LC, EVT RetVT,
                       const SDValue *Ops, unsigned NumOps, bool isSigned,
                       DebugLoc dl);
-	std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
-									                               SDNode *Node, bool isSigned);
-	std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
+  
+  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
+                                                 SDNode *Node, bool isSigned);
+  std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
 
   SDValue PromoteTargetBoolean(SDValue Bool, EVT VT);
   void ReplaceValueWith(SDValue From, SDValue To);
@@ -206,8 +213,10 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   // Integer Result Promotion.
   void PromoteIntegerResult(SDNode *N, unsigned ResNo);
+  SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_AssertSext(SDNode *N);
   SDValue PromoteIntRes_AssertZext(SDNode *N);
+  SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
   SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
   SDValue PromoteIntRes_Atomic2(AtomicSDNode *N);
   SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
@@ -215,6 +224,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
   SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
   SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
+  SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
   SDValue PromoteIntRes_BITCAST(SDNode *N);
   SDValue PromoteIntRes_BSWAP(SDNode *N);
   SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
@@ -232,6 +242,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_SDIV(SDNode *N);
   SDValue PromoteIntRes_SELECT(SDNode *N);
+  SDValue PromoteIntRes_VSELECT(SDNode *N);
   SDValue PromoteIntRes_SELECT_CC(SDNode *N);
   SDValue PromoteIntRes_SETCC(SDNode *N);
   SDValue PromoteIntRes_SHL(SDNode *N);
@@ -249,6 +260,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   // Integer Operand Promotion.
   bool PromoteIntegerOperand(SDNode *N, unsigned OperandNo);
   SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
+  SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
   SDValue PromoteIntOp_BITCAST(SDNode *N);
   SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
   SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
@@ -264,6 +276,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
+  SDValue PromoteIntOp_VSETCC(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_Shift(SDNode *N);
   SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
   SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
@@ -289,6 +302,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   // Integer Result Expansion.
   void ExpandIntegerResult(SDNode *N, unsigned ResNo);
+  void ExpandIntRes_MERGE_VALUES      (SDNode *N, unsigned ResNo,
+                                       SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_ANY_EXTEND        (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_AssertSext        (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_AssertZext        (SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -320,6 +335,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void ExpandIntRes_UADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_XMULO             (SDNode *N, SDValue &Lo, SDValue &Hi);
 
+  void ExpandIntRes_ATOMIC_LOAD       (SDNode *N, SDValue &Lo, SDValue &Hi);
+
   void ExpandShiftByConstant(SDNode *N, unsigned Amt,
                              SDValue &Lo, SDValue &Hi);
   bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -339,6 +356,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue ExpandIntOp_TRUNCATE(SDNode *N);
   SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
   SDValue ExpandIntOp_RETURNADDR(SDNode *N);
+  SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
 
   void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
                                   ISD::CondCode &CCCode, DebugLoc dl);
@@ -362,6 +380,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   // Result Float to Integer Conversion.
   void SoftenFloatResult(SDNode *N, unsigned OpNo);
+  SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
   SDValue SoftenFloatRes_BITCAST(SDNode *N);
   SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
   SDValue SoftenFloatRes_ConstantFP(ConstantFPSDNode *N);
@@ -488,6 +507,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   // Vector Result Scalarization: <1 x ty> -> ty.
   void ScalarizeVectorResult(SDNode *N, unsigned OpNo);
+  SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
   SDValue ScalarizeVecRes_BinOp(SDNode *N);
   SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
   SDValue ScalarizeVecRes_InregOp(SDNode *N);
@@ -559,6 +579,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
   SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
+  SDValue SplitVecOp_VSETCC(SDNode *N);
   SDValue SplitVecOp_FP_ROUND(SDNode *N);
 
   //===--------------------------------------------------------------------===//
@@ -581,6 +602,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   // Widen Vector Result Promotion.
   void WidenVectorResult(SDNode *N, unsigned ResNo);
+  SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
   SDValue WidenVecRes_BITCAST(SDNode* N);
   SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
   SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
@@ -677,7 +699,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
 
   // Generic Result Splitting.
-  void SplitRes_MERGE_VALUES(SDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
+                             SDValue &Lo, SDValue &Hi);
   void SplitRes_SELECT      (SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitRes_SELECT_CC   (SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitRes_UNDEF       (SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -699,6 +722,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   }
 
   // Generic Result Expansion.
+  void ExpandRes_MERGE_VALUES      (SDNode *N, unsigned ResNo,
+                                    SDValue &Lo, SDValue &Hi);
   void ExpandRes_BITCAST           (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandRes_BUILD_PAIR        (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandRes_EXTRACT_ELEMENT   (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 85ea6b662044..8e7e4985e4d0 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -31,6 +31,11 @@ using namespace llvm;
 // These routines assume that the Lo/Hi part is stored first in memory on
 // little/big-endian machines, followed by the Hi/Lo part.  This means that
 // they cannot be used as is on vectors, for which Lo is always stored first.
+void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
+                                              SDValue &Lo, SDValue &Hi) {
+  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
+  GetExpandedOp(Op, Lo, Hi);
+}
 
 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
   EVT OutVT = N->getValueType(0);
@@ -426,37 +431,34 @@ SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
 // bytes; for integers and floats it is Lo first if and only if the machine is
 // little-endian).
 
-void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N,
+void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
                                              SDValue &Lo, SDValue &Hi) {
-  // A MERGE_VALUES node can produce any number of values.  We know that the
-  // first illegal one needs to be expanded into Lo/Hi.
-  unsigned i;
-
-  // The string of legal results gets turned into input operands, which have
-  // the same type.
-  for (i = 0; isTypeLegal(N->getValueType(i)); ++i)
-    ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i)));
-
-  // The first illegal result must be the one that needs to be expanded.
-  GetSplitOp(N->getOperand(i), Lo, Hi);
-
-  // Legalize the rest of the results into the input operands whether they are
-  // legal or not.
-  unsigned e = N->getNumValues();
-  for (++i; i != e; ++i)
-    ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i)));
+  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
+  GetSplitOp(Op, Lo, Hi);
 }
 
 void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo,
                                        SDValue &Hi) {
-  SDValue LL, LH, RL, RH;
+  SDValue LL, LH, RL, RH, CL, CH;
   DebugLoc dl = N->getDebugLoc();
   GetSplitOp(N->getOperand(1), LL, LH);
   GetSplitOp(N->getOperand(2), RL, RH);
 
   SDValue Cond = N->getOperand(0);
-  Lo = DAG.getNode(ISD::SELECT, dl, LL.getValueType(), Cond, LL, RL);
-  Hi = DAG.getNode(ISD::SELECT, dl, LH.getValueType(), Cond, LH, RH);
+  CL = CH = Cond;
+  if (Cond.getValueType().isVector()) {
+    assert(Cond.getValueType().getVectorElementType() == MVT::i1 &&
+           "Condition legalized before result?");
+    unsigned NumElements = Cond.getValueType().getVectorNumElements();
+    EVT VCondTy = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElements / 2);
+    CL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VCondTy, Cond,
+                     DAG.getIntPtrConstant(0));
+    CH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VCondTy, Cond,
+                     DAG.getIntPtrConstant(NumElements / 2));
+  }
+
+  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL);
+  Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH);
 }
 
 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index ffff10ce2948..f815b00db5d6 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -61,6 +61,9 @@ class VectorLegalizer {
   // Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
   // SINT_TO_FLOAT and SHR on vectors isn't legal.
   SDValue ExpandUINT_TO_FLOAT(SDValue Op);
+  // Implement vselect in terms of XOR, AND, OR when blend is not supported
+  // by the target.
+  SDValue ExpandVSELECT(SDValue Op);
   SDValue ExpandFNEG(SDValue Op);
   // Implements vector promotion; this is essentially just bitcasting the
   // operands to a different type and bitcasting the result back to the
@@ -157,8 +160,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::CTLZ:
   case ISD::CTPOP:
   case ISD::SELECT:
+  case ISD::VSELECT:
   case ISD::SELECT_CC:
-  case ISD::VSETCC:
+  case ISD::SETCC:
   case ISD::ZERO_EXTEND:
   case ISD::ANY_EXTEND:
   case ISD::TRUNCATE:
@@ -210,11 +214,13 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
     // FALL THROUGH
   }
   case TargetLowering::Expand:
-    if (Node->getOpcode() == ISD::UINT_TO_FP)
+    if (Node->getOpcode() == ISD::VSELECT)
+      Result = ExpandVSELECT(Op);
+    else if (Node->getOpcode() == ISD::UINT_TO_FP)
       Result = ExpandUINT_TO_FLOAT(Op);
     else if (Node->getOpcode() == ISD::FNEG)
       Result = ExpandFNEG(Op);
-    else if (Node->getOpcode() == ISD::VSETCC)
+    else if (Node->getOpcode() == ISD::SETCC)
       Result = UnrollVSETCC(Op);
     else
       Result = DAG.UnrollVectorOp(Op.getNode());
@@ -256,9 +262,41 @@ SDValue VectorLegalizer::PromoteVectorOp(SDValue Op) {
   return DAG.getNode(ISD::BITCAST, dl, VT, Op);
 }
 
+SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
+  // Implement VSELECT in terms of XOR, AND, OR
+  // on platforms which do not support blend natively.
+  EVT VT =  Op.getOperand(0).getValueType();
+  DebugLoc DL = Op.getDebugLoc();
+
+  SDValue Mask = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  SDValue Op2 = Op.getOperand(2);
+
+  // If we can't even use the basic vector operations of
+  // AND,OR,XOR, we will have to scalarize the op.
+  if (!TLI.isOperationLegalOrCustom(ISD::AND, VT) ||
+      !TLI.isOperationLegalOrCustom(ISD::XOR, VT) ||
+      !TLI.isOperationLegalOrCustom(ISD::OR, VT))
+        return DAG.UnrollVectorOp(Op.getNode());
+
+  assert(VT.getSizeInBits() == Op.getOperand(1).getValueType().getSizeInBits()
+         && "Invalid mask size");
+  // Bitcast the operands to be the same type as the mask.
+  // This is needed when we select between FP types because
+  // the mask is a vector of integers.
+  Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
+  Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
+
+  SDValue AllOnes = DAG.getConstant(
+    APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), VT);
+  SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes);
+
+  Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
+  Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
+  return DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
+}
+
 SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
-
-
   EVT VT = Op.getOperand(0).getValueType();
   DebugLoc DL = Op.getDebugLoc();
 
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index b5698f9c6738..107a42b2951c 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -44,8 +44,10 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
     N->dump(&DAG);
     dbgs() << "\n";
 #endif
-    llvm_unreachable("Do not know how to scalarize the result of this operator!");
+    report_fatal_error("Do not know how to scalarize the result of this "
+                       "operator!\n");
 
+  case ISD::MERGE_VALUES:      R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
   case ISD::BITCAST:           R = ScalarizeVecRes_BITCAST(N); break;
   case ISD::BUILD_VECTOR:      R = N->getOperand(0); break;
   case ISD::CONVERT_RNDSAT:    R = ScalarizeVecRes_CONVERT_RNDSAT(N); break;
@@ -62,8 +64,6 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SETCC:             R = ScalarizeVecRes_SETCC(N); break;
   case ISD::UNDEF:             R = ScalarizeVecRes_UNDEF(N); break;
   case ISD::VECTOR_SHUFFLE:    R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
-  case ISD::VSETCC:            R = ScalarizeVecRes_VSETCC(N); break;
-
   case ISD::ANY_EXTEND:
   case ISD::CTLZ:
   case ISD::CTPOP:
@@ -129,6 +129,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
                      LHS.getValueType(), LHS, RHS);
 }
 
+SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
+                                                       unsigned ResNo) {
+  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
+  return GetScalarizedVector(Op);
+}
+
 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
   EVT NewVT = N->getValueType(0).getVectorElementType();
   return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
@@ -237,6 +243,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
+  assert(N->getValueType(0).isVector() ==
+         N->getOperand(0).getValueType().isVector() &&
+         "Scalar/Vector type mismatch");
+
+  if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N);
+
   SDValue LHS = GetScalarizedVector(N->getOperand(0));
   SDValue RHS = GetScalarizedVector(N->getOperand(1));
   DebugLoc DL = N->getDebugLoc();
@@ -259,35 +271,23 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
+  assert(N->getValueType(0).isVector() &&
+         N->getOperand(0).getValueType().isVector() &&
+         "Operand types must be vectors");
+
   SDValue LHS = GetScalarizedVector(N->getOperand(0));
   SDValue RHS = GetScalarizedVector(N->getOperand(1));
   EVT NVT = N->getValueType(0).getVectorElementType();
-  EVT SVT = TLI.getSetCCResultType(LHS.getValueType());
   DebugLoc DL = N->getDebugLoc();
 
   // Turn it into a scalar SETCC.
-  SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2));
-
-  // VSETCC always returns a sign-extended value, while SETCC may not.  The
-  // SETCC result type may not match the vector element type.  Correct these.
-  if (NVT.bitsLE(SVT)) {
-    // The SETCC result type is bigger than the vector element type.
-    // Ensure the SETCC result is sign-extended.
-    if (TLI.getBooleanContents() !=
-        TargetLowering::ZeroOrNegativeOneBooleanContent)
-      Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res,
-                        DAG.getValueType(MVT::i1));
-    // Truncate to the final type.
-    return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res);
-  }
-
-  // The SETCC result type is smaller than the vector element type.
-  // If the SetCC result is not sign-extended, chop it down to MVT::i1.
-  if (TLI.getBooleanContents() !=
-        TargetLowering::ZeroOrNegativeOneBooleanContent)
-    Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res);
-  // Sign extend to the final type.
-  return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res);
+  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
+                            N->getOperand(2));
+  // Vectors may have a different boolean contents to scalars.  Promote the
+  // value appropriately.
+  ISD::NodeType ExtendCode =
+    TargetLowering::getExtendForContent(TLI.getBooleanContents(true));
+  return DAG.getNode(ExtendCode, DL, NVT, Res);
 }
 
 
@@ -415,7 +415,8 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
 #endif
     llvm_unreachable("Do not know how to split the result of this operator!");
 
-  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break;
+  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
+  case ISD::VSELECT:
   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
@@ -432,7 +433,6 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
     break;
   case ISD::SETCC:
-  case ISD::VSETCC:
     SplitVecRes_SETCC(N, Lo, Hi);
     break;
   case ISD::VECTOR_SHUFFLE:
@@ -524,12 +524,11 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
 
   // Handle some special cases efficiently.
   switch (getTypeAction(InVT)) {
-  default:
-    assert(false && "Unknown type action!");
   case TargetLowering::TypeLegal:
   case TargetLowering::TypePromoteInteger:
   case TargetLowering::TypeSoftenFloat:
   case TargetLowering::TypeScalarizeVector:
+  case TargetLowering::TypeWidenVector:
     break;
   case TargetLowering::TypeExpandInteger:
   case TargetLowering::TypeExpandFloat:
@@ -670,7 +669,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
   // Store the new element.  This may be larger than the vector element type,
   // so use a truncating store.
   SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
-  const Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
+  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
   unsigned Alignment =
     TLI.getTargetData()->getPrefTypeAlignment(VecType);
   Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT,
@@ -740,6 +739,10 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
 }
 
 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
+  assert(N->getValueType(0).isVector() &&
+         N->getOperand(0).getValueType().isVector() &&
+         "Operand types must be vectors");
+
   EVT LoVT, HiVT;
   DebugLoc DL = N->getDebugLoc();
   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
@@ -965,7 +968,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
       dbgs() << "\n";
 #endif
       llvm_unreachable("Do not know how to split this operator's operand!");
-
+    case ISD::SETCC:             Res = SplitVecOp_VSETCC(N); break;
     case ISD::BITCAST:           Res = SplitVecOp_BITCAST(N); break;
     case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
     case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
@@ -1163,6 +1166,26 @@ SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
                      &Elts[0], Elts.size());
 }
 
+SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
+  assert(N->getValueType(0).isVector() &&
+         N->getOperand(0).getValueType().isVector() &&
+         "Operand types must be vectors");
+  // The result has a legal vector type, but the input needs splitting.
+  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
+  DebugLoc DL = N->getDebugLoc();
+  GetSplitVector(N->getOperand(0), Lo0, Hi0);
+  GetSplitVector(N->getOperand(1), Lo1, Hi1);
+  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
+  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
+  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
+
+  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
+  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
+  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
+  return PromoteTargetBoolean(Con, N->getValueType(0));
+}
+
+
 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
   // The result has a legal vector type, but the input needs splitting.
   EVT ResVT = N->getValueType(0);
@@ -1205,6 +1228,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
 #endif
     llvm_unreachable("Do not know how to widen the result of this operator!");
 
+  case ISD::MERGE_VALUES:      Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
   case ISD::BITCAST:           Res = WidenVecRes_BITCAST(N); break;
   case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
   case ISD::CONCAT_VECTORS:    Res = WidenVecRes_CONCAT_VECTORS(N); break;
@@ -1222,10 +1246,6 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::VECTOR_SHUFFLE:
     Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
     break;
-  case ISD::VSETCC:
-    Res = WidenVecRes_VSETCC(N);
-    break;
-
   case ISD::ADD:
   case ISD::AND:
   case ISD::BSWAP:
@@ -1557,6 +1577,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
                      WidenVT, WidenLHS, DAG.getValueType(ExtVT));
 }
 
+SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
+  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
+  return GetWidenedVector(WidenVec);
+}
+
 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
   SDValue InOp = N->getOperand(0);
   EVT InVT = InOp.getValueType();
@@ -1661,6 +1686,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   DebugLoc dl = N->getDebugLoc();
   unsigned WidenNumElts = WidenVT.getVectorNumElements();
+  unsigned NumInElts = InVT.getVectorNumElements();
   unsigned NumOperands = N->getNumOperands();
 
   bool InputWidened = false; // Indicates we need to widen the input.
@@ -1686,17 +1712,17 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
         if (N->getOperand(i).getOpcode() != ISD::UNDEF)
           break;
 
-      if (i > NumOperands)
+      if (i == NumOperands)
         // Everything but the first operand is an UNDEF so just return the
         // widened first operand.
         return GetWidenedVector(N->getOperand(0));
 
       if (NumOperands == 2) {
         // Replace concat of two operands with a shuffle.
-        SmallVector<int, 16> MaskOps(WidenNumElts);
-        for (unsigned i=0; i < WidenNumElts/2; ++i) {
+        SmallVector<int, 16> MaskOps(WidenNumElts, -1);
+        for (unsigned i = 0; i < NumInElts; ++i) {
           MaskOps[i] = i;
-          MaskOps[i+WidenNumElts/2] = i+WidenNumElts;
+          MaskOps[i + NumInElts] = i + WidenNumElts;
         }
         return DAG.getVectorShuffle(WidenVT, dl,
                                     GetWidenedVector(N->getOperand(0)),
@@ -1708,7 +1734,6 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
 
   // Fall back to use extracts and build vector.
   EVT EltVT = WidenVT.getVectorElementType();
-  unsigned NumInElts = InVT.getVectorNumElements();
   SmallVector<SDValue, 16> Ops(WidenNumElts);
   unsigned Idx = 0;
   for (unsigned i=0; i < NumOperands; ++i) {
@@ -1916,6 +1941,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
+  assert(N->getValueType(0).isVector() ==
+         N->getOperand(0).getValueType().isVector() &&
+         "Scalar/Vector type mismatch");
+  if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N);
+
   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
@@ -1954,6 +1984,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
+  assert(N->getValueType(0).isVector() &&
+         N->getOperand(0).getValueType().isVector() &&
+         "Operands must be vectors");
   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   unsigned WidenNumElts = WidenVT.getVectorNumElements();
 
@@ -1970,7 +2003,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
   assert(InOp1.getValueType() == WidenInVT &&
          InOp2.getValueType() == WidenInVT &&
          "Input not widened to expected type!");
-  return DAG.getNode(ISD::VSETCC, N->getDebugLoc(),
+  (void)WidenInVT;
+  return DAG.getNode(ISD::SETCC, N->getDebugLoc(),
                      WidenVT, InOp1, InOp2, N->getOperand(2));
 }
 
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 12b183804c28..e757defd3895 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -2621,6 +2621,39 @@ bool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) {
   return false;
 }
 
+/// canClobberReachingPhysRegUse - True if SU would clobber one of it's
+/// successor's explicit physregs whose definition can reach DepSU.
+/// i.e. DepSU should not be scheduled above SU.
+static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU,
+                                         ScheduleDAGRRList *scheduleDAG,
+                                         const TargetInstrInfo *TII,
+                                         const TargetRegisterInfo *TRI) {
+  const unsigned *ImpDefs
+    = TII->get(SU->getNode()->getMachineOpcode()).getImplicitDefs();
+  if(!ImpDefs)
+    return false;
+
+  for (SUnit::const_succ_iterator SI = SU->Succs.begin(), SE = SU->Succs.end();
+       SI != SE; ++SI) {
+    SUnit *SuccSU = SI->getSUnit();
+    for (SUnit::const_pred_iterator PI = SuccSU->Preds.begin(),
+           PE = SuccSU->Preds.end(); PI != PE; ++PI) {
+      if (!PI->isAssignedRegDep())
+        continue;
+
+      for (const unsigned *ImpDef = ImpDefs; *ImpDef; ++ImpDef) {
+        // Return true if SU clobbers this physical register use and the
+        // definition of the register reaches from DepSU. IsReachable queries a
+        // topological forward sort of the DAG (following the successors).
+        if (TRI->regsOverlap(*ImpDef, PI->getReg()) &&
+            scheduleDAG->IsReachable(DepSU, PI->getSUnit()))
+          return true;
+      }
+    }
+  }
+  return false;
+}
+
 /// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's
 /// physical register defs.
 static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU,
@@ -2837,7 +2870,8 @@ void RegReductionPQBase::AddPseudoTwoAddrDeps() {
             SuccOpc == TargetOpcode::INSERT_SUBREG ||
             SuccOpc == TargetOpcode::SUBREG_TO_REG)
           continue;
-        if ((!canClobber(SuccSU, DUSU) ||
+        if (!canClobberReachingPhysRegUse(SuccSU, SU, scheduleDAG, TII, TRI) &&
+            (!canClobber(SuccSU, DUSU) ||
              (isLiveOut && !hasOnlyLiveOutUses(SuccSU)) ||
              (!SU->isCommutable && SuccSU->isCommutable)) &&
             !scheduleDAG->IsReachable(SuccSU, SU)) {
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 35ea0bb940b5..20bea8e4c9e9 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -403,7 +403,7 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
     ID.AddInteger(CP->getAlignment());
     ID.AddInteger(CP->getOffset());
     if (CP->isMachineConstantPoolEntry())
-      CP->getMachineCPVal()->AddSelectionDAGCSEId(ID);
+      CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
     else
       ID.AddPointer(CP->getConstVal());
     ID.AddInteger(CP->getTargetFlags());
@@ -432,7 +432,9 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
   case ISD::ATOMIC_LOAD_MIN:
   case ISD::ATOMIC_LOAD_MAX:
   case ISD::ATOMIC_LOAD_UMIN:
-  case ISD::ATOMIC_LOAD_UMAX: {
+  case ISD::ATOMIC_LOAD_UMAX:
+  case ISD::ATOMIC_LOAD:
+  case ISD::ATOMIC_STORE: {
     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
     ID.AddInteger(AT->getMemoryVT().getRawBits());
     ID.AddInteger(AT->getRawSubclassData());
@@ -769,11 +771,14 @@ static void VerifyNodeCommon(SDNode *N) {
     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
            "Wrong number of operands!");
     EVT EltVT = N->getValueType(0).getVectorElementType();
-    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
+    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
       assert((I->getValueType() == EltVT ||
              (EltVT.isInteger() && I->getValueType().isInteger() &&
               EltVT.bitsLE(I->getValueType()))) &&
             "Wrong operand type!");
+      assert(I->getValueType() == N->getOperand(0).getValueType() &&
+             "Operands must all have the same type");
+    }
     break;
   }
   }
@@ -821,7 +826,7 @@ static void VerifyMachineNode(SDNode *N) {
 /// given type.
 ///
 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
-  const Type *Ty = VT == MVT::iPTR ?
+  Type *Ty = VT == MVT::iPTR ?
                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
                    VT.getTypeForEVT(*getContext());
 
@@ -876,6 +881,12 @@ void SelectionDAG::clear() {
   DbgInfo->clear();
 }
 
+SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) {
+  return VT.bitsGT(Op.getValueType()) ?
+    getNode(ISD::ANY_EXTEND, DL, VT, Op) :
+    getNode(ISD::TRUNCATE, DL, VT, Op);
+}
+
 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) {
   return VT.bitsGT(Op.getValueType()) ?
     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
@@ -925,13 +936,25 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) {
   assert(VT.isInteger() && "Cannot create FP integer constant!");
 
   EVT EltVT = VT.getScalarType();
-  assert(Val.getBitWidth() == EltVT.getSizeInBits() &&
-         "APInt size does not match type size!");
+  const ConstantInt *Elt = &Val;
 
+  // In some cases the vector type is legal but the element type is illegal and
+  // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
+  // inserted value (the type does not need to match the vector element type).
+  // Any extra bits introduced will be truncated away.
+  if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) ==
+      TargetLowering::TypePromoteInteger) {
+   EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT);
+   APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
+   Elt = ConstantInt::get(*getContext(), NewVal);
+  }
+
+  assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
+         "APInt size does not match type size!");
   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
-  ID.AddPointer(&Val);
+  ID.AddPointer(Elt);
   void *IP = 0;
   SDNode *N = NULL;
   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
@@ -939,7 +962,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) {
       return SDValue(N, 0);
 
   if (!N) {
-    N = new (NodeAllocator) ConstantSDNode(isT, &Val, EltVT);
+    N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT);
     CSEMap.InsertNode(N, IP);
     AllNodes.push_back(N);
   }
@@ -1131,7 +1154,7 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
   ID.AddInteger(Alignment);
   ID.AddInteger(Offset);
-  C->AddSelectionDAGCSEId(ID);
+  C->addSelectionDAGCSEId(ID);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
@@ -1432,7 +1455,7 @@ SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
   unsigned ByteSize = VT.getStoreSize();
-  const Type *Ty = VT.getTypeForEVT(*getContext());
+  Type *Ty = VT.getTypeForEVT(*getContext());
   unsigned StackAlign =
   std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign);
 
@@ -1445,8 +1468,8 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
                             VT2.getStoreSizeInBits())/8;
-  const Type *Ty1 = VT1.getTypeForEVT(*getContext());
-  const Type *Ty2 = VT2.getTypeForEVT(*getContext());
+  Type *Ty1 = VT1.getTypeForEVT(*getContext());
+  Type *Ty2 = VT2.getTypeForEVT(*getContext());
   const TargetData *TD = TLI.getTargetData();
   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
                             TD->getPrefTypeAlignment(Ty2));
@@ -1718,8 +1741,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask,
     // The boolean result conforms to getBooleanContents.  Fall through.
   case ISD::SETCC:
     // If we know the result of a setcc has the top bits zero, use this info.
-    if (TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent &&
-        BitWidth > 1)
+    if (TLI.getBooleanContents(Op.getValueType().isVector()) ==
+        TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1)
       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
     return;
   case ISD::SHL:
@@ -2153,7 +2176,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
     // The boolean result conforms to getBooleanContents.  Fall through.
   case ISD::SETCC:
     // If setcc returns 0/-1, all bits are sign bits.
-    if (TLI.getBooleanContents() ==
+    if (TLI.getBooleanContents(Op.getValueType().isVector()) ==
         TargetLowering::ZeroOrNegativeOneBooleanContent)
       return VTBits;
     break;
@@ -2437,7 +2460,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL,
                               APFloat::rmTowardZero, &ignored);
         if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
           break;
-        APInt api(VT.getSizeInBits(), 2, x);
+        APInt api(VT.getSizeInBits(), x);
         return getConstant(api, VT);
       }
       case ISD::BITCAST:
@@ -2777,6 +2800,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
            "Vector element counts must match in FP_ROUND_INREG");
     assert(EVT.bitsLE(VT) && "Not rounding down!");
+    (void)EVT;
     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
     break;
   }
@@ -2884,6 +2908,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
     assert(!N1.getValueType().isVector() && !VT.isVector() &&
            (N1.getValueType().isInteger() == VT.isInteger()) &&
+           N1.getValueType() != VT &&
            "Wrong types for EXTRACT_ELEMENT!");
 
     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
@@ -3425,7 +3450,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
     return SDValue();
 
   if (DstAlignCanChange) {
-    const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
+    Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
@@ -3514,7 +3539,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
     return SDValue();
 
   if (DstAlignCanChange) {
-    const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
+    Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
@@ -3589,7 +3614,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl,
     return SDValue();
 
   if (DstAlignCanChange) {
-    const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
+    Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
@@ -3782,7 +3807,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
     return Result;
 
   // Emit a library call.
-  const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext());
+  Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   Entry.Node = Dst; Entry.Ty = IntPtrTy;
@@ -3815,7 +3840,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Chain, SDValue Ptr, SDValue Cmp,
                                 SDValue Swp, MachinePointerInfo PtrInfo,
-                                unsigned Alignment) {
+                                unsigned Alignment,
+                                AtomicOrdering Ordering,
+                                SynchronizationScope SynchScope) {                                
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(MemVT);
 
@@ -3823,18 +3850,23 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
   unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
   // For now, atomics are considered to be volatile always.
+  // FIXME: Volatile isn't really correct; we should keep track of atomic
+  // orderings in the memoperand.
   Flags |= MachineMemOperand::MOVolatile;
 
   MachineMemOperand *MMO =
     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
 
-  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO);
+  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
+                   Ordering, SynchScope);
 }
 
 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Chain,
                                 SDValue Ptr, SDValue Cmp,
-                                SDValue Swp, MachineMemOperand *MMO) {
+                                SDValue Swp, MachineMemOperand *MMO,
+                                AtomicOrdering Ordering,
+                                SynchronizationScope SynchScope) {
   assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
 
@@ -3851,7 +3883,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
     return SDValue(E, 0);
   }
   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
-                                               Ptr, Cmp, Swp, MMO);
+                                               Ptr, Cmp, Swp, MMO, Ordering,
+                                               SynchScope);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDValue(N, 0);
@@ -3861,27 +3894,39 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Chain,
                                 SDValue Ptr, SDValue Val,
                                 const Value* PtrVal,
-                                unsigned Alignment) {
+                                unsigned Alignment,
+                                AtomicOrdering Ordering,
+                                SynchronizationScope SynchScope) {
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(MemVT);
 
   MachineFunction &MF = getMachineFunction();
-  unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+  // A monotonic store does not load; a release store "loads" in the sense
+  // that other stores cannot be sunk past it.
+  // (An atomicrmw obviously both loads and stores.)
+  unsigned Flags = MachineMemOperand::MOStore;
+  if (Opcode != ISD::ATOMIC_STORE || Ordering > Monotonic)
+    Flags |= MachineMemOperand::MOLoad;
 
   // For now, atomics are considered to be volatile always.
+  // FIXME: Volatile isn't really correct; we should keep track of atomic
+  // orderings in the memoperand.
   Flags |= MachineMemOperand::MOVolatile;
 
   MachineMemOperand *MMO =
     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
                             MemVT.getStoreSize(), Alignment);
 
-  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
+  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
+                   Ordering, SynchScope);
 }
 
 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Chain,
                                 SDValue Ptr, SDValue Val,
-                                MachineMemOperand *MMO) {
+                                MachineMemOperand *MMO,
+                                AtomicOrdering Ordering,
+                                SynchronizationScope SynchScope) {
   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
           Opcode == ISD::ATOMIC_LOAD_SUB ||
           Opcode == ISD::ATOMIC_LOAD_AND ||
@@ -3892,12 +3937,14 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
           Opcode == ISD::ATOMIC_LOAD_MAX ||
           Opcode == ISD::ATOMIC_LOAD_UMIN ||
           Opcode == ISD::ATOMIC_LOAD_UMAX ||
-          Opcode == ISD::ATOMIC_SWAP) &&
+          Opcode == ISD::ATOMIC_SWAP ||
+          Opcode == ISD::ATOMIC_STORE) &&
          "Invalid Atomic Op");
 
   EVT VT = Val.getValueType();
 
-  SDVTList VTs = getVTList(VT, MVT::Other);
+  SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
+                                               getVTList(VT, MVT::Other);
   FoldingSetNodeID ID;
   ID.AddInteger(MemVT.getRawBits());
   SDValue Ops[] = {Chain, Ptr, Val};
@@ -3908,7 +3955,63 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
     return SDValue(E, 0);
   }
   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
-                                               Ptr, Val, MMO);
+                                               Ptr, Val, MMO,
+                                               Ordering, SynchScope);
+  CSEMap.InsertNode(N, IP);
+  AllNodes.push_back(N);
+  return SDValue(N, 0);
+}
+
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
+                                EVT VT, SDValue Chain,
+                                SDValue Ptr,
+                                const Value* PtrVal,
+                                unsigned Alignment,
+                                AtomicOrdering Ordering,
+                                SynchronizationScope SynchScope) {
+  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
+    Alignment = getEVTAlignment(MemVT);
+
+  MachineFunction &MF = getMachineFunction();
+  // A monotonic load does not store; an acquire load "stores" in the sense
+  // that other loads cannot be hoisted past it.
+  unsigned Flags = MachineMemOperand::MOLoad;
+  if (Ordering > Monotonic)
+    Flags |= MachineMemOperand::MOStore;
+
+  // For now, atomics are considered to be volatile always.
+  // FIXME: Volatile isn't really correct; we should keep track of atomic
+  // orderings in the memoperand.
+  Flags |= MachineMemOperand::MOVolatile;
+
+  MachineMemOperand *MMO =
+    MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
+                            MemVT.getStoreSize(), Alignment);
+
+  return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,
+                   Ordering, SynchScope);
+}
+
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
+                                EVT VT, SDValue Chain,
+                                SDValue Ptr,
+                                MachineMemOperand *MMO,
+                                AtomicOrdering Ordering,
+                                SynchronizationScope SynchScope) {
+  assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
+
+  SDVTList VTs = getVTList(VT, MVT::Other);
+  FoldingSetNodeID ID;
+  ID.AddInteger(MemVT.getRawBits());
+  SDValue Ops[] = {Chain, Ptr};
+  AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
+  void* IP = 0;
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+    cast<AtomicSDNode>(E)->refineAlignment(MMO);
+    return SDValue(E, 0);
+  }
+  SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
+                                               Ptr, MMO, Ordering, SynchScope);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDValue(N, 0);
@@ -5769,6 +5872,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
 #endif
   case ISD::PREFETCH:      return "Prefetch";
   case ISD::MEMBARRIER:    return "MemBarrier";
+  case ISD::ATOMIC_FENCE:    return "AtomicFence";
   case ISD::ATOMIC_CMP_SWAP:    return "AtomicCmpSwap";
   case ISD::ATOMIC_SWAP:        return "AtomicSwap";
   case ISD::ATOMIC_LOAD_ADD:    return "AtomicLoadAdd";
@@ -5781,6 +5885,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::ATOMIC_LOAD_MAX:    return "AtomicLoadMax";
   case ISD::ATOMIC_LOAD_UMIN:   return "AtomicLoadUMin";
   case ISD::ATOMIC_LOAD_UMAX:   return "AtomicLoadUMax";
+  case ISD::ATOMIC_LOAD:        return "AtomicLoad";
+  case ISD::ATOMIC_STORE:       return "AtomicStore";
   case ISD::PCMARKER:      return "PCMarker";
   case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
   case ISD::SRCVALUE:      return "SrcValue";
@@ -5896,8 +6002,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
 
   case ISD::FPOWI:  return "fpowi";
   case ISD::SETCC:       return "setcc";
-  case ISD::VSETCC:      return "vsetcc";
   case ISD::SELECT:      return "select";
+  case ISD::VSELECT:     return "vselect";
   case ISD::SELECT_CC:   return "select_cc";
   case ISD::INSERT_VECTOR_ELT:   return "insert_vector_elt";
   case ISD::EXTRACT_VECTOR_ELT:  return "extract_vector_elt";
@@ -5985,7 +6091,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::CTLZ:    return "ctlz";
 
   // Trampolines
-  case ISD::TRAMPOLINE: return "trampoline";
+  case ISD::INIT_TRAMPOLINE: return "init_trampoline";
+  case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline";
 
   case ISD::CONDCODE:
     switch (cast<CondCodeSDNode>(this)->get()) {
@@ -6245,8 +6352,7 @@ void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const {
 
 static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N,
                                   const SelectionDAG *G, unsigned depth,
-                                  unsigned indent)
-{
+                                  unsigned indent) {
   if (depth == 0)
     return;
 
@@ -6340,6 +6446,10 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
                                 &Operands[0], Operands.size()));
       break;
+    case ISD::VSELECT:
+      Scalars.push_back(getNode(ISD::SELECT, dl, EltVT,
+                                &Operands[0], Operands.size()));
+      break;
     case ISD::SHL:
     case ISD::SRA:
     case ISD::SRL:
@@ -6427,6 +6537,8 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
           Align = TD->getPreferredAlignment(GVar);
         }
       }
+      if (!Align)
+        Align = TLI.getTargetData()->getABITypeAlignment(GV->getType());
     }
     return MinAlign(Align, GVOffset);
   }
@@ -6528,7 +6640,7 @@ unsigned GlobalAddressSDNode::getAddressSpace() const {
 }
 
 
-const Type *ConstantPoolSDNode::getType() const {
+Type *ConstantPoolSDNode::getType() const {
   if (isMachineConstantPoolEntry())
     return Val.MachineCPVal->getType();
   return Val.ConstVal->getType();
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 81b03ee76a5c..7ed46a6952d1 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -578,7 +578,7 @@ namespace {
       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
 
     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
-                 unsigned Reg, const Type *Ty) {
+                 unsigned Reg, Type *Ty) {
       ComputeValueVTs(tli, Ty, ValueVTs);
 
       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
@@ -788,6 +788,18 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
   if (HasMatching)
     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
+  else if (!Regs.empty() &&
+           TargetRegisterInfo::isVirtualRegister(Regs.front())) {
+    // Put the register class of the virtual registers in the flag word.  That
+    // way, later passes can recompute register class constraints for inline
+    // assembly as well as normal instructions.
+    // Don't do this for tied operands that can use the regclass information
+    // from the def.
+    const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+    const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
+    Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
+  }
+
   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
   Ops.push_back(Res);
 
@@ -805,6 +817,7 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa) {
   AA = &aa;
   GFI = gfi;
   TD = DAG.getTarget().getTargetData();
+  LPadToCallSiteMap.clear();
 }
 
 /// clear - Clear out the current SelectionDAG and the associated
@@ -956,7 +969,7 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
   }
 }
 
-// getValue - Return an SDValue for the given Value.
+/// getValue - Return an SDValue for the given Value.
 SDValue SelectionDAGBuilder::getValue(const Value *V) {
   // If we already have an SDValue for this value, use it. It's important
   // to do this first, so that we don't create a CopyFromReg if we already
@@ -971,7 +984,7 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
     unsigned InReg = It->second;
     RegsForValue RFV(*DAG.getContext(), TLI, InReg, V->getType());
     SDValue Chain = DAG.getEntryNode();
-    N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain,NULL);
+    N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain, NULL);
     resolveDanglingDebugInfo(V, N);
     return N;
   }
@@ -1069,7 +1082,7 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
       return DAG.getBlockAddress(BA, VT);
 
-    const VectorType *VecTy = cast<VectorType>(V->getType());
+    VectorType *VecTy = cast<VectorType>(V->getType());
     unsigned NumElements = VecTy->getNumElements();
 
     // Now that we know the number and type of the elements, get that number of
@@ -1277,15 +1290,17 @@ uint32_t SelectionDAGBuilder::getEdgeWeight(MachineBasicBlock *Src,
   BranchProbabilityInfo *BPI = FuncInfo.BPI;
   if (!BPI)
     return 0;
-  BasicBlock *SrcBB = const_cast<BasicBlock*>(Src->getBasicBlock());
-  BasicBlock *DstBB = const_cast<BasicBlock*>(Dst->getBasicBlock());
+  const BasicBlock *SrcBB = Src->getBasicBlock();
+  const BasicBlock *DstBB = Dst->getBasicBlock();
   return BPI->getEdgeWeight(SrcBB, DstBB);
 }
 
-void SelectionDAGBuilder::addSuccessorWithWeight(MachineBasicBlock *Src,
-                                                 MachineBasicBlock *Dst) {
-  uint32_t weight = getEdgeWeight(Src, Dst);
-  Src->addSuccessor(Dst, weight);
+void SelectionDAGBuilder::
+addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
+                       uint32_t Weight /* = 0 */) {
+  if (!Weight)
+    Weight = getEdgeWeight(Src, Dst);
+  Src->addSuccessor(Dst, Weight);
 }
 
 
@@ -1558,8 +1573,8 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
   }
 
   // Update successor info
-  addSuccessorWithWeight(SwitchBB, CB.TrueBB);
-  addSuccessorWithWeight(SwitchBB, CB.FalseBB);
+  addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
+  addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
 
   // Set NextBlock to be the MBB immediately after the current one, if any.
   // This is used to avoid emitting unnecessary branches to the next block.
@@ -1677,7 +1692,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
     UsePtrType = true;
   else {
     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
-      if ((uint64_t)((int64_t)B.Cases[i].Mask >> VT.getSizeInBits()) + 1 >= 2) {
+      if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
         // Switch table case range are encoded into series of masks.
         // Just use pointer type, it's guaranteed to fit.
         UsePtrType = true;
@@ -1808,6 +1823,49 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
 void SelectionDAGBuilder::visitUnwind(const UnwindInst &I) {
 }
 
+void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
+  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
+}
+
+void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
+  assert(FuncInfo.MBB->isLandingPad() &&
+         "Call to landingpad not in landing pad!");
+
+  MachineBasicBlock *MBB = FuncInfo.MBB;
+  MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
+  AddLandingPadInfo(LP, MMI, MBB);
+
+  SmallVector<EVT, 2> ValueVTs;
+  ComputeValueVTs(TLI, LP.getType(), ValueVTs);
+
+  // Insert the EXCEPTIONADDR instruction.
+  assert(FuncInfo.MBB->isLandingPad() &&
+         "Call to eh.exception not in landing pad!");
+  SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other);
+  SDValue Ops[2];
+  Ops[0] = DAG.getRoot();
+  SDValue Op1 = DAG.getNode(ISD::EXCEPTIONADDR, getCurDebugLoc(), VTs, Ops, 1);
+  SDValue Chain = Op1.getValue(1);
+
+  // Insert the EHSELECTION instruction.
+  VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other);
+  Ops[0] = Op1;
+  Ops[1] = Chain;
+  SDValue Op2 = DAG.getNode(ISD::EHSELECTION, getCurDebugLoc(), VTs, Ops, 2);
+  Chain = Op2.getValue(1);
+  Op2 = DAG.getSExtOrTrunc(Op2, getCurDebugLoc(), MVT::i32);
+
+  Ops[0] = Op1;
+  Ops[1] = Op2;
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
+                            DAG.getVTList(&ValueVTs[0], ValueVTs.size()),
+                            &Ops[0], 2);
+
+  std::pair<SDValue, SDValue> RetPair = std::make_pair(Res, Chain);
+  setValue(&LP, RetPair.first);
+  DAG.setRoot(RetPair.second);
+}
+
 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
 /// small case ranges).
 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
@@ -1866,8 +1924,8 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
                                     ISD::SETEQ);
 
         // Update successor info.
-        SwitchBB->addSuccessor(Small.BB);
-        SwitchBB->addSuccessor(Default);
+        addSuccessorWithWeight(SwitchBB, Small.BB);
+        addSuccessorWithWeight(SwitchBB, Default);
 
         // Insert the true branch.
         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
@@ -1923,7 +1981,11 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
       CC = ISD::SETLE;
       LHS = I->Low; MHS = SV; RHS = I->High;
     }
-    CaseBlock CB(CC, LHS, RHS, MHS, I->BB, FallThrough, CurBlock);
+
+    uint32_t ExtraWeight = I->ExtraWeight;
+    CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
+                 /* me */ CurBlock,
+                 /* trueweight */ ExtraWeight / 2, /* falseweight */ ExtraWeight / 2);
 
     // If emitting the first comparison, just call visitSwitchCase to emit the
     // code into the current block.  Otherwise, push the CaseBlock onto the
@@ -1953,10 +2015,10 @@ static APInt ComputeRange(const APInt &First, const APInt &Last) {
 }
 
 /// handleJTSwitchCase - Emit jumptable for current switch case range
-bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR,
-                                             CaseRecVector& WorkList,
-                                             const Value* SV,
-                                             MachineBasicBlock* Default,
+bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
+                                             CaseRecVector &WorkList,
+                                             const Value *SV,
+                                             MachineBasicBlock *Default,
                                              MachineBasicBlock *SwitchBB) {
   Case& FrontCase = *CR.Range.first;
   Case& BackCase  = *(CR.Range.second-1);
@@ -1965,8 +2027,7 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR,
   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
 
   APInt TSize(First.getBitWidth(), 0);
-  for (CaseItr I = CR.Range.first, E = CR.Range.second;
-       I!=E; ++I)
+  for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
     TSize += I->size();
 
   if (!areJTsAllowed(TLI) || TSize.ult(4))
@@ -2044,7 +2105,6 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR,
     visitJumpTableHeader(JT, JTH, SwitchBB);
 
   JTCases.push_back(JumpTableBlock(JTH, JT));
-
   return true;
 }
 
@@ -2318,12 +2378,17 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
                                        const SwitchInst& SI) {
   size_t numCmps = 0;
 
+  BranchProbabilityInfo *BPI = FuncInfo.BPI;
   // Start with "simple" cases
   for (size_t i = 1; i < SI.getNumSuccessors(); ++i) {
-    MachineBasicBlock *SMBB = FuncInfo.MBBMap[SI.getSuccessor(i)];
+    BasicBlock *SuccBB = SI.getSuccessor(i);
+    MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
+
+    uint32_t ExtraWeight = BPI ? BPI->getEdgeWeight(SI.getParent(), SuccBB) : 0;
+
     Cases.push_back(Case(SI.getSuccessorValue(i),
                          SI.getSuccessorValue(i),
-                         SMBB));
+                         SMBB, ExtraWeight));
   }
   std::sort(Cases.begin(), Cases.end(), CaseCmp());
 
@@ -2343,6 +2408,16 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
         I->High = J->High;
         J = Cases.erase(J);
+
+        if (BranchProbabilityInfo *BPI = FuncInfo.BPI) {
+          uint32_t CurWeight = currentBB->getBasicBlock() ?
+            BPI->getEdgeWeight(SI.getParent(), currentBB->getBasicBlock()) : 16;
+          uint32_t NextWeight = nextBB->getBasicBlock() ?
+            BPI->getEdgeWeight(SI.getParent(), nextBB->getBasicBlock()) : 16;
+
+          BPI->setEdgeWeight(SI.getParent(), currentBB->getBasicBlock(),
+                             CurWeight + NextWeight);
+        }
       } else {
         I = J++;
       }
@@ -2379,7 +2454,7 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
 
   // If there is only the default destination, branch to it if it is not the
   // next basic block.  Otherwise, just fall through.
-  if (SI.getNumOperands() == 2) {
+  if (SI.getNumCases() == 1) {
     // Update machine-CFG edges.
 
     // If this is not a fall-through branch, emit the branch.
@@ -2399,12 +2474,12 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
   size_t numCmps = Clusterify(Cases, SI);
   DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
                << ". Total compares: " << numCmps << '\n');
-  numCmps = 0;
+  (void)numCmps;
 
   // Get the Value to be switched on and default basic blocks, which will be
   // inserted into CaseBlock records, representing basic blocks in the binary
   // search tree.
-  const Value *SV = SI.getOperand(0);
+  const Value *SV = SI.getCondition();
 
   // Push the initial CaseRec onto the worklist
   CaseRecVector WorkList;
@@ -2458,7 +2533,7 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
 
 void SelectionDAGBuilder::visitFSub(const User &I) {
   // -0.0 - X --> fneg
-  const Type *Ty = I.getType();
+  Type *Ty = I.getType();
   if (isa<Constant>(I.getOperand(0)) &&
       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
     SDValue Op2 = getValue(I.getOperand(1));
@@ -2562,10 +2637,12 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
   SDValue Cond     = getValue(I.getOperand(0));
   SDValue TrueVal  = getValue(I.getOperand(1));
   SDValue FalseVal = getValue(I.getOperand(2));
+  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
+    ISD::VSELECT : ISD::SELECT;
 
   for (unsigned i = 0; i != NumValues; ++i)
-    Values[i] = DAG.getNode(ISD::SELECT, getCurDebugLoc(),
-                          TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
+    Values[i] = DAG.getNode(OpCode, getCurDebugLoc(),
+                            TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
                             Cond,
                             SDValue(TrueVal.getNode(),
                                     TrueVal.getResNo() + i),
@@ -2778,7 +2855,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
     // Analyze the access pattern of the vector to see if we can extract
     // two subvectors and do the shuffle. The analysis is done by calculating
     // the range of elements the mask access on both vectors.
-    int MinRange[2] = { SrcNumElts+1, SrcNumElts+1};
+    int MinRange[2] = { static_cast<int>(SrcNumElts+1),
+                        static_cast<int>(SrcNumElts+1)};
     int MaxRange[2] = {-1, -1};
 
     for (unsigned i = 0; i != MaskNumElts; ++i) {
@@ -2886,8 +2964,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
   const Value *Op0 = I.getOperand(0);
   const Value *Op1 = I.getOperand(1);
-  const Type *AggTy = I.getType();
-  const Type *ValTy = Op1->getType();
+  Type *AggTy = I.getType();
+  Type *ValTy = Op1->getType();
   bool IntoUndef = isa<UndefValue>(Op0);
   bool FromUndef = isa<UndefValue>(Op1);
 
@@ -2927,8 +3005,8 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
 
 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
   const Value *Op0 = I.getOperand(0);
-  const Type *AggTy = Op0->getType();
-  const Type *ValTy = I.getType();
+  Type *AggTy = Op0->getType();
+  Type *ValTy = I.getType();
   bool OutOfUndef = isa<UndefValue>(Op0);
 
   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
@@ -2961,12 +3039,12 @@ void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
 
 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
   SDValue N = getValue(I.getOperand(0));
-  const Type *Ty = I.getOperand(0)->getType();
+  Type *Ty = I.getOperand(0)->getType();
 
   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
        OI != E; ++OI) {
     const Value *Idx = *OI;
-    if (const StructType *StTy = dyn_cast<StructType>(Ty)) {
+    if (StructType *StTy = dyn_cast<StructType>(Ty)) {
       unsigned Field = cast<ConstantInt>(Idx)->getZExtValue();
       if (Field) {
         // N = N + Offset
@@ -3037,7 +3115,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
   if (FuncInfo.StaticAllocaMap.count(&I))
     return;   // getValue will auto-populate this.
 
-  const Type *Ty = I.getAllocatedType();
+  Type *Ty = I.getAllocatedType();
   uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty);
   unsigned Align =
     std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
@@ -3084,10 +3162,13 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
 }
 
 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
+  if (I.isAtomic())
+    return visitAtomicLoad(I);
+
   const Value *SV = I.getOperand(0);
   SDValue Ptr = getValue(SV);
 
-  const Type *Ty = I.getType();
+  Type *Ty = I.getType();
 
   bool isVolatile = I.isVolatile();
   bool isNonTemporal = I.getMetadata("nontemporal") != 0;
@@ -3161,6 +3242,9 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
 }
 
 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
+  if (I.isAtomic())
+    return visitAtomicStore(I);
+
   const Value *SrcV = I.getOperand(0);
   const Value *PtrV = I.getOperand(1);
 
@@ -3211,6 +3295,179 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
   DAG.setRoot(StoreNode);
 }
 
+static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
+                                    SynchronizationScope Scope,
+                                    bool Before, DebugLoc dl,
+                                    SelectionDAG &DAG,
+                                    const TargetLowering &TLI) {
+  // Fence, if necessary
+  if (Before) {
+    if (Order == AcquireRelease || Order == SequentiallyConsistent)
+      Order = Release;
+    else if (Order == Acquire || Order == Monotonic)
+      return Chain;
+  } else {
+    if (Order == AcquireRelease)
+      Order = Acquire;
+    else if (Order == Release || Order == Monotonic)
+      return Chain;
+  }
+  SDValue Ops[3];
+  Ops[0] = Chain;
+  Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
+  Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
+  return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3);
+}
+
+void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
+  DebugLoc dl = getCurDebugLoc();
+  AtomicOrdering Order = I.getOrdering();
+  SynchronizationScope Scope = I.getSynchScope();
+
+  SDValue InChain = getRoot();
+
+  if (TLI.getInsertFencesForAtomic())
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
+                                   DAG, TLI);
+
+  SDValue L =
+    DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
+                  getValue(I.getCompareOperand()).getValueType().getSimpleVT(),
+                  InChain,
+                  getValue(I.getPointerOperand()),
+                  getValue(I.getCompareOperand()),
+                  getValue(I.getNewValOperand()),
+                  MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Scope);
+
+  SDValue OutChain = L.getValue(1);
+
+  if (TLI.getInsertFencesForAtomic())
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
+                                    DAG, TLI);
+
+  setValue(&I, L);
+  DAG.setRoot(OutChain);
+}
+
+void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
+  DebugLoc dl = getCurDebugLoc();
+  ISD::NodeType NT;
+  switch (I.getOperation()) {
+  default: llvm_unreachable("Unknown atomicrmw operation"); return;
+  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
+  case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
+  case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
+  case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
+  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
+  case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
+  case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
+  case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
+  case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
+  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
+  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
+  }
+  AtomicOrdering Order = I.getOrdering();
+  SynchronizationScope Scope = I.getSynchScope();
+
+  SDValue InChain = getRoot();
+
+  if (TLI.getInsertFencesForAtomic())
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
+                                   DAG, TLI);
+
+  SDValue L =
+    DAG.getAtomic(NT, dl,
+                  getValue(I.getValOperand()).getValueType().getSimpleVT(),
+                  InChain,
+                  getValue(I.getPointerOperand()),
+                  getValue(I.getValOperand()),
+                  I.getPointerOperand(), 0 /* Alignment */,
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Scope);
+
+  SDValue OutChain = L.getValue(1);
+
+  if (TLI.getInsertFencesForAtomic())
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
+                                    DAG, TLI);
+
+  setValue(&I, L);
+  DAG.setRoot(OutChain);
+}
+
+void SelectionDAGBuilder::visitFence(const FenceInst &I) {
+  DebugLoc dl = getCurDebugLoc();
+  SDValue Ops[3];
+  Ops[0] = getRoot();
+  Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
+  Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
+  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3));
+}
+
+void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
+  DebugLoc dl = getCurDebugLoc();
+  AtomicOrdering Order = I.getOrdering();
+  SynchronizationScope Scope = I.getSynchScope();
+
+  SDValue InChain = getRoot();
+
+  EVT VT = EVT::getEVT(I.getType());
+
+  if (I.getAlignment() * 8 < VT.getSizeInBits())
+    report_fatal_error("Cannot generate unaligned atomic load");
+
+  SDValue L =
+    DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
+                  getValue(I.getPointerOperand()),
+                  I.getPointerOperand(), I.getAlignment(),
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Scope);
+
+  SDValue OutChain = L.getValue(1);
+
+  if (TLI.getInsertFencesForAtomic())
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
+                                    DAG, TLI);
+
+  setValue(&I, L);
+  DAG.setRoot(OutChain);
+}
+
+void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
+  DebugLoc dl = getCurDebugLoc();
+
+  AtomicOrdering Order = I.getOrdering();
+  SynchronizationScope Scope = I.getSynchScope();
+
+  SDValue InChain = getRoot();
+
+  EVT VT = EVT::getEVT(I.getValueOperand()->getType());
+
+  if (I.getAlignment() * 8 < VT.getSizeInBits())
+    report_fatal_error("Cannot generate unaligned atomic store");
+
+  if (TLI.getInsertFencesForAtomic())
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
+                                   DAG, TLI);
+
+  SDValue OutChain =
+    DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
+                  InChain,
+                  getValue(I.getPointerOperand()),
+                  getValue(I.getValueOperand()),
+                  I.getPointerOperand(), I.getAlignment(),
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Scope);
+
+  if (TLI.getInsertFencesForAtomic())
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
+                                    DAG, TLI);
+
+  DAG.setRoot(OutChain);
+}
+
 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
 /// node.
 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
@@ -3290,7 +3547,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
   }
 
   if (!I.getType()->isVoidTy()) {
-    if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
+    if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
       EVT VT = TLI.getValueType(PTy);
       Result = DAG.getNode(ISD::BITCAST, getCurDebugLoc(), VT, Result);
     }
@@ -3337,25 +3594,6 @@ getF32Constant(SelectionDAG &DAG, unsigned Flt) {
   return DAG.getConstantFP(APFloat(APInt(32, Flt)), MVT::f32);
 }
 
-/// Inlined utility function to implement binary input atomic intrinsics for
-/// visitIntrinsicCall: I is a call instruction
-///                     Op is the associated NodeType for I
-const char *
-SelectionDAGBuilder::implVisitBinaryAtomic(const CallInst& I,
-                                           ISD::NodeType Op) {
-  SDValue Root = getRoot();
-  SDValue L =
-    DAG.getAtomic(Op, getCurDebugLoc(),
-                  getValue(I.getArgOperand(1)).getValueType().getSimpleVT(),
-                  Root,
-                  getValue(I.getArgOperand(0)),
-                  getValue(I.getArgOperand(1)),
-                  I.getArgOperand(0));
-  setValue(&I, L);
-  DAG.setRoot(L.getValue(1));
-  return 0;
-}
-
 // implVisitAluOverflow - Lower arithmetic overflow instrinsics.
 const char *
 SelectionDAGBuilder::implVisitAluOverflow(const CallInst &I, ISD::NodeType Op) {
@@ -4154,17 +4392,12 @@ SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable,
     return false;
 
   unsigned Reg = 0;
-  if (Arg->hasByValAttr()) {
-    // Byval arguments' frame index is recorded during argument lowering.
-    // Use this info directly.
-    Reg = TRI->getFrameRegister(MF);
-    Offset = FuncInfo.getByValArgumentFrameIndex(Arg);
-    // If byval argument ofset is not recorded then ignore this.
-    if (!Offset)
-      Reg = 0;
-  }
+  // Some arguments' frame index is recorded during argument lowering.
+  Offset = FuncInfo.getArgumentFrameIndex(Arg);
+  if (Offset)
+      Reg = TRI->getFrameRegister(MF);
 
-  if (N.getNode()) {
+  if (!Reg && N.getNode()) {
     if (N.getOpcode() == ISD::CopyFromReg)
       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
     else
@@ -4295,7 +4528,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
     MDNode *Variable = DI.getVariable();
     const Value *Address = DI.getAddress();
-    if (!Address || !DIVariable(DI.getVariable()).Verify())
+    if (!Address || !DIVariable(Variable).Verify())
       return 0;
 
     // Build an entry in DbgOrdering.  Debug info input nodes get an SDNodeOrder
@@ -4385,7 +4618,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     // debug info exists.
     ++SDNodeOrder;
     SDDbgValue *SDV;
-    if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) {
+    if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
       SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder);
       DAG.AddDbgValue(SDV, 0, false);
     } else {
@@ -4514,9 +4747,24 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     MMI.setCurrentCallSite(CI->getZExtValue());
     return 0;
   }
+  case Intrinsic::eh_sjlj_functioncontext: {
+    // Get and store the index of the function context.
+    MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+    AllocaInst *FnCtx =
+      cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
+    int FI = FuncInfo.StaticAllocaMap[FnCtx];
+    MFI->setFunctionContextIndex(FI);
+    return 0;
+  }
   case Intrinsic::eh_sjlj_setjmp: {
-    setValue(&I, DAG.getNode(ISD::EH_SJLJ_SETJMP, dl, MVT::i32, getRoot(),
-                             getValue(I.getArgOperand(0))));
+    SDValue Ops[2];
+    Ops[0] = getRoot();
+    Ops[1] = getValue(I.getArgOperand(0));
+    SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, dl,
+                             DAG.getVTList(MVT::i32, MVT::Other),
+                             Ops, 2);
+    setValue(&I, Op.getValue(0));
+    DAG.setRoot(Op.getValue(1));
     return 0;
   }
   case Intrinsic::eh_sjlj_longjmp: {
@@ -4778,12 +5026,15 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
     Ops[5] = DAG.getSrcValue(F);
 
-    Res = DAG.getNode(ISD::TRAMPOLINE, dl,
-                      DAG.getVTList(TLI.getPointerTy(), MVT::Other),
-                      Ops, 6);
+    Res = DAG.getNode(ISD::INIT_TRAMPOLINE, dl, MVT::Other, Ops, 6);
 
-    setValue(&I, Res);
-    DAG.setRoot(Res.getValue(1));
+    DAG.setRoot(Res);
+    return 0;
+  }
+  case Intrinsic::adjust_trampoline: {
+    setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, dl,
+                             TLI.getPointerTy(),
+                             getValue(I.getArgOperand(0))));
     return 0;
   }
   case Intrinsic::gcroot:
@@ -4857,51 +5108,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
                                         rw==1)); /* write */
     return 0;
   }
-  case Intrinsic::memory_barrier: {
-    SDValue Ops[6];
-    Ops[0] = getRoot();
-    for (int x = 1; x < 6; ++x)
-      Ops[x] = getValue(I.getArgOperand(x - 1));
-
-    DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6));
-    return 0;
-  }
-  case Intrinsic::atomic_cmp_swap: {
-    SDValue Root = getRoot();
-    SDValue L =
-      DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, getCurDebugLoc(),
-                    getValue(I.getArgOperand(1)).getValueType().getSimpleVT(),
-                    Root,
-                    getValue(I.getArgOperand(0)),
-                    getValue(I.getArgOperand(1)),
-                    getValue(I.getArgOperand(2)),
-                    MachinePointerInfo(I.getArgOperand(0)));
-    setValue(&I, L);
-    DAG.setRoot(L.getValue(1));
-    return 0;
-  }
-  case Intrinsic::atomic_load_add:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_ADD);
-  case Intrinsic::atomic_load_sub:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_SUB);
-  case Intrinsic::atomic_load_or:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_OR);
-  case Intrinsic::atomic_load_xor:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_XOR);
-  case Intrinsic::atomic_load_and:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_AND);
-  case Intrinsic::atomic_load_nand:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_NAND);
-  case Intrinsic::atomic_load_max:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MAX);
-  case Intrinsic::atomic_load_min:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MIN);
-  case Intrinsic::atomic_load_umin:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMIN);
-  case Intrinsic::atomic_load_umax:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX);
-  case Intrinsic::atomic_swap:
-    return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP);
 
   case Intrinsic::invariant_start:
   case Intrinsic::lifetime_start:
@@ -4918,9 +5124,9 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
                                       bool isTailCall,
                                       MachineBasicBlock *LandingPad) {
-  const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
-  const FunctionType *FTy = cast<FunctionType>(PT->getElementType());
-  const Type *RetTy = FTy->getReturnType();
+  PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
+  FunctionType *FTy = cast<FunctionType>(PT->getElementType());
+  Type *RetTy = FTy->getReturnType();
   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
   MCSymbol *BeginLabel = 0;
 
@@ -4949,7 +5155,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
                       FTy->getReturnType());
     MachineFunction &MF = DAG.getMachineFunction();
     DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
-    const Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType());
+    Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType());
 
     DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI.getPointerTy());
     Entry.Node = DemoteStackSlot;
@@ -4997,6 +5203,8 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
     unsigned CallSiteIndex = MMI.getCurrentCallSite();
     if (CallSiteIndex) {
       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
+      LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
+
       // Now that the call site is handled, stop tracking it.
       MMI.setCurrentCallSite(0);
     }
@@ -5037,7 +5245,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
     // The instruction result is the result of loading from the
     // hidden sret parameter.
     SmallVector<EVT, 1> PVTs;
-    const Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType());
+    Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType());
 
     ComputeValueVTs(TLI, PtrRetTy, PVTs);
     assert(PVTs.size() == 1 && "Pointers should fit in one register");
@@ -5130,7 +5338,7 @@ static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
 }
 
 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
-                             const Type *LoadTy,
+                             Type *LoadTy,
                              SelectionDAGBuilder &Builder) {
 
   // Check to see if this load can be trivially constant folded, e.g. if the
@@ -5193,7 +5401,7 @@ bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
   if (Size && IsOnlyUsedInZeroEqualityComparison(&I)) {
     bool ActuallyDoIt = true;
     MVT LoadVT;
-    const Type *LoadTy;
+    Type *LoadTy;
     switch (Size->getZExtValue()) {
     default:
       LoadVT = MVT::Other;
@@ -5261,14 +5469,14 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
 
   // See if any floating point values are being passed to this function. This is
   // used to emit an undefined reference to fltused on Windows.
-  const FunctionType *FT =
+  FunctionType *FT =
     cast<FunctionType>(I.getCalledValue()->getType()->getContainedType(0));
   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
   if (FT->isVarArg() &&
       !MMI.callsExternalVAFunctionWithFloatingPointArguments()) {
     for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
-      const Type* T = I.getArgOperand(i)->getType();
-      for (po_iterator<const Type*> i = po_begin(T), e = po_end(T);
+      Type* T = I.getArgOperand(i)->getType();
+      for (po_iterator<Type*> i = po_begin(T), e = po_end(T);
            i != e; ++i) {
         if (!i->isFloatingPointTy()) continue;
         MMI.setCallsExternalVAFunctionWithFloatingPointArguments(true);
@@ -5412,20 +5620,20 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
     if (isa<BasicBlock>(CallOperandVal))
       return TLI.getPointerTy();
 
-    const llvm::Type *OpTy = CallOperandVal->getType();
+    llvm::Type *OpTy = CallOperandVal->getType();
 
     // FIXME: code duplicated from TargetLowering::ParseConstraints().
     // If this is an indirect operand, the operand is a pointer to the
     // accessed type.
     if (isIndirect) {
-      const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
+      llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
       if (!PtrTy)
         report_fatal_error("Indirect operand for inline asm not a pointer!");
       OpTy = PtrTy->getElementType();
     }
 
     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
-    if (const StructType *STy = dyn_cast<StructType>(OpTy))
+    if (StructType *STy = dyn_cast<StructType>(OpTy))
       if (STy->getNumElements() == 1)
         OpTy = STy->getElementType(0);
 
@@ -5637,9 +5845,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
 
       // The return value of the call is this value.  As such, there is no
       // corresponding argument.
-      assert(!CS.getType()->isVoidTy() &&
-             "Bad inline asm!");
-      if (const StructType *STy = dyn_cast<StructType>(CS.getType())) {
+      assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
+      if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
         OpVT = TLI.getValueType(STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
@@ -5707,9 +5914,11 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
 
       if (OpInfo.ConstraintVT != Input.ConstraintVT) {
 	std::pair<unsigned, const TargetRegisterClass*> MatchRC =
-	  TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, OpInfo.ConstraintVT);
+	  TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode,
+                                           OpInfo.ConstraintVT);
 	std::pair<unsigned, const TargetRegisterClass*> InputRC =
-	  TLI.getRegForInlineAsmConstraint(Input.ConstraintCode, Input.ConstraintVT);
+	  TLI.getRegForInlineAsmConstraint(Input.ConstraintCode,
+                                           Input.ConstraintVT);
         if ((OpInfo.ConstraintVT.isInteger() !=
              Input.ConstraintVT.isInteger()) ||
             (MatchRC.second != InputRC.second)) {
@@ -5750,7 +5959,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
       } else {
         // Otherwise, create a stack slot and emit a store to it before the
         // asm.
-        const Type *Ty = OpVal->getType();
+        Type *Ty = OpVal->getType();
         uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty);
         unsigned Align  = TLI.getTargetData()->getPrefTypeAlignment(Ty);
         MachineFunction &MF = DAG.getMachineFunction();
@@ -6111,7 +6320,7 @@ void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
 /// FIXME: When all targets are
 /// migrated to using LowerCall, this hook should be integrated into SDISel.
 std::pair<SDValue, SDValue>
-TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
+TargetLowering::LowerCallTo(SDValue Chain, Type *RetTy,
                             bool RetSExt, bool RetZExt, bool isVarArg,
                             bool isInreg, unsigned NumFixedArgs,
                             CallingConv::ID CallConv, bool isTailCall,
@@ -6128,7 +6337,7 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
     for (unsigned Value = 0, NumValues = ValueVTs.size();
          Value != NumValues; ++Value) {
       EVT VT = ValueVTs[Value];
-      const Type *ArgTy = VT.getTypeForEVT(RetTy->getContext());
+      Type *ArgTy = VT.getTypeForEVT(RetTy->getContext());
       SDValue Op = SDValue(Args[i].Node.getNode(),
                            Args[i].Node.getResNo() + Value);
       ISD::ArgFlagsTy Flags;
@@ -6145,8 +6354,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
         Flags.setSRet();
       if (Args[i].isByVal) {
         Flags.setByVal();
-        const PointerType *Ty = cast<PointerType>(Args[i].Ty);
-        const Type *ElementTy = Ty->getElementType();
+        PointerType *Ty = cast<PointerType>(Args[i].Ty);
+        Type *ElementTy = Ty->getElementType();
         Flags.setByValSize(getTargetData()->getTypeAllocSize(ElementTy));
         // For ByVal, alignment should come from FE.  BE will guess if this
         // info is not there but there are cases it cannot get right.
@@ -6356,7 +6565,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
     for (unsigned Value = 0, NumValues = ValueVTs.size();
          Value != NumValues; ++Value) {
       EVT VT = ValueVTs[Value];
-      const Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
+      Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
       ISD::ArgFlagsTy Flags;
       unsigned OriginalAlignment =
         TD->getABITypeAlignment(ArgTy);
@@ -6371,8 +6580,8 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
         Flags.setSRet();
       if (F.paramHasAttr(Idx, Attribute::ByVal)) {
         Flags.setByVal();
-        const PointerType *Ty = cast<PointerType>(I->getType());
-        const Type *ElementTy = Ty->getElementType();
+        PointerType *Ty = cast<PointerType>(I->getType());
+        Type *ElementTy = Ty->getElementType();
         Flags.setByValSize(TD->getTypeAllocSize(ElementTy));
         // For ByVal, alignment should be passed from FE.  BE will guess if
         // this info is not there but there are cases it cannot get right.
@@ -6487,15 +6696,22 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
     if (ArgValues.empty())
       continue;
 
-    // Note down frame index for byval arguments.
-    if (I->hasByValAttr())
-      if (FrameIndexSDNode *FI =
-          dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
-        FuncInfo->setByValArgumentFrameIndex(I, FI->getIndex());
+    // Note down frame index.
+    if (FrameIndexSDNode *FI =
+	dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
+      FuncInfo->setArgumentFrameIndex(I, FI->getIndex());
 
     SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues,
                                      SDB->getCurDebugLoc());
+
     SDB->setValue(I, Res);
+    if (!EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
+      if (LoadSDNode *LNode = 
+          dyn_cast<LoadSDNode>(Res.getOperand(0).getNode()))
+        if (FrameIndexSDNode *FI =
+            dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
+        FuncInfo->setArgumentFrameIndex(I, FI->getIndex());
+    }
 
     // If this argument is live outside of the entry block, insert a copy from
     // wherever we got it to the vreg that other BB's will reference it as.
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index a0884ebf5d56..0a21ca3472ca 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -132,10 +132,13 @@ class SelectionDAGBuilder {
     Constant* Low;
     Constant* High;
     MachineBasicBlock* BB;
+    uint32_t ExtraWeight;
+
+    Case() : Low(0), High(0), BB(0), ExtraWeight(0) { }
+    Case(Constant* low, Constant* high, MachineBasicBlock* bb,
+         uint32_t extraweight) : Low(low), High(high), BB(bb),
+         ExtraWeight(extraweight) { }
 
-    Case() : Low(0), High(0), BB(0) { }
-    Case(Constant* low, Constant* high, MachineBasicBlock* bb) :
-      Low(low), High(high), BB(bb) { }
     APInt size() const {
       const APInt &rHigh = cast<ConstantInt>(High)->getValue();
       const APInt &rLow  = cast<ConstantInt>(Low)->getValue();
@@ -203,20 +206,30 @@ class SelectionDAGBuilder {
     CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
               const Value *cmpmiddle,
               MachineBasicBlock *truebb, MachineBasicBlock *falsebb,
-              MachineBasicBlock *me)
+              MachineBasicBlock *me,
+              uint32_t trueweight = 0, uint32_t falseweight = 0)
       : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs),
-        TrueBB(truebb), FalseBB(falsebb), ThisBB(me) {}
+        TrueBB(truebb), FalseBB(falsebb), ThisBB(me),
+        TrueWeight(trueweight), FalseWeight(falseweight) { }
+
     // CC - the condition code to use for the case block's setcc node
     ISD::CondCode CC;
+
     // CmpLHS/CmpRHS/CmpMHS - The LHS/MHS/RHS of the comparison to emit.
     // Emit by default LHS op RHS. MHS is used for range comparisons:
     // If MHS is not null: (LHS <= MHS) and (MHS <= RHS).
     const Value *CmpLHS, *CmpMHS, *CmpRHS;
+
     // TrueBB/FalseBB - the block to branch to if the setcc is true/false.
     MachineBasicBlock *TrueBB, *FalseBB;
+
     // ThisBB - the block into which to emit the code for the setcc and branches
     MachineBasicBlock *ThisBB;
+
+    // TrueWeight/FalseWeight - branch weights.
+    uint32_t TrueWeight, FalseWeight;
   };
+
   struct JumpTable {
     JumpTable(unsigned R, unsigned J, MachineBasicBlock *M,
               MachineBasicBlock *D): Reg(R), JTI(J), MBB(M), Default(D) {}
@@ -307,6 +320,9 @@ class SelectionDAGBuilder {
   /// GFI - Garbage collection metadata for the function.
   GCFunctionInfo *GFI;
 
+  /// LPadToCallSiteMap - Map a landing pad to the call site indexes.
+  DenseMap<MachineBasicBlock*, SmallVector<unsigned, 4> > LPadToCallSiteMap;
+
   /// HasTailCall - This is set to true if a call in the current
   /// block has been translated as a tail call. In this case,
   /// no subsequent DAG nodes should be created.
@@ -436,7 +452,8 @@ class SelectionDAGBuilder {
                                 MachineBasicBlock *SwitchBB);
 
   uint32_t getEdgeWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst);
-  void addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst);
+  void addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
+                              uint32_t Weight = 0);
 public:
   void visitSwitchCase(CaseBlock &CB,
                        MachineBasicBlock *SwitchBB);
@@ -453,6 +470,7 @@ class SelectionDAGBuilder {
 private:
   // These all get lowered before this pass.
   void visitInvoke(const InvokeInst &I);
+  void visitResume(const ResumeInst &I);
   void visitUnwind(const UnwindInst &I);
 
   void visitBinary(const User &I, unsigned OpCode);
@@ -497,6 +515,7 @@ class SelectionDAGBuilder {
 
   void visitExtractValue(const ExtractValueInst &I);
   void visitInsertValue(const InsertValueInst &I);
+  void visitLandingPad(const LandingPadInst &I);
 
   void visitGetElementPtr(const User &I);
   void visitSelect(const User &I);
@@ -504,10 +523,15 @@ class SelectionDAGBuilder {
   void visitAlloca(const AllocaInst &I);
   void visitLoad(const LoadInst &I);
   void visitStore(const StoreInst &I);
+  void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
+  void visitAtomicRMW(const AtomicRMWInst &I);
+  void visitFence(const FenceInst &I);
   void visitPHI(const PHINode &I);
   void visitCall(const CallInst &I);
   bool visitMemCmpCall(const CallInst &I);
-  
+  void visitAtomicLoad(const LoadInst &I);
+  void visitAtomicStore(const StoreInst &I);
+
   void visitInlineAsm(ImmutableCallSite CS);
   const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
   void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
@@ -531,7 +555,6 @@ class SelectionDAGBuilder {
     llvm_unreachable("UserOp2 should not exist at instruction selection time!");
   }
   
-  const char *implVisitBinaryAtomic(const CallInst& I, ISD::NodeType Op);
   const char *implVisitAluOverflow(const CallInst &I, ISD::NodeType Op);
 
   void HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB);
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 87bb296b8c79..68b9146adfe1 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -177,6 +177,13 @@ TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   return 0;
 }
 
+void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI,
+                                                   SDNode *Node) const {
+  assert(!MI->getDesc().hasPostISelHook() &&
+         "If a target marks an instruction with 'hasPostISelHook', "
+         "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
+}
+
 //===----------------------------------------------------------------------===//
 // SelectionDAGISel code
 //===----------------------------------------------------------------------===//
@@ -463,6 +470,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
     GroupName = "Instruction Selection and Scheduling";
   std::string BlockName;
   int BlockNumber = -1;
+  (void)BlockNumber;
 #ifdef NDEBUG
   if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||
       ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs ||
@@ -677,21 +685,26 @@ void SelectionDAGISel::DoInstructionSelection() {
 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
 /// do other setup for EH landing-pad blocks.
 void SelectionDAGISel::PrepareEHLandingPad() {
+  MachineBasicBlock *MBB = FuncInfo->MBB;
+
   // Add a label to mark the beginning of the landing pad.  Deletion of the
   // landing pad can thus be detected via the MachineModuleInfo.
-  MCSymbol *Label = MF->getMMI().addLandingPad(FuncInfo->MBB);
+  MCSymbol *Label = MF->getMMI().addLandingPad(MBB);
 
+  // Assign the call site to the landing pad's begin label.
+  MF->getMMI().setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
+    
   const MCInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL);
-  BuildMI(*FuncInfo->MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
+  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
     .addSym(Label);
 
   // Mark exception register as live in.
   unsigned Reg = TLI.getExceptionAddressRegister();
-  if (Reg) FuncInfo->MBB->addLiveIn(Reg);
+  if (Reg) MBB->addLiveIn(Reg);
 
   // Mark exception selector register as live in.
   Reg = TLI.getExceptionSelectorRegister();
-  if (Reg) FuncInfo->MBB->addLiveIn(Reg);
+  if (Reg) MBB->addLiveIn(Reg);
 
   // FIXME: Hack around an exception handling flaw (PR1508): the personality
   // function and list of typeids logically belong to the invoke (or, if you
@@ -704,7 +717,7 @@ void SelectionDAGISel::PrepareEHLandingPad() {
   // in exceptions not being caught because no typeids are associated with
   // the invoke.  This may not be the only way things can go wrong, but it
   // is the only way we try to work around for the moment.
-  const BasicBlock *LLVMBB = FuncInfo->MBB->getBasicBlock();
+  const BasicBlock *LLVMBB = MBB->getBasicBlock();
   const BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator());
 
   if (Br && Br->isUnconditional()) { // Critical edge?
@@ -719,8 +732,6 @@ void SelectionDAGISel::PrepareEHLandingPad() {
   }
 }
 
-
-
 /// TryToFoldFastISelLoad - We're checking to see if we can fold the specified
 /// load into the specified FoldInst.  Note that we could have a sequence where
 /// multiple LLVM IR instructions are folded into the same machineinstr.  For
@@ -741,7 +752,7 @@ bool SelectionDAGISel::TryToFoldFastISelLoad(const LoadInst *LI,
   // isn't one of the folded instructions, then we can't succeed here.  Handle
   // this by scanning the single-use users of the load until we get to FoldInst.
   unsigned MaxUsers = 6;  // Don't scan down huge single-use chains of instrs.
-  
+
   const Instruction *TheUser = LI->use_back();
   while (TheUser != FoldInst &&   // Scan up until we find FoldInst.
          // Stay in the right block.
@@ -750,10 +761,15 @@ bool SelectionDAGISel::TryToFoldFastISelLoad(const LoadInst *LI,
     // If there are multiple or no uses of this instruction, then bail out.
     if (!TheUser->hasOneUse())
       return false;
-    
+
     TheUser = TheUser->use_back();
   }
-  
+
+  // If we didn't find the fold instruction, then we failed to collapse the
+  // sequence.
+  if (TheUser != FoldInst)
+    return false;
+
   // Don't try to fold volatile loads.  Target has to deal with alignment
   // constraints.
   if (LI->isVolatile()) return false;
@@ -802,6 +818,7 @@ static bool isFoldedOrDeadInstruction(const Instruction *I,
   return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
          !isa<TerminatorInst>(I) && // Terminators aren't folded.
          !isa<DbgInfoIntrinsic>(I) &&  // Debug instructions aren't folded.
+         !isa<LandingPadInst>(I) &&    // Landingpad instructions aren't folded.
          !FuncInfo->isExportedInst(I); // Exported instrs must be computed.
 }
 
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 2626ac3bbb2a..907d8d9da1af 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -317,7 +317,7 @@ static void InitLibcallNames(const char **Names) {
   Names[RTLIB::SYNC_FETCH_AND_OR_8] = "__sync_fetch_and_or_8";
   Names[RTLIB::SYNC_FETCH_AND_XOR_1] = "__sync_fetch_and_xor_1";
   Names[RTLIB::SYNC_FETCH_AND_XOR_2] = "__sync_fetch_and_xor_2";
-  Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and-xor_4";
+  Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and_xor_4";
   Names[RTLIB::SYNC_FETCH_AND_XOR_8] = "__sync_fetch_and_xor_8";
   Names[RTLIB::SYNC_FETCH_AND_NAND_1] = "__sync_fetch_and_nand_1";
   Names[RTLIB::SYNC_FETCH_AND_NAND_2] = "__sync_fetch_and_nand_2";
@@ -609,6 +609,7 @@ TargetLowering::TargetLowering(const TargetMachine &tm,
   ExceptionPointerRegister = 0;
   ExceptionSelectorRegister = 0;
   BooleanContents = UndefinedBooleanContent;
+  BooleanVectorContents = UndefinedBooleanContent;
   SchedPreferenceInfo = Sched::Latency;
   JumpBufSize = 0;
   JumpBufAlignment = 0;
@@ -617,6 +618,7 @@ TargetLowering::TargetLowering(const TargetMachine &tm,
   PrefLoopAlignment = 0;
   MinStackArgumentAlignment = 1;
   ShouldFoldAtomicFences = false;
+  InsertFencesForAtomic = false;
 
   InitLibcallNames(LibcallRoutineNames);
   InitCmpLibcallCCs(CmpLibcallCCs);
@@ -914,7 +916,8 @@ const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
 }
 
 
-MVT::SimpleValueType TargetLowering::getSetCCResultType(EVT VT) const {
+EVT TargetLowering::getSetCCResultType(EVT VT) const {
+  assert(!VT.isVector() && "No default SetCC type for vectors!");
   return PointerTy.SimpleTy;
 }
 
@@ -996,7 +999,7 @@ unsigned TargetLowering::getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
 /// type of the given function.  This does not require a DAG or a return value,
 /// and is suitable for use before any DAGs for the function are constructed.
 /// TODO: Move this out of TargetLowering.cpp.
-void llvm::GetReturnInfo(const Type* ReturnType, Attributes attr,
+void llvm::GetReturnInfo(Type* ReturnType, Attributes attr,
                          SmallVectorImpl<ISD::OutputArg> &Outs,
                          const TargetLowering &TLI,
                          SmallVectorImpl<uint64_t> *Offsets) {
@@ -1054,7 +1057,7 @@ void llvm::GetReturnInfo(const Type* ReturnType, Attributes attr,
 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
 /// function arguments in the caller parameter area.  This is the actual
 /// alignment, not its logarithm.
-unsigned TargetLowering::getByValTypeAlignment(const Type *Ty) const {
+unsigned TargetLowering::getByValTypeAlignment(Type *Ty) const {
   return TD->getCallFrameTypeAlignment(Ty);
 }
 
@@ -1764,17 +1767,16 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
     break;
   }
   case ISD::AssertZext: {
-    // Demand all the bits of the input that are demanded in the output.
-    // The low bits are obvious; the high bits are demanded because we're
-    // asserting that they're zero here.
-    if (SimplifyDemandedBits(Op.getOperand(0), NewMask,
+    // AssertZext demands all of the high bits, plus any of the low bits
+    // demanded by its users.
+    EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+    APInt InMask = APInt::getLowBitsSet(BitWidth,
+                                        VT.getSizeInBits());
+    if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | NewMask,
                              KnownZero, KnownOne, TLO, Depth+1))
       return true;
     assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
 
-    EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
-    APInt InMask = APInt::getLowBitsSet(BitWidth,
-                                        VT.getSizeInBits());
     KnownZero |= ~InMask & NewMask;
     break;
   }
@@ -2191,7 +2193,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         }
       } else if (N1C->getAPIntValue() == 1 &&
                  (VT == MVT::i1 ||
-                  getBooleanContents() == ZeroOrOneBooleanContent)) {
+                  getBooleanContents(false) == ZeroOrOneBooleanContent)) {
         SDValue Op0 = N0;
         if (Op0.getOpcode() == ISD::TRUNCATE)
           Op0 = Op0.getOperand(0);
@@ -2758,16 +2760,8 @@ getRegForInlineAsmConstraint(const std::string &Constraint,
 
     // If none of the value types for this register class are valid, we
     // can't use it.  For example, 64-bit reg classes on 32-bit targets.
-    bool isLegal = false;
-    for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end();
-         I != E; ++I) {
-      if (isTypeLegal(*I)) {
-        isLegal = true;
-        break;
-      }
-    }
-
-    if (!isLegal) continue;
+    if (!isLegalRC(RC))
+      continue;
 
     for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
          I != E; ++I) {
@@ -2840,7 +2834,7 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints(
       // corresponding argument.
       assert(!CS.getType()->isVoidTy() &&
              "Bad inline asm!");
-      if (const StructType *STy = dyn_cast<StructType>(CS.getType())) {
+      if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
         OpInfo.ConstraintVT = getValueType(STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
@@ -2857,16 +2851,16 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints(
     }
 
     if (OpInfo.CallOperandVal) {
-      const llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
+      llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
       if (OpInfo.isIndirect) {
-        const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
+        llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
         if (!PtrTy)
           report_fatal_error("Indirect operand for inline asm not a pointer!");
         OpTy = PtrTy->getElementType();
       }
 
       // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
-      if (const StructType *STy = dyn_cast<StructType>(OpTy))
+      if (StructType *STy = dyn_cast<StructType>(OpTy))
         if (STy->getNumElements() == 1)
           OpTy = STy->getElementType(0);
 
@@ -3187,7 +3181,7 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
 /// isLegalAddressingMode - Return true if the addressing mode represented
 /// by AM is legal for this target, for a load/store of the specified type.
 bool TargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                           const Type *Ty) const {
+                                           Type *Ty) const {
   // The default implementation of this implements a conservative RISCy, r+r and
   // r+i addr mode.
 
diff --git a/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp b/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp
index 5a253a4d97e4..2609256c8ffa 100644
--- a/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp
+++ b/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp
@@ -61,7 +61,7 @@ namespace {
   private:
     bool IsNullValue(Value *V);
     Constant *GetFrameMap(Function &F);
-    const Type* GetConcreteStackEntryType(Function &F);
+    Type* GetConcreteStackEntryType(Function &F);
     void CollectRoots(Function &F);
     static GetElementPtrInst *CreateGEP(LLVMContext &Context, 
                                         IRBuilder<> &B, Value *BasePtr,
@@ -109,13 +109,15 @@ namespace {
         State = 1;
 
       case 1:
-        // Find all 'return' and 'unwind' instructions.
+        // Find all 'return', 'resume', and 'unwind' instructions.
         while (StateBB != StateE) {
           BasicBlock *CurBB = StateBB++;
 
-          // Branches and invokes do not escape, only unwind and return do.
+          // Branches and invokes do not escape, only unwind, resume, and return
+          // do.
           TerminatorInst *TI = CurBB->getTerminator();
-          if (!isa<UnwindInst>(TI) && !isa<ReturnInst>(TI))
+          if (!isa<UnwindInst>(TI) && !isa<ReturnInst>(TI) &&
+              !isa<ResumeInst>(TI))
             continue;
 
           Builder.SetInsertPoint(TI->getParent(), TI);
@@ -139,9 +141,19 @@ namespace {
           return 0;
 
         // Create a cleanup block.
-        BasicBlock *CleanupBB = BasicBlock::Create(F.getContext(),
-                                                   CleanupBBName, &F);
-        UnwindInst *UI = new UnwindInst(F.getContext(), CleanupBB);
+        LLVMContext &C = F.getContext();
+        BasicBlock *CleanupBB = BasicBlock::Create(C, CleanupBBName, &F);
+        Type *ExnTy = StructType::get(Type::getInt8PtrTy(C),
+                                      Type::getInt32Ty(C), NULL);
+        Constant *PersFn =
+          F.getParent()->
+          getOrInsertFunction("__gcc_personality_v0",
+                              FunctionType::get(Type::getInt32Ty(C), true));
+        LandingPadInst *LPad = LandingPadInst::Create(ExnTy, PersFn, 1,
+                                                      "cleanup.lpad",
+                                                      CleanupBB);
+        LPad->setCleanup(true);
+        ResumeInst *RI = ResumeInst::Create(LPad, CleanupBB);
 
         // Transform the 'call' instructions into 'invoke's branching to the
         // cleanup block. Go in reverse order to make prettier BB names.
@@ -172,7 +184,7 @@ namespace {
           delete CI;
         }
 
-        Builder.SetInsertPoint(UI->getParent(), UI);
+        Builder.SetInsertPoint(RI->getParent(), RI);
         return &Builder;
       }
     }
@@ -190,7 +202,7 @@ ShadowStackGC::ShadowStackGC() : Head(0), StackEntryTy(0) {
 
 Constant *ShadowStackGC::GetFrameMap(Function &F) {
   // doInitialization creates the abstract type of this value.
-  const Type *VoidPtr = Type::getInt8PtrTy(F.getContext());
+  Type *VoidPtr = Type::getInt8PtrTy(F.getContext());
 
   // Truncate the ShadowStackDescriptor if some metadata is null.
   unsigned NumMeta = 0;
@@ -203,7 +215,7 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) {
   }
   Metadata.resize(NumMeta);
 
-  const Type *Int32Ty = Type::getInt32Ty(F.getContext());
+  Type *Int32Ty = Type::getInt32Ty(F.getContext());
   
   Constant *BaseElts[] = {
     ConstantInt::get(Int32Ty, Roots.size(), false),
@@ -216,7 +228,7 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) {
   };
 
   Type *EltTys[] = { DescriptorElts[0]->getType(),DescriptorElts[1]->getType()};
-  StructType *STy = StructType::createNamed("gc_map."+utostr(NumMeta), EltTys);
+  StructType *STy = StructType::create(EltTys, "gc_map."+utostr(NumMeta));
   
   Constant *FrameMap = ConstantStruct::get(STy, DescriptorElts);
 
@@ -241,17 +253,17 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) {
                           ConstantInt::get(Type::getInt32Ty(F.getContext()), 0),
                           ConstantInt::get(Type::getInt32Ty(F.getContext()), 0)
                           };
-  return ConstantExpr::getGetElementPtr(GV, GEPIndices, 2);
+  return ConstantExpr::getGetElementPtr(GV, GEPIndices);
 }
 
-const Type* ShadowStackGC::GetConcreteStackEntryType(Function &F) {
+Type* ShadowStackGC::GetConcreteStackEntryType(Function &F) {
   // doInitialization creates the generic version of this type.
   std::vector<Type*> EltTys;
   EltTys.push_back(StackEntryTy);
   for (size_t I = 0; I != Roots.size(); I++)
     EltTys.push_back(Roots[I].second->getAllocatedType());
   
-  return StructType::createNamed("gc_stackentry."+F.getName().str(), EltTys);
+  return StructType::create(EltTys, "gc_stackentry."+F.getName().str());
 }
 
 /// doInitialization - If this module uses the GC intrinsics, find them now. If
@@ -267,7 +279,7 @@ bool ShadowStackGC::initializeCustomLowering(Module &M) {
   EltTys.push_back(Type::getInt32Ty(M.getContext()));
   // Specifies length of variable length array. 
   EltTys.push_back(Type::getInt32Ty(M.getContext()));
-  FrameMapTy = StructType::createNamed("gc_map", EltTys);
+  FrameMapTy = StructType::create(EltTys, "gc_map");
   PointerType *FrameMapPtrTy = PointerType::getUnqual(FrameMapTy);
 
   // struct StackEntry {
@@ -276,13 +288,13 @@ bool ShadowStackGC::initializeCustomLowering(Module &M) {
   //   void *Roots[];          // Stack roots (in-place array, so we pretend).
   // };
   
-  StackEntryTy = StructType::createNamed(M.getContext(), "gc_stackentry");
+  StackEntryTy = StructType::create(M.getContext(), "gc_stackentry");
   
   EltTys.clear();
   EltTys.push_back(PointerType::getUnqual(StackEntryTy));
   EltTys.push_back(FrameMapPtrTy);
   StackEntryTy->setBody(EltTys);
-  const PointerType *StackEntryPtrTy = PointerType::getUnqual(StackEntryTy);
+  PointerType *StackEntryPtrTy = PointerType::getUnqual(StackEntryTy);
 
   // Get the root chain if it already exists.
   Head = M.getGlobalVariable("llvm_gc_root_chain");
@@ -340,7 +352,7 @@ ShadowStackGC::CreateGEP(LLVMContext &Context, IRBuilder<> &B, Value *BasePtr,
   Value *Indices[] = { ConstantInt::get(Type::getInt32Ty(Context), 0),
                        ConstantInt::get(Type::getInt32Ty(Context), Idx),
                        ConstantInt::get(Type::getInt32Ty(Context), Idx2) };
-  Value* Val = B.CreateGEP(BasePtr, Indices, Indices + 3, Name);
+  Value* Val = B.CreateGEP(BasePtr, Indices, Name);
 
   assert(isa<GetElementPtrInst>(Val) && "Unexpected folded constant");
 
@@ -352,7 +364,7 @@ ShadowStackGC::CreateGEP(LLVMContext &Context, IRBuilder<> &B, Value *BasePtr,
                          int Idx, const char *Name) {
   Value *Indices[] = { ConstantInt::get(Type::getInt32Ty(Context), 0),
                        ConstantInt::get(Type::getInt32Ty(Context), Idx) };
-  Value *Val = B.CreateGEP(BasePtr, Indices, Indices + 2, Name);
+  Value *Val = B.CreateGEP(BasePtr, Indices, Name);
 
   assert(isa<GetElementPtrInst>(Val) && "Unexpected folded constant");
 
@@ -373,7 +385,7 @@ bool ShadowStackGC::performCustomLowering(Function &F) {
 
   // Build the constant map and figure the type of the shadow stack entry.
   Value *FrameMap = GetFrameMap(F);
-  const Type *ConcreteStackEntryTy = GetConcreteStackEntryType(F);
+  Type *ConcreteStackEntryTy = GetConcreteStackEntryType(F);
 
   // Build the shadow stack entry at the very start of the function.
   BasicBlock::iterator IP = F.getEntryBlock().begin();
diff --git a/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp
index 65a33da93afe..ded2459d4278 100644
--- a/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp
+++ b/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp
@@ -21,26 +21,31 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/Module.h"
 #include "llvm/Pass.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/IRBuilder.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
 #include <set>
 using namespace llvm;
 
+static cl::opt<bool> DisableOldSjLjEH("disable-old-sjlj-eh", cl::Hidden,
+    cl::desc("Disable the old SjLj EH preparation pass"));
+
 STATISTIC(NumInvokes, "Number of invokes replaced");
 STATISTIC(NumUnwinds, "Number of unwinds replaced");
 STATISTIC(NumSpilled, "Number of registers live across unwind edges");
 
 namespace {
   class SjLjEHPass : public FunctionPass {
-
     const TargetLowering *TLI;
-
-    const Type *FunctionContextTy;
+    Type *FunctionContextTy;
     Constant *RegisterFn;
     Constant *UnregisterFn;
     Constant *BuiltinSetjmpFn;
@@ -53,8 +58,9 @@ namespace {
     Constant *ExceptionFn;
     Constant *CallSiteFn;
     Constant *DispatchSetupFn;
-
+    Constant *FuncCtxFn;
     Value *CallSite;
+    DenseMap<InvokeInst*, BasicBlock*> LPadSuccMap;
   public:
     static char ID; // Pass identification, replacement for typeid
     explicit SjLjEHPass(const TargetLowering *tli = NULL)
@@ -62,16 +68,22 @@ namespace {
     bool doInitialization(Module &M);
     bool runOnFunction(Function &F);
 
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const { }
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {}
     const char *getPassName() const {
       return "SJLJ Exception Handling preparation";
     }
 
   private:
+    bool setupEntryBlockAndCallSites(Function &F);
+    Value *setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads);
+    void lowerIncomingArguments(Function &F);
+    void lowerAcrossUnwindEdges(Function &F, ArrayRef<InvokeInst*> Invokes);
+
     void insertCallSiteStore(Instruction *I, int Number, Value *CallSite);
     void markInvokeCallSite(InvokeInst *II, int InvokeNo, Value *CallSite,
                             SwitchInst *CatchSwitch);
     void splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes);
+    void splitLandingPad(InvokeInst *II);
     bool insertSjLjEHSupport(Function &F);
   };
 } // end anonymous namespace
@@ -116,6 +128,7 @@ bool SjLjEHPass::doInitialization(Module &M) {
   CallSiteFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_callsite);
   DispatchSetupFn
     = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_dispatch_setup);
+  FuncCtxFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_functioncontext);
   PersonalityFn = 0;
 
   return true;
@@ -131,6 +144,42 @@ void SjLjEHPass::insertCallSiteStore(Instruction *I, int Number,
   new StoreInst(CallSiteNoC, CallSite, true, I);  // volatile
 }
 
+/// splitLandingPad - Split a landing pad. This takes considerable care because
+/// of PHIs and other nasties. The problem is that the jump table needs to jump
+/// to the landing pad block. However, the landing pad block can be jumped to
+/// only by an invoke instruction. So we clone the landingpad instruction into
+/// its own basic block, have the invoke jump to there. The landingpad
+/// instruction's basic block's successor is now the target for the jump table.
+///
+/// But because of PHI nodes, we need to create another basic block for the jump
+/// table to jump to. This is definitely a hack, because the values for the PHI
+/// nodes may not be defined on the edge from the jump table. But that's okay,
+/// because the jump table is simply a construct to mimic what is happening in
+/// the CFG. So the values are mysteriously there, even though there is no value
+/// for the PHI from the jump table's edge (hence calling this a hack).
+void SjLjEHPass::splitLandingPad(InvokeInst *II) {
+  SmallVector<BasicBlock*, 2> NewBBs;
+  SplitLandingPadPredecessors(II->getUnwindDest(), II->getParent(),
+                              ".1", ".2", this, NewBBs);
+
+  // Create an empty block so that the jump table has something to jump to
+  // which doesn't have any PHI nodes.
+  BasicBlock *LPad = NewBBs[0];
+  BasicBlock *Succ = *succ_begin(LPad);
+  BasicBlock *JumpTo = BasicBlock::Create(II->getContext(), "jt.land",
+                                          LPad->getParent(), Succ);
+  LPad->getTerminator()->eraseFromParent();
+  BranchInst::Create(JumpTo, LPad);
+  BranchInst::Create(Succ, JumpTo);
+  LPadSuccMap[II] = JumpTo;
+
+  for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); ++I) {
+    PHINode *PN = cast<PHINode>(I);
+    Value *Val = PN->removeIncomingValue(LPad, false);
+    PN->addIncoming(Val, JumpTo);
+  }
+}
+
 /// markInvokeCallSite - Insert code to mark the call_site for this invoke
 void SjLjEHPass::markInvokeCallSite(InvokeInst *II, int InvokeNo,
                                     Value *CallSite,
@@ -140,11 +189,15 @@ void SjLjEHPass::markInvokeCallSite(InvokeInst *II, int InvokeNo,
   // The runtime comes back to the dispatcher with the call_site - 1 in
   // the context. Odd, but there it is.
   ConstantInt *SwitchValC = ConstantInt::get(Type::getInt32Ty(II->getContext()),
-                                            InvokeNo - 1);
+                                             InvokeNo - 1);
 
   // If the unwind edge has phi nodes, split the edge.
   if (isa<PHINode>(II->getUnwindDest()->begin())) {
-    SplitCriticalEdge(II, 1, this);
+    // FIXME: New EH - This if-condition will be always true in the new scheme.
+    if (II->getUnwindDest()->isLandingPad())
+      splitLandingPad(II);
+    else
+      SplitCriticalEdge(II, 1, this);
 
     // If there are any phi nodes left, they must have a single predecessor.
     while (PHINode *PN = dyn_cast<PHINode>(II->getUnwindDest()->begin())) {
@@ -161,7 +214,12 @@ void SjLjEHPass::markInvokeCallSite(InvokeInst *II, int InvokeNo,
   CallInst::Create(CallSiteFn, CallSiteNoC, "", II);
 
   // Add a switch case to our unwind block.
-  CatchSwitch->addCase(SwitchValC, II->getUnwindDest());
+  if (BasicBlock *SuccBB = LPadSuccMap[II]) {
+    CatchSwitch->addCase(SwitchValC, SuccBB);
+  } else {
+    CatchSwitch->addCase(SwitchValC, II->getUnwindDest());
+  }
+
   // We still want this to look like an invoke so we emit the LSDA properly,
   // so we don't transform the invoke into a call here.
 }
@@ -187,10 +245,16 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) {
   for (unsigned i = 0, e = Invokes.size(); i != e; ++i) {
     InvokeInst *II = Invokes[i];
     SplitCriticalEdge(II, 0, this);
-    SplitCriticalEdge(II, 1, this);
+
+    // FIXME: New EH - This if-condition will be always true in the new scheme.
+    if (II->getUnwindDest()->isLandingPad())
+      splitLandingPad(II);
+    else
+      SplitCriticalEdge(II, 1, this);
+
     assert(!isa<PHINode>(II->getNormalDest()) &&
            !isa<PHINode>(II->getUnwindDest()) &&
-           "critical edge splitting left single entry phi nodes?");
+           "Critical edge splitting left single entry phi nodes?");
   }
 
   Function *F = Invokes.back()->getParent()->getParent();
@@ -204,7 +268,7 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) {
     ++AfterAllocaInsertPt;
   for (Function::arg_iterator AI = F->arg_begin(), E = F->arg_end();
        AI != E; ++AI) {
-    const Type *Ty = AI->getType();
+    Type *Ty = AI->getType();
     // Aggregate types can't be cast, but are legal argument types, so we have
     // to handle them differently. We use an extract/insert pair as a
     // lightweight method to achieve the same goal.
@@ -283,9 +347,8 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) {
       bool NeedsSpill = false;
       for (unsigned i = 0, e = Invokes.size(); i != e; ++i) {
         BasicBlock *UnwindBlock = Invokes[i]->getUnwindDest();
-        if (UnwindBlock != BB && LiveBBs.count(UnwindBlock)) {
+        if (UnwindBlock != BB && LiveBBs.count(UnwindBlock))
           NeedsSpill = true;
-        }
       }
 
       // If we decided we need a spill, do it.
@@ -299,6 +362,44 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) {
     }
 }
 
+/// CreateLandingPadLoad - Load the exception handling values and insert them
+/// into a structure.
+static Instruction *CreateLandingPadLoad(Function &F, Value *ExnAddr,
+                                         Value *SelAddr,
+                                         BasicBlock::iterator InsertPt) {
+  Value *Exn = new LoadInst(ExnAddr, "exn", false,
+                            InsertPt);
+  Type *Ty = Type::getInt8PtrTy(F.getContext());
+  Exn = CastInst::Create(Instruction::IntToPtr, Exn, Ty, "", InsertPt);
+  Value *Sel = new LoadInst(SelAddr, "sel", false, InsertPt);
+
+  Ty = StructType::get(Exn->getType(), Sel->getType(), NULL);
+  InsertValueInst *LPadVal = InsertValueInst::Create(llvm::UndefValue::get(Ty),
+                                                     Exn, 0,
+                                                     "lpad.val", InsertPt);
+  return InsertValueInst::Create(LPadVal, Sel, 1, "lpad.val", InsertPt);
+}
+
+/// ReplaceLandingPadVal - Replace the landingpad instruction's value with a
+/// load from the stored values (via CreateLandingPadLoad). This looks through
+/// PHI nodes, and removes them if they are dead.
+static void ReplaceLandingPadVal(Function &F, Instruction *Inst, Value *ExnAddr,
+                                 Value *SelAddr) {
+  if (Inst->use_empty()) return;
+
+  while (!Inst->use_empty()) {
+    Instruction *I = cast<Instruction>(Inst->use_back());
+
+    if (PHINode *PN = dyn_cast<PHINode>(I)) {
+      ReplaceLandingPadVal(F, PN, ExnAddr, SelAddr);
+      if (PN->use_empty()) PN->eraseFromParent();
+      continue;
+    }
+
+    I->replaceUsesOfWith(Inst, CreateLandingPadLoad(F, ExnAddr, SelAddr, I));
+  }
+}
+
 bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   SmallVector<ReturnInst*,16> Returns;
   SmallVector<UnwindInst*,16> Unwinds;
@@ -337,10 +438,23 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   SmallVector<CallInst*,16> EH_Exceptions;
   SmallVector<Instruction*,16> JmpbufUpdatePoints;
 
-  // Note: Skip the entry block since there's nothing there that interests
-  // us. eh.selector and eh.exception shouldn't ever be there, and we
-  // want to disregard any allocas that are there.
-  for (Function::iterator BB = F.begin(), E = F.end(); ++BB != E;) {
+  for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
+    // Note: Skip the entry block since there's nothing there that interests
+    // us. eh.selector and eh.exception shouldn't ever be there, and we
+    // want to disregard any allocas that are there.
+    // 
+    // FIXME: This is awkward. The new EH scheme won't need to skip the entry
+    //        block.
+    if (BB == F.begin()) {
+      if (InvokeInst *II = dyn_cast<InvokeInst>(F.begin()->getTerminator())) {
+        // FIXME: This will be always non-NULL in the new EH.
+        if (LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst())
+          if (!PersonalityFn) PersonalityFn = LPI->getPersonalityFn();
+      }
+
+      continue;
+    }
+
     for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
       if (CallInst *CI = dyn_cast<CallInst>(I)) {
         if (CI->getCalledFunction() == SelectorFn) {
@@ -353,6 +467,10 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
         }
       } else if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
         JmpbufUpdatePoints.push_back(AI);
+      } else if (InvokeInst *II = dyn_cast<InvokeInst>(I)) {
+        // FIXME: This will be always non-NULL in the new EH.
+        if (LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst())
+          if (!PersonalityFn) PersonalityFn = LPI->getPersonalityFn();
       }
     }
   }
@@ -371,6 +489,16 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   // invoke's.
   splitLiveRangesAcrossInvokes(Invokes);
 
+
+  SmallVector<LandingPadInst*, 16> LandingPads;
+  for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
+    if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator()))
+      // FIXME: This will be always non-NULL in the new EH.
+      if (LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst())
+        LandingPads.push_back(LPI);
+  }
+
+
   BasicBlock *EntryBB = F.begin();
   // Create an alloca for the incoming jump buffer ptr and the new jump buffer
   // that needs to be restored on all exits from the function.  This is an
@@ -381,27 +509,25 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
                    "fcn_context", F.begin()->begin());
 
   Value *Idxs[2];
-  const Type *Int32Ty = Type::getInt32Ty(F.getContext());
+  Type *Int32Ty = Type::getInt32Ty(F.getContext());
   Value *Zero = ConstantInt::get(Int32Ty, 0);
   // We need to also keep around a reference to the call_site field
   Idxs[0] = Zero;
   Idxs[1] = ConstantInt::get(Int32Ty, 1);
-  CallSite = GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2,
-                                       "call_site",
+  CallSite = GetElementPtrInst::Create(FunctionContext, Idxs, "call_site",
                                        EntryBB->getTerminator());
 
   // The exception selector comes back in context->data[1]
   Idxs[1] = ConstantInt::get(Int32Ty, 2);
-  Value *FCData = GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2,
-                                            "fc_data",
+  Value *FCData = GetElementPtrInst::Create(FunctionContext, Idxs, "fc_data",
                                             EntryBB->getTerminator());
   Idxs[1] = ConstantInt::get(Int32Ty, 1);
-  Value *SelectorAddr = GetElementPtrInst::Create(FCData, Idxs, Idxs+2,
+  Value *SelectorAddr = GetElementPtrInst::Create(FCData, Idxs,
                                                   "exc_selector_gep",
                                                   EntryBB->getTerminator());
   // The exception value comes back in context->data[0]
   Idxs[1] = Zero;
-  Value *ExceptionAddr = GetElementPtrInst::Create(FCData, Idxs, Idxs+2,
+  Value *ExceptionAddr = GetElementPtrInst::Create(FCData, Idxs,
                                                    "exception_gep",
                                                    EntryBB->getTerminator());
 
@@ -423,13 +549,16 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
     // instruction hasn't already been removed.
     if (!I->getParent()) continue;
     Value *Val = new LoadInst(ExceptionAddr, "exception", true, I);
-    const Type *Ty = Type::getInt8PtrTy(F.getContext());
+    Type *Ty = Type::getInt8PtrTy(F.getContext());
     Val = CastInst::Create(Instruction::IntToPtr, Val, Ty, "", I);
 
     I->replaceAllUsesWith(Val);
     I->eraseFromParent();
   }
 
+  for (unsigned i = 0, e = LandingPads.size(); i != e; ++i)
+    ReplaceLandingPadVal(F, LandingPads[i], ExceptionAddr, SelectorAddr);
+
   // The entry block changes to have the eh.sjlj.setjmp, with a conditional
   // branch to a dispatch block for non-zero returns. If we return normally,
   // we're not handling an exception and just register the function context and
@@ -466,8 +595,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   Idxs[0] = Zero;
   Idxs[1] = ConstantInt::get(Int32Ty, 4);
   Value *LSDAFieldPtr =
-    GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2,
-                              "lsda_gep",
+    GetElementPtrInst::Create(FunctionContext, Idxs, "lsda_gep",
                               EntryBB->getTerminator());
   Value *LSDA = CallInst::Create(LSDAAddrFn, "lsda_addr",
                                  EntryBB->getTerminator());
@@ -475,8 +603,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
 
   Idxs[1] = ConstantInt::get(Int32Ty, 3);
   Value *PersonalityFieldPtr =
-    GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2,
-                              "lsda_gep",
+    GetElementPtrInst::Create(FunctionContext, Idxs, "lsda_gep",
                               EntryBB->getTerminator());
   new StoreInst(PersonalityFn, PersonalityFieldPtr, true,
                 EntryBB->getTerminator());
@@ -484,12 +611,11 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   // Save the frame pointer.
   Idxs[1] = ConstantInt::get(Int32Ty, 5);
   Value *JBufPtr
-    = GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2,
-                                "jbuf_gep",
+    = GetElementPtrInst::Create(FunctionContext, Idxs, "jbuf_gep",
                                 EntryBB->getTerminator());
   Idxs[1] = ConstantInt::get(Int32Ty, 0);
   Value *FramePtr =
-    GetElementPtrInst::Create(JBufPtr, Idxs, Idxs+2, "jbuf_fp_gep",
+    GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_fp_gep",
                               EntryBB->getTerminator());
 
   Value *Val = CallInst::Create(FrameAddrFn,
@@ -501,7 +627,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   // Save the stack pointer.
   Idxs[1] = ConstantInt::get(Int32Ty, 2);
   Value *StackPtr =
-    GetElementPtrInst::Create(JBufPtr, Idxs, Idxs+2, "jbuf_sp_gep",
+    GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_sp_gep",
                               EntryBB->getTerminator());
 
   Val = CallInst::Create(StackAddrFn, "sp", EntryBB->getTerminator());
@@ -513,7 +639,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
                      Type::getInt8PtrTy(F.getContext()), "",
                      EntryBB->getTerminator());
   Value *DispatchVal = CallInst::Create(BuiltinSetjmpFn, SetjmpArg,
-                                        "dispatch",
+                                        "",
                                         EntryBB->getTerminator());
 
   // Add a call to dispatch_setup after the setjmp call. This is expanded to any
@@ -554,6 +680,8 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
         if (Callee != SelectorFn && Callee != ExceptionFn
             && !CI->doesNotThrow())
           insertCallSiteStore(CI, -1, CallSite);
+      } else if (ResumeInst *RI = dyn_cast<ResumeInst>(I)) {
+        insertCallSiteStore(RI, -1, CallSite);
       }
   }
 
@@ -582,7 +710,317 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) {
   return true;
 }
 
+/// setupFunctionContext - Allocate the function context on the stack and fill
+/// it with all of the data that we know at this point.
+Value *SjLjEHPass::
+setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads) {
+  BasicBlock *EntryBB = F.begin();
+
+  // Create an alloca for the incoming jump buffer ptr and the new jump buffer
+  // that needs to be restored on all exits from the function. This is an alloca
+  // because the value needs to be added to the global context list.
+  unsigned Align =
+    TLI->getTargetData()->getPrefTypeAlignment(FunctionContextTy);
+  AllocaInst *FuncCtx =
+    new AllocaInst(FunctionContextTy, 0, Align, "fn_context", EntryBB->begin());
+
+  // Fill in the function context structure.
+  Value *Idxs[2];
+  Type *Int32Ty = Type::getInt32Ty(F.getContext());
+  Value *Zero = ConstantInt::get(Int32Ty, 0);
+  Value *One = ConstantInt::get(Int32Ty, 1);
+
+  // Keep around a reference to the call_site field.
+  Idxs[0] = Zero;
+  Idxs[1] = One;
+  CallSite = GetElementPtrInst::Create(FuncCtx, Idxs, "call_site",
+                                       EntryBB->getTerminator());
+
+  // Reference the __data field.
+  Idxs[1] = ConstantInt::get(Int32Ty, 2);
+  Value *FCData = GetElementPtrInst::Create(FuncCtx, Idxs, "__data",
+                                            EntryBB->getTerminator());
+
+  // The exception value comes back in context->__data[0].
+  Idxs[1] = Zero;
+  Value *ExceptionAddr = GetElementPtrInst::Create(FCData, Idxs,
+                                                   "exception_gep",
+                                                   EntryBB->getTerminator());
+
+  // The exception selector comes back in context->__data[1].
+  Idxs[1] = One;
+  Value *SelectorAddr = GetElementPtrInst::Create(FCData, Idxs,
+                                                  "exn_selector_gep",
+                                                  EntryBB->getTerminator());
+
+  for (unsigned I = 0, E = LPads.size(); I != E; ++I) {
+    LandingPadInst *LPI = LPads[I];
+    IRBuilder<> Builder(LPI->getParent()->getFirstInsertionPt());
+
+    Value *ExnVal = Builder.CreateLoad(ExceptionAddr, true, "exn_val");
+    ExnVal = Builder.CreateIntToPtr(ExnVal, Type::getInt8PtrTy(F.getContext()));
+    Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val");
+
+    Type *LPadType = LPI->getType();
+    Value *LPadVal = UndefValue::get(LPadType);
+    LPadVal = Builder.CreateInsertValue(LPadVal, ExnVal, 0, "lpad.val");
+    LPadVal = Builder.CreateInsertValue(LPadVal, SelVal, 1, "lpad.val");
+
+    LPI->replaceAllUsesWith(LPadVal);
+  }
+
+  // Personality function
+  Idxs[1] = ConstantInt::get(Int32Ty, 3);
+  if (!PersonalityFn)
+    PersonalityFn = LPads[0]->getPersonalityFn();
+  Value *PersonalityFieldPtr =
+    GetElementPtrInst::Create(FuncCtx, Idxs, "pers_fn_gep",
+                              EntryBB->getTerminator());
+  new StoreInst(PersonalityFn, PersonalityFieldPtr, true,
+                EntryBB->getTerminator());
+
+  // LSDA address
+  Idxs[1] = ConstantInt::get(Int32Ty, 4);
+  Value *LSDAFieldPtr = GetElementPtrInst::Create(FuncCtx, Idxs, "lsda_gep",
+                                                  EntryBB->getTerminator());
+  Value *LSDA = CallInst::Create(LSDAAddrFn, "lsda_addr",
+                                 EntryBB->getTerminator());
+  new StoreInst(LSDA, LSDAFieldPtr, true, EntryBB->getTerminator());
+
+  return FuncCtx;
+}
+
+/// lowerIncomingArguments - To avoid having to handle incoming arguments
+/// specially, we lower each arg to a copy instruction in the entry block. This
+/// ensures that the argument value itself cannot be live out of the entry
+/// block.
+void SjLjEHPass::lowerIncomingArguments(Function &F) {
+  BasicBlock::iterator AfterAllocaInsPt = F.begin()->begin();
+  while (isa<AllocaInst>(AfterAllocaInsPt) &&
+         isa<ConstantInt>(cast<AllocaInst>(AfterAllocaInsPt)->getArraySize()))
+    ++AfterAllocaInsPt;
+
+  for (Function::arg_iterator
+         AI = F.arg_begin(), AE = F.arg_end(); AI != AE; ++AI) {
+    Type *Ty = AI->getType();
+
+    // Aggregate types can't be cast, but are legal argument types, so we have
+    // to handle them differently. We use an extract/insert pair as a
+    // lightweight method to achieve the same goal.
+    if (isa<StructType>(Ty) || isa<ArrayType>(Ty) || isa<VectorType>(Ty)) {
+      Instruction *EI = ExtractValueInst::Create(AI, 0, "", AfterAllocaInsPt);
+      Instruction *NI = InsertValueInst::Create(AI, EI, 0);
+      NI->insertAfter(EI);
+      AI->replaceAllUsesWith(NI);
+
+      // Set the operand of the instructions back to the AllocaInst.
+      EI->setOperand(0, AI);
+      NI->setOperand(0, AI);
+    } else {
+      // This is always a no-op cast because we're casting AI to AI->getType()
+      // so src and destination types are identical. BitCast is the only
+      // possibility.
+      CastInst *NC =
+        new BitCastInst(AI, AI->getType(), AI->getName() + ".tmp",
+                        AfterAllocaInsPt);
+      AI->replaceAllUsesWith(NC);
+
+      // Set the operand of the cast instruction back to the AllocaInst.
+      // Normally it's forbidden to replace a CastInst's operand because it
+      // could cause the opcode to reflect an illegal conversion. However, we're
+      // replacing it here with the same value it was constructed with.  We do
+      // this because the above replaceAllUsesWith() clobbered the operand, but
+      // we want this one to remain.
+      NC->setOperand(0, AI);
+    }
+  }
+}
+
+/// lowerAcrossUnwindEdges - Find all variables which are alive across an unwind
+/// edge and spill them.
+void SjLjEHPass::lowerAcrossUnwindEdges(Function &F,
+                                        ArrayRef<InvokeInst*> Invokes) {
+  // Finally, scan the code looking for instructions with bad live ranges.
+  for (Function::iterator
+         BB = F.begin(), BBE = F.end(); BB != BBE; ++BB) {
+    for (BasicBlock::iterator
+           II = BB->begin(), IIE = BB->end(); II != IIE; ++II) {
+      // Ignore obvious cases we don't have to handle. In particular, most
+      // instructions either have no uses or only have a single use inside the
+      // current block. Ignore them quickly.
+      Instruction *Inst = II;
+      if (Inst->use_empty()) continue;
+      if (Inst->hasOneUse() &&
+          cast<Instruction>(Inst->use_back())->getParent() == BB &&
+          !isa<PHINode>(Inst->use_back())) continue;
+
+      // If this is an alloca in the entry block, it's not a real register
+      // value.
+      if (AllocaInst *AI = dyn_cast<AllocaInst>(Inst))
+        if (isa<ConstantInt>(AI->getArraySize()) && BB == F.begin())
+          continue;
+
+      // Avoid iterator invalidation by copying users to a temporary vector.
+      SmallVector<Instruction*, 16> Users;
+      for (Value::use_iterator
+             UI = Inst->use_begin(), E = Inst->use_end(); UI != E; ++UI) {
+        Instruction *User = cast<Instruction>(*UI);
+        if (User->getParent() != BB || isa<PHINode>(User))
+          Users.push_back(User);
+      }
+
+      // Find all of the blocks that this value is live in.
+      std::set<BasicBlock*> LiveBBs;
+      LiveBBs.insert(Inst->getParent());
+      while (!Users.empty()) {
+        Instruction *U = Users.back();
+        Users.pop_back();
+
+        if (!isa<PHINode>(U)) {
+          MarkBlocksLiveIn(U->getParent(), LiveBBs);
+        } else {
+          // Uses for a PHI node occur in their predecessor block.
+          PHINode *PN = cast<PHINode>(U);
+          for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
+            if (PN->getIncomingValue(i) == Inst)
+              MarkBlocksLiveIn(PN->getIncomingBlock(i), LiveBBs);
+        }
+      }
+
+      // Now that we know all of the blocks that this thing is live in, see if
+      // it includes any of the unwind locations.
+      bool NeedsSpill = false;
+      for (unsigned i = 0, e = Invokes.size(); i != e; ++i) {
+        BasicBlock *UnwindBlock = Invokes[i]->getUnwindDest();
+        if (UnwindBlock != BB && LiveBBs.count(UnwindBlock)) {
+          NeedsSpill = true;
+        }
+      }
+
+      // If we decided we need a spill, do it.
+      // FIXME: Spilling this way is overkill, as it forces all uses of
+      // the value to be reloaded from the stack slot, even those that aren't
+      // in the unwind blocks. We should be more selective.
+      if (NeedsSpill) {
+        ++NumSpilled;
+        DemoteRegToStack(*Inst, true);
+      }
+    }
+  }
+}
+
+/// setupEntryBlockAndCallSites - Setup the entry block by creating and filling
+/// the function context and marking the call sites with the appropriate
+/// values. These values are used by the DWARF EH emitter.
+bool SjLjEHPass::setupEntryBlockAndCallSites(Function &F) {
+  SmallVector<ReturnInst*,     16> Returns;
+  SmallVector<InvokeInst*,     16> Invokes;
+  SmallVector<LandingPadInst*, 16> LPads;
+
+  // Look through the terminators of the basic blocks to find invokes.
+  for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
+    if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator())) {
+      Invokes.push_back(II);
+      LPads.push_back(II->getUnwindDest()->getLandingPadInst());
+    } else if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) {
+      Returns.push_back(RI);
+    }
+
+  if (Invokes.empty()) return false;
+
+  lowerIncomingArguments(F);
+  lowerAcrossUnwindEdges(F, Invokes);
+
+  Value *FuncCtx = setupFunctionContext(F, LPads);
+  BasicBlock *EntryBB = F.begin();
+  Type *Int32Ty = Type::getInt32Ty(F.getContext());
+
+  Value *Idxs[2] = {
+    ConstantInt::get(Int32Ty, 0), 0
+  };
+
+  // Get a reference to the jump buffer.
+  Idxs[1] = ConstantInt::get(Int32Ty, 5);
+  Value *JBufPtr = GetElementPtrInst::Create(FuncCtx, Idxs, "jbuf_gep",
+                                             EntryBB->getTerminator());
+
+  // Save the frame pointer.
+  Idxs[1] = ConstantInt::get(Int32Ty, 0);
+  Value *FramePtr = GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_fp_gep",
+                                              EntryBB->getTerminator());
+
+  Value *Val = CallInst::Create(FrameAddrFn,
+                                ConstantInt::get(Int32Ty, 0),
+                                "fp",
+                                EntryBB->getTerminator());
+  new StoreInst(Val, FramePtr, true, EntryBB->getTerminator());
+
+  // Save the stack pointer.
+  Idxs[1] = ConstantInt::get(Int32Ty, 2);
+  Value *StackPtr = GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_sp_gep",
+                                              EntryBB->getTerminator());
+
+  Val = CallInst::Create(StackAddrFn, "sp", EntryBB->getTerminator());
+  new StoreInst(Val, StackPtr, true, EntryBB->getTerminator());
+
+  // Call the setjmp instrinsic. It fills in the rest of the jmpbuf.
+  Value *SetjmpArg = CastInst::Create(Instruction::BitCast, JBufPtr,
+                                      Type::getInt8PtrTy(F.getContext()), "",
+                                      EntryBB->getTerminator());
+  CallInst::Create(BuiltinSetjmpFn, SetjmpArg, "", EntryBB->getTerminator());
+
+  // Store a pointer to the function context so that the back-end will know
+  // where to look for it.
+  Value *FuncCtxArg = CastInst::Create(Instruction::BitCast, FuncCtx,
+                                       Type::getInt8PtrTy(F.getContext()), "",
+                                       EntryBB->getTerminator());
+  CallInst::Create(FuncCtxFn, FuncCtxArg, "", EntryBB->getTerminator());
+
+  // At this point, we are all set up, update the invoke instructions to mark
+  // their call_site values.
+  for (unsigned I = 0, E = Invokes.size(); I != E; ++I) {
+    insertCallSiteStore(Invokes[I], I + 1, CallSite);
+
+    ConstantInt *CallSiteNum =
+      ConstantInt::get(Type::getInt32Ty(F.getContext()), I + 1);
+
+    // Record the call site value for the back end so it stays associated with
+    // the invoke.
+    CallInst::Create(CallSiteFn, CallSiteNum, "", Invokes[I]);
+  }
+
+  // Mark call instructions that aren't nounwind as no-action (call_site ==
+  // -1). Skip the entry block, as prior to then, no function context has been
+  // created for this function and any unexpected exceptions thrown will go
+  // directly to the caller's context, which is what we want anyway, so no need
+  // to do anything here.
+  for (Function::iterator BB = F.begin(), E = F.end(); ++BB != E;)
+    for (BasicBlock::iterator I = BB->begin(), end = BB->end(); I != end; ++I)
+      if (CallInst *CI = dyn_cast<CallInst>(I)) {
+        if (!CI->doesNotThrow())
+          insertCallSiteStore(CI, -1, CallSite);
+      } else if (ResumeInst *RI = dyn_cast<ResumeInst>(I)) {
+        insertCallSiteStore(RI, -1, CallSite);
+      }
+
+  // Register the function context and make sure it's known to not throw
+  CallInst *Register = CallInst::Create(RegisterFn, FuncCtx, "",
+                                        EntryBB->getTerminator());
+  Register->setDoesNotThrow();
+
+  // Finally, for any returns from this function, if this function contains an
+  // invoke, add a call to unregister the function context.
+  for (unsigned I = 0, E = Returns.size(); I != E; ++I)
+    CallInst::Create(UnregisterFn, FuncCtx, "", Returns[I]);
+
+  return true;
+}
+
 bool SjLjEHPass::runOnFunction(Function &F) {
-  bool Res = insertSjLjEHSupport(F);
+  bool Res = false;
+  if (!DisableOldSjLjEH)
+    Res = insertSjLjEHSupport(F);
+  else
+    Res = setupEntryBlockAndCallSites(F);
   return Res;
 }
diff --git a/contrib/llvm/lib/CodeGen/SpillPlacement.cpp b/contrib/llvm/lib/CodeGen/SpillPlacement.cpp
index 694961863261..6f33f5465ca2 100644
--- a/contrib/llvm/lib/CodeGen/SpillPlacement.cpp
+++ b/contrib/llvm/lib/CodeGen/SpillPlacement.cpp
@@ -220,6 +220,7 @@ void SpillPlacement::addConstraints(ArrayRef<BlockConstraint> LiveBlocks) {
       0,           // DontCare,
       1,           // PrefReg,
       -1,          // PrefSpill
+      0,           // PrefBoth
       -HUGE_VALF   // MustSpill
     };
 
@@ -239,6 +240,22 @@ void SpillPlacement::addConstraints(ArrayRef<BlockConstraint> LiveBlocks) {
   }
 }
 
+/// addPrefSpill - Same as addConstraints(PrefSpill)
+void SpillPlacement::addPrefSpill(ArrayRef<unsigned> Blocks, bool Strong) {
+  for (ArrayRef<unsigned>::iterator I = Blocks.begin(), E = Blocks.end();
+       I != E; ++I) {
+    float Freq = getBlockFrequency(*I);
+    if (Strong)
+      Freq += Freq;
+    unsigned ib = bundles->getBundle(*I, 0);
+    unsigned ob = bundles->getBundle(*I, 1);
+    activate(ib);
+    activate(ob);
+    nodes[ib].addBias(-Freq, 1);
+    nodes[ob].addBias(-Freq, 0);
+  }
+}
+
 void SpillPlacement::addLinks(ArrayRef<unsigned> Links) {
   for (ArrayRef<unsigned>::iterator I = Links.begin(), E = Links.end(); I != E;
        ++I) {
diff --git a/contrib/llvm/lib/CodeGen/SpillPlacement.h b/contrib/llvm/lib/CodeGen/SpillPlacement.h
index 6952ad800965..fc412f817cdb 100644
--- a/contrib/llvm/lib/CodeGen/SpillPlacement.h
+++ b/contrib/llvm/lib/CodeGen/SpillPlacement.h
@@ -71,6 +71,7 @@ class SpillPlacement  : public MachineFunctionPass {
     DontCare,  ///< Block doesn't care / variable not live.
     PrefReg,   ///< Block entry/exit prefers a register.
     PrefSpill, ///< Block entry/exit prefers a stack slot.
+    PrefBoth,  ///< Block entry prefers both register and stack.
     MustSpill  ///< A register is impossible, variable must be spilled.
   };
 
@@ -79,6 +80,11 @@ class SpillPlacement  : public MachineFunctionPass {
     unsigned Number;            ///< Basic block number (from MBB::getNumber()).
     BorderConstraint Entry : 8; ///< Constraint on block entry.
     BorderConstraint Exit : 8;  ///< Constraint on block exit.
+
+    /// True when this block changes the value of the live range. This means
+    /// the block has a non-PHI def.  When this is false, a live-in value on
+    /// the stack can be live-out on the stack without inserting a spill.
+    bool ChangesValue;
   };
 
   /// prepare - Reset state and prepare for a new spill placement computation.
@@ -96,6 +102,14 @@ class SpillPlacement  : public MachineFunctionPass {
   ///                   live out.
   void addConstraints(ArrayRef<BlockConstraint> LiveBlocks);
 
+  /// addPrefSpill - Add PrefSpill constraints to all blocks listed.  This is
+  /// equivalent to calling addConstraint with identical BlockConstraints with
+  /// Entry = Exit = PrefSpill, and ChangesValue = false.
+  ///
+  /// @param Blocks Array of block numbers that prefer to spill in and out.
+  /// @param Strong When true, double the negative bias for these blocks.
+  void addPrefSpill(ArrayRef<unsigned> Blocks, bool Strong);
+
   /// addLinks - Add transparent blocks with the given numbers.
   void addLinks(ArrayRef<unsigned> Links);
 
diff --git a/contrib/llvm/lib/CodeGen/SplitKit.cpp b/contrib/llvm/lib/CodeGen/SplitKit.cpp
index 761cab7ce850..63627800af69 100644
--- a/contrib/llvm/lib/CodeGen/SplitKit.cpp
+++ b/contrib/llvm/lib/CodeGen/SplitKit.cpp
@@ -20,6 +20,7 @@
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -178,45 +179,55 @@ bool SplitAnalysis::calcLiveBlockInfo() {
         return false;
     } else {
       // This block has uses. Find the first and last uses in the block.
-      BI.FirstUse = *UseI;
-      assert(BI.FirstUse >= Start);
+      BI.FirstInstr = *UseI;
+      assert(BI.FirstInstr >= Start);
       do ++UseI;
       while (UseI != UseE && *UseI < Stop);
-      BI.LastUse = UseI[-1];
-      assert(BI.LastUse < Stop);
+      BI.LastInstr = UseI[-1];
+      assert(BI.LastInstr < Stop);
 
       // LVI is the first live segment overlapping MBB.
       BI.LiveIn = LVI->start <= Start;
 
+      // When not live in, the first use should be a def.
+      if (!BI.LiveIn) {
+        assert(LVI->start == LVI->valno->def && "Dangling LiveRange start");
+        assert(LVI->start == BI.FirstInstr && "First instr should be a def");
+        BI.FirstDef = BI.FirstInstr;
+      }
+
       // Look for gaps in the live range.
       BI.LiveOut = true;
       while (LVI->end < Stop) {
         SlotIndex LastStop = LVI->end;
         if (++LVI == LVE || LVI->start >= Stop) {
           BI.LiveOut = false;
-          BI.LastUse = LastStop;
+          BI.LastInstr = LastStop;
           break;
         }
+
         if (LastStop < LVI->start) {
           // There is a gap in the live range. Create duplicate entries for the
           // live-in snippet and the live-out snippet.
           ++NumGapBlocks;
 
           // Push the Live-in part.
-          BI.LiveThrough = false;
           BI.LiveOut = false;
           UseBlocks.push_back(BI);
-          UseBlocks.back().LastUse = LastStop;
+          UseBlocks.back().LastInstr = LastStop;
 
           // Set up BI for the live-out part.
           BI.LiveIn = false;
           BI.LiveOut = true;
-          BI.FirstUse = LVI->start;
+          BI.FirstInstr = BI.FirstDef = LVI->start;
         }
+
+        // A LiveRange that starts in the middle of the block must be a def.
+        assert(LVI->start == LVI->valno->def && "Dangling LiveRange start");
+        if (!BI.FirstDef)
+          BI.FirstDef = LVI->start;
       }
 
-      // Don't set LiveThrough when the block has a gap.
-      BI.LiveThrough = BI.LiveIn && BI.LiveOut;
       UseBlocks.push_back(BI);
 
       // LVI is now at LVE or LVI->end >= Stop.
@@ -299,17 +310,21 @@ SplitEditor::SplitEditor(SplitAnalysis &sa,
     TRI(*vrm.getMachineFunction().getTarget().getRegisterInfo()),
     Edit(0),
     OpenIdx(0),
+    SpillMode(SM_Partition),
     RegAssign(Allocator)
 {}
 
-void SplitEditor::reset(LiveRangeEdit &lre) {
-  Edit = &lre;
+void SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) {
+  Edit = &LRE;
+  SpillMode = SM;
   OpenIdx = 0;
   RegAssign.clear();
   Values.clear();
 
-  // We don't need to clear LiveOutCache, only LiveOutSeen entries are read.
-  LiveOutSeen.clear();
+  // Reset the LiveRangeCalc instances needed for this spill mode.
+  LRCalc[0].reset(&VRM.getMachineFunction());
+  if (SpillMode)
+    LRCalc[1].reset(&VRM.getMachineFunction());
 
   // We don't need an AliasAnalysis since we will only be performing
   // cheap-as-a-copy remats anyway.
@@ -340,7 +355,8 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx,
 
   // Use insert for lookup, so we can add missing values with a second lookup.
   std::pair<ValueMap::iterator, bool> InsP =
-    Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), VNI));
+    Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id),
+                                 ValueForcePair(VNI, false)));
 
   // This was the first time (RegIdx, ParentVNI) was mapped.
   // Keep it as a simple def without any liveness.
@@ -348,11 +364,11 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx,
     return VNI;
 
   // If the previous value was a simple mapping, add liveness for it now.
-  if (VNInfo *OldVNI = InsP.first->second) {
+  if (VNInfo *OldVNI = InsP.first->second.getPointer()) {
     SlotIndex Def = OldVNI->def;
     LI->addRange(LiveRange(Def, Def.getNextSlot(), OldVNI));
-    // No longer a simple mapping.
-    InsP.first->second = 0;
+    // No longer a simple mapping.  Switch to a complex, non-forced mapping.
+    InsP.first->second = ValueForcePair();
   }
 
   // This is a complex mapping, add liveness for VNI
@@ -362,230 +378,24 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx,
   return VNI;
 }
 
-void SplitEditor::markComplexMapped(unsigned RegIdx, const VNInfo *ParentVNI) {
+void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI) {
   assert(ParentVNI && "Mapping  NULL value");
-  VNInfo *&VNI = Values[std::make_pair(RegIdx, ParentVNI->id)];
+  ValueForcePair &VFP = Values[std::make_pair(RegIdx, ParentVNI->id)];
+  VNInfo *VNI = VFP.getPointer();
 
-  // ParentVNI was either unmapped or already complex mapped. Either way.
-  if (!VNI)
+  // ParentVNI was either unmapped or already complex mapped. Either way, just
+  // set the force bit.
+  if (!VNI) {
+    VFP.setInt(true);
     return;
+  }
 
   // This was previously a single mapping. Make sure the old def is represented
   // by a trivial live range.
   SlotIndex Def = VNI->def;
   Edit->get(RegIdx)->addRange(LiveRange(Def, Def.getNextSlot(), VNI));
-  VNI = 0;
-}
-
-// extendRange - Extend the live range to reach Idx.
-// Potentially create phi-def values.
-void SplitEditor::extendRange(unsigned RegIdx, SlotIndex Idx) {
-  assert(Idx.isValid() && "Invalid SlotIndex");
-  MachineBasicBlock *IdxMBB = LIS.getMBBFromIndex(Idx);
-  assert(IdxMBB && "No MBB at Idx");
-  LiveInterval *LI = Edit->get(RegIdx);
-
-  // Is there a def in the same MBB we can extend?
-  if (LI->extendInBlock(LIS.getMBBStartIdx(IdxMBB), Idx))
-    return;
-
-  // Now for the fun part. We know that ParentVNI potentially has multiple defs,
-  // and we may need to create even more phi-defs to preserve VNInfo SSA form.
-  // Perform a search for all predecessor blocks where we know the dominating
-  // VNInfo.
-  VNInfo *VNI = findReachingDefs(LI, IdxMBB, Idx.getNextSlot());
-
-  // When there were multiple different values, we may need new PHIs.
-  if (!VNI)
-    return updateSSA();
-
-  // Poor man's SSA update for the single-value case.
-  LiveOutPair LOP(VNI, MDT[LIS.getMBBFromIndex(VNI->def)]);
-  for (SmallVectorImpl<LiveInBlock>::iterator I = LiveInBlocks.begin(),
-         E = LiveInBlocks.end(); I != E; ++I) {
-    MachineBasicBlock *MBB = I->DomNode->getBlock();
-    SlotIndex Start = LIS.getMBBStartIdx(MBB);
-    if (I->Kill.isValid())
-      LI->addRange(LiveRange(Start, I->Kill, VNI));
-    else {
-      LiveOutCache[MBB] = LOP;
-      LI->addRange(LiveRange(Start, LIS.getMBBEndIdx(MBB), VNI));
-    }
-  }
-}
-
-/// findReachingDefs - Search the CFG for known live-out values.
-/// Add required live-in blocks to LiveInBlocks.
-VNInfo *SplitEditor::findReachingDefs(LiveInterval *LI,
-                                      MachineBasicBlock *KillMBB,
-                                      SlotIndex Kill) {
-  // Initialize the live-out cache the first time it is needed.
-  if (LiveOutSeen.empty()) {
-    unsigned N = VRM.getMachineFunction().getNumBlockIDs();
-    LiveOutSeen.resize(N);
-    LiveOutCache.resize(N);
-  }
-
-  // Blocks where LI should be live-in.
-  SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB);
-
-  // Remember if we have seen more than one value.
-  bool UniqueVNI = true;
-  VNInfo *TheVNI = 0;
-
-  // Using LiveOutCache as a visited set, perform a BFS for all reaching defs.
-  for (unsigned i = 0; i != WorkList.size(); ++i) {
-    MachineBasicBlock *MBB = WorkList[i];
-    assert(!MBB->pred_empty() && "Value live-in to entry block?");
-    for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
-           PE = MBB->pred_end(); PI != PE; ++PI) {
-       MachineBasicBlock *Pred = *PI;
-       LiveOutPair &LOP = LiveOutCache[Pred];
-
-       // Is this a known live-out block?
-       if (LiveOutSeen.test(Pred->getNumber())) {
-         if (VNInfo *VNI = LOP.first) {
-           if (TheVNI && TheVNI != VNI)
-             UniqueVNI = false;
-           TheVNI = VNI;
-         }
-         continue;
-       }
-
-       // First time. LOP is garbage and must be cleared below.
-       LiveOutSeen.set(Pred->getNumber());
-
-       // Does Pred provide a live-out value?
-       SlotIndex Start, Last;
-       tie(Start, Last) = LIS.getSlotIndexes()->getMBBRange(Pred);
-       Last = Last.getPrevSlot();
-       VNInfo *VNI = LI->extendInBlock(Start, Last);
-       LOP.first = VNI;
-       if (VNI) {
-         LOP.second = MDT[LIS.getMBBFromIndex(VNI->def)];
-         if (TheVNI && TheVNI != VNI)
-           UniqueVNI = false;
-         TheVNI = VNI;
-         continue;
-       }
-       LOP.second = 0;
-
-       // No, we need a live-in value for Pred as well
-       if (Pred != KillMBB)
-          WorkList.push_back(Pred);
-       else
-          // Loopback to KillMBB, so value is really live through.
-         Kill = SlotIndex();
-    }
-  }
-
-  // Transfer WorkList to LiveInBlocks in reverse order.
-  // This ordering works best with updateSSA().
-  LiveInBlocks.clear();
-  LiveInBlocks.reserve(WorkList.size());
-  while(!WorkList.empty())
-    LiveInBlocks.push_back(MDT[WorkList.pop_back_val()]);
-
-  // The kill block may not be live-through.
-  assert(LiveInBlocks.back().DomNode->getBlock() == KillMBB);
-  LiveInBlocks.back().Kill = Kill;
-
-  return UniqueVNI ? TheVNI : 0;
-}
-
-void SplitEditor::updateSSA() {
-  // This is essentially the same iterative algorithm that SSAUpdater uses,
-  // except we already have a dominator tree, so we don't have to recompute it.
-  unsigned Changes;
-  do {
-    Changes = 0;
-    // Propagate live-out values down the dominator tree, inserting phi-defs
-    // when necessary.
-    for (SmallVectorImpl<LiveInBlock>::iterator I = LiveInBlocks.begin(),
-           E = LiveInBlocks.end(); I != E; ++I) {
-      MachineDomTreeNode *Node = I->DomNode;
-      // Skip block if the live-in value has already been determined.
-      if (!Node)
-        continue;
-      MachineBasicBlock *MBB = Node->getBlock();
-      MachineDomTreeNode *IDom = Node->getIDom();
-      LiveOutPair IDomValue;
-
-      // We need a live-in value to a block with no immediate dominator?
-      // This is probably an unreachable block that has survived somehow.
-      bool needPHI = !IDom || !LiveOutSeen.test(IDom->getBlock()->getNumber());
-
-      // IDom dominates all of our predecessors, but it may not be their
-      // immediate dominator. Check if any of them have live-out values that are
-      // properly dominated by IDom. If so, we need a phi-def here.
-      if (!needPHI) {
-        IDomValue = LiveOutCache[IDom->getBlock()];
-        for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
-               PE = MBB->pred_end(); PI != PE; ++PI) {
-          LiveOutPair Value = LiveOutCache[*PI];
-          if (!Value.first || Value.first == IDomValue.first)
-            continue;
-          // This predecessor is carrying something other than IDomValue.
-          // It could be because IDomValue hasn't propagated yet, or it could be
-          // because MBB is in the dominance frontier of that value.
-          if (MDT.dominates(IDom, Value.second)) {
-            needPHI = true;
-            break;
-          }
-        }
-      }
-
-      // The value may be live-through even if Kill is set, as can happen when
-      // we are called from extendRange. In that case LiveOutSeen is true, and
-      // LiveOutCache indicates a foreign or missing value.
-      LiveOutPair &LOP = LiveOutCache[MBB];
-
-      // Create a phi-def if required.
-      if (needPHI) {
-        ++Changes;
-        SlotIndex Start = LIS.getMBBStartIdx(MBB);
-        unsigned RegIdx = RegAssign.lookup(Start);
-        LiveInterval *LI = Edit->get(RegIdx);
-        VNInfo *VNI = LI->getNextValue(Start, 0, LIS.getVNInfoAllocator());
-        VNI->setIsPHIDef(true);
-        I->Value = VNI;
-        // This block is done, we know the final value.
-        I->DomNode = 0;
-        if (I->Kill.isValid())
-          LI->addRange(LiveRange(Start, I->Kill, VNI));
-        else {
-          LI->addRange(LiveRange(Start, LIS.getMBBEndIdx(MBB), VNI));
-          LOP = LiveOutPair(VNI, Node);
-        }
-      } else if (IDomValue.first) {
-        // No phi-def here. Remember incoming value.
-        I->Value = IDomValue.first;
-        if (I->Kill.isValid())
-          continue;
-        // Propagate IDomValue if needed:
-        // MBB is live-out and doesn't define its own value.
-        if (LOP.second != Node && LOP.first != IDomValue.first) {
-          ++Changes;
-          LOP = IDomValue;
-        }
-      }
-    }
-  } while (Changes);
-
-  // The values in LiveInBlocks are now accurate. No more phi-defs are needed
-  // for these blocks, so we can color the live ranges.
-  for (SmallVectorImpl<LiveInBlock>::iterator I = LiveInBlocks.begin(),
-         E = LiveInBlocks.end(); I != E; ++I) {
-    if (!I->DomNode)
-      continue;
-    assert(I->Value && "No live-in value found");
-    MachineBasicBlock *MBB = I->DomNode->getBlock();
-    SlotIndex Start = LIS.getMBBStartIdx(MBB);
-    unsigned RegIdx = RegAssign.lookup(Start);
-    LiveInterval *LI = Edit->get(RegIdx);
-    LI->addRange(LiveRange(Start, I->Kill.isValid() ?
-                                  I->Kill : LIS.getMBBEndIdx(MBB), I->Value));
-  }
+  // Mark as complex mapped, forced.
+  VFP = ValueForcePair(0, true);
 }
 
 VNInfo *SplitEditor::defFromParent(unsigned RegIdx,
@@ -710,17 +520,28 @@ SlotIndex SplitEditor::leaveIntvAfter(SlotIndex Idx) {
   DEBUG(dbgs() << "    leaveIntvAfter " << Idx);
 
   // The interval must be live beyond the instruction at Idx.
-  Idx = Idx.getBoundaryIndex();
-  VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
+  SlotIndex Boundary = Idx.getBoundaryIndex();
+  VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Boundary);
   if (!ParentVNI) {
     DEBUG(dbgs() << ": not live\n");
-    return Idx.getNextSlot();
+    return Boundary.getNextSlot();
   }
   DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
-
-  MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
+  MachineInstr *MI = LIS.getInstructionFromIndex(Boundary);
   assert(MI && "No instruction at index");
-  VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(),
+
+  // In spill mode, make live ranges as short as possible by inserting the copy
+  // before MI.  This is only possible if that instruction doesn't redefine the
+  // value.  The inserted COPY is not a kill, and we don't need to recompute
+  // the source live range.  The spiller also won't try to hoist this copy.
+  if (SpillMode && !SlotIndex::isSameInstr(ParentVNI->def, Idx) &&
+      MI->readsVirtualRegister(Edit->getReg())) {
+    forceRecompute(0, ParentVNI);
+    defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI);
+    return Idx;
+  }
+
+  VNInfo *VNI = defFromParent(0, ParentVNI, Boundary, *MI->getParent(),
                               llvm::next(MachineBasicBlock::iterator(MI)));
   return VNI->def;
 }
@@ -730,7 +551,7 @@ SlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) {
   DEBUG(dbgs() << "    leaveIntvBefore " << Idx);
 
   // The interval must be live into the instruction at Idx.
-  Idx = Idx.getBoundaryIndex();
+  Idx = Idx.getBaseIndex();
   VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
   if (!ParentVNI) {
     DEBUG(dbgs() << ": not live\n");
@@ -770,19 +591,219 @@ void SplitEditor::overlapIntv(SlotIndex Start, SlotIndex End) {
   assert(LIS.getMBBFromIndex(Start) == LIS.getMBBFromIndex(End) &&
          "Range cannot span basic blocks");
 
-  // The complement interval will be extended as needed by extendRange().
+  // The complement interval will be extended as needed by LRCalc.extend().
   if (ParentVNI)
-    markComplexMapped(0, ParentVNI);
+    forceRecompute(0, ParentVNI);
   DEBUG(dbgs() << "    overlapIntv [" << Start << ';' << End << "):");
   RegAssign.insert(Start, End, OpenIdx);
   DEBUG(dump());
 }
 
+//===----------------------------------------------------------------------===//
+//                                  Spill modes
+//===----------------------------------------------------------------------===//
+
+void SplitEditor::removeBackCopies(SmallVectorImpl<VNInfo*> &Copies) {
+  LiveInterval *LI = Edit->get(0);
+  DEBUG(dbgs() << "Removing " << Copies.size() << " back-copies.\n");
+  RegAssignMap::iterator AssignI;
+  AssignI.setMap(RegAssign);
+
+  for (unsigned i = 0, e = Copies.size(); i != e; ++i) {
+    VNInfo *VNI = Copies[i];
+    SlotIndex Def = VNI->def;
+    MachineInstr *MI = LIS.getInstructionFromIndex(Def);
+    assert(MI && "No instruction for back-copy");
+
+    MachineBasicBlock *MBB = MI->getParent();
+    MachineBasicBlock::iterator MBBI(MI);
+    bool AtBegin;
+    do AtBegin = MBBI == MBB->begin();
+    while (!AtBegin && (--MBBI)->isDebugValue());
+
+    DEBUG(dbgs() << "Removing " << Def << '\t' << *MI);
+    LI->removeValNo(VNI);
+    LIS.RemoveMachineInstrFromMaps(MI);
+    MI->eraseFromParent();
+
+    // Adjust RegAssign if a register assignment is killed at VNI->def.  We
+    // want to avoid calculating the live range of the source register if
+    // possible.
+    AssignI.find(VNI->def.getPrevSlot());
+    if (!AssignI.valid() || AssignI.start() >= Def)
+      continue;
+    // If MI doesn't kill the assigned register, just leave it.
+    if (AssignI.stop() != Def)
+      continue;
+    unsigned RegIdx = AssignI.value();
+    if (AtBegin || !MBBI->readsVirtualRegister(Edit->getReg())) {
+      DEBUG(dbgs() << "  cannot find simple kill of RegIdx " << RegIdx << '\n');
+      forceRecompute(RegIdx, Edit->getParent().getVNInfoAt(Def));
+    } else {
+      SlotIndex Kill = LIS.getInstructionIndex(MBBI).getDefIndex();
+      DEBUG(dbgs() << "  move kill to " << Kill << '\t' << *MBBI);
+      AssignI.setStop(Kill);
+    }
+  }
+}
+
+MachineBasicBlock*
+SplitEditor::findShallowDominator(MachineBasicBlock *MBB,
+                                  MachineBasicBlock *DefMBB) {
+  if (MBB == DefMBB)
+    return MBB;
+  assert(MDT.dominates(DefMBB, MBB) && "MBB must be dominated by the def.");
+
+  const MachineLoopInfo &Loops = SA.Loops;
+  const MachineLoop *DefLoop = Loops.getLoopFor(DefMBB);
+  MachineDomTreeNode *DefDomNode = MDT[DefMBB];
+
+  // Best candidate so far.
+  MachineBasicBlock *BestMBB = MBB;
+  unsigned BestDepth = UINT_MAX;
+
+  for (;;) {
+    const MachineLoop *Loop = Loops.getLoopFor(MBB);
+
+    // MBB isn't in a loop, it doesn't get any better.  All dominators have a
+    // higher frequency by definition.
+    if (!Loop) {
+      DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#"
+                   << MBB->getNumber() << " at depth 0\n");
+      return MBB;
+    }
+
+    // We'll never be able to exit the DefLoop.
+    if (Loop == DefLoop) {
+      DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#"
+                   << MBB->getNumber() << " in the same loop\n");
+      return MBB;
+    }
+
+    // Least busy dominator seen so far.
+    unsigned Depth = Loop->getLoopDepth();
+    if (Depth < BestDepth) {
+      BestMBB = MBB;
+      BestDepth = Depth;
+      DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#"
+                   << MBB->getNumber() << " at depth " << Depth << '\n');
+    }
+
+    // Leave loop by going to the immediate dominator of the loop header.
+    // This is a bigger stride than simply walking up the dominator tree.
+    MachineDomTreeNode *IDom = MDT[Loop->getHeader()]->getIDom();
+
+    // Too far up the dominator tree?
+    if (!IDom || !MDT.dominates(DefDomNode, IDom))
+      return BestMBB;
+
+    MBB = IDom->getBlock();
+  }
+}
+
+void SplitEditor::hoistCopiesForSize() {
+  // Get the complement interval, always RegIdx 0.
+  LiveInterval *LI = Edit->get(0);
+  LiveInterval *Parent = &Edit->getParent();
+
+  // Track the nearest common dominator for all back-copies for each ParentVNI,
+  // indexed by ParentVNI->id.
+  typedef std::pair<MachineBasicBlock*, SlotIndex> DomPair;
+  SmallVector<DomPair, 8> NearestDom(Parent->getNumValNums());
+
+  // Find the nearest common dominator for parent values with multiple
+  // back-copies.  If a single back-copy dominates, put it in DomPair.second.
+  for (LiveInterval::vni_iterator VI = LI->vni_begin(), VE = LI->vni_end();
+       VI != VE; ++VI) {
+    VNInfo *VNI = *VI;
+    VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
+    assert(ParentVNI && "Parent not live at complement def");
+
+    // Don't hoist remats.  The complement is probably going to disappear
+    // completely anyway.
+    if (Edit->didRematerialize(ParentVNI))
+      continue;
+
+    MachineBasicBlock *ValMBB = LIS.getMBBFromIndex(VNI->def);
+    DomPair &Dom = NearestDom[ParentVNI->id];
+
+    // Keep directly defined parent values.  This is either a PHI or an
+    // instruction in the complement range.  All other copies of ParentVNI
+    // should be eliminated.
+    if (VNI->def == ParentVNI->def) {
+      DEBUG(dbgs() << "Direct complement def at " << VNI->def << '\n');
+      Dom = DomPair(ValMBB, VNI->def);
+      continue;
+    }
+    // Skip the singly mapped values.  There is nothing to gain from hoisting a
+    // single back-copy.
+    if (Values.lookup(std::make_pair(0, ParentVNI->id)).getPointer()) {
+      DEBUG(dbgs() << "Single complement def at " << VNI->def << '\n');
+      continue;
+    }
+
+    if (!Dom.first) {
+      // First time we see ParentVNI.  VNI dominates itself.
+      Dom = DomPair(ValMBB, VNI->def);
+    } else if (Dom.first == ValMBB) {
+      // Two defs in the same block.  Pick the earlier def.
+      if (!Dom.second.isValid() || VNI->def < Dom.second)
+        Dom.second = VNI->def;
+    } else {
+      // Different basic blocks. Check if one dominates.
+      MachineBasicBlock *Near =
+        MDT.findNearestCommonDominator(Dom.first, ValMBB);
+      if (Near == ValMBB)
+        // Def ValMBB dominates.
+        Dom = DomPair(ValMBB, VNI->def);
+      else if (Near != Dom.first)
+        // None dominate. Hoist to common dominator, need new def.
+        Dom = DomPair(Near, SlotIndex());
+    }
+
+    DEBUG(dbgs() << "Multi-mapped complement " << VNI->id << '@' << VNI->def
+                 << " for parent " << ParentVNI->id << '@' << ParentVNI->def
+                 << " hoist to BB#" << Dom.first->getNumber() << ' '
+                 << Dom.second << '\n');
+  }
+
+  // Insert the hoisted copies.
+  for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) {
+    DomPair &Dom = NearestDom[i];
+    if (!Dom.first || Dom.second.isValid())
+      continue;
+    // This value needs a hoisted copy inserted at the end of Dom.first.
+    VNInfo *ParentVNI = Parent->getValNumInfo(i);
+    MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(ParentVNI->def);
+    // Get a less loopy dominator than Dom.first.
+    Dom.first = findShallowDominator(Dom.first, DefMBB);
+    SlotIndex Last = LIS.getMBBEndIdx(Dom.first).getPrevSlot();
+    Dom.second =
+      defFromParent(0, ParentVNI, Last, *Dom.first,
+                    LIS.getLastSplitPoint(Edit->getParent(), Dom.first))->def;
+  }
+
+  // Remove redundant back-copies that are now known to be dominated by another
+  // def with the same value.
+  SmallVector<VNInfo*, 8> BackCopies;
+  for (LiveInterval::vni_iterator VI = LI->vni_begin(), VE = LI->vni_end();
+       VI != VE; ++VI) {
+    VNInfo *VNI = *VI;
+    VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
+    const DomPair &Dom = NearestDom[ParentVNI->id];
+    if (!Dom.first || Dom.second == VNI->def)
+      continue;
+    BackCopies.push_back(VNI);
+    forceRecompute(0, ParentVNI);
+  }
+  removeBackCopies(BackCopies);
+}
+
+
 /// transferValues - Transfer all possible values to the new live ranges.
-/// Values that were rematerialized are left alone, they need extendRange().
+/// Values that were rematerialized are left alone, they need LRCalc.extend().
 bool SplitEditor::transferValues() {
   bool Skipped = false;
-  LiveInBlocks.clear();
   RegAssignMap::const_iterator AssignI = RegAssign.begin();
   for (LiveInterval::const_iterator ParentI = Edit->getParent().begin(),
          ParentE = Edit->getParent().end(); ParentI != ParentE; ++ParentI) {
@@ -812,28 +833,23 @@ bool SplitEditor::transferValues() {
       LiveInterval *LI = Edit->get(RegIdx);
 
       // Check for a simply defined value that can be blitted directly.
-      if (VNInfo *VNI = Values.lookup(std::make_pair(RegIdx, ParentVNI->id))) {
+      ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id));
+      if (VNInfo *VNI = VFP.getPointer()) {
         DEBUG(dbgs() << ':' << VNI->id);
         LI->addRange(LiveRange(Start, End, VNI));
         Start = End;
         continue;
       }
 
-      // Skip rematerialized values, we need to use extendRange() and
-      // extendPHIKillRanges() to completely recompute the live ranges.
-      if (Edit->didRematerialize(ParentVNI)) {
-        DEBUG(dbgs() << "(remat)");
+      // Skip values with forced recomputation.
+      if (VFP.getInt()) {
+        DEBUG(dbgs() << "(recalc)");
         Skipped = true;
         Start = End;
         continue;
       }
 
-      // Initialize the live-out cache the first time it is needed.
-      if (LiveOutSeen.empty()) {
-        unsigned N = VRM.getMachineFunction().getNumBlockIDs();
-        LiveOutSeen.resize(N);
-        LiveOutCache.resize(N);
-      }
+      LiveRangeCalc &LRC = getLRCalc(RegIdx);
 
       // This value has multiple defs in RegIdx, but it wasn't rematerialized,
       // so the live range is accurate. Add live-in blocks in [Start;End) to the
@@ -844,15 +860,13 @@ bool SplitEditor::transferValues() {
 
       // The first block may be live-in, or it may have its own def.
       if (Start != BlockStart) {
-        VNInfo *VNI = LI->extendInBlock(BlockStart,
-                                        std::min(BlockEnd, End).getPrevSlot());
+        VNInfo *VNI = LI->extendInBlock(BlockStart, std::min(BlockEnd, End));
         assert(VNI && "Missing def for complex mapped value");
         DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber());
         // MBB has its own def. Is it also live-out?
-        if (BlockEnd <= End) {
-          LiveOutSeen.set(MBB->getNumber());
-          LiveOutCache[MBB] = LiveOutPair(VNI, MDT[MBB]);
-        }
+        if (BlockEnd <= End)
+          LRC.setLiveOutValue(MBB, VNI);
+
         // Skip to the next block for live-in.
         ++MBB;
         BlockStart = BlockEnd;
@@ -866,25 +880,19 @@ bool SplitEditor::transferValues() {
         if (BlockStart == ParentVNI->def) {
           // This block has the def of a parent PHI, so it isn't live-in.
           assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?");
-          VNInfo *VNI = LI->extendInBlock(BlockStart,
-                                         std::min(BlockEnd, End).getPrevSlot());
+          VNInfo *VNI = LI->extendInBlock(BlockStart, std::min(BlockEnd, End));
           assert(VNI && "Missing def for complex mapped parent PHI");
-          if (End >= BlockEnd) {
-            // Live-out as well.
-            LiveOutSeen.set(MBB->getNumber());
-            LiveOutCache[MBB] = LiveOutPair(VNI, MDT[MBB]);
-          }
+          if (End >= BlockEnd)
+            LRC.setLiveOutValue(MBB, VNI); // Live-out as well.
         } else {
-          // This block needs a live-in value.
-          LiveInBlocks.push_back(MDT[MBB]);
-          // The last block covered may not be live-out.
+          // This block needs a live-in value.  The last block covered may not
+          // be live-out.
           if (End < BlockEnd)
-            LiveInBlocks.back().Kill = End;
+            LRC.addLiveInBlock(LI, MDT[MBB], End);
           else {
-            // Live-out, but we need updateSSA to tell us the value.
-            LiveOutSeen.set(MBB->getNumber());
-            LiveOutCache[MBB] = LiveOutPair((VNInfo*)0,
-                                            (MachineDomTreeNode*)0);
+            // Live-through, and we don't know the value.
+            LRC.addLiveInBlock(LI, MDT[MBB]);
+            LRC.setLiveOutValue(MBB, 0);
           }
         }
         BlockStart = BlockEnd;
@@ -895,8 +903,11 @@ bool SplitEditor::transferValues() {
     DEBUG(dbgs() << '\n');
   }
 
-  if (!LiveInBlocks.empty())
-    updateSSA();
+  LRCalc[0].calculateValues(LIS.getSlotIndexes(), &MDT,
+                            &LIS.getVNInfoAllocator());
+  if (SpillMode)
+    LRCalc[1].calculateValues(LIS.getSlotIndexes(), &MDT,
+                              &LIS.getVNInfoAllocator());
 
   return Skipped;
 }
@@ -909,16 +920,20 @@ void SplitEditor::extendPHIKillRanges() {
     if (PHIVNI->isUnused() || !PHIVNI->isPHIDef())
       continue;
     unsigned RegIdx = RegAssign.lookup(PHIVNI->def);
+    LiveInterval *LI = Edit->get(RegIdx);
+    LiveRangeCalc &LRC = getLRCalc(RegIdx);
     MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def);
     for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
          PE = MBB->pred_end(); PI != PE; ++PI) {
-      SlotIndex End = LIS.getMBBEndIdx(*PI).getPrevSlot();
+      SlotIndex End = LIS.getMBBEndIdx(*PI);
+      SlotIndex LastUse = End.getPrevSlot();
       // The predecessor may not have a live-out value. That is OK, like an
       // undef PHI operand.
-      if (Edit->getParent().liveAt(End)) {
-        assert(RegAssign.lookup(End) == RegIdx &&
+      if (Edit->getParent().liveAt(LastUse)) {
+        assert(RegAssign.lookup(LastUse) == RegIdx &&
                "Different register assignment in phi predecessor");
-        extendRange(RegIdx, End);
+        LRC.extend(LI, End,
+                   LIS.getSlotIndexes(), &MDT, &LIS.getVNInfoAllocator());
       }
     }
   }
@@ -938,25 +953,22 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) {
       continue;
     }
 
-    // <undef> operands don't really read the register, so just assign them to
-    // the complement.
-    if (MO.isUse() && MO.isUndef()) {
-      MO.setReg(Edit->get(0)->reg);
-      continue;
-    }
-
+    // <undef> operands don't really read the register, so it doesn't matter
+    // which register we choose.  When the use operand is tied to a def, we must
+    // use the same register as the def, so just do that always.
     SlotIndex Idx = LIS.getInstructionIndex(MI);
-    if (MO.isDef())
+    if (MO.isDef() || MO.isUndef())
       Idx = MO.isEarlyClobber() ? Idx.getUseIndex() : Idx.getDefIndex();
 
     // Rewrite to the mapped register at Idx.
     unsigned RegIdx = RegAssign.lookup(Idx);
-    MO.setReg(Edit->get(RegIdx)->reg);
+    LiveInterval *LI = Edit->get(RegIdx);
+    MO.setReg(LI->reg);
     DEBUG(dbgs() << "  rewr BB#" << MI->getParent()->getNumber() << '\t'
                  << Idx << ':' << RegIdx << '\t' << *MI);
 
     // Extend liveness to Idx if the instruction reads reg.
-    if (!ExtendRanges)
+    if (!ExtendRanges || MO.isUndef())
       continue;
 
     // Skip instructions that don't read Reg.
@@ -971,7 +983,8 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) {
     } else
       Idx = Idx.getUseIndex();
 
-    extendRange(RegIdx, Idx);
+    getLRCalc(RegIdx).extend(LI, Idx.getNextSlot(), LIS.getSlotIndexes(),
+                             &MDT, &LIS.getVNInfoAllocator());
   }
 }
 
@@ -1019,11 +1032,24 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
     VNI->setIsPHIDef(ParentVNI->isPHIDef());
     VNI->setCopy(ParentVNI->getCopy());
 
-    // Mark rematted values as complex everywhere to force liveness computation.
+    // Force rematted values to be recomputed everywhere.
     // The new live ranges may be truncated.
     if (Edit->didRematerialize(ParentVNI))
       for (unsigned i = 0, e = Edit->size(); i != e; ++i)
-        markComplexMapped(i, ParentVNI);
+        forceRecompute(i, ParentVNI);
+  }
+
+  // Hoist back-copies to the complement interval when in spill mode.
+  switch (SpillMode) {
+  case SM_Partition:
+    // Leave all back-copies as is.
+    break;
+  case SM_Size:
+    hoistCopiesForSize();
+    break;
+  case SM_Speed:
+    llvm_unreachable("Spill mode 'speed' not implemented yet");
+    break;
   }
 
   // Transfer the simply mapped values, check if any are skipped.
@@ -1081,50 +1107,39 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
 //                            Single Block Splitting
 //===----------------------------------------------------------------------===//
 
-/// getMultiUseBlocks - if CurLI has more than one use in a basic block, it
-/// may be an advantage to split CurLI for the duration of the block.
-bool SplitAnalysis::getMultiUseBlocks(BlockPtrSet &Blocks) {
-  // If CurLI is local to one block, there is no point to splitting it.
-  if (UseBlocks.size() <= 1)
+bool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI,
+                                           bool SingleInstrs) const {
+  // Always split for multiple instructions.
+  if (!BI.isOneInstr())
+    return true;
+  // Don't split for single instructions unless explicitly requested.
+  if (!SingleInstrs)
     return false;
-  // Add blocks with multiple uses.
-  for (unsigned i = 0, e = UseBlocks.size(); i != e; ++i) {
-    const BlockInfo &BI = UseBlocks[i];
-    if (BI.FirstUse == BI.LastUse)
-      continue;
-    Blocks.insert(BI.MBB);
-  }
-  return !Blocks.empty();
+  // Splitting a live-through range always makes progress.
+  if (BI.LiveIn && BI.LiveOut)
+    return true;
+  // No point in isolating a copy. It has no register class constraints.
+  if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike())
+    return false;
+  // Finally, don't isolate an end point that was created by earlier splits.
+  return isOriginalEndpoint(BI.FirstInstr);
 }
 
 void SplitEditor::splitSingleBlock(const SplitAnalysis::BlockInfo &BI) {
   openIntv();
   SlotIndex LastSplitPoint = SA.getLastSplitPoint(BI.MBB->getNumber());
-  SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstUse,
+  SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstInstr,
     LastSplitPoint));
-  if (!BI.LiveOut || BI.LastUse < LastSplitPoint) {
-    useIntv(SegStart, leaveIntvAfter(BI.LastUse));
+  if (!BI.LiveOut || BI.LastInstr < LastSplitPoint) {
+    useIntv(SegStart, leaveIntvAfter(BI.LastInstr));
   } else {
       // The last use is after the last valid split point.
     SlotIndex SegStop = leaveIntvBefore(LastSplitPoint);
     useIntv(SegStart, SegStop);
-    overlapIntv(SegStop, BI.LastUse);
+    overlapIntv(SegStop, BI.LastInstr);
   }
 }
 
-/// splitSingleBlocks - Split CurLI into a separate live interval inside each
-/// basic block in Blocks.
-void SplitEditor::splitSingleBlocks(const SplitAnalysis::BlockPtrSet &Blocks) {
-  DEBUG(dbgs() << "  splitSingleBlocks for " << Blocks.size() << " blocks.\n");
-  ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA.getUseBlocks();
-  for (unsigned i = 0; i != UseBlocks.size(); ++i) {
-    const SplitAnalysis::BlockInfo &BI = UseBlocks[i];
-    if (Blocks.count(BI.MBB))
-      splitSingleBlock(BI);
-  }
-  finish();
-}
-
 
 //===----------------------------------------------------------------------===//
 //                    Global Live Range Splitting Support
@@ -1149,6 +1164,12 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
 
   assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks");
 
+  assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block");
+  assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf");
+  assert((!EnterAfter || EnterAfter >= Start) && "Interference before block");
+
+  MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum);
+
   if (!IntvOut) {
     DEBUG(dbgs() << ", spill on entry.\n");
     //
@@ -1157,7 +1178,6 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
     //    -____________    Spill on entry.
     //
     selectIntv(IntvIn);
-    MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum);
     SlotIndex Idx = leaveIntvAtTop(*MBB);
     assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
     (void)Idx;
@@ -1172,7 +1192,6 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
     //    ___________--    Reload on exit.
     //
     selectIntv(IntvOut);
-    MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum);
     SlotIndex Idx = enterIntvAtEnd(*MBB);
     assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
     (void)Idx;
@@ -1192,6 +1211,7 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
 
   // We cannot legally insert splits after LSP.
   SlotIndex LSP = SA.getLastSplitPoint(MBBNum);
+  assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf");
 
   if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter ||
                   LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) {
@@ -1201,10 +1221,14 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
     //    |-----------|    Live through.
     //    ------=======    Switch intervals between interference.
     //
-    SlotIndex Cut = (LeaveBefore && LeaveBefore < LSP) ? LeaveBefore : LSP;
     selectIntv(IntvOut);
-    SlotIndex Idx = enterIntvBefore(Cut);
-    useIntv(Idx, Stop);
+    SlotIndex Idx;
+    if (LeaveBefore && LeaveBefore < LSP) {
+      Idx = enterIntvBefore(LeaveBefore);
+      useIntv(Idx, Stop);
+    } else {
+      Idx = enterIntvAtEnd(*MBB);
+    }
     selectIntv(IntvIn);
     useIntv(Start, Idx);
     assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
@@ -1238,7 +1262,7 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
   tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
 
   DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop
-               << "), uses " << BI.FirstUse << '-' << BI.LastUse
+               << "), uses " << BI.FirstInstr << '-' << BI.LastInstr
                << ", reg-in " << IntvIn << ", leave before " << LeaveBefore
                << (BI.LiveOut ? ", stack-out" : ", killed in block"));
 
@@ -1246,7 +1270,7 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
   assert(BI.LiveIn && "Must be live-in");
   assert((!LeaveBefore || LeaveBefore > Start) && "Bad interference");
 
-  if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastUse)) {
+  if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastInstr)) {
     DEBUG(dbgs() << " before interference.\n");
     //
     //               <<<    Interference after kill.
@@ -1254,13 +1278,13 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
     //     =========        Use IntvIn everywhere.
     //
     selectIntv(IntvIn);
-    useIntv(Start, BI.LastUse);
+    useIntv(Start, BI.LastInstr);
     return;
   }
 
   SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber());
 
-  if (!LeaveBefore || LeaveBefore > BI.LastUse.getBoundaryIndex()) {
+  if (!LeaveBefore || LeaveBefore > BI.LastInstr.getBoundaryIndex()) {
     //
     //               <<<    Possible interference after last use.
     //     |---o---o---|    Live-out on stack.
@@ -1271,17 +1295,17 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
     //     ============     Copy to stack after LSP, overlap IntvIn.
     //            \_____    Stack interval is live-out.
     //
-    if (BI.LastUse < LSP) {
+    if (BI.LastInstr < LSP) {
       DEBUG(dbgs() << ", spill after last use before interference.\n");
       selectIntv(IntvIn);
-      SlotIndex Idx = leaveIntvAfter(BI.LastUse);
+      SlotIndex Idx = leaveIntvAfter(BI.LastInstr);
       useIntv(Start, Idx);
       assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
     } else {
       DEBUG(dbgs() << ", spill before last split point.\n");
       selectIntv(IntvIn);
       SlotIndex Idx = leaveIntvBefore(LSP);
-      overlapIntv(Idx, BI.LastUse);
+      overlapIntv(Idx, BI.LastInstr);
       useIntv(Start, Idx);
       assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
     }
@@ -1295,13 +1319,13 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
   (void)LocalIntv;
   DEBUG(dbgs() << ", creating local interval " << LocalIntv << ".\n");
 
-  if (!BI.LiveOut || BI.LastUse < LSP) {
+  if (!BI.LiveOut || BI.LastInstr < LSP) {
     //
     //           <<<<<<<    Interference overlapping uses.
     //     |---o---o---|    Live-out on stack.
     //     =====----____    Leave IntvIn before interference, then spill.
     //
-    SlotIndex To = leaveIntvAfter(BI.LastUse);
+    SlotIndex To = leaveIntvAfter(BI.LastInstr);
     SlotIndex From = enterIntvBefore(LeaveBefore);
     useIntv(From, To);
     selectIntv(IntvIn);
@@ -1316,7 +1340,7 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
   //            \_____    Stack interval is live-out.
   //
   SlotIndex To = leaveIntvBefore(LSP);
-  overlapIntv(To, BI.LastUse);
+  overlapIntv(To, BI.LastInstr);
   SlotIndex From = enterIntvBefore(std::min(To, LeaveBefore));
   useIntv(From, To);
   selectIntv(IntvIn);
@@ -1330,7 +1354,7 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
   tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
 
   DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop
-               << "), uses " << BI.FirstUse << '-' << BI.LastUse
+               << "), uses " << BI.FirstInstr << '-' << BI.LastInstr
                << ", reg-out " << IntvOut << ", enter after " << EnterAfter
                << (BI.LiveIn ? ", stack-in" : ", defined in block"));
 
@@ -1340,7 +1364,7 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
   assert(BI.LiveOut && "Must be live-out");
   assert((!EnterAfter || EnterAfter < LSP) && "Bad interference");
 
-  if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstUse)) {
+  if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstInstr)) {
     DEBUG(dbgs() << " after interference.\n");
     //
     //    >>>>             Interference before def.
@@ -1348,11 +1372,11 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
     //        =========    Use IntvOut everywhere.
     //
     selectIntv(IntvOut);
-    useIntv(BI.FirstUse, Stop);
+    useIntv(BI.FirstInstr, Stop);
     return;
   }
 
-  if (!EnterAfter || EnterAfter < BI.FirstUse.getBaseIndex()) {
+  if (!EnterAfter || EnterAfter < BI.FirstInstr.getBaseIndex()) {
     DEBUG(dbgs() << ", reload after interference.\n");
     //
     //    >>>>             Interference before def.
@@ -1360,7 +1384,7 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
     //    ____=========    Enter IntvOut before first use.
     //
     selectIntv(IntvOut);
-    SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstUse));
+    SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstInstr));
     useIntv(Idx, Stop);
     assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
     return;
@@ -1381,6 +1405,6 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
   assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
 
   openIntv();
-  SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstUse));
+  SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstInstr));
   useIntv(From, Idx);
 }
diff --git a/contrib/llvm/lib/CodeGen/SplitKit.h b/contrib/llvm/lib/CodeGen/SplitKit.h
index 7948b725f856..d8fc2122a3c7 100644
--- a/contrib/llvm/lib/CodeGen/SplitKit.h
+++ b/contrib/llvm/lib/CodeGen/SplitKit.h
@@ -15,13 +15,11 @@
 #ifndef LLVM_CODEGEN_SPLITKIT_H
 #define LLVM_CODEGEN_SPLITKIT_H
 
+#include "LiveRangeCalc.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/IntervalMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/CodeGen/SlotIndexes.h"
 
 namespace llvm {
 
@@ -38,12 +36,6 @@ class VirtRegMap;
 class VNInfo;
 class raw_ostream;
 
-/// At some point we should just include MachineDominators.h:
-class MachineDominatorTree;
-template <class NodeT> class DomTreeNodeBase;
-typedef DomTreeNodeBase<MachineBasicBlock> MachineDomTreeNode;
-
-
 /// SplitAnalysis - Analyze a LiveInterval, looking for live range splitting
 /// opportunities.
 class SplitAnalysis {
@@ -76,16 +68,16 @@ class SplitAnalysis {
   ///
   struct BlockInfo {
     MachineBasicBlock *MBB;
-    SlotIndex FirstUse;   ///< First instr using current reg.
-    SlotIndex LastUse;    ///< Last instr using current reg.
-    bool LiveThrough;     ///< Live in whole block (Templ 5. above).
+    SlotIndex FirstInstr; ///< First instr accessing current reg.
+    SlotIndex LastInstr;  ///< Last instr accessing current reg.
+    SlotIndex FirstDef;   ///< First non-phi valno->def, or SlotIndex().
     bool LiveIn;          ///< Current reg is live in.
     bool LiveOut;         ///< Current reg is live out.
 
     /// isOneInstr - Returns true when this BlockInfo describes a single
     /// instruction.
     bool isOneInstr() const {
-      return SlotIndex::isSameInstr(FirstUse, LastUse);
+      return SlotIndex::isSameInstr(FirstInstr, LastInstr);
     }
   };
 
@@ -185,10 +177,15 @@ class SplitAnalysis {
 
   typedef SmallPtrSet<const MachineBasicBlock*, 16> BlockPtrSet;
 
-  /// getMultiUseBlocks - Add basic blocks to Blocks that may benefit from
-  /// having CurLI split to a new live interval. Return true if Blocks can be
-  /// passed to SplitEditor::splitSingleBlocks.
-  bool getMultiUseBlocks(BlockPtrSet &Blocks);
+  /// shouldSplitSingleBlock - Returns true if it would help to create a local
+  /// live range for the instructions in BI. There is normally no benefit to
+  /// creating a live range for a single instruction, but it does enable
+  /// register class inflation if the instruction has a restricted register
+  /// class.
+  ///
+  /// @param BI           The block to be isolated.
+  /// @param SingleInstrs True when single instructions should be isolated.
+  bool shouldSplitSingleBlock(const BlockInfo &BI, bool SingleInstrs) const;
 };
 
 
@@ -212,6 +209,36 @@ class SplitEditor {
   const TargetInstrInfo &TII;
   const TargetRegisterInfo &TRI;
 
+public:
+
+  /// ComplementSpillMode - Select how the complement live range should be
+  /// created.  SplitEditor automatically creates interval 0 to contain
+  /// anything that isn't added to another interval.  This complement interval
+  /// can get quite complicated, and it can sometimes be an advantage to allow
+  /// it to overlap the other intervals.  If it is going to spill anyway, no
+  /// registers are wasted by keeping a value in two places at the same time.
+  enum ComplementSpillMode {
+    /// SM_Partition(Default) - Try to create the complement interval so it
+    /// doesn't overlap any other intervals, and the original interval is
+    /// partitioned.  This may require a large number of back copies and extra
+    /// PHI-defs.  Only segments marked with overlapIntv will be overlapping.
+    SM_Partition,
+
+    /// SM_Size - Overlap intervals to minimize the number of inserted COPY
+    /// instructions.  Copies to the complement interval are hoisted to their
+    /// common dominator, so only one COPY is required per value in the
+    /// complement interval.  This also means that no extra PHI-defs need to be
+    /// inserted in the complement interval.
+    SM_Size,
+
+    /// SM_Speed - Overlap intervals to minimize the expected execution
+    /// frequency of the inserted copies.  This is very similar to SM_Size, but
+    /// the complement interval may get some extra PHI-defs.
+    SM_Speed
+  };
+
+private:
+
   /// Edit - The current parent register and new intervals created.
   LiveRangeEdit *Edit;
 
@@ -220,6 +247,9 @@ class SplitEditor {
   /// openIntv will be 1.
   unsigned OpenIdx;
 
+  /// The current spill mode, selected by reset().
+  ComplementSpillMode SpillMode;
+
   typedef IntervalMap<SlotIndex, unsigned> RegAssignMap;
 
   /// Allocator for the interval map. This will eventually be shared with
@@ -231,65 +261,34 @@ class SplitEditor {
   /// Idx.
   RegAssignMap RegAssign;
 
-  typedef DenseMap<std::pair<unsigned, unsigned>, VNInfo*> ValueMap;
+  typedef PointerIntPair<VNInfo*, 1> ValueForcePair;
+  typedef DenseMap<std::pair<unsigned, unsigned>, ValueForcePair> ValueMap;
 
   /// Values - keep track of the mapping from parent values to values in the new
   /// intervals. Given a pair (RegIdx, ParentVNI->id), Values contains:
   ///
   /// 1. No entry - the value is not mapped to Edit.get(RegIdx).
-  /// 2. Null - the value is mapped to multiple values in Edit.get(RegIdx).
-  ///    Each value is represented by a minimal live range at its def.
-  /// 3. A non-null VNInfo - the value is mapped to a single new value.
+  /// 2. (Null, false) - the value is mapped to multiple values in
+  ///    Edit.get(RegIdx).  Each value is represented by a minimal live range at
+  ///    its def.  The full live range can be inferred exactly from the range
+  ///    of RegIdx in RegAssign.
+  /// 3. (Null, true).  As above, but the ranges in RegAssign are too large, and
+  ///    the live range must be recomputed using LiveRangeCalc::extend().
+  /// 4. (VNI, false) The value is mapped to a single new value.
   ///    The new value has no live ranges anywhere.
   ValueMap Values;
 
-  typedef std::pair<VNInfo*, MachineDomTreeNode*> LiveOutPair;
-  typedef IndexedMap<LiveOutPair, MBB2NumberFunctor> LiveOutMap;
+  /// LRCalc - Cache for computing live ranges and SSA update.  Each instance
+  /// can only handle non-overlapping live ranges, so use a separate
+  /// LiveRangeCalc instance for the complement interval when in spill mode.
+  LiveRangeCalc LRCalc[2];
 
-  // LiveOutCache - Map each basic block where a new register is live out to the
-  // live-out value and its defining block.
-  // One of these conditions shall be true:
-  //
-  //  1. !LiveOutCache.count(MBB)
-  //  2. LiveOutCache[MBB].second.getNode() == MBB
-  //  3. forall P in preds(MBB): LiveOutCache[P] == LiveOutCache[MBB]
-  //
-  // This is only a cache, the values can be computed as:
-  //
-  //  VNI = Edit.get(RegIdx)->getVNInfoAt(LIS.getMBBEndIdx(MBB))
-  //  Node = mbt_[LIS.getMBBFromIndex(VNI->def)]
-  //
-  // The cache is also used as a visited set by extendRange(). It can be shared
-  // by all the new registers because at most one is live out of each block.
-  LiveOutMap LiveOutCache;
-
-  // LiveOutSeen - Indexed by MBB->getNumber(), a bit is set for each valid
-  // entry in LiveOutCache.
-  BitVector LiveOutSeen;
-
-  /// LiveInBlock - Info for updateSSA() about a block where a register is
-  /// live-in.
-  /// The updateSSA caller provides DomNode and Kill inside MBB, updateSSA()
-  /// adds the computed live-in value.
-  struct LiveInBlock {
-    // Dominator tree node for the block.
-    // Cleared by updateSSA when the final value has been determined.
-    MachineDomTreeNode *DomNode;
-
-    // Live-in value filled in by updateSSA once it is known.
-    VNInfo *Value;
-
-    // Position in block where the live-in range ends, or SlotIndex() if the
-    // range passes through the block.
-    SlotIndex Kill;
-
-    LiveInBlock(MachineDomTreeNode *node) : DomNode(node), Value(0) {}
-  };
-
-  /// LiveInBlocks - List of live-in blocks used by findReachingDefs() and
-  /// updateSSA(). This list is usually empty, it exists here to avoid frequent
-  /// reallocations.
-  SmallVector<LiveInBlock, 16> LiveInBlocks;
+  /// getLRCalc - Return the LRCalc to use for RegIdx.  In spill mode, the
+  /// complement interval can overlap the other intervals, so it gets its own
+  /// LRCalc instance.  When not in spill mode, all intervals can share one.
+  LiveRangeCalc &getLRCalc(unsigned RegIdx) {
+    return LRCalc[SpillMode != SM_Partition && RegIdx != 0];
+  }
 
   /// defValue - define a value in RegIdx from ParentVNI at Idx.
   /// Idx does not have to be ParentVNI->def, but it must be contained within
@@ -298,9 +297,11 @@ class SplitEditor {
   /// Return the new LI value.
   VNInfo *defValue(unsigned RegIdx, const VNInfo *ParentVNI, SlotIndex Idx);
 
-  /// markComplexMapped - Mark ParentVNI as complex mapped in RegIdx regardless
-  /// of the number of defs.
-  void markComplexMapped(unsigned RegIdx, const VNInfo *ParentVNI);
+  /// forceRecompute - Force the live range of ParentVNI in RegIdx to be
+  /// recomputed by LiveRangeCalc::extend regardless of the number of defs.
+  /// This is used for values whose live range doesn't match RegAssign exactly.
+  /// They could have rematerialized, or back-copies may have been moved.
+  void forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI);
 
   /// defFromParent - Define Reg from ParentVNI at UseIdx using either
   /// rematerialization or a COPY from parent. Return the new value.
@@ -310,22 +311,18 @@ class SplitEditor {
                         MachineBasicBlock &MBB,
                         MachineBasicBlock::iterator I);
 
-  /// extendRange - Extend the live range of Edit.get(RegIdx) so it reaches Idx.
-  /// Insert PHIDefs as needed to preserve SSA form.
-  void extendRange(unsigned RegIdx, SlotIndex Idx);
+  /// removeBackCopies - Remove the copy instructions that defines the values
+  /// in the vector in the complement interval.
+  void removeBackCopies(SmallVectorImpl<VNInfo*> &Copies);
 
-  /// findReachingDefs - Starting from MBB, add blocks to LiveInBlocks until all
-  /// reaching defs for LI are found.
-  /// @param LI   Live interval whose value is needed.
-  /// @param MBB  Block where LI should be live-in.
-  /// @param Kill Kill point in MBB.
-  /// @return Unique value seen, or NULL.
-  VNInfo *findReachingDefs(LiveInterval *LI, MachineBasicBlock *MBB,
-                           SlotIndex Kill);
+  /// getShallowDominator - Returns the least busy dominator of MBB that is
+  /// also dominated by DefMBB.  Busy is measured by loop depth.
+  MachineBasicBlock *findShallowDominator(MachineBasicBlock *MBB,
+                                          MachineBasicBlock *DefMBB);
 
-  /// updateSSA - Compute and insert PHIDefs such that all blocks in
-  // LiveInBlocks get a known live-in value. Add live ranges to the blocks.
-  void updateSSA();
+  /// hoistCopiesForSize - Hoist back-copies to the complement interval in a
+  /// way that minimizes code size. This implements the SM_Size spill mode.
+  void hoistCopiesForSize();
 
   /// transferValues - Transfer values to the new ranges.
   /// Return true if any ranges were skipped.
@@ -348,7 +345,7 @@ class SplitEditor {
               MachineDominatorTree&);
 
   /// reset - Prepare for a new split.
-  void reset(LiveRangeEdit&);
+  void reset(LiveRangeEdit&, ComplementSpillMode = SM_Partition);
 
   /// Create a new virtual register and live interval.
   /// Return the interval index, starting from 1. Interval index 0 is the
@@ -423,10 +420,6 @@ class SplitEditor {
   /// split, and doesn't call finish().
   void splitSingleBlock(const SplitAnalysis::BlockInfo &BI);
 
-  /// splitSingleBlocks - Split CurLI into a separate live interval inside each
-  /// basic block in Blocks.
-  void splitSingleBlocks(const SplitAnalysis::BlockPtrSet &Blocks);
-
   /// splitLiveThroughBlock - Split CurLI in the given block such that it
   /// enters the block in IntvIn and leaves it in IntvOut. There may be uses in
   /// the block, but they will be ignored when placing split points.
diff --git a/contrib/llvm/lib/CodeGen/Splitter.cpp b/contrib/llvm/lib/CodeGen/Splitter.cpp
index ec75df4b7d1f..77973b72bbc8 100644
--- a/contrib/llvm/lib/CodeGen/Splitter.cpp
+++ b/contrib/llvm/lib/CodeGen/Splitter.cpp
@@ -11,7 +11,6 @@
 
 #include "Splitter.h"
 
-#include "RegisterCoalescer.h"
 #include "llvm/Module.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
@@ -20,6 +19,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -262,7 +262,7 @@ namespace llvm {
     au.addPreserved<MachineDominatorTree>();
     au.addRequired<MachineLoopInfo>();
     au.addPreserved<MachineLoopInfo>();
-    au.addPreserved<RegisterCoalescer>();
+    au.addPreservedID(RegisterCoalescerPassID);
     au.addPreserved<CalculateSpillWeights>();
     au.addPreserved<LiveStacks>();
     au.addRequired<SlotIndexes>();
diff --git a/contrib/llvm/lib/CodeGen/StackProtector.cpp b/contrib/llvm/lib/CodeGen/StackProtector.cpp
index d3cbd15b64e8..1f0e5a2711ae 100644
--- a/contrib/llvm/lib/CodeGen/StackProtector.cpp
+++ b/contrib/llvm/lib/CodeGen/StackProtector.cpp
@@ -123,7 +123,7 @@ bool StackProtector::RequiresStackProtector() const {
           // protectors.
           return true;
 
-        if (const ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType())) {
+        if (ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType())) {
           // We apparently only care about character arrays.
           if (!AT->getElementType()->isIntegerTy(8))
             continue;
@@ -165,7 +165,7 @@ bool StackProtector::InsertStackProtectors() {
       //     StackGuard = load __stack_chk_guard
       //     call void @llvm.stackprotect.create(StackGuard, StackGuardSlot)
       // 
-      const PointerType *PtrTy = Type::getInt8PtrTy(RI->getContext());
+      PointerType *PtrTy = Type::getInt8PtrTy(RI->getContext());
       unsigned AddressSpace, Offset;
       if (TLI->getStackCookieLocation(AddressSpace, Offset)) {
         Constant *OffsetVal =
diff --git a/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp b/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp
index 227eb47e6827..260cc0ee50a5 100644
--- a/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp
+++ b/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp
@@ -47,6 +47,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/Debug.h"
 using namespace llvm;
@@ -673,7 +674,7 @@ void StrongPHIElimination::InsertCopiesForPHI(MachineInstr *PHI,
     if (PHIColor && SrcColor == PHIColor) {
       LiveInterval &SrcInterval = LI->getInterval(SrcReg);
       SlotIndex PredIndex = LI->getMBBEndIdx(PredBB);
-      VNInfo *SrcVNI = SrcInterval.getVNInfoAt(PredIndex.getPrevIndex());
+      VNInfo *SrcVNI = SrcInterval.getVNInfoBefore(PredIndex);
       assert(SrcVNI);
       SrcVNI->setHasPHIKill(true);
       continue;
diff --git a/contrib/llvm/lib/CodeGen/TailDuplication.cpp b/contrib/llvm/lib/CodeGen/TailDuplication.cpp
index 6b801cbf6e1e..3a6211a0f3e6 100644
--- a/contrib/llvm/lib/CodeGen/TailDuplication.cpp
+++ b/contrib/llvm/lib/CodeGen/TailDuplication.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
diff --git a/contrib/llvm/lib/CodeGen/TargetInstrInfoImpl.cpp b/contrib/llvm/lib/CodeGen/TargetInstrInfoImpl.cpp
index 86e71d8ccbb6..f32678f12b0a 100644
--- a/contrib/llvm/lib/CodeGen/TargetInstrInfoImpl.cpp
+++ b/contrib/llvm/lib/CodeGen/TargetInstrInfoImpl.cpp
@@ -74,23 +74,25 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI,
 
   assert(MI->getOperand(Idx1).isReg() && MI->getOperand(Idx2).isReg() &&
          "This only knows how to commute register operands so far");
+  unsigned Reg0 = HasDef ? MI->getOperand(0).getReg() : 0;
   unsigned Reg1 = MI->getOperand(Idx1).getReg();
   unsigned Reg2 = MI->getOperand(Idx2).getReg();
   bool Reg1IsKill = MI->getOperand(Idx1).isKill();
   bool Reg2IsKill = MI->getOperand(Idx2).isKill();
-  bool ChangeReg0 = false;
-  if (HasDef && MI->getOperand(0).getReg() == Reg1) {
-    // Must be two address instruction!
-    assert(MI->getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
-           "Expecting a two-address instruction!");
+  // If destination is tied to either of the commuted source register, then
+  // it must be updated.
+  if (HasDef && Reg0 == Reg1 &&
+      MI->getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) {
     Reg2IsKill = false;
-    ChangeReg0 = true;
+    Reg0 = Reg2;
+  } else if (HasDef && Reg0 == Reg2 &&
+             MI->getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) {
+    Reg1IsKill = false;
+    Reg0 = Reg1;
   }
 
   if (NewMI) {
     // Create a new instruction.
-    unsigned Reg0 = HasDef
-      ? (ChangeReg0 ? Reg2 : MI->getOperand(0).getReg()) : 0;
     bool Reg0IsDead = HasDef ? MI->getOperand(0).isDead() : false;
     MachineFunction &MF = *MI->getParent()->getParent();
     if (HasDef)
@@ -104,8 +106,8 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI,
         .addReg(Reg1, getKillRegState(Reg2IsKill));
   }
 
-  if (ChangeReg0)
-    MI->getOperand(0).setReg(Reg2);
+  if (HasDef)
+    MI->getOperand(0).setReg(Reg0);
   MI->getOperand(Idx2).setReg(Reg1);
   MI->getOperand(Idx1).setReg(Reg2);
   MI->getOperand(Idx2).setIsKill(Reg1IsKill);
@@ -160,6 +162,42 @@ bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI,
   return MadeChange;
 }
 
+bool TargetInstrInfoImpl::hasLoadFromStackSlot(const MachineInstr *MI,
+                                        const MachineMemOperand *&MMO,
+                                        int &FrameIndex) const {
+  for (MachineInstr::mmo_iterator o = MI->memoperands_begin(),
+         oe = MI->memoperands_end();
+       o != oe;
+       ++o) {
+    if ((*o)->isLoad() && (*o)->getValue())
+      if (const FixedStackPseudoSourceValue *Value =
+          dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) {
+        FrameIndex = Value->getFrameIndex();
+        MMO = *o;
+        return true;
+      }
+  }
+  return false;
+}
+
+bool TargetInstrInfoImpl::hasStoreToStackSlot(const MachineInstr *MI,
+                                       const MachineMemOperand *&MMO,
+                                       int &FrameIndex) const {
+  for (MachineInstr::mmo_iterator o = MI->memoperands_begin(),
+         oe = MI->memoperands_end();
+       o != oe;
+       ++o) {
+    if ((*o)->isStore() && (*o)->getValue())
+      if (const FixedStackPseudoSourceValue *Value =
+          dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) {
+        FrameIndex = Value->getFrameIndex();
+        MMO = *o;
+        return true;
+      }
+  }
+  return false;
+}
+
 void TargetInstrInfoImpl::reMaterialize(MachineBasicBlock &MBB,
                                         MachineBasicBlock::iterator I,
                                         unsigned DestReg,
@@ -324,6 +362,19 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI,
   const TargetInstrInfo &TII = *TM.getInstrInfo();
   const TargetRegisterInfo &TRI = *TM.getRegisterInfo();
 
+  // Remat clients assume operand 0 is the defined register.
+  if (!MI->getNumOperands() || !MI->getOperand(0).isReg())
+    return false;
+  unsigned DefReg = MI->getOperand(0).getReg();
+
+  // A sub-register definition can only be rematerialized if the instruction
+  // doesn't read the other parts of the register.  Otherwise it is really a
+  // read-modify-write operation on the full virtual register which cannot be
+  // moved safely.
+  if (TargetRegisterInfo::isVirtualRegister(DefReg) &&
+      MI->getOperand(0).getSubReg() && MI->readsVirtualRegister(DefReg))
+    return false;
+
   // A load from a fixed stack slot can be rematerialized. This may be
   // redundant with subsequent checks, but it's target-independent,
   // simple, and a common case.
@@ -383,8 +434,9 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI,
       continue;
     }
 
-    // Only allow one virtual-register def, and that in the first operand.
-    if (MO.isDef() != (i == 0))
+    // Only allow one virtual-register def.  There may be multiple defs of the
+    // same virtual register, though.
+    if (MO.isDef() && Reg != DefReg)
       return false;
 
     // Don't allow any virtual-register uses. Rematting an instruction with
diff --git a/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index a3c562013b59..fb87154232a0 100644
--- a/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -43,153 +43,6 @@ using namespace dwarf;
 //                                  ELF
 //===----------------------------------------------------------------------===//
 
-TargetLoweringObjectFileELF::TargetLoweringObjectFileELF()
-  : TargetLoweringObjectFile(),
-    TLSDataSection(0),
-    TLSBSSSection(0),
-    DataRelSection(0),
-    DataRelLocalSection(0),
-    DataRelROSection(0),
-    DataRelROLocalSection(0),
-    MergeableConst4Section(0),
-    MergeableConst8Section(0),
-    MergeableConst16Section(0) {
-}
-
-void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx,
-                                             const TargetMachine &TM) {
-  TargetLoweringObjectFile::Initialize(Ctx, TM);
-
-  BSSSection =
-    getContext().getELFSection(".bss", ELF::SHT_NOBITS,
-                               ELF::SHF_WRITE |ELF::SHF_ALLOC,
-                               SectionKind::getBSS());
-
-  TextSection =
-    getContext().getELFSection(".text", ELF::SHT_PROGBITS,
-                               ELF::SHF_EXECINSTR |
-                               ELF::SHF_ALLOC,
-                               SectionKind::getText());
-
-  DataSection =
-    getContext().getELFSection(".data", ELF::SHT_PROGBITS,
-                               ELF::SHF_WRITE |ELF::SHF_ALLOC,
-                               SectionKind::getDataRel());
-
-  ReadOnlySection =
-    getContext().getELFSection(".rodata", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC,
-                               SectionKind::getReadOnly());
-
-  TLSDataSection =
-    getContext().getELFSection(".tdata", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC | ELF::SHF_TLS |
-                               ELF::SHF_WRITE,
-                               SectionKind::getThreadData());
-
-  TLSBSSSection =
-    getContext().getELFSection(".tbss", ELF::SHT_NOBITS,
-                               ELF::SHF_ALLOC | ELF::SHF_TLS |
-                               ELF::SHF_WRITE,
-                               SectionKind::getThreadBSS());
-
-  DataRelSection =
-    getContext().getELFSection(".data.rel", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_WRITE,
-                               SectionKind::getDataRel());
-
-  DataRelLocalSection =
-    getContext().getELFSection(".data.rel.local", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_WRITE,
-                               SectionKind::getDataRelLocal());
-
-  DataRelROSection =
-    getContext().getELFSection(".data.rel.ro", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_WRITE,
-                               SectionKind::getReadOnlyWithRel());
-
-  DataRelROLocalSection =
-    getContext().getELFSection(".data.rel.ro.local", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_WRITE,
-                               SectionKind::getReadOnlyWithRelLocal());
-
-  MergeableConst4Section =
-    getContext().getELFSection(".rodata.cst4", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_MERGE,
-                               SectionKind::getMergeableConst4());
-
-  MergeableConst8Section =
-    getContext().getELFSection(".rodata.cst8", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_MERGE,
-                               SectionKind::getMergeableConst8());
-
-  MergeableConst16Section =
-    getContext().getELFSection(".rodata.cst16", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_MERGE,
-                               SectionKind::getMergeableConst16());
-
-  StaticCtorSection =
-    getContext().getELFSection(".ctors", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_WRITE,
-                               SectionKind::getDataRel());
-
-  StaticDtorSection =
-    getContext().getELFSection(".dtors", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC |ELF::SHF_WRITE,
-                               SectionKind::getDataRel());
-
-  // Exception Handling Sections.
-
-  // FIXME: We're emitting LSDA info into a readonly section on ELF, even though
-  // it contains relocatable pointers.  In PIC mode, this is probably a big
-  // runtime hit for C++ apps.  Either the contents of the LSDA need to be
-  // adjusted or this should be a data section.
-  LSDASection =
-    getContext().getELFSection(".gcc_except_table", ELF::SHT_PROGBITS,
-                               ELF::SHF_ALLOC,
-                               SectionKind::getReadOnly());
-  // Debug Info Sections.
-  DwarfAbbrevSection =
-    getContext().getELFSection(".debug_abbrev", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfInfoSection =
-    getContext().getELFSection(".debug_info", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfLineSection =
-    getContext().getELFSection(".debug_line", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfFrameSection =
-    getContext().getELFSection(".debug_frame", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfPubNamesSection =
-    getContext().getELFSection(".debug_pubnames", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfPubTypesSection =
-    getContext().getELFSection(".debug_pubtypes", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfStrSection =
-    getContext().getELFSection(".debug_str", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfLocSection =
-    getContext().getELFSection(".debug_loc", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfARangesSection =
-    getContext().getELFSection(".debug_aranges", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfRangesSection =
-    getContext().getELFSection(".debug_ranges", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-  DwarfMacroInfoSection =
-    getContext().getELFSection(".debug_macinfo", ELF::SHT_PROGBITS, 0,
-                               SectionKind::getMetadata());
-}
-
-const MCSection *TargetLoweringObjectFileELF::getEHFrameSection() const {
-  return getContext().getELFSection(".eh_frame", ELF::SHT_PROGBITS,
-                                    ELF::SHF_ALLOC,
-                                    SectionKind::getDataRel());
-}
-
 MCSymbol *
 TargetLoweringObjectFileELF::getCFIPersonalitySymbol(const GlobalValue *GV,
                                                      Mangler *Mang,
@@ -493,221 +346,6 @@ getExprForDwarfGlobalReference(const GlobalValue *GV, Mangler *Mang,
 //                                 MachO
 //===----------------------------------------------------------------------===//
 
-TargetLoweringObjectFileMachO::TargetLoweringObjectFileMachO()
-  : TargetLoweringObjectFile(),
-    TLSDataSection(0),
-    TLSBSSSection(0),
-    TLSTLVSection(0),
-    TLSThreadInitSection(0),
-    CStringSection(0),
-    UStringSection(0),
-    TextCoalSection(0),
-    ConstTextCoalSection(0),
-    ConstDataSection(0),
-    DataCoalSection(0),
-    DataCommonSection(0),
-    DataBSSSection(0),
-    FourByteConstantSection(0),
-    EightByteConstantSection(0),
-    SixteenByteConstantSection(0),
-    LazySymbolPointerSection(0),
-    NonLazySymbolPointerSection(0) {
-}
-
-void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx,
-                                               const TargetMachine &TM) {
-  IsFunctionEHFrameSymbolPrivate = false;
-  SupportsWeakOmittedEHFrame = false;
-
-  // .comm doesn't support alignment before Leopard.
-  Triple T(((LLVMTargetMachine&)TM).getTargetTriple());
-  if (T.isMacOSX() && T.isMacOSXVersionLT(10, 5))
-    CommDirectiveSupportsAlignment = false;
-
-  TargetLoweringObjectFile::Initialize(Ctx, TM);
-
-  TextSection // .text
-    = getContext().getMachOSection("__TEXT", "__text",
-                                   MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS,
-                                   SectionKind::getText());
-  DataSection // .data
-    = getContext().getMachOSection("__DATA", "__data", 0,
-                                   SectionKind::getDataRel());
-
-  TLSDataSection // .tdata
-    = getContext().getMachOSection("__DATA", "__thread_data",
-                                   MCSectionMachO::S_THREAD_LOCAL_REGULAR,
-                                   SectionKind::getDataRel());
-  TLSBSSSection // .tbss
-    = getContext().getMachOSection("__DATA", "__thread_bss",
-                                   MCSectionMachO::S_THREAD_LOCAL_ZEROFILL,
-                                   SectionKind::getThreadBSS());
-
-  // TODO: Verify datarel below.
-  TLSTLVSection // .tlv
-    = getContext().getMachOSection("__DATA", "__thread_vars",
-                                   MCSectionMachO::S_THREAD_LOCAL_VARIABLES,
-                                   SectionKind::getDataRel());
-
-  TLSThreadInitSection
-    = getContext().getMachOSection("__DATA", "__thread_init",
-                          MCSectionMachO::S_THREAD_LOCAL_INIT_FUNCTION_POINTERS,
-                          SectionKind::getDataRel());
-
-  CStringSection // .cstring
-    = getContext().getMachOSection("__TEXT", "__cstring",
-                                   MCSectionMachO::S_CSTRING_LITERALS,
-                                   SectionKind::getMergeable1ByteCString());
-  UStringSection
-    = getContext().getMachOSection("__TEXT","__ustring", 0,
-                                   SectionKind::getMergeable2ByteCString());
-  FourByteConstantSection // .literal4
-    = getContext().getMachOSection("__TEXT", "__literal4",
-                                   MCSectionMachO::S_4BYTE_LITERALS,
-                                   SectionKind::getMergeableConst4());
-  EightByteConstantSection // .literal8
-    = getContext().getMachOSection("__TEXT", "__literal8",
-                                   MCSectionMachO::S_8BYTE_LITERALS,
-                                   SectionKind::getMergeableConst8());
-
-  // ld_classic doesn't support .literal16 in 32-bit mode, and ld64 falls back
-  // to using it in -static mode.
-  SixteenByteConstantSection = 0;
-  if (TM.getRelocationModel() != Reloc::Static &&
-      TM.getTargetData()->getPointerSize() == 32)
-    SixteenByteConstantSection =   // .literal16
-      getContext().getMachOSection("__TEXT", "__literal16",
-                                   MCSectionMachO::S_16BYTE_LITERALS,
-                                   SectionKind::getMergeableConst16());
-
-  ReadOnlySection  // .const
-    = getContext().getMachOSection("__TEXT", "__const", 0,
-                                   SectionKind::getReadOnly());
-
-  TextCoalSection
-    = getContext().getMachOSection("__TEXT", "__textcoal_nt",
-                                   MCSectionMachO::S_COALESCED |
-                                   MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS,
-                                   SectionKind::getText());
-  ConstTextCoalSection
-    = getContext().getMachOSection("__TEXT", "__const_coal",
-                                   MCSectionMachO::S_COALESCED,
-                                   SectionKind::getReadOnly());
-  ConstDataSection  // .const_data
-    = getContext().getMachOSection("__DATA", "__const", 0,
-                                   SectionKind::getReadOnlyWithRel());
-  DataCoalSection
-    = getContext().getMachOSection("__DATA","__datacoal_nt",
-                                   MCSectionMachO::S_COALESCED,
-                                   SectionKind::getDataRel());
-  DataCommonSection
-    = getContext().getMachOSection("__DATA","__common",
-                                   MCSectionMachO::S_ZEROFILL,
-                                   SectionKind::getBSS());
-  DataBSSSection
-    = getContext().getMachOSection("__DATA","__bss", MCSectionMachO::S_ZEROFILL,
-                                   SectionKind::getBSS());
-
-
-  LazySymbolPointerSection
-    = getContext().getMachOSection("__DATA", "__la_symbol_ptr",
-                                   MCSectionMachO::S_LAZY_SYMBOL_POINTERS,
-                                   SectionKind::getMetadata());
-  NonLazySymbolPointerSection
-    = getContext().getMachOSection("__DATA", "__nl_symbol_ptr",
-                                   MCSectionMachO::S_NON_LAZY_SYMBOL_POINTERS,
-                                   SectionKind::getMetadata());
-
-  if (TM.getRelocationModel() == Reloc::Static) {
-    StaticCtorSection
-      = getContext().getMachOSection("__TEXT", "__constructor", 0,
-                                     SectionKind::getDataRel());
-    StaticDtorSection
-      = getContext().getMachOSection("__TEXT", "__destructor", 0,
-                                     SectionKind::getDataRel());
-  } else {
-    StaticCtorSection
-      = getContext().getMachOSection("__DATA", "__mod_init_func",
-                                     MCSectionMachO::S_MOD_INIT_FUNC_POINTERS,
-                                     SectionKind::getDataRel());
-    StaticDtorSection
-      = getContext().getMachOSection("__DATA", "__mod_term_func",
-                                     MCSectionMachO::S_MOD_TERM_FUNC_POINTERS,
-                                     SectionKind::getDataRel());
-  }
-
-  // Exception Handling.
-  LSDASection = getContext().getMachOSection("__TEXT", "__gcc_except_tab", 0,
-                                             SectionKind::getReadOnlyWithRel());
-
-  if (T.isMacOSX() && !T.isMacOSXVersionLT(10, 6))
-    CompactUnwindSection =
-      getContext().getMachOSection("__LD", "__compact_unwind",
-                                   MCSectionMachO::S_ATTR_DEBUG,
-                                   SectionKind::getReadOnly());
-
-  // Debug Information.
-  DwarfAbbrevSection =
-    getContext().getMachOSection("__DWARF", "__debug_abbrev",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfInfoSection =
-    getContext().getMachOSection("__DWARF", "__debug_info",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfLineSection =
-    getContext().getMachOSection("__DWARF", "__debug_line",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfFrameSection =
-    getContext().getMachOSection("__DWARF", "__debug_frame",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfPubNamesSection =
-    getContext().getMachOSection("__DWARF", "__debug_pubnames",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfPubTypesSection =
-    getContext().getMachOSection("__DWARF", "__debug_pubtypes",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfStrSection =
-    getContext().getMachOSection("__DWARF", "__debug_str",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfLocSection =
-    getContext().getMachOSection("__DWARF", "__debug_loc",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfARangesSection =
-    getContext().getMachOSection("__DWARF", "__debug_aranges",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfRangesSection =
-    getContext().getMachOSection("__DWARF", "__debug_ranges",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfMacroInfoSection =
-    getContext().getMachOSection("__DWARF", "__debug_macinfo",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-  DwarfDebugInlineSection =
-    getContext().getMachOSection("__DWARF", "__debug_inlined",
-                                 MCSectionMachO::S_ATTR_DEBUG,
-                                 SectionKind::getMetadata());
-
-  TLSExtraDataSection = TLSTLVSection;
-}
-
-const MCSection *TargetLoweringObjectFileMachO::getEHFrameSection() const {
-  return getContext().getMachOSection("__TEXT", "__eh_frame",
-                                      MCSectionMachO::S_COALESCED |
-                                      MCSectionMachO::S_ATTR_NO_TOC |
-                                      MCSectionMachO::S_ATTR_STRIP_STATIC_SYMS |
-                                      MCSectionMachO::S_ATTR_LIVE_SUPPORT,
-                                      SectionKind::getReadOnly());
-}
-
 const MCSection *TargetLoweringObjectFileMachO::
 getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
                          Mangler *Mang, const TargetMachine &TM) const {
@@ -905,183 +543,10 @@ getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
   return SSym;
 }
 
-unsigned TargetLoweringObjectFileMachO::getPersonalityEncoding() const {
-  return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-}
-
-unsigned TargetLoweringObjectFileMachO::getLSDAEncoding() const {
-  return DW_EH_PE_pcrel;
-}
-
-unsigned TargetLoweringObjectFileMachO::getFDEEncoding(bool CFI) const {
-  return DW_EH_PE_pcrel;
-}
-
-unsigned TargetLoweringObjectFileMachO::getTTypeEncoding() const {
-  return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-}
-
 //===----------------------------------------------------------------------===//
 //                                  COFF
 //===----------------------------------------------------------------------===//
 
-TargetLoweringObjectFileCOFF::TargetLoweringObjectFileCOFF()
-  : TargetLoweringObjectFile(),
-    DrectveSection(0),
-    PDataSection(0),
-    XDataSection(0) {
-}
-
-void TargetLoweringObjectFileCOFF::Initialize(MCContext &Ctx,
-                                              const TargetMachine &TM) {
-  TargetLoweringObjectFile::Initialize(Ctx, TM);
-  TextSection =
-    getContext().getCOFFSection(".text",
-                                COFF::IMAGE_SCN_CNT_CODE |
-                                COFF::IMAGE_SCN_MEM_EXECUTE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getText());
-  DataSection =
-    getContext().getCOFFSection(".data",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ |
-                                COFF::IMAGE_SCN_MEM_WRITE,
-                                SectionKind::getDataRel());
-  ReadOnlySection =
-    getContext().getCOFFSection(".rdata",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getReadOnly());
-  StaticCtorSection =
-    getContext().getCOFFSection(".ctors",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ |
-                                COFF::IMAGE_SCN_MEM_WRITE,
-                                SectionKind::getDataRel());
-  StaticDtorSection =
-    getContext().getCOFFSection(".dtors",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ |
-                                COFF::IMAGE_SCN_MEM_WRITE,
-                                SectionKind::getDataRel());
-
-  // FIXME: We're emitting LSDA info into a readonly section on COFF, even
-  // though it contains relocatable pointers.  In PIC mode, this is probably a
-  // big runtime hit for C++ apps.  Either the contents of the LSDA need to be
-  // adjusted or this should be a data section.
-  LSDASection =
-    getContext().getCOFFSection(".gcc_except_table",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getReadOnly());
-  // Debug info.
-  DwarfAbbrevSection =
-    getContext().getCOFFSection(".debug_abbrev",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfInfoSection =
-    getContext().getCOFFSection(".debug_info",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfLineSection =
-    getContext().getCOFFSection(".debug_line",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfFrameSection =
-    getContext().getCOFFSection(".debug_frame",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfPubNamesSection =
-    getContext().getCOFFSection(".debug_pubnames",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfPubTypesSection =
-    getContext().getCOFFSection(".debug_pubtypes",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfStrSection =
-    getContext().getCOFFSection(".debug_str",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfLocSection =
-    getContext().getCOFFSection(".debug_loc",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfARangesSection =
-    getContext().getCOFFSection(".debug_aranges",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfRangesSection =
-    getContext().getCOFFSection(".debug_ranges",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-  DwarfMacroInfoSection =
-    getContext().getCOFFSection(".debug_macinfo",
-                                COFF::IMAGE_SCN_MEM_DISCARDABLE |
-                                COFF::IMAGE_SCN_MEM_READ,
-                                SectionKind::getMetadata());
-
-  DrectveSection =
-    getContext().getCOFFSection(".drectve",
-                                COFF::IMAGE_SCN_LNK_INFO,
-                                SectionKind::getMetadata());
-
-  PDataSection =
-    getContext().getCOFFSection(".pdata",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ |
-                                COFF::IMAGE_SCN_MEM_WRITE,
-                                SectionKind::getDataRel());
-
-  XDataSection =
-    getContext().getCOFFSection(".xdata",
-                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_MEM_READ |
-                                COFF::IMAGE_SCN_MEM_WRITE,
-                                SectionKind::getDataRel());
-}
-
-const MCSection *TargetLoweringObjectFileCOFF::getEHFrameSection() const {
-  return getContext().getCOFFSection(".eh_frame",
-                                     COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                     COFF::IMAGE_SCN_MEM_READ |
-                                     COFF::IMAGE_SCN_MEM_WRITE,
-                                     SectionKind::getDataRel());
-}
-
-const MCSection *TargetLoweringObjectFileCOFF::getWin64EHFuncTableSection(
-                                                       StringRef suffix) const {
-  if (suffix == "")
-    return PDataSection;
-  return getContext().getCOFFSection((".pdata"+suffix).str(),
-                                     COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                     COFF::IMAGE_SCN_MEM_READ |
-                                     COFF::IMAGE_SCN_MEM_WRITE,
-                                     SectionKind::getDataRel());
-}
-
-const MCSection *TargetLoweringObjectFileCOFF::getWin64EHTableSection(
-                                                       StringRef suffix) const {
-  if (suffix == "")
-    return XDataSection;
-  return getContext().getCOFFSection((".xdata"+suffix).str(),
-                                     COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                     COFF::IMAGE_SCN_MEM_READ |
-                                     COFF::IMAGE_SCN_MEM_WRITE,
-                                     SectionKind::getDataRel());
-}
-
-
 static unsigned
 getCOFFSectionFlags(SectionKind K) {
   unsigned Flags = 0;
diff --git a/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index 6d6244e4f879..d87937822280 100644
--- a/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -177,6 +177,10 @@ char &llvm::TwoAddressInstructionPassID = TwoAddressInstructionPass::ID;
 bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB,
                                            MachineInstr *MI, unsigned SavedReg,
                                            MachineBasicBlock::iterator OldPos) {
+  // FIXME: Shouldn't we be trying to do this before we three-addressify the
+  // instruction?  After this transformation is done, we no longer need
+  // the instruction to be in three-address form.
+
   // Check if it's safe to move this instruction.
   bool SeenStore = true; // Be conservative.
   if (!MI->isSafeToMove(TII, AA, SeenStore))
@@ -217,7 +221,11 @@ bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB,
     break;
   }
 
-  if (!KillMI || KillMI->getParent() != MBB || KillMI == MI)
+  // If we find the instruction that kills SavedReg, and it is in an
+  // appropriate location, we can try to sink the current instruction
+  // past it.
+  if (!KillMI || KillMI->getParent() != MBB || KillMI == MI ||
+      KillMI->getDesc().isTerminator())
     return false;
 
   // If any of the definitions are used by another instruction between the
@@ -1041,6 +1049,9 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) {
   DEBUG(dbgs() << "********** Function: " 
         << MF.getFunction()->getName() << '\n');
 
+  // This pass takes the function out of SSA form.
+  MRI->leaveSSA();
+
   // ReMatRegs - Keep track of the registers whose def's are remat'ed.
   BitVector ReMatRegs(MRI->getNumVirtRegs());
 
diff --git a/contrib/llvm/lib/CodeGen/VirtRegMap.cpp b/contrib/llvm/lib/CodeGen/VirtRegMap.cpp
index 7557979ec9bb..8a1cdc01c494 100644
--- a/contrib/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/contrib/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -41,8 +41,8 @@
 #include <algorithm>
 using namespace llvm;
 
-STATISTIC(NumSpills  , "Number of register spills");
-STATISTIC(NumIdCopies, "Number of identity moves eliminated after rewriting");
+STATISTIC(NumSpillSlots, "Number of spill slots allocated");
+STATISTIC(NumIdCopies,   "Number of identity moves eliminated after rewriting");
 
 //===----------------------------------------------------------------------===//
 //  VirtRegMap implementation
@@ -111,6 +111,7 @@ unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) {
   unsigned Idx = SS-LowSpillSlot;
   while (Idx >= SpillSlotToUsesMap.size())
     SpillSlotToUsesMap.resize(SpillSlotToUsesMap.size()*2);
+  ++NumSpillSlots;
   return SS;
 }
 
@@ -130,7 +131,6 @@ int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) {
   assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
          "attempt to assign stack slot to already spilled register");
   const TargetRegisterClass* RC = MF->getRegInfo().getRegClass(virtReg);
-  ++NumSpills;
   return Virt2StackSlotMap[virtReg] = createSpillSlot(RC);
 }
 
@@ -285,14 +285,24 @@ void VirtRegMap::rewrite(SlotIndexes *Indexes) {
         // Preserve semantics of sub-register operands.
         if (MO.getSubReg()) {
           // A virtual register kill refers to the whole register, so we may
-          // have to add <imp-use,kill> operands for the super-register.
-          if (MO.isUse()) {
-            if (MO.isKill() && !MO.isUndef())
-              SuperKills.push_back(PhysReg);
-          } else if (MO.isDead())
-            SuperDeads.push_back(PhysReg);
-          else
-            SuperDefs.push_back(PhysReg);
+          // have to add <imp-use,kill> operands for the super-register.  A
+          // partial redef always kills and redefines the super-register.
+          if (MO.readsReg() && (MO.isDef() || MO.isKill()))
+            SuperKills.push_back(PhysReg);
+
+          if (MO.isDef()) {
+            // The <def,undef> flag only makes sense for sub-register defs, and
+            // we are substituting a full physreg.  An <imp-use,kill> operand
+            // from the SuperKills list will represent the partial read of the
+            // super-register.
+            MO.setIsUndef(false);
+
+            // Also add implicit defs for the super-register.
+            if (MO.isDead())
+              SuperDeads.push_back(PhysReg);
+            else
+              SuperDefs.push_back(PhysReg);
+          }
 
           // PhysReg operands cannot have subregister indexes.
           PhysReg = TRI->getSubReg(PhysReg, MO.getSubReg());
diff --git a/contrib/llvm/lib/CompilerDriver/Action.cpp b/contrib/llvm/lib/CompilerDriver/Action.cpp
deleted file mode 100644
index a8d625c7ac04..000000000000
--- a/contrib/llvm/lib/CompilerDriver/Action.cpp
+++ /dev/null
@@ -1,134 +0,0 @@
-//===--- Action.cpp - The LLVM Compiler Driver ------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Action class - implementation and auxiliary functions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CompilerDriver/Action.h"
-#include "llvm/CompilerDriver/BuiltinOptions.h"
-#include "llvm/CompilerDriver/Error.h"
-#include "llvm/CompilerDriver/Main.h"
-
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/SystemUtils.h"
-#include "llvm/Support/Program.h"
-#include "llvm/Support/TimeValue.h"
-
-#include <stdexcept>
-#include <string>
-
-using namespace llvm;
-using namespace llvmc;
-
-namespace llvmc {
-
-extern const char* ProgramName;
-
-}
-
-namespace {
-
-  void PrintString (const std::string& str) {
-    errs() << str << ' ';
-  }
-
-  void PrintCommand (const std::string& Cmd, const StrVector& Args) {
-    errs() << Cmd << ' ';
-    std::for_each(Args.begin(), Args.end(), &PrintString);
-    errs() << '\n';
-  }
-
-  bool IsSegmentationFault (int returnCode) {
-#ifdef LLVM_ON_WIN32
-    return (returnCode >= 0xc0000000UL)
-#else
-    return (returnCode < 0);
-#endif
-  }
-
-  int ExecuteProgram (const std::string& name, const StrVector& args) {
-    sys::Path prog(name);
-
-    if (sys::path::is_relative(prog.str())) {
-      prog = PrependMainExecutablePath(name, ProgramName,
-                                       (void *)(intptr_t)&Main);
-
-      if (!prog.canExecute()) {
-        prog = sys::Program::FindProgramByName(name);
-        if (prog.isEmpty()) {
-          PrintError("Can't find program '" + name + "'");
-          return -1;
-        }
-      }
-    }
-    if (!prog.canExecute()) {
-      PrintError("Program '" + name + "' is not executable.");
-      return -1;
-    }
-
-    // Build the command line vector and the redirects array.
-    const sys::Path* redirects[3] = {0,0,0};
-    sys::Path stdout_redirect;
-
-    std::vector<const char*> argv;
-    argv.reserve((args.size()+2));
-    argv.push_back(name.c_str());
-
-    for (StrVector::const_iterator B = args.begin(), E = args.end();
-         B!=E; ++B) {
-      if (*B == ">") {
-        ++B;
-        stdout_redirect.set(*B);
-        redirects[1] = &stdout_redirect;
-      }
-      else {
-        argv.push_back((*B).c_str());
-      }
-    }
-    argv.push_back(0);  // null terminate list.
-
-    // Invoke the program.
-    int ret = sys::Program::ExecuteAndWait(prog, &argv[0], 0, &redirects[0]);
-
-    if (IsSegmentationFault(ret)) {
-      errs() << "Segmentation fault: ";
-      PrintCommand(name, args);
-    }
-
-    return ret;
-  }
-}
-
-namespace llvmc {
-  void AppendToGlobalTimeLog (const std::string& cmd, double time);
-}
-
-int llvmc::Action::Execute () const {
-  if (DryRun || VerboseMode)
-    PrintCommand(Command_, Args_);
-
-  if (!DryRun) {
-    if (Time) {
-      sys::TimeValue now = sys::TimeValue::now();
-      int ret = ExecuteProgram(Command_, Args_);
-      sys::TimeValue now2 = sys::TimeValue::now();
-      now2 -= now;
-      double elapsed = now2.seconds()  + now2.microseconds()  / 1000000.0;
-      AppendToGlobalTimeLog(Command_, elapsed);
-
-      return ret;
-    }
-    else {
-      return ExecuteProgram(Command_, Args_);
-    }
-  }
-
-  return 0;
-}
diff --git a/contrib/llvm/lib/CompilerDriver/BuiltinOptions.cpp b/contrib/llvm/lib/CompilerDriver/BuiltinOptions.cpp
deleted file mode 100644
index 38442038d738..000000000000
--- a/contrib/llvm/lib/CompilerDriver/BuiltinOptions.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-//===--- BuiltinOptions.cpp - The LLVM Compiler Driver ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Definitions of all global command-line option variables.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CompilerDriver/BuiltinOptions.h"
-
-#ifdef ENABLE_LLVMC_DYNAMIC_PLUGINS
-#include "llvm/Support/PluginLoader.h"
-#endif
-
-namespace cl = llvm::cl;
-
-namespace llvmc {
-
-cl::list<std::string> InputFilenames(cl::Positional, cl::desc("<input file>"),
-                                     cl::ZeroOrMore);
-cl::opt<std::string> OutputFilename("o", cl::desc("Output file name"),
-                                    cl::value_desc("file"), cl::Prefix);
-cl::opt<std::string> TempDirname("temp-dir", cl::desc("Temp dir name"),
-                                 cl::value_desc("<directory>"), cl::Prefix);
-cl::list<std::string> Languages("x",
-          cl::desc("Specify the language of the following input files"),
-          cl::ZeroOrMore);
-
-cl::opt<bool> DryRun("dry-run",
-                     cl::desc("Only pretend to run commands"));
-cl::opt<bool> Time("time", cl::desc("Time individual commands"));
-cl::opt<bool> VerboseMode("v",
-                          cl::desc("Enable verbose mode"));
-
-cl::opt<bool> CheckGraph("check-graph",
-                         cl::desc("Check the compilation graph for errors"),
-                         cl::Hidden);
-cl::opt<bool> WriteGraph("write-graph",
-                         cl::desc("Write compilation-graph.dot file"),
-                         cl::Hidden);
-cl::opt<bool> ViewGraph("view-graph",
-                         cl::desc("Show compilation graph in GhostView"),
-                         cl::Hidden);
-
-cl::opt<SaveTempsEnum::Values> SaveTemps
-("save-temps", cl::desc("Keep temporary files"),
- cl::init(SaveTempsEnum::Unset),
- cl::values(clEnumValN(SaveTempsEnum::Obj, "obj",
-                       "Save files in the directory specified with -o"),
-            clEnumValN(SaveTempsEnum::Cwd, "cwd",
-                       "Use current working directory"),
-            clEnumValN(SaveTempsEnum::Obj, "", "Same as 'cwd'"),
-            clEnumValEnd),
- cl::ValueOptional);
-
-}  // End namespace llvmc.
diff --git a/contrib/llvm/lib/CompilerDriver/CompilationGraph.cpp b/contrib/llvm/lib/CompilerDriver/CompilationGraph.cpp
deleted file mode 100644
index 33c6566499b8..000000000000
--- a/contrib/llvm/lib/CompilerDriver/CompilationGraph.cpp
+++ /dev/null
@@ -1,655 +0,0 @@
-//===--- CompilationGraph.cpp - The LLVM Compiler Driver --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Compilation graph - implementation.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CompilerDriver/BuiltinOptions.h"
-#include "llvm/CompilerDriver/CompilationGraph.h"
-#include "llvm/CompilerDriver/Error.h"
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/DOTGraphTraits.h"
-#include "llvm/Support/GraphWriter.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <algorithm>
-#include <cstring>
-#include <iterator>
-#include <limits>
-#include <queue>
-
-using namespace llvm;
-using namespace llvmc;
-
-namespace llvmc {
-
-  const std::string* LanguageMap::GetLanguage(const sys::Path& File) const {
-    // Remove the '.'.
-    StringRef suf = sys::path::extension(File.str()).substr(1);
-    LanguageMap::const_iterator Lang =
-      this->find(suf.empty() ? "*empty*" : suf);
-    if (Lang == this->end()) {
-      PrintError("File '" + File.str() + "' has unknown suffix '"
-                 + suf.str() + '\'');
-      return 0;
-    }
-    return &Lang->second;
-  }
-}
-
-namespace {
-
-  /// ChooseEdge - Return the edge with the maximum weight. Returns 0 on error.
-  template <class C>
-  const Edge* ChooseEdge(const C& EdgesContainer,
-                         const InputLanguagesSet& InLangs,
-                         const std::string& NodeName = "root") {
-    const Edge* MaxEdge = 0;
-    int MaxWeight = 0;
-    bool SingleMax = true;
-
-    // TODO: fix calculation of SingleMax.
-    for (typename C::const_iterator B = EdgesContainer.begin(),
-           E = EdgesContainer.end(); B != E; ++B) {
-      const Edge* e = B->getPtr();
-      int EW = e->Weight(InLangs);
-      if (EW < 0) {
-        // (error) invocation in TableGen -> we don't need to print an error
-        // message.
-        return 0;
-      }
-      if (EW > MaxWeight) {
-        MaxEdge = e;
-        MaxWeight = EW;
-        SingleMax = true;
-      } else if (EW == MaxWeight) {
-        SingleMax = false;
-      }
-    }
-
-    if (!SingleMax) {
-      PrintError("Node " + NodeName + ": multiple maximal outward edges found!"
-                 " Most probably a specification error.");
-      return 0;
-    }
-    if (!MaxEdge) {
-      PrintError("Node " + NodeName + ": no maximal outward edge found!"
-                 " Most probably a specification error.");
-      return 0;
-    }
-    return MaxEdge;
-  }
-
-}
-
-void Node::AddEdge(Edge* Edg) {
-  // If there already was an edge between two nodes, modify it instead
-  // of adding a new edge.
-  const std::string& ToolName = Edg->ToolName();
-  for (container_type::iterator B = OutEdges.begin(), E = OutEdges.end();
-       B != E; ++B) {
-    if ((*B)->ToolName() == ToolName) {
-      llvm::IntrusiveRefCntPtr<Edge>(Edg).swap(*B);
-      return;
-    }
-  }
-  OutEdges.push_back(llvm::IntrusiveRefCntPtr<Edge>(Edg));
-}
-
-CompilationGraph::CompilationGraph() {
-  NodesMap["root"] = Node(this);
-}
-
-Node* CompilationGraph::getNode(const std::string& ToolName) {
-  nodes_map_type::iterator I = NodesMap.find(ToolName);
-  if (I == NodesMap.end()) {
-    PrintError("Node " + ToolName + " is not in the graph");
-    return 0;
-  }
-  return &I->second;
-}
-
-const Node* CompilationGraph::getNode(const std::string& ToolName) const {
-  nodes_map_type::const_iterator I = NodesMap.find(ToolName);
-  if (I == NodesMap.end()) {
-    PrintError("Node " + ToolName + " is not in the graph!");
-    return 0;
-  }
-  return &I->second;
-}
-
-// Find the tools list corresponding to the given language name.
-const CompilationGraph::tools_vector_type*
-CompilationGraph::getToolsVector(const std::string& LangName) const
-{
-  tools_map_type::const_iterator I = ToolsMap.find(LangName);
-  if (I == ToolsMap.end()) {
-    PrintError("No tool corresponding to the language " + LangName + " found");
-    return 0;
-  }
-  return &I->second;
-}
-
-void CompilationGraph::insertNode(Tool* V) {
-  if (NodesMap.count(V->Name()) == 0)
-    NodesMap[V->Name()] = Node(this, V);
-}
-
-int CompilationGraph::insertEdge(const std::string& A, Edge* Edg) {
-  Node* B = getNode(Edg->ToolName());
-  if (B == 0)
-    return 1;
-
-  if (A == "root") {
-    const char** InLangs = B->ToolPtr->InputLanguages();
-    for (;*InLangs; ++InLangs)
-      ToolsMap[*InLangs].push_back(IntrusiveRefCntPtr<Edge>(Edg));
-    NodesMap["root"].AddEdge(Edg);
-  }
-  else {
-    Node* N = getNode(A);
-    if (N == 0)
-      return 1;
-
-    N->AddEdge(Edg);
-  }
-  // Increase the inward edge counter.
-  B->IncrInEdges();
-
-  return 0;
-}
-
-// Pass input file through the chain until we bump into a Join node or
-// a node that says that it is the last.
-int CompilationGraph::PassThroughGraph (const sys::Path& InFile,
-                                        const Node* StartNode,
-                                        const InputLanguagesSet& InLangs,
-                                        const sys::Path& TempDir,
-                                        const LanguageMap& LangMap) const {
-  sys::Path In = InFile;
-  const Node* CurNode = StartNode;
-
-  while(true) {
-    Tool* CurTool = CurNode->ToolPtr.getPtr();
-
-    if (CurTool->IsJoin()) {
-      JoinTool& JT = static_cast<JoinTool&>(*CurTool);
-      JT.AddToJoinList(In);
-      break;
-    }
-
-    Action CurAction;
-    if (int ret = CurTool->GenerateAction(CurAction, In, CurNode->HasChildren(),
-                                          TempDir, InLangs, LangMap)) {
-      return ret;
-    }
-
-    if (int ret = CurAction.Execute())
-      return ret;
-
-    if (CurAction.StopCompilation())
-      return 0;
-
-    const Edge* Edg = ChooseEdge(CurNode->OutEdges, InLangs, CurNode->Name());
-    if (Edg == 0)
-      return 1;
-
-    CurNode = getNode(Edg->ToolName());
-    if (CurNode == 0)
-      return 1;
-
-    In = CurAction.OutFile();
-  }
-
-  return 0;
-}
-
-// Find the head of the toolchain corresponding to the given file.
-// Also, insert an input language into InLangs.
-const Node* CompilationGraph::
-FindToolChain(const sys::Path& In, const std::string* ForceLanguage,
-              InputLanguagesSet& InLangs, const LanguageMap& LangMap) const {
-
-  // Determine the input language.
-  const std::string* InLang = (ForceLanguage ? ForceLanguage
-                               : LangMap.GetLanguage(In));
-  if (InLang == 0)
-    return 0;
-  const std::string& InLanguage = *InLang;
-
-  // Add the current input language to the input language set.
-  InLangs.insert(InLanguage);
-
-  // Find the toolchain for the input language.
-  const tools_vector_type* pTV = getToolsVector(InLanguage);
-  if (pTV == 0)
-    return 0;
-
-  const tools_vector_type& TV = *pTV;
-  if (TV.empty()) {
-    PrintError("No toolchain corresponding to language "
-               + InLanguage + " found");
-    return 0;
-  }
-
-  const Edge* Edg = ChooseEdge(TV, InLangs);
-  if (Edg == 0)
-    return 0;
-
-  return getNode(Edg->ToolName());
-}
-
-// Helper function used by Build().
-// Traverses initial portions of the toolchains (up to the first Join node).
-// This function is also responsible for handling the -x option.
-int CompilationGraph::BuildInitial (InputLanguagesSet& InLangs,
-                                    const sys::Path& TempDir,
-                                    const LanguageMap& LangMap) {
-  // This is related to -x option handling.
-  cl::list<std::string>::const_iterator xIter = Languages.begin(),
-    xBegin = xIter, xEnd = Languages.end();
-  bool xEmpty = true;
-  const std::string* xLanguage = 0;
-  unsigned xPos = 0, xPosNext = 0, filePos = 0;
-
-  if (xIter != xEnd) {
-    xEmpty = false;
-    xPos = Languages.getPosition(xIter - xBegin);
-    cl::list<std::string>::const_iterator xNext = llvm::next(xIter);
-    xPosNext = (xNext == xEnd) ? std::numeric_limits<unsigned>::max()
-      : Languages.getPosition(xNext - xBegin);
-    xLanguage = (*xIter == "none") ? 0 : &(*xIter);
-  }
-
-  // For each input file:
-  for (cl::list<std::string>::const_iterator B = InputFilenames.begin(),
-         CB = B, E = InputFilenames.end(); B != E; ++B) {
-    sys::Path In = sys::Path(*B);
-
-    // Code for handling the -x option.
-    // Output: std::string* xLanguage (can be NULL).
-    if (!xEmpty) {
-      filePos = InputFilenames.getPosition(B - CB);
-
-      if (xPos < filePos) {
-        if (filePos < xPosNext) {
-          xLanguage = (*xIter == "none") ? 0 : &(*xIter);
-        }
-        else { // filePos >= xPosNext
-          // Skip xIters while filePos > xPosNext
-          while (filePos > xPosNext) {
-            ++xIter;
-            xPos = xPosNext;
-
-            cl::list<std::string>::const_iterator xNext = llvm::next(xIter);
-            if (xNext == xEnd)
-              xPosNext = std::numeric_limits<unsigned>::max();
-            else
-              xPosNext = Languages.getPosition(xNext - xBegin);
-            xLanguage = (*xIter == "none") ? 0 : &(*xIter);
-          }
-        }
-      }
-    }
-
-    // Find the toolchain corresponding to this file.
-    const Node* N = FindToolChain(In, xLanguage, InLangs, LangMap);
-    if (N == 0)
-      return 1;
-    // Pass file through the chain starting at head.
-    if (int ret = PassThroughGraph(In, N, InLangs, TempDir, LangMap))
-      return ret;
-  }
-
-  return 0;
-}
-
-// Sort the nodes in topological order.
-int CompilationGraph::TopologicalSort(std::vector<const Node*>& Out) {
-  std::queue<const Node*> Q;
-
-  Node* Root = getNode("root");
-  if (Root == 0)
-    return 1;
-
-  Q.push(Root);
-
-  while (!Q.empty()) {
-    const Node* A = Q.front();
-    Q.pop();
-    Out.push_back(A);
-    for (Node::const_iterator EB = A->EdgesBegin(), EE = A->EdgesEnd();
-         EB != EE; ++EB) {
-      Node* B = getNode((*EB)->ToolName());
-      if (B == 0)
-        return 1;
-
-      B->DecrInEdges();
-      if (B->HasNoInEdges())
-        Q.push(B);
-    }
-  }
-
-  return 0;
-}
-
-namespace {
-  bool NotJoinNode(const Node* N) {
-    return N->ToolPtr ? !N->ToolPtr->IsJoin() : true;
-  }
-}
-
-// Call TopologicalSort and filter the resulting list to include
-// only Join nodes.
-int CompilationGraph::
-TopologicalSortFilterJoinNodes(std::vector<const Node*>& Out) {
-  std::vector<const Node*> TopSorted;
-  if (int ret = TopologicalSort(TopSorted))
-    return ret;
-  std::remove_copy_if(TopSorted.begin(), TopSorted.end(),
-                      std::back_inserter(Out), NotJoinNode);
-
-  return 0;
-}
-
-int CompilationGraph::Build (const sys::Path& TempDir,
-                             const LanguageMap& LangMap) {
-  InputLanguagesSet InLangs;
-  bool WasSomeActionGenerated = !InputFilenames.empty();
-
-  // Traverse initial parts of the toolchains and fill in InLangs.
-  if (int ret = BuildInitial(InLangs, TempDir, LangMap))
-    return ret;
-
-  std::vector<const Node*> JTV;
-  if (int ret = TopologicalSortFilterJoinNodes(JTV))
-    return ret;
-
-  // For all join nodes in topological order:
-  for (std::vector<const Node*>::iterator B = JTV.begin(), E = JTV.end();
-       B != E; ++B) {
-
-    const Node* CurNode = *B;
-    JoinTool* JT = &static_cast<JoinTool&>(*CurNode->ToolPtr.getPtr());
-
-    // Are there any files in the join list?
-    if (JT->JoinListEmpty() && !(JT->WorksOnEmpty() && InputFilenames.empty()))
-      continue;
-
-    WasSomeActionGenerated = true;
-    Action CurAction;
-    if (int ret = JT->GenerateAction(CurAction, CurNode->HasChildren(),
-                                     TempDir, InLangs, LangMap)) {
-      return ret;
-    }
-
-    if (int ret = CurAction.Execute())
-      return ret;
-
-    if (CurAction.StopCompilation())
-      return 0;
-
-    const Edge* Edg = ChooseEdge(CurNode->OutEdges, InLangs, CurNode->Name());
-    if (Edg == 0)
-      return 1;
-
-    const Node* NextNode = getNode(Edg->ToolName());
-    if (NextNode == 0)
-      return 1;
-
-    if (int ret = PassThroughGraph(sys::Path(CurAction.OutFile()), NextNode,
-                                   InLangs, TempDir, LangMap)) {
-      return ret;
-    }
-  }
-
-  if (!WasSomeActionGenerated) {
-    PrintError("no input files");
-    return 1;
-  }
-
-  return 0;
-}
-
-int CompilationGraph::CheckLanguageNames() const {
-  int ret = 0;
-
-  // Check that names for output and input languages on all edges do match.
-  for (const_nodes_iterator B = this->NodesMap.begin(),
-         E = this->NodesMap.end(); B != E; ++B) {
-
-    const Node & N1 = B->second;
-    if (N1.ToolPtr) {
-      for (Node::const_iterator EB = N1.EdgesBegin(), EE = N1.EdgesEnd();
-           EB != EE; ++EB) {
-        const Node* N2 = this->getNode((*EB)->ToolName());
-        if (N2 == 0)
-          return 1;
-
-        if (!N2->ToolPtr) {
-          ++ret;
-          errs() << "Error: there is an edge from '" << N1.ToolPtr->Name()
-                 << "' back to the root!\n\n";
-          continue;
-        }
-
-        const char** OutLangs = N1.ToolPtr->OutputLanguages();
-        const char** InLangs = N2->ToolPtr->InputLanguages();
-        bool eq = false;
-        const char* OutLang = 0;
-        for (;*OutLangs; ++OutLangs) {
-          OutLang = *OutLangs;
-          for (;*InLangs; ++InLangs) {
-            if (std::strcmp(OutLang, *InLangs) == 0) {
-              eq = true;
-              break;
-            }
-          }
-        }
-
-        if (!eq) {
-          ++ret;
-          errs() << "Error: Output->input language mismatch in the edge '"
-                 << N1.ToolPtr->Name() << "' -> '" << N2->ToolPtr->Name()
-                 << "'!\n"
-                 << "Expected one of { ";
-
-          InLangs = N2->ToolPtr->InputLanguages();
-          for (;*InLangs; ++InLangs) {
-            errs() << '\'' << *InLangs << (*(InLangs+1) ? "', " : "'");
-          }
-
-          errs() << " }, but got '" << OutLang << "'!\n\n";
-        }
-
-      }
-    }
-  }
-
-  return ret;
-}
-
-int CompilationGraph::CheckMultipleDefaultEdges() const {
-  int ret = 0;
-  InputLanguagesSet Dummy;
-
-  // For all nodes, just iterate over the outgoing edges and check if there is
-  // more than one edge with maximum weight.
-  for (const_nodes_iterator B = this->NodesMap.begin(),
-         E = this->NodesMap.end(); B != E; ++B) {
-    const Node& N = B->second;
-    int MaxWeight = -1024;
-
-    // Ignore the root node.
-    if (!N.ToolPtr)
-      continue;
-
-    for (Node::const_iterator EB = N.EdgesBegin(), EE = N.EdgesEnd();
-         EB != EE; ++EB) {
-      int EdgeWeight = (*EB)->Weight(Dummy);
-      if (EdgeWeight > MaxWeight) {
-        MaxWeight = EdgeWeight;
-      }
-      else if (EdgeWeight == MaxWeight) {
-        ++ret;
-        errs() << "Error: there are multiple maximal edges stemming from the '"
-               << N.ToolPtr->Name() << "' node!\n\n";
-        break;
-      }
-    }
-  }
-
-  return ret;
-}
-
-int CompilationGraph::CheckCycles() {
-  unsigned deleted = 0;
-  std::queue<Node*> Q;
-
-  Node* Root = getNode("root");
-  if (Root == 0)
-    return 1;
-
-  Q.push(Root);
-
-  // Try to delete all nodes that have no ingoing edges, starting from the
-  // root. If there are any nodes left after this operation, then we have a
-  // cycle. This relies on '--check-graph' not performing the topological sort.
-  while (!Q.empty()) {
-    Node* A = Q.front();
-    Q.pop();
-    ++deleted;
-
-    for (Node::iterator EB = A->EdgesBegin(), EE = A->EdgesEnd();
-         EB != EE; ++EB) {
-      Node* B = getNode((*EB)->ToolName());
-      if (B == 0)
-        return 1;
-
-      B->DecrInEdges();
-      if (B->HasNoInEdges())
-        Q.push(B);
-    }
-  }
-
-  if (deleted != NodesMap.size()) {
-    errs() << "Error: there are cycles in the compilation graph!\n"
-           << "Try inspecting the diagram produced by "
-           << "'llvmc --view-graph'.\n\n";
-    return 1;
-  }
-
-  return 0;
-}
-
-int CompilationGraph::Check () {
-  // We try to catch as many errors as we can in one go.
-  int errs = 0;
-  int ret = 0;
-
-  // Check that output/input language names match.
-  ret = this->CheckLanguageNames();
-  if (ret < 0)
-    return 1;
-  errs += ret;
-
-  // Check for multiple default edges.
-  ret = this->CheckMultipleDefaultEdges();
-  if (ret < 0)
-    return 1;
-  errs += ret;
-
-  // Check for cycles.
-  ret = this->CheckCycles();
-  if (ret < 0)
-    return 1;
-  errs += ret;
-
-  return errs;
-}
-
-// Code related to graph visualization.
-
-namespace {
-
-std::string SquashStrArray (const char** StrArr) {
-  std::string ret;
-
-  for (; *StrArr; ++StrArr) {
-    if (*(StrArr + 1)) {
-      ret += *StrArr;
-      ret +=  ", ";
-    }
-    else {
-      ret += *StrArr;
-    }
-  }
-
-  return ret;
-}
-
-} // End anonymous namespace.
-
-namespace llvm {
-  template <>
-  struct DOTGraphTraits<llvmc::CompilationGraph*>
-    : public DefaultDOTGraphTraits
-  {
-    DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {}
-
-    template<typename GraphType>
-    static std::string getNodeLabel(const Node* N, const GraphType&)
-    {
-      if (N->ToolPtr)
-        if (N->ToolPtr->IsJoin())
-          return N->Name() + "\n (join" +
-            (N->HasChildren() ? ")"
-             : std::string(": ") +
-             SquashStrArray(N->ToolPtr->OutputLanguages()) + ')');
-        else
-          return N->Name();
-      else
-        return "root";
-    }
-
-    template<typename EdgeIter>
-    static std::string getEdgeSourceLabel(const Node* N, EdgeIter I) {
-      if (N->ToolPtr) {
-        return SquashStrArray(N->ToolPtr->OutputLanguages());
-      }
-      else {
-        return SquashStrArray(I->ToolPtr->InputLanguages());
-      }
-    }
-  };
-
-} // End namespace llvm
-
-int CompilationGraph::writeGraph(const std::string& OutputFilename) {
-  std::string ErrorInfo;
-  raw_fd_ostream O(OutputFilename.c_str(), ErrorInfo);
-
-  if (ErrorInfo.empty()) {
-    errs() << "Writing '"<< OutputFilename << "' file...";
-    llvm::WriteGraph(O, this);
-    errs() << "done.\n";
-  }
-  else {
-    PrintError("Error opening file '" + OutputFilename + "' for writing!");
-    return 1;
-  }
-
-  return 0;
-}
-
-void CompilationGraph::viewGraph() {
-  llvm::ViewGraph(this, "compilation-graph");
-}
diff --git a/contrib/llvm/lib/CompilerDriver/Main.cpp b/contrib/llvm/lib/CompilerDriver/Main.cpp
deleted file mode 100644
index 7120027f7ce0..000000000000
--- a/contrib/llvm/lib/CompilerDriver/Main.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-//===--- Main.cpp - The LLVM Compiler Driver --------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  llvmc::Main function - driver entry point.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CompilerDriver/AutoGenerated.h"
-#include "llvm/CompilerDriver/BuiltinOptions.h"
-#include "llvm/CompilerDriver/CompilationGraph.h"
-#include "llvm/CompilerDriver/Error.h"
-
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Path.h"
-
-#include <sstream>
-#include <string>
-
-namespace cl = llvm::cl;
-namespace sys = llvm::sys;
-using namespace llvmc;
-
-namespace {
-
-  std::stringstream* GlobalTimeLog;
-
-  /// GetTempDir - Get the temporary directory location. Returns non-zero value
-  /// on error.
-  int GetTempDir(sys::Path& tempDir) {
-    // The --temp-dir option.
-    if (!TempDirname.empty()) {
-      tempDir = TempDirname;
-    }
-    // GCC 4.5-style -save-temps handling.
-    else if (SaveTemps == SaveTempsEnum::Unset) {
-      tempDir = sys::Path::GetTemporaryDirectory();
-      return 0;
-    }
-    else if (SaveTemps == SaveTempsEnum::Obj && !OutputFilename.empty()) {
-      tempDir = sys::path::parent_path(OutputFilename);
-    }
-    else {
-      // SaveTemps == Cwd --> use current dir (leave tempDir empty).
-      return 0;
-    }
-
-    bool Exists;
-    if (llvm::sys::fs::exists(tempDir.str(), Exists) || !Exists) {
-      std::string ErrMsg;
-      if (tempDir.createDirectoryOnDisk(true, &ErrMsg)) {
-        PrintError(ErrMsg);
-        return 1;
-      }
-    }
-
-    return 0;
-  }
-
-  /// BuildTargets - A small wrapper for CompilationGraph::Build. Returns
-  /// non-zero value in case of error.
-  int BuildTargets(CompilationGraph& graph, const LanguageMap& langMap) {
-    int ret;
-    sys::Path tempDir;
-    bool toDelete = (SaveTemps == SaveTempsEnum::Unset);
-
-    if (int ret = GetTempDir(tempDir))
-      return ret;
-
-    ret = graph.Build(tempDir, langMap);
-
-    if (toDelete)
-      tempDir.eraseFromDisk(true);
-
-    return ret;
-  }
-}
-
-namespace llvmc {
-
-// Used to implement -time option. External linkage is intentional.
-void AppendToGlobalTimeLog(const std::string& cmd, double time) {
-  *GlobalTimeLog << "# " << cmd << ' ' << time << '\n';
-}
-
-// Sometimes user code wants to access the argv[0] value.
-const char* ProgramName;
-
-int Main(int argc, char** argv) {
-  int ret = 0;
-  LanguageMap langMap;
-  CompilationGraph graph;
-
-  ProgramName = argv[0];
-
-  cl::ParseCommandLineOptions
-    (argc, argv,
-     /* Overview = */ "LLVM Compiler Driver (Work In Progress)",
-     /* ReadResponseFiles = */ false);
-
-  if (int ret = autogenerated::RunInitialization(langMap, graph))
-    return ret;
-
-  if (CheckGraph) {
-    ret = graph.Check();
-    if (!ret)
-      llvm::errs() << "check-graph: no errors found.\n";
-
-    return ret;
-  }
-
-  if (ViewGraph) {
-    graph.viewGraph();
-    if (!WriteGraph)
-      return 0;
-  }
-
-  if (WriteGraph) {
-    const std::string& Out = (OutputFilename.empty()
-                              ? std::string("compilation-graph.dot")
-                              : OutputFilename);
-    return graph.writeGraph(Out);
-  }
-
-  if (Time) {
-    GlobalTimeLog = new std::stringstream;
-    GlobalTimeLog->precision(2);
-  }
-
-  ret = BuildTargets(graph, langMap);
-
-  if (Time) {
-    llvm::errs() << GlobalTimeLog->str();
-    delete GlobalTimeLog;
-  }
-
-  return ret;
-}
-
-} // end namespace llvmc
diff --git a/contrib/llvm/lib/CompilerDriver/Tool.cpp b/contrib/llvm/lib/CompilerDriver/Tool.cpp
deleted file mode 100644
index 876759aa72b0..000000000000
--- a/contrib/llvm/lib/CompilerDriver/Tool.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-//===--- Tool.cpp - The LLVM Compiler Driver --------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Tool base class - implementation details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CompilerDriver/BuiltinOptions.h"
-#include "llvm/CompilerDriver/Tool.h"
-
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/Path.h"
-
-#include <algorithm>
-
-using namespace llvm;
-using namespace llvmc;
-
-namespace {
-  sys::Path MakeTempFile(const sys::Path& TempDir, const std::string& BaseName,
-                         const std::string& Suffix) {
-    sys::Path Out;
-
-    // Make sure we don't end up with path names like '/file.o' if the
-    // TempDir is empty.
-    if (TempDir.empty()) {
-      Out.set(BaseName);
-    }
-    else {
-      Out = TempDir;
-      Out.appendComponent(BaseName);
-    }
-    Out.appendSuffix(Suffix);
-    // NOTE: makeUnique always *creates* a unique temporary file,
-    // which is good, since there will be no races. However, some
-    // tools do not like it when the output file already exists, so
-    // they need to be placated with -f or something like that.
-    Out.makeUnique(true, NULL);
-    return Out;
-  }
-}
-
-sys::Path Tool::OutFilename(const sys::Path& In,
-                            const sys::Path& TempDir,
-                            bool StopCompilation,
-                            const char* OutputSuffix) const {
-  sys::Path Out;
-
-  if (StopCompilation) {
-    if (!OutputFilename.empty()) {
-      Out.set(OutputFilename);
-    }
-    else if (IsJoin()) {
-      Out.set("a");
-      Out.appendSuffix(OutputSuffix);
-    }
-    else {
-      Out.set(sys::path::stem(In.str()));
-      Out.appendSuffix(OutputSuffix);
-    }
-  }
-  else {
-    if (IsJoin())
-      Out = MakeTempFile(TempDir, "tmp", OutputSuffix);
-    else
-      Out = MakeTempFile(TempDir, sys::path::stem(In.str()), OutputSuffix);
-  }
-  return Out;
-}
-
-namespace {
-  template <class A, class B>
-  bool CompareFirst (std::pair<A,B> p1, std::pair<A,B> p2) {
-    return std::less<A>()(p1.first, p2.first);
-  }
-}
-
-StrVector Tool::SortArgs(ArgsVector& Args) const {
-  StrVector Out;
-
-  // HACK: this won't be needed when we'll migrate away from CommandLine.
-  std::stable_sort(Args.begin(), Args.end(),
-                   &CompareFirst<unsigned, std::string>);
-  for (ArgsVector::iterator B = Args.begin(), E = Args.end(); B != E; ++B) {
-    Out.push_back(B->second);
-  }
-
-  return Out;
-}
diff --git a/contrib/llvm/lib/DebugInfo/DIContext.cpp b/contrib/llvm/lib/DebugInfo/DIContext.cpp
new file mode 100644
index 000000000000..e2fd55fd6ef8
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DIContext.cpp
@@ -0,0 +1,24 @@
+//===-- DIContext.cpp -----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/DIContext.h"
+#include "DWARFContext.h"
+using namespace llvm;
+
+DIContext::~DIContext() {}
+
+DIContext *DIContext::getDWARFContext(bool isLittleEndian,
+                                      StringRef infoSection,
+                                      StringRef abbrevSection,
+                                      StringRef aRangeSection,
+                                      StringRef lineSection,
+                                      StringRef stringSection) {
+  return new DWARFContextInMemory(isLittleEndian, infoSection, abbrevSection,
+                                  aRangeSection, lineSection, stringSection);
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.cpp b/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.cpp
new file mode 100644
index 000000000000..0df692c3a3b7
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.cpp
@@ -0,0 +1,83 @@
+//===-- DWARFAbbreviationDeclaration.cpp ----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFAbbreviationDeclaration.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+using namespace dwarf;
+
+bool
+DWARFAbbreviationDeclaration::extract(DataExtractor data, uint32_t* offset_ptr){
+  return extract(data, offset_ptr, data.getULEB128(offset_ptr));
+}
+
+bool
+DWARFAbbreviationDeclaration::extract(DataExtractor data, uint32_t* offset_ptr,
+                                      uint32_t code) {
+  Code = code;
+  Attributes.clear();
+  if (Code) {
+    Tag = data.getULEB128(offset_ptr);
+    HasChildren = data.getU8(offset_ptr);
+
+    while (data.isValidOffset(*offset_ptr)) {
+      uint16_t attr = data.getULEB128(offset_ptr);
+      uint16_t form = data.getULEB128(offset_ptr);
+
+      if (attr && form)
+        Attributes.push_back(DWARFAttribute(attr, form));
+      else
+        break;
+    }
+
+    return Tag != 0;
+  } else {
+    Tag = 0;
+    HasChildren = false;
+  }
+
+  return false;
+}
+
+void DWARFAbbreviationDeclaration::dump(raw_ostream &OS) const {
+  const char *tagString = TagString(getTag());
+  OS << '[' << getCode() << "] ";
+  if (tagString)
+    OS << tagString;
+  else
+    OS << format("DW_TAG_Unknown_%x", getTag());
+  OS << "\tDW_CHILDREN_" << (hasChildren() ? "yes" : "no") << '\n';
+  for (unsigned i = 0, e = Attributes.size(); i != e; ++i) {
+    OS << '\t';
+    const char *attrString = AttributeString(Attributes[i].getAttribute());
+    if (attrString)
+      OS << attrString;
+    else
+      OS << format("DW_AT_Unknown_%x", Attributes[i].getAttribute());
+    OS << '\t';
+    const char *formString = FormEncodingString(Attributes[i].getForm());
+    if (formString)
+      OS << formString;
+    else
+      OS << format("DW_FORM_Unknown_%x", Attributes[i].getForm());
+    OS << '\n';
+  }
+  OS << '\n';
+}
+
+uint32_t
+DWARFAbbreviationDeclaration::findAttributeIndex(uint16_t attr) const {
+  for (uint32_t i = 0, e = Attributes.size(); i != e; ++i) {
+    if (Attributes[i].getAttribute() == attr)
+      return i;
+  }
+  return -1U;
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.h b/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.h
new file mode 100644
index 000000000000..2463a3cc0494
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.h
@@ -0,0 +1,54 @@
+//===-- DWARFAbbreviationDeclaration.h --------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFABBREVIATIONDECLARATION_H
+#define LLVM_DEBUGINFO_DWARFABBREVIATIONDECLARATION_H
+
+#include "DWARFAttribute.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/DataExtractor.h"
+
+namespace llvm {
+
+class raw_ostream;
+
+class DWARFAbbreviationDeclaration {
+  uint32_t Code;
+  uint32_t Tag;
+  bool HasChildren;
+  SmallVector<DWARFAttribute, 8> Attributes;
+public:
+  enum { InvalidCode = 0 };
+  DWARFAbbreviationDeclaration()
+    : Code(InvalidCode), Tag(0), HasChildren(0) {}
+
+  uint32_t getCode() const { return Code; }
+  uint32_t getTag() const { return Tag; }
+  bool hasChildren() const { return HasChildren; }
+  uint32_t getNumAttributes() const { return Attributes.size(); }
+  uint16_t getAttrByIndex(uint32_t idx) const {
+    return Attributes.size() > idx ? Attributes[idx].getAttribute() : 0;
+  }
+  uint16_t getFormByIndex(uint32_t idx) const {
+    return Attributes.size() > idx ? Attributes[idx].getForm() : 0;
+  }
+
+  uint32_t findAttributeIndex(uint16_t attr) const;
+  bool extract(DataExtractor data, uint32_t* offset_ptr);
+  bool extract(DataExtractor data, uint32_t* offset_ptr, uint32_t code);
+  bool isValid() const { return Code != 0 && Tag != 0; }
+  void dump(raw_ostream &OS) const;
+  const SmallVectorImpl<DWARFAttribute> &getAttributes() const {
+    return Attributes;
+  }
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFAttribute.h b/contrib/llvm/lib/DebugInfo/DWARFAttribute.h
new file mode 100644
index 000000000000..6f49b637c6a6
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFAttribute.h
@@ -0,0 +1,30 @@
+//===-- DWARFAttribute.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFATTRIBUTE_H
+#define LLVM_DEBUGINFO_DWARFATTRIBUTE_H
+
+#include "llvm/Support/DataTypes.h"
+
+namespace llvm {
+
+class DWARFAttribute {
+  uint16_t Attribute;
+  uint16_t Form;
+  public:
+  DWARFAttribute(uint16_t attr, uint16_t form)
+    : Attribute(attr), Form(form) {}
+
+  uint16_t getAttribute() const { return Attribute; }
+  uint16_t getForm() const { return Form; }
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.cpp b/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.cpp
new file mode 100644
index 000000000000..24bf97ff608a
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.cpp
@@ -0,0 +1,238 @@
+//===-- DWARFCompileUnit.cpp ----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFCompileUnit.h"
+#include "DWARFContext.h"
+#include "DWARFFormValue.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+using namespace dwarf;
+
+DataExtractor DWARFCompileUnit::getDebugInfoExtractor() const {
+  return DataExtractor(Context.getInfoSection(),
+                       Context.isLittleEndian(), getAddressByteSize());
+}
+
+bool DWARFCompileUnit::extract(DataExtractor debug_info, uint32_t *offset_ptr) {
+  clear();
+
+  Offset = *offset_ptr;
+
+  if (debug_info.isValidOffset(*offset_ptr)) {
+    uint64_t abbrOffset;
+    const DWARFDebugAbbrev *abbr = Context.getDebugAbbrev();
+    Length = debug_info.getU32(offset_ptr);
+    Version = debug_info.getU16(offset_ptr);
+    abbrOffset = debug_info.getU32(offset_ptr);
+    AddrSize = debug_info.getU8(offset_ptr);
+
+    bool lengthOK = debug_info.isValidOffset(getNextCompileUnitOffset()-1);
+    bool versionOK = DWARFContext::isSupportedVersion(Version);
+    bool abbrOffsetOK = Context.getAbbrevSection().size() > abbrOffset;
+    bool addrSizeOK = AddrSize == 4 || AddrSize == 8;
+
+    if (lengthOK && versionOK && addrSizeOK && abbrOffsetOK && abbr != NULL) {
+      Abbrevs = abbr->getAbbreviationDeclarationSet(abbrOffset);
+      return true;
+    }
+
+    // reset the offset to where we tried to parse from if anything went wrong
+    *offset_ptr = Offset;
+  }
+
+  return false;
+}
+
+uint32_t
+DWARFCompileUnit::extract(uint32_t offset, DataExtractor debug_info_data,
+                          const DWARFAbbreviationDeclarationSet *abbrevs) {
+  clear();
+
+  Offset = offset;
+
+  if (debug_info_data.isValidOffset(offset)) {
+    Length = debug_info_data.getU32(&offset);
+    Version = debug_info_data.getU16(&offset);
+    bool abbrevsOK = debug_info_data.getU32(&offset) == abbrevs->getOffset();
+    Abbrevs = abbrevs;
+    AddrSize = debug_info_data.getU8 (&offset);
+
+    bool versionOK = DWARFContext::isSupportedVersion(Version);
+    bool addrSizeOK = AddrSize == 4 || AddrSize == 8;
+
+    if (versionOK && addrSizeOK && abbrevsOK &&
+        debug_info_data.isValidOffset(offset))
+      return offset;
+  }
+  return 0;
+}
+
+void DWARFCompileUnit::clear() {
+  Offset = 0;
+  Length = 0;
+  Version = 0;
+  Abbrevs = 0;
+  AddrSize = 0;
+  BaseAddr = 0;
+  DieArray.clear();
+}
+
+void DWARFCompileUnit::dump(raw_ostream &OS) {
+  OS << format("0x%08x", Offset) << ": Compile Unit:"
+     << " length = " << format("0x%08x", Length)
+     << " version = " << format("0x%04x", Version)
+     << " abbr_offset = " << format("0x%04x", Abbrevs->getOffset())
+     << " addr_size = " << format("0x%02x", AddrSize)
+     << " (next CU at " << format("0x%08x", getNextCompileUnitOffset())
+     << ")\n";
+
+  getCompileUnitDIE(false)->dump(OS, this, -1U);
+}
+
+void DWARFCompileUnit::setDIERelations() {
+  if (DieArray.empty())
+    return;
+  DWARFDebugInfoEntryMinimal *die_array_begin = &DieArray.front();
+  DWARFDebugInfoEntryMinimal *die_array_end = &DieArray.back();
+  DWARFDebugInfoEntryMinimal *curr_die;
+  // We purposely are skipping the last element in the array in the loop below
+  // so that we can always have a valid next item
+  for (curr_die = die_array_begin; curr_die < die_array_end; ++curr_die) {
+    // Since our loop doesn't include the last element, we can always
+    // safely access the next die in the array.
+    DWARFDebugInfoEntryMinimal *next_die = curr_die + 1;
+
+    const DWARFAbbreviationDeclaration *curr_die_abbrev =
+      curr_die->getAbbreviationDeclarationPtr();
+
+    if (curr_die_abbrev) {
+      // Normal DIE
+      if (curr_die_abbrev->hasChildren())
+        next_die->setParent(curr_die);
+      else
+        curr_die->setSibling(next_die);
+    } else {
+      // NULL DIE that terminates a sibling chain
+      DWARFDebugInfoEntryMinimal *parent = curr_die->getParent();
+      if (parent)
+        parent->setSibling(next_die);
+    }
+  }
+
+  // Since we skipped the last element, we need to fix it up!
+  if (die_array_begin < die_array_end)
+    curr_die->setParent(die_array_begin);
+}
+
+size_t DWARFCompileUnit::extractDIEsIfNeeded(bool cu_die_only) {
+  const size_t initial_die_array_size = DieArray.size();
+  if ((cu_die_only && initial_die_array_size > 0) ||
+      initial_die_array_size > 1)
+    return 0; // Already parsed
+
+  // Set the offset to that of the first DIE and calculate the start of the
+  // next compilation unit header.
+  uint32_t offset = getFirstDIEOffset();
+  uint32_t next_cu_offset = getNextCompileUnitOffset();
+
+  DWARFDebugInfoEntryMinimal die;
+  // Keep a flat array of the DIE for binary lookup by DIE offset
+  uint32_t depth = 0;
+  // We are in our compile unit, parse starting at the offset
+  // we were told to parse
+
+  const uint8_t *fixed_form_sizes =
+    DWARFFormValue::getFixedFormSizesForAddressSize(getAddressByteSize());
+
+  while (offset < next_cu_offset &&
+         die.extractFast(this, fixed_form_sizes, &offset)) {
+
+    if (depth == 0) {
+      uint64_t base_addr =
+        die.getAttributeValueAsUnsigned(this, DW_AT_low_pc, -1U);
+      if (base_addr == -1U)
+        base_addr = die.getAttributeValueAsUnsigned(this, DW_AT_entry_pc, 0);
+      setBaseAddress(base_addr);
+    }
+
+    if (cu_die_only) {
+      addDIE(die);
+      return 1;
+    }
+    else if (depth == 0 && initial_die_array_size == 1) {
+      // Don't append the CU die as we already did that
+    } else {
+      addDIE (die);
+    }
+
+    const DWARFAbbreviationDeclaration *abbrDecl =
+      die.getAbbreviationDeclarationPtr();
+    if (abbrDecl) {
+      // Normal DIE
+      if (abbrDecl->hasChildren())
+        ++depth;
+    } else {
+      // NULL DIE.
+      if (depth > 0)
+        --depth;
+      if (depth == 0)
+        break;  // We are done with this compile unit!
+    }
+
+  }
+
+  // Give a little bit of info if we encounter corrupt DWARF (our offset
+  // should always terminate at or before the start of the next compilation
+  // unit header).
+  if (offset > next_cu_offset) {
+    fprintf (stderr, "warning: DWARF compile unit extends beyond its bounds cu 0x%8.8x at 0x%8.8x'\n", getOffset(), offset);
+  }
+
+  setDIERelations();
+  return DieArray.size();
+}
+
+void DWARFCompileUnit::clearDIEs(bool keep_compile_unit_die) {
+  if (DieArray.size() > 1) {
+    // std::vectors never get any smaller when resized to a smaller size,
+    // or when clear() or erase() are called, the size will report that it
+    // is smaller, but the memory allocated remains intact (call capacity()
+    // to see this). So we need to create a temporary vector and swap the
+    // contents which will cause just the internal pointers to be swapped
+    // so that when "tmp_array" goes out of scope, it will destroy the
+    // contents.
+
+    // Save at least the compile unit DIE
+    std::vector<DWARFDebugInfoEntryMinimal> tmpArray;
+    DieArray.swap(tmpArray);
+    if (keep_compile_unit_die)
+      DieArray.push_back(tmpArray.front());
+  }
+}
+
+void
+DWARFCompileUnit::buildAddressRangeTable(DWARFDebugAranges *debug_aranges,
+                                         bool clear_dies_if_already_not_parsed){
+  // This function is usually called if there in no .debug_aranges section
+  // in order to produce a compile unit level set of address ranges that
+  // is accurate. If the DIEs weren't parsed, then we don't want all dies for
+  // all compile units to stay loaded when they weren't needed. So we can end
+  // up parsing the DWARF and then throwing them all away to keep memory usage
+  // down.
+  const bool clear_dies = extractDIEsIfNeeded(false) > 1;
+
+  DieArray[0].buildAddressRangeTable(this, debug_aranges);
+
+  // Keep memory down by clearing DIEs if this generate function
+  // caused them to be parsed.
+  if (clear_dies)
+    clearDIEs(true);
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.h b/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.h
new file mode 100644
index 000000000000..d9167292a9df
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.h
@@ -0,0 +1,111 @@
+//===-- DWARFCompileUnit.h --------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFCOMPILEUNIT_H
+#define LLVM_DEBUGINFO_DWARFCOMPILEUNIT_H
+
+#include "DWARFDebugAbbrev.h"
+#include "DWARFDebugInfoEntry.h"
+#include <vector>
+
+namespace llvm {
+
+class DWARFContext;
+class raw_ostream;
+
+class DWARFCompileUnit {
+  DWARFContext &Context;
+
+  uint32_t Offset;
+  uint32_t Length;
+  uint16_t Version;
+  const DWARFAbbreviationDeclarationSet *Abbrevs;
+  uint8_t AddrSize;
+  uint64_t BaseAddr;
+  // The compile unit debug information entry item.
+  std::vector<DWARFDebugInfoEntryMinimal> DieArray;
+public:
+  DWARFCompileUnit(DWARFContext &context) : Context(context) {
+    clear();
+  }
+
+  DWARFContext &getContext() const { return Context; }
+  DataExtractor getDebugInfoExtractor() const;
+
+  bool extract(DataExtractor debug_info, uint32_t* offset_ptr);
+  uint32_t extract(uint32_t offset, DataExtractor debug_info_data,
+                   const DWARFAbbreviationDeclarationSet *abbrevs);
+
+  /// extractDIEsIfNeeded - Parses a compile unit and indexes its DIEs if it
+  /// hasn't already been done.
+  size_t extractDIEsIfNeeded(bool cu_die_only);
+  void clear();
+  void dump(raw_ostream &OS);
+  uint32_t getOffset() const { return Offset; }
+  /// Size in bytes of the compile unit header.
+  uint32_t getSize() const { return 11; }
+  bool containsDIEOffset(uint32_t die_offset) const {
+    return die_offset >= getFirstDIEOffset() &&
+      die_offset < getNextCompileUnitOffset();
+  }
+  uint32_t getFirstDIEOffset() const { return Offset + getSize(); }
+  uint32_t getNextCompileUnitOffset() const { return Offset + Length + 4; }
+  /// Size in bytes of the .debug_info data associated with this compile unit.
+  size_t getDebugInfoSize() const { return Length + 4 - getSize(); }
+  uint32_t getLength() const { return Length; }
+  uint16_t getVersion() const { return Version; }
+  const DWARFAbbreviationDeclarationSet *getAbbreviations() const {
+    return Abbrevs;
+  }
+  uint8_t getAddressByteSize() const { return AddrSize; }
+  uint64_t getBaseAddress() const { return BaseAddr; }
+
+  void setBaseAddress(uint64_t base_addr) {
+    BaseAddr = base_addr;
+  }
+
+  const DWARFDebugInfoEntryMinimal *
+  getCompileUnitDIE(bool extract_cu_die_only = true) {
+    extractDIEsIfNeeded(extract_cu_die_only);
+    if (DieArray.empty())
+      return NULL;
+    return &DieArray[0];
+  }
+
+  /// setDIERelations - We read in all of the DIE entries into our flat list
+  /// of DIE entries and now we need to go back through all of them and set the
+  /// parent, sibling and child pointers for quick DIE navigation.
+  void setDIERelations();
+
+  void addDIE(DWARFDebugInfoEntryMinimal &die) {
+    // The average bytes per DIE entry has been seen to be
+    // around 14-20 so lets pre-reserve the needed memory for
+    // our DIE entries accordingly. Search forward for "Compute
+    // average bytes per DIE" to see #if'ed out code that does
+    // that determination.
+
+    // Only reserve the memory if we are adding children of
+    // the main compile unit DIE. The compile unit DIE is always
+    // the first entry, so if our size is 1, then we are adding
+    // the first compile unit child DIE and should reserve
+    // the memory.
+    if (DieArray.empty())
+      DieArray.reserve(getDebugInfoSize() / 14);
+    DieArray.push_back(die);
+  }
+
+  void clearDIEs(bool keep_compile_unit_die);
+
+  void buildAddressRangeTable(DWARFDebugAranges *debug_aranges,
+                              bool clear_dies_if_already_not_parsed);
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFContext.cpp b/contrib/llvm/lib/DebugInfo/DWARFContext.cpp
new file mode 100644
index 000000000000..e1ac398b1011
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFContext.cpp
@@ -0,0 +1,167 @@
+//===-- DWARFContext.cpp --------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFContext.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+using namespace llvm;
+using namespace dwarf;
+
+void DWARFContext::dump(raw_ostream &OS) {
+  OS << ".debug_abbrev contents:\n";
+  getDebugAbbrev()->dump(OS);
+
+  OS << "\n.debug_info contents:\n";
+  for (unsigned i = 0, e = getNumCompileUnits(); i != e; ++i)
+    getCompileUnitAtIndex(i)->dump(OS);
+
+  OS << "\n.debug_aranges contents:\n";
+  DataExtractor arangesData(getARangeSection(), isLittleEndian(), 0);
+  uint32_t offset = 0;
+  DWARFDebugArangeSet set;
+  while (set.extract(arangesData, &offset))
+    set.dump(OS);
+
+  OS << "\n.debug_lines contents:\n";
+  for (unsigned i = 0, e = getNumCompileUnits(); i != e; ++i) {
+    DWARFCompileUnit *cu = getCompileUnitAtIndex(i);
+    unsigned stmtOffset =
+      cu->getCompileUnitDIE()->getAttributeValueAsUnsigned(cu, DW_AT_stmt_list,
+                                                           -1U);
+    if (stmtOffset != -1U) {
+      DataExtractor lineData(getLineSection(), isLittleEndian(),
+                             cu->getAddressByteSize());
+      DWARFDebugLine::DumpingState state(OS);
+      DWARFDebugLine::parseStatementTable(lineData, &stmtOffset, state);
+    }
+  }
+
+  OS << "\n.debug_str contents:\n";
+  DataExtractor strData(getStringSection(), isLittleEndian(), 0);
+  offset = 0;
+  uint32_t lastOffset = 0;
+  while (const char *s = strData.getCStr(&offset)) {
+    OS << format("0x%8.8x: \"%s\"\n", lastOffset, s);
+    lastOffset = offset;
+  }
+}
+
+const DWARFDebugAbbrev *DWARFContext::getDebugAbbrev() {
+  if (Abbrev)
+    return Abbrev.get();
+
+  DataExtractor abbrData(getAbbrevSection(), isLittleEndian(), 0);
+
+  Abbrev.reset(new DWARFDebugAbbrev());
+  Abbrev->parse(abbrData);
+  return Abbrev.get();
+}
+
+const DWARFDebugAranges *DWARFContext::getDebugAranges() {
+  if (Aranges)
+    return Aranges.get();
+
+  DataExtractor arangesData(getARangeSection(), isLittleEndian(), 0);
+
+  Aranges.reset(new DWARFDebugAranges());
+  Aranges->extract(arangesData);
+  if (Aranges->isEmpty()) // No aranges in file, generate them from the DIEs.
+    Aranges->generate(this);
+  return Aranges.get();
+}
+
+const DWARFDebugLine::LineTable *
+DWARFContext::getLineTableForCompileUnit(DWARFCompileUnit *cu) {
+  if (!Line)
+    Line.reset(new DWARFDebugLine());
+
+  unsigned stmtOffset =
+    cu->getCompileUnitDIE()->getAttributeValueAsUnsigned(cu, DW_AT_stmt_list,
+                                                         -1U);
+  if (stmtOffset == -1U)
+    return 0; // No line table for this compile unit.
+
+  // See if the line table is cached.
+  if (const DWARFDebugLine::LineTable *lt = Line->getLineTable(stmtOffset))
+    return lt;
+
+  // We have to parse it first.
+  DataExtractor lineData(getLineSection(), isLittleEndian(),
+                         cu->getAddressByteSize());
+  return Line->getOrParseLineTable(lineData, stmtOffset);
+}
+
+void DWARFContext::parseCompileUnits() {
+  uint32_t offset = 0;
+  const DataExtractor &debug_info_data = DataExtractor(getInfoSection(),
+                                                       isLittleEndian(), 0);
+  while (debug_info_data.isValidOffset(offset)) {
+    CUs.push_back(DWARFCompileUnit(*this));
+    if (!CUs.back().extract(debug_info_data, &offset)) {
+      CUs.pop_back();
+      break;
+    }
+
+    offset = CUs.back().getNextCompileUnitOffset();
+  }
+}
+
+namespace {
+  struct OffsetComparator {
+    bool operator()(const DWARFCompileUnit &LHS,
+                    const DWARFCompileUnit &RHS) const {
+      return LHS.getOffset() < RHS.getOffset();
+    }
+    bool operator()(const DWARFCompileUnit &LHS, uint32_t RHS) const {
+      return LHS.getOffset() < RHS;
+    }
+    bool operator()(uint32_t LHS, const DWARFCompileUnit &RHS) const {
+      return LHS < RHS.getOffset();
+    }
+  };
+}
+
+DWARFCompileUnit *DWARFContext::getCompileUnitForOffset(uint32_t offset) {
+  if (CUs.empty())
+    parseCompileUnits();
+
+  DWARFCompileUnit *i = std::lower_bound(CUs.begin(), CUs.end(), offset,
+                                         OffsetComparator());
+  if (i != CUs.end())
+    return &*i;
+  return 0;
+}
+
+DILineInfo DWARFContext::getLineInfoForAddress(uint64_t address) {
+  // First, get the offset of the compile unit.
+  uint32_t cuOffset = getDebugAranges()->findAddress(address);
+  // Retrieve the compile unit.
+  DWARFCompileUnit *cu = getCompileUnitForOffset(cuOffset);
+  if (!cu)
+    return DILineInfo("<invalid>", 0, 0);
+  // Get the line table for this compile unit.
+  const DWARFDebugLine::LineTable *lineTable = getLineTableForCompileUnit(cu);
+  if (!lineTable)
+    return DILineInfo("<invalid>", 0, 0);
+  // Get the index of the row we're looking for in the line table.
+  uint64_t hiPC =
+    cu->getCompileUnitDIE()->getAttributeValueAsUnsigned(cu, DW_AT_high_pc,
+                                                         -1ULL);
+  uint32_t rowIndex = lineTable->lookupAddress(address, hiPC);
+  if (rowIndex == -1U)
+    return DILineInfo("<invalid>", 0, 0);
+
+  // From here, contruct the DILineInfo.
+  const DWARFDebugLine::Row &row = lineTable->Rows[rowIndex];
+  const std::string &fileName = lineTable->Prologue.FileNames[row.File-1].Name;
+
+  return DILineInfo(fileName.c_str(), row.Line, row.Column);
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFContext.h b/contrib/llvm/lib/DebugInfo/DWARFContext.h
new file mode 100644
index 000000000000..746a4639f277
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFContext.h
@@ -0,0 +1,118 @@
+//===-- DWARFContext.h ------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===/
+
+#ifndef LLVM_DEBUGINFO_DWARFCONTEXT_H
+#define LLVM_DEBUGINFO_DWARFCONTEXT_H
+
+#include "DWARFCompileUnit.h"
+#include "DWARFDebugAranges.h"
+#include "DWARFDebugLine.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+/// DWARFContext
+/// This data structure is the top level entity that deals with dwarf debug
+/// information parsing. The actual data is supplied through pure virtual
+/// methods that a concrete implementation provides.
+class DWARFContext : public DIContext {
+  bool IsLittleEndian;
+
+  SmallVector<DWARFCompileUnit, 1> CUs;
+  OwningPtr<DWARFDebugAbbrev> Abbrev;
+  OwningPtr<DWARFDebugAranges> Aranges;
+  OwningPtr<DWARFDebugLine> Line;
+
+  DWARFContext(DWARFContext &); // = delete
+  DWARFContext &operator=(DWARFContext &); // = delete
+
+  /// Read compile units from the debug_info section and store them in CUs.
+  void parseCompileUnits();
+protected:
+  DWARFContext(bool isLittleEndian) : IsLittleEndian(isLittleEndian) {}
+public:
+  virtual void dump(raw_ostream &OS);
+  /// Get the number of compile units in this context.
+  unsigned getNumCompileUnits() {
+    if (CUs.empty())
+      parseCompileUnits();
+    return CUs.size();
+  }
+  /// Get the compile unit at the specified index for this compile unit.
+  DWARFCompileUnit *getCompileUnitAtIndex(unsigned index) {
+    if (CUs.empty())
+      parseCompileUnits();
+    return &CUs[index];
+  }
+
+  /// Return the compile unit that includes an offset (relative to .debug_info).
+  DWARFCompileUnit *getCompileUnitForOffset(uint32_t offset);
+
+  /// Get a pointer to the parsed DebugAbbrev object.
+  const DWARFDebugAbbrev *getDebugAbbrev();
+
+  /// Get a pointer to the parsed DebugAranges object.
+  const DWARFDebugAranges *getDebugAranges();
+
+  /// Get a pointer to a parsed line table corresponding to a compile unit.
+  const DWARFDebugLine::LineTable *
+  getLineTableForCompileUnit(DWARFCompileUnit *cu);
+
+  virtual DILineInfo getLineInfoForAddress(uint64_t address);
+
+  bool isLittleEndian() const { return IsLittleEndian; }
+
+  virtual StringRef getInfoSection() = 0;
+  virtual StringRef getAbbrevSection() = 0;
+  virtual StringRef getARangeSection() = 0;
+  virtual StringRef getLineSection() = 0;
+  virtual StringRef getStringSection() = 0;
+
+  static bool isSupportedVersion(unsigned version) {
+    return version == 2 || version == 3;
+  }
+};
+
+
+/// DWARFContextInMemory is the simplest possible implementation of a
+/// DWARFContext. It assumes all content is available in memory and stores
+/// pointers to it.
+class DWARFContextInMemory : public DWARFContext {
+  StringRef InfoSection;
+  StringRef AbbrevSection;
+  StringRef ARangeSection;
+  StringRef LineSection;
+  StringRef StringSection;
+public:
+  DWARFContextInMemory(bool isLittleEndian,
+                       StringRef infoSection,
+                       StringRef abbrevSection,
+                       StringRef aRangeSection,
+                       StringRef lineSection,
+                       StringRef stringSection)
+    : DWARFContext(isLittleEndian),
+      InfoSection(infoSection),
+      AbbrevSection(abbrevSection),
+      ARangeSection(aRangeSection),
+      LineSection(lineSection),
+      StringSection(stringSection)
+    {}
+
+  virtual StringRef getInfoSection() { return InfoSection; }
+  virtual StringRef getAbbrevSection() { return AbbrevSection; }
+  virtual StringRef getARangeSection() { return ARangeSection; }
+  virtual StringRef getLineSection() { return LineSection; }
+  virtual StringRef getStringSection() { return StringSection; }
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.cpp b/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.cpp
new file mode 100644
index 000000000000..a11ae3f2908e
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.cpp
@@ -0,0 +1,106 @@
+//===-- DWARFDebugAbbrev.cpp ----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFDebugAbbrev.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+bool DWARFAbbreviationDeclarationSet::extract(DataExtractor data,
+                                              uint32_t* offset_ptr) {
+  const uint32_t beginOffset = *offset_ptr;
+  Offset = beginOffset;
+  clear();
+  DWARFAbbreviationDeclaration abbrevDeclaration;
+  uint32_t prevAbbrAode = 0;
+  while (abbrevDeclaration.extract(data, offset_ptr)) {
+    Decls.push_back(abbrevDeclaration);
+    if (IdxOffset == 0) {
+      IdxOffset = abbrevDeclaration.getCode();
+    } else {
+      if (prevAbbrAode + 1 != abbrevDeclaration.getCode())
+        IdxOffset = UINT32_MAX;// Out of order indexes, we can't do O(1) lookups
+    }
+    prevAbbrAode = abbrevDeclaration.getCode();
+  }
+  return beginOffset != *offset_ptr;
+}
+
+void DWARFAbbreviationDeclarationSet::dump(raw_ostream &OS) const {
+  for (unsigned i = 0, e = Decls.size(); i != e; ++i)
+    Decls[i].dump(OS);
+}
+
+const DWARFAbbreviationDeclaration*
+DWARFAbbreviationDeclarationSet::getAbbreviationDeclaration(uint32_t abbrCode)
+  const {
+  if (IdxOffset == UINT32_MAX) {
+    DWARFAbbreviationDeclarationCollConstIter pos;
+    DWARFAbbreviationDeclarationCollConstIter end = Decls.end();
+    for (pos = Decls.begin(); pos != end; ++pos) {
+      if (pos->getCode() == abbrCode)
+        return &(*pos);
+    }
+  } else {
+    uint32_t idx = abbrCode - IdxOffset;
+    if (idx < Decls.size())
+      return &Decls[idx];
+  }
+  return NULL;
+}
+
+DWARFDebugAbbrev::DWARFDebugAbbrev() :
+  AbbrevCollMap(),
+  PrevAbbrOffsetPos(AbbrevCollMap.end()) {}
+
+
+void DWARFDebugAbbrev::parse(DataExtractor data) {
+  uint32_t offset = 0;
+
+  while (data.isValidOffset(offset)) {
+    uint32_t initial_cu_offset = offset;
+    DWARFAbbreviationDeclarationSet abbrevDeclSet;
+
+    if (abbrevDeclSet.extract(data, &offset))
+      AbbrevCollMap[initial_cu_offset] = abbrevDeclSet;
+    else
+      break;
+  }
+  PrevAbbrOffsetPos = AbbrevCollMap.end();
+}
+
+void DWARFDebugAbbrev::dump(raw_ostream &OS) const {
+  if (AbbrevCollMap.empty()) {
+    OS << "< EMPTY >\n";
+    return;
+  }
+
+  DWARFAbbreviationDeclarationCollMapConstIter pos;
+  for (pos = AbbrevCollMap.begin(); pos != AbbrevCollMap.end(); ++pos) {
+    OS << format("Abbrev table for offset: 0x%8.8x\n", pos->first);
+    pos->second.dump(OS);
+  }
+}
+
+const DWARFAbbreviationDeclarationSet*
+DWARFDebugAbbrev::getAbbreviationDeclarationSet(uint64_t cu_abbr_offset) const {
+  DWARFAbbreviationDeclarationCollMapConstIter end = AbbrevCollMap.end();
+  DWARFAbbreviationDeclarationCollMapConstIter pos;
+  if (PrevAbbrOffsetPos != end &&
+      PrevAbbrOffsetPos->first == cu_abbr_offset) {
+    return &(PrevAbbrOffsetPos->second);
+  } else {
+    pos = AbbrevCollMap.find(cu_abbr_offset);
+    PrevAbbrOffsetPos = pos;
+  }
+
+  if (pos != AbbrevCollMap.end())
+    return &(pos->second);
+  return NULL;
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.h b/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.h
new file mode 100644
index 000000000000..03189b132127
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.h
@@ -0,0 +1,73 @@
+//===-- DWARFDebugAbbrev.h --------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFDEBUGABBREV_H
+#define LLVM_DEBUGINFO_DWARFDEBUGABBREV_H
+
+#include "DWARFAbbreviationDeclaration.h"
+#include <list>
+#include <map>
+#include <vector>
+
+namespace llvm {
+
+typedef std::vector<DWARFAbbreviationDeclaration>
+  DWARFAbbreviationDeclarationColl;
+typedef DWARFAbbreviationDeclarationColl::iterator
+  DWARFAbbreviationDeclarationCollIter;
+typedef DWARFAbbreviationDeclarationColl::const_iterator
+  DWARFAbbreviationDeclarationCollConstIter;
+
+class DWARFAbbreviationDeclarationSet {
+  uint64_t Offset;
+  uint32_t IdxOffset;
+  std::vector<DWARFAbbreviationDeclaration> Decls;
+  public:
+  DWARFAbbreviationDeclarationSet()
+    : Offset(0), IdxOffset(0) {}
+
+  DWARFAbbreviationDeclarationSet(uint64_t offset, uint32_t idxOffset)
+    : Offset(offset), IdxOffset(idxOffset) {}
+
+  void clear() {
+    IdxOffset = 0;
+    Decls.clear();
+  }
+  uint64_t getOffset() const { return Offset; }
+  void dump(raw_ostream &OS) const;
+  bool extract(DataExtractor data, uint32_t* offset_ptr);
+
+  const DWARFAbbreviationDeclaration *
+    getAbbreviationDeclaration(uint32_t abbrCode) const;
+};
+
+class DWARFDebugAbbrev {
+public:
+  typedef std::map<uint64_t, DWARFAbbreviationDeclarationSet>
+    DWARFAbbreviationDeclarationCollMap;
+  typedef DWARFAbbreviationDeclarationCollMap::iterator
+    DWARFAbbreviationDeclarationCollMapIter;
+  typedef DWARFAbbreviationDeclarationCollMap::const_iterator
+    DWARFAbbreviationDeclarationCollMapConstIter;
+
+private:
+  DWARFAbbreviationDeclarationCollMap AbbrevCollMap;
+  mutable DWARFAbbreviationDeclarationCollMapConstIter PrevAbbrOffsetPos;
+
+public:
+  DWARFDebugAbbrev();
+  const DWARFAbbreviationDeclarationSet *
+    getAbbreviationDeclarationSet(uint64_t cu_abbr_offset) const;
+  void dump(raw_ostream &OS) const;
+  void parse(DataExtractor data);
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.cpp b/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.cpp
new file mode 100644
index 000000000000..b0c0354383b9
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.cpp
@@ -0,0 +1,150 @@
+//===-- DWARFDebugArangeSet.cpp -------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFDebugArangeSet.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <cassert>
+using namespace llvm;
+
+void DWARFDebugArangeSet::clear() {
+  Offset = -1U;
+  std::memset(&Header, 0, sizeof(Header));
+  ArangeDescriptors.clear();
+}
+
+void DWARFDebugArangeSet::compact() {
+  if (ArangeDescriptors.empty())
+    return;
+
+  // Iterate through all arange descriptors and combine any ranges that
+  // overlap or have matching boundaries. The ArangeDescriptors are assumed
+  // to be in ascending order.
+  uint32_t i = 0;
+  while (i + 1 < ArangeDescriptors.size()) {
+    if (ArangeDescriptors[i].getEndAddress() >= ArangeDescriptors[i+1].Address){
+      // The current range ends at or exceeds the start of the next address
+      // range. Compute the max end address between the two and use that to
+      // make the new length.
+      const uint64_t max_end_addr =
+        std::max(ArangeDescriptors[i].getEndAddress(),
+                 ArangeDescriptors[i+1].getEndAddress());
+      ArangeDescriptors[i].Length = max_end_addr - ArangeDescriptors[i].Address;
+      // Now remove the next entry as it was just combined with the previous one
+      ArangeDescriptors.erase(ArangeDescriptors.begin()+i+1);
+    } else {
+      // Discontiguous address range, just proceed to the next one.
+      ++i;
+    }
+  }
+}
+
+bool
+DWARFDebugArangeSet::extract(DataExtractor data, uint32_t *offset_ptr) {
+  if (data.isValidOffset(*offset_ptr)) {
+    ArangeDescriptors.clear();
+    Offset = *offset_ptr;
+
+    // 7.20 Address Range Table
+    //
+    // Each set of entries in the table of address ranges contained in
+    // the .debug_aranges section begins with a header consisting of: a
+    // 4-byte length containing the length of the set of entries for this
+    // compilation unit, not including the length field itself; a 2-byte
+    // version identifier containing the value 2 for DWARF Version 2; a
+    // 4-byte offset into the.debug_infosection; a 1-byte unsigned integer
+    // containing the size in bytes of an address (or the offset portion of
+    // an address for segmented addressing) on the target system; and a
+    // 1-byte unsigned integer containing the size in bytes of a segment
+    // descriptor on the target system. This header is followed by a series
+    // of tuples. Each tuple consists of an address and a length, each in
+    // the size appropriate for an address on the target architecture.
+    Header.Length = data.getU32(offset_ptr);
+    Header.Version = data.getU16(offset_ptr);
+    Header.CuOffset = data.getU32(offset_ptr);
+    Header.AddrSize = data.getU8(offset_ptr);
+    Header.SegSize = data.getU8(offset_ptr);
+
+    // Perform basic validation of the header fields.
+    if (!data.isValidOffsetForDataOfSize(Offset, Header.Length) ||
+        (Header.AddrSize != 4 && Header.AddrSize != 8)) {
+      clear();
+      return false;
+    }
+
+    // The first tuple following the header in each set begins at an offset
+    // that is a multiple of the size of a single tuple (that is, twice the
+    // size of an address). The header is padded, if necessary, to the
+    // appropriate boundary.
+    const uint32_t header_size = *offset_ptr - Offset;
+    const uint32_t tuple_size = Header.AddrSize * 2;
+    uint32_t first_tuple_offset = 0;
+    while (first_tuple_offset < header_size)
+      first_tuple_offset += tuple_size;
+
+    *offset_ptr = Offset + first_tuple_offset;
+
+    Descriptor arangeDescriptor;
+
+    assert(sizeof(arangeDescriptor.Address) == sizeof(arangeDescriptor.Length));
+    assert(sizeof(arangeDescriptor.Address) >= Header.AddrSize);
+
+    while (data.isValidOffset(*offset_ptr)) {
+      arangeDescriptor.Address = data.getUnsigned(offset_ptr, Header.AddrSize);
+      arangeDescriptor.Length = data.getUnsigned(offset_ptr, Header.AddrSize);
+
+      // Each set of tuples is terminated by a 0 for the address and 0
+      // for the length.
+      if (arangeDescriptor.Address || arangeDescriptor.Length)
+        ArangeDescriptors.push_back(arangeDescriptor);
+      else
+        break; // We are done if we get a zero address and length
+    }
+
+    return !ArangeDescriptors.empty();
+  }
+  return false;
+}
+
+void DWARFDebugArangeSet::dump(raw_ostream &OS) const {
+  OS << format("Address Range Header: length = 0x%8.8x, version = 0x%4.4x, ",
+               Header.Length, Header.Version)
+     << format("cu_offset = 0x%8.8x, addr_size = 0x%2.2x, seg_size = 0x%2.2x\n",
+               Header.CuOffset, Header.AddrSize, Header.SegSize);
+
+  const uint32_t hex_width = Header.AddrSize * 2;
+  for (DescriptorConstIter pos = ArangeDescriptors.begin(),
+       end = ArangeDescriptors.end(); pos != end; ++pos)
+    OS << format("[0x%*.*llx -", hex_width, hex_width, pos->Address)
+       << format(" 0x%*.*llx)\n", hex_width, hex_width, pos->getEndAddress());
+}
+
+
+namespace {
+  class DescriptorContainsAddress {
+    const uint64_t Address;
+  public:
+    DescriptorContainsAddress(uint64_t address) : Address(address) {}
+    bool operator()(const DWARFDebugArangeSet::Descriptor &desc) const {
+      return Address >= desc.Address && Address < (desc.Address + desc.Length);
+    }
+  };
+}
+
+uint32_t DWARFDebugArangeSet::findAddress(uint64_t address) const {
+  DescriptorConstIter end = ArangeDescriptors.end();
+  DescriptorConstIter pos =
+    std::find_if(ArangeDescriptors.begin(), end, // Range
+                 DescriptorContainsAddress(address)); // Predicate
+  if (pos != end)
+    return Header.CuOffset;
+
+  return -1U;
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.h b/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.h
new file mode 100644
index 000000000000..9a2a6d0f0037
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.h
@@ -0,0 +1,75 @@
+//===-- DWARFDebugArangeSet.h -----------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFDEBUGARANGESET_H
+#define LLVM_DEBUGINFO_DWARFDEBUGARANGESET_H
+
+#include "llvm/Support/DataExtractor.h"
+#include <vector>
+
+namespace llvm {
+
+class raw_ostream;
+
+class DWARFDebugArangeSet {
+public:
+  struct Header {
+    // The total length of the entries for that set, not including the length
+    // field itself.
+    uint32_t Length;
+    // The offset from the beginning of the .debug_info section of the
+    // compilation unit entry referenced by the table.
+    uint32_t CuOffset;
+    // The DWARF version number.
+    uint16_t Version;
+    // The size in bytes of an address on the target architecture. For segmented
+    // addressing, this is the size of the offset portion of the address.
+    uint8_t AddrSize;
+    // The size in bytes of a segment descriptor on the target architecture.
+    // If the target system uses a flat address space, this value is 0.
+    uint8_t SegSize;
+  };
+
+  struct Descriptor {
+    uint64_t Address;
+    uint64_t Length;
+    uint64_t getEndAddress() const { return Address + Length; }
+  };
+
+private:
+  typedef std::vector<Descriptor> DescriptorColl;
+  typedef DescriptorColl::iterator DescriptorIter;
+  typedef DescriptorColl::const_iterator DescriptorConstIter;
+
+  uint32_t Offset;
+  Header Header;
+  DescriptorColl ArangeDescriptors;
+
+public:
+  DWARFDebugArangeSet() { clear(); }
+  void clear();
+  void compact();
+  bool extract(DataExtractor data, uint32_t *offset_ptr);
+  void dump(raw_ostream &OS) const;
+
+  uint32_t getCompileUnitDIEOffset() const { return Header.CuOffset; }
+  uint32_t getOffsetOfNextEntry() const { return Offset + Header.Length + 4; }
+  uint32_t findAddress(uint64_t address) const;
+  uint32_t getNumDescriptors() const { return ArangeDescriptors.size(); }
+  const struct Header &getHeader() const { return Header; }
+  const Descriptor *getDescriptor(uint32_t i) const {
+    if (i < ArangeDescriptors.size())
+      return &ArangeDescriptors[i];
+    return NULL;
+  }
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.cpp b/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.cpp
new file mode 100644
index 000000000000..576d37d7813a
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.cpp
@@ -0,0 +1,223 @@
+//===-- DWARFDebugAranges.cpp -----------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFDebugAranges.h"
+#include "DWARFCompileUnit.h"
+#include "DWARFContext.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <cassert>
+using namespace llvm;
+
+// Compare function DWARFDebugAranges::Range structures
+static bool RangeLessThan(const DWARFDebugAranges::Range &range1,
+                          const DWARFDebugAranges::Range &range2) {
+  return range1.LoPC < range2.LoPC;
+}
+
+namespace {
+  class CountArangeDescriptors {
+  public:
+    CountArangeDescriptors(uint32_t &count_ref) : Count(count_ref) {}
+    void operator()(const DWARFDebugArangeSet &set) {
+      Count += set.getNumDescriptors();
+    }
+    uint32_t &Count;
+  };
+
+  class AddArangeDescriptors {
+  public:
+    AddArangeDescriptors(DWARFDebugAranges::RangeColl &ranges)
+      : RangeCollection(ranges) {}
+    void operator()(const DWARFDebugArangeSet& set) {
+      const DWARFDebugArangeSet::Descriptor* arange_desc_ptr;
+      DWARFDebugAranges::Range range;
+      range.Offset = set.getCompileUnitDIEOffset();
+
+      for (uint32_t i=0; (arange_desc_ptr = set.getDescriptor(i)) != NULL; ++i){
+        range.LoPC = arange_desc_ptr->Address;
+        range.Length = arange_desc_ptr->Length;
+
+        // Insert each item in increasing address order so binary searching
+        // can later be done!
+        DWARFDebugAranges::RangeColl::iterator insert_pos =
+          std::lower_bound(RangeCollection.begin(), RangeCollection.end(),
+                           range, RangeLessThan);
+        RangeCollection.insert(insert_pos, range);
+      }
+    }
+    DWARFDebugAranges::RangeColl& RangeCollection;
+  };
+}
+
+bool DWARFDebugAranges::extract(DataExtractor debug_aranges_data) {
+  if (debug_aranges_data.isValidOffset(0)) {
+    uint32_t offset = 0;
+
+    typedef std::vector<DWARFDebugArangeSet> SetCollection;
+    typedef SetCollection::const_iterator SetCollectionIter;
+    SetCollection sets;
+
+    DWARFDebugArangeSet set;
+    Range range;
+    while (set.extract(debug_aranges_data, &offset))
+      sets.push_back(set);
+
+    uint32_t count = 0;
+
+    std::for_each(sets.begin(), sets.end(), CountArangeDescriptors(count));
+
+    if (count > 0) {
+      Aranges.reserve(count);
+      AddArangeDescriptors range_adder(Aranges);
+      std::for_each(sets.begin(), sets.end(), range_adder);
+    }
+  }
+  return false;
+}
+
+bool DWARFDebugAranges::generate(DWARFContext *ctx) {
+  clear();
+  if (ctx) {
+    const uint32_t num_compile_units = ctx->getNumCompileUnits();
+    for (uint32_t cu_idx = 0; cu_idx < num_compile_units; ++cu_idx) {
+      DWARFCompileUnit *cu = ctx->getCompileUnitAtIndex(cu_idx);
+      if (cu)
+        cu->buildAddressRangeTable(this, true);
+    }
+  }
+  return !isEmpty();
+}
+
+void DWARFDebugAranges::dump(raw_ostream &OS) const {
+  const uint32_t num_ranges = getNumRanges();
+  for (uint32_t i = 0; i < num_ranges; ++i) {
+    const Range &range = Aranges[i];
+    OS << format("0x%8.8x: [0x%8.8llx - 0x%8.8llx)\n", range.Offset,
+                 (uint64_t)range.LoPC, (uint64_t)range.HiPC());
+  }
+}
+
+void DWARFDebugAranges::Range::dump(raw_ostream &OS) const {
+  OS << format("{0x%8.8x}: [0x%8.8llx - 0x%8.8llx)\n", Offset, LoPC, HiPC());
+}
+
+void DWARFDebugAranges::appendRange(uint32_t offset, uint64_t low_pc,
+                                    uint64_t high_pc) {
+  if (!Aranges.empty()) {
+    if (Aranges.back().Offset == offset && Aranges.back().HiPC() == low_pc) {
+      Aranges.back().setHiPC(high_pc);
+      return;
+    }
+  }
+  Aranges.push_back(Range(low_pc, high_pc, offset));
+}
+
+void DWARFDebugAranges::sort(bool minimize, uint32_t n) {
+  const size_t orig_arange_size = Aranges.size();
+  // Size of one? If so, no sorting is needed
+  if (orig_arange_size <= 1)
+    return;
+  // Sort our address range entries
+  std::stable_sort(Aranges.begin(), Aranges.end(), RangeLessThan);
+
+  if (!minimize)
+    return;
+
+  // Most address ranges are contiguous from function to function
+  // so our new ranges will likely be smaller. We calculate the size
+  // of the new ranges since although std::vector objects can be resized,
+  // the will never reduce their allocated block size and free any excesss
+  // memory, so we might as well start a brand new collection so it is as
+  // small as possible.
+
+  // First calculate the size of the new minimal arange vector
+  // so we don't have to do a bunch of re-allocations as we
+  // copy the new minimal stuff over to the new collection.
+  size_t minimal_size = 1;
+  for (size_t i = 1; i < orig_arange_size; ++i) {
+    if (!Range::SortedOverlapCheck(Aranges[i-1], Aranges[i], n))
+      ++minimal_size;
+  }
+
+  // If the sizes are the same, then no consecutive aranges can be
+  // combined, we are done.
+  if (minimal_size == orig_arange_size)
+    return;
+
+  // Else, make a new RangeColl that _only_ contains what we need.
+  RangeColl minimal_aranges;
+  minimal_aranges.resize(minimal_size);
+  uint32_t j = 0;
+  minimal_aranges[j] = Aranges[0];
+  for (size_t i = 1; i < orig_arange_size; ++i) {
+    if(Range::SortedOverlapCheck (minimal_aranges[j], Aranges[i], n)) {
+      minimal_aranges[j].setHiPC (Aranges[i].HiPC());
+    } else {
+      // Only increment j if we aren't merging
+      minimal_aranges[++j] = Aranges[i];
+    }
+  }
+  assert (j+1 == minimal_size);
+
+  // Now swap our new minimal aranges into place. The local
+  // minimal_aranges will then contian the old big collection
+  // which will get freed.
+  minimal_aranges.swap(Aranges);
+}
+
+uint32_t DWARFDebugAranges::findAddress(uint64_t address) const {
+  if (!Aranges.empty()) {
+    Range range(address);
+    RangeCollIterator begin = Aranges.begin();
+    RangeCollIterator end = Aranges.end();
+    RangeCollIterator pos = lower_bound(begin, end, range, RangeLessThan);
+
+    if (pos != end && pos->LoPC <= address && address < pos->HiPC()) {
+      return pos->Offset;
+    } else if (pos != begin) {
+      --pos;
+      if (pos->LoPC <= address && address < pos->HiPC())
+        return (*pos).Offset;
+    }
+  }
+  return -1U;
+}
+
+bool
+DWARFDebugAranges::allRangesAreContiguous(uint64_t &LoPC, uint64_t &HiPC) const{
+  if (Aranges.empty())
+    return false;
+
+  uint64_t next_addr = 0;
+  RangeCollIterator begin = Aranges.begin();
+  for (RangeCollIterator pos = begin, end = Aranges.end(); pos != end;
+       ++pos) {
+    if (pos != begin && pos->LoPC != next_addr)
+      return false;
+    next_addr = pos->HiPC();
+  }
+  // We checked for empty at the start of function so front() will be valid.
+  LoPC = Aranges.front().LoPC;
+  // We checked for empty at the start of function so back() will be valid.
+  HiPC = Aranges.back().HiPC();
+  return true;
+}
+
+bool DWARFDebugAranges::getMaxRange(uint64_t &LoPC, uint64_t &HiPC) const {
+  if (Aranges.empty())
+    return false;
+  // We checked for empty at the start of function so front() will be valid.
+  LoPC = Aranges.front().LoPC;
+  // We checked for empty at the start of function so back() will be valid.
+  HiPC = Aranges.back().HiPC();
+  return true;
+}
+
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.h b/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.h
new file mode 100644
index 000000000000..12afb60beb40
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.h
@@ -0,0 +1,98 @@
+//===-- DWARFDebugAranges.h -------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFDEBUGARANGES_H
+#define LLVM_DEBUGINFO_DWARFDEBUGARANGES_H
+
+#include "DWARFDebugArangeSet.h"
+#include <list>
+
+namespace llvm {
+
+class DWARFContext;
+
+class DWARFDebugAranges {
+public:
+  struct Range {
+    explicit Range(uint64_t lo = -1ULL, uint64_t hi = -1ULL,
+                   uint32_t off = -1U)
+      : LoPC(lo), Length(hi-lo), Offset(off) {}
+
+    void clear() {
+      LoPC = -1ULL;
+      Length = 0;
+      Offset = -1U;
+    }
+
+    void setHiPC(uint64_t HiPC) {
+      if (HiPC == -1ULL || HiPC <= LoPC)
+        Length = 0;
+      else
+        Length = HiPC - LoPC;
+    }
+    uint64_t HiPC() const {
+      if (Length)
+        return LoPC + Length;
+      return -1ULL;
+    }
+    bool isValidRange() const { return Length > 0; }
+
+    static bool SortedOverlapCheck(const Range &curr_range,
+                                   const Range &next_range, uint32_t n) {
+      if (curr_range.Offset != next_range.Offset)
+        return false;
+      return curr_range.HiPC() + n >= next_range.LoPC;
+    }
+
+    bool contains(const Range &range) const {
+      return LoPC <= range.LoPC && range.HiPC() <= HiPC();
+    }
+
+    void dump(raw_ostream &OS) const;
+    uint64_t LoPC; // Start of address range
+    uint32_t Length; // End of address range (not including this address)
+    uint32_t Offset; // Offset of the compile unit or die
+  };
+
+  void clear() { Aranges.clear(); }
+  bool allRangesAreContiguous(uint64_t& LoPC, uint64_t& HiPC) const;
+  bool getMaxRange(uint64_t& LoPC, uint64_t& HiPC) const;
+  bool extract(DataExtractor debug_aranges_data);
+  bool generate(DWARFContext *ctx);
+
+  // Use append range multiple times and then call sort
+  void appendRange(uint32_t cu_offset, uint64_t low_pc, uint64_t high_pc);
+  void sort(bool minimize, uint32_t n);
+
+  const Range *rangeAtIndex(uint32_t idx) const {
+    if (idx < Aranges.size())
+      return &Aranges[idx];
+    return NULL;
+  }
+  void dump(raw_ostream &OS) const;
+  uint32_t findAddress(uint64_t address) const;
+  bool isEmpty() const { return Aranges.empty(); }
+  uint32_t getNumRanges() const { return Aranges.size(); }
+
+  uint32_t offsetAtIndex(uint32_t idx) const {
+    if (idx < Aranges.size())
+      return Aranges[idx].Offset;
+    return -1U;
+  }
+
+  typedef std::vector<Range>              RangeColl;
+  typedef RangeColl::const_iterator       RangeCollIterator;
+
+private:
+  RangeColl Aranges;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.cpp b/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.cpp
new file mode 100644
index 000000000000..1b089adbe13b
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.cpp
@@ -0,0 +1,444 @@
+//===-- DWARFDebugInfoEntry.cpp --------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFDebugInfoEntry.h"
+#include "DWARFCompileUnit.h"
+#include "DWARFContext.h"
+#include "DWARFDebugAbbrev.h"
+#include "DWARFFormValue.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+using namespace dwarf;
+
+void DWARFDebugInfoEntryMinimal::dump(raw_ostream &OS,
+                                      const DWARFCompileUnit *cu,
+                                      unsigned recurseDepth,
+                                      unsigned indent) const {
+  DataExtractor debug_info_data = cu->getDebugInfoExtractor();
+  uint32_t offset = Offset;
+
+  if (debug_info_data.isValidOffset(offset)) {
+    uint64_t abbrCode = debug_info_data.getULEB128(&offset);
+
+    OS << format("\n0x%8.8x: ", Offset);
+    if (abbrCode) {
+      if (AbbrevDecl) {
+        const char *tagString = TagString(getTag());
+        if (tagString)
+          OS.indent(indent) << tagString;
+        else
+          OS.indent(indent) << format("DW_TAG_Unknown_%x", getTag());
+        OS << format(" [%u] %c\n", abbrCode,
+                     AbbrevDecl->hasChildren() ? '*' : ' ');
+
+        // Dump all data in the .debug_info for the attributes
+        const uint32_t numAttributes = AbbrevDecl->getNumAttributes();
+        for (uint32_t i = 0; i != numAttributes; ++i) {
+          uint16_t attr = AbbrevDecl->getAttrByIndex(i);
+          uint16_t form = AbbrevDecl->getFormByIndex(i);
+          dumpAttribute(OS, cu, &offset, attr, form, indent);
+        }
+
+        const DWARFDebugInfoEntryMinimal *child = getFirstChild();
+        if (recurseDepth > 0 && child) {
+          while (child) {
+            child->dump(OS, cu, recurseDepth-1, indent+2);
+            child = child->getSibling();
+          }
+        }
+      } else {
+        OS << "Abbreviation code not found in 'debug_abbrev' class for code: "
+           << abbrCode << '\n';
+      }
+    } else {
+      OS.indent(indent) << "NULL\n";
+    }
+  }
+}
+
+void DWARFDebugInfoEntryMinimal::dumpAttribute(raw_ostream &OS,
+                                               const DWARFCompileUnit *cu,
+                                               uint32_t* offset_ptr,
+                                               uint16_t attr,
+                                               uint16_t form,
+                                               unsigned indent) const {
+  OS << format("0x%8.8x: ", *offset_ptr);
+  OS.indent(indent+2);
+  const char *attrString = AttributeString(attr);
+  if (attrString)
+    OS << attrString;
+  else
+    OS << format("DW_AT_Unknown_%x", attr);
+  const char *formString = FormEncodingString(form);
+  if (formString)
+    OS << " [" << formString << ']';
+  else
+    OS << format(" [DW_FORM_Unknown_%x]", form);
+
+  DWARFFormValue formValue(form);
+
+  if (!formValue.extractValue(cu->getDebugInfoExtractor(), offset_ptr, cu))
+    return;
+
+  OS << "\t(";
+  formValue.dump(OS, cu);
+  OS << ")\n";
+}
+
+bool DWARFDebugInfoEntryMinimal::extractFast(const DWARFCompileUnit *cu,
+                                             const uint8_t *fixed_form_sizes,
+                                             uint32_t *offset_ptr) {
+  Offset = *offset_ptr;
+
+  DataExtractor debug_info_data = cu->getDebugInfoExtractor();
+  uint64_t abbrCode = debug_info_data.getULEB128(offset_ptr);
+
+  assert (fixed_form_sizes); // For best performance this should be specified!
+
+  if (abbrCode) {
+    uint32_t offset = *offset_ptr;
+
+    AbbrevDecl = cu->getAbbreviations()->getAbbreviationDeclaration(abbrCode);
+
+    // Skip all data in the .debug_info for the attributes
+    const uint32_t numAttributes = AbbrevDecl->getNumAttributes();
+    uint32_t i;
+    uint16_t form;
+    for (i=0; i<numAttributes; ++i) {
+      form = AbbrevDecl->getFormByIndex(i);
+
+      const uint8_t fixed_skip_size = fixed_form_sizes[form];
+      if (fixed_skip_size)
+        offset += fixed_skip_size;
+      else {
+        bool form_is_indirect = false;
+        do {
+          form_is_indirect = false;
+          uint32_t form_size = 0;
+          switch (form) {
+          // Blocks if inlined data that have a length field and the data bytes
+          // inlined in the .debug_info.
+          case DW_FORM_block:
+            form_size = debug_info_data.getULEB128(&offset);
+            break;
+          case DW_FORM_block1:
+            form_size = debug_info_data.getU8(&offset);
+            break;
+          case DW_FORM_block2:
+            form_size = debug_info_data.getU16(&offset);
+            break;
+          case DW_FORM_block4:
+            form_size = debug_info_data.getU32(&offset);
+            break;
+
+          // Inlined NULL terminated C-strings
+          case DW_FORM_string:
+            debug_info_data.getCStr(&offset);
+            break;
+
+          // Compile unit address sized values
+          case DW_FORM_addr:
+          case DW_FORM_ref_addr:
+            form_size = cu->getAddressByteSize();
+            break;
+
+          // 1 byte values
+          case DW_FORM_data1:
+          case DW_FORM_flag:
+          case DW_FORM_ref1:
+            form_size = 1;
+            break;
+
+          // 2 byte values
+          case DW_FORM_data2:
+          case DW_FORM_ref2:
+            form_size = 2;
+            break;
+
+          // 4 byte values
+          case DW_FORM_strp:
+          case DW_FORM_data4:
+          case DW_FORM_ref4:
+            form_size = 4;
+            break;
+
+          // 8 byte values
+          case DW_FORM_data8:
+          case DW_FORM_ref8:
+            form_size = 8;
+            break;
+
+          // signed or unsigned LEB 128 values
+          case DW_FORM_sdata:
+          case DW_FORM_udata:
+          case DW_FORM_ref_udata:
+            debug_info_data.getULEB128(&offset);
+            break;
+
+          case DW_FORM_indirect:
+            form_is_indirect = true;
+            form = debug_info_data.getULEB128(&offset);
+            break;
+
+          default:
+            *offset_ptr = Offset;
+            return false;
+          }
+          offset += form_size;
+
+        } while (form_is_indirect);
+      }
+    }
+    *offset_ptr = offset;
+    return true;
+  } else {
+    AbbrevDecl = NULL;
+    return true; // NULL debug tag entry
+  }
+
+  return false;
+}
+
+bool
+DWARFDebugInfoEntryMinimal::extract(const DWARFCompileUnit *cu,
+                                    uint32_t *offset_ptr) {
+  DataExtractor debug_info_data = cu->getDebugInfoExtractor();
+  const uint32_t cu_end_offset = cu->getNextCompileUnitOffset();
+  const uint8_t cu_addr_size = cu->getAddressByteSize();
+  uint32_t offset = *offset_ptr;
+  if ((offset < cu_end_offset) && debug_info_data.isValidOffset(offset)) {
+    Offset = offset;
+
+    uint64_t abbrCode = debug_info_data.getULEB128(&offset);
+
+    if (abbrCode) {
+      AbbrevDecl = cu->getAbbreviations()->getAbbreviationDeclaration(abbrCode);
+
+      if (AbbrevDecl) {
+        uint16_t tag = AbbrevDecl->getTag();
+
+        bool isCompileUnitTag = tag == DW_TAG_compile_unit;
+        if(cu && isCompileUnitTag)
+          const_cast<DWARFCompileUnit*>(cu)->setBaseAddress(0);
+
+        // Skip all data in the .debug_info for the attributes
+        const uint32_t numAttributes = AbbrevDecl->getNumAttributes();
+        for (uint32_t i = 0; i != numAttributes; ++i) {
+          uint16_t attr = AbbrevDecl->getAttrByIndex(i);
+          uint16_t form = AbbrevDecl->getFormByIndex(i);
+
+          if (isCompileUnitTag &&
+              ((attr == DW_AT_entry_pc) || (attr == DW_AT_low_pc))) {
+            DWARFFormValue form_value(form);
+            if (form_value.extractValue(debug_info_data, &offset, cu)) {
+              if (attr == DW_AT_low_pc || attr == DW_AT_entry_pc)
+                const_cast<DWARFCompileUnit*>(cu)
+                  ->setBaseAddress(form_value.getUnsigned());
+            }
+          } else {
+            bool form_is_indirect = false;
+            do {
+              form_is_indirect = false;
+              register uint32_t form_size = 0;
+              switch (form) {
+              // Blocks if inlined data that have a length field and the data
+              // bytes // inlined in the .debug_info
+              case DW_FORM_block:
+                form_size = debug_info_data.getULEB128(&offset);
+                break;
+              case DW_FORM_block1:
+                form_size = debug_info_data.getU8(&offset);
+                break;
+              case DW_FORM_block2:
+                form_size = debug_info_data.getU16(&offset);
+                break;
+              case DW_FORM_block4:
+                form_size = debug_info_data.getU32(&offset);
+                break;
+
+              // Inlined NULL terminated C-strings
+              case DW_FORM_string:
+                debug_info_data.getCStr(&offset);
+                break;
+
+              // Compile unit address sized values
+              case DW_FORM_addr:
+              case DW_FORM_ref_addr:
+                form_size = cu_addr_size;
+                break;
+
+              // 1 byte values
+              case DW_FORM_data1:
+              case DW_FORM_flag:
+              case DW_FORM_ref1:
+                form_size = 1;
+                break;
+
+              // 2 byte values
+              case DW_FORM_data2:
+              case DW_FORM_ref2:
+                form_size = 2;
+                break;
+
+                // 4 byte values
+              case DW_FORM_strp:
+                form_size = 4;
+                break;
+
+              case DW_FORM_data4:
+              case DW_FORM_ref4:
+                form_size = 4;
+                break;
+
+              // 8 byte values
+              case DW_FORM_data8:
+              case DW_FORM_ref8:
+                form_size = 8;
+                break;
+
+              // signed or unsigned LEB 128 values
+              case DW_FORM_sdata:
+              case DW_FORM_udata:
+              case DW_FORM_ref_udata:
+                debug_info_data.getULEB128(&offset);
+                break;
+
+              case DW_FORM_indirect:
+                form = debug_info_data.getULEB128(&offset);
+                form_is_indirect = true;
+                break;
+
+              default:
+                *offset_ptr = offset;
+                return false;
+              }
+
+              offset += form_size;
+            } while (form_is_indirect);
+          }
+        }
+        *offset_ptr = offset;
+        return true;
+      }
+    } else {
+      AbbrevDecl = NULL;
+      *offset_ptr = offset;
+      return true;    // NULL debug tag entry
+    }
+  }
+
+  return false;
+}
+
+uint32_t
+DWARFDebugInfoEntryMinimal::getAttributeValue(const DWARFCompileUnit *cu,
+                                              const uint16_t attr,
+                                              DWARFFormValue &form_value,
+                                              uint32_t *end_attr_offset_ptr)
+                                              const {
+  if (AbbrevDecl) {
+    uint32_t attr_idx = AbbrevDecl->findAttributeIndex(attr);
+
+    if (attr_idx != -1U) {
+      uint32_t offset = getOffset();
+
+      DataExtractor debug_info_data = cu->getDebugInfoExtractor();
+
+      // Skip the abbreviation code so we are at the data for the attributes
+      debug_info_data.getULEB128(&offset);
+
+      uint32_t idx = 0;
+      while (idx < attr_idx)
+        DWARFFormValue::skipValue(AbbrevDecl->getFormByIndex(idx++),
+                                  debug_info_data, &offset, cu);
+
+      const uint32_t attr_offset = offset;
+      form_value = DWARFFormValue(AbbrevDecl->getFormByIndex(idx));
+      if (form_value.extractValue(debug_info_data, &offset, cu)) {
+        if (end_attr_offset_ptr)
+          *end_attr_offset_ptr = offset;
+        return attr_offset;
+      }
+    }
+  }
+
+  return 0;
+}
+
+const char*
+DWARFDebugInfoEntryMinimal::getAttributeValueAsString(
+    const DWARFCompileUnit* cu,
+    const uint16_t attr,
+    const char* fail_value) const {
+  DWARFFormValue form_value;
+  if (getAttributeValue(cu, attr, form_value)) {
+    DataExtractor stringExtractor(cu->getContext().getStringSection(),
+        false, 0);
+    return form_value.getAsCString(&stringExtractor);
+  }
+  return fail_value;
+}
+
+uint64_t
+DWARFDebugInfoEntryMinimal::getAttributeValueAsUnsigned(
+    const DWARFCompileUnit* cu,
+    const uint16_t attr,
+    uint64_t fail_value) const {
+  DWARFFormValue form_value;
+  if (getAttributeValue(cu, attr, form_value))
+      return form_value.getUnsigned();
+  return fail_value;
+}
+
+int64_t
+DWARFDebugInfoEntryMinimal::getAttributeValueAsSigned(
+    const DWARFCompileUnit* cu,
+    const uint16_t attr,
+    int64_t fail_value) const {
+  DWARFFormValue form_value;
+  if (getAttributeValue(cu, attr, form_value))
+      return form_value.getSigned();
+  return fail_value;
+}
+
+uint64_t
+DWARFDebugInfoEntryMinimal::getAttributeValueAsReference(
+                                                  const DWARFCompileUnit* cu,
+                                                  const uint16_t attr,
+                                                  uint64_t fail_value) const {
+  DWARFFormValue form_value;
+  if (getAttributeValue(cu, attr, form_value))
+      return form_value.getReference(cu);
+  return fail_value;
+}
+
+void
+DWARFDebugInfoEntryMinimal::buildAddressRangeTable(const DWARFCompileUnit *cu,
+                                               DWARFDebugAranges *debug_aranges)
+                                                   const {
+  if (AbbrevDecl) {
+    uint16_t tag = AbbrevDecl->getTag();
+    if (tag == DW_TAG_subprogram) {
+      uint64_t hi_pc = -1ULL;
+      uint64_t lo_pc = getAttributeValueAsUnsigned(cu, DW_AT_low_pc, -1ULL);
+      if (lo_pc != -1ULL)
+        hi_pc = getAttributeValueAsUnsigned(cu, DW_AT_high_pc, -1ULL);
+      if (hi_pc != -1ULL)
+        debug_aranges->appendRange(cu->getOffset(), lo_pc, hi_pc);
+    }
+
+    const DWARFDebugInfoEntryMinimal *child = getFirstChild();
+    while (child) {
+      child->buildAddressRangeTable(cu, debug_aranges);
+      child = child->getSibling();
+    }
+  }
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.h b/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.h
new file mode 100644
index 000000000000..aff2e8556729
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.h
@@ -0,0 +1,135 @@
+//===-- DWARFDebugInfoEntry.h -----------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFDEBUGINFOENTRY_H
+#define LLVM_DEBUGINFO_DWARFDEBUGINFOENTRY_H
+
+#include "DWARFAbbreviationDeclaration.h"
+#include "llvm/Support/DataTypes.h"
+
+namespace llvm {
+
+class DWARFDebugAranges;
+class DWARFCompileUnit;
+class DWARFContext;
+class DWARFFormValue;
+
+/// DWARFDebugInfoEntryMinimal - A DIE with only the minimum required data.
+class DWARFDebugInfoEntryMinimal {
+  /// Offset within the .debug_info of the start of this entry.
+  uint64_t Offset;
+
+  /// How many to subtract from "this" to get the parent.
+  /// If zero this die has no parent.
+  uint32_t ParentIdx;
+
+  /// How many to add to "this" to get the sibling.
+  uint32_t SiblingIdx;
+
+  const DWARFAbbreviationDeclaration *AbbrevDecl;
+public:
+  DWARFDebugInfoEntryMinimal()
+    : Offset(0), ParentIdx(0), SiblingIdx(0), AbbrevDecl(0) {}
+
+  void dump(raw_ostream &OS, const DWARFCompileUnit *cu,
+            unsigned recurseDepth, unsigned indent = 0) const;
+  void dumpAttribute(raw_ostream &OS, const DWARFCompileUnit *cu,
+                     uint32_t *offset_ptr, uint16_t attr, uint16_t form,
+                     unsigned indent = 0) const;
+
+  bool extractFast(const DWARFCompileUnit *cu, const uint8_t *fixed_form_sizes,
+                   uint32_t *offset_ptr);
+
+  /// Extract a debug info entry for a given compile unit from the
+  /// .debug_info and .debug_abbrev data starting at the given offset.
+  bool extract(const DWARFCompileUnit *cu, uint32_t *offset_ptr);
+
+  uint32_t getTag() const { return AbbrevDecl ? AbbrevDecl->getTag() : 0; }
+  bool isNULL() const { return AbbrevDecl == 0; }
+  uint64_t getOffset() const { return Offset; }
+  uint32_t getNumAttributes() const {
+    return !isNULL() ? AbbrevDecl->getNumAttributes() : 0;
+  }
+  bool hasChildren() const { return !isNULL() && AbbrevDecl->hasChildren(); }
+
+  // We know we are kept in a vector of contiguous entries, so we know
+  // our parent will be some index behind "this".
+  DWARFDebugInfoEntryMinimal *getParent() {
+    return ParentIdx > 0 ? this - ParentIdx : 0;
+  }
+  const DWARFDebugInfoEntryMinimal *getParent() const {
+    return ParentIdx > 0 ? this - ParentIdx : 0;
+  }
+  // We know we are kept in a vector of contiguous entries, so we know
+  // our sibling will be some index after "this".
+  DWARFDebugInfoEntryMinimal *getSibling() {
+    return SiblingIdx > 0 ? this + SiblingIdx : 0;
+  }
+  const DWARFDebugInfoEntryMinimal *getSibling() const {
+    return SiblingIdx > 0 ? this + SiblingIdx : 0;
+  }
+  // We know we are kept in a vector of contiguous entries, so we know
+  // we don't need to store our child pointer, if we have a child it will
+  // be the next entry in the list...
+  DWARFDebugInfoEntryMinimal *getFirstChild() {
+    return hasChildren() ? this + 1 : 0;
+  }
+  const DWARFDebugInfoEntryMinimal *getFirstChild() const {
+    return hasChildren() ? this + 1 : 0;
+  }
+
+  void setParent(DWARFDebugInfoEntryMinimal *parent) {
+    if (parent) {
+      // We know we are kept in a vector of contiguous entries, so we know
+      // our parent will be some index behind "this".
+      ParentIdx = this - parent;
+    } else
+      ParentIdx = 0;
+  }
+  void setSibling(DWARFDebugInfoEntryMinimal *sibling) {
+    if (sibling) {
+      // We know we are kept in a vector of contiguous entries, so we know
+      // our sibling will be some index after "this".
+      SiblingIdx = sibling - this;
+      sibling->setParent(getParent());
+    } else
+      SiblingIdx = 0;
+  }
+
+  const DWARFAbbreviationDeclaration *getAbbreviationDeclarationPtr() const {
+    return AbbrevDecl;
+  }
+
+  uint32_t getAttributeValue(const DWARFCompileUnit *cu,
+                             const uint16_t attr, DWARFFormValue &formValue,
+                             uint32_t *end_attr_offset_ptr = 0) const;
+
+  const char* getAttributeValueAsString(const DWARFCompileUnit* cu,
+                                        const uint16_t attr,
+                                        const char *fail_value) const;
+
+  uint64_t getAttributeValueAsUnsigned(const DWARFCompileUnit *cu,
+                                       const uint16_t attr,
+                                       uint64_t fail_value) const;
+
+  uint64_t getAttributeValueAsReference(const DWARFCompileUnit *cu,
+                                        const uint16_t attr,
+                                        uint64_t fail_value) const;
+
+  int64_t getAttributeValueAsSigned(const DWARFCompileUnit* cu,
+                                    const uint16_t attr,
+                                    int64_t fail_value) const;
+
+  void buildAddressRangeTable(const DWARFCompileUnit *cu,
+                              DWARFDebugAranges *debug_aranges) const;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugLine.cpp b/contrib/llvm/lib/DebugInfo/DWARFDebugLine.cpp
new file mode 100644
index 000000000000..fe1ef78b026e
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugLine.cpp
@@ -0,0 +1,475 @@
+//===-- DWARFDebugLine.cpp ------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFDebugLine.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+using namespace llvm;
+using namespace dwarf;
+
+void DWARFDebugLine::Prologue::dump(raw_ostream &OS) const {
+  OS << "Line table prologue:\n"
+     << format("   total_length: 0x%8.8x\n", TotalLength)
+     << format("        version: %u\n", Version)
+     << format("prologue_length: 0x%8.8x\n", PrologueLength)
+     << format("min_inst_length: %u\n", MinInstLength)
+     << format("default_is_stmt: %u\n", DefaultIsStmt)
+     << format("      line_base: %i\n", LineBase)
+     << format("     line_range: %u\n", LineRange)
+     << format("    opcode_base: %u\n", OpcodeBase);
+
+  for (uint32_t i = 0; i < StandardOpcodeLengths.size(); ++i)
+    OS << format("standard_opcode_lengths[%s] = %u\n", LNStandardString(i+1),
+                 StandardOpcodeLengths[i]);
+
+  if (!IncludeDirectories.empty())
+    for (uint32_t i = 0; i < IncludeDirectories.size(); ++i)
+      OS << format("include_directories[%3u] = '", i+1)
+         << IncludeDirectories[i] << "'\n";
+
+  if (!FileNames.empty()) {
+    OS << "                Dir  Mod Time   File Len   File Name\n"
+       << "                ---- ---------- ---------- -----------"
+          "----------------\n";
+    for (uint32_t i = 0; i < FileNames.size(); ++i) {
+      const FileNameEntry& fileEntry = FileNames[i];
+      OS << format("file_names[%3u] %4u ", i+1, fileEntry.DirIdx)
+         << format("0x%8.8x 0x%8.8x ", fileEntry.ModTime, fileEntry.Length)
+         << fileEntry.Name << '\n';
+    }
+  }
+}
+
+void DWARFDebugLine::Row::postAppend() {
+  BasicBlock = false;
+  PrologueEnd = false;
+  EpilogueBegin = false;
+}
+
+void DWARFDebugLine::Row::reset(bool default_is_stmt) {
+  Address = 0;
+  Line = 1;
+  Column = 0;
+  File = 1;
+  Isa = 0;
+  IsStmt = default_is_stmt;
+  BasicBlock = false;
+  EndSequence = false;
+  PrologueEnd = false;
+  EpilogueBegin = false;
+}
+
+void DWARFDebugLine::Row::dump(raw_ostream &OS) const {
+  OS << format("0x%16.16llx %6u %6u", Address, Line, Column)
+     << format(" %6u %3u ", File, Isa)
+     << (IsStmt ? " is_stmt" : "")
+     << (BasicBlock ? " basic_block" : "")
+     << (PrologueEnd ? " prologue_end" : "")
+     << (EpilogueBegin ? " epilogue_begin" : "")
+     << (EndSequence ? " end_sequence" : "")
+     << '\n';
+}
+
+void DWARFDebugLine::LineTable::dump(raw_ostream &OS) const {
+  Prologue.dump(OS);
+  OS << '\n';
+
+  if (!Rows.empty()) {
+    OS << "Address            Line   Column File   ISA Flags\n"
+       << "------------------ ------ ------ ------ --- -------------\n";
+    for (std::vector<Row>::const_iterator pos = Rows.begin(),
+         end = Rows.end(); pos != end; ++pos)
+      pos->dump(OS);
+  }
+}
+
+DWARFDebugLine::State::~State() {}
+
+void DWARFDebugLine::State::appendRowToMatrix(uint32_t offset) {
+  ++row;  // Increase the row number.
+  LineTable::appendRow(*this);
+  Row::postAppend();
+}
+
+DWARFDebugLine::DumpingState::~DumpingState() {}
+
+void DWARFDebugLine::DumpingState::finalize(uint32_t offset) {
+  LineTable::dump(OS);
+}
+
+const DWARFDebugLine::LineTable *
+DWARFDebugLine::getLineTable(uint32_t offset) const {
+  LineTableConstIter pos = LineTableMap.find(offset);
+  if (pos != LineTableMap.end())
+    return &pos->second;
+  return 0;
+}
+
+const DWARFDebugLine::LineTable *
+DWARFDebugLine::getOrParseLineTable(DataExtractor debug_line_data,
+                                    uint32_t offset) {
+  std::pair<LineTableIter, bool> pos =
+    LineTableMap.insert(LineTableMapTy::value_type(offset, LineTable()));
+  if (pos.second) {
+    // Parse and cache the line table for at this offset.
+    State state;
+    if (!parseStatementTable(debug_line_data, &offset, state))
+      return 0;
+    pos.first->second = state;
+  }
+  return &pos.first->second;
+}
+
+bool
+DWARFDebugLine::parsePrologue(DataExtractor debug_line_data,
+                              uint32_t *offset_ptr, Prologue *prologue) {
+  const uint32_t prologue_offset = *offset_ptr;
+
+  prologue->clear();
+  prologue->TotalLength = debug_line_data.getU32(offset_ptr);
+  prologue->Version = debug_line_data.getU16(offset_ptr);
+  if (prologue->Version != 2)
+    return false;
+
+  prologue->PrologueLength = debug_line_data.getU32(offset_ptr);
+  const uint32_t end_prologue_offset = prologue->PrologueLength + *offset_ptr;
+  prologue->MinInstLength = debug_line_data.getU8(offset_ptr);
+  prologue->DefaultIsStmt = debug_line_data.getU8(offset_ptr);
+  prologue->LineBase = debug_line_data.getU8(offset_ptr);
+  prologue->LineRange = debug_line_data.getU8(offset_ptr);
+  prologue->OpcodeBase = debug_line_data.getU8(offset_ptr);
+
+  prologue->StandardOpcodeLengths.reserve(prologue->OpcodeBase-1);
+  for (uint32_t i = 1; i < prologue->OpcodeBase; ++i) {
+    uint8_t op_len = debug_line_data.getU8(offset_ptr);
+    prologue->StandardOpcodeLengths.push_back(op_len);
+  }
+
+  while (*offset_ptr < end_prologue_offset) {
+    const char *s = debug_line_data.getCStr(offset_ptr);
+    if (s && s[0])
+      prologue->IncludeDirectories.push_back(s);
+    else
+      break;
+  }
+
+  while (*offset_ptr < end_prologue_offset) {
+    const char *name = debug_line_data.getCStr(offset_ptr);
+    if (name && name[0]) {
+      FileNameEntry fileEntry;
+      fileEntry.Name = name;
+      fileEntry.DirIdx = debug_line_data.getULEB128(offset_ptr);
+      fileEntry.ModTime = debug_line_data.getULEB128(offset_ptr);
+      fileEntry.Length = debug_line_data.getULEB128(offset_ptr);
+      prologue->FileNames.push_back(fileEntry);
+    } else {
+      break;
+    }
+  }
+
+  if (*offset_ptr != end_prologue_offset) {
+    fprintf(stderr, "warning: parsing line table prologue at 0x%8.8x should"
+                    " have ended at 0x%8.8x but it ended ad 0x%8.8x\n",
+            prologue_offset, end_prologue_offset, *offset_ptr);
+  }
+  return end_prologue_offset;
+}
+
+bool
+DWARFDebugLine::parseStatementTable(DataExtractor debug_line_data,
+                                    uint32_t *offset_ptr, State &state) {
+  const uint32_t debug_line_offset = *offset_ptr;
+
+  Prologue *prologue = &state.Prologue;
+
+  if (!parsePrologue(debug_line_data, offset_ptr, prologue)) {
+    // Restore our offset and return false to indicate failure!
+    *offset_ptr = debug_line_offset;
+    return false;
+  }
+
+  const uint32_t end_offset = debug_line_offset + prologue->TotalLength +
+                              sizeof(prologue->TotalLength);
+
+  state.reset();
+
+  while (*offset_ptr < end_offset) {
+    uint8_t opcode = debug_line_data.getU8(offset_ptr);
+
+    if (opcode == 0) {
+      // Extended Opcodes always start with a zero opcode followed by
+      // a uleb128 length so you can skip ones you don't know about
+      uint32_t ext_offset = *offset_ptr;
+      uint64_t len = debug_line_data.getULEB128(offset_ptr);
+      uint32_t arg_size = len - (*offset_ptr - ext_offset);
+
+      uint8_t sub_opcode = debug_line_data.getU8(offset_ptr);
+      switch (sub_opcode) {
+      case DW_LNE_end_sequence:
+        // Set the end_sequence register of the state machine to true and
+        // append a row to the matrix using the current values of the
+        // state-machine registers. Then reset the registers to the initial
+        // values specified above. Every statement program sequence must end
+        // with a DW_LNE_end_sequence instruction which creates a row whose
+        // address is that of the byte after the last target machine instruction
+        // of the sequence.
+        state.EndSequence = true;
+        state.appendRowToMatrix(*offset_ptr);
+        state.reset();
+        break;
+
+      case DW_LNE_set_address:
+        // Takes a single relocatable address as an operand. The size of the
+        // operand is the size appropriate to hold an address on the target
+        // machine. Set the address register to the value given by the
+        // relocatable address. All of the other statement program opcodes
+        // that affect the address register add a delta to it. This instruction
+        // stores a relocatable value into it instead.
+        state.Address = debug_line_data.getAddress(offset_ptr);
+        break;
+
+      case DW_LNE_define_file:
+        // Takes 4 arguments. The first is a null terminated string containing
+        // a source file name. The second is an unsigned LEB128 number
+        // representing the directory index of the directory in which the file
+        // was found. The third is an unsigned LEB128 number representing the
+        // time of last modification of the file. The fourth is an unsigned
+        // LEB128 number representing the length in bytes of the file. The time
+        // and length fields may contain LEB128(0) if the information is not
+        // available.
+        //
+        // The directory index represents an entry in the include_directories
+        // section of the statement program prologue. The index is LEB128(0)
+        // if the file was found in the current directory of the compilation,
+        // LEB128(1) if it was found in the first directory in the
+        // include_directories section, and so on. The directory index is
+        // ignored for file names that represent full path names.
+        //
+        // The files are numbered, starting at 1, in the order in which they
+        // appear; the names in the prologue come before names defined by
+        // the DW_LNE_define_file instruction. These numbers are used in the
+        // the file register of the state machine.
+        {
+          FileNameEntry fileEntry;
+          fileEntry.Name = debug_line_data.getCStr(offset_ptr);
+          fileEntry.DirIdx = debug_line_data.getULEB128(offset_ptr);
+          fileEntry.ModTime = debug_line_data.getULEB128(offset_ptr);
+          fileEntry.Length = debug_line_data.getULEB128(offset_ptr);
+          prologue->FileNames.push_back(fileEntry);
+        }
+        break;
+
+      default:
+        // Length doesn't include the zero opcode byte or the length itself, but
+        // it does include the sub_opcode, so we have to adjust for that below
+        (*offset_ptr) += arg_size;
+        break;
+      }
+    } else if (opcode < prologue->OpcodeBase) {
+      switch (opcode) {
+      // Standard Opcodes
+      case DW_LNS_copy:
+        // Takes no arguments. Append a row to the matrix using the
+        // current values of the state-machine registers. Then set
+        // the basic_block register to false.
+        state.appendRowToMatrix(*offset_ptr);
+        break;
+
+      case DW_LNS_advance_pc:
+        // Takes a single unsigned LEB128 operand, multiplies it by the
+        // min_inst_length field of the prologue, and adds the
+        // result to the address register of the state machine.
+        state.Address += debug_line_data.getULEB128(offset_ptr) *
+                         prologue->MinInstLength;
+        break;
+
+      case DW_LNS_advance_line:
+        // Takes a single signed LEB128 operand and adds that value to
+        // the line register of the state machine.
+        state.Line += debug_line_data.getSLEB128(offset_ptr);
+        break;
+
+      case DW_LNS_set_file:
+        // Takes a single unsigned LEB128 operand and stores it in the file
+        // register of the state machine.
+        state.File = debug_line_data.getULEB128(offset_ptr);
+        break;
+
+      case DW_LNS_set_column:
+        // Takes a single unsigned LEB128 operand and stores it in the
+        // column register of the state machine.
+        state.Column = debug_line_data.getULEB128(offset_ptr);
+        break;
+
+      case DW_LNS_negate_stmt:
+        // Takes no arguments. Set the is_stmt register of the state
+        // machine to the logical negation of its current value.
+        state.IsStmt = !state.IsStmt;
+        break;
+
+      case DW_LNS_set_basic_block:
+        // Takes no arguments. Set the basic_block register of the
+        // state machine to true
+        state.BasicBlock = true;
+        break;
+
+      case DW_LNS_const_add_pc:
+        // Takes no arguments. Add to the address register of the state
+        // machine the address increment value corresponding to special
+        // opcode 255. The motivation for DW_LNS_const_add_pc is this:
+        // when the statement program needs to advance the address by a
+        // small amount, it can use a single special opcode, which occupies
+        // a single byte. When it needs to advance the address by up to
+        // twice the range of the last special opcode, it can use
+        // DW_LNS_const_add_pc followed by a special opcode, for a total
+        // of two bytes. Only if it needs to advance the address by more
+        // than twice that range will it need to use both DW_LNS_advance_pc
+        // and a special opcode, requiring three or more bytes.
+        {
+          uint8_t adjust_opcode = 255 - prologue->OpcodeBase;
+          uint64_t addr_offset = (adjust_opcode / prologue->LineRange) *
+                                 prologue->MinInstLength;
+          state.Address += addr_offset;
+        }
+        break;
+
+      case DW_LNS_fixed_advance_pc:
+        // Takes a single uhalf operand. Add to the address register of
+        // the state machine the value of the (unencoded) operand. This
+        // is the only extended opcode that takes an argument that is not
+        // a variable length number. The motivation for DW_LNS_fixed_advance_pc
+        // is this: existing assemblers cannot emit DW_LNS_advance_pc or
+        // special opcodes because they cannot encode LEB128 numbers or
+        // judge when the computation of a special opcode overflows and
+        // requires the use of DW_LNS_advance_pc. Such assemblers, however,
+        // can use DW_LNS_fixed_advance_pc instead, sacrificing compression.
+        state.Address += debug_line_data.getU16(offset_ptr);
+        break;
+
+      case DW_LNS_set_prologue_end:
+        // Takes no arguments. Set the prologue_end register of the
+        // state machine to true
+        state.PrologueEnd = true;
+        break;
+
+      case DW_LNS_set_epilogue_begin:
+        // Takes no arguments. Set the basic_block register of the
+        // state machine to true
+        state.EpilogueBegin = true;
+        break;
+
+      case DW_LNS_set_isa:
+        // Takes a single unsigned LEB128 operand and stores it in the
+        // column register of the state machine.
+        state.Isa = debug_line_data.getULEB128(offset_ptr);
+        break;
+
+      default:
+        // Handle any unknown standard opcodes here. We know the lengths
+        // of such opcodes because they are specified in the prologue
+        // as a multiple of LEB128 operands for each opcode.
+        {
+          assert(opcode - 1U < prologue->StandardOpcodeLengths.size());
+          uint8_t opcode_length = prologue->StandardOpcodeLengths[opcode - 1];
+          for (uint8_t i=0; i<opcode_length; ++i)
+            debug_line_data.getULEB128(offset_ptr);
+        }
+        break;
+      }
+    } else {
+      // Special Opcodes
+
+      // A special opcode value is chosen based on the amount that needs
+      // to be added to the line and address registers. The maximum line
+      // increment for a special opcode is the value of the line_base
+      // field in the header, plus the value of the line_range field,
+      // minus 1 (line base + line range - 1). If the desired line
+      // increment is greater than the maximum line increment, a standard
+      // opcode must be used instead of a special opcode. The "address
+      // advance" is calculated by dividing the desired address increment
+      // by the minimum_instruction_length field from the header. The
+      // special opcode is then calculated using the following formula:
+      //
+      //  opcode = (desired line increment - line_base) +
+      //           (line_range * address advance) + opcode_base
+      //
+      // If the resulting opcode is greater than 255, a standard opcode
+      // must be used instead.
+      //
+      // To decode a special opcode, subtract the opcode_base from the
+      // opcode itself to give the adjusted opcode. The amount to
+      // increment the address register is the result of the adjusted
+      // opcode divided by the line_range multiplied by the
+      // minimum_instruction_length field from the header. That is:
+      //
+      //  address increment = (adjusted opcode / line_range) *
+      //                      minimum_instruction_length
+      //
+      // The amount to increment the line register is the line_base plus
+      // the result of the adjusted opcode modulo the line_range. That is:
+      //
+      // line increment = line_base + (adjusted opcode % line_range)
+
+      uint8_t adjust_opcode = opcode - prologue->OpcodeBase;
+      uint64_t addr_offset = (adjust_opcode / prologue->LineRange) *
+                             prologue->MinInstLength;
+      int32_t line_offset = prologue->LineBase +
+                            (adjust_opcode % prologue->LineRange);
+      state.Line += line_offset;
+      state.Address += addr_offset;
+      state.appendRowToMatrix(*offset_ptr);
+    }
+  }
+
+  state.finalize(*offset_ptr);
+
+  return end_offset;
+}
+
+static bool findMatchingAddress(const DWARFDebugLine::Row& row1,
+                                const DWARFDebugLine::Row& row2) {
+  return row1.Address < row2.Address;
+}
+
+uint32_t
+DWARFDebugLine::LineTable::lookupAddress(uint64_t address,
+                                         uint64_t cu_high_pc) const {
+  uint32_t index = UINT32_MAX;
+  if (!Rows.empty()) {
+    // Use the lower_bound algorithm to perform a binary search since we know
+    // that our line table data is ordered by address.
+    DWARFDebugLine::Row row;
+    row.Address = address;
+    typedef std::vector<Row>::const_iterator iterator;
+    iterator begin_pos = Rows.begin();
+    iterator end_pos = Rows.end();
+    iterator pos = std::lower_bound(begin_pos, end_pos, row,
+                                    findMatchingAddress);
+    if (pos == end_pos) {
+      if (address < cu_high_pc)
+        return Rows.size()-1;
+    } else {
+      // Rely on fact that we are using a std::vector and we can do
+      // pointer arithmetic to find the row index (which will be one less
+      // that what we found since it will find the first position after
+      // the current address) since std::vector iterators are just
+      // pointers to the container type.
+      index = pos - begin_pos;
+      if (pos->Address > address) {
+        if (index > 0)
+          --index;
+        else
+          index = UINT32_MAX;
+      }
+    }
+  }
+  return index; // Failed to find address.
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugLine.h b/contrib/llvm/lib/DebugInfo/DWARFDebugLine.h
new file mode 100644
index 000000000000..bc6a70b11120
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFDebugLine.h
@@ -0,0 +1,190 @@
+//===-- DWARFDebugLine.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFDEBUGLINE_H
+#define LLVM_DEBUGINFO_DWARFDEBUGLINE_H
+
+#include "llvm/Support/DataExtractor.h"
+#include <map>
+#include <string>
+#include <vector>
+
+namespace llvm {
+
+class raw_ostream;
+
+class DWARFDebugLine {
+public:
+  struct FileNameEntry {
+    FileNameEntry() : DirIdx(0), ModTime(0), Length(0) {}
+
+    std::string Name;
+    uint64_t DirIdx;
+    uint64_t ModTime;
+    uint64_t Length;
+  };
+
+  struct Prologue {
+    Prologue()
+      : TotalLength(0), Version(0), PrologueLength(0), MinInstLength(0),
+        DefaultIsStmt(0), LineBase(0), LineRange(0), OpcodeBase(0) {}
+
+    // The size in bytes of the statement information for this compilation unit
+    // (not including the total_length field itself).
+    uint32_t TotalLength;
+    // Version identifier for the statement information format.
+    uint16_t Version;
+    // The number of bytes following the prologue_length field to the beginning
+    // of the first byte of the statement program itself.
+    uint32_t PrologueLength;
+    // The size in bytes of the smallest target machine instruction. Statement
+    // program opcodes that alter the address register first multiply their
+    // operands by this value.
+    uint8_t MinInstLength;
+    // The initial value of theis_stmtregister.
+    uint8_t DefaultIsStmt;
+    // This parameter affects the meaning of the special opcodes. See below.
+    int8_t LineBase;
+    // This parameter affects the meaning of the special opcodes. See below.
+    uint8_t LineRange;
+    // The number assigned to the first special opcode.
+    uint8_t OpcodeBase;
+    std::vector<uint8_t> StandardOpcodeLengths;
+    std::vector<std::string> IncludeDirectories;
+    std::vector<FileNameEntry> FileNames;
+
+    // Length of the prologue in bytes.
+    uint32_t getLength() const {
+      return PrologueLength + sizeof(TotalLength) + sizeof(Version) +
+             sizeof(PrologueLength);
+    }
+    // Length of the line table data in bytes (not including the prologue).
+    uint32_t getStatementTableLength() const {
+      return TotalLength + sizeof(TotalLength) - getLength();
+    }
+    int32_t getMaxLineIncrementForSpecialOpcode() const {
+      return LineBase + (int8_t)LineRange - 1;
+    }
+    void dump(raw_ostream &OS) const;
+    void clear() {
+      TotalLength = Version = PrologueLength = 0;
+      MinInstLength = LineBase = LineRange = OpcodeBase = 0;
+      StandardOpcodeLengths.clear();
+      IncludeDirectories.clear();
+      FileNames.clear();
+    }
+  };
+
+  // Standard .debug_line state machine structure.
+  struct Row {
+    Row(bool default_is_stmt = false) { reset(default_is_stmt); }
+    /// Called after a row is appended to the matrix.
+    void postAppend();
+    void reset(bool default_is_stmt);
+    void dump(raw_ostream &OS) const;
+
+    // The program-counter value corresponding to a machine instruction
+    // generated by the compiler.
+    uint64_t Address;
+    // An unsigned integer indicating a source line number. Lines are numbered
+    // beginning at 1. The compiler may emit the value 0 in cases where an
+    // instruction cannot be attributed to any source line.
+    uint32_t Line;
+    // An unsigned integer indicating a column number within a source line.
+    // Columns are numbered beginning at 1. The value 0 is reserved to indicate
+    // that a statement begins at the 'left edge' of the line.
+    uint16_t Column;
+    // An unsigned integer indicating the identity of the source file
+    // corresponding to a machine instruction.
+    uint16_t File;
+    // An unsigned integer whose value encodes the applicable instruction set
+    // architecture for the current instruction.
+    uint8_t Isa;
+    // A boolean indicating that the current instruction is the beginning of a
+    // statement.
+    uint8_t IsStmt:1,
+            // A boolean indicating that the current instruction is the
+            // beginning of a basic block.
+            BasicBlock:1,
+            // A boolean indicating that the current address is that of the
+            // first byte after the end of a sequence of target machine
+            // instructions.
+            EndSequence:1,
+            // A boolean indicating that the current address is one (of possibly
+            // many) where execution should be suspended for an entry breakpoint
+            // of a function.
+            PrologueEnd:1,
+            // A boolean indicating that the current address is one (of possibly
+            // many) where execution should be suspended for an exit breakpoint
+            // of a function.
+            EpilogueBegin:1;
+  };
+
+  struct LineTable {
+    void appendRow(const DWARFDebugLine::Row &state) { Rows.push_back(state); }
+    void clear() {
+      Prologue.clear();
+      Rows.clear();
+    }
+
+    uint32_t lookupAddress(uint64_t address, uint64_t cu_high_pc) const;
+    void dump(raw_ostream &OS) const;
+
+    struct Prologue Prologue;
+    std::vector<Row> Rows;
+  };
+
+  struct State : public Row, public LineTable {
+    // Special row codes.
+    enum {
+      StartParsingLineTable = 0,
+      DoneParsingLineTable = -1
+    };
+
+    State() : row(StartParsingLineTable) {}
+    virtual ~State();
+
+    virtual void appendRowToMatrix(uint32_t offset);
+    virtual void finalize(uint32_t offset) { row = DoneParsingLineTable; }
+    virtual void reset() { Row::reset(Prologue.DefaultIsStmt); }
+
+    // The row number that starts at zero for the prologue, and increases for
+    // each row added to the matrix.
+    unsigned row;
+  };
+
+  struct DumpingState : public State {
+    DumpingState(raw_ostream &OS) : OS(OS) {}
+    virtual ~DumpingState();
+    virtual void finalize(uint32_t offset);
+  private:
+    raw_ostream &OS;
+  };
+
+  static bool parsePrologue(DataExtractor debug_line_data, uint32_t *offset_ptr,
+                            Prologue *prologue);
+  /// Parse a single line table (prologue and all rows).
+  static bool parseStatementTable(DataExtractor debug_line_data,
+                                  uint32_t *offset_ptr, State &state);
+
+  const LineTable *getLineTable(uint32_t offset) const;
+  const LineTable *getOrParseLineTable(DataExtractor debug_line_data,
+                                       uint32_t offset);
+
+private:
+  typedef std::map<uint32_t, LineTable> LineTableMapTy;
+  typedef LineTableMapTy::iterator LineTableIter;
+  typedef LineTableMapTy::const_iterator LineTableConstIter;
+
+  LineTableMapTy LineTableMap;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/DebugInfo/DWARFFormValue.cpp b/contrib/llvm/lib/DebugInfo/DWARFFormValue.cpp
new file mode 100644
index 000000000000..705efe5549b6
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFFormValue.cpp
@@ -0,0 +1,427 @@
+//===-- DWARFFormValue.cpp ------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFFormValue.h"
+#include "DWARFCompileUnit.h"
+#include "DWARFContext.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+using namespace llvm;
+using namespace dwarf;
+
+static const uint8_t form_sizes_addr4[] = {
+  0, // 0x00 unused
+  4, // 0x01 DW_FORM_addr
+  0, // 0x02 unused
+  0, // 0x03 DW_FORM_block2
+  0, // 0x04 DW_FORM_block4
+  2, // 0x05 DW_FORM_data2
+  4, // 0x06 DW_FORM_data4
+  8, // 0x07 DW_FORM_data8
+  0, // 0x08 DW_FORM_string
+  0, // 0x09 DW_FORM_block
+  0, // 0x0a DW_FORM_block1
+  1, // 0x0b DW_FORM_data1
+  1, // 0x0c DW_FORM_flag
+  0, // 0x0d DW_FORM_sdata
+  4, // 0x0e DW_FORM_strp
+  0, // 0x0f DW_FORM_udata
+  4, // 0x10 DW_FORM_ref_addr
+  1, // 0x11 DW_FORM_ref1
+  2, // 0x12 DW_FORM_ref2
+  4, // 0x13 DW_FORM_ref4
+  8, // 0x14 DW_FORM_ref8
+  0, // 0x15 DW_FORM_ref_udata
+  0, // 0x16 DW_FORM_indirect
+};
+
+static const uint8_t form_sizes_addr8[] = {
+  0, // 0x00 unused
+  8, // 0x01 DW_FORM_addr
+  0, // 0x02 unused
+  0, // 0x03 DW_FORM_block2
+  0, // 0x04 DW_FORM_block4
+  2, // 0x05 DW_FORM_data2
+  4, // 0x06 DW_FORM_data4
+  8, // 0x07 DW_FORM_data8
+  0, // 0x08 DW_FORM_string
+  0, // 0x09 DW_FORM_block
+  0, // 0x0a DW_FORM_block1
+  1, // 0x0b DW_FORM_data1
+  1, // 0x0c DW_FORM_flag
+  0, // 0x0d DW_FORM_sdata
+  4, // 0x0e DW_FORM_strp
+  0, // 0x0f DW_FORM_udata
+  8, // 0x10 DW_FORM_ref_addr
+  1, // 0x11 DW_FORM_ref1
+  2, // 0x12 DW_FORM_ref2
+  4, // 0x13 DW_FORM_ref4
+  8, // 0x14 DW_FORM_ref8
+  0, // 0x15 DW_FORM_ref_udata
+  0, // 0x16 DW_FORM_indirect
+};
+
+const uint8_t *
+DWARFFormValue::getFixedFormSizesForAddressSize(uint8_t addr_size) {
+  switch (addr_size) {
+  case 4: return form_sizes_addr4;
+  case 8: return form_sizes_addr8;
+  }
+  return NULL;
+}
+
+bool
+DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
+                             const DWARFCompileUnit *cu) {
+  bool indirect = false;
+  bool is_block = false;
+  Value.data = NULL;
+  // Read the value for the form into value and follow and DW_FORM_indirect
+  // instances we run into
+  do {
+    indirect = false;
+    switch (Form) {
+    case DW_FORM_addr:
+    case DW_FORM_ref_addr:
+      Value.uval = data.getUnsigned(offset_ptr, cu->getAddressByteSize());
+      break;
+    case DW_FORM_block:
+      Value.uval = data.getULEB128(offset_ptr);
+      is_block = true;
+      break;
+    case DW_FORM_block1:
+      Value.uval = data.getU8(offset_ptr);
+      is_block = true;
+      break;
+    case DW_FORM_block2:
+      Value.uval = data.getU16(offset_ptr);
+      is_block = true;
+      break;
+    case DW_FORM_block4:
+      Value.uval = data.getU32(offset_ptr);
+      is_block = true;
+      break;
+    case DW_FORM_data1:
+    case DW_FORM_ref1:
+    case DW_FORM_flag:
+      Value.uval = data.getU8(offset_ptr);
+      break;
+    case DW_FORM_data2:
+    case DW_FORM_ref2:
+      Value.uval = data.getU16(offset_ptr);
+      break;
+    case DW_FORM_data4:
+    case DW_FORM_ref4:
+      Value.uval = data.getU32(offset_ptr);
+      break;
+    case DW_FORM_data8:
+    case DW_FORM_ref8:
+      Value.uval = data.getU64(offset_ptr);
+      break;
+    case DW_FORM_sdata:
+      Value.sval = data.getSLEB128(offset_ptr);
+      break;
+    case DW_FORM_strp:
+      Value.uval = data.getU32(offset_ptr);
+      break;
+    case DW_FORM_udata:
+    case DW_FORM_ref_udata:
+      Value.uval = data.getULEB128(offset_ptr);
+      break;
+    case DW_FORM_string:
+      Value.cstr = data.getCStr(offset_ptr);
+      // Set the string value to also be the data for inlined cstr form
+      // values only so we can tell the differnence between DW_FORM_string
+      // and DW_FORM_strp form values
+      Value.data = (uint8_t*)Value.cstr;
+      break;
+    case DW_FORM_indirect:
+      Form = data.getULEB128(offset_ptr);
+      indirect = true;
+      break;
+    default:
+      return false;
+    }
+  } while (indirect);
+
+  if (is_block) {
+    StringRef str = data.getData().substr(*offset_ptr, Value.uval);
+    Value.data = NULL;
+    if (!str.empty()) {
+      Value.data = reinterpret_cast<const uint8_t *>(str.data());
+      *offset_ptr += Value.uval;
+    }
+  }
+
+  return true;
+}
+
+bool
+DWARFFormValue::skipValue(DataExtractor debug_info_data, uint32_t* offset_ptr,
+                          const DWARFCompileUnit *cu) const {
+  return DWARFFormValue::skipValue(Form, debug_info_data, offset_ptr, cu);
+}
+
+bool
+DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
+                          uint32_t *offset_ptr, const DWARFCompileUnit *cu) {
+  bool indirect = false;
+  do {
+    indirect = false;
+    switch (form) {
+    // Blocks if inlined data that have a length field and the data bytes
+    // inlined in the .debug_info
+    case DW_FORM_block: {
+      uint64_t size = debug_info_data.getULEB128(offset_ptr);
+      *offset_ptr += size;
+      return true;
+    }
+    case DW_FORM_block1: {
+      uint8_t size = debug_info_data.getU8(offset_ptr);
+      *offset_ptr += size;
+      return true;
+    }
+    case DW_FORM_block2: {
+      uint16_t size = debug_info_data.getU16(offset_ptr);
+      *offset_ptr += size;
+      return true;
+    }
+    case DW_FORM_block4: {
+      uint32_t size = debug_info_data.getU32(offset_ptr);
+      *offset_ptr += size;
+      return true;
+    }
+
+    // Inlined NULL terminated C-strings
+    case DW_FORM_string:
+      debug_info_data.getCStr(offset_ptr);
+      return true;
+
+    // Compile unit address sized values
+    case DW_FORM_addr:
+    case DW_FORM_ref_addr:
+      *offset_ptr += cu->getAddressByteSize();
+      return true;
+
+    // 1 byte values
+    case DW_FORM_data1:
+    case DW_FORM_flag:
+    case DW_FORM_ref1:
+      *offset_ptr += 1;
+      return true;
+
+    // 2 byte values
+    case DW_FORM_data2:
+    case DW_FORM_ref2:
+      *offset_ptr += 2;
+      return true;
+
+    // 4 byte values
+    case DW_FORM_strp:
+    case DW_FORM_data4:
+    case DW_FORM_ref4:
+      *offset_ptr += 4;
+      return true;
+
+    // 8 byte values
+    case DW_FORM_data8:
+    case DW_FORM_ref8:
+      *offset_ptr += 8;
+      return true;
+
+    // signed or unsigned LEB 128 values
+    //  case DW_FORM_APPLE_db_str:
+    case DW_FORM_sdata:
+    case DW_FORM_udata:
+    case DW_FORM_ref_udata:
+      debug_info_data.getULEB128(offset_ptr);
+      return true;
+
+    case DW_FORM_indirect:
+      indirect = true;
+      form = debug_info_data.getULEB128(offset_ptr);
+      break;
+    default:
+      return false;
+    }
+  } while (indirect);
+  return true;
+}
+
+void
+DWARFFormValue::dump(raw_ostream &OS, const DWARFCompileUnit *cu) const {
+  DataExtractor debug_str_data(cu->getContext().getStringSection(), true, 0);
+  uint64_t uvalue = getUnsigned();
+  bool cu_relative_offset = false;
+
+  switch (Form) {
+  case DW_FORM_addr:      OS << format("0x%016x", uvalue); break;
+  case DW_FORM_flag:
+  case DW_FORM_data1:     OS << format("0x%02x", uvalue);  break;
+  case DW_FORM_data2:     OS << format("0x%04x", uvalue);  break;
+  case DW_FORM_data4:     OS << format("0x%08x", uvalue);  break;
+  case DW_FORM_data8:     OS << format("0x%016x", uvalue); break;
+  case DW_FORM_string:
+    OS << '"';
+    OS.write_escaped(getAsCString(NULL));
+    OS << '"';
+    break;
+  case DW_FORM_block:
+  case DW_FORM_block1:
+  case DW_FORM_block2:
+  case DW_FORM_block4:
+    if (uvalue > 0) {
+      switch (Form) {
+      case DW_FORM_block:  OS << format("<0x%llx> ", uvalue);            break;
+      case DW_FORM_block1: OS << format("<0x%2.2x> ", (uint8_t)uvalue);  break;
+      case DW_FORM_block2: OS << format("<0x%4.4x> ", (uint16_t)uvalue); break;
+      case DW_FORM_block4: OS << format("<0x%8.8x> ", (uint32_t)uvalue); break;
+      default: break;
+      }
+
+      const uint8_t* data_ptr = Value.data;
+      if (data_ptr) {
+        // uvalue contains size of block
+        const uint8_t* end_data_ptr = data_ptr + uvalue;
+        while (data_ptr < end_data_ptr) {
+          OS << format("%2.2x ", *data_ptr);
+          ++data_ptr;
+        }
+      }
+      else
+        OS << "NULL";
+    }
+    break;
+
+  case DW_FORM_sdata:     OS << getSigned();   break;
+  case DW_FORM_udata:     OS << getUnsigned(); break;
+  case DW_FORM_strp: {
+    OS << format(" .debug_str[0x%8.8x] = ", (uint32_t)uvalue);
+    const char* dbg_str = getAsCString(&debug_str_data);
+    if (dbg_str) {
+      OS << '"';
+      OS.write_escaped(dbg_str);
+      OS << '"';
+    }
+    break;
+  }
+  case DW_FORM_ref_addr:
+    OS << format("0x%016x", uvalue);
+    break;
+  case DW_FORM_ref1:
+    cu_relative_offset = true;
+    OS << format("cu + 0x%2.2x", (uint8_t)uvalue);
+    break;
+  case DW_FORM_ref2:
+    cu_relative_offset = true;
+    OS << format("cu + 0x%4.4x", (uint16_t)uvalue);
+    break;
+  case DW_FORM_ref4:
+    cu_relative_offset = true;
+    OS << format("cu + 0x%4.4x", (uint32_t)uvalue);
+    break;
+  case DW_FORM_ref8:
+    cu_relative_offset = true;
+    OS << format("cu + 0x%8.8llx", uvalue);
+    break;
+  case DW_FORM_ref_udata:
+    cu_relative_offset = true;
+    OS << format("cu + 0x%llx", uvalue);
+    break;
+
+    // All DW_FORM_indirect attributes should be resolved prior to calling
+    // this function
+  case DW_FORM_indirect:
+    OS << "DW_FORM_indirect";
+    break;
+  default:
+    OS << format("DW_FORM(0x%4.4x)", Form);
+    break;
+  }
+
+  if (cu_relative_offset)
+    OS << format(" => {0x%8.8x}", (uvalue + (cu ? cu->getOffset() : 0)));
+}
+
+const char*
+DWARFFormValue::getAsCString(const DataExtractor *debug_str_data_ptr) const {
+  if (isInlinedCStr()) {
+    return Value.cstr;
+  } else if (debug_str_data_ptr) {
+    uint32_t offset = Value.uval;
+    return debug_str_data_ptr->getCStr(&offset);
+  }
+  return NULL;
+}
+
+uint64_t DWARFFormValue::getReference(const DWARFCompileUnit *cu) const {
+  uint64_t die_offset = Value.uval;
+  switch (Form) {
+  case DW_FORM_ref1:
+  case DW_FORM_ref2:
+  case DW_FORM_ref4:
+  case DW_FORM_ref8:
+  case DW_FORM_ref_udata:
+      die_offset += (cu ? cu->getOffset() : 0);
+      break;
+  default:
+      break;
+  }
+
+  return die_offset;
+}
+
+bool
+DWARFFormValue::resolveCompileUnitReferences(const DWARFCompileUnit *cu) {
+  switch (Form) {
+  case DW_FORM_ref1:
+  case DW_FORM_ref2:
+  case DW_FORM_ref4:
+  case DW_FORM_ref8:
+  case DW_FORM_ref_udata:
+    Value.uval += cu->getOffset();
+    Form = DW_FORM_ref_addr;
+    return true;
+  default:
+    break;
+  }
+  return false;
+}
+
+const uint8_t *DWARFFormValue::BlockData() const {
+  if (!isInlinedCStr())
+    return Value.data;
+  return NULL;
+}
+
+bool DWARFFormValue::isBlockForm(uint16_t form) {
+  switch (form) {
+  case DW_FORM_block:
+  case DW_FORM_block1:
+  case DW_FORM_block2:
+  case DW_FORM_block4:
+    return true;
+  }
+  return false;
+}
+
+bool DWARFFormValue::isDataForm(uint16_t form) {
+  switch (form) {
+  case DW_FORM_sdata:
+  case DW_FORM_udata:
+  case DW_FORM_data1:
+  case DW_FORM_data2:
+  case DW_FORM_data4:
+  case DW_FORM_data8:
+    return true;
+  }
+  return false;
+}
diff --git a/contrib/llvm/lib/DebugInfo/DWARFFormValue.h b/contrib/llvm/lib/DebugInfo/DWARFFormValue.h
new file mode 100644
index 000000000000..22ac0116646e
--- /dev/null
+++ b/contrib/llvm/lib/DebugInfo/DWARFFormValue.h
@@ -0,0 +1,78 @@
+//===-- DWARFFormValue.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFFORMVALUE_H
+#define LLVM_DEBUGINFO_DWARFFORMVALUE_H
+
+#include "llvm/Support/DataExtractor.h"
+
+namespace llvm {
+
+class DWARFCompileUnit;
+class raw_ostream;
+
+class DWARFFormValue {
+public:
+  struct ValueType {
+    ValueType() : data(NULL) {
+      uval = 0;
+    }
+
+    union {
+      uint64_t uval;
+      int64_t sval;
+      const char* cstr;
+    };
+    const uint8_t* data;
+  };
+
+  enum {
+    eValueTypeInvalid = 0,
+    eValueTypeUnsigned,
+    eValueTypeSigned,
+    eValueTypeCStr,
+    eValueTypeBlock
+  };
+
+private:
+  uint16_t Form;   // Form for this value.
+  ValueType Value; // Contains all data for the form.
+
+public:
+  DWARFFormValue(uint16_t form = 0) : Form(form) {}
+  uint16_t getForm() const { return Form; }
+  const ValueType& value() const { return Value; }
+  void dump(raw_ostream &OS, const DWARFCompileUnit* cu) const;
+  bool extractValue(DataExtractor data, uint32_t *offset_ptr,
+                    const DWARFCompileUnit *cu);
+  bool isInlinedCStr() const {
+    return Value.data != NULL && Value.data == (uint8_t*)Value.cstr;
+  }
+  const uint8_t *BlockData() const;
+  uint64_t getReference(const DWARFCompileUnit* cu) const;
+
+  /// Resolve any compile unit specific references so that we don't need
+  /// the compile unit at a later time in order to work with the form
+  /// value.
+  bool resolveCompileUnitReferences(const DWARFCompileUnit* cu);
+  uint64_t getUnsigned() const { return Value.uval; }
+  int64_t getSigned() const { return Value.sval; }
+  const char *getAsCString(const DataExtractor *debug_str_data_ptr) const;
+  bool skipValue(DataExtractor debug_info_data, uint32_t *offset_ptr,
+                 const DWARFCompileUnit *cu) const;
+  static bool skipValue(uint16_t form, DataExtractor debug_info_data,
+                        uint32_t *offset_ptr, const DWARFCompileUnit *cu);
+  static bool isBlockForm(uint16_t form);
+  static bool isDataForm(uint16_t form);
+  static const uint8_t *getFixedFormSizesForAddressSize(uint8_t addr_size);
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index 7652090af8f5..525877b68900 100644
--- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -93,7 +93,7 @@ class GVMemoryBlock : public CallbackVH {
   /// \brief Returns the address the GlobalVariable should be written into.  The
   /// GVMemoryBlock object prefixes that.
   static char *Create(const GlobalVariable *GV, const TargetData& TD) {
-    const Type *ElTy = GV->getType()->getElementType();
+    Type *ElTy = GV->getType()->getElementType();
     size_t GVSize = (size_t)TD.getTypeAllocSize(ElTy);
     void *RawMemory = ::operator new(
       TargetData::RoundUpAlignment(sizeof(GVMemoryBlock),
@@ -272,7 +272,7 @@ void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE,
   Array = new char[(InputArgv.size()+1)*PtrSize];
 
   DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array << "\n");
-  const Type *SBytePtr = Type::getInt8PtrTy(C);
+  Type *SBytePtr = Type::getInt8PtrTy(C);
 
   for (unsigned i = 0; i != InputArgv.size(); ++i) {
     unsigned Size = InputArgv[i].size()+1;
@@ -361,8 +361,8 @@ int ExecutionEngine::runFunctionAsMain(Function *Fn,
 
   // Check main() type
   unsigned NumArgs = Fn->getFunctionType()->getNumParams();
-  const FunctionType *FTy = Fn->getFunctionType();
-  const Type* PPInt8Ty = Type::getInt8PtrTy(Fn->getContext())->getPointerTo();
+  FunctionType *FTy = Fn->getFunctionType();
+  Type* PPInt8Ty = Type::getInt8PtrTy(Fn->getContext())->getPointerTo();
 
   // Check the argument types.
   if (NumArgs > 3)
@@ -422,6 +422,7 @@ ExecutionEngine *ExecutionEngine::createJIT(Module *M,
                                             JITMemoryManager *JMM,
                                             CodeGenOpt::Level OptLevel,
                                             bool GVsWithCode,
+                                            Reloc::Model RM,
                                             CodeModel::Model CMM) {
   if (ExecutionEngine::JITCtor == 0) {
     if (ErrorStr)
@@ -436,9 +437,8 @@ ExecutionEngine *ExecutionEngine::createJIT(Module *M,
   SmallVector<std::string, 1> MAttrs;
 
   TargetMachine *TM =
-          EngineBuilder::selectTarget(M, MArch, MCPU, MAttrs, ErrorStr);
+    EngineBuilder::selectTarget(M, MArch, MCPU, MAttrs, RM, CMM, ErrorStr);
   if (!TM || (ErrorStr && ErrorStr->length() > 0)) return 0;
-  TM->setCodeModel(CMM);
 
   return ExecutionEngine::JITCtor(M, ErrorStr, JMM, OptLevel, GVsWithCode, TM);
 }
@@ -465,10 +465,9 @@ ExecutionEngine *EngineBuilder::create() {
   // Unless the interpreter was explicitly selected or the JIT is not linked,
   // try making a JIT.
   if (WhichEngine & EngineKind::JIT) {
-    if (TargetMachine *TM =
-        EngineBuilder::selectTarget(M, MArch, MCPU, MAttrs, ErrorStr)) {
-      TM->setCodeModel(CMModel);
-
+    if (TargetMachine *TM = EngineBuilder::selectTarget(M, MArch, MCPU, MAttrs,
+                                                        RelocModel, CMModel,
+                                                        ErrorStr)) {
       if (UseMCJIT && ExecutionEngine::MCJITCtor) {
         ExecutionEngine *EE =
           ExecutionEngine::MCJITCtor(M, ErrorStr, JMM, OptLevel,
@@ -548,8 +547,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       // Compute the index
       GenericValue Result = getConstantValue(Op0);
       SmallVector<Value*, 8> Indices(CE->op_begin()+1, CE->op_end());
-      uint64_t Offset =
-        TD->getIndexedOffset(Op0->getType(), &Indices[0], Indices.size());
+      uint64_t Offset = TD->getIndexedOffset(Op0->getType(), Indices);
 
       char* tmp = (char*) Result.PointerVal;
       Result = PTOGV(tmp + Offset);
@@ -651,7 +649,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
     }
     case Instruction::BitCast: {
       GenericValue GV = getConstantValue(Op0);
-      const Type* DestTy = CE->getType();
+      Type* DestTy = CE->getType();
       switch (Op0->getType()->getTypeID()) {
         default: llvm_unreachable("Invalid bitcast operand");
         case Type::IntegerTyID:
@@ -847,7 +845,7 @@ static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst,
 }
 
 void ExecutionEngine::StoreValueToMemory(const GenericValue &Val,
-                                         GenericValue *Ptr, const Type *Ty) {
+                                         GenericValue *Ptr, Type *Ty) {
   const unsigned StoreBytes = getTargetData()->getTypeStoreSize(Ty);
 
   switch (Ty->getTypeID()) {
@@ -909,7 +907,7 @@ static void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes) {
 ///
 void ExecutionEngine::LoadValueFromMemory(GenericValue &Result,
                                           GenericValue *Ptr,
-                                          const Type *Ty) {
+                                          Type *Ty) {
   const unsigned LoadBytes = getTargetData()->getTypeStoreSize(Ty);
 
   switch (Ty->getTypeID()) {
@@ -932,7 +930,7 @@ void ExecutionEngine::LoadValueFromMemory(GenericValue &Result,
     // FIXME: Will not trap if loading a signaling NaN.
     uint64_t y[2];
     memcpy(y, Ptr, 10);
-    Result.IntVal = APInt(80, 2, y);
+    Result.IntVal = APInt(80, y);
     break;
   }
   default:
@@ -986,7 +984,7 @@ void ExecutionEngine::emitGlobals() {
   // Loop over all of the global variables in the program, allocating the memory
   // to hold them.  If there is more than one module, do a prepass over globals
   // to figure out how the different modules should link together.
-  std::map<std::pair<std::string, const Type*>,
+  std::map<std::pair<std::string, Type*>,
            const GlobalValue*> LinkedGlobalsMap;
 
   if (Modules.size() != 1) {
@@ -1101,7 +1099,7 @@ void ExecutionEngine::EmitGlobalVariable(const GlobalVariable *GV) {
   if (!GV->isThreadLocal())
     InitializeMemory(GV->getInitializer(), GA);
 
-  const Type *ElTy = GV->getType()->getElementType();
+  Type *ElTy = GV->getType()->getElementType();
   size_t GVSize = (size_t)getTargetData()->getTypeAllocSize(ElTy);
   NumInitBytes += (unsigned)GVSize;
   ++NumGlobals;
diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
index 498063bf6555..27917da07a2c 100644
--- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
@@ -51,7 +51,7 @@ static void SetValue(Value *V, GenericValue Val, ExecutionContext &SF) {
      break
 
 static void executeFAddInst(GenericValue &Dest, GenericValue Src1,
-                            GenericValue Src2, const Type *Ty) {
+                            GenericValue Src2, Type *Ty) {
   switch (Ty->getTypeID()) {
     IMPLEMENT_BINARY_OPERATOR(+, Float);
     IMPLEMENT_BINARY_OPERATOR(+, Double);
@@ -62,7 +62,7 @@ static void executeFAddInst(GenericValue &Dest, GenericValue Src1,
 }
 
 static void executeFSubInst(GenericValue &Dest, GenericValue Src1,
-                            GenericValue Src2, const Type *Ty) {
+                            GenericValue Src2, Type *Ty) {
   switch (Ty->getTypeID()) {
     IMPLEMENT_BINARY_OPERATOR(-, Float);
     IMPLEMENT_BINARY_OPERATOR(-, Double);
@@ -73,7 +73,7 @@ static void executeFSubInst(GenericValue &Dest, GenericValue Src1,
 }
 
 static void executeFMulInst(GenericValue &Dest, GenericValue Src1,
-                            GenericValue Src2, const Type *Ty) {
+                            GenericValue Src2, Type *Ty) {
   switch (Ty->getTypeID()) {
     IMPLEMENT_BINARY_OPERATOR(*, Float);
     IMPLEMENT_BINARY_OPERATOR(*, Double);
@@ -84,7 +84,7 @@ static void executeFMulInst(GenericValue &Dest, GenericValue Src1,
 }
 
 static void executeFDivInst(GenericValue &Dest, GenericValue Src1, 
-                            GenericValue Src2, const Type *Ty) {
+                            GenericValue Src2, Type *Ty) {
   switch (Ty->getTypeID()) {
     IMPLEMENT_BINARY_OPERATOR(/, Float);
     IMPLEMENT_BINARY_OPERATOR(/, Double);
@@ -95,7 +95,7 @@ static void executeFDivInst(GenericValue &Dest, GenericValue Src1,
 }
 
 static void executeFRemInst(GenericValue &Dest, GenericValue Src1, 
-                            GenericValue Src2, const Type *Ty) {
+                            GenericValue Src2, Type *Ty) {
   switch (Ty->getTypeID()) {
   case Type::FloatTyID:
     Dest.FloatVal = fmod(Src1.FloatVal, Src2.FloatVal);
@@ -125,7 +125,7 @@ static void executeFRemInst(GenericValue &Dest, GenericValue Src1,
       break;
 
 static GenericValue executeICMP_EQ(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(eq,Ty);
@@ -138,7 +138,7 @@ static GenericValue executeICMP_EQ(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_NE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(ne,Ty);
@@ -151,7 +151,7 @@ static GenericValue executeICMP_NE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_ULT(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(ult,Ty);
@@ -164,7 +164,7 @@ static GenericValue executeICMP_ULT(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_SLT(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(slt,Ty);
@@ -177,7 +177,7 @@ static GenericValue executeICMP_SLT(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_UGT(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(ugt,Ty);
@@ -190,7 +190,7 @@ static GenericValue executeICMP_UGT(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_SGT(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(sgt,Ty);
@@ -203,7 +203,7 @@ static GenericValue executeICMP_SGT(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_ULE(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(ule,Ty);
@@ -216,7 +216,7 @@ static GenericValue executeICMP_ULE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_SLE(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(sle,Ty);
@@ -229,7 +229,7 @@ static GenericValue executeICMP_SLE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_UGE(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(uge,Ty);
@@ -242,7 +242,7 @@ static GenericValue executeICMP_UGE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeICMP_SGE(GenericValue Src1, GenericValue Src2,
-                                    const Type *Ty) {
+                                    Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_INTEGER_ICMP(sge,Ty);
@@ -256,7 +256,7 @@ static GenericValue executeICMP_SGE(GenericValue Src1, GenericValue Src2,
 
 void Interpreter::visitICmpInst(ICmpInst &I) {
   ExecutionContext &SF = ECStack.back();
-  const Type *Ty    = I.getOperand(0)->getType();
+  Type *Ty    = I.getOperand(0)->getType();
   GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
   GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
   GenericValue R;   // Result
@@ -286,7 +286,7 @@ void Interpreter::visitICmpInst(ICmpInst &I) {
      break
 
 static GenericValue executeFCMP_OEQ(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_FCMP(==, Float);
@@ -299,7 +299,7 @@ static GenericValue executeFCMP_OEQ(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeFCMP_ONE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_FCMP(!=, Float);
@@ -313,7 +313,7 @@ static GenericValue executeFCMP_ONE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeFCMP_OLE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_FCMP(<=, Float);
@@ -326,7 +326,7 @@ static GenericValue executeFCMP_OLE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeFCMP_OGE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_FCMP(>=, Float);
@@ -339,7 +339,7 @@ static GenericValue executeFCMP_OGE(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeFCMP_OLT(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_FCMP(<, Float);
@@ -352,7 +352,7 @@ static GenericValue executeFCMP_OLT(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeFCMP_OGT(GenericValue Src1, GenericValue Src2,
-                                     const Type *Ty) {
+                                     Type *Ty) {
   GenericValue Dest;
   switch (Ty->getTypeID()) {
     IMPLEMENT_FCMP(>, Float);
@@ -377,49 +377,49 @@ static GenericValue executeFCMP_OGT(GenericValue Src1, GenericValue Src2,
 
 
 static GenericValue executeFCMP_UEQ(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   IMPLEMENT_UNORDERED(Ty, Src1, Src2)
   return executeFCMP_OEQ(Src1, Src2, Ty);
 }
 
 static GenericValue executeFCMP_UNE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   IMPLEMENT_UNORDERED(Ty, Src1, Src2)
   return executeFCMP_ONE(Src1, Src2, Ty);
 }
 
 static GenericValue executeFCMP_ULE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   IMPLEMENT_UNORDERED(Ty, Src1, Src2)
   return executeFCMP_OLE(Src1, Src2, Ty);
 }
 
 static GenericValue executeFCMP_UGE(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   IMPLEMENT_UNORDERED(Ty, Src1, Src2)
   return executeFCMP_OGE(Src1, Src2, Ty);
 }
 
 static GenericValue executeFCMP_ULT(GenericValue Src1, GenericValue Src2,
-                                   const Type *Ty) {
+                                   Type *Ty) {
   GenericValue Dest;
   IMPLEMENT_UNORDERED(Ty, Src1, Src2)
   return executeFCMP_OLT(Src1, Src2, Ty);
 }
 
 static GenericValue executeFCMP_UGT(GenericValue Src1, GenericValue Src2,
-                                     const Type *Ty) {
+                                     Type *Ty) {
   GenericValue Dest;
   IMPLEMENT_UNORDERED(Ty, Src1, Src2)
   return executeFCMP_OGT(Src1, Src2, Ty);
 }
 
 static GenericValue executeFCMP_ORD(GenericValue Src1, GenericValue Src2,
-                                     const Type *Ty) {
+                                     Type *Ty) {
   GenericValue Dest;
   if (Ty->isFloatTy())
     Dest.IntVal = APInt(1,(Src1.FloatVal == Src1.FloatVal && 
@@ -431,7 +431,7 @@ static GenericValue executeFCMP_ORD(GenericValue Src1, GenericValue Src2,
 }
 
 static GenericValue executeFCMP_UNO(GenericValue Src1, GenericValue Src2,
-                                     const Type *Ty) {
+                                     Type *Ty) {
   GenericValue Dest;
   if (Ty->isFloatTy())
     Dest.IntVal = APInt(1,(Src1.FloatVal != Src1.FloatVal || 
@@ -444,7 +444,7 @@ static GenericValue executeFCMP_UNO(GenericValue Src1, GenericValue Src2,
 
 void Interpreter::visitFCmpInst(FCmpInst &I) {
   ExecutionContext &SF = ECStack.back();
-  const Type *Ty    = I.getOperand(0)->getType();
+  Type *Ty    = I.getOperand(0)->getType();
   GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
   GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
   GenericValue R;   // Result
@@ -475,7 +475,7 @@ void Interpreter::visitFCmpInst(FCmpInst &I) {
 }
 
 static GenericValue executeCmpInst(unsigned predicate, GenericValue Src1, 
-                                   GenericValue Src2, const Type *Ty) {
+                                   GenericValue Src2, Type *Ty) {
   GenericValue Result;
   switch (predicate) {
   case ICmpInst::ICMP_EQ:    return executeICMP_EQ(Src1, Src2, Ty);
@@ -520,7 +520,7 @@ static GenericValue executeCmpInst(unsigned predicate, GenericValue Src1,
 
 void Interpreter::visitBinaryOperator(BinaryOperator &I) {
   ExecutionContext &SF = ECStack.back();
-  const Type *Ty    = I.getOperand(0)->getType();
+  Type *Ty    = I.getOperand(0)->getType();
   GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
   GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
   GenericValue R;   // Result
@@ -585,7 +585,7 @@ void Interpreter::exitCalled(GenericValue GV) {
 /// care of switching to the normal destination BB, if we are returning
 /// from an invoke.
 ///
-void Interpreter::popStackAndReturnValueToCaller(const Type *RetTy,
+void Interpreter::popStackAndReturnValueToCaller(Type *RetTy,
                                                  GenericValue Result) {
   // Pop the current stack frame.
   ECStack.pop_back();
@@ -613,7 +613,7 @@ void Interpreter::popStackAndReturnValueToCaller(const Type *RetTy,
 
 void Interpreter::visitReturnInst(ReturnInst &I) {
   ExecutionContext &SF = ECStack.back();
-  const Type *RetTy = Type::getVoidTy(I.getContext());
+  Type *RetTy = Type::getVoidTy(I.getContext());
   GenericValue Result;
 
   // Save away the return value... (if we are not 'ret void')
@@ -662,18 +662,21 @@ void Interpreter::visitBranchInst(BranchInst &I) {
 
 void Interpreter::visitSwitchInst(SwitchInst &I) {
   ExecutionContext &SF = ECStack.back();
-  GenericValue CondVal = getOperandValue(I.getOperand(0), SF);
-  const Type *ElTy = I.getOperand(0)->getType();
+  Value* Cond = I.getCondition();
+  Type *ElTy = Cond->getType();
+  GenericValue CondVal = getOperandValue(Cond, SF);
 
   // Check to see if any of the cases match...
   BasicBlock *Dest = 0;
-  for (unsigned i = 2, e = I.getNumOperands(); i != e; i += 2)
-    if (executeICMP_EQ(CondVal, getOperandValue(I.getOperand(i), SF), ElTy)
-        .IntVal != 0) {
-      Dest = cast<BasicBlock>(I.getOperand(i+1));
+  unsigned NumCases = I.getNumCases();
+  // Skip the first item since that's the default case.
+  for (unsigned i = 1; i < NumCases; ++i) {
+    GenericValue CaseVal = getOperandValue(I.getCaseValue(i), SF);
+    if (executeICMP_EQ(CondVal, CaseVal, ElTy).IntVal != 0) {
+      Dest = cast<BasicBlock>(I.getSuccessor(i));
       break;
     }
-
+  }
   if (!Dest) Dest = I.getDefaultDest();   // No cases matched: use default
   SwitchToNewBasicBlock(Dest, SF);
 }
@@ -730,7 +733,7 @@ void Interpreter::SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF){
 void Interpreter::visitAllocaInst(AllocaInst &I) {
   ExecutionContext &SF = ECStack.back();
 
-  const Type *Ty = I.getType()->getElementType();  // Type to be allocated
+  Type *Ty = I.getType()->getElementType();  // Type to be allocated
 
   // Get the number of elements being allocated by the array...
   unsigned NumElements = 
@@ -767,7 +770,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I,
   uint64_t Total = 0;
 
   for (; I != E; ++I) {
-    if (const StructType *STy = dyn_cast<StructType>(*I)) {
+    if (StructType *STy = dyn_cast<StructType>(*I)) {
       const StructLayout *SLO = TD.getStructLayout(STy);
 
       const ConstantInt *CPU = cast<ConstantInt>(I.getOperand());
@@ -775,7 +778,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I,
 
       Total += SLO->getElementOffset(Index);
     } else {
-      const SequentialType *ST = cast<SequentialType>(*I);
+      SequentialType *ST = cast<SequentialType>(*I);
       // Get the index number for the array... which must be long type...
       GenericValue IdxGV = getOperandValue(I.getOperand(), SF);
 
@@ -929,34 +932,34 @@ void Interpreter::visitAShr(BinaryOperator &I) {
   SetValue(&I, Dest, SF);
 }
 
-GenericValue Interpreter::executeTruncInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeTruncInst(Value *SrcVal, Type *DstTy,
                                            ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
-  const IntegerType *DITy = cast<IntegerType>(DstTy);
+  IntegerType *DITy = cast<IntegerType>(DstTy);
   unsigned DBitWidth = DITy->getBitWidth();
   Dest.IntVal = Src.IntVal.trunc(DBitWidth);
   return Dest;
 }
 
-GenericValue Interpreter::executeSExtInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeSExtInst(Value *SrcVal, Type *DstTy,
                                           ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
-  const IntegerType *DITy = cast<IntegerType>(DstTy);
+  IntegerType *DITy = cast<IntegerType>(DstTy);
   unsigned DBitWidth = DITy->getBitWidth();
   Dest.IntVal = Src.IntVal.sext(DBitWidth);
   return Dest;
 }
 
-GenericValue Interpreter::executeZExtInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeZExtInst(Value *SrcVal, Type *DstTy,
                                           ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
-  const IntegerType *DITy = cast<IntegerType>(DstTy);
+  IntegerType *DITy = cast<IntegerType>(DstTy);
   unsigned DBitWidth = DITy->getBitWidth();
   Dest.IntVal = Src.IntVal.zext(DBitWidth);
   return Dest;
 }
 
-GenericValue Interpreter::executeFPTruncInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeFPTruncInst(Value *SrcVal, Type *DstTy,
                                              ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(SrcVal->getType()->isDoubleTy() && DstTy->isFloatTy() &&
@@ -965,7 +968,7 @@ GenericValue Interpreter::executeFPTruncInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeFPExtInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeFPExtInst(Value *SrcVal, Type *DstTy,
                                            ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(SrcVal->getType()->isFloatTy() && DstTy->isDoubleTy() &&
@@ -974,9 +977,9 @@ GenericValue Interpreter::executeFPExtInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeFPToUIInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeFPToUIInst(Value *SrcVal, Type *DstTy,
                                             ExecutionContext &SF) {
-  const Type *SrcTy = SrcVal->getType();
+  Type *SrcTy = SrcVal->getType();
   uint32_t DBitWidth = cast<IntegerType>(DstTy)->getBitWidth();
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(SrcTy->isFloatingPointTy() && "Invalid FPToUI instruction");
@@ -988,9 +991,9 @@ GenericValue Interpreter::executeFPToUIInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeFPToSIInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeFPToSIInst(Value *SrcVal, Type *DstTy,
                                             ExecutionContext &SF) {
-  const Type *SrcTy = SrcVal->getType();
+  Type *SrcTy = SrcVal->getType();
   uint32_t DBitWidth = cast<IntegerType>(DstTy)->getBitWidth();
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(SrcTy->isFloatingPointTy() && "Invalid FPToSI instruction");
@@ -1002,7 +1005,7 @@ GenericValue Interpreter::executeFPToSIInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeUIToFPInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeUIToFPInst(Value *SrcVal, Type *DstTy,
                                             ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(DstTy->isFloatingPointTy() && "Invalid UIToFP instruction");
@@ -1014,7 +1017,7 @@ GenericValue Interpreter::executeUIToFPInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeSIToFPInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeSIToFPInst(Value *SrcVal, Type *DstTy,
                                             ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(DstTy->isFloatingPointTy() && "Invalid SIToFP instruction");
@@ -1027,7 +1030,7 @@ GenericValue Interpreter::executeSIToFPInst(Value *SrcVal, const Type *DstTy,
 
 }
 
-GenericValue Interpreter::executePtrToIntInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executePtrToIntInst(Value *SrcVal, Type *DstTy,
                                               ExecutionContext &SF) {
   uint32_t DBitWidth = cast<IntegerType>(DstTy)->getBitWidth();
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
@@ -1037,7 +1040,7 @@ GenericValue Interpreter::executePtrToIntInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeIntToPtrInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeIntToPtrInst(Value *SrcVal, Type *DstTy,
                                               ExecutionContext &SF) {
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(DstTy->isPointerTy() && "Invalid PtrToInt instruction");
@@ -1050,10 +1053,10 @@ GenericValue Interpreter::executeIntToPtrInst(Value *SrcVal, const Type *DstTy,
   return Dest;
 }
 
-GenericValue Interpreter::executeBitCastInst(Value *SrcVal, const Type *DstTy,
+GenericValue Interpreter::executeBitCastInst(Value *SrcVal, Type *DstTy,
                                              ExecutionContext &SF) {
   
-  const Type *SrcTy = SrcVal->getType();
+  Type *SrcTy = SrcVal->getType();
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   if (DstTy->isPointerTy()) {
     assert(SrcTy->isPointerTy() && "Invalid BitCast");
@@ -1155,7 +1158,7 @@ void Interpreter::visitVAArgInst(VAArgInst &I) {
   GenericValue Dest;
   GenericValue Src = ECStack[VAList.UIntPairVal.first]
                       .VarArgs[VAList.UIntPairVal.second];
-  const Type *Ty = I.getType();
+  Type *Ty = I.getType();
   switch (Ty->getTypeID()) {
     case Type::IntegerTyID: Dest.IntVal = Src.IntVal;
     IMPLEMENT_VAARG(Pointer);
@@ -1222,7 +1225,7 @@ GenericValue Interpreter::getConstantExprValue (ConstantExpr *CE,
   GenericValue Op0 = getOperandValue(CE->getOperand(0), SF);
   GenericValue Op1 = getOperandValue(CE->getOperand(1), SF);
   GenericValue Dest;
-  const Type * Ty = CE->getOperand(0)->getType();
+  Type * Ty = CE->getOperand(0)->getType();
   switch (CE->getOpcode()) {
   case Instruction::Add:  Dest.IntVal = Op0.IntVal + Op1.IntVal; break;
   case Instruction::Sub:  Dest.IntVal = Op0.IntVal - Op1.IntVal; break;
diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
index f7e2a4df951e..055875c9456a 100644
--- a/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
@@ -48,7 +48,7 @@ using namespace llvm;
 
 static ManagedStatic<sys::Mutex> FunctionsLock;
 
-typedef GenericValue (*ExFunc)(const FunctionType *,
+typedef GenericValue (*ExFunc)(FunctionType *,
                                const std::vector<GenericValue> &);
 static ManagedStatic<std::map<const Function *, ExFunc> > ExportedFunctions;
 static std::map<std::string, ExFunc> FuncNames;
@@ -60,7 +60,7 @@ static ManagedStatic<std::map<const Function *, RawFunc> > RawFunctions;
 
 static Interpreter *TheInterpreter;
 
-static char getTypeID(const Type *Ty) {
+static char getTypeID(Type *Ty) {
   switch (Ty->getTypeID()) {
   case Type::VoidTyID:    return 'V';
   case Type::IntegerTyID:
@@ -91,7 +91,7 @@ static ExFunc lookupFunction(const Function *F) {
   // Function not found, look it up... start by figuring out what the
   // composite function name should be.
   std::string ExtName = "lle_";
-  const FunctionType *FT = F->getFunctionType();
+  FunctionType *FT = F->getFunctionType();
   for (unsigned i = 0, e = FT->getNumContainedTypes(); i != e; ++i)
     ExtName += getTypeID(FT->getContainedType(i));
   ExtName + "_" + F->getNameStr();
@@ -109,7 +109,7 @@ static ExFunc lookupFunction(const Function *F) {
 }
 
 #ifdef USE_LIBFFI
-static ffi_type *ffiTypeFor(const Type *Ty) {
+static ffi_type *ffiTypeFor(Type *Ty) {
   switch (Ty->getTypeID()) {
     case Type::VoidTyID: return &ffi_type_void;
     case Type::IntegerTyID:
@@ -129,7 +129,7 @@ static ffi_type *ffiTypeFor(const Type *Ty) {
   return NULL;
 }
 
-static void *ffiValueFor(const Type *Ty, const GenericValue &AV,
+static void *ffiValueFor(Type *Ty, const GenericValue &AV,
                          void *ArgDataPtr) {
   switch (Ty->getTypeID()) {
     case Type::IntegerTyID:
@@ -181,7 +181,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
                       const std::vector<GenericValue> &ArgVals,
                       const TargetData *TD, GenericValue &Result) {
   ffi_cif cif;
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
   const unsigned NumArgs = F->arg_size();
 
   // TODO: We don't have type information about the remaining arguments, because
@@ -197,7 +197,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
   for (Function::const_arg_iterator A = F->arg_begin(), E = F->arg_end();
        A != E; ++A) {
     const unsigned ArgNo = A->getArgNo();
-    const Type *ArgTy = FTy->getParamType(ArgNo);
+    Type *ArgTy = FTy->getParamType(ArgNo);
     args[ArgNo] = ffiTypeFor(ArgTy);
     ArgBytes += TD->getTypeStoreSize(ArgTy);
   }
@@ -209,12 +209,12 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
   for (Function::const_arg_iterator A = F->arg_begin(), E = F->arg_end();
        A != E; ++A) {
     const unsigned ArgNo = A->getArgNo();
-    const Type *ArgTy = FTy->getParamType(ArgNo);
+    Type *ArgTy = FTy->getParamType(ArgNo);
     values[ArgNo] = ffiValueFor(ArgTy, ArgVals[ArgNo], ArgDataPtr);
     ArgDataPtr += TD->getTypeStoreSize(ArgTy);
   }
 
-  const Type *RetTy = FTy->getReturnType();
+  Type *RetTy = FTy->getReturnType();
   ffi_type *rtype = ffiTypeFor(RetTy);
 
   if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, NumArgs, rtype, &args[0]) == FFI_OK) {
@@ -304,7 +304,7 @@ GenericValue Interpreter::callExternalFunction(Function *F,
 extern "C" {  // Don't add C++ manglings to llvm mangling :)
 
 // void atexit(Function*)
-GenericValue lle_X_atexit(const FunctionType *FT,
+GenericValue lle_X_atexit(FunctionType *FT,
                           const std::vector<GenericValue> &Args) {
   assert(Args.size() == 1);
   TheInterpreter->addAtExitHandler((Function*)GVTOP(Args[0]));
@@ -314,14 +314,14 @@ GenericValue lle_X_atexit(const FunctionType *FT,
 }
 
 // void exit(int)
-GenericValue lle_X_exit(const FunctionType *FT,
+GenericValue lle_X_exit(FunctionType *FT,
                         const std::vector<GenericValue> &Args) {
   TheInterpreter->exitCalled(Args[0]);
   return GenericValue();
 }
 
 // void abort(void)
-GenericValue lle_X_abort(const FunctionType *FT,
+GenericValue lle_X_abort(FunctionType *FT,
                          const std::vector<GenericValue> &Args) {
   //FIXME: should we report or raise here?
   //report_fatal_error("Interpreted program raised SIGABRT");
@@ -331,7 +331,7 @@ GenericValue lle_X_abort(const FunctionType *FT,
 
 // int sprintf(char *, const char *, ...) - a very rough implementation to make
 // output useful.
-GenericValue lle_X_sprintf(const FunctionType *FT,
+GenericValue lle_X_sprintf(FunctionType *FT,
                            const std::vector<GenericValue> &Args) {
   char *OutputBuffer = (char *)GVTOP(Args[0]);
   const char *FmtStr = (const char *)GVTOP(Args[1]);
@@ -413,7 +413,7 @@ GenericValue lle_X_sprintf(const FunctionType *FT,
 
 // int printf(const char *, ...) - a very rough implementation to make output
 // useful.
-GenericValue lle_X_printf(const FunctionType *FT,
+GenericValue lle_X_printf(FunctionType *FT,
                           const std::vector<GenericValue> &Args) {
   char Buffer[10000];
   std::vector<GenericValue> NewArgs;
@@ -425,7 +425,7 @@ GenericValue lle_X_printf(const FunctionType *FT,
 }
 
 // int sscanf(const char *format, ...);
-GenericValue lle_X_sscanf(const FunctionType *FT,
+GenericValue lle_X_sscanf(FunctionType *FT,
                           const std::vector<GenericValue> &args) {
   assert(args.size() < 10 && "Only handle up to 10 args to sscanf right now!");
 
@@ -440,7 +440,7 @@ GenericValue lle_X_sscanf(const FunctionType *FT,
 }
 
 // int scanf(const char *format, ...);
-GenericValue lle_X_scanf(const FunctionType *FT,
+GenericValue lle_X_scanf(FunctionType *FT,
                          const std::vector<GenericValue> &args) {
   assert(args.size() < 10 && "Only handle up to 10 args to scanf right now!");
 
@@ -456,7 +456,7 @@ GenericValue lle_X_scanf(const FunctionType *FT,
 
 // int fprintf(FILE *, const char *, ...) - a very rough implementation to make
 // output useful.
-GenericValue lle_X_fprintf(const FunctionType *FT,
+GenericValue lle_X_fprintf(FunctionType *FT,
                            const std::vector<GenericValue> &Args) {
   assert(Args.size() >= 2);
   char Buffer[10000];
diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h
index bfebe3debfcd..ee2b4596f38f 100644
--- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h
+++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h
@@ -174,7 +174,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
 
   void visitVAArgInst(VAArgInst &I);
   void visitInstruction(Instruction &I) {
-    errs() << I;
+    errs() << I << "\n";
     llvm_unreachable("Instruction not interpretable yet!");
   }
 
@@ -207,33 +207,33 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
   void initializeExternalFunctions();
   GenericValue getConstantExprValue(ConstantExpr *CE, ExecutionContext &SF);
   GenericValue getOperandValue(Value *V, ExecutionContext &SF);
-  GenericValue executeTruncInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeTruncInst(Value *SrcVal, Type *DstTy,
                                 ExecutionContext &SF);
-  GenericValue executeSExtInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeSExtInst(Value *SrcVal, Type *DstTy,
                                ExecutionContext &SF);
-  GenericValue executeZExtInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeZExtInst(Value *SrcVal, Type *DstTy,
                                ExecutionContext &SF);
-  GenericValue executeFPTruncInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeFPTruncInst(Value *SrcVal, Type *DstTy,
                                   ExecutionContext &SF);
-  GenericValue executeFPExtInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeFPExtInst(Value *SrcVal, Type *DstTy,
                                 ExecutionContext &SF);
-  GenericValue executeFPToUIInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeFPToUIInst(Value *SrcVal, Type *DstTy,
                                  ExecutionContext &SF);
-  GenericValue executeFPToSIInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeFPToSIInst(Value *SrcVal, Type *DstTy,
                                  ExecutionContext &SF);
-  GenericValue executeUIToFPInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeUIToFPInst(Value *SrcVal, Type *DstTy,
                                  ExecutionContext &SF);
-  GenericValue executeSIToFPInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeSIToFPInst(Value *SrcVal, Type *DstTy,
                                  ExecutionContext &SF);
-  GenericValue executePtrToIntInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executePtrToIntInst(Value *SrcVal, Type *DstTy,
                                    ExecutionContext &SF);
-  GenericValue executeIntToPtrInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeIntToPtrInst(Value *SrcVal, Type *DstTy,
                                    ExecutionContext &SF);
-  GenericValue executeBitCastInst(Value *SrcVal, const Type *DstTy,
+  GenericValue executeBitCastInst(Value *SrcVal, Type *DstTy,
                                   ExecutionContext &SF);
   GenericValue executeCastOperation(Instruction::CastOps opcode, Value *SrcVal, 
-                                    const Type *Ty, ExecutionContext &SF);
-  void popStackAndReturnValueToCaller(const Type *RetTy, GenericValue Result);
+                                    Type *Ty, ExecutionContext &SF);
+  void popStackAndReturnValueToCaller(Type *RetTy, GenericValue Result);
 
 };
 
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/Intercept.cpp b/contrib/llvm/lib/ExecutionEngine/JIT/Intercept.cpp
index fa8bee460427..2251a8e6b077 100644
--- a/contrib/llvm/lib/ExecutionEngine/JIT/Intercept.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/JIT/Intercept.cpp
@@ -52,6 +52,7 @@ static void runAtExitHandlers() {
 #include <sys/stat.h>
 #endif
 #include <fcntl.h>
+#include <unistd.h>
 /* stat functions are redirecting to __xstat with a version number.  On x86-64
  * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
  * available as an exported symbol, so we have to add it explicitly.
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/JIT.cpp b/contrib/llvm/lib/ExecutionEngine/JIT/JIT.cpp
index 445d2d0670c8..d773009065b5 100644
--- a/contrib/llvm/lib/ExecutionEngine/JIT/JIT.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/JIT/JIT.cpp
@@ -390,8 +390,8 @@ GenericValue JIT::runFunction(Function *F,
 
   void *FPtr = getPointerToFunction(F);
   assert(FPtr && "Pointer to fn's code was null after getPointerToFunction");
-  const FunctionType *FTy = F->getFunctionType();
-  const Type *RetTy = FTy->getReturnType();
+  FunctionType *FTy = F->getFunctionType();
+  Type *RetTy = FTy->getReturnType();
 
   assert((FTy->getNumParams() == ArgValues.size() ||
           (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) &&
@@ -500,7 +500,7 @@ GenericValue JIT::runFunction(Function *F,
   SmallVector<Value*, 8> Args;
   for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) {
     Constant *C = 0;
-    const Type *ArgTy = FTy->getParamType(i);
+    Type *ArgTy = FTy->getParamType(i);
     const GenericValue &AV = ArgValues[i];
     switch (ArgTy->getTypeID()) {
     default: llvm_unreachable("Unknown argument type for function call!");
@@ -788,7 +788,7 @@ char* JIT::getMemoryForGV(const GlobalVariable* GV) {
   // be allocated into the same buffer, but in general globals are allocated
   // through the memory manager which puts them near the code but not in the
   // same buffer.
-  const Type *GlobalType = GV->getType()->getElementType();
+  Type *GlobalType = GV->getType()->getElementType();
   size_t S = getTargetData()->getTypeAllocSize(GlobalType);
   size_t A = getTargetData()->getPreferredAlignment(GV);
   if (GV->isThreadLocal()) {
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/JIT.h b/contrib/llvm/lib/ExecutionEngine/JIT/JIT.h
index b879fc36e59b..92dcb0e99586 100644
--- a/contrib/llvm/lib/ExecutionEngine/JIT/JIT.h
+++ b/contrib/llvm/lib/ExecutionEngine/JIT/JIT.h
@@ -100,9 +100,10 @@ class JIT : public ExecutionEngine {
                                  CodeGenOpt::Level OptLevel =
                                    CodeGenOpt::Default,
                                  bool GVsWithCode = true,
-                                 CodeModel::Model CMM = CodeModel::Default) {
+                                 Reloc::Model RM = Reloc::Default,
+                                 CodeModel::Model CMM = CodeModel::JITDefault) {
     return ExecutionEngine::createJIT(M, Err, JMM, OptLevel, GVsWithCode,
-                                      CMM);
+                                      RM, CMM);
   }
 
   virtual void addModule(Module *M);
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp b/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp
index ddb0d5478596..8f84ac7b4126 100644
--- a/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp
@@ -18,12 +18,12 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/JITCodeEmitter.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/Support/ErrorHandling.h"
+#include "llvm/MC/MachineLocation.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetFrameLowering.h"
@@ -45,7 +45,7 @@ unsigned char* JITDwarfEmitter::EmitDwarfTable(MachineFunction& F,
   TD = TM.getTargetData();
   stackGrowthDirection = TM.getFrameLowering()->getStackGrowthDirection();
   RI = TM.getRegisterInfo();
-  TFI = TM.getFrameLowering();
+  MAI = TM.getMCAsmInfo();
   JCE = &jce;
 
   unsigned char* ExceptionTable = EmitExceptionTable(&F, StartFunction,
@@ -523,9 +523,7 @@ JITDwarfEmitter::EmitCommonEHFrame(const Function* Personality) const {
     JCE->emitULEB128Bytes(dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4);
   }
 
-  std::vector<MachineMove> Moves;
-  TFI->getInitialFrameState(Moves);
-  EmitFrameMoves(0, Moves);
+  EmitFrameMoves(0, MAI->getInitialFrameState());
 
   JCE->emitAlignmentWithFill(PointerSize, dwarf::DW_CFA_nop);
 
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.h b/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.h
index e1d00454d8d2..8dc99abc4224 100644
--- a/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.h
+++ b/contrib/llvm/lib/ExecutionEngine/JIT/JITDwarfEmitter.h
@@ -22,8 +22,8 @@ class JITCodeEmitter;
 class MachineFunction;
 class MachineModuleInfo;
 class MachineMove;
+class MCAsmInfo;
 class TargetData;
-class TargetFrameLowering;
 class TargetMachine;
 class TargetRegisterInfo;
 
@@ -31,7 +31,7 @@ class JITDwarfEmitter {
   const TargetData* TD;
   JITCodeEmitter* JCE;
   const TargetRegisterInfo* RI;
-  const TargetFrameLowering *TFI;
+  const MCAsmInfo *MAI;
   MachineModuleInfo* MMI;
   JIT& Jit;
   bool stackGrowthDirection;
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/JITEmitter.cpp b/contrib/llvm/lib/ExecutionEngine/JIT/JITEmitter.cpp
index d046b8aea641..24020ee6d689 100644
--- a/contrib/llvm/lib/ExecutionEngine/JIT/JITEmitter.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/JIT/JITEmitter.cpp
@@ -668,6 +668,7 @@ void *JITResolver::JITCompilerFn(void *Stub) {
     DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName()
           << "' In stub ptr = " << Stub << " actual ptr = "
           << ActualPtr << "\n");
+    (void)ActualPtr;
 
     Result = JR->TheJIT->getPointerToFunction(F);
   }
@@ -770,7 +771,7 @@ static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
     MachineConstantPoolEntry CPE = Constants[i];
     unsigned AlignMask = CPE.getAlignment() - 1;
     Size = (Size + AlignMask) & ~AlignMask;
-    const Type *Ty = CPE.getType();
+    Type *Ty = CPE.getType();
     Size += TD->getTypeAllocSize(Ty);
   }
   return Size;
@@ -1098,7 +1099,7 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
     DEBUG(dbgs() << "JIT:   CP" << i << " at [0x";
           dbgs().write_hex(CAddr) << "]\n");
 
-    const Type *Ty = CPE.Val.ConstVal->getType();
+    Type *Ty = CPE.Val.ConstVal->getType();
     Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty);
   }
 }
diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/CMakeLists.txt b/contrib/llvm/lib/ExecutionEngine/MCJIT/CMakeLists.txt
deleted file mode 100644
index f7ed176fef78..000000000000
--- a/contrib/llvm/lib/ExecutionEngine/MCJIT/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMMCJIT
-  MCJIT.cpp
-  TargetSelect.cpp
-  )
diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/Intercept.cpp b/contrib/llvm/lib/ExecutionEngine/MCJIT/Intercept.cpp
index e431c848d630..f83f4282e016 100644
--- a/contrib/llvm/lib/ExecutionEngine/MCJIT/Intercept.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/Intercept.cpp
@@ -52,6 +52,7 @@ static void runAtExitHandlers() {
 #include <sys/stat.h>
 #endif
 #include <fcntl.h>
+#include <unistd.h>
 /* stat functions are redirecting to __xstat with a version number.  On x86-64
  * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
  * available as an exported symbol, so we have to add it explicitly.
diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp
index 4475f4d5c0d8..7c8a740dc862 100644
--- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp
@@ -59,6 +59,7 @@ MCJIT::MCJIT(Module *m, TargetMachine *tm, TargetJITInfo &tji,
              bool AllocateGVsWithCode)
   : ExecutionEngine(m), TM(tm), MemMgr(MM), M(m), OS(Buffer), Dyld(MM) {
 
+  setTargetData(TM->getTargetData());
   PM.add(new TargetData(*TM->getTargetData()));
 
   // Turn the machine code intermediate representation into bytes in memory
@@ -124,8 +125,8 @@ GenericValue MCJIT::runFunction(Function *F,
 
   void *FPtr = getPointerToFunction(F);
   assert(FPtr && "Pointer to fn's code was null after getPointerToFunction");
-  const FunctionType *FTy = F->getFunctionType();
-  const Type *RetTy = FTy->getReturnType();
+  FunctionType *FTy = F->getFunctionType();
+  Type *RetTy = FTy->getReturnType();
 
   assert((FTy->getNumParams() == ArgValues.size() ||
           (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) &&
@@ -216,6 +217,6 @@ GenericValue MCJIT::runFunction(Function *F,
     }
   }
 
-  assert("Full-featured argument passing not supported yet!");
+  assert(0 && "Full-featured argument passing not supported yet!");
   return GenericValue();
 }
diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/Makefile b/contrib/llvm/lib/ExecutionEngine/MCJIT/Makefile
deleted file mode 100644
index 967efbc0efa4..000000000000
--- a/contrib/llvm/lib/ExecutionEngine/MCJIT/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-##===- lib/ExecutionEngine/MCJIT/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-LIBRARYNAME = LLVMMCJIT
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt
deleted file mode 100644
index 9e53f8757ec0..000000000000
--- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-add_llvm_library(LLVMRuntimeDyld
-  RuntimeDyld.cpp
-  )
diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Makefile b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Makefile
deleted file mode 100644
index 5d6f26d950fe..000000000000
--- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-##===- lib/ExecutionEngine/MCJIT/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-LIBRARYNAME = LLVMRuntimeDyld
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index bcdfb04801a5..7190a3c36fe9 100644
--- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -143,7 +143,7 @@ class RuntimeDyldMachO : public RuntimeDyldImpl {
 
   bool isCompatibleFormat(const MemoryBuffer *InputBuffer) const {
     return isKnownFormat(InputBuffer);
-  };
+  }
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp b/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp
index f51aff3603b8..004b8656bf22 100644
--- a/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp
+++ b/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp
@@ -17,11 +17,11 @@
 #include "llvm/Module.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/MC/SubtargetFeature.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Host.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
 /// selectTarget - Pick a target either via -march or by guessing the native
@@ -30,6 +30,8 @@ TargetMachine *EngineBuilder::selectTarget(Module *Mod,
                               StringRef MArch,
                               StringRef MCPU,
                               const SmallVectorImpl<std::string>& MAttrs,
+                              Reloc::Model RM,
+                              CodeModel::Model CM,
                               std::string *ErrorStr) {
   Triple TheTriple(Mod->getTargetTriple());
   if (TheTriple.getTriple().empty())
@@ -83,8 +85,9 @@ TargetMachine *EngineBuilder::selectTarget(Module *Mod,
   }
 
   // Allocate a target...
-  TargetMachine *Target =
-    TheTarget->createTargetMachine(TheTriple.getTriple(), MCPU, FeaturesStr);
+  TargetMachine *Target = TheTarget->createTargetMachine(TheTriple.getTriple(),
+                                                         MCPU, FeaturesStr,
+                                                         RM, CM);
   assert(Target && "Could not allocate target machine!");
   return Target;
 }
diff --git a/contrib/llvm/lib/Linker/LinkModules.cpp b/contrib/llvm/lib/Linker/LinkModules.cpp
index 55aa9bf18887..03a962e3be5d 100644
--- a/contrib/llvm/lib/Linker/LinkModules.cpp
+++ b/contrib/llvm/lib/Linker/LinkModules.cpp
@@ -14,9 +14,12 @@
 #include "llvm/Linker.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
+#include "llvm/Instructions.h"
 #include "llvm/Module.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 using namespace llvm;
 
@@ -139,7 +142,7 @@ bool TypeMapTy::areTypesIsomorphic(Type *DstTy, Type *SrcTy) {
       return false;
   } else if (StructType *DSTy = dyn_cast<StructType>(DstTy)) {
     StructType *SSTy = cast<StructType>(SrcTy);
-    if (DSTy->isAnonymous() != SSTy->isAnonymous() ||
+    if (DSTy->isLiteral() != SSTy->isLiteral() ||
         DSTy->isPacked() != SSTy->isPacked())
       return false;
   } else if (ArrayType *DATy = dyn_cast<ArrayType>(DstTy)) {
@@ -223,7 +226,7 @@ Type *TypeMapTy::getImpl(Type *Ty) {
   
   // If this is not a named struct type, then just map all of the elements and
   // then rebuild the type from inside out.
-  if (!isa<StructType>(Ty) || cast<StructType>(Ty)->isAnonymous()) {
+  if (!isa<StructType>(Ty) || cast<StructType>(Ty)->isLiteral()) {
     // If there are no element types to map, then the type is itself.  This is
     // true for the anonymous {} struct, things like 'float', integers, etc.
     if (Ty->getNumContainedTypes() == 0)
@@ -261,7 +264,7 @@ Type *TypeMapTy::getImpl(Type *Ty) {
                                       cast<PointerType>(Ty)->getAddressSpace());
     case Type::FunctionTyID:
       return *Entry = FunctionType::get(ElementTypes[0],
-                                        ArrayRef<Type*>(ElementTypes).slice(1),
+                                        makeArrayRef(ElementTypes).slice(1),
                                         cast<FunctionType>(Ty)->isVarArg());
     case Type::StructTyID:
       // Note that this is only reached for anonymous structs.
@@ -302,7 +305,7 @@ Type *TypeMapTy::getImpl(Type *Ty) {
   // Otherwise we create a new type and resolve its body later.  This will be
   // resolved by the top level of get().
   DefinitionsToResolve.push_back(STy);
-  return *Entry = StructType::createNamed(STy->getContext(), "");
+  return *Entry = StructType::create(STy->getContext());
 }
 
 
@@ -333,10 +336,16 @@ namespace {
     
     std::vector<AppendingVarInfo> AppendingVars;
     
+    unsigned Mode; // Mode to treat source module.
+    
+    // Set of items not to link in from source.
+    SmallPtrSet<const Value*, 16> DoNotLinkFromSource;
+    
   public:
     std::string ErrorMsg;
     
-    ModuleLinker(Module *dstM, Module *srcM) : DstM(dstM), SrcM(srcM) { }
+    ModuleLinker(Module *dstM, Module *srcM, unsigned mode)
+      : DstM(dstM), SrcM(srcM), Mode(mode) { }
     
     bool run();
     
@@ -596,9 +605,9 @@ bool ModuleLinker::linkAppendingVarProto(GlobalVariable *DstGV,
   DstGV->replaceAllUsesWith(ConstantExpr::getBitCast(NG, DstGV->getType()));
   DstGV->eraseFromParent();
   
-  // Zap the initializer in the source variable so we don't try to link it.
-  SrcGV->setInitializer(0);
-  SrcGV->setLinkage(GlobalValue::ExternalLinkage);
+  // Track the source variable so we don't try to link it.
+  DoNotLinkFromSource.insert(SrcGV);
+  
   return false;
 }
 
@@ -633,11 +642,10 @@ bool ModuleLinker::linkGlobalProto(GlobalVariable *SGV) {
       // Make sure to remember this mapping.
       ValueMap[SGV] = ConstantExpr::getBitCast(DGV,TypeMap.get(SGV->getType()));
       
-      // Destroy the source global's initializer (and convert it to a prototype)
-      // so that we don't attempt to copy it over when processing global
-      // initializers.
-      SGV->setInitializer(0);
-      SGV->setLinkage(GlobalValue::ExternalLinkage);
+      // Track the source global so that we don't attempt to copy it over when 
+      // processing global initializers.
+      DoNotLinkFromSource.insert(SGV);
+      
       return false;
     }
   }
@@ -682,8 +690,10 @@ bool ModuleLinker::linkFunctionProto(Function *SF) {
       // Make sure to remember this mapping.
       ValueMap[SF] = ConstantExpr::getBitCast(DGV, TypeMap.get(SF->getType()));
       
-      // Remove the body from the source module so we don't attempt to remap it.
-      SF->deleteBody();
+      // Track the function from the source module so we don't attempt to remap 
+      // it.
+      DoNotLinkFromSource.insert(SF);
+      
       return false;
     }
   }
@@ -722,8 +732,9 @@ bool ModuleLinker::linkAliasProto(GlobalAlias *SGA) {
       // Make sure to remember this mapping.
       ValueMap[SGA] = ConstantExpr::getBitCast(DGV,TypeMap.get(SGA->getType()));
       
-      // Remove the body from the source module so we don't attempt to remap it.
-      SGA->setAliasee(0);
+      // Track the alias from the source module so we don't attempt to remap it.
+      DoNotLinkFromSource.insert(SGA);
+      
       return false;
     }
   }
@@ -779,7 +790,9 @@ void ModuleLinker::linkGlobalInits() {
   // Loop over all of the globals in the src module, mapping them over as we go
   for (Module::const_global_iterator I = SrcM->global_begin(),
        E = SrcM->global_end(); I != E; ++I) {
-    if (!I->hasInitializer()) continue;      // Only process initialized GV's.
+    
+    // Only process initialized GV's or ones not already in dest.
+    if (!I->hasInitializer() || DoNotLinkFromSource.count(I)) continue;          
     
     // Grab destination global variable.
     GlobalVariable *DGV = cast<GlobalVariable>(ValueMap[I]);
@@ -805,31 +818,42 @@ void ModuleLinker::linkFunctionBody(Function *Dst, Function *Src) {
     ValueMap[I] = DI;
   }
 
-  // Splice the body of the source function into the dest function.
-  Dst->getBasicBlockList().splice(Dst->end(), Src->getBasicBlockList());
-
-  // At this point, all of the instructions and values of the function are now
-  // copied over.  The only problem is that they are still referencing values in
-  // the Source function as operands.  Loop through all of the operands of the
-  // functions and patch them up to point to the local versions.
-  for (Function::iterator BB = Dst->begin(), BE = Dst->end(); BB != BE; ++BB)
-    for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
-      RemapInstruction(I, ValueMap, RF_IgnoreMissingEntries, &TypeMap);
-
+  if (Mode == Linker::DestroySource) {
+    // Splice the body of the source function into the dest function.
+    Dst->getBasicBlockList().splice(Dst->end(), Src->getBasicBlockList());
+    
+    // At this point, all of the instructions and values of the function are now
+    // copied over.  The only problem is that they are still referencing values in
+    // the Source function as operands.  Loop through all of the operands of the
+    // functions and patch them up to point to the local versions.
+    for (Function::iterator BB = Dst->begin(), BE = Dst->end(); BB != BE; ++BB)
+      for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
+        RemapInstruction(I, ValueMap, RF_IgnoreMissingEntries, &TypeMap);
+    
+  } else {
+    // Clone the body of the function into the dest function.
+    SmallVector<ReturnInst*, 8> Returns; // Ignore returns.
+    CloneFunctionInto(Dst, Src, ValueMap, false, Returns);
+  }
+  
   // There is no need to map the arguments anymore.
   for (Function::arg_iterator I = Src->arg_begin(), E = Src->arg_end();
        I != E; ++I)
     ValueMap.erase(I);
+  
 }
 
 
 void ModuleLinker::linkAliasBodies() {
   for (Module::alias_iterator I = SrcM->alias_begin(), E = SrcM->alias_end();
-       I != E; ++I)
+       I != E; ++I) {
+    if (DoNotLinkFromSource.count(I))
+      continue;
     if (Constant *Aliasee = I->getAliasee()) {
       GlobalAlias *DA = cast<GlobalAlias>(ValueMap[I]);
       DA->setAliasee(MapValue(Aliasee, ValueMap, RF_None, &TypeMap));
     }
+  }
 }
 
 /// linkNamedMDNodes - Insert all of the named mdnodes in Src into the Dest
@@ -891,16 +915,10 @@ bool ModuleLinker::run() {
   StringRef ModuleId = SrcM->getModuleIdentifier();
   if (!ModuleId.empty())
     DstM->removeLibrary(sys::path::stem(ModuleId));
-
   
   // Loop over all of the linked values to compute type mappings.
   computeTypeMapping();
 
-  // Remap all of the named mdnoes in Src into the DstM module. We do this
-  // after linking GlobalValues so that MDNodes that reference GlobalValues
-  // are properly remapped.
-  linkNamedMDNodes();
-
   // Insert all of the globals in src into the DstM module... without linking
   // initializers (which could refer to functions not yet mapped over).
   for (Module::global_iterator I = SrcM->global_begin(),
@@ -933,7 +951,17 @@ bool ModuleLinker::run() {
   // Link in the function bodies that are defined in the source module into
   // DstM.
   for (Module::iterator SF = SrcM->begin(), E = SrcM->end(); SF != E; ++SF) {
-    if (SF->isDeclaration()) continue;      // No body if function is external.
+    
+    // Skip if not linking from source.
+    if (DoNotLinkFromSource.count(SF)) continue;
+    
+    // Skip if no body (function is external) or materialize.
+    if (SF->isDeclaration()) {
+      if (!SF->isMaterializable())
+        continue;
+      if (SF->Materialize(&ErrorMsg))
+        return true;
+    }
     
     linkFunctionBody(cast<Function>(ValueMap[SF]), SF);
   }
@@ -941,6 +969,11 @@ bool ModuleLinker::run() {
   // Resolve all uses of aliases with aliasees.
   linkAliasBodies();
 
+  // Remap all of the named mdnoes in Src into the DstM module. We do this
+  // after linking GlobalValues so that MDNodes that reference GlobalValues
+  // are properly remapped.
+  linkNamedMDNodes();
+
   // Now that all of the types from the source are used, resolve any structs
   // copied over to the dest that didn't exist there.
   TypeMap.linkDefinedTypeBodies();
@@ -957,8 +990,9 @@ bool ModuleLinker::run() {
 // error occurs, true is returned and ErrorMsg (if not null) is set to indicate
 // the problem.  Upon failure, the Dest module could be in a modified state, and
 // shouldn't be relied on to be consistent.
-bool Linker::LinkModules(Module *Dest, Module *Src, std::string *ErrorMsg) {
-  ModuleLinker TheLinker(Dest, Src);
+bool Linker::LinkModules(Module *Dest, Module *Src, unsigned Mode, 
+                         std::string *ErrorMsg) {
+  ModuleLinker TheLinker(Dest, Src, Mode);
   if (TheLinker.run()) {
     if (ErrorMsg) *ErrorMsg = TheLinker.ErrorMsg;
     return true;
diff --git a/contrib/llvm/lib/Linker/Linker.cpp b/contrib/llvm/lib/Linker/Linker.cpp
index fba91da5ddd1..59fbceb6a308 100644
--- a/contrib/llvm/lib/Linker/Linker.cpp
+++ b/contrib/llvm/lib/Linker/Linker.cpp
@@ -141,6 +141,14 @@ static inline sys::Path IsLibrary(StringRef Name,
   if (FullPath.isBitcodeFile())    // .so file containing bitcode?
     return FullPath;
 
+  // Try libX form, to make it possible to add dependency on the
+  // specific version of .so, like liblzma.so.1.0.0
+  FullPath.eraseSuffix();
+  if (FullPath.isDynamicLibrary())  // Native shared library?
+    return FullPath;
+  if (FullPath.isBitcodeFile())    // .so file containing bitcode?
+    return FullPath;
+
   // Not found .. fall through
 
   // Indicate that the library was not found in the directory.
diff --git a/contrib/llvm/lib/MC/ELFObjectWriter.cpp b/contrib/llvm/lib/MC/ELFObjectWriter.cpp
index 59e1b8eb8105..3d16de5604f6 100644
--- a/contrib/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/contrib/llvm/lib/MC/ELFObjectWriter.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -23,13 +24,13 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ELF.h"
-#include "llvm/Target/TargetAsmBackend.h"
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringSwitch.h"
 
-#include "../Target/X86/X86FixupKinds.h"
-#include "../Target/ARM/ARMFixupKinds.h"
+#include "../Target/X86/MCTargetDesc/X86FixupKinds.h"
+#include "../Target/ARM/MCTargetDesc/ARMFixupKinds.h"
+#include "../Target/PowerPC/MCTargetDesc/PPCFixupKinds.h"
 
 #include <vector>
 using namespace llvm;
@@ -124,12 +125,12 @@ void ELFObjectWriter::WriteHeader(uint64_t SectionDataSize,
 
   // e_shnum     = # of section header ents
   if (NumberOfSections >= ELF::SHN_LORESERVE)
-    Write16(0);
+    Write16(ELF::SHN_UNDEF);
   else
     Write16(NumberOfSections);
 
   // e_shstrndx  = Section # of '.shstrtab'
-  if (NumberOfSections >= ELF::SHN_LORESERVE)
+  if (ShstrtabIndex >= ELF::SHN_LORESERVE)
     Write16(ELF::SHN_XINDEX);
   else
     Write16(ShstrtabIndex);
@@ -301,7 +302,8 @@ void ELFObjectWriter::WriteSymbolTable(MCDataFragment *SymtabF,
     if (Section.getType() == ELF::SHT_RELA ||
         Section.getType() == ELF::SHT_REL ||
         Section.getType() == ELF::SHT_STRTAB ||
-        Section.getType() == ELF::SHT_SYMTAB)
+        Section.getType() == ELF::SHT_SYMTAB ||
+        Section.getType() == ELF::SHT_SYMTAB_SHNDX)
       continue;
     WriteSymbolEntry(SymtabF, ShndxF, 0, ELF::STT_SECTION, 0, 0,
                      ELF::STV_DEFAULT, SectionIndexMap.lookup(&Section), false);
@@ -447,8 +449,16 @@ void ELFObjectWriter::RecordRelocation(const MCAssembler &Asm,
   uint64_t RelocOffset = Layout.getFragmentOffset(Fragment) +
     Fixup.getOffset();
 
+  adjustFixupOffset(Fixup, RelocOffset);
+
   if (!hasRelocationAddend())
     Addend = 0;
+
+  if (is64Bit())
+    assert(isInt<64>(Addend));
+  else
+    assert(isInt<32>(Addend));
+
   ELFRelocationEntry ERE(RelocOffset, Index, Type, RelocSymbol, Addend);
   Relocations[Fragment->getParent()].push_back(ERE);
 }
@@ -656,6 +666,9 @@ void ELFObjectWriter::ComputeSymbolTable(MCAssembler &Asm,
     ExternalSymbolData[i].SymbolData->setIndex(Index++);
   for (unsigned i = 0, e = UndefinedSymbolData.size(); i != e; ++i)
     UndefinedSymbolData[i].SymbolData->setIndex(Index++);
+
+  if (NumRegularSections > ELF::SHN_LORESERVE)
+    NeedsSymtabShndx = true;
 }
 
 void ELFObjectWriter::CreateRelocationSections(MCAssembler &Asm,
@@ -992,11 +1005,10 @@ void ELFObjectWriter::WriteSection(MCAssembler &Asm,
     // Nothing to do.
     break;
 
-  case ELF::SHT_GROUP: {
+  case ELF::SHT_GROUP:
     sh_link = SymbolTableIndex;
     sh_info = GroupSymbolIndex;
     break;
-  }
 
   default:
     assert(0 && "FIXME: sh_type value not supported!");
@@ -1224,7 +1236,7 @@ void ELFObjectWriter::WriteObject(MCAssembler &Asm,
 
   FileOff = OS.tell();
 
-  // ... and then the remainting sections ...
+  // ... and then the remaining sections ...
   for (unsigned i = NumRegularSections + 1; i < NumSections; ++i)
     WriteDataSectionData(Asm, Layout, *Sections[i]);
 }
@@ -1252,6 +1264,11 @@ MCObjectWriter *llvm::createELFObjectWriter(MCELFObjectTargetWriter *MOTW,
       return new ARMELFObjectWriter(MOTW, OS, IsLittleEndian); break;
     case ELF::EM_MBLAZE:
       return new MBlazeELFObjectWriter(MOTW, OS, IsLittleEndian); break;
+    case ELF::EM_PPC:
+    case ELF::EM_PPC64:
+      return new PPCELFObjectWriter(MOTW, OS, IsLittleEndian); break;
+    case ELF::EM_MIPS:
+      return new MipsELFObjectWriter(MOTW, OS, IsLittleEndian); break;
     default: llvm_unreachable("Unsupported architecture"); break;
   }
 }
@@ -1503,6 +1520,76 @@ unsigned ARMELFObjectWriter::GetRelocTypeInner(const MCValue &Target,
   return Type;
 }
 
+//===- PPCELFObjectWriter -------------------------------------------===//
+
+PPCELFObjectWriter::PPCELFObjectWriter(MCELFObjectTargetWriter *MOTW,
+                                             raw_ostream &_OS,
+                                             bool IsLittleEndian)
+  : ELFObjectWriter(MOTW, _OS, IsLittleEndian) {
+}
+
+PPCELFObjectWriter::~PPCELFObjectWriter() {
+}
+
+unsigned PPCELFObjectWriter::GetRelocType(const MCValue &Target,
+                                             const MCFixup &Fixup,
+                                             bool IsPCRel,
+                                             bool IsRelocWithSymbol,
+                                             int64_t Addend) {
+  // determine the type of the relocation
+  unsigned Type;
+  if (IsPCRel) {
+    switch ((unsigned)Fixup.getKind()) {
+    default:
+      llvm_unreachable("Unimplemented");
+    case PPC::fixup_ppc_br24:
+      Type = ELF::R_PPC_REL24;
+      break;
+    case FK_PCRel_4:
+      Type = ELF::R_PPC_REL32;
+      break;
+    }
+  } else {
+    switch ((unsigned)Fixup.getKind()) {
+      default: llvm_unreachable("invalid fixup kind!");
+    case PPC::fixup_ppc_br24:
+      Type = ELF::R_PPC_ADDR24;
+      break;
+    case PPC::fixup_ppc_brcond14:
+      Type = ELF::R_PPC_ADDR14_BRTAKEN; // XXX: or BRNTAKEN?_
+      break;
+    case PPC::fixup_ppc_ha16:
+      Type = ELF::R_PPC_ADDR16_HA;
+      break;
+    case PPC::fixup_ppc_lo16:
+      Type = ELF::R_PPC_ADDR16_LO;
+      break;
+    case PPC::fixup_ppc_lo14:
+      Type = ELF::R_PPC_ADDR14;
+      break;
+    case FK_Data_4:
+      Type = ELF::R_PPC_ADDR32;
+      break;
+    case FK_Data_2:
+      Type = ELF::R_PPC_ADDR16;
+      break;
+    }
+  }
+  return Type;
+}
+
+void
+PPCELFObjectWriter::adjustFixupOffset(const MCFixup &Fixup, uint64_t &RelocOffset) {
+  switch ((unsigned)Fixup.getKind()) {
+    case PPC::fixup_ppc_ha16:
+    case PPC::fixup_ppc_lo16:
+      RelocOffset += 2;
+      break;
+    default:
+      break;
+  }
+}
+
 //===- MBlazeELFObjectWriter -------------------------------------------===//
 
 MBlazeELFObjectWriter::MBlazeELFObjectWriter(MCELFObjectTargetWriter *MOTW,
@@ -1624,7 +1711,6 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target,
       default: llvm_unreachable("invalid fixup kind!");
       case FK_Data_8: Type = ELF::R_X86_64_64; break;
       case X86::reloc_signed_4byte:
-        assert(isInt<32>(Target.getConstant()));
         switch (Modifier) {
         default:
           llvm_unreachable("Unimplemented");
@@ -1728,3 +1814,19 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target,
 
   return Type;
 }
+
+MipsELFObjectWriter::MipsELFObjectWriter(MCELFObjectTargetWriter *MOTW,
+                                         raw_ostream &_OS,
+                                         bool IsLittleEndian)
+  : ELFObjectWriter(MOTW, _OS, IsLittleEndian) {}
+
+MipsELFObjectWriter::~MipsELFObjectWriter() {}
+
+unsigned MipsELFObjectWriter::GetRelocType(const MCValue &Target,
+                                           const MCFixup &Fixup,
+                                           bool IsPCRel,
+                                           bool IsRelocWithSymbol,
+                                           int64_t Addend) {
+  // tbd
+  return 1;
+}
diff --git a/contrib/llvm/lib/MC/ELFObjectWriter.h b/contrib/llvm/lib/MC/ELFObjectWriter.h
index 7593099fb447..862b085c76bf 100644
--- a/contrib/llvm/lib/MC/ELFObjectWriter.h
+++ b/contrib/llvm/lib/MC/ELFObjectWriter.h
@@ -347,6 +347,7 @@ class ELFObjectWriter : public MCObjectWriter {
     virtual unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                                   bool IsPCRel, bool IsRelocWithSymbol,
                                   int64_t Addend) = 0;
+    virtual void adjustFixupOffset(const MCFixup &Fixup, uint64_t &RelocOffset) { }
   };
 
   //===- X86ELFObjectWriter -------------------------------------------===//
@@ -395,6 +396,22 @@ class ELFObjectWriter : public MCObjectWriter {
     
   };
 
+  //===- PPCELFObjectWriter -------------------------------------------===//
+
+  class PPCELFObjectWriter : public ELFObjectWriter {
+  public:
+    PPCELFObjectWriter(MCELFObjectTargetWriter *MOTW,
+                          raw_ostream &_OS,
+                          bool IsLittleEndian);
+
+    virtual ~PPCELFObjectWriter();
+  protected:
+    virtual unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
+                                  bool IsPCRel, bool IsRelocWithSymbol,
+                                  int64_t Addend);
+    virtual void adjustFixupOffset(const MCFixup &Fixup, uint64_t &RelocOffset);
+  };
+
   //===- MBlazeELFObjectWriter -------------------------------------------===//
 
   class MBlazeELFObjectWriter : public ELFObjectWriter {
@@ -409,6 +426,21 @@ class ELFObjectWriter : public MCObjectWriter {
                                   bool IsPCRel, bool IsRelocWithSymbol,
                                   int64_t Addend);
   };
+
+  //===- MipsELFObjectWriter -------------------------------------------===//
+
+  class MipsELFObjectWriter : public ELFObjectWriter {
+  public:
+    MipsELFObjectWriter(MCELFObjectTargetWriter *MOTW,
+                        raw_ostream &_OS,
+                        bool IsLittleEndian);
+
+    virtual ~MipsELFObjectWriter();
+  protected:
+    virtual unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
+                                  bool IsPCRel, bool IsRelocWithSymbol,
+                                  int64_t Addend);
+  };
 }
 
 #endif
diff --git a/contrib/llvm/lib/MC/TargetAsmBackend.cpp b/contrib/llvm/lib/MC/MCAsmBackend.cpp
similarity index 78%
rename from contrib/llvm/lib/MC/TargetAsmBackend.cpp
rename to contrib/llvm/lib/MC/MCAsmBackend.cpp
index 192755742535..2c150f456cf6 100644
--- a/contrib/llvm/lib/MC/TargetAsmBackend.cpp
+++ b/contrib/llvm/lib/MC/MCAsmBackend.cpp
@@ -1,4 +1,4 @@
-//===-- TargetAsmBackend.cpp - Target Assembly Backend ---------------------==//
+//===-- MCAsmBackend.cpp - Target MC Assembly Backend ----------------------==//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,19 +7,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmBackend.h"
+#include "llvm/MC/MCAsmBackend.h"
 using namespace llvm;
 
-TargetAsmBackend::TargetAsmBackend()
+MCAsmBackend::MCAsmBackend()
   : HasReliableSymbolDifference(false)
 {
 }
 
-TargetAsmBackend::~TargetAsmBackend() {
+MCAsmBackend::~MCAsmBackend() {
 }
 
 const MCFixupKindInfo &
-TargetAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   static const MCFixupKindInfo Builtins[] = {
     { "FK_Data_1", 0, 8, 0 },
     { "FK_Data_2", 0, 16, 0 },
diff --git a/contrib/llvm/lib/MC/MCAsmInfo.cpp b/contrib/llvm/lib/MC/MCAsmInfo.cpp
index 502b60b0edf4..95861bc61c27 100644
--- a/contrib/llvm/lib/MC/MCAsmInfo.cpp
+++ b/contrib/llvm/lib/MC/MCAsmInfo.cpp
@@ -29,6 +29,7 @@ MCAsmInfo::MCAsmInfo() {
   HasSubsectionsViaSymbols = false;
   HasMachoZeroFillDirective = false;
   HasMachoTBSSDirective = false;
+  StructorOutputOrder = Structors::ReversePriorityOrder;
   HasStaticCtorDtorReferenceInStaticMode = false;
   LinkerRequiresNonEmptyDwarfLines = false;
   MaxInstLength = 4;
@@ -42,6 +43,9 @@ MCAsmInfo::MCAsmInfo() {
   LinkerPrivateGlobalPrefix = "";
   InlineAsmStart = "APP";
   InlineAsmEnd = "NO_APP";
+  Code16Directive = ".code16";
+  Code32Directive = ".code32";
+  Code64Directive = ".code64";
   AssemblerDialect = 0;
   AllowQuotesInName = false;
   AllowNameToStartWithDigit = false;
@@ -53,6 +57,12 @@ MCAsmInfo::MCAsmInfo() {
   Data16bitsDirective = "\t.short\t";
   Data32bitsDirective = "\t.long\t";
   Data64bitsDirective = "\t.quad\t";
+  DataBegin = "$d.";
+  CodeBegin = "$a.";
+  JT8Begin = "$d.";
+  JT16Begin = "$d.";
+  JT32Begin = "$d.";
+  SupportsDataRegions = false;
   SunStyleELFSectionSwitchSyntax = false;
   UsesELFSectionDirectiveForBSS = false;
   AlignDirective = "\t.align\t";
@@ -62,7 +72,7 @@ MCAsmInfo::MCAsmInfo() {
   GlobalDirective = "\t.globl\t";
   HasSetDirective = true;
   HasAggressiveSymbolFolding = true;
-  HasLCOMMDirective = false;
+  LCOMMDirectiveType = LCOMM::None;
   COMMDirectiveAlignmentIsInBytes = true;
   HasDotTypeDotSizeDirective = true;
   HasSingleParameterDotFile = true;
diff --git a/contrib/llvm/lib/MC/MCAsmInfoCOFF.cpp b/contrib/llvm/lib/MC/MCAsmInfoCOFF.cpp
index 7fc7d7abb232..434d9103a71a 100644
--- a/contrib/llvm/lib/MC/MCAsmInfoCOFF.cpp
+++ b/contrib/llvm/lib/MC/MCAsmInfoCOFF.cpp
@@ -19,7 +19,7 @@ using namespace llvm;
 MCAsmInfoCOFF::MCAsmInfoCOFF() {
   GlobalPrefix = "_";
   COMMDirectiveAlignmentIsInBytes = false;
-  HasLCOMMDirective = true;
+  LCOMMDirectiveType = LCOMM::ByteAlignment;
   HasDotTypeDotSizeDirective = false;
   HasSingleParameterDotFile = false;
   PrivateGlobalPrefix = "L";  // Prefix for private global symbols
@@ -27,11 +27,14 @@ MCAsmInfoCOFF::MCAsmInfoCOFF() {
   LinkOnceDirective = "\t.linkonce discard\n";
   
   // Doesn't support visibility:
-  HiddenVisibilityAttr = ProtectedVisibilityAttr = MCSA_Invalid;
+  HiddenVisibilityAttr = HiddenDeclarationVisibilityAttr = MCSA_Invalid;
+  ProtectedVisibilityAttr = MCSA_Invalid;
 
   // Set up DWARF directives
   HasLEB128 = true;  // Target asm supports leb128 directives (little-endian)
   SupportsDebugInformation = true;
   DwarfSectionOffsetDirective = "\t.secrel32\t";
   HasMicrosoftFastStdCallMangling = true;
+
+  SupportsDataRegions = false;
 }
diff --git a/contrib/llvm/lib/MC/MCAsmInfoDarwin.cpp b/contrib/llvm/lib/MC/MCAsmInfoDarwin.cpp
index 5851cb0391d8..b20e338f7904 100644
--- a/contrib/llvm/lib/MC/MCAsmInfoDarwin.cpp
+++ b/contrib/llvm/lib/MC/MCAsmInfoDarwin.cpp
@@ -39,8 +39,16 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
   ZeroDirective = "\t.space\t";  // ".space N" emits N zeros.
   HasMachoZeroFillDirective = true;  // Uses .zerofill
   HasMachoTBSSDirective = true; // Uses .tbss
+  StructorOutputOrder = Structors::PriorityOrder;
   HasStaticCtorDtorReferenceInStaticMode = true;
 
+  CodeBegin = "L$start$code$";
+  DataBegin = "L$start$data$";
+  JT8Begin  = "L$start$jt8$";
+  JT16Begin = "L$start$jt16$";
+  JT32Begin = "L$start$jt32$";
+  SupportsDataRegions = true;
+
   // FIXME: Darwin 10 and newer don't need this.
   LinkerRequiresNonEmptyDwarfLines = true;
 
diff --git a/contrib/llvm/lib/MC/MCAsmStreamer.cpp b/contrib/llvm/lib/MC/MCAsmStreamer.cpp
index d5d08e8f69fb..3fcbb05907bc 100644
--- a/contrib/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCAsmStreamer.cpp
@@ -15,8 +15,12 @@
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -25,9 +29,6 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormattedStream.h"
-#include "llvm/Target/TargetAsmBackend.h"
-#include "llvm/Target/TargetAsmInfo.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
 #include <cctype>
 using namespace llvm;
 
@@ -40,7 +41,7 @@ class MCAsmStreamer : public MCStreamer {
 private:
   OwningPtr<MCInstPrinter> InstPrinter;
   OwningPtr<MCCodeEmitter> Emitter;
-  OwningPtr<TargetAsmBackend> AsmBackend;
+  OwningPtr<MCAsmBackend> AsmBackend;
 
   SmallString<128> CommentToEmit;
   raw_svector_ostream CommentStream;
@@ -63,7 +64,7 @@ class MCAsmStreamer : public MCStreamer {
   MCAsmStreamer(MCContext &Context, formatted_raw_ostream &os,
                 bool isVerboseAsm, bool useLoc, bool useCFI,
                 MCInstPrinter *printer, MCCodeEmitter *emitter,
-                TargetAsmBackend *asmbackend,
+                MCAsmBackend *asmbackend,
                 bool showInst)
     : MCStreamer(Context), OS(os), MAI(Context.getAsmInfo()),
       InstPrinter(printer), Emitter(emitter), AsmBackend(asmbackend),
@@ -157,7 +158,9 @@ class MCAsmStreamer : public MCStreamer {
   ///
   /// @param Symbol - The common symbol to emit.
   /// @param Size - The size of the common symbol.
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size);
+  /// @param Size - The alignment of the common symbol in bytes.
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment);
 
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
                             unsigned Size = 0, unsigned ByteAlignment = 0);
@@ -334,8 +337,9 @@ void MCAsmStreamer::EmitAssemblerFlag(MCAssemblerFlag Flag) {
   default: assert(0 && "Invalid flag!");
   case MCAF_SyntaxUnified:         OS << "\t.syntax unified"; break;
   case MCAF_SubsectionsViaSymbols: OS << ".subsections_via_symbols"; break;
-  case MCAF_Code16:                OS << "\t.code\t16"; break;
-  case MCAF_Code32:                OS << "\t.code\t32"; break;
+  case MCAF_Code16:                OS << '\t'<< MAI.getCode16Directive(); break;
+  case MCAF_Code32:                OS << '\t'<< MAI.getCode32Directive(); break;
+  case MCAF_Code64:                OS << '\t'<< MAI.getCode64Directive(); break;
   }
   EmitEOL();
 }
@@ -482,9 +486,16 @@ void MCAsmStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
 ///
 /// @param Symbol - The common symbol to emit.
 /// @param Size - The size of the common symbol.
-void MCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {
-  assert(MAI.hasLCOMMDirective() && "Doesn't have .lcomm, can't emit it!");
+void MCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                          unsigned ByteAlign) {
+  assert(MAI.getLCOMMDirectiveType() != LCOMM::None &&
+         "Doesn't have .lcomm, can't emit it!");
   OS << "\t.lcomm\t" << *Symbol << ',' << Size;
+  if (ByteAlign > 1) {
+    assert(MAI.getLCOMMDirectiveType() == LCOMM::ByteAlignment &&
+           "Alignment not supported on .lcomm!");
+    OS << ',' << ByteAlign;
+  }
   EmitEOL();
 }
 
@@ -827,8 +838,8 @@ void MCAsmStreamer::EmitCFIEndProc() {
 
 void MCAsmStreamer::EmitRegisterName(int64_t Register) {
   if (InstPrinter && !MAI.useDwarfRegNumForCFI()) {
-    const TargetAsmInfo &TAI = getContext().getTargetAsmInfo();
-    unsigned LLVMRegister = TAI.getLLVMRegNum(Register, true);
+    const MCRegisterInfo &MRI = getContext().getRegisterInfo();
+    unsigned LLVMRegister = MRI.getLLVMRegNum(Register, true);
     InstPrinter->printRegName(OS, LLVMRegister);
   } else {
     OS << Register;
@@ -994,6 +1005,19 @@ void MCAsmStreamer::EmitWin64EHHandler(const MCSymbol *Sym, bool Unwind,
   EmitEOL();
 }
 
+static const MCSection *getWin64EHTableSection(StringRef suffix,
+                                               MCContext &context) {
+  // FIXME: This doesn't belong in MCObjectFileInfo. However,
+  /// this duplicate code in MCWin64EH.cpp.
+  if (suffix == "")
+    return context.getObjectFileInfo()->getXDataSection();
+  return context.getCOFFSection((".xdata"+suffix).str(),
+                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                                COFF::IMAGE_SCN_MEM_READ |
+                                COFF::IMAGE_SCN_MEM_WRITE,
+                                SectionKind::getDataRel());
+}
+
 void MCAsmStreamer::EmitWin64EHHandlerData() {
   MCStreamer::EmitWin64EHHandlerData();
 
@@ -1003,8 +1027,7 @@ void MCAsmStreamer::EmitWin64EHHandlerData() {
   // data block is visible.
   MCWin64EHUnwindInfo *CurFrame = getCurrentW64UnwindInfo();
   StringRef suffix=MCWin64EHUnwindEmitter::GetSectionSuffix(CurFrame->Function);
-  const MCSection *xdataSect =
-    getContext().getTargetAsmInfo().getWin64EHTableSection(suffix);
+  const MCSection *xdataSect = getWin64EHTableSection(suffix, getContext());
   if (xdataSect)
     SwitchSectionNoChange(xdataSect);
 
@@ -1221,7 +1244,7 @@ void MCAsmStreamer::EmitInstruction(const MCInst &Inst) {
 
   // If we have an AsmPrinter, use that to print, otherwise print the MCInst.
   if (InstPrinter)
-    InstPrinter->printInst(&Inst, OS);
+    InstPrinter->printInst(&Inst, OS, "");
   else
     Inst.print(OS, &MAI);
   EmitEOL();
@@ -1249,8 +1272,8 @@ MCStreamer *llvm::createAsmStreamer(MCContext &Context,
                                     formatted_raw_ostream &OS,
                                     bool isVerboseAsm, bool useLoc,
                                     bool useCFI, MCInstPrinter *IP,
-                                    MCCodeEmitter *CE, TargetAsmBackend *TAB,
+                                    MCCodeEmitter *CE, MCAsmBackend *MAB,
                                     bool ShowInst) {
   return new MCAsmStreamer(Context, OS, isVerboseAsm, useLoc, useCFI,
-                           IP, CE, TAB, ShowInst);
+                           IP, CE, MAB, ShowInst);
 }
diff --git a/contrib/llvm/lib/MC/MCAssembler.cpp b/contrib/llvm/lib/MC/MCAssembler.cpp
index 527a63caaee5..06c8aec91917 100644
--- a/contrib/llvm/lib/MC/MCAssembler.cpp
+++ b/contrib/llvm/lib/MC/MCAssembler.cpp
@@ -18,6 +18,7 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCDwarf.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
@@ -25,8 +26,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmBackend.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
 
@@ -194,7 +194,7 @@ MCSymbolData::MCSymbolData(const MCSymbol &_Symbol, MCFragment *_Fragment,
 
 /* *** */
 
-MCAssembler::MCAssembler(MCContext &Context_, TargetAsmBackend &Backend_,
+MCAssembler::MCAssembler(MCContext &Context_, MCAsmBackend &Backend_,
                          MCCodeEmitter &Emitter_, MCObjectWriter &Writer_,
                          raw_ostream &OS_)
   : Context(Context_), Backend(Backend_), Emitter(Emitter_), Writer(Writer_),
diff --git a/contrib/llvm/lib/MC/MCAtom.cpp b/contrib/llvm/lib/MC/MCAtom.cpp
new file mode 100644
index 000000000000..d71444324f3d
--- /dev/null
+++ b/contrib/llvm/lib/MC/MCAtom.cpp
@@ -0,0 +1,97 @@
+//===- lib/MC/MCAtom.cpp - MCAtom implementation --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCAtom.h"
+#include "llvm/MC/MCModule.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+void MCAtom::addInst(const MCInst &I, uint64_t Address, unsigned Size) {
+  assert(Type == TextAtom && "Trying to add MCInst to a non-text atom!");
+
+  assert(Address < End+Size &&
+         "Instruction not contiguous with end of atom!");
+  if (Address > End)
+    Parent->remap(this, Begin, End+Size);
+
+  Text.push_back(std::make_pair(Address, I));
+}
+
+void MCAtom::addData(const MCData &D) {
+  assert(Type == DataAtom && "Trying to add MCData to a non-data atom!");
+  Parent->remap(this, Begin, End+1);
+
+  Data.push_back(D);
+}
+
+MCAtom *MCAtom::split(uint64_t SplitPt) {
+  assert((SplitPt > Begin && SplitPt <= End) &&
+         "Splitting at point not contained in atom!");
+
+  // Compute the new begin/end points.
+  uint64_t LeftBegin = Begin;
+  uint64_t LeftEnd = SplitPt - 1;
+  uint64_t RightBegin = SplitPt;
+  uint64_t RightEnd = End;
+
+  // Remap this atom to become the lower of the two new ones.
+  Parent->remap(this, LeftBegin, LeftEnd);
+
+  // Create a new atom for the higher atom.
+  MCAtom *RightAtom = Parent->createAtom(Type, RightBegin, RightEnd);
+
+  // Split the contents of the original atom between it and the new one.  The
+  // precise method depends on whether this is a data or a text atom.
+  if (isDataAtom()) {
+    std::vector<MCData>::iterator I = Data.begin() + (RightBegin - LeftBegin);
+
+    assert(I != Data.end() && "Split point not found in range!");
+
+    std::copy(I, Data.end(), RightAtom->Data.end());
+    Data.erase(I, Data.end());
+  } else if (isTextAtom()) {
+    std::vector<std::pair<uint64_t, MCInst> >::iterator I = Text.begin();
+
+    while (I != Text.end() && I->first < SplitPt) ++I;
+
+    assert(I != Text.end() && "Split point not found in disassembly!");
+    assert(I->first == SplitPt &&
+           "Split point does not fall on instruction boundary!");
+
+    std::copy(I, Text.end(), RightAtom->Text.end());
+    Text.erase(I, Text.end());
+  } else
+    llvm_unreachable("Unknown atom type!");
+
+  return RightAtom;
+}
+
+void MCAtom::truncate(uint64_t TruncPt) {
+  assert((TruncPt >= Begin && TruncPt < End) &&
+         "Truncation point not contained in atom!");
+
+  Parent->remap(this, Begin, TruncPt);
+
+  if (isDataAtom()) {
+    Data.resize(TruncPt - Begin + 1);
+  } else if (isTextAtom()) {
+    std::vector<std::pair<uint64_t, MCInst> >::iterator I = Text.begin();
+
+    while (I != Text.end() && I->first <= TruncPt) ++I;
+
+    assert(I != Text.end() && "Truncation point not found in disassembly!");
+    assert(I->first == TruncPt+1 &&
+           "Truncation point does not fall on instruction boundary");
+
+    Text.erase(I, Text.end());
+  } else
+    llvm_unreachable("Unknown atom type!");
+}
+
diff --git a/contrib/llvm/lib/MC/MCCodeGenInfo.cpp b/contrib/llvm/lib/MC/MCCodeGenInfo.cpp
new file mode 100644
index 000000000000..236e7de68a8a
--- /dev/null
+++ b/contrib/llvm/lib/MC/MCCodeGenInfo.cpp
@@ -0,0 +1,21 @@
+//===-- MCCodeGenInfo.cpp - Target CodeGen Info -----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file tracks information about the target which can affect codegen,
+// asm parsing, and asm printing. For example, relocation model.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCCodeGenInfo.h"
+using namespace llvm;
+
+void MCCodeGenInfo::InitMCCodeGenInfo(Reloc::Model RM, CodeModel::Model CM) {
+  RelocationModel = RM;
+  CMModel = CM;
+}
diff --git a/contrib/llvm/lib/MC/MCContext.cpp b/contrib/llvm/lib/MC/MCContext.cpp
index 8faa72ecb4e8..82690ee3b3e9 100644
--- a/contrib/llvm/lib/MC/MCContext.cpp
+++ b/contrib/llvm/lib/MC/MCContext.cpp
@@ -9,13 +9,14 @@
 
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCLabel.h"
 #include "llvm/MC/MCDwarf.h"
-#include "llvm/Target/TargetAsmInfo.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/ELF.h"
@@ -26,8 +27,9 @@ typedef StringMap<const MCSectionELF*> ELFUniqueMapTy;
 typedef StringMap<const MCSectionCOFF*> COFFUniqueMapTy;
 
 
-MCContext::MCContext(const MCAsmInfo &mai, const TargetAsmInfo *tai) :
-  MAI(mai), TAI(tai),
+MCContext::MCContext(const MCAsmInfo &mai, const MCRegisterInfo &mri,
+                     const MCObjectFileInfo *mofi) :
+  MAI(mai), MRI(mri), MOFI(mofi),
   Allocator(), Symbols(Allocator), UsedNames(Allocator),
   NextUniqueID(0),
   CurrentDwarfLoc(0,0,0,DWARF2_FLAG_IS_STMT,0,0),
@@ -54,8 +56,6 @@ MCContext::~MCContext() {
 
   // If the stream for the .secure_log_unique directive was created free it.
   delete (raw_ostream*)SecureLog;
-
-  delete TAI;
 }
 
 //===----------------------------------------------------------------------===//
@@ -279,7 +279,8 @@ unsigned MCContext::GetDwarfFile(StringRef FileName, unsigned FileNumber) {
   } else {
     StringRef Directory = Slash.first;
     Name = Slash.second;
-    for (DirIndex = 0; DirIndex < MCDwarfDirs.size(); DirIndex++) {
+    DirIndex = 0;
+    for (unsigned End = MCDwarfDirs.size(); DirIndex < End; DirIndex++) {
       if (Directory == MCDwarfDirs[DirIndex])
         break;
     }
diff --git a/contrib/llvm/lib/MC/MCDisassembler/Disassembler.cpp b/contrib/llvm/lib/MC/MCDisassembler/Disassembler.cpp
index 5480b4b12b2c..16e66dc98e74 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/Disassembler.cpp
+++ b/contrib/llvm/lib/MC/MCDisassembler/Disassembler.cpp
@@ -1,4 +1,4 @@
-//===-- lib/MC/Disassembler.cpp - Disassembler Public C Interface -*- C -*-===//
+//===-- lib/MC/Disassembler.cpp - Disassembler Public C Interface ---------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -11,15 +11,14 @@
 #include "llvm-c/Disassembler.h"
 
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmInfo.h"  // FIXME.
-#include "llvm/Target/TargetMachine.h"  // FIXME.
-#include "llvm/Target/TargetSelect.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/MemoryObject.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
 
 namespace llvm {
 class Target;
@@ -38,10 +37,7 @@ LLVMDisasmContextRef LLVMCreateDisasm(const char *TripleName, void *DisInfo,
                                       LLVMSymbolLookupCallback SymbolLookUp) {
   // Initialize targets and assembly printers/parsers.
   llvm::InitializeAllTargetInfos();
-  // FIXME: We shouldn't need to initialize the Target(Machine)s.
-  llvm::InitializeAllTargets();
-  llvm::InitializeAllMCAsmInfos();
-  llvm::InitializeAllAsmPrinters();
+  llvm::InitializeAllTargetMCs();
   llvm::InitializeAllAsmParsers();
   llvm::InitializeAllDisassemblers();
 
@@ -54,41 +50,38 @@ LLVMDisasmContextRef LLVMCreateDisasm(const char *TripleName, void *DisInfo,
   const MCAsmInfo *MAI = TheTarget->createMCAsmInfo(TripleName);
   assert(MAI && "Unable to create target asm info!");
 
+  const MCRegisterInfo *MRI = TheTarget->createMCRegInfo(TripleName);
+  assert(MRI && "Unable to create target register info!");
+
   // Package up features to be passed to target/subtarget
   std::string FeaturesStr;
   std::string CPU;
 
-  // FIXME: We shouldn't need to do this (and link in codegen).
-  //        When we split this out, we should do it in a way that makes
-  //        it straightforward to switch subtargets on the fly.
-  TargetMachine *TM = TheTarget->createTargetMachine(TripleName, CPU,
-                                                     FeaturesStr);
-  assert(TM && "Unable to create target machine!");
-
-  // Get the target assembler info needed to setup the context.
-  const TargetAsmInfo *tai = new TargetAsmInfo(*TM);
-  assert(tai && "Unable to create target assembler!");
+  const MCSubtargetInfo *STI = TheTarget->createMCSubtargetInfo(TripleName, CPU,
+                                                                FeaturesStr);
+  assert(STI && "Unable to create subtarget info!");
 
   // Set up the MCContext for creating symbols and MCExpr's.
-  MCContext *Ctx = new MCContext(*MAI, tai);
+  MCContext *Ctx = new MCContext(*MAI, *MRI, 0);
   assert(Ctx && "Unable to create MCContext!");
 
   // Set up disassembler.
-  MCDisassembler *DisAsm = TheTarget->createMCDisassembler();
+  MCDisassembler *DisAsm = TheTarget->createMCDisassembler(*STI);
   assert(DisAsm && "Unable to create disassembler!");
-  DisAsm->setupForSymbolicDisassembly(GetOpInfo, DisInfo, Ctx);
+  DisAsm->setupForSymbolicDisassembly(GetOpInfo, SymbolLookUp, DisInfo, Ctx);
 
   // Set up the instruction printer.
   int AsmPrinterVariant = MAI->getAssemblerDialect();
   MCInstPrinter *IP = TheTarget->createMCInstPrinter(AsmPrinterVariant,
-                                                     *MAI);
+                                                     *MAI, *STI);
   assert(IP && "Unable to create instruction printer!");
 
   LLVMDisasmContext *DC = new LLVMDisasmContext(TripleName, DisInfo, TagType,
                                                 GetOpInfo, SymbolLookUp,
-                                                TheTarget, MAI, TM, tai, Ctx,
-                                                DisAsm, IP);
+                                                TheTarget, MAI, MRI,
+                                                Ctx, DisAsm, IP);
   assert(DC && "Allocation failure!");
+
   return DC;
 }
 
@@ -147,18 +140,35 @@ size_t LLVMDisasmInstruction(LLVMDisasmContextRef DCR, uint8_t *Bytes,
   MCInst Inst;
   const MCDisassembler *DisAsm = DC->getDisAsm();
   MCInstPrinter *IP = DC->getIP();
-  if (!DisAsm->getInstruction(Inst, Size, MemoryObject, PC, /*REMOVE*/ nulls()))
+  MCDisassembler::DecodeStatus S;
+  S = DisAsm->getInstruction(Inst, Size, MemoryObject, PC,
+                             /*REMOVE*/ nulls(), DC->CommentStream);
+  switch (S) {
+  case MCDisassembler::Fail:
+  case MCDisassembler::SoftFail:
+    // FIXME: Do something different for soft failure modes?
     return 0;
 
-  SmallVector<char, 64> InsnStr;
-  raw_svector_ostream OS(InsnStr);
-  IP->printInst(&Inst, OS);
-  OS.flush();
+  case MCDisassembler::Success: {
+    DC->CommentStream.flush();
+    StringRef Comments = DC->CommentsToEmit.str();
 
-  assert(OutStringSize != 0 && "Output buffer cannot be zero size");
-  size_t OutputSize = std::min(OutStringSize-1, InsnStr.size());
-  std::memcpy(OutString, InsnStr.data(), OutputSize);
-  OutString[OutputSize] = '\0'; // Terminate string.
+    SmallVector<char, 64> InsnStr;
+    raw_svector_ostream OS(InsnStr);
+    IP->printInst(&Inst, OS, Comments);
+    OS.flush();
 
-  return Size;
+    // Tell the comment stream that the vector changed underneath it.
+    DC->CommentsToEmit.clear();
+    DC->CommentStream.resync();
+
+    assert(OutStringSize != 0 && "Output buffer cannot be zero size");
+    size_t OutputSize = std::min(OutStringSize-1, InsnStr.size());
+    std::memcpy(OutString, InsnStr.data(), OutputSize);
+    OutString[OutputSize] = '\0'; // Terminate string.
+
+    return Size;
+  }
+  }
+  return 0;
 }
diff --git a/contrib/llvm/lib/MC/MCDisassembler/Disassembler.h b/contrib/llvm/lib/MC/MCDisassembler/Disassembler.h
index f0ec42a017a4..238ff7d50025 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/Disassembler.h
+++ b/contrib/llvm/lib/MC/MCDisassembler/Disassembler.h
@@ -20,15 +20,16 @@
 #include "llvm-c/Disassembler.h"
 #include <string>
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
-class TargetAsmInfo;
 class MCContext;
 class MCAsmInfo;
 class MCDisassembler;
 class MCInstPrinter; 
+class MCRegisterInfo;
 class Target;
-class TargetMachine;
 
 //
 // This is the disassembler context returned by LLVMCreateDisasm().
@@ -58,12 +59,8 @@ class LLVMDisasmContext {
   const Target *TheTarget;
   // The assembly information for the target architecture.
   llvm::OwningPtr<const llvm::MCAsmInfo> MAI;
-  // The target machine instance.
-  llvm::OwningPtr<llvm::TargetMachine> TM;
-  // The disassembler for the target architecture.
-  // FIXME: using llvm::OwningPtr<const llvm::TargetAsmInfo> causes a malloc
-  //        error when this LLVMDisasmContext is deleted.
-  const TargetAsmInfo *Tai;
+  // The register information for the target architecture.
+  llvm::OwningPtr<const llvm::MCRegisterInfo> MRI;
   // The assembly context for creating symbols and MCExprs.
   llvm::OwningPtr<const llvm::MCContext> Ctx;
   // The disassembler for the target architecture.
@@ -72,22 +69,28 @@ class LLVMDisasmContext {
   llvm::OwningPtr<llvm::MCInstPrinter> IP;
 
 public:
+  // Comment stream and backing vector.
+  SmallString<128> CommentsToEmit;
+  raw_svector_ostream CommentStream;
+
   LLVMDisasmContext(std::string tripleName, void *disInfo, int tagType,
                     LLVMOpInfoCallback getOpInfo,
                     LLVMSymbolLookupCallback symbolLookUp,
                     const Target *theTarget, const MCAsmInfo *mAI,
-                    llvm::TargetMachine *tM, const TargetAsmInfo *tai,
+                    const MCRegisterInfo *mRI,
                     llvm::MCContext *ctx, const MCDisassembler *disAsm,
                     MCInstPrinter *iP) : TripleName(tripleName),
                     DisInfo(disInfo), TagType(tagType), GetOpInfo(getOpInfo),
-                    SymbolLookUp(symbolLookUp), TheTarget(theTarget), Tai(tai) {
-    TM.reset(tM);
+                    SymbolLookUp(symbolLookUp), TheTarget(theTarget),
+                    CommentStream(CommentsToEmit) {
     MAI.reset(mAI);
+    MRI.reset(mRI);
     Ctx.reset(ctx);
     DisAsm.reset(disAsm);
     IP.reset(iP);
   }
   const MCDisassembler *getDisAsm() const { return DisAsm.get(); }
+  const MCAsmInfo *getAsmInfo() const { return MAI.get(); }
   MCInstPrinter *getIP() { return IP.get(); }
 };
 
diff --git a/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.cpp b/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.cpp
index bdd99afe1ae4..83362a21f77b 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.cpp
+++ b/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.cpp
@@ -22,20 +22,19 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCTargetAsmLexer.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/MemoryObject.h"
 #include "llvm/Support/SourceMgr.h"
-#include "llvm/Target/TargetAsmLexer.h"
-#include "llvm/Target/TargetAsmParser.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetSelect.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
 using namespace llvm;
 
 bool EDDisassembler::sInitialized = false;
@@ -106,9 +105,7 @@ void EDDisassembler::initialize() {
   sInitialized = true;
   
   InitializeAllTargetInfos();
-  InitializeAllTargets();
-  InitializeAllMCAsmInfos();
-  InitializeAllAsmPrinters();
+  InitializeAllTargetMCs();
   InitializeAllAsmParsers();
   InitializeAllDisassemblers();
 }
@@ -169,24 +166,24 @@ EDDisassembler::EDDisassembler(CPUKey &key) :
   if (!Tgt)
     return;
   
-  std::string CPU;
-  std::string featureString;
-  TargetMachine.reset(Tgt->createTargetMachine(tripleString, CPU,
-                                               featureString));
+  MRI.reset(Tgt->createMCRegInfo(tripleString));
 
-  const TargetRegisterInfo *registerInfo = TargetMachine->getRegisterInfo();
-  
-  if (!registerInfo)
+  if (!MRI)
     return;
-    
-  initMaps(*registerInfo);
+
+  initMaps(*MRI);
   
   AsmInfo.reset(Tgt->createMCAsmInfo(tripleString));
   
   if (!AsmInfo)
     return;
 
-  Disassembler.reset(Tgt->createMCDisassembler());
+  STI.reset(Tgt->createMCSubtargetInfo(tripleString, "", ""));
+  
+  if (!STI)
+    return;
+
+  Disassembler.reset(Tgt->createMCDisassembler(*STI));
   
   if (!Disassembler)
     return;
@@ -195,16 +192,16 @@ EDDisassembler::EDDisassembler(CPUKey &key) :
   
   InstString.reset(new std::string);
   InstStream.reset(new raw_string_ostream(*InstString));
-  InstPrinter.reset(Tgt->createMCInstPrinter(LLVMSyntaxVariant, *AsmInfo));
+  InstPrinter.reset(Tgt->createMCInstPrinter(LLVMSyntaxVariant, *AsmInfo, *STI));
   
   if (!InstPrinter)
     return;
     
   GenericAsmLexer.reset(new AsmLexer(*AsmInfo));
-  SpecificAsmLexer.reset(Tgt->createAsmLexer(*AsmInfo));
+  SpecificAsmLexer.reset(Tgt->createMCAsmLexer(*MRI, *AsmInfo));
   SpecificAsmLexer->InstallLexer(*GenericAsmLexer);
   
-  initMaps(*TargetMachine->getRegisterInfo());
+  initMaps(*MRI);
     
   Valid = true;
 }
@@ -247,14 +244,17 @@ EDInst *EDDisassembler::createInst(EDByteReaderCallback byteReader,
   MCInst* inst = new MCInst;
   uint64_t byteSize;
   
-  if (!Disassembler->getInstruction(*inst,
-                                    byteSize,
-                                    memoryObject,
-                                    address,
-                                    ErrorStream)) {
+  MCDisassembler::DecodeStatus S;
+  S = Disassembler->getInstruction(*inst, byteSize, memoryObject, address,
+                                   ErrorStream, nulls());
+  switch (S) {
+  case MCDisassembler::Fail:
+  case MCDisassembler::SoftFail:
+    // FIXME: Do something different on soft failure mode?
     delete inst;
     return NULL;
-  } else {
+    
+  case MCDisassembler::Success: {
     const llvm::EDInstInfo *thisInstInfo = NULL;
 
     if (InstInfos) {
@@ -264,9 +264,11 @@ EDInst *EDDisassembler::createInst(EDByteReaderCallback byteReader,
     EDInst* sdInst = new EDInst(inst, byteSize, *this, thisInstInfo);
     return sdInst;
   }
+  }
+  return NULL;
 }
 
-void EDDisassembler::initMaps(const TargetRegisterInfo &registerInfo) {
+void EDDisassembler::initMaps(const MCRegisterInfo &registerInfo) {
   unsigned numRegisters = registerInfo.getNumRegs();
   unsigned registerIndex;
   
@@ -325,7 +327,7 @@ bool EDDisassembler::registerIsProgramCounter(unsigned registerID) {
 int EDDisassembler::printInst(std::string &str, MCInst &inst) {
   PrinterMutex.acquire();
   
-  InstPrinter->printInst(&inst, *InstStream);
+  InstPrinter->printInst(&inst, *InstStream, "");
   InstStream->flush();
   str = *InstString;
   InstString->clear();
@@ -368,16 +370,16 @@ int EDDisassembler::parseInst(SmallVectorImpl<MCParsedAsmOperand*> &operands,
   SourceMgr sourceMgr;
   sourceMgr.setDiagHandler(diag_handler, static_cast<void*>(this));
   sourceMgr.AddNewSourceBuffer(buf, SMLoc()); // ownership of buf handed over
-  MCContext context(*AsmInfo, NULL);
+  MCContext context(*AsmInfo, *MRI, NULL);
   OwningPtr<MCStreamer> streamer(createNullStreamer(context));
-  OwningPtr<MCAsmParser> genericParser(createMCAsmParser(*Tgt, sourceMgr,
+  OwningPtr<MCAsmParser> genericParser(createMCAsmParser(sourceMgr,
                                                          context, *streamer,
                                                          *AsmInfo));
 
   StringRef triple = tripleFromArch(Key.Arch);
   OwningPtr<MCSubtargetInfo> STI(Tgt->createMCSubtargetInfo(triple, "", ""));
-  OwningPtr<TargetAsmParser> TargetParser(Tgt->createAsmParser(*STI,
-                                                               *genericParser));
+  OwningPtr<MCTargetAsmParser>
+    TargetParser(Tgt->createMCAsmParser(*STI, *genericParser));
   
   AsmToken OpcodeToken = genericParser->Lex();
   AsmToken NextToken = genericParser->Lex();  // consume next token, because specificParser expects us to
diff --git a/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.h b/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.h
index 11d69c151cf9..38c22038c510 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.h
+++ b/contrib/llvm/lib/MC/MCDisassembler/EDDisassembler.h
@@ -29,24 +29,23 @@
 
 namespace llvm {
 class AsmLexer;
+class AsmParser;
 class AsmToken;
 class MCContext;
 class MCAsmInfo;
 class MCAsmLexer;
-class AsmParser;
-class TargetAsmLexer;
-class TargetAsmParser;
 class MCDisassembler;
 class MCInstPrinter;
 class MCInst;
 class MCParsedAsmOperand;
+class MCRegisterInfo;
 class MCStreamer;
 class MCSubtargetInfo;
+class MCTargetAsmLexer;
+class MCTargetAsmParser;
 template <typename T> class SmallVectorImpl;
 class SourceMgr;
 class Target;
-class TargetMachine;
-class TargetRegisterInfo;
 
 struct EDInstInfo;
 struct EDInst;
@@ -136,10 +135,12 @@ struct EDDisassembler {
   CPUKey Key;
   /// The LLVM target corresponding to the disassembler
   const llvm::Target *Tgt;
-  /// The target machine instance.
-  llvm::OwningPtr<llvm::TargetMachine> TargetMachine;
   /// The assembly information for the target architecture
   llvm::OwningPtr<const llvm::MCAsmInfo> AsmInfo;
+  /// The subtarget information for the target architecture
+  llvm::OwningPtr<const llvm::MCSubtargetInfo> STI;
+  // The register information for the target architecture.
+  llvm::OwningPtr<const llvm::MCRegisterInfo> MRI;
   /// The disassembler for the target architecture
   llvm::OwningPtr<const llvm::MCDisassembler> Disassembler;
   /// The output string for the instruction printer; must be guarded with 
@@ -160,7 +161,7 @@ struct EDDisassembler {
   /// The target-specific lexer for use in tokenizing strings, in
   ///   target-independent and target-specific portions
   llvm::OwningPtr<llvm::AsmLexer> GenericAsmLexer;
-  llvm::OwningPtr<llvm::TargetAsmLexer> SpecificAsmLexer;
+  llvm::OwningPtr<llvm::MCTargetAsmLexer> SpecificAsmLexer;
   /// The guard for the above
   llvm::sys::Mutex ParserMutex;
   /// The LLVM number used for the target disassembly syntax variant
@@ -216,7 +217,7 @@ struct EDDisassembler {
   ///   info
   ///
   /// @arg registerInfo - the register information to use as a source
-  void initMaps(const llvm::TargetRegisterInfo &registerInfo);
+  void initMaps(const llvm::MCRegisterInfo &registerInfo);
   /// nameWithRegisterID - Returns the name (owned by the EDDisassembler) of a 
   ///   register for a given register ID, or NULL on failure
   ///
diff --git a/contrib/llvm/lib/MC/MCDisassembler/EDInst.h b/contrib/llvm/lib/MC/MCDisassembler/EDInst.h
index ceb9505028de..6b78dc826c92 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/EDInst.h
+++ b/contrib/llvm/lib/MC/MCDisassembler/EDInst.h
@@ -73,7 +73,7 @@ struct EDInst {
   std::string String;
   /// The order in which operands from the InstInfo's operand information appear
   /// in String
-  const char* OperandOrder;
+  const signed char* OperandOrder;
   
   /// The result of the parseOperands() function
   CachedResult ParseResult;
diff --git a/contrib/llvm/lib/MC/MCDisassembler/EDToken.cpp b/contrib/llvm/lib/MC/MCDisassembler/EDToken.cpp
index de770b41ef35..5f6c9df4812a 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/EDToken.cpp
+++ b/contrib/llvm/lib/MC/MCDisassembler/EDToken.cpp
@@ -87,14 +87,18 @@ int EDToken::registerID(unsigned &registerID) const {
 
 int EDToken::tokenize(std::vector<EDToken*> &tokens,
                       std::string &str,
-                      const char *operandOrder,
+                      const signed char *operandOrder,
                       EDDisassembler &disassembler) {
   SmallVector<MCParsedAsmOperand*, 5> parsedOperands;
   SmallVector<AsmToken, 10> asmTokens;
   
   if (disassembler.parseInst(parsedOperands, asmTokens, str))
+  {
+    for (unsigned i = 0, e = parsedOperands.size(); i != e; ++i)
+      delete parsedOperands[i];
     return -1;
-  
+  }
+      
   SmallVectorImpl<MCParsedAsmOperand*>::iterator operandIterator;
   unsigned int operandIndex;
   SmallVectorImpl<AsmToken>::iterator tokenIterator;
diff --git a/contrib/llvm/lib/MC/MCDisassembler/EDToken.h b/contrib/llvm/lib/MC/MCDisassembler/EDToken.h
index ba467078686a..384079b72eec 100644
--- a/contrib/llvm/lib/MC/MCDisassembler/EDToken.h
+++ b/contrib/llvm/lib/MC/MCDisassembler/EDToken.h
@@ -125,7 +125,7 @@ struct EDToken {
   //                      assembly syntax
   static int tokenize(std::vector<EDToken*> &tokens,
                       std::string &str,
-                      const char *operandOrder,
+                      const signed char *operandOrder,
                       EDDisassembler &disassembler);
   
   /// getString - Directs a character pointer to the string, returning 0 on
diff --git a/contrib/llvm/lib/MC/MCDwarf.cpp b/contrib/llvm/lib/MC/MCDwarf.cpp
index ad86db13d510..4658a3093fab 100644
--- a/contrib/llvm/lib/MC/MCDwarf.cpp
+++ b/contrib/llvm/lib/MC/MCDwarf.cpp
@@ -7,17 +7,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCDwarf.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCObjectWriter.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetAsmInfo.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -196,7 +197,7 @@ static inline void EmitDwarfLineTable(MCStreamer *MCOS,
   MCOS->EmitLabel(SectionEnd);
 
   // Switch back the the dwarf line section.
-  MCOS->SwitchSection(context.getTargetAsmInfo().getDwarfLineSection());
+  MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfLineSection());
 
   const MCAsmInfo &asmInfo = MCOS->getContext().getAsmInfo();
   MCOS->EmitDwarfAdvanceLineAddr(INT64_MAX, LastLabel, SectionEnd,
@@ -209,7 +210,7 @@ static inline void EmitDwarfLineTable(MCStreamer *MCOS,
 void MCDwarfFileTable::Emit(MCStreamer *MCOS) {
   MCContext &context = MCOS->getContext();
   // Switch to the section where the table will be emitted into.
-  MCOS->SwitchSection(context.getTargetAsmInfo().getDwarfLineSection());
+  MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfLineSection());
 
   // Create a symbol at the beginning of this section.
   MCSymbol *LineStartSym = context.CreateTempSymbol();
@@ -485,11 +486,11 @@ static void EmitPersonality(MCStreamer &streamer, const MCSymbol &symbol,
 }
 
 static const MachineLocation TranslateMachineLocation(
-                                                  const TargetAsmInfo &TAI,
+                                                  const MCRegisterInfo &MRI,
                                                   const MachineLocation &Loc) {
   unsigned Reg = Loc.getReg() == MachineLocation::VirtualFP ?
     MachineLocation::VirtualFP :
-    unsigned(TAI.getDwarfRegNum(Loc.getReg(), true));
+    unsigned(MRI.getDwarfRegNum(Loc.getReg(), true));
   const MachineLocation &NewLoc = Loc.isReg() ?
     MachineLocation(Reg) : MachineLocation(Reg, Loc.getOffset());
   return NewLoc;
@@ -503,10 +504,11 @@ namespace {
     bool IsEH;
     const MCSymbol *SectionStart;
   public:
-    FrameEmitterImpl(bool usingCFI, bool isEH, const MCSymbol *sectionStart) :
-      CFAOffset(0), CIENum(0), UsingCFI(usingCFI), IsEH(isEH),
-      SectionStart(sectionStart) {
-    }
+    FrameEmitterImpl(bool usingCFI, bool isEH)
+      : CFAOffset(0), CIENum(0), UsingCFI(usingCFI), IsEH(isEH),
+        SectionStart(0) {}
+
+    void setSectionStart(const MCSymbol *Label) { SectionStart = Label; }
 
     /// EmitCompactUnwind - Emit the unwind information in a compact way. If
     /// we're successful, return 'true'. Otherwise, return 'false' and it will
@@ -687,11 +689,8 @@ void FrameEmitterImpl::EmitCFIInstructions(MCStreamer &streamer,
 /// normal CIE and FDE.
 bool FrameEmitterImpl::EmitCompactUnwind(MCStreamer &Streamer,
                                          const MCDwarfFrameInfo &Frame) {
-#if 1
-  return false;
-#else
   MCContext &Context = Streamer.getContext();
-  const TargetAsmInfo &TAI = Context.getTargetAsmInfo();
+  const MCObjectFileInfo *MOFI = Context.getObjectFileInfo();
   bool VerboseAsm = Streamer.isVerboseAsm();
 
   // range-start range-length  compact-unwind-enc personality-func   lsda
@@ -716,19 +715,17 @@ bool FrameEmitterImpl::EmitCompactUnwind(MCStreamer &Streamer,
   //   .quad __gxx_personality
   //   .quad except_tab1
 
-  uint32_t Encoding =
-    TAI.getCompactUnwindEncoding(Frame.Instructions,
-                                 getDataAlignmentFactor(Streamer), IsEH);
+  uint32_t Encoding = Frame.CompactUnwindEncoding;
   if (!Encoding) return false;
 
   // The encoding needs to know we have an LSDA.
   if (Frame.Lsda)
     Encoding |= 0x40000000;
 
-  Streamer.SwitchSection(TAI.getCompactUnwindSection());
+  Streamer.SwitchSection(MOFI->getCompactUnwindSection());
 
   // Range Start
-  unsigned FDEEncoding = TAI.getFDEEncoding(UsingCFI);
+  unsigned FDEEncoding = MOFI->getFDEEncoding(UsingCFI);
   unsigned Size = getSizeForEncoding(Streamer, FDEEncoding);
   if (VerboseAsm) Streamer.AddComment("Range Start");
   Streamer.EmitSymbolValue(Frame.Function, Size);
@@ -745,6 +742,7 @@ bool FrameEmitterImpl::EmitCompactUnwind(MCStreamer &Streamer,
                                       Twine(llvm::utohexstr(Encoding)));
   Streamer.EmitIntValue(Encoding, Size);
 
+
   // Personality Function
   Size = getSizeForEncoding(Streamer, dwarf::DW_EH_PE_absptr);
   if (VerboseAsm) Streamer.AddComment("Personality Function");
@@ -762,7 +760,6 @@ bool FrameEmitterImpl::EmitCompactUnwind(MCStreamer &Streamer,
     Streamer.EmitIntValue(0, Size); // No LSDA
 
   return true;
-#endif
 }
 
 const MCSymbol &FrameEmitterImpl::EmitCIE(MCStreamer &streamer,
@@ -771,11 +768,12 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCStreamer &streamer,
                                           const MCSymbol *lsda,
                                           unsigned lsdaEncoding) {
   MCContext &context = streamer.getContext();
-  const TargetAsmInfo &TAI = context.getTargetAsmInfo();
+  const MCRegisterInfo &MRI = context.getRegisterInfo();
+  const MCObjectFileInfo *MOFI = context.getObjectFileInfo();
   bool verboseAsm = streamer.isVerboseAsm();
 
   MCSymbol *sectionStart;
-  if (TAI.isFunctionEHFrameSymbolPrivate() || !IsEH)
+  if (MOFI->isFunctionEHFrameSymbolPrivate() || !IsEH)
     sectionStart = context.CreateTempSymbol();
   else
     sectionStart = context.GetOrCreateSymbol(Twine("EH_frame") + Twine(CIENum));
@@ -824,7 +822,7 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCStreamer &streamer,
 
   // Return Address Register
   if (verboseAsm) streamer.AddComment("CIE Return Address Column");
-  streamer.EmitULEB128IntValue(TAI.getDwarfRARegNum(true));
+  streamer.EmitULEB128IntValue(MRI.getDwarfRegNum(MRI.getRARegister(), true));
 
   // Augmentation Data Length (optional)
 
@@ -858,21 +856,22 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCStreamer &streamer,
       EmitEncodingByte(streamer, lsdaEncoding, "LSDA Encoding");
 
     // Encoding of the FDE pointers
-    EmitEncodingByte(streamer, TAI.getFDEEncoding(UsingCFI),
+    EmitEncodingByte(streamer, MOFI->getFDEEncoding(UsingCFI),
                      "FDE Encoding");
   }
 
   // Initial Instructions
 
-  const std::vector<MachineMove> &Moves = TAI.getInitialFrameState();
+  const MCAsmInfo &MAI = context.getAsmInfo();
+  const std::vector<MachineMove> &Moves = MAI.getInitialFrameState();
   std::vector<MCCFIInstruction> Instructions;
 
   for (int i = 0, n = Moves.size(); i != n; ++i) {
     MCSymbol *Label = Moves[i].getLabel();
     const MachineLocation &Dst =
-      TranslateMachineLocation(TAI, Moves[i].getDestination());
+      TranslateMachineLocation(MRI, Moves[i].getDestination());
     const MachineLocation &Src =
-      TranslateMachineLocation(TAI, Moves[i].getSource());
+      TranslateMachineLocation(MRI, Moves[i].getSource());
     MCCFIInstruction Inst(Label, Dst, Src);
     Instructions.push_back(Inst);
   }
@@ -893,10 +892,10 @@ MCSymbol *FrameEmitterImpl::EmitFDE(MCStreamer &streamer,
   MCContext &context = streamer.getContext();
   MCSymbol *fdeStart = context.CreateTempSymbol();
   MCSymbol *fdeEnd = context.CreateTempSymbol();
-  const TargetAsmInfo &TAI = context.getTargetAsmInfo();
+  const MCObjectFileInfo *MOFI = context.getObjectFileInfo();
   bool verboseAsm = streamer.isVerboseAsm();
 
-  if (!TAI.isFunctionEHFrameSymbolPrivate() && IsEH) {
+  if (IsEH && frame.Function && !MOFI->isFunctionEHFrameSymbolPrivate()) {
     MCSymbol *EHSym =
       context.GetOrCreateSymbol(frame.Function->getName() + Twine(".eh"));
     streamer.EmitEHSymAttributes(frame.Function, EHSym);
@@ -925,7 +924,7 @@ MCSymbol *FrameEmitterImpl::EmitFDE(MCStreamer &streamer,
     streamer.EmitSymbolValue(&cieStart, 4);
   }
 
-  unsigned fdeEncoding = TAI.getFDEEncoding(UsingCFI);
+  unsigned fdeEncoding = MOFI->getFDEEncoding(UsingCFI);
   unsigned size = getSizeForEncoding(streamer, fdeEncoding);
 
   // PC Begin
@@ -1011,26 +1010,34 @@ void MCDwarfFrameEmitter::Emit(MCStreamer &Streamer,
                                bool UsingCFI,
                                bool IsEH) {
   MCContext &Context = Streamer.getContext();
-  const TargetAsmInfo &TAI = Context.getTargetAsmInfo();
-  const MCSection &Section = IsEH ? *TAI.getEHFrameSection() :
-                                    *TAI.getDwarfFrameSection();
+  MCObjectFileInfo *MOFI =
+    const_cast<MCObjectFileInfo*>(Context.getObjectFileInfo());
+  FrameEmitterImpl Emitter(UsingCFI, IsEH);
+  ArrayRef<MCDwarfFrameInfo> FrameArray = Streamer.getFrameInfos();
+
+  // Emit the compact unwind info if available.
+  // FIXME: This emits both the compact unwind and the old CIE/FDE
+  //        information. Only one of those is needed.
+  if (IsEH && MOFI->getCompactUnwindSection())
+    for (unsigned i = 0, n = Streamer.getNumFrameInfos(); i < n; ++i) {
+      const MCDwarfFrameInfo &Frame = Streamer.getFrameInfo(i);
+      if (!Frame.CompactUnwindEncoding)
+        Emitter.EmitCompactUnwind(Streamer, Frame);
+    }
+
+  const MCSection &Section = IsEH ? *MOFI->getEHFrameSection() :
+                                    *MOFI->getDwarfFrameSection();
   Streamer.SwitchSection(&Section);
   MCSymbol *SectionStart = Context.CreateTempSymbol();
   Streamer.EmitLabel(SectionStart);
+  Emitter.setSectionStart(SectionStart);
 
   MCSymbol *FDEEnd = NULL;
   DenseMap<CIEKey, const MCSymbol*> CIEStarts;
-  FrameEmitterImpl Emitter(UsingCFI, IsEH, SectionStart);
 
   const MCSymbol *DummyDebugKey = NULL;
-  for (unsigned i = 0, n = Streamer.getNumFrameInfos(); i < n; ++i) {
-    const MCDwarfFrameInfo &Frame = Streamer.getFrameInfo(i);
-    if (IsEH && TAI.getCompactUnwindSection() &&
-        Emitter.EmitCompactUnwind(Streamer, Frame)) {
-      FDEEnd = NULL;
-      continue;
-    }
-
+  for (unsigned i = 0, n = FrameArray.size(); i < n; ++i) {
+    const MCDwarfFrameInfo &Frame = FrameArray[i];
     CIEKey Key(Frame.Personality, Frame.PersonalityEncoding,
                Frame.LsdaEncoding);
     const MCSymbol *&CIEStart = IsEH ? CIEStarts[Key] : DummyDebugKey;
diff --git a/contrib/llvm/lib/MC/MCELF.cpp b/contrib/llvm/lib/MC/MCELF.cpp
index 2c3f8e8f7866..dad2e7ba9878 100644
--- a/contrib/llvm/lib/MC/MCELF.cpp
+++ b/contrib/llvm/lib/MC/MCELF.cpp
@@ -16,7 +16,6 @@
 #include "llvm/MC/MCELFSymbolFlags.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/Support/ELF.h"
-#include "llvm/Target/TargetAsmBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/lib/MC/MCELFStreamer.cpp b/contrib/llvm/lib/MC/MCELFStreamer.cpp
index 49340edbed5e..9ada08ea9530 100644
--- a/contrib/llvm/lib/MC/MCELFStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCELFStreamer.cpp
@@ -21,11 +21,11 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetAsmBackend.h"
 
 using namespace llvm;
 
@@ -53,8 +53,9 @@ void MCELFStreamer::EmitLabel(MCSymbol *Symbol) {
 void MCELFStreamer::EmitAssemblerFlag(MCAssemblerFlag Flag) {
   switch (Flag) {
   case MCAF_SyntaxUnified: return; // no-op here.
-  case MCAF_Code16: return; // no-op here.
-  case MCAF_Code32: return; // no-op here.
+  case MCAF_Code16: return; // Change parsing mode; no-op here.
+  case MCAF_Code32: return; // Change parsing mode; no-op here.
+  case MCAF_Code64: return; // Change parsing mode; no-op here.
   case MCAF_SubsectionsViaSymbols:
     getAssembler().setSubsectionsViaSymbols(true);
     return;
@@ -219,14 +220,14 @@ void MCELFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   SD.setSize(MCConstantExpr::Create(Size, getContext()));
 }
 
-void MCELFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {
+void MCELFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                          unsigned ByteAlignment) {
   // FIXME: Should this be caught and done earlier?
   MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
   MCELF::SetBinding(SD, ELF::STB_LOCAL);
   SD.setExternal(false);
   BindingExplicitlySet.insert(Symbol);
-  // FIXME: ByteAlignment is not needed here, but is required.
-  EmitCommonSymbol(Symbol, Size, 1);
+  EmitCommonSymbol(Symbol, Size, ByteAlignment);
 }
 
 void MCELFStreamer::EmitBytes(StringRef Data, unsigned AddrSpace) {
@@ -374,10 +375,10 @@ void MCELFStreamer::Finish() {
   this->MCObjectStreamer::Finish();
 }
 
-MCStreamer *llvm::createELFStreamer(MCContext &Context, TargetAsmBackend &TAB,
+MCStreamer *llvm::createELFStreamer(MCContext &Context, MCAsmBackend &MAB,
                                     raw_ostream &OS, MCCodeEmitter *CE,
                                     bool RelaxAll, bool NoExecStack) {
-  MCELFStreamer *S = new MCELFStreamer(Context, TAB, OS, CE);
+  MCELFStreamer *S = new MCELFStreamer(Context, MAB, OS, CE);
   if (RelaxAll)
     S->getAssembler().setRelaxAll(true);
   if (NoExecStack)
diff --git a/contrib/llvm/lib/MC/MCELFStreamer.h b/contrib/llvm/lib/MC/MCELFStreamer.h
index 855e7e9ca60f..10bf77580998 100644
--- a/contrib/llvm/lib/MC/MCELFStreamer.h
+++ b/contrib/llvm/lib/MC/MCELFStreamer.h
@@ -25,11 +25,11 @@ namespace llvm {
 
 class MCELFStreamer : public MCObjectStreamer {
 public:
-  MCELFStreamer(MCContext &Context, TargetAsmBackend &TAB,
+  MCELFStreamer(MCContext &Context, MCAsmBackend &TAB,
                   raw_ostream &OS, MCCodeEmitter *Emitter)
     : MCObjectStreamer(Context, TAB, OS, Emitter) {}
 
-  MCELFStreamer(MCContext &Context, TargetAsmBackend &TAB,
+  MCELFStreamer(MCContext &Context, MCAsmBackend &TAB,
                 raw_ostream &OS, MCCodeEmitter *Emitter,
                 MCAssembler *Assembler)
     : MCObjectStreamer(Context, TAB, OS, Emitter, Assembler) {}
@@ -74,7 +74,8 @@ class MCELFStreamer : public MCObjectStreamer {
      SD.setSize(Value);
   }
 
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size);
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment);
 
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
                             unsigned Size = 0, unsigned ByteAlignment = 0) {
diff --git a/contrib/llvm/lib/MC/MCExpr.cpp b/contrib/llvm/lib/MC/MCExpr.cpp
index fcf1aabb5a86..da297fb1d95a 100644
--- a/contrib/llvm/lib/MC/MCExpr.cpp
+++ b/contrib/llvm/lib/MC/MCExpr.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetAsmBackend.h"
 using namespace llvm;
 
 namespace {
diff --git a/contrib/llvm/lib/MC/MCInstPrinter.cpp b/contrib/llvm/lib/MC/MCInstPrinter.cpp
index 81a939ffaaef..2317a2891f8b 100644
--- a/contrib/llvm/lib/MC/MCInstPrinter.cpp
+++ b/contrib/llvm/lib/MC/MCInstPrinter.cpp
@@ -8,7 +8,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
 MCInstPrinter::~MCInstPrinter() {
@@ -23,3 +25,12 @@ StringRef MCInstPrinter::getOpcodeName(unsigned Opcode) const {
 void MCInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   assert(0 && "Target should implement this");
 }
+
+void MCInstPrinter::printAnnotation(raw_ostream &OS, StringRef Annot) {
+  if (!Annot.empty()) {
+    if (CommentStream)
+      (*CommentStream) << Annot;
+    else
+      OS << " " << MAI.getCommentString() << " " << Annot;
+  }
+}
diff --git a/contrib/llvm/lib/MC/MCInstrAnalysis.cpp b/contrib/llvm/lib/MC/MCInstrAnalysis.cpp
new file mode 100644
index 000000000000..7736702f35a4
--- /dev/null
+++ b/contrib/llvm/lib/MC/MCInstrAnalysis.cpp
@@ -0,0 +1,21 @@
+//===-- MCInstrAnalysis.cpp - InstrDesc target hooks ------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCInstrAnalysis.h"
+using namespace llvm;
+
+uint64_t MCInstrAnalysis::evaluateBranch(const MCInst &Inst, uint64_t Addr,
+                                         uint64_t Size) const {
+  if (Inst.getNumOperands() == 0 ||
+      Info->get(Inst.getOpcode()).OpInfo[0].OperandType != MCOI::OPERAND_PCREL)
+    return -1ULL;
+
+  int64_t Imm = Inst.getOperand(0).getImm();
+  return Addr+Size+Imm;
+}
diff --git a/contrib/llvm/lib/MC/MCLoggingStreamer.cpp b/contrib/llvm/lib/MC/MCLoggingStreamer.cpp
index 309752ec5f02..3fe8ac72c8ec 100644
--- a/contrib/llvm/lib/MC/MCLoggingStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCLoggingStreamer.cpp
@@ -133,9 +133,10 @@ class MCLoggingStreamer : public MCStreamer {
     return Child->EmitCommonSymbol(Symbol, Size, ByteAlignment);
   }
 
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment) {
     LogCall("EmitLocalCommonSymbol");
-    return Child->EmitLocalCommonSymbol(Symbol, Size);
+    return Child->EmitLocalCommonSymbol(Symbol, Size, ByteAlignment);
   }
   
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
diff --git a/contrib/llvm/lib/MC/MCMachOStreamer.cpp b/contrib/llvm/lib/MC/MCMachOStreamer.cpp
index 1b21249ca321..aa35815dd19c 100644
--- a/contrib/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCMachOStreamer.cpp
@@ -20,10 +20,10 @@
 #include "llvm/MC/MCMachOSymbolFlags.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCDwarf.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetAsmBackend.h"
 
 using namespace llvm;
 
@@ -34,9 +34,9 @@ class MCMachOStreamer : public MCObjectStreamer {
   virtual void EmitInstToData(const MCInst &Inst);
 
 public:
-  MCMachOStreamer(MCContext &Context, TargetAsmBackend &TAB,
+  MCMachOStreamer(MCContext &Context, MCAsmBackend &MAB,
                   raw_ostream &OS, MCCodeEmitter *Emitter)
-    : MCObjectStreamer(Context, TAB, OS, Emitter) {}
+    : MCObjectStreamer(Context, MAB, OS, Emitter) {}
 
   /// @name MCStreamer Interface
   /// @{
@@ -67,7 +67,8 @@ class MCMachOStreamer : public MCObjectStreamer {
   virtual void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
     assert(0 && "macho doesn't support this directive");
   }
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment) {
     assert(0 && "macho doesn't support this directive");
   }
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
@@ -143,8 +144,9 @@ void MCMachOStreamer::EmitAssemblerFlag(MCAssemblerFlag Flag) {
   // Do any generic stuff we need to do.
   switch (Flag) {
   case MCAF_SyntaxUnified: return; // no-op here.
-  case MCAF_Code16: return; // no-op here.
-  case MCAF_Code32: return; // no-op here.
+  case MCAF_Code16: return; // Change parsing mode; no-op here.
+  case MCAF_Code32: return; // Change parsing mode; no-op here.
+  case MCAF_Code64: return; // Change parsing mode; no-op here.
   case MCAF_SubsectionsViaSymbols:
     getAssembler().setSubsectionsViaSymbols(true);
     return;
@@ -207,8 +209,8 @@ void MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   case MCSA_ELF_TypeCommon:
   case MCSA_ELF_TypeNoType:
   case MCSA_ELF_TypeGnuUniqueObject:
-  case MCSA_IndirectSymbol:
   case MCSA_Hidden:
+  case MCSA_IndirectSymbol:
   case MCSA_Internal:
   case MCSA_Protected:
   case MCSA_Weak:
@@ -410,10 +412,10 @@ void MCMachOStreamer::Finish() {
   this->MCObjectStreamer::Finish();
 }
 
-MCStreamer *llvm::createMachOStreamer(MCContext &Context, TargetAsmBackend &TAB,
+MCStreamer *llvm::createMachOStreamer(MCContext &Context, MCAsmBackend &MAB,
                                       raw_ostream &OS, MCCodeEmitter *CE,
                                       bool RelaxAll) {
-  MCMachOStreamer *S = new MCMachOStreamer(Context, TAB, OS, CE);
+  MCMachOStreamer *S = new MCMachOStreamer(Context, MAB, OS, CE);
   if (RelaxAll)
     S->getAssembler().setRelaxAll(true);
   return S;
diff --git a/contrib/llvm/lib/MC/MCModule.cpp b/contrib/llvm/lib/MC/MCModule.cpp
new file mode 100644
index 000000000000..b1d09d945a39
--- /dev/null
+++ b/contrib/llvm/lib/MC/MCModule.cpp
@@ -0,0 +1,45 @@
+//===- lib/MC/MCModule.cpp - MCModule implementation --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCAtom.h"
+#include "llvm/MC/MCModule.h"
+
+using namespace llvm;
+
+MCAtom *MCModule::createAtom(MCAtom::AtomType Type,
+                             uint64_t Begin, uint64_t End) {
+  assert(Begin < End && "Creating MCAtom with endpoints reversed?");
+
+  // Check for atoms already covering this range.
+  IntervalMap<uint64_t, MCAtom*>::iterator I = OffsetMap.find(Begin);
+  assert((!I.valid() || I.start() < End) && "Offset range already occupied!");
+
+  // Create the new atom and add it to our maps.
+  MCAtom *NewAtom = new MCAtom(Type, this, Begin, End);
+  AtomAllocationTracker.insert(NewAtom);
+  OffsetMap.insert(Begin, End, NewAtom);
+  return NewAtom;
+}
+
+// remap - Update the interval mapping for an atom.
+void MCModule::remap(MCAtom *Atom, uint64_t NewBegin, uint64_t NewEnd) {
+  // Find and erase the old mapping.
+  IntervalMap<uint64_t, MCAtom*>::iterator I = OffsetMap.find(Atom->Begin);
+  assert(I.valid() && "Atom offset not found in module!");
+  assert(*I == Atom && "Previous atom mapping was invalid!");
+  I.erase();
+
+  // Insert the new mapping.
+  OffsetMap.insert(NewBegin, NewEnd, Atom);
+
+  // Update the atom internal bounds.
+  Atom->Begin = NewBegin;
+  Atom->End = NewEnd;
+}
+
diff --git a/contrib/llvm/lib/MC/MCNullStreamer.cpp b/contrib/llvm/lib/MC/MCNullStreamer.cpp
index 9577af010205..a6c0adb6793f 100644
--- a/contrib/llvm/lib/MC/MCNullStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCNullStreamer.cpp
@@ -59,8 +59,8 @@ namespace {
     virtual void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {}
     virtual void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                   unsigned ByteAlignment) {}
-    virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {}
-
+    virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                       unsigned ByteAlignment) {}
     virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
                               unsigned Size = 0, unsigned ByteAlignment = 0) {}
     virtual void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
diff --git a/contrib/llvm/lib/MC/MCObjectFileInfo.cpp b/contrib/llvm/lib/MC/MCObjectFileInfo.cpp
new file mode 100644
index 000000000000..df8b99d2be6c
--- /dev/null
+++ b/contrib/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -0,0 +1,554 @@
+//===-- MObjectFileInfo.cpp - Object File Information ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCSectionCOFF.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSectionMachO.h"
+#include "llvm/ADT/Triple.h"
+using namespace llvm;
+
+void MCObjectFileInfo::InitMachOMCObjectFileInfo(Triple T) {
+  // MachO
+  IsFunctionEHFrameSymbolPrivate = false;
+  SupportsWeakOmittedEHFrame = false;
+
+  PersonalityEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel
+    | dwarf::DW_EH_PE_sdata4;
+  LSDAEncoding = FDEEncoding = FDECFIEncoding = dwarf::DW_EH_PE_pcrel;
+  TTypeEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
+    dwarf::DW_EH_PE_sdata4;
+
+  // .comm doesn't support alignment before Leopard.
+  if (T.isMacOSX() && T.isMacOSXVersionLT(10, 5))
+    CommDirectiveSupportsAlignment = false;
+
+  TextSection // .text
+    = Ctx->getMachOSection("__TEXT", "__text",
+                           MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS,
+                           SectionKind::getText());
+  DataSection // .data
+    = Ctx->getMachOSection("__DATA", "__data", 0,
+                           SectionKind::getDataRel());
+
+  TLSDataSection // .tdata
+    = Ctx->getMachOSection("__DATA", "__thread_data",
+                           MCSectionMachO::S_THREAD_LOCAL_REGULAR,
+                           SectionKind::getDataRel());
+  TLSBSSSection // .tbss
+    = Ctx->getMachOSection("__DATA", "__thread_bss",
+                           MCSectionMachO::S_THREAD_LOCAL_ZEROFILL,
+                           SectionKind::getThreadBSS());
+
+  // TODO: Verify datarel below.
+  TLSTLVSection // .tlv
+    = Ctx->getMachOSection("__DATA", "__thread_vars",
+                           MCSectionMachO::S_THREAD_LOCAL_VARIABLES,
+                           SectionKind::getDataRel());
+
+  TLSThreadInitSection
+    = Ctx->getMachOSection("__DATA", "__thread_init",
+                           MCSectionMachO::S_THREAD_LOCAL_INIT_FUNCTION_POINTERS,
+                           SectionKind::getDataRel());
+
+  CStringSection // .cstring
+    = Ctx->getMachOSection("__TEXT", "__cstring",
+                           MCSectionMachO::S_CSTRING_LITERALS,
+                           SectionKind::getMergeable1ByteCString());
+  UStringSection
+    = Ctx->getMachOSection("__TEXT","__ustring", 0,
+                           SectionKind::getMergeable2ByteCString());
+  FourByteConstantSection // .literal4
+    = Ctx->getMachOSection("__TEXT", "__literal4",
+                           MCSectionMachO::S_4BYTE_LITERALS,
+                           SectionKind::getMergeableConst4());
+  EightByteConstantSection // .literal8
+    = Ctx->getMachOSection("__TEXT", "__literal8",
+                           MCSectionMachO::S_8BYTE_LITERALS,
+                           SectionKind::getMergeableConst8());
+
+  // ld_classic doesn't support .literal16 in 32-bit mode, and ld64 falls back
+  // to using it in -static mode.
+  SixteenByteConstantSection = 0;
+  if (RelocM != Reloc::Static &&
+      T.getArch() != Triple::x86_64 && T.getArch() != Triple::ppc64)
+    SixteenByteConstantSection =   // .literal16
+      Ctx->getMachOSection("__TEXT", "__literal16",
+                           MCSectionMachO::S_16BYTE_LITERALS,
+                           SectionKind::getMergeableConst16());
+
+  ReadOnlySection  // .const
+    = Ctx->getMachOSection("__TEXT", "__const", 0,
+                           SectionKind::getReadOnly());
+
+  TextCoalSection
+    = Ctx->getMachOSection("__TEXT", "__textcoal_nt",
+                           MCSectionMachO::S_COALESCED |
+                           MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS,
+                           SectionKind::getText());
+  ConstTextCoalSection
+    = Ctx->getMachOSection("__TEXT", "__const_coal",
+                           MCSectionMachO::S_COALESCED,
+                           SectionKind::getReadOnly());
+  ConstDataSection  // .const_data
+    = Ctx->getMachOSection("__DATA", "__const", 0,
+                           SectionKind::getReadOnlyWithRel());
+  DataCoalSection
+    = Ctx->getMachOSection("__DATA","__datacoal_nt",
+                           MCSectionMachO::S_COALESCED,
+                           SectionKind::getDataRel());
+  DataCommonSection
+    = Ctx->getMachOSection("__DATA","__common",
+                           MCSectionMachO::S_ZEROFILL,
+                           SectionKind::getBSS());
+  DataBSSSection
+    = Ctx->getMachOSection("__DATA","__bss", MCSectionMachO::S_ZEROFILL,
+                           SectionKind::getBSS());
+
+
+  LazySymbolPointerSection
+    = Ctx->getMachOSection("__DATA", "__la_symbol_ptr",
+                           MCSectionMachO::S_LAZY_SYMBOL_POINTERS,
+                           SectionKind::getMetadata());
+  NonLazySymbolPointerSection
+    = Ctx->getMachOSection("__DATA", "__nl_symbol_ptr",
+                           MCSectionMachO::S_NON_LAZY_SYMBOL_POINTERS,
+                           SectionKind::getMetadata());
+
+  if (RelocM == Reloc::Static) {
+    StaticCtorSection
+      = Ctx->getMachOSection("__TEXT", "__constructor", 0,
+                             SectionKind::getDataRel());
+    StaticDtorSection
+      = Ctx->getMachOSection("__TEXT", "__destructor", 0,
+                             SectionKind::getDataRel());
+  } else {
+    StaticCtorSection
+      = Ctx->getMachOSection("__DATA", "__mod_init_func",
+                             MCSectionMachO::S_MOD_INIT_FUNC_POINTERS,
+                             SectionKind::getDataRel());
+    StaticDtorSection
+      = Ctx->getMachOSection("__DATA", "__mod_term_func",
+                             MCSectionMachO::S_MOD_TERM_FUNC_POINTERS,
+                             SectionKind::getDataRel());
+  }
+
+  // Exception Handling.
+  LSDASection = Ctx->getMachOSection("__TEXT", "__gcc_except_tab", 0,
+                                     SectionKind::getReadOnlyWithRel());
+
+  if (T.isMacOSX() && !T.isMacOSXVersionLT(10, 6))
+    CompactUnwindSection =
+      Ctx->getMachOSection("__LD", "__compact_unwind",
+                           MCSectionMachO::S_ATTR_DEBUG,
+                           SectionKind::getReadOnly());
+
+  // Debug Information.
+  DwarfAbbrevSection =
+    Ctx->getMachOSection("__DWARF", "__debug_abbrev",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfInfoSection =
+    Ctx->getMachOSection("__DWARF", "__debug_info",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfLineSection =
+    Ctx->getMachOSection("__DWARF", "__debug_line",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfFrameSection =
+    Ctx->getMachOSection("__DWARF", "__debug_frame",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfPubNamesSection =
+    Ctx->getMachOSection("__DWARF", "__debug_pubnames",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfPubTypesSection =
+    Ctx->getMachOSection("__DWARF", "__debug_pubtypes",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfStrSection =
+    Ctx->getMachOSection("__DWARF", "__debug_str",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfLocSection =
+    Ctx->getMachOSection("__DWARF", "__debug_loc",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfARangesSection =
+    Ctx->getMachOSection("__DWARF", "__debug_aranges",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfRangesSection =
+    Ctx->getMachOSection("__DWARF", "__debug_ranges",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfMacroInfoSection =
+    Ctx->getMachOSection("__DWARF", "__debug_macinfo",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+  DwarfDebugInlineSection =
+    Ctx->getMachOSection("__DWARF", "__debug_inlined",
+                         MCSectionMachO::S_ATTR_DEBUG,
+                         SectionKind::getMetadata());
+
+  TLSExtraDataSection = TLSTLVSection;
+}
+
+void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) {
+  if (T.getArch() == Triple::x86) {
+    PersonalityEncoding = (RelocM == Reloc::PIC_)
+      ? dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4
+      : dwarf::DW_EH_PE_absptr;
+    LSDAEncoding = (RelocM == Reloc::PIC_)
+      ? dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4
+      : dwarf::DW_EH_PE_absptr;
+    FDEEncoding = FDECFIEncoding = (RelocM == Reloc::PIC_)
+      ? dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4
+      : dwarf::DW_EH_PE_absptr;
+    TTypeEncoding = (RelocM == Reloc::PIC_)
+      ? dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4
+      : dwarf::DW_EH_PE_absptr;
+  } else if (T.getArch() == Triple::x86_64) {
+    FDECFIEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4;
+
+    if (RelocM == Reloc::PIC_) {
+      PersonalityEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
+        ((CMModel == CodeModel::Small || CMModel == CodeModel::Medium)
+         ? dwarf::DW_EH_PE_sdata4 : dwarf::DW_EH_PE_sdata8);
+      LSDAEncoding = dwarf::DW_EH_PE_pcrel |
+        (CMModel == CodeModel::Small
+         ? dwarf::DW_EH_PE_sdata4 : dwarf::DW_EH_PE_sdata8);
+      FDEEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4;
+      TTypeEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
+        ((CMModel == CodeModel::Small || CMModel == CodeModel::Medium)
+         ? dwarf::DW_EH_PE_sdata4 : dwarf::DW_EH_PE_sdata8);
+    } else {
+      PersonalityEncoding =
+        (CMModel == CodeModel::Small || CMModel == CodeModel::Medium)
+        ? dwarf::DW_EH_PE_udata4 : dwarf::DW_EH_PE_absptr;
+      LSDAEncoding = (CMModel == CodeModel::Small)
+        ? dwarf::DW_EH_PE_udata4 : dwarf::DW_EH_PE_absptr;
+      FDEEncoding = dwarf::DW_EH_PE_udata4;
+      TTypeEncoding = (CMModel == CodeModel::Small)
+        ? dwarf::DW_EH_PE_udata4 : dwarf::DW_EH_PE_absptr;
+    }
+  }
+
+  // ELF
+  BSSSection =
+    Ctx->getELFSection(".bss", ELF::SHT_NOBITS,
+                       ELF::SHF_WRITE |ELF::SHF_ALLOC,
+                       SectionKind::getBSS());
+
+  TextSection =
+    Ctx->getELFSection(".text", ELF::SHT_PROGBITS,
+                       ELF::SHF_EXECINSTR |
+                       ELF::SHF_ALLOC,
+                       SectionKind::getText());
+
+  DataSection =
+    Ctx->getELFSection(".data", ELF::SHT_PROGBITS,
+                       ELF::SHF_WRITE |ELF::SHF_ALLOC,
+                       SectionKind::getDataRel());
+
+  ReadOnlySection =
+    Ctx->getELFSection(".rodata", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC,
+                       SectionKind::getReadOnly());
+
+  TLSDataSection =
+    Ctx->getELFSection(".tdata", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC | ELF::SHF_TLS |
+                       ELF::SHF_WRITE,
+                       SectionKind::getThreadData());
+
+  TLSBSSSection =
+    Ctx->getELFSection(".tbss", ELF::SHT_NOBITS,
+                       ELF::SHF_ALLOC | ELF::SHF_TLS |
+                       ELF::SHF_WRITE,
+                       SectionKind::getThreadBSS());
+
+  DataRelSection =
+    Ctx->getELFSection(".data.rel", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_WRITE,
+                       SectionKind::getDataRel());
+
+  DataRelLocalSection =
+    Ctx->getELFSection(".data.rel.local", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_WRITE,
+                       SectionKind::getDataRelLocal());
+
+  DataRelROSection =
+    Ctx->getELFSection(".data.rel.ro", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_WRITE,
+                       SectionKind::getReadOnlyWithRel());
+
+  DataRelROLocalSection =
+    Ctx->getELFSection(".data.rel.ro.local", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_WRITE,
+                       SectionKind::getReadOnlyWithRelLocal());
+
+  MergeableConst4Section =
+    Ctx->getELFSection(".rodata.cst4", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_MERGE,
+                       SectionKind::getMergeableConst4());
+
+  MergeableConst8Section =
+    Ctx->getELFSection(".rodata.cst8", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_MERGE,
+                       SectionKind::getMergeableConst8());
+
+  MergeableConst16Section =
+    Ctx->getELFSection(".rodata.cst16", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_MERGE,
+                       SectionKind::getMergeableConst16());
+
+  StaticCtorSection =
+    Ctx->getELFSection(".ctors", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_WRITE,
+                       SectionKind::getDataRel());
+
+  StaticDtorSection =
+    Ctx->getELFSection(".dtors", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC |ELF::SHF_WRITE,
+                       SectionKind::getDataRel());
+
+  // Exception Handling Sections.
+
+  // FIXME: We're emitting LSDA info into a readonly section on ELF, even though
+  // it contains relocatable pointers.  In PIC mode, this is probably a big
+  // runtime hit for C++ apps.  Either the contents of the LSDA need to be
+  // adjusted or this should be a data section.
+  LSDASection =
+    Ctx->getELFSection(".gcc_except_table", ELF::SHT_PROGBITS,
+                       ELF::SHF_ALLOC,
+                       SectionKind::getReadOnly());
+
+  // Debug Info Sections.
+  DwarfAbbrevSection =
+    Ctx->getELFSection(".debug_abbrev", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfInfoSection =
+    Ctx->getELFSection(".debug_info", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfLineSection =
+    Ctx->getELFSection(".debug_line", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfFrameSection =
+    Ctx->getELFSection(".debug_frame", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfPubNamesSection =
+    Ctx->getELFSection(".debug_pubnames", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfPubTypesSection =
+    Ctx->getELFSection(".debug_pubtypes", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfStrSection =
+    Ctx->getELFSection(".debug_str", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfLocSection =
+    Ctx->getELFSection(".debug_loc", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfARangesSection =
+    Ctx->getELFSection(".debug_aranges", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfRangesSection =
+    Ctx->getELFSection(".debug_ranges", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfMacroInfoSection =
+    Ctx->getELFSection(".debug_macinfo", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+}
+
+
+void MCObjectFileInfo::InitCOFFMCObjectFileInfo(Triple T) {
+  // COFF
+  TextSection =
+    Ctx->getCOFFSection(".text",
+                        COFF::IMAGE_SCN_CNT_CODE |
+                        COFF::IMAGE_SCN_MEM_EXECUTE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getText());
+  DataSection =
+    Ctx->getCOFFSection(".data",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ |
+                        COFF::IMAGE_SCN_MEM_WRITE,
+                        SectionKind::getDataRel());
+  ReadOnlySection =
+    Ctx->getCOFFSection(".rdata",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getReadOnly());
+  StaticCtorSection =
+    Ctx->getCOFFSection(".ctors",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ |
+                        COFF::IMAGE_SCN_MEM_WRITE,
+                        SectionKind::getDataRel());
+  StaticDtorSection =
+    Ctx->getCOFFSection(".dtors",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ |
+                        COFF::IMAGE_SCN_MEM_WRITE,
+                        SectionKind::getDataRel());
+
+  // FIXME: We're emitting LSDA info into a readonly section on COFF, even
+  // though it contains relocatable pointers.  In PIC mode, this is probably a
+  // big runtime hit for C++ apps.  Either the contents of the LSDA need to be
+  // adjusted or this should be a data section.
+  LSDASection =
+    Ctx->getCOFFSection(".gcc_except_table",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getReadOnly());
+
+  // Debug info.
+  DwarfAbbrevSection =
+    Ctx->getCOFFSection(".debug_abbrev",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfInfoSection =
+    Ctx->getCOFFSection(".debug_info",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfLineSection =
+    Ctx->getCOFFSection(".debug_line",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfFrameSection =
+    Ctx->getCOFFSection(".debug_frame",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfPubNamesSection =
+    Ctx->getCOFFSection(".debug_pubnames",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfPubTypesSection =
+    Ctx->getCOFFSection(".debug_pubtypes",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfStrSection =
+    Ctx->getCOFFSection(".debug_str",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfLocSection =
+    Ctx->getCOFFSection(".debug_loc",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfARangesSection =
+    Ctx->getCOFFSection(".debug_aranges",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfRangesSection =
+    Ctx->getCOFFSection(".debug_ranges",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+  DwarfMacroInfoSection =
+    Ctx->getCOFFSection(".debug_macinfo",
+                        COFF::IMAGE_SCN_MEM_DISCARDABLE |
+                        COFF::IMAGE_SCN_MEM_READ,
+                        SectionKind::getMetadata());
+
+  DrectveSection =
+    Ctx->getCOFFSection(".drectve",
+                        COFF::IMAGE_SCN_LNK_INFO,
+                        SectionKind::getMetadata());
+
+  PDataSection =
+    Ctx->getCOFFSection(".pdata",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ |
+                        COFF::IMAGE_SCN_MEM_WRITE,
+                        SectionKind::getDataRel());
+
+  XDataSection =
+    Ctx->getCOFFSection(".xdata",
+                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                        COFF::IMAGE_SCN_MEM_READ |
+                        COFF::IMAGE_SCN_MEM_WRITE,
+                        SectionKind::getDataRel());
+}
+
+void MCObjectFileInfo::InitMCObjectFileInfo(StringRef TT, Reloc::Model relocm,
+                                            CodeModel::Model cm,
+                                            MCContext &ctx) {
+  RelocM = relocm;
+  CMModel = cm;
+  Ctx = &ctx;
+
+  // Common.
+  CommDirectiveSupportsAlignment = true;
+  SupportsWeakOmittedEHFrame = true;
+  IsFunctionEHFrameSymbolPrivate = true;
+
+  PersonalityEncoding = LSDAEncoding = FDEEncoding = FDECFIEncoding =
+    TTypeEncoding = dwarf::DW_EH_PE_absptr;
+
+  EHFrameSection = 0;           // Created on demand.
+  CompactUnwindSection = 0;     // Used only by selected targets.
+
+  Triple T(TT);
+  Triple::ArchType Arch = T.getArch();
+  // FIXME: Checking for Arch here to filter out bogus triples such as
+  // cellspu-apple-darwin. Perhaps we should fix in Triple?
+  if ((Arch == Triple::x86 || Arch == Triple::x86_64 ||
+       Arch == Triple::arm || Arch == Triple::thumb ||
+       Arch == Triple::ppc || Arch == Triple::ppc64 ||
+       Arch == Triple::UnknownArch) &&
+      (T.isOSDarwin() || T.getEnvironment() == Triple::MachO)) {
+    Env = IsMachO;
+    InitMachOMCObjectFileInfo(T);
+  } else if ((Arch == Triple::x86 || Arch == Triple::x86_64) &&
+             (T.getOS() == Triple::MinGW32 || T.getOS() == Triple::Cygwin ||
+              T.getOS() == Triple::Win32)) {
+    Env = IsCOFF;
+    InitCOFFMCObjectFileInfo(T);
+  } else {
+    Env = IsELF;
+    InitELFMCObjectFileInfo(T);
+  }
+}
+
+void MCObjectFileInfo::InitEHFrameSection() {
+  if (Env == IsMachO)
+    EHFrameSection =
+      Ctx->getMachOSection("__TEXT", "__eh_frame",
+                           MCSectionMachO::S_COALESCED |
+                           MCSectionMachO::S_ATTR_NO_TOC |
+                           MCSectionMachO::S_ATTR_STRIP_STATIC_SYMS |
+                           MCSectionMachO::S_ATTR_LIVE_SUPPORT,
+                           SectionKind::getReadOnly());
+  else if (Env == IsELF)
+    EHFrameSection =
+      Ctx->getELFSection(".eh_frame", ELF::SHT_PROGBITS,
+                         ELF::SHF_ALLOC,
+                         SectionKind::getDataRel());
+  else
+    EHFrameSection =
+      Ctx->getCOFFSection(".eh_frame",
+                          COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                          COFF::IMAGE_SCN_MEM_READ |
+                          COFF::IMAGE_SCN_MEM_WRITE,
+                          SectionKind::getDataRel());
+}
diff --git a/contrib/llvm/lib/MC/MCObjectStreamer.cpp b/contrib/llvm/lib/MC/MCObjectStreamer.cpp
index 8635aac00302..a04ae0812a3e 100644
--- a/contrib/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCObjectStreamer.cpp
@@ -17,10 +17,10 @@
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/TargetAsmBackend.h"
+#include "llvm/MC/MCAsmBackend.h"
 using namespace llvm;
 
-MCObjectStreamer::MCObjectStreamer(MCContext &Context, TargetAsmBackend &TAB,
+MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB,
                                    raw_ostream &OS, MCCodeEmitter *Emitter_)
   : MCStreamer(Context),
     Assembler(new MCAssembler(Context, TAB,
@@ -30,7 +30,7 @@ MCObjectStreamer::MCObjectStreamer(MCContext &Context, TargetAsmBackend &TAB,
 {
 }
 
-MCObjectStreamer::MCObjectStreamer(MCContext &Context, TargetAsmBackend &TAB,
+MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB,
                                    raw_ostream &OS, MCCodeEmitter *Emitter_,
                                    MCAssembler *_Assembler)
   : MCStreamer(Context), Assembler(_Assembler), CurSectionData(0)
diff --git a/contrib/llvm/lib/MC/MCParser/AsmLexer.cpp b/contrib/llvm/lib/MC/MCParser/AsmLexer.cpp
index 0c1f8f0df774..c76052d66e00 100644
--- a/contrib/llvm/lib/MC/MCParser/AsmLexer.cpp
+++ b/contrib/llvm/lib/MC/MCParser/AsmLexer.cpp
@@ -24,6 +24,7 @@ using namespace llvm;
 AsmLexer::AsmLexer(const MCAsmInfo &_MAI) : MAI(_MAI)  {
   CurBuf = NULL;
   CurPtr = NULL;
+  isAtStartOfLine = true;
 }
 
 AsmLexer::~AsmLexer() {
@@ -146,7 +147,7 @@ AsmToken AsmLexer::LexLineComment() {
   // FIXME: This is broken if we happen to a comment at the end of a file, which
   // was .included, and which doesn't end with a newline.
   int CurChar = getNextChar();
-  while (CurChar != '\n' && CurChar != '\n' && CurChar != EOF)
+  while (CurChar != '\n' && CurChar != '\r' && CurChar != EOF)
     CurChar = getNextChar();
 
   if (CurChar == EOF)
@@ -334,6 +335,17 @@ StringRef AsmLexer::LexUntilEndOfStatement() {
   return StringRef(TokStart, CurPtr-TokStart);
 }
 
+StringRef AsmLexer::LexUntilEndOfLine() {
+  TokStart = CurPtr;
+
+  while (*CurPtr != '\n' &&
+         *CurPtr != '\r' &&
+         (*CurPtr != 0 || CurPtr != CurBuf->getBufferEnd())) {
+    ++CurPtr;
+  }
+  return StringRef(TokStart, CurPtr-TokStart);
+}
+
 bool AsmLexer::isAtStartOfComment(char Char) {
   // FIXME: This won't work for multi-character comment indicators like "//".
   return Char == *MAI.getCommentString();
@@ -349,14 +361,29 @@ AsmToken AsmLexer::LexToken() {
   // This always consumes at least one character.
   int CurChar = getNextChar();
 
-  if (isAtStartOfComment(CurChar))
+  if (isAtStartOfComment(CurChar)) {
+    // If this comment starts with a '#', then return the Hash token and let
+    // the assembler parser see if it can be parsed as a cpp line filename
+    // comment. We do this only if we are at the start of a line.
+    if (CurChar == '#' && isAtStartOfLine)
+      return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
+    isAtStartOfLine = true;
     return LexLineComment();
+  }
   if (isAtStatementSeparator(TokStart)) {
     CurPtr += strlen(MAI.getSeparatorString()) - 1;
     return AsmToken(AsmToken::EndOfStatement,
                     StringRef(TokStart, strlen(MAI.getSeparatorString())));
   }
 
+  // If we're missing a newline at EOF, make sure we still get an
+  // EndOfStatement token before the Eof token.
+  if (CurChar == EOF && !isAtStartOfLine) {
+    isAtStartOfLine = true;
+    return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
+  }
+
+  isAtStartOfLine = false;
   switch (CurChar) {
   default:
     // Handle identifier: [a-zA-Z_.][a-zA-Z0-9_$.@]*
@@ -373,6 +400,7 @@ AsmToken AsmLexer::LexToken() {
     return LexToken();
   case '\n': // FALL THROUGH.
   case '\r':
+    isAtStartOfLine = true;
     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
   case ':': return AsmToken(AsmToken::Colon, StringRef(TokStart, 1));
   case '+': return AsmToken(AsmToken::Plus, StringRef(TokStart, 1));
diff --git a/contrib/llvm/lib/MC/MCParser/AsmParser.cpp b/contrib/llvm/lib/MC/MCParser/AsmParser.cpp
index 0c181f39611e..16487579abfc 100644
--- a/contrib/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/contrib/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -18,22 +18,22 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCParser/AsmCond.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/MC/MCDwarf.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetAsmInfo.h"
-#include "llvm/Target/TargetAsmParser.h"
 #include <cctype>
 #include <vector>
 using namespace llvm;
@@ -87,6 +87,8 @@ class AsmParser : public MCAsmParser {
   MCStreamer &Out;
   const MCAsmInfo &MAI;
   SourceMgr &SrcMgr;
+  SourceMgr::DiagHandlerTy SavedDiagHandler;
+  void *SavedDiagContext;
   MCAsmParserExtension *GenericParser;
   MCAsmParserExtension *PlatformParser;
 
@@ -115,8 +117,13 @@ class AsmParser : public MCAsmParser {
   /// Flag tracking whether any errors have been encountered.
   unsigned HadError : 1;
 
+  /// The values from the last parsed cpp hash file line comment if any.
+  StringRef CppHashFilename;
+  int64_t CppHashLineNumber;
+  SMLoc CppHashLoc;
+
 public:
-  AsmParser(const Target &T, SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
+  AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
             const MCAsmInfo &MAI);
   ~AsmParser();
 
@@ -153,6 +160,8 @@ class AsmParser : public MCAsmParser {
   void CheckForValidSection();
 
   bool ParseStatement();
+  void EatToEndOfLine();
+  bool ParseCppHashLineFilenameComment(const SMLoc &L);
 
   bool HandleMacroEntry(StringRef Name, SMLoc NameLoc, const Macro *M);
   bool expandMacro(SmallString<256> &Buf, StringRef Body,
@@ -166,6 +175,7 @@ class AsmParser : public MCAsmParser {
                     bool ShowLine = true) const {
     SrcMgr.PrintMessage(Loc, Msg, Type, ShowLine);
   }
+  static void DiagHandler(const SMDiagnostic &Diag, void *Context);
 
   /// EnterIncludeFile - Enter the specified file. This returns true on failure.
   bool EnterIncludeFile(const std::string &Filename);
@@ -338,11 +348,16 @@ extern MCAsmParserExtension *createCOFFAsmParser();
 
 enum { DEFAULT_ADDRSPACE = 0 };
 
-AsmParser::AsmParser(const Target &T, SourceMgr &_SM, MCContext &_Ctx,
+AsmParser::AsmParser(SourceMgr &_SM, MCContext &_Ctx,
                      MCStreamer &_Out, const MCAsmInfo &_MAI)
   : Lexer(_MAI), Ctx(_Ctx), Out(_Out), MAI(_MAI), SrcMgr(_SM),
     GenericParser(new GenericAsmParser), PlatformParser(0),
-    CurBuffer(0), MacrosEnabled(true) {
+    CurBuffer(0), MacrosEnabled(true), CppHashLineNumber(0) {
+  // Save the old handler.
+  SavedDiagHandler = SrcMgr.getDiagHandler();
+  SavedDiagContext = SrcMgr.getDiagContext();
+  // Set our own handler which calls the saved handler.
+  SrcMgr.setDiagHandler(DiagHandler, this);
   Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer));
 
   // Initialize the generic parser.
@@ -740,9 +755,12 @@ AsmParser::ApplyModifierToExpr(const MCExpr *E,
 
 /// ParseExpression - Parse an expression and return it.
 ///
-///  expr ::= expr +,- expr          -> lowest.
-///  expr ::= expr |,^,&,! expr      -> middle.
-///  expr ::= expr *,/,%,<<,>> expr  -> highest.
+///  expr ::= expr &&,|| expr               -> lowest.
+///  expr ::= expr |,^,&,! expr
+///  expr ::= expr ==,!=,<>,<,<=,>,>= expr
+///  expr ::= expr <<,>> expr
+///  expr ::= expr +,- expr
+///  expr ::= expr *,/,% expr               -> highest.
 ///  expr ::= primaryexpr
 ///
 bool AsmParser::ParseExpression(const MCExpr *&Res, SMLoc &EndLoc) {
@@ -809,7 +827,7 @@ static unsigned getBinOpPrecedence(AsmToken::TokenKind K,
   default:
     return 0;    // not a binop.
 
-    // Lowest Precedence: &&, ||, @
+    // Lowest Precedence: &&, ||
   case AsmToken::AmpAmp:
     Kind = MCBinaryExpr::LAnd;
     return 1;
@@ -852,30 +870,32 @@ static unsigned getBinOpPrecedence(AsmToken::TokenKind K,
     Kind = MCBinaryExpr::GTE;
     return 3;
 
+    // Intermediate Precedence: <<, >>
+  case AsmToken::LessLess:
+    Kind = MCBinaryExpr::Shl;
+    return 4;
+  case AsmToken::GreaterGreater:
+    Kind = MCBinaryExpr::Shr;
+    return 4;
+
     // High Intermediate Precedence: +, -
   case AsmToken::Plus:
     Kind = MCBinaryExpr::Add;
-    return 4;
+    return 5;
   case AsmToken::Minus:
     Kind = MCBinaryExpr::Sub;
-    return 4;
+    return 5;
 
-    // Highest Precedence: *, /, %, <<, >>
+    // Highest Precedence: *, /, %
   case AsmToken::Star:
     Kind = MCBinaryExpr::Mul;
-    return 5;
+    return 6;
   case AsmToken::Slash:
     Kind = MCBinaryExpr::Div;
-    return 5;
+    return 6;
   case AsmToken::Percent:
     Kind = MCBinaryExpr::Mod;
-    return 5;
-  case AsmToken::LessLess:
-    Kind = MCBinaryExpr::Shl;
-    return 5;
-  case AsmToken::GreaterGreater:
-    Kind = MCBinaryExpr::Shr;
-    return 5;
+    return 6;
   }
 }
 
@@ -932,10 +952,8 @@ bool AsmParser::ParseStatement() {
   StringRef IDVal;
   int64_t LocalLabelVal = -1;
   // A full line comment is a '#' as the first token.
-  if (Lexer.is(AsmToken::Hash)) {
-    EatToEndOfStatement();
-    return false;
-  }
+  if (Lexer.is(AsmToken::Hash))
+    return ParseCppHashLineFilenameComment(IDLoc);
 
   // Allow an integer followed by a ':' as a directional local label.
   if (Lexer.is(AsmToken::Integer)) {
@@ -1117,15 +1135,8 @@ bool AsmParser::ParseStatement() {
 
     if (IDVal == ".globl" || IDVal == ".global")
       return ParseDirectiveSymbolAttribute(MCSA_Global);
-    // ELF only? Should it be here?
-    if (IDVal == ".local")
-      return ParseDirectiveSymbolAttribute(MCSA_Local);
-    if (IDVal == ".hidden")
-      return ParseDirectiveSymbolAttribute(MCSA_Hidden);
     if (IDVal == ".indirect_symbol")
       return ParseDirectiveSymbolAttribute(MCSA_IndirectSymbol);
-    if (IDVal == ".internal")
-      return ParseDirectiveSymbolAttribute(MCSA_Internal);
     if (IDVal == ".lazy_reference")
       return ParseDirectiveSymbolAttribute(MCSA_LazyReference);
     if (IDVal == ".no_dead_strip")
@@ -1134,12 +1145,8 @@ bool AsmParser::ParseStatement() {
       return ParseDirectiveSymbolAttribute(MCSA_SymbolResolver);
     if (IDVal == ".private_extern")
       return ParseDirectiveSymbolAttribute(MCSA_PrivateExtern);
-    if (IDVal == ".protected")
-      return ParseDirectiveSymbolAttribute(MCSA_Protected);
     if (IDVal == ".reference")
       return ParseDirectiveSymbolAttribute(MCSA_Reference);
-    if (IDVal == ".weak")
-      return ParseDirectiveSymbolAttribute(MCSA_Weak);
     if (IDVal == ".weak_definition")
       return ParseDirectiveSymbolAttribute(MCSA_WeakDefinition);
     if (IDVal == ".weak_reference")
@@ -1157,7 +1164,7 @@ bool AsmParser::ParseStatement() {
     if (IDVal == ".include")
       return ParseDirectiveInclude();
 
-    if (IDVal == ".code16" || IDVal == ".code32" || IDVal == ".code64")
+    if (IDVal == ".code16")
       return TokError(Twine(IDVal) + " not supported yet");
 
     // Look up the handler in the handler table.
@@ -1215,6 +1222,108 @@ bool AsmParser::ParseStatement() {
   return false;
 }
 
+/// EatToEndOfLine uses the Lexer to eat the characters to the end of the line
+/// since they may not be able to be tokenized to get to the end of line token.
+void AsmParser::EatToEndOfLine() {
+  if (!Lexer.is(AsmToken::EndOfStatement))
+    Lexer.LexUntilEndOfLine();
+ // Eat EOL.
+ Lex();
+}
+
+/// ParseCppHashLineFilenameComment as this:
+///   ::= # number "filename"
+/// or just as a full line comment if it doesn't have a number and a string.
+bool AsmParser::ParseCppHashLineFilenameComment(const SMLoc &L) {
+  Lex(); // Eat the hash token.
+
+  if (getLexer().isNot(AsmToken::Integer)) {
+    // Consume the line since in cases it is not a well-formed line directive,
+    // as if were simply a full line comment.
+    EatToEndOfLine();
+    return false;
+  }
+
+  int64_t LineNumber = getTok().getIntVal();
+  Lex();
+
+  if (getLexer().isNot(AsmToken::String)) {
+    EatToEndOfLine();
+    return false;
+  }
+
+  StringRef Filename = getTok().getString();
+  // Get rid of the enclosing quotes.
+  Filename = Filename.substr(1, Filename.size()-2);
+
+  // Save the SMLoc, Filename and LineNumber for later use by diagnostics.
+  CppHashLoc = L;
+  CppHashFilename = Filename;
+  CppHashLineNumber = LineNumber;
+
+  // Ignore any trailing characters, they're just comment.
+  EatToEndOfLine();
+  return false;
+}
+
+/// DiagHandler - will use the the last parsed cpp hash line filename comment
+/// for the Filename and LineNo if any in the diagnostic.
+void AsmParser::DiagHandler(const SMDiagnostic &Diag, void *Context) {
+  const AsmParser *Parser = static_cast<const AsmParser*>(Context);
+  raw_ostream &OS = errs();
+
+  const SourceMgr &DiagSrcMgr = *Diag.getSourceMgr();
+  const SMLoc &DiagLoc = Diag.getLoc();
+  int DiagBuf = DiagSrcMgr.FindBufferContainingLoc(DiagLoc);
+  int CppHashBuf = Parser->SrcMgr.FindBufferContainingLoc(Parser->CppHashLoc);
+
+  // Like SourceMgr::PrintMessage() we need to print the include stack if any
+  // before printing the message.
+  int DiagCurBuffer = DiagSrcMgr.FindBufferContainingLoc(DiagLoc);
+  if (!Parser->SavedDiagHandler && DiagCurBuffer > 0) {
+     SMLoc ParentIncludeLoc = DiagSrcMgr.getParentIncludeLoc(DiagCurBuffer);
+     DiagSrcMgr.PrintIncludeStack(ParentIncludeLoc, OS);
+  }
+
+  // If we have not parsed a cpp hash line filename comment or the source 
+  // manager changed or buffer changed (like in a nested include) then just
+  // print the normal diagnostic using its Filename and LineNo.
+  if (!Parser->CppHashLineNumber ||
+      &DiagSrcMgr != &Parser->SrcMgr ||
+      DiagBuf != CppHashBuf) {
+    if (Parser->SavedDiagHandler)
+      Parser->SavedDiagHandler(Diag, Parser->SavedDiagContext);
+    else
+      Diag.Print(0, OS);
+    return;
+  }
+
+  // Use the CppHashFilename and calculate a line number based on the 
+  // CppHashLoc and CppHashLineNumber relative to this Diag's SMLoc for
+  // the diagnostic.
+  const std::string Filename = Parser->CppHashFilename;
+
+  int DiagLocLineNo = DiagSrcMgr.FindLineNumber(DiagLoc, DiagBuf);
+  int CppHashLocLineNo =
+      Parser->SrcMgr.FindLineNumber(Parser->CppHashLoc, CppHashBuf);
+  int LineNo = Parser->CppHashLineNumber - 1 +
+               (DiagLocLineNo - CppHashLocLineNo);
+
+  SMDiagnostic NewDiag(*Diag.getSourceMgr(),
+                       Diag.getLoc(),
+                       Filename,
+                       LineNo,
+                       Diag.getColumnNo(),
+                       Diag.getMessage(),
+                       Diag.getLineContents(),
+                       Diag.getShowLine());
+
+  if (Parser->SavedDiagHandler)
+    Parser->SavedDiagHandler(NewDiag, Parser->SavedDiagContext);
+  else
+    NewDiag.Print(0, OS);
+}
+
 bool AsmParser::expandMacro(SmallString<256> &Buf, StringRef Body,
                             const std::vector<StringRef> &Parameters,
                             const std::vector<std::vector<AsmToken> > &A,
@@ -1923,12 +2032,17 @@ bool AsmParser::ParseDirectiveSymbolAttribute(MCSymbolAttr Attr) {
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
     for (;;) {
       StringRef Name;
+      SMLoc Loc = getTok().getLoc();
 
       if (ParseIdentifier(Name))
-        return TokError("expected identifier in directive");
+        return Error(Loc, "expected identifier in directive");
 
       MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
 
+      // Assembler local symbols don't make any sense here. Complain loudly.
+      if (Sym->isTemporary())
+        return Error(Loc, "non-local symbol required in directive");
+
       getStreamer().EmitSymbolAttribute(Sym, Attr);
 
       if (getLexer().is(AsmToken::EndOfStatement))
@@ -2416,7 +2530,7 @@ bool GenericAsmParser::ParseRegisterOrRegisterNumber(int64_t &Register,
     if (getParser().getTargetParser().ParseRegister(RegNo, DirectiveLoc,
       DirectiveLoc))
       return true;
-    Register = getContext().getTargetAsmInfo().getDwarfRegNum(RegNo, true);
+    Register = getContext().getRegisterInfo().getDwarfRegNum(RegNo, true);
   } else
     return getParser().ParseAbsoluteExpression(Register);
 
@@ -2724,8 +2838,8 @@ bool GenericAsmParser::ParseDirectiveLEB128(StringRef DirName, SMLoc) {
 
 
 /// \brief Create an MCAsmParser instance.
-MCAsmParser *llvm::createMCAsmParser(const Target &T, SourceMgr &SM,
+MCAsmParser *llvm::createMCAsmParser(SourceMgr &SM,
                                      MCContext &C, MCStreamer &Out,
                                      const MCAsmInfo &MAI) {
-  return new AsmParser(T, SM, C, Out, MAI);
+  return new AsmParser(SM, C, Out, MAI);
 }
diff --git a/contrib/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/contrib/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index 66ad384c7db2..185b5168bd6c 100644
--- a/contrib/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/contrib/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -8,15 +8,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/Target/TargetAsmInfo.h"
-#include "llvm/Target/TargetAsmParser.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/COFF.h"
 using namespace llvm;
 
@@ -72,6 +73,7 @@ class COFFAsmParser : public MCAsmParserExtension {
                                                               ".seh_pushframe");
     AddDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveEndProlog>(
                                                             ".seh_endprologue");
+    AddDirectiveHandler<&COFFAsmParser::ParseDirectiveSymbolAttribute>(".weak");
   }
 
   bool ParseSectionDirectiveText(StringRef, SMLoc) {
@@ -118,12 +120,44 @@ class COFFAsmParser : public MCAsmParserExtension {
 
   bool ParseAtUnwindOrAtExcept(bool &unwind, bool &except);
   bool ParseSEHRegisterNumber(unsigned &RegNo);
+  bool ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc);
 public:
   COFFAsmParser() {}
 };
 
 } // end annonomous namespace.
 
+/// ParseDirectiveSymbolAttribute
+///  ::= { ".weak", ... } [ identifier ( , identifier )* ]
+bool COFFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
+  MCSymbolAttr Attr = StringSwitch<MCSymbolAttr>(Directive)
+    .Case(".weak", MCSA_Weak)
+    .Default(MCSA_Invalid);
+  assert(Attr != MCSA_Invalid && "unexpected symbol attribute directive!");
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    for (;;) {
+      StringRef Name;
+
+      if (getParser().ParseIdentifier(Name))
+        return TokError("expected identifier in directive");
+
+      MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+
+      getStreamer().EmitSymbolAttribute(Sym, Attr);
+
+      if (getLexer().is(AsmToken::EndOfStatement))
+        break;
+
+      if (getLexer().isNot(AsmToken::Comma))
+        return TokError("unexpected token in directive");
+      Lex();
+    }
+  }
+
+  Lex();
+  return false;
+}
+
 bool COFFAsmParser::ParseSectionSwitch(StringRef Section,
                                        unsigned Characteristics,
                                        SectionKind Kind) {
@@ -401,12 +435,16 @@ bool COFFAsmParser::ParseAtUnwindOrAtExcept(bool &unwind, bool &except) {
 bool COFFAsmParser::ParseSEHRegisterNumber(unsigned &RegNo) {
   SMLoc startLoc = getLexer().getLoc();
   if (getLexer().is(AsmToken::Percent)) {
-    const TargetAsmInfo &TAI = getContext().getTargetAsmInfo();
+    const MCRegisterInfo &MRI = getContext().getRegisterInfo();
     SMLoc endLoc;
     unsigned LLVMRegNo;
     if (getParser().getTargetParser().ParseRegister(LLVMRegNo,startLoc,endLoc))
       return true;
 
+#if 0
+    // FIXME: TargetAsmInfo::getCalleeSavedRegs() commits a serious layering
+    // violation so this validation code is disabled.
+
     // Check that this is a non-volatile register.
     const unsigned *NVRegs = TAI.getCalleeSavedRegs();
     unsigned i;
@@ -415,8 +453,9 @@ bool COFFAsmParser::ParseSEHRegisterNumber(unsigned &RegNo) {
         break;
     if (NVRegs[i] == 0)
       return Error(startLoc, "expected non-volatile register");
+#endif
 
-    int SEHRegNo = TAI.getSEHRegNum(LLVMRegNo);
+    int SEHRegNo = MRI.getSEHRegNum(LLVMRegNo);
     if (SEHRegNo < 0)
       return Error(startLoc,"register can't be represented in SEH unwind info");
     RegNo = SEHRegNo;
diff --git a/contrib/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/contrib/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index dcf689a6f0e7..d89112645cd8 100644
--- a/contrib/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/contrib/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -47,12 +47,17 @@ class ELFAsmParser : public MCAsmParserExtension {
     AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveRoData>(".rodata");
     AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveTData>(".tdata");
     AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveTBSS>(".tbss");
-    AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveDataRel>(".data.rel");
-    AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveDataRelRo>(".data.rel.ro");
-    AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveDataRelRoLocal>(".data.rel.ro.local");
-    AddDirectiveHandler<&ELFAsmParser::ParseSectionDirectiveEhFrame>(".eh_frame");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseSectionDirectiveDataRel>(".data.rel");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseSectionDirectiveDataRelRo>(".data.rel.ro");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseSectionDirectiveDataRelRoLocal>(".data.rel.ro.local");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseSectionDirectiveEhFrame>(".eh_frame");
     AddDirectiveHandler<&ELFAsmParser::ParseDirectiveSection>(".section");
-    AddDirectiveHandler<&ELFAsmParser::ParseDirectivePushSection>(".pushsection");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseDirectivePushSection>(".pushsection");
     AddDirectiveHandler<&ELFAsmParser::ParseDirectivePopSection>(".popsection");
     AddDirectiveHandler<&ELFAsmParser::ParseDirectiveSize>(".size");
     AddDirectiveHandler<&ELFAsmParser::ParseDirectivePrevious>(".previous");
@@ -60,6 +65,14 @@ class ELFAsmParser : public MCAsmParserExtension {
     AddDirectiveHandler<&ELFAsmParser::ParseDirectiveIdent>(".ident");
     AddDirectiveHandler<&ELFAsmParser::ParseDirectiveSymver>(".symver");
     AddDirectiveHandler<&ELFAsmParser::ParseDirectiveWeakref>(".weakref");
+    AddDirectiveHandler<&ELFAsmParser::ParseDirectiveSymbolAttribute>(".weak");
+    AddDirectiveHandler<&ELFAsmParser::ParseDirectiveSymbolAttribute>(".local");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseDirectiveSymbolAttribute>(".protected");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseDirectiveSymbolAttribute>(".internal");
+    AddDirectiveHandler<
+      &ELFAsmParser::ParseDirectiveSymbolAttribute>(".hidden");
   }
 
   // FIXME: Part of this logic is duplicated in the MCELFStreamer. What is
@@ -129,6 +142,7 @@ class ELFAsmParser : public MCAsmParserExtension {
   bool ParseDirectiveIdent(StringRef, SMLoc);
   bool ParseDirectiveSymver(StringRef, SMLoc);
   bool ParseDirectiveWeakref(StringRef, SMLoc);
+  bool ParseDirectiveSymbolAttribute(StringRef, SMLoc);
 
 private:
   bool ParseSectionName(StringRef &SectionName);
@@ -136,6 +150,41 @@ class ELFAsmParser : public MCAsmParserExtension {
 
 }
 
+/// ParseDirectiveSymbolAttribute
+///  ::= { ".local", ".weak", ... } [ identifier ( , identifier )* ]
+bool ELFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
+  MCSymbolAttr Attr = StringSwitch<MCSymbolAttr>(Directive)
+    .Case(".weak", MCSA_Weak)
+    .Case(".local", MCSA_Local)
+    .Case(".hidden", MCSA_Hidden)
+    .Case(".internal", MCSA_Internal)
+    .Case(".protected", MCSA_Protected)
+    .Default(MCSA_Invalid);
+  assert(Attr != MCSA_Invalid && "unexpected symbol attribute directive!");
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    for (;;) {
+      StringRef Name;
+
+      if (getParser().ParseIdentifier(Name))
+        return TokError("expected identifier in directive");
+
+      MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+
+      getStreamer().EmitSymbolAttribute(Sym, Attr);
+
+      if (getLexer().is(AsmToken::EndOfStatement))
+        break;
+
+      if (getLexer().isNot(AsmToken::Comma))
+        return TokError("unexpected token in directive");
+      Lex();
+    }
+  }
+
+  Lex();
+  return false;
+}
+
 bool ELFAsmParser::ParseSectionSwitch(StringRef Section, unsigned Type,
                                       unsigned Flags, SectionKind Kind) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
diff --git a/contrib/llvm/lib/MC/MCParser/MCAsmParser.cpp b/contrib/llvm/lib/MC/MCParser/MCAsmParser.cpp
index 4030e41036aa..5239ec753e77 100644
--- a/contrib/llvm/lib/MC/MCParser/MCAsmParser.cpp
+++ b/contrib/llvm/lib/MC/MCParser/MCAsmParser.cpp
@@ -8,13 +8,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCParser/MCAsmParser.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Target/TargetAsmParser.h"
+#include "llvm/ADT/Twine.h"
 using namespace llvm;
 
 MCAsmParser::MCAsmParser() : TargetParser(0), ShowParsedOperands(0) {
@@ -23,7 +23,7 @@ MCAsmParser::MCAsmParser() : TargetParser(0), ShowParsedOperands(0) {
 MCAsmParser::~MCAsmParser() {
 }
 
-void MCAsmParser::setTargetParser(TargetAsmParser &P) {
+void MCAsmParser::setTargetParser(MCTargetAsmParser &P) {
   assert(!TargetParser && "Target parser is already initialized!");
   TargetParser = &P;
   TargetParser->Initialize(*this);
diff --git a/contrib/llvm/lib/MC/MCParser/TargetAsmParser.cpp b/contrib/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp
similarity index 64%
rename from contrib/llvm/lib/MC/MCParser/TargetAsmParser.cpp
rename to contrib/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp
index 512f6b044911..6fb1ba4216f8 100644
--- a/contrib/llvm/lib/MC/MCParser/TargetAsmParser.cpp
+++ b/contrib/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp
@@ -1,4 +1,4 @@
-//===-- TargetAsmParser.cpp - Target Assembly Parser -----------------------==//
+//===-- MCTargetAsmParser.cpp - Target Assembly Parser ---------------------==//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,13 +7,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmParser.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 using namespace llvm;
 
-TargetAsmParser::TargetAsmParser()
+MCTargetAsmParser::MCTargetAsmParser()
   : AvailableFeatures(0)
 {
 }
 
-TargetAsmParser::~TargetAsmParser() {
+MCTargetAsmParser::~MCTargetAsmParser() {
 }
diff --git a/contrib/llvm/lib/MC/MCPureStreamer.cpp b/contrib/llvm/lib/MC/MCPureStreamer.cpp
index 6098e6b8f38b..086c9229bcdc 100644
--- a/contrib/llvm/lib/MC/MCPureStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCPureStreamer.cpp
@@ -28,7 +28,7 @@ class MCPureStreamer : public MCObjectStreamer {
   virtual void EmitInstToData(const MCInst &Inst);
 
 public:
-  MCPureStreamer(MCContext &Context, TargetAsmBackend &TAB,
+  MCPureStreamer(MCContext &Context, MCAsmBackend &TAB,
                  raw_ostream &OS, MCCodeEmitter *Emitter)
     : MCObjectStreamer(Context, TAB, OS, Emitter) {}
 
@@ -86,7 +86,8 @@ class MCPureStreamer : public MCObjectStreamer {
   virtual void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
     report_fatal_error("unsupported directive in pure streamer");
   }
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment) {
     report_fatal_error("unsupported directive in pure streamer");
   }
   virtual void EmitFileDirective(StringRef Filename) {
@@ -228,7 +229,7 @@ void MCPureStreamer::Finish() {
   this->MCObjectStreamer::Finish();
 }
 
-MCStreamer *llvm::createPureStreamer(MCContext &Context, TargetAsmBackend &TAB,
+MCStreamer *llvm::createPureStreamer(MCContext &Context, MCAsmBackend &MAB,
                                      raw_ostream &OS, MCCodeEmitter *CE) {
-  return new MCPureStreamer(Context, TAB, OS, CE);
+  return new MCPureStreamer(Context, MAB, OS, CE);
 }
diff --git a/contrib/llvm/lib/MC/MCStreamer.cpp b/contrib/llvm/lib/MC/MCStreamer.cpp
index 6e96b78e315b..3afa22b0d0be 100644
--- a/contrib/llvm/lib/MC/MCStreamer.cpp
+++ b/contrib/llvm/lib/MC/MCStreamer.cpp
@@ -16,13 +16,17 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
 #include <cstdlib>
 using namespace llvm;
 
 MCStreamer::MCStreamer(MCContext &Ctx) : Context(Ctx), EmitEHFrame(true),
                                          EmitDebugFrame(false),
-                                         CurrentW64UnwindInfo(0) {
+                                         CurrentW64UnwindInfo(0),
+                                         LastSymbol(0),
+                                         UniqueCodeBeginSuffix(0),
+                                         UniqueDataBeginSuffix(0) {
   const MCSection *section = NULL;
   SectionStack.push_back(std::make_pair(section, section));
 }
@@ -171,10 +175,94 @@ void MCStreamer::EmitLabel(MCSymbol *Symbol) {
   assert(!Symbol->isVariable() && "Cannot emit a variable symbol!");
   assert(getCurrentSection() && "Cannot emit before setting section!");
   Symbol->setSection(*getCurrentSection());
+  LastSymbol = Symbol;
+}
 
-  StringRef Prefix = getContext().getAsmInfo().getPrivateGlobalPrefix();
-  if (!Symbol->getName().startswith(Prefix))
-    LastNonPrivate = Symbol;
+void MCStreamer::EmitDataRegion() {
+  if (RegionIndicator == Data) return;
+
+  MCContext &Context = getContext();
+  const MCAsmInfo &MAI = Context.getAsmInfo();
+  if (!MAI.getSupportsDataRegions()) return;
+
+  // Generate a unique symbol name.
+  MCSymbol *NewSym = Context.GetOrCreateSymbol(
+      Twine(MAI.getDataBeginLabelName()) +
+        utostr(UniqueDataBeginSuffix++));
+  EmitLabel(NewSym);
+
+  RegionIndicator = Data;
+}
+
+void MCStreamer::EmitCodeRegion() {
+  if (RegionIndicator == Code) return;
+
+  MCContext &Context = getContext();
+  const MCAsmInfo &MAI = Context.getAsmInfo();
+  if (!MAI.getSupportsDataRegions()) return;
+
+  // Generate a unique symbol name.
+  MCSymbol *NewSym = Context.GetOrCreateSymbol(
+      Twine(MAI.getCodeBeginLabelName()) +
+        utostr(UniqueCodeBeginSuffix++));
+  EmitLabel(NewSym);
+
+  RegionIndicator = Code;
+}
+
+void MCStreamer::EmitJumpTable8Region() {
+  if (RegionIndicator == JumpTable8) return;
+
+  MCContext &Context = getContext();
+  const MCAsmInfo &MAI = Context.getAsmInfo();
+  if (!MAI.getSupportsDataRegions()) return;
+
+  // Generate a unique symbol name.
+  MCSymbol *NewSym = Context.GetOrCreateSymbol(
+      Twine(MAI.getJumpTable8BeginLabelName()) +
+        utostr(UniqueDataBeginSuffix++));
+  EmitLabel(NewSym);
+
+  RegionIndicator = JumpTable8;
+}
+
+void MCStreamer::EmitJumpTable16Region() {
+  if (RegionIndicator == JumpTable16) return;
+
+  MCContext &Context = getContext();
+  const MCAsmInfo &MAI = Context.getAsmInfo();
+  if (!MAI.getSupportsDataRegions()) return;
+
+  // Generate a unique symbol name.
+  MCSymbol *NewSym = Context.GetOrCreateSymbol(
+      Twine(MAI.getJumpTable16BeginLabelName()) +
+        utostr(UniqueDataBeginSuffix++));
+  EmitLabel(NewSym);
+
+  RegionIndicator = JumpTable16;
+}
+
+
+void MCStreamer::EmitJumpTable32Region() {
+  if (RegionIndicator == JumpTable32) return;
+
+  MCContext &Context = getContext();
+  const MCAsmInfo &MAI = Context.getAsmInfo();
+  if (!MAI.getSupportsDataRegions()) return;
+
+  // Generate a unique symbol name.
+  MCSymbol *NewSym = Context.GetOrCreateSymbol(
+      Twine(MAI.getJumpTable32BeginLabelName()) +
+        utostr(UniqueDataBeginSuffix++));
+  EmitLabel(NewSym);
+
+  RegionIndicator = JumpTable32;
+}
+
+void MCStreamer::EmitCompactUnwindEncoding(uint32_t CompactUnwindEncoding) {
+  EnsureValidFrame();
+  MCDwarfFrameInfo *CurFrame = getCurrentFrameInfo();
+  CurFrame->CompactUnwindEncoding = CompactUnwindEncoding;
 }
 
 void MCStreamer::EmitCFISections(bool EH, bool Debug) {
@@ -187,11 +275,22 @@ void MCStreamer::EmitCFIStartProc() {
   MCDwarfFrameInfo *CurFrame = getCurrentFrameInfo();
   if (CurFrame && !CurFrame->End)
     report_fatal_error("Starting a frame before finishing the previous one!");
+
   MCDwarfFrameInfo Frame;
-  Frame.Begin = getContext().CreateTempSymbol();
-  Frame.Function = LastNonPrivate;
-  EmitLabel(Frame.Begin);
+  Frame.Function = LastSymbol;
+
+  // If the function is externally visible, we need to create a local
+  // symbol to avoid relocations.
+  StringRef Prefix = getContext().getAsmInfo().getPrivateGlobalPrefix();
+  if (LastSymbol && LastSymbol->getName().startswith(Prefix)) {
+    Frame.Begin = LastSymbol;
+  } else {
+    Frame.Begin = getContext().CreateTempSymbol();
+    EmitLabel(Frame.Begin);
+  }
+
   FrameInfos.push_back(Frame);
+  RegionIndicator = Code;
 }
 
 void MCStreamer::EmitCFIEndProc() {
diff --git a/contrib/llvm/lib/Target/TargetAsmLexer.cpp b/contrib/llvm/lib/MC/MCTargetAsmLexer.cpp
similarity index 56%
rename from contrib/llvm/lib/Target/TargetAsmLexer.cpp
rename to contrib/llvm/lib/MC/MCTargetAsmLexer.cpp
index d4893ff2521e..c01c914cecd2 100644
--- a/contrib/llvm/lib/Target/TargetAsmLexer.cpp
+++ b/contrib/llvm/lib/MC/MCTargetAsmLexer.cpp
@@ -1,4 +1,4 @@
-//===-- llvm/Target/TargetAsmLexer.cpp - Target Assembly Lexer ------------===//
+//===-- llvm/MC/MCTargetAsmLexer.cpp - Target Assembly Lexer --------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,8 +7,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmLexer.h"
+#include "llvm/MC/MCTargetAsmLexer.h"
 using namespace llvm;
 
-TargetAsmLexer::TargetAsmLexer(const Target &T) : TheTarget(T), Lexer(NULL) {}
-TargetAsmLexer::~TargetAsmLexer() {}
+MCTargetAsmLexer::MCTargetAsmLexer(const Target &T)
+  : TheTarget(T), Lexer(NULL) {
+}
+MCTargetAsmLexer::~MCTargetAsmLexer() {}
diff --git a/contrib/llvm/lib/MC/MCWin64EH.cpp b/contrib/llvm/lib/MC/MCWin64EH.cpp
index e698384a49f1..79e66fcdf2c0 100644
--- a/contrib/llvm/lib/MC/MCWin64EH.cpp
+++ b/contrib/llvm/lib/MC/MCWin64EH.cpp
@@ -10,10 +10,11 @@
 #include "llvm/MC/MCWin64EH.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/Target/TargetAsmInfo.h"
+#include "llvm/ADT/Twine.h"
 
 namespace llvm {
 
@@ -220,14 +221,36 @@ StringRef MCWin64EHUnwindEmitter::GetSectionSuffix(const MCSymbol *func) {
   return "";
 }
 
+static const MCSection *getWin64EHTableSection(StringRef suffix,
+                                               MCContext &context) {
+  if (suffix == "")
+    return context.getObjectFileInfo()->getXDataSection();
+
+  return context.getCOFFSection((".xdata"+suffix).str(),
+                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                                COFF::IMAGE_SCN_MEM_READ |
+                                COFF::IMAGE_SCN_MEM_WRITE,
+                                SectionKind::getDataRel());
+}
+
+static const MCSection *getWin64EHFuncTableSection(StringRef suffix,
+                                                   MCContext &context) {
+  if (suffix == "")
+    return context.getObjectFileInfo()->getPDataSection();
+  return context.getCOFFSection((".pdata"+suffix).str(),
+                                COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                                COFF::IMAGE_SCN_MEM_READ |
+                                COFF::IMAGE_SCN_MEM_WRITE,
+                                SectionKind::getDataRel());
+}
+
 void MCWin64EHUnwindEmitter::EmitUnwindInfo(MCStreamer &streamer,
                                             MCWin64EHUnwindInfo *info) {
   // Switch sections (the static function above is meant to be called from
   // here and from Emit().
   MCContext &context = streamer.getContext();
-  const TargetAsmInfo &TAI = context.getTargetAsmInfo();
   const MCSection *xdataSect =
-    TAI.getWin64EHTableSection(GetSectionSuffix(info->Function));
+    getWin64EHTableSection(GetSectionSuffix(info->Function), context);
   streamer.SwitchSection(xdataSect);
 
   llvm::EmitUnwindInfo(streamer, info);
@@ -236,11 +259,10 @@ void MCWin64EHUnwindEmitter::EmitUnwindInfo(MCStreamer &streamer,
 void MCWin64EHUnwindEmitter::Emit(MCStreamer &streamer) {
   MCContext &context = streamer.getContext();
   // Emit the unwind info structs first.
-  const TargetAsmInfo &TAI = context.getTargetAsmInfo();
   for (unsigned i = 0; i < streamer.getNumW64UnwindInfos(); ++i) {
     MCWin64EHUnwindInfo &info = streamer.getW64UnwindInfo(i);
     const MCSection *xdataSect =
-      TAI.getWin64EHTableSection(GetSectionSuffix(info.Function));
+      getWin64EHTableSection(GetSectionSuffix(info.Function), context);
     streamer.SwitchSection(xdataSect);
     llvm::EmitUnwindInfo(streamer, &info);
   }
@@ -248,7 +270,7 @@ void MCWin64EHUnwindEmitter::Emit(MCStreamer &streamer) {
   for (unsigned i = 0; i < streamer.getNumW64UnwindInfos(); ++i) {
     MCWin64EHUnwindInfo &info = streamer.getW64UnwindInfo(i);
     const MCSection *pdataSect =
-      TAI.getWin64EHFuncTableSection(GetSectionSuffix(info.Function));
+      getWin64EHFuncTableSection(GetSectionSuffix(info.Function), context);
     streamer.SwitchSection(pdataSect);
     EmitRuntimeFunction(streamer, &info);
   }
diff --git a/contrib/llvm/lib/MC/MachObjectWriter.cpp b/contrib/llvm/lib/MC/MachObjectWriter.cpp
index 69efe231ad6e..a9219ad29c65 100644
--- a/contrib/llvm/lib/MC/MachObjectWriter.cpp
+++ b/contrib/llvm/lib/MC/MachObjectWriter.cpp
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -21,7 +22,6 @@
 #include "llvm/MC/MCValue.h"
 #include "llvm/Object/MachOFormat.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetAsmBackend.h"
 
 #include <vector>
 using namespace llvm;
@@ -291,7 +291,7 @@ void MachObjectWriter::WriteNlist(MachSymbolData &MSD,
   const MCSymbol &Symbol = Data.getSymbol();
   uint8_t Type = 0;
   uint16_t Flags = Data.getFlags();
-  uint32_t Address = 0;
+  uint64_t Address = 0;
 
   // Set the N_TYPE bits. See <mach-o/nlist.h>.
   //
@@ -590,14 +590,28 @@ IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
         return false;
       return true;
     }
+    // For Darwin x86_64, there is one special case when the reference IsPCRel.
+    // If the fragment with the reference does not have a base symbol but meets
+    // the simple way of dealing with this, in that it is a temporary symbol in
+    // the same atom then it is assumed to be fully resolved.  This is needed so
+    // a relocation entry is not created and so the static linker does not
+    // mess up the reference later.
+    else if(!FB.getAtom() &&
+            SA.isTemporary() && SA.isInSection() && &SecA == &SecB){
+      return true;
+    }
   } else {
     if (!TargetObjectWriter->useAggressiveSymbolFolding())
       return false;
   }
 
-  const MCFragment &FA = *Asm.getSymbolData(SA).getFragment();
+  const MCFragment *FA = Asm.getSymbolData(SA).getFragment();
 
-  A_Base = FA.getAtom();
+  // Bail if the symbol has no fragment.
+  if (!FA)
+    return false;
+
+  A_Base = FA->getAtom();
   if (!A_Base)
     return false;
 
@@ -613,7 +627,8 @@ IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
   return false;
 }
 
-void MachObjectWriter::WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) {
+void MachObjectWriter::WriteObject(MCAssembler &Asm,
+				   const MCAsmLayout &Layout) {
   unsigned NumSections = Asm.size();
 
   // The section data starts after the header, the segment load command (and
diff --git a/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp b/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp
index 101237aabb02..b15e225fc2a3 100644
--- a/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -33,7 +33,7 @@
 
 #include "llvm/Support/TimeValue.h"
 
-#include "../Target/X86/X86FixupKinds.h"
+#include "../Target/X86/MCTargetDesc/X86FixupKinds.h"
 
 #include <cstdio>
 
diff --git a/contrib/llvm/lib/MC/WinCOFFStreamer.cpp b/contrib/llvm/lib/MC/WinCOFFStreamer.cpp
index 6c36c1231c83..7409daf39085 100644
--- a/contrib/llvm/lib/MC/WinCOFFStreamer.cpp
+++ b/contrib/llvm/lib/MC/WinCOFFStreamer.cpp
@@ -24,13 +24,13 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCWin64EH.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmBackend.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/ADT/StringMap.h"
 
 #include "llvm/Support/COFF.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
@@ -40,7 +40,7 @@ class WinCOFFStreamer : public MCObjectStreamer {
   MCSymbol const *CurSymbol;
 
   WinCOFFStreamer(MCContext &Context,
-                  TargetAsmBackend &TAB,
+                  MCAsmBackend &MAB,
                   MCCodeEmitter &CE,
                   raw_ostream &OS);
 
@@ -63,7 +63,8 @@ class WinCOFFStreamer : public MCObjectStreamer {
   virtual void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value);
   virtual void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                 unsigned ByteAlignment);
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size);
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment);
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
                             unsigned Size,unsigned ByteAlignment);
   virtual void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
@@ -123,10 +124,10 @@ class WinCOFFStreamer : public MCObjectStreamer {
 } // end anonymous namespace.
 
 WinCOFFStreamer::WinCOFFStreamer(MCContext &Context,
-                                 TargetAsmBackend &TAB,
+                                 MCAsmBackend &MAB,
                                  MCCodeEmitter &CE,
                                  raw_ostream &OS)
-    : MCObjectStreamer(Context, TAB, OS, &CE)
+    : MCObjectStreamer(Context, MAB, OS, &CE)
     , CurSymbol(NULL) {
 }
 
@@ -304,11 +305,12 @@ void WinCOFFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   AddCommonSymbol(Symbol, Size, ByteAlignment, true);
 }
 
-void WinCOFFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {
+void WinCOFFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                            unsigned ByteAlignment) {
   assert((Symbol->isInSection()
          ? Symbol->getSection().getVariant() == MCSection::SV_COFF
          : true) && "Got non COFF section in the COFF backend!");
-  AddCommonSymbol(Symbol, Size, 1, false);
+  AddCommonSymbol(Symbol, Size, ByteAlignment, false);
 }
 
 void WinCOFFStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
@@ -395,11 +397,11 @@ void WinCOFFStreamer::Finish() {
 namespace llvm
 {
   MCStreamer *createWinCOFFStreamer(MCContext &Context,
-                                    TargetAsmBackend &TAB,
+                                    MCAsmBackend &MAB,
                                     MCCodeEmitter &CE,
                                     raw_ostream &OS,
                                     bool RelaxAll) {
-    WinCOFFStreamer *S = new WinCOFFStreamer(Context, TAB, CE, OS);
+    WinCOFFStreamer *S = new WinCOFFStreamer(Context, MAB, CE, OS);
     S->getAssembler().setRelaxAll(RelaxAll);
     return S;
   }
diff --git a/contrib/llvm/lib/Object/Archive.cpp b/contrib/llvm/lib/Object/Archive.cpp
new file mode 100644
index 000000000000..e2eaff53c1f1
--- /dev/null
+++ b/contrib/llvm/lib/Object/Archive.cpp
@@ -0,0 +1,172 @@
+//===- Archive.cpp - ar File Format implementation --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the ArchiveObjectFile class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/Archive.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace llvm;
+using namespace object;
+
+namespace {
+const StringRef Magic = "!<arch>\n";
+
+struct ArchiveMemberHeader {
+  char Name[16];
+  char LastModified[12];
+  char UID[6];
+  char GID[6];
+  char AccessMode[8];
+  char Size[10]; //< Size of data, not including header or padding.
+  char Terminator[2];
+
+  ///! Get the name without looking up long names.
+  StringRef getName() const {
+    char EndCond = Name[0] == '/' ? ' ' : '/';
+    StringRef::size_type end = StringRef(Name, sizeof(Name)).find(EndCond);
+    if (end == StringRef::npos)
+      end = sizeof(Name);
+    assert(end <= sizeof(Name) && end > 0);
+    // Don't include the EndCond if there is one.
+    return StringRef(Name, end);
+  }
+
+  uint64_t getSize() const {
+    APInt ret;
+    StringRef(Size, sizeof(Size)).getAsInteger(10, ret);
+    return ret.getZExtValue();
+  }
+};
+
+const ArchiveMemberHeader *ToHeader(const char *base) {
+  return reinterpret_cast<const ArchiveMemberHeader *>(base);
+}
+}
+
+Archive::Child Archive::Child::getNext() const {
+  size_t SpaceToSkip = sizeof(ArchiveMemberHeader) +
+    ToHeader(Data.data())->getSize();
+  // If it's odd, add 1 to make it even.
+  if (SpaceToSkip & 1)
+    ++SpaceToSkip;
+
+  const char *NextLoc = Data.data() + SpaceToSkip;
+
+  // Check to see if this is past the end of the archive.
+  if (NextLoc >= Parent->Data->getBufferEnd())
+    return Child(Parent, StringRef(0, 0));
+
+  size_t NextSize = sizeof(ArchiveMemberHeader) +
+    ToHeader(NextLoc)->getSize();
+
+  return Child(Parent, StringRef(NextLoc, NextSize));
+}
+
+error_code Archive::Child::getName(StringRef &Result) const {
+  StringRef name = ToHeader(Data.data())->getName();
+  // Check if it's a special name.
+  if (name[0] == '/') {
+    if (name.size() == 1) { // Linker member.
+      Result = name;
+      return object_error::success;
+    }
+    if (name.size() == 2 && name[1] == '/') { // String table.
+      Result = name;
+      return object_error::success;
+    }
+    // It's a long name.
+    // Get the offset.
+    APInt offset;
+    name.substr(1).getAsInteger(10, offset);
+    const char *addr = Parent->StringTable->Data.begin()
+                       + sizeof(ArchiveMemberHeader)
+                       + offset.getZExtValue();
+    // Verify it.
+    if (Parent->StringTable == Parent->end_children()
+        || addr < (Parent->StringTable->Data.begin()
+                   + sizeof(ArchiveMemberHeader))
+        || addr > (Parent->StringTable->Data.begin()
+                   + sizeof(ArchiveMemberHeader)
+                   + Parent->StringTable->getSize()))
+      return object_error::parse_failed;
+    Result = addr;
+    return object_error::success;
+  }
+  // It's a simple name.
+  if (name[name.size() - 1] == '/')
+    Result = name.substr(0, name.size() - 1);
+  else
+    Result = name;
+  return object_error::success;
+}
+
+uint64_t Archive::Child::getSize() const {
+  return ToHeader(Data.data())->getSize();
+}
+
+MemoryBuffer *Archive::Child::getBuffer() const {
+  StringRef name;
+  if (getName(name)) return NULL;
+  return MemoryBuffer::getMemBuffer(Data.substr(sizeof(ArchiveMemberHeader),
+                                                getSize()),
+                                    name,
+                                    false);
+}
+
+error_code Archive::Child::getAsBinary(OwningPtr<Binary> &Result) const {
+  OwningPtr<Binary> ret;
+  if (error_code ec =
+    createBinary(getBuffer(), ret))
+    return ec;
+  Result.swap(ret);
+  return object_error::success;
+}
+
+Archive::Archive(MemoryBuffer *source, error_code &ec)
+  : Binary(Binary::isArchive, source)
+  , StringTable(Child(this, StringRef(0, 0))) {
+  // Check for sufficient magic.
+  if (!source || source->getBufferSize()
+                 < (8 + sizeof(ArchiveMemberHeader) + 2) // Smallest archive.
+              || StringRef(source->getBufferStart(), 8) != Magic) {
+    ec = object_error::invalid_file_type;
+    return;
+  }
+
+  // Get the string table. It's the 3rd member.
+  child_iterator StrTable = begin_children();
+  child_iterator e = end_children();
+  for (int i = 0; StrTable != e && i < 2; ++StrTable, ++i) {}
+
+  // Check to see if there were 3 members, or the 3rd member wasn't named "//".
+  StringRef name;
+  if (StrTable != e && !StrTable->getName(name) && name == "//")
+    StringTable = StrTable;
+
+  ec = object_error::success;
+}
+
+Archive::child_iterator Archive::begin_children() const {
+  const char *Loc = Data->getBufferStart() + Magic.size();
+  size_t Size = sizeof(ArchiveMemberHeader) +
+    ToHeader(Loc)->getSize();
+  return Child(this, StringRef(Loc, Size));
+}
+
+Archive::child_iterator Archive::end_children() const {
+  return Child(this, StringRef(0, 0));
+}
+
+namespace llvm {
+
+} // end namespace llvm
diff --git a/contrib/llvm/lib/Object/Binary.cpp b/contrib/llvm/lib/Object/Binary.cpp
index 4b31c7557dd3..4e528d8ea565 100644
--- a/contrib/llvm/lib/Object/Binary.cpp
+++ b/contrib/llvm/lib/Object/Binary.cpp
@@ -17,8 +17,9 @@
 #include "llvm/Support/Path.h"
 
 // Include headers for createBinary.
-#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Object/ObjectFile.h"
 
 using namespace llvm;
 using namespace object;
@@ -50,6 +51,12 @@ error_code object::createBinary(MemoryBuffer *Source,
                                 static_cast<unsigned>(Source->getBufferSize()));
   error_code ec;
   switch (type) {
+    case sys::Archive_FileType: {
+      OwningPtr<Binary> ret(new Archive(scopedSource.take(), ec));
+      if (ec) return ec;
+      Result.swap(ret);
+      return object_error::success;
+    }
     case sys::ELF_Relocatable_FileType:
     case sys::ELF_Executable_FileType:
     case sys::ELF_SharedObject_FileType:
@@ -90,7 +97,7 @@ error_code object::createBinary(MemoryBuffer *Source,
 
 error_code object::createBinary(StringRef Path, OwningPtr<Binary> &Result) {
   OwningPtr<MemoryBuffer> File;
-  if (error_code ec = MemoryBuffer::getFile(Path, File))
+  if (error_code ec = MemoryBuffer::getFileOrSTDIN(Path, File))
     return ec;
   return createBinary(File.take(), Result);
 }
diff --git a/contrib/llvm/lib/Object/CMakeLists.txt b/contrib/llvm/lib/Object/CMakeLists.txt
deleted file mode 100644
index 6a6814fd37d9..000000000000
--- a/contrib/llvm/lib/Object/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-add_llvm_library(LLVMObject
-  MachOObject.cpp
-  ObjectFile.cpp
-  COFFObjectFile.cpp
-  ELFObjectFile.cpp
-  )
diff --git a/contrib/llvm/lib/Object/COFFObjectFile.cpp b/contrib/llvm/lib/Object/COFFObjectFile.cpp
index 07de6bc99973..750c34d12a18 100644
--- a/contrib/llvm/lib/Object/COFFObjectFile.cpp
+++ b/contrib/llvm/lib/Object/COFFObjectFile.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Object/COFF.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 
@@ -114,7 +115,7 @@ error_code COFFObjectFile::getSymbolNext(DataRefImpl Symb,
   return object_error::success;
 }
 
-error_code COFFObjectFile::getSymbolAddress(DataRefImpl Symb,
+error_code COFFObjectFile::getSymbolOffset(DataRefImpl Symb,
                                             uint64_t &Result) const {
   const coff_symbol *symb = toSymb(Symb);
   const coff_section *Section = NULL;
@@ -132,6 +133,55 @@ error_code COFFObjectFile::getSymbolAddress(DataRefImpl Symb,
   return object_error::success;
 }
 
+error_code COFFObjectFile::getSymbolAddress(DataRefImpl Symb,
+                                            uint64_t &Result) const {
+  const coff_symbol *symb = toSymb(Symb);
+  const coff_section *Section = NULL;
+  if (error_code ec = getSection(symb->SectionNumber, Section))
+    return ec;
+  char Type;
+  if (error_code ec = getSymbolNMTypeChar(Symb, Type))
+    return ec;
+  if (Type == 'U' || Type == 'w')
+    Result = UnknownAddressOrSize;
+  else if (Section)
+    Result = reinterpret_cast<uintptr_t>(base() +
+                                         Section->PointerToRawData +
+                                         symb->Value);
+  else
+    Result = reinterpret_cast<uintptr_t>(base() + symb->Value);
+  return object_error::success;
+}
+
+error_code COFFObjectFile::getSymbolType(DataRefImpl Symb,
+                                         SymbolRef::SymbolType &Result) const {
+  const coff_symbol *symb = toSymb(Symb);
+  Result = SymbolRef::ST_Other;
+  if (symb->StorageClass == COFF::IMAGE_SYM_CLASS_EXTERNAL &&
+      symb->SectionNumber == COFF::IMAGE_SYM_UNDEFINED) {
+    Result = SymbolRef::ST_External;
+  } else {
+    if (symb->Type.ComplexType == COFF::IMAGE_SYM_DTYPE_FUNCTION) {
+      Result = SymbolRef::ST_Function;
+    } else {
+      char Type;
+      if (error_code ec = getSymbolNMTypeChar(Symb, Type))
+        return ec;
+      if (Type == 'r' || Type == 'R') {
+        Result = SymbolRef::ST_Data;
+      }
+    }
+  }
+  return object_error::success;
+}
+
+error_code COFFObjectFile::isSymbolGlobal(DataRefImpl Symb,
+                                          bool &Result) const {
+  const coff_symbol *symb = toSymb(Symb);
+  Result = (symb->StorageClass == COFF::IMAGE_SYM_CLASS_EXTERNAL);
+  return object_error::success;
+}
+
 error_code COFFObjectFile::getSymbolSize(DataRefImpl Symb,
                                          uint64_t &Result) const {
   // FIXME: Return the correct size. This requires looking at all the symbols
@@ -286,6 +336,15 @@ error_code COFFObjectFile::getSectionContents(DataRefImpl Sec,
   return object_error::success;
 }
 
+error_code COFFObjectFile::getSectionAlignment(DataRefImpl Sec,
+                                               uint64_t &Res) const {
+  const coff_section *sec = toSec(Sec);
+  if (!sec)
+    return object_error::parse_failed;
+  Res = uint64_t(1) << (((sec->Characteristics & 0x00F00000) >> 20) - 1);
+  return object_error::success;
+}
+
 error_code COFFObjectFile::isSectionText(DataRefImpl Sec,
                                          bool &Result) const {
   const coff_section *sec = toSec(Sec);
@@ -293,14 +352,61 @@ error_code COFFObjectFile::isSectionText(DataRefImpl Sec,
   return object_error::success;
 }
 
+error_code COFFObjectFile::isSectionData(DataRefImpl Sec,
+                                         bool &Result) const {
+  const coff_section *sec = toSec(Sec);
+  Result = sec->Characteristics & COFF::IMAGE_SCN_CNT_INITIALIZED_DATA;
+  return object_error::success;
+}
+
+error_code COFFObjectFile::isSectionBSS(DataRefImpl Sec,
+                                        bool &Result) const {
+  const coff_section *sec = toSec(Sec);
+  Result = sec->Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA;
+  return object_error::success;
+}
+
 error_code COFFObjectFile::sectionContainsSymbol(DataRefImpl Sec,
                                                  DataRefImpl Symb,
                                                  bool &Result) const {
-  // FIXME: Unimplemented.
-  Result = false;
+  const coff_section *sec = toSec(Sec);
+  const coff_symbol *symb = toSymb(Symb);
+  const coff_section *symb_sec;
+  if (error_code ec = getSection(symb->SectionNumber, symb_sec)) return ec;
+  if (symb_sec == sec)
+    Result = true;
+  else
+    Result = false;
   return object_error::success;
 }
 
+relocation_iterator COFFObjectFile::getSectionRelBegin(DataRefImpl Sec) const {
+  const coff_section *sec = toSec(Sec);
+  DataRefImpl ret;
+  std::memset(&ret, 0, sizeof(ret));
+  if (sec->NumberOfRelocations == 0)
+    ret.p = 0;
+  else
+    ret.p = reinterpret_cast<uintptr_t>(base() + sec->PointerToRelocations);
+
+  return relocation_iterator(RelocationRef(ret, this));
+}
+
+relocation_iterator COFFObjectFile::getSectionRelEnd(DataRefImpl Sec) const {
+  const coff_section *sec = toSec(Sec);
+  DataRefImpl ret;
+  std::memset(&ret, 0, sizeof(ret));
+  if (sec->NumberOfRelocations == 0)
+    ret.p = 0;
+  else
+    ret.p = reinterpret_cast<uintptr_t>(
+              reinterpret_cast<const coff_relocation*>(
+                base() + sec->PointerToRelocations)
+              + sec->NumberOfRelocations);
+
+  return relocation_iterator(RelocationRef(ret, this));
+}
+
 COFFObjectFile::COFFObjectFile(MemoryBuffer *Object, error_code &ec)
   : ObjectFile(Binary::isCOFF, Object, ec) {
   // Check that we at least have enough room for a header.
@@ -327,7 +433,7 @@ COFFObjectFile::COFFObjectFile(MemoryBuffer *Object, error_code &ec)
   Header = reinterpret_cast<const coff_file_header *>(base() + HeaderStart);
   if (!checkAddr(Data, ec, uintptr_t(Header), sizeof(coff_file_header)))
     return;
-  
+
   SectionTable =
     reinterpret_cast<const coff_section *>( base()
                                           + HeaderStart
@@ -360,18 +466,18 @@ COFFObjectFile::COFFObjectFile(MemoryBuffer *Object, error_code &ec)
     ec = object_error::parse_failed;
     return;
   }
-  
+
   ec = object_error::success;
 }
 
-ObjectFile::symbol_iterator COFFObjectFile::begin_symbols() const {
+symbol_iterator COFFObjectFile::begin_symbols() const {
   DataRefImpl ret;
   std::memset(&ret, 0, sizeof(DataRefImpl));
   ret.p = reinterpret_cast<intptr_t>(SymbolTable);
   return symbol_iterator(SymbolRef(ret, this));
 }
 
-ObjectFile::symbol_iterator COFFObjectFile::end_symbols() const {
+symbol_iterator COFFObjectFile::end_symbols() const {
   // The symbol table ends where the string table begins.
   DataRefImpl ret;
   std::memset(&ret, 0, sizeof(DataRefImpl));
@@ -379,14 +485,14 @@ ObjectFile::symbol_iterator COFFObjectFile::end_symbols() const {
   return symbol_iterator(SymbolRef(ret, this));
 }
 
-ObjectFile::section_iterator COFFObjectFile::begin_sections() const {
+section_iterator COFFObjectFile::begin_sections() const {
   DataRefImpl ret;
   std::memset(&ret, 0, sizeof(DataRefImpl));
   ret.p = reinterpret_cast<intptr_t>(SectionTable);
   return section_iterator(SectionRef(ret, this));
 }
 
-ObjectFile::section_iterator COFFObjectFile::end_sections() const {
+section_iterator COFFObjectFile::end_sections() const {
   DataRefImpl ret;
   std::memset(&ret, 0, sizeof(DataRefImpl));
   ret.p = reinterpret_cast<intptr_t>(SectionTable + Header->NumberOfSections);
@@ -445,6 +551,121 @@ error_code COFFObjectFile::getString(uint32_t offset,
   return object_error::success;
 }
 
+error_code COFFObjectFile::getSymbol(uint32_t index,
+                                     const coff_symbol *&Result) const {
+  if (index > 0 && index < Header->NumberOfSymbols)
+    Result = SymbolTable + index;
+  else
+    return object_error::parse_failed;
+  return object_error::success;
+}
+
+const coff_relocation *COFFObjectFile::toRel(DataRefImpl Rel) const {
+  return reinterpret_cast<const coff_relocation*>(Rel.p);
+}
+error_code COFFObjectFile::getRelocationNext(DataRefImpl Rel,
+                                             RelocationRef &Res) const {
+  Rel.p = reinterpret_cast<uintptr_t>(
+            reinterpret_cast<const coff_relocation*>(Rel.p) + 1);
+  Res = RelocationRef(Rel, this);
+  return object_error::success;
+}
+error_code COFFObjectFile::getRelocationAddress(DataRefImpl Rel,
+                                                uint64_t &Res) const {
+  Res = toRel(Rel)->VirtualAddress;
+  return object_error::success;
+}
+error_code COFFObjectFile::getRelocationSymbol(DataRefImpl Rel,
+                                               SymbolRef &Res) const {
+  const coff_relocation* R = toRel(Rel);
+  DataRefImpl Symb;
+  Symb.p = reinterpret_cast<uintptr_t>(SymbolTable + R->SymbolTableIndex);
+  Res = SymbolRef(Symb, this);
+  return object_error::success;
+}
+error_code COFFObjectFile::getRelocationType(DataRefImpl Rel,
+                                             uint32_t &Res) const {
+  const coff_relocation* R = toRel(Rel);
+  Res = R->Type;
+  return object_error::success;
+}
+
+#define LLVM_COFF_SWITCH_RELOC_TYPE_NAME(enum) \
+  case COFF::enum: res = #enum; break;
+
+error_code COFFObjectFile::getRelocationTypeName(DataRefImpl Rel,
+                                          SmallVectorImpl<char> &Result) const {
+  const coff_relocation *reloc = toRel(Rel);
+  StringRef res;
+  switch (Header->Machine) {
+  case COFF::IMAGE_FILE_MACHINE_AMD64:
+    switch (reloc->Type) {
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_ABSOLUTE);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_ADDR64);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_ADDR32);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_ADDR32NB);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_REL32);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_REL32_1);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_REL32_2);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_REL32_3);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_REL32_4);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_REL32_5);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_SECTION);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_SECREL);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_SECREL7);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_TOKEN);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_SREL32);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_PAIR);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_AMD64_SSPAN32);
+    default:
+      res = "Unknown";
+    }
+    break;
+  case COFF::IMAGE_FILE_MACHINE_I386:
+    switch (reloc->Type) {
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_ABSOLUTE);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_DIR16);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_REL16);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_DIR32);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_DIR32NB);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_SEG12);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_SECTION);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_SECREL);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_TOKEN);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_SECREL7);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_I386_REL32);
+    default:
+      res = "Unknown";
+    }
+    break;
+  default:
+    res = "Unknown";
+  }
+  Result.append(res.begin(), res.end());
+  return object_error::success;
+}
+
+#undef LLVM_COFF_SWITCH_RELOC_TYPE_NAME
+
+error_code COFFObjectFile::getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                       int64_t &Res) const {
+  Res = 0;
+  return object_error::success;
+}
+error_code COFFObjectFile::getRelocationValueString(DataRefImpl Rel,
+                                          SmallVectorImpl<char> &Result) const {
+  const coff_relocation *reloc = toRel(Rel);
+  const coff_symbol *symb = 0;
+  if (error_code ec = getSymbol(reloc->SymbolTableIndex, symb)) return ec;
+  DataRefImpl sym;
+  ::memset(&sym, 0, sizeof(sym));
+  sym.p = reinterpret_cast<uintptr_t>(symb);
+  StringRef symname;
+  if (error_code ec = getSymbolName(sym, symname)) return ec;
+  Result.append(symname.begin(), symname.end());
+  return object_error::success;
+}
+
 namespace llvm {
 
   ObjectFile *ObjectFile::createCOFFObjectFile(MemoryBuffer *Object) {
diff --git a/contrib/llvm/lib/Object/ELFObjectFile.cpp b/contrib/llvm/lib/Object/ELFObjectFile.cpp
index e2ff4dfc0384..257d08cadff3 100644
--- a/contrib/llvm/lib/Object/ELFObjectFile.cpp
+++ b/contrib/llvm/lib/Object/ELFObjectFile.cpp
@@ -14,11 +14,14 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
 #include <limits>
 #include <utility>
 
@@ -175,6 +178,81 @@ struct Elf_Sym_Impl : Elf_Sym_Base<target_endianness, is64Bits> {
 };
 }
 
+namespace {
+template<support::endianness target_endianness, bool is64Bits, bool isRela>
+struct Elf_Rel_Base;
+
+template<support::endianness target_endianness>
+struct Elf_Rel_Base<target_endianness, false, false> {
+  LLVM_ELF_IMPORT_TYPES(target_endianness, false)
+  Elf_Addr      r_offset; // Location (file byte offset, or program virtual addr)
+  Elf_Word      r_info;  // Symbol table index and type of relocation to apply
+};
+
+template<support::endianness target_endianness>
+struct Elf_Rel_Base<target_endianness, true, false> {
+  LLVM_ELF_IMPORT_TYPES(target_endianness, true)
+  Elf_Addr      r_offset; // Location (file byte offset, or program virtual addr)
+  Elf_Xword     r_info;   // Symbol table index and type of relocation to apply
+};
+
+template<support::endianness target_endianness>
+struct Elf_Rel_Base<target_endianness, false, true> {
+  LLVM_ELF_IMPORT_TYPES(target_endianness, false)
+  Elf_Addr      r_offset; // Location (file byte offset, or program virtual addr)
+  Elf_Word      r_info;   // Symbol table index and type of relocation to apply
+  Elf_Sword     r_addend; // Compute value for relocatable field by adding this
+};
+
+template<support::endianness target_endianness>
+struct Elf_Rel_Base<target_endianness, true, true> {
+  LLVM_ELF_IMPORT_TYPES(target_endianness, true)
+  Elf_Addr      r_offset; // Location (file byte offset, or program virtual addr)
+  Elf_Xword     r_info;   // Symbol table index and type of relocation to apply
+  Elf_Sxword    r_addend; // Compute value for relocatable field by adding this.
+};
+
+template<support::endianness target_endianness, bool is64Bits, bool isRela>
+struct Elf_Rel_Impl;
+
+template<support::endianness target_endianness, bool isRela>
+struct Elf_Rel_Impl<target_endianness, true, isRela>
+       : Elf_Rel_Base<target_endianness, true, isRela> {
+  using Elf_Rel_Base<target_endianness, true, isRela>::r_info;
+  LLVM_ELF_IMPORT_TYPES(target_endianness, true)
+
+  // These accessors and mutators correspond to the ELF64_R_SYM, ELF64_R_TYPE,
+  // and ELF64_R_INFO macros defined in the ELF specification:
+  uint64_t getSymbol() const { return (r_info >> 32); }
+  unsigned char getType() const {
+    return (unsigned char) (r_info & 0xffffffffL);
+  }
+  void setSymbol(uint64_t s) { setSymbolAndType(s, getType()); }
+  void setType(unsigned char t) { setSymbolAndType(getSymbol(), t); }
+  void setSymbolAndType(uint64_t s, unsigned char t) {
+    r_info = (s << 32) + (t&0xffffffffL);
+  }
+};
+
+template<support::endianness target_endianness, bool isRela>
+struct Elf_Rel_Impl<target_endianness, false, isRela>
+       : Elf_Rel_Base<target_endianness, false, isRela> {
+  using Elf_Rel_Base<target_endianness, false, isRela>::r_info;
+  LLVM_ELF_IMPORT_TYPES(target_endianness, false)
+
+  // These accessors and mutators correspond to the ELF32_R_SYM, ELF32_R_TYPE,
+  // and ELF32_R_INFO macros defined in the ELF specification:
+  uint32_t getSymbol() const { return (r_info >> 8); }
+  unsigned char getType() const { return (unsigned char) (r_info & 0x0ff); }
+  void setSymbol(uint32_t s) { setSymbolAndType(s, getType()); }
+  void setType(unsigned char t) { setSymbolAndType(getSymbol(), t); }
+  void setSymbolAndType(uint32_t s, unsigned char t) {
+    r_info = (s << 8) + t;
+  }
+};
+
+}
+
 namespace {
 template<support::endianness target_endianness, bool is64Bits>
 class ELFObjectFile : public ObjectFile {
@@ -182,6 +260,8 @@ class ELFObjectFile : public ObjectFile {
 
   typedef Elf_Shdr_Impl<target_endianness, is64Bits> Elf_Shdr;
   typedef Elf_Sym_Impl<target_endianness, is64Bits> Elf_Sym;
+  typedef Elf_Rel_Impl<target_endianness, is64Bits, false> Elf_Rel;
+  typedef Elf_Rel_Impl<target_endianness, is64Bits, true> Elf_Rela;
 
   struct Elf_Ehdr {
     unsigned char e_ident[ELF::EI_NIDENT]; // ELF Identification bytes
@@ -206,37 +286,81 @@ class ELFObjectFile : public ObjectFile {
     unsigned char getDataEncoding() const { return e_ident[ELF::EI_DATA]; }
   };
 
-  typedef SmallVector<const Elf_Shdr*, 1> SymbolTableSections_t;
+  typedef SmallVector<const Elf_Shdr*, 1> Sections_t;
+  typedef DenseMap<unsigned, unsigned> IndexMap_t;
+  typedef DenseMap<const Elf_Shdr*, SmallVector<uint32_t, 1> > RelocMap_t;
 
   const Elf_Ehdr *Header;
   const Elf_Shdr *SectionHeaderTable;
   const Elf_Shdr *dot_shstrtab_sec; // Section header string table.
   const Elf_Shdr *dot_strtab_sec;   // Symbol header string table.
-  SymbolTableSections_t SymbolTableSections;
+  Sections_t SymbolTableSections;
+  IndexMap_t SymbolTableSectionsIndexMap;
+  DenseMap<const Elf_Sym*, ELF::Elf64_Word> ExtendedSymbolTable;
+
+  /// @brief Map sections to an array of relocation sections that reference
+  ///        them sorted by section index.
+  RelocMap_t SectionRelocMap;
+
+  /// @brief Get the relocation section that contains \a Rel.
+  const Elf_Shdr *getRelSection(DataRefImpl Rel) const {
+    return getSection(Rel.w.b);
+  }
 
   void            validateSymbol(DataRefImpl Symb) const;
+  bool            isRelocationHasAddend(DataRefImpl Rel) const;
+  template<typename T>
+  const T        *getEntry(uint16_t Section, uint32_t Entry) const;
+  template<typename T>
+  const T        *getEntry(const Elf_Shdr *Section, uint32_t Entry) const;
   const Elf_Sym  *getSymbol(DataRefImpl Symb) const;
   const Elf_Shdr *getSection(DataRefImpl index) const;
-  const Elf_Shdr *getSection(uint16_t index) const;
-  const char     *getString(uint16_t section, uint32_t offset) const;
+  const Elf_Shdr *getSection(uint32_t index) const;
+  const Elf_Rel  *getRel(DataRefImpl Rel) const;
+  const Elf_Rela *getRela(DataRefImpl Rela) const;
+  const char     *getString(uint32_t section, uint32_t offset) const;
   const char     *getString(const Elf_Shdr *section, uint32_t offset) const;
+  error_code      getSymbolName(const Elf_Sym *Symb, StringRef &Res) const;
 
 protected:
   virtual error_code getSymbolNext(DataRefImpl Symb, SymbolRef &Res) const;
   virtual error_code getSymbolName(DataRefImpl Symb, StringRef &Res) const;
+  virtual error_code getSymbolOffset(DataRefImpl Symb, uint64_t &Res) const;
   virtual error_code getSymbolAddress(DataRefImpl Symb, uint64_t &Res) const;
   virtual error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const;
   virtual error_code getSymbolNMTypeChar(DataRefImpl Symb, char &Res) const;
   virtual error_code isSymbolInternal(DataRefImpl Symb, bool &Res) const;
+  virtual error_code isSymbolGlobal(DataRefImpl Symb, bool &Res) const;
+  virtual error_code getSymbolType(DataRefImpl Symb, SymbolRef::SymbolType &Res) const;
 
   virtual error_code getSectionNext(DataRefImpl Sec, SectionRef &Res) const;
   virtual error_code getSectionName(DataRefImpl Sec, StringRef &Res) const;
   virtual error_code getSectionAddress(DataRefImpl Sec, uint64_t &Res) const;
   virtual error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const;
   virtual error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const;
+  virtual error_code getSectionAlignment(DataRefImpl Sec, uint64_t &Res) const;
   virtual error_code isSectionText(DataRefImpl Sec, bool &Res) const;
+  virtual error_code isSectionData(DataRefImpl Sec, bool &Res) const;
+  virtual error_code isSectionBSS(DataRefImpl Sec, bool &Res) const;
   virtual error_code sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb,
                                            bool &Result) const;
+  virtual relocation_iterator getSectionRelBegin(DataRefImpl Sec) const;
+  virtual relocation_iterator getSectionRelEnd(DataRefImpl Sec) const;
+
+  virtual error_code getRelocationNext(DataRefImpl Rel,
+                                       RelocationRef &Res) const;
+  virtual error_code getRelocationAddress(DataRefImpl Rel,
+                                          uint64_t &Res) const;
+  virtual error_code getRelocationSymbol(DataRefImpl Rel,
+                                         SymbolRef &Res) const;
+  virtual error_code getRelocationType(DataRefImpl Rel,
+                                       uint32_t &Res) const;
+  virtual error_code getRelocationTypeName(DataRefImpl Rel,
+                                           SmallVectorImpl<char> &Result) const;
+  virtual error_code getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                 int64_t &Res) const;
+  virtual error_code getRelocationValueString(DataRefImpl Rel,
+                                           SmallVectorImpl<char> &Result) const;
 
 public:
   ELFObjectFile(MemoryBuffer *Object, error_code &ec);
@@ -248,6 +372,11 @@ class ELFObjectFile : public ObjectFile {
   virtual uint8_t getBytesInAddress() const;
   virtual StringRef getFileFormatName() const;
   virtual unsigned getArch() const;
+
+  uint64_t getNumSections() const;
+  uint64_t getStringTableIndex() const;
+  ELF::Elf64_Word getSymbolTableIndex(const Elf_Sym *symb) const;
+  const Elf_Shdr *getSection(const Elf_Sym *symb) const;
 };
 } // end namespace
 
@@ -299,29 +428,37 @@ error_code ELFObjectFile<target_endianness, is64Bits>
                         ::getSymbolName(DataRefImpl Symb,
                                         StringRef &Result) const {
   validateSymbol(Symb);
-  const Elf_Sym  *symb = getSymbol(Symb);
-  if (symb->st_name == 0) {
-    const Elf_Shdr *section = getSection(symb->st_shndx);
-    if (!section)
-      Result = "";
-    else
-      Result = getString(dot_shstrtab_sec, section->sh_name);
-    return object_error::success;
-  }
+  const Elf_Sym *symb = getSymbol(Symb);
+  return getSymbolName(symb, Result);
+}
 
-  // Use the default symbol table name section.
-  Result = getString(dot_strtab_sec, symb->st_name);
-  return object_error::success;
+template<support::endianness target_endianness, bool is64Bits>
+ELF::Elf64_Word ELFObjectFile<target_endianness, is64Bits>
+                      ::getSymbolTableIndex(const Elf_Sym *symb) const {
+  if (symb->st_shndx == ELF::SHN_XINDEX)
+    return ExtendedSymbolTable.lookup(symb);
+  return symb->st_shndx;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+const typename ELFObjectFile<target_endianness, is64Bits>::Elf_Shdr *
+ELFObjectFile<target_endianness, is64Bits>
+                             ::getSection(const Elf_Sym *symb) const {
+  if (symb->st_shndx == ELF::SHN_XINDEX)
+    return getSection(ExtendedSymbolTable.lookup(symb));
+  if (symb->st_shndx >= ELF::SHN_LORESERVE)
+    return 0;
+  return getSection(symb->st_shndx);
 }
 
 template<support::endianness target_endianness, bool is64Bits>
 error_code ELFObjectFile<target_endianness, is64Bits>
-                        ::getSymbolAddress(DataRefImpl Symb,
-                                           uint64_t &Result) const {
+                        ::getSymbolOffset(DataRefImpl Symb,
+                                          uint64_t &Result) const {
   validateSymbol(Symb);
   const Elf_Sym  *symb = getSymbol(Symb);
   const Elf_Shdr *Section;
-  switch (symb->st_shndx) {
+  switch (getSymbolTableIndex(symb)) {
   case ELF::SHN_COMMON:
    // Undefined symbols have no address yet.
   case ELF::SHN_UNDEF:
@@ -330,7 +467,7 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   case ELF::SHN_ABS:
     Result = symb->st_value;
     return object_error::success;
-  default: Section = getSection(symb->st_shndx);
+  default: Section = getSection(symb);
   }
 
   switch (symb->getType()) {
@@ -348,6 +485,43 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   }
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getSymbolAddress(DataRefImpl Symb,
+                                           uint64_t &Result) const {
+  validateSymbol(Symb);
+  const Elf_Sym  *symb = getSymbol(Symb);
+  const Elf_Shdr *Section;
+  switch (getSymbolTableIndex(symb)) {
+  case ELF::SHN_COMMON: // Fall through.
+   // Undefined symbols have no address yet.
+  case ELF::SHN_UNDEF:
+    Result = UnknownAddressOrSize;
+    return object_error::success;
+  case ELF::SHN_ABS:
+    Result = reinterpret_cast<uintptr_t>(base()+symb->st_value);
+    return object_error::success;
+  default: Section = getSection(symb);
+  }
+  const uint8_t* addr = base();
+  if (Section)
+    addr += Section->sh_offset;
+  switch (symb->getType()) {
+  case ELF::STT_SECTION:
+    Result = reinterpret_cast<uintptr_t>(addr);
+    return object_error::success;
+  case ELF::STT_FUNC: // Fall through.
+  case ELF::STT_OBJECT: // Fall through.
+  case ELF::STT_NOTYPE:
+    addr += symb->st_value;
+    Result = reinterpret_cast<uintptr_t>(addr);
+    return object_error::success;
+  default:
+    Result = UnknownAddressOrSize;
+    return object_error::success;
+  }
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 error_code ELFObjectFile<target_endianness, is64Bits>
                         ::getSymbolSize(DataRefImpl Symb,
@@ -366,7 +540,7 @@ error_code ELFObjectFile<target_endianness, is64Bits>
                                               char &Result) const {
   validateSymbol(Symb);
   const Elf_Sym  *symb = getSymbol(Symb);
-  const Elf_Shdr *Section = getSection(symb->st_shndx);
+  const Elf_Shdr *Section = getSection(symb);
 
   char ret = '?';
 
@@ -389,7 +563,7 @@ error_code ELFObjectFile<target_endianness, is64Bits>
     }
   }
 
-  switch (symb->st_shndx) {
+  switch (getSymbolTableIndex(symb)) {
   case ELF::SHN_UNDEF:
     if (ret == '?')
       ret = 'U';
@@ -401,7 +575,7 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   switch (symb->getBinding()) {
   case ELF::STB_GLOBAL: ret = ::toupper(ret); break;
   case ELF::STB_WEAK:
-    if (symb->st_shndx == ELF::SHN_UNDEF)
+    if (getSymbolTableIndex(symb) == ELF::SHN_UNDEF)
       ret = 'w';
     else
       if (symb->getType() == ELF::STT_OBJECT)
@@ -416,7 +590,8 @@ error_code ELFObjectFile<target_endianness, is64Bits>
       return ec;
     Result = StringSwitch<char>(name)
       .StartsWith(".debug", 'N')
-      .StartsWith(".note", 'n');
+      .StartsWith(".note", 'n')
+      .Default('?');
     return object_error::success;
   }
 
@@ -424,6 +599,43 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   return object_error::success;
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getSymbolType(DataRefImpl Symb,
+                                        SymbolRef::SymbolType &Result) const {
+  validateSymbol(Symb);
+  const Elf_Sym  *symb = getSymbol(Symb);
+
+  if (getSymbolTableIndex(symb) == ELF::SHN_UNDEF) {
+    Result = SymbolRef::ST_External;
+    return object_error::success;
+  }
+
+  switch (symb->getType()) {
+  case ELF::STT_FUNC:
+    Result = SymbolRef::ST_Function;
+    break;
+  case ELF::STT_OBJECT:
+    Result = SymbolRef::ST_Data;
+    break;
+  default:
+    Result = SymbolRef::ST_Other;
+    break;
+  }
+  return object_error::success;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::isSymbolGlobal(DataRefImpl Symb,
+                                        bool &Result) const {
+  validateSymbol(Symb);
+  const Elf_Sym  *symb = getSymbol(Symb);
+
+  Result = symb->getBinding() == ELF::STB_GLOBAL;
+  return object_error::success;
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 error_code ELFObjectFile<target_endianness, is64Bits>
                         ::isSymbolInternal(DataRefImpl Symb,
@@ -485,6 +697,15 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   return object_error::success;
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getSectionAlignment(DataRefImpl Sec,
+                                              uint64_t &Result) const {
+  const Elf_Shdr *sec = reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  Result = sec->sh_addralign;
+  return object_error::success;
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 error_code ELFObjectFile<target_endianness, is64Bits>
                         ::isSectionText(DataRefImpl Sec,
@@ -497,6 +718,32 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   return object_error::success;
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::isSectionData(DataRefImpl Sec,
+                                        bool &Result) const {
+  const Elf_Shdr *sec = reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  if (sec->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE)
+      && sec->sh_type == ELF::SHT_PROGBITS)
+    Result = true;
+  else
+    Result = false;
+  return object_error::success;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::isSectionBSS(DataRefImpl Sec,
+                                       bool &Result) const {
+  const Elf_Shdr *sec = reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  if (sec->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE)
+      && sec->sh_type == ELF::SHT_NOBITS)
+    Result = true;
+  else
+    Result = false;
+  return object_error::success;
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 error_code ELFObjectFile<target_endianness, is64Bits>
                           ::sectionContainsSymbol(DataRefImpl Sec,
@@ -507,6 +754,330 @@ error_code ELFObjectFile<target_endianness, is64Bits>
   return object_error::success;
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+relocation_iterator ELFObjectFile<target_endianness, is64Bits>
+                                 ::getSectionRelBegin(DataRefImpl Sec) const {
+  DataRefImpl RelData;
+  memset(&RelData, 0, sizeof(RelData));
+  const Elf_Shdr *sec = reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  typename RelocMap_t::const_iterator ittr = SectionRelocMap.find(sec);
+  if (sec != 0 && ittr != SectionRelocMap.end()) {
+    RelData.w.a = getSection(ittr->second[0])->sh_info;
+    RelData.w.b = ittr->second[0];
+    RelData.w.c = 0;
+  }
+  return relocation_iterator(RelocationRef(RelData, this));
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+relocation_iterator ELFObjectFile<target_endianness, is64Bits>
+                                 ::getSectionRelEnd(DataRefImpl Sec) const {
+  DataRefImpl RelData;
+  memset(&RelData, 0, sizeof(RelData));
+  const Elf_Shdr *sec = reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  typename RelocMap_t::const_iterator ittr = SectionRelocMap.find(sec);
+  if (sec != 0 && ittr != SectionRelocMap.end()) {
+    // Get the index of the last relocation section for this section.
+    std::size_t relocsecindex = ittr->second[ittr->second.size() - 1];
+    const Elf_Shdr *relocsec = getSection(relocsecindex);
+    RelData.w.a = relocsec->sh_info;
+    RelData.w.b = relocsecindex;
+    RelData.w.c = relocsec->sh_size / relocsec->sh_entsize;
+  }
+  return relocation_iterator(RelocationRef(RelData, this));
+}
+
+// Relocations
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationNext(DataRefImpl Rel,
+                                            RelocationRef &Result) const {
+  ++Rel.w.c;
+  const Elf_Shdr *relocsec = getSection(Rel.w.b);
+  if (Rel.w.c >= (relocsec->sh_size / relocsec->sh_entsize)) {
+    // We have reached the end of the relocations for this section. See if there
+    // is another relocation section.
+    typename RelocMap_t::mapped_type relocseclist =
+      SectionRelocMap.lookup(getSection(Rel.w.a));
+
+    // Do a binary search for the current reloc section index (which must be
+    // present). Then get the next one.
+    typename RelocMap_t::mapped_type::const_iterator loc =
+      std::lower_bound(relocseclist.begin(), relocseclist.end(), Rel.w.b);
+    ++loc;
+
+    // If there is no next one, don't do anything. The ++Rel.w.c above sets Rel
+    // to the end iterator.
+    if (loc != relocseclist.end()) {
+      Rel.w.b = *loc;
+      Rel.w.a = 0;
+    }
+  }
+  Result = RelocationRef(Rel, this);
+  return object_error::success;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationSymbol(DataRefImpl Rel,
+                                              SymbolRef &Result) const {
+  uint32_t symbolIdx;
+  const Elf_Shdr *sec = getSection(Rel.w.b);
+  switch (sec->sh_type) {
+    default :
+      report_fatal_error("Invalid section type in Rel!");
+    case ELF::SHT_REL : {
+      symbolIdx = getRel(Rel)->getSymbol();
+      break;
+    }
+    case ELF::SHT_RELA : {
+      symbolIdx = getRela(Rel)->getSymbol();
+      break;
+    }
+  }
+  DataRefImpl SymbolData;
+  IndexMap_t::const_iterator it = SymbolTableSectionsIndexMap.find(sec->sh_link);
+  if (it == SymbolTableSectionsIndexMap.end())
+    report_fatal_error("Relocation symbol table not found!");
+  SymbolData.d.a = symbolIdx;
+  SymbolData.d.b = it->second;
+  Result = SymbolRef(SymbolData, this);
+  return object_error::success;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationAddress(DataRefImpl Rel,
+                                               uint64_t &Result) const {
+  uint64_t offset;
+  const Elf_Shdr *sec = getSection(Rel.w.b);
+  switch (sec->sh_type) {
+    default :
+      report_fatal_error("Invalid section type in Rel!");
+    case ELF::SHT_REL : {
+      offset = getRel(Rel)->r_offset;
+      break;
+    }
+    case ELF::SHT_RELA : {
+      offset = getRela(Rel)->r_offset;
+      break;
+    }
+  }
+
+  Result = offset;
+  return object_error::success;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationType(DataRefImpl Rel,
+                                            uint32_t &Result) const {
+  const Elf_Shdr *sec = getSection(Rel.w.b);
+  switch (sec->sh_type) {
+    default :
+      report_fatal_error("Invalid section type in Rel!");
+    case ELF::SHT_REL : {
+      Result = getRel(Rel)->getType();
+      break;
+    }
+    case ELF::SHT_RELA : {
+      Result = getRela(Rel)->getType();
+      break;
+    }
+  }
+  return object_error::success;
+}
+
+#define LLVM_ELF_SWITCH_RELOC_TYPE_NAME(enum) \
+  case ELF::enum: res = #enum; break;
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationTypeName(DataRefImpl Rel,
+                                          SmallVectorImpl<char> &Result) const {
+  const Elf_Shdr *sec = getSection(Rel.w.b);
+  uint8_t type;
+  StringRef res;
+  switch (sec->sh_type) {
+    default :
+      return object_error::parse_failed;
+    case ELF::SHT_REL : {
+      type = getRel(Rel)->getType();
+      break;
+    }
+    case ELF::SHT_RELA : {
+      type = getRela(Rel)->getType();
+      break;
+    }
+  }
+  switch (Header->e_machine) {
+  case ELF::EM_X86_64:
+    switch (type) {
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_NONE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_PC32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GOT32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_PLT32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_COPY);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GLOB_DAT);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_JUMP_SLOT);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_RELATIVE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GOTPCREL);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_32S);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_16);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_PC16);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_8);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_PC8);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_DTPMOD64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_DTPOFF64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_TPOFF64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_TLSGD);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_TLSLD);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_DTPOFF32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GOTTPOFF);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_TPOFF32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_PC64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GOTOFF64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GOTPC32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_SIZE32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_SIZE64);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_GOTPC32_TLSDESC);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_TLSDESC_CALL);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_X86_64_TLSDESC);
+    default:
+      res = "Unknown";
+    }
+    break;
+  case ELF::EM_386:
+    switch (type) {
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_NONE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_PC32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_GOT32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_PLT32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_COPY);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_GLOB_DAT);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_JUMP_SLOT);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_RELATIVE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_GOTOFF);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_GOTPC);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_32PLT);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_TPOFF);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_IE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GOTIE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LE);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GD);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LDM);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_16);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_PC16);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_8);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_PC8);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GD_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GD_PUSH);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GD_CALL);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GD_POP);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LDM_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LDM_PUSH);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LDM_CALL);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LDM_POP);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LDO_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_IE_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_LE_32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_DTPMOD32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_DTPOFF32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_TPOFF32);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_GOTDESC);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_DESC_CALL);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_TLS_DESC);
+      LLVM_ELF_SWITCH_RELOC_TYPE_NAME(R_386_IRELATIVE);
+    default:
+      res = "Unknown";
+    }
+    break;
+  default:
+    res = "Unknown";
+  }
+  Result.append(res.begin(), res.end());
+  return object_error::success;
+}
+
+#undef LLVM_ELF_SWITCH_RELOC_TYPE_NAME
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                      int64_t &Result) const {
+  const Elf_Shdr *sec = getSection(Rel.w.b);
+  switch (sec->sh_type) {
+    default :
+      report_fatal_error("Invalid section type in Rel!");
+    case ELF::SHT_REL : {
+      Result = 0;
+      return object_error::success;
+    }
+    case ELF::SHT_RELA : {
+      Result = getRela(Rel)->r_addend;
+      return object_error::success;
+    }
+  }
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getRelocationValueString(DataRefImpl Rel,
+                                          SmallVectorImpl<char> &Result) const {
+  const Elf_Shdr *sec = getSection(Rel.w.b);
+  uint8_t type;
+  StringRef res;
+  int64_t addend = 0;
+  uint16_t symbol_index = 0;
+  switch (sec->sh_type) {
+    default :
+      return object_error::parse_failed;
+    case ELF::SHT_REL : {
+      type = getRel(Rel)->getType();
+      symbol_index = getRel(Rel)->getSymbol();
+      // TODO: Read implicit addend from section data.
+      break;
+    }
+    case ELF::SHT_RELA : {
+      type = getRela(Rel)->getType();
+      symbol_index = getRela(Rel)->getSymbol();
+      addend = getRela(Rel)->r_addend;
+      break;
+    }
+  }
+  const Elf_Sym *symb = getEntry<Elf_Sym>(sec->sh_link, symbol_index);
+  StringRef symname;
+  if (error_code ec = getSymbolName(symb, symname))
+    return ec;
+  switch (Header->e_machine) {
+  case ELF::EM_X86_64:
+    switch (type) {
+    case ELF::R_X86_64_32S:
+      res = symname;
+      break;
+    case ELF::R_X86_64_PC32: {
+        std::string fmtbuf;
+        raw_string_ostream fmt(fmtbuf);
+        fmt << symname << (addend < 0 ? "" : "+") << addend << "-P";
+        fmt.flush();
+        Result.append(fmtbuf.begin(), fmtbuf.end());
+      }
+      break;
+    default:
+      res = "Unknown";
+    }
+    break;
+  default:
+    res = "Unknown";
+  }
+  if (Result.empty())
+    Result.append(res.begin(), res.end());
+  return object_error::success;
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 ELFObjectFile<target_endianness, is64Bits>::ELFObjectFile(MemoryBuffer *Object
                                                           , error_code &ec)
@@ -521,25 +1092,41 @@ ELFObjectFile<target_endianness, is64Bits>::ELFObjectFile(MemoryBuffer *Object
 
   SectionHeaderTable =
     reinterpret_cast<const Elf_Shdr *>(base() + Header->e_shoff);
-  uint32_t SectionTableSize = Header->e_shnum * Header->e_shentsize;
+  uint64_t SectionTableSize = getNumSections() * Header->e_shentsize;
   if (!(  (const uint8_t *)SectionHeaderTable + SectionTableSize
          <= base() + Data->getBufferSize()))
     // FIXME: Proper error handling.
     report_fatal_error("Section table goes past end of file!");
 
 
-  // To find the symbol tables we walk the section table to find SHT_STMTAB.
-  for (const char *i = reinterpret_cast<const char *>(SectionHeaderTable),
-                  *e = i + Header->e_shnum * Header->e_shentsize;
-                   i != e; i += Header->e_shentsize) {
-    const Elf_Shdr *sh = reinterpret_cast<const Elf_Shdr*>(i);
+  // To find the symbol tables we walk the section table to find SHT_SYMTAB.
+  const Elf_Shdr* SymbolTableSectionHeaderIndex = 0;
+  const Elf_Shdr* sh = reinterpret_cast<const Elf_Shdr*>(SectionHeaderTable);
+  for (uint64_t i = 0, e = getNumSections(); i != e; ++i) {
+    if (sh->sh_type == ELF::SHT_SYMTAB_SHNDX) {
+      if (SymbolTableSectionHeaderIndex)
+        // FIXME: Proper error handling.
+        report_fatal_error("More than one .symtab_shndx!");
+      SymbolTableSectionHeaderIndex = sh;
+    }
     if (sh->sh_type == ELF::SHT_SYMTAB) {
+      SymbolTableSectionsIndexMap[i] = SymbolTableSections.size();
       SymbolTableSections.push_back(sh);
     }
+    if (sh->sh_type == ELF::SHT_REL || sh->sh_type == ELF::SHT_RELA) {
+      SectionRelocMap[getSection(sh->sh_info)].push_back(i);
+    }
+    ++sh;
+  }
+
+  // Sort section relocation lists by index.
+  for (typename RelocMap_t::iterator i = SectionRelocMap.begin(),
+                                     e = SectionRelocMap.end(); i != e; ++i) {
+    std::sort(i->second.begin(), i->second.end());
   }
 
   // Get string table sections.
-  dot_shstrtab_sec = getSection(Header->e_shstrndx);
+  dot_shstrtab_sec = getSection(getStringTableIndex());
   if (dot_shstrtab_sec) {
     // Verify that the last byte in the string table in a null.
     if (((const char*)base() + dot_shstrtab_sec->sh_offset)
@@ -550,7 +1137,7 @@ ELFObjectFile<target_endianness, is64Bits>::ELFObjectFile(MemoryBuffer *Object
 
   // Merge this into the above loop.
   for (const char *i = reinterpret_cast<const char *>(SectionHeaderTable),
-                  *e = i + Header->e_shnum * Header->e_shentsize;
+                  *e = i + getNumSections() * Header->e_shentsize;
                    i != e; i += Header->e_shentsize) {
     const Elf_Shdr *sh = reinterpret_cast<const Elf_Shdr*>(i);
     if (sh->sh_type == ELF::SHT_STRTAB) {
@@ -567,11 +1154,26 @@ ELFObjectFile<target_endianness, is64Bits>::ELFObjectFile(MemoryBuffer *Object
       }
     }
   }
+
+  // Build symbol name side-mapping if there is one.
+  if (SymbolTableSectionHeaderIndex) {
+    const Elf_Word *ShndxTable = reinterpret_cast<const Elf_Word*>(base() +
+                                      SymbolTableSectionHeaderIndex->sh_offset);
+    error_code ec;
+    for (symbol_iterator si = begin_symbols(),
+                         se = end_symbols(); si != se; si.increment(ec)) {
+      if (ec)
+        report_fatal_error("Fewer extended symbol table entries than symbols!");
+      if (*ShndxTable != ELF::SHN_UNDEF)
+        ExtendedSymbolTable[getSymbol(si->getRawDataRefImpl())] = *ShndxTable;
+      ++ShndxTable;
+    }
+  }
 }
 
 template<support::endianness target_endianness, bool is64Bits>
-ObjectFile::symbol_iterator ELFObjectFile<target_endianness, is64Bits>
-                                         ::begin_symbols() const {
+symbol_iterator ELFObjectFile<target_endianness, is64Bits>
+                             ::begin_symbols() const {
   DataRefImpl SymbolData;
   memset(&SymbolData, 0, sizeof(SymbolData));
   if (SymbolTableSections.size() == 0) {
@@ -585,8 +1187,8 @@ ObjectFile::symbol_iterator ELFObjectFile<target_endianness, is64Bits>
 }
 
 template<support::endianness target_endianness, bool is64Bits>
-ObjectFile::symbol_iterator ELFObjectFile<target_endianness, is64Bits>
-                                         ::end_symbols() const {
+symbol_iterator ELFObjectFile<target_endianness, is64Bits>
+                             ::end_symbols() const {
   DataRefImpl SymbolData;
   memset(&SymbolData, 0, sizeof(SymbolData));
   SymbolData.d.a = std::numeric_limits<uint32_t>::max();
@@ -595,8 +1197,8 @@ ObjectFile::symbol_iterator ELFObjectFile<target_endianness, is64Bits>
 }
 
 template<support::endianness target_endianness, bool is64Bits>
-ObjectFile::section_iterator ELFObjectFile<target_endianness, is64Bits>
-                                          ::begin_sections() const {
+section_iterator ELFObjectFile<target_endianness, is64Bits>
+                              ::begin_sections() const {
   DataRefImpl ret;
   memset(&ret, 0, sizeof(DataRefImpl));
   ret.p = reinterpret_cast<intptr_t>(base() + Header->e_shoff);
@@ -604,13 +1206,13 @@ ObjectFile::section_iterator ELFObjectFile<target_endianness, is64Bits>
 }
 
 template<support::endianness target_endianness, bool is64Bits>
-ObjectFile::section_iterator ELFObjectFile<target_endianness, is64Bits>
-                                          ::end_sections() const {
+section_iterator ELFObjectFile<target_endianness, is64Bits>
+                              ::end_sections() const {
   DataRefImpl ret;
   memset(&ret, 0, sizeof(DataRefImpl));
   ret.p = reinterpret_cast<intptr_t>(base()
                                      + Header->e_shoff
-                                     + (Header->e_shentsize * Header->e_shnum));
+                                     + (Header->e_shentsize*getNumSections()));
   return section_iterator(SectionRef(ret, this));
 }
 
@@ -629,6 +1231,8 @@ StringRef ELFObjectFile<target_endianness, is64Bits>
       return "ELF32-i386";
     case ELF::EM_X86_64:
       return "ELF32-x86-64";
+    case ELF::EM_ARM:
+      return "ELF32-arm";
     default:
       return "ELF32-unknown";
     }
@@ -654,26 +1258,75 @@ unsigned ELFObjectFile<target_endianness, is64Bits>::getArch() const {
     return Triple::x86;
   case ELF::EM_X86_64:
     return Triple::x86_64;
+  case ELF::EM_ARM:
+    return Triple::arm;
   default:
     return Triple::UnknownArch;
   }
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+uint64_t ELFObjectFile<target_endianness, is64Bits>::getNumSections() const {
+  if (Header->e_shnum == ELF::SHN_UNDEF)
+    return SectionHeaderTable->sh_size;
+  return Header->e_shnum;
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+uint64_t
+ELFObjectFile<target_endianness, is64Bits>::getStringTableIndex() const {
+  if (Header->e_shnum == ELF::SHN_UNDEF) {
+    if (Header->e_shstrndx == ELF::SHN_HIRESERVE)
+      return SectionHeaderTable->sh_link;
+    if (Header->e_shstrndx >= getNumSections())
+      return 0;
+  }
+  return Header->e_shstrndx;
+}
+
+
+template<support::endianness target_endianness, bool is64Bits>
+template<typename T>
+inline const T *
+ELFObjectFile<target_endianness, is64Bits>::getEntry(uint16_t Section,
+                                                     uint32_t Entry) const {
+  return getEntry<T>(getSection(Section), Entry);
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+template<typename T>
+inline const T *
+ELFObjectFile<target_endianness, is64Bits>::getEntry(const Elf_Shdr * Section,
+                                                     uint32_t Entry) const {
+  return reinterpret_cast<const T *>(
+           base()
+           + Section->sh_offset
+           + (Entry * Section->sh_entsize));
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 const typename ELFObjectFile<target_endianness, is64Bits>::Elf_Sym *
 ELFObjectFile<target_endianness, is64Bits>::getSymbol(DataRefImpl Symb) const {
-  const Elf_Shdr *sec = SymbolTableSections[Symb.d.b];
-  return reinterpret_cast<const Elf_Sym *>(
-           base()
-           + sec->sh_offset
-           + (Symb.d.a * sec->sh_entsize));
+  return getEntry<Elf_Sym>(SymbolTableSections[Symb.d.b], Symb.d.a);
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+const typename ELFObjectFile<target_endianness, is64Bits>::Elf_Rel *
+ELFObjectFile<target_endianness, is64Bits>::getRel(DataRefImpl Rel) const {
+  return getEntry<Elf_Rel>(Rel.w.b, Rel.w.c);
+}
+
+template<support::endianness target_endianness, bool is64Bits>
+const typename ELFObjectFile<target_endianness, is64Bits>::Elf_Rela *
+ELFObjectFile<target_endianness, is64Bits>::getRela(DataRefImpl Rela) const {
+  return getEntry<Elf_Rela>(Rela.w.b, Rela.w.c);
 }
 
 template<support::endianness target_endianness, bool is64Bits>
 const typename ELFObjectFile<target_endianness, is64Bits>::Elf_Shdr *
 ELFObjectFile<target_endianness, is64Bits>::getSection(DataRefImpl Symb) const {
   const Elf_Shdr *sec = getSection(Symb.d.b);
-  if (sec->sh_type != ELF::SHT_SYMTAB)
+  if (sec->sh_type != ELF::SHT_SYMTAB || sec->sh_type != ELF::SHT_DYNSYM)
     // FIXME: Proper error handling.
     report_fatal_error("Invalid symbol table section!");
   return sec;
@@ -681,10 +1334,10 @@ ELFObjectFile<target_endianness, is64Bits>::getSection(DataRefImpl Symb) const {
 
 template<support::endianness target_endianness, bool is64Bits>
 const typename ELFObjectFile<target_endianness, is64Bits>::Elf_Shdr *
-ELFObjectFile<target_endianness, is64Bits>::getSection(uint16_t index) const {
-  if (index == 0 || index >= ELF::SHN_LORESERVE)
+ELFObjectFile<target_endianness, is64Bits>::getSection(uint32_t index) const {
+  if (index == 0)
     return 0;
-  if (!SectionHeaderTable || index >= Header->e_shnum)
+  if (!SectionHeaderTable || index >= getNumSections())
     // FIXME: Proper error handling.
     report_fatal_error("Invalid section index!");
 
@@ -695,7 +1348,7 @@ ELFObjectFile<target_endianness, is64Bits>::getSection(uint16_t index) const {
 
 template<support::endianness target_endianness, bool is64Bits>
 const char *ELFObjectFile<target_endianness, is64Bits>
-                         ::getString(uint16_t section,
+                         ::getString(uint32_t section,
                                      ELF::Elf32_Word offset) const {
   return getString(getSection(section), offset);
 }
@@ -711,6 +1364,24 @@ const char *ELFObjectFile<target_endianness, is64Bits>
   return (const char *)base() + section->sh_offset + offset;
 }
 
+template<support::endianness target_endianness, bool is64Bits>
+error_code ELFObjectFile<target_endianness, is64Bits>
+                        ::getSymbolName(const Elf_Sym *symb,
+                                        StringRef &Result) const {
+  if (symb->st_name == 0) {
+    const Elf_Shdr *section = getSection(symb);
+    if (!section)
+      Result = "";
+    else
+      Result = getString(dot_shstrtab_sec, section->sh_name);
+    return object_error::success;
+  }
+
+  // Use the default symbol table name section.
+  Result = getString(dot_strtab_sec, symb->st_name);
+  return object_error::success;
+}
+
 // EI_CLASS, EI_DATA.
 static std::pair<unsigned char, unsigned char>
 getElfArchType(MemoryBuffer *Object) {
diff --git a/contrib/llvm/lib/Object/MachOObject.cpp b/contrib/llvm/lib/Object/MachOObject.cpp
index 9890febfb616..9cdac8681ddd 100644
--- a/contrib/llvm/lib/Object/MachOObject.cpp
+++ b/contrib/llvm/lib/Object/MachOObject.cpp
@@ -9,6 +9,7 @@
 
 #include "llvm/Object/MachOObject.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/SwapByteOrder.h"
@@ -243,6 +244,18 @@ void MachOObject::ReadDysymtabLoadCommand(const LoadCommandInfo &LCI,
   ReadInMemoryStruct(*this, Buffer->getBuffer(), LCI.Offset, Res);
 }
 
+template<>
+void SwapStruct(macho::LinkeditDataLoadCommand &Value) {
+  SwapValue(Value.Type);
+  SwapValue(Value.Size);
+  SwapValue(Value.DataOffset);
+  SwapValue(Value.DataSize);
+}
+void MachOObject::ReadLinkeditDataLoadCommand(const LoadCommandInfo &LCI,
+                    InMemoryStruct<macho::LinkeditDataLoadCommand> &Res) const {
+  ReadInMemoryStruct(*this, Buffer->getBuffer(), LCI.Offset, Res);
+}
+
 template<>
 void SwapStruct(macho::IndirectSymbolTableEntry &Value) {
   SwapValue(Value.Index);
@@ -343,6 +356,31 @@ void MachOObject::ReadSymbol64TableEntry(uint64_t SymbolTableOffset,
   ReadInMemoryStruct(*this, Buffer->getBuffer(), Offset, Res);
 }
 
+
+void MachOObject::ReadULEB128s(uint64_t Index,
+                               SmallVectorImpl<uint64_t> &Out) const {
+  const char *ptr = Buffer->getBufferStart() + Index;
+  uint64_t data = 0;
+  uint64_t delta = 0;
+  uint32_t shift = 0;
+  while (true) {
+    assert(ptr < Buffer->getBufferEnd() && "index out of bounds");
+    assert(shift < 64 && "too big for uint64_t");
+
+    uint8_t byte = *ptr++;
+    delta |= ((byte & 0x7F) << shift);
+    shift += 7;
+    if (byte < 0x80) {
+      if (delta == 0)
+        break;
+      data += delta;
+      Out.push_back(data);
+      delta = 0;
+      shift = 0;
+    }
+  }
+}
+
 /* ** */
 // Object Dumping Facilities
 void MachOObject::dump() const { print(dbgs()); dbgs() << '\n'; }
diff --git a/contrib/llvm/lib/Object/MachOObjectFile.cpp b/contrib/llvm/lib/Object/MachOObjectFile.cpp
index 26a6e136d753..507df5865eb1 100644
--- a/contrib/llvm/lib/Object/MachOObjectFile.cpp
+++ b/contrib/llvm/lib/Object/MachOObjectFile.cpp
@@ -13,11 +13,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/Triple.h"
+#include "llvm/Object/MachO.h"
 #include "llvm/Object/MachOFormat.h"
-#include "llvm/Object/MachOObject.h"
-#include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/MachO.h"
 
 #include <cctype>
 #include <cstring>
@@ -27,56 +25,24 @@ using namespace llvm;
 using namespace object;
 
 namespace llvm {
+namespace object {
 
-typedef MachOObject::LoadCommandInfo LoadCommandInfo;
-
-class MachOObjectFile : public ObjectFile {
-public:
-  MachOObjectFile(MemoryBuffer *Object, MachOObject *MOO, error_code &ec)
+MachOObjectFile::MachOObjectFile(MemoryBuffer *Object, MachOObject *MOO,
+                                 error_code &ec)
     : ObjectFile(Binary::isMachO, Object, ec),
       MachOObj(MOO),
-      RegisteredStringTable(std::numeric_limits<uint32_t>::max()) {}
+      RegisteredStringTable(std::numeric_limits<uint32_t>::max()) {
+  DataRefImpl DRI;
+  DRI.d.a = DRI.d.b = 0;
+  moveToNextSection(DRI);
+  uint32_t LoadCommandCount = MachOObj->getHeader().NumLoadCommands;
+  while (DRI.d.a < LoadCommandCount) {
+    Sections.push_back(DRI);
+    DRI.d.b++;
+    moveToNextSection(DRI);
+  }
+}
 
-  virtual symbol_iterator begin_symbols() const;
-  virtual symbol_iterator end_symbols() const;
-  virtual section_iterator begin_sections() const;
-  virtual section_iterator end_sections() const;
-
-  virtual uint8_t getBytesInAddress() const;
-  virtual StringRef getFileFormatName() const;
-  virtual unsigned getArch() const;
-
-protected:
-  virtual error_code getSymbolNext(DataRefImpl Symb, SymbolRef &Res) const;
-  virtual error_code getSymbolName(DataRefImpl Symb, StringRef &Res) const;
-  virtual error_code getSymbolAddress(DataRefImpl Symb, uint64_t &Res) const;
-  virtual error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const;
-  virtual error_code getSymbolNMTypeChar(DataRefImpl Symb, char &Res) const;
-  virtual error_code isSymbolInternal(DataRefImpl Symb, bool &Res) const;
-
-  virtual error_code getSectionNext(DataRefImpl Sec, SectionRef &Res) const;
-  virtual error_code getSectionName(DataRefImpl Sec, StringRef &Res) const;
-  virtual error_code getSectionAddress(DataRefImpl Sec, uint64_t &Res) const;
-  virtual error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const;
-  virtual error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const;
-  virtual error_code isSectionText(DataRefImpl Sec, bool &Res) const;
-  virtual error_code sectionContainsSymbol(DataRefImpl DRI, DataRefImpl S,
-                                           bool &Result) const;
-
-private:
-  MachOObject *MachOObj;
-  mutable uint32_t RegisteredStringTable;
-
-  void moveToNextSection(DataRefImpl &DRI) const;
-  void getSymbolTableEntry(DataRefImpl DRI,
-                           InMemoryStruct<macho::SymbolTableEntry> &Res) const;
-  void getSymbol64TableEntry(DataRefImpl DRI,
-                          InMemoryStruct<macho::Symbol64TableEntry> &Res) const;
-  void moveToNextSymbol(DataRefImpl &DRI) const;
-  void getSection(DataRefImpl DRI, InMemoryStruct<macho::Section> &Res) const;
-  void getSection64(DataRefImpl DRI,
-                    InMemoryStruct<macho::Section64> &Res) const;
-};
 
 ObjectFile *ObjectFile::createMachOObjectFile(MemoryBuffer *Buffer) {
   error_code ec;
@@ -158,6 +124,27 @@ error_code MachOObjectFile::getSymbolName(DataRefImpl DRI,
   return object_error::success;
 }
 
+error_code MachOObjectFile::getSymbolOffset(DataRefImpl DRI,
+                                             uint64_t &Result) const {
+  uint64_t SectionOffset;
+  uint8_t SectionIndex;
+  if (MachOObj->is64Bit()) {
+    InMemoryStruct<macho::Symbol64TableEntry> Entry;
+    getSymbol64TableEntry(DRI, Entry);
+    Result = Entry->Value;
+    SectionIndex = Entry->SectionIndex;
+  } else {
+    InMemoryStruct<macho::SymbolTableEntry> Entry;
+    getSymbolTableEntry(DRI, Entry);
+    Result = Entry->Value;
+    SectionIndex = Entry->SectionIndex;
+  }
+  getSectionAddress(Sections[SectionIndex-1], SectionOffset);
+  Result -= SectionOffset;
+
+  return object_error::success;
+}
+
 error_code MachOObjectFile::getSymbolAddress(DataRefImpl DRI,
                                              uint64_t &Result) const {
   if (MachOObj->is64Bit()) {
@@ -227,7 +214,51 @@ error_code MachOObjectFile::isSymbolInternal(DataRefImpl DRI,
   return object_error::success;
 }
 
-ObjectFile::symbol_iterator MachOObjectFile::begin_symbols() const {
+error_code MachOObjectFile::isSymbolGlobal(DataRefImpl Symb, bool &Res) const {
+
+  if (MachOObj->is64Bit()) {
+    InMemoryStruct<macho::Symbol64TableEntry> Entry;
+    getSymbol64TableEntry(Symb, Entry);
+    Res = Entry->Type & MachO::NlistMaskExternal;
+  } else {
+    InMemoryStruct<macho::SymbolTableEntry> Entry;
+    getSymbolTableEntry(Symb, Entry);
+    Res = Entry->Type & MachO::NlistMaskExternal;
+  }
+  return object_error::success;
+}
+
+error_code MachOObjectFile::getSymbolType(DataRefImpl Symb,
+                                          SymbolRef::SymbolType &Res) const {
+  uint8_t n_type;
+  if (MachOObj->is64Bit()) {
+    InMemoryStruct<macho::Symbol64TableEntry> Entry;
+    getSymbol64TableEntry(Symb, Entry);
+    n_type = Entry->Type;
+  } else {
+    InMemoryStruct<macho::SymbolTableEntry> Entry;
+    getSymbolTableEntry(Symb, Entry);
+    n_type = Entry->Type;
+  }
+  Res = SymbolRef::ST_Other;
+
+  // If this is a STAB debugging symbol, we can do nothing more.
+  if (n_type & MachO::NlistMaskStab)
+    return object_error::success;
+
+  switch (n_type & MachO::NlistMaskType) {
+    case MachO::NListTypeUndefined :
+      Res = SymbolRef::ST_External;
+      break;
+    case MachO::NListTypeSection :
+      Res = SymbolRef::ST_Function;
+      break;
+  }
+  return object_error::success;
+}
+
+
+symbol_iterator MachOObjectFile::begin_symbols() const {
   // DRI.d.a = segment number; DRI.d.b = symbol index.
   DataRefImpl DRI;
   DRI.d.a = DRI.d.b = 0;
@@ -235,7 +266,7 @@ ObjectFile::symbol_iterator MachOObjectFile::begin_symbols() const {
   return symbol_iterator(SymbolRef(DRI, this));
 }
 
-ObjectFile::symbol_iterator MachOObjectFile::end_symbols() const {
+symbol_iterator MachOObjectFile::end_symbols() const {
   DataRefImpl DRI;
   DRI.d.a = MachOObj->getHeader().NumLoadCommands;
   DRI.d.b = 0;
@@ -283,6 +314,13 @@ MachOObjectFile::getSection(DataRefImpl DRI,
   MachOObj->ReadSection(LCI, DRI.d.b, Res);
 }
 
+std::size_t MachOObjectFile::getSectionIndex(DataRefImpl Sec) const {
+  SectionList::const_iterator loc =
+    std::find(Sections.begin(), Sections.end(), Sec);
+  assert(loc != Sections.end() && "Sec is not a valid section!");
+  return std::distance(Sections.begin(), loc);
+}
+
 void
 MachOObjectFile::getSection64(DataRefImpl DRI,
                             InMemoryStruct<macho::Section64> &Res) const {
@@ -371,6 +409,20 @@ error_code MachOObjectFile::getSectionContents(DataRefImpl DRI,
   return object_error::success;
 }
 
+error_code MachOObjectFile::getSectionAlignment(DataRefImpl DRI,
+                                                uint64_t &Result) const {
+  if (is64BitLoadCommand(MachOObj, DRI)) {
+    InMemoryStruct<macho::Section64> Sect;
+    getSection64(DRI, Sect);
+    Result = uint64_t(1) << Sect->Align;
+  } else {
+    InMemoryStruct<macho::Section> Sect;
+    getSection(DRI, Sect);
+    Result = uint64_t(1) << Sect->Align;
+  }
+  return object_error::success;
+}
+
 error_code MachOObjectFile::isSectionText(DataRefImpl DRI,
                                           bool &Result) const {
   if (is64BitLoadCommand(MachOObj, DRI)) {
@@ -385,35 +437,185 @@ error_code MachOObjectFile::isSectionText(DataRefImpl DRI,
   return object_error::success;
 }
 
-error_code MachOObjectFile::sectionContainsSymbol(DataRefImpl Sec,
-                                                  DataRefImpl Symb,
-                                                  bool &Result) const {
-  if (MachOObj->is64Bit()) {
-    InMemoryStruct<macho::Symbol64TableEntry> Entry;
-    getSymbol64TableEntry(Symb, Entry);
-    Result = Entry->SectionIndex == 1 + Sec.d.a + Sec.d.b;
-  } else {
-    InMemoryStruct<macho::SymbolTableEntry> Entry;
-    getSymbolTableEntry(Symb, Entry);
-    Result = Entry->SectionIndex == 1 + Sec.d.a + Sec.d.b;
-  }
+error_code MachOObjectFile::isSectionData(DataRefImpl DRI,
+                                          bool &Result) const {
+  // FIXME: Unimplemented.
+  Result = false;
   return object_error::success;
 }
 
-ObjectFile::section_iterator MachOObjectFile::begin_sections() const {
+error_code MachOObjectFile::isSectionBSS(DataRefImpl DRI,
+                                         bool &Result) const {
+  // FIXME: Unimplemented.
+  Result = false;
+  return object_error::success;
+}
+
+error_code MachOObjectFile::sectionContainsSymbol(DataRefImpl Sec,
+                                                  DataRefImpl Symb,
+                                                  bool &Result) const {
+  SymbolRef::SymbolType ST;
+  getSymbolType(Symb, ST);
+  if (ST == SymbolRef::ST_External) {
+    Result = false;
+    return object_error::success;
+  }
+
+  uint64_t SectBegin, SectEnd;
+  getSectionAddress(Sec, SectBegin);
+  getSectionSize(Sec, SectEnd);
+  SectEnd += SectBegin;
+
+  if (MachOObj->is64Bit()) {
+    InMemoryStruct<macho::Symbol64TableEntry> Entry;
+    getSymbol64TableEntry(Symb, Entry);
+    uint64_t SymAddr= Entry->Value;
+    Result = (SymAddr >= SectBegin) && (SymAddr < SectEnd);
+  } else {
+    InMemoryStruct<macho::SymbolTableEntry> Entry;
+    getSymbolTableEntry(Symb, Entry);
+    uint64_t SymAddr= Entry->Value;
+    Result = (SymAddr >= SectBegin) && (SymAddr < SectEnd);
+  }
+
+  return object_error::success;
+}
+
+relocation_iterator MachOObjectFile::getSectionRelBegin(DataRefImpl Sec) const {
+  DataRefImpl ret;
+  ret.d.a = 0;
+  ret.d.b = getSectionIndex(Sec);
+  return relocation_iterator(RelocationRef(ret, this));
+}
+relocation_iterator MachOObjectFile::getSectionRelEnd(DataRefImpl Sec) const {
+  uint32_t last_reloc;
+  if (is64BitLoadCommand(MachOObj, Sec)) {
+    InMemoryStruct<macho::Section64> Sect;
+    getSection64(Sec, Sect);
+    last_reloc = Sect->NumRelocationTableEntries;
+  } else {
+    InMemoryStruct<macho::Section> Sect;
+    getSection(Sec, Sect);
+    last_reloc = Sect->NumRelocationTableEntries;
+  }
+  DataRefImpl ret;
+  ret.d.a = last_reloc;
+  ret.d.b = getSectionIndex(Sec);
+  return relocation_iterator(RelocationRef(ret, this));
+}
+
+section_iterator MachOObjectFile::begin_sections() const {
   DataRefImpl DRI;
   DRI.d.a = DRI.d.b = 0;
   moveToNextSection(DRI);
   return section_iterator(SectionRef(DRI, this));
 }
 
-ObjectFile::section_iterator MachOObjectFile::end_sections() const {
+section_iterator MachOObjectFile::end_sections() const {
   DataRefImpl DRI;
   DRI.d.a = MachOObj->getHeader().NumLoadCommands;
   DRI.d.b = 0;
   return section_iterator(SectionRef(DRI, this));
 }
 
+/*===-- Relocations -------------------------------------------------------===*/
+
+void MachOObjectFile::
+getRelocation(DataRefImpl Rel,
+              InMemoryStruct<macho::RelocationEntry> &Res) const {
+  uint32_t relOffset;
+  if (MachOObj->is64Bit()) {
+    InMemoryStruct<macho::Section64> Sect;
+    getSection64(Sections[Rel.d.b], Sect);
+    relOffset = Sect->RelocationTableOffset;
+  } else {
+    InMemoryStruct<macho::Section> Sect;
+    getSection(Sections[Rel.d.b], Sect);
+    relOffset = Sect->RelocationTableOffset;
+  }
+  MachOObj->ReadRelocationEntry(relOffset, Rel.d.a, Res);
+}
+error_code MachOObjectFile::getRelocationNext(DataRefImpl Rel,
+                                              RelocationRef &Res) const {
+  ++Rel.d.a;
+  Res = RelocationRef(Rel, this);
+  return object_error::success;
+}
+error_code MachOObjectFile::getRelocationAddress(DataRefImpl Rel,
+                                                 uint64_t &Res) const {
+  const uint8_t* sectAddress = base();
+  if (MachOObj->is64Bit()) {
+    InMemoryStruct<macho::Section64> Sect;
+    getSection64(Sections[Rel.d.b], Sect);
+    sectAddress += Sect->Offset;
+  } else {
+    InMemoryStruct<macho::Section> Sect;
+    getSection(Sections[Rel.d.b], Sect);
+    sectAddress += Sect->Offset;
+  }
+  InMemoryStruct<macho::RelocationEntry> RE;
+  getRelocation(Rel, RE);
+  Res = reinterpret_cast<uintptr_t>(sectAddress + RE->Word0);
+  return object_error::success;
+}
+error_code MachOObjectFile::getRelocationSymbol(DataRefImpl Rel,
+                                                SymbolRef &Res) const {
+  InMemoryStruct<macho::RelocationEntry> RE;
+  getRelocation(Rel, RE);
+  uint32_t SymbolIdx = RE->Word1 & 0xffffff;
+  bool isExtern = (RE->Word1 >> 27) & 1;
+
+  DataRefImpl Sym;
+  Sym.d.a = Sym.d.b = 0;
+  moveToNextSymbol(Sym);
+  if (isExtern) {
+    for (unsigned i = 0; i < SymbolIdx; i++) {
+      Sym.d.b++;
+      moveToNextSymbol(Sym);
+      assert(Sym.d.a < MachOObj->getHeader().NumLoadCommands &&
+             "Relocation symbol index out of range!");
+    }
+  }
+  Res = SymbolRef(Sym, this);
+  return object_error::success;
+}
+error_code MachOObjectFile::getRelocationType(DataRefImpl Rel,
+                                              uint32_t &Res) const {
+  InMemoryStruct<macho::RelocationEntry> RE;
+  getRelocation(Rel, RE);
+  Res = RE->Word1;
+  return object_error::success;
+}
+error_code MachOObjectFile::getRelocationTypeName(DataRefImpl Rel,
+                                          SmallVectorImpl<char> &Result) const {
+  return object_error::success;
+}
+error_code MachOObjectFile::getRelocationAdditionalInfo(DataRefImpl Rel,
+                                                        int64_t &Res) const {
+  InMemoryStruct<macho::RelocationEntry> RE;
+  getRelocation(Rel, RE);
+  bool isExtern = (RE->Word1 >> 27) & 1;
+  Res = 0;
+  if (!isExtern) {
+    const uint8_t* sectAddress = base();
+    if (MachOObj->is64Bit()) {
+      InMemoryStruct<macho::Section64> Sect;
+      getSection64(Sections[Rel.d.b], Sect);
+      sectAddress += Sect->Offset;
+    } else {
+      InMemoryStruct<macho::Section> Sect;
+      getSection(Sections[Rel.d.b], Sect);
+      sectAddress += Sect->Offset;
+    }
+    Res = reinterpret_cast<uintptr_t>(sectAddress);
+  }
+  return object_error::success;
+}
+error_code MachOObjectFile::getRelocationValueString(DataRefImpl Rel,
+                                          SmallVectorImpl<char> &Result) const {
+  return object_error::success;
+}
+
 /*===-- Miscellaneous -----------------------------------------------------===*/
 
 uint8_t MachOObjectFile::getBytesInAddress() const {
@@ -465,5 +667,5 @@ unsigned MachOObjectFile::getArch() const {
   }
 }
 
+} // end namespace object
 } // end namespace llvm
-
diff --git a/contrib/llvm/lib/Object/Makefile b/contrib/llvm/lib/Object/Makefile
deleted file mode 100644
index 79388dc97f1a..000000000000
--- a/contrib/llvm/lib/Object/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-##===- lib/Object/Makefile ---------------------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../..
-LIBRARYNAME = LLVMObject
-BUILD_ARCHIVE := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Object/Object.cpp b/contrib/llvm/lib/Object/Object.cpp
index 9a373ad21bd2..2ea8db978670 100644
--- a/contrib/llvm/lib/Object/Object.cpp
+++ b/contrib/llvm/lib/Object/Object.cpp
@@ -27,8 +27,8 @@ void LLVMDisposeObjectFile(LLVMObjectFileRef ObjectFile) {
 }
 
 LLVMSectionIteratorRef LLVMGetSections(LLVMObjectFileRef ObjectFile) {
-  ObjectFile::section_iterator SI = unwrap(ObjectFile)->begin_sections();
-  return wrap(new ObjectFile::section_iterator(SI));
+  section_iterator SI = unwrap(ObjectFile)->begin_sections();
+  return wrap(new section_iterator(SI));
 }
 
 void LLVMDisposeSectionIterator(LLVMSectionIteratorRef SI) {
diff --git a/contrib/llvm/lib/Object/ObjectFile.cpp b/contrib/llvm/lib/Object/ObjectFile.cpp
index a7798df33fe5..69d8ed0e5e9a 100644
--- a/contrib/llvm/lib/Object/ObjectFile.cpp
+++ b/contrib/llvm/lib/Object/ObjectFile.cpp
@@ -45,6 +45,7 @@ ObjectFile *ObjectFile::createObjectFile(MemoryBuffer *Object) {
     case sys::Mach_O_DynamicLinker_FileType:
     case sys::Mach_O_Bundle_FileType:
     case sys::Mach_O_DynamicallyLinkedSharedLibStub_FileType:
+    case sys::Mach_O_DSYMCompanion_FileType:
       return createMachOObjectFile(Object);
     case sys::COFF_FileType:
       return createCOFFObjectFile(Object);
diff --git a/contrib/llvm/lib/Support/APFloat.cpp b/contrib/llvm/lib/Support/APFloat.cpp
index c64da6e137ea..f2388944929b 100644
--- a/contrib/llvm/lib/Support/APFloat.cpp
+++ b/contrib/llvm/lib/Support/APFloat.cpp
@@ -832,6 +832,7 @@ APFloat::incrementSignificand()
 
   /* Our callers should never cause us to overflow.  */
   assert(carry == 0);
+  (void)carry;
 }
 
 /* Add the significand of the RHS.  Returns the carry flag.  */
@@ -926,6 +927,7 @@ APFloat::multiplySignificand(const APFloat &rhs, const APFloat *addend)
     APFloat extendedAddend(*addend);
     status = extendedAddend.convert(extendedSemantics, rmTowardZero, &ignored);
     assert(status == opOK);
+    (void)status;
     lost_fraction = addOrSubtractSignificand(extendedAddend, false);
 
     /* Restore our state.  */
@@ -1190,7 +1192,7 @@ APFloat::normalize(roundingMode rounding_mode,
 
   if (omsb) {
     /* OMSB is numbered from 1.  We want to place it in the integer
-       bit numbered PRECISON if possible, with a compensating change in
+       bit numbered PRECISION if possible, with a compensating change in
        the exponent.  */
     exponentChange = omsb - semantics->precision;
 
@@ -1389,6 +1391,7 @@ APFloat::addOrSubtractSignificand(const APFloat &rhs, bool subtract)
     /* The code above is intended to ensure that no borrow is
        necessary.  */
     assert(!carry);
+    (void)carry;
   } else {
     if (bits > 0) {
       APFloat temp_rhs(rhs);
@@ -1402,6 +1405,7 @@ APFloat::addOrSubtractSignificand(const APFloat &rhs, bool subtract)
 
     /* We have a guard bit; generating a carry cannot happen.  */
     assert(!carry);
+    (void)carry;
   }
 
   return lost_fraction;
@@ -2098,7 +2102,7 @@ APFloat::convertToInteger(APSInt &result,
   opStatus status = convertToInteger(
     parts.data(), bitWidth, result.isSigned(), rounding_mode, isExact);
   // Keeps the original signed-ness.
-  result = APInt(bitWidth, (unsigned)parts.size(), parts.data());
+  result = APInt(bitWidth, parts);
   return status;
 }
 
@@ -2121,7 +2125,7 @@ APFloat::convertFromUnsignedParts(const integerPart *src,
   dstCount = partCount();
   precision = semantics->precision;
 
-  /* We want the most significant PRECISON bits of SRC.  There may not
+  /* We want the most significant PRECISION bits of SRC.  There may not
      be that many; extract what we can.  */
   if (precision <= omsb) {
     exponent = omsb - 1;
@@ -2192,7 +2196,7 @@ APFloat::convertFromZeroExtendedInteger(const integerPart *parts,
                                         roundingMode rounding_mode)
 {
   unsigned int partCount = partCountForBits(width);
-  APInt api = APInt(width, partCount, parts);
+  APInt api = APInt(width, makeArrayRef(parts, partCount));
 
   sign = false;
   if (isSigned && APInt::tcExtractBit(parts, width - 1)) {
@@ -2746,7 +2750,7 @@ APFloat::convertF80LongDoubleAPFloatToAPInt() const
   words[0] = mysignificand;
   words[1] =  ((uint64_t)(sign & 1) << 15) |
               (myexponent & 0x7fffLL);
-  return APInt(80, 2, words);
+  return APInt(80, words);
 }
 
 APInt
@@ -2791,7 +2795,7 @@ APFloat::convertPPCDoubleDoubleAPFloatToAPInt() const
   words[1] =  ((uint64_t)(sign2 & 1) << 63) |
               ((myexponent2 & 0x7ff) <<  52) |
               (mysignificand2 & 0xfffffffffffffLL);
-  return APInt(128, 2, words);
+  return APInt(128, words);
 }
 
 APInt
@@ -2827,7 +2831,7 @@ APFloat::convertQuadrupleAPFloatToAPInt() const
              ((myexponent & 0x7fff) << 48) |
              (mysignificand2 & 0xffffffffffffLL);
 
-  return APInt(128, 2, words);
+  return APInt(128, words);
 }
 
 APInt
@@ -3239,8 +3243,9 @@ APFloat APFloat::getLargest(const fltSemantics &Sem, bool Negative) {
     significand[i] = ~((integerPart) 0);
 
   // ...and then clear the top bits for internal consistency.
-  significand[N-1] &=
-    (((integerPart) 1) << ((Sem.precision % integerPartWidth) - 1)) - 1;
+  if (Sem.precision % integerPartWidth != 0)
+    significand[N-1] &=
+      (((integerPart) 1) << (Sem.precision % integerPartWidth)) - 1;
 
   return Val;
 }
@@ -3270,7 +3275,7 @@ APFloat APFloat::getSmallestNormalized(const fltSemantics &Sem, bool Negative) {
   Val.exponent = Sem.minExponent;
   Val.zeroSignificand();
   Val.significandParts()[partCountForBits(Sem.precision)-1] |=
-    (((integerPart) 1) << ((Sem.precision % integerPartWidth) - 1));
+    (((integerPart) 1) << ((Sem.precision - 1) % integerPartWidth));
 
   return Val;
 }
@@ -3413,8 +3418,8 @@ void APFloat::toString(SmallVectorImpl<char> &Str,
   // Decompose the number into an APInt and an exponent.
   int exp = exponent - ((int) semantics->precision - 1);
   APInt significand(semantics->precision,
-                    partCountForBits(semantics->precision),
-                    significandParts());
+                    makeArrayRef(significandParts(),
+                                 partCountForBits(semantics->precision)));
 
   // Set FormatPrecision if zero.  We want to do this before we
   // truncate trailing zeros, as those are part of the precision.
@@ -3451,7 +3456,7 @@ void APFloat::toString(SmallVectorImpl<char> &Str,
     //                 <= semantics->precision + e * 137 / 59
     //   (log_2(5) ~ 2.321928 < 2.322034 ~ 137/59)
 
-    unsigned precision = semantics->precision + 137 * texp / 59;
+    unsigned precision = semantics->precision + (137 * texp + 136) / 59;
 
     // Multiply significand by 5^e.
     //   N * 5^0101 == N * 5^(1*1) * 5^(0*2) * 5^(1*4) * 5^(0*8)
diff --git a/contrib/llvm/lib/Support/APInt.cpp b/contrib/llvm/lib/Support/APInt.cpp
index 76265d445f45..3774c5223c46 100644
--- a/contrib/llvm/lib/Support/APInt.cpp
+++ b/contrib/llvm/lib/Support/APInt.cpp
@@ -48,18 +48,20 @@ inline static uint64_t* getMemory(unsigned numWords) {
 inline static unsigned getDigit(char cdigit, uint8_t radix) {
   unsigned r;
 
-  if (radix == 16) {
+  if (radix == 16 || radix == 36) {
     r = cdigit - '0';
     if (r <= 9)
       return r;
 
     r = cdigit - 'A';
-    if (r <= 5)
+    if (r <= radix - 11U)
       return r + 10;
 
     r = cdigit - 'a';
-    if (r <= 5)
+    if (r <= radix - 11U)
       return r + 10;
+    
+    radix = 10;
   }
 
   r = cdigit - '0';
@@ -83,25 +85,33 @@ void APInt::initSlowCase(const APInt& that) {
   memcpy(pVal, that.pVal, getNumWords() * APINT_WORD_SIZE);
 }
 
-
-APInt::APInt(unsigned numBits, unsigned numWords, const uint64_t bigVal[])
-  : BitWidth(numBits), VAL(0) {
+void APInt::initFromArray(ArrayRef<uint64_t> bigVal) {
   assert(BitWidth && "Bitwidth too small");
-  assert(bigVal && "Null pointer detected!");
+  assert(bigVal.data() && "Null pointer detected!");
   if (isSingleWord())
     VAL = bigVal[0];
   else {
     // Get memory, cleared to 0
     pVal = getClearedMemory(getNumWords());
     // Calculate the number of words to copy
-    unsigned words = std::min<unsigned>(numWords, getNumWords());
+    unsigned words = std::min<unsigned>(bigVal.size(), getNumWords());
     // Copy the words from bigVal to pVal
-    memcpy(pVal, bigVal, words * APINT_WORD_SIZE);
+    memcpy(pVal, bigVal.data(), words * APINT_WORD_SIZE);
   }
   // Make sure unused high bits are cleared
   clearUnusedBits();
 }
 
+APInt::APInt(unsigned numBits, ArrayRef<uint64_t> bigVal)
+  : BitWidth(numBits), VAL(0) {
+  initFromArray(bigVal);
+}
+
+APInt::APInt(unsigned numBits, unsigned numWords, const uint64_t bigVal[])
+  : BitWidth(numBits), VAL(0) {
+  initFromArray(makeArrayRef(bigVal, numWords));
+}
+
 APInt::APInt(unsigned numbits, StringRef Str, uint8_t radix)
   : BitWidth(numbits), VAL(0) {
   assert(BitWidth && "Bitwidth too small");
@@ -376,6 +386,7 @@ APInt& APInt::operator*=(const APInt& RHS) {
   clearAllBits();
   unsigned wordsToCopy = destWords >= getNumWords() ? getNumWords() : destWords;
   memcpy(pVal, dest, wordsToCopy * APINT_WORD_SIZE);
+  clearUnusedBits();
 
   // delete dest array and return
   delete[] dest;
@@ -461,7 +472,7 @@ APInt APInt::operator*(const APInt& RHS) const {
     return APInt(BitWidth, VAL * RHS.VAL);
   APInt Result(*this);
   Result *= RHS;
-  return Result.clearUnusedBits();
+  return Result;
 }
 
 APInt APInt::operator+(const APInt& RHS) const {
@@ -613,8 +624,9 @@ void APInt::flipBit(unsigned bitPosition) {
 
 unsigned APInt::getBitsNeeded(StringRef str, uint8_t radix) {
   assert(!str.empty() && "Invalid string length");
-  assert((radix == 10 || radix == 8 || radix == 16 || radix == 2) &&
-         "Radix should be 2, 8, 10, or 16!");
+  assert((radix == 10 || radix == 8 || radix == 16 || radix == 2 || 
+          radix == 36) &&
+         "Radix should be 2, 8, 10, 16, or 36!");
 
   size_t slen = str.size();
 
@@ -636,6 +648,8 @@ unsigned APInt::getBitsNeeded(StringRef str, uint8_t radix) {
   if (radix == 16)
     return slen * 4 + isNegative;
 
+  // FIXME: base 36
+  
   // This is grossly inefficient but accurate. We could probably do something
   // with a computation of roughly slen*64/20 and then adjust by the value of
   // the first few digits. But, I'm not sure how accurate that could be.
@@ -644,7 +658,9 @@ unsigned APInt::getBitsNeeded(StringRef str, uint8_t radix) {
   // be too large. This avoids the assertion in the constructor. This
   // calculation doesn't work appropriately for the numbers 0-9, so just use 4
   // bits in that case.
-  unsigned sufficient = slen == 1 ? 4 : slen * 64/18;
+  unsigned sufficient 
+    = radix == 10? (slen == 1 ? 4 : slen * 64/18)
+                 : (slen == 1 ? 7 : slen * 16/3);
 
   // Convert to the actual binary value.
   APInt tmp(sufficient, StringRef(p, slen), radix);
@@ -2107,8 +2123,9 @@ APInt APInt::sshl_ov(unsigned ShAmt, bool &Overflow) const {
 void APInt::fromString(unsigned numbits, StringRef str, uint8_t radix) {
   // Check our assumptions here
   assert(!str.empty() && "Invalid string length");
-  assert((radix == 10 || radix == 8 || radix == 16 || radix == 2) &&
-         "Radix should be 2, 8, 10, or 16!");
+  assert((radix == 10 || radix == 8 || radix == 16 || radix == 2 || 
+          radix == 36) &&
+         "Radix should be 2, 8, 10, 16, or 36!");
 
   StringRef::iterator p = str.begin();
   size_t slen = str.size();
@@ -2165,7 +2182,8 @@ void APInt::fromString(unsigned numbits, StringRef str, uint8_t radix) {
 
 void APInt::toString(SmallVectorImpl<char> &Str, unsigned Radix,
                      bool Signed, bool formatAsCLiteral) const {
-  assert((Radix == 10 || Radix == 8 || Radix == 16 || Radix == 2) &&
+  assert((Radix == 10 || Radix == 8 || Radix == 16 || Radix == 2 || 
+          Radix == 36) &&
          "Radix should be 2, 8, 10, or 16!");
 
   const char *Prefix = "";
@@ -2195,7 +2213,7 @@ void APInt::toString(SmallVectorImpl<char> &Str, unsigned Radix,
     return;
   }
 
-  static const char Digits[] = "0123456789ABCDEF";
+  static const char Digits[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
   if (isSingleWord()) {
     char Buffer[65];
@@ -2249,7 +2267,7 @@ void APInt::toString(SmallVectorImpl<char> &Str, unsigned Radix,
   // For the 2, 8 and 16 bit cases, we can just shift instead of divide
   // because the number of bits per digit (1, 3 and 4 respectively) divides
   // equaly.  We just shift until the value is zero.
-  if (Radix != 10) {
+  if (Radix == 2 || Radix == 8 || Radix == 16) {
     // Just shift tmp right for each digit width until it becomes zero
     unsigned ShiftAmt = (Radix == 16 ? 4 : (Radix == 8 ? 3 : 1));
     unsigned MaskAmt = Radix - 1;
@@ -2260,7 +2278,7 @@ void APInt::toString(SmallVectorImpl<char> &Str, unsigned Radix,
       Tmp = Tmp.lshr(ShiftAmt);
     }
   } else {
-    APInt divisor(4, 10);
+    APInt divisor(Radix == 10? 4 : 8, Radix);
     while (Tmp != 0) {
       APInt APdigit(1, 0);
       APInt tmp2(Tmp.getBitWidth(), 0);
diff --git a/contrib/llvm/lib/Support/Atomic.cpp b/contrib/llvm/lib/Support/Atomic.cpp
index 8214521f9827..94760cc069fc 100644
--- a/contrib/llvm/lib/Support/Atomic.cpp
+++ b/contrib/llvm/lib/Support/Atomic.cpp
@@ -22,7 +22,7 @@ using namespace llvm;
 #endif
 
 void sys::MemoryFence() {
-#if !defined(LLVM_MULTITHREADED) || LLVM_MULTITHREADED == 0
+#if LLVM_HAS_ATOMICS == 0
   return;
 #else
 #  if defined(__GNUC__)
@@ -38,7 +38,7 @@ void sys::MemoryFence() {
 sys::cas_flag sys::CompareAndSwap(volatile sys::cas_flag* ptr,
                                   sys::cas_flag new_value,
                                   sys::cas_flag old_value) {
-#if !defined(LLVM_MULTITHREADED) || LLVM_MULTITHREADED == 0
+#if LLVM_HAS_ATOMICS == 0
   sys::cas_flag result = *ptr;
   if (result == old_value)
     *ptr = new_value;
@@ -53,7 +53,7 @@ sys::cas_flag sys::CompareAndSwap(volatile sys::cas_flag* ptr,
 }
 
 sys::cas_flag sys::AtomicIncrement(volatile sys::cas_flag* ptr) {
-#if !defined(LLVM_MULTITHREADED) || LLVM_MULTITHREADED == 0
+#if LLVM_HAS_ATOMICS == 0
   ++(*ptr);
   return *ptr;
 #elif defined(__GNUC__)
@@ -66,7 +66,7 @@ sys::cas_flag sys::AtomicIncrement(volatile sys::cas_flag* ptr) {
 }
 
 sys::cas_flag sys::AtomicDecrement(volatile sys::cas_flag* ptr) {
-#if !defined(LLVM_MULTITHREADED) || LLVM_MULTITHREADED == 0
+#if LLVM_HAS_ATOMICS == 0
   --(*ptr);
   return *ptr;
 #elif defined(__GNUC__)
@@ -79,7 +79,7 @@ sys::cas_flag sys::AtomicDecrement(volatile sys::cas_flag* ptr) {
 }
 
 sys::cas_flag sys::AtomicAdd(volatile sys::cas_flag* ptr, sys::cas_flag val) {
-#if !defined(LLVM_MULTITHREADED) || LLVM_MULTITHREADED == 0
+#if LLVM_HAS_ATOMICS == 0
   *ptr += val;
   return *ptr;
 #elif defined(__GNUC__)
diff --git a/contrib/llvm/lib/Support/BlockFrequency.cpp b/contrib/llvm/lib/Support/BlockFrequency.cpp
new file mode 100644
index 000000000000..a63bf83f2039
--- /dev/null
+++ b/contrib/llvm/lib/Support/BlockFrequency.cpp
@@ -0,0 +1,126 @@
+//====--------------- lib/Support/BlockFrequency.cpp -----------*- C++ -*-====//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements Block Frequency class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/BranchProbability.h"
+#include "llvm/Support/BlockFrequency.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+
+using namespace llvm;
+
+namespace {
+
+/// mult96bit - Multiply FREQ by N and store result in W array.
+void mult96bit(uint64_t freq, uint32_t N, uint64_t W[2]) {
+  uint64_t u0 = freq & UINT32_MAX;
+  uint64_t u1 = freq >> 32;
+
+  // Represent 96-bit value as w[2]:w[1]:w[0];
+  uint32_t w[3] = { 0, 0, 0 };
+
+  uint64_t t = u0 * N;
+  uint64_t k = t >> 32;
+  w[0] = t;
+  t = u1 * N + k;
+  w[1] = t;
+  w[2] = t >> 32;
+
+  // W[1] - higher bits.
+  // W[0] - lower bits.
+  W[0] = w[0] + ((uint64_t) w[1] << 32);
+  W[1] = w[2];
+}
+
+
+/// div96bit - Divide 96-bit value stored in W array by D. Return 64-bit frequency.
+uint64_t div96bit(uint64_t W[2], uint32_t D) {
+  uint64_t y = W[0];
+  uint64_t x = W[1];
+  int i;
+
+  for (i = 1; i <= 64 && x; ++i) {
+    uint32_t t = (int)x >> 31;
+    x = (x << 1) | (y >> 63);
+    y = y << 1;
+    if ((x | t) >= D) {
+      x -= D;
+      ++y;
+    }
+  }
+
+  return y << (64 - i + 1);
+}
+
+}
+
+
+BlockFrequency &BlockFrequency::operator*=(const BranchProbability &Prob) {
+  uint32_t n = Prob.getNumerator();
+  uint32_t d = Prob.getDenominator();
+
+  assert(n <= d && "Probability must be less or equal to 1.");
+
+  // If we can overflow use 96-bit operations.
+  if (n > 0 && Frequency > UINT64_MAX / n) {
+    // 96-bit value represented as W[1]:W[0].
+    uint64_t W[2];
+
+    // Probability is less or equal to 1 which means that results must fit
+    // 64-bit.
+    mult96bit(Frequency, n, W);
+    Frequency = div96bit(W, d);
+    return *this;
+  }
+
+  Frequency *= n;
+  Frequency /= d;
+  return *this;
+}
+
+const BlockFrequency
+BlockFrequency::operator*(const BranchProbability &Prob) const {
+  BlockFrequency Freq(Frequency);
+  Freq *= Prob;
+  return Freq;
+}
+
+BlockFrequency &BlockFrequency::operator+=(const BlockFrequency &Freq) {
+  uint64_t Before = Freq.Frequency;
+  Frequency += Freq.Frequency;
+
+  // If overflow, set frequency to the maximum value.
+  if (Frequency < Before)
+    Frequency = UINT64_MAX;
+
+  return *this;
+}
+
+const BlockFrequency
+BlockFrequency::operator+(const BlockFrequency &Prob) const {
+  BlockFrequency Freq(Frequency);
+  Freq += Prob;
+  return Freq;
+}
+
+void BlockFrequency::print(raw_ostream &OS) const {
+  OS << Frequency;
+}
+
+namespace llvm {
+
+raw_ostream &operator<<(raw_ostream &OS, const BlockFrequency &Freq) {
+  Freq.print(OS);
+  return OS;
+}
+
+}
diff --git a/contrib/llvm/lib/Support/BranchProbability.cpp b/contrib/llvm/lib/Support/BranchProbability.cpp
index 97342da3f078..49d04ed83653 100644
--- a/contrib/llvm/lib/Support/BranchProbability.cpp
+++ b/contrib/llvm/lib/Support/BranchProbability.cpp
@@ -24,9 +24,8 @@ BranchProbability::BranchProbability(uint32_t n, uint32_t d) {
   D = d;
 }
 
-raw_ostream &BranchProbability::print(raw_ostream &OS) const {
+void BranchProbability::print(raw_ostream &OS) const {
   OS << N << " / " << D << " = " << ((double)N / D);
-  return OS;
 }
 
 void BranchProbability::dump() const {
diff --git a/lib/libc/amd64/gen/modf.S b/contrib/llvm/lib/Support/COPYRIGHT.regex
similarity index 53%
rename from lib/libc/amd64/gen/modf.S
rename to contrib/llvm/lib/Support/COPYRIGHT.regex
index 691aad82a415..a6392fd37c3d 100644
--- a/lib/libc/amd64/gen/modf.S
+++ b/contrib/llvm/lib/Support/COPYRIGHT.regex
@@ -1,9 +1,30 @@
+$OpenBSD: COPYRIGHT,v 1.3 2003/06/02 20:18:36 millert Exp $
+
+Copyright 1992, 1993, 1994 Henry Spencer.  All rights reserved.
+This software is not subject to any license of the American Telephone
+and Telegraph Company or of the Regents of the University of California.
+
+Permission is granted to anyone to use this software for any purpose on
+any computer system, and to alter it and redistribute it, subject
+to the following restrictions:
+
+1. The author is not responsible for the consequences of use of this
+   software, no matter how awful, even if they arise from flaws in it.
+
+2. The origin of this software must not be misrepresented, either by
+   explicit claim or by omission.  Since few users ever read sources,
+   credits must appear in the documentation.
+
+3. Altered versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.  Since few users
+   ever read sources, credits must appear in the documentation.
+
+4. This notice may not be removed or altered.
+
+=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
 /*-
- * Copyright (c) 1990 The Regents of the University of California.
- * All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * Sean Eric Fagan.
+ * Copyright (c) 1994
+ *	The Regents of the University of California.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -13,7 +34,7 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
+ * 3. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
@@ -29,63 +50,5 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	from: @(#)modf.s	5.5 (Berkeley) 3/18/91
+ *	@(#)COPYRIGHT	8.1 (Berkeley) 3/16/94
  */
-
-#include <machine/asm.h>
-#if defined(LIBC_SCCS)
-	RCSID("$NetBSD: modf.S,v 1.5 1997/07/16 14:37:18 christos Exp $")
-#endif
-__FBSDID("$FreeBSD$");
-
-/*
- * modf(value, iptr): return fractional part of value, and stores the
- * integral part into iptr (a pointer to double).
- *
- * Written by Sean Eric Fagan (sef@kithrup.COM)
- * Sun Mar 11 20:27:30 PST 1990
- */
-
-/* With CHOP mode on, frndint behaves as TRUNC does.  Useful. */
-ENTRY(modf)
-
-	/*
-	 * Set chop mode.
-	 */
-	fnstcw	-12(%rsp)
-	movw	-12(%rsp),%dx
-	orw	$3072,%dx
-	movw	%dx,-16(%rsp)
-	fldcw	-16(%rsp)
-
-	/*
-	 * Get integral part.
-	 */
-	movsd	%xmm0,-24(%rsp)
-	fldl	-24(%rsp)
-	frndint
-	fstpl	-8(%rsp)
-
-	/*
-	 * Restore control word.
-	 */
-	fldcw	-12(%rsp)
-
-	/*
-	 * Store integral part.
-	 */
-	movsd	-8(%rsp),%xmm0
-	movsd	%xmm0,(%rdi)
-
-	/*
-	 * Get fractional part and return it.
-	 */
-	fldl	-24(%rsp)
-	fsubl	-8(%rsp)
-	fstpl	-8(%rsp)
-	movsd   -8(%rsp),%xmm0
-
-	ret
-END(modf)
-
-	.section .note.GNU-stack,"",%progbits
diff --git a/contrib/llvm/lib/Support/CommandLine.cpp b/contrib/llvm/lib/Support/CommandLine.cpp
index 92c60a9631f7..238adcce0a12 100644
--- a/contrib/llvm/lib/Support/CommandLine.cpp
+++ b/contrib/llvm/lib/Support/CommandLine.cpp
@@ -23,7 +23,6 @@
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/Path.h"
 #include "llvm/ADT/OwningPtr.h"
@@ -45,6 +44,7 @@ TEMPLATE_INSTANTIATION(class basic_parser<bool>);
 TEMPLATE_INSTANTIATION(class basic_parser<boolOrDefault>);
 TEMPLATE_INSTANTIATION(class basic_parser<int>);
 TEMPLATE_INSTANTIATION(class basic_parser<unsigned>);
+TEMPLATE_INSTANTIATION(class basic_parser<unsigned long long>);
 TEMPLATE_INSTANTIATION(class basic_parser<double>);
 TEMPLATE_INSTANTIATION(class basic_parser<float>);
 TEMPLATE_INSTANTIATION(class basic_parser<std::string>);
@@ -63,6 +63,7 @@ void parser<bool>::anchor() {}
 void parser<boolOrDefault>::anchor() {}
 void parser<int>::anchor() {}
 void parser<unsigned>::anchor() {}
+void parser<unsigned long long>::anchor() {}
 void parser<double>::anchor() {}
 void parser<float>::anchor() {}
 void parser<std::string>::anchor() {}
@@ -1006,6 +1007,16 @@ bool parser<unsigned>::parse(Option &O, StringRef ArgName,
   return false;
 }
 
+// parser<unsigned long long> implementation
+//
+bool parser<unsigned long long>::parse(Option &O, StringRef ArgName,
+                                      StringRef Arg, unsigned long long &Value){
+
+  if (Arg.getAsInteger(0, Value))
+    return O.error("'" + Arg + "' value invalid for uint argument!");
+  return false;
+}
+
 // parser<double>/parser<float> implementation
 //
 static bool parseDouble(Option &O, StringRef Arg, double &Value) {
@@ -1151,6 +1162,7 @@ PRINT_OPT_DIFF(bool)
 PRINT_OPT_DIFF(boolOrDefault)
 PRINT_OPT_DIFF(int)
 PRINT_OPT_DIFF(unsigned)
+PRINT_OPT_DIFF(unsigned long long)
 PRINT_OPT_DIFF(double)
 PRINT_OPT_DIFF(float)
 PRINT_OPT_DIFF(char)
@@ -1330,10 +1342,7 @@ void cl::PrintOptionValues() {
 
 static void (*OverrideVersionPrinter)() = 0;
 
-static int TargetArraySortFn(const void *LHS, const void *RHS) {
-  typedef std::pair<const char *, const Target*> pair_ty;
-  return strcmp(((const pair_ty*)LHS)->first, ((const pair_ty*)RHS)->first);
-}
+static std::vector<void (*)()>* ExtraVersionPrinters = 0;
 
 namespace {
 class VersionPrinter {
@@ -1357,41 +1366,31 @@ class VersionPrinter {
     std::string CPU = sys::getHostCPUName();
     if (CPU == "generic") CPU = "(unknown)";
     OS << ".\n"
-#if defined(ENABLE_TIMESTAMPS) && ENABLE_TIMESTAMPS == 1
+#if (ENABLE_TIMESTAMPS == 1)
        << "  Built " << __DATE__ << " (" << __TIME__ << ").\n"
 #endif
        << "  Host: " << sys::getHostTriple() << '\n'
-       << "  Host CPU: " << CPU << '\n'
-       << '\n'
-       << "  Registered Targets:\n";
-
-    std::vector<std::pair<const char *, const Target*> > Targets;
-    size_t Width = 0;
-    for (TargetRegistry::iterator it = TargetRegistry::begin(),
-           ie = TargetRegistry::end(); it != ie; ++it) {
-      Targets.push_back(std::make_pair(it->getName(), &*it));
-      Width = std::max(Width, strlen(Targets.back().first));
-    }
-    if (!Targets.empty())
-      qsort(&Targets[0], Targets.size(), sizeof(Targets[0]),
-            TargetArraySortFn);
-
-    for (unsigned i = 0, e = Targets.size(); i != e; ++i) {
-      OS << "    " << Targets[i].first;
-      OS.indent(Width - strlen(Targets[i].first)) << " - "
-             << Targets[i].second->getShortDescription() << '\n';
-    }
-    if (Targets.empty())
-      OS << "    (none)\n";
+       << "  Host CPU: " << CPU << '\n';
   }
   void operator=(bool OptionWasSpecified) {
     if (!OptionWasSpecified) return;
 
-    if (OverrideVersionPrinter == 0) {
-      print();
+    if (OverrideVersionPrinter != 0) {
+      (*OverrideVersionPrinter)();
       exit(1);
     }
-    (*OverrideVersionPrinter)();
+    print();
+
+    // Iterate over any registered extra printers and call them to add further
+    // information.
+    if (ExtraVersionPrinters != 0) {
+      outs() << '\n';
+      for (std::vector<void (*)()>::iterator I = ExtraVersionPrinters->begin(),
+                                             E = ExtraVersionPrinters->end();
+           I != E; ++I)
+        (*I)();
+    }
+
     exit(1);
   }
 };
@@ -1424,3 +1423,10 @@ void cl::PrintVersionMessage() {
 void cl::SetVersionPrinter(void (*func)()) {
   OverrideVersionPrinter = func;
 }
+
+void cl::AddExtraVersionPrinter(void (*func)()) {
+  if (ExtraVersionPrinters == 0)
+    ExtraVersionPrinters = new std::vector<void (*)()>;
+
+  ExtraVersionPrinters->push_back(func);
+}
diff --git a/contrib/llvm/lib/Support/ConstantRange.cpp b/contrib/llvm/lib/Support/ConstantRange.cpp
index 81382d08dc23..c29cb53fb9c5 100644
--- a/contrib/llvm/lib/Support/ConstantRange.cpp
+++ b/contrib/llvm/lib/Support/ConstantRange.cpp
@@ -21,11 +21,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Constants.h"
+#include "llvm/InstrTypes.h"
 #include "llvm/Support/ConstantRange.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Instructions.h"
 using namespace llvm;
 
 /// Initialize a full (the default) or empty set for the specified type.
@@ -56,56 +55,56 @@ ConstantRange ConstantRange::makeICmpRegion(unsigned Pred,
 
   uint32_t W = CR.getBitWidth();
   switch (Pred) {
-    default: assert(!"Invalid ICmp predicate to makeICmpRegion()");
-    case ICmpInst::ICMP_EQ:
+    default: assert(0 && "Invalid ICmp predicate to makeICmpRegion()");
+    case CmpInst::ICMP_EQ:
       return CR;
-    case ICmpInst::ICMP_NE:
+    case CmpInst::ICMP_NE:
       if (CR.isSingleElement())
         return ConstantRange(CR.getUpper(), CR.getLower());
       return ConstantRange(W);
-    case ICmpInst::ICMP_ULT: {
+    case CmpInst::ICMP_ULT: {
       APInt UMax(CR.getUnsignedMax());
       if (UMax.isMinValue())
         return ConstantRange(W, /* empty */ false);
       return ConstantRange(APInt::getMinValue(W), UMax);
     }
-    case ICmpInst::ICMP_SLT: {
+    case CmpInst::ICMP_SLT: {
       APInt SMax(CR.getSignedMax());
       if (SMax.isMinSignedValue())
         return ConstantRange(W, /* empty */ false);
       return ConstantRange(APInt::getSignedMinValue(W), SMax);
     }
-    case ICmpInst::ICMP_ULE: {
+    case CmpInst::ICMP_ULE: {
       APInt UMax(CR.getUnsignedMax());
       if (UMax.isMaxValue())
         return ConstantRange(W);
       return ConstantRange(APInt::getMinValue(W), UMax + 1);
     }
-    case ICmpInst::ICMP_SLE: {
+    case CmpInst::ICMP_SLE: {
       APInt SMax(CR.getSignedMax());
       if (SMax.isMaxSignedValue())
         return ConstantRange(W);
       return ConstantRange(APInt::getSignedMinValue(W), SMax + 1);
     }
-    case ICmpInst::ICMP_UGT: {
+    case CmpInst::ICMP_UGT: {
       APInt UMin(CR.getUnsignedMin());
       if (UMin.isMaxValue())
         return ConstantRange(W, /* empty */ false);
       return ConstantRange(UMin + 1, APInt::getNullValue(W));
     }
-    case ICmpInst::ICMP_SGT: {
+    case CmpInst::ICMP_SGT: {
       APInt SMin(CR.getSignedMin());
       if (SMin.isMaxSignedValue())
         return ConstantRange(W, /* empty */ false);
       return ConstantRange(SMin + 1, APInt::getSignedMinValue(W));
     }
-    case ICmpInst::ICMP_UGE: {
+    case CmpInst::ICMP_UGE: {
       APInt UMin(CR.getUnsignedMin());
       if (UMin.isMinValue())
         return ConstantRange(W);
       return ConstantRange(UMin, APInt::getNullValue(W));
     }
-    case ICmpInst::ICMP_SGE: {
+    case CmpInst::ICMP_SGE: {
       APInt SMin(CR.getSignedMin());
       if (SMin.isMinSignedValue())
         return ConstantRange(W);
diff --git a/contrib/llvm/lib/Support/CrashRecoveryContext.cpp b/contrib/llvm/lib/Support/CrashRecoveryContext.cpp
index 899c3890d78a..263114c06f98 100644
--- a/contrib/llvm/lib/Support/CrashRecoveryContext.cpp
+++ b/contrib/llvm/lib/Support/CrashRecoveryContext.cpp
@@ -12,6 +12,7 @@
 #include "llvm/Config/config.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/ThreadLocal.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <setjmp.h>
 #include <cstdio>
 using namespace llvm;
@@ -123,7 +124,56 @@ CrashRecoveryContext::unregisterCleanup(CrashRecoveryContextCleanup *cleanup) {
 
 #ifdef LLVM_ON_WIN32
 
-// FIXME: No real Win32 implementation currently.
+#include "Windows/Windows.h"
+
+// On Windows, we can make use of vectored exception handling to
+// catch most crashing situations.  Note that this does mean
+// we will be alerted of exceptions *before* structured exception
+// handling has the opportunity to catch it.  But that isn't likely
+// to cause problems because nowhere in the project is SEH being
+// used.
+//
+// Vectored exception handling is built on top of SEH, and so it
+// works on a per-thread basis.
+//
+// The vectored exception handler functionality was added in Windows
+// XP, so if support for older versions of Windows is required,
+// it will have to be added.
+//
+// If we want to support as far back as Win2k, we could use the
+// SetUnhandledExceptionFilter API, but there's a risk of that
+// being entirely overwritten (it's not a chain).
+
+static LONG CALLBACK ExceptionHandler(PEXCEPTION_POINTERS ExceptionInfo)
+{
+  // Lookup the current thread local recovery object.
+  const CrashRecoveryContextImpl *CRCI = CurrentContext.get();
+
+  if (!CRCI) {
+    // Something has gone horribly wrong, so let's just tell everyone
+    // to keep searching
+    CrashRecoveryContext::Disable();
+    return EXCEPTION_CONTINUE_SEARCH;
+  }
+
+  // TODO: We can capture the stack backtrace here and store it on the
+  // implementation if we so choose.
+
+  // Handle the crash
+  const_cast<CrashRecoveryContextImpl*>(CRCI)->HandleCrash();
+
+  // Note that we don't actually get here because HandleCrash calls
+  // longjmp, which means the HandleCrash function never returns.
+  llvm_unreachable("Handled the crash, should have longjmp'ed out of here");
+  return EXCEPTION_CONTINUE_SEARCH;
+}
+
+// Because the Enable and Disable calls are static, it means that
+// there may not actually be an Impl available, or even a current
+// CrashRecoveryContext at all.  So we make use of a thread-local
+// exception table.  The handles contained in here will either be
+// non-NULL, valid VEH handles, or NULL.
+static sys::ThreadLocal<const void> sCurrentExceptionHandle;
 
 void CrashRecoveryContext::Enable() {
   sys::ScopedLock L(gCrashRecoveryContexMutex);
@@ -132,6 +182,13 @@ void CrashRecoveryContext::Enable() {
     return;
 
   gCrashRecoveryEnabled = true;
+
+  // We can set up vectored exception handling now.  We will install our
+  // handler as the front of the list, though there's no assurances that
+  // it will remain at the front (another call could install itself before
+  // our handler).  This 1) isn't likely, and 2) shouldn't cause problems.
+  PVOID handle = ::AddVectoredExceptionHandler(1, ExceptionHandler);
+  sCurrentExceptionHandle.set(handle);
 }
 
 void CrashRecoveryContext::Disable() {
@@ -141,6 +198,15 @@ void CrashRecoveryContext::Disable() {
     return;
 
   gCrashRecoveryEnabled = false;
+
+  PVOID currentHandle = const_cast<PVOID>(sCurrentExceptionHandle.get());
+  if (currentHandle) {
+    // Now we can remove the vectored exception handler from the chain
+    ::RemoveVectoredExceptionHandler(currentHandle);
+
+    // Reset the handle in our thread-local set.
+    sCurrentExceptionHandle.set(NULL);
+  }
 }
 
 #else
diff --git a/contrib/llvm/lib/Support/DataExtractor.cpp b/contrib/llvm/lib/Support/DataExtractor.cpp
new file mode 100644
index 000000000000..b946c1df8363
--- /dev/null
+++ b/contrib/llvm/lib/Support/DataExtractor.cpp
@@ -0,0 +1,175 @@
+//===-- DataExtractor.cpp -------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/SwapByteOrder.h"
+using namespace llvm;
+
+template <typename T>
+static T getU(uint32_t *offset_ptr, const DataExtractor *de,
+              bool isLittleEndian, const char *Data) {
+  T val = 0;
+  uint32_t offset = *offset_ptr;
+  if (de->isValidOffsetForDataOfSize(offset, sizeof(val))) {
+    std::memcpy(&val, &Data[offset], sizeof(val));
+    if (sys::isLittleEndianHost() != isLittleEndian)
+      val = sys::SwapByteOrder(val);
+
+    // Advance the offset
+    *offset_ptr += sizeof(val);
+  }
+  return val;
+}
+
+template <typename T>
+static T *getUs(uint32_t *offset_ptr, T *dst, uint32_t count,
+                const DataExtractor *de, bool isLittleEndian, const char *Data){
+  uint32_t offset = *offset_ptr;
+
+  if (count > 0 && de->isValidOffsetForDataOfSize(offset, sizeof(*dst)*count)) {
+    for (T *value_ptr = dst, *end = dst + count; value_ptr != end;
+        ++value_ptr, offset += sizeof(*dst))
+      *value_ptr = getU<T>(offset_ptr, de, isLittleEndian, Data);
+    // Advance the offset
+    *offset_ptr = offset;
+    // Return a non-NULL pointer to the converted data as an indicator of
+    // success
+    return dst;
+  }
+  return NULL;
+}
+
+uint8_t DataExtractor::getU8(uint32_t *offset_ptr) const {
+  return getU<uint8_t>(offset_ptr, this, IsLittleEndian, Data.data());
+}
+
+uint8_t *
+DataExtractor::getU8(uint32_t *offset_ptr, uint8_t *dst, uint32_t count) const {
+  return getUs<uint8_t>(offset_ptr, dst, count, this, IsLittleEndian,
+                       Data.data());
+}
+
+
+uint16_t DataExtractor::getU16(uint32_t *offset_ptr) const {
+  return getU<uint16_t>(offset_ptr, this, IsLittleEndian, Data.data());
+}
+
+uint16_t *DataExtractor::getU16(uint32_t *offset_ptr, uint16_t *dst,
+                                uint32_t count) const {
+  return getUs<uint16_t>(offset_ptr, dst, count, this, IsLittleEndian,
+                        Data.data());
+}
+
+uint32_t DataExtractor::getU32(uint32_t *offset_ptr) const {
+  return getU<uint32_t>(offset_ptr, this, IsLittleEndian, Data.data());
+}
+
+uint32_t *DataExtractor::getU32(uint32_t *offset_ptr, uint32_t *dst,
+                                uint32_t count) const {
+  return getUs<uint32_t>(offset_ptr, dst, count, this, IsLittleEndian,
+                        Data.data());;
+}
+
+uint64_t DataExtractor::getU64(uint32_t *offset_ptr) const {
+  return getU<uint64_t>(offset_ptr, this, IsLittleEndian, Data.data());
+}
+
+uint64_t *DataExtractor::getU64(uint32_t *offset_ptr, uint64_t *dst,
+                                uint32_t count) const {
+  return getUs<uint64_t>(offset_ptr, dst, count, this, IsLittleEndian,
+                        Data.data());
+}
+
+uint64_t
+DataExtractor::getUnsigned(uint32_t *offset_ptr, uint32_t byte_size) const {
+  switch (byte_size) {
+  case 1:
+    return getU8(offset_ptr);
+  case 2:
+    return getU16(offset_ptr);
+  case 4:
+    return getU32(offset_ptr);
+  case 8:
+    return getU64(offset_ptr);
+  }
+  llvm_unreachable("getUnsigned unhandled case!");
+}
+
+int64_t
+DataExtractor::getSigned(uint32_t *offset_ptr, uint32_t byte_size) const {
+  switch (byte_size) {
+  case 1:
+    return (int8_t)getU8(offset_ptr);
+  case 2:
+    return (int16_t)getU16(offset_ptr);
+  case 4:
+    return (int32_t)getU32(offset_ptr);
+  case 8:
+    return (int64_t)getU64(offset_ptr);
+  }
+  llvm_unreachable("getSigned unhandled case!");
+}
+
+const char *DataExtractor::getCStr(uint32_t *offset_ptr) const {
+  uint32_t offset = *offset_ptr;
+  StringRef::size_type pos = Data.find('\0', offset);
+  if (pos != StringRef::npos) {
+    *offset_ptr = pos + 1;
+    return Data.data() + offset;
+  }
+  return NULL;
+}
+
+uint64_t DataExtractor::getULEB128(uint32_t *offset_ptr) const {
+  uint64_t result = 0;
+  if (Data.empty())
+    return 0;
+
+  unsigned shift = 0;
+  uint32_t offset = *offset_ptr;
+  uint8_t byte = 0;
+
+  while (isValidOffset(offset)) {
+    byte = Data[offset++];
+    result |= (byte & 0x7f) << shift;
+    shift += 7;
+    if ((byte & 0x80) == 0)
+      break;
+  }
+
+  *offset_ptr = offset;
+  return result;
+}
+
+int64_t DataExtractor::getSLEB128(uint32_t *offset_ptr) const {
+  int64_t result = 0;
+  if (Data.empty())
+    return 0;
+
+  unsigned shift = 0;
+  uint32_t offset = *offset_ptr;
+  uint8_t byte = 0;
+
+  while (isValidOffset(offset)) {
+    byte = Data[offset++];
+    result |= (byte & 0x7f) << shift;
+    shift += 7;
+    if ((byte & 0x80) == 0)
+      break;
+  }
+
+  // Sign bit of byte is 2nd high order bit (0x40)
+  if (shift < 64 && (byte & 0x40))
+    result |= -(1 << shift);
+
+  *offset_ptr = offset;
+  return result;
+}
diff --git a/contrib/llvm/lib/Support/Disassembler.cpp b/contrib/llvm/lib/Support/Disassembler.cpp
index 6362aff43a9d..c6d73bcad3e4 100644
--- a/contrib/llvm/lib/Support/Disassembler.cpp
+++ b/contrib/llvm/lib/Support/Disassembler.cpp
@@ -1,4 +1,4 @@
-//===- lib/System/Disassembler.cpp ------------------------------*- C++ -*-===//
+//===- lib/Support/Disassembler.cpp -----------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/contrib/llvm/lib/Support/Dwarf.cpp b/contrib/llvm/lib/Support/Dwarf.cpp
index 0813321b6f79..95a9550f9663 100644
--- a/contrib/llvm/lib/Support/Dwarf.cpp
+++ b/contrib/llvm/lib/Support/Dwarf.cpp
@@ -82,6 +82,19 @@ const char *llvm::dwarf::TagString(unsigned Tag) {
   case DW_TAG_arg_variable:              return "DW_TAG_arg_variable";
   case DW_TAG_return_variable:           return "DW_TAG_return_variable";
   case DW_TAG_vector_type:               return "DW_TAG_vector_type";
+  case DW_TAG_rvalue_reference_type:     return "DW_TAG_rvalue_reference_type";
+  case DW_TAG_template_alias:            return "DW_TAG_template_alias";
+  case DW_TAG_MIPS_loop:                 return "DW_TAG_MIPS_loop";
+  case DW_TAG_type_unit:                 return "DW_TAG_type_unit";
+  case DW_TAG_format_label:              return "DW_TAG_format_label";
+  case DW_TAG_function_template:         return "DW_TAG_function_template";
+  case DW_TAG_class_template:            return "DW_TAG_class_template";
+  case DW_TAG_GNU_template_template_param:
+    return "DW_TAG_GNU_template_template_param";
+  case DW_TAG_GNU_template_parameter_pack:
+    return "DW_TAG_GNU_template_parameter_pack";
+  case DW_TAG_GNU_formal_parameter_pack:
+    return "DW_TAG_GNU_formal_parameter_pack";
   }
   return 0;
 }
@@ -186,7 +199,30 @@ const char *llvm::dwarf::AttributeString(unsigned Attribute) {
   case DW_AT_elemental:                  return "DW_AT_elemental";
   case DW_AT_pure:                       return "DW_AT_pure";
   case DW_AT_recursive:                  return "DW_AT_recursive";
+  case DW_AT_signature:                  return "DW_AT_signature";
+  case DW_AT_main_subprogram:            return "DW_AT_main_subprogram";
+  case DW_AT_data_bit_offset:            return "DW_AT_data_bit_offset";
+  case DW_AT_const_expr:                 return "DW_AT_const_expr";
+  case DW_AT_enum_class:                 return "DW_AT_enum_class";
+  case DW_AT_linkage_name:               return "DW_AT_linkage_name";
+  case DW_AT_MIPS_loop_begin:            return "DW_AT_MIPS_loop_begin";
+  case DW_AT_MIPS_tail_loop_begin:       return "DW_AT_MIPS_tail_loop_begin";
+  case DW_AT_MIPS_epilog_begin:          return "DW_AT_MIPS_epilog_begin";
+  case DW_AT_MIPS_loop_unroll_factor:    return "DW_AT_MIPS_loop_unroll_factor";
+  case DW_AT_MIPS_software_pipeline_depth:
+    return "DW_AT_MIPS_software_pipeline_depth";
   case DW_AT_MIPS_linkage_name:          return "DW_AT_MIPS_linkage_name";
+  case DW_AT_MIPS_stride:                return "DW_AT_MIPS_stride";
+  case DW_AT_MIPS_abstract_name:         return "DW_AT_MIPS_abstract_name";
+  case DW_AT_MIPS_clone_origin:          return "DW_AT_MIPS_clone_origin";
+  case DW_AT_MIPS_has_inlines:           return "DW_AT_MIPS_has_inlines";
+  case DW_AT_MIPS_stride_byte:           return "DW_AT_MIPS_stride_byte";
+  case DW_AT_MIPS_stride_elem:           return "DW_AT_MIPS_stride_elem";
+  case DW_AT_MIPS_ptr_dopetype:          return "DW_AT_MIPS_ptr_dopetype";
+  case DW_AT_MIPS_allocatable_dopetype:
+    return "DW_AT_MIPS_allocatable_dopetype";
+  case DW_AT_MIPS_assumed_shape_dopetype:
+    return "DW_AT_MIPS_assumed_shape_dopetype";
   case DW_AT_sf_names:                   return "DW_AT_sf_names";
   case DW_AT_src_info:                   return "DW_AT_src_info";
   case DW_AT_mac_info:                   return "DW_AT_mac_info";
@@ -194,6 +230,8 @@ const char *llvm::dwarf::AttributeString(unsigned Attribute) {
   case DW_AT_body_begin:                 return "DW_AT_body_begin";
   case DW_AT_body_end:                   return "DW_AT_body_end";
   case DW_AT_GNU_vector:                 return "DW_AT_GNU_vector";
+  case DW_AT_GNU_template_name:          return "DW_AT_GNU_template_name";
+  case DW_AT_MIPS_assumed_size:          return "DW_AT_MIPS_assumed_size";
   case DW_AT_lo_user:                    return "DW_AT_lo_user";
   case DW_AT_hi_user:                    return "DW_AT_hi_user";
   case DW_AT_APPLE_optimized:            return "DW_AT_APPLE_optimized";
@@ -237,6 +275,10 @@ const char *llvm::dwarf::FormEncodingString(unsigned Encoding) {
   case DW_FORM_ref8:                     return "DW_FORM_ref8";
   case DW_FORM_ref_udata:                return "DW_FORM_ref_udata";
   case DW_FORM_indirect:                 return "DW_FORM_indirect";
+  case DW_FORM_sec_offset:               return "DW_FORM_sec_offset";
+  case DW_FORM_exprloc:                  return "DW_FORM_exprloc";
+  case DW_FORM_flag_present:             return "DW_FORM_flag_present";
+  case DW_FORM_ref_sig8:                 return "DW_FORM_ref_sig8";
   }
   return 0;
 }
@@ -397,6 +439,8 @@ const char *llvm::dwarf::OperationEncodingString(unsigned Encoding) {
   case DW_OP_form_tls_address:           return "DW_OP_form_tls_address";
   case DW_OP_call_frame_cfa:             return "DW_OP_call_frame_cfa";
   case DW_OP_bit_piece:                  return "DW_OP_bit_piece";
+  case DW_OP_implicit_value:             return "DW_OP_implicit_value";
+  case DW_OP_stack_value:                return "DW_OP_stack_value";
   case DW_OP_lo_user:                    return "DW_OP_lo_user";
   case DW_OP_hi_user:                    return "DW_OP_hi_user";
   }
@@ -416,6 +460,7 @@ const char *llvm::dwarf::AttributeEncodingString(unsigned Encoding) {
   case DW_ATE_unsigned:                  return "DW_ATE_unsigned";
   case DW_ATE_unsigned_char:             return "DW_ATE_unsigned_char";
   case DW_ATE_imaginary_float:           return "DW_ATE_imaginary_float";
+  case DW_ATE_UTF:                       return "DW_ATE_UTF";
   case DW_ATE_packed_decimal:            return "DW_ATE_packed_decimal";
   case DW_ATE_numeric_string:            return "DW_ATE_numeric_string";
   case DW_ATE_edited:                    return "DW_ATE_edited";
@@ -602,6 +647,7 @@ const char *llvm::dwarf::LNExtendedString(unsigned Encoding) {
   case DW_LNE_end_sequence:              return "DW_LNE_end_sequence";
   case DW_LNE_set_address:               return "DW_LNE_set_address";
   case DW_LNE_define_file:               return "DW_LNE_define_file";
+  case DW_LNE_set_discriminator:         return "DW_LNE_set_discriminator";
   case DW_LNE_lo_user:                   return "DW_LNE_lo_user";
   case DW_LNE_hi_user:                   return "DW_LNE_hi_user";
   }
@@ -651,6 +697,9 @@ const char *llvm::dwarf::CallFrameString(unsigned Encoding) {
   case DW_CFA_val_offset:                return "DW_CFA_val_offset";
   case DW_CFA_val_offset_sf:             return "DW_CFA_val_offset_sf";
   case DW_CFA_val_expression:            return "DW_CFA_val_expression";
+  case DW_CFA_MIPS_advance_loc8:         return "DW_CFA_MIPS_advance_loc8";
+  case DW_CFA_GNU_window_save:           return "DW_CFA_GNU_window_save";
+  case DW_CFA_GNU_args_size:             return "DW_CFA_GNU_args_size";
   case DW_CFA_lo_user:                   return "DW_CFA_lo_user";
   case DW_CFA_hi_user:                   return "DW_CFA_hi_user";
   }
diff --git a/contrib/llvm/lib/Support/DynamicLibrary.cpp b/contrib/llvm/lib/Support/DynamicLibrary.cpp
index 455c3801cc68..fb02c07e4af8 100644
--- a/contrib/llvm/lib/Support/DynamicLibrary.cpp
+++ b/contrib/llvm/lib/Support/DynamicLibrary.cpp
@@ -9,28 +9,26 @@
 //
 //  This header file implements the operating system DynamicLibrary concept.
 //
-// FIXME: This file leaks the ExplicitSymbols and OpenedHandles vector, and is
-// not thread safe!
+// FIXME: This file leaks ExplicitSymbols and OpenedHandles!
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Config/config.h"
 #include <cstdio>
 #include <cstring>
-#include <map>
-#include <vector>
 
 // Collection of symbol name/value pairs to be searched prior to any libraries.
-static std::map<std::string, void*> *ExplicitSymbols = 0;
+static llvm::StringMap<void *> *ExplicitSymbols = 0;
 
 namespace {
 
 struct ExplicitSymbolsDeleter {
   ~ExplicitSymbolsDeleter() {
-    if (ExplicitSymbols)
-      delete ExplicitSymbols;
+    delete ExplicitSymbols;
   }
 };
 
@@ -38,13 +36,22 @@ struct ExplicitSymbolsDeleter {
 
 static ExplicitSymbolsDeleter Dummy;
 
-void llvm::sys::DynamicLibrary::AddSymbol(const char* symbolName,
+
+static llvm::sys::SmartMutex<true>& getMutex() {
+  static llvm::sys::SmartMutex<true> HandlesMutex;
+  return HandlesMutex;
+}
+
+void llvm::sys::DynamicLibrary::AddSymbol(StringRef symbolName,
                                           void *symbolValue) {
+  SmartScopedLock<true> lock(getMutex());
   if (ExplicitSymbols == 0)
-    ExplicitSymbols = new std::map<std::string, void*>();
+    ExplicitSymbols = new llvm::StringMap<void*>();
   (*ExplicitSymbols)[symbolName] = symbolValue;
 }
 
+char llvm::sys::DynamicLibrary::Invalid = 0;
+
 #ifdef LLVM_ON_WIN32
 
 #include "Windows/DynamicLibrary.inc"
@@ -61,66 +68,78 @@ using namespace llvm::sys;
 //===          independent code.
 //===----------------------------------------------------------------------===//
 
-static std::vector<void *> *OpenedHandles = 0;
+static DenseSet<void *> *OpenedHandles = 0;
 
+DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *filename,
+                                                   std::string *errMsg) {
+  SmartScopedLock<true> lock(getMutex());
 
-static SmartMutex<true>& getMutex() {
-  static SmartMutex<true> HandlesMutex;
-  return HandlesMutex;
-}
-
-
-bool DynamicLibrary::LoadLibraryPermanently(const char *Filename,
-                                            std::string *ErrMsg) {
-  void *H = dlopen(Filename, RTLD_LAZY|RTLD_GLOBAL);
-  if (H == 0) {
-    if (ErrMsg) *ErrMsg = dlerror();
-    return true;
+  void *handle = dlopen(filename, RTLD_LAZY|RTLD_GLOBAL);
+  if (handle == 0) {
+    if (errMsg) *errMsg = dlerror();
+    return DynamicLibrary();
   }
+
 #ifdef __CYGWIN__
   // Cygwin searches symbols only in the main
   // with the handle of dlopen(NULL, RTLD_GLOBAL).
-  if (Filename == NULL)
-    H = RTLD_DEFAULT;
+  if (filename == NULL)
+    handle = RTLD_DEFAULT;
 #endif
-  SmartScopedLock<true> Lock(getMutex());
+
   if (OpenedHandles == 0)
-    OpenedHandles = new std::vector<void *>();
-  OpenedHandles->push_back(H);
-  return false;
+    OpenedHandles = new DenseSet<void *>();
+
+  // If we've already loaded this library, dlclose() the handle in order to
+  // keep the internal refcount at +1.
+  if (!OpenedHandles->insert(handle).second)
+    dlclose(handle);
+
+  return DynamicLibrary(handle);
 }
+
+void *DynamicLibrary::getAddressOfSymbol(const char *symbolName) {
+  if (!isValid())
+    return NULL;
+  return dlsym(Data, symbolName);
+}
+
 #else
 
 using namespace llvm;
 using namespace llvm::sys;
 
-bool DynamicLibrary::LoadLibraryPermanently(const char *Filename,
-                                            std::string *ErrMsg) {
-  if (ErrMsg) *ErrMsg = "dlopen() not supported on this platform";
-  return true;
+DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *filename,
+                                                   std::string *errMsg) {
+  if (errMsg) *errMsg = "dlopen() not supported on this platform";
+  return DynamicLibrary();
 }
+
+void *DynamicLibrary::getAddressOfSymbol(const char *symbolName) {
+  return NULL;
+}
+
 #endif
 
 namespace llvm {
 void *SearchForAddressOfSpecialSymbol(const char* symbolName);
 }
 
-void* DynamicLibrary::SearchForAddressOfSymbol(const char* symbolName) {
+void* DynamicLibrary::SearchForAddressOfSymbol(const char *symbolName) {
+  SmartScopedLock<true> Lock(getMutex());
+
   // First check symbols added via AddSymbol().
   if (ExplicitSymbols) {
-    std::map<std::string, void *>::iterator I =
-      ExplicitSymbols->find(symbolName);
-    std::map<std::string, void *>::iterator E = ExplicitSymbols->end();
+    StringMap<void *>::iterator i = ExplicitSymbols->find(symbolName);
 
-    if (I != E)
-      return I->second;
+    if (i != ExplicitSymbols->end())
+      return i->second;
   }
 
 #if HAVE_DLFCN_H
   // Now search the libraries.
-  SmartScopedLock<true> Lock(getMutex());
   if (OpenedHandles) {
-    for (std::vector<void *>::iterator I = OpenedHandles->begin(),
+    for (DenseSet<void *>::iterator I = OpenedHandles->begin(),
          E = OpenedHandles->end(); I != E; ++I) {
       //lt_ptr ptr = lt_dlsym(*I, symbolName);
       void *ptr = dlsym(*I, symbolName);
diff --git a/contrib/llvm/lib/Support/FoldingSet.cpp b/contrib/llvm/lib/Support/FoldingSet.cpp
index 1568342e9c9d..17b827132f57 100644
--- a/contrib/llvm/lib/Support/FoldingSet.cpp
+++ b/contrib/llvm/lib/Support/FoldingSet.cpp
@@ -64,10 +64,8 @@ void FoldingSetNodeID::AddPointer(const void *Ptr) {
   // depend on the host.  It doesn't matter however, because hashing on
   // pointer values in inherently unstable.  Nothing  should depend on the 
   // ordering of nodes in the folding set.
-  intptr_t PtrI = (intptr_t)Ptr;
-  Bits.push_back(unsigned(PtrI));
-  if (sizeof(intptr_t) > sizeof(unsigned))
-    Bits.push_back(unsigned(uint64_t(PtrI) >> 32));
+  Bits.append(reinterpret_cast<unsigned *>(&Ptr),
+              reinterpret_cast<unsigned *>(&Ptr+1));
 }
 void FoldingSetNodeID::AddInteger(signed I) {
   Bits.push_back(I);
diff --git a/contrib/llvm/lib/Support/Host.cpp b/contrib/llvm/lib/Support/Host.cpp
index c525a1228129..a19e4b41189b 100644
--- a/contrib/llvm/lib/Support/Host.cpp
+++ b/contrib/llvm/lib/Support/Host.cpp
@@ -213,13 +213,13 @@ std::string sys::getHostCPUName() {
       case 30: // Intel(R) Core(TM) i7 CPU         870  @ 2.93GHz.
                // As found in a Summer 2010 model iMac.
       case 37: // Intel Core i7, laptop version.
+      case 44: // Intel Core i7 processor and Intel Xeon processor. All
+               // processors are manufactured using the 32 nm process.
         return "corei7";
 
       // SandyBridge:
       case 42: // Intel Core i7 processor. All processors are manufactured
                // using the 32 nm process.
-      case 44: // Intel Core i7 processor and Intel Xeon processor. All
-               // processors are manufactured using the 32 nm process.
       case 45:
         return "corei7-avx";
 
diff --git a/contrib/llvm/lib/Support/IncludeFile.cpp b/contrib/llvm/lib/Support/IncludeFile.cpp
index 5da88261ce53..e67acb3d1d67 100644
--- a/contrib/llvm/lib/Support/IncludeFile.cpp
+++ b/contrib/llvm/lib/Support/IncludeFile.cpp
@@ -1,4 +1,4 @@
-//===- lib/System/IncludeFile.cpp - Ensure Linking Of Implementation -----===//
+//===- lib/Support/IncludeFile.cpp - Ensure Linking Of Implementation -----===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/contrib/llvm/lib/Support/Memory.cpp b/contrib/llvm/lib/Support/Memory.cpp
index ac7af0ae8bdf..ceaaa99b33f2 100644
--- a/contrib/llvm/lib/Support/Memory.cpp
+++ b/contrib/llvm/lib/Support/Memory.cpp
@@ -16,6 +16,10 @@
 #include "llvm/Support/Valgrind.h"
 #include "llvm/Config/config.h"
 
+#if defined(__mips__)
+#include <sys/cachectl.h>
+#endif
+
 namespace llvm {
 using namespace sys;
 }
@@ -66,6 +70,8 @@ void llvm::sys::Memory::InvalidateInstructionCache(const void *Addr,
   char *Start = (char*) Addr;
   char *End = Start + Len;
   __clear_cache(Start, End);
+#  elif defined(__mips__)
+  cacheflush((char*)Addr, Len, BCACHE);
 #  endif
 
 #endif  // end apple
diff --git a/contrib/llvm/lib/Support/MemoryBuffer.cpp b/contrib/llvm/lib/Support/MemoryBuffer.cpp
index d264be9aced1..0771af5fee07 100644
--- a/contrib/llvm/lib/Support/MemoryBuffer.cpp
+++ b/contrib/llvm/lib/Support/MemoryBuffer.cpp
@@ -275,16 +275,16 @@ static bool shouldUseMmap(int FD,
 
 error_code MemoryBuffer::getOpenFile(int FD, const char *Filename,
                                      OwningPtr<MemoryBuffer> &result,
-                                     size_t FileSize, size_t MapSize,
-                                     off_t Offset,
+                                     uint64_t FileSize, uint64_t MapSize,
+                                     int64_t Offset,
                                      bool RequiresNullTerminator) {
   static int PageSize = sys::Process::GetPageSize();
 
   // Default is to map the full file.
-  if (MapSize == size_t(-1)) {
+  if (MapSize == uint64_t(-1)) {
     // If we don't know the file size, use fstat to find out.  fstat on an open
     // file descriptor is cheaper than stat on a random path.
-    if (FileSize == size_t(-1)) {
+    if (FileSize == uint64_t(-1)) {
       struct stat FileInfo;
       // TODO: This should use fstat64 when available.
       if (fstat(FD, &FileInfo) == -1) {
diff --git a/contrib/llvm/lib/Support/MemoryObject.cpp b/contrib/llvm/lib/Support/MemoryObject.cpp
index 91e3ecd23a2e..b20ab8923813 100644
--- a/contrib/llvm/lib/Support/MemoryObject.cpp
+++ b/contrib/llvm/lib/Support/MemoryObject.cpp
@@ -19,8 +19,11 @@ int MemoryObject::readBytes(uint64_t address,
                             uint64_t* copied) const {
   uint64_t current = address;
   uint64_t limit = getBase() + getExtent();
-  
-  while (current - address < size && current < limit) {
+
+  if (current + size > limit)
+    return -1;
+
+  while (current - address < size) {
     if (readByte(current, &buf[(current - address)]))
       return -1;
     
diff --git a/contrib/llvm/lib/Support/Mutex.cpp b/contrib/llvm/lib/Support/Mutex.cpp
index b408973bbad1..8874e943f4c2 100644
--- a/contrib/llvm/lib/Support/Mutex.cpp
+++ b/contrib/llvm/lib/Support/Mutex.cpp
@@ -152,6 +152,6 @@ MutexImpl::tryacquire()
 #elif defined( LLVM_ON_WIN32)
 #include "Windows/Mutex.inc"
 #else
-#warning Neither LLVM_ON_UNIX nor LLVM_ON_WIN32 was set in System/Mutex.cpp
+#warning Neither LLVM_ON_UNIX nor LLVM_ON_WIN32 was set in Support/Mutex.cpp
 #endif
 #endif
diff --git a/contrib/llvm/lib/Support/Path.cpp b/contrib/llvm/lib/Support/Path.cpp
index 8fbaf2d42bf9..e5b7cd3bfbc2 100644
--- a/contrib/llvm/lib/Support/Path.cpp
+++ b/contrib/llvm/lib/Support/Path.cpp
@@ -121,7 +121,7 @@ sys::IdentifyFileType(const char *magic, unsigned length) {
         case 7: return Mach_O_DynamicLinker_FileType;
         case 8: return Mach_O_Bundle_FileType;
         case 9: return Mach_O_DynamicallyLinkedSharedLibStub_FileType;
-        case 10: break; // FIXME: MH_DSYM companion file with only debug.
+        case 10: return Mach_O_DSYMCompanion_FileType;
       }
       break;
     }
diff --git a/contrib/llvm/lib/Support/PathV2.cpp b/contrib/llvm/lib/Support/PathV2.cpp
index 896c94c071bc..bebe442e2478 100644
--- a/contrib/llvm/lib/Support/PathV2.cpp
+++ b/contrib/llvm/lib/Support/PathV2.cpp
@@ -490,6 +490,36 @@ bool is_separator(char value) {
   }
 }
 
+void system_temp_directory(bool erasedOnReboot, SmallVectorImpl<char> &result) {
+  result.clear();
+  
+  // Check whether the temporary directory is specified by an environment
+  // variable.
+  const char *EnvironmentVariable;
+#ifdef LLVM_ON_WIN32
+  EnvironmentVariable = "TEMP";
+#else
+  EnvironmentVariable = "TMPDIR";
+#endif
+  if (char *RequestedDir = getenv(EnvironmentVariable)) {
+    result.append(RequestedDir, RequestedDir + strlen(RequestedDir));
+    return;
+  }
+    
+  // Fall back to a system default.
+  const char *DefaultResult;
+#ifdef LLVM_ON_WIN32
+  (void)erasedOnReboot;
+  DefaultResult = "C:\\TEMP";
+#else
+  if (erasedOnReboot)
+    DefaultResult = "/tmp";
+  else
+    DefaultResult = "/var/tmp";
+#endif
+  result.append(DefaultResult, DefaultResult + strlen(DefaultResult));
+}
+  
 bool has_root_name(const Twine &path) {
   SmallString<128> path_storage;
   StringRef p = path.toStringRef(path_storage);
@@ -626,7 +656,7 @@ error_code create_directories(const Twine &path, bool &existed) {
   if (error_code ec = fs::exists(parent, parent_exists)) return ec;
 
   if (!parent_exists)
-    return create_directories(parent, existed);
+    if (error_code ec = create_directories(parent, existed)) return ec;
 
   return create_directory(p, existed);
 }
@@ -682,14 +712,12 @@ bool is_other(file_status status) {
          !is_symlink(status);
 }
 
-void directory_entry::replace_filename(const Twine &filename, file_status st,
-                                       file_status symlink_st) {
+void directory_entry::replace_filename(const Twine &filename, file_status st) {
   SmallString<128> path(Path.begin(), Path.end());
   path::remove_filename(path);
   path::append(path, filename);
   Path = path.str();
   Status = st;
-  SymlinkStatus = symlink_st;
 }
 
 error_code has_magic(const Twine &path, const Twine &magic, bool &result) {
diff --git a/contrib/llvm/lib/Support/PrettyStackTrace.cpp b/contrib/llvm/lib/Support/PrettyStackTrace.cpp
index 082b7012eb23..ef3307317c4a 100644
--- a/contrib/llvm/lib/Support/PrettyStackTrace.cpp
+++ b/contrib/llvm/lib/Support/PrettyStackTrace.cpp
@@ -62,7 +62,7 @@ extern "C" {
 CRASH_REPORTER_CLIENT_HIDDEN 
 struct crashreporter_annotations_t gCRAnnotations 
         __attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) 
-        = { CRASHREPORTER_ANNOTATIONS_VERSION, 0, 0, 0, 0 };
+        = { CRASHREPORTER_ANNOTATIONS_VERSION, 0, 0, 0, 0, 0, 0 };
 }
 #elif defined (__APPLE__) && HAVE_CRASHREPORTER_INFO
 static const char *__crashreporter_info__ = 0;
diff --git a/contrib/llvm/lib/Support/RWMutex.cpp b/contrib/llvm/lib/Support/RWMutex.cpp
index fc02f9cf7c11..d0b1e10b56fb 100644
--- a/contrib/llvm/lib/Support/RWMutex.cpp
+++ b/contrib/llvm/lib/Support/RWMutex.cpp
@@ -152,6 +152,6 @@ RWMutexImpl::writer_release()
 #elif defined( LLVM_ON_WIN32)
 #include "Windows/RWMutex.inc"
 #else
-#warning Neither LLVM_ON_UNIX nor LLVM_ON_WIN32 was set in System/Mutex.cpp
+#warning Neither LLVM_ON_UNIX nor LLVM_ON_WIN32 was set in Support/Mutex.cpp
 #endif
 #endif
diff --git a/contrib/llvm/lib/Support/SearchForAddressOfSpecialSymbol.cpp b/contrib/llvm/lib/Support/SearchForAddressOfSpecialSymbol.cpp
index d63830185c32..2d23902bb85c 100644
--- a/contrib/llvm/lib/Support/SearchForAddressOfSpecialSymbol.cpp
+++ b/contrib/llvm/lib/Support/SearchForAddressOfSpecialSymbol.cpp
@@ -28,21 +28,6 @@ static void *DoSearch(const char* symbolName) {
 
 #ifdef __APPLE__
   {
-    EXPLICIT_SYMBOL(__ashldi3);
-    EXPLICIT_SYMBOL(__ashrdi3);
-    EXPLICIT_SYMBOL(__cmpdi2);
-    EXPLICIT_SYMBOL(__divdi3);
-    EXPLICIT_SYMBOL(__fixdfdi);
-    EXPLICIT_SYMBOL(__fixsfdi);
-    EXPLICIT_SYMBOL(__fixunsdfdi);
-    EXPLICIT_SYMBOL(__fixunssfdi);
-    EXPLICIT_SYMBOL(__floatdidf);
-    EXPLICIT_SYMBOL(__floatdisf);
-    EXPLICIT_SYMBOL(__lshrdi3);
-    EXPLICIT_SYMBOL(__moddi3);
-    EXPLICIT_SYMBOL(__udivdi3);
-    EXPLICIT_SYMBOL(__umoddi3);
-
     // __eprintf is sometimes used for assert() handling on x86.
     //
     // FIXME: Currently disabled when using Clang, as we don't always have our
diff --git a/contrib/llvm/lib/Support/StringExtras.cpp b/contrib/llvm/lib/Support/StringExtras.cpp
index eb2fa084218a..49c5ac4252c8 100644
--- a/contrib/llvm/lib/Support/StringExtras.cpp
+++ b/contrib/llvm/lib/Support/StringExtras.cpp
@@ -51,11 +51,10 @@ std::pair<StringRef, StringRef> llvm::getToken(StringRef Source,
 void llvm::SplitString(StringRef Source,
                        SmallVectorImpl<StringRef> &OutFragments,
                        StringRef Delimiters) {
-  StringRef S2, S;
-  tie(S2, S) = getToken(Source, Delimiters);
-  while (!S2.empty()) {
-    OutFragments.push_back(S2);
-    tie(S2, S) = getToken(S, Delimiters);
+  std::pair<StringRef, StringRef> S = getToken(Source, Delimiters);
+  while (!S.first.empty()) {
+    OutFragments.push_back(S.first);
+    S = getToken(S.second, Delimiters);
   }
 }
 
diff --git a/contrib/llvm/lib/Support/StringRef.cpp b/contrib/llvm/lib/Support/StringRef.cpp
index 8c3fc094cd11..b5b4f9476026 100644
--- a/contrib/llvm/lib/Support/StringRef.cpp
+++ b/contrib/llvm/lib/Support/StringRef.cpp
@@ -46,12 +46,12 @@ int StringRef::compare_lower(StringRef RHS) const {
 /// compare_numeric - Compare strings, handle embedded numbers.
 int StringRef::compare_numeric(StringRef RHS) const {
   for (size_t I = 0, E = min(Length, RHS.Length); I != E; ++I) {
-    if (Data[I] == RHS.Data[I])
-      continue;
+    // Check for sequences of digits.
     if (ascii_isdigit(Data[I]) && ascii_isdigit(RHS.Data[I])) {
-      // The longer sequence of numbers is larger. This doesn't really handle
-      // prefixed zeros well.
-      for (size_t J = I+1; J != E+1; ++J) {
+      // The longer sequence of numbers is considered larger.
+      // This doesn't really handle prefixed zeros well.
+      size_t J;
+      for (J = I + 1; J != E + 1; ++J) {
         bool ld = J < Length && ascii_isdigit(Data[J]);
         bool rd = J < RHS.Length && ascii_isdigit(RHS.Data[J]);
         if (ld != rd)
@@ -59,8 +59,15 @@ int StringRef::compare_numeric(StringRef RHS) const {
         if (!rd)
           break;
       }
+      // The two number sequences have the same length (J-I), just memcmp them.
+      if (int Res = compareMemory(Data + I, RHS.Data + I, J - I))
+        return Res < 0 ? -1 : 1;
+      // Identical number sequences, continue search after the numbers.
+      I = J - 1;
+      continue;
     }
-    return (unsigned char)Data[I] < (unsigned char)RHS.Data[I] ? -1 : 1;
+    if (Data[I] != RHS.Data[I])
+      return (unsigned char)Data[I] < (unsigned char)RHS.Data[I] ? -1 : 1;
   }
   if (Length == RHS.Length)
     return 0;
diff --git a/contrib/llvm/lib/Support/TargetRegistry.cpp b/contrib/llvm/lib/Support/TargetRegistry.cpp
index 293a5d7a0168..7497bfe035c6 100644
--- a/contrib/llvm/lib/Support/TargetRegistry.cpp
+++ b/contrib/llvm/lib/Support/TargetRegistry.cpp
@@ -7,9 +7,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Host.h"
+#include "llvm/Support/raw_ostream.h"
 #include <cassert>
+#include <vector>
 using namespace llvm;
 
 // Clients are responsible for avoid race conditions in registration.
@@ -90,3 +94,29 @@ const Target *TargetRegistry::getClosestTargetForJIT(std::string &Error) {
   return TheTarget;
 }
 
+static int TargetArraySortFn(const void *LHS, const void *RHS) {
+  typedef std::pair<StringRef, const Target*> pair_ty;
+  return ((const pair_ty*)LHS)->first.compare(((const pair_ty*)RHS)->first);
+}
+
+void TargetRegistry::printRegisteredTargetsForVersion() {
+  std::vector<std::pair<StringRef, const Target*> > Targets;
+  size_t Width = 0;
+  for (TargetRegistry::iterator I = TargetRegistry::begin(),
+       E = TargetRegistry::end();
+       I != E; ++I) {
+    Targets.push_back(std::make_pair(I->getName(), &*I));
+    Width = std::max(Width, Targets.back().first.size());
+  }
+  array_pod_sort(Targets.begin(), Targets.end(), TargetArraySortFn);
+
+  raw_ostream &OS = outs();
+  OS << "  Registered Targets:\n";
+  for (unsigned i = 0, e = Targets.size(); i != e; ++i) {
+    OS << "    " << Targets[i].first;
+    OS.indent(Width - Targets[i].first.size()) << " - "
+      << Targets[i].second->getShortDescription() << '\n';
+  }
+  if (Targets.empty())
+    OS << "    (none)\n";
+}
diff --git a/contrib/llvm/lib/Support/ThreadLocal.cpp b/contrib/llvm/lib/Support/ThreadLocal.cpp
index 6b43048da155..fdb251c0a36b 100644
--- a/contrib/llvm/lib/Support/ThreadLocal.cpp
+++ b/contrib/llvm/lib/Support/ThreadLocal.cpp
@@ -79,6 +79,6 @@ void ThreadLocalImpl::removeInstance() {
 #elif defined( LLVM_ON_WIN32)
 #include "Windows/ThreadLocal.inc"
 #else
-#warning Neither LLVM_ON_UNIX nor LLVM_ON_WIN32 was set in System/ThreadLocal.cpp
+#warning Neither LLVM_ON_UNIX nor LLVM_ON_WIN32 set in Support/ThreadLocal.cpp
 #endif
 #endif
diff --git a/contrib/llvm/lib/Support/Threading.cpp b/contrib/llvm/lib/Support/Threading.cpp
index b62b1a94b234..8f0bb93eb4d1 100644
--- a/contrib/llvm/lib/Support/Threading.cpp
+++ b/contrib/llvm/lib/Support/Threading.cpp
@@ -24,7 +24,7 @@ static bool multithreaded_mode = false;
 static sys::Mutex* global_lock = 0;
 
 bool llvm::llvm_start_multithreaded() {
-#if defined(LLVM_MULTITHREADED) && LLVM_MULTITHREADED == 1
+#if ENABLE_THREADS != 0
   assert(!multithreaded_mode && "Already multithreaded!");
   multithreaded_mode = true;
   global_lock = new sys::Mutex(true);
@@ -39,7 +39,7 @@ bool llvm::llvm_start_multithreaded() {
 }
 
 void llvm::llvm_stop_multithreaded() {
-#if defined(LLVM_MULTITHREADED) && LLVM_MULTITHREADED == 1
+#if ENABLE_THREADS != 0
   assert(multithreaded_mode && "Not currently multithreaded!");
 
   // We fence here to insure that all threaded operations are complete BEFORE we
@@ -63,7 +63,7 @@ void llvm::llvm_release_global_lock() {
   if (multithreaded_mode) global_lock->release();
 }
 
-#if defined(LLVM_MULTITHREADED) && LLVM_MULTITHREADED == 1 && defined(HAVE_PTHREAD_H)
+#if ENABLE_THREADS != 0 && defined(HAVE_PTHREAD_H)
 #include <pthread.h>
 
 struct ThreadInfo {
@@ -102,11 +102,42 @@ void llvm::llvm_execute_on_thread(void (*Fn)(void*), void *UserData,
  error:
   ::pthread_attr_destroy(&Attr);
 }
+#elif ENABLE_THREADS!=0 && defined(LLVM_ON_WIN32)
+#include "Windows/Windows.h"
+#include <process.h>
 
+struct ThreadInfo {
+  void (*func)(void*);
+  void *param;
+};
+
+static unsigned __stdcall ThreadCallback(void *param) {
+  struct ThreadInfo *info = reinterpret_cast<struct ThreadInfo *>(param);
+  info->func(info->param);
+
+  return 0;
+}
+
+void llvm::llvm_execute_on_thread(void (*Fn)(void*), void *UserData,
+                                  unsigned RequestedStackSize) {
+  struct ThreadInfo param = { Fn, UserData };
+
+  HANDLE hThread = (HANDLE)::_beginthreadex(NULL,
+                                            RequestedStackSize, ThreadCallback,
+                                            &param, 0, NULL);
+
+  if (hThread) {
+    // We actually don't care whether the wait succeeds or fails, in
+    // the same way we don't care whether the pthread_join call succeeds
+    // or fails.  There's not much we could do if this were to fail. But
+    // on success, this call will wait until the thread finishes executing
+    // before returning.
+    (void)::WaitForSingleObject(hThread, INFINITE);
+    ::CloseHandle(hThread);
+  }
+}
 #else
-
-// No non-pthread implementation, currently.
-
+// Support for non-Win32, non-pthread implementation.
 void llvm::llvm_execute_on_thread(void (*Fn)(void*), void *UserData,
                                   unsigned RequestedStackSize) {
   (void) RequestedStackSize;
diff --git a/contrib/llvm/lib/Support/Triple.cpp b/contrib/llvm/lib/Support/Triple.cpp
index 7e094ee78f36..c61af372d79c 100644
--- a/contrib/llvm/lib/Support/Triple.cpp
+++ b/contrib/llvm/lib/Support/Triple.cpp
@@ -8,16 +8,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/Triple.h"
-
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/Twine.h"
-#include <cassert>
 #include <cstring>
 using namespace llvm;
 
-//
-
 const char *Triple::getArchTypeName(ArchType Kind) {
   switch (Kind) {
   case InvalidArch: return "<invalid>";
@@ -29,6 +24,8 @@ const char *Triple::getArchTypeName(ArchType Kind) {
   case cellspu: return "cellspu";
   case mips:    return "mips";
   case mipsel:  return "mipsel";
+  case mips64:  return "mips64";
+  case mips64el:return "mips64el";
   case msp430:  return "msp430";
   case ppc64:   return "powerpc64";
   case ppc:     return "powerpc";
@@ -43,6 +40,8 @@ const char *Triple::getArchTypeName(ArchType Kind) {
   case mblaze:  return "mblaze";
   case ptx32:   return "ptx32";
   case ptx64:   return "ptx64";
+  case le32:    return "le32";
+  case amdil:   return "amdil";
   }
 
   return "<invalid>";
@@ -77,6 +76,8 @@ const char *Triple::getArchTypePrefix(ArchType Kind) {
 
   case ptx32:   return "ptx";
   case ptx64:   return "ptx";
+  case le32:    return "le32";
+  case amdil:   return "amdil";
   }
 }
 
@@ -102,6 +103,7 @@ const char *Triple::getOSTypeName(OSType Kind) {
   case DragonFly: return "dragonfly";
   case FreeBSD: return "freebsd";
   case IOS: return "ios";
+  case KFreeBSD: return "kfreebsd";
   case Linux: return "linux";
   case Lv2: return "lv2";
   case MacOSX: return "macosx";
@@ -114,6 +116,7 @@ const char *Triple::getOSTypeName(OSType Kind) {
   case Haiku: return "haiku";
   case Minix: return "minix";
   case RTEMS: return "rtems";
+  case NativeClient: return "nacl";
   }
 
   return "<invalid>";
@@ -144,10 +147,16 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
     return mips;
   if (Name == "mipsel")
     return mipsel;
+  if (Name == "mips64")
+    return mips64;
+  if (Name == "mips64el")
+    return mips64el;
   if (Name == "msp430")
     return msp430;
   if (Name == "ppc64")
     return ppc64;
+  if (Name == "ppc32")
+    return ppc;
   if (Name == "ppc")
     return ppc;
   if (Name == "mblaze")
@@ -172,6 +181,10 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
     return ptx32;
   if (Name == "ptx64")
     return ptx64;
+  if (Name == "le32")
+    return le32;
+  if (Name == "amdil")
+      return amdil;
 
   return UnknownArch;
 }
@@ -207,13 +220,16 @@ Triple::ArchType Triple::getArchTypeForDarwinArchName(StringRef Str) {
 
   // This is derived from the driver driver.
   if (Str == "arm" || Str == "armv4t" || Str == "armv5" || Str == "xscale" ||
-      Str == "armv6" || Str == "armv7")
+      Str == "armv6" || Str == "armv7" || Str == "armv7f" || Str == "armv7k" ||
+      Str == "armv7s")
     return Triple::arm;
 
   if (Str == "ptx32")
     return Triple::ptx32;
   if (Str == "ptx64")
     return Triple::ptx64;
+  if (Str == "amdil")
+      return Triple::amdil;
 
   return Triple::UnknownArch;
 }
@@ -249,6 +265,10 @@ const char *Triple::getArchNameForAssembler() {
     return "ptx32";
   if (Str == "ptx64")
     return "ptx64";
+  if (Str == "le32")
+    return "le32";
+  if (Str == "amdil")
+      return "amdil";
   return NULL;
 }
 
@@ -288,6 +308,10 @@ Triple::ArchType Triple::ParseArch(StringRef ArchName) {
   else if (ArchName == "mipsel" || ArchName == "mipsallegrexel" ||
            ArchName == "psp")
     return mipsel;
+  else if (ArchName == "mips64" || ArchName == "mips64eb")
+    return mips64;
+  else if (ArchName == "mips64el")
+    return mips64el;
   else if (ArchName == "sparc")
     return sparc;
   else if (ArchName == "sparcv9")
@@ -302,6 +326,10 @@ Triple::ArchType Triple::ParseArch(StringRef ArchName) {
     return ptx32;
   else if (ArchName == "ptx64")
     return ptx64;
+  else if (ArchName == "le32")
+    return le32;
+  else if (ArchName == "amdil")
+      return amdil;
   else
     return UnknownArch;
 }
@@ -330,6 +358,8 @@ Triple::OSType Triple::ParseOS(StringRef OSName) {
     return FreeBSD;
   else if (OSName.startswith("ios"))
     return IOS;
+  else if (OSName.startswith("kfreebsd"))
+    return KFreeBSD;
   else if (OSName.startswith("linux"))
     return Linux;
   else if (OSName.startswith("lv2"))
@@ -354,6 +384,8 @@ Triple::OSType Triple::ParseOS(StringRef OSName) {
     return Minix;
   else if (OSName.startswith("rtems"))
     return RTEMS;
+  else if (OSName.startswith("nacl"))
+    return NativeClient;
   else
     return UnknownOS;
 }
diff --git a/contrib/llvm/lib/Support/Twine.cpp b/contrib/llvm/lib/Support/Twine.cpp
index d62123cc985e..3d04bc34f0eb 100644
--- a/contrib/llvm/lib/Support/Twine.cpp
+++ b/contrib/llvm/lib/Support/Twine.cpp
@@ -16,7 +16,7 @@ using namespace llvm;
 std::string Twine::str() const {
   // If we're storing only a std::string, just return it.
   if (LHSKind == StdStringKind && RHSKind == EmptyKind)
-    return *static_cast<const std::string*>(LHS);
+    return *LHS.stdString;
 
   // Otherwise, flatten and copy the contents first.
   SmallString<256> Vec;
@@ -40,9 +40,9 @@ StringRef Twine::toNullTerminatedStringRef(SmallVectorImpl<char> &Out) const {
     switch (getLHSKind()) {
     case CStringKind:
       // Already null terminated, yay!
-      return StringRef(static_cast<const char*>(LHS));
+      return StringRef(LHS.cString);
     case StdStringKind: {
-      const std::string *str = static_cast<const std::string*>(LHS);
+      const std::string *str = LHS.stdString;
       return StringRef(str->c_str(), str->size());
     }
     default:
@@ -55,48 +55,51 @@ StringRef Twine::toNullTerminatedStringRef(SmallVectorImpl<char> &Out) const {
   return StringRef(Out.data(), Out.size());
 }
 
-void Twine::printOneChild(raw_ostream &OS, const void *Ptr,
+void Twine::printOneChild(raw_ostream &OS, Child Ptr,
                           NodeKind Kind) const {
   switch (Kind) {
   case Twine::NullKind: break;
   case Twine::EmptyKind: break;
   case Twine::TwineKind:
-    static_cast<const Twine*>(Ptr)->print(OS);
+    Ptr.twine->print(OS);
     break;
   case Twine::CStringKind:
-    OS << static_cast<const char*>(Ptr);
+    OS << Ptr.cString;
     break;
   case Twine::StdStringKind:
-    OS << *static_cast<const std::string*>(Ptr);
+    OS << *Ptr.stdString;
     break;
   case Twine::StringRefKind:
-    OS << *static_cast<const StringRef*>(Ptr);
+    OS << *Ptr.stringRef;
+    break;
+  case Twine::CharKind:
+    OS << Ptr.character;
     break;
   case Twine::DecUIKind:
-    OS << (unsigned)(uintptr_t)Ptr;
+    OS << Ptr.decUI;
     break;
   case Twine::DecIKind:
-    OS << (int)(intptr_t)Ptr;
+    OS << Ptr.decI;
     break;
   case Twine::DecULKind:
-    OS << *static_cast<const unsigned long*>(Ptr);
+    OS << *Ptr.decUL;
     break;
   case Twine::DecLKind:
-    OS << *static_cast<const long*>(Ptr);
+    OS << *Ptr.decL;
     break;
   case Twine::DecULLKind:
-    OS << *static_cast<const unsigned long long*>(Ptr);
+    OS << *Ptr.decULL;
     break;
   case Twine::DecLLKind:
-    OS << *static_cast<const long long*>(Ptr);
+    OS << *Ptr.decLL;
     break;
   case Twine::UHexKind:
-    OS.write_hex(*static_cast<const uint64_t*>(Ptr));
+    OS.write_hex(*Ptr.uHex);
     break;
   }
 }
 
-void Twine::printOneChildRepr(raw_ostream &OS, const void *Ptr,
+void Twine::printOneChildRepr(raw_ostream &OS, Child Ptr,
                               NodeKind Kind) const {
   switch (Kind) {
   case Twine::NullKind:
@@ -105,40 +108,43 @@ void Twine::printOneChildRepr(raw_ostream &OS, const void *Ptr,
     OS << "empty"; break;
   case Twine::TwineKind:
     OS << "rope:";
-    static_cast<const Twine*>(Ptr)->printRepr(OS);
+    Ptr.twine->printRepr(OS);
     break;
   case Twine::CStringKind:
     OS << "cstring:\""
-       << static_cast<const char*>(Ptr) << "\"";
+       << Ptr.cString << "\"";
     break;
   case Twine::StdStringKind:
     OS << "std::string:\""
-       << static_cast<const std::string*>(Ptr) << "\"";
+       << Ptr.stdString << "\"";
     break;
   case Twine::StringRefKind:
     OS << "stringref:\""
-       << static_cast<const StringRef*>(Ptr) << "\"";
+       << Ptr.stringRef << "\"";
+    break;
+  case Twine::CharKind:
+    OS << "char:\"" << Ptr.character << "\"";
     break;
   case Twine::DecUIKind:
-    OS << "decUI:\"" << (unsigned)(uintptr_t)Ptr << "\"";
+    OS << "decUI:\"" << Ptr.decUI << "\"";
     break;
   case Twine::DecIKind:
-    OS << "decI:\"" << (int)(intptr_t)Ptr << "\"";
+    OS << "decI:\"" << Ptr.decI << "\"";
     break;
   case Twine::DecULKind:
-    OS << "decUL:\"" << *static_cast<const unsigned long*>(Ptr) << "\"";
+    OS << "decUL:\"" << *Ptr.decUL << "\"";
     break;
   case Twine::DecLKind:
-    OS << "decL:\"" << *static_cast<const long*>(Ptr) << "\"";
+    OS << "decL:\"" << *Ptr.decL << "\"";
     break;
   case Twine::DecULLKind:
-    OS << "decULL:\"" << *static_cast<const unsigned long long*>(Ptr) << "\"";
+    OS << "decULL:\"" << *Ptr.decULL << "\"";
     break;
   case Twine::DecLLKind:
-    OS << "decLL:\"" << *static_cast<const long long*>(Ptr) << "\"";
+    OS << "decLL:\"" << *Ptr.decLL << "\"";
     break;
   case Twine::UHexKind:
-    OS << "uhex:\"" << static_cast<const uint64_t*>(Ptr) << "\"";
+    OS << "uhex:\"" << Ptr.uHex << "\"";
     break;
   }
 }
diff --git a/contrib/llvm/lib/Support/Unix/Host.inc b/contrib/llvm/lib/Support/Unix/Host.inc
index eacb08b3fb00..1e438e75fa43 100644
--- a/contrib/llvm/lib/Support/Unix/Host.inc
+++ b/contrib/llvm/lib/Support/Unix/Host.inc
@@ -22,6 +22,7 @@
 #include <sys/utsname.h>
 #include <cctype>
 #include <string>
+#include <cstdlib> // ::getenv
 
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Support/Unix/Path.inc b/contrib/llvm/lib/Support/Unix/Path.inc
index f295b92e4a5b..85c7c4022f48 100644
--- a/contrib/llvm/lib/Support/Unix/Path.inc
+++ b/contrib/llvm/lib/Support/Unix/Path.inc
@@ -252,8 +252,8 @@ Path::GetUserHomeDirectory() {
 Path
 Path::GetCurrentDirectory() {
   char pathname[MAXPATHLEN];
-  if (!getcwd(pathname,MAXPATHLEN)) {
-    assert (false && "Could not query current working directory.");
+  if (!getcwd(pathname, MAXPATHLEN)) {
+    assert(false && "Could not query current working directory.");
     return Path();
   }
 
diff --git a/contrib/llvm/lib/Support/Unix/PathV2.inc b/contrib/llvm/lib/Support/Unix/PathV2.inc
index 03ff28367e44..bbbc344661be 100644
--- a/contrib/llvm/lib/Support/Unix/PathV2.inc
+++ b/contrib/llvm/lib/Support/Unix/PathV2.inc
@@ -42,6 +42,9 @@
 #if HAVE_STDIO_H
 #include <stdio.h>
 #endif
+#if HAVE_LIMITS_H
+#include <limits.h>
+#endif
 
 using namespace llvm;
 
@@ -342,19 +345,22 @@ error_code status(const Twine &path, file_status &result) {
 }
 
 error_code unique_file(const Twine &model, int &result_fd,
-                             SmallVectorImpl<char> &result_path) {
+                             SmallVectorImpl<char> &result_path,
+                             bool makeAbsolute) {
   SmallString<128> Model;
   model.toVector(Model);
   // Null terminate.
   Model.c_str();
 
-  // Make model absolute by prepending a temp directory if it's not already.
-  bool absolute = path::is_absolute(Twine(Model));
-  if (!absolute) {
-    SmallString<128> TDir;
-    if (error_code ec = TempDir(TDir)) return ec;
-    path::append(TDir, Twine(Model));
-    Model.swap(TDir);
+  if (makeAbsolute) {
+    // Make model absolute by prepending a temp directory if it's not already.
+    bool absolute = path::is_absolute(Twine(Model));
+    if (!absolute) {
+      SmallString<128> TDir;
+      if (error_code ec = TempDir(TDir)) return ec;
+      path::append(TDir, Twine(Model));
+      Model.swap(TDir);
+    }
   }
 
   // Replace '%' with random chars. From here on, DO NOT modify model. It may be
diff --git a/contrib/llvm/lib/Support/Unix/Process.inc b/contrib/llvm/lib/Support/Unix/Process.inc
index 5cdb11ccebc4..da440fd48f3d 100644
--- a/contrib/llvm/lib/Support/Unix/Process.inc
+++ b/contrib/llvm/lib/Support/Unix/Process.inc
@@ -293,3 +293,7 @@ const char *Process::OutputBold(bool bg) {
 const char *Process::ResetColor() {
   return "\033[0m";
 }
+
+void Process::SetWorkingDirectory(std::string Path) {
+  ::chdir(Path.c_str());
+}
diff --git a/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc b/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc
index fc5f5809cb40..83da82a949ca 100644
--- a/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc
+++ b/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc
@@ -39,7 +39,7 @@ using namespace sys;
 //===          and must not be UNIX code.
 //===----------------------------------------------------------------------===//
 
-static std::vector<HMODULE> OpenedHandles;
+static DenseSet<HMODULE> *OpenedHandles;
 
 extern "C" {
 
@@ -63,30 +63,43 @@ extern "C" {
 #endif
         stricmp(ModuleName, "msvcrt20") != 0 &&
         stricmp(ModuleName, "msvcrt40") != 0) {
-      OpenedHandles.push_back((HMODULE)ModuleBase);
+      OpenedHandles->insert((HMODULE)ModuleBase);
     }
     return TRUE;
   }
 }
 
-bool DynamicLibrary::LoadLibraryPermanently(const char *filename,
-                                            std::string *ErrMsg) {
-  if (filename) {
-    HMODULE a_handle = LoadLibrary(filename);
+DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *filename,
+                                                   std::string *errMsg) {
+  SmartScopedLock<true> lock(getMutex());
 
-    if (a_handle == 0)
-      return MakeErrMsg(ErrMsg, std::string(filename) + ": Can't open : ");
+  if (!filename) {
+    // When no file is specified, enumerate all DLLs and EXEs in the process.
+    if (OpenedHandles == 0)
+      OpenedHandles = new DenseSet<HMODULE>();
 
-    OpenedHandles.push_back(a_handle);
-  } else {
-    // When no file is specified, enumerate all DLLs and EXEs in the
-    // process.
     EnumerateLoadedModules(GetCurrentProcess(), ELM_Callback, 0);
+    // Dummy library that represents "search all handles".
+    // This is mostly to ensure that the return value still shows up as "valid".
+    return DynamicLibrary(&OpenedHandles);
+  }
+  
+  HMODULE a_handle = LoadLibrary(filename);
+
+  if (a_handle == 0) {
+    MakeErrMsg(errMsg, std::string(filename) + ": Can't open : ");
+    return DynamicLibrary();
   }
 
-  // Because we don't remember the handle, we will never free it; hence,
-  // it is loaded permanently.
-  return false;
+  if (OpenedHandles == 0)
+    OpenedHandles = new DenseSet<HMODULE>();
+
+  // If we've already loaded this library, FreeLibrary() the handle in order to
+  // keep the internal refcount at +1.
+  if (!OpenedHandles->insert(a_handle).second)
+    FreeLibrary(a_handle);
+
+  return DynamicLibrary(a_handle);
 }
 
 // Stack probing routines are in the support library (e.g. libgcc), but we don't
@@ -101,21 +114,24 @@ bool DynamicLibrary::LoadLibraryPermanently(const char *filename,
 #undef EXPLICIT_SYMBOL2
 
 void* DynamicLibrary::SearchForAddressOfSymbol(const char* symbolName) {
+  SmartScopedLock<true> Lock(getMutex());
+
   // First check symbols added via AddSymbol().
   if (ExplicitSymbols) {
-    std::map<std::string, void *>::iterator I =
-      ExplicitSymbols->find(symbolName);
-    std::map<std::string, void *>::iterator E = ExplicitSymbols->end();
-    if (I != E)
-      return I->second;
+    StringMap<void *>::iterator i = ExplicitSymbols->find(symbolName);
+
+    if (i != ExplicitSymbols->end())
+      return i->second;
   }
 
   // Now search the libraries.
-  for (std::vector<HMODULE>::iterator I = OpenedHandles.begin(),
-       E = OpenedHandles.end(); I != E; ++I) {
-    FARPROC ptr = GetProcAddress((HMODULE)*I, symbolName);
-    if (ptr) {
-      return (void *)(intptr_t)ptr;
+  if (OpenedHandles) {
+    for (DenseSet<HMODULE>::iterator I = OpenedHandles->begin(),
+         E = OpenedHandles->end(); I != E; ++I) {
+      FARPROC ptr = GetProcAddress((HMODULE)*I, symbolName);
+      if (ptr) {
+        return (void *)(intptr_t)ptr;
+      }
     }
   }
 
@@ -134,4 +150,14 @@ void* DynamicLibrary::SearchForAddressOfSymbol(const char* symbolName) {
   return 0;
 }
 
+
+void *DynamicLibrary::getAddressOfSymbol(const char *symbolName) {
+  if (!isValid())
+    return NULL;
+  if (Data == &OpenedHandles)
+    return SearchForAddressOfSymbol(symbolName);
+  return (void *)(intptr_t)GetProcAddress((HMODULE)Data, symbolName);
+}
+
+
 }
diff --git a/contrib/llvm/lib/Support/Windows/Memory.inc b/contrib/llvm/lib/Support/Windows/Memory.inc
index 9f69e7367e6f..fcc72837c456 100644
--- a/contrib/llvm/lib/Support/Windows/Memory.inc
+++ b/contrib/llvm/lib/Support/Windows/Memory.inc
@@ -32,11 +32,16 @@ MemoryBlock Memory::AllocateRWX(size_t NumBytes,
   static const size_t pageSize = Process::GetPageSize();
   size_t NumPages = (NumBytes+pageSize-1)/pageSize;
 
-  //FIXME: support NearBlock if ever needed on Win64.
+  PVOID start = NearBlock ? static_cast<unsigned char *>(NearBlock->base()) +
+                                NearBlock->size() : NULL;
 
-  void *pa = VirtualAlloc(NULL, NumPages*pageSize, MEM_COMMIT,
+  void *pa = VirtualAlloc(start, NumPages*pageSize, MEM_RESERVE | MEM_COMMIT,
                   PAGE_EXECUTE_READWRITE);
   if (pa == NULL) {
+    if (NearBlock) {
+      // Try again without the NearBlock hint
+      return AllocateRWX(NumBytes, NULL, ErrMsg);
+    }
     MakeErrMsg(ErrMsg, "Can't allocate RWX Memory: ");
     return MemoryBlock();
   }
@@ -54,20 +59,62 @@ bool Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) {
   return false;
 }
 
+static DWORD getProtection(const void *addr) {
+  MEMORY_BASIC_INFORMATION info;
+  if (sizeof(info) == ::VirtualQuery(addr, &info, sizeof(info))) {
+    return info.Protect;
+  }
+  return 0;
+}
+
 bool Memory::setWritable(MemoryBlock &M, std::string *ErrMsg) {
+  if (!setRangeWritable(M.Address, M.Size)) {
+    return MakeErrMsg(ErrMsg, "Cannot set memory to writeable: ");
+  }
   return true;
 }
 
 bool Memory::setExecutable(MemoryBlock &M, std::string *ErrMsg) {
-  return false;
-}
-
-bool Memory::setRangeWritable(const void *Addr, size_t Size) {
+  if (!setRangeExecutable(M.Address, M.Size)) {
+    return MakeErrMsg(ErrMsg, "Cannot set memory to executable: ");
+  }
   return true;
 }
 
+bool Memory::setRangeWritable(const void *Addr, size_t Size) {
+  DWORD prot = getProtection(Addr);
+  if (!prot)
+    return false;
+
+  if (prot == PAGE_EXECUTE || prot == PAGE_EXECUTE_READ) {
+    prot = PAGE_EXECUTE_READWRITE;
+  } else if (prot == PAGE_NOACCESS || prot == PAGE_READONLY) {
+    prot = PAGE_READWRITE;
+  }
+
+  DWORD oldProt;
+  sys::Memory::InvalidateInstructionCache(Addr, Size);
+  return ::VirtualProtect(const_cast<LPVOID>(Addr), Size, prot, &oldProt)
+            == TRUE;
+}
+
 bool Memory::setRangeExecutable(const void *Addr, size_t Size) {
-  return false;
+  DWORD prot = getProtection(Addr);
+  if (!prot)
+    return false;
+
+  if (prot == PAGE_NOACCESS) {
+    prot = PAGE_EXECUTE;
+  } else if (prot == PAGE_READONLY) {
+    prot = PAGE_EXECUTE_READ;
+  } else if (prot == PAGE_READWRITE) {
+    prot = PAGE_EXECUTE_READWRITE;
+  }
+
+  DWORD oldProt;
+  sys::Memory::InvalidateInstructionCache(Addr, Size);
+  return ::VirtualProtect(const_cast<LPVOID>(Addr), Size, prot, &oldProt)
+            == TRUE;
 }
 
 }
diff --git a/contrib/llvm/lib/Support/Windows/PathV2.inc b/contrib/llvm/lib/Support/Windows/PathV2.inc
index af71b73cd693..bc597b2dcc89 100644
--- a/contrib/llvm/lib/Support/Windows/PathV2.inc
+++ b/contrib/llvm/lib/Support/Windows/PathV2.inc
@@ -445,13 +445,35 @@ error_code file_size(const Twine &path, uint64_t &result) {
   return success;
 }
 
+static bool isReservedName(StringRef path) {
+  // This list of reserved names comes from MSDN, at:
+  // http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx
+  static const char *sReservedNames[] = { "nul", "con", "prn", "aux",
+                              "com1", "com2", "com3", "com4", "com5", "com6",
+                              "com7", "com8", "com9", "lpt1", "lpt2", "lpt3",
+                              "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9" };
+
+  // First, check to see if this is a device namespace, which always
+  // starts with \\.\, since device namespaces are not legal file paths.
+  if (path.startswith("\\\\.\\"))
+    return true;
+
+  // Then compare against the list of ancient reserved names
+  for (size_t i = 0; i < sizeof(sReservedNames) / sizeof(const char *); ++i) {
+    if (path.equals_lower(sReservedNames[i]))
+      return true;
+  }
+
+  // The path isn't what we consider reserved.
+  return false;
+}
+
 error_code status(const Twine &path, file_status &result) {
   SmallString<128> path_storage;
   SmallVector<wchar_t, 128> path_utf16;
 
   StringRef path8 = path.toStringRef(path_storage);
-  // FIXME: We should detect as many "special file name" as possible.
-  if (path8.compare_lower("nul") == 0) {
+  if (isReservedName(path8)) {
     result = file_status(file_type::character_file);
     return success;
   }
@@ -501,7 +523,8 @@ handle_status_error:
 }
 
 error_code unique_file(const Twine &model, int &result_fd,
-                             SmallVectorImpl<char> &result_path) {
+                             SmallVectorImpl<char> &result_path,
+                             bool makeAbsolute) {
   // Use result_path as temp storage.
   result_path.set_size(0);
   StringRef m = model.toStringRef(result_path);
@@ -509,17 +532,19 @@ error_code unique_file(const Twine &model, int &result_fd,
   SmallVector<wchar_t, 128> model_utf16;
   if (error_code ec = UTF8ToUTF16(m, model_utf16)) return ec;
 
-  // Make model absolute by prepending a temp directory if it's not already.
-  bool absolute = path::is_absolute(m);
-
-  if (!absolute) {
-    SmallVector<wchar_t, 64> temp_dir;
-    if (error_code ec = TempDir(temp_dir)) return ec;
-    // Handle c: by removing it.
-    if (model_utf16.size() > 2 && model_utf16[1] == L':') {
-      model_utf16.erase(model_utf16.begin(), model_utf16.begin() + 2);
+  if (makeAbsolute) {
+    // Make model absolute by prepending a temp directory if it's not already.
+    bool absolute = path::is_absolute(m);
+  
+    if (!absolute) {
+      SmallVector<wchar_t, 64> temp_dir;
+      if (error_code ec = TempDir(temp_dir)) return ec;
+      // Handle c: by removing it.
+      if (model_utf16.size() > 2 && model_utf16[1] == L':') {
+        model_utf16.erase(model_utf16.begin(), model_utf16.begin() + 2);
+      }
+      model_utf16.insert(model_utf16.begin(), temp_dir.begin(), temp_dir.end());
     }
-    model_utf16.insert(model_utf16.begin(), temp_dir.begin(), temp_dir.end());
   }
 
   // Replace '%' with random chars. From here on, DO NOT modify model. It may be
diff --git a/contrib/llvm/lib/Support/Windows/Process.inc b/contrib/llvm/lib/Support/Windows/Process.inc
index 06a7f0054d50..fe54eb1a7972 100644
--- a/contrib/llvm/lib/Support/Windows/Process.inc
+++ b/contrib/llvm/lib/Support/Windows/Process.inc
@@ -15,6 +15,7 @@
 #include <psapi.h>
 #include <malloc.h>
 #include <io.h>
+#include <direct.h>
 
 #ifdef __MINGW32__
  #if (HAVE_LIBPSAPI != 1)
@@ -219,4 +220,8 @@ const char *Process::ResetColor() {
   return 0;
 }
 
+void Process::SetWorkingDirectory(std::string Path) {
+  ::_chdir(Path.c_str());
+}
+
 }
diff --git a/contrib/llvm/lib/Support/Windows/RWMutex.inc b/contrib/llvm/lib/Support/Windows/RWMutex.inc
index 471f8fa294be..26b9bba9052c 100644
--- a/contrib/llvm/lib/Support/Windows/RWMutex.inc
+++ b/contrib/llvm/lib/Support/Windows/RWMutex.inc
@@ -18,39 +18,115 @@
 
 #include "Windows.h"
 
-// FIXME: Windows does not have reader-writer locks pre-Vista.  If you want
-// real reader-writer locks, you a threads implementation for Windows.
-
 namespace llvm {
 using namespace sys;
 
+// Windows has slim read-writer lock support on Vista and higher, so we
+// will attempt to load the APIs.  If they exist, we will use them, and
+// if not, we will fall back on critical sections.  When we drop support
+// for XP, we can stop lazy-loading these APIs and just use them directly.
+#if defined(__MINGW32__)
+  // Taken from WinNT.h
+  typedef struct _RTL_SRWLOCK {
+    PVOID Ptr;
+  } RTL_SRWLOCK, *PRTL_SRWLOCK;
+
+  // Taken from WinBase.h
+  typedef RTL_SRWLOCK SRWLOCK, *PSRWLOCK;
+#endif
+
+static VOID (WINAPI *fpInitializeSRWLock)(PSRWLOCK lock) = NULL;
+static VOID (WINAPI *fpAcquireSRWLockExclusive)(PSRWLOCK lock) = NULL;
+static VOID (WINAPI *fpAcquireSRWLockShared)(PSRWLOCK lock) = NULL;
+static VOID (WINAPI *fpReleaseSRWLockExclusive)(PSRWLOCK lock) = NULL;
+static VOID (WINAPI *fpReleaseSRWLockShared)(PSRWLOCK lock) = NULL;
+
+static bool sHasSRW = false;
+
+static bool loadSRW() {
+  static bool sChecked = false;
+  if (!sChecked) {
+    sChecked = true;
+
+    HMODULE hLib = ::LoadLibrary(TEXT("Kernel32"));
+    if (hLib) {
+      fpInitializeSRWLock =
+        (VOID (WINAPI *)(PSRWLOCK))::GetProcAddress(hLib,
+                                               "InitializeSRWLock");
+      fpAcquireSRWLockExclusive =
+        (VOID (WINAPI *)(PSRWLOCK))::GetProcAddress(hLib,
+                                               "AcquireSRWLockExclusive");
+      fpAcquireSRWLockShared =
+        (VOID (WINAPI *)(PSRWLOCK))::GetProcAddress(hLib,
+                                               "AcquireSRWLockShared");
+      fpReleaseSRWLockExclusive =
+        (VOID (WINAPI *)(PSRWLOCK))::GetProcAddress(hLib,
+                                               "ReleaseSRWLockExclusive");
+      fpReleaseSRWLockShared =
+        (VOID (WINAPI *)(PSRWLOCK))::GetProcAddress(hLib,
+                                               "ReleaseSRWLockShared");
+      ::FreeLibrary(hLib);
+
+	  if (fpInitializeSRWLock != NULL) {
+	    sHasSRW = true;
+	  }
+    }
+  }
+  return sHasSRW;
+}
+
 RWMutexImpl::RWMutexImpl() {
-  data_ = calloc(1, sizeof(CRITICAL_SECTION));
-  InitializeCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  if (loadSRW()) {
+    data_ = calloc(1, sizeof(SRWLOCK));
+    fpInitializeSRWLock(static_cast<PSRWLOCK>(data_));
+  } else {
+    data_ = calloc(1, sizeof(CRITICAL_SECTION));
+    InitializeCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  }
 }
 
 RWMutexImpl::~RWMutexImpl() {
-  DeleteCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
-  free(data_);
+  if (sHasSRW) {
+    // Nothing to do in the case of slim reader/writers
+  } else {
+    DeleteCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+    free(data_);
+  }
 }
 
 bool RWMutexImpl::reader_acquire() {
-  EnterCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  if (sHasSRW) {
+    fpAcquireSRWLockShared(static_cast<PSRWLOCK>(data_));
+  } else {
+    EnterCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  }
   return true;
 }
 
 bool RWMutexImpl::reader_release() {
-  LeaveCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  if (sHasSRW) {
+    fpReleaseSRWLockShared(static_cast<PSRWLOCK>(data_));
+  } else {
+    LeaveCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  }
   return true;
 }
 
 bool RWMutexImpl::writer_acquire() {
-  EnterCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  if (sHasSRW) {
+    fpAcquireSRWLockExclusive(static_cast<PSRWLOCK>(data_));
+  } else {
+    EnterCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  }
   return true;
 }
 
 bool RWMutexImpl::writer_release() {
-  LeaveCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  if (sHasSRW) {
+    fpReleaseSRWLockExclusive(static_cast<PSRWLOCK>(data_));
+  } else {
+    LeaveCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
+  }
   return true;
 }
 
diff --git a/contrib/llvm/lib/Support/Windows/Signals.inc b/contrib/llvm/lib/Support/Windows/Signals.inc
index 14f3f21f02a1..0d4b8a26b023 100644
--- a/contrib/llvm/lib/Support/Windows/Signals.inc
+++ b/contrib/llvm/lib/Support/Windows/Signals.inc
@@ -23,14 +23,133 @@
 #endif
 #include <psapi.h>
 
-#ifdef __MINGW32__
+#ifdef _MSC_VER
+ #pragma comment(lib, "psapi.lib")
+ #pragma comment(lib, "dbghelp.lib")
+#elif __MINGW32__
  #if ((HAVE_LIBIMAGEHLP != 1) || (HAVE_LIBPSAPI != 1))
   #error "libimagehlp.a & libpsapi.a should be present"
  #endif
-#else
- #pragma comment(lib, "psapi.lib")
- #pragma comment(lib, "dbghelp.lib")
-#endif
+ // The version of g++ that comes with MinGW does *not* properly understand
+ // the ll format specifier for printf. However, MinGW passes the format
+ // specifiers on to the MSVCRT entirely, and the CRT understands the ll
+ // specifier. So these warnings are spurious in this case. Since we compile
+ // with -Wall, this will generate these warnings which should be ignored. So
+ // we will turn off the warnings for this just file. However, MinGW also does
+ // not support push and pop for diagnostics, so we have to manually turn it
+ // back on at the end of the file.
+ #pragma GCC diagnostic ignored "-Wformat"
+ #pragma GCC diagnostic ignored "-Wformat-extra-args"
+
+ #if !defined(__MINGW64_VERSION_MAJOR)
+ // MinGW.org does not have updated support for the 64-bit versions of the
+ // DebugHlp APIs. So we will have to load them manually. The structures and
+ // method signatures were pulled from DbgHelp.h in the Windows Platform SDK,
+ // and adjusted for brevity.
+ typedef struct _IMAGEHLP_LINE64 {
+   DWORD    SizeOfStruct;
+   PVOID    Key;
+   DWORD    LineNumber;
+   PCHAR    FileName;
+   DWORD64  Address;
+ } IMAGEHLP_LINE64, *PIMAGEHLP_LINE64;
+
+ typedef struct _IMAGEHLP_SYMBOL64 {
+   DWORD   SizeOfStruct;
+   DWORD64 Address;
+   DWORD   Size;
+   DWORD   Flags;
+   DWORD   MaxNameLength;
+   CHAR    Name[1];
+ } IMAGEHLP_SYMBOL64, *PIMAGEHLP_SYMBOL64;
+
+ typedef struct _tagADDRESS64 {
+   DWORD64       Offset;
+   WORD          Segment;
+   ADDRESS_MODE  Mode;
+ } ADDRESS64, *LPADDRESS64;
+
+ typedef struct _KDHELP64 {
+   DWORD64   Thread;
+   DWORD   ThCallbackStack;
+   DWORD   ThCallbackBStore;
+   DWORD   NextCallback;
+   DWORD   FramePointer;
+   DWORD64   KiCallUserMode;
+   DWORD64   KeUserCallbackDispatcher;
+   DWORD64   SystemRangeStart;
+   DWORD64   KiUserExceptionDispatcher;
+   DWORD64   StackBase;
+   DWORD64   StackLimit;
+   DWORD64   Reserved[5];
+ } KDHELP64, *PKDHELP64;
+
+ typedef struct _tagSTACKFRAME64 {
+   ADDRESS64   AddrPC;
+   ADDRESS64   AddrReturn;
+   ADDRESS64   AddrFrame;
+   ADDRESS64   AddrStack;
+   ADDRESS64   AddrBStore;
+   PVOID       FuncTableEntry;
+   DWORD64     Params[4];
+   BOOL        Far;
+   BOOL        Virtual;
+   DWORD64     Reserved[3];
+   KDHELP64    KdHelp;
+ } STACKFRAME64, *LPSTACKFRAME64;
+
+typedef BOOL (__stdcall *PREAD_PROCESS_MEMORY_ROUTINE64)(HANDLE hProcess,
+                      DWORD64 qwBaseAddress, PVOID lpBuffer, DWORD nSize,
+                      LPDWORD lpNumberOfBytesRead);
+
+typedef PVOID (__stdcall *PFUNCTION_TABLE_ACCESS_ROUTINE64)( HANDLE ahProcess,
+                      DWORD64 AddrBase);
+
+typedef DWORD64 (__stdcall *PGET_MODULE_BASE_ROUTINE64)(HANDLE hProcess,
+                      DWORD64 Address);
+
+typedef DWORD64 (__stdcall *PTRANSLATE_ADDRESS_ROUTINE64)(HANDLE hProcess,
+                      HANDLE hThread, LPADDRESS64 lpaddr);
+
+typedef BOOL (WINAPI *fpStackWalk64)(DWORD, HANDLE, HANDLE, LPSTACKFRAME64,
+                      PVOID, PREAD_PROCESS_MEMORY_ROUTINE64,
+                      PFUNCTION_TABLE_ACCESS_ROUTINE64,
+                      PGET_MODULE_BASE_ROUTINE64,
+                      PTRANSLATE_ADDRESS_ROUTINE64);
+static fpStackWalk64 StackWalk64;
+
+typedef DWORD64 (WINAPI *fpSymGetModuleBase64)(HANDLE, DWORD64);
+static fpSymGetModuleBase64 SymGetModuleBase64;
+
+typedef BOOL (WINAPI *fpSymGetSymFromAddr64)(HANDLE, DWORD64,
+                      PDWORD64, PIMAGEHLP_SYMBOL64);
+static fpSymGetSymFromAddr64 SymGetSymFromAddr64;
+
+typedef BOOL (WINAPI *fpSymGetLineFromAddr64)(HANDLE, DWORD64,
+                      PDWORD, PIMAGEHLP_LINE64);
+static fpSymGetLineFromAddr64 SymGetLineFromAddr64;
+
+typedef PVOID (WINAPI *fpSymFunctionTableAccess64)(HANDLE, DWORD64);
+static fpSymFunctionTableAccess64 SymFunctionTableAccess64;
+
+static bool load64BitDebugHelp(void) {
+  HMODULE hLib = ::LoadLibrary("Dbghelp.dll");
+  if (hLib) {
+    StackWalk64 = (fpStackWalk64)
+                      ::GetProcAddress(hLib, "StackWalk64");
+    SymGetModuleBase64 = (fpSymGetModuleBase64)
+                      ::GetProcAddress(hLib, "SymGetModuleBase64");
+    SymGetSymFromAddr64 = (fpSymGetSymFromAddr64)
+                      ::GetProcAddress(hLib, "SymGetSymFromAddr64");
+    SymGetLineFromAddr64 = (fpSymGetLineFromAddr64)
+                      ::GetProcAddress(hLib, "SymGetLineFromAddr64");
+    SymFunctionTableAccess64 = (fpSymFunctionTableAccess64)
+                     ::GetProcAddress(hLib, "SymFunctionTableAccess64");
+  }
+  return StackWalk64 != NULL;
+}
+ #endif // !defined(__MINGW64_VERSION_MAJOR)
+#endif // __MINGW32__
 
 // Forward declare.
 static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep);
@@ -90,6 +209,18 @@ static int CRTReportHook(int ReportType, char *Message, int *Return) {
 #endif
 
 static void RegisterHandler() {
+#if __MINGW32__ && !defined(__MINGW64_VERSION_MAJOR)
+  // On MinGW.org, we need to load up the symbols explicitly, because the
+  // Win32 framework they include does not have support for the 64-bit
+  // versions of the APIs we need.  If we cannot load up the APIs (which
+  // would be unexpected as they should exist on every version of Windows
+  // we support), we will bail out since there would be nothing to report.
+  if (!load64BitDebugHelp()) {
+    assert(false && "These APIs should always be available");
+    return;
+  }
+#endif
+
   if (RegisteredUnhandledExceptionFilter) {
     EnterCriticalSection(&CriticalSection);
     return;
@@ -213,20 +344,28 @@ void llvm::sys::RunInterruptHandlers() {
 static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep) {
   Cleanup();
 
-#ifdef _WIN64
-  // TODO: provide a x64 friendly version of the following
-#else
-
   // Initialize the STACKFRAME structure.
-  STACKFRAME StackFrame;
+  STACKFRAME64 StackFrame;
   memset(&StackFrame, 0, sizeof(StackFrame));
 
+  DWORD machineType;
+#if defined(_M_X64)
+  machineType = IMAGE_FILE_MACHINE_AMD64;
+  StackFrame.AddrPC.Offset = ep->ContextRecord->Rip;
+  StackFrame.AddrPC.Mode = AddrModeFlat;
+  StackFrame.AddrStack.Offset = ep->ContextRecord->Rsp;
+  StackFrame.AddrStack.Mode = AddrModeFlat;
+  StackFrame.AddrFrame.Offset = ep->ContextRecord->Rbp;
+  StackFrame.AddrFrame.Mode = AddrModeFlat;
+#elif defined(_M_IX86)
+  machineType = IMAGE_FILE_MACHINE_I386;
   StackFrame.AddrPC.Offset = ep->ContextRecord->Eip;
   StackFrame.AddrPC.Mode = AddrModeFlat;
   StackFrame.AddrStack.Offset = ep->ContextRecord->Esp;
   StackFrame.AddrStack.Mode = AddrModeFlat;
   StackFrame.AddrFrame.Offset = ep->ContextRecord->Ebp;
   StackFrame.AddrFrame.Mode = AddrModeFlat;
+#endif
 
   HANDLE hProcess = GetCurrentProcess();
   HANDLE hThread = GetCurrentThread();
@@ -236,9 +375,9 @@ static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep) {
   SymInitialize(hProcess, NULL, TRUE);
 
   while (true) {
-    if (!StackWalk(IMAGE_FILE_MACHINE_I386, hProcess, hThread, &StackFrame,
-                   ep->ContextRecord, NULL, SymFunctionTableAccess,
-                   SymGetModuleBase, NULL)) {
+    if (!StackWalk64(machineType, hProcess, hThread, &StackFrame,
+                   ep->ContextRecord, NULL, SymFunctionTableAccess64,
+                   SymGetModuleBase64, NULL)) {
       break;
     }
 
@@ -246,54 +385,66 @@ static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep) {
       break;
 
     // Print the PC in hexadecimal.
-    DWORD PC = StackFrame.AddrPC.Offset;
-    fprintf(stderr, "%08lX", PC);
+    DWORD64 PC = StackFrame.AddrPC.Offset;
+#if defined(_M_X64)
+    fprintf(stderr, "0x%016llX", PC);
+#elif defined(_M_IX86)
+    fprintf(stderr, "0x%08lX", static_cast<DWORD>(PC));
+#endif
 
     // Print the parameters.  Assume there are four.
+#if defined(_M_X64)
+    fprintf(stderr, " (0x%016llX 0x%016llX 0x%016llX 0x%016llX)",
+                StackFrame.Params[0],
+                StackFrame.Params[1],
+                StackFrame.Params[2],
+                StackFrame.Params[3]);
+#elif defined(_M_IX86)
     fprintf(stderr, " (0x%08lX 0x%08lX 0x%08lX 0x%08lX)",
-            StackFrame.Params[0],
-            StackFrame.Params[1], StackFrame.Params[2], StackFrame.Params[3]);
-
+                static_cast<DWORD>(StackFrame.Params[0]),
+                static_cast<DWORD>(StackFrame.Params[1]),
+                static_cast<DWORD>(StackFrame.Params[2]),
+                static_cast<DWORD>(StackFrame.Params[3]));
+#endif
     // Verify the PC belongs to a module in this process.
-    if (!SymGetModuleBase(hProcess, PC)) {
+    if (!SymGetModuleBase64(hProcess, PC)) {
       fputs(" <unknown module>\n", stderr);
       continue;
     }
 
     // Print the symbol name.
     char buffer[512];
-    IMAGEHLP_SYMBOL *symbol = reinterpret_cast<IMAGEHLP_SYMBOL *>(buffer);
-    memset(symbol, 0, sizeof(IMAGEHLP_SYMBOL));
-    symbol->SizeOfStruct = sizeof(IMAGEHLP_SYMBOL);
-    symbol->MaxNameLength = 512 - sizeof(IMAGEHLP_SYMBOL);
+    IMAGEHLP_SYMBOL64 *symbol = reinterpret_cast<IMAGEHLP_SYMBOL64 *>(buffer);
+    memset(symbol, 0, sizeof(IMAGEHLP_SYMBOL64));
+    symbol->SizeOfStruct = sizeof(IMAGEHLP_SYMBOL64);
+    symbol->MaxNameLength = 512 - sizeof(IMAGEHLP_SYMBOL64);
 
-    DWORD dwDisp;
-    if (!SymGetSymFromAddr(hProcess, PC, &dwDisp, symbol)) {
+    DWORD64 dwDisp;
+    if (!SymGetSymFromAddr64(hProcess, PC, &dwDisp, symbol)) {
       fputc('\n', stderr);
       continue;
     }
 
     buffer[511] = 0;
     if (dwDisp > 0)
-      fprintf(stderr, ", %s()+%04lu bytes(s)", symbol->Name, dwDisp);
+      fprintf(stderr, ", %s() + 0x%llX bytes(s)", symbol->Name, dwDisp);
     else
       fprintf(stderr, ", %s", symbol->Name);
 
     // Print the source file and line number information.
-    IMAGEHLP_LINE line;
+    IMAGEHLP_LINE64 line;
+    DWORD dwLineDisp;
     memset(&line, 0, sizeof(line));
     line.SizeOfStruct = sizeof(line);
-    if (SymGetLineFromAddr(hProcess, PC, &dwDisp, &line)) {
+    if (SymGetLineFromAddr64(hProcess, PC, &dwLineDisp, &line)) {
       fprintf(stderr, ", %s, line %lu", line.FileName, line.LineNumber);
-      if (dwDisp > 0)
-        fprintf(stderr, "+%04lu byte(s)", dwDisp);
+      if (dwLineDisp > 0)
+        fprintf(stderr, " + 0x%lX byte(s)", dwLineDisp);
     }
 
     fputc('\n', stderr);
   }
 
-#endif
-
   if (ExitOnUnhandledExceptions)
     _exit(-3);
 
@@ -326,3 +477,12 @@ static BOOL WINAPI LLVMConsoleCtrlHandler(DWORD dwCtrlType) {
   LeaveCriticalSection(&CriticalSection);
   return FALSE;
 }
+
+#if __MINGW32__
+ // We turned these warnings off for this file so that MinGW-g++ doesn't
+ // complain about the ll format specifiers used.  Now we are turning the
+ // warnings back on.  If MinGW starts to support diagnostic stacks, we can
+ // replace this with a pop.
+ #pragma GCC diagnostic warning "-Wformat"
+ #pragma GCC diagnostic warning "-Wformat-extra-args"
+#endif
diff --git a/contrib/llvm/lib/Support/Windows/Windows.h b/contrib/llvm/lib/Support/Windows/Windows.h
index 4a1553b599d7..67b6f015114f 100644
--- a/contrib/llvm/lib/Support/Windows/Windows.h
+++ b/contrib/llvm/lib/Support/Windows/Windows.h
@@ -19,9 +19,9 @@
 // mingw-w64 tends to define it as 0x0502 in its headers.
 #undef _WIN32_WINNT
 
-// Require at least Windows 2000 API.
-#define _WIN32_WINNT 0x0500
-#define _WIN32_IE    0x0500 // MinGW at it again.
+// Require at least Windows XP(5.1) API.
+#define _WIN32_WINNT 0x0501
+#define _WIN32_IE    0x0600 // MinGW at it again.
 #define WIN32_LEAN_AND_MEAN
 
 #include "llvm/Config/config.h" // Get build system configuration settings
diff --git a/contrib/llvm/lib/Support/raw_ostream.cpp b/contrib/llvm/lib/Support/raw_ostream.cpp
index 5a71fa3d8cea..4927e9a7b9d4 100644
--- a/contrib/llvm/lib/Support/raw_ostream.cpp
+++ b/contrib/llvm/lib/Support/raw_ostream.cpp
@@ -84,7 +84,7 @@ void raw_ostream::SetBuffered() {
 }
 
 void raw_ostream::SetBufferAndMode(char *BufferStart, size_t Size,
-                                    BufferKind Mode) {
+                                   BufferKind Mode) {
   assert(((Mode == Unbuffered && BufferStart == 0 && Size == 0) ||
           (Mode != Unbuffered && BufferStart && Size)) &&
          "stream must be unbuffered or have at least one byte");
@@ -121,7 +121,8 @@ raw_ostream &raw_ostream::operator<<(unsigned long N) {
 raw_ostream &raw_ostream::operator<<(long N) {
   if (N <  0) {
     *this << '-';
-    N = -N;
+    // Avoid undefined behavior on LONG_MIN with a cast.
+    N = -(unsigned long)N;
   }
 
   return this->operator<<(static_cast<unsigned long>(N));
@@ -284,7 +285,7 @@ raw_ostream &raw_ostream::write(unsigned char C) {
 
 raw_ostream &raw_ostream::write(const char *Ptr, size_t Size) {
   // Group exceptional cases into a single branch.
-  if (BUILTIN_EXPECT(OutBufCur+Size > OutBufEnd, false)) {
+  if (BUILTIN_EXPECT(size_t(OutBufEnd - OutBufCur) < Size, false)) {
     if (BUILTIN_EXPECT(!OutBufStart, false)) {
       if (BufferMode == Unbuffered) {
         write_impl(Ptr, Size);
diff --git a/contrib/llvm/utils/TableGen/Error.cpp b/contrib/llvm/lib/TableGen/Error.cpp
similarity index 96%
rename from contrib/llvm/utils/TableGen/Error.cpp
rename to contrib/llvm/lib/TableGen/Error.cpp
index 3f6cda897729..5b2cbbfec4b5 100644
--- a/contrib/llvm/utils/TableGen/Error.cpp
+++ b/contrib/llvm/lib/TableGen/Error.cpp
@@ -12,7 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Error.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/raw_ostream.h"
 
diff --git a/contrib/llvm/lib/TableGen/Main.cpp b/contrib/llvm/lib/TableGen/Main.cpp
new file mode 100644
index 000000000000..01bc55e9898b
--- /dev/null
+++ b/contrib/llvm/lib/TableGen/Main.cpp
@@ -0,0 +1,124 @@
+//===- Main.cpp - Top-Level TableGen implementation -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// TableGen is a tool which can be used to build up a description of something,
+// then invoke one or more "tablegen backends" to emit information about the
+// description in some predefined format.  In practice, this is used by the LLVM
+// code generators to automate generation of a code generator through a
+// high-level description of the target.
+//
+//===----------------------------------------------------------------------===//
+
+#include "TGParser.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/system_error.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenAction.h"
+#include <algorithm>
+#include <cstdio>
+using namespace llvm;
+
+namespace {
+  cl::opt<std::string>
+  OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"),
+                 cl::init("-"));
+
+  cl::opt<std::string>
+  DependFilename("d", cl::desc("Dependency filename"), cl::value_desc("filename"),
+                 cl::init(""));
+
+  cl::opt<std::string>
+  InputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
+
+  cl::list<std::string>
+  IncludeDirs("I", cl::desc("Directory of include files"),
+              cl::value_desc("directory"), cl::Prefix);
+}
+
+namespace llvm {
+
+int TableGenMain(char *argv0, TableGenAction &Action) {
+  RecordKeeper Records;
+
+  try {
+    // Parse the input file.
+    OwningPtr<MemoryBuffer> File;
+    if (error_code ec = MemoryBuffer::getFileOrSTDIN(InputFilename.c_str(), File)) {
+      errs() << "Could not open input file '" << InputFilename << "': "
+             << ec.message() <<"\n";
+      return 1;
+    }
+    MemoryBuffer *F = File.take();
+
+    // Tell SrcMgr about this buffer, which is what TGParser will pick up.
+    SrcMgr.AddNewSourceBuffer(F, SMLoc());
+
+    // Record the location of the include directory so that the lexer can find
+    // it later.
+    SrcMgr.setIncludeDirs(IncludeDirs);
+
+    TGParser Parser(SrcMgr, Records);
+
+    if (Parser.ParseFile())
+      return 1;
+
+    std::string Error;
+    tool_output_file Out(OutputFilename.c_str(), Error);
+    if (!Error.empty()) {
+      errs() << argv0 << ": error opening " << OutputFilename
+        << ":" << Error << "\n";
+      return 1;
+    }
+    if (!DependFilename.empty()) {
+      if (OutputFilename == "-") {
+        errs() << argv0 << ": the option -d must be used together with -o\n";
+        return 1;
+      }
+      tool_output_file DepOut(DependFilename.c_str(), Error);
+      if (!Error.empty()) {
+        errs() << argv0 << ": error opening " << DependFilename
+          << ":" << Error << "\n";
+        return 1;
+      }
+      DepOut.os() << OutputFilename << ":";
+      const std::vector<std::string> &Dependencies = Parser.getDependencies();
+      for (std::vector<std::string>::const_iterator I = Dependencies.begin(),
+                                                          E = Dependencies.end();
+           I != E; ++I) {
+        DepOut.os() << " " << (*I);
+      }
+      DepOut.os() << "\n";
+      DepOut.keep();
+    }
+
+    if (Action(Out.os(), Records))
+      return 1;
+
+    // Declare success.
+    Out.keep();
+    return 0;
+
+  } catch (const TGError &Error) {
+    PrintError(Error);
+  } catch (const std::string &Error) {
+    PrintError(Error);
+  } catch (const char *Error) {
+    PrintError(Error);
+  } catch (...) {
+    errs() << argv0 << ": Unknown unexpected exception occurred.\n";
+  }
+
+  return 1;
+}
+
+}
diff --git a/contrib/llvm/utils/TableGen/Record.cpp b/contrib/llvm/lib/TableGen/Record.cpp
similarity index 68%
rename from contrib/llvm/utils/TableGen/Record.cpp
rename to contrib/llvm/lib/TableGen/Record.cpp
index 730eca1b3ca5..b7c51cae953c 100644
--- a/contrib/llvm/utils/TableGen/Record.cpp
+++ b/contrib/llvm/lib/TableGen/Record.cpp
@@ -11,20 +11,84 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Record.h"
-#include "Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
 
 using namespace llvm;
 
+//===----------------------------------------------------------------------===//
+//    std::string wrapper for DenseMap purposes
+//===----------------------------------------------------------------------===//
+
+/// TableGenStringKey - This is a wrapper for std::string suitable for
+/// using as a key to a DenseMap.  Because there isn't a particularly
+/// good way to indicate tombstone or empty keys for strings, we want
+/// to wrap std::string to indicate that this is a "special" string
+/// not expected to take on certain values (those of the tombstone and
+/// empty keys).  This makes things a little safer as it clarifies
+/// that DenseMap is really not appropriate for general strings.
+
+class TableGenStringKey {
+public:
+  TableGenStringKey(const std::string &str) : data(str) {}
+  TableGenStringKey(const char *str) : data(str) {}
+
+  const std::string &str() const { return data; }
+  
+private:
+  std::string data;
+};
+
+/// Specialize DenseMapInfo for TableGenStringKey.
+namespace llvm {
+
+template<> struct DenseMapInfo<TableGenStringKey> {
+  static inline TableGenStringKey getEmptyKey() {
+    TableGenStringKey Empty("<<<EMPTY KEY>>>");
+    return Empty;
+  }
+  static inline TableGenStringKey getTombstoneKey() {
+    TableGenStringKey Tombstone("<<<TOMBSTONE KEY>>>");
+    return Tombstone;
+  }
+  static unsigned getHashValue(const TableGenStringKey& Val) {
+    return HashString(Val.str());
+  }
+  static bool isEqual(const TableGenStringKey& LHS,
+                      const TableGenStringKey& RHS) {
+    return LHS.str() == RHS.str();
+  }
+};
+
+}
+
 //===----------------------------------------------------------------------===//
 //    Type implementations
 //===----------------------------------------------------------------------===//
 
+BitRecTy BitRecTy::Shared;
+IntRecTy IntRecTy::Shared;
+StringRecTy StringRecTy::Shared;
+CodeRecTy CodeRecTy::Shared;
+DagRecTy DagRecTy::Shared;
+
 void RecTy::dump() const { print(errs()); }
 
+ListRecTy *RecTy::getListTy() {
+  if (!ListTy)
+    ListTy = new ListRecTy(this);
+  return ListTy;
+}
+
 Init *BitRecTy::convertValue(BitsInit *BI) {
   if (BI->getNumBits() != 1) return 0; // Only accept if just one bit!
   return BI->getBit(0);
@@ -38,7 +102,7 @@ Init *BitRecTy::convertValue(IntInit *II) {
   int64_t Val = II->getValue();
   if (Val != 0 && Val != 1) return 0;  // Only accept 0 or 1 for a bit!
 
-  return new BitInit(Val != 0);
+  return BitInit::get(Val != 0);
 }
 
 Init *BitRecTy::convertValue(TypedInit *VI) {
@@ -47,23 +111,32 @@ Init *BitRecTy::convertValue(TypedInit *VI) {
   return 0;
 }
 
+BitsRecTy *BitsRecTy::get(unsigned Sz) {
+  static std::vector<BitsRecTy*> Shared;
+  if (Sz >= Shared.size())
+    Shared.resize(Sz + 1);
+  BitsRecTy *&Ty = Shared[Sz];
+  if (!Ty)
+    Ty = new BitsRecTy(Sz);
+  return Ty;
+}
+
 std::string BitsRecTy::getAsString() const {
   return "bits<" + utostr(Size) + ">";
 }
 
 Init *BitsRecTy::convertValue(UnsetInit *UI) {
-  BitsInit *Ret = new BitsInit(Size);
+  SmallVector<Init *, 16> NewBits(Size);
 
   for (unsigned i = 0; i != Size; ++i)
-    Ret->setBit(i, new UnsetInit());
-  return Ret;
+    NewBits[i] = UnsetInit::get();
+
+  return BitsInit::get(NewBits);
 }
 
 Init *BitsRecTy::convertValue(BitInit *UI) {
   if (Size != 1) return 0;  // Can only convert single bit.
-  BitsInit *Ret = new BitsInit(1);
-  Ret->setBit(0, UI);
-  return Ret;
+          return BitsInit::get(UI);
 }
 
 /// canFitInBitfield - Return true if the number of bits is large enough to hold
@@ -83,11 +156,12 @@ Init *BitsRecTy::convertValue(IntInit *II) {
   if (!canFitInBitfield(Value, Size))
     return 0;
 
-  BitsInit *Ret = new BitsInit(Size);
-  for (unsigned i = 0; i != Size; ++i)
-    Ret->setBit(i, new BitInit(Value & (1LL << i)));
+  SmallVector<Init *, 16> NewBits(Size);
 
-  return Ret;
+  for (unsigned i = 0; i != Size; ++i)
+    NewBits[i] = BitInit::get(Value & (1LL << i));
+
+  return BitsInit::get(NewBits);
 }
 
 Init *BitsRecTy::convertValue(BitsInit *BI) {
@@ -100,17 +174,15 @@ Init *BitsRecTy::convertValue(BitsInit *BI) {
 Init *BitsRecTy::convertValue(TypedInit *VI) {
   if (BitsRecTy *BRT = dynamic_cast<BitsRecTy*>(VI->getType()))
     if (BRT->Size == Size) {
-      BitsInit *Ret = new BitsInit(Size);
+      SmallVector<Init *, 16> NewBits(Size);
+ 
       for (unsigned i = 0; i != Size; ++i)
-        Ret->setBit(i, new VarBitInit(VI, i));
-      return Ret;
+        NewBits[i] = VarBitInit::get(VI, i);
+      return BitsInit::get(NewBits);
     }
 
-  if (Size == 1 && dynamic_cast<BitRecTy*>(VI->getType())) {
-    BitsInit *Ret = new BitsInit(1);
-    Ret->setBit(0, VI);
-    return Ret;
-  }
+  if (Size == 1 && dynamic_cast<BitRecTy*>(VI->getType()))
+    return BitsInit::get(VI);
 
   if (TernOpInit *Tern = dynamic_cast<TernOpInit*>(VI)) {
     if (Tern->getOpcode() == TernOpInit::IF) {
@@ -126,30 +198,31 @@ Init *BitsRecTy::convertValue(TypedInit *VI) {
         int64_t RHSVal = RHSi->getValue();
 
         if (canFitInBitfield(MHSVal, Size) && canFitInBitfield(RHSVal, Size)) {
-          BitsInit *Ret = new BitsInit(Size);
+          SmallVector<Init *, 16> NewBits(Size);
 
           for (unsigned i = 0; i != Size; ++i)
-            Ret->setBit(i, new TernOpInit(TernOpInit::IF, LHS,
-                                          new IntInit((MHSVal & (1LL << i)) ? 1 : 0),
-                                          new IntInit((RHSVal & (1LL << i)) ? 1 : 0),
-                                          VI->getType()));
+            NewBits[i] =
+              TernOpInit::get(TernOpInit::IF, LHS,
+                              IntInit::get((MHSVal & (1LL << i)) ? 1 : 0),
+                              IntInit::get((RHSVal & (1LL << i)) ? 1 : 0),
+                              VI->getType());
 
-          return Ret;
+          return BitsInit::get(NewBits);
         }
       } else {
         BitsInit *MHSbs = dynamic_cast<BitsInit*>(MHS);
         BitsInit *RHSbs = dynamic_cast<BitsInit*>(RHS);
 
         if (MHSbs && RHSbs) {
-          BitsInit *Ret = new BitsInit(Size);
+          SmallVector<Init *, 16> NewBits(Size);
 
           for (unsigned i = 0; i != Size; ++i)
-            Ret->setBit(i, new TernOpInit(TernOpInit::IF, LHS,
-                                          MHSbs->getBit(i),
-                                          RHSbs->getBit(i),
-                                          VI->getType()));
+            NewBits[i] = TernOpInit::get(TernOpInit::IF, LHS,
+                                         MHSbs->getBit(i),
+                                         RHSbs->getBit(i),
+                                         VI->getType());
 
-          return Ret;
+          return BitsInit::get(NewBits);
         }
       }
     }
@@ -159,7 +232,7 @@ Init *BitsRecTy::convertValue(TypedInit *VI) {
 }
 
 Init *IntRecTy::convertValue(BitInit *BI) {
-  return new IntInit(BI->getValue());
+  return IntInit::get(BI->getValue());
 }
 
 Init *IntRecTy::convertValue(BitsInit *BI) {
@@ -170,7 +243,7 @@ Init *IntRecTy::convertValue(BitsInit *BI) {
     } else {
       return 0;
     }
-  return new IntInit(Result);
+  return IntInit::get(Result);
 }
 
 Init *IntRecTy::convertValue(TypedInit *TI) {
@@ -184,7 +257,7 @@ Init *StringRecTy::convertValue(UnOpInit *BO) {
     Init *L = BO->getOperand()->convertInitializerTo(this);
     if (L == 0) return 0;
     if (L != BO->getOperand())
-      return new UnOpInit(UnOpInit::CAST, L, new StringRecTy);
+      return UnOpInit::get(UnOpInit::CAST, L, new StringRecTy);
     return BO;
   }
 
@@ -197,7 +270,7 @@ Init *StringRecTy::convertValue(BinOpInit *BO) {
     Init *R = BO->getRHS()->convertInitializerTo(this);
     if (L == 0 || R == 0) return 0;
     if (L != BO->getLHS() || R != BO->getRHS())
-      return new BinOpInit(BinOpInit::STRCONCAT, L, R, new StringRecTy);
+      return BinOpInit::get(BinOpInit::STRCONCAT, L, R, new StringRecTy);
     return BO;
   }
 
@@ -231,7 +304,7 @@ Init *ListRecTy::convertValue(ListInit *LI) {
     return 0;
   }
 
-  return new ListInit(Elements, new ListRecTy(Ty));
+  return ListInit::get(Elements, this);
 }
 
 Init *ListRecTy::convertValue(TypedInit *TI) {
@@ -259,7 +332,7 @@ Init *DagRecTy::convertValue(UnOpInit *BO) {
     Init *L = BO->getOperand()->convertInitializerTo(this);
     if (L == 0) return 0;
     if (L != BO->getOperand())
-      return new UnOpInit(UnOpInit::CAST, L, new DagRecTy);
+      return UnOpInit::get(UnOpInit::CAST, L, new DagRecTy);
     return BO;
   }
   return 0;
@@ -271,12 +344,16 @@ Init *DagRecTy::convertValue(BinOpInit *BO) {
     Init *R = BO->getRHS()->convertInitializerTo(this);
     if (L == 0 || R == 0) return 0;
     if (L != BO->getLHS() || R != BO->getRHS())
-      return new BinOpInit(BinOpInit::CONCAT, L, R, new DagRecTy);
+      return BinOpInit::get(BinOpInit::CONCAT, L, R, new DagRecTy);
     return BO;
   }
   return 0;
 }
 
+RecordRecTy *RecordRecTy::get(Record *R) {
+  return &dynamic_cast<RecordRecTy&>(*R->getDefInit()->getType());
+}
+
 std::string RecordRecTy::getAsString() const {
   return Rec->getName();
 }
@@ -326,7 +403,7 @@ RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
               iend = T1SuperClasses.end();
             i != iend;
             ++i) {
-          RecordRecTy *SuperRecTy1 = new RecordRecTy(*i);
+          RecordRecTy *SuperRecTy1 = RecordRecTy::get(*i);
           RecTy *NewType1 = resolveTypes(SuperRecTy1, T2);
           if (NewType1 != 0) {
             if (NewType1 != SuperRecTy1) {
@@ -345,7 +422,7 @@ RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
               iend = T2SuperClasses.end();
             i != iend;
             ++i) {
-          RecordRecTy *SuperRecTy2 = new RecordRecTy(*i);
+          RecordRecTy *SuperRecTy2 = RecordRecTy::get(*i);
           RecTy *NewType2 = resolveTypes(T1, SuperRecTy2);
           if (NewType2 != 0) {
             if (NewType2 != SuperRecTy2) {
@@ -369,16 +446,60 @@ RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
 
 void Init::dump() const { return print(errs()); }
 
-Init *BitsInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) {
-  BitsInit *BI = new BitsInit(Bits.size());
+UnsetInit *UnsetInit::get() {
+  static UnsetInit TheInit;
+  return &TheInit;
+}
+
+BitInit *BitInit::get(bool V) {
+  static BitInit True(true);
+  static BitInit False(false);
+
+  return V ? &True : &False;
+}
+
+static void
+ProfileBitsInit(FoldingSetNodeID &ID, ArrayRef<Init *> Range) {
+  ID.AddInteger(Range.size());
+
+  for (ArrayRef<Init *>::iterator i = Range.begin(),
+         iend = Range.end();
+       i != iend;
+       ++i)
+    ID.AddPointer(*i);
+}
+
+BitsInit *BitsInit::get(ArrayRef<Init *> Range) {
+  typedef FoldingSet<BitsInit> Pool;
+  static Pool ThePool;  
+
+  FoldingSetNodeID ID;
+  ProfileBitsInit(ID, Range);
+
+  void *IP = 0;
+  if (BitsInit *I = ThePool.FindNodeOrInsertPos(ID, IP))
+    return I;
+
+  BitsInit *I = new BitsInit(Range);
+  ThePool.InsertNode(I, IP);
+
+  return I;
+}
+
+void BitsInit::Profile(FoldingSetNodeID &ID) const {
+  ProfileBitsInit(ID, Bits);
+}
+
+Init *
+BitsInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
+  SmallVector<Init *, 16> NewBits(Bits.size());
+
   for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
-    if (Bits[i] >= getNumBits()) {
-      delete BI;
+    if (Bits[i] >= getNumBits())
       return 0;
-    }
-    BI->setBit(i, getBit(Bits[i]));
+    NewBits[i] = getBit(Bits[i]);
   }
-  return BI;
+  return BitsInit::get(NewBits);
 }
 
 std::string BitsInit::getAsString() const {
@@ -396,9 +517,9 @@ std::string BitsInit::getAsString() const {
 // resolveReferences - If there are any field references that refer to fields
 // that have been filled in, we can propagate the values now.
 //
-Init *BitsInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *BitsInit::resolveReferences(Record &R, const RecordVal *RV) const {
   bool Changed = false;
-  BitsInit *New = new BitsInit(getNumBits());
+  SmallVector<Init *, 16> NewBits(getNumBits());
 
   for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
     Init *B;
@@ -409,40 +530,107 @@ Init *BitsInit::resolveReferences(Record &R, const RecordVal *RV) {
       CurBit = CurBit->resolveReferences(R, RV);
       Changed |= B != CurBit;
     } while (B != CurBit);
-    New->setBit(i, CurBit);
+    NewBits[i] = CurBit;
   }
 
   if (Changed)
-    return New;
-  delete New;
-  return this;
+    return BitsInit::get(NewBits);
+
+  return const_cast<BitsInit *>(this);
+}
+
+IntInit *IntInit::get(int64_t V) {
+  typedef DenseMap<int64_t, IntInit *> Pool;
+  static Pool ThePool;
+
+  IntInit *&I = ThePool[V];
+  if (!I) I = new IntInit(V);
+  return I;
 }
 
 std::string IntInit::getAsString() const {
   return itostr(Value);
 }
 
-Init *IntInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) {
-  BitsInit *BI = new BitsInit(Bits.size());
+Init *
+IntInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
+  SmallVector<Init *, 16> NewBits(Bits.size());
 
   for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
-    if (Bits[i] >= 64) {
-      delete BI;
+    if (Bits[i] >= 64)
       return 0;
-    }
-    BI->setBit(i, new BitInit(Value & (INT64_C(1) << Bits[i])));
+
+    NewBits[i] = BitInit::get(Value & (INT64_C(1) << Bits[i]));
   }
-  return BI;
+  return BitsInit::get(NewBits);
 }
 
-Init *ListInit::convertInitListSlice(const std::vector<unsigned> &Elements) {
+StringInit *StringInit::get(const std::string &V) {
+  typedef StringMap<StringInit *> Pool;
+  static Pool ThePool;
+
+  StringInit *&I = ThePool[V];
+  if (!I) I = new StringInit(V);
+  return I;
+}
+
+CodeInit *CodeInit::get(const std::string &V) {
+  typedef StringMap<CodeInit *> Pool;
+  static Pool ThePool;
+
+  CodeInit *&I = ThePool[V];
+  if (!I) I = new CodeInit(V);
+  return I;
+}
+
+static void ProfileListInit(FoldingSetNodeID &ID,
+                            ArrayRef<Init *> Range,
+                            RecTy *EltTy) {
+  ID.AddInteger(Range.size());
+  ID.AddPointer(EltTy);
+
+  for (ArrayRef<Init *>::iterator i = Range.begin(),
+         iend = Range.end();
+       i != iend;
+       ++i)
+    ID.AddPointer(*i);
+}
+
+ListInit *ListInit::get(ArrayRef<Init *> Range, RecTy *EltTy) {
+  typedef FoldingSet<ListInit> Pool;
+  static Pool ThePool;
+
+  // Just use the FoldingSetNodeID to compute a hash.  Use a DenseMap
+  // for actual storage.
+  FoldingSetNodeID ID;
+  ProfileListInit(ID, Range, EltTy);
+
+  void *IP = 0;
+  if (ListInit *I = ThePool.FindNodeOrInsertPos(ID, IP))
+    return I;
+
+  ListInit *I = new ListInit(Range, EltTy);
+  ThePool.InsertNode(I, IP);
+  return I;
+}
+
+void ListInit::Profile(FoldingSetNodeID &ID) const {
+  ListRecTy *ListType = dynamic_cast<ListRecTy *>(getType());
+  assert(ListType && "Bad type for ListInit!");
+  RecTy *EltTy = ListType->getElementType();
+
+  ProfileListInit(ID, Values, EltTy);
+}
+
+Init *
+ListInit::convertInitListSlice(const std::vector<unsigned> &Elements) const {
   std::vector<Init*> Vals;
   for (unsigned i = 0, e = Elements.size(); i != e; ++i) {
     if (Elements[i] >= getSize())
       return 0;
     Vals.push_back(getElement(Elements[i]));
   }
-  return new ListInit(Vals, getType());
+  return ListInit::get(Vals, getType());
 }
 
 Record *ListInit::getElementAsRecord(unsigned i) const {
@@ -452,7 +640,7 @@ Record *ListInit::getElementAsRecord(unsigned i) const {
   return DI->getDef();
 }
 
-Init *ListInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *ListInit::resolveReferences(Record &R, const RecordVal *RV) const {
   std::vector<Init*> Resolved;
   Resolved.reserve(getSize());
   bool Changed = false;
@@ -470,12 +658,12 @@ Init *ListInit::resolveReferences(Record &R, const RecordVal *RV) {
   }
 
   if (Changed)
-    return new ListInit(Resolved, getType());
-  return this;
+    return ListInit::get(Resolved, getType());
+  return const_cast<ListInit *>(this);
 }
 
 Init *ListInit::resolveListElementReference(Record &R, const RecordVal *IRV,
-                                            unsigned Elt) {
+                                            unsigned Elt) const {
   if (Elt >= getSize())
     return 0;  // Out of range reference.
   Init *E = getElement(Elt);
@@ -497,7 +685,7 @@ std::string ListInit::getAsString() const {
 }
 
 Init *OpInit::resolveBitReference(Record &R, const RecordVal *IRV,
-                                  unsigned Bit) {
+                                  unsigned Bit) const {
   Init *Folded = Fold(&R, 0);
 
   if (Folded != this) {
@@ -511,20 +699,41 @@ Init *OpInit::resolveBitReference(Record &R, const RecordVal *IRV,
 }
 
 Init *OpInit::resolveListElementReference(Record &R, const RecordVal *IRV,
-                                          unsigned Elt) {
-  Init *Folded = Fold(&R, 0);
+                                          unsigned Elt) const {
+  Init *Resolved = resolveReferences(R, IRV);
+  OpInit *OResolved = dynamic_cast<OpInit *>(Resolved);
+  if (OResolved) {
+    Resolved = OResolved->Fold(&R, 0);
+  }
 
-  if (Folded != this) {
-    TypedInit *Typed = dynamic_cast<TypedInit *>(Folded);
+  if (Resolved != this) {
+    TypedInit *Typed = dynamic_cast<TypedInit *>(Resolved); 
+    assert(Typed && "Expected typed init for list reference");
     if (Typed) {
-      return Typed->resolveListElementReference(R, IRV, Elt);
+      Init *New = Typed->resolveListElementReference(R, IRV, Elt);
+      if (New)
+        return New;
+      return VarListElementInit::get(Typed, Elt);
     }
   }
 
   return 0;
 }
 
-Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
+UnOpInit *UnOpInit::get(UnaryOp opc, Init *lhs, RecTy *Type) {
+  typedef std::pair<std::pair<unsigned, Init *>, RecTy *> Key;
+
+  typedef DenseMap<Key, UnOpInit *> Pool;
+  static Pool ThePool;  
+
+  Key TheKey(std::make_pair(std::make_pair(opc, lhs), Type));
+
+  UnOpInit *&I = ThePool[TheKey];
+  if (!I) I = new UnOpInit(opc, lhs, Type);
+  return I;
+}
+
+Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   default: assert(0 && "Unknown unop");
   case CAST: {
@@ -536,7 +745,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
 
       DefInit *LHSd = dynamic_cast<DefInit*>(LHS);
       if (LHSd) {
-        return new StringInit(LHSd->getDef()->getName());
+        return StringInit::get(LHSd->getDef()->getName());
       }
     } else {
       StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
@@ -548,7 +757,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
           if (const RecordVal *RV = CurRec->getValue(Name)) {
             if (RV->getType() != getType())
               throw "type mismatch in cast";
-            return new VarInit(Name, RV->getType());
+            return VarInit::get(Name, RV->getType());
           }
 
           std::string TemplateArgName = CurRec->getName()+":"+Name;
@@ -559,7 +768,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
             if (RV->getType() != getType())
               throw "type mismatch in cast";
 
-            return new VarInit(TemplateArgName, RV->getType());
+            return VarInit::get(TemplateArgName, RV->getType());
           }
         }
 
@@ -572,12 +781,12 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
             if (RV->getType() != getType())
               throw "type mismatch in cast";
 
-            return new VarInit(MCName, RV->getType());
+            return VarInit::get(MCName, RV->getType());
           }
         }
 
         if (Record *D = (CurRec->getRecords()).getDef(Name))
-          return new DefInit(D);
+          return DefInit::get(D);
 
         throw TGError(CurRec->getLoc(), "Undefined reference:'" + Name + "'\n");
       }
@@ -602,8 +811,13 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
         assert(0 && "Empty list in cdr");
         return 0;
       }
-      ListInit *Result = new ListInit(LHSl->begin()+1, LHSl->end(),
-                                      LHSl->getType());
+      // Note the +1.  We can't just pass the result of getValues()
+      // directly.
+      ArrayRef<Init *>::iterator begin = LHSl->getValues().begin()+1;
+      ArrayRef<Init *>::iterator end   = LHSl->getValues().end();
+      ListInit *Result =
+        ListInit::get(ArrayRef<Init *>(begin, end - begin),
+                      LHSl->getType());
       return Result;
     }
     break;
@@ -612,31 +826,31 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
     ListInit *LHSl = dynamic_cast<ListInit*>(LHS);
     if (LHSl) {
       if (LHSl->getSize() == 0) {
-        return new IntInit(1);
+        return IntInit::get(1);
       } else {
-        return new IntInit(0);
+        return IntInit::get(0);
       }
     }
     StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
     if (LHSs) {
       if (LHSs->getValue().empty()) {
-        return new IntInit(1);
+        return IntInit::get(1);
       } else {
-        return new IntInit(0);
+        return IntInit::get(0);
       }
     }
 
     break;
   }
   }
-  return this;
+  return const_cast<UnOpInit *>(this);
 }
 
-Init *UnOpInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *UnOpInit::resolveReferences(Record &R, const RecordVal *RV) const {
   Init *lhs = LHS->resolveReferences(R, RV);
 
   if (LHS != lhs)
-    return (new UnOpInit(getOpcode(), lhs, getType()))->Fold(&R, 0);
+    return (UnOpInit::get(getOpcode(), lhs, getType()))->Fold(&R, 0);
   return Fold(&R, 0);
 }
 
@@ -651,7 +865,25 @@ std::string UnOpInit::getAsString() const {
   return Result + "(" + LHS->getAsString() + ")";
 }
 
-Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
+BinOpInit *BinOpInit::get(BinaryOp opc, Init *lhs,
+                          Init *rhs, RecTy *Type) {
+  typedef std::pair<
+    std::pair<std::pair<unsigned, Init *>, Init *>,
+    RecTy *
+    > Key;
+
+  typedef DenseMap<Key, BinOpInit *> Pool;
+  static Pool ThePool;  
+
+  Key TheKey(std::make_pair(std::make_pair(std::make_pair(opc, lhs), rhs),
+                            Type));
+
+  BinOpInit *&I = ThePool[TheKey];
+  if (!I) I = new BinOpInit(opc, lhs, rhs, Type);
+  return I;
+}
+
+Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   default: assert(0 && "Unknown binop");
   case CONCAT: {
@@ -672,7 +904,7 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
         Args.push_back(RHSs->getArg(i));
         ArgNames.push_back(RHSs->getArgName(i));
       }
-      return new DagInit(LHSs->getOperator(), "", Args, ArgNames);
+      return DagInit::get(LHSs->getOperator(), "", Args, ArgNames);
     }
     break;
   }
@@ -680,26 +912,26 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
     StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
     StringInit *RHSs = dynamic_cast<StringInit*>(RHS);
     if (LHSs && RHSs)
-      return new StringInit(LHSs->getValue() + RHSs->getValue());
+      return StringInit::get(LHSs->getValue() + RHSs->getValue());
     break;
   }
   case EQ: {
     // try to fold eq comparison for 'bit' and 'int', otherwise fallback
     // to string objects.
     IntInit* L =
-      dynamic_cast<IntInit*>(LHS->convertInitializerTo(new IntRecTy()));
+      dynamic_cast<IntInit*>(LHS->convertInitializerTo(IntRecTy::get()));
     IntInit* R =
-      dynamic_cast<IntInit*>(RHS->convertInitializerTo(new IntRecTy()));
+      dynamic_cast<IntInit*>(RHS->convertInitializerTo(IntRecTy::get()));
 
     if (L && R)
-      return new IntInit(L->getValue() == R->getValue());
+      return IntInit::get(L->getValue() == R->getValue());
 
     StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
     StringInit *RHSs = dynamic_cast<StringInit*>(RHS);
 
     // Make sure we've resolved
     if (LHSs && RHSs)
-      return new IntInit(LHSs->getValue() == RHSs->getValue());
+      return IntInit::get(LHSs->getValue() == RHSs->getValue());
 
     break;
   }
@@ -717,20 +949,20 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
       case SRA: Result = LHSv >> RHSv; break;
       case SRL: Result = (uint64_t)LHSv >> (uint64_t)RHSv; break;
       }
-      return new IntInit(Result);
+      return IntInit::get(Result);
     }
     break;
   }
   }
-  return this;
+  return const_cast<BinOpInit *>(this);
 }
 
-Init *BinOpInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *BinOpInit::resolveReferences(Record &R, const RecordVal *RV) const {
   Init *lhs = LHS->resolveReferences(R, RV);
   Init *rhs = RHS->resolveReferences(R, RV);
 
   if (LHS != lhs || RHS != rhs)
-    return (new BinOpInit(getOpcode(), lhs, rhs, getType()))->Fold(&R, 0);
+    return (BinOpInit::get(getOpcode(), lhs, rhs, getType()))->Fold(&R, 0);
   return Fold(&R, 0);
 }
 
@@ -747,6 +979,31 @@ std::string BinOpInit::getAsString() const {
   return Result + "(" + LHS->getAsString() + ", " + RHS->getAsString() + ")";
 }
 
+TernOpInit *TernOpInit::get(TernaryOp opc, Init *lhs,
+                                  Init *mhs, Init *rhs,
+                                  RecTy *Type) {
+  typedef std::pair<
+    std::pair<
+      std::pair<std::pair<unsigned, RecTy *>, Init *>,
+      Init *
+      >,
+    Init *
+    > Key;
+
+  typedef DenseMap<Key, TernOpInit *> Pool;
+  static Pool ThePool;
+
+  Key TheKey(std::make_pair(std::make_pair(std::make_pair(std::make_pair(opc,
+                                                                         Type),
+                                                          lhs),
+                                           mhs),
+                            rhs));
+
+  TernOpInit *&I = ThePool[TheKey];
+  if (!I) I = new TernOpInit(opc, lhs, mhs, rhs, Type);
+  return I;
+}
+
 static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
                            Record *CurRec, MultiClass *CurMultiClass);
 
@@ -787,12 +1044,11 @@ static Init *EvaluateOperation(OpInit *RHSo, Init *LHS, Init *Arg,
   }
 
   // Now run the operator and use its result as the new leaf
-  OpInit *NewOp = RHSo->clone(NewOperands);
+  const OpInit *NewOp = RHSo->clone(NewOperands);
   Init *NewVal = NewOp->Fold(CurRec, CurMultiClass);
-  if (NewVal != NewOp) {
-    delete NewOp;
+  if (NewVal != NewOp)
     return NewVal;
-  }
+
   return 0;
 }
 
@@ -843,13 +1099,13 @@ static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
         args.push_back(std::make_pair(Arg, ArgName));
       }
 
-      return new DagInit(Val, "", args);
+      return DagInit::get(Val, "", args);
     }
     if (MHSl) {
       std::vector<Init *> NewOperands;
       std::vector<Init *> NewList(MHSl->begin(), MHSl->end());
 
-      for (ListInit::iterator li = NewList.begin(),
+      for (std::vector<Init *>::iterator li = NewList.begin(),
              liend = NewList.end();
            li != liend;
            ++li) {
@@ -865,20 +1121,18 @@ static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
         }
 
         // Now run the operator and use its result as the new list item
-        OpInit *NewOp = RHSo->clone(NewOperands);
+        const OpInit *NewOp = RHSo->clone(NewOperands);
         Init *NewItem = NewOp->Fold(CurRec, CurMultiClass);
-        if (NewItem != NewOp) {
+        if (NewItem != NewOp)
           *li = NewItem;
-          delete NewOp;
-        }
       }
-      return new ListInit(NewList, MHSl->getType());
+      return ListInit::get(NewList, MHSl->getType());
     }
   }
   return 0;
 }
 
-Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
+Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   default: assert(0 && "Unknown binop");
   case SUBST: {
@@ -902,14 +1156,14 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
         if (LHSd->getAsString() == RHSd->getAsString()) {
           Val = MHSd->getDef();
         }
-        return new DefInit(Val);
+        return DefInit::get(Val);
       }
       if (RHSv) {
         std::string Val = RHSv->getName();
         if (LHSv->getAsString() == RHSv->getAsString()) {
           Val = MHSv->getName();
         }
-        return new VarInit(Val, getType());
+        return VarInit::get(Val, getType());
       }
       if (RHSs) {
         std::string Val = RHSs->getValue();
@@ -924,7 +1178,7 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
           idx = found +  MHSs->getValue().size();
         } while (found != std::string::npos);
 
-        return new StringInit(Val);
+        return StringInit::get(Val);
       }
     }
     break;
@@ -941,7 +1195,7 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
 
   case IF: {
     IntInit *LHSi = dynamic_cast<IntInit*>(LHS);
-    if (Init *I = LHS->convertInitializerTo(new IntRecTy()))
+    if (Init *I = LHS->convertInitializerTo(IntRecTy::get()))
       LHSi = dynamic_cast<IntInit*>(I);
     if (LHSi) {
       if (LHSi->getValue()) {
@@ -954,26 +1208,27 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) {
   }
   }
 
-  return this;
+  return const_cast<TernOpInit *>(this);
 }
 
-Init *TernOpInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *TernOpInit::resolveReferences(Record &R,
+                                    const RecordVal *RV) const {
   Init *lhs = LHS->resolveReferences(R, RV);
 
   if (Opc == IF && lhs != LHS) {
     IntInit *Value = dynamic_cast<IntInit*>(lhs);
-    if (Init *I = lhs->convertInitializerTo(new IntRecTy()))
+    if (Init *I = lhs->convertInitializerTo(IntRecTy::get()))
       Value = dynamic_cast<IntInit*>(I);
     if (Value != 0) {
       // Short-circuit
       if (Value->getValue()) {
         Init *mhs = MHS->resolveReferences(R, RV);
-        return (new TernOpInit(getOpcode(), lhs, mhs,
-                               RHS, getType()))->Fold(&R, 0);
+        return (TernOpInit::get(getOpcode(), lhs, mhs,
+                                RHS, getType()))->Fold(&R, 0);
       } else {
         Init *rhs = RHS->resolveReferences(R, RV);
-        return (new TernOpInit(getOpcode(), lhs, MHS,
-                               rhs, getType()))->Fold(&R, 0);
+        return (TernOpInit::get(getOpcode(), lhs, MHS,
+                                rhs, getType()))->Fold(&R, 0);
       }
     }
   }
@@ -982,7 +1237,8 @@ Init *TernOpInit::resolveReferences(Record &R, const RecordVal *RV) {
   Init *rhs = RHS->resolveReferences(R, RV);
 
   if (LHS != lhs || MHS != mhs || RHS != rhs)
-    return (new TernOpInit(getOpcode(), lhs, mhs, rhs, getType()))->Fold(&R, 0);
+    return (TernOpInit::get(getOpcode(), lhs, mhs, rhs,
+                            getType()))->Fold(&R, 0);
   return Fold(&R, 0);
 }
 
@@ -1008,39 +1264,53 @@ RecTy *TypedInit::getFieldType(const std::string &FieldName) const {
   return 0;
 }
 
-Init *TypedInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) {
+Init *
+TypedInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
   BitsRecTy *T = dynamic_cast<BitsRecTy*>(getType());
   if (T == 0) return 0;  // Cannot subscript a non-bits variable.
   unsigned NumBits = T->getNumBits();
 
-  BitsInit *BI = new BitsInit(Bits.size());
+  SmallVector<Init *, 16> NewBits(Bits.size());
   for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
-    if (Bits[i] >= NumBits) {
-      delete BI;
+    if (Bits[i] >= NumBits)
       return 0;
-    }
-    BI->setBit(i, new VarBitInit(this, Bits[i]));
+
+    NewBits[i] = VarBitInit::get(const_cast<TypedInit *>(this), Bits[i]);
   }
-  return BI;
+  return BitsInit::get(NewBits);
 }
 
-Init *TypedInit::convertInitListSlice(const std::vector<unsigned> &Elements) {
+Init *
+TypedInit::convertInitListSlice(const std::vector<unsigned> &Elements) const {
   ListRecTy *T = dynamic_cast<ListRecTy*>(getType());
   if (T == 0) return 0;  // Cannot subscript a non-list variable.
 
   if (Elements.size() == 1)
-    return new VarListElementInit(this, Elements[0]);
+    return VarListElementInit::get(const_cast<TypedInit *>(this), Elements[0]);
 
   std::vector<Init*> ListInits;
   ListInits.reserve(Elements.size());
   for (unsigned i = 0, e = Elements.size(); i != e; ++i)
-    ListInits.push_back(new VarListElementInit(this, Elements[i]));
-  return new ListInit(ListInits, T);
+    ListInits.push_back(VarListElementInit::get(const_cast<TypedInit *>(this),
+                                                Elements[i]));
+  return ListInit::get(ListInits, T);
 }
 
 
+VarInit *VarInit::get(const std::string &VN, RecTy *T) {
+  typedef std::pair<RecTy *, TableGenStringKey> Key;
+  typedef DenseMap<Key, VarInit *> Pool;
+  static Pool ThePool;
+
+  Key TheKey(std::make_pair(T, VN));
+
+  VarInit *&I = ThePool[TheKey];
+  if (!I) I = new VarInit(VN, T);
+  return I;
+}
+
 Init *VarInit::resolveBitReference(Record &R, const RecordVal *IRV,
-                                   unsigned Bit) {
+                                   unsigned Bit) const {
   if (R.isTemplateArg(getName())) return 0;
   if (IRV && IRV->getName() != getName()) return 0;
 
@@ -1060,8 +1330,9 @@ Init *VarInit::resolveBitReference(Record &R, const RecordVal *IRV,
   return 0;
 }
 
-Init *VarInit::resolveListElementReference(Record &R, const RecordVal *IRV,
-                                           unsigned Elt) {
+Init *VarInit::resolveListElementReference(Record &R,
+                                           const RecordVal *IRV,
+                                           unsigned Elt) const {
   if (R.isTemplateArg(getName())) return 0;
   if (IRV && IRV->getName() != getName()) return 0;
 
@@ -1069,9 +1340,9 @@ Init *VarInit::resolveListElementReference(Record &R, const RecordVal *IRV,
   assert(RV && "Reference to a non-existent variable?");
   ListInit *LI = dynamic_cast<ListInit*>(RV->getValue());
   if (!LI) {
-    VarInit *VI = dynamic_cast<VarInit*>(RV->getValue());
+    TypedInit *VI = dynamic_cast<TypedInit*>(RV->getValue());
     assert(VI && "Invalid list element!");
-    return new VarListElementInit(VI, Elt);
+    return VarListElementInit::get(VI, Elt);
   }
 
   if (Elt >= LI->getSize())
@@ -1114,48 +1385,91 @@ Init *VarInit::getFieldInit(Record &R, const RecordVal *RV,
 /// If a value is set for the variable later, this method will be called on
 /// users of the value to allow the value to propagate out.
 ///
-Init *VarInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *VarInit::resolveReferences(Record &R, const RecordVal *RV) const {
   if (RecordVal *Val = R.getValue(VarName))
     if (RV == Val || (RV == 0 && !dynamic_cast<UnsetInit*>(Val->getValue())))
       return Val->getValue();
-  return this;
+  return const_cast<VarInit *>(this);
+}
+
+VarBitInit *VarBitInit::get(TypedInit *T, unsigned B) {
+  typedef std::pair<TypedInit *, unsigned> Key;
+  typedef DenseMap<Key, VarBitInit *> Pool;
+
+  static Pool ThePool;
+
+  Key TheKey(std::make_pair(T, B));
+
+  VarBitInit *&I = ThePool[TheKey];
+  if (!I) I = new VarBitInit(T, B);
+  return I;
 }
 
 std::string VarBitInit::getAsString() const {
    return TI->getAsString() + "{" + utostr(Bit) + "}";
 }
 
-Init *VarBitInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *VarBitInit::resolveReferences(Record &R, const RecordVal *RV) const {
   if (Init *I = getVariable()->resolveBitReference(R, RV, getBitNum()))
     return I;
-  return this;
+  return const_cast<VarBitInit *>(this);
+}
+
+VarListElementInit *VarListElementInit::get(TypedInit *T,
+                                            unsigned E) {
+  typedef std::pair<TypedInit *, unsigned> Key;
+  typedef DenseMap<Key, VarListElementInit *> Pool;
+
+  static Pool ThePool;
+
+  Key TheKey(std::make_pair(T, E));
+
+  VarListElementInit *&I = ThePool[TheKey];
+  if (!I) I = new VarListElementInit(T, E);
+  return I;
 }
 
 std::string VarListElementInit::getAsString() const {
   return TI->getAsString() + "[" + utostr(Element) + "]";
 }
 
-Init *VarListElementInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *
+VarListElementInit::resolveReferences(Record &R, const RecordVal *RV) const {
   if (Init *I = getVariable()->resolveListElementReference(R, RV,
                                                            getElementNum()))
     return I;
-  return this;
+  return const_cast<VarListElementInit *>(this);
 }
 
 Init *VarListElementInit::resolveBitReference(Record &R, const RecordVal *RV,
-                                              unsigned Bit) {
+                                              unsigned Bit) const {
   // FIXME: This should be implemented, to support references like:
   // bit B = AA[0]{1};
   return 0;
 }
 
-Init *VarListElementInit::
-resolveListElementReference(Record &R, const RecordVal *RV, unsigned Elt) {
-  // FIXME: This should be implemented, to support references like:
-  // int B = AA[0][1];
+Init *VarListElementInit:: resolveListElementReference(Record &R,
+                                                       const RecordVal *RV,
+                                                       unsigned Elt) const {
+  Init *Result = TI->resolveListElementReference(R, RV, Element);
+  
+  if (Result) {
+    TypedInit *TInit = dynamic_cast<TypedInit *>(Result);
+    if (TInit) {
+      Init *Result2 = TInit->resolveListElementReference(R, RV, Elt);
+      if (Result2) return Result2;
+      return new VarListElementInit(TInit, Elt);
+    }
+    return Result;
+  }
+ 
   return 0;
 }
 
+DefInit *DefInit::get(Record *R) {
+  return R->getDefInit();
+}
+
 RecTy *DefInit::getFieldType(const std::string &FieldName) const {
   if (const RecordVal *RV = Def->getValue(FieldName))
     return RV->getType();
@@ -1172,8 +1486,20 @@ std::string DefInit::getAsString() const {
   return Def->getName();
 }
 
+FieldInit *FieldInit::get(Init *R, const std::string &FN) {
+  typedef std::pair<Init *, TableGenStringKey> Key;
+  typedef DenseMap<Key, FieldInit *> Pool;
+  static Pool ThePool;  
+
+  Key TheKey(std::make_pair(R, FN));
+
+  FieldInit *&I = ThePool[TheKey];
+  if (!I) I = new FieldInit(R, FN);
+  return I;
+}
+
 Init *FieldInit::resolveBitReference(Record &R, const RecordVal *RV,
-                                     unsigned Bit) {
+                                     unsigned Bit) const {
   if (Init *BitsVal = Rec->getFieldInit(R, RV, FieldName))
     if (BitsInit *BI = dynamic_cast<BitsInit*>(BitsVal)) {
       assert(Bit < BI->getNumBits() && "Bit reference out of range!");
@@ -1186,7 +1512,7 @@ Init *FieldInit::resolveBitReference(Record &R, const RecordVal *RV,
 }
 
 Init *FieldInit::resolveListElementReference(Record &R, const RecordVal *RV,
-                                             unsigned Elt) {
+                                             unsigned Elt) const {
   if (Init *ListVal = Rec->getFieldInit(R, RV, FieldName))
     if (ListInit *LI = dynamic_cast<ListInit*>(ListVal)) {
       if (Elt >= LI->getSize()) return 0;
@@ -1201,22 +1527,83 @@ Init *FieldInit::resolveListElementReference(Record &R, const RecordVal *RV,
   return 0;
 }
 
-Init *FieldInit::resolveReferences(Record &R, const RecordVal *RV) {
+Init *FieldInit::resolveReferences(Record &R, const RecordVal *RV) const {
   Init *NewRec = RV ? Rec->resolveReferences(R, RV) : Rec;
 
   Init *BitsVal = NewRec->getFieldInit(R, RV, FieldName);
   if (BitsVal) {
     Init *BVR = BitsVal->resolveReferences(R, RV);
-    return BVR->isComplete() ? BVR : this;
+    return BVR->isComplete() ? BVR : const_cast<FieldInit *>(this);
   }
 
   if (NewRec != Rec) {
-    return new FieldInit(NewRec, FieldName);
+    return FieldInit::get(NewRec, FieldName);
   }
-  return this;
+  return const_cast<FieldInit *>(this);
 }
 
-Init *DagInit::resolveReferences(Record &R, const RecordVal *RV) {
+void ProfileDagInit(FoldingSetNodeID &ID,
+                    Init *V,
+                    const std::string &VN,
+                    ArrayRef<Init *> ArgRange,
+                    ArrayRef<std::string> NameRange) {
+  ID.AddPointer(V);
+  ID.AddString(VN);
+
+  ArrayRef<Init *>::iterator Arg  = ArgRange.begin();
+  ArrayRef<std::string>::iterator  Name = NameRange.begin();
+  while (Arg != ArgRange.end()) {
+    assert(Name != NameRange.end() && "Arg name underflow!");
+    ID.AddPointer(*Arg++);
+    ID.AddString(*Name++);
+  }
+  assert(Name == NameRange.end() && "Arg name overflow!");
+}
+
+DagInit *
+DagInit::get(Init *V, const std::string &VN,
+             ArrayRef<Init *> ArgRange,
+             ArrayRef<std::string> NameRange) {
+  typedef FoldingSet<DagInit> Pool;
+  static Pool ThePool;  
+
+  FoldingSetNodeID ID;
+  ProfileDagInit(ID, V, VN, ArgRange, NameRange);
+
+  void *IP = 0;
+  if (DagInit *I = ThePool.FindNodeOrInsertPos(ID, IP))
+    return I;
+
+  DagInit *I = new DagInit(V, VN, ArgRange, NameRange);
+  ThePool.InsertNode(I, IP);
+
+  return I;
+}
+
+DagInit *
+DagInit::get(Init *V, const std::string &VN,
+             const std::vector<std::pair<Init*, std::string> > &args) {
+  typedef std::pair<Init*, std::string> PairType;
+
+  std::vector<Init *> Args;
+  std::vector<std::string> Names;
+
+  for (std::vector<PairType>::const_iterator i = args.begin(),
+         iend = args.end();
+       i != iend;
+       ++i) {
+    Args.push_back(i->first);
+    Names.push_back(i->second);
+  }
+
+  return DagInit::get(V, VN, Args, Names);
+}
+
+void DagInit::Profile(FoldingSetNodeID &ID) const {
+  ProfileDagInit(ID, Val, ValName, Args, ArgNames);
+}
+
+Init *DagInit::resolveReferences(Record &R, const RecordVal *RV) const {
   std::vector<Init*> NewArgs;
   for (unsigned i = 0, e = Args.size(); i != e; ++i)
     NewArgs.push_back(Args[i]->resolveReferences(R, RV));
@@ -1224,9 +1611,9 @@ Init *DagInit::resolveReferences(Record &R, const RecordVal *RV) {
   Init *Op = Val->resolveReferences(R, RV);
 
   if (Args != NewArgs || Op != Val)
-    return new DagInit(Op, ValName, NewArgs, ArgNames);
+    return DagInit::get(Op, ValName, NewArgs, ArgNames);
 
-  return this;
+  return const_cast<DagInit *>(this);
 }
 
 
@@ -1250,12 +1637,24 @@ std::string DagInit::getAsString() const {
 //    Other implementations
 //===----------------------------------------------------------------------===//
 
-RecordVal::RecordVal(const std::string &N, RecTy *T, unsigned P)
+RecordVal::RecordVal(Init *N, RecTy *T, unsigned P)
   : Name(N), Ty(T), Prefix(P) {
-  Value = Ty->convertValue(new UnsetInit());
+  Value = Ty->convertValue(UnsetInit::get());
   assert(Value && "Cannot create unset value for current type!");
 }
 
+RecordVal::RecordVal(const std::string &N, RecTy *T, unsigned P)
+  : Name(StringInit::get(N)), Ty(T), Prefix(P) {
+  Value = Ty->convertValue(UnsetInit::get());
+  assert(Value && "Cannot create unset value for current type!");
+}
+
+const std::string &RecordVal::getName() const {
+  StringInit *NameString = dynamic_cast<StringInit *>(Name);
+  assert(NameString && "RecordVal name is not a string!");
+  return NameString->getValue();
+}
+
 void RecordVal::dump() const { errs() << *this; }
 
 void RecordVal::print(raw_ostream &OS, bool PrintSem) const {
@@ -1270,16 +1669,61 @@ void RecordVal::print(raw_ostream &OS, bool PrintSem) const {
 
 unsigned Record::LastID = 0;
 
-void Record::setName(const std::string &Name) {
-  if (TrackedRecords.getDef(getName()) == this) {
-    TrackedRecords.removeDef(getName());
-    this->Name = Name;
+void Record::checkName() {
+  // Ensure the record name has string type.
+  const TypedInit *TypedName = dynamic_cast<const TypedInit *>(Name);
+  assert(TypedName && "Record name is not typed!");
+  RecTy *Type = TypedName->getType();
+  if (dynamic_cast<StringRecTy *>(Type) == 0) {
+    llvm_unreachable("Record name is not a string!");
+  }
+}
+
+DefInit *Record::getDefInit() {
+  if (!TheInit)
+    TheInit = new DefInit(this, new RecordRecTy(this));
+  return TheInit;
+}
+
+const std::string &Record::getName() const {
+  const StringInit *NameString =
+    dynamic_cast<const StringInit *>(Name);
+  assert(NameString && "Record name is not a string!");
+  return NameString->getValue();
+}
+
+void Record::setName(Init *NewName) {
+  if (TrackedRecords.getDef(Name->getAsUnquotedString()) == this) {
+    TrackedRecords.removeDef(Name->getAsUnquotedString());
+    Name = NewName;
     TrackedRecords.addDef(this);
   } else {
-    TrackedRecords.removeClass(getName());
-    this->Name = Name;
+    TrackedRecords.removeClass(Name->getAsUnquotedString());
+    Name = NewName;
     TrackedRecords.addClass(this);
   }
+  checkName();
+  // Since the Init for the name was changed, see if we can resolve
+  // any of it using members of the Record.
+  Init *ComputedName = Name->resolveReferences(*this, 0);
+  if (ComputedName != Name) {
+    setName(ComputedName);
+  }
+  // DO NOT resolve record values to the name at this point because
+  // there might be default values for arguments of this def.  Those
+  // arguments might not have been resolved yet so we don't want to
+  // prematurely assume values for those arguments were not passed to
+  // this def.
+  //
+  // Nonetheless, it may be that some of this Record's values
+  // reference the record name.  Indeed, the reason for having the
+  // record name be an Init is to provide this flexibility.  The extra
+  // resolve steps after completely instantiating defs takes care of
+  // this.  See TGParser::ParseDef and TGParser::ParseDefm.
+}
+
+void Record::setName(const std::string &Name) {
+  setName(StringInit::get(Name));
 }
 
 /// resolveReferencesTo - If anything in this record refers to RV, replace the
@@ -1351,7 +1795,7 @@ std::string Record::getValueAsString(StringRef FieldName) const {
     throw "Record `" + getName() + "' does not have a field named `" +
           FieldName.str() + "'!\n";
 
-  if (const StringInit *SI = dynamic_cast<const StringInit*>(R->getValue()))
+  if (StringInit *SI = dynamic_cast<StringInit*>(R->getValue()))
     return SI->getValue();
   throw "Record `" + getName() + "', field `" + FieldName.str() +
         "' does not have a string initializer!";
@@ -1516,7 +1960,7 @@ std::string Record::getValueAsCode(StringRef FieldName) const {
     throw "Record `" + getName() + "' does not have a field named `" +
       FieldName.str() + "'!\n";
 
-  if (const CodeInit *CI = dynamic_cast<const CodeInit*>(R->getValue()))
+  if (CodeInit *CI = dynamic_cast<CodeInit*>(R->getValue()))
     return CI->getValue();
   throw "Record `" + getName() + "', field `" + FieldName.str() +
     "' does not have a code initializer!";
diff --git a/contrib/llvm/utils/TableGen/TGLexer.cpp b/contrib/llvm/lib/TableGen/TGLexer.cpp
similarity index 90%
rename from contrib/llvm/utils/TableGen/TGLexer.cpp
rename to contrib/llvm/lib/TableGen/TGLexer.cpp
index b4b90ff64eb6..8c1b4290548d 100644
--- a/contrib/llvm/utils/TableGen/TGLexer.cpp
+++ b/contrib/llvm/lib/TableGen/TGLexer.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "TGLexer.h"
-#include "Error.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Config/config.h"
@@ -208,40 +208,40 @@ tgtok::TokKind TGLexer::LexVarName() {
 tgtok::TokKind TGLexer::LexIdentifier() {
   // The first letter is [a-zA-Z_#].
   const char *IdentStart = TokStart;
-  
+
   // Match the rest of the identifier regex: [0-9a-zA-Z_#]*
   while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_' ||
          *CurPtr == '#')
     ++CurPtr;
-  
-  
+
   // Check to see if this identifier is a keyword.
-  unsigned Len = CurPtr-IdentStart;
-  
-  if (Len == 3 && !memcmp(IdentStart, "int", 3)) return tgtok::Int;
-  if (Len == 3 && !memcmp(IdentStart, "bit", 3)) return tgtok::Bit;
-  if (Len == 4 && !memcmp(IdentStart, "bits", 4)) return tgtok::Bits;
-  if (Len == 6 && !memcmp(IdentStart, "string", 6)) return tgtok::String;
-  if (Len == 4 && !memcmp(IdentStart, "list", 4)) return tgtok::List;
-  if (Len == 4 && !memcmp(IdentStart, "code", 4)) return tgtok::Code;
-  if (Len == 3 && !memcmp(IdentStart, "dag", 3)) return tgtok::Dag;
-  
-  if (Len == 5 && !memcmp(IdentStart, "class", 5)) return tgtok::Class;
-  if (Len == 3 && !memcmp(IdentStart, "def", 3)) return tgtok::Def;
-  if (Len == 4 && !memcmp(IdentStart, "defm", 4)) return tgtok::Defm;
-  if (Len == 10 && !memcmp(IdentStart, "multiclass", 10))
-    return tgtok::MultiClass;
-  if (Len == 5 && !memcmp(IdentStart, "field", 5)) return tgtok::Field;
-  if (Len == 3 && !memcmp(IdentStart, "let", 3)) return tgtok::Let;
-  if (Len == 2 && !memcmp(IdentStart, "in", 2)) return tgtok::In;
-  
-  if (Len == 7 && !memcmp(IdentStart, "include", 7)) {
+  StringRef Str(IdentStart, CurPtr-IdentStart);
+
+  if (Str == "include") {
     if (LexInclude()) return tgtok::Error;
     return Lex();
   }
-    
-  CurStrVal.assign(IdentStart, CurPtr);
-  return tgtok::Id;
+
+  tgtok::TokKind Kind = StringSwitch<tgtok::TokKind>(Str)
+    .Case("int", tgtok::Int)
+    .Case("bit", tgtok::Bit)
+    .Case("bits", tgtok::Bits)
+    .Case("string", tgtok::String)
+    .Case("list", tgtok::List)
+    .Case("code", tgtok::Code)
+    .Case("dag", tgtok::Dag)
+    .Case("class", tgtok::Class)
+    .Case("def", tgtok::Def)
+    .Case("defm", tgtok::Defm)
+    .Case("multiclass", tgtok::MultiClass)
+    .Case("field", tgtok::Field)
+    .Case("let", tgtok::Let)
+    .Case("in", tgtok::In)
+    .Default(tgtok::Id);
+
+  if (Kind == tgtok::Id)
+    CurStrVal.assign(Str.begin(), Str.end());
+  return Kind;
 }
 
 /// LexInclude - We just read the "include" token.  Get the string token that
diff --git a/contrib/llvm/utils/TableGen/TGLexer.h b/contrib/llvm/lib/TableGen/TGLexer.h
similarity index 100%
rename from contrib/llvm/utils/TableGen/TGLexer.h
rename to contrib/llvm/lib/TableGen/TGLexer.h
diff --git a/contrib/llvm/utils/TableGen/TGParser.cpp b/contrib/llvm/lib/TableGen/TGParser.cpp
similarity index 88%
rename from contrib/llvm/utils/TableGen/TGParser.cpp
rename to contrib/llvm/lib/TableGen/TGParser.cpp
index 59097f986f79..e7f00baf4931 100644
--- a/contrib/llvm/utils/TableGen/TGParser.cpp
+++ b/contrib/llvm/lib/TableGen/TGParser.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "TGParser.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include <algorithm>
 #include <sstream>
@@ -106,9 +106,9 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, const std::string &ValName,
       return Error(Loc, "Value '" + ValName + "' is not a bits type");
 
     // Convert the incoming value to a bits type of the appropriate size...
-    Init *BI = V->convertInitializerTo(new BitsRecTy(BitList.size()));
+    Init *BI = V->convertInitializerTo(BitsRecTy::get(BitList.size()));
     if (BI == 0) {
-      V->convertInitializerTo(new BitsRecTy(BitList.size()));
+      V->convertInitializerTo(BitsRecTy::get(BitList.size()));
       return Error(Loc, "Initializer is not compatible with bit range");
     }
 
@@ -116,22 +116,22 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, const std::string &ValName,
     BitsInit *BInit = dynamic_cast<BitsInit*>(BI);
     assert(BInit != 0);
 
-    BitsInit *NewVal = new BitsInit(CurVal->getNumBits());
+    SmallVector<Init *, 16> NewBits(CurVal->getNumBits());
 
     // Loop over bits, assigning values as appropriate.
     for (unsigned i = 0, e = BitList.size(); i != e; ++i) {
       unsigned Bit = BitList[i];
-      if (NewVal->getBit(Bit))
+      if (NewBits[Bit])
         return Error(Loc, "Cannot set bit #" + utostr(Bit) + " of value '" +
                      ValName + "' more than once");
-      NewVal->setBit(Bit, BInit->getBit(i));
+      NewBits[Bit] = BInit->getBit(i);
     }
 
     for (unsigned i = 0, e = CurVal->getNumBits(); i != e; ++i)
-      if (NewVal->getBit(i) == 0)
-        NewVal->setBit(i, CurVal->getBit(i));
+      if (NewBits[i] == 0)
+        NewBits[i] = CurVal->getBit(i);
 
-    V = NewVal;
+    V = BitsInit::get(NewBits);
   }
 
   if (RV->setValue(V))
@@ -581,13 +581,13 @@ bool TGParser::ParseOptionalBitList(std::vector<unsigned> &Ranges) {
 RecTy *TGParser::ParseType() {
   switch (Lex.getCode()) {
   default: TokError("Unknown token when expecting a type"); return 0;
-  case tgtok::String: Lex.Lex(); return new StringRecTy();
-  case tgtok::Bit:    Lex.Lex(); return new BitRecTy();
-  case tgtok::Int:    Lex.Lex(); return new IntRecTy();
-  case tgtok::Code:   Lex.Lex(); return new CodeRecTy();
-  case tgtok::Dag:    Lex.Lex(); return new DagRecTy();
+  case tgtok::String: Lex.Lex(); return StringRecTy::get();
+  case tgtok::Bit:    Lex.Lex(); return BitRecTy::get();
+  case tgtok::Int:    Lex.Lex(); return IntRecTy::get();
+  case tgtok::Code:   Lex.Lex(); return CodeRecTy::get();
+  case tgtok::Dag:    Lex.Lex(); return DagRecTy::get();
   case tgtok::Id:
-    if (Record *R = ParseClassID()) return new RecordRecTy(R);
+    if (Record *R = ParseClassID()) return RecordRecTy::get(R);
     return 0;
   case tgtok::Bits: {
     if (Lex.Lex() != tgtok::less) { // Eat 'bits'
@@ -604,7 +604,7 @@ RecTy *TGParser::ParseType() {
       return 0;
     }
     Lex.Lex();  // Eat '>'
-    return new BitsRecTy(Val);
+    return BitsRecTy::get(Val);
   }
   case tgtok::List: {
     if (Lex.Lex() != tgtok::less) { // Eat 'bits'
@@ -620,7 +620,7 @@ RecTy *TGParser::ParseType() {
       return 0;
     }
     Lex.Lex();  // Eat '>'
-    return new ListRecTy(SubType);
+    return ListRecTy::get(SubType);
   }
   }
 }
@@ -647,13 +647,16 @@ Init *TGParser::ParseIDValue(Record *CurRec,
                              const std::string &Name, SMLoc NameLoc) {
   if (CurRec) {
     if (const RecordVal *RV = CurRec->getValue(Name))
-      return new VarInit(Name, RV->getType());
+      return VarInit::get(Name, RV->getType());
 
     std::string TemplateArgName = CurRec->getName()+":"+Name;
+    if (CurMultiClass)
+      TemplateArgName = CurMultiClass->Rec.getName()+"::"+TemplateArgName;
+
     if (CurRec->isTemplateArg(TemplateArgName)) {
       const RecordVal *RV = CurRec->getValue(TemplateArgName);
       assert(RV && "Template arg doesn't exist??");
-      return new VarInit(TemplateArgName, RV->getType());
+      return VarInit::get(TemplateArgName, RV->getType());
     }
   }
 
@@ -662,12 +665,12 @@ Init *TGParser::ParseIDValue(Record *CurRec,
     if (CurMultiClass->Rec.isTemplateArg(MCName)) {
       const RecordVal *RV = CurMultiClass->Rec.getValue(MCName);
       assert(RV && "Template arg doesn't exist??");
-      return new VarInit(MCName, RV->getType());
+      return VarInit::get(MCName, RV->getType());
     }
   }
 
   if (Record *D = Records.getDef(Name))
-    return new DefInit(D);
+    return DefInit::get(D);
 
   Error(NameLoc, "Variable not defined: '" + Name + "'");
   return 0;
@@ -715,7 +718,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
     case tgtok::XEmpty:
       Lex.Lex();  // eat the operation
       Code = UnOpInit::EMPTY;
-      Type = new IntRecTy;
+      Type = IntRecTy::get();
       break;
     }
     if (Lex.getCode() != tgtok::l_paren) {
@@ -767,7 +770,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
           if (Code == UnOpInit::HEAD) {
             Type = Itemt->getType();
           } else {
-            Type = new ListRecTy(Itemt->getType());
+            Type = ListRecTy::get(Itemt->getType());
           }
         } else {
           assert(LHSt && "expected list type argument in unary operator");
@@ -790,7 +793,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
       return 0;
     }
     Lex.Lex();  // eat the ')'
-    return (new UnOpInit(Code, LHS, Type))->Fold(CurRec, CurMultiClass);
+    return (UnOpInit::get(Code, LHS, Type))->Fold(CurRec, CurMultiClass);
   }
 
   case tgtok::XConcat:
@@ -808,14 +811,14 @@ Init *TGParser::ParseOperation(Record *CurRec) {
 
     switch (OpTok) {
     default: assert(0 && "Unhandled code!");
-    case tgtok::XConcat: Code = BinOpInit::CONCAT; Type = new DagRecTy(); break;
-    case tgtok::XSRA:    Code = BinOpInit::SRA;    Type = new IntRecTy(); break;
-    case tgtok::XSRL:    Code = BinOpInit::SRL;    Type = new IntRecTy(); break;
-    case tgtok::XSHL:    Code = BinOpInit::SHL;    Type = new IntRecTy(); break;
-    case tgtok::XEq:     Code = BinOpInit::EQ;     Type = new BitRecTy(); break;
+    case tgtok::XConcat: Code = BinOpInit::CONCAT;Type = DagRecTy::get(); break;
+    case tgtok::XSRA:    Code = BinOpInit::SRA;   Type = IntRecTy::get(); break;
+    case tgtok::XSRL:    Code = BinOpInit::SRL;   Type = IntRecTy::get(); break;
+    case tgtok::XSHL:    Code = BinOpInit::SHL;   Type = IntRecTy::get(); break;
+    case tgtok::XEq:     Code = BinOpInit::EQ;    Type = BitRecTy::get(); break;
     case tgtok::XStrConcat:
       Code = BinOpInit::STRCONCAT;
-      Type = new StringRecTy();
+      Type = StringRecTy::get();
       break;
     }
 
@@ -848,14 +851,14 @@ Init *TGParser::ParseOperation(Record *CurRec) {
     if (Code == BinOpInit::STRCONCAT) {
       while (InitList.size() > 2) {
         Init *RHS = InitList.pop_back_val();
-        RHS = (new BinOpInit(Code, InitList.back(), RHS, Type))
-                      ->Fold(CurRec, CurMultiClass);
+        RHS = (BinOpInit::get(Code, InitList.back(), RHS, Type))
+                           ->Fold(CurRec, CurMultiClass);
         InitList.back() = RHS;
       }
     }
 
     if (InitList.size() == 2)
-      return (new BinOpInit(Code, InitList[0], InitList[1], Type))
+      return (BinOpInit::get(Code, InitList[0], InitList[1], Type))
         ->Fold(CurRec, CurMultiClass);
 
     Error(OpLoc, "expected two operands to operator");
@@ -932,14 +935,14 @@ Init *TGParser::ParseOperation(Record *CurRec) {
 
         if (MHSbits && RHSbits &&
             MHSbits->getNumBits() == RHSbits->getNumBits()) {
-          Type = new BitRecTy();
+          Type = BitRecTy::get();
           break;
         } else {
           BitInit *MHSbit = dynamic_cast<BitInit*>(MHS);
           BitInit *RHSbit = dynamic_cast<BitInit*>(RHS);
 
           if (MHSbit && RHSbit) {
-            Type = new BitRecTy();
+            Type = BitRecTy::get();
             break;
           }
         }
@@ -982,7 +985,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
       break;
     }
     }
-    return (new TernOpInit(Code, LHS, MHS, RHS, Type))->Fold(CurRec,
+    return (TernOpInit::get(Code, LHS, MHS, RHS, Type))->Fold(CurRec,
                                                              CurMultiClass);
   }
   }
@@ -1042,7 +1045,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType) {
   Init *R = 0;
   switch (Lex.getCode()) {
   default: TokError("Unknown token when parsing a value"); break;
-  case tgtok::IntVal: R = new IntInit(Lex.getCurIntVal()); Lex.Lex(); break;
+  case tgtok::IntVal: R = IntInit::get(Lex.getCurIntVal()); Lex.Lex(); break;
   case tgtok::StrVal: {
     std::string Val = Lex.getCurStrVal();
     Lex.Lex();
@@ -1053,15 +1056,15 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType) {
       Lex.Lex();
     }
 
-    R = new StringInit(Val);
+    R = StringInit::get(Val);
     break;
   }
   case tgtok::CodeFragment:
-    R = new CodeInit(Lex.getCurStrVal());
+    R = CodeInit::get(Lex.getCurStrVal());
     Lex.Lex();
     break;
   case tgtok::question:
-    R = new UnsetInit();
+    R = UnsetInit::get();
     Lex.Lex();
     break;
   case tgtok::Id: {
@@ -1110,7 +1113,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType) {
     Records.addDef(NewRec);
 
     // The result of the expression is a reference to the new record.
-    return new DefInit(NewRec);
+    return DefInit::get(NewRec);
   }
   case tgtok::l_brace: {           // Value ::= '{' ValueList '}'
     SMLoc BraceLoc = Lex.getLoc();
@@ -1127,17 +1130,18 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType) {
     }
     Lex.Lex();  // eat the '}'
 
-    BitsInit *Result = new BitsInit(Vals.size());
+    SmallVector<Init *, 16> NewBits(Vals.size());
+
     for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
-      Init *Bit = Vals[i]->convertInitializerTo(new BitRecTy());
+      Init *Bit = Vals[i]->convertInitializerTo(BitRecTy::get());
       if (Bit == 0) {
         Error(BraceLoc, "Element #" + utostr(i) + " (" + Vals[i]->getAsString()+
               ") is not convertable to a bit");
         return 0;
       }
-      Result->setBit(Vals.size()-i-1, Bit);
+      NewBits[Vals.size()-i-1] = Bit;
     }
-    return Result;
+    return BitsInit::get(NewBits);
   }
   case tgtok::l_square: {          // Value ::= '[' ValueList ']'
     Lex.Lex(); // eat the '['
@@ -1236,7 +1240,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType) {
       DeducedEltTy = EltTy;
     }
 
-    return new ListInit(Vals, DeducedEltTy);
+    return ListInit::get(Vals, DeducedEltTy);
   }
   case tgtok::l_paren: {         // Value ::= '(' IDValue DagArgList ')'
     Lex.Lex();   // eat the '('
@@ -1271,7 +1275,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType) {
     }
     Lex.Lex();  // eat the ')'
 
-    return new DagInit(Operator, OperatorName, DagArgs);
+    return DagInit::get(Operator, OperatorName, DagArgs);
   }
 
   case tgtok::XHead:
@@ -1361,7 +1365,7 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType) {
                  Result->getAsString() + "'");
         return 0;
       }
-      Result = new FieldInit(Result, Lex.getCurStrVal());
+      Result = FieldInit::get(Result, Lex.getCurStrVal());
       Lex.Lex();  // eat field name
       break;
     }
@@ -1415,6 +1419,10 @@ std::vector<Init*> TGParser::ParseValueList(Record *CurRec, Record *ArgsRec,
   if (ArgsRec != 0 && EltTy == 0) {
     const std::vector<std::string> &TArgs = ArgsRec->getTemplateArgs();
     const RecordVal *RV = ArgsRec->getValue(TArgs[ArgN]);
+    if (!RV) {
+      errs() << "Cannot find template arg " << ArgN << " (" << TArgs[ArgN]
+        << ")\n";
+    }
     assert(RV && "Template argument record not found??");
     ItemType = RV->getType();
     ++ArgN;
@@ -1689,6 +1697,9 @@ bool TGParser::ParseDef(MultiClass *CurMultiClass) {
     return true;
 
   if (CurMultiClass == 0)  // Def's in multiclasses aren't really defs.
+    // See Record::setName().  This resolve step will see any new name
+    // for the def that might have been created when resolving
+    // inheritance, values and arguments above.
     CurRec->resolveReferences();
 
   // If ObjectBody has template arguments, it's an error.
@@ -1709,7 +1720,6 @@ bool TGParser::ParseDef(MultiClass *CurMultiClass) {
   return false;
 }
 
-
 /// ParseClass - Parse a tblgen class definition.
 ///
 ///   ClassInst ::= CLASS ID TemplateArgList? ObjectBody
@@ -1911,6 +1921,115 @@ bool TGParser::ParseMultiClass() {
   return false;
 }
 
+Record *TGParser::
+InstantiateMulticlassDef(MultiClass &MC,
+                         Record *DefProto,
+                         const std::string &DefmPrefix,
+                         SMLoc DefmPrefixLoc) {
+  // Add in the defm name.  If the defm prefix is empty, give each
+  // instantiated def a unique name.  Otherwise, if "#NAME#" exists in the
+  // name, substitute the prefix for #NAME#.  Otherwise, use the defm name
+  // as a prefix.
+  std::string DefName = DefProto->getName();
+  if (DefmPrefix.empty()) {
+    DefName = GetNewAnonymousName();
+  } else {
+    std::string::size_type idx = DefName.find("#NAME#");
+    if (idx != std::string::npos) {
+      DefName.replace(idx, 6, DefmPrefix);
+    } else {
+      // Add the suffix to the defm name to get the new name.
+      DefName = DefmPrefix + DefName;
+    }
+  }
+
+  Record *CurRec = new Record(DefName, DefmPrefixLoc, Records);
+
+  SubClassReference Ref;
+  Ref.RefLoc = DefmPrefixLoc;
+  Ref.Rec = DefProto;
+  AddSubClass(CurRec, Ref);
+
+  return CurRec;
+}
+
+bool TGParser::ResolveMulticlassDefArgs(MultiClass &MC,
+                                        Record *CurRec,
+                                        SMLoc DefmPrefixLoc,
+                                        SMLoc SubClassLoc,
+                                        const std::vector<std::string> &TArgs,
+                                        std::vector<Init *> &TemplateVals,
+                                        bool DeleteArgs) {
+  // Loop over all of the template arguments, setting them to the specified
+  // value or leaving them as the default if necessary.
+  for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
+    // Check if a value is specified for this temp-arg.
+    if (i < TemplateVals.size()) {
+      // Set it now.
+      if (SetValue(CurRec, DefmPrefixLoc, TArgs[i], std::vector<unsigned>(),
+                   TemplateVals[i]))
+        return true;
+        
+      // Resolve it next.
+      CurRec->resolveReferencesTo(CurRec->getValue(TArgs[i]));
+
+      if (DeleteArgs)
+        // Now remove it.
+        CurRec->removeValue(TArgs[i]);
+        
+    } else if (!CurRec->getValue(TArgs[i])->getValue()->isComplete()) {
+      return Error(SubClassLoc, "value not specified for template argument #"+
+                   utostr(i) + " (" + TArgs[i] + ") of multiclassclass '" +
+                   MC.Rec.getName() + "'");
+    }
+  }
+  return false;
+}
+
+bool TGParser::ResolveMulticlassDef(MultiClass &MC,
+                                    Record *CurRec,
+                                    Record *DefProto,
+                                    SMLoc DefmPrefixLoc) {
+  // If the mdef is inside a 'let' expression, add to each def.
+  for (unsigned i = 0, e = LetStack.size(); i != e; ++i)
+    for (unsigned j = 0, e = LetStack[i].size(); j != e; ++j)
+      if (SetValue(CurRec, LetStack[i][j].Loc, LetStack[i][j].Name,
+                   LetStack[i][j].Bits, LetStack[i][j].Value))
+        return Error(DefmPrefixLoc, "when instantiating this defm");
+
+  // Ensure redefinition doesn't happen.
+  if (Records.getDef(CurRec->getName()))
+    return Error(DefmPrefixLoc, "def '" + CurRec->getName() + 
+                 "' already defined, instantiating defm with subdef '" + 
+                 DefProto->getName() + "'");
+
+  // Don't create a top level definition for defm inside multiclasses,
+  // instead, only update the prototypes and bind the template args
+  // with the new created definition.
+  if (CurMultiClass) {
+    for (unsigned i = 0, e = CurMultiClass->DefPrototypes.size();
+         i != e; ++i)
+      if (CurMultiClass->DefPrototypes[i]->getName() == CurRec->getName())
+        return Error(DefmPrefixLoc, "defm '" + CurRec->getName() +
+                     "' already defined in this multiclass!");
+    CurMultiClass->DefPrototypes.push_back(CurRec);
+
+    // Copy the template arguments for the multiclass into the new def.
+    const std::vector<std::string> &TA =
+      CurMultiClass->Rec.getTemplateArgs();
+
+    for (unsigned i = 0, e = TA.size(); i != e; ++i) {
+      const RecordVal *RV = CurMultiClass->Rec.getValue(TA[i]);
+      assert(RV && "Template arg doesn't exist?");
+      CurRec->addValue(*RV);
+    }
+  } else {
+    Records.addDef(CurRec);
+  }
+
+  return false;
+}
+
 /// ParseDefm - Parse the instantiation of a multiclass.
 ///
 ///   DefMInst ::= DEFM ID ':' DefmSubClassRef ';'
@@ -1960,99 +2079,19 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
     for (unsigned i = 0, e = MC->DefPrototypes.size(); i != e; ++i) {
       Record *DefProto = MC->DefPrototypes[i];
 
-      // Add in the defm name.  If the defm prefix is empty, give each
-      // instantiated def a unique name.  Otherwise, if "#NAME#" exists in the
-      // name, substitute the prefix for #NAME#.  Otherwise, use the defm name
-      // as a prefix.
-      std::string DefName = DefProto->getName();
-      if (DefmPrefix.empty()) {
-        DefName = GetNewAnonymousName();
-      } else {
-        std::string::size_type idx = DefName.find("#NAME#");
-        if (idx != std::string::npos) {
-          DefName.replace(idx, 6, DefmPrefix);
-        } else {
-          // Add the suffix to the defm name to get the new name.
-          DefName = DefmPrefix + DefName;
-        }
-      }
+      Record *CurRec = InstantiateMulticlassDef(*MC, DefProto, DefmPrefix, DefmPrefixLoc);
 
-      Record *CurRec = new Record(DefName, DefmPrefixLoc, Records);
+      if (ResolveMulticlassDefArgs(*MC, CurRec, DefmPrefixLoc, SubClassLoc,
+                                   TArgs, TemplateVals, true/*Delete args*/))
+        return Error(SubClassLoc, "could not instantiate def");
 
-      SubClassReference Ref;
-      Ref.RefLoc = DefmPrefixLoc;
-      Ref.Rec = DefProto;
-      AddSubClass(CurRec, Ref);
-
-      // Loop over all of the template arguments, setting them to the specified
-      // value or leaving them as the default if necessary.
-      for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
-        // Check if a value is specified for this temp-arg.
-        if (i < TemplateVals.size()) {
-          // Set it now.
-          if (SetValue(CurRec, DefmPrefixLoc, TArgs[i], std::vector<unsigned>(),
-                       TemplateVals[i]))
-            return true;
-
-          // Resolve it next.
-          CurRec->resolveReferencesTo(CurRec->getValue(TArgs[i]));
-
-          // Now remove it.
-          CurRec->removeValue(TArgs[i]);
-
-        } else if (!CurRec->getValue(TArgs[i])->getValue()->isComplete()) {
-          return Error(SubClassLoc,
-                       "value not specified for template argument #"+
-                       utostr(i) + " (" + TArgs[i] + ") of multiclassclass '" +
-                       MC->Rec.getName() + "'");
-        }
-      }
-
-      // If the mdef is inside a 'let' expression, add to each def.
-      for (unsigned i = 0, e = LetStack.size(); i != e; ++i)
-        for (unsigned j = 0, e = LetStack[i].size(); j != e; ++j)
-          if (SetValue(CurRec, LetStack[i][j].Loc, LetStack[i][j].Name,
-                       LetStack[i][j].Bits, LetStack[i][j].Value)) {
-            Error(DefmPrefixLoc, "when instantiating this defm");
-            return true;
-          }
-
-      // Ensure redefinition doesn't happen.
-      if (Records.getDef(CurRec->getName()))
-        return Error(DefmPrefixLoc, "def '" + CurRec->getName() +
-                     "' already defined, instantiating defm with subdef '" +
-                     DefProto->getName() + "'");
-
-      // Don't create a top level definition for defm inside multiclasses,
-      // instead, only update the prototypes and bind the template args
-      // with the new created definition.
-      if (CurMultiClass) {
-        for (unsigned i = 0, e = CurMultiClass->DefPrototypes.size();
-             i != e; ++i) {
-          if (CurMultiClass->DefPrototypes[i]->getName() == CurRec->getName()) {
-            Error(DefmPrefixLoc, "defm '" + CurRec->getName() +
-                  "' already defined in this multiclass!");
-            return 0;
-          }
-        }
-        CurMultiClass->DefPrototypes.push_back(CurRec);
-
-        // Copy the template arguments for the multiclass into the new def.
-        const std::vector<std::string> &TA =
-          CurMultiClass->Rec.getTemplateArgs();
-
-        for (unsigned i = 0, e = TA.size(); i != e; ++i) {
-          const RecordVal *RV = CurMultiClass->Rec.getValue(TA[i]);
-          assert(RV && "Template arg doesn't exist?");
-          CurRec->addValue(*RV);
-        }
-      } else {
-        Records.addDef(CurRec);
-      }
+      if (ResolveMulticlassDef(*MC, CurRec, DefProto, DefmPrefixLoc))
+        return Error(SubClassLoc, "could not instantiate def");
 
       NewRecDefs.push_back(CurRec);
     }
 
+
     if (Lex.getCode() != tgtok::comma) break;
     Lex.Lex(); // eat ','.
 
@@ -2101,6 +2140,9 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
 
   if (!CurMultiClass)
     for (unsigned i = 0, e = NewRecDefs.size(); i != e; ++i)
+      // See Record::setName().  This resolve step will see any new
+      // name for the def that might have been created when resolving
+      // inheritance, values and arguments above.
       NewRecDefs[i]->resolveReferences();
 
   if (Lex.getCode() != tgtok::semi)
diff --git a/contrib/llvm/utils/TableGen/TGParser.h b/contrib/llvm/lib/TableGen/TGParser.h
similarity index 82%
rename from contrib/llvm/utils/TableGen/TGParser.h
rename to contrib/llvm/lib/TableGen/TGParser.h
index dce7e1dec94d..db8a62029746 100644
--- a/contrib/llvm/utils/TableGen/TGParser.h
+++ b/contrib/llvm/lib/TableGen/TGParser.h
@@ -15,7 +15,7 @@
 #define TGPARSER_H
 
 #include "TGLexer.h"
-#include "Error.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/SourceMgr.h"
 #include <map>
@@ -24,7 +24,7 @@ namespace llvm {
   class Record;
   class RecordVal;
   class RecordKeeper;
-  struct RecTy;
+  class RecTy;
   class Init;
   struct MultiClass;
   struct SubClassReference;
@@ -83,6 +83,21 @@ class TGParser {
   bool ParseObject(MultiClass *MC);
   bool ParseClass();
   bool ParseMultiClass();
+  Record *InstantiateMulticlassDef(MultiClass &MC,
+                                   Record *DefProto,
+                                   const std::string &DefmPrefix,
+                                   SMLoc DefmPrefixLoc);
+  bool ResolveMulticlassDefArgs(MultiClass &MC,
+                                Record *DefProto,
+                                SMLoc DefmPrefixLoc,
+                                SMLoc SubClassLoc,
+                                const std::vector<std::string> &TArgs,
+                                std::vector<Init *> &TemplateVals,
+                                bool DeleteArgs);
+  bool ResolveMulticlassDef(MultiClass &MC,
+                            Record *CurRec,
+                            Record *DefProto,
+                            SMLoc DefmPrefixLoc);
   bool ParseDefm(MultiClass *CurMultiClass);
   bool ParseDef(MultiClass *CurMultiClass);
   bool ParseTopLevelLet(MultiClass *CurMultiClass);
diff --git a/contrib/llvm/utils/TableGen/TableGenBackend.cpp b/contrib/llvm/lib/TableGen/TableGenBackend.cpp
similarity index 92%
rename from contrib/llvm/utils/TableGen/TableGenBackend.cpp
rename to contrib/llvm/lib/TableGen/TableGenBackend.cpp
index b3e33b5f9c24..29588db324cf 100644
--- a/contrib/llvm/utils/TableGen/TableGenBackend.cpp
+++ b/contrib/llvm/lib/TableGen/TableGenBackend.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "TableGenBackend.h"
-#include "Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include "llvm/TableGen/Record.h"
 using namespace llvm;
 
 void TableGenBackend::EmitSourceFileHeader(const std::string &Desc,
diff --git a/contrib/llvm/lib/Target/ARM/ARM.h b/contrib/llvm/lib/Target/ARM/ARM.h
index 08dc340f8541..16d0da3b8ac7 100644
--- a/contrib/llvm/lib/Target/ARM/ARM.h
+++ b/contrib/llvm/lib/Target/ARM/ARM.h
@@ -15,7 +15,7 @@
 #ifndef TARGET_ARM_H
 #define TARGET_ARM_H
 
-#include "ARMBaseInfo.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
 #include "MCTargetDesc/ARMMCTargetDesc.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -29,19 +29,7 @@ class ARMBaseTargetMachine;
 class FunctionPass;
 class JITCodeEmitter;
 class MachineInstr;
-class MCCodeEmitter;
 class MCInst;
-class MCInstrInfo;
-class MCObjectWriter;
-class MCSubtargetInfo;
-class TargetAsmBackend;
-class formatted_raw_ostream;
-
-MCCodeEmitter *createARMMCCodeEmitter(const MCInstrInfo &MCII,
-                                      const MCSubtargetInfo &STI,
-                                      MCContext &Ctx);
-
-TargetAsmBackend *createARMAsmBackend(const Target &, const std::string &);
 
 FunctionPass *createARMISelDag(ARMBaseTargetMachine &TM,
                                CodeGenOpt::Level OptLevel);
@@ -53,7 +41,6 @@ FunctionPass *createARMLoadStoreOptimizationPass(bool PreAlloc = false);
 FunctionPass *createARMExpandPseudoPass();
 FunctionPass *createARMGlobalMergePass(const TargetLowering* tli);
 FunctionPass *createARMConstantIslandPass();
-FunctionPass *createNEONMoveFixPass();
 FunctionPass *createMLxExpansionPass();
 FunctionPass *createThumb2ITBlockPass();
 FunctionPass *createThumb2SizeReductionPass();
@@ -61,12 +48,6 @@ FunctionPass *createThumb2SizeReductionPass();
 void LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                   ARMAsmPrinter &AP);
 
-/// createARMMachObjectWriter - Construct an ARM Mach-O object writer.
-MCObjectWriter *createARMMachObjectWriter(raw_ostream &OS,
-                                          bool Is64Bit,
-                                          uint32_t CPUType,
-                                          uint32_t CPUSubtype);
-
 } // end namespace llvm;
 
 #endif
diff --git a/contrib/llvm/lib/Target/ARM/ARM.td b/contrib/llvm/lib/Target/ARM/ARM.td
index cf333ccd49ba..5c727ad6e343 100644
--- a/contrib/llvm/lib/Target/ARM/ARM.td
+++ b/contrib/llvm/lib/Target/ARM/ARM.td
@@ -23,6 +23,9 @@ include "llvm/Target/Target.td"
 def ModeThumb  : SubtargetFeature<"thumb-mode", "InThumbMode", "true",
                                   "Thumb mode">;
 
+def ModeNaCl   : SubtargetFeature<"nacl-mode", "InNaClMode", "true",
+                                  "Native client mode">;
+
 //===----------------------------------------------------------------------===//
 // ARM Subtarget features.
 //
@@ -85,12 +88,16 @@ def FeatureAvoidPartialCPSR : SubtargetFeature<"avoid-partial-cpsr",
 
 /// Some M architectures don't have the DSP extension (v7E-M vs. v7M)
 def FeatureDSPThumb2 : SubtargetFeature<"t2dsp", "Thumb2DSP", "true",
-                                 "Supports v7 DSP instructions in Thumb2.">;
+                                 "Supports v7 DSP instructions in Thumb2">;
 
 // Multiprocessing extension.
 def FeatureMP : SubtargetFeature<"mp", "HasMPExtension", "true",
                                  "Supports Multiprocessing extension">;
 
+// M-series ISA?
+def FeatureMClass : SubtargetFeature<"mclass", "IsMClass", "true",
+                                     "Is microcontroller profile ('M' series)">;
+
 // ARM ISAs.
 def HasV4TOps   : SubtargetFeature<"v4t", "HasV4TOps", "true",
                                    "Support ARM v4T instructions">;
@@ -105,7 +112,7 @@ def HasV6Ops    : SubtargetFeature<"v6", "HasV6Ops", "true",
                                    [HasV5TEOps]>;
 def HasV6T2Ops  : SubtargetFeature<"v6t2", "HasV6T2Ops", "true",
                                    "Support ARM v6t2 instructions",
-                                   [HasV6Ops, FeatureThumb2, FeatureDSPThumb2]>;
+                                   [HasV6Ops, FeatureThumb2]>;
 def HasV7Ops    : SubtargetFeature<"v7", "HasV7Ops", "true",
                                    "Support ARM v7 instructions",
                                    [HasV6T2Ops]>;
@@ -182,12 +189,14 @@ def : Processor<"mpcore",           ARMV6Itineraries, [HasV6Ops, FeatureVFP2,
 
 // V6M Processors.
 def : Processor<"cortex-m0",        ARMV6Itineraries, [HasV6Ops, FeatureNoARM,
-                                                       FeatureDB]>;
+                                                       FeatureDB, FeatureMClass]>;
 
 // V6T2 Processors.
-def : Processor<"arm1156t2-s",      ARMV6Itineraries, [HasV6T2Ops]>;
+def : Processor<"arm1156t2-s",      ARMV6Itineraries, [HasV6T2Ops,
+                                                       FeatureDSPThumb2]>;
 def : Processor<"arm1156t2f-s",     ARMV6Itineraries, [HasV6T2Ops, FeatureVFP2,
-                                                       FeatureHasSlowFPVMLx]>;
+                                                       FeatureHasSlowFPVMLx,
+                                                       FeatureDSPThumb2]>;
 
 // V7a Processors.
 def : Processor<"cortex-a8",        CortexA8Itineraries,
@@ -203,14 +212,14 @@ def : Processor<"cortex-a9-mp",     CortexA9Itineraries,
 // V7M Processors.
 def : ProcNoItin<"cortex-m3",       [HasV7Ops,
                                      FeatureThumb2, FeatureNoARM, FeatureDB,
-                                     FeatureHWDiv]>;
+                                     FeatureHWDiv, FeatureMClass]>;
 
 // V7EM Processors.
 def : ProcNoItin<"cortex-m4",       [HasV7Ops,
                                      FeatureThumb2, FeatureNoARM, FeatureDB,
                                      FeatureHWDiv, FeatureDSPThumb2,
                                      FeatureT2XtPk, FeatureVFP2,
-                                     FeatureVFPOnlySP]>;
+                                     FeatureVFPOnlySP, FeatureMClass]>;
 
 //===----------------------------------------------------------------------===//
 // Register File Description
diff --git a/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index dbc3ee41f3da..ea3319fb0e03 100644
--- a/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -15,15 +15,15 @@
 #define DEBUG_TYPE "asm-printer"
 #include "ARM.h"
 #include "ARMAsmPrinter.h"
-#include "ARMAddressingModes.h"
 #include "ARMBuildAttrs.h"
 #include "ARMBaseRegisterInfo.h"
 #include "ARMConstantPoolValue.h"
 #include "ARMMachineFunctionInfo.h"
-#include "ARMMCExpr.h"
 #include "ARMTargetMachine.h"
 #include "ARMTargetObjectFile.h"
 #include "InstPrinter/ARMInstPrinter.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
+#include "MCTargetDesc/ARMMCExpr.h"
 #include "llvm/Analysis/DebugInfo.h"
 #include "llvm/Constants.h"
 #include "llvm/Module.h"
@@ -45,13 +45,13 @@
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cctype>
 using namespace llvm;
@@ -92,7 +92,7 @@ namespace {
       case ARMBuildAttrs::Advanced_SIMD_arch:
       case ARMBuildAttrs::VFP_arch:
         Streamer.EmitRawText(StringRef("\t.fpu ") + LowercaseString(String));
-        break;    
+        break;
       default: assert(0 && "Unsupported Text attribute in ASM Mode"); break;
       }
     }
@@ -100,13 +100,41 @@ namespace {
   };
 
   class ObjectAttributeEmitter : public AttributeEmitter {
+    // This structure holds all attributes, accounting for
+    // their string/numeric value, so we can later emmit them
+    // in declaration order, keeping all in the same vector
+    struct AttributeItemType {
+      enum {
+        HiddenAttribute = 0,
+        NumericAttribute,
+        TextAttribute
+      } Type;
+      unsigned Tag;
+      unsigned IntValue;
+      StringRef StringValue;
+    } AttributeItem;
+
     MCObjectStreamer &Streamer;
     StringRef CurrentVendor;
-    SmallString<64> Contents;
+    SmallVector<AttributeItemType, 64> Contents;
+
+    // Account for the ULEB/String size of each item,
+    // not just the number of items
+    size_t ContentsSize;
+    // FIXME: this should be in a more generic place, but
+    // getULEBSize() is in MCAsmInfo and will be moved to MCDwarf
+    size_t getULEBSize(int Value) {
+      size_t Size = 0;
+      do {
+        Value >>= 7;
+        Size += sizeof(int8_t); // Is this really necessary?
+      } while (Value);
+      return Size;
+    }
 
   public:
     ObjectAttributeEmitter(MCObjectStreamer &Streamer_) :
-      Streamer(Streamer_), CurrentVendor("") { }
+      Streamer(Streamer_), CurrentVendor(""), ContentsSize(0) { }
 
     void MaybeSwitchVendor(StringRef Vendor) {
       assert(!Vendor.empty() && "Vendor cannot be empty.");
@@ -124,20 +152,32 @@ namespace {
     }
 
     void EmitAttribute(unsigned Attribute, unsigned Value) {
-      // FIXME: should be ULEB
-      Contents += Attribute;
-      Contents += Value;
+      AttributeItemType attr = {
+        AttributeItemType::NumericAttribute,
+        Attribute,
+        Value,
+        StringRef("")
+      };
+      ContentsSize += getULEBSize(Attribute);
+      ContentsSize += getULEBSize(Value);
+      Contents.push_back(attr);
     }
 
     void EmitTextAttribute(unsigned Attribute, StringRef String) {
-      Contents += Attribute;
-      Contents += UppercaseString(String);
-      Contents += 0;
+      AttributeItemType attr = {
+        AttributeItemType::TextAttribute,
+        Attribute,
+        0,
+        String
+      };
+      ContentsSize += getULEBSize(Attribute);
+      // String + \0
+      ContentsSize += String.size()+1;
+
+      Contents.push_back(attr);
     }
 
     void Finish() {
-      const size_t ContentsSize = Contents.size();
-
       // Vendor size + Vendor name + '\0'
       const size_t VendorHeaderSize = 4 + CurrentVendor.size() + 1;
 
@@ -151,7 +191,23 @@ namespace {
       Streamer.EmitIntValue(ARMBuildAttrs::File, 1);
       Streamer.EmitIntValue(TagHeaderSize + ContentsSize, 4);
 
-      Streamer.EmitBytes(Contents, 0);
+      // Size should have been accounted for already, now
+      // emit each field as its type (ULEB or String)
+      for (unsigned int i=0; i<Contents.size(); ++i) {
+        AttributeItemType item = Contents[i];
+        Streamer.EmitULEB128IntValue(item.Tag, 0);
+        switch (item.Type) {
+        case AttributeItemType::NumericAttribute:
+          Streamer.EmitULEB128IntValue(item.IntValue, 0);
+          break;
+        case AttributeItemType::TextAttribute:
+          Streamer.EmitBytes(UppercaseString(item.StringValue), 0);
+          Streamer.EmitIntValue(0, 1); // '\0'
+          break;
+        default:
+          assert(0 && "Invalid attribute type");
+        }
+      }
 
       Contents.clear();
     }
@@ -184,7 +240,7 @@ void ARMAsmPrinter::EmitDwarfRegOp(const MachineLocation &MLoc) const {
       // S registers are described as bit-pieces of a register
       // S[2x] = DW_OP_regx(256 + (x>>1)) DW_OP_bit_piece(32, 0)
       // S[2x+1] = DW_OP_regx(256 + (x>>1)) DW_OP_bit_piece(32, 32)
-      
+
       unsigned SReg = Reg - ARM::S0;
       bool odd = SReg & 0x1;
       unsigned Rx = 256 + (SReg >> 1);
@@ -209,12 +265,13 @@ void ARMAsmPrinter::EmitDwarfRegOp(const MachineLocation &MLoc) const {
     } else if (Reg >= ARM::Q0 && Reg <= ARM::Q15) {
       assert(ARM::Q0 + 15 == ARM::Q15 && "Unexpected ARM Q register numbering");
       // Q registers Q0-Q15 are described by composing two D registers together.
-      // Qx = DW_OP_regx(256+2x) DW_OP_piece(8) DW_OP_regx(256+2x+1) DW_OP_piece(8)
+      // Qx = DW_OP_regx(256+2x) DW_OP_piece(8) DW_OP_regx(256+2x+1)
+      // DW_OP_piece(8)
 
       unsigned QReg = Reg - ARM::Q0;
       unsigned D1 = 256 + 2 * QReg;
       unsigned D2 = D1 + 1;
-      
+
       OutStreamer.AddComment("DW_OP_regx for Q register: D1");
       EmitInt8(dwarf::DW_OP_regx);
       EmitULEB128(D1);
@@ -233,6 +290,8 @@ void ARMAsmPrinter::EmitDwarfRegOp(const MachineLocation &MLoc) const {
 }
 
 void ARMAsmPrinter::EmitFunctionEntryLabel() {
+  OutStreamer.ForceCodeRegion();
+
   if (AFI->isThumbFunction()) {
     OutStreamer.EmitAssemblerFlag(MCAF_Code16);
     OutStreamer.EmitThumbFunc(CurrentFnSym);
@@ -395,16 +454,16 @@ bool ARMAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
       // This takes advantage of the 2 operand-ness of ldm/stm and that we've
       // already got the operands in registers that are operands to the
       // inline asm statement.
-      
+
       O << "{" << ARMInstPrinter::getRegisterName(RegBegin);
-      
+
       // FIXME: The register allocator not only may not have given us the
       // registers in sequence, but may not be in ascending registers. This
       // will require changes in the register allocator that'll need to be
       // propagated down here if the operands change.
       unsigned RegOps = OpNum + 1;
       while (MI->getOperand(RegOps).isReg()) {
-        O << ", " 
+        O << ", "
           << ARMInstPrinter::getRegisterName(MI->getOperand(RegOps).getReg());
         RegOps++;
       }
@@ -413,14 +472,34 @@ bool ARMAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 
       return false;
     }
+    case 'R': // The most significant register of a pair.
+    case 'Q': { // The least significant register of a pair.
+      if (OpNum == 0)
+        return true;
+      const MachineOperand &FlagsOP = MI->getOperand(OpNum - 1);
+      if (!FlagsOP.isImm())
+        return true;
+      unsigned Flags = FlagsOP.getImm();
+      unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
+      if (NumVals != 2)
+        return true;
+      unsigned RegOp = ExtraCode[0] == 'Q' ? OpNum : OpNum + 1;
+      if (RegOp >= MI->getNumOperands())
+        return true;
+      const MachineOperand &MO = MI->getOperand(RegOp);
+      if (!MO.isReg())
+        return true;
+      unsigned Reg = MO.getReg();
+      O << ARMInstPrinter::getRegisterName(Reg);
+      return false;
+    }
+
     // These modifiers are not yet supported.
     case 'p': // The high single-precision register of a VFP double-precision
               // register.
     case 'e': // The low doubleword register of a NEON quad register.
     case 'f': // The high doubleword register of a NEON quad register.
     case 'h': // A range of VFP/NEON registers suitable for VLD1/VST1.
-    case 'Q': // The least significant register of a pair.
-    case 'R': // The most significant register of a pair.
     case 'H': // The highest-numbered register of a pair.
       return true;
     }
@@ -437,7 +516,7 @@ bool ARMAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
   // Does this asm operand have a single letter operand modifier?
   if (ExtraCode && ExtraCode[0]) {
     if (ExtraCode[1] != 0) return true; // Unknown modifier.
-    
+
     switch (ExtraCode[0]) {
       case 'A': // A memory operand for a VLD1/VST1 instruction.
       default: return true;  // Unknown modifier.
@@ -448,7 +527,7 @@ bool ARMAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
         return false;
     }
   }
-  
+
   const MachineOperand &MO = MI->getOperand(OpNum);
   assert(MO.isReg() && "unexpected inline asm memory operand");
   O << "[" << ARMInstPrinter::getRegisterName(MO.getReg()) << "]";
@@ -772,13 +851,19 @@ EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
     OS << MAI->getPrivateGlobalPrefix() << "_LSDA_" << getFunctionNumber();
     MCSym = OutContext.GetOrCreateSymbol(OS.str());
   } else if (ACPV->isBlockAddress()) {
-    MCSym = GetBlockAddressSymbol(ACPV->getBlockAddress());
+    const BlockAddress *BA =
+      cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress();
+    MCSym = GetBlockAddressSymbol(BA);
   } else if (ACPV->isGlobalValue()) {
-    const GlobalValue *GV = ACPV->getGV();
+    const GlobalValue *GV = cast<ARMConstantPoolConstant>(ACPV)->getGV();
     MCSym = GetARMGVSymbol(GV);
+  } else if (ACPV->isMachineBasicBlock()) {
+    const MachineBasicBlock *MBB = cast<ARMConstantPoolMBB>(ACPV)->getMBB();
+    MCSym = MBB->getSymbol();
   } else {
     assert(ACPV->isExtSymbol() && "unrecognized constant pool value");
-    MCSym = GetExternalSymbolSymbol(ACPV->getSymbol());
+    const char *Sym = cast<ARMConstantPoolSymbol>(ACPV)->getSymbol();
+    MCSym = GetExternalSymbolSymbol(Sym);
   }
 
   // Create an MCSymbol for the reference.
@@ -822,6 +907,9 @@ void ARMAsmPrinter::EmitJumpTable(const MachineInstr *MI) {
   const MachineOperand &MO2 = MI->getOperand(OpNum+1); // Unique Id
   unsigned JTI = MO1.getIndex();
 
+  // Tag the jump table appropriately for precise disassembly.
+  OutStreamer.EmitJumpTable32Region();
+
   // Emit a label for the jump table.
   MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel2(JTI, MO2.getImm());
   OutStreamer.EmitLabel(JTISymbol);
@@ -847,6 +935,11 @@ void ARMAsmPrinter::EmitJumpTable(const MachineInstr *MI) {
       Expr = MCBinaryExpr::CreateSub(Expr, MCSymbolRefExpr::Create(JTISymbol,
                                                                    OutContext),
                                      OutContext);
+    // If we're generating a table of Thumb addresses in static relocation
+    // model, we need to add one to keep interworking correctly.
+    else if (AFI->isThumbFunction())
+      Expr = MCBinaryExpr::CreateAdd(Expr, MCConstantExpr::Create(1,OutContext),
+                                     OutContext);
     OutStreamer.EmitValue(Expr, 4);
   }
 }
@@ -859,6 +952,14 @@ void ARMAsmPrinter::EmitJump2Table(const MachineInstr *MI) {
   unsigned JTI = MO1.getIndex();
 
   // Emit a label for the jump table.
+  if (MI->getOpcode() == ARM::t2TBB_JT) {
+    OutStreamer.EmitJumpTable8Region();
+  } else if (MI->getOpcode() == ARM::t2TBH_JT) {
+    OutStreamer.EmitJumpTable16Region();
+  } else {
+    OutStreamer.EmitJumpTable32Region();
+  }
+
   MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel2(JTI, MO2.getImm());
   OutStreamer.EmitLabel(JTISymbol);
 
@@ -881,6 +982,8 @@ void ARMAsmPrinter::EmitJump2Table(const MachineInstr *MI) {
       MCInst BrInst;
       BrInst.setOpcode(ARM::t2B);
       BrInst.addOperand(MCOperand::CreateExpr(MBBSymbolExpr));
+      BrInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
+      BrInst.addOperand(MCOperand::CreateReg(0));
       OutStreamer.EmitInstruction(BrInst);
       continue;
     }
@@ -994,7 +1097,8 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
            i != NumOps; ++i)
         RegList.push_back(MI->getOperand(i).getReg());
       break;
-    case ARM::STR_PRE:
+    case ARM::STR_PRE_IMM:
+    case ARM::STR_PRE_REG:
       assert(MI->getOperand(2).getReg() == ARM::SP &&
              "Only stack pointer as a source reg is supported");
       RegList.push_back(SrcReg);
@@ -1074,10 +1178,20 @@ extern cl::opt<bool> EnableARMEHABI;
 #include "ARMGenMCPseudoLowering.inc"
 
 void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
+  if (MI->getOpcode() != ARM::CONSTPOOL_ENTRY)
+    OutStreamer.EmitCodeRegion();
+
+  // Emit unwinding stuff for frame-related instructions
+  if (EnableARMEHABI && MI->getFlag(MachineInstr::FrameSetup))
+    EmitUnwindingInstruction(MI);
+
   // Do any auto-generated pseudo lowerings.
   if (emitPseudoExpansionLowering(OutStreamer, MI))
     return;
 
+  assert(!convertAddSubFlagsOpcode(MI->getOpcode()) &&
+         "Pseudo flag setting opcode should be expanded early");
+
   // Check for manual lowerings.
   unsigned Opc = MI->getOpcode();
   switch (Opc) {
@@ -1372,6 +1486,10 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     unsigned CPIdx   = (unsigned)MI->getOperand(1).getIndex();
 
     EmitAlignment(2);
+
+    // Mark the constant pool entry as data if we're not already in a data
+    // region.
+    OutStreamer.EmitDataRegion();
     OutStreamer.EmitLabel(GetCPISymbol(LabelId));
 
     const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPIdx];
@@ -1379,7 +1497,6 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       EmitMachineConstantPoolValue(MCPE.Val.MachineCPVal);
     else
       EmitGlobalConstant(MCPE.Val.ConstVal);
-
     return;
   }
   case ARM::t2BR_JT: {
@@ -1590,6 +1707,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       MCInst TmpInst;
       TmpInst.setOpcode(ARM::tB);
       TmpInst.addOperand(MCOperand::CreateExpr(SymbolExpr));
+      TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
+      TmpInst.addOperand(MCOperand::CreateReg(0));
       OutStreamer.EmitInstruction(TmpInst);
     }
     {
@@ -1804,10 +1923,6 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   MCInst TmpInst;
   LowerARMMachineInstrToMCInst(MI, TmpInst, *this);
 
-  // Emit unwinding stuff for frame-related instructions
-  if (EnableARMEHABI && MI->getFlag(MachineInstr::FrameSetup))
-    EmitUnwindingInstruction(MI);
-
   OutStreamer.EmitInstruction(TmpInst);
 }
 
@@ -1815,20 +1930,9 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 // Target Registry Stuff
 //===----------------------------------------------------------------------===//
 
-static MCInstPrinter *createARMMCInstPrinter(const Target &T,
-                                             unsigned SyntaxVariant,
-                                             const MCAsmInfo &MAI) {
-  if (SyntaxVariant == 0)
-    return new ARMInstPrinter(MAI);
-  return 0;
-}
-
 // Force static initialization.
 extern "C" void LLVMInitializeARMAsmPrinter() {
   RegisterAsmPrinter<ARMAsmPrinter> X(TheARMTarget);
   RegisterAsmPrinter<ARMAsmPrinter> Y(TheThumbTarget);
-
-  TargetRegistry::RegisterMCInstPrinter(TheARMTarget, createARMMCInstPrinter);
-  TargetRegistry::RegisterMCInstPrinter(TheThumbTarget, createARMMCInstPrinter);
 }
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 649bd7d5ce3f..408edfc20d4c 100644
--- a/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -13,11 +13,11 @@
 
 #include "ARMBaseInstrInfo.h"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMConstantPoolValue.h"
 #include "ARMHazardRecognizer.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMRegisterInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/Constants.h"
 #include "llvm/Function.h"
 #include "llvm/GlobalValue.h"
@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/CommandLine.h"
@@ -45,6 +46,10 @@ static cl::opt<bool>
 EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
                cl::desc("Enable ARM 2-addr to 3-addr conv"));
 
+static cl::opt<bool>
+WidenVMOVS("widen-vmovs", cl::Hidden,
+           cl::desc("Widen ARM vmovs to vmovd when possible"));
+
 /// ARM_MLxEntry - Record information about MLA / MLS instructions.
 struct ARM_MLxEntry {
   unsigned MLxOpc;     // MLA / MLS opcode
@@ -171,7 +176,7 @@ ARMBaseInstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm);
       unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt);
       UpdateMI = BuildMI(MF, MI->getDebugLoc(),
-                         get(isSub ? ARM::SUBrs : ARM::ADDrs), WBReg)
+                         get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg)
         .addReg(BaseReg).addReg(OffReg).addReg(0).addImm(SOOpc)
         .addImm(Pred).addReg(0).addReg(0);
     } else
@@ -399,6 +404,7 @@ ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
     ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
   int BccOpc = !AFI->isThumbFunction()
     ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
+  bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
 
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
@@ -406,9 +412,12 @@ ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
          "ARM branch conditions have two components!");
 
   if (FBB == 0) {
-    if (Cond.empty()) // Unconditional branch?
-      BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
-    else
+    if (Cond.empty()) { // Unconditional branch?
+      if (isThumb)
+        BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).addImm(ARMCC::AL).addReg(0);
+      else
+        BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
+    } else
       BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB)
         .addImm(Cond[0].getImm()).addReg(Cond[1].getReg());
     return 1;
@@ -417,7 +426,10 @@ ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
   // Two-way conditional branch.
   BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB)
     .addImm(Cond[0].getImm()).addReg(Cond[1].getReg());
-  BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
+  if (isThumb)
+    BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).addImm(ARMCC::AL).addReg(0);
+  else
+    BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
   return 2;
 }
 
@@ -627,7 +639,7 @@ void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   bool SPRDest = ARM::SPRRegClass.contains(DestReg);
   bool SPRSrc  = ARM::SPRRegClass.contains(SrcReg);
 
-  unsigned Opc;
+  unsigned Opc = 0;
   if (SPRDest && SPRSrc)
     Opc = ARM::VMOVS;
   else if (GPRDest && SPRSrc)
@@ -638,19 +650,40 @@ void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     Opc = ARM::VMOVD;
   else if (ARM::QPRRegClass.contains(DestReg, SrcReg))
     Opc = ARM::VORRq;
-  else if (ARM::QQPRRegClass.contains(DestReg, SrcReg))
-    Opc = ARM::VMOVQQ;
-  else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg))
-    Opc = ARM::VMOVQQQQ;
-  else
-    llvm_unreachable("Impossible reg-to-reg copy");
 
-  MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
-  MIB.addReg(SrcReg, getKillRegState(KillSrc));
-  if (Opc == ARM::VORRq)
+  if (Opc) {
+    MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
     MIB.addReg(SrcReg, getKillRegState(KillSrc));
-  if (Opc != ARM::VMOVQQ && Opc != ARM::VMOVQQQQ)
+    if (Opc == ARM::VORRq)
+      MIB.addReg(SrcReg, getKillRegState(KillSrc));
     AddDefaultPred(MIB);
+    return;
+  }
+
+  // Generate instructions for VMOVQQ and VMOVQQQQ pseudos in place.
+  if (ARM::QQPRRegClass.contains(DestReg, SrcReg) ||
+      ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) {
+    const TargetRegisterInfo *TRI = &getRegisterInfo();
+    assert(ARM::qsub_0 + 3 == ARM::qsub_3 && "Expected contiguous enum.");
+    unsigned EndSubReg = ARM::QQPRRegClass.contains(DestReg, SrcReg) ?
+      ARM::qsub_1 : ARM::qsub_3;
+    for (unsigned i = ARM::qsub_0, e = EndSubReg + 1; i != e; ++i) {
+      unsigned Dst = TRI->getSubReg(DestReg, i);
+      unsigned Src = TRI->getSubReg(SrcReg, i);
+      MachineInstrBuilder Mov =
+        AddDefaultPred(BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VORRq))
+                       .addReg(Dst, RegState::Define)
+                       .addReg(Src, getKillRegState(KillSrc))
+                       .addReg(Src, getKillRegState(KillSrc)));
+      if (i == EndSubReg) {
+        Mov->addRegisterDefined(DestReg, TRI);
+        if (KillSrc)
+          Mov->addRegisterKilled(SrcReg, TRI);
+      }
+    }
+    return;
+  }
+  llvm_unreachable("Impossible reg-to-reg copy");
 }
 
 static const
@@ -683,82 +716,84 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                             MFI.getObjectSize(FI),
                             Align);
 
-  // tGPR is used sometimes in ARM instructions that need to avoid using
-  // certain registers.  Just treat it as GPR here. Likewise, rGPR.
-  if (RC == ARM::tGPRRegisterClass || RC == ARM::tcGPRRegisterClass
-      || RC == ARM::rGPRRegisterClass)
-    RC = ARM::GPRRegisterClass;
-
-  switch (RC->getID()) {
-  case ARM::GPRRegClassID:
-    AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STRi12))
+  switch (RC->getSize()) {
+    case 4:
+      if (ARM::GPRRegClass.hasSubClassEq(RC)) {
+        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STRi12))
                    .addReg(SrcReg, getKillRegState(isKill))
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
-    break;
-  case ARM::SPRRegClassID:
-    AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRS))
+      } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
+        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRS))
                    .addReg(SrcReg, getKillRegState(isKill))
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
-    break;
-  case ARM::DPRRegClassID:
-  case ARM::DPR_VFP2RegClassID:
-  case ARM::DPR_8RegClassID:
-    AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRD))
+      } else
+        llvm_unreachable("Unknown reg class!");
+      break;
+    case 8:
+      if (ARM::DPRRegClass.hasSubClassEq(RC)) {
+        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRD))
                    .addReg(SrcReg, getKillRegState(isKill))
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
-    break;
-  case ARM::QPRRegClassID:
-  case ARM::QPR_VFP2RegClassID:
-  case ARM::QPR_8RegClassID:
-    if (Align >= 16 && getRegisterInfo().needsStackRealignment(MF)) {
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1q64Pseudo))
+      } else
+        llvm_unreachable("Unknown reg class!");
+      break;
+    case 16:
+      if (ARM::QPRRegClass.hasSubClassEq(RC)) {
+        if (Align >= 16 && getRegisterInfo().needsStackRealignment(MF)) {
+          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1q64Pseudo))
                      .addFrameIndex(FI).addImm(16)
                      .addReg(SrcReg, getKillRegState(isKill))
                      .addMemOperand(MMO));
-    } else {
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMQIA))
+        } else {
+          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMQIA))
                      .addReg(SrcReg, getKillRegState(isKill))
                      .addFrameIndex(FI)
                      .addMemOperand(MMO));
-    }
-    break;
-  case ARM::QQPRRegClassID:
-  case ARM::QQPR_VFP2RegClassID:
-    if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
-      // FIXME: It's possible to only store part of the QQ register if the
-      // spilled def has a sub-register index.
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1d64QPseudo))
+        }
+      } else
+        llvm_unreachable("Unknown reg class!");
+      break;
+    case 32:
+      if (ARM::QQPRRegClass.hasSubClassEq(RC)) {
+        if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
+          // FIXME: It's possible to only store part of the QQ register if the
+          // spilled def has a sub-register index.
+          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1d64QPseudo))
                      .addFrameIndex(FI).addImm(16)
                      .addReg(SrcReg, getKillRegState(isKill))
                      .addMemOperand(MMO));
-    } else {
-      MachineInstrBuilder MIB =
-        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
+        } else {
+          MachineInstrBuilder MIB =
+          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
                        .addFrameIndex(FI))
-        .addMemOperand(MMO);
-      MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
-      MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
-      MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
-            AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
-    }
-    break;
-  case ARM::QQQQPRRegClassID: {
-    MachineInstrBuilder MIB =
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
-                     .addFrameIndex(FI))
-      .addMemOperand(MMO);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
-    MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
-          AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
-    break;
-  }
-  default:
-    llvm_unreachable("Unknown regclass!");
+                       .addMemOperand(MMO);
+          MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
+          MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
+          MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
+                AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
+        }
+      } else
+        llvm_unreachable("Unknown reg class!");
+      break;
+    case 64:
+      if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
+        MachineInstrBuilder MIB =
+          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
+                         .addFrameIndex(FI))
+                         .addMemOperand(MMO);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
+        MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
+              AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
+      } else
+        llvm_unreachable("Unknown reg class!");
+      break;
+    default:
+      llvm_unreachable("Unknown reg class!");
   }
 }
 
@@ -809,6 +844,12 @@ ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr *MI,
   return 0;
 }
 
+unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr *MI,
+                                                    int &FrameIndex) const {
+  const MachineMemOperand *Dummy;
+  return MI->getDesc().mayStore() && hasStoreToStackSlot(MI, Dummy, FrameIndex);
+}
+
 void ARMBaseInstrInfo::
 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
@@ -826,72 +867,77 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                             MFI.getObjectSize(FI),
                             Align);
 
-  // tGPR is used sometimes in ARM instructions that need to avoid using
-  // certain registers.  Just treat it as GPR here.
-  if (RC == ARM::tGPRRegisterClass || RC == ARM::tcGPRRegisterClass
-      || RC == ARM::rGPRRegisterClass)
-    RC = ARM::GPRRegisterClass;
+  switch (RC->getSize()) {
+  case 4:
+    if (ARM::GPRRegClass.hasSubClassEq(RC)) {
+      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
+                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
 
-  switch (RC->getID()) {
-  case ARM::GPRRegClassID:
-    AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
+    } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
+      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
+    } else
+      llvm_unreachable("Unknown reg class!");
     break;
-  case ARM::SPRRegClassID:
-    AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
+  case 8:
+    if (ARM::DPRRegClass.hasSubClassEq(RC)) {
+      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
+    } else
+      llvm_unreachable("Unknown reg class!");
     break;
-  case ARM::DPRRegClassID:
-  case ARM::DPR_VFP2RegClassID:
-  case ARM::DPR_8RegClassID:
-    AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
-                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
-    break;
-  case ARM::QPRRegClassID:
-  case ARM::QPR_VFP2RegClassID:
-  case ARM::QPR_8RegClassID:
-    if (Align >= 16 && getRegisterInfo().needsStackRealignment(MF)) {
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1q64Pseudo), DestReg)
+  case 16:
+    if (ARM::QPRRegClass.hasSubClassEq(RC)) {
+      if (Align >= 16 && getRegisterInfo().needsStackRealignment(MF)) {
+        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1q64Pseudo), DestReg)
                      .addFrameIndex(FI).addImm(16)
                      .addMemOperand(MMO));
-    } else {
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
-                     .addFrameIndex(FI)
-                     .addMemOperand(MMO));
-    }
+      } else {
+        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
+                       .addFrameIndex(FI)
+                       .addMemOperand(MMO));
+      }
+    } else
+      llvm_unreachable("Unknown reg class!");
     break;
-  case ARM::QQPRRegClassID:
-  case ARM::QQPR_VFP2RegClassID:
-    if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
+  case 32:
+    if (ARM::QQPRRegClass.hasSubClassEq(RC)) {
+      if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
+        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
                      .addFrameIndex(FI).addImm(16)
                      .addMemOperand(MMO));
-    } else {
-      MachineInstrBuilder MIB =
+      } else {
+        MachineInstrBuilder MIB =
         AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
                        .addFrameIndex(FI))
-        .addMemOperand(MMO);
+                       .addMemOperand(MMO);
+        MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::Define, TRI);
+        MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::Define, TRI);
+        MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::Define, TRI);
+        MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::Define, TRI);
+        MIB.addReg(DestReg, RegState::Define | RegState::Implicit);
+      }
+    } else
+      llvm_unreachable("Unknown reg class!");
+    break;
+  case 64:
+    if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
+      MachineInstrBuilder MIB =
+      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
+                     .addFrameIndex(FI))
+                     .addMemOperand(MMO);
       MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::Define, TRI);
       MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::Define, TRI);
       MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::Define, TRI);
-            AddDReg(MIB, DestReg, ARM::dsub_3, RegState::Define, TRI);
-    }
+      MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::Define, TRI);
+      MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::Define, TRI);
+      MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::Define, TRI);
+      MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::Define, TRI);
+      MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::Define, TRI);
+      MIB.addReg(DestReg, RegState::Define | RegState::Implicit);
+    } else
+      llvm_unreachable("Unknown reg class!");
     break;
-  case ARM::QQQQPRRegClassID: {
-    MachineInstrBuilder MIB =
-      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
-                     .addFrameIndex(FI))
-      .addMemOperand(MMO);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::Define, TRI);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::Define, TRI);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::Define, TRI);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::Define, TRI);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::Define, TRI);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::Define, TRI);
-    MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::Define, TRI);
-    AddDReg(MIB, DestReg, ARM::dsub_7, RegState::Define, TRI);
-    break;
-  }
   default:
     llvm_unreachable("Unknown regclass!");
   }
@@ -944,6 +990,78 @@ ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
   return 0;
 }
 
+unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr *MI,
+                                             int &FrameIndex) const {
+  const MachineMemOperand *Dummy;
+  return MI->getDesc().mayLoad() && hasLoadFromStackSlot(MI, Dummy, FrameIndex);
+}
+
+bool ARMBaseInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const{
+  // This hook gets to expand COPY instructions before they become
+  // copyPhysReg() calls.  Look for VMOVS instructions that can legally be
+  // widened to VMOVD.  We prefer the VMOVD when possible because it may be
+  // changed into a VORR that can go down the NEON pipeline.
+  if (!WidenVMOVS || !MI->isCopy())
+    return false;
+
+  // Look for a copy between even S-registers.  That is where we keep floats
+  // when using NEON v2f32 instructions for f32 arithmetic.
+  unsigned DstRegS = MI->getOperand(0).getReg();
+  unsigned SrcRegS = MI->getOperand(1).getReg();
+  if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS))
+    return false;
+
+  const TargetRegisterInfo *TRI = &getRegisterInfo();
+  unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0,
+                                              &ARM::DPRRegClass);
+  unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0,
+                                              &ARM::DPRRegClass);
+  if (!DstRegD || !SrcRegD)
+    return false;
+
+  // We want to widen this into a DstRegD = VMOVD SrcRegD copy.  This is only
+  // legal if the COPY already defines the full DstRegD, and it isn't a
+  // sub-register insertion.
+  if (!MI->definesRegister(DstRegD, TRI) || MI->readsRegister(DstRegD, TRI))
+    return false;
+
+  // A dead copy shouldn't show up here, but reject it just in case.
+  if (MI->getOperand(0).isDead())
+    return false;
+
+  // All clear, widen the COPY.
+  DEBUG(dbgs() << "widening:    " << *MI);
+
+  // Get rid of the old <imp-def> of DstRegD.  Leave it if it defines a Q-reg
+  // or some other super-register.
+  int ImpDefIdx = MI->findRegisterDefOperandIdx(DstRegD);
+  if (ImpDefIdx != -1)
+    MI->RemoveOperand(ImpDefIdx);
+
+  // Change the opcode and operands.
+  MI->setDesc(get(ARM::VMOVD));
+  MI->getOperand(0).setReg(DstRegD);
+  MI->getOperand(1).setReg(SrcRegD);
+  AddDefaultPred(MachineInstrBuilder(MI));
+
+  // We are now reading SrcRegD instead of SrcRegS.  This may upset the
+  // register scavenger and machine verifier, so we need to indicate that we
+  // are reading an undefined value from SrcRegD, but a proper value from
+  // SrcRegS.
+  MI->getOperand(1).setIsUndef();
+  MachineInstrBuilder(MI).addReg(SrcRegS, RegState::Implicit);
+
+  // SrcRegD may actually contain an unrelated value in the ssub_1
+  // sub-register.  Don't kill it.  Only kill the ssub_0 sub-register.
+  if (MI->getOperand(1).isKill()) {
+    MI->getOperand(1).setIsKill(false);
+    MI->addRegisterKilled(SrcRegS, TRI, true);
+  }
+
+  DEBUG(dbgs() << "replaced by: " << *MI);
+  return true;
+}
+
 MachineInstr*
 ARMBaseInstrInfo::emitFrameIndexDebugValue(MachineFunction &MF,
                                            int FrameIx, uint64_t Offset,
@@ -974,17 +1092,24 @@ static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
   // instructions, so that's probably OK, but is PIC always correct when
   // we get here?
   if (ACPV->isGlobalValue())
-    NewCPV = new ARMConstantPoolValue(ACPV->getGV(), PCLabelId,
-                                      ARMCP::CPValue, 4);
+    NewCPV = ARMConstantPoolConstant::
+      Create(cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId,
+             ARMCP::CPValue, 4);
   else if (ACPV->isExtSymbol())
-    NewCPV = new ARMConstantPoolValue(MF.getFunction()->getContext(),
-                                      ACPV->getSymbol(), PCLabelId, 4);
+    NewCPV = ARMConstantPoolSymbol::
+      Create(MF.getFunction()->getContext(),
+             cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4);
   else if (ACPV->isBlockAddress())
-    NewCPV = new ARMConstantPoolValue(ACPV->getBlockAddress(), PCLabelId,
-                                      ARMCP::CPBlockAddress, 4);
+    NewCPV = ARMConstantPoolConstant::
+      Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId,
+             ARMCP::CPBlockAddress, 4);
   else if (ACPV->isLSDA())
-    NewCPV = new ARMConstantPoolValue(MF.getFunction(), PCLabelId,
-                                      ARMCP::CPLSDA, 4);
+    NewCPV = ARMConstantPoolConstant::Create(MF.getFunction(), PCLabelId,
+                                             ARMCP::CPLSDA, 4);
+  else if (ACPV->isMachineBasicBlock())
+    NewCPV = ARMConstantPoolMBB::
+      Create(MF.getFunction()->getContext(),
+             cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4);
   else
     llvm_unreachable("Unexpected ARM constantpool value type!!");
   CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlignment());
@@ -1289,7 +1414,7 @@ isProfitableToIfCvt(MachineBasicBlock &TMBB,
   // Attempt to estimate the relative costs of predication versus branching.
   unsigned TUnpredCost = Probability.getNumerator() * TCycles;
   TUnpredCost /= Probability.getDenominator();
-    
+
   uint32_t Comp = Probability.getDenominator() - Probability.getNumerator();
   unsigned FUnpredCost = Comp * FCycles;
   FUnpredCost /= Probability.getDenominator();
@@ -1330,6 +1455,57 @@ int llvm::getMatchingCondBranchOpcode(int Opc) {
 }
 
 
+/// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
+/// instruction is encoded with an 'S' bit is determined by the optional CPSR
+/// def operand.
+///
+/// This will go away once we can teach tblgen how to set the optional CPSR def
+/// operand itself.
+struct AddSubFlagsOpcodePair {
+  unsigned PseudoOpc;
+  unsigned MachineOpc;
+};
+
+static AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
+  {ARM::ADDSri, ARM::ADDri},
+  {ARM::ADDSrr, ARM::ADDrr},
+  {ARM::ADDSrsi, ARM::ADDrsi},
+  {ARM::ADDSrsr, ARM::ADDrsr},
+
+  {ARM::SUBSri, ARM::SUBri},
+  {ARM::SUBSrr, ARM::SUBrr},
+  {ARM::SUBSrsi, ARM::SUBrsi},
+  {ARM::SUBSrsr, ARM::SUBrsr},
+
+  {ARM::RSBSri, ARM::RSBri},
+  {ARM::RSBSrr, ARM::RSBrr},
+  {ARM::RSBSrsi, ARM::RSBrsi},
+  {ARM::RSBSrsr, ARM::RSBrsr},
+
+  {ARM::t2ADDSri, ARM::t2ADDri},
+  {ARM::t2ADDSrr, ARM::t2ADDrr},
+  {ARM::t2ADDSrs, ARM::t2ADDrs},
+
+  {ARM::t2SUBSri, ARM::t2SUBri},
+  {ARM::t2SUBSrr, ARM::t2SUBrr},
+  {ARM::t2SUBSrs, ARM::t2SUBrs},
+
+  {ARM::t2RSBSri, ARM::t2RSBri},
+  {ARM::t2RSBSrs, ARM::t2RSBrs},
+};
+
+unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
+  static const int NPairs =
+    sizeof(AddSubFlagsOpcodeMap) / sizeof(AddSubFlagsOpcodePair);
+  for (AddSubFlagsOpcodePair *OpcPair = &AddSubFlagsOpcodeMap[0],
+         *End = &AddSubFlagsOpcodeMap[NPairs]; OpcPair != End; ++OpcPair) {
+    if (OldOpc == OpcPair->PseudoOpc) {
+      return OpcPair->MachineOpc;
+    }
+  }
+  return 0;
+}
+
 void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator &MBBI, DebugLoc dl,
                                unsigned DestReg, unsigned BaseReg, int NumBytes,
@@ -1862,7 +2038,6 @@ ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
   case ARM::STMIB_UPD:
   case ARM::tLDMIA:
   case ARM::tLDMIA_UPD:
-  case ARM::tSTMIA:
   case ARM::tSTMIA_UPD:
   case ARM::tPOP_RET:
   case ARM::tPOP:
@@ -2128,7 +2303,6 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   case ARM::STMDA_UPD:
   case ARM::STMDB_UPD:
   case ARM::STMIB_UPD:
-  case ARM::tSTMIA:
   case ARM::tSTMIA_UPD:
   case ARM::tPOP_RET:
   case ARM::tPOP:
@@ -2567,6 +2741,15 @@ hasLowDefLatency(const InstrItineraryData *ItinData,
   return false;
 }
 
+bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr *MI,
+                                         StringRef &ErrInfo) const {
+  if (convertAddSubFlagsOpcode(MI->getOpcode())) {
+    ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
+    return false;
+  }
+  return true;
+}
+
 bool
 ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
                                      unsigned &AddSubOpc,
@@ -2582,3 +2765,66 @@ ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
   HasLane = Entry.HasLane;
   return true;
 }
+
+//===----------------------------------------------------------------------===//
+// Execution domains.
+//===----------------------------------------------------------------------===//
+//
+// Some instructions go down the NEON pipeline, some go down the VFP pipeline,
+// and some can go down both.  The vmov instructions go down the VFP pipeline,
+// but they can be changed to vorr equivalents that are executed by the NEON
+// pipeline.
+//
+// We use the following execution domain numbering:
+//
+enum ARMExeDomain {
+  ExeGeneric = 0,
+  ExeVFP = 1,
+  ExeNEON = 2
+};
+//
+// Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
+//
+std::pair<uint16_t, uint16_t>
+ARMBaseInstrInfo::getExecutionDomain(const MachineInstr *MI) const {
+  // VMOVD is a VFP instruction, but can be changed to NEON if it isn't
+  // predicated.
+  if (MI->getOpcode() == ARM::VMOVD && !isPredicated(MI))
+    return std::make_pair(ExeVFP, (1<<ExeVFP) | (1<<ExeNEON));
+
+  // No other instructions can be swizzled, so just determine their domain.
+  unsigned Domain = MI->getDesc().TSFlags & ARMII::DomainMask;
+
+  if (Domain & ARMII::DomainNEON)
+    return std::make_pair(ExeNEON, 0);
+
+  // Certain instructions can go either way on Cortex-A8.
+  // Treat them as NEON instructions.
+  if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
+    return std::make_pair(ExeNEON, 0);
+
+  if (Domain & ARMII::DomainVFP)
+    return std::make_pair(ExeVFP, 0);
+
+  return std::make_pair(ExeGeneric, 0);
+}
+
+void
+ARMBaseInstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const {
+  // We only know how to change VMOVD into VORR.
+  assert(MI->getOpcode() == ARM::VMOVD && "Can only swizzle VMOVD");
+  if (Domain != ExeNEON)
+    return;
+
+  // Zap the predicate operands.
+  assert(!isPredicated(MI) && "Cannot predicate a VORRd");
+  MI->RemoveOperand(3);
+  MI->RemoveOperand(2);
+
+  // Change to a VORRd which requires two identical use operands.
+  MI->setDesc(get(ARM::VORRd));
+
+  // Add the extra source operand and new predicates.
+  // This will go before any implicit ops.
+  AddDefaultPred(MachineInstrBuilder(MI).addOperand(MI->getOperand(1)));
+}
diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 507e8974bf7b..0f9f32179a31 100644
--- a/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -27,146 +27,6 @@ namespace llvm {
   class ARMSubtarget;
   class ARMBaseRegisterInfo;
 
-/// ARMII - This namespace holds all of the target specific flags that
-/// instruction info tracks.
-///
-namespace ARMII {
-  enum {
-    //===------------------------------------------------------------------===//
-    // Instruction Flags.
-
-    //===------------------------------------------------------------------===//
-    // This four-bit field describes the addressing mode used.
-    AddrModeMask  = 0x1f, // The AddrMode enums are declared in ARMBaseInfo.h
-
-    // IndexMode - Unindex, pre-indexed, or post-indexed are valid for load
-    // and store ops only.  Generic "updating" flag is used for ld/st multiple.
-    // The index mode enums are declared in ARMBaseInfo.h
-    IndexModeShift = 5,
-    IndexModeMask  = 3 << IndexModeShift,
-
-    //===------------------------------------------------------------------===//
-    // Instruction encoding formats.
-    //
-    FormShift     = 7,
-    FormMask      = 0x3f << FormShift,
-
-    // Pseudo instructions
-    Pseudo        = 0  << FormShift,
-
-    // Multiply instructions
-    MulFrm        = 1  << FormShift,
-
-    // Branch instructions
-    BrFrm         = 2  << FormShift,
-    BrMiscFrm     = 3  << FormShift,
-
-    // Data Processing instructions
-    DPFrm         = 4  << FormShift,
-    DPSoRegFrm    = 5  << FormShift,
-
-    // Load and Store
-    LdFrm         = 6  << FormShift,
-    StFrm         = 7  << FormShift,
-    LdMiscFrm     = 8  << FormShift,
-    StMiscFrm     = 9  << FormShift,
-    LdStMulFrm    = 10 << FormShift,
-
-    LdStExFrm     = 11 << FormShift,
-
-    // Miscellaneous arithmetic instructions
-    ArithMiscFrm  = 12 << FormShift,
-    SatFrm        = 13 << FormShift,
-
-    // Extend instructions
-    ExtFrm        = 14 << FormShift,
-
-    // VFP formats
-    VFPUnaryFrm   = 15 << FormShift,
-    VFPBinaryFrm  = 16 << FormShift,
-    VFPConv1Frm   = 17 << FormShift,
-    VFPConv2Frm   = 18 << FormShift,
-    VFPConv3Frm   = 19 << FormShift,
-    VFPConv4Frm   = 20 << FormShift,
-    VFPConv5Frm   = 21 << FormShift,
-    VFPLdStFrm    = 22 << FormShift,
-    VFPLdStMulFrm = 23 << FormShift,
-    VFPMiscFrm    = 24 << FormShift,
-
-    // Thumb format
-    ThumbFrm      = 25 << FormShift,
-
-    // Miscelleaneous format
-    MiscFrm       = 26 << FormShift,
-
-    // NEON formats
-    NGetLnFrm     = 27 << FormShift,
-    NSetLnFrm     = 28 << FormShift,
-    NDupFrm       = 29 << FormShift,
-    NLdStFrm      = 30 << FormShift,
-    N1RegModImmFrm= 31 << FormShift,
-    N2RegFrm      = 32 << FormShift,
-    NVCVTFrm      = 33 << FormShift,
-    NVDupLnFrm    = 34 << FormShift,
-    N2RegVShLFrm  = 35 << FormShift,
-    N2RegVShRFrm  = 36 << FormShift,
-    N3RegFrm      = 37 << FormShift,
-    N3RegVShFrm   = 38 << FormShift,
-    NVExtFrm      = 39 << FormShift,
-    NVMulSLFrm    = 40 << FormShift,
-    NVTBLFrm      = 41 << FormShift,
-
-    //===------------------------------------------------------------------===//
-    // Misc flags.
-
-    // UnaryDP - Indicates this is a unary data processing instruction, i.e.
-    // it doesn't have a Rn operand.
-    UnaryDP       = 1 << 13,
-
-    // Xform16Bit - Indicates this Thumb2 instruction may be transformed into
-    // a 16-bit Thumb instruction if certain conditions are met.
-    Xform16Bit    = 1 << 14,
-
-    //===------------------------------------------------------------------===//
-    // Code domain.
-    DomainShift   = 15,
-    DomainMask    = 7 << DomainShift,
-    DomainGeneral = 0 << DomainShift,
-    DomainVFP     = 1 << DomainShift,
-    DomainNEON    = 2 << DomainShift,
-    DomainNEONA8  = 4 << DomainShift,
-
-    //===------------------------------------------------------------------===//
-    // Field shifts - such shifts are used to set field while generating
-    // machine instructions.
-    //
-    // FIXME: This list will need adjusting/fixing as the MC code emitter
-    // takes shape and the ARMCodeEmitter.cpp bits go away.
-    ShiftTypeShift = 4,
-
-    M_BitShift     = 5,
-    ShiftImmShift  = 5,
-    ShiftShift     = 7,
-    N_BitShift     = 7,
-    ImmHiShift     = 8,
-    SoRotImmShift  = 8,
-    RegRsShift     = 8,
-    ExtRotImmShift = 10,
-    RegRdLoShift   = 12,
-    RegRdShift     = 12,
-    RegRdHiShift   = 16,
-    RegRnShift     = 16,
-    S_BitShift     = 20,
-    W_BitShift     = 21,
-    AM3_I_BitShift = 22,
-    D_BitShift     = 22,
-    U_BitShift     = 23,
-    P_BitShift     = 24,
-    I_BitShift     = 25,
-    CondShift      = 28
-  };
-}
-
 class ARMBaseInstrInfo : public ARMGenInstrInfo {
   const ARMSubtarget &Subtarget;
 
@@ -241,6 +101,10 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
                                        int &FrameIndex) const;
   virtual unsigned isStoreToStackSlot(const MachineInstr *MI,
                                       int &FrameIndex) const;
+  virtual unsigned isLoadFromStackSlotPostFE(const MachineInstr *MI,
+                                             int &FrameIndex) const;
+  virtual unsigned isStoreToStackSlotPostFE(const MachineInstr *MI,
+                                            int &FrameIndex) const;
 
   virtual void copyPhysReg(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator I, DebugLoc DL,
@@ -259,6 +123,8 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
                                     const TargetRegisterClass *RC,
                                     const TargetRegisterInfo *TRI) const;
 
+  virtual bool expandPostRAPseudo(MachineBasicBlock::iterator MI) const;
+
   virtual MachineInstr *emitFrameIndexDebugValue(MachineFunction &MF,
                                                  int FrameIx,
                                                  uint64_t Offset,
@@ -346,6 +212,12 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
   int getOperandLatency(const InstrItineraryData *ItinData,
                         SDNode *DefNode, unsigned DefIdx,
                         SDNode *UseNode, unsigned UseIdx) const;
+
+  /// VFP/NEON execution domains.
+  std::pair<uint16_t, uint16_t>
+  getExecutionDomain(const MachineInstr *MI) const;
+  void setExecutionDomain(MachineInstr *MI, unsigned Domain) const;
+
 private:
   int getVLDMDefCycle(const InstrItineraryData *ItinData,
                       const MCInstrDesc &DefMCID,
@@ -382,6 +254,9 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
   bool hasLowDefLatency(const InstrItineraryData *ItinData,
                         const MachineInstr *DefMI, unsigned DefIdx) const;
 
+  /// verifyInstruction - Perform target specific instruction verification.
+  bool verifyInstruction(const MachineInstr *MI, StringRef &ErrInfo) const;
+
 private:
   /// Modeling special VFP / NEON fp MLA / MLS hazards.
 
@@ -464,6 +339,12 @@ ARMCC::CondCodes getInstrPredicate(const MachineInstr *MI, unsigned &PredReg);
 
 int getMatchingCondBranchOpcode(int Opc);
 
+
+/// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether
+/// the instruction is encoded with an 'S' bit is determined by the optional
+/// CPSR def operand.
+unsigned convertAddSubFlagsOpcode(unsigned OldOpc);
+
 /// emitARMRegPlusImmediate / emitT2RegPlusImmediate - Emits a series of
 /// instructions to materializea destreg = basereg + immediate in ARM / Thumb2
 /// code.
diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index ba422952ac1a..7c42342229a2 100644
--- a/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -12,13 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMBaseRegisterInfo.h"
 #include "ARMFrameLowering.h"
 #include "ARMInstrInfo.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMSubtarget.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Function.h"
@@ -27,7 +27,6 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/Support/Debug.h"
@@ -57,7 +56,7 @@ EnableBasePointer("arm-use-base-pointer", cl::Hidden, cl::init(true),
 
 ARMBaseRegisterInfo::ARMBaseRegisterInfo(const ARMBaseInstrInfo &tii,
                                          const ARMSubtarget &sti)
-  : ARMGenRegisterInfo(), TII(tii), STI(sti),
+  : ARMGenRegisterInfo(ARM::LR), TII(tii), STI(sti),
     FramePtr((STI.isTargetDarwin() || STI.isThumb()) ? ARM::R7 : ARM::R11),
     BasePtr(ARM::R6) {
 }
@@ -354,7 +353,7 @@ const TargetRegisterClass*
 ARMBaseRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC)
                                                                          const {
   const TargetRegisterClass *Super = RC;
-  TargetRegisterClass::sc_iterator I = RC->superclasses_begin();
+  TargetRegisterClass::sc_iterator I = RC->getSuperClasses();
   do {
     switch (Super->getID()) {
     case ARM::GPRRegClassID:
@@ -375,6 +374,13 @@ ARMBaseRegisterInfo::getPointerRegClass(unsigned Kind) const {
   return ARM::GPRRegisterClass;
 }
 
+const TargetRegisterClass *
+ARMBaseRegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
+  if (RC == &ARM::CCRRegClass)
+    return 0;  // Can't copy CCR registers.
+  return RC;
+}
+
 unsigned
 ARMBaseRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
                                          MachineFunction &MF) const {
@@ -487,19 +493,19 @@ ARMBaseRegisterInfo::getRawAllocationOrder(const TargetRegisterClass *RC,
 
     if (!TFI->hasFP(MF)) {
       if (!STI.isR9Reserved())
-        return ArrayRef<unsigned>(GPREven1);
+        return makeArrayRef(GPREven1);
       else
-        return ArrayRef<unsigned>(GPREven4);
+        return makeArrayRef(GPREven4);
     } else if (FramePtr == ARM::R7) {
       if (!STI.isR9Reserved())
-        return ArrayRef<unsigned>(GPREven2);
+        return makeArrayRef(GPREven2);
       else
-        return ArrayRef<unsigned>(GPREven5);
+        return makeArrayRef(GPREven5);
     } else { // FramePtr == ARM::R11
       if (!STI.isR9Reserved())
-        return ArrayRef<unsigned>(GPREven3);
+        return makeArrayRef(GPREven3);
       else
-        return ArrayRef<unsigned>(GPREven6);
+        return makeArrayRef(GPREven6);
     }
   } else if (HintType == ARMRI::RegPairOdd) {
     if (isPhysicalRegister(HintReg) && getRegisterPairOdd(HintReg, MF) == 0)
@@ -509,19 +515,19 @@ ARMBaseRegisterInfo::getRawAllocationOrder(const TargetRegisterClass *RC,
 
     if (!TFI->hasFP(MF)) {
       if (!STI.isR9Reserved())
-        return ArrayRef<unsigned>(GPROdd1);
+        return makeArrayRef(GPROdd1);
       else
-        return ArrayRef<unsigned>(GPROdd4);
+        return makeArrayRef(GPROdd4);
     } else if (FramePtr == ARM::R7) {
       if (!STI.isR9Reserved())
-        return ArrayRef<unsigned>(GPROdd2);
+        return makeArrayRef(GPROdd2);
       else
-        return ArrayRef<unsigned>(GPROdd5);
+        return makeArrayRef(GPROdd5);
     } else { // FramePtr == ARM::R11
       if (!STI.isR9Reserved())
-        return ArrayRef<unsigned>(GPROdd3);
+        return makeArrayRef(GPROdd3);
       else
-        return ArrayRef<unsigned>(GPROdd6);
+        return makeArrayRef(GPROdd6);
     }
   }
   return RC->getRawAllocationOrder(MF);
@@ -649,10 +655,6 @@ cannotEliminateFrame(const MachineFunction &MF) const {
     || needsStackRealignment(MF);
 }
 
-unsigned ARMBaseRegisterInfo::getRARegister() const {
-  return ARM::LR;
-}
-
 unsigned
 ARMBaseRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
@@ -672,99 +674,54 @@ unsigned ARMBaseRegisterInfo::getEHHandlerRegister() const {
   return 0;
 }
 
-int ARMBaseRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  return ARMGenRegisterInfo::getDwarfRegNumFull(RegNum, 0);
-}
-
-int ARMBaseRegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  return ARMGenRegisterInfo::getLLVMRegNumFull(DwarfRegNo,0);
-}
-
 unsigned ARMBaseRegisterInfo::getRegisterPairEven(unsigned Reg,
                                               const MachineFunction &MF) const {
   switch (Reg) {
   default: break;
   // Return 0 if either register of the pair is a special register.
   // So no R12, etc.
-  case ARM::R1:
-    return ARM::R0;
-  case ARM::R3:
-    return ARM::R2;
-  case ARM::R5:
-    return ARM::R4;
+  case ARM::R1: return ARM::R0;
+  case ARM::R3: return ARM::R2;
+  case ARM::R5: return ARM::R4;
   case ARM::R7:
     return (isReservedReg(MF, ARM::R7) || isReservedReg(MF, ARM::R6))
       ? 0 : ARM::R6;
-  case ARM::R9:
-    return isReservedReg(MF, ARM::R9)  ? 0 :ARM::R8;
-  case ARM::R11:
-    return isReservedReg(MF, ARM::R11) ? 0 : ARM::R10;
+  case ARM::R9: return isReservedReg(MF, ARM::R9)  ? 0 :ARM::R8;
+  case ARM::R11: return isReservedReg(MF, ARM::R11) ? 0 : ARM::R10;
 
-  case ARM::S1:
-    return ARM::S0;
-  case ARM::S3:
-    return ARM::S2;
-  case ARM::S5:
-    return ARM::S4;
-  case ARM::S7:
-    return ARM::S6;
-  case ARM::S9:
-    return ARM::S8;
-  case ARM::S11:
-    return ARM::S10;
-  case ARM::S13:
-    return ARM::S12;
-  case ARM::S15:
-    return ARM::S14;
-  case ARM::S17:
-    return ARM::S16;
-  case ARM::S19:
-    return ARM::S18;
-  case ARM::S21:
-    return ARM::S20;
-  case ARM::S23:
-    return ARM::S22;
-  case ARM::S25:
-    return ARM::S24;
-  case ARM::S27:
-    return ARM::S26;
-  case ARM::S29:
-    return ARM::S28;
-  case ARM::S31:
-    return ARM::S30;
+  case ARM::S1: return ARM::S0;
+  case ARM::S3: return ARM::S2;
+  case ARM::S5: return ARM::S4;
+  case ARM::S7: return ARM::S6;
+  case ARM::S9: return ARM::S8;
+  case ARM::S11: return ARM::S10;
+  case ARM::S13: return ARM::S12;
+  case ARM::S15: return ARM::S14;
+  case ARM::S17: return ARM::S16;
+  case ARM::S19: return ARM::S18;
+  case ARM::S21: return ARM::S20;
+  case ARM::S23: return ARM::S22;
+  case ARM::S25: return ARM::S24;
+  case ARM::S27: return ARM::S26;
+  case ARM::S29: return ARM::S28;
+  case ARM::S31: return ARM::S30;
 
-  case ARM::D1:
-    return ARM::D0;
-  case ARM::D3:
-    return ARM::D2;
-  case ARM::D5:
-    return ARM::D4;
-  case ARM::D7:
-    return ARM::D6;
-  case ARM::D9:
-    return ARM::D8;
-  case ARM::D11:
-    return ARM::D10;
-  case ARM::D13:
-    return ARM::D12;
-  case ARM::D15:
-    return ARM::D14;
-  case ARM::D17:
-    return ARM::D16;
-  case ARM::D19:
-    return ARM::D18;
-  case ARM::D21:
-    return ARM::D20;
-  case ARM::D23:
-    return ARM::D22;
-  case ARM::D25:
-    return ARM::D24;
-  case ARM::D27:
-    return ARM::D26;
-  case ARM::D29:
-    return ARM::D28;
-  case ARM::D31:
-    return ARM::D30;
+  case ARM::D1: return ARM::D0;
+  case ARM::D3: return ARM::D2;
+  case ARM::D5: return ARM::D4;
+  case ARM::D7: return ARM::D6;
+  case ARM::D9: return ARM::D8;
+  case ARM::D11: return ARM::D10;
+  case ARM::D13: return ARM::D12;
+  case ARM::D15: return ARM::D14;
+  case ARM::D17: return ARM::D16;
+  case ARM::D19: return ARM::D18;
+  case ARM::D21: return ARM::D20;
+  case ARM::D23: return ARM::D22;
+  case ARM::D25: return ARM::D24;
+  case ARM::D27: return ARM::D26;
+  case ARM::D29: return ARM::D28;
+  case ARM::D31: return ARM::D30;
   }
 
   return 0;
@@ -776,85 +733,48 @@ unsigned ARMBaseRegisterInfo::getRegisterPairOdd(unsigned Reg,
   default: break;
   // Return 0 if either register of the pair is a special register.
   // So no R12, etc.
-  case ARM::R0:
-    return ARM::R1;
-  case ARM::R2:
-    return ARM::R3;
-  case ARM::R4:
-    return ARM::R5;
+  case ARM::R0: return ARM::R1;
+  case ARM::R2: return ARM::R3;
+  case ARM::R4: return ARM::R5;
   case ARM::R6:
     return (isReservedReg(MF, ARM::R7) || isReservedReg(MF, ARM::R6))
       ? 0 : ARM::R7;
-  case ARM::R8:
-    return isReservedReg(MF, ARM::R9)  ? 0 :ARM::R9;
-  case ARM::R10:
-    return isReservedReg(MF, ARM::R11) ? 0 : ARM::R11;
+  case ARM::R8: return isReservedReg(MF, ARM::R9)  ? 0 :ARM::R9;
+  case ARM::R10: return isReservedReg(MF, ARM::R11) ? 0 : ARM::R11;
 
-  case ARM::S0:
-    return ARM::S1;
-  case ARM::S2:
-    return ARM::S3;
-  case ARM::S4:
-    return ARM::S5;
-  case ARM::S6:
-    return ARM::S7;
-  case ARM::S8:
-    return ARM::S9;
-  case ARM::S10:
-    return ARM::S11;
-  case ARM::S12:
-    return ARM::S13;
-  case ARM::S14:
-    return ARM::S15;
-  case ARM::S16:
-    return ARM::S17;
-  case ARM::S18:
-    return ARM::S19;
-  case ARM::S20:
-    return ARM::S21;
-  case ARM::S22:
-    return ARM::S23;
-  case ARM::S24:
-    return ARM::S25;
-  case ARM::S26:
-    return ARM::S27;
-  case ARM::S28:
-    return ARM::S29;
-  case ARM::S30:
-    return ARM::S31;
+  case ARM::S0: return ARM::S1;
+  case ARM::S2: return ARM::S3;
+  case ARM::S4: return ARM::S5;
+  case ARM::S6: return ARM::S7;
+  case ARM::S8: return ARM::S9;
+  case ARM::S10: return ARM::S11;
+  case ARM::S12: return ARM::S13;
+  case ARM::S14: return ARM::S15;
+  case ARM::S16: return ARM::S17;
+  case ARM::S18: return ARM::S19;
+  case ARM::S20: return ARM::S21;
+  case ARM::S22: return ARM::S23;
+  case ARM::S24: return ARM::S25;
+  case ARM::S26: return ARM::S27;
+  case ARM::S28: return ARM::S29;
+  case ARM::S30: return ARM::S31;
 
-  case ARM::D0:
-    return ARM::D1;
-  case ARM::D2:
-    return ARM::D3;
-  case ARM::D4:
-    return ARM::D5;
-  case ARM::D6:
-    return ARM::D7;
-  case ARM::D8:
-    return ARM::D9;
-  case ARM::D10:
-    return ARM::D11;
-  case ARM::D12:
-    return ARM::D13;
-  case ARM::D14:
-    return ARM::D15;
-  case ARM::D16:
-    return ARM::D17;
-  case ARM::D18:
-    return ARM::D19;
-  case ARM::D20:
-    return ARM::D21;
-  case ARM::D22:
-    return ARM::D23;
-  case ARM::D24:
-    return ARM::D25;
-  case ARM::D26:
-    return ARM::D27;
-  case ARM::D28:
-    return ARM::D29;
-  case ARM::D30:
-    return ARM::D31;
+  case ARM::D0: return ARM::D1;
+  case ARM::D2: return ARM::D3;
+  case ARM::D4: return ARM::D5;
+  case ARM::D6: return ARM::D7;
+  case ARM::D8: return ARM::D9;
+  case ARM::D10: return ARM::D11;
+  case ARM::D12: return ARM::D13;
+  case ARM::D14: return ARM::D15;
+  case ARM::D16: return ARM::D17;
+  case ARM::D18: return ARM::D19;
+  case ARM::D20: return ARM::D21;
+  case ARM::D22: return ARM::D23;
+  case ARM::D24: return ARM::D25;
+  case ARM::D26: return ARM::D27;
+  case ARM::D28: return ARM::D29;
+  case ARM::D30: return ARM::D31;
   }
 
   return 0;
@@ -1111,11 +1031,11 @@ materializeFrameBaseRegister(MachineBasicBlock *MBB,
   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
   MRI.constrainRegClass(BaseReg, TII.getRegClass(MCID, 0, this));
 
-  MachineInstrBuilder MIB = BuildMI(*MBB, Ins, DL, MCID, BaseReg)
-    .addFrameIndex(FrameIdx).addImm(Offset);
+  MachineInstrBuilder MIB = AddDefaultPred(BuildMI(*MBB, Ins, DL, MCID, BaseReg)
+    .addFrameIndex(FrameIdx).addImm(Offset));
 
   if (!AFI->isThumb1OnlyFunction())
-    AddDefaultCC(AddDefaultPred(MIB));
+    AddDefaultCC(MIB);
 }
 
 void
@@ -1143,6 +1063,7 @@ ARMBaseRegisterInfo::resolveFrameIndex(MachineBasicBlock::iterator I,
     Done = rewriteT2FrameIndex(MI, i, BaseReg, Off, TII);
   }
   assert (Done && "Unable to resolve frame index!");
+  (void)Done;
 }
 
 bool ARMBaseRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h b/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h
index b4b4059e7361..fee17ff3c1ca 100644
--- a/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h
+++ b/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h
@@ -33,19 +33,6 @@ namespace ARMRI {
   };
 }
 
-/// isARMLowRegister - Returns true if the register is low register r0-r7.
-///
-static inline bool isARMLowRegister(unsigned Reg) {
-  using namespace ARM;
-  switch (Reg) {
-  case R0:  case R1:  case R2:  case R3:
-  case R4:  case R5:  case R6:  case R7:
-    return true;
-  default:
-    return false;
-  }
-}
-
 /// isARMArea1Register - Returns true if the register is a low register (r0-r7)
 /// or a stack/pc register that we should push/pop.
 static inline bool isARMArea1Register(unsigned Reg, bool isDarwin) {
@@ -129,6 +116,8 @@ class ARMBaseRegisterInfo : public ARMGenRegisterInfo {
                                        unsigned &NewSubIdx) const;
 
   const TargetRegisterClass *getPointerRegClass(unsigned Kind = 0) const;
+  const TargetRegisterClass*
+  getCrossCopyRegClass(const TargetRegisterClass *RC) const;
 
   const TargetRegisterClass*
   getLargestLegalSuperClass(const TargetRegisterClass *RC) const;
@@ -164,7 +153,6 @@ class ARMBaseRegisterInfo : public ARMGenRegisterInfo {
   bool cannotEliminateFrame(const MachineFunction &MF) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
   unsigned getBaseRegister() const { return BasePtr; }
 
@@ -172,9 +160,6 @@ class ARMBaseRegisterInfo : public ARMGenRegisterInfo {
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
 
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
-
   bool isLowRegister(unsigned Reg) const;
 
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMCodeEmitter.cpp b/contrib/llvm/lib/Target/ARM/ARMCodeEmitter.cpp
index d6fca6277501..4148d4ab10e9 100644
--- a/contrib/llvm/lib/Target/ARM/ARMCodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMCodeEmitter.cpp
@@ -14,12 +14,12 @@
 
 #define DEBUG_TYPE "jit"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMConstantPoolValue.h"
 #include "ARMInstrInfo.h"
 #include "ARMRelocations.h"
 #include "ARMSubtarget.h"
 #include "ARMTargetMachine.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Function.h"
@@ -161,11 +161,11 @@ namespace {
     //  are already handled elsewhere. They are placeholders to allow this
     //  encoder to continue to function until the MC encoder is sufficiently
     //  far along that this one can be eliminated entirely.
-    unsigned NEONThumb2DataIPostEncoder(const MachineInstr &MI, unsigned Val) 
+    unsigned NEONThumb2DataIPostEncoder(const MachineInstr &MI, unsigned Val)
       const { return 0; }
-    unsigned NEONThumb2LoadStorePostEncoder(const MachineInstr &MI,unsigned Val) 
+    unsigned NEONThumb2LoadStorePostEncoder(const MachineInstr &MI,unsigned Val)
       const { return 0; }
-    unsigned NEONThumb2DupPostEncoder(const MachineInstr &MI,unsigned Val) 
+    unsigned NEONThumb2DupPostEncoder(const MachineInstr &MI,unsigned Val)
       const { return 0; }
     unsigned VFPThumb2PostEncoder(const MachineInstr&MI, unsigned Val)
       const { return 0; }
@@ -189,13 +189,17 @@ namespace {
       unsigned Op) const { return 0; }
     unsigned getARMBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
+    unsigned getARMBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
+      const { return 0; }
     unsigned getCCOutOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
     unsigned getSOImmOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
     unsigned getT2SOImmOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
-    unsigned getSORegOpValue(const MachineInstr &MI, unsigned Op)
+    unsigned getSORegRegOpValue(const MachineInstr &MI, unsigned Op)
+      const { return 0; }
+    unsigned getSORegImmOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
     unsigned getThumbAddrModeRegRegOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
@@ -203,8 +207,12 @@ namespace {
       const { return 0; }
     unsigned getT2AddrModeImm8OpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
+    unsigned getT2Imm8s4OpValue(const MachineInstr &MI, unsigned Op)
+      const { return 0; }
     unsigned getT2AddrModeImm8s4OpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
+    unsigned getT2AddrModeImm0_1020s4OpValue(const MachineInstr &MI,unsigned Op)
+      const { return 0; }
     unsigned getT2AddrModeImm8OffsetOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
     unsigned getT2AddrModeImm12OffsetOpValue(const MachineInstr &MI,unsigned Op)
@@ -213,10 +221,6 @@ namespace {
       const { return 0; }
     unsigned getT2SORegOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
-    unsigned getRotImmOpValue(const MachineInstr &MI, unsigned Op)
-      const { return 0; }
-    unsigned getImmMinusOneOpValue(const MachineInstr &MI, unsigned Op)
-      const { return 0; }
     unsigned getT2AdrLabelOpValue(const MachineInstr &MI, unsigned Op)
       const { return 0; }
     unsigned getAddrMode6AddressOpValue(const MachineInstr &MI, unsigned Op)
@@ -230,8 +234,6 @@ namespace {
       const { return 0; }
     unsigned getBitfieldInvertedMaskOpValue(const MachineInstr &MI,
                                             unsigned Op) const { return 0; }
-    unsigned getMsbOpValue(const MachineInstr &MI,
-                           unsigned Op) const { return 0; }
     unsigned getSsatBitPosValue(const MachineInstr &MI,
                                 unsigned Op) const { return 0; }
     uint32_t getLdStmModeOpValue(const MachineInstr &MI, unsigned OpIdx)
@@ -268,6 +270,8 @@ namespace {
       const { return 0;}
     uint32_t getAddrMode2OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
       const { return 0;}
+    uint32_t getPostIdxRegOpValue(const MachineInstr &MI, unsigned OpIdx)
+      const { return 0;}
     uint32_t getAddrMode3OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
       const { return 0;}
     uint32_t getAddrMode3OpValue(const MachineInstr &MI, unsigned Op)
@@ -632,15 +636,16 @@ void ARMCodeEmitter::emitConstPoolInstruction(const MachineInstr &MI) {
           << (void*)MCE.getCurrentPCValue() << " " << *ACPV << '\n');
 
     assert(ACPV->isGlobalValue() && "unsupported constant pool value");
-    const GlobalValue *GV = ACPV->getGV();
+    const GlobalValue *GV = cast<ARMConstantPoolConstant>(ACPV)->getGV();
     if (GV) {
       Reloc::Model RelocM = TM.getRelocationModel();
       emitGlobalAddress(GV, ARM::reloc_arm_machine_cp_entry,
                         isa<Function>(GV),
                         Subtarget->GVIsIndirectSymbol(GV, RelocM),
                         (intptr_t)ACPV);
-     } else  {
-      emitExternalSymbolAddress(ACPV->getSymbol(), ARM::reloc_arm_absolute);
+    } else  {
+      const char *Sym = cast<ARMConstantPoolSymbol>(ACPV)->getSymbol();
+      emitExternalSymbolAddress(Sym, ARM::reloc_arm_absolute);
     }
     emitWordLE(0);
   } else {
@@ -983,7 +988,7 @@ unsigned ARMCodeEmitter::getMachineSoImmOpValue(unsigned SoImm) {
 
 unsigned ARMCodeEmitter::getAddrModeSBit(const MachineInstr &MI,
                                          const MCInstrDesc &MCID) const {
-  for (unsigned i = MI.getNumOperands(), e = MCID.getNumOperands(); i >= e; --i){
+  for (unsigned i = MI.getNumOperands(), e = MCID.getNumOperands(); i >= e;--i){
     const MachineOperand &MO = MI.getOperand(i-1);
     if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)
       return 1 << ARMII::S_BitShift;
diff --git a/contrib/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/contrib/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index f45ebdc53500..3e3a4134c704 100644
--- a/contrib/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -15,10 +15,10 @@
 
 #define DEBUG_TYPE "arm-cp-islands"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMInstrInfo.h"
 #include "Thumb2InstrInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
@@ -739,7 +739,11 @@ MachineBasicBlock *ARMConstantIslands::SplitBlockBeforeInstr(MachineInstr *MI) {
   // There doesn't seem to be meaningful DebugInfo available; this doesn't
   // correspond to anything in the source.
   unsigned Opc = isThumb ? (isThumb2 ? ARM::t2B : ARM::tB) : ARM::B;
-  BuildMI(OrigBB, DebugLoc(), TII->get(Opc)).addMBB(NewBB);
+  if (!isThumb)
+    BuildMI(OrigBB, DebugLoc(), TII->get(Opc)).addMBB(NewBB);
+  else
+    BuildMI(OrigBB, DebugLoc(), TII->get(Opc)).addMBB(NewBB)
+            .addImm(ARMCC::AL).addReg(0);
   ++NumSplit;
 
   // Update the CFG.  All succs of OrigBB are now succs of NewBB.
@@ -1151,7 +1155,11 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex,
     // targets will be exchanged, and the altered branch may be out of
     // range, so the machinery has to know about it.
     int UncondBr = isThumb ? ((isThumb2) ? ARM::t2B : ARM::tB) : ARM::B;
-    BuildMI(UserMBB, DebugLoc(), TII->get(UncondBr)).addMBB(NewMBB);
+    if (!isThumb)
+      BuildMI(UserMBB, DebugLoc(), TII->get(UncondBr)).addMBB(NewMBB);
+    else
+      BuildMI(UserMBB, DebugLoc(), TII->get(UncondBr)).addMBB(NewMBB)
+              .addImm(ARMCC::AL).addReg(0);
     unsigned MaxDisp = getUnconditionalBrDisp(UncondBr);
     ImmBranches.push_back(ImmBranch(&UserMBB->back(),
                           MaxDisp, false, UncondBr));
@@ -1512,7 +1520,11 @@ ARMConstantIslands::FixUpConditionalBr(MachineFunction &MF, ImmBranch &Br) {
     .addMBB(NextBB).addImm(CC).addReg(CCReg);
   Br.MI = &MBB->back();
   BBSizes[MBB->getNumber()] += TII->GetInstSizeInBytes(&MBB->back());
-  BuildMI(MBB, DebugLoc(), TII->get(Br.UncondBr)).addMBB(DestBB);
+  if (isThumb)
+    BuildMI(MBB, DebugLoc(), TII->get(Br.UncondBr)).addMBB(DestBB)
+            .addImm(ARMCC::AL).addReg(0);
+  else
+    BuildMI(MBB, DebugLoc(), TII->get(Br.UncondBr)).addMBB(DestBB);
   BBSizes[MBB->getNumber()] += TII->GetInstSizeInBytes(&MBB->back());
   unsigned MaxDisp = getUnconditionalBrDisp(Br.UncondBr);
   ImmBranches.push_back(ImmBranch(&MBB->back(), MaxDisp, false, Br.UncondBr));
@@ -1891,7 +1903,8 @@ AdjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB)
   // There doesn't seem to be meaningful DebugInfo available; this doesn't
   // correspond directly to anything in the source.
   assert (isThumb2 && "Adjusting for TB[BH] but not in Thumb2?");
-  BuildMI(NewBB, DebugLoc(), TII->get(ARM::t2B)).addMBB(BB);
+  BuildMI(NewBB, DebugLoc(), TII->get(ARM::t2B)).addMBB(BB)
+          .addImm(ARMCC::AL).addReg(0);
 
   // Update internal data structures to account for the newly inserted MBB.
   MF.RenumberBlocks(NewBB);
diff --git a/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp b/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp
index 165a1d849ad5..aadfd4779db3 100644
--- a/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp
@@ -17,79 +17,57 @@
 #include "llvm/Constants.h"
 #include "llvm/GlobalValue.h"
 #include "llvm/Type.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdlib>
 using namespace llvm;
 
-ARMConstantPoolValue::ARMConstantPoolValue(const Constant *cval, unsigned id,
-                                           ARMCP::ARMCPKind K,
+//===----------------------------------------------------------------------===//
+// ARMConstantPoolValue
+//===----------------------------------------------------------------------===//
+
+ARMConstantPoolValue::ARMConstantPoolValue(Type *Ty, unsigned id,
+                                           ARMCP::ARMCPKind kind,
                                            unsigned char PCAdj,
-                                           ARMCP::ARMCPModifier Modif,
-                                           bool AddCA)
-  : MachineConstantPoolValue((const Type*)cval->getType()),
-    CVal(cval), S(NULL), LabelId(id), Kind(K), PCAdjust(PCAdj),
-    Modifier(Modif), AddCurrentAddress(AddCA) {}
+                                           ARMCP::ARMCPModifier modifier,
+                                           bool addCurrentAddress)
+  : MachineConstantPoolValue(Ty), LabelId(id), Kind(kind),
+    PCAdjust(PCAdj), Modifier(modifier),
+    AddCurrentAddress(addCurrentAddress) {}
 
-ARMConstantPoolValue::ARMConstantPoolValue(LLVMContext &C,
-                                           const char *s, unsigned id,
+ARMConstantPoolValue::ARMConstantPoolValue(LLVMContext &C, unsigned id,
+                                           ARMCP::ARMCPKind kind,
                                            unsigned char PCAdj,
-                                           ARMCP::ARMCPModifier Modif,
-                                           bool AddCA)
-  : MachineConstantPoolValue((const Type*)Type::getInt32Ty(C)),
-    CVal(NULL), S(strdup(s)), LabelId(id), Kind(ARMCP::CPExtSymbol),
-    PCAdjust(PCAdj), Modifier(Modif), AddCurrentAddress(AddCA) {}
+                                           ARMCP::ARMCPModifier modifier,
+                                           bool addCurrentAddress)
+  : MachineConstantPoolValue((Type*)Type::getInt32Ty(C)),
+    LabelId(id), Kind(kind), PCAdjust(PCAdj), Modifier(modifier),
+    AddCurrentAddress(addCurrentAddress) {}
 
-ARMConstantPoolValue::ARMConstantPoolValue(const GlobalValue *gv,
-                                           ARMCP::ARMCPModifier Modif)
-  : MachineConstantPoolValue((const Type*)Type::getInt32Ty(gv->getContext())),
-    CVal(gv), S(NULL), LabelId(0), Kind(ARMCP::CPValue), PCAdjust(0),
-    Modifier(Modif), AddCurrentAddress(false) {}
+ARMConstantPoolValue::~ARMConstantPoolValue() {}
 
-const GlobalValue *ARMConstantPoolValue::getGV() const {
-  return dyn_cast_or_null<GlobalValue>(CVal);
-}
-
-const BlockAddress *ARMConstantPoolValue::getBlockAddress() const {
-  return dyn_cast_or_null<BlockAddress>(CVal);
-}
-
-static bool CPV_streq(const char *S1, const char *S2) {
-  if (S1 == S2)
-    return true;
-  if (S1 && S2 && strcmp(S1, S2) == 0)
-    return true;
-  return false;
+const char *ARMConstantPoolValue::getModifierText() const {
+  switch (Modifier) {
+  default: llvm_unreachable("Unknown modifier!");
+    // FIXME: Are these case sensitive? It'd be nice to lower-case all the
+    // strings if that's legal.
+  case ARMCP::no_modifier: return "none";
+  case ARMCP::TLSGD:       return "tlsgd";
+  case ARMCP::GOT:         return "GOT";
+  case ARMCP::GOTOFF:      return "GOTOFF";
+  case ARMCP::GOTTPOFF:    return "gottpoff";
+  case ARMCP::TPOFF:       return "tpoff";
+  }
 }
 
 int ARMConstantPoolValue::getExistingMachineCPValue(MachineConstantPool *CP,
                                                     unsigned Alignment) {
-  unsigned AlignMask = Alignment - 1;
-  const std::vector<MachineConstantPoolEntry> Constants = CP->getConstants();
-  for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
-    if (Constants[i].isMachineConstantPoolEntry() &&
-        (Constants[i].getAlignment() & AlignMask) == 0) {
-      ARMConstantPoolValue *CPV =
-        (ARMConstantPoolValue *)Constants[i].Val.MachineCPVal;
-      if (CPV->CVal == CVal &&
-          CPV->LabelId == LabelId &&
-          CPV->PCAdjust == PCAdjust &&
-          CPV_streq(CPV->S, S) &&
-          CPV->Modifier == Modifier)
-        return i;
-    }
-  }
-
+  assert(false && "Shouldn't be calling this directly!");
   return -1;
 }
 
-ARMConstantPoolValue::~ARMConstantPoolValue() {
-  free((void*)S);
-}
-
 void
-ARMConstantPoolValue::AddSelectionDAGCSEId(FoldingSetNodeID &ID) {
-  ID.AddPointer(CVal);
-  ID.AddPointer(S);
+ARMConstantPoolValue::addSelectionDAGCSEId(FoldingSetNodeID &ID) {
   ID.AddInteger(LabelId);
   ID.AddInteger(PCAdjust);
 }
@@ -97,9 +75,7 @@ ARMConstantPoolValue::AddSelectionDAGCSEId(FoldingSetNodeID &ID) {
 bool
 ARMConstantPoolValue::hasSameValue(ARMConstantPoolValue *ACPV) {
   if (ACPV->Kind == Kind &&
-      ACPV->CVal == CVal &&
       ACPV->PCAdjust == PCAdjust &&
-      CPV_streq(ACPV->S, S) &&
       ACPV->Modifier == Modifier) {
     if (ACPV->LabelId == LabelId)
       return true;
@@ -115,12 +91,7 @@ void ARMConstantPoolValue::dump() const {
   errs() << "  " << *this;
 }
 
-
 void ARMConstantPoolValue::print(raw_ostream &O) const {
-  if (CVal)
-    O << CVal->getName();
-  else
-    O << S;
   if (Modifier) O << "(" << getModifierText() << ")";
   if (PCAdjust != 0) {
     O << "-(LPC" << LabelId << "+" << (unsigned)PCAdjust;
@@ -128,3 +99,221 @@ void ARMConstantPoolValue::print(raw_ostream &O) const {
     O << ")";
   }
 }
+
+//===----------------------------------------------------------------------===//
+// ARMConstantPoolConstant
+//===----------------------------------------------------------------------===//
+
+ARMConstantPoolConstant::ARMConstantPoolConstant(Type *Ty,
+                                                 const Constant *C,
+                                                 unsigned ID,
+                                                 ARMCP::ARMCPKind Kind,
+                                                 unsigned char PCAdj,
+                                                 ARMCP::ARMCPModifier Modifier,
+                                                 bool AddCurrentAddress)
+  : ARMConstantPoolValue(Ty, ID, Kind, PCAdj, Modifier, AddCurrentAddress),
+    CVal(C) {}
+
+ARMConstantPoolConstant::ARMConstantPoolConstant(const Constant *C,
+                                                 unsigned ID,
+                                                 ARMCP::ARMCPKind Kind,
+                                                 unsigned char PCAdj,
+                                                 ARMCP::ARMCPModifier Modifier,
+                                                 bool AddCurrentAddress)
+  : ARMConstantPoolValue((Type*)C->getType(), ID, Kind, PCAdj, Modifier,
+                         AddCurrentAddress),
+    CVal(C) {}
+
+ARMConstantPoolConstant *
+ARMConstantPoolConstant::Create(const Constant *C, unsigned ID) {
+  return new ARMConstantPoolConstant(C, ID, ARMCP::CPValue, 0,
+                                     ARMCP::no_modifier, false);
+}
+
+ARMConstantPoolConstant *
+ARMConstantPoolConstant::Create(const GlobalValue *GV,
+                                ARMCP::ARMCPModifier Modifier) {
+  return new ARMConstantPoolConstant((Type*)Type::getInt32Ty(GV->getContext()),
+                                     GV, 0, ARMCP::CPValue, 0,
+                                     Modifier, false);
+}
+
+ARMConstantPoolConstant *
+ARMConstantPoolConstant::Create(const Constant *C, unsigned ID,
+                                ARMCP::ARMCPKind Kind, unsigned char PCAdj) {
+  return new ARMConstantPoolConstant(C, ID, Kind, PCAdj,
+                                     ARMCP::no_modifier, false);
+}
+
+ARMConstantPoolConstant *
+ARMConstantPoolConstant::Create(const Constant *C, unsigned ID,
+                                ARMCP::ARMCPKind Kind, unsigned char PCAdj,
+                                ARMCP::ARMCPModifier Modifier,
+                                bool AddCurrentAddress) {
+  return new ARMConstantPoolConstant(C, ID, Kind, PCAdj, Modifier,
+                                     AddCurrentAddress);
+}
+
+const GlobalValue *ARMConstantPoolConstant::getGV() const {
+  return dyn_cast_or_null<GlobalValue>(CVal);
+}
+
+const BlockAddress *ARMConstantPoolConstant::getBlockAddress() const {
+  return dyn_cast_or_null<BlockAddress>(CVal);
+}
+
+int ARMConstantPoolConstant::getExistingMachineCPValue(MachineConstantPool *CP,
+                                                       unsigned Alignment) {
+  unsigned AlignMask = Alignment - 1;
+  const std::vector<MachineConstantPoolEntry> Constants = CP->getConstants();
+  for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+    if (Constants[i].isMachineConstantPoolEntry() &&
+        (Constants[i].getAlignment() & AlignMask) == 0) {
+      ARMConstantPoolValue *CPV =
+        (ARMConstantPoolValue *)Constants[i].Val.MachineCPVal;
+      ARMConstantPoolConstant *APC = dyn_cast<ARMConstantPoolConstant>(CPV);
+      if (!APC) continue;
+      if (APC->CVal == CVal && equals(APC))
+        return i;
+    }
+  }
+
+  return -1;
+}
+
+bool ARMConstantPoolConstant::hasSameValue(ARMConstantPoolValue *ACPV) {
+  const ARMConstantPoolConstant *ACPC = dyn_cast<ARMConstantPoolConstant>(ACPV);
+  return ACPC && ACPC->CVal == CVal && ARMConstantPoolValue::hasSameValue(ACPV);
+}
+
+void ARMConstantPoolConstant::addSelectionDAGCSEId(FoldingSetNodeID &ID) {
+  ID.AddPointer(CVal);
+  ARMConstantPoolValue::addSelectionDAGCSEId(ID);
+}
+
+void ARMConstantPoolConstant::print(raw_ostream &O) const {
+  O << CVal->getName();
+  ARMConstantPoolValue::print(O);
+}
+
+//===----------------------------------------------------------------------===//
+// ARMConstantPoolSymbol
+//===----------------------------------------------------------------------===//
+
+ARMConstantPoolSymbol::ARMConstantPoolSymbol(LLVMContext &C, const char *s,
+                                             unsigned id,
+                                             unsigned char PCAdj,
+                                             ARMCP::ARMCPModifier Modifier,
+                                             bool AddCurrentAddress)
+  : ARMConstantPoolValue(C, id, ARMCP::CPExtSymbol, PCAdj, Modifier,
+                         AddCurrentAddress),
+    S(strdup(s)) {}
+
+ARMConstantPoolSymbol::~ARMConstantPoolSymbol() {
+  free((void*)S);
+}
+
+ARMConstantPoolSymbol *
+ARMConstantPoolSymbol::Create(LLVMContext &C, const char *s,
+                              unsigned ID, unsigned char PCAdj) {
+  return new ARMConstantPoolSymbol(C, s, ID, PCAdj, ARMCP::no_modifier, false);
+}
+
+static bool CPV_streq(const char *S1, const char *S2) {
+  if (S1 == S2)
+    return true;
+  if (S1 && S2 && strcmp(S1, S2) == 0)
+    return true;
+  return false;
+}
+
+int ARMConstantPoolSymbol::getExistingMachineCPValue(MachineConstantPool *CP,
+                                                     unsigned Alignment) {
+  unsigned AlignMask = Alignment - 1;
+  const std::vector<MachineConstantPoolEntry> Constants = CP->getConstants();
+  for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+    if (Constants[i].isMachineConstantPoolEntry() &&
+        (Constants[i].getAlignment() & AlignMask) == 0) {
+      ARMConstantPoolValue *CPV =
+        (ARMConstantPoolValue *)Constants[i].Val.MachineCPVal;
+      ARMConstantPoolSymbol *APS = dyn_cast<ARMConstantPoolSymbol>(CPV);
+      if (!APS) continue;
+
+      if (CPV_streq(APS->S, S) && equals(APS))
+        return i;
+    }
+  }
+
+  return -1;
+}
+
+bool ARMConstantPoolSymbol::hasSameValue(ARMConstantPoolValue *ACPV) {
+  const ARMConstantPoolSymbol *ACPS = dyn_cast<ARMConstantPoolSymbol>(ACPV);
+  return ACPS && CPV_streq(ACPS->S, S) &&
+    ARMConstantPoolValue::hasSameValue(ACPV);
+}
+
+void ARMConstantPoolSymbol::addSelectionDAGCSEId(FoldingSetNodeID &ID) {
+  ID.AddPointer(S);
+  ARMConstantPoolValue::addSelectionDAGCSEId(ID);
+}
+
+void ARMConstantPoolSymbol::print(raw_ostream &O) const {
+  O << S;
+  ARMConstantPoolValue::print(O);
+}
+
+//===----------------------------------------------------------------------===//
+// ARMConstantPoolMBB
+//===----------------------------------------------------------------------===//
+
+ARMConstantPoolMBB::ARMConstantPoolMBB(LLVMContext &C,
+                                       const MachineBasicBlock *mbb,
+                                       unsigned id, unsigned char PCAdj,
+                                       ARMCP::ARMCPModifier Modifier,
+                                       bool AddCurrentAddress)
+  : ARMConstantPoolValue(C, id, ARMCP::CPMachineBasicBlock, PCAdj,
+                         Modifier, AddCurrentAddress),
+    MBB(mbb) {}
+
+ARMConstantPoolMBB *ARMConstantPoolMBB::Create(LLVMContext &C,
+                                               const MachineBasicBlock *mbb,
+                                               unsigned ID,
+                                               unsigned char PCAdj) {
+  return new ARMConstantPoolMBB(C, mbb, ID, PCAdj, ARMCP::no_modifier, false);
+}
+
+int ARMConstantPoolMBB::getExistingMachineCPValue(MachineConstantPool *CP,
+                                                  unsigned Alignment) {
+  unsigned AlignMask = Alignment - 1;
+  const std::vector<MachineConstantPoolEntry> Constants = CP->getConstants();
+  for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+    if (Constants[i].isMachineConstantPoolEntry() &&
+        (Constants[i].getAlignment() & AlignMask) == 0) {
+      ARMConstantPoolValue *CPV =
+        (ARMConstantPoolValue *)Constants[i].Val.MachineCPVal;
+      ARMConstantPoolMBB *APMBB = dyn_cast<ARMConstantPoolMBB>(CPV);
+      if (!APMBB) continue;
+
+      if (APMBB->MBB == MBB && equals(APMBB))
+        return i;
+    }
+  }
+
+  return -1;
+}
+
+bool ARMConstantPoolMBB::hasSameValue(ARMConstantPoolValue *ACPV) {
+  const ARMConstantPoolMBB *ACPMBB = dyn_cast<ARMConstantPoolMBB>(ACPV);
+  return ACPMBB && ACPMBB->MBB == MBB &&
+    ARMConstantPoolValue::hasSameValue(ACPV);
+}
+
+void ARMConstantPoolMBB::addSelectionDAGCSEId(FoldingSetNodeID &ID) {
+  ID.AddPointer(MBB);
+  ARMConstantPoolValue::addSelectionDAGCSEId(ID);
+}
+
+void ARMConstantPoolMBB::print(raw_ostream &O) const {
+  ARMConstantPoolValue::print(O);
+}
diff --git a/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.h b/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.h
index d008811c40e4..0d0def32b7d8 100644
--- a/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.h
+++ b/contrib/llvm/lib/Target/ARM/ARMConstantPoolValue.h
@@ -20,17 +20,19 @@
 
 namespace llvm {
 
-class Constant;
 class BlockAddress;
+class Constant;
 class GlobalValue;
 class LLVMContext;
+class MachineBasicBlock;
 
 namespace ARMCP {
   enum ARMCPKind {
     CPValue,
     CPExtSymbol,
     CPBlockAddress,
-    CPLSDA
+    CPLSDA,
+    CPMachineBasicBlock
   };
 
   enum ARMCPModifier {
@@ -47,8 +49,6 @@ namespace ARMCP {
 /// represent PC-relative displacement between the address of the load
 /// instruction and the constant being loaded, i.e. (&GV-(LPIC+8)).
 class ARMConstantPoolValue : public MachineConstantPoolValue {
-  const Constant *CVal;    // Constant being loaded.
-  const char *S;           // ExtSymbol being loaded.
   unsigned LabelId;        // Label id of the load.
   ARMCP::ARMCPKind Kind;   // Kind of constant.
   unsigned char PCAdjust;  // Extra adjustment if constantpool is pc-relative.
@@ -56,60 +56,54 @@ class ARMConstantPoolValue : public MachineConstantPoolValue {
   ARMCP::ARMCPModifier Modifier;   // GV modifier i.e. (&GV(modifier)-(LPIC+8))
   bool AddCurrentAddress;
 
-public:
-  ARMConstantPoolValue(const Constant *cval, unsigned id,
-                       ARMCP::ARMCPKind Kind = ARMCP::CPValue,
-                       unsigned char PCAdj = 0,
-                       ARMCP::ARMCPModifier Modifier = ARMCP::no_modifier,
-                       bool AddCurrentAddress = false);
-  ARMConstantPoolValue(LLVMContext &C, const char *s, unsigned id,
-                       unsigned char PCAdj = 0,
-                       ARMCP::ARMCPModifier Modifier = ARMCP::no_modifier,
-                       bool AddCurrentAddress = false);
-  ARMConstantPoolValue(const GlobalValue *GV, ARMCP::ARMCPModifier Modifier);
-  ARMConstantPoolValue();
-  ~ARMConstantPoolValue();
+protected:
+  ARMConstantPoolValue(Type *Ty, unsigned id, ARMCP::ARMCPKind Kind,
+                       unsigned char PCAdj, ARMCP::ARMCPModifier Modifier,
+                       bool AddCurrentAddress);
+
+  ARMConstantPoolValue(LLVMContext &C, unsigned id, ARMCP::ARMCPKind Kind,
+                       unsigned char PCAdj, ARMCP::ARMCPModifier Modifier,
+                       bool AddCurrentAddress);
+public:
+  virtual ~ARMConstantPoolValue();
 
-  const GlobalValue *getGV() const;
-  const char *getSymbol() const { return S; }
-  const BlockAddress *getBlockAddress() const;
   ARMCP::ARMCPModifier getModifier() const { return Modifier; }
-  const char *getModifierText() const {
-    switch (Modifier) {
-    default: llvm_unreachable("Unknown modifier!");
-    // FIXME: Are these case sensitive? It'd be nice to lower-case all the
-    // strings if that's legal.
-    case ARMCP::no_modifier: return "none";
-    case ARMCP::TLSGD:       return "tlsgd";
-    case ARMCP::GOT:         return "GOT";
-    case ARMCP::GOTOFF:      return "GOTOFF";
-    case ARMCP::GOTTPOFF:    return "gottpoff";
-    case ARMCP::TPOFF:       return "tpoff";
-    }
-  }
+  const char *getModifierText() const;
   bool hasModifier() const { return Modifier != ARMCP::no_modifier; }
+
   bool mustAddCurrentAddress() const { return AddCurrentAddress; }
+
   unsigned getLabelId() const { return LabelId; }
   unsigned char getPCAdjustment() const { return PCAdjust; }
+
   bool isGlobalValue() const { return Kind == ARMCP::CPValue; }
   bool isExtSymbol() const { return Kind == ARMCP::CPExtSymbol; }
-  bool isBlockAddress() { return Kind == ARMCP::CPBlockAddress; }
-  bool isLSDA() { return Kind == ARMCP::CPLSDA; }
+  bool isBlockAddress() const { return Kind == ARMCP::CPBlockAddress; }
+  bool isLSDA() const { return Kind == ARMCP::CPLSDA; }
+  bool isMachineBasicBlock() const{ return Kind == ARMCP::CPMachineBasicBlock; }
 
   virtual unsigned getRelocationInfo() const { return 2; }
 
   virtual int getExistingMachineCPValue(MachineConstantPool *CP,
                                         unsigned Alignment);
 
-  virtual void AddSelectionDAGCSEId(FoldingSetNodeID &ID);
+  virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID);
 
-  /// hasSameValue - Return true if this ARM constpool value
-  /// can share the same constantpool entry as another ARM constpool value.
-  bool hasSameValue(ARMConstantPoolValue *ACPV);
+  /// hasSameValue - Return true if this ARM constpool value can share the same
+  /// constantpool entry as another ARM constpool value.
+  virtual bool hasSameValue(ARMConstantPoolValue *ACPV);
 
+  bool equals(const ARMConstantPoolValue *A) const {
+    return this->LabelId == A->LabelId &&
+      this->PCAdjust == A->PCAdjust &&
+      this->Modifier == A->Modifier;
+  }
+
+  virtual void print(raw_ostream &O) const;
   void print(raw_ostream *O) const { if (O) print(*O); }
-  void print(raw_ostream &O) const;
   void dump() const;
+
+  static bool classof(const ARMConstantPoolValue *) { return true; }
 };
 
 inline raw_ostream &operator<<(raw_ostream &O, const ARMConstantPoolValue &V) {
@@ -117,6 +111,123 @@ inline raw_ostream &operator<<(raw_ostream &O, const ARMConstantPoolValue &V) {
   return O;
 }
 
+/// ARMConstantPoolConstant - ARM-specific constant pool values for Constants,
+/// Functions, and BlockAddresses.
+class ARMConstantPoolConstant : public ARMConstantPoolValue {
+  const Constant *CVal;         // Constant being loaded.
+
+  ARMConstantPoolConstant(const Constant *C,
+                          unsigned ID,
+                          ARMCP::ARMCPKind Kind,
+                          unsigned char PCAdj,
+                          ARMCP::ARMCPModifier Modifier,
+                          bool AddCurrentAddress);
+  ARMConstantPoolConstant(Type *Ty, const Constant *C,
+                          unsigned ID,
+                          ARMCP::ARMCPKind Kind,
+                          unsigned char PCAdj,
+                          ARMCP::ARMCPModifier Modifier,
+                          bool AddCurrentAddress);
+
+public:
+  static ARMConstantPoolConstant *Create(const Constant *C, unsigned ID);
+  static ARMConstantPoolConstant *Create(const GlobalValue *GV,
+                                         ARMCP::ARMCPModifier Modifier);
+  static ARMConstantPoolConstant *Create(const Constant *C, unsigned ID,
+                                         ARMCP::ARMCPKind Kind,
+                                         unsigned char PCAdj);
+  static ARMConstantPoolConstant *Create(const Constant *C, unsigned ID,
+                                         ARMCP::ARMCPKind Kind,
+                                         unsigned char PCAdj,
+                                         ARMCP::ARMCPModifier Modifier,
+                                         bool AddCurrentAddress);
+
+  const GlobalValue *getGV() const;
+  const BlockAddress *getBlockAddress() const;
+
+  virtual int getExistingMachineCPValue(MachineConstantPool *CP,
+                                        unsigned Alignment);
+
+  /// hasSameValue - Return true if this ARM constpool value can share the same
+  /// constantpool entry as another ARM constpool value.
+  virtual bool hasSameValue(ARMConstantPoolValue *ACPV);
+
+  virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID);
+
+  virtual void print(raw_ostream &O) const;
+  static bool classof(const ARMConstantPoolValue *APV) {
+    return APV->isGlobalValue() || APV->isBlockAddress() || APV->isLSDA();
+  }
+  static bool classof(const ARMConstantPoolConstant *) { return true; }
+};
+
+/// ARMConstantPoolSymbol - ARM-specific constantpool values for external
+/// symbols.
+class ARMConstantPoolSymbol : public ARMConstantPoolValue {
+  const char *S;                // ExtSymbol being loaded.
+
+  ARMConstantPoolSymbol(LLVMContext &C, const char *s, unsigned id,
+                        unsigned char PCAdj, ARMCP::ARMCPModifier Modifier,
+                        bool AddCurrentAddress);
+
+public:
+  ~ARMConstantPoolSymbol();
+
+  static ARMConstantPoolSymbol *Create(LLVMContext &C, const char *s,
+                                       unsigned ID, unsigned char PCAdj);
+
+  const char *getSymbol() const { return S; }
+
+  virtual int getExistingMachineCPValue(MachineConstantPool *CP,
+                                        unsigned Alignment);
+
+  virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID);
+
+  /// hasSameValue - Return true if this ARM constpool value can share the same
+  /// constantpool entry as another ARM constpool value.
+  virtual bool hasSameValue(ARMConstantPoolValue *ACPV);
+
+  virtual void print(raw_ostream &O) const;
+
+  static bool classof(const ARMConstantPoolValue *ACPV) {
+    return ACPV->isExtSymbol();
+  }
+  static bool classof(const ARMConstantPoolSymbol *) { return true; }
+};
+
+/// ARMConstantPoolMBB - ARM-specific constantpool value of a machine basic
+/// block.
+class ARMConstantPoolMBB : public ARMConstantPoolValue {
+  const MachineBasicBlock *MBB; // Machine basic block.
+
+  ARMConstantPoolMBB(LLVMContext &C, const MachineBasicBlock *mbb, unsigned id,
+                     unsigned char PCAdj, ARMCP::ARMCPModifier Modifier,
+                     bool AddCurrentAddress);
+
+public:
+  static ARMConstantPoolMBB *Create(LLVMContext &C,
+                                    const MachineBasicBlock *mbb,
+                                    unsigned ID, unsigned char PCAdj);
+
+  const MachineBasicBlock *getMBB() const { return MBB; }
+
+  virtual int getExistingMachineCPValue(MachineConstantPool *CP,
+                                        unsigned Alignment);
+
+  virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID);
+
+  /// hasSameValue - Return true if this ARM constpool value can share the same
+  /// constantpool entry as another ARM constpool value.
+  virtual bool hasSameValue(ARMConstantPoolValue *ACPV);
+
+  virtual void print(raw_ostream &O) const;
+
+  static bool classof(const ARMConstantPoolValue *ACPV) {
+    return ACPV->isMachineBasicBlock();
+  }
+  static bool classof(const ARMConstantPoolMBB *) { return true; }
+};
+
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/contrib/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 94b72fdb9a7e..7872cb90f4e7 100644
--- a/contrib/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -16,19 +16,24 @@
 
 #define DEBUG_TYPE "arm-pseudo"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMBaseRegisterInfo.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMRegisterInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/raw_ostream.h" // FIXME: for debug only. remove!
 using namespace llvm;
 
+static cl::opt<bool>
+VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
+                cl::desc("Verify machine code after expanding ARM pseudos"));
+
 namespace {
   class ARMExpandPseudo : public MachineFunctionPass {
   public:
@@ -741,8 +746,22 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
       MI.eraseFromParent();
       return true;
     }
-    case ARM::MOVCCs: {
-      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVs),
+    case ARM::MOVCCsi: {
+      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
+              (MI.getOperand(1).getReg()))
+        .addReg(MI.getOperand(2).getReg(),
+                getKillRegState(MI.getOperand(2).isKill()))
+        .addImm(MI.getOperand(3).getImm())
+        .addImm(MI.getOperand(4).getImm()) // 'pred'
+        .addReg(MI.getOperand(5).getReg())
+        .addReg(0); // 's' bit
+
+      MI.eraseFromParent();
+      return true;
+    }
+
+    case ARM::MOVCCsr: {
+      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
               (MI.getOperand(1).getReg()))
         .addReg(MI.getOperand(2).getReg(),
                 getKillRegState(MI.getOperand(2).isKill()))
@@ -837,10 +856,9 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     case ARM::MOVsrl_flag:
     case ARM::MOVsra_flag: {
       // These are just fancy MOVs insructions.
-      AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVs),
+      AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
                              MI.getOperand(0).getReg())
                      .addOperand(MI.getOperand(1))
-                     .addReg(0)
                      .addImm(ARM_AM::getSORegOpc((Opcode == ARM::MOVsrl_flag ?
                                                   ARM_AM::lsr : ARM_AM::asr),
                                                  1)))
@@ -851,10 +869,9 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     case ARM::RRX: {
       // This encodes as "MOVs Rd, Rm, rrx
       MachineInstrBuilder MIB =
-        AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVs),
+        AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(),TII->get(ARM::MOVsi),
                                MI.getOperand(0).getReg())
                        .addOperand(MI.getOperand(1))
-                       .addOperand(MI.getOperand(1))
                        .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0)))
         .addReg(0);
       TransferImpOps(MI, MIB, MIB);
@@ -953,34 +970,6 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
       ExpandMOV32BitImm(MBB, MBBI);
       return true;
 
-    case ARM::VMOVQQ: {
-      unsigned DstReg = MI.getOperand(0).getReg();
-      bool DstIsDead = MI.getOperand(0).isDead();
-      unsigned EvenDst = TRI->getSubReg(DstReg, ARM::qsub_0);
-      unsigned OddDst  = TRI->getSubReg(DstReg, ARM::qsub_1);
-      unsigned SrcReg = MI.getOperand(1).getReg();
-      bool SrcIsKill = MI.getOperand(1).isKill();
-      unsigned EvenSrc = TRI->getSubReg(SrcReg, ARM::qsub_0);
-      unsigned OddSrc  = TRI->getSubReg(SrcReg, ARM::qsub_1);
-      MachineInstrBuilder Even =
-        AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(),
-                               TII->get(ARM::VORRq))
-                       .addReg(EvenDst,
-                               RegState::Define | getDeadRegState(DstIsDead))
-                       .addReg(EvenSrc, getKillRegState(SrcIsKill))
-                       .addReg(EvenSrc, getKillRegState(SrcIsKill)));
-      MachineInstrBuilder Odd =
-        AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(),
-                               TII->get(ARM::VORRq))
-                       .addReg(OddDst,
-                               RegState::Define | getDeadRegState(DstIsDead))
-                       .addReg(OddSrc, getKillRegState(SrcIsKill))
-                       .addReg(OddSrc, getKillRegState(SrcIsKill)));
-      TransferImpOps(MI, Even, Odd);
-      MI.eraseFromParent();
-      return true;
-    }
-
     case ARM::VLDMQIA: {
       unsigned NewOpc = ARM::VLDMDIA;
       MachineInstrBuilder MIB =
@@ -1316,6 +1305,8 @@ bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
   for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E;
        ++MFI)
     Modified |= ExpandMBB(*MFI);
+  if (VerifyARMPseudo)
+    MF.verify(this, "After expanding ARM pseudo instructions.");
   return Modified;
 }
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMFastISel.cpp b/contrib/llvm/lib/Target/ARM/ARMFastISel.cpp
index f469d7efe11a..9bc7ef21d8a3 100644
--- a/contrib/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -14,13 +14,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMCallingConv.h"
 #include "ARMRegisterInfo.h"
 #include "ARMTargetMachine.h"
 #include "ARMSubtarget.h"
 #include "ARMConstantPoolValue.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CallingConv.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/GlobalVariable.h"
@@ -171,8 +171,8 @@ class ARMFastISel : public FastISel {
 
     // Utility routines.
   private:
-    bool isTypeLegal(const Type *Ty, MVT &VT);
-    bool isLoadTypeLegal(const Type *Ty, MVT &VT);
+    bool isTypeLegal(Type *Ty, MVT &VT);
+    bool isLoadTypeLegal(Type *Ty, MVT &VT);
     bool ARMEmitLoad(EVT VT, unsigned &ResultReg, Address &Addr);
     bool ARMEmitStore(EVT VT, unsigned SrcReg, Address &Addr);
     bool ARMComputeAddress(const Value *Obj, Address &Addr);
@@ -502,11 +502,19 @@ unsigned ARMFastISel::ARMMaterializeFP(const ConstantFP *CFP, EVT VT) {
   // This checks to see if we can use VFP3 instructions to materialize
   // a constant, otherwise we have to go through the constant pool.
   if (TLI.isFPImmLegal(Val, VT)) {
-    unsigned Opc = is64bit ? ARM::FCONSTD : ARM::FCONSTS;
+    int Imm;
+    unsigned Opc;
+    if (is64bit) {
+      Imm = ARM_AM::getFP64Imm(Val);
+      Opc = ARM::FCONSTD;
+    } else {
+      Imm = ARM_AM::getFP32Imm(Val);
+      Opc = ARM::FCONSTS;
+    }
     unsigned DestReg = createResultReg(TLI.getRegClassFor(VT));
     AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc),
                             DestReg)
-                    .addFPImm(CFP));
+                    .addImm(Imm));
     return DestReg;
   }
 
@@ -590,8 +598,9 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, EVT VT) {
   // Grab index.
   unsigned PCAdj = (RelocM != Reloc::PIC_) ? 0 : (Subtarget->isThumb() ? 4 : 8);
   unsigned Id = AFI->createPICLabelUId();
-  ARMConstantPoolValue *CPV = new ARMConstantPoolValue(GV, Id,
-                                                       ARMCP::CPValue, PCAdj);
+  ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(GV, Id,
+                                                              ARMCP::CPValue,
+                                                              PCAdj);
   unsigned Idx = MCP.getConstantPoolIndex(CPV, Align);
 
   // Load value.
@@ -615,8 +624,8 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, EVT VT) {
   if (Subtarget->GVIsIndirectSymbol(GV, RelocM)) {
     unsigned NewDestReg = createResultReg(TLI.getRegClassFor(VT));
     if (isThumb)
-      MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(ARM::t2LDRi12),
-                    NewDestReg)
+      MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL,
+                    TII.get(ARM::t2LDRi12), NewDestReg)
             .addReg(DestReg)
             .addImm(0);
     else
@@ -673,7 +682,7 @@ unsigned ARMFastISel::TargetMaterializeAlloca(const AllocaInst *AI) {
   return 0;
 }
 
-bool ARMFastISel::isTypeLegal(const Type *Ty, MVT &VT) {
+bool ARMFastISel::isTypeLegal(Type *Ty, MVT &VT) {
   EVT evt = TLI.getValueType(Ty, true);
 
   // Only handle simple types.
@@ -685,7 +694,7 @@ bool ARMFastISel::isTypeLegal(const Type *Ty, MVT &VT) {
   return TLI.isTypeLegal(VT);
 }
 
-bool ARMFastISel::isLoadTypeLegal(const Type *Ty, MVT &VT) {
+bool ARMFastISel::isLoadTypeLegal(Type *Ty, MVT &VT) {
   if (isTypeLegal(Ty, VT)) return true;
 
   // If this is a type than can be sign or zero-extended to a basic operation
@@ -714,7 +723,7 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
     U = C;
   }
 
-  if (const PointerType *Ty = dyn_cast<PointerType>(Obj->getType()))
+  if (PointerType *Ty = dyn_cast<PointerType>(Obj->getType()))
     if (Ty->getAddressSpace() > 255)
       // Fast instruction selection doesn't support the special
       // address spaces.
@@ -749,7 +758,7 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
       for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
            i != e; ++i, ++GTI) {
         const Value *Op = *i;
-        if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+        if (StructType *STy = dyn_cast<StructType>(*GTI)) {
           const StructLayout *SL = TD.getStructLayout(STy);
           unsigned Idx = cast<ConstantInt>(Op)->getZExtValue();
           TmpOffset += SL->getElementOffset(Idx);
@@ -946,6 +955,10 @@ bool ARMFastISel::ARMEmitLoad(EVT VT, unsigned &ResultReg, Address &Addr) {
 }
 
 bool ARMFastISel::SelectLoad(const Instruction *I) {
+  // Atomic loads need special handling.
+  if (cast<LoadInst>(I)->isAtomic())
+    return false;
+
   // Verify we have a legal type before going any further.
   MVT VT;
   if (!isLoadTypeLegal(I->getType(), VT))
@@ -1008,6 +1021,10 @@ bool ARMFastISel::SelectStore(const Instruction *I) {
   Value *Op0 = I->getOperand(0);
   unsigned SrcReg = 0;
 
+  // Atomic stores need special handling.
+  if (cast<StoreInst>(I)->isAtomic())
+    return false;
+
   // Verify we have a legal type before going any further.
   MVT VT;
   if (!isLoadTypeLegal(I->getOperand(0)->getType(), VT))
@@ -1085,7 +1102,7 @@ bool ARMFastISel::SelectBranch(const Instruction *I) {
   // TODO: Factor this out.
   if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {
     MVT SourceVT;
-    const Type *Ty = CI->getOperand(0)->getType();
+    Type *Ty = CI->getOperand(0)->getType();
     if (CI->hasOneUse() && (CI->getParent() == I->getParent())
         && isTypeLegal(Ty, SourceVT)) {
       bool isFloat = (Ty->isDoubleTy() || Ty->isFloatTy());
@@ -1201,7 +1218,7 @@ bool ARMFastISel::SelectCmp(const Instruction *I) {
   const CmpInst *CI = cast<CmpInst>(I);
 
   MVT VT;
-  const Type *Ty = CI->getOperand(0)->getType();
+  Type *Ty = CI->getOperand(0)->getType();
   if (!isTypeLegal(Ty, VT))
     return false;
 
@@ -1309,7 +1326,7 @@ bool ARMFastISel::SelectSIToFP(const Instruction *I) {
   if (!Subtarget->hasVFP2()) return false;
 
   MVT DstVT;
-  const Type *Ty = I->getType();
+  Type *Ty = I->getType();
   if (!isTypeLegal(Ty, DstVT))
     return false;
 
@@ -1328,7 +1345,7 @@ bool ARMFastISel::SelectSIToFP(const Instruction *I) {
   unsigned Opc;
   if (Ty->isFloatTy()) Opc = ARM::VSITOS;
   else if (Ty->isDoubleTy()) Opc = ARM::VSITOD;
-  else return 0;
+  else return false;
 
   unsigned ResultReg = createResultReg(TLI.getRegClassFor(DstVT));
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc),
@@ -1343,7 +1360,7 @@ bool ARMFastISel::SelectFPToSI(const Instruction *I) {
   if (!Subtarget->hasVFP2()) return false;
 
   MVT DstVT;
-  const Type *RetTy = I->getType();
+  Type *RetTy = I->getType();
   if (!isTypeLegal(RetTy, DstVT))
     return false;
 
@@ -1351,10 +1368,10 @@ bool ARMFastISel::SelectFPToSI(const Instruction *I) {
   if (Op == 0) return false;
 
   unsigned Opc;
-  const Type *OpTy = I->getOperand(0)->getType();
+  Type *OpTy = I->getOperand(0)->getType();
   if (OpTy->isFloatTy()) Opc = ARM::VTOSIZS;
   else if (OpTy->isDoubleTy()) Opc = ARM::VTOSIZD;
-  else return 0;
+  else return false;
 
   // f64->s32 or f32->s32 both need an intermediate f32 reg.
   unsigned ResultReg = createResultReg(TLI.getRegClassFor(MVT::f32));
@@ -1401,7 +1418,7 @@ bool ARMFastISel::SelectSelect(const Instruction *I) {
 
 bool ARMFastISel::SelectSDiv(const Instruction *I) {
   MVT VT;
-  const Type *Ty = I->getType();
+  Type *Ty = I->getType();
   if (!isTypeLegal(Ty, VT))
     return false;
 
@@ -1429,7 +1446,7 @@ bool ARMFastISel::SelectSDiv(const Instruction *I) {
 
 bool ARMFastISel::SelectSRem(const Instruction *I) {
   MVT VT;
-  const Type *Ty = I->getType();
+  Type *Ty = I->getType();
   if (!isTypeLegal(Ty, VT))
     return false;
 
@@ -1456,7 +1473,7 @@ bool ARMFastISel::SelectBinaryOp(const Instruction *I, unsigned ISDOpcode) {
   // operations, but can't figure out how to. Just use the vfp instructions
   // if we have them.
   // FIXME: It'd be nice to use NEON instructions.
-  const Type *Ty = I->getType();
+  Type *Ty = I->getType();
   bool isFloat = (Ty->isDoubleTy() || Ty->isFloatTy());
   if (isFloat && !Subtarget->hasVFP2())
     return false;
@@ -1711,7 +1728,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
 
     // Analyze operands of the call, assigning locations to each operand.
     SmallVector<CCValAssign, 16> ValLocs;
-    CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, TM, ValLocs, I->getContext());
+    CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, TM, ValLocs,I->getContext());
     CCInfo.AnalyzeReturn(Outs, CCAssignFnForCall(CC, true /* is Ret */));
 
     const Value *RV = Ret->getOperand(0);
@@ -1778,7 +1795,7 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
   CallingConv::ID CC = TLI.getLibcallCallingConv(Call);
 
   // Handle *simple* calls for now.
-  const Type *RetTy = I->getType();
+  Type *RetTy = I->getType();
   MVT RetVT;
   if (RetTy->isVoidTy())
     RetVT = MVT::isVoid;
@@ -1802,7 +1819,7 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
     unsigned Arg = getRegForValue(Op);
     if (Arg == 0) return false;
 
-    const Type *ArgTy = Op->getType();
+    Type *ArgTy = Op->getType();
     MVT ArgVT;
     if (!isTypeLegal(ArgTy, ArgVT)) return false;
 
@@ -1870,13 +1887,13 @@ bool ARMFastISel::SelectCall(const Instruction *I) {
   // TODO: Avoid some calling conventions?
 
   // Let SDISel handle vararg functions.
-  const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
-  const FunctionType *FTy = cast<FunctionType>(PT->getElementType());
+  PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
+  FunctionType *FTy = cast<FunctionType>(PT->getElementType());
   if (FTy->isVarArg())
     return false;
 
   // Handle *simple* calls for now.
-  const Type *RetTy = I->getType();
+  Type *RetTy = I->getType();
   MVT RetVT;
   if (RetTy->isVoidTy())
     RetVT = MVT::isVoid;
@@ -1915,7 +1932,7 @@ bool ARMFastISel::SelectCall(const Instruction *I) {
         CS.paramHasAttr(AttrInd, Attribute::ByVal))
       return false;
 
-    const Type *ArgTy = (*i)->getType();
+    Type *ArgTy = (*i)->getType();
     MVT ArgVT;
     if (!isTypeLegal(ArgTy, ArgVT))
       return false;
@@ -1969,9 +1986,9 @@ bool ARMFastISel::SelectIntCast(const Instruction *I) {
   // On ARM, in general, integer casts don't involve legal types; this code
   // handles promotable integers.  The high bits for a type smaller than
   // the register size are assumed to be undefined.
-  const Type *DestTy = I->getType();
+  Type *DestTy = I->getType();
   Value *Op = I->getOperand(0);
-  const Type *SrcTy = Op->getType();
+  Type *SrcTy = Op->getType();
 
   EVT SrcVT, DestVT;
   SrcVT = TLI.getValueType(SrcTy, true);
@@ -2002,16 +2019,18 @@ bool ARMFastISel::SelectIntCast(const Instruction *I) {
   switch (SrcVT.getSimpleVT().SimpleTy) {
   default: return false;
   case MVT::i16:
+    if (!Subtarget->hasV6Ops()) return false;
     if (isZext)
-      Opc = isThumb ? ARM::t2UXTHr : ARM::UXTHr;
+      Opc = isThumb ? ARM::t2UXTH : ARM::UXTH;
     else
-      Opc = isThumb ? ARM::t2SXTHr : ARM::SXTHr;
+      Opc = isThumb ? ARM::t2SXTH : ARM::SXTH;
     break;
   case MVT::i8:
+    if (!Subtarget->hasV6Ops()) return false;
     if (isZext)
-      Opc = isThumb ? ARM::t2UXTBr : ARM::UXTBr;
+      Opc = isThumb ? ARM::t2UXTB : ARM::UXTB;
     else
-      Opc = isThumb ? ARM::t2SXTBr : ARM::SXTBr;
+      Opc = isThumb ? ARM::t2SXTB : ARM::SXTB;
     break;
   case MVT::i1:
     if (isZext) {
@@ -2033,6 +2052,8 @@ bool ARMFastISel::SelectIntCast(const Instruction *I) {
         .addReg(SrcReg);
   if (isBoolZext)
     MIB.addImm(1);
+  else
+    MIB.addImm(0);
   AddOptionalDefs(MIB);
   UpdateValueMap(I, DestReg);
   return true;
diff --git a/contrib/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/contrib/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 381b404519e2..2d1de6fe8e9d 100644
--- a/contrib/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -12,10 +12,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "ARMFrameLowering.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMBaseRegisterInfo.h"
 #include "ARMMachineFunctionInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -93,7 +93,8 @@ static bool isCSRestore(MachineInstr *MI,
         return false;
     return true;
   }
-  if ((MI->getOpcode() == ARM::LDR_POST ||
+  if ((MI->getOpcode() == ARM::LDR_POST_IMM ||
+       MI->getOpcode() == ARM::LDR_POST_REG ||
        MI->getOpcode() == ARM::t2LDR_POST) &&
       isCalleeSavedRegister(MI->getOperand(0).getReg(), CSRegs) &&
       MI->getOperand(1).getReg() == ARM::SP)
@@ -413,6 +414,9 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
         MIB.addExternalSymbol(JumpTarget.getSymbolName(),
                               JumpTarget.getTargetFlags());
       }
+
+      // Add the default predicate in Thumb mode.
+      if (STI.isThumb()) MIB.addImm(ARMCC::AL).addReg(0);
     } else if (RetOpcode == ARM::TCRETURNri) {
       BuildMI(MBB, MBBI, dl,
               TII.get(STI.isThumb() ? ARM::tTAILJMPr : ARM::TAILJMPr)).
@@ -502,7 +506,7 @@ ARMFrameLowering::ResolveFrameIndexReference(const MachineFunction &MF,
         }
       }
     } else if (AFI->isThumb2Function()) {
-      // Use  add <rd>, sp, #<imm8> 
+      // Use  add <rd>, sp, #<imm8>
       //      ldr <rd>, [sp, #<imm8>]
       // if at all possible to save space.
       if (Offset >= 0 && (Offset & 3) == 0 && Offset <= 1020)
@@ -587,14 +591,8 @@ void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc),
                                         ARM::SP)
         .addReg(Regs[0].first, getKillRegState(Regs[0].second))
-        .addReg(ARM::SP).setMIFlags(MIFlags);
-      // ARM mode needs an extra reg0 here due to addrmode2. Will go away once
-      // that refactoring is complete (eventually).
-      if (StrOpc == ARM::STR_PRE) {
-        MIB.addReg(0);
-        MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::sub, 4, ARM_AM::no_shift));
-      } else
-        MIB.addImm(-4);
+        .addReg(ARM::SP).setMIFlags(MIFlags)
+        .addImm(-4);
       AddDefaultPred(MIB);
     }
     Regs.clear();
@@ -651,8 +649,10 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
                        .addReg(ARM::SP));
       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
         MIB.addReg(Regs[i], getDefRegState(true));
-      if (DeleteRet)
+      if (DeleteRet) {
+        MIB->copyImplicitOps(&*MI);
         MI->eraseFromParent();
+      }
       MI = MIB;
     } else if (Regs.size() == 1) {
       // If we adjusted the reg to PC from LR above, switch it back here. We
@@ -665,7 +665,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
           .addReg(ARM::SP);
       // ARM mode needs an extra reg0 here due to addrmode2. Will go away once
       // that refactoring is complete (eventually).
-      if (LdrOpc == ARM::LDR_POST) {
+      if (LdrOpc == ARM::LDR_POST_REG || LdrOpc == ARM::LDR_POST_IMM) {
         MIB.addReg(0);
         MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::add, 4, ARM_AM::no_shift));
       } else
@@ -687,7 +687,8 @@ bool ARMFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
 
   unsigned PushOpc = AFI->isThumbFunction() ? ARM::t2STMDB_UPD : ARM::STMDB_UPD;
-  unsigned PushOneOpc = AFI->isThumbFunction() ? ARM::t2STR_PRE : ARM::STR_PRE;
+  unsigned PushOneOpc = AFI->isThumbFunction() ?
+    ARM::t2STR_PRE : ARM::STR_PRE_IMM;
   unsigned FltOpc = ARM::VSTMDDB_UPD;
   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea1Register,
                MachineInstr::FrameSetup);
@@ -711,7 +712,7 @@ bool ARMFrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
   bool isVarArg = AFI->getVarArgsRegSaveSize() > 0;
 
   unsigned PopOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
-  unsigned LdrOpc = AFI->isThumbFunction() ? ARM::t2LDR_POST : ARM::LDR_POST;
+  unsigned LdrOpc = AFI->isThumbFunction() ? ARM::t2LDR_POST :ARM::LDR_POST_IMM;
   unsigned FltOpc = ARM::VLDMDIA_UPD;
   emitPopInst(MBB, MI, CSI, FltOpc, 0, isVarArg, true, &isARMArea3Register);
   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
diff --git a/contrib/llvm/lib/Target/ARM/ARMGlobalMerge.cpp b/contrib/llvm/lib/Target/ARM/ARMGlobalMerge.cpp
index 8d77b2d8383e..5f863ea241ca 100644
--- a/contrib/llvm/lib/Target/ARM/ARMGlobalMerge.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMGlobalMerge.cpp
@@ -100,8 +100,8 @@ namespace {
       GlobalCmp(const TargetData *td) : TD(td) { }
 
       bool operator()(const GlobalVariable *GV1, const GlobalVariable *GV2) {
-        const Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
-        const Type *Ty2 = cast<PointerType>(GV2->getType())->getElementType();
+        Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
+        Type *Ty2 = cast<PointerType>(GV2->getType())->getElementType();
 
         return (TD->getTypeAllocSize(Ty1) < TD->getTypeAllocSize(Ty2));
       }
@@ -123,7 +123,7 @@ bool ARMGlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
   // FIXME: Find better heuristics
   std::stable_sort(Globals.begin(), Globals.end(), GlobalCmp(TD));
 
-  const Type *Int32Ty = Type::getInt32Ty(M.getContext());
+  Type *Int32Ty = Type::getInt32Ty(M.getContext());
 
   for (size_t i = 0, e = Globals.size(); i != e; ) {
     size_t j = 0;
@@ -150,7 +150,7 @@ bool ARMGlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
         ConstantInt::get(Int32Ty, 0),
         ConstantInt::get(Int32Ty, k-i)
       };
-      Constant *GEP = ConstantExpr::getInBoundsGetElementPtr(MergedGV, Idx, 2);
+      Constant *GEP = ConstantExpr::getInBoundsGetElementPtr(MergedGV, Idx);
       Globals[k]->replaceAllUsesWith(GEP);
       Globals[k]->eraseFromParent();
     }
@@ -176,7 +176,7 @@ bool ARMGlobalMerge::doInitialization(Module &M) {
 
     // Ignore fancy-aligned globals for now.
     unsigned Alignment = I->getAlignment();
-    const Type *Ty = I->getType()->getElementType();
+    Type *Ty = I->getType()->getElementType();
     if (Alignment > TD->getABITypeAlignment(Ty))
       continue;
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/contrib/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 2c9481b86c55..5ee009c04c5b 100644
--- a/contrib/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -14,8 +14,8 @@
 #define DEBUG_TYPE "arm-isel"
 #include "ARM.h"
 #include "ARMBaseInstrInfo.h"
-#include "ARMAddressingModes.h"
 #include "ARMTargetMachine.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CallingConv.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
@@ -47,6 +47,11 @@ CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
   cl::desc("Check fp vmla / vmls hazard at isel time"),
   cl::init(true));
 
+static cl::opt<bool>
+DisableARMIntABS("disable-arm-int-abs", cl::Hidden,
+  cl::desc("Enable / disable ARM integer abs transform"),
+  cl::init(false));
+
 //===--------------------------------------------------------------------===//
 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
 /// instructions for SelectionDAG operations.
@@ -90,13 +95,20 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   bool hasNoVMLxHazardUse(SDNode *N) const;
   bool isShifterOpProfitable(const SDValue &Shift,
                              ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
-  bool SelectShifterOperandReg(SDValue N, SDValue &A,
+  bool SelectRegShifterOperand(SDValue N, SDValue &A,
                                SDValue &B, SDValue &C,
                                bool CheckProfitability = true);
-  bool SelectShiftShifterOperandReg(SDValue N, SDValue &A,
+  bool SelectImmShifterOperand(SDValue N, SDValue &A,
+                               SDValue &B, bool CheckProfitability = true);
+  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
                                     SDValue &B, SDValue &C) {
     // Don't apply the profitability check
-    return SelectShifterOperandReg(N, A, B, C, false);
+    return SelectRegShifterOperand(N, A, B, C, false);
+  }
+  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
+                                    SDValue &B) {
+    // Don't apply the profitability check
+    return SelectImmShifterOperand(N, A, B, false);
   }
 
   bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
@@ -122,8 +134,13 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
     return true;
   }
 
-  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
+  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
                              SDValue &Offset, SDValue &Opc);
+  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
+                             SDValue &Offset, SDValue &Opc);
+  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
+                             SDValue &Offset, SDValue &Opc);
+  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
   bool SelectAddrMode3(SDValue N, SDValue &Base,
                        SDValue &Offset, SDValue &Opc);
   bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
@@ -240,8 +257,13 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
                                ARMCC::CondCodes CCVal, SDValue CCR,
                                SDValue InFlag);
 
+  // Select special operations if node forms integer ABS pattern
+  SDNode *SelectABSOp(SDNode *N);
+
   SDNode *SelectConcatVector(SDNode *N);
 
+  SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
+
   /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
   /// inline asm expressions.
   virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
@@ -291,10 +313,10 @@ static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
 /// (N * Scale) where (N in [\arg RangeMin, \arg RangeMax).
 ///
 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
-static bool isScaledConstantInRange(SDValue Node, unsigned Scale,
+static bool isScaledConstantInRange(SDValue Node, int Scale,
                                     int RangeMin, int RangeMax,
                                     int &ScaledConstant) {
-  assert(Scale && "Invalid scale!");
+  assert(Scale > 0 && "Invalid scale!");
 
   // Check that this is a constant.
   const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
@@ -365,15 +387,14 @@ bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
   return ShOpcVal == ARM_AM::lsl && ShAmt == 2;
 }
 
-bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue N,
+bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
                                               SDValue &BaseReg,
-                                              SDValue &ShReg,
                                               SDValue &Opc,
                                               bool CheckProfitability) {
   if (DisableShifterOp)
     return false;
 
-  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
+  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
 
   // Don't match base register only case. That is matched to a separate
   // lower complexity pattern with explicit register operand.
@@ -381,19 +402,42 @@ bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue N,
 
   BaseReg = N.getOperand(0);
   unsigned ShImmVal = 0;
-  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
-    ShReg = CurDAG->getRegister(0, MVT::i32);
-    ShImmVal = RHS->getZExtValue() & 31;
-  } else {
-    ShReg = N.getOperand(1);
-    if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
-      return false;
-  }
+  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
+  if (!RHS) return false;
+  ShImmVal = RHS->getZExtValue() & 31;
   Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
                                   MVT::i32);
   return true;
 }
 
+bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
+                                              SDValue &BaseReg,
+                                              SDValue &ShReg,
+                                              SDValue &Opc,
+                                              bool CheckProfitability) {
+  if (DisableShifterOp)
+    return false;
+
+  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
+
+  // Don't match base register only case. That is matched to a separate
+  // lower complexity pattern with explicit register operand.
+  if (ShOpcVal == ARM_AM::no_shift) return false;
+
+  BaseReg = N.getOperand(0);
+  unsigned ShImmVal = 0;
+  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
+  if (RHS) return false;
+
+  ShReg = N.getOperand(1);
+  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
+    return false;
+  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
+                                  MVT::i32);
+  return true;
+}
+
+
 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
                                           SDValue &Base,
                                           SDValue &OffImm) {
@@ -483,13 +527,10 @@ bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
       return false;
   }
 
-  if (Subtarget->isCortexA9() && !N.hasOneUse())
-    // Compute R +/- (R << N) and reuse it.
-    return false;
-
   // Otherwise this is R +/- [possibly shifted] R.
   ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
-  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
+  ARM_AM::ShiftOpc ShOpcVal =
+    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
   unsigned ShAmt = 0;
 
   Base   = N.getOperand(0);
@@ -515,16 +556,14 @@ bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
   // Try matching (R shl C) + (R).
   if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
       !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
-    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
+    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
     if (ShOpcVal != ARM_AM::no_shift) {
       // Check to see if the RHS of the shift is a constant, if not, we can't
       // fold it.
       if (ConstantSDNode *Sh =
           dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
         ShAmt = Sh->getZExtValue();
-        if (!Subtarget->isCortexA9() ||
-            (N.hasOneUse() &&
-             isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt))) {
+        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
           Offset = N.getOperand(0).getOperand(0);
           Base = N.getOperand(1);
         } else {
@@ -630,7 +669,8 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
 
   // Otherwise this is R +/- [possibly shifted] R.
   ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
-  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
+  ARM_AM::ShiftOpc ShOpcVal =
+    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
   unsigned ShAmt = 0;
 
   Base   = N.getOperand(0);
@@ -656,16 +696,14 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
   // Try matching (R shl C) + (R).
   if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
       !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
-    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
+    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
     if (ShOpcVal != ARM_AM::no_shift) {
       // Check to see if the RHS of the shift is a constant, if not, we can't
       // fold it.
       if (ConstantSDNode *Sh =
           dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
         ShAmt = Sh->getZExtValue();
-        if (!Subtarget->isCortexA9() ||
-            (N.hasOneUse() &&
-             isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt))) {
+        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
           Offset = N.getOperand(0).getOperand(0);
           Base = N.getOperand(1);
         } else {
@@ -683,7 +721,7 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
   return AM2_SHOP;
 }
 
-bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
+bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
                                             SDValue &Offset, SDValue &Opc) {
   unsigned Opcode = Op->getOpcode();
   ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
@@ -692,16 +730,11 @@ bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
   ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
     ? ARM_AM::add : ARM_AM::sub;
   int Val;
-  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
-    Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
-                                                      ARM_AM::no_shift),
-                                    MVT::i32);
-    return true;
-  }
+  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
+    return false;
 
   Offset = N;
-  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
+  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
   unsigned ShAmt = 0;
   if (ShOpcVal != ARM_AM::no_shift) {
     // Check to see if the RHS of the shift is a constant, if not, we can't fold
@@ -724,6 +757,50 @@ bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
   return true;
 }
 
+bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
+                                            SDValue &Offset, SDValue &Opc) {
+  unsigned Opcode = Op->getOpcode();
+  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
+    ? cast<LoadSDNode>(Op)->getAddressingMode()
+    : cast<StoreSDNode>(Op)->getAddressingMode();
+  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
+    ? ARM_AM::add : ARM_AM::sub;
+  int Val;
+  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
+    if (AddSub == ARM_AM::sub) Val *= -1;
+    Offset = CurDAG->getRegister(0, MVT::i32);
+    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
+    return true;
+  }
+
+  return false;
+}
+
+
+bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
+                                            SDValue &Offset, SDValue &Opc) {
+  unsigned Opcode = Op->getOpcode();
+  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
+    ? cast<LoadSDNode>(Op)->getAddressingMode()
+    : cast<StoreSDNode>(Op)->getAddressingMode();
+  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
+    ? ARM_AM::add : ARM_AM::sub;
+  int Val;
+  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
+    Offset = CurDAG->getRegister(0, MVT::i32);
+    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
+                                                      ARM_AM::no_shift),
+                                    MVT::i32);
+    return true;
+  }
+
+  return false;
+}
+
+bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
+  Base = N;
+  return true;
+}
 
 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
                                       SDValue &Base, SDValue &Offset,
@@ -1079,7 +1156,7 @@ bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
   if (DisableShifterOp)
     return false;
 
-  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
+  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
 
   // Don't match base register only case. That is matched to a separate
   // lower complexity pattern with explicit register operand.
@@ -1208,21 +1285,15 @@ bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
       return false;
   }
 
-  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
-    // Compute R + (R << [1,2,3]) and reuse it.
-    Base = N;
-    return false;
-  }
-
   // Look for (R + R) or (R + (R << [1,2,3])).
   unsigned ShAmt = 0;
   Base   = N.getOperand(0);
   OffReg = N.getOperand(1);
 
   // Swap if it is ((R << c) + R).
-  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
+  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
   if (ShOpcVal != ARM_AM::lsl) {
-    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
+    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
     if (ShOpcVal == ARM_AM::lsl)
       std::swap(Base, OffReg);
   }
@@ -1266,10 +1337,19 @@ SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
   bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
   unsigned Opcode = 0;
   bool Match = false;
-  if (LoadedVT == MVT::i32 &&
-      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
-    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
+  if (LoadedVT == MVT::i32 && isPre &&
+      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
+    Opcode = ARM::LDR_PRE_IMM;
     Match = true;
+  } else if (LoadedVT == MVT::i32 && !isPre &&
+      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
+    Opcode = ARM::LDR_POST_IMM;
+    Match = true;
+  } else if (LoadedVT == MVT::i32 &&
+      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
+    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
+    Match = true;
+
   } else if (LoadedVT == MVT::i16 &&
              SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
     Match = true;
@@ -1283,20 +1363,37 @@ SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
         Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
       }
     } else {
-      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
+      if (isPre &&
+          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
         Match = true;
-        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
+        Opcode = ARM::LDRB_PRE_IMM;
+      } else if (!isPre &&
+                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
+        Match = true;
+        Opcode = ARM::LDRB_POST_IMM;
+      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
+        Match = true;
+        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
       }
     }
   }
 
   if (Match) {
-    SDValue Chain = LD->getChain();
-    SDValue Base = LD->getBasePtr();
-    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
-                     CurDAG->getRegister(0, MVT::i32), Chain };
-    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
-                                  MVT::Other, Ops, 6);
+    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
+      SDValue Chain = LD->getChain();
+      SDValue Base = LD->getBasePtr();
+      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
+                       CurDAG->getRegister(0, MVT::i32), Chain };
+      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
+                                    MVT::i32, MVT::Other, Ops, 5);
+    } else {
+      SDValue Chain = LD->getChain();
+      SDValue Base = LD->getBasePtr();
+      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
+                       CurDAG->getRegister(0, MVT::i32), Chain };
+      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
+                                    MVT::i32, MVT::Other, Ops, 6);
+    }
   }
 
   return NULL;
@@ -1966,7 +2063,8 @@ SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
                                 Srl_imm)) {
         assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
 
-        unsigned Width = CountTrailingOnes_32(And_imm);
+        // Note: The width operand is encoded as width-1.
+        unsigned Width = CountTrailingOnes_32(And_imm) - 1;
         unsigned LSB = Srl_imm;
         SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
         SDValue Ops[] = { N->getOperand(0).getOperand(0),
@@ -1986,7 +2084,8 @@ SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
     unsigned Srl_imm = 0;
     if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
       assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
-      unsigned Width = 32 - Srl_imm;
+      // Note: The width operand is encoded as width-1.
+      unsigned Width = 32 - Srl_imm - 1;
       int LSB = Srl_imm - Shl_imm;
       if (LSB < 0)
         return NULL;
@@ -2034,10 +2133,16 @@ SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
   SDValue CPTmp0;
   SDValue CPTmp1;
   SDValue CPTmp2;
-  if (SelectShifterOperandReg(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
+  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
+    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
+    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag };
+    return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6);
+  }
+
+  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
     SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
     SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
-    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
+    return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7);
   }
   return 0;
 }
@@ -2198,6 +2303,56 @@ SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
   return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
 }
 
+/// Target-specific DAG combining for ISD::XOR.
+/// Target-independent combining lowers SELECT_CC nodes of the form
+/// select_cc setg[ge] X,  0,  X, -X
+/// select_cc setgt    X, -1,  X, -X
+/// select_cc setl[te] X,  0, -X,  X
+/// select_cc setlt    X,  1, -X,  X
+/// which represent Integer ABS into:
+/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
+/// ARM instruction selection detects the latter and matches it to
+/// ARM::ABS or ARM::t2ABS machine node.
+SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
+  SDValue XORSrc0 = N->getOperand(0);
+  SDValue XORSrc1 = N->getOperand(1);
+  DebugLoc DL = N->getDebugLoc();
+  EVT VT = N->getValueType(0);
+
+  if (DisableARMIntABS)
+    return NULL;
+
+  if (Subtarget->isThumb1Only())
+    return NULL;
+
+  if (XORSrc0.getOpcode() != ISD::ADD ||
+    XORSrc1.getOpcode() != ISD::SRA)
+    return NULL;
+
+  SDValue ADDSrc0 = XORSrc0.getOperand(0);
+  SDValue ADDSrc1 = XORSrc0.getOperand(1);
+  SDValue SRASrc0 = XORSrc1.getOperand(0);
+  SDValue SRASrc1 = XORSrc1.getOperand(1);
+  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
+  EVT XType = SRASrc0.getValueType();
+  unsigned Size = XType.getSizeInBits() - 1;
+
+  if (ADDSrc1 == XORSrc1  &&
+      ADDSrc0 == SRASrc0 &&
+      XType.isInteger() &&
+      SRAConstant != NULL &&
+      Size == SRAConstant->getZExtValue()) {
+
+    unsigned Opcode = ARM::ABS;
+    if (Subtarget->isThumb2())
+      Opcode = ARM::t2ABS;
+
+    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
+  }
+
+  return NULL;
+}
+
 SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
   // The only time a CONCAT_VECTORS operation can have legal types is when
   // two 64-bit vectors are concatenated to a 128-bit vector.
@@ -2207,6 +2362,25 @@ SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
   return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
 }
 
+SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
+  SmallVector<SDValue, 6> Ops;
+  Ops.push_back(Node->getOperand(1)); // Ptr
+  Ops.push_back(Node->getOperand(2)); // Low part of Val1
+  Ops.push_back(Node->getOperand(3)); // High part of Val1
+  if (Opc == ARM::ATOMCMPXCHG6432) {
+    Ops.push_back(Node->getOperand(4)); // Low part of Val2
+    Ops.push_back(Node->getOperand(5)); // High part of Val2
+  }
+  Ops.push_back(Node->getOperand(0)); // Chain
+  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
+  MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
+  SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
+                                           MVT::i32, MVT::i32, MVT::Other,
+                                           Ops.data() ,Ops.size());
+  cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1);
+  return ResNode;
+}
+
 SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   DebugLoc dl = N->getDebugLoc();
 
@@ -2215,6 +2389,14 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
 
   switch (N->getOpcode()) {
   default: break;
+  case ISD::XOR: {
+    // Select special operations if XOR node forms integer ABS pattern
+    SDNode *ResNode = SelectABSOp(N);
+    if (ResNode)
+      return ResNode;
+    // Other cases are autogenerated.
+    break;
+  }
   case ISD::Constant: {
     unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
     bool UseCP = true;
@@ -2269,8 +2451,9 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
     int FI = cast<FrameIndexSDNode>(N)->getIndex();
     SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
     if (Subtarget->isThumb1Only()) {
-      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
-                                  CurDAG->getTargetConstant(0, MVT::i32));
+      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
+                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
+      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4);
     } else {
       unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
                       ARM::t2ADDri : ARM::ADDri);
@@ -2307,7 +2490,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
           return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
         } else {
           SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
-          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
+          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7);
         }
       }
       if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
@@ -2323,7 +2506,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
           return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
         } else {
           SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
-          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
+          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7);
         }
       }
     }
@@ -2986,6 +3169,23 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
 
   case ISD::CONCAT_VECTORS:
     return SelectConcatVector(N);
+
+  case ARMISD::ATOMOR64_DAG:
+    return SelectAtomic64(N, ARM::ATOMOR6432);
+  case ARMISD::ATOMXOR64_DAG:
+    return SelectAtomic64(N, ARM::ATOMXOR6432);
+  case ARMISD::ATOMADD64_DAG:
+    return SelectAtomic64(N, ARM::ATOMADD6432);
+  case ARMISD::ATOMSUB64_DAG:
+    return SelectAtomic64(N, ARM::ATOMSUB6432);
+  case ARMISD::ATOMNAND64_DAG:
+    return SelectAtomic64(N, ARM::ATOMNAND6432);
+  case ARMISD::ATOMAND64_DAG:
+    return SelectAtomic64(N, ARM::ATOMAND6432);
+  case ARMISD::ATOMSWAP64_DAG:
+    return SelectAtomic64(N, ARM::ATOMSWAP6432);
+  case ARMISD::ATOMCMPXCHG64_DAG:
+    return SelectAtomic64(N, ARM::ATOMCMPXCHG6432);
   }
 
   return SelectCode(N);
diff --git a/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp b/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp
index cf8c5baa8e7d..e44e35673aeb 100644
--- a/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -14,7 +14,6 @@
 
 #define DEBUG_TYPE "arm-isel"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMCallingConv.h"
 #include "ARMConstantPoolValue.h"
 #include "ARMISelLowering.h"
@@ -24,6 +23,7 @@
 #include "ARMSubtarget.h"
 #include "ARMTargetMachine.h"
 #include "ARMTargetObjectFile.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CallingConv.h"
 #include "llvm/Constants.h"
 #include "llvm/Function.h"
@@ -38,6 +38,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -106,7 +107,7 @@ void ARMTargetLowering::addTypeForNEON(EVT VT, EVT PromotedLdStVT,
 
   EVT ElemTy = VT.getVectorElementType();
   if (ElemTy != MVT::i64 && ElemTy != MVT::f64)
-    setOperationAction(ISD::VSETCC, VT.getSimpleVT(), Custom);
+    setOperationAction(ISD::SETCC, VT.getSimpleVT(), Custom);
   setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT.getSimpleVT(), Custom);
   if (ElemTy != MVT::i32) {
     setOperationAction(ISD::SINT_TO_FP, VT.getSimpleVT(), Expand);
@@ -178,6 +179,8 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   RegInfo = TM.getRegisterInfo();
   Itins = TM.getInstrItineraryData();
 
+  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
+
   if (Subtarget->isTargetDarwin()) {
     // Uses VFP for Thumb libfuncs if available.
     if (Subtarget->isThumb() && Subtarget->hasVFP2()) {
@@ -419,6 +422,13 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
     setLibcallName(RTLIB::MEMSET,  "__aeabi_memset");
   }
 
+  // Use divmod compiler-rt calls for iOS 5.0 and later.
+  if (Subtarget->getTargetTriple().getOS() == Triple::IOS &&
+      !Subtarget->getTargetTriple().isOSVersionLT(5, 0)) {
+    setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
+    setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
+  }
+
   if (Subtarget->isThumb1Only())
     addRegisterClass(MVT::i32, ARM::tGPRRegisterClass);
   else
@@ -453,7 +463,7 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
     setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
     setOperationAction(ISD::FREM, MVT::v2f64, Expand);
     setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
-    setOperationAction(ISD::VSETCC, MVT::v2f64, Expand);
+    setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
     setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
     setOperationAction(ISD::FABS, MVT::v2f64, Expand);
     setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
@@ -485,8 +495,8 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
     setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
     setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
     setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
-    setOperationAction(ISD::VSETCC, MVT::v1i64, Expand);
-    setOperationAction(ISD::VSETCC, MVT::v2i64, Expand);
+    setOperationAction(ISD::SETCC, MVT::v1i64, Expand);
+    setOperationAction(ISD::SETCC, MVT::v2i64, Expand);
     // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
     // a destination type that is wider than the source.
     setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
@@ -551,6 +561,14 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   setOperationAction(ISD::SRL,       MVT::i64, Custom);
   setOperationAction(ISD::SRA,       MVT::i64, Custom);
 
+  if (!Subtarget->isThumb1Only()) {
+    // FIXME: We should do this for Thumb1 as well.
+    setOperationAction(ISD::ADDC,    MVT::i32, Custom);
+    setOperationAction(ISD::ADDE,    MVT::i32, Custom);
+    setOperationAction(ISD::SUBC,    MVT::i32, Custom);
+    setOperationAction(ISD::SUBE,    MVT::i32, Custom);
+  }
+
   // ARM does not have ROTL.
   setOperationAction(ISD::ROTL,  MVT::i32, Expand);
   setOperationAction(ISD::CTTZ,  MVT::i32, Custom);
@@ -596,62 +614,46 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
   // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
   // the default expansion.
+  // FIXME: This should be checking for v6k, not just v6.
   if (Subtarget->hasDataBarrier() ||
       (Subtarget->hasV6Ops() && !Subtarget->isThumb())) {
     // membarrier needs custom lowering; the rest are legal and handled
     // normally.
     setOperationAction(ISD::MEMBARRIER, MVT::Other, Custom);
+    setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
+    // Custom lowering for 64-bit ops
+    setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_SWAP,  MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i64, Custom);
+    // Automatically insert fences (dmb ist) around ATOMIC_SWAP etc.
+    setInsertFencesForAtomic(true);
   } else {
     // Set them all for expansion, which will force libcalls.
     setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
-    setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i16, Expand);
+    setOperationAction(ISD::ATOMIC_FENCE,   MVT::Other, Expand);
     setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_SWAP,      MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_SWAP,      MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_SWAP,      MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i8,  Expand);
-    setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i16, Expand);
     setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand);
+    // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
+    // Unordered/Monotonic case.
+    setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
+    setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
     // Since the libcalls include locking, fold in the fences
     setShouldFoldAtomicFences(true);
   }
-  // 64-bit versions are always libcalls (for now)
-  setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_SWAP,      MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i64, Expand);
-  setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Expand);
 
   setOperationAction(ISD::PREFETCH,         MVT::Other, Custom);
 
@@ -839,6 +841,11 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::SRA_FLAG:      return "ARMISD::SRA_FLAG";
   case ARMISD::RRX:           return "ARMISD::RRX";
 
+  case ARMISD::ADDC:          return "ARMISD::ADDC";
+  case ARMISD::ADDE:          return "ARMISD::ADDE";
+  case ARMISD::SUBC:          return "ARMISD::SUBC";
+  case ARMISD::SUBE:          return "ARMISD::SUBE";
+
   case ARMISD::VMOVRRD:       return "ARMISD::VMOVRRD";
   case ARMISD::VMOVDRR:       return "ARMISD::VMOVDRR";
 
@@ -935,6 +942,11 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   }
 }
 
+EVT ARMTargetLowering::getSetCCResultType(EVT VT) const {
+  if (!VT.isVector()) return getPointerTy();
+  return VT.changeVectorElementTypeToInteger();
+}
+
 /// getRegClassFor - Return the register class that should be used for the
 /// specified value type.
 TargetRegisterClass *ARMTargetLowering::getRegClassFor(EVT VT) const {
@@ -1210,8 +1222,8 @@ ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
   MachineFunction &MF = DAG.getMachineFunction();
   bool IsStructRet    = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
   bool IsSibCall = false;
-  // Temporarily disable tail calls so things don't break.
-  if (!EnableARMTailCalls)
+  // Disable tail calls if they're not supported.
+  if (!EnableARMTailCalls && !Subtarget->supportsTailCall())
     isTailCall = false;
   if (isTailCall) {
     // Check if it's really possible to do a tail call.
@@ -1336,10 +1348,12 @@ ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
       SDValue Src = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg, SrcOffset);
       SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset,
                                          MVT::i32);
+      // TODO: Disable AlwaysInline when it becomes possible
+      //       to emit a nested call sequence.
       MemOpChains.push_back(DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode,
                                           Flags.getByValAlign(),
                                           /*isVolatile=*/false,
-                                          /*AlwaysInline=*/false,
+                                          /*AlwaysInline=*/true,
                                           MachinePointerInfo(0),
                                           MachinePointerInfo(0)));
 
@@ -1404,9 +1418,9 @@ ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
       const GlobalValue *GV = G->getGlobal();
       // Create a constant pool entry for the callee address
       unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-      ARMConstantPoolValue *CPV = new ARMConstantPoolValue(GV,
-                                                           ARMPCLabelIndex,
-                                                           ARMCP::CPValue, 0);
+      ARMConstantPoolValue *CPV =
+        ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0);
+
       // Get the address of the callee into a register
       SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
@@ -1419,8 +1433,9 @@ ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
 
       // Create a constant pool entry for the callee address
       unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-      ARMConstantPoolValue *CPV = new ARMConstantPoolValue(*DAG.getContext(),
-                                                       Sym, ARMPCLabelIndex, 0);
+      ARMConstantPoolValue *CPV =
+        ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
+                                      ARMPCLabelIndex, 0);
       // Get the address of the callee into a register
       SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
@@ -1441,9 +1456,8 @@ ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
     // tBX takes a register source operand.
     if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
       unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-      ARMConstantPoolValue *CPV = new ARMConstantPoolValue(GV,
-                                                           ARMPCLabelIndex,
-                                                           ARMCP::CPValue, 4);
+      ARMConstantPoolValue *CPV =
+        ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 4);
       SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
       Callee = DAG.getLoad(getPointerTy(), dl,
@@ -1470,8 +1484,9 @@ ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
     const char *Sym = S->getSymbol();
     if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
       unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-      ARMConstantPoolValue *CPV = new ARMConstantPoolValue(*DAG.getContext(),
-                                                       Sym, ARMPCLabelIndex, 4);
+      ARMConstantPoolValue *CPV =
+        ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
+                                      ARMPCLabelIndex, 4);
       SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
       Callee = DAG.getLoad(getPointerTy(), dl,
@@ -1940,9 +1955,9 @@ SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
   } else {
     unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
     ARMPCLabelIndex = AFI->createPICLabelUId();
-    ARMConstantPoolValue *CPV = new ARMConstantPoolValue(BA, ARMPCLabelIndex,
-                                                         ARMCP::CPBlockAddress,
-                                                         PCAdj);
+    ARMConstantPoolValue *CPV =
+      ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
+                                      ARMCP::CPBlockAddress, PCAdj);
     CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
   }
   CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
@@ -1966,8 +1981,8 @@ ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
   ARMConstantPoolValue *CPV =
-    new ARMConstantPoolValue(GA->getGlobal(), ARMPCLabelIndex,
-                             ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
+    ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
+                                    ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
   SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4);
   Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
   Argument = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Argument,
@@ -1982,11 +1997,11 @@ ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
   ArgListTy Args;
   ArgListEntry Entry;
   Entry.Node = Argument;
-  Entry.Ty = (const Type *) Type::getInt32Ty(*DAG.getContext());
+  Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
   Args.push_back(Entry);
   // FIXME: is there useful debug info available here?
   std::pair<SDValue, SDValue> CallResult =
-    LowerCallTo(Chain, (const Type *) Type::getInt32Ty(*DAG.getContext()),
+    LowerCallTo(Chain, (Type *) Type::getInt32Ty(*DAG.getContext()),
                 false, false, false, false,
                 0, CallingConv::C, false, /*isReturnValueUsed=*/true,
                 DAG.getExternalSymbol("__tls_get_addr", PtrVT), Args, DAG, dl);
@@ -2013,8 +2028,9 @@ ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
     // Initial exec model.
     unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
     ARMConstantPoolValue *CPV =
-      new ARMConstantPoolValue(GA->getGlobal(), ARMPCLabelIndex,
-                               ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF, true);
+      ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
+                                      ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
+                                      true);
     Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
     Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
     Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
@@ -2030,7 +2046,8 @@ ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
                          false, false, 0);
   } else {
     // local exec model
-    ARMConstantPoolValue *CPV = new ARMConstantPoolValue(GV, ARMCP::TPOFF);
+    ARMConstantPoolValue *CPV =
+      ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF);
     Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
     Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
     Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
@@ -2066,7 +2083,8 @@ SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
   if (RelocM == Reloc::PIC_) {
     bool UseGOTOFF = GV->hasLocalLinkage() || GV->hasHiddenVisibility();
     ARMConstantPoolValue *CPV =
-      new ARMConstantPoolValue(GV, UseGOTOFF ? ARMCP::GOTOFF : ARMCP::GOT);
+      ARMConstantPoolConstant::Create(GV,
+                                      UseGOTOFF ? ARMCP::GOTOFF : ARMCP::GOT);
     SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
     CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
     SDValue Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
@@ -2135,7 +2153,8 @@ SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
     ARMPCLabelIndex = AFI->createPICLabelUId();
     unsigned PCAdj = (RelocM != Reloc::PIC_) ? 0 : (Subtarget->isThumb()?4:8);
     ARMConstantPoolValue *CPV =
-      new ARMConstantPoolValue(GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj);
+      ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue,
+                                      PCAdj);
     CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
   }
   CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
@@ -2167,9 +2186,9 @@ SDValue ARMTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op,
   EVT PtrVT = getPointerTy();
   DebugLoc dl = Op.getDebugLoc();
   unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
-  ARMConstantPoolValue *CPV = new ARMConstantPoolValue(*DAG.getContext(),
-                                                       "_GLOBAL_OFFSET_TABLE_",
-                                                       ARMPCLabelIndex, PCAdj);
+  ARMConstantPoolValue *CPV =
+    ARMConstantPoolSymbol::Create(*DAG.getContext(), "_GLOBAL_OFFSET_TABLE_",
+                                  ARMPCLabelIndex, PCAdj);
   SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
   CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
   SDValue Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr,
@@ -2191,7 +2210,8 @@ SDValue
 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
   DebugLoc dl = Op.getDebugLoc();
   SDValue Val = DAG.getConstant(0, MVT::i32);
-  return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl, MVT::i32, Op.getOperand(0),
+  return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
+                     DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
                      Op.getOperand(1), Val);
 }
 
@@ -2224,8 +2244,8 @@ ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
     unsigned PCAdj = (RelocM != Reloc::PIC_)
       ? 0 : (Subtarget->isThumb() ? 4 : 8);
     ARMConstantPoolValue *CPV =
-      new ARMConstantPoolValue(MF.getFunction(), ARMPCLabelIndex,
-                               ARMCP::CPLSDA, PCAdj);
+      ARMConstantPoolConstant::Create(MF.getFunction(), ARMPCLabelIndex,
+                                      ARMCP::CPLSDA, PCAdj);
     CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
     CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
     SDValue Result =
@@ -2277,6 +2297,25 @@ static SDValue LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG,
                      DAG.getConstant(DMBOpt, MVT::i32));
 }
 
+
+static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
+                                 const ARMSubtarget *Subtarget) {
+  // FIXME: handle "fence singlethread" more efficiently.
+  DebugLoc dl = Op.getDebugLoc();
+  if (!Subtarget->hasDataBarrier()) {
+    // Some ARMv6 cpus can support data barriers with an mcr instruction.
+    // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
+    // here.
+    assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
+           "Unexpected ISD::MEMBARRIER encountered. Should be libcall!");
+    return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
+                       DAG.getConstant(0, MVT::i32));
+  }
+
+  return DAG.getNode(ARMISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0),
+                     DAG.getConstant(ARM_MB::ISH, MVT::i32));
+}
+
 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
                              const ARMSubtarget *Subtarget) {
   // ARM pre v5TE and Thumb1 does not have preload instructions.
@@ -2754,7 +2793,7 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
     SDValue ARMcc;
     SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
     SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
-    return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp);
+    return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR,Cmp);
   }
 
   ARMCC::CondCodes CondCode, CondCode2;
@@ -2993,8 +3032,8 @@ static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
   DebugLoc dl = Op.getDebugLoc();
 
-  EVT OperandVT = Op.getOperand(0).getValueType();
-  assert(OperandVT == MVT::v4i16 && "Invalid type for custom lowering!");
+  assert(Op.getOperand(0).getValueType() == MVT::v4i16 &&
+         "Invalid type for custom lowering!");
   if (VT != MVT::v4f32)
     return DAG.UnrollVectorOp(Op.getNode());
 
@@ -3905,8 +3944,7 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       }
 
       // Try an immediate VMVN.
-      uint64_t NegatedImm = (SplatBits.getZExtValue() ^
-                             ((1LL << SplatBitSize) - 1));
+      uint64_t NegatedImm = (~SplatBits).getZExtValue();
       Val = isNEONModifiedImm(NegatedImm,
                                       SplatUndef.getZExtValue(), SplatBitSize,
                                       DAG, VmovVT, VT.is128BitVector(),
@@ -4019,6 +4057,14 @@ SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
       // A shuffle can only come from building a vector from various
       // elements of other vectors.
       return SDValue();
+    } else if (V.getOperand(0).getValueType().getVectorElementType() !=
+               VT.getVectorElementType()) {
+      // This code doesn't know how to handle shuffles where the vector
+      // element types do not match (this happens because type legalization
+      // promotes the return type of EXTRACT_VECTOR_ELT).
+      // FIXME: It might be appropriate to extend this code to handle
+      // mismatched types.
+      return SDValue();
     }
 
     // Record this extraction against the appropriate vector if possible...
@@ -4819,6 +4865,71 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
   return N0;
 }
 
+static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
+  EVT VT = Op.getNode()->getValueType(0);
+  SDVTList VTs = DAG.getVTList(VT, MVT::i32);
+
+  unsigned Opc;
+  bool ExtraOp = false;
+  switch (Op.getOpcode()) {
+  default: assert(0 && "Invalid code");
+  case ISD::ADDC: Opc = ARMISD::ADDC; break;
+  case ISD::ADDE: Opc = ARMISD::ADDE; ExtraOp = true; break;
+  case ISD::SUBC: Opc = ARMISD::SUBC; break;
+  case ISD::SUBE: Opc = ARMISD::SUBE; ExtraOp = true; break;
+  }
+
+  if (!ExtraOp)
+    return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0),
+                       Op.getOperand(1));
+  return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0),
+                     Op.getOperand(1), Op.getOperand(2));
+}
+
+static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
+  // Monotonic load/store is legal for all targets
+  if (cast<AtomicSDNode>(Op)->getOrdering() <= Monotonic)
+    return Op;
+
+  // Aquire/Release load/store is not legal for targets without a
+  // dmb or equivalent available.
+  return SDValue();
+}
+
+
+static void
+ReplaceATOMIC_OP_64(SDNode *Node, SmallVectorImpl<SDValue>& Results,
+                    SelectionDAG &DAG, unsigned NewOp) {
+  DebugLoc dl = Node->getDebugLoc();
+  assert (Node->getValueType(0) == MVT::i64 &&
+          "Only know how to expand i64 atomics");
+
+  SmallVector<SDValue, 6> Ops;
+  Ops.push_back(Node->getOperand(0)); // Chain
+  Ops.push_back(Node->getOperand(1)); // Ptr
+  // Low part of Val1
+  Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
+                            Node->getOperand(2), DAG.getIntPtrConstant(0)));
+  // High part of Val1
+  Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
+                            Node->getOperand(2), DAG.getIntPtrConstant(1)));
+  if (NewOp == ARMISD::ATOMCMPXCHG64_DAG) {
+    // High part of Val1
+    Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
+                              Node->getOperand(3), DAG.getIntPtrConstant(0)));
+    // High part of Val2
+    Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
+                              Node->getOperand(3), DAG.getIntPtrConstant(1)));
+  }
+  SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
+  SDValue Result =
+    DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops.data(), Ops.size(), MVT::i64,
+                            cast<MemSDNode>(Node)->getMemOperand());
+  SDValue OpsF[] = { Result.getValue(0), Result.getValue(1) };
+  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
+  Results.push_back(Result.getValue(2));
+}
+
 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Don't know how to custom lower this!");
@@ -4834,6 +4945,7 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::BR_JT:         return LowerBR_JT(Op, DAG);
   case ISD::VASTART:       return LowerVASTART(Op, DAG);
   case ISD::MEMBARRIER:    return LowerMEMBARRIER(Op, DAG, Subtarget);
+  case ISD::ATOMIC_FENCE:  return LowerATOMIC_FENCE(Op, DAG, Subtarget);
   case ISD::PREFETCH:      return LowerPREFETCH(Op, DAG, Subtarget);
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:    return LowerINT_TO_FP(Op, DAG);
@@ -4856,7 +4968,7 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SRL_PARTS:
   case ISD::SRA_PARTS:     return LowerShiftRightParts(Op, DAG);
   case ISD::CTTZ:          return LowerCTTZ(Op.getNode(), DAG, Subtarget);
-  case ISD::VSETCC:        return LowerVSETCC(Op, DAG);
+  case ISD::SETCC:         return LowerVSETCC(Op, DAG);
   case ISD::BUILD_VECTOR:  return LowerBUILD_VECTOR(Op, DAG, Subtarget);
   case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
@@ -4865,6 +4977,12 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::MUL:           return LowerMUL(Op, DAG);
   case ISD::SDIV:          return LowerSDIV(Op, DAG);
   case ISD::UDIV:          return LowerUDIV(Op, DAG);
+  case ISD::ADDC:
+  case ISD::ADDE:
+  case ISD::SUBC:
+  case ISD::SUBE:          return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
+  case ISD::ATOMIC_LOAD:
+  case ISD::ATOMIC_STORE:  return LowerAtomicLoadStore(Op, DAG);
   }
   return SDValue();
 }
@@ -4886,6 +5004,30 @@ void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::SRA:
     Res = Expand64BitShift(N, DAG, Subtarget);
     break;
+  case ISD::ATOMIC_LOAD_ADD:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMADD64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_AND:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMAND64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_NAND:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMNAND64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_OR:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMOR64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_SUB:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMSUB64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_XOR:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMXOR64_DAG);
+    return;
+  case ISD::ATOMIC_SWAP:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMSWAP64_DAG);
+    return;
+  case ISD::ATOMIC_CMP_SWAP:
+    ReplaceATOMIC_OP_64(N, Results, DAG, ARMISD::ATOMCMPXCHG64_DAG);
+    return;
   }
   if (Res.getNode())
     Results.push_back(Res);
@@ -4963,7 +5105,10 @@ ARMTargetLowering::EmitAtomicCmpSwap(MachineInstr *MI,
   //   cmp dest, oldval
   //   bne exitMBB
   BB = loop1MBB;
-  AddDefaultPred(BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr));
+  MachineInstrBuilder MIB = BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
+  if (ldrOpc == ARM::t2LDREX)
+    MIB.addImm(0);
+  AddDefaultPred(MIB);
   AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
                  .addReg(dest).addReg(oldval));
   BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
@@ -4976,8 +5121,10 @@ ARMTargetLowering::EmitAtomicCmpSwap(MachineInstr *MI,
   //   cmp scratch, #0
   //   bne loop1MBB
   BB = loop2MBB;
-  AddDefaultPred(BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(newval)
-                 .addReg(ptr));
+  MIB = BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(newval).addReg(ptr);
+  if (strOpc == ARM::t2STREX)
+    MIB.addImm(0);
+  AddDefaultPred(MIB);
   AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
                  .addReg(scratch).addImm(0));
   BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
@@ -5063,7 +5210,10 @@ ARMTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
   //   bne- loopMBB
   //   fallthrough --> exitMBB
   BB = loopMBB;
-  AddDefaultPred(BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr));
+  MachineInstrBuilder MIB = BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
+  if (ldrOpc == ARM::t2LDREX)
+    MIB.addImm(0);
+  AddDefaultPred(MIB);
   if (BinOpcode) {
     // operand order needs to go the other way for NAND
     if (BinOpcode == ARM::BICrr || BinOpcode == ARM::t2BICrr)
@@ -5074,8 +5224,10 @@ ARMTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
                      addReg(dest).addReg(incr)).addReg(0);
   }
 
-  AddDefaultPred(BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(scratch2)
-                 .addReg(ptr));
+  MIB = BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(scratch2).addReg(ptr);
+  if (strOpc == ARM::t2STREX)
+    MIB.addImm(0);
+  AddDefaultPred(MIB);
   AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
                  .addReg(scratch).addImm(0));
   BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
@@ -5125,12 +5277,12 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   case 1:
     ldrOpc = isThumb2 ? ARM::t2LDREXB : ARM::LDREXB;
     strOpc = isThumb2 ? ARM::t2STREXB : ARM::STREXB;
-    extendOpc = isThumb2 ? ARM::t2SXTBr : ARM::SXTBr;
+    extendOpc = isThumb2 ? ARM::t2SXTB : ARM::SXTB;
     break;
   case 2:
     ldrOpc = isThumb2 ? ARM::t2LDREXH : ARM::LDREXH;
     strOpc = isThumb2 ? ARM::t2STREXH : ARM::STREXH;
-    extendOpc = isThumb2 ? ARM::t2SXTHr : ARM::SXTHr;
+    extendOpc = isThumb2 ? ARM::t2SXTH : ARM::SXTH;
     break;
   case 4:
     ldrOpc = isThumb2 ? ARM::t2LDREX : ARM::LDREX;
@@ -5170,12 +5322,17 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   //   bne- loopMBB
   //   fallthrough --> exitMBB
   BB = loopMBB;
-  AddDefaultPred(BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr));
+  MachineInstrBuilder MIB = BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
+  if (ldrOpc == ARM::t2LDREX)
+    MIB.addImm(0);
+  AddDefaultPred(MIB);
 
   // Sign extend the value, if necessary.
   if (signExtend && extendOpc) {
     oldval = MRI.createVirtualRegister(ARM::GPRRegisterClass);
-    AddDefaultPred(BuildMI(BB, dl, TII->get(extendOpc), oldval).addReg(dest));
+    AddDefaultPred(BuildMI(BB, dl, TII->get(extendOpc), oldval)
+                     .addReg(dest)
+                     .addImm(0));
   }
 
   // Build compare and cmov instructions.
@@ -5184,8 +5341,10 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2MOVCCr : ARM::MOVCCr), scratch2)
          .addReg(oldval).addReg(incr).addImm(Cond).addReg(ARM::CPSR);
 
-  AddDefaultPred(BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(scratch2)
-                 .addReg(ptr));
+  MIB = BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(scratch2).addReg(ptr);
+  if (strOpc == ARM::t2STREX)
+    MIB.addImm(0);
+  AddDefaultPred(MIB);
   AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
                  .addReg(scratch).addImm(0));
   BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
@@ -5203,6 +5362,589 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   return BB;
 }
 
+MachineBasicBlock *
+ARMTargetLowering::EmitAtomicBinary64(MachineInstr *MI, MachineBasicBlock *BB,
+                                      unsigned Op1, unsigned Op2,
+                                      bool NeedsCarry, bool IsCmpxchg) const {
+  // This also handles ATOMIC_SWAP, indicated by Op1==0.
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+
+  const BasicBlock *LLVM_BB = BB->getBasicBlock();
+  MachineFunction *MF = BB->getParent();
+  MachineFunction::iterator It = BB;
+  ++It;
+
+  unsigned destlo = MI->getOperand(0).getReg();
+  unsigned desthi = MI->getOperand(1).getReg();
+  unsigned ptr = MI->getOperand(2).getReg();
+  unsigned vallo = MI->getOperand(3).getReg();
+  unsigned valhi = MI->getOperand(4).getReg();
+  DebugLoc dl = MI->getDebugLoc();
+  bool isThumb2 = Subtarget->isThumb2();
+
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+  if (isThumb2) {
+    MRI.constrainRegClass(destlo, ARM::rGPRRegisterClass);
+    MRI.constrainRegClass(desthi, ARM::rGPRRegisterClass);
+    MRI.constrainRegClass(ptr, ARM::rGPRRegisterClass);
+  }
+
+  unsigned ldrOpc = isThumb2 ? ARM::t2LDREXD : ARM::LDREXD;
+  unsigned strOpc = isThumb2 ? ARM::t2STREXD : ARM::STREXD;
+
+  MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *contBB = 0, *cont2BB = 0;
+  if (IsCmpxchg) {
+    contBB = MF->CreateMachineBasicBlock(LLVM_BB);
+    cont2BB = MF->CreateMachineBasicBlock(LLVM_BB);
+  }
+  MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  MF->insert(It, loopMBB);
+  if (IsCmpxchg) {
+    MF->insert(It, contBB);
+    MF->insert(It, cont2BB);
+  }
+  MF->insert(It, exitMBB);
+
+  // Transfer the remainder of BB and its successor edges to exitMBB.
+  exitMBB->splice(exitMBB->begin(), BB,
+                  llvm::next(MachineBasicBlock::iterator(MI)),
+                  BB->end());
+  exitMBB->transferSuccessorsAndUpdatePHIs(BB);
+
+  TargetRegisterClass *TRC =
+    isThumb2 ? ARM::tGPRRegisterClass : ARM::GPRRegisterClass;
+  unsigned storesuccess = MRI.createVirtualRegister(TRC);
+
+  //  thisMBB:
+  //   ...
+  //   fallthrough --> loopMBB
+  BB->addSuccessor(loopMBB);
+
+  //  loopMBB:
+  //   ldrexd r2, r3, ptr
+  //   <binopa> r0, r2, incr
+  //   <binopb> r1, r3, incr
+  //   strexd storesuccess, r0, r1, ptr
+  //   cmp storesuccess, #0
+  //   bne- loopMBB
+  //   fallthrough --> exitMBB
+  //
+  // Note that the registers are explicitly specified because there is not any
+  // way to force the register allocator to allocate a register pair.
+  //
+  // FIXME: The hardcoded registers are not necessary for Thumb2, but we
+  // need to properly enforce the restriction that the two output registers
+  // for ldrexd must be different.
+  BB = loopMBB;
+  // Load
+  AddDefaultPred(BuildMI(BB, dl, TII->get(ldrOpc))
+                 .addReg(ARM::R2, RegState::Define)
+                 .addReg(ARM::R3, RegState::Define).addReg(ptr));
+  // Copy r2/r3 into dest.  (This copy will normally be coalesced.)
+  BuildMI(BB, dl, TII->get(TargetOpcode::COPY), destlo).addReg(ARM::R2);
+  BuildMI(BB, dl, TII->get(TargetOpcode::COPY), desthi).addReg(ARM::R3);
+
+  if (IsCmpxchg) {
+    // Add early exit
+    for (unsigned i = 0; i < 2; i++) {
+      AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr :
+                                                         ARM::CMPrr))
+                     .addReg(i == 0 ? destlo : desthi)
+                     .addReg(i == 0 ? vallo : valhi));
+      BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
+        .addMBB(exitMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
+      BB->addSuccessor(exitMBB);
+      BB->addSuccessor(i == 0 ? contBB : cont2BB);
+      BB = (i == 0 ? contBB : cont2BB);
+    }
+
+    // Copy to physregs for strexd
+    unsigned setlo = MI->getOperand(5).getReg();
+    unsigned sethi = MI->getOperand(6).getReg();
+    BuildMI(BB, dl, TII->get(TargetOpcode::COPY), ARM::R0).addReg(setlo);
+    BuildMI(BB, dl, TII->get(TargetOpcode::COPY), ARM::R1).addReg(sethi);
+  } else if (Op1) {
+    // Perform binary operation
+    AddDefaultPred(BuildMI(BB, dl, TII->get(Op1), ARM::R0)
+                   .addReg(destlo).addReg(vallo))
+        .addReg(NeedsCarry ? ARM::CPSR : 0, getDefRegState(NeedsCarry));
+    AddDefaultPred(BuildMI(BB, dl, TII->get(Op2), ARM::R1)
+                   .addReg(desthi).addReg(valhi)).addReg(0);
+  } else {
+    // Copy to physregs for strexd
+    BuildMI(BB, dl, TII->get(TargetOpcode::COPY), ARM::R0).addReg(vallo);
+    BuildMI(BB, dl, TII->get(TargetOpcode::COPY), ARM::R1).addReg(valhi);
+  }
+
+  // Store
+  AddDefaultPred(BuildMI(BB, dl, TII->get(strOpc), storesuccess)
+                 .addReg(ARM::R0).addReg(ARM::R1).addReg(ptr));
+  // Cmp+jump
+  AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
+                 .addReg(storesuccess).addImm(0));
+  BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
+    .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
+
+  BB->addSuccessor(loopMBB);
+  BB->addSuccessor(exitMBB);
+
+  //  exitMBB:
+  //   ...
+  BB = exitMBB;
+
+  MI->eraseFromParent();   // The instruction is gone now.
+
+  return BB;
+}
+
+/// EmitBasePointerRecalculation - For functions using a base pointer, we
+/// rematerialize it (via the frame pointer).
+void ARMTargetLowering::
+EmitBasePointerRecalculation(MachineInstr *MI, MachineBasicBlock *MBB,
+                             MachineBasicBlock *DispatchBB) const {
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+  const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
+  MachineFunction &MF = *MI->getParent()->getParent();
+  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
+
+  if (!RI.hasBasePointer(MF)) return;
+
+  MachineBasicBlock::iterator MBBI = MI;
+
+  int32_t NumBytes = AFI->getFramePtrSpillOffset();
+  unsigned FramePtr = RI.getFrameRegister(MF);
+  assert(MF.getTarget().getFrameLowering()->hasFP(MF) &&
+         "Base pointer without frame pointer?");
+
+  if (AFI->isThumb2Function())
+    llvm::emitT2RegPlusImmediate(*MBB, MBBI, MI->getDebugLoc(), ARM::R6,
+                                 FramePtr, -NumBytes, ARMCC::AL, 0, *AII);
+  else if (AFI->isThumbFunction())
+    llvm::emitThumbRegPlusImmediate(*MBB, MBBI, MI->getDebugLoc(), ARM::R6,
+                                    FramePtr, -NumBytes, *AII, RI);
+  else
+    llvm::emitARMRegPlusImmediate(*MBB, MBBI, MI->getDebugLoc(), ARM::R6,
+                                  FramePtr, -NumBytes, ARMCC::AL, 0, *AII);
+
+  if (!RI.needsStackRealignment(MF)) return;
+
+  // If there's dynamic realignment, adjust for it.
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  unsigned MaxAlign = MFI->getMaxAlignment();
+  assert(!AFI->isThumb1OnlyFunction());
+
+  // Emit bic r6, r6, MaxAlign
+  unsigned bicOpc = AFI->isThumbFunction() ? ARM::t2BICri : ARM::BICri;
+  AddDefaultCC(
+    AddDefaultPred(
+      BuildMI(*MBB, MBBI, MI->getDebugLoc(), TII->get(bicOpc), ARM::R6)
+      .addReg(ARM::R6, RegState::Kill)
+      .addImm(MaxAlign - 1)));
+}
+
+/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
+/// registers the function context.
+void ARMTargetLowering::
+SetupEntryBlockForSjLj(MachineInstr *MI, MachineBasicBlock *MBB,
+                       MachineBasicBlock *DispatchBB, int FI) const {
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+  DebugLoc dl = MI->getDebugLoc();
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo *MRI = &MF->getRegInfo();
+  MachineConstantPool *MCP = MF->getConstantPool();
+  ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
+  const Function *F = MF->getFunction();
+
+  bool isThumb = Subtarget->isThumb();
+  bool isThumb2 = Subtarget->isThumb2();
+
+  unsigned PCLabelId = AFI->createPICLabelUId();
+  unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
+  ARMConstantPoolValue *CPV =
+    ARMConstantPoolMBB::Create(F->getContext(), DispatchBB, PCLabelId, PCAdj);
+  unsigned CPI = MCP->getConstantPoolIndex(CPV, 4);
+
+  const TargetRegisterClass *TRC =
+    isThumb ? ARM::tGPRRegisterClass : ARM::GPRRegisterClass;
+
+  // Grab constant pool and fixed stack memory operands.
+  MachineMemOperand *CPMMO =
+    MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(),
+                             MachineMemOperand::MOLoad, 4, 4);
+
+  MachineMemOperand *FIMMOSt =
+    MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(FI),
+                             MachineMemOperand::MOStore, 4, 4);
+
+  EmitBasePointerRecalculation(MI, MBB, DispatchBB);
+
+  // Load the address of the dispatch MBB into the jump buffer.
+  if (isThumb2) {
+    // Incoming value: jbuf
+    //   ldr.n  r5, LCPI1_1
+    //   orr    r5, r5, #1
+    //   add    r5, pc
+    //   str    r5, [$jbuf, #+4] ; &jbuf[1]
+    unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1)
+                   .addConstantPoolIndex(CPI)
+                   .addMemOperand(CPMMO));
+    // Set the low bit because of thumb mode.
+    unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
+    AddDefaultCC(
+      AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2)
+                     .addReg(NewVReg1, RegState::Kill)
+                     .addImm(0x01)));
+    unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
+    BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3)
+      .addReg(NewVReg2, RegState::Kill)
+      .addImm(PCLabelId);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12))
+                   .addReg(NewVReg3, RegState::Kill)
+                   .addFrameIndex(FI)
+                   .addImm(36)  // &jbuf[1] :: pc
+                   .addMemOperand(FIMMOSt));
+  } else if (isThumb) {
+    // Incoming value: jbuf
+    //   ldr.n  r1, LCPI1_4
+    //   add    r1, pc
+    //   mov    r2, #1
+    //   orrs   r1, r2
+    //   add    r2, $jbuf, #+4 ; &jbuf[1]
+    //   str    r1, [r2]
+    unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1)
+                   .addConstantPoolIndex(CPI)
+                   .addMemOperand(CPMMO));
+    unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
+    BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2)
+      .addReg(NewVReg1, RegState::Kill)
+      .addImm(PCLabelId);
+    // Set the low bit because of thumb mode.
+    unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3)
+                   .addReg(ARM::CPSR, RegState::Define)
+                   .addImm(1));
+    unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4)
+                   .addReg(ARM::CPSR, RegState::Define)
+                   .addReg(NewVReg2, RegState::Kill)
+                   .addReg(NewVReg3, RegState::Kill));
+    unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tADDrSPi), NewVReg5)
+                   .addFrameIndex(FI)
+                   .addImm(36)); // &jbuf[1] :: pc
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi))
+                   .addReg(NewVReg4, RegState::Kill)
+                   .addReg(NewVReg5, RegState::Kill)
+                   .addImm(0)
+                   .addMemOperand(FIMMOSt));
+  } else {
+    // Incoming value: jbuf
+    //   ldr  r1, LCPI1_1
+    //   add  r1, pc, r1
+    //   str  r1, [$jbuf, #+4] ; &jbuf[1]
+    unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12),  NewVReg1)
+                   .addConstantPoolIndex(CPI)
+                   .addImm(0)
+                   .addMemOperand(CPMMO));
+    unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2)
+                   .addReg(NewVReg1, RegState::Kill)
+                   .addImm(PCLabelId));
+    AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12))
+                   .addReg(NewVReg2, RegState::Kill)
+                   .addFrameIndex(FI)
+                   .addImm(36)  // &jbuf[1] :: pc
+                   .addMemOperand(FIMMOSt));
+  }
+}
+
+MachineBasicBlock *ARMTargetLowering::
+EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+  DebugLoc dl = MI->getDebugLoc();
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo *MRI = &MF->getRegInfo();
+  ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
+  MachineFrameInfo *MFI = MF->getFrameInfo();
+  int FI = MFI->getFunctionContextIndex();
+
+  const TargetRegisterClass *TRC =
+    Subtarget->isThumb() ? ARM::tGPRRegisterClass : ARM::GPRRegisterClass;
+
+  // Get a mapping of the call site numbers to all of the landing pads they're
+  // associated with.
+  DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2> > CallSiteNumToLPad;
+  unsigned MaxCSNum = 0;
+  MachineModuleInfo &MMI = MF->getMMI();
+  for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E; ++BB) {
+    if (!BB->isLandingPad()) continue;
+
+    // FIXME: We should assert that the EH_LABEL is the first MI in the landing
+    // pad.
+    for (MachineBasicBlock::iterator
+           II = BB->begin(), IE = BB->end(); II != IE; ++II) {
+      if (!II->isEHLabel()) continue;
+
+      MCSymbol *Sym = II->getOperand(0).getMCSymbol();
+      if (!MMI.hasCallSiteLandingPad(Sym)) continue;
+
+      SmallVectorImpl<unsigned> &CallSiteIdxs = MMI.getCallSiteLandingPad(Sym);
+      for (SmallVectorImpl<unsigned>::iterator
+             CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end();
+           CSI != CSE; ++CSI) {
+        CallSiteNumToLPad[*CSI].push_back(BB);
+        MaxCSNum = std::max(MaxCSNum, *CSI);
+      }
+      break;
+    }
+  }
+
+  // Get an ordered list of the machine basic blocks for the jump table.
+  std::vector<MachineBasicBlock*> LPadList;
+  SmallPtrSet<MachineBasicBlock*, 64> InvokeBBs;
+  LPadList.reserve(CallSiteNumToLPad.size());
+  for (unsigned I = 1; I <= MaxCSNum; ++I) {
+    SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
+    for (SmallVectorImpl<MachineBasicBlock*>::iterator
+           II = MBBList.begin(), IE = MBBList.end(); II != IE; ++II) {
+      LPadList.push_back(*II);
+      InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end());
+    }
+  }
+
+  assert(!LPadList.empty() &&
+         "No landing pad destinations for the dispatch jump table!");
+
+  // Create the jump table and associated information.
+  MachineJumpTableInfo *JTI =
+    MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
+  unsigned MJTI = JTI->createJumpTableIndex(LPadList);
+  unsigned UId = AFI->createJumpTableUId();
+
+  // Create the MBBs for the dispatch code.
+
+  // Shove the dispatch's address into the return slot in the function context.
+  MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
+  DispatchBB->setIsLandingPad();
+
+  MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
+  BuildMI(TrapBB, dl, TII->get(Subtarget->isThumb() ? ARM::tTRAP : ARM::TRAP));
+  DispatchBB->addSuccessor(TrapBB);
+
+  MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
+  DispatchBB->addSuccessor(DispContBB);
+
+  // Insert and renumber MBBs.
+  MachineBasicBlock *Last = &MF->back();
+  MF->insert(MF->end(), DispatchBB);
+  MF->insert(MF->end(), DispContBB);
+  MF->insert(MF->end(), TrapBB);
+  MF->RenumberBlocks(Last);
+
+  // Insert code into the entry block that creates and registers the function
+  // context.
+  SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
+
+  MachineMemOperand *FIMMOLd =
+    MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(FI),
+                             MachineMemOperand::MOLoad |
+                             MachineMemOperand::MOVolatile, 4, 4);
+
+  if (Subtarget->isThumb2()) {
+    unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1)
+                   .addFrameIndex(FI)
+                   .addImm(4)
+                   .addMemOperand(FIMMOLd));
+    AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri))
+                   .addReg(NewVReg1)
+                   .addImm(LPadList.size()));
+    BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc))
+      .addMBB(TrapBB)
+      .addImm(ARMCC::HI)
+      .addReg(ARM::CPSR);
+
+    unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT),NewVReg2)
+                   .addJumpTableIndex(MJTI)
+                   .addImm(UId));
+
+    unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
+    AddDefaultCC(
+      AddDefaultPred(
+        BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg3)
+        .addReg(NewVReg2, RegState::Kill)
+        .addReg(NewVReg1)
+        .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
+
+    BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
+      .addReg(NewVReg3, RegState::Kill)
+      .addReg(NewVReg1)
+      .addJumpTableIndex(MJTI)
+      .addImm(UId);
+  } else if (Subtarget->isThumb()) {
+    unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
+                   .addFrameIndex(FI)
+                   .addImm(1)
+                   .addMemOperand(FIMMOLd));
+
+    AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8))
+                   .addReg(NewVReg1)
+                   .addImm(LPadList.size()));
+    BuildMI(DispatchBB, dl, TII->get(ARM::tBcc))
+      .addMBB(TrapBB)
+      .addImm(ARMCC::HI)
+      .addReg(ARM::CPSR);
+
+    unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2)
+                   .addReg(ARM::CPSR, RegState::Define)
+                   .addReg(NewVReg1)
+                   .addImm(2));
+
+    unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
+                   .addJumpTableIndex(MJTI)
+                   .addImm(UId));
+
+    unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
+                   .addReg(ARM::CPSR, RegState::Define)
+                   .addReg(NewVReg2, RegState::Kill)
+                   .addReg(NewVReg3));
+
+    MachineMemOperand *JTMMOLd =
+      MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(),
+                               MachineMemOperand::MOLoad, 4, 4);
+
+    unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
+                   .addReg(NewVReg4, RegState::Kill)
+                   .addImm(0)
+                   .addMemOperand(JTMMOLd));
+
+    unsigned NewVReg6 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6)
+                   .addReg(ARM::CPSR, RegState::Define)
+                   .addReg(NewVReg5, RegState::Kill)
+                   .addReg(NewVReg3));
+
+    BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
+      .addReg(NewVReg6, RegState::Kill)
+      .addJumpTableIndex(MJTI)
+      .addImm(UId);
+  } else {
+    unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
+                   .addFrameIndex(FI)
+                   .addImm(4)
+                   .addMemOperand(FIMMOLd));
+    AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPri))
+                   .addReg(NewVReg1)
+                   .addImm(LPadList.size()));
+    BuildMI(DispatchBB, dl, TII->get(ARM::Bcc))
+      .addMBB(TrapBB)
+      .addImm(ARMCC::HI)
+      .addReg(ARM::CPSR);
+
+    unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
+    AddDefaultCC(
+      AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg2)
+                     .addReg(NewVReg1)
+                     .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
+    unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg3)
+                   .addJumpTableIndex(MJTI)
+                   .addImm(UId));
+
+    MachineMemOperand *JTMMOLd =
+      MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(),
+                               MachineMemOperand::MOLoad, 4, 4);
+    unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
+    AddDefaultPred(
+      BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg4)
+      .addReg(NewVReg2, RegState::Kill)
+      .addReg(NewVReg3)
+      .addImm(0)
+      .addMemOperand(JTMMOLd));
+
+    BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
+      .addReg(NewVReg4, RegState::Kill)
+      .addReg(NewVReg3)
+      .addJumpTableIndex(MJTI)
+      .addImm(UId);
+  }
+
+  // Add the jump table entries as successors to the MBB.
+  MachineBasicBlock *PrevMBB = 0;
+  for (std::vector<MachineBasicBlock*>::iterator
+         I = LPadList.begin(), E = LPadList.end(); I != E; ++I) {
+    MachineBasicBlock *CurMBB = *I;
+    if (PrevMBB != CurMBB)
+      DispContBB->addSuccessor(CurMBB);
+    PrevMBB = CurMBB;
+  }
+
+  const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
+  const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
+  const unsigned *SavedRegs = RI.getCalleeSavedRegs(MF);
+  for (SmallPtrSet<MachineBasicBlock*, 64>::iterator
+         I = InvokeBBs.begin(), E = InvokeBBs.end(); I != E; ++I) {
+    MachineBasicBlock *BB = *I;
+
+    // Remove the landing pad successor from the invoke block and replace it
+    // with the new dispatch block.
+    for (MachineBasicBlock::succ_iterator
+           SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) {
+      MachineBasicBlock *SMBB = *SI;
+      if (SMBB->isLandingPad()) {
+        BB->removeSuccessor(SMBB);
+        SMBB->setIsLandingPad(false);
+      }
+    }
+
+    BB->addSuccessor(DispatchBB);
+
+    // Find the invoke call and mark all of the callee-saved registers as
+    // 'implicit defined' so that they're spilled. This prevents code from
+    // moving instructions to before the EH block, where they will never be
+    // executed.
+    for (MachineBasicBlock::reverse_iterator
+           II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
+      if (!II->getDesc().isCall()) continue;
+
+      DenseMap<unsigned, bool> DefRegs;
+      for (MachineInstr::mop_iterator
+             OI = II->operands_begin(), OE = II->operands_end();
+           OI != OE; ++OI) {
+        if (!OI->isReg()) continue;
+        DefRegs[OI->getReg()] = true;
+      }
+
+      MachineInstrBuilder MIB(&*II);
+
+      for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
+        if (!TRC->contains(SavedRegs[i])) continue;
+        if (!DefRegs[SavedRegs[i]])
+          MIB.addReg(SavedRegs[i], RegState::ImplicitDefine | RegState::Dead);
+      }
+
+      break;
+    }
+  }
+
+  // The instruction is gone now.
+  MI->eraseFromParent();
+
+  return MBB;
+}
+
 static
 MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
   for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(),
@@ -5212,72 +5954,6 @@ MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
   llvm_unreachable("Expecting a BB with two successors!");
 }
 
-// FIXME: This opcode table should obviously be expressed in the target
-// description. We probably just need a "machine opcode" value in the pseudo
-// instruction. But the ideal solution maybe to simply remove the "S" version
-// of the opcode altogether.
-struct AddSubFlagsOpcodePair {
-  unsigned PseudoOpc;
-  unsigned MachineOpc;
-};
-
-static AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
-  {ARM::ADCSri, ARM::ADCri},
-  {ARM::ADCSrr, ARM::ADCrr},
-  {ARM::ADCSrs, ARM::ADCrs},
-  {ARM::SBCSri, ARM::SBCri},
-  {ARM::SBCSrr, ARM::SBCrr},
-  {ARM::SBCSrs, ARM::SBCrs},
-  {ARM::RSBSri, ARM::RSBri},
-  {ARM::RSBSrr, ARM::RSBrr},
-  {ARM::RSBSrs, ARM::RSBrs},
-  {ARM::RSCSri, ARM::RSCri},
-  {ARM::RSCSrs, ARM::RSCrs},
-  {ARM::t2ADCSri, ARM::t2ADCri},
-  {ARM::t2ADCSrr, ARM::t2ADCrr},
-  {ARM::t2ADCSrs, ARM::t2ADCrs},
-  {ARM::t2SBCSri, ARM::t2SBCri},
-  {ARM::t2SBCSrr, ARM::t2SBCrr},
-  {ARM::t2SBCSrs, ARM::t2SBCrs},
-  {ARM::t2RSBSri, ARM::t2RSBri},
-  {ARM::t2RSBSrs, ARM::t2RSBrs},
-};
-
-// Convert and Add or Subtract with Carry and Flags to a generic opcode with
-// CPSR<def> operand. e.g. ADCS (...) -> ADC (... CPSR<def>).
-//
-// FIXME: Somewhere we should assert that CPSR<def> is in the correct
-// position to be recognized by the target descrition as the 'S' bit.
-bool ARMTargetLowering::RemapAddSubWithFlags(MachineInstr *MI,
-                                             MachineBasicBlock *BB) const {
-  unsigned OldOpc = MI->getOpcode();
-  unsigned NewOpc = 0;
-
-  // This is only called for instructions that need remapping, so iterating over
-  // the tiny opcode table is not costly.
-  static const int NPairs =
-    sizeof(AddSubFlagsOpcodeMap) / sizeof(AddSubFlagsOpcodePair);
-  for (AddSubFlagsOpcodePair *Pair = &AddSubFlagsOpcodeMap[0],
-         *End = &AddSubFlagsOpcodeMap[NPairs]; Pair != End; ++Pair) {
-    if (OldOpc == Pair->PseudoOpc) {
-      NewOpc = Pair->MachineOpc;
-      break;
-    }
-  }
-  if (!NewOpc)
-    return false;
-
-  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-  DebugLoc dl = MI->getDebugLoc();
-  MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
-  for (unsigned i = 0; i < MI->getNumOperands(); ++i)
-    MIB.addOperand(MI->getOperand(i));
-  AddDefaultPred(MIB);
-  MIB.addReg(ARM::CPSR, RegState::Define); // S bit
-  MI->eraseFromParent();
-  return true;
-}
-
 MachineBasicBlock *
 ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
                                                MachineBasicBlock *BB) const {
@@ -5286,12 +5962,61 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   bool isThumb2 = Subtarget->isThumb2();
   switch (MI->getOpcode()) {
   default: {
-    if (RemapAddSubWithFlags(MI, BB))
-      return BB;
-
     MI->dump();
     llvm_unreachable("Unexpected instr type to insert");
   }
+  // The Thumb2 pre-indexed stores have the same MI operands, they just
+  // define them differently in the .td files from the isel patterns, so
+  // they need pseudos.
+  case ARM::t2STR_preidx:
+    MI->setDesc(TII->get(ARM::t2STR_PRE));
+    return BB;
+  case ARM::t2STRB_preidx:
+    MI->setDesc(TII->get(ARM::t2STRB_PRE));
+    return BB;
+  case ARM::t2STRH_preidx:
+    MI->setDesc(TII->get(ARM::t2STRH_PRE));
+    return BB;
+
+  case ARM::STRi_preidx:
+  case ARM::STRBi_preidx: {
+    unsigned NewOpc = MI->getOpcode() == ARM::STRi_preidx ?
+      ARM::STR_PRE_IMM : ARM::STRB_PRE_IMM;
+    // Decode the offset.
+    unsigned Offset = MI->getOperand(4).getImm();
+    bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub;
+    Offset = ARM_AM::getAM2Offset(Offset);
+    if (isSub)
+      Offset = -Offset;
+
+    MachineMemOperand *MMO = *MI->memoperands_begin();
+    BuildMI(*BB, MI, dl, TII->get(NewOpc))
+      .addOperand(MI->getOperand(0))  // Rn_wb
+      .addOperand(MI->getOperand(1))  // Rt
+      .addOperand(MI->getOperand(2))  // Rn
+      .addImm(Offset)                 // offset (skip GPR==zero_reg)
+      .addOperand(MI->getOperand(5))  // pred
+      .addOperand(MI->getOperand(6))
+      .addMemOperand(MMO);
+    MI->eraseFromParent();
+    return BB;
+  }
+  case ARM::STRr_preidx:
+  case ARM::STRBr_preidx:
+  case ARM::STRH_preidx: {
+    unsigned NewOpc;
+    switch (MI->getOpcode()) {
+    default: llvm_unreachable("unexpected opcode!");
+    case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
+    case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
+    case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
+    }
+    MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
+    for (unsigned i = 0; i < MI->getNumOperands(); ++i)
+      MIB.addOperand(MI->getOperand(i));
+    MI->eraseFromParent();
+    return BB;
+  }
   case ARM::ATOMIC_LOAD_ADD_I8:
      return EmitAtomicBinary(MI, BB, 1, isThumb2 ? ARM::t2ADDrr : ARM::ADDrr);
   case ARM::ATOMIC_LOAD_ADD_I16:
@@ -5370,6 +6095,31 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case ARM::ATOMIC_CMP_SWAP_I16: return EmitAtomicCmpSwap(MI, BB, 2);
   case ARM::ATOMIC_CMP_SWAP_I32: return EmitAtomicCmpSwap(MI, BB, 4);
 
+
+  case ARM::ATOMADD6432:
+    return EmitAtomicBinary64(MI, BB, isThumb2 ? ARM::t2ADDrr : ARM::ADDrr,
+                              isThumb2 ? ARM::t2ADCrr : ARM::ADCrr,
+                              /*NeedsCarry*/ true);
+  case ARM::ATOMSUB6432:
+    return EmitAtomicBinary64(MI, BB, isThumb2 ? ARM::t2SUBrr : ARM::SUBrr,
+                              isThumb2 ? ARM::t2SBCrr : ARM::SBCrr,
+                              /*NeedsCarry*/ true);
+  case ARM::ATOMOR6432:
+    return EmitAtomicBinary64(MI, BB, isThumb2 ? ARM::t2ORRrr : ARM::ORRrr,
+                              isThumb2 ? ARM::t2ORRrr : ARM::ORRrr);
+  case ARM::ATOMXOR6432:
+    return EmitAtomicBinary64(MI, BB, isThumb2 ? ARM::t2EORrr : ARM::EORrr,
+                              isThumb2 ? ARM::t2EORrr : ARM::EORrr);
+  case ARM::ATOMAND6432:
+    return EmitAtomicBinary64(MI, BB, isThumb2 ? ARM::t2ANDrr : ARM::ANDrr,
+                              isThumb2 ? ARM::t2ANDrr : ARM::ANDrr);
+  case ARM::ATOMSWAP6432:
+    return EmitAtomicBinary64(MI, BB, 0, 0, false);
+  case ARM::ATOMCMPXCHG6432:
+    return EmitAtomicBinary64(MI, BB, isThumb2 ? ARM::t2SUBrr : ARM::SUBrr,
+                              isThumb2 ? ARM::t2SBCrr : ARM::SBCrr,
+                              /*NeedsCarry*/ false, /*IsCmpxchg*/true);
+
   case ARM::tMOVCCr_pseudo: {
     // To "insert" a SELECT_CC instruction, we actually have to insert the
     // diamond control-flow pattern.  The incoming instruction knows the
@@ -5461,13 +6211,159 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
 
     BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
       .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR);
-    BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2B : ARM::B))
-      .addMBB(exitMBB);
+    if (isThumb2)
+      AddDefaultPred(BuildMI(BB, dl, TII->get(ARM::t2B)).addMBB(exitMBB));
+    else
+      BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB);
 
     MI->eraseFromParent();   // The pseudo instruction is gone now.
     return BB;
   }
+
+  case ARM::ABS:
+  case ARM::t2ABS: {
+    // To insert an ABS instruction, we have to insert the
+    // diamond control-flow pattern.  The incoming instruction knows the
+    // source vreg to test against 0, the destination vreg to set,
+    // the condition code register to branch on, the
+    // true/false values to select between, and a branch opcode to use. 
+    // It transforms
+    //     V1 = ABS V0
+    // into
+    //     V2 = MOVS V0
+    //     BCC                      (branch to SinkBB if V0 >= 0)
+    //     RSBBB: V3 = RSBri V2, 0  (compute ABS if V2 < 0)
+    //     SinkBB: V1 = PHI(V2, V3)     
+    const BasicBlock *LLVM_BB = BB->getBasicBlock();
+    MachineFunction::iterator BBI = BB;
+    ++BBI;
+    MachineFunction *Fn = BB->getParent();
+    MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB);
+    MachineBasicBlock *SinkBB  = Fn->CreateMachineBasicBlock(LLVM_BB);
+    Fn->insert(BBI, RSBBB);
+    Fn->insert(BBI, SinkBB);
+
+    unsigned int ABSSrcReg = MI->getOperand(1).getReg();
+    unsigned int ABSDstReg = MI->getOperand(0).getReg();
+    bool isThumb2 = Subtarget->isThumb2();
+    MachineRegisterInfo &MRI = Fn->getRegInfo();
+    // In Thumb mode S must not be specified if source register is the SP or
+    // PC and if destination register is the SP, so restrict register class
+    unsigned NewMovDstReg = MRI.createVirtualRegister(
+      isThumb2 ? ARM::rGPRRegisterClass : ARM::GPRRegisterClass);
+    unsigned NewRsbDstReg = MRI.createVirtualRegister(
+      isThumb2 ? ARM::rGPRRegisterClass : ARM::GPRRegisterClass);
+
+    // Transfer the remainder of BB and its successor edges to sinkMBB.
+    SinkBB->splice(SinkBB->begin(), BB,
+      llvm::next(MachineBasicBlock::iterator(MI)),
+      BB->end());
+    SinkBB->transferSuccessorsAndUpdatePHIs(BB);
+
+    BB->addSuccessor(RSBBB);
+    BB->addSuccessor(SinkBB);
+
+    // fall through to SinkMBB
+    RSBBB->addSuccessor(SinkBB);
+
+    // insert a movs at the end of BB
+    BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2MOVr : ARM::MOVr),
+      NewMovDstReg)
+      .addReg(ABSSrcReg, RegState::Kill)
+      .addImm((unsigned)ARMCC::AL).addReg(0)
+      .addReg(ARM::CPSR, RegState::Define);
+
+    // insert a bcc with opposite CC to ARMCC::MI at the end of BB
+    BuildMI(BB, dl, 
+      TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB)
+      .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR);
+
+    // insert rsbri in RSBBB
+    // Note: BCC and rsbri will be converted into predicated rsbmi
+    // by if-conversion pass
+    BuildMI(*RSBBB, RSBBB->begin(), dl, 
+      TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
+      .addReg(NewMovDstReg, RegState::Kill)
+      .addImm(0).addImm((unsigned)ARMCC::AL).addReg(0).addReg(0);
+
+    // insert PHI in SinkBB, 
+    // reuse ABSDstReg to not change uses of ABS instruction
+    BuildMI(*SinkBB, SinkBB->begin(), dl,
+      TII->get(ARM::PHI), ABSDstReg)
+      .addReg(NewRsbDstReg).addMBB(RSBBB)
+      .addReg(NewMovDstReg).addMBB(BB);
+
+    // remove ABS instruction
+    MI->eraseFromParent(); 
+
+    // return last added BB
+    return SinkBB;
   }
+  }
+}
+
+void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI,
+                                                      SDNode *Node) const {
+  const MCInstrDesc &MCID = MI->getDesc();
+  if (!MCID.hasPostISelHook()) {
+    assert(!convertAddSubFlagsOpcode(MI->getOpcode()) &&
+           "Pseudo flag-setting opcodes must be marked with 'hasPostISelHook'");
+    return;
+  }
+
+  // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
+  // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
+  // operand is still set to noreg. If needed, set the optional operand's
+  // register to CPSR, and remove the redundant implicit def.
+  //
+  // e.g. ADCS (...opt:%noreg, CPSR<imp-def>) -> ADC (... opt:CPSR<def>).
+
+  // Rename pseudo opcodes.
+  unsigned NewOpc = convertAddSubFlagsOpcode(MI->getOpcode());
+  if (NewOpc) {
+    const ARMBaseInstrInfo *TII =
+      static_cast<const ARMBaseInstrInfo*>(getTargetMachine().getInstrInfo());
+    MI->setDesc(TII->get(NewOpc));
+  }
+  unsigned ccOutIdx = MCID.getNumOperands() - 1;
+
+  // Any ARM instruction that sets the 's' bit should specify an optional
+  // "cc_out" operand in the last operand position.
+  if (!MCID.hasOptionalDef() || !MCID.OpInfo[ccOutIdx].isOptionalDef()) {
+    assert(!NewOpc && "Optional cc_out operand required");
+    return;
+  }
+  // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
+  // since we already have an optional CPSR def.
+  bool definesCPSR = false;
+  bool deadCPSR = false;
+  for (unsigned i = MCID.getNumOperands(), e = MI->getNumOperands();
+       i != e; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
+      definesCPSR = true;
+      if (MO.isDead())
+        deadCPSR = true;
+      MI->RemoveOperand(i);
+      break;
+    }
+  }
+  if (!definesCPSR) {
+    assert(!NewOpc && "Optional cc_out operand required");
+    return;
+  }
+  assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
+  if (deadCPSR) {
+    assert(!MI->getOperand(ccOutIdx).getReg() &&
+           "expect uninitialized optional cc_out operand");
+    return;
+  }
+
+  // If this instruction was defined with an optional CPSR def and its dag node
+  // had a live implicit CPSR def, then activate the optional CPSR def.
+  MachineOperand &MO = MI->getOperand(ccOutIdx);
+  MO.setReg(ARM::CPSR);
+  MO.setIsDef(true);
 }
 
 //===----------------------------------------------------------------------===//
@@ -6975,7 +7871,8 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   SDValue FalseVal = N->getOperand(0);
   SDValue TrueVal = N->getOperand(1);
   SDValue ARMcc = N->getOperand(2);
-  ARMCC::CondCodes CC = (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
+  ARMCC::CondCodes CC =
+    (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
 
   // Simplify
   //   mov     r1, r0
@@ -6995,7 +7892,7 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   //   movne   r0, y
   /// FIXME: Turn this into a target neutral optimization?
   SDValue Res;
-  if (CC == ARMCC::NE && FalseVal == RHS) {
+  if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
     Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
                       N->getOperand(3), Cmp);
   } else if (CC == ARMCC::EQ && TrueVal == RHS) {
@@ -7235,7 +8132,7 @@ bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
 /// isLegalAddressingMode - Return true if the addressing mode represented
 /// by AM is legal for this target, for a load/store of the specified type.
 bool ARMTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              const Type *Ty) const {
+                                              Type *Ty) const {
   EVT VT = getValueType(Ty, true);
   if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
     return false;
@@ -7351,7 +8248,8 @@ static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
 
     if (Ptr->getOpcode() == ISD::ADD) {
       isInc = true;
-      ARM_AM::ShiftOpc ShOpcVal= ARM_AM::getShiftOpcForNode(Ptr->getOperand(0));
+      ARM_AM::ShiftOpc ShOpcVal=
+        ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode());
       if (ShOpcVal != ARM_AM::no_shift) {
         Base = Ptr->getOperand(1);
         Offset = Ptr->getOperand(0);
@@ -7536,7 +8434,7 @@ bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const {
     if (AsmPieces.size() == 3 &&
         AsmPieces[0] == "rev" && AsmPieces[1] == "$0" && AsmPieces[2] == "$1" &&
         IA->getConstraintString().compare(0, 4, "=l,l") == 0) {
-      const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
+      IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
       if (Ty && Ty->getBitWidth() == 32)
         return IntrinsicLowering::LowerToByteSwap(CI);
     }
@@ -7559,6 +8457,9 @@ ARMTargetLowering::getConstraintType(const std::string &Constraint) const {
     case 'x': return C_RegisterClass;
     case 't': return C_RegisterClass;
     case 'j': return C_Other; // Constant for movw.
+      // An address with a single base register. Due to the way we
+      // currently handle addresses it is the same as an 'r' memory constraint.
+    case 'Q': return C_Memory;
     }
   } else if (Constraint.size() == 2) {
     switch (Constraint[0]) {
@@ -7582,7 +8483,7 @@ ARMTargetLowering::getSingleConstraintMatchWeight(
     // but allow it at the lowest weight.
   if (CallOperandVal == NULL)
     return CW_Default;
-  const Type *type = CallOperandVal->getType();
+  Type *type = CallOperandVal->getType();
   // Look at the constraint type.
   switch (*constraint) {
   default:
@@ -7618,7 +8519,7 @@ ARMTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
         return RCPair(0U, ARM::GPRRegisterClass);
     case 'h': // High regs or no regs.
       if (Subtarget->isThumb())
-	return RCPair(0U, ARM::hGPRRegisterClass);
+        return RCPair(0U, ARM::hGPRRegisterClass);
       break;
     case 'r':
       return RCPair(0U, ARM::GPRRegisterClass);
@@ -7632,15 +8533,15 @@ ARMTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
       break;
     case 'x':
       if (VT == MVT::f32)
-	return RCPair(0U, ARM::SPR_8RegisterClass);
+        return RCPair(0U, ARM::SPR_8RegisterClass);
       if (VT.getSizeInBits() == 64)
-	return RCPair(0U, ARM::DPR_8RegisterClass);
+        return RCPair(0U, ARM::DPR_8RegisterClass);
       if (VT.getSizeInBits() == 128)
-	return RCPair(0U, ARM::QPR_8RegisterClass);
+        return RCPair(0U, ARM::QPR_8RegisterClass);
       break;
     case 't':
       if (VT == MVT::f32)
-	return RCPair(0U, ARM::SPRRegisterClass);
+        return RCPair(0U, ARM::SPRRegisterClass);
       break;
     }
   }
@@ -7680,12 +8581,12 @@ void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 
     switch (ConstraintLetter) {
       case 'j':
-	// Constant suitable for movw, must be between 0 and
-	// 65535.
-	if (Subtarget->hasV6T2Ops())
-	  if (CVal >= 0 && CVal <= 65535)
-	    break;
-	return;
+        // Constant suitable for movw, must be between 0 and
+        // 65535.
+        if (Subtarget->hasV6T2Ops())
+          if (CVal >= 0 && CVal <= 65535)
+            break;
+        return;
       case 'I':
         if (Subtarget->isThumb1Only()) {
           // This must be a constant between 0 and 255, for ADD
@@ -7823,50 +8724,6 @@ ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
   return false;
 }
 
-int ARM::getVFPf32Imm(const APFloat &FPImm) {
-  APInt Imm = FPImm.bitcastToAPInt();
-  uint32_t Sign = Imm.lshr(31).getZExtValue() & 1;
-  int32_t Exp = (Imm.lshr(23).getSExtValue() & 0xff) - 127;  // -126 to 127
-  int64_t Mantissa = Imm.getZExtValue() & 0x7fffff;  // 23 bits
-
-  // We can handle 4 bits of mantissa.
-  // mantissa = (16+UInt(e:f:g:h))/16.
-  if (Mantissa & 0x7ffff)
-    return -1;
-  Mantissa >>= 19;
-  if ((Mantissa & 0xf) != Mantissa)
-    return -1;
-
-  // We can handle 3 bits of exponent: exp == UInt(NOT(b):c:d)-3
-  if (Exp < -3 || Exp > 4)
-    return -1;
-  Exp = ((Exp+3) & 0x7) ^ 4;
-
-  return ((int)Sign << 7) | (Exp << 4) | Mantissa;
-}
-
-int ARM::getVFPf64Imm(const APFloat &FPImm) {
-  APInt Imm = FPImm.bitcastToAPInt();
-  uint64_t Sign = Imm.lshr(63).getZExtValue() & 1;
-  int64_t Exp = (Imm.lshr(52).getSExtValue() & 0x7ff) - 1023;   // -1022 to 1023
-  uint64_t Mantissa = Imm.getZExtValue() & 0xfffffffffffffLL;
-
-  // We can handle 4 bits of mantissa.
-  // mantissa = (16+UInt(e:f:g:h))/16.
-  if (Mantissa & 0xffffffffffffLL)
-    return -1;
-  Mantissa >>= 48;
-  if ((Mantissa & 0xf) != Mantissa)
-    return -1;
-
-  // We can handle 3 bits of exponent: exp == UInt(NOT(b):c:d)-3
-  if (Exp < -3 || Exp > 4)
-    return -1;
-  Exp = ((Exp+3) & 0x7) ^ 4;
-
-  return ((int)Sign << 7) | (Exp << 4) | Mantissa;
-}
-
 bool ARM::isBitFieldInvertedMask(unsigned v) {
   if (v == 0xffffffff)
     return 0;
@@ -7889,9 +8746,9 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
   if (!Subtarget->hasVFP3())
     return false;
   if (VT == MVT::f32)
-    return ARM::getVFPf32Imm(Imm) != -1;
+    return ARM_AM::getFP32Imm(Imm) != -1;
   if (VT == MVT::f64)
-    return ARM::getVFPf64Imm(Imm) != -1;
+    return ARM_AM::getFP64Imm(Imm) != -1;
   return false;
 }
 
@@ -7933,7 +8790,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     // Conservatively set memVT to the entire set of vectors stored.
     unsigned NumElts = 0;
     for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
-      const Type *ArgTy = I.getArgOperand(ArgI)->getType();
+      Type *ArgTy = I.getArgOperand(ArgI)->getType();
       if (!ArgTy->isVectorTy())
         break;
       NumElts += getTargetData()->getTypeAllocSize(ArgTy) / 8;
diff --git a/contrib/llvm/lib/Target/ARM/ARMISelLowering.h b/contrib/llvm/lib/Target/ARM/ARMISelLowering.h
index 980fb404887e..5da9b27fca68 100644
--- a/contrib/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/contrib/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -71,6 +71,11 @@ namespace llvm {
       SRA_FLAG,     // V,Flag = sra_flag X -> sra X, 1 + save carry out.
       RRX,          // V = RRX X, Flag     -> srl X, 1 + shift in carry flag.
 
+      ADDC,         // Add with carry
+      ADDE,         // Add using carry
+      SUBC,         // Sub with carry
+      SUBE,         // Sub using carry
+
       VMOVRRD,      // double to two gprs.
       VMOVDRR,      // Two gprs to double.
 
@@ -206,18 +211,22 @@ namespace llvm {
       VST4_UPD,
       VST2LN_UPD,
       VST3LN_UPD,
-      VST4LN_UPD
+      VST4LN_UPD,
+
+      // 64-bit atomic ops (value split into two registers)
+      ATOMADD64_DAG,
+      ATOMSUB64_DAG,
+      ATOMOR64_DAG,
+      ATOMXOR64_DAG,
+      ATOMAND64_DAG,
+      ATOMNAND64_DAG,
+      ATOMSWAP64_DAG,
+      ATOMCMPXCHG64_DAG
     };
   }
 
   /// Define some predicates that are used for node matching.
   namespace ARM {
-    /// getVFPf32Imm / getVFPf64Imm - If the given fp immediate can be
-    /// materialized with a VMOV.f32 / VMOV.f64 (i.e. fconsts / fconstd)
-    /// instruction, returns its 8-bit integer representation. Otherwise,
-    /// returns -1.
-    int getVFPf32Imm(const APFloat &FPImm);
-    int getVFPf64Imm(const APFloat &FPImm);
     bool isBitFieldInvertedMask(unsigned v);
   }
 
@@ -240,10 +249,16 @@ namespace llvm {
 
     virtual const char *getTargetNodeName(unsigned Opcode) const;
 
+    /// getSetCCResultType - Return the value type to use for ISD::SETCC.
+    virtual EVT getSetCCResultType(EVT VT) const;
+
     virtual MachineBasicBlock *
       EmitInstrWithCustomInserter(MachineInstr *MI,
                                   MachineBasicBlock *MBB) const;
 
+    virtual void
+    AdjustInstrPostInstrSelection(MachineInstr *MI, SDNode *Node) const;
+
     SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const;
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
@@ -256,7 +271,7 @@ namespace llvm {
 
     /// isLegalAddressingMode - Return true if the addressing mode represented
     /// by AM is legal for this target, for a load/store of the specified type.
-    virtual bool isLegalAddressingMode(const AddrMode &AM, const Type *Ty)const;
+    virtual bool isLegalAddressingMode(const AddrMode &AM, Type *Ty)const;
     bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const;
 
     /// isLegalICmpImmediate - Return true if the specified immediate is legal
@@ -485,12 +500,28 @@ namespace llvm {
                                         MachineBasicBlock *BB,
                                         unsigned Size,
                                         unsigned BinOpcode) const;
+    MachineBasicBlock *EmitAtomicBinary64(MachineInstr *MI,
+                                          MachineBasicBlock *BB,
+                                          unsigned Op1,
+                                          unsigned Op2,
+                                          bool NeedsCarry = false,
+                                          bool IsCmpxchg = false) const;
     MachineBasicBlock * EmitAtomicBinaryMinMax(MachineInstr *MI,
                                                MachineBasicBlock *BB,
                                                unsigned Size,
                                                bool signExtend,
                                                ARMCC::CondCodes Cond) const;
 
+    void EmitBasePointerRecalculation(MachineInstr *MI, MachineBasicBlock *MBB,
+                                      MachineBasicBlock *DispatchBB) const;
+
+    void SetupEntryBlockForSjLj(MachineInstr *MI,
+                                MachineBasicBlock *MBB,
+                                MachineBasicBlock *DispatchBB, int FI) const;
+
+    MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr *MI,
+                                             MachineBasicBlock *MBB) const;
+
     bool RemapAddSubWithFlags(MachineInstr *MI, MachineBasicBlock *BB) const;
   };
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrFormats.td b/contrib/llvm/lib/Target/ARM/ARMInstrFormats.td
index 3ccf22f80b7d..7cbc9111dec3 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrFormats.td
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrFormats.td
@@ -25,7 +25,7 @@ def BrFrm         : Format<2>;
 def BrMiscFrm     : Format<3>;
 
 def DPFrm         : Format<4>;
-def DPSoRegFrm    : Format<5>;
+def DPSoRegRegFrm    : Format<5>;
 
 def LdFrm         : Format<6>;
 def StFrm         : Format<7>;
@@ -68,6 +68,7 @@ def N3RegVShFrm   : Format<38>;
 def NVExtFrm      : Format<39>;
 def NVMulSLFrm    : Format<40>;
 def NVTBLFrm      : Format<41>;
+def DPSoRegImmFrm  : Format<42>;
 
 // Misc flags.
 
@@ -130,39 +131,15 @@ def VFPNeonA8Domain : Domain<5>; // Instructions in VFP & Neon under A8
 // ARM special operands.
 //
 
-def CondCodeOperand : AsmOperandClass {
-  let Name = "CondCode";
-  let SuperClasses = [];
-}
-
-def CCOutOperand : AsmOperandClass {
-  let Name = "CCOut";
-  let SuperClasses = [];
-}
-
-def MemBarrierOptOperand : AsmOperandClass {
-  let Name = "MemBarrierOpt";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseMemBarrierOptOperand";
-}
-
-def ProcIFlagsOperand : AsmOperandClass {
-  let Name = "ProcIFlags";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseProcIFlagsOperand";
-}
-
-def MSRMaskOperand : AsmOperandClass {
-  let Name = "MSRMask";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseMSRMaskOperand";
-}
-
 // ARM imod and iflag operands, used only by the CPS instruction.
 def imod_op : Operand<i32> {
   let PrintMethod = "printCPSIMod";
 }
 
+def ProcIFlagsOperand : AsmOperandClass {
+  let Name = "ProcIFlags";
+  let ParserMethod = "parseProcIFlagsOperand";
+}
 def iflags_op : Operand<i32> {
   let PrintMethod = "printCPSIFlag";
   let ParserMatchClass = ProcIFlagsOperand;
@@ -170,17 +147,21 @@ def iflags_op : Operand<i32> {
 
 // ARM Predicate operand. Default to 14 = always (AL). Second part is CC
 // register whose default is 0 (no register).
-def pred : PredicateOperand<OtherVT, (ops i32imm, CCR),
+def CondCodeOperand : AsmOperandClass { let Name = "CondCode"; }
+def pred : PredicateOperand<OtherVT, (ops i32imm, i32imm),
                                      (ops (i32 14), (i32 zero_reg))> {
   let PrintMethod = "printPredicateOperand";
   let ParserMatchClass = CondCodeOperand;
+  let DecoderMethod = "DecodePredicateOperand";
 }
 
 // Conditional code result for instructions whose 's' bit is set, e.g. subs.
+def CCOutOperand : AsmOperandClass { let Name = "CCOut"; }
 def cc_out : OptionalDefOperand<OtherVT, (ops CCR), (ops (i32 zero_reg))> {
   let EncoderMethod = "getCCOutOpValue";
   let PrintMethod = "printSBitModifierOperand";
   let ParserMatchClass = CCOutOperand;
+  let DecoderMethod = "DecodeCCOutOperand";
 }
 
 // Same as cc_out except it defaults to setting CPSR.
@@ -188,16 +169,27 @@ def s_cc_out : OptionalDefOperand<OtherVT, (ops CCR), (ops (i32 CPSR))> {
   let EncoderMethod = "getCCOutOpValue";
   let PrintMethod = "printSBitModifierOperand";
   let ParserMatchClass = CCOutOperand;
+  let DecoderMethod = "DecodeCCOutOperand";
 }
 
 // ARM special operands for disassembly only.
 //
+def SetEndAsmOperand : AsmOperandClass {
+  let Name = "SetEndImm";
+  let ParserMethod = "parseSetEndImm";
+}
 def setend_op : Operand<i32> {
   let PrintMethod = "printSetendOperand";
+  let ParserMatchClass = SetEndAsmOperand;
 }
 
+def MSRMaskOperand : AsmOperandClass {
+  let Name = "MSRMask";
+  let ParserMethod = "parseMSRMaskOperand";
+}
 def msr_mask : Operand<i32> {
   let PrintMethod = "printMSRMaskOperand";
+  let DecoderMethod = "DecodeMSRMask";
   let ParserMatchClass = MSRMaskOperand;
 }
 
@@ -211,21 +203,40 @@ def msr_mask : Operand<i32> {
 //     64       64 - <imm> is encoded in imm6<5:0>
 def shr_imm8  : Operand<i32> {
   let EncoderMethod = "getShiftRight8Imm";
+  let DecoderMethod = "DecodeShiftRight8Imm";
 }
 def shr_imm16 : Operand<i32> {
   let EncoderMethod = "getShiftRight16Imm";
+  let DecoderMethod = "DecodeShiftRight16Imm";
 }
 def shr_imm32 : Operand<i32> {
   let EncoderMethod = "getShiftRight32Imm";
+  let DecoderMethod = "DecodeShiftRight32Imm";
 }
 def shr_imm64 : Operand<i32> {
   let EncoderMethod = "getShiftRight64Imm";
+  let DecoderMethod = "DecodeShiftRight64Imm";
 }
 
+//===----------------------------------------------------------------------===//
+// ARM Assembler alias templates.
+//
+class ARMInstAlias<string Asm, dag Result, bit Emit = 0b1>
+      : InstAlias<Asm, Result, Emit>, Requires<[IsARM]>;
+class  tInstAlias<string Asm, dag Result, bit Emit = 0b1>
+      : InstAlias<Asm, Result, Emit>, Requires<[IsThumb]>;
+class t2InstAlias<string Asm, dag Result, bit Emit = 0b1>
+      : InstAlias<Asm, Result, Emit>, Requires<[IsThumb2]>;
+class VFP2InstAlias<string Asm, dag Result, bit Emit = 0b1>
+      : InstAlias<Asm, Result, Emit>, Requires<[HasVFP2]>;
+class VFP3InstAlias<string Asm, dag Result, bit Emit = 0b1>
+      : InstAlias<Asm, Result, Emit>, Requires<[HasVFP3]>;
+
 //===----------------------------------------------------------------------===//
 // ARM Instruction templates.
 //
 
+
 class InstTemplate<AddrMode am, int sz, IndexMode im,
                    Format f, Domain d, string cstr, InstrItinClass itin>
   : Instruction {
@@ -240,17 +251,22 @@ class InstTemplate<AddrMode am, int sz, IndexMode im,
   Domain D = d;
   bit isUnaryDataProc = 0;
   bit canXformTo16Bit = 0;
+  // The instruction is a 16-bit flag setting Thumb instruction. Used
+  // by the parser to determine whether to require the 'S' suffix on the
+  // mnemonic (when not in an IT block) or preclude it (when in an IT block).
+  bit thumbArithFlagSetting = 0;
 
   // If this is a pseudo instruction, mark it isCodeGenOnly.
   let isCodeGenOnly = !eq(!cast<string>(f), "Pseudo");
 
-  // The layout of TSFlags should be kept in sync with ARMBaseInstrInfo.h.
+  // The layout of TSFlags should be kept in sync with ARMBaseInfo.h.
   let TSFlags{4-0}   = AM.Value;
   let TSFlags{6-5}   = IndexModeBits;
   let TSFlags{12-7} = Form;
   let TSFlags{13}    = isUnaryDataProc;
   let TSFlags{14}    = canXformTo16Bit;
   let TSFlags{17-15} = D.Value;
+  let TSFlags{18}    = thumbArithFlagSetting;
 
   let Constraints = cstr;
   let Itinerary = itin;
@@ -262,13 +278,17 @@ class Encoding {
 
 class InstARM<AddrMode am, int sz, IndexMode im,
               Format f, Domain d, string cstr, InstrItinClass itin>
-  : InstTemplate<am, sz, im, f, d, cstr, itin>, Encoding;
+  : InstTemplate<am, sz, im, f, d, cstr, itin>, Encoding {
+  let DecoderNamespace = "ARM";
+}
 
 // This Encoding-less class is used by Thumb1 to specify the encoding bits later
 // on by adding flavors to specific instructions.
 class InstThumb<AddrMode am, int sz, IndexMode im,
                 Format f, Domain d, string cstr, InstrItinClass itin>
-  : InstTemplate<am, sz, im, f, d, cstr, itin>;
+  : InstTemplate<am, sz, im, f, d, cstr, itin> {
+  let DecoderNamespace = "Thumb";
+}
 
 class PseudoInst<dag oops, dag iops, InstrItinClass itin, list<dag> pattern>
   : InstTemplate<AddrModeNone, 0, IndexModeNone, Pseudo,
@@ -426,11 +446,11 @@ class AIldrex<bits<2> opcod, dag oops, dag iops, InstrItinClass itin,
   : I<oops, iops, AddrModeNone, 4, IndexModeNone, LdStExFrm, itin,
       opc, asm, "", pattern> {
   bits<4> Rt;
-  bits<4> Rn;
+  bits<4> addr;
   let Inst{27-23} = 0b00011;
   let Inst{22-21} = opcod;
   let Inst{20}    = 1;
-  let Inst{19-16} = Rn;
+  let Inst{19-16} = addr;
   let Inst{15-12} = Rt;
   let Inst{11-0}  = 0b111110011111;
 }
@@ -450,14 +470,14 @@ class AIstrex<bits<2> opcod, dag oops, dag iops, InstrItinClass itin,
   let Inst{3-0}   = Rt;
 }
 class AIswp<bit b, dag oops, dag iops, string opc, list<dag> pattern>
-  : AI<oops, iops, MiscFrm, NoItinerary, opc, "\t$Rt, $Rt2, [$Rn]", pattern> {
+  : AI<oops, iops, MiscFrm, NoItinerary, opc, "\t$Rt, $Rt2, $addr", pattern> {
   bits<4> Rt;
   bits<4> Rt2;
-  bits<4> Rn;
+  bits<4> addr;
   let Inst{27-23} = 0b00010;
   let Inst{22} = b;
   let Inst{21-20} = 0b00;
-  let Inst{19-16} = Rn;
+  let Inst{19-16} = addr;
   let Inst{15-12} = Rt;
   let Inst{11-4} = 0b00001001;
   let Inst{3-0} = Rt2;
@@ -515,22 +535,41 @@ class AI2ldstidx<bit isLd, bit isByte, bit isPre, dag oops, dag iops,
   let Inst{20}    = isLd; // L bit
   let Inst{15-12} = Rt;
 }
-class AI2stridx<bit isByte, bit isPre, dag oops, dag iops,
+class AI2stridx_reg<bit isByte, bit isPre, dag oops, dag iops,
                 IndexMode im, Format f, InstrItinClass itin, string opc,
                 string asm, string cstr, list<dag> pattern>
   : AI2ldstidx<0, isByte, isPre, oops, iops, im, f, itin, opc, asm, cstr,
                pattern> {
   // AM2 store w/ two operands: (GPR, am2offset)
-  // {13}     1 == Rm, 0 == imm12
   // {12}     isAdd
   // {11-0}   imm12/Rm
   bits<14> offset;
   bits<4> Rn;
-  let Inst{25} = offset{13};
+  let Inst{25} = 1;
+  let Inst{23} = offset{12};
+  let Inst{19-16} = Rn;
+  let Inst{11-5} = offset{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = offset{3-0};
+}
+
+class AI2stridx_imm<bit isByte, bit isPre, dag oops, dag iops,
+                IndexMode im, Format f, InstrItinClass itin, string opc,
+                string asm, string cstr, list<dag> pattern>
+  : AI2ldstidx<0, isByte, isPre, oops, iops, im, f, itin, opc, asm, cstr,
+               pattern> {
+  // AM2 store w/ two operands: (GPR, am2offset)
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> Rn;
+  let Inst{25} = 0;
   let Inst{23} = offset{12};
   let Inst{19-16} = Rn;
   let Inst{11-0} = offset{11-0};
 }
+
+
 // FIXME: Merge with the above class when addrmode2 gets used for STR, STRB
 // but for now use this class for STRT and STRBT.
 class AI2stridxT<bit isByte, bit isPre, dag oops, dag iops,
@@ -568,9 +607,11 @@ class AI3ld<bits<4> op, bit op20, dag oops, dag iops, Format f,
   let Inst{11-8}  = addr{7-4};    // imm7_4/zero
   let Inst{7-4}   = op;
   let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
+
+  let DecoderMethod = "DecodeAddrMode3Instruction";
 }
 
-class AI3ldstidx<bits<4> op, bit op20, bit isLd, bit isPre, dag oops, dag iops,
+class AI3ldstidx<bits<4> op, bit op20, bit isPre, dag oops, dag iops,
                 IndexMode im, Format f, InstrItinClass itin, string opc,
                 string asm, string cstr, list<dag> pattern>
   : I<oops, iops, AddrMode3, 4, im, f, itin,
@@ -586,48 +627,24 @@ class AI3ldstidx<bits<4> op, bit op20, bit isLd, bit isPre, dag oops, dag iops,
 
 // FIXME: Merge with the above class when addrmode2 gets used for LDR, LDRB
 // but for now use this class for LDRSBT, LDRHT, LDSHT.
-class AI3ldstidxT<bits<4> op, bit op20, bit isLd, bit isPre, dag oops, dag iops,
+class AI3ldstidxT<bits<4> op, bit isLoad, dag oops, dag iops,
                   IndexMode im, Format f, InstrItinClass itin, string opc,
                   string asm, string cstr, list<dag> pattern>
-  : I<oops, iops, AddrMode3, 4, im, f, itin,
-      opc, asm, cstr, pattern> {
+  : I<oops, iops, AddrMode3, 4, im, f, itin, opc, asm, cstr, pattern> {
   // {13}     1 == imm8, 0 == Rm
   // {12-9}   Rn
   // {8}      isAdd
   // {7-4}    imm7_4/zero
   // {3-0}    imm3_0/Rm
-  bits<14> addr;
+  bits<4> addr;
   bits<4> Rt;
   let Inst{27-25} = 0b000;
-  let Inst{24}    = isPre;        // P bit
-  let Inst{23}    = addr{8};      // U bit
-  let Inst{22}    = addr{13};     // 1 == imm8, 0 == Rm
-  let Inst{20}    = op20;         // L bit
-  let Inst{19-16} = addr{12-9};   // Rn
+  let Inst{24}    = 0;            // P bit
+  let Inst{21}    = 1;
+  let Inst{20}    = isLoad;       // L bit
+  let Inst{19-16} = addr;         // Rn
   let Inst{15-12} = Rt;           // Rt
-  let Inst{11-8}  = addr{7-4};    // imm7_4/zero
   let Inst{7-4}   = op;
-  let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
-  let AsmMatchConverter = "CvtLdWriteBackRegAddrMode3";
-}
-
-class AI3stridx<bits<4> op, bit isByte, bit isPre, dag oops, dag iops,
-                IndexMode im, Format f, InstrItinClass itin, string opc,
-                string asm, string cstr, list<dag> pattern>
-  : AI2ldstidx<0, isByte, isPre, oops, iops, im, f, itin, opc, asm, cstr,
-               pattern> {
-  // AM3 store w/ two operands: (GPR, am3offset)
-  bits<14> offset;
-  bits<4> Rt;
-  bits<4> Rn;
-  let Inst{27-25} = 0b000;
-  let Inst{23}    = offset{8};
-  let Inst{22}    = offset{9};
-  let Inst{19-16} = Rn;
-  let Inst{15-12} = Rt;           // Rt
-  let Inst{11-8}  = offset{7-4};  // imm7_4/zero
-  let Inst{7-4}   = op;
-  let Inst{3-0}   = offset{3-0};  // imm3_0/Rm
 }
 
 // stores
@@ -648,75 +665,7 @@ class AI3str<bits<4> op, dag oops, dag iops, Format f, InstrItinClass itin,
   let Inst{11-8}  = addr{7-4};    // imm7_4/zero
   let Inst{7-4}   = op;
   let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
-}
-
-// Pre-indexed stores
-class AI3sthpr<dag oops, dag iops, Format f, InstrItinClass itin,
-               string opc, string asm, string cstr, list<dag> pattern>
-  : I<oops, iops, AddrMode3, 4, IndexModePre, f, itin,
-      opc, asm, cstr, pattern> {
-  let Inst{4}     = 1;
-  let Inst{5}     = 1; // H bit
-  let Inst{6}     = 0; // S bit
-  let Inst{7}     = 1;
-  let Inst{20}    = 0; // L bit
-  let Inst{21}    = 1; // W bit
-  let Inst{24}    = 1; // P bit
-  let Inst{27-25} = 0b000;
-}
-class AI3stdpr<dag oops, dag iops, Format f, InstrItinClass itin,
-             string opc, string asm, string cstr, list<dag> pattern>
-  : I<oops, iops, AddrMode3, 4, IndexModePre, f, itin,
-      opc, asm, cstr, pattern> {
-  let Inst{4}     = 1;
-  let Inst{5}     = 1; // H bit
-  let Inst{6}     = 1; // S bit
-  let Inst{7}     = 1;
-  let Inst{20}    = 0; // L bit
-  let Inst{21}    = 1; // W bit
-  let Inst{24}    = 1; // P bit
-  let Inst{27-25} = 0b000;
-}
-
-// Post-indexed stores
-class AI3sthpo<dag oops, dag iops, Format f, InstrItinClass itin,
-               string opc, string asm, string cstr, list<dag> pattern>
-  : I<oops, iops, AddrMode3, 4, IndexModePost, f, itin,
-      opc, asm, cstr,pattern> {
-  // {13}     1 == imm8, 0 == Rm
-  // {12-9}   Rn
-  // {8}      isAdd
-  // {7-4}    imm7_4/zero
-  // {3-0}    imm3_0/Rm
-  bits<14> addr;
-  bits<4> Rt;
-  let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
-  let Inst{4}     = 1;
-  let Inst{5}     = 1; // H bit
-  let Inst{6}     = 0; // S bit
-  let Inst{7}     = 1;
-  let Inst{11-8}  = addr{7-4};    // imm7_4/zero
-  let Inst{15-12} = Rt;           // Rt
-  let Inst{19-16} = addr{12-9};   // Rn
-  let Inst{20}    = 0; // L bit
-  let Inst{21}    = 0; // W bit
-  let Inst{22}    = addr{13};     // 1 == imm8, 0 == Rm
-  let Inst{23}    = addr{8};      // U bit
-  let Inst{24}    = 0; // P bit
-  let Inst{27-25} = 0b000;
-}
-class AI3stdpo<dag oops, dag iops, Format f, InstrItinClass itin,
-             string opc, string asm, string cstr, list<dag> pattern>
-  : I<oops, iops, AddrMode3, 4, IndexModePost, f, itin,
-      opc, asm, cstr, pattern> {
-  let Inst{4}     = 1;
-  let Inst{5}     = 1; // H bit
-  let Inst{6}     = 1; // S bit
-  let Inst{7}     = 1;
-  let Inst{20}    = 0; // L bit
-  let Inst{21}    = 0; // W bit
-  let Inst{24}    = 0; // P bit
-  let Inst{27-25} = 0b000;
+  let DecoderMethod = "DecodeAddrMode3Instruction";
 }
 
 // addrmode4 instructions
@@ -843,6 +792,23 @@ class AMiscA1I<bits<8> opcod, bits<4> opc7_4, dag oops, dag iops,
 }
 
 // PKH instructions
+def PKHLSLAsmOperand : AsmOperandClass {
+  let Name = "PKHLSLImm";
+  let ParserMethod = "parsePKHLSLImm";
+}
+def pkh_lsl_amt: Operand<i32>, ImmLeaf<i32, [{ return Imm >= 0 && Imm < 32; }]>{
+  let PrintMethod = "printPKHLSLShiftImm";
+  let ParserMatchClass = PKHLSLAsmOperand;
+}
+def PKHASRAsmOperand : AsmOperandClass {
+  let Name = "PKHASRImm";
+  let ParserMethod = "parsePKHASRImm";
+}
+def pkh_asr_amt: Operand<i32>, ImmLeaf<i32, [{ return Imm > 0 && Imm <= 32; }]>{
+  let PrintMethod = "printPKHASRShiftImm";
+  let ParserMatchClass = PKHASRAsmOperand;
+}
+
 class APKHI<bits<8> opcod, bit tb, dag oops, dag iops, InstrItinClass itin,
             string opc, string asm, list<dag> pattern>
   : I<oops, iops, AddrModeNone, 4, IndexModeNone, ArithMiscFrm, itin,
@@ -850,11 +816,11 @@ class APKHI<bits<8> opcod, bit tb, dag oops, dag iops, InstrItinClass itin,
   bits<4> Rd;
   bits<4> Rn;
   bits<4> Rm;
-  bits<8> sh;
+  bits<5> sh;
   let Inst{27-20} = opcod;
   let Inst{19-16} = Rn;
   let Inst{15-12} = Rd;
-  let Inst{11-7}  = sh{7-3};
+  let Inst{11-7}  = sh;
   let Inst{6}     = tb;
   let Inst{5-4}   = 0b01;
   let Inst{3-0}   = Rm;
@@ -949,7 +915,9 @@ class Thumb1sI<dag oops, dag iops, AddrMode am, int sz,
   let InOperandList = !con(iops, (ins pred:$p));
   let AsmString = !strconcat(opc, "${s}${p}", asm);
   let Pattern = pattern;
+  let thumbArithFlagSetting = 1;
   list<Predicate> Predicates = [IsThumb, IsThumb1Only];
+  let DecoderNamespace = "ThumbSBit";
 }
 
 class T1sI<dag oops, dag iops, InstrItinClass itin,
@@ -1071,6 +1039,7 @@ class Thumb2I<dag oops, dag iops, AddrMode am, int sz,
   let AsmString = !strconcat(opc, "${p}", asm);
   let Pattern = pattern;
   list<Predicate> Predicates = [IsThumb2];
+  let DecoderNamespace = "Thumb2";
 }
 
 // Same as Thumb2I except it can optionally modify CPSR. Note it's modeled as an
@@ -1091,6 +1060,7 @@ class Thumb2sI<dag oops, dag iops, AddrMode am, int sz,
   let AsmString = !strconcat(opc, "${s}${p}", asm);
   let Pattern = pattern;
   list<Predicate> Predicates = [IsThumb2];
+  let DecoderNamespace = "Thumb2";
 }
 
 // Special cases
@@ -1103,6 +1073,7 @@ class Thumb2XI<dag oops, dag iops, AddrMode am, int sz,
   let AsmString = asm;
   let Pattern = pattern;
   list<Predicate> Predicates = [IsThumb2];
+  let DecoderNamespace = "Thumb2";
 }
 
 class ThumbXI<dag oops, dag iops, AddrMode am, int sz,
@@ -1114,6 +1085,7 @@ class ThumbXI<dag oops, dag iops, AddrMode am, int sz,
   let AsmString = asm;
   let Pattern = pattern;
   list<Predicate> Predicates = [IsThumb, IsThumb1Only];
+  let DecoderNamespace = "Thumb";
 }
 
 class T2I<dag oops, dag iops, InstrItinClass itin,
@@ -1132,8 +1104,8 @@ class T2Ipc<dag oops, dag iops, InstrItinClass itin,
             string opc, string asm, list<dag> pattern>
   : Thumb2I<oops, iops, AddrModeT2_pc, 4, itin, opc, asm, "", pattern>;
 class T2Ii8s4<bit P, bit W, bit isLoad, dag oops, dag iops, InstrItinClass itin,
-              string opc, string asm, list<dag> pattern>
-  : Thumb2I<oops, iops, AddrModeT2_i8s4, 4, itin, opc, asm, "",
+              string opc, string asm, string cstr, list<dag> pattern>
+  : Thumb2I<oops, iops, AddrModeT2_i8s4, 4, itin, opc, asm, cstr,
             pattern> {
   bits<4> Rt;
   bits<4> Rt2;
@@ -1149,6 +1121,26 @@ class T2Ii8s4<bit P, bit W, bit isLoad, dag oops, dag iops, InstrItinClass itin,
   let Inst{11-8}  = Rt2{3-0};
   let Inst{7-0}   = addr{7-0};
 }
+class T2Ii8s4post<bit P, bit W, bit isLoad, dag oops, dag iops,
+                  InstrItinClass itin, string opc, string asm, string cstr,
+                  list<dag> pattern>
+  : Thumb2I<oops, iops, AddrModeT2_i8s4, 4, itin, opc, asm, cstr,
+            pattern> {
+  bits<4> Rt;
+  bits<4> Rt2;
+  bits<4> addr;
+  bits<9> imm;
+  let Inst{31-25} = 0b1110100;
+  let Inst{24}    = P;
+  let Inst{23}    = imm{8};
+  let Inst{22}    = 1;
+  let Inst{21}    = W;
+  let Inst{20}    = isLoad;
+  let Inst{19-16} = addr;
+  let Inst{15-12} = Rt{3-0};
+  let Inst{11-8}  = Rt2{3-0};
+  let Inst{7-0}   = imm{7-0};
+}
 
 class T2sI<dag oops, dag iops, InstrItinClass itin,
            string opc, string asm, list<dag> pattern>
@@ -1172,8 +1164,8 @@ class T2XIt<dag oops, dag iops, InstrItinClass itin,
             string asm, string cstr, list<dag> pattern>
   : Thumb2XI<oops, iops, AddrModeNone, 4, itin, asm, cstr, pattern>;
 
-// T2Iidxldst - Thumb2 indexed load / store instructions.
-class T2Iidxldst<bit signed, bits<2> opcod, bit load, bit pre,
+// T2Ipreldst - Thumb2 pre-indexed load / store instructions.
+class T2Ipreldst<bit signed, bits<2> opcod, bit load, bit pre,
                  dag oops, dag iops,
                  AddrMode am, IndexMode im, InstrItinClass itin,
                  string opc, string asm, string cstr, list<dag> pattern>
@@ -1183,25 +1175,60 @@ class T2Iidxldst<bit signed, bits<2> opcod, bit load, bit pre,
   let AsmString = !strconcat(opc, "${p}", asm);
   let Pattern = pattern;
   list<Predicate> Predicates = [IsThumb2];
+  let DecoderNamespace = "Thumb2";
+
+  bits<4> Rt;
+  bits<13> addr;
   let Inst{31-27} = 0b11111;
   let Inst{26-25} = 0b00;
   let Inst{24}    = signed;
   let Inst{23}    = 0;
   let Inst{22-21} = opcod;
   let Inst{20}    = load;
+  let Inst{19-16} = addr{12-9};
+  let Inst{15-12} = Rt{3-0};
   let Inst{11}    = 1;
   // (P, W) = (1, 1) Pre-indexed or (0, 1) Post-indexed
   let Inst{10}    = pre; // The P bit.
+  let Inst{9}     = addr{8}; // Sign bit
   let Inst{8}     = 1; // The W bit.
+  let Inst{7-0}   = addr{7-0};
 
-  bits<9> addr;
-  let Inst{7-0} = addr{7-0};
-  let Inst{9}   = addr{8}; // Sign bit
+  let DecoderMethod = "DecodeT2LdStPre";
+}
+
+// T2Ipostldst - Thumb2 post-indexed load / store instructions.
+class T2Ipostldst<bit signed, bits<2> opcod, bit load, bit pre,
+                 dag oops, dag iops,
+                 AddrMode am, IndexMode im, InstrItinClass itin,
+                 string opc, string asm, string cstr, list<dag> pattern>
+  : InstARM<am, 4, im, ThumbFrm, GenericDomain, cstr, itin> {
+  let OutOperandList = oops;
+  let InOperandList = !con(iops, (ins pred:$p));
+  let AsmString = !strconcat(opc, "${p}", asm);
+  let Pattern = pattern;
+  list<Predicate> Predicates = [IsThumb2];
+  let DecoderNamespace = "Thumb2";
 
   bits<4> Rt;
   bits<4> Rn;
+  bits<9> offset;
+  let Inst{31-27} = 0b11111;
+  let Inst{26-25} = 0b00;
+  let Inst{24}    = signed;
+  let Inst{23}    = 0;
+  let Inst{22-21} = opcod;
+  let Inst{20}    = load;
+  let Inst{19-16} = Rn;
   let Inst{15-12} = Rt{3-0};
-  let Inst{19-16} = Rn{3-0};
+  let Inst{11}    = 1;
+  // (P, W) = (1, 1) Pre-indexed or (0, 1) Post-indexed
+  let Inst{10}    = pre; // The P bit.
+  let Inst{9}     = offset{8}; // Sign bit
+  let Inst{8}     = 1; // The W bit.
+  let Inst{7-0}   = offset{7-0};
+
+  let DecoderMethod = "DecodeT2LdStPre";
 }
 
 // Tv5Pat - Same as Pat<>, but requires V5T Thumb mode.
@@ -1242,6 +1269,7 @@ class VFPI<dag oops, dag iops, AddrMode am, int sz,
   let AsmString = !strconcat(opc, "${p}", asm);
   let Pattern = pattern;
   let PostEncoderMethod = "VFPThumb2PostEncoder";
+  let DecoderNamespace = "VFP";
   list<Predicate> Predicates = [HasVFP2];
 }
 
@@ -1257,6 +1285,7 @@ class VFPXI<dag oops, dag iops, AddrMode am, int sz,
   let AsmString = asm;
   let Pattern = pattern;
   let PostEncoderMethod = "VFPThumb2PostEncoder";
+  let DecoderNamespace = "VFP";
   list<Predicate> Predicates = [HasVFP2];
 }
 
@@ -1574,6 +1603,7 @@ class NeonI<dag oops, dag iops, AddrMode am, IndexMode im, Format f,
   let AsmString = !strconcat(opc, "${p}", ".", dt, "\t", asm);
   let Pattern = pattern;
   list<Predicate> Predicates = [HasNEON];
+  let DecoderNamespace = "NEON";
 }
 
 // Same as NeonI except it does not have a "data type" specifier.
@@ -1586,6 +1616,7 @@ class NeonXI<dag oops, dag iops, AddrMode am, IndexMode im, Format f,
   let AsmString = !strconcat(opc, "${p}", "\t", asm);
   let Pattern = pattern;
   list<Predicate> Predicates = [HasNEON];
+  let DecoderNamespace = "NEON";
 }
 
 class NLdSt<bit op23, bits<2> op21_20, bits<4> op11_8, bits<4> op7_4,
@@ -1600,6 +1631,7 @@ class NLdSt<bit op23, bits<2> op21_20, bits<4> op11_8, bits<4> op7_4,
   let Inst{7-4}   = op7_4;
 
   let PostEncoderMethod = "NEONThumb2LoadStorePostEncoder";
+  let DecoderNamespace = "NEONLoadStore";
 
   bits<5> Vd;
   bits<6> Rn;
@@ -1643,6 +1675,7 @@ class NDataI<dag oops, dag iops, Format f, InstrItinClass itin,
           pattern> {
   let Inst{31-25} = 0b1111001;
   let PostEncoderMethod = "NEONThumb2DataIPostEncoder";
+  let DecoderNamespace = "NEONData";
 }
 
 class NDataXI<dag oops, dag iops, Format f, InstrItinClass itin,
@@ -1651,6 +1684,7 @@ class NDataXI<dag oops, dag iops, Format f, InstrItinClass itin,
            cstr, pattern> {
   let Inst{31-25} = 0b1111001;
   let PostEncoderMethod = "NEONThumb2DataIPostEncoder";
+  let DecoderNamespace = "NEONData";
 }
 
 // NEON "one register and a modified immediate" format.
@@ -1677,6 +1711,7 @@ class N1ModImm<bit op23, bits<3> op21_19, bits<4> op11_8, bit op7, bit op6,
   let Inst{24}    = SIMM{7};
   let Inst{18-16} = SIMM{6-4};
   let Inst{3-0}   = SIMM{3-0};
+  let DecoderMethod = "DecodeNEONModImmInstruction";
 }
 
 // NEON 2 vector register format.
@@ -1874,6 +1909,7 @@ class NVLaneOp<bits<8> opcod1, bits<4> opcod2, bits<2> opcod3,
   list<Predicate> Predicates = [HasNEON];
 
   let PostEncoderMethod = "NEONThumb2DupPostEncoder";
+  let DecoderNamespace = "NEONDup";
 
   bits<5> V;
   bits<4> R;
@@ -1915,7 +1951,6 @@ class NVDupLane<bits<4> op19_16, bit op6, dag oops, dag iops,
 
   bits<5> Vd;
   bits<5> Vm;
-  bits<4> lane;
 
   let Inst{22}     = Vd{4};
   let Inst{15-12} = Vd{3-0};
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMInstrInfo.cpp
index adcbf1806fe3..48da03f63bb9 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrInfo.cpp
@@ -13,8 +13,8 @@
 
 #include "ARMInstrInfo.h"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMMachineFunctionInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -30,14 +30,18 @@ ARMInstrInfo::ARMInstrInfo(const ARMSubtarget &STI)
 unsigned ARMInstrInfo::getUnindexedOpcode(unsigned Opc) const {
   switch (Opc) {
   default: break;
-  case ARM::LDR_PRE:
-  case ARM::LDR_POST:
+  case ARM::LDR_PRE_IMM:
+  case ARM::LDR_PRE_REG:
+  case ARM::LDR_POST_IMM:
+  case ARM::LDR_POST_REG:
     return ARM::LDRi12;
   case ARM::LDRH_PRE:
   case ARM::LDRH_POST:
     return ARM::LDRH;
-  case ARM::LDRB_PRE:
-  case ARM::LDRB_POST:
+  case ARM::LDRB_PRE_IMM:
+  case ARM::LDRB_PRE_REG:
+  case ARM::LDRB_POST_IMM:
+  case ARM::LDRB_POST_REG:
     return ARM::LDRBi12;
   case ARM::LDRSH_PRE:
   case ARM::LDRSH_POST:
@@ -45,14 +49,18 @@ unsigned ARMInstrInfo::getUnindexedOpcode(unsigned Opc) const {
   case ARM::LDRSB_PRE:
   case ARM::LDRSB_POST:
     return ARM::LDRSB;
-  case ARM::STR_PRE:
-  case ARM::STR_POST:
+  case ARM::STR_PRE_IMM:
+  case ARM::STR_PRE_REG:
+  case ARM::STR_POST_IMM:
+  case ARM::STR_POST_REG:
     return ARM::STRi12;
   case ARM::STRH_PRE:
   case ARM::STRH_POST:
     return ARM::STRH;
-  case ARM::STRB_PRE:
-  case ARM::STRB_POST:
+  case ARM::STRB_PRE_IMM:
+  case ARM::STRB_PRE_REG:
+  case ARM::STRB_POST_IMM:
+  case ARM::STRB_POST_REG:
     return ARM::STRBi12;
   }
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrInfo.td b/contrib/llvm/lib/Target/ARM/ARMInstrInfo.td
index a42dd1a54ec7..2cf0f09ffc64 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -70,6 +70,18 @@ def SDT_ARMTCRET : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
 def SDT_ARMBFI : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                       SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
 
+def SDTBinaryArithWithFlags : SDTypeProfile<2, 2,
+                                            [SDTCisSameAs<0, 2>,
+                                             SDTCisSameAs<0, 3>,
+                                             SDTCisInt<0>, SDTCisVT<1, i32>]>;
+
+// SDTBinaryArithWithFlagsInOut - RES1, CPSR = op LHS, RHS, CPSR
+def SDTBinaryArithWithFlagsInOut : SDTypeProfile<2, 3,
+                                            [SDTCisSameAs<0, 2>,
+                                             SDTCisSameAs<0, 3>,
+                                             SDTCisInt<0>,
+                                             SDTCisVT<1, i32>,
+                                             SDTCisVT<4, i32>]>;
 // Node definitions.
 def ARMWrapper       : SDNode<"ARMISD::Wrapper",     SDTIntUnaryOp>;
 def ARMWrapperDYN    : SDNode<"ARMISD::WrapperDYN",  SDTIntUnaryOp>;
@@ -120,6 +132,12 @@ def ARMsrl_flag      : SDNode<"ARMISD::SRL_FLAG", SDTIntUnaryOp, [SDNPOutGlue]>;
 def ARMsra_flag      : SDNode<"ARMISD::SRA_FLAG", SDTIntUnaryOp, [SDNPOutGlue]>;
 def ARMrrx           : SDNode<"ARMISD::RRX"     , SDTIntUnaryOp, [SDNPInGlue ]>;
 
+def ARMaddc          : SDNode<"ARMISD::ADDC",  SDTBinaryArithWithFlags,
+                              [SDNPCommutative]>;
+def ARMsubc          : SDNode<"ARMISD::SUBC",  SDTBinaryArithWithFlags>;
+def ARMadde          : SDNode<"ARMISD::ADDE",  SDTBinaryArithWithFlagsInOut>;
+def ARMsube          : SDNode<"ARMISD::SUBE",  SDTBinaryArithWithFlagsInOut>;
+
 def ARMthread_pointer: SDNode<"ARMISD::THREAD_POINTER", SDT_ARMThreadPointer>;
 def ARMeh_sjlj_setjmp: SDNode<"ARMISD::EH_SJLJ_SETJMP",
                                SDT_ARMEH_SJLJ_Setjmp, [SDNPHasChain]>;
@@ -187,10 +205,16 @@ def IsThumb          : Predicate<"Subtarget->isThumb()">,
 def IsThumb1Only     : Predicate<"Subtarget->isThumb1Only()">;
 def IsThumb2         : Predicate<"Subtarget->isThumb2()">,
                                  AssemblerPredicate<"ModeThumb,FeatureThumb2">;
+def IsMClass         : Predicate<"Subtarget->isMClass()">,
+                                 AssemblerPredicate<"FeatureMClass">;
+def IsARClass        : Predicate<"!Subtarget->isMClass()">,
+                                 AssemblerPredicate<"!FeatureMClass">;
 def IsARM            : Predicate<"!Subtarget->isThumb()">,
                                  AssemblerPredicate<"!ModeThumb">;
 def IsDarwin         : Predicate<"Subtarget->isTargetDarwin()">;
 def IsNotDarwin      : Predicate<"!Subtarget->isTargetDarwin()">;
+def IsNaCl           : Predicate<"Subtarget->isTargetNaCl()">,
+                                 AssemblerPredicate<"ModeNaCl">;
 
 // FIXME: Eventually this will be just "hasV6T2Ops".
 def UseMovt          : Predicate<"Subtarget->useMovt()">;
@@ -263,24 +287,11 @@ def imm0_65535 : Operand<i32>, ImmLeaf<i32, [{
   let ParserMatchClass = Imm0_65535AsmOperand;
 }
 
+class BinOpWithFlagFrag<dag res> :
+      PatFrag<(ops node:$LHS, node:$RHS, node:$FLAG), res>;
 class BinOpFrag<dag res> : PatFrag<(ops node:$LHS, node:$RHS), res>;
 class UnOpFrag <dag res> : PatFrag<(ops node:$Src), res>;
 
-/// adde and sube predicates - True based on whether the carry flag output
-/// will be needed or not.
-def adde_dead_carry :
-  PatFrag<(ops node:$LHS, node:$RHS), (adde node:$LHS, node:$RHS),
-  [{return !N->hasAnyUseOfValue(1);}]>;
-def sube_dead_carry :
-  PatFrag<(ops node:$LHS, node:$RHS), (sube node:$LHS, node:$RHS),
-  [{return !N->hasAnyUseOfValue(1);}]>;
-def adde_live_carry :
-  PatFrag<(ops node:$LHS, node:$RHS), (adde node:$LHS, node:$RHS),
-  [{return N->hasAnyUseOfValue(1);}]>;
-def sube_live_carry :
-  PatFrag<(ops node:$LHS, node:$RHS), (sube node:$LHS, node:$RHS),
-  [{return N->hasAnyUseOfValue(1);}]>;
-
 // An 'and' node with a single use.
 def and_su : PatFrag<(ops node:$lhs, node:$rhs), (and node:$lhs, node:$rhs), [{
   return N->hasOneUse();
@@ -315,6 +326,7 @@ def fsub_mlx : PatFrag<(ops node:$lhs, node:$rhs),(fsub node:$lhs, node:$rhs),[{
 def brtarget : Operand<OtherVT> {
   let EncoderMethod = "getBranchTargetOpValue";
   let OperandType = "OPERAND_PCREL";
+  let DecoderMethod = "DecodeT2BROperand";
 }
 
 // FIXME: get rid of this one?
@@ -345,39 +357,35 @@ def bl_target : Operand<i32> {
   let OperandType = "OPERAND_PCREL";
 }
 
+def blx_target : Operand<i32> {
+  // Encoded the same as branch targets.
+  let EncoderMethod = "getARMBLXTargetOpValue";
+  let OperandType = "OPERAND_PCREL";
+}
 
 // A list of registers separated by comma. Used by load/store multiple.
-def RegListAsmOperand : AsmOperandClass {
-  let Name = "RegList";
-  let SuperClasses = [];
-}
-
-def DPRRegListAsmOperand : AsmOperandClass {
-  let Name = "DPRRegList";
-  let SuperClasses = [];
-}
-
-def SPRRegListAsmOperand : AsmOperandClass {
-  let Name = "SPRRegList";
-  let SuperClasses = [];
-}
-
+def RegListAsmOperand : AsmOperandClass { let Name = "RegList"; }
 def reglist : Operand<i32> {
   let EncoderMethod = "getRegisterListOpValue";
   let ParserMatchClass = RegListAsmOperand;
   let PrintMethod = "printRegisterList";
+  let DecoderMethod = "DecodeRegListOperand";
 }
 
+def DPRRegListAsmOperand : AsmOperandClass { let Name = "DPRRegList"; }
 def dpr_reglist : Operand<i32> {
   let EncoderMethod = "getRegisterListOpValue";
   let ParserMatchClass = DPRRegListAsmOperand;
   let PrintMethod = "printRegisterList";
+  let DecoderMethod = "DecodeDPRRegListOperand";
 }
 
+def SPRRegListAsmOperand : AsmOperandClass { let Name = "SPRRegList"; }
 def spr_reglist : Operand<i32> {
   let EncoderMethod = "getRegisterListOpValue";
   let ParserMatchClass = SPRRegListAsmOperand;
   let PrintMethod = "printRegisterList";
+  let DecoderMethod = "DecodeSPRRegListOperand";
 }
 
 // An operand for the CONSTPOOL_ENTRY pseudo-instruction.
@@ -397,56 +405,99 @@ def adrlabel : Operand<i32> {
 
 def neon_vcvt_imm32 : Operand<i32> {
   let EncoderMethod = "getNEONVcvtImm32OpValue";
+  let DecoderMethod = "DecodeVCVTImmOperand";
 }
 
 // rot_imm: An integer that encodes a rotate amount. Must be 8, 16, or 24.
-def rot_imm : Operand<i32>, ImmLeaf<i32, [{
-    int32_t v = (int32_t)Imm;
-    return v == 8 || v == 16 || v == 24; }]> {
-  let EncoderMethod = "getRotImmOpValue";
+def rot_imm_XFORM: SDNodeXForm<imm, [{
+  switch (N->getZExtValue()){
+  default: assert(0);
+  case 0:  return CurDAG->getTargetConstant(0, MVT::i32);
+  case 8:  return CurDAG->getTargetConstant(1, MVT::i32);
+  case 16: return CurDAG->getTargetConstant(2, MVT::i32);
+  case 24: return CurDAG->getTargetConstant(3, MVT::i32);
+  }
+}]>;
+def RotImmAsmOperand : AsmOperandClass {
+  let Name = "RotImm";
+  let ParserMethod = "parseRotImm";
 }
-
-def ShifterAsmOperand : AsmOperandClass {
-  let Name = "Shifter";
-  let SuperClasses = [];
+def rot_imm : Operand<i32>, PatLeaf<(i32 imm), [{
+    int32_t v = N->getZExtValue();
+    return v == 8 || v == 16 || v == 24; }],
+    rot_imm_XFORM> {
+  let PrintMethod = "printRotImmOperand";
+  let ParserMatchClass = RotImmAsmOperand;
 }
 
 // shift_imm: An integer that encodes a shift amount and the type of shift
-// (currently either asr or lsl) using the same encoding used for the
-// immediates in so_reg operands.
+// (asr or lsl). The 6-bit immediate encodes as:
+//    {5}     0 ==> lsl
+//            1     asr
+//    {4-0}   imm5 shift amount.
+//            asr #32 encoded as imm5 == 0.
+def ShifterImmAsmOperand : AsmOperandClass {
+  let Name = "ShifterImm";
+  let ParserMethod = "parseShifterImm";
+}
 def shift_imm : Operand<i32> {
   let PrintMethod = "printShiftImmOperand";
-  let ParserMatchClass = ShifterAsmOperand;
+  let ParserMatchClass = ShifterImmAsmOperand;
 }
 
-def ShiftedRegAsmOperand : AsmOperandClass {
-  let Name = "ShiftedReg";
-}
-
-// shifter_operand operands: so_reg and so_imm.
-def so_reg : Operand<i32>,    // reg reg imm
-             ComplexPattern<i32, 3, "SelectShifterOperandReg",
-                            [shl,srl,sra,rotr]> {
-  let EncoderMethod = "getSORegOpValue";
-  let PrintMethod = "printSORegOperand";
+// shifter_operand operands: so_reg_reg, so_reg_imm, and so_imm.
+def ShiftedRegAsmOperand : AsmOperandClass { let Name = "RegShiftedReg"; }
+def so_reg_reg : Operand<i32>,  // reg reg imm
+                 ComplexPattern<i32, 3, "SelectRegShifterOperand",
+                                [shl, srl, sra, rotr]> {
+  let EncoderMethod = "getSORegRegOpValue";
+  let PrintMethod = "printSORegRegOperand";
+  let DecoderMethod = "DecodeSORegRegOperand";
   let ParserMatchClass = ShiftedRegAsmOperand;
-  let MIOperandInfo = (ops GPR, GPR, shift_imm);
+  let MIOperandInfo = (ops GPRnopc, GPRnopc, i32imm);
 }
+
+def ShiftedImmAsmOperand : AsmOperandClass { let Name = "RegShiftedImm"; }
+def so_reg_imm : Operand<i32>, // reg imm
+                 ComplexPattern<i32, 2, "SelectImmShifterOperand",
+                                [shl, srl, sra, rotr]> {
+  let EncoderMethod = "getSORegImmOpValue";
+  let PrintMethod = "printSORegImmOperand";
+  let DecoderMethod = "DecodeSORegImmOperand";
+  let ParserMatchClass = ShiftedImmAsmOperand;
+  let MIOperandInfo = (ops GPR, i32imm);
+}
+
 // FIXME: Does this need to be distinct from so_reg?
-def shift_so_reg : Operand<i32>,    // reg reg imm
-                   ComplexPattern<i32, 3, "SelectShiftShifterOperandReg",
+def shift_so_reg_reg : Operand<i32>,    // reg reg imm
+                   ComplexPattern<i32, 3, "SelectShiftRegShifterOperand",
                                   [shl,srl,sra,rotr]> {
-  let EncoderMethod = "getSORegOpValue";
-  let PrintMethod = "printSORegOperand";
-  let MIOperandInfo = (ops GPR, GPR, shift_imm);
+  let EncoderMethod = "getSORegRegOpValue";
+  let PrintMethod = "printSORegRegOperand";
+  let DecoderMethod = "DecodeSORegRegOperand";
+  let MIOperandInfo = (ops GPR, GPR, i32imm);
 }
 
+// FIXME: Does this need to be distinct from so_reg?
+def shift_so_reg_imm : Operand<i32>,    // reg reg imm
+                   ComplexPattern<i32, 2, "SelectShiftImmShifterOperand",
+                                  [shl,srl,sra,rotr]> {
+  let EncoderMethod = "getSORegImmOpValue";
+  let PrintMethod = "printSORegImmOperand";
+  let DecoderMethod = "DecodeSORegImmOperand";
+  let MIOperandInfo = (ops GPR, i32imm);
+}
+
+
 // so_imm - Match a 32-bit shifter_operand immediate operand, which is an
 // 8-bit immediate rotated by an arbitrary number of bits.
+def SOImmAsmOperand: AsmOperandClass { let Name = "ARMSOImm"; }
 def so_imm : Operand<i32>, ImmLeaf<i32, [{
     return ARM_AM::getSOImmVal(Imm) != -1;
   }]> {
   let EncoderMethod = "getSOImmOpValue";
+  let ParserMatchClass = SOImmAsmOperand;
+  let DecoderMethod = "DecodeSOImmOperand";
 }
 
 // Break so_imm's up into two pieces.  This handles immediates with up to 16
@@ -464,7 +515,7 @@ def arm_i32imm : PatLeaf<(imm), [{
   return ARM_AM::isSOImmTwoPartVal((unsigned)N->getZExtValue());
 }]>;
 
-/// imm0_7 predicate - Immediate in the range [0,31].
+/// imm0_7 predicate - Immediate in the range [0,7].
 def Imm0_7AsmOperand: AsmOperandClass { let Name = "Imm0_7"; }
 def imm0_7 : Operand<i32>, ImmLeaf<i32, [{
   return Imm >= 0 && Imm < 8;
@@ -472,7 +523,7 @@ def imm0_7 : Operand<i32>, ImmLeaf<i32, [{
   let ParserMatchClass = Imm0_7AsmOperand;
 }
 
-/// imm0_15 predicate - Immediate in the range [0,31].
+/// imm0_15 predicate - Immediate in the range [0,15].
 def Imm0_15AsmOperand: AsmOperandClass { let Name = "Imm0_15"; }
 def imm0_15 : Operand<i32>, ImmLeaf<i32, [{
   return Imm >= 0 && Imm < 16;
@@ -481,68 +532,83 @@ def imm0_15 : Operand<i32>, ImmLeaf<i32, [{
 }
 
 /// imm0_31 predicate - True if the 32-bit immediate is in the range [0,31].
+def Imm0_31AsmOperand: AsmOperandClass { let Name = "Imm0_31"; }
 def imm0_31 : Operand<i32>, ImmLeaf<i32, [{
   return Imm >= 0 && Imm < 32;
-}]>;
-
-/// imm0_31_m1 - Matches and prints like imm0_31, but encodes as 'value - 1'.
-def imm0_31_m1 : Operand<i32>, ImmLeaf<i32, [{
-  return Imm >= 0 && Imm < 32;
 }]> {
-  let EncoderMethod = "getImmMinusOneOpValue";
+  let ParserMatchClass = Imm0_31AsmOperand;
 }
 
-// i32imm_hilo16 - For movt/movw - sets the MC Encoder method.
-// The imm is split into imm{15-12}, imm{11-0}
-//
-def i32imm_hilo16 : Operand<i32> {
-  let EncoderMethod = "getHiLo16ImmOpValue";
+/// imm0_255 predicate - Immediate in the range [0,255].
+def Imm0_255AsmOperand : AsmOperandClass { let Name = "Imm0_255"; }
+def imm0_255 : Operand<i32>, ImmLeaf<i32, [{ return Imm >= 0 && Imm < 256; }]> {
+  let ParserMatchClass = Imm0_255AsmOperand;
 }
 
+// imm0_65535_expr - For movt/movw - 16-bit immediate that can also reference
+// a relocatable expression.
+//
+// FIXME: This really needs a Thumb version separate from the ARM version.
+// While the range is the same, and can thus use the same match class,
+// the encoding is different so it should have a different encoder method.
+def Imm0_65535ExprAsmOperand: AsmOperandClass { let Name = "Imm0_65535Expr"; }
+def imm0_65535_expr : Operand<i32> {
+  let EncoderMethod = "getHiLo16ImmOpValue";
+  let ParserMatchClass = Imm0_65535ExprAsmOperand;
+}
+
+/// imm24b - True if the 32-bit immediate is encodable in 24 bits.
+def Imm24bitAsmOperand: AsmOperandClass { let Name = "Imm24bit"; }
+def imm24b : Operand<i32>, ImmLeaf<i32, [{
+  return Imm >= 0 && Imm <= 0xffffff;
+}]> {
+  let ParserMatchClass = Imm24bitAsmOperand;
+}
+
+
 /// bf_inv_mask_imm predicate - An AND mask to clear an arbitrary width bitfield
 /// e.g., 0xf000ffff
+def BitfieldAsmOperand : AsmOperandClass {
+  let Name = "Bitfield";
+  let ParserMethod = "parseBitfield";
+}
 def bf_inv_mask_imm : Operand<i32>,
                       PatLeaf<(imm), [{
   return ARM::isBitFieldInvertedMask(N->getZExtValue());
 }] > {
   let EncoderMethod = "getBitfieldInvertedMaskOpValue";
   let PrintMethod = "printBitfieldInvMaskImmOperand";
+  let DecoderMethod = "DecodeBitfieldMaskOperand";
+  let ParserMatchClass = BitfieldAsmOperand;
 }
 
-/// lsb_pos_imm - position of the lsb bit, used by BFI4p and t2BFI4p
-def lsb_pos_imm : Operand<i32>, ImmLeaf<i32, [{
-  return isInt<5>(Imm);
+def imm1_32_XFORM: SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant((int)N->getZExtValue() - 1, MVT::i32);
 }]>;
-
-/// width_imm - number of bits to be copied, used by BFI4p and t2BFI4p
-def width_imm : Operand<i32>, ImmLeaf<i32, [{
-  return Imm > 0 &&  Imm <= 32;
-}] > {
-  let EncoderMethod = "getMsbOpValue";
+def Imm1_32AsmOperand: AsmOperandClass { let Name = "Imm1_32"; }
+def imm1_32 : Operand<i32>, PatLeaf<(imm), [{
+   uint64_t Imm = N->getZExtValue();
+   return Imm > 0 && Imm <= 32;
+ }],
+    imm1_32_XFORM> {
+  let PrintMethod = "printImmPlusOneOperand";
+  let ParserMatchClass = Imm1_32AsmOperand;
 }
 
-def ssat_imm : Operand<i32>, ImmLeaf<i32, [{
-  return Imm > 0 && Imm <= 32;
-}]> {
-  let EncoderMethod = "getSsatBitPosValue";
+def imm1_16_XFORM: SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant((int)N->getZExtValue() - 1, MVT::i32);
+}]>;
+def Imm1_16AsmOperand: AsmOperandClass { let Name = "Imm1_16"; }
+def imm1_16 : Operand<i32>, PatLeaf<(imm), [{ return Imm > 0 && Imm <= 16; }],
+    imm1_16_XFORM> {
+  let PrintMethod = "printImmPlusOneOperand";
+  let ParserMatchClass = Imm1_16AsmOperand;
 }
 
 // Define ARM specific addressing modes.
-
-def MemMode2AsmOperand : AsmOperandClass {
-  let Name = "MemMode2";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseMemMode2Operand";
-}
-
-def MemMode3AsmOperand : AsmOperandClass {
-  let Name = "MemMode3";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseMemMode3Operand";
-}
-
 // addrmode_imm12 := reg +/- imm12
 //
+def MemImm12OffsetAsmOperand : AsmOperandClass { let Name = "MemImm12Offset"; }
 def addrmode_imm12 : Operand<i32>,
                      ComplexPattern<i32, 2, "SelectAddrModeImm12", []> {
   // 12-bit immediate operand. Note that instructions using this encode
@@ -551,53 +617,129 @@ def addrmode_imm12 : Operand<i32>,
 
   let EncoderMethod = "getAddrModeImm12OpValue";
   let PrintMethod = "printAddrModeImm12Operand";
+  let DecoderMethod = "DecodeAddrModeImm12Operand";
+  let ParserMatchClass = MemImm12OffsetAsmOperand;
   let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
 }
 // ldst_so_reg := reg +/- reg shop imm
 //
+def MemRegOffsetAsmOperand : AsmOperandClass { let Name = "MemRegOffset"; }
 def ldst_so_reg : Operand<i32>,
                   ComplexPattern<i32, 3, "SelectLdStSOReg", []> {
   let EncoderMethod = "getLdStSORegOpValue";
   // FIXME: Simplify the printer
   let PrintMethod = "printAddrMode2Operand";
-  let MIOperandInfo = (ops GPR:$base, GPR:$offsreg, i32imm:$offsimm);
+  let DecoderMethod = "DecodeSORegMemOperand";
+  let ParserMatchClass = MemRegOffsetAsmOperand;
+  let MIOperandInfo = (ops GPR:$base, GPRnopc:$offsreg, i32imm:$shift);
 }
 
+// postidx_imm8 := +/- [0,255]
+//
+// 9 bit value:
+//  {8}       1 is imm8 is non-negative. 0 otherwise.
+//  {7-0}     [0,255] imm8 value.
+def PostIdxImm8AsmOperand : AsmOperandClass { let Name = "PostIdxImm8"; }
+def postidx_imm8 : Operand<i32> {
+  let PrintMethod = "printPostIdxImm8Operand";
+  let ParserMatchClass = PostIdxImm8AsmOperand;
+  let MIOperandInfo = (ops i32imm);
+}
+
+// postidx_imm8s4 := +/- [0,1020]
+//
+// 9 bit value:
+//  {8}       1 is imm8 is non-negative. 0 otherwise.
+//  {7-0}     [0,255] imm8 value, scaled by 4.
+def PostIdxImm8s4AsmOperand : AsmOperandClass { let Name = "PostIdxImm8s4"; }
+def postidx_imm8s4 : Operand<i32> {
+  let PrintMethod = "printPostIdxImm8s4Operand";
+  let ParserMatchClass = PostIdxImm8s4AsmOperand;
+  let MIOperandInfo = (ops i32imm);
+}
+
+
+// postidx_reg := +/- reg
+//
+def PostIdxRegAsmOperand : AsmOperandClass {
+  let Name = "PostIdxReg";
+  let ParserMethod = "parsePostIdxReg";
+}
+def postidx_reg : Operand<i32> {
+  let EncoderMethod = "getPostIdxRegOpValue";
+  let DecoderMethod = "DecodePostIdxReg";
+  let PrintMethod = "printPostIdxRegOperand";
+  let ParserMatchClass = PostIdxRegAsmOperand;
+  let MIOperandInfo = (ops GPR, i32imm);
+}
+
+
 // addrmode2 := reg +/- imm12
 //           := reg +/- reg shop imm
 //
+// FIXME: addrmode2 should be refactored the rest of the way to always
+// use explicit imm vs. reg versions above (addrmode_imm12 and ldst_so_reg).
+def AddrMode2AsmOperand : AsmOperandClass { let Name = "AddrMode2"; }
 def addrmode2 : Operand<i32>,
                 ComplexPattern<i32, 3, "SelectAddrMode2", []> {
   let EncoderMethod = "getAddrMode2OpValue";
   let PrintMethod = "printAddrMode2Operand";
-  let ParserMatchClass = MemMode2AsmOperand;
+  let ParserMatchClass = AddrMode2AsmOperand;
   let MIOperandInfo = (ops GPR:$base, GPR:$offsreg, i32imm:$offsimm);
 }
 
-def am2offset : Operand<i32>,
-                ComplexPattern<i32, 2, "SelectAddrMode2Offset",
+def PostIdxRegShiftedAsmOperand : AsmOperandClass {
+  let Name = "PostIdxRegShifted";
+  let ParserMethod = "parsePostIdxReg";
+}
+def am2offset_reg : Operand<i32>,
+                ComplexPattern<i32, 2, "SelectAddrMode2OffsetReg",
                 [], [SDNPWantRoot]> {
   let EncoderMethod = "getAddrMode2OffsetOpValue";
   let PrintMethod = "printAddrMode2OffsetOperand";
+  // When using this for assembly, it's always as a post-index offset.
+  let ParserMatchClass = PostIdxRegShiftedAsmOperand;
   let MIOperandInfo = (ops GPR, i32imm);
 }
 
+// FIXME: am2offset_imm should only need the immediate, not the GPR. Having
+// the GPR is purely vestigal at this point.
+def AM2OffsetImmAsmOperand : AsmOperandClass { let Name = "AM2OffsetImm"; }
+def am2offset_imm : Operand<i32>,
+                ComplexPattern<i32, 2, "SelectAddrMode2OffsetImm",
+                [], [SDNPWantRoot]> {
+  let EncoderMethod = "getAddrMode2OffsetOpValue";
+  let PrintMethod = "printAddrMode2OffsetOperand";
+  let ParserMatchClass = AM2OffsetImmAsmOperand;
+  let MIOperandInfo = (ops GPR, i32imm);
+}
+
+
 // addrmode3 := reg +/- reg
 // addrmode3 := reg +/- imm8
 //
+// FIXME: split into imm vs. reg versions.
+def AddrMode3AsmOperand : AsmOperandClass { let Name = "AddrMode3"; }
 def addrmode3 : Operand<i32>,
                 ComplexPattern<i32, 3, "SelectAddrMode3", []> {
   let EncoderMethod = "getAddrMode3OpValue";
   let PrintMethod = "printAddrMode3Operand";
-  let ParserMatchClass = MemMode3AsmOperand;
+  let ParserMatchClass = AddrMode3AsmOperand;
   let MIOperandInfo = (ops GPR:$base, GPR:$offsreg, i32imm:$offsimm);
 }
 
+// FIXME: split into imm vs. reg versions.
+// FIXME: parser method to handle +/- register.
+def AM3OffsetAsmOperand : AsmOperandClass {
+  let Name = "AM3Offset";
+  let ParserMethod = "parseAM3Offset";
+}
 def am3offset : Operand<i32>,
                 ComplexPattern<i32, 2, "SelectAddrMode3Offset",
                                [], [SDNPWantRoot]> {
   let EncoderMethod = "getAddrMode3OffsetOpValue";
   let PrintMethod = "printAddrMode3OffsetOperand";
+  let ParserMatchClass = AM3OffsetAsmOperand;
   let MIOperandInfo = (ops GPR, i32imm);
 }
 
@@ -608,28 +750,28 @@ def ldstm_mode : OptionalDefOperand<OtherVT, (ops i32), (ops (i32 1))> {
   let PrintMethod = "printLdStmModeOperand";
 }
 
-def MemMode5AsmOperand : AsmOperandClass {
-  let Name = "MemMode5";
-  let SuperClasses = [];
-}
-
 // addrmode5 := reg +/- imm8*4
 //
+def AddrMode5AsmOperand : AsmOperandClass { let Name = "AddrMode5"; }
 def addrmode5 : Operand<i32>,
                 ComplexPattern<i32, 2, "SelectAddrMode5", []> {
   let PrintMethod = "printAddrMode5Operand";
-  let MIOperandInfo = (ops GPR:$base, i32imm);
-  let ParserMatchClass = MemMode5AsmOperand;
   let EncoderMethod = "getAddrMode5OpValue";
+  let DecoderMethod = "DecodeAddrMode5Operand";
+  let ParserMatchClass = AddrMode5AsmOperand;
+  let MIOperandInfo = (ops GPR:$base, i32imm);
 }
 
 // addrmode6 := reg with optional alignment
 //
+def AddrMode6AsmOperand : AsmOperandClass { let Name = "AlignedMemory"; }
 def addrmode6 : Operand<i32>,
                 ComplexPattern<i32, 2, "SelectAddrMode6", [], [SDNPWantParent]>{
   let PrintMethod = "printAddrMode6Operand";
-  let MIOperandInfo = (ops GPR:$addr, i32imm);
+  let MIOperandInfo = (ops GPR:$addr, i32imm:$align);
   let EncoderMethod = "getAddrMode6AddressOpValue";
+  let DecoderMethod = "DecodeAddrMode6Operand";
+  let ParserMatchClass = AddrMode6AsmOperand;
 }
 
 def am6offset : Operand<i32>,
@@ -638,6 +780,7 @@ def am6offset : Operand<i32>,
   let PrintMethod = "printAddrMode6OffsetOperand";
   let MIOperandInfo = (ops GPR);
   let EncoderMethod = "getAddrMode6OffsetOpValue";
+  let DecoderMethod = "DecodeGPRRegisterClass";
 }
 
 // Special version of addrmode6 to handle alignment encoding for VST1/VLD1
@@ -666,19 +809,15 @@ def addrmodepc : Operand<i32>,
   let MIOperandInfo = (ops GPR, i32imm);
 }
 
-def MemMode7AsmOperand : AsmOperandClass {
-  let Name = "MemMode7";
-  let SuperClasses = [];
-}
-
-// addrmode7 := reg
-// Used by load/store exclusive instructions. Useful to enable right assembly
-// parsing and printing. Not used for any codegen matching.
+// addr_offset_none := reg
 //
-def addrmode7 : Operand<i32> {
+def MemNoOffsetAsmOperand : AsmOperandClass { let Name = "MemNoOffset"; }
+def addr_offset_none : Operand<i32>,
+                       ComplexPattern<i32, 1, "SelectAddrOffsetNone", []> {
   let PrintMethod = "printAddrMode7Operand";
-  let MIOperandInfo = (ops GPR);
-  let ParserMatchClass = MemMode7AsmOperand;
+  let DecoderMethod = "DecodeAddrMode7Operand";
+  let ParserMatchClass = MemNoOffsetAsmOperand;
+  let MIOperandInfo = (ops GPR:$base);
 }
 
 def nohash_imm : Operand<i32> {
@@ -687,25 +826,30 @@ def nohash_imm : Operand<i32> {
 
 def CoprocNumAsmOperand : AsmOperandClass {
   let Name = "CoprocNum";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseCoprocNumOperand";
+  let ParserMethod = "parseCoprocNumOperand";
+}
+def p_imm : Operand<i32> {
+  let PrintMethod = "printPImmediate";
+  let ParserMatchClass = CoprocNumAsmOperand;
+  let DecoderMethod = "DecodeCoprocessor";
 }
 
 def CoprocRegAsmOperand : AsmOperandClass {
   let Name = "CoprocReg";
-  let SuperClasses = [];
-  let ParserMethod = "tryParseCoprocRegOperand";
+  let ParserMethod = "parseCoprocRegOperand";
 }
-
-def p_imm : Operand<i32> {
-  let PrintMethod = "printPImmediate";
-  let ParserMatchClass = CoprocNumAsmOperand;
-}
-
 def c_imm : Operand<i32> {
   let PrintMethod = "printCImmediate";
   let ParserMatchClass = CoprocRegAsmOperand;
 }
+def CoprocOptionAsmOperand : AsmOperandClass {
+  let Name = "CoprocOption";
+  let ParserMethod = "parseCoprocOptionOperand";
+}
+def coproc_option_imm : Operand<i32> {
+  let PrintMethod = "printCoprocOptionImm";
+  let ParserMatchClass = CoprocOptionAsmOperand;
+}
 
 //===----------------------------------------------------------------------===//
 
@@ -748,16 +892,37 @@ multiclass AsI1_bin_irs<bits<4> opcod, string opc,
     let Inst{11-4} = 0b00000000;
     let Inst{3-0} = Rm;
   }
-  def rs : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift), DPSoRegFrm,
+
+  def rsi : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm,
                iis, opc, "\t$Rd, $Rn, $shift",
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_reg:$shift))]> {
+               [(set GPR:$Rd, (opnode GPR:$Rn, so_reg_imm:$shift))]> {
     bits<4> Rd;
     bits<4> Rn;
     bits<12> shift;
     let Inst{25} = 0;
     let Inst{19-16} = Rn;
     let Inst{15-12} = Rd;
-    let Inst{11-0} = shift;
+    let Inst{11-5} = shift{11-5};
+    let Inst{4} = 0;
+    let Inst{3-0} = shift{3-0};
+  }
+
+  def rsr : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_reg:$shift), DPSoRegRegFrm,
+               iis, opc, "\t$Rd, $Rn, $shift",
+               [(set GPR:$Rd, (opnode GPR:$Rn, so_reg_reg:$shift))]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-8} = shift{11-8};
+    let Inst{7} = 0;
+    let Inst{6-5} = shift{6-5};
+    let Inst{4} = 1;
+    let Inst{3-0} = shift{3-0};
   }
 
   // Assembly aliases for optional destination operand when it's the same
@@ -773,56 +938,172 @@ multiclass AsI1_bin_irs<bits<4> opcod, string opc,
                                                     cc_out:$s)>,
      Requires<[IsARM]>;
   def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
-     (!cast<Instruction>(!strconcat(baseOpc, "rs")) GPR:$Rdn, GPR:$Rdn,
-                                                    so_reg:$shift, pred:$p,
+     (!cast<Instruction>(!strconcat(baseOpc, "rsi")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_imm:$shift, pred:$p,
                                                     cc_out:$s)>,
      Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rsr")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_reg:$shift, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+
 }
 
-/// AI1_bin_s_irs - Similar to AsI1_bin_irs except it sets the 's' bit so the
-/// instruction modifies the CPSR register.
-let isCodeGenOnly = 1, Defs = [CPSR] in {
-multiclass AI1_bin_s_irs<bits<4> opcod, string opc,
+/// AsI1_rbin_irs - Same as AsI1_bin_irs except the order of operands are
+/// reversed.  The 'rr' form is only defined for the disassembler; for codegen
+/// it is equivalent to the AsI1_bin_irs counterpart.
+multiclass AsI1_rbin_irs<bits<4> opcod, string opc,
                      InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
-                         PatFrag opnode, bit Commutable = 0> {
-  def ri : AI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm), DPFrm,
+                        PatFrag opnode, string baseOpc, bit Commutable = 0> {
+  // The register-immediate version is re-materializable. This is useful
+  // in particular for taking the address of a local.
+  let isReMaterializable = 1 in {
+  def ri : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm), DPFrm,
                iii, opc, "\t$Rd, $Rn, $imm",
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_imm:$imm))]> {
+               [(set GPR:$Rd, (opnode so_imm:$imm, GPR:$Rn))]> {
     bits<4> Rd;
     bits<4> Rn;
     bits<12> imm;
     let Inst{25} = 1;
-    let Inst{20} = 1;
     let Inst{19-16} = Rn;
     let Inst{15-12} = Rd;
     let Inst{11-0} = imm;
   }
-  def rr : AI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), DPFrm,
+  }
+  def rr : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), DPFrm,
                iir, opc, "\t$Rd, $Rn, $Rm",
-               [(set GPR:$Rd, (opnode GPR:$Rn, GPR:$Rm))]> {
+               [/* pattern left blank */]> {
     bits<4> Rd;
     bits<4> Rn;
     bits<4> Rm;
-    let isCommutable = Commutable;
-    let Inst{25} = 0;
-    let Inst{20} = 1;
-    let Inst{19-16} = Rn;
-    let Inst{15-12} = Rd;
     let Inst{11-4} = 0b00000000;
+    let Inst{25} = 0;
     let Inst{3-0} = Rm;
+    let Inst{15-12} = Rd;
+    let Inst{19-16} = Rn;
   }
-  def rs : AI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift), DPSoRegFrm,
+
+  def rsi : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm,
                iis, opc, "\t$Rd, $Rn, $shift",
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_reg:$shift))]> {
+               [(set GPR:$Rd, (opnode so_reg_imm:$shift, GPR:$Rn))]> {
     bits<4> Rd;
     bits<4> Rn;
     bits<12> shift;
     let Inst{25} = 0;
-    let Inst{20} = 1;
     let Inst{19-16} = Rn;
     let Inst{15-12} = Rd;
-    let Inst{11-0} = shift;
+    let Inst{11-5} = shift{11-5};
+    let Inst{4} = 0;
+    let Inst{3-0} = shift{3-0};
   }
+
+  def rsr : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_reg:$shift), DPSoRegRegFrm,
+               iis, opc, "\t$Rd, $Rn, $shift",
+               [(set GPR:$Rd, (opnode so_reg_reg:$shift, GPR:$Rn))]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-8} = shift{11-8};
+    let Inst{7} = 0;
+    let Inst{6-5} = shift{6-5};
+    let Inst{4} = 1;
+    let Inst{3-0} = shift{3-0};
+  }
+
+  // Assembly aliases for optional destination operand when it's the same
+  // as the source operand.
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $imm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "ri")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_imm:$imm, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) GPR:$Rdn, GPR:$Rdn,
+                                                    GPR:$Rm, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rsi")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_imm:$shift, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rsr")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_reg:$shift, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+
+}
+
+/// AsI1_rbin_s_is - Same as AsI1_rbin_s_is except it sets 's' bit by default.
+///
+/// These opcodes will be converted to the real non-S opcodes by
+/// AdjustInstrPostInstrSelection after giving then an optional CPSR operand.
+let hasPostISelHook = 1, isCodeGenOnly = 1, isPseudo = 1, Defs = [CPSR] in {
+multiclass AsI1_rbin_s_is<bits<4> opcod, string opc,
+                     InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
+                        PatFrag opnode, bit Commutable = 0> {
+  def ri : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm), DPFrm,
+               iii, opc, "\t$Rd, $Rn, $imm",
+               [(set GPR:$Rd, CPSR, (opnode so_imm:$imm, GPR:$Rn))]>;
+
+  def rr : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), DPFrm,
+               iir, opc, "\t$Rd, $Rn, $Rm",
+               [/* pattern left blank */]>;
+
+  def rsi : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm,
+               iis, opc, "\t$Rd, $Rn, $shift",
+               [(set GPR:$Rd, CPSR, (opnode so_reg_imm:$shift, GPR:$Rn))]>;
+
+  def rsr : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_reg:$shift), DPSoRegRegFrm,
+               iis, opc, "\t$Rd, $Rn, $shift",
+               [(set GPR:$Rd, CPSR, (opnode so_reg_reg:$shift, GPR:$Rn))]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-8} = shift{11-8};
+    let Inst{7} = 0;
+    let Inst{6-5} = shift{6-5};
+    let Inst{4} = 1;
+    let Inst{3-0} = shift{3-0};
+  }
+}
+}
+
+/// AsI1_bin_s_irs - Same as AsI1_bin_irs except it sets the 's' bit by default.
+///
+/// These opcodes will be converted to the real non-S opcodes by
+/// AdjustInstrPostInstrSelection after giving then an optional CPSR operand.
+let hasPostISelHook = 1, isCodeGenOnly = 1, isPseudo = 1, Defs = [CPSR] in {
+multiclass AsI1_bin_s_irs<bits<4> opcod, string opc,
+                     InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
+                         PatFrag opnode, bit Commutable = 0> {
+  def ri : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm), DPFrm,
+               iii, opc, "\t$Rd, $Rn, $imm",
+               [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, so_imm:$imm))]>;
+  def rr : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), DPFrm,
+               iir, opc, "\t$Rd, $Rn, $Rm",
+               [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, GPR:$Rm))]>;
+  def rsi : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm,
+               iis, opc, "\t$Rd, $Rn, $shift",
+               [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, so_reg_imm:$shift))]>;
+
+  def rsr : AsI1<opcod, (outs GPR:$Rd),
+               (ins GPR:$Rn, so_reg_reg:$shift), DPSoRegRegFrm,
+               iis, opc, "\t$Rd, $Rn, $shift",
+               [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, so_reg_reg:$shift))]>;
 }
 }
 
@@ -857,128 +1138,102 @@ multiclass AI1_cmp_irs<bits<4> opcod, string opc,
     let Inst{11-4} = 0b00000000;
     let Inst{3-0} = Rm;
   }
-  def rs : AI1<opcod, (outs), (ins GPR:$Rn, so_reg:$shift), DPSoRegFrm, iis,
+  def rsi : AI1<opcod, (outs),
+               (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm, iis,
                opc, "\t$Rn, $shift",
-               [(opnode GPR:$Rn, so_reg:$shift)]> {
+               [(opnode GPR:$Rn, so_reg_imm:$shift)]> {
     bits<4> Rn;
     bits<12> shift;
     let Inst{25} = 0;
     let Inst{20} = 1;
     let Inst{19-16} = Rn;
     let Inst{15-12} = 0b0000;
-    let Inst{11-0} = shift;
+    let Inst{11-5} = shift{11-5};
+    let Inst{4} = 0;
+    let Inst{3-0} = shift{3-0};
   }
+  def rsr : AI1<opcod, (outs),
+               (ins GPR:$Rn, so_reg_reg:$shift), DPSoRegRegFrm, iis,
+               opc, "\t$Rn, $shift",
+               [(opnode GPR:$Rn, so_reg_reg:$shift)]> {
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{20} = 1;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = 0b0000;
+    let Inst{11-8} = shift{11-8};
+    let Inst{7} = 0;
+    let Inst{6-5} = shift{6-5};
+    let Inst{4} = 1;
+    let Inst{3-0} = shift{3-0};
+  }
+
 }
 }
 
 /// AI_ext_rrot - A unary operation with two forms: one whose operand is a
 /// register and one whose operand is a register rotated by 8/16/24.
 /// FIXME: Remove the 'r' variant. Its rot_imm is zero.
-multiclass AI_ext_rrot<bits<8> opcod, string opc, PatFrag opnode> {
-  def r     : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rm),
-                 IIC_iEXTr, opc, "\t$Rd, $Rm",
-                 [(set GPR:$Rd, (opnode GPR:$Rm))]>,
-              Requires<[IsARM, HasV6]> {
-    bits<4> Rd;
-    bits<4> Rm;
-    let Inst{19-16} = 0b1111;
-    let Inst{15-12} = Rd;
-    let Inst{11-10} = 0b00;
-    let Inst{3-0}   = Rm;
-  }
-  def r_rot : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rm, rot_imm:$rot),
-                 IIC_iEXTr, opc, "\t$Rd, $Rm, ror $rot",
-                 [(set GPR:$Rd, (opnode (rotr GPR:$Rm, rot_imm:$rot)))]>,
-              Requires<[IsARM, HasV6]> {
-    bits<4> Rd;
-    bits<4> Rm;
-    bits<2> rot;
-    let Inst{19-16} = 0b1111;
-    let Inst{15-12} = Rd;
-    let Inst{11-10} = rot;
-    let Inst{3-0}   = Rm;
-  }
+class AI_ext_rrot<bits<8> opcod, string opc, PatFrag opnode>
+  : AExtI<opcod, (outs GPRnopc:$Rd), (ins GPRnopc:$Rm, rot_imm:$rot),
+          IIC_iEXTr, opc, "\t$Rd, $Rm$rot",
+          [(set GPRnopc:$Rd, (opnode (rotr GPRnopc:$Rm, rot_imm:$rot)))]>,
+       Requires<[IsARM, HasV6]> {
+  bits<4> Rd;
+  bits<4> Rm;
+  bits<2> rot;
+  let Inst{19-16} = 0b1111;
+  let Inst{15-12} = Rd;
+  let Inst{11-10} = rot;
+  let Inst{3-0}   = Rm;
 }
 
-multiclass AI_ext_rrot_np<bits<8> opcod, string opc> {
-  def r     : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rm),
-                 IIC_iEXTr, opc, "\t$Rd, $Rm",
-                 [/* For disassembly only; pattern left blank */]>,
-              Requires<[IsARM, HasV6]> {
-    let Inst{19-16} = 0b1111;
-    let Inst{11-10} = 0b00;
-  }
-  def r_rot : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rm, rot_imm:$rot),
-                 IIC_iEXTr, opc, "\t$Rd, $Rm, ror $rot",
-                 [/* For disassembly only; pattern left blank */]>,
-              Requires<[IsARM, HasV6]> {
-    bits<2> rot;
-    let Inst{19-16} = 0b1111;
-    let Inst{11-10} = rot;
-  }
+class AI_ext_rrot_np<bits<8> opcod, string opc>
+  : AExtI<opcod, (outs GPRnopc:$Rd), (ins GPRnopc:$Rm, rot_imm:$rot),
+          IIC_iEXTr, opc, "\t$Rd, $Rm$rot", []>,
+       Requires<[IsARM, HasV6]> {
+  bits<2> rot;
+  let Inst{19-16} = 0b1111;
+  let Inst{11-10} = rot;
 }
 
 /// AI_exta_rrot - A binary operation with two forms: one whose operand is a
 /// register and one whose operand is a register rotated by 8/16/24.
-multiclass AI_exta_rrot<bits<8> opcod, string opc, PatFrag opnode> {
-  def rr     : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-                  IIC_iEXTAr, opc, "\t$Rd, $Rn, $Rm",
-                  [(set GPR:$Rd, (opnode GPR:$Rn, GPR:$Rm))]>,
-               Requires<[IsARM, HasV6]> {
-    bits<4> Rd;
-    bits<4> Rm;
-    bits<4> Rn;
-    let Inst{19-16} = Rn;
-    let Inst{15-12} = Rd;
-    let Inst{11-10} = 0b00;
-    let Inst{9-4}   = 0b000111;
-    let Inst{3-0}   = Rm;
-  }
-  def rr_rot : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm,
-                                             rot_imm:$rot),
-                  IIC_iEXTAr, opc, "\t$Rd, $Rn, $Rm, ror $rot",
-                  [(set GPR:$Rd, (opnode GPR:$Rn,
-                                          (rotr GPR:$Rm, rot_imm:$rot)))]>,
-                  Requires<[IsARM, HasV6]> {
-    bits<4> Rd;
-    bits<4> Rm;
-    bits<4> Rn;
-    bits<2> rot;
-    let Inst{19-16} = Rn;
-    let Inst{15-12} = Rd;
-    let Inst{11-10} = rot;
-    let Inst{9-4}   = 0b000111;
-    let Inst{3-0}   = Rm;
-  }
+class AI_exta_rrot<bits<8> opcod, string opc, PatFrag opnode>
+  : AExtI<opcod, (outs GPRnopc:$Rd), (ins GPR:$Rn, GPRnopc:$Rm, rot_imm:$rot),
+          IIC_iEXTAr, opc, "\t$Rd, $Rn, $Rm$rot",
+          [(set GPRnopc:$Rd, (opnode GPR:$Rn,
+                                     (rotr GPRnopc:$Rm, rot_imm:$rot)))]>,
+        Requires<[IsARM, HasV6]> {
+  bits<4> Rd;
+  bits<4> Rm;
+  bits<4> Rn;
+  bits<2> rot;
+  let Inst{19-16} = Rn;
+  let Inst{15-12} = Rd;
+  let Inst{11-10} = rot;
+  let Inst{9-4}   = 0b000111;
+  let Inst{3-0}   = Rm;
 }
 
-// For disassembly only.
-multiclass AI_exta_rrot_np<bits<8> opcod, string opc> {
-  def rr     : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-                  IIC_iEXTAr, opc, "\t$Rd, $Rn, $Rm",
-                  [/* For disassembly only; pattern left blank */]>,
-               Requires<[IsARM, HasV6]> {
-    let Inst{11-10} = 0b00;
-  }
-  def rr_rot : AExtI<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm,
-                                             rot_imm:$rot),
-                  IIC_iEXTAr, opc, "\t$Rd, $Rn, $Rm, ror $rot",
-                  [/* For disassembly only; pattern left blank */]>,
-                  Requires<[IsARM, HasV6]> {
-    bits<4> Rn;
-    bits<2> rot;
-    let Inst{19-16} = Rn;
-    let Inst{11-10} = rot;
-  }
+class AI_exta_rrot_np<bits<8> opcod, string opc>
+  : AExtI<opcod, (outs GPRnopc:$Rd), (ins GPR:$Rn, GPRnopc:$Rm, rot_imm:$rot),
+          IIC_iEXTAr, opc, "\t$Rd, $Rn, $Rm$rot", []>,
+       Requires<[IsARM, HasV6]> {
+  bits<4> Rn;
+  bits<2> rot;
+  let Inst{19-16} = Rn;
+  let Inst{11-10} = rot;
 }
 
 /// AI1_adde_sube_irs - Define instructions and patterns for adde and sube.
 multiclass AI1_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
                              string baseOpc, bit Commutable = 0> {
-  let Uses = [CPSR] in {
+  let hasPostISelHook = 1, Defs = [CPSR], Uses = [CPSR] in {
   def ri : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm),
                 DPFrm, IIC_iALUi, opc, "\t$Rd, $Rn, $imm",
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_imm:$imm))]>,
+               [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, so_imm:$imm, CPSR))]>,
                Requires<[IsARM]> {
     bits<4> Rd;
     bits<4> Rn;
@@ -990,7 +1245,7 @@ multiclass AI1_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
   }
   def rr : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
                 DPFrm, IIC_iALUr, opc, "\t$Rd, $Rn, $Rm",
-               [(set GPR:$Rd, (opnode GPR:$Rn, GPR:$Rm))]>,
+               [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, GPR:$Rm, CPSR))]>,
                Requires<[IsARM]> {
     bits<4> Rd;
     bits<4> Rn;
@@ -1002,19 +1257,40 @@ multiclass AI1_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
     let Inst{15-12} = Rd;
     let Inst{19-16} = Rn;
   }
-  def rs : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift),
-                DPSoRegFrm, IIC_iALUsr, opc, "\t$Rd, $Rn, $shift",
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_reg:$shift))]>,
+  def rsi : AsI1<opcod, (outs GPR:$Rd),
+                (ins GPR:$Rn, so_reg_imm:$shift),
+                DPSoRegImmFrm, IIC_iALUsr, opc, "\t$Rd, $Rn, $shift",
+              [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, so_reg_imm:$shift, CPSR))]>,
                Requires<[IsARM]> {
     bits<4> Rd;
     bits<4> Rn;
     bits<12> shift;
     let Inst{25} = 0;
-    let Inst{11-0} = shift;
-    let Inst{15-12} = Rd;
     let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-5} = shift{11-5};
+    let Inst{4} = 0;
+    let Inst{3-0} = shift{3-0};
+  }
+  def rsr : AsI1<opcod, (outs GPR:$Rd),
+                (ins GPR:$Rn, so_reg_reg:$shift),
+                DPSoRegRegFrm, IIC_iALUsr, opc, "\t$Rd, $Rn, $shift",
+              [(set GPR:$Rd, CPSR, (opnode GPR:$Rn, so_reg_reg:$shift, CPSR))]>,
+               Requires<[IsARM]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-8} = shift{11-8};
+    let Inst{7} = 0;
+    let Inst{6-5} = shift{6-5};
+    let Inst{4} = 1;
+    let Inst{3-0} = shift{3-0};
   }
   }
+
   // Assembly aliases for optional destination operand when it's the same
   // as the source operand.
   def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $imm"),
@@ -1028,28 +1304,99 @@ multiclass AI1_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
                                                     cc_out:$s)>,
      Requires<[IsARM]>;
   def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
-     (!cast<Instruction>(!strconcat(baseOpc, "rs")) GPR:$Rdn, GPR:$Rdn,
-                                                    so_reg:$shift, pred:$p,
+     (!cast<Instruction>(!strconcat(baseOpc, "rsi")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_imm:$shift, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rsr")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_reg:$shift, pred:$p,
                                                     cc_out:$s)>,
      Requires<[IsARM]>;
 }
 
-// Carry setting variants
-// NOTE: CPSR def omitted because it will be handled by the custom inserter.
-let usesCustomInserter = 1 in {
-multiclass AI1_adde_sube_s_irs<PatFrag opnode, bit Commutable = 0> {
-  def ri : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm),
-               4, IIC_iALUi,
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_imm:$imm))]>;
-  def rr : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-               4, IIC_iALUr,
-               [(set GPR:$Rd, (opnode GPR:$Rn, GPR:$Rm))]> {
-    let isCommutable = Commutable;
+/// AI1_rsc_irs - Define instructions and patterns for rsc
+multiclass AI1_rsc_irs<bits<4> opcod, string opc, PatFrag opnode,
+                       string baseOpc> {
+  let hasPostISelHook = 1, Defs = [CPSR], Uses = [CPSR] in {
+  def ri : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm),
+                DPFrm, IIC_iALUi, opc, "\t$Rd, $Rn, $imm",
+               [(set GPR:$Rd, CPSR, (opnode so_imm:$imm, GPR:$Rn, CPSR))]>,
+               Requires<[IsARM]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> imm;
+    let Inst{25} = 1;
+    let Inst{15-12} = Rd;
+    let Inst{19-16} = Rn;
+    let Inst{11-0} = imm;
   }
-  def rs : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift),
-               4, IIC_iALUsr,
-               [(set GPR:$Rd, (opnode GPR:$Rn, so_reg:$shift))]>;
-}
+  def rr : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
+                DPFrm, IIC_iALUr, opc, "\t$Rd, $Rn, $Rm",
+               [/* pattern left blank */]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<4> Rm;
+    let Inst{11-4} = 0b00000000;
+    let Inst{25} = 0;
+    let Inst{3-0} = Rm;
+    let Inst{15-12} = Rd;
+    let Inst{19-16} = Rn;
+  }
+  def rsi : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_reg_imm:$shift),
+                DPSoRegImmFrm, IIC_iALUsr, opc, "\t$Rd, $Rn, $shift",
+              [(set GPR:$Rd, CPSR, (opnode so_reg_imm:$shift, GPR:$Rn, CPSR))]>,
+               Requires<[IsARM]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-5} = shift{11-5};
+    let Inst{4} = 0;
+    let Inst{3-0} = shift{3-0};
+  }
+  def rsr : AsI1<opcod, (outs GPR:$Rd), (ins GPR:$Rn, so_reg_reg:$shift),
+                DPSoRegRegFrm, IIC_iALUsr, opc, "\t$Rd, $Rn, $shift",
+              [(set GPR:$Rd, CPSR, (opnode so_reg_reg:$shift, GPR:$Rn, CPSR))]>,
+               Requires<[IsARM]> {
+    bits<4> Rd;
+    bits<4> Rn;
+    bits<12> shift;
+    let Inst{25} = 0;
+    let Inst{19-16} = Rn;
+    let Inst{15-12} = Rd;
+    let Inst{11-8} = shift{11-8};
+    let Inst{7} = 0;
+    let Inst{6-5} = shift{6-5};
+    let Inst{4} = 1;
+    let Inst{3-0} = shift{3-0};
+  }
+  }
+
+  // Assembly aliases for optional destination operand when it's the same
+  // as the source operand.
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $imm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "ri")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_imm:$imm, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) GPR:$Rdn, GPR:$Rdn,
+                                                    GPR:$Rm, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rsi")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_imm:$shift, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
+  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rsr")) GPR:$Rdn, GPR:$Rdn,
+                                                    so_reg_reg:$shift, pred:$p,
+                                                    cc_out:$s)>,
+     Requires<[IsARM]>;
 }
 
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
@@ -1082,6 +1429,37 @@ multiclass AI_ldr1<bit isByte, string opc, InstrItinClass iii,
 }
 }
 
+let canFoldAsLoad = 1, isReMaterializable = 1 in {
+multiclass AI_ldr1nopc<bit isByte, string opc, InstrItinClass iii,
+           InstrItinClass iir, PatFrag opnode> {
+  // Note: We use the complex addrmode_imm12 rather than just an input
+  // GPR and a constrained immediate so that we can use this to match
+  // frame index references and avoid matching constant pool references.
+  def i12: AI2ldst<0b010, 1, isByte, (outs GPRnopc:$Rt), (ins addrmode_imm12:$addr),
+                   AddrMode_i12, LdFrm, iii, opc, "\t$Rt, $addr",
+                  [(set GPRnopc:$Rt, (opnode addrmode_imm12:$addr))]> {
+    bits<4>  Rt;
+    bits<17> addr;
+    let Inst{23}    = addr{12};     // U (add = ('U' == 1))
+    let Inst{19-16} = addr{16-13};  // Rn
+    let Inst{15-12} = Rt;
+    let Inst{11-0}  = addr{11-0};   // imm12
+  }
+  def rs : AI2ldst<0b011, 1, isByte, (outs GPRnopc:$Rt), (ins ldst_so_reg:$shift),
+                  AddrModeNone, LdFrm, iir, opc, "\t$Rt, $shift",
+                 [(set GPRnopc:$Rt, (opnode ldst_so_reg:$shift))]> {
+    bits<4>  Rt;
+    bits<17> shift;
+    let shift{4}    = 0;            // Inst{4} = 0
+    let Inst{23}    = shift{12};    // U (add = ('U' == 1))
+    let Inst{19-16} = shift{16-13}; // Rn
+    let Inst{15-12} = Rt;
+    let Inst{11-0}  = shift{11-0};
+  }
+}
+}
+
+
 multiclass AI_str1<bit isByte, string opc, InstrItinClass iii,
            InstrItinClass iir, PatFrag opnode> {
   // Note: We use the complex addrmode_imm12 rather than just an input
@@ -1110,6 +1488,37 @@ multiclass AI_str1<bit isByte, string opc, InstrItinClass iii,
     let Inst{11-0}  = shift{11-0};
   }
 }
+
+multiclass AI_str1nopc<bit isByte, string opc, InstrItinClass iii,
+           InstrItinClass iir, PatFrag opnode> {
+  // Note: We use the complex addrmode_imm12 rather than just an input
+  // GPR and a constrained immediate so that we can use this to match
+  // frame index references and avoid matching constant pool references.
+  def i12 : AI2ldst<0b010, 0, isByte, (outs),
+                   (ins GPRnopc:$Rt, addrmode_imm12:$addr),
+                   AddrMode_i12, StFrm, iii, opc, "\t$Rt, $addr",
+                  [(opnode GPRnopc:$Rt, addrmode_imm12:$addr)]> {
+    bits<4> Rt;
+    bits<17> addr;
+    let Inst{23}    = addr{12};     // U (add = ('U' == 1))
+    let Inst{19-16} = addr{16-13};  // Rn
+    let Inst{15-12} = Rt;
+    let Inst{11-0}  = addr{11-0};   // imm12
+  }
+  def rs : AI2ldst<0b011, 0, isByte, (outs), (ins GPRnopc:$Rt, ldst_so_reg:$shift),
+                  AddrModeNone, StFrm, iir, opc, "\t$Rt, $shift",
+                 [(opnode GPRnopc:$Rt, ldst_so_reg:$shift)]> {
+    bits<4> Rt;
+    bits<17> shift;
+    let shift{4}    = 0;            // Inst{4} = 0
+    let Inst{23}    = shift{12};    // U (add = ('U' == 1))
+    let Inst{19-16} = shift{16-13}; // Rn
+    let Inst{15-12} = Rt;
+    let Inst{11-0}  = shift{11-0};
+  }
+}
+
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
@@ -1140,42 +1549,66 @@ PseudoInst<(outs), (ins i32imm:$amt, pred:$p), NoItinerary,
            [(ARMcallseq_start timm:$amt)]>;
 }
 
-def NOP : AI<(outs), (ins), MiscFrm, NoItinerary, "nop", "",
-             [/* For disassembly only; pattern left blank */]>,
+// Atomic pseudo-insts which will be lowered to ldrexd/strexd loops.
+// (These psuedos use a hand-written selection code).
+let usesCustomInserter = 1, Defs = [CPSR], mayLoad = 1, mayStore = 1 in {
+def ATOMOR6432   : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMXOR6432  : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMADD6432  : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMSUB6432  : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMNAND6432 : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMAND6432  : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMSWAP6432 : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                              (ins GPR:$addr, GPR:$src1, GPR:$src2),
+                              NoItinerary, []>;
+def ATOMCMPXCHG6432 : PseudoInst<(outs GPR:$dst1, GPR:$dst2),
+                                 (ins GPR:$addr, GPR:$cmp1, GPR:$cmp2,
+                                      GPR:$set1, GPR:$set2),
+                                 NoItinerary, []>;
+}
+
+def NOP : AI<(outs), (ins), MiscFrm, NoItinerary, "nop", "", []>,
           Requires<[IsARM, HasV6T2]> {
   let Inst{27-16} = 0b001100100000;
   let Inst{15-8} = 0b11110000;
   let Inst{7-0} = 0b00000000;
 }
 
-def YIELD : AI<(outs), (ins), MiscFrm, NoItinerary, "yield", "",
-             [/* For disassembly only; pattern left blank */]>,
+def YIELD : AI<(outs), (ins), MiscFrm, NoItinerary, "yield", "", []>,
           Requires<[IsARM, HasV6T2]> {
   let Inst{27-16} = 0b001100100000;
   let Inst{15-8} = 0b11110000;
   let Inst{7-0} = 0b00000001;
 }
 
-def WFE : AI<(outs), (ins), MiscFrm, NoItinerary, "wfe", "",
-             [/* For disassembly only; pattern left blank */]>,
+def WFE : AI<(outs), (ins), MiscFrm, NoItinerary, "wfe", "", []>,
           Requires<[IsARM, HasV6T2]> {
   let Inst{27-16} = 0b001100100000;
   let Inst{15-8} = 0b11110000;
   let Inst{7-0} = 0b00000010;
 }
 
-def WFI : AI<(outs), (ins), MiscFrm, NoItinerary, "wfi", "",
-             [/* For disassembly only; pattern left blank */]>,
+def WFI : AI<(outs), (ins), MiscFrm, NoItinerary, "wfi", "", []>,
           Requires<[IsARM, HasV6T2]> {
   let Inst{27-16} = 0b001100100000;
   let Inst{15-8} = 0b11110000;
   let Inst{7-0} = 0b00000011;
 }
 
-def SEL : AI<(outs GPR:$dst), (ins GPR:$a, GPR:$b), DPFrm, NoItinerary, "sel",
-             "\t$dst, $a, $b",
-             [/* For disassembly only; pattern left blank */]>,
-          Requires<[IsARM, HasV6]> {
+def SEL : AI<(outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), DPFrm, NoItinerary, "sel",
+             "\t$Rd, $Rn, $Rm", []>, Requires<[IsARM, HasV6]> {
   bits<4> Rd;
   bits<4> Rn;
   bits<4> Rm;
@@ -1188,8 +1621,7 @@ def SEL : AI<(outs GPR:$dst), (ins GPR:$a, GPR:$b), DPFrm, NoItinerary, "sel",
 }
 
 def SEV : AI<(outs), (ins), MiscFrm, NoItinerary, "sev", "",
-             [/* For disassembly only; pattern left blank */]>,
-          Requires<[IsARM, HasV6T2]> {
+             []>, Requires<[IsARM, HasV6T2]> {
   let Inst{27-16} = 0b001100100000;
   let Inst{15-8} = 0b11110000;
   let Inst{7-0} = 0b00000100;
@@ -1206,14 +1638,11 @@ def BKPT : AI<(outs), (ins imm0_65535:$val), MiscFrm, NoItinerary,
   let Inst{7-4} = 0b0111;
 }
 
-// Change Processor State is a system instruction -- for disassembly and
-// parsing only.
-// FIXME: Since the asm parser has currently no clean way to handle optional
-// operands, create 3 versions of the same instruction. Once there's a clean
-// framework to represent optional operands, change this behavior.
+// Change Processor State
+// FIXME: We should use InstAlias to handle the optional operands.
 class CPS<dag iops, string asm_ops>
   : AXI<(outs), iops, MiscFrm, NoItinerary, !strconcat("cps", asm_ops),
-        [/* For disassembly only; pattern left blank */]>, Requires<[IsARM]> {
+        []>, Requires<[IsARM]> {
   bits<2> imod;
   bits<3> iflags;
   bits<5> mode;
@@ -1229,17 +1658,18 @@ class CPS<dag iops, string asm_ops>
   let Inst{4-0}   = mode;
 }
 
+let DecoderMethod = "DecodeCPSInstruction" in {
 let M = 1 in
-  def CPS3p : CPS<(ins imod_op:$imod, iflags_op:$iflags, i32imm:$mode),
+  def CPS3p : CPS<(ins imod_op:$imod, iflags_op:$iflags, imm0_31:$mode),
                   "$imod\t$iflags, $mode">;
 let mode = 0, M = 0 in
   def CPS2p : CPS<(ins imod_op:$imod, iflags_op:$iflags), "$imod\t$iflags">;
 
 let imod = 0, iflags = 0, M = 1 in
-  def CPS1p : CPS<(ins i32imm:$mode), "\t$mode">;
+  def CPS1p : CPS<(ins imm0_31:$mode), "\t$mode">;
+}
 
 // Preload signals the memory system of possible future data/instruction access.
-// These are for disassembly only.
 multiclass APreLoad<bits<1> read, bits<1> data, string opc> {
 
   def i12 : AXI<(outs), (ins addrmode_imm12:$addr), MiscFrm, IIC_Preload,
@@ -1271,6 +1701,7 @@ multiclass APreLoad<bits<1> read, bits<1> data, string opc> {
     let Inst{19-16} = shift{16-13}; // Rn
     let Inst{15-12} = 0b1111;
     let Inst{11-0}  = shift{11-0};
+    let Inst{4} = 0;
   }
 }
 
@@ -1278,10 +1709,8 @@ defm PLD  : APreLoad<1, 1, "pld">,  Requires<[IsARM]>;
 defm PLDW : APreLoad<0, 1, "pldw">, Requires<[IsARM,HasV7,HasMP]>;
 defm PLI  : APreLoad<1, 0, "pli">,  Requires<[IsARM,HasV7]>;
 
-def SETEND : AXI<(outs),(ins setend_op:$end), MiscFrm, NoItinerary,
-                 "setend\t$end",
-                 [/* For disassembly only; pattern left blank */]>,
-               Requires<[IsARM]> {
+def SETEND : AXI<(outs), (ins setend_op:$end), MiscFrm, NoItinerary,
+                 "setend\t$end", []>, Requires<[IsARM]> {
   bits<1> end;
   let Inst{31-10} = 0b1111000100000001000000;
   let Inst{9} = end;
@@ -1351,14 +1780,17 @@ let neverHasSideEffects = 1, isReMaterializable = 1 in
 // the instruction. The {24-21} opcode bits are set by the fixup, as we don't
 // know until then which form of the instruction will be used.
 def ADR : AI1<{0,?,?,0}, (outs GPR:$Rd), (ins adrlabel:$label),
-                 MiscFrm, IIC_iALUi, "adr", "\t$Rd, #$label", []> {
+                 MiscFrm, IIC_iALUi, "adr", "\t$Rd, $label", []> {
   bits<4> Rd;
-  bits<12> label;
+  bits<14> label;
   let Inst{27-25} = 0b001;
+  let Inst{24} = 0;
+  let Inst{23-22} = label{13-12};
+  let Inst{21} = 0;
   let Inst{20} = 0;
   let Inst{19-16} = 0b1111;
   let Inst{15-12} = Rd;
-  let Inst{11-0} = label;
+  let Inst{11-0} = label{11-0};
 }
 def LEApcrel : ARMPseudoInst<(outs GPR:$Rd), (ins i32imm:$label, pred:$p),
                     4, IIC_iALUi, []>;
@@ -1424,6 +1856,7 @@ let isCall = 1,
     let Inst{31-28} = 0b1110;
     bits<24> func;
     let Inst{23-0} = func;
+    let DecoderMethod = "DecodeBranchImmInstruction";
   }
 
   def BL_pred : ABI<0b1011, (outs), (ins bl_target:$func, variable_ops),
@@ -1432,6 +1865,7 @@ let isCall = 1,
                 Requires<[IsARM, IsNotDarwin]> {
     bits<24> func;
     let Inst{23-0} = func;
+    let DecoderMethod = "DecodeBranchImmInstruction";
   }
 
   // ARMv5T and above
@@ -1516,6 +1950,7 @@ let isBranch = 1, isTerminator = 1 in {
                [/*(ARMbrcond bb:$target, imm:$cc, CCR:$ccr)*/]> {
     bits<24> target;
     let Inst{23-0} = target;
+    let DecoderMethod = "DecodeBranchImmInstruction";
   }
 
   let isBarrier = 1 in {
@@ -1549,9 +1984,9 @@ let isBranch = 1, isTerminator = 1 in {
 
 }
 
-// BLX (immediate) -- for disassembly only
-def BLXi : AXI<(outs), (ins br_target:$target), BrMiscFrm, NoItinerary,
-               "blx\t$target", [/* pattern left blank */]>,
+// BLX (immediate)
+def BLXi : AXI<(outs), (ins blx_target:$target), BrMiscFrm, NoItinerary,
+               "blx\t$target", []>,
            Requires<[IsARM, HasV5T]> {
   let Inst{31-25} = 0b1111101;
   bits<25> target;
@@ -1614,64 +2049,100 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1 in {
   }
 }
 
-
-
-
-
-// Secure Monitor Call is a system instruction -- for disassembly only
-def SMC : ABI<0b0001, (outs), (ins i32imm:$opt), NoItinerary, "smc", "\t$opt",
-              [/* For disassembly only; pattern left blank */]> {
+// Secure Monitor Call is a system instruction.
+def SMC : ABI<0b0001, (outs), (ins imm0_15:$opt), NoItinerary, "smc", "\t$opt",
+              []> {
   bits<4> opt;
   let Inst{23-4} = 0b01100000000000000111;
   let Inst{3-0} = opt;
 }
 
-// Supervisor Call (Software Interrupt) -- for disassembly only
+// Supervisor Call (Software Interrupt)
 let isCall = 1, Uses = [SP] in {
-def SVC : ABI<0b1111, (outs), (ins i32imm:$svc), IIC_Br, "svc", "\t$svc",
-              [/* For disassembly only; pattern left blank */]> {
+def SVC : ABI<0b1111, (outs), (ins imm24b:$svc), IIC_Br, "svc", "\t$svc", []> {
   bits<24> svc;
   let Inst{23-0} = svc;
 }
 }
 
-// Store Return State is a system instruction -- for disassembly only
-let isCodeGenOnly = 1 in {  // FIXME: This should not use submode!
-def SRSW : ABXI<{1,0,0,?}, (outs), (ins ldstm_mode:$amode, i32imm:$mode),
-                NoItinerary, "srs${amode}\tsp!, $mode",
-                [/* For disassembly only; pattern left blank */]> {
+// Store Return State
+class SRSI<bit wb, string asm>
+  : XI<(outs), (ins imm0_31:$mode), AddrModeNone, 4, IndexModeNone, BrFrm,
+       NoItinerary, asm, "", []> {
+  bits<5> mode;
   let Inst{31-28} = 0b1111;
-  let Inst{22-20} = 0b110; // W = 1
-  let Inst{19-8} = 0xd05;
-  let Inst{7-5} = 0b000;
+  let Inst{27-25} = 0b100;
+  let Inst{22} = 1;
+  let Inst{21} = wb;
+  let Inst{20} = 0;
+  let Inst{19-16} = 0b1101;  // SP
+  let Inst{15-5} = 0b00000101000;
+  let Inst{4-0} = mode;
 }
 
-def SRS  : ABXI<{1,0,0,?}, (outs), (ins ldstm_mode:$amode, i32imm:$mode),
-                NoItinerary, "srs${amode}\tsp, $mode",
-                [/* For disassembly only; pattern left blank */]> {
-  let Inst{31-28} = 0b1111;
-  let Inst{22-20} = 0b100; // W = 0
-  let Inst{19-8} = 0xd05;
-  let Inst{7-5} = 0b000;
+def SRSDA : SRSI<0, "srsda\tsp, $mode"> {
+  let Inst{24-23} = 0;
+}
+def SRSDA_UPD : SRSI<1, "srsda\tsp!, $mode"> {
+  let Inst{24-23} = 0;
+}
+def SRSDB : SRSI<0, "srsdb\tsp, $mode"> {
+  let Inst{24-23} = 0b10;
+}
+def SRSDB_UPD : SRSI<1, "srsdb\tsp!, $mode"> {
+  let Inst{24-23} = 0b10;
+}
+def SRSIA : SRSI<0, "srsia\tsp, $mode"> {
+  let Inst{24-23} = 0b01;
+}
+def SRSIA_UPD : SRSI<1, "srsia\tsp!, $mode"> {
+  let Inst{24-23} = 0b01;
+}
+def SRSIB : SRSI<0, "srsib\tsp, $mode"> {
+  let Inst{24-23} = 0b11;
+}
+def SRSIB_UPD : SRSI<1, "srsib\tsp!, $mode"> {
+  let Inst{24-23} = 0b11;
 }
 
-// Return From Exception is a system instruction -- for disassembly only
-def RFEW : ABXI<{1,0,0,?}, (outs), (ins ldstm_mode:$amode, GPR:$base),
-                NoItinerary, "rfe${amode}\t$base!",
-                [/* For disassembly only; pattern left blank */]> {
+// Return From Exception
+class RFEI<bit wb, string asm>
+  : XI<(outs), (ins GPR:$Rn), AddrModeNone, 4, IndexModeNone, BrFrm,
+       NoItinerary, asm, "", []> {
+  bits<4> Rn;
   let Inst{31-28} = 0b1111;
-  let Inst{22-20} = 0b011; // W = 1
-  let Inst{15-0} = 0x0a00;
+  let Inst{27-25} = 0b100;
+  let Inst{22} = 0;
+  let Inst{21} = wb;
+  let Inst{20} = 1;
+  let Inst{19-16} = Rn;
+  let Inst{15-0} = 0xa00;
 }
 
-def RFE  : ABXI<{1,0,0,?}, (outs), (ins ldstm_mode:$amode, GPR:$base),
-                NoItinerary, "rfe${amode}\t$base",
-                [/* For disassembly only; pattern left blank */]> {
-  let Inst{31-28} = 0b1111;
-  let Inst{22-20} = 0b001; // W = 0
-  let Inst{15-0} = 0x0a00;
+def RFEDA : RFEI<0, "rfeda\t$Rn"> {
+  let Inst{24-23} = 0;
+}
+def RFEDA_UPD : RFEI<1, "rfeda\t$Rn!"> {
+  let Inst{24-23} = 0;
+}
+def RFEDB : RFEI<0, "rfedb\t$Rn"> {
+  let Inst{24-23} = 0b10;
+}
+def RFEDB_UPD : RFEI<1, "rfedb\t$Rn!"> {
+  let Inst{24-23} = 0b10;
+}
+def RFEIA : RFEI<0, "rfeia\t$Rn"> {
+  let Inst{24-23} = 0b01;
+}
+def RFEIA_UPD : RFEI<1, "rfeia\t$Rn!"> {
+  let Inst{24-23} = 0b01;
+}
+def RFEIB : RFEI<0, "rfeib\t$Rn"> {
+  let Inst{24-23} = 0b11;
+}
+def RFEIB_UPD : RFEI<1, "rfeib\t$Rn!"> {
+  let Inst{24-23} = 0b11;
 }
-} // isCodeGenOnly = 1
 
 //===----------------------------------------------------------------------===//
 //  Load / store Instructions.
@@ -1682,16 +2153,16 @@ def RFE  : ABXI<{1,0,0,?}, (outs), (ins ldstm_mode:$amode, GPR:$base),
 
 defm LDR  : AI_ldr1<0, "ldr", IIC_iLoad_r, IIC_iLoad_si,
                     UnOpFrag<(load node:$Src)>>;
-defm LDRB : AI_ldr1<1, "ldrb", IIC_iLoad_bh_r, IIC_iLoad_bh_si,
+defm LDRB : AI_ldr1nopc<1, "ldrb", IIC_iLoad_bh_r, IIC_iLoad_bh_si,
                     UnOpFrag<(zextloadi8 node:$Src)>>;
 defm STR  : AI_str1<0, "str", IIC_iStore_r, IIC_iStore_si,
                    BinOpFrag<(store node:$LHS, node:$RHS)>>;
-defm STRB : AI_str1<1, "strb", IIC_iStore_bh_r, IIC_iStore_bh_si,
+defm STRB : AI_str1nopc<1, "strb", IIC_iStore_bh_r, IIC_iStore_bh_si,
                    BinOpFrag<(truncstorei8 node:$LHS, node:$RHS)>>;
 
 // Special LDR for loads from non-pc-relative constpools.
 let canFoldAsLoad = 1, mayLoad = 1, neverHasSideEffects = 1,
-    isReMaterializable = 1 in
+    isReMaterializable = 1, isCodeGenOnly = 1 in
 def LDRcp : AI2ldst<0b010, 1, 0, (outs GPR:$Rt), (ins addrmode_imm12:$addr),
                  AddrMode_i12, LdFrm, IIC_iLoad_r, "ldr", "\t$Rt, $addr",
                  []> {
@@ -1727,34 +2198,65 @@ def LDRD : AI3ld<0b1101, 0, (outs GPR:$Rd, GPR:$dst2),
 
 // Indexed loads
 multiclass AI2_ldridx<bit isByte, string opc, InstrItinClass itin> {
-  def _PRE  : AI2ldstidx<1, isByte, 1, (outs GPR:$Rt, GPR:$Rn_wb),
-                      (ins addrmode2:$addr), IndexModePre, LdFrm, itin,
+  def _PRE_IMM  : AI2ldstidx<1, isByte, 1, (outs GPR:$Rt, GPR:$Rn_wb),
+                      (ins addrmode_imm12:$addr), IndexModePre, LdFrm, itin,
                       opc, "\t$Rt, $addr!", "$addr.base = $Rn_wb", []> {
-    // {17-14}  Rn
-    // {13}     1 == Rm, 0 == imm12
-    // {12}     isAdd
-    // {11-0}   imm12/Rm
-    bits<18> addr;
-    let Inst{25} = addr{13};
+    bits<17> addr;
+    let Inst{25} = 0;
     let Inst{23} = addr{12};
-    let Inst{19-16} = addr{17-14};
+    let Inst{19-16} = addr{16-13};
     let Inst{11-0} = addr{11-0};
-    let AsmMatchConverter = "CvtLdWriteBackRegAddrMode2";
+    let DecoderMethod = "DecodeLDRPreImm";
+    let AsmMatchConverter = "cvtLdWriteBackRegAddrModeImm12";
   }
-  def _POST : AI2ldstidx<1, isByte, 0, (outs GPR:$Rt, GPR:$Rn_wb),
-                      (ins GPR:$Rn, am2offset:$offset),
+
+  def _PRE_REG  : AI2ldstidx<1, isByte, 1, (outs GPR:$Rt, GPR:$Rn_wb),
+                      (ins ldst_so_reg:$addr), IndexModePre, LdFrm, itin,
+                      opc, "\t$Rt, $addr!", "$addr.base = $Rn_wb", []> {
+    bits<17> addr;
+    let Inst{25} = 1;
+    let Inst{23} = addr{12};
+    let Inst{19-16} = addr{16-13};
+    let Inst{11-0} = addr{11-0};
+    let Inst{4} = 0;
+    let DecoderMethod = "DecodeLDRPreReg";
+    let AsmMatchConverter = "cvtLdWriteBackRegAddrMode2";
+  }
+
+  def _POST_REG : AI2ldstidx<1, isByte, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                       (ins addr_offset_none:$addr, am2offset_reg:$offset),
+                       IndexModePost, LdFrm, itin,
+                       opc, "\t$Rt, $addr, $offset",
+                       "$addr.base = $Rn_wb", []> {
+     // {12}     isAdd
+     // {11-0}   imm12/Rm
+     bits<14> offset;
+     bits<4> addr;
+     let Inst{25} = 1;
+     let Inst{23} = offset{12};
+     let Inst{19-16} = addr;
+     let Inst{11-0} = offset{11-0};
+
+    let DecoderMethod = "DecodeAddrMode2IdxInstruction";
+   }
+
+   def _POST_IMM : AI2ldstidx<1, isByte, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                       (ins addr_offset_none:$addr, am2offset_imm:$offset),
                       IndexModePost, LdFrm, itin,
-                      opc, "\t$Rt, [$Rn], $offset", "$Rn = $Rn_wb", []> {
-    // {13}     1 == Rm, 0 == imm12
+                      opc, "\t$Rt, $addr, $offset",
+                      "$addr.base = $Rn_wb", []> {
     // {12}     isAdd
     // {11-0}   imm12/Rm
     bits<14> offset;
-    bits<4> Rn;
-    let Inst{25} = offset{13};
+    bits<4> addr;
+    let Inst{25} = 0;
     let Inst{23} = offset{12};
-    let Inst{19-16} = Rn;
+    let Inst{19-16} = addr;
     let Inst{11-0} = offset{11-0};
+
+    let DecoderMethod = "DecodeAddrMode2IdxInstruction";
   }
+
 }
 
 let mayLoad = 1, neverHasSideEffects = 1 in {
@@ -1762,8 +2264,8 @@ defm LDR  : AI2_ldridx<0, "ldr", IIC_iLoad_ru>;
 defm LDRB : AI2_ldridx<1, "ldrb", IIC_iLoad_bh_ru>;
 }
 
-multiclass AI3_ldridx<bits<4> op, bit op20, string opc, InstrItinClass itin> {
-  def _PRE  : AI3ldstidx<op, op20, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
+multiclass AI3_ldridx<bits<4> op, string opc, InstrItinClass itin> {
+  def _PRE  : AI3ldstidx<op, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
                         (ins addrmode3:$addr), IndexModePre,
                         LdMiscFrm, itin,
                         opc, "\t$Rt, $addr!", "$addr.base = $Rn_wb", []> {
@@ -1773,27 +2275,31 @@ multiclass AI3_ldridx<bits<4> op, bit op20, string opc, InstrItinClass itin> {
     let Inst{19-16} = addr{12-9};   // Rn
     let Inst{11-8}  = addr{7-4};    // imm7_4/zero
     let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
+    let AsmMatchConverter = "cvtLdWriteBackRegAddrMode3";
+    let DecoderMethod = "DecodeAddrMode3Instruction";
   }
-  def _POST : AI3ldstidx<op, op20, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
-                        (ins GPR:$Rn, am3offset:$offset), IndexModePost,
-                        LdMiscFrm, itin,
-                        opc, "\t$Rt, [$Rn], $offset", "$Rn = $Rn_wb", []> {
+  def _POST : AI3ldstidx<op, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                        (ins addr_offset_none:$addr, am3offset:$offset),
+                        IndexModePost, LdMiscFrm, itin,
+                        opc, "\t$Rt, $addr, $offset", "$addr.base = $Rn_wb",
+                        []> {
     bits<10> offset;
-    bits<4> Rn;
+    bits<4> addr;
     let Inst{23}    = offset{8};      // U bit
     let Inst{22}    = offset{9};      // 1 == imm8, 0 == Rm
-    let Inst{19-16} = Rn;
+    let Inst{19-16} = addr;
     let Inst{11-8}  = offset{7-4};    // imm7_4/zero
     let Inst{3-0}   = offset{3-0};    // imm3_0/Rm
+    let DecoderMethod = "DecodeAddrMode3Instruction";
   }
 }
 
 let mayLoad = 1, neverHasSideEffects = 1 in {
-defm LDRH  : AI3_ldridx<0b1011, 1, "ldrh", IIC_iLoad_bh_ru>;
-defm LDRSH : AI3_ldridx<0b1111, 1, "ldrsh", IIC_iLoad_bh_ru>;
-defm LDRSB : AI3_ldridx<0b1101, 1, "ldrsb", IIC_iLoad_bh_ru>;
+defm LDRH  : AI3_ldridx<0b1011, "ldrh", IIC_iLoad_bh_ru>;
+defm LDRSH : AI3_ldridx<0b1111, "ldrsh", IIC_iLoad_bh_ru>;
+defm LDRSB : AI3_ldridx<0b1101, "ldrsb", IIC_iLoad_bh_ru>;
 let hasExtraDefRegAllocReq = 1 in {
-def LDRD_PRE : AI3ldstidx<0b1101, 0, 1, 1, (outs GPR:$Rt, GPR:$Rt2, GPR:$Rn_wb),
+def LDRD_PRE : AI3ldstidx<0b1101, 0, 1, (outs GPR:$Rt, GPR:$Rt2, GPR:$Rn_wb),
                           (ins addrmode3:$addr), IndexModePre,
                           LdMiscFrm, IIC_iLoad_d_ru,
                           "ldrd", "\t$Rt, $Rt2, $addr!",
@@ -1804,70 +2310,128 @@ def LDRD_PRE : AI3ldstidx<0b1101, 0, 1, 1, (outs GPR:$Rt, GPR:$Rt2, GPR:$Rn_wb),
   let Inst{19-16} = addr{12-9};   // Rn
   let Inst{11-8}  = addr{7-4};    // imm7_4/zero
   let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
+  let DecoderMethod = "DecodeAddrMode3Instruction";
+  let AsmMatchConverter = "cvtLdrdPre";
 }
-def LDRD_POST: AI3ldstidx<0b1101, 0, 1, 0, (outs GPR:$Rt, GPR:$Rt2, GPR:$Rn_wb),
-                          (ins GPR:$Rn, am3offset:$offset), IndexModePost,
-                          LdMiscFrm, IIC_iLoad_d_ru,
-                          "ldrd", "\t$Rt, $Rt2, [$Rn], $offset",
-                          "$Rn = $Rn_wb", []> {
+def LDRD_POST: AI3ldstidx<0b1101, 0, 0, (outs GPR:$Rt, GPR:$Rt2, GPR:$Rn_wb),
+                          (ins addr_offset_none:$addr, am3offset:$offset),
+                          IndexModePost, LdMiscFrm, IIC_iLoad_d_ru,
+                          "ldrd", "\t$Rt, $Rt2, $addr, $offset",
+                          "$addr.base = $Rn_wb", []> {
   bits<10> offset;
-  bits<4> Rn;
+  bits<4> addr;
   let Inst{23}    = offset{8};      // U bit
   let Inst{22}    = offset{9};      // 1 == imm8, 0 == Rm
-  let Inst{19-16} = Rn;
+  let Inst{19-16} = addr;
   let Inst{11-8}  = offset{7-4};    // imm7_4/zero
   let Inst{3-0}   = offset{3-0};    // imm3_0/Rm
+  let DecoderMethod = "DecodeAddrMode3Instruction";
 }
 } // hasExtraDefRegAllocReq = 1
 } // mayLoad = 1, neverHasSideEffects = 1
 
-// LDRT, LDRBT, LDRSBT, LDRHT, LDRSHT are for disassembly only.
+// LDRT, LDRBT, LDRSBT, LDRHT, LDRSHT.
 let mayLoad = 1, neverHasSideEffects = 1 in {
-def LDRT : AI2ldstidx<1, 0, 0, (outs GPR:$Rt, GPR:$base_wb),
-                   (ins addrmode2:$addr), IndexModePost, LdFrm, IIC_iLoad_ru,
-                   "ldrt", "\t$Rt, $addr", "$addr.base = $base_wb", []> {
-  // {17-14}  Rn
-  // {13}     1 == Rm, 0 == imm12
+def LDRT_POST_REG : AI2ldstidx<1, 0, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                    (ins addr_offset_none:$addr, am2offset_reg:$offset),
+                    IndexModePost, LdFrm, IIC_iLoad_ru,
+                    "ldrt", "\t$Rt, $addr, $offset",
+                    "$addr.base = $Rn_wb", []> {
   // {12}     isAdd
   // {11-0}   imm12/Rm
-  bits<18> addr;
-  let Inst{25} = addr{13};
-  let Inst{23} = addr{12};
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 1;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
-  let Inst{19-16} = addr{17-14};
-  let Inst{11-0} = addr{11-0};
-  let AsmMatchConverter = "CvtLdWriteBackRegAddrMode2";
+  let Inst{19-16} = addr;
+  let Inst{11-5} = offset{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = offset{3-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
-def LDRBT : AI2ldstidx<1, 1, 0, (outs GPR:$Rt, GPR:$base_wb),
-                  (ins addrmode2:$addr), IndexModePost, LdFrm, IIC_iLoad_bh_ru,
-                  "ldrbt", "\t$Rt, $addr", "$addr.base = $base_wb", []> {
-  // {17-14}  Rn
-  // {13}     1 == Rm, 0 == imm12
+
+def LDRT_POST_IMM : AI2ldstidx<1, 0, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                    (ins addr_offset_none:$addr, am2offset_imm:$offset),
+                   IndexModePost, LdFrm, IIC_iLoad_ru,
+                   "ldrt", "\t$Rt, $addr, $offset",
+                   "$addr.base = $Rn_wb", []> {
   // {12}     isAdd
   // {11-0}   imm12/Rm
-  bits<18> addr;
-  let Inst{25} = addr{13};
-  let Inst{23} = addr{12};
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 0;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
-  let Inst{19-16} = addr{17-14};
-  let Inst{11-0} = addr{11-0};
-  let AsmMatchConverter = "CvtLdWriteBackRegAddrMode2";
+  let Inst{19-16} = addr;
+  let Inst{11-0} = offset{11-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
-def LDRSBT : AI3ldstidxT<0b1101, 1, 1, 0, (outs GPR:$Rt, GPR:$base_wb),
-             (ins addrmode3:$addr), IndexModePost, LdMiscFrm, IIC_iLoad_bh_ru,
-             "ldrsbt", "\t$Rt, $addr", "$addr.base = $base_wb", []> {
+
+def LDRBT_POST_REG : AI2ldstidx<1, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                     (ins addr_offset_none:$addr, am2offset_reg:$offset),
+                     IndexModePost, LdFrm, IIC_iLoad_bh_ru,
+                     "ldrbt", "\t$Rt, $addr, $offset",
+                     "$addr.base = $Rn_wb", []> {
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 1;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
+  let Inst{19-16} = addr;
+  let Inst{11-5} = offset{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = offset{3-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
-def LDRHT  : AI3ldstidxT<0b1011, 1, 1, 0, (outs GPR:$Rt, GPR:$base_wb),
-             (ins addrmode3:$addr), IndexModePost, LdMiscFrm, IIC_iLoad_bh_ru,
-             "ldrht", "\t$Rt, $addr", "$addr.base = $base_wb", []> {
+
+def LDRBT_POST_IMM : AI2ldstidx<1, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                     (ins addr_offset_none:$addr, am2offset_imm:$offset),
+                    IndexModePost, LdFrm, IIC_iLoad_bh_ru,
+                    "ldrbt", "\t$Rt, $addr, $offset",
+                    "$addr.base = $Rn_wb", []> {
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 0;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
+  let Inst{19-16} = addr;
+  let Inst{11-0} = offset{11-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
-def LDRSHT : AI3ldstidxT<0b1111, 1, 1, 0, (outs GPR:$Rt, GPR:$base_wb),
-             (ins addrmode3:$addr), IndexModePost, LdMiscFrm, IIC_iLoad_bh_ru,
-             "ldrsht", "\t$Rt, $addr", "$addr.base = $base_wb", []> {
-  let Inst{21} = 1; // overwrite
+
+multiclass AI3ldrT<bits<4> op, string opc> {
+  def i : AI3ldstidxT<op, 1, (outs GPR:$Rt, GPR:$base_wb),
+                      (ins addr_offset_none:$addr, postidx_imm8:$offset),
+                      IndexModePost, LdMiscFrm, IIC_iLoad_bh_ru, opc,
+                      "\t$Rt, $addr, $offset", "$addr.base = $base_wb", []> {
+    bits<9> offset;
+    let Inst{23} = offset{8};
+    let Inst{22} = 1;
+    let Inst{11-8} = offset{7-4};
+    let Inst{3-0} = offset{3-0};
+    let AsmMatchConverter = "cvtLdExtTWriteBackImm";
+  }
+  def r : AI3ldstidxT<op, 1, (outs GPR:$Rt, GPR:$base_wb),
+                      (ins addr_offset_none:$addr, postidx_reg:$Rm),
+                      IndexModePost, LdMiscFrm, IIC_iLoad_bh_ru, opc,
+                      "\t$Rt, $addr, $Rm", "$addr.base = $base_wb", []> {
+    bits<5> Rm;
+    let Inst{23} = Rm{4};
+    let Inst{22} = 0;
+    let Inst{11-8} = 0;
+    let Inst{3-0} = Rm{3-0};
+    let AsmMatchConverter = "cvtLdExtTWriteBackReg";
+  }
 }
+
+defm LDRSBT : AI3ldrT<0b1101, "ldrsbt">;
+defm LDRHT  : AI3ldrT<0b1011, "ldrht">;
+defm LDRSHT : AI3ldrT<0b1111, "ldrsht">;
 }
 
 // Store
@@ -1881,98 +2445,302 @@ def STRH : AI3str<0b1011, (outs), (ins GPR:$Rt, addrmode3:$addr), StMiscFrm,
 let mayStore = 1, neverHasSideEffects = 1, hasExtraSrcRegAllocReq = 1 in
 def STRD : AI3str<0b1111, (outs), (ins GPR:$Rt, GPR:$src2, addrmode3:$addr),
                StMiscFrm, IIC_iStore_d_r,
-               "strd", "\t$Rt, $src2, $addr", []>, Requires<[IsARM, HasV5TE]>;
+               "strd", "\t$Rt, $src2, $addr", []>,
+           Requires<[IsARM, HasV5TE]> {
+  let Inst{21} = 0;
+}
 
 // Indexed stores
-def STR_PRE  : AI2stridx<0, 1, (outs GPR:$Rn_wb),
-                     (ins GPR:$Rt, GPR:$Rn, am2offset:$offset),
-                     IndexModePre, StFrm, IIC_iStore_ru,
-                     "str", "\t$Rt, [$Rn, $offset]!",
-                     "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
-                     [(set GPR:$Rn_wb,
-                      (pre_store GPR:$Rt, GPR:$Rn, am2offset:$offset))]>;
+multiclass AI2_stridx<bit isByte, string opc, InstrItinClass itin> {
+  def _PRE_IMM : AI2ldstidx<0, isByte, 1, (outs GPR:$Rn_wb),
+                            (ins GPR:$Rt, addrmode_imm12:$addr), IndexModePre,
+                            StFrm, itin,
+                            opc, "\t$Rt, $addr!", "$addr.base = $Rn_wb", []> {
+    bits<17> addr;
+    let Inst{25} = 0;
+    let Inst{23}    = addr{12};     // U (add = ('U' == 1))
+    let Inst{19-16} = addr{16-13};  // Rn
+    let Inst{11-0}  = addr{11-0};   // imm12
+    let AsmMatchConverter = "cvtStWriteBackRegAddrModeImm12";
+    let DecoderMethod = "DecodeSTRPreImm";
+  }
 
-def STR_POST : AI2stridx<0, 0, (outs GPR:$Rn_wb),
-                     (ins GPR:$Rt, GPR:$Rn, am2offset:$offset),
-                     IndexModePost, StFrm, IIC_iStore_ru,
-                     "str", "\t$Rt, [$Rn], $offset",
-                     "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
-                     [(set GPR:$Rn_wb,
-                      (post_store GPR:$Rt, GPR:$Rn, am2offset:$offset))]>;
+  def _PRE_REG  : AI2ldstidx<0, isByte, 1, (outs GPR:$Rn_wb),
+                      (ins GPR:$Rt, ldst_so_reg:$addr),
+                      IndexModePre, StFrm, itin,
+                      opc, "\t$Rt, $addr!", "$addr.base = $Rn_wb", []> {
+    bits<17> addr;
+    let Inst{25} = 1;
+    let Inst{23}    = addr{12};    // U (add = ('U' == 1))
+    let Inst{19-16} = addr{16-13}; // Rn
+    let Inst{11-0}  = addr{11-0};
+    let Inst{4}     = 0;           // Inst{4} = 0
+    let AsmMatchConverter = "cvtStWriteBackRegAddrMode2";
+    let DecoderMethod = "DecodeSTRPreReg";
+  }
+  def _POST_REG : AI2ldstidx<0, isByte, 0, (outs GPR:$Rn_wb),
+                (ins GPR:$Rt, addr_offset_none:$addr, am2offset_reg:$offset),
+                IndexModePost, StFrm, itin,
+                opc, "\t$Rt, $addr, $offset",
+                "$addr.base = $Rn_wb", []> {
+     // {12}     isAdd
+     // {11-0}   imm12/Rm
+     bits<14> offset;
+     bits<4> addr;
+     let Inst{25} = 1;
+     let Inst{23} = offset{12};
+     let Inst{19-16} = addr;
+     let Inst{11-0} = offset{11-0};
 
-def STRB_PRE : AI2stridx<1, 1, (outs GPR:$Rn_wb),
-                     (ins GPR:$Rt, GPR:$Rn, am2offset:$offset),
-                     IndexModePre, StFrm, IIC_iStore_bh_ru,
-                     "strb", "\t$Rt, [$Rn, $offset]!",
-                     "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
-                     [(set GPR:$Rn_wb, (pre_truncsti8 GPR:$Rt,
-                                        GPR:$Rn, am2offset:$offset))]>;
-def STRB_POST: AI2stridx<1, 0, (outs GPR:$Rn_wb),
-                     (ins GPR:$Rt, GPR:$Rn, am2offset:$offset),
-                     IndexModePost, StFrm, IIC_iStore_bh_ru,
-                     "strb", "\t$Rt, [$Rn], $offset",
-                     "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
-                     [(set GPR:$Rn_wb, (post_truncsti8 GPR:$Rt,
-                                        GPR:$Rn, am2offset:$offset))]>;
+    let DecoderMethod = "DecodeAddrMode2IdxInstruction";
+   }
 
-def STRH_PRE : AI3stridx<0b1011, 0, 1, (outs GPR:$Rn_wb),
-                     (ins GPR:$Rt, GPR:$Rn, am3offset:$offset),
-                     IndexModePre, StMiscFrm, IIC_iStore_ru,
-                     "strh", "\t$Rt, [$Rn, $offset]!",
-                     "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
-                     [(set GPR:$Rn_wb,
-                      (pre_truncsti16 GPR:$Rt, GPR:$Rn, am3offset:$offset))]>;
+   def _POST_IMM : AI2ldstidx<0, isByte, 0, (outs GPR:$Rn_wb),
+                (ins GPR:$Rt, addr_offset_none:$addr, am2offset_imm:$offset),
+                IndexModePost, StFrm, itin,
+                opc, "\t$Rt, $addr, $offset",
+                "$addr.base = $Rn_wb", []> {
+    // {12}     isAdd
+    // {11-0}   imm12/Rm
+    bits<14> offset;
+    bits<4> addr;
+    let Inst{25} = 0;
+    let Inst{23} = offset{12};
+    let Inst{19-16} = addr;
+    let Inst{11-0} = offset{11-0};
 
-def STRH_POST: AI3stridx<0b1011, 0, 0, (outs GPR:$Rn_wb),
-                     (ins GPR:$Rt, GPR:$Rn, am3offset:$offset),
-                     IndexModePost, StMiscFrm, IIC_iStore_bh_ru,
-                     "strh", "\t$Rt, [$Rn], $offset",
-                     "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
-                     [(set GPR:$Rn_wb, (post_truncsti16 GPR:$Rt,
-                                        GPR:$Rn, am3offset:$offset))]>;
+    let DecoderMethod = "DecodeAddrMode2IdxInstruction";
+  }
+}
+
+let mayStore = 1, neverHasSideEffects = 1 in {
+defm STR  : AI2_stridx<0, "str", IIC_iStore_ru>;
+defm STRB : AI2_stridx<1, "strb", IIC_iStore_bh_ru>;
+}
+
+def : ARMPat<(post_store GPR:$Rt, addr_offset_none:$addr,
+                         am2offset_reg:$offset),
+             (STR_POST_REG GPR:$Rt, addr_offset_none:$addr,
+                           am2offset_reg:$offset)>;
+def : ARMPat<(post_store GPR:$Rt, addr_offset_none:$addr,
+                         am2offset_imm:$offset),
+             (STR_POST_IMM GPR:$Rt, addr_offset_none:$addr,
+                           am2offset_imm:$offset)>;
+def : ARMPat<(post_truncsti8 GPR:$Rt, addr_offset_none:$addr,
+                             am2offset_reg:$offset),
+             (STRB_POST_REG GPR:$Rt, addr_offset_none:$addr,
+                            am2offset_reg:$offset)>;
+def : ARMPat<(post_truncsti8 GPR:$Rt, addr_offset_none:$addr,
+                             am2offset_imm:$offset),
+             (STRB_POST_IMM GPR:$Rt, addr_offset_none:$addr,
+                            am2offset_imm:$offset)>;
+
+// Pseudo-instructions for pattern matching the pre-indexed stores. We can't
+// put the patterns on the instruction definitions directly as ISel wants
+// the address base and offset to be separate operands, not a single
+// complex operand like we represent the instructions themselves. The
+// pseudos map between the two.
+let usesCustomInserter = 1,
+    Constraints = "$Rn = $Rn_wb,@earlyclobber $Rn_wb" in {
+def STRi_preidx: ARMPseudoInst<(outs GPR:$Rn_wb),
+               (ins GPR:$Rt, GPR:$Rn, am2offset_imm:$offset, pred:$p),
+               4, IIC_iStore_ru,
+            [(set GPR:$Rn_wb,
+                  (pre_store GPR:$Rt, GPR:$Rn, am2offset_imm:$offset))]>;
+def STRr_preidx: ARMPseudoInst<(outs GPR:$Rn_wb),
+               (ins GPR:$Rt, GPR:$Rn, am2offset_reg:$offset, pred:$p),
+               4, IIC_iStore_ru,
+            [(set GPR:$Rn_wb,
+                  (pre_store GPR:$Rt, GPR:$Rn, am2offset_reg:$offset))]>;
+def STRBi_preidx: ARMPseudoInst<(outs GPR:$Rn_wb),
+               (ins GPR:$Rt, GPR:$Rn, am2offset_imm:$offset, pred:$p),
+               4, IIC_iStore_ru,
+            [(set GPR:$Rn_wb,
+                  (pre_truncsti8 GPR:$Rt, GPR:$Rn, am2offset_imm:$offset))]>;
+def STRBr_preidx: ARMPseudoInst<(outs GPR:$Rn_wb),
+               (ins GPR:$Rt, GPR:$Rn, am2offset_reg:$offset, pred:$p),
+               4, IIC_iStore_ru,
+            [(set GPR:$Rn_wb,
+                  (pre_truncsti8 GPR:$Rt, GPR:$Rn, am2offset_reg:$offset))]>;
+def STRH_preidx: ARMPseudoInst<(outs GPR:$Rn_wb),
+               (ins GPR:$Rt, GPR:$Rn, am3offset:$offset, pred:$p),
+               4, IIC_iStore_ru,
+            [(set GPR:$Rn_wb,
+                  (pre_truncsti16 GPR:$Rt, GPR:$Rn, am3offset:$offset))]>;
+}
+
+
+
+def STRH_PRE  : AI3ldstidx<0b1011, 0, 1, (outs GPR:$Rn_wb),
+                           (ins GPR:$Rt, addrmode3:$addr), IndexModePre,
+                           StMiscFrm, IIC_iStore_bh_ru,
+                           "strh", "\t$Rt, $addr!", "$addr.base = $Rn_wb", []> {
+  bits<14> addr;
+  let Inst{23}    = addr{8};      // U bit
+  let Inst{22}    = addr{13};     // 1 == imm8, 0 == Rm
+  let Inst{19-16} = addr{12-9};   // Rn
+  let Inst{11-8}  = addr{7-4};    // imm7_4/zero
+  let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
+  let AsmMatchConverter = "cvtStWriteBackRegAddrMode3";
+  let DecoderMethod = "DecodeAddrMode3Instruction";
+}
+
+def STRH_POST : AI3ldstidx<0b1011, 0, 0, (outs GPR:$Rn_wb),
+                       (ins GPR:$Rt, addr_offset_none:$addr, am3offset:$offset),
+                       IndexModePost, StMiscFrm, IIC_iStore_bh_ru,
+                       "strh", "\t$Rt, $addr, $offset", "$addr.base = $Rn_wb",
+                   [(set GPR:$Rn_wb, (post_truncsti16 GPR:$Rt,
+                                                      addr_offset_none:$addr,
+                                                      am3offset:$offset))]> {
+  bits<10> offset;
+  bits<4> addr;
+  let Inst{23}    = offset{8};      // U bit
+  let Inst{22}    = offset{9};      // 1 == imm8, 0 == Rm
+  let Inst{19-16} = addr;
+  let Inst{11-8}  = offset{7-4};    // imm7_4/zero
+  let Inst{3-0}   = offset{3-0};    // imm3_0/Rm
+  let DecoderMethod = "DecodeAddrMode3Instruction";
+}
 
-// For disassembly only
 let mayStore = 1, neverHasSideEffects = 1, hasExtraSrcRegAllocReq = 1 in {
-def STRD_PRE : AI3stdpr<(outs GPR:$base_wb),
-                     (ins GPR:$src1, GPR:$src2, GPR:$base, am3offset:$offset),
-                     StMiscFrm, IIC_iStore_d_ru,
-                     "strd", "\t$src1, $src2, [$base, $offset]!",
-                     "$base = $base_wb", []>;
+def STRD_PRE : AI3ldstidx<0b1111, 0, 1, (outs GPR:$Rn_wb),
+                          (ins GPR:$Rt, GPR:$Rt2, addrmode3:$addr),
+                          IndexModePre, StMiscFrm, IIC_iStore_d_ru,
+                          "strd", "\t$Rt, $Rt2, $addr!",
+                          "$addr.base = $Rn_wb", []> {
+  bits<14> addr;
+  let Inst{23}    = addr{8};      // U bit
+  let Inst{22}    = addr{13};     // 1 == imm8, 0 == Rm
+  let Inst{19-16} = addr{12-9};   // Rn
+  let Inst{11-8}  = addr{7-4};    // imm7_4/zero
+  let Inst{3-0}   = addr{3-0};    // imm3_0/Rm
+  let DecoderMethod = "DecodeAddrMode3Instruction";
+  let AsmMatchConverter = "cvtStrdPre";
+}
 
-// For disassembly only
-def STRD_POST: AI3stdpo<(outs GPR:$base_wb),
-                     (ins GPR:$src1, GPR:$src2, GPR:$base, am3offset:$offset),
-                     StMiscFrm, IIC_iStore_d_ru,
-                     "strd", "\t$src1, $src2, [$base], $offset",
-                     "$base = $base_wb", []>;
+def STRD_POST: AI3ldstidx<0b1111, 0, 0, (outs GPR:$Rn_wb),
+                          (ins GPR:$Rt, GPR:$Rt2, addr_offset_none:$addr,
+                               am3offset:$offset),
+                          IndexModePost, StMiscFrm, IIC_iStore_d_ru,
+                          "strd", "\t$Rt, $Rt2, $addr, $offset",
+                          "$addr.base = $Rn_wb", []> {
+  bits<10> offset;
+  bits<4> addr;
+  let Inst{23}    = offset{8};      // U bit
+  let Inst{22}    = offset{9};      // 1 == imm8, 0 == Rm
+  let Inst{19-16} = addr;
+  let Inst{11-8}  = offset{7-4};    // imm7_4/zero
+  let Inst{3-0}   = offset{3-0};    // imm3_0/Rm
+  let DecoderMethod = "DecodeAddrMode3Instruction";
+}
 } // mayStore = 1, neverHasSideEffects = 1, hasExtraSrcRegAllocReq = 1
 
-// STRT, STRBT, and STRHT are for disassembly only.
+// STRT, STRBT, and STRHT
 
-def STRT : AI2stridxT<0, 0, (outs GPR:$Rn_wb), (ins GPR:$Rt, addrmode2:$addr),
-                     IndexModePost, StFrm, IIC_iStore_ru,
-                     "strt", "\t$Rt, $addr", "$addr.base = $Rn_wb",
-                     [/* For disassembly only; pattern left blank */]> {
+def STRBT_POST_REG : AI2ldstidx<0, 1, 0, (outs GPR:$Rn_wb),
+                   (ins GPR:$Rt, addr_offset_none:$addr, am2offset_reg:$offset),
+                   IndexModePost, StFrm, IIC_iStore_bh_ru,
+                   "strbt", "\t$Rt, $addr, $offset",
+                   "$addr.base = $Rn_wb", []> {
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 1;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
-  let AsmMatchConverter = "CvtStWriteBackRegAddrMode2";
+  let Inst{19-16} = addr;
+  let Inst{11-5} = offset{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = offset{3-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
 
-def STRBT : AI2stridxT<1, 0, (outs GPR:$Rn_wb), (ins GPR:$Rt, addrmode2:$addr),
-                      IndexModePost, StFrm, IIC_iStore_bh_ru,
-                      "strbt", "\t$Rt, $addr", "$addr.base = $Rn_wb",
-                      [/* For disassembly only; pattern left blank */]> {
+def STRBT_POST_IMM : AI2ldstidx<0, 1, 0, (outs GPR:$Rn_wb),
+                   (ins GPR:$Rt, addr_offset_none:$addr, am2offset_imm:$offset),
+                   IndexModePost, StFrm, IIC_iStore_bh_ru,
+                   "strbt", "\t$Rt, $addr, $offset",
+                   "$addr.base = $Rn_wb", []> {
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 0;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
-  let AsmMatchConverter = "CvtStWriteBackRegAddrMode2";
+  let Inst{19-16} = addr;
+  let Inst{11-0} = offset{11-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
 
-def STRHT: AI3sthpo<(outs GPR:$base_wb), (ins GPR:$Rt, addrmode3:$addr),
-                    StMiscFrm, IIC_iStore_bh_ru,
-                    "strht", "\t$Rt, $addr", "$addr.base = $base_wb",
-                    [/* For disassembly only; pattern left blank */]> {
+let mayStore = 1, neverHasSideEffects = 1 in {
+def STRT_POST_REG : AI2ldstidx<0, 0, 0, (outs GPR:$Rn_wb),
+                   (ins GPR:$Rt, addr_offset_none:$addr, am2offset_reg:$offset),
+                   IndexModePost, StFrm, IIC_iStore_ru,
+                   "strt", "\t$Rt, $addr, $offset",
+                   "$addr.base = $Rn_wb", []> {
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 1;
+  let Inst{23} = offset{12};
   let Inst{21} = 1; // overwrite
-  let AsmMatchConverter = "CvtStWriteBackRegAddrMode3";
+  let Inst{19-16} = addr;
+  let Inst{11-5} = offset{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = offset{3-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
 }
 
+def STRT_POST_IMM : AI2ldstidx<0, 0, 0, (outs GPR:$Rn_wb),
+                   (ins GPR:$Rt, addr_offset_none:$addr, am2offset_imm:$offset),
+                   IndexModePost, StFrm, IIC_iStore_ru,
+                   "strt", "\t$Rt, $addr, $offset",
+                   "$addr.base = $Rn_wb", []> {
+  // {12}     isAdd
+  // {11-0}   imm12/Rm
+  bits<14> offset;
+  bits<4> addr;
+  let Inst{25} = 0;
+  let Inst{23} = offset{12};
+  let Inst{21} = 1; // overwrite
+  let Inst{19-16} = addr;
+  let Inst{11-0} = offset{11-0};
+  let DecoderMethod = "DecodeAddrMode2IdxInstruction";
+}
+}
+
+
+multiclass AI3strT<bits<4> op, string opc> {
+  def i : AI3ldstidxT<op, 0, (outs GPR:$base_wb),
+                    (ins GPR:$Rt, addr_offset_none:$addr, postidx_imm8:$offset),
+                    IndexModePost, StMiscFrm, IIC_iStore_bh_ru, opc,
+                    "\t$Rt, $addr, $offset", "$addr.base = $base_wb", []> {
+    bits<9> offset;
+    let Inst{23} = offset{8};
+    let Inst{22} = 1;
+    let Inst{11-8} = offset{7-4};
+    let Inst{3-0} = offset{3-0};
+    let AsmMatchConverter = "cvtStExtTWriteBackImm";
+  }
+  def r : AI3ldstidxT<op, 0, (outs GPR:$base_wb),
+                      (ins GPR:$Rt, addr_offset_none:$addr, postidx_reg:$Rm),
+                      IndexModePost, StMiscFrm, IIC_iStore_bh_ru, opc,
+                      "\t$Rt, $addr, $Rm", "$addr.base = $base_wb", []> {
+    bits<5> Rm;
+    let Inst{23} = Rm{4};
+    let Inst{22} = 0;
+    let Inst{11-8} = 0;
+    let Inst{3-0} = Rm{3-0};
+    let AsmMatchConverter = "cvtStExtTWriteBackReg";
+  }
+}
+
+
+defm STRHT : AI3strT<0b1011, "strht">;
+
+
 //===----------------------------------------------------------------------===//
 //  Load / store multiple Instructions.
 //
@@ -1996,6 +2764,8 @@ multiclass arm_ldst_mult<string asm, bit L_bit, Format f,
     let Inst{24-23} = 0b01;       // Increment After
     let Inst{21}    = 1;          // Writeback
     let Inst{20}    = L_bit;
+
+    let DecoderMethod = "DecodeMemMultipleWritebackInstruction";
   }
   def DA :
     AXI4<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
@@ -2012,6 +2782,8 @@ multiclass arm_ldst_mult<string asm, bit L_bit, Format f,
     let Inst{24-23} = 0b00;       // Decrement After
     let Inst{21}    = 1;          // Writeback
     let Inst{20}    = L_bit;
+
+    let DecoderMethod = "DecodeMemMultipleWritebackInstruction";
   }
   def DB :
     AXI4<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
@@ -2028,6 +2800,8 @@ multiclass arm_ldst_mult<string asm, bit L_bit, Format f,
     let Inst{24-23} = 0b10;       // Decrement Before
     let Inst{21}    = 1;          // Writeback
     let Inst{20}    = L_bit;
+
+    let DecoderMethod = "DecodeMemMultipleWritebackInstruction";
   }
   def IB :
     AXI4<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
@@ -2044,6 +2818,8 @@ multiclass arm_ldst_mult<string asm, bit L_bit, Format f,
     let Inst{24-23} = 0b11;       // Increment Before
     let Inst{21}    = 1;          // Writeback
     let Inst{20}    = L_bit;
+
+    let DecoderMethod = "DecodeMemMultipleWritebackInstruction";
   }
 }
 
@@ -2084,6 +2860,9 @@ def MOVr : AsI1<0b1101, (outs GPR:$Rd), (ins GPR:$Rm), DPFrm, IIC_iMOVr,
   let Inst{15-12} = Rd;
 }
 
+def : ARMInstAlias<"movs${p} $Rd, $Rm", 
+                   (MOVr GPR:$Rd, GPR:$Rm, pred:$p, CPSR)>;
+
 // A version for the smaller set of tail call registers.
 let neverHasSideEffects = 1 in
 def MOVr_TC : AsI1<0b1101, (outs tcGPR:$Rd), (ins tcGPR:$Rm), DPFrm,
@@ -2097,15 +2876,33 @@ def MOVr_TC : AsI1<0b1101, (outs tcGPR:$Rd), (ins tcGPR:$Rm), DPFrm,
   let Inst{15-12} = Rd;
 }
 
-def MOVs : AsI1<0b1101, (outs GPR:$Rd), (ins shift_so_reg:$src),
-                DPSoRegFrm, IIC_iMOVsr,
-                "mov", "\t$Rd, $src", [(set GPR:$Rd, shift_so_reg:$src)]>,
+def MOVsr : AsI1<0b1101, (outs GPRnopc:$Rd), (ins shift_so_reg_reg:$src),
+                DPSoRegRegFrm, IIC_iMOVsr,
+                "mov", "\t$Rd, $src",
+                [(set GPRnopc:$Rd, shift_so_reg_reg:$src)]>, UnaryDP {
+  bits<4> Rd;
+  bits<12> src;
+  let Inst{15-12} = Rd;
+  let Inst{19-16} = 0b0000;
+  let Inst{11-8} = src{11-8};
+  let Inst{7} = 0;
+  let Inst{6-5} = src{6-5};
+  let Inst{4} = 1;
+  let Inst{3-0} = src{3-0};
+  let Inst{25} = 0;
+}
+
+def MOVsi : AsI1<0b1101, (outs GPR:$Rd), (ins shift_so_reg_imm:$src),
+                DPSoRegImmFrm, IIC_iMOVsr,
+                "mov", "\t$Rd, $src", [(set GPR:$Rd, shift_so_reg_imm:$src)]>,
                 UnaryDP {
   bits<4> Rd;
   bits<12> src;
   let Inst{15-12} = Rd;
   let Inst{19-16} = 0b0000;
-  let Inst{11-0} = src;
+  let Inst{11-5} = src{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = src{3-0};
   let Inst{25} = 0;
 }
 
@@ -2121,7 +2918,7 @@ def MOVi : AsI1<0b1101, (outs GPR:$Rd), (ins so_imm:$imm), DPFrm, IIC_iMOVi,
 }
 
 let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in
-def MOVi16 : AI1<0b1000, (outs GPR:$Rd), (ins i32imm_hilo16:$imm),
+def MOVi16 : AI1<0b1000, (outs GPR:$Rd), (ins imm0_65535_expr:$imm),
                  DPFrm, IIC_iMOVi,
                  "movw", "\t$Rd, $imm",
                  [(set GPR:$Rd, imm0_65535:$imm)]>,
@@ -2133,16 +2930,22 @@ def MOVi16 : AI1<0b1000, (outs GPR:$Rd), (ins i32imm_hilo16:$imm),
   let Inst{19-16} = imm{15-12};
   let Inst{20} = 0;
   let Inst{25} = 1;
+  let DecoderMethod = "DecodeArmMOVTWInstruction";
 }
 
+def : InstAlias<"mov${p} $Rd, $imm",
+                (MOVi16 GPR:$Rd, imm0_65535_expr:$imm, pred:$p)>,
+        Requires<[IsARM]>;
+
 def MOVi16_ga_pcrel : PseudoInst<(outs GPR:$Rd),
                                 (ins i32imm:$addr, pclabel:$id), IIC_iMOVi, []>;
 
 let Constraints = "$src = $Rd" in {
-def MOVTi16 : AI1<0b1010, (outs GPR:$Rd), (ins GPR:$src, i32imm_hilo16:$imm),
+def MOVTi16 : AI1<0b1010, (outs GPRnopc:$Rd),
+                  (ins GPR:$src, imm0_65535_expr:$imm),
                   DPFrm, IIC_iMOVi,
                   "movt", "\t$Rd, $imm",
-                  [(set GPR:$Rd,
+                  [(set GPRnopc:$Rd,
                         (or (and GPR:$src, 0xffff),
                             lo16AllZero:$imm))]>, UnaryDP,
                   Requires<[IsARM, HasV6T2]> {
@@ -2153,6 +2956,7 @@ def MOVTi16 : AI1<0b1010, (outs GPR:$Rd), (ins GPR:$src, i32imm_hilo16:$imm),
   let Inst{19-16} = imm{15-12};
   let Inst{20} = 0;
   let Inst{25} = 1;
+  let DecoderMethod = "DecodeArmMOVTWInstruction";
 }
 
 def MOVTi16_ga_pcrel : PseudoInst<(outs GPR:$Rd),
@@ -2186,30 +2990,28 @@ def MOVsra_flag : PseudoInst<(outs GPR:$dst), (ins GPR:$src), IIC_iMOVsi,
 
 // Sign extenders
 
-defm SXTB  : AI_ext_rrot<0b01101010,
+def SXTB  : AI_ext_rrot<0b01101010,
                          "sxtb", UnOpFrag<(sext_inreg node:$Src, i8)>>;
-defm SXTH  : AI_ext_rrot<0b01101011,
+def SXTH  : AI_ext_rrot<0b01101011,
                          "sxth", UnOpFrag<(sext_inreg node:$Src, i16)>>;
 
-defm SXTAB : AI_exta_rrot<0b01101010,
+def SXTAB : AI_exta_rrot<0b01101010,
                "sxtab", BinOpFrag<(add node:$LHS, (sext_inreg node:$RHS, i8))>>;
-defm SXTAH : AI_exta_rrot<0b01101011,
+def SXTAH : AI_exta_rrot<0b01101011,
                "sxtah", BinOpFrag<(add node:$LHS, (sext_inreg node:$RHS,i16))>>;
 
-// For disassembly only
-defm SXTB16  : AI_ext_rrot_np<0b01101000, "sxtb16">;
+def SXTB16  : AI_ext_rrot_np<0b01101000, "sxtb16">;
 
-// For disassembly only
-defm SXTAB16 : AI_exta_rrot_np<0b01101000, "sxtab16">;
+def SXTAB16 : AI_exta_rrot_np<0b01101000, "sxtab16">;
 
 // Zero extenders
 
 let AddedComplexity = 16 in {
-defm UXTB   : AI_ext_rrot<0b01101110,
+def UXTB   : AI_ext_rrot<0b01101110,
                           "uxtb"  , UnOpFrag<(and node:$Src, 0x000000FF)>>;
-defm UXTH   : AI_ext_rrot<0b01101111,
+def UXTH   : AI_ext_rrot<0b01101111,
                           "uxth"  , UnOpFrag<(and node:$Src, 0x0000FFFF)>>;
-defm UXTB16 : AI_ext_rrot<0b01101100,
+def UXTB16 : AI_ext_rrot<0b01101100,
                           "uxtb16", UnOpFrag<(and node:$Src, 0x00FF00FF)>>;
 
 // FIXME: This pattern incorrectly assumes the shl operator is a rotate.
@@ -2217,23 +3019,22 @@ defm UXTB16 : AI_ext_rrot<0b01101100,
 //        instead so we can include a check for masking back in the upper
 //        eight bits of the source into the lower eight bits of the result.
 //def : ARMV6Pat<(and (shl GPR:$Src, (i32 8)), 0xFF00FF),
-//               (UXTB16r_rot GPR:$Src, 24)>;
+//               (UXTB16r_rot GPR:$Src, 3)>;
 def : ARMV6Pat<(and (srl GPR:$Src, (i32 8)), 0xFF00FF),
-               (UXTB16r_rot GPR:$Src, 8)>;
+               (UXTB16 GPR:$Src, 1)>;
 
-defm UXTAB : AI_exta_rrot<0b01101110, "uxtab",
+def UXTAB : AI_exta_rrot<0b01101110, "uxtab",
                         BinOpFrag<(add node:$LHS, (and node:$RHS, 0x00FF))>>;
-defm UXTAH : AI_exta_rrot<0b01101111, "uxtah",
+def UXTAH : AI_exta_rrot<0b01101111, "uxtah",
                         BinOpFrag<(add node:$LHS, (and node:$RHS, 0xFFFF))>>;
 }
 
 // This isn't safe in general, the add is two 16-bit units, not a 32-bit add.
-// For disassembly only
-defm UXTAB16 : AI_exta_rrot_np<0b01101100, "uxtab16">;
+def UXTAB16 : AI_exta_rrot_np<0b01101100, "uxtab16">;
 
 
-def SBFX  : I<(outs GPR:$Rd),
-              (ins GPR:$Rn, imm0_31:$lsb, imm0_31_m1:$width),
+def SBFX  : I<(outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, imm0_31:$lsb, imm1_32:$width),
                AddrMode1, 4, IndexModeNone, DPFrm, IIC_iUNAsi,
                "sbfx", "\t$Rd, $Rn, $lsb, $width", "", []>,
                Requires<[IsARM, HasV6T2]> {
@@ -2250,7 +3051,7 @@ def SBFX  : I<(outs GPR:$Rd),
 }
 
 def UBFX  : I<(outs GPR:$Rd),
-              (ins GPR:$Rn, imm0_31:$lsb, imm0_31_m1:$width),
+              (ins GPR:$Rn, imm0_31:$lsb, imm1_32:$width),
                AddrMode1, 4, IndexModeNone, DPFrm, IIC_iUNAsi,
                "ubfx", "\t$Rd, $Rn, $lsb, $width", "", []>,
                Requires<[IsARM, HasV6T2]> {
@@ -2278,148 +3079,58 @@ defm SUB  : AsI1_bin_irs<0b0010, "sub",
                          BinOpFrag<(sub  node:$LHS, node:$RHS)>, "SUB">;
 
 // ADD and SUB with 's' bit set.
-defm ADDS : AI1_bin_s_irs<0b0100, "adds",
+//
+// Currently, t2ADDS/t2SUBS are pseudo opcodes that exist only in the
+// selection DAG. They are "lowered" to real t2ADD/t2SUB opcodes by
+// AdjustInstrPostInstrSelection where we determine whether or not to
+// set the "s" bit based on CPSR liveness.
+//
+// FIXME: Eliminate t2ADDS/t2SUBS pseudo opcodes after adding tablegen
+// support for an optional CPSR definition that corresponds to the DAG
+// node's second value. We can then eliminate the implicit def of CPSR.
+defm ADDS : AsI1_bin_s_irs<0b0100, "add",
                           IIC_iALUi, IIC_iALUr, IIC_iALUsr,
-                          BinOpFrag<(addc node:$LHS, node:$RHS)>, 1>;
-defm SUBS : AI1_bin_s_irs<0b0010, "subs",
+                          BinOpFrag<(ARMaddc node:$LHS, node:$RHS)>, 1>;
+defm SUBS : AsI1_bin_s_irs<0b0010, "sub",
                           IIC_iALUi, IIC_iALUr, IIC_iALUsr,
-                          BinOpFrag<(subc node:$LHS, node:$RHS)>>;
+                          BinOpFrag<(ARMsubc node:$LHS, node:$RHS)>>;
 
 defm ADC : AI1_adde_sube_irs<0b0101, "adc",
-                          BinOpFrag<(adde_dead_carry node:$LHS, node:$RHS)>,
+                  BinOpWithFlagFrag<(ARMadde node:$LHS, node:$RHS, node:$FLAG)>,
                           "ADC", 1>;
 defm SBC : AI1_adde_sube_irs<0b0110, "sbc",
-                          BinOpFrag<(sube_dead_carry node:$LHS, node:$RHS)>,
+                  BinOpWithFlagFrag<(ARMsube node:$LHS, node:$RHS, node:$FLAG)>,
                           "SBC">;
 
-// ADC and SUBC with 's' bit set.
-let usesCustomInserter = 1 in {
-defm ADCS : AI1_adde_sube_s_irs<
-              BinOpFrag<(adde_live_carry node:$LHS, node:$RHS)>, 1>;
-defm SBCS : AI1_adde_sube_s_irs<
-              BinOpFrag<(sube_live_carry node:$LHS, node:$RHS) >>;
-}
+defm RSB  : AsI1_rbin_irs <0b0011, "rsb",
+                         IIC_iALUi, IIC_iALUr, IIC_iALUsr,
+                         BinOpFrag<(sub node:$LHS, node:$RHS)>, "RSB">;
 
-def RSBri : AsI1<0b0011, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm), DPFrm,
-                 IIC_iALUi, "rsb", "\t$Rd, $Rn, $imm",
-                 [(set GPR:$Rd, (sub so_imm:$imm, GPR:$Rn))]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<12> imm;
-  let Inst{25} = 1;
-  let Inst{15-12} = Rd;
-  let Inst{19-16} = Rn;
-  let Inst{11-0} = imm;
-}
+// FIXME: Eliminate them if we can write def : Pat patterns which defines
+// CPSR and the implicit def of CPSR is not needed.
+defm RSBS : AsI1_rbin_s_is<0b0011, "rsb",
+                         IIC_iALUi, IIC_iALUr, IIC_iALUsr,
+                         BinOpFrag<(ARMsubc node:$LHS, node:$RHS)>>;
 
-// The reg/reg form is only defined for the disassembler; for codegen it is
-// equivalent to SUBrr.
-def RSBrr : AsI1<0b0011, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), DPFrm,
-                 IIC_iALUr, "rsb", "\t$Rd, $Rn, $Rm",
-                 [/* For disassembly only; pattern left blank */]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<4> Rm;
-  let Inst{11-4} = 0b00000000;
-  let Inst{25} = 0;
-  let Inst{3-0} = Rm;
-  let Inst{15-12} = Rd;
-  let Inst{19-16} = Rn;
-}
-
-def RSBrs : AsI1<0b0011, (outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift),
-                 DPSoRegFrm, IIC_iALUsr, "rsb", "\t$Rd, $Rn, $shift",
-                 [(set GPR:$Rd, (sub so_reg:$shift, GPR:$Rn))]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<12> shift;
-  let Inst{25} = 0;
-  let Inst{11-0} = shift;
-  let Inst{15-12} = Rd;
-  let Inst{19-16} = Rn;
-}
-
-// RSB with 's' bit set.
-// NOTE: CPSR def omitted because it will be handled by the custom inserter.
-let usesCustomInserter = 1 in {
-def RSBSri : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm),
-                 4, IIC_iALUi,
-                 [(set GPR:$Rd, (subc so_imm:$imm, GPR:$Rn))]>;
-def RSBSrr : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-                 4, IIC_iALUr,
-                 [/* For disassembly only; pattern left blank */]>;
-def RSBSrs : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift),
-                 4, IIC_iALUsr,
-                 [(set GPR:$Rd, (subc so_reg:$shift, GPR:$Rn))]>;
-}
-
-let Uses = [CPSR] in {
-def RSCri : AsI1<0b0111, (outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm),
-                 DPFrm, IIC_iALUi, "rsc", "\t$Rd, $Rn, $imm",
-                 [(set GPR:$Rd, (sube_dead_carry so_imm:$imm, GPR:$Rn))]>,
-                 Requires<[IsARM]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<12> imm;
-  let Inst{25} = 1;
-  let Inst{15-12} = Rd;
-  let Inst{19-16} = Rn;
-  let Inst{11-0} = imm;
-}
-// The reg/reg form is only defined for the disassembler; for codegen it is
-// equivalent to SUBrr.
-def RSCrr : AsI1<0b0111, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-                 DPFrm, IIC_iALUr, "rsc", "\t$Rd, $Rn, $Rm",
-                 [/* For disassembly only; pattern left blank */]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<4> Rm;
-  let Inst{11-4} = 0b00000000;
-  let Inst{25} = 0;
-  let Inst{3-0} = Rm;
-  let Inst{15-12} = Rd;
-  let Inst{19-16} = Rn;
-}
-def RSCrs : AsI1<0b0111, (outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift),
-                 DPSoRegFrm, IIC_iALUsr, "rsc", "\t$Rd, $Rn, $shift",
-                 [(set GPR:$Rd, (sube_dead_carry so_reg:$shift, GPR:$Rn))]>,
-                 Requires<[IsARM]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<12> shift;
-  let Inst{25} = 0;
-  let Inst{11-0} = shift;
-  let Inst{15-12} = Rd;
-  let Inst{19-16} = Rn;
-}
-}
-
-// NOTE: CPSR def omitted because it will be handled by the custom inserter.
-let usesCustomInserter = 1, Uses = [CPSR] in {
-def RSCSri : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, so_imm:$imm),
-                  4, IIC_iALUi,
-                  [(set GPR:$Rd, (sube_dead_carry so_imm:$imm, GPR:$Rn))]>;
-def RSCSrs : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$Rn, so_reg:$shift),
-                  4, IIC_iALUsr,
-                  [(set GPR:$Rd, (sube_dead_carry so_reg:$shift, GPR:$Rn))]>;
-}
+defm RSC : AI1_rsc_irs<0b0111, "rsc",
+                  BinOpWithFlagFrag<(ARMsube node:$LHS, node:$RHS, node:$FLAG)>,
+                       "RSC">;
 
 // (sub X, imm) gets canonicalized to (add X, -imm).  Match this form.
 // The assume-no-carry-in form uses the negation of the input since add/sub
 // assume opposite meanings of the carry flag (i.e., carry == !borrow).
 // See the definition of AddWithCarry() in the ARM ARM A2.2.1 for the gory
 // details.
-def : ARMPat<(add    GPR:$src, so_imm_neg:$imm),
-             (SUBri  GPR:$src, so_imm_neg:$imm)>;
-def : ARMPat<(addc   GPR:$src, so_imm_neg:$imm),
-             (SUBSri GPR:$src, so_imm_neg:$imm)>;
+def : ARMPat<(add     GPR:$src, so_imm_neg:$imm),
+             (SUBri   GPR:$src, so_imm_neg:$imm)>;
+def : ARMPat<(ARMaddc GPR:$src, so_imm_neg:$imm),
+             (SUBSri  GPR:$src, so_imm_neg:$imm)>;
+
 // The with-carry-in form matches bitwise not instead of the negation.
 // Effectively, the inverse interpretation of the carry flag already accounts
 // for part of the negation.
-def : ARMPat<(adde_dead_carry   GPR:$src, so_imm_not:$imm),
-             (SBCri  GPR:$src, so_imm_not:$imm)>;
-def : ARMPat<(adde_live_carry   GPR:$src, so_imm_not:$imm),
-             (SBCSri GPR:$src, so_imm_not:$imm)>;
+def : ARMPat<(ARMadde GPR:$src, so_imm_not:$imm, CPSR),
+             (SBCri   GPR:$src, so_imm_not:$imm)>;
 
 // Note: These are implemented in C++ code, because they have to generate
 // ADD/SUBrs instructions, which use a complex pattern that a xform function
@@ -2427,12 +3138,13 @@ def : ARMPat<(adde_live_carry   GPR:$src, so_imm_not:$imm),
 // (mul X, 2^n+1) -> (add (X << n), X)
 // (mul X, 2^n-1) -> (rsb X, (X << n))
 
-// ARM Arithmetic Instruction -- for disassembly only
+// ARM Arithmetic Instruction
 // GPR:$dst = GPR:$a op GPR:$b
 class AAI<bits<8> op27_20, bits<8> op11_4, string opc,
-          list<dag> pattern = [/* For disassembly only; pattern left blank */],
-          dag iops = (ins GPR:$Rn, GPR:$Rm), string asm = "\t$Rd, $Rn, $Rm">
-  : AI<(outs GPR:$Rd), iops, DPFrm, IIC_iALUr, opc, asm, pattern> {
+          list<dag> pattern = [],
+          dag iops = (ins GPRnopc:$Rn, GPRnopc:$Rm),
+          string asm = "\t$Rd, $Rn, $Rm">
+  : AI<(outs GPRnopc:$Rd), iops, DPFrm, IIC_iALUr, opc, asm, pattern> {
   bits<4> Rn;
   bits<4> Rd;
   bits<4> Rm;
@@ -2443,17 +3155,19 @@ class AAI<bits<8> op27_20, bits<8> op11_4, string opc,
   let Inst{3-0}   = Rm;
 }
 
-// Saturating add/subtract -- for disassembly only
+// Saturating add/subtract
 
 def QADD    : AAI<0b00010000, 0b00000101, "qadd",
-                  [(set GPR:$Rd, (int_arm_qadd GPR:$Rm, GPR:$Rn))],
-                  (ins GPR:$Rm, GPR:$Rn), "\t$Rd, $Rm, $Rn">;
+                  [(set GPRnopc:$Rd, (int_arm_qadd GPRnopc:$Rm, GPRnopc:$Rn))],
+                  (ins GPRnopc:$Rm, GPRnopc:$Rn), "\t$Rd, $Rm, $Rn">;
 def QSUB    : AAI<0b00010010, 0b00000101, "qsub",
-                  [(set GPR:$Rd, (int_arm_qsub GPR:$Rm, GPR:$Rn))],
-                  (ins GPR:$Rm, GPR:$Rn), "\t$Rd, $Rm, $Rn">;
-def QDADD   : AAI<0b00010100, 0b00000101, "qdadd", [], (ins GPR:$Rm, GPR:$Rn),
+                  [(set GPRnopc:$Rd, (int_arm_qsub GPRnopc:$Rm, GPRnopc:$Rn))],
+                  (ins GPRnopc:$Rm, GPRnopc:$Rn), "\t$Rd, $Rm, $Rn">;
+def QDADD   : AAI<0b00010100, 0b00000101, "qdadd", [],
+                  (ins GPRnopc:$Rm, GPRnopc:$Rn),
                   "\t$Rd, $Rm, $Rn">;
-def QDSUB   : AAI<0b00010110, 0b00000101, "qdsub", [], (ins GPR:$Rm, GPR:$Rn),
+def QDSUB   : AAI<0b00010110, 0b00000101, "qdsub", [],
+                  (ins GPRnopc:$Rm, GPRnopc:$Rn),
                   "\t$Rd, $Rm, $Rn">;
 
 def QADD16  : AAI<0b01100010, 0b11110001, "qadd16">;
@@ -2469,7 +3183,7 @@ def UQSAX   : AAI<0b01100110, 0b11110101, "uqsax">;
 def UQSUB16 : AAI<0b01100110, 0b11110111, "uqsub16">;
 def UQSUB8  : AAI<0b01100110, 0b11111111, "uqsub8">;
 
-// Signed/Unsigned add/subtract -- for disassembly only
+// Signed/Unsigned add/subtract
 
 def SASX   : AAI<0b01100001, 0b11110011, "sasx">;
 def SADD16 : AAI<0b01100001, 0b11110001, "sadd16">;
@@ -2484,7 +3198,7 @@ def USAX   : AAI<0b01100101, 0b11110101, "usax">;
 def USUB16 : AAI<0b01100101, 0b11110111, "usub16">;
 def USUB8  : AAI<0b01100101, 0b11111111, "usub8">;
 
-// Signed/Unsigned halving add/subtract -- for disassembly only
+// Signed/Unsigned halving add/subtract
 
 def SHASX   : AAI<0b01100011, 0b11110011, "shasx">;
 def SHADD16 : AAI<0b01100011, 0b11110001, "shadd16">;
@@ -2499,7 +3213,7 @@ def UHSAX   : AAI<0b01100111, 0b11110101, "uhsax">;
 def UHSUB16 : AAI<0b01100111, 0b11110111, "uhsub16">;
 def UHSUB8  : AAI<0b01100111, 0b11111111, "uhsub8">;
 
-// Unsigned Sum of Absolute Differences [and Accumulate] -- for disassembly only
+// Unsigned Sum of Absolute Differences [and Accumulate].
 
 def USAD8  : AI<(outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
                 MulFrm /* for convenience */, NoItinerary, "usad8",
@@ -2531,11 +3245,11 @@ def USADA8 : AI<(outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
   let Inst{3-0} = Rn;
 }
 
-// Signed/Unsigned saturate -- for disassembly only
+// Signed/Unsigned saturate
 
-def SSAT : AI<(outs GPR:$Rd), (ins ssat_imm:$sat_imm, GPR:$a, shift_imm:$sh),
-              SatFrm, NoItinerary, "ssat", "\t$Rd, $sat_imm, $a$sh",
-              [/* For disassembly only; pattern left blank */]> {
+def SSAT : AI<(outs GPRnopc:$Rd),
+              (ins imm1_32:$sat_imm, GPRnopc:$Rn, shift_imm:$sh),
+              SatFrm, NoItinerary, "ssat", "\t$Rd, $sat_imm, $Rn$sh", []> {
   bits<4> Rd;
   bits<5> sat_imm;
   bits<4> Rn;
@@ -2544,14 +3258,14 @@ def SSAT : AI<(outs GPR:$Rd), (ins ssat_imm:$sat_imm, GPR:$a, shift_imm:$sh),
   let Inst{5-4} = 0b01;
   let Inst{20-16} = sat_imm;
   let Inst{15-12} = Rd;
-  let Inst{11-7} = sh{7-3};
-  let Inst{6} = sh{0};
+  let Inst{11-7} = sh{4-0};
+  let Inst{6} = sh{5};
   let Inst{3-0} = Rn;
 }
 
-def SSAT16 : AI<(outs GPR:$Rd), (ins ssat_imm:$sat_imm, GPR:$Rn), SatFrm,
-                NoItinerary, "ssat16", "\t$Rd, $sat_imm, $Rn",
-                [/* For disassembly only; pattern left blank */]> {
+def SSAT16 : AI<(outs GPRnopc:$Rd),
+                (ins imm1_16:$sat_imm, GPRnopc:$Rn), SatFrm,
+                NoItinerary, "ssat16", "\t$Rd, $sat_imm, $Rn", []> {
   bits<4> Rd;
   bits<4> sat_imm;
   bits<4> Rn;
@@ -2562,9 +3276,9 @@ def SSAT16 : AI<(outs GPR:$Rd), (ins ssat_imm:$sat_imm, GPR:$Rn), SatFrm,
   let Inst{3-0} = Rn;
 }
 
-def USAT : AI<(outs GPR:$Rd), (ins i32imm:$sat_imm, GPR:$a, shift_imm:$sh),
-              SatFrm, NoItinerary, "usat", "\t$Rd, $sat_imm, $a$sh",
-              [/* For disassembly only; pattern left blank */]> {
+def USAT : AI<(outs GPRnopc:$Rd),
+              (ins imm0_31:$sat_imm, GPRnopc:$Rn, shift_imm:$sh),
+              SatFrm, NoItinerary, "usat", "\t$Rd, $sat_imm, $Rn$sh", []> {
   bits<4> Rd;
   bits<5> sat_imm;
   bits<4> Rn;
@@ -2572,15 +3286,15 @@ def USAT : AI<(outs GPR:$Rd), (ins i32imm:$sat_imm, GPR:$a, shift_imm:$sh),
   let Inst{27-21} = 0b0110111;
   let Inst{5-4} = 0b01;
   let Inst{15-12} = Rd;
-  let Inst{11-7} = sh{7-3};
-  let Inst{6} = sh{0};
+  let Inst{11-7} = sh{4-0};
+  let Inst{6} = sh{5};
   let Inst{20-16} = sat_imm;
   let Inst{3-0} = Rn;
 }
 
-def USAT16 : AI<(outs GPR:$Rd), (ins i32imm:$sat_imm, GPR:$a), SatFrm,
-                NoItinerary, "usat16", "\t$Rd, $sat_imm, $a",
-                [/* For disassembly only; pattern left blank */]> {
+def USAT16 : AI<(outs GPRnopc:$Rd),
+                (ins imm0_15:$sat_imm, GPRnopc:$Rn), SatFrm,
+                NoItinerary, "usat16", "\t$Rd, $sat_imm, $Rn", []> {
   bits<4> Rd;
   bits<4> sat_imm;
   bits<4> Rn;
@@ -2591,8 +3305,10 @@ def USAT16 : AI<(outs GPR:$Rd), (ins i32imm:$sat_imm, GPR:$a), SatFrm,
   let Inst{3-0} = Rn;
 }
 
-def : ARMV6Pat<(int_arm_ssat GPR:$a, imm:$pos), (SSAT imm:$pos, GPR:$a, 0)>;
-def : ARMV6Pat<(int_arm_usat GPR:$a, imm:$pos), (USAT imm:$pos, GPR:$a, 0)>;
+def : ARMV6Pat<(int_arm_ssat GPRnopc:$a, imm:$pos),
+               (SSAT imm:$pos, GPRnopc:$a, 0)>;
+def : ARMV6Pat<(int_arm_usat GPRnopc:$a, imm:$pos),
+               (USAT imm:$pos, GPRnopc:$a, 0)>;
 
 //===----------------------------------------------------------------------===//
 //  Bitwise Instructions.
@@ -2611,6 +3327,10 @@ defm BIC   : AsI1_bin_irs<0b1110, "bic",
                           IIC_iBITi, IIC_iBITr, IIC_iBITsr,
                           BinOpFrag<(and node:$LHS, (not node:$RHS))>, "BIC">;
 
+// FIXME: bf_inv_mask_imm should be two operands, the lsb and the msb, just
+// like in the actual instruction encoding. The complexity of mapping the mask
+// to the lsb/msb pair should be handled by ISel, not encapsulated in the
+// instruction description.
 def BFC    : I<(outs GPR:$Rd), (ins GPR:$src, bf_inv_mask_imm:$imm),
                AddrMode1, 4, IndexModeNone, DPFrm, IIC_iUNAsi,
                "bfc", "\t$Rd, $imm", "$src = $Rd",
@@ -2622,16 +3342,16 @@ def BFC    : I<(outs GPR:$Rd), (ins GPR:$src, bf_inv_mask_imm:$imm),
   let Inst{6-0}   = 0b0011111;
   let Inst{15-12} = Rd;
   let Inst{11-7}  = imm{4-0}; // lsb
-  let Inst{20-16} = imm{9-5}; // width
+  let Inst{20-16} = imm{9-5}; // msb
 }
 
 // A8.6.18  BFI - Bitfield insert (Encoding A1)
-def BFI    : I<(outs GPR:$Rd), (ins GPR:$src, GPR:$Rn, bf_inv_mask_imm:$imm),
-               AddrMode1, 4, IndexModeNone, DPFrm, IIC_iUNAsi,
-               "bfi", "\t$Rd, $Rn, $imm", "$src = $Rd",
-               [(set GPR:$Rd, (ARMbfi GPR:$src, GPR:$Rn,
-                                bf_inv_mask_imm:$imm))]>,
-               Requires<[IsARM, HasV6T2]> {
+def BFI:I<(outs GPRnopc:$Rd), (ins GPRnopc:$src, GPR:$Rn, bf_inv_mask_imm:$imm),
+          AddrMode1, 4, IndexModeNone, DPFrm, IIC_iUNAsi,
+          "bfi", "\t$Rd, $Rn, $imm", "$src = $Rd",
+          [(set GPRnopc:$Rd, (ARMbfi GPRnopc:$src, GPR:$Rn,
+                           bf_inv_mask_imm:$imm))]>,
+          Requires<[IsARM, HasV6T2]> {
   bits<4> Rd;
   bits<4> Rn;
   bits<10> imm;
@@ -2643,25 +3363,6 @@ def BFI    : I<(outs GPR:$Rd), (ins GPR:$src, GPR:$Rn, bf_inv_mask_imm:$imm),
   let Inst{3-0}   = Rn;
 }
 
-// GNU as only supports this form of bfi (w/ 4 arguments)
-let isAsmParserOnly = 1 in
-def BFI4p : I<(outs GPR:$Rd), (ins GPR:$src, GPR:$Rn,
-                                   lsb_pos_imm:$lsb, width_imm:$width),
-               AddrMode1, 4, IndexModeNone, DPFrm, IIC_iUNAsi,
-               "bfi", "\t$Rd, $Rn, $lsb, $width", "$src = $Rd",
-               []>, Requires<[IsARM, HasV6T2]> {
-  bits<4> Rd;
-  bits<4> Rn;
-  bits<5> lsb;
-  bits<5> width;
-  let Inst{27-21} = 0b0111110;
-  let Inst{6-4}   = 0b001; // Rn: Inst{3-0} != 15
-  let Inst{15-12} = Rd;
-  let Inst{11-7}  = lsb;
-  let Inst{20-16} = width; // Custom encoder => lsb+width-1
-  let Inst{3-0}   = Rn;
-}
-
 def  MVNr  : AsI1<0b1111, (outs GPR:$Rd), (ins GPR:$Rm), DPFrm, IIC_iMVNr,
                   "mvn", "\t$Rd, $Rm",
                   [(set GPR:$Rd, (not GPR:$Rm))]>, UnaryDP {
@@ -2673,15 +3374,31 @@ def  MVNr  : AsI1<0b1111, (outs GPR:$Rd), (ins GPR:$Rm), DPFrm, IIC_iMVNr,
   let Inst{15-12} = Rd;
   let Inst{3-0} = Rm;
 }
-def  MVNs  : AsI1<0b1111, (outs GPR:$Rd), (ins so_reg:$shift), DPSoRegFrm,
-                  IIC_iMVNsr, "mvn", "\t$Rd, $shift",
-                  [(set GPR:$Rd, (not so_reg:$shift))]>, UnaryDP {
+def  MVNsi  : AsI1<0b1111, (outs GPR:$Rd), (ins so_reg_imm:$shift),
+                  DPSoRegImmFrm, IIC_iMVNsr, "mvn", "\t$Rd, $shift",
+                  [(set GPR:$Rd, (not so_reg_imm:$shift))]>, UnaryDP {
   bits<4> Rd;
   bits<12> shift;
   let Inst{25} = 0;
   let Inst{19-16} = 0b0000;
   let Inst{15-12} = Rd;
-  let Inst{11-0} = shift;
+  let Inst{11-5} = shift{11-5};
+  let Inst{4} = 0;
+  let Inst{3-0} = shift{3-0};
+}
+def  MVNsr  : AsI1<0b1111, (outs GPR:$Rd), (ins so_reg_reg:$shift),
+                  DPSoRegRegFrm, IIC_iMVNsr, "mvn", "\t$Rd, $shift",
+                  [(set GPR:$Rd, (not so_reg_reg:$shift))]>, UnaryDP {
+  bits<4> Rd;
+  bits<12> shift;
+  let Inst{25} = 0;
+  let Inst{19-16} = 0b0000;
+  let Inst{15-12} = Rd;
+  let Inst{11-8} = shift{11-8};
+  let Inst{7} = 0;
+  let Inst{6-5} = shift{6-5};
+  let Inst{4} = 1;
+  let Inst{3-0} = shift{3-0};
 }
 let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in
 def  MVNi  : AsI1<0b1111, (outs GPR:$Rd), (ins so_imm:$imm), DPFrm,
@@ -2820,8 +3537,8 @@ def UMAAL : AMul1I <0b0000010, (outs GPR:$RdLo, GPR:$RdHi),
   bits<4> RdHi;
   bits<4> Rm;
   bits<4> Rn;
-  let Inst{19-16} = RdLo;
-  let Inst{15-12} = RdHi;
+  let Inst{19-16} = RdHi;
+  let Inst{15-12} = RdLo;
   let Inst{11-8}  = Rm;
   let Inst{3-0}   = Rn;
 }
@@ -2855,8 +3572,7 @@ def SMMUL : AMul2I <0b0111010, 0b0001, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
 }
 
 def SMMULR : AMul2I <0b0111010, 0b0011, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-               IIC_iMUL32, "smmulr", "\t$Rd, $Rn, $Rm",
-               [/* For disassembly only; pattern left blank */]>,
+               IIC_iMUL32, "smmulr", "\t$Rd, $Rn, $Rm", []>,
             Requires<[IsARM, HasV6]> {
   let Inst{15-12} = 0b1111;
 }
@@ -2869,8 +3585,7 @@ def SMMLA : AMul2Ia <0b0111010, 0b0001, (outs GPR:$Rd),
 
 def SMMLAR : AMul2Ia <0b0111010, 0b0011, (outs GPR:$Rd),
                (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
-               IIC_iMAC32, "smmlar", "\t$Rd, $Rn, $Rm, $Ra",
-               [/* For disassembly only; pattern left blank */]>,
+               IIC_iMAC32, "smmlar", "\t$Rd, $Rn, $Rm, $Ra", []>,
             Requires<[IsARM, HasV6]>;
 
 def SMMLS : AMul2Ia <0b0111010, 0b1101, (outs GPR:$Rd),
@@ -2881,8 +3596,7 @@ def SMMLS : AMul2Ia <0b0111010, 0b1101, (outs GPR:$Rd),
 
 def SMMLSR : AMul2Ia <0b0111010, 0b1111, (outs GPR:$Rd),
                (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
-               IIC_iMAC32, "smmlsr", "\t$Rd, $Rn, $Rm, $Ra",
-               [/* For disassembly only; pattern left blank */]>,
+               IIC_iMAC32, "smmlsr", "\t$Rd, $Rn, $Rm, $Ra", []>,
             Requires<[IsARM, HasV6]>;
 
 multiclass AI_smul<string opc, PatFrag opnode> {
@@ -2925,92 +3639,95 @@ multiclass AI_smul<string opc, PatFrag opnode> {
 
 
 multiclass AI_smla<string opc, PatFrag opnode> {
-  def BB : AMulxyIa<0b0001000, 0b00, (outs GPR:$Rd),
-              (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  let DecoderMethod = "DecodeSMLAInstruction" in {
+  def BB : AMulxyIa<0b0001000, 0b00, (outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "bb"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPR:$Rd, (add GPR:$Ra,
-                               (opnode (sext_inreg GPR:$Rn, i16),
-                                       (sext_inreg GPR:$Rm, i16))))]>,
+              [(set GPRnopc:$Rd, (add GPR:$Ra,
+                               (opnode (sext_inreg GPRnopc:$Rn, i16),
+                                       (sext_inreg GPRnopc:$Rm, i16))))]>,
            Requires<[IsARM, HasV5TE]>;
 
-  def BT : AMulxyIa<0b0001000, 0b10, (outs GPR:$Rd),
-              (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def BT : AMulxyIa<0b0001000, 0b10, (outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "bt"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPR:$Rd, (add GPR:$Ra, (opnode (sext_inreg GPR:$Rn, i16),
-                                                   (sra GPR:$Rm, (i32 16)))))]>,
+              [(set GPRnopc:$Rd,
+                    (add GPR:$Ra, (opnode (sext_inreg GPRnopc:$Rn, i16),
+                                          (sra GPRnopc:$Rm, (i32 16)))))]>,
            Requires<[IsARM, HasV5TE]>;
 
-  def TB : AMulxyIa<0b0001000, 0b01, (outs GPR:$Rd),
-              (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def TB : AMulxyIa<0b0001000, 0b01, (outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "tb"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPR:$Rd, (add GPR:$Ra, (opnode (sra GPR:$Rn, (i32 16)),
-                                                (sext_inreg GPR:$Rm, i16))))]>,
+              [(set GPRnopc:$Rd,
+                    (add GPR:$Ra, (opnode (sra GPRnopc:$Rn, (i32 16)),
+                                          (sext_inreg GPRnopc:$Rm, i16))))]>,
            Requires<[IsARM, HasV5TE]>;
 
-  def TT : AMulxyIa<0b0001000, 0b11, (outs GPR:$Rd),
-              (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def TT : AMulxyIa<0b0001000, 0b11, (outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "tt"), "\t$Rd, $Rn, $Rm, $Ra",
-             [(set GPR:$Rd, (add GPR:$Ra, (opnode (sra GPR:$Rn, (i32 16)),
-                                                   (sra GPR:$Rm, (i32 16)))))]>,
+             [(set GPRnopc:$Rd,
+                   (add GPR:$Ra, (opnode (sra GPRnopc:$Rn, (i32 16)),
+                                         (sra GPRnopc:$Rm, (i32 16)))))]>,
             Requires<[IsARM, HasV5TE]>;
 
-  def WB : AMulxyIa<0b0001001, 0b00, (outs GPR:$Rd),
-              (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def WB : AMulxyIa<0b0001001, 0b00, (outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "wb"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPR:$Rd, (add GPR:$Ra, (sra (opnode GPR:$Rn,
-                                      (sext_inreg GPR:$Rm, i16)), (i32 16))))]>,
+              [(set GPRnopc:$Rd,
+                    (add GPR:$Ra, (sra (opnode GPRnopc:$Rn,
+                                  (sext_inreg GPRnopc:$Rm, i16)), (i32 16))))]>,
            Requires<[IsARM, HasV5TE]>;
 
-  def WT : AMulxyIa<0b0001001, 0b10, (outs GPR:$Rd),
-              (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def WT : AMulxyIa<0b0001001, 0b10, (outs GPRnopc:$Rd),
+              (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "wt"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPR:$Rd, (add GPR:$Ra, (sra (opnode GPR:$Rn,
-                                        (sra GPR:$Rm, (i32 16))), (i32 16))))]>,
+              [(set GPRnopc:$Rd,
+                 (add GPR:$Ra, (sra (opnode GPRnopc:$Rn,
+                                    (sra GPRnopc:$Rm, (i32 16))), (i32 16))))]>,
             Requires<[IsARM, HasV5TE]>;
+  }
 }
 
 defm SMUL : AI_smul<"smul", BinOpFrag<(mul node:$LHS, node:$RHS)>>;
 defm SMLA : AI_smla<"smla", BinOpFrag<(mul node:$LHS, node:$RHS)>>;
 
-// Halfword multiply accumulate long: SMLAL<x><y> -- for disassembly only
-def SMLALBB : AMulxyI64<0b0001010, 0b00, (outs GPR:$RdLo, GPR:$RdHi),
-                      (ins GPR:$Rn, GPR:$Rm),
-                      IIC_iMAC64, "smlalbb", "\t$RdLo, $RdHi, $Rn, $Rm",
-                      [/* For disassembly only; pattern left blank */]>,
+// Halfword multiply accumulate long: SMLAL<x><y>.
+def SMLALBB : AMulxyI64<0b0001010, 0b00, (outs GPRnopc:$RdLo, GPRnopc:$RdHi),
+                      (ins GPRnopc:$Rn, GPRnopc:$Rm),
+                      IIC_iMAC64, "smlalbb", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
               Requires<[IsARM, HasV5TE]>;
 
-def SMLALBT : AMulxyI64<0b0001010, 0b10, (outs GPR:$RdLo, GPR:$RdHi),
-                      (ins GPR:$Rn, GPR:$Rm),
-                      IIC_iMAC64, "smlalbt", "\t$RdLo, $RdHi, $Rn, $Rm",
-                      [/* For disassembly only; pattern left blank */]>,
+def SMLALBT : AMulxyI64<0b0001010, 0b10, (outs GPRnopc:$RdLo, GPRnopc:$RdHi),
+                      (ins GPRnopc:$Rn, GPRnopc:$Rm),
+                      IIC_iMAC64, "smlalbt", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
               Requires<[IsARM, HasV5TE]>;
 
-def SMLALTB : AMulxyI64<0b0001010, 0b01, (outs GPR:$RdLo, GPR:$RdHi),
-                      (ins GPR:$Rn, GPR:$Rm),
-                      IIC_iMAC64, "smlaltb", "\t$RdLo, $RdHi, $Rn, $Rm",
-                      [/* For disassembly only; pattern left blank */]>,
+def SMLALTB : AMulxyI64<0b0001010, 0b01, (outs GPRnopc:$RdLo, GPRnopc:$RdHi),
+                      (ins GPRnopc:$Rn, GPRnopc:$Rm),
+                      IIC_iMAC64, "smlaltb", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
               Requires<[IsARM, HasV5TE]>;
 
-def SMLALTT : AMulxyI64<0b0001010, 0b11, (outs GPR:$RdLo, GPR:$RdHi),
-                      (ins GPR:$Rn, GPR:$Rm),
-                      IIC_iMAC64, "smlaltt", "\t$RdLo, $RdHi, $Rn, $Rm",
-                      [/* For disassembly only; pattern left blank */]>,
+def SMLALTT : AMulxyI64<0b0001010, 0b11, (outs GPRnopc:$RdLo, GPRnopc:$RdHi),
+                      (ins GPRnopc:$Rn, GPRnopc:$Rm),
+                      IIC_iMAC64, "smlaltt", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
               Requires<[IsARM, HasV5TE]>;
 
-// Helper class for AI_smld -- for disassembly only
+// Helper class for AI_smld.
 class AMulDualIbase<bit long, bit sub, bit swap, dag oops, dag iops,
                     InstrItinClass itin, string opc, string asm>
   : AI<oops, iops, MulFrm, itin, opc, asm, []>, Requires<[IsARM, HasV6]> {
   bits<4> Rn;
   bits<4> Rm;
-  let Inst{4}     = 1;
-  let Inst{5}     = swap;
-  let Inst{6}     = sub;
-  let Inst{7}     = 0;
-  let Inst{21-20} = 0b00;
-  let Inst{22}    = long;
   let Inst{27-23} = 0b01110;
+  let Inst{22}    = long;
+  let Inst{21-20} = 0b00;
   let Inst{11-8}  = Rm;
+  let Inst{7}     = 0;
+  let Inst{6}     = sub;
+  let Inst{5}     = swap;
+  let Inst{4}     = 1;
   let Inst{3-0}   = Rn;
 }
 class AMulDualI<bit long, bit sub, bit swap, dag oops, dag iops,
@@ -3024,6 +3741,8 @@ class AMulDualIa<bit long, bit sub, bit swap, dag oops, dag iops,
                 InstrItinClass itin, string opc, string asm>
   : AMulDualIbase<long, sub, swap, oops, iops, itin, opc, asm> {
   bits<4> Ra;
+  bits<4> Rd;
+  let Inst{19-16} = Rd;
   let Inst{15-12} = Ra;
 }
 class AMulDualI64<bit long, bit sub, bit swap, dag oops, dag iops,
@@ -3037,18 +3756,20 @@ class AMulDualI64<bit long, bit sub, bit swap, dag oops, dag iops,
 
 multiclass AI_smld<bit sub, string opc> {
 
-  def D : AMulDualIa<0, sub, 0, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def D : AMulDualIa<0, sub, 0, (outs GPRnopc:$Rd),
+                  (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
                   NoItinerary, !strconcat(opc, "d"), "\t$Rd, $Rn, $Rm, $Ra">;
 
-  def DX: AMulDualIa<0, sub, 1, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
+  def DX: AMulDualIa<0, sub, 1, (outs GPRnopc:$Rd),
+                  (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
                   NoItinerary, !strconcat(opc, "dx"), "\t$Rd, $Rn, $Rm, $Ra">;
 
-  def LD: AMulDualI64<1, sub, 0, (outs GPR:$RdLo,GPR:$RdHi),
-                  (ins GPR:$Rn, GPR:$Rm), NoItinerary,
+  def LD: AMulDualI64<1, sub, 0, (outs GPRnopc:$RdLo, GPRnopc:$RdHi),
+                  (ins GPRnopc:$Rn, GPRnopc:$Rm), NoItinerary,
                   !strconcat(opc, "ld"), "\t$RdLo, $RdHi, $Rn, $Rm">;
 
-  def LDX : AMulDualI64<1, sub, 1, (outs GPR:$RdLo,GPR:$RdHi),
-                  (ins GPR:$Rn, GPR:$Rm), NoItinerary,
+  def LDX : AMulDualI64<1, sub, 1, (outs GPRnopc:$RdLo, GPRnopc:$RdHi),
+                  (ins GPRnopc:$Rn, GPRnopc:$Rm), NoItinerary,
                   !strconcat(opc, "ldx"),"\t$RdLo, $RdHi, $Rn, $Rm">;
 
 }
@@ -3058,10 +3779,10 @@ defm SMLS : AI_smld<1, "smls">;
 
 multiclass AI_sdml<bit sub, string opc> {
 
-  def D : AMulDualI<0, sub, 0, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-                    NoItinerary, !strconcat(opc, "d"), "\t$Rd, $Rn, $Rm">;
-  def DX : AMulDualI<0, sub, 1, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
-                    NoItinerary, !strconcat(opc, "dx"), "\t$Rd, $Rn, $Rm">;
+  def D:AMulDualI<0, sub, 0, (outs GPRnopc:$Rd), (ins GPRnopc:$Rn, GPRnopc:$Rm),
+                  NoItinerary, !strconcat(opc, "d"), "\t$Rd, $Rn, $Rm">;
+  def DX:AMulDualI<0, sub, 1, (outs GPRnopc:$Rd),(ins GPRnopc:$Rn, GPRnopc:$Rm),
+                  NoItinerary, !strconcat(opc, "dx"), "\t$Rd, $Rn, $Rm">;
 }
 
 defm SMUA : AI_sdml<0, "smua">;
@@ -3100,55 +3821,38 @@ def : ARMV6Pat<(or (sra (shl GPR:$Rm, (i32 24)), (i32 16)),
                    (and (srl GPR:$Rm, (i32 8)), 0xFF)),
                (REVSH GPR:$Rm)>;
 
-def lsl_shift_imm : SDNodeXForm<imm, [{
-  unsigned Sh = ARM_AM::getSORegOpc(ARM_AM::lsl, N->getZExtValue());
-  return CurDAG->getTargetConstant(Sh, MVT::i32);
-}]>;
-
-def lsl_amt : ImmLeaf<i32, [{
-  return Imm > 0 && Imm < 32;
-}], lsl_shift_imm>;
-
-def PKHBT : APKHI<0b01101000, 0, (outs GPR:$Rd),
-                              (ins GPR:$Rn, GPR:$Rm, shift_imm:$sh),
+def PKHBT : APKHI<0b01101000, 0, (outs GPRnopc:$Rd),
+                              (ins GPRnopc:$Rn, GPRnopc:$Rm, pkh_lsl_amt:$sh),
                IIC_iALUsi, "pkhbt", "\t$Rd, $Rn, $Rm$sh",
-               [(set GPR:$Rd, (or (and GPR:$Rn, 0xFFFF),
-                                  (and (shl GPR:$Rm, lsl_amt:$sh),
-                                       0xFFFF0000)))]>,
+               [(set GPRnopc:$Rd, (or (and GPRnopc:$Rn, 0xFFFF),
+                                      (and (shl GPRnopc:$Rm, pkh_lsl_amt:$sh),
+                                           0xFFFF0000)))]>,
                Requires<[IsARM, HasV6]>;
 
 // Alternate cases for PKHBT where identities eliminate some nodes.
-def : ARMV6Pat<(or (and GPR:$Rn, 0xFFFF), (and GPR:$Rm, 0xFFFF0000)),
-               (PKHBT GPR:$Rn, GPR:$Rm, 0)>;
-def : ARMV6Pat<(or (and GPR:$Rn, 0xFFFF), (shl GPR:$Rm, imm16_31:$sh)),
-               (PKHBT GPR:$Rn, GPR:$Rm, (lsl_shift_imm imm16_31:$sh))>;
-
-def asr_shift_imm : SDNodeXForm<imm, [{
-  unsigned Sh = ARM_AM::getSORegOpc(ARM_AM::asr, N->getZExtValue());
-  return CurDAG->getTargetConstant(Sh, MVT::i32);
-}]>;
-
-def asr_amt : ImmLeaf<i32, [{
-  return Imm > 0 && Imm <= 32;
-}], asr_shift_imm>;
+def : ARMV6Pat<(or (and GPRnopc:$Rn, 0xFFFF), (and GPRnopc:$Rm, 0xFFFF0000)),
+               (PKHBT GPRnopc:$Rn, GPRnopc:$Rm, 0)>;
+def : ARMV6Pat<(or (and GPRnopc:$Rn, 0xFFFF), (shl GPRnopc:$Rm, imm16_31:$sh)),
+               (PKHBT GPRnopc:$Rn, GPRnopc:$Rm, imm16_31:$sh)>;
 
 // Note: Shifts of 1-15 bits will be transformed to srl instead of sra and
 // will match the pattern below.
-def PKHTB : APKHI<0b01101000, 1, (outs GPR:$Rd),
-                              (ins GPR:$Rn, GPR:$Rm, shift_imm:$sh),
+def PKHTB : APKHI<0b01101000, 1, (outs GPRnopc:$Rd),
+                              (ins GPRnopc:$Rn, GPRnopc:$Rm, pkh_asr_amt:$sh),
                IIC_iBITsi, "pkhtb", "\t$Rd, $Rn, $Rm$sh",
-               [(set GPR:$Rd, (or (and GPR:$Rn, 0xFFFF0000),
-                                  (and (sra GPR:$Rm, asr_amt:$sh),
-                                       0xFFFF)))]>,
+               [(set GPRnopc:$Rd, (or (and GPRnopc:$Rn, 0xFFFF0000),
+                                      (and (sra GPRnopc:$Rm, pkh_asr_amt:$sh),
+                                           0xFFFF)))]>,
                Requires<[IsARM, HasV6]>;
 
 // Alternate cases for PKHTB where identities eliminate some nodes.  Note that
 // a shift amount of 0 is *not legal* here, it is PKHBT instead.
-def : ARMV6Pat<(or (and GPR:$src1, 0xFFFF0000), (srl GPR:$src2, imm16_31:$sh)),
-               (PKHTB GPR:$src1, GPR:$src2, (asr_shift_imm imm16_31:$sh))>;
-def : ARMV6Pat<(or (and GPR:$src1, 0xFFFF0000),
-                   (and (srl GPR:$src2, imm1_15:$sh), 0xFFFF)),
-               (PKHTB GPR:$src1, GPR:$src2, (asr_shift_imm imm1_15:$sh))>;
+def : ARMV6Pat<(or (and GPRnopc:$src1, 0xFFFF0000),
+                   (srl GPRnopc:$src2, imm16_31:$sh)),
+               (PKHTB GPRnopc:$src1, GPRnopc:$src2, imm16_31:$sh)>;
+def : ARMV6Pat<(or (and GPRnopc:$src1, 0xFFFF0000),
+                   (and (srl GPRnopc:$src2, imm1_15:$sh), 0xFFFF)),
+               (PKHTB GPRnopc:$src1, GPRnopc:$src2, imm1_15:$sh)>;
 
 //===----------------------------------------------------------------------===//
 //  Comparison Instructions...
@@ -3163,8 +3867,10 @@ def : ARMPat<(ARMcmpZ GPR:$src, so_imm:$imm),
              (CMPri   GPR:$src, so_imm:$imm)>;
 def : ARMPat<(ARMcmpZ GPR:$src, GPR:$rhs),
              (CMPrr   GPR:$src, GPR:$rhs)>;
-def : ARMPat<(ARMcmpZ GPR:$src, so_reg:$rhs),
-             (CMPrs   GPR:$src, so_reg:$rhs)>;
+def : ARMPat<(ARMcmpZ GPR:$src, so_reg_imm:$rhs),
+             (CMPrsi   GPR:$src, so_reg_imm:$rhs)>;
+def : ARMPat<(ARMcmpZ GPR:$src, so_reg_reg:$rhs),
+             (CMPrsr   GPR:$src, so_reg_reg:$rhs)>;
 
 // FIXME: We have to be careful when using the CMN instruction and comparison
 // with 0. One would expect these two pieces of code should give identical
@@ -3250,15 +3956,23 @@ def MOVCCr : ARMPseudoInst<(outs GPR:$Rd), (ins GPR:$false, GPR:$Rm, pred:$p),
                            4, IIC_iCMOVr,
   [/*(set GPR:$Rd, (ARMcmov GPR:$false, GPR:$Rm, imm:$cc, CCR:$ccr))*/]>,
       RegConstraint<"$false = $Rd">;
-def MOVCCs : ARMPseudoInst<(outs GPR:$Rd),
-                           (ins GPR:$false, so_reg:$shift, pred:$p),
+def MOVCCsi : ARMPseudoInst<(outs GPR:$Rd),
+                           (ins GPR:$false, so_reg_imm:$shift, pred:$p),
                            4, IIC_iCMOVsr,
-  [/*(set GPR:$Rd, (ARMcmov GPR:$false, so_reg:$shift, imm:$cc, CCR:$ccr))*/]>,
+  [/*(set GPR:$Rd, (ARMcmov GPR:$false, so_reg_imm:$shift,
+                            imm:$cc, CCR:$ccr))*/]>,
       RegConstraint<"$false = $Rd">;
+def MOVCCsr : ARMPseudoInst<(outs GPR:$Rd),
+                           (ins GPR:$false, so_reg_reg:$shift, pred:$p),
+                           4, IIC_iCMOVsr,
+  [/*(set GPR:$Rd, (ARMcmov GPR:$false, so_reg_reg:$shift,
+                            imm:$cc, CCR:$ccr))*/]>,
+      RegConstraint<"$false = $Rd">;
+
 
 let isMoveImm = 1 in
 def MOVCCi16 : ARMPseudoInst<(outs GPR:$Rd),
-                             (ins GPR:$false, i32imm_hilo16:$imm, pred:$p),
+                             (ins GPR:$false, imm0_65535_expr:$imm, pred:$p),
                              4, IIC_iMOVi,
                              []>,
       RegConstraint<"$false = $Rd">, Requires<[IsARM, HasV6T2]>;
@@ -3288,9 +4002,14 @@ def MVNCCi : ARMPseudoInst<(outs GPR:$Rd),
 // Atomic operations intrinsics
 //
 
+def MemBarrierOptOperand : AsmOperandClass {
+  let Name = "MemBarrierOpt";
+  let ParserMethod = "parseMemBarrierOptOperand";
+}
 def memb_opt : Operand<i32> {
   let PrintMethod = "printMemBOption";
   let ParserMatchClass = MemBarrierOptOperand;
+  let DecoderMethod = "DecodeMemBarrierOption";
 }
 
 // memory barriers protect the atomic sequences
@@ -3321,8 +4040,16 @@ def ISB : AInoP<(outs), (ins memb_opt:$opt), MiscFrm, NoItinerary,
   let Inst{3-0} = opt;
 }
 
+// Pseudo isntruction that combines movs + predicated rsbmi
+// to implement integer ABS
+let usesCustomInserter = 1, Defs = [CPSR] in {
+def ABS : ARMPseudoInst<
+  (outs GPR:$dst), (ins GPR:$src),
+  8, NoItinerary, []>;
+}
+
 let usesCustomInserter = 1 in {
-  let Uses = [CPSR] in {
+  let Defs = [CPSR] in {
     def ATOMIC_LOAD_ADD_I8 : PseudoInst<
       (outs GPR:$dst), (ins GPR:$ptr, GPR:$incr), NoItinerary,
       [(set GPR:$dst, (atomic_load_add_8 GPR:$ptr, GPR:$incr))]>;
@@ -3437,44 +4164,47 @@ let usesCustomInserter = 1 in {
 }
 
 let mayLoad = 1 in {
-def LDREXB : AIldrex<0b10, (outs GPR:$Rt), (ins addrmode7:$addr), NoItinerary,
+def LDREXB : AIldrex<0b10, (outs GPR:$Rt), (ins addr_offset_none:$addr),
+                     NoItinerary,
                     "ldrexb", "\t$Rt, $addr", []>;
-def LDREXH : AIldrex<0b11, (outs GPR:$Rt), (ins addrmode7:$addr), NoItinerary,
-                    "ldrexh", "\t$Rt, $addr", []>;
-def LDREX  : AIldrex<0b00, (outs GPR:$Rt), (ins addrmode7:$addr), NoItinerary,
-                    "ldrex", "\t$Rt, $addr", []>;
+def LDREXH : AIldrex<0b11, (outs GPR:$Rt), (ins addr_offset_none:$addr),
+                     NoItinerary, "ldrexh", "\t$Rt, $addr", []>;
+def LDREX  : AIldrex<0b00, (outs GPR:$Rt), (ins addr_offset_none:$addr),
+                     NoItinerary, "ldrex", "\t$Rt, $addr", []>;
 let hasExtraDefRegAllocReq = 1 in
-  def LDREXD : AIldrex<0b01, (outs GPR:$Rt, GPR:$Rt2), (ins addrmode7:$addr),
-                      NoItinerary, "ldrexd", "\t$Rt, $Rt2, $addr", []>;
+def LDREXD: AIldrex<0b01, (outs GPR:$Rt, GPR:$Rt2),(ins addr_offset_none:$addr),
+                      NoItinerary, "ldrexd", "\t$Rt, $Rt2, $addr", []> {
+  let DecoderMethod = "DecodeDoubleRegLoad";
+}
 }
 
 let mayStore = 1, Constraints = "@earlyclobber $Rd" in {
-def STREXB : AIstrex<0b10, (outs GPR:$Rd), (ins GPR:$Rt, addrmode7:$addr),
+def STREXB: AIstrex<0b10, (outs GPR:$Rd), (ins GPR:$Rt, addr_offset_none:$addr),
                     NoItinerary, "strexb", "\t$Rd, $Rt, $addr", []>;
-def STREXH : AIstrex<0b11, (outs GPR:$Rd), (ins GPR:$Rt, addrmode7:$addr),
+def STREXH: AIstrex<0b11, (outs GPR:$Rd), (ins GPR:$Rt, addr_offset_none:$addr),
                     NoItinerary, "strexh", "\t$Rd, $Rt, $addr", []>;
-def STREX  : AIstrex<0b00, (outs GPR:$Rd), (ins GPR:$Rt, addrmode7:$addr),
+def STREX : AIstrex<0b00, (outs GPR:$Rd), (ins GPR:$Rt, addr_offset_none:$addr),
                     NoItinerary, "strex", "\t$Rd, $Rt, $addr", []>;
 }
 
 let hasExtraSrcRegAllocReq = 1, Constraints = "@earlyclobber $Rd" in
 def STREXD : AIstrex<0b01, (outs GPR:$Rd),
-                    (ins GPR:$Rt, GPR:$Rt2, addrmode7:$addr),
-                    NoItinerary, "strexd", "\t$Rd, $Rt, $Rt2, $addr", []>;
+                    (ins GPR:$Rt, GPR:$Rt2, addr_offset_none:$addr),
+                    NoItinerary, "strexd", "\t$Rd, $Rt, $Rt2, $addr", []> {
+  let DecoderMethod = "DecodeDoubleRegStore";
+}
 
-// Clear-Exclusive is for disassembly only.
-def CLREX : AXI<(outs), (ins), MiscFrm, NoItinerary, "clrex",
-                [/* For disassembly only; pattern left blank */]>,
+def CLREX : AXI<(outs), (ins), MiscFrm, NoItinerary, "clrex", []>,
             Requires<[IsARM, HasV7]>  {
   let Inst{31-0} = 0b11110101011111111111000000011111;
 }
 
-// SWP/SWPB are deprecated in V6/V7 and for disassembly only.
-let mayLoad = 1 in {
-def SWP  : AIswp<0, (outs GPR:$Rt), (ins GPR:$Rt2, GPR:$Rn), "swp",
-             [/* For disassembly only; pattern left blank */]>;
-def SWPB : AIswp<1, (outs GPR:$Rt), (ins GPR:$Rt2, GPR:$Rn), "swpb",
-             [/* For disassembly only; pattern left blank */]>;
+// SWP/SWPB are deprecated in V6/V7.
+let mayLoad = 1, mayStore = 1 in {
+def SWP : AIswp<0, (outs GPR:$Rt), (ins GPR:$Rt2, addr_offset_none:$addr),
+                "swp", []>;
+def SWPB: AIswp<1, (outs GPR:$Rt), (ins GPR:$Rt2, addr_offset_none:$addr),
+                "swpb", []>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -3526,108 +4256,171 @@ def CDP2 : ABXI<0b1110, (outs), (ins p_imm:$cop, imm0_15:$opc1,
 
 class ACI<dag oops, dag iops, string opc, string asm,
           IndexMode im = IndexModeNone>
-  : InoP<oops, iops, AddrModeNone, 4, im, BrFrm, NoItinerary,
-         opc, asm, "", [/* For disassembly only; pattern left blank */]> {
+  : I<oops, iops, AddrModeNone, 4, im, BrFrm, NoItinerary,
+      opc, asm, "", []> {
   let Inst{27-25} = 0b110;
 }
-
-multiclass LdStCop<bits<4> op31_28, bit load, dag ops, string opc, string cond>{
-
-  def _OFFSET : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd, addrmode2:$addr), ops),
-      !strconcat(opc, cond), "\tp$cop, cr$CRd, $addr"> {
-    let Inst{31-28} = op31_28;
+class ACInoP<dag oops, dag iops, string opc, string asm,
+          IndexMode im = IndexModeNone>
+  : InoP<oops, iops, AddrModeNone, 4, im, BrFrm, NoItinerary,
+         opc, asm, "", []> {
+  let Inst{31-28} = 0b1111;
+  let Inst{27-25} = 0b110;
+}
+multiclass LdStCop<bit load, bit Dbit, string asm> {
+  def _OFFSET : ACI<(outs), (ins p_imm:$cop, c_imm:$CRd, addrmode5:$addr),
+                    asm, "\t$cop, $CRd, $addr"> {
+    bits<13> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 1; // P = 1
+    let Inst{23} = addr{8};
+    let Inst{22} = Dbit;
     let Inst{21} = 0; // W = 0
-    let Inst{22} = 0; // D = 0
     let Inst{20} = load;
+    let Inst{19-16} = addr{12-9};
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = addr{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
-  def _PRE : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd, addrmode2:$addr), ops),
-      !strconcat(opc, cond), "\tp$cop, cr$CRd, $addr!", IndexModePre> {
-    let Inst{31-28} = op31_28;
+  def _PRE : ACI<(outs), (ins p_imm:$cop, c_imm:$CRd, addrmode5:$addr),
+                 asm, "\t$cop, $CRd, $addr!", IndexModePre> {
+    bits<13> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 1; // P = 1
+    let Inst{23} = addr{8};
+    let Inst{22} = Dbit;
     let Inst{21} = 1; // W = 1
-    let Inst{22} = 0; // D = 0
     let Inst{20} = load;
+    let Inst{19-16} = addr{12-9};
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = addr{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
-  def _POST : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd, addrmode2:$addr), ops),
-      !strconcat(opc, cond), "\tp$cop, cr$CRd, $addr", IndexModePost> {
-    let Inst{31-28} = op31_28;
+  def _POST: ACI<(outs), (ins p_imm:$cop, c_imm:$CRd, addr_offset_none:$addr,
+                              postidx_imm8s4:$offset),
+                 asm, "\t$cop, $CRd, $addr, $offset", IndexModePost> {
+    bits<9> offset;
+    bits<4> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 0; // P = 0
+    let Inst{23} = offset{8};
+    let Inst{22} = Dbit;
     let Inst{21} = 1; // W = 1
-    let Inst{22} = 0; // D = 0
     let Inst{20} = load;
+    let Inst{19-16} = addr;
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = offset{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
   def _OPTION : ACI<(outs),
-      !con((ins nohash_imm:$cop,nohash_imm:$CRd,GPR:$base, nohash_imm:$option),
-            ops),
-      !strconcat(opc, cond), "\tp$cop, cr$CRd, [$base], \\{$option\\}"> {
-    let Inst{31-28} = op31_28;
+                    (ins p_imm:$cop, c_imm:$CRd, addr_offset_none:$addr,
+                         coproc_option_imm:$option),
+      asm, "\t$cop, $CRd, $addr, $option"> {
+    bits<8> option;
+    bits<4> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 0; // P = 0
     let Inst{23} = 1; // U = 1
+    let Inst{22} = Dbit;
     let Inst{21} = 0; // W = 0
-    let Inst{22} = 0; // D = 0
     let Inst{20} = load;
+    let Inst{19-16} = addr;
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = option;
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
-  def L_OFFSET : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd, addrmode2:$addr), ops),
-      !strconcat(!strconcat(opc, "l"), cond), "\tp$cop, cr$CRd, $addr"> {
-    let Inst{31-28} = op31_28;
+}
+multiclass LdSt2Cop<bit load, bit Dbit, string asm> {
+  def _OFFSET : ACInoP<(outs), (ins p_imm:$cop, c_imm:$CRd, addrmode5:$addr),
+                       asm, "\t$cop, $CRd, $addr"> {
+    bits<13> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 1; // P = 1
+    let Inst{23} = addr{8};
+    let Inst{22} = Dbit;
     let Inst{21} = 0; // W = 0
-    let Inst{22} = 1; // D = 1
     let Inst{20} = load;
+    let Inst{19-16} = addr{12-9};
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = addr{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
-  def L_PRE : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd, addrmode2:$addr), ops),
-      !strconcat(!strconcat(opc, "l"), cond), "\tp$cop, cr$CRd, $addr!",
-      IndexModePre> {
-    let Inst{31-28} = op31_28;
+  def _PRE : ACInoP<(outs), (ins p_imm:$cop, c_imm:$CRd, addrmode5:$addr),
+                    asm, "\t$cop, $CRd, $addr!", IndexModePre> {
+    bits<13> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 1; // P = 1
+    let Inst{23} = addr{8};
+    let Inst{22} = Dbit;
     let Inst{21} = 1; // W = 1
-    let Inst{22} = 1; // D = 1
     let Inst{20} = load;
+    let Inst{19-16} = addr{12-9};
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = addr{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
-  def L_POST : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd, addrmode2:$addr), ops),
-      !strconcat(!strconcat(opc, "l"), cond), "\tp$cop, cr$CRd, $addr",
-      IndexModePost> {
-    let Inst{31-28} = op31_28;
+  def _POST: ACInoP<(outs), (ins p_imm:$cop, c_imm:$CRd, addr_offset_none:$addr,
+                                 postidx_imm8s4:$offset),
+                 asm, "\t$cop, $CRd, $addr, $offset", IndexModePost> {
+    bits<9> offset;
+    bits<4> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 0; // P = 0
+    let Inst{23} = offset{8};
+    let Inst{22} = Dbit;
     let Inst{21} = 1; // W = 1
-    let Inst{22} = 1; // D = 1
     let Inst{20} = load;
+    let Inst{19-16} = addr;
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = offset{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
-
-  def L_OPTION : ACI<(outs),
-      !con((ins nohash_imm:$cop, nohash_imm:$CRd,GPR:$base,nohash_imm:$option),
-            ops),
-      !strconcat(!strconcat(opc, "l"), cond),
-      "\tp$cop, cr$CRd, [$base], \\{$option\\}"> {
-    let Inst{31-28} = op31_28;
+  def _OPTION : ACInoP<(outs),
+                       (ins p_imm:$cop, c_imm:$CRd, addr_offset_none:$addr,
+                            coproc_option_imm:$option),
+      asm, "\t$cop, $CRd, $addr, $option"> {
+    bits<8> option;
+    bits<4> addr;
+    bits<4> cop;
+    bits<4> CRd;
     let Inst{24} = 0; // P = 0
     let Inst{23} = 1; // U = 1
+    let Inst{22} = Dbit;
     let Inst{21} = 0; // W = 0
-    let Inst{22} = 1; // D = 1
     let Inst{20} = load;
+    let Inst{19-16} = addr;
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = option;
+    let DecoderMethod = "DecodeCopMemInstruction";
   }
 }
 
-defm LDC  : LdStCop<{?,?,?,?}, 1, (ins pred:$p), "ldc",  "${p}">;
-defm LDC2 : LdStCop<0b1111,    1, (ins),         "ldc2", "">;
-defm STC  : LdStCop<{?,?,?,?}, 0, (ins pred:$p), "stc",  "${p}">;
-defm STC2 : LdStCop<0b1111,    0, (ins),         "stc2", "">;
+defm LDC   : LdStCop <1, 0, "ldc">;
+defm LDCL  : LdStCop <1, 1, "ldcl">;
+defm STC   : LdStCop <0, 0, "stc">;
+defm STCL  : LdStCop <0, 1, "stcl">;
+defm LDC2  : LdSt2Cop<1, 0, "ldc2">;
+defm LDC2L : LdSt2Cop<1, 1, "ldc2l">;
+defm STC2  : LdSt2Cop<0, 0, "stc2">;
+defm STC2L : LdSt2Cop<0, 1, "stc2l">;
 
 //===----------------------------------------------------------------------===//
-// Move between coprocessor and ARM core register -- for disassembly only
+// Move between coprocessor and ARM core register.
 //
 
 class MovRCopro<string opc, bit direction, dag oops, dag iops,
@@ -3660,8 +4453,8 @@ def MCR : MovRCopro<"mcr", 0 /* from ARM core register to coprocessor */,
                                   imm:$CRm, imm:$opc2)]>;
 def MRC : MovRCopro<"mrc", 1 /* from coprocessor to ARM core register */,
                     (outs GPR:$Rt),
-                    (ins p_imm:$cop, i32imm:$opc1, c_imm:$CRn, c_imm:$CRm,
-                         i32imm:$opc2), []>;
+                    (ins p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm,
+                         imm0_7:$opc2), []>;
 
 def : ARMPat<(int_arm_mrc imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2),
              (MRC imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
@@ -3697,15 +4490,14 @@ def MCR2 : MovRCopro2<"mcr2", 0 /* from ARM core register to coprocessor */,
                                      imm:$CRm, imm:$opc2)]>;
 def MRC2 : MovRCopro2<"mrc2", 1 /* from coprocessor to ARM core register */,
                       (outs GPR:$Rt),
-                      (ins p_imm:$cop, i32imm:$opc1, c_imm:$CRn, c_imm:$CRm,
-                           i32imm:$opc2), []>;
+                      (ins p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm,
+                           imm0_7:$opc2), []>;
 
 def : ARMV5TPat<(int_arm_mrc2 imm:$cop, imm:$opc1, imm:$CRn,
                               imm:$CRm, imm:$opc2),
                 (MRC2 imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
 
-class MovRRCopro<string opc, bit direction,
-                 list<dag> pattern = [/* For disassembly only */]>
+class MovRRCopro<string opc, bit direction, list<dag> pattern = []>
   : ABI<0b1100, (outs), (ins p_imm:$cop, imm0_15:$opc1,
         GPR:$Rt, GPR:$Rt2, c_imm:$CRm),
         NoItinerary, opc, "\t$cop, $opc1, $Rt, $Rt2, $CRm", pattern> {
@@ -3730,8 +4522,7 @@ def MCRR : MovRRCopro<"mcrr", 0 /* from ARM core register to coprocessor */,
                                      imm:$CRm)]>;
 def MRRC : MovRRCopro<"mrrc", 1 /* from coprocessor to ARM core register */>;
 
-class MovRRCopro2<string opc, bit direction,
-                  list<dag> pattern = [/* For disassembly only */]>
+class MovRRCopro2<string opc, bit direction, list<dag> pattern = []>
   : ABXI<0b1100, (outs), (ins p_imm:$cop, imm0_15:$opc1,
          GPR:$Rt, GPR:$Rt2, c_imm:$CRm), NoItinerary,
          !strconcat(opc, "\t$cop, $opc1, $Rt, $Rt2, $CRm"), pattern> {
@@ -3758,20 +4549,22 @@ def MCRR2 : MovRRCopro2<"mcrr2", 0 /* from ARM core register to coprocessor */,
 def MRRC2 : MovRRCopro2<"mrrc2", 1 /* from coprocessor to ARM core register */>;
 
 //===----------------------------------------------------------------------===//
-// Move between special register and ARM core register -- for disassembly only
+// Move between special register and ARM core register
 //
 
 // Move to ARM core register from Special Register
-def MRS : ABI<0b0001, (outs GPR:$Rd), (ins), NoItinerary, "mrs", "\t$Rd, cpsr",
-              [/* For disassembly only; pattern left blank */]> {
+def MRS : ABI<0b0001, (outs GPR:$Rd), (ins), NoItinerary,
+              "mrs", "\t$Rd, apsr", []> {
   bits<4> Rd;
   let Inst{23-16} = 0b00001111;
   let Inst{15-12} = Rd;
   let Inst{7-4} = 0b0000;
 }
 
-def MRSsys : ABI<0b0001, (outs GPR:$Rd), (ins), NoItinerary,"mrs","\t$Rd, spsr",
-              [/* For disassembly only; pattern left blank */]> {
+def : InstAlias<"mrs${p} $Rd, cpsr", (MRS GPR:$Rd, pred:$p)>, Requires<[IsARM]>;
+
+def MRSsys : ABI<0b0001, (outs GPR:$Rd), (ins), NoItinerary,
+                 "mrs", "\t$Rd, spsr", []> {
   bits<4> Rd;
   let Inst{23-16} = 0b01001111;
   let Inst{15-12} = Rd;
@@ -3785,8 +4578,7 @@ def MRSsys : ABI<0b0001, (outs GPR:$Rd), (ins), NoItinerary,"mrs","\t$Rd, spsr",
 // operand contains the special register (R Bit) in bit 4 and bits 3-0 contains
 // the mask with the fields to be accessed in the special register.
 def MSR : ABI<0b0001, (outs), (ins msr_mask:$mask, GPR:$Rn), NoItinerary,
-              "msr", "\t$mask, $Rn",
-              [/* For disassembly only; pattern left blank */]> {
+              "msr", "\t$mask, $Rn", []> {
   bits<5> mask;
   bits<4> Rn;
 
@@ -3800,8 +4592,7 @@ def MSR : ABI<0b0001, (outs), (ins msr_mask:$mask, GPR:$Rn), NoItinerary,
 }
 
 def MSRi : ABI<0b0011, (outs), (ins msr_mask:$mask,  so_imm:$a), NoItinerary,
-               "msr", "\t$mask, $a",
-               [/* For disassembly only; pattern left blank */]> {
+               "msr", "\t$mask, $a", []> {
   bits<5> mask;
   bits<12> a;
 
@@ -4030,6 +4821,47 @@ def : ARMV5TEPat<(add GPR:$acc,
 def : ARMPat<(ARMMemBarrierMCR GPR:$zero), (MCR 15, 0, GPR:$zero, 7, 10, 5)>,
          Requires<[IsARM, HasV6]>;
 
+// SXT/UXT with no rotate
+let AddedComplexity = 16 in {
+def : ARMV6Pat<(and GPR:$Src, 0x000000FF), (UXTB GPR:$Src, 0)>;
+def : ARMV6Pat<(and GPR:$Src, 0x0000FFFF), (UXTH GPR:$Src, 0)>;
+def : ARMV6Pat<(and GPR:$Src, 0x00FF00FF), (UXTB16 GPR:$Src, 0)>;
+def : ARMV6Pat<(add GPR:$Rn, (and GPR:$Rm, 0x00FF)),
+               (UXTAB GPR:$Rn, GPR:$Rm, 0)>;
+def : ARMV6Pat<(add GPR:$Rn, (and GPR:$Rm, 0xFFFF)),
+               (UXTAH GPR:$Rn, GPR:$Rm, 0)>;
+}
+
+def : ARMV6Pat<(sext_inreg GPR:$Src, i8),  (SXTB GPR:$Src, 0)>;
+def : ARMV6Pat<(sext_inreg GPR:$Src, i16), (SXTH GPR:$Src, 0)>;
+
+def : ARMV6Pat<(add GPR:$Rn, (sext_inreg GPRnopc:$Rm, i8)),
+               (SXTAB GPR:$Rn, GPRnopc:$Rm, 0)>;
+def : ARMV6Pat<(add GPR:$Rn, (sext_inreg GPRnopc:$Rm, i16)),
+               (SXTAH GPR:$Rn, GPRnopc:$Rm, 0)>;
+
+// Atomic load/store patterns
+def : ARMPat<(atomic_load_8 ldst_so_reg:$src),
+             (LDRBrs ldst_so_reg:$src)>;
+def : ARMPat<(atomic_load_8 addrmode_imm12:$src),
+             (LDRBi12 addrmode_imm12:$src)>;
+def : ARMPat<(atomic_load_16 addrmode3:$src),
+             (LDRH addrmode3:$src)>;
+def : ARMPat<(atomic_load_32 ldst_so_reg:$src),
+             (LDRrs ldst_so_reg:$src)>;
+def : ARMPat<(atomic_load_32 addrmode_imm12:$src),
+             (LDRi12 addrmode_imm12:$src)>;
+def : ARMPat<(atomic_store_8 ldst_so_reg:$ptr, GPR:$val),
+             (STRBrs GPR:$val, ldst_so_reg:$ptr)>;
+def : ARMPat<(atomic_store_8 addrmode_imm12:$ptr, GPR:$val),
+             (STRBi12 GPR:$val, addrmode_imm12:$ptr)>;
+def : ARMPat<(atomic_store_16 addrmode3:$ptr, GPR:$val),
+             (STRH GPR:$val, addrmode3:$ptr)>;
+def : ARMPat<(atomic_store_32 ldst_so_reg:$ptr, GPR:$val),
+             (STRrs GPR:$val, ldst_so_reg:$ptr)>;
+def : ARMPat<(atomic_store_32 addrmode_imm12:$ptr, GPR:$val),
+             (STRi12 GPR:$val, addrmode_imm12:$ptr)>;
+
 
 //===----------------------------------------------------------------------===//
 // Thumb Support
@@ -4070,7 +4902,103 @@ def : MnemonicAlias<"swi", "svc">;
 // Load / Store Multiple
 def : MnemonicAlias<"ldmfd", "ldm">;
 def : MnemonicAlias<"ldmia", "ldm">;
+def : MnemonicAlias<"ldmea", "ldmdb">;
 def : MnemonicAlias<"stmfd", "stmdb">;
 def : MnemonicAlias<"stmia", "stm">;
 def : MnemonicAlias<"stmea", "stm">;
 
+// PKHBT/PKHTB with default shift amount. PKHTB is equivalent to PKHBT when the
+// shift amount is zero (i.e., unspecified).
+def : InstAlias<"pkhbt${p} $Rd, $Rn, $Rm",
+                (PKHBT GPRnopc:$Rd, GPRnopc:$Rn, GPRnopc:$Rm, 0, pred:$p)>,
+        Requires<[IsARM, HasV6]>;
+def : InstAlias<"pkhtb${p} $Rd, $Rn, $Rm",
+                (PKHBT GPRnopc:$Rd, GPRnopc:$Rn, GPRnopc:$Rm, 0, pred:$p)>,
+        Requires<[IsARM, HasV6]>;
+
+// PUSH/POP aliases for STM/LDM
+def : ARMInstAlias<"push${p} $regs", (STMDB_UPD SP, pred:$p, reglist:$regs)>;
+def : ARMInstAlias<"pop${p} $regs", (LDMIA_UPD SP, pred:$p, reglist:$regs)>;
+
+// SSAT/USAT optional shift operand.
+def : ARMInstAlias<"ssat${p} $Rd, $sat_imm, $Rn",
+                (SSAT GPRnopc:$Rd, imm1_32:$sat_imm, GPRnopc:$Rn, 0, pred:$p)>;
+def : ARMInstAlias<"usat${p} $Rd, $sat_imm, $Rn",
+                (USAT GPRnopc:$Rd, imm0_31:$sat_imm, GPRnopc:$Rn, 0, pred:$p)>;
+
+
+// Extend instruction optional rotate operand.
+def : ARMInstAlias<"sxtab${p} $Rd, $Rn, $Rm",
+                (SXTAB GPRnopc:$Rd, GPR:$Rn, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"sxtah${p} $Rd, $Rn, $Rm",
+                (SXTAH GPRnopc:$Rd, GPR:$Rn, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"sxtab16${p} $Rd, $Rn, $Rm",
+                (SXTAB16 GPRnopc:$Rd, GPR:$Rn, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"sxtb${p} $Rd, $Rm",
+                (SXTB GPRnopc:$Rd, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"sxtb16${p} $Rd, $Rm",
+                (SXTB16 GPRnopc:$Rd, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"sxth${p} $Rd, $Rm",
+                (SXTH GPRnopc:$Rd, GPRnopc:$Rm, 0, pred:$p)>;
+
+def : ARMInstAlias<"uxtab${p} $Rd, $Rn, $Rm",
+                (UXTAB GPRnopc:$Rd, GPR:$Rn, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"uxtah${p} $Rd, $Rn, $Rm",
+                (UXTAH GPRnopc:$Rd, GPR:$Rn, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"uxtab16${p} $Rd, $Rn, $Rm",
+                (UXTAB16 GPRnopc:$Rd, GPR:$Rn, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"uxtb${p} $Rd, $Rm",
+                (UXTB GPRnopc:$Rd, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"uxtb16${p} $Rd, $Rm",
+                (UXTB16 GPRnopc:$Rd, GPRnopc:$Rm, 0, pred:$p)>;
+def : ARMInstAlias<"uxth${p} $Rd, $Rm",
+                (UXTH GPRnopc:$Rd, GPRnopc:$Rm, 0, pred:$p)>;
+
+
+// RFE aliases
+def : MnemonicAlias<"rfefa", "rfeda">;
+def : MnemonicAlias<"rfeea", "rfedb">;
+def : MnemonicAlias<"rfefd", "rfeia">;
+def : MnemonicAlias<"rfeed", "rfeib">;
+def : MnemonicAlias<"rfe", "rfeia">;
+
+// SRS aliases
+def : MnemonicAlias<"srsfa", "srsda">;
+def : MnemonicAlias<"srsea", "srsdb">;
+def : MnemonicAlias<"srsfd", "srsia">;
+def : MnemonicAlias<"srsed", "srsib">;
+def : MnemonicAlias<"srs", "srsia">;
+
+// QSAX == QSUBADDX
+def : MnemonicAlias<"qsubaddx", "qsax">;
+// SASX == SADDSUBX
+def : MnemonicAlias<"saddsubx", "sasx">;
+// SHASX == SHADDSUBX
+def : MnemonicAlias<"shaddsubx", "shasx">;
+// SHSAX == SHSUBADDX
+def : MnemonicAlias<"shsubaddx", "shsax">;
+// SSAX == SSUBADDX
+def : MnemonicAlias<"ssubaddx", "ssax">;
+// UASX == UADDSUBX
+def : MnemonicAlias<"uaddsubx", "uasx">;
+// UHASX == UHADDSUBX
+def : MnemonicAlias<"uhaddsubx", "uhasx">;
+// UHSAX == UHSUBADDX
+def : MnemonicAlias<"uhsubaddx", "uhsax">;
+// UQASX == UQADDSUBX
+def : MnemonicAlias<"uqaddsubx", "uqasx">;
+// UQSAX == UQSUBADDX
+def : MnemonicAlias<"uqsubaddx", "uqsax">;
+// USAX == USUBADDX
+def : MnemonicAlias<"usubaddx", "usax">;
+
+// LDRSBT/LDRHT/LDRSHT post-index offset if optional.
+// Note that the write-back output register is a dummy operand for MC (it's
+// only meaningful for codegen), so we just pass zero here.
+// FIXME: tblgen not cooperating with argument conversions.
+//def : InstAlias<"ldrsbt${p} $Rt, $addr",
+//                (LDRSBTi GPR:$Rt, GPR:$Rt, addr_offset_none:$addr, 0,pred:$p)>;
+//def : InstAlias<"ldrht${p} $Rt, $addr",
+//                (LDRHTi GPR:$Rt, GPR:$Rt, addr_offset_none:$addr, 0, pred:$p)>;
+//def : InstAlias<"ldrsht${p} $Rt, $addr",
+//                (LDRSHTi GPR:$Rt, GPR:$Rt, addr_offset_none:$addr, 0, pred:$p)>;
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrNEON.td b/contrib/llvm/lib/Target/ARM/ARMInstrNEON.td
index 0df62f456343..7aad18695bb3 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -11,6 +11,35 @@
 //
 //===----------------------------------------------------------------------===//
 
+
+//===----------------------------------------------------------------------===//
+// NEON-specific Operands.
+//===----------------------------------------------------------------------===//
+def VectorIndex8Operand  : AsmOperandClass { let Name = "VectorIndex8"; }
+def VectorIndex16Operand : AsmOperandClass { let Name = "VectorIndex16"; }
+def VectorIndex32Operand : AsmOperandClass { let Name = "VectorIndex32"; }
+def VectorIndex8 : Operand<i32>, ImmLeaf<i32, [{
+  return ((uint64_t)Imm) < 8;
+}]> {
+  let ParserMatchClass = VectorIndex8Operand;
+  let PrintMethod = "printVectorIndex";
+  let MIOperandInfo = (ops i32imm);
+}
+def VectorIndex16 : Operand<i32>, ImmLeaf<i32, [{
+  return ((uint64_t)Imm) < 4;
+}]> {
+  let ParserMatchClass = VectorIndex16Operand;
+  let PrintMethod = "printVectorIndex";
+  let MIOperandInfo = (ops i32imm);
+}
+def VectorIndex32 : Operand<i32>, ImmLeaf<i32, [{
+  return ((uint64_t)Imm) < 2;
+}]> {
+  let ParserMatchClass = VectorIndex32Operand;
+  let PrintMethod = "printVectorIndex";
+  let MIOperandInfo = (ops i32imm);
+}
+
 //===----------------------------------------------------------------------===//
 // NEON-specific DAG Nodes.
 //===----------------------------------------------------------------------===//
@@ -175,7 +204,8 @@ class VLDQQWBPseudo<InstrItinClass itin>
                 (ins addrmode6:$addr, am6offset:$offset), itin,
                 "$addr.addr = $wb">;
 class VLDQQQQPseudo<InstrItinClass itin>
-  : PseudoNLdSt<(outs QQQQPR:$dst), (ins addrmode6:$addr, QQQQPR:$src),itin,"">;
+  : PseudoNLdSt<(outs QQQQPR:$dst), (ins addrmode6:$addr, QQQQPR:$src),itin,
+                "$src = $dst">;
 class VLDQQQQWBPseudo<InstrItinClass itin>
   : PseudoNLdSt<(outs QQQQPR:$dst, GPR:$wb),
                 (ins addrmode6:$addr, am6offset:$offset, QQQQPR:$src), itin,
@@ -190,6 +220,7 @@ class VLD1D<bits<4> op7_4, string Dt>
           "vld1", Dt, "\\{$Vd\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 class VLD1Q<bits<4> op7_4, string Dt>
   : NLdSt<0,0b10,0b1010,op7_4, (outs DPR:$Vd, DPR:$dst2),
@@ -197,6 +228,7 @@ class VLD1Q<bits<4> op7_4, string Dt>
           "vld1", Dt, "\\{$Vd, $dst2\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def  VLD1d8   : VLD1D<{0,0,0,?}, "8">;
@@ -221,6 +253,7 @@ class VLD1DWB<bits<4> op7_4, string Dt>
           "vld1", Dt, "\\{$Vd\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 class VLD1QWB<bits<4> op7_4, string Dt>
   : NLdSt<0,0b10,0b1010,op7_4, (outs DPR:$Vd, DPR:$dst2, GPR:$wb),
@@ -228,6 +261,7 @@ class VLD1QWB<bits<4> op7_4, string Dt>
           "vld1", Dt, "\\{$Vd, $dst2\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def VLD1d8_UPD  : VLD1DWB<{0,0,0,?}, "8">;
@@ -252,12 +286,14 @@ class VLD1D3<bits<4> op7_4, string Dt>
           "\\{$Vd, $dst2, $dst3\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 class VLD1D3WB<bits<4> op7_4, string Dt>
   : NLdSt<0,0b10,0b0110,op7_4, (outs DPR:$Vd, DPR:$dst2, DPR:$dst3, GPR:$wb),
           (ins addrmode6:$Rn, am6offset:$Rm), IIC_VLD1x3u, "vld1", Dt,
           "\\{$Vd, $dst2, $dst3\\}, $Rn$Rm", "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def VLD1d8T      : VLD1D3<{0,0,0,?}, "8">;
@@ -280,6 +316,7 @@ class VLD1D4<bits<4> op7_4, string Dt>
           "\\{$Vd, $dst2, $dst3, $dst4\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 class VLD1D4WB<bits<4> op7_4, string Dt>
   : NLdSt<0,0b10,0b0010,op7_4,
@@ -288,6 +325,7 @@ class VLD1D4WB<bits<4> op7_4, string Dt>
           "\\{$Vd, $dst2, $dst3, $dst4\\}, $Rn$Rm", "$Rn.addr = $wb",
           []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def VLD1d8Q      : VLD1D4<{0,0,?,?}, "8">;
@@ -310,6 +348,7 @@ class VLD2D<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vld2", Dt, "\\{$Vd, $dst2\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 class VLD2Q<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b10, 0b0011, op7_4,
@@ -318,6 +357,7 @@ class VLD2Q<bits<4> op7_4, string Dt>
           "vld2", Dt, "\\{$Vd, $dst2, $dst3, $dst4\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def  VLD2d8   : VLD2D<0b1000, {0,0,?,?}, "8">;
@@ -343,6 +383,7 @@ class VLD2DWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vld2", Dt, "\\{$Vd, $dst2\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 class VLD2QWB<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b10, 0b0011, op7_4,
@@ -351,6 +392,7 @@ class VLD2QWB<bits<4> op7_4, string Dt>
           "vld2", Dt, "\\{$Vd, $dst2, $dst3, $dst4\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def VLD2d8_UPD  : VLD2DWB<0b1000, {0,0,?,?}, "8">;
@@ -384,6 +426,7 @@ class VLD3D<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vld3", Dt, "\\{$Vd, $dst2, $dst3\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def  VLD3d8   : VLD3D<0b0100, {0,0,0,?}, "8">;
@@ -402,6 +445,7 @@ class VLD3DWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vld3", Dt, "\\{$Vd, $dst2, $dst3\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def VLD3d8_UPD  : VLD3DWB<0b0100, {0,0,0,?}, "8">;
@@ -441,6 +485,7 @@ class VLD4D<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vld4", Dt, "\\{$Vd, $dst2, $dst3, $dst4\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def  VLD4d8   : VLD4D<0b0000, {0,0,?,?}, "8">;
@@ -459,6 +504,7 @@ class VLD4DWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vld4", Dt, "\\{$Vd, $dst2, $dst3, $dst4\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVLDInstruction";
 }
 
 def VLD4d8_UPD  : VLD4DWB<0b0000, {0,0,?,?}, "8">;
@@ -530,6 +576,7 @@ class VLD1LN<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
                                          (i32 (LoadOp addrmode6:$Rn)),
                                          imm:$lane))]> {
   let Rm = 0b1111;
+  let DecoderMethod = "DecodeVLD1LN";
 }
 class VLD1LN32<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
              PatFrag LoadOp>
@@ -541,6 +588,7 @@ class VLD1LN32<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
                                          (i32 (LoadOp addrmode6oneL32:$Rn)),
                                          imm:$lane))]> {
   let Rm = 0b1111;
+  let DecoderMethod = "DecodeVLD1LN";
 }
 class VLD1QLNPseudo<ValueType Ty, PatFrag LoadOp> : VLDQLNPseudo<IIC_VLD1ln> {
   let Pattern = [(set QPR:$dst, (vector_insert (Ty QPR:$src),
@@ -580,7 +628,9 @@ class VLD1LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           (ins addrmode6:$Rn, am6offset:$Rm,
            DPR:$src, nohash_imm:$lane), IIC_VLD1lnu, "vld1", Dt,
           "\\{$Vd[$lane]\\}, $Rn$Rm",
-          "$src = $Vd, $Rn.addr = $wb", []>;
+          "$src = $Vd, $Rn.addr = $wb", []> {
+  let DecoderMethod = "DecodeVLD1LN";
+}
 
 def VLD1LNd8_UPD  : VLD1LNWB<0b0000, {?,?,?,0}, "8"> {
   let Inst{7-5} = lane{2-0};
@@ -607,6 +657,7 @@ class VLD2LN<bits<4> op11_8, bits<4> op7_4, string Dt>
           "$src1 = $Vd, $src2 = $dst2", []> {
   let Rm = 0b1111;
   let Inst{4}   = Rn{4};
+  let DecoderMethod = "DecodeVLD2LN";
 }
 
 def VLD2LNd8  : VLD2LN<0b0001, {?,?,?,?}, "8"> {
@@ -642,6 +693,7 @@ class VLD2LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           "\\{$Vd[$lane], $dst2[$lane]\\}, $Rn$Rm",
           "$src1 = $Vd, $src2 = $dst2, $Rn.addr = $wb", []> {
   let Inst{4}   = Rn{4};
+  let DecoderMethod = "DecodeVLD2LN";
 }
 
 def VLD2LNd8_UPD  : VLD2LNWB<0b0001, {?,?,?,?}, "8"> {
@@ -676,6 +728,7 @@ class VLD3LN<bits<4> op11_8, bits<4> op7_4, string Dt>
           "\\{$Vd[$lane], $dst2[$lane], $dst3[$lane]\\}, $Rn",
           "$src1 = $Vd, $src2 = $dst2, $src3 = $dst3", []> {
   let Rm = 0b1111;
+  let DecoderMethod = "DecodeVLD3LN";
 }
 
 def VLD3LNd8  : VLD3LN<0b0010, {?,?,?,0}, "8"> {
@@ -712,7 +765,9 @@ class VLD3LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           IIC_VLD3lnu, "vld3", Dt,
           "\\{$Vd[$lane], $dst2[$lane], $dst3[$lane]\\}, $Rn$Rm",
           "$src1 = $Vd, $src2 = $dst2, $src3 = $dst3, $Rn.addr = $wb",
-          []>;
+          []> {
+  let DecoderMethod = "DecodeVLD3LN";
+}
 
 def VLD3LNd8_UPD  : VLD3LNWB<0b0010, {?,?,?,0}, "8"> {
   let Inst{7-5} = lane{2-0};
@@ -748,6 +803,7 @@ class VLD4LN<bits<4> op11_8, bits<4> op7_4, string Dt>
           "$src1 = $Vd, $src2 = $dst2, $src3 = $dst3, $src4 = $dst4", []> {
   let Rm = 0b1111;
   let Inst{4}   = Rn{4};
+  let DecoderMethod = "DecodeVLD4LN";
 }
 
 def VLD4LNd8  : VLD4LN<0b0011, {?,?,?,?}, "8"> {
@@ -788,6 +844,7 @@ class VLD4LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
 "$src1 = $Vd, $src2 = $dst2, $src3 = $dst3, $src4 = $dst4, $Rn.addr = $wb",
           []> {
   let Inst{4}   = Rn{4};
+  let DecoderMethod = "DecodeVLD4LN"  ;
 }
 
 def VLD4LNd8_UPD  : VLD4LNWB<0b0011, {?,?,?,?}, "8"> {
@@ -825,6 +882,7 @@ class VLD1DUP<bits<4> op7_4, string Dt, ValueType Ty, PatFrag LoadOp>
           [(set DPR:$Vd, (Ty (NEONvdup (i32 (LoadOp addrmode6dup:$Rn)))))]> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD1DupInstruction";
 }
 class VLD1QDUPPseudo<ValueType Ty, PatFrag LoadOp> : VLDQPseudo<IIC_VLD1dup> {
   let Pattern = [(set QPR:$dst,
@@ -852,6 +910,7 @@ class VLD1QDUP<bits<4> op7_4, string Dt>
           "vld1", Dt, "\\{$Vd[], $dst2[]\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD1DupInstruction";
 }
 
 def VLD1DUPq8  : VLD1QDUP<{0,0,1,0}, "8">;
@@ -864,12 +923,14 @@ class VLD1DUPWB<bits<4> op7_4, string Dt>
           (ins addrmode6dup:$Rn, am6offset:$Rm), IIC_VLD1dupu,
           "vld1", Dt, "\\{$Vd[]\\}, $Rn$Rm", "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD1DupInstruction";
 }
 class VLD1QDUPWB<bits<4> op7_4, string Dt>
   : NLdSt<1, 0b10, 0b1100, op7_4, (outs DPR:$Vd, DPR:$dst2, GPR:$wb),
           (ins addrmode6dup:$Rn, am6offset:$Rm), IIC_VLD1dupu,
           "vld1", Dt, "\\{$Vd[], $dst2[]\\}, $Rn$Rm", "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD1DupInstruction";
 }
 
 def VLD1DUPd8_UPD  : VLD1DUPWB<{0,0,0,0}, "8">;
@@ -891,6 +952,7 @@ class VLD2DUP<bits<4> op7_4, string Dt>
           "vld2", Dt, "\\{$Vd[], $dst2[]\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD2DupInstruction";
 }
 
 def VLD2DUPd8  : VLD2DUP<{0,0,0,?}, "8">;
@@ -912,6 +974,7 @@ class VLD2DUPWB<bits<4> op7_4, string Dt>
           (ins addrmode6dup:$Rn, am6offset:$Rm), IIC_VLD2dupu,
           "vld2", Dt, "\\{$Vd[], $dst2[]\\}, $Rn$Rm", "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD2DupInstruction";
 }
 
 def VLD2DUPd8_UPD  : VLD2DUPWB<{0,0,0,0}, "8">;
@@ -932,7 +995,8 @@ class VLD3DUP<bits<4> op7_4, string Dt>
           (ins addrmode6dup:$Rn), IIC_VLD3dup,
           "vld3", Dt, "\\{$Vd[], $dst2[], $dst3[]\\}, $Rn", "", []> {
   let Rm = 0b1111;
-  let Inst{4} = Rn{4};
+  let Inst{4} = 0;
+  let DecoderMethod = "DecodeVLD3DupInstruction";
 }
 
 def VLD3DUPd8  : VLD3DUP<{0,0,0,?}, "8">;
@@ -954,7 +1018,8 @@ class VLD3DUPWB<bits<4> op7_4, string Dt>
           (ins addrmode6dup:$Rn, am6offset:$Rm), IIC_VLD3dupu,
           "vld3", Dt, "\\{$Vd[], $dst2[], $dst3[]\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
-  let Inst{4} = Rn{4};
+  let Inst{4} = 0;
+  let DecoderMethod = "DecodeVLD3DupInstruction";
 }
 
 def VLD3DUPd8_UPD  : VLD3DUPWB<{0,0,0,0}, "8">;
@@ -977,6 +1042,7 @@ class VLD4DUP<bits<4> op7_4, string Dt>
           "vld4", Dt, "\\{$Vd[], $dst2[], $dst3[], $dst4[]\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD4DupInstruction";
 }
 
 def VLD4DUPd8  : VLD4DUP<{0,0,0,?}, "8">;
@@ -1000,6 +1066,7 @@ class VLD4DUPWB<bits<4> op7_4, string Dt>
           "vld4", Dt, "\\{$Vd[], $dst2[], $dst3[], $dst4[]\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVLD4DupInstruction";
 }
 
 def VLD4DUPd8_UPD  : VLD4DUPWB<{0,0,0,0}, "8">;
@@ -1045,6 +1112,7 @@ class VST1D<bits<4> op7_4, string Dt>
           IIC_VST1, "vst1", Dt, "\\{$Vd\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 class VST1Q<bits<4> op7_4, string Dt>
   : NLdSt<0,0b00,0b1010,op7_4, (outs),
@@ -1052,6 +1120,7 @@ class VST1Q<bits<4> op7_4, string Dt>
           "vst1", Dt, "\\{$Vd, $src2\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def  VST1d8   : VST1D<{0,0,0,?}, "8">;
@@ -1075,6 +1144,7 @@ class VST1DWB<bits<4> op7_4, string Dt>
           (ins addrmode6:$Rn, am6offset:$Rm, DPR:$Vd), IIC_VST1u,
           "vst1", Dt, "\\{$Vd\\}, $Rn$Rm", "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 class VST1QWB<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b00, 0b1010, op7_4, (outs GPR:$wb),
@@ -1082,6 +1152,7 @@ class VST1QWB<bits<4> op7_4, string Dt>
           IIC_VST1x2u, "vst1", Dt, "\\{$Vd, $src2\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def VST1d8_UPD  : VST1DWB<{0,0,0,?}, "8">;
@@ -1106,6 +1177,7 @@ class VST1D3<bits<4> op7_4, string Dt>
           IIC_VST1x3, "vst1", Dt, "\\{$Vd, $src2, $src3\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 class VST1D3WB<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b00, 0b0110, op7_4, (outs GPR:$wb),
@@ -1114,6 +1186,7 @@ class VST1D3WB<bits<4> op7_4, string Dt>
           IIC_VST1x3u, "vst1", Dt, "\\{$Vd, $src2, $src3\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def VST1d8T      : VST1D3<{0,0,0,?}, "8">;
@@ -1137,6 +1210,7 @@ class VST1D4<bits<4> op7_4, string Dt>
           []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 class VST1D4WB<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b00, 0b0010, op7_4, (outs GPR:$wb),
@@ -1145,6 +1219,7 @@ class VST1D4WB<bits<4> op7_4, string Dt>
           "vst1", Dt, "\\{$Vd, $src2, $src3, $src4\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def VST1d8Q      : VST1D4<{0,0,?,?}, "8">;
@@ -1167,6 +1242,7 @@ class VST2D<bits<4> op11_8, bits<4> op7_4, string Dt>
           IIC_VST2, "vst2", Dt, "\\{$Vd, $src2\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 class VST2Q<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b00, 0b0011, op7_4, (outs),
@@ -1175,6 +1251,7 @@ class VST2Q<bits<4> op7_4, string Dt>
           "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def  VST2d8   : VST2D<0b1000, {0,0,?,?}, "8">;
@@ -1200,6 +1277,7 @@ class VST2DWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           IIC_VST2u, "vst2", Dt, "\\{$Vd, $src2\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 class VST2QWB<bits<4> op7_4, string Dt>
   : NLdSt<0, 0b00, 0b0011, op7_4, (outs GPR:$wb),
@@ -1208,6 +1286,7 @@ class VST2QWB<bits<4> op7_4, string Dt>
           "vst2", Dt, "\\{$Vd, $src2, $src3, $src4\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def VST2d8_UPD  : VST2DWB<0b1000, {0,0,?,?}, "8">;
@@ -1241,6 +1320,7 @@ class VST3D<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vst3", Dt, "\\{$Vd, $src2, $src3\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def  VST3d8   : VST3D<0b0100, {0,0,0,?}, "8">;
@@ -1259,6 +1339,7 @@ class VST3DWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           "vst3", Dt, "\\{$Vd, $src2, $src3\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def VST3d8_UPD  : VST3DWB<0b0100, {0,0,0,?}, "8">;
@@ -1298,6 +1379,7 @@ class VST4D<bits<4> op11_8, bits<4> op7_4, string Dt>
           "", []> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def  VST4d8   : VST4D<0b0000, {0,0,?,?}, "8">;
@@ -1316,6 +1398,7 @@ class VST4DWB<bits<4> op11_8, bits<4> op7_4, string Dt>
            "vst4", Dt, "\\{$Vd, $src2, $src3, $src4\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{5-4} = Rn{5-4};
+  let DecoderMethod = "DecodeVSTInstruction";
 }
 
 def VST4d8_UPD  : VST4DWB<0b0000, {0,0,?,?}, "8">;
@@ -1381,6 +1464,7 @@ class VST1LN<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
           IIC_VST1ln, "vst1", Dt, "\\{$Vd[$lane]\\}, $Rn", "",
           [(StoreOp (ExtractOp (Ty DPR:$Vd), imm:$lane), addrmode6:$Rn)]> {
   let Rm = 0b1111;
+  let DecoderMethod = "DecodeVST1LN";
 }
 class VST1LN32<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
              PatFrag StoreOp, SDNode ExtractOp>
@@ -1389,6 +1473,7 @@ class VST1LN32<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
           IIC_VST1ln, "vst1", Dt, "\\{$Vd[$lane]\\}, $Rn", "",
           [(StoreOp (ExtractOp (Ty DPR:$Vd), imm:$lane), addrmode6oneL32:$Rn)]>{
   let Rm = 0b1111;
+  let DecoderMethod = "DecodeVST1LN";
 }
 class VST1QLNPseudo<ValueType Ty, PatFrag StoreOp, SDNode ExtractOp>
   : VSTQLNPseudo<IIC_VST1ln> {
@@ -1429,7 +1514,9 @@ class VST1LNWB<bits<4> op11_8, bits<4> op7_4, string Dt, ValueType Ty,
           "\\{$Vd[$lane]\\}, $Rn$Rm",
           "$Rn.addr = $wb",
           [(set GPR:$wb, (StoreOp (ExtractOp (Ty DPR:$Vd), imm:$lane),
-                                  addrmode6:$Rn, am6offset:$Rm))]>;
+                                  addrmode6:$Rn, am6offset:$Rm))]> {
+  let DecoderMethod = "DecodeVST1LN";
+}
 class VST1QLNWBPseudo<ValueType Ty, PatFrag StoreOp, SDNode ExtractOp>
   : VSTQLNWBPseudo<IIC_VST1lnu> {
   let Pattern = [(set GPR:$wb, (StoreOp (ExtractOp (Ty QPR:$src), imm:$lane),
@@ -1465,6 +1552,7 @@ class VST2LN<bits<4> op11_8, bits<4> op7_4, string Dt>
           "", []> {
   let Rm = 0b1111;
   let Inst{4}   = Rn{4};
+  let DecoderMethod = "DecodeVST2LN";
 }
 
 def VST2LNd8  : VST2LN<0b0001, {?,?,?,?}, "8"> {
@@ -1502,6 +1590,7 @@ class VST2LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
           "\\{$src1[$lane], $src2[$lane]\\}, $addr$offset",
           "$addr.addr = $wb", []> {
   let Inst{4}   = Rn{4};
+  let DecoderMethod = "DecodeVST2LN";
 }
 
 def VST2LNd8_UPD  : VST2LNWB<0b0001, {?,?,?,?}, "8"> {
@@ -1535,6 +1624,7 @@ class VST3LN<bits<4> op11_8, bits<4> op7_4, string Dt>
            nohash_imm:$lane), IIC_VST3ln, "vst3", Dt,
           "\\{$Vd[$lane], $src2[$lane], $src3[$lane]\\}, $Rn", "", []> {
   let Rm = 0b1111;
+  let DecoderMethod = "DecodeVST3LN";
 }
 
 def VST3LNd8  : VST3LN<0b0010, {?,?,?,0}, "8"> {
@@ -1569,7 +1659,9 @@ class VST3LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
            DPR:$Vd, DPR:$src2, DPR:$src3, nohash_imm:$lane),
           IIC_VST3lnu, "vst3", Dt,
           "\\{$Vd[$lane], $src2[$lane], $src3[$lane]\\}, $Rn$Rm",
-          "$Rn.addr = $wb", []>;
+          "$Rn.addr = $wb", []> {
+  let DecoderMethod = "DecodeVST3LN";
+}
 
 def VST3LNd8_UPD  : VST3LNWB<0b0010, {?,?,?,0}, "8"> {
   let Inst{7-5} = lane{2-0};
@@ -1604,6 +1696,7 @@ class VST4LN<bits<4> op11_8, bits<4> op7_4, string Dt>
           "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVST4LN";
 }
 
 def VST4LNd8  : VST4LN<0b0011, {?,?,?,?}, "8"> {
@@ -1642,6 +1735,7 @@ class VST4LNWB<bits<4> op11_8, bits<4> op7_4, string Dt>
   "\\{$Vd[$lane], $src2[$lane], $src3[$lane], $src4[$lane]\\}, $Rn$Rm",
           "$Rn.addr = $wb", []> {
   let Inst{4} = Rn{4};
+  let DecoderMethod = "DecodeVST4LN";
 }
 
 def VST4LNd8_UPD  : VST4LNWB<0b0011, {?,?,?,?}, "8"> {
@@ -4039,6 +4133,7 @@ class N2VLShMax<bit op24, bit op23, bits<6> op21_16, bits<4> op11_8, bit op7,
   : N2VLSh<op24, op23, op11_8, op7, op6, op4, OpcodeStr, Dt,
            ResTy, OpTy, OpNode> {
   let Inst{21-16} = op21_16;
+  let DecoderMethod = "DecodeVSHLMaxInstruction";
 }
 def  VSHLLi8  : N2VLShMax<1, 1, 0b110010, 0b0011, 0, 0, 0, "vshll", "i8",
                           v8i16, v8i8, NEONvshlli>;
@@ -4219,16 +4314,6 @@ def : InstAlias<"vmov${p} $Vd, $Vm",
 def : InstAlias<"vmov${p} $Vd, $Vm",
                 (VORRq QPR:$Vd, QPR:$Vm, QPR:$Vm, pred:$p)>;
 
-let neverHasSideEffects = 1 in {
-// Pseudo vector move instructions for QQ and QQQQ registers. This should
-// be expanded after register allocation is completed.
-def  VMOVQQ   : PseudoInst<(outs QQPR:$dst), (ins QQPR:$src),
-                NoItinerary, []>;
-
-def  VMOVQQQQ : PseudoInst<(outs QQQQPR:$dst), (ins QQQQPR:$src),
-                NoItinerary, []>;
-} // neverHasSideEffects
-
 //   VMOV     : Vector Move (Immediate)
 
 let isReMaterializable = 1 in {
@@ -4462,36 +4547,42 @@ def : Pat<(v4f32 (NEONvdup (f32 (bitconvert GPR:$R)))), (VDUP32q GPR:$R)>;
 //   VDUP     : Vector Duplicate Lane (from scalar to all elements)
 
 class VDUPLND<bits<4> op19_16, string OpcodeStr, string Dt,
-              ValueType Ty>
-  : NVDupLane<op19_16, 0, (outs DPR:$Vd), (ins DPR:$Vm, nohash_imm:$lane),
-              IIC_VMOVD, OpcodeStr, Dt, "$Vd, $Vm[$lane]",
+              ValueType Ty, Operand IdxTy>
+  : NVDupLane<op19_16, 0, (outs DPR:$Vd), (ins DPR:$Vm, IdxTy:$lane),
+              IIC_VMOVD, OpcodeStr, Dt, "$Vd, $Vm$lane",
               [(set DPR:$Vd, (Ty (NEONvduplane (Ty DPR:$Vm), imm:$lane)))]>;
 
 class VDUPLNQ<bits<4> op19_16, string OpcodeStr, string Dt,
-              ValueType ResTy, ValueType OpTy>
-  : NVDupLane<op19_16, 1, (outs QPR:$Vd), (ins DPR:$Vm, nohash_imm:$lane),
-              IIC_VMOVQ, OpcodeStr, Dt, "$Vd, $Vm[$lane]",
+              ValueType ResTy, ValueType OpTy, Operand IdxTy>
+  : NVDupLane<op19_16, 1, (outs QPR:$Vd), (ins DPR:$Vm, IdxTy:$lane),
+              IIC_VMOVQ, OpcodeStr, Dt, "$Vd, $Vm$lane",
               [(set QPR:$Vd, (ResTy (NEONvduplane (OpTy DPR:$Vm),
-                                      imm:$lane)))]>;
+                                      VectorIndex32:$lane)))]>;
 
 // Inst{19-16} is partially specified depending on the element size.
 
-def VDUPLN8d  : VDUPLND<{?,?,?,1}, "vdup", "8", v8i8> {
+def VDUPLN8d  : VDUPLND<{?,?,?,1}, "vdup", "8", v8i8, VectorIndex8> {
+  bits<3> lane;
   let Inst{19-17} = lane{2-0};
 }
-def VDUPLN16d : VDUPLND<{?,?,1,0}, "vdup", "16", v4i16> {
+def VDUPLN16d : VDUPLND<{?,?,1,0}, "vdup", "16", v4i16, VectorIndex16> {
+  bits<2> lane;
   let Inst{19-18} = lane{1-0};
 }
-def VDUPLN32d : VDUPLND<{?,1,0,0}, "vdup", "32", v2i32> {
+def VDUPLN32d : VDUPLND<{?,1,0,0}, "vdup", "32", v2i32, VectorIndex32> {
+  bits<1> lane;
   let Inst{19} = lane{0};
 }
-def VDUPLN8q  : VDUPLNQ<{?,?,?,1}, "vdup", "8", v16i8, v8i8> {
+def VDUPLN8q  : VDUPLNQ<{?,?,?,1}, "vdup", "8", v16i8, v8i8, VectorIndex8> {
+  bits<3> lane;
   let Inst{19-17} = lane{2-0};
 }
-def VDUPLN16q : VDUPLNQ<{?,?,1,0}, "vdup", "16", v8i16, v4i16> {
+def VDUPLN16q : VDUPLNQ<{?,?,1,0}, "vdup", "16", v8i16, v4i16, VectorIndex16> {
+  bits<2> lane;
   let Inst{19-18} = lane{1-0};
 }
-def VDUPLN32q : VDUPLNQ<{?,1,0,0}, "vdup", "32", v4i32, v2i32> {
+def VDUPLN32q : VDUPLNQ<{?,1,0,0}, "vdup", "32", v4i32, v2i32, VectorIndex32> {
+  bits<1> lane;
   let Inst{19} = lane{0};
 }
 
@@ -4753,6 +4844,7 @@ def  VZIPq32  : N2VQShuffle<0b10, 0b00011, IIC_VPERMQ3, "vzip", "32">;
 // Vector Table Lookup and Table Extension.
 
 //   VTBL     : Vector Table Lookup
+let DecoderMethod = "DecodeTBLInstruction" in {
 def  VTBL1
   : N3V<1,1,0b11,0b1000,0,0, (outs DPR:$Vd),
         (ins DPR:$Vn, DPR:$Vm), NVTBLFrm, IIC_VTB1,
@@ -4815,6 +4907,7 @@ def  VTBX3Pseudo
 def  VTBX4Pseudo
   : PseudoNeonI<(outs DPR:$dst), (ins DPR:$orig, QQPR:$tbl, DPR:$src),
                 IIC_VTBX4, "$orig = $dst", []>;
+} // DecoderMethod = "DecodeTBLInstruction"
 
 //===----------------------------------------------------------------------===//
 // NEON instructions for single-precision FP math
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrThumb.td b/contrib/llvm/lib/Target/ARM/ARMInstrThumb.td
index bfe83eceb13f..cedb54799db0 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrThumb.td
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrThumb.td
@@ -19,6 +19,19 @@ def ARMtcall : SDNode<"ARMISD::tCALL", SDT_ARMcall,
                       [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                        SDNPVariadic]>;
 
+def imm_sr_XFORM: SDNodeXForm<imm, [{
+  unsigned Imm = N->getZExtValue();
+  return CurDAG->getTargetConstant((Imm == 32 ? 0 : Imm), MVT::i32);
+}]>;
+def ThumbSRImmAsmOperand: AsmOperandClass { let Name = "ImmThumbSR"; }
+def imm_sr : Operand<i32>, PatLeaf<(imm), [{
+  uint64_t Imm = N->getZExtValue();
+  return Imm > 0 && Imm <= 32;
+}], imm_sr_XFORM> {
+  let PrintMethod = "printThumbSRImm";
+  let ParserMatchClass = ThumbSRImmAsmOperand;
+}
+
 def imm_neg_XFORM : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(-(int)N->getZExtValue(), MVT::i32);
 }]>;
@@ -30,10 +43,6 @@ def imm0_7_neg : PatLeaf<(i32 imm), [{
   return (uint32_t)-N->getZExtValue() < 8;
 }], imm_neg_XFORM>;
 
-def imm0_255_asmoperand : AsmOperandClass { let Name = "Imm0_255"; }
-def imm0_255 : Operand<i32>, ImmLeaf<i32, [{ return Imm >= 0 && Imm < 256; }]> {
-  let ParserMatchClass = imm0_255_asmoperand;
-}
 def imm0_255_comp : PatLeaf<(i32 imm), [{
   return ~((uint32_t)N->getZExtValue()) < 256;
 }]>;
@@ -69,8 +78,17 @@ def t_adrlabel : Operand<i32> {
 }
 
 // Scaled 4 immediate.
-def t_imm_s4 : Operand<i32> {
+def t_imm0_1020s4_asmoperand: AsmOperandClass { let Name = "Imm0_1020s4"; }
+def t_imm0_1020s4 : Operand<i32> {
   let PrintMethod = "printThumbS4ImmOperand";
+  let ParserMatchClass = t_imm0_1020s4_asmoperand;
+  let OperandType = "OPERAND_IMMEDIATE";
+}
+
+def t_imm0_508s4_asmoperand: AsmOperandClass { let Name = "Imm0_508s4"; }
+def t_imm0_508s4 : Operand<i32> {
+  let PrintMethod = "printThumbS4ImmOperand";
+  let ParserMatchClass = t_imm0_508s4_asmoperand;
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
@@ -79,113 +97,129 @@ def t_imm_s4 : Operand<i32> {
 let OperandType = "OPERAND_PCREL" in {
 def t_brtarget : Operand<OtherVT> {
   let EncoderMethod = "getThumbBRTargetOpValue";
+  let DecoderMethod = "DecodeThumbBROperand";
 }
 
 def t_bcctarget : Operand<i32> {
   let EncoderMethod = "getThumbBCCTargetOpValue";
+  let DecoderMethod = "DecodeThumbBCCTargetOperand";
 }
 
 def t_cbtarget : Operand<i32> {
   let EncoderMethod = "getThumbCBTargetOpValue";
+  let DecoderMethod = "DecodeThumbCmpBROperand";
 }
 
 def t_bltarget : Operand<i32> {
   let EncoderMethod = "getThumbBLTargetOpValue";
+  let DecoderMethod = "DecodeThumbBLTargetOperand";
 }
 
 def t_blxtarget : Operand<i32> {
   let EncoderMethod = "getThumbBLXTargetOpValue";
+  let DecoderMethod = "DecodeThumbBLXOffset";
 }
 }
 
-def MemModeRegThumbAsmOperand : AsmOperandClass {
-  let Name = "MemModeRegThumb";
-  let SuperClasses = [];
-}
-
-def MemModeImmThumbAsmOperand : AsmOperandClass {
-  let Name = "MemModeImmThumb";
-  let SuperClasses = [];
-}
-
 // t_addrmode_rr := reg + reg
 //
+def t_addrmode_rr_asm_operand : AsmOperandClass { let Name = "MemThumbRR"; }
 def t_addrmode_rr : Operand<i32>,
                     ComplexPattern<i32, 2, "SelectThumbAddrModeRR", []> {
   let EncoderMethod = "getThumbAddrModeRegRegOpValue";
   let PrintMethod = "printThumbAddrModeRROperand";
+  let DecoderMethod = "DecodeThumbAddrModeRR";
+  let ParserMatchClass = t_addrmode_rr_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, tGPR:$offsreg);
 }
 
 // t_addrmode_rrs := reg + reg
 //
+// We use separate scaled versions because the Select* functions need
+// to explicitly check for a matching constant and return false here so that
+// the reg+imm forms will match instead. This is a horrible way to do that,
+// as it forces tight coupling between the methods, but it's how selectiondag
+// currently works.
 def t_addrmode_rrs1 : Operand<i32>,
                       ComplexPattern<i32, 2, "SelectThumbAddrModeRI5S1", []> {
   let EncoderMethod = "getThumbAddrModeRegRegOpValue";
   let PrintMethod = "printThumbAddrModeRROperand";
+  let DecoderMethod = "DecodeThumbAddrModeRR";
+  let ParserMatchClass = t_addrmode_rr_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, tGPR:$offsreg);
-  let ParserMatchClass = MemModeRegThumbAsmOperand;
 }
 def t_addrmode_rrs2 : Operand<i32>,
                       ComplexPattern<i32, 2, "SelectThumbAddrModeRI5S2", []> {
   let EncoderMethod = "getThumbAddrModeRegRegOpValue";
+  let DecoderMethod = "DecodeThumbAddrModeRR";
   let PrintMethod = "printThumbAddrModeRROperand";
+  let ParserMatchClass = t_addrmode_rr_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, tGPR:$offsreg);
-  let ParserMatchClass = MemModeRegThumbAsmOperand;
 }
 def t_addrmode_rrs4 : Operand<i32>,
                       ComplexPattern<i32, 2, "SelectThumbAddrModeRI5S4", []> {
   let EncoderMethod = "getThumbAddrModeRegRegOpValue";
+  let DecoderMethod = "DecodeThumbAddrModeRR";
   let PrintMethod = "printThumbAddrModeRROperand";
+  let ParserMatchClass = t_addrmode_rr_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, tGPR:$offsreg);
-  let ParserMatchClass = MemModeRegThumbAsmOperand;
 }
 
 // t_addrmode_is4 := reg + imm5 * 4
 //
+def t_addrmode_is4_asm_operand : AsmOperandClass { let Name = "MemThumbRIs4"; }
 def t_addrmode_is4 : Operand<i32>,
                      ComplexPattern<i32, 2, "SelectThumbAddrModeImm5S4", []> {
   let EncoderMethod = "getAddrModeISOpValue";
+  let DecoderMethod = "DecodeThumbAddrModeIS";
   let PrintMethod = "printThumbAddrModeImm5S4Operand";
+  let ParserMatchClass = t_addrmode_is4_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemModeImmThumbAsmOperand;
 }
 
 // t_addrmode_is2 := reg + imm5 * 2
 //
+def t_addrmode_is2_asm_operand : AsmOperandClass { let Name = "MemThumbRIs2"; }
 def t_addrmode_is2 : Operand<i32>,
                      ComplexPattern<i32, 2, "SelectThumbAddrModeImm5S2", []> {
   let EncoderMethod = "getAddrModeISOpValue";
+  let DecoderMethod = "DecodeThumbAddrModeIS";
   let PrintMethod = "printThumbAddrModeImm5S2Operand";
+  let ParserMatchClass = t_addrmode_is2_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemModeImmThumbAsmOperand;
 }
 
 // t_addrmode_is1 := reg + imm5
 //
+def t_addrmode_is1_asm_operand : AsmOperandClass { let Name = "MemThumbRIs1"; }
 def t_addrmode_is1 : Operand<i32>,
                      ComplexPattern<i32, 2, "SelectThumbAddrModeImm5S1", []> {
   let EncoderMethod = "getAddrModeISOpValue";
+  let DecoderMethod = "DecodeThumbAddrModeIS";
   let PrintMethod = "printThumbAddrModeImm5S1Operand";
+  let ParserMatchClass = t_addrmode_is1_asm_operand;
   let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemModeImmThumbAsmOperand;
 }
 
 // t_addrmode_sp := sp + imm8 * 4
 //
+// FIXME: This really shouldn't have an explicit SP operand at all. It should
+// be implicit, just like in the instruction encoding itself.
+def t_addrmode_sp_asm_operand : AsmOperandClass { let Name = "MemThumbSPI"; }
 def t_addrmode_sp : Operand<i32>,
                     ComplexPattern<i32, 2, "SelectThumbAddrModeSP", []> {
   let EncoderMethod = "getAddrModeThumbSPOpValue";
+  let DecoderMethod = "DecodeThumbAddrModeSP";
   let PrintMethod = "printThumbAddrModeSPOperand";
+  let ParserMatchClass = t_addrmode_sp_asm_operand;
   let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemModeImmThumbAsmOperand;
 }
 
 // t_addrmode_pc := <label> => pc + imm8 * 4
 //
 def t_addrmode_pc : Operand<i32> {
   let EncoderMethod = "getAddrModePCOpValue";
-  let ParserMatchClass = MemModeImmThumbAsmOperand;
+  let DecoderMethod = "DecodeThumbAddrModePC";
 }
 
 //===----------------------------------------------------------------------===//
@@ -207,68 +241,52 @@ def tADJCALLSTACKDOWN :
             Requires<[IsThumb, IsThumb1Only]>;
 }
 
-// T1Disassembly - A simple class to make encoding some disassembly patterns
-// easier and less verbose.
-class T1Disassembly<bits<2> op1, bits<8> op2>
+class T1SystemEncoding<bits<8> opc>
   : T1Encoding<0b101111> {
-  let Inst{9-8} = op1;
-  let Inst{7-0} = op2;
+  let Inst{9-8} = 0b11;
+  let Inst{7-0} = opc;
 }
 
-def tNOP : T1pI<(outs), (ins), NoItinerary, "nop", "",
-                [/* For disassembly only; pattern left blank */]>,
-           T1Disassembly<0b11, 0x00>; // A8.6.110
+def tNOP : T1pI<(outs), (ins), NoItinerary, "nop", "", []>,
+           T1SystemEncoding<0x00>, // A8.6.110
+        Requires<[IsThumb2]>;
 
-def tYIELD : T1pI<(outs), (ins), NoItinerary, "yield", "",
-                  [/* For disassembly only; pattern left blank */]>,
-           T1Disassembly<0b11, 0x10>; // A8.6.410
+def tYIELD : T1pI<(outs), (ins), NoItinerary, "yield", "", []>,
+           T1SystemEncoding<0x10>; // A8.6.410
 
-def tWFE : T1pI<(outs), (ins), NoItinerary, "wfe", "",
-                [/* For disassembly only; pattern left blank */]>,
-           T1Disassembly<0b11, 0x20>; // A8.6.408
+def tWFE : T1pI<(outs), (ins), NoItinerary, "wfe", "", []>,
+           T1SystemEncoding<0x20>; // A8.6.408
 
-def tWFI : T1pI<(outs), (ins), NoItinerary, "wfi", "",
-                [/* For disassembly only; pattern left blank */]>,
-           T1Disassembly<0b11, 0x30>; // A8.6.409
+def tWFI : T1pI<(outs), (ins), NoItinerary, "wfi", "", []>,
+           T1SystemEncoding<0x30>; // A8.6.409
 
-def tSEV : T1pI<(outs), (ins), NoItinerary, "sev", "",
-                [/* For disassembly only; pattern left blank */]>,
-           T1Disassembly<0b11, 0x40>; // A8.6.157
+def tSEV : T1pI<(outs), (ins), NoItinerary, "sev", "", []>,
+           T1SystemEncoding<0x40>; // A8.6.157
 
-// The i32imm operand $val can be used by a debugger to store more information
+// The imm operand $val can be used by a debugger to store more information
 // about the breakpoint.
-def tBKPT : T1I<(outs), (ins i32imm:$val), NoItinerary, "bkpt\t$val",
-                [/* For disassembly only; pattern left blank */]>,
-           T1Disassembly<0b10, {?,?,?,?,?,?,?,?}> {
+def tBKPT : T1I<(outs), (ins imm0_255:$val), NoItinerary, "bkpt\t$val",
+                []>,
+           T1Encoding<0b101111> {
+  let Inst{9-8} = 0b10;
   // A8.6.22
   bits<8> val;
   let Inst{7-0} = val;
 }
 
-def tSETENDBE : T1I<(outs), (ins), NoItinerary, "setend\tbe",
-                    [/* For disassembly only; pattern left blank */]>,
-                T1Encoding<0b101101> {
+def tSETEND : T1I<(outs), (ins setend_op:$end), NoItinerary, "setend\t$end",
+                  []>, T1Encoding<0b101101> {
+  bits<1> end;
   // A8.6.156
   let Inst{9-5} = 0b10010;
   let Inst{4}   = 1;
-  let Inst{3}   = 1;            // Big-Endian
-  let Inst{2-0} = 0b000;
-}
-
-def tSETENDLE : T1I<(outs), (ins), NoItinerary, "setend\tle",
-                    [/* For disassembly only; pattern left blank */]>,
-                T1Encoding<0b101101> {
-  // A8.6.156
-  let Inst{9-5} = 0b10010;
-  let Inst{4}   = 1;
-  let Inst{3}   = 0;            // Little-Endian
+  let Inst{3}   = end;
   let Inst{2-0} = 0b000;
 }
 
 // Change Processor State is a system instruction -- for disassembly only.
 def tCPS : T1I<(outs), (ins imod_op:$imod, iflags_op:$iflags),
-                NoItinerary, "cps$imod $iflags",
-                [/* For disassembly only; pattern left blank */]>,
+                NoItinerary, "cps$imod $iflags", []>,
            T1Misc<0b0110011> {
   // A8.6.38 & B6.1.1
   bit imod;
@@ -277,6 +295,7 @@ def tCPS : T1I<(outs), (ins imod_op:$imod, iflags_op:$iflags),
   let Inst{4}   = imod;
   let Inst{3}   = 0;
   let Inst{2-0} = iflags;
+  let DecoderMethod = "DecodeThumbCPS";
 }
 
 // For both thumb1 and thumb2.
@@ -290,70 +309,70 @@ def tPICADD : TIt<(outs GPR:$dst), (ins GPR:$lhs, pclabel:$cp), IIC_iALUr, "",
   let Inst{2-0} = dst;
 }
 
-// PC relative add (ADR).
-def tADDrPCi : T1I<(outs tGPR:$dst), (ins t_imm_s4:$rhs), IIC_iALUi,
-                   "add\t$dst, pc, $rhs", []>,
-               T1Encoding<{1,0,1,0,0,?}> {
-  // A6.2 & A8.6.10
-  bits<3> dst;
-  bits<8> rhs;
-  let Inst{10-8} = dst;
-  let Inst{7-0}  = rhs;
-}
-
 // ADD <Rd>, sp, #<imm8>
-// This is rematerializable, which is particularly useful for taking the
-// address of locals.
-let isReMaterializable = 1 in
-def tADDrSPi : T1I<(outs tGPR:$dst), (ins GPR:$sp, t_imm_s4:$rhs), IIC_iALUi,
-                   "add\t$dst, $sp, $rhs", []>,
+// FIXME: This should not be marked as having side effects, and it should be
+// rematerializable. Clearing the side effect bit causes miscompilations,
+// probably because the instruction can be moved around.
+def tADDrSPi : T1pI<(outs tGPR:$dst), (ins GPRsp:$sp, t_imm0_1020s4:$imm),
+                    IIC_iALUi, "add", "\t$dst, $sp, $imm", []>,
                T1Encoding<{1,0,1,0,1,?}> {
   // A6.2 & A8.6.8
   bits<3> dst;
-  bits<8> rhs;
+  bits<8> imm;
   let Inst{10-8} = dst;
-  let Inst{7-0}  = rhs;
+  let Inst{7-0}  = imm;
+  let DecoderMethod = "DecodeThumbAddSpecialReg";
 }
 
 // ADD sp, sp, #<imm7>
-def tADDspi : TIt<(outs GPR:$dst), (ins GPR:$lhs, t_imm_s4:$rhs), IIC_iALUi,
-                  "add\t$dst, $rhs", []>,
+def tADDspi : T1pIt<(outs GPRsp:$Rdn), (ins GPRsp:$Rn, t_imm0_508s4:$imm),
+                     IIC_iALUi, "add", "\t$Rdn, $imm", []>,
               T1Misc<{0,0,0,0,0,?,?}> {
   // A6.2.5 & A8.6.8
-  bits<7> rhs;
-  let Inst{6-0} = rhs;
+  bits<7> imm;
+  let Inst{6-0} = imm;
+  let DecoderMethod = "DecodeThumbAddSPImm";
 }
 
 // SUB sp, sp, #<imm7>
 // FIXME: The encoding and the ASM string don't match up.
-def tSUBspi : TIt<(outs GPR:$dst), (ins GPR:$lhs, t_imm_s4:$rhs), IIC_iALUi,
-                  "sub\t$dst, $rhs", []>,
+def tSUBspi : T1pIt<(outs GPRsp:$Rdn), (ins GPRsp:$Rn, t_imm0_508s4:$imm),
+                    IIC_iALUi, "sub", "\t$Rdn, $imm", []>,
               T1Misc<{0,0,0,0,1,?,?}> {
   // A6.2.5 & A8.6.214
-  bits<7> rhs;
-  let Inst{6-0} = rhs;
+  bits<7> imm;
+  let Inst{6-0} = imm;
+  let DecoderMethod = "DecodeThumbAddSPImm";
 }
 
+// Can optionally specify SP as a three operand instruction.
+def : tInstAlias<"add${p} sp, sp, $imm",
+                 (tADDspi SP, t_imm0_508s4:$imm, pred:$p)>;
+def : tInstAlias<"sub${p} sp, sp, $imm",
+                 (tSUBspi SP, t_imm0_508s4:$imm, pred:$p)>;
+
 // ADD <Rm>, sp
-def tADDrSP : TIt<(outs GPR:$dst), (ins GPR:$lhs, GPR:$rhs), IIC_iALUr,
-                  "add\t$dst, $rhs", []>,
+def tADDrSP : T1pIt<(outs GPR:$Rdn), (ins GPR:$Rn, GPRsp:$sp), IIC_iALUr,
+                  "add", "\t$Rdn, $sp, $Rn", []>,
               T1Special<{0,0,?,?}> {
   // A8.6.9 Encoding T1
-  bits<4> dst;
-  let Inst{7}   = dst{3};
+  bits<4> Rdn;
+  let Inst{7}   = Rdn{3};
   let Inst{6-3} = 0b1101;
-  let Inst{2-0} = dst{2-0};
+  let Inst{2-0} = Rdn{2-0};
+  let DecoderMethod = "DecodeThumbAddSPReg";
 }
 
 // ADD sp, <Rm>
-def tADDspr : TIt<(outs GPR:$dst), (ins GPR:$lhs, GPR:$rhs), IIC_iALUr,
-                  "add\t$dst, $rhs", []>,
+def tADDspr : T1pIt<(outs GPRsp:$Rdn), (ins GPRsp:$Rn, GPR:$Rm), IIC_iALUr,
+                  "add", "\t$Rdn, $Rm", []>,
               T1Special<{0,0,?,?}> {
   // A8.6.9 Encoding T2
-  bits<4> dst;
+  bits<4> Rm;
   let Inst{7} = 1;
-  let Inst{6-3} = dst;
+  let Inst{6-3} = Rm;
   let Inst{2-0} = 0b101;
+  let DecoderMethod = "DecodeThumbAddSPReg";
 }
 
 //===----------------------------------------------------------------------===//
@@ -390,11 +409,12 @@ let isCall = 1,
   Uses = [SP] in {
   // Also used for Thumb2
   def tBL  : TIx2<0b11110, 0b11, 1,
-                  (outs), (ins t_bltarget:$func, variable_ops), IIC_Br,
-                  "bl\t$func",
+                  (outs), (ins pred:$p, t_bltarget:$func, variable_ops), IIC_Br,
+                  "bl${p}\t$func",
                   [(ARMtcall tglobaladdr:$func)]>,
              Requires<[IsThumb, IsNotDarwin]> {
-    bits<21> func;
+    bits<22> func;
+    let Inst{26} = func{21};
     let Inst{25-16} = func{20-11};
     let Inst{13} = 1;
     let Inst{11} = 1;
@@ -403,8 +423,8 @@ let isCall = 1,
 
   // ARMv5T and above, also used for Thumb2
   def tBLXi : TIx2<0b11110, 0b11, 0,
-                   (outs), (ins t_blxtarget:$func, variable_ops), IIC_Br,
-                   "blx\t$func",
+                 (outs), (ins pred:$p, t_blxtarget:$func, variable_ops), IIC_Br,
+                   "blx${p}\t$func",
                    [(ARMcall tglobaladdr:$func)]>,
               Requires<[IsThumb, HasV5T, IsNotDarwin]> {
     bits<21> func;
@@ -416,8 +436,8 @@ let isCall = 1,
   }
 
   // Also used for Thumb2
-  def tBLXr : TI<(outs), (ins GPR:$func, variable_ops), IIC_Br,
-                  "blx\t$func",
+  def tBLXr : TI<(outs), (ins pred:$p, GPR:$func, variable_ops), IIC_Br,
+                  "blx${p}\t$func",
                   [(ARMtcall GPR:$func)]>,
               Requires<[IsThumb, HasV5T, IsNotDarwin]>,
               T1Special<{1,1,1,?}> { // A6.2.3 & A8.6.24;
@@ -440,43 +460,22 @@ let isCall = 1,
   Defs = [R0,  R1,  R2,  R3,  R9,  R12, LR, QQQQ0, QQQQ2, QQQQ3, CPSR, FPSCR],
   Uses = [R7, SP] in {
   // Also used for Thumb2
-  def tBLr9 : TIx2<0b11110, 0b11, 1,
-                   (outs), (ins pred:$p, t_bltarget:$func, variable_ops),
-                   IIC_Br, "bl${p}\t$func",
-                   [(ARMtcall tglobaladdr:$func)]>,
-              Requires<[IsThumb, IsDarwin]> {
-    bits<21> func;
-    let Inst{25-16} = func{20-11};
-    let Inst{13} = 1;
-    let Inst{11} = 1;
-    let Inst{10-0} = func{10-0};
-  }
+  def tBLr9 : tPseudoExpand<(outs), (ins pred:$p, t_bltarget:$func, variable_ops),
+                          4, IIC_Br, [(ARMtcall tglobaladdr:$func)],
+                          (tBL pred:$p, t_bltarget:$func)>,
+              Requires<[IsThumb, IsDarwin]>;
 
   // ARMv5T and above, also used for Thumb2
-  def tBLXi_r9 : TIx2<0b11110, 0b11, 0,
-                      (outs), (ins pred:$p, t_blxtarget:$func, variable_ops),
-                      IIC_Br, "blx${p}\t$func",
-                      [(ARMcall tglobaladdr:$func)]>,
-                 Requires<[IsThumb, HasV5T, IsDarwin]> {
-    bits<21> func;
-    let Inst{25-16} = func{20-11};
-    let Inst{13} = 1;
-    let Inst{11} = 1;
-    let Inst{10-1} = func{10-1};
-    let Inst{0} = 0; // func{0} is assumed zero
-  }
+  def tBLXi_r9 : tPseudoExpand<(outs), (ins pred:$p, t_blxtarget:$func, variable_ops),
+                      4, IIC_Br, [(ARMcall tglobaladdr:$func)],
+                      (tBLXi pred:$p, t_blxtarget:$func)>,
+                 Requires<[IsThumb, HasV5T, IsDarwin]>;
 
   // Also used for Thumb2
-  def tBLXr_r9 : TI<(outs), (ins pred:$p, GPR:$func, variable_ops), IIC_Br,
-                    "blx${p}\t$func",
-                    [(ARMtcall GPR:$func)]>,
-                 Requires<[IsThumb, HasV5T, IsDarwin]>,
-                 T1Special<{1,1,1,?}> {
-    // A6.2.3 & A8.6.24
-    bits<4> func;
-    let Inst{6-3} = func;
-    let Inst{2-0} = 0b000;
-  }
+  def tBLXr_r9 : tPseudoExpand<(outs), (ins pred:$p, GPR:$func, variable_ops),
+                    2, IIC_Br, [(ARMtcall GPR:$func)],
+                    (tBLXr pred:$p, GPR:$func)>,
+                 Requires<[IsThumb, HasV5T, IsDarwin]>;
 
   // ARMv4T
   def tBXr9_CALL : tPseudoInst<(outs), (ins tGPR:$func, variable_ops),
@@ -487,8 +486,8 @@ let isCall = 1,
 
 let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
   let isPredicable = 1 in
-  def tB   : T1I<(outs), (ins t_brtarget:$target), IIC_Br,
-                 "b\t$target", [(br bb:$target)]>,
+  def tB   : T1pI<(outs), (ins t_brtarget:$target), IIC_Br,
+                 "b", "\t$target", [(br bb:$target)]>,
              T1Encoding<{1,1,1,0,0,?}> {
     bits<11> target;
     let Inst{10-0} = target;
@@ -498,8 +497,8 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
   // Just a pseudo for a tBL instruction. Needed to let regalloc know about
   // the clobber of LR.
   let Defs = [LR] in
-  def tBfar : tPseudoExpand<(outs), (ins t_bltarget:$target),
-                          4, IIC_Br, [], (tBL t_bltarget:$target)>;
+  def tBfar : tPseudoExpand<(outs), (ins t_bltarget:$target, pred:$p),
+                          4, IIC_Br, [], (tBL pred:$p, t_bltarget:$target)>;
 
   def tBR_JTr : tPseudoInst<(outs),
                       (ins tGPR:$target, i32imm:$jt, i32imm:$id),
@@ -522,31 +521,6 @@ let isBranch = 1, isTerminator = 1 in
   let Inst{7-0} = target;
 }
 
-// Compare and branch on zero / non-zero
-let isBranch = 1, isTerminator = 1 in {
-  def tCBZ  : T1I<(outs), (ins tGPR:$Rn, t_cbtarget:$target), IIC_Br,
-                  "cbz\t$Rn, $target", []>,
-              T1Misc<{0,0,?,1,?,?,?}> {
-    // A8.6.27
-    bits<6> target;
-    bits<3> Rn;
-    let Inst{9}   = target{5};
-    let Inst{7-3} = target{4-0};
-    let Inst{2-0} = Rn;
-  }
-
-  def tCBNZ : T1I<(outs), (ins tGPR:$cmp, t_cbtarget:$target), IIC_Br,
-                  "cbnz\t$cmp, $target", []>,
-              T1Misc<{1,0,?,1,?,?,?}> {
-    // A8.6.27
-    bits<6> target;
-    bits<3> Rn;
-    let Inst{9}   = target{5};
-    let Inst{7-3} = target{4-0};
-    let Inst{2-0} = Rn;
-  }
-}
-
 // Tail calls
 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1 in {
   // Darwin versions.
@@ -562,9 +536,10 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1 in {
   // Non-Darwin versions (the difference is R9).
   let Defs = [R0, R1, R2, R3, R12, QQQQ0, QQQQ2, QQQQ3, PC],
       Uses = [SP] in {
-    def tTAILJMPdND : tPseudoExpand<(outs), (ins t_brtarget:$dst, variable_ops),
+    def tTAILJMPdND : tPseudoExpand<(outs),
+                   (ins t_brtarget:$dst, pred:$p, variable_ops),
                    4, IIC_Br, [],
-                   (tB t_brtarget:$dst)>,
+                   (tB t_brtarget:$dst, pred:$p)>,
                  Requires<[IsThumb, IsNotDarwin]>;
     def tTAILJMPrND : tPseudoExpand<(outs), (ins tcGPR:$dst, variable_ops),
                      4, IIC_Br, [],
@@ -574,11 +549,11 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1 in {
 }
 
 
-// A8.6.218 Supervisor Call (Software Interrupt) -- for disassembly only
+// A8.6.218 Supervisor Call (Software Interrupt)
 // A8.6.16 B: Encoding T1
 // If Inst{11-8} == 0b1111 then SEE SVC
 let isCall = 1, Uses = [SP] in
-def tSVC : T1pI<(outs), (ins i32imm:$imm), IIC_Br,
+def tSVC : T1pI<(outs), (ins imm0_255:$imm), IIC_Br,
                 "svc", "\t$imm", []>, Encoding16 {
   bits<8> imm;
   let Inst{15-12} = 0b1101;
@@ -653,17 +628,17 @@ defm tLDRH : thumb_ld_rr_ri_enc<0b101, 0b1000, t_addrmode_rrs2,
 
 let AddedComplexity = 10 in
 def tLDRSB :                    // A8.6.80
-  T1pILdStEncode<0b011, (outs tGPR:$dst), (ins t_addrmode_rr:$addr),
+  T1pILdStEncode<0b011, (outs tGPR:$Rt), (ins t_addrmode_rr:$addr),
                  AddrModeT1_1, IIC_iLoad_bh_r,
-                 "ldrsb", "\t$dst, $addr",
-                 [(set tGPR:$dst, (sextloadi8 t_addrmode_rr:$addr))]>;
+                 "ldrsb", "\t$Rt, $addr",
+                 [(set tGPR:$Rt, (sextloadi8 t_addrmode_rr:$addr))]>;
 
 let AddedComplexity = 10 in
 def tLDRSH :                    // A8.6.84
-  T1pILdStEncode<0b111, (outs tGPR:$dst), (ins t_addrmode_rr:$addr),
+  T1pILdStEncode<0b111, (outs tGPR:$Rt), (ins t_addrmode_rr:$addr),
                  AddrModeT1_2, IIC_iLoad_bh_r,
-                 "ldrsh", "\t$dst, $addr",
-                 [(set tGPR:$dst, (sextloadi16 t_addrmode_rr:$addr))]>;
+                 "ldrsh", "\t$Rt, $addr",
+                 [(set tGPR:$Rt, (sextloadi16 t_addrmode_rr:$addr))]>;
 
 let canFoldAsLoad = 1 in
 def tLDRspi : T1pIs<(outs tGPR:$Rt), (ins t_addrmode_sp:$addr), IIC_iLoad_i,
@@ -678,7 +653,7 @@ def tLDRspi : T1pIs<(outs tGPR:$Rt), (ins t_addrmode_sp:$addr), IIC_iLoad_i,
 
 // Load tconstpool
 // FIXME: Use ldr.n to work around a Darwin assembler bug.
-let canFoldAsLoad = 1, isReMaterializable = 1 in
+let canFoldAsLoad = 1, isReMaterializable = 1, isCodeGenOnly = 1 in
 def tLDRpci : T1pIs<(outs tGPR:$Rt), (ins t_addrmode_pc:$addr), IIC_iLoad_i,
                   "ldr", ".n\t$Rt, $addr",
                   [(set tGPR:$Rt, (load (ARMWrapper tconstpool:$addr)))]>,
@@ -736,42 +711,53 @@ def tSTRspi : T1pIs<(outs), (ins tGPR:$Rt, t_addrmode_sp:$addr), IIC_iStore_i,
 //  Load / store multiple Instructions.
 //
 
-multiclass thumb_ldst_mult<string asm, InstrItinClass itin,
-                           InstrItinClass itin_upd, bits<6> T1Enc,
-                           bit L_bit> {
-  def IA :
-    T1I<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
-        itin, !strconcat(asm, "ia${p}\t$Rn, $regs"), []>,
-       T1Encoding<T1Enc> {
-    bits<3> Rn;
-    bits<8> regs;
-    let Inst{10-8} = Rn;
-    let Inst{7-0}  = regs;
-  }
-  def IA_UPD :
-    T1It<(outs GPR:$wb), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
-         itin_upd, !strconcat(asm, "ia${p}\t$Rn!, $regs"), "$Rn = $wb", []>,
-        T1Encoding<T1Enc> {
-    bits<3> Rn;
-    bits<8> regs;
-    let Inst{10-8} = Rn;
-    let Inst{7-0}  = regs;
-  }
-}
-
 // These require base address to be written back or one of the loaded regs.
 let neverHasSideEffects = 1 in {
 
 let mayLoad = 1, hasExtraDefRegAllocReq = 1 in
-defm tLDM : thumb_ldst_mult<"ldm", IIC_iLoad_m, IIC_iLoad_mu,
-                            {1,1,0,0,1,?}, 1>;
+def tLDMIA : T1I<(outs), (ins tGPR:$Rn, pred:$p, reglist:$regs, variable_ops),
+        IIC_iLoad_m, "ldm${p}\t$Rn, $regs", []>, T1Encoding<{1,1,0,0,1,?}> {
+  bits<3> Rn;
+  bits<8> regs;
+  let Inst{10-8} = Rn;
+  let Inst{7-0}  = regs;
+}
 
+// Writeback version is just a pseudo, as there's no encoding difference.
+// Writeback happens iff the base register is not in the destination register
+// list.
+def tLDMIA_UPD :
+    InstTemplate<AddrModeNone, 0, IndexModeNone, Pseudo, GenericDomain,
+                 "$Rn = $wb", IIC_iLoad_mu>,
+    PseudoInstExpansion<(tLDMIA tGPR:$Rn, pred:$p, reglist:$regs)> {
+  let Size = 2;
+  let OutOperandList = (outs GPR:$wb);
+  let InOperandList = (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops);
+  let Pattern = [];
+  let isCodeGenOnly = 1;
+  let isPseudo = 1;
+  list<Predicate> Predicates = [IsThumb];
+}
+
+// There is no non-writeback version of STM for Thumb.
 let mayStore = 1, hasExtraSrcRegAllocReq = 1 in
-defm tSTM : thumb_ldst_mult<"stm", IIC_iStore_m, IIC_iStore_mu,
-                            {1,1,0,0,0,?}, 0>;
+def tSTMIA_UPD : Thumb1I<(outs GPR:$wb),
+                         (ins tGPR:$Rn, pred:$p, reglist:$regs, variable_ops),
+                         AddrModeNone, 2, IIC_iStore_mu,
+                         "stm${p}\t$Rn!, $regs", "$Rn = $wb", []>,
+                     T1Encoding<{1,1,0,0,0,?}> {
+  bits<3> Rn;
+  bits<8> regs;
+  let Inst{10-8} = Rn;
+  let Inst{7-0}  = regs;
+}
 
 } // neverHasSideEffects
 
+def : InstAlias<"ldm${p} $Rn!, $regs",
+                (tLDMIA tGPR:$Rn, pred:$p, reglist:$regs)>,
+        Requires<[IsThumb, IsThumb1Only]>;
+
 let mayLoad = 1, Uses = [SP], Defs = [SP], hasExtraDefRegAllocReq = 1 in
 def tPOP : T1I<(outs), (ins pred:$p, reglist:$regs, variable_ops),
                IIC_iPop,
@@ -876,7 +862,7 @@ def tADC :                      // A8.6.2
 
 // Add immediate
 def tADDi3 :                    // A8.6.4 T1
-  T1sIGenEncodeImm<0b01110, (outs tGPR:$Rd), (ins tGPR:$Rm, i32imm:$imm3),
+  T1sIGenEncodeImm<0b01110, (outs tGPR:$Rd), (ins tGPR:$Rm, imm0_7:$imm3),
                    IIC_iALUi,
                    "add", "\t$Rd, $Rm, $imm3",
                    [(set tGPR:$Rd, (add tGPR:$Rm, imm0_7:$imm3))]> {
@@ -885,8 +871,8 @@ def tADDi3 :                    // A8.6.4 T1
 }
 
 def tADDi8 :                    // A8.6.4 T2
-  T1sItGenEncodeImm<{1,1,0,?,?}, (outs tGPR:$Rdn), (ins tGPR:$Rn, i32imm:$imm8),
-                    IIC_iALUi,
+  T1sItGenEncodeImm<{1,1,0,?,?}, (outs tGPR:$Rdn),
+                    (ins tGPR:$Rn, imm0_255:$imm8), IIC_iALUi,
                     "add", "\t$Rdn, $imm8",
                     [(set tGPR:$Rdn, (add tGPR:$Rn, imm8_255:$imm8))]>;
 
@@ -920,10 +906,10 @@ def tAND :                      // A8.6.12
 
 // ASR immediate
 def tASRri :                    // A8.6.14
-  T1sIGenEncodeImm<{0,1,0,?,?}, (outs tGPR:$Rd), (ins tGPR:$Rm, i32imm:$imm5),
+  T1sIGenEncodeImm<{0,1,0,?,?}, (outs tGPR:$Rd), (ins tGPR:$Rm, imm_sr:$imm5),
                    IIC_iMOVsi,
                    "asr", "\t$Rd, $Rm, $imm5",
-                   [(set tGPR:$Rd, (sra tGPR:$Rm, (i32 imm:$imm5)))]> {
+                   [(set tGPR:$Rd, (sra tGPR:$Rm, (i32 imm_sr:$imm5)))]> {
   bits<5> imm5;
   let Inst{10-6} = imm5;
 }
@@ -962,7 +948,7 @@ def tCMNz :                     // A8.6.33
 
 // CMP immediate
 let isCompare = 1, Defs = [CPSR] in {
-def tCMPi8 : T1pI<(outs), (ins tGPR:$Rn, i32imm:$imm8), IIC_iCMPi,
+def tCMPi8 : T1pI<(outs), (ins tGPR:$Rn, imm0_255:$imm8), IIC_iCMPi,
                   "cmp", "\t$Rn, $imm8",
                   [(ARMcmp tGPR:$Rn, imm0_255:$imm8)]>,
              T1General<{1,0,1,?,?}> {
@@ -1003,7 +989,7 @@ def tEOR :                      // A8.6.45
 
 // LSL immediate
 def tLSLri :                    // A8.6.88
-  T1sIGenEncodeImm<{0,0,0,?,?}, (outs tGPR:$Rd), (ins tGPR:$Rm, i32imm:$imm5),
+  T1sIGenEncodeImm<{0,0,0,?,?}, (outs tGPR:$Rd), (ins tGPR:$Rm, imm0_31:$imm5),
                    IIC_iMOVsi,
                    "lsl", "\t$Rd, $Rm, $imm5",
                    [(set tGPR:$Rd, (shl tGPR:$Rm, (i32 imm:$imm5)))]> {
@@ -1020,10 +1006,10 @@ def tLSLrr :                    // A8.6.89
 
 // LSR immediate
 def tLSRri :                    // A8.6.90
-  T1sIGenEncodeImm<{0,0,1,?,?}, (outs tGPR:$Rd), (ins tGPR:$Rm, i32imm:$imm5),
+  T1sIGenEncodeImm<{0,0,1,?,?}, (outs tGPR:$Rd), (ins tGPR:$Rm, imm_sr:$imm5),
                    IIC_iMOVsi,
                    "lsr", "\t$Rd, $Rm, $imm5",
-                   [(set tGPR:$Rd, (srl tGPR:$Rm, (i32 imm:$imm5)))]> {
+                   [(set tGPR:$Rd, (srl tGPR:$Rm, (i32 imm_sr:$imm5)))]> {
   bits<5> imm5;
   let Inst{10-6} = imm5;
 }
@@ -1047,6 +1033,10 @@ def tMOVi8 : T1sI<(outs tGPR:$Rd), (ins imm0_255:$imm8), IIC_iMOVi,
   let Inst{10-8} = Rd;
   let Inst{7-0}  = imm8;
 }
+// Because we have an explicit tMOVSr below, we need an alias to handle
+// the immediate "movs" form here. Blech.
+def : tInstAlias <"movs $Rdn, $imm",
+                 (tMOVi8 tGPR:$Rdn, CPSR, imm0_255:$imm, 14, 0)>;
 
 // A7-73: MOV(2) - mov setting flag.
 
@@ -1077,10 +1067,19 @@ def tMOVSr      : T1I<(outs tGPR:$Rd), (ins tGPR:$Rm), IIC_iMOVr,
 // Multiply register
 let isCommutable = 1 in
 def tMUL :                      // A8.6.105 T1
-  T1sItDPEncode<0b1101, (outs tGPR:$Rdn), (ins tGPR:$Rn, tGPR:$Rm),
-                IIC_iMUL32,
-                "mul", "\t$Rdn, $Rm, $Rdn",
-                [(set tGPR:$Rdn, (mul tGPR:$Rn, tGPR:$Rm))]>;
+  Thumb1sI<(outs tGPR:$Rd), (ins tGPR:$Rn, tGPR:$Rm), AddrModeNone, 2,
+           IIC_iMUL32, "mul", "\t$Rd, $Rn, $Rm", "$Rm = $Rd",
+           [(set tGPR:$Rd, (mul tGPR:$Rn, tGPR:$Rm))]>,
+      T1DataProcessing<0b1101> {
+  bits<3> Rd;
+  bits<3> Rn;
+  let Inst{5-3} = Rn;
+  let Inst{2-0} = Rd;
+  let AsmMatchConverter = "cvtThumbMultiply";
+}
+
+def :tInstAlias<"mul${s}${p} $Rdm, $Rn", (tMUL tGPR:$Rdm, s_cc_out:$s, tGPR:$Rn,
+                                               pred:$p)>;
 
 // Move inverse register
 def tMVN :                      // A8.6.107
@@ -1132,6 +1131,9 @@ def tRSB :                      // A8.6.141
                "rsb", "\t$Rd, $Rn, #0",
                [(set tGPR:$Rd, (ineg tGPR:$Rn))]>;
 
+def : tInstAlias<"neg${s}${p} $Rd, $Rm",
+                 (tRSB tGPR:$Rd, s_cc_out:$s, tGPR:$Rm, pred:$p)>;
+
 // Subtract with carry register
 let Uses = [CPSR] in
 def tSBC :                      // A8.6.151
@@ -1142,7 +1144,7 @@ def tSBC :                      // A8.6.151
 
 // Subtract immediate
 def tSUBi3 :                    // A8.6.210 T1
-  T1sIGenEncodeImm<0b01111, (outs tGPR:$Rd), (ins tGPR:$Rm, i32imm:$imm3),
+  T1sIGenEncodeImm<0b01111, (outs tGPR:$Rd), (ins tGPR:$Rm, imm0_7:$imm3),
                    IIC_iALUi,
                    "sub", "\t$Rd, $Rm, $imm3",
                    [(set tGPR:$Rd, (add tGPR:$Rm, imm0_7_neg:$imm3))]> {
@@ -1151,8 +1153,8 @@ def tSUBi3 :                    // A8.6.210 T1
 }
 
 def tSUBi8 :                    // A8.6.210 T2
-  T1sItGenEncodeImm<{1,1,1,?,?}, (outs tGPR:$Rdn), (ins tGPR:$Rn, i32imm:$imm8),
-                    IIC_iALUi,
+  T1sItGenEncodeImm<{1,1,1,?,?}, (outs tGPR:$Rdn),
+                    (ins tGPR:$Rn, imm0_255:$imm8), IIC_iALUi,
                     "sub", "\t$Rdn, $imm8",
                     [(set tGPR:$Rdn, (add tGPR:$Rn, imm8_255_neg:$imm8))]>;
 
@@ -1163,8 +1165,6 @@ def tSUBrr :                    // A8.6.212
                 "sub", "\t$Rd, $Rn, $Rm",
                 [(set tGPR:$Rd, (sub tGPR:$Rn, tGPR:$Rm))]>;
 
-// TODO: A7-96: STMIA - store multiple.
-
 // Sign-extend byte
 def tSXTB :                     // A8.6.222
   T1pIMiscEncode<{0,0,1,0,0,1,?}, (outs tGPR:$Rd), (ins tGPR:$Rm),
@@ -1216,12 +1216,13 @@ let usesCustomInserter = 1 in  // Expanded after instruction selection.
 // assembler.
 
 def tADR : T1I<(outs tGPR:$Rd), (ins t_adrlabel:$addr, pred:$p),
-               IIC_iALUi, "adr{$p}\t$Rd, #$addr", []>,
+               IIC_iALUi, "adr{$p}\t$Rd, $addr", []>,
                T1Encoding<{1,0,1,0,0,?}> {
   bits<3> Rd;
   bits<8> addr;
   let Inst{10-8} = Rd;
   let Inst{7-0} = addr;
+  let DecoderMethod = "DecodeThumbAddSpecialReg";
 }
 
 let neverHasSideEffects = 1, isReMaterializable = 1 in
@@ -1361,6 +1362,31 @@ def : T1Pat<(sextloadi16 t_addrmode_rrs2:$addr),
 def : T1Pat<(sextloadi16 t_addrmode_is2:$addr),
             (tASRri (tLSLri (tLDRHi t_addrmode_is2:$addr), 16), 16)>;
 
+def : T1Pat<(atomic_load_8 t_addrmode_is1:$src),
+             (tLDRBi t_addrmode_is1:$src)>;
+def : T1Pat<(atomic_load_8 t_addrmode_rrs1:$src),
+             (tLDRBr t_addrmode_rrs1:$src)>;
+def : T1Pat<(atomic_load_16 t_addrmode_is2:$src),
+             (tLDRHi t_addrmode_is2:$src)>;
+def : T1Pat<(atomic_load_16 t_addrmode_rrs2:$src),
+             (tLDRHr t_addrmode_rrs2:$src)>;
+def : T1Pat<(atomic_load_32 t_addrmode_is4:$src),
+             (tLDRi t_addrmode_is4:$src)>;
+def : T1Pat<(atomic_load_32 t_addrmode_rrs4:$src),
+             (tLDRr t_addrmode_rrs4:$src)>;
+def : T1Pat<(atomic_store_8 t_addrmode_is1:$ptr, tGPR:$val),
+             (tSTRBi tGPR:$val, t_addrmode_is1:$ptr)>;
+def : T1Pat<(atomic_store_8 t_addrmode_rrs1:$ptr, tGPR:$val),
+             (tSTRBr tGPR:$val, t_addrmode_rrs1:$ptr)>;
+def : T1Pat<(atomic_store_16 t_addrmode_is2:$ptr, tGPR:$val),
+             (tSTRHi tGPR:$val, t_addrmode_is2:$ptr)>;
+def : T1Pat<(atomic_store_16 t_addrmode_rrs2:$ptr, tGPR:$val),
+             (tSTRHr tGPR:$val, t_addrmode_rrs2:$ptr)>;
+def : T1Pat<(atomic_store_32 t_addrmode_is4:$ptr, tGPR:$val),
+             (tSTRi tGPR:$val, t_addrmode_is4:$ptr)>;
+def : T1Pat<(atomic_store_32 t_addrmode_rrs4:$ptr, tGPR:$val),
+             (tSTRr tGPR:$val, t_addrmode_rrs4:$ptr)>;
+
 // Large immediate handling.
 
 // Two piece imms.
@@ -1395,3 +1421,16 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
                   2, IIC_Br, [(brind GPR:$Rm)],
                   (tMOVr PC, GPR:$Rm, pred:$p)>;
 }
+
+
+// In Thumb1, "nop" is encoded as a "mov r8, r8". Technically, the bf00
+// encoding is available on ARMv6K, but we don't differentiate that finely.
+def : InstAlias<"nop", (tMOVr R8, R8, 14, 0)>,Requires<[IsThumb, IsThumb1Only]>;
+
+
+// For round-trip assembly/disassembly, we have to handle a CPS instruction
+// without any iflags. That's not, strictly speaking, valid syntax, but it's
+// a useful extention and assembles to defined behaviour (the insn does
+// nothing).
+def : tInstAlias<"cps$imod", (tCPS imod_op:$imod, 0)>;
+def : tInstAlias<"cps$imod", (tCPS imod_op:$imod, 0)>;
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td b/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td
index c2c6cbcac0f5..471ec2967487 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -12,13 +12,32 @@
 //===----------------------------------------------------------------------===//
 
 // IT block predicate field
+def it_pred_asmoperand : AsmOperandClass {
+  let Name = "ITCondCode";
+  let ParserMethod = "parseITCondCode";
+}
 def it_pred : Operand<i32> {
   let PrintMethod = "printMandatoryPredicateOperand";
+  let ParserMatchClass = it_pred_asmoperand;
 }
 
 // IT block condition mask
+def it_mask_asmoperand : AsmOperandClass { let Name = "ITMask"; }
 def it_mask : Operand<i32> {
   let PrintMethod = "printThumbITMask";
+  let ParserMatchClass = it_mask_asmoperand;
+}
+
+// t2_shift_imm: An integer that encodes a shift amount and the type of shift
+// (asr or lsl). The 6-bit immediate encodes as:
+//    {5}     0 ==> lsl
+//            1     asr
+//    {4-0}   imm5 shift amount.
+//            asr #32 not allowed
+def t2_shift_imm : Operand<i32> {
+  let PrintMethod = "printShiftImmOperand";
+  let ParserMatchClass = ShifterImmAsmOperand;
+  let DecoderMethod = "DecodeT2ShifterImmOperand";
 }
 
 // Shifted operands. No register controlled shifts for Thumb2.
@@ -28,6 +47,8 @@ def t2_so_reg : Operand<i32>,    // reg imm
                                [shl,srl,sra,rotr]> {
   let EncoderMethod = "getT2SORegOpValue";
   let PrintMethod = "printT2SOOperand";
+  let DecoderMethod = "DecodeSORegImmOperand";
+  let ParserMatchClass = ShiftedImmAsmOperand;
   let MIOperandInfo = (ops rGPR, i32imm);
 }
 
@@ -50,6 +71,7 @@ def t2_so_imm : Operand<i32>, ImmLeaf<i32, [{
   }]> {
   let ParserMatchClass = t2_so_imm_asmoperand;
   let EncoderMethod = "getT2SOImmOpValue";
+  let DecoderMethod = "DecodeT2SOImm";
 }
 
 // t2_so_imm_not - Match an immediate that is a complement
@@ -65,11 +87,6 @@ def t2_so_imm_neg : Operand<i32>,
   return ARM_AM::getT2SOImmVal(-((uint32_t)N->getZExtValue())) != -1;
 }], t2_so_imm_neg_XFORM>;
 
-/// imm1_31 predicate - True if the 32-bit immediate is in the range [1,31].
-def imm1_31 : ImmLeaf<i32, [{
-  return (int32_t)Imm >= 1 && (int32_t)Imm < 32;
-}]>;
-
 /// imm0_4095 predicate - True if the 32-bit immediate is in the range [0.4095].
 def imm0_4095 : Operand<i32>,
                 ImmLeaf<i32, [{
@@ -96,17 +113,20 @@ def lo5AllOne : PatLeaf<(i32 imm), [{
 // Define Thumb2 specific addressing modes.
 
 // t2addrmode_imm12  := reg + imm12
+def t2addrmode_imm12_asmoperand : AsmOperandClass {let Name="MemUImm12Offset";}
 def t2addrmode_imm12 : Operand<i32>,
                        ComplexPattern<i32, 2, "SelectT2AddrModeImm12", []> {
   let PrintMethod = "printAddrModeImm12Operand";
   let EncoderMethod = "getAddrModeImm12OpValue";
+  let DecoderMethod = "DecodeT2AddrModeImm12";
+  let ParserMatchClass = t2addrmode_imm12_asmoperand;
   let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemMode5AsmOperand;
 }
 
 // t2ldrlabel  := imm12
 def t2ldrlabel : Operand<i32> {
   let EncoderMethod = "getAddrModeImm12OpValue";
+  let PrintMethod = "printT2LdrLabelOperand";
 }
 
 
@@ -116,13 +136,36 @@ def t2adrlabel : Operand<i32> {
 }
 
 
+// t2addrmode_posimm8  := reg + imm8
+def MemPosImm8OffsetAsmOperand : AsmOperandClass {let Name="MemPosImm8Offset";}
+def t2addrmode_posimm8 : Operand<i32> {
+  let PrintMethod = "printT2AddrModeImm8Operand";
+  let EncoderMethod = "getT2AddrModeImm8OpValue";
+  let DecoderMethod = "DecodeT2AddrModeImm8";
+  let ParserMatchClass = MemPosImm8OffsetAsmOperand;
+  let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
+}
+
+// t2addrmode_negimm8  := reg - imm8
+def MemNegImm8OffsetAsmOperand : AsmOperandClass {let Name="MemNegImm8Offset";}
+def t2addrmode_negimm8 : Operand<i32>,
+                      ComplexPattern<i32, 2, "SelectT2AddrModeImm8", []> {
+  let PrintMethod = "printT2AddrModeImm8Operand";
+  let EncoderMethod = "getT2AddrModeImm8OpValue";
+  let DecoderMethod = "DecodeT2AddrModeImm8";
+  let ParserMatchClass = MemNegImm8OffsetAsmOperand;
+  let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
+}
+
 // t2addrmode_imm8  := reg +/- imm8
+def MemImm8OffsetAsmOperand : AsmOperandClass { let Name = "MemImm8Offset"; }
 def t2addrmode_imm8 : Operand<i32>,
                       ComplexPattern<i32, 2, "SelectT2AddrModeImm8", []> {
   let PrintMethod = "printT2AddrModeImm8Operand";
   let EncoderMethod = "getT2AddrModeImm8OpValue";
+  let DecoderMethod = "DecodeT2AddrModeImm8";
+  let ParserMatchClass = MemImm8OffsetAsmOperand;
   let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemMode5AsmOperand;
 }
 
 def t2am_imm8_offset : Operand<i32>,
@@ -130,38 +173,61 @@ def t2am_imm8_offset : Operand<i32>,
                                       [], [SDNPWantRoot]> {
   let PrintMethod = "printT2AddrModeImm8OffsetOperand";
   let EncoderMethod = "getT2AddrModeImm8OffsetOpValue";
-  let ParserMatchClass = MemMode5AsmOperand;
+  let DecoderMethod = "DecodeT2Imm8";
 }
 
 // t2addrmode_imm8s4  := reg +/- (imm8 << 2)
+def MemImm8s4OffsetAsmOperand : AsmOperandClass {let Name = "MemImm8s4Offset";}
 def t2addrmode_imm8s4 : Operand<i32> {
   let PrintMethod = "printT2AddrModeImm8s4Operand";
   let EncoderMethod = "getT2AddrModeImm8s4OpValue";
+  let DecoderMethod = "DecodeT2AddrModeImm8s4";
+  let ParserMatchClass = MemImm8s4OffsetAsmOperand;
   let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
-  let ParserMatchClass = MemMode5AsmOperand;
 }
 
+def t2am_imm8s4_offset_asmoperand : AsmOperandClass { let Name = "Imm8s4"; }
 def t2am_imm8s4_offset : Operand<i32> {
   let PrintMethod = "printT2AddrModeImm8s4OffsetOperand";
+  let EncoderMethod = "getT2Imm8s4OpValue";
+  let DecoderMethod = "DecodeT2Imm8S4";
+}
+
+// t2addrmode_imm0_1020s4  := reg + (imm8 << 2)
+def MemImm0_1020s4OffsetAsmOperand : AsmOperandClass {
+  let Name = "MemImm0_1020s4Offset";
+}
+def t2addrmode_imm0_1020s4 : Operand<i32> {
+  let PrintMethod = "printT2AddrModeImm0_1020s4Operand";
+  let EncoderMethod = "getT2AddrModeImm0_1020s4OpValue";
+  let DecoderMethod = "DecodeT2AddrModeImm0_1020s4";
+  let ParserMatchClass = MemImm0_1020s4OffsetAsmOperand;
+  let MIOperandInfo = (ops GPRnopc:$base, i32imm:$offsimm);
 }
 
 // t2addrmode_so_reg  := reg + (reg << imm2)
+def t2addrmode_so_reg_asmoperand : AsmOperandClass {let Name="T2MemRegOffset";}
 def t2addrmode_so_reg : Operand<i32>,
                         ComplexPattern<i32, 3, "SelectT2AddrModeSoReg", []> {
   let PrintMethod = "printT2AddrModeSoRegOperand";
   let EncoderMethod = "getT2AddrModeSORegOpValue";
+  let DecoderMethod = "DecodeT2AddrModeSOReg";
+  let ParserMatchClass = t2addrmode_so_reg_asmoperand;
   let MIOperandInfo = (ops GPR:$base, rGPR:$offsreg, i32imm:$offsimm);
-  let ParserMatchClass = MemMode5AsmOperand;
 }
 
-// t2addrmode_reg := reg
-// Used by load/store exclusive instructions. Useful to enable right assembly
-// parsing and printing. Not used for any codegen matching.
-//
-def t2addrmode_reg : Operand<i32> {
-  let PrintMethod = "printAddrMode7Operand";
-  let MIOperandInfo = (ops GPR);
-  let ParserMatchClass = MemMode7AsmOperand;
+// Addresses for the TBB/TBH instructions.
+def addrmode_tbb_asmoperand : AsmOperandClass { let Name = "MemTBB"; }
+def addrmode_tbb : Operand<i32> {
+  let PrintMethod = "printAddrModeTBB";
+  let ParserMatchClass = addrmode_tbb_asmoperand;
+  let MIOperandInfo = (ops GPR:$Rn, rGPR:$Rm);
+}
+def addrmode_tbh_asmoperand : AsmOperandClass { let Name = "MemTBH"; }
+def addrmode_tbh : Operand<i32> {
+  let PrintMethod = "printAddrModeTBH";
+  let ParserMatchClass = addrmode_tbh_asmoperand;
+  let MIOperandInfo = (ops GPR:$Rn, rGPR:$Rm);
 }
 
 //===----------------------------------------------------------------------===//
@@ -419,47 +485,6 @@ class T2MulLong<bits<3> opc22_20, bits<4> opc7_4,
 }
 
 
-/// T2I_un_irs - Defines a set of (op reg, {so_imm|r|so_reg}) patterns for a
-/// unary operation that produces a value. These are predicable and can be
-/// changed to modify CPSR.
-multiclass T2I_un_irs<bits<4> opcod, string opc,
-                     InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
-                      PatFrag opnode, bit Cheap = 0, bit ReMat = 0> {
-   // shifted imm
-   def i : T2sOneRegImm<(outs rGPR:$Rd), (ins t2_so_imm:$imm), iii,
-                opc, "\t$Rd, $imm",
-                [(set rGPR:$Rd, (opnode t2_so_imm:$imm))]> {
-     let isAsCheapAsAMove = Cheap;
-     let isReMaterializable = ReMat;
-     let Inst{31-27} = 0b11110;
-     let Inst{25} = 0;
-     let Inst{24-21} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15} = 0;
-   }
-   // register
-   def r : T2sTwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm), iir,
-                opc, ".w\t$Rd, $Rm",
-                [(set rGPR:$Rd, (opnode rGPR:$Rm))]> {
-     let Inst{31-27} = 0b11101;
-     let Inst{26-25} = 0b01;
-     let Inst{24-21} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{14-12} = 0b000; // imm3
-     let Inst{7-6} = 0b00; // imm2
-     let Inst{5-4} = 0b00; // type
-   }
-   // shifted register
-   def s : T2sOneRegShiftedReg<(outs rGPR:$Rd), (ins t2_so_reg:$ShiftedRm), iis,
-                opc, ".w\t$Rd, $ShiftedRm",
-                [(set rGPR:$Rd, (opnode t2_so_reg:$ShiftedRm))]> {
-     let Inst{31-27} = 0b11101;
-     let Inst{26-25} = 0b01;
-     let Inst{24-21} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-   }
-}
-
 /// T2I_bin_irs - Defines a set of (op reg, {so_imm|r|so_reg}) patterns for a
 /// binary operation that produces a value. These are predicable and can be
 /// changed to modify CPSR.
@@ -500,21 +525,18 @@ multiclass T2I_bin_irs<bits<4> opcod, string opc,
    }
   // Assembly aliases for optional destination operand when it's the same
   // as the source operand.
-  def : InstAlias<!strconcat(opc, "${s}${p} $Rdn, $imm"),
+  def : t2InstAlias<!strconcat(opc, "${s}${p} $Rdn, $imm"),
      (!cast<Instruction>(!strconcat(baseOpc, "ri")) rGPR:$Rdn, rGPR:$Rdn,
                                                     t2_so_imm:$imm, pred:$p,
-                                                    cc_out:$s)>,
-     Requires<[IsThumb2]>;
-  def : InstAlias<!strconcat(opc, "${s}${p}", wide, " $Rdn, $Rm"),
+                                                    cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", wide, " $Rdn, $Rm"),
      (!cast<Instruction>(!strconcat(baseOpc, "rr")) rGPR:$Rdn, rGPR:$Rdn,
                                                     rGPR:$Rm, pred:$p,
-                                                    cc_out:$s)>,
-     Requires<[IsThumb2]>;
-  def : InstAlias<!strconcat(opc, "${s}${p}", wide, " $Rdn, $shift"),
+                                                    cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", wide, " $Rdn, $shift"),
      (!cast<Instruction>(!strconcat(baseOpc, "rs")) rGPR:$Rdn, rGPR:$Rdn,
                                                     t2_so_reg:$shift, pred:$p,
-                                                    cc_out:$s)>,
-     Requires<[IsThumb2]>;
+                                                    cc_out:$s)>;
 }
 
 /// T2I_bin_w_irs - Same as T2I_bin_irs except these operations need
@@ -522,7 +544,27 @@ multiclass T2I_bin_irs<bits<4> opcod, string opc,
 multiclass T2I_bin_w_irs<bits<4> opcod, string opc,
                      InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
                          PatFrag opnode, string baseOpc, bit Commutable = 0> :
-    T2I_bin_irs<opcod, opc, iii, iir, iis, opnode, baseOpc, Commutable, ".w">;
+    T2I_bin_irs<opcod, opc, iii, iir, iis, opnode, baseOpc, Commutable, ".w"> {
+  // Assembler aliases w/o the ".w" suffix.
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rd, $Rn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) rGPR:$Rd, rGPR:$Rn,
+                                                    rGPR:$Rm, pred:$p,
+                                                    cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rd, $Rn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rs")) rGPR:$Rd, rGPR:$Rn,
+                                                    t2_so_reg:$shift, pred:$p,
+                                                    cc_out:$s)>;
+
+  // and with the optional destination operand, too.
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rdn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) rGPR:$Rdn, rGPR:$Rdn,
+                                                    rGPR:$Rm, pred:$p,
+                                                    cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rdn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rs")) rGPR:$Rdn, rGPR:$Rdn,
+                                                    t2_so_reg:$shift, pred:$p,
+                                                    cc_out:$s)>;
+}
 
 /// T2I_rbin_is - Same as T2I_bin_irs except the order of operands are
 /// reversed.  The 'rr' form is only defined for the disassembler; for codegen
@@ -563,45 +605,28 @@ multiclass T2I_rbin_irs<bits<4> opcod, string opc, PatFrag opnode> {
 
 /// T2I_bin_s_irs - Similar to T2I_bin_irs except it sets the 's' bit so the
 /// instruction modifies the CPSR register.
-let isCodeGenOnly = 1, Defs = [CPSR] in {
+///
+/// These opcodes will be converted to the real non-S opcodes by
+/// AdjustInstrPostInstrSelection after giving then an optional CPSR operand.
+let hasPostISelHook = 1, isCodeGenOnly = 1, isPseudo = 1, Defs = [CPSR] in {
 multiclass T2I_bin_s_irs<bits<4> opcod, string opc,
                      InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
                          PatFrag opnode, bit Commutable = 0> {
    // shifted imm
-   def ri : T2TwoRegImm<
+   def ri : T2sTwoRegImm<
                 (outs rGPR:$Rd), (ins GPR:$Rn, t2_so_imm:$imm), iii,
-                !strconcat(opc, "s"), ".w\t$Rd, $Rn, $imm",
-                [(set rGPR:$Rd, (opnode GPR:$Rn, t2_so_imm:$imm))]> {
-     let Inst{31-27} = 0b11110;
-     let Inst{25} = 0;
-     let Inst{24-21} = opcod;
-     let Inst{20} = 1; // The S bit.
-     let Inst{15} = 0;
-   }
+                opc, ".w\t$Rd, $Rn, $imm",
+                [(set rGPR:$Rd, CPSR, (opnode GPR:$Rn, t2_so_imm:$imm))]>;
    // register
-   def rr : T2ThreeReg<
+   def rr : T2sThreeReg<
                 (outs rGPR:$Rd), (ins GPR:$Rn, rGPR:$Rm), iir,
-                !strconcat(opc, "s"), ".w\t$Rd, $Rn, $Rm",
-                [(set rGPR:$Rd, (opnode GPR:$Rn, rGPR:$Rm))]> {
-     let isCommutable = Commutable;
-     let Inst{31-27} = 0b11101;
-     let Inst{26-25} = 0b01;
-     let Inst{24-21} = opcod;
-     let Inst{20} = 1; // The S bit.
-     let Inst{14-12} = 0b000; // imm3
-     let Inst{7-6} = 0b00; // imm2
-     let Inst{5-4} = 0b00; // type
-   }
+                opc, ".w\t$Rd, $Rn, $Rm",
+                [(set rGPR:$Rd, CPSR, (opnode GPR:$Rn, rGPR:$Rm))]>;
    // shifted register
-   def rs : T2TwoRegShiftedReg<
+   def rs : T2sTwoRegShiftedReg<
                 (outs rGPR:$Rd), (ins GPR:$Rn, t2_so_reg:$ShiftedRm), iis,
-                !strconcat(opc, "s"), ".w\t$Rd, $Rn, $ShiftedRm",
-                [(set rGPR:$Rd, (opnode GPR:$Rn, t2_so_reg:$ShiftedRm))]> {
-     let Inst{31-27} = 0b11101;
-     let Inst{26-25} = 0b01;
-     let Inst{24-21} = opcod;
-     let Inst{20} = 1; // The S bit.
-   }
+                opc, ".w\t$Rd, $Rn, $ShiftedRm",
+               [(set rGPR:$Rd, CPSR, (opnode GPR:$Rn, t2_so_reg:$ShiftedRm))]>;
 }
 }
 
@@ -614,9 +639,9 @@ multiclass T2I_bin_ii12rs<bits<3> op23_21, string opc, PatFrag opnode,
    // in particular for taking the address of a local.
    let isReMaterializable = 1 in {
    def ri : T2sTwoRegImm<
-                 (outs rGPR:$Rd), (ins GPR:$Rn, t2_so_imm:$imm), IIC_iALUi,
-                 opc, ".w\t$Rd, $Rn, $imm",
-                 [(set rGPR:$Rd, (opnode GPR:$Rn, t2_so_imm:$imm))]> {
+               (outs GPRnopc:$Rd), (ins GPRnopc:$Rn, t2_so_imm:$imm), IIC_iALUi,
+               opc, ".w\t$Rd, $Rn, $imm",
+               [(set GPRnopc:$Rd, (opnode GPRnopc:$Rn, t2_so_imm:$imm))]> {
      let Inst{31-27} = 0b11110;
      let Inst{25} = 0;
      let Inst{24} = 1;
@@ -626,9 +651,9 @@ multiclass T2I_bin_ii12rs<bits<3> op23_21, string opc, PatFrag opnode,
    }
    // 12-bit imm
    def ri12 : T2I<
-                  (outs rGPR:$Rd), (ins GPR:$Rn, imm0_4095:$imm), IIC_iALUi,
+                  (outs GPRnopc:$Rd), (ins GPR:$Rn, imm0_4095:$imm), IIC_iALUi,
                   !strconcat(opc, "w"), "\t$Rd, $Rn, $imm",
-                  [(set rGPR:$Rd, (opnode GPR:$Rn, imm0_4095:$imm))]> {
+                  [(set GPRnopc:$Rd, (opnode GPR:$Rn, imm0_4095:$imm))]> {
      bits<4> Rd;
      bits<4> Rn;
      bits<12> imm;
@@ -644,9 +669,9 @@ multiclass T2I_bin_ii12rs<bits<3> op23_21, string opc, PatFrag opnode,
      let Inst{7-0} = imm{7-0};
    }
    // register
-   def rr : T2sThreeReg<(outs rGPR:$Rd), (ins GPR:$Rn, rGPR:$Rm), IIC_iALUr,
-                 opc, ".w\t$Rd, $Rn, $Rm",
-                 [(set rGPR:$Rd, (opnode GPR:$Rn, rGPR:$Rm))]> {
+   def rr : T2sThreeReg<(outs GPRnopc:$Rd), (ins GPRnopc:$Rn, rGPR:$Rm),
+                 IIC_iALUr, opc, ".w\t$Rd, $Rn, $Rm",
+                 [(set GPRnopc:$Rd, (opnode GPRnopc:$Rn, rGPR:$Rm))]> {
      let isCommutable = Commutable;
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
@@ -658,9 +683,9 @@ multiclass T2I_bin_ii12rs<bits<3> op23_21, string opc, PatFrag opnode,
    }
    // shifted register
    def rs : T2sTwoRegShiftedReg<
-                 (outs rGPR:$Rd), (ins GPR:$Rn, t2_so_reg:$ShiftedRm),
+                 (outs GPRnopc:$Rd), (ins GPRnopc:$Rn, t2_so_reg:$ShiftedRm),
                  IIC_iALUsi, opc, ".w\t$Rd, $Rn, $ShiftedRm",
-                 [(set rGPR:$Rd, (opnode GPR:$Rn, t2_so_reg:$ShiftedRm))]> {
+              [(set GPRnopc:$Rd, (opnode GPRnopc:$Rn, t2_so_reg:$ShiftedRm))]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
      let Inst{24} = 1;
@@ -671,13 +696,13 @@ multiclass T2I_bin_ii12rs<bits<3> op23_21, string opc, PatFrag opnode,
 /// T2I_adde_sube_irs - Defines a set of (op reg, {so_imm|r|so_reg}) patterns
 /// for a binary operation that produces a value and use the carry
 /// bit. It's not predicable.
-let Uses = [CPSR] in {
+let Defs = [CPSR], Uses = [CPSR] in {
 multiclass T2I_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
                              bit Commutable = 0> {
    // shifted imm
    def ri : T2sTwoRegImm<(outs rGPR:$Rd), (ins rGPR:$Rn, t2_so_imm:$imm),
                  IIC_iALUi, opc, "\t$Rd, $Rn, $imm",
-                 [(set rGPR:$Rd, (opnode rGPR:$Rn, t2_so_imm:$imm))]>,
+               [(set rGPR:$Rd, CPSR, (opnode rGPR:$Rn, t2_so_imm:$imm, CPSR))]>,
                  Requires<[IsThumb2]> {
      let Inst{31-27} = 0b11110;
      let Inst{25} = 0;
@@ -687,7 +712,7 @@ multiclass T2I_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
    // register
    def rr : T2sThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iALUr,
                  opc, ".w\t$Rd, $Rn, $Rm",
-                 [(set rGPR:$Rd, (opnode rGPR:$Rn, rGPR:$Rm))]>,
+                 [(set rGPR:$Rd, CPSR, (opnode rGPR:$Rn, rGPR:$Rm, CPSR))]>,
                  Requires<[IsThumb2]> {
      let isCommutable = Commutable;
      let Inst{31-27} = 0b11101;
@@ -701,7 +726,7 @@ multiclass T2I_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
    def rs : T2sTwoRegShiftedReg<
                  (outs rGPR:$Rd), (ins rGPR:$Rn, t2_so_reg:$ShiftedRm),
                  IIC_iALUsi, opc, ".w\t$Rd, $Rn, $ShiftedRm",
-                 [(set rGPR:$Rd, (opnode rGPR:$Rn, t2_so_reg:$ShiftedRm))]>,
+         [(set rGPR:$Rd, CPSR, (opnode rGPR:$Rn, t2_so_reg:$ShiftedRm, CPSR))]>,
                  Requires<[IsThumb2]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
@@ -710,64 +735,35 @@ multiclass T2I_adde_sube_irs<bits<4> opcod, string opc, PatFrag opnode,
 }
 }
 
-// Carry setting variants
-// NOTE: CPSR def omitted because it will be handled by the custom inserter.
-let usesCustomInserter = 1 in {
-multiclass T2I_adde_sube_s_irs<PatFrag opnode, bit Commutable = 0> {
-   // shifted imm
-   def ri : t2PseudoInst<(outs rGPR:$Rd), (ins rGPR:$Rn, t2_so_imm:$imm),
-                4, IIC_iALUi,
-                [(set rGPR:$Rd, (opnode rGPR:$Rn, t2_so_imm:$imm))]>;
-   // register
-   def rr : t2PseudoInst<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm),
-                4, IIC_iALUr,
-                [(set rGPR:$Rd, (opnode rGPR:$Rn, rGPR:$Rm))]> {
-     let isCommutable = Commutable;
-   }
-   // shifted register
-   def rs : t2PseudoInst<
-                (outs rGPR:$Rd), (ins rGPR:$Rn, t2_so_reg:$ShiftedRm),
-                4, IIC_iALUsi,
-                [(set rGPR:$Rd, (opnode rGPR:$Rn, t2_so_reg:$ShiftedRm))]>;
-}
-}
-
 /// T2I_rbin_s_is - Same as T2I_rbin_irs except sets 's' bit and the register
 /// version is not needed since this is only for codegen.
-let isCodeGenOnly = 1, Defs = [CPSR] in {
+///
+/// These opcodes will be converted to the real non-S opcodes by
+/// AdjustInstrPostInstrSelection after giving then an optional CPSR operand.
+let hasPostISelHook = 1, isCodeGenOnly = 1, isPseudo = 1, Defs = [CPSR] in {
 multiclass T2I_rbin_s_is<bits<4> opcod, string opc, PatFrag opnode> {
    // shifted imm
-   def ri : T2TwoRegImm<
+   def ri : T2sTwoRegImm<
                 (outs rGPR:$Rd), (ins rGPR:$Rn, t2_so_imm:$imm), IIC_iALUi,
-                !strconcat(opc, "s"), ".w\t$Rd, $Rn, $imm",
-                [(set rGPR:$Rd, (opnode t2_so_imm:$imm, rGPR:$Rn))]> {
-     let Inst{31-27} = 0b11110;
-     let Inst{25} = 0;
-     let Inst{24-21} = opcod;
-     let Inst{20} = 1; // The S bit.
-     let Inst{15} = 0;
-   }
+                opc, ".w\t$Rd, $Rn, $imm",
+                [(set rGPR:$Rd, CPSR, (opnode t2_so_imm:$imm, rGPR:$Rn))]>;
    // shifted register
-   def rs : T2TwoRegShiftedReg<
+   def rs : T2sTwoRegShiftedReg<
                 (outs rGPR:$Rd), (ins rGPR:$Rn, t2_so_reg:$ShiftedRm),
-                IIC_iALUsi, !strconcat(opc, "s"), "\t$Rd, $Rn, $ShiftedRm",
-                [(set rGPR:$Rd, (opnode t2_so_reg:$ShiftedRm, rGPR:$Rn))]> {
-     let Inst{31-27} = 0b11101;
-     let Inst{26-25} = 0b01;
-     let Inst{24-21} = opcod;
-     let Inst{20} = 1; // The S bit.
-   }
+                IIC_iALUsi, opc, "\t$Rd, $Rn, $ShiftedRm",
+              [(set rGPR:$Rd, CPSR, (opnode t2_so_reg:$ShiftedRm, rGPR:$Rn))]>;
 }
 }
 
 /// T2I_sh_ir - Defines a set of (op reg, {so_imm|r}) patterns for a shift /
 //  rotate operation that produces a value.
-multiclass T2I_sh_ir<bits<2> opcod, string opc, PatFrag opnode> {
+multiclass T2I_sh_ir<bits<2> opcod, string opc, Operand ty, PatFrag opnode,
+                     string baseOpc> {
    // 5-bit imm
    def ri : T2sTwoRegShiftImm<
-                 (outs rGPR:$Rd), (ins rGPR:$Rm, i32imm:$imm), IIC_iMOVsi,
+                 (outs rGPR:$Rd), (ins rGPR:$Rm, ty:$imm), IIC_iMOVsi,
                  opc, ".w\t$Rd, $Rm, $imm",
-                 [(set rGPR:$Rd, (opnode rGPR:$Rm, imm1_31:$imm))]> {
+                 [(set rGPR:$Rd, (opnode rGPR:$Rm, (i32 ty:$imm)))]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-21} = 0b010010;
      let Inst{19-16} = 0b1111; // Rn
@@ -784,20 +780,50 @@ multiclass T2I_sh_ir<bits<2> opcod, string opc, PatFrag opnode> {
      let Inst{15-12} = 0b1111;
      let Inst{7-4} = 0b0000;
    }
+
+  // Optional destination register
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", ".w $Rdn, $imm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "ri")) rGPR:$Rdn, rGPR:$Rdn,
+                                                    ty:$imm, pred:$p,
+                                                    cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", ".w $Rdn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) rGPR:$Rdn, rGPR:$Rdn,
+                                                    rGPR:$Rm, pred:$p,
+                                                    cc_out:$s)>;
+
+  // Assembler aliases w/o the ".w" suffix.
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rd, $Rn, $imm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "ri")) rGPR:$Rd, rGPR:$Rn,
+                                                    ty:$imm, pred:$p,
+                                                   cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rd, $Rn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) rGPR:$Rd, rGPR:$Rn,
+                                                    rGPR:$Rm, pred:$p,
+                                                    cc_out:$s)>;
+
+  // and with the optional destination operand, too.
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rdn, $imm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "ri")) rGPR:$Rdn, rGPR:$Rdn,
+                                                    ty:$imm, pred:$p,
+                                                    cc_out:$s)>;
+  def : t2InstAlias<!strconcat(opc, "${s}${p}", " $Rdn, $Rm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rr")) rGPR:$Rdn, rGPR:$Rdn,
+                                                    rGPR:$Rm, pred:$p,
+                                                    cc_out:$s)>;
 }
 
 /// T2I_cmp_irs - Defines a set of (op r, {so_imm|r|so_reg}) cmp / test
 /// patterns. Similar to T2I_bin_irs except the instruction does not produce
 /// a explicit result, only implicitly set CPSR.
-let isCompare = 1, Defs = [CPSR] in {
 multiclass T2I_cmp_irs<bits<4> opcod, string opc,
                      InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
-                       PatFrag opnode> {
+                       PatFrag opnode, string baseOpc> {
+let isCompare = 1, Defs = [CPSR] in {
    // shifted imm
    def ri : T2OneRegCmpImm<
-                (outs), (ins GPR:$Rn, t2_so_imm:$imm), iii,
+                (outs), (ins GPRnopc:$Rn, t2_so_imm:$imm), iii,
                 opc, ".w\t$Rn, $imm",
-                [(opnode GPR:$Rn, t2_so_imm:$imm)]> {
+                [(opnode GPRnopc:$Rn, t2_so_imm:$imm)]> {
      let Inst{31-27} = 0b11110;
      let Inst{25} = 0;
      let Inst{24-21} = opcod;
@@ -807,9 +833,9 @@ multiclass T2I_cmp_irs<bits<4> opcod, string opc,
    }
    // register
    def rr : T2TwoRegCmp<
-                (outs), (ins GPR:$lhs, rGPR:$rhs), iir,
-                opc, ".w\t$lhs, $rhs",
-                [(opnode GPR:$lhs, rGPR:$rhs)]> {
+                (outs), (ins GPRnopc:$Rn, rGPR:$Rm), iir,
+                opc, ".w\t$Rn, $Rm",
+                [(opnode GPRnopc:$Rn, rGPR:$Rm)]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
      let Inst{24-21} = opcod;
@@ -821,9 +847,9 @@ multiclass T2I_cmp_irs<bits<4> opcod, string opc,
    }
    // shifted register
    def rs : T2OneRegCmpShiftedReg<
-                (outs), (ins GPR:$Rn, t2_so_reg:$ShiftedRm), iis,
+                (outs), (ins GPRnopc:$Rn, t2_so_reg:$ShiftedRm), iis,
                 opc, ".w\t$Rn, $ShiftedRm",
-                [(opnode GPR:$Rn, t2_so_reg:$ShiftedRm)]> {
+                [(opnode GPRnopc:$Rn, t2_so_reg:$ShiftedRm)]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
      let Inst{24-21} = opcod;
@@ -831,55 +857,60 @@ multiclass T2I_cmp_irs<bits<4> opcod, string opc,
      let Inst{11-8} = 0b1111; // Rd
    }
 }
+
+  // Assembler aliases w/o the ".w" suffix.
+  // No alias here for 'rr' version as not all instantiations of this
+  // multiclass want one (CMP in particular, does not).
+  def : t2InstAlias<!strconcat(opc, "${p}", " $Rn, $imm"),
+     (!cast<Instruction>(!strconcat(baseOpc, "ri")) GPRnopc:$Rn,
+                                                    t2_so_imm:$imm, pred:$p)>;
+  def : t2InstAlias<!strconcat(opc, "${p}", " $Rn, $shift"),
+     (!cast<Instruction>(!strconcat(baseOpc, "rs")) GPRnopc:$Rn,
+                                                    t2_so_reg:$shift,
+                                                    pred:$p)>;
 }
 
 /// T2I_ld - Defines a set of (op r, {imm12|imm8|so_reg}) load patterns.
 multiclass T2I_ld<bit signed, bits<2> opcod, string opc,
-                  InstrItinClass iii, InstrItinClass iis, PatFrag opnode> {
-  def i12 : T2Ii12<(outs GPR:$Rt), (ins t2addrmode_imm12:$addr), iii,
+                  InstrItinClass iii, InstrItinClass iis, RegisterClass target,
+                  PatFrag opnode> {
+  def i12 : T2Ii12<(outs target:$Rt), (ins t2addrmode_imm12:$addr), iii,
                    opc, ".w\t$Rt, $addr",
-                   [(set GPR:$Rt, (opnode t2addrmode_imm12:$addr))]> {
-    let Inst{31-27} = 0b11111;
-    let Inst{26-25} = 0b00;
+                   [(set target:$Rt, (opnode t2addrmode_imm12:$addr))]> {
+    bits<4> Rt;
+    bits<17> addr;
+    let Inst{31-25} = 0b1111100;
     let Inst{24} = signed;
     let Inst{23} = 1;
     let Inst{22-21} = opcod;
     let Inst{20} = 1; // load
-
-    bits<4> Rt;
-    let Inst{15-12} = Rt;
-
-    bits<17> addr;
-    let addr{12}    = 1;           // add = TRUE
     let Inst{19-16} = addr{16-13}; // Rn
-    let Inst{23}    = addr{12};    // U
+    let Inst{15-12} = Rt;
     let Inst{11-0}  = addr{11-0};  // imm
   }
-  def i8  : T2Ii8 <(outs GPR:$Rt), (ins t2addrmode_imm8:$addr), iii,
+  def i8  : T2Ii8 <(outs target:$Rt), (ins t2addrmode_negimm8:$addr), iii,
                    opc, "\t$Rt, $addr",
-                   [(set GPR:$Rt, (opnode t2addrmode_imm8:$addr))]> {
+                   [(set target:$Rt, (opnode t2addrmode_negimm8:$addr))]> {
+    bits<4> Rt;
+    bits<13> addr;
     let Inst{31-27} = 0b11111;
     let Inst{26-25} = 0b00;
     let Inst{24} = signed;
     let Inst{23} = 0;
     let Inst{22-21} = opcod;
     let Inst{20} = 1; // load
+    let Inst{19-16} = addr{12-9}; // Rn
+    let Inst{15-12} = Rt;
     let Inst{11} = 1;
     // Offset: index==TRUE, wback==FALSE
     let Inst{10} = 1; // The P bit.
-    let Inst{8} = 0; // The W bit.
-
-    bits<4> Rt;
-    let Inst{15-12} = Rt;
-
-    bits<13> addr;
-    let Inst{19-16} = addr{12-9}; // Rn
     let Inst{9}     = addr{8};    // U
+    let Inst{8} = 0; // The W bit.
     let Inst{7-0}   = addr{7-0};  // imm
   }
-  def s   : T2Iso <(outs GPR:$Rt), (ins t2addrmode_so_reg:$addr), iis,
+  def s   : T2Iso <(outs target:$Rt), (ins t2addrmode_so_reg:$addr), iis,
                    opc, ".w\t$Rt, $addr",
-                   [(set GPR:$Rt, (opnode t2addrmode_so_reg:$addr))]> {
+                   [(set target:$Rt, (opnode t2addrmode_so_reg:$addr))]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-25} = 0b00;
     let Inst{24} = signed;
@@ -895,12 +926,14 @@ multiclass T2I_ld<bit signed, bits<2> opcod, string opc,
     let Inst{19-16} = addr{9-6}; // Rn
     let Inst{3-0}   = addr{5-2}; // Rm
     let Inst{5-4}   = addr{1-0}; // imm
+
+    let DecoderMethod = "DecodeT2LoadShift";
   }
 
   // FIXME: Is the pci variant actually needed?
-  def pci : T2Ipc <(outs GPR:$Rt), (ins t2ldrlabel:$addr), iii,
+  def pci : T2Ipc <(outs target:$Rt), (ins t2ldrlabel:$addr), iii,
                    opc, ".w\t$Rt, $addr",
-                   [(set GPR:$Rt, (opnode (ARMWrapper tconstpool:$addr)))]> {
+                   [(set target:$Rt, (opnode (ARMWrapper tconstpool:$addr)))]> {
     let isReMaterializable = 1;
     let Inst{31-27} = 0b11111;
     let Inst{26-25} = 0b00;
@@ -918,10 +951,11 @@ multiclass T2I_ld<bit signed, bits<2> opcod, string opc,
 
 /// T2I_st - Defines a set of (op r, {imm12|imm8|so_reg}) store patterns.
 multiclass T2I_st<bits<2> opcod, string opc,
-                  InstrItinClass iii, InstrItinClass iis, PatFrag opnode> {
-  def i12 : T2Ii12<(outs), (ins GPR:$Rt, t2addrmode_imm12:$addr), iii,
+                  InstrItinClass iii, InstrItinClass iis, RegisterClass target,
+                  PatFrag opnode> {
+  def i12 : T2Ii12<(outs), (ins target:$Rt, t2addrmode_imm12:$addr), iii,
                    opc, ".w\t$Rt, $addr",
-                   [(opnode GPR:$Rt, t2addrmode_imm12:$addr)]> {
+                   [(opnode target:$Rt, t2addrmode_imm12:$addr)]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0001;
     let Inst{22-21} = opcod;
@@ -936,9 +970,9 @@ multiclass T2I_st<bits<2> opcod, string opc,
     let Inst{23}    = addr{12};    // U
     let Inst{11-0}  = addr{11-0};  // imm
   }
-  def i8  : T2Ii8 <(outs), (ins GPR:$Rt, t2addrmode_imm8:$addr), iii,
+  def i8  : T2Ii8 <(outs), (ins target:$Rt, t2addrmode_negimm8:$addr), iii,
                    opc, "\t$Rt, $addr",
-                   [(opnode GPR:$Rt, t2addrmode_imm8:$addr)]> {
+                   [(opnode target:$Rt, t2addrmode_negimm8:$addr)]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0000;
     let Inst{22-21} = opcod;
@@ -956,9 +990,9 @@ multiclass T2I_st<bits<2> opcod, string opc,
     let Inst{9}     = addr{8};    // U
     let Inst{7-0}   = addr{7-0};  // imm
   }
-  def s   : T2Iso <(outs), (ins GPR:$Rt, t2addrmode_so_reg:$addr), iis,
+  def s   : T2Iso <(outs), (ins target:$Rt, t2addrmode_so_reg:$addr), iis,
                    opc, ".w\t$Rt, $addr",
-                   [(opnode GPR:$Rt, t2addrmode_so_reg:$addr)]> {
+                   [(opnode target:$Rt, t2addrmode_so_reg:$addr)]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0000;
     let Inst{22-21} = opcod;
@@ -977,146 +1011,81 @@ multiclass T2I_st<bits<2> opcod, string opc,
 
 /// T2I_ext_rrot - A unary operation with two forms: one whose operand is a
 /// register and one whose operand is a register rotated by 8/16/24.
-multiclass T2I_ext_rrot<bits<3> opcod, string opc, PatFrag opnode> {
-  def r     : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm), IIC_iEXTr,
-                  opc, ".w\t$Rd, $Rm",
-                 [(set rGPR:$Rd, (opnode rGPR:$Rm))]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-     let Inst{5-4} = 0b00; // rotate
-   }
-  def r_rot : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm, rot_imm:$rot), IIC_iEXTr,
-                  opc, ".w\t$Rd, $Rm, ror $rot",
-                 [(set rGPR:$Rd, (opnode (rotr rGPR:$Rm, rot_imm:$rot)))]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
+class T2I_ext_rrot<bits<3> opcod, string opc, PatFrag opnode>
+  : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm, rot_imm:$rot), IIC_iEXTr,
+             opc, ".w\t$Rd, $Rm$rot",
+             [(set rGPR:$Rd, (opnode (rotr rGPR:$Rm, rot_imm:$rot)))]>,
+             Requires<[IsThumb2]> {
+   let Inst{31-27} = 0b11111;
+   let Inst{26-23} = 0b0100;
+   let Inst{22-20} = opcod;
+   let Inst{19-16} = 0b1111; // Rn
+   let Inst{15-12} = 0b1111;
+   let Inst{7} = 1;
 
-     bits<2> rot;
-     let Inst{5-4} = rot{1-0}; // rotate
-   }
+   bits<2> rot;
+   let Inst{5-4} = rot{1-0}; // rotate
 }
 
 // UXTB16 - Requres T2ExtractPack, does not need the .w qualifier.
-multiclass T2I_ext_rrot_uxtb16<bits<3> opcod, string opc, PatFrag opnode> {
-  def r     : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm), IIC_iEXTr,
-                  opc, "\t$Rd, $Rm",
-                 [(set rGPR:$Rd, (opnode rGPR:$Rm))]>,
-                 Requires<[HasT2ExtractPack, IsThumb2]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-     let Inst{5-4} = 0b00; // rotate
-   }
-  def r_rot : T2TwoReg<(outs rGPR:$dst), (ins rGPR:$Rm, rot_imm:$rot),
-                  IIC_iEXTr, opc, "\t$dst, $Rm, ror $rot",
-                 [(set rGPR:$dst, (opnode (rotr rGPR:$Rm, rot_imm:$rot)))]>,
-                 Requires<[HasT2ExtractPack, IsThumb2]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-
-     bits<2> rot;
-     let Inst{5-4} = rot{1-0}; // rotate
-   }
+class T2I_ext_rrot_uxtb16<bits<3> opcod, string opc, PatFrag opnode>
+  : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm, rot_imm:$rot),
+             IIC_iEXTr, opc, "\t$Rd, $Rm$rot",
+            [(set rGPR:$Rd, (opnode (rotr rGPR:$Rm, rot_imm:$rot)))]>,
+          Requires<[HasT2ExtractPack, IsThumb2]> {
+  bits<2> rot;
+  let Inst{31-27} = 0b11111;
+  let Inst{26-23} = 0b0100;
+  let Inst{22-20} = opcod;
+  let Inst{19-16} = 0b1111; // Rn
+  let Inst{15-12} = 0b1111;
+  let Inst{7} = 1;
+  let Inst{5-4} = rot;
 }
 
 // SXTB16 - Requres T2ExtractPack, does not need the .w qualifier, no pattern
 // supported yet.
-multiclass T2I_ext_rrot_sxtb16<bits<3> opcod, string opc> {
-  def r     : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm), IIC_iEXTr,
-                  opc, "\t$Rd, $Rm", []>,
+class T2I_ext_rrot_sxtb16<bits<3> opcod, string opc>
+  : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm, rot_imm:$rot), IIC_iEXTr,
+             opc, "\t$Rd, $Rm$rot", []>,
           Requires<[IsThumb2, HasT2ExtractPack]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-     let Inst{5-4} = 0b00; // rotate
-   }
-  def r_rot : T2TwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm, i32imm:$rot), IIC_iEXTr,
-                  opc, "\t$Rd, $Rm, ror $rot", []>,
-          Requires<[IsThumb2, HasT2ExtractPack]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{19-16} = 0b1111; // Rn
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-
-      bits<2> rot;
-      let Inst{5-4} = rot{1-0}; // rotate
-   }
+  bits<2> rot;
+  let Inst{31-27} = 0b11111;
+  let Inst{26-23} = 0b0100;
+  let Inst{22-20} = opcod;
+  let Inst{19-16} = 0b1111; // Rn
+  let Inst{15-12} = 0b1111;
+  let Inst{7} = 1;
+  let Inst{5-4} = rot;
 }
 
 /// T2I_exta_rrot - A binary operation with two forms: one whose operand is a
 /// register and one whose operand is a register rotated by 8/16/24.
-multiclass T2I_exta_rrot<bits<3> opcod, string opc, PatFrag opnode> {
-  def rr     : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iEXTAr,
-                  opc, "\t$Rd, $Rn, $Rm",
-                  [(set rGPR:$Rd, (opnode rGPR:$Rn, rGPR:$Rm))]>,
-                  Requires<[HasT2ExtractPack, IsThumb2]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-     let Inst{5-4} = 0b00; // rotate
-   }
-  def rr_rot : T2ThreeReg<(outs rGPR:$Rd),
-                  (ins rGPR:$Rn, rGPR:$Rm, rot_imm:$rot),
-                  IIC_iEXTAsr, opc, "\t$Rd, $Rn, $Rm, ror $rot",
-                  [(set rGPR:$Rd, (opnode rGPR:$Rn,
-                                          (rotr rGPR:$Rm, rot_imm:$rot)))]>,
-                  Requires<[HasT2ExtractPack, IsThumb2]> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-
-     bits<2> rot;
-     let Inst{5-4} = rot{1-0}; // rotate
-   }
+class T2I_exta_rrot<bits<3> opcod, string opc, PatFrag opnode>
+  : T2ThreeReg<(outs rGPR:$Rd),
+               (ins rGPR:$Rn, rGPR:$Rm, rot_imm:$rot),
+               IIC_iEXTAsr, opc, "\t$Rd, $Rn, $Rm$rot",
+             [(set rGPR:$Rd, (opnode rGPR:$Rn, (rotr rGPR:$Rm,rot_imm:$rot)))]>,
+           Requires<[HasT2ExtractPack, IsThumb2]> {
+  bits<2> rot;
+  let Inst{31-27} = 0b11111;
+  let Inst{26-23} = 0b0100;
+  let Inst{22-20} = opcod;
+  let Inst{15-12} = 0b1111;
+  let Inst{7} = 1;
+  let Inst{5-4} = rot;
 }
 
-// DO variant - disassembly only, no pattern
-
-multiclass T2I_exta_rrot_DO<bits<3> opcod, string opc> {
-  def rr     : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iEXTAr,
-                  opc, "\t$Rd, $Rn, $Rm", []> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-     let Inst{5-4} = 0b00; // rotate
-   }
-  def rr_rot :T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, i32imm:$rot),
-                  IIC_iEXTAsr, opc, "\t$Rd, $Rn, $Rm, ror $rot", []> {
-     let Inst{31-27} = 0b11111;
-     let Inst{26-23} = 0b0100;
-     let Inst{22-20} = opcod;
-     let Inst{15-12} = 0b1111;
-     let Inst{7} = 1;
-
-     bits<2> rot;
-     let Inst{5-4} = rot{1-0}; // rotate
-   }
+class T2I_exta_rrot_np<bits<3> opcod, string opc>
+  : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm,rot_imm:$rot),
+               IIC_iEXTAsr, opc, "\t$Rd, $Rn, $Rm$rot", []> {
+  bits<2> rot;
+  let Inst{31-27} = 0b11111;
+  let Inst{26-23} = 0b0100;
+  let Inst{22-20} = opcod;
+  let Inst{15-12} = 0b1111;
+  let Inst{7} = 1;
+  let Inst{5-4} = rot;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1143,7 +1112,7 @@ class T2PCOneRegImm<dag oops, dag iops, InstrItinClass itin,
 // assembler.
 def t2ADR : T2PCOneRegImm<(outs rGPR:$Rd),
               (ins t2adrlabel:$addr, pred:$p),
-              IIC_iALUi, "adr{$p}.w\t$Rd, #$addr", []> {
+              IIC_iALUi, "adr{$p}.w\t$Rd, $addr", []> {
   let Inst{31-27} = 0b11110;
   let Inst{25-24} = 0b10;
   // Inst{23:21} = '11' (add = FALSE) or '00' (add = TRUE)
@@ -1160,6 +1129,8 @@ def t2ADR : T2PCOneRegImm<(outs rGPR:$Rd),
   let Inst{26}    = addr{11};
   let Inst{14-12} = addr{10-8};
   let Inst{7-0}   = addr{7-0};
+
+  let DecoderMethod = "DecodeT2Adr";
 }
 
 let neverHasSideEffects = 1, isReMaterializable = 1 in
@@ -1177,33 +1148,33 @@ def t2LEApcrelJT : t2PseudoInst<(outs rGPR:$Rd),
 
 // Load
 let canFoldAsLoad = 1, isReMaterializable = 1  in
-defm t2LDR   : T2I_ld<0, 0b10, "ldr", IIC_iLoad_i, IIC_iLoad_si,
+defm t2LDR   : T2I_ld<0, 0b10, "ldr", IIC_iLoad_i, IIC_iLoad_si, GPR,
                       UnOpFrag<(load node:$Src)>>;
 
 // Loads with zero extension
 defm t2LDRH  : T2I_ld<0, 0b01, "ldrh", IIC_iLoad_bh_i, IIC_iLoad_bh_si,
-                      UnOpFrag<(zextloadi16 node:$Src)>>;
+                      rGPR, UnOpFrag<(zextloadi16 node:$Src)>>;
 defm t2LDRB  : T2I_ld<0, 0b00, "ldrb", IIC_iLoad_bh_i, IIC_iLoad_bh_si,
-                      UnOpFrag<(zextloadi8  node:$Src)>>;
+                      rGPR, UnOpFrag<(zextloadi8  node:$Src)>>;
 
 // Loads with sign extension
 defm t2LDRSH : T2I_ld<1, 0b01, "ldrsh", IIC_iLoad_bh_i, IIC_iLoad_bh_si,
-                      UnOpFrag<(sextloadi16 node:$Src)>>;
+                      rGPR, UnOpFrag<(sextloadi16 node:$Src)>>;
 defm t2LDRSB : T2I_ld<1, 0b00, "ldrsb", IIC_iLoad_bh_i, IIC_iLoad_bh_si,
-                      UnOpFrag<(sextloadi8  node:$Src)>>;
+                      rGPR, UnOpFrag<(sextloadi8  node:$Src)>>;
 
 let mayLoad = 1, neverHasSideEffects = 1, hasExtraDefRegAllocReq = 1 in {
 // Load doubleword
 def t2LDRDi8  : T2Ii8s4<1, 0, 1, (outs rGPR:$Rt, rGPR:$Rt2),
                         (ins t2addrmode_imm8s4:$addr),
-                        IIC_iLoad_d_i, "ldrd", "\t$Rt, $Rt2, $addr", []>;
+                        IIC_iLoad_d_i, "ldrd", "\t$Rt, $Rt2, $addr", "", []>;
 } // mayLoad = 1, neverHasSideEffects = 1, hasExtraDefRegAllocReq = 1
 
 // zextload i1 -> zextload i8
 def : T2Pat<(zextloadi1 t2addrmode_imm12:$addr),
             (t2LDRBi12  t2addrmode_imm12:$addr)>;
-def : T2Pat<(zextloadi1 t2addrmode_imm8:$addr),
-            (t2LDRBi8   t2addrmode_imm8:$addr)>;
+def : T2Pat<(zextloadi1 t2addrmode_negimm8:$addr),
+            (t2LDRBi8   t2addrmode_negimm8:$addr)>;
 def : T2Pat<(zextloadi1 t2addrmode_so_reg:$addr),
             (t2LDRBs    t2addrmode_so_reg:$addr)>;
 def : T2Pat<(zextloadi1 (ARMWrapper tconstpool:$addr)),
@@ -1214,8 +1185,8 @@ def : T2Pat<(zextloadi1 (ARMWrapper tconstpool:$addr)),
 // earlier?
 def : T2Pat<(extloadi1  t2addrmode_imm12:$addr),
             (t2LDRBi12  t2addrmode_imm12:$addr)>;
-def : T2Pat<(extloadi1  t2addrmode_imm8:$addr),
-            (t2LDRBi8   t2addrmode_imm8:$addr)>;
+def : T2Pat<(extloadi1  t2addrmode_negimm8:$addr),
+            (t2LDRBi8   t2addrmode_negimm8:$addr)>;
 def : T2Pat<(extloadi1  t2addrmode_so_reg:$addr),
             (t2LDRBs    t2addrmode_so_reg:$addr)>;
 def : T2Pat<(extloadi1  (ARMWrapper tconstpool:$addr)),
@@ -1223,8 +1194,8 @@ def : T2Pat<(extloadi1  (ARMWrapper tconstpool:$addr)),
 
 def : T2Pat<(extloadi8  t2addrmode_imm12:$addr),
             (t2LDRBi12  t2addrmode_imm12:$addr)>;
-def : T2Pat<(extloadi8  t2addrmode_imm8:$addr),
-            (t2LDRBi8   t2addrmode_imm8:$addr)>;
+def : T2Pat<(extloadi8  t2addrmode_negimm8:$addr),
+            (t2LDRBi8   t2addrmode_negimm8:$addr)>;
 def : T2Pat<(extloadi8  t2addrmode_so_reg:$addr),
             (t2LDRBs    t2addrmode_so_reg:$addr)>;
 def : T2Pat<(extloadi8  (ARMWrapper tconstpool:$addr)),
@@ -1232,8 +1203,8 @@ def : T2Pat<(extloadi8  (ARMWrapper tconstpool:$addr)),
 
 def : T2Pat<(extloadi16 t2addrmode_imm12:$addr),
             (t2LDRHi12  t2addrmode_imm12:$addr)>;
-def : T2Pat<(extloadi16 t2addrmode_imm8:$addr),
-            (t2LDRHi8   t2addrmode_imm8:$addr)>;
+def : T2Pat<(extloadi16 t2addrmode_negimm8:$addr),
+            (t2LDRHi8   t2addrmode_negimm8:$addr)>;
 def : T2Pat<(extloadi16 t2addrmode_so_reg:$addr),
             (t2LDRHs    t2addrmode_so_reg:$addr)>;
 def : T2Pat<(extloadi16 (ARMWrapper tconstpool:$addr)),
@@ -1247,83 +1218,86 @@ def : T2Pat<(extloadi16 (ARMWrapper tconstpool:$addr)),
 // Indexed loads
 
 let mayLoad = 1, neverHasSideEffects = 1 in {
-def t2LDR_PRE  : T2Iidxldst<0, 0b10, 1, 1, (outs GPR:$Rt, GPR:$Rn),
+def t2LDR_PRE  : T2Ipreldst<0, 0b10, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
                             (ins t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iLoad_iu,
-                            "ldr", "\t$Rt, $addr!", "$addr.base = $Rn",
-                            []>;
+                            "ldr", "\t$Rt, $addr!", "$addr.base = $Rn_wb",
+                            []> {
+  let AsmMatchConverter = "cvtLdWriteBackRegT2AddrModeImm8";
+}
 
-def t2LDR_POST : T2Iidxldst<0, 0b10, 1, 0, (outs GPR:$Rt, GPR:$Rn),
-                            (ins GPR:$base, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iLoad_iu,
-                          "ldr", "\t$Rt, [$Rn], $addr", "$base = $Rn",
-                            []>;
+def t2LDR_POST : T2Ipostldst<0, 0b10, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                          (ins addr_offset_none:$Rn, t2am_imm8_offset:$offset),
+                          AddrModeT2_i8, IndexModePost, IIC_iLoad_iu,
+                          "ldr", "\t$Rt, $Rn$offset", "$Rn = $Rn_wb", []>;
 
-def t2LDRB_PRE : T2Iidxldst<0, 0b00, 1, 1, (outs GPR:$Rt, GPR:$Rn),
+def t2LDRB_PRE : T2Ipreldst<0, 0b00, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
                             (ins t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iLoad_bh_iu,
-                            "ldrb", "\t$Rt, $addr!", "$addr.base = $Rn",
-                            []>;
-def t2LDRB_POST : T2Iidxldst<0, 0b00, 1, 0, (outs GPR:$Rt, GPR:$Rn),
-                            (ins GPR:$base, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
-                         "ldrb", "\t$Rt, [$Rn], $addr", "$base = $Rn",
-                            []>;
+                            "ldrb", "\t$Rt, $addr!", "$addr.base = $Rn_wb",
+                            []> {
+  let AsmMatchConverter = "cvtLdWriteBackRegT2AddrModeImm8";
+}
+def t2LDRB_POST : T2Ipostldst<0, 0b00, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                          (ins addr_offset_none:$Rn, t2am_imm8_offset:$offset),
+                          AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
+                          "ldrb", "\t$Rt, $Rn$offset", "$Rn = $Rn_wb", []>;
 
-def t2LDRH_PRE : T2Iidxldst<0, 0b01, 1, 1, (outs GPR:$Rt, GPR:$Rn),
+def t2LDRH_PRE : T2Ipreldst<0, 0b01, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
                             (ins t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iLoad_bh_iu,
-                            "ldrh", "\t$Rt, $addr!", "$addr.base = $Rn",
-                            []>;
-def t2LDRH_POST : T2Iidxldst<0, 0b01, 1, 0, (outs GPR:$Rt, GPR:$Rn),
-                            (ins GPR:$base, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
-                         "ldrh", "\t$Rt, [$Rn], $addr", "$base = $Rn",
-                            []>;
+                            "ldrh", "\t$Rt, $addr!", "$addr.base = $Rn_wb",
+                            []> {
+  let AsmMatchConverter = "cvtLdWriteBackRegT2AddrModeImm8";
+}
+def t2LDRH_POST : T2Ipostldst<0, 0b01, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                          (ins addr_offset_none:$Rn, t2am_imm8_offset:$offset),
+                          AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
+                          "ldrh", "\t$Rt, $Rn$offset", "$Rn = $Rn_wb", []>;
 
-def t2LDRSB_PRE : T2Iidxldst<1, 0b00, 1, 1, (outs GPR:$Rt, GPR:$Rn),
+def t2LDRSB_PRE : T2Ipreldst<1, 0b00, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
                             (ins t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iLoad_bh_iu,
-                            "ldrsb", "\t$Rt, $addr!", "$addr.base = $Rn",
-                            []>;
-def t2LDRSB_POST : T2Iidxldst<1, 0b00, 1, 0, (outs GPR:$Rt, GPR:$Rn),
-                            (ins GPR:$base, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
-                        "ldrsb", "\t$Rt, [$Rn], $addr", "$base = $Rn",
-                            []>;
+                            "ldrsb", "\t$Rt, $addr!", "$addr.base = $Rn_wb",
+                            []> {
+  let AsmMatchConverter = "cvtLdWriteBackRegT2AddrModeImm8";
+}
+def t2LDRSB_POST : T2Ipostldst<1, 0b00, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                          (ins addr_offset_none:$Rn, t2am_imm8_offset:$offset),
+                          AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
+                          "ldrsb", "\t$Rt, $Rn$offset", "$Rn = $Rn_wb", []>;
 
-def t2LDRSH_PRE : T2Iidxldst<1, 0b01, 1, 1, (outs GPR:$Rt, GPR:$Rn),
+def t2LDRSH_PRE : T2Ipreldst<1, 0b01, 1, 1, (outs GPR:$Rt, GPR:$Rn_wb),
                             (ins t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iLoad_bh_iu,
-                            "ldrsh", "\t$Rt, $addr!", "$addr.base = $Rn",
-                            []>;
-def t2LDRSH_POST : T2Iidxldst<1, 0b01, 1, 0, (outs GPR:$dst, GPR:$Rn),
-                            (ins GPR:$base, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
-                        "ldrsh", "\t$dst, [$Rn], $addr", "$base = $Rn",
-                            []>;
+                            "ldrsh", "\t$Rt, $addr!", "$addr.base = $Rn_wb",
+                            []> {
+  let AsmMatchConverter = "cvtLdWriteBackRegT2AddrModeImm8";
+}
+def t2LDRSH_POST : T2Ipostldst<1, 0b01, 1, 0, (outs GPR:$Rt, GPR:$Rn_wb),
+                          (ins addr_offset_none:$Rn, t2am_imm8_offset:$offset),
+                          AddrModeT2_i8, IndexModePost, IIC_iLoad_bh_iu,
+                          "ldrsh", "\t$Rt, $Rn$offset", "$Rn = $Rn_wb", []>;
 } // mayLoad = 1, neverHasSideEffects = 1
 
-// LDRT, LDRBT, LDRHT, LDRSBT, LDRSHT all have offset mode (PUW=0b110) and are
-// for disassembly only.
+// LDRT, LDRBT, LDRHT, LDRSBT, LDRSHT all have offset mode (PUW=0b110).
 // Ref: A8.6.57 LDR (immediate, Thumb) Encoding T4
 class T2IldT<bit signed, bits<2> type, string opc, InstrItinClass ii>
-  : T2Ii8<(outs rGPR:$Rt), (ins t2addrmode_imm8:$addr), ii, opc,
+  : T2Ii8<(outs rGPR:$Rt), (ins t2addrmode_posimm8:$addr), ii, opc,
           "\t$Rt, $addr", []> {
+  bits<4> Rt;
+  bits<13> addr;
   let Inst{31-27} = 0b11111;
   let Inst{26-25} = 0b00;
   let Inst{24} = signed;
   let Inst{23} = 0;
   let Inst{22-21} = type;
   let Inst{20} = 1; // load
+  let Inst{19-16} = addr{12-9};
+  let Inst{15-12} = Rt;
   let Inst{11} = 1;
   let Inst{10-8} = 0b110; // PUW.
-
-  bits<4> Rt;
-  bits<13> addr;
-  let Inst{15-12} = Rt;
-  let Inst{19-16} = addr{12-9};
-  let Inst{7-0}   = addr{7-0};
+  let Inst{7-0} = addr{7-0};
 }
 
 def t2LDRT   : T2IldT<0, 0b10, "ldrt", IIC_iLoad_i>;
@@ -1333,67 +1307,97 @@ def t2LDRSBT : T2IldT<1, 0b00, "ldrsbt", IIC_iLoad_bh_i>;
 def t2LDRSHT : T2IldT<1, 0b01, "ldrsht", IIC_iLoad_bh_i>;
 
 // Store
-defm t2STR :T2I_st<0b10,"str", IIC_iStore_i, IIC_iStore_si,
+defm t2STR :T2I_st<0b10,"str", IIC_iStore_i, IIC_iStore_si, GPR,
                    BinOpFrag<(store node:$LHS, node:$RHS)>>;
 defm t2STRB:T2I_st<0b00,"strb", IIC_iStore_bh_i, IIC_iStore_bh_si,
-                   BinOpFrag<(truncstorei8 node:$LHS, node:$RHS)>>;
+                   rGPR, BinOpFrag<(truncstorei8 node:$LHS, node:$RHS)>>;
 defm t2STRH:T2I_st<0b01,"strh", IIC_iStore_bh_i, IIC_iStore_bh_si,
-                   BinOpFrag<(truncstorei16 node:$LHS, node:$RHS)>>;
+                   rGPR, BinOpFrag<(truncstorei16 node:$LHS, node:$RHS)>>;
 
 // Store doubleword
 let mayLoad = 1, neverHasSideEffects = 1, hasExtraSrcRegAllocReq = 1 in
 def t2STRDi8 : T2Ii8s4<1, 0, 0, (outs),
                        (ins GPR:$Rt, GPR:$Rt2, t2addrmode_imm8s4:$addr),
-               IIC_iStore_d_r, "strd", "\t$Rt, $Rt2, $addr", []>;
+               IIC_iStore_d_r, "strd", "\t$Rt, $Rt2, $addr", "", []>;
 
 // Indexed stores
-def t2STR_PRE  : T2Iidxldst<0, 0b10, 0, 1, (outs GPR:$base_wb),
-                            (ins GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr),
+def t2STR_PRE  : T2Ipreldst<0, 0b10, 0, 1, (outs GPRnopc:$Rn_wb),
+                            (ins rGPR:$Rt, t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iStore_iu,
-                         "str", "\t$Rt, [$Rn, $addr]!",
-                         "$Rn = $base_wb,@earlyclobber $base_wb",
-             [(set GPR:$base_wb,
-                   (pre_store GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr))]>;
-
-def t2STR_POST : T2Iidxldst<0, 0b10, 0, 0, (outs GPR:$base_wb),
-                            (ins GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iStore_iu,
-                          "str", "\t$Rt, [$Rn], $addr",
-                          "$Rn = $base_wb,@earlyclobber $base_wb",
-             [(set GPR:$base_wb,
-                  (post_store GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr))]>;
-
-def t2STRH_PRE  : T2Iidxldst<0, 0b01, 0, 1, (outs GPR:$base_wb),
-                            (ins GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr),
+                            "str", "\t$Rt, $addr!",
+                            "$addr.base = $Rn_wb,@earlyclobber $Rn_wb", []> {
+  let AsmMatchConverter = "cvtStWriteBackRegT2AddrModeImm8";
+}
+def t2STRH_PRE  : T2Ipreldst<0, 0b01, 0, 1, (outs GPRnopc:$Rn_wb),
+                            (ins rGPR:$Rt, t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iStore_iu,
-                        "strh", "\t$Rt, [$Rn, $addr]!",
-                        "$Rn = $base_wb,@earlyclobber $base_wb",
-        [(set GPR:$base_wb,
-              (pre_truncsti16 GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr))]>;
+                        "strh", "\t$Rt, $addr!",
+                        "$addr.base = $Rn_wb,@earlyclobber $Rn_wb", []> {
+  let AsmMatchConverter = "cvtStWriteBackRegT2AddrModeImm8";
+}
 
-def t2STRH_POST : T2Iidxldst<0, 0b01, 0, 0, (outs GPR:$base_wb),
-                            (ins GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr),
-                            AddrModeT2_i8, IndexModePost, IIC_iStore_bh_iu,
-                         "strh", "\t$Rt, [$Rn], $addr",
-                         "$Rn = $base_wb,@earlyclobber $base_wb",
-       [(set GPR:$base_wb,
-             (post_truncsti16 GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr))]>;
-
-def t2STRB_PRE  : T2Iidxldst<0, 0b00, 0, 1, (outs GPR:$base_wb),
-                            (ins GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr),
+def t2STRB_PRE  : T2Ipreldst<0, 0b00, 0, 1, (outs GPRnopc:$Rn_wb),
+                            (ins rGPR:$Rt, t2addrmode_imm8:$addr),
                             AddrModeT2_i8, IndexModePre, IIC_iStore_bh_iu,
-                        "strb", "\t$Rt, [$Rn, $addr]!",
-                        "$Rn = $base_wb,@earlyclobber $base_wb",
-         [(set GPR:$base_wb,
-               (pre_truncsti8 GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr))]>;
+                        "strb", "\t$Rt, $addr!",
+                        "$addr.base = $Rn_wb,@earlyclobber $Rn_wb", []> {
+  let AsmMatchConverter = "cvtStWriteBackRegT2AddrModeImm8";
+}
 
-def t2STRB_POST : T2Iidxldst<0, 0b00, 0, 0, (outs GPR:$base_wb),
-                            (ins GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr),
+def t2STR_POST : T2Ipostldst<0, 0b10, 0, 0, (outs GPRnopc:$Rn_wb),
+                            (ins rGPR:$Rt, addr_offset_none:$Rn,
+                                 t2am_imm8_offset:$offset),
+                            AddrModeT2_i8, IndexModePost, IIC_iStore_iu,
+                          "str", "\t$Rt, $Rn$offset",
+                          "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
+             [(set GPRnopc:$Rn_wb,
+                  (post_store rGPR:$Rt, addr_offset_none:$Rn,
+                              t2am_imm8_offset:$offset))]>;
+
+def t2STRH_POST : T2Ipostldst<0, 0b01, 0, 0, (outs GPRnopc:$Rn_wb),
+                            (ins rGPR:$Rt, addr_offset_none:$Rn,
+                                 t2am_imm8_offset:$offset),
                             AddrModeT2_i8, IndexModePost, IIC_iStore_bh_iu,
-                         "strb", "\t$Rt, [$Rn], $addr",
-                         "$Rn = $base_wb,@earlyclobber $base_wb",
-        [(set GPR:$base_wb,
-              (post_truncsti8 GPR:$Rt, GPR:$Rn, t2am_imm8_offset:$addr))]>;
+                         "strh", "\t$Rt, $Rn$offset",
+                         "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
+       [(set GPRnopc:$Rn_wb,
+             (post_truncsti16 rGPR:$Rt, addr_offset_none:$Rn,
+                              t2am_imm8_offset:$offset))]>;
+
+def t2STRB_POST : T2Ipostldst<0, 0b00, 0, 0, (outs GPRnopc:$Rn_wb),
+                            (ins rGPR:$Rt, addr_offset_none:$Rn,
+                                 t2am_imm8_offset:$offset),
+                            AddrModeT2_i8, IndexModePost, IIC_iStore_bh_iu,
+                         "strb", "\t$Rt, $Rn$offset",
+                         "$Rn = $Rn_wb,@earlyclobber $Rn_wb",
+        [(set GPRnopc:$Rn_wb,
+              (post_truncsti8 rGPR:$Rt, addr_offset_none:$Rn,
+                              t2am_imm8_offset:$offset))]>;
+
+// Pseudo-instructions for pattern matching the pre-indexed stores. We can't
+// put the patterns on the instruction definitions directly as ISel wants
+// the address base and offset to be separate operands, not a single
+// complex operand like we represent the instructions themselves. The
+// pseudos map between the two.
+let usesCustomInserter = 1,
+    Constraints = "$Rn = $Rn_wb,@earlyclobber $Rn_wb" in {
+def t2STR_preidx: t2PseudoInst<(outs GPRnopc:$Rn_wb),
+               (ins rGPR:$Rt, GPRnopc:$Rn, t2am_imm8_offset:$offset, pred:$p),
+               4, IIC_iStore_ru,
+      [(set GPRnopc:$Rn_wb,
+            (pre_store rGPR:$Rt, GPRnopc:$Rn, t2am_imm8_offset:$offset))]>;
+def t2STRB_preidx: t2PseudoInst<(outs GPRnopc:$Rn_wb),
+               (ins rGPR:$Rt, GPRnopc:$Rn, t2am_imm8_offset:$offset, pred:$p),
+               4, IIC_iStore_ru,
+      [(set GPRnopc:$Rn_wb,
+            (pre_truncsti8 rGPR:$Rt, GPRnopc:$Rn, t2am_imm8_offset:$offset))]>;
+def t2STRH_preidx: t2PseudoInst<(outs GPRnopc:$Rn_wb),
+               (ins rGPR:$Rt, GPRnopc:$Rn, t2am_imm8_offset:$offset, pred:$p),
+               4, IIC_iStore_ru,
+      [(set GPRnopc:$Rn_wb,
+            (pre_truncsti16 rGPR:$Rt, GPRnopc:$Rn, t2am_imm8_offset:$offset))]>;
+}
+
 
 // STRT, STRBT, STRHT all have offset mode (PUW=0b110) and are for disassembly
 // only.
@@ -1424,21 +1428,31 @@ def t2STRHT  : T2IstT<0b01, "strht", IIC_iStore_bh_i>;
 // ldrd / strd pre / post variants
 // For disassembly only.
 
-def t2LDRD_PRE  : T2Ii8s4<1, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2),
-                 (ins GPR:$base, t2am_imm8s4_offset:$imm), IIC_iLoad_d_ru,
-                 "ldrd", "\t$Rt, $Rt2, [$base, $imm]!", []>;
+def t2LDRD_PRE  : T2Ii8s4<1, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2, GPR:$wb),
+                 (ins t2addrmode_imm8s4:$addr), IIC_iLoad_d_ru,
+                 "ldrd", "\t$Rt, $Rt2, $addr!", "$addr.base = $wb", []> {
+  let AsmMatchConverter = "cvtT2LdrdPre";
+  let DecoderMethod = "DecodeT2LDRDPreInstruction";
+}
 
-def t2LDRD_POST : T2Ii8s4<0, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2),
-                 (ins GPR:$base, t2am_imm8s4_offset:$imm), IIC_iLoad_d_ru,
-                 "ldrd", "\t$Rt, $Rt2, [$base], $imm", []>;
+def t2LDRD_POST : T2Ii8s4post<0, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2, GPR:$wb),
+                 (ins addr_offset_none:$addr, t2am_imm8s4_offset:$imm),
+                 IIC_iLoad_d_ru, "ldrd", "\t$Rt, $Rt2, $addr$imm",
+                 "$addr.base = $wb", []>;
 
-def t2STRD_PRE  : T2Ii8s4<1, 1, 0, (outs),
-                 (ins rGPR:$Rt, rGPR:$Rt2, GPR:$base, t2am_imm8s4_offset:$imm),
-                 IIC_iStore_d_ru, "strd", "\t$Rt, $Rt2, [$base, $imm]!", []>;
+def t2STRD_PRE  : T2Ii8s4<1, 1, 0, (outs GPR:$wb),
+                 (ins rGPR:$Rt, rGPR:$Rt2, t2addrmode_imm8s4:$addr),
+                 IIC_iStore_d_ru, "strd", "\t$Rt, $Rt2, $addr!",
+                 "$addr.base = $wb", []> {
+  let AsmMatchConverter = "cvtT2StrdPre";
+  let DecoderMethod = "DecodeT2STRDPreInstruction";
+}
 
-def t2STRD_POST : T2Ii8s4<0, 1, 0, (outs),
-                 (ins rGPR:$Rt, rGPR:$Rt2, GPR:$base, t2am_imm8s4_offset:$imm),
-                 IIC_iStore_d_ru, "strd", "\t$Rt, $Rt2, [$base], $imm", []>;
+def t2STRD_POST : T2Ii8s4post<0, 1, 0, (outs GPR:$wb),
+                 (ins rGPR:$Rt, rGPR:$Rt2, addr_offset_none:$addr,
+                      t2am_imm8s4_offset:$imm),
+                 IIC_iStore_d_ru, "strd", "\t$Rt, $Rt2, $addr$imm",
+                 "$addr.base = $wb", []>;
 
 // T2Ipl (Preload Data/Instruction) signals the memory system of possible future
 // data/instruction access.  These are for disassembly only.
@@ -1463,9 +1477,9 @@ multiclass T2Ipl<bits<1> write, bits<1> instr, string opc> {
     let Inst{11-0}  = addr{11-0};  // imm12
   }
 
-  def i8 : T2Ii8<(outs), (ins t2addrmode_imm8:$addr), IIC_Preload, opc,
+  def i8 : T2Ii8<(outs), (ins t2addrmode_negimm8:$addr), IIC_Preload, opc,
                 "\t$addr",
-               [(ARMPreload t2addrmode_imm8:$addr, (i32 write), (i32 instr))]> {
+            [(ARMPreload t2addrmode_negimm8:$addr, (i32 write), (i32 instr))]> {
     let Inst{31-25} = 0b1111100;
     let Inst{24} = instr;
     let Inst{23} = 0; // U = 0
@@ -1496,6 +1510,8 @@ multiclass T2Ipl<bits<1> write, bits<1> instr, string opc> {
     let Inst{19-16} = addr{9-6}; // Rn
     let Inst{3-0}   = addr{5-2}; // Rm
     let Inst{5-4}   = addr{1-0}; // imm2
+
+    let DecoderMethod = "DecodeT2LoadShift";
   }
 }
 
@@ -1507,11 +1523,11 @@ defm t2PLI  : T2Ipl<0, 1, "pli">,  Requires<[IsThumb2,HasV7]>;
 //  Load / store multiple Instructions.
 //
 
-multiclass thumb2_ldst_mult<string asm, InstrItinClass itin,
+multiclass thumb2_ld_mult<string asm, InstrItinClass itin,
                             InstrItinClass itin_upd, bit L_bit> {
   def IA :
     T2XI<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
-         itin, !strconcat(asm, "ia${p}.w\t$Rn, $regs"), []> {
+         itin, !strconcat(asm, "${p}.w\t$Rn, $regs"), []> {
     bits<4>  Rn;
     bits<16> regs;
 
@@ -1522,11 +1538,12 @@ multiclass thumb2_ldst_mult<string asm, InstrItinClass itin,
     let Inst{21}    = 0;        // No writeback
     let Inst{20}    = L_bit;
     let Inst{19-16} = Rn;
-    let Inst{15-0}  = regs;
+    let Inst{15}    = 0;
+    let Inst{14-0}  = regs{14-0};
   }
   def IA_UPD :
     T2XIt<(outs GPR:$wb), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
-          itin_upd, !strconcat(asm, "ia${p}.w\t$Rn!, $regs"), "$Rn = $wb", []> {
+          itin_upd, !strconcat(asm, "${p}.w\t$Rn!, $regs"), "$Rn = $wb", []> {
     bits<4>  Rn;
     bits<16> regs;
 
@@ -1537,11 +1554,12 @@ multiclass thumb2_ldst_mult<string asm, InstrItinClass itin,
     let Inst{21}    = 1;        // Writeback
     let Inst{20}    = L_bit;
     let Inst{19-16} = Rn;
-    let Inst{15-0}  = regs;
+    let Inst{15}    = 0;
+    let Inst{14-0}  = regs{14-0};
   }
   def DB :
     T2XI<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
-         itin, !strconcat(asm, "db${p}.w\t$Rn, $regs"), []> {
+         itin, !strconcat(asm, "db${p}\t$Rn, $regs"), []> {
     bits<4>  Rn;
     bits<16> regs;
 
@@ -1552,11 +1570,12 @@ multiclass thumb2_ldst_mult<string asm, InstrItinClass itin,
     let Inst{21}    = 0;        // No writeback
     let Inst{20}    = L_bit;
     let Inst{19-16} = Rn;
-    let Inst{15-0}  = regs;
+    let Inst{15}    = 0;
+    let Inst{14-0}  = regs{14-0};
   }
   def DB_UPD :
     T2XIt<(outs GPR:$wb), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
-          itin_upd, !strconcat(asm, "db${p}.w\t$Rn, $regs"), "$Rn = $wb", []> {
+          itin_upd, !strconcat(asm, "db${p}\t$Rn!, $regs"), "$Rn = $wb", []> {
     bits<4>  Rn;
     bits<16> regs;
 
@@ -1567,17 +1586,95 @@ multiclass thumb2_ldst_mult<string asm, InstrItinClass itin,
     let Inst{21}    = 1;        // Writeback
     let Inst{20}    = L_bit;
     let Inst{19-16} = Rn;
-    let Inst{15-0}  = regs;
+    let Inst{15}    = 0;
+    let Inst{14-0}  = regs{14-0};
   }
 }
 
 let neverHasSideEffects = 1 in {
 
 let mayLoad = 1, hasExtraDefRegAllocReq = 1 in
-defm t2LDM : thumb2_ldst_mult<"ldm", IIC_iLoad_m, IIC_iLoad_mu, 1>;
+defm t2LDM : thumb2_ld_mult<"ldm", IIC_iLoad_m, IIC_iLoad_mu, 1>;
+
+multiclass thumb2_st_mult<string asm, InstrItinClass itin,
+                            InstrItinClass itin_upd, bit L_bit> {
+  def IA :
+    T2XI<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
+         itin, !strconcat(asm, "${p}.w\t$Rn, $regs"), []> {
+    bits<4>  Rn;
+    bits<16> regs;
+
+    let Inst{31-27} = 0b11101;
+    let Inst{26-25} = 0b00;
+    let Inst{24-23} = 0b01;     // Increment After
+    let Inst{22}    = 0;
+    let Inst{21}    = 0;        // No writeback
+    let Inst{20}    = L_bit;
+    let Inst{19-16} = Rn;
+    let Inst{15}    = 0;
+    let Inst{14}    = regs{14};
+    let Inst{13}    = 0;
+    let Inst{12-0}  = regs{12-0};
+  }
+  def IA_UPD :
+    T2XIt<(outs GPR:$wb), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
+          itin_upd, !strconcat(asm, "${p}.w\t$Rn!, $regs"), "$Rn = $wb", []> {
+    bits<4>  Rn;
+    bits<16> regs;
+
+    let Inst{31-27} = 0b11101;
+    let Inst{26-25} = 0b00;
+    let Inst{24-23} = 0b01;     // Increment After
+    let Inst{22}    = 0;
+    let Inst{21}    = 1;        // Writeback
+    let Inst{20}    = L_bit;
+    let Inst{19-16} = Rn;
+    let Inst{15}    = 0;
+    let Inst{14}    = regs{14};
+    let Inst{13}    = 0;
+    let Inst{12-0}  = regs{12-0};
+  }
+  def DB :
+    T2XI<(outs), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
+         itin, !strconcat(asm, "db${p}\t$Rn, $regs"), []> {
+    bits<4>  Rn;
+    bits<16> regs;
+
+    let Inst{31-27} = 0b11101;
+    let Inst{26-25} = 0b00;
+    let Inst{24-23} = 0b10;     // Decrement Before
+    let Inst{22}    = 0;
+    let Inst{21}    = 0;        // No writeback
+    let Inst{20}    = L_bit;
+    let Inst{19-16} = Rn;
+    let Inst{15}    = 0;
+    let Inst{14}    = regs{14};
+    let Inst{13}    = 0;
+    let Inst{12-0}  = regs{12-0};
+  }
+  def DB_UPD :
+    T2XIt<(outs GPR:$wb), (ins GPR:$Rn, pred:$p, reglist:$regs, variable_ops),
+          itin_upd, !strconcat(asm, "db${p}\t$Rn!, $regs"), "$Rn = $wb", []> {
+    bits<4>  Rn;
+    bits<16> regs;
+
+    let Inst{31-27} = 0b11101;
+    let Inst{26-25} = 0b00;
+    let Inst{24-23} = 0b10;     // Decrement Before
+    let Inst{22}    = 0;
+    let Inst{21}    = 1;        // Writeback
+    let Inst{20}    = L_bit;
+    let Inst{19-16} = Rn;
+    let Inst{15}    = 0;
+    let Inst{14}    = regs{14};
+    let Inst{13}    = 0;
+    let Inst{12-0}  = regs{12-0};
+  }
+}
+
 
 let mayStore = 1, hasExtraSrcRegAllocReq = 1 in
-defm t2STM : thumb2_ldst_mult<"stm", IIC_iStore_m, IIC_iStore_mu, 0>;
+defm t2STM : thumb2_st_mult<"stm", IIC_iStore_m, IIC_iStore_mu, 0>;
 
 } // neverHasSideEffects
 
@@ -1587,7 +1684,7 @@ defm t2STM : thumb2_ldst_mult<"stm", IIC_iStore_m, IIC_iStore_mu, 0>;
 //
 
 let neverHasSideEffects = 1 in
-def t2MOVr : T2sTwoReg<(outs GPR:$Rd), (ins GPR:$Rm), IIC_iMOVr,
+def t2MOVr : T2sTwoReg<(outs GPRnopc:$Rd), (ins GPR:$Rm), IIC_iMOVr,
                    "mov", ".w\t$Rd, $Rm", []> {
   let Inst{31-27} = 0b11101;
   let Inst{26-25} = 0b01;
@@ -1596,6 +1693,10 @@ def t2MOVr : T2sTwoReg<(outs GPR:$Rd), (ins GPR:$Rm), IIC_iMOVr,
   let Inst{14-12} = 0b000;
   let Inst{7-4} = 0b0000;
 }
+def : t2InstAlias<"movs${p}.w $Rd, $Rm", (t2MOVr GPRnopc:$Rd, GPR:$Rm,
+                                                 pred:$p, CPSR)>;
+def : t2InstAlias<"movs${p} $Rd, $Rm", (t2MOVr GPRnopc:$Rd, GPR:$Rm,
+                                               pred:$p, CPSR)>;
 
 // AddedComplexity to ensure isel tries t2MOVi before t2MOVi16.
 let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1,
@@ -1610,12 +1711,20 @@ def t2MOVi : T2sOneRegImm<(outs rGPR:$Rd), (ins t2_so_imm:$imm), IIC_iMOVi,
   let Inst{15} = 0;
 }
 
-def : InstAlias<"mov${s}${p} $Rd, $imm", (t2MOVi rGPR:$Rd, t2_so_imm:$imm,
-                                                 pred:$p, cc_out:$s)>,
-                Requires<[IsThumb2]>;
+// cc_out is handled as part of the explicit mnemonic in the parser for 'mov'.
+// Use aliases to get that to play nice here.
+def : t2InstAlias<"movs${p}.w $Rd, $imm", (t2MOVi rGPR:$Rd, t2_so_imm:$imm,
+                                                pred:$p, CPSR)>;
+def : t2InstAlias<"movs${p} $Rd, $imm", (t2MOVi rGPR:$Rd, t2_so_imm:$imm,
+                                                pred:$p, CPSR)>;
+
+def : t2InstAlias<"mov${p}.w $Rd, $imm", (t2MOVi rGPR:$Rd, t2_so_imm:$imm,
+                                                 pred:$p, zero_reg)>;
+def : t2InstAlias<"mov${p} $Rd, $imm", (t2MOVi rGPR:$Rd, t2_so_imm:$imm,
+                                               pred:$p, zero_reg)>;
 
 let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in
-def t2MOVi16 : T2I<(outs rGPR:$Rd), (ins i32imm_hilo16:$imm), IIC_iMOVi,
+def t2MOVi16 : T2I<(outs rGPR:$Rd), (ins imm0_65535_expr:$imm), IIC_iMOVi,
                    "movw", "\t$Rd, $imm",
                    [(set rGPR:$Rd, imm0_65535:$imm)]> {
   let Inst{31-27} = 0b11110;
@@ -1632,6 +1741,7 @@ def t2MOVi16 : T2I<(outs rGPR:$Rd), (ins i32imm_hilo16:$imm), IIC_iMOVi,
   let Inst{26}    = imm{11};
   let Inst{14-12} = imm{10-8};
   let Inst{7-0}   = imm{7-0};
+  let DecoderMethod = "DecodeT2MOVTWInstruction";
 }
 
 def t2MOVi16_ga_pcrel : PseudoInst<(outs rGPR:$Rd),
@@ -1639,7 +1749,7 @@ def t2MOVi16_ga_pcrel : PseudoInst<(outs rGPR:$Rd),
 
 let Constraints = "$src = $Rd" in {
 def t2MOVTi16 : T2I<(outs rGPR:$Rd),
-                    (ins rGPR:$src, i32imm_hilo16:$imm), IIC_iMOVi,
+                    (ins rGPR:$src, imm0_65535_expr:$imm), IIC_iMOVi,
                     "movt", "\t$Rd, $imm",
                     [(set rGPR:$Rd,
                           (or (and rGPR:$src, 0xffff), lo16AllZero:$imm))]> {
@@ -1657,6 +1767,7 @@ def t2MOVTi16 : T2I<(outs rGPR:$Rd),
   let Inst{26}    = imm{11};
   let Inst{14-12} = imm{10-8};
   let Inst{7-0}   = imm{7-0};
+  let DecoderMethod = "DecodeT2MOVTWInstruction";
 }
 
 def t2MOVTi16_ga_pcrel : PseudoInst<(outs rGPR:$Rd),
@@ -1671,28 +1782,26 @@ def : T2Pat<(or rGPR:$src, 0xffff0000), (t2MOVTi16 rGPR:$src, 0xffff)>;
 
 // Sign extenders
 
-defm t2SXTB  : T2I_ext_rrot<0b100, "sxtb",
+def t2SXTB  : T2I_ext_rrot<0b100, "sxtb",
                               UnOpFrag<(sext_inreg node:$Src, i8)>>;
-defm t2SXTH  : T2I_ext_rrot<0b000, "sxth",
+def t2SXTH  : T2I_ext_rrot<0b000, "sxth",
                               UnOpFrag<(sext_inreg node:$Src, i16)>>;
-defm t2SXTB16 : T2I_ext_rrot_sxtb16<0b010, "sxtb16">;
+def t2SXTB16 : T2I_ext_rrot_sxtb16<0b010, "sxtb16">;
 
-defm t2SXTAB : T2I_exta_rrot<0b100, "sxtab",
+def t2SXTAB : T2I_exta_rrot<0b100, "sxtab",
                         BinOpFrag<(add node:$LHS, (sext_inreg node:$RHS, i8))>>;
-defm t2SXTAH : T2I_exta_rrot<0b000, "sxtah",
+def t2SXTAH : T2I_exta_rrot<0b000, "sxtah",
                         BinOpFrag<(add node:$LHS, (sext_inreg node:$RHS,i16))>>;
-defm t2SXTAB16 : T2I_exta_rrot_DO<0b010, "sxtab16">;
-
-// TODO: SXT(A){B|H}16 - done for disassembly only
+def t2SXTAB16 : T2I_exta_rrot_np<0b010, "sxtab16">;
 
 // Zero extenders
 
 let AddedComplexity = 16 in {
-defm t2UXTB   : T2I_ext_rrot<0b101, "uxtb",
+def t2UXTB   : T2I_ext_rrot<0b101, "uxtb",
                                UnOpFrag<(and node:$Src, 0x000000FF)>>;
-defm t2UXTH   : T2I_ext_rrot<0b001, "uxth",
+def t2UXTH   : T2I_ext_rrot<0b001, "uxth",
                                UnOpFrag<(and node:$Src, 0x0000FFFF)>>;
-defm t2UXTB16 : T2I_ext_rrot_uxtb16<0b011, "uxtb16",
+def t2UXTB16 : T2I_ext_rrot_uxtb16<0b011, "uxtb16",
                                UnOpFrag<(and node:$Src, 0x00FF00FF)>>;
 
 // FIXME: This pattern incorrectly assumes the shl operator is a rotate.
@@ -1700,17 +1809,17 @@ defm t2UXTB16 : T2I_ext_rrot_uxtb16<0b011, "uxtb16",
 //        instead so we can include a check for masking back in the upper
 //        eight bits of the source into the lower eight bits of the result.
 //def : T2Pat<(and (shl rGPR:$Src, (i32 8)), 0xFF00FF),
-//            (t2UXTB16r_rot rGPR:$Src, 24)>,
+//            (t2UXTB16 rGPR:$Src, 3)>,
 //          Requires<[HasT2ExtractPack, IsThumb2]>;
 def : T2Pat<(and (srl rGPR:$Src, (i32 8)), 0xFF00FF),
-            (t2UXTB16r_rot rGPR:$Src, 8)>,
+            (t2UXTB16 rGPR:$Src, 1)>,
         Requires<[HasT2ExtractPack, IsThumb2]>;
 
-defm t2UXTAB : T2I_exta_rrot<0b101, "uxtab",
+def t2UXTAB : T2I_exta_rrot<0b101, "uxtab",
                            BinOpFrag<(add node:$LHS, (and node:$RHS, 0x00FF))>>;
-defm t2UXTAH : T2I_exta_rrot<0b001, "uxtah",
+def t2UXTAH : T2I_exta_rrot<0b001, "uxtah",
                            BinOpFrag<(add node:$LHS, (and node:$RHS, 0xFFFF))>>;
-defm t2UXTAB16 : T2I_exta_rrot_DO<0b011, "uxtab16">;
+def t2UXTAB16 : T2I_exta_rrot_np<0b011, "uxtab16">;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1723,27 +1832,37 @@ defm t2SUB  : T2I_bin_ii12rs<0b101, "sub",
                              BinOpFrag<(sub  node:$LHS, node:$RHS)>>;
 
 // ADD and SUB with 's' bit set. No 12-bit immediate (T4) variants.
+//
+// Currently, t2ADDS/t2SUBS are pseudo opcodes that exist only in the
+// selection DAG. They are "lowered" to real t2ADD/t2SUB opcodes by
+// AdjustInstrPostInstrSelection where we determine whether or not to
+// set the "s" bit based on CPSR liveness.
+//
+// FIXME: Eliminate t2ADDS/t2SUBS pseudo opcodes after adding tablegen
+// support for an optional CPSR definition that corresponds to the DAG
+// node's second value. We can then eliminate the implicit def of CPSR.
 defm t2ADDS : T2I_bin_s_irs <0b1000, "add",
                              IIC_iALUi, IIC_iALUr, IIC_iALUsi,
-                             BinOpFrag<(addc node:$LHS, node:$RHS)>, 1>;
+                             BinOpFrag<(ARMaddc node:$LHS, node:$RHS)>, 1>;
 defm t2SUBS : T2I_bin_s_irs <0b1101, "sub",
                              IIC_iALUi, IIC_iALUr, IIC_iALUsi,
-                             BinOpFrag<(subc node:$LHS, node:$RHS)>>;
+                             BinOpFrag<(ARMsubc node:$LHS, node:$RHS)>>;
 
+let hasPostISelHook = 1 in {
 defm t2ADC  : T2I_adde_sube_irs<0b1010, "adc",
-                          BinOpFrag<(adde_dead_carry node:$LHS, node:$RHS)>, 1>;
+              BinOpWithFlagFrag<(ARMadde node:$LHS, node:$RHS, node:$FLAG)>, 1>;
 defm t2SBC  : T2I_adde_sube_irs<0b1011, "sbc",
-                          BinOpFrag<(sube_dead_carry node:$LHS, node:$RHS)>>;
-defm t2ADCS : T2I_adde_sube_s_irs<BinOpFrag<(adde_live_carry node:$LHS,
-                                                             node:$RHS)>, 1>;
-defm t2SBCS : T2I_adde_sube_s_irs<BinOpFrag<(sube_live_carry node:$LHS,
-                                                             node:$RHS)>>;
+              BinOpWithFlagFrag<(ARMsube node:$LHS, node:$RHS, node:$FLAG)>>;
+}
 
 // RSB
 defm t2RSB  : T2I_rbin_irs  <0b1110, "rsb",
                              BinOpFrag<(sub  node:$LHS, node:$RHS)>>;
+
+// FIXME: Eliminate them if we can write def : Pat patterns which defines
+// CPSR and the implicit def of CPSR is not needed.
 defm t2RSBS : T2I_rbin_s_is <0b1110, "rsb",
-                             BinOpFrag<(subc node:$LHS, node:$RHS)>>;
+                             BinOpFrag<(ARMsubc node:$LHS, node:$RHS)>>;
 
 // (sub X, imm) gets canonicalized to (add X, -imm).  Match this form.
 // The assume-no-carry-in form uses the negation of the input since add/sub
@@ -1760,23 +1879,18 @@ def : T2Pat<(add        GPR:$src, t2_so_imm_neg:$imm),
 def : T2Pat<(add        GPR:$src, imm0_4095_neg:$imm),
             (t2SUBri12  GPR:$src, imm0_4095_neg:$imm)>;
 let AddedComplexity = 1 in
-def : T2Pat<(addc       rGPR:$src, imm0_255_neg:$imm),
+def : T2Pat<(ARMaddc    rGPR:$src, imm0_255_neg:$imm),
             (t2SUBSri   rGPR:$src, imm0_255_neg:$imm)>;
-def : T2Pat<(addc       rGPR:$src, t2_so_imm_neg:$imm),
+def : T2Pat<(ARMaddc    rGPR:$src, t2_so_imm_neg:$imm),
             (t2SUBSri   rGPR:$src, t2_so_imm_neg:$imm)>;
 // The with-carry-in form matches bitwise not instead of the negation.
 // Effectively, the inverse interpretation of the carry flag already accounts
 // for part of the negation.
 let AddedComplexity = 1 in
-def : T2Pat<(adde_dead_carry       rGPR:$src, imm0_255_not:$imm),
+def : T2Pat<(ARMadde    rGPR:$src, imm0_255_not:$imm, CPSR),
             (t2SBCri    rGPR:$src, imm0_255_not:$imm)>;
-def : T2Pat<(adde_dead_carry       rGPR:$src, t2_so_imm_not:$imm),
+def : T2Pat<(ARMadde    rGPR:$src, t2_so_imm_not:$imm, CPSR),
             (t2SBCri    rGPR:$src, t2_so_imm_not:$imm)>;
-let AddedComplexity = 1 in
-def : T2Pat<(adde_live_carry       rGPR:$src, imm0_255_not:$imm),
-            (t2SBCSri   rGPR:$src, imm0_255_not:$imm)>;
-def : T2Pat<(adde_live_carry       rGPR:$src, t2_so_imm_not:$imm),
-            (t2SBCSri   rGPR:$src, t2_so_imm_not:$imm)>;
 
 // Select Bytes -- for disassembly only
 
@@ -1893,8 +2007,7 @@ class T2FourReg_mac<bit long, bits<3> op22_20, bits<4> op7_4, dag oops,
   let Inst{7-4}   = op7_4;
 }
 
-// Unsigned Sum of Absolute Differences [and Accumulate] -- for disassembly only
-
+// Unsigned Sum of Absolute Differences [and Accumulate].
 def t2USAD8   : T2ThreeReg_mac<0, 0b111, 0b0000, (outs rGPR:$Rd),
                                            (ins rGPR:$Rn, rGPR:$Rm),
                         NoItinerary, "usad8", "\t$Rd, $Rn, $Rm", []>,
@@ -1906,8 +2019,7 @@ def t2USADA8  : T2FourReg_mac<0, 0b111, 0b0000, (outs rGPR:$Rd),
                         "usada8", "\t$Rd, $Rn, $Rm, $Ra", []>,
           Requires<[IsThumb2, HasThumb2DSP]>;
 
-// Signed/Unsigned saturate -- for disassembly only
-
+// Signed/Unsigned saturate.
 class T2SatI<dag oops, dag iops, InstrItinClass itin,
            string opc, string asm, list<dag> pattern>
   : T2I<oops, iops, itin, opc, asm, pattern> {
@@ -1918,26 +2030,26 @@ class T2SatI<dag oops, dag iops, InstrItinClass itin,
 
   let Inst{11-8}  = Rd;
   let Inst{19-16} = Rn;
-  let Inst{4-0}   = sat_imm{4-0};
-  let Inst{21}    = sh{6};
+  let Inst{4-0}   = sat_imm;
+  let Inst{21}    = sh{5};
   let Inst{14-12} = sh{4-2};
   let Inst{7-6}   = sh{1-0};
 }
 
 def t2SSAT: T2SatI<
-              (outs rGPR:$Rd), (ins ssat_imm:$sat_imm, rGPR:$Rn, shift_imm:$sh),
-              NoItinerary, "ssat", "\t$Rd, $sat_imm, $Rn$sh",
-              [/* For disassembly only; pattern left blank */]> {
+              (outs rGPR:$Rd),
+              (ins imm1_32:$sat_imm, rGPR:$Rn, t2_shift_imm:$sh),
+              NoItinerary, "ssat", "\t$Rd, $sat_imm, $Rn$sh", []> {
   let Inst{31-27} = 0b11110;
   let Inst{25-22} = 0b1100;
   let Inst{20} = 0;
   let Inst{15} = 0;
+  let Inst{5}  = 0;
 }
 
 def t2SSAT16: T2SatI<
-                (outs rGPR:$Rd), (ins ssat_imm:$sat_imm, rGPR:$Rn), NoItinerary,
-                "ssat16", "\t$Rd, $sat_imm, $Rn",
-                [/* For disassembly only; pattern left blank */]>,
+                (outs rGPR:$Rd), (ins imm1_16:$sat_imm, rGPR:$Rn), NoItinerary,
+                "ssat16", "\t$Rd, $sat_imm, $Rn", []>,
           Requires<[IsThumb2, HasThumb2DSP]> {
   let Inst{31-27} = 0b11110;
   let Inst{25-22} = 0b1100;
@@ -1946,30 +2058,30 @@ def t2SSAT16: T2SatI<
   let Inst{21} = 1;        // sh = '1'
   let Inst{14-12} = 0b000; // imm3 = '000'
   let Inst{7-6} = 0b00;    // imm2 = '00'
+  let Inst{5-4} = 0b00;
 }
 
 def t2USAT: T2SatI<
-                (outs rGPR:$Rd), (ins i32imm:$sat_imm, rGPR:$Rn, shift_imm:$sh),
-                NoItinerary, "usat", "\t$Rd, $sat_imm, $Rn$sh",
-                [/* For disassembly only; pattern left blank */]> {
+               (outs rGPR:$Rd),
+               (ins imm0_31:$sat_imm, rGPR:$Rn, t2_shift_imm:$sh),
+                NoItinerary, "usat", "\t$Rd, $sat_imm, $Rn$sh", []> {
   let Inst{31-27} = 0b11110;
   let Inst{25-22} = 0b1110;
   let Inst{20} = 0;
   let Inst{15} = 0;
 }
 
-def t2USAT16: T2SatI<(outs rGPR:$dst), (ins i32imm:$sat_imm, rGPR:$Rn),
+def t2USAT16: T2SatI<(outs rGPR:$Rd), (ins imm0_15:$sat_imm, rGPR:$Rn),
                      NoItinerary,
-                     "usat16", "\t$dst, $sat_imm, $Rn",
-                     [/* For disassembly only; pattern left blank */]>,
+                     "usat16", "\t$Rd, $sat_imm, $Rn", []>,
           Requires<[IsThumb2, HasThumb2DSP]> {
-  let Inst{31-27} = 0b11110;
-  let Inst{25-22} = 0b1110;
+  let Inst{31-22} = 0b1111001110;
   let Inst{20} = 0;
   let Inst{15} = 0;
   let Inst{21} = 1;        // sh = '1'
   let Inst{14-12} = 0b000; // imm3 = '000'
   let Inst{7-6} = 0b00;    // imm2 = '00'
+  let Inst{5-4} = 0b00;
 }
 
 def : T2Pat<(int_arm_ssat GPR:$a, imm:$pos), (t2SSAT imm:$pos, GPR:$a, 0)>;
@@ -1979,10 +2091,14 @@ def : T2Pat<(int_arm_usat GPR:$a, imm:$pos), (t2USAT imm:$pos, GPR:$a, 0)>;
 //  Shift and rotate Instructions.
 //
 
-defm t2LSL  : T2I_sh_ir<0b00, "lsl", BinOpFrag<(shl  node:$LHS, node:$RHS)>>;
-defm t2LSR  : T2I_sh_ir<0b01, "lsr", BinOpFrag<(srl  node:$LHS, node:$RHS)>>;
-defm t2ASR  : T2I_sh_ir<0b10, "asr", BinOpFrag<(sra  node:$LHS, node:$RHS)>>;
-defm t2ROR  : T2I_sh_ir<0b11, "ror", BinOpFrag<(rotr node:$LHS, node:$RHS)>>;
+defm t2LSL  : T2I_sh_ir<0b00, "lsl", imm0_31,
+                        BinOpFrag<(shl  node:$LHS, node:$RHS)>, "t2LSL">;
+defm t2LSR  : T2I_sh_ir<0b01, "lsr", imm_sr,
+                        BinOpFrag<(srl  node:$LHS, node:$RHS)>, "t2LSR">;
+defm t2ASR  : T2I_sh_ir<0b10, "asr", imm_sr,
+                        BinOpFrag<(sra  node:$LHS, node:$RHS)>, "t2ASR">;
+defm t2ROR  : T2I_sh_ir<0b11, "ror", imm0_31,
+                        BinOpFrag<(rotr node:$LHS, node:$RHS)>, "t2ROR">;
 
 // (rotr x, (and y, 0x...1f)) ==> (ROR x, y)
 def : Pat<(rotr rGPR:$lhs, (and rGPR:$rhs, lo5AllOne)),
@@ -2090,7 +2206,7 @@ def t2BFC : T2BitFI<(outs rGPR:$Rd), (ins rGPR:$src, bf_inv_mask_imm:$imm),
 }
 
 def t2SBFX: T2TwoRegBitFI<
-                (outs rGPR:$Rd), (ins rGPR:$Rn, imm0_31:$lsb, imm0_31_m1:$msb),
+                (outs rGPR:$Rd), (ins rGPR:$Rn, imm0_31:$lsb, imm1_32:$msb),
                  IIC_iUNAsi, "sbfx", "\t$Rd, $Rn, $lsb, $msb", []> {
   let Inst{31-27} = 0b11110;
   let Inst{25} = 1;
@@ -2099,7 +2215,7 @@ def t2SBFX: T2TwoRegBitFI<
 }
 
 def t2UBFX: T2TwoRegBitFI<
-                (outs rGPR:$Rd), (ins rGPR:$Rn, imm0_31:$lsb, imm0_31_m1:$msb),
+                (outs rGPR:$Rd), (ins rGPR:$Rn, imm0_31:$lsb, imm1_32:$msb),
                  IIC_iUNAsi, "ubfx", "\t$Rd, $Rn, $lsb, $msb", []> {
   let Inst{31-27} = 0b11110;
   let Inst{25} = 1;
@@ -2125,26 +2241,6 @@ let Constraints = "$src = $Rd" in {
     let msb{4-0} = imm{9-5};
     let lsb{4-0} = imm{4-0};
   }
-
-  // GNU as only supports this form of bfi (w/ 4 arguments)
-  let isAsmParserOnly = 1 in
-  def t2BFI4p : T2TwoRegBitFI<(outs rGPR:$Rd),
-                  (ins rGPR:$src, rGPR:$Rn, lsb_pos_imm:$lsbit,
-                       width_imm:$width),
-                  IIC_iBITi, "bfi", "\t$Rd, $Rn, $lsbit, $width",
-                  []> {
-    let Inst{31-27} = 0b11110;
-    let Inst{26} = 0; // should be 0.
-    let Inst{25} = 1;
-    let Inst{24-20} = 0b10110;
-    let Inst{15} = 0;
-    let Inst{5} = 0; // should be 0.
-
-    bits<5> lsbit;
-    bits<5> width;
-    let msb{4-0} = width; // Custom encoder => lsb+width-1
-    let lsb{4-0} = lsbit;
-  }
 }
 
 defm t2ORN  : T2I_bin_irs<0b0011, "orn",
@@ -2152,13 +2248,53 @@ defm t2ORN  : T2I_bin_irs<0b0011, "orn",
                           BinOpFrag<(or  node:$LHS, (not node:$RHS))>,
                           "t2ORN", 0, "">;
 
+/// T2I_un_irs - Defines a set of (op reg, {so_imm|r|so_reg}) patterns for a
+/// unary operation that produces a value. These are predicable and can be
+/// changed to modify CPSR.
+multiclass T2I_un_irs<bits<4> opcod, string opc,
+                     InstrItinClass iii, InstrItinClass iir, InstrItinClass iis,
+                      PatFrag opnode, bit Cheap = 0, bit ReMat = 0> {
+   // shifted imm
+   def i : T2sOneRegImm<(outs rGPR:$Rd), (ins t2_so_imm:$imm), iii,
+                opc, "\t$Rd, $imm",
+                [(set rGPR:$Rd, (opnode t2_so_imm:$imm))]> {
+     let isAsCheapAsAMove = Cheap;
+     let isReMaterializable = ReMat;
+     let Inst{31-27} = 0b11110;
+     let Inst{25} = 0;
+     let Inst{24-21} = opcod;
+     let Inst{19-16} = 0b1111; // Rn
+     let Inst{15} = 0;
+   }
+   // register
+   def r : T2sTwoReg<(outs rGPR:$Rd), (ins rGPR:$Rm), iir,
+                opc, ".w\t$Rd, $Rm",
+                [(set rGPR:$Rd, (opnode rGPR:$Rm))]> {
+     let Inst{31-27} = 0b11101;
+     let Inst{26-25} = 0b01;
+     let Inst{24-21} = opcod;
+     let Inst{19-16} = 0b1111; // Rn
+     let Inst{14-12} = 0b000; // imm3
+     let Inst{7-6} = 0b00; // imm2
+     let Inst{5-4} = 0b00; // type
+   }
+   // shifted register
+   def s : T2sOneRegShiftedReg<(outs rGPR:$Rd), (ins t2_so_reg:$ShiftedRm), iis,
+                opc, ".w\t$Rd, $ShiftedRm",
+                [(set rGPR:$Rd, (opnode t2_so_reg:$ShiftedRm))]> {
+     let Inst{31-27} = 0b11101;
+     let Inst{26-25} = 0b01;
+     let Inst{24-21} = opcod;
+     let Inst{19-16} = 0b1111; // Rn
+   }
+}
+
 // Prefer over of t2EORri ra, rb, -1 because mvn has 16-bit version
 let AddedComplexity = 1 in
 defm t2MVN  : T2I_un_irs <0b0011, "mvn",
                           IIC_iMVNi, IIC_iMVNr, IIC_iMVNsi,
                           UnOpFrag<(not node:$Src)>, 1, 1>;
 
-
 let AddedComplexity = 1 in
 def : T2Pat<(and     rGPR:$src, t2_so_imm_not:$imm),
             (t2BICri rGPR:$src, t2_so_imm_not:$imm)>;
@@ -2209,9 +2345,9 @@ def t2MLS: T2FourReg<
 let neverHasSideEffects = 1 in {
 let isCommutable = 1 in {
 def t2SMULL : T2MulLong<0b000, 0b0000,
-                  (outs rGPR:$Rd, rGPR:$Ra),
+                  (outs rGPR:$RdLo, rGPR:$RdHi),
                   (ins rGPR:$Rn, rGPR:$Rm), IIC_iMUL64,
-                   "smull", "\t$Rd, $Ra, $Rn, $Rm", []>;
+                   "smull", "\t$RdLo, $RdHi, $Rn, $Rm", []>;
 
 def t2UMULL : T2MulLong<0b010, 0b0000,
                   (outs rGPR:$RdLo, rGPR:$RdHi),
@@ -2468,7 +2604,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
 defm t2SMUL : T2I_smul<"smul", BinOpFrag<(mul node:$LHS, node:$RHS)>>;
 defm t2SMLA : T2I_smla<"smla", BinOpFrag<(mul node:$LHS, node:$RHS)>>;
 
-// Halfword multiple accumulate long: SMLAL<x><y> -- for disassembly only
+// Halfword multiple accumulate long: SMLAL<x><y>
 def t2SMLALBB : T2FourReg_mac<1, 0b100, 0b1000, (outs rGPR:$Ra,rGPR:$Rd),
          (ins rGPR:$Rn,rGPR:$Rm), IIC_iMAC64, "smlalbb", "\t$Ra, $Rd, $Rn, $Rm",
            [/* For disassembly only; pattern left blank */]>,
@@ -2487,8 +2623,6 @@ def t2SMLALTT : T2FourReg_mac<1, 0b100, 0b1011, (outs rGPR:$Ra,rGPR:$Rd),
           Requires<[IsThumb2, HasThumb2DSP]>;
 
 // Dual halfword multiple: SMUAD, SMUSD, SMLAD, SMLSD, SMLALD, SMLSLD
-// These are for disassembly only.
-
 def t2SMUAD: T2ThreeReg_mac<
             0, 0b010, 0b0000, (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm),
             IIC_iMAC32, "smuad", "\t$Rd, $Rn, $Rm", []>,
@@ -2513,7 +2647,7 @@ def t2SMUSDX:T2ThreeReg_mac<
           Requires<[IsThumb2, HasThumb2DSP]> {
   let Inst{15-12} = 0b1111;
 }
-def t2SMLAD   : T2ThreeReg_mac<
+def t2SMLAD   : T2FourReg_mac<
             0, 0b010, 0b0000, (outs rGPR:$Rd),
             (ins rGPR:$Rn, rGPR:$Rm, rGPR:$Ra), IIC_iMAC32, "smlad",
             "\t$Rd, $Rn, $Rm, $Ra", []>,
@@ -2532,20 +2666,20 @@ def t2SMLSDX  : T2FourReg_mac<0, 0b100, 0b0001, (outs rGPR:$Rd),
             "\t$Rd, $Rn, $Rm, $Ra", []>,
           Requires<[IsThumb2, HasThumb2DSP]>;
 def t2SMLALD  : T2FourReg_mac<1, 0b100, 0b1100, (outs rGPR:$Ra,rGPR:$Rd),
-                        (ins rGPR:$Rm, rGPR:$Rn), IIC_iMAC64, "smlald",
-                        "\t$Ra, $Rd, $Rm, $Rn", []>,
+                        (ins rGPR:$Rn, rGPR:$Rm), IIC_iMAC64, "smlald",
+                        "\t$Ra, $Rd, $Rn, $Rm", []>,
           Requires<[IsThumb2, HasThumb2DSP]>;
 def t2SMLALDX : T2FourReg_mac<1, 0b100, 0b1101, (outs rGPR:$Ra,rGPR:$Rd),
-                        (ins rGPR:$Rm,rGPR:$Rn), IIC_iMAC64, "smlaldx",
-                        "\t$Ra, $Rd, $Rm, $Rn", []>,
+                        (ins rGPR:$Rn,rGPR:$Rm), IIC_iMAC64, "smlaldx",
+                        "\t$Ra, $Rd, $Rn, $Rm", []>,
           Requires<[IsThumb2, HasThumb2DSP]>;
 def t2SMLSLD  : T2FourReg_mac<1, 0b101, 0b1100, (outs rGPR:$Ra,rGPR:$Rd),
-                        (ins rGPR:$Rm,rGPR:$Rn), IIC_iMAC64, "smlsld",
-                        "\t$Ra, $Rd, $Rm, $Rn", []>,
+                        (ins rGPR:$Rn,rGPR:$Rm), IIC_iMAC64, "smlsld",
+                        "\t$Ra, $Rd, $Rn, $Rm", []>,
           Requires<[IsThumb2, HasThumb2DSP]>;
 def t2SMLSLDX : T2FourReg_mac<1, 0b101, 0b1101, (outs rGPR:$Ra,rGPR:$Rd),
                         (ins rGPR:$Rm,rGPR:$Rn), IIC_iMAC64, "smlsldx",
-                        "\t$Ra, $Rd, $Rm, $Rn", []>,
+                        "\t$Ra, $Rd, $Rn, $Rm", []>,
           Requires<[IsThumb2, HasThumb2DSP]>;
 
 //===----------------------------------------------------------------------===//
@@ -2613,10 +2747,10 @@ def : T2Pat<(or (sra (shl rGPR:$Rm, (i32 24)), (i32 16)),
             (t2REVSH rGPR:$Rm)>;
 
 def t2PKHBT : T2ThreeReg<
-            (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, shift_imm:$sh),
+            (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, pkh_lsl_amt:$sh),
                   IIC_iBITsi, "pkhbt", "\t$Rd, $Rn, $Rm$sh",
                   [(set rGPR:$Rd, (or (and rGPR:$Rn, 0xFFFF),
-                                      (and (shl rGPR:$Rm, lsl_amt:$sh),
+                                      (and (shl rGPR:$Rm, pkh_lsl_amt:$sh),
                                            0xFFFF0000)))]>,
                   Requires<[HasT2ExtractPack, IsThumb2]> {
   let Inst{31-27} = 0b11101;
@@ -2625,9 +2759,9 @@ def t2PKHBT : T2ThreeReg<
   let Inst{5} = 0; // BT form
   let Inst{4} = 0;
 
-  bits<8> sh;
-  let Inst{14-12} = sh{7-5};
-  let Inst{7-6}   = sh{4-3};
+  bits<5> sh;
+  let Inst{14-12} = sh{4-2};
+  let Inst{7-6}   = sh{1-0};
 }
 
 // Alternate cases for PKHBT where identities eliminate some nodes.
@@ -2635,16 +2769,16 @@ def : T2Pat<(or (and rGPR:$src1, 0xFFFF), (and rGPR:$src2, 0xFFFF0000)),
             (t2PKHBT rGPR:$src1, rGPR:$src2, 0)>,
             Requires<[HasT2ExtractPack, IsThumb2]>;
 def : T2Pat<(or (and rGPR:$src1, 0xFFFF), (shl rGPR:$src2, imm16_31:$sh)),
-            (t2PKHBT rGPR:$src1, rGPR:$src2, (lsl_shift_imm imm16_31:$sh))>,
+            (t2PKHBT rGPR:$src1, rGPR:$src2, imm16_31:$sh)>,
             Requires<[HasT2ExtractPack, IsThumb2]>;
 
 // Note: Shifts of 1-15 bits will be transformed to srl instead of sra and
 // will match the pattern below.
 def t2PKHTB : T2ThreeReg<
-                  (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, shift_imm:$sh),
+                  (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, pkh_asr_amt:$sh),
                   IIC_iBITsi, "pkhtb", "\t$Rd, $Rn, $Rm$sh",
                   [(set rGPR:$Rd, (or (and rGPR:$Rn, 0xFFFF0000),
-                                       (and (sra rGPR:$Rm, asr_amt:$sh),
+                                       (and (sra rGPR:$Rm, pkh_asr_amt:$sh),
                                             0xFFFF)))]>,
                   Requires<[HasT2ExtractPack, IsThumb2]> {
   let Inst{31-27} = 0b11101;
@@ -2653,19 +2787,19 @@ def t2PKHTB : T2ThreeReg<
   let Inst{5} = 1; // TB form
   let Inst{4} = 0;
 
-  bits<8> sh;
-  let Inst{14-12} = sh{7-5};
-  let Inst{7-6}   = sh{4-3};
+  bits<5> sh;
+  let Inst{14-12} = sh{4-2};
+  let Inst{7-6}   = sh{1-0};
 }
 
 // Alternate cases for PKHTB where identities eliminate some nodes.  Note that
 // a shift amount of 0 is *not legal* here, it is PKHBT instead.
 def : T2Pat<(or (and rGPR:$src1, 0xFFFF0000), (srl rGPR:$src2, imm16_31:$sh)),
-            (t2PKHTB rGPR:$src1, rGPR:$src2, (asr_shift_imm imm16_31:$sh))>,
+            (t2PKHTB rGPR:$src1, rGPR:$src2, imm16_31:$sh)>,
             Requires<[HasT2ExtractPack, IsThumb2]>;
 def : T2Pat<(or (and rGPR:$src1, 0xFFFF0000),
                 (and (srl rGPR:$src2, imm1_15:$sh), 0xFFFF)),
-            (t2PKHTB rGPR:$src1, rGPR:$src2, (asr_shift_imm imm1_15:$sh))>,
+            (t2PKHTB rGPR:$src1, rGPR:$src2, imm1_15:$sh)>,
             Requires<[HasT2ExtractPack, IsThumb2]>;
 
 //===----------------------------------------------------------------------===//
@@ -2673,14 +2807,14 @@ def : T2Pat<(or (and rGPR:$src1, 0xFFFF0000),
 //
 defm t2CMP  : T2I_cmp_irs<0b1101, "cmp",
                           IIC_iCMPi, IIC_iCMPr, IIC_iCMPsi,
-                          BinOpFrag<(ARMcmp node:$LHS, node:$RHS)>>;
+                          BinOpFrag<(ARMcmp node:$LHS, node:$RHS)>, "t2CMP">;
 
-def : T2Pat<(ARMcmpZ  GPR:$lhs, t2_so_imm:$imm),
-            (t2CMPri  GPR:$lhs, t2_so_imm:$imm)>;
-def : T2Pat<(ARMcmpZ  GPR:$lhs, rGPR:$rhs),
-            (t2CMPrr  GPR:$lhs, rGPR:$rhs)>;
-def : T2Pat<(ARMcmpZ  GPR:$lhs, t2_so_reg:$rhs),
-            (t2CMPrs  GPR:$lhs, t2_so_reg:$rhs)>;
+def : T2Pat<(ARMcmpZ  GPRnopc:$lhs, t2_so_imm:$imm),
+            (t2CMPri  GPRnopc:$lhs, t2_so_imm:$imm)>;
+def : T2Pat<(ARMcmpZ  GPRnopc:$lhs, rGPR:$rhs),
+            (t2CMPrr  GPRnopc:$lhs, rGPR:$rhs)>;
+def : T2Pat<(ARMcmpZ  GPRnopc:$lhs, t2_so_reg:$rhs),
+            (t2CMPrs  GPRnopc:$lhs, t2_so_reg:$rhs)>;
 
 //FIXME: Disable CMN, as CCodes are backwards from compare expectations
 //       Compare-to-zero still works out, just not the relationals
@@ -2688,20 +2822,23 @@ def : T2Pat<(ARMcmpZ  GPR:$lhs, t2_so_reg:$rhs),
 //                          BinOpFrag<(ARMcmp node:$LHS,(ineg node:$RHS))>>;
 defm t2CMNz : T2I_cmp_irs<0b1000, "cmn",
                           IIC_iCMPi, IIC_iCMPr, IIC_iCMPsi,
-                          BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>>;
+                          BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>,
+                          "t2CMNz">;
 
 //def : T2Pat<(ARMcmp  GPR:$src, t2_so_imm_neg:$imm),
 //            (t2CMNri GPR:$src, t2_so_imm_neg:$imm)>;
 
-def : T2Pat<(ARMcmpZ  GPR:$src, t2_so_imm_neg:$imm),
-            (t2CMNzri GPR:$src, t2_so_imm_neg:$imm)>;
+def : T2Pat<(ARMcmpZ  GPRnopc:$src, t2_so_imm_neg:$imm),
+            (t2CMNzri GPRnopc:$src, t2_so_imm_neg:$imm)>;
 
 defm t2TST  : T2I_cmp_irs<0b0000, "tst",
                           IIC_iTSTi, IIC_iTSTr, IIC_iTSTsi,
-                         BinOpFrag<(ARMcmpZ (and_su node:$LHS, node:$RHS), 0)>>;
+                         BinOpFrag<(ARMcmpZ (and_su node:$LHS, node:$RHS), 0)>,
+                          "t2TST">;
 defm t2TEQ  : T2I_cmp_irs<0b0100, "teq",
                           IIC_iTSTi, IIC_iTSTr, IIC_iTSTsi,
-                         BinOpFrag<(ARMcmpZ (xor_su node:$LHS, node:$RHS), 0)>>;
+                         BinOpFrag<(ARMcmpZ (xor_su node:$LHS, node:$RHS), 0)>,
+                          "t2TEQ">;
 
 // Conditional moves
 // FIXME: should be able to write a pattern for ARMcmov, but can't use
@@ -2723,7 +2860,7 @@ def t2MOVCCi : t2PseudoInst<(outs rGPR:$Rd),
 // FIXME: Pseudo-ize these. For now, just mark codegen only.
 let isCodeGenOnly = 1 in {
 let isMoveImm = 1 in
-def t2MOVCCi16 : T2I<(outs rGPR:$Rd), (ins rGPR:$false, i32imm_hilo16:$imm),
+def t2MOVCCi16 : T2I<(outs rGPR:$Rd), (ins rGPR:$false, imm0_65535_expr:$imm),
                       IIC_iCMOVi,
                       "movw", "\t$Rd, $imm", []>,
                       RegConstraint<"$false = $Rd"> {
@@ -2807,20 +2944,19 @@ def t2DMB : AInoP<(outs), (ins memb_opt:$opt), ThumbFrm, NoItinerary,
 }
 
 def t2DSB : AInoP<(outs), (ins memb_opt:$opt), ThumbFrm, NoItinerary,
-                  "dsb", "\t$opt",
-                  [/* For disassembly only; pattern left blank */]>,
+                  "dsb", "\t$opt", []>,
                   Requires<[IsThumb, HasDB]> {
   bits<4> opt;
   let Inst{31-4} = 0xf3bf8f4;
   let Inst{3-0} = opt;
 }
 
-// ISB has only full system option -- for disassembly only
-def t2ISB : AInoP<(outs), (ins), ThumbFrm, NoItinerary, "isb", "",
-                  [/* For disassembly only; pattern left blank */]>,
-                  Requires<[IsThumb2, HasV7]> {
+def t2ISB : AInoP<(outs), (ins memb_opt:$opt), ThumbFrm, NoItinerary,
+                  "isb", "\t$opt",
+                  []>, Requires<[IsThumb2, HasDB]> {
+  bits<4> opt;
   let Inst{31-4} = 0xf3bf8f6;
-  let Inst{3-0} = 0b1111;
+  let Inst{3-0} = opt;
 }
 
 class T2I_ldrex<bits<2> opcod, dag oops, dag iops, AddrMode am, int sz,
@@ -2858,28 +2994,27 @@ class T2I_strex<bits<2> opcod, dag oops, dag iops, AddrMode am, int sz,
 }
 
 let mayLoad = 1 in {
-def t2LDREXB : T2I_ldrex<0b00, (outs rGPR:$Rt), (ins t2addrmode_reg:$addr),
+def t2LDREXB : T2I_ldrex<0b00, (outs rGPR:$Rt), (ins addr_offset_none:$addr),
                          AddrModeNone, 4, NoItinerary,
                          "ldrexb", "\t$Rt, $addr", "", []>;
-def t2LDREXH : T2I_ldrex<0b01, (outs rGPR:$Rt), (ins t2addrmode_reg:$addr),
+def t2LDREXH : T2I_ldrex<0b01, (outs rGPR:$Rt), (ins addr_offset_none:$addr),
                          AddrModeNone, 4, NoItinerary,
                          "ldrexh", "\t$Rt, $addr", "", []>;
-def t2LDREX  : Thumb2I<(outs rGPR:$Rt), (ins t2addrmode_reg:$addr),
+def t2LDREX  : Thumb2I<(outs rGPR:$Rt), (ins t2addrmode_imm0_1020s4:$addr),
                        AddrModeNone, 4, NoItinerary,
                        "ldrex", "\t$Rt, $addr", "", []> {
+  bits<4> Rt;
+  bits<12> addr;
   let Inst{31-27} = 0b11101;
   let Inst{26-20} = 0b0000101;
-  let Inst{11-8} = 0b1111;
-  let Inst{7-0} = 0b00000000; // imm8 = 0
-
-  bits<4> Rt;
-  bits<4> addr;
-  let Inst{19-16} = addr;
+  let Inst{19-16} = addr{11-8};
   let Inst{15-12} = Rt;
+  let Inst{11-8} = 0b1111;
+  let Inst{7-0} = addr{7-0};
 }
 let hasExtraDefRegAllocReq = 1 in
 def t2LDREXD : T2I_ldrex<0b11, (outs rGPR:$Rt, rGPR:$Rt2),
-                         (ins t2addrmode_reg:$addr),
+                         (ins addr_offset_none:$addr),
                          AddrModeNone, 4, NoItinerary,
                          "ldrexd", "\t$Rt, $Rt2, $addr", "",
                          [], {?, ?, ?, ?}> {
@@ -2890,33 +3025,33 @@ def t2LDREXD : T2I_ldrex<0b11, (outs rGPR:$Rt, rGPR:$Rt2),
 
 let mayStore = 1, Constraints = "@earlyclobber $Rd" in {
 def t2STREXB : T2I_strex<0b00, (outs rGPR:$Rd),
-                         (ins rGPR:$Rt, t2addrmode_reg:$addr),
+                         (ins rGPR:$Rt, addr_offset_none:$addr),
                          AddrModeNone, 4, NoItinerary,
                          "strexb", "\t$Rd, $Rt, $addr", "", []>;
 def t2STREXH : T2I_strex<0b01, (outs rGPR:$Rd),
-                         (ins rGPR:$Rt, t2addrmode_reg:$addr),
+                         (ins rGPR:$Rt, addr_offset_none:$addr),
                          AddrModeNone, 4, NoItinerary,
                          "strexh", "\t$Rd, $Rt, $addr", "", []>;
-def t2STREX  : Thumb2I<(outs rGPR:$Rd), (ins rGPR:$Rt, t2addrmode_reg:$addr),
+def t2STREX  : Thumb2I<(outs rGPR:$Rd), (ins rGPR:$Rt,
+                             t2addrmode_imm0_1020s4:$addr),
                   AddrModeNone, 4, NoItinerary,
                   "strex", "\t$Rd, $Rt, $addr", "",
                   []> {
+  bits<4> Rd;
+  bits<4> Rt;
+  bits<12> addr;
   let Inst{31-27} = 0b11101;
   let Inst{26-20} = 0b0000100;
-  let Inst{7-0} = 0b00000000; // imm8 = 0
-
-  bits<4> Rd;
-  bits<4> addr;
-  bits<4> Rt;
-  let Inst{11-8}  = Rd;
-  let Inst{19-16} = addr;
+  let Inst{19-16} = addr{11-8};
   let Inst{15-12} = Rt;
+  let Inst{11-8}  = Rd;
+  let Inst{7-0} = addr{7-0};
 }
 }
 
 let hasExtraSrcRegAllocReq = 1, Constraints = "@earlyclobber $Rd" in
 def t2STREXD : T2I_strex<0b11, (outs rGPR:$Rd),
-                         (ins rGPR:$Rt, rGPR:$Rt2, t2addrmode_reg:$addr),
+                         (ins rGPR:$Rt, rGPR:$Rt2, addr_offset_none:$addr),
                          AddrModeNone, 4, NoItinerary,
                          "strexd", "\t$Rd, $Rt, $Rt2, $addr", "", [],
                          {?, ?, ?, ?}> {
@@ -2924,9 +3059,7 @@ def t2STREXD : T2I_strex<0b11, (outs rGPR:$Rd),
   let Inst{11-8} = Rt2;
 }
 
-// Clear-Exclusive is for disassembly only.
-def t2CLREX : T2XI<(outs), (ins), NoItinerary, "clrex",
-                   [/* For disassembly only; pattern left blank */]>,
+def t2CLREX : T2I<(outs), (ins), NoItinerary, "clrex", "", []>,
             Requires<[IsThumb2, HasV7]>  {
   let Inst{31-16} = 0xf3bf;
   let Inst{15-14} = 0b10;
@@ -2986,8 +3119,8 @@ def t2LDMIA_RET: t2PseudoExpand<(outs GPR:$wb), (ins GPR:$Rn, pred:$p,
 
 let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
 let isPredicable = 1 in
-def t2B   : T2XI<(outs), (ins uncondbrtarget:$target), IIC_Br,
-                 "b.w\t$target",
+def t2B   : T2I<(outs), (ins uncondbrtarget:$target), IIC_Br,
+                 "b", ".w\t$target",
                  [(br bb:$target)]> {
   let Inst{31-27} = 0b11110;
   let Inst{15-14} = 0b10;
@@ -3009,15 +3142,13 @@ def t2BR_JT : t2PseudoInst<(outs),
 
 // FIXME: Add a non-pc based case that can be predicated.
 def t2TBB_JT : t2PseudoInst<(outs),
-        (ins GPR:$index, i32imm:$jt, i32imm:$id),
-         0, IIC_Br, []>;
+        (ins GPR:$index, i32imm:$jt, i32imm:$id), 0, IIC_Br, []>;
 
 def t2TBH_JT : t2PseudoInst<(outs),
-        (ins GPR:$index, i32imm:$jt, i32imm:$id),
-         0, IIC_Br, []>;
+        (ins GPR:$index, i32imm:$jt, i32imm:$id), 0, IIC_Br, []>;
 
-def t2TBB : T2I<(outs), (ins GPR:$Rn, GPR:$Rm), IIC_Br,
-                    "tbb", "\t[$Rn, $Rm]", []> {
+def t2TBB : T2I<(outs), (ins addrmode_tbb:$addr), IIC_Br,
+                    "tbb", "\t$addr", []> {
   bits<4> Rn;
   bits<4> Rm;
   let Inst{31-20} = 0b111010001101;
@@ -3025,10 +3156,12 @@ def t2TBB : T2I<(outs), (ins GPR:$Rn, GPR:$Rm), IIC_Br,
   let Inst{15-5} = 0b11110000000;
   let Inst{4} = 0; // B form
   let Inst{3-0} = Rm;
+
+  let DecoderMethod = "DecodeThumbTableBranch";
 }
 
-def t2TBH : T2I<(outs), (ins GPR:$Rn, GPR:$Rm), IIC_Br,
-                   "tbh", "\t[$Rn, $Rm, lsl #1]", []> {
+def t2TBH : T2I<(outs), (ins addrmode_tbh:$addr), IIC_Br,
+                   "tbh", "\t$addr", []> {
   bits<4> Rn;
   bits<4> Rm;
   let Inst{31-20} = 0b111010001101;
@@ -3036,13 +3169,15 @@ def t2TBH : T2I<(outs), (ins GPR:$Rn, GPR:$Rm), IIC_Br,
   let Inst{15-5} = 0b11110000000;
   let Inst{4} = 1; // H form
   let Inst{3-0} = Rm;
+
+  let DecoderMethod = "DecodeThumbTableBranch";
 }
 } // isNotDuplicable, isIndirectBranch
 
 } // isBranch, isTerminator, isBarrier
 
 // FIXME: should be able to write a pattern for ARMBrcond, but can't use
-// a two-value operand where a dag node expects two operands. :(
+// a two-value operand where a dag node expects ", "two operands. :(
 let isBranch = 1, isTerminator = 1 in
 def t2Bcc : T2I<(outs), (ins brtarget:$target), IIC_Br,
                 "b", ".w\t$target",
@@ -3060,6 +3195,8 @@ def t2Bcc : T2I<(outs), (ins brtarget:$target), IIC_Br,
   let Inst{13} = target{18};
   let Inst{21-16} = target{17-12};
   let Inst{10-0} = target{11-1};
+
+  let DecoderMethod = "DecodeThumb2BCCInstruction";
 }
 
 // Tail calls. The Darwin version of thumb tail calls uses a t2 branch, so
@@ -3068,9 +3205,10 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1 in {
   // Darwin version.
   let Defs = [R0, R1, R2, R3, R9, R12, QQQQ0, QQQQ2, QQQQ3, PC],
       Uses = [SP] in
-  def tTAILJMPd: tPseudoExpand<(outs), (ins uncondbrtarget:$dst, variable_ops),
+  def tTAILJMPd: tPseudoExpand<(outs),
+                   (ins uncondbrtarget:$dst, pred:$p, variable_ops),
                    4, IIC_Br, [],
-                   (t2B uncondbrtarget:$dst)>,
+                   (t2B uncondbrtarget:$dst, pred:$p)>,
                  Requires<[IsThumb2, IsDarwin]>;
 }
 
@@ -3087,30 +3225,55 @@ def t2IT : Thumb2XI<(outs), (ins it_pred:$cc, it_mask:$mask),
   bits<4> mask;
   let Inst{7-4} = cc;
   let Inst{3-0} = mask;
+
+  let DecoderMethod = "DecodeIT";
 }
 
 // Branch and Exchange Jazelle -- for disassembly only
 // Rm = Inst{19-16}
-def t2BXJ : T2I<(outs), (ins rGPR:$func), NoItinerary, "bxj", "\t$func",
-              [/* For disassembly only; pattern left blank */]> {
+def t2BXJ : T2I<(outs), (ins rGPR:$func), NoItinerary, "bxj", "\t$func", []> {
+  bits<4> func;
   let Inst{31-27} = 0b11110;
   let Inst{26} = 0;
   let Inst{25-20} = 0b111100;
-  let Inst{15-14} = 0b10;
-  let Inst{12} = 0;
-
-  bits<4> func;
   let Inst{19-16} = func;
+  let Inst{15-0} = 0b1000111100000000;
 }
 
-// Change Processor State is a system instruction -- for disassembly and
-// parsing only.
+// Compare and branch on zero / non-zero
+let isBranch = 1, isTerminator = 1 in {
+  def tCBZ  : T1I<(outs), (ins tGPR:$Rn, t_cbtarget:$target), IIC_Br,
+                  "cbz\t$Rn, $target", []>,
+              T1Misc<{0,0,?,1,?,?,?}>,
+              Requires<[IsThumb2]> {
+    // A8.6.27
+    bits<6> target;
+    bits<3> Rn;
+    let Inst{9}   = target{5};
+    let Inst{7-3} = target{4-0};
+    let Inst{2-0} = Rn;
+  }
+
+  def tCBNZ : T1I<(outs), (ins tGPR:$Rn, t_cbtarget:$target), IIC_Br,
+                  "cbnz\t$Rn, $target", []>,
+              T1Misc<{1,0,?,1,?,?,?}>,
+              Requires<[IsThumb2]> {
+    // A8.6.27
+    bits<6> target;
+    bits<3> Rn;
+    let Inst{9}   = target{5};
+    let Inst{7-3} = target{4-0};
+    let Inst{2-0} = Rn;
+  }
+}
+
+
+// Change Processor State is a system instruction.
 // FIXME: Since the asm parser has currently no clean way to handle optional
 // operands, create 3 versions of the same instruction. Once there's a clean
 // framework to represent optional operands, change this behavior.
 class t2CPS<dag iops, string asm_op> : T2XI<(outs), iops, NoItinerary,
-            !strconcat("cps", asm_op),
-            [/* For disassembly only; pattern left blank */]> {
+            !strconcat("cps", asm_op), []> {
   bits<2> imod;
   bits<3> iflags;
   bits<5> mode;
@@ -3126,6 +3289,7 @@ class t2CPS<dag iops, string asm_op> : T2XI<(outs), iops, NoItinerary,
   let Inst{8}     = M;
   let Inst{7-5}   = iflags;
   let Inst{4-0}   = mode;
+  let DecoderMethod = "DecodeT2CPSInstruction";
 }
 
 let M = 1 in
@@ -3135,14 +3299,12 @@ let mode = 0, M = 0 in
   def t2CPS2p : t2CPS<(ins imod_op:$imod, iflags_op:$iflags),
                       "$imod.w\t$iflags">;
 let imod = 0, iflags = 0, M = 1 in
-  def t2CPS1p : t2CPS<(ins i32imm:$mode), "\t$mode">;
+  def t2CPS1p : t2CPS<(ins imm0_31:$mode), "\t$mode">;
 
 // A6.3.4 Branches and miscellaneous control
 // Table A6-14 Change Processor State, and hint instructions
-// Helper class for disassembly only.
 class T2I_hint<bits<8> op7_0, string opc, string asm>
-  : T2I<(outs), (ins), NoItinerary, opc, asm,
-        [/* For disassembly only; pattern left blank */]> {
+  : T2I<(outs), (ins), NoItinerary, opc, asm, []> {
   let Inst{31-20} = 0xf3a;
   let Inst{19-16} = 0b1111;
   let Inst{15-14} = 0b10;
@@ -3158,20 +3320,17 @@ def t2WFI   : T2I_hint<0b00000011, "wfi",   ".w">;
 def t2SEV   : T2I_hint<0b00000100, "sev",   ".w">;
 
 def t2DBG : T2I<(outs), (ins imm0_15:$opt), NoItinerary, "dbg", "\t$opt", []> {
-  let Inst{31-20} = 0xf3a;
-  let Inst{15-14} = 0b10;
-  let Inst{12} = 0;
-  let Inst{10-8} = 0b000;
-  let Inst{7-4} = 0b1111;
-
   bits<4> opt;
+  let Inst{31-20} = 0b111100111010;
+  let Inst{19-16} = 0b1111;
+  let Inst{15-8} = 0b10000000;
+  let Inst{7-4} = 0b1111;
   let Inst{3-0} = opt;
 }
 
-// Secure Monitor Call is a system instruction -- for disassembly only
+// Secure Monitor Call is a system instruction.
 // Option = Inst{19-16}
-def t2SMC : T2I<(outs), (ins i32imm:$opt), NoItinerary, "smc", "\t$opt",
-                [/* For disassembly only; pattern left blank */]> {
+def t2SMC : T2I<(outs), (ins imm0_15:$opt), NoItinerary, "smc", "\t$opt", []> {
   let Inst{31-27} = 0b11110;
   let Inst{26-20} = 0b1111111;
   let Inst{15-12} = 0b1000;
@@ -3180,32 +3339,30 @@ def t2SMC : T2I<(outs), (ins i32imm:$opt), NoItinerary, "smc", "\t$opt",
   let Inst{19-16} = opt;
 }
 
-class T2SRS<bits<12> op31_20,
-           dag oops, dag iops, InstrItinClass itin,
-          string opc, string asm, list<dag> pattern>
+class T2SRS<bits<2> Op, bit W, dag oops, dag iops, InstrItinClass itin,
+            string opc, string asm, list<dag> pattern>
   : T2I<oops, iops, itin, opc, asm, pattern> {
-  let Inst{31-20} = op31_20{11-0};
-
   bits<5> mode;
+  let Inst{31-25} = 0b1110100;
+  let Inst{24-23} = Op;
+  let Inst{22} = 0;
+  let Inst{21} = W;
+  let Inst{20-16} = 0b01101;
+  let Inst{15-5} = 0b11000000000;
   let Inst{4-0} = mode{4-0};
 }
 
-// Store Return State is a system instruction -- for disassembly only
-def t2SRSDBW : T2SRS<0b111010000010,
-                   (outs),(ins i32imm:$mode),NoItinerary,"srsdb","\tsp!, $mode",
-                   [/* For disassembly only; pattern left blank */]>;
-def t2SRSDB  : T2SRS<0b111010000000,
-                   (outs),(ins i32imm:$mode),NoItinerary,"srsdb","\tsp, $mode",
-                   [/* For disassembly only; pattern left blank */]>;
-def t2SRSIAW : T2SRS<0b111010011010,
-                   (outs),(ins i32imm:$mode),NoItinerary,"srsia","\tsp!, $mode",
-                   [/* For disassembly only; pattern left blank */]>;
-def t2SRSIA  : T2SRS<0b111010011000,
-                   (outs), (ins i32imm:$mode),NoItinerary,"srsia","\tsp, $mode",
-                   [/* For disassembly only; pattern left blank */]>;
-
-// Return From Exception is a system instruction -- for disassembly only
+// Store Return State is a system instruction.
+def t2SRSDB_UPD : T2SRS<0b00, 1, (outs), (ins imm0_31:$mode), NoItinerary,
+                        "srsdb", "\tsp!, $mode", []>;
+def t2SRSDB  : T2SRS<0b00, 0, (outs), (ins imm0_31:$mode), NoItinerary,
+                     "srsdb","\tsp, $mode", []>;
+def t2SRSIA_UPD : T2SRS<0b11, 1, (outs), (ins imm0_31:$mode), NoItinerary,
+                        "srsia","\tsp!, $mode", []>;
+def t2SRSIA  : T2SRS<0b11, 0, (outs), (ins imm0_31:$mode), NoItinerary,
+                     "srsia","\tsp, $mode", []>;
 
+// Return From Exception is a system instruction.
 class T2RFE<bits<12> op31_20, dag oops, dag iops, InstrItinClass itin,
           string opc, string asm, list<dag> pattern>
   : T2I<oops, iops, itin, opc, asm, pattern> {
@@ -3277,53 +3434,186 @@ def t2LDRpci_pic : PseudoInst<(outs rGPR:$dst), (ins i32imm:$addr, pclabel:$cp),
                                            imm:$cp))]>,
                Requires<[IsThumb2]>;
 
+// Pseudo isntruction that combines movs + predicated rsbmi 
+// to implement integer ABS
+let usesCustomInserter = 1, Defs = [CPSR] in {
+def t2ABS : PseudoInst<(outs rGPR:$dst), (ins rGPR:$src),
+                       NoItinerary, []>, Requires<[IsThumb2]>;
+}
+
+//===----------------------------------------------------------------------===//
+// Coprocessor load/store -- for disassembly only
+//
+class T2CI<bits<4> op31_28, dag oops, dag iops, string opc, string asm>
+  : T2I<oops, iops, NoItinerary, opc, asm, []> {
+  let Inst{31-28} = op31_28;
+  let Inst{27-25} = 0b110;
+}
+
+multiclass t2LdStCop<bits<4> op31_28, bit load, bit Dbit, string asm> {
+  def _OFFSET : T2CI<op31_28,
+                     (outs), (ins p_imm:$cop, c_imm:$CRd, addrmode5:$addr),
+                     asm, "\t$cop, $CRd, $addr"> {
+    bits<13> addr;
+    bits<4> cop;
+    bits<4> CRd;
+    let Inst{24} = 1; // P = 1
+    let Inst{23} = addr{8};
+    let Inst{22} = Dbit;
+    let Inst{21} = 0; // W = 0
+    let Inst{20} = load;
+    let Inst{19-16} = addr{12-9};
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = addr{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
+  }
+  def _PRE : T2CI<op31_28,
+                  (outs), (ins p_imm:$cop, c_imm:$CRd, addrmode5:$addr),
+                  asm, "\t$cop, $CRd, $addr!"> {
+    bits<13> addr;
+    bits<4> cop;
+    bits<4> CRd;
+    let Inst{24} = 1; // P = 1
+    let Inst{23} = addr{8};
+    let Inst{22} = Dbit;
+    let Inst{21} = 1; // W = 1
+    let Inst{20} = load;
+    let Inst{19-16} = addr{12-9};
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = addr{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
+  }
+  def _POST: T2CI<op31_28,
+                  (outs), (ins p_imm:$cop, c_imm:$CRd, addr_offset_none:$addr,
+                               postidx_imm8s4:$offset),
+                 asm, "\t$cop, $CRd, $addr, $offset"> {
+    bits<9> offset;
+    bits<4> addr;
+    bits<4> cop;
+    bits<4> CRd;
+    let Inst{24} = 0; // P = 0
+    let Inst{23} = offset{8};
+    let Inst{22} = Dbit;
+    let Inst{21} = 1; // W = 1
+    let Inst{20} = load;
+    let Inst{19-16} = addr;
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = offset{7-0};
+    let DecoderMethod = "DecodeCopMemInstruction";
+  }
+  def _OPTION : T2CI<op31_28, (outs),
+                     (ins p_imm:$cop, c_imm:$CRd, addr_offset_none:$addr,
+                          coproc_option_imm:$option),
+      asm, "\t$cop, $CRd, $addr, $option"> {
+    bits<8> option;
+    bits<4> addr;
+    bits<4> cop;
+    bits<4> CRd;
+    let Inst{24} = 0; // P = 0
+    let Inst{23} = 1; // U = 1
+    let Inst{22} = Dbit;
+    let Inst{21} = 0; // W = 0
+    let Inst{20} = load;
+    let Inst{19-16} = addr;
+    let Inst{15-12} = CRd;
+    let Inst{11-8} = cop;
+    let Inst{7-0} = option;
+    let DecoderMethod = "DecodeCopMemInstruction";
+  }
+}
+
+defm t2LDC   : t2LdStCop<0b1110, 1, 0, "ldc">;
+defm t2LDCL  : t2LdStCop<0b1110, 1, 1, "ldcl">;
+defm t2STC   : t2LdStCop<0b1110, 0, 0, "stc">;
+defm t2STCL  : t2LdStCop<0b1110, 0, 1, "stcl">;
+defm t2LDC2  : t2LdStCop<0b1111, 1, 0, "ldc2">;
+defm t2LDC2L : t2LdStCop<0b1111, 1, 1, "ldc2l">;
+defm t2STC2  : t2LdStCop<0b1111, 0, 0, "stc2">;
+defm t2STC2L : t2LdStCop<0b1111, 0, 1, "stc2l">;
+
+
 //===----------------------------------------------------------------------===//
 // Move between special register and ARM core register -- for disassembly only
 //
+// Move to ARM core register from Special Register
 
-class T2SpecialReg<bits<12> op31_20, bits<2> op15_14, bits<1> op12,
-          dag oops, dag iops, InstrItinClass itin,
-          string opc, string asm, list<dag> pattern>
-  : T2I<oops, iops, itin, opc, asm, pattern> {
-  let Inst{31-20} = op31_20{11-0};
-  let Inst{15-14} = op15_14{1-0};
-  let Inst{12} = op12{0};
+// A/R class MRS.
+//
+// A/R class can only move from CPSR or SPSR.
+def t2MRS_AR : T2I<(outs GPR:$Rd), (ins), NoItinerary, "mrs", "\t$Rd, apsr", []>,
+               Requires<[IsThumb2,IsARClass]> {
+  bits<4> Rd;
+  let Inst{31-12} = 0b11110011111011111000;
+  let Inst{11-8} = Rd;
+  let Inst{7-0} = 0b0000;
 }
 
-class T2MRS<bits<12> op31_20, bits<2> op15_14, bits<1> op12,
-          dag oops, dag iops, InstrItinClass itin,
-          string opc, string asm, list<dag> pattern>
-  : T2SpecialReg<op31_20, op15_14, op12, oops, iops, itin, opc, asm, pattern> {
+def : t2InstAlias<"mrs${p} $Rd, cpsr", (t2MRS_AR GPR:$Rd, pred:$p)>;
+
+def t2MRSsys_AR: T2I<(outs GPR:$Rd), (ins), NoItinerary, "mrs", "\t$Rd, spsr", []>,
+                 Requires<[IsThumb2,IsARClass]> {
   bits<4> Rd;
+  let Inst{31-12} = 0b11110011111111111000;
+  let Inst{11-8} = Rd;
+  let Inst{7-0} = 0b0000;
+}
+
+// M class MRS.
+//
+// This MRS has a mask field in bits 7-0 and can take more values than
+// the A/R class (a full msr_mask).
+def t2MRS_M : T2I<(outs rGPR:$Rd), (ins msr_mask:$mask), NoItinerary,
+                  "mrs", "\t$Rd, $mask", []>,
+              Requires<[IsThumb2,IsMClass]> {
+  bits<4> Rd;
+  bits<8> mask;
+  let Inst{31-12} = 0b11110011111011111000;
   let Inst{11-8} = Rd;
   let Inst{19-16} = 0b1111;
+  let Inst{7-0} = mask;
 }
 
-def t2MRS : T2MRS<0b111100111110, 0b10, 0,
-                (outs rGPR:$Rd), (ins), NoItinerary, "mrs", "\t$Rd, cpsr",
-                [/* For disassembly only; pattern left blank */]>;
-def t2MRSsys : T2MRS<0b111100111111, 0b10, 0,
-                   (outs rGPR:$Rd), (ins), NoItinerary, "mrs", "\t$Rd, spsr",
-                   [/* For disassembly only; pattern left blank */]>;
 
 // Move from ARM core register to Special Register
 //
+// A/R class MSR.
+//
 // No need to have both system and application versions, the encodings are the
 // same and the assembly parser has no way to distinguish between them. The mask
 // operand contains the special register (R Bit) in bit 4 and bits 3-0 contains
 // the mask with the fields to be accessed in the special register.
-def t2MSR : T2SpecialReg<0b111100111000 /* op31-20 */, 0b10 /* op15-14 */,
-                         0 /* op12 */, (outs), (ins msr_mask:$mask, rGPR:$Rn),
-                         NoItinerary, "msr", "\t$mask, $Rn",
-                         [/* For disassembly only; pattern left blank */]> {
+def t2MSR_AR : T2I<(outs), (ins msr_mask:$mask, rGPR:$Rn),
+                   NoItinerary, "msr", "\t$mask, $Rn", []>,
+               Requires<[IsThumb2,IsARClass]> {
   bits<5> mask;
   bits<4> Rn;
-  let Inst{19-16} = Rn;
+  let Inst{31-21} = 0b11110011100;
   let Inst{20}    = mask{4}; // R Bit
-  let Inst{13}    = 0b0;
+  let Inst{19-16} = Rn;
+  let Inst{15-12} = 0b1000;
   let Inst{11-8}  = mask{3-0};
+  let Inst{7-0}   = 0;
 }
 
+// M class MSR.
+//
+// Move from ARM core register to Special Register
+def t2MSR_M : T2I<(outs), (ins msr_mask:$SYSm, rGPR:$Rn),
+                  NoItinerary, "msr", "\t$SYSm, $Rn", []>,
+              Requires<[IsThumb2,IsMClass]> {
+  bits<8> SYSm;
+  bits<4> Rn;
+  let Inst{31-21} = 0b11110011100;
+  let Inst{20}    = 0b0;
+  let Inst{19-16} = Rn;
+  let Inst{15-12} = 0b1000;
+  let Inst{7-0}  = SYSm;
+}
+
+
 //===----------------------------------------------------------------------===//
 // Move between coprocessor and ARM core register
 //
@@ -3389,13 +3679,12 @@ def t2MCR2 : t2MovRCopro<0b1111, "mcr2", 0,
 
 /* from coprocessor to ARM core register */
 def t2MRC : t2MovRCopro<0b1110, "mrc", 1,
-           (outs GPR:$Rt),
-           (ins p_imm:$cop, i32imm:$opc1, c_imm:$CRn, c_imm:$CRm, i32imm:$opc2),
-           []>;
+             (outs GPR:$Rt), (ins p_imm:$cop, imm0_7:$opc1, c_imm:$CRn,
+                                  c_imm:$CRm, imm0_7:$opc2), []>;
 
 def t2MRC2 : t2MovRCopro<0b1111, "mrc2", 1,
-             (outs GPR:$Rt), (ins p_imm:$cop, i32imm:$opc1, c_imm:$CRn,
-                                  c_imm:$CRm, i32imm:$opc2), []>;
+             (outs GPR:$Rt), (ins p_imm:$cop, imm0_7:$opc1, c_imm:$CRn,
+                                  c_imm:$CRm, imm0_7:$opc2), []>;
 
 def : T2v6Pat<(int_arm_mrc  imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2),
               (t2MRC imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
@@ -3465,3 +3754,269 @@ def t2CDP2 : T2Cop<0b1111, (outs), (ins p_imm:$cop, imm0_15:$opc1,
   let Inst{19-16} = CRn;
   let Inst{23-20} = opc1;
 }
+
+
+
+//===----------------------------------------------------------------------===//
+// Non-Instruction Patterns
+//
+
+// SXT/UXT with no rotate
+let AddedComplexity = 16 in {
+def : T2Pat<(and rGPR:$Rm, 0x000000FF), (t2UXTB rGPR:$Rm, 0)>,
+           Requires<[IsThumb2]>;
+def : T2Pat<(and rGPR:$Rm, 0x0000FFFF), (t2UXTH rGPR:$Rm, 0)>,
+           Requires<[IsThumb2]>;
+def : T2Pat<(and rGPR:$Rm, 0x00FF00FF), (t2UXTB16 rGPR:$Rm, 0)>,
+           Requires<[HasT2ExtractPack, IsThumb2]>;
+def : T2Pat<(add rGPR:$Rn, (and rGPR:$Rm, 0x00FF)),
+            (t2UXTAB rGPR:$Rn, rGPR:$Rm, 0)>,
+           Requires<[HasT2ExtractPack, IsThumb2]>;
+def : T2Pat<(add rGPR:$Rn, (and rGPR:$Rm, 0xFFFF)),
+            (t2UXTAH rGPR:$Rn, rGPR:$Rm, 0)>,
+           Requires<[HasT2ExtractPack, IsThumb2]>;
+}
+
+def : T2Pat<(sext_inreg rGPR:$Src, i8),  (t2SXTB rGPR:$Src, 0)>,
+           Requires<[IsThumb2]>;
+def : T2Pat<(sext_inreg rGPR:$Src, i16), (t2SXTH rGPR:$Src, 0)>,
+           Requires<[IsThumb2]>;
+def : T2Pat<(add rGPR:$Rn, (sext_inreg rGPR:$Rm, i8)),
+            (t2SXTAB rGPR:$Rn, rGPR:$Rm, 0)>,
+           Requires<[HasT2ExtractPack, IsThumb2]>;
+def : T2Pat<(add rGPR:$Rn, (sext_inreg rGPR:$Rm, i16)),
+            (t2SXTAH rGPR:$Rn, rGPR:$Rm, 0)>,
+           Requires<[HasT2ExtractPack, IsThumb2]>;
+
+// Atomic load/store patterns
+def : T2Pat<(atomic_load_8   t2addrmode_imm12:$addr),
+            (t2LDRBi12  t2addrmode_imm12:$addr)>;
+def : T2Pat<(atomic_load_8   t2addrmode_negimm8:$addr),
+            (t2LDRBi8   t2addrmode_negimm8:$addr)>;
+def : T2Pat<(atomic_load_8   t2addrmode_so_reg:$addr),
+            (t2LDRBs    t2addrmode_so_reg:$addr)>;
+def : T2Pat<(atomic_load_16  t2addrmode_imm12:$addr),
+            (t2LDRHi12  t2addrmode_imm12:$addr)>;
+def : T2Pat<(atomic_load_16  t2addrmode_negimm8:$addr),
+            (t2LDRHi8   t2addrmode_negimm8:$addr)>;
+def : T2Pat<(atomic_load_16  t2addrmode_so_reg:$addr),
+            (t2LDRHs    t2addrmode_so_reg:$addr)>;
+def : T2Pat<(atomic_load_32  t2addrmode_imm12:$addr),
+            (t2LDRi12   t2addrmode_imm12:$addr)>;
+def : T2Pat<(atomic_load_32  t2addrmode_negimm8:$addr),
+            (t2LDRi8    t2addrmode_negimm8:$addr)>;
+def : T2Pat<(atomic_load_32  t2addrmode_so_reg:$addr),
+            (t2LDRs     t2addrmode_so_reg:$addr)>;
+def : T2Pat<(atomic_store_8  t2addrmode_imm12:$addr, GPR:$val),
+            (t2STRBi12  GPR:$val, t2addrmode_imm12:$addr)>;
+def : T2Pat<(atomic_store_8  t2addrmode_negimm8:$addr, GPR:$val),
+            (t2STRBi8   GPR:$val, t2addrmode_negimm8:$addr)>;
+def : T2Pat<(atomic_store_8  t2addrmode_so_reg:$addr, GPR:$val),
+            (t2STRBs    GPR:$val, t2addrmode_so_reg:$addr)>;
+def : T2Pat<(atomic_store_16 t2addrmode_imm12:$addr, GPR:$val),
+            (t2STRHi12  GPR:$val, t2addrmode_imm12:$addr)>;
+def : T2Pat<(atomic_store_16 t2addrmode_negimm8:$addr, GPR:$val),
+            (t2STRHi8   GPR:$val, t2addrmode_negimm8:$addr)>;
+def : T2Pat<(atomic_store_16 t2addrmode_so_reg:$addr, GPR:$val),
+            (t2STRHs    GPR:$val, t2addrmode_so_reg:$addr)>;
+def : T2Pat<(atomic_store_32 t2addrmode_imm12:$addr, GPR:$val),
+            (t2STRi12   GPR:$val, t2addrmode_imm12:$addr)>;
+def : T2Pat<(atomic_store_32 t2addrmode_negimm8:$addr, GPR:$val),
+            (t2STRi8    GPR:$val, t2addrmode_negimm8:$addr)>;
+def : T2Pat<(atomic_store_32 t2addrmode_so_reg:$addr, GPR:$val),
+            (t2STRs     GPR:$val, t2addrmode_so_reg:$addr)>;
+
+
+//===----------------------------------------------------------------------===//
+// Assembler aliases
+//
+
+// Aliases for ADC without the ".w" optional width specifier.
+def : t2InstAlias<"adc${s}${p} $Rd, $Rn, $Rm",
+                  (t2ADCrr rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"adc${s}${p} $Rd, $Rn, $ShiftedRm",
+                  (t2ADCrs rGPR:$Rd, rGPR:$Rn, t2_so_reg:$ShiftedRm,
+                           pred:$p, cc_out:$s)>;
+
+// Aliases for SBC without the ".w" optional width specifier.
+def : t2InstAlias<"sbc${s}${p} $Rd, $Rn, $Rm",
+                  (t2SBCrr rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"sbc${s}${p} $Rd, $Rn, $ShiftedRm",
+                  (t2SBCrs rGPR:$Rd, rGPR:$Rn, t2_so_reg:$ShiftedRm,
+                           pred:$p, cc_out:$s)>;
+
+// Aliases for ADD without the ".w" optional width specifier.
+def : t2InstAlias<"add${s}${p} $Rd, $Rn, $imm",
+        (t2ADDri GPRnopc:$Rd, GPRnopc:$Rn, t2_so_imm:$imm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"add${p} $Rd, $Rn, $imm",
+           (t2ADDri12 GPRnopc:$Rd, GPR:$Rn, imm0_4095:$imm, pred:$p)>;
+def : t2InstAlias<"add${s}${p} $Rd, $Rn, $Rm",
+              (t2ADDrr GPRnopc:$Rd, GPRnopc:$Rn, rGPR:$Rm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"add${s}${p} $Rd, $Rn, $ShiftedRm",
+                  (t2ADDrs GPRnopc:$Rd, GPRnopc:$Rn, t2_so_reg:$ShiftedRm,
+                           pred:$p, cc_out:$s)>;
+
+// Aliases for SUB without the ".w" optional width specifier.
+def : t2InstAlias<"sub${s}${p} $Rd, $Rn, $imm",
+        (t2SUBri GPRnopc:$Rd, GPRnopc:$Rn, t2_so_imm:$imm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"sub${p} $Rd, $Rn, $imm",
+           (t2SUBri12 GPRnopc:$Rd, GPR:$Rn, imm0_4095:$imm, pred:$p)>;
+def : t2InstAlias<"sub${s}${p} $Rd, $Rn, $Rm",
+              (t2SUBrr GPRnopc:$Rd, GPRnopc:$Rn, rGPR:$Rm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"sub${s}${p} $Rd, $Rn, $ShiftedRm",
+                  (t2SUBrs GPRnopc:$Rd, GPRnopc:$Rn, t2_so_reg:$ShiftedRm,
+                           pred:$p, cc_out:$s)>;
+
+// Alias for compares without the ".w" optional width specifier.
+def : t2InstAlias<"cmn${p} $Rn, $Rm",
+                  (t2CMNzrr GPRnopc:$Rn, rGPR:$Rm, pred:$p)>;
+def : t2InstAlias<"teq${p} $Rn, $Rm",
+                  (t2TEQrr GPRnopc:$Rn, rGPR:$Rm, pred:$p)>;
+def : t2InstAlias<"tst${p} $Rn, $Rm",
+                  (t2TSTrr GPRnopc:$Rn, rGPR:$Rm, pred:$p)>;
+
+// Memory barriers
+def : InstAlias<"dmb", (t2DMB 0xf)>, Requires<[IsThumb2, HasDB]>;
+def : InstAlias<"dsb", (t2DSB 0xf)>, Requires<[IsThumb2, HasDB]>;
+def : InstAlias<"isb", (t2ISB 0xf)>, Requires<[IsThumb2, HasDB]>;
+
+// Alias for LDR, LDRB, LDRH, LDRSB, and LDRSH without the ".w" optional
+// width specifier.
+def : t2InstAlias<"ldr${p} $Rt, $addr",
+                  (t2LDRi12 GPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+def : t2InstAlias<"ldrb${p} $Rt, $addr",
+                  (t2LDRBi12 rGPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+def : t2InstAlias<"ldrh${p} $Rt, $addr",
+                  (t2LDRHi12 rGPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+def : t2InstAlias<"ldrsb${p} $Rt, $addr",
+                  (t2LDRSBi12 rGPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+def : t2InstAlias<"ldrsh${p} $Rt, $addr",
+                  (t2LDRSHi12 rGPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+
+def : t2InstAlias<"ldr${p} $Rt, $addr",
+                  (t2LDRs GPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+def : t2InstAlias<"ldrb${p} $Rt, $addr",
+                  (t2LDRBs rGPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+def : t2InstAlias<"ldrh${p} $Rt, $addr",
+                  (t2LDRHs rGPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+def : t2InstAlias<"ldrsb${p} $Rt, $addr",
+                  (t2LDRSBs rGPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+def : t2InstAlias<"ldrsh${p} $Rt, $addr",
+                  (t2LDRSHs rGPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+
+// Alias for MVN without the ".w" optional width specifier.
+def : t2InstAlias<"mvn${s}${p} $Rd, $Rm",
+           (t2MVNr rGPR:$Rd, rGPR:$Rm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"mvn${s}${p} $Rd, $ShiftedRm",
+           (t2MVNs rGPR:$Rd, t2_so_reg:$ShiftedRm, pred:$p, cc_out:$s)>;
+
+// PKHBT/PKHTB with default shift amount. PKHTB is equivalent to PKHBT when the
+// shift amount is zero (i.e., unspecified).
+def : InstAlias<"pkhbt${p} $Rd, $Rn, $Rm",
+                (t2PKHBT rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>,
+            Requires<[HasT2ExtractPack, IsThumb2]>;
+def : InstAlias<"pkhtb${p} $Rd, $Rn, $Rm",
+                (t2PKHBT rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>,
+            Requires<[HasT2ExtractPack, IsThumb2]>;
+
+// PUSH/POP aliases for STM/LDM
+def : t2InstAlias<"push${p}.w $regs", (t2STMDB_UPD SP, pred:$p, reglist:$regs)>;
+def : t2InstAlias<"push${p} $regs", (t2STMDB_UPD SP, pred:$p, reglist:$regs)>;
+def : t2InstAlias<"pop${p}.w $regs", (t2LDMIA_UPD SP, pred:$p, reglist:$regs)>;
+def : t2InstAlias<"pop${p} $regs", (t2LDMIA_UPD SP, pred:$p, reglist:$regs)>;
+
+// Alias for REV/REV16/REVSH without the ".w" optional width specifier.
+def : t2InstAlias<"rev${p} $Rd, $Rm", (t2REV rGPR:$Rd, rGPR:$Rm, pred:$p)>;
+def : t2InstAlias<"rev16${p} $Rd, $Rm", (t2REV16 rGPR:$Rd, rGPR:$Rm, pred:$p)>;
+def : t2InstAlias<"revsh${p} $Rd, $Rm", (t2REVSH rGPR:$Rd, rGPR:$Rm, pred:$p)>;
+
+
+// Alias for RSB without the ".w" optional width specifier, and with optional
+// implied destination register.
+def : t2InstAlias<"rsb${s}${p} $Rd, $Rn, $imm",
+           (t2RSBri rGPR:$Rd, rGPR:$Rn, t2_so_imm:$imm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"rsb${s}${p} $Rdn, $imm",
+           (t2RSBri rGPR:$Rdn, rGPR:$Rdn, t2_so_imm:$imm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"rsb${s}${p} $Rdn, $Rm",
+           (t2RSBrr rGPR:$Rdn, rGPR:$Rdn, rGPR:$Rm, pred:$p, cc_out:$s)>;
+def : t2InstAlias<"rsb${s}${p} $Rdn, $ShiftedRm",
+           (t2RSBrs rGPR:$Rdn, rGPR:$Rdn, t2_so_reg:$ShiftedRm, pred:$p,
+                    cc_out:$s)>;
+
+// SSAT/USAT optional shift operand.
+def : t2InstAlias<"ssat${p} $Rd, $sat_imm, $Rn",
+                  (t2SSAT rGPR:$Rd, imm1_32:$sat_imm, rGPR:$Rn, 0, pred:$p)>;
+def : t2InstAlias<"usat${p} $Rd, $sat_imm, $Rn",
+                  (t2USAT rGPR:$Rd, imm0_31:$sat_imm, rGPR:$Rn, 0, pred:$p)>;
+
+// STM w/o the .w suffix.
+def : t2InstAlias<"stm${p} $Rn, $regs",
+                  (t2STMIA GPR:$Rn, pred:$p, reglist:$regs)>;
+
+// Alias for STR, STRB, and STRH without the ".w" optional
+// width specifier.
+def : t2InstAlias<"str${p} $Rt, $addr",
+                  (t2STRi12 GPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+def : t2InstAlias<"strb${p} $Rt, $addr",
+                  (t2STRBi12 rGPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+def : t2InstAlias<"strh${p} $Rt, $addr",
+                  (t2STRHi12 rGPR:$Rt, t2addrmode_imm12:$addr, pred:$p)>;
+
+def : t2InstAlias<"str${p} $Rt, $addr",
+                  (t2STRs GPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+def : t2InstAlias<"strb${p} $Rt, $addr",
+                  (t2STRBs rGPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+def : t2InstAlias<"strh${p} $Rt, $addr",
+                  (t2STRHs rGPR:$Rt, t2addrmode_so_reg:$addr, pred:$p)>;
+
+// Extend instruction optional rotate operand.
+def : t2InstAlias<"sxtab${p} $Rd, $Rn, $Rm",
+                (t2SXTAB rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"sxtah${p} $Rd, $Rn, $Rm",
+                (t2SXTAH rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"sxtab16${p} $Rd, $Rn, $Rm",
+                (t2SXTAB16 rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>;
+
+def : t2InstAlias<"sxtb${p} $Rd, $Rm",
+                (t2SXTB rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"sxtb16${p} $Rd, $Rm",
+                (t2SXTB16 rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"sxth${p} $Rd, $Rm",
+                (t2SXTH rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"sxtb${p}.w $Rd, $Rm",
+                (t2SXTB rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"sxth${p}.w $Rd, $Rm",
+                (t2SXTH rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+
+def : t2InstAlias<"uxtab${p} $Rd, $Rn, $Rm",
+                (t2UXTAB rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"uxtah${p} $Rd, $Rn, $Rm",
+                (t2UXTAH rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"uxtab16${p} $Rd, $Rn, $Rm",
+                (t2UXTAB16 rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"uxtb${p} $Rd, $Rm",
+                (t2UXTB rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"uxtb16${p} $Rd, $Rm",
+                (t2UXTB16 rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"uxth${p} $Rd, $Rm",
+                (t2UXTH rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+
+def : t2InstAlias<"uxtb${p}.w $Rd, $Rm",
+                (t2UXTB rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+def : t2InstAlias<"uxth${p}.w $Rd, $Rm",
+                (t2UXTH rGPR:$Rd, rGPR:$Rm, 0, pred:$p)>;
+
+// Extend instruction w/o the ".w" optional width specifier.
+def : t2InstAlias<"uxtb${p} $Rd, $Rm$rot",
+                  (t2UXTB rGPR:$Rd, rGPR:$Rm, rot_imm:$rot, pred:$p)>;
+def : t2InstAlias<"uxtb16${p} $Rd, $Rm$rot",
+                  (t2UXTB16 rGPR:$Rd, rGPR:$Rm, rot_imm:$rot, pred:$p)>;
+def : t2InstAlias<"uxth${p} $Rd, $Rm$rot",
+                  (t2UXTH rGPR:$Rd, rGPR:$Rm, rot_imm:$rot, pred:$p)>;
+
+def : t2InstAlias<"sxtb${p} $Rd, $Rm$rot",
+                  (t2SXTB rGPR:$Rd, rGPR:$Rm, rot_imm:$rot, pred:$p)>;
+def : t2InstAlias<"sxtb16${p} $Rd, $Rm$rot",
+                  (t2SXTB16 rGPR:$Rd, rGPR:$Rm, rot_imm:$rot, pred:$p)>;
+def : t2InstAlias<"sxth${p} $Rd, $Rm$rot",
+                  (t2SXTH rGPR:$Rd, rGPR:$Rm, rot_imm:$rot, pred:$p)>;
diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrVFP.td b/contrib/llvm/lib/Target/ARM/ARMInstrVFP.td
index f1f3cb9c2ecd..e746cf20d032 100644
--- a/contrib/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/contrib/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -31,18 +31,34 @@ def arm_fmdrr  : SDNode<"ARMISD::VMOVDRR", SDT_VMOVDRR>;
 // Operand Definitions.
 //
 
+// 8-bit floating-point immediate encodings.
+def FPImmOperand : AsmOperandClass {
+  let Name = "FPImm";
+  let ParserMethod = "parseFPImm";
+}
+
 def vfp_f32imm : Operand<f32>,
                  PatLeaf<(f32 fpimm), [{
-      return ARM::getVFPf32Imm(N->getValueAPF()) != -1;
-    }]> {
-  let PrintMethod = "printVFPf32ImmOperand";
+      return ARM_AM::getFP32Imm(N->getValueAPF()) != -1;
+    }], SDNodeXForm<fpimm, [{
+      APFloat InVal = N->getValueAPF();
+      uint32_t enc = ARM_AM::getFP32Imm(InVal);
+      return CurDAG->getTargetConstant(enc, MVT::i32);
+    }]>> {
+  let PrintMethod = "printFPImmOperand";
+  let ParserMatchClass = FPImmOperand;
 }
 
 def vfp_f64imm : Operand<f64>,
                  PatLeaf<(f64 fpimm), [{
-      return ARM::getVFPf64Imm(N->getValueAPF()) != -1;
-    }]> {
-  let PrintMethod = "printVFPf64ImmOperand";
+      return ARM_AM::getFP64Imm(N->getValueAPF()) != -1;
+    }], SDNodeXForm<fpimm, [{
+      APFloat InVal = N->getValueAPF();
+      uint32_t enc = ARM_AM::getFP64Imm(InVal);
+      return CurDAG->getTargetConstant(enc, MVT::i32);
+    }]>> {
+  let PrintMethod = "printFPImmOperand";
+  let ParserMatchClass = FPImmOperand;
 }
 
 
@@ -385,26 +401,26 @@ def VCVTSD  : VFPAI<(outs SPR:$Sd), (ins DPR:$Dm), VFPUnaryFrm,
 // Between half-precision and single-precision.  For disassembly only.
 
 // FIXME: Verify encoding after integrated assembler is working.
-def VCVTBSH: ASuI<0b11101, 0b11, 0b0010, 0b01, 0, (outs SPR:$dst), (ins SPR:$a),
-                 /* FIXME */ IIC_fpCVTSH, "vcvtb", ".f32.f16\t$dst, $a",
+def VCVTBSH: ASuI<0b11101, 0b11, 0b0010, 0b01, 0, (outs SPR:$Sd), (ins SPR:$Sm),
+                 /* FIXME */ IIC_fpCVTSH, "vcvtb", ".f32.f16\t$Sd, $Sm",
                  [/* For disassembly only; pattern left blank */]>;
 
 def : ARMPat<(f32_to_f16 SPR:$a),
              (i32 (COPY_TO_REGCLASS (VCVTBSH SPR:$a), GPR))>;
 
-def VCVTBHS: ASuI<0b11101, 0b11, 0b0011, 0b01, 0, (outs SPR:$dst), (ins SPR:$a),
-                 /* FIXME */ IIC_fpCVTHS, "vcvtb", ".f16.f32\t$dst, $a",
+def VCVTBHS: ASuI<0b11101, 0b11, 0b0011, 0b01, 0, (outs SPR:$Sd), (ins SPR:$Sm),
+                 /* FIXME */ IIC_fpCVTHS, "vcvtb", ".f16.f32\t$Sd, $Sm",
                  [/* For disassembly only; pattern left blank */]>;
 
 def : ARMPat<(f16_to_f32 GPR:$a),
              (VCVTBHS (COPY_TO_REGCLASS GPR:$a, SPR))>;
 
-def VCVTTSH: ASuI<0b11101, 0b11, 0b0010, 0b11, 0, (outs SPR:$dst), (ins SPR:$a),
-                 /* FIXME */ IIC_fpCVTSH, "vcvtt", ".f32.f16\t$dst, $a",
+def VCVTTSH: ASuI<0b11101, 0b11, 0b0010, 0b11, 0, (outs SPR:$Sd), (ins SPR:$Sm),
+                 /* FIXME */ IIC_fpCVTSH, "vcvtt", ".f32.f16\t$Sd, $Sm",
                  [/* For disassembly only; pattern left blank */]>;
 
-def VCVTTHS: ASuI<0b11101, 0b11, 0b0011, 0b11, 0, (outs SPR:$dst), (ins SPR:$a),
-                 /* FIXME */ IIC_fpCVTHS, "vcvtt", ".f16.f32\t$dst, $a",
+def VCVTTHS: ASuI<0b11101, 0b11, 0b0011, 0b11, 0, (outs SPR:$Sd), (ins SPR:$Sm),
+                 /* FIXME */ IIC_fpCVTHS, "vcvtt", ".f16.f32\t$Sd, $Sm",
                  [/* For disassembly only; pattern left blank */]>;
 
 def VNEGD  : ADuI<0b11101, 0b11, 0b0001, 0b01, 0,
@@ -511,14 +527,25 @@ def VMOVRRD  : AVConv3I<0b11000101, 0b1011,
 }
 
 def VMOVRRS  : AVConv3I<0b11000101, 0b1010,
-                      (outs GPR:$wb, GPR:$dst2), (ins SPR:$src1, SPR:$src2),
-                 IIC_fpMOVDI, "vmov", "\t$wb, $dst2, $src1, $src2",
+                      (outs GPR:$Rt, GPR:$Rt2), (ins SPR:$src1, SPR:$src2),
+                 IIC_fpMOVDI, "vmov", "\t$Rt, $Rt2, $src1, $src2",
                  [/* For disassembly only; pattern left blank */]> {
+  bits<5> src1;
+  bits<4> Rt;
+  bits<4> Rt2;
+
+  // Encode instruction operands.
+  let Inst{3-0}   = src1{3-0};
+  let Inst{5}     = src1{4};
+  let Inst{15-12} = Rt;
+  let Inst{19-16} = Rt2;
+
   let Inst{7-6} = 0b00;
 
   // Some single precision VFP instructions may be executed on both NEON and VFP
   // pipelines.
   let D = VFPNeonDomain;
+  let DecoderMethod = "DecodeVMOVRRS";
 }
 } // neverHasSideEffects
 
@@ -552,11 +579,24 @@ def VMOVSRR : AVConv5I<0b11000100, 0b1010,
                      (outs SPR:$dst1, SPR:$dst2), (ins GPR:$src1, GPR:$src2),
                 IIC_fpMOVID, "vmov", "\t$dst1, $dst2, $src1, $src2",
                 [/* For disassembly only; pattern left blank */]> {
+  // Instruction operands.
+  bits<5> dst1;
+  bits<4> src1;
+  bits<4> src2;
+
+  // Encode instruction operands.
+  let Inst{3-0}   = dst1{3-0};
+  let Inst{5}     = dst1{4};
+  let Inst{15-12} = src1;
+  let Inst{19-16} = src2;
+
   let Inst{7-6} = 0b00;
 
   // Some single precision VFP instructions may be executed on both NEON and VFP
   // pipelines.
   let D = VFPNeonDomain;
+
+  let DecoderMethod = "DecodeVMOVSRR";
 }
 
 // FMRDH: SPR -> GPR
@@ -1084,45 +1124,42 @@ def FCONSTD : VFPAI<(outs DPR:$Dd), (ins vfp_f64imm:$imm),
                     VFPMiscFrm, IIC_fpUNA64,
                     "vmov", ".f64\t$Dd, $imm",
                     [(set DPR:$Dd, vfp_f64imm:$imm)]>, Requires<[HasVFP3]> {
-  // Instruction operands.
-  bits<5>  Dd;
-  bits<32> imm;
+  bits<5> Dd;
+  bits<8> imm;
 
-  // Encode instruction operands.
-  let Inst{15-12} = Dd{3-0};
-  let Inst{22}    = Dd{4};
-  let Inst{19}    = imm{31};
-  let Inst{18-16} = imm{22-20};
-  let Inst{3-0}   = imm{19-16};
-
-  // Encode remaining instruction bits.
   let Inst{27-23} = 0b11101;
+  let Inst{22}    = Dd{4};
   let Inst{21-20} = 0b11;
+  let Inst{19-16} = imm{7-4};
+  let Inst{15-12} = Dd{3-0};
   let Inst{11-9}  = 0b101;
   let Inst{8}     = 1;          // Double precision.
   let Inst{7-4}   = 0b0000;
+  let Inst{3-0}   = imm{3-0};
 }
 
 def FCONSTS : VFPAI<(outs SPR:$Sd), (ins vfp_f32imm:$imm),
                      VFPMiscFrm, IIC_fpUNA32,
                      "vmov", ".f32\t$Sd, $imm",
                      [(set SPR:$Sd, vfp_f32imm:$imm)]>, Requires<[HasVFP3]> {
-  // Instruction operands.
-  bits<5>  Sd;
-  bits<32> imm;
+  bits<5> Sd;
+  bits<8> imm;
 
-  // Encode instruction operands.
-  let Inst{15-12} = Sd{4-1};
-  let Inst{22}    = Sd{0};
-  let Inst{19}    = imm{31};    // The immediate is handled as a double.
-  let Inst{18-16} = imm{22-20};
-  let Inst{3-0}   = imm{19-16};
-
-  // Encode remaining instruction bits.
   let Inst{27-23} = 0b11101;
+  let Inst{22}    = Sd{0};
   let Inst{21-20} = 0b11;
+  let Inst{19-16} = imm{7-4};
+  let Inst{15-12} = Sd{4-1};
   let Inst{11-9}  = 0b101;
   let Inst{8}     = 0;          // Single precision.
   let Inst{7-4}   = 0b0000;
+  let Inst{3-0}   = imm{3-0};
 }
 }
+
+//===----------------------------------------------------------------------===//
+// Assembler aliases.
+//
+
+def : VFP2InstAlias<"fmstat${p}", (FMSTAT pred:$p)>;
+
diff --git a/contrib/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/contrib/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index c6efea1d7806..faa8ba76845e 100644
--- a/contrib/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -14,10 +14,10 @@
 
 #define DEBUG_TYPE "arm-ldst-opt"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMRegisterInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Function.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
@@ -763,9 +764,9 @@ static unsigned getPreIndexedLoadStoreOpcode(unsigned Opc,
                                              ARM_AM::AddrOpc Mode) {
   switch (Opc) {
   case ARM::LDRi12:
-    return ARM::LDR_PRE;
+    return ARM::LDR_PRE_IMM;
   case ARM::STRi12:
-    return ARM::STR_PRE;
+    return ARM::STR_PRE_IMM;
   case ARM::VLDRS:
     return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
   case ARM::VLDRD:
@@ -789,9 +790,9 @@ static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc,
                                               ARM_AM::AddrOpc Mode) {
   switch (Opc) {
   case ARM::LDRi12:
-    return ARM::LDR_POST;
+    return ARM::LDR_POST_IMM;
   case ARM::STRi12:
-    return ARM::STR_POST;
+    return ARM::STR_POST_IMM;
   case ARM::VLDRS:
     return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
   case ARM::VLDRD:
@@ -892,12 +893,6 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
   if (!DoMerge)
     return false;
 
-  unsigned Offset = 0;
-  if (isAM2)
-    Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
-  else if (!isAM5)
-    Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
-
   if (isAM5) {
     // VLDM[SD}_UPD, VSTM[SD]_UPD
     // (There are no base-updating versions of VLDR/VSTR instructions, but the
@@ -911,28 +906,44 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
       .addReg(MO.getReg(), (isLd ? getDefRegState(true) :
                             getKillRegState(MO.isKill())));
   } else if (isLd) {
-    if (isAM2)
-      // LDR_PRE, LDR_POST,
-      BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
-        .addReg(Base, RegState::Define)
-        .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
-    else
+    if (isAM2) {
+      // LDR_PRE, LDR_POST
+      if (NewOpc == ARM::LDR_PRE_IMM || NewOpc == ARM::LDRB_PRE_IMM) {
+        int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
+        BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
+          .addReg(Base, RegState::Define)
+          .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
+      } else {
+        int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
+        BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
+          .addReg(Base, RegState::Define)
+          .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
+      }
+    } else {
+      int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
       // t2LDR_PRE, t2LDR_POST
       BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
         .addReg(Base, RegState::Define)
         .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
+    }
   } else {
     MachineOperand &MO = MI->getOperand(0);
-    if (isAM2)
+    // FIXME: post-indexed stores use am2offset_imm, which still encodes
+    // the vestigal zero-reg offset register. When that's fixed, this clause
+    // can be removed entirely.
+    if (isAM2 && NewOpc == ARM::STR_POST_IMM) {
+      int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
       // STR_PRE, STR_POST
       BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
         .addReg(MO.getReg(), getKillRegState(MO.isKill()))
         .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
-    else
+    } else {
+      int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
       // t2STR_PRE, t2STR_POST
       BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
         .addReg(MO.getReg(), getKillRegState(MO.isKill()))
         .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
+    }
   }
   MBB.erase(MBBI);
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMMCInstLower.cpp b/contrib/llvm/lib/Target/ARM/ARMMCInstLower.cpp
index 7411b599f0fa..daa126def401 100644
--- a/contrib/llvm/lib/Target/ARM/ARMMCInstLower.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMMCInstLower.cpp
@@ -14,7 +14,7 @@
 
 #include "ARM.h"
 #include "ARMAsmPrinter.h"
-#include "ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCExpr.h"
 #include "llvm/Constants.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/MC/MCExpr.h"
diff --git a/contrib/llvm/lib/Target/ARM/ARMRegisterInfo.td b/contrib/llvm/lib/Target/ARM/ARMRegisterInfo.td
index 76eb496bde42..036822d18ad2 100644
--- a/contrib/llvm/lib/Target/ARM/ARMRegisterInfo.td
+++ b/contrib/llvm/lib/Target/ARM/ARMRegisterInfo.td
@@ -182,8 +182,10 @@ def QQQQ3 : ARMReg<3, "qqqq3", [QQ6, QQ7]>;
 
 // Current Program Status Register.
 def CPSR    : ARMReg<0, "cpsr">;
-def FPSCR   : ARMReg<1, "fpscr">;
-def ITSTATE : ARMReg<2, "itstate">;
+def APSR    : ARMReg<1, "apsr">;
+def SPSR    : ARMReg<2, "spsr">;
+def FPSCR   : ARMReg<3, "fpscr">;
+def ITSTATE : ARMReg<4, "itstate">;
 
 // Special Registers - only available in privileged mode.
 def FPSID   : ARMReg<0, "fpsid">;
@@ -213,6 +215,23 @@ def GPR : RegisterClass<"ARM", [i32], 32, (add (sequence "R%u", 0, 12),
   }];
 }
 
+// GPRs without the PC.  Some ARM instructions do not allow the PC in
+// certain operand slots, particularly as the destination.  Primarily
+// useful for disassembly.
+def GPRnopc : RegisterClass<"ARM", [i32], 32, (sub GPR, PC)> {
+  let AltOrders = [(add LR, GPRnopc), (trunc GPRnopc, 8)];
+  let AltOrderSelect = [{
+      return 1 + MF.getTarget().getSubtarget<ARMSubtarget>().isThumb1Only();
+  }];
+}
+
+// GPRsp - Only the SP is legal. Used by Thumb1 instructions that want the
+// implied SP argument list.
+// FIXME: It would be better to not use this at all and refactor the
+// instructions to not have SP an an explicit argument. That makes
+// frame index resolution a bit trickier, though.
+def GPRsp : RegisterClass<"ARM", [i32], 32, (add SP)>;
+
 // restricted GPR register class. Many Thumb2 instructions allow the full
 // register range for operands, but have undefined behaviours when PC
 // or SP (R13 or R15) are used. The ARM ISA refers to these operands
@@ -328,5 +347,6 @@ def QQQQPR : RegisterClass<"ARM", [v8i64], 256, (sequence "QQQQ%u", 0, 3)> {
 
 // Condition code registers.
 def CCR : RegisterClass<"ARM", [i32], 32, (add CPSR)> {
+  let CopyCost = -1;  // Don't allow copying of status registers.
   let isAllocatable = 0;
 }
diff --git a/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index ef0aaf2a59eb..a3a3d58841db 100644
--- a/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -138,13 +138,12 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
 // Adjust parameters for memset, EABI uses format (ptr, size, value),
 // GNU library uses (ptr, value, size)
 // See RTABI section 4.3.4
-SDValue
-ARMSelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
-                                             SDValue Chain, SDValue Dst,
-                                             SDValue Src, SDValue Size,
-                                             unsigned Align, bool isVolatile,
-                                             MachinePointerInfo DstPtrInfo) const
-{
+SDValue ARMSelectionDAGInfo::
+EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
+                        SDValue Chain, SDValue Dst,
+                        SDValue Src, SDValue Size,
+                        unsigned Align, bool isVolatile,
+                        MachinePointerInfo DstPtrInfo) const {
   // Use default for non AAPCS subtargets
   if (!Subtarget->isAAPCS_ABI())
     return SDValue();
@@ -155,7 +154,7 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
   TargetLowering::ArgListEntry Entry;
 
   // First argument: data pointer
-  const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*DAG.getContext());
   Entry.Node = Dst;
   Entry.Ty = IntPtrTy;
   Args.push_back(Entry);
diff --git a/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h b/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
index ec1bf5ca1941..6419a737295a 100644
--- a/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
+++ b/contrib/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
@@ -14,10 +14,27 @@
 #ifndef ARMSELECTIONDAGINFO_H
 #define ARMSELECTIONDAGINFO_H
 
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/Target/TargetSelectionDAGInfo.h"
 
 namespace llvm {
 
+namespace ARM_AM {
+  static inline ShiftOpc getShiftOpcForNode(unsigned Opcode) {
+    switch (Opcode) {
+    default:          return ARM_AM::no_shift;
+    case ISD::SHL:    return ARM_AM::lsl;
+    case ISD::SRL:    return ARM_AM::lsr;
+    case ISD::SRA:    return ARM_AM::asr;
+    case ISD::ROTR:   return ARM_AM::ror;
+    //case ISD::ROTL:  // Only if imm -> turn into ROTR.
+    // Can't handle RRX here, because it would require folding a flag into
+    // the addressing mode.  :(  This causes us to miss certain things.
+    //case ARMISD::RRX: return ARM_AM::rrx;
+    }
+  }
+}  // end namespace ARM_AM
+
 class ARMSelectionDAGInfo : public TargetSelectionDAGInfo {
   /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
   /// make the right decision when generating code for different targets.
diff --git a/contrib/llvm/lib/Target/ARM/ARMSubtarget.cpp b/contrib/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 1cab9e44ce75..247d6be59ad4 100644
--- a/contrib/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -53,11 +53,14 @@ ARMSubtarget::ARMSubtarget(const std::string &TT, const std::string &CPU,
   , HasVMLxForwarding(false)
   , SlowFPBrcc(false)
   , InThumbMode(false)
+  , InNaClMode(false)
   , HasThumb2(false)
+  , IsMClass(false)
   , NoARM(false)
   , PostRAScheduler(false)
   , IsR9Reserved(ReserveR9)
   , UseMovt(false)
+  , SupportsTailCall(false)
   , HasFP16(false)
   , HasD16(false)
   , HasHardwareDivide(false)
@@ -111,6 +114,8 @@ ARMSubtarget::ARMSubtarget(const std::string &TT, const std::string &CPU,
   else {
     IsR9Reserved = ReserveR9 | !HasV6Ops;
     UseMovt = DarwinUseMOVT && hasV6T2Ops();
+    const Triple &T = getTargetTriple();
+    SupportsTailCall = T.getOS() == Triple::IOS && !T.isOSVersionLT(5, 0);
   }
 
   if (!isThumb() || hasThumb2())
diff --git a/contrib/llvm/lib/Target/ARM/ARMSubtarget.h b/contrib/llvm/lib/Target/ARM/ARMSubtarget.h
index c6508723a576..b63e1085fb83 100644
--- a/contrib/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/contrib/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -70,9 +70,16 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// InThumbMode - True if compiling for Thumb, false for ARM.
   bool InThumbMode;
 
+  /// InNaClMode - True if targeting Native Client
+  bool InNaClMode;
+
   /// HasThumb2 - True if Thumb2 instructions are supported.
   bool HasThumb2;
 
+  /// IsMClass - True if the subtarget belongs to the 'M' profile of CPUs - 
+  /// v6m, v7m for example.
+  bool IsMClass;
+
   /// NoARM - True if subtarget does not support ARM mode execution.
   bool NoARM;
 
@@ -86,6 +93,11 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// imms (including global addresses).
   bool UseMovt;
 
+  /// SupportsTailCall - True if the OS supports tail call. The dynamic linker
+  /// must be able to synthesize call stubs for interworking between ARM and
+  /// Thumb.
+  bool SupportsTailCall;
+
   /// HasFP16 - True if subtarget supports half-precision FP (We support VFP+HF
   /// only so far)
   bool HasFP16;
@@ -209,6 +221,9 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   const Triple &getTargetTriple() const { return TargetTriple; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
+  bool isTargetNaCl() const {
+    return TargetTriple.getOS() == Triple::NativeClient;
+  }
   bool isTargetELF() const { return !isTargetDarwin(); }
 
   bool isAPCS_ABI() const { return TargetABI == ARM_ABI_APCS; }
@@ -218,10 +233,13 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool isThumb1Only() const { return InThumbMode && !HasThumb2; }
   bool isThumb2() const { return InThumbMode && HasThumb2; }
   bool hasThumb2() const { return HasThumb2; }
+  bool isMClass() const { return IsMClass; }
+  bool isARClass() const { return !IsMClass; }
 
   bool isR9Reserved() const { return IsR9Reserved; }
 
   bool useMovt() const { return UseMovt && hasV6T2Ops(); }
+  bool supportsTailCall() const { return SupportsTailCall; }
 
   bool allowsUnalignedMem() const { return AllowsUnalignedMem; }
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/contrib/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index f0b176ad6981..96b1e89b0df0 100644
--- a/contrib/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -15,77 +15,50 @@
 #include "ARM.h"
 #include "llvm/PassManager.h"
 #include "llvm/CodeGen/Passes.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 using namespace llvm;
 
-// This is duplicated code. Refactor this.
-static MCStreamer *createMCStreamer(const Target &T, const std::string &TT,
-                                    MCContext &Ctx, TargetAsmBackend &TAB,
-                                    raw_ostream &OS,
-                                    MCCodeEmitter *Emitter,
-                                    bool RelaxAll,
-                                    bool NoExecStack) {
-  Triple TheTriple(TT);
-
-  if (TheTriple.isOSDarwin())
-    return createMachOStreamer(Ctx, TAB, OS, Emitter, RelaxAll);
-
-  if (TheTriple.isOSWindows()) {
-    llvm_unreachable("ARM does not support Windows COFF format");
-    return NULL;
-  }
-
-  return createELFStreamer(Ctx, TAB, OS, Emitter, RelaxAll, NoExecStack);
-}
+static cl::opt<bool>
+EnableGlobalMerge("global-merge", cl::Hidden,
+                  cl::desc("Enable global merge pass"),
+                  cl::init(true));
 
 extern "C" void LLVMInitializeARMTarget() {
   // Register the target.
   RegisterTargetMachine<ARMTargetMachine> X(TheARMTarget);
   RegisterTargetMachine<ThumbTargetMachine> Y(TheThumbTarget);
-
-  // Register the MC Code Emitter
-  TargetRegistry::RegisterCodeEmitter(TheARMTarget, createARMMCCodeEmitter);
-  TargetRegistry::RegisterCodeEmitter(TheThumbTarget, createARMMCCodeEmitter);
-
-  // Register the asm backend.
-  TargetRegistry::RegisterAsmBackend(TheARMTarget, createARMAsmBackend);
-  TargetRegistry::RegisterAsmBackend(TheThumbTarget, createARMAsmBackend);
-
-  // Register the object streamer.
-  TargetRegistry::RegisterObjectStreamer(TheARMTarget, createMCStreamer);
-  TargetRegistry::RegisterObjectStreamer(TheThumbTarget, createMCStreamer);
-
 }
 
 /// TargetMachine ctor - Create an ARM architecture model.
 ///
-ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T,
-                                           const std::string &TT,
-                                           const std::string &CPU,
-                                           const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, StringRef TT,
+                                           StringRef CPU, StringRef FS,
+                                           Reloc::Model RM, CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS),
     JITInfo(),
     InstrItins(Subtarget.getInstrItineraryData()) {
-  DefRelocModel = getRelocationModel();
-
   // Default to soft float ABI
   if (FloatABIType == FloatABI::Default)
     FloatABIType = FloatABI::Soft;
 }
 
-ARMTargetMachine::ARMTargetMachine(const Target &T, const std::string &TT,
-                                   const std::string &CPU,
-                                   const std::string &FS)
-  : ARMBaseTargetMachine(T, TT, CPU, FS), InstrInfo(Subtarget),
+ARMTargetMachine::ARMTargetMachine(const Target &T, StringRef TT,
+                                   StringRef CPU, StringRef FS,
+                                   Reloc::Model RM, CodeModel::Model CM)
+  : ARMBaseTargetMachine(T, TT, CPU, FS, RM, CM), InstrInfo(Subtarget),
     DataLayout(Subtarget.isAPCS_ABI() ?
                std::string("e-p:32:32-f64:32:64-i64:32:64-"
-                           "v128:32:128-v64:32:64-n32") :
+                           "v128:32:128-v64:32:64-n32-S32") :
+               Subtarget.isAAPCS_ABI() ?
                std::string("e-p:32:32-f64:64:64-i64:64:64-"
-                           "v128:64:128-v64:64:64-n32")),
+                           "v128:64:128-v64:64:64-n32-S64") :
+               std::string("e-p:32:32-f64:64:64-i64:64:64-"
+                           "v128:64:128-v64:64:64-n32-S32")),
     ELFWriterInfo(*this),
     TLInfo(*this),
     TSInfo(*this),
@@ -95,20 +68,24 @@ ARMTargetMachine::ARMTargetMachine(const Target &T, const std::string &TT,
                        "support ARM mode execution!");
 }
 
-ThumbTargetMachine::ThumbTargetMachine(const Target &T, const std::string &TT,
-                                       const std::string &CPU,
-                                       const std::string &FS)
-  : ARMBaseTargetMachine(T, TT, CPU, FS),
+ThumbTargetMachine::ThumbTargetMachine(const Target &T, StringRef TT,
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM)
+  : ARMBaseTargetMachine(T, TT, CPU, FS, RM, CM),
     InstrInfo(Subtarget.hasThumb2()
               ? ((ARMBaseInstrInfo*)new Thumb2InstrInfo(Subtarget))
               : ((ARMBaseInstrInfo*)new Thumb1InstrInfo(Subtarget))),
     DataLayout(Subtarget.isAPCS_ABI() ?
                std::string("e-p:32:32-f64:32:64-i64:32:64-"
                            "i16:16:32-i8:8:32-i1:8:32-"
-                           "v128:32:128-v64:32:64-a:0:32-n32") :
+                           "v128:32:128-v64:32:64-a:0:32-n32-S32") :
+               Subtarget.isAAPCS_ABI() ?
                std::string("e-p:32:32-f64:64:64-i64:64:64-"
                            "i16:16:32-i8:8:32-i1:8:32-"
-                           "v128:64:128-v64:64:64-a:0:32-n32")),
+                           "v128:64:128-v64:64:64-a:0:32-n32-S64") :
+               std::string("e-p:32:32-f64:64:64-i64:64:64-"
+                           "i16:16:32-i8:8:32-i1:8:32-"
+                           "v128:64:128-v64:64:64-a:0:32-n32-S32")),
     ELFWriterInfo(*this),
     TLInfo(*this),
     TSInfo(*this),
@@ -117,10 +94,9 @@ ThumbTargetMachine::ThumbTargetMachine(const Target &T, const std::string &TT,
               : (ARMFrameLowering*)new Thumb1FrameLowering(Subtarget)) {
 }
 
-// Pass Pipeline Configuration
 bool ARMBaseTargetMachine::addPreISel(PassManagerBase &PM,
                                       CodeGenOpt::Level OptLevel) {
-  if (OptLevel != CodeGenOpt::None)
+  if (OptLevel != CodeGenOpt::None && EnableGlobalMerge)
     PM.add(createARMGlobalMergePass(getTargetLowering()));
 
   return false;
@@ -139,7 +115,6 @@ bool ARMBaseTargetMachine::addPreRegAlloc(PassManagerBase &PM,
     PM.add(createARMLoadStoreOptimizationPass(true));
   if (OptLevel != CodeGenOpt::None && Subtarget.isCortexA9())
     PM.add(createMLxExpansionPass());
-
   return true;
 }
 
@@ -150,7 +125,7 @@ bool ARMBaseTargetMachine::addPreSched2(PassManagerBase &PM,
     if (!Subtarget.isThumb1Only())
       PM.add(createARMLoadStoreOptimizationPass());
     if (Subtarget.hasNEON())
-      PM.add(createNEONMoveFixPass());
+      PM.add(createExecutionDependencyFixPass(&ARM::DPRRegClass));
   }
 
   // Expand some pseudo instructions into multiple instructions to allow
@@ -179,10 +154,6 @@ bool ARMBaseTargetMachine::addPreEmitPass(PassManagerBase &PM,
 bool ARMBaseTargetMachine::addCodeEmitter(PassManagerBase &PM,
                                           CodeGenOpt::Level OptLevel,
                                           JITCodeEmitter &JCE) {
-  // FIXME: Move this to TargetJITInfo!
-  if (DefRelocModel == Reloc::Default)
-    setRelocationModel(Reloc::Static);
-
   // Machine code emitter pass for ARM.
   PM.add(createARMJITCodeEmitterPass(*this, JCE));
   return false;
diff --git a/contrib/llvm/lib/Target/ARM/ARMTargetMachine.h b/contrib/llvm/lib/Target/ARM/ARMTargetMachine.h
index bc3d46a50ea5..c8c601c30171 100644
--- a/contrib/llvm/lib/Target/ARM/ARMTargetMachine.h
+++ b/contrib/llvm/lib/Target/ARM/ARMTargetMachine.h
@@ -37,11 +37,11 @@ class ARMBaseTargetMachine : public LLVMTargetMachine {
 private:
   ARMJITInfo          JITInfo;
   InstrItineraryData  InstrItins;
-  Reloc::Model        DefRelocModel;    // Reloc model before it's overridden.
 
 public:
-  ARMBaseTargetMachine(const Target &T, const std::string &TT,
-                       const std::string &CPU, const std::string &FS);
+  ARMBaseTargetMachine(const Target &T, StringRef TT,
+                       StringRef CPU, StringRef FS,
+                       Reloc::Model RM, CodeModel::Model CM);
 
   virtual       ARMJITInfo       *getJITInfo()         { return &JITInfo; }
   virtual const ARMSubtarget  *getSubtargetImpl() const { return &Subtarget; }
@@ -69,8 +69,9 @@ class ARMTargetMachine : public ARMBaseTargetMachine {
   ARMSelectionDAGInfo TSInfo;
   ARMFrameLowering    FrameLowering;
  public:
-  ARMTargetMachine(const Target &T, const std::string &TT,
-                   const std::string &CPU, const std::string &FS);
+  ARMTargetMachine(const Target &T, StringRef TT,
+                   StringRef CPU, StringRef FS,
+                   Reloc::Model RM, CodeModel::Model CM);
 
   virtual const ARMRegisterInfo  *getRegisterInfo() const {
     return &InstrInfo.getRegisterInfo();
@@ -108,8 +109,9 @@ class ThumbTargetMachine : public ARMBaseTargetMachine {
   // Either Thumb1FrameLowering or ARMFrameLowering.
   OwningPtr<ARMFrameLowering> FrameLowering;
 public:
-  ThumbTargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS);
+  ThumbTargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 
   /// returns either Thumb1RegisterInfo or Thumb2RegisterInfo
   virtual const ARMBaseRegisterInfo *getRegisterInfo() const {
diff --git a/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmLexer.cpp b/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmLexer.cpp
index d9a5fa223b4b..14d35ba54654 100644
--- a/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmLexer.cpp
+++ b/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmLexer.cpp
@@ -7,16 +7,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARM.h"
-#include "ARMTargetMachine.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
 
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCTargetAsmLexer.h"
 
-#include "llvm/Target/TargetAsmLexer.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
@@ -30,7 +29,7 @@ using namespace llvm;
 
 namespace {
 
-class ARMBaseAsmLexer : public TargetAsmLexer {
+class ARMBaseAsmLexer : public MCTargetAsmLexer {
   const MCAsmInfo &AsmInfo;
 
   const AsmToken &lexDefinite() {
@@ -43,7 +42,7 @@ class ARMBaseAsmLexer : public TargetAsmLexer {
 
   rmap_ty RegisterMap;
 
-  void InitRegisterMap(const TargetRegisterInfo *info) {
+  void InitRegisterMap(const MCRegisterInfo *info) {
     unsigned numRegs = info->getNumRegs();
 
     for (unsigned i = 0; i < numRegs; ++i) {
@@ -77,33 +76,23 @@ class ARMBaseAsmLexer : public TargetAsmLexer {
   }
 public:
   ARMBaseAsmLexer(const Target &T, const MCAsmInfo &MAI)
-    : TargetAsmLexer(T), AsmInfo(MAI) {
+    : MCTargetAsmLexer(T), AsmInfo(MAI) {
   }
 };
 
 class ARMAsmLexer : public ARMBaseAsmLexer {
 public:
-  ARMAsmLexer(const Target &T, const MCAsmInfo &MAI)
+  ARMAsmLexer(const Target &T, const MCRegisterInfo &MRI, const MCAsmInfo &MAI)
     : ARMBaseAsmLexer(T, MAI) {
-    std::string tripleString("arm-unknown-unknown");
-    std::string featureString;
-    std::string CPU;
-    OwningPtr<const TargetMachine>
-      targetMachine(T.createTargetMachine(tripleString, CPU, featureString));
-    InitRegisterMap(targetMachine->getRegisterInfo());
+    InitRegisterMap(&MRI);
   }
 };
 
 class ThumbAsmLexer : public ARMBaseAsmLexer {
 public:
-  ThumbAsmLexer(const Target &T, const MCAsmInfo &MAI)
+  ThumbAsmLexer(const Target &T, const MCRegisterInfo &MRI,const MCAsmInfo &MAI)
     : ARMBaseAsmLexer(T, MAI) {
-    std::string tripleString("thumb-unknown-unknown");
-    std::string featureString;
-    std::string CPU;
-    OwningPtr<const TargetMachine>
-      targetMachine(T.createTargetMachine(tripleString, CPU, featureString));
-    InitRegisterMap(targetMachine->getRegisterInfo());
+    InitRegisterMap(&MRI);
   }
 };
 
@@ -149,6 +138,6 @@ AsmToken ARMBaseAsmLexer::LexTokenUAL() {
 }
 
 extern "C" void LLVMInitializeARMAsmLexer() {
-  RegisterAsmLexer<ARMAsmLexer> X(TheARMTarget);
-  RegisterAsmLexer<ThumbAsmLexer> Y(TheThumbTarget);
+  RegisterMCAsmLexer<ARMAsmLexer> X(TheARMTarget);
+  RegisterMCAsmLexer<ThumbAsmLexer> Y(TheThumbTarget);
 }
diff --git a/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index a4741270c7a5..24f15b4694ff 100644
--- a/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -7,11 +7,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARM.h"
-#include "ARMAddressingModes.h"
-#include "ARMMCExpr.h"
-#include "ARMBaseRegisterInfo.h"
-#include "ARMSubtarget.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
+#include "MCTargetDesc/ARMMCExpr.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
@@ -20,12 +18,17 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmParser.h"
+#include "llvm/MC/MCTargetAsmParser.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -37,49 +40,65 @@ namespace {
 
 class ARMOperand;
 
-class ARMAsmParser : public TargetAsmParser {
+class ARMAsmParser : public MCTargetAsmParser {
   MCSubtargetInfo &STI;
   MCAsmParser &Parser;
 
+  struct {
+    ARMCC::CondCodes Cond;    // Condition for IT block.
+    unsigned Mask:4;          // Condition mask for instructions.
+                              // Starting at first 1 (from lsb).
+                              //   '1'  condition as indicated in IT.
+                              //   '0'  inverse of condition (else).
+                              // Count of instructions in IT block is
+                              // 4 - trailingzeroes(mask)
+
+    bool FirstCond;           // Explicit flag for when we're parsing the
+                              // First instruction in the IT block. It's
+                              // implied in the mask, so needs special
+                              // handling.
+
+    unsigned CurPosition;     // Current position in parsing of IT
+                              // block. In range [0,3]. Initialized
+                              // according to count of instructions in block.
+                              // ~0U if no active IT block.
+  } ITState;
+  bool inITBlock() { return ITState.CurPosition != ~0U;}
+  void forwardITPosition() {
+    if (!inITBlock()) return;
+    // Move to the next instruction in the IT block, if there is one. If not,
+    // mark the block as done.
+    unsigned TZ = CountTrailingZeros_32(ITState.Mask);
+    if (++ITState.CurPosition == 5 - TZ)
+      ITState.CurPosition = ~0U; // Done with the IT block after this.
+  }
+
+
   MCAsmParser &getParser() const { return Parser; }
   MCAsmLexer &getLexer() const { return Parser.getLexer(); }
 
   void Warning(SMLoc L, const Twine &Msg) { Parser.Warning(L, Msg); }
   bool Error(SMLoc L, const Twine &Msg) { return Parser.Error(L, Msg); }
 
-  int TryParseRegister();
-  virtual bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc);
-  bool TryParseRegisterWithWriteBack(SmallVectorImpl<MCParsedAsmOperand*> &);
-  int TryParseShiftRegister(SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool ParseRegisterList(SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool ParseMemory(SmallVectorImpl<MCParsedAsmOperand*> &,
-                   ARMII::AddrMode AddrMode);
-  bool ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &, StringRef Mnemonic);
-  bool ParsePrefix(ARMMCExpr::VariantKind &RefKind);
-  const MCExpr *ApplyPrefixToExpr(const MCExpr *E,
-                                  MCSymbolRefExpr::VariantKind Variant);
+  int tryParseRegister();
+  bool tryParseRegisterWithWriteBack(SmallVectorImpl<MCParsedAsmOperand*> &);
+  int tryParseShiftRegister(SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool parseRegisterList(SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool parseMemory(SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool parseOperand(SmallVectorImpl<MCParsedAsmOperand*> &, StringRef Mnemonic);
+  bool parsePrefix(ARMMCExpr::VariantKind &RefKind);
+  bool parseMemRegOffsetShift(ARM_AM::ShiftOpc &ShiftType,
+                              unsigned &ShiftAmount);
+  bool parseDirectiveWord(unsigned Size, SMLoc L);
+  bool parseDirectiveThumb(SMLoc L);
+  bool parseDirectiveThumbFunc(SMLoc L);
+  bool parseDirectiveCode(SMLoc L);
+  bool parseDirectiveSyntax(SMLoc L);
 
-
-  bool ParseMemoryOffsetReg(bool &Negative,
-                            bool &OffsetRegShifted,
-                            enum ARM_AM::ShiftOpc &ShiftType,
-                            const MCExpr *&ShiftAmount,
-                            const MCExpr *&Offset,
-                            bool &OffsetIsReg,
-                            int &OffsetRegNum,
-                            SMLoc &E);
-  bool ParseShift(enum ARM_AM::ShiftOpc &St,
-                  const MCExpr *&ShiftAmount, SMLoc &E);
-  bool ParseDirectiveWord(unsigned Size, SMLoc L);
-  bool ParseDirectiveThumb(SMLoc L);
-  bool ParseDirectiveThumbFunc(SMLoc L);
-  bool ParseDirectiveCode(SMLoc L);
-  bool ParseDirectiveSyntax(SMLoc L);
-
-  bool MatchAndEmitInstruction(SMLoc IDLoc,
-                               SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                               MCStreamer &Out);
-  void GetMnemonicAcceptInfo(StringRef Mnemonic, bool &CanAcceptCarrySet,
+  StringRef splitMnemonic(StringRef Mnemonic, unsigned &PredicationCode,
+                          bool &CarrySetting, unsigned &ProcessorIMod,
+                          StringRef &ITMask);
+  void getMnemonicAcceptInfo(StringRef Mnemonic, bool &CanAcceptCarrySet,
                              bool &CanAcceptPredicationCode);
 
   bool isThumb() const {
@@ -89,10 +108,22 @@ class ARMAsmParser : public TargetAsmParser {
   bool isThumbOne() const {
     return isThumb() && (STI.getFeatureBits() & ARM::FeatureThumb2) == 0;
   }
+  bool isThumbTwo() const {
+    return isThumb() && (STI.getFeatureBits() & ARM::FeatureThumb2);
+  }
+  bool hasV6Ops() const {
+    return STI.getFeatureBits() & ARM::HasV6Ops;
+  }
+  bool hasV7Ops() const {
+    return STI.getFeatureBits() & ARM::HasV7Ops;
+  }
   void SwitchMode() {
     unsigned FB = ComputeAvailableFeatures(STI.ToggleFeature(ARM::ModeThumb));
     setAvailableFeatures(FB);
   }
+  bool isMClass() const {
+    return STI.getFeatureBits() & ARM::FeatureMClass;
+  }
 
   /// @name Auto-generated Match Functions
   /// {
@@ -102,43 +133,108 @@ class ARMAsmParser : public TargetAsmParser {
 
   /// }
 
-  OperandMatchResultTy tryParseCoprocNumOperand(
+  OperandMatchResultTy parseITCondCode(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parseCoprocNumOperand(
     SmallVectorImpl<MCParsedAsmOperand*>&);
-  OperandMatchResultTy tryParseCoprocRegOperand(
+  OperandMatchResultTy parseCoprocRegOperand(
     SmallVectorImpl<MCParsedAsmOperand*>&);
-  OperandMatchResultTy tryParseMemBarrierOptOperand(
+  OperandMatchResultTy parseCoprocOptionOperand(
     SmallVectorImpl<MCParsedAsmOperand*>&);
-  OperandMatchResultTy tryParseProcIFlagsOperand(
+  OperandMatchResultTy parseMemBarrierOptOperand(
     SmallVectorImpl<MCParsedAsmOperand*>&);
-  OperandMatchResultTy tryParseMSRMaskOperand(
+  OperandMatchResultTy parseProcIFlagsOperand(
     SmallVectorImpl<MCParsedAsmOperand*>&);
-  OperandMatchResultTy tryParseMemMode2Operand(
-    SmallVectorImpl<MCParsedAsmOperand*>&);
-  OperandMatchResultTy tryParseMemMode3Operand(
+  OperandMatchResultTy parseMSRMaskOperand(
     SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parsePKHImm(SmallVectorImpl<MCParsedAsmOperand*> &O,
+                                   StringRef Op, int Low, int High);
+  OperandMatchResultTy parsePKHLSLImm(SmallVectorImpl<MCParsedAsmOperand*> &O) {
+    return parsePKHImm(O, "lsl", 0, 31);
+  }
+  OperandMatchResultTy parsePKHASRImm(SmallVectorImpl<MCParsedAsmOperand*> &O) {
+    return parsePKHImm(O, "asr", 1, 32);
+  }
+  OperandMatchResultTy parseSetEndImm(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parseShifterImm(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parseRotImm(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parseBitfield(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parsePostIdxReg(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parseAM3Offset(SmallVectorImpl<MCParsedAsmOperand*>&);
+  OperandMatchResultTy parseFPImm(SmallVectorImpl<MCParsedAsmOperand*>&);
 
   // Asm Match Converter Methods
-  bool CvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+  bool cvtT2LdrdPre(MCInst &Inst, unsigned Opcode,
+                    const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtT2StrdPre(MCInst &Inst, unsigned Opcode,
+                    const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool CvtStWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+  bool cvtStWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool CvtLdWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+  bool cvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool CvtStWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+  bool cvtLdWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtStWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+                                  const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtStWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+                                  const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtStWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+                                  const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtLdExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+                             const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtLdExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+                             const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtStExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+                             const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtStExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+                             const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtLdrdPre(MCInst &Inst, unsigned Opcode,
+                  const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtStrdPre(MCInst &Inst, unsigned Opcode,
+                  const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtLdWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+                                  const SmallVectorImpl<MCParsedAsmOperand*> &);
+  bool cvtThumbMultiply(MCInst &Inst, unsigned Opcode,
+                        const SmallVectorImpl<MCParsedAsmOperand*> &);
+
+  bool validateInstruction(MCInst &Inst,
+                           const SmallVectorImpl<MCParsedAsmOperand*> &Ops);
+  void processInstruction(MCInst &Inst,
+                          const SmallVectorImpl<MCParsedAsmOperand*> &Ops);
+  bool shouldOmitCCOutOperand(StringRef Mnemonic,
+                              SmallVectorImpl<MCParsedAsmOperand*> &Operands);
 
 public:
+  enum ARMMatchResultTy {
+    Match_RequiresITBlock = FIRST_TARGET_MATCH_RESULT_TY,
+    Match_RequiresNotITBlock,
+    Match_RequiresV6,
+    Match_RequiresThumb2
+  };
+
   ARMAsmParser(MCSubtargetInfo &_STI, MCAsmParser &_Parser)
-    : TargetAsmParser(), STI(_STI), Parser(_Parser) {
+    : MCTargetAsmParser(), STI(_STI), Parser(_Parser) {
     MCAsmParserExtension::Initialize(_Parser);
 
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
+
+    // Not in an ITBlock to start with.
+    ITState.CurPosition = ~0U;
   }
 
-  virtual bool ParseInstruction(StringRef Name, SMLoc NameLoc,
-                                SmallVectorImpl<MCParsedAsmOperand*> &Operands);
-  virtual bool ParseDirective(AsmToken DirectiveID);
+  // Implementation of the MCTargetAsmParser interface:
+  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc);
+  bool ParseInstruction(StringRef Name, SMLoc NameLoc,
+                        SmallVectorImpl<MCParsedAsmOperand*> &Operands);
+  bool ParseDirective(AsmToken DirectiveID);
+
+  unsigned checkTargetMatchPredicate(MCInst &Inst);
+
+  bool MatchAndEmitInstruction(SMLoc IDLoc,
+                               SmallVectorImpl<MCParsedAsmOperand*> &Operands,
+                               MCStreamer &Out);
 };
 } // end anonymous namespace
 
@@ -148,22 +244,30 @@ namespace {
 /// instruction.
 class ARMOperand : public MCParsedAsmOperand {
   enum KindTy {
-    CondCode,
-    CCOut,
-    CoprocNum,
-    CoprocReg,
-    Immediate,
-    MemBarrierOpt,
-    Memory,
-    MSRMask,
-    ProcIFlags,
-    Register,
-    RegisterList,
-    DPRRegisterList,
-    SPRRegisterList,
-    ShiftedRegister,
-    Shifter,
-    Token
+    k_CondCode,
+    k_CCOut,
+    k_ITCondMask,
+    k_CoprocNum,
+    k_CoprocReg,
+    k_CoprocOption,
+    k_Immediate,
+    k_FPImmediate,
+    k_MemBarrierOpt,
+    k_Memory,
+    k_PostIndexRegister,
+    k_MSRMask,
+    k_ProcIFlags,
+    k_VectorIndex,
+    k_Register,
+    k_RegisterList,
+    k_DPRRegisterList,
+    k_SPRRegisterList,
+    k_ShiftedRegister,
+    k_ShiftedImmediate,
+    k_ShifterImmediate,
+    k_RotateImmediate,
+    k_BitfieldDescriptor,
+    k_Token
   } Kind;
 
   SMLoc StartLoc, EndLoc;
@@ -175,12 +279,20 @@ class ARMOperand : public MCParsedAsmOperand {
     } CC;
 
     struct {
-      ARM_MB::MemBOpt Val;
-    } MBOpt;
+      unsigned Val;
+    } Cop;
 
     struct {
       unsigned Val;
-    } Cop;
+    } CoprocOption;
+
+    struct {
+      unsigned Mask:4;
+    } ITMask;
+
+    struct {
+      ARM_MB::MemBOpt Val;
+    } MBOpt;
 
     struct {
       ARM_PROC::IFlags Val;
@@ -199,38 +311,61 @@ class ARMOperand : public MCParsedAsmOperand {
       unsigned RegNum;
     } Reg;
 
+    struct {
+      unsigned Val;
+    } VectorIndex;
+
     struct {
       const MCExpr *Val;
     } Imm;
 
+    struct {
+      unsigned Val;       // encoded 8-bit representation
+    } FPImm;
+
     /// Combined record for all forms of ARM address expressions.
     struct {
-      ARMII::AddrMode AddrMode;
       unsigned BaseRegNum;
-      union {
-        unsigned RegNum;     ///< Offset register num, when OffsetIsReg.
-        const MCExpr *Value; ///< Offset value, when !OffsetIsReg.
-      } Offset;
-      const MCExpr *ShiftAmount;     // used when OffsetRegShifted is true
-      enum ARM_AM::ShiftOpc ShiftType; // used when OffsetRegShifted is true
-      unsigned OffsetRegShifted : 1; // only used when OffsetIsReg is true
-      unsigned Preindexed       : 1;
-      unsigned Postindexed      : 1;
-      unsigned OffsetIsReg      : 1;
-      unsigned Negative         : 1; // only used when OffsetIsReg is true
-      unsigned Writeback        : 1;
-    } Mem;
+      // Offset is in OffsetReg or OffsetImm. If both are zero, no offset
+      // was specified.
+      const MCConstantExpr *OffsetImm;  // Offset immediate value
+      unsigned OffsetRegNum;    // Offset register num, when OffsetImm == NULL
+      ARM_AM::ShiftOpc ShiftType; // Shift type for OffsetReg
+      unsigned ShiftImm;        // shift for OffsetReg.
+      unsigned Alignment;       // 0 = no alignment specified
+                                // n = alignment in bytes (8, 16, or 32)
+      unsigned isNegative : 1;  // Negated OffsetReg? (~'U' bit)
+    } Memory;
 
     struct {
+      unsigned RegNum;
+      bool isAdd;
       ARM_AM::ShiftOpc ShiftTy;
+      unsigned ShiftImm;
+    } PostIdxReg;
+
+    struct {
+      bool isASR;
       unsigned Imm;
-    } Shift;
+    } ShifterImm;
     struct {
       ARM_AM::ShiftOpc ShiftTy;
       unsigned SrcReg;
       unsigned ShiftReg;
       unsigned ShiftImm;
-    } ShiftedReg;
+    } RegShiftedReg;
+    struct {
+      ARM_AM::ShiftOpc ShiftTy;
+      unsigned SrcReg;
+      unsigned ShiftImm;
+    } RegShiftedImm;
+    struct {
+      unsigned Imm;
+    } RotImm;
+    struct {
+      unsigned LSB;
+      unsigned Width;
+    } Bitfield;
   };
 
   ARMOperand(KindTy K) : MCParsedAsmOperand(), Kind(K) {}
@@ -240,45 +375,69 @@ class ARMOperand : public MCParsedAsmOperand {
     StartLoc = o.StartLoc;
     EndLoc = o.EndLoc;
     switch (Kind) {
-    case CondCode:
+    case k_CondCode:
       CC = o.CC;
       break;
-    case Token:
+    case k_ITCondMask:
+      ITMask = o.ITMask;
+      break;
+    case k_Token:
       Tok = o.Tok;
       break;
-    case CCOut:
-    case Register:
+    case k_CCOut:
+    case k_Register:
       Reg = o.Reg;
       break;
-    case RegisterList:
-    case DPRRegisterList:
-    case SPRRegisterList:
+    case k_RegisterList:
+    case k_DPRRegisterList:
+    case k_SPRRegisterList:
       Registers = o.Registers;
       break;
-    case CoprocNum:
-    case CoprocReg:
+    case k_CoprocNum:
+    case k_CoprocReg:
       Cop = o.Cop;
       break;
-    case Immediate:
+    case k_CoprocOption:
+      CoprocOption = o.CoprocOption;
+      break;
+    case k_Immediate:
       Imm = o.Imm;
       break;
-    case MemBarrierOpt:
+    case k_FPImmediate:
+      FPImm = o.FPImm;
+      break;
+    case k_MemBarrierOpt:
       MBOpt = o.MBOpt;
       break;
-    case Memory:
-      Mem = o.Mem;
+    case k_Memory:
+      Memory = o.Memory;
       break;
-    case MSRMask:
+    case k_PostIndexRegister:
+      PostIdxReg = o.PostIdxReg;
+      break;
+    case k_MSRMask:
       MMask = o.MMask;
       break;
-    case ProcIFlags:
+    case k_ProcIFlags:
       IFlags = o.IFlags;
       break;
-    case Shifter:
-      Shift = o.Shift;
+    case k_ShifterImmediate:
+      ShifterImm = o.ShifterImm;
       break;
-    case ShiftedRegister:
-      ShiftedReg = o.ShiftedReg;
+    case k_ShiftedRegister:
+      RegShiftedReg = o.RegShiftedReg;
+      break;
+    case k_ShiftedImmediate:
+      RegShiftedImm = o.RegShiftedImm;
+      break;
+    case k_RotateImmediate:
+      RotImm = o.RotImm;
+      break;
+    case k_BitfieldDescriptor:
+      Bitfield = o.Bitfield;
+      break;
+    case k_VectorIndex:
+      VectorIndex = o.VectorIndex;
       break;
     }
   }
@@ -289,94 +448,96 @@ class ARMOperand : public MCParsedAsmOperand {
   SMLoc getEndLoc() const { return EndLoc; }
 
   ARMCC::CondCodes getCondCode() const {
-    assert(Kind == CondCode && "Invalid access!");
+    assert(Kind == k_CondCode && "Invalid access!");
     return CC.Val;
   }
 
   unsigned getCoproc() const {
-    assert((Kind == CoprocNum || Kind == CoprocReg) && "Invalid access!");
+    assert((Kind == k_CoprocNum || Kind == k_CoprocReg) && "Invalid access!");
     return Cop.Val;
   }
 
   StringRef getToken() const {
-    assert(Kind == Token && "Invalid access!");
+    assert(Kind == k_Token && "Invalid access!");
     return StringRef(Tok.Data, Tok.Length);
   }
 
   unsigned getReg() const {
-    assert((Kind == Register || Kind == CCOut) && "Invalid access!");
+    assert((Kind == k_Register || Kind == k_CCOut) && "Invalid access!");
     return Reg.RegNum;
   }
 
   const SmallVectorImpl<unsigned> &getRegList() const {
-    assert((Kind == RegisterList || Kind == DPRRegisterList ||
-            Kind == SPRRegisterList) && "Invalid access!");
+    assert((Kind == k_RegisterList || Kind == k_DPRRegisterList ||
+            Kind == k_SPRRegisterList) && "Invalid access!");
     return Registers;
   }
 
   const MCExpr *getImm() const {
-    assert(Kind == Immediate && "Invalid access!");
+    assert(Kind == k_Immediate && "Invalid access!");
     return Imm.Val;
   }
 
+  unsigned getFPImm() const {
+    assert(Kind == k_FPImmediate && "Invalid access!");
+    return FPImm.Val;
+  }
+
+  unsigned getVectorIndex() const {
+    assert(Kind == k_VectorIndex && "Invalid access!");
+    return VectorIndex.Val;
+  }
+
   ARM_MB::MemBOpt getMemBarrierOpt() const {
-    assert(Kind == MemBarrierOpt && "Invalid access!");
+    assert(Kind == k_MemBarrierOpt && "Invalid access!");
     return MBOpt.Val;
   }
 
   ARM_PROC::IFlags getProcIFlags() const {
-    assert(Kind == ProcIFlags && "Invalid access!");
+    assert(Kind == k_ProcIFlags && "Invalid access!");
     return IFlags.Val;
   }
 
   unsigned getMSRMask() const {
-    assert(Kind == MSRMask && "Invalid access!");
+    assert(Kind == k_MSRMask && "Invalid access!");
     return MMask.Val;
   }
 
-  /// @name Memory Operand Accessors
-  /// @{
-  ARMII::AddrMode getMemAddrMode() const {
-    return Mem.AddrMode;
+  bool isCoprocNum() const { return Kind == k_CoprocNum; }
+  bool isCoprocReg() const { return Kind == k_CoprocReg; }
+  bool isCoprocOption() const { return Kind == k_CoprocOption; }
+  bool isCondCode() const { return Kind == k_CondCode; }
+  bool isCCOut() const { return Kind == k_CCOut; }
+  bool isITMask() const { return Kind == k_ITCondMask; }
+  bool isITCondCode() const { return Kind == k_CondCode; }
+  bool isImm() const { return Kind == k_Immediate; }
+  bool isFPImm() const { return Kind == k_FPImmediate; }
+  bool isImm8s4() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return ((Value & 3) == 0) && Value >= -1020 && Value <= 1020;
   }
-  unsigned getMemBaseRegNum() const {
-    return Mem.BaseRegNum;
+  bool isImm0_1020s4() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return ((Value & 3) == 0) && Value >= 0 && Value <= 1020;
   }
-  unsigned getMemOffsetRegNum() const {
-    assert(Mem.OffsetIsReg && "Invalid access!");
-    return Mem.Offset.RegNum;
+  bool isImm0_508s4() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return ((Value & 3) == 0) && Value >= 0 && Value <= 508;
   }
-  const MCExpr *getMemOffset() const {
-    assert(!Mem.OffsetIsReg && "Invalid access!");
-    return Mem.Offset.Value;
-  }
-  unsigned getMemOffsetRegShifted() const {
-    assert(Mem.OffsetIsReg && "Invalid access!");
-    return Mem.OffsetRegShifted;
-  }
-  const MCExpr *getMemShiftAmount() const {
-    assert(Mem.OffsetIsReg && Mem.OffsetRegShifted && "Invalid access!");
-    return Mem.ShiftAmount;
-  }
-  enum ARM_AM::ShiftOpc getMemShiftType() const {
-    assert(Mem.OffsetIsReg && Mem.OffsetRegShifted && "Invalid access!");
-    return Mem.ShiftType;
-  }
-  bool getMemPreindexed() const { return Mem.Preindexed; }
-  bool getMemPostindexed() const { return Mem.Postindexed; }
-  bool getMemOffsetIsReg() const { return Mem.OffsetIsReg; }
-  bool getMemNegative() const { return Mem.Negative; }
-  bool getMemWriteback() const { return Mem.Writeback; }
-
-  /// @}
-
-  bool isCoprocNum() const { return Kind == CoprocNum; }
-  bool isCoprocReg() const { return Kind == CoprocReg; }
-  bool isCondCode() const { return Kind == CondCode; }
-  bool isCCOut() const { return Kind == CCOut; }
-  bool isImm() const { return Kind == Immediate; }
   bool isImm0_255() const {
-    if (Kind != Immediate)
+    if (Kind != k_Immediate)
       return false;
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
@@ -384,7 +545,7 @@ class ARMOperand : public MCParsedAsmOperand {
     return Value >= 0 && Value < 256;
   }
   bool isImm0_7() const {
-    if (Kind != Immediate)
+    if (Kind != k_Immediate)
       return false;
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
@@ -392,130 +553,365 @@ class ARMOperand : public MCParsedAsmOperand {
     return Value >= 0 && Value < 8;
   }
   bool isImm0_15() const {
-    if (Kind != Immediate)
+    if (Kind != k_Immediate)
       return false;
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
     int64_t Value = CE->getValue();
     return Value >= 0 && Value < 16;
   }
+  bool isImm0_31() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value >= 0 && Value < 32;
+  }
+  bool isImm1_16() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value > 0 && Value < 17;
+  }
+  bool isImm1_32() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value > 0 && Value < 33;
+  }
   bool isImm0_65535() const {
-    if (Kind != Immediate)
+    if (Kind != k_Immediate)
       return false;
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
     int64_t Value = CE->getValue();
     return Value >= 0 && Value < 65536;
   }
+  bool isImm0_65535Expr() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    // If it's not a constant expression, it'll generate a fixup and be
+    // handled later.
+    if (!CE) return true;
+    int64_t Value = CE->getValue();
+    return Value >= 0 && Value < 65536;
+  }
+  bool isImm24bit() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value >= 0 && Value <= 0xffffff;
+  }
+  bool isImmThumbSR() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value > 0 && Value < 33;
+  }
+  bool isPKHLSLImm() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value >= 0 && Value < 32;
+  }
+  bool isPKHASRImm() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value > 0 && Value <= 32;
+  }
+  bool isARMSOImm() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return ARM_AM::getSOImmVal(Value) != -1;
+  }
   bool isT2SOImm() const {
-    if (Kind != Immediate)
+    if (Kind != k_Immediate)
       return false;
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
     int64_t Value = CE->getValue();
     return ARM_AM::getT2SOImmVal(Value) != -1;
   }
-  bool isReg() const { return Kind == Register; }
-  bool isRegList() const { return Kind == RegisterList; }
-  bool isDPRRegList() const { return Kind == DPRRegisterList; }
-  bool isSPRRegList() const { return Kind == SPRRegisterList; }
-  bool isToken() const { return Kind == Token; }
-  bool isMemBarrierOpt() const { return Kind == MemBarrierOpt; }
-  bool isMemory() const { return Kind == Memory; }
-  bool isShifter() const { return Kind == Shifter; }
-  bool isShiftedReg() const { return Kind == ShiftedRegister; }
-  bool isMemMode2() const {
-    if (getMemAddrMode() != ARMII::AddrMode2)
+  bool isSetEndImm() const {
+    if (Kind != k_Immediate)
       return false;
-
-    if (getMemOffsetIsReg())
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return Value == 1 || Value == 0;
+  }
+  bool isReg() const { return Kind == k_Register; }
+  bool isRegList() const { return Kind == k_RegisterList; }
+  bool isDPRRegList() const { return Kind == k_DPRRegisterList; }
+  bool isSPRRegList() const { return Kind == k_SPRRegisterList; }
+  bool isToken() const { return Kind == k_Token; }
+  bool isMemBarrierOpt() const { return Kind == k_MemBarrierOpt; }
+  bool isMemory() const { return Kind == k_Memory; }
+  bool isShifterImm() const { return Kind == k_ShifterImmediate; }
+  bool isRegShiftedReg() const { return Kind == k_ShiftedRegister; }
+  bool isRegShiftedImm() const { return Kind == k_ShiftedImmediate; }
+  bool isRotImm() const { return Kind == k_RotateImmediate; }
+  bool isBitfield() const { return Kind == k_BitfieldDescriptor; }
+  bool isPostIdxRegShifted() const { return Kind == k_PostIndexRegister; }
+  bool isPostIdxReg() const {
+    return Kind == k_PostIndexRegister && PostIdxReg.ShiftTy == ARM_AM::no_shift;
+  }
+  bool isMemNoOffset(bool alignOK = false) const {
+    if (!isMemory())
+      return false;
+    // No offset of any kind.
+    return Memory.OffsetRegNum == 0 && Memory.OffsetImm == 0 &&
+     (alignOK || Memory.Alignment == 0);
+  }
+  bool isAlignedMemory() const {
+    return isMemNoOffset(true);
+  }
+  bool isAddrMode2() const {
+    if (!isMemory() || Memory.Alignment != 0) return false;
+    // Check for register offset.
+    if (Memory.OffsetRegNum) return true;
+    // Immediate offset in range [-4095, 4095].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val > -4096 && Val < 4096;
+  }
+  bool isAM2OffsetImm() const {
+    if (Kind != k_Immediate)
+      return false;
+    // Immediate offset in range [-4095, 4095].
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Val = CE->getValue();
+    return Val > -4096 && Val < 4096;
+  }
+  bool isAddrMode3() const {
+    if (!isMemory() || Memory.Alignment != 0) return false;
+    // No shifts are legal for AM3.
+    if (Memory.ShiftType != ARM_AM::no_shift) return false;
+    // Check for register offset.
+    if (Memory.OffsetRegNum) return true;
+    // Immediate offset in range [-255, 255].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val > -256 && Val < 256;
+  }
+  bool isAM3Offset() const {
+    if (Kind != k_Immediate && Kind != k_PostIndexRegister)
+      return false;
+    if (Kind == k_PostIndexRegister)
+      return PostIdxReg.ShiftTy == ARM_AM::no_shift;
+    // Immediate offset in range [-255, 255].
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Val = CE->getValue();
+    // Special case, #-0 is INT32_MIN.
+    return (Val > -256 && Val < 256) || Val == INT32_MIN;
+  }
+  bool isAddrMode5() const {
+    if (!isMemory() || Memory.Alignment != 0) return false;
+    // Check for register offset.
+    if (Memory.OffsetRegNum) return false;
+    // Immediate offset in range [-1020, 1020] and a multiple of 4.
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return (Val >= -1020 && Val <= 1020 && ((Val & 3) == 0)) ||
+           Val == INT32_MIN;
+  }
+  bool isMemTBB() const {
+    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+        Memory.ShiftType != ARM_AM::no_shift || Memory.Alignment != 0)
+      return false;
+    return true;
+  }
+  bool isMemTBH() const {
+    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+        Memory.ShiftType != ARM_AM::lsl || Memory.ShiftImm != 1 ||
+        Memory.Alignment != 0 )
+      return false;
+    return true;
+  }
+  bool isMemRegOffset() const {
+    if (!isMemory() || !Memory.OffsetRegNum || Memory.Alignment != 0)
+      return false;
+    return true;
+  }
+  bool isT2MemRegOffset() const {
+    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+        Memory.Alignment != 0)
+      return false;
+    // Only lsl #{0, 1, 2, 3} allowed.
+    if (Memory.ShiftType == ARM_AM::no_shift)
+      return true;
+    if (Memory.ShiftType != ARM_AM::lsl || Memory.ShiftImm > 3)
+      return false;
+    return true;
+  }
+  bool isMemThumbRR() const {
+    // Thumb reg+reg addressing is simple. Just two registers, a base and
+    // an offset. No shifts, negations or any other complicating factors.
+    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+        Memory.ShiftType != ARM_AM::no_shift || Memory.Alignment != 0)
+      return false;
+    return isARMLowRegister(Memory.BaseRegNum) &&
+      (!Memory.OffsetRegNum || isARMLowRegister(Memory.OffsetRegNum));
+  }
+  bool isMemThumbRIs4() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+        !isARMLowRegister(Memory.BaseRegNum) || Memory.Alignment != 0)
+      return false;
+    // Immediate offset, multiple of 4 in range [0, 124].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= 0 && Val <= 124 && (Val % 4) == 0;
+  }
+  bool isMemThumbRIs2() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+        !isARMLowRegister(Memory.BaseRegNum) || Memory.Alignment != 0)
+      return false;
+    // Immediate offset, multiple of 4 in range [0, 62].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= 0 && Val <= 62 && (Val % 2) == 0;
+  }
+  bool isMemThumbRIs1() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+        !isARMLowRegister(Memory.BaseRegNum) || Memory.Alignment != 0)
+      return false;
+    // Immediate offset in range [0, 31].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= 0 && Val <= 31;
+  }
+  bool isMemThumbSPI() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+        Memory.BaseRegNum != ARM::SP || Memory.Alignment != 0)
+      return false;
+    // Immediate offset, multiple of 4 in range [0, 1020].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= 0 && Val <= 1020 && (Val % 4) == 0;
+  }
+  bool isMemImm8s4Offset() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+      return false;
+    // Immediate offset a multiple of 4 in range [-1020, 1020].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= -1020 && Val <= 1020 && (Val & 3) == 0;
+  }
+  bool isMemImm0_1020s4Offset() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+      return false;
+    // Immediate offset a multiple of 4 in range [0, 1020].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= 0 && Val <= 1020 && (Val & 3) == 0;
+  }
+  bool isMemImm8Offset() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+      return false;
+    // Immediate offset in range [-255, 255].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return (Val == INT32_MIN) || (Val > -256 && Val < 256);
+  }
+  bool isMemPosImm8Offset() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+      return false;
+    // Immediate offset in range [0, 255].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val >= 0 && Val < 256;
+  }
+  bool isMemNegImm8Offset() const {
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+      return false;
+    // Immediate offset in range [-255, -1].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return Val > -256 && Val < 0;
+  }
+  bool isMemUImm12Offset() const {
+    // If we have an immediate that's not a constant, treat it as a label
+    // reference needing a fixup. If it is a constant, it's something else
+    // and we reject it.
+    if (Kind == k_Immediate && !isa<MCConstantExpr>(getImm()))
       return true;
 
-    if (getMemNegative() &&
-        !(getMemPostindexed() || getMemPreindexed()))
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
-
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    if (!CE) return false;
-    int64_t Value = CE->getValue();
-
-    // The offset must be in the range 0-4095 (imm12).
-    if (Value > 4095 || Value < -4095)
-      return false;
-
-    return true;
+    // Immediate offset in range [0, 4095].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return (Val >= 0 && Val < 4096);
   }
-  bool isMemMode3() const {
-    if (getMemAddrMode() != ARMII::AddrMode3)
-      return false;
-
-    if (getMemOffsetIsReg()) {
-      if (getMemOffsetRegShifted())
-        return false; // No shift with offset reg allowed
+  bool isMemImm12Offset() const {
+    // If we have an immediate that's not a constant, treat it as a label
+    // reference needing a fixup. If it is a constant, it's something else
+    // and we reject it.
+    if (Kind == k_Immediate && !isa<MCConstantExpr>(getImm()))
       return true;
-    }
 
-    if (getMemNegative() &&
-        !(getMemPostindexed() || getMemPreindexed()))
+    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
-
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
+    // Immediate offset in range [-4095, 4095].
+    if (!Memory.OffsetImm) return true;
+    int64_t Val = Memory.OffsetImm->getValue();
+    return (Val > -4096 && Val < 4096) || (Val == INT32_MIN);
+  }
+  bool isPostIdxImm8() const {
+    if (Kind != k_Immediate)
+      return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
-    int64_t Value = CE->getValue();
-
-    // The offset must be in the range 0-255 (imm8).
-    if (Value > 255 || Value < -255)
-      return false;
-
-    return true;
+    int64_t Val = CE->getValue();
+    return (Val > -256 && Val < 256) || (Val == INT32_MIN);
   }
-  bool isMemMode5() const {
-    if (!isMemory() || getMemOffsetIsReg() || getMemWriteback() ||
-        getMemNegative())
+  bool isPostIdxImm8s4() const {
+    if (Kind != k_Immediate)
       return false;
-
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     if (!CE) return false;
-
-    // The offset must be a multiple of 4 in the range 0-1020.
-    int64_t Value = CE->getValue();
-    return ((Value & 0x3) == 0 && Value <= 1020 && Value >= -1020);
+    int64_t Val = CE->getValue();
+    return ((Val & 3) == 0 && Val >= -1020 && Val <= 1020) ||
+      (Val == INT32_MIN);
   }
-  bool isMemMode7() const {
-    if (!isMemory() ||
-        getMemPreindexed() ||
-        getMemPostindexed() ||
-        getMemOffsetIsReg() ||
-        getMemNegative() ||
-        getMemWriteback())
-      return false;
 
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    if (!CE) return false;
+  bool isMSRMask() const { return Kind == k_MSRMask; }
+  bool isProcIFlags() const { return Kind == k_ProcIFlags; }
 
-    if (CE->getValue())
-      return false;
-
-    return true;
+  bool isVectorIndex8() const {
+    if (Kind != k_VectorIndex) return false;
+    return VectorIndex.Val < 8;
   }
-  bool isMemModeRegThumb() const {
-    if (!isMemory() || !getMemOffsetIsReg() || getMemWriteback())
-      return false;
-    return true;
+  bool isVectorIndex16() const {
+    if (Kind != k_VectorIndex) return false;
+    return VectorIndex.Val < 4;
   }
-  bool isMemModeImmThumb() const {
-    if (!isMemory() || getMemOffsetIsReg() || getMemWriteback())
-      return false;
-
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    if (!CE) return false;
-
-    // The offset must be a multiple of 4 in the range 0-124.
-    uint64_t Value = CE->getValue();
-    return ((Value & 0x3) == 0 && Value <= 124);
+  bool isVectorIndex32() const {
+    if (Kind != k_VectorIndex) return false;
+    return VectorIndex.Val < 2;
   }
-  bool isMSRMask() const { return Kind == MSRMask; }
-  bool isProcIFlags() const { return Kind == ProcIFlags; }
+
+
 
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     // Add as immediates when possible.  Null MCExpr = 0.
@@ -544,6 +940,21 @@ class ARMOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::CreateImm(getCoproc()));
   }
 
+  void addCoprocOptionOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(CoprocOption.Val));
+  }
+
+  void addITMaskOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(ITMask.Mask));
+  }
+
+  void addITCondCodeOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(unsigned(getCondCode())));
+  }
+
   void addCCOutOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::CreateReg(getReg()));
@@ -554,22 +965,27 @@ class ARMOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::CreateReg(getReg()));
   }
 
-  void addShiftedRegOperands(MCInst &Inst, unsigned N) const {
+  void addRegShiftedRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 3 && "Invalid number of operands!");
-    assert(isShiftedReg() && "addShiftedRegOperands() on non ShiftedReg!");
-    assert((ShiftedReg.ShiftReg == 0 ||
-            ARM_AM::getSORegOffset(ShiftedReg.ShiftImm) == 0) &&
-           "Invalid shifted register operand!");
-    Inst.addOperand(MCOperand::CreateReg(ShiftedReg.SrcReg));
-    Inst.addOperand(MCOperand::CreateReg(ShiftedReg.ShiftReg));
+    assert(isRegShiftedReg() && "addRegShiftedRegOperands() on non RegShiftedReg!");
+    Inst.addOperand(MCOperand::CreateReg(RegShiftedReg.SrcReg));
+    Inst.addOperand(MCOperand::CreateReg(RegShiftedReg.ShiftReg));
     Inst.addOperand(MCOperand::CreateImm(
-      ARM_AM::getSORegOpc(ShiftedReg.ShiftTy, ShiftedReg.ShiftImm)));
+      ARM_AM::getSORegOpc(RegShiftedReg.ShiftTy, RegShiftedReg.ShiftImm)));
   }
 
-  void addShifterOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
+  void addRegShiftedImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    assert(isRegShiftedImm() && "addRegShiftedImmOperands() on non RegShiftedImm!");
+    Inst.addOperand(MCOperand::CreateReg(RegShiftedImm.SrcReg));
     Inst.addOperand(MCOperand::CreateImm(
-      ARM_AM::getSORegOpc(Shift.ShiftTy, 0)));
+      ARM_AM::getSORegOpc(RegShiftedImm.ShiftTy, RegShiftedImm.ShiftImm)));
+  }
+
+  void addShifterImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm((ShifterImm.isASR << 5) |
+                                         ShifterImm.Imm));
   }
 
   void addRegListOperands(MCInst &Inst, unsigned N) const {
@@ -588,11 +1004,57 @@ class ARMOperand : public MCParsedAsmOperand {
     addRegListOperands(Inst, N);
   }
 
+  void addRotImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // Encoded as val>>3. The printer handles display as 8, 16, 24.
+    Inst.addOperand(MCOperand::CreateImm(RotImm.Imm >> 3));
+  }
+
+  void addBitfieldOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // Munge the lsb/width into a bitfield mask.
+    unsigned lsb = Bitfield.LSB;
+    unsigned width = Bitfield.Width;
+    // Make a 32-bit mask w/ the referenced bits clear and all other bits set.
+    uint32_t Mask = ~(((uint32_t)0xffffffff >> lsb) << (32 - width) >>
+                      (32 - (lsb + width)));
+    Inst.addOperand(MCOperand::CreateImm(Mask));
+  }
+
   void addImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     addExpr(Inst, getImm());
   }
 
+  void addFPImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(getFPImm()));
+  }
+
+  void addImm8s4Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // FIXME: We really want to scale the value here, but the LDRD/STRD
+    // instruction don't encode operands that way yet.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+  }
+
+  void addImm0_1020s4Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // The immediate is scaled by four in the encoding and is stored
+    // in the MCInst as such. Lop off the low two bits here.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    Inst.addOperand(MCOperand::CreateImm(CE->getValue() / 4));
+  }
+
+  void addImm0_508s4Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // The immediate is scaled by four in the encoding and is stored
+    // in the MCInst as such. Lop off the low two bits here.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    Inst.addOperand(MCOperand::CreateImm(CE->getValue() / 4));
+  }
+
   void addImm0_255Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     addExpr(Inst, getImm());
@@ -608,137 +1070,344 @@ class ARMOperand : public MCParsedAsmOperand {
     addExpr(Inst, getImm());
   }
 
+  void addImm0_31Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    addExpr(Inst, getImm());
+  }
+
+  void addImm1_16Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // The constant encodes as the immediate-1, and we store in the instruction
+    // the bits as encoded, so subtract off one here.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    Inst.addOperand(MCOperand::CreateImm(CE->getValue() - 1));
+  }
+
+  void addImm1_32Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // The constant encodes as the immediate-1, and we store in the instruction
+    // the bits as encoded, so subtract off one here.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    Inst.addOperand(MCOperand::CreateImm(CE->getValue() - 1));
+  }
+
   void addImm0_65535Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     addExpr(Inst, getImm());
   }
 
+  void addImm0_65535ExprOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    addExpr(Inst, getImm());
+  }
+
+  void addImm24bitOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    addExpr(Inst, getImm());
+  }
+
+  void addImmThumbSROperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // The constant encodes as the immediate, except for 32, which encodes as
+    // zero.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    unsigned Imm = CE->getValue();
+    Inst.addOperand(MCOperand::CreateImm((Imm == 32 ? 0 : Imm)));
+  }
+
+  void addPKHLSLImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    addExpr(Inst, getImm());
+  }
+
+  void addPKHASRImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // An ASR value of 32 encodes as 0, so that's how we want to add it to
+    // the instruction as well.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    int Val = CE->getValue();
+    Inst.addOperand(MCOperand::CreateImm(Val == 32 ? 0 : Val));
+  }
+
+  void addARMSOImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    addExpr(Inst, getImm());
+  }
+
   void addT2SOImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     addExpr(Inst, getImm());
   }
 
+  void addSetEndImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    addExpr(Inst, getImm());
+  }
+
   void addMemBarrierOptOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::CreateImm(unsigned(getMemBarrierOpt())));
   }
 
-  void addMemMode7Operands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && isMemMode7() && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseRegNum()));
-
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    (void)CE;
-    assert((CE || CE->getValue() == 0) &&
-           "No offset operand support in mode 7");
+  void addMemNoOffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
   }
 
-  void addMemMode2Operands(MCInst &Inst, unsigned N) const {
-    assert(isMemMode2() && "Invalid mode or number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseRegNum()));
-    unsigned IdxMode = (getMemPreindexed() | getMemPostindexed() << 1);
+  void addAlignedMemoryOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Memory.Alignment));
+  }
 
-    if (getMemOffsetIsReg()) {
-      Inst.addOperand(MCOperand::CreateReg(getMemOffsetRegNum()));
+  void addAddrMode2Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 3 && "Invalid number of operands!");
+    int32_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
+    if (!Memory.OffsetRegNum) {
+      ARM_AM::AddrOpc AddSub = Val < 0 ? ARM_AM::sub : ARM_AM::add;
+      // Special case for #-0
+      if (Val == INT32_MIN) Val = 0;
+      if (Val < 0) Val = -Val;
+      Val = ARM_AM::getAM2Opc(AddSub, Val, ARM_AM::no_shift);
+    } else {
+      // For register offset, we encode the shift type and negation flag
+      // here.
+      Val = ARM_AM::getAM2Opc(Memory.isNegative ? ARM_AM::sub : ARM_AM::add,
+                              Memory.ShiftImm, Memory.ShiftType);
+    }
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
 
-      ARM_AM::AddrOpc AMOpc = getMemNegative() ? ARM_AM::sub : ARM_AM::add;
-      ARM_AM::ShiftOpc ShOpc = ARM_AM::no_shift;
-      int64_t ShiftAmount = 0;
+  void addAM2OffsetImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    assert(CE && "non-constant AM2OffsetImm operand!");
+    int32_t Val = CE->getValue();
+    ARM_AM::AddrOpc AddSub = Val < 0 ? ARM_AM::sub : ARM_AM::add;
+    // Special case for #-0
+    if (Val == INT32_MIN) Val = 0;
+    if (Val < 0) Val = -Val;
+    Val = ARM_AM::getAM2Opc(AddSub, Val, ARM_AM::no_shift);
+    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
 
-      if (getMemOffsetRegShifted()) {
-        ShOpc = getMemShiftType();
-        const MCConstantExpr *CE =
-                   dyn_cast<MCConstantExpr>(getMemShiftAmount());
-        ShiftAmount = CE->getValue();
-      }
+  void addAddrMode3Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 3 && "Invalid number of operands!");
+    int32_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
+    if (!Memory.OffsetRegNum) {
+      ARM_AM::AddrOpc AddSub = Val < 0 ? ARM_AM::sub : ARM_AM::add;
+      // Special case for #-0
+      if (Val == INT32_MIN) Val = 0;
+      if (Val < 0) Val = -Val;
+      Val = ARM_AM::getAM3Opc(AddSub, Val);
+    } else {
+      // For register offset, we encode the shift type and negation flag
+      // here.
+      Val = ARM_AM::getAM3Opc(Memory.isNegative ? ARM_AM::sub : ARM_AM::add, 0);
+    }
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
 
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM2Opc(AMOpc, ShiftAmount,
-                                           ShOpc, IdxMode)));
+  void addAM3OffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    if (Kind == k_PostIndexRegister) {
+      int32_t Val =
+        ARM_AM::getAM3Opc(PostIdxReg.isAdd ? ARM_AM::add : ARM_AM::sub, 0);
+      Inst.addOperand(MCOperand::CreateReg(PostIdxReg.RegNum));
+      Inst.addOperand(MCOperand::CreateImm(Val));
       return;
     }
 
-    // Create a operand placeholder to always yield the same number of operands.
+    // Constant offset.
+    const MCConstantExpr *CE = static_cast<const MCConstantExpr*>(getImm());
+    int32_t Val = CE->getValue();
+    ARM_AM::AddrOpc AddSub = Val < 0 ? ARM_AM::sub : ARM_AM::add;
+    // Special case for #-0
+    if (Val == INT32_MIN) Val = 0;
+    if (Val < 0) Val = -Val;
+    Val = ARM_AM::getAM3Opc(AddSub, Val);
     Inst.addOperand(MCOperand::CreateReg(0));
-
-    // FIXME: #-0 is encoded differently than #0. Does the parser preserve
-    // the difference?
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    assert(CE && "Non-constant mode 2 offset operand!");
-    int64_t Offset = CE->getValue();
-
-    if (Offset >= 0)
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM2Opc(ARM_AM::add,
-                                           Offset, ARM_AM::no_shift, IdxMode)));
-    else
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM2Opc(ARM_AM::sub,
-                                          -Offset, ARM_AM::no_shift, IdxMode)));
+    Inst.addOperand(MCOperand::CreateImm(Val));
   }
 
-  void addMemMode3Operands(MCInst &Inst, unsigned N) const {
-    assert(isMemMode3() && "Invalid mode or number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseRegNum()));
-    unsigned IdxMode = (getMemPreindexed() | getMemPostindexed() << 1);
+  void addAddrMode5Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    // The lower two bits are always zero and as such are not encoded.
+    int32_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() / 4 : 0;
+    ARM_AM::AddrOpc AddSub = Val < 0 ? ARM_AM::sub : ARM_AM::add;
+    // Special case for #-0
+    if (Val == INT32_MIN) Val = 0;
+    if (Val < 0) Val = -Val;
+    Val = ARM_AM::getAM5Opc(AddSub, Val);
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
 
-    if (getMemOffsetIsReg()) {
-      Inst.addOperand(MCOperand::CreateReg(getMemOffsetRegNum()));
+  void addMemImm8s4OffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
 
-      ARM_AM::AddrOpc AMOpc = getMemNegative() ? ARM_AM::sub : ARM_AM::add;
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM3Opc(AMOpc, 0,
-                                                             IdxMode)));
+  void addMemImm0_1020s4OffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    // The lower two bits are always zero and as such are not encoded.
+    int32_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() / 4 : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addMemImm8OffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addMemPosImm8OffsetOperands(MCInst &Inst, unsigned N) const {
+    addMemImm8OffsetOperands(Inst, N);
+  }
+
+  void addMemNegImm8OffsetOperands(MCInst &Inst, unsigned N) const {
+    addMemImm8OffsetOperands(Inst, N);
+  }
+
+  void addMemUImm12OffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    // If this is an immediate, it's a label reference.
+    if (Kind == k_Immediate) {
+      addExpr(Inst, getImm());
+      Inst.addOperand(MCOperand::CreateImm(0));
       return;
     }
 
-    // Create a operand placeholder to always yield the same number of operands.
-    Inst.addOperand(MCOperand::CreateReg(0));
-
-    // FIXME: #-0 is encoded differently than #0. Does the parser preserve
-    // the difference?
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    assert(CE && "Non-constant mode 3 offset operand!");
-    int64_t Offset = CE->getValue();
-
-    if (Offset >= 0)
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM3Opc(ARM_AM::add,
-                                           Offset, IdxMode)));
-    else
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM3Opc(ARM_AM::sub,
-                                           -Offset, IdxMode)));
+    // Otherwise, it's a normal memory reg+offset.
+    int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
   }
 
-  void addMemMode5Operands(MCInst &Inst, unsigned N) const {
-    assert(N == 2 && isMemMode5() && "Invalid number of operands!");
+  void addMemImm12OffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    // If this is an immediate, it's a label reference.
+    if (Kind == k_Immediate) {
+      addExpr(Inst, getImm());
+      Inst.addOperand(MCOperand::CreateImm(0));
+      return;
+    }
 
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseRegNum()));
-    assert(!getMemOffsetIsReg() && "Invalid mode 5 operand");
-
-    // FIXME: #-0 is encoded differently than #0. Does the parser preserve
-    // the difference?
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    assert(CE && "Non-constant mode 5 offset operand!");
-
-    // The MCInst offset operand doesn't include the low two bits (like
-    // the instruction encoding).
-    int64_t Offset = CE->getValue() / 4;
-    if (Offset >= 0)
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(ARM_AM::add,
-                                                             Offset)));
-    else
-      Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(ARM_AM::sub,
-                                                             -Offset)));
+    // Otherwise, it's a normal memory reg+offset.
+    int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
   }
 
-  void addMemModeRegThumbOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 2 && isMemModeRegThumb() && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseRegNum()));
-    Inst.addOperand(MCOperand::CreateReg(getMemOffsetRegNum()));
+  void addMemTBBOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
   }
 
-  void addMemModeImmThumbOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 2 && isMemModeImmThumb() && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseRegNum()));
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemOffset());
-    assert(CE && "Non-constant mode offset operand!");
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+  void addMemTBHOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+  }
+
+  void addMemRegOffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 3 && "Invalid number of operands!");
+    unsigned Val = ARM_AM::getAM2Opc(Memory.isNegative ? ARM_AM::sub : ARM_AM::add,
+                                     Memory.ShiftImm, Memory.ShiftType);
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addT2MemRegOffsetOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 3 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Memory.ShiftImm));
+  }
+
+  void addMemThumbRROperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+  }
+
+  void addMemThumbRIs4Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue() / 4) : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addMemThumbRIs2Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue() / 2) : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addMemThumbRIs1Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue()) : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addMemThumbSPIOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue() / 4) : 0;
+    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::CreateImm(Val));
+  }
+
+  void addPostIdxImm8Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    assert(CE && "non-constant post-idx-imm8 operand!");
+    int Imm = CE->getValue();
+    bool isAdd = Imm >= 0;
+    if (Imm == INT32_MIN) Imm = 0;
+    Imm = (Imm < 0 ? -Imm : Imm) | (int)isAdd << 8;
+    Inst.addOperand(MCOperand::CreateImm(Imm));
+  }
+
+  void addPostIdxImm8s4Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    assert(CE && "non-constant post-idx-imm8s4 operand!");
+    int Imm = CE->getValue();
+    bool isAdd = Imm >= 0;
+    if (Imm == INT32_MIN) Imm = 0;
+    // Immediate is scaled by 4.
+    Imm = ((Imm < 0 ? -Imm : Imm) / 4) | (int)isAdd << 8;
+    Inst.addOperand(MCOperand::CreateImm(Imm));
+  }
+
+  void addPostIdxRegOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(PostIdxReg.RegNum));
+    Inst.addOperand(MCOperand::CreateImm(PostIdxReg.isAdd));
+  }
+
+  void addPostIdxRegShiftedOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(PostIdxReg.RegNum));
+    // The sign, shift type, and shift amount are encoded in a single operand
+    // using the AM2 encoding helpers.
+    ARM_AM::AddrOpc opc = PostIdxReg.isAdd ? ARM_AM::add : ARM_AM::sub;
+    unsigned Imm = ARM_AM::getAM2Opc(opc, PostIdxReg.ShiftImm,
+                                     PostIdxReg.ShiftTy);
+    Inst.addOperand(MCOperand::CreateImm(Imm));
   }
 
   void addMSRMaskOperands(MCInst &Inst, unsigned N) const {
@@ -751,10 +1420,33 @@ class ARMOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::CreateImm(unsigned(getProcIFlags())));
   }
 
+  void addVectorIndex8Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+  }
+
+  void addVectorIndex16Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+  }
+
+  void addVectorIndex32Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+  }
+
   virtual void print(raw_ostream &OS) const;
 
+  static ARMOperand *CreateITMask(unsigned Mask, SMLoc S) {
+    ARMOperand *Op = new ARMOperand(k_ITCondMask);
+    Op->ITMask.Mask = Mask;
+    Op->StartLoc = S;
+    Op->EndLoc = S;
+    return Op;
+  }
+
   static ARMOperand *CreateCondCode(ARMCC::CondCodes CC, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(CondCode);
+    ARMOperand *Op = new ARMOperand(k_CondCode);
     Op->CC.Val = CC;
     Op->StartLoc = S;
     Op->EndLoc = S;
@@ -762,7 +1454,7 @@ class ARMOperand : public MCParsedAsmOperand {
   }
 
   static ARMOperand *CreateCoprocNum(unsigned CopVal, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(CoprocNum);
+    ARMOperand *Op = new ARMOperand(k_CoprocNum);
     Op->Cop.Val = CopVal;
     Op->StartLoc = S;
     Op->EndLoc = S;
@@ -770,15 +1462,23 @@ class ARMOperand : public MCParsedAsmOperand {
   }
 
   static ARMOperand *CreateCoprocReg(unsigned CopVal, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(CoprocReg);
+    ARMOperand *Op = new ARMOperand(k_CoprocReg);
     Op->Cop.Val = CopVal;
     Op->StartLoc = S;
     Op->EndLoc = S;
     return Op;
   }
 
+  static ARMOperand *CreateCoprocOption(unsigned Val, SMLoc S, SMLoc E) {
+    ARMOperand *Op = new ARMOperand(k_CoprocOption);
+    Op->Cop.Val = Val;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
   static ARMOperand *CreateCCOut(unsigned RegNum, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(CCOut);
+    ARMOperand *Op = new ARMOperand(k_CCOut);
     Op->Reg.RegNum = RegNum;
     Op->StartLoc = S;
     Op->EndLoc = S;
@@ -786,7 +1486,7 @@ class ARMOperand : public MCParsedAsmOperand {
   }
 
   static ARMOperand *CreateToken(StringRef Str, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(Token);
+    ARMOperand *Op = new ARMOperand(k_Token);
     Op->Tok.Data = Str.data();
     Op->Tok.Length = Str.size();
     Op->StartLoc = S;
@@ -795,7 +1495,7 @@ class ARMOperand : public MCParsedAsmOperand {
   }
 
   static ARMOperand *CreateReg(unsigned RegNum, SMLoc S, SMLoc E) {
-    ARMOperand *Op = new ARMOperand(Register);
+    ARMOperand *Op = new ARMOperand(k_Register);
     Op->Reg.RegNum = RegNum;
     Op->StartLoc = S;
     Op->EndLoc = E;
@@ -807,20 +1507,52 @@ class ARMOperand : public MCParsedAsmOperand {
                                            unsigned ShiftReg,
                                            unsigned ShiftImm,
                                            SMLoc S, SMLoc E) {
-    ARMOperand *Op = new ARMOperand(ShiftedRegister);
-    Op->ShiftedReg.ShiftTy = ShTy;
-    Op->ShiftedReg.SrcReg = SrcReg;
-    Op->ShiftedReg.ShiftReg = ShiftReg;
-    Op->ShiftedReg.ShiftImm = ShiftImm;
+    ARMOperand *Op = new ARMOperand(k_ShiftedRegister);
+    Op->RegShiftedReg.ShiftTy = ShTy;
+    Op->RegShiftedReg.SrcReg = SrcReg;
+    Op->RegShiftedReg.ShiftReg = ShiftReg;
+    Op->RegShiftedReg.ShiftImm = ShiftImm;
     Op->StartLoc = S;
     Op->EndLoc = E;
     return Op;
   }
 
-  static ARMOperand *CreateShifter(ARM_AM::ShiftOpc ShTy,
+  static ARMOperand *CreateShiftedImmediate(ARM_AM::ShiftOpc ShTy,
+                                            unsigned SrcReg,
+                                            unsigned ShiftImm,
+                                            SMLoc S, SMLoc E) {
+    ARMOperand *Op = new ARMOperand(k_ShiftedImmediate);
+    Op->RegShiftedImm.ShiftTy = ShTy;
+    Op->RegShiftedImm.SrcReg = SrcReg;
+    Op->RegShiftedImm.ShiftImm = ShiftImm;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  static ARMOperand *CreateShifterImm(bool isASR, unsigned Imm,
                                    SMLoc S, SMLoc E) {
-    ARMOperand *Op = new ARMOperand(Shifter);
-    Op->Shift.ShiftTy = ShTy;
+    ARMOperand *Op = new ARMOperand(k_ShifterImmediate);
+    Op->ShifterImm.isASR = isASR;
+    Op->ShifterImm.Imm = Imm;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  static ARMOperand *CreateRotImm(unsigned Imm, SMLoc S, SMLoc E) {
+    ARMOperand *Op = new ARMOperand(k_RotateImmediate);
+    Op->RotImm.Imm = Imm;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  static ARMOperand *CreateBitfield(unsigned LSB, unsigned Width,
+                                    SMLoc S, SMLoc E) {
+    ARMOperand *Op = new ARMOperand(k_BitfieldDescriptor);
+    Op->Bitfield.LSB = LSB;
+    Op->Bitfield.Width = Width;
     Op->StartLoc = S;
     Op->EndLoc = E;
     return Op;
@@ -829,12 +1561,13 @@ class ARMOperand : public MCParsedAsmOperand {
   static ARMOperand *
   CreateRegList(const SmallVectorImpl<std::pair<unsigned, SMLoc> > &Regs,
                 SMLoc StartLoc, SMLoc EndLoc) {
-    KindTy Kind = RegisterList;
+    KindTy Kind = k_RegisterList;
 
-    if (ARM::DPRRegClass.contains(Regs.front().first))
-      Kind = DPRRegisterList;
-    else if (ARM::SPRRegClass.contains(Regs.front().first))
-      Kind = SPRRegisterList;
+    if (ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Regs.front().first))
+      Kind = k_DPRRegisterList;
+    else if (ARMMCRegisterClasses[ARM::SPRRegClassID].
+             contains(Regs.front().first))
+      Kind = k_SPRRegisterList;
 
     ARMOperand *Op = new ARMOperand(Kind);
     for (SmallVectorImpl<std::pair<unsigned, SMLoc> >::const_iterator
@@ -846,55 +1579,68 @@ class ARMOperand : public MCParsedAsmOperand {
     return Op;
   }
 
+  static ARMOperand *CreateVectorIndex(unsigned Idx, SMLoc S, SMLoc E,
+                                       MCContext &Ctx) {
+    ARMOperand *Op = new ARMOperand(k_VectorIndex);
+    Op->VectorIndex.Val = Idx;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
   static ARMOperand *CreateImm(const MCExpr *Val, SMLoc S, SMLoc E) {
-    ARMOperand *Op = new ARMOperand(Immediate);
+    ARMOperand *Op = new ARMOperand(k_Immediate);
     Op->Imm.Val = Val;
     Op->StartLoc = S;
     Op->EndLoc = E;
     return Op;
   }
 
-  static ARMOperand *CreateMem(ARMII::AddrMode AddrMode, unsigned BaseRegNum,
-                               bool OffsetIsReg, const MCExpr *Offset,
-                               int OffsetRegNum, bool OffsetRegShifted,
-                               enum ARM_AM::ShiftOpc ShiftType,
-                               const MCExpr *ShiftAmount, bool Preindexed,
-                               bool Postindexed, bool Negative, bool Writeback,
+  static ARMOperand *CreateFPImm(unsigned Val, SMLoc S, MCContext &Ctx) {
+    ARMOperand *Op = new ARMOperand(k_FPImmediate);
+    Op->FPImm.Val = Val;
+    Op->StartLoc = S;
+    Op->EndLoc = S;
+    return Op;
+  }
+
+  static ARMOperand *CreateMem(unsigned BaseRegNum,
+                               const MCConstantExpr *OffsetImm,
+                               unsigned OffsetRegNum,
+                               ARM_AM::ShiftOpc ShiftType,
+                               unsigned ShiftImm,
+                               unsigned Alignment,
+                               bool isNegative,
                                SMLoc S, SMLoc E) {
-    assert((OffsetRegNum == -1 || OffsetIsReg) &&
-           "OffsetRegNum must imply OffsetIsReg!");
-    assert((!OffsetRegShifted || OffsetIsReg) &&
-           "OffsetRegShifted must imply OffsetIsReg!");
-    assert((Offset || OffsetIsReg) &&
-           "Offset must exists unless register offset is used!");
-    assert((!ShiftAmount || (OffsetIsReg && OffsetRegShifted)) &&
-           "Cannot have shift amount without shifted register offset!");
-    assert((!Offset || !OffsetIsReg) &&
-           "Cannot have expression offset and register offset!");
-
-    ARMOperand *Op = new ARMOperand(Memory);
-    Op->Mem.AddrMode = AddrMode;
-    Op->Mem.BaseRegNum = BaseRegNum;
-    Op->Mem.OffsetIsReg = OffsetIsReg;
-    if (OffsetIsReg)
-      Op->Mem.Offset.RegNum = OffsetRegNum;
-    else
-      Op->Mem.Offset.Value = Offset;
-    Op->Mem.OffsetRegShifted = OffsetRegShifted;
-    Op->Mem.ShiftType = ShiftType;
-    Op->Mem.ShiftAmount = ShiftAmount;
-    Op->Mem.Preindexed = Preindexed;
-    Op->Mem.Postindexed = Postindexed;
-    Op->Mem.Negative = Negative;
-    Op->Mem.Writeback = Writeback;
+    ARMOperand *Op = new ARMOperand(k_Memory);
+    Op->Memory.BaseRegNum = BaseRegNum;
+    Op->Memory.OffsetImm = OffsetImm;
+    Op->Memory.OffsetRegNum = OffsetRegNum;
+    Op->Memory.ShiftType = ShiftType;
+    Op->Memory.ShiftImm = ShiftImm;
+    Op->Memory.Alignment = Alignment;
+    Op->Memory.isNegative = isNegative;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
 
+  static ARMOperand *CreatePostIdxReg(unsigned RegNum, bool isAdd,
+                                      ARM_AM::ShiftOpc ShiftTy,
+                                      unsigned ShiftImm,
+                                      SMLoc S, SMLoc E) {
+    ARMOperand *Op = new ARMOperand(k_PostIndexRegister);
+    Op->PostIdxReg.RegNum = RegNum;
+    Op->PostIdxReg.isAdd = isAdd;
+    Op->PostIdxReg.ShiftTy = ShiftTy;
+    Op->PostIdxReg.ShiftImm = ShiftImm;
     Op->StartLoc = S;
     Op->EndLoc = E;
     return Op;
   }
 
   static ARMOperand *CreateMemBarrierOpt(ARM_MB::MemBOpt Opt, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(MemBarrierOpt);
+    ARMOperand *Op = new ARMOperand(k_MemBarrierOpt);
     Op->MBOpt.Val = Opt;
     Op->StartLoc = S;
     Op->EndLoc = S;
@@ -902,7 +1648,7 @@ class ARMOperand : public MCParsedAsmOperand {
   }
 
   static ARMOperand *CreateProcIFlags(ARM_PROC::IFlags IFlags, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(ProcIFlags);
+    ARMOperand *Op = new ARMOperand(k_ProcIFlags);
     Op->IFlags.Val = IFlags;
     Op->StartLoc = S;
     Op->EndLoc = S;
@@ -910,7 +1656,7 @@ class ARMOperand : public MCParsedAsmOperand {
   }
 
   static ARMOperand *CreateMSRMask(unsigned MMask, SMLoc S) {
-    ARMOperand *Op = new ARMOperand(MSRMask);
+    ARMOperand *Op = new ARMOperand(k_MSRMask);
     Op->MMask.Val = MMask;
     Op->StartLoc = S;
     Op->EndLoc = S;
@@ -922,53 +1668,56 @@ class ARMOperand : public MCParsedAsmOperand {
 
 void ARMOperand::print(raw_ostream &OS) const {
   switch (Kind) {
-  case CondCode:
+  case k_FPImmediate:
+    OS << "<fpimm " << getFPImm() << "(" << ARM_AM::getFPImmFloat(getFPImm())
+       << ") >";
+    break;
+  case k_CondCode:
     OS << "<ARMCC::" << ARMCondCodeToString(getCondCode()) << ">";
     break;
-  case CCOut:
+  case k_CCOut:
     OS << "<ccout " << getReg() << ">";
     break;
-  case CoprocNum:
+  case k_ITCondMask: {
+    static char MaskStr[][6] = { "()", "(t)", "(e)", "(tt)", "(et)", "(te)",
+      "(ee)", "(ttt)", "(ett)", "(tet)", "(eet)", "(tte)", "(ete)",
+      "(tee)", "(eee)" };
+    assert((ITMask.Mask & 0xf) == ITMask.Mask);
+    OS << "<it-mask " << MaskStr[ITMask.Mask] << ">";
+    break;
+  }
+  case k_CoprocNum:
     OS << "<coprocessor number: " << getCoproc() << ">";
     break;
-  case CoprocReg:
+  case k_CoprocReg:
     OS << "<coprocessor register: " << getCoproc() << ">";
     break;
-  case MSRMask:
+  case k_CoprocOption:
+    OS << "<coprocessor option: " << CoprocOption.Val << ">";
+    break;
+  case k_MSRMask:
     OS << "<mask: " << getMSRMask() << ">";
     break;
-  case Immediate:
+  case k_Immediate:
     getImm()->print(OS);
     break;
-  case MemBarrierOpt:
+  case k_MemBarrierOpt:
     OS << "<ARM_MB::" << MemBOptToString(getMemBarrierOpt()) << ">";
     break;
-  case Memory:
+  case k_Memory:
     OS << "<memory "
-       << "am:" << ARMII::AddrModeToString(getMemAddrMode())
-       << " base:" << getMemBaseRegNum();
-    if (getMemOffsetIsReg()) {
-      OS << " offset:<register " << getMemOffsetRegNum();
-      if (getMemOffsetRegShifted()) {
-        OS << " offset-shift-type:" << getMemShiftType();
-        OS << " offset-shift-amount:" << *getMemShiftAmount();
-      }
-    } else {
-      OS << " offset:" << *getMemOffset();
-    }
-    if (getMemOffsetIsReg())
-      OS << " (offset-is-reg)";
-    if (getMemPreindexed())
-      OS << " (pre-indexed)";
-    if (getMemPostindexed())
-      OS << " (post-indexed)";
-    if (getMemNegative())
-      OS << " (negative)";
-    if (getMemWriteback())
-      OS << " (writeback)";
+       << " base:" << Memory.BaseRegNum;
     OS << ">";
     break;
-  case ProcIFlags: {
+  case k_PostIndexRegister:
+    OS << "post-idx register " << (PostIdxReg.isAdd ? "" : "-")
+       << PostIdxReg.RegNum;
+    if (PostIdxReg.ShiftTy != ARM_AM::no_shift)
+      OS << ARM_AM::getShiftOpcStr(PostIdxReg.ShiftTy) << " "
+         << PostIdxReg.ShiftImm;
+    OS << ">";
+    break;
+  case k_ProcIFlags: {
     OS << "<ARM_PROC::";
     unsigned IFlags = getProcIFlags();
     for (int i=2; i >= 0; --i)
@@ -977,23 +1726,38 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << ">";
     break;
   }
-  case Register:
+  case k_Register:
     OS << "<register " << getReg() << ">";
     break;
-  case Shifter:
-    OS << "<shifter " << ARM_AM::getShiftOpcStr(Shift.ShiftTy) << ">";
+  case k_ShifterImmediate:
+    OS << "<shift " << (ShifterImm.isASR ? "asr" : "lsl")
+       << " #" << ShifterImm.Imm << ">";
     break;
-  case ShiftedRegister:
-    OS << "<so_reg"
-       << ShiftedReg.SrcReg
-       << ARM_AM::getShiftOpcStr(ARM_AM::getSORegShOp(ShiftedReg.ShiftImm))
-       << ", " << ShiftedReg.ShiftReg << ", "
-       << ARM_AM::getSORegOffset(ShiftedReg.ShiftImm)
+  case k_ShiftedRegister:
+    OS << "<so_reg_reg "
+       << RegShiftedReg.SrcReg
+       << ARM_AM::getShiftOpcStr(ARM_AM::getSORegShOp(RegShiftedReg.ShiftImm))
+       << ", " << RegShiftedReg.ShiftReg << ", "
+       << ARM_AM::getSORegOffset(RegShiftedReg.ShiftImm)
        << ">";
     break;
-  case RegisterList:
-  case DPRRegisterList:
-  case SPRRegisterList: {
+  case k_ShiftedImmediate:
+    OS << "<so_reg_imm "
+       << RegShiftedImm.SrcReg
+       << ARM_AM::getShiftOpcStr(ARM_AM::getSORegShOp(RegShiftedImm.ShiftImm))
+       << ", " << ARM_AM::getSORegOffset(RegShiftedImm.ShiftImm)
+       << ">";
+    break;
+  case k_RotateImmediate:
+    OS << "<ror " << " #" << (RotImm.Imm * 8) << ">";
+    break;
+  case k_BitfieldDescriptor:
+    OS << "<bitfield " << "lsb: " << Bitfield.LSB
+       << ", width: " << Bitfield.Width << ">";
+    break;
+  case k_RegisterList:
+  case k_DPRRegisterList:
+  case k_SPRRegisterList: {
     OS << "<register_list ";
 
     const SmallVectorImpl<unsigned> &RegList = getRegList();
@@ -1006,9 +1770,12 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << ">";
     break;
   }
-  case Token:
+  case k_Token:
     OS << "'" << getToken() << "'";
     break;
+  case k_VectorIndex:
+    OS << "<vectorindex " << getVectorIndex() << ">";
+    break;
   }
 }
 
@@ -1021,7 +1788,7 @@ static unsigned MatchRegisterName(StringRef Name);
 
 bool ARMAsmParser::ParseRegister(unsigned &RegNo,
                                  SMLoc &StartLoc, SMLoc &EndLoc) {
-  RegNo = TryParseRegister();
+  RegNo = tryParseRegister();
 
   return (RegNo == (unsigned)-1);
 }
@@ -1030,9 +1797,9 @@ bool ARMAsmParser::ParseRegister(unsigned &RegNo,
 /// and if it is a register name the token is eaten and the register number is
 /// returned.  Otherwise return -1.
 ///
-int ARMAsmParser::TryParseRegister() {
+int ARMAsmParser::tryParseRegister() {
   const AsmToken &Tok = Parser.getTok();
-  assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
+  if (Tok.isNot(AsmToken::Identifier)) return -1;
 
   // FIXME: Validate register for the current architecture; we have to do
   // validation later, so maybe there is no need for this here.
@@ -1050,6 +1817,39 @@ int ARMAsmParser::TryParseRegister() {
   if (!RegNum) return -1;
 
   Parser.Lex(); // Eat identifier token.
+
+#if 0
+  // Also check for an index operand. This is only legal for vector registers,
+  // but that'll get caught OK in operand matching, so we don't need to
+  // explicitly filter everything else out here.
+  if (Parser.getTok().is(AsmToken::LBrac)) {
+    SMLoc SIdx = Parser.getTok().getLoc();
+    Parser.Lex(); // Eat left bracket token.
+
+    const MCExpr *ImmVal;
+    SMLoc ExprLoc = Parser.getTok().getLoc();
+    if (getParser().ParseExpression(ImmVal))
+      return MatchOperand_ParseFail;
+    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
+    if (!MCE) {
+      TokError("immediate value expected for vector index");
+      return MatchOperand_ParseFail;
+    }
+
+    SMLoc E = Parser.getTok().getLoc();
+    if (Parser.getTok().isNot(AsmToken::RBrac)) {
+      Error(E, "']' expected");
+      return MatchOperand_ParseFail;
+    }
+
+    Parser.Lex(); // Eat right bracket token.
+
+    Operands.push_back(ARMOperand::CreateVectorIndex(MCE->getValue(),
+                                                     SIdx, E,
+                                                     getContext()));
+  }
+#endif
+
   return RegNum;
 }
 
@@ -1058,7 +1858,7 @@ int ARMAsmParser::TryParseRegister() {
 // occurs, return -1. An irrecoverable error is one where tokens have been
 // consumed in the process of trying to parse the shifter (i.e., when it is
 // indeed a shifter operand, but malformed).
-int ARMAsmParser::TryParseShiftRegister(
+int ARMAsmParser::tryParseShiftRegister(
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
@@ -1120,7 +1920,7 @@ int ARMAsmParser::TryParseShiftRegister(
         return -1;
       }
     } else if (Parser.getTok().is(AsmToken::Identifier)) {
-      ShiftReg = TryParseRegister();
+      ShiftReg = tryParseRegister();
       SMLoc L = Parser.getTok().getLoc();
       if (ShiftReg == -1) {
         Error (L, "expected immediate or register in shift operand");
@@ -1133,8 +1933,12 @@ int ARMAsmParser::TryParseShiftRegister(
     }
   }
 
-  Operands.push_back(ARMOperand::CreateShiftedRegister(ShiftTy, SrcReg,
-                                                       ShiftReg, Imm,
+  if (ShiftReg && ShiftTy != ARM_AM::rrx)
+    Operands.push_back(ARMOperand::CreateShiftedRegister(ShiftTy, SrcReg,
+                                                         ShiftReg, Imm,
+                                               S, Parser.getTok().getLoc()));
+  else
+    Operands.push_back(ARMOperand::CreateShiftedImmediate(ShiftTy, SrcReg, Imm,
                                                S, Parser.getTok().getLoc()));
 
   return 0;
@@ -1148,9 +1952,9 @@ int ARMAsmParser::TryParseShiftRegister(
 /// TODO this is likely to change to allow different register types and or to
 /// parse for a specific register type.
 bool ARMAsmParser::
-TryParseRegisterWithWriteBack(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+tryParseRegisterWithWriteBack(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
-  int RegNo = TryParseRegister();
+  int RegNo = tryParseRegister();
   if (RegNo == -1)
     return true;
 
@@ -1161,6 +1965,37 @@ TryParseRegisterWithWriteBack(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
     Operands.push_back(ARMOperand::CreateToken(ExclaimTok.getString(),
                                                ExclaimTok.getLoc()));
     Parser.Lex(); // Eat exclaim token
+    return false;
+  }
+
+  // Also check for an index operand. This is only legal for vector registers,
+  // but that'll get caught OK in operand matching, so we don't need to
+  // explicitly filter everything else out here.
+  if (Parser.getTok().is(AsmToken::LBrac)) {
+    SMLoc SIdx = Parser.getTok().getLoc();
+    Parser.Lex(); // Eat left bracket token.
+
+    const MCExpr *ImmVal;
+    SMLoc ExprLoc = Parser.getTok().getLoc();
+    if (getParser().ParseExpression(ImmVal))
+      return MatchOperand_ParseFail;
+    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
+    if (!MCE) {
+      TokError("immediate value expected for vector index");
+      return MatchOperand_ParseFail;
+    }
+
+    SMLoc E = Parser.getTok().getLoc();
+    if (Parser.getTok().isNot(AsmToken::RBrac)) {
+      Error(E, "']' expected");
+      return MatchOperand_ParseFail;
+    }
+
+    Parser.Lex(); // Eat right bracket token.
+
+    Operands.push_back(ARMOperand::CreateVectorIndex(MCE->getValue(),
+                                                     SIdx, E,
+                                                     getContext()));
   }
 
   return false;
@@ -1209,14 +2044,50 @@ static int MatchCoprocessorOperandName(StringRef Name, char CoprocOp) {
   return -1;
 }
 
-/// tryParseCoprocNumOperand - Try to parse an coprocessor number operand. The
+/// parseITCondCode - Try to parse a condition code for an IT instruction.
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseITCondCode(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+  const AsmToken &Tok = Parser.getTok();
+  if (!Tok.is(AsmToken::Identifier))
+    return MatchOperand_NoMatch;
+  unsigned CC = StringSwitch<unsigned>(Tok.getString())
+    .Case("eq", ARMCC::EQ)
+    .Case("ne", ARMCC::NE)
+    .Case("hs", ARMCC::HS)
+    .Case("cs", ARMCC::HS)
+    .Case("lo", ARMCC::LO)
+    .Case("cc", ARMCC::LO)
+    .Case("mi", ARMCC::MI)
+    .Case("pl", ARMCC::PL)
+    .Case("vs", ARMCC::VS)
+    .Case("vc", ARMCC::VC)
+    .Case("hi", ARMCC::HI)
+    .Case("ls", ARMCC::LS)
+    .Case("ge", ARMCC::GE)
+    .Case("lt", ARMCC::LT)
+    .Case("gt", ARMCC::GT)
+    .Case("le", ARMCC::LE)
+    .Case("al", ARMCC::AL)
+    .Default(~0U);
+  if (CC == ~0U)
+    return MatchOperand_NoMatch;
+  Parser.Lex(); // Eat the token.
+
+  Operands.push_back(ARMOperand::CreateCondCode(ARMCC::CondCodes(CC), S));
+
+  return MatchOperand_Success;
+}
+
+/// parseCoprocNumOperand - Try to parse an coprocessor number operand. The
 /// token must be an Identifier when called, and if it is a coprocessor
 /// number, the token is eaten and the operand is added to the operand list.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseCoprocNumOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+parseCoprocNumOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
-  assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
+  if (Tok.isNot(AsmToken::Identifier))
+    return MatchOperand_NoMatch;
 
   int Num = MatchCoprocessorOperandName(Tok.getString(), 'p');
   if (Num == -1)
@@ -1227,14 +2098,15 @@ tryParseCoprocNumOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   return MatchOperand_Success;
 }
 
-/// tryParseCoprocRegOperand - Try to parse an coprocessor register operand. The
+/// parseCoprocRegOperand - Try to parse an coprocessor register operand. The
 /// token must be an Identifier when called, and if it is a coprocessor
 /// number, the token is eaten and the operand is added to the operand list.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseCoprocRegOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+parseCoprocRegOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
-  assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
+  if (Tok.isNot(AsmToken::Identifier))
+    return MatchOperand_NoMatch;
 
   int Reg = MatchCoprocessorOperandName(Tok.getString(), 'c');
   if (Reg == -1)
@@ -1245,93 +2117,155 @@ tryParseCoprocRegOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   return MatchOperand_Success;
 }
 
-/// Parse a register list, return it if successful else return null.  The first
-/// token must be a '{' when called.
+/// parseCoprocOptionOperand - Try to parse an coprocessor option operand.
+/// coproc_option : '{' imm0_255 '}'
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseCoprocOptionOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+
+  // If this isn't a '{', this isn't a coprocessor immediate operand.
+  if (Parser.getTok().isNot(AsmToken::LCurly))
+    return MatchOperand_NoMatch;
+  Parser.Lex(); // Eat the '{'
+
+  const MCExpr *Expr;
+  SMLoc Loc = Parser.getTok().getLoc();
+  if (getParser().ParseExpression(Expr)) {
+    Error(Loc, "illegal expression");
+    return MatchOperand_ParseFail;
+  }
+  const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr);
+  if (!CE || CE->getValue() < 0 || CE->getValue() > 255) {
+    Error(Loc, "coprocessor option must be an immediate in range [0, 255]");
+    return MatchOperand_ParseFail;
+  }
+  int Val = CE->getValue();
+
+  // Check for and consume the closing '}'
+  if (Parser.getTok().isNot(AsmToken::RCurly))
+    return MatchOperand_ParseFail;
+  SMLoc E = Parser.getTok().getLoc();
+  Parser.Lex(); // Eat the '}'
+
+  Operands.push_back(ARMOperand::CreateCoprocOption(Val, S, E));
+  return MatchOperand_Success;
+}
+
+// For register list parsing, we need to map from raw GPR register numbering
+// to the enumeration values. The enumeration values aren't sorted by
+// register number due to our using "sp", "lr" and "pc" as canonical names.
+static unsigned getNextRegister(unsigned Reg) {
+  // If this is a GPR, we need to do it manually, otherwise we can rely
+  // on the sort ordering of the enumeration since the other reg-classes
+  // are sane.
+  if (!ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg))
+    return Reg + 1;
+  switch(Reg) {
+  default: assert(0 && "Invalid GPR number!");
+  case ARM::R0:  return ARM::R1;  case ARM::R1:  return ARM::R2;
+  case ARM::R2:  return ARM::R3;  case ARM::R3:  return ARM::R4;
+  case ARM::R4:  return ARM::R5;  case ARM::R5:  return ARM::R6;
+  case ARM::R6:  return ARM::R7;  case ARM::R7:  return ARM::R8;
+  case ARM::R8:  return ARM::R9;  case ARM::R9:  return ARM::R10;
+  case ARM::R10: return ARM::R11; case ARM::R11: return ARM::R12;
+  case ARM::R12: return ARM::SP;  case ARM::SP:  return ARM::LR;
+  case ARM::LR:  return ARM::PC;  case ARM::PC:  return ARM::R0;
+  }
+}
+
+/// Parse a register list.
 bool ARMAsmParser::
-ParseRegisterList(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+parseRegisterList(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   assert(Parser.getTok().is(AsmToken::LCurly) &&
          "Token is not a Left Curly Brace");
   SMLoc S = Parser.getTok().getLoc();
+  Parser.Lex(); // Eat '{' token.
+  SMLoc RegLoc = Parser.getTok().getLoc();
 
-  // Read the rest of the registers in the list.
-  unsigned PrevRegNum = 0;
-  SmallVector<std::pair<unsigned, SMLoc>, 32> Registers;
+  // Check the first register in the list to see what register class
+  // this is a list of.
+  int Reg = tryParseRegister();
+  if (Reg == -1)
+    return Error(RegLoc, "register expected");
 
-  do {
-    bool IsRange = Parser.getTok().is(AsmToken::Minus);
-    Parser.Lex(); // Eat non-identifier token.
+  MCRegisterClass *RC;
+  if (ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg))
+    RC = &ARMMCRegisterClasses[ARM::GPRRegClassID];
+  else if (ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Reg))
+    RC = &ARMMCRegisterClasses[ARM::DPRRegClassID];
+  else if (ARMMCRegisterClasses[ARM::SPRRegClassID].contains(Reg))
+    RC = &ARMMCRegisterClasses[ARM::SPRRegClassID];
+  else
+    return Error(RegLoc, "invalid register in register list");
 
-    const AsmToken &RegTok = Parser.getTok();
-    SMLoc RegLoc = RegTok.getLoc();
-    if (RegTok.isNot(AsmToken::Identifier)) {
-      Error(RegLoc, "register expected");
-      return true;
+  // The reglist instructions have at most 16 registers, so reserve
+  // space for that many.
+  SmallVector<std::pair<unsigned, SMLoc>, 16> Registers;
+  // Store the first register.
+  Registers.push_back(std::pair<unsigned, SMLoc>(Reg, RegLoc));
+
+  // This starts immediately after the first register token in the list,
+  // so we can see either a comma or a minus (range separator) as a legal
+  // next token.
+  while (Parser.getTok().is(AsmToken::Comma) ||
+         Parser.getTok().is(AsmToken::Minus)) {
+    if (Parser.getTok().is(AsmToken::Minus)) {
+      Parser.Lex(); // Eat the comma.
+      SMLoc EndLoc = Parser.getTok().getLoc();
+      int EndReg = tryParseRegister();
+      if (EndReg == -1)
+        return Error(EndLoc, "register expected");
+      // If the register is the same as the start reg, there's nothing
+      // more to do.
+      if (Reg == EndReg)
+        continue;
+      // The register must be in the same register class as the first.
+      if (!RC->contains(EndReg))
+        return Error(EndLoc, "invalid register in register list");
+      // Ranges must go from low to high.
+      if (getARMRegisterNumbering(Reg) > getARMRegisterNumbering(EndReg))
+        return Error(EndLoc, "bad range in register list");
+
+      // Add all the registers in the range to the register list.
+      while (Reg != EndReg) {
+        Reg = getNextRegister(Reg);
+        Registers.push_back(std::pair<unsigned, SMLoc>(Reg, RegLoc));
+      }
+      continue;
     }
-
-    int RegNum = TryParseRegister();
-    if (RegNum == -1) {
-      Error(RegLoc, "register expected");
-      return true;
-    }
-
-    if (IsRange) {
-      int Reg = PrevRegNum;
-      do {
-        ++Reg;
-        Registers.push_back(std::make_pair(Reg, RegLoc));
-      } while (Reg != RegNum);
-    } else {
-      Registers.push_back(std::make_pair(RegNum, RegLoc));
-    }
-
-    PrevRegNum = RegNum;
-  } while (Parser.getTok().is(AsmToken::Comma) ||
-           Parser.getTok().is(AsmToken::Minus));
-
-  // Process the right curly brace of the list.
-  const AsmToken &RCurlyTok = Parser.getTok();
-  if (RCurlyTok.isNot(AsmToken::RCurly)) {
-    Error(RCurlyTok.getLoc(), "'}' expected");
-    return true;
+    Parser.Lex(); // Eat the comma.
+    RegLoc = Parser.getTok().getLoc();
+    int OldReg = Reg;
+    Reg = tryParseRegister();
+    if (Reg == -1)
+      return Error(RegLoc, "register expected");
+    // The register must be in the same register class as the first.
+    if (!RC->contains(Reg))
+      return Error(RegLoc, "invalid register in register list");
+    // List must be monotonically increasing.
+    if (getARMRegisterNumbering(Reg) <= getARMRegisterNumbering(OldReg))
+      return Error(RegLoc, "register list not in ascending order");
+    // VFP register lists must also be contiguous.
+    // It's OK to use the enumeration values directly here rather, as the
+    // VFP register classes have the enum sorted properly.
+    if (RC != &ARMMCRegisterClasses[ARM::GPRRegClassID] &&
+        Reg != OldReg + 1)
+      return Error(RegLoc, "non-contiguous register range");
+    Registers.push_back(std::pair<unsigned, SMLoc>(Reg, RegLoc));
   }
 
-  SMLoc E = RCurlyTok.getLoc();
-  Parser.Lex(); // Eat right curly brace token.
-
-  // Verify the register list.
-  SmallVectorImpl<std::pair<unsigned, SMLoc> >::const_iterator
-    RI = Registers.begin(), RE = Registers.end();
-
-  unsigned HighRegNum = getARMRegisterNumbering(RI->first);
-  bool EmittedWarning = false;
-
-  DenseMap<unsigned, bool> RegMap;
-  RegMap[HighRegNum] = true;
-
-  for (++RI; RI != RE; ++RI) {
-    const std::pair<unsigned, SMLoc> &RegInfo = *RI;
-    unsigned Reg = getARMRegisterNumbering(RegInfo.first);
-
-    if (RegMap[Reg]) {
-      Error(RegInfo.second, "register duplicated in register list");
-      return true;
-    }
-
-    if (!EmittedWarning && Reg < HighRegNum)
-      Warning(RegInfo.second,
-              "register not in ascending order in register list");
-
-    RegMap[Reg] = true;
-    HighRegNum = std::max(Reg, HighRegNum);
-  }
+  SMLoc E = Parser.getTok().getLoc();
+  if (Parser.getTok().isNot(AsmToken::RCurly))
+    return Error(E, "'}' expected");
+  Parser.Lex(); // Eat '}' token.
 
   Operands.push_back(ARMOperand::CreateRegList(Registers, S, E));
   return false;
 }
 
-/// tryParseMemBarrierOptOperand - Try to parse DSB/DMB data barrier options.
+/// parseMemBarrierOptOperand - Try to parse DSB/DMB data barrier options.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseMemBarrierOptOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+parseMemBarrierOptOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
   assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
@@ -1360,28 +2294,32 @@ tryParseMemBarrierOptOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   return MatchOperand_Success;
 }
 
-/// tryParseProcIFlagsOperand - Try to parse iflags from CPS instruction.
+/// parseProcIFlagsOperand - Try to parse iflags from CPS instruction.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseProcIFlagsOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+parseProcIFlagsOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
   assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
   StringRef IFlagsStr = Tok.getString();
 
+  // An iflags string of "none" is interpreted to mean that none of the AIF
+  // bits are set.  Not a terribly useful instruction, but a valid encoding.
   unsigned IFlags = 0;
-  for (int i = 0, e = IFlagsStr.size(); i != e; ++i) {
-    unsigned Flag = StringSwitch<unsigned>(IFlagsStr.substr(i, 1))
-    .Case("a", ARM_PROC::A)
-    .Case("i", ARM_PROC::I)
-    .Case("f", ARM_PROC::F)
-    .Default(~0U);
+  if (IFlagsStr != "none") {
+        for (int i = 0, e = IFlagsStr.size(); i != e; ++i) {
+      unsigned Flag = StringSwitch<unsigned>(IFlagsStr.substr(i, 1))
+        .Case("a", ARM_PROC::A)
+        .Case("i", ARM_PROC::I)
+        .Case("f", ARM_PROC::F)
+        .Default(~0U);
 
-    // If some specific iflag is already set, it means that some letter is
-    // present more than once, this is not acceptable.
-    if (Flag == ~0U || (IFlags & Flag))
-      return MatchOperand_NoMatch;
+      // If some specific iflag is already set, it means that some letter is
+      // present more than once, this is not acceptable.
+      if (Flag == ~0U || (IFlags & Flag))
+        return MatchOperand_NoMatch;
 
-    IFlags |= Flag;
+      IFlags |= Flag;
+    }
   }
 
   Parser.Lex(); // Eat identifier token.
@@ -1389,18 +2327,49 @@ tryParseProcIFlagsOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   return MatchOperand_Success;
 }
 
-/// tryParseMSRMaskOperand - Try to parse mask flags from MSR instruction.
+/// parseMSRMaskOperand - Try to parse mask flags from MSR instruction.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseMSRMaskOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+parseMSRMaskOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
   assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
   StringRef Mask = Tok.getString();
 
+  if (isMClass()) {
+    // See ARMv6-M 10.1.1
+    unsigned FlagsVal = StringSwitch<unsigned>(Mask)
+      .Case("apsr", 0)
+      .Case("iapsr", 1)
+      .Case("eapsr", 2)
+      .Case("xpsr", 3)
+      .Case("ipsr", 5)
+      .Case("epsr", 6)
+      .Case("iepsr", 7)
+      .Case("msp", 8)
+      .Case("psp", 9)
+      .Case("primask", 16)
+      .Case("basepri", 17)
+      .Case("basepri_max", 18)
+      .Case("faultmask", 19)
+      .Case("control", 20)
+      .Default(~0U);
+    
+    if (FlagsVal == ~0U)
+      return MatchOperand_NoMatch;
+
+    if (!hasV7Ops() && FlagsVal >= 17 && FlagsVal <= 19)
+      // basepri, basepri_max and faultmask only valid for V7m.
+      return MatchOperand_NoMatch;
+    
+    Parser.Lex(); // Eat identifier token.
+    Operands.push_back(ARMOperand::CreateMSRMask(FlagsVal, S));
+    return MatchOperand_Success;
+  }
+
   // Split spec_reg from flag, example: CPSR_sxf => "CPSR" and "sxf"
   size_t Start = 0, Next = Mask.find('_');
   StringRef Flags = "";
-  StringRef SpecReg = Mask.slice(Start, Next);
+  std::string SpecReg = LowercaseString(Mask.slice(Start, Next));
   if (Next != StringRef::npos)
     Flags = Mask.slice(Next+1, Mask.size());
 
@@ -1411,7 +2380,7 @@ tryParseMSRMaskOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
 
   if (SpecReg == "apsr") {
     FlagsVal = StringSwitch<unsigned>(Flags)
-    .Case("nzcvq",  0x8) // same as CPSR_c
+    .Case("nzcvq",  0x8) // same as CPSR_f
     .Case("g",      0x4) // same as CPSR_s
     .Case("nzcvqg", 0xc) // same as CPSR_fs
     .Default(~0U);
@@ -1420,7 +2389,7 @@ tryParseMSRMaskOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
       if (!Flags.empty())
         return MatchOperand_NoMatch;
       else
-        FlagsVal = 0; // No flag
+        FlagsVal = 8; // No flag
     }
   } else if (SpecReg == "cpsr" || SpecReg == "spsr") {
     if (Flags == "all") // cpsr_all is an alias for cpsr_fc
@@ -1455,96 +2424,680 @@ tryParseMSRMaskOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   return MatchOperand_Success;
 }
 
-/// tryParseMemMode2Operand - Try to parse memory addressing mode 2 operand.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseMemMode2Operand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
-  assert(Parser.getTok().is(AsmToken::LBrac) && "Token is not a \"[\"");
+parsePKHImm(SmallVectorImpl<MCParsedAsmOperand*> &Operands, StringRef Op,
+            int Low, int High) {
+  const AsmToken &Tok = Parser.getTok();
+  if (Tok.isNot(AsmToken::Identifier)) {
+    Error(Parser.getTok().getLoc(), Op + " operand expected.");
+    return MatchOperand_ParseFail;
+  }
+  StringRef ShiftName = Tok.getString();
+  std::string LowerOp = LowercaseString(Op);
+  std::string UpperOp = UppercaseString(Op);
+  if (ShiftName != LowerOp && ShiftName != UpperOp) {
+    Error(Parser.getTok().getLoc(), Op + " operand expected.");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat shift type token.
 
-  if (ParseMemory(Operands, ARMII::AddrMode2))
-    return MatchOperand_NoMatch;
+  // There must be a '#' and a shift amount.
+  if (Parser.getTok().isNot(AsmToken::Hash)) {
+    Error(Parser.getTok().getLoc(), "'#' expected");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat hash token.
+
+  const MCExpr *ShiftAmount;
+  SMLoc Loc = Parser.getTok().getLoc();
+  if (getParser().ParseExpression(ShiftAmount)) {
+    Error(Loc, "illegal expression");
+    return MatchOperand_ParseFail;
+  }
+  const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(ShiftAmount);
+  if (!CE) {
+    Error(Loc, "constant expression expected");
+    return MatchOperand_ParseFail;
+  }
+  int Val = CE->getValue();
+  if (Val < Low || Val > High) {
+    Error(Loc, "immediate value out of range");
+    return MatchOperand_ParseFail;
+  }
+
+  Operands.push_back(ARMOperand::CreateImm(CE, Loc, Parser.getTok().getLoc()));
 
   return MatchOperand_Success;
 }
 
-/// tryParseMemMode3Operand - Try to parse memory addressing mode 3 operand.
 ARMAsmParser::OperandMatchResultTy ARMAsmParser::
-tryParseMemMode3Operand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
-  assert(Parser.getTok().is(AsmToken::LBrac) && "Token is not a \"[\"");
+parseSetEndImm(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  const AsmToken &Tok = Parser.getTok();
+  SMLoc S = Tok.getLoc();
+  if (Tok.isNot(AsmToken::Identifier)) {
+    Error(Tok.getLoc(), "'be' or 'le' operand expected");
+    return MatchOperand_ParseFail;
+  }
+  int Val = StringSwitch<int>(Tok.getString())
+    .Case("be", 1)
+    .Case("le", 0)
+    .Default(-1);
+  Parser.Lex(); // Eat the token.
 
-  if (ParseMemory(Operands, ARMII::AddrMode3))
-    return MatchOperand_NoMatch;
+  if (Val == -1) {
+    Error(Tok.getLoc(), "'be' or 'le' operand expected");
+    return MatchOperand_ParseFail;
+  }
+  Operands.push_back(ARMOperand::CreateImm(MCConstantExpr::Create(Val,
+                                                                  getContext()),
+                                           S, Parser.getTok().getLoc()));
+  return MatchOperand_Success;
+}
+
+/// parseShifterImm - Parse the shifter immediate operand for SSAT/USAT
+/// instructions. Legal values are:
+///     lsl #n  'n' in [0,31]
+///     asr #n  'n' in [1,32]
+///             n == 32 encoded as n == 0.
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseShifterImm(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  const AsmToken &Tok = Parser.getTok();
+  SMLoc S = Tok.getLoc();
+  if (Tok.isNot(AsmToken::Identifier)) {
+    Error(S, "shift operator 'asr' or 'lsl' expected");
+    return MatchOperand_ParseFail;
+  }
+  StringRef ShiftName = Tok.getString();
+  bool isASR;
+  if (ShiftName == "lsl" || ShiftName == "LSL")
+    isASR = false;
+  else if (ShiftName == "asr" || ShiftName == "ASR")
+    isASR = true;
+  else {
+    Error(S, "shift operator 'asr' or 'lsl' expected");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat the operator.
+
+  // A '#' and a shift amount.
+  if (Parser.getTok().isNot(AsmToken::Hash)) {
+    Error(Parser.getTok().getLoc(), "'#' expected");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat hash token.
+
+  const MCExpr *ShiftAmount;
+  SMLoc E = Parser.getTok().getLoc();
+  if (getParser().ParseExpression(ShiftAmount)) {
+    Error(E, "malformed shift expression");
+    return MatchOperand_ParseFail;
+  }
+  const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(ShiftAmount);
+  if (!CE) {
+    Error(E, "shift amount must be an immediate");
+    return MatchOperand_ParseFail;
+  }
+
+  int64_t Val = CE->getValue();
+  if (isASR) {
+    // Shift amount must be in [1,32]
+    if (Val < 1 || Val > 32) {
+      Error(E, "'asr' shift amount must be in range [1,32]");
+      return MatchOperand_ParseFail;
+    }
+    // asr #32 encoded as asr #0, but is not allowed in Thumb2 mode.
+    if (isThumb() && Val == 32) {
+      Error(E, "'asr #32' shift amount not allowed in Thumb mode");
+      return MatchOperand_ParseFail;
+    }
+    if (Val == 32) Val = 0;
+  } else {
+    // Shift amount must be in [1,32]
+    if (Val < 0 || Val > 31) {
+      Error(E, "'lsr' shift amount must be in range [0,31]");
+      return MatchOperand_ParseFail;
+    }
+  }
+
+  E = Parser.getTok().getLoc();
+  Operands.push_back(ARMOperand::CreateShifterImm(isASR, Val, S, E));
 
   return MatchOperand_Success;
 }
 
-/// CvtLdWriteBackRegAddrMode2 - Convert parsed operands to MCInst.
+/// parseRotImm - Parse the shifter immediate operand for SXTB/UXTB family
+/// of instructions. Legal values are:
+///     ror #n  'n' in {0, 8, 16, 24}
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseRotImm(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  const AsmToken &Tok = Parser.getTok();
+  SMLoc S = Tok.getLoc();
+  if (Tok.isNot(AsmToken::Identifier))
+    return MatchOperand_NoMatch;
+  StringRef ShiftName = Tok.getString();
+  if (ShiftName != "ror" && ShiftName != "ROR")
+    return MatchOperand_NoMatch;
+  Parser.Lex(); // Eat the operator.
+
+  // A '#' and a rotate amount.
+  if (Parser.getTok().isNot(AsmToken::Hash)) {
+    Error(Parser.getTok().getLoc(), "'#' expected");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat hash token.
+
+  const MCExpr *ShiftAmount;
+  SMLoc E = Parser.getTok().getLoc();
+  if (getParser().ParseExpression(ShiftAmount)) {
+    Error(E, "malformed rotate expression");
+    return MatchOperand_ParseFail;
+  }
+  const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(ShiftAmount);
+  if (!CE) {
+    Error(E, "rotate amount must be an immediate");
+    return MatchOperand_ParseFail;
+  }
+
+  int64_t Val = CE->getValue();
+  // Shift amount must be in {0, 8, 16, 24} (0 is undocumented extension)
+  // normally, zero is represented in asm by omitting the rotate operand
+  // entirely.
+  if (Val != 8 && Val != 16 && Val != 24 && Val != 0) {
+    Error(E, "'ror' rotate amount must be 8, 16, or 24");
+    return MatchOperand_ParseFail;
+  }
+
+  E = Parser.getTok().getLoc();
+  Operands.push_back(ARMOperand::CreateRotImm(Val, S, E));
+
+  return MatchOperand_Success;
+}
+
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseBitfield(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+  // The bitfield descriptor is really two operands, the LSB and the width.
+  if (Parser.getTok().isNot(AsmToken::Hash)) {
+    Error(Parser.getTok().getLoc(), "'#' expected");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat hash token.
+
+  const MCExpr *LSBExpr;
+  SMLoc E = Parser.getTok().getLoc();
+  if (getParser().ParseExpression(LSBExpr)) {
+    Error(E, "malformed immediate expression");
+    return MatchOperand_ParseFail;
+  }
+  const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(LSBExpr);
+  if (!CE) {
+    Error(E, "'lsb' operand must be an immediate");
+    return MatchOperand_ParseFail;
+  }
+
+  int64_t LSB = CE->getValue();
+  // The LSB must be in the range [0,31]
+  if (LSB < 0 || LSB > 31) {
+    Error(E, "'lsb' operand must be in the range [0,31]");
+    return MatchOperand_ParseFail;
+  }
+  E = Parser.getTok().getLoc();
+
+  // Expect another immediate operand.
+  if (Parser.getTok().isNot(AsmToken::Comma)) {
+    Error(Parser.getTok().getLoc(), "too few operands");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat hash token.
+  if (Parser.getTok().isNot(AsmToken::Hash)) {
+    Error(Parser.getTok().getLoc(), "'#' expected");
+    return MatchOperand_ParseFail;
+  }
+  Parser.Lex(); // Eat hash token.
+
+  const MCExpr *WidthExpr;
+  if (getParser().ParseExpression(WidthExpr)) {
+    Error(E, "malformed immediate expression");
+    return MatchOperand_ParseFail;
+  }
+  CE = dyn_cast<MCConstantExpr>(WidthExpr);
+  if (!CE) {
+    Error(E, "'width' operand must be an immediate");
+    return MatchOperand_ParseFail;
+  }
+
+  int64_t Width = CE->getValue();
+  // The LSB must be in the range [1,32-lsb]
+  if (Width < 1 || Width > 32 - LSB) {
+    Error(E, "'width' operand must be in the range [1,32-lsb]");
+    return MatchOperand_ParseFail;
+  }
+  E = Parser.getTok().getLoc();
+
+  Operands.push_back(ARMOperand::CreateBitfield(LSB, Width, S, E));
+
+  return MatchOperand_Success;
+}
+
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parsePostIdxReg(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Check for a post-index addressing register operand. Specifically:
+  // postidx_reg := '+' register {, shift}
+  //              | '-' register {, shift}
+  //              | register {, shift}
+
+  // This method must return MatchOperand_NoMatch without consuming any tokens
+  // in the case where there is no match, as other alternatives take other
+  // parse methods.
+  AsmToken Tok = Parser.getTok();
+  SMLoc S = Tok.getLoc();
+  bool haveEaten = false;
+  bool isAdd = true;
+  int Reg = -1;
+  if (Tok.is(AsmToken::Plus)) {
+    Parser.Lex(); // Eat the '+' token.
+    haveEaten = true;
+  } else if (Tok.is(AsmToken::Minus)) {
+    Parser.Lex(); // Eat the '-' token.
+    isAdd = false;
+    haveEaten = true;
+  }
+  if (Parser.getTok().is(AsmToken::Identifier))
+    Reg = tryParseRegister();
+  if (Reg == -1) {
+    if (!haveEaten)
+      return MatchOperand_NoMatch;
+    Error(Parser.getTok().getLoc(), "register expected");
+    return MatchOperand_ParseFail;
+  }
+  SMLoc E = Parser.getTok().getLoc();
+
+  ARM_AM::ShiftOpc ShiftTy = ARM_AM::no_shift;
+  unsigned ShiftImm = 0;
+  if (Parser.getTok().is(AsmToken::Comma)) {
+    Parser.Lex(); // Eat the ','.
+    if (parseMemRegOffsetShift(ShiftTy, ShiftImm))
+      return MatchOperand_ParseFail;
+  }
+
+  Operands.push_back(ARMOperand::CreatePostIdxReg(Reg, isAdd, ShiftTy,
+                                                  ShiftImm, S, E));
+
+  return MatchOperand_Success;
+}
+
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseAM3Offset(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Check for a post-index addressing register operand. Specifically:
+  // am3offset := '+' register
+  //              | '-' register
+  //              | register
+  //              | # imm
+  //              | # + imm
+  //              | # - imm
+
+  // This method must return MatchOperand_NoMatch without consuming any tokens
+  // in the case where there is no match, as other alternatives take other
+  // parse methods.
+  AsmToken Tok = Parser.getTok();
+  SMLoc S = Tok.getLoc();
+
+  // Do immediates first, as we always parse those if we have a '#'.
+  if (Parser.getTok().is(AsmToken::Hash)) {
+    Parser.Lex(); // Eat the '#'.
+    // Explicitly look for a '-', as we need to encode negative zero
+    // differently.
+    bool isNegative = Parser.getTok().is(AsmToken::Minus);
+    const MCExpr *Offset;
+    if (getParser().ParseExpression(Offset))
+      return MatchOperand_ParseFail;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Offset);
+    if (!CE) {
+      Error(S, "constant expression expected");
+      return MatchOperand_ParseFail;
+    }
+    SMLoc E = Tok.getLoc();
+    // Negative zero is encoded as the flag value INT32_MIN.
+    int32_t Val = CE->getValue();
+    if (isNegative && Val == 0)
+      Val = INT32_MIN;
+
+    Operands.push_back(
+      ARMOperand::CreateImm(MCConstantExpr::Create(Val, getContext()), S, E));
+
+    return MatchOperand_Success;
+  }
+
+
+  bool haveEaten = false;
+  bool isAdd = true;
+  int Reg = -1;
+  if (Tok.is(AsmToken::Plus)) {
+    Parser.Lex(); // Eat the '+' token.
+    haveEaten = true;
+  } else if (Tok.is(AsmToken::Minus)) {
+    Parser.Lex(); // Eat the '-' token.
+    isAdd = false;
+    haveEaten = true;
+  }
+  if (Parser.getTok().is(AsmToken::Identifier))
+    Reg = tryParseRegister();
+  if (Reg == -1) {
+    if (!haveEaten)
+      return MatchOperand_NoMatch;
+    Error(Parser.getTok().getLoc(), "register expected");
+    return MatchOperand_ParseFail;
+  }
+  SMLoc E = Parser.getTok().getLoc();
+
+  Operands.push_back(ARMOperand::CreatePostIdxReg(Reg, isAdd, ARM_AM::no_shift,
+                                                  0, S, E));
+
+  return MatchOperand_Success;
+}
+
+/// cvtT2LdrdPre - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
 bool ARMAsmParser::
-CvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
-                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+cvtT2LdrdPre(MCInst &Inst, unsigned Opcode,
+             const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Rt, Rt2
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
-
+  ((ARMOperand*)Operands[3])->addRegOperands(Inst, 1);
   // Create a writeback register dummy placeholder.
-  Inst.addOperand(MCOperand::CreateImm(0));
-
-  ((ARMOperand*)Operands[3])->addMemMode2Operands(Inst, 3);
+  Inst.addOperand(MCOperand::CreateReg(0));
+  // addr
+  ((ARMOperand*)Operands[4])->addMemImm8s4OffsetOperands(Inst, 2);
+  // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
   return true;
 }
 
-/// CvtStWriteBackRegAddrMode2 - Convert parsed operands to MCInst.
+/// cvtT2StrdPre - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
 bool ARMAsmParser::
-CvtStWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
-                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+cvtT2StrdPre(MCInst &Inst, unsigned Opcode,
+             const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
-  Inst.addOperand(MCOperand::CreateImm(0));
+  Inst.addOperand(MCOperand::CreateReg(0));
+  // Rt, Rt2
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
-  ((ARMOperand*)Operands[3])->addMemMode2Operands(Inst, 3);
+  ((ARMOperand*)Operands[3])->addRegOperands(Inst, 1);
+  // addr
+  ((ARMOperand*)Operands[4])->addMemImm8s4OffsetOperands(Inst, 2);
+  // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
   return true;
 }
 
-/// CvtLdWriteBackRegAddrMode3 - Convert parsed operands to MCInst.
+/// cvtLdWriteBackRegT2AddrModeImm8 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
 bool ARMAsmParser::
-CvtLdWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
 
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
 
-  ((ARMOperand*)Operands[3])->addMemMode3Operands(Inst, 3);
+  ((ARMOperand*)Operands[3])->addMemImm8OffsetOperands(Inst, 2);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
   return true;
 }
 
-/// CvtStWriteBackRegAddrMode3 - Convert parsed operands to MCInst.
+/// cvtStWriteBackRegT2AddrModeImm8 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
 bool ARMAsmParser::
-CvtStWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+cvtStWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
-  ((ARMOperand*)Operands[3])->addMemMode3Operands(Inst, 3);
+  ((ARMOperand*)Operands[3])->addMemImm8OffsetOperands(Inst, 2);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
   return true;
 }
 
+/// cvtLdWriteBackRegAddrMode2 - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+
+  ((ARMOperand*)Operands[3])->addAddrMode2Operands(Inst, 3);
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtLdWriteBackRegAddrModeImm12 - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtLdWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+
+  ((ARMOperand*)Operands[3])->addMemImm12OffsetOperands(Inst, 2);
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+
+/// cvtStWriteBackRegAddrModeImm12 - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtStWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  ((ARMOperand*)Operands[3])->addMemImm12OffsetOperands(Inst, 2);
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtStWriteBackRegAddrMode2 - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtStWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  ((ARMOperand*)Operands[3])->addAddrMode2Operands(Inst, 3);
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtStWriteBackRegAddrMode3 - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtStWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  ((ARMOperand*)Operands[3])->addAddrMode3Operands(Inst, 3);
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtLdExtTWriteBackImm - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtLdExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+                      const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Rt
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  // addr
+  ((ARMOperand*)Operands[3])->addMemNoOffsetOperands(Inst, 1);
+  // offset
+  ((ARMOperand*)Operands[4])->addPostIdxImm8Operands(Inst, 1);
+  // pred
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtLdExtTWriteBackReg - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtLdExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+                      const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Rt
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  // addr
+  ((ARMOperand*)Operands[3])->addMemNoOffsetOperands(Inst, 1);
+  // offset
+  ((ARMOperand*)Operands[4])->addPostIdxRegOperands(Inst, 2);
+  // pred
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtStExtTWriteBackImm - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtStExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+                      const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  // Rt
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  // addr
+  ((ARMOperand*)Operands[3])->addMemNoOffsetOperands(Inst, 1);
+  // offset
+  ((ARMOperand*)Operands[4])->addPostIdxImm8Operands(Inst, 1);
+  // pred
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtStExtTWriteBackReg - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtStExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+                      const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  // Rt
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  // addr
+  ((ARMOperand*)Operands[3])->addMemNoOffsetOperands(Inst, 1);
+  // offset
+  ((ARMOperand*)Operands[4])->addPostIdxRegOperands(Inst, 2);
+  // pred
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtLdrdPre - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtLdrdPre(MCInst &Inst, unsigned Opcode,
+           const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Rt, Rt2
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  ((ARMOperand*)Operands[3])->addRegOperands(Inst, 1);
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  // addr
+  ((ARMOperand*)Operands[4])->addAddrMode3Operands(Inst, 3);
+  // pred
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtStrdPre - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtStrdPre(MCInst &Inst, unsigned Opcode,
+           const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  // Rt, Rt2
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  ((ARMOperand*)Operands[3])->addRegOperands(Inst, 1);
+  // addr
+  ((ARMOperand*)Operands[4])->addAddrMode3Operands(Inst, 3);
+  // pred
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtLdWriteBackRegAddrMode3 - Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtLdWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+                         const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
+  // Create a writeback register dummy placeholder.
+  Inst.addOperand(MCOperand::CreateImm(0));
+  ((ARMOperand*)Operands[3])->addAddrMode3Operands(Inst, 3);
+  ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
+  return true;
+}
+
+/// cvtThumbMultiple- Convert parsed operands to MCInst.
+/// Needed here because the Asm Gen Matcher can't handle properly tied operands
+/// when they refer multiple MIOperands inside a single one.
+bool ARMAsmParser::
+cvtThumbMultiply(MCInst &Inst, unsigned Opcode,
+           const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // The second source operand must be the same register as the destination
+  // operand.
+  if (Operands.size() == 6 &&
+      (((ARMOperand*)Operands[3])->getReg() !=
+       ((ARMOperand*)Operands[5])->getReg()) &&
+      (((ARMOperand*)Operands[3])->getReg() !=
+       ((ARMOperand*)Operands[4])->getReg())) {
+    Error(Operands[3]->getStartLoc(),
+          "destination register must match source register");
+    return false;
+  }
+  ((ARMOperand*)Operands[3])->addRegOperands(Inst, 1);
+  ((ARMOperand*)Operands[1])->addCCOutOperands(Inst, 1);
+  ((ARMOperand*)Operands[4])->addRegOperands(Inst, 1);
+  // If we have a three-operand form, use that, else the second source operand
+  // is just the destination operand again.
+  if (Operands.size() == 6)
+    ((ARMOperand*)Operands[5])->addRegOperands(Inst, 1);
+  else
+    Inst.addOperand(Inst.getOperand(0));
+  ((ARMOperand*)Operands[2])->addCondCodeOperands(Inst, 2);
+
+  return true;
+}
+
 /// Parse an ARM memory expression, return false if successful else return true
 /// or an error.  The first token must be a '[' when called.
-///
-/// TODO Only preindexing and postindexing addressing are started, unindexed
-/// with option, etc are still to do.
 bool ARMAsmParser::
-ParseMemory(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-            ARMII::AddrMode AddrMode = ARMII::AddrModeNone) {
+parseMemory(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S, E;
   assert(Parser.getTok().is(AsmToken::LBrac) &&
          "Token is not a Left Bracket");
@@ -1552,185 +3105,178 @@ ParseMemory(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
   Parser.Lex(); // Eat left bracket token.
 
   const AsmToken &BaseRegTok = Parser.getTok();
-  if (BaseRegTok.isNot(AsmToken::Identifier)) {
-    Error(BaseRegTok.getLoc(), "register expected");
-    return true;
-  }
-  int BaseRegNum = TryParseRegister();
-  if (BaseRegNum == -1) {
-    Error(BaseRegTok.getLoc(), "register expected");
-    return true;
-  }
+  int BaseRegNum = tryParseRegister();
+  if (BaseRegNum == -1)
+    return Error(BaseRegTok.getLoc(), "register expected");
 
   // The next token must either be a comma or a closing bracket.
   const AsmToken &Tok = Parser.getTok();
   if (!Tok.is(AsmToken::Comma) && !Tok.is(AsmToken::RBrac))
-    return true;
+    return Error(Tok.getLoc(), "malformed memory operand");
 
-  bool Preindexed = false;
-  bool Postindexed = false;
-  bool OffsetIsReg = false;
-  bool Negative = false;
-  bool Writeback = false;
-  ARMOperand *WBOp = 0;
-  int OffsetRegNum = -1;
-  bool OffsetRegShifted = false;
-  enum ARM_AM::ShiftOpc ShiftType = ARM_AM::lsl;
-  const MCExpr *ShiftAmount = 0;
-  const MCExpr *Offset = 0;
-
-  // First look for preindexed address forms, that is after the "[Rn" we now
-  // have to see if the next token is a comma.
-  if (Tok.is(AsmToken::Comma)) {
-    Preindexed = true;
-    Parser.Lex(); // Eat comma token.
-
-    if (ParseMemoryOffsetReg(Negative, OffsetRegShifted, ShiftType, ShiftAmount,
-                             Offset, OffsetIsReg, OffsetRegNum, E))
-      return true;
-    const AsmToken &RBracTok = Parser.getTok();
-    if (RBracTok.isNot(AsmToken::RBrac)) {
-      Error(RBracTok.getLoc(), "']' expected");
-      return true;
-    }
-    E = RBracTok.getLoc();
-    Parser.Lex(); // Eat right bracket token.
-
-    const AsmToken &ExclaimTok = Parser.getTok();
-    if (ExclaimTok.is(AsmToken::Exclaim)) {
-      // None of addrmode3 instruction uses "!"
-      if (AddrMode == ARMII::AddrMode3)
-        return true;
-
-      WBOp = ARMOperand::CreateToken(ExclaimTok.getString(),
-                                     ExclaimTok.getLoc());
-      Writeback = true;
-      Parser.Lex(); // Eat exclaim token
-    } else { // In addressing mode 2, pre-indexed mode always end with "!"
-      if (AddrMode == ARMII::AddrMode2)
-        Preindexed = false;
-    }
-  } else {
-    // The "[Rn" we have so far was not followed by a comma.
-
-    // If there's anything other than the right brace, this is a post indexing
-    // addressing form.
+  if (Tok.is(AsmToken::RBrac)) {
     E = Tok.getLoc();
     Parser.Lex(); // Eat right bracket token.
 
-    const AsmToken &NextTok = Parser.getTok();
+    Operands.push_back(ARMOperand::CreateMem(BaseRegNum, 0, 0, ARM_AM::no_shift,
+                                             0, 0, false, S, E));
 
-    if (NextTok.isNot(AsmToken::EndOfStatement)) {
-      Postindexed = true;
-      Writeback = true;
-
-      if (NextTok.isNot(AsmToken::Comma)) {
-        Error(NextTok.getLoc(), "',' expected");
-        return true;
-      }
-
-      Parser.Lex(); // Eat comma token.
-
-      if (ParseMemoryOffsetReg(Negative, OffsetRegShifted, ShiftType,
-                               ShiftAmount, Offset, OffsetIsReg, OffsetRegNum,
-                               E))
-        return true;
+    // If there's a pre-indexing writeback marker, '!', just add it as a token
+    // operand. It's rather odd, but syntactically valid.
+    if (Parser.getTok().is(AsmToken::Exclaim)) {
+      Operands.push_back(ARMOperand::CreateToken("!",Parser.getTok().getLoc()));
+      Parser.Lex(); // Eat the '!'.
     }
+
+    return false;
   }
 
-  // Force Offset to exist if used.
-  if (!OffsetIsReg) {
-    if (!Offset)
-      Offset = MCConstantExpr::Create(0, getContext());
-  } else {
-    if (AddrMode == ARMII::AddrMode3 && OffsetRegShifted) {
-      Error(E, "shift amount not supported");
-      return true;
+  assert(Tok.is(AsmToken::Comma) && "Lost comma in memory operand?!");
+  Parser.Lex(); // Eat the comma.
+
+  // If we have a ':', it's an alignment specifier.
+  if (Parser.getTok().is(AsmToken::Colon)) {
+    Parser.Lex(); // Eat the ':'.
+    E = Parser.getTok().getLoc();
+
+    const MCExpr *Expr;
+    if (getParser().ParseExpression(Expr))
+     return true;
+
+    // The expression has to be a constant. Memory references with relocations
+    // don't come through here, as they use the <label> forms of the relevant
+    // instructions.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr);
+    if (!CE)
+      return Error (E, "constant expression expected");
+
+    unsigned Align = 0;
+    switch (CE->getValue()) {
+    default:
+      return Error(E, "alignment specifier must be 64, 128, or 256 bits");
+    case 64:  Align = 8; break;
+    case 128: Align = 16; break;
+    case 256: Align = 32; break;
     }
-  }
 
-  Operands.push_back(ARMOperand::CreateMem(AddrMode, BaseRegNum, OffsetIsReg,
-                                     Offset, OffsetRegNum, OffsetRegShifted,
-                                     ShiftType, ShiftAmount, Preindexed,
-                                     Postindexed, Negative, Writeback, S, E));
-  if (WBOp)
-    Operands.push_back(WBOp);
+    // Now we should have the closing ']'
+    E = Parser.getTok().getLoc();
+    if (Parser.getTok().isNot(AsmToken::RBrac))
+      return Error(E, "']' expected");
+    Parser.Lex(); // Eat right bracket token.
 
-  return false;
-}
+    // Don't worry about range checking the value here. That's handled by
+    // the is*() predicates.
+    Operands.push_back(ARMOperand::CreateMem(BaseRegNum, 0, 0,
+                                             ARM_AM::no_shift, 0, Align,
+                                             false, S, E));
 
-/// Parse the offset of a memory operand after we have seen "[Rn," or "[Rn],"
-/// we will parse the following (were +/- means that a plus or minus is
-/// optional):
-///   +/-Rm
-///   +/-Rm, shift
-///   #offset
-/// we return false on success or an error otherwise.
-bool ARMAsmParser::ParseMemoryOffsetReg(bool &Negative,
-                                        bool &OffsetRegShifted,
-                                        enum ARM_AM::ShiftOpc &ShiftType,
-                                        const MCExpr *&ShiftAmount,
-                                        const MCExpr *&Offset,
-                                        bool &OffsetIsReg,
-                                        int &OffsetRegNum,
-                                        SMLoc &E) {
-  Negative = false;
-  OffsetRegShifted = false;
-  OffsetIsReg = false;
-  OffsetRegNum = -1;
-  const AsmToken &NextTok = Parser.getTok();
-  E = NextTok.getLoc();
-  if (NextTok.is(AsmToken::Plus))
-    Parser.Lex(); // Eat plus token.
-  else if (NextTok.is(AsmToken::Minus)) {
-    Negative = true;
-    Parser.Lex(); // Eat minus token
-  }
-  // See if there is a register following the "[Rn," or "[Rn]," we have so far.
-  const AsmToken &OffsetRegTok = Parser.getTok();
-  if (OffsetRegTok.is(AsmToken::Identifier)) {
-    SMLoc CurLoc = OffsetRegTok.getLoc();
-    OffsetRegNum = TryParseRegister();
-    if (OffsetRegNum != -1) {
-      OffsetIsReg = true;
-      E = CurLoc;
+    // If there's a pre-indexing writeback marker, '!', just add it as a token
+    // operand.
+    if (Parser.getTok().is(AsmToken::Exclaim)) {
+      Operands.push_back(ARMOperand::CreateToken("!",Parser.getTok().getLoc()));
+      Parser.Lex(); // Eat the '!'.
     }
+
+    return false;
   }
 
-  // If we parsed a register as the offset then there can be a shift after that.
-  if (OffsetRegNum != -1) {
-    // Look for a comma then a shift
-    const AsmToken &Tok = Parser.getTok();
-    if (Tok.is(AsmToken::Comma)) {
-      Parser.Lex(); // Eat comma token.
-
-      const AsmToken &Tok = Parser.getTok();
-      if (ParseShift(ShiftType, ShiftAmount, E))
-        return Error(Tok.getLoc(), "shift expected");
-      OffsetRegShifted = true;
-    }
-  }
-  else { // the "[Rn," or "[Rn,]" we have so far was not followed by "Rm"
-    // Look for #offset following the "[Rn," or "[Rn],"
-    const AsmToken &HashTok = Parser.getTok();
-    if (HashTok.isNot(AsmToken::Hash))
-      return Error(HashTok.getLoc(), "'#' expected");
-
-    Parser.Lex(); // Eat hash token.
+  // If we have a '#', it's an immediate offset, else assume it's a register
+  // offset.
+  if (Parser.getTok().is(AsmToken::Hash)) {
+    Parser.Lex(); // Eat the '#'.
+    E = Parser.getTok().getLoc();
 
+    bool isNegative = getParser().getTok().is(AsmToken::Minus);
+    const MCExpr *Offset;
     if (getParser().ParseExpression(Offset))
      return true;
-    E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+
+    // The expression has to be a constant. Memory references with relocations
+    // don't come through here, as they use the <label> forms of the relevant
+    // instructions.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Offset);
+    if (!CE)
+      return Error (E, "constant expression expected");
+
+    // If the constant was #-0, represent it as INT32_MIN.
+    int32_t Val = CE->getValue();
+    if (isNegative && Val == 0)
+      CE = MCConstantExpr::Create(INT32_MIN, getContext());
+
+    // Now we should have the closing ']'
+    E = Parser.getTok().getLoc();
+    if (Parser.getTok().isNot(AsmToken::RBrac))
+      return Error(E, "']' expected");
+    Parser.Lex(); // Eat right bracket token.
+
+    // Don't worry about range checking the value here. That's handled by
+    // the is*() predicates.
+    Operands.push_back(ARMOperand::CreateMem(BaseRegNum, CE, 0,
+                                             ARM_AM::no_shift, 0, 0,
+                                             false, S, E));
+
+    // If there's a pre-indexing writeback marker, '!', just add it as a token
+    // operand.
+    if (Parser.getTok().is(AsmToken::Exclaim)) {
+      Operands.push_back(ARMOperand::CreateToken("!",Parser.getTok().getLoc()));
+      Parser.Lex(); // Eat the '!'.
+    }
+
+    return false;
   }
+
+  // The register offset is optionally preceded by a '+' or '-'
+  bool isNegative = false;
+  if (Parser.getTok().is(AsmToken::Minus)) {
+    isNegative = true;
+    Parser.Lex(); // Eat the '-'.
+  } else if (Parser.getTok().is(AsmToken::Plus)) {
+    // Nothing to do.
+    Parser.Lex(); // Eat the '+'.
+  }
+
+  E = Parser.getTok().getLoc();
+  int OffsetRegNum = tryParseRegister();
+  if (OffsetRegNum == -1)
+    return Error(E, "register expected");
+
+  // If there's a shift operator, handle it.
+  ARM_AM::ShiftOpc ShiftType = ARM_AM::no_shift;
+  unsigned ShiftImm = 0;
+  if (Parser.getTok().is(AsmToken::Comma)) {
+    Parser.Lex(); // Eat the ','.
+    if (parseMemRegOffsetShift(ShiftType, ShiftImm))
+      return true;
+  }
+
+  // Now we should have the closing ']'
+  E = Parser.getTok().getLoc();
+  if (Parser.getTok().isNot(AsmToken::RBrac))
+    return Error(E, "']' expected");
+  Parser.Lex(); // Eat right bracket token.
+
+  Operands.push_back(ARMOperand::CreateMem(BaseRegNum, 0, OffsetRegNum,
+                                           ShiftType, ShiftImm, 0, isNegative,
+                                           S, E));
+
+  // If there's a pre-indexing writeback marker, '!', just add it as a token
+  // operand.
+  if (Parser.getTok().is(AsmToken::Exclaim)) {
+    Operands.push_back(ARMOperand::CreateToken("!",Parser.getTok().getLoc()));
+    Parser.Lex(); // Eat the '!'.
+  }
+
   return false;
 }
 
-/// ParseShift as one of these two:
+/// parseMemRegOffsetShift - one of these two:
 ///   ( lsl | lsr | asr | ror ) , # shift_amount
 ///   rrx
-/// and returns true if it parses a shift otherwise it returns false.
-bool ARMAsmParser::ParseShift(ARM_AM::ShiftOpc &St,
-                              const MCExpr *&ShiftAmount, SMLoc &E) {
+/// return true if it parses a shift otherwise it returns false.
+bool ARMAsmParser::parseMemRegOffsetShift(ARM_AM::ShiftOpc &St,
+                                          unsigned &Amount) {
+  SMLoc Loc = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
   if (Tok.isNot(AsmToken::Identifier))
     return true;
@@ -1746,28 +3292,86 @@ bool ARMAsmParser::ParseShift(ARM_AM::ShiftOpc &St,
   else if (ShiftName == "rrx" || ShiftName == "RRX")
     St = ARM_AM::rrx;
   else
-    return true;
+    return Error(Loc, "illegal shift operator");
   Parser.Lex(); // Eat shift type token.
 
-  // Rrx stands alone.
-  if (St == ARM_AM::rrx)
-    return false;
+  // rrx stands alone.
+  Amount = 0;
+  if (St != ARM_AM::rrx) {
+    Loc = Parser.getTok().getLoc();
+    // A '#' and a shift amount.
+    const AsmToken &HashTok = Parser.getTok();
+    if (HashTok.isNot(AsmToken::Hash))
+      return Error(HashTok.getLoc(), "'#' expected");
+    Parser.Lex(); // Eat hash token.
 
-  // Otherwise, there must be a '#' and a shift amount.
-  const AsmToken &HashTok = Parser.getTok();
-  if (HashTok.isNot(AsmToken::Hash))
-    return Error(HashTok.getLoc(), "'#' expected");
-  Parser.Lex(); // Eat hash token.
-
-  if (getParser().ParseExpression(ShiftAmount))
-    return true;
+    const MCExpr *Expr;
+    if (getParser().ParseExpression(Expr))
+      return true;
+    // Range check the immediate.
+    // lsl, ror: 0 <= imm <= 31
+    // lsr, asr: 0 <= imm <= 32
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr);
+    if (!CE)
+      return Error(Loc, "shift amount must be an immediate");
+    int64_t Imm = CE->getValue();
+    if (Imm < 0 ||
+        ((St == ARM_AM::lsl || St == ARM_AM::ror) && Imm > 31) ||
+        ((St == ARM_AM::lsr || St == ARM_AM::asr) && Imm > 32))
+      return Error(Loc, "immediate shift value out of range");
+    Amount = Imm;
+  }
 
   return false;
 }
 
+/// parseFPImm - A floating point immediate expression operand.
+ARMAsmParser::OperandMatchResultTy ARMAsmParser::
+parseFPImm(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+
+  if (Parser.getTok().isNot(AsmToken::Hash))
+    return MatchOperand_NoMatch;
+  Parser.Lex(); // Eat the '#'.
+
+  // Handle negation, as that still comes through as a separate token.
+  bool isNegative = false;
+  if (Parser.getTok().is(AsmToken::Minus)) {
+    isNegative = true;
+    Parser.Lex();
+  }
+  const AsmToken &Tok = Parser.getTok();
+  if (Tok.is(AsmToken::Real)) {
+    APFloat RealVal(APFloat::IEEEdouble, Tok.getString());
+    uint64_t IntVal = RealVal.bitcastToAPInt().getZExtValue();
+    // If we had a '-' in front, toggle the sign bit.
+    IntVal ^= (uint64_t)isNegative << 63;
+    int Val = ARM_AM::getFP64Imm(APInt(64, IntVal));
+    Parser.Lex(); // Eat the token.
+    if (Val == -1) {
+      TokError("floating point value out of range");
+      return MatchOperand_ParseFail;
+    }
+    Operands.push_back(ARMOperand::CreateFPImm(Val, S, getContext()));
+    return MatchOperand_Success;
+  }
+  if (Tok.is(AsmToken::Integer)) {
+    int64_t Val = Tok.getIntVal();
+    Parser.Lex(); // Eat the token.
+    if (Val > 255 || Val < 0) {
+      TokError("encoded floating point value out of range");
+      return MatchOperand_ParseFail;
+    }
+    Operands.push_back(ARMOperand::CreateFPImm(Val, S, getContext()));
+    return MatchOperand_Success;
+  }
+
+  TokError("invalid floating point immediate");
+  return MatchOperand_ParseFail;
+}
 /// Parse a arm instruction operand.  For now this parses the operand regardless
 /// of the mnemonic.
-bool ARMAsmParser::ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
+bool ARMAsmParser::parseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
                                 StringRef Mnemonic) {
   SMLoc S, E;
 
@@ -1787,13 +3391,20 @@ bool ARMAsmParser::ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
     Error(Parser.getTok().getLoc(), "unexpected token in operand");
     return true;
   case AsmToken::Identifier: {
-    if (!TryParseRegisterWithWriteBack(Operands))
+    // If this is VMRS, check for the apsr_nzcv operand.
+    if (!tryParseRegisterWithWriteBack(Operands))
       return false;
-    int Res = TryParseShiftRegister(Operands);
+    int Res = tryParseShiftRegister(Operands);
     if (Res == 0) // success
       return false;
     else if (Res == -1) // irrecoverable error
       return true;
+    if (Mnemonic == "vmrs" && Parser.getTok().getString() == "apsr_nzcv") {
+      S = Parser.getTok().getLoc();
+      Parser.Lex();
+      Operands.push_back(ARMOperand::CreateToken("apsr_nzcv", S));
+      return false;
+    }
 
     // Fall though for the Identifier case that is not a register or a
     // special name.
@@ -1811,26 +3422,36 @@ bool ARMAsmParser::ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
     return false;
   }
   case AsmToken::LBrac:
-    return ParseMemory(Operands);
+    return parseMemory(Operands);
   case AsmToken::LCurly:
-    return ParseRegisterList(Operands);
-  case AsmToken::Hash:
+    return parseRegisterList(Operands);
+  case AsmToken::Hash: {
     // #42 -> immediate.
     // TODO: ":lower16:" and ":upper16:" modifiers after # before immediate
     S = Parser.getTok().getLoc();
     Parser.Lex();
+    bool isNegative = Parser.getTok().is(AsmToken::Minus);
     const MCExpr *ImmVal;
     if (getParser().ParseExpression(ImmVal))
       return true;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(ImmVal);
+    if (!CE) {
+      Error(S, "constant expression expected");
+      return MatchOperand_ParseFail;
+    }
+    int32_t Val = CE->getValue();
+    if (isNegative && Val == 0)
+      ImmVal = MCConstantExpr::Create(INT32_MIN, getContext());
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
     Operands.push_back(ARMOperand::CreateImm(ImmVal, S, E));
     return false;
+  }
   case AsmToken::Colon: {
     // ":lower16:" and ":upper16:" expression prefixes
     // FIXME: Check it's an expression prefix,
     // e.g. (FOO - :lower16:BAR) isn't legal.
     ARMMCExpr::VariantKind RefKind;
-    if (ParsePrefix(RefKind))
+    if (parsePrefix(RefKind))
       return true;
 
     const MCExpr *SubExprVal;
@@ -1846,9 +3467,9 @@ bool ARMAsmParser::ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
   }
 }
 
-// ParsePrefix - Parse ARM 16-bit relocations expression prefix, i.e.
+// parsePrefix - Parse ARM 16-bit relocations expression prefix, i.e.
 //  :lower16: and :upper16:.
-bool ARMAsmParser::ParsePrefix(ARMMCExpr::VariantKind &RefKind) {
+bool ARMAsmParser::parsePrefix(ARMMCExpr::VariantKind &RefKind) {
   RefKind = ARMMCExpr::VK_ARM_None;
 
   // :lower16: and :upper16: modifiers
@@ -1879,55 +3500,16 @@ bool ARMAsmParser::ParsePrefix(ARMMCExpr::VariantKind &RefKind) {
   return false;
 }
 
-const MCExpr *
-ARMAsmParser::ApplyPrefixToExpr(const MCExpr *E,
-                                MCSymbolRefExpr::VariantKind Variant) {
-  // Recurse over the given expression, rebuilding it to apply the given variant
-  // to the leftmost symbol.
-  if (Variant == MCSymbolRefExpr::VK_None)
-    return E;
-
-  switch (E->getKind()) {
-  case MCExpr::Target:
-    llvm_unreachable("Can't handle target expr yet");
-  case MCExpr::Constant:
-    llvm_unreachable("Can't handle lower16/upper16 of constant yet");
-
-  case MCExpr::SymbolRef: {
-    const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(E);
-
-    if (SRE->getKind() != MCSymbolRefExpr::VK_None)
-      return 0;
-
-    return MCSymbolRefExpr::Create(&SRE->getSymbol(), Variant, getContext());
-  }
-
-  case MCExpr::Unary:
-    llvm_unreachable("Can't handle unary expressions yet");
-
-  case MCExpr::Binary: {
-    const MCBinaryExpr *BE = cast<MCBinaryExpr>(E);
-    const MCExpr *LHS = ApplyPrefixToExpr(BE->getLHS(), Variant);
-    const MCExpr *RHS = BE->getRHS();
-    if (!LHS)
-      return 0;
-
-    return MCBinaryExpr::Create(BE->getOpcode(), LHS, RHS, getContext());
-  }
-  }
-
-  assert(0 && "Invalid expression kind!");
-  return 0;
-}
-
 /// \brief Given a mnemonic, split out possible predication code and carry
 /// setting letters to form a canonical mnemonic and flags.
 //
 // FIXME: Would be nice to autogen this.
-static StringRef SplitMnemonic(StringRef Mnemonic,
-                               unsigned &PredicationCode,
-                               bool &CarrySetting,
-                               unsigned &ProcessorIMod) {
+// FIXME: This is a bit of a maze of special cases.
+StringRef ARMAsmParser::splitMnemonic(StringRef Mnemonic,
+                                      unsigned &PredicationCode,
+                                      bool &CarrySetting,
+                                      unsigned &ProcessorIMod,
+                                      StringRef &ITMask) {
   PredicationCode = ARMCC::AL;
   CarrySetting = false;
   ProcessorIMod = 0;
@@ -1935,23 +3517,22 @@ static StringRef SplitMnemonic(StringRef Mnemonic,
   // Ignore some mnemonics we know aren't predicated forms.
   //
   // FIXME: Would be nice to autogen this.
-  if (Mnemonic == "teq" || Mnemonic == "vceq" ||
-      Mnemonic == "movs" ||
-      Mnemonic == "svc" ||
-      (Mnemonic == "mls" || Mnemonic == "smmls" || Mnemonic == "vcls" ||
-       Mnemonic == "vmls" || Mnemonic == "vnmls") ||
-      Mnemonic == "vacge" || Mnemonic == "vcge" ||
-      Mnemonic == "vclt" ||
-      Mnemonic == "vacgt" || Mnemonic == "vcgt" ||
-      Mnemonic == "vcle" ||
-      (Mnemonic == "smlal" || Mnemonic == "umaal" || Mnemonic == "umlal" ||
-       Mnemonic == "vabal" || Mnemonic == "vmlal" || Mnemonic == "vpadal" ||
-       Mnemonic == "vqdmlal" || Mnemonic == "bics"))
+  if ((Mnemonic == "movs" && isThumb()) ||
+      Mnemonic == "teq"   || Mnemonic == "vceq"   || Mnemonic == "svc"   ||
+      Mnemonic == "mls"   || Mnemonic == "smmls"  || Mnemonic == "vcls"  ||
+      Mnemonic == "vmls"  || Mnemonic == "vnmls"  || Mnemonic == "vacge" ||
+      Mnemonic == "vcge"  || Mnemonic == "vclt"   || Mnemonic == "vacgt" ||
+      Mnemonic == "vcgt"  || Mnemonic == "vcle"   || Mnemonic == "smlal" ||
+      Mnemonic == "umaal" || Mnemonic == "umlal"  || Mnemonic == "vabal" ||
+      Mnemonic == "vmlal" || Mnemonic == "vpadal" || Mnemonic == "vqdmlal")
     return Mnemonic;
 
   // First, split out any predication code. Ignore mnemonics we know aren't
   // predicated but do have a carry-set and so weren't caught above.
-  if (Mnemonic != "adcs") {
+  if (Mnemonic != "adcs" && Mnemonic != "bics" && Mnemonic != "movs" &&
+      Mnemonic != "muls" && Mnemonic != "smlals" && Mnemonic != "smulls" &&
+      Mnemonic != "umlals" && Mnemonic != "umulls" && Mnemonic != "lsls" &&
+      Mnemonic != "sbcs" && Mnemonic != "rscs") {
     unsigned CC = StringSwitch<unsigned>(Mnemonic.substr(Mnemonic.size()-2))
       .Case("eq", ARMCC::EQ)
       .Case("ne", ARMCC::NE)
@@ -1980,11 +3561,12 @@ static StringRef SplitMnemonic(StringRef Mnemonic,
   // Next, determine if we have a carry setting bit. We explicitly ignore all
   // the instructions we know end in 's'.
   if (Mnemonic.endswith("s") &&
-      !(Mnemonic == "asrs" || Mnemonic == "cps" || Mnemonic == "mls" ||
-        Mnemonic == "movs" || Mnemonic == "mrs" || Mnemonic == "smmls" ||
-        Mnemonic == "vabs" || Mnemonic == "vcls" || Mnemonic == "vmls" ||
-        Mnemonic == "vmrs" || Mnemonic == "vnmls" || Mnemonic == "vqabs" ||
-        Mnemonic == "vrecps" || Mnemonic == "vrsqrts")) {
+      !(Mnemonic == "cps" || Mnemonic == "mls" ||
+        Mnemonic == "mrs" || Mnemonic == "smmls" || Mnemonic == "vabs" ||
+        Mnemonic == "vcls" || Mnemonic == "vmls" || Mnemonic == "vmrs" ||
+        Mnemonic == "vnmls" || Mnemonic == "vqabs" || Mnemonic == "vrecps" ||
+        Mnemonic == "vrsqrts" || Mnemonic == "srs" ||
+        (Mnemonic == "movs" && isThumb()))) {
     Mnemonic = Mnemonic.slice(0, Mnemonic.size() - 1);
     CarrySetting = true;
   }
@@ -2004,6 +3586,12 @@ static StringRef SplitMnemonic(StringRef Mnemonic,
     }
   }
 
+  // The "it" instruction has the condition mask on the end of the mnemonic.
+  if (Mnemonic.startswith("it")) {
+    ITMask = Mnemonic.slice(2, Mnemonic.size());
+    Mnemonic = Mnemonic.slice(0, 2);
+  }
+
   return Mnemonic;
 }
 
@@ -2012,37 +3600,154 @@ static StringRef SplitMnemonic(StringRef Mnemonic,
 //
 // FIXME: It would be nice to autogen this.
 void ARMAsmParser::
-GetMnemonicAcceptInfo(StringRef Mnemonic, bool &CanAcceptCarrySet,
+getMnemonicAcceptInfo(StringRef Mnemonic, bool &CanAcceptCarrySet,
                       bool &CanAcceptPredicationCode) {
   if (Mnemonic == "and" || Mnemonic == "lsl" || Mnemonic == "lsr" ||
       Mnemonic == "rrx" || Mnemonic == "ror" || Mnemonic == "sub" ||
-      Mnemonic == "smull" || Mnemonic == "add" || Mnemonic == "adc" ||
+      Mnemonic == "add" || Mnemonic == "adc" ||
       Mnemonic == "mul" || Mnemonic == "bic" || Mnemonic == "asr" ||
-      Mnemonic == "umlal" || Mnemonic == "orr" || Mnemonic == "mvn" ||
+      Mnemonic == "orr" || Mnemonic == "mvn" ||
       Mnemonic == "rsb" || Mnemonic == "rsc" || Mnemonic == "orn" ||
-      Mnemonic == "sbc" || Mnemonic == "mla" || Mnemonic == "umull" ||
-      Mnemonic == "eor" || Mnemonic == "smlal" ||
-      (Mnemonic == "mov" && !isThumbOne())) {
+      Mnemonic == "sbc" || Mnemonic == "eor" || Mnemonic == "neg" ||
+      (!isThumb() && (Mnemonic == "smull" || Mnemonic == "mov" ||
+                      Mnemonic == "mla" || Mnemonic == "smlal" ||
+                      Mnemonic == "umlal" || Mnemonic == "umull"))) {
     CanAcceptCarrySet = true;
-  } else {
+  } else
     CanAcceptCarrySet = false;
-  }
 
   if (Mnemonic == "cbnz" || Mnemonic == "setend" || Mnemonic == "dmb" ||
       Mnemonic == "cps" || Mnemonic == "mcr2" || Mnemonic == "it" ||
       Mnemonic == "mcrr2" || Mnemonic == "cbz" || Mnemonic == "cdp2" ||
       Mnemonic == "trap" || Mnemonic == "mrc2" || Mnemonic == "mrrc2" ||
-      Mnemonic == "dsb" || Mnemonic == "movs" || Mnemonic == "isb" ||
-      Mnemonic == "clrex" || Mnemonic.startswith("cps")) {
+      Mnemonic == "dsb" || Mnemonic == "isb" || Mnemonic == "setend" ||
+      (Mnemonic == "clrex" && !isThumb()) ||
+      (Mnemonic == "nop" && isThumbOne()) ||
+      ((Mnemonic == "pld" || Mnemonic == "pli" || Mnemonic == "pldw" ||
+        Mnemonic == "ldc2" || Mnemonic == "ldc2l" ||
+        Mnemonic == "stc2" || Mnemonic == "stc2l") && !isThumb()) ||
+      ((Mnemonic.startswith("rfe") || Mnemonic.startswith("srs")) &&
+       !isThumb()) ||
+      Mnemonic.startswith("cps") || (Mnemonic == "movs" && isThumbOne())) {
     CanAcceptPredicationCode = false;
-  } else {
+  } else
     CanAcceptPredicationCode = true;
-  }
 
-  if (isThumb())
+  if (isThumb()) {
     if (Mnemonic == "bkpt" || Mnemonic == "mcr" || Mnemonic == "mcrr" ||
         Mnemonic == "mrc" || Mnemonic == "mrrc" || Mnemonic == "cdp")
       CanAcceptPredicationCode = false;
+  }
+}
+
+bool ARMAsmParser::shouldOmitCCOutOperand(StringRef Mnemonic,
+                               SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // FIXME: This is all horribly hacky. We really need a better way to deal
+  // with optional operands like this in the matcher table.
+
+  // The 'mov' mnemonic is special. One variant has a cc_out operand, while
+  // another does not. Specifically, the MOVW instruction does not. So we
+  // special case it here and remove the defaulted (non-setting) cc_out
+  // operand if that's the instruction we're trying to match.
+  //
+  // We do this as post-processing of the explicit operands rather than just
+  // conditionally adding the cc_out in the first place because we need
+  // to check the type of the parsed immediate operand.
+  if (Mnemonic == "mov" && Operands.size() > 4 && !isThumb() &&
+      !static_cast<ARMOperand*>(Operands[4])->isARMSOImm() &&
+      static_cast<ARMOperand*>(Operands[4])->isImm0_65535Expr() &&
+      static_cast<ARMOperand*>(Operands[1])->getReg() == 0)
+    return true;
+
+  // Register-register 'add' for thumb does not have a cc_out operand
+  // when there are only two register operands.
+  if (isThumb() && Mnemonic == "add" && Operands.size() == 5 &&
+      static_cast<ARMOperand*>(Operands[3])->isReg() &&
+      static_cast<ARMOperand*>(Operands[4])->isReg() &&
+      static_cast<ARMOperand*>(Operands[1])->getReg() == 0)
+    return true;
+  // Register-register 'add' for thumb does not have a cc_out operand
+  // when it's an ADD Rdm, SP, {Rdm|#imm0_255} instruction. We do
+  // have to check the immediate range here since Thumb2 has a variant
+  // that can handle a different range and has a cc_out operand.
+  if (((isThumb() && Mnemonic == "add") ||
+       (isThumbTwo() && Mnemonic == "sub")) &&
+      Operands.size() == 6 &&
+      static_cast<ARMOperand*>(Operands[3])->isReg() &&
+      static_cast<ARMOperand*>(Operands[4])->isReg() &&
+      static_cast<ARMOperand*>(Operands[4])->getReg() == ARM::SP &&
+      static_cast<ARMOperand*>(Operands[1])->getReg() == 0 &&
+      (static_cast<ARMOperand*>(Operands[5])->isReg() ||
+       static_cast<ARMOperand*>(Operands[5])->isImm0_1020s4()))
+    return true;
+  // For Thumb2, add/sub immediate does not have a cc_out operand for the
+  // imm0_4095 variant. That's the least-preferred variant when
+  // selecting via the generic "add" mnemonic, so to know that we
+  // should remove the cc_out operand, we have to explicitly check that
+  // it's not one of the other variants. Ugh.
+  if (isThumbTwo() && (Mnemonic == "add" || Mnemonic == "sub") &&
+      Operands.size() == 6 &&
+      static_cast<ARMOperand*>(Operands[3])->isReg() &&
+      static_cast<ARMOperand*>(Operands[4])->isReg() &&
+      static_cast<ARMOperand*>(Operands[5])->isImm()) {
+    // Nest conditions rather than one big 'if' statement for readability.
+    //
+    // If either register is a high reg, it's either one of the SP
+    // variants (handled above) or a 32-bit encoding, so we just
+    // check against T3.
+    if ((!isARMLowRegister(static_cast<ARMOperand*>(Operands[3])->getReg()) ||
+         !isARMLowRegister(static_cast<ARMOperand*>(Operands[4])->getReg())) &&
+        static_cast<ARMOperand*>(Operands[5])->isT2SOImm())
+      return false;
+    // If both registers are low, we're in an IT block, and the immediate is
+    // in range, we should use encoding T1 instead, which has a cc_out.
+    if (inITBlock() &&
+        isARMLowRegister(static_cast<ARMOperand*>(Operands[3])->getReg()) &&
+        isARMLowRegister(static_cast<ARMOperand*>(Operands[4])->getReg()) &&
+        static_cast<ARMOperand*>(Operands[5])->isImm0_7())
+      return false;
+
+    // Otherwise, we use encoding T4, which does not have a cc_out
+    // operand.
+    return true;
+  }
+
+  // The thumb2 multiply instruction doesn't have a CCOut register, so
+  // if we have a "mul" mnemonic in Thumb mode, check if we'll be able to
+  // use the 16-bit encoding or not.
+  if (isThumbTwo() && Mnemonic == "mul" && Operands.size() == 6 &&
+      static_cast<ARMOperand*>(Operands[1])->getReg() == 0 &&
+      static_cast<ARMOperand*>(Operands[3])->isReg() &&
+      static_cast<ARMOperand*>(Operands[4])->isReg() &&
+      static_cast<ARMOperand*>(Operands[5])->isReg() &&
+      // If the registers aren't low regs, the destination reg isn't the
+      // same as one of the source regs, or the cc_out operand is zero
+      // outside of an IT block, we have to use the 32-bit encoding, so
+      // remove the cc_out operand.
+      (!isARMLowRegister(static_cast<ARMOperand*>(Operands[3])->getReg()) ||
+       !isARMLowRegister(static_cast<ARMOperand*>(Operands[4])->getReg()) ||
+       !inITBlock() ||
+       (static_cast<ARMOperand*>(Operands[3])->getReg() !=
+        static_cast<ARMOperand*>(Operands[5])->getReg() &&
+        static_cast<ARMOperand*>(Operands[3])->getReg() !=
+        static_cast<ARMOperand*>(Operands[4])->getReg())))
+    return true;
+
+
+
+  // Register-register 'add/sub' for thumb does not have a cc_out operand
+  // when it's an ADD/SUB SP, #imm. Be lenient on count since there's also
+  // the "add/sub SP, SP, #imm" version. If the follow-up operands aren't
+  // right, this will result in better diagnostics (which operand is off)
+  // anyway.
+  if (isThumb() && (Mnemonic == "add" || Mnemonic == "sub") &&
+      (Operands.size() == 5 || Operands.size() == 6) &&
+      static_cast<ARMOperand*>(Operands[3])->isReg() &&
+      static_cast<ARMOperand*>(Operands[3])->getReg() == ARM::SP &&
+      static_cast<ARMOperand*>(Operands[1])->getReg() == 0)
+    return true;
+
+  return false;
 }
 
 /// Parse an arm instruction mnemonic followed by its operands.
@@ -2050,16 +3755,51 @@ bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create the leading tokens for the mnemonic, split by '.' characters.
   size_t Start = 0, Next = Name.find('.');
-  StringRef Head = Name.slice(Start, Next);
+  StringRef Mnemonic = Name.slice(Start, Next);
 
   // Split out the predication code and carry setting flag from the mnemonic.
   unsigned PredicationCode;
   unsigned ProcessorIMod;
   bool CarrySetting;
-  Head = SplitMnemonic(Head, PredicationCode, CarrySetting,
-                       ProcessorIMod);
+  StringRef ITMask;
+  Mnemonic = splitMnemonic(Mnemonic, PredicationCode, CarrySetting,
+                           ProcessorIMod, ITMask);
 
-  Operands.push_back(ARMOperand::CreateToken(Head, NameLoc));
+  // In Thumb1, only the branch (B) instruction can be predicated.
+  if (isThumbOne() && PredicationCode != ARMCC::AL && Mnemonic != "b") {
+    Parser.EatToEndOfStatement();
+    return Error(NameLoc, "conditional execution not supported in Thumb1");
+  }
+
+  Operands.push_back(ARMOperand::CreateToken(Mnemonic, NameLoc));
+
+  // Handle the IT instruction ITMask. Convert it to a bitmask. This
+  // is the mask as it will be for the IT encoding if the conditional
+  // encoding has a '1' as it's bit0 (i.e. 't' ==> '1'). In the case
+  // where the conditional bit0 is zero, the instruction post-processing
+  // will adjust the mask accordingly.
+  if (Mnemonic == "it") {
+    SMLoc Loc = SMLoc::getFromPointer(NameLoc.getPointer() + 2);
+    if (ITMask.size() > 3) {
+      Parser.EatToEndOfStatement();
+      return Error(Loc, "too many conditions on IT instruction");
+    }
+    unsigned Mask = 8;
+    for (unsigned i = ITMask.size(); i != 0; --i) {
+      char pos = ITMask[i - 1];
+      if (pos != 't' && pos != 'e') {
+        Parser.EatToEndOfStatement();
+        return Error(Loc, "illegal IT block condition mask '" + ITMask + "'");
+      }
+      Mask >>= 1;
+      if (ITMask[i - 1] == 't')
+        Mask |= 8;
+    }
+    Operands.push_back(ARMOperand::CreateITMask(Mask, Loc));
+  }
+
+  // FIXME: This is all a pretty gross hack. We should automatically handle
+  // optional operands like this via tblgen.
 
   // Next, add the CCOut and ConditionCode operands, if needed.
   //
@@ -2069,34 +3809,36 @@ bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
   // the matcher deal with finding the right instruction or generating an
   // appropriate error.
   bool CanAcceptCarrySet, CanAcceptPredicationCode;
-  GetMnemonicAcceptInfo(Head, CanAcceptCarrySet, CanAcceptPredicationCode);
+  getMnemonicAcceptInfo(Mnemonic, CanAcceptCarrySet, CanAcceptPredicationCode);
 
   // If we had a carry-set on an instruction that can't do that, issue an
   // error.
   if (!CanAcceptCarrySet && CarrySetting) {
     Parser.EatToEndOfStatement();
-    return Error(NameLoc, "instruction '" + Head +
+    return Error(NameLoc, "instruction '" + Mnemonic +
                  "' can not set flags, but 's' suffix specified");
   }
+  // If we had a predication code on an instruction that can't do that, issue an
+  // error.
+  if (!CanAcceptPredicationCode && PredicationCode != ARMCC::AL) {
+    Parser.EatToEndOfStatement();
+    return Error(NameLoc, "instruction '" + Mnemonic +
+                 "' is not predicable, but condition code specified");
+  }
 
   // Add the carry setting operand, if necessary.
-  //
-  // FIXME: It would be awesome if we could somehow invent a location such that
-  // match errors on this operand would print a nice diagnostic about how the
-  // 's' character in the mnemonic resulted in a CCOut operand.
-  if (CanAcceptCarrySet)
+  if (CanAcceptCarrySet) {
+    SMLoc Loc = SMLoc::getFromPointer(NameLoc.getPointer() + Mnemonic.size());
     Operands.push_back(ARMOperand::CreateCCOut(CarrySetting ? ARM::CPSR : 0,
-                                               NameLoc));
+                                               Loc));
+  }
 
   // Add the predication code operand, if necessary.
   if (CanAcceptPredicationCode) {
+    SMLoc Loc = SMLoc::getFromPointer(NameLoc.getPointer() + Mnemonic.size() +
+                                      CarrySetting);
     Operands.push_back(ARMOperand::CreateCondCode(
-                         ARMCC::CondCodes(PredicationCode), NameLoc));
-  } else {
-    // This mnemonic can't ever accept a predication code, but the user wrote
-    // one (or misspelled another mnemonic).
-
-    // FIXME: Issue a nice error.
+                         ARMCC::CondCodes(PredicationCode), Loc));
   }
 
   // Add the processor imod operand, if necessary.
@@ -2104,11 +3846,6 @@ bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
     Operands.push_back(ARMOperand::CreateImm(
           MCConstantExpr::Create(ProcessorIMod, getContext()),
                                  NameLoc, NameLoc));
-  } else {
-    // This mnemonic can't ever accept a imod, but the user wrote
-    // one (or misspelled another mnemonic).
-
-    // FIXME: Issue a nice error.
   }
 
   // Add the remaining tokens in the mnemonic.
@@ -2117,13 +3854,19 @@ bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
     Next = Name.find('.', Start + 1);
     StringRef ExtraToken = Name.slice(Start, Next);
 
-    Operands.push_back(ARMOperand::CreateToken(ExtraToken, NameLoc));
+    // For now, we're only parsing Thumb1 (for the most part), so
+    // just ignore ".n" qualifiers. We'll use them to restrict
+    // matching when we do Thumb2.
+    if (ExtraToken != ".n") {
+      SMLoc Loc = SMLoc::getFromPointer(NameLoc.getPointer() + Start);
+      Operands.push_back(ARMOperand::CreateToken(ExtraToken, Loc));
+    }
   }
 
   // Read the remaining operands.
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
     // Read the first operand.
-    if (ParseOperand(Operands, Head)) {
+    if (parseOperand(Operands, Mnemonic)) {
       Parser.EatToEndOfStatement();
       return true;
     }
@@ -2132,7 +3875,7 @@ bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
       Parser.Lex();  // Eat the comma.
 
       // Parse and remember the operand.
-      if (ParseOperand(Operands, Head)) {
+      if (parseOperand(Operands, Mnemonic)) {
         Parser.EatToEndOfStatement();
         return true;
       }
@@ -2140,75 +3883,548 @@ bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
   }
 
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    SMLoc Loc = getLexer().getLoc();
     Parser.EatToEndOfStatement();
-    return TokError("unexpected token in argument list");
+    return Error(Loc, "unexpected token in argument list");
   }
 
   Parser.Lex(); // Consume the EndOfStatement
+
+  // Some instructions, mostly Thumb, have forms for the same mnemonic that
+  // do and don't have a cc_out optional-def operand. With some spot-checks
+  // of the operand list, we can figure out which variant we're trying to
+  // parse and adjust accordingly before actually matching. We shouldn't ever
+  // try to remove a cc_out operand that was explicitly set on the the
+  // mnemonic, of course (CarrySetting == true). Reason number #317 the
+  // table driven matcher doesn't fit well with the ARM instruction set.
+  if (!CarrySetting && shouldOmitCCOutOperand(Mnemonic, Operands)) {
+    ARMOperand *Op = static_cast<ARMOperand*>(Operands[1]);
+    Operands.erase(Operands.begin() + 1);
+    delete Op;
+  }
+
+  // ARM mode 'blx' need special handling, as the register operand version
+  // is predicable, but the label operand version is not. So, we can't rely
+  // on the Mnemonic based checking to correctly figure out when to put
+  // a k_CondCode operand in the list. If we're trying to match the label
+  // version, remove the k_CondCode operand here.
+  if (!isThumb() && Mnemonic == "blx" && Operands.size() == 3 &&
+      static_cast<ARMOperand*>(Operands[2])->isImm()) {
+    ARMOperand *Op = static_cast<ARMOperand*>(Operands[1]);
+    Operands.erase(Operands.begin() + 1);
+    delete Op;
+  }
+
+  // The vector-compare-to-zero instructions have a literal token "#0" at
+  // the end that comes to here as an immediate operand. Convert it to a
+  // token to play nicely with the matcher.
+  if ((Mnemonic == "vceq" || Mnemonic == "vcge" || Mnemonic == "vcgt" ||
+      Mnemonic == "vcle" || Mnemonic == "vclt") && Operands.size() == 6 &&
+      static_cast<ARMOperand*>(Operands[5])->isImm()) {
+    ARMOperand *Op = static_cast<ARMOperand*>(Operands[5]);
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op->getImm());
+    if (CE && CE->getValue() == 0) {
+      Operands.erase(Operands.begin() + 5);
+      Operands.push_back(ARMOperand::CreateToken("#0", Op->getStartLoc()));
+      delete Op;
+    }
+  }
+  // VCMP{E} does the same thing, but with a different operand count.
+  if ((Mnemonic == "vcmp" || Mnemonic == "vcmpe") && Operands.size() == 5 &&
+      static_cast<ARMOperand*>(Operands[4])->isImm()) {
+    ARMOperand *Op = static_cast<ARMOperand*>(Operands[4]);
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op->getImm());
+    if (CE && CE->getValue() == 0) {
+      Operands.erase(Operands.begin() + 4);
+      Operands.push_back(ARMOperand::CreateToken("#0", Op->getStartLoc()));
+      delete Op;
+    }
+  }
+  // Similarly, the Thumb1 "RSB" instruction has a literal "#0" on the
+  // end. Convert it to a token here.
+  if (Mnemonic == "rsb" && isThumb() && Operands.size() == 6 &&
+      static_cast<ARMOperand*>(Operands[5])->isImm()) {
+    ARMOperand *Op = static_cast<ARMOperand*>(Operands[5]);
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op->getImm());
+    if (CE && CE->getValue() == 0) {
+      Operands.erase(Operands.begin() + 5);
+      Operands.push_back(ARMOperand::CreateToken("#0", Op->getStartLoc()));
+      delete Op;
+    }
+  }
+
   return false;
 }
 
+// Validate context-sensitive operand constraints.
+
+// return 'true' if register list contains non-low GPR registers,
+// 'false' otherwise. If Reg is in the register list or is HiReg, set
+// 'containsReg' to true.
+static bool checkLowRegisterList(MCInst Inst, unsigned OpNo, unsigned Reg,
+                                 unsigned HiReg, bool &containsReg) {
+  containsReg = false;
+  for (unsigned i = OpNo; i < Inst.getNumOperands(); ++i) {
+    unsigned OpReg = Inst.getOperand(i).getReg();
+    if (OpReg == Reg)
+      containsReg = true;
+    // Anything other than a low register isn't legal here.
+    if (!isARMLowRegister(OpReg) && (!HiReg || OpReg != HiReg))
+      return true;
+  }
+  return false;
+}
+
+// Check if the specified regisgter is in the register list of the inst,
+// starting at the indicated operand number.
+static bool listContainsReg(MCInst &Inst, unsigned OpNo, unsigned Reg) {
+  for (unsigned i = OpNo; i < Inst.getNumOperands(); ++i) {
+    unsigned OpReg = Inst.getOperand(i).getReg();
+    if (OpReg == Reg)
+      return true;
+  }
+  return false;
+}
+
+// FIXME: We would really prefer to have MCInstrInfo (the wrapper around
+// the ARMInsts array) instead. Getting that here requires awkward
+// API changes, though. Better way?
+namespace llvm {
+extern MCInstrDesc ARMInsts[];
+}
+static MCInstrDesc &getInstDesc(unsigned Opcode) {
+  return ARMInsts[Opcode];
+}
+
+// FIXME: We would really like to be able to tablegen'erate this.
+bool ARMAsmParser::
+validateInstruction(MCInst &Inst,
+                    const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  MCInstrDesc &MCID = getInstDesc(Inst.getOpcode());
+  SMLoc Loc = Operands[0]->getStartLoc();
+  // Check the IT block state first.
+  // NOTE: In Thumb mode, the BKPT instruction has the interesting property of
+  // being allowed in IT blocks, but not being predicable.  It just always
+  // executes.
+  if (inITBlock() && Inst.getOpcode() != ARM::tBKPT) {
+    unsigned bit = 1;
+    if (ITState.FirstCond)
+      ITState.FirstCond = false;
+    else
+      bit = (ITState.Mask >> (5 - ITState.CurPosition)) & 1;
+    // The instruction must be predicable.
+    if (!MCID.isPredicable())
+      return Error(Loc, "instructions in IT block must be predicable");
+    unsigned Cond = Inst.getOperand(MCID.findFirstPredOperandIdx()).getImm();
+    unsigned ITCond = bit ? ITState.Cond :
+      ARMCC::getOppositeCondition(ITState.Cond);
+    if (Cond != ITCond) {
+      // Find the condition code Operand to get its SMLoc information.
+      SMLoc CondLoc;
+      for (unsigned i = 1; i < Operands.size(); ++i)
+        if (static_cast<ARMOperand*>(Operands[i])->isCondCode())
+          CondLoc = Operands[i]->getStartLoc();
+      return Error(CondLoc, "incorrect condition in IT block; got '" +
+                   StringRef(ARMCondCodeToString(ARMCC::CondCodes(Cond))) +
+                   "', but expected '" +
+                   ARMCondCodeToString(ARMCC::CondCodes(ITCond)) + "'");
+    }
+  // Check for non-'al' condition codes outside of the IT block.
+  } else if (isThumbTwo() && MCID.isPredicable() &&
+             Inst.getOperand(MCID.findFirstPredOperandIdx()).getImm() !=
+             ARMCC::AL && Inst.getOpcode() != ARM::tB &&
+             Inst.getOpcode() != ARM::t2B)
+    return Error(Loc, "predicated instructions must be in IT block");
+
+  switch (Inst.getOpcode()) {
+  case ARM::LDRD:
+  case ARM::LDRD_PRE:
+  case ARM::LDRD_POST:
+  case ARM::LDREXD: {
+    // Rt2 must be Rt + 1.
+    unsigned Rt = getARMRegisterNumbering(Inst.getOperand(0).getReg());
+    unsigned Rt2 = getARMRegisterNumbering(Inst.getOperand(1).getReg());
+    if (Rt2 != Rt + 1)
+      return Error(Operands[3]->getStartLoc(),
+                   "destination operands must be sequential");
+    return false;
+  }
+  case ARM::STRD: {
+    // Rt2 must be Rt + 1.
+    unsigned Rt = getARMRegisterNumbering(Inst.getOperand(0).getReg());
+    unsigned Rt2 = getARMRegisterNumbering(Inst.getOperand(1).getReg());
+    if (Rt2 != Rt + 1)
+      return Error(Operands[3]->getStartLoc(),
+                   "source operands must be sequential");
+    return false;
+  }
+  case ARM::STRD_PRE:
+  case ARM::STRD_POST:
+  case ARM::STREXD: {
+    // Rt2 must be Rt + 1.
+    unsigned Rt = getARMRegisterNumbering(Inst.getOperand(1).getReg());
+    unsigned Rt2 = getARMRegisterNumbering(Inst.getOperand(2).getReg());
+    if (Rt2 != Rt + 1)
+      return Error(Operands[3]->getStartLoc(),
+                   "source operands must be sequential");
+    return false;
+  }
+  case ARM::SBFX:
+  case ARM::UBFX: {
+    // width must be in range [1, 32-lsb]
+    unsigned lsb = Inst.getOperand(2).getImm();
+    unsigned widthm1 = Inst.getOperand(3).getImm();
+    if (widthm1 >= 32 - lsb)
+      return Error(Operands[5]->getStartLoc(),
+                   "bitfield width must be in range [1,32-lsb]");
+    return false;
+  }
+  case ARM::tLDMIA: {
+    // If we're parsing Thumb2, the .w variant is available and handles
+    // most cases that are normally illegal for a Thumb1 LDM
+    // instruction. We'll make the transformation in processInstruction()
+    // if necessary.
+    //
+    // Thumb LDM instructions are writeback iff the base register is not
+    // in the register list.
+    unsigned Rn = Inst.getOperand(0).getReg();
+    bool hasWritebackToken =
+      (static_cast<ARMOperand*>(Operands[3])->isToken() &&
+       static_cast<ARMOperand*>(Operands[3])->getToken() == "!");
+    bool listContainsBase;
+    if (checkLowRegisterList(Inst, 3, Rn, 0, listContainsBase) && !isThumbTwo())
+      return Error(Operands[3 + hasWritebackToken]->getStartLoc(),
+                   "registers must be in range r0-r7");
+    // If we should have writeback, then there should be a '!' token.
+    if (!listContainsBase && !hasWritebackToken && !isThumbTwo())
+      return Error(Operands[2]->getStartLoc(),
+                   "writeback operator '!' expected");
+    // If we should not have writeback, there must not be a '!'. This is
+    // true even for the 32-bit wide encodings.
+    if (listContainsBase && hasWritebackToken)
+      return Error(Operands[3]->getStartLoc(),
+                   "writeback operator '!' not allowed when base register "
+                   "in register list");
+
+    break;
+  }
+  case ARM::t2LDMIA_UPD: {
+    if (listContainsReg(Inst, 3, Inst.getOperand(0).getReg()))
+      return Error(Operands[4]->getStartLoc(),
+                   "writeback operator '!' not allowed when base register "
+                   "in register list");
+    break;
+  }
+  case ARM::tPOP: {
+    bool listContainsBase;
+    if (checkLowRegisterList(Inst, 3, 0, ARM::PC, listContainsBase))
+      return Error(Operands[2]->getStartLoc(),
+                   "registers must be in range r0-r7 or pc");
+    break;
+  }
+  case ARM::tPUSH: {
+    bool listContainsBase;
+    if (checkLowRegisterList(Inst, 3, 0, ARM::LR, listContainsBase))
+      return Error(Operands[2]->getStartLoc(),
+                   "registers must be in range r0-r7 or lr");
+    break;
+  }
+  case ARM::tSTMIA_UPD: {
+    bool listContainsBase;
+    if (checkLowRegisterList(Inst, 4, 0, 0, listContainsBase) && !isThumbTwo())
+      return Error(Operands[4]->getStartLoc(),
+                   "registers must be in range r0-r7");
+    break;
+  }
+  }
+
+  return false;
+}
+
+void ARMAsmParser::
+processInstruction(MCInst &Inst,
+                   const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  switch (Inst.getOpcode()) {
+  case ARM::LDMIA_UPD:
+    // If this is a load of a single register via a 'pop', then we should use
+    // a post-indexed LDR instruction instead, per the ARM ARM.
+    if (static_cast<ARMOperand*>(Operands[0])->getToken() == "pop" &&
+        Inst.getNumOperands() == 5) {
+      MCInst TmpInst;
+      TmpInst.setOpcode(ARM::LDR_POST_IMM);
+      TmpInst.addOperand(Inst.getOperand(4)); // Rt
+      TmpInst.addOperand(Inst.getOperand(0)); // Rn_wb
+      TmpInst.addOperand(Inst.getOperand(1)); // Rn
+      TmpInst.addOperand(MCOperand::CreateReg(0));  // am2offset
+      TmpInst.addOperand(MCOperand::CreateImm(4));
+      TmpInst.addOperand(Inst.getOperand(2)); // CondCode
+      TmpInst.addOperand(Inst.getOperand(3));
+      Inst = TmpInst;
+    }
+    break;
+  case ARM::STMDB_UPD:
+    // If this is a store of a single register via a 'push', then we should use
+    // a pre-indexed STR instruction instead, per the ARM ARM.
+    if (static_cast<ARMOperand*>(Operands[0])->getToken() == "push" &&
+        Inst.getNumOperands() == 5) {
+      MCInst TmpInst;
+      TmpInst.setOpcode(ARM::STR_PRE_IMM);
+      TmpInst.addOperand(Inst.getOperand(0)); // Rn_wb
+      TmpInst.addOperand(Inst.getOperand(4)); // Rt
+      TmpInst.addOperand(Inst.getOperand(1)); // addrmode_imm12
+      TmpInst.addOperand(MCOperand::CreateImm(-4));
+      TmpInst.addOperand(Inst.getOperand(2)); // CondCode
+      TmpInst.addOperand(Inst.getOperand(3));
+      Inst = TmpInst;
+    }
+    break;
+  case ARM::tADDi8:
+    // If the immediate is in the range 0-7, we want tADDi3 iff Rd was
+    // explicitly specified. From the ARM ARM: "Encoding T1 is preferred
+    // to encoding T2 if <Rd> is specified and encoding T2 is preferred
+    // to encoding T1 if <Rd> is omitted."
+    if (Inst.getOperand(3).getImm() < 8 && Operands.size() == 6)
+      Inst.setOpcode(ARM::tADDi3);
+    break;
+  case ARM::tSUBi8:
+    // If the immediate is in the range 0-7, we want tADDi3 iff Rd was
+    // explicitly specified. From the ARM ARM: "Encoding T1 is preferred
+    // to encoding T2 if <Rd> is specified and encoding T2 is preferred
+    // to encoding T1 if <Rd> is omitted."
+    if (Inst.getOperand(3).getImm() < 8 && Operands.size() == 6)
+      Inst.setOpcode(ARM::tSUBi3);
+    break;
+  case ARM::tB:
+    // A Thumb conditional branch outside of an IT block is a tBcc.
+    if (Inst.getOperand(1).getImm() != ARMCC::AL && !inITBlock())
+      Inst.setOpcode(ARM::tBcc);
+    break;
+  case ARM::t2B:
+    // A Thumb2 conditional branch outside of an IT block is a t2Bcc.
+    if (Inst.getOperand(1).getImm() != ARMCC::AL && !inITBlock())
+      Inst.setOpcode(ARM::t2Bcc);
+    break;
+  case ARM::t2Bcc:
+    // If the conditional is AL or we're in an IT block, we really want t2B.
+    if (Inst.getOperand(1).getImm() == ARMCC::AL || inITBlock())
+      Inst.setOpcode(ARM::t2B);
+    break;
+  case ARM::tBcc:
+    // If the conditional is AL, we really want tB.
+    if (Inst.getOperand(1).getImm() == ARMCC::AL)
+      Inst.setOpcode(ARM::tB);
+    break;
+  case ARM::tLDMIA: {
+    // If the register list contains any high registers, or if the writeback
+    // doesn't match what tLDMIA can do, we need to use the 32-bit encoding
+    // instead if we're in Thumb2. Otherwise, this should have generated
+    // an error in validateInstruction().
+    unsigned Rn = Inst.getOperand(0).getReg();
+    bool hasWritebackToken =
+      (static_cast<ARMOperand*>(Operands[3])->isToken() &&
+       static_cast<ARMOperand*>(Operands[3])->getToken() == "!");
+    bool listContainsBase;
+    if (checkLowRegisterList(Inst, 3, Rn, 0, listContainsBase) ||
+        (!listContainsBase && !hasWritebackToken) ||
+        (listContainsBase && hasWritebackToken)) {
+      // 16-bit encoding isn't sufficient. Switch to the 32-bit version.
+      assert (isThumbTwo());
+      Inst.setOpcode(hasWritebackToken ? ARM::t2LDMIA_UPD : ARM::t2LDMIA);
+      // If we're switching to the updating version, we need to insert
+      // the writeback tied operand.
+      if (hasWritebackToken)
+        Inst.insert(Inst.begin(),
+                    MCOperand::CreateReg(Inst.getOperand(0).getReg()));
+    }
+    break;
+  }
+  case ARM::tSTMIA_UPD: {
+    // If the register list contains any high registers, we need to use
+    // the 32-bit encoding instead if we're in Thumb2. Otherwise, this
+    // should have generated an error in validateInstruction().
+    unsigned Rn = Inst.getOperand(0).getReg();
+    bool listContainsBase;
+    if (checkLowRegisterList(Inst, 4, Rn, 0, listContainsBase)) {
+      // 16-bit encoding isn't sufficient. Switch to the 32-bit version.
+      assert (isThumbTwo());
+      Inst.setOpcode(ARM::t2STMIA_UPD);
+    }
+    break;
+  }
+  case ARM::t2MOVi: {
+    // If we can use the 16-bit encoding and the user didn't explicitly
+    // request the 32-bit variant, transform it here.
+    if (isARMLowRegister(Inst.getOperand(0).getReg()) &&
+        Inst.getOperand(1).getImm() <= 255 &&
+        ((!inITBlock() && Inst.getOperand(2).getImm() == ARMCC::AL &&
+         Inst.getOperand(4).getReg() == ARM::CPSR) ||
+        (inITBlock() && Inst.getOperand(4).getReg() == 0)) &&
+        (!static_cast<ARMOperand*>(Operands[2])->isToken() ||
+         static_cast<ARMOperand*>(Operands[2])->getToken() != ".w")) {
+      // The operands aren't in the same order for tMOVi8...
+      MCInst TmpInst;
+      TmpInst.setOpcode(ARM::tMOVi8);
+      TmpInst.addOperand(Inst.getOperand(0));
+      TmpInst.addOperand(Inst.getOperand(4));
+      TmpInst.addOperand(Inst.getOperand(1));
+      TmpInst.addOperand(Inst.getOperand(2));
+      TmpInst.addOperand(Inst.getOperand(3));
+      Inst = TmpInst;
+    }
+    break;
+  }
+  case ARM::t2MOVr: {
+    // If we can use the 16-bit encoding and the user didn't explicitly
+    // request the 32-bit variant, transform it here.
+    if (isARMLowRegister(Inst.getOperand(0).getReg()) &&
+        isARMLowRegister(Inst.getOperand(1).getReg()) &&
+        Inst.getOperand(2).getImm() == ARMCC::AL &&
+        Inst.getOperand(4).getReg() == ARM::CPSR &&
+        (!static_cast<ARMOperand*>(Operands[2])->isToken() ||
+         static_cast<ARMOperand*>(Operands[2])->getToken() != ".w")) {
+      // The operands aren't the same for tMOV[S]r... (no cc_out)
+      MCInst TmpInst;
+      TmpInst.setOpcode(Inst.getOperand(4).getReg() ? ARM::tMOVSr : ARM::tMOVr);
+      TmpInst.addOperand(Inst.getOperand(0));
+      TmpInst.addOperand(Inst.getOperand(1));
+      TmpInst.addOperand(Inst.getOperand(2));
+      TmpInst.addOperand(Inst.getOperand(3));
+      Inst = TmpInst;
+    }
+    break;
+  }
+  case ARM::t2SXTH:
+  case ARM::t2SXTB:
+  case ARM::t2UXTH:
+  case ARM::t2UXTB: {
+    // If we can use the 16-bit encoding and the user didn't explicitly
+    // request the 32-bit variant, transform it here.
+    if (isARMLowRegister(Inst.getOperand(0).getReg()) &&
+        isARMLowRegister(Inst.getOperand(1).getReg()) &&
+        Inst.getOperand(2).getImm() == 0 &&
+        (!static_cast<ARMOperand*>(Operands[2])->isToken() ||
+         static_cast<ARMOperand*>(Operands[2])->getToken() != ".w")) {
+      unsigned NewOpc;
+      switch (Inst.getOpcode()) {
+      default: llvm_unreachable("Illegal opcode!");
+      case ARM::t2SXTH: NewOpc = ARM::tSXTH; break;
+      case ARM::t2SXTB: NewOpc = ARM::tSXTB; break;
+      case ARM::t2UXTH: NewOpc = ARM::tUXTH; break;
+      case ARM::t2UXTB: NewOpc = ARM::tUXTB; break;
+      }
+      // The operands aren't the same for thumb1 (no rotate operand).
+      MCInst TmpInst;
+      TmpInst.setOpcode(NewOpc);
+      TmpInst.addOperand(Inst.getOperand(0));
+      TmpInst.addOperand(Inst.getOperand(1));
+      TmpInst.addOperand(Inst.getOperand(3));
+      TmpInst.addOperand(Inst.getOperand(4));
+      Inst = TmpInst;
+    }
+    break;
+  }
+  case ARM::t2IT: {
+    // The mask bits for all but the first condition are represented as
+    // the low bit of the condition code value implies 't'. We currently
+    // always have 1 implies 't', so XOR toggle the bits if the low bit
+    // of the condition code is zero. The encoding also expects the low
+    // bit of the condition to be encoded as bit 4 of the mask operand,
+    // so mask that in if needed
+    MCOperand &MO = Inst.getOperand(1);
+    unsigned Mask = MO.getImm();
+    unsigned OrigMask = Mask;
+    unsigned TZ = CountTrailingZeros_32(Mask);
+    if ((Inst.getOperand(0).getImm() & 1) == 0) {
+      assert(Mask && TZ <= 3 && "illegal IT mask value!");
+      for (unsigned i = 3; i != TZ; --i)
+        Mask ^= 1 << i;
+    } else
+      Mask |= 0x10;
+    MO.setImm(Mask);
+
+    // Set up the IT block state according to the IT instruction we just
+    // matched.
+    assert(!inITBlock() && "nested IT blocks?!");
+    ITState.Cond = ARMCC::CondCodes(Inst.getOperand(0).getImm());
+    ITState.Mask = OrigMask; // Use the original mask, not the updated one.
+    ITState.CurPosition = 0;
+    ITState.FirstCond = true;
+    break;
+  }
+  }
+}
+
+unsigned ARMAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
+  // 16-bit thumb arithmetic instructions either require or preclude the 'S'
+  // suffix depending on whether they're in an IT block or not.
+  unsigned Opc = Inst.getOpcode();
+  MCInstrDesc &MCID = getInstDesc(Opc);
+  if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
+    assert(MCID.hasOptionalDef() &&
+           "optionally flag setting instruction missing optional def operand");
+    assert(MCID.NumOperands == Inst.getNumOperands() &&
+           "operand count mismatch!");
+    // Find the optional-def operand (cc_out).
+    unsigned OpNo;
+    for (OpNo = 0;
+         !MCID.OpInfo[OpNo].isOptionalDef() && OpNo < MCID.NumOperands;
+         ++OpNo)
+      ;
+    // If we're parsing Thumb1, reject it completely.
+    if (isThumbOne() && Inst.getOperand(OpNo).getReg() != ARM::CPSR)
+      return Match_MnemonicFail;
+    // If we're parsing Thumb2, which form is legal depends on whether we're
+    // in an IT block.
+    if (isThumbTwo() && Inst.getOperand(OpNo).getReg() != ARM::CPSR &&
+        !inITBlock())
+      return Match_RequiresITBlock;
+    if (isThumbTwo() && Inst.getOperand(OpNo).getReg() == ARM::CPSR &&
+        inITBlock())
+      return Match_RequiresNotITBlock;
+  }
+  // Some high-register supporting Thumb1 encodings only allow both registers
+  // to be from r0-r7 when in Thumb2.
+  else if (Opc == ARM::tADDhirr && isThumbOne() &&
+           isARMLowRegister(Inst.getOperand(1).getReg()) &&
+           isARMLowRegister(Inst.getOperand(2).getReg()))
+    return Match_RequiresThumb2;
+  // Others only require ARMv6 or later.
+  else if (Opc == ARM::tMOVr && isThumbOne() && !hasV6Ops() &&
+           isARMLowRegister(Inst.getOperand(0).getReg()) &&
+           isARMLowRegister(Inst.getOperand(1).getReg()))
+    return Match_RequiresV6;
+  return Match_Success;
+}
+
 bool ARMAsmParser::
 MatchAndEmitInstruction(SMLoc IDLoc,
                         SmallVectorImpl<MCParsedAsmOperand*> &Operands,
                         MCStreamer &Out) {
   MCInst Inst;
   unsigned ErrorInfo;
-  MatchResultTy MatchResult, MatchResult2;
+  unsigned MatchResult;
   MatchResult = MatchInstructionImpl(Operands, Inst, ErrorInfo);
-  if (MatchResult != Match_Success) {
-    // If we get a Match_InvalidOperand it might be some arithmetic instruction
-    // that does not update the condition codes.  So try adding a CCOut operand
-    // with a value of reg0.
-    if (MatchResult == Match_InvalidOperand) {
-      Operands.insert(Operands.begin() + 1,
-                      ARMOperand::CreateCCOut(0,
-                                  ((ARMOperand*)Operands[0])->getStartLoc()));
-      MatchResult2 = MatchInstructionImpl(Operands, Inst, ErrorInfo);
-      if (MatchResult2 == Match_Success)
-        MatchResult = Match_Success;
-      else {
-        ARMOperand *CCOut = ((ARMOperand*)Operands[1]);
-        Operands.erase(Operands.begin() + 1);
-        delete CCOut;
-      }
-    }
-    // If we get a Match_MnemonicFail it might be some arithmetic instruction
-    // that updates the condition codes if it ends in 's'.  So see if the
-    // mnemonic ends in 's' and if so try removing the 's' and adding a CCOut
-    // operand with a value of CPSR.
-    else if (MatchResult == Match_MnemonicFail) {
-      // Get the instruction mnemonic, which is the first token.
-      StringRef Mnemonic = ((ARMOperand*)Operands[0])->getToken();
-      if (Mnemonic.substr(Mnemonic.size()-1) == "s") {
-        // removed the 's' from the mnemonic for matching.
-        StringRef MnemonicNoS = Mnemonic.slice(0, Mnemonic.size() - 1);
-        SMLoc NameLoc = ((ARMOperand*)Operands[0])->getStartLoc();
-        ARMOperand *OldMnemonic = ((ARMOperand*)Operands[0]);
-        Operands.erase(Operands.begin());
-        delete OldMnemonic;
-        Operands.insert(Operands.begin(),
-                        ARMOperand::CreateToken(MnemonicNoS, NameLoc));
-        Operands.insert(Operands.begin() + 1,
-                        ARMOperand::CreateCCOut(ARM::CPSR, NameLoc));
-        MatchResult2 = MatchInstructionImpl(Operands, Inst, ErrorInfo);
-        if (MatchResult2 == Match_Success)
-          MatchResult = Match_Success;
-        else {
-          ARMOperand *OldMnemonic = ((ARMOperand*)Operands[0]);
-          Operands.erase(Operands.begin());
-          delete OldMnemonic;
-          Operands.insert(Operands.begin(),
-                          ARMOperand::CreateToken(Mnemonic, NameLoc));
-          ARMOperand *CCOut = ((ARMOperand*)Operands[1]);
-          Operands.erase(Operands.begin() + 1);
-          delete CCOut;
-        }
-      }
-    }
-  }
   switch (MatchResult) {
+  default: break;
   case Match_Success:
+    // Context sensitive operand constraints aren't handled by the matcher,
+    // so check them here.
+    if (validateInstruction(Inst, Operands)) {
+      // Still progress the IT block, otherwise one wrong condition causes
+      // nasty cascading errors.
+      forwardITPosition();
+      return true;
+    }
+
+    // Some instructions need post-processing to, for example, tweak which
+    // encoding is selected.
+    processInstruction(Inst, Operands);
+
+    // Only move forward at the very end so that everything in validate
+    // and process gets a consistent answer about whether we're in an IT
+    // block.
+    forwardITPosition();
+
     Out.EmitInstruction(Inst);
     return false;
   case Match_MissingFeature:
@@ -2227,34 +4443,43 @@ MatchAndEmitInstruction(SMLoc IDLoc,
     return Error(ErrorLoc, "invalid operand for instruction");
   }
   case Match_MnemonicFail:
-    return Error(IDLoc, "unrecognized instruction mnemonic");
+    return Error(IDLoc, "invalid instruction");
   case Match_ConversionFail:
-    return Error(IDLoc, "unable to convert operands to instruction");
+    // The converter function will have already emited a diagnostic.
+    return true;
+  case Match_RequiresNotITBlock:
+    return Error(IDLoc, "flag setting instruction only valid outside IT block");
+  case Match_RequiresITBlock:
+    return Error(IDLoc, "instruction only valid inside IT block");
+  case Match_RequiresV6:
+    return Error(IDLoc, "instruction variant requires ARMv6 or later");
+  case Match_RequiresThumb2:
+    return Error(IDLoc, "instruction variant requires Thumb2");
   }
 
   llvm_unreachable("Implement any new match types added!");
   return true;
 }
 
-/// ParseDirective parses the arm specific directives
+/// parseDirective parses the arm specific directives
 bool ARMAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getIdentifier();
   if (IDVal == ".word")
-    return ParseDirectiveWord(4, DirectiveID.getLoc());
+    return parseDirectiveWord(4, DirectiveID.getLoc());
   else if (IDVal == ".thumb")
-    return ParseDirectiveThumb(DirectiveID.getLoc());
+    return parseDirectiveThumb(DirectiveID.getLoc());
   else if (IDVal == ".thumb_func")
-    return ParseDirectiveThumbFunc(DirectiveID.getLoc());
+    return parseDirectiveThumbFunc(DirectiveID.getLoc());
   else if (IDVal == ".code")
-    return ParseDirectiveCode(DirectiveID.getLoc());
+    return parseDirectiveCode(DirectiveID.getLoc());
   else if (IDVal == ".syntax")
-    return ParseDirectiveSyntax(DirectiveID.getLoc());
+    return parseDirectiveSyntax(DirectiveID.getLoc());
   return true;
 }
 
-/// ParseDirectiveWord
+/// parseDirectiveWord
 ///  ::= .word [ expression (, expression)* ]
-bool ARMAsmParser::ParseDirectiveWord(unsigned Size, SMLoc L) {
+bool ARMAsmParser::parseDirectiveWord(unsigned Size, SMLoc L) {
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
     for (;;) {
       const MCExpr *Value;
@@ -2277,9 +4502,9 @@ bool ARMAsmParser::ParseDirectiveWord(unsigned Size, SMLoc L) {
   return false;
 }
 
-/// ParseDirectiveThumb
+/// parseDirectiveThumb
 ///  ::= .thumb
-bool ARMAsmParser::ParseDirectiveThumb(SMLoc L) {
+bool ARMAsmParser::parseDirectiveThumb(SMLoc L) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
     return Error(L, "unexpected token in directive");
   Parser.Lex();
@@ -2290,9 +4515,9 @@ bool ARMAsmParser::ParseDirectiveThumb(SMLoc L) {
   return false;
 }
 
-/// ParseDirectiveThumbFunc
+/// parseDirectiveThumbFunc
 ///  ::= .thumbfunc symbol_name
-bool ARMAsmParser::ParseDirectiveThumbFunc(SMLoc L) {
+bool ARMAsmParser::parseDirectiveThumbFunc(SMLoc L) {
   const MCAsmInfo &MAI = getParser().getStreamer().getContext().getAsmInfo();
   bool isMachO = MAI.hasSubsectionsViaSymbols();
   StringRef Name;
@@ -2322,9 +4547,9 @@ bool ARMAsmParser::ParseDirectiveThumbFunc(SMLoc L) {
   return false;
 }
 
-/// ParseDirectiveSyntax
+/// parseDirectiveSyntax
 ///  ::= .syntax unified | divided
-bool ARMAsmParser::ParseDirectiveSyntax(SMLoc L) {
+bool ARMAsmParser::parseDirectiveSyntax(SMLoc L) {
   const AsmToken &Tok = Parser.getTok();
   if (Tok.isNot(AsmToken::Identifier))
     return Error(L, "unexpected token in .syntax directive");
@@ -2345,9 +4570,9 @@ bool ARMAsmParser::ParseDirectiveSyntax(SMLoc L) {
   return false;
 }
 
-/// ParseDirectiveCode
+/// parseDirectiveCode
 ///  ::= .code 16 | 32
-bool ARMAsmParser::ParseDirectiveCode(SMLoc L) {
+bool ARMAsmParser::parseDirectiveCode(SMLoc L) {
   const AsmToken &Tok = Parser.getTok();
   if (Tok.isNot(AsmToken::Integer))
     return Error(L, "unexpected token in .code directive");
@@ -2380,8 +4605,8 @@ extern "C" void LLVMInitializeARMAsmLexer();
 
 /// Force static initialization.
 extern "C" void LLVMInitializeARMAsmParser() {
-  RegisterAsmParser<ARMAsmParser> X(TheARMTarget);
-  RegisterAsmParser<ARMAsmParser> Y(TheThumbTarget);
+  RegisterMCAsmParser<ARMAsmParser> X(TheARMTarget);
+  RegisterMCAsmParser<ARMAsmParser> Y(TheThumbTarget);
   LLVMInitializeARMAsmLexer();
 }
 
diff --git a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index bdce2c4cf896..8f2f813b676a 100644
--- a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -6,578 +6,325 @@
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
-//
-// This file is part of the ARM Disassembler.
-// It contains code to implement the public interfaces of ARMDisassembler and
-// ThumbDisassembler, both of which are instances of MCDisassembler.
-//
-//===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "arm-disassembler"
 
-#include "ARMDisassembler.h"
-#include "ARMDisassemblerCore.h"
-
-#include "llvm/ADT/OwningPtr.h"
+#include "ARM.h"
+#include "ARMRegisterInfo.h"
+#include "ARMSubtarget.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
+#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
 #include "llvm/MC/EDInstInfo.h"
 #include "llvm/MC/MCInst.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDisassembler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MemoryObject.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
-//#define DEBUG(X) do { X; } while (0)
-
-/// ARMGenDecoderTables.inc - ARMDecoderTables.inc is tblgen'ed from
-/// ARMDecoderEmitter.cpp TableGen backend.  It contains:
-///
-/// o Mappings from opcode to ARM/Thumb instruction format
-///
-/// o static uint16_t decodeInstruction(uint32_t insn) - the decoding function
-/// for an ARM instruction.
-///
-/// o static uint16_t decodeThumbInstruction(field_t insn) - the decoding
-/// function for a Thumb instruction.
-///
-#include "ARMGenDecoderTables.inc"
-
-#include "ARMGenEDInfo.inc"
-
 using namespace llvm;
 
-/// showBitVector - Use the raw_ostream to log a diagnostic message describing
-/// the inidividual bits of the instruction.
-///
-static inline void showBitVector(raw_ostream &os, const uint32_t &insn) {
-  // Split the bit position markers into more than one lines to fit 80 columns.
-  os << " 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11"
-     << " 10  9  8  7  6  5  4  3  2  1  0 \n";
-  os << "---------------------------------------------------------------"
-     << "----------------------------------\n";
-  os << '|';
-  for (unsigned i = 32; i != 0; --i) {
-    if (insn >> (i - 1) & 0x01)
-      os << " 1";
-    else
-      os << " 0";
-    os << (i%4 == 1 ? '|' : ':');
+typedef MCDisassembler::DecodeStatus DecodeStatus;
+
+namespace {
+/// ARMDisassembler - ARM disassembler for all ARM platforms.
+class ARMDisassembler : public MCDisassembler {
+public:
+  /// Constructor     - Initializes the disassembler.
+  ///
+  ARMDisassembler(const MCSubtargetInfo &STI) :
+    MCDisassembler(STI) {
   }
-  os << '\n';
-  // Split the bit position markers into more than one lines to fit 80 columns.
-  os << "---------------------------------------------------------------"
-     << "----------------------------------\n";
-  os << '\n';
+
+  ~ARMDisassembler() {
+  }
+
+  /// getInstruction - See MCDisassembler.
+  DecodeStatus getInstruction(MCInst &instr,
+                              uint64_t &size,
+                              const MemoryObject &region,
+                              uint64_t address,
+                              raw_ostream &vStream,
+                              raw_ostream &cStream) const;
+
+  /// getEDInfo - See MCDisassembler.
+  EDInstInfo *getEDInfo() const;
+private:
+};
+
+/// ThumbDisassembler - Thumb disassembler for all Thumb platforms.
+class ThumbDisassembler : public MCDisassembler {
+public:
+  /// Constructor     - Initializes the disassembler.
+  ///
+  ThumbDisassembler(const MCSubtargetInfo &STI) :
+    MCDisassembler(STI) {
+  }
+
+  ~ThumbDisassembler() {
+  }
+
+  /// getInstruction - See MCDisassembler.
+  DecodeStatus getInstruction(MCInst &instr,
+                              uint64_t &size,
+                              const MemoryObject &region,
+                              uint64_t address,
+                              raw_ostream &vStream,
+                              raw_ostream &cStream) const;
+
+  /// getEDInfo - See MCDisassembler.
+  EDInstInfo *getEDInfo() const;
+private:
+  mutable std::vector<unsigned> ITBlock;
+  DecodeStatus AddThumbPredicate(MCInst&) const;
+  void UpdateThumbVFPPredicate(MCInst&) const;
+};
 }
 
-/// decodeARMInstruction is a decorator function which tries special cases of
-/// instruction matching before calling the auto-generated decoder function.
-static unsigned decodeARMInstruction(uint32_t &insn) {
-  if (slice(insn, 31, 28) == 15)
-    goto AutoGenedDecoder;
-
-  // Special case processing, if any, goes here....
-
-  // LLVM combines the offset mode of A8.6.197 & A8.6.198 into STRB.
-  // The insufficient encoding information of the combined instruction confuses
-  // the decoder wrt BFC/BFI.  Therefore, we try to recover here.
-  // For BFC, Inst{27-21} = 0b0111110 & Inst{6-0} = 0b0011111.
-  // For BFI, Inst{27-21} = 0b0111110 & Inst{6-4} = 0b001 & Inst{3-0} =! 0b1111.
-  if (slice(insn, 27, 21) == 0x3e && slice(insn, 6, 4) == 1) {
-    if (slice(insn, 3, 0) == 15)
-      return ARM::BFC;
-    else
-      return ARM::BFI;
-  }
-
-  // Ditto for STRBT, which is a super-instruction for A8.6.199 Encodings
-  // A1 & A2.
-  // As a result, the decoder fails to deocode USAT properly.
-  if (slice(insn, 27, 21) == 0x37 && slice(insn, 5, 4) == 1)
-    return ARM::USAT;
-  // As a result, the decoder fails to deocode UQADD16 properly.
-  if (slice(insn, 27, 20) == 0x66 && slice(insn, 7, 4) == 1)
-    return ARM::UQADD16;
-
-  // Ditto for ADDSrs, which is a super-instruction for A8.6.7 & A8.6.8.
-  // As a result, the decoder fails to decode UMULL properly.
-  if (slice(insn, 27, 21) == 0x04 && slice(insn, 7, 4) == 9) {
-    return ARM::UMULL;
-  }
-
-  // Ditto for STR_PRE, which is a super-instruction for A8.6.194 & A8.6.195.
-  // As a result, the decoder fails to decode SBFX properly.
-  if (slice(insn, 27, 21) == 0x3d && slice(insn, 6, 4) == 5)
-    return ARM::SBFX;
-
-  // And STRB_PRE, which is a super-instruction for A8.6.197 & A8.6.198.
-  // As a result, the decoder fails to decode UBFX properly.
-  if (slice(insn, 27, 21) == 0x3f && slice(insn, 6, 4) == 5)
-    return ARM::UBFX;
-
-  // Ditto for STRT, which is a super-instruction for A8.6.210 Encoding A1 & A2.
-  // As a result, the decoder fails to deocode SSAT properly.
-  if (slice(insn, 27, 21) == 0x35 && slice(insn, 5, 4) == 1)
-    return ARM::SSAT;
-
-  // Ditto for RSCrs, which is a super-instruction for A8.6.146 & A8.6.147.
-  // As a result, the decoder fails to decode STRHT/LDRHT/LDRSHT/LDRSBT.
-  if (slice(insn, 27, 24) == 0) {
-    switch (slice(insn, 21, 20)) {
-    case 2:
-      switch (slice(insn, 7, 4)) {
-      case 11:
-        return ARM::STRHT;
-      default:
-        break; // fallthrough
-      }
-      break;
-    case 3:
-      switch (slice(insn, 7, 4)) {
-      case 11:
-        return ARM::LDRHT;
-      case 13:
-        return ARM::LDRSBT;
-      case 15:
-        return ARM::LDRSHT;
-      default:
-        break; // fallthrough
-      }
-      break;
-    default:
-      break;   // fallthrough
-    }
-  }
-
-  // Ditto for SBCrs, which is a super-instruction for A8.6.152 & A8.6.153.
-  // As a result, the decoder fails to decode STRH_Post/LDRD_POST/STRD_POST
-  // properly.
-  if (slice(insn, 27, 25) == 0 && slice(insn, 20, 20) == 0) {
-    unsigned PW = slice(insn, 24, 24) << 1 | slice(insn, 21, 21);
-    switch (slice(insn, 7, 4)) {
-    case 11:
-      switch (PW) {
-      case 2: // Offset
-        return ARM::STRH;
-      case 3: // Pre-indexed
-        return ARM::STRH_PRE;
-      case 0: // Post-indexed
-        return ARM::STRH_POST;
-      default:
-        break; // fallthrough
-      }
-      break;
-    case 13:
-      switch (PW) {
-      case 2: // Offset
-        return ARM::LDRD;
-      case 3: // Pre-indexed
-        return ARM::LDRD_PRE;
-      case 0: // Post-indexed
-        return ARM::LDRD_POST;
-      default:
-        break; // fallthrough
-      }
-      break;
-    case 15:
-      switch (PW) {
-      case 2: // Offset
-        return ARM::STRD;
-      case 3: // Pre-indexed
-        return ARM::STRD_PRE;
-      case 0: // Post-indexed
-        return ARM::STRD_POST;
-      default:
-        break; // fallthrough
-      }
-      break;
-    default:
-      break; // fallthrough
-    }
-  }
-
-  // Ditto for SBCSSrs, which is a super-instruction for A8.6.152 & A8.6.153.
-  // As a result, the decoder fails to decode LDRH_POST/LDRSB_POST/LDRSH_POST
-  // properly.
-  if (slice(insn, 27, 25) == 0 && slice(insn, 20, 20) == 1) {
-    unsigned PW = slice(insn, 24, 24) << 1 | slice(insn, 21, 21);
-    switch (slice(insn, 7, 4)) {
-    case 11:
-      switch (PW) {
-      case 2: // Offset
-        return ARM::LDRH;
-      case 3: // Pre-indexed
-        return ARM::LDRH_PRE;
-      case 0: // Post-indexed
-        return ARM::LDRH_POST;
-      default:
-        break; // fallthrough
-      }
-      break;
-    case 13:
-      switch (PW) {
-      case 2: // Offset
-        return ARM::LDRSB;
-      case 3: // Pre-indexed
-        return ARM::LDRSB_PRE;
-      case 0: // Post-indexed
-        return ARM::LDRSB_POST;
-      default:
-        break; // fallthrough
-      }
-      break;
-    case 15:
-      switch (PW) {
-      case 2: // Offset
-        return ARM::LDRSH;
-      case 3: // Pre-indexed
-        return ARM::LDRSH_PRE;
-      case 0: // Post-indexed
-        return ARM::LDRSH_POST;
-      default:
-        break; // fallthrough
-      }
-      break;
-    default:
-      break; // fallthrough
-    }
-  }
-
-AutoGenedDecoder:
-  // Calling the auto-generated decoder function.
-  return decodeInstruction(insn);
-}
-
-// Helper function for special case handling of LDR (literal) and friends.
-// See, for example, A6.3.7 Load word: Table A6-18 Load word.
-// See A8.6.57 T3, T4 & A8.6.60 T2 and friends for why we morphed the opcode
-// before returning it.
-static unsigned T2Morph2LoadLiteral(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return Opcode; // Return unmorphed opcode.
-
-  case ARM::t2LDR_POST:   case ARM::t2LDR_PRE:
-  case ARM::t2LDRi12:     case ARM::t2LDRi8:
-  case ARM::t2LDRs:       case ARM::t2LDRT:
-    return ARM::t2LDRpci;
-
-  case ARM::t2LDRB_POST:  case ARM::t2LDRB_PRE:
-  case ARM::t2LDRBi12:    case ARM::t2LDRBi8:
-  case ARM::t2LDRBs:      case ARM::t2LDRBT:
-    return ARM::t2LDRBpci;
-
-  case ARM::t2LDRH_POST:  case ARM::t2LDRH_PRE:
-  case ARM::t2LDRHi12:    case ARM::t2LDRHi8:
-  case ARM::t2LDRHs:      case ARM::t2LDRHT:
-    return ARM::t2LDRHpci;
-
-  case ARM::t2LDRSB_POST:  case ARM::t2LDRSB_PRE:
-  case ARM::t2LDRSBi12:    case ARM::t2LDRSBi8:
-  case ARM::t2LDRSBs:      case ARM::t2LDRSBT:
-    return ARM::t2LDRSBpci;
-
-  case ARM::t2LDRSH_POST:  case ARM::t2LDRSH_PRE:
-  case ARM::t2LDRSHi12:    case ARM::t2LDRSHi8:
-  case ARM::t2LDRSHs:      case ARM::t2LDRSHT:
-    return ARM::t2LDRSHpci;
-  }
-}
-
-// Helper function for special case handling of PLD (literal) and friends.
-// See A8.6.117 T1 & T2 and friends for why we morphed the opcode
-// before returning it.
-static unsigned T2Morph2PLDLiteral(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return Opcode; // Return unmorphed opcode.
-
-  case ARM::t2PLDi8:   case ARM::t2PLDs:
-  case ARM::t2PLDWi12: case ARM::t2PLDWi8:
-  case ARM::t2PLDWs:
-    return ARM::t2PLDi12;
-
-  case ARM::t2PLIi8:   case ARM::t2PLIs:
-    return ARM::t2PLIi12;
-  }
-}
-
-/// decodeThumbSideEffect is a decorator function which can potentially twiddle
-/// the instruction or morph the returned opcode under Thumb2.
-///
-/// First it checks whether the insn is a NEON or VFP instr; if true, bit
-/// twiddling could be performed on insn to turn it into an ARM NEON/VFP
-/// equivalent instruction and decodeInstruction is called with the transformed
-/// insn.
-///
-/// Next, there is special handling for Load byte/halfword/word instruction by
-/// checking whether Rn=0b1111 and call T2Morph2LoadLiteral() on the decoded
-/// Thumb2 instruction.  See comments below for further details.
-///
-/// Finally, one last check is made to see whether the insn is a NEON/VFP and
-/// decodeInstruction(insn) is invoked on the original insn.
-///
-/// Otherwise, decodeThumbInstruction is called with the original insn.
-static unsigned decodeThumbSideEffect(bool IsThumb2, unsigned &insn) {
-  if (IsThumb2) {
-    uint16_t op1 = slice(insn, 28, 27);
-    uint16_t op2 = slice(insn, 26, 20);
-
-    // A6.3 32-bit Thumb instruction encoding
-    // Table A6-9 32-bit Thumb instruction encoding
-
-    // The coprocessor instructions of interest are transformed to their ARM
-    // equivalents.
-
-    // --------- Transform Begin Marker ---------
-    if ((op1 == 1 || op1 == 3) && slice(op2, 6, 4) == 7) {
-      // A7.4 Advanced SIMD data-processing instructions
-      // U bit of Thumb corresponds to Inst{24} of ARM.
-      uint16_t U = slice(op1, 1, 1);
-
-      // Inst{28-24} of ARM = {1,0,0,1,U};
-      uint16_t bits28_24 = 9 << 1 | U;
-      DEBUG(showBitVector(errs(), insn));
-      setSlice(insn, 28, 24, bits28_24);
-      return decodeInstruction(insn);
-    }
-
-    if (op1 == 3 && slice(op2, 6, 4) == 1 && slice(op2, 0, 0) == 0) {
-      // A7.7 Advanced SIMD element or structure load/store instructions
-      // Inst{27-24} of Thumb = 0b1001
-      // Inst{27-24} of ARM   = 0b0100
-      DEBUG(showBitVector(errs(), insn));
-      setSlice(insn, 27, 24, 4);
-      return decodeInstruction(insn);
-    }
-    // --------- Transform End Marker ---------
-
-    unsigned unmorphed = decodeThumbInstruction(insn);
-
-    // See, for example, A6.3.7 Load word: Table A6-18 Load word.
-    // See A8.6.57 T3, T4 & A8.6.60 T2 and friends for why we morphed the opcode
-    // before returning it to our caller.
-    if (op1 == 3 && slice(op2, 6, 5) == 0 && slice(op2, 0, 0) == 1
-        && slice(insn, 19, 16) == 15) {
-      unsigned morphed = T2Morph2LoadLiteral(unmorphed);
-      if (morphed != unmorphed)
-        return morphed;
-    }
-
-    // See, for example, A8.6.117 PLD,PLDW (immediate) T1 & T2, and friends for
-    // why we morphed the opcode before returning it to our caller.
-    if (slice(insn, 31, 25) == 0x7C && slice(insn, 15, 12) == 0xF
-        && slice(insn, 22, 22) == 0 && slice(insn, 20, 20) == 1
-        && slice(insn, 19, 16) == 15) {
-      unsigned morphed = T2Morph2PLDLiteral(unmorphed);
-      if (morphed != unmorphed)
-        return morphed;
-    }
-
-    // One last check for NEON/VFP instructions.
-    if ((op1 == 1 || op1 == 3) && slice(op2, 6, 6) == 1)
-      return decodeInstruction(insn);
-
-    // Fall through.
-  }
-
-  return decodeThumbInstruction(insn);
-}
-
-//
-// Public interface for the disassembler
-//
-
-bool ARMDisassembler::getInstruction(MCInst &MI,
-                                     uint64_t &Size,
-                                     const MemoryObject &Region,
-                                     uint64_t Address,
-                                     raw_ostream &os) const {
-  // The machine instruction.
-  uint32_t insn;
-  uint8_t bytes[4];
-
-  // We want to read exactly 4 bytes of data.
-  if (Region.readBytes(Address, 4, (uint8_t*)bytes, NULL) == -1)
-    return false;
-
-  // Encoded as a small-endian 32-bit word in the stream.
-  insn = (bytes[3] << 24) |
-         (bytes[2] << 16) |
-         (bytes[1] <<  8) |
-         (bytes[0] <<  0);
-
-  unsigned Opcode = decodeARMInstruction(insn);
-  ARMFormat Format = ARMFormats[Opcode];
-  Size = 4;
-
-  DEBUG({
-      errs() << "\nOpcode=" << Opcode << " Name=" <<ARMUtils::OpcodeName(Opcode)
-             << " Format=" << stringForARMFormat(Format) << '(' << (int)Format
-             << ")\n";
-      showBitVector(errs(), insn);
-    });
-
-  OwningPtr<ARMBasicMCBuilder> Builder(CreateMCBuilder(Opcode, Format));
-  if (!Builder)
-    return false;
-
-  Builder->setupBuilderForSymbolicDisassembly(getLLVMOpInfoCallback(),
-                                              getDisInfoBlock(), getMCContext(),
-                                              Address);
-
-  if (!Builder->Build(MI, insn))
-    return false;
-
-  return true;
-}
-
-bool ThumbDisassembler::getInstruction(MCInst &MI,
-                                       uint64_t &Size,
-                                       const MemoryObject &Region,
-                                       uint64_t Address,
-                                       raw_ostream &os) const {
-  // The Thumb instruction stream is a sequence of halfwords.
-
-  // This represents the first halfword as well as the machine instruction
-  // passed to decodeThumbInstruction().  For 16-bit Thumb instruction, the top
-  // halfword of insn is 0x00 0x00; otherwise, the first halfword is moved to
-  // the top half followed by the second halfword.
-  unsigned insn = 0;
-  // Possible second halfword.
-  uint16_t insn1 = 0;
-
-  // A6.1 Thumb instruction set encoding
-  //
-  // If bits [15:11] of the halfword being decoded take any of the following
-  // values, the halfword is the first halfword of a 32-bit instruction:
-  // o 0b11101
-  // o 0b11110
-  // o 0b11111.
-  //
-  // Otherwise, the halfword is a 16-bit instruction.
-
-  // Read 2 bytes of data first.
-  uint8_t bytes[2];
-  if (Region.readBytes(Address, 2, (uint8_t*)bytes, NULL) == -1)
-    return false;
-
-  // Encoded as a small-endian 16-bit halfword in the stream.
-  insn = (bytes[1] << 8) | bytes[0];
-  unsigned bits15_11 = slice(insn, 15, 11);
-  bool IsThumb2 = false;
-
-  // 32-bit instructions if the bits [15:11] of the halfword matches
-  // { 0b11101 /* 0x1D */, 0b11110 /* 0x1E */, ob11111 /* 0x1F */ }.
-  if (bits15_11 == 0x1D || bits15_11 == 0x1E || bits15_11 == 0x1F) {
-    IsThumb2 = true;
-    if (Region.readBytes(Address + 2, 2, (uint8_t*)bytes, NULL) == -1)
+static bool Check(DecodeStatus &Out, DecodeStatus In) {
+  switch (In) {
+    case MCDisassembler::Success:
+      // Out stays the same.
+      return true;
+    case MCDisassembler::SoftFail:
+      Out = In;
+      return true;
+    case MCDisassembler::Fail:
+      Out = In;
       return false;
-    // Encoded as a small-endian 16-bit halfword in the stream.
-    insn1 = (bytes[1] << 8) | bytes[0];
-    insn = (insn << 16 | insn1);
   }
-
-  // The insn could potentially be bit-twiddled in order to be decoded as an ARM
-  // NEON/VFP opcode.  In such case, the modified insn is later disassembled as
-  // an ARM NEON/VFP instruction.
-  //
-  // This is a short term solution for lack of encoding bits specified for the
-  // Thumb2 NEON/VFP instructions.  The long term solution could be adding some
-  // infrastructure to have each instruction support more than one encodings.
-  // Which encoding is used would be based on which subtarget the compiler/
-  // disassembler is working with at the time.  This would allow the sharing of
-  // the NEON patterns between ARM and Thumb2, as well as potential greater
-  // sharing between the regular ARM instructions and the 32-bit wide Thumb2
-  // instructions as well.
-  unsigned Opcode = decodeThumbSideEffect(IsThumb2, insn);
-
-  ARMFormat Format = ARMFormats[Opcode];
-  Size = IsThumb2 ? 4 : 2;
-
-  DEBUG({
-      errs() << "Opcode=" << Opcode << " Name=" << ARMUtils::OpcodeName(Opcode)
-             << " Format=" << stringForARMFormat(Format) << '(' << (int)Format
-             << ")\n";
-      showBitVector(errs(), insn);
-    });
-
-  OwningPtr<ARMBasicMCBuilder> Builder(CreateMCBuilder(Opcode, Format));
-  if (!Builder)
-    return false;
-
-  Builder->SetSession(const_cast<Session *>(&SO));
-
-  Builder->setupBuilderForSymbolicDisassembly(getLLVMOpInfoCallback(),
-                                              getDisInfoBlock(), getMCContext(),
-                                              Address);
-
-  if (!Builder->Build(MI, insn))
-    return false;
-
-  return true;
+  return false;
 }
 
-// A8.6.50
-// Valid return values are {1, 2, 3, 4}, with 0 signifying an error condition.
-static unsigned short CountITSize(unsigned ITMask) {
-  // First count the trailing zeros of the IT mask.
-  unsigned TZ = CountTrailingZeros_32(ITMask);
-  if (TZ > 3) {
-    DEBUG(errs() << "Encoding error: IT Mask '0000'");
-    return 0;
-  }
-  return (4 - TZ);
+
+// Forward declare these because the autogenerated code will reference them.
+// Definitions are further down.
+static DecodeStatus DecodeGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeGPRnopcRegisterClass(llvm::MCInst &Inst,
+                                               unsigned RegNo, uint64_t Address,
+                                               const void *Decoder);
+static DecodeStatus DecodetGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecodetcGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecoderGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeDPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeDPR_8RegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeDPR_VFP2RegisterClass(llvm::MCInst &Inst,
+                                                unsigned RegNo,
+                                                uint64_t Address,
+                                                const void *Decoder);
+static DecodeStatus DecodeQPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder);
+
+static DecodeStatus DecodePredicateOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeCCOutOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSOImmOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeRegListOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSPRRegListOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeDPRRegListOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+
+static DecodeStatus DecodeBitfieldMaskOperand(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeCopMemInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeAddrMode2IdxInstruction(llvm::MCInst &Inst,
+                                                  unsigned Insn,
+                                                  uint64_t Address,
+                                                  const void *Decoder);
+static DecodeStatus DecodeSORegMemOperand(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeAddrMode3Instruction(llvm::MCInst &Inst,unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSORegImmOperand(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSORegRegOperand(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+
+static DecodeStatus DecodeMemMultipleWritebackInstruction(llvm::MCInst & Inst,
+                                                  unsigned Insn,
+                                                  uint64_t Adddress,
+                                                  const void *Decoder);
+static DecodeStatus DecodeT2MOVTWInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeArmMOVTWInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSMLAInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeCPSInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2CPSInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeAddrModeImm12Operand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeAddrMode5Operand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeAddrMode7Operand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeBranchImmInstruction(llvm::MCInst &Inst,unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVCVTImmOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeAddrMode6Operand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLDInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVSTInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD1DupInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD2DupInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD3DupInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD4DupInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeNEONModImmInstruction(llvm::MCInst &Inst,unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVSHLMaxInstruction(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeShiftRight8Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeShiftRight16Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeShiftRight32Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeShiftRight64Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeTBLInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodePostIdxReg(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeCoprocessor(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeMemBarrierOption(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeMSRMask(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeDoubleRegLoad(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeDoubleRegStore(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeLDRPreImm(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeLDRPreReg(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSTRPreImm(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeSTRPreReg(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD1LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD2LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD3LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVLD4LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVST1LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVST2LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVST3LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVST4LN(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVMOVSRR(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeVMOVRRS(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+
+static DecodeStatus DecodeThumbAddSpecialReg(llvm::MCInst &Inst, uint16_t Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbBROperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2BROperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbCmpBROperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbAddrModeRR(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbAddrModeIS(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbAddrModePC(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbAddrModeSP(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2AddrModeSOReg(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2LoadShift(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2Imm8S4(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2AddrModeImm8s4(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2AddrModeImm0_1020s4(llvm::MCInst &Inst,unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2Imm8(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2AddrModeImm8(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbAddSPImm(llvm::MCInst &Inst, uint16_t Val,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbAddSPReg(llvm::MCInst &Inst, uint16_t Insn,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbCPS(llvm::MCInst &Inst, uint16_t Insn,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbBLXOffset(llvm::MCInst &Inst, unsigned Insn,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2AddrModeImm12(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbTableBranch(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumb2BCCInstruction(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2SOImm(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbBCCTargetOperand(llvm::MCInst &Inst,unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeThumbBLTargetOperand(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeIT(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2LDRDPreInstruction(llvm::MCInst &Inst,unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2STRDPreInstruction(llvm::MCInst &Inst,unsigned Insn,
+                               uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2Adr(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2LdStPre(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+static DecodeStatus DecodeT2ShifterImmOperand(llvm::MCInst &Inst, unsigned Val,
+                                uint64_t Address, const void *Decoder);
+
+
+
+#include "ARMGenDisassemblerTables.inc"
+#include "ARMGenInstrInfo.inc"
+#include "ARMGenEDInfo.inc"
+
+static MCDisassembler *createARMDisassembler(const Target &T, const MCSubtargetInfo &STI) {
+  return new ARMDisassembler(STI);
 }
 
-/// Init ITState.  Note that at least one bit is always 1 in mask.
-bool Session::InitIT(unsigned short bits7_0) {
-  ITCounter = CountITSize(slice(bits7_0, 3, 0));
-  if (ITCounter == 0)
-    return false;
-
-  // A8.6.50 IT
-  unsigned short FirstCond = slice(bits7_0, 7, 4);
-  if (FirstCond == 0xF) {
-    DEBUG(errs() << "Encoding error: IT FirstCond '1111'");
-    return false;
-  }
-  if (FirstCond == 0xE && ITCounter != 1) {
-    DEBUG(errs() << "Encoding error: IT FirstCond '1110' && Mask != '1000'");
-    return false;
-  }
-
-  ITState = bits7_0;
-
-  return true;
-}
-
-/// Update ITState if necessary.
-void Session::UpdateIT() {
-  assert(ITCounter);
-  --ITCounter;
-  if (ITCounter == 0)
-    ITState = 0;
-  else {
-    unsigned short NewITState4_0 = slice(ITState, 4, 0) << 1;
-    setSlice(ITState, 4, 0, NewITState4_0);
-  }
-}
-
-static MCDisassembler *createARMDisassembler(const Target &T) {
-  return new ARMDisassembler;
-}
-
-static MCDisassembler *createThumbDisassembler(const Target &T) {
-  return new ThumbDisassembler;
-}
-
-extern "C" void LLVMInitializeARMDisassembler() {
-  // Register the disassembler.
-  TargetRegistry::RegisterMCDisassembler(TheARMTarget,
-                                         createARMDisassembler);
-  TargetRegistry::RegisterMCDisassembler(TheThumbTarget,
-                                         createThumbDisassembler);
+static MCDisassembler *createThumbDisassembler(const Target &T, const MCSubtargetInfo &STI) {
+  return new ThumbDisassembler(STI);
 }
 
 EDInstInfo *ARMDisassembler::getEDInfo() const {
@@ -587,3 +334,3749 @@ EDInstInfo *ARMDisassembler::getEDInfo() const {
 EDInstInfo *ThumbDisassembler::getEDInfo() const {
   return instInfoARM;
 }
+
+DecodeStatus ARMDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
+                                             const MemoryObject &Region,
+                                             uint64_t Address,
+                                             raw_ostream &os,
+                                             raw_ostream &cs) const {
+  CommentStream = &cs;
+
+  uint8_t bytes[4];
+
+  assert(!(STI.getFeatureBits() & ARM::ModeThumb) &&
+         "Asked to disassemble an ARM instruction but Subtarget is in Thumb mode!");
+
+  // We want to read exactly 4 bytes of data.
+  if (Region.readBytes(Address, 4, (uint8_t*)bytes, NULL) == -1) {
+    Size = 0;
+    return MCDisassembler::Fail;
+  }
+
+  // Encoded as a small-endian 32-bit word in the stream.
+  uint32_t insn = (bytes[3] << 24) |
+                  (bytes[2] << 16) |
+                  (bytes[1] <<  8) |
+                  (bytes[0] <<  0);
+
+  // Calling the auto-generated decoder function.
+  DecodeStatus result = decodeARMInstruction32(MI, insn, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    return result;
+  }
+
+  // VFP and NEON instructions, similarly, are shared between ARM
+  // and Thumb modes.
+  MI.clear();
+  result = decodeVFPInstruction32(MI, insn, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    return result;
+  }
+
+  MI.clear();
+  result = decodeNEONDataInstruction32(MI, insn, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    // Add a fake predicate operand, because we share these instruction
+    // definitions with Thumb2 where these instructions are predicable.
+    if (!DecodePredicateOperand(MI, 0xE, Address, this))
+      return MCDisassembler::Fail;
+    return result;
+  }
+
+  MI.clear();
+  result = decodeNEONLoadStoreInstruction32(MI, insn, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    // Add a fake predicate operand, because we share these instruction
+    // definitions with Thumb2 where these instructions are predicable.
+    if (!DecodePredicateOperand(MI, 0xE, Address, this))
+      return MCDisassembler::Fail;
+    return result;
+  }
+
+  MI.clear();
+  result = decodeNEONDupInstruction32(MI, insn, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    // Add a fake predicate operand, because we share these instruction
+    // definitions with Thumb2 where these instructions are predicable.
+    if (!DecodePredicateOperand(MI, 0xE, Address, this))
+      return MCDisassembler::Fail;
+    return result;
+  }
+
+  MI.clear();
+
+  Size = 0;
+  return MCDisassembler::Fail;
+}
+
+namespace llvm {
+extern MCInstrDesc ARMInsts[];
+}
+
+/// tryAddingSymbolicOperand - trys to add a symbolic operand in place of the
+/// immediate Value in the MCInst.  The immediate Value has had any PC
+/// adjustment made by the caller.  If the instruction is a branch instruction
+/// then isBranch is true, else false.  If the getOpInfo() function was set as
+/// part of the setupForSymbolicDisassembly() call then that function is called
+/// to get any symbolic information at the Address for this instruction.  If
+/// that returns non-zero then the symbolic information it returns is used to
+/// create an MCExpr and that is added as an operand to the MCInst.  If
+/// getOpInfo() returns zero and isBranch is true then a symbol look up for
+/// Value is done and if a symbol is found an MCExpr is created with that, else
+/// an MCExpr with Value is created.  This function returns true if it adds an
+/// operand to the MCInst and false otherwise.
+static bool tryAddingSymbolicOperand(uint64_t Address, int32_t Value,
+                                     bool isBranch, uint64_t InstSize,
+                                     MCInst &MI, const void *Decoder) {
+  const MCDisassembler *Dis = static_cast<const MCDisassembler*>(Decoder);
+  LLVMOpInfoCallback getOpInfo = Dis->getLLVMOpInfoCallback();
+  if (!getOpInfo)
+    return false;
+
+  struct LLVMOpInfo1 SymbolicOp;
+  SymbolicOp.Value = Value;
+  void *DisInfo = Dis->getDisInfoBlock();
+  if (!getOpInfo(DisInfo, Address, 0 /* Offset */, InstSize, 1, &SymbolicOp)) {
+    if (isBranch) {
+      LLVMSymbolLookupCallback SymbolLookUp =
+                                            Dis->getLLVMSymbolLookupCallback();
+      if (SymbolLookUp) {
+        uint64_t ReferenceType;
+        ReferenceType = LLVMDisassembler_ReferenceType_In_Branch;
+        const char *ReferenceName;
+        const char *Name = SymbolLookUp(DisInfo, Value, &ReferenceType, Address,
+                                        &ReferenceName);
+        if (Name) {
+          SymbolicOp.AddSymbol.Name = Name;
+          SymbolicOp.AddSymbol.Present = true;
+          SymbolicOp.Value = 0;
+        }
+        else {
+          SymbolicOp.Value = Value;
+        }
+        if(ReferenceType == LLVMDisassembler_ReferenceType_Out_SymbolStub)
+          (*Dis->CommentStream) << "symbol stub for: " << ReferenceName;
+      }
+      else {
+        return false;
+      }
+    }
+    else {
+      return false;
+    }
+  }
+
+  MCContext *Ctx = Dis->getMCContext();
+  const MCExpr *Add = NULL;
+  if (SymbolicOp.AddSymbol.Present) {
+    if (SymbolicOp.AddSymbol.Name) {
+      StringRef Name(SymbolicOp.AddSymbol.Name);
+      MCSymbol *Sym = Ctx->GetOrCreateSymbol(Name);
+      Add = MCSymbolRefExpr::Create(Sym, *Ctx);
+    } else {
+      Add = MCConstantExpr::Create(SymbolicOp.AddSymbol.Value, *Ctx);
+    }
+  }
+
+  const MCExpr *Sub = NULL;
+  if (SymbolicOp.SubtractSymbol.Present) {
+    if (SymbolicOp.SubtractSymbol.Name) {
+      StringRef Name(SymbolicOp.SubtractSymbol.Name);
+      MCSymbol *Sym = Ctx->GetOrCreateSymbol(Name);
+      Sub = MCSymbolRefExpr::Create(Sym, *Ctx);
+    } else {
+      Sub = MCConstantExpr::Create(SymbolicOp.SubtractSymbol.Value, *Ctx);
+    }
+  }
+
+  const MCExpr *Off = NULL;
+  if (SymbolicOp.Value != 0)
+    Off = MCConstantExpr::Create(SymbolicOp.Value, *Ctx);
+
+  const MCExpr *Expr;
+  if (Sub) {
+    const MCExpr *LHS;
+    if (Add)
+      LHS = MCBinaryExpr::CreateSub(Add, Sub, *Ctx);
+    else
+      LHS = MCUnaryExpr::CreateMinus(Sub, *Ctx);
+    if (Off != 0)
+      Expr = MCBinaryExpr::CreateAdd(LHS, Off, *Ctx);
+    else
+      Expr = LHS;
+  } else if (Add) {
+    if (Off != 0)
+      Expr = MCBinaryExpr::CreateAdd(Add, Off, *Ctx);
+    else
+      Expr = Add;
+  } else {
+    if (Off != 0)
+      Expr = Off;
+    else
+      Expr = MCConstantExpr::Create(0, *Ctx);
+  }
+
+  if (SymbolicOp.VariantKind == LLVMDisassembler_VariantKind_ARM_HI16)
+    MI.addOperand(MCOperand::CreateExpr(ARMMCExpr::CreateUpper16(Expr, *Ctx)));
+  else if (SymbolicOp.VariantKind == LLVMDisassembler_VariantKind_ARM_LO16)
+    MI.addOperand(MCOperand::CreateExpr(ARMMCExpr::CreateLower16(Expr, *Ctx)));
+  else if (SymbolicOp.VariantKind == LLVMDisassembler_VariantKind_None)
+    MI.addOperand(MCOperand::CreateExpr(Expr));
+  else 
+    assert(0 && "bad SymbolicOp.VariantKind");
+
+  return true;
+}
+
+/// tryAddingPcLoadReferenceComment - trys to add a comment as to what is being
+/// referenced by a load instruction with the base register that is the Pc.
+/// These can often be values in a literal pool near the Address of the
+/// instruction.  The Address of the instruction and its immediate Value are
+/// used as a possible literal pool entry.  The SymbolLookUp call back will
+/// return the name of a symbol referenced by the the literal pool's entry if
+/// the referenced address is that of a symbol.  Or it will return a pointer to
+/// a literal 'C' string if the referenced address of the literal pool's entry
+/// is an address into a section with 'C' string literals.
+static void tryAddingPcLoadReferenceComment(uint64_t Address, int Value,
+					    const void *Decoder) {
+  const MCDisassembler *Dis = static_cast<const MCDisassembler*>(Decoder);
+  LLVMSymbolLookupCallback SymbolLookUp = Dis->getLLVMSymbolLookupCallback();
+  if (SymbolLookUp) {
+    void *DisInfo = Dis->getDisInfoBlock();
+    uint64_t ReferenceType;
+    ReferenceType = LLVMDisassembler_ReferenceType_In_PCrel_Load;
+    const char *ReferenceName;
+    (void)SymbolLookUp(DisInfo, Value, &ReferenceType, Address, &ReferenceName);
+    if(ReferenceType == LLVMDisassembler_ReferenceType_Out_LitPool_SymAddr ||
+       ReferenceType == LLVMDisassembler_ReferenceType_Out_LitPool_CstrAddr)
+      (*Dis->CommentStream) << "literal pool for: " << ReferenceName;
+  }
+}
+
+// Thumb1 instructions don't have explicit S bits.  Rather, they
+// implicitly set CPSR.  Since it's not represented in the encoding, the
+// auto-generated decoder won't inject the CPSR operand.  We need to fix
+// that as a post-pass.
+static void AddThumb1SBit(MCInst &MI, bool InITBlock) {
+  const MCOperandInfo *OpInfo = ARMInsts[MI.getOpcode()].OpInfo;
+  unsigned short NumOps = ARMInsts[MI.getOpcode()].NumOperands;
+  MCInst::iterator I = MI.begin();
+  for (unsigned i = 0; i < NumOps; ++i, ++I) {
+    if (I == MI.end()) break;
+    if (OpInfo[i].isOptionalDef() && OpInfo[i].RegClass == ARM::CCRRegClassID) {
+      if (i > 0 && OpInfo[i-1].isPredicate()) continue;
+      MI.insert(I, MCOperand::CreateReg(InITBlock ? 0 : ARM::CPSR));
+      return;
+    }
+  }
+
+  MI.insert(I, MCOperand::CreateReg(InITBlock ? 0 : ARM::CPSR));
+}
+
+// Most Thumb instructions don't have explicit predicates in the
+// encoding, but rather get their predicates from IT context.  We need
+// to fix up the predicate operands using this context information as a
+// post-pass.
+MCDisassembler::DecodeStatus
+ThumbDisassembler::AddThumbPredicate(MCInst &MI) const {
+  MCDisassembler::DecodeStatus S = Success;
+
+  // A few instructions actually have predicates encoded in them.  Don't
+  // try to overwrite it if we're seeing one of those.
+  switch (MI.getOpcode()) {
+    case ARM::tBcc:
+    case ARM::t2Bcc:
+    case ARM::tCBZ:
+    case ARM::tCBNZ:
+    case ARM::tCPS:
+    case ARM::t2CPS3p:
+    case ARM::t2CPS2p:
+    case ARM::t2CPS1p:
+    case ARM::tMOVSr:
+    case ARM::tSETEND:
+      // Some instructions (mostly conditional branches) are not
+      // allowed in IT blocks.
+      if (!ITBlock.empty())
+        S = SoftFail;
+      else
+        return Success;
+      break;
+    case ARM::tB:
+    case ARM::t2B:
+    case ARM::t2TBB:
+    case ARM::t2TBH:
+      // Some instructions (mostly unconditional branches) can
+      // only appears at the end of, or outside of, an IT.
+      if (ITBlock.size() > 1)
+        S = SoftFail;
+      break;
+    default:
+      break;
+  }
+
+  // If we're in an IT block, base the predicate on that.  Otherwise,
+  // assume a predicate of AL.
+  unsigned CC;
+  if (!ITBlock.empty()) {
+    CC = ITBlock.back();
+    if (CC == 0xF)
+      CC = ARMCC::AL;
+    ITBlock.pop_back();
+  } else
+    CC = ARMCC::AL;
+
+  const MCOperandInfo *OpInfo = ARMInsts[MI.getOpcode()].OpInfo;
+  unsigned short NumOps = ARMInsts[MI.getOpcode()].NumOperands;
+  MCInst::iterator I = MI.begin();
+  for (unsigned i = 0; i < NumOps; ++i, ++I) {
+    if (I == MI.end()) break;
+    if (OpInfo[i].isPredicate()) {
+      I = MI.insert(I, MCOperand::CreateImm(CC));
+      ++I;
+      if (CC == ARMCC::AL)
+        MI.insert(I, MCOperand::CreateReg(0));
+      else
+        MI.insert(I, MCOperand::CreateReg(ARM::CPSR));
+      return S;
+    }
+  }
+
+  I = MI.insert(I, MCOperand::CreateImm(CC));
+  ++I;
+  if (CC == ARMCC::AL)
+    MI.insert(I, MCOperand::CreateReg(0));
+  else
+    MI.insert(I, MCOperand::CreateReg(ARM::CPSR));
+
+  return S;
+}
+
+// Thumb VFP instructions are a special case.  Because we share their
+// encodings between ARM and Thumb modes, and they are predicable in ARM
+// mode, the auto-generated decoder will give them an (incorrect)
+// predicate operand.  We need to rewrite these operands based on the IT
+// context as a post-pass.
+void ThumbDisassembler::UpdateThumbVFPPredicate(MCInst &MI) const {
+  unsigned CC;
+  if (!ITBlock.empty()) {
+    CC = ITBlock.back();
+    ITBlock.pop_back();
+  } else
+    CC = ARMCC::AL;
+
+  const MCOperandInfo *OpInfo = ARMInsts[MI.getOpcode()].OpInfo;
+  MCInst::iterator I = MI.begin();
+  unsigned short NumOps = ARMInsts[MI.getOpcode()].NumOperands;
+  for (unsigned i = 0; i < NumOps; ++i, ++I) {
+    if (OpInfo[i].isPredicate() ) {
+      I->setImm(CC);
+      ++I;
+      if (CC == ARMCC::AL)
+        I->setReg(0);
+      else
+        I->setReg(ARM::CPSR);
+      return;
+    }
+  }
+}
+
+DecodeStatus ThumbDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
+                                               const MemoryObject &Region,
+                                               uint64_t Address,
+                                               raw_ostream &os,
+                                               raw_ostream &cs) const {
+  CommentStream = &cs;
+
+  uint8_t bytes[4];
+
+  assert((STI.getFeatureBits() & ARM::ModeThumb) &&
+         "Asked to disassemble in Thumb mode but Subtarget is in ARM mode!");
+
+  // We want to read exactly 2 bytes of data.
+  if (Region.readBytes(Address, 2, (uint8_t*)bytes, NULL) == -1) {
+    Size = 0;
+    return MCDisassembler::Fail;
+  }
+
+  uint16_t insn16 = (bytes[1] << 8) | bytes[0];
+  DecodeStatus result = decodeThumbInstruction16(MI, insn16, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 2;
+    Check(result, AddThumbPredicate(MI));
+    return result;
+  }
+
+  MI.clear();
+  result = decodeThumbSBitInstruction16(MI, insn16, Address, this, STI);
+  if (result) {
+    Size = 2;
+    bool InITBlock = !ITBlock.empty();
+    Check(result, AddThumbPredicate(MI));
+    AddThumb1SBit(MI, InITBlock);
+    return result;
+  }
+
+  MI.clear();
+  result = decodeThumb2Instruction16(MI, insn16, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 2;
+
+    // Nested IT blocks are UNPREDICTABLE.  Must be checked before we add
+    // the Thumb predicate.
+    if (MI.getOpcode() == ARM::t2IT && !ITBlock.empty())
+      result = MCDisassembler::SoftFail;
+
+    Check(result, AddThumbPredicate(MI));
+
+    // If we find an IT instruction, we need to parse its condition
+    // code and mask operands so that we can apply them correctly
+    // to the subsequent instructions.
+    if (MI.getOpcode() == ARM::t2IT) {
+
+      // (3 - the number of trailing zeros) is the number of then / else.
+      unsigned firstcond = MI.getOperand(0).getImm();
+      unsigned Mask = MI.getOperand(1).getImm();
+      unsigned CondBit0 = Mask >> 4 & 1;
+      unsigned NumTZ = CountTrailingZeros_32(Mask);
+      assert(NumTZ <= 3 && "Invalid IT mask!");
+      for (unsigned Pos = 3, e = NumTZ; Pos > e; --Pos) {
+        bool T = ((Mask >> Pos) & 1) == CondBit0;
+        if (T)
+          ITBlock.insert(ITBlock.begin(), firstcond);
+        else
+          ITBlock.insert(ITBlock.begin(), firstcond ^ 1);
+      }
+
+      ITBlock.push_back(firstcond);
+    }
+
+    return result;
+  }
+
+  // We want to read exactly 4 bytes of data.
+  if (Region.readBytes(Address, 4, (uint8_t*)bytes, NULL) == -1) {
+    Size = 0;
+    return MCDisassembler::Fail;
+  }
+
+  uint32_t insn32 = (bytes[3] <<  8) |
+                    (bytes[2] <<  0) |
+                    (bytes[1] << 24) |
+                    (bytes[0] << 16);
+  MI.clear();
+  result = decodeThumbInstruction32(MI, insn32, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    bool InITBlock = ITBlock.size();
+    Check(result, AddThumbPredicate(MI));
+    AddThumb1SBit(MI, InITBlock);
+    return result;
+  }
+
+  MI.clear();
+  result = decodeThumb2Instruction32(MI, insn32, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    Check(result, AddThumbPredicate(MI));
+    return result;
+  }
+
+  MI.clear();
+  result = decodeVFPInstruction32(MI, insn32, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    UpdateThumbVFPPredicate(MI);
+    return result;
+  }
+
+  MI.clear();
+  result = decodeNEONDupInstruction32(MI, insn32, Address, this, STI);
+  if (result != MCDisassembler::Fail) {
+    Size = 4;
+    Check(result, AddThumbPredicate(MI));
+    return result;
+  }
+
+  if (fieldFromInstruction32(insn32, 24, 8) == 0xF9) {
+    MI.clear();
+    uint32_t NEONLdStInsn = insn32;
+    NEONLdStInsn &= 0xF0FFFFFF;
+    NEONLdStInsn |= 0x04000000;
+    result = decodeNEONLoadStoreInstruction32(MI, NEONLdStInsn, Address, this, STI);
+    if (result != MCDisassembler::Fail) {
+      Size = 4;
+      Check(result, AddThumbPredicate(MI));
+      return result;
+    }
+  }
+
+  if (fieldFromInstruction32(insn32, 24, 4) == 0xF) {
+    MI.clear();
+    uint32_t NEONDataInsn = insn32;
+    NEONDataInsn &= 0xF0FFFFFF; // Clear bits 27-24
+    NEONDataInsn |= (NEONDataInsn & 0x10000000) >> 4; // Move bit 28 to bit 24
+    NEONDataInsn |= 0x12000000; // Set bits 28 and 25
+    result = decodeNEONDataInstruction32(MI, NEONDataInsn, Address, this, STI);
+    if (result != MCDisassembler::Fail) {
+      Size = 4;
+      Check(result, AddThumbPredicate(MI));
+      return result;
+    }
+  }
+
+  Size = 0;
+  return MCDisassembler::Fail;
+}
+
+
+extern "C" void LLVMInitializeARMDisassembler() {
+  TargetRegistry::RegisterMCDisassembler(TheARMTarget,
+                                         createARMDisassembler);
+  TargetRegistry::RegisterMCDisassembler(TheThumbTarget,
+                                         createThumbDisassembler);
+}
+
+static const unsigned GPRDecoderTable[] = {
+  ARM::R0, ARM::R1, ARM::R2, ARM::R3,
+  ARM::R4, ARM::R5, ARM::R6, ARM::R7,
+  ARM::R8, ARM::R9, ARM::R10, ARM::R11,
+  ARM::R12, ARM::SP, ARM::LR, ARM::PC
+};
+
+static DecodeStatus DecodeGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo > 15)
+    return MCDisassembler::Fail;
+
+  unsigned Register = GPRDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::CreateReg(Register));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus
+DecodeGPRnopcRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                           uint64_t Address, const void *Decoder) {
+  if (RegNo == 15) return MCDisassembler::Fail;
+  return DecodeGPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
+static DecodeStatus DecodetGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo > 7)
+    return MCDisassembler::Fail;
+  return DecodeGPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
+static DecodeStatus DecodetcGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  unsigned Register = 0;
+  switch (RegNo) {
+    case 0:
+      Register = ARM::R0;
+      break;
+    case 1:
+      Register = ARM::R1;
+      break;
+    case 2:
+      Register = ARM::R2;
+      break;
+    case 3:
+      Register = ARM::R3;
+      break;
+    case 9:
+      Register = ARM::R9;
+      break;
+    case 12:
+      Register = ARM::R12;
+      break;
+    default:
+      return MCDisassembler::Fail;
+    }
+
+  Inst.addOperand(MCOperand::CreateReg(Register));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecoderGPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo == 13 || RegNo == 15) return MCDisassembler::Fail;
+  return DecodeGPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
+static const unsigned SPRDecoderTable[] = {
+     ARM::S0,  ARM::S1,  ARM::S2,  ARM::S3,
+     ARM::S4,  ARM::S5,  ARM::S6,  ARM::S7,
+     ARM::S8,  ARM::S9, ARM::S10, ARM::S11,
+    ARM::S12, ARM::S13, ARM::S14, ARM::S15,
+    ARM::S16, ARM::S17, ARM::S18, ARM::S19,
+    ARM::S20, ARM::S21, ARM::S22, ARM::S23,
+    ARM::S24, ARM::S25, ARM::S26, ARM::S27,
+    ARM::S28, ARM::S29, ARM::S30, ARM::S31
+};
+
+static DecodeStatus DecodeSPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo > 31)
+    return MCDisassembler::Fail;
+
+  unsigned Register = SPRDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::CreateReg(Register));
+  return MCDisassembler::Success;
+}
+
+static const unsigned DPRDecoderTable[] = {
+     ARM::D0,  ARM::D1,  ARM::D2,  ARM::D3,
+     ARM::D4,  ARM::D5,  ARM::D6,  ARM::D7,
+     ARM::D8,  ARM::D9, ARM::D10, ARM::D11,
+    ARM::D12, ARM::D13, ARM::D14, ARM::D15,
+    ARM::D16, ARM::D17, ARM::D18, ARM::D19,
+    ARM::D20, ARM::D21, ARM::D22, ARM::D23,
+    ARM::D24, ARM::D25, ARM::D26, ARM::D27,
+    ARM::D28, ARM::D29, ARM::D30, ARM::D31
+};
+
+static DecodeStatus DecodeDPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo > 31)
+    return MCDisassembler::Fail;
+
+  unsigned Register = DPRDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::CreateReg(Register));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeDPR_8RegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo > 7)
+    return MCDisassembler::Fail;
+  return DecodeDPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
+static DecodeStatus
+DecodeDPR_VFP2RegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                            uint64_t Address, const void *Decoder) {
+  if (RegNo > 15)
+    return MCDisassembler::Fail;
+  return DecodeDPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
+static const unsigned QPRDecoderTable[] = {
+     ARM::Q0,  ARM::Q1,  ARM::Q2,  ARM::Q3,
+     ARM::Q4,  ARM::Q5,  ARM::Q6,  ARM::Q7,
+     ARM::Q8,  ARM::Q9, ARM::Q10, ARM::Q11,
+    ARM::Q12, ARM::Q13, ARM::Q14, ARM::Q15
+};
+
+
+static DecodeStatus DecodeQPRRegisterClass(llvm::MCInst &Inst, unsigned RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  if (RegNo > 31)
+    return MCDisassembler::Fail;
+  RegNo >>= 1;
+
+  unsigned Register = QPRDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::CreateReg(Register));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodePredicateOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  if (Val == 0xF) return MCDisassembler::Fail;
+  // AL predicate is not allowed on Thumb1 branches.
+  if (Inst.getOpcode() == ARM::tBcc && Val == 0xE)
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(Val));
+  if (Val == ARMCC::AL) {
+    Inst.addOperand(MCOperand::CreateReg(0));
+  } else
+    Inst.addOperand(MCOperand::CreateReg(ARM::CPSR));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeCCOutOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  if (Val)
+    Inst.addOperand(MCOperand::CreateReg(ARM::CPSR));
+  else
+    Inst.addOperand(MCOperand::CreateReg(0));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeSOImmOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  uint32_t imm = Val & 0xFF;
+  uint32_t rot = (Val & 0xF00) >> 7;
+  uint32_t rot_imm = (imm >> rot) | (imm << ((32-rot) & 0x1F));
+  Inst.addOperand(MCOperand::CreateImm(rot_imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeSORegImmOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rm = fieldFromInstruction32(Val, 0, 4);
+  unsigned type = fieldFromInstruction32(Val, 5, 2);
+  unsigned imm = fieldFromInstruction32(Val, 7, 5);
+
+  // Register-immediate
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  ARM_AM::ShiftOpc Shift = ARM_AM::lsl;
+  switch (type) {
+    case 0:
+      Shift = ARM_AM::lsl;
+      break;
+    case 1:
+      Shift = ARM_AM::lsr;
+      break;
+    case 2:
+      Shift = ARM_AM::asr;
+      break;
+    case 3:
+      Shift = ARM_AM::ror;
+      break;
+  }
+
+  if (Shift == ARM_AM::ror && imm == 0)
+    Shift = ARM_AM::rrx;
+
+  unsigned Op = Shift | (imm << 3);
+  Inst.addOperand(MCOperand::CreateImm(Op));
+
+  return S;
+}
+
+static DecodeStatus DecodeSORegRegOperand(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rm = fieldFromInstruction32(Val, 0, 4);
+  unsigned type = fieldFromInstruction32(Val, 5, 2);
+  unsigned Rs = fieldFromInstruction32(Val, 8, 4);
+
+  // Register-register
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rs, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  ARM_AM::ShiftOpc Shift = ARM_AM::lsl;
+  switch (type) {
+    case 0:
+      Shift = ARM_AM::lsl;
+      break;
+    case 1:
+      Shift = ARM_AM::lsr;
+      break;
+    case 2:
+      Shift = ARM_AM::asr;
+      break;
+    case 3:
+      Shift = ARM_AM::ror;
+      break;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(Shift));
+
+  return S;
+}
+
+static DecodeStatus DecodeRegListOperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  bool writebackLoad = false;
+  unsigned writebackReg = 0;
+  switch (Inst.getOpcode()) {
+    default:
+      break;
+    case ARM::LDMIA_UPD:
+    case ARM::LDMDB_UPD:
+    case ARM::LDMIB_UPD:
+    case ARM::LDMDA_UPD:
+    case ARM::t2LDMIA_UPD:
+    case ARM::t2LDMDB_UPD:
+      writebackLoad = true;
+      writebackReg = Inst.getOperand(0).getReg();
+      break;
+  }
+
+  // Empty register lists are not allowed.
+  if (CountPopulation_32(Val) == 0) return MCDisassembler::Fail;
+  for (unsigned i = 0; i < 16; ++i) {
+    if (Val & (1 << i)) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, i, Address, Decoder)))
+        return MCDisassembler::Fail;
+      // Writeback not allowed if Rn is in the target list.
+      if (writebackLoad && writebackReg == Inst.end()[-1].getReg())
+        Check(S, MCDisassembler::SoftFail);
+    }
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeSPRRegListOperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Vd = fieldFromInstruction32(Val, 8, 4);
+  unsigned regs = Val & 0xFF;
+
+  if (!Check(S, DecodeSPRRegisterClass(Inst, Vd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  for (unsigned i = 0; i < (regs - 1); ++i) {
+    if (!Check(S, DecodeSPRRegisterClass(Inst, ++Vd, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeDPRRegListOperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Vd = fieldFromInstruction32(Val, 8, 4);
+  unsigned regs = (Val & 0xFF) / 2;
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Vd, Address, Decoder)))
+      return MCDisassembler::Fail;
+  for (unsigned i = 0; i < (regs - 1); ++i) {
+    if (!Check(S, DecodeDPRRegisterClass(Inst, ++Vd, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeBitfieldMaskOperand(llvm::MCInst &Inst, unsigned Val,
+                                      uint64_t Address, const void *Decoder) {
+  // This operand encodes a mask of contiguous zeros between a specified MSB
+  // and LSB.  To decode it, we create the mask of all bits MSB-and-lower,
+  // the mask of all bits LSB-and-lower, and then xor them to create
+  // the mask of that's all ones on [msb, lsb].  Finally we not it to
+  // create the final mask.
+  unsigned msb = fieldFromInstruction32(Val, 5, 5);
+  unsigned lsb = fieldFromInstruction32(Val, 0, 5);
+
+  DecodeStatus S = MCDisassembler::Success;
+  if (lsb > msb) Check(S, MCDisassembler::SoftFail);
+
+  uint32_t msb_mask = 0xFFFFFFFF;
+  if (msb != 31) msb_mask = (1U << (msb+1)) - 1;
+  uint32_t lsb_mask = (1U << lsb) - 1;
+
+  Inst.addOperand(MCOperand::CreateImm(~(msb_mask ^ lsb_mask)));
+  return S;
+}
+
+static DecodeStatus DecodeCopMemInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned CRd = fieldFromInstruction32(Insn, 12, 4);
+  unsigned coproc = fieldFromInstruction32(Insn, 8, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 8);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned U = fieldFromInstruction32(Insn, 23, 1);
+
+  switch (Inst.getOpcode()) {
+    case ARM::LDC_OFFSET:
+    case ARM::LDC_PRE:
+    case ARM::LDC_POST:
+    case ARM::LDC_OPTION:
+    case ARM::LDCL_OFFSET:
+    case ARM::LDCL_PRE:
+    case ARM::LDCL_POST:
+    case ARM::LDCL_OPTION:
+    case ARM::STC_OFFSET:
+    case ARM::STC_PRE:
+    case ARM::STC_POST:
+    case ARM::STC_OPTION:
+    case ARM::STCL_OFFSET:
+    case ARM::STCL_PRE:
+    case ARM::STCL_POST:
+    case ARM::STCL_OPTION:
+    case ARM::t2LDC_OFFSET:
+    case ARM::t2LDC_PRE:
+    case ARM::t2LDC_POST:
+    case ARM::t2LDC_OPTION:
+    case ARM::t2LDCL_OFFSET:
+    case ARM::t2LDCL_PRE:
+    case ARM::t2LDCL_POST:
+    case ARM::t2LDCL_OPTION:
+    case ARM::t2STC_OFFSET:
+    case ARM::t2STC_PRE:
+    case ARM::t2STC_POST:
+    case ARM::t2STC_OPTION:
+    case ARM::t2STCL_OFFSET:
+    case ARM::t2STCL_PRE:
+    case ARM::t2STCL_POST:
+    case ARM::t2STCL_OPTION:
+      if (coproc == 0xA || coproc == 0xB)
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(coproc));
+  Inst.addOperand(MCOperand::CreateImm(CRd));
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  unsigned P = fieldFromInstruction32(Insn, 24, 1);
+  unsigned W = fieldFromInstruction32(Insn, 21, 1);
+
+  bool writeback = (P == 0) || (W == 1);
+  unsigned idx_mode = 0;
+  if (P && writeback)
+    idx_mode = ARMII::IndexModePre;
+  else if (!P && writeback)
+    idx_mode = ARMII::IndexModePost;
+
+  switch (Inst.getOpcode()) {
+    case ARM::t2LDC2_OFFSET:
+    case ARM::t2LDC2L_OFFSET:
+    case ARM::t2LDC2_PRE:
+    case ARM::t2LDC2L_PRE:
+    case ARM::t2STC2_OFFSET:
+    case ARM::t2STC2L_OFFSET:
+    case ARM::t2STC2_PRE:
+    case ARM::t2STC2L_PRE:
+    case ARM::LDC2_OFFSET:
+    case ARM::LDC2L_OFFSET:
+    case ARM::LDC2_PRE:
+    case ARM::LDC2L_PRE:
+    case ARM::STC2_OFFSET:
+    case ARM::STC2L_OFFSET:
+    case ARM::STC2_PRE:
+    case ARM::STC2L_PRE:
+    case ARM::t2LDC_OFFSET:
+    case ARM::t2LDCL_OFFSET:
+    case ARM::t2LDC_PRE:
+    case ARM::t2LDCL_PRE:
+    case ARM::t2STC_OFFSET:
+    case ARM::t2STCL_OFFSET:
+    case ARM::t2STC_PRE:
+    case ARM::t2STCL_PRE:
+    case ARM::LDC_OFFSET:
+    case ARM::LDCL_OFFSET:
+    case ARM::LDC_PRE:
+    case ARM::LDCL_PRE:
+    case ARM::STC_OFFSET:
+    case ARM::STCL_OFFSET:
+    case ARM::STC_PRE:
+    case ARM::STCL_PRE:
+      imm = ARM_AM::getAM5Opc(U ? ARM_AM::add : ARM_AM::sub, imm);
+      Inst.addOperand(MCOperand::CreateImm(imm));
+      break;
+    case ARM::t2LDC2_POST:
+    case ARM::t2LDC2L_POST:
+    case ARM::t2STC2_POST:
+    case ARM::t2STC2L_POST:
+    case ARM::LDC2_POST:
+    case ARM::LDC2L_POST:
+    case ARM::STC2_POST:
+    case ARM::STC2L_POST:
+    case ARM::t2LDC_POST:
+    case ARM::t2LDCL_POST:
+    case ARM::t2STC_POST:
+    case ARM::t2STCL_POST:
+    case ARM::LDC_POST:
+    case ARM::LDCL_POST:
+    case ARM::STC_POST:
+    case ARM::STCL_POST:
+      imm |= U << 8;
+      // fall through.
+    default:
+      // The 'option' variant doesn't encode 'U' in the immediate since
+      // the immediate is unsigned [0,255].
+      Inst.addOperand(MCOperand::CreateImm(imm));
+      break;
+  }
+
+  switch (Inst.getOpcode()) {
+    case ARM::LDC_OFFSET:
+    case ARM::LDC_PRE:
+    case ARM::LDC_POST:
+    case ARM::LDC_OPTION:
+    case ARM::LDCL_OFFSET:
+    case ARM::LDCL_PRE:
+    case ARM::LDCL_POST:
+    case ARM::LDCL_OPTION:
+    case ARM::STC_OFFSET:
+    case ARM::STC_PRE:
+    case ARM::STC_POST:
+    case ARM::STC_OPTION:
+    case ARM::STCL_OFFSET:
+    case ARM::STCL_PRE:
+    case ARM::STCL_POST:
+    case ARM::STCL_OPTION:
+      if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  return S;
+}
+
+static DecodeStatus
+DecodeAddrMode2IdxInstruction(llvm::MCInst &Inst, unsigned Insn,
+                              uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 12);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned reg = fieldFromInstruction32(Insn, 25, 1);
+  unsigned P = fieldFromInstruction32(Insn, 24, 1);
+  unsigned W = fieldFromInstruction32(Insn, 21, 1);
+
+  // On stores, the writeback operand precedes Rt.
+  switch (Inst.getOpcode()) {
+    case ARM::STR_POST_IMM:
+    case ARM::STR_POST_REG:
+    case ARM::STRB_POST_IMM:
+    case ARM::STRB_POST_REG:
+    case ARM::STRT_POST_REG:
+    case ARM::STRT_POST_IMM:
+    case ARM::STRBT_POST_REG:
+    case ARM::STRBT_POST_IMM:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  // On loads, the writeback operand comes after Rt.
+  switch (Inst.getOpcode()) {
+    case ARM::LDR_POST_IMM:
+    case ARM::LDR_POST_REG:
+    case ARM::LDRB_POST_IMM:
+    case ARM::LDRB_POST_REG:
+    case ARM::LDRBT_POST_REG:
+    case ARM::LDRBT_POST_IMM:
+    case ARM::LDRT_POST_REG:
+    case ARM::LDRT_POST_IMM:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  ARM_AM::AddrOpc Op = ARM_AM::add;
+  if (!fieldFromInstruction32(Insn, 23, 1))
+    Op = ARM_AM::sub;
+
+  bool writeback = (P == 0) || (W == 1);
+  unsigned idx_mode = 0;
+  if (P && writeback)
+    idx_mode = ARMII::IndexModePre;
+  else if (!P && writeback)
+    idx_mode = ARMII::IndexModePost;
+
+  if (writeback && (Rn == 15 || Rn == Rt))
+    S = MCDisassembler::SoftFail; // UNPREDICTABLE
+
+  if (reg) {
+    if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rm, Address, Decoder)))
+      return MCDisassembler::Fail;
+    ARM_AM::ShiftOpc Opc = ARM_AM::lsl;
+    switch( fieldFromInstruction32(Insn, 5, 2)) {
+      case 0:
+        Opc = ARM_AM::lsl;
+        break;
+      case 1:
+        Opc = ARM_AM::lsr;
+        break;
+      case 2:
+        Opc = ARM_AM::asr;
+        break;
+      case 3:
+        Opc = ARM_AM::ror;
+        break;
+      default:
+        return MCDisassembler::Fail;
+    }
+    unsigned amt = fieldFromInstruction32(Insn, 7, 5);
+    unsigned imm = ARM_AM::getAM2Opc(Op, amt, Opc, idx_mode);
+
+    Inst.addOperand(MCOperand::CreateImm(imm));
+  } else {
+    Inst.addOperand(MCOperand::CreateReg(0));
+    unsigned tmp = ARM_AM::getAM2Opc(Op, imm, ARM_AM::lsl, idx_mode);
+    Inst.addOperand(MCOperand::CreateImm(tmp));
+  }
+
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeSORegMemOperand(llvm::MCInst &Inst, unsigned Val,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 13, 4);
+  unsigned Rm = fieldFromInstruction32(Val,  0, 4);
+  unsigned type = fieldFromInstruction32(Val, 5, 2);
+  unsigned imm = fieldFromInstruction32(Val, 7, 5);
+  unsigned U = fieldFromInstruction32(Val, 12, 1);
+
+  ARM_AM::ShiftOpc ShOp = ARM_AM::lsl;
+  switch (type) {
+    case 0:
+      ShOp = ARM_AM::lsl;
+      break;
+    case 1:
+      ShOp = ARM_AM::lsr;
+      break;
+    case 2:
+      ShOp = ARM_AM::asr;
+      break;
+    case 3:
+      ShOp = ARM_AM::ror;
+      break;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  unsigned shift;
+  if (U)
+    shift = ARM_AM::getAM2Opc(ARM_AM::add, imm, ShOp);
+  else
+    shift = ARM_AM::getAM2Opc(ARM_AM::sub, imm, ShOp);
+  Inst.addOperand(MCOperand::CreateImm(shift));
+
+  return S;
+}
+
+static DecodeStatus
+DecodeAddrMode3Instruction(llvm::MCInst &Inst, unsigned Insn,
+                           uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned type = fieldFromInstruction32(Insn, 22, 1);
+  unsigned imm = fieldFromInstruction32(Insn, 8, 4);
+  unsigned U = ((~fieldFromInstruction32(Insn, 23, 1)) & 1) << 8;
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned W = fieldFromInstruction32(Insn, 21, 1);
+  unsigned P = fieldFromInstruction32(Insn, 24, 1);
+
+  bool writeback = (W == 1) | (P == 0);
+
+  // For {LD,ST}RD, Rt must be even, else undefined.
+  switch (Inst.getOpcode()) {
+    case ARM::STRD:
+    case ARM::STRD_PRE:
+    case ARM::STRD_POST:
+    case ARM::LDRD:
+    case ARM::LDRD_PRE:
+    case ARM::LDRD_POST:
+      if (Rt & 0x1) return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  if (writeback) { // Writeback
+    if (P)
+      U |= ARMII::IndexModePre << 9;
+    else
+      U |= ARMII::IndexModePost << 9;
+
+    // On stores, the writeback operand precedes Rt.
+    switch (Inst.getOpcode()) {
+    case ARM::STRD:
+    case ARM::STRD_PRE:
+    case ARM::STRD_POST:
+    case ARM::STRH:
+    case ARM::STRH_PRE:
+    case ARM::STRH_POST:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+    }
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  switch (Inst.getOpcode()) {
+    case ARM::STRD:
+    case ARM::STRD_PRE:
+    case ARM::STRD_POST:
+    case ARM::LDRD:
+    case ARM::LDRD_PRE:
+    case ARM::LDRD_POST:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rt+1, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  if (writeback) {
+    // On loads, the writeback operand comes after Rt.
+    switch (Inst.getOpcode()) {
+    case ARM::LDRD:
+    case ARM::LDRD_PRE:
+    case ARM::LDRD_POST:
+    case ARM::LDRH:
+    case ARM::LDRH_PRE:
+    case ARM::LDRH_POST:
+    case ARM::LDRSH:
+    case ARM::LDRSH_PRE:
+    case ARM::LDRSH_POST:
+    case ARM::LDRSB:
+    case ARM::LDRSB_PRE:
+    case ARM::LDRSB_POST:
+    case ARM::LDRHTr:
+    case ARM::LDRSBTr:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+    }
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (type) {
+    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::CreateImm(U | (imm << 4) | Rm));
+  } else {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    Inst.addOperand(MCOperand::CreateImm(U));
+  }
+
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeRFEInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned mode = fieldFromInstruction32(Insn, 23, 2);
+
+  switch (mode) {
+    case 0:
+      mode = ARM_AM::da;
+      break;
+    case 1:
+      mode = ARM_AM::ia;
+      break;
+    case 2:
+      mode = ARM_AM::db;
+      break;
+    case 3:
+      mode = ARM_AM::ib;
+      break;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(mode));
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeMemMultipleWritebackInstruction(llvm::MCInst &Inst,
+                                  unsigned Insn,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned reglist = fieldFromInstruction32(Insn, 0, 16);
+
+  if (pred == 0xF) {
+    switch (Inst.getOpcode()) {
+      case ARM::LDMDA:
+        Inst.setOpcode(ARM::RFEDA);
+        break;
+      case ARM::LDMDA_UPD:
+        Inst.setOpcode(ARM::RFEDA_UPD);
+        break;
+      case ARM::LDMDB:
+        Inst.setOpcode(ARM::RFEDB);
+        break;
+      case ARM::LDMDB_UPD:
+        Inst.setOpcode(ARM::RFEDB_UPD);
+        break;
+      case ARM::LDMIA:
+        Inst.setOpcode(ARM::RFEIA);
+        break;
+      case ARM::LDMIA_UPD:
+        Inst.setOpcode(ARM::RFEIA_UPD);
+        break;
+      case ARM::LDMIB:
+        Inst.setOpcode(ARM::RFEIB);
+        break;
+      case ARM::LDMIB_UPD:
+        Inst.setOpcode(ARM::RFEIB_UPD);
+        break;
+      case ARM::STMDA:
+        Inst.setOpcode(ARM::SRSDA);
+        break;
+      case ARM::STMDA_UPD:
+        Inst.setOpcode(ARM::SRSDA_UPD);
+        break;
+      case ARM::STMDB:
+        Inst.setOpcode(ARM::SRSDB);
+        break;
+      case ARM::STMDB_UPD:
+        Inst.setOpcode(ARM::SRSDB_UPD);
+        break;
+      case ARM::STMIA:
+        Inst.setOpcode(ARM::SRSIA);
+        break;
+      case ARM::STMIA_UPD:
+        Inst.setOpcode(ARM::SRSIA_UPD);
+        break;
+      case ARM::STMIB:
+        Inst.setOpcode(ARM::SRSIB);
+        break;
+      case ARM::STMIB_UPD:
+        Inst.setOpcode(ARM::SRSIB_UPD);
+        break;
+      default:
+        if (!Check(S, MCDisassembler::Fail)) return MCDisassembler::Fail;
+    }
+
+    // For stores (which become SRS's, the only operand is the mode.
+    if (fieldFromInstruction32(Insn, 20, 1) == 0) {
+      Inst.addOperand(
+          MCOperand::CreateImm(fieldFromInstruction32(Insn, 0, 4)));
+      return S;
+    }
+
+    return DecodeRFEInstruction(Inst, Insn, Address, Decoder);
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail; // Tied
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeRegListOperand(Inst, reglist, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeCPSInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  unsigned imod = fieldFromInstruction32(Insn, 18, 2);
+  unsigned M = fieldFromInstruction32(Insn, 17, 1);
+  unsigned iflags = fieldFromInstruction32(Insn, 6, 3);
+  unsigned mode = fieldFromInstruction32(Insn, 0, 5);
+
+  DecodeStatus S = MCDisassembler::Success;
+
+  // imod == '01' --> UNPREDICTABLE
+  // NOTE: Even though this is technically UNPREDICTABLE, we choose to
+  // return failure here.  The '01' imod value is unprintable, so there's
+  // nothing useful we could do even if we returned UNPREDICTABLE.
+
+  if (imod == 1) return MCDisassembler::Fail;
+
+  if (imod && M) {
+    Inst.setOpcode(ARM::CPS3p);
+    Inst.addOperand(MCOperand::CreateImm(imod));
+    Inst.addOperand(MCOperand::CreateImm(iflags));
+    Inst.addOperand(MCOperand::CreateImm(mode));
+  } else if (imod && !M) {
+    Inst.setOpcode(ARM::CPS2p);
+    Inst.addOperand(MCOperand::CreateImm(imod));
+    Inst.addOperand(MCOperand::CreateImm(iflags));
+    if (mode) S = MCDisassembler::SoftFail;
+  } else if (!imod && M) {
+    Inst.setOpcode(ARM::CPS1p);
+    Inst.addOperand(MCOperand::CreateImm(mode));
+    if (iflags) S = MCDisassembler::SoftFail;
+  } else {
+    // imod == '00' && M == '0' --> UNPREDICTABLE
+    Inst.setOpcode(ARM::CPS1p);
+    Inst.addOperand(MCOperand::CreateImm(mode));
+    S = MCDisassembler::SoftFail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeT2CPSInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  unsigned imod = fieldFromInstruction32(Insn, 9, 2);
+  unsigned M = fieldFromInstruction32(Insn, 8, 1);
+  unsigned iflags = fieldFromInstruction32(Insn, 5, 3);
+  unsigned mode = fieldFromInstruction32(Insn, 0, 5);
+
+  DecodeStatus S = MCDisassembler::Success;
+
+  // imod == '01' --> UNPREDICTABLE
+  // NOTE: Even though this is technically UNPREDICTABLE, we choose to
+  // return failure here.  The '01' imod value is unprintable, so there's
+  // nothing useful we could do even if we returned UNPREDICTABLE.
+
+  if (imod == 1) return MCDisassembler::Fail;
+
+  if (imod && M) {
+    Inst.setOpcode(ARM::t2CPS3p);
+    Inst.addOperand(MCOperand::CreateImm(imod));
+    Inst.addOperand(MCOperand::CreateImm(iflags));
+    Inst.addOperand(MCOperand::CreateImm(mode));
+  } else if (imod && !M) {
+    Inst.setOpcode(ARM::t2CPS2p);
+    Inst.addOperand(MCOperand::CreateImm(imod));
+    Inst.addOperand(MCOperand::CreateImm(iflags));
+    if (mode) S = MCDisassembler::SoftFail;
+  } else if (!imod && M) {
+    Inst.setOpcode(ARM::t2CPS1p);
+    Inst.addOperand(MCOperand::CreateImm(mode));
+    if (iflags) S = MCDisassembler::SoftFail;
+  } else {
+    // imod == '00' && M == '0' --> UNPREDICTABLE
+    Inst.setOpcode(ARM::t2CPS1p);
+    Inst.addOperand(MCOperand::CreateImm(mode));
+    S = MCDisassembler::SoftFail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeT2MOVTWInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 8, 4);
+  unsigned imm = 0;
+
+  imm |= (fieldFromInstruction32(Insn, 0, 8) << 0);
+  imm |= (fieldFromInstruction32(Insn, 12, 3) << 8);
+  imm |= (fieldFromInstruction32(Insn, 16, 4) << 12);
+  imm |= (fieldFromInstruction32(Insn, 26, 1) << 11);
+
+  if (Inst.getOpcode() == ARM::t2MOVTi16)
+    if (!Check(S, DecoderGPRRegisterClass(Inst, Rd, Address, Decoder)))
+      return MCDisassembler::Fail;
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (!tryAddingSymbolicOperand(Address, imm, false, 4, Inst, Decoder))
+    Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return S;
+}
+
+static DecodeStatus DecodeArmMOVTWInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned imm = 0;
+
+  imm |= (fieldFromInstruction32(Insn, 0, 12) << 0);
+  imm |= (fieldFromInstruction32(Insn, 16, 4) << 12);
+
+  if (Inst.getOpcode() == ARM::MOVTi16)
+    if (!Check(S, DecoderGPRRegisterClass(Inst, Rd, Address, Decoder)))
+      return MCDisassembler::Fail;
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (!tryAddingSymbolicOperand(Address, imm, false, 4, Inst, Decoder))
+    Inst.addOperand(MCOperand::CreateImm(imm));
+
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeSMLAInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 8, 4);
+  unsigned Ra = fieldFromInstruction32(Insn, 12, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+
+  if (pred == 0xF)
+    return DecodeCPSInstruction(Inst, Insn, Address, Decoder);
+
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Ra, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeAddrModeImm12Operand(llvm::MCInst &Inst, unsigned Val,
+                           uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned add = fieldFromInstruction32(Val, 12, 1);
+  unsigned imm = fieldFromInstruction32(Val, 0, 12);
+  unsigned Rn = fieldFromInstruction32(Val, 13, 4);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (!add) imm *= -1;
+  if (imm == 0 && !add) imm = INT32_MIN;
+  Inst.addOperand(MCOperand::CreateImm(imm));
+  if (Rn == 15)
+    tryAddingPcLoadReferenceComment(Address, Address + imm + 8, Decoder);
+
+  return S;
+}
+
+static DecodeStatus DecodeAddrMode5Operand(llvm::MCInst &Inst, unsigned Val,
+                                   uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 9, 4);
+  unsigned U = fieldFromInstruction32(Val, 8, 1);
+  unsigned imm = fieldFromInstruction32(Val, 0, 8);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (U)
+    Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(ARM_AM::add, imm)));
+  else
+    Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(ARM_AM::sub, imm)));
+
+  return S;
+}
+
+static DecodeStatus DecodeAddrMode7Operand(llvm::MCInst &Inst, unsigned Val,
+                                   uint64_t Address, const void *Decoder) {
+  return DecodeGPRRegisterClass(Inst, Val, Address, Decoder);
+}
+
+static DecodeStatus
+DecodeBranchImmInstruction(llvm::MCInst &Inst, unsigned Insn,
+                           uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 24) << 2;
+
+  if (pred == 0xF) {
+    Inst.setOpcode(ARM::BLXi);
+    imm |= fieldFromInstruction32(Insn, 24, 1) << 1;
+    Inst.addOperand(MCOperand::CreateImm(SignExtend32<26>(imm)));
+    return S;
+  }
+
+  if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<26>(imm) + 8, true,
+                                4, Inst, Decoder))
+    Inst.addOperand(MCOperand::CreateImm(SignExtend32<26>(imm)));
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+
+static DecodeStatus DecodeVCVTImmOperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(64 - Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeAddrMode6Operand(llvm::MCInst &Inst, unsigned Val,
+                                   uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rm = fieldFromInstruction32(Val, 0, 4);
+  unsigned align = fieldFromInstruction32(Val, 4, 2);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!align)
+    Inst.addOperand(MCOperand::CreateImm(0));
+  else
+    Inst.addOperand(MCOperand::CreateImm(4 << align));
+
+  return S;
+}
+
+static DecodeStatus DecodeVLDInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                   uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned wb = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  Rn |= fieldFromInstruction32(Insn, 4, 2) << 4;
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+
+  // First output register
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  // Second output register
+  switch (Inst.getOpcode()) {
+    case ARM::VLD1q8:
+    case ARM::VLD1q16:
+    case ARM::VLD1q32:
+    case ARM::VLD1q64:
+    case ARM::VLD1q8_UPD:
+    case ARM::VLD1q16_UPD:
+    case ARM::VLD1q32_UPD:
+    case ARM::VLD1q64_UPD:
+    case ARM::VLD1d8T:
+    case ARM::VLD1d16T:
+    case ARM::VLD1d32T:
+    case ARM::VLD1d64T:
+    case ARM::VLD1d8T_UPD:
+    case ARM::VLD1d16T_UPD:
+    case ARM::VLD1d32T_UPD:
+    case ARM::VLD1d64T_UPD:
+    case ARM::VLD1d8Q:
+    case ARM::VLD1d16Q:
+    case ARM::VLD1d32Q:
+    case ARM::VLD1d64Q:
+    case ARM::VLD1d8Q_UPD:
+    case ARM::VLD1d16Q_UPD:
+    case ARM::VLD1d32Q_UPD:
+    case ARM::VLD1d64Q_UPD:
+    case ARM::VLD2d8:
+    case ARM::VLD2d16:
+    case ARM::VLD2d32:
+    case ARM::VLD2d8_UPD:
+    case ARM::VLD2d16_UPD:
+    case ARM::VLD2d32_UPD:
+    case ARM::VLD2q8:
+    case ARM::VLD2q16:
+    case ARM::VLD2q32:
+    case ARM::VLD2q8_UPD:
+    case ARM::VLD2q16_UPD:
+    case ARM::VLD2q32_UPD:
+    case ARM::VLD3d8:
+    case ARM::VLD3d16:
+    case ARM::VLD3d32:
+    case ARM::VLD3d8_UPD:
+    case ARM::VLD3d16_UPD:
+    case ARM::VLD3d32_UPD:
+    case ARM::VLD4d8:
+    case ARM::VLD4d16:
+    case ARM::VLD4d32:
+    case ARM::VLD4d8_UPD:
+    case ARM::VLD4d16_UPD:
+    case ARM::VLD4d32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+1)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VLD2b8:
+    case ARM::VLD2b16:
+    case ARM::VLD2b32:
+    case ARM::VLD2b8_UPD:
+    case ARM::VLD2b16_UPD:
+    case ARM::VLD2b32_UPD:
+    case ARM::VLD3q8:
+    case ARM::VLD3q16:
+    case ARM::VLD3q32:
+    case ARM::VLD3q8_UPD:
+    case ARM::VLD3q16_UPD:
+    case ARM::VLD3q32_UPD:
+    case ARM::VLD4q8:
+    case ARM::VLD4q16:
+    case ARM::VLD4q32:
+    case ARM::VLD4q8_UPD:
+    case ARM::VLD4q16_UPD:
+    case ARM::VLD4q32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+2)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+    default:
+      break;
+  }
+
+  // Third output register
+  switch(Inst.getOpcode()) {
+    case ARM::VLD1d8T:
+    case ARM::VLD1d16T:
+    case ARM::VLD1d32T:
+    case ARM::VLD1d64T:
+    case ARM::VLD1d8T_UPD:
+    case ARM::VLD1d16T_UPD:
+    case ARM::VLD1d32T_UPD:
+    case ARM::VLD1d64T_UPD:
+    case ARM::VLD1d8Q:
+    case ARM::VLD1d16Q:
+    case ARM::VLD1d32Q:
+    case ARM::VLD1d64Q:
+    case ARM::VLD1d8Q_UPD:
+    case ARM::VLD1d16Q_UPD:
+    case ARM::VLD1d32Q_UPD:
+    case ARM::VLD1d64Q_UPD:
+    case ARM::VLD2q8:
+    case ARM::VLD2q16:
+    case ARM::VLD2q32:
+    case ARM::VLD2q8_UPD:
+    case ARM::VLD2q16_UPD:
+    case ARM::VLD2q32_UPD:
+    case ARM::VLD3d8:
+    case ARM::VLD3d16:
+    case ARM::VLD3d32:
+    case ARM::VLD3d8_UPD:
+    case ARM::VLD3d16_UPD:
+    case ARM::VLD3d32_UPD:
+    case ARM::VLD4d8:
+    case ARM::VLD4d16:
+    case ARM::VLD4d32:
+    case ARM::VLD4d8_UPD:
+    case ARM::VLD4d16_UPD:
+    case ARM::VLD4d32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+2)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VLD3q8:
+    case ARM::VLD3q16:
+    case ARM::VLD3q32:
+    case ARM::VLD3q8_UPD:
+    case ARM::VLD3q16_UPD:
+    case ARM::VLD3q32_UPD:
+    case ARM::VLD4q8:
+    case ARM::VLD4q16:
+    case ARM::VLD4q32:
+    case ARM::VLD4q8_UPD:
+    case ARM::VLD4q16_UPD:
+    case ARM::VLD4q32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+4)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  // Fourth output register
+  switch (Inst.getOpcode()) {
+    case ARM::VLD1d8Q:
+    case ARM::VLD1d16Q:
+    case ARM::VLD1d32Q:
+    case ARM::VLD1d64Q:
+    case ARM::VLD1d8Q_UPD:
+    case ARM::VLD1d16Q_UPD:
+    case ARM::VLD1d32Q_UPD:
+    case ARM::VLD1d64Q_UPD:
+    case ARM::VLD2q8:
+    case ARM::VLD2q16:
+    case ARM::VLD2q32:
+    case ARM::VLD2q8_UPD:
+    case ARM::VLD2q16_UPD:
+    case ARM::VLD2q32_UPD:
+    case ARM::VLD4d8:
+    case ARM::VLD4d16:
+    case ARM::VLD4d32:
+    case ARM::VLD4d8_UPD:
+    case ARM::VLD4d16_UPD:
+    case ARM::VLD4d32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+3)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VLD4q8:
+    case ARM::VLD4q16:
+    case ARM::VLD4q32:
+    case ARM::VLD4q8_UPD:
+    case ARM::VLD4q16_UPD:
+    case ARM::VLD4q32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+6)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  // Writeback operand
+  switch (Inst.getOpcode()) {
+    case ARM::VLD1d8_UPD:
+    case ARM::VLD1d16_UPD:
+    case ARM::VLD1d32_UPD:
+    case ARM::VLD1d64_UPD:
+    case ARM::VLD1q8_UPD:
+    case ARM::VLD1q16_UPD:
+    case ARM::VLD1q32_UPD:
+    case ARM::VLD1q64_UPD:
+    case ARM::VLD1d8T_UPD:
+    case ARM::VLD1d16T_UPD:
+    case ARM::VLD1d32T_UPD:
+    case ARM::VLD1d64T_UPD:
+    case ARM::VLD1d8Q_UPD:
+    case ARM::VLD1d16Q_UPD:
+    case ARM::VLD1d32Q_UPD:
+    case ARM::VLD1d64Q_UPD:
+    case ARM::VLD2d8_UPD:
+    case ARM::VLD2d16_UPD:
+    case ARM::VLD2d32_UPD:
+    case ARM::VLD2q8_UPD:
+    case ARM::VLD2q16_UPD:
+    case ARM::VLD2q32_UPD:
+    case ARM::VLD2b8_UPD:
+    case ARM::VLD2b16_UPD:
+    case ARM::VLD2b32_UPD:
+    case ARM::VLD3d8_UPD:
+    case ARM::VLD3d16_UPD:
+    case ARM::VLD3d32_UPD:
+    case ARM::VLD3q8_UPD:
+    case ARM::VLD3q16_UPD:
+    case ARM::VLD3q32_UPD:
+    case ARM::VLD4d8_UPD:
+    case ARM::VLD4d16_UPD:
+    case ARM::VLD4d32_UPD:
+    case ARM::VLD4q8_UPD:
+    case ARM::VLD4q16_UPD:
+    case ARM::VLD4q32_UPD:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, wb, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  // AddrMode6 Base (register+alignment)
+  if (!Check(S, DecodeAddrMode6Operand(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  // AddrMode6 Offset (register)
+  if (Rm == 0xD)
+    Inst.addOperand(MCOperand::CreateReg(0));
+  else if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeVSTInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned wb = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  Rn |= fieldFromInstruction32(Insn, 4, 2) << 4;
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+
+  // Writeback Operand
+  switch (Inst.getOpcode()) {
+    case ARM::VST1d8_UPD:
+    case ARM::VST1d16_UPD:
+    case ARM::VST1d32_UPD:
+    case ARM::VST1d64_UPD:
+    case ARM::VST1q8_UPD:
+    case ARM::VST1q16_UPD:
+    case ARM::VST1q32_UPD:
+    case ARM::VST1q64_UPD:
+    case ARM::VST1d8T_UPD:
+    case ARM::VST1d16T_UPD:
+    case ARM::VST1d32T_UPD:
+    case ARM::VST1d64T_UPD:
+    case ARM::VST1d8Q_UPD:
+    case ARM::VST1d16Q_UPD:
+    case ARM::VST1d32Q_UPD:
+    case ARM::VST1d64Q_UPD:
+    case ARM::VST2d8_UPD:
+    case ARM::VST2d16_UPD:
+    case ARM::VST2d32_UPD:
+    case ARM::VST2q8_UPD:
+    case ARM::VST2q16_UPD:
+    case ARM::VST2q32_UPD:
+    case ARM::VST2b8_UPD:
+    case ARM::VST2b16_UPD:
+    case ARM::VST2b32_UPD:
+    case ARM::VST3d8_UPD:
+    case ARM::VST3d16_UPD:
+    case ARM::VST3d32_UPD:
+    case ARM::VST3q8_UPD:
+    case ARM::VST3q16_UPD:
+    case ARM::VST3q32_UPD:
+    case ARM::VST4d8_UPD:
+    case ARM::VST4d16_UPD:
+    case ARM::VST4d32_UPD:
+    case ARM::VST4q8_UPD:
+    case ARM::VST4q16_UPD:
+    case ARM::VST4q32_UPD:
+      if (!Check(S, DecodeGPRRegisterClass(Inst, wb, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  // AddrMode6 Base (register+alignment)
+  if (!Check(S, DecodeAddrMode6Operand(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  // AddrMode6 Offset (register)
+  if (Rm == 0xD)
+    Inst.addOperand(MCOperand::CreateReg(0));
+  else if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+
+  // First input register
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  // Second input register
+  switch (Inst.getOpcode()) {
+    case ARM::VST1q8:
+    case ARM::VST1q16:
+    case ARM::VST1q32:
+    case ARM::VST1q64:
+    case ARM::VST1q8_UPD:
+    case ARM::VST1q16_UPD:
+    case ARM::VST1q32_UPD:
+    case ARM::VST1q64_UPD:
+    case ARM::VST1d8T:
+    case ARM::VST1d16T:
+    case ARM::VST1d32T:
+    case ARM::VST1d64T:
+    case ARM::VST1d8T_UPD:
+    case ARM::VST1d16T_UPD:
+    case ARM::VST1d32T_UPD:
+    case ARM::VST1d64T_UPD:
+    case ARM::VST1d8Q:
+    case ARM::VST1d16Q:
+    case ARM::VST1d32Q:
+    case ARM::VST1d64Q:
+    case ARM::VST1d8Q_UPD:
+    case ARM::VST1d16Q_UPD:
+    case ARM::VST1d32Q_UPD:
+    case ARM::VST1d64Q_UPD:
+    case ARM::VST2d8:
+    case ARM::VST2d16:
+    case ARM::VST2d32:
+    case ARM::VST2d8_UPD:
+    case ARM::VST2d16_UPD:
+    case ARM::VST2d32_UPD:
+    case ARM::VST2q8:
+    case ARM::VST2q16:
+    case ARM::VST2q32:
+    case ARM::VST2q8_UPD:
+    case ARM::VST2q16_UPD:
+    case ARM::VST2q32_UPD:
+    case ARM::VST3d8:
+    case ARM::VST3d16:
+    case ARM::VST3d32:
+    case ARM::VST3d8_UPD:
+    case ARM::VST3d16_UPD:
+    case ARM::VST3d32_UPD:
+    case ARM::VST4d8:
+    case ARM::VST4d16:
+    case ARM::VST4d32:
+    case ARM::VST4d8_UPD:
+    case ARM::VST4d16_UPD:
+    case ARM::VST4d32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+1)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VST2b8:
+    case ARM::VST2b16:
+    case ARM::VST2b32:
+    case ARM::VST2b8_UPD:
+    case ARM::VST2b16_UPD:
+    case ARM::VST2b32_UPD:
+    case ARM::VST3q8:
+    case ARM::VST3q16:
+    case ARM::VST3q32:
+    case ARM::VST3q8_UPD:
+    case ARM::VST3q16_UPD:
+    case ARM::VST3q32_UPD:
+    case ARM::VST4q8:
+    case ARM::VST4q16:
+    case ARM::VST4q32:
+    case ARM::VST4q8_UPD:
+    case ARM::VST4q16_UPD:
+    case ARM::VST4q32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+2)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  // Third input register
+  switch (Inst.getOpcode()) {
+    case ARM::VST1d8T:
+    case ARM::VST1d16T:
+    case ARM::VST1d32T:
+    case ARM::VST1d64T:
+    case ARM::VST1d8T_UPD:
+    case ARM::VST1d16T_UPD:
+    case ARM::VST1d32T_UPD:
+    case ARM::VST1d64T_UPD:
+    case ARM::VST1d8Q:
+    case ARM::VST1d16Q:
+    case ARM::VST1d32Q:
+    case ARM::VST1d64Q:
+    case ARM::VST1d8Q_UPD:
+    case ARM::VST1d16Q_UPD:
+    case ARM::VST1d32Q_UPD:
+    case ARM::VST1d64Q_UPD:
+    case ARM::VST2q8:
+    case ARM::VST2q16:
+    case ARM::VST2q32:
+    case ARM::VST2q8_UPD:
+    case ARM::VST2q16_UPD:
+    case ARM::VST2q32_UPD:
+    case ARM::VST3d8:
+    case ARM::VST3d16:
+    case ARM::VST3d32:
+    case ARM::VST3d8_UPD:
+    case ARM::VST3d16_UPD:
+    case ARM::VST3d32_UPD:
+    case ARM::VST4d8:
+    case ARM::VST4d16:
+    case ARM::VST4d32:
+    case ARM::VST4d8_UPD:
+    case ARM::VST4d16_UPD:
+    case ARM::VST4d32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+2)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VST3q8:
+    case ARM::VST3q16:
+    case ARM::VST3q32:
+    case ARM::VST3q8_UPD:
+    case ARM::VST3q16_UPD:
+    case ARM::VST3q32_UPD:
+    case ARM::VST4q8:
+    case ARM::VST4q16:
+    case ARM::VST4q32:
+    case ARM::VST4q8_UPD:
+    case ARM::VST4q16_UPD:
+    case ARM::VST4q32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+4)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  // Fourth input register
+  switch (Inst.getOpcode()) {
+    case ARM::VST1d8Q:
+    case ARM::VST1d16Q:
+    case ARM::VST1d32Q:
+    case ARM::VST1d64Q:
+    case ARM::VST1d8Q_UPD:
+    case ARM::VST1d16Q_UPD:
+    case ARM::VST1d32Q_UPD:
+    case ARM::VST1d64Q_UPD:
+    case ARM::VST2q8:
+    case ARM::VST2q16:
+    case ARM::VST2q32:
+    case ARM::VST2q8_UPD:
+    case ARM::VST2q16_UPD:
+    case ARM::VST2q32_UPD:
+    case ARM::VST4d8:
+    case ARM::VST4d16:
+    case ARM::VST4d32:
+    case ARM::VST4d8_UPD:
+    case ARM::VST4d16_UPD:
+    case ARM::VST4d32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+3)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VST4q8:
+    case ARM::VST4q16:
+    case ARM::VST4q32:
+    case ARM::VST4q8_UPD:
+    case ARM::VST4q16_UPD:
+    case ARM::VST4q32_UPD:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+6)%32, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeVLD1DupInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned align = fieldFromInstruction32(Insn, 4, 1);
+  unsigned size = fieldFromInstruction32(Insn, 6, 2);
+  unsigned regs = fieldFromInstruction32(Insn, 5, 1) + 1;
+
+  align *= (1 << size);
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (regs == 2) {
+    if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+1)%32, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+  if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+
+  if (Rm == 0xD)
+    Inst.addOperand(MCOperand::CreateReg(0));
+  else if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeVLD2DupInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned align = fieldFromInstruction32(Insn, 4, 1);
+  unsigned size = 1 << fieldFromInstruction32(Insn, 6, 2);
+  unsigned inc = fieldFromInstruction32(Insn, 5, 1) + 1;
+  align *= 2*size;
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+inc)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+
+  if (Rm == 0xD)
+    Inst.addOperand(MCOperand::CreateReg(0));
+  else if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeVLD3DupInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned inc = fieldFromInstruction32(Insn, 5, 1) + 1;
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+inc)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+2*inc)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(0));
+
+  if (Rm == 0xD)
+    Inst.addOperand(MCOperand::CreateReg(0));
+  else if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeVLD4DupInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned size = fieldFromInstruction32(Insn, 6, 2);
+  unsigned inc = fieldFromInstruction32(Insn, 5, 1) + 1;
+  unsigned align = fieldFromInstruction32(Insn, 4, 1);
+
+  if (size == 0x3) {
+    size = 4;
+    align = 16;
+  } else {
+    if (size == 2) {
+      size = 1 << size;
+      align *= 8;
+    } else {
+      size = 1 << size;
+      align *= 4*size;
+    }
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+inc)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+2*inc)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, (Rd+3*inc)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+
+  if (Rm == 0xD)
+    Inst.addOperand(MCOperand::CreateReg(0));
+  else if (Rm != 0xF) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus
+DecodeNEONModImmInstruction(llvm::MCInst &Inst, unsigned Insn,
+                            uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned imm = fieldFromInstruction32(Insn, 0, 4);
+  imm |= fieldFromInstruction32(Insn, 16, 3) << 4;
+  imm |= fieldFromInstruction32(Insn, 24, 1) << 7;
+  imm |= fieldFromInstruction32(Insn, 8, 4) << 8;
+  imm |= fieldFromInstruction32(Insn, 5, 1) << 12;
+  unsigned Q = fieldFromInstruction32(Insn, 6, 1);
+
+  if (Q) {
+    if (!Check(S, DecodeQPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  } else {
+    if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  switch (Inst.getOpcode()) {
+    case ARM::VORRiv4i16:
+    case ARM::VORRiv2i32:
+    case ARM::VBICiv4i16:
+    case ARM::VBICiv2i32:
+      if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    case ARM::VORRiv8i16:
+    case ARM::VORRiv4i32:
+    case ARM::VBICiv8i16:
+    case ARM::VBICiv4i32:
+      if (!Check(S, DecodeQPRRegisterClass(Inst, Rd, Address, Decoder)))
+        return MCDisassembler::Fail;
+      break;
+    default:
+      break;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeVSHLMaxInstruction(llvm::MCInst &Inst, unsigned Insn,
+                                        uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  Rm |= fieldFromInstruction32(Insn, 5, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 18, 2);
+
+  if (!Check(S, DecodeQPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(8 << size));
+
+  return S;
+}
+
+static DecodeStatus DecodeShiftRight8Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(8 - Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeShiftRight16Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(16 - Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeShiftRight32Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(32 - Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeShiftRight64Imm(llvm::MCInst &Inst, unsigned Val,
+                               uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(64 - Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeTBLInstruction(llvm::MCInst &Inst, unsigned Insn,
+                               uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  Rn |= fieldFromInstruction32(Insn, 7, 1) << 4;
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  Rm |= fieldFromInstruction32(Insn, 5, 1) << 4;
+  unsigned op = fieldFromInstruction32(Insn, 6, 1);
+  unsigned length = fieldFromInstruction32(Insn, 8, 2) + 1;
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (op) {
+    if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail; // Writeback
+  }
+
+  for (unsigned i = 0; i < length; ++i) {
+    if (!Check(S, DecodeDPRRegisterClass(Inst, (Rn+i)%32, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeThumbAddSpecialReg(llvm::MCInst &Inst, uint16_t Insn,
+                                     uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned dst = fieldFromInstruction16(Insn, 8, 3);
+  unsigned imm = fieldFromInstruction16(Insn, 0, 8);
+
+  if (!Check(S, DecodetGPRRegisterClass(Inst, dst, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  switch(Inst.getOpcode()) {
+    default:
+      return MCDisassembler::Fail;
+    case ARM::tADR:
+      break; // tADR does not explicitly represent the PC as an operand.
+    case ARM::tADDrSPi:
+      Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+      break;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(imm));
+  return S;
+}
+
+static DecodeStatus DecodeThumbBROperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(SignExtend32<12>(Val << 1)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeT2BROperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(SignExtend32<21>(Val)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeThumbCmpBROperand(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateImm(SignExtend32<7>(Val << 1)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeThumbAddrModeRR(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 0, 3);
+  unsigned Rm = fieldFromInstruction32(Val, 3, 3);
+
+  if (!Check(S, DecodetGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodetGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeThumbAddrModeIS(llvm::MCInst &Inst, unsigned Val,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 0, 3);
+  unsigned imm = fieldFromInstruction32(Val, 3, 5);
+
+  if (!Check(S, DecodetGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return S;
+}
+
+static DecodeStatus DecodeThumbAddrModePC(llvm::MCInst &Inst, unsigned Val,
+                                  uint64_t Address, const void *Decoder) {
+  unsigned imm = Val << 2;
+
+  Inst.addOperand(MCOperand::CreateImm(imm));
+  tryAddingPcLoadReferenceComment(Address, (Address & ~2u) + imm + 4, Decoder);
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeThumbAddrModeSP(llvm::MCInst &Inst, unsigned Val,
+                                  uint64_t Address, const void *Decoder) {
+  Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+  Inst.addOperand(MCOperand::CreateImm(Val));
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeT2AddrModeSOReg(llvm::MCInst &Inst, unsigned Val,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 6, 4);
+  unsigned Rm = fieldFromInstruction32(Val, 2, 4);
+  unsigned imm = fieldFromInstruction32(Val, 0, 2);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return S;
+}
+
+static DecodeStatus DecodeT2LoadShift(llvm::MCInst &Inst, unsigned Insn,
+                              uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  switch (Inst.getOpcode()) {
+    case ARM::t2PLDs:
+    case ARM::t2PLDWs:
+    case ARM::t2PLIs:
+      break;
+    default: {
+      unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+      if (!Check(S, DecoderGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+    }
+  }
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  if (Rn == 0xF) {
+    switch (Inst.getOpcode()) {
+      case ARM::t2LDRBs:
+        Inst.setOpcode(ARM::t2LDRBpci);
+        break;
+      case ARM::t2LDRHs:
+        Inst.setOpcode(ARM::t2LDRHpci);
+        break;
+      case ARM::t2LDRSHs:
+        Inst.setOpcode(ARM::t2LDRSHpci);
+        break;
+      case ARM::t2LDRSBs:
+        Inst.setOpcode(ARM::t2LDRSBpci);
+        break;
+      case ARM::t2PLDs:
+        Inst.setOpcode(ARM::t2PLDi12);
+        Inst.addOperand(MCOperand::CreateReg(ARM::PC));
+        break;
+      default:
+        return MCDisassembler::Fail;
+    }
+
+    int imm = fieldFromInstruction32(Insn, 0, 12);
+    if (!fieldFromInstruction32(Insn, 23, 1)) imm *= -1;
+    Inst.addOperand(MCOperand::CreateImm(imm));
+
+    return S;
+  }
+
+  unsigned addrmode = fieldFromInstruction32(Insn, 4, 2);
+  addrmode |= fieldFromInstruction32(Insn, 0, 4) << 2;
+  addrmode |= fieldFromInstruction32(Insn, 16, 4) << 6;
+  if (!Check(S, DecodeT2AddrModeSOReg(Inst, addrmode, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeT2Imm8S4(llvm::MCInst &Inst, unsigned Val,
+                           uint64_t Address, const void *Decoder) {
+  int imm = Val & 0xFF;
+  if (!(Val & 0x100)) imm *= -1;
+  Inst.addOperand(MCOperand::CreateImm(imm << 2));
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeT2AddrModeImm8s4(llvm::MCInst &Inst, unsigned Val,
+                                   uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 9, 4);
+  unsigned imm = fieldFromInstruction32(Val, 0, 9);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeT2Imm8S4(Inst, imm, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeT2AddrModeImm0_1020s4(llvm::MCInst &Inst,unsigned Val,
+                                   uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 8, 4);
+  unsigned imm = fieldFromInstruction32(Val, 0, 8);
+
+  if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return S;
+}
+
+static DecodeStatus DecodeT2Imm8(llvm::MCInst &Inst, unsigned Val,
+                         uint64_t Address, const void *Decoder) {
+  int imm = Val & 0xFF;
+  if (Val == 0)
+    imm = INT32_MIN;
+  else if (!(Val & 0x100))
+    imm *= -1;
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return MCDisassembler::Success;
+}
+
+
+static DecodeStatus DecodeT2AddrModeImm8(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 9, 4);
+  unsigned imm = fieldFromInstruction32(Val, 0, 9);
+
+  // Some instructions always use an additive offset.
+  switch (Inst.getOpcode()) {
+    case ARM::t2LDRT:
+    case ARM::t2LDRBT:
+    case ARM::t2LDRHT:
+    case ARM::t2LDRSBT:
+    case ARM::t2LDRSHT:
+    case ARM::t2STRT:
+    case ARM::t2STRBT:
+    case ARM::t2STRHT:
+      imm |= 0x100;
+      break;
+    default:
+      break;
+  }
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeT2Imm8(Inst, imm, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeT2LdStPre(llvm::MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned addr = fieldFromInstruction32(Insn, 0, 8);
+  addr |= fieldFromInstruction32(Insn, 9, 1) << 8;
+  addr |= Rn << 9;
+  unsigned load = fieldFromInstruction32(Insn, 20, 1);
+
+  if (!load) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (load) {
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+
+  if (!Check(S, DecodeT2AddrModeImm8(Inst, addr, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeT2AddrModeImm12(llvm::MCInst &Inst, unsigned Val,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Val, 13, 4);
+  unsigned imm = fieldFromInstruction32(Val, 0, 12);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return S;
+}
+
+
+static DecodeStatus DecodeThumbAddSPImm(llvm::MCInst &Inst, uint16_t Insn,
+                                uint64_t Address, const void *Decoder) {
+  unsigned imm = fieldFromInstruction16(Insn, 0, 7);
+
+  Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+  Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+  Inst.addOperand(MCOperand::CreateImm(imm));
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeThumbAddSPReg(llvm::MCInst &Inst, uint16_t Insn,
+                                uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  if (Inst.getOpcode() == ARM::tADDrSP) {
+    unsigned Rdm = fieldFromInstruction16(Insn, 0, 3);
+    Rdm |= fieldFromInstruction16(Insn, 7, 1) << 3;
+
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rdm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rdm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+  } else if (Inst.getOpcode() == ARM::tADDspr) {
+    unsigned Rm = fieldFromInstruction16(Insn, 3, 4);
+
+    Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+    Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+
+  return S;
+}
+
+static DecodeStatus DecodeThumbCPS(llvm::MCInst &Inst, uint16_t Insn,
+                           uint64_t Address, const void *Decoder) {
+  unsigned imod = fieldFromInstruction16(Insn, 4, 1) | 0x2;
+  unsigned flags = fieldFromInstruction16(Insn, 0, 3);
+
+  Inst.addOperand(MCOperand::CreateImm(imod));
+  Inst.addOperand(MCOperand::CreateImm(flags));
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodePostIdxReg(llvm::MCInst &Inst, unsigned Insn,
+                             uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned add = fieldFromInstruction32(Insn, 4, 1);
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(add));
+
+  return S;
+}
+
+static DecodeStatus DecodeThumbBLXOffset(llvm::MCInst &Inst, unsigned Val,
+                                 uint64_t Address, const void *Decoder) {
+  if (!tryAddingSymbolicOperand(Address, 
+                                (Address & ~2u) + SignExtend32<22>(Val << 1) + 4,
+                                true, 4, Inst, Decoder))
+    Inst.addOperand(MCOperand::CreateImm(SignExtend32<22>(Val << 1)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeCoprocessor(llvm::MCInst &Inst, unsigned Val,
+                              uint64_t Address, const void *Decoder) {
+  if (Val == 0xA || Val == 0xB)
+    return MCDisassembler::Fail;
+
+  Inst.addOperand(MCOperand::CreateImm(Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus
+DecodeThumbTableBranch(llvm::MCInst &Inst, unsigned Insn,
+                       uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+
+  if (Rn == ARM::SP) S = MCDisassembler::SoftFail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  return S;
+}
+
+static DecodeStatus
+DecodeThumb2BCCInstruction(llvm::MCInst &Inst, unsigned Insn,
+                           uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned pred = fieldFromInstruction32(Insn, 22, 4);
+  if (pred == 0xE || pred == 0xF) {
+    unsigned opc = fieldFromInstruction32(Insn, 4, 28);
+    switch (opc) {
+      default:
+        return MCDisassembler::Fail;
+      case 0xf3bf8f4:
+        Inst.setOpcode(ARM::t2DSB);
+        break;
+      case 0xf3bf8f5:
+        Inst.setOpcode(ARM::t2DMB);
+        break;
+      case 0xf3bf8f6:
+        Inst.setOpcode(ARM::t2ISB);
+        break;
+    }
+
+    unsigned imm = fieldFromInstruction32(Insn, 0, 4);
+    return DecodeMemBarrierOption(Inst, imm, Address, Decoder);
+  }
+
+  unsigned brtarget = fieldFromInstruction32(Insn, 0, 11) << 1;
+  brtarget |= fieldFromInstruction32(Insn, 11, 1) << 19;
+  brtarget |= fieldFromInstruction32(Insn, 13, 1) << 18;
+  brtarget |= fieldFromInstruction32(Insn, 16, 6) << 12;
+  brtarget |= fieldFromInstruction32(Insn, 26, 1) << 20;
+
+  if (!Check(S, DecodeT2BROperand(Inst, brtarget, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+// Decode a shifted immediate operand.  These basically consist
+// of an 8-bit value, and a 4-bit directive that specifies either
+// a splat operation or a rotation.
+static DecodeStatus DecodeT2SOImm(llvm::MCInst &Inst, unsigned Val,
+                          uint64_t Address, const void *Decoder) {
+  unsigned ctrl = fieldFromInstruction32(Val, 10, 2);
+  if (ctrl == 0) {
+    unsigned byte = fieldFromInstruction32(Val, 8, 2);
+    unsigned imm = fieldFromInstruction32(Val, 0, 8);
+    switch (byte) {
+      case 0:
+        Inst.addOperand(MCOperand::CreateImm(imm));
+        break;
+      case 1:
+        Inst.addOperand(MCOperand::CreateImm((imm << 16) | imm));
+        break;
+      case 2:
+        Inst.addOperand(MCOperand::CreateImm((imm << 24) | (imm << 8)));
+        break;
+      case 3:
+        Inst.addOperand(MCOperand::CreateImm((imm << 24) | (imm << 16) |
+                                             (imm << 8)  |  imm));
+        break;
+    }
+  } else {
+    unsigned unrot = fieldFromInstruction32(Val, 0, 7) | 0x80;
+    unsigned rot = fieldFromInstruction32(Val, 7, 5);
+    unsigned imm = (unrot >> rot) | (unrot << ((32-rot)&31));
+    Inst.addOperand(MCOperand::CreateImm(imm));
+  }
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus
+DecodeThumbBCCTargetOperand(llvm::MCInst &Inst, unsigned Val,
+                            uint64_t Address, const void *Decoder){
+  Inst.addOperand(MCOperand::CreateImm(Val << 1));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeThumbBLTargetOperand(llvm::MCInst &Inst, unsigned Val,
+                                       uint64_t Address, const void *Decoder){
+  Inst.addOperand(MCOperand::CreateImm(SignExtend32<22>(Val << 1)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeMemBarrierOption(llvm::MCInst &Inst, unsigned Val,
+                                   uint64_t Address, const void *Decoder) {
+  switch (Val) {
+  default:
+    return MCDisassembler::Fail;
+  case 0xF: // SY
+  case 0xE: // ST
+  case 0xB: // ISH
+  case 0xA: // ISHST
+  case 0x7: // NSH
+  case 0x6: // NSHST
+  case 0x3: // OSH
+  case 0x2: // OSHST
+    break;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeMSRMask(llvm::MCInst &Inst, unsigned Val,
+                          uint64_t Address, const void *Decoder) {
+  if (!Val) return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(Val));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeDoubleRegLoad(llvm::MCInst &Inst, unsigned Insn,
+                                        uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+
+  if ((Rt & 1) || Rt == 0xE || Rn == 0xF) return MCDisassembler::Fail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt+1, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+
+static DecodeStatus DecodeDoubleRegStore(llvm::MCInst &Inst, unsigned Insn,
+                                         uint64_t Address, const void *Decoder){
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rt = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if ((Rt & 1) || Rt == 0xE || Rn == 0xF) return MCDisassembler::Fail;
+  if (Rd == Rn || Rd == Rt || Rd == Rt+1) return MCDisassembler::Fail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt+1, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeLDRPreImm(llvm::MCInst &Inst, unsigned Insn,
+                            uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 12);
+  imm |= fieldFromInstruction32(Insn, 16, 4) << 13;
+  imm |= fieldFromInstruction32(Insn, 23, 1) << 12;
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+
+  if (Rn == 0xF || Rn == Rt) S = MCDisassembler::SoftFail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeAddrModeImm12Operand(Inst, imm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeLDRPreReg(llvm::MCInst &Inst, unsigned Insn,
+                            uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 12);
+  imm |= fieldFromInstruction32(Insn, 16, 4) << 13;
+  imm |= fieldFromInstruction32(Insn, 23, 1) << 12;
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+
+  if (Rn == 0xF || Rn == Rt) S = MCDisassembler::SoftFail;
+  if (Rm == 0xF) S = MCDisassembler::SoftFail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeSORegMemOperand(Inst, imm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+
+static DecodeStatus DecodeSTRPreImm(llvm::MCInst &Inst, unsigned Insn,
+                            uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 12);
+  imm |= fieldFromInstruction32(Insn, 16, 4) << 13;
+  imm |= fieldFromInstruction32(Insn, 23, 1) << 12;
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+
+  if (Rn == 0xF || Rn == Rt) S = MCDisassembler::SoftFail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeAddrModeImm12Operand(Inst, imm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeSTRPreReg(llvm::MCInst &Inst, unsigned Insn,
+                            uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned imm = fieldFromInstruction32(Insn, 0, 12);
+  imm |= fieldFromInstruction32(Insn, 16, 4) << 13;
+  imm |= fieldFromInstruction32(Insn, 23, 1) << 12;
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+
+  if (Rn == 0xF || Rn == Rt) S = MCDisassembler::SoftFail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeSORegMemOperand(Inst, imm, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeVLD1LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 5, 3);
+      break;
+    case 1:
+      if (fieldFromInstruction32(Insn, 5, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 6, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 4, 2) != 0)
+        align = 4;
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+        return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+static DecodeStatus DecodeVST1LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 5, 3);
+      break;
+    case 1:
+      if (fieldFromInstruction32(Insn, 5, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 6, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 4, 2) != 0)
+        align = 4;
+  }
+
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+
+static DecodeStatus DecodeVLD2LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  unsigned inc = 1;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      index = fieldFromInstruction32(Insn, 5, 3);
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 2;
+      break;
+    case 1:
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 4;
+      if (fieldFromInstruction32(Insn, 5, 1))
+        inc = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 5, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 4, 1) != 0)
+        align = 8;
+      if (fieldFromInstruction32(Insn, 6, 1))
+        inc = 2;
+      break;
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+        return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+static DecodeStatus DecodeVST2LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  unsigned inc = 1;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      index = fieldFromInstruction32(Insn, 5, 3);
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 2;
+      break;
+    case 1:
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 4;
+      if (fieldFromInstruction32(Insn, 5, 1))
+        inc = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 5, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 4, 1) != 0)
+        align = 8;
+      if (fieldFromInstruction32(Insn, 6, 1))
+        inc = 2;
+      break;
+  }
+
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+        return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+
+static DecodeStatus DecodeVLD3LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  unsigned inc = 1;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 5, 3);
+      break;
+    case 1:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 5, 1))
+        inc = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 4, 2))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 6, 1))
+        inc = 2;
+      break;
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+static DecodeStatus DecodeVST3LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  unsigned inc = 1;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 5, 3);
+      break;
+    case 1:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 5, 1))
+        inc = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 4, 2))
+        return MCDisassembler::Fail; // UNDEFINED
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 6, 1))
+        inc = 2;
+      break;
+  }
+
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+
+static DecodeStatus DecodeVLD4LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  unsigned inc = 1;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 4;
+      index = fieldFromInstruction32(Insn, 5, 3);
+      break;
+    case 1:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 8;
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 5, 1))
+        inc = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 4, 2))
+        align = 4 << fieldFromInstruction32(Insn, 4, 2);
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 6, 1))
+        inc = 2;
+      break;
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+3*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+      return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+        return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+3*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+static DecodeStatus DecodeVST4LN(llvm::MCInst &Inst, unsigned Insn,
+                         uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm = fieldFromInstruction32(Insn, 0, 4);
+  unsigned Rd = fieldFromInstruction32(Insn, 12, 4);
+  Rd |= fieldFromInstruction32(Insn, 22, 1) << 4;
+  unsigned size = fieldFromInstruction32(Insn, 10, 2);
+
+  unsigned align = 0;
+  unsigned index = 0;
+  unsigned inc = 1;
+  switch (size) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 4;
+      index = fieldFromInstruction32(Insn, 5, 3);
+      break;
+    case 1:
+      if (fieldFromInstruction32(Insn, 4, 1))
+        align = 8;
+      index = fieldFromInstruction32(Insn, 6, 2);
+      if (fieldFromInstruction32(Insn, 5, 1))
+        inc = 2;
+      break;
+    case 2:
+      if (fieldFromInstruction32(Insn, 4, 2))
+        align = 4 << fieldFromInstruction32(Insn, 4, 2);
+      index = fieldFromInstruction32(Insn, 7, 1);
+      if (fieldFromInstruction32(Insn, 6, 1))
+        inc = 2;
+      break;
+  }
+
+  if (Rm != 0xF) { // Writeback
+    if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  }
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(align));
+  if (Rm != 0xF) {
+    if (Rm != 0xD) {
+      if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
+    return MCDisassembler::Fail;
+    } else
+      Inst.addOperand(MCOperand::CreateReg(0));
+  }
+
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+3*inc, Address, Decoder)))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::CreateImm(index));
+
+  return S;
+}
+
+static DecodeStatus DecodeVMOVSRR(llvm::MCInst &Inst, unsigned Insn,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+  unsigned Rt  = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rt2 = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm  = fieldFromInstruction32(Insn,  0, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  Rm |= fieldFromInstruction32(Insn, 5, 1) << 4;
+
+  if (Rt == 0xF || Rt2 == 0xF || Rm == 0x1F)
+    S = MCDisassembler::SoftFail;
+
+  if (!Check(S, DecodeSPRRegisterClass(Inst, Rm  , Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeSPRRegisterClass(Inst, Rm+1, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt  , Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt2 , Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeVMOVRRS(llvm::MCInst &Inst, unsigned Insn,
+                                  uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+  unsigned Rt  = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rt2 = fieldFromInstruction32(Insn, 16, 4);
+  unsigned Rm  = fieldFromInstruction32(Insn,  0, 4);
+  unsigned pred = fieldFromInstruction32(Insn, 28, 4);
+  Rm |= fieldFromInstruction32(Insn, 5, 1) << 4;
+
+  if (Rt == 0xF || Rt2 == 0xF || Rm == 0x1F)
+    S = MCDisassembler::SoftFail;
+
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt  , Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeGPRRegisterClass(Inst, Rt2 , Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeSPRRegisterClass(Inst, Rm  , Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodeSPRRegisterClass(Inst, Rm+1, Address, Decoder)))
+    return MCDisassembler::Fail;
+  if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeIT(llvm::MCInst &Inst, unsigned Insn,
+                             uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+  unsigned pred = fieldFromInstruction16(Insn, 4, 4);
+  // The InstPrinter needs to have the low bit of the predicate in
+  // the mask operand to be able to print it properly.
+  unsigned mask = fieldFromInstruction16(Insn, 0, 5);
+
+  if (pred == 0xF) {
+    pred = 0xE;
+    S = MCDisassembler::SoftFail;
+  }
+
+  if ((mask & 0xF) == 0) {
+    // Preserve the high bit of the mask, which is the low bit of
+    // the predicate.
+    mask &= 0x10;
+    mask |= 0x8;
+    S = MCDisassembler::SoftFail;
+  }
+
+  Inst.addOperand(MCOperand::CreateImm(pred));
+  Inst.addOperand(MCOperand::CreateImm(mask));
+  return S;
+}
+
+static DecodeStatus
+DecodeT2LDRDPreInstruction(llvm::MCInst &Inst, unsigned Insn,
+                           uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rt2 = fieldFromInstruction32(Insn, 8, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned addr = fieldFromInstruction32(Insn, 0, 8);
+  unsigned W = fieldFromInstruction32(Insn, 21, 1);
+  unsigned U = fieldFromInstruction32(Insn, 23, 1);
+  unsigned P = fieldFromInstruction32(Insn, 24, 1);
+  bool writeback = (W == 1) | (P == 0);
+
+  addr |= (U << 8) | (Rn << 9);
+
+  if (writeback && (Rn == Rt || Rn == Rt2))
+    Check(S, MCDisassembler::SoftFail);
+  if (Rt == Rt2)
+    Check(S, MCDisassembler::SoftFail);
+
+  // Rt
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  // Rt2
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rt2, Address, Decoder)))
+    return MCDisassembler::Fail;
+  // Writeback operand
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  // addr
+  if (!Check(S, DecodeT2AddrModeImm8s4(Inst, addr, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus
+DecodeT2STRDPreInstruction(llvm::MCInst &Inst, unsigned Insn,
+                           uint64_t Address, const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  unsigned Rt = fieldFromInstruction32(Insn, 12, 4);
+  unsigned Rt2 = fieldFromInstruction32(Insn, 8, 4);
+  unsigned Rn = fieldFromInstruction32(Insn, 16, 4);
+  unsigned addr = fieldFromInstruction32(Insn, 0, 8);
+  unsigned W = fieldFromInstruction32(Insn, 21, 1);
+  unsigned U = fieldFromInstruction32(Insn, 23, 1);
+  unsigned P = fieldFromInstruction32(Insn, 24, 1);
+  bool writeback = (W == 1) | (P == 0);
+
+  addr |= (U << 8) | (Rn << 9);
+
+  if (writeback && (Rn == Rt || Rn == Rt2))
+    Check(S, MCDisassembler::SoftFail);
+
+  // Writeback operand
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rn, Address, Decoder)))
+    return MCDisassembler::Fail;
+  // Rt
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rt, Address, Decoder)))
+    return MCDisassembler::Fail;
+  // Rt2
+  if (!Check(S, DecoderGPRRegisterClass(Inst, Rt2, Address, Decoder)))
+    return MCDisassembler::Fail;
+  // addr
+  if (!Check(S, DecodeT2AddrModeImm8s4(Inst, addr, Address, Decoder)))
+    return MCDisassembler::Fail;
+
+  return S;
+}
+
+static DecodeStatus DecodeT2Adr(llvm::MCInst &Inst, uint32_t Insn,
+                                uint64_t Address, const void *Decoder) {
+  unsigned sign1 = fieldFromInstruction32(Insn, 21, 1);
+  unsigned sign2 = fieldFromInstruction32(Insn, 23, 1);
+  if (sign1 != sign2) return MCDisassembler::Fail;
+
+  unsigned Val = fieldFromInstruction32(Insn, 0, 8);
+  Val |= fieldFromInstruction32(Insn, 12, 3) << 8;
+  Val |= fieldFromInstruction32(Insn, 26, 1) << 11;
+  Val |= sign1 << 12;
+  Inst.addOperand(MCOperand::CreateImm(SignExtend32<13>(Val)));
+
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeT2ShifterImmOperand(llvm::MCInst &Inst, uint32_t Val,
+                                              uint64_t Address,
+                                              const void *Decoder) {
+  DecodeStatus S = MCDisassembler::Success;
+
+  // Shift of "asr #32" is not allowed in Thumb2 mode.
+  if (Val == 0x20) S = MCDisassembler::SoftFail;
+  Inst.addOperand(MCOperand::CreateImm(Val));
+  return S;
+}
+
diff --git a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.h b/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.h
deleted file mode 100644
index 0a74a3866eed..000000000000
--- a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.h
+++ /dev/null
@@ -1,99 +0,0 @@
-//===- ARMDisassembler.h - Disassembler for ARM/Thumb ISA -------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is part of the ARM Disassembler.
-// It contains the header for ARMDisassembler and ThumbDisassembler, both are
-// subclasses of MCDisassembler.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef ARMDISASSEMBLER_H
-#define ARMDISASSEMBLER_H
-
-#include "llvm/MC/MCDisassembler.h"
-
-namespace llvm {
-  
-class MCInst;
-class MemoryObject;
-class raw_ostream;
-  
-struct EDInstInfo;
-  
-/// ARMDisassembler - ARM disassembler for all ARM platforms.
-class ARMDisassembler : public MCDisassembler {
-public:
-  /// Constructor     - Initializes the disassembler.
-  ///
-  ARMDisassembler() :
-    MCDisassembler() {
-  }
-
-  ~ARMDisassembler() {
-  }
-
-  /// getInstruction - See MCDisassembler.
-  bool getInstruction(MCInst &instr,
-                      uint64_t &size,
-                      const MemoryObject &region,
-                      uint64_t address,
-                      raw_ostream &vStream) const;
-  
-  /// getEDInfo - See MCDisassembler.
-  EDInstInfo *getEDInfo() const;
-private:
-};
-
-// Forward declaration.
-class ARMBasicMCBuilder;
-
-/// Session - Keep track of the IT Block progression.
-class Session {
-  friend class ARMBasicMCBuilder;
-public:
-  Session() : ITCounter(0), ITState(0) {}
-  ~Session() {}
-  /// InitIT - Initializes ITCounter/ITState.
-  bool InitIT(unsigned short bits7_0);
-  /// UpdateIT - Updates ITCounter/ITState as IT Block progresses.
-  void UpdateIT();
-
-private:
-  unsigned ITCounter; // Possible values: 0, 1, 2, 3, 4.
-  unsigned ITState;   // A2.5.2 Consists of IT[7:5] and IT[4:0] initially.
-};
-
-/// ThumbDisassembler - Thumb disassembler for all ARM platforms.
-class ThumbDisassembler : public MCDisassembler {
-public:
-  /// Constructor     - Initializes the disassembler.
-  ///
-  ThumbDisassembler() :
-    MCDisassembler(), SO() {
-  }
-
-  ~ThumbDisassembler() {
-  }
-
-  /// getInstruction - See MCDisassembler.
-  bool getInstruction(MCInst &instr,
-                      uint64_t &size,
-                      const MemoryObject &region,
-                      uint64_t address,
-                      raw_ostream &vStream) const;
-  
-  /// getEDInfo - See MCDisassembler.
-  EDInstInfo *getEDInfo() const;
-private:
-  Session SO;
-};
-
-} // namespace llvm
-  
-#endif
diff --git a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassemblerCore.cpp b/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassemblerCore.cpp
deleted file mode 100644
index d89c80a9d457..000000000000
--- a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassemblerCore.cpp
+++ /dev/null
@@ -1,3818 +0,0 @@
-//===- ARMDisassemblerCore.cpp - ARM disassembler helpers -------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is part of the ARM Disassembler.
-// It contains code to represent the core concepts of Builder and DisassembleFP
-// to solve the problem of disassembling an ARM instr.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "arm-disassembler"
-
-#include "ARMDisassemblerCore.h"
-#include "ARMAddressingModes.h"
-#include "ARMMCExpr.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-
-//#define DEBUG(X) do { X; } while (0)
-
-/// ARMGenInstrInfo.inc - ARMGenInstrInfo.inc contains the static const
-/// MCInstrDesc ARMInsts[] definition and the MCOperandInfo[]'s describing the
-/// operand info for each ARMInsts[i].
-///
-/// Together with an instruction's encoding format, we can take advantage of the
-/// NumOperands and the OpInfo fields of the target instruction description in
-/// the quest to build out the MCOperand list for an MCInst.
-///
-/// The general guideline is that with a known format, the number of dst and src
-/// operands are well-known.  The dst is built first, followed by the src
-/// operand(s).  The operands not yet used at this point are for the Implicit
-/// Uses and Defs by this instr.  For the Uses part, the pred:$p operand is
-/// defined with two components:
-///
-/// def pred { // Operand PredicateOperand
-///   ValueType Type = OtherVT;
-///   string PrintMethod = "printPredicateOperand";
-///   string AsmOperandLowerMethod = ?;
-///   dag MIOperandInfo = (ops i32imm, CCR);
-///   AsmOperandClass ParserMatchClass = ImmAsmOperand;
-///   dag DefaultOps = (ops (i32 14), (i32 zero_reg));
-/// }
-///
-/// which is manifested by the MCOperandInfo[] of:
-///
-/// { 0, 0|(1<<MCOI::Predicate), 0 },
-/// { ARM::CCRRegClassID, 0|(1<<MCOI::Predicate), 0 }
-///
-/// So the first predicate MCOperand corresponds to the immediate part of the
-/// ARM condition field (Inst{31-28}), and the second predicate MCOperand
-/// corresponds to a register kind of ARM::CPSR.
-///
-/// For the Defs part, in the simple case of only cc_out:$s, we have:
-///
-/// def cc_out { // Operand OptionalDefOperand
-///   ValueType Type = OtherVT;
-///   string PrintMethod = "printSBitModifierOperand";
-///   string AsmOperandLowerMethod = ?;
-///   dag MIOperandInfo = (ops CCR);
-///   AsmOperandClass ParserMatchClass = ImmAsmOperand;
-///   dag DefaultOps = (ops (i32 zero_reg));
-/// }
-///
-/// which is manifested by the one MCOperandInfo of:
-///
-/// { ARM::CCRRegClassID, 0|(1<<MCOI::OptionalDef), 0 }
-///
-
-namespace llvm {
-extern MCInstrDesc ARMInsts[];
-}
-
-using namespace llvm;
-
-const char *ARMUtils::OpcodeName(unsigned Opcode) {
-  return ARMInsts[Opcode].Name;
-}
-
-// Return the register enum Based on RegClass and the raw register number.
-// FIXME: Auto-gened?
-static unsigned
-getRegisterEnum(BO B, unsigned RegClassID, unsigned RawRegister) {
-  if (RegClassID == ARM::rGPRRegClassID) {
-    // Check for The register numbers 13 and 15 that are not permitted for many
-    // Thumb register specifiers.
-    if (RawRegister == 13 || RawRegister == 15) {
-      B->SetErr(-1);
-      return 0;
-    }
-    // For this purpose, we can treat rGPR as if it were GPR.
-    RegClassID = ARM::GPRRegClassID;
-  }
-
-  // See also decodeNEONRd(), decodeNEONRn(), decodeNEONRm().
-  // A7.3 register encoding
-  //     Qd -> bit[12] == 0
-  //     Qn -> bit[16] == 0
-  //     Qm -> bit[0]  == 0
-  //
-  // If one of these bits is 1, the instruction is UNDEFINED.
-  if (RegClassID == ARM::QPRRegClassID && slice(RawRegister, 0, 0) == 1) {
-    B->SetErr(-1);
-    return 0;
-  }
-  unsigned RegNum =
-    RegClassID == ARM::QPRRegClassID ? RawRegister >> 1 : RawRegister;
-
-  switch (RegNum) {
-  default:
-    break;
-  case 0:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R0;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D0;
-    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
-    case ARM::QPR_VFP2RegClassID:
-      return ARM::Q0;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S0;
-    }
-    break;
-  case 1:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R1;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D1;
-    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
-    case ARM::QPR_VFP2RegClassID:
-      return ARM::Q1;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S1;
-    }
-    break;
-  case 2:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R2;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D2;
-    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
-    case ARM::QPR_VFP2RegClassID:
-      return ARM::Q2;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S2;
-    }
-    break;
-  case 3:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R3;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D3;
-    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
-    case ARM::QPR_VFP2RegClassID:
-      return ARM::Q3;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S3;
-    }
-    break;
-  case 4:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R4;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D4;
-    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q4;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S4;
-    }
-    break;
-  case 5:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R5;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D5;
-    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q5;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S5;
-    }
-    break;
-  case 6:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R6;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D6;
-    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q6;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S6;
-    }
-    break;
-  case 7:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R7;
-    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
-    case ARM::DPR_VFP2RegClassID:
-      return ARM::D7;
-    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q7;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S7;
-    }
-    break;
-  case 8:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::R8;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D8;
-    case ARM::QPRRegClassID: return ARM::Q8;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S8;
-    }
-    break;
-  case 9:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::R9;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D9;
-    case ARM::QPRRegClassID: return ARM::Q9;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S9;
-    }
-    break;
-  case 10:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::R10;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D10;
-    case ARM::QPRRegClassID: return ARM::Q10;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S10;
-    }
-    break;
-  case 11:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::R11;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D11;
-    case ARM::QPRRegClassID: return ARM::Q11;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S11;
-    }
-    break;
-  case 12:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::R12;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D12;
-    case ARM::QPRRegClassID: return ARM::Q12;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S12;
-    }
-    break;
-  case 13:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::SP;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D13;
-    case ARM::QPRRegClassID: return ARM::Q13;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S13;
-    }
-    break;
-  case 14:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::LR;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D14;
-    case ARM::QPRRegClassID: return ARM::Q14;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S14;
-    }
-    break;
-  case 15:
-    switch (RegClassID) {
-    case ARM::GPRRegClassID: return ARM::PC;
-    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D15;
-    case ARM::QPRRegClassID: return ARM::Q15;
-    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S15;
-    }
-    break;
-  case 16:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D16;
-    case ARM::SPRRegClassID: return ARM::S16;
-    }
-    break;
-  case 17:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D17;
-    case ARM::SPRRegClassID: return ARM::S17;
-    }
-    break;
-  case 18:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D18;
-    case ARM::SPRRegClassID: return ARM::S18;
-    }
-    break;
-  case 19:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D19;
-    case ARM::SPRRegClassID: return ARM::S19;
-    }
-    break;
-  case 20:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D20;
-    case ARM::SPRRegClassID: return ARM::S20;
-    }
-    break;
-  case 21:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D21;
-    case ARM::SPRRegClassID: return ARM::S21;
-    }
-    break;
-  case 22:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D22;
-    case ARM::SPRRegClassID: return ARM::S22;
-    }
-    break;
-  case 23:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D23;
-    case ARM::SPRRegClassID: return ARM::S23;
-    }
-    break;
-  case 24:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D24;
-    case ARM::SPRRegClassID: return ARM::S24;
-    }
-    break;
-  case 25:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D25;
-    case ARM::SPRRegClassID: return ARM::S25;
-    }
-    break;
-  case 26:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D26;
-    case ARM::SPRRegClassID: return ARM::S26;
-    }
-    break;
-  case 27:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D27;
-    case ARM::SPRRegClassID: return ARM::S27;
-    }
-    break;
-  case 28:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D28;
-    case ARM::SPRRegClassID: return ARM::S28;
-    }
-    break;
-  case 29:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D29;
-    case ARM::SPRRegClassID: return ARM::S29;
-    }
-    break;
-  case 30:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D30;
-    case ARM::SPRRegClassID: return ARM::S30;
-    }
-    break;
-  case 31:
-    switch (RegClassID) {
-    case ARM::DPRRegClassID: return ARM::D31;
-    case ARM::SPRRegClassID: return ARM::S31;
-    }
-    break;
-  }
-  DEBUG(errs() << "Invalid (RegClassID, RawRegister) combination\n");
-  // Encoding error.  Mark the builder with error code != 0.
-  B->SetErr(-1);
-  return 0;
-}
-
-///////////////////////////////
-//                           //
-//     Utility Functions     //
-//                           //
-///////////////////////////////
-
-// Extract/Decode Rd: Inst{15-12}.
-static inline unsigned decodeRd(uint32_t insn) {
-  return (insn >> ARMII::RegRdShift) & ARMII::GPRRegMask;
-}
-
-// Extract/Decode Rn: Inst{19-16}.
-static inline unsigned decodeRn(uint32_t insn) {
-  return (insn >> ARMII::RegRnShift) & ARMII::GPRRegMask;
-}
-
-// Extract/Decode Rm: Inst{3-0}.
-static inline unsigned decodeRm(uint32_t insn) {
-  return (insn & ARMII::GPRRegMask);
-}
-
-// Extract/Decode Rs: Inst{11-8}.
-static inline unsigned decodeRs(uint32_t insn) {
-  return (insn >> ARMII::RegRsShift) & ARMII::GPRRegMask;
-}
-
-static inline unsigned getCondField(uint32_t insn) {
-  return (insn >> ARMII::CondShift);
-}
-
-static inline unsigned getIBit(uint32_t insn) {
-  return (insn >> ARMII::I_BitShift) & 1;
-}
-
-static inline unsigned getAM3IBit(uint32_t insn) {
-  return (insn >> ARMII::AM3_I_BitShift) & 1;
-}
-
-static inline unsigned getPBit(uint32_t insn) {
-  return (insn >> ARMII::P_BitShift) & 1;
-}
-
-static inline unsigned getUBit(uint32_t insn) {
-  return (insn >> ARMII::U_BitShift) & 1;
-}
-
-static inline unsigned getPUBits(uint32_t insn) {
-  return (insn >> ARMII::U_BitShift) & 3;
-}
-
-static inline unsigned getSBit(uint32_t insn) {
-  return (insn >> ARMII::S_BitShift) & 1;
-}
-
-static inline unsigned getWBit(uint32_t insn) {
-  return (insn >> ARMII::W_BitShift) & 1;
-}
-
-static inline unsigned getDBit(uint32_t insn) {
-  return (insn >> ARMII::D_BitShift) & 1;
-}
-
-static inline unsigned getNBit(uint32_t insn) {
-  return (insn >> ARMII::N_BitShift) & 1;
-}
-
-static inline unsigned getMBit(uint32_t insn) {
-  return (insn >> ARMII::M_BitShift) & 1;
-}
-
-// See A8.4 Shifts applied to a register.
-//     A8.4.2 Register controlled shifts.
-//
-// getShiftOpcForBits - getShiftOpcForBits translates from the ARM encoding bits
-// into llvm enums for shift opcode.  The API clients should pass in the value
-// encoded with two bits, so the assert stays to signal a wrong API usage.
-//
-// A8-12: DecodeRegShift()
-static inline ARM_AM::ShiftOpc getShiftOpcForBits(unsigned bits) {
-  switch (bits) {
-  default: assert(0 && "No such value"); return ARM_AM::no_shift;
-  case 0:  return ARM_AM::lsl;
-  case 1:  return ARM_AM::lsr;
-  case 2:  return ARM_AM::asr;
-  case 3:  return ARM_AM::ror;
-  }
-}
-
-// See A8.4 Shifts applied to a register.
-//     A8.4.1 Constant shifts.
-//
-// getImmShiftSE - getImmShiftSE translates from the raw ShiftOpc and raw Imm5
-// encodings into the intended ShiftOpc and shift amount.
-//
-// A8-11: DecodeImmShift()
-static inline void getImmShiftSE(ARM_AM::ShiftOpc &ShOp, unsigned &ShImm) {
-  if (ShImm != 0)
-    return;
-  switch (ShOp) {
-  case ARM_AM::no_shift:
-  case ARM_AM::rrx:
-    break;
-  case ARM_AM::lsl:
-    ShOp = ARM_AM::no_shift;
-    break;
-  case ARM_AM::lsr:
-  case ARM_AM::asr:
-    ShImm = 32;
-    break;
-  case ARM_AM::ror:
-    ShOp = ARM_AM::rrx;
-    break;
-  }
-}
-
-// getAMSubModeForBits - getAMSubModeForBits translates from the ARM encoding
-// bits Inst{24-23} (P(24) and U(23)) into llvm enums for AMSubMode.  The API
-// clients should pass in the value encoded with two bits, so the assert stays
-// to signal a wrong API usage.
-static inline ARM_AM::AMSubMode getAMSubModeForBits(unsigned bits) {
-  switch (bits) {
-  default: assert(0 && "No such value"); return ARM_AM::bad_am_submode;
-  case 1:  return ARM_AM::ia;   // P=0 U=1
-  case 3:  return ARM_AM::ib;   // P=1 U=1
-  case 0:  return ARM_AM::da;   // P=0 U=0
-  case 2:  return ARM_AM::db;   // P=1 U=0
-  }
-}
-
-////////////////////////////////////////////
-//                                        //
-//    Disassemble function definitions    //
-//                                        //
-////////////////////////////////////////////
-
-/// There is a separate Disassemble*Frm function entry for disassembly of an ARM
-/// instr into a list of MCOperands in the appropriate order, with possible dst,
-/// followed by possible src(s).
-///
-/// The processing of the predicate, and the 'S' modifier bit, if MI modifies
-/// the CPSR, is factored into ARMBasicMCBuilder's method named
-/// TryPredicateAndSBitModifier.
-
-static bool DisassemblePseudo(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
-
-  assert(0 && "Unexpected pseudo instruction!");
-  return false;
-}
-
-// A8.6.94 MLA
-// if d == 15 || n == 15 || m == 15 || a == 15 then UNPREDICTABLE;
-//
-// A8.6.105 MUL
-// if d == 15 || n == 15 || m == 15 then UNPREDICTABLE;
-//
-// A8.6.246 UMULL
-// if dLo == 15 || dHi == 15 || n == 15 || m == 15 then UNPREDICTABLE;
-// if dHi == dLo then UNPREDICTABLE;
-static bool BadRegsMulFrm(unsigned Opcode, uint32_t insn) {
-  unsigned R19_16 = slice(insn, 19, 16);
-  unsigned R15_12 = slice(insn, 15, 12);
-  unsigned R11_8  = slice(insn, 11, 8);
-  unsigned R3_0   = slice(insn, 3, 0);
-  switch (Opcode) {
-  default:
-    // Did we miss an opcode?
-    DEBUG(errs() << "BadRegsMulFrm: unexpected opcode!");
-    return false;
-  case ARM::MLA:     case ARM::MLS:     case ARM::SMLABB:  case ARM::SMLABT:
-  case ARM::SMLATB:  case ARM::SMLATT:  case ARM::SMLAWB:  case ARM::SMLAWT:
-  case ARM::SMMLA:   case ARM::SMMLAR:  case ARM::SMMLS:   case ARM::SMMLSR:
-  case ARM::USADA8:
-    if (R19_16 == 15 || R15_12 == 15 || R11_8 == 15 || R3_0 == 15)
-      return true;
-    return false;
-  case ARM::MUL:     case ARM::SMMUL:   case ARM::SMMULR:
-  case ARM::SMULBB:  case ARM::SMULBT:  case ARM::SMULTB:  case ARM::SMULTT:
-  case ARM::SMULWB:  case ARM::SMULWT:  case ARM::SMUAD:   case ARM::SMUADX:
-  // A8.6.167 SMLAD & A8.6.172 SMLSD
-  case ARM::SMLAD:   case ARM::SMLADX:  case ARM::SMLSD:   case ARM::SMLSDX:
-  case ARM::USAD8:
-    if (R19_16 == 15 || R11_8 == 15 || R3_0 == 15)
-      return true;
-    return false;
-  case ARM::SMLAL:   case ARM::SMULL:   case ARM::UMAAL:   case ARM::UMLAL:
-  case ARM::UMULL:
-  case ARM::SMLALBB: case ARM::SMLALBT: case ARM::SMLALTB: case ARM::SMLALTT:
-  case ARM::SMLALD:  case ARM::SMLALDX: case ARM::SMLSLD:  case ARM::SMLSLDX:
-    if (R19_16 == 15 || R15_12 == 15 || R11_8 == 15 || R3_0 == 15)
-      return true;
-    if (R19_16 == R15_12)
-      return true;
-    return false;;
-  }
-}
-
-// Multiply Instructions.
-// MLA, MLS, SMLABB, SMLABT, SMLATB, SMLATT, SMLAWB, SMLAWT, SMMLA, SMMLAR,
-// SMMLS, SMMLAR, SMLAD, SMLADX, SMLSD, SMLSDX, and USADA8 (for convenience):
-//     Rd{19-16} Rn{3-0} Rm{11-8} Ra{15-12}
-// But note that register checking for {SMLAD, SMLADX, SMLSD, SMLSDX} is
-// only for {d, n, m}.
-//
-// MUL, SMMUL, SMMULR, SMULBB, SMULBT, SMULTB, SMULTT, SMULWB, SMULWT, SMUAD,
-// SMUADX, and USAD8 (for convenience):
-//     Rd{19-16} Rn{3-0} Rm{11-8}
-//
-// SMLAL, SMULL, UMAAL, UMLAL, UMULL, SMLALBB, SMLALBT, SMLALTB, SMLALTT,
-// SMLALD, SMLADLX, SMLSLD, SMLSLDX:
-//     RdLo{15-12} RdHi{19-16} Rn{3-0} Rm{11-8}
-//
-// The mapping of the multiply registers to the "regular" ARM registers, where
-// there are convenience decoder functions, is:
-//
-// Inst{15-12} => Rd
-// Inst{19-16} => Rn
-// Inst{3-0} => Rm
-// Inst{11-8} => Rs
-static bool DisassembleMulFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  unsigned short NumDefs = MCID.getNumDefs();
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumDefs > 0 && "NumDefs should be greater than 0 for MulFrm");
-  assert(NumOps >= 3
-         && OpInfo[0].RegClass == ARM::GPRRegClassID
-         && OpInfo[1].RegClass == ARM::GPRRegClassID
-         && OpInfo[2].RegClass == ARM::GPRRegClassID
-         && "Expect three register operands");
-
-  // Sanity check for the register encodings.
-  if (BadRegsMulFrm(Opcode, insn))
-    return false;
-
-  // Instructions with two destination registers have RdLo{15-12} first.
-  if (NumDefs == 2) {
-    assert(NumOps >= 4 && OpInfo[3].RegClass == ARM::GPRRegClassID &&
-           "Expect 4th register operand");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn))));
-    ++OpIdx;
-  }
-
-  // The destination register: RdHi{19-16} or Rd{19-16}.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-
-  // The two src regsiters: Rn{3-0}, then Rm{11-8}.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRm(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRs(insn))));
-  OpIdx += 3;
-
-  // Many multiply instructions (e.g., MLA) have three src registers.
-  // The third register operand is Ra{15-12}.
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn))));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// Helper routines for disassembly of coprocessor instructions.
-
-static bool LdStCopOpcode(unsigned Opcode) {
-  if ((Opcode >= ARM::LDC2L_OFFSET && Opcode <= ARM::LDC_PRE) ||
-      (Opcode >= ARM::STC2L_OFFSET && Opcode <= ARM::STC_PRE))
-    return true;
-  return false;
-}
-static bool CoprocessorOpcode(unsigned Opcode) {
-  if (LdStCopOpcode(Opcode))
-    return true;
-
-  switch (Opcode) {
-  default:
-    return false;
-  case ARM::CDP:  case ARM::CDP2:
-  case ARM::MCR:  case ARM::MCR2:  case ARM::MRC:  case ARM::MRC2:
-  case ARM::MCRR: case ARM::MCRR2: case ARM::MRRC: case ARM::MRRC2:
-    return true;
-  }
-}
-static inline unsigned GetCoprocessor(uint32_t insn) {
-  return slice(insn, 11, 8);
-}
-static inline unsigned GetCopOpc1(uint32_t insn, bool CDP) {
-  return CDP ? slice(insn, 23, 20) : slice(insn, 23, 21);
-}
-static inline unsigned GetCopOpc2(uint32_t insn) {
-  return slice(insn, 7, 5);
-}
-static inline unsigned GetCopOpc(uint32_t insn) {
-  return slice(insn, 7, 4);
-}
-// Most of the operands are in immediate forms, except Rd and Rn, which are ARM
-// core registers.
-//
-// CDP, CDP2:                cop opc1 CRd CRn CRm opc2
-//
-// MCR, MCR2, MRC, MRC2:     cop opc1 Rd CRn CRm opc2
-//
-// MCRR, MCRR2, MRRC, MRRc2: cop opc Rd Rn CRm
-//
-// LDC_OFFSET, LDC_PRE, LDC_POST: cop CRd Rn R0 [+/-]imm8:00
-// and friends
-// STC_OFFSET, STC_PRE, STC_POST: cop CRd Rn R0 [+/-]imm8:00
-// and friends
-//                                        <-- addrmode2 -->
-//
-// LDC_OPTION:                    cop CRd Rn imm8
-// and friends
-// STC_OPTION:                    cop CRd Rn imm8
-// and friends
-//
-static bool DisassembleCoprocessor(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 4 && "Num of operands >= 4 for coprocessor instr");
-
-  unsigned &OpIdx = NumOpsAdded;
-  // A8.6.92
-  // if coproc == '101x' then SEE "Advanced SIMD and VFP"
-  // But since the special instructions have more explicit encoding bits
-  // specified, if coproc == 10 or 11, we should reject it as invalid.
-  unsigned coproc = GetCoprocessor(insn);
-  if ((Opcode == ARM::MCR || Opcode == ARM::MCRR ||
-       Opcode == ARM::MRC || Opcode == ARM::MRRC) &&
-      (coproc == 10 || coproc == 11)) {
-    DEBUG(errs() << "Encoding error: coproc == 10 or 11 for MCR[R]/MR[R]C\n");
-    return false;
-  }
-
-  bool OneCopOpc = (Opcode == ARM::MCRR || Opcode == ARM::MCRR2 ||
-                    Opcode == ARM::MRRC || Opcode == ARM::MRRC2);
-
-  // CDP/CDP2 has no GPR operand; the opc1 operand is also wider (Inst{23-20}).
-  bool NoGPR = (Opcode == ARM::CDP || Opcode == ARM::CDP2);
-  bool LdStCop = LdStCopOpcode(Opcode);
-  bool RtOut = (Opcode == ARM::MRC || Opcode == ARM::MRC2);
-
-  OpIdx = 0;
-
-  if (RtOut) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn))));
-    ++OpIdx;
-  }
-  MI.addOperand(MCOperand::CreateImm(coproc));
-  ++OpIdx;
-
-  if (LdStCop) {
-    // Unindex if P:W = 0b00 --> _OPTION variant
-    unsigned PW = getPBit(insn) << 1 | getWBit(insn);
-
-    MI.addOperand(MCOperand::CreateImm(decodeRd(insn)));
-
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    OpIdx += 2;
-
-    if (PW) {
-      MI.addOperand(MCOperand::CreateReg(0));
-      ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
-      const MCInstrDesc &MCID = ARMInsts[Opcode];
-      unsigned IndexMode =
-                 (MCID.TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
-      unsigned Offset = ARM_AM::getAM2Opc(AddrOpcode, slice(insn, 7, 0) << 2,
-                                          ARM_AM::no_shift, IndexMode);
-      MI.addOperand(MCOperand::CreateImm(Offset));
-      OpIdx += 2;
-    } else {
-      MI.addOperand(MCOperand::CreateImm(slice(insn, 7, 0)));
-      ++OpIdx;
-    }
-  } else {
-    MI.addOperand(MCOperand::CreateImm(OneCopOpc ? GetCopOpc(insn)
-                                                 : GetCopOpc1(insn, NoGPR)));
-    ++OpIdx;
-
-    if (!RtOut) {
-      MI.addOperand(NoGPR ? MCOperand::CreateImm(decodeRd(insn))
-                          : MCOperand::CreateReg(
-                                getRegisterEnum(B, ARM::GPRRegClassID,
-                                                decodeRd(insn))));
-      ++OpIdx;
-    }
-
-    MI.addOperand(OneCopOpc ? MCOperand::CreateReg(
-                                getRegisterEnum(B, ARM::GPRRegClassID,
-                                                decodeRn(insn)))
-                            : MCOperand::CreateImm(decodeRn(insn)));
-
-    MI.addOperand(MCOperand::CreateImm(decodeRm(insn)));
-
-    OpIdx += 2;
-
-    if (!OneCopOpc) {
-      MI.addOperand(MCOperand::CreateImm(GetCopOpc2(insn)));
-      ++OpIdx;
-    }
-  }
-
-  return true;
-}
-
-// Branch Instructions.
-// BL: SignExtend(Imm24:'00', 32)
-// Bcc, BL_pred: SignExtend(Imm24:'00', 32) Pred0 Pred1
-// SMC: ZeroExtend(imm4, 32)
-// SVC: ZeroExtend(Imm24, 32)
-//
-// Various coprocessor instructions are assigned BrFrm arbitrarily.
-// Delegates to DisassembleCoprocessor() helper function.
-//
-// MRS/MRSsys: Rd
-// MSR/MSRsys: Rm mask=Inst{19-16}
-// BXJ:        Rm
-// MSRi/MSRsysi: so_imm
-// SRSW/SRS: ldstm_mode:$amode mode_imm
-// RFEW/RFE: ldstm_mode:$amode Rn
-static bool DisassembleBrFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  if (CoprocessorOpcode(Opcode))
-    return DisassembleCoprocessor(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  // MRS and MRSsys take one GPR reg Rd.
-  if (Opcode == ARM::MRS || Opcode == ARM::MRSsys) {
-    assert(NumOps >= 1 && OpInfo[0].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn))));
-    NumOpsAdded = 1;
-    return true;
-  }
-  // BXJ takes one GPR reg Rm.
-  if (Opcode == ARM::BXJ) {
-    assert(NumOps >= 1 && OpInfo[0].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    NumOpsAdded = 1;
-    return true;
-  }
-  // MSR take a mask, followed by one GPR reg Rm. The mask contains the R Bit in
-  // bit 4, and the special register fields in bits 3-0.
-  if (Opcode == ARM::MSR) {
-    assert(NumOps >= 1 && OpInfo[1].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 22, 22) << 4 /* R Bit */ |
-                                       slice(insn, 19, 16) /* Special Reg */ ));
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    NumOpsAdded = 2;
-    return true;
-  }
-  // MSRi take a mask, followed by one so_imm operand. The mask contains the
-  // R Bit in bit 4, and the special register fields in bits 3-0.
-  if (Opcode == ARM::MSRi) {
-    // A5.2.11 MSR (immediate), and hints & B6.1.6 MSR (immediate)
-    // The hints instructions have more specific encodings, so if mask == 0,
-    // we should reject this as an invalid instruction.
-    if (slice(insn, 19, 16) == 0)
-      return false;
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 22, 22) << 4 /* R Bit */ |
-                                       slice(insn, 19, 16) /* Special Reg */ ));
-    // SOImm is 4-bit rotate amount in bits 11-8 with 8-bit imm in bits 7-0.
-    // A5.2.4 Rotate amount is twice the numeric value of Inst{11-8}.
-    // See also ARMAddressingModes.h: getSOImmValImm() and getSOImmValRot().
-    unsigned Rot = (insn >> ARMII::SoRotImmShift) & 0xF;
-    unsigned Imm = insn & 0xFF;
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::rotr32(Imm, 2*Rot)));
-    NumOpsAdded = 2;
-    return true;
-  }
-  if (Opcode == ARM::SRSW || Opcode == ARM::SRS ||
-      Opcode == ARM::RFEW || Opcode == ARM::RFE) {
-    ARM_AM::AMSubMode SubMode = getAMSubModeForBits(getPUBits(insn));
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM4ModeImm(SubMode)));
-
-    if (Opcode == ARM::SRSW || Opcode == ARM::SRS)
-      MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));
-    else
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         decodeRn(insn))));
-    NumOpsAdded = 3;
-    return true;
-  }
-
-  assert((Opcode == ARM::Bcc || Opcode == ARM::BL || Opcode == ARM::BL_pred
-          || Opcode == ARM::SMC || Opcode == ARM::SVC) &&
-         "Unexpected Opcode");
-
-  assert(NumOps >= 1 && OpInfo[0].RegClass < 0 && "Imm operand expected");
-
-  int Imm32 = 0;
-  if (Opcode == ARM::SMC) {
-    // ZeroExtend(imm4, 32) where imm24 = Inst{3-0}.
-    Imm32 = slice(insn, 3, 0);
-  } else if (Opcode == ARM::SVC) {
-    // ZeroExtend(imm24, 32) where imm24 = Inst{23-0}.
-    Imm32 = slice(insn, 23, 0);
-  } else {
-    // SignExtend(imm24:'00', 32) where imm24 = Inst{23-0}.
-    unsigned Imm26 = slice(insn, 23, 0) << 2;
-    //Imm32 = signextend<signed int, 26>(Imm26);
-    Imm32 = SignExtend32<26>(Imm26);
-  }
-
-  MI.addOperand(MCOperand::CreateImm(Imm32));
-  NumOpsAdded = 1;
-
-  return true;
-}
-
-// Misc. Branch Instructions.
-// BX_RET, MOVPCLR
-// BLX, BLX_pred, BX, BX_pred
-// BLXi
-static bool DisassembleBrMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  // BX_RET and MOVPCLR have only two predicate operands; do an early return.
-  if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR)
-    return true;
-
-  // BLX and BX take one GPR reg.
-  if (Opcode == ARM::BLX || Opcode == ARM::BLX_pred ||
-      Opcode == ARM::BX || Opcode == ARM::BX_pred) {
-    assert(NumOps >= 1 && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    OpIdx = 1;
-    return true;
-  }
-
-  // BLXi takes imm32 (the PC offset).
-  if (Opcode == ARM::BLXi) {
-    assert(NumOps >= 1 && OpInfo[0].RegClass < 0 && "Imm operand expected");
-    // SignExtend(imm24:H:'0', 32) where imm24 = Inst{23-0} and H = Inst{24}.
-    unsigned Imm26 = slice(insn, 23, 0) << 2 | slice(insn, 24, 24) << 1;
-    int Imm32 = SignExtend32<26>(Imm26);
-    MI.addOperand(MCOperand::CreateImm(Imm32));
-    OpIdx = 1;
-    return true;
-  }
-
-  return false;
-}
-
-static inline bool getBFCInvMask(uint32_t insn, uint32_t &mask) {
-  uint32_t lsb = slice(insn, 11, 7);
-  uint32_t msb = slice(insn, 20, 16);
-  uint32_t Val = 0;
-  if (msb < lsb) {
-    DEBUG(errs() << "Encoding error: msb < lsb\n");
-    return false;
-  }
-
-  for (uint32_t i = lsb; i <= msb; ++i)
-    Val |= (1 << i);
-  mask = ~Val;
-  return true;
-}
-
-// Standard data-processing instructions allow PC as a register specifier,
-// but we should reject other DPFrm instructions with PC as registers.
-static bool BadRegsDPFrm(unsigned Opcode, uint32_t insn) {
-  switch (Opcode) {
-  default:
-    // Did we miss an opcode?
-    if (decodeRd(insn) == 15 || decodeRn(insn) == 15 || decodeRm(insn) == 15) {
-      DEBUG(errs() << "DPFrm with bad reg specifier(s)\n");
-      return true;
-    }
-  case ARM::ADCrr:  case ARM::ADDSrr: case ARM::ADDrr:  case ARM::ANDrr:
-  case ARM::BICrr:  case ARM::CMNzrr: case ARM::CMPrr:  case ARM::EORrr:
-  case ARM::ORRrr:  case ARM::RSBrr:  case ARM::RSCrr:  case ARM::SBCrr:
-  case ARM::SUBSrr: case ARM::SUBrr:  case ARM::TEQrr:  case ARM::TSTrr:
-    return false;
-  }
-}
-
-// A major complication is the fact that some of the saturating add/subtract
-// operations have Rd Rm Rn, instead of the "normal" Rd Rn Rm.
-// They are QADD, QDADD, QDSUB, and QSUB.
-static bool DisassembleDPFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  unsigned short NumDefs = MCID.getNumDefs();
-  bool isUnary = isUnaryDP(MCID.TSFlags);
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  // Disassemble register def if there is one.
-  if (NumDefs && (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID)) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn))));
-    ++OpIdx;
-  }
-
-  // Now disassemble the src operands.
-  if (OpIdx >= NumOps)
-    return false;
-
-  // Special-case handling of BFC/BFI/SBFX/UBFX.
-  if (Opcode == ARM::BFC || Opcode == ARM::BFI) {
-    // A8.6.17 BFC & A8.6.18 BFI
-    // Sanity check Rd.
-    if (decodeRd(insn) == 15)
-      return false;
-    MI.addOperand(MCOperand::CreateReg(0));
-    if (Opcode == ARM::BFI) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         decodeRm(insn))));
-      ++OpIdx;
-    }
-    uint32_t mask = 0;
-    if (!getBFCInvMask(insn, mask))
-      return false;
-
-    MI.addOperand(MCOperand::CreateImm(mask));
-    OpIdx += 2;
-    return true;
-  }
-  if (Opcode == ARM::SBFX || Opcode == ARM::UBFX) {
-    // Sanity check Rd and Rm.
-    if (decodeRd(insn) == 15 || decodeRm(insn) == 15)
-      return false;
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 11, 7)));
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 20, 16) + 1));
-    OpIdx += 3;
-    return true;
-  }
-
-  bool RmRn = (Opcode == ARM::QADD || Opcode == ARM::QDADD ||
-               Opcode == ARM::QDSUB || Opcode == ARM::QSUB);
-
-  // BinaryDP has an Rn operand.
-  if (!isUnary) {
-    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, ARM::GPRRegClassID,
-                                    RmRn ? decodeRm(insn) : decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  // If this is a two-address operand, skip it, e.g., MOVCCr operand 1.
-  if (isUnary && (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)) {
-    MI.addOperand(MCOperand::CreateReg(0));
-    ++OpIdx;
-  }
-
-  // Now disassemble operand 2.
-  if (OpIdx >= NumOps)
-    return false;
-
-  if (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) {
-    // We have a reg/reg form.
-    // Assert disabled because saturating operations, e.g., A8.6.127 QASX, are
-    // routed here as well.
-    // assert(getIBit(insn) == 0 && "I_Bit != '0' reg/reg form");
-    if (BadRegsDPFrm(Opcode, insn))
-      return false;
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, ARM::GPRRegClassID,
-                                    RmRn? decodeRn(insn) : decodeRm(insn))));
-    ++OpIdx;
-  } else if (Opcode == ARM::MOVi16 || Opcode == ARM::MOVTi16) {
-    // These two instructions don't allow d as 15.
-    if (decodeRd(insn) == 15)
-      return false;
-    // We have an imm16 = imm4:imm12 (imm4=Inst{19:16}, imm12 = Inst{11:0}).
-    assert(getIBit(insn) == 1 && "I_Bit != '1' reg/imm form");
-    unsigned Imm16 = slice(insn, 19, 16) << 12 | slice(insn, 11, 0);
-    if (!B->tryAddingSymbolicOperand(Imm16, 4, MI))
-      MI.addOperand(MCOperand::CreateImm(Imm16));
-    ++OpIdx;
-  } else {
-    // We have a reg/imm form.
-    // SOImm is 4-bit rotate amount in bits 11-8 with 8-bit imm in bits 7-0.
-    // A5.2.4 Rotate amount is twice the numeric value of Inst{11-8}.
-    // See also ARMAddressingModes.h: getSOImmValImm() and getSOImmValRot().
-    assert(getIBit(insn) == 1 && "I_Bit != '1' reg/imm form");
-    unsigned Rot = (insn >> ARMII::SoRotImmShift) & 0xF;
-    unsigned Imm = insn & 0xFF;
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::rotr32(Imm, 2*Rot)));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-static bool DisassembleDPSoRegFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  unsigned short NumDefs = MCID.getNumDefs();
-  bool isUnary = isUnaryDP(MCID.TSFlags);
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  // Disassemble register def if there is one.
-  if (NumDefs && (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID)) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn))));
-    ++OpIdx;
-  }
-
-  // Disassemble the src operands.
-  if (OpIdx >= NumOps)
-    return false;
-
-  // BinaryDP has an Rn operand.
-  if (!isUnary) {
-    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  // If this is a two-address operand, skip it, e.g., MOVCCs operand 1.
-  if (isUnary && (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)) {
-    MI.addOperand(MCOperand::CreateReg(0));
-    ++OpIdx;
-  }
-
-  // Disassemble operand 2, which consists of three components.
-  if (OpIdx + 2 >= NumOps)
-    return false;
-
-  assert((OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) &&
-         (OpInfo[OpIdx+1].RegClass == ARM::GPRRegClassID) &&
-         (OpInfo[OpIdx+2].RegClass < 0) &&
-         "Expect 3 reg operands");
-
-  // Register-controlled shifts have Inst{7} = 0 and Inst{4} = 1.
-  unsigned Rs = slice(insn, 4, 4);
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRm(insn))));
-  if (Rs) {
-    // If Inst{7} != 0, we should reject this insn as an invalid encoding.
-    if (slice(insn, 7, 7))
-      return false;
-
-    // A8.6.3 ADC (register-shifted register)
-    // if d == 15 || n == 15 || m == 15 || s == 15 then UNPREDICTABLE;
-    // 
-    // This also accounts for shift instructions (register) where, fortunately,
-    // Inst{19-16} = 0b0000.
-    // A8.6.89 LSL (register)
-    // if d == 15 || n == 15 || m == 15 then UNPREDICTABLE;
-    if (decodeRd(insn) == 15 || decodeRn(insn) == 15 ||
-        decodeRm(insn) == 15 || decodeRs(insn) == 15)
-      return false;
-    
-    // Register-controlled shifts: [Rm, Rs, shift].
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRs(insn))));
-    // Inst{6-5} encodes the shift opcode.
-    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(ShOp, 0)));
-  } else {
-    // Constant shifts: [Rm, reg0, shift_imm].
-    MI.addOperand(MCOperand::CreateReg(0)); // NoRegister
-    // Inst{6-5} encodes the shift opcode.
-    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
-    // Inst{11-7} encodes the imm5 shift amount.
-    unsigned ShImm = slice(insn, 11, 7);
-
-    // A8.4.1.  Possible rrx or shift amount of 32...
-    getImmShiftSE(ShOp, ShImm);
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(ShOp, ShImm)));
-  }
-  OpIdx += 3;
-
-  return true;
-}
-
-static bool BadRegsLdStFrm(unsigned Opcode, uint32_t insn, bool Store, bool WBack,
-                           bool Imm) {
-  const StringRef Name = ARMInsts[Opcode].Name;
-  unsigned Rt = decodeRd(insn);
-  unsigned Rn = decodeRn(insn);
-  unsigned Rm = decodeRm(insn);
-  unsigned P  = getPBit(insn);
-  unsigned W  = getWBit(insn);
-
-  if (Store) {
-    // Only STR (immediate, register) allows PC as the source.
-    if (Name.startswith("STRB") && Rt == 15) {
-      DEBUG(errs() << "if t == 15 then UNPREDICTABLE\n");
-      return true;
-    }
-    if (WBack && (Rn == 15 || Rn == Rt)) {
-      DEBUG(errs() << "if wback && (n == 15 || n == t) then UNPREDICTABLE\n");
-      return true;
-    }
-    if (!Imm && Rm == 15) {
-      DEBUG(errs() << "if m == 15 then UNPREDICTABLE\n");
-      return true;
-    }
-  } else {
-    // Only LDR (immediate, register) allows PC as the destination.
-    if (Name.startswith("LDRB") && Rt == 15) {
-      DEBUG(errs() << "if t == 15 then UNPREDICTABLE\n");
-      return true;
-    }
-    if (Imm) {
-      // Immediate
-      if (Rn == 15) {
-        // The literal form must be in offset mode; it's an encoding error
-        // otherwise.
-        if (!(P == 1 && W == 0)) {
-          DEBUG(errs() << "Ld literal form with !(P == 1 && W == 0)\n");
-          return true;
-        }
-        // LDRB (literal) does not allow PC as the destination.
-        if (Opcode != ARM::LDRi12 && Rt == 15) {
-          DEBUG(errs() << "if t == 15 then UNPREDICTABLE\n");
-          return true;
-        }
-      } else {
-        // Write back while Rn == Rt does not make sense.
-        if (WBack && (Rn == Rt)) {
-          DEBUG(errs() << "if wback && n == t then UNPREDICTABLE\n");
-          return true;
-        }
-      }
-    } else {
-      // Register
-      if (Rm == 15) {
-        DEBUG(errs() << "if m == 15 then UNPREDICTABLE\n");
-        return true;
-      }
-      if (WBack && (Rn == 15 || Rn == Rt)) {
-        DEBUG(errs() << "if wback && (n == 15 || n == t) then UNPREDICTABLE\n");
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
-static bool DisassembleLdStFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, bool isStore, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  bool isPrePost = isPrePostLdSt(MCID.TSFlags);
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  if (!OpInfo) return false;
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(((!isStore && MCID.getNumDefs() > 0) ||
-          (isStore && (MCID.getNumDefs() == 0 || isPrePost)))
-         && "Invalid arguments");
-
-  // Operand 0 of a pre- and post-indexed store is the address base writeback.
-  if (isPrePost && isStore) {
-    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  // Disassemble the dst/src operand.
-  if (OpIdx >= NumOps)
-    return false;
-
-  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-         "Reg operand expected");
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  ++OpIdx;
-
-  // After dst of a pre- and post-indexed load is the address base writeback.
-  if (isPrePost && !isStore) {
-    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  // Disassemble the base operand.
-  if (OpIdx >= NumOps)
-    return false;
-
-  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-         "Reg operand expected");
-  assert((!isPrePost || (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1))
-         && "Index mode or tied_to operand expected");
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  ++OpIdx;
-
-  // For reg/reg form, base reg is followed by +/- reg shop imm.
-  // For immediate form, it is followed by +/- imm12.
-  // See also ARMAddressingModes.h (Addressing Mode #2).
-  if (OpIdx + 1 >= NumOps)
-    return false;
-
-  if (BadRegsLdStFrm(Opcode, insn, isStore, isPrePost, getIBit(insn)==0))
-    return false;
-
-  ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
-  unsigned IndexMode =
-               (MCID.TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
-  if (getIBit(insn) == 0) {
-    // For pre- and post-indexed case, add a reg0 operand (Addressing Mode #2).
-    // Otherwise, skip the reg operand since for addrmode_imm12, Rn has already
-    // been populated.
-    if (isPrePost) {
-      MI.addOperand(MCOperand::CreateReg(0));
-      OpIdx += 1;
-    }
-
-    unsigned Imm12 = slice(insn, 11, 0);
-    if (Opcode == ARM::LDRBi12 || Opcode == ARM::LDRi12 ||
-        Opcode == ARM::STRBi12 || Opcode == ARM::STRi12) {
-      // Disassemble the 12-bit immediate offset, which is the second operand in
-      // $addrmode_imm12 => (ops GPR:$base, i32imm:$offsimm).    
-      int Offset = AddrOpcode == ARM_AM::add ? 1 * Imm12 : -1 * Imm12;
-      MI.addOperand(MCOperand::CreateImm(Offset));
-    } else {
-      // Disassemble the 12-bit immediate offset, which is the second operand in
-      // $am2offset => (ops GPR, i32imm).
-      unsigned Offset = ARM_AM::getAM2Opc(AddrOpcode, Imm12, ARM_AM::no_shift,
-                                          IndexMode);
-      MI.addOperand(MCOperand::CreateImm(Offset));
-    }
-    OpIdx += 1;
-  } else {
-    // If Inst{25} = 1 and Inst{4} != 0, we should reject this as invalid.
-    if (slice(insn,4,4) == 1)
-      return false;
-
-    // Disassemble the offset reg (Rm), shift type, and immediate shift length.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    // Inst{6-5} encodes the shift opcode.
-    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
-    // Inst{11-7} encodes the imm5 shift amount.
-    unsigned ShImm = slice(insn, 11, 7);
-
-    // A8.4.1.  Possible rrx or shift amount of 32...
-    getImmShiftSE(ShOp, ShImm);
-    MI.addOperand(MCOperand::CreateImm(
-                    ARM_AM::getAM2Opc(AddrOpcode, ShImm, ShOp, IndexMode)));
-    OpIdx += 2;
-  }
-
-  return true;
-}
-
-static bool DisassembleLdFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  return DisassembleLdStFrm(MI, Opcode, insn, NumOps, NumOpsAdded, false, B);
-}
-
-static bool DisassembleStFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  return DisassembleLdStFrm(MI, Opcode, insn, NumOps, NumOpsAdded, true, B);
-}
-
-static bool HasDualReg(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-  case ARM::LDRD: case ARM::LDRD_PRE: case ARM::LDRD_POST:
-  case ARM::STRD: case ARM::STRD_PRE: case ARM::STRD_POST:
-    return true;
-  }
-}
-
-static bool DisassembleLdStMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, bool isStore, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  bool isPrePost = isPrePostLdSt(MCID.TSFlags);
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  if (!OpInfo) return false;
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(((!isStore && MCID.getNumDefs() > 0) ||
-          (isStore && (MCID.getNumDefs() == 0 || isPrePost)))
-         && "Invalid arguments");
-
-  // Operand 0 of a pre- and post-indexed store is the address base writeback.
-  if (isPrePost && isStore) {
-    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  // Disassemble the dst/src operand.
-  if (OpIdx >= NumOps)
-    return false;
-
-  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-         "Reg operand expected");
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  ++OpIdx;
-
-  // Fill in LDRD and STRD's second operand Rt operand.
-  if (HasDualReg(Opcode)) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn) + 1)));
-    ++OpIdx;
-  }
-
-  // After dst of a pre- and post-indexed load is the address base writeback.
-  if (isPrePost && !isStore) {
-    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  // Disassemble the base operand.
-  if (OpIdx >= NumOps)
-    return false;
-
-  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-         "Reg operand expected");
-  assert((!isPrePost || (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1))
-         && "Offset mode or tied_to operand expected");
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  ++OpIdx;
-
-  // For reg/reg form, base reg is followed by +/- reg.
-  // For immediate form, it is followed by +/- imm8.
-  // See also ARMAddressingModes.h (Addressing Mode #3).
-  if (OpIdx + 1 >= NumOps)
-    return false;
-
-  assert((OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) &&
-         (OpInfo[OpIdx+1].RegClass < 0) &&
-         "Expect 1 reg operand followed by 1 imm operand");
-
-  ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
-  unsigned IndexMode =
-                 (MCID.TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
-  if (getAM3IBit(insn) == 1) {
-    MI.addOperand(MCOperand::CreateReg(0));
-
-    // Disassemble the 8-bit immediate offset.
-    unsigned Imm4H = (insn >> ARMII::ImmHiShift) & 0xF;
-    unsigned Imm4L = insn & 0xF;
-    unsigned Offset = ARM_AM::getAM3Opc(AddrOpcode, (Imm4H << 4) | Imm4L,
-                                        IndexMode);
-    MI.addOperand(MCOperand::CreateImm(Offset));
-  } else {
-    // Disassemble the offset reg (Rm).
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    unsigned Offset = ARM_AM::getAM3Opc(AddrOpcode, 0, IndexMode);
-    MI.addOperand(MCOperand::CreateImm(Offset));
-  }
-  OpIdx += 2;
-
-  return true;
-}
-
-static bool DisassembleLdMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  return DisassembleLdStMiscFrm(MI, Opcode, insn, NumOps, NumOpsAdded, false,
-                                B);
-}
-
-static bool DisassembleStMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  return DisassembleLdStMiscFrm(MI, Opcode, insn, NumOps, NumOpsAdded, true, B);
-}
-
-// The algorithm for disassembly of LdStMulFrm is different from others because
-// it explicitly populates the two predicate operands after the base register.
-// After that, we need to populate the reglist with each affected register
-// encoded as an MCOperand.
-static bool DisassembleLdStMulFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 4 && "LdStMulFrm expects NumOps >= 4");
-  NumOpsAdded = 0;
-
-  unsigned Base = getRegisterEnum(B, ARM::GPRRegClassID, decodeRn(insn));
-
-  // Writeback to base, if necessary.
-  if (Opcode == ARM::LDMIA_UPD || Opcode == ARM::STMIA_UPD ||
-      Opcode == ARM::LDMDA_UPD || Opcode == ARM::STMDA_UPD ||
-      Opcode == ARM::LDMDB_UPD || Opcode == ARM::STMDB_UPD ||
-      Opcode == ARM::LDMIB_UPD || Opcode == ARM::STMIB_UPD) {
-    MI.addOperand(MCOperand::CreateReg(Base));
-    ++NumOpsAdded;
-  }
-
-  // Add the base register operand.
-  MI.addOperand(MCOperand::CreateReg(Base));
-
-  // Handling the two predicate operands before the reglist.
-  int64_t CondVal = getCondField(insn);
-  if (CondVal == 0xF)
-    return false;
-  MI.addOperand(MCOperand::CreateImm(CondVal));
-  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
-
-  NumOpsAdded += 3;
-
-  // Fill the variadic part of reglist.
-  unsigned RegListBits = insn & ((1 << 16) - 1);
-  for (unsigned i = 0; i < 16; ++i) {
-    if ((RegListBits >> i) & 1) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         i)));
-      ++NumOpsAdded;
-    }
-  }
-
-  return true;
-}
-
-// LDREX, LDREXB, LDREXH: Rd Rn
-// LDREXD:                Rd Rd+1 Rn
-// STREX, STREXB, STREXH: Rd Rm Rn
-// STREXD:                Rd Rm Rm+1 Rn
-//
-// SWP, SWPB:             Rd Rm Rn
-static bool DisassembleLdStExFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2
-         && OpInfo[0].RegClass == ARM::GPRRegClassID
-         && OpInfo[1].RegClass == ARM::GPRRegClassID
-         && "Expect 2 reg operands");
-
-  bool isStore = slice(insn, 20, 20) == 0;
-  bool isDW = (Opcode == ARM::LDREXD || Opcode == ARM::STREXD);
-
-  // Add the destination operand.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  ++OpIdx;
-
-  // Store register Exclusive needs a source operand.
-  if (isStore) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    ++OpIdx;
-
-    if (isDW) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         decodeRm(insn)+1)));
-      ++OpIdx;
-    }
-  } else if (isDW) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRd(insn)+1)));
-    ++OpIdx;
-  }
-
-  // Finally add the pointer operand.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  ++OpIdx;
-
-  return true;
-}
-
-// Misc. Arithmetic Instructions.
-// CLZ: Rd Rm
-// PKHBT, PKHTB: Rd Rn Rm , LSL/ASR #imm5
-// RBIT, REV, REV16, REVSH: Rd Rm
-static bool DisassembleArithMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2
-         && OpInfo[0].RegClass == ARM::GPRRegClassID
-         && OpInfo[1].RegClass == ARM::GPRRegClassID
-         && "Expect 2 reg operands");
-
-  bool ThreeReg = NumOps > 2 && OpInfo[2].RegClass == ARM::GPRRegClassID;
-
-  // Sanity check the registers, which should not be 15.
-  if (decodeRd(insn) == 15 || decodeRm(insn) == 15)
-    return false;
-  if (ThreeReg && decodeRn(insn) == 15)
-    return false;
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  ++OpIdx;
-
-  if (ThreeReg) {
-    assert(NumOps >= 4 && "Expect >= 4 operands");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRm(insn))));
-  ++OpIdx;
-
-  // If there is still an operand info left which is an immediate operand, add
-  // an additional imm5 LSL/ASR operand.
-  if (ThreeReg && OpInfo[OpIdx].RegClass < 0
-      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-    // Extract the 5-bit immediate field Inst{11-7}.
-    unsigned ShiftAmt = (insn >> ARMII::ShiftShift) & 0x1F;
-    ARM_AM::ShiftOpc Opc = ARM_AM::no_shift;
-    if (Opcode == ARM::PKHBT)
-      Opc = ARM_AM::lsl;
-    else if (Opcode == ARM::PKHTB)
-      Opc = ARM_AM::asr;
-    getImmShiftSE(Opc, ShiftAmt);
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(Opc, ShiftAmt)));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-/// DisassembleSatFrm - Disassemble saturate instructions:
-/// SSAT, SSAT16, USAT, and USAT16.
-static bool DisassembleSatFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  // A8.6.183 SSAT
-  // if d == 15 || n == 15 then UNPREDICTABLE;
-  if (decodeRd(insn) == 15 || decodeRm(insn) == 15)
-    return false;
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  NumOpsAdded = MCID.getNumOperands() - 2; // ignore predicate operands
-
-  // Disassemble register def.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-
-  unsigned Pos = slice(insn, 20, 16);
-  if (Opcode == ARM::SSAT || Opcode == ARM::SSAT16)
-    Pos += 1;
-  MI.addOperand(MCOperand::CreateImm(Pos));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRm(insn))));
-
-  if (NumOpsAdded == 4) {
-    ARM_AM::ShiftOpc Opc = (slice(insn, 6, 6) != 0 ? ARM_AM::asr : ARM_AM::lsl);
-    // Inst{11-7} encodes the imm5 shift amount.
-    unsigned ShAmt = slice(insn, 11, 7);
-    if (ShAmt == 0) {
-      // A8.6.183.  Possible ASR shift amount of 32...
-      if (Opc == ARM_AM::asr)
-        ShAmt = 32;
-      else
-        Opc = ARM_AM::no_shift;
-    }
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(Opc, ShAmt)));
-  }
-  return true;
-}
-
-// Extend instructions.
-// SXT* and UXT*: Rd [Rn] Rm [rot_imm].
-// The 2nd operand register is Rn and the 3rd operand regsiter is Rm for the
-// three register operand form.  Otherwise, Rn=0b1111 and only Rm is used.
-static bool DisassembleExtFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  // A8.6.220 SXTAB
-  // if d == 15 || m == 15 then UNPREDICTABLE;
-  if (decodeRd(insn) == 15 || decodeRm(insn) == 15)
-    return false;
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2
-         && OpInfo[0].RegClass == ARM::GPRRegClassID
-         && OpInfo[1].RegClass == ARM::GPRRegClassID
-         && "Expect 2 reg operands");
-
-  bool ThreeReg = NumOps > 2 && OpInfo[2].RegClass == ARM::GPRRegClassID;
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  ++OpIdx;
-
-  if (ThreeReg) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRm(insn))));
-  ++OpIdx;
-
-  // If there is still an operand info left which is an immediate operand, add
-  // an additional rotate immediate operand.
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-    // Extract the 2-bit rotate field Inst{11-10}.
-    unsigned rot = (insn >> ARMII::ExtRotImmShift) & 3;
-    // Rotation by 8, 16, or 24 bits.
-    MI.addOperand(MCOperand::CreateImm(rot << 3));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-/////////////////////////////////////
-//                                 //
-//    Utility Functions For VFP    //
-//                                 //
-/////////////////////////////////////
-
-// Extract/Decode Dd/Sd:
-//
-// SP => d = UInt(Vd:D)
-// DP => d = UInt(D:Vd)
-static unsigned decodeVFPRd(uint32_t insn, bool isSPVFP) {
-  return isSPVFP ? (decodeRd(insn) << 1 | getDBit(insn))
-                 : (decodeRd(insn) | getDBit(insn) << 4);
-}
-
-// Extract/Decode Dn/Sn:
-//
-// SP => n = UInt(Vn:N)
-// DP => n = UInt(N:Vn)
-static unsigned decodeVFPRn(uint32_t insn, bool isSPVFP) {
-  return isSPVFP ? (decodeRn(insn) << 1 | getNBit(insn))
-                 : (decodeRn(insn) | getNBit(insn) << 4);
-}
-
-// Extract/Decode Dm/Sm:
-//
-// SP => m = UInt(Vm:M)
-// DP => m = UInt(M:Vm)
-static unsigned decodeVFPRm(uint32_t insn, bool isSPVFP) {
-  return isSPVFP ? (decodeRm(insn) << 1 | getMBit(insn))
-                 : (decodeRm(insn) | getMBit(insn) << 4);
-}
-
-// A7.5.1
-static APInt VFPExpandImm(unsigned char byte, unsigned N) {
-  assert(N == 32 || N == 64);
-
-  uint64_t Result;
-  unsigned bit6 = slice(byte, 6, 6);
-  if (N == 32) {
-    Result = slice(byte, 7, 7) << 31 | slice(byte, 5, 0) << 19;
-    if (bit6)
-      Result |= 0x1f << 25;
-    else
-      Result |= 0x1 << 30;
-  } else {
-    Result = (uint64_t)slice(byte, 7, 7) << 63 |
-             (uint64_t)slice(byte, 5, 0) << 48;
-    if (bit6)
-      Result |= 0xffULL << 54;
-    else
-      Result |= 0x1ULL << 62;
-  }
-  return APInt(N, Result);
-}
-
-// VFP Unary Format Instructions:
-//
-// VCMP[E]ZD, VCMP[E]ZS: compares one floating-point register with zero
-// VCVTDS, VCVTSD: converts between double-precision and single-precision
-// The rest of the instructions have homogeneous [VFP]Rd and [VFP]Rm registers.
-static bool DisassembleVFPUnaryFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 1 && "VFPUnaryFrm expects NumOps >= 1");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  unsigned RegClass = OpInfo[OpIdx].RegClass;
-  assert((RegClass == ARM::SPRRegClassID || RegClass == ARM::DPRRegClassID) &&
-         "Reg operand expected");
-  bool isSP = (RegClass == ARM::SPRRegClassID);
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass, decodeVFPRd(insn, isSP))));
-  ++OpIdx;
-
-  // Early return for compare with zero instructions.
-  if (Opcode == ARM::VCMPEZD || Opcode == ARM::VCMPEZS
-      || Opcode == ARM::VCMPZD || Opcode == ARM::VCMPZS)
-    return true;
-
-  RegClass = OpInfo[OpIdx].RegClass;
-  assert((RegClass == ARM::SPRRegClassID || RegClass == ARM::DPRRegClassID) &&
-         "Reg operand expected");
-  isSP = (RegClass == ARM::SPRRegClassID);
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass, decodeVFPRm(insn, isSP))));
-  ++OpIdx;
-
-  return true;
-}
-
-// All the instructions have homogeneous [VFP]Rd, [VFP]Rn, and [VFP]Rm regs.
-// Some of them have operand constraints which tie the first operand in the
-// InOperandList to that of the dst.  As far as asm printing is concerned, this
-// tied_to operand is simply skipped.
-static bool DisassembleVFPBinaryFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 3 && "VFPBinaryFrm expects NumOps >= 3");
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  unsigned RegClass = OpInfo[OpIdx].RegClass;
-  assert((RegClass == ARM::SPRRegClassID || RegClass == ARM::DPRRegClassID) &&
-         "Reg operand expected");
-  bool isSP = (RegClass == ARM::SPRRegClassID);
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass, decodeVFPRd(insn, isSP))));
-  ++OpIdx;
-
-  // Skip tied_to operand constraint.
-  if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1) {
-    assert(NumOps >= 4 && "Expect >=4 operands");
-    MI.addOperand(MCOperand::CreateReg(0));
-    ++OpIdx;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass, decodeVFPRn(insn, isSP))));
-  ++OpIdx;
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass, decodeVFPRm(insn, isSP))));
-  ++OpIdx;
-
-  return true;
-}
-
-// A8.6.295 vcvt (floating-point <-> integer)
-// Int to FP: VSITOD, VSITOS, VUITOD, VUITOS
-// FP to Int: VTOSI[Z|R]D, VTOSI[Z|R]S, VTOUI[Z|R]D, VTOUI[Z|R]S
-//
-// A8.6.297 vcvt (floating-point and fixed-point)
-// Dd|Sd Dd|Sd(TIED_TO) #fbits(= 16|32 - UInt(imm4:i))
-static bool DisassembleVFPConv1Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 2 && "VFPConv1Frm expects NumOps >= 2");
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  if (!OpInfo) return false;
-
-  bool SP = slice(insn, 8, 8) == 0; // A8.6.295 & A8.6.297
-  bool fixed_point = slice(insn, 17, 17) == 1; // A8.6.297
-  unsigned RegClassID = SP ? ARM::SPRRegClassID : ARM::DPRRegClassID;
-
-  if (fixed_point) {
-    // A8.6.297
-    assert(NumOps >= 3 && "Expect >= 3 operands");
-    int size = slice(insn, 7, 7) == 0 ? 16 : 32;
-    int fbits = size - (slice(insn,3,0) << 1 | slice(insn,5,5));
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, RegClassID,
-                                    decodeVFPRd(insn, SP))));
-
-    assert(MCID.getOperandConstraint(1, MCOI::TIED_TO) != -1 &&
-           "Tied to operand expected");
-    MI.addOperand(MI.getOperand(0));
-
-    assert(OpInfo[2].RegClass < 0 && !OpInfo[2].isPredicate() &&
-           !OpInfo[2].isOptionalDef() && "Imm operand expected");
-    MI.addOperand(MCOperand::CreateImm(fbits));
-
-    NumOpsAdded = 3;
-  } else {
-    // A8.6.295
-    // The Rd (destination) and Rm (source) bits have different interpretations
-    // depending on their single-precisonness.
-    unsigned d, m;
-    if (slice(insn, 18, 18) == 1) { // to_integer operation
-      d = decodeVFPRd(insn, true /* Is Single Precision */);
-      MI.addOperand(MCOperand::CreateReg(
-                      getRegisterEnum(B, ARM::SPRRegClassID, d)));
-      m = decodeVFPRm(insn, SP);
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassID, m)));
-    } else {
-      d = decodeVFPRd(insn, SP);
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassID, d)));
-      m = decodeVFPRm(insn, true /* Is Single Precision */);
-      MI.addOperand(MCOperand::CreateReg(
-                      getRegisterEnum(B, ARM::SPRRegClassID, m)));
-    }
-    NumOpsAdded = 2;
-  }
-
-  return true;
-}
-
-// VMOVRS - A8.6.330
-// Rt => Rd; Sn => UInt(Vn:N)
-static bool DisassembleVFPConv2Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 2 && "VFPConv2Frm expects NumOps >= 2");
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::SPRRegClassID,
-                                                     decodeVFPRn(insn, true))));
-  NumOpsAdded = 2;
-  return true;
-}
-
-// VMOVRRD - A8.6.332
-// Rt => Rd; Rt2 => Rn; Dm => UInt(M:Vm)
-//
-// VMOVRRS - A8.6.331
-// Rt => Rd; Rt2 => Rn; Sm => UInt(Vm:M); Sm1 = Sm+1
-static bool DisassembleVFPConv3Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 3 && "VFPConv3Frm expects NumOps >= 3");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  OpIdx = 2;
-
-  if (OpInfo[OpIdx].RegClass == ARM::SPRRegClassID) {
-    unsigned Sm = decodeVFPRm(insn, true);
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::SPRRegClassID,
-                                                       Sm)));
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::SPRRegClassID,
-                                                       Sm+1)));
-    OpIdx += 2;
-  } else {
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, ARM::DPRRegClassID,
-                                    decodeVFPRm(insn, false))));
-    ++OpIdx;
-  }
-  return true;
-}
-
-// VMOVSR - A8.6.330
-// Rt => Rd; Sn => UInt(Vn:N)
-static bool DisassembleVFPConv4Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 2 && "VFPConv4Frm expects NumOps >= 2");
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::SPRRegClassID,
-                                                     decodeVFPRn(insn, true))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  NumOpsAdded = 2;
-  return true;
-}
-
-// VMOVDRR - A8.6.332
-// Rt => Rd; Rt2 => Rn; Dm => UInt(M:Vm)
-//
-// VMOVRRS - A8.6.331
-// Rt => Rd; Rt2 => Rn; Sm => UInt(Vm:M); Sm1 = Sm+1
-static bool DisassembleVFPConv5Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 3 && "VFPConv5Frm expects NumOps >= 3");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  if (OpInfo[OpIdx].RegClass == ARM::SPRRegClassID) {
-    unsigned Sm = decodeVFPRm(insn, true);
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::SPRRegClassID,
-                                                       Sm)));
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::SPRRegClassID,
-                                                       Sm+1)));
-    OpIdx += 2;
-  } else {
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, ARM::DPRRegClassID,
-                                    decodeVFPRm(insn, false))));
-    ++OpIdx;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  OpIdx += 2;
-  return true;
-}
-
-// VFP Load/Store Instructions.
-// VLDRD, VLDRS, VSTRD, VSTRS
-static bool DisassembleVFPLdStFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 3 && "VFPLdStFrm expects NumOps >= 3");
-
-  bool isSPVFP = (Opcode == ARM::VLDRS || Opcode == ARM::VSTRS);
-  unsigned RegClassID = isSPVFP ? ARM::SPRRegClassID : ARM::DPRRegClassID;
-
-  // Extract Dd/Sd for operand 0.
-  unsigned RegD = decodeVFPRd(insn, isSPVFP);
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassID, RegD)));
-
-  unsigned Base = getRegisterEnum(B, ARM::GPRRegClassID, decodeRn(insn));
-  MI.addOperand(MCOperand::CreateReg(Base));
-
-  // Next comes the AM5 Opcode.
-  ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
-  unsigned char Imm8 = insn & 0xFF;
-  MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(AddrOpcode, Imm8)));
-
-  NumOpsAdded = 3;
-
-  return true;
-}
-
-// VFP Load/Store Multiple Instructions.
-// We have an optional write back reg, the base, and two predicate operands.
-// It is then followed by a reglist of either DPR(s) or SPR(s).
-//
-// VLDMD[_UPD], VLDMS[_UPD], VSTMD[_UPD], VSTMS[_UPD]
-static bool DisassembleVFPLdStMulFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 4 && "VFPLdStMulFrm expects NumOps >= 4");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  unsigned Base = getRegisterEnum(B, ARM::GPRRegClassID, decodeRn(insn));
-
-  // Writeback to base, if necessary.
-  if (Opcode == ARM::VLDMDIA_UPD || Opcode == ARM::VLDMSIA_UPD ||
-      Opcode == ARM::VLDMDDB_UPD || Opcode == ARM::VLDMSDB_UPD ||
-      Opcode == ARM::VSTMDIA_UPD || Opcode == ARM::VSTMSIA_UPD ||
-      Opcode == ARM::VSTMDDB_UPD || Opcode == ARM::VSTMSDB_UPD) {
-    MI.addOperand(MCOperand::CreateReg(Base));
-    ++OpIdx;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(Base));
-
-  // Handling the two predicate operands before the reglist.
-  int64_t CondVal = getCondField(insn);
-  if (CondVal == 0xF)
-    return false;
-  MI.addOperand(MCOperand::CreateImm(CondVal));
-  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
-
-  OpIdx += 3;
-
-  bool isSPVFP = (Opcode == ARM::VLDMSIA     ||
-                  Opcode == ARM::VLDMSIA_UPD || Opcode == ARM::VLDMSDB_UPD ||
-                  Opcode == ARM::VSTMSIA     ||
-                  Opcode == ARM::VSTMSIA_UPD || Opcode == ARM::VSTMSDB_UPD);
-  unsigned RegClassID = isSPVFP ? ARM::SPRRegClassID : ARM::DPRRegClassID;
-
-  // Extract Dd/Sd.
-  unsigned RegD = decodeVFPRd(insn, isSPVFP);
-
-  // Fill the variadic part of reglist.
-  unsigned char Imm8 = insn & 0xFF;
-  unsigned Regs = isSPVFP ? Imm8 : Imm8/2;
-
-  // Apply some sanity checks before proceeding.
-  if (Regs == 0 || (RegD + Regs) > 32 || (!isSPVFP && Regs > 16))
-    return false;
-
-  for (unsigned i = 0; i < Regs; ++i) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassID,
-                                                       RegD + i)));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// Misc. VFP Instructions.
-// FMSTAT (vmrs with Rt=0b1111, i.e., to apsr_nzcv and no register operand)
-// FCONSTD (DPR and a VFPf64Imm operand)
-// FCONSTS (SPR and a VFPf32Imm operand)
-// VMRS/VMSR (GPR operand)
-static bool DisassembleVFPMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  if (Opcode == ARM::FMSTAT)
-    return true;
-
-  assert(NumOps >= 2 && "VFPMiscFrm expects >=2 operands");
-
-  unsigned RegEnum = 0;
-  switch (OpInfo[0].RegClass) {
-  case ARM::DPRRegClassID:
-    RegEnum = getRegisterEnum(B, ARM::DPRRegClassID, decodeVFPRd(insn, false));
-    break;
-  case ARM::SPRRegClassID:
-    RegEnum = getRegisterEnum(B, ARM::SPRRegClassID, decodeVFPRd(insn, true));
-    break;
-  case ARM::GPRRegClassID:
-    RegEnum = getRegisterEnum(B, ARM::GPRRegClassID, decodeRd(insn));
-    break;
-  default:
-    assert(0 && "Invalid reg class id");
-    return false;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(RegEnum));
-  ++OpIdx;
-
-  // Extract/decode the f64/f32 immediate.
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-        && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-    // The asm syntax specifies the floating point value, not the 8-bit literal.
-    APInt immRaw = VFPExpandImm(slice(insn,19,16) << 4 | slice(insn, 3, 0),
-                             Opcode == ARM::FCONSTD ? 64 : 32);
-    APFloat immFP = APFloat(immRaw, true);
-    double imm = Opcode == ARM::FCONSTD ? immFP.convertToDouble() :
-      immFP.convertToFloat();
-    MI.addOperand(MCOperand::CreateFPImm(imm));
-
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// DisassembleThumbFrm() is defined in ThumbDisassemblerCore.h file.
-#include "ThumbDisassemblerCore.h"
-
-/////////////////////////////////////////////////////
-//                                                 //
-//     Utility Functions For ARM Advanced SIMD     //
-//                                                 //
-/////////////////////////////////////////////////////
-
-// The following NEON namings are based on A8.6.266 VABA, VABAL.  Notice that
-// A8.6.303 VDUP (ARM core register)'s D/Vd pair is the N/Vn pair of VABA/VABAL.
-
-// A7.3 Register encoding
-
-// Extract/Decode NEON D/Vd:
-//
-// Note that for quadword, Qd = UInt(D:Vd<3:1>) = Inst{22:15-13}, whereas for
-// doubleword, Dd = UInt(D:Vd).  We compensate for this difference by
-// handling it in the getRegisterEnum() utility function.
-// D = Inst{22}, Vd = Inst{15-12}
-static unsigned decodeNEONRd(uint32_t insn) {
-  return ((insn >> ARMII::NEON_D_BitShift) & 1) << 4
-    | ((insn >> ARMII::NEON_RegRdShift) & ARMII::NEONRegMask);
-}
-
-// Extract/Decode NEON N/Vn:
-//
-// Note that for quadword, Qn = UInt(N:Vn<3:1>) = Inst{7:19-17}, whereas for
-// doubleword, Dn = UInt(N:Vn).  We compensate for this difference by
-// handling it in the getRegisterEnum() utility function.
-// N = Inst{7}, Vn = Inst{19-16}
-static unsigned decodeNEONRn(uint32_t insn) {
-  return ((insn >> ARMII::NEON_N_BitShift) & 1) << 4
-    | ((insn >> ARMII::NEON_RegRnShift) & ARMII::NEONRegMask);
-}
-
-// Extract/Decode NEON M/Vm:
-//
-// Note that for quadword, Qm = UInt(M:Vm<3:1>) = Inst{5:3-1}, whereas for
-// doubleword, Dm = UInt(M:Vm).  We compensate for this difference by
-// handling it in the getRegisterEnum() utility function.
-// M = Inst{5}, Vm = Inst{3-0}
-static unsigned decodeNEONRm(uint32_t insn) {
-  return ((insn >> ARMII::NEON_M_BitShift) & 1) << 4
-    | ((insn >> ARMII::NEON_RegRmShift) & ARMII::NEONRegMask);
-}
-
-namespace {
-enum ElemSize {
-  ESizeNA = 0,
-  ESize8 = 8,
-  ESize16 = 16,
-  ESize32 = 32,
-  ESize64 = 64
-};
-} // End of unnamed namespace
-
-// size        field -> Inst{11-10}
-// index_align field -> Inst{7-4}
-//
-// The Lane Index interpretation depends on the Data Size:
-//   8  (encoded as size = 0b00) -> Index = index_align[3:1]
-//   16 (encoded as size = 0b01) -> Index = index_align[3:2]
-//   32 (encoded as size = 0b10) -> Index = index_align[3]
-//
-// Ref: A8.6.317 VLD4 (single 4-element structure to one lane).
-static unsigned decodeLaneIndex(uint32_t insn) {
-  unsigned size = insn >> 10 & 3;
-  assert((size == 0 || size == 1 || size == 2) &&
-         "Encoding error: size should be either 0, 1, or 2");
-
-  unsigned index_align = insn >> 4 & 0xF;
-  return (index_align >> 1) >> size;
-}
-
-// imm64 = AdvSIMDExpandImm(op, cmode, i:imm3:imm4)
-// op = Inst{5}, cmode = Inst{11-8}
-// i = Inst{24} (ARM architecture)
-// imm3 = Inst{18-16}, imm4 = Inst{3-0}
-// Ref: Table A7-15 Modified immediate values for Advanced SIMD instructions.
-static uint64_t decodeN1VImm(uint32_t insn, ElemSize esize) {
-  unsigned char op = (insn >> 5) & 1;
-  unsigned char cmode = (insn >> 8) & 0xF;
-  unsigned char Imm8 = ((insn >> 24) & 1) << 7 |
-                       ((insn >> 16) & 7) << 4 |
-                       (insn & 0xF);
-  return (op << 12) | (cmode << 8) | Imm8;
-}
-
-// A8.6.339 VMUL, VMULL (by scalar)
-// ESize16 => m = Inst{2-0} (Vm<2:0>) D0-D7
-// ESize32 => m = Inst{3-0} (Vm<3:0>) D0-D15
-static unsigned decodeRestrictedDm(uint32_t insn, ElemSize esize) {
-  switch (esize) {
-  case ESize16:
-    return insn & 7;
-  case ESize32:
-    return insn & 0xF;
-  default:
-    assert(0 && "Unreachable code!");
-    return 0;
-  }
-}
-
-// A8.6.339 VMUL, VMULL (by scalar)
-// ESize16 => index = Inst{5:3} (M:Vm<3>) D0-D7
-// ESize32 => index = Inst{5}   (M)       D0-D15
-static unsigned decodeRestrictedDmIndex(uint32_t insn, ElemSize esize) {
-  switch (esize) {
-  case ESize16:
-    return (((insn >> 5) & 1) << 1) | ((insn >> 3) & 1);
-  case ESize32:
-    return (insn >> 5) & 1;
-  default:
-    assert(0 && "Unreachable code!");
-    return 0;
-  }
-}
-
-// A8.6.296 VCVT (between floating-point and fixed-point, Advanced SIMD)
-// (64 - <fbits>) is encoded as imm6, i.e., Inst{21-16}.
-static unsigned decodeVCVTFractionBits(uint32_t insn) {
-  return 64 - ((insn >> 16) & 0x3F);
-}
-
-// A8.6.302 VDUP (scalar)
-// ESize8  => index = Inst{19-17}
-// ESize16 => index = Inst{19-18}
-// ESize32 => index = Inst{19}
-static unsigned decodeNVLaneDupIndex(uint32_t insn, ElemSize esize) {
-  switch (esize) {
-  case ESize8:
-    return (insn >> 17) & 7;
-  case ESize16:
-    return (insn >> 18) & 3;
-  case ESize32:
-    return (insn >> 19) & 1;
-  default:
-    assert(0 && "Unspecified element size!");
-    return 0;
-  }
-}
-
-// A8.6.328 VMOV (ARM core register to scalar)
-// A8.6.329 VMOV (scalar to ARM core register)
-// ESize8  => index = Inst{21:6-5}
-// ESize16 => index = Inst{21:6}
-// ESize32 => index = Inst{21}
-static unsigned decodeNVLaneOpIndex(uint32_t insn, ElemSize esize) {
-  switch (esize) {
-  case ESize8:
-    return ((insn >> 21) & 1) << 2 | ((insn >> 5) & 3);
-  case ESize16:
-    return ((insn >> 21) & 1) << 1 | ((insn >> 6) & 1);
-  case ESize32:
-    return ((insn >> 21) & 1);
-  default:
-    assert(0 && "Unspecified element size!");
-    return 0;
-  }
-}
-
-// Imm6 = Inst{21-16}, L = Inst{7}
-//
-// LeftShift == true (A8.6.367 VQSHL, A8.6.387 VSLI):
-// case L:imm6 of
-//   '0001xxx' => esize = 8; shift_amount = imm6 - 8
-//   '001xxxx' => esize = 16; shift_amount = imm6 - 16
-//   '01xxxxx' => esize = 32; shift_amount = imm6 - 32
-//   '1xxxxxx' => esize = 64; shift_amount = imm6
-//
-// LeftShift == false (A8.6.376 VRSHR, A8.6.368 VQSHRN):
-// case L:imm6 of
-//   '0001xxx' => esize = 8; shift_amount = 16 - imm6
-//   '001xxxx' => esize = 16; shift_amount = 32 - imm6
-//   '01xxxxx' => esize = 32; shift_amount = 64 - imm6
-//   '1xxxxxx' => esize = 64; shift_amount = 64 - imm6
-//
-static unsigned decodeNVSAmt(uint32_t insn, bool LeftShift) {
-  ElemSize esize = ESizeNA;
-  unsigned L = (insn >> 7) & 1;
-  unsigned imm6 = (insn >> 16) & 0x3F;
-  if (L == 0) {
-    if (imm6 >> 3 == 1)
-      esize = ESize8;
-    else if (imm6 >> 4 == 1)
-      esize = ESize16;
-    else if (imm6 >> 5 == 1)
-      esize = ESize32;
-    else
-      assert(0 && "Wrong encoding of Inst{7:21-16}!");
-  } else
-    esize = ESize64;
-
-  if (LeftShift)
-    return esize == ESize64 ? imm6 : (imm6 - esize);
-  else
-    return esize == ESize64 ? (esize - imm6) : (2*esize - imm6);
-}
-
-// A8.6.305 VEXT
-// Imm4 = Inst{11-8}
-static unsigned decodeN3VImm(uint32_t insn) {
-  return (insn >> 8) & 0xF;
-}
-
-// VLD*
-//   D[d] D[d2] ... Rn [TIED_TO Rn] align [Rm]
-// VLD*LN*
-//   D[d] D[d2] ... Rn [TIED_TO Rn] align [Rm] TIED_TO ... imm(idx)
-// VST*
-//   Rn [TIED_TO Rn] align [Rm] D[d] D[d2] ...
-// VST*LN*
-//   Rn [TIED_TO Rn] align [Rm] D[d] D[d2] ... [imm(idx)]
-//
-// Correctly set VLD*/VST*'s TIED_TO GPR, as the asm printer needs it.
-static bool DisassembleNLdSt0(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, bool Store, bool DblSpaced,
-    unsigned alignment, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-
-  // At least one DPR register plus addressing mode #6.
-  assert(NumOps >= 3 && "Expect >= 3 operands");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  // We have homogeneous NEON registers for Load/Store.
-  unsigned RegClass = 0;
-
-  // Double-spaced registers have increments of 2.
-  unsigned Inc = DblSpaced ? 2 : 1;
-
-  unsigned Rn = decodeRn(insn);
-  unsigned Rm = decodeRm(insn);
-  unsigned Rd = decodeNEONRd(insn);
-
-  // A7.7.1 Advanced SIMD addressing mode.
-  bool WB = Rm != 15;
-
-  // LLVM Addressing Mode #6.
-  unsigned RmEnum = 0;
-  if (WB && Rm != 13)
-    RmEnum = getRegisterEnum(B, ARM::GPRRegClassID, Rm);
-
-  if (Store) {
-    // Consume possible WB, AddrMode6, possible increment reg, the DPR/QPR's,
-    // then possible lane index.
-    assert(OpIdx < NumOps && OpInfo[0].RegClass == ARM::GPRRegClassID &&
-           "Reg operand expected");
-
-    if (WB) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         Rn)));
-      ++OpIdx;
-    }
-
-    assert((OpIdx+1) < NumOps && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           OpInfo[OpIdx + 1].RegClass < 0 && "Addrmode #6 Operands expected");
-    // addrmode6 := (ops GPR:$addr, i32imm)
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       Rn)));
-    MI.addOperand(MCOperand::CreateImm(alignment)); // Alignment
-    OpIdx += 2;
-
-    if (WB) {
-      MI.addOperand(MCOperand::CreateReg(RmEnum));
-      ++OpIdx;
-    }
-
-    assert(OpIdx < NumOps &&
-           (OpInfo[OpIdx].RegClass == ARM::DPRRegClassID ||
-            OpInfo[OpIdx].RegClass == ARM::QPRRegClassID) &&
-           "Reg operand expected");
-
-    RegClass = OpInfo[OpIdx].RegClass;
-    while (OpIdx < NumOps && (unsigned)OpInfo[OpIdx].RegClass == RegClass) {
-      MI.addOperand(MCOperand::CreateReg(
-                      getRegisterEnum(B, RegClass, Rd)));
-      Rd += Inc;
-      ++OpIdx;
-    }
-
-    // Handle possible lane index.
-    if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-        && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-      MI.addOperand(MCOperand::CreateImm(decodeLaneIndex(insn)));
-      ++OpIdx;
-    }
-
-  } else {
-    // Consume the DPR/QPR's, possible WB, AddrMode6, possible incrment reg,
-    // possible TIED_TO DPR/QPR's (ignored), then possible lane index.
-    RegClass = OpInfo[0].RegClass;
-
-    while (OpIdx < NumOps && (unsigned)OpInfo[OpIdx].RegClass == RegClass) {
-      MI.addOperand(MCOperand::CreateReg(
-                      getRegisterEnum(B, RegClass, Rd)));
-      Rd += Inc;
-      ++OpIdx;
-    }
-
-    if (WB) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         Rn)));
-      ++OpIdx;
-    }
-
-    assert((OpIdx+1) < NumOps && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
-           OpInfo[OpIdx + 1].RegClass < 0 && "Addrmode #6 Operands expected");
-    // addrmode6 := (ops GPR:$addr, i32imm)
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       Rn)));
-    MI.addOperand(MCOperand::CreateImm(alignment)); // Alignment
-    OpIdx += 2;
-
-    if (WB) {
-      MI.addOperand(MCOperand::CreateReg(RmEnum));
-      ++OpIdx;
-    }
-
-    while (OpIdx < NumOps && (unsigned)OpInfo[OpIdx].RegClass == RegClass) {
-      assert(MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1 &&
-             "Tied to operand expected");
-      MI.addOperand(MCOperand::CreateReg(0));
-      ++OpIdx;
-    }
-
-    // Handle possible lane index.
-    if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-        && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-      MI.addOperand(MCOperand::CreateImm(decodeLaneIndex(insn)));
-      ++OpIdx;
-    }
-  }
-
-  // Accessing registers past the end of the NEON register file is not
-  // defined.
-  if (Rd > 32)
-    return false;
-
-  return true;
-}
-
-// A8.6.308, A8.6.311, A8.6.314, A8.6.317.
-static bool Align4OneLaneInst(unsigned elem, unsigned size,
-    unsigned index_align, unsigned & alignment) {
-  unsigned bits = 0;
-  switch (elem) {
-  default:
-    return false;
-  case 1:
-    // A8.6.308
-    if (size == 0)
-      return slice(index_align, 0, 0) == 0;
-    else if (size == 1) {
-      bits = slice(index_align, 1, 0);
-      if (bits != 0 && bits != 1)
-        return false;
-      if (bits == 1)
-        alignment = 16;
-      return true;
-    } else if (size == 2) {
-      bits = slice(index_align, 2, 0);
-      if (bits != 0 && bits != 3)
-        return false;
-      if (bits == 3)
-        alignment = 32;
-      return true;;
-    }
-    return true;
-  case 2:
-    // A8.6.311
-    if (size == 0) {
-      if (slice(index_align, 0, 0) == 1)
-        alignment = 16;
-      return true;
-    } if (size == 1) {
-      if (slice(index_align, 0, 0) == 1)
-        alignment = 32;
-      return true;
-    } else if (size == 2) {
-      if (slice(index_align, 1, 1) != 0)
-        return false;
-      if (slice(index_align, 0, 0) == 1)
-        alignment = 64;
-      return true;;
-    }
-    return true;
-  case 3:
-    // A8.6.314
-    if (size == 0) {
-      if (slice(index_align, 0, 0) != 0)
-        return false;
-      return true;
-    } if (size == 1) {
-      if (slice(index_align, 0, 0) != 0)
-        return false;
-      return true;
-      return true;
-    } else if (size == 2) {
-      if (slice(index_align, 1, 0) != 0)
-        return false;
-      return true;;
-    }
-    return true;
-  case 4:
-    // A8.6.317
-    if (size == 0) {
-      if (slice(index_align, 0, 0) == 1)
-        alignment = 32;
-      return true;
-    } if (size == 1) {
-      if (slice(index_align, 0, 0) == 1)
-        alignment = 64;
-      return true;
-    } else if (size == 2) {
-      bits = slice(index_align, 1, 0);
-      if (bits == 3)
-        return false;
-      if (bits == 1)
-        alignment = 64;
-      else if (bits == 2)
-        alignment = 128;
-      return true;;
-    }
-    return true;
-  }
-}
-
-// A7.7
-// If L (Inst{21}) == 0, store instructions.
-// Find out about double-spaced-ness of the Opcode and pass it on to
-// DisassembleNLdSt0().
-static bool DisassembleNLdSt(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const StringRef Name = ARMInsts[Opcode].Name;
-  bool DblSpaced = false;
-  // 0 represents standard alignment, i.e., unaligned data access.
-  unsigned alignment = 0;
-
-  unsigned elem = 0; // legal values: {1, 2, 3, 4}
-  if (Name.startswith("VST1") || Name.startswith("VLD1"))
-    elem = 1;
-
-  if (Name.startswith("VST2") || Name.startswith("VLD2"))
-    elem = 2;
-
-  if (Name.startswith("VST3") || Name.startswith("VLD3"))
-    elem = 3;
-
-  if (Name.startswith("VST4") || Name.startswith("VLD4"))
-    elem = 4;
-
-  if (Name.find("LN") != std::string::npos) {
-    // To one lane instructions.
-    // See, for example, 8.6.317 VLD4 (single 4-element structure to one lane).
-
-    // Utility function takes number of elements, size, and index_align.
-    if (!Align4OneLaneInst(elem,
-                           slice(insn, 11, 10),
-                           slice(insn, 7, 4),
-                           alignment))
-      return false;
-
-    // <size> == 16 && Inst{5} == 1 --> DblSpaced = true
-    if (Name.endswith("16") || Name.endswith("16_UPD"))
-      DblSpaced = slice(insn, 5, 5) == 1;
-
-    // <size> == 32 && Inst{6} == 1 --> DblSpaced = true
-    if (Name.endswith("32") || Name.endswith("32_UPD"))
-      DblSpaced = slice(insn, 6, 6) == 1;
-  } else if (Name.find("DUP") != std::string::npos) {
-    // Single element (or structure) to all lanes.
-    // Inst{9-8} encodes the number of element(s) in the structure, with:
-    // 0b00 (VLD1DUP) (for this, a bit makes sense only for data size 16 and 32.
-    // 0b01 (VLD2DUP)
-    // 0b10 (VLD3DUP) (for this, a bit must be encoded as 0)
-    // 0b11 (VLD4DUP)
-    //
-    // Inst{7-6} encodes the data size, with:
-    // 0b00 => 8, 0b01 => 16, 0b10 => 32
-    //
-    // Inst{4} (the a bit) encodes the align action (0: standard alignment)
-    unsigned elem = slice(insn, 9, 8) + 1;
-    unsigned a = slice(insn, 4, 4);
-    if (elem != 3) {
-      // 0b11 is not a valid encoding for Inst{7-6}.
-      if (slice(insn, 7, 6) == 3)
-        return false;
-      unsigned data_size = 8 << slice(insn, 7, 6);
-      // For VLD1DUP, a bit makes sense only for data size of 16 and 32.
-      if (a && data_size == 8)
-        return false;
-
-      // Now we can calculate the alignment!
-      if (a)
-        alignment = elem * data_size;
-    } else {
-      if (a) {
-        // A8.6.315 VLD3 (single 3-element structure to all lanes)
-        // The a bit must be encoded as 0.
-        return false;
-      }
-    }
-  } else {
-    // Multiple n-element structures with type encoded as Inst{11-8}.
-    // See, for example, A8.6.316 VLD4 (multiple 4-element structures).
-
-    // Inst{5-4} encodes alignment.
-    unsigned align = slice(insn, 5, 4);
-    switch (align) {
-    default:
-      break;
-    case 1:
-      alignment = 64; break;
-    case 2:
-      alignment = 128; break;
-    case 3:
-      alignment = 256; break;
-    }
-
-    unsigned type = slice(insn, 11, 8);
-    // Reject UNDEFINED instructions based on type and align.
-    // Plus set DblSpaced flag where appropriate.
-    switch (elem) {
-    default:
-      break;
-    case 1:
-      // n == 1
-      // A8.6.307 & A8.6.391
-      if ((type == 7  && slice(align, 1, 1) == 1) ||
-          (type == 10 && align == 3) ||
-          (type == 6  && slice(align, 1, 1) == 1))
-        return false;
-      break;
-    case 2:
-      // n == 2 && type == 0b1001 -> DblSpaced = true
-      // A8.6.310 & A8.6.393
-      if ((type == 8 || type == 9) && align == 3)
-        return false;
-      DblSpaced = (type == 9);
-      break;
-    case 3:
-      // n == 3 && type == 0b0101 -> DblSpaced = true
-      // A8.6.313 & A8.6.395
-      if (slice(insn, 7, 6) == 3 || slice(align, 1, 1) == 1)
-        return false;
-      DblSpaced = (type == 5);
-      break;
-    case 4:
-      // n == 4 && type == 0b0001 -> DblSpaced = true
-      // A8.6.316 & A8.6.397
-      if (slice(insn, 7, 6) == 3)
-        return false;
-      DblSpaced = (type == 1);
-      break;
-    }
-  }
-  return DisassembleNLdSt0(MI, Opcode, insn, NumOps, NumOpsAdded,
-                           slice(insn, 21, 21) == 0, DblSpaced, alignment/8, B);
-}
-
-// VMOV (immediate)
-//   Qd/Dd imm
-// VBIC (immediate)
-// VORR (immediate)
-//   Qd/Dd imm src(=Qd/Dd)
-static bool DisassembleN1RegModImmFrm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-
-  assert(NumOps >= 2 &&
-         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
-          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
-         (OpInfo[1].RegClass < 0) &&
-         "Expect 1 reg operand followed by 1 imm operand");
-
-  // Qd/Dd = Inst{22:15-12} => NEON Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[0].RegClass,
-                                                     decodeNEONRd(insn))));
-
-  ElemSize esize = ESizeNA;
-  switch (Opcode) {
-  case ARM::VMOVv8i8:
-  case ARM::VMOVv16i8:
-    esize = ESize8;
-    break;
-  case ARM::VMOVv4i16:
-  case ARM::VMOVv8i16:
-  case ARM::VMVNv4i16:
-  case ARM::VMVNv8i16:
-  case ARM::VBICiv4i16:
-  case ARM::VBICiv8i16:
-  case ARM::VORRiv4i16:
-  case ARM::VORRiv8i16:
-    esize = ESize16;
-    break;
-  case ARM::VMOVv2i32:
-  case ARM::VMOVv4i32:
-  case ARM::VMVNv2i32:
-  case ARM::VMVNv4i32:
-  case ARM::VBICiv2i32:
-  case ARM::VBICiv4i32:
-  case ARM::VORRiv2i32:
-  case ARM::VORRiv4i32:
-    esize = ESize32;
-    break;
-  case ARM::VMOVv1i64:
-  case ARM::VMOVv2i64:
-    esize = ESize64;
-    break;
-  default:
-    assert(0 && "Unexpected opcode!");
-    return false;
-  }
-
-  // One register and a modified immediate value.
-  // Add the imm operand.
-  MI.addOperand(MCOperand::CreateImm(decodeN1VImm(insn, esize)));
-
-  NumOpsAdded = 2;
-
-  // VBIC/VORRiv*i* variants have an extra $src = $Vd to be filled in.
-  if (NumOps >= 3 &&
-      (OpInfo[2].RegClass == ARM::DPRRegClassID ||
-       OpInfo[2].RegClass == ARM::QPRRegClassID)) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[0].RegClass,
-                                                     decodeNEONRd(insn))));
-    NumOpsAdded += 1;
-  }
-
-  return true;
-}
-
-namespace {
-enum N2VFlag {
-  N2V_None,
-  N2V_VectorDupLane,
-  N2V_VectorConvert_Between_Float_Fixed
-};
-} // End of unnamed namespace
-
-// Vector Convert [between floating-point and fixed-point]
-//   Qd/Dd Qm/Dm [fbits]
-//
-// Vector Duplicate Lane (from scalar to all elements) Instructions.
-// VDUPLN16d, VDUPLN16q, VDUPLN32d, VDUPLN32q, VDUPLN8d, VDUPLN8q:
-//   Qd/Dd Dm index
-//
-// Vector Move Long:
-//   Qd Dm
-//
-// Vector Move Narrow:
-//   Dd Qm
-//
-// Others
-static bool DisassembleNVdVmOptImm(MCInst &MI, unsigned Opc, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, N2VFlag Flag, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opc];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-
-  assert(NumOps >= 2 &&
-         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
-          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
-         (OpInfo[1].RegClass == ARM::DPRRegClassID ||
-          OpInfo[1].RegClass == ARM::QPRRegClassID) &&
-         "Expect >= 2 operands and first 2 as reg operands");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  ElemSize esize = ESizeNA;
-  if (Flag == N2V_VectorDupLane) {
-    // VDUPLN has its index embedded.  Its size can be inferred from the Opcode.
-    assert(Opc >= ARM::VDUPLN16d && Opc <= ARM::VDUPLN8q &&
-           "Unexpected Opcode");
-    esize = (Opc == ARM::VDUPLN8d || Opc == ARM::VDUPLN8q) ? ESize8
-       : ((Opc == ARM::VDUPLN16d || Opc == ARM::VDUPLN16q) ? ESize16
-                                                           : ESize32);
-  }
-
-  // Qd/Dd = Inst{22:15-12} => NEON Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeNEONRd(insn))));
-  ++OpIdx;
-
-  // VPADAL...
-  if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1) {
-    // TIED_TO operand.
-    MI.addOperand(MCOperand::CreateReg(0));
-    ++OpIdx;
-  }
-
-  // Dm = Inst{5:3-0} => NEON Rm
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeNEONRm(insn))));
-  ++OpIdx;
-
-  // VZIP and others have two TIED_TO reg operands.
-  int Idx;
-  while (OpIdx < NumOps &&
-         (Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-    // Add TIED_TO operand.
-    MI.addOperand(MI.getOperand(Idx));
-    ++OpIdx;
-  }
-
-  // Add the imm operand, if required.
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-
-    unsigned imm = 0xFFFFFFFF;
-
-    if (Flag == N2V_VectorDupLane)
-      imm = decodeNVLaneDupIndex(insn, esize);
-    if (Flag == N2V_VectorConvert_Between_Float_Fixed)
-      imm = decodeVCVTFractionBits(insn);
-
-    assert(imm != 0xFFFFFFFF && "Internal error");
-    MI.addOperand(MCOperand::CreateImm(imm));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-static bool DisassembleN2RegFrm(MCInst &MI, unsigned Opc, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVmOptImm(MI, Opc, insn, NumOps, NumOpsAdded,
-                                N2V_None, B);
-}
-static bool DisassembleNVCVTFrm(MCInst &MI, unsigned Opc, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVmOptImm(MI, Opc, insn, NumOps, NumOpsAdded,
-                                N2V_VectorConvert_Between_Float_Fixed, B);
-}
-static bool DisassembleNVecDupLnFrm(MCInst &MI, unsigned Opc, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVmOptImm(MI, Opc, insn, NumOps, NumOpsAdded,
-                                N2V_VectorDupLane, B);
-}
-
-// Vector Shift [Accumulate] Instructions.
-// Qd/Dd [Qd/Dd (TIED_TO)] Qm/Dm ShiftAmt
-//
-// Vector Shift Left Long (with maximum shift count) Instructions.
-// VSHLLi16, VSHLLi32, VSHLLi8: Qd Dm imm (== size)
-//
-static bool DisassembleNVectorShift(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, bool LeftShift, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-
-  assert(NumOps >= 3 &&
-         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
-          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
-         (OpInfo[1].RegClass == ARM::DPRRegClassID ||
-          OpInfo[1].RegClass == ARM::QPRRegClassID) &&
-         "Expect >= 3 operands and first 2 as reg operands");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  // Qd/Dd = Inst{22:15-12} => NEON Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeNEONRd(insn))));
-  ++OpIdx;
-
-  if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1) {
-    // TIED_TO operand.
-    MI.addOperand(MCOperand::CreateReg(0));
-    ++OpIdx;
-  }
-
-  assert((OpInfo[OpIdx].RegClass == ARM::DPRRegClassID ||
-          OpInfo[OpIdx].RegClass == ARM::QPRRegClassID) &&
-         "Reg operand expected");
-
-  // Qm/Dm = Inst{5:3-0} => NEON Rm
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeNEONRm(insn))));
-  ++OpIdx;
-
-  assert(OpInfo[OpIdx].RegClass < 0 && "Imm operand expected");
-
-  // Add the imm operand.
-
-  // VSHLL has maximum shift count as the imm, inferred from its size.
-  unsigned Imm;
-  switch (Opcode) {
-  default:
-    Imm = decodeNVSAmt(insn, LeftShift);
-    break;
-  case ARM::VSHLLi8:
-    Imm = 8;
-    break;
-  case ARM::VSHLLi16:
-    Imm = 16;
-    break;
-  case ARM::VSHLLi32:
-    Imm = 32;
-    break;
-  }
-  MI.addOperand(MCOperand::CreateImm(Imm));
-  ++OpIdx;
-
-  return true;
-}
-
-// Left shift instructions.
-static bool DisassembleN2RegVecShLFrm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVectorShift(MI, Opcode, insn, NumOps, NumOpsAdded, true,
-                                 B);
-}
-// Right shift instructions have different shift amount interpretation.
-static bool DisassembleN2RegVecShRFrm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVectorShift(MI, Opcode, insn, NumOps, NumOpsAdded, false,
-                                 B);
-}
-
-namespace {
-enum N3VFlag {
-  N3V_None,
-  N3V_VectorExtract,
-  N3V_VectorShift,
-  N3V_Multiply_By_Scalar
-};
-} // End of unnamed namespace
-
-// NEON Three Register Instructions with Optional Immediate Operand
-//
-// Vector Extract Instructions.
-// Qd/Dd Qn/Dn Qm/Dm imm4
-//
-// Vector Shift (Register) Instructions.
-// Qd/Dd Qm/Dm Qn/Dn (notice the order of m, n)
-//
-// Vector Multiply [Accumulate/Subtract] [Long] By Scalar Instructions.
-// Qd/Dd Qn/Dn RestrictedDm index
-//
-// Others
-static bool DisassembleNVdVnVmOptImm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, N3VFlag Flag, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-
-  // No checking for OpInfo[2] because of MOVDneon/MOVQ with only two regs.
-  assert(NumOps >= 3 &&
-         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
-          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
-         (OpInfo[1].RegClass == ARM::DPRRegClassID ||
-          OpInfo[1].RegClass == ARM::QPRRegClassID) &&
-         "Expect >= 3 operands and first 2 as reg operands");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  bool VdVnVm = Flag == N3V_VectorShift ? false : true;
-  bool IsImm4 = Flag == N3V_VectorExtract ? true : false;
-  bool IsDmRestricted = Flag == N3V_Multiply_By_Scalar ? true : false;
-  ElemSize esize = ESizeNA;
-  if (Flag == N3V_Multiply_By_Scalar) {
-    unsigned size = (insn >> 20) & 3;
-    if (size == 1) esize = ESize16;
-    if (size == 2) esize = ESize32;
-    assert (esize == ESize16 || esize == ESize32);
-  }
-
-  // Qd/Dd = Inst{22:15-12} => NEON Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeNEONRd(insn))));
-  ++OpIdx;
-
-  // VABA, VABAL, VBSLd, VBSLq, ...
-  if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1) {
-    // TIED_TO operand.
-    MI.addOperand(MCOperand::CreateReg(0));
-    ++OpIdx;
-  }
-
-  // Dn = Inst{7:19-16} => NEON Rn
-  // or
-  // Dm = Inst{5:3-0} => NEON Rm
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                  VdVnVm ? decodeNEONRn(insn)
-                                         : decodeNEONRm(insn))));
-  ++OpIdx;
-
-  // Dm = Inst{5:3-0} => NEON Rm
-  // or
-  // Dm is restricted to D0-D7 if size is 16, D0-D15 otherwise
-  // or
-  // Dn = Inst{7:19-16} => NEON Rn
-  unsigned m = VdVnVm ? (IsDmRestricted ? decodeRestrictedDm(insn, esize)
-                                        : decodeNEONRm(insn))
-                      : decodeNEONRn(insn);
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, OpInfo[OpIdx].RegClass, m)));
-  ++OpIdx;
-
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-    // Add the imm operand.
-    unsigned Imm = 0;
-    if (IsImm4)
-      Imm = decodeN3VImm(insn);
-    else if (IsDmRestricted)
-      Imm = decodeRestrictedDmIndex(insn, esize);
-    else {
-      assert(0 && "Internal error: unreachable code!");
-      return false;
-    }
-
-    MI.addOperand(MCOperand::CreateImm(Imm));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-static bool DisassembleN3RegFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                  N3V_None, B);
-}
-static bool DisassembleN3RegVecShFrm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                  N3V_VectorShift, B);
-}
-static bool DisassembleNVecExtractFrm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                  N3V_VectorExtract, B);
-}
-static bool DisassembleNVecMulScalarFrm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                  N3V_Multiply_By_Scalar, B);
-}
-
-// Vector Table Lookup
-//
-// VTBL1, VTBX1: Dd [Dd(TIED_TO)] Dn Dm
-// VTBL2, VTBX2: Dd [Dd(TIED_TO)] Dn Dn+1 Dm
-// VTBL3, VTBX3: Dd [Dd(TIED_TO)] Dn Dn+1 Dn+2 Dm
-// VTBL4, VTBX4: Dd [Dd(TIED_TO)] Dn Dn+1 Dn+2 Dn+3 Dm
-static bool DisassembleNVTBLFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::DPRRegClassID &&
-         OpInfo[1].RegClass == ARM::DPRRegClassID &&
-         OpInfo[2].RegClass == ARM::DPRRegClassID &&
-         "Expect >= 3 operands and first 3 as reg operands");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  unsigned Rn = decodeNEONRn(insn);
-
-  // {Dn} encoded as len = 0b00
-  // {Dn Dn+1} encoded as len = 0b01
-  // {Dn Dn+1 Dn+2 } encoded as len = 0b10
-  // {Dn Dn+1 Dn+2 Dn+3} encoded as len = 0b11
-  unsigned Len = slice(insn, 9, 8) + 1;
-
-  // Dd (the destination vector)
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::DPRRegClassID,
-                                                     decodeNEONRd(insn))));
-  ++OpIdx;
-
-  // Process tied_to operand constraint.
-  int Idx;
-  if ((Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-    MI.addOperand(MI.getOperand(Idx));
-    ++OpIdx;
-  }
-
-  // Do the <list> now.
-  for (unsigned i = 0; i < Len; ++i) {
-    assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::DPRRegClassID &&
-           "Reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::DPRRegClassID,
-                                                       Rn + i)));
-    ++OpIdx;
-  }
-
-  // Dm (the index vector)
-  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::DPRRegClassID &&
-         "Reg operand (index vector) expected");
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::DPRRegClassID,
-                                                     decodeNEONRm(insn))));
-  ++OpIdx;
-
-  return true;
-}
-
-// Vector Get Lane (move scalar to ARM core register) Instructions.
-// VGETLNi32, VGETLNs16, VGETLNs8, VGETLNu16, VGETLNu8: Rt Dn index
-static bool DisassembleNGetLnFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  if (!OpInfo) return false;
-
-  assert(MCID.getNumDefs() == 1 && NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::GPRRegClassID &&
-         OpInfo[1].RegClass == ARM::DPRRegClassID &&
-         OpInfo[2].RegClass < 0 &&
-         "Expect >= 3 operands with one dst operand");
-
-  ElemSize esize =
-    Opcode == ARM::VGETLNi32 ? ESize32
-      : ((Opcode == ARM::VGETLNs16 || Opcode == ARM::VGETLNu16) ? ESize16
-                                                                : ESize8);
-
-  // Rt = Inst{15-12} => ARM Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-
-  // Dn = Inst{7:19-16} => NEON Rn
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::DPRRegClassID,
-                                                     decodeNEONRn(insn))));
-
-  MI.addOperand(MCOperand::CreateImm(decodeNVLaneOpIndex(insn, esize)));
-
-  NumOpsAdded = 3;
-  return true;
-}
-
-// Vector Set Lane (move ARM core register to scalar) Instructions.
-// VSETLNi16, VSETLNi32, VSETLNi8: Dd Dd (TIED_TO) Rt index
-static bool DisassembleNSetLnFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  if (!OpInfo) return false;
-
-  assert(MCID.getNumDefs() == 1 && NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::DPRRegClassID &&
-         OpInfo[1].RegClass == ARM::DPRRegClassID &&
-         MCID.getOperandConstraint(1, MCOI::TIED_TO) != -1 &&
-         OpInfo[2].RegClass == ARM::GPRRegClassID &&
-         OpInfo[3].RegClass < 0 &&
-         "Expect >= 3 operands with one dst operand");
-
-  ElemSize esize =
-    Opcode == ARM::VSETLNi8 ? ESize8
-                            : (Opcode == ARM::VSETLNi16 ? ESize16
-                                                        : ESize32);
-
-  // Dd = Inst{7:19-16} => NEON Rn
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::DPRRegClassID,
-                                                     decodeNEONRn(insn))));
-
-  // TIED_TO operand.
-  MI.addOperand(MCOperand::CreateReg(0));
-
-  // Rt = Inst{15-12} => ARM Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-
-  MI.addOperand(MCOperand::CreateImm(decodeNVLaneOpIndex(insn, esize)));
-
-  NumOpsAdded = 4;
-  return true;
-}
-
-// Vector Duplicate Instructions (from ARM core register to all elements).
-// VDUP8d, VDUP16d, VDUP32d, VDUP8q, VDUP16q, VDUP32q: Qd/Dd Rt
-static bool DisassembleNDupFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-
-  assert(NumOps >= 2 &&
-         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
-          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
-         OpInfo[1].RegClass == ARM::GPRRegClassID &&
-         "Expect >= 2 operands and first 2 as reg operand");
-
-  unsigned RegClass = OpInfo[0].RegClass;
-
-  // Qd/Dd = Inst{7:19-16} => NEON Rn
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClass,
-                                                     decodeNEONRn(insn))));
-
-  // Rt = Inst{15-12} => ARM Rd
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-
-  NumOpsAdded = 2;
-  return true;
-}
-
-static inline bool PreLoadOpcode(unsigned Opcode) {
-  switch(Opcode) {
-  case ARM::PLDi12:  case ARM::PLDrs:
-  case ARM::PLDWi12: case ARM::PLDWrs:
-  case ARM::PLIi12:  case ARM::PLIrs:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static bool DisassemblePreLoadFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  // Preload Data/Instruction requires either 2 or 3 operands.
-  // PLDi12, PLDWi12, PLIi12: addrmode_imm12
-  // PLDrs, PLDWrs, PLIrs:    ldst_so_reg
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-
-  if (Opcode == ARM::PLDi12 || Opcode == ARM::PLDWi12
-      || Opcode == ARM::PLIi12) {
-    unsigned Imm12 = slice(insn, 11, 0);
-    bool Negative = getUBit(insn) == 0;
-
-    // A8.6.118 PLD (literal) PLDWi12 with Rn=PC is transformed to PLDi12.
-    if (Opcode == ARM::PLDWi12 && slice(insn, 19, 16) == 0xF) {
-      DEBUG(errs() << "Rn == '1111': PLDWi12 morphed to PLDi12\n");
-      MI.setOpcode(ARM::PLDi12);
-    }
-    
-    // -0 is represented specially. All other values are as normal.
-    int Offset = Negative ? -1 * Imm12 : Imm12;
-    if (Imm12 == 0 && Negative)
-      Offset = INT32_MIN;
-
-    MI.addOperand(MCOperand::CreateImm(Offset));
-    NumOpsAdded = 2;
-  } else {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-
-    ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
-
-    // Inst{6-5} encodes the shift opcode.
-    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
-    // Inst{11-7} encodes the imm5 shift amount.
-    unsigned ShImm = slice(insn, 11, 7);
-
-    // A8.4.1.  Possible rrx or shift amount of 32...
-    getImmShiftSE(ShOp, ShImm);
-    MI.addOperand(MCOperand::CreateImm(
-                    ARM_AM::getAM2Opc(AddrOpcode, ShImm, ShOp)));
-    NumOpsAdded = 3;
-  }
-
-  return true;
-}
-
-static bool DisassembleMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  if (Opcode == ARM::DMB || Opcode == ARM::DSB || Opcode == ARM::ISB) {
-    // Inst{3-0} encodes the memory barrier option for the variants.
-    unsigned opt = slice(insn, 3, 0);
-    switch (opt) {
-    case ARM_MB::SY:  case ARM_MB::ST:
-    case ARM_MB::ISH: case ARM_MB::ISHST:
-    case ARM_MB::NSH: case ARM_MB::NSHST:
-    case ARM_MB::OSH: case ARM_MB::OSHST:
-      MI.addOperand(MCOperand::CreateImm(opt));
-      NumOpsAdded = 1;
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  switch (Opcode) {
-  case ARM::CLREX:
-  case ARM::NOP:
-  case ARM::TRAP:
-  case ARM::YIELD:
-  case ARM::WFE:
-  case ARM::WFI:
-  case ARM::SEV:
-    return true;
-  case ARM::SWP:
-  case ARM::SWPB:
-    // SWP, SWPB: Rd Rm Rn
-    // Delegate to DisassembleLdStExFrm()....
-    return DisassembleLdStExFrm(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-  default:
-    break;
-  }
-
-  if (Opcode == ARM::SETEND) {
-    NumOpsAdded = 1;
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 9, 9)));
-    return true;
-  }
-
-  // FIXME: To enable correct asm parsing and disasm of CPS we need 3 different
-  // opcodes which match the same real instruction. This is needed since there's
-  // no current handling of optional arguments. Fix here when a better handling
-  // of optional arguments is implemented.
-  if (Opcode == ARM::CPS3p) {   // M = 1
-    // Let's reject these impossible imod values by returning false:
-    // 1. (imod=0b01)
-    //
-    // AsmPrinter cannot handle imod=0b00, plus (imod=0b00,M=1,iflags!=0) is an
-    // invalid combination, so we just check for imod=0b00 here.
-    if (slice(insn, 19, 18) == 0 || slice(insn, 19, 18) == 1)
-      return false;
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 18))); // imod
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 8, 6)));   // iflags
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));   // mode
-    NumOpsAdded = 3;
-    return true;
-  }
-  if (Opcode == ARM::CPS2p) { // mode = 0, M = 0
-    // Let's reject these impossible imod values by returning false:
-    // 1. (imod=0b00,M=0)
-    // 2. (imod=0b01)
-    if (slice(insn, 19, 18) == 0 || slice(insn, 19, 18) == 1)
-      return false;
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 18))); // imod
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 8, 6)));   // iflags
-    NumOpsAdded = 2;
-    return true;
-  }
-  if (Opcode == ARM::CPS1p) { // imod = 0, iflags = 0, M = 1
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0))); // mode
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  // DBG has its option specified in Inst{3-0}.
-  if (Opcode == ARM::DBG) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 3, 0)));
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  // BKPT takes an imm32 val equal to ZeroExtend(Inst{19-8:3-0}).
-  if (Opcode == ARM::BKPT) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 8) << 4 |
-                                       slice(insn, 3, 0)));
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  if (PreLoadOpcode(Opcode))
-    return DisassemblePreLoadFrm(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-
-  assert(0 && "Unexpected misc instruction!");
-  return false;
-}
-
-/// FuncPtrs - FuncPtrs maps ARMFormat to its corresponding DisassembleFP.
-/// We divide the disassembly task into different categories, with each one
-/// corresponding to a specific instruction encoding format.  There could be
-/// exceptions when handling a specific format, and that is why the Opcode is
-/// also present in the function prototype.
-static const DisassembleFP FuncPtrs[] = {
-  &DisassemblePseudo,
-  &DisassembleMulFrm,
-  &DisassembleBrFrm,
-  &DisassembleBrMiscFrm,
-  &DisassembleDPFrm,
-  &DisassembleDPSoRegFrm,
-  &DisassembleLdFrm,
-  &DisassembleStFrm,
-  &DisassembleLdMiscFrm,
-  &DisassembleStMiscFrm,
-  &DisassembleLdStMulFrm,
-  &DisassembleLdStExFrm,
-  &DisassembleArithMiscFrm,
-  &DisassembleSatFrm,
-  &DisassembleExtFrm,
-  &DisassembleVFPUnaryFrm,
-  &DisassembleVFPBinaryFrm,
-  &DisassembleVFPConv1Frm,
-  &DisassembleVFPConv2Frm,
-  &DisassembleVFPConv3Frm,
-  &DisassembleVFPConv4Frm,
-  &DisassembleVFPConv5Frm,
-  &DisassembleVFPLdStFrm,
-  &DisassembleVFPLdStMulFrm,
-  &DisassembleVFPMiscFrm,
-  &DisassembleThumbFrm,
-  &DisassembleMiscFrm,
-  &DisassembleNGetLnFrm,
-  &DisassembleNSetLnFrm,
-  &DisassembleNDupFrm,
-
-  // VLD and VST (including one lane) Instructions.
-  &DisassembleNLdSt,
-
-  // A7.4.6 One register and a modified immediate value
-  // 1-Register Instructions with imm.
-  // LLVM only defines VMOVv instructions.
-  &DisassembleN1RegModImmFrm,
-
-  // 2-Register Instructions with no imm.
-  &DisassembleN2RegFrm,
-
-  // 2-Register Instructions with imm (vector convert float/fixed point).
-  &DisassembleNVCVTFrm,
-
-  // 2-Register Instructions with imm (vector dup lane).
-  &DisassembleNVecDupLnFrm,
-
-  // Vector Shift Left Instructions.
-  &DisassembleN2RegVecShLFrm,
-
-  // Vector Shift Righ Instructions, which has different interpretation of the
-  // shift amount from the imm6 field.
-  &DisassembleN2RegVecShRFrm,
-
-  // 3-Register Data-Processing Instructions.
-  &DisassembleN3RegFrm,
-
-  // Vector Shift (Register) Instructions.
-  // D:Vd M:Vm N:Vn (notice that M:Vm is the first operand)
-  &DisassembleN3RegVecShFrm,
-
-  // Vector Extract Instructions.
-  &DisassembleNVecExtractFrm,
-
-  // Vector [Saturating Rounding Doubling] Multiply [Accumulate/Subtract] [Long]
-  // By Scalar Instructions.
-  &DisassembleNVecMulScalarFrm,
-
-  // Vector Table Lookup uses byte indexes in a control vector to look up byte
-  // values in a table and generate a new vector.
-  &DisassembleNVTBLFrm,
-
-  NULL
-};
-
-/// BuildIt - BuildIt performs the build step for this ARM Basic MC Builder.
-/// The general idea is to set the Opcode for the MCInst, followed by adding
-/// the appropriate MCOperands to the MCInst.  ARM Basic MC Builder delegates
-/// to the Format-specific disassemble function for disassembly, followed by
-/// TryPredicateAndSBitModifier() to do PredicateOperand and OptionalDefOperand
-/// which follow the Dst/Src Operands.
-bool ARMBasicMCBuilder::BuildIt(MCInst &MI, uint32_t insn) {
-  // Stage 1 sets the Opcode.
-  MI.setOpcode(Opcode);
-  // If the number of operands is zero, we're done!
-  if (NumOps == 0)
-    return true;
-
-  // Stage 2 calls the format-specific disassemble function to build the operand
-  // list.
-  if (Disasm == NULL)
-    return false;
-  unsigned NumOpsAdded = 0;
-  bool OK = (*Disasm)(MI, Opcode, insn, NumOps, NumOpsAdded, this);
-
-  if (!OK || this->Err != 0) return false;
-  if (NumOpsAdded >= NumOps)
-    return true;
-
-  // Stage 3 deals with operands unaccounted for after stage 2 is finished.
-  // FIXME: Should this be done selectively?
-  return TryPredicateAndSBitModifier(MI, Opcode, insn, NumOps - NumOpsAdded);
-}
-
-// A8.3 Conditional execution
-// A8.3.1 Pseudocode details of conditional execution
-// Condition bits '111x' indicate the instruction is always executed.
-static uint32_t CondCode(uint32_t CondField) {
-  if (CondField == 0xF)
-    return ARMCC::AL;
-  return CondField;
-}
-
-/// DoPredicateOperands - DoPredicateOperands process the predicate operands
-/// of some Thumb instructions which come before the reglist operands.  It
-/// returns true if the two predicate operands have been processed.
-bool ARMBasicMCBuilder::DoPredicateOperands(MCInst& MI, unsigned Opcode,
-    uint32_t /* insn */, unsigned short NumOpsRemaining) {
-
-  assert(NumOpsRemaining > 0 && "Invalid argument");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned Idx = MI.getNumOperands();
-
-  // First, we check whether this instr specifies the PredicateOperand through
-  // a pair of MCOperandInfos with isPredicate() property.
-  if (NumOpsRemaining >= 2 &&
-      OpInfo[Idx].isPredicate() && OpInfo[Idx+1].isPredicate() &&
-      OpInfo[Idx].RegClass < 0 &&
-      OpInfo[Idx+1].RegClass == ARM::CCRRegClassID)
-  {
-    // If we are inside an IT block, get the IT condition bits maintained via
-    // ARMBasicMCBuilder::ITState[7:0], through ARMBasicMCBuilder::GetITCond().
-    // See also A2.5.2.
-    if (InITBlock())
-      MI.addOperand(MCOperand::CreateImm(GetITCond()));
-    else
-      MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
-    return true;
-  }
-
-  return false;
-}
-
-/// TryPredicateAndSBitModifier - TryPredicateAndSBitModifier tries to process
-/// the possible Predicate and SBitModifier, to build the remaining MCOperand
-/// constituents.
-bool ARMBasicMCBuilder::TryPredicateAndSBitModifier(MCInst& MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOpsRemaining) {
-
-  assert(NumOpsRemaining > 0 && "Invalid argument");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  const std::string &Name = ARMInsts[Opcode].Name;
-  unsigned Idx = MI.getNumOperands();
-  uint64_t TSFlags = ARMInsts[Opcode].TSFlags;
-
-  // First, we check whether this instr specifies the PredicateOperand through
-  // a pair of MCOperandInfos with isPredicate() property.
-  if (NumOpsRemaining >= 2 &&
-      OpInfo[Idx].isPredicate() && OpInfo[Idx+1].isPredicate() &&
-      OpInfo[Idx].RegClass < 0 &&
-      OpInfo[Idx+1].RegClass == ARM::CCRRegClassID)
-  {
-    // If we are inside an IT block, get the IT condition bits maintained via
-    // ARMBasicMCBuilder::ITState[7:0], through ARMBasicMCBuilder::GetITCond().
-    // See also A2.5.2.
-    if (InITBlock())
-      MI.addOperand(MCOperand::CreateImm(GetITCond()));
-    else {
-      if (Name.length() > 1 && Name[0] == 't') {
-        // Thumb conditional branch instructions have their cond field embedded,
-        // like ARM.
-        //
-        // A8.6.16 B
-        // Check for undefined encodings.
-        unsigned cond;
-        if (Name == "t2Bcc") {
-          if ((cond = slice(insn, 25, 22)) >= 14)
-            return false;
-          MI.addOperand(MCOperand::CreateImm(CondCode(cond)));
-        } else if (Name == "tBcc") {
-          if ((cond = slice(insn, 11, 8)) == 14)
-            return false;
-          MI.addOperand(MCOperand::CreateImm(CondCode(cond)));
-        } else
-          MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
-      } else {
-        // ARM instructions get their condition field from Inst{31-28}.
-        // We should reject Inst{31-28} = 0b1111 as invalid encoding.
-        if (!isNEONDomain(TSFlags) && getCondField(insn) == 0xF)
-          return false;
-        MI.addOperand(MCOperand::CreateImm(CondCode(getCondField(insn))));
-      }
-    }
-    MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
-    Idx += 2;
-    NumOpsRemaining -= 2;
-  }
-
-  if (NumOpsRemaining == 0)
-    return true;
-
-  // Next, if OptionalDefOperand exists, we check whether the 'S' bit is set.
-  if (OpInfo[Idx].isOptionalDef() && OpInfo[Idx].RegClass==ARM::CCRRegClassID) {
-    MI.addOperand(MCOperand::CreateReg(getSBit(insn) == 1 ? ARM::CPSR : 0));
-    --NumOpsRemaining;
-  }
-
-  if (NumOpsRemaining == 0)
-    return true;
-  else
-    return false;
-}
-
-/// RunBuildAfterHook - RunBuildAfterHook performs operations deemed necessary
-/// after BuildIt is finished.
-bool ARMBasicMCBuilder::RunBuildAfterHook(bool Status, MCInst &MI,
-    uint32_t insn) {
-
-  if (!SP) return Status;
-
-  if (Opcode == ARM::t2IT)
-    Status = SP->InitIT(slice(insn, 7, 0)) ? Status : false;
-  else if (InITBlock())
-    SP->UpdateIT();
-
-  return Status;
-}
-
-/// Opcode, Format, and NumOperands make up an ARM Basic MCBuilder.
-ARMBasicMCBuilder::ARMBasicMCBuilder(unsigned opc, ARMFormat format,
-                                     unsigned short num)
-  : Opcode(opc), Format(format), NumOps(num), SP(0), Err(0) {
-  unsigned Idx = (unsigned)format;
-  assert(Idx < (array_lengthof(FuncPtrs) - 1) && "Unknown format");
-  Disasm = FuncPtrs[Idx];
-}
-
-/// CreateMCBuilder - Return an ARMBasicMCBuilder that can build up the MC
-/// infrastructure of an MCInst given the Opcode and Format of the instr.
-/// Return NULL if it fails to create/return a proper builder.  API clients
-/// are responsible for freeing up of the allocated memory.  Cacheing can be
-/// performed by the API clients to improve performance.
-ARMBasicMCBuilder *llvm::CreateMCBuilder(unsigned Opcode, ARMFormat Format) {
-  // For "Unknown format", fail by returning a NULL pointer.
-  if ((unsigned)Format >= (array_lengthof(FuncPtrs) - 1)) {
-    DEBUG(errs() << "Unknown format\n");
-    return 0;
-  }
-
-  return new ARMBasicMCBuilder(Opcode, Format,
-                               ARMInsts[Opcode].getNumOperands());
-}
-
-/// tryAddingSymbolicOperand - tryAddingSymbolicOperand trys to add a symbolic
-/// operand in place of the immediate Value in the MCInst.  The immediate
-/// Value has had any PC adjustment made by the caller.  If the getOpInfo()
-/// function was set as part of the setupBuilderForSymbolicDisassembly() call
-/// then that function is called to get any symbolic information at the
-/// builder's Address for this instrution.  If that returns non-zero then the
-/// symbolic information it returns is used to create an MCExpr and that is
-/// added as an operand to the MCInst.  This function returns true if it adds
-/// an operand to the MCInst and false otherwise.
-bool ARMBasicMCBuilder::tryAddingSymbolicOperand(uint64_t Value,
-                                                 uint64_t InstSize,
-                                                 MCInst &MI) {
-  if (!GetOpInfo)
-    return false;
-
-  struct LLVMOpInfo1 SymbolicOp;
-  SymbolicOp.Value = Value;
-  if (!GetOpInfo(DisInfo, Address, 0 /* Offset */, InstSize, 1, &SymbolicOp))
-    return false;
-
-  const MCExpr *Add = NULL;
-  if (SymbolicOp.AddSymbol.Present) {
-    if (SymbolicOp.AddSymbol.Name) {
-      StringRef Name(SymbolicOp.AddSymbol.Name);
-      MCSymbol *Sym = Ctx->GetOrCreateSymbol(Name);
-      Add = MCSymbolRefExpr::Create(Sym, *Ctx);
-    } else {
-      Add = MCConstantExpr::Create(SymbolicOp.AddSymbol.Value, *Ctx);
-    }
-  }
-
-  const MCExpr *Sub = NULL;
-  if (SymbolicOp.SubtractSymbol.Present) {
-    if (SymbolicOp.SubtractSymbol.Name) {
-      StringRef Name(SymbolicOp.SubtractSymbol.Name);
-      MCSymbol *Sym = Ctx->GetOrCreateSymbol(Name);
-      Sub = MCSymbolRefExpr::Create(Sym, *Ctx);
-    } else {
-      Sub = MCConstantExpr::Create(SymbolicOp.SubtractSymbol.Value, *Ctx);
-    }
-  }
-
-  const MCExpr *Off = NULL;
-  if (SymbolicOp.Value != 0)
-    Off = MCConstantExpr::Create(SymbolicOp.Value, *Ctx);
-
-  const MCExpr *Expr;
-  if (Sub) {
-    const MCExpr *LHS;
-    if (Add)
-      LHS = MCBinaryExpr::CreateSub(Add, Sub, *Ctx);
-    else
-      LHS = MCUnaryExpr::CreateMinus(Sub, *Ctx);
-    if (Off != 0)
-      Expr = MCBinaryExpr::CreateAdd(LHS, Off, *Ctx);
-    else
-      Expr = LHS;
-  } else if (Add) {
-    if (Off != 0)
-      Expr = MCBinaryExpr::CreateAdd(Add, Off, *Ctx);
-    else
-      Expr = Add;
-  } else {
-    if (Off != 0)
-      Expr = Off;
-    else
-      Expr = MCConstantExpr::Create(0, *Ctx);
-  }
-
-  if (SymbolicOp.VariantKind == LLVMDisassembler_VariantKind_ARM_HI16)
-    MI.addOperand(MCOperand::CreateExpr(ARMMCExpr::CreateUpper16(Expr, *Ctx)));
-  else if (SymbolicOp.VariantKind == LLVMDisassembler_VariantKind_ARM_LO16)
-    MI.addOperand(MCOperand::CreateExpr(ARMMCExpr::CreateLower16(Expr, *Ctx)));
-  else if (SymbolicOp.VariantKind == LLVMDisassembler_VariantKind_None)
-    MI.addOperand(MCOperand::CreateExpr(Expr));
-  else 
-    assert("bad SymbolicOp.VariantKind");
-
-  return true;
-}
diff --git a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassemblerCore.h b/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassemblerCore.h
deleted file mode 100644
index a7ba14141c0a..000000000000
--- a/contrib/llvm/lib/Target/ARM/Disassembler/ARMDisassemblerCore.h
+++ /dev/null
@@ -1,336 +0,0 @@
-//===- ARMDisassemblerCore.h - ARM disassembler helpers ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is part of the ARM Disassembler.
-//
-// The first part defines the enumeration type of ARM instruction format, which
-// specifies the encoding used by the instruction, as well as a helper function
-// to convert the enums to printable char strings.
-//
-// It also contains code to represent the concepts of Builder and DisassembleFP
-// to solve the problem of disassembling an ARM instr.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef ARMDISASSEMBLERCORE_H
-#define ARMDISASSEMBLERCORE_H
-
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCSymbol.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/Target/TargetInstrInfo.h"
-#include "llvm-c/Disassembler.h"
-#include "ARMBaseInstrInfo.h"
-#include "ARMRegisterInfo.h"
-#include "ARMDisassembler.h"
-
-namespace llvm {
-class MCContext;
-
-class ARMUtils {
-public:
-  static const char *OpcodeName(unsigned Opcode);
-};
-
-/////////////////////////////////////////////////////
-//                                                 //
-//  Enums and Utilities for ARM Instruction Format //
-//                                                 //
-/////////////////////////////////////////////////////
-
-#define ARM_FORMATS                   \
-  ENTRY(ARM_FORMAT_PSEUDO,         0) \
-  ENTRY(ARM_FORMAT_MULFRM,         1) \
-  ENTRY(ARM_FORMAT_BRFRM,          2) \
-  ENTRY(ARM_FORMAT_BRMISCFRM,      3) \
-  ENTRY(ARM_FORMAT_DPFRM,          4) \
-  ENTRY(ARM_FORMAT_DPSOREGFRM,     5) \
-  ENTRY(ARM_FORMAT_LDFRM,          6) \
-  ENTRY(ARM_FORMAT_STFRM,          7) \
-  ENTRY(ARM_FORMAT_LDMISCFRM,      8) \
-  ENTRY(ARM_FORMAT_STMISCFRM,      9) \
-  ENTRY(ARM_FORMAT_LDSTMULFRM,    10) \
-  ENTRY(ARM_FORMAT_LDSTEXFRM,     11) \
-  ENTRY(ARM_FORMAT_ARITHMISCFRM,  12) \
-  ENTRY(ARM_FORMAT_SATFRM,        13) \
-  ENTRY(ARM_FORMAT_EXTFRM,        14) \
-  ENTRY(ARM_FORMAT_VFPUNARYFRM,   15) \
-  ENTRY(ARM_FORMAT_VFPBINARYFRM,  16) \
-  ENTRY(ARM_FORMAT_VFPCONV1FRM,   17) \
-  ENTRY(ARM_FORMAT_VFPCONV2FRM,   18) \
-  ENTRY(ARM_FORMAT_VFPCONV3FRM,   19) \
-  ENTRY(ARM_FORMAT_VFPCONV4FRM,   20) \
-  ENTRY(ARM_FORMAT_VFPCONV5FRM,   21) \
-  ENTRY(ARM_FORMAT_VFPLDSTFRM,    22) \
-  ENTRY(ARM_FORMAT_VFPLDSTMULFRM, 23) \
-  ENTRY(ARM_FORMAT_VFPMISCFRM,    24) \
-  ENTRY(ARM_FORMAT_THUMBFRM,      25) \
-  ENTRY(ARM_FORMAT_MISCFRM,       26) \
-  ENTRY(ARM_FORMAT_NEONGETLNFRM,  27) \
-  ENTRY(ARM_FORMAT_NEONSETLNFRM,  28) \
-  ENTRY(ARM_FORMAT_NEONDUPFRM,    29) \
-  ENTRY(ARM_FORMAT_NLdSt,         30) \
-  ENTRY(ARM_FORMAT_N1RegModImm,   31) \
-  ENTRY(ARM_FORMAT_N2Reg,         32) \
-  ENTRY(ARM_FORMAT_NVCVT,         33) \
-  ENTRY(ARM_FORMAT_NVecDupLn,     34) \
-  ENTRY(ARM_FORMAT_N2RegVecShL,   35) \
-  ENTRY(ARM_FORMAT_N2RegVecShR,   36) \
-  ENTRY(ARM_FORMAT_N3Reg,         37) \
-  ENTRY(ARM_FORMAT_N3RegVecSh,    38) \
-  ENTRY(ARM_FORMAT_NVecExtract,   39) \
-  ENTRY(ARM_FORMAT_NVecMulScalar, 40) \
-  ENTRY(ARM_FORMAT_NVTBL,         41)
-
-// ARM instruction format specifies the encoding used by the instruction.
-#define ENTRY(n, v) n = v,
-typedef enum {
-  ARM_FORMATS
-  ARM_FORMAT_NA
-} ARMFormat;
-#undef ENTRY
-
-// Converts enum to const char*.
-static const inline char *stringForARMFormat(ARMFormat form) {
-#define ENTRY(n, v) case n: return #n;
-  switch(form) {
-    ARM_FORMATS
-  case ARM_FORMAT_NA:
-  default:
-    return "";
-  }
-#undef ENTRY
-}
-
-/// Expands on the enum definitions from ARMBaseInstrInfo.h.
-/// They are being used by the disassembler implementation.
-namespace ARMII {
-  enum {
-    NEONRegMask = 15,
-    GPRRegMask = 15,
-    NEON_RegRdShift = 12,
-    NEON_D_BitShift = 22,
-    NEON_RegRnShift = 16,
-    NEON_N_BitShift = 7,
-    NEON_RegRmShift = 0,
-    NEON_M_BitShift = 5
-  };
-}
-
-/// Utility function for extracting [From, To] bits from a uint32_t.
-static inline unsigned slice(uint32_t Bits, unsigned From, unsigned To) {
-  assert(From < 32 && To < 32 && From >= To);
-  return (Bits >> To) & ((1 << (From - To + 1)) - 1);
-}
-
-/// Utility function for setting [From, To] bits to Val for a uint32_t.
-static inline void setSlice(unsigned &Bits, unsigned From, unsigned To,
-                            unsigned Val) {
-  assert(From < 32 && To < 32 && From >= To);
-  uint32_t Mask = ((1 << (From - To + 1)) - 1);
-  Bits &= ~(Mask << To);
-  Bits |= (Val & Mask) << To;
-}
-
-// Return an integer result equal to the number of bits of x that are ones.
-static inline uint32_t
-BitCount (uint64_t x)
-{
-    // c accumulates the total bits set in x
-    uint32_t c;
-    for (c = 0; x; ++c)
-    {
-        x &= x - 1; // clear the least significant bit set
-    }
-    return c;
-}
-
-static inline bool
-BitIsSet (const uint64_t value, const uint64_t bit)
-{
-    return (value & (1ull << bit)) != 0;
-}
-
-static inline bool
-BitIsClear (const uint64_t value, const uint64_t bit)
-{
-    return (value & (1ull << bit)) == 0;
-}
-
-/// Various utilities for checking the target specific flags.
-
-/// A unary data processing instruction doesn't have an Rn operand.
-static inline bool isUnaryDP(uint64_t TSFlags) {
-  return (TSFlags & ARMII::UnaryDP);
-}
-
-/// A NEON Domain instruction has cond field (Inst{31-28}) as 0b1111.
-static inline bool isNEONDomain(uint64_t TSFlags) {
-  return (TSFlags & ARMII::DomainNEON) ||
-         (TSFlags & ARMII::DomainNEONA8);
-}
-
-/// This four-bit field describes the addressing mode used.
-/// See also ARMBaseInstrInfo.h.
-static inline unsigned getAddrMode(uint64_t TSFlags) {
-  return (TSFlags & ARMII::AddrModeMask);
-}
-
-/// {IndexModePre, IndexModePost}
-/// Only valid for load and store ops.
-/// See also ARMBaseInstrInfo.h.
-static inline unsigned getIndexMode(uint64_t TSFlags) {
-  return (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
-}
-
-/// Pre-/post-indexed operations define an extra $base_wb in the OutOperandList.
-static inline bool isPrePostLdSt(uint64_t TSFlags) {
-  return (TSFlags & ARMII::IndexModeMask) != 0;
-}
-
-// Forward declaration.
-class ARMBasicMCBuilder;
-
-// Builder Object is mostly ignored except in some Thumb disassemble functions.
-typedef ARMBasicMCBuilder *BO;
-
-/// DisassembleFP - DisassembleFP points to a function that disassembles an insn
-/// and builds the MCOperand list upon disassembly.  It returns false on failure
-/// or true on success.  The number of operands added is updated upon success.
-typedef bool (*DisassembleFP)(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder);
-
-/// CreateMCBuilder - Return an ARMBasicMCBuilder that can build up the MC
-/// infrastructure of an MCInst given the Opcode and Format of the instr.
-/// Return NULL if it fails to create/return a proper builder.  API clients
-/// are responsible for freeing up of the allocated memory.  Cacheing can be
-/// performed by the API clients to improve performance.
-extern ARMBasicMCBuilder *CreateMCBuilder(unsigned Opcode, ARMFormat Format);
-
-/// ARMBasicMCBuilder - ARMBasicMCBuilder represents an ARM MCInst builder that
-/// knows how to build up the MCOperand list.
-class ARMBasicMCBuilder {
-  friend ARMBasicMCBuilder *CreateMCBuilder(unsigned Opcode, ARMFormat Format);
-  unsigned Opcode;
-  ARMFormat Format;
-  unsigned short NumOps;
-  DisassembleFP Disasm;
-  Session *SP;
-  int Err; // !=0 if the builder encounters some error condition during build.
-
-private:
-  /// Opcode, Format, and NumOperands make up an ARM Basic MCBuilder.
-  ARMBasicMCBuilder(unsigned opc, ARMFormat format, unsigned short num);
-
-public:
-  ARMBasicMCBuilder(ARMBasicMCBuilder &B)
-    : Opcode(B.Opcode), Format(B.Format), NumOps(B.NumOps), Disasm(B.Disasm),
-      SP(B.SP), GetOpInfo(0), DisInfo(0), Ctx(0) {
-    Err = 0;
-  }
-
-  virtual ~ARMBasicMCBuilder() {}
-
-  void SetSession(Session *sp) {
-    SP = sp;
-  }
-
-  void SetErr(int ErrCode) {
-    Err = ErrCode;
-  }
-
-  /// DoPredicateOperands - DoPredicateOperands process the predicate operands
-  /// of some Thumb instructions which come before the reglist operands.  It
-  /// returns true if the two predicate operands have been processed.
-  bool DoPredicateOperands(MCInst& MI, unsigned Opcode,
-      uint32_t insn, unsigned short NumOpsRemaning);
-  
-  /// TryPredicateAndSBitModifier - TryPredicateAndSBitModifier tries to process
-  /// the possible Predicate and SBitModifier, to build the remaining MCOperand
-  /// constituents.
-  bool TryPredicateAndSBitModifier(MCInst& MI, unsigned Opcode,
-      uint32_t insn, unsigned short NumOpsRemaning);
-
-  /// InITBlock - InITBlock returns true if we are inside an IT block.
-  bool InITBlock() {
-    if (SP)
-      return SP->ITCounter > 0;
-
-    return false;
-  }
-
-  /// Build - Build delegates to BuildIt to perform the heavy liftling.  After
-  /// that, it invokes RunBuildAfterHook where some housekeepings can be done.
-  virtual bool Build(MCInst &MI, uint32_t insn) {
-    bool Status = BuildIt(MI, insn);
-    return RunBuildAfterHook(Status, MI, insn);
-  }
-
-  /// BuildIt - BuildIt performs the build step for this ARM Basic MC Builder.
-  /// The general idea is to set the Opcode for the MCInst, followed by adding
-  /// the appropriate MCOperands to the MCInst.  ARM Basic MC Builder delegates
-  /// to the Format-specific disassemble function for disassembly, followed by
-  /// TryPredicateAndSBitModifier() for PredicateOperand and OptionalDefOperand
-  /// which follow the Dst/Src Operands.
-  virtual bool BuildIt(MCInst &MI, uint32_t insn);
-
-  /// RunBuildAfterHook - RunBuildAfterHook performs operations deemed necessary
-  /// after BuildIt is finished.
-  virtual bool RunBuildAfterHook(bool Status, MCInst &MI, uint32_t insn);
-
-private:
-  /// Get condition of the current IT instruction.
-  unsigned GetITCond() {
-    assert(SP);
-    return slice(SP->ITState, 7, 4);
-  }
-
-private:
-  //
-  // Hooks for symbolic disassembly via the public 'C' interface.
-  //
-  // The function to get the symbolic information for operands.
-  LLVMOpInfoCallback GetOpInfo;
-  // The pointer to the block of symbolic information for above call back.
-  void *DisInfo;
-  // The assembly context for creating symbols and MCExprs in place of
-  // immediate operands when there is symbolic information.
-  MCContext *Ctx;
-  // The address of the instruction being disassembled.
-  uint64_t Address;
-
-public:
-  void setupBuilderForSymbolicDisassembly(LLVMOpInfoCallback getOpInfo,
-                                          void *disInfo, MCContext *ctx,
-                                          uint64_t address) {
-    GetOpInfo = getOpInfo;
-    DisInfo = disInfo;
-    Ctx = ctx;
-    Address = address;
-  }
-
-  uint64_t getBuilderAddress() const { return Address; }
-
-  /// tryAddingSymbolicOperand - tryAddingSymbolicOperand trys to add a symbolic
-  /// operand in place of the immediate Value in the MCInst.  The immediate
-  /// Value has had any PC adjustment made by the caller.  If the getOpInfo()
-  /// function was set as part of the setupBuilderForSymbolicDisassembly() call
-  /// then that function is called to get any symbolic information at the
-  /// builder's Address for this instrution.  If that returns non-zero then the
-  /// symbolic information it returns is used to create an MCExpr and that is
-  /// added as an operand to the MCInst.  This function returns true if it adds
-  /// an operand to the MCInst and false otherwise.
-  bool tryAddingSymbolicOperand(uint64_t Value, uint64_t InstSize, MCInst &MI);
-
-};
-
-} // namespace llvm
-
-#endif
diff --git a/contrib/llvm/lib/Target/ARM/Disassembler/ThumbDisassemblerCore.h b/contrib/llvm/lib/Target/ARM/Disassembler/ThumbDisassemblerCore.h
deleted file mode 100644
index 834c6f65295d..000000000000
--- a/contrib/llvm/lib/Target/ARM/Disassembler/ThumbDisassemblerCore.h
+++ /dev/null
@@ -1,2459 +0,0 @@
-//===- ThumbDisassemblerCore.h - Thumb disassembler helpers -----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is part of the ARM Disassembler.
-// It contains code for disassembling a Thumb instr.  It is to be included by
-// ARMDisassemblerCore.cpp because it contains the static DisassembleThumbFrm()
-// function which acts as the dispatcher to disassemble a Thumb instruction.
-//
-//===----------------------------------------------------------------------===//
-
-///////////////////////////////
-//                           //
-//     Utility Functions     //
-//                           //
-///////////////////////////////
-
-// Utilities for 16-bit Thumb instructions.
-/*
-15 14 13 12 11 10  9  8  7  6  5  4  3  2  1  0
-               [  tRt ]
-                      [ tRm ]  [ tRn ]  [ tRd ]
-                         D  [   Rm   ]  [  Rd ]
-
-                      [ imm3]
-               [    imm5    ]
-                   i     [    imm5   ]
-                            [       imm7      ]
-                         [       imm8         ]
-               [             imm11            ]
-
-            [   cond  ]
-*/
-
-// Extract tRt: Inst{10-8}.
-static inline unsigned getT1tRt(uint32_t insn) {
-  return slice(insn, 10, 8);
-}
-
-// Extract tRm: Inst{8-6}.
-static inline unsigned getT1tRm(uint32_t insn) {
-  return slice(insn, 8, 6);
-}
-
-// Extract tRn: Inst{5-3}.
-static inline unsigned getT1tRn(uint32_t insn) {
-  return slice(insn, 5, 3);
-}
-
-// Extract tRd: Inst{2-0}.
-static inline unsigned getT1tRd(uint32_t insn) {
-  return slice(insn, 2, 0);
-}
-
-// Extract [D:Rd]: Inst{7:2-0}.
-static inline unsigned getT1Rd(uint32_t insn) {
-  return slice(insn, 7, 7) << 3 | slice(insn, 2, 0);
-}
-
-// Extract Rm: Inst{6-3}.
-static inline unsigned getT1Rm(uint32_t insn) {
-  return slice(insn, 6, 3);
-}
-
-// Extract imm3: Inst{8-6}.
-static inline unsigned getT1Imm3(uint32_t insn) {
-  return slice(insn, 8, 6);
-}
-
-// Extract imm5: Inst{10-6}.
-static inline unsigned getT1Imm5(uint32_t insn) {
-  return slice(insn, 10, 6);
-}
-
-// Extract i:imm5: Inst{9:7-3}.
-static inline unsigned getT1Imm6(uint32_t insn) {
-  return slice(insn, 9, 9) << 5 | slice(insn, 7, 3);
-}
-
-// Extract imm7: Inst{6-0}.
-static inline unsigned getT1Imm7(uint32_t insn) {
-  return slice(insn, 6, 0);
-}
-
-// Extract imm8: Inst{7-0}.
-static inline unsigned getT1Imm8(uint32_t insn) {
-  return slice(insn, 7, 0);
-}
-
-// Extract imm11: Inst{10-0}.
-static inline unsigned getT1Imm11(uint32_t insn) {
-  return slice(insn, 10, 0);
-}
-
-// Extract cond: Inst{11-8}.
-static inline unsigned getT1Cond(uint32_t insn) {
-  return slice(insn, 11, 8);
-}
-
-static inline bool IsGPR(unsigned RegClass) {
-  return RegClass == ARM::GPRRegClassID || RegClass == ARM::rGPRRegClassID;
-}
-
-// Utilities for 32-bit Thumb instructions.
-
-static inline bool BadReg(uint32_t n) { return n == 13 || n == 15; }
-
-// Extract imm4: Inst{19-16}.
-static inline unsigned getImm4(uint32_t insn) {
-  return slice(insn, 19, 16);
-}
-
-// Extract imm3: Inst{14-12}.
-static inline unsigned getImm3(uint32_t insn) {
-  return slice(insn, 14, 12);
-}
-
-// Extract imm8: Inst{7-0}.
-static inline unsigned getImm8(uint32_t insn) {
-  return slice(insn, 7, 0);
-}
-
-// A8.6.61 LDRB (immediate, Thumb) and friends
-// +/-: Inst{9}
-// imm8: Inst{7-0}
-static inline int decodeImm8(uint32_t insn) {
-  int Offset = getImm8(insn);
-  return slice(insn, 9, 9) ? Offset : -Offset;
-}
-
-// Extract imm12: Inst{11-0}.
-static inline unsigned getImm12(uint32_t insn) {
-  return slice(insn, 11, 0);
-}
-
-// A8.6.63 LDRB (literal) and friends
-// +/-: Inst{23}
-// imm12: Inst{11-0}
-static inline int decodeImm12(uint32_t insn) {
-  int Offset = getImm12(insn);
-  return slice(insn, 23, 23) ? Offset : -Offset;
-}
-
-// Extract imm2: Inst{7-6}.
-static inline unsigned getImm2(uint32_t insn) {
-  return slice(insn, 7, 6);
-}
-
-// For BFI, BFC, t2SBFX, and t2UBFX.
-// Extract lsb: Inst{14-12:7-6}.
-static inline unsigned getLsb(uint32_t insn) {
-  return getImm3(insn) << 2 | getImm2(insn);
-}
-
-// For BFI and BFC.
-// Extract msb: Inst{4-0}.
-static inline unsigned getMsb(uint32_t insn) {
-  return slice(insn, 4, 0);
-}
-
-// For t2SBFX and t2UBFX.
-// Extract widthminus1: Inst{4-0}.
-static inline unsigned getWidthMinus1(uint32_t insn) {
-  return slice(insn, 4, 0);
-}
-
-// For t2ADDri12 and t2SUBri12.
-// imm12 = i:imm3:imm8;
-static inline unsigned getIImm3Imm8(uint32_t insn) {
-  return slice(insn, 26, 26) << 11 | getImm3(insn) << 8 | getImm8(insn);
-}
-
-// For t2MOVi16 and t2MOVTi16.
-// imm16 = imm4:i:imm3:imm8;
-static inline unsigned getImm16(uint32_t insn) {
-  return getImm4(insn) << 12 | slice(insn, 26, 26) << 11 |
-    getImm3(insn) << 8 | getImm8(insn);
-}
-
-// Inst{5-4} encodes the shift type.
-static inline unsigned getShiftTypeBits(uint32_t insn) {
-  return slice(insn, 5, 4);
-}
-
-// Inst{14-12}:Inst{7-6} encodes the imm5 shift amount.
-static inline unsigned getShiftAmtBits(uint32_t insn) {
-  return getImm3(insn) << 2 | getImm2(insn);
-}
-
-// A8.6.17 BFC
-// Encoding T1 ARMv6T2, ARMv7
-// LLVM-specific encoding for #<lsb> and #<width>
-static inline bool getBitfieldInvMask(uint32_t insn, uint32_t &mask) {
-  uint32_t lsb = getImm3(insn) << 2 | getImm2(insn);
-  uint32_t msb = getMsb(insn);
-  uint32_t Val = 0;
-  if (msb < lsb) {
-    DEBUG(errs() << "Encoding error: msb < lsb\n");
-    return false;
-  }
-  for (uint32_t i = lsb; i <= msb; ++i)
-    Val |= (1 << i);
-  mask = ~Val;
-  return true;
-}
-
-// A8.4 Shifts applied to a register
-// A8.4.1 Constant shifts
-// A8.4.3 Pseudocode details of instruction-specified shifts and rotates
-//
-// decodeImmShift() returns the shift amount and the the shift opcode.
-// Note that, as of Jan-06-2010, LLVM does not support rrx shifted operands yet.
-static inline unsigned decodeImmShift(unsigned bits2, unsigned imm5,
-                                      ARM_AM::ShiftOpc &ShOp) {
-
-  assert(imm5 < 32 && "Invalid imm5 argument");
-  switch (bits2) {
-  default: assert(0 && "No such value");
-  case 0:
-    ShOp = (imm5 == 0 ? ARM_AM::no_shift : ARM_AM::lsl);
-    return imm5;
-  case 1:
-    ShOp = ARM_AM::lsr;
-    return (imm5 == 0 ? 32 : imm5);
-  case 2:
-    ShOp = ARM_AM::asr;
-    return (imm5 == 0 ? 32 : imm5);
-  case 3:
-    ShOp = (imm5 == 0 ? ARM_AM::rrx : ARM_AM::ror);
-    return (imm5 == 0 ? 1 : imm5);
-  }
-}
-
-// A6.3.2 Modified immediate constants in Thumb instructions
-//
-// ThumbExpandImm() returns the modified immediate constant given an imm12 for
-// Thumb data-processing instructions with modified immediate.
-// See also A6.3.1 Data-processing (modified immediate).
-static inline unsigned ThumbExpandImm(unsigned imm12) {
-  assert(imm12 <= 0xFFF && "Invalid imm12 argument");
-
-  // If the leading two bits is 0b00, the modified immediate constant is
-  // obtained by splatting the low 8 bits into the first byte, every other byte,
-  // or every byte of a 32-bit value.
-  //
-  // Otherwise, a rotate right of '1':imm12<6:0> by the amount imm12<11:7> is
-  // performed.
-
-  if (slice(imm12, 11, 10) == 0) {
-    unsigned short control = slice(imm12, 9, 8);
-    unsigned imm8 = slice(imm12, 7, 0);
-    switch (control) {
-    default:
-      assert(0 && "No such value");
-      return 0;
-    case 0:
-      return imm8;
-    case 1:
-      return imm8 << 16 | imm8;
-    case 2:
-      return imm8 << 24 | imm8 << 8;
-    case 3:
-      return imm8 << 24 | imm8 << 16 | imm8 << 8 | imm8;
-    }
-  } else {
-    // A rotate is required.
-    unsigned Val = 1 << 7 | slice(imm12, 6, 0);
-    unsigned Amt = slice(imm12, 11, 7);
-    return ARM_AM::rotr32(Val, Amt);
-  }
-}
-
-static inline int decodeImm32_B_EncodingT3(uint32_t insn) {
-  bool S = slice(insn, 26, 26);
-  bool J1 = slice(insn, 13, 13);
-  bool J2 = slice(insn, 11, 11);
-  unsigned Imm21 = slice(insn, 21, 16) << 12 | slice(insn, 10, 0) << 1;
-  if (S) Imm21 |= 1 << 20;
-  if (J2) Imm21 |= 1 << 19;
-  if (J1) Imm21 |= 1 << 18;
-
-  return SignExtend32<21>(Imm21);
-}
-
-static inline int decodeImm32_B_EncodingT4(uint32_t insn) {
-  unsigned S = slice(insn, 26, 26);
-  bool I1 = slice(insn, 13, 13) == S;
-  bool I2 = slice(insn, 11, 11) == S;
-  unsigned Imm25 = slice(insn, 25, 16) << 12 | slice(insn, 10, 0) << 1;
-  if (S) Imm25 |= 1 << 24;
-  if (I1) Imm25 |= 1 << 23;
-  if (I2) Imm25 |= 1 << 22;
-
-  return SignExtend32<25>(Imm25);
-}
-
-static inline int decodeImm32_BL(uint32_t insn) {
-  unsigned S = slice(insn, 26, 26);
-  bool I1 = slice(insn, 13, 13) == S;
-  bool I2 = slice(insn, 11, 11) == S;
-  unsigned Imm25 = slice(insn, 25, 16) << 12 | slice(insn, 10, 0) << 1;
-  if (S) Imm25 |= 1 << 24;
-  if (I1) Imm25 |= 1 << 23;
-  if (I2) Imm25 |= 1 << 22;
-
-  return SignExtend32<25>(Imm25);
-}
-
-static inline int decodeImm32_BLX(uint32_t insn) {
-  unsigned S = slice(insn, 26, 26);
-  bool I1 = slice(insn, 13, 13) == S;
-  bool I2 = slice(insn, 11, 11) == S;
-  unsigned Imm25 = slice(insn, 25, 16) << 12 | slice(insn, 10, 1) << 2;
-  if (S) Imm25 |= 1 << 24;
-  if (I1) Imm25 |= 1 << 23;
-  if (I2) Imm25 |= 1 << 22;
-
-  return SignExtend32<25>(Imm25);
-}
-
-// See, for example, A8.6.221 SXTAB16.
-static inline unsigned decodeRotate(uint32_t insn) {
-  unsigned rotate = slice(insn, 5, 4);
-  return rotate << 3;
-}
-
-///////////////////////////////////////////////
-//                                           //
-// Thumb1 instruction disassembly functions. //
-//                                           //
-///////////////////////////////////////////////
-
-// See "Utilities for 16-bit Thumb instructions" for register naming convention.
-
-// A6.2.1 Shift (immediate), add, subtract, move, and compare
-//
-// shift immediate:         tRd CPSR tRn imm5
-// add/sub register:        tRd CPSR tRn tRm
-// add/sub 3-bit immediate: tRd CPSR tRn imm3
-// add/sub 8-bit immediate: tRt CPSR tRt(TIED_TO) imm8
-// mov/cmp immediate:       tRt [CPSR] imm8 (CPSR present for mov)
-//
-// Special case:
-// tMOVSr:                  tRd tRn
-static bool DisassembleThumb1General(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID
-         && "Invalid arguments");
-
-  bool Imm3 = (Opcode == ARM::tADDi3 || Opcode == ARM::tSUBi3);
-
-  // Use Rt implies use imm8.
-  bool UseRt = (Opcode == ARM::tADDi8 || Opcode == ARM::tSUBi8 ||
-                Opcode == ARM::tMOVi8 || Opcode == ARM::tCMPi8);
-
-  // Add the destination operand.
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, ARM::tGPRRegClassID,
-                                  UseRt ? getT1tRt(insn) : getT1tRd(insn))));
-  ++OpIdx;
-
-  // Check whether the next operand to be added is a CCR Register.
-  if (OpInfo[OpIdx].RegClass == ARM::CCRRegClassID) {
-    assert(OpInfo[OpIdx].isOptionalDef() && "Optional def operand expected");
-    MI.addOperand(MCOperand::CreateReg(B->InITBlock() ? 0 : ARM::CPSR));
-    ++OpIdx;
-  }
-
-  // Check whether the next operand to be added is a Thumb1 Register.
-  assert(OpIdx < NumOps && "More operands expected");
-  if (OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID) {
-    // For UseRt, the reg operand is tied to the first reg operand.
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, ARM::tGPRRegClassID,
-                                    UseRt ? getT1tRt(insn) : getT1tRn(insn))));
-    ++OpIdx;
-  }
-
-  // Special case for tMOVSr.
-  if (OpIdx == NumOps)
-    return true;
-
-  // The next available operand is either a reg operand or an imm operand.
-  if (OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID) {
-    // Three register operand instructions.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                       getT1tRm(insn))));
-  } else {
-    assert(OpInfo[OpIdx].RegClass < 0 &&
-           !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()
-           && "Pure imm operand expected");
-    unsigned Imm = 0;
-    if (UseRt)
-      Imm = getT1Imm8(insn);
-    else if (Imm3)
-      Imm = getT1Imm3(insn);
-    else {
-      Imm = getT1Imm5(insn);
-      ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 12, 11));
-      getImmShiftSE(ShOp, Imm);
-    }
-    MI.addOperand(MCOperand::CreateImm(Imm));
-  }
-  ++OpIdx;
-
-  return true;
-}
-
-// A6.2.2 Data-processing
-//
-// tCMPr, tTST, tCMN: tRd tRn
-// tMVN, tRSB:        tRd CPSR tRn
-// Others:            tRd CPSR tRd(TIED_TO) tRn
-static bool DisassembleThumb1DP(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
-         (OpInfo[1].RegClass == ARM::CCRRegClassID
-          || OpInfo[1].RegClass == ARM::tGPRRegClassID)
-         && "Invalid arguments");
-
-  // Add the destination operand.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRd(insn))));
-  ++OpIdx;
-
-  // Check whether the next operand to be added is a CCR Register.
-  if (OpInfo[OpIdx].RegClass == ARM::CCRRegClassID) {
-    assert(OpInfo[OpIdx].isOptionalDef() && "Optional def operand expected");
-    MI.addOperand(MCOperand::CreateReg(B->InITBlock() ? 0 : ARM::CPSR));
-    ++OpIdx;
-  }
-
-  // We have either { tRd(TIED_TO), tRn } or { tRn } remaining.
-  // Process the TIED_TO operand first.
-
-  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID
-         && "Thumb reg operand expected");
-  int Idx;
-  if ((Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-    // The reg operand is tied to the first reg operand.
-    MI.addOperand(MI.getOperand(Idx));
-    ++OpIdx;
-  }
-
-  // Process possible next reg operand.
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID) {
-    // Add tRn operand.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                       getT1tRn(insn))));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// A6.2.3 Special data instructions and branch and exchange
-//
-// tADDhirr: Rd Rd(TIED_TO) Rm
-// tCMPhir:  Rd Rm
-// tMOVr, tMOVgpr2gpr, tMOVgpr2tgpr, tMOVtgpr2gpr: Rd|tRd Rm|tRn
-// tBX: Rm
-// tBX_RET: 0 operand
-// tBX_RET_vararg: Rm
-// tBLXr_r9: Rm
-// tBRIND: Rm
-static bool DisassembleThumb1Special(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  // tBX_RET has 0 operand.
-  if (NumOps == 0)
-    return true;
-
-  // BX/BLX/tBRIND (indirect branch, i.e, mov pc, Rm) has 1 reg operand: Rm.
-  if (Opcode==ARM::tBLXr_r9 || Opcode==ARM::tBX || Opcode==ARM::tBRIND) {
-    if (Opcode == ARM::tBLXr_r9) {
-      // Handling the two predicate operands before the reg operand.
-      if (!B->DoPredicateOperands(MI, Opcode, insn, NumOps))
-        return false;
-      NumOpsAdded += 2;
-    }
-
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       getT1Rm(insn))));
-    NumOpsAdded += 1;
-
-    if (Opcode == ARM::tBX) {
-      // Handling the two predicate operands after the reg operand.
-      if (!B->DoPredicateOperands(MI, Opcode, insn, NumOps))
-        return false;
-      NumOpsAdded += 2;
-    }
-
-    return true;
-  }
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  // Add the destination operand.
-  unsigned RegClass = OpInfo[OpIdx].RegClass;
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass,
-                                  IsGPR(RegClass) ? getT1Rd(insn)
-                                                  : getT1tRd(insn))));
-  ++OpIdx;
-
-  // We have either { Rd(TIED_TO), Rm } or { Rm|tRn } remaining.
-  // Process the TIED_TO operand first.
-
-  assert(OpIdx < NumOps && "More operands expected");
-  int Idx;
-  if ((Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-    // The reg operand is tied to the first reg operand.
-    MI.addOperand(MI.getOperand(Idx));
-    ++OpIdx;
-  }
-
-  // The next reg operand is either Rm or tRn.
-  assert(OpIdx < NumOps && "More operands expected");
-  RegClass = OpInfo[OpIdx].RegClass;
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RegClass,
-                                  IsGPR(RegClass) ? getT1Rm(insn)
-                                                  : getT1tRn(insn))));
-  ++OpIdx;
-
-  return true;
-}
-
-// A8.6.59 LDR (literal)
-//
-// tLDRpci: tRt imm8*4
-static bool DisassembleThumb1LdPC(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
-         (OpInfo[1].RegClass < 0 &&
-          !OpInfo[1].isPredicate() &&
-          !OpInfo[1].isOptionalDef())
-         && "Invalid arguments");
-
-  // Add the destination operand.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRt(insn))));
-
-  // And the (imm8 << 2) operand.
-  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn) << 2));
-
-  NumOpsAdded = 2;
-
-  return true;
-}
-
-// Thumb specific addressing modes (see ARMInstrThumb.td):
-//
-// t_addrmode_rr := reg + reg
-//
-// t_addrmode_s4 := reg + reg
-//                  reg + imm5 * 4
-//
-// t_addrmode_s2 := reg + reg
-//                  reg + imm5 * 2
-//
-// t_addrmode_s1 := reg + reg
-//                  reg + imm5
-//
-// t_addrmode_sp := sp + imm8 * 4
-//
-
-// A8.6.63 LDRB (literal)
-// A8.6.79 LDRSB (literal)
-// A8.6.75 LDRH (literal)
-// A8.6.83 LDRSH (literal)
-// A8.6.59 LDR (literal)
-//
-// These instrs calculate an address from the PC value and an immediate offset.
-// Rd Rn=PC (+/-)imm12 (+ if Inst{23} == 0b1)
-static bool DisassembleThumb2Ldpci(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 2 &&
-         OpInfo[0].RegClass == ARM::GPRRegClassID &&
-         OpInfo[1].RegClass < 0 &&
-         "Expect >= 2 operands, first as reg, and second as imm operand");
-
-  // Build the register operand, followed by the (+/-)imm12 immediate.
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRd(insn))));
-
-  MI.addOperand(MCOperand::CreateImm(decodeImm12(insn)));
-
-  NumOpsAdded = 2;
-
-  return true;
-}
-
-
-// A6.2.4 Load/store single data item
-//
-// Load/Store Register (reg|imm):      tRd tRn imm5|tRm
-// Load Register Signed Byte|Halfword: tRd tRn tRm
-static bool DisassembleThumb1LdSt(unsigned opA, MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  assert(NumOps >= 2
-         && OpInfo[0].RegClass == ARM::tGPRRegClassID
-         && OpInfo[1].RegClass == ARM::tGPRRegClassID
-         && "Expect >= 2 operands and first two as thumb reg operands");
-
-  // Add the destination reg and the base reg.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRd(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRn(insn))));
-  OpIdx = 2;
-
-  // We have either { imm5 } or { tRm } remaining.
-  // Note that STR/LDR (register) should skip the imm5 offset operand for
-  // t_addrmode_s[1|2|4].
-
-  assert(OpIdx < NumOps && "More operands expected");
-
-  if (OpInfo[OpIdx].RegClass < 0 && !OpInfo[OpIdx].isPredicate() &&
-      !OpInfo[OpIdx].isOptionalDef()) {
-    // Table A6-5 16-bit Thumb Load/store instructions
-    // opA = 0b0101 for STR/LDR (register) and friends.
-    // Otherwise, we have STR/LDR (immediate) and friends.
-    assert(opA != 5 && "Immediate operand expected for this opcode");
-    MI.addOperand(MCOperand::CreateImm(getT1Imm5(insn)));
-    ++OpIdx;
-  } else {
-    // The next reg operand is tRm, the offset.
-    assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID
-           && "Thumb reg operand expected");
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                       getT1tRm(insn))));
-    ++OpIdx;
-  }
-  return true;
-}
-
-// A6.2.4 Load/store single data item
-//
-// Load/Store Register SP relative: tRt ARM::SP imm8
-static bool DisassembleThumb1LdStSP(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert((Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi)
-         && "Unexpected opcode");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::tGPRRegClassID &&
-         OpInfo[1].RegClass == ARM::GPRRegClassID &&
-         (OpInfo[2].RegClass < 0 &&
-          !OpInfo[2].isPredicate() &&
-          !OpInfo[2].isOptionalDef())
-         && "Invalid arguments");
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRt(insn))));
-  MI.addOperand(MCOperand::CreateReg(ARM::SP));
-  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn)));
-  NumOpsAdded = 3;
-  return true;
-}
-
-// Table A6-1 16-bit Thumb instruction encoding
-// A8.6.10 ADR
-//
-// tADDrPCi: tRt imm8
-static bool DisassembleThumb1AddPCi(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(Opcode == ARM::tADDrPCi && "Unexpected opcode");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
-         (OpInfo[1].RegClass < 0 &&
-          !OpInfo[1].isPredicate() &&
-          !OpInfo[1].isOptionalDef())
-         && "Invalid arguments");
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRt(insn))));
-  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn)));
-  NumOpsAdded = 2;
-  return true;
-}
-
-// Table A6-1 16-bit Thumb instruction encoding
-// A8.6.8 ADD (SP plus immediate)
-//
-// tADDrSPi: tRt ARM::SP imm8
-static bool DisassembleThumb1AddSPi(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(Opcode == ARM::tADDrSPi && "Unexpected opcode");
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::tGPRRegClassID &&
-         OpInfo[1].RegClass == ARM::GPRRegClassID &&
-         (OpInfo[2].RegClass < 0 &&
-          !OpInfo[2].isPredicate() &&
-          !OpInfo[2].isOptionalDef())
-         && "Invalid arguments");
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRt(insn))));
-  MI.addOperand(MCOperand::CreateReg(ARM::SP));
-  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn)));
-  NumOpsAdded = 3;
-  return true;
-}
-
-// tPUSH, tPOP: Pred-Imm Pred-CCR register_list
-//
-// where register_list = low registers + [lr] for PUSH or
-//                       low registers + [pc] for POP
-//
-// "low registers" is specified by Inst{7-0}
-// lr|pc is specified by Inst{8}
-static bool DisassembleThumb1PushPop(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert((Opcode == ARM::tPUSH || Opcode == ARM::tPOP) && "Unexpected opcode");
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  // Handling the two predicate operands before the reglist.
-  if (B->DoPredicateOperands(MI, Opcode, insn, NumOps))
-    OpIdx += 2;
-  else {
-    DEBUG(errs() << "Expected predicate operands not found.\n");
-    return false;
-  }
-
-  unsigned RegListBits = slice(insn, 8, 8) << (Opcode == ARM::tPUSH ? 14 : 15)
-    | slice(insn, 7, 0);
-
-  // Fill the variadic part of reglist.
-  for (unsigned i = 0; i < 16; ++i) {
-    if ((RegListBits >> i) & 1) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         i)));
-      ++OpIdx;
-    }
-  }
-
-  return true;
-}
-
-// A6.2.5 Miscellaneous 16-bit instructions
-// Delegate to DisassembleThumb1PushPop() for tPUSH & tPOP.
-//
-// tADDspi, tSUBspi: ARM::SP ARM::SP(TIED_TO) imm7
-// t2IT:             firstcond=Inst{7-4} mask=Inst{3-0}
-// tCBNZ, tCBZ:      tRd imm6*2
-// tBKPT:            imm8
-// tNOP, tSEV, tYIELD, tWFE, tWFI:
-//   no operand (except predicate pair)
-// tSETENDBE, tSETENDLE, :
-//   no operand
-// Others:           tRd tRn
-static bool DisassembleThumb1Misc(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  if (NumOps == 0)
-    return true;
-
-  if (Opcode == ARM::tPUSH || Opcode == ARM::tPOP)
-    return DisassembleThumb1PushPop(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-
-  // Predicate operands are handled elsewhere.
-  if (NumOps == 2 &&
-      OpInfo[0].isPredicate() && OpInfo[1].isPredicate() &&
-      OpInfo[0].RegClass < 0 && OpInfo[1].RegClass == ARM::CCRRegClassID) {
-    return true;
-  }
-
-  if (Opcode == ARM::tADDspi || Opcode == ARM::tSUBspi) {
-    // Special case handling for tADDspi and tSUBspi.
-    // A8.6.8 ADD (SP plus immediate) & A8.6.215 SUB (SP minus immediate)
-    MI.addOperand(MCOperand::CreateReg(ARM::SP));
-    MI.addOperand(MCOperand::CreateReg(ARM::SP));
-    MI.addOperand(MCOperand::CreateImm(getT1Imm7(insn)));
-    NumOpsAdded = 3;
-    return true;
-  }
-
-  if (Opcode == ARM::t2IT) {
-    // Special case handling for If-Then.
-    // A8.6.50 IT
-    // Tag the (firstcond[0] bit << 4) along with mask.
-
-    // firstcond
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 7, 4)));
-
-    // firstcond[0] and mask
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));
-    NumOpsAdded = 2;
-    return true;
-  }
-
-  if (Opcode == ARM::tBKPT) {
-    MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn))); // breakpoint value
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  // CPS has a singleton $opt operand that contains the following information:
-  // The first op would be 0b10 as enable and 0b11 as disable in regular ARM,
-  // but in Thumb it's is 0 as enable and 1 as disable. So map it to ARM's
-  // default one. The second get the AIF flags from Inst{2-0}.
-  if (Opcode == ARM::tCPS) {
-    MI.addOperand(MCOperand::CreateImm(2 + slice(insn, 4, 4)));
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 2, 0)));
-    NumOpsAdded = 2;
-    return true;
-  }
-
-  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
-         (OpInfo[1].RegClass < 0 || OpInfo[1].RegClass==ARM::tGPRRegClassID)
-         && "Expect >=2 operands");
-
-  // Add the destination operand.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                     getT1tRd(insn))));
-
-  if (OpInfo[1].RegClass == ARM::tGPRRegClassID) {
-    // Two register instructions.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                       getT1tRn(insn))));
-  } else {
-    // CBNZ, CBZ
-    assert((Opcode == ARM::tCBNZ || Opcode == ARM::tCBZ) &&"Unexpected opcode");
-    MI.addOperand(MCOperand::CreateImm(getT1Imm6(insn) * 2));
-  }
-
-  NumOpsAdded = 2;
-
-  return true;
-}
-
-// A8.6.53  LDM / LDMIA
-// A8.6.189 STM / STMIA
-//
-// tLDMIA_UPD/tSTMIA_UPD: tRt tRt AM4ModeImm Pred-Imm Pred-CCR register_list
-// tLDMIA:                tRt AM4ModeImm Pred-Imm Pred-CCR register_list
-static bool DisassembleThumb1LdStMul(bool Ld, MCInst &MI, unsigned Opcode,
-                                     uint32_t insn, unsigned short NumOps,
-                                     unsigned &NumOpsAdded, BO B) {
-  assert((Opcode == ARM::tLDMIA || Opcode == ARM::tLDMIA_UPD ||
-          Opcode == ARM::tSTMIA_UPD) && "Unexpected opcode");
-
-  unsigned tRt = getT1tRt(insn);
-  NumOpsAdded = 0;
-
-  // WB register, if necessary.
-  if (Opcode == ARM::tLDMIA_UPD || Opcode == ARM::tSTMIA_UPD) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       tRt)));
-    ++NumOpsAdded;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     tRt)));
-  ++NumOpsAdded;
-
-  // Handling the two predicate operands before the reglist.
-  if (B->DoPredicateOperands(MI, Opcode, insn, NumOps)) {
-    NumOpsAdded += 2;
-  } else {
-    DEBUG(errs() << "Expected predicate operands not found.\n");
-    return false;
-  }
-
-  unsigned RegListBits = slice(insn, 7, 0);
-  if (BitCount(RegListBits) < 1) {
-    DEBUG(errs() << "if BitCount(registers) < 1 then UNPREDICTABLE\n");
-    return false;
-  }
-
-  // Fill the variadic part of reglist.
-  for (unsigned i = 0; i < 8; ++i)
-    if ((RegListBits >> i) & 1) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::tGPRRegClassID,
-                                                         i)));
-      ++NumOpsAdded;
-    }
-
-  return true;
-}
-
-static bool DisassembleThumb1LdMul(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  return DisassembleThumb1LdStMul(true, MI, Opcode, insn, NumOps, NumOpsAdded,
-                                  B);
-}
-
-static bool DisassembleThumb1StMul(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  return DisassembleThumb1LdStMul(false, MI, Opcode, insn, NumOps, NumOpsAdded,
-                                  B);
-}
-
-// A8.6.16 B Encoding T1
-// cond = Inst{11-8} & imm8 = Inst{7-0}
-// imm32 = SignExtend(imm8:'0', 32)
-//
-// tBcc: offset Pred-Imm Pred-CCR
-// tSVC: imm8 Pred-Imm Pred-CCR
-// tTRAP: 0 operand (early return)
-static bool DisassembleThumb1CondBr(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
-
-  if (Opcode == ARM::tTRAP)
-    return true;
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps == 3 && OpInfo[0].RegClass < 0 &&
-         OpInfo[1].isPredicate() && OpInfo[2].RegClass == ARM::CCRRegClassID
-         && "Exactly 3 operands expected");
-
-  unsigned Imm8 = getT1Imm8(insn);
-  MI.addOperand(MCOperand::CreateImm(
-                  Opcode == ARM::tBcc ? SignExtend32<9>(Imm8 << 1)
-                                      : (int)Imm8));
-
-  // Predicate operands by ARMBasicMCBuilder::TryPredicateAndSBitModifier().
-  // But note that for tBcc, if cond = '1110' then UNDEFINED.
-  if (Opcode == ARM::tBcc && slice(insn, 11, 8) == 14) {
-    DEBUG(errs() << "if cond = '1110' then UNDEFINED\n");
-    return false;
-  }
-  NumOpsAdded = 1;
-
-  return true;
-}
-
-// A8.6.16 B Encoding T2
-// imm11 = Inst{10-0}
-// imm32 = SignExtend(imm11:'0', 32)
-//
-// tB: offset
-static bool DisassembleThumb1Br(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps == 1 && OpInfo[0].RegClass < 0 && "1 imm operand expected");
-
-  unsigned Imm11 = getT1Imm11(insn);
-
-  MI.addOperand(MCOperand::CreateImm(SignExtend32<12>(Imm11 << 1)));
-
-  NumOpsAdded = 1;
-
-  return true;
-
-}
-
-// See A6.2 16-bit Thumb instruction encoding for instruction classes
-// corresponding to op.
-//
-// Table A6-1 16-bit Thumb instruction encoding (abridged)
-// op    Instruction or instruction class
-// ------  --------------------------------------------------------------------
-// 00xxxx  Shift (immediate), add, subtract, move, and compare on page A6-7
-// 010000  Data-processing on page A6-8
-// 010001  Special data instructions and branch and exchange on page A6-9
-// 01001x  Load from Literal Pool, see LDR (literal) on page A8-122
-// 0101xx  Load/store single data item on page A6-10
-// 011xxx
-// 100xxx
-// 10100x  Generate PC-relative address, see ADR on page A8-32
-// 10101x  Generate SP-relative address, see ADD (SP plus immediate) on
-//         page A8-28
-// 1011xx  Miscellaneous 16-bit instructions on page A6-11
-// 11000x  Store multiple registers, see STM / STMIA / STMEA on page A8-374
-// 11001x  Load multiple registers, see LDM / LDMIA / LDMFD on page A8-110 a
-// 1101xx  Conditional branch, and Supervisor Call on page A6-13
-// 11100x  Unconditional Branch, see B on page A8-44
-//
-static bool DisassembleThumb1(uint16_t op, MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  unsigned op1 = slice(op, 5, 4);
-  unsigned op2 = slice(op, 3, 2);
-  unsigned op3 = slice(op, 1, 0);
-  unsigned opA = slice(op, 5, 2);
-  switch (op1) {
-  case 0:
-    // A6.2.1 Shift (immediate), add, subtract, move, and compare
-    return DisassembleThumb1General(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-  case 1:
-    switch (op2) {
-    case 0:
-      switch (op3) {
-      case 0:
-        // A6.2.2 Data-processing
-        return DisassembleThumb1DP(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      case 1:
-        // A6.2.3 Special data instructions and branch and exchange
-        return DisassembleThumb1Special(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                        B);
-      default:
-        // A8.6.59 LDR (literal)
-        return DisassembleThumb1LdPC(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      }
-      break;
-    default:
-      // A6.2.4 Load/store single data item
-      return DisassembleThumb1LdSt(opA, MI, Opcode, insn, NumOps, NumOpsAdded,
-                                   B);
-      break;
-    }
-    break;
-  case 2:
-    switch (op2) {
-    case 0:
-      // A6.2.4 Load/store single data item
-      return DisassembleThumb1LdSt(opA, MI, Opcode, insn, NumOps, NumOpsAdded,
-                                   B);
-    case 1:
-      // A6.2.4 Load/store single data item
-      return DisassembleThumb1LdStSP(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-    case 2:
-      if (op3 <= 1) {
-        // A8.6.10 ADR
-        return DisassembleThumb1AddPCi(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                       B);
-      } else {
-        // A8.6.8 ADD (SP plus immediate)
-        return DisassembleThumb1AddSPi(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                       B);
-      }
-    default:
-      // A6.2.5 Miscellaneous 16-bit instructions
-      return DisassembleThumb1Misc(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-    }
-    break;
-  case 3:
-    switch (op2) {
-    case 0:
-      if (op3 <= 1) {
-        // A8.6.189 STM / STMIA / STMEA
-        return DisassembleThumb1StMul(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      } else {
-        // A8.6.53 LDM / LDMIA / LDMFD
-        return DisassembleThumb1LdMul(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      }
-    case 1:
-      // A6.2.6 Conditional branch, and Supervisor Call
-      return DisassembleThumb1CondBr(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-    case 2:
-      // Unconditional Branch, see B on page A8-44
-      return DisassembleThumb1Br(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-    default:
-      assert(0 && "Unreachable code");
-      break;
-    }
-    break;
-  default:
-    assert(0 && "Unreachable code");
-    break;
-  }
-
-  return false;
-}
-
-///////////////////////////////////////////////
-//                                           //
-// Thumb2 instruction disassembly functions. //
-//                                           //
-///////////////////////////////////////////////
-
-///////////////////////////////////////////////////////////
-//                                                       //
-// Note: the register naming follows the ARM convention! //
-//                                                       //
-///////////////////////////////////////////////////////////
-
-static inline bool Thumb2SRSOpcode(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-  case ARM::t2SRSDBW: case ARM::t2SRSDB:
-  case ARM::t2SRSIAW: case ARM::t2SRSIA:
-    return true;
-  }
-}
-
-static inline bool Thumb2RFEOpcode(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-  case ARM::t2RFEDBW: case ARM::t2RFEDB:
-  case ARM::t2RFEIAW: case ARM::t2RFEIA:
-    return true;
-  }
-}
-
-// t2SRS[IA|DB]W/t2SRS[IA|DB]: mode_imm = Inst{4-0}
-static bool DisassembleThumb2SRS(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded) {
-  MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));
-  NumOpsAdded = 1;
-  return true;
-}
-
-// t2RFE[IA|DB]W/t2RFE[IA|DB]: Rn
-static bool DisassembleThumb2RFE(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-  unsigned Rn = decodeRn(insn);
-  if (Rn == 15) {
-    DEBUG(errs() << "if n == 15 then UNPREDICTABLE\n");
-    return false;
-  }
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B,ARM::GPRRegClassID,Rn)));
-  NumOpsAdded = 1;
-  return true;
-}
-
-static bool DisassembleThumb2LdStMul(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  if (Thumb2SRSOpcode(Opcode))
-    return DisassembleThumb2SRS(MI, Opcode, insn, NumOps, NumOpsAdded);
-
-  if (Thumb2RFEOpcode(Opcode))
-    return DisassembleThumb2RFE(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-
-  assert((Opcode == ARM::t2LDMIA || Opcode == ARM::t2LDMIA_UPD ||
-          Opcode == ARM::t2LDMDB || Opcode == ARM::t2LDMDB_UPD ||
-          Opcode == ARM::t2STMIA || Opcode == ARM::t2STMIA_UPD ||
-          Opcode == ARM::t2STMDB || Opcode == ARM::t2STMDB_UPD)
-         && "Unexpected opcode");
-  assert(NumOps >= 4 && "Thumb2 LdStMul expects NumOps >= 4");
-
-  NumOpsAdded = 0;
-
-  unsigned Base = getRegisterEnum(B, ARM::GPRRegClassID, decodeRn(insn));
-
-  // Writeback to base.
-  if (Opcode == ARM::t2LDMIA_UPD || Opcode == ARM::t2LDMDB_UPD ||
-      Opcode == ARM::t2STMIA_UPD || Opcode == ARM::t2STMDB_UPD) {
-    MI.addOperand(MCOperand::CreateReg(Base));
-    ++NumOpsAdded;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(Base));
-  ++NumOpsAdded;
-
-  // Handling the two predicate operands before the reglist.
-  if (B->DoPredicateOperands(MI, Opcode, insn, NumOps)) {
-    NumOpsAdded += 2;
-  } else {
-    DEBUG(errs() << "Expected predicate operands not found.\n");
-    return false;
-  }
-
-  unsigned RegListBits = insn & ((1 << 16) - 1);
-
-  // Fill the variadic part of reglist.
-  for (unsigned i = 0; i < 16; ++i)
-    if ((RegListBits >> i) & 1) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                         i)));
-      ++NumOpsAdded;
-    }
-
-  return true;
-}
-
-// t2LDREX: Rd Rn
-// t2LDREXD: Rd Rs Rn
-// t2LDREXB, t2LDREXH: Rd Rn
-// t2STREX: Rs Rd Rn
-// t2STREXD: Rm Rd Rs Rn
-// t2STREXB, t2STREXH: Rm Rd Rn
-static bool DisassembleThumb2LdStEx(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2
-         && OpInfo[0].RegClass > 0
-         && OpInfo[1].RegClass > 0
-         && "Expect >=2 operands and first two as reg operands");
-
-  bool isStore = (ARM::t2STREX <= Opcode && Opcode <= ARM::t2STREXH);
-  bool isSW = (Opcode == ARM::t2LDREX || Opcode == ARM::t2STREX);
-  bool isDW = (Opcode == ARM::t2LDREXD || Opcode == ARM::t2STREXD);
-
-  unsigned Rt  = decodeRd(insn);
-  unsigned Rt2 = decodeRs(insn); // But note that this is Rd for t2STREX.
-  unsigned Rd  = decodeRm(insn);
-  unsigned Rn  = decodeRn(insn);
-
-  // Some sanity checking first.
-  if (isStore) {
-    // if d == n || d == t then UNPREDICTABLE
-    // if d == n || d == t || d == t2 then UNPREDICTABLE
-    if (isDW) {
-      if (Rd == Rn || Rd == Rt || Rd == Rt2) {
-        DEBUG(errs() << "if d == n || d == t || d == t2 then UNPREDICTABLE\n");
-        return false;
-      }
-    } else {
-      if (isSW) {
-        if (Rt2 == Rn || Rt2 == Rt) {
-          DEBUG(errs() << "if d == n || d == t then UNPREDICTABLE\n");
-          return false;
-        }
-      } else {
-        if (Rd == Rn || Rd == Rt) {
-          DEBUG(errs() << "if d == n || d == t then UNPREDICTABLE\n");
-          return false;
-        }
-      }
-    }
-  } else {
-    // Load
-    // A8.6.71 LDREXD
-    // if t == t2 then UNPREDICTABLE
-    if (isDW && Rt == Rt2) {
-      DEBUG(errs() << "if t == t2 then UNPREDICTABLE\n");
-      return false;
-    }
-  }
-
-  // Add the destination operand for store.
-  if (isStore) {
-    MI.addOperand(MCOperand::CreateReg(
-                    getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                    isSW ? Rt2 : Rd)));
-    ++OpIdx;
-  }
-
-  // Source operand for store and destination operand for load.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     Rt)));
-  ++OpIdx;
-
-  // Thumb2 doubleword complication: with an extra source/destination operand.
-  if (isDW) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B,OpInfo[OpIdx].RegClass,
-                                                       Rt2)));
-    ++OpIdx;
-  }
-
-  // Finally add the pointer operand.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     Rn)));
-  ++OpIdx;
-
-  return true;
-}
-
-// t2LDRDi8: Rd Rs Rn imm8s4 (offset mode)
-// t2LDRDpci: Rd Rs imm8s4 (Not decoded, prefer the generic t2LDRDi8 version)
-// t2STRDi8: Rd Rs Rn imm8s4 (offset mode)
-//
-// Ditto for t2LDRD_PRE, t2LDRD_POST, t2STRD_PRE, t2STRD_POST, which are for
-// disassembly only and do not have a tied_to writeback base register operand.
-static bool DisassembleThumb2LdStDual(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-  if (!OpInfo) return false;
-
-  assert(NumOps >= 4
-         && OpInfo[0].RegClass > 0
-         && OpInfo[0].RegClass == OpInfo[1].RegClass
-         && OpInfo[2].RegClass > 0
-         && OpInfo[3].RegClass < 0
-         && "Expect >= 4 operands and first 3 as reg operands");
-
-  // Thumnb allows for specifying Rt and Rt2, unlike ARM (which has Rt2==Rt+1).
-  unsigned Rt  = decodeRd(insn);
-  unsigned Rt2 = decodeRs(insn);
-  unsigned Rn  = decodeRn(insn);
-
-  // Some sanity checking first.
-
-  // A8.6.67 LDRD (literal) has its W bit as (0).
-  if (Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2LDRD_PRE || Opcode == ARM::t2LDRD_POST) {
-    if (Rn == 15 && slice(insn, 21, 21) != 0)
-      return false;
-  } else {
-    // For Dual Store, PC cannot be used as the base register.
-    if (Rn == 15) {
-      DEBUG(errs() << "if n == 15 then UNPREDICTABLE\n");
-      return false;
-    }
-  }
-  if (Rt == Rt2) {
-    DEBUG(errs() << "if t == t2 then UNPREDICTABLE\n");
-    return false;
-  }
-  if (Opcode != ARM::t2LDRDi8 && Opcode != ARM::t2STRDi8) {
-    if (Rn == Rt || Rn == Rt2) {
-      DEBUG(errs() << "if wback && (n == t || n == t2) then UNPREDICTABLE\n");
-      return false;
-    }
-  }
-
-  // Add the <Rt> <Rt2> operands.
-  unsigned RegClassPair = OpInfo[0].RegClass;
-  unsigned RegClassBase = OpInfo[2].RegClass;
-  
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassPair,
-                                                     decodeRd(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassPair,
-                                                     decodeRs(insn))));
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RegClassBase,
-                                                     decodeRn(insn))));
-
-  // Finally add (+/-)imm8*4, depending on the U bit.
-  int Offset = getImm8(insn) * 4;
-  if (getUBit(insn) == 0)
-    Offset = -Offset;
-  MI.addOperand(MCOperand::CreateImm(Offset));
-  NumOpsAdded = 4;
-
-  return true;
-}
-
-// t2TBB, t2TBH: Rn Rm Pred-Imm Pred-CCR
-static bool DisassembleThumb2TB(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  assert(NumOps >= 2 && "Expect >= 2 operands");
-
-  // The generic version of TBB/TBH needs a base register.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  // Add the index register.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRm(insn))));
-  NumOpsAdded = 2;
-
-  return true;
-}
-
-static inline bool Thumb2ShiftOpcode(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-  case ARM::t2MOVCClsl: case ARM::t2MOVCClsr:
-  case ARM::t2MOVCCasr: case ARM::t2MOVCCror:
-  case ARM::t2LSLri:    case ARM::t2LSRri:
-  case ARM::t2ASRri:    case ARM::t2RORri:
-    return true;
-  }
-}
-
-// A6.3.11 Data-processing (shifted register)
-//
-// Two register operands (Rn=0b1111 no 1st operand reg): Rs Rm
-// Two register operands (Rs=0b1111 no dst operand reg): Rn Rm
-// Three register operands: Rs Rn Rm
-// Three register operands: (Rn=0b1111 Conditional Move) Rs Ro(TIED_TO) Rm
-//
-// Constant shifts t2_so_reg is a 2-operand unit corresponding to the Thumb2
-// register with shift forms: (Rm, ConstantShiftSpecifier).
-// Constant shift specifier: Imm = (ShOp | ShAmt<<3).
-//
-// There are special instructions, like t2MOVsra_flag and t2MOVsrl_flag, which
-// only require two register operands: Rd, Rm in ARM Reference Manual terms, and
-// nothing else, because the shift amount is already specified.
-// Similar case holds for t2MOVrx, t2ADDrr, ..., etc.
-static bool DisassembleThumb2DPSoReg(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  // Special case handling.
-  if (Opcode == ARM::t2BR_JT) {
-    assert(NumOps == 4
-           && OpInfo[0].RegClass == ARM::GPRRegClassID
-           && OpInfo[1].RegClass == ARM::GPRRegClassID
-           && OpInfo[2].RegClass < 0
-           && OpInfo[3].RegClass < 0
-           && "Exactly 4 operands expect and first two as reg operands");
-    // Only need to populate the src reg operand.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-    MI.addOperand(MCOperand::CreateReg(0));
-    MI.addOperand(MCOperand::CreateImm(0));
-    MI.addOperand(MCOperand::CreateImm(0));
-    NumOpsAdded = 4;
-    return true;
-  }
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2
-         && (OpInfo[0].RegClass == ARM::GPRRegClassID ||
-             OpInfo[0].RegClass == ARM::rGPRRegClassID)
-         && (OpInfo[1].RegClass == ARM::GPRRegClassID ||
-             OpInfo[1].RegClass == ARM::rGPRRegClassID)
-         && "Expect >= 2 operands and first two as reg operands");
-
-  bool ThreeReg = (NumOps > 2 && (OpInfo[2].RegClass == ARM::GPRRegClassID ||
-                                  OpInfo[2].RegClass == ARM::rGPRRegClassID));
-  bool NoDstReg = (decodeRs(insn) == 0xF);
-
-  // Build the register operands, followed by the constant shift specifier.
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, OpInfo[0].RegClass,
-                                  NoDstReg ? decodeRn(insn) : decodeRs(insn))));
-  ++OpIdx;
-
-  if (ThreeReg) {
-    int Idx;
-    if ((Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-      // Process tied_to operand constraint.
-      MI.addOperand(MI.getOperand(Idx));
-      ++OpIdx;
-    } else if (!NoDstReg) {
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[1].RegClass,
-                                                         decodeRn(insn))));
-      ++OpIdx;
-    } else {
-      DEBUG(errs() << "Thumb2 encoding error: d==15 for three-reg operands.\n");
-      return false;
-    }
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeRm(insn))));
-  ++OpIdx;
-
-  if (NumOps == OpIdx)
-    return true;
-
-  if (OpInfo[OpIdx].RegClass < 0 && !OpInfo[OpIdx].isPredicate()
-      && !OpInfo[OpIdx].isOptionalDef()) {
-
-    if (Thumb2ShiftOpcode(Opcode)) {
-      unsigned Imm = getShiftAmtBits(insn);
-      ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 5, 4));
-      getImmShiftSE(ShOp, Imm);
-      MI.addOperand(MCOperand::CreateImm(Imm));
-    } else {
-      // Build the constant shift specifier operand.
-      unsigned bits2 = getShiftTypeBits(insn);
-      unsigned imm5 = getShiftAmtBits(insn);
-      ARM_AM::ShiftOpc ShOp = ARM_AM::no_shift;
-      unsigned ShAmt = decodeImmShift(bits2, imm5, ShOp);
-      MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(ShOp, ShAmt)));
-    }
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// A6.3.1 Data-processing (modified immediate)
-//
-// Two register operands: Rs Rn ModImm
-// One register operands (Rs=0b1111 no explicit dest reg): Rn ModImm
-// One register operands (Rn=0b1111 no explicit src reg): Rs ModImm -
-// {t2MOVi, t2MVNi}
-//
-// ModImm = ThumbExpandImm(i:imm3:imm8)
-static bool DisassembleThumb2DPModImm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  unsigned RdRegClassID = OpInfo[0].RegClass;
-  assert(NumOps >= 2 && (RdRegClassID == ARM::GPRRegClassID ||
-                         RdRegClassID == ARM::rGPRRegClassID)
-         && "Expect >= 2 operands and first one as reg operand");
-
-  unsigned RnRegClassID = OpInfo[1].RegClass;
-  bool TwoReg = (RnRegClassID == ARM::GPRRegClassID
-                 || RnRegClassID == ARM::rGPRRegClassID);
-  bool NoDstReg = (decodeRs(insn) == 0xF);
-
-  // Build the register operands, followed by the modified immediate.
-
-  MI.addOperand(MCOperand::CreateReg(
-                  getRegisterEnum(B, RdRegClassID,
-                                  NoDstReg ? decodeRn(insn) : decodeRs(insn))));
-  ++OpIdx;
-
-  if (TwoReg) {
-    if (NoDstReg) {
-      DEBUG(errs()<<"Thumb2 encoding error: d==15 for DPModImm 2-reg instr.\n");
-      return false;
-    }
-    int Idx;
-    if ((Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-      // The reg operand is tied to the first reg operand.
-      MI.addOperand(MI.getOperand(Idx));
-    } else {
-      // Add second reg operand.
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RnRegClassID,
-                                                         decodeRn(insn))));
-    }
-    ++OpIdx;
-  }
-
-  // The modified immediate operand should come next.
-  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0 &&
-         !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()
-         && "Pure imm operand expected");
-
-  // i:imm3:imm8
-  // A6.3.2 Modified immediate constants in Thumb instructions
-  unsigned imm12 = getIImm3Imm8(insn);
-  MI.addOperand(MCOperand::CreateImm(ThumbExpandImm(imm12)));
-  ++OpIdx;
-
-  return true;
-}
-
-static inline bool Thumb2SaturateOpcode(unsigned Opcode) {
-  switch (Opcode) {
-  case ARM::t2SSAT: case ARM::t2SSAT16:
-  case ARM::t2USAT: case ARM::t2USAT16:
-    return true;
-  default:
-    return false;
-  }
-}
-
-/// DisassembleThumb2Sat - Disassemble Thumb2 saturate instructions:
-/// o t2SSAT, t2USAT: Rs sat_pos Rn shamt
-/// o t2SSAT16, t2USAT16: Rs sat_pos Rn
-static bool DisassembleThumb2Sat(MCInst &MI, unsigned Opcode, uint32_t insn,
-                                 unsigned &NumOpsAdded, BO B) {
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  NumOpsAdded = MCID.getNumOperands() - 2; // ignore predicate operands
-
-  // Disassemble the register def.
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRs(insn))));
-
-  unsigned Pos = slice(insn, 4, 0);
-  if (Opcode == ARM::t2SSAT || Opcode == ARM::t2SSAT16)
-    Pos += 1;
-  MI.addOperand(MCOperand::CreateImm(Pos));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRn(insn))));
-
-  if (NumOpsAdded == 4) {
-    ARM_AM::ShiftOpc Opc = (slice(insn, 21, 21) != 0 ?
-                            ARM_AM::asr : ARM_AM::lsl);
-    // Inst{14-12:7-6} encodes the imm5 shift amount.
-    unsigned ShAmt = slice(insn, 14, 12) << 2 | slice(insn, 7, 6);
-    if (ShAmt == 0) {
-      if (Opc == ARM_AM::asr)
-        ShAmt = 32;
-      else
-        Opc = ARM_AM::no_shift;
-    }
-    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(Opc, ShAmt)));
-  }
-  return true;
-}
-
-// A6.3.3 Data-processing (plain binary immediate)
-//
-// o t2ADDri12, t2SUBri12: Rs Rn imm12
-// o t2LEApcrel (ADR): Rs imm12
-// o t2BFC (BFC): Rs Ro(TIED_TO) bf_inv_mask_imm
-// o t2BFI (BFI): Rs Ro(TIED_TO) Rn bf_inv_mask_imm
-// o t2MOVi16: Rs imm16
-// o t2MOVTi16: Rs imm16
-// o t2SBFX (SBFX): Rs Rn lsb width
-// o t2UBFX (UBFX): Rs Rn lsb width
-// o t2BFI (BFI): Rs Rn lsb width
-static bool DisassembleThumb2DPBinImm(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  unsigned RdRegClassID = OpInfo[0].RegClass;
-  assert(NumOps >= 2 && (RdRegClassID == ARM::GPRRegClassID ||
-                         RdRegClassID == ARM::rGPRRegClassID)
-         && "Expect >= 2 operands and first one as reg operand");
-
-  unsigned RnRegClassID = OpInfo[1].RegClass;
-  bool TwoReg = (RnRegClassID == ARM::GPRRegClassID
-                 || RnRegClassID == ARM::rGPRRegClassID);
-
-  // Build the register operand(s), followed by the immediate(s).
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RdRegClassID,
-                                                     decodeRs(insn))));
-  ++OpIdx;
-
-  if (TwoReg) {
-    assert(NumOps >= 3 && "Expect >= 3 operands");
-    int Idx;
-    if ((Idx = MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO)) != -1) {
-      // Process tied_to operand constraint.
-      MI.addOperand(MI.getOperand(Idx));
-    } else {
-      // Add src reg operand.
-      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RnRegClassID,
-                                                         decodeRn(insn))));
-    }
-    ++OpIdx;
-  }
-
-  if (Opcode == ARM::t2BFI) {
-    // Add val reg operand.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, RnRegClassID,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  assert(OpInfo[OpIdx].RegClass < 0 && !OpInfo[OpIdx].isPredicate()
-         && !OpInfo[OpIdx].isOptionalDef()
-         && "Pure imm operand expected");
-
-  // Pre-increment OpIdx.
-  ++OpIdx;
-
-  if (Opcode == ARM::t2ADDri12 || Opcode == ARM::t2SUBri12
-      || Opcode == ARM::t2LEApcrel)
-    MI.addOperand(MCOperand::CreateImm(getIImm3Imm8(insn)));
-  else if (Opcode == ARM::t2MOVi16 || Opcode == ARM::t2MOVTi16) {
-    if (!B->tryAddingSymbolicOperand(getImm16(insn), 4, MI))
-      MI.addOperand(MCOperand::CreateImm(getImm16(insn)));
-  } else if (Opcode == ARM::t2BFC || Opcode == ARM::t2BFI) {
-    uint32_t mask = 0;
-    if (getBitfieldInvMask(insn, mask))
-      MI.addOperand(MCOperand::CreateImm(mask));
-    else
-      return false;
-  } else {
-    // Handle the case of: lsb width
-    assert((Opcode == ARM::t2SBFX || Opcode == ARM::t2UBFX)
-            && "Unexpected opcode");
-    MI.addOperand(MCOperand::CreateImm(getLsb(insn)));
-    MI.addOperand(MCOperand::CreateImm(getWidthMinus1(insn) + 1));
-
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// A6.3.4 Table A6-15 Miscellaneous control instructions
-// A8.6.41 DMB
-// A8.6.42 DSB
-// A8.6.49 ISB
-static inline bool t2MiscCtrlInstr(uint32_t insn) {
-  if (slice(insn, 31, 20) == 0xf3b && slice(insn, 15, 14) == 2 &&
-      slice(insn, 12, 12) == 0)
-    return true;
-
-  return false;
-}
-
-// A6.3.4 Branches and miscellaneous control
-//
-// A8.6.16 B
-// Branches: t2B, t2Bcc -> imm operand
-//
-// Branches: t2TPsoft -> no operand
-//
-// A8.6.23 BL, BLX (immediate)
-// Branches (defined in ARMInstrThumb.td): tBLr9, tBLXi_r9 -> imm operand
-//
-// A8.6.26
-// t2BXJ -> Rn
-//
-// Miscellaneous control:
-//   -> no operand (except pred-imm pred-ccr for CLREX, memory barrier variants)
-//
-// Hint: t2NOP, t2YIELD, t2WFE, t2WFI, t2SEV
-//   -> no operand (except pred-imm pred-ccr)
-//
-// t2DBG -> imm4 = Inst{3-0}
-//
-// t2MRS/t2MRSsys -> Rs
-// t2MSR/t2MSRsys -> Rn mask=Inst{11-8}
-// t2SMC -> imm4 = Inst{19-16}
-static bool DisassembleThumb2BrMiscCtrl(MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  if (NumOps == 0)
-    return true;
-
-  if (Opcode == ARM::t2DMB || Opcode == ARM::t2DSB) {
-    // Inst{3-0} encodes the memory barrier option for the variants.
-    unsigned opt = slice(insn, 3, 0);
-    switch (opt) {
-    case ARM_MB::SY:  case ARM_MB::ST:
-    case ARM_MB::ISH: case ARM_MB::ISHST:
-    case ARM_MB::NSH: case ARM_MB::NSHST:
-    case ARM_MB::OSH: case ARM_MB::OSHST:
-      MI.addOperand(MCOperand::CreateImm(opt));
-      NumOpsAdded = 1;
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  if (t2MiscCtrlInstr(insn))
-    return true;
-
-  switch (Opcode) {
-  case ARM::t2CLREX:
-  case ARM::t2NOP:
-  case ARM::t2YIELD:
-  case ARM::t2WFE:
-  case ARM::t2WFI:
-  case ARM::t2SEV:
-    return true;
-  default:
-    break;
-  }
-
-  // FIXME: To enable correct asm parsing and disasm of CPS we need 3 different
-  // opcodes which match the same real instruction. This is needed since there's
-  // no current handling of optional arguments. Fix here when a better handling
-  // of optional arguments is implemented.
-  if (Opcode == ARM::t2CPS3p) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 10, 9))); // imod
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 7, 5)));  // iflags
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));  // mode
-    NumOpsAdded = 3;
-    return true;
-  }
-  if (Opcode == ARM::t2CPS2p) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 10, 9))); // imod
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 7, 5)));  // iflags
-    NumOpsAdded = 2;
-    return true;
-  }
-  if (Opcode == ARM::t2CPS1p) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0))); // mode
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  // DBG has its option specified in Inst{3-0}.
-  if (Opcode == ARM::t2DBG) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 3, 0)));
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  // MRS and MRSsys take one GPR reg Rs.
-  if (Opcode == ARM::t2MRS || Opcode == ARM::t2MRSsys) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRs(insn))));
-    NumOpsAdded = 1;
-    return true;
-  }
-  // BXJ takes one GPR reg Rn.
-  if (Opcode == ARM::t2BXJ) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    NumOpsAdded = 1;
-    return true;
-  }
-  // MSR take a mask, followed by one GPR reg Rn. The mask contains the R Bit in
-  // bit 4, and the special register fields in bits 3-0.
-  if (Opcode == ARM::t2MSR) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 20, 20) << 4 /* R Bit */ |
-                                       slice(insn, 11, 8) /* Special Reg */));
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRn(insn))));
-    NumOpsAdded = 2;
-    return true;
-  }
-  // SMC take imm4.
-  if (Opcode == ARM::t2SMC) {
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 16)));
-    NumOpsAdded = 1;
-    return true;
-  }
-
-  // Some instructions have predicate operands first before the immediate.
-  if (Opcode == ARM::tBLXi_r9 || Opcode == ARM::tBLr9) {
-    // Handling the two predicate operands before the imm operand.
-    if (B->DoPredicateOperands(MI, Opcode, insn, NumOps))
-      NumOpsAdded += 2;
-    else {
-      DEBUG(errs() << "Expected predicate operands not found.\n");
-      return false;
-    }
-  }
-
-  // Add the imm operand.
-  int Offset = 0;
-
-  switch (Opcode) {
-  default:
-    assert(0 && "Unexpected opcode");
-    return false;
-  case ARM::t2B:
-    Offset = decodeImm32_B_EncodingT4(insn);
-    break;
-  case ARM::t2Bcc:
-    Offset = decodeImm32_B_EncodingT3(insn);
-    break;
-  case ARM::tBLr9:
-    Offset = decodeImm32_BL(insn);
-    break;
-  case ARM::tBLXi_r9:
-    Offset = decodeImm32_BLX(insn);
-    break;
-  }
-
-  if (!B->tryAddingSymbolicOperand(Offset + B->getBuilderAddress() + 4, 4, MI))
-    MI.addOperand(MCOperand::CreateImm(Offset));
-
-  // This is an increment as some predicate operands may have been added first.
-  NumOpsAdded += 1;
-
-  return true;
-}
-
-static inline bool Thumb2PreloadOpcode(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-  case ARM::t2PLDi12:   case ARM::t2PLDi8:
-  case ARM::t2PLDs:
-  case ARM::t2PLDWi12:  case ARM::t2PLDWi8:
-  case ARM::t2PLDWs:
-  case ARM::t2PLIi12:   case ARM::t2PLIi8:
-  case ARM::t2PLIs:
-    return true;
-  }
-}
-
-static bool DisassembleThumb2PreLoad(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  // Preload Data/Instruction requires either 2 or 3 operands.
-  // t2PLDi12, t2PLDi8, t2PLDpci: Rn [+/-]imm12/imm8
-  // t2PLDr:                      Rn Rm
-  // t2PLDs:                      Rn Rm imm2=Inst{5-4}
-  // Same pattern applies for t2PLDW* and t2PLI*.
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2 &&
-         OpInfo[0].RegClass == ARM::GPRRegClassID &&
-         "Expect >= 2 operands and first one as reg operand");
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                     decodeRn(insn))));
-  ++OpIdx;
-
-  if (OpInfo[OpIdx].RegClass == ARM::rGPRRegClassID) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::GPRRegClassID,
-                                                       decodeRm(insn))));
-  } else {
-    assert(OpInfo[OpIdx].RegClass < 0 && !OpInfo[OpIdx].isPredicate()
-           && !OpInfo[OpIdx].isOptionalDef()
-           && "Pure imm operand expected");
-    int Offset = 0;
-    if (Opcode == ARM::t2PLDi8 || Opcode == ARM::t2PLDWi8 ||
-        Opcode == ARM::t2PLIi8) {
-      // A8.6.117 Encoding T2: add = FALSE
-      unsigned Imm8 = getImm8(insn);
-      Offset = -1 * Imm8;
-    } else {
-      // The i12 forms.  See, for example, A8.6.117 Encoding T1.
-      // Note that currently t2PLDi12 also handles the previously named t2PLDpci
-      // opcode, that's why we use decodeImm12(insn) which returns +/- imm12.
-      Offset = decodeImm12(insn);
-    }
-    MI.addOperand(MCOperand::CreateImm(Offset));
-  }
-  ++OpIdx;
-
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0 &&
-      !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-    // Fills in the shift amount for t2PLDs, t2PLDWs, t2PLIs.
-    MI.addOperand(MCOperand::CreateImm(slice(insn, 5, 4)));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-static bool BadRegsThumb2LdSt(unsigned Opcode, uint32_t insn, bool Load,
-      unsigned R0, unsigned R1, unsigned R2, bool UseRm, bool WB) {
-
-  // Inst{22-21} encodes the data item transferred for load/store.
-  // For single word, it is encoded as ob10.
-  bool Word = (slice(insn, 22, 21) == 2);
-  bool Half = (slice(insn, 22, 21) == 1);
-  bool Byte = (slice(insn, 22, 21) == 0);
-
-  if (UseRm && BadReg(R2)) {
-    DEBUG(errs() << "if BadReg(m) then UNPREDICTABLE\n");
-    return true;
-  }
-
-  if (Load) {
-    if (!Word && R0 == 13) {
-      DEBUG(errs() << "if t == 13 then UNPREDICTABLE\n");
-      return true;
-    }
-    if (Byte) {
-      if (WB && R0 == 15 && slice(insn, 10, 8) == 3)  {
-        // A8.6.78 LDRSB (immediate) Encoding T2 (errata markup 8.0)
-        DEBUG(errs() << "if t == 15 && PUW == '011' then UNPREDICTABLE\n");
-        return true;
-      }
-    }
-    // A6.3.8 Load halfword, memory hints
-    if (Half) {
-      if (WB) {
-        if (R0 == R1)  {
-          // A8.6.82 LDRSH (immediate) Encoding T2
-          DEBUG(errs() << "if WB && n == t then UNPREDICTABLE\n");
-          return true;
-        }
-        if (R0 == 15 && slice(insn, 10, 8) == 3)  {
-          // A8.6.82 LDRSH (immediate) Encoding T2 (errata markup 8.0)
-          DEBUG(errs() << "if t == 15 && PUW == '011' then UNPREDICTABLE\n");
-          return true;
-        }
-      } else {
-        if (Opcode == ARM::t2LDRHi8 || Opcode == ARM::t2LDRSHi8) {
-          if (R0 == 15 && slice(insn, 10, 8) == 4) {
-            // A8.6.82 LDRSH (immediate) Encoding T2
-            DEBUG(errs() << "if Rt == '1111' and PUW == '100' then SEE"
-                         << " \"Unallocated memory hints\"\n");
-            return true;
-          }
-        } else {
-          if (R0 == 15) {
-            // A8.6.82 LDRSH (immediate) Encoding T1
-            DEBUG(errs() << "if Rt == '1111' then SEE"
-                         << " \"Unallocated memory hints\"\n");
-            return true;
-          }
-        }
-      }
-    }
-  } else {
-    if (WB && R0 == R1) {
-      DEBUG(errs() << "if wback && n == t then UNPREDICTABLE\n");
-      return true;
-    }
-    if ((WB && R0 == 15) || (!WB && R1 == 15)) {
-      DEBUG(errs() << "if Rn == '1111' then UNDEFINED\n");
-      return true;
-    }
-    if (Word) {
-      if ((WB && R1 == 15) || (!WB && R0 == 15)) {
-        DEBUG(errs() << "if t == 15 then UNPREDICTABLE\n");
-        return true;
-      }
-    } else {
-      if ((WB && BadReg(R1)) || (!WB && BadReg(R0))) {
-        DEBUG(errs() << "if BadReg(t) then UNPREDICTABLE\n");
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
-// A6.3.10 Store single data item
-// A6.3.9 Load byte, memory hints
-// A6.3.8 Load halfword, memory hints
-// A6.3.7 Load word
-//
-// For example,
-//
-// t2LDRi12:   Rd Rn (+)imm12
-// t2LDRi8:    Rd Rn (+/-)imm8 (+ if Inst{9} == 0b1)
-// t2LDRs:     Rd Rn Rm ConstantShiftSpecifier (see also
-//             DisassembleThumb2DPSoReg)
-// t2LDR_POST: Rd Rn Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
-// t2LDR_PRE:  Rd Rn Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
-//
-// t2STRi12:   Rd Rn (+)imm12
-// t2STRi8:    Rd Rn (+/-)imm8 (+ if Inst{9} == 0b1)
-// t2STRs:     Rd Rn Rm ConstantShiftSpecifier (see also
-//             DisassembleThumb2DPSoReg)
-// t2STR_POST: Rn Rd Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
-// t2STR_PRE:  Rn Rd Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
-//
-// Note that for indexed modes, the Rn(TIED_TO) operand needs to be populated
-// correctly, as LLVM AsmPrinter depends on it.  For indexed stores, the first
-// operand is Rn; for all the other instructions, Rd is the first operand.
-//
-// Delegates to DisassembleThumb2PreLoad() for preload data/instruction.
-// Delegates to DisassembleThumb2Ldpci() for load * literal operations.
-static bool DisassembleThumb2LdSt(bool Load, MCInst &MI, unsigned Opcode,
-    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  unsigned Rn = decodeRn(insn);
-
-  if (Thumb2PreloadOpcode(Opcode))
-    return DisassembleThumb2PreLoad(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-
-  // See, for example, A6.3.7 Load word: Table A6-18 Load word.
-  if (Load && Rn == 15)
-    return DisassembleThumb2Ldpci(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 3 &&
-         OpInfo[0].RegClass > 0 &&
-         OpInfo[1].RegClass > 0 &&
-         "Expect >= 3 operands and first two as reg operands");
-
-  bool ThreeReg = (OpInfo[2].RegClass > 0);
-  bool TIED_TO = ThreeReg && MCID.getOperandConstraint(2, MCOI::TIED_TO) != -1;
-  bool Imm12 = !ThreeReg && slice(insn, 23, 23) == 1; // ARMInstrThumb2.td
-
-  // Build the register operands, followed by the immediate.
-  unsigned R0 = 0, R1 = 0, R2 = 0;
-  unsigned Rd = decodeRd(insn);
-  int Imm = 0;
-
-  if (!Load && TIED_TO) {
-    R0 = Rn;
-    R1 = Rd;
-  } else {
-    R0 = Rd;
-    R1 = Rn;
-  }
-  if (ThreeReg) {
-    if (TIED_TO) {
-      R2 = Rn;
-      Imm = decodeImm8(insn);
-    } else {
-      R2 = decodeRm(insn);
-      // See, for example, A8.6.64 LDRB (register).
-      // And ARMAsmPrinter::printT2AddrModeSoRegOperand().
-      // LSL is the default shift opc, and LLVM does not expect it to be encoded
-      // as part of the immediate operand.
-      // Imm = ARM_AM::getSORegOpc(ARM_AM::lsl, slice(insn, 5, 4));
-      Imm = slice(insn, 5, 4);
-    }
-  } else {
-    if (Imm12)
-      Imm = getImm12(insn);
-    else
-      Imm = decodeImm8(insn);
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     R0)));
-  ++OpIdx;
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     R1)));
-  ++OpIdx;
-
-  if (ThreeReg) {
-    // This could be an offset register or a TIED_TO register.
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B,OpInfo[OpIdx].RegClass,
-                                                       R2)));
-    ++OpIdx;
-  }
-
-  if (BadRegsThumb2LdSt(Opcode, insn, Load, R0, R1, R2, ThreeReg & !TIED_TO,
-                        TIED_TO))
-    return false;
-
-  assert(OpInfo[OpIdx].RegClass < 0 && !OpInfo[OpIdx].isPredicate()
-         && !OpInfo[OpIdx].isOptionalDef()
-         && "Pure imm operand expected");
-
-  MI.addOperand(MCOperand::CreateImm(Imm));
-  ++OpIdx;
-
-  return true;
-}
-
-// A6.3.12 Data-processing (register)
-//
-// Two register operands [rotate]:   Rs Rm [rotation(= (rotate:'000'))]
-// Three register operands only:     Rs Rn Rm
-// Three register operands [rotate]: Rs Rn Rm [rotation(= (rotate:'000'))]
-//
-// Parallel addition and subtraction 32-bit Thumb instructions: Rs Rn Rm
-//
-// Miscellaneous operations: Rs [Rn] Rm
-static bool DisassembleThumb2DPReg(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCInstrDesc &MCID = ARMInsts[Opcode];
-  const MCOperandInfo *OpInfo = MCID.OpInfo;
-  unsigned &OpIdx = NumOpsAdded;
-
-  OpIdx = 0;
-
-  assert(NumOps >= 2 &&
-         OpInfo[0].RegClass > 0 &&
-         OpInfo[1].RegClass > 0 &&
-         "Expect >= 2 operands and first two as reg operands");
-
-  // Build the register operands, followed by the optional rotation amount.
-
-  bool ThreeReg = NumOps > 2 && OpInfo[2].RegClass > 0;
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeRs(insn))));
-  ++OpIdx;
-
-  if (ThreeReg) {
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B,OpInfo[OpIdx].RegClass,
-                                                       decodeRn(insn))));
-    ++OpIdx;
-  }
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, OpInfo[OpIdx].RegClass,
-                                                     decodeRm(insn))));
-  ++OpIdx;
-
-  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass < 0
-      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
-    // Add the rotation amount immediate.
-    MI.addOperand(MCOperand::CreateImm(decodeRotate(insn)));
-    ++OpIdx;
-  }
-
-  return true;
-}
-
-// A6.3.16 Multiply, multiply accumulate, and absolute difference
-//
-// t2MLA, t2MLS, t2SMMLA, t2SMMLS: Rs Rn Rm Ra=Inst{15-12}
-// t2MUL, t2SMMUL:                 Rs Rn Rm
-// t2SMLA[BB|BT|TB|TT|WB|WT]:      Rs Rn Rm Ra=Inst{15-12}
-// t2SMUL[BB|BT|TB|TT|WB|WT]:      Rs Rn Rm
-//
-// Dual halfword multiply: t2SMUAD[X], t2SMUSD[X], t2SMLAD[X], t2SMLSD[X]:
-//   Rs Rn Rm Ra=Inst{15-12}
-//
-// Unsigned Sum of Absolute Differences [and Accumulate]
-//    Rs Rn Rm [Ra=Inst{15-12}]
-static bool DisassembleThumb2Mul(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-
-  assert(NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::rGPRRegClassID &&
-         OpInfo[1].RegClass == ARM::rGPRRegClassID &&
-         OpInfo[2].RegClass == ARM::rGPRRegClassID &&
-         "Expect >= 3 operands and first three as reg operands");
-
-  // Build the register operands.
-
-  bool FourReg = NumOps > 3 && OpInfo[3].RegClass == ARM::rGPRRegClassID;
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRs(insn))));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRn(insn))));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRm(insn))));
-
-  if (FourReg)
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                       decodeRd(insn))));
-
-  NumOpsAdded = FourReg ? 4 : 3;
-
-  return true;
-}
-
-// A6.3.17 Long multiply, long multiply accumulate, and divide
-//
-// t2SMULL, t2UMULL, t2SMLAL, t2UMLAL, t2UMAAL: RdLo RdHi Rn Rm
-// where RdLo = Inst{15-12} and RdHi = Inst{11-8}
-//
-// Halfword multiple accumulate long: t2SMLAL<x><y>: RdLo RdHi Rn Rm
-// where RdLo = Inst{15-12} and RdHi = Inst{11-8}
-//
-// Dual halfword multiple: t2SMLALD[X], t2SMLSLD[X]: RdLo RdHi Rn Rm
-// where RdLo = Inst{15-12} and RdHi = Inst{11-8}
-//
-// Signed/Unsigned divide: t2SDIV, t2UDIV: Rs Rn Rm
-static bool DisassembleThumb2LongMul(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO B) {
-
-  const MCOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
-
-  assert(NumOps >= 3 &&
-         OpInfo[0].RegClass == ARM::rGPRRegClassID &&
-         OpInfo[1].RegClass == ARM::rGPRRegClassID &&
-         OpInfo[2].RegClass == ARM::rGPRRegClassID &&
-         "Expect >= 3 operands and first three as reg operands");
-
-  bool FourReg = NumOps > 3 && OpInfo[3].RegClass == ARM::rGPRRegClassID;
-
-  // Build the register operands.
-
-  if (FourReg)
-    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                       decodeRd(insn))));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRs(insn))));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRn(insn))));
-
-  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(B, ARM::rGPRRegClassID,
-                                                     decodeRm(insn))));
-
-  if (FourReg)
-    NumOpsAdded = 4;
-  else
-    NumOpsAdded = 3;
-
-  return true;
-}
-
-// See A6.3 32-bit Thumb instruction encoding for instruction classes
-// corresponding to (op1, op2, op).
-//
-// Table A6-9 32-bit Thumb instruction encoding
-// op1  op2    op  Instruction class, see
-// ---  -------  --  -----------------------------------------------------------
-// 01  00xx0xx  -  Load/store multiple on page A6-23
-//     00xx1xx  -  Load/store dual, load/store exclusive, table branch on
-//                 page A6-24
-//     01xxxxx  -  Data-processing (shifted register) on page A6-31
-//     1xxxxxx  -  Coprocessor instructions on page A6-40
-// 10  x0xxxxx  0  Data-processing (modified immediate) on page A6-15
-//     x1xxxxx  0  Data-processing (plain binary immediate) on page A6-19
-//         -    1  Branches and miscellaneous control on page A6-20
-// 11  000xxx0  -  Store single data item on page A6-30
-//     001xxx0  -  Advanced SIMD element or structure load/store instructions
-//                 on page A7-27
-//     00xx001  - Load byte, memory hints on page A6-28
-//     00xx011  -  Load halfword, memory hints on page A6-26
-//     00xx101  -  Load word on page A6-25
-//     00xx111  -  UNDEFINED
-//     010xxxx  -  Data-processing (register) on page A6-33
-//     0110xxx  -  Multiply, multiply accumulate, and absolute difference on
-//                 page A6-38
-//     0111xxx  -  Long multiply, long multiply accumulate, and divide on
-//                 page A6-39
-//     1xxxxxx  -  Coprocessor instructions on page A6-40
-//
-static bool DisassembleThumb2(uint16_t op1, uint16_t op2, uint16_t op,
-    MCInst &MI, unsigned Opcode, uint32_t insn, unsigned short NumOps,
-    unsigned &NumOpsAdded, BO B) {
-
-  switch (op1) {
-  case 1:
-    if (slice(op2, 6, 5) == 0) {
-      if (slice(op2, 2, 2) == 0) {
-        // Load/store multiple.
-        return DisassembleThumb2LdStMul(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                        B);
-      }
-
-      // Load/store dual, load/store exclusive, table branch, otherwise.
-      assert(slice(op2, 2, 2) == 1 && "Thumb2 encoding error!");
-      if ((ARM::t2LDREX <= Opcode && Opcode <= ARM::t2LDREXH) ||
-          (ARM::t2STREX <= Opcode && Opcode <= ARM::t2STREXH)) {
-        // Load/store exclusive.
-        return DisassembleThumb2LdStEx(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                       B);
-      }
-      if (Opcode == ARM::t2LDRDi8 ||
-          Opcode == ARM::t2LDRD_PRE || Opcode == ARM::t2LDRD_POST ||
-          Opcode == ARM::t2STRDi8 ||
-          Opcode == ARM::t2STRD_PRE || Opcode == ARM::t2STRD_POST) {
-        // Load/store dual.
-        return DisassembleThumb2LdStDual(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                         B);
-      }
-      if (Opcode == ARM::t2TBB || Opcode == ARM::t2TBH) {
-        // Table branch.
-        return DisassembleThumb2TB(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      }
-    } else if (slice(op2, 6, 5) == 1) {
-      // Data-processing (shifted register).
-      return DisassembleThumb2DPSoReg(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-    }
-
-    // FIXME: A6.3.18 Coprocessor instructions
-    // But see ThumbDisassembler::getInstruction().
-
-    break;
-  case 2:
-    if (op == 0) {
-      if (slice(op2, 5, 5) == 0)
-        // Data-processing (modified immediate)
-        return DisassembleThumb2DPModImm(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                         B);
-      if (Thumb2SaturateOpcode(Opcode))
-        return DisassembleThumb2Sat(MI, Opcode, insn, NumOpsAdded, B);
-
-      // Data-processing (plain binary immediate)
-      return DisassembleThumb2DPBinImm(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                       B);
-    }
-    // Branches and miscellaneous control on page A6-20.
-    return DisassembleThumb2BrMiscCtrl(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                       B);
-  case 3:
-    switch (slice(op2, 6, 5)) {
-    case 0:
-      // Load/store instructions...
-      if (slice(op2, 0, 0) == 0) {
-        if (slice(op2, 4, 4) == 0) {
-          // Store single data item on page A6-30
-          return DisassembleThumb2LdSt(false, MI,Opcode,insn,NumOps,NumOpsAdded,
-                                       B);
-        } else {
-          // FIXME: Advanced SIMD element or structure load/store instructions.
-          // But see ThumbDisassembler::getInstruction().
-          ;
-        }
-      } else {
-        // Table A6-9 32-bit Thumb instruction encoding: Load byte|halfword|word
-        return DisassembleThumb2LdSt(true, MI, Opcode, insn, NumOps,
-                                     NumOpsAdded, B);
-      }
-      break;
-    case 1:
-      if (slice(op2, 4, 4) == 0) {
-        // A6.3.12 Data-processing (register)
-        return DisassembleThumb2DPReg(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      } else if (slice(op2, 3, 3) == 0) {
-        // A6.3.16 Multiply, multiply accumulate, and absolute difference
-        return DisassembleThumb2Mul(MI, Opcode, insn, NumOps, NumOpsAdded, B);
-      } else {
-        // A6.3.17 Long multiply, long multiply accumulate, and divide
-        return DisassembleThumb2LongMul(MI, Opcode, insn, NumOps, NumOpsAdded,
-                                        B);
-      }
-      break;
-    default:
-      // FIXME: A6.3.18 Coprocessor instructions
-      // But see ThumbDisassembler::getInstruction().
-      ;
-      break;
-    }
-
-    break;
-  default:
-    assert(0 && "Thumb2 encoding error!");
-    break;
-  }
-
-  return false;
-}
-
-static bool DisassembleThumbFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
-    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder) {
-
-  uint16_t HalfWord = slice(insn, 31, 16);
-
-  if (HalfWord == 0) {
-    // A6.2 16-bit Thumb instruction encoding
-    // op = bits[15:10]
-    uint16_t op = slice(insn, 15, 10);
-    return DisassembleThumb1(op, MI, Opcode, insn, NumOps, NumOpsAdded,
-                             Builder);
-  }
-
-  unsigned bits15_11 = slice(HalfWord, 15, 11);
-
-  // A6.1 Thumb instruction set encoding
-  if (!(bits15_11 == 0x1D || bits15_11 == 0x1E || bits15_11 == 0x1F)) {
-    assert("Bits[15:11] first halfword of Thumb2 instruction is out of range");
-    return false;
-  }
-
-  // A6.3 32-bit Thumb instruction encoding
-
-  uint16_t op1 = slice(HalfWord, 12, 11);
-  uint16_t op2 = slice(HalfWord, 10, 4);
-  uint16_t op = slice(insn, 15, 15);
-
-  return DisassembleThumb2(op1, op2, op, MI, Opcode, insn, NumOps, NumOpsAdded,
-                           Builder);
-}
diff --git a/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp b/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
index 78d3e477975c..ccdac3ebeb47 100644
--- a/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
+++ b/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
@@ -12,9 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "asm-printer"
-#include "ARMBaseInfo.h"
 #include "ARMInstPrinter.h"
-#include "ARMAddressingModes.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
@@ -25,6 +25,23 @@ using namespace llvm;
 #define GET_INSTRUCTION_NAME
 #include "ARMGenAsmWriter.inc"
 
+/// translateShiftImm - Convert shift immediate from 0-31 to 1-32 for printing.
+///
+/// getSORegOffset returns an integer from 0-31, representing '32' as 0.
+static unsigned translateShiftImm(unsigned imm) {
+  if (imm == 0)
+    return 32;
+  return imm;
+}
+
+
+ARMInstPrinter::ARMInstPrinter(const MCAsmInfo &MAI,
+                               const MCSubtargetInfo &STI) :
+  MCInstPrinter(MAI) {
+  // Initialize the set of available features.
+  setAvailableFeatures(STI.getFeatureBits());
+}
+
 StringRef ARMInstPrinter::getOpcodeName(unsigned Opcode) const {
   return getInstructionName(Opcode);
 }
@@ -33,11 +50,12 @@ void ARMInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   OS << getRegisterName(RegNo);
 }
 
-void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
+void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
+                               StringRef Annot) {
   unsigned Opcode = MI->getOpcode();
 
   // Check for MOVs and print canonical forms, instead.
-  if (Opcode == ARM::MOVs) {
+  if (Opcode == ARM::MOVsr) {
     // FIXME: Thumb variants?
     const MCOperand &Dst = MI->getOperand(0);
     const MCOperand &MO1 = MI->getOperand(1);
@@ -51,20 +69,36 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
     O << '\t' << getRegisterName(Dst.getReg())
       << ", " << getRegisterName(MO1.getReg());
 
-    if (ARM_AM::getSORegShOp(MO3.getImm()) == ARM_AM::rrx)
-      return;
-
-    O << ", ";
-
-    if (MO2.getReg()) {
-      O << getRegisterName(MO2.getReg());
-      assert(ARM_AM::getSORegOffset(MO3.getImm()) == 0);
-    } else {
-      O << "#" << ARM_AM::getSORegOffset(MO3.getImm());
-    }
+    O << ", " << getRegisterName(MO2.getReg());
+    assert(ARM_AM::getSORegOffset(MO3.getImm()) == 0);
+    printAnnotation(O, Annot);
     return;
   }
 
+  if (Opcode == ARM::MOVsi) {
+    // FIXME: Thumb variants?
+    const MCOperand &Dst = MI->getOperand(0);
+    const MCOperand &MO1 = MI->getOperand(1);
+    const MCOperand &MO2 = MI->getOperand(2);
+
+    O << '\t' << ARM_AM::getShiftOpcStr(ARM_AM::getSORegShOp(MO2.getImm()));
+    printSBitModifierOperand(MI, 5, O);
+    printPredicateOperand(MI, 3, O);
+
+    O << '\t' << getRegisterName(Dst.getReg())
+      << ", " << getRegisterName(MO1.getReg());
+
+    if (ARM_AM::getSORegShOp(MO2.getImm()) == ARM_AM::rrx) {
+      printAnnotation(O, Annot);
+      return;
+    }
+
+    O << ", #" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()));
+    printAnnotation(O, Annot);
+    return;
+  }
+
+
   // A8.6.123 PUSH
   if ((Opcode == ARM::STMDB_UPD || Opcode == ARM::t2STMDB_UPD) &&
       MI->getOperand(0).getReg() == ARM::SP) {
@@ -74,6 +108,15 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
       O << ".w";
     O << '\t';
     printRegisterList(MI, 4, O);
+    printAnnotation(O, Annot);
+    return;
+  }
+  if (Opcode == ARM::STR_PRE_IMM && MI->getOperand(2).getReg() == ARM::SP &&
+      MI->getOperand(3).getImm() == -4) {
+    O << '\t' << "push";
+    printPredicateOperand(MI, 4, O);
+    O << "\t{" << getRegisterName(MI->getOperand(1).getReg()) << "}";
+    printAnnotation(O, Annot);
     return;
   }
 
@@ -86,8 +129,18 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
       O << ".w";
     O << '\t';
     printRegisterList(MI, 4, O);
+    printAnnotation(O, Annot);
     return;
   }
+  if (Opcode == ARM::LDR_POST_IMM && MI->getOperand(2).getReg() == ARM::SP &&
+      MI->getOperand(4).getImm() == 4) {
+    O << '\t' << "pop";
+    printPredicateOperand(MI, 5, O);
+    O << "\t{" << getRegisterName(MI->getOperand(0).getReg()) << "}";
+    printAnnotation(O, Annot);
+    return;
+  }
+
 
   // A8.6.355 VPUSH
   if ((Opcode == ARM::VSTMSDB_UPD || Opcode == ARM::VSTMDDB_UPD) &&
@@ -96,6 +149,7 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
     printPredicateOperand(MI, 2, O);
     O << '\t';
     printRegisterList(MI, 4, O);
+    printAnnotation(O, Annot);
     return;
   }
 
@@ -106,10 +160,40 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
     printPredicateOperand(MI, 2, O);
     O << '\t';
     printRegisterList(MI, 4, O);
+    printAnnotation(O, Annot);
+    return;
+  }
+
+  if (Opcode == ARM::tLDMIA) {
+    bool Writeback = true;
+    unsigned BaseReg = MI->getOperand(0).getReg();
+    for (unsigned i = 3; i < MI->getNumOperands(); ++i) {
+      if (MI->getOperand(i).getReg() == BaseReg)
+        Writeback = false;
+    }
+
+    O << "\tldm";
+
+    printPredicateOperand(MI, 1, O);
+    O << '\t' << getRegisterName(BaseReg);
+    if (Writeback) O << "!";
+    O << ", ";
+    printRegisterList(MI, 3, O);
+    printAnnotation(O, Annot);
+    return;
+  }
+
+  // Thumb1 NOP
+  if (Opcode == ARM::tMOVr && MI->getOperand(0).getReg() == ARM::R8 &&
+      MI->getOperand(1).getReg() == ARM::R8) {
+    O << "\tnop";
+    printPredicateOperand(MI, 2, O);
+    printAnnotation(O, Annot);
     return;
   }
 
   printInstruction(MI, O);
+  printAnnotation(O, Annot);
 }
 
 void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
@@ -122,16 +206,38 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << '#' << Op.getImm();
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    O << *Op.getExpr();
+    // If a symbolic branch target was added as a constant expression then print
+    // that address in hex.
+    const MCConstantExpr *BranchTarget = dyn_cast<MCConstantExpr>(Op.getExpr());
+    int64_t Address;
+    if (BranchTarget && BranchTarget->EvaluateAsAbsolute(Address)) {
+      O << "0x";
+      O.write_hex(Address);
+    }
+    else {
+      // Otherwise, just print the expression.
+      O << *Op.getExpr();
+    }
   }
 }
 
+void ARMInstPrinter::printT2LdrLabelOperand(const MCInst *MI, unsigned OpNum,
+                                       raw_ostream &O) {
+  const MCOperand &MO1 = MI->getOperand(OpNum);
+  if (MO1.isExpr())
+    O << *MO1.getExpr();
+  else if (MO1.isImm())
+    O << "[pc, #" << MO1.getImm() << "]";
+  else
+    llvm_unreachable("Unknown LDR label operand?");
+}
+
 // so_reg is a 4-operand unit corresponding to register forms of the A5.1
 // "Addressing Mode 1 - Data-processing operands" forms.  This includes:
 //    REG 0   0           - e.g. R5
 //    REG REG 0,SH_OPC    - e.g. R5, ROR R3
 //    REG 0   IMM,SH_OPC  - e.g. R5, LSL #3
-void ARMInstPrinter::printSORegOperand(const MCInst *MI, unsigned OpNum,
+void ARMInstPrinter::printSORegRegOperand(const MCInst *MI, unsigned OpNum,
                                        raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(OpNum);
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
@@ -144,14 +250,27 @@ void ARMInstPrinter::printSORegOperand(const MCInst *MI, unsigned OpNum,
   O << ", " << ARM_AM::getShiftOpcStr(ShOpc);
   if (ShOpc == ARM_AM::rrx)
     return;
-  if (MO2.getReg()) {
-    O << ' ' << getRegisterName(MO2.getReg());
-    assert(ARM_AM::getSORegOffset(MO3.getImm()) == 0);
-  } else if (ShOpc != ARM_AM::rrx) {
-    O << " #" << ARM_AM::getSORegOffset(MO3.getImm());
-  }
+  
+  O << ' ' << getRegisterName(MO2.getReg());
+  assert(ARM_AM::getSORegOffset(MO3.getImm()) == 0);
 }
 
+void ARMInstPrinter::printSORegImmOperand(const MCInst *MI, unsigned OpNum,
+                                       raw_ostream &O) {
+  const MCOperand &MO1 = MI->getOperand(OpNum);
+  const MCOperand &MO2 = MI->getOperand(OpNum+1);
+
+  O << getRegisterName(MO1.getReg());
+
+  // Print the shift opc.
+  ARM_AM::ShiftOpc ShOpc = ARM_AM::getSORegShOp(MO2.getImm());
+  O << ", " << ARM_AM::getShiftOpcStr(ShOpc);
+  if (ShOpc == ARM_AM::rrx)
+    return;
+  O << " #" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()));
+}
+
+
 //===--------------------------------------------------------------------===//
 // Addressing Mode #2
 //===--------------------------------------------------------------------===//
@@ -209,6 +328,22 @@ void ARMInstPrinter::printAM2PostIndexOp(const MCInst *MI, unsigned Op,
     << " #" << ShImm;
 }
 
+void ARMInstPrinter::printAddrModeTBB(const MCInst *MI, unsigned Op,
+                                           raw_ostream &O) {
+  const MCOperand &MO1 = MI->getOperand(Op);
+  const MCOperand &MO2 = MI->getOperand(Op+1);
+  O << "[" << getRegisterName(MO1.getReg()) << ", "
+    << getRegisterName(MO2.getReg()) << "]";
+}
+
+void ARMInstPrinter::printAddrModeTBH(const MCInst *MI, unsigned Op,
+                                           raw_ostream &O) {
+  const MCOperand &MO1 = MI->getOperand(Op);
+  const MCOperand &MO2 = MI->getOperand(Op+1);
+  O << "[" << getRegisterName(MO1.getReg()) << ", "
+    << getRegisterName(MO2.getReg()) << ", lsl #1]";
+}
+
 void ARMInstPrinter::printAddrMode2Operand(const MCInst *MI, unsigned Op,
                                            raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(Op);
@@ -284,7 +419,7 @@ void ARMInstPrinter::printAM3PreOrOffsetIndexOp(const MCInst *MI, unsigned Op,
   O << '[' << getRegisterName(MO1.getReg());
 
   if (MO2.getReg()) {
-    O << ", " << (char)ARM_AM::getAM3Op(MO3.getImm())
+    O << ", " << getAddrOpcStr(ARM_AM::getAM3Op(MO3.getImm()))
       << getRegisterName(MO2.getReg()) << ']';
     return;
   }
@@ -315,8 +450,8 @@ void ARMInstPrinter::printAddrMode3OffsetOperand(const MCInst *MI,
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
   if (MO1.getReg()) {
-    O << (char)ARM_AM::getAM3Op(MO2.getImm())
-    << getRegisterName(MO1.getReg());
+    O << getAddrOpcStr(ARM_AM::getAM3Op(MO2.getImm()))
+      << getRegisterName(MO1.getReg());
     return;
   }
 
@@ -326,6 +461,31 @@ void ARMInstPrinter::printAddrMode3OffsetOperand(const MCInst *MI,
     << ImmOffs;
 }
 
+void ARMInstPrinter::printPostIdxImm8Operand(const MCInst *MI,
+                                             unsigned OpNum,
+                                             raw_ostream &O) {
+  const MCOperand &MO = MI->getOperand(OpNum);
+  unsigned Imm = MO.getImm();
+  O << '#' << ((Imm & 256) ? "" : "-") << (Imm & 0xff);
+}
+
+void ARMInstPrinter::printPostIdxRegOperand(const MCInst *MI, unsigned OpNum,
+                                            raw_ostream &O) {
+  const MCOperand &MO1 = MI->getOperand(OpNum);
+  const MCOperand &MO2 = MI->getOperand(OpNum+1);
+
+  O << (MO2.getImm() ? "" : "-") << getRegisterName(MO1.getReg());
+}
+
+void ARMInstPrinter::printPostIdxImm8s4Operand(const MCInst *MI,
+                                             unsigned OpNum,
+                                             raw_ostream &O) {
+  const MCOperand &MO = MI->getOperand(OpNum);
+  unsigned Imm = MO.getImm();
+  O << '#' << ((Imm & 256) ? "" : "-") << ((Imm & 0xff) << 2);
+}
+
+
 void ARMInstPrinter::printLdStmModeOperand(const MCInst *MI, unsigned OpNum,
                                            raw_ostream &O) {
   ARM_AM::AMSubMode Mode = ARM_AM::getAM4SubMode(MI->getOperand(OpNum)
@@ -345,7 +505,9 @@ void ARMInstPrinter::printAddrMode5Operand(const MCInst *MI, unsigned OpNum,
 
   O << "[" << getRegisterName(MO1.getReg());
 
-  if (unsigned ImmOffs = ARM_AM::getAM5Offset(MO2.getImm())) {
+  unsigned ImmOffs = ARM_AM::getAM5Offset(MO2.getImm());
+  unsigned Op = ARM_AM::getAM5Op(MO2.getImm());
+  if (ImmOffs || Op == ARM_AM::sub) {
     O << ", #"
       << ARM_AM::getAddrOpcStr(ARM_AM::getAM5Op(MO2.getImm()))
       << ImmOffs * 4;
@@ -402,20 +564,31 @@ void ARMInstPrinter::printMemBOption(const MCInst *MI, unsigned OpNum,
 void ARMInstPrinter::printShiftImmOperand(const MCInst *MI, unsigned OpNum,
                                           raw_ostream &O) {
   unsigned ShiftOp = MI->getOperand(OpNum).getImm();
-  ARM_AM::ShiftOpc Opc = ARM_AM::getSORegShOp(ShiftOp);
-  switch (Opc) {
-  case ARM_AM::no_shift:
+  bool isASR = (ShiftOp & (1 << 5)) != 0;
+  unsigned Amt = ShiftOp & 0x1f;
+  if (isASR)
+    O << ", asr #" << (Amt == 0 ? 32 : Amt);
+  else if (Amt)
+    O << ", lsl #" << Amt;
+}
+
+void ARMInstPrinter::printPKHLSLShiftImm(const MCInst *MI, unsigned OpNum,
+                                         raw_ostream &O) {
+  unsigned Imm = MI->getOperand(OpNum).getImm();
+  if (Imm == 0)
     return;
-  case ARM_AM::lsl:
-    O << ", lsl #";
-    break;
-  case ARM_AM::asr:
-    O << ", asr #";
-    break;
-  default:
-    assert(0 && "unexpected shift opcode for shift immediate operand");
-  }
-  O << ARM_AM::getSORegOffset(ShiftOp);
+  assert(Imm > 0 && Imm < 32 && "Invalid PKH shift immediate value!");
+  O << ", lsl #" << Imm;
+}
+
+void ARMInstPrinter::printPKHASRShiftImm(const MCInst *MI, unsigned OpNum,
+                                         raw_ostream &O) {
+  unsigned Imm = MI->getOperand(OpNum).getImm();
+  // A shift amount of 32 is encoded as 0.
+  if (Imm == 0)
+    Imm = 32;
+  assert(Imm > 0 && Imm <= 32 && "Invalid PKH shift immediate value!");
+  O << ", asr #" << Imm;
 }
 
 void ARMInstPrinter::printRegisterList(const MCInst *MI, unsigned OpNum,
@@ -450,6 +623,9 @@ void ARMInstPrinter::printCPSIFlag(const MCInst *MI, unsigned OpNum,
   for (int i=2; i >= 0; --i)
     if (IFlags & (1 << i))
       O << ARM_PROC::IFlagsToString(1 << i);
+
+  if (IFlags == 0)
+    O << "none";
 }
 
 void ARMInstPrinter::printMSRMaskOperand(const MCInst *MI, unsigned OpNum,
@@ -458,10 +634,43 @@ void ARMInstPrinter::printMSRMaskOperand(const MCInst *MI, unsigned OpNum,
   unsigned SpecRegRBit = Op.getImm() >> 4;
   unsigned Mask = Op.getImm() & 0xf;
 
+  if (getAvailableFeatures() & ARM::FeatureMClass) {
+    switch (Op.getImm()) {
+    default: assert(0 && "Unexpected mask value!");
+    case 0: O << "apsr"; return;
+    case 1: O << "iapsr"; return;
+    case 2: O << "eapsr"; return;
+    case 3: O << "xpsr"; return;
+    case 5: O << "ipsr"; return;
+    case 6: O << "epsr"; return;
+    case 7: O << "iepsr"; return;
+    case 8: O << "msp"; return;
+    case 9: O << "psp"; return;
+    case 16: O << "primask"; return;
+    case 17: O << "basepri"; return;
+    case 18: O << "basepri_max"; return;
+    case 19: O << "faultmask"; return;
+    case 20: O << "control"; return;
+    }
+  }
+
+  // As special cases, CPSR_f, CPSR_s and CPSR_fs prefer printing as
+  // APSR_nzcvq, APSR_g and APSRnzcvqg, respectively.
+  if (!SpecRegRBit && (Mask == 8 || Mask == 4 || Mask == 12)) {
+    O << "APSR_";
+    switch (Mask) {
+    default: assert(0);
+    case 4:  O << "g"; return;
+    case 8:  O << "nzcvq"; return;
+    case 12: O << "nzcvqg"; return;
+    }
+    llvm_unreachable("Unexpected mask value!");
+  }
+
   if (SpecRegRBit)
-    O << "spsr";
+    O << "SPSR";
   else
-    O << "cpsr";
+    O << "CPSR";
 
   if (Mask) {
     O << '_';
@@ -501,15 +710,20 @@ void ARMInstPrinter::printNoHashImmediate(const MCInst *MI, unsigned OpNum,
 }
 
 void ARMInstPrinter::printPImmediate(const MCInst *MI, unsigned OpNum,
-                                          raw_ostream &O) {
+                                     raw_ostream &O) {
   O << "p" << MI->getOperand(OpNum).getImm();
 }
 
 void ARMInstPrinter::printCImmediate(const MCInst *MI, unsigned OpNum,
-                                          raw_ostream &O) {
+                                     raw_ostream &O) {
   O << "c" << MI->getOperand(OpNum).getImm();
 }
 
+void ARMInstPrinter::printCoprocOptionImm(const MCInst *MI, unsigned OpNum,
+                                          raw_ostream &O) {
+  O << "{" << MI->getOperand(OpNum).getImm() << "}";
+}
+
 void ARMInstPrinter::printPCLabel(const MCInst *MI, unsigned OpNum,
                                   raw_ostream &O) {
   llvm_unreachable("Unhandled PC-relative pseudo-instruction!");
@@ -517,7 +731,13 @@ void ARMInstPrinter::printPCLabel(const MCInst *MI, unsigned OpNum,
 
 void ARMInstPrinter::printThumbS4ImmOperand(const MCInst *MI, unsigned OpNum,
                                             raw_ostream &O) {
-  O << "#" <<  MI->getOperand(OpNum).getImm() * 4;
+  O << "#" << MI->getOperand(OpNum).getImm() * 4;
+}
+
+void ARMInstPrinter::printThumbSRImm(const MCInst *MI, unsigned OpNum,
+                                     raw_ostream &O) {
+  unsigned Imm = MI->getOperand(OpNum).getImm();
+  O << "#" << (Imm == 0 ? 32 : Imm);
 }
 
 void ARMInstPrinter::printThumbITMask(const MCInst *MI, unsigned OpNum,
@@ -610,7 +830,7 @@ void ARMInstPrinter::printT2SOOperand(const MCInst *MI, unsigned OpNum,
   ARM_AM::ShiftOpc ShOpc = ARM_AM::getSORegShOp(MO2.getImm());
   O << ", " << ARM_AM::getShiftOpcStr(ShOpc);
   if (ShOpc != ARM_AM::rrx)
-    O << " #" << ARM_AM::getSORegOffset(MO2.getImm());
+    O << " #" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()));
 }
 
 void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum,
@@ -647,7 +867,9 @@ void ARMInstPrinter::printT2AddrModeImm8Operand(const MCInst *MI,
 
   int32_t OffImm = (int32_t)MO2.getImm();
   // Don't print +0.
-  if (OffImm < 0)
+  if (OffImm == INT32_MIN)
+    O << ", #-0";
+  else if (OffImm < 0)
     O << ", #-" << -OffImm;
   else if (OffImm > 0)
     O << ", #" << OffImm;
@@ -671,6 +893,18 @@ void ARMInstPrinter::printT2AddrModeImm8s4Operand(const MCInst *MI,
   O << "]";
 }
 
+void ARMInstPrinter::printT2AddrModeImm0_1020s4Operand(const MCInst *MI,
+                                                       unsigned OpNum,
+                                                       raw_ostream &O) {
+  const MCOperand &MO1 = MI->getOperand(OpNum);
+  const MCOperand &MO2 = MI->getOperand(OpNum+1);
+
+  O << "[" << getRegisterName(MO1.getReg());
+  if (MO2.getImm())
+    O << ", #" << MO2.getImm() * 4;
+  O << "]";
+}
+
 void ARMInstPrinter::printT2AddrModeImm8OffsetOperand(const MCInst *MI,
                                                       unsigned OpNum,
                                                       raw_ostream &O) {
@@ -678,9 +912,9 @@ void ARMInstPrinter::printT2AddrModeImm8OffsetOperand(const MCInst *MI,
   int32_t OffImm = (int32_t)MO1.getImm();
   // Don't print +0.
   if (OffImm < 0)
-    O << "#-" << -OffImm;
-  else if (OffImm > 0)
-    O << "#" << OffImm;
+    O << ", #-" << -OffImm;
+  else
+    O << ", #" << OffImm;
 }
 
 void ARMInstPrinter::printT2AddrModeImm8s4OffsetOperand(const MCInst *MI,
@@ -689,10 +923,13 @@ void ARMInstPrinter::printT2AddrModeImm8s4OffsetOperand(const MCInst *MI,
   const MCOperand &MO1 = MI->getOperand(OpNum);
   int32_t OffImm = (int32_t)MO1.getImm() / 4;
   // Don't print +0.
-  if (OffImm < 0)
-    O << "#-" << -OffImm * 4;
-  else if (OffImm > 0)
-    O << "#" << OffImm * 4;
+  if (OffImm != 0) {
+    O << ", ";
+    if (OffImm < 0)
+      O << "#-" << -OffImm * 4;
+    else if (OffImm > 0)
+      O << "#" << OffImm * 4;
+  }
 }
 
 void ARMInstPrinter::printT2AddrModeSoRegOperand(const MCInst *MI,
@@ -715,39 +952,10 @@ void ARMInstPrinter::printT2AddrModeSoRegOperand(const MCInst *MI,
   O << "]";
 }
 
-void ARMInstPrinter::printVFPf32ImmOperand(const MCInst *MI, unsigned OpNum,
-                                           raw_ostream &O) {
+void ARMInstPrinter::printFPImmOperand(const MCInst *MI, unsigned OpNum,
+                                       raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
-  O << '#';
-  if (MO.isFPImm()) {
-    O << (float)MO.getFPImm();
-  } else {
-    union {
-      uint32_t I;
-      float F;
-    } FPUnion;
-
-    FPUnion.I = MO.getImm();
-    O << FPUnion.F;
-  }
-}
-
-void ARMInstPrinter::printVFPf64ImmOperand(const MCInst *MI, unsigned OpNum,
-                                           raw_ostream &O) {
-  const MCOperand &MO = MI->getOperand(OpNum);
-  O << '#';
-  if (MO.isFPImm()) {
-    O << MO.getFPImm();
-  } else {
-    // We expect the binary encoding of a floating point number here.
-    union {
-      uint64_t I;
-      double D;
-    } FPUnion;
-
-    FPUnion.I = MO.getImm();
-    O << FPUnion.D;
-  }
+  O << '#' << ARM_AM::getFPImmFloat(MO.getImm());
 }
 
 void ARMInstPrinter::printNEONModImmOperand(const MCInst *MI, unsigned OpNum,
@@ -757,3 +965,28 @@ void ARMInstPrinter::printNEONModImmOperand(const MCInst *MI, unsigned OpNum,
   uint64_t Val = ARM_AM::decodeNEONModImm(EncodedImm, EltBits);
   O << "#0x" << utohexstr(Val);
 }
+
+void ARMInstPrinter::printImmPlusOneOperand(const MCInst *MI, unsigned OpNum,
+                                            raw_ostream &O) {
+  unsigned Imm = MI->getOperand(OpNum).getImm();
+  O << "#" << Imm + 1;
+}
+
+void ARMInstPrinter::printRotImmOperand(const MCInst *MI, unsigned OpNum,
+                                        raw_ostream &O) {
+  unsigned Imm = MI->getOperand(OpNum).getImm();
+  if (Imm == 0)
+    return;
+  O << ", ror #";
+  switch (Imm) {
+  default: assert (0 && "illegal ror immediate!");
+  case 1: O << "8"; break;
+  case 2: O << "16"; break;
+  case 3: O << "24"; break;
+  }
+}
+
+void ARMInstPrinter::printVectorIndex(const MCInst *MI, unsigned OpNum,
+                                      raw_ostream &O) {
+  O << "[" << MI->getOperand(OpNum).getImm() << "]";
+}
diff --git a/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.h b/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
index d5f238bb8a61..5c2173fcde62 100644
--- a/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
+++ b/contrib/llvm/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
@@ -15,6 +15,7 @@
 #define ARMINSTPRINTER_H
 
 #include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 
 namespace llvm {
 
@@ -22,10 +23,9 @@ class MCOperand;
 
 class ARMInstPrinter : public MCInstPrinter {
 public:
-  ARMInstPrinter(const MCAsmInfo &MAI)
-    : MCInstPrinter(MAI) {}
+    ARMInstPrinter(const MCAsmInfo &MAI, const MCSubtargetInfo &STI);
 
-  virtual void printInst(const MCInst *MI, raw_ostream &O);
+  virtual void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot);
   virtual StringRef getOpcodeName(unsigned Opcode) const;
   virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
 
@@ -38,8 +38,11 @@ class ARMInstPrinter : public MCInstPrinter {
 
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
 
-  void printSORegOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printSORegRegOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printSORegImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
 
+  void printAddrModeTBB(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printAddrModeTBH(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printAddrMode2Operand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printAM2PostIndexOp(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printAM2PreOrOffsetIndexOp(const MCInst *MI, unsigned OpNum,
@@ -48,11 +51,15 @@ class ARMInstPrinter : public MCInstPrinter {
                                    raw_ostream &O);
 
   void printAddrMode3Operand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
-  void printAM3PostIndexOp(const MCInst *MI, unsigned OpNum, raw_ostream &O);
-  void printAM3PreOrOffsetIndexOp(const MCInst *MI, unsigned OpNum,
-                                  raw_ostream &O);
   void printAddrMode3OffsetOperand(const MCInst *MI, unsigned OpNum,
                                    raw_ostream &O);
+  void printAM3PostIndexOp(const MCInst *MI, unsigned Op, raw_ostream &O);
+  void printAM3PreOrOffsetIndexOp(const MCInst *MI, unsigned Op,raw_ostream &O);
+  void printPostIdxImm8Operand(const MCInst *MI, unsigned OpNum,
+                               raw_ostream &O);
+  void printPostIdxRegOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printPostIdxImm8s4Operand(const MCInst *MI, unsigned OpNum,
+                               raw_ostream &O);
 
   void printLdStmModeOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printAddrMode5Operand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
@@ -65,8 +72,11 @@ class ARMInstPrinter : public MCInstPrinter {
                                       raw_ostream &O);
   void printMemBOption(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printShiftImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printPKHLSLShiftImm(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printPKHASRShiftImm(const MCInst *MI, unsigned OpNum, raw_ostream &O);
 
   void printThumbS4ImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printThumbSRImm(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printThumbITMask(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printThumbAddrModeRROperand(const MCInst *MI, unsigned OpNum,
                                    raw_ostream &O);
@@ -88,6 +98,8 @@ class ARMInstPrinter : public MCInstPrinter {
                                   raw_ostream &O);
   void printT2AddrModeImm8s4Operand(const MCInst *MI, unsigned OpNum,
                                     raw_ostream &O);
+  void printT2AddrModeImm0_1020s4Operand(const MCInst *MI, unsigned OpNum,
+                                    raw_ostream &O);
   void printT2AddrModeImm8OffsetOperand(const MCInst *MI, unsigned OpNum,
                                         raw_ostream &O);
   void printT2AddrModeImm8s4OffsetOperand(const MCInst *MI, unsigned OpNum,
@@ -108,11 +120,15 @@ class ARMInstPrinter : public MCInstPrinter {
   void printNoHashImmediate(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printPImmediate(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printCImmediate(const MCInst *MI, unsigned OpNum, raw_ostream &O);
-  void printVFPf32ImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
-  void printVFPf64ImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printCoprocOptionImm(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printFPImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printNEONModImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printImmPlusOneOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printRotImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
 
   void printPCLabel(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printT2LdrLabelOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printVectorIndex(const MCInst *MI, unsigned OpNum, raw_ostream &O);
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/ARM/InstPrinter/CMakeLists.txt b/contrib/llvm/lib/Target/ARM/InstPrinter/CMakeLists.txt
deleted file mode 100644
index 18645c0864a3..000000000000
--- a/contrib/llvm/lib/Target/ARM/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMARMAsmPrinter
-  ARMInstPrinter.cpp
-  )
-add_dependencies(LLVMARMAsmPrinter ARMCodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/ARM/InstPrinter/Makefile b/contrib/llvm/lib/Target/ARM/InstPrinter/Makefile
deleted file mode 100644
index 65d372e44b88..000000000000
--- a/contrib/llvm/lib/Target/ARM/InstPrinter/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/ARM/AsmPrinter/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMARMAsmPrinter
-
-# Hack: we need to include 'main' arm target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/ARM/ARMAddressingModes.h b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
similarity index 87%
rename from contrib/llvm/lib/Target/ARM/ARMAddressingModes.h
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
index 595708fa7881..9982fa68a578 100644
--- a/contrib/llvm/lib/Target/ARM/ARMAddressingModes.h
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
@@ -14,7 +14,8 @@
 #ifndef LLVM_TARGET_ARM_ARMADDRESSINGMODES_H
 #define LLVM_TARGET_ARM_ARMADDRESSINGMODES_H
 
-#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
 
@@ -32,7 +33,8 @@ namespace ARM_AM {
   };
 
   enum AddrOpc {
-    add = '+', sub = '-'
+    sub = 0,
+    add
   };
 
   static inline const char *getAddrOpcStr(AddrOpc Op) {
@@ -60,20 +62,6 @@ namespace ARM_AM {
     }
   }
 
-  static inline ShiftOpc getShiftOpcForNode(SDValue N) {
-    switch (N.getOpcode()) {
-    default:          return ARM_AM::no_shift;
-    case ISD::SHL:    return ARM_AM::lsl;
-    case ISD::SRL:    return ARM_AM::lsr;
-    case ISD::SRA:    return ARM_AM::asr;
-    case ISD::ROTR:   return ARM_AM::ror;
-    //case ISD::ROTL:  // Only if imm -> turn into ROTR.
-    // Can't handle RRX here, because it would require folding a flag into
-    // the addressing mode.  :(  This causes us to miss certain things.
-    //case ARMISD::RRX: return ARM_AM::rrx;
-    }
-  }
-
   enum AMSubMode {
     bad_am_submode = 0,
     ia,
@@ -588,6 +576,90 @@ namespace ARM_AM {
 
   AMSubMode getLoadStoreMultipleSubMode(int Opcode);
 
+  //===--------------------------------------------------------------------===//
+  // Floating-point Immediates
+  //
+  static inline float getFPImmFloat(unsigned Imm) {
+    // We expect an 8-bit binary encoding of a floating-point number here.
+    union {
+      uint32_t I;
+      float F;
+    } FPUnion;
+
+    uint8_t Sign = (Imm >> 7) & 0x1;
+    uint8_t Exp = (Imm >> 4) & 0x7;
+    uint8_t Mantissa = Imm & 0xf;
+
+    //   8-bit FP    iEEEE Float Encoding
+    //   abcd efgh   aBbbbbbc defgh000 00000000 00000000
+    //
+    // where B = NOT(b);
+
+    FPUnion.I = 0;
+    FPUnion.I |= Sign << 31;
+    FPUnion.I |= ((Exp & 0x4) != 0 ? 0 : 1) << 30;
+    FPUnion.I |= ((Exp & 0x4) != 0 ? 0x1f : 0) << 25;
+    FPUnion.I |= (Exp & 0x3) << 23;
+    FPUnion.I |= Mantissa << 19;
+    return FPUnion.F;
+  }
+
+  /// getFP32Imm - Return an 8-bit floating-point version of the 32-bit
+  /// floating-point value. If the value cannot be represented as an 8-bit
+  /// floating-point value, then return -1.
+  static inline int getFP32Imm(const APInt &Imm) {
+    uint32_t Sign = Imm.lshr(31).getZExtValue() & 1;
+    int32_t Exp = (Imm.lshr(23).getSExtValue() & 0xff) - 127;  // -126 to 127
+    int64_t Mantissa = Imm.getZExtValue() & 0x7fffff;  // 23 bits
+
+    // We can handle 4 bits of mantissa.
+    // mantissa = (16+UInt(e:f:g:h))/16.
+    if (Mantissa & 0x7ffff)
+      return -1;
+    Mantissa >>= 19;
+    if ((Mantissa & 0xf) != Mantissa)
+      return -1;
+
+    // We can handle 3 bits of exponent: exp == UInt(NOT(b):c:d)-3
+    if (Exp < -3 || Exp > 4)
+      return -1;
+    Exp = ((Exp+3) & 0x7) ^ 4;
+
+    return ((int)Sign << 7) | (Exp << 4) | Mantissa;
+  }
+
+  static inline int getFP32Imm(const APFloat &FPImm) {
+    return getFP32Imm(FPImm.bitcastToAPInt());
+  }
+
+  /// getFP64Imm - Return an 8-bit floating-point version of the 64-bit
+  /// floating-point value. If the value cannot be represented as an 8-bit
+  /// floating-point value, then return -1.
+  static inline int getFP64Imm(const APInt &Imm) {
+    uint64_t Sign = Imm.lshr(63).getZExtValue() & 1;
+    int64_t Exp = (Imm.lshr(52).getSExtValue() & 0x7ff) - 1023; // -1022 to 1023
+    uint64_t Mantissa = Imm.getZExtValue() & 0xfffffffffffffULL;
+
+    // We can handle 4 bits of mantissa.
+    // mantissa = (16+UInt(e:f:g:h))/16.
+    if (Mantissa & 0xffffffffffffULL)
+      return -1;
+    Mantissa >>= 48;
+    if ((Mantissa & 0xf) != Mantissa)
+      return -1;
+
+    // We can handle 3 bits of exponent: exp == UInt(NOT(b):c:d)-3
+    if (Exp < -3 || Exp > 4)
+      return -1;
+    Exp = ((Exp+3) & 0x7) ^ 4;
+
+    return ((int)Sign << 7) | (Exp << 4) | Mantissa;
+  }
+
+  static inline int getFP64Imm(const APFloat &FPImm) {
+    return getFP64Imm(FPImm.bitcastToAPInt());
+  }
+
 } // end namespace ARM_AM
 } // end namespace llvm
 
diff --git a/contrib/llvm/lib/Target/ARM/ARMAsmBackend.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
similarity index 88%
rename from contrib/llvm/lib/Target/ARM/ARMAsmBackend.cpp
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 5e438a976732..c31c5e6b8452 100644
--- a/contrib/llvm/lib/Target/ARM/ARMAsmBackend.cpp
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -7,9 +7,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARM.h"
-#include "ARMAddressingModes.h"
-#include "ARMFixupKinds.h"
+#include "MCTargetDesc/ARMMCTargetDesc.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
+#include "MCTargetDesc/ARMFixupKinds.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCDirectives.h"
@@ -19,12 +20,12 @@
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Object/MachOFormat.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetAsmBackend.h"
-#include "llvm/Target/TargetRegistry.h"
 using namespace llvm;
 
 namespace {
@@ -35,13 +36,24 @@ class ARMELFObjectWriter : public MCELFObjectTargetWriter {
                               /*HasRelocationAddend*/ false) {}
 };
 
-class ARMAsmBackend : public TargetAsmBackend {
+class ARMAsmBackend : public MCAsmBackend {
+  const MCSubtargetInfo* STI;
   bool isThumbMode;  // Currently emitting Thumb code.
 public:
-  ARMAsmBackend(const Target &T) : TargetAsmBackend(), isThumbMode(false) {}
+  ARMAsmBackend(const Target &T, const StringRef TT)
+    : MCAsmBackend(), STI(ARM_MC::createARMMCSubtargetInfo(TT, "", "")),
+      isThumbMode(TT.startswith("thumb")) {}
+
+  ~ARMAsmBackend() {
+    delete STI;
+  }
 
   unsigned getNumFixupKinds() const { return ARM::NumTargetFixupKinds; }
 
+  bool hasNOP() const {
+    return (STI->getFeatureBits() & ARM::HasV6T2Ops) != 0;
+  }
+
   const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const {
     const static MCFixupKindInfo Infos[ARM::NumTargetFixupKinds] = {
 // This table *must* be in the order that the fixup_* kinds are defined in
@@ -65,9 +77,9 @@ class ARMAsmBackend : public TargetAsmBackend {
 { "fixup_t2_uncondbranch",   0,            32,  MCFixupKindInfo::FKF_IsPCRel },
 { "fixup_arm_thumb_br",      0,            16,  MCFixupKindInfo::FKF_IsPCRel },
 { "fixup_arm_thumb_bl",      0,            32,  MCFixupKindInfo::FKF_IsPCRel },
-{ "fixup_arm_thumb_blx",     7,            21,  MCFixupKindInfo::FKF_IsPCRel },
+{ "fixup_arm_thumb_blx",     0,            32,  MCFixupKindInfo::FKF_IsPCRel },
 { "fixup_arm_thumb_cb",      0,            16,  MCFixupKindInfo::FKF_IsPCRel },
-{ "fixup_arm_thumb_cp",      1,             8,  MCFixupKindInfo::FKF_IsPCRel },
+{ "fixup_arm_thumb_cp",      0,             8,  MCFixupKindInfo::FKF_IsPCRel },
 { "fixup_arm_thumb_bcc",     0,             8,  MCFixupKindInfo::FKF_IsPCRel },
 // movw / movt: 16-bits immediate but scattered into two chunks 0 - 12, 16 - 19.
 { "fixup_arm_movt_hi16",     0,            20,  0 },
@@ -81,7 +93,7 @@ class ARMAsmBackend : public TargetAsmBackend {
     };
 
     if (Kind < FirstTargetFixupKind)
-      return TargetAsmBackend::getFixupKindInfo(Kind);
+      return MCAsmBackend::getFixupKindInfo(Kind);
 
     assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
            "Invalid kind!");
@@ -123,20 +135,28 @@ void ARMAsmBackend::RelaxInstruction(const MCInst &Inst, MCInst &Res) const {
 }
 
 bool ARMAsmBackend::WriteNopData(uint64_t Count, MCObjectWriter *OW) const {
+  const uint16_t Thumb1_16bitNopEncoding = 0x46c0; // using MOV r8,r8
+  const uint16_t Thumb2_16bitNopEncoding = 0xbf00; // NOP
+  const uint32_t ARMv4_NopEncoding = 0xe1a0000; // using MOV r0,r0
+  const uint32_t ARMv6T2_NopEncoding = 0xe3207800; // NOP
   if (isThumb()) {
-    // FIXME: 0xbf00 is the ARMv7 value. For v6 and before, we'll need to
-    // use 0x46c0 (which is a 'mov r8, r8' insn).
+    const uint16_t nopEncoding = hasNOP() ? Thumb2_16bitNopEncoding
+                                          : Thumb1_16bitNopEncoding;
     uint64_t NumNops = Count / 2;
     for (uint64_t i = 0; i != NumNops; ++i)
-      OW->Write16(0xbf00);
+      OW->Write16(nopEncoding);
     if (Count & 1)
       OW->Write8(0);
     return true;
   }
   // ARM mode
+  const uint32_t nopEncoding = hasNOP() ? ARMv6T2_NopEncoding
+                                        : ARMv4_NopEncoding;
   uint64_t NumNops = Count / 4;
   for (uint64_t i = 0; i != NumNops; ++i)
-    OW->Write32(0xe1a00000);
+    OW->Write32(nopEncoding);
+  // FIXME: should this function return false when unable to write exactly
+  // 'Count' bytes with NOP encodings?
   switch (Count % 4) {
   default: break; // No leftover bytes to write
   case 1: OW->Write8(0); break;
@@ -163,8 +183,6 @@ static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
   case ARM::fixup_arm_movw_lo16_pcrel: {
     unsigned Hi4 = (Value & 0xF000) >> 12;
     unsigned Lo12 = Value & 0x0FFF;
-    assert ((((int64_t)Value) >= -0x8000) && (((int64_t)Value) <= 0x7fff) &&
-            "Out of range pc-relative fixup value!");
     // inst{19-16} = Hi4;
     // inst{11-0} = Lo12;
     Value = (Hi4 << 16) | (Lo12);
@@ -185,10 +203,6 @@ static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
     // inst{26} = i;
     // inst{14-12} = Mid3;
     // inst{7-0} = Lo8;
-    // The value comes in as the whole thing, not just the portion required
-    // for this fixup, so we need to mask off the bits not handled by this
-    // portion (lo vs. hi).
-    Value &= 0xffff;
     Value = (Hi4 << 16) | (i << 26) | (Mid3 << 12) | (Lo8);
     uint64_t swapped = (Value & 0xFFFF0000) >> 16;
     swapped |= (Value & 0x0000FFFF) << 16;
@@ -382,8 +396,9 @@ namespace {
 class ELFARMAsmBackend : public ARMAsmBackend {
 public:
   Triple::OSType OSType;
-  ELFARMAsmBackend(const Target &T, Triple::OSType _OSType)
-    : ARMAsmBackend(T), OSType(_OSType) { }
+  ELFARMAsmBackend(const Target &T, const StringRef TT,
+                   Triple::OSType _OSType)
+    : ARMAsmBackend(T, TT), OSType(_OSType) { }
 
   void ApplyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
                   uint64_t Value) const;
@@ -414,8 +429,9 @@ void ELFARMAsmBackend::ApplyFixup(const MCFixup &Fixup, char *Data,
 class DarwinARMAsmBackend : public ARMAsmBackend {
 public:
   const object::mach::CPUSubtypeARM Subtype;
-  DarwinARMAsmBackend(const Target &T, object::mach::CPUSubtypeARM st)
-    : ARMAsmBackend(T), Subtype(st) { }
+  DarwinARMAsmBackend(const Target &T, const StringRef TT,
+                      object::mach::CPUSubtypeARM st)
+    : ARMAsmBackend(T, TT), Subtype(st) { }
 
   MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
     return createARMMachObjectWriter(OS, /*Is64Bit=*/false,
@@ -492,25 +508,24 @@ void DarwinARMAsmBackend::ApplyFixup(const MCFixup &Fixup, char *Data,
 
 } // end anonymous namespace
 
-TargetAsmBackend *llvm::createARMAsmBackend(const Target &T,
-                                            const std::string &TT) {
+MCAsmBackend *llvm::createARMAsmBackend(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin()) {
     if (TheTriple.getArchName() == "armv4t" ||
         TheTriple.getArchName() == "thumbv4t")
-      return new DarwinARMAsmBackend(T, object::mach::CSARM_V4T);
+      return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V4T);
     else if (TheTriple.getArchName() == "armv5e" ||
         TheTriple.getArchName() == "thumbv5e")
-      return new DarwinARMAsmBackend(T, object::mach::CSARM_V5TEJ);
+      return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V5TEJ);
     else if (TheTriple.getArchName() == "armv6" ||
         TheTriple.getArchName() == "thumbv6")
-      return new DarwinARMAsmBackend(T, object::mach::CSARM_V6);
-    return new DarwinARMAsmBackend(T, object::mach::CSARM_V7);
+      return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V6);
+    return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V7);
   }
 
   if (TheTriple.isOSWindows())
     assert(0 && "Windows not supported on ARM");
 
-  return new ELFARMAsmBackend(T, Triple(TT).getOS());
+  return new ELFARMAsmBackend(T, TT, Triple(TT).getOS());
 }
diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseInfo.h b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
similarity index 65%
rename from contrib/llvm/lib/Target/ARM/ARMBaseInfo.h
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
index 458f7dd1f784..ec4b6ffcfe83 100644
--- a/contrib/llvm/lib/Target/ARM/ARMBaseInfo.h
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
@@ -17,12 +17,9 @@
 #ifndef ARMBASEINFO_H
 #define ARMBASEINFO_H
 
-#include "MCTargetDesc/ARMMCTargetDesc.h"
+#include "ARMMCTargetDesc.h"
 #include "llvm/Support/ErrorHandling.h"
 
-// Note that the following auto-generated files only defined enum types, and
-// so are safe to include here.
-
 namespace llvm {
 
 // Enums corresponding to ARM condition codes
@@ -191,6 +188,22 @@ inline static unsigned getARMRegisterNumbering(unsigned Reg) {
   }
 }
 
+/// isARMLowRegister - Returns true if the register is a low register (r0-r7).
+///
+static inline bool isARMLowRegister(unsigned Reg) {
+  using namespace ARM;
+  switch (Reg) {
+  case R0:  case R1:  case R2:  case R3:
+  case R4:  case R5:  case R6:  case R7:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/// ARMII - This namespace holds all of the target specific flags that
+/// instruction info tracks.
+///
 namespace ARMII {
 
   /// ARM Index Modes
@@ -287,6 +300,148 @@ namespace ARMII {
     /// call operand.
     MO_PLT
   };
+
+  enum {
+    //===------------------------------------------------------------------===//
+    // Instruction Flags.
+
+    //===------------------------------------------------------------------===//
+    // This four-bit field describes the addressing mode used.
+    AddrModeMask  = 0x1f, // The AddrMode enums are declared in ARMBaseInfo.h
+
+    // IndexMode - Unindex, pre-indexed, or post-indexed are valid for load
+    // and store ops only.  Generic "updating" flag is used for ld/st multiple.
+    // The index mode enums are declared in ARMBaseInfo.h
+    IndexModeShift = 5,
+    IndexModeMask  = 3 << IndexModeShift,
+
+    //===------------------------------------------------------------------===//
+    // Instruction encoding formats.
+    //
+    FormShift     = 7,
+    FormMask      = 0x3f << FormShift,
+
+    // Pseudo instructions
+    Pseudo        = 0  << FormShift,
+
+    // Multiply instructions
+    MulFrm        = 1  << FormShift,
+
+    // Branch instructions
+    BrFrm         = 2  << FormShift,
+    BrMiscFrm     = 3  << FormShift,
+
+    // Data Processing instructions
+    DPFrm         = 4  << FormShift,
+    DPSoRegFrm    = 5  << FormShift,
+
+    // Load and Store
+    LdFrm         = 6  << FormShift,
+    StFrm         = 7  << FormShift,
+    LdMiscFrm     = 8  << FormShift,
+    StMiscFrm     = 9  << FormShift,
+    LdStMulFrm    = 10 << FormShift,
+
+    LdStExFrm     = 11 << FormShift,
+
+    // Miscellaneous arithmetic instructions
+    ArithMiscFrm  = 12 << FormShift,
+    SatFrm        = 13 << FormShift,
+
+    // Extend instructions
+    ExtFrm        = 14 << FormShift,
+
+    // VFP formats
+    VFPUnaryFrm   = 15 << FormShift,
+    VFPBinaryFrm  = 16 << FormShift,
+    VFPConv1Frm   = 17 << FormShift,
+    VFPConv2Frm   = 18 << FormShift,
+    VFPConv3Frm   = 19 << FormShift,
+    VFPConv4Frm   = 20 << FormShift,
+    VFPConv5Frm   = 21 << FormShift,
+    VFPLdStFrm    = 22 << FormShift,
+    VFPLdStMulFrm = 23 << FormShift,
+    VFPMiscFrm    = 24 << FormShift,
+
+    // Thumb format
+    ThumbFrm      = 25 << FormShift,
+
+    // Miscelleaneous format
+    MiscFrm       = 26 << FormShift,
+
+    // NEON formats
+    NGetLnFrm     = 27 << FormShift,
+    NSetLnFrm     = 28 << FormShift,
+    NDupFrm       = 29 << FormShift,
+    NLdStFrm      = 30 << FormShift,
+    N1RegModImmFrm= 31 << FormShift,
+    N2RegFrm      = 32 << FormShift,
+    NVCVTFrm      = 33 << FormShift,
+    NVDupLnFrm    = 34 << FormShift,
+    N2RegVShLFrm  = 35 << FormShift,
+    N2RegVShRFrm  = 36 << FormShift,
+    N3RegFrm      = 37 << FormShift,
+    N3RegVShFrm   = 38 << FormShift,
+    NVExtFrm      = 39 << FormShift,
+    NVMulSLFrm    = 40 << FormShift,
+    NVTBLFrm      = 41 << FormShift,
+
+    //===------------------------------------------------------------------===//
+    // Misc flags.
+
+    // UnaryDP - Indicates this is a unary data processing instruction, i.e.
+    // it doesn't have a Rn operand.
+    UnaryDP       = 1 << 13,
+
+    // Xform16Bit - Indicates this Thumb2 instruction may be transformed into
+    // a 16-bit Thumb instruction if certain conditions are met.
+    Xform16Bit    = 1 << 14,
+
+    // ThumbArithFlagSetting - The instruction is a 16-bit flag setting Thumb
+    // instruction. Used by the parser to determine whether to require the 'S'
+    // suffix on the mnemonic (when not in an IT block) or preclude it (when
+    // in an IT block).
+    ThumbArithFlagSetting = 1 << 18,
+
+    //===------------------------------------------------------------------===//
+    // Code domain.
+    DomainShift   = 15,
+    DomainMask    = 7 << DomainShift,
+    DomainGeneral = 0 << DomainShift,
+    DomainVFP     = 1 << DomainShift,
+    DomainNEON    = 2 << DomainShift,
+    DomainNEONA8  = 4 << DomainShift,
+
+    //===------------------------------------------------------------------===//
+    // Field shifts - such shifts are used to set field while generating
+    // machine instructions.
+    //
+    // FIXME: This list will need adjusting/fixing as the MC code emitter
+    // takes shape and the ARMCodeEmitter.cpp bits go away.
+    ShiftTypeShift = 4,
+
+    M_BitShift     = 5,
+    ShiftImmShift  = 5,
+    ShiftShift     = 7,
+    N_BitShift     = 7,
+    ImmHiShift     = 8,
+    SoRotImmShift  = 8,
+    RegRsShift     = 8,
+    ExtRotImmShift = 10,
+    RegRdLoShift   = 12,
+    RegRdShift     = 12,
+    RegRdHiShift   = 16,
+    RegRnShift     = 16,
+    S_BitShift     = 20,
+    W_BitShift     = 21,
+    AM3_I_BitShift = 22,
+    D_BitShift     = 22,
+    U_BitShift     = 23,
+    P_BitShift     = 24,
+    I_BitShift     = 25,
+    CondShift      = 28
+  };
+
 } // end namespace ARMII
 
 } // end namespace llvm;
diff --git a/contrib/llvm/lib/Target/ARM/ARMFixupKinds.h b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h
similarity index 100%
rename from contrib/llvm/lib/Target/ARM/ARMFixupKinds.h
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h
diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index 53b4c95d3801..1c109e015280 100644
--- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -52,6 +52,9 @@ ARMMCAsmInfoDarwin::ARMMCAsmInfoDarwin() {
   AsmTransCBE = arm_asm_table;
   Data64bitsDirective = 0;
   CommentString = "@";
+  Code16Directive = ".code\t16";
+  Code32Directive = ".code\t32";
+
   SupportsDebugInformation = true;
 
   // Exceptions handling
@@ -64,12 +67,14 @@ ARMELFMCAsmInfo::ARMELFMCAsmInfo() {
 
   Data64bitsDirective = 0;
   CommentString = "@";
+  PrivateGlobalPrefix = ".L";
+  Code16Directive = ".code\t16";
+  Code32Directive = ".code\t32";
+
+  WeakRefDirective = "\t.weak\t";
+  LCOMMDirectiveType = LCOMM::NoAlignment;
 
   HasLEB128 = true;
-  PrivateGlobalPrefix = ".L";
-  WeakRefDirective = "\t.weak\t";
-  HasLCOMMDirective = true;
-
   SupportsDebugInformation = true;
 
   // Exceptions handling
diff --git a/contrib/llvm/lib/Target/ARM/ARMMCCodeEmitter.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
similarity index 82%
rename from contrib/llvm/lib/Target/ARM/ARMMCCodeEmitter.cpp
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index 39be3f0e39f8..865c3e22b842 100644
--- a/contrib/llvm/lib/Target/ARM/ARMMCCodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -12,17 +12,18 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "mccodeemitter"
-#include "ARM.h"
-#include "ARMAddressingModes.h"
-#include "ARMFixupKinds.h"
-#include "ARMInstrInfo.h"
-#include "ARMMCExpr.h"
-#include "ARMSubtarget.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
+#include "MCTargetDesc/ARMFixupKinds.h"
+#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCTargetDesc.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -112,11 +113,13 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
   /// immediate Thumb2 direct branch target.
   uint32_t getUnconditionalBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
                                   SmallVectorImpl<MCFixup> &Fixups) const;
-  
+
   /// getARMBranchTargetOpValue - Return encoding info for 24-bit immediate
   /// branch target.
   uint32_t getARMBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
                                      SmallVectorImpl<MCFixup> &Fixups) const;
+  uint32_t getARMBLXTargetOpValue(const MCInst &MI, unsigned OpIdx,
+                                     SmallVectorImpl<MCFixup> &Fixups) const;
 
   /// getAdrLabelOpValue - Return encoding info for 12-bit immediate
   /// ADR label target.
@@ -142,6 +145,16 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
   uint32_t getT2AddrModeImm8s4OpValue(const MCInst &MI, unsigned OpIdx,
                                    SmallVectorImpl<MCFixup> &Fixups) const;
 
+  /// getT2AddrModeImm0_1020s4OpValue - Return encoding info for 'reg + imm8<<2'
+  /// operand.
+  uint32_t getT2AddrModeImm0_1020s4OpValue(const MCInst &MI, unsigned OpIdx,
+                                   SmallVectorImpl<MCFixup> &Fixups) const;
+
+  /// getT2Imm8s4OpValue - Return encoding info for '+/- imm8<<2'
+  /// operand.
+  uint32_t getT2Imm8s4OpValue(const MCInst &MI, unsigned OpIdx,
+                              SmallVectorImpl<MCFixup> &Fixups) const;
+
 
   /// getLdStSORegOpValue - Return encoding info for 'reg +/- reg shop imm'
   /// operand as needed by load/store instructions.
@@ -183,6 +196,10 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
   uint32_t getAddrMode2OffsetOpValue(const MCInst &MI, unsigned OpIdx,
                                      SmallVectorImpl<MCFixup> &Fixups) const;
 
+  /// getPostIdxRegOpValue - Return encoding for postidx_reg operands.
+  uint32_t getPostIdxRegOpValue(const MCInst &MI, unsigned OpIdx,
+                                SmallVectorImpl<MCFixup> &Fixups) const;
+
   /// getAddrMode3OffsetOpValue - Return encoding for am3offset operands.
   uint32_t getAddrMode3OffsetOpValue(const MCInst &MI, unsigned OpIdx,
                                      SmallVectorImpl<MCFixup> &Fixups) const;
@@ -251,27 +268,13 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
     SmallVectorImpl<MCFixup> &Fixups) const;
 
   /// getSORegOpValue - Return an encoded so_reg shifted register value.
-  unsigned getSORegOpValue(const MCInst &MI, unsigned Op,
+  unsigned getSORegRegOpValue(const MCInst &MI, unsigned Op,
+                           SmallVectorImpl<MCFixup> &Fixups) const;
+  unsigned getSORegImmOpValue(const MCInst &MI, unsigned Op,
                            SmallVectorImpl<MCFixup> &Fixups) const;
   unsigned getT2SORegOpValue(const MCInst &MI, unsigned Op,
                              SmallVectorImpl<MCFixup> &Fixups) const;
 
-  unsigned getRotImmOpValue(const MCInst &MI, unsigned Op,
-                            SmallVectorImpl<MCFixup> &Fixups) const {
-    switch (MI.getOperand(Op).getImm()) {
-    default: assert (0 && "Not a valid rot_imm value!");
-    case 0:  return 0;
-    case 8:  return 1;
-    case 16: return 2;
-    case 24: return 3;
-    }
-  }
-
-  unsigned getImmMinusOneOpValue(const MCInst &MI, unsigned Op,
-                                 SmallVectorImpl<MCFixup> &Fixups) const {
-    return MI.getOperand(Op).getImm() - 1;
-  }
-
   unsigned getNEONVcvtImm32OpValue(const MCInst &MI, unsigned Op,
                                    SmallVectorImpl<MCFixup> &Fixups) const {
     return 64 - MI.getOperand(Op).getImm();
@@ -280,12 +283,6 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
   unsigned getBitfieldInvertedMaskOpValue(const MCInst &MI, unsigned Op,
                                       SmallVectorImpl<MCFixup> &Fixups) const;
 
-  unsigned getMsbOpValue(const MCInst &MI, unsigned Op,
-                         SmallVectorImpl<MCFixup> &Fixups) const;
-
-  unsigned getSsatBitPosValue(const MCInst &MI, unsigned Op,
-                              SmallVectorImpl<MCFixup> &Fixups) const;
-
   unsigned getRegisterListOpValue(const MCInst &MI, unsigned Op,
                                   SmallVectorImpl<MCFixup> &Fixups) const;
   unsigned getAddrMode6AddressOpValue(const MCInst &MI, unsigned Op,
@@ -306,6 +303,9 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
   unsigned getShiftRight64Imm(const MCInst &MI, unsigned Op,
                               SmallVectorImpl<MCFixup> &Fixups) const;
 
+  unsigned getThumbSRImmOpValue(const MCInst &MI, unsigned Op,
+                                 SmallVectorImpl<MCFixup> &Fixups) const;
+
   unsigned NEONThumb2DataIPostEncoder(const MCInst &MI,
                                       unsigned EncodedValue) const;
   unsigned NEONThumb2LoadStorePostEncoder(const MCInst &MI,
@@ -439,8 +439,10 @@ EncodeAddrModeOpValues(const MCInst &MI, unsigned OpIdx, unsigned &Reg,
   bool isAdd = true;
 
   // Special value for #-0
-  if (SImm == INT32_MIN)
+  if (SImm == INT32_MIN) {
     SImm = 0;
+    isAdd = false;
+  }
 
   // Immediate is always encoded as positive. The 'U' bit controls add vs sub.
   if (SImm < 0) {
@@ -470,11 +472,34 @@ static uint32_t getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
   return 0;
 }
 
+// Thumb BL and BLX use a strange offset encoding where bits 22 and 21 are
+// determined by negating them and XOR'ing them with bit 23.
+static int32_t encodeThumbBLOffset(int32_t offset) {
+  offset >>= 1;
+  uint32_t S  = (offset & 0x800000) >> 23;
+  uint32_t J1 = (offset & 0x400000) >> 22;
+  uint32_t J2 = (offset & 0x200000) >> 21;
+  J1 = (~J1 & 0x1);
+  J2 = (~J2 & 0x1);
+  J1 ^= S;
+  J2 ^= S;
+
+  offset &= ~0x600000;
+  offset |= J1 << 22;
+  offset |= J2 << 21;
+
+  return offset;
+}
+
 /// getThumbBLTargetOpValue - Return encoding info for immediate branch target.
 uint32_t ARMMCCodeEmitter::
 getThumbBLTargetOpValue(const MCInst &MI, unsigned OpIdx,
                         SmallVectorImpl<MCFixup> &Fixups) const {
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_bl, Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_bl,
+                                    Fixups);
+  return encodeThumbBLOffset(MO.getImm());
 }
 
 /// getThumbBLXTargetOpValue - Return encoding info for Thumb immediate
@@ -482,28 +507,43 @@ getThumbBLTargetOpValue(const MCInst &MI, unsigned OpIdx,
 uint32_t ARMMCCodeEmitter::
 getThumbBLXTargetOpValue(const MCInst &MI, unsigned OpIdx,
                          SmallVectorImpl<MCFixup> &Fixups) const {
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_blx, Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_blx,
+                                    Fixups);
+  return encodeThumbBLOffset(MO.getImm());
 }
 
 /// getThumbBRTargetOpValue - Return encoding info for Thumb branch target.
 uint32_t ARMMCCodeEmitter::
 getThumbBRTargetOpValue(const MCInst &MI, unsigned OpIdx,
                         SmallVectorImpl<MCFixup> &Fixups) const {
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_br, Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_br,
+                                    Fixups);
+  return (MO.getImm() >> 1);
 }
 
 /// getThumbBCCTargetOpValue - Return encoding info for Thumb branch target.
 uint32_t ARMMCCodeEmitter::
 getThumbBCCTargetOpValue(const MCInst &MI, unsigned OpIdx,
                          SmallVectorImpl<MCFixup> &Fixups) const {
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_bcc, Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_bcc,
+                                    Fixups);
+  return (MO.getImm() >> 1);
 }
 
 /// getThumbCBTargetOpValue - Return encoding info for Thumb branch target.
 uint32_t ARMMCCodeEmitter::
 getThumbCBTargetOpValue(const MCInst &MI, unsigned OpIdx,
                         SmallVectorImpl<MCFixup> &Fixups) const {
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_cb, Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_cb, Fixups);
+  return (MO.getImm() >> 1);
 }
 
 /// Return true if this branch has a non-always predication
@@ -513,9 +553,9 @@ static bool HasConditionalBranch(const MCInst &MI) {
     for (int i = 0; i < NumOp-1; ++i) {
       const MCOperand &MCOp1 = MI.getOperand(i);
       const MCOperand &MCOp2 = MI.getOperand(i + 1);
-      if (MCOp1.isImm() && MCOp2.isReg() && 
+      if (MCOp1.isImm() && MCOp2.isReg() &&
           (MCOp2.getReg() == 0 || MCOp2.getReg() == ARM::CPSR)) {
-        if (ARMCC::CondCodes(MCOp1.getImm()) != ARMCC::AL) 
+        if (ARMCC::CondCodes(MCOp1.getImm()) != ARMCC::AL)
           return true;
       }
     }
@@ -541,15 +581,32 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
 uint32_t ARMMCCodeEmitter::
 getARMBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
                           SmallVectorImpl<MCFixup> &Fixups) const {
-  if (HasConditionalBranch(MI)) 
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr()) {
+    if (HasConditionalBranch(MI))
+      return ::getBranchTargetOpValue(MI, OpIdx,
+                                      ARM::fixup_arm_condbranch, Fixups);
     return ::getBranchTargetOpValue(MI, OpIdx,
-                                    ARM::fixup_arm_condbranch, Fixups);
-  return ::getBranchTargetOpValue(MI, OpIdx, 
-                                  ARM::fixup_arm_uncondbranch, Fixups);
+                                    ARM::fixup_arm_uncondbranch, Fixups);
+  }
+
+  return MO.getImm() >> 2;
 }
 
+uint32_t ARMMCCodeEmitter::
+getARMBLXTargetOpValue(const MCInst &MI, unsigned OpIdx,
+                          SmallVectorImpl<MCFixup> &Fixups) const {
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr()) {
+    if (HasConditionalBranch(MI))
+      return ::getBranchTargetOpValue(MI, OpIdx,
+                                      ARM::fixup_arm_condbranch, Fixups);
+    return ::getBranchTargetOpValue(MI, OpIdx,
+                                    ARM::fixup_arm_uncondbranch, Fixups);
+  }
 
-
+  return MO.getImm() >> 1;
+}
 
 /// getUnconditionalBranchTargetOpValue - Return encoding info for 24-bit
 /// immediate branch target.
@@ -579,9 +636,18 @@ getUnconditionalBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
 uint32_t ARMMCCodeEmitter::
 getAdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
                    SmallVectorImpl<MCFixup> &Fixups) const {
-  assert(MI.getOperand(OpIdx).isExpr() && "Unexpected adr target type!");
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_adr_pcrel_12,
-                                  Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_adr_pcrel_12,
+                                    Fixups);
+  int32_t offset = MO.getImm();
+  uint32_t Val = 0x2000;
+  if (offset < 0) {
+    Val = 0x1000;
+    offset *= -1;
+  }
+  Val |= offset;
+  return Val;
 }
 
 /// getAdrLabelOpValue - Return encoding info for 12-bit immediate ADR label
@@ -589,9 +655,16 @@ getAdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
 uint32_t ARMMCCodeEmitter::
 getT2AdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
                    SmallVectorImpl<MCFixup> &Fixups) const {
-  assert(MI.getOperand(OpIdx).isExpr() && "Unexpected adr target type!");
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_t2_adr_pcrel_12,
-                                  Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_t2_adr_pcrel_12,
+                                    Fixups);
+  int32_t Val = MO.getImm();
+  if (Val < 0) {
+    Val *= -1;
+    Val |= 0x1000;
+  }
+  return Val;
 }
 
 /// getAdrLabelOpValue - Return encoding info for 8-bit immediate ADR label
@@ -599,9 +672,11 @@ getT2AdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
 uint32_t ARMMCCodeEmitter::
 getThumbAdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
                    SmallVectorImpl<MCFixup> &Fixups) const {
-  assert(MI.getOperand(OpIdx).isExpr() && "Unexpected adr target type!");
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_thumb_adr_pcrel_10,
-                                  Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_thumb_adr_pcrel_10,
+                                    Fixups);
+  return MO.getImm();
 }
 
 /// getThumbAddrModeRegRegOpValue - Return encoding info for 'reg + reg'
@@ -635,17 +710,26 @@ getAddrModeImm12OpValue(const MCInst &MI, unsigned OpIdx,
     Imm12 = 0;
     isAdd = false ; // 'U' bit is set as part of the fixup.
 
-    assert(MO.isExpr() && "Unexpected machine operand type!");
-    const MCExpr *Expr = MO.getExpr();
+    if (MO.isExpr()) {
+      const MCExpr *Expr = MO.getExpr();
 
-    MCFixupKind Kind;
-    if (isThumb2())
-      Kind = MCFixupKind(ARM::fixup_t2_ldst_pcrel_12);
-    else
-      Kind = MCFixupKind(ARM::fixup_arm_ldst_pcrel_12);
-    Fixups.push_back(MCFixup::Create(0, Expr, Kind));
+      MCFixupKind Kind;
+      if (isThumb2())
+        Kind = MCFixupKind(ARM::fixup_t2_ldst_pcrel_12);
+      else
+        Kind = MCFixupKind(ARM::fixup_arm_ldst_pcrel_12);
+      Fixups.push_back(MCFixup::Create(0, Expr, Kind));
 
-    ++MCNumCPRelocations;
+      ++MCNumCPRelocations;
+    } else {
+      Reg = ARM::PC;
+      int32_t Offset = MO.getImm();
+      if (Offset < 0) {
+        Offset *= -1;
+        isAdd = false;
+      }
+      Imm12 = Offset;
+    }
   } else
     isAdd = EncodeAddrModeOpValues(MI, OpIdx, Reg, Imm12, Fixups);
 
@@ -657,6 +741,37 @@ getAddrModeImm12OpValue(const MCInst &MI, unsigned OpIdx,
   return Binary;
 }
 
+/// getT2Imm8s4OpValue - Return encoding info for
+/// '+/- imm8<<2' operand.
+uint32_t ARMMCCodeEmitter::
+getT2Imm8s4OpValue(const MCInst &MI, unsigned OpIdx,
+                   SmallVectorImpl<MCFixup> &Fixups) const {
+  // FIXME: The immediate operand should have already been encoded like this
+  // before ever getting here. The encoder method should just need to combine
+  // the MI operands for the register and the offset into a single
+  // representation for the complex operand in the .td file. This isn't just
+  // style, unfortunately. As-is, we can't represent the distinct encoding
+  // for #-0.
+
+  // {8}    = (U)nsigned (add == '1', sub == '0')
+  // {7-0}  = imm8
+  int32_t Imm8 = MI.getOperand(OpIdx).getImm();
+  bool isAdd = Imm8 >= 0;
+
+  // Immediate is always encoded as positive. The 'U' bit controls add vs sub.
+  if (Imm8 < 0)
+    Imm8 = -Imm8;
+
+  // Scaled by 4.
+  Imm8 /= 4;
+
+  uint32_t Binary = Imm8 & 0xff;
+  // Immediate is always encoded as positive. The 'U' bit controls add vs sub.
+  if (isAdd)
+    Binary |= (1 << 8);
+  return Binary;
+}
+
 /// getT2AddrModeImm8s4OpValue - Return encoding info for
 /// 'reg +/- imm8<<2' operand.
 uint32_t ARMMCCodeEmitter::
@@ -683,6 +798,12 @@ getT2AddrModeImm8s4OpValue(const MCInst &MI, unsigned OpIdx,
   } else
     isAdd = EncodeAddrModeOpValues(MI, OpIdx, Reg, Imm8, Fixups);
 
+  // FIXME: The immediate operand should have already been encoded like this
+  // before ever getting here. The encoder method should just need to combine
+  // the MI operands for the register and the offset into a single
+  // representation for the complex operand in the .td file. This isn't just
+  // style, unfortunately. As-is, we can't represent the distinct encoding
+  // for #-0.
   uint32_t Binary = (Imm8 >> 2) & 0xff;
   // Immediate is always encoded as positive. The 'U' bit controls add vs sub.
   if (isAdd)
@@ -691,6 +812,20 @@ getT2AddrModeImm8s4OpValue(const MCInst &MI, unsigned OpIdx,
   return Binary;
 }
 
+/// getT2AddrModeImm0_1020s4OpValue - Return encoding info for
+/// 'reg + imm8<<2' operand.
+uint32_t ARMMCCodeEmitter::
+getT2AddrModeImm0_1020s4OpValue(const MCInst &MI, unsigned OpIdx,
+                        SmallVectorImpl<MCFixup> &Fixups) const {
+  // {11-8} = reg
+  // {7-0}  = imm8
+  const MCOperand &MO = MI.getOperand(OpIdx);
+  const MCOperand &MO1 = MI.getOperand(OpIdx + 1);
+  unsigned Reg = getARMRegisterNumbering(MO.getReg());
+  unsigned Imm8 = MO1.getImm();
+  return (Reg << 8) | Imm8;
+}
+
 // FIXME: This routine assumes that a binary
 // expression will always result in a PCRel expression
 // In reality, its only true if one or more subexpressions
@@ -817,6 +952,17 @@ getAddrMode2OffsetOpValue(const MCInst &MI, unsigned OpIdx,
   return Binary | (isAdd << 12) | (isReg << 13);
 }
 
+uint32_t ARMMCCodeEmitter::
+getPostIdxRegOpValue(const MCInst &MI, unsigned OpIdx,
+                     SmallVectorImpl<MCFixup> &Fixups) const {
+  // {4}      isAdd
+  // {3-0}    Rm
+  const MCOperand &MO = MI.getOperand(OpIdx);
+  const MCOperand &MO1 = MI.getOperand(OpIdx+1);
+  bool isAdd = MO1.getImm() != 0;
+  return getARMRegisterNumbering(MO.getReg()) | (isAdd << 4);
+}
+
 uint32_t ARMMCCodeEmitter::
 getAddrMode3OffsetOpValue(const MCInst &MI, unsigned OpIdx,
                           SmallVectorImpl<MCFixup> &Fixups) const {
@@ -891,7 +1037,10 @@ getAddrModeISOpValue(const MCInst &MI, unsigned OpIdx,
 uint32_t ARMMCCodeEmitter::
 getAddrModePCOpValue(const MCInst &MI, unsigned OpIdx,
                      SmallVectorImpl<MCFixup> &Fixups) const {
-  return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_cp, Fixups);
+  const MCOperand MO = MI.getOperand(OpIdx);
+  if (MO.isExpr())
+    return ::getBranchTargetOpValue(MI, OpIdx, ARM::fixup_arm_thumb_cp, Fixups);
+  return (MO.getImm() >> 2);
 }
 
 /// getAddrMode5OpValue - Return encoding info for 'reg +/- imm10' operand.
@@ -934,20 +1083,17 @@ getAddrMode5OpValue(const MCInst &MI, unsigned OpIdx,
 }
 
 unsigned ARMMCCodeEmitter::
-getSORegOpValue(const MCInst &MI, unsigned OpIdx,
+getSORegRegOpValue(const MCInst &MI, unsigned OpIdx,
                 SmallVectorImpl<MCFixup> &Fixups) const {
   // Sub-operands are [reg, reg, imm]. The first register is Rm, the reg to be
-  // shifted. The second is either Rs, the amount to shift by, or reg0 in which
-  // case the imm contains the amount to shift by.
+  // shifted. The second is Rs, the amount to shift by, and the third specifies
+  // the type of the shift.
   //
   // {3-0} = Rm.
-  // {4}   = 1 if reg shift, 0 if imm shift
+  // {4}   = 1
   // {6-5} = type
-  //    If reg shift:
-  //      {11-8} = Rs
-  //      {7}    = 0
-  //    else (imm shift)
-  //      {11-7} = imm
+  // {11-8} = Rs
+  // {7}    = 0
 
   const MCOperand &MO  = MI.getOperand(OpIdx);
   const MCOperand &MO1 = MI.getOperand(OpIdx + 1);
@@ -966,45 +1112,70 @@ getSORegOpValue(const MCInst &MI, unsigned OpIdx,
     // LSR - 0011
     // ASR - 0101
     // ROR - 0111
-    // RRX - 0110 and bit[11:8] clear.
     switch (SOpc) {
     default: llvm_unreachable("Unknown shift opc!");
     case ARM_AM::lsl: SBits = 0x1; break;
     case ARM_AM::lsr: SBits = 0x3; break;
     case ARM_AM::asr: SBits = 0x5; break;
     case ARM_AM::ror: SBits = 0x7; break;
-    case ARM_AM::rrx: SBits = 0x6; break;
-    }
-  } else {
-    // Set shift operand (bit[6:4]).
-    // LSL - 000
-    // LSR - 010
-    // ASR - 100
-    // ROR - 110
-    switch (SOpc) {
-    default: llvm_unreachable("Unknown shift opc!");
-    case ARM_AM::lsl: SBits = 0x0; break;
-    case ARM_AM::lsr: SBits = 0x2; break;
-    case ARM_AM::asr: SBits = 0x4; break;
-    case ARM_AM::ror: SBits = 0x6; break;
     }
   }
 
   Binary |= SBits << 4;
-  if (SOpc == ARM_AM::rrx)
-    return Binary;
 
-  // Encode the shift operation Rs or shift_imm (except rrx).
-  if (Rs) {
-    // Encode Rs bit[11:8].
-    assert(ARM_AM::getSORegOffset(MO2.getImm()) == 0);
-    return Binary | (getARMRegisterNumbering(Rs) << ARMII::RegRsShift);
+  // Encode the shift operation Rs.
+  // Encode Rs bit[11:8].
+  assert(ARM_AM::getSORegOffset(MO2.getImm()) == 0);
+  return Binary | (getARMRegisterNumbering(Rs) << ARMII::RegRsShift);
+}
+
+unsigned ARMMCCodeEmitter::
+getSORegImmOpValue(const MCInst &MI, unsigned OpIdx,
+                SmallVectorImpl<MCFixup> &Fixups) const {
+  // Sub-operands are [reg, imm]. The first register is Rm, the reg to be
+  // shifted. The second is the amount to shift by.
+  //
+  // {3-0} = Rm.
+  // {4}   = 0
+  // {6-5} = type
+  // {11-7} = imm
+
+  const MCOperand &MO  = MI.getOperand(OpIdx);
+  const MCOperand &MO1 = MI.getOperand(OpIdx + 1);
+  ARM_AM::ShiftOpc SOpc = ARM_AM::getSORegShOp(MO1.getImm());
+
+  // Encode Rm.
+  unsigned Binary = getARMRegisterNumbering(MO.getReg());
+
+  // Encode the shift opcode.
+  unsigned SBits = 0;
+
+  // Set shift operand (bit[6:4]).
+  // LSL - 000
+  // LSR - 010
+  // ASR - 100
+  // ROR - 110
+  // RRX - 110 and bit[11:8] clear.
+  switch (SOpc) {
+  default: llvm_unreachable("Unknown shift opc!");
+  case ARM_AM::lsl: SBits = 0x0; break;
+  case ARM_AM::lsr: SBits = 0x2; break;
+  case ARM_AM::asr: SBits = 0x4; break;
+  case ARM_AM::ror: SBits = 0x6; break;
+  case ARM_AM::rrx:
+    Binary |= 0x60;
+    return Binary;
   }
 
   // Encode shift_imm bit[11:7].
-  return Binary | ARM_AM::getSORegOffset(MO2.getImm()) << 7;
+  Binary |= SBits << 4;
+  unsigned Offset = ARM_AM::getSORegOffset(MO1.getImm());
+  assert(Offset && "Offset must be in range 1-32!");
+  if (Offset == 32) Offset = 0;
+  return Binary | (Offset << 7);
 }
 
+
 unsigned ARMMCCodeEmitter::
 getT2AddrModeSORegOpValue(const MCInst &MI, unsigned OpNum,
                 SmallVectorImpl<MCFixup> &Fixups) const {
@@ -1106,6 +1277,7 @@ getT2SORegOpValue(const MCInst &MI, unsigned OpIdx,
   case ARM_AM::lsl: SBits = 0x0; break;
   case ARM_AM::lsr: SBits = 0x2; break;
   case ARM_AM::asr: SBits = 0x4; break;
+  case ARM_AM::rrx: // FALLTHROUGH
   case ARM_AM::ror: SBits = 0x6; break;
   }
 
@@ -1130,24 +1302,6 @@ getBitfieldInvertedMaskOpValue(const MCInst &MI, unsigned Op,
   return lsb | (msb << 5);
 }
 
-unsigned ARMMCCodeEmitter::
-getMsbOpValue(const MCInst &MI, unsigned Op,
-              SmallVectorImpl<MCFixup> &Fixups) const {
-  // MSB - 5 bits.
-  uint32_t lsb = MI.getOperand(Op-1).getImm();
-  uint32_t width = MI.getOperand(Op).getImm();
-  uint32_t msb = lsb+width-1;
-  assert (width != 0 && msb < 32 && "Illegal bit width!");
-  return msb;
-}
-
-unsigned ARMMCCodeEmitter::
-getSsatBitPosValue(const MCInst &MI, unsigned Op,
-                   SmallVectorImpl<MCFixup> &Fixups) const {
-  // For ssat instructions, the bit position should be encoded decremented by 1
-  return MI.getOperand(Op).getImm()-1;
-}
-
 unsigned ARMMCCodeEmitter::
 getRegisterListOpValue(const MCInst &MI, unsigned Op,
                        SmallVectorImpl<MCFixup> &Fixups) const {
@@ -1158,8 +1312,8 @@ getRegisterListOpValue(const MCInst &MI, unsigned Op,
   // LDM/STM:
   //   {15-0}  = Bitfield of GPRs.
   unsigned Reg = MI.getOperand(Op).getReg();
-  bool SPRRegs = ARM::SPRRegClass.contains(Reg);
-  bool DPRRegs = ARM::DPRRegClass.contains(Reg);
+  bool SPRRegs = llvm::ARMMCRegisterClasses[ARM::SPRRegClassID].contains(Reg);
+  bool DPRRegs = llvm::ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Reg);
 
   unsigned Binary = 0;
 
@@ -1299,7 +1453,7 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
     Size = Desc.getSize();
   else
     llvm_unreachable("Unexpected instruction size!");
-  
+
   uint32_t Binary = getBinaryCodeForInstr(MI, Fixups);
   // Thumb 32-bit wide instructions need to emit the high order halfword
   // first.
diff --git a/contrib/llvm/lib/Target/ARM/ARMMCExpr.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
similarity index 100%
rename from contrib/llvm/lib/Target/ARM/ARMMCExpr.cpp
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
diff --git a/contrib/llvm/lib/Target/ARM/ARMMCExpr.h b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
similarity index 100%
rename from contrib/llvm/lib/Target/ARM/ARMMCExpr.h
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
index f8fcf2b8aff1..a55c41075d40 100644
--- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
@@ -13,10 +13,16 @@
 
 #include "ARMMCTargetDesc.h"
 #include "ARMMCAsmInfo.h"
+#include "ARMBaseInfo.h"
+#include "InstPrinter/ARMInstPrinter.h"
+#include "llvm/MC/MCCodeGenInfo.h"
+#include "llvm/MC/MCInstrAnalysis.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_REGINFO_MC_DESC
 #include "ARMGenRegisterInfo.inc"
@@ -35,7 +41,7 @@ std::string ARM_MC::ParseARMTriple(StringRef TT) {
   unsigned Len = TT.size();
   unsigned Idx = 0;
 
-  // FIXME: Enahnce Triple helper class to extract ARM version.
+  // FIXME: Enhance Triple helper class to extract ARM version.
   bool isThumb = false;
   if (Len >= 5 && TT.substr(0, 4) == "armv")
     Idx = 4;
@@ -50,18 +56,21 @@ std::string ARM_MC::ParseARMTriple(StringRef TT) {
     unsigned SubVer = TT[Idx];
     if (SubVer >= '7' && SubVer <= '9') {
       if (Len >= Idx+2 && TT[Idx+1] == 'm') {
-        // v7m: FeatureNoARM, FeatureDB, FeatureHWDiv
-        ARMArchFeature = "+v7,+noarm,+db,+hwdiv";
+        // v7m: FeatureNoARM, FeatureDB, FeatureHWDiv, FeatureMClass
+        ARMArchFeature = "+v7,+noarm,+db,+hwdiv,+mclass";
       } else if (Len >= Idx+3 && TT[Idx+1] == 'e'&& TT[Idx+2] == 'm') {
         // v7em: FeatureNoARM, FeatureDB, FeatureHWDiv, FeatureDSPThumb2,
-        //       FeatureT2XtPk
-        ARMArchFeature = "+v7,+noarm,+db,+hwdiv,+t2dsp,t2xtpk";
+        //       FeatureT2XtPk, FeatureMClass
+        ARMArchFeature = "+v7,+noarm,+db,+hwdiv,+t2dsp,t2xtpk,+mclass";
       } else
-        // v7a: FeatureNEON, FeatureDB, FeatureDSPThumb2
-        ARMArchFeature = "+v7,+neon,+db,+t2dsp";
+        // v7a: FeatureNEON, FeatureDB, FeatureDSPThumb2, FeatureT2XtPk
+        ARMArchFeature = "+v7,+neon,+db,+t2dsp,+t2xtpk";
     } else if (SubVer == '6') {
       if (Len >= Idx+3 && TT[Idx+1] == 't' && TT[Idx+2] == '2')
         ARMArchFeature = "+v6t2";
+      else if (Len >= Idx+2 && TT[Idx+1] == 'm')
+        // v6m: FeatureNoARM, FeatureMClass
+        ARMArchFeature = "+v6t2,+noarm,+mclass";
       else
         ARMArchFeature = "+v6";
     } else if (SubVer == '5') {
@@ -80,6 +89,14 @@ std::string ARM_MC::ParseARMTriple(StringRef TT) {
       ARMArchFeature += ",+thumb-mode";
   }
 
+  Triple TheTriple(TT);
+  if (TheTriple.getOS() == Triple::NativeClient) {
+    if (ARMArchFeature.empty())
+      ARMArchFeature = "+nacl-mode";
+    else
+      ARMArchFeature += ",+nacl-mode";
+  }
+
   return ARMArchFeature;
 }
 
@@ -98,36 +115,18 @@ MCSubtargetInfo *ARM_MC::createARMMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-// Force static initialization.
-extern "C" void LLVMInitializeARMMCSubtargetInfo() {
-  TargetRegistry::RegisterMCSubtargetInfo(TheARMTarget,
-                                          ARM_MC::createARMMCSubtargetInfo);
-  TargetRegistry::RegisterMCSubtargetInfo(TheThumbTarget,
-                                          ARM_MC::createARMMCSubtargetInfo);
-}
-
 static MCInstrInfo *createARMMCInstrInfo() {
   MCInstrInfo *X = new MCInstrInfo();
   InitARMMCInstrInfo(X);
   return X;
 }
 
-extern "C" void LLVMInitializeARMMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheARMTarget, createARMMCInstrInfo);
-  TargetRegistry::RegisterMCInstrInfo(TheThumbTarget, createARMMCInstrInfo);
-}
-
-static MCRegisterInfo *createARMMCRegisterInfo() {
+static MCRegisterInfo *createARMMCRegisterInfo(StringRef Triple) {
   MCRegisterInfo *X = new MCRegisterInfo();
-  InitARMMCRegisterInfo(X);
+  InitARMMCRegisterInfo(X, ARM::LR);
   return X;
 }
 
-extern "C" void LLVMInitializeARMMCRegInfo() {
-  TargetRegistry::RegisterMCRegInfo(TheARMTarget, createARMMCRegisterInfo);
-  TargetRegistry::RegisterMCRegInfo(TheThumbTarget, createARMMCRegisterInfo);
-}
-
 static MCAsmInfo *createARMMCAsmInfo(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
 
@@ -137,8 +136,128 @@ static MCAsmInfo *createARMMCAsmInfo(const Target &T, StringRef TT) {
   return new ARMELFMCAsmInfo();
 }
 
-extern "C" void LLVMInitializeARMMCAsmInfo() {
-  // Register the target asm info.
+static MCCodeGenInfo *createARMMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                             CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  if (RM == Reloc::Default) {
+    Triple TheTriple(TT);
+    // Default relocation model on Darwin is PIC, not DynamicNoPIC.
+    RM = TheTriple.isOSDarwin() ? Reloc::PIC_ : Reloc::DynamicNoPIC;
+  }
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+// This is duplicated code. Refactor this.
+static MCStreamer *createMCStreamer(const Target &T, StringRef TT,
+                                    MCContext &Ctx, MCAsmBackend &MAB,
+                                    raw_ostream &OS,
+                                    MCCodeEmitter *Emitter,
+                                    bool RelaxAll,
+                                    bool NoExecStack) {
+  Triple TheTriple(TT);
+
+  if (TheTriple.isOSDarwin())
+    return createMachOStreamer(Ctx, MAB, OS, Emitter, RelaxAll);
+
+  if (TheTriple.isOSWindows()) {
+    llvm_unreachable("ARM does not support Windows COFF format");
+    return NULL;
+  }
+
+  return createELFStreamer(Ctx, MAB, OS, Emitter, RelaxAll, NoExecStack);
+}
+
+static MCInstPrinter *createARMMCInstPrinter(const Target &T,
+                                             unsigned SyntaxVariant,
+                                             const MCAsmInfo &MAI,
+                                             const MCSubtargetInfo &STI) {
+  if (SyntaxVariant == 0)
+    return new ARMInstPrinter(MAI, STI);
+  return 0;
+}
+
+namespace {
+
+class ARMMCInstrAnalysis : public MCInstrAnalysis {
+public:
+  ARMMCInstrAnalysis(const MCInstrInfo *Info) : MCInstrAnalysis(Info) {}
+
+  virtual bool isUnconditionalBranch(const MCInst &Inst) const {
+    // BCCs with the "always" predicate are unconditional branches.
+    if (Inst.getOpcode() == ARM::Bcc && Inst.getOperand(1).getImm()==ARMCC::AL)
+      return true;
+    return MCInstrAnalysis::isUnconditionalBranch(Inst);
+  }
+
+  virtual bool isConditionalBranch(const MCInst &Inst) const {
+    // BCCs with the "always" predicate are unconditional branches.
+    if (Inst.getOpcode() == ARM::Bcc && Inst.getOperand(1).getImm()==ARMCC::AL)
+      return false;
+    return MCInstrAnalysis::isConditionalBranch(Inst);
+  }
+
+  uint64_t evaluateBranch(const MCInst &Inst, uint64_t Addr,
+                          uint64_t Size) const {
+    // We only handle PCRel branches for now.
+    if (Info->get(Inst.getOpcode()).OpInfo[0].OperandType!=MCOI::OPERAND_PCREL)
+      return -1ULL;
+
+    int64_t Imm = Inst.getOperand(0).getImm();
+    // FIXME: This is not right for thumb.
+    return Addr+Imm+8; // In ARM mode the PC is always off by 8 bytes.
+  }
+};
+
+}
+
+static MCInstrAnalysis *createARMMCInstrAnalysis(const MCInstrInfo *Info) {
+  return new ARMMCInstrAnalysis(Info);
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeARMTargetMC() {
+  // Register the MC asm info.
   RegisterMCAsmInfoFn A(TheARMTarget, createARMMCAsmInfo);
   RegisterMCAsmInfoFn B(TheThumbTarget, createARMMCAsmInfo);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheARMTarget, createARMMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(TheThumbTarget, createARMMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheARMTarget, createARMMCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(TheThumbTarget, createARMMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheARMTarget, createARMMCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(TheThumbTarget, createARMMCRegisterInfo);
+
+  // Register the MC subtarget info.
+  TargetRegistry::RegisterMCSubtargetInfo(TheARMTarget,
+                                          ARM_MC::createARMMCSubtargetInfo);
+  TargetRegistry::RegisterMCSubtargetInfo(TheThumbTarget,
+                                          ARM_MC::createARMMCSubtargetInfo);
+
+  // Register the MC instruction analyzer.
+  TargetRegistry::RegisterMCInstrAnalysis(TheARMTarget,
+                                          createARMMCInstrAnalysis);
+  TargetRegistry::RegisterMCInstrAnalysis(TheThumbTarget,
+                                          createARMMCInstrAnalysis);
+
+  // Register the MC Code Emitter
+  TargetRegistry::RegisterMCCodeEmitter(TheARMTarget, createARMMCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(TheThumbTarget, createARMMCCodeEmitter);
+
+  // Register the asm backend.
+  TargetRegistry::RegisterMCAsmBackend(TheARMTarget, createARMAsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(TheThumbTarget, createARMAsmBackend);
+
+  // Register the object streamer.
+  TargetRegistry::RegisterMCObjectStreamer(TheARMTarget, createMCStreamer);
+  TargetRegistry::RegisterMCObjectStreamer(TheThumbTarget, createMCStreamer);
+
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(TheARMTarget, createARMMCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(TheThumbTarget, createARMMCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index 74701e3516dc..9b3d3bd32183 100644
--- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -14,12 +14,19 @@
 #ifndef ARMMCTARGETDESC_H
 #define ARMMCTARGETDESC_H
 
+#include "llvm/Support/DataTypes.h"
 #include <string>
 
 namespace llvm {
+class MCAsmBackend;
+class MCCodeEmitter;
+class MCContext;
+class MCInstrInfo;
+class MCObjectWriter;
 class MCSubtargetInfo;
-class Target;
 class StringRef;
+class Target;
+class raw_ostream;
 
 extern Target TheARMTarget, TheThumbTarget;
 
@@ -33,6 +40,18 @@ namespace ARM_MC {
                                             StringRef FS);
 }
 
+MCCodeEmitter *createARMMCCodeEmitter(const MCInstrInfo &MCII,
+                                      const MCSubtargetInfo &STI,
+                                      MCContext &Ctx);
+
+MCAsmBackend *createARMAsmBackend(const Target &T, StringRef TT);
+
+/// createARMMachObjectWriter - Construct an ARM Mach-O object writer.
+MCObjectWriter *createARMMachObjectWriter(raw_ostream &OS,
+                                          bool Is64Bit,
+                                          uint32_t CPUType,
+                                          uint32_t CPUSubtype);
+
 } // End llvm namespace
 
 // Defines symbolic names for ARM registers.  This defines a mapping from
diff --git a/contrib/llvm/lib/Target/ARM/ARMMachObjectWriter.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
similarity index 99%
rename from contrib/llvm/lib/Target/ARM/ARMMachObjectWriter.cpp
rename to contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index a36e47da06d4..352c73e84df1 100644
--- a/contrib/llvm/lib/Target/ARM/ARMMachObjectWriter.cpp
+++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARM.h"
-#include "ARMFixupKinds.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
+#include "MCTargetDesc/ARMFixupKinds.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCAsmLayout.h"
@@ -19,7 +19,6 @@
 #include "llvm/MC/MCValue.h"
 #include "llvm/Object/MachOFormat.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetAsmBackend.h"
 using namespace llvm;
 using namespace llvm::object;
 
diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 68daf42c9191..000000000000
--- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-add_llvm_library(LLVMARMDesc
-  ARMMCTargetDesc.cpp
-  ARMMCAsmInfo.cpp
-  )
-
-# Hack: we need to include 'main' target directory to grab private headers
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/..)
diff --git a/contrib/llvm/lib/Target/ARM/NEONMoveFix.cpp b/contrib/llvm/lib/Target/ARM/NEONMoveFix.cpp
deleted file mode 100644
index c85d1e99705a..000000000000
--- a/contrib/llvm/lib/Target/ARM/NEONMoveFix.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-//===-- NEONMoveFix.cpp - Convert vfp reg-reg moves into neon ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "neon-mov-fix"
-#include "ARM.h"
-#include "ARMMachineFunctionInfo.h"
-#include "ARMInstrInfo.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace llvm;
-
-STATISTIC(NumVMovs, "Number of reg-reg moves converted");
-
-namespace {
-  struct NEONMoveFixPass : public MachineFunctionPass {
-    static char ID;
-    NEONMoveFixPass() : MachineFunctionPass(ID) {}
-
-    virtual bool runOnMachineFunction(MachineFunction &Fn);
-
-    virtual const char *getPassName() const {
-      return "NEON reg-reg move conversion";
-    }
-
-  private:
-    const TargetRegisterInfo *TRI;
-    const ARMBaseInstrInfo *TII;
-    bool isA8;
-
-    typedef DenseMap<unsigned, const MachineInstr*> RegMap;
-
-    bool InsertMoves(MachineBasicBlock &MBB);
-  };
-  char NEONMoveFixPass::ID = 0;
-}
-
-static bool inNEONDomain(unsigned Domain, bool isA8) {
-  return (Domain & ARMII::DomainNEON) ||
-    (isA8 && (Domain & ARMII::DomainNEONA8));
-}
-
-bool NEONMoveFixPass::InsertMoves(MachineBasicBlock &MBB) {
-  RegMap Defs;
-  bool Modified = false;
-
-  // Walk over MBB tracking the def points of the registers.
-  MachineBasicBlock::iterator MII = MBB.begin(), E = MBB.end();
-  MachineBasicBlock::iterator NextMII;
-  for (; MII != E; MII = NextMII) {
-    NextMII = llvm::next(MII);
-    MachineInstr *MI = &*MII;
-
-    if (MI->getOpcode() == ARM::VMOVD &&
-        !TII->isPredicated(MI)) {
-      unsigned SrcReg = MI->getOperand(1).getReg();
-      // If we do not find an instruction defining the reg, this means the
-      // register should be live-in for this BB. It's always to better to use
-      // NEON reg-reg moves.
-      unsigned Domain = ARMII::DomainNEON;
-      RegMap::iterator DefMI = Defs.find(SrcReg);
-      if (DefMI != Defs.end()) {
-        Domain = DefMI->second->getDesc().TSFlags & ARMII::DomainMask;
-        // Instructions in general domain are subreg accesses.
-        // Map them to NEON reg-reg moves.
-        if (Domain == ARMII::DomainGeneral)
-          Domain = ARMII::DomainNEON;
-      }
-
-      if (inNEONDomain(Domain, isA8)) {
-        // Convert VMOVD to VORRd
-        unsigned DestReg = MI->getOperand(0).getReg();
-
-        DEBUG({errs() << "vmov convert: "; MI->dump();});
-
-        // It's safe to ignore imp-defs / imp-uses here, since:
-        //  - We're running late, no intelligent condegen passes should be run
-        //    afterwards
-        //  - The imp-defs / imp-uses are superregs only, we don't care about
-        //    them.
-        AddDefaultPred(BuildMI(MBB, *MI, MI->getDebugLoc(),
-                             TII->get(ARM::VORRd), DestReg)
-          .addReg(SrcReg).addReg(SrcReg));
-        MBB.erase(MI);
-        MachineBasicBlock::iterator I = prior(NextMII);
-        MI = &*I;
-
-        DEBUG({errs() << "        into: "; MI->dump();});
-
-        Modified = true;
-        ++NumVMovs;
-      } else {
-        assert((Domain & ARMII::DomainVFP) && "Invalid domain!");
-        // Do nothing.
-      }
-    }
-
-    // Update def information.
-    for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
-      const MachineOperand& MO = MI->getOperand(i);
-      if (!MO.isReg() || !MO.isDef())
-        continue;
-      unsigned MOReg = MO.getReg();
-
-      Defs[MOReg] = MI;
-      // Catch aliases as well.
-      for (const unsigned *R = TRI->getAliasSet(MOReg); *R; ++R)
-        Defs[*R] = MI;
-    }
-  }
-
-  return Modified;
-}
-
-bool NEONMoveFixPass::runOnMachineFunction(MachineFunction &Fn) {
-  ARMFunctionInfo *AFI = Fn.getInfo<ARMFunctionInfo>();
-  const TargetMachine &TM = Fn.getTarget();
-
-  if (AFI->isThumb1OnlyFunction())
-    return false;
-
-  TRI = TM.getRegisterInfo();
-  TII = static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo());
-  isA8 = TM.getSubtarget<ARMSubtarget>().isCortexA8();
-
-  bool Modified = false;
-  for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
-       ++MFI) {
-    MachineBasicBlock &MBB = *MFI;
-    Modified |= InsertMoves(MBB);
-  }
-
-  return Modified;
-}
-
-/// createNEONMoveFixPass - Returns an instance of the NEON reg-reg moves fix
-/// pass.
-FunctionPass *llvm::createNEONMoveFixPass() {
-  return new NEONMoveFixPass();
-}
diff --git a/contrib/llvm/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp b/contrib/llvm/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
index 163a0a987584..500e3de82db3 100644
--- a/contrib/llvm/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "ARM.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheARMTarget, llvm::TheThumbTarget;
diff --git a/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
index c258870e48a5..d8481778c0da 100644
--- a/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -21,7 +21,7 @@
 
 using namespace llvm;
 
-bool Thumb1FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
+bool Thumb1FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const{
   const MachineFrameInfo *FFI = MF.getFrameInfo();
   unsigned CFSize = FFI->getMaxCallFrameSize();
   // It's not always a good idea to include the call frame as part of the
@@ -133,9 +133,9 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF) const {
 
   // Adjust FP so it point to the stack slot that contains the previous FP.
   if (hasFP(MF)) {
-    BuildMI(MBB, MBBI, dl, TII.get(ARM::tADDrSPi), FramePtr)
+    AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tADDrSPi), FramePtr)
       .addFrameIndex(FramePtrSpillFI).addImm(0)
-      .setMIFlags(MachineInstr::FrameSetup);
+      .setMIFlags(MachineInstr::FrameSetup));
     if (NumBytes > 508)
       // If offset is > 508 then sp cannot be adjusted in a single instruction,
       // try restoring from fp instead.
@@ -155,6 +155,11 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF) const {
   AFI->setGPRCalleeSavedArea2Size(GPRCS2Size);
   AFI->setDPRCalleeSavedAreaSize(DPRCSSize);
 
+  // Thumb1 does not currently support dynamic stack realignment.  Report a
+  // fatal error rather then silently generate bad code.
+  if (RegInfo->needsStackRealignment(MF))
+      report_fatal_error("Dynamic stack realignment not supported for thumb1.");
+
   // If we need a base pointer, set it up here. It's whatever the value
   // of the stack pointer is at this point. Any variable size objects
   // will be allocated after this, so we can still use the base pointer
diff --git a/contrib/llvm/lib/Target/ARM/Thumb1RegisterInfo.cpp b/contrib/llvm/lib/Target/ARM/Thumb1RegisterInfo.cpp
index 4eb0b6c93e1d..e8ed482a66fa 100644
--- a/contrib/llvm/lib/Target/ARM/Thumb1RegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/Thumb1RegisterInfo.cpp
@@ -13,12 +13,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMSubtarget.h"
 #include "Thumb1InstrInfo.h"
 #include "Thumb1RegisterInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Function.h"
@@ -27,7 +27,6 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
@@ -182,7 +181,6 @@ void llvm::emitThumbRegPlusImmediate(MachineBasicBlock &MBB,
   int Opc = 0;
   int ExtraOpc = 0;
   bool NeedCC = false;
-  bool NeedPred = false;
 
   if (DestReg == BaseReg && BaseReg == ARM::SP) {
     assert(isMul4 && "Thumb sp inc / dec size must be multiple of 4!");
@@ -217,7 +215,7 @@ void llvm::emitThumbRegPlusImmediate(MachineBasicBlock &MBB,
     } else {
       Opc = isSub ? ARM::tSUBi8 : ARM::tADDi8;
       NumBits = 8;
-      NeedPred = NeedCC = true;
+      NeedCC = true;
     }
     isTwoAddr = true;
   }
@@ -241,7 +239,8 @@ void llvm::emitThumbRegPlusImmediate(MachineBasicBlock &MBB,
       Bytes -= ThisVal;
       const MCInstrDesc &MCID = TII.get(isSub ? ARM::tSUBi3 : ARM::tADDi3);
       const MachineInstrBuilder MIB =
-        AddDefaultT1CC(BuildMI(MBB, MBBI, dl, MCID, DestReg).setMIFlags(MIFlags));
+        AddDefaultT1CC(BuildMI(MBB, MBBI, dl, MCID, DestReg)
+                         .setMIFlags(MIFlags));
       AddDefaultPred(MIB.addReg(BaseReg, RegState::Kill).addImm(ThisVal));
     } else {
       AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), DestReg)
@@ -262,18 +261,15 @@ void llvm::emitThumbRegPlusImmediate(MachineBasicBlock &MBB,
       if (NeedCC)
         MIB = AddDefaultT1CC(MIB);
       MIB.addReg(DestReg).addImm(ThisVal);
-      if (NeedPred)
-        MIB = AddDefaultPred(MIB);
+      MIB = AddDefaultPred(MIB);
       MIB.setMIFlags(MIFlags);
-    }
-    else {
+    } else {
       bool isKill = BaseReg != ARM::SP;
       MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg);
       if (NeedCC)
         MIB = AddDefaultT1CC(MIB);
       MIB.addReg(BaseReg, getKillRegState(isKill)).addImm(ThisVal);
-      if (NeedPred)
-        MIB = AddDefaultPred(MIB);
+      MIB = AddDefaultPred(MIB);
       MIB.setMIFlags(MIFlags);
 
       BaseReg = DestReg;
@@ -285,7 +281,7 @@ void llvm::emitThumbRegPlusImmediate(MachineBasicBlock &MBB,
         Scale = 1;
         Chunk = ((1 << NumBits) - 1) * Scale;
         Opc = isSub ? ARM::tSUBi8 : ARM::tADDi8;
-        NeedPred = NeedCC = isTwoAddr = true;
+        NeedCC = isTwoAddr = true;
       }
     }
   }
@@ -405,7 +401,6 @@ rewriteFrameIndex(MachineBasicBlock::iterator II, unsigned FrameRegIdx,
     unsigned Scale = 1;
     if (FrameReg != ARM::SP) {
       Opcode = ARM::tADDi3;
-      MI.setDesc(TII.get(Opcode));
       NumBits = 3;
     } else {
       NumBits = 8;
@@ -419,10 +414,9 @@ rewriteFrameIndex(MachineBasicBlock::iterator II, unsigned FrameRegIdx,
       // Turn it into a move.
       MI.setDesc(TII.get(ARM::tMOVr));
       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
-      // Remove offset and add predicate operands.
+      // Remove offset
       MI.RemoveOperand(FrameRegIdx+1);
       MachineInstrBuilder MIB(&MI);
-      AddDefaultPred(MIB);
       return true;
     }
 
@@ -431,6 +425,7 @@ rewriteFrameIndex(MachineBasicBlock::iterator II, unsigned FrameRegIdx,
     if (((Offset / Scale) & ~Mask) == 0) {
       // Replace the FrameIndex with sp / fp
       if (Opcode == ARM::tADDi3) {
+        MI.setDesc(TII.get(Opcode));
         removeOperands(MI, FrameRegIdx);
         MachineInstrBuilder MIB(&MI);
         AddDefaultPred(AddDefaultT1CC(MIB).addReg(FrameReg)
@@ -459,6 +454,7 @@ rewriteFrameIndex(MachineBasicBlock::iterator II, unsigned FrameRegIdx,
       // r0 = add sp, 255*4
       // r0 = add r0, (imm - 255*4)
       if (Opcode == ARM::tADDi3) {
+        MI.setDesc(TII.get(Opcode));
         removeOperands(MI, FrameRegIdx);
         MachineInstrBuilder MIB(&MI);
         AddDefaultPred(AddDefaultT1CC(MIB).addReg(FrameReg).addImm(Mask));
@@ -479,10 +475,6 @@ rewriteFrameIndex(MachineBasicBlock::iterator II, unsigned FrameRegIdx,
       MI.setDesc(TII.get(ARM::tADDhirr));
       MI.getOperand(FrameRegIdx).ChangeToRegister(DestReg, false, false, true);
       MI.getOperand(FrameRegIdx+1).ChangeToRegister(FrameReg, false);
-      if (Opcode == ARM::tADDi3) {
-        MachineInstrBuilder MIB(&MI);
-        AddDefaultPred(MIB);
-      }
     }
     return true;
   } else {
@@ -545,9 +537,9 @@ Thumb1RegisterInfo::resolveFrameIndex(MachineBasicBlock::iterator I,
     ++i;
     assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
   }
-  bool Done = false;
-  Done = rewriteFrameIndex(MI, i, BaseReg, Off, TII);
+  bool Done = rewriteFrameIndex(MI, i, BaseReg, Off, TII);
   assert (Done && "Unable to resolve frame index!");
+  (void)Done;
 }
 
 /// saveScavengerRegister - Spill the register so it can be used by the
diff --git a/contrib/llvm/lib/Target/ARM/Thumb2ITBlockPass.cpp b/contrib/llvm/lib/Target/ARM/Thumb2ITBlockPass.cpp
index 360ec009e201..b6274007b237 100644
--- a/contrib/llvm/lib/Target/ARM/Thumb2ITBlockPass.cpp
+++ b/contrib/llvm/lib/Target/ARM/Thumb2ITBlockPass.cpp
@@ -124,6 +124,27 @@ Thumb2ITBlockPass::MoveCopyOutOfITBlock(MachineInstr *MI,
   if (Uses.count(DstReg) || Defs.count(SrcReg))
     return false;
 
+  // If the CPSR is defined by this copy, then we don't want to move it. E.g.,
+  // if we have:
+  //
+  //   movs  r1, r1
+  //   rsb   r1, 0
+  //   movs  r2, r2
+  //   rsb   r2, 0
+  //
+  // we don't want this to be converted to:
+  //
+  //   movs  r1, r1
+  //   movs  r2, r2
+  //   itt   mi
+  //   rsb   r1, 0
+  //   rsb   r2, 0
+  //
+  const MCInstrDesc &MCID = MI->getDesc();
+  if (MCID.hasOptionalDef() &&
+      MI->getOperand(MCID.getNumOperands() - 1).getReg() == ARM::CPSR)
+    return false;
+
   // Then peek at the next instruction to see if it's predicated on CC or OCC.
   // If not, then there is nothing to be gained by moving the copy.
   MachineBasicBlock::iterator I = MI; ++I;
diff --git a/contrib/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp b/contrib/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
index 51b56aaeb008..cf040c822de9 100644
--- a/contrib/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/contrib/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -14,9 +14,9 @@
 #include "Thumb2InstrInfo.h"
 #include "ARM.h"
 #include "ARMConstantPoolValue.h"
-#include "ARMAddressingModes.h"
 #include "ARMMachineFunctionInfo.h"
 #include "Thumb2InstrInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
@@ -122,7 +122,8 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     const TargetRegisterClass *RC,
                     const TargetRegisterInfo *TRI) const {
   if (RC == ARM::GPRRegisterClass   || RC == ARM::tGPRRegisterClass ||
-      RC == ARM::tcGPRRegisterClass || RC == ARM::rGPRRegisterClass) {
+      RC == ARM::tcGPRRegisterClass || RC == ARM::rGPRRegisterClass ||
+      RC == ARM::GPRnopcRegisterClass) {
     DebugLoc DL;
     if (I != MBB.end()) DL = I->getDebugLoc();
 
@@ -149,7 +150,8 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      const TargetRegisterClass *RC,
                      const TargetRegisterInfo *TRI) const {
   if (RC == ARM::GPRRegisterClass   || RC == ARM::tGPRRegisterClass ||
-      RC == ARM::tcGPRRegisterClass || RC == ARM::rGPRRegisterClass) {
+      RC == ARM::tcGPRRegisterClass || RC == ARM::rGPRRegisterClass ||
+      RC == ARM::GPRnopcRegisterClass) {
     DebugLoc DL;
     if (I != MBB.end()) DL = I->getDebugLoc();
 
@@ -233,9 +235,8 @@ void llvm::emitT2RegPlusImmediate(MachineBasicBlock &MBB,
       if (DestReg == ARM::SP && (ThisVal < ((1 << 7)-1) * 4)) {
         assert((ThisVal & 3) == 0 && "Stack update is not multiple of 4?");
         Opc = isSub ? ARM::tSUBspi : ARM::tADDspi;
-        // FIXME: Fix Thumb1 immediate encoding.
-        BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
-          .addReg(BaseReg).addImm(ThisVal/4).setMIFlags(MIFlags);
+        AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
+          .addReg(BaseReg).addImm(ThisVal/4).setMIFlags(MIFlags));
         NumBytes = 0;
         continue;
       }
diff --git a/contrib/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp b/contrib/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
index c741a6e8a5b7..89a155c5a7f5 100644
--- a/contrib/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
+++ b/contrib/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
@@ -9,11 +9,11 @@
 
 #define DEBUG_TYPE "t2-reduce-size"
 #include "ARM.h"
-#include "ARMAddressingModes.h"
 #include "ARMBaseRegisterInfo.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMSubtarget.h"
 #include "Thumb2InstrInfo.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -97,11 +97,11 @@ namespace {
     { ARM::t2SUBrr, ARM::tSUBrr,  0,             0,   0,    1,   0,  0,0, 0,0 },
     { ARM::t2SUBSri,ARM::tSUBi3,  ARM::tSUBi8,   3,   8,    1,   1,  2,2, 0,0 },
     { ARM::t2SUBSrr,ARM::tSUBrr,  0,             0,   0,    1,   0,  2,0, 0,0 },
-    { ARM::t2SXTBr, ARM::tSXTB,   0,             0,   0,    1,   0,  1,0, 0,0 },
-    { ARM::t2SXTHr, ARM::tSXTH,   0,             0,   0,    1,   0,  1,0, 0,0 },
+    { ARM::t2SXTB,  ARM::tSXTB,   0,             0,   0,    1,   0,  1,0, 0,1 },
+    { ARM::t2SXTH,  ARM::tSXTH,   0,             0,   0,    1,   0,  1,0, 0,1 },
     { ARM::t2TSTrr, ARM::tTST,    0,             0,   0,    1,   0,  2,0, 0,0 },
-    { ARM::t2UXTBr, ARM::tUXTB,   0,             0,   0,    1,   0,  1,0, 0,0 },
-    { ARM::t2UXTHr, ARM::tUXTH,   0,             0,   0,    1,   0,  1,0, 0,0 },
+    { ARM::t2UXTB,  ARM::tUXTB,   0,             0,   0,    1,   0,  1,0, 0,1 },
+    { ARM::t2UXTH,  ARM::tUXTH,   0,             0,   0,    1,   0,  1,0, 0,1 },
 
     // FIXME: Clean this up after splitting each Thumb load / store opcode
     // into multiple ones.
@@ -507,6 +507,7 @@ Thumb2SizeReduce::ReduceSpecial(MachineBasicBlock &MBB, MachineInstr *MI,
       .addOperand(MI->getOperand(0))
       .addOperand(MI->getOperand(1))
       .addImm(Imm / 4); // The tADDrSPi has an implied scale by four.
+    AddDefaultPred(MIB);
 
     // Transfer MI flags.
     MIB.setMIFlags(MI->getFlags());
@@ -546,6 +547,10 @@ Thumb2SizeReduce::ReduceSpecial(MachineBasicBlock &MBB, MachineInstr *MI,
   }
   case ARM::t2RSBri:
   case ARM::t2RSBSri:
+  case ARM::t2SXTB:
+  case ARM::t2SXTH:
+  case ARM::t2UXTB:
+  case ARM::t2UXTH:
     if (MI->getOperand(2).getImm() == 0)
       return ReduceToNarrow(MBB, MI, Entry, LiveCPSR, CPSRDef);
     break;
@@ -742,7 +747,11 @@ Thumb2SizeReduce::ReduceToNarrow(MachineBasicBlock &MBB, MachineInstr *MI,
     if (i < NumOps && MCID.OpInfo[i].isOptionalDef())
       continue;
     if ((MCID.getOpcode() == ARM::t2RSBSri ||
-         MCID.getOpcode() == ARM::t2RSBri) && i == 2)
+         MCID.getOpcode() == ARM::t2RSBri ||
+         MCID.getOpcode() == ARM::t2SXTB ||
+         MCID.getOpcode() == ARM::t2SXTH ||
+         MCID.getOpcode() == ARM::t2UXTB ||
+         MCID.getOpcode() == ARM::t2UXTH) && i == 2)
       // Skip the zero immediate operand, it's now implicit.
       continue;
     bool isPred = (i < NumOps && MCID.OpInfo[i].isPredicate());
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaAsmPrinter.cpp b/contrib/llvm/lib/Target/Alpha/AlphaAsmPrinter.cpp
index 46ae286895a9..5dce06ac86a5 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaAsmPrinter.cpp
@@ -26,8 +26,8 @@
 #include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaISelDAGToDAG.cpp b/contrib/llvm/lib/Target/Alpha/AlphaISelDAGToDAG.cpp
index 7b91fea54af4..f877c65cd61f 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaISelDAGToDAG.cpp
@@ -80,7 +80,7 @@ namespace {
             // Otherwise we don't know that the it's okay to zapnot this entire
             // byte.  Only do this iff we can prove that the missing bits are
             // already null, so the bytezap doesn't need to really null them.
-            BitsToCheck |= ~Constant & (0xFF << 8*i);
+            BitsToCheck |= ~Constant & (0xFFULL << 8*i);
           }
         }
       }
@@ -114,9 +114,8 @@ namespace {
       if (!x) return 0;
       unsigned at = CountLeadingZeros_64(x);
       uint64_t complow = 1ULL << (63 - at);
-      uint64_t comphigh = 1ULL << (64 - at);
-      //cerr << x << ":" << complow << ":" << comphigh << "\n";
-      if (abs64(complow - x) <= abs64(comphigh - x))
+      uint64_t comphigh = complow << 1;
+      if (x - complow <= comphigh - x)
         return complow;
       else
         return comphigh;
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.cpp b/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.cpp
index de003fb4c65e..3057eb8c57fb 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.cpp
@@ -49,6 +49,7 @@ AlphaTargetLowering::AlphaTargetLowering(TargetMachine &TM)
   // Set up the TargetLowering object.
   //I am having problems with shr n i8 1
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
 
   addRegisterClass(MVT::i64, Alpha::GPRCRegisterClass);
   addRegisterClass(MVT::f64, Alpha::F8RCRegisterClass);
@@ -153,6 +154,9 @@ AlphaTargetLowering::AlphaTargetLowering(TargetMachine &TM)
   setOperationAction(ISD::JumpTable, MVT::i64, Custom);
   setOperationAction(ISD::JumpTable, MVT::i32, Custom);
 
+  setOperationAction(ISD::ATOMIC_LOAD,  MVT::i32, Expand);
+  setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Expand);
+
   setStackPointerRegisterToSaveRestore(Alpha::R30);
 
   setJumpBufSize(272);
@@ -160,10 +164,12 @@ AlphaTargetLowering::AlphaTargetLowering(TargetMachine &TM)
 
   setMinFunctionAlignment(4);
 
+  setInsertFencesForAtomic(true);
+
   computeRegisterProperties();
 }
 
-MVT::SimpleValueType AlphaTargetLowering::getSetCCResultType(EVT VT) const {
+EVT AlphaTargetLowering::getSetCCResultType(EVT VT) const {
   return MVT::i64;
 }
 
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.h b/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.h
index 13383f4430f9..80f8efaea5d2 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.h
+++ b/contrib/llvm/lib/Target/Alpha/AlphaISelLowering.h
@@ -66,7 +66,7 @@ namespace llvm {
     virtual MVT getShiftAmountTy(EVT LHSTy) const { return MVT::i64; }
 
     /// getSetCCResultType - Get the SETCC result ValueType
-    virtual MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    virtual EVT getSetCCResultType(EVT VT) const;
 
     /// LowerOperation - Provide custom lowering hooks for some operations.
     ///
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.cpp b/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.cpp
index 4dcec8f31750..8df2ed75f625 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.cpp
@@ -16,7 +16,6 @@
 #include "AlphaMachineFunctionInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.td b/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.td
index b20171224e29..c8c9377c3d8d 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.td
+++ b/contrib/llvm/lib/Target/Alpha/AlphaInstrInfo.td
@@ -607,6 +607,8 @@ def : Pat<(membarrier (i64 imm), (i64 imm), (i64 imm), (i64 1), (i64 imm)),
 def : Pat<(membarrier (i64 imm), (i64 imm), (i64 imm), (i64 imm), (i64 imm)),
           (MB)>;
 
+def : Pat<(atomic_fence (imm), (imm)), (MB)>;
+
 //Basic Floating point ops
 
 //Floats
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.cpp b/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.cpp
index df8f157266e1..8b6230fa2a24 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.cpp
@@ -21,7 +21,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -40,8 +39,7 @@
 using namespace llvm;
 
 AlphaRegisterInfo::AlphaRegisterInfo(const TargetInstrInfo &tii)
-  : AlphaGenRegisterInfo(),
-    TII(tii) {
+  : AlphaGenRegisterInfo(Alpha::R26), TII(tii) {
 }
 
 static long getUpper16(long l) {
@@ -178,10 +176,6 @@ AlphaRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 }
 
-unsigned AlphaRegisterInfo::getRARegister() const {
-  return Alpha::R26;
-}
-
 unsigned AlphaRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
 
@@ -198,16 +192,6 @@ unsigned AlphaRegisterInfo::getEHHandlerRegister() const {
   return 0;
 }
 
-int AlphaRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  llvm_unreachable("What is the dwarf register number");
-  return -1;
-}
-
-int AlphaRegisterInfo::getLLVMRegNum(unsigned DwarfRegNum, bool isEH) const {
-  llvm_unreachable("What is the dwarf register number");
-  return -1;
-}
-
 std::string AlphaRegisterInfo::getPrettyName(unsigned reg)
 {
   std::string s(AlphaRegDesc[reg].Name);
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.h b/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.h
index 1072bf73f199..e35be273c7cd 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.h
+++ b/contrib/llvm/lib/Target/Alpha/AlphaRegisterInfo.h
@@ -42,16 +42,12 @@ struct AlphaRegisterInfo : public AlphaGenRegisterInfo {
                            int SPAdj, RegScavenger *RS = NULL) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   // Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
 
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
-
   static std::string getPrettyName(unsigned reg);
 };
 
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaSubtarget.cpp b/contrib/llvm/lib/Target/Alpha/AlphaSubtarget.cpp
index 624a5e2ebd09..bd55ce9e315a 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaSubtarget.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaSubtarget.cpp
@@ -13,7 +13,6 @@
 
 #include "AlphaSubtarget.h"
 #include "Alpha.h"
-#include "llvm/Target/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.cpp b/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.cpp
index 3b65d41be892..fc9a6771a900 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.cpp
@@ -14,7 +14,7 @@
 #include "AlphaTargetMachine.h"
 #include "llvm/PassManager.h"
 #include "llvm/Support/FormattedStream.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 extern "C" void LLVMInitializeAlphaTarget() { 
@@ -22,19 +22,17 @@ extern "C" void LLVMInitializeAlphaTarget() {
   RegisterTargetMachine<AlphaTargetMachine> X(TheAlphaTarget);
 }
 
-AlphaTargetMachine::AlphaTargetMachine(const Target &T, const std::string &TT,
-                                       const std::string &CPU,
-                                       const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+AlphaTargetMachine::AlphaTargetMachine(const Target &T, StringRef TT,
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     DataLayout("e-f128:128:128-n64"),
     FrameLowering(Subtarget),
     Subtarget(TT, CPU, FS),
     TLInfo(*this),
     TSInfo(*this) {
-  setRelocationModel(Reloc::PIC_);
 }
 
-
 //===----------------------------------------------------------------------===//
 // Pass Pipeline Configuration
 //===----------------------------------------------------------------------===//
diff --git a/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.h b/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.h
index cf00e5875d34..48bb948a7945 100644
--- a/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.h
+++ b/contrib/llvm/lib/Target/Alpha/AlphaTargetMachine.h
@@ -36,8 +36,9 @@ class AlphaTargetMachine : public LLVMTargetMachine {
   AlphaSelectionDAGInfo TSInfo;
 
 public:
-  AlphaTargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS);
+  AlphaTargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 
   virtual const AlphaInstrInfo *getInstrInfo() const { return &InstrInfo; }
   virtual const TargetFrameLowering  *getFrameLowering() const {
diff --git a/contrib/llvm/lib/Target/Alpha/MCTargetDesc/AlphaMCTargetDesc.cpp b/contrib/llvm/lib/Target/Alpha/MCTargetDesc/AlphaMCTargetDesc.cpp
index 562052b6df67..4ad021ca6761 100644
--- a/contrib/llvm/lib/Target/Alpha/MCTargetDesc/AlphaMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/Alpha/MCTargetDesc/AlphaMCTargetDesc.cpp
@@ -13,10 +13,11 @@
 
 #include "AlphaMCTargetDesc.h"
 #include "AlphaMCAsmInfo.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "AlphaGenInstrInfo.inc"
@@ -36,8 +37,10 @@ static MCInstrInfo *createAlphaMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeAlphaMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheAlphaTarget, createAlphaMCInstrInfo);
+static MCRegisterInfo *createAlphaMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitAlphaMCRegisterInfo(X, Alpha::R26);
+  return X;
 }
 
 static MCSubtargetInfo *createAlphaMCSubtargetInfo(StringRef TT, StringRef CPU,
@@ -47,11 +50,29 @@ static MCSubtargetInfo *createAlphaMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeAlphaMCSubtargetInfo() {
+static MCCodeGenInfo *createAlphaMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                               CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->InitMCCodeGenInfo(Reloc::PIC_, CM);
+  return X;
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeAlphaTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfo<AlphaMCAsmInfo> X(TheAlphaTarget);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheAlphaTarget,
+                                        createAlphaMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheAlphaTarget, createAlphaMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheAlphaTarget, createAlphaMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(TheAlphaTarget,
                                           createAlphaMCSubtargetInfo);
 }
-
-extern "C" void LLVMInitializeAlphaMCAsmInfo() {
-  RegisterMCAsmInfo<AlphaMCAsmInfo> X(TheAlphaTarget);
-}
diff --git a/contrib/llvm/lib/Target/Alpha/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/Alpha/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index ad0dd26aafb1..000000000000
--- a/contrib/llvm/lib/Target/Alpha/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMAlphaDesc
-  AlphaMCTargetDesc.cpp
-  AlphaMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/Alpha/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/Alpha/MCTargetDesc/Makefile
deleted file mode 100644
index d55175fa69dc..000000000000
--- a/contrib/llvm/lib/Target/Alpha/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/Alpha/TargetDesc/Makefile ----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMAlphaDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/Alpha/TargetInfo/AlphaTargetInfo.cpp b/contrib/llvm/lib/Target/Alpha/TargetInfo/AlphaTargetInfo.cpp
index f7099b9ae975..bdc69e788bcc 100644
--- a/contrib/llvm/lib/Target/Alpha/TargetInfo/AlphaTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/Alpha/TargetInfo/AlphaTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "Alpha.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 llvm::Target llvm::TheAlphaTarget;
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinAsmPrinter.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinAsmPrinter.cpp
index 6ba258beb2b8..ed9844e1bee4 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinAsmPrinter.cpp
@@ -29,9 +29,9 @@
 #include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinFrameLowering.h b/contrib/llvm/lib/Target/Blackfin/BlackfinFrameLowering.h
index 726fa2c063f7..169aa8e3011d 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinFrameLowering.h
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinFrameLowering.h
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef ALPHA_FRAMEINFO_H
-#define ALPHA_FRAMEINFO_H
+#ifndef BLACKFIN_FRAMEINFO_H
+#define BLACKFIN_FRAMEINFO_H
 
 #include "Blackfin.h"
 #include "BlackfinSubtarget.h"
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.cpp
index d5728324de87..7d4c45fdf665 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.cpp
@@ -42,6 +42,7 @@ using namespace llvm;
 BlackfinTargetLowering::BlackfinTargetLowering(TargetMachine &TM)
   : TargetLowering(TM, new TargetLoweringObjectFileELF()) {
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
   setStackPointerRegisterToSaveRestore(BF::SP);
   setIntDivIsCheap(false);
 
@@ -99,6 +100,7 @@ BlackfinTargetLowering::BlackfinTargetLowering(TargetMachine &TM)
 
   // Blackfin has no intrinsics for these particular operations.
   setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
   setOperationAction(ISD::BSWAP, MVT::i32, Expand);
 
   setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
@@ -134,7 +136,7 @@ const char *BlackfinTargetLowering::getTargetNodeName(unsigned Opcode) const {
   }
 }
 
-MVT::SimpleValueType BlackfinTargetLowering::getSetCCResultType(EVT VT) const {
+EVT BlackfinTargetLowering::getSetCCResultType(EVT VT) const {
   // SETCC always sets the CC register. Technically that is an i1 register, but
   // that type is not legal, so we treat it as an i32 register.
   return MVT::i32;
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.h b/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.h
index b65775b9285d..90908baaae9d 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.h
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinISelLowering.h
@@ -33,7 +33,7 @@ namespace llvm {
   public:
     BlackfinTargetLowering(TargetMachine &TM);
     virtual MVT getShiftAmountTy(EVT LHSTy) const { return MVT::i16; }
-    virtual MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    virtual EVT getSetCCResultType(EVT VT) const;
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
     virtual void ReplaceNodeResults(SDNode *N,
                                     SmallVectorImpl<SDValue> &Results,
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinInstrInfo.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinInstrInfo.cpp
index d190ae7984b2..c06a919708d6 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinInstrInfo.cpp
@@ -16,10 +16,10 @@
 #include "Blackfin.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_CTOR
 #include "BlackfinGenInstrInfo.inc"
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.cpp
index ae8ee9e2a1a2..71356768dd5d 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.cpp
@@ -34,7 +34,7 @@ namespace bfinIntrinsic {
 
 }
 
-std::string BlackfinIntrinsicInfo::getName(unsigned IntrID, const Type **Tys,
+std::string BlackfinIntrinsicInfo::getName(unsigned IntrID, Type **Tys,
                                            unsigned numTys) const {
   static const char *const names[] = {
 #define GET_INTRINSIC_NAME_TABLE
@@ -81,8 +81,8 @@ bool BlackfinIntrinsicInfo::isOverloaded(unsigned IntrID) const {
 #include "BlackfinGenIntrinsics.inc"
 #undef GET_INTRINSIC_ATTRIBUTES
 
-static const FunctionType *getType(LLVMContext &Context, unsigned id) {
-  const Type *ResultTy = NULL;
+static FunctionType *getType(LLVMContext &Context, unsigned id) {
+  Type *ResultTy = NULL;
   std::vector<Type*> ArgTys;
   bool IsVarArg = false;
   
@@ -94,7 +94,7 @@ static const FunctionType *getType(LLVMContext &Context, unsigned id) {
 }
 
 Function *BlackfinIntrinsicInfo::getDeclaration(Module *M, unsigned IntrID,
-                                                const Type **Tys,
+                                                Type **Tys,
                                                 unsigned numTy) const {
   assert(!isOverloaded(IntrID) && "Blackfin intrinsics are not overloaded");
   AttrListPtr AList = getAttributes((bfinIntrinsic::ID) IntrID);
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.h b/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.h
index 7c4b5a9967d2..f05db5ad7cd9 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.h
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinIntrinsicInfo.h
@@ -19,11 +19,11 @@ namespace llvm {
 
   class BlackfinIntrinsicInfo : public TargetIntrinsicInfo {
   public:
-    std::string getName(unsigned IntrID, const Type **Tys = 0,
+    std::string getName(unsigned IntrID, Type **Tys = 0,
                         unsigned numTys = 0) const;
     unsigned lookupName(const char *Name, unsigned Len) const;
     bool isOverloaded(unsigned IID) const;
-    Function *getDeclaration(Module *M, unsigned ID, const Type **Tys = 0,
+    Function *getDeclaration(Module *M, unsigned ID, Type **Tys = 0,
                              unsigned numTys = 0) const;
   };
 
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.cpp
index 3a7c104ee055..0d415c5f342b 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.cpp
@@ -20,7 +20,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
@@ -37,7 +36,7 @@ using namespace llvm;
 
 BlackfinRegisterInfo::BlackfinRegisterInfo(BlackfinSubtarget &st,
                                            const TargetInstrInfo &tii)
-  : BlackfinGenRegisterInfo(), Subtarget(st), TII(tii) {}
+  : BlackfinGenRegisterInfo(BF::RETS), Subtarget(st), TII(tii) {}
 
 const unsigned*
 BlackfinRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
@@ -327,10 +326,6 @@ BlackfinRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 }
 
-unsigned BlackfinRegisterInfo::getRARegister() const {
-  return BF::RETS;
-}
-
 unsigned
 BlackfinRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
@@ -347,14 +342,3 @@ unsigned BlackfinRegisterInfo::getEHHandlerRegister() const {
   llvm_unreachable("What is the exception handler register");
   return 0;
 }
-
-int BlackfinRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  llvm_unreachable("What is the dwarf register number");
-  return -1;
-}
-
-int BlackfinRegisterInfo::getLLVMRegNum(unsigned DwarfRegNum,
-                                        bool isEH) const {
-  llvm_unreachable("What is the dwarf register number");
-  return -1;
-}
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.h b/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.h
index 86f45c17c625..6ac22af793e0 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.h
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinRegisterInfo.h
@@ -53,15 +53,11 @@ namespace llvm {
                              int SPAdj, RegScavenger *RS = NULL) const;
 
     unsigned getFrameRegister(const MachineFunction &MF) const;
-    unsigned getRARegister() const;
 
     // Exception handling queries.
     unsigned getEHExceptionRegister() const;
     unsigned getEHHandlerRegister() const;
 
-    int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-    int getLLVMRegNum(unsigned RegNum, bool isEH) const;
-
     // Utility functions
     void adjustRegister(MachineBasicBlock &MBB,
                         MachineBasicBlock::iterator I,
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinSubtarget.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinSubtarget.cpp
index ec919cdf0b90..0bdce09177ed 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinSubtarget.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinSubtarget.cpp
@@ -13,7 +13,7 @@
 
 #include "BlackfinSubtarget.h"
 #include "Blackfin.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.cpp b/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.cpp
index a1c9f1c05e0d..a4ae46b90fa0 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.cpp
@@ -13,7 +13,7 @@
 #include "BlackfinTargetMachine.h"
 #include "Blackfin.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
 
@@ -22,10 +22,12 @@ extern "C" void LLVMInitializeBlackfinTarget() {
 }
 
 BlackfinTargetMachine::BlackfinTargetMachine(const Target &T,
-                                             const std::string &TT,
-                                             const std::string &CPU,
-                                             const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+                                             StringRef TT,
+                                             StringRef CPU,
+                                             StringRef FS,
+                                             Reloc::Model RM,
+                                             CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     DataLayout("e-p:32:32-i64:32-f64:32-n32"),
     Subtarget(TT, CPU, FS),
     TLInfo(*this),
diff --git a/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.h b/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.h
index bd7dc84f04ae..c85337fe237f 100644
--- a/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.h
+++ b/contrib/llvm/lib/Target/Blackfin/BlackfinTargetMachine.h
@@ -35,8 +35,9 @@ namespace llvm {
     BlackfinFrameLowering FrameLowering;
     BlackfinIntrinsicInfo IntrinsicInfo;
   public:
-    BlackfinTargetMachine(const Target &T, const std::string &TT,
-                          const std::string &CPU, const std::string &FS);
+    BlackfinTargetMachine(const Target &T, StringRef TT,
+                          StringRef CPU, StringRef FS,
+                          Reloc::Model RM, CodeModel::Model CM);
 
     virtual const BlackfinInstrInfo *getInstrInfo() const { return &InstrInfo; }
     virtual const TargetFrameLowering *getFrameLowering() const {
diff --git a/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/BlackfinMCTargetDesc.cpp b/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/BlackfinMCTargetDesc.cpp
index 0fa1471ae3e7..272e3c2bbb75 100644
--- a/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/BlackfinMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/BlackfinMCTargetDesc.cpp
@@ -13,10 +13,11 @@
 
 #include "BlackfinMCTargetDesc.h"
 #include "BlackfinMCAsmInfo.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "BlackfinGenInstrInfo.inc"
@@ -36,12 +37,12 @@ static MCInstrInfo *createBlackfinMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeBlackfinMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheBlackfinTarget,
-                                      createBlackfinMCInstrInfo);
+static MCRegisterInfo *createBlackfinMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitBlackfinMCRegisterInfo(X, BF::RETS);
+  return X;
 }
 
-
 static MCSubtargetInfo *createBlackfinMCSubtargetInfo(StringRef TT,
                                                       StringRef CPU,
                                                       StringRef FS) {
@@ -50,11 +51,31 @@ static MCSubtargetInfo *createBlackfinMCSubtargetInfo(StringRef TT,
   return X;
 }
 
-extern "C" void LLVMInitializeBlackfinMCSubtargetInfo() {
+static MCCodeGenInfo *createBlackfinMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                                  CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeBlackfinTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfo<BlackfinMCAsmInfo> X(TheBlackfinTarget);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheBlackfinTarget,
+                                        createBlackfinMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheBlackfinTarget,
+                                      createBlackfinMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheBlackfinTarget,
+                                    createBlackfinMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(TheBlackfinTarget,
                                           createBlackfinMCSubtargetInfo);
 }
-
-extern "C" void LLVMInitializeBlackfinMCAsmInfo() {
-  RegisterMCAsmInfo<BlackfinMCAsmInfo> X(TheBlackfinTarget);
-}
diff --git a/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 8cd924f9236f..000000000000
--- a/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMBlackfinDesc
-  BlackfinMCTargetDesc.cpp
-  BlackfinMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/Makefile
deleted file mode 100644
index 6b26101f4473..000000000000
--- a/contrib/llvm/lib/Target/Blackfin/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/Blackfin/TargetDesc/Makefile -------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMBlackfinDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/Blackfin/TargetInfo/BlackfinTargetInfo.cpp b/contrib/llvm/lib/Target/Blackfin/TargetInfo/BlackfinTargetInfo.cpp
index 402e0afde81d..57f1d3e95fbf 100644
--- a/contrib/llvm/lib/Target/Blackfin/TargetInfo/BlackfinTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/Blackfin/TargetInfo/BlackfinTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "Blackfin.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/CBackend/CBackend.cpp b/contrib/llvm/lib/Target/CBackend/CBackend.cpp
index 415beb1dd1cd..69d8c46a5024 100644
--- a/contrib/llvm/lib/Target/CBackend/CBackend.cpp
+++ b/contrib/llvm/lib/Target/CBackend/CBackend.cpp
@@ -37,10 +37,11 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/CFG.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -48,6 +49,7 @@
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/InstVisitor.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Config/config.h"
 #include <algorithm>
@@ -62,12 +64,6 @@ extern "C" void LLVMInitializeCBackendTarget() {
   RegisterTargetMachine<CTargetMachine> X(TheCBackendTarget);
 }
 
-extern "C" void LLVMInitializeCBackendMCAsmInfo() {}
-
-extern "C" void LLVMInitializeCBackendMCInstrInfo() {}
-
-extern "C" void LLVMInitializeCBackendMCSubtargetInfo() {}
-
 namespace {
   class CBEMCAsmInfo : public MCAsmInfo {
   public:
@@ -86,6 +82,8 @@ namespace {
     LoopInfo *LI;
     const Module *TheModule;
     const MCAsmInfo* TAsm;
+    const MCRegisterInfo *MRI;
+    const MCObjectFileInfo *MOFI;
     MCContext *TCtx;
     const TargetData* TD;
     
@@ -99,14 +97,14 @@ namespace {
 
     /// UnnamedStructIDs - This contains a unique ID for each struct that is
     /// either anonymous or has no name.
-    DenseMap<const StructType*, unsigned> UnnamedStructIDs;
+    DenseMap<StructType*, unsigned> UnnamedStructIDs;
     
   public:
     static char ID;
     explicit CWriter(formatted_raw_ostream &o)
       : FunctionPass(ID), Out(o), IL(0), Mang(0), LI(0),
-        TheModule(0), TAsm(0), TCtx(0), TD(0), OpaqueCounter(0),
-        NextAnonValueNumber(0) {
+        TheModule(0), TAsm(0), MRI(0), MOFI(0), TCtx(0), TD(0),
+        OpaqueCounter(0), NextAnonValueNumber(0) {
       initializeLoopInfoPass(*PassRegistry::getPassRegistry());
       FPCounter = 0;
     }
@@ -145,6 +143,8 @@ namespace {
       delete Mang;
       delete TCtx;
       delete TAsm;
+      delete MRI;
+      delete MOFI;
       FPConstantMap.clear();
       ByValParams.clear();
       intrinsicPrototypesAlreadyGenerated.clear();
@@ -152,20 +152,20 @@ namespace {
       return false;
     }
 
-    raw_ostream &printType(raw_ostream &Out, const Type *Ty,
+    raw_ostream &printType(raw_ostream &Out, Type *Ty,
                            bool isSigned = false,
                            const std::string &VariableName = "",
                            bool IgnoreName = false,
                            const AttrListPtr &PAL = AttrListPtr());
-    raw_ostream &printSimpleType(raw_ostream &Out, const Type *Ty,
+    raw_ostream &printSimpleType(raw_ostream &Out, Type *Ty,
                                  bool isSigned,
                                  const std::string &NameSoFar = "");
 
     void printStructReturnPointerFunctionType(raw_ostream &Out,
                                               const AttrListPtr &PAL,
-                                              const PointerType *Ty);
+                                              PointerType *Ty);
 
-    std::string getStructName(const StructType *ST);
+    std::string getStructName(StructType *ST);
     
     /// writeOperandDeref - Print the result of dereferencing the specified
     /// operand with '*'.  This is equivalent to printing '*' then using
@@ -188,7 +188,7 @@ namespace {
     void writeOperandWithCast(Value* Operand, const ICmpInst &I);
     bool writeInstructionCast(const Instruction &I);
 
-    void writeMemoryAccess(Value *Operand, const Type *OperandType,
+    void writeMemoryAccess(Value *Operand, Type *OperandType,
                            bool IsVolatile, unsigned Alignment);
 
   private :
@@ -200,7 +200,7 @@ namespace {
     void printIntrinsicDefinition(const Function &F, raw_ostream &Out);
 
     void printModuleTypes();
-    void printContainedStructs(const Type *Ty, SmallPtrSet<const Type *, 16> &);
+    void printContainedStructs(Type *Ty, SmallPtrSet<Type *, 16> &);
     void printFloatingPointConstants(Function &F);
     void printFloatingPointConstants(const Constant *C);
     void printFunctionSignature(const Function *F, bool Prototype);
@@ -209,7 +209,7 @@ namespace {
     void printBasicBlock(BasicBlock *BB);
     void printLoop(Loop *L);
 
-    void printCast(unsigned opcode, const Type *SrcTy, const Type *DstTy);
+    void printCast(unsigned opcode, Type *SrcTy, Type *DstTy);
     void printConstant(Constant *CPV, bool Static);
     void printConstantWithCast(Constant *CPV, unsigned Opcode);
     bool printConstExprCast(const ConstantExpr *CE, bool Static);
@@ -288,10 +288,12 @@ namespace {
     void visitInvokeInst(InvokeInst &I) {
       llvm_unreachable("Lowerinvoke pass didn't work!");
     }
-
     void visitUnwindInst(UnwindInst &I) {
       llvm_unreachable("Lowerinvoke pass didn't work!");
     }
+    void visitResumeInst(ResumeInst &I) {
+      llvm_unreachable("DwarfEHPrepare pass didn't work!");
+    }
     void visitUnreachableInst(UnreachableInst &I);
 
     void visitPHINode(PHINode &I);
@@ -360,8 +362,8 @@ static std::string CBEMangle(const std::string &S) {
   return Result;
 }
 
-std::string CWriter::getStructName(const StructType *ST) {
-  if (!ST->isAnonymous() && !ST->getName().empty())
+std::string CWriter::getStructName(StructType *ST) {
+  if (!ST->isLiteral() && !ST->getName().empty())
     return CBEMangle("l_"+ST->getName().str());
   
   return "l_unnamed_" + utostr(UnnamedStructIDs[ST]);
@@ -373,20 +375,20 @@ std::string CWriter::getStructName(const StructType *ST) {
 /// print it as "Struct (*)(...)", for struct return functions.
 void CWriter::printStructReturnPointerFunctionType(raw_ostream &Out,
                                                    const AttrListPtr &PAL,
-                                                   const PointerType *TheTy) {
-  const FunctionType *FTy = cast<FunctionType>(TheTy->getElementType());
+                                                   PointerType *TheTy) {
+  FunctionType *FTy = cast<FunctionType>(TheTy->getElementType());
   std::string tstr;
   raw_string_ostream FunctionInnards(tstr);
   FunctionInnards << " (*) (";
   bool PrintedType = false;
 
   FunctionType::param_iterator I = FTy->param_begin(), E = FTy->param_end();
-  const Type *RetTy = cast<PointerType>(*I)->getElementType();
+  Type *RetTy = cast<PointerType>(*I)->getElementType();
   unsigned Idx = 1;
   for (++I, ++Idx; I != E; ++I, ++Idx) {
     if (PrintedType)
       FunctionInnards << ", ";
-    const Type *ArgTy = *I;
+    Type *ArgTy = *I;
     if (PAL.paramHasAttr(Idx, Attribute::ByVal)) {
       assert(ArgTy->isPointerTy());
       ArgTy = cast<PointerType>(ArgTy)->getElementType();
@@ -408,7 +410,7 @@ void CWriter::printStructReturnPointerFunctionType(raw_ostream &Out,
 }
 
 raw_ostream &
-CWriter::printSimpleType(raw_ostream &Out, const Type *Ty, bool isSigned,
+CWriter::printSimpleType(raw_ostream &Out, Type *Ty, bool isSigned,
                          const std::string &NameSoFar) {
   assert((Ty->isPrimitiveType() || Ty->isIntegerTy() || Ty->isVectorTy()) &&
          "Invalid type for printSimpleType");
@@ -444,7 +446,7 @@ CWriter::printSimpleType(raw_ostream &Out, const Type *Ty, bool isSigned,
                      " __attribute__((vector_size(64))) " + NameSoFar);
 
   case Type::VectorTyID: {
-    const VectorType *VTy = cast<VectorType>(Ty);
+    VectorType *VTy = cast<VectorType>(Ty);
     return printSimpleType(Out, VTy->getElementType(), isSigned,
                      " __attribute__((vector_size(" +
                      utostr(TD->getTypeAllocSize(VTy)) + " ))) " + NameSoFar);
@@ -461,7 +463,7 @@ CWriter::printSimpleType(raw_ostream &Out, const Type *Ty, bool isSigned,
 // Pass the Type* and the variable name and this prints out the variable
 // declaration.
 //
-raw_ostream &CWriter::printType(raw_ostream &Out, const Type *Ty,
+raw_ostream &CWriter::printType(raw_ostream &Out, Type *Ty,
                                 bool isSigned, const std::string &NameSoFar,
                                 bool IgnoreName, const AttrListPtr &PAL) {
   if (Ty->isPrimitiveType() || Ty->isIntegerTy() || Ty->isVectorTy()) {
@@ -471,14 +473,14 @@ raw_ostream &CWriter::printType(raw_ostream &Out, const Type *Ty,
 
   switch (Ty->getTypeID()) {
   case Type::FunctionTyID: {
-    const FunctionType *FTy = cast<FunctionType>(Ty);
+    FunctionType *FTy = cast<FunctionType>(Ty);
     std::string tstr;
     raw_string_ostream FunctionInnards(tstr);
     FunctionInnards << " (" << NameSoFar << ") (";
     unsigned Idx = 1;
     for (FunctionType::param_iterator I = FTy->param_begin(),
            E = FTy->param_end(); I != E; ++I) {
-      const Type *ArgTy = *I;
+      Type *ArgTy = *I;
       if (PAL.paramHasAttr(Idx, Attribute::ByVal)) {
         assert(ArgTy->isPointerTy());
         ArgTy = cast<PointerType>(ArgTy)->getElementType();
@@ -502,7 +504,7 @@ raw_ostream &CWriter::printType(raw_ostream &Out, const Type *Ty,
     return Out;
   }
   case Type::StructTyID: {
-    const StructType *STy = cast<StructType>(Ty);
+    StructType *STy = cast<StructType>(Ty);
     
     // Check to see if the type is named.
     if (!IgnoreName)
@@ -523,7 +525,7 @@ raw_ostream &CWriter::printType(raw_ostream &Out, const Type *Ty,
   }
 
   case Type::PointerTyID: {
-    const PointerType *PTy = cast<PointerType>(Ty);
+    PointerType *PTy = cast<PointerType>(Ty);
     std::string ptrName = "*" + NameSoFar;
 
     if (PTy->getElementType()->isArrayTy() ||
@@ -537,7 +539,7 @@ raw_ostream &CWriter::printType(raw_ostream &Out, const Type *Ty,
   }
 
   case Type::ArrayTyID: {
-    const ArrayType *ATy = cast<ArrayType>(Ty);
+    ArrayType *ATy = cast<ArrayType>(Ty);
     unsigned NumElements = ATy->getNumElements();
     if (NumElements == 0) NumElements = 1;
     // Arrays are wrapped in structs to allow them to have normal
@@ -560,7 +562,7 @@ void CWriter::printConstantArray(ConstantArray *CPA, bool Static) {
   // As a special case, print the array as a string if it is an array of
   // ubytes or an array of sbytes with positive values.
   //
-  const Type *ETy = CPA->getType()->getElementType();
+  Type *ETy = CPA->getType()->getElementType();
   bool isString = (ETy == Type::getInt8Ty(CPA->getContext()) ||
                    ETy == Type::getInt8Ty(CPA->getContext()));
 
@@ -682,7 +684,7 @@ static bool isFPCSafeToPrint(const ConstantFP *CFP) {
 /// Print out the casting for a cast operation. This does the double casting
 /// necessary for conversion to the destination type, if necessary.
 /// @brief Print a cast
-void CWriter::printCast(unsigned opc, const Type *SrcTy, const Type *DstTy) {
+void CWriter::printCast(unsigned opc, Type *SrcTy, Type *DstTy) {
   // Print the destination type cast
   switch (opc) {
     case Instruction::UIToFP:
@@ -917,7 +919,7 @@ void CWriter::printConstant(Constant *CPV, bool Static) {
   }
 
   if (ConstantInt *CI = dyn_cast<ConstantInt>(CPV)) {
-    const Type* Ty = CI->getType();
+    Type* Ty = CI->getType();
     if (Ty == Type::getInt1Ty(CPV->getContext()))
       Out << (CI->getZExtValue() ? '1' : '0');
     else if (Ty == Type::getInt32Ty(CPV->getContext()))
@@ -1027,7 +1029,7 @@ void CWriter::printConstant(Constant *CPV, bool Static) {
       printConstantArray(CA, Static);
     } else {
       assert(isa<ConstantAggregateZero>(CPV) || isa<UndefValue>(CPV));
-      const ArrayType *AT = cast<ArrayType>(CPV->getType());
+      ArrayType *AT = cast<ArrayType>(CPV->getType());
       Out << '{';
       if (AT->getNumElements()) {
         Out << ' ';
@@ -1054,7 +1056,7 @@ void CWriter::printConstant(Constant *CPV, bool Static) {
       printConstantVector(CV, Static);
     } else {
       assert(isa<ConstantAggregateZero>(CPV) || isa<UndefValue>(CPV));
-      const VectorType *VT = cast<VectorType>(CPV->getType());
+      VectorType *VT = cast<VectorType>(CPV->getType());
       Out << "{ ";
       Constant *CZ = Constant::getNullValue(VT->getElementType());
       printConstant(CZ, Static);
@@ -1074,7 +1076,7 @@ void CWriter::printConstant(Constant *CPV, bool Static) {
       Out << ")";
     }
     if (isa<ConstantAggregateZero>(CPV) || isa<UndefValue>(CPV)) {
-      const StructType *ST = cast<StructType>(CPV->getType());
+      StructType *ST = cast<StructType>(CPV->getType());
       Out << '{';
       if (ST->getNumElements()) {
         Out << ' ';
@@ -1123,7 +1125,7 @@ void CWriter::printConstant(Constant *CPV, bool Static) {
 // care of detecting that case and printing the cast for the ConstantExpr.
 bool CWriter::printConstExprCast(const ConstantExpr* CE, bool Static) {
   bool NeedsExplicitCast = false;
-  const Type *Ty = CE->getOperand(0)->getType();
+  Type *Ty = CE->getOperand(0)->getType();
   bool TypeIsSigned = false;
   switch (CE->getOpcode()) {
   case Instruction::Add:
@@ -1175,7 +1177,7 @@ bool CWriter::printConstExprCast(const ConstantExpr* CE, bool Static) {
 void CWriter::printConstantWithCast(Constant* CPV, unsigned Opcode) {
 
   // Extract the operand's type, we'll need it.
-  const Type* OpTy = CPV->getType();
+  Type* OpTy = CPV->getType();
 
   // Indicate whether to do the cast or not.
   bool shouldCast = false;
@@ -1267,7 +1269,7 @@ std::string CWriter::GetValueName(const Value *Operand) {
 void CWriter::writeInstComputationInline(Instruction &I) {
   // We can't currently support integer types other than 1, 8, 16, 32, 64.
   // Validate this.
-  const Type *Ty = I.getType();
+  Type *Ty = I.getType();
   if (Ty->isIntegerTy() && (Ty!=Type::getInt1Ty(I.getContext()) &&
         Ty!=Type::getInt8Ty(I.getContext()) &&
         Ty!=Type::getInt16Ty(I.getContext()) &&
@@ -1330,7 +1332,7 @@ void CWriter::writeOperand(Value *Operand, bool Static) {
 // This function takes care of detecting that case and printing the cast
 // for the Instruction.
 bool CWriter::writeInstructionCast(const Instruction &I) {
-  const Type *Ty = I.getOperand(0)->getType();
+  Type *Ty = I.getOperand(0)->getType();
   switch (I.getOpcode()) {
   case Instruction::Add:
   case Instruction::Sub:
@@ -1362,7 +1364,7 @@ bool CWriter::writeInstructionCast(const Instruction &I) {
 void CWriter::writeOperandWithCast(Value* Operand, unsigned Opcode) {
 
   // Extract the operand's type, we'll need it.
-  const Type* OpTy = Operand->getType();
+  Type* OpTy = Operand->getType();
 
   // Indicate whether to do the cast or not.
   bool shouldCast = false;
@@ -1430,7 +1432,7 @@ void CWriter::writeOperandWithCast(Value* Operand, const ICmpInst &Cmp) {
   bool castIsSigned = Cmp.isSigned();
 
   // If the operand was a pointer, convert to a large integer type.
-  const Type* OpTy = Operand->getType();
+  Type* OpTy = Operand->getType();
   if (OpTy->isPointerTy())
     OpTy = TD->getIntPtrType(Operand->getContext());
 
@@ -1665,7 +1667,8 @@ bool CWriter::doInitialization(Module &M) {
     TAsm = Match->createMCAsmInfo(Triple);
 #endif
   TAsm = new CBEMCAsmInfo();
-  TCtx = new MCContext(*TAsm, NULL);
+  MRI  = new MCRegisterInfo();
+  TCtx = new MCContext(*TAsm, *MRI, NULL);
   Mang = new Mangler(*TCtx, *TD);
 
   // Keep track of which functions are static ctors/dtors so they can have
@@ -2049,7 +2052,7 @@ void CWriter::printModuleTypes() {
   for (unsigned i = 0, e = StructTypes.size(); i != e; ++i) {
     StructType *ST = StructTypes[i];
 
-    if (ST->isAnonymous() || ST->getName().empty())
+    if (ST->isLiteral() || ST->getName().empty())
       UnnamedStructIDs[ST] = NextTypeID++;
 
     std::string Name = getStructName(ST);
@@ -2060,7 +2063,7 @@ void CWriter::printModuleTypes() {
   Out << '\n';
 
   // Keep track of which structures have been printed so far.
-  SmallPtrSet<const Type *, 16> StructPrinted;
+  SmallPtrSet<Type *, 16> StructPrinted;
 
   // Loop over all structures then push them into the stack so they are
   // printed in the correct order.
@@ -2077,8 +2080,8 @@ void CWriter::printModuleTypes() {
 //
 // TODO:  Make this work properly with vector types
 //
-void CWriter::printContainedStructs(const Type *Ty,
-                                SmallPtrSet<const Type *, 16> &StructPrinted) {
+void CWriter::printContainedStructs(Type *Ty,
+                                SmallPtrSet<Type *, 16> &StructPrinted) {
   // Don't walk through pointers.
   if (Ty->isPointerTy() || Ty->isPrimitiveType() || Ty->isIntegerTy())
     return;
@@ -2088,7 +2091,7 @@ void CWriter::printContainedStructs(const Type *Ty,
        E = Ty->subtype_end(); I != E; ++I)
     printContainedStructs(*I, StructPrinted);
 
-  if (const StructType *ST = dyn_cast<StructType>(Ty)) {
+  if (StructType *ST = dyn_cast<StructType>(Ty)) {
     // Check to see if we have already printed this struct.
     if (!StructPrinted.insert(Ty)) return;
     
@@ -2120,7 +2123,7 @@ void CWriter::printFunctionSignature(const Function *F, bool Prototype) {
   }
 
   // Loop over the arguments, printing them...
-  const FunctionType *FT = cast<FunctionType>(F->getFunctionType());
+  FunctionType *FT = cast<FunctionType>(F->getFunctionType());
   const AttrListPtr &PAL = F->getAttributes();
 
   std::string tstr;
@@ -2150,7 +2153,7 @@ void CWriter::printFunctionSignature(const Function *F, bool Prototype) {
           ArgName = GetValueName(I);
         else
           ArgName = "";
-        const Type *ArgTy = I->getType();
+        Type *ArgTy = I->getType();
         if (PAL.paramHasAttr(Idx, Attribute::ByVal)) {
           ArgTy = cast<PointerType>(ArgTy)->getElementType();
           ByValParams.insert(I);
@@ -2177,7 +2180,7 @@ void CWriter::printFunctionSignature(const Function *F, bool Prototype) {
 
     for (; I != E; ++I) {
       if (PrintedArg) FunctionInnards << ", ";
-      const Type *ArgTy = *I;
+      Type *ArgTy = *I;
       if (PAL.paramHasAttr(Idx, Attribute::ByVal)) {
         assert(ArgTy->isPointerTy());
         ArgTy = cast<PointerType>(ArgTy)->getElementType();
@@ -2205,7 +2208,7 @@ void CWriter::printFunctionSignature(const Function *F, bool Prototype) {
   FunctionInnards << ')';
 
   // Get the return tpe for the function.
-  const Type *RetTy;
+  Type *RetTy;
   if (!isStructReturn)
     RetTy = F->getReturnType();
   else {
@@ -2222,8 +2225,8 @@ void CWriter::printFunctionSignature(const Function *F, bool Prototype) {
 static inline bool isFPIntBitCast(const Instruction &I) {
   if (!isa<BitCastInst>(I))
     return false;
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DstTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DstTy = I.getType();
   return (SrcTy->isFloatingPointTy() && DstTy->isIntegerTy()) ||
          (DstTy->isFloatingPointTy() && SrcTy->isIntegerTy());
 }
@@ -2237,7 +2240,7 @@ void CWriter::printFunction(Function &F) {
 
   // If this is a struct return function, handle the result with magic.
   if (isStructReturn) {
-    const Type *StructTy =
+    Type *StructTy =
       cast<PointerType>(F.arg_begin()->getType())->getElementType();
     Out << "  ";
     printType(Out, StructTy, false, "StructReturn");
@@ -2380,22 +2383,29 @@ void CWriter::visitReturnInst(ReturnInst &I) {
 
 void CWriter::visitSwitchInst(SwitchInst &SI) {
 
+  Value* Cond = SI.getCondition();
+
   Out << "  switch (";
-  writeOperand(SI.getOperand(0));
+  writeOperand(Cond);
   Out << ") {\n  default:\n";
   printPHICopiesForSuccessor (SI.getParent(), SI.getDefaultDest(), 2);
   printBranchToBlock(SI.getParent(), SI.getDefaultDest(), 2);
   Out << ";\n";
-  for (unsigned i = 2, e = SI.getNumOperands(); i != e; i += 2) {
+
+  unsigned NumCases = SI.getNumCases();
+  // Skip the first item since that's the default case.
+  for (unsigned i = 1; i < NumCases; ++i) {
+    ConstantInt* CaseVal = SI.getCaseValue(i);
+    BasicBlock* Succ = SI.getSuccessor(i);
     Out << "  case ";
-    writeOperand(SI.getOperand(i));
+    writeOperand(CaseVal);
     Out << ":\n";
-    BasicBlock *Succ = cast<BasicBlock>(SI.getOperand(i+1));
     printPHICopiesForSuccessor (SI.getParent(), Succ, 2);
     printBranchToBlock(SI.getParent(), Succ, 2);
     if (Function::iterator(Succ) == llvm::next(Function::iterator(SI.getParent())))
       Out << "    break;\n";
   }
+
   Out << "  }\n";
 }
 
@@ -2656,7 +2666,7 @@ void CWriter::visitFCmpInst(FCmpInst &I) {
   Out << ")";
 }
 
-static const char * getFloatBitCastField(const Type *Ty) {
+static const char * getFloatBitCastField(Type *Ty) {
   switch (Ty->getTypeID()) {
     default: llvm_unreachable("Invalid Type");
     case Type::FloatTyID:  return "Float";
@@ -2672,8 +2682,8 @@ static const char * getFloatBitCastField(const Type *Ty) {
 }
 
 void CWriter::visitCastInst(CastInst &I) {
-  const Type *DstTy = I.getType();
-  const Type *SrcTy = I.getOperand(0)->getType();
+  Type *DstTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
   if (isFPIntBitCast(I)) {
     Out << '(';
     // These int<->float and long<->double casts need to be handled specially
@@ -2719,7 +2729,7 @@ void CWriter::visitSelectInst(SelectInst &I) {
 
 // Returns the macro name or value of the max or min of an integer type
 // (as defined in limits.h).
-static void printLimitValue(const IntegerType &Ty, bool isSigned, bool isMax,
+static void printLimitValue(IntegerType &Ty, bool isSigned, bool isMax,
                             raw_ostream &Out) {
   const char* type;
   const char* sprefix = "";
@@ -2745,16 +2755,16 @@ static void printLimitValue(const IntegerType &Ty, bool isSigned, bool isMax,
 }
 
 #ifndef NDEBUG
-static bool isSupportedIntegerSize(const IntegerType &T) {
+static bool isSupportedIntegerSize(IntegerType &T) {
   return T.getBitWidth() == 8 || T.getBitWidth() == 16 ||
          T.getBitWidth() == 32 || T.getBitWidth() == 64;
 }
 #endif
 
 void CWriter::printIntrinsicDefinition(const Function &F, raw_ostream &Out) {
-  const FunctionType *funT = F.getFunctionType();
-  const Type *retT = F.getReturnType();
-  const IntegerType *elemT = cast<IntegerType>(funT->getParamType(1));
+  FunctionType *funT = F.getFunctionType();
+  Type *retT = F.getReturnType();
+  IntegerType *elemT = cast<IntegerType>(funT->getParamType(1));
 
   assert(isSupportedIntegerSize(*elemT) &&
          "CBackend does not support arbitrary size integers.");
@@ -2829,7 +2839,6 @@ void CWriter::lowerIntrinsics(Function &F) {
         if (Function *F = CI->getCalledFunction())
           switch (F->getIntrinsicID()) {
           case Intrinsic::not_intrinsic:
-          case Intrinsic::memory_barrier:
           case Intrinsic::vastart:
           case Intrinsic::vacopy:
           case Intrinsic::vaend:
@@ -2908,8 +2917,8 @@ void CWriter::visitCallInst(CallInst &I) {
 
   Value *Callee = I.getCalledValue();
 
-  const PointerType  *PTy   = cast<PointerType>(Callee->getType());
-  const FunctionType *FTy   = cast<FunctionType>(PTy->getElementType());
+  PointerType  *PTy   = cast<PointerType>(Callee->getType());
+  FunctionType *FTy   = cast<FunctionType>(PTy->getElementType());
 
   // If this is a call to a struct-return function, assign to the first
   // parameter instead of passing it to the call.
@@ -3020,9 +3029,6 @@ bool CWriter::visitBuiltinCall(CallInst &I, Intrinsic::ID ID,
     WroteCallee = true;
     return false;
   }
-  case Intrinsic::memory_barrier:
-    Out << "__sync_synchronize()";
-    return true;
   case Intrinsic::vastart:
     Out << "0; ";
 
@@ -3217,7 +3223,7 @@ void CWriter::visitInlineAsm(CallInst &CI) {
   std::vector<std::pair<Value*, int> > ResultVals;
   if (CI.getType() == Type::getVoidTy(CI.getContext()))
     ;
-  else if (const StructType *ST = dyn_cast<StructType>(CI.getType())) {
+  else if (StructType *ST = dyn_cast<StructType>(CI.getType())) {
     for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i)
       ResultVals.push_back(std::make_pair(&CI, (int)i));
   } else {
@@ -3348,7 +3354,7 @@ void CWriter::printGEPExpression(Value *Ptr, gep_type_iterator I,
   // Find out if the last index is into a vector.  If so, we have to print this
   // specially.  Since vectors can't have elements of indexable type, only the
   // last index could possibly be of a vector element.
-  const VectorType *LastIndexIsVector = 0;
+  VectorType *LastIndexIsVector = 0;
   {
     for (gep_type_iterator TmpI = I; TmpI != E; ++TmpI)
       LastIndexIsVector = dyn_cast<VectorType>(*TmpI);
@@ -3421,7 +3427,7 @@ void CWriter::printGEPExpression(Value *Ptr, gep_type_iterator I,
   Out << ")";
 }
 
-void CWriter::writeMemoryAccess(Value *Operand, const Type *OperandType,
+void CWriter::writeMemoryAccess(Value *Operand, Type *OperandType,
                                 bool IsVolatile, unsigned Alignment) {
 
   bool IsUnaligned = Alignment &&
@@ -3463,7 +3469,7 @@ void CWriter::visitStoreInst(StoreInst &I) {
   Out << " = ";
   Value *Operand = I.getOperand(0);
   Constant *BitMask = 0;
-  if (const IntegerType* ITy = dyn_cast<IntegerType>(Operand->getType()))
+  if (IntegerType* ITy = dyn_cast<IntegerType>(Operand->getType()))
     if (!ITy->isPowerOf2ByteWidth())
       // We have a bit width that doesn't match an even power-of-2 byte
       // size. Consequently we must & the value with the type's bit mask
@@ -3492,7 +3498,7 @@ void CWriter::visitVAArgInst(VAArgInst &I) {
 }
 
 void CWriter::visitInsertElementInst(InsertElementInst &I) {
-  const Type *EltTy = I.getType()->getElementType();
+  Type *EltTy = I.getType()->getElementType();
   writeOperand(I.getOperand(0));
   Out << ";\n  ";
   Out << "((";
@@ -3507,7 +3513,7 @@ void CWriter::visitInsertElementInst(InsertElementInst &I) {
 void CWriter::visitExtractElementInst(ExtractElementInst &I) {
   // We know that our operand is not inlined.
   Out << "((";
-  const Type *EltTy =
+  Type *EltTy =
     cast<VectorType>(I.getOperand(0)->getType())->getElementType();
   printType(Out, PointerType::getUnqual(EltTy));
   Out << ")(&" << GetValueName(I.getOperand(0)) << "))[";
@@ -3519,9 +3525,9 @@ void CWriter::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   Out << "(";
   printType(Out, SVI.getType());
   Out << "){ ";
-  const VectorType *VT = SVI.getType();
+  VectorType *VT = SVI.getType();
   unsigned NumElts = VT->getNumElements();
-  const Type *EltTy = VT->getElementType();
+  Type *EltTy = VT->getElementType();
 
   for (unsigned i = 0; i != NumElts; ++i) {
     if (i) Out << ", ";
@@ -3557,9 +3563,9 @@ void CWriter::visitInsertValueInst(InsertValueInst &IVI) {
   Out << GetValueName(&IVI);
   for (const unsigned *b = IVI.idx_begin(), *i = b, *e = IVI.idx_end();
        i != e; ++i) {
-    const Type *IndexedTy =
+    Type *IndexedTy =
       ExtractValueInst::getIndexedType(IVI.getOperand(0)->getType(),
-                                       ArrayRef<unsigned>(b, i+1));
+                                       makeArrayRef(b, i+1));
     if (IndexedTy->isArrayTy())
       Out << ".array[" << *i << "]";
     else
@@ -3579,9 +3585,9 @@ void CWriter::visitExtractValueInst(ExtractValueInst &EVI) {
     Out << GetValueName(EVI.getOperand(0));
     for (const unsigned *b = EVI.idx_begin(), *i = b, *e = EVI.idx_end();
          i != e; ++i) {
-      const Type *IndexedTy =
+      Type *IndexedTy =
         ExtractValueInst::getIndexedType(EVI.getOperand(0)->getType(),
-                                         ArrayRef<unsigned>(b, i+1));
+                                         makeArrayRef(b, i+1));
       if (IndexedTy->isArrayTy())
         Out << ".array[" << *i << "]";
       else
diff --git a/contrib/llvm/lib/Target/CBackend/CTargetMachine.h b/contrib/llvm/lib/Target/CBackend/CTargetMachine.h
index e64216be0bdc..4f1ca974ded9 100644
--- a/contrib/llvm/lib/Target/CBackend/CTargetMachine.h
+++ b/contrib/llvm/lib/Target/CBackend/CTargetMachine.h
@@ -20,8 +20,9 @@
 namespace llvm {
 
 struct CTargetMachine : public TargetMachine {
-  CTargetMachine(const Target &T, const std::string &TT,
-                 const std::string &CPU, const std::string &FS)
+  CTargetMachine(const Target &T, StringRef TT,
+                 StringRef CPU, StringRef FS,
+                 Reloc::Model RM, CodeModel::Model CM)
     : TargetMachine(T, TT, CPU, FS) {}
 
   virtual bool addPassesToEmitFile(PassManagerBase &PM,
diff --git a/contrib/llvm/lib/Target/CBackend/TargetInfo/CBackendTargetInfo.cpp b/contrib/llvm/lib/Target/CBackend/TargetInfo/CBackendTargetInfo.cpp
index f7e8ff254848..e8274ff9ce5a 100644
--- a/contrib/llvm/lib/Target/CBackend/TargetInfo/CBackendTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/CBackend/TargetInfo/CBackendTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "CTargetMachine.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheCBackendTarget;
@@ -17,3 +17,5 @@ Target llvm::TheCBackendTarget;
 extern "C" void LLVMInitializeCBackendTargetInfo() { 
   RegisterTarget<> X(TheCBackendTarget, "c", "C backend");
 }
+
+extern "C" void LLVMInitializeCBackendTargetMC() {}
diff --git a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 85fb258eac2c..000000000000
--- a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMCellSPUDesc
-  SPUMCTargetDesc.cpp
-  SPUMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/Makefile
deleted file mode 100644
index 10d9a42239ad..000000000000
--- a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/CellSPU/TargetDesc/Makefile --------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMCellSPUDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.cpp b/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.cpp
index 26c5a4bc7b33..d5af2a88aed1 100644
--- a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.cpp
@@ -13,10 +13,12 @@
 
 #include "SPUMCTargetDesc.h"
 #include "SPUMCAsmInfo.h"
+#include "llvm/MC/MachineLocation.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "SPUGenInstrInfo.inc"
@@ -35,8 +37,10 @@ static MCInstrInfo *createSPUMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeCellSPUMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheCellSPUTarget, createSPUMCInstrInfo);
+static MCRegisterInfo *createCellSPUMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitSPUMCRegisterInfo(X, SPU::R0);
+  return X;
 }
 
 static MCSubtargetInfo *createSPUMCSubtargetInfo(StringRef TT, StringRef CPU,
@@ -46,11 +50,43 @@ static MCSubtargetInfo *createSPUMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeCellSPUMCSubtargetInfo() {
+static MCAsmInfo *createSPUMCAsmInfo(const Target &T, StringRef TT) {
+  MCAsmInfo *MAI = new SPULinuxMCAsmInfo(T, TT);
+
+  // Initial state of the frame pointer is R1.
+  MachineLocation Dst(MachineLocation::VirtualFP);
+  MachineLocation Src(SPU::R1, 0);
+  MAI->addInitialFrameState(0, Dst, Src);
+
+  return MAI;
+}
+
+static MCCodeGenInfo *createSPUMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                      CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  // For the time being, use static relocations, since there's really no
+  // support for PIC yet.
+  X->InitMCCodeGenInfo(Reloc::Static, CM);
+  return X;
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeCellSPUTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfoFn X(TheCellSPUTarget, createSPUMCAsmInfo);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheCellSPUTarget,
+                                        createSPUMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheCellSPUTarget, createSPUMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheCellSPUTarget,
+                                    createCellSPUMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(TheCellSPUTarget,
                                           createSPUMCSubtargetInfo);
 }
-
-extern "C" void LLVMInitializeCellSPUMCAsmInfo() {
-  RegisterMCAsmInfo<SPULinuxMCAsmInfo> X(TheCellSPUTarget);
-}
diff --git a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.h b/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.h
index c5c037d4de44..a3717b0bfc10 100644
--- a/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.h
+++ b/contrib/llvm/lib/Target/CellSPU/MCTargetDesc/SPUMCTargetDesc.h
@@ -1,4 +1,4 @@
-//===-- SPUMCTargetDesc.h - Alpha Target Descriptions ---------*- C++ -*-===//
+//===-- SPUMCTargetDesc.h - CellSPU Target Descriptions ---------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file provides Alpha specific target descriptions.
+// This file provides CellSPU specific target descriptions.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUAsmPrinter.cpp b/contrib/llvm/lib/Target/CellSPU/SPUAsmPrinter.cpp
index fd96694b32fe..90b5270a9dae 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPUAsmPrinter.cpp
@@ -29,10 +29,10 @@
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.cpp b/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.cpp
index a3e7e73ae30a..093f99f28711 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.cpp
@@ -181,18 +181,6 @@ void SPUFrameLowering::emitPrologue(MachineFunction &MF) const {
       MachineLocation FPSrc(MachineLocation::VirtualFP);
       Moves.push_back(MachineMove(ReadyLabel, FPDst, FPSrc));
     }
-  } else {
-    // This is a leaf function -- insert a branch hint iff there are
-    // sufficient number instructions in the basic block. Note that
-    // this is just a best guess based on the basic block's size.
-    if (MBB.size() >= (unsigned) SPUFrameLowering::branchHintPenalty()) {
-      MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-      dl = MBBI->getDebugLoc();
-
-      // Insert terminator label
-      BuildMI(MBB, MBBI, dl, TII.get(SPU::PROLOG_LABEL))
-        .addSym(MMI.getContext().CreateTempSymbol());
-    }
   }
 }
 
@@ -249,14 +237,6 @@ void SPUFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-void SPUFrameLowering::getInitialFrameState(std::vector<MachineMove> &Moves)
-                                                                         const {
-  // Initial state of the frame pointer is R1.
-  MachineLocation Dst(MachineLocation::VirtualFP);
-  MachineLocation Src(SPU::R1, 0);
-  Moves.push_back(MachineMove(0, Dst, Src));
-}
-
 void SPUFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                                         RegScavenger *RS) const{
   // Mark LR and SP unused, since the prolog spills them to stack and
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.h b/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.h
index 4fee72d946a2..b837f2cf94e1 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.h
+++ b/contrib/llvm/lib/Target/CellSPU/SPUFrameLowering.h
@@ -43,9 +43,6 @@ namespace llvm {
     void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                               RegScavenger *RS = NULL) const;
 
-    //! Perform target-specific stack frame setup.
-    void getInitialFrameState(std::vector<MachineMove> &Moves) const;
-
     //! Return a function's saved spill slots
     /*!
       For CellSPU, a function's saved spill slots is just the link register.
@@ -77,17 +74,6 @@ namespace llvm {
     static int FItoStackOffset(int frame_index) {
       return frame_index * stackSlotSize();
     }
-    //! Number of instructions required to overcome hint-for-branch latency
-    /*!
-      HBR (hint-for-branch) instructions can be inserted when, for example,
-      we know that a given function is going to be called, such as printf(),
-      in the control flow graph. HBRs are only inserted if a sufficient number
-      of instructions occurs between the HBR and the target. Currently, HBRs
-      take 6 cycles, ergo, the magic number 6.
-     */
-    static int branchHintPenalty() {
-      return 6;
-    }
   };
 }
 
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.cpp b/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.cpp
index f0ceee214149..ac33111f74ae 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.cpp
@@ -69,7 +69,7 @@ namespace {
     TargetLowering::ArgListEntry Entry;
     for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
       EVT ArgVT = Op.getOperand(i).getValueType();
-      const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+      Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
       Entry.Node = Op.getOperand(i);
       Entry.Ty = ArgTy;
       Entry.isSExt = isSigned;
@@ -80,7 +80,7 @@ namespace {
                                            TLI.getPointerTy());
 
     // Splice the libcall in wherever FindInputOutputChains tells us to.
-    const Type *RetTy =
+    Type *RetTy =
                 Op.getNode()->getValueType(0).getTypeForEVT(*DAG.getContext());
     std::pair<SDValue, SDValue> CallInfo =
             TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false,
@@ -174,6 +174,7 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
 
   // SPU has no intrinsics for these particular operations:
   setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
 
   // SPU has no division/remainder instructions
   setOperationAction(ISD::SREM,    MVT::i8,   Expand);
@@ -401,6 +402,9 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
        i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
     MVT::SimpleValueType VT = (MVT::SimpleValueType)i;
 
+    // Set operation actions to legal types only.
+    if (!isTypeLegal(VT)) continue;
+
     // add/sub are legal for all supported vector VT's.
     setOperationAction(ISD::ADD,     VT, Legal);
     setOperationAction(ISD::SUB,     VT, Legal);
@@ -438,6 +442,7 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
   setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
 
   setBooleanContents(ZeroOrNegativeOneBooleanContent);
+  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); // FIXME: Is this correct?
 
   setStackPointerRegisterToSaveRestore(SPU::R1);
 
@@ -497,7 +502,7 @@ SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
 // Return the Cell SPU's SETCC result type
 //===----------------------------------------------------------------------===//
 
-MVT::SimpleValueType SPUTargetLowering::getSetCCResultType(EVT VT) const {
+EVT SPUTargetLowering::getSetCCResultType(EVT VT) const {
   // i8, i16 and i32 are valid SETCC result types
   MVT::SimpleValueType retval;
 
@@ -2727,6 +2732,7 @@ static SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG)
   // the type to extend from needs to be i64 or i32.
   assert((OpVT == MVT::i128 && (Op0VT == MVT::i64 || Op0VT == MVT::i32)) &&
           "LowerSIGN_EXTEND: input and/or output operand have wrong size");
+  (void)OpVT;
 
   // Create shuffle mask
   unsigned mask1 = 0x10101010; // byte 0 - 3 and 4 - 7
@@ -3216,7 +3222,7 @@ SPUTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 /// isLegalAddressImmediate - Return true if the integer value can be used
 /// as the offset of the target addressing mode.
 bool SPUTargetLowering::isLegalAddressImmediate(int64_t V,
-                                                const Type *Ty) const {
+                                                Type *Ty) const {
   // SPU's addresses are 256K:
   return (V > -(1 << 18) && V < (1 << 18) - 1);
 }
@@ -3239,7 +3245,7 @@ bool SPUTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
 
 bool
 SPUTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                         const Type * ) const{
+                                         Type * ) const{
 
   // A-form: 18bit absolute address.
   if (AM.BaseGV && !AM.HasBaseReg && AM.Scale == 0 && AM.BaseOffs == 0)
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.h b/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.h
index d23f6cc48c71..aa4a1687278a 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.h
+++ b/contrib/llvm/lib/Target/CellSPU/SPUISelLowering.h
@@ -107,7 +107,7 @@ namespace llvm {
     virtual const char *getTargetNodeName(unsigned Opcode) const;
 
     /// getSetCCResultType - Return the ValueType for ISD::SETCC
-    virtual MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    virtual EVT getSetCCResultType(EVT VT) const;
 
     virtual MVT getShiftAmountTy(EVT LHSTy) const { return MVT::i32; }
 
@@ -147,7 +147,7 @@ namespace llvm {
 
     /// isLegalAddressImmediate - Return true if the integer value can be used
     /// as the offset of the target addressing mode.
-    virtual bool isLegalAddressImmediate(int64_t V, const Type *Ty) const;
+    virtual bool isLegalAddressImmediate(int64_t V, Type *Ty) const;
     virtual bool isLegalAddressImmediate(GlobalValue *) const;
 
     virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const;
@@ -179,7 +179,7 @@ namespace llvm {
     virtual bool isLegalICmpImmediate(int64_t Imm) const;
 
     virtual bool isLegalAddressingMode(const AddrMode &AM,
-                                       const Type *Ty) const;
+                                       Type *Ty) const;
   };
 }
 
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.cpp b/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.cpp
index e67b10c7984d..007bc0e02c7e 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.cpp
@@ -17,9 +17,9 @@
 #include "SPUHazardRecognizers.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define GET_INSTRINFO_CTOR
@@ -290,6 +290,8 @@ static void removeHBR( MachineBasicBlock &MBB) {
     if (I->getOpcode() == SPU::HBRA ||
         I->getOpcode() == SPU::HBR_LABEL){
       I=MBB.erase(I);
+      if (I == MBB.end())
+        break;
     }
   }
 }
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.td b/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.td
index e103c9b6a5af..f76ebd75bfef 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.td
+++ b/contrib/llvm/lib/Target/CellSPU/SPUInstrInfo.td
@@ -1594,8 +1594,8 @@ multiclass BitwiseOrImm
 {
   def v4i32: ORIVecInst<v4i32, v4i32Uns10Imm>;
 
-  def r32: ORIInst<(outs R32C:$rT), (ins R32C:$rA, u10imm_i32:$val),
-                   [(set R32C:$rT, (or R32C:$rA, i32ImmUns10:$val))]>;
+  def r32: ORIInst<(outs R32C:$rT), (ins R32C:$rA, s10imm_i32:$val),
+                   [(set R32C:$rT, (or R32C:$rA, i32ImmSExt10:$val))]>;
 
   // i16i32: hacked version of the ori instruction to extend 16-bit quantities
   // to 32-bit quantities. used exclusively to match "anyext" conversions (vide
@@ -3467,8 +3467,10 @@ let isBranch = 1, isTerminator = 1, hasCtrlDep = 1 in {
         [/* no pattern */]>;
 
     // Indirect branch
-    def BI:
-      BIForm<0b00010101100, "bi\t$func", [(brind R32C:$func)]>;
+    let isIndirectBranch = 1 in {
+      def BI:
+        BIForm<0b00010101100, "bi\t$func", [(brind R32C:$func)]>;
+    }
   }
 
   // Conditional branches:
diff --git a/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.cpp b/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.cpp
index 19896c0b4be9..bbac6fd0be54 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.cpp
@@ -25,7 +25,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -187,7 +186,7 @@ unsigned SPURegisterInfo::getRegisterNumbering(unsigned RegEnum) {
 
 SPURegisterInfo::SPURegisterInfo(const SPUSubtarget &subtarget,
                                  const TargetInstrInfo &tii) :
-  SPUGenRegisterInfo(), Subtarget(subtarget), TII(tii)
+  SPUGenRegisterInfo(SPU::R0), Subtarget(subtarget), TII(tii)
 {
 }
 
@@ -310,28 +309,12 @@ SPURegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
   }
 }
 
-unsigned
-SPURegisterInfo::getRARegister() const
-{
-  return SPU::R0;
-}
-
 unsigned
 SPURegisterInfo::getFrameRegister(const MachineFunction &MF) const
 {
   return SPU::R1;
 }
 
-int
-SPURegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  // FIXME: Most probably dwarf numbers differs for Linux and Darwin
-  return SPUGenRegisterInfo::getDwarfRegNumFull(RegNum, 0);
-}
-
-int SPURegisterInfo::getLLVMRegNum(unsigned RegNum, bool isEH) const {
-  return SPUGenRegisterInfo::getLLVMRegNumFull(RegNum, 0);
-}
-
 int
 SPURegisterInfo::convertDFormToXForm(int dFormOpcode) const
 {
diff --git a/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.h b/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.h
index 5e014f8adbfc..b7818a47abd7 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.h
+++ b/contrib/llvm/lib/Target/CellSPU/SPURegisterInfo.h
@@ -74,8 +74,6 @@ namespace llvm {
     void eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
                              RegScavenger *RS = NULL) const;
 
-    //! Get return address register (LR, aka R0)
-    unsigned getRARegister() const;
     //! Get the stack frame register (SP, aka R1)
     unsigned getFrameRegister(const MachineFunction &MF) const;
 
@@ -83,10 +81,6 @@ namespace llvm {
     // New methods added:
     //------------------------------------------------------------------------
 
-    //! Get DWARF debugging register number
-    int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-    int getLLVMRegNum(unsigned RegNum, bool isEH) const;
-
     //! Convert D-form load/store to X-form load/store
     /*!
       Converts a regiser displacement load/store into a register-indexed
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUSubtarget.cpp b/contrib/llvm/lib/Target/CellSPU/SPUSubtarget.cpp
index 856dc82f786b..43335abf0ac2 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUSubtarget.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPUSubtarget.cpp
@@ -14,7 +14,7 @@
 #include "SPUSubtarget.h"
 #include "SPU.h"
 #include "SPURegisterInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/ADT/SmallVector.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.cpp b/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.cpp
index 3542a2b87e43..93a7f6e36501 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.cpp
@@ -16,7 +16,8 @@
 #include "llvm/PassManager.h"
 #include "llvm/CodeGen/RegAllocRegistry.h"
 #include "llvm/CodeGen/SchedulerRegistry.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
 
@@ -31,9 +32,10 @@ SPUFrameLowering::getCalleeSaveSpillSlots(unsigned &NumEntries) const {
   return &LR[0];
 }
 
-SPUTargetMachine::SPUTargetMachine(const Target &T, const std::string &TT,
-                                   const std::string &CPU,const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+SPUTargetMachine::SPUTargetMachine(const Target &T, StringRef TT,
+                                   StringRef CPU, StringRef FS,
+                                   Reloc::Model RM, CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS),
     DataLayout(Subtarget.getTargetDataString()),
     InstrInfo(*this),
@@ -41,9 +43,6 @@ SPUTargetMachine::SPUTargetMachine(const Target &T, const std::string &TT,
     TLInfo(*this),
     TSInfo(*this),
     InstrItins(Subtarget.getInstrItineraryData()) {
-  // For the time being, use static relocations, since there's really no
-  // support for PIC yet.
-  setRelocationModel(Reloc::Static);
 }
 
 //===----------------------------------------------------------------------===//
@@ -59,8 +58,16 @@ bool SPUTargetMachine::addInstSelector(PassManagerBase &PM,
 
 // passes to run just before printing the assembly
 bool SPUTargetMachine::
-addPreEmitPass(PassManagerBase &PM, CodeGenOpt::Level OptLevel) 
-{
+addPreEmitPass(PassManagerBase &PM, CodeGenOpt::Level OptLevel) {
+  // load the TCE instruction scheduler, if available via
+  // loaded plugins
+  typedef llvm::FunctionPass* (*BuilderFunc)(const char*);
+  BuilderFunc schedulerCreator =
+    (BuilderFunc)(intptr_t)sys::DynamicLibrary::SearchForAddressOfSymbol(
+          "createTCESchedulerPass");
+  if (schedulerCreator != NULL)
+      PM.add(schedulerCreator("cellspu"));
+
   //align instructions with nops/lnops for dual issue
   PM.add(createSPUNopFillerPass(*this));
   return true;
diff --git a/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.h b/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.h
index d96f86dcaeb0..fffe77cabba3 100644
--- a/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.h
+++ b/contrib/llvm/lib/Target/CellSPU/SPUTargetMachine.h
@@ -38,8 +38,9 @@ class SPUTargetMachine : public LLVMTargetMachine {
   SPUSelectionDAGInfo TSInfo;
   InstrItineraryData  InstrItins;
 public:
-  SPUTargetMachine(const Target &T, const std::string &TT,
-                   const std::string &CPU, const std::string &FS);
+  SPUTargetMachine(const Target &T, StringRef TT,
+                   StringRef CPU, StringRef FS,
+                   Reloc::Model RM, CodeModel::Model CM);
 
   /// Return the subtarget implementation object
   virtual const SPUSubtarget     *getSubtargetImpl() const {
diff --git a/contrib/llvm/lib/Target/CellSPU/TargetInfo/CellSPUTargetInfo.cpp b/contrib/llvm/lib/Target/CellSPU/TargetInfo/CellSPUTargetInfo.cpp
index 049ea236e992..84aadfad6f8d 100644
--- a/contrib/llvm/lib/Target/CellSPU/TargetInfo/CellSPUTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/CellSPU/TargetInfo/CellSPUTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "SPU.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheCellSPUTarget;
diff --git a/contrib/llvm/lib/Target/CppBackend/CPPBackend.cpp b/contrib/llvm/lib/Target/CppBackend/CPPBackend.cpp
index 10d18f61c7e2..ae0e3c404b76 100644
--- a/contrib/llvm/lib/Target/CppBackend/CPPBackend.cpp
+++ b/contrib/llvm/lib/Target/CppBackend/CPPBackend.cpp
@@ -29,7 +29,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormattedStream.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Config/config.h"
 #include <algorithm>
@@ -77,22 +77,12 @@ extern "C" void LLVMInitializeCppBackendTarget() {
   RegisterTargetMachine<CPPTargetMachine> X(TheCppBackendTarget);
 }
 
-extern "C" void LLVMInitializeCppBackendMCAsmInfo() {}
-
-extern "C" void LLVMInitializeCppBackendMCInstrInfo() {
-  RegisterMCInstrInfo<MCInstrInfo> X(TheCppBackendTarget);
-}
-
-extern "C" void LLVMInitializeCppBackendMCSubtargetInfo() {
-  RegisterMCSubtargetInfo<MCSubtargetInfo> X(TheCppBackendTarget);
-}
-
 namespace {
-  typedef std::vector<const Type*> TypeList;
-  typedef std::map<const Type*,std::string> TypeMap;
+  typedef std::vector<Type*> TypeList;
+  typedef std::map<Type*,std::string> TypeMap;
   typedef std::map<const Value*,std::string> ValueMap;
   typedef std::set<std::string> NameSet;
-  typedef std::set<const Type*> TypeSet;
+  typedef std::set<Type*> TypeSet;
   typedef std::set<const Value*> ValueSet;
   typedef std::map<const Value*,std::string> ForwardRefMap;
 
@@ -143,14 +133,14 @@ namespace {
     void printEscapedString(const std::string& str);
     void printCFP(const ConstantFP* CFP);
 
-    std::string getCppName(const Type* val);
-    inline void printCppName(const Type* val);
+    std::string getCppName(Type* val);
+    inline void printCppName(Type* val);
 
     std::string getCppName(const Value* val);
     inline void printCppName(const Value* val);
 
     void printAttributes(const AttrListPtr &PAL, const std::string &name);
-    void printType(const Type* Ty);
+    void printType(Type* Ty);
     void printTypes(const Module* M);
 
     void printConstant(const Constant *CPV);
@@ -164,7 +154,7 @@ namespace {
     void printFunctionHead(const Function *F);
     void printFunctionBody(const Function *F);
     void printInstruction(const Instruction *I, const std::string& bbname);
-    std::string getOpName(Value*);
+    std::string getOpName(const Value*);
 
     void printModuleBody();
   };
@@ -184,7 +174,7 @@ static inline void sanitize(std::string &str) {
       str[i] = '_';
 }
 
-static std::string getTypePrefix(const Type *Ty) {
+static std::string getTypePrefix(Type *Ty) {
   switch (Ty->getTypeID()) {
   case Type::VoidTyID:     return "void_";
   case Type::IntegerTyID:
@@ -339,7 +329,7 @@ void CppWriter::printEscapedString(const std::string &Str) {
   }
 }
 
-std::string CppWriter::getCppName(const Type* Ty) {
+std::string CppWriter::getCppName(Type* Ty) {
   // First, handle the primitive types .. easy
   if (Ty->isPrimitiveType() || Ty->isIntegerTy()) {
     switch (Ty->getTypeID()) {
@@ -379,7 +369,7 @@ std::string CppWriter::getCppName(const Type* Ty) {
 
   // See if the type has a name in the symboltable and build accordingly
   std::string name;
-  if (const StructType *STy = dyn_cast<StructType>(Ty))
+  if (StructType *STy = dyn_cast<StructType>(Ty))
     if (STy->hasName())
       name = STy->getName();
   
@@ -393,7 +383,7 @@ std::string CppWriter::getCppName(const Type* Ty) {
   return TypeNames[Ty] = name;
 }
 
-void CppWriter::printCppName(const Type* Ty) {
+void CppWriter::printCppName(Type* Ty) {
   printEscapedString(getCppName(Ty));
 }
 
@@ -480,6 +470,9 @@ void CppWriter::printAttributes(const AttrListPtr &PAL,
       HANDLE_ATTR(NoImplicitFloat);
       HANDLE_ATTR(Naked);
       HANDLE_ATTR(InlineHint);
+      HANDLE_ATTR(ReturnsTwice);
+      HANDLE_ATTR(UWTable);
+      HANDLE_ATTR(NonLazyBind);
 #undef HANDLE_ATTR
       if (attrs & Attribute::StackAlignment)
         Out << " | Attribute::constructStackAlignmentFromInt("
@@ -499,7 +492,7 @@ void CppWriter::printAttributes(const AttrListPtr &PAL,
   }
 }
 
-void CppWriter::printType(const Type* Ty) {
+void CppWriter::printType(Type* Ty) {
   // We don't print definitions for primitive types
   if (Ty->isPrimitiveType() || Ty->isIntegerTy())
     return;
@@ -514,13 +507,13 @@ void CppWriter::printType(const Type* Ty) {
   // Print the type definition
   switch (Ty->getTypeID()) {
   case Type::FunctionTyID:  {
-    const FunctionType* FT = cast<FunctionType>(Ty);
+    FunctionType* FT = cast<FunctionType>(Ty);
     Out << "std::vector<Type*>" << typeName << "_args;";
     nl(Out);
     FunctionType::param_iterator PI = FT->param_begin();
     FunctionType::param_iterator PE = FT->param_end();
     for (; PI != PE; ++PI) {
-      const Type* argTy = static_cast<const Type*>(*PI);
+      Type* argTy = static_cast<Type*>(*PI);
       printType(argTy);
       std::string argName(getCppName(argTy));
       Out << typeName << "_args.push_back(" << argName;
@@ -539,13 +532,21 @@ void CppWriter::printType(const Type* Ty) {
     break;
   }
   case Type::StructTyID: {
-    const StructType* ST = cast<StructType>(Ty);
-    if (!ST->isAnonymous()) {
-      Out << "StructType *" << typeName << " = ";
-      Out << "StructType::createNamed(mod->getContext(), \"";
+    StructType* ST = cast<StructType>(Ty);
+    if (!ST->isLiteral()) {
+      Out << "StructType *" << typeName << " = mod->getTypeByName(\"";
       printEscapedString(ST->getName());
       Out << "\");";
       nl(Out);
+      Out << "if (!" << typeName << ") {";
+      nl(Out);
+      Out << typeName << " = ";
+      Out << "StructType::create(mod->getContext(), \"";
+      printEscapedString(ST->getName());
+      Out << "\");";
+      nl(Out);
+      Out << "}";
+      nl(Out);
       // Indicate that this type is now defined.
       DefinedTypes.insert(Ty);
     }
@@ -555,7 +556,7 @@ void CppWriter::printType(const Type* Ty) {
     StructType::element_iterator EI = ST->element_begin();
     StructType::element_iterator EE = ST->element_end();
     for (; EI != EE; ++EI) {
-      const Type* fieldTy = static_cast<const Type*>(*EI);
+      Type* fieldTy = static_cast<Type*>(*EI);
       printType(fieldTy);
       std::string fieldName(getCppName(fieldTy));
       Out << typeName << "_fields.push_back(" << fieldName;
@@ -563,21 +564,27 @@ void CppWriter::printType(const Type* Ty) {
       nl(Out);
     }
 
-    if (ST->isAnonymous()) {
+    if (ST->isLiteral()) {
       Out << "StructType *" << typeName << " = ";
       Out << "StructType::get(" << "mod->getContext(), ";
     } else {
+      Out << "if (" << typeName << "->isOpaque()) {";
+      nl(Out);
       Out << typeName << "->setBody(";
     }
 
     Out << typeName << "_fields, /*isPacked=*/"
         << (ST->isPacked() ? "true" : "false") << ");";
     nl(Out);
+    if (!ST->isLiteral()) {
+      Out << "}";
+      nl(Out);
+    }
     break;
   }
   case Type::ArrayTyID: {
-    const ArrayType* AT = cast<ArrayType>(Ty);
-    const Type* ET = AT->getElementType();
+    ArrayType* AT = cast<ArrayType>(Ty);
+    Type* ET = AT->getElementType();
     printType(ET);
     if (DefinedTypes.find(Ty) == DefinedTypes.end()) {
       std::string elemName(getCppName(ET));
@@ -589,8 +596,8 @@ void CppWriter::printType(const Type* Ty) {
     break;
   }
   case Type::PointerTyID: {
-    const PointerType* PT = cast<PointerType>(Ty);
-    const Type* ET = PT->getElementType();
+    PointerType* PT = cast<PointerType>(Ty);
+    Type* ET = PT->getElementType();
     printType(ET);
     if (DefinedTypes.find(Ty) == DefinedTypes.end()) {
       std::string elemName(getCppName(ET));
@@ -602,8 +609,8 @@ void CppWriter::printType(const Type* Ty) {
     break;
   }
   case Type::VectorTyID: {
-    const VectorType* PT = cast<VectorType>(Ty);
-    const Type* ET = PT->getElementType();
+    VectorType* PT = cast<VectorType>(Ty);
+    Type* ET = PT->getElementType();
     printType(ET);
     if (DefinedTypes.find(Ty) == DefinedTypes.end()) {
       std::string elemName(getCppName(ET));
@@ -766,9 +773,7 @@ void CppWriter::printConstant(const Constant *CV) {
       Out << "Constant* " << constName
           << " = ConstantExpr::getGetElementPtr("
           << getCppName(CE->getOperand(0)) << ", "
-          << "&" << constName << "_indices[0], "
-          << constName << "_indices.size()"
-          << ");";
+          << constName << "_indices);";
     } else if (CE->isCast()) {
       printConstant(CE->getOperand(0));
       Out << "Constant* " << constName << " = ConstantExpr::getCast(";
@@ -988,7 +993,7 @@ void CppWriter::printVariableBody(const GlobalVariable *GV) {
   }
 }
 
-std::string CppWriter::getOpName(Value* V) {
+std::string CppWriter::getOpName(const Value* V) {
   if (!isa<Instruction>(V) || DefinedValues.find(V) != DefinedValues.end())
     return getCppName(V);
 
@@ -1053,14 +1058,17 @@ void CppWriter::printInstruction(const Instruction *I,
   case Instruction::Switch: {
     const SwitchInst *SI = cast<SwitchInst>(I);
     Out << "SwitchInst* " << iName << " = SwitchInst::Create("
-        << opNames[0] << ", "
-        << opNames[1] << ", "
+        << getOpName(SI->getCondition()) << ", "
+        << getOpName(SI->getDefaultDest()) << ", "
         << SI->getNumCases() << ", " << bbname << ");";
     nl(Out);
-    for (unsigned i = 2; i != SI->getNumOperands(); i += 2) {
+    unsigned NumCases = SI->getNumCases();
+    for (unsigned i = 1; i < NumCases; ++i) {
+      const ConstantInt* CaseVal = SI->getCaseValue(i);
+      const BasicBlock* BB = SI->getSuccessor(i);
       Out << iName << "->addCase("
-          << opNames[i] << ", "
-          << opNames[i+1] << ");";
+          << getOpName(CaseVal) << ", "
+          << getOpName(BB) << ");";
       nl(Out);
     }
     break;
@@ -1076,6 +1084,11 @@ void CppWriter::printInstruction(const Instruction *I,
     }
     break;
   }
+  case Instruction::Resume: {
+    Out << "ResumeInst::Create(mod->getContext(), " << opNames[0]
+        << ", " << bbname << ");";
+    break;
+  }
   case Instruction::Invoke: {
     const InvokeInst* inv = cast<InvokeInst>(I);
     Out << "std::vector<Value*> " << iName << "_params;";
@@ -1090,8 +1103,7 @@ void CppWriter::printInstruction(const Instruction *I,
         << getOpName(inv->getCalledFunction()) << ", "
         << getOpName(inv->getNormalDest()) << ", "
         << getOpName(inv->getUnwindDest()) << ", "
-        << iName << "_params.begin(), "
-        << iName << "_params.end(), \"";
+        << iName << "_params, \"";
     printEscapedString(inv->getName());
     Out << "\", " << bbname << ");";
     nl(Out) << iName << "->setCallingConv(";
@@ -1252,8 +1264,7 @@ void CppWriter::printInstruction(const Instruction *I,
         nl(Out);
       }
       Out << "Instruction* " << iName << " = GetElementPtrInst::Create("
-          << opNames[0] << ", " << iName << "_indices.begin(), "
-          << iName << "_indices.end()";
+          << opNames[0] << ", " << iName << "_indices";
     }
     Out << ", \"";
     printEscapedString(gep->getName());
@@ -1304,7 +1315,7 @@ void CppWriter::printInstruction(const Instruction *I,
     case Instruction::PtrToInt: Out << "PtrToIntInst"; break;
     case Instruction::IntToPtr: Out << "IntToPtrInst"; break;
     case Instruction::BitCast:  Out << "BitCastInst"; break;
-    default: assert(!"Unreachable"); break;
+    default: assert(0 && "Unreachable"); break;
     }
     Out << "(" << opNames[0] << ", "
         << getCppName(cst->getType()) << ", \"";
@@ -1331,8 +1342,7 @@ void CppWriter::printInstruction(const Instruction *I,
       }
       Out << "CallInst* " << iName << " = CallInst::Create("
           << opNames[call->getNumArgOperands()] << ", "
-          << iName << "_params.begin(), "
-          << iName << "_params.end(), \"";
+          << iName << "_params, \"";
     } else if (call->getNumArgOperands() == 1) {
       Out << "CallInst* " << iName << " = CallInst::Create("
           << opNames[call->getNumArgOperands()] << ", " << opNames[0] << ", \"";
@@ -1415,7 +1425,7 @@ void CppWriter::printInstruction(const Instruction *I,
     Out << "ExtractValueInst* " << getCppName(evi)
         << " = ExtractValueInst::Create(" << opNames[0]
         << ", "
-        << iName << "_indices.begin(), " << iName << "_indices.end(), \"";
+        << iName << "_indices, \"";
     printEscapedString(evi->getName());
     Out << "\", " << bbname << ");";
     break;
@@ -1432,7 +1442,7 @@ void CppWriter::printInstruction(const Instruction *I,
     Out << "InsertValueInst* " << getCppName(ivi)
         << " = InsertValueInst::Create(" << opNames[0]
         << ", " << opNames[1] << ", "
-        << iName << "_indices.begin(), " << iName << "_indices.end(), \"";
+        << iName << "_indices, \"";
     printEscapedString(ivi->getName());
     Out << "\", " << bbname << ");";
     break;
@@ -1542,13 +1552,12 @@ void CppWriter::printFunctionUses(const Function* F) {
 
 void CppWriter::printFunctionHead(const Function* F) {
   nl(Out) << "Function* " << getCppName(F);
-  if (is_inline) {
-    Out << " = mod->getFunction(\"";
-    printEscapedString(F->getName());
-    Out << "\", " << getCppName(F->getFunctionType()) << ");";
-    nl(Out) << "if (!" << getCppName(F) << ") {";
-    nl(Out) << getCppName(F);
-  }
+  Out << " = mod->getFunction(\"";
+  printEscapedString(F->getName());
+  Out << "\");";
+  nl(Out) << "if (!" << getCppName(F) << ") {";
+  nl(Out) << getCppName(F);
+
   Out<< " = Function::Create(";
   nl(Out,1) << "/*Type=*/" << getCppName(F->getFunctionType()) << ",";
   nl(Out) << "/*Linkage=*/";
@@ -1585,10 +1594,8 @@ void CppWriter::printFunctionHead(const Function* F) {
     Out << "->setGC(\"" << F->getGC() << "\");";
     nl(Out);
   }
-  if (is_inline) {
-    Out << "}";
-    nl(Out);
-  }
+  Out << "}";
+  nl(Out);
   printAttributes(F->getAttributes(), getCppName(F));
   printCppName(F);
   Out << "->setAttributes(" << getCppName(F) << "_PAL);";
@@ -1873,7 +1880,7 @@ void CppWriter::printVariable(const std::string& fname,
 
 void CppWriter::printType(const std::string &fname,
                           const std::string &typeName) {
-  const Type* Ty = TheModule->getTypeByName(typeName);
+  Type* Ty = TheModule->getTypeByName(typeName);
   if (!Ty) {
     error(std::string("Type '") + typeName + "' not found in input module");
     return;
diff --git a/contrib/llvm/lib/Target/CppBackend/CPPTargetMachine.h b/contrib/llvm/lib/Target/CppBackend/CPPTargetMachine.h
index 7322e3e34f00..287e53727139 100644
--- a/contrib/llvm/lib/Target/CppBackend/CPPTargetMachine.h
+++ b/contrib/llvm/lib/Target/CppBackend/CPPTargetMachine.h
@@ -22,8 +22,9 @@ namespace llvm {
 class formatted_raw_ostream;
 
 struct CPPTargetMachine : public TargetMachine {
-  CPPTargetMachine(const Target &T, const std::string &TT,
-                   const std::string &CPU, const std::string &FS)
+  CPPTargetMachine(const Target &T, StringRef TT,
+                   StringRef CPU, StringRef FS,
+                   Reloc::Model RM, CodeModel::Model CM)
     : TargetMachine(T, TT, CPU, FS) {}
 
   virtual bool addPassesToEmitFile(PassManagerBase &PM,
diff --git a/contrib/llvm/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp b/contrib/llvm/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp
index d0aeb12499c5..a8ac0a282cd1 100644
--- a/contrib/llvm/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "CPPTargetMachine.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheCppBackendTarget;
@@ -24,3 +24,5 @@ extern "C" void LLVMInitializeCppBackendTargetInfo() {
                                   "C++ backend",
                                   &CppBackend_TripleMatchQuality);
 }
+
+extern "C" void LLVMInitializeCppBackendTargetMC() {}
diff --git a/contrib/llvm/lib/Target/MBlaze/AsmParser/CMakeLists.txt b/contrib/llvm/lib/Target/MBlaze/AsmParser/CMakeLists.txt
deleted file mode 100644
index 87e7cb5da561..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/AsmParser/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. 
-                     ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMMBlazeAsmParser
-  MBlazeAsmLexer.cpp
-  MBlazeAsmParser.cpp
-  )
-
diff --git a/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmLexer.cpp b/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmLexer.cpp
index 15965964452a..2d357bb9674d 100644
--- a/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmLexer.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmLexer.cpp
@@ -7,8 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MBlaze.h"
-#include "MBlazeTargetMachine.h"
+#include "MCTargetDesc/MBlazeBaseInfo.h"
 
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
@@ -17,10 +16,10 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCTargetAsmLexer.h"
 
-#include "llvm/Target/TargetAsmLexer.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #include <string>
 #include <map>
@@ -29,7 +28,7 @@ using namespace llvm;
 
 namespace {
   
-  class MBlazeBaseAsmLexer : public TargetAsmLexer {
+  class MBlazeBaseAsmLexer : public MCTargetAsmLexer {
     const MCAsmInfo &AsmInfo;
     
     const AsmToken &lexDefinite() {
@@ -42,7 +41,7 @@ namespace {
     
     rmap_ty RegisterMap;
     
-    void InitRegisterMap(const TargetRegisterInfo *info) {
+    void InitRegisterMap(const MCRegisterInfo *info) {
       unsigned numRegs = info->getNumRegs();
 
       for (unsigned i = 0; i < numRegs; ++i) {
@@ -76,20 +75,16 @@ namespace {
     }
   public:
     MBlazeBaseAsmLexer(const Target &T, const MCAsmInfo &MAI)
-      : TargetAsmLexer(T), AsmInfo(MAI) {
+      : MCTargetAsmLexer(T), AsmInfo(MAI) {
     }
   };
   
   class MBlazeAsmLexer : public MBlazeBaseAsmLexer {
   public:
-    MBlazeAsmLexer(const Target &T, const MCAsmInfo &MAI)
+    MBlazeAsmLexer(const Target &T, const MCRegisterInfo &MRI,
+                   const MCAsmInfo &MAI)
       : MBlazeBaseAsmLexer(T, MAI) {
-      std::string tripleString("mblaze-unknown-unknown");
-      std::string featureString;
-      std::string CPU;
-      OwningPtr<const TargetMachine> 
-        targetMachine(T.createTargetMachine(tripleString, CPU, featureString));
-      InitRegisterMap(targetMachine->getRegisterInfo());
+      InitRegisterMap(&MRI);
     }
   };
 }
@@ -123,6 +118,6 @@ AsmToken MBlazeBaseAsmLexer::LexTokenUAL() {
 }
 
 extern "C" void LLVMInitializeMBlazeAsmLexer() {
-  RegisterAsmLexer<MBlazeAsmLexer> X(TheMBlazeTarget);
+  RegisterMCAsmLexer<MBlazeAsmLexer> X(TheMBlazeTarget);
 }
 
diff --git a/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp b/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp
index eebd9d878943..97d311c15107 100644
--- a/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp
@@ -7,19 +7,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MBlaze.h"
-#include "MBlazeSubtarget.h"
-#include "MBlazeRegisterInfo.h"
-#include "MBlazeISelLowering.h"
+#include "MCTargetDesc/MBlazeBaseInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmParser.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
@@ -30,7 +27,7 @@ using namespace llvm;
 namespace {
 struct MBlazeOperand;
 
-class MBlazeAsmParser : public TargetAsmParser {
+class MBlazeAsmParser : public MCTargetAsmParser {
   MCAsmParser &Parser;
 
   MCAsmParser &getParser() const { return Parser; }
@@ -64,7 +61,7 @@ class MBlazeAsmParser : public TargetAsmParser {
 
 public:
   MBlazeAsmParser(MCSubtargetInfo &_STI, MCAsmParser &_Parser)
-    : TargetAsmParser(), Parser(_Parser) {}
+    : MCTargetAsmParser(), Parser(_Parser) {}
 
   virtual bool ParseInstruction(StringRef Name, SMLoc NameLoc,
                                 SmallVectorImpl<MCParsedAsmOperand*> &Operands);
@@ -286,19 +283,19 @@ void MBlazeOperand::print(raw_ostream &OS) const {
     break;
   case Register:
     OS << "<register R";
-    OS << MBlazeRegisterInfo::getRegisterNumbering(getReg()) << ">";
+    OS << getMBlazeRegisterNumbering(getReg()) << ">";
     break;
   case Token:
     OS << "'" << getToken() << "'";
     break;
   case Memory: {
     OS << "<memory R";
-    OS << MBlazeRegisterInfo::getRegisterNumbering(getMemBase());
+    OS << getMBlazeRegisterNumbering(getMemBase());
     OS << ", ";
 
     unsigned RegOff = getMemOffReg();
     if (RegOff)
-      OS << "R" << MBlazeRegisterInfo::getRegisterNumbering(RegOff);
+      OS << "R" << getMBlazeRegisterNumbering(RegOff);
     else
       OS << getMemOff();
     OS << ">";
@@ -326,6 +323,7 @@ MatchAndEmitInstruction(SMLoc IDLoc,
   unsigned ErrorInfo;
 
   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo)) {
+  default: break;
   case Match_Success:
     Out.EmitInstruction(Inst);
     return false;
@@ -521,7 +519,7 @@ ParseInstruction(StringRef Name, SMLoc NameLoc,
   return false;
 }
 
-/// ParseDirective parses the arm specific directives
+/// ParseDirective parses the MBlaze specific directives
 bool MBlazeAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getIdentifier();
   if (IDVal == ".word")
@@ -558,7 +556,7 @@ extern "C" void LLVMInitializeMBlazeAsmLexer();
 
 /// Force static initialization.
 extern "C" void LLVMInitializeMBlazeAsmParser() {
-  RegisterAsmParser<MBlazeAsmParser> X(TheMBlazeTarget);
+  RegisterMCAsmParser<MBlazeAsmParser> X(TheMBlazeTarget);
   LLVMInitializeMBlazeAsmLexer();
 }
 
diff --git a/contrib/llvm/lib/Target/MBlaze/AsmParser/Makefile b/contrib/llvm/lib/Target/MBlaze/AsmParser/Makefile
deleted file mode 100644
index 611a0f473f73..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/AsmParser/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/ARM/AsmParser/Makefile -------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMBlazeAsmParser
-
-# Hack: we need to include 'main' MBlaze target directory for private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/MBlaze/Disassembler/CMakeLists.txt b/contrib/llvm/lib/Target/MBlaze/Disassembler/CMakeLists.txt
deleted file mode 100644
index 9376e68a35cf..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/Disassembler/CMakeLists.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. 
-                     ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMMBlazeDisassembler
-  MBlazeDisassembler.cpp
-  )
-
-# workaround for hanging compilation on MSVC9 and 10
-if( MSVC_VERSION EQUAL 1500 OR MSVC_VERSION EQUAL 1600 )
-set_property(
-  SOURCE MBlazeDisassembler.cpp
-  PROPERTY COMPILE_FLAGS "/Od"
-  )
-endif()
-
-add_dependencies(LLVMMBlazeDisassembler MBlazeCodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.cpp b/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.cpp
index 88d80a12eb3a..fd761f1ca8c1 100644
--- a/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.cpp
@@ -20,9 +20,9 @@
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MemoryObject.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
 // #include "MBlazeGenDecoderTables.inc"
@@ -60,27 +60,27 @@ static unsigned mblazeBinary2Opcode[] = {
 };
 
 static unsigned getRD(uint32_t insn) {
-  if (!MBlazeRegisterInfo::isRegister((insn>>21)&0x1F))
+  if (!isMBlazeRegister((insn>>21)&0x1F))
     return UNSUPPORTED;
-  return MBlazeRegisterInfo::getRegisterFromNumbering((insn>>21)&0x1F);
+  return getMBlazeRegisterFromNumbering((insn>>21)&0x1F);
 }
 
 static unsigned getRA(uint32_t insn) {
-  if (!MBlazeRegisterInfo::getRegisterFromNumbering((insn>>16)&0x1F))
+  if (!getMBlazeRegisterFromNumbering((insn>>16)&0x1F))
     return UNSUPPORTED;
-  return MBlazeRegisterInfo::getRegisterFromNumbering((insn>>16)&0x1F);
+  return getMBlazeRegisterFromNumbering((insn>>16)&0x1F);
 }
 
 static unsigned getRB(uint32_t insn) {
-  if (!MBlazeRegisterInfo::getRegisterFromNumbering((insn>>11)&0x1F))
+  if (!getMBlazeRegisterFromNumbering((insn>>11)&0x1F))
     return UNSUPPORTED;
-  return MBlazeRegisterInfo::getRegisterFromNumbering((insn>>11)&0x1F);
+  return getMBlazeRegisterFromNumbering((insn>>11)&0x1F);
 }
 
 static int64_t getRS(uint32_t insn) {
-  if (!MBlazeRegisterInfo::isSpecialRegister(insn&0x3FFF))
+  if (!isSpecialMBlazeRegister(insn&0x3FFF))
     return UNSUPPORTED;
-  return MBlazeRegisterInfo::getSpecialRegisterFromNumbering(insn&0x3FFF);
+  return getSpecialMBlazeRegisterFromNumbering(insn&0x3FFF);
 }
 
 static int64_t getIMM(uint32_t insn) {
@@ -493,11 +493,12 @@ EDInstInfo *MBlazeDisassembler::getEDInfo() const {
 // Public interface for the disassembler
 //
 
-bool MBlazeDisassembler::getInstruction(MCInst &instr,
+MCDisassembler::DecodeStatus MBlazeDisassembler::getInstruction(MCInst &instr,
                                         uint64_t &size,
                                         const MemoryObject &region,
                                         uint64_t address,
-                                        raw_ostream &vStream) const {
+                                        raw_ostream &vStream,
+                                        raw_ostream &cStream) const {
   // The machine instruction.
   uint32_t insn;
   uint64_t read;
@@ -508,7 +509,7 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   // We want to read exactly 4 bytes of data.
   if (region.readBytes(address, 4, (uint8_t*)bytes, &read) == -1 || read < 4)
-    return false;
+    return Fail;
 
   // Encoded as a big-endian 32-bit word in the stream.
   insn = (bytes[0]<<24) | (bytes[1]<<16) | (bytes[2]<< 8) | (bytes[3]<<0);
@@ -517,7 +518,7 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
   // that it is a valid instruction.
   unsigned opcode = getOPCODE(insn);
   if (opcode == UNSUPPORTED)
-    return false;
+    return Fail;
 
   instr.setOpcode(opcode);
 
@@ -529,11 +530,11 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
   uint64_t tsFlags = MBlazeInsts[opcode].TSFlags;
   switch ((tsFlags & MBlazeII::FormMask)) {
   default: 
-    return false;
+    return Fail;
 
   case MBlazeII::FRRRR:
     if (RD == UNSUPPORTED || RA == UNSUPPORTED || RB == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RB));
     instr.addOperand(MCOperand::CreateReg(RA));
@@ -541,7 +542,7 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   case MBlazeII::FRRR:
     if (RD == UNSUPPORTED || RA == UNSUPPORTED || RB == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RA));
     instr.addOperand(MCOperand::CreateReg(RB));
@@ -550,23 +551,23 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
   case MBlazeII::FRI:
     switch (opcode) {
     default: 
-      return false;
+      return Fail;
     case MBlaze::MFS:
       if (RD == UNSUPPORTED)
-        return false;
+        return Fail;
       instr.addOperand(MCOperand::CreateReg(RD));
       instr.addOperand(MCOperand::CreateImm(insn&0x3FFF));
       break;
     case MBlaze::MTS:
       if (RA == UNSUPPORTED)
-        return false;
+        return Fail;
       instr.addOperand(MCOperand::CreateImm(insn&0x3FFF));
       instr.addOperand(MCOperand::CreateReg(RA));
       break;
     case MBlaze::MSRSET:
     case MBlaze::MSRCLR:
       if (RD == UNSUPPORTED)
-        return false;
+        return Fail;
       instr.addOperand(MCOperand::CreateReg(RD));
       instr.addOperand(MCOperand::CreateImm(insn&0x7FFF));
       break;
@@ -575,7 +576,7 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   case MBlazeII::FRRI:
     if (RD == UNSUPPORTED || RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RA));
     switch (opcode) {
@@ -592,35 +593,35 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   case MBlazeII::FCRR:
     if (RA == UNSUPPORTED || RB == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RA));
     instr.addOperand(MCOperand::CreateReg(RB));
     break;
 
   case MBlazeII::FCRI:
     if (RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RA));
     instr.addOperand(MCOperand::CreateImm(getIMM(insn)));
     break;
 
   case MBlazeII::FRCR:
     if (RD == UNSUPPORTED || RB == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RB));
     break;
 
   case MBlazeII::FRCI:
     if (RD == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateImm(getIMM(insn)));
     break;
 
   case MBlazeII::FCCR:
     if (RB == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RB));
     break;
 
@@ -630,7 +631,7 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   case MBlazeII::FRRCI:
     if (RD == UNSUPPORTED || RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RA));
     instr.addOperand(MCOperand::CreateImm(getSHT(insn)));
@@ -638,35 +639,35 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   case MBlazeII::FRRC:
     if (RD == UNSUPPORTED || RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RA));
     break;
 
   case MBlazeII::FRCX:
     if (RD == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateImm(getFSL(insn)));
     break;
 
   case MBlazeII::FRCS:
     if (RD == UNSUPPORTED || RS == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateReg(RS));
     break;
 
   case MBlazeII::FCRCS:
     if (RS == UNSUPPORTED || RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RS));
     instr.addOperand(MCOperand::CreateReg(RA));
     break;
 
   case MBlazeII::FCRCX:
     if (RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RA));
     instr.addOperand(MCOperand::CreateImm(getFSL(insn)));
     break;
@@ -677,13 +678,13 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
 
   case MBlazeII::FCR:
     if (RB == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RB));
     break;
 
   case MBlazeII::FRIR:
     if (RD == UNSUPPORTED || RA == UNSUPPORTED)
-      return false;
+      return Fail;
     instr.addOperand(MCOperand::CreateReg(RD));
     instr.addOperand(MCOperand::CreateImm(getIMM(insn)));
     instr.addOperand(MCOperand::CreateReg(RA));
@@ -693,11 +694,12 @@ bool MBlazeDisassembler::getInstruction(MCInst &instr,
   // We always consume 4 bytes of data on success
   size = 4;
 
-  return true;
+  return Success;
 }
 
-static MCDisassembler *createMBlazeDisassembler(const Target &T) {
-  return new MBlazeDisassembler;
+static MCDisassembler *createMBlazeDisassembler(const Target &T,
+                                                const MCSubtargetInfo &STI) {
+  return new MBlazeDisassembler(STI);
 }
 
 extern "C" void LLVMInitializeMBlazeDisassembler() {
diff --git a/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.h b/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.h
index d05eced0bacf..0ac0d89efbe7 100644
--- a/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.h
+++ b/contrib/llvm/lib/Target/MBlaze/Disassembler/MBlazeDisassembler.h
@@ -32,19 +32,20 @@ class MBlazeDisassembler : public MCDisassembler {
 public:
   /// Constructor     - Initializes the disassembler.
   ///
-  MBlazeDisassembler() :
-    MCDisassembler() {
+  MBlazeDisassembler(const MCSubtargetInfo &STI) :
+    MCDisassembler(STI) {
   }
 
   ~MBlazeDisassembler() {
   }
 
   /// getInstruction - See MCDisassembler.
-  bool getInstruction(MCInst &instr,
+  MCDisassembler::DecodeStatus getInstruction(MCInst &instr,
                       uint64_t &size,
                       const MemoryObject &region,
                       uint64_t address,
-                      raw_ostream &vStream) const;
+                      raw_ostream &vStream,
+                      raw_ostream &cStream) const;
 
   /// getEDInfo - See MCDisassembler.
   EDInstInfo *getEDInfo() const;
diff --git a/contrib/llvm/lib/Target/MBlaze/Disassembler/Makefile b/contrib/llvm/lib/Target/MBlaze/Disassembler/Makefile
deleted file mode 100644
index 0530b3286bc4..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/Disassembler/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/MBlaze/Disassembler/Makefile -------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMBlazeDisassembler
-
-# Hack: we need to include 'main' MBlaze target directory to grab headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/MBlaze/InstPrinter/CMakeLists.txt b/contrib/llvm/lib/Target/MBlaze/InstPrinter/CMakeLists.txt
deleted file mode 100644
index 242a573036e6..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. 
-                     ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMMBlazeAsmPrinter
-    MBlazeInstPrinter.cpp
-  )
-
-add_dependencies(LLVMMBlazeAsmPrinter MBlazeCodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.cpp b/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.cpp
index a7fd287990b7..a1f1dbc7a23b 100644
--- a/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.cpp
@@ -25,8 +25,10 @@ using namespace llvm;
 // Include the auto-generated portion of the assembly writer.
 #include "MBlazeGenAsmWriter.inc"
 
-void MBlazeInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
+void MBlazeInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
+                                  StringRef Annot) {
   printInstruction(MI, O);
+  printAnnotation(O, Annot);
 }
 
 void MBlazeInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
diff --git a/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.h b/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.h
index eacca410b986..570ab08a07aa 100644
--- a/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.h
+++ b/contrib/llvm/lib/Target/MBlaze/InstPrinter/MBlazeInstPrinter.h
@@ -24,7 +24,7 @@ namespace llvm {
     MBlazeInstPrinter(const MCAsmInfo &MAI)
       : MCInstPrinter(MAI) {}
 
-    virtual void printInst(const MCInst *MI, raw_ostream &O);
+    virtual void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot);
 
     // Autogenerated by tblgen.
     void printInstruction(const MCInst *MI, raw_ostream &O);
diff --git a/contrib/llvm/lib/Target/MBlaze/InstPrinter/Makefile b/contrib/llvm/lib/Target/MBlaze/InstPrinter/Makefile
deleted file mode 100644
index 9fb6e869d945..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/InstPrinter/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/MBlaze/AsmPrinter/Makefile ---------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMBlazeAsmPrinter
-
-# Hack: we need to include 'main' MBlaze target directory to grab
-#       private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlaze.h b/contrib/llvm/lib/Target/MBlaze/MBlaze.h
index 3390794c9375..1399b85d34d2 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlaze.h
+++ b/contrib/llvm/lib/Target/MBlaze/MBlaze.h
@@ -15,6 +15,7 @@
 #ifndef TARGET_MBLAZE_H
 #define TARGET_MBLAZE_H
 
+#include "MCTargetDesc/MBlazeBaseInfo.h"
 #include "MCTargetDesc/MBlazeMCTargetDesc.h"
 #include "llvm/Target/TargetMachine.h"
 
@@ -22,17 +23,6 @@ namespace llvm {
   class MBlazeTargetMachine;
   class FunctionPass;
   class MachineCodeEmitter;
-  class MCCodeEmitter;
-  class MCInstrInfo;
-  class MCSubtargetInfo;
-  class TargetAsmBackend;
-  class formatted_raw_ostream;
-
-  MCCodeEmitter *createMBlazeMCCodeEmitter(const MCInstrInfo &MCII,
-                                           const MCSubtargetInfo &STI,
-                                           MCContext &Ctx);
-  
-  TargetAsmBackend *createMBlazeAsmBackend(const Target &, const std::string &);
 
   FunctionPass *createMBlazeISelDag(MBlazeTargetMachine &TM);
   FunctionPass *createMBlazeDelaySlotFillerPass(MBlazeTargetMachine &TM);
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeAsmPrinter.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeAsmPrinter.cpp
index 0016df569b93..97bd083fdd15 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeAsmPrinter.cpp
@@ -38,10 +38,10 @@
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cctype>
 
@@ -136,19 +136,17 @@ void MBlazeAsmPrinter::printSavedRegsBitmask() {
   const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     unsigned Reg = CSI[i].getReg();
-    unsigned RegNum = MBlazeRegisterInfo::getRegisterNumbering(Reg);
+    unsigned RegNum = getMBlazeRegisterNumbering(Reg);
     if (MBlaze::GPRRegisterClass->contains(Reg))
       CPUBitmask |= (1 << RegNum);
   }
 
   // Return Address and Frame registers must also be set in CPUBitmask.
   if (TFI->hasFP(*MF))
-    CPUBitmask |= (1 << MBlazeRegisterInfo::
-                getRegisterNumbering(RI.getFrameRegister(*MF)));
+    CPUBitmask |= (1 <<  getMBlazeRegisterNumbering(RI.getFrameRegister(*MF)));
 
   if (MFI->adjustsStack())
-    CPUBitmask |= (1 << MBlazeRegisterInfo::
-                getRegisterNumbering(RI.getRARegister()));
+    CPUBitmask |= (1 << getMBlazeRegisterNumbering(RI.getRARegister()));
 
   // Print CPUBitmask
   OutStreamer.EmitRawText("\t.mask\t0x" + Twine::utohexstr(CPUBitmask));
@@ -318,18 +316,7 @@ isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const {
   return I == Pred->end() || !I->getDesc().isBarrier();
 }
 
-static MCInstPrinter *createMBlazeMCInstPrinter(const Target &T,
-                                                unsigned SyntaxVariant,
-                                                const MCAsmInfo &MAI) {
-  if (SyntaxVariant == 0)
-    return new MBlazeInstPrinter(MAI);
-  return 0;
-}
-
 // Force static initialization.
 extern "C" void LLVMInitializeMBlazeAsmPrinter() {
   RegisterAsmPrinter<MBlazeAsmPrinter> X(TheMBlazeTarget);
-  TargetRegistry::RegisterMCInstPrinter(TheMBlazeTarget,
-                                        createMBlazeMCInstPrinter);
-
 }
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeFrameLowering.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeFrameLowering.cpp
index e7639025cf1a..f28d5a77d49c 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeFrameLowering.cpp
@@ -1,4 +1,4 @@
-//=======- MBlazeFrameLowering.cpp - MBlaze Frame Information ------*- C++ -*-====//
+//===- MBlazeFrameLowering.cpp - MBlaze Frame Information ------*- C++ -*-====//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.cpp
index 62dfdcc2fd10..8ec548f1437d 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.cpp
@@ -59,6 +59,7 @@ MBlazeTargetLowering::MBlazeTargetLowering(MBlazeTargetMachine &TM)
   // MBlaze does not have i1 type, so use i32 for
   // setcc operations results (slt, sgt, ...).
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
 
   // Set up the register classes
   addRegisterClass(MVT::i32, MBlaze::GPRRegisterClass);
@@ -187,7 +188,7 @@ MBlazeTargetLowering::MBlazeTargetLowering(MBlazeTargetMachine &TM)
   computeRegisterProperties();
 }
 
-MVT::SimpleValueType MBlazeTargetLowering::getSetCCResultType(EVT VT) const {
+EVT MBlazeTargetLowering::getSetCCResultType(EVT VT) const {
   return MVT::i32;
 }
 
@@ -964,13 +965,13 @@ LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
     // The last register argument that must be saved is MBlaze::R10
     TargetRegisterClass *RC = MBlaze::GPRRegisterClass;
 
-    unsigned Begin = MBlazeRegisterInfo::getRegisterNumbering(MBlaze::R5);
-    unsigned Start = MBlazeRegisterInfo::getRegisterNumbering(ArgRegEnd+1);
-    unsigned End   = MBlazeRegisterInfo::getRegisterNumbering(MBlaze::R10);
+    unsigned Begin = getMBlazeRegisterNumbering(MBlaze::R5);
+    unsigned Start = getMBlazeRegisterNumbering(ArgRegEnd+1);
+    unsigned End   = getMBlazeRegisterNumbering(MBlaze::R10);
     unsigned StackLoc = Start - Begin + 1;
 
     for (; Start <= End; ++Start, ++StackLoc) {
-      unsigned Reg = MBlazeRegisterInfo::getRegisterFromNumbering(Start);
+      unsigned Reg = getMBlazeRegisterFromNumbering(Start);
       unsigned LiveReg = MF.addLiveIn(Reg, RC);
       SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, LiveReg, MVT::i32);
 
@@ -1096,7 +1097,7 @@ MBlazeTargetLowering::getSingleConstraintMatchWeight(
     // but allow it at the lowest weight.
   if (CallOperandVal == NULL)
     return CW_Default;
-  const Type *type = CallOperandVal->getType();
+  Type *type = CallOperandVal->getType();
   // Look at the constraint type.
   switch (*constraint) {
   default:
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.h b/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.h
index bb128da3c7c0..8b49bc3de0cc 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.h
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeISelLowering.h
@@ -102,7 +102,7 @@ namespace llvm {
     virtual const char *getTargetNodeName(unsigned Opcode) const;
 
     /// getSetCCResultType - get the ISD::SETCC result ValueType
-    MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    EVT getSetCCResultType(EVT VT) const;
 
   private:
     // Subtarget Info
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.cpp
index 188f10a3972e..7ae05b367cba 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.cpp
@@ -17,9 +17,9 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 
 #define GET_INSTRINFO_CTOR
@@ -239,7 +239,8 @@ unsigned MBlazeInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
   return 2;
 }
 
-bool MBlazeInstrInfo::ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
+bool MBlazeInstrInfo::ReverseBranchCondition(SmallVectorImpl<MachineOperand>
+                                               &Cond) const {
   assert(Cond.size() == 2 && "Invalid MBlaze branch opcode!");
   switch (Cond[0].getImm()) {
   default:            return true;
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.h b/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.h
index 79f962b349bf..7174405a49d9 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.h
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.h
@@ -166,62 +166,6 @@ namespace MBlaze {
   }
 }
 
-/// MBlazeII - This namespace holds all of the target specific flags that
-/// instruction info tracks.
-///
-namespace MBlazeII {
-  enum {
-    // PseudoFrm - This represents an instruction that is a pseudo instruction
-    // or one that has not been implemented yet.  It is illegal to code generate
-    // it, but tolerated for intermediate implementation stages.
-    FPseudo = 0,
-    FRRR,
-    FRRI,
-    FCRR,
-    FCRI,
-    FRCR,
-    FRCI,
-    FCCR,
-    FCCI,
-    FRRCI,
-    FRRC,
-    FRCX,
-    FRCS,
-    FCRCS,
-    FCRCX,
-    FCX,
-    FCR,
-    FRIR,
-    FRRRR,
-    FRI,
-    FC,
-    FormMask = 63
-
-    //===------------------------------------------------------------------===//
-    // MBlaze Specific MachineOperand flags.
-    // MO_NO_FLAG,
-
-    /// MO_GOT - Represents the offset into the global offset table at which
-    /// the address the relocation entry symbol resides during execution.
-    // MO_GOT,
-
-    /// MO_GOT_CALL - Represents the offset into the global offset table at
-    /// which the address of a call site relocation entry symbol resides
-    /// during execution. This is different from the above since this flag
-    /// can only be present in call instructions.
-    // MO_GOT_CALL,
-
-    /// MO_GPREL - Represents the offset from the current gp value to be used
-    /// for the relocatable object file being produced.
-    // MO_GPREL,
-
-    /// MO_ABS_HILO - Represents the hi or low part of an absolute symbol
-    /// address.
-    // MO_ABS_HILO
-
-  };
-}
-
 class MBlazeInstrInfo : public MBlazeGenInstrInfo {
   MBlazeTargetMachine &TM;
   const MBlazeRegisterInfo RI;
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.td b/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.td
index 950f2d77029b..1d8c987a0856 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.td
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeInstrInfo.td
@@ -442,17 +442,19 @@ let Predicates=[HasMul] in {
 //===----------------------------------------------------------------------===//
 
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
-  def LBU  :  LoadM<0x30, 0x000, "lbu    ">;
-  def LBUR :  LoadM<0x30, 0x200, "lbur   ">;
+  let neverHasSideEffects = 1 in {
+    def LBU  :  LoadM<0x30, 0x000, "lbu    ">;
+    def LBUR :  LoadM<0x30, 0x200, "lbur   ">;
 
-  def LHU  :  LoadM<0x31, 0x000, "lhu    ">;
-  def LHUR :  LoadM<0x31, 0x200, "lhur   ">;
+    def LHU  :  LoadM<0x31, 0x000, "lhu    ">;
+    def LHUR :  LoadM<0x31, 0x200, "lhur   ">;
 
-  def LW   :  LoadM<0x32, 0x000, "lw     ">;
-  def LWR  :  LoadM<0x32, 0x200, "lwr    ">;
+    def LW   :  LoadM<0x32, 0x000, "lw     ">;
+    def LWR  :  LoadM<0x32, 0x200, "lwr    ">;
 
-  let Defs = [CARRY] in {
-    def LWX  :  LoadM<0x32, 0x400, "lwx    ">;
+    let Defs = [CARRY] in {
+      def LWX  :  LoadM<0x32, 0x400, "lwx    ">;
+    }
   }
 
   def LBUI : LoadMI<0x38, "lbui   ", zextloadi8>;
@@ -877,6 +879,9 @@ def : Pat<(zextloadi8 xaddr:$addr), (i32 (LBU xaddr:$addr))>;
 // Peepholes
 def : Pat<(store (i32 0), iaddr:$dst), (SWI (i32 R0), iaddr:$dst)>;
 
+// Atomic fence
+def : Pat<(atomic_fence (imm), (imm)), (MEMBARRIER)>;
+
 //===----------------------------------------------------------------------===//
 // Floating Point Support
 //===----------------------------------------------------------------------===//
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp
index 32d67b264a20..ea81dd63d195 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp
@@ -37,7 +37,7 @@ namespace mblazeIntrinsic {
 #undef GET_LLVM_INTRINSIC_FOR_GCC_BUILTIN
 }
 
-std::string MBlazeIntrinsicInfo::getName(unsigned IntrID, const Type **Tys,
+std::string MBlazeIntrinsicInfo::getName(unsigned IntrID, Type **Tys,
                                          unsigned numTys) const {
   static const char *const names[] = {
 #define GET_INTRINSIC_NAME_TABLE
@@ -90,8 +90,8 @@ bool MBlazeIntrinsicInfo::isOverloaded(unsigned IntrID) const {
 #include "MBlazeGenIntrinsics.inc"
 #undef GET_INTRINSIC_ATTRIBUTES
 
-static const FunctionType *getType(LLVMContext &Context, unsigned id) {
-  const Type *ResultTy = NULL;
+static FunctionType *getType(LLVMContext &Context, unsigned id) {
+  Type *ResultTy = NULL;
   std::vector<Type*> ArgTys;
   bool IsVarArg = false;
 
@@ -103,7 +103,7 @@ static const FunctionType *getType(LLVMContext &Context, unsigned id) {
 }
 
 Function *MBlazeIntrinsicInfo::getDeclaration(Module *M, unsigned IntrID,
-                                                const Type **Tys,
+                                                Type **Tys,
                                                 unsigned numTy) const {
   assert(!isOverloaded(IntrID) && "MBlaze intrinsics are not overloaded");
   AttrListPtr AList = getAttributes((mblazeIntrinsic::ID) IntrID);
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.h b/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.h
index 9804c7723bee..80760d87e00a 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.h
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeIntrinsicInfo.h
@@ -19,12 +19,12 @@ namespace llvm {
 
   class MBlazeIntrinsicInfo : public TargetIntrinsicInfo {
   public:
-    std::string getName(unsigned IntrID, const Type **Tys = 0,
+    std::string getName(unsigned IntrID, Type **Tys = 0,
                         unsigned numTys = 0) const;
     unsigned lookupName(const char *Name, unsigned Len) const;
     unsigned lookupGCCName(const char *Name) const;
     bool isOverloaded(unsigned IID) const;
-    Function *getDeclaration(Module *M, unsigned ID, const Type **Tys = 0,
+    Function *getDeclaration(Module *M, unsigned ID, Type **Tys = 0,
                              unsigned numTys = 0) const;
   };
 
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
index f0b201a66170..9788ba9e6021 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
@@ -25,7 +25,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -44,164 +43,7 @@ using namespace llvm;
 
 MBlazeRegisterInfo::
 MBlazeRegisterInfo(const MBlazeSubtarget &ST, const TargetInstrInfo &tii)
-  : MBlazeGenRegisterInfo(), Subtarget(ST), TII(tii) {}
-
-/// getRegisterNumbering - Given the enum value for some register, e.g.
-/// MBlaze::R0, return the number that it corresponds to (e.g. 0).
-unsigned MBlazeRegisterInfo::getRegisterNumbering(unsigned RegEnum) {
-  switch (RegEnum) {
-    case MBlaze::R0     : return 0;
-    case MBlaze::R1     : return 1;
-    case MBlaze::R2     : return 2;
-    case MBlaze::R3     : return 3;
-    case MBlaze::R4     : return 4;
-    case MBlaze::R5     : return 5;
-    case MBlaze::R6     : return 6;
-    case MBlaze::R7     : return 7;
-    case MBlaze::R8     : return 8;
-    case MBlaze::R9     : return 9;
-    case MBlaze::R10    : return 10;
-    case MBlaze::R11    : return 11;
-    case MBlaze::R12    : return 12;
-    case MBlaze::R13    : return 13;
-    case MBlaze::R14    : return 14;
-    case MBlaze::R15    : return 15;
-    case MBlaze::R16    : return 16;
-    case MBlaze::R17    : return 17;
-    case MBlaze::R18    : return 18;
-    case MBlaze::R19    : return 19;
-    case MBlaze::R20    : return 20;
-    case MBlaze::R21    : return 21;
-    case MBlaze::R22    : return 22;
-    case MBlaze::R23    : return 23;
-    case MBlaze::R24    : return 24;
-    case MBlaze::R25    : return 25;
-    case MBlaze::R26    : return 26;
-    case MBlaze::R27    : return 27;
-    case MBlaze::R28    : return 28;
-    case MBlaze::R29    : return 29;
-    case MBlaze::R30    : return 30;
-    case MBlaze::R31    : return 31;
-    case MBlaze::RPC    : return 0x0000;
-    case MBlaze::RMSR   : return 0x0001;
-    case MBlaze::REAR   : return 0x0003;
-    case MBlaze::RESR   : return 0x0005;
-    case MBlaze::RFSR   : return 0x0007;
-    case MBlaze::RBTR   : return 0x000B;
-    case MBlaze::REDR   : return 0x000D;
-    case MBlaze::RPID   : return 0x1000;
-    case MBlaze::RZPR   : return 0x1001;
-    case MBlaze::RTLBX  : return 0x1002;
-    case MBlaze::RTLBLO : return 0x1003;
-    case MBlaze::RTLBHI : return 0x1004;
-    case MBlaze::RPVR0  : return 0x2000;
-    case MBlaze::RPVR1  : return 0x2001;
-    case MBlaze::RPVR2  : return 0x2002;
-    case MBlaze::RPVR3  : return 0x2003;
-    case MBlaze::RPVR4  : return 0x2004;
-    case MBlaze::RPVR5  : return 0x2005;
-    case MBlaze::RPVR6  : return 0x2006;
-    case MBlaze::RPVR7  : return 0x2007;
-    case MBlaze::RPVR8  : return 0x2008;
-    case MBlaze::RPVR9  : return 0x2009;
-    case MBlaze::RPVR10 : return 0x200A;
-    case MBlaze::RPVR11 : return 0x200B;
-    default: llvm_unreachable("Unknown register number!");
-  }
-  return 0; // Not reached
-}
-
-/// getRegisterFromNumbering - Given the enum value for some register, e.g.
-/// MBlaze::R0, return the number that it corresponds to (e.g. 0).
-unsigned MBlazeRegisterInfo::getRegisterFromNumbering(unsigned Reg) {
-  switch (Reg) {
-    case 0  : return MBlaze::R0;
-    case 1  : return MBlaze::R1;
-    case 2  : return MBlaze::R2;
-    case 3  : return MBlaze::R3;
-    case 4  : return MBlaze::R4;
-    case 5  : return MBlaze::R5;
-    case 6  : return MBlaze::R6;
-    case 7  : return MBlaze::R7;
-    case 8  : return MBlaze::R8;
-    case 9  : return MBlaze::R9;
-    case 10 : return MBlaze::R10;
-    case 11 : return MBlaze::R11;
-    case 12 : return MBlaze::R12;
-    case 13 : return MBlaze::R13;
-    case 14 : return MBlaze::R14;
-    case 15 : return MBlaze::R15;
-    case 16 : return MBlaze::R16;
-    case 17 : return MBlaze::R17;
-    case 18 : return MBlaze::R18;
-    case 19 : return MBlaze::R19;
-    case 20 : return MBlaze::R20;
-    case 21 : return MBlaze::R21;
-    case 22 : return MBlaze::R22;
-    case 23 : return MBlaze::R23;
-    case 24 : return MBlaze::R24;
-    case 25 : return MBlaze::R25;
-    case 26 : return MBlaze::R26;
-    case 27 : return MBlaze::R27;
-    case 28 : return MBlaze::R28;
-    case 29 : return MBlaze::R29;
-    case 30 : return MBlaze::R30;
-    case 31 : return MBlaze::R31;
-    default: llvm_unreachable("Unknown register number!");
-  }
-  return 0; // Not reached
-}
-
-unsigned MBlazeRegisterInfo::getSpecialRegisterFromNumbering(unsigned Reg) {
-  switch (Reg) {
-    case 0x0000 : return MBlaze::RPC;
-    case 0x0001 : return MBlaze::RMSR;
-    case 0x0003 : return MBlaze::REAR;
-    case 0x0005 : return MBlaze::RESR;
-    case 0x0007 : return MBlaze::RFSR;
-    case 0x000B : return MBlaze::RBTR;
-    case 0x000D : return MBlaze::REDR;
-    case 0x1000 : return MBlaze::RPID;
-    case 0x1001 : return MBlaze::RZPR;
-    case 0x1002 : return MBlaze::RTLBX;
-    case 0x1003 : return MBlaze::RTLBLO;
-    case 0x1004 : return MBlaze::RTLBHI;
-    case 0x2000 : return MBlaze::RPVR0;
-    case 0x2001 : return MBlaze::RPVR1;
-    case 0x2002 : return MBlaze::RPVR2;
-    case 0x2003 : return MBlaze::RPVR3;
-    case 0x2004 : return MBlaze::RPVR4;
-    case 0x2005 : return MBlaze::RPVR5;
-    case 0x2006 : return MBlaze::RPVR6;
-    case 0x2007 : return MBlaze::RPVR7;
-    case 0x2008 : return MBlaze::RPVR8;
-    case 0x2009 : return MBlaze::RPVR9;
-    case 0x200A : return MBlaze::RPVR10;
-    case 0x200B : return MBlaze::RPVR11;
-    default: llvm_unreachable("Unknown register number!");
-  }
-  return 0; // Not reached
-}
-
-bool MBlazeRegisterInfo::isRegister(unsigned Reg) {
-  return Reg <= 31;
-}
-
-bool MBlazeRegisterInfo::isSpecialRegister(unsigned Reg) {
-  switch (Reg) {
-    case 0x0000 : case 0x0001 : case 0x0003 : case 0x0005 : 
-    case 0x0007 : case 0x000B : case 0x000D : case 0x1000 : 
-    case 0x1001 : case 0x1002 : case 0x1003 : case 0x1004 : 
-    case 0x2000 : case 0x2001 : case 0x2002 : case 0x2003 : 
-    case 0x2004 : case 0x2005 : case 0x2006 : case 0x2007 : 
-    case 0x2008 : case 0x2009 : case 0x200A : case 0x200B : 
-      return true;
-
-    default:
-      return false;
-  }
-  return false; // Not reached
-}
+  : MBlazeGenRegisterInfo(MBlaze::R15), Subtarget(ST), TII(tii) {}
 
 unsigned MBlazeRegisterInfo::getPICCallReg() {
   return MBlaze::R20;
@@ -334,10 +176,6 @@ processFunctionBeforeFrameFinalized(MachineFunction &MF) const {
     MFI->setObjectOffset(MBlazeFI->getGPFI(), MBlazeFI->getGPStackOffset());
 }
 
-unsigned MBlazeRegisterInfo::getRARegister() const {
-  return MBlaze::R15;
-}
-
 unsigned MBlazeRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
 
@@ -353,11 +191,3 @@ unsigned MBlazeRegisterInfo::getEHHandlerRegister() const {
   llvm_unreachable("What is the exception handler register");
   return 0;
 }
-
-int MBlazeRegisterInfo::getDwarfRegNum(unsigned RegNo, bool isEH) const {
-  return MBlazeGenRegisterInfo::getDwarfRegNumFull(RegNo,0);
-}
-
-int MBlazeRegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  return MBlazeGenRegisterInfo::getLLVMRegNumFull(DwarfRegNo,0);
-}
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.h b/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.h
index 7ebce21d3a80..7e4b269cb887 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.h
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeRegisterInfo.h
@@ -42,14 +42,6 @@ struct MBlazeRegisterInfo : public MBlazeGenRegisterInfo {
   MBlazeRegisterInfo(const MBlazeSubtarget &Subtarget,
                      const TargetInstrInfo &tii);
 
-  /// getRegisterNumbering - Given the enum value for some register, e.g.
-  /// MBlaze::RA, return the number that it corresponds to (e.g. 31).
-  static unsigned getRegisterNumbering(unsigned RegEnum);
-  static unsigned getRegisterFromNumbering(unsigned RegEnum);
-  static unsigned getSpecialRegisterFromNumbering(unsigned RegEnum);
-  static bool isRegister(unsigned RegEnum);
-  static bool isSpecialRegister(unsigned RegEnum);
-
   /// Get PIC indirect call register
   static unsigned getPICCallReg();
 
@@ -69,15 +61,11 @@ struct MBlazeRegisterInfo : public MBlazeGenRegisterInfo {
   void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
 
   /// Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   /// Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
-
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeSubtarget.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeSubtarget.cpp
index eda141daf2b3..7e5667f55c15 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeSubtarget.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeSubtarget.cpp
@@ -15,7 +15,7 @@
 #include "MBlaze.h"
 #include "MBlazeRegisterInfo.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.cpp
index 7208874aef1d..7bff53ef8717 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.cpp
@@ -16,48 +16,13 @@
 #include "llvm/PassManager.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 using namespace llvm;
 
-static MCStreamer *createMCStreamer(const Target &T, const std::string &TT,
-                                    MCContext &Ctx, TargetAsmBackend &TAB,
-                                    raw_ostream &_OS,
-                                    MCCodeEmitter *_Emitter,
-                                    bool RelaxAll,
-                                    bool NoExecStack) {
-  Triple TheTriple(TT);
-
-  if (TheTriple.isOSDarwin()) {
-    llvm_unreachable("MBlaze does not support Darwin MACH-O format");
-    return NULL;
-  }
-
-  if (TheTriple.isOSWindows()) {
-    llvm_unreachable("MBlaze does not support Windows COFF format");
-    return NULL;
-  }
-
-  return createELFStreamer(Ctx, TAB, _OS, _Emitter, RelaxAll, NoExecStack);
-}
-
-
 extern "C" void LLVMInitializeMBlazeTarget() {
   // Register the target.
   RegisterTargetMachine<MBlazeTargetMachine> X(TheMBlazeTarget);
-
-  // Register the MC code emitter
-  TargetRegistry::RegisterCodeEmitter(TheMBlazeTarget,
-                                      llvm::createMBlazeMCCodeEmitter);
-
-  // Register the asm backend
-  TargetRegistry::RegisterAsmBackend(TheMBlazeTarget,
-                                     createMBlazeAsmBackend);
-
-  // Register the object streamer
-  TargetRegistry::RegisterObjectStreamer(TheMBlazeTarget,
-                                         createMCStreamer);
-
 }
 
 // DataLayout --> Big-endian, 32-bit pointer/ABI/alignment
@@ -67,21 +32,16 @@ extern "C" void LLVMInitializeMBlazeTarget() {
 // offset from the stack/frame pointer, using StackGrowsUp enables
 // an easier handling.
 MBlazeTargetMachine::
-MBlazeTargetMachine(const Target &T, const std::string &TT,
-                    const std::string &CPU, const std::string &FS):
-  LLVMTargetMachine(T, TT, CPU, FS),
+MBlazeTargetMachine(const Target &T, StringRef TT,
+                    StringRef CPU, StringRef FS,
+                    Reloc::Model RM, CodeModel::Model CM):
+  LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
   Subtarget(TT, CPU, FS),
   DataLayout("E-p:32:32:32-i8:8:8-i16:16:16"),
   InstrInfo(*this),
   FrameLowering(Subtarget),
   TLInfo(*this), TSInfo(*this), ELFWriterInfo(*this),
   InstrItins(Subtarget.getInstrItineraryData()) {
-  if (getRelocationModel() == Reloc::Default) {
-      setRelocationModel(Reloc::Static);
-  }
-
-  if (getCodeModel() == CodeModel::Default)
-    setCodeModel(CodeModel::Small);
 }
 
 // Install an instruction selector pass using
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.h b/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.h
index cd6caafbf309..c1bc08aeb505 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.h
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeTargetMachine.h
@@ -41,8 +41,9 @@ namespace llvm {
     InstrItineraryData     InstrItins;
 
   public:
-    MBlazeTargetMachine(const Target &T, const std::string &TT,
-                        const std::string &CPU, const std::string &FS);
+    MBlazeTargetMachine(const Target &T, StringRef TT,
+                        StringRef CPU, StringRef FS,
+                        Reloc::Model RM, CodeModel::Model CM);
 
     virtual const MBlazeInstrInfo *getInstrInfo() const
     { return &InstrInfo; }
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp b/contrib/llvm/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp
index abd1b0b62c7d..f66ea302d9fe 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp
@@ -69,7 +69,7 @@ IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
   if (Kind.isMergeable1ByteCString())
     return false;
 
-  const Type *Ty = GV->getType()->getElementType();
+  Type *Ty = GV->getType()->getElementType();
   return IsInSmallSection(TM.getTargetData()->getTypeAllocSize(Ty));
 }
 
diff --git a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 3d15708c35b8..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMMBlazeDesc
-  MBlazeMCTargetDesc.cpp
-  MBlazeMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeAsmBackend.cpp b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp
similarity index 91%
rename from contrib/llvm/lib/Target/MBlaze/MBlazeAsmBackend.cpp
rename to contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp
index 08f14c365957..08f7d46a58f9 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeAsmBackend.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp
@@ -7,10 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmBackend.h"
-#include "MBlaze.h"
-#include "MBlazeELFWriterInfo.h"
-#include "llvm/ADT/Twine.h"
+#include "MCTargetDesc/MBlazeMCTargetDesc.h"
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCELFObjectWriter.h"
@@ -20,11 +18,11 @@
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCValue.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmBackend.h"
 using namespace llvm;
 
 static unsigned getFixupKindSize(unsigned Kind) {
@@ -48,10 +46,10 @@ class MBlazeELFObjectWriter : public MCELFObjectTargetWriter {
                               /*HasRelocationAddend*/ true) {}
 };
 
-class MBlazeAsmBackend : public TargetAsmBackend {
+class MBlazeAsmBackend : public MCAsmBackend {
 public:
   MBlazeAsmBackend(const Target &T)
-    : TargetAsmBackend() {
+    : MCAsmBackend() {
   }
 
   unsigned getNumFixupKinds() const {
@@ -148,8 +146,7 @@ void ELFMBlazeAsmBackend::ApplyFixup(const MCFixup &Fixup, char *Data,
 }
 } // end anonymous namespace
 
-TargetAsmBackend *llvm::createMBlazeAsmBackend(const Target &T,
-                                            const std::string &TT) {
+MCAsmBackend *llvm::createMBlazeAsmBackend(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin())
diff --git a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeBaseInfo.h b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeBaseInfo.h
new file mode 100644
index 000000000000..776dbc4d8678
--- /dev/null
+++ b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeBaseInfo.h
@@ -0,0 +1,240 @@
+//===-- MBlazeBaseInfo.h - Top level definitions for MBlaze -- --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains small standalone helper functions and enum definitions for
+// the MBlaze target useful for the compiler back-end and the MC libraries.
+// As such, it deliberately does not include references to LLVM core
+// code gen types, passes, etc..
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MBlazeBASEINFO_H
+#define MBlazeBASEINFO_H
+
+#include "MBlazeMCTargetDesc.h"
+#include "llvm/Support/ErrorHandling.h"
+
+namespace llvm {
+
+/// MBlazeII - This namespace holds all of the target specific flags that
+/// instruction info tracks.
+///
+namespace MBlazeII {
+  enum {
+    // PseudoFrm - This represents an instruction that is a pseudo instruction
+    // or one that has not been implemented yet.  It is illegal to code generate
+    // it, but tolerated for intermediate implementation stages.
+    FPseudo = 0,
+    FRRR,
+    FRRI,
+    FCRR,
+    FCRI,
+    FRCR,
+    FRCI,
+    FCCR,
+    FCCI,
+    FRRCI,
+    FRRC,
+    FRCX,
+    FRCS,
+    FCRCS,
+    FCRCX,
+    FCX,
+    FCR,
+    FRIR,
+    FRRRR,
+    FRI,
+    FC,
+    FormMask = 63
+
+    //===------------------------------------------------------------------===//
+    // MBlaze Specific MachineOperand flags.
+    // MO_NO_FLAG,
+
+    /// MO_GOT - Represents the offset into the global offset table at which
+    /// the address the relocation entry symbol resides during execution.
+    // MO_GOT,
+
+    /// MO_GOT_CALL - Represents the offset into the global offset table at
+    /// which the address of a call site relocation entry symbol resides
+    /// during execution. This is different from the above since this flag
+    /// can only be present in call instructions.
+    // MO_GOT_CALL,
+
+    /// MO_GPREL - Represents the offset from the current gp value to be used
+    /// for the relocatable object file being produced.
+    // MO_GPREL,
+
+    /// MO_ABS_HILO - Represents the hi or low part of an absolute symbol
+    /// address.
+    // MO_ABS_HILO
+
+  };
+}
+
+static inline bool isMBlazeRegister(unsigned Reg) {
+  return Reg <= 31;
+}
+
+static inline bool isSpecialMBlazeRegister(unsigned Reg) {
+  switch (Reg) {
+    case 0x0000 : case 0x0001 : case 0x0003 : case 0x0005 : 
+    case 0x0007 : case 0x000B : case 0x000D : case 0x1000 : 
+    case 0x1001 : case 0x1002 : case 0x1003 : case 0x1004 : 
+    case 0x2000 : case 0x2001 : case 0x2002 : case 0x2003 : 
+    case 0x2004 : case 0x2005 : case 0x2006 : case 0x2007 : 
+    case 0x2008 : case 0x2009 : case 0x200A : case 0x200B : 
+      return true;
+
+    default:
+      return false;
+  }
+  return false; // Not reached
+}
+
+/// getMBlazeRegisterNumbering - Given the enum value for some register, e.g.
+/// MBlaze::R0, return the number that it corresponds to (e.g. 0).
+static inline unsigned getMBlazeRegisterNumbering(unsigned RegEnum) {
+  switch (RegEnum) {
+    case MBlaze::R0     : return 0;
+    case MBlaze::R1     : return 1;
+    case MBlaze::R2     : return 2;
+    case MBlaze::R3     : return 3;
+    case MBlaze::R4     : return 4;
+    case MBlaze::R5     : return 5;
+    case MBlaze::R6     : return 6;
+    case MBlaze::R7     : return 7;
+    case MBlaze::R8     : return 8;
+    case MBlaze::R9     : return 9;
+    case MBlaze::R10    : return 10;
+    case MBlaze::R11    : return 11;
+    case MBlaze::R12    : return 12;
+    case MBlaze::R13    : return 13;
+    case MBlaze::R14    : return 14;
+    case MBlaze::R15    : return 15;
+    case MBlaze::R16    : return 16;
+    case MBlaze::R17    : return 17;
+    case MBlaze::R18    : return 18;
+    case MBlaze::R19    : return 19;
+    case MBlaze::R20    : return 20;
+    case MBlaze::R21    : return 21;
+    case MBlaze::R22    : return 22;
+    case MBlaze::R23    : return 23;
+    case MBlaze::R24    : return 24;
+    case MBlaze::R25    : return 25;
+    case MBlaze::R26    : return 26;
+    case MBlaze::R27    : return 27;
+    case MBlaze::R28    : return 28;
+    case MBlaze::R29    : return 29;
+    case MBlaze::R30    : return 30;
+    case MBlaze::R31    : return 31;
+    case MBlaze::RPC    : return 0x0000;
+    case MBlaze::RMSR   : return 0x0001;
+    case MBlaze::REAR   : return 0x0003;
+    case MBlaze::RESR   : return 0x0005;
+    case MBlaze::RFSR   : return 0x0007;
+    case MBlaze::RBTR   : return 0x000B;
+    case MBlaze::REDR   : return 0x000D;
+    case MBlaze::RPID   : return 0x1000;
+    case MBlaze::RZPR   : return 0x1001;
+    case MBlaze::RTLBX  : return 0x1002;
+    case MBlaze::RTLBLO : return 0x1003;
+    case MBlaze::RTLBHI : return 0x1004;
+    case MBlaze::RPVR0  : return 0x2000;
+    case MBlaze::RPVR1  : return 0x2001;
+    case MBlaze::RPVR2  : return 0x2002;
+    case MBlaze::RPVR3  : return 0x2003;
+    case MBlaze::RPVR4  : return 0x2004;
+    case MBlaze::RPVR5  : return 0x2005;
+    case MBlaze::RPVR6  : return 0x2006;
+    case MBlaze::RPVR7  : return 0x2007;
+    case MBlaze::RPVR8  : return 0x2008;
+    case MBlaze::RPVR9  : return 0x2009;
+    case MBlaze::RPVR10 : return 0x200A;
+    case MBlaze::RPVR11 : return 0x200B;
+    default: llvm_unreachable("Unknown register number!");
+  }
+  return 0; // Not reached
+}
+
+/// getRegisterFromNumbering - Given the enum value for some register, e.g.
+/// MBlaze::R0, return the number that it corresponds to (e.g. 0).
+static inline unsigned getMBlazeRegisterFromNumbering(unsigned Reg) {
+  switch (Reg) {
+    case 0  : return MBlaze::R0;
+    case 1  : return MBlaze::R1;
+    case 2  : return MBlaze::R2;
+    case 3  : return MBlaze::R3;
+    case 4  : return MBlaze::R4;
+    case 5  : return MBlaze::R5;
+    case 6  : return MBlaze::R6;
+    case 7  : return MBlaze::R7;
+    case 8  : return MBlaze::R8;
+    case 9  : return MBlaze::R9;
+    case 10 : return MBlaze::R10;
+    case 11 : return MBlaze::R11;
+    case 12 : return MBlaze::R12;
+    case 13 : return MBlaze::R13;
+    case 14 : return MBlaze::R14;
+    case 15 : return MBlaze::R15;
+    case 16 : return MBlaze::R16;
+    case 17 : return MBlaze::R17;
+    case 18 : return MBlaze::R18;
+    case 19 : return MBlaze::R19;
+    case 20 : return MBlaze::R20;
+    case 21 : return MBlaze::R21;
+    case 22 : return MBlaze::R22;
+    case 23 : return MBlaze::R23;
+    case 24 : return MBlaze::R24;
+    case 25 : return MBlaze::R25;
+    case 26 : return MBlaze::R26;
+    case 27 : return MBlaze::R27;
+    case 28 : return MBlaze::R28;
+    case 29 : return MBlaze::R29;
+    case 30 : return MBlaze::R30;
+    case 31 : return MBlaze::R31;
+    default: llvm_unreachable("Unknown register number!");
+  }
+  return 0; // Not reached
+}
+
+static inline unsigned getSpecialMBlazeRegisterFromNumbering(unsigned Reg) {
+  switch (Reg) {
+    case 0x0000 : return MBlaze::RPC;
+    case 0x0001 : return MBlaze::RMSR;
+    case 0x0003 : return MBlaze::REAR;
+    case 0x0005 : return MBlaze::RESR;
+    case 0x0007 : return MBlaze::RFSR;
+    case 0x000B : return MBlaze::RBTR;
+    case 0x000D : return MBlaze::REDR;
+    case 0x1000 : return MBlaze::RPID;
+    case 0x1001 : return MBlaze::RZPR;
+    case 0x1002 : return MBlaze::RTLBX;
+    case 0x1003 : return MBlaze::RTLBLO;
+    case 0x1004 : return MBlaze::RTLBHI;
+    case 0x2000 : return MBlaze::RPVR0;
+    case 0x2001 : return MBlaze::RPVR1;
+    case 0x2002 : return MBlaze::RPVR2;
+    case 0x2003 : return MBlaze::RPVR3;
+    case 0x2004 : return MBlaze::RPVR4;
+    case 0x2005 : return MBlaze::RPVR5;
+    case 0x2006 : return MBlaze::RPVR6;
+    case 0x2007 : return MBlaze::RPVR7;
+    case 0x2008 : return MBlaze::RPVR8;
+    case 0x2009 : return MBlaze::RPVR9;
+    case 0x200A : return MBlaze::RPVR10;
+    case 0x200B : return MBlaze::RPVR11;
+    default: llvm_unreachable("Unknown register number!");
+  }
+  return 0; // Not reached
+}
+
+} // end namespace llvm;
+
+#endif
diff --git a/contrib/llvm/lib/Target/MBlaze/MBlazeMCCodeEmitter.cpp b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp
similarity index 97%
rename from contrib/llvm/lib/Target/MBlaze/MBlazeMCCodeEmitter.cpp
rename to contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp
index ddc636d0ce64..1514557bf00b 100644
--- a/contrib/llvm/lib/Target/MBlaze/MBlazeMCCodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp
@@ -12,11 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "mccodeemitter"
-#include "MBlaze.h"
-#include "MBlazeInstrInfo.h"
+#include "MCTargetDesc/MBlazeBaseInfo.h"
+#include "MCTargetDesc/MBlazeMCTargetDesc.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/ADT/Statistic.h"
@@ -106,7 +108,7 @@ MCCodeEmitter *llvm::createMBlazeMCCodeEmitter(const MCInstrInfo &MCII,
 unsigned MBlazeMCCodeEmitter::getMachineOpValue(const MCInst &MI,
                                              const MCOperand &MO) const {
   if (MO.isReg())
-    return MBlazeRegisterInfo::getRegisterNumbering(MO.getReg());
+    return getMBlazeRegisterNumbering(MO.getReg());
   else if (MO.isImm())
     return static_cast<unsigned>(MO.getImm());
   else if (MO.isExpr())
diff --git a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.cpp b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.cpp
index 20d6c0bd2156..43ae281519c2 100644
--- a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.cpp
@@ -13,10 +13,14 @@
 
 #include "MBlazeMCTargetDesc.h"
 #include "MBlazeMCAsmInfo.h"
+#include "InstPrinter/MBlazeInstPrinter.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "MBlazeGenInstrInfo.inc"
@@ -36,8 +40,10 @@ static MCInstrInfo *createMBlazeMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeMBlazeMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheMBlazeTarget, createMBlazeMCInstrInfo);
+static MCRegisterInfo *createMBlazeMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitMBlazeMCRegisterInfo(X, MBlaze::R15);
+  return X;
 }
 
 static MCSubtargetInfo *createMBlazeMCSubtargetInfo(StringRef TT, StringRef CPU,
@@ -47,11 +53,6 @@ static MCSubtargetInfo *createMBlazeMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeMBlazeMCSubtargetInfo() {
-  TargetRegistry::RegisterMCSubtargetInfo(TheMBlazeTarget,
-                                          createMBlazeMCSubtargetInfo);
-}
-
 static MCAsmInfo *createMCAsmInfo(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
   switch (TheTriple.getOS()) {
@@ -60,6 +61,80 @@ static MCAsmInfo *createMCAsmInfo(const Target &T, StringRef TT) {
   }
 }
 
-extern "C" void LLVMInitializeMBlazeMCAsmInfo() {
-  RegisterMCAsmInfoFn X(TheMBlazeTarget, createMCAsmInfo);
+static MCCodeGenInfo *createMBlazeMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                                CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  if (RM == Reloc::Default)
+    RM = Reloc::Static;
+  if (CM == CodeModel::Default)
+    CM = CodeModel::Small;
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+static MCStreamer *createMCStreamer(const Target &T, StringRef TT,
+                                    MCContext &Ctx, MCAsmBackend &MAB,
+                                    raw_ostream &_OS,
+                                    MCCodeEmitter *_Emitter,
+                                    bool RelaxAll,
+                                    bool NoExecStack) {
+  Triple TheTriple(TT);
+
+  if (TheTriple.isOSDarwin()) {
+    llvm_unreachable("MBlaze does not support Darwin MACH-O format");
+    return NULL;
+  }
+
+  if (TheTriple.isOSWindows()) {
+    llvm_unreachable("MBlaze does not support Windows COFF format");
+    return NULL;
+  }
+
+  return createELFStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll, NoExecStack);
+}
+
+static MCInstPrinter *createMBlazeMCInstPrinter(const Target &T,
+                                                unsigned SyntaxVariant,
+                                                const MCAsmInfo &MAI,
+                                                const MCSubtargetInfo &STI) {
+  if (SyntaxVariant == 0)
+    return new MBlazeInstPrinter(MAI);
+  return 0;
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeMBlazeTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfoFn X(TheMBlazeTarget, createMCAsmInfo);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheMBlazeTarget,
+                                        createMBlazeMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheMBlazeTarget, createMBlazeMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheMBlazeTarget,
+                                    createMBlazeMCRegisterInfo);
+
+  // Register the MC subtarget info.
+  TargetRegistry::RegisterMCSubtargetInfo(TheMBlazeTarget,
+                                          createMBlazeMCSubtargetInfo);
+
+  // Register the MC code emitter
+  TargetRegistry::RegisterMCCodeEmitter(TheMBlazeTarget,
+                                        llvm::createMBlazeMCCodeEmitter);
+
+  // Register the asm backend
+  TargetRegistry::RegisterMCAsmBackend(TheMBlazeTarget,
+                                       createMBlazeAsmBackend);
+
+  // Register the object streamer
+  TargetRegistry::RegisterMCObjectStreamer(TheMBlazeTarget,
+                                           createMCStreamer);
+
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(TheMBlazeTarget,
+                                        createMBlazeMCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h
index b14772ef060b..deff5cb078f9 100644
--- a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h
+++ b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h
@@ -15,12 +15,23 @@
 #define MBLAZEMCTARGETDESC_H
 
 namespace llvm {
+class MCAsmBackend;
+class MCContext;
+class MCCodeEmitter;
+class MCInstrInfo;
 class MCSubtargetInfo;
 class Target;
 class StringRef;
+class formatted_raw_ostream;
 
 extern Target TheMBlazeTarget;
 
+MCCodeEmitter *createMBlazeMCCodeEmitter(const MCInstrInfo &MCII,
+                                         const MCSubtargetInfo &STI,
+                                         MCContext &Ctx);
+  
+MCAsmBackend *createMBlazeAsmBackend(const Target &T, StringRef TT);
+
 } // End llvm namespace
 
 // Defines symbolic names for MBlaze registers.  This defines a mapping from
diff --git a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/Makefile
deleted file mode 100644
index 71075ffbf47c..000000000000
--- a/contrib/llvm/lib/Target/MBlaze/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/MBlaze/TargetDesc/Makefile ---------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMBlazeDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/MBlaze/TargetInfo/MBlazeTargetInfo.cpp b/contrib/llvm/lib/Target/MBlaze/TargetInfo/MBlazeTargetInfo.cpp
index 16e01dbfdeed..71210d8db466 100644
--- a/contrib/llvm/lib/Target/MBlaze/TargetInfo/MBlazeTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/MBlaze/TargetInfo/MBlazeTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "MBlaze.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheMBlazeTarget;
diff --git a/contrib/llvm/lib/Target/MSP430/InstPrinter/CMakeLists.txt b/contrib/llvm/lib/Target/MSP430/InstPrinter/CMakeLists.txt
deleted file mode 100644
index f5458d59a821..000000000000
--- a/contrib/llvm/lib/Target/MSP430/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMMSP430AsmPrinter
-  MSP430InstPrinter.cpp
-  )
-add_dependencies(LLVMMSP430AsmPrinter MSP430CodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp b/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
index e10d4fe7ca16..5d6c6ad93dbe 100644
--- a/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
+++ b/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
@@ -25,8 +25,10 @@ using namespace llvm;
 // Include the auto-generated portion of the assembly writer.
 #include "MSP430GenAsmWriter.inc"
 
-void MSP430InstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
+void MSP430InstPrinter::printInst(const MCInst *MI, raw_ostream &O,
+                                  StringRef Annot) {
   printInstruction(MI, O);
+  printAnnotation(O, Annot);
 }
 
 void MSP430InstPrinter::printPCRelImmOperand(const MCInst *MI, unsigned OpNo,
diff --git a/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h b/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h
index 50d98b7c41fd..a1984a8aec19 100644
--- a/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h
+++ b/contrib/llvm/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h
@@ -22,9 +22,9 @@ namespace llvm {
   class MSP430InstPrinter : public MCInstPrinter {
   public:
     MSP430InstPrinter(const MCAsmInfo &MAI)
-      : MCInstPrinter(MAI) {}
+        : MCInstPrinter(MAI) {}
 
-    virtual void printInst(const MCInst *MI, raw_ostream &O);
+    virtual void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot);
 
     // Autogenerated by tblgen.
     void printInstruction(const MCInst *MI, raw_ostream &O);
diff --git a/contrib/llvm/lib/Target/MSP430/InstPrinter/Makefile b/contrib/llvm/lib/Target/MSP430/InstPrinter/Makefile
deleted file mode 100644
index a5293ab8a234..000000000000
--- a/contrib/llvm/lib/Target/MSP430/InstPrinter/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/MSP430/AsmPrinter/Makefile ---------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMSP430AsmPrinter
-
-# Hack: we need to include 'main' MSP430 target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/MSP430/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 0f3ebd303924..000000000000
--- a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMMSP430Desc
-  MSP430MCTargetDesc.cpp
-  MSP430MCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp b/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
index 43a704d7a7df..fda70b81dc8c 100644
--- a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
@@ -13,10 +13,12 @@
 
 #include "MSP430MCTargetDesc.h"
 #include "MSP430MCAsmInfo.h"
+#include "InstPrinter/MSP430InstPrinter.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "MSP430GenInstrInfo.inc"
@@ -29,18 +31,18 @@
 
 using namespace llvm;
 
-
 static MCInstrInfo *createMSP430MCInstrInfo() {
   MCInstrInfo *X = new MCInstrInfo();
   InitMSP430MCInstrInfo(X);
   return X;
 }
 
-extern "C" void LLVMInitializeMSP430MCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheMSP430Target, createMSP430MCInstrInfo);
+static MCRegisterInfo *createMSP430MCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitMSP430MCRegisterInfo(X, MSP430::PCW);
+  return X;
 }
 
-
 static MCSubtargetInfo *createMSP430MCSubtargetInfo(StringRef TT, StringRef CPU,
                                                     StringRef FS) {
   MCSubtargetInfo *X = new MCSubtargetInfo();
@@ -48,11 +50,42 @@ static MCSubtargetInfo *createMSP430MCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeMSP430MCSubtargetInfo() {
-  TargetRegistry::RegisterMCSubtargetInfo(TheMSP430Target,
-                                          createMSP430MCSubtargetInfo);
+static MCCodeGenInfo *createMSP430MCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                                CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
 }
 
-extern "C" void LLVMInitializeMSP430MCAsmInfo() {
-  RegisterMCAsmInfo<MSP430MCAsmInfo> X(TheMSP430Target);
+static MCInstPrinter *createMSP430MCInstPrinter(const Target &T,
+                                                unsigned SyntaxVariant,
+                                                const MCAsmInfo &MAI,
+                                                const MCSubtargetInfo &STI) {
+  if (SyntaxVariant == 0)
+    return new MSP430InstPrinter(MAI);
+  return 0;
+}
+
+extern "C" void LLVMInitializeMSP430TargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfo<MSP430MCAsmInfo> X(TheMSP430Target);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheMSP430Target,
+                                        createMSP430MCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheMSP430Target, createMSP430MCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheMSP430Target,
+                                    createMSP430MCRegisterInfo);
+
+  // Register the MC subtarget info.
+  TargetRegistry::RegisterMCSubtargetInfo(TheMSP430Target,
+                                          createMSP430MCSubtargetInfo);
+
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(TheMSP430Target,
+                                        createMSP430MCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h b/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h
index 0d8a6bdb44f9..35f259076441 100644
--- a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h
+++ b/contrib/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef ALPHAMCTARGETDESC_H
-#define ALPHAMCTARGETDESC_H
+#ifndef MSP430MCTARGETDESC_H
+#define MSP430MCTARGETDESC_H
 
 namespace llvm {
 class MCSubtargetInfo;
diff --git a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/MSP430/MCTargetDesc/Makefile
deleted file mode 100644
index bb857998eef9..000000000000
--- a/contrib/llvm/lib/Target/MSP430/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/MSP430/TargetDesc/Makefile ---------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMSP430Desc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp b/contrib/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp
index 2042056617ac..883654943b64 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp
@@ -32,9 +32,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
@@ -163,17 +161,7 @@ void MSP430AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   OutStreamer.EmitInstruction(TmpInst);
 }
 
-static MCInstPrinter *createMSP430MCInstPrinter(const Target &T,
-                                                unsigned SyntaxVariant,
-                                                const MCAsmInfo &MAI) {
-  if (SyntaxVariant == 0)
-    return new MSP430InstPrinter(MAI);
-  return 0;
-}
-
 // Force static initialization.
 extern "C" void LLVMInitializeMSP430AsmPrinter() {
   RegisterAsmPrinter<MSP430AsmPrinter> X(TheMSP430Target);
-  TargetRegistry::RegisterMCInstPrinter(TheMSP430Target,
-                                        createMSP430MCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index 0a3eab1f7a66..dc374315171f 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -79,6 +79,7 @@ MSP430TargetLowering::MSP430TargetLowering(MSP430TargetMachine &tm) :
 
   setStackPointerRegisterToSaveRestore(MSP430::SPW);
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
   setSchedulingPreference(Sched::Latency);
 
   // We have post-incremented loads / stores.
@@ -987,8 +988,8 @@ const char *MSP430TargetLowering::getTargetNodeName(unsigned Opcode) const {
   }
 }
 
-bool MSP430TargetLowering::isTruncateFree(const Type *Ty1,
-                                          const Type *Ty2) const {
+bool MSP430TargetLowering::isTruncateFree(Type *Ty1,
+                                          Type *Ty2) const {
   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
     return false;
 
@@ -1002,7 +1003,7 @@ bool MSP430TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
   return (VT1.getSizeInBits() > VT2.getSizeInBits());
 }
 
-bool MSP430TargetLowering::isZExtFree(const Type *Ty1, const Type *Ty2) const {
+bool MSP430TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
   // MSP430 implicitly zero-extends 8-bit results in 16-bit registers.
   return 0 && Ty1->isIntegerTy(8) && Ty2->isIntegerTy(16);
 }
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.h b/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.h
index bd660a0bb039..237f60435736 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.h
+++ b/contrib/llvm/lib/Target/MSP430/MSP430ISelLowering.h
@@ -102,7 +102,7 @@ namespace llvm {
     /// isTruncateFree - Return true if it's free to truncate a value of type
     /// Ty1 to type Ty2. e.g. On msp430 it's free to truncate a i16 value in
     /// register R15W to i8 by referencing its sub-register R15B.
-    virtual bool isTruncateFree(const Type *Ty1, const Type *Ty2) const;
+    virtual bool isTruncateFree(Type *Ty1, Type *Ty2) const;
     virtual bool isTruncateFree(EVT VT1, EVT VT2) const;
 
     /// isZExtFree - Return true if any actual instruction that defines a value
@@ -113,7 +113,7 @@ namespace llvm {
     /// necessarily apply to truncate instructions. e.g. on msp430, all
     /// instructions that define 8-bit values implicit zero-extend the result
     /// out to 16 bits.
-    virtual bool isZExtFree(const Type *Ty1, const Type *Ty2) const;
+    virtual bool isZExtFree(Type *Ty1, Type *Ty2) const;
     virtual bool isZExtFree(EVT VT1, EVT VT2) const;
 
     MachineBasicBlock* EmitInstrWithCustomInserter(MachineInstr *MI,
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp b/contrib/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
index 846d09361b33..ffd43183c5d9 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
@@ -20,8 +20,8 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_CTOR
 #include "MSP430GenInstrInfo.inc"
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp b/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp
index 1cc60bba3a55..9049c4bf8f65 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp
@@ -34,7 +34,7 @@ using namespace llvm;
 // FIXME: Provide proper call frame setup / destroy opcodes.
 MSP430RegisterInfo::MSP430RegisterInfo(MSP430TargetMachine &tm,
                                        const TargetInstrInfo &tii)
-  : MSP430GenRegisterInfo(), TM(tm), TII(tii) {
+  : MSP430GenRegisterInfo(MSP430::PCW), TM(tm), TII(tii) {
   StackAlign = TM.getFrameLowering()->getStackAlignment();
 }
 
@@ -233,22 +233,8 @@ MSP430RegisterInfo::processFunctionBeforeFrameFinalized(MachineFunction &MF)
   }
 }
 
-unsigned MSP430RegisterInfo::getRARegister() const {
-  return MSP430::PCW;
-}
-
 unsigned MSP430RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
 
   return TFI->hasFP(MF) ? MSP430::FPW : MSP430::SPW;
 }
-
-int MSP430RegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  llvm_unreachable("Not implemented yet!");
-  return 0;
-}
-
-int MSP430RegisterInfo::getLLVMRegNum(unsigned RegNum, bool isEH) const {
-  llvm_unreachable("Not implemented yet!");
-  return 0;
-}
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.h b/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.h
index fb70594ab37c..10a3d5320636 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.h
+++ b/contrib/llvm/lib/Target/MSP430/MSP430RegisterInfo.h
@@ -58,12 +58,7 @@ struct MSP430RegisterInfo : public MSP430GenRegisterInfo {
   void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
-
-  //! Get DWARF debugging register number
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430Subtarget.cpp b/contrib/llvm/lib/Target/MSP430/MSP430Subtarget.cpp
index b58c50afb982..3ee14d9f7a83 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430Subtarget.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MSP430Subtarget.cpp
@@ -13,7 +13,7 @@
 
 #include "MSP430Subtarget.h"
 #include "MSP430.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp b/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp
index 971f512141e8..4dd893326e3f 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp
+++ b/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp
@@ -16,7 +16,7 @@
 #include "llvm/PassManager.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/MC/MCAsmInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 extern "C" void LLVMInitializeMSP430Target() {
@@ -25,10 +25,11 @@ extern "C" void LLVMInitializeMSP430Target() {
 }
 
 MSP430TargetMachine::MSP430TargetMachine(const Target &T,
-                                         const std::string &TT,
-                                         const std::string &CPU,
-                                         const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+                                         StringRef TT,
+                                         StringRef CPU,
+                                         StringRef FS,
+                                         Reloc::Model RM, CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS),
     // FIXME: Check TargetData string.
     DataLayout("e-p:16:16:16-i8:8:8-i16:16:16-i32:16:32-n8:16"),
diff --git a/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.h b/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.h
index 2a9eea0bcd82..eb483dc8706f 100644
--- a/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.h
+++ b/contrib/llvm/lib/Target/MSP430/MSP430TargetMachine.h
@@ -38,8 +38,9 @@ class MSP430TargetMachine : public LLVMTargetMachine {
   MSP430FrameLowering    FrameLowering;
 
 public:
-  MSP430TargetMachine(const Target &T, const std::string &TT,
-                      const std::string &CPU, const std::string &FS);
+  MSP430TargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM);
 
   virtual const TargetFrameLowering *getFrameLowering() const {
     return &FrameLowering;
diff --git a/contrib/llvm/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp b/contrib/llvm/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
index f9ca5c49c979..8b3e01ecf52c 100644
--- a/contrib/llvm/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
+++ b/contrib/llvm/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "MSP430.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheMSP430Target;
diff --git a/contrib/llvm/lib/Target/Mangler.cpp b/contrib/llvm/lib/Target/Mangler.cpp
index 46c687b64001..53ad155f376c 100644
--- a/contrib/llvm/lib/Target/Mangler.cpp
+++ b/contrib/llvm/lib/Target/Mangler.cpp
@@ -159,7 +159,7 @@ static void AddFastCallStdCallSuffix(SmallVectorImpl<char> &OutName,
   unsigned ArgWords = 0;
   for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end();
        AI != AE; ++AI) {
-    const Type *Ty = AI->getType();
+    Type *Ty = AI->getType();
     // 'Dereference' type in case of byval parameter attribute
     if (AI->hasByValAttr())
       Ty = cast<PointerType>(Ty)->getElementType();
@@ -214,7 +214,7 @@ void Mangler::getNameWithPrefix(SmallVectorImpl<char> &OutName,
     
       // fastcall and stdcall functions usually need @42 at the end to specify
       // the argument info.
-      const FunctionType *FT = F->getFunctionType();
+      FunctionType *FT = F->getFunctionType();
       if ((CC == CallingConv::X86_FastCall || CC == CallingConv::X86_StdCall) &&
           // "Pure" variadic functions do not receive @0 suffix.
           (!FT->isVarArg() || FT->getNumParams() == 0 ||
diff --git a/contrib/llvm/lib/Target/Mips/InstPrinter/CMakeLists.txt b/contrib/llvm/lib/Target/Mips/InstPrinter/CMakeLists.txt
deleted file mode 100644
index 8852fd4126e6..000000000000
--- a/contrib/llvm/lib/Target/Mips/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMMipsAsmPrinter
-  MipsInstPrinter.cpp
-  )
-add_dependencies(LLVMMipsAsmPrinter MipsCodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/Mips/InstPrinter/Makefile b/contrib/llvm/lib/Target/Mips/InstPrinter/Makefile
deleted file mode 100644
index 63e38ef3e6aa..000000000000
--- a/contrib/llvm/lib/Target/Mips/InstPrinter/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/Mips/AsmPrinter/Makefile --------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMipsAsmPrinter
-
-# Hack: we need to include 'main' arm target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp b/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
index 41c1dd3919b4..3dafc6134488 100644
--- a/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
+++ b/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
@@ -1,4 +1,4 @@
-//===-- MipsInstPrinter.cpp - Convert Mips MCInst to assembly syntax --------===//
+//===-- MipsInstPrinter.cpp - Convert Mips MCInst to assembly syntax ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -69,8 +69,10 @@ void MipsInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   OS << '$' << LowercaseString(getRegisterName(RegNo));
 }
 
-void MipsInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
+void MipsInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
+                                StringRef Annot) {
   printInstruction(MI, O);
+  printAnnotation(O, Annot);
 }
 
 void MipsInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
diff --git a/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.h b/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.h
index 680208eb819b..5c1116538c61 100644
--- a/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.h
+++ b/contrib/llvm/lib/Target/Mips/InstPrinter/MipsInstPrinter.h
@@ -1,4 +1,4 @@
-//===-- MipsInstPrinter.h - Convert Mips MCInst to assembly syntax ----------===//
+//===-- MipsInstPrinter.h - Convert Mips MCInst to assembly syntax --------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -86,7 +86,7 @@ class MipsInstPrinter : public MCInstPrinter {
   
   virtual StringRef getOpcodeName(unsigned Opcode) const;
   virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
-  virtual void printInst(const MCInst *MI, raw_ostream &O);
+  virtual void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot);
   
 private:
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 97de75db5347..000000000000
--- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMMipsDesc
-  MipsMCTargetDesc.cpp
-  MipsMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/Mips/MCTargetDesc/Makefile
deleted file mode 100644
index 7fe2086a6e00..000000000000
--- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/Mips/TargetDesc/Makefile -----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMMipsDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
new file mode 100644
index 000000000000..f190ec42c776
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -0,0 +1,117 @@
+#include "MCTargetDesc/MipsMCTargetDesc.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCDirectives.h"
+#include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCMachObjectWriter.h"
+#include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSectionMachO.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Object/MachOFormat.h"
+#include "llvm/Support/ELF.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+namespace {
+class MipsELFObjectWriter : public MCELFObjectTargetWriter {
+public:
+  MipsELFObjectWriter(bool is64Bit, Triple::OSType OSType, uint16_t EMachine,
+                      bool HasRelocationAddend)
+    : MCELFObjectTargetWriter(is64Bit, OSType, EMachine,
+                              HasRelocationAddend) {}
+};
+
+class MipsAsmBackend : public MCAsmBackend {
+public:
+  MipsAsmBackend(const Target &T)
+    : MCAsmBackend() {}
+
+  unsigned getNumFixupKinds() const {
+    return 1;   //tbd
+  }
+
+  /// ApplyFixup - Apply the \arg Value for given \arg Fixup into the provided
+  /// data fragment, at the offset specified by the fixup and following the
+  /// fixup kind as appropriate.
+  void ApplyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
+                  uint64_t Value) const {
+  }
+
+  /// @name Target Relaxation Interfaces
+  /// @{
+
+  /// MayNeedRelaxation - Check whether the given instruction may need
+  /// relaxation.
+  ///
+  /// \param Inst - The instruction to test.
+  bool MayNeedRelaxation(const MCInst &Inst) const {
+    return false;
+  }
+
+  /// RelaxInstruction - Relax the instruction in the given fragment to the next
+  /// wider instruction.
+  ///
+  /// \param Inst - The instruction to relax, which may be the same as the
+  /// output.
+  /// \parm Res [output] - On return, the relaxed instruction.
+  void RelaxInstruction(const MCInst &Inst, MCInst &Res) const {
+  }
+  
+  /// @}
+
+  /// WriteNopData - Write an (optimal) nop sequence of Count bytes to the given
+  /// output. If the target cannot generate such a sequence, it should return an
+  /// error.
+  ///
+  /// \return - True on success.
+  bool WriteNopData(uint64_t Count, MCObjectWriter *OW) const {
+    return false;
+  }
+};
+
+class MipsEB_AsmBackend : public MipsAsmBackend {
+public:
+  Triple::OSType OSType;
+
+  MipsEB_AsmBackend(const Target &T, Triple::OSType _OSType)
+    : MipsAsmBackend(T), OSType(_OSType) {}
+
+  MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
+    return createELFObjectWriter(createELFObjectTargetWriter(),
+                                 OS, /*IsLittleEndian*/ false);
+  }
+
+  MCELFObjectTargetWriter *createELFObjectTargetWriter() const {
+    return new MipsELFObjectWriter(false, OSType, ELF::EM_MIPS, false);
+  }
+};
+
+class MipsEL_AsmBackend : public MipsAsmBackend {
+public:
+  Triple::OSType OSType;
+
+  MipsEL_AsmBackend(const Target &T, Triple::OSType _OSType)
+    : MipsAsmBackend(T), OSType(_OSType) {}
+
+  MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
+    return createELFObjectWriter(createELFObjectTargetWriter(),
+                                 OS, /*IsLittleEndian*/ true);
+  }
+
+  MCELFObjectTargetWriter *createELFObjectTargetWriter() const {
+    return new MipsELFObjectWriter(false, OSType, ELF::EM_MIPS, false);
+  }
+};
+}
+
+MCAsmBackend *llvm::createMipsAsmBackend(const Target &T, StringRef TT) {
+  Triple TheTriple(TT);
+
+  // just return little endian for now
+  //
+  return new MipsEL_AsmBackend(T, Triple(TT).getOS());
+}
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
new file mode 100644
index 000000000000..f7a6fa949091
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
@@ -0,0 +1,113 @@
+//===-- MipsBaseInfo.h - Top level definitions for ARM ------- --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains small standalone helper functions and enum definitions for
+// the Mips target useful for the compiler back-end and the MC libraries.
+//
+//===----------------------------------------------------------------------===//
+#ifndef MIPSBASEINFO_H
+#define MIPSBASEINFO_H
+
+#include "MipsMCTargetDesc.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
+
+namespace llvm {
+/// getMipsRegisterNumbering - Given the enum value for some register,
+/// return the number that it corresponds to.
+inline static unsigned getMipsRegisterNumbering(unsigned RegEnum)
+{
+  switch (RegEnum) {
+  case Mips::ZERO: case Mips::ZERO_64: case Mips::F0: case Mips::D0_64:
+  case Mips::D0:
+    return 0;
+  case Mips::AT: case Mips::AT_64: case Mips::F1: case Mips::D1_64:
+    return 1;
+  case Mips::V0: case Mips::V0_64: case Mips::F2: case Mips::D2_64:
+  case Mips::D1:
+    return 2;
+  case Mips::V1: case Mips::V1_64: case Mips::F3: case Mips::D3_64:
+    return 3;
+  case Mips::A0: case Mips::A0_64: case Mips::F4: case Mips::D4_64:
+  case Mips::D2:
+    return 4;
+  case Mips::A1: case Mips::A1_64: case Mips::F5: case Mips::D5_64:
+    return 5;
+  case Mips::A2: case Mips::A2_64: case Mips::F6: case Mips::D6_64:
+  case Mips::D3:
+    return 6;
+  case Mips::A3: case Mips::A3_64: case Mips::F7: case Mips::D7_64:
+    return 7;
+  case Mips::T0: case Mips::T0_64: case Mips::F8: case Mips::D8_64:
+  case Mips::D4:
+    return 8;
+  case Mips::T1: case Mips::T1_64: case Mips::F9: case Mips::D9_64:
+    return 9;
+  case Mips::T2: case Mips::T2_64: case Mips::F10: case Mips::D10_64:
+  case Mips::D5:
+    return 10;
+  case Mips::T3: case Mips::T3_64: case Mips::F11: case Mips::D11_64:
+    return 11;
+  case Mips::T4: case Mips::T4_64: case Mips::F12: case Mips::D12_64:
+  case Mips::D6:
+    return 12;
+  case Mips::T5: case Mips::T5_64: case Mips::F13: case Mips::D13_64:
+    return 13;
+  case Mips::T6: case Mips::T6_64: case Mips::F14: case Mips::D14_64:
+  case Mips::D7:
+    return 14;
+  case Mips::T7: case Mips::T7_64: case Mips::F15: case Mips::D15_64:
+    return 15;
+  case Mips::S0: case Mips::S0_64: case Mips::F16: case Mips::D16_64:
+  case Mips::D8:
+    return 16;
+  case Mips::S1: case Mips::S1_64: case Mips::F17: case Mips::D17_64:
+    return 17;
+  case Mips::S2: case Mips::S2_64: case Mips::F18: case Mips::D18_64:
+  case Mips::D9:
+    return 18;
+  case Mips::S3: case Mips::S3_64: case Mips::F19: case Mips::D19_64:
+    return 19;
+  case Mips::S4: case Mips::S4_64: case Mips::F20: case Mips::D20_64:
+  case Mips::D10:
+    return 20;
+  case Mips::S5: case Mips::S5_64: case Mips::F21: case Mips::D21_64:
+    return 21;
+  case Mips::S6: case Mips::S6_64: case Mips::F22: case Mips::D22_64:
+  case Mips::D11:
+    return 22;
+  case Mips::S7: case Mips::S7_64: case Mips::F23: case Mips::D23_64:
+    return 23;
+  case Mips::T8: case Mips::T8_64: case Mips::F24: case Mips::D24_64:
+  case Mips::D12:
+    return 24;
+  case Mips::T9: case Mips::T9_64: case Mips::F25: case Mips::D25_64:
+    return 25;
+  case Mips::K0: case Mips::K0_64: case Mips::F26: case Mips::D26_64:
+  case Mips::D13:
+    return 26;
+  case Mips::K1: case Mips::K1_64: case Mips::F27: case Mips::D27_64:
+    return 27;
+  case Mips::GP: case Mips::GP_64: case Mips::F28: case Mips::D28_64:
+  case Mips::D14:
+    return 28;
+  case Mips::SP: case Mips::SP_64: case Mips::F29: case Mips::D29_64:
+    return 29;
+  case Mips::FP: case Mips::FP_64: case Mips::F30: case Mips::D30_64:
+  case Mips::D15: 
+    return 30;
+  case Mips::RA: case Mips::RA_64: case Mips::F31: case Mips::D31_64:
+    return 31;
+  default: llvm_unreachable("Unknown register number!");
+  }
+  return 0; // Not reached
+}
+}
+
+#endif
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h
new file mode 100644
index 000000000000..8b099eab91fd
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h
@@ -0,0 +1,90 @@
+#ifndef LLVM_Mips_MipsFIXUPKINDS_H
+#define LLVM_Mips_MipsFIXUPKINDS_H
+
+//===-- Mips/MipsFixupKinds.h - Mips Specific Fixup Entries --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+#include "llvm/MC/MCFixup.h"
+
+namespace llvm {
+namespace Mips {
+    enum Fixups {
+        // fixup_Mips_xxx - R_MIPS_NONE
+        fixup_Mips_NONE = FirstTargetFixupKind,
+
+        // fixup_Mips_xxx - R_MIPS_16.
+        fixup_Mips_16,
+
+        // fixup_Mips_xxx - R_MIPS_32.
+        fixup_Mips_32,
+
+        // fixup_Mips_xxx - R_MIPS_REL32.
+        fixup_Mips_REL32,
+
+        // fixup_Mips_xxx - R_MIPS_26.
+        fixup_Mips_26,
+
+        // fixup_Mips_xxx - R_MIPS_HI16.
+        fixup_Mips_HI16,
+
+        // fixup_Mips_xxx - R_MIPS_LO16.
+        fixup_Mips_LO16,
+
+        // fixup_Mips_xxx - R_MIPS_GPREL16.
+        fixup_Mips_GPREL16,
+
+        // fixup_Mips_xxx - R_MIPS_LITERAL.
+        fixup_Mips_LITERAL,
+
+        // fixup_Mips_xxx - R_MIPS_GOT16.
+        fixup_Mips_GOT16,
+
+        // fixup_Mips_xxx - R_MIPS_PC16.
+        fixup_Mips_PC16,
+
+        // fixup_Mips_xxx - R_MIPS_CALL16.
+        fixup_Mips_CALL16,
+
+        // fixup_Mips_xxx - R_MIPS_GPREL32.
+        fixup_Mips_GPREL32,
+
+        // fixup_Mips_xxx - R_MIPS_SHIFT5.
+        fixup_Mips_SHIFT5,
+
+        // fixup_Mips_xxx - R_MIPS_SHIFT6.
+        fixup_Mips_SHIFT6,
+
+        // fixup_Mips_xxx - R_MIPS_64.
+        fixup_Mips_64,
+
+        // fixup_Mips_xxx - R_MIPS_TLS_GD.
+        fixup_Mips_TLSGD,
+
+        // fixup_Mips_xxx - R_MIPS_TLS_GOTTPREL.
+        fixup_Mips_GOTTPREL,
+
+        // fixup_Mips_xxx - R_MIPS_TLS_TPREL_HI16.
+        fixup_Mips_TPREL_HI,
+
+        // fixup_Mips_xxx - R_MIPS_TLS_TPREL_LO16.
+        fixup_Mips_TPREL_LO,
+
+        // fixup_Mips_xxx - yyy. // This should become R_MIPS_PC16
+        fixup_Mips_Branch_PCRel,
+
+        // Marker
+        LastTargetFixupKind,
+        NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
+    };
+} // namespace llvm
+} // namespace Mips
+
+
+#endif /* LLVM_Mips_MipsFIXUPKINDS_H */
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
index 5d9242500f6d..71ae80498995 100644
--- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
@@ -18,7 +18,8 @@ using namespace llvm;
 
 MipsMCAsmInfo::MipsMCAsmInfo(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
-  if (TheTriple.getArch() == Triple::mips)
+  if ((TheTriple.getArch() == Triple::mips) ||
+      (TheTriple.getArch() == Triple::mips64))
     IsLittleEndian = false;
 
   AlignmentIsInBytes          = false;
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
new file mode 100644
index 000000000000..d66de23ba115
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -0,0 +1,52 @@
+//===-- MipsMCCodeEmitter.cpp - Convert Mips code to machine code ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MipsMCCodeEmitter class.
+//
+//===----------------------------------------------------------------------===//
+//
+#define DEBUG_TYPE "mccodeemitter"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Support/raw_ostream.h"
+#include "MCTargetDesc/MipsMCTargetDesc.h"
+
+using namespace llvm;
+
+namespace {
+class MipsMCCodeEmitter : public MCCodeEmitter {
+  MipsMCCodeEmitter(const MipsMCCodeEmitter &); // DO NOT IMPLEMENT
+  void operator=(const MipsMCCodeEmitter &); // DO NOT IMPLEMENT
+  const MCInstrInfo &MCII;
+  const MCSubtargetInfo &STI;
+
+public:
+  MipsMCCodeEmitter(const MCInstrInfo &mcii, const MCSubtargetInfo &sti,
+                    MCContext &ctx)
+    : MCII(mcii), STI(sti) {}
+
+  ~MipsMCCodeEmitter() {}
+
+  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+                         SmallVectorImpl<MCFixup> &Fixups) const {
+  }
+}; // class MipsMCCodeEmitter
+}  // namespace
+
+MCCodeEmitter *llvm::createMipsMCCodeEmitter(const MCInstrInfo &MCII,
+                                             const MCSubtargetInfo &STI,
+                                             MCContext &Ctx) {
+  return new MipsMCCodeEmitter(MCII, STI, Ctx);
+}
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
index 06f0d0bfb6b9..1f9e3ddf13c8 100644
--- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
@@ -13,10 +13,14 @@
 
 #include "MipsMCTargetDesc.h"
 #include "MipsMCAsmInfo.h"
+#include "InstPrinter/MipsInstPrinter.h"
+#include "llvm/MC/MachineLocation.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "MipsGenInstrInfo.inc"
@@ -35,11 +39,12 @@ static MCInstrInfo *createMipsMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeMipsMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheMipsTarget, createMipsMCInstrInfo);
+static MCRegisterInfo *createMipsMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitMipsMCRegisterInfo(X, Mips::RA);
+  return X;
 }
 
-
 static MCSubtargetInfo *createMipsMCSubtargetInfo(StringRef TT, StringRef CPU,
                                                   StringRef FS) {
   MCSubtargetInfo *X = new MCSubtargetInfo();
@@ -47,12 +52,111 @@ static MCSubtargetInfo *createMipsMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeMipsMCSubtargetInfo() {
-  TargetRegistry::RegisterMCSubtargetInfo(TheMipsTarget,
-                                          createMipsMCSubtargetInfo);
+static MCAsmInfo *createMipsMCAsmInfo(const Target &T, StringRef TT) {
+  MCAsmInfo *MAI = new MipsMCAsmInfo(T, TT);
+
+  MachineLocation Dst(MachineLocation::VirtualFP);
+  MachineLocation Src(Mips::SP, 0);
+  MAI->addInitialFrameState(0, Dst, Src);
+
+  return MAI;
 }
 
-extern "C" void LLVMInitializeMipsMCAsmInfo() {
-  RegisterMCAsmInfo<MipsMCAsmInfo> X(TheMipsTarget);
-  RegisterMCAsmInfo<MipsMCAsmInfo> Y(TheMipselTarget);
+static MCCodeGenInfo *createMipsMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                              CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  if (RM == Reloc::Default)
+    RM = Reloc::PIC_;
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+static MCInstPrinter *createMipsMCInstPrinter(const Target &T,
+                                              unsigned SyntaxVariant,
+                                              const MCAsmInfo &MAI,
+                                              const MCSubtargetInfo &STI) {
+  return new MipsInstPrinter(MAI);
+}
+
+static MCStreamer *createMCStreamer(const Target &T, StringRef TT,
+                                    MCContext &Ctx, MCAsmBackend &MAB,
+                                    raw_ostream &_OS,
+                                    MCCodeEmitter *_Emitter,
+                                    bool RelaxAll,
+                                    bool NoExecStack) {
+  Triple TheTriple(TT);
+
+  return createELFStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll, NoExecStack);
+}
+
+extern "C" void LLVMInitializeMipsTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfoFn X(TheMipsTarget, createMipsMCAsmInfo);
+  RegisterMCAsmInfoFn Y(TheMipselTarget, createMipsMCAsmInfo);
+  RegisterMCAsmInfoFn A(TheMips64Target, createMipsMCAsmInfo);
+  RegisterMCAsmInfoFn B(TheMips64elTarget, createMipsMCAsmInfo);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheMipsTarget,
+                                        createMipsMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(TheMipselTarget,
+                                        createMipsMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(TheMips64Target,
+                                        createMipsMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(TheMips64elTarget,
+                                        createMipsMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheMipsTarget, createMipsMCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(TheMipselTarget, createMipsMCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(TheMips64Target, createMipsMCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(TheMips64elTarget, createMipsMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheMipsTarget, createMipsMCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(TheMipselTarget, createMipsMCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(TheMips64Target, createMipsMCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(TheMips64elTarget,
+                                    createMipsMCRegisterInfo);
+
+  // Register the MC Code Emitter
+  TargetRegistry::RegisterMCCodeEmitter(TheMipsTarget, createMipsMCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(TheMipselTarget,
+                                        createMipsMCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(TheMips64Target,
+                                        createMipsMCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(TheMips64elTarget,
+                                        createMipsMCCodeEmitter);
+
+  // Register the object streamer.
+  TargetRegistry::RegisterMCObjectStreamer(TheMipsTarget, createMCStreamer);
+  TargetRegistry::RegisterMCObjectStreamer(TheMipselTarget, createMCStreamer);
+  TargetRegistry::RegisterMCObjectStreamer(TheMips64Target, createMCStreamer);
+  TargetRegistry::RegisterMCObjectStreamer(TheMips64elTarget, createMCStreamer);
+
+  // Register the asm backend.
+  TargetRegistry::RegisterMCAsmBackend(TheMipsTarget, createMipsAsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(TheMipselTarget, createMipsAsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(TheMips64Target, createMipsAsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(TheMips64elTarget, createMipsAsmBackend);
+
+  // Register the MC subtarget info.
+  TargetRegistry::RegisterMCSubtargetInfo(TheMipsTarget,
+                                          createMipsMCSubtargetInfo);
+  TargetRegistry::RegisterMCSubtargetInfo(TheMipselTarget,
+                                          createMipsMCSubtargetInfo);
+  TargetRegistry::RegisterMCSubtargetInfo(TheMips64Target,
+                                          createMipsMCSubtargetInfo);
+  TargetRegistry::RegisterMCSubtargetInfo(TheMips64elTarget,
+                                          createMipsMCSubtargetInfo);
+
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(TheMipsTarget,
+                                        createMipsMCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(TheMipselTarget,
+                                        createMipsMCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(TheMips64Target,
+                                        createMipsMCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(TheMips64elTarget,
+                                        createMipsMCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
index 3d18f114c8bd..7a0042ad889e 100644
--- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
+++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
@@ -1,4 +1,4 @@
-//===-- AlphaMCTargetDesc.h - Alpha Target Descriptions ---------*- C++ -*-===//
+//===-- MipsMCTargetDesc.h - Mips Target Descriptions -----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,21 +7,32 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file provides Alpha specific target descriptions.
+// This file provides Mips specific target descriptions.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef ALPHAMCTARGETDESC_H
-#define ALPHAMCTARGETDESC_H
+#ifndef MIPSMCTARGETDESC_H
+#define MIPSMCTARGETDESC_H
 
 namespace llvm {
+class MCAsmBackend;
+class MCInstrInfo;
+class MCCodeEmitter;
+class MCContext;
 class MCSubtargetInfo;
-class Target;
 class StringRef;
+class Target;
 
 extern Target TheMipsTarget;
 extern Target TheMipselTarget;
+extern Target TheMips64Target;
+extern Target TheMips64elTarget;
 
+MCCodeEmitter *createMipsMCCodeEmitter(const MCInstrInfo &MCII,
+                                       const MCSubtargetInfo &STI,
+                                       MCContext &Ctx);
+
+MCAsmBackend *createMipsAsmBackend(const Target &T, StringRef TT);
 } // End llvm namespace
 
 // Defines symbolic names for Mips registers.  This defines a mapping from
diff --git a/contrib/llvm/lib/Target/Mips/Mips.h b/contrib/llvm/lib/Target/Mips/Mips.h
index 984b5adfc5f3..bacecf20b920 100644
--- a/contrib/llvm/lib/Target/Mips/Mips.h
+++ b/contrib/llvm/lib/Target/Mips/Mips.h
@@ -29,6 +29,9 @@ namespace llvm {
   FunctionPass *createMipsExpandPseudoPass(MipsTargetMachine &TM);
   FunctionPass *createMipsEmitGPRestorePass(MipsTargetMachine &TM);
 
+  FunctionPass *createMipsJITCodeEmitterPass(MipsTargetMachine &TM,
+                                             JITCodeEmitter &JCE);
+
 } // end namespace llvm;
 
 #endif
diff --git a/contrib/llvm/lib/Target/Mips/Mips.td b/contrib/llvm/lib/Target/Mips/Mips.td
index 433cd57f34e0..39c2c164f664 100644
--- a/contrib/llvm/lib/Target/Mips/Mips.td
+++ b/contrib/llvm/lib/Target/Mips/Mips.td
@@ -38,6 +38,10 @@ def FeatureSingleFloat : SubtargetFeature<"single-float", "IsSingleFloat",
                                 "true", "Only supports single precision float">;
 def FeatureO32         : SubtargetFeature<"o32", "MipsABI", "O32",
                                 "Enable o32 ABI">;
+def FeatureN32         : SubtargetFeature<"n32", "MipsABI", "N32",
+                                "Enable n32 ABI">;
+def FeatureN64         : SubtargetFeature<"n64", "MipsABI", "N64",
+                                "Enable n64 ABI">;
 def FeatureEABI        : SubtargetFeature<"eabi", "MipsABI", "EABI",
                                 "Enable eabi ABI">;
 def FeatureVFPU        : SubtargetFeature<"vfpu", "HasVFPU",
@@ -54,16 +58,19 @@ def FeatureSwap        : SubtargetFeature<"swap", "HasSwap", "true",
                                 "Enable 'byte/half swap' instructions.">;
 def FeatureBitCount    : SubtargetFeature<"bitcount", "HasBitCount", "true",
                                 "Enable 'count leading bits' instructions.">;
-def FeatureMips1       : SubtargetFeature<"mips1", "MipsArchVersion", "Mips1",
-                                "Mips1 ISA Support">;
-def FeatureMips2       : SubtargetFeature<"mips2", "MipsArchVersion", "Mips2",
-                                "Mips2 ISA Support">;
 def FeatureMips32      : SubtargetFeature<"mips32", "MipsArchVersion", "Mips32",
                                 "Mips32 ISA Support",
                                 [FeatureCondMov, FeatureBitCount]>;
 def FeatureMips32r2    : SubtargetFeature<"mips32r2", "MipsArchVersion",
                                 "Mips32r2", "Mips32r2 ISA Support",
                                 [FeatureMips32, FeatureSEInReg]>;
+def FeatureMips64      : SubtargetFeature<"mips64", "MipsArchVersion",
+                                "Mips64", "Mips64 ISA Support",
+                                [FeatureGP64Bit, FeatureFP64Bit,
+                                 FeatureMips32]>;
+def FeatureMips64r2    : SubtargetFeature<"mips64r2", "MipsArchVersion",
+                                "Mips64r2", "Mips64r2 ISA Support",
+                                [FeatureMips64, FeatureMips32r2]>;
 
 //===----------------------------------------------------------------------===//
 // Mips processors supported.
@@ -72,21 +79,10 @@ def FeatureMips32r2    : SubtargetFeature<"mips32r2", "MipsArchVersion",
 class Proc<string Name, list<SubtargetFeature> Features>
  : Processor<Name, MipsGenericItineraries, Features>;
 
-def : Proc<"mips1", [FeatureMips1]>;
-def : Proc<"r2000", [FeatureMips1]>;
-def : Proc<"r3000", [FeatureMips1]>;
-
-def : Proc<"mips2", [FeatureMips2]>;
-def : Proc<"r6000", [FeatureMips2]>;
-
+def : Proc<"mips32r1", [FeatureMips32]>;
 def : Proc<"4ke", [FeatureMips32r2]>;
-
-// Allegrex is a 32bit subset of r4000, both for integer and fp registers,
-// but much more similar to Mips2 than Mips3. It also contains some of
-// Mips32/Mips32r2 instructions and a custom vector fpu processor.
-def : Proc<"allegrex", [FeatureMips2, FeatureSingleFloat, FeatureEABI,
-      FeatureVFPU, FeatureSEInReg, FeatureCondMov, FeatureMulDivAdd,
-      FeatureMinMax, FeatureSwap, FeatureBitCount]>;
+def : Proc<"mips64r1", [FeatureMips64]>;
+def : Proc<"mips64r2", [FeatureMips64r2]>;
 
 def MipsAsmWriter : AsmWriter {
   string AsmWriterClassName  = "InstPrinter";
diff --git a/contrib/llvm/lib/Target/Mips/Mips64InstrInfo.td b/contrib/llvm/lib/Target/Mips/Mips64InstrInfo.td
new file mode 100644
index 000000000000..49b0223643bb
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/Mips64InstrInfo.td
@@ -0,0 +1,214 @@
+//===- Mips64InstrInfo.td - Mips64 Instruction Information -*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes Mips64 instructions.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Mips Operand, Complex Patterns and Transformations Definitions.
+//===----------------------------------------------------------------------===//
+
+// Instruction operand types
+def shamt_64       : Operand<i64>;
+
+// Unsigned Operand
+def uimm16_64      : Operand<i64> {
+  let PrintMethod = "printUnsignedImm";
+}
+
+// Transformation Function - get Imm - 32.
+def Subtract32 : SDNodeXForm<imm, [{
+  return getI32Imm((unsigned)N->getZExtValue() - 32);
+}]>;
+
+// imm32_63 predicate - True if imm is in range [32, 63].
+def imm32_63 : ImmLeaf<i64,
+                       [{return (int32_t)Imm >= 32 && (int32_t)Imm < 64;}],
+                       Subtract32>;
+
+//===----------------------------------------------------------------------===//
+// Instructions specific format
+//===----------------------------------------------------------------------===//
+// Shifts
+class LogicR_shift_rotate_imm64<bits<6> func, bits<5> _rs, string instr_asm,
+                                SDNode OpNode, PatFrag PF>:
+  FR<0x00, func, (outs CPU64Regs:$dst), (ins CPU64Regs:$b, shamt_64:$c),
+     !strconcat(instr_asm, "\t$dst, $b, $c"),
+     [(set CPU64Regs:$dst, (OpNode CPU64Regs:$b, (i64 PF:$c)))],
+     IIAlu> {
+  let rs = _rs;
+}
+
+class LogicR_shift_rotate_reg64<bits<6> func, bits<5> _shamt, string instr_asm,
+                                SDNode OpNode>:
+  FR<0x00, func, (outs CPU64Regs:$dst), (ins CPU64Regs:$c, CPU64Regs:$b),
+     !strconcat(instr_asm, "\t$dst, $b, $c"),
+     [(set CPU64Regs:$dst, (OpNode CPU64Regs:$b, CPU64Regs:$c))], IIAlu> {
+  let shamt = _shamt;
+}
+
+// Mul, Div
+let Defs = [HI64, LO64] in {
+  let isCommutable = 1 in
+  class Mul64<bits<6> func, string instr_asm, InstrItinClass itin>:
+    FR<0x00, func, (outs), (ins CPU64Regs:$a, CPU64Regs:$b),
+       !strconcat(instr_asm, "\t$a, $b"), [], itin>;
+
+  class Div64<SDNode op, bits<6> func, string instr_asm, InstrItinClass itin>:
+              FR<0x00, func, (outs), (ins CPU64Regs:$a, CPU64Regs:$b),
+              !strconcat(instr_asm, "\t$$zero, $a, $b"),
+              [(op CPU64Regs:$a, CPU64Regs:$b)], itin>;
+}
+
+// Move from Hi/Lo
+let shamt = 0 in {
+let rs = 0, rt = 0 in
+class MoveFromLOHI64<bits<6> func, string instr_asm>:
+  FR<0x00, func, (outs CPU64Regs:$dst), (ins),
+     !strconcat(instr_asm, "\t$dst"), [], IIHiLo>;
+
+let rt = 0, rd = 0 in
+class MoveToLOHI64<bits<6> func, string instr_asm>:
+  FR<0x00, func, (outs), (ins CPU64Regs:$src),
+     !strconcat(instr_asm, "\t$src"), [], IIHiLo>;
+}
+
+// Count Leading Ones/Zeros in Word
+class CountLeading64<bits<6> func, string instr_asm, list<dag> pattern>:
+  FR<0x1c, func, (outs CPU64Regs:$dst), (ins CPU64Regs:$src),
+     !strconcat(instr_asm, "\t$dst, $src"), pattern, IIAlu>,
+     Requires<[HasBitCount]> {
+  let shamt = 0;
+  let rt = rd;
+}
+
+//===----------------------------------------------------------------------===//
+// Instruction definition
+//===----------------------------------------------------------------------===//
+
+/// Arithmetic Instructions (ALU Immediate)
+def DADDiu   : ArithLogicI<0x19, "daddiu", add, simm16_64, immSExt16,
+                           CPU64Regs>;
+def DANDi    : ArithLogicI<0x0c, "andi", and, uimm16_64, immZExt16, CPU64Regs>;
+def SLTi64   : SetCC_I<0x0a, "slti", setlt, simm16_64, immSExt16, CPU64Regs>;
+def SLTiu64  : SetCC_I<0x0b, "sltiu", setult, simm16_64, immSExt16, CPU64Regs>;
+def ORi64    : ArithLogicI<0x0d, "ori", or, uimm16_64, immZExt16, CPU64Regs>;
+def XORi64   : ArithLogicI<0x0e, "xori", xor, uimm16_64, immZExt16, CPU64Regs>;
+
+/// Arithmetic Instructions (3-Operand, R-Type)
+def DADDu    : ArithLogicR<0x00, 0x2d, "daddu", add, IIAlu, CPU64Regs, 1>;
+def DSUBu    : ArithLogicR<0x00, 0x2f, "dsubu", sub, IIAlu, CPU64Regs>;
+def SLT64    : SetCC_R<0x00, 0x2a, "slt", setlt, CPU64Regs>;
+def SLTu64   : SetCC_R<0x00, 0x2b, "sltu", setult, CPU64Regs>;
+def AND64    : ArithLogicR<0x00, 0x24, "and", and, IIAlu, CPU64Regs, 1>;
+def OR64     : ArithLogicR<0x00, 0x25, "or", or, IIAlu, CPU64Regs, 1>;
+def XOR64    : ArithLogicR<0x00, 0x26, "xor", xor, IIAlu, CPU64Regs, 1>;
+def NOR64    : LogicNOR<0x00, 0x27, "nor", CPU64Regs>;
+
+/// Shift Instructions
+def DSLL     : LogicR_shift_rotate_imm64<0x38, 0x00, "dsll", shl, immZExt5>;
+def DSRL     : LogicR_shift_rotate_imm64<0x3a, 0x00, "dsrl", srl, immZExt5>;
+def DSRA     : LogicR_shift_rotate_imm64<0x3b, 0x00, "dsra", sra, immZExt5>;
+def DSLL32   : LogicR_shift_rotate_imm64<0x3c, 0x00, "dsll32", shl, imm32_63>;
+def DSRL32   : LogicR_shift_rotate_imm64<0x3e, 0x00, "dsrl32", srl, imm32_63>;
+def DSRA32   : LogicR_shift_rotate_imm64<0x3f, 0x00, "dsra32", sra, imm32_63>;
+def DSLLV    : LogicR_shift_rotate_reg64<0x24, 0x00, "dsllv", shl>;
+def DSRLV    : LogicR_shift_rotate_reg64<0x26, 0x00, "dsrlv", srl>;
+def DSRAV    : LogicR_shift_rotate_reg64<0x27, 0x00, "dsrav", sra>;
+
+// Rotate Instructions
+let Predicates = [HasMips64r2] in {
+  def DROTR    : LogicR_shift_rotate_imm64<0x3a, 0x01, "drotr", rotr, immZExt5>;
+  def DROTR32  : LogicR_shift_rotate_imm64<0x3e, 0x01, "drotr32", rotr,
+                                           imm32_63>;
+  def DROTRV   : LogicR_shift_rotate_reg64<0x16, 0x01, "drotrv", rotr>;
+}
+
+/// Load and Store Instructions
+///  aligned 
+defm LB64    : LoadM64<0x20, "lb",  sextloadi8>;
+defm LBu64   : LoadM64<0x24, "lbu", zextloadi8>;
+defm LH64    : LoadM64<0x21, "lh",  sextloadi16_a>;
+defm LHu64   : LoadM64<0x25, "lhu", zextloadi16_a>;
+defm LW64    : LoadM64<0x23, "lw",  sextloadi32_a>;
+defm LWu64   : LoadM64<0x27, "lwu", zextloadi32_a>;
+defm SB64    : StoreM64<0x28, "sb", truncstorei8>;
+defm SH64    : StoreM64<0x29, "sh", truncstorei16_a>;
+defm SW64    : StoreM64<0x2b, "sw", truncstorei32_a>;
+defm LD      : LoadM64<0x37, "ld",  load_a>;
+defm SD      : StoreM64<0x3f, "sd", store_a>;
+
+///  unaligned
+defm ULH64     : LoadM64<0x21, "ulh",  sextloadi16_u, 1>;
+defm ULHu64    : LoadM64<0x25, "ulhu", zextloadi16_u, 1>;
+defm ULW64     : LoadM64<0x23, "ulw",  sextloadi32_u, 1>;
+defm USH64     : StoreM64<0x29, "ush", truncstorei16_u, 1>;
+defm USW64     : StoreM64<0x2b, "usw", truncstorei32_u, 1>;
+defm ULD       : LoadM64<0x37, "uld",  load_u, 1>;
+defm USD       : StoreM64<0x3f, "usd", store_u, 1>;
+
+/// Jump and Branch Instructions
+def BEQ64  : CBranch<0x04, "beq", seteq, CPU64Regs>;
+def BNE64  : CBranch<0x05, "bne", setne, CPU64Regs>;
+def BGEZ64 : CBranchZero<0x01, 1, "bgez", setge, CPU64Regs>;
+def BGTZ64 : CBranchZero<0x07, 0, "bgtz", setgt, CPU64Regs>;
+def BLEZ64 : CBranchZero<0x07, 0, "blez", setle, CPU64Regs>;
+def BLTZ64 : CBranchZero<0x01, 0, "bltz", setlt, CPU64Regs>;
+
+/// Multiply and Divide Instructions.
+def DMULT    : Mul64<0x1c, "dmult", IIImul>;
+def DMULTu   : Mul64<0x1d, "dmultu", IIImul>;
+def DSDIV    : Div64<MipsDivRem, 0x1e, "ddiv", IIIdiv>;
+def DUDIV    : Div64<MipsDivRemU, 0x1f, "ddivu", IIIdiv>;
+
+let Defs = [HI64] in
+  def MTHI64  : MoveToLOHI64<0x11, "mthi">;
+let Defs = [LO64] in
+  def MTLO64  : MoveToLOHI64<0x13, "mtlo">;
+
+let Uses = [HI64] in
+  def MFHI64  : MoveFromLOHI64<0x10, "mfhi">;
+let Uses = [LO64] in
+  def MFLO64  : MoveFromLOHI64<0x12, "mflo">;
+
+/// Count Leading
+def DCLZ : CountLeading64<0x24, "dclz",
+                          [(set CPU64Regs:$dst, (ctlz CPU64Regs:$src))]>;
+def DCLO : CountLeading64<0x25, "dclo",
+                          [(set CPU64Regs:$dst, (ctlz (not CPU64Regs:$src)))]>;
+
+//===----------------------------------------------------------------------===//
+//  Arbitrary patterns that map to one or more instructions
+//===----------------------------------------------------------------------===//
+
+// Small immediates
+def : Pat<(i64 immSExt16:$in),
+          (DADDiu ZERO_64, imm:$in)>;
+def : Pat<(i64 immZExt16:$in),
+          (ORi64 ZERO_64, imm:$in)>;
+
+// zextloadi32_u
+def : Pat<(zextloadi32_u addr:$a), (DSRL (DSLL (ULW64_P8 addr:$a), 32), 32)>,
+      Requires<[IsN64]>;
+def : Pat<(zextloadi32_u addr:$a), (DSRL (DSLL (ULW64 addr:$a), 32), 32)>,
+      Requires<[NotN64]>;
+
+// hi/lo relocs
+def : Pat<(i64 (MipsLo tglobaladdr:$in)), (DADDiu ZERO_64, tglobaladdr:$in)>;
+
+defm : BrcondPats<CPU64Regs, BEQ64, BNE64, SLT64, SLTu64, SLTi64, SLTiu64,
+                  ZERO_64>;
+
+// setcc patterns
+defm : SeteqPats<CPU64Regs, SLTiu64, XOR64, SLTu64, ZERO_64>;
+defm : SetlePats<CPU64Regs, SLT64, SLTu64>;
+defm : SetgtPats<CPU64Regs, SLT64, SLTu64>;
+defm : SetgePats<CPU64Regs, SLT64, SLTu64>;
+defm : SetgeImmPats<CPU64Regs, SLTi64, SLTiu64>;
diff --git a/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp b/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
index 69e03bd29724..0e826812d076 100644
--- a/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -18,6 +18,7 @@
 #include "MipsInstrInfo.h"
 #include "MipsMachineFunction.h"
 #include "MipsMCInstLower.h"
+#include "MipsMCSymbolRefExpr.h"
 #include "InstPrinter/MipsInstPrinter.h"
 #include "llvm/BasicBlock.h"
 #include "llvm/Instructions.h"
@@ -25,6 +26,7 @@
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCInst.h"
@@ -33,15 +35,22 @@
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Analysis/DebugInfo.h"
 
 using namespace llvm;
 
+static bool isUnalignedLoadStore(unsigned Opc) {
+  return Opc == Mips::ULW    || Opc == Mips::ULH    || Opc == Mips::ULHu ||
+         Opc == Mips::USW    || Opc == Mips::USH    ||
+         Opc == Mips::ULW_P8 || Opc == Mips::ULH_P8 || Opc == Mips::ULHu_P8 ||
+         Opc == Mips::USW_P8 || Opc == Mips::USH_P8;
+}
+
 void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   SmallString<128> Str;
   raw_svector_ostream OS(Str);
@@ -52,8 +61,21 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   }
 
   MipsMCInstLower MCInstLowering(Mang, *MF, *this);
+  unsigned Opc = MI->getOpcode();
   MCInst TmpInst0;
   MCInstLowering.Lower(MI, TmpInst0);
+  
+  // Enclose unaligned load or store with .macro & .nomacro directives.
+  if (isUnalignedLoadStore(Opc)) {
+    MCInst Directive;
+    Directive.setOpcode(Mips::MACRO);
+    OutStreamer.EmitInstruction(Directive);
+    OutStreamer.EmitInstruction(TmpInst0);
+    Directive.setOpcode(Mips::NOMACRO);
+    OutStreamer.EmitInstruction(Directive);
+    return;
+  }
+
   OutStreamer.EmitInstruction(TmpInst0);
 }
 
@@ -180,7 +202,6 @@ void MipsAsmPrinter::emitFrameDirective() {
 const char *MipsAsmPrinter::getCurrentABIString() const {
   switch (Subtarget->getTargetABI()) {
   case MipsSubtarget::O32:  return "abi32";
-  case MipsSubtarget::O64:  return "abiO64";
   case MipsSubtarget::N32:  return "abiN32";
   case MipsSubtarget::N64:  return "abi64";
   case MipsSubtarget::EABI: return "eabi32"; // TODO: handle eabi64
@@ -304,6 +325,11 @@ void MipsAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
   case MipsII::MO_GOTTPREL: O << "%gottprel("; break;
   case MipsII::MO_TPREL_HI: O << "%tprel_hi("; break;
   case MipsII::MO_TPREL_LO: O << "%tprel_lo("; break;
+  case MipsII::MO_GPOFF_HI: O << "%hi(%neg(%gp_rel("; break;
+  case MipsII::MO_GPOFF_LO: O << "%lo(%neg(%gp_rel("; break;
+  case MipsII::MO_GOT_DISP: O << "%got_disp("; break;
+  case MipsII::MO_GOT_PAGE: O << "%got_page("; break;
+  case MipsII::MO_GOT_OFST: O << "%got_ofst("; break;
   }
 
   switch (MO.getType()) {
@@ -424,17 +450,9 @@ void MipsAsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
 }
 
 // Force static initialization.
-static MCInstPrinter *createMipsMCInstPrinter(const Target &T,
-                                              unsigned SyntaxVariant,
-                                              const MCAsmInfo &MAI) {
-  return new MipsInstPrinter(MAI);
-}
-
 extern "C" void LLVMInitializeMipsAsmPrinter() {
   RegisterAsmPrinter<MipsAsmPrinter> X(TheMipsTarget);
   RegisterAsmPrinter<MipsAsmPrinter> Y(TheMipselTarget);
-
-  TargetRegistry::RegisterMCInstPrinter(TheMipsTarget, createMipsMCInstPrinter);
-  TargetRegistry::RegisterMCInstPrinter(TheMipselTarget,
-                                        createMipsMCInstPrinter);
+  RegisterAsmPrinter<MipsAsmPrinter> A(TheMips64Target);
+  RegisterAsmPrinter<MipsAsmPrinter> B(TheMips64elTarget);
 }
diff --git a/contrib/llvm/lib/Target/Mips/MipsCallingConv.td b/contrib/llvm/lib/Target/Mips/MipsCallingConv.td
index 876f0fcc83ea..0ae4ef6fbad4 100644
--- a/contrib/llvm/lib/Target/Mips/MipsCallingConv.td
+++ b/contrib/llvm/lib/Target/Mips/MipsCallingConv.td
@@ -30,6 +30,55 @@ def RetCC_MipsO32 : CallingConv<[
   CCIfType<[f64], CCIfSubtarget<"isNotSingleFloat()", CCAssignToReg<[D0, D1]>>>
 ]>;
 
+//===----------------------------------------------------------------------===//
+// Mips N32/64 Calling Convention
+//===----------------------------------------------------------------------===//
+
+def CC_MipsN : CallingConv<[
+  // FIXME: Handle byval, complex and float double parameters.
+
+  // Promote i8/i16/i32 arguments to i64.
+  CCIfType<[i8, i16, i32], CCPromoteToType<i64>>,
+
+  // Integer arguments are passed in integer registers.
+  CCIfType<[i64], CCAssignToRegWithShadow<[A0_64, A1_64, A2_64, A3_64,
+                                           T0_64, T1_64, T2_64, T3_64],
+                                          [D12_64, D13_64, D14_64, D15_64,
+                                           D16_64, D17_64, D18_64, D19_64]>>,
+
+  // f32 arguments are passed in single precision FP registers.
+  CCIfType<[f32], CCAssignToRegWithShadow<[F12, F13, F14, F15,
+                                           F16, F17, F18, F19],
+                                          [A0_64, A1_64, A2_64, A3_64,
+                                           T0_64, T1_64, T2_64, T3_64]>>,
+
+  // f64 arguments are passed in double precision FP registers.
+  CCIfType<[f64], CCAssignToRegWithShadow<[D12_64, D13_64, D14_64, D15_64,
+                                           D16_64, D17_64, D18_64, D19_64],
+                                          [A0_64, A1_64, A2_64, A3_64,
+                                           T0_64, T1_64, T2_64, T3_64]>>,
+
+  // All stack parameter slots become 64-bit doublewords and are 8-byte aligned.
+  CCIfType<[i64, f64], CCAssignToStack<8, 8>>,
+  CCIfType<[f32], CCAssignToStack<4, 8>>
+]>;
+
+def RetCC_MipsN : CallingConv<[
+  // FIXME: Handle complex and float double return values.
+
+  // i32 are returned in registers V0, V1
+  CCIfType<[i32], CCAssignToReg<[V0, V1]>>,
+
+  // i64 are returned in registers V0_64, V1_64
+  CCIfType<[i64], CCAssignToReg<[V0_64, V1_64]>>,
+
+  // f32 are returned in registers F0, F2
+  CCIfType<[f32], CCAssignToReg<[F0, F2]>>,
+
+  // f64 are returned in registers D0, D2
+  CCIfType<[f64], CCAssignToReg<[D0_64, D2_64]>>
+]>;
+
 //===----------------------------------------------------------------------===//
 // Mips EABI Calling Convention
 //===----------------------------------------------------------------------===//
@@ -77,10 +126,14 @@ def RetCC_MipsEABI : CallingConv<[
 //===----------------------------------------------------------------------===//
 
 def CC_Mips : CallingConv<[
-  CCIfSubtarget<"isABI_EABI()", CCDelegateTo<CC_MipsEABI>>
+  CCIfSubtarget<"isABI_EABI()", CCDelegateTo<CC_MipsEABI>>,
+  CCIfSubtarget<"isABI_N32()", CCDelegateTo<CC_MipsN>>,
+  CCIfSubtarget<"isABI_N64()", CCDelegateTo<CC_MipsN>>
 ]>;
 
 def RetCC_Mips : CallingConv<[
   CCIfSubtarget<"isABI_EABI()", CCDelegateTo<RetCC_MipsEABI>>,
+  CCIfSubtarget<"isABI_N32()", CCDelegateTo<RetCC_MipsN>>,
+  CCIfSubtarget<"isABI_N64()", CCDelegateTo<RetCC_MipsN>>,
   CCDelegateTo<RetCC_MipsO32>
 ]>;
diff --git a/contrib/llvm/lib/Target/Mips/MipsCodeEmitter.cpp b/contrib/llvm/lib/Target/Mips/MipsCodeEmitter.cpp
new file mode 100644
index 000000000000..9220d9c7ab4f
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MipsCodeEmitter.cpp
@@ -0,0 +1,245 @@
+//===-- Mips/MipsCodeEmitter.cpp - Convert Mips code to machine code -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file contains the pass that transforms the Mips machine instructions
+// into relocatable machine code.
+//
+//===---------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "jit"
+#include "Mips.h"
+#include "MipsInstrInfo.h"
+#include "MipsRelocations.h"
+#include "MipsSubtarget.h"
+#include "MipsTargetMachine.h"
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Function.h"
+#include "llvm/PassManager.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#ifndef NDEBUG
+#include <iomanip>
+#endif
+
+#include "llvm/CodeGen/MachineOperand.h"
+
+using namespace llvm;
+
+STATISTIC(NumEmitted, "Number of machine instructions emitted");
+
+namespace {
+
+class MipsCodeEmitter : public MachineFunctionPass {
+  MipsJITInfo *JTI;
+  const MipsInstrInfo *II;
+  const TargetData *TD;
+  const MipsSubtarget *Subtarget;
+  TargetMachine &TM;
+  JITCodeEmitter &MCE;
+  const std::vector<MachineConstantPoolEntry> *MCPEs;
+  const std::vector<MachineJumpTableEntry> *MJTEs;
+  bool IsPIC;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const {
+    AU.addRequired<MachineModuleInfo> ();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  static char ID;
+
+  public:
+    MipsCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce) :
+      MachineFunctionPass(ID), JTI(0),
+        II((const MipsInstrInfo *) tm.getInstrInfo()),
+        TD(tm.getTargetData()), TM(tm), MCE(mce), MCPEs(0), MJTEs(0),
+        IsPIC(TM.getRelocationModel() == Reloc::PIC_) {
+    }
+
+    bool runOnMachineFunction(MachineFunction &MF);
+
+    virtual const char *getPassName() const {
+      return "Mips Machine Code Emitter";
+    }
+
+    /// getBinaryCodeForInstr - This function, generated by the
+    /// CodeEmitterGenerator using TableGen, produces the binary encoding for
+    /// machine instructions.
+    unsigned getBinaryCodeForInstr(const MachineInstr &MI) const;
+
+    void emitInstruction(const MachineInstr &MI);
+
+  private:
+
+    void emitWordLE(unsigned Word);
+
+    /// Routines that handle operands which add machine relocations which are
+    /// fixed up by the relocation stage.
+    void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+                                bool MayNeedFarStub) const;
+    void emitExternalSymbolAddress(const char *ES, unsigned Reloc) const;
+    void emitConstPoolAddress(unsigned CPI, unsigned Reloc) const;
+    void emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const;
+    void emitMachineBasicBlock(MachineBasicBlock *BB, unsigned Reloc) const;
+
+    /// getMachineOpValue - Return binary encoding of operand. If the machine
+    /// operand requires relocation, record the relocation and return zero.
+    unsigned getMachineOpValue(const MachineInstr &MI,
+                               const MachineOperand &MO) const;
+
+    unsigned getRelocation(const MachineInstr &MI,
+                           const MachineOperand &MO) const;
+
+  };
+}
+
+char MipsCodeEmitter::ID = 0;
+
+bool MipsCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
+  JTI = ((MipsTargetMachine&) MF.getTarget()).getJITInfo();
+  II = ((const MipsTargetMachine&) MF.getTarget()).getInstrInfo();
+  TD = ((const MipsTargetMachine&) MF.getTarget()).getTargetData();
+  Subtarget = &TM.getSubtarget<MipsSubtarget> ();
+  MCPEs = &MF.getConstantPool()->getConstants();
+  MJTEs = 0;
+  if (MF.getJumpTableInfo()) MJTEs = &MF.getJumpTableInfo()->getJumpTables();
+  JTI->Initialize(MF, IsPIC);
+  MCE.setModuleInfo(&getAnalysis<MachineModuleInfo> ());
+
+  do {
+    DEBUG(errs() << "JITTing function '"
+        << MF.getFunction()->getName() << "'\n");
+    MCE.startFunction(MF);
+
+    for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+        MBB != E; ++MBB){
+      MCE.StartMachineBasicBlock(MBB);
+      for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end();
+          I != E; ++I)
+        emitInstruction(*I);
+    }
+  } while (MCE.finishFunction(MF));
+
+  return false;
+}
+
+unsigned MipsCodeEmitter::getRelocation(const MachineInstr &MI,
+                                        const MachineOperand &MO) const {
+  // NOTE: This relocations are for static.
+  uint64_t TSFlags = MI.getDesc().TSFlags;
+  uint64_t Form = TSFlags & MipsII::FormMask;
+  if (Form == MipsII::FrmJ)
+    return Mips::reloc_mips_26;
+  if ((Form == MipsII::FrmI || Form == MipsII::FrmFI)
+       && MI.getDesc().isBranch())
+    return Mips::reloc_mips_branch;
+  if (Form == MipsII::FrmI && MI.getOpcode() == Mips::LUi)
+    return Mips::reloc_mips_hi;
+  return Mips::reloc_mips_lo;
+}
+
+/// getMachineOpValue - Return binary encoding of operand. If the machine
+/// operand requires relocation, record the relocation and return zero.
+unsigned MipsCodeEmitter::getMachineOpValue(const MachineInstr &MI,
+                                           const MachineOperand &MO) const {
+  if (MO.isReg())
+    return MipsRegisterInfo::getRegisterNumbering(MO.getReg());
+  else if (MO.isImm())
+    return static_cast<unsigned>(MO.getImm());
+  else if (MO.isGlobal())
+    emitGlobalAddress(MO.getGlobal(), getRelocation(MI, MO), true);
+  else if (MO.isSymbol())
+    emitExternalSymbolAddress(MO.getSymbolName(), getRelocation(MI, MO));
+  else if (MO.isCPI())
+    emitConstPoolAddress(MO.getIndex(), getRelocation(MI, MO));
+  else if (MO.isJTI())
+    emitJumpTableAddress(MO.getIndex(), getRelocation(MI, MO));
+  else if (MO.isMBB())
+    emitMachineBasicBlock(MO.getMBB(), getRelocation(MI, MO));
+  else
+    llvm_unreachable("Unable to encode MachineOperand!");
+  return 0;
+}
+
+void MipsCodeEmitter::emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+                                                bool MayNeedFarStub) const {
+  MCE.addRelocation(MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+                             const_cast<GlobalValue *>(GV), 0, MayNeedFarStub));
+}
+
+void MipsCodeEmitter::
+emitExternalSymbolAddress(const char *ES, unsigned Reloc) const {
+  MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+                                                 Reloc, ES, 0, 0, false));
+}
+
+void MipsCodeEmitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc) const {
+  MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+                                                    Reloc, CPI, 0, false));
+}
+
+void MipsCodeEmitter::
+emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const {
+  MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
+                                                    Reloc, JTIndex, 0, false));
+}
+
+void MipsCodeEmitter::emitMachineBasicBlock(MachineBasicBlock *BB,
+                                           unsigned Reloc) const {
+  MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+                                             Reloc, BB));
+}
+
+void MipsCodeEmitter::emitInstruction(const MachineInstr &MI) {
+  DEBUG(errs() << "JIT: " << (void*)MCE.getCurrentPCValue() << ":\t" << MI);
+
+  MCE.processDebugLoc(MI.getDebugLoc(), true);
+
+  // Skip pseudo instructions.
+  if ((MI.getDesc().TSFlags & MipsII::FormMask) == MipsII::Pseudo)
+    return;
+
+  ++NumEmitted;  // Keep track of the # of mi's emitted
+
+  switch (MI.getOpcode()) {
+  default:
+    emitWordLE(getBinaryCodeForInstr(MI));
+    break;
+  }
+
+  MCE.processDebugLoc(MI.getDebugLoc(), false);
+}
+
+void MipsCodeEmitter::emitWordLE(unsigned Word) {
+  DEBUG(errs() << "  0x";
+        errs().write_hex(Word) << "\n");
+  MCE.emitWordLE(Word);
+}
+
+/// createMipsJITCodeEmitterPass - Return a pass that emits the collected Mips
+/// code to the specified MCE object.
+FunctionPass *llvm::createMipsJITCodeEmitterPass(MipsTargetMachine &TM,
+    JITCodeEmitter &JCE) {
+  return new MipsCodeEmitter(TM, JCE);
+}
+
+unsigned MipsCodeEmitter::getBinaryCodeForInstr(const MachineInstr &MI) const {
+ // this function will be automatically generated by the CodeEmitterGenerator
+ // using TableGen
+ return 0;
+}
diff --git a/contrib/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp b/contrib/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
index c3a6211399cd..be3b7a02ec31 100644
--- a/contrib/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Simple pass to fills delay slots with NOPs.
+// Simple pass to fills delay slots with useful instructions.
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,18 +17,31 @@
 #include "MipsTargetMachine.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 
 using namespace llvm;
 
 STATISTIC(FilledSlots, "Number of delay slots filled");
+STATISTIC(UsefulSlots, "Number of delay slots filled with instructions that"
+                       " are not NOP.");
+
+static cl::opt<bool> EnableDelaySlotFiller(
+  "enable-mips-delay-filler",
+  cl::init(false),
+  cl::desc("Fill the Mips delay slots useful instructions."),
+  cl::Hidden);
 
 namespace {
   struct Filler : public MachineFunctionPass {
 
     TargetMachine &TM;
     const TargetInstrInfo *TII;
+    MachineBasicBlock::iterator LastFiller;
 
     static char ID;
     Filler(TargetMachine &tm)
@@ -47,31 +60,61 @@ namespace {
       return Changed;
     }
 
+    bool isDelayFiller(MachineBasicBlock &MBB,
+                       MachineBasicBlock::iterator candidate);
+
+    void insertCallUses(MachineBasicBlock::iterator MI,
+                        SmallSet<unsigned, 32>& RegDefs,
+                        SmallSet<unsigned, 32>& RegUses);
+
+    void insertDefsUses(MachineBasicBlock::iterator MI,
+                        SmallSet<unsigned, 32>& RegDefs,
+                        SmallSet<unsigned, 32>& RegUses);
+
+    bool IsRegInSet(SmallSet<unsigned, 32>& RegSet,
+                    unsigned Reg);
+
+    bool delayHasHazard(MachineBasicBlock::iterator candidate,
+                        bool &sawLoad, bool &sawStore,
+                        SmallSet<unsigned, 32> &RegDefs,
+                        SmallSet<unsigned, 32> &RegUses);
+
+    bool
+    findDelayInstr(MachineBasicBlock &MBB, MachineBasicBlock::iterator slot,
+                   MachineBasicBlock::iterator &Filler);
+
+
   };
   char Filler::ID = 0;
 } // end of anonymous namespace
 
 /// runOnMachineBasicBlock - Fill in delay slots for the given basic block.
-/// Currently, we fill delay slots with NOPs. We assume there is only one
-/// delay slot per delayed instruction.
+/// We assume there is only one delay slot per delayed instruction.
 bool Filler::
-runOnMachineBasicBlock(MachineBasicBlock &MBB)
-{
+runOnMachineBasicBlock(MachineBasicBlock &MBB) {
   bool Changed = false;
-  for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
-    const MCInstrDesc& MCid = I->getDesc();
-    if (MCid.hasDelaySlot() &&
-        (TM.getSubtarget<MipsSubtarget>().isMips1() ||
-         MCid.isCall() || MCid.isBranch() || MCid.isReturn())) {
-      MachineBasicBlock::iterator J = I;
-      ++J;
-      BuildMI(MBB, J, I->getDebugLoc(), TII->get(Mips::NOP));
+  LastFiller = MBB.end();
+
+  for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I)
+    if (I->getDesc().hasDelaySlot()) {
       ++FilledSlots;
       Changed = true;
-    }
-  }
 
+      MachineBasicBlock::iterator D;
+
+      if (EnableDelaySlotFiller && findDelayInstr(MBB, I, D)) {
+        MBB.splice(llvm::next(I), &MBB, D);
+        ++UsefulSlots;
+      }
+      else 
+        BuildMI(MBB, llvm::next(I), I->getDebugLoc(), TII->get(Mips::NOP));
+
+      // Record the filler instruction that filled the delay slot.
+      // The instruction after it will be visited in the next iteration.
+      LastFiller = ++I;
+     }
   return Changed;
+
 }
 
 /// createMipsDelaySlotFillerPass - Returns a pass that fills in delay
@@ -80,3 +123,134 @@ FunctionPass *llvm::createMipsDelaySlotFillerPass(MipsTargetMachine &tm) {
   return new Filler(tm);
 }
 
+bool Filler::findDelayInstr(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator slot,
+                            MachineBasicBlock::iterator &Filler) {
+  SmallSet<unsigned, 32> RegDefs;
+  SmallSet<unsigned, 32> RegUses;
+
+  insertDefsUses(slot, RegDefs, RegUses);
+
+  bool sawLoad = false;
+  bool sawStore = false;
+
+  for (MachineBasicBlock::reverse_iterator I(slot); I != MBB.rend(); ++I) {
+    // skip debug value
+    if (I->isDebugValue())
+      continue;
+
+    // Convert to forward iterator.
+    MachineBasicBlock::iterator FI(llvm::next(I).base());
+
+    if (I->hasUnmodeledSideEffects()
+        || I->isInlineAsm()
+        || I->isLabel()
+        || FI == LastFiller
+        || I->getDesc().isPseudo()
+        //
+        // Should not allow:
+        // ERET, DERET or WAIT, PAUSE. Need to add these to instruction
+        // list. TBD.
+        )
+      break;
+
+    if (delayHasHazard(FI, sawLoad, sawStore, RegDefs, RegUses)) {
+      insertDefsUses(FI, RegDefs, RegUses);
+      continue;
+    }
+
+    Filler = FI;
+    return true;
+  }
+
+  return false;
+}
+
+bool Filler::delayHasHazard(MachineBasicBlock::iterator candidate,
+                            bool &sawLoad,
+                            bool &sawStore,
+                            SmallSet<unsigned, 32> &RegDefs,
+                            SmallSet<unsigned, 32> &RegUses) {
+  if (candidate->isImplicitDef() || candidate->isKill())
+    return true;
+
+  MCInstrDesc MCID = candidate->getDesc();
+  // Loads or stores cannot be moved past a store to the delay slot
+  // and stores cannot be moved past a load. 
+  if (MCID.mayLoad()) {
+    if (sawStore)
+      return true;
+    sawLoad = true;
+  }
+
+  if (MCID.mayStore()) {
+    if (sawStore)
+      return true;
+    sawStore = true;
+    if (sawLoad)
+      return true;
+  }
+
+  assert((!MCID.isCall() && !MCID.isReturn()) &&
+         "Cannot put calls or returns in delay slot.");
+
+  for (unsigned i = 0, e = candidate->getNumOperands(); i!= e; ++i) {
+    const MachineOperand &MO = candidate->getOperand(i);
+    unsigned Reg;
+
+    if (!MO.isReg() || !(Reg = MO.getReg()))
+      continue; // skip
+
+    if (MO.isDef()) {
+      // check whether Reg is defined or used before delay slot.
+      if (IsRegInSet(RegDefs, Reg) || IsRegInSet(RegUses, Reg))
+        return true;
+    }
+    if (MO.isUse()) {
+      // check whether Reg is defined before delay slot.
+      if (IsRegInSet(RegDefs, Reg))
+        return true;
+    }
+  }
+  return false;
+}
+
+// Insert Defs and Uses of MI into the sets RegDefs and RegUses.
+void Filler::insertDefsUses(MachineBasicBlock::iterator MI,
+                            SmallSet<unsigned, 32>& RegDefs,
+                            SmallSet<unsigned, 32>& RegUses) {
+  // If MI is a call or return, just examine the explicit non-variadic operands.
+  MCInstrDesc MCID = MI->getDesc();
+  unsigned e = MCID.isCall() || MCID.isReturn() ? MCID.getNumOperands() :
+                                                  MI->getNumOperands();
+  
+  // Add RA to RegDefs to prevent users of RA from going into delay slot. 
+  if (MCID.isCall())
+    RegDefs.insert(Mips::RA);
+
+  for (unsigned i = 0; i != e; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    unsigned Reg;
+
+    if (!MO.isReg() || !(Reg = MO.getReg()))
+      continue;
+
+    if (MO.isDef())
+      RegDefs.insert(Reg);
+    else if (MO.isUse())
+      RegUses.insert(Reg);
+  }
+}
+
+//returns true if the Reg or its alias is in the RegSet.
+bool Filler::IsRegInSet(SmallSet<unsigned, 32>& RegSet, unsigned Reg) {
+  if (RegSet.count(Reg))
+    return true;
+  // check Aliased Registers
+  for (const unsigned *Alias = TM.getRegisterInfo()->getAliasSet(Reg);
+       *Alias; ++Alias)
+    if (RegSet.count(*Alias))
+      return true;
+
+  return false;
+}
diff --git a/contrib/llvm/lib/Target/Mips/MipsFrameLowering.cpp b/contrib/llvm/lib/Target/Mips/MipsFrameLowering.cpp
index a0f90a0d487a..22d1e47b1a2b 100644
--- a/contrib/llvm/lib/Target/Mips/MipsFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsFrameLowering.cpp
@@ -254,9 +254,15 @@ void MipsFrameLowering::emitPrologue(MachineFunction &MF) const {
   }
 
   // Restore GP from the saved stack location
-  if (MipsFI->needGPSaveRestore())
-    BuildMI(MBB, MBBI, dl, TII.get(Mips::CPRESTORE))
-      .addImm(MFI->getObjectOffset(MipsFI->getGPFI()));
+  if (MipsFI->needGPSaveRestore()) {
+    unsigned Offset = MFI->getObjectOffset(MipsFI->getGPFI());
+    BuildMI(MBB, MBBI, dl, TII.get(Mips::CPRESTORE)).addImm(Offset);
+
+    if (Offset >= 0x8000) {
+      BuildMI(MBB, llvm::prior(MBBI), dl, TII.get(Mips::MACRO));
+      BuildMI(MBB, MBBI, dl, TII.get(Mips::NOMACRO));
+    }
+  }
 }
 
 void MipsFrameLowering::emitEpilogue(MachineFunction &MF,
@@ -300,13 +306,6 @@ void MipsFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-void
-MipsFrameLowering::getInitialFrameState(std::vector<MachineMove> &Moves) const {
-  MachineLocation Dst(MachineLocation::VirtualFP);
-  MachineLocation Src(Mips::SP, 0);
-  Moves.push_back(MachineMove(0, Dst, Src));
-}
-
 void MipsFrameLowering::
 processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                      RegScavenger *RS) const {
diff --git a/contrib/llvm/lib/Target/Mips/MipsFrameLowering.h b/contrib/llvm/lib/Target/Mips/MipsFrameLowering.h
index 78c78eea5b9a..c24975614c8d 100644
--- a/contrib/llvm/lib/Target/Mips/MipsFrameLowering.h
+++ b/contrib/llvm/lib/Target/Mips/MipsFrameLowering.h
@@ -27,7 +27,8 @@ class MipsFrameLowering : public TargetFrameLowering {
 
 public:
   explicit MipsFrameLowering(const MipsSubtarget &sti)
-    : TargetFrameLowering(StackGrowsDown, 8, 0), STI(sti) {
+    : TargetFrameLowering(StackGrowsDown, sti.hasMips64() ? 16 : 8, 0),
+      STI(sti) {
   }
 
   bool targetHandlesStackFrameRounding() const;
@@ -39,8 +40,6 @@ class MipsFrameLowering : public TargetFrameLowering {
 
   bool hasFP(const MachineFunction &MF) const;
 
-  void getInitialFrameState(std::vector<MachineMove> &Moves) const;
-  
   void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                             RegScavenger *RS) const;
 };
diff --git a/contrib/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp b/contrib/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
index 90aaeb60d06f..9c831ede9dbf 100644
--- a/contrib/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -86,9 +86,6 @@ class MipsDAGToDAGISel : public SelectionDAGISel {
   // Complex Pattern.
   bool SelectAddr(SDValue N, SDValue &Base, SDValue &Offset);
 
-  SDNode *SelectLoadFp64(SDNode *N);
-  SDNode *SelectStoreFp64(SDNode *N);
-
   // getI32Imm - Return a target constant with the specified
   // value, of type i32.
   inline SDValue getI32Imm(unsigned Imm) {
@@ -114,17 +111,20 @@ SDNode *MipsDAGToDAGISel::getGlobalBaseReg() {
 /// Used on Mips Load/Store instructions
 bool MipsDAGToDAGISel::
 SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
+  EVT ValTy = Addr.getValueType();
+  unsigned GPReg = ValTy == MVT::i32 ? Mips::GP : Mips::GP_64;
+
   // if Address is FI, get the TargetFrameIndex.
   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
-    Base   = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
-    Offset = CurDAG->getTargetConstant(0, MVT::i32);
+    Base   = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
+    Offset = CurDAG->getTargetConstant(0, ValTy);
     return true;
   }
 
   // on PIC code Load GA
   if (TM.getRelocationModel() == Reloc::PIC_) {
     if (Addr.getOpcode() == MipsISD::WrapperPIC) {
-      Base   = CurDAG->getRegister(Mips::GP, MVT::i32);
+      Base   = CurDAG->getRegister(GPReg, ValTy);
       Offset = Addr.getOperand(0);
       return true;
     }
@@ -133,7 +133,7 @@ SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
         Addr.getOpcode() == ISD::TargetGlobalAddress))
       return false;
     else if (Addr.getOpcode() == ISD::TargetGlobalTLSAddress) {
-      Base   = CurDAG->getRegister(Mips::GP, MVT::i32);
+      Base   = CurDAG->getRegister(GPReg, ValTy);
       Offset = Addr;
       return true;
     }
@@ -147,11 +147,11 @@ SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
       // If the first operand is a FI, get the TargetFI Node
       if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>
                                   (Addr.getOperand(0)))
-        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
+        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
       else
         Base = Addr.getOperand(0);
 
-      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
+      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), ValTy);
       return true;
     }
   }
@@ -180,134 +180,10 @@ SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
   }
 
   Base   = Addr;
-  Offset = CurDAG->getTargetConstant(0, MVT::i32);
+  Offset = CurDAG->getTargetConstant(0, ValTy);
   return true;
 }
 
-SDNode *MipsDAGToDAGISel::SelectLoadFp64(SDNode *N) {
-  MVT::SimpleValueType NVT =
-    N->getValueType(0).getSimpleVT().SimpleTy;
-
-  if (!Subtarget.isMips1() || NVT != MVT::f64)
-    return NULL;
-
-  LoadSDNode *LN = cast<LoadSDNode>(N);
-  if (LN->getExtensionType() != ISD::NON_EXTLOAD ||
-      LN->getAddressingMode() != ISD::UNINDEXED)
-    return NULL;
-
-  SDValue Chain = N->getOperand(0);
-  SDValue N1 = N->getOperand(1);
-  SDValue Offset0, Offset1, Base;
-
-  if (!SelectAddr(N1, Base, Offset0) ||
-      N1.getValueType() != MVT::i32)
-    return NULL;
-
-  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
-  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
-  DebugLoc dl = N->getDebugLoc();
-
-  // The second load should start after for 4 bytes.
-  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Offset0))
-    Offset1 = CurDAG->getTargetConstant(C->getSExtValue()+4, MVT::i32);
-  else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Offset0))
-    Offset1 = CurDAG->getTargetConstantPool(CP->getConstVal(),
-                                            MVT::i32,
-                                            CP->getAlignment(),
-                                            CP->getOffset()+4,
-                                            CP->getTargetFlags());
-  else
-    return NULL;
-
-  // Choose the offsets depending on the endianess
-  if (TM.getTargetData()->isBigEndian())
-    std::swap(Offset0, Offset1);
-
-  // Instead of:
-  //    ldc $f0, X($3)
-  // Generate:
-  //    lwc $f0, X($3)
-  //    lwc $f1, X+4($3)
-  SDNode *LD0 = CurDAG->getMachineNode(Mips::LWC1, dl, MVT::f32,
-                                       MVT::Other, Base, Offset0, Chain);
-  SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
-                                                 dl, NVT), 0);
-  SDValue I0 = CurDAG->getTargetInsertSubreg(Mips::sub_fpeven, dl,
-                            MVT::f64, Undef, SDValue(LD0, 0));
-
-  SDNode *LD1 = CurDAG->getMachineNode(Mips::LWC1, dl, MVT::f32,
-                                       MVT::Other, Base, Offset1, SDValue(LD0, 1));
-  SDValue I1 = CurDAG->getTargetInsertSubreg(Mips::sub_fpodd, dl,
-                            MVT::f64, I0, SDValue(LD1, 0));
-
-  ReplaceUses(SDValue(N, 0), I1);
-  ReplaceUses(SDValue(N, 1), Chain);
-  cast<MachineSDNode>(LD0)->setMemRefs(MemRefs0, MemRefs0 + 1);
-  cast<MachineSDNode>(LD1)->setMemRefs(MemRefs0, MemRefs0 + 1);
-  return I1.getNode();
-}
-
-SDNode *MipsDAGToDAGISel::SelectStoreFp64(SDNode *N) {
-
-  if (!Subtarget.isMips1() ||
-      N->getOperand(1).getValueType() != MVT::f64)
-    return NULL;
-
-  SDValue Chain = N->getOperand(0);
-
-  StoreSDNode *SN = cast<StoreSDNode>(N);
-  if (SN->isTruncatingStore() || SN->getAddressingMode() != ISD::UNINDEXED)
-    return NULL;
-
-  SDValue N1 = N->getOperand(1);
-  SDValue N2 = N->getOperand(2);
-  SDValue Offset0, Offset1, Base;
-
-  if (!SelectAddr(N2, Base, Offset0) ||
-      N1.getValueType() != MVT::f64 ||
-      N2.getValueType() != MVT::i32)
-    return NULL;
-
-  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
-  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
-  DebugLoc dl = N->getDebugLoc();
-
-  // Get the even and odd part from the f64 register
-  SDValue FPOdd = CurDAG->getTargetExtractSubreg(Mips::sub_fpodd,
-                                                 dl, MVT::f32, N1);
-  SDValue FPEven = CurDAG->getTargetExtractSubreg(Mips::sub_fpeven,
-                                                 dl, MVT::f32, N1);
-
-  // The second store should start after for 4 bytes.
-  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Offset0))
-    Offset1 = CurDAG->getTargetConstant(C->getSExtValue()+4, MVT::i32);
-  else
-    return NULL;
-
-  // Choose the offsets depending on the endianess
-  if (TM.getTargetData()->isBigEndian())
-    std::swap(Offset0, Offset1);
-
-  // Instead of:
-  //    sdc $f0, X($3)
-  // Generate:
-  //    swc $f0, X($3)
-  //    swc $f1, X+4($3)
-  SDValue Ops0[] = { FPEven, Base, Offset0, Chain };
-  Chain = SDValue(CurDAG->getMachineNode(Mips::SWC1, dl,
-                                       MVT::Other, Ops0, 4), 0);
-  cast<MachineSDNode>(Chain.getNode())->setMemRefs(MemRefs0, MemRefs0 + 1);
-
-  SDValue Ops1[] = { FPOdd, Base, Offset1, Chain };
-  Chain = SDValue(CurDAG->getMachineNode(Mips::SWC1, dl,
-                                       MVT::Other, Ops1, 4), 0);
-  cast<MachineSDNode>(Chain.getNode())->setMemRefs(MemRefs0, MemRefs0 + 1);
-
-  ReplaceUses(SDValue(N, 0), Chain);
-  return Chain.getNode();
-}
-
 /// Select instructions not customized! Used for
 /// expanded, promoted and normal instructions
 SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
@@ -364,6 +240,8 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
     /// Mul with two results
     case ISD::SMUL_LOHI:
     case ISD::UMUL_LOHI: {
+      assert(Node->getValueType(0) != MVT::i64 &&
+             "64-bit multiplication with two results not handled.");
       SDValue Op1 = Node->getOperand(0);
       SDValue Op2 = Node->getOperand(1);
 
@@ -389,21 +267,29 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
 
     /// Special Muls
     case ISD::MUL:
-      if (Subtarget.isMips32())
+      // Mips32 has a 32-bit three operand mul instruction.
+      if (Subtarget.hasMips32() && Node->getValueType(0) == MVT::i32)
         break;
     case ISD::MULHS:
     case ISD::MULHU: {
+      assert((Opcode == ISD::MUL || Node->getValueType(0) != MVT::i64) &&
+             "64-bit MULH* not handled.");
+      EVT Ty = Node->getValueType(0);
       SDValue MulOp1 = Node->getOperand(0);
       SDValue MulOp2 = Node->getOperand(1);
 
-      unsigned MulOp  = (Opcode == ISD::MULHU ? Mips::MULTu : Mips::MULT);
+      unsigned MulOp  = (Opcode == ISD::MULHU ?
+                         Mips::MULTu :
+                         (Ty == MVT::i32 ? Mips::MULT : Mips::DMULT));
       SDNode *MulNode = CurDAG->getMachineNode(MulOp, dl,
                                                MVT::Glue, MulOp1, MulOp2);
 
       SDValue InFlag = SDValue(MulNode, 0);
 
-      if (Opcode == ISD::MUL)
-        return CurDAG->getMachineNode(Mips::MFLO, dl, MVT::i32, InFlag);
+      if (Opcode == ISD::MUL) {
+        unsigned Opc = (Ty == MVT::i32 ? Mips::MFLO : Mips::MFLO64);
+        return CurDAG->getMachineNode(Opc, dl, Ty, InFlag);
+      }
       else
         return CurDAG->getMachineNode(Mips::MFHI, dl, MVT::i32, InFlag);
     }
@@ -417,31 +303,12 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
       if (Node->getValueType(0) == MVT::f64 && CN->isExactlyValue(+0.0)) {
         SDValue Zero = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
                                         Mips::ZERO, MVT::i32);
-        SDValue Undef = SDValue(
-          CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::f64), 0);
-        SDNode *MTC = CurDAG->getMachineNode(Mips::MTC1, dl, MVT::f32, Zero);
-        SDValue I0 = CurDAG->getTargetInsertSubreg(Mips::sub_fpeven, dl,
-                            MVT::f64, Undef, SDValue(MTC, 0));
-        SDValue I1 = CurDAG->getTargetInsertSubreg(Mips::sub_fpodd, dl,
-                            MVT::f64, I0, SDValue(MTC, 0));
-        ReplaceUses(SDValue(Node, 0), I1);
-        return I1.getNode();
+        return CurDAG->getMachineNode(Mips::BuildPairF64, dl, MVT::f64, Zero,
+                                      Zero);
       }
       break;
     }
 
-    case ISD::LOAD:
-      if (SDNode *ResNode = SelectLoadFp64(Node))
-        return ResNode;
-      // Other cases are autogenerated.
-      break;
-
-    case ISD::STORE:
-      if (SDNode *ResNode = SelectStoreFp64(Node))
-        return ResNode;
-      // Other cases are autogenerated.
-      break;
-
     case MipsISD::ThreadPointer: {
       unsigned SrcReg = Mips::HWR29;
       unsigned DestReg = Mips::V1;
diff --git a/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp b/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp
index b4f4b1b4bf04..1932e745c593 100644
--- a/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -35,6 +35,18 @@
 #include "llvm/Support/ErrorHandling.h"
 using namespace llvm;
 
+// If I is a shifted mask, set the size (Size) and the first bit of the 
+// mask (Pos), and return true.
+// For example, if I is 0x003ff800, (Pos, Size) = (11, 11).  
+static bool IsShiftedMask(uint64_t I, uint64_t &Pos, uint64_t &Size) {
+  if (!isUInt<32>(I) || !isShiftedMask_32(I))
+     return false;
+
+  Size = CountPopulation_32(I);
+  Pos = CountTrailingZeros_32(I);
+  return true;
+}
+
 const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   switch (Opcode) {
   case MipsISD::JmpLink:           return "MipsISD::JmpLink";
@@ -61,27 +73,38 @@ const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case MipsISD::ExtractElementF64: return "MipsISD::ExtractElementF64";
   case MipsISD::WrapperPIC:        return "MipsISD::WrapperPIC";
   case MipsISD::DynAlloc:          return "MipsISD::DynAlloc";
+  case MipsISD::Sync:              return "MipsISD::Sync";
+  case MipsISD::Ext:               return "MipsISD::Ext";
+  case MipsISD::Ins:               return "MipsISD::Ins";
   default:                         return NULL;
   }
 }
 
 MipsTargetLowering::
 MipsTargetLowering(MipsTargetMachine &TM)
-  : TargetLowering(TM, new MipsTargetObjectFile()) {
-  Subtarget = &TM.getSubtarget<MipsSubtarget>();
+  : TargetLowering(TM, new MipsTargetObjectFile()),
+    Subtarget(&TM.getSubtarget<MipsSubtarget>()),
+    HasMips64(Subtarget->hasMips64()), IsN64(Subtarget->isABI_N64()) {
 
   // Mips does not have i1 type, so use i32 for
   // setcc operations results (slt, sgt, ...).
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
 
   // Set up the register classes
   addRegisterClass(MVT::i32, Mips::CPURegsRegisterClass);
   addRegisterClass(MVT::f32, Mips::FGR32RegisterClass);
 
+  if (HasMips64)
+    addRegisterClass(MVT::i64, Mips::CPU64RegsRegisterClass);
+
   // When dealing with single precision only, use libcalls
-  if (!Subtarget->isSingleFloat())
-    if (!Subtarget->isFP64bit())
+  if (!Subtarget->isSingleFloat()) {
+    if (HasMips64)
+      addRegisterClass(MVT::f64, Mips::FGR64RegisterClass);
+    else
       addRegisterClass(MVT::f64, Mips::AFGR64RegisterClass);
+  }
 
   // Load extented operations for i1 types must be promoted
   setLoadExtAction(ISD::EXTLOAD,  MVT::i1,  Promote);
@@ -100,6 +123,7 @@ MipsTargetLowering(MipsTargetMachine &TM)
 
   // Mips Custom Operations
   setOperationAction(ISD::GlobalAddress,      MVT::i32,   Custom);
+  setOperationAction(ISD::GlobalAddress,      MVT::i64,   Custom);
   setOperationAction(ISD::BlockAddress,       MVT::i32,   Custom);
   setOperationAction(ISD::GlobalTLSAddress,   MVT::i32,   Custom);
   setOperationAction(ISD::JumpTable,          MVT::i32,   Custom);
@@ -115,6 +139,10 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setOperationAction(ISD::SREM, MVT::i32, Expand);
   setOperationAction(ISD::UDIV, MVT::i32, Expand);
   setOperationAction(ISD::UREM, MVT::i32, Expand);
+  setOperationAction(ISD::SDIV, MVT::i64, Expand);
+  setOperationAction(ISD::SREM, MVT::i64, Expand);
+  setOperationAction(ISD::UDIV, MVT::i64, Expand);
+  setOperationAction(ISD::UREM, MVT::i64, Expand);
 
   // Operations not directly supported by Mips.
   setOperationAction(ISD::BR_JT,             MVT::Other, Expand);
@@ -126,10 +154,14 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setOperationAction(ISD::CTPOP,             MVT::i32,   Expand);
   setOperationAction(ISD::CTTZ,              MVT::i32,   Expand);
   setOperationAction(ISD::ROTL,              MVT::i32,   Expand);
+  setOperationAction(ISD::ROTL,              MVT::i64,   Expand);
 
-  if (!Subtarget->isMips32r2())
+  if (!Subtarget->hasMips32r2())
     setOperationAction(ISD::ROTR, MVT::i32,   Expand);
 
+  if (!Subtarget->hasMips64r2())
+    setOperationAction(ISD::ROTR, MVT::i64,   Expand);
+
   setOperationAction(ISD::SHL_PARTS,         MVT::i32,   Expand);
   setOperationAction(ISD::SRA_PARTS,         MVT::i32,   Expand);
   setOperationAction(ISD::SRL_PARTS,         MVT::i32,   Expand);
@@ -159,7 +191,14 @@ MipsTargetLowering(MipsTargetMachine &TM)
   // Use the default for now
   setOperationAction(ISD::STACKSAVE,         MVT::Other, Expand);
   setOperationAction(ISD::STACKRESTORE,      MVT::Other, Expand);
-  setOperationAction(ISD::MEMBARRIER,        MVT::Other, Expand);
+
+  setOperationAction(ISD::MEMBARRIER,        MVT::Other, Custom);
+  setOperationAction(ISD::ATOMIC_FENCE,      MVT::Other, Custom);  
+
+  setOperationAction(ISD::ATOMIC_LOAD,       MVT::i32,    Expand);  
+  setOperationAction(ISD::ATOMIC_STORE,      MVT::i32,    Expand);  
+
+  setInsertFencesForAtomic(true);
 
   if (Subtarget->isSingleFloat())
     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
@@ -180,6 +219,8 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setTargetDAGCombine(ISD::SDIVREM);
   setTargetDAGCombine(ISD::UDIVREM);
   setTargetDAGCombine(ISD::SETCC);
+  setTargetDAGCombine(ISD::AND);
+  setTargetDAGCombine(ISD::OR);
 
   setMinFunctionAlignment(2);
 
@@ -190,7 +231,12 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setExceptionSelectorRegister(Mips::A1);
 }
 
-MVT::SimpleValueType MipsTargetLowering::getSetCCResultType(EVT VT) const {
+bool MipsTargetLowering::allowsUnalignedMemoryAccesses(EVT VT) const {
+  MVT::SimpleValueType SVT = VT.getSimpleVT().SimpleTy;
+  return SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16; 
+}
+
+EVT MipsTargetLowering::getSetCCResultType(EVT VT) const {
   return MVT::i32;
 }
 
@@ -348,7 +394,7 @@ static SDValue PerformADDECombine(SDNode *N, SelectionDAG& DAG,
   if (DCI.isBeforeLegalize())
     return SDValue();
 
-  if (Subtarget->isMips32() && SelectMadd(N, &DAG))
+  if (Subtarget->hasMips32() && SelectMadd(N, &DAG))
     return SDValue(N, 0);
 
   return SDValue();
@@ -360,7 +406,7 @@ static SDValue PerformSUBECombine(SDNode *N, SelectionDAG& DAG,
   if (DCI.isBeforeLegalize())
     return SDValue();
 
-  if (Subtarget->isMips32() && SelectMsub(N, &DAG))
+  if (Subtarget->hasMips32() && SelectMsub(N, &DAG))
     return SDValue(N, 0);
 
   return SDValue();
@@ -372,6 +418,9 @@ static SDValue PerformDivRemCombine(SDNode *N, SelectionDAG& DAG,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
+  EVT Ty = N->getValueType(0);
+  unsigned LO = (Ty == MVT::i32) ? Mips::LO : Mips::LO64; 
+  unsigned HI = (Ty == MVT::i32) ? Mips::HI : Mips::HI64; 
   unsigned opc = N->getOpcode() == ISD::SDIVREM ? MipsISD::DivRem :
                                                   MipsISD::DivRemU;
   DebugLoc dl = N->getDebugLoc();
@@ -383,7 +432,7 @@ static SDValue PerformDivRemCombine(SDNode *N, SelectionDAG& DAG,
 
   // insert MFLO
   if (N->hasAnyUseOfValue(0)) {
-    SDValue CopyFromLo = DAG.getCopyFromReg(InChain, dl, Mips::LO, MVT::i32,
+    SDValue CopyFromLo = DAG.getCopyFromReg(InChain, dl, LO, Ty,
                                             InGlue);
     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), CopyFromLo);
     InChain = CopyFromLo.getValue(1);
@@ -393,7 +442,7 @@ static SDValue PerformDivRemCombine(SDNode *N, SelectionDAG& DAG,
   // insert MFHI
   if (N->hasAnyUseOfValue(1)) {
     SDValue CopyFromHi = DAG.getCopyFromReg(InChain, dl,
-                                            Mips::HI, MVT::i32, InGlue);
+                                            HI, Ty, InGlue);
     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), CopyFromHi);
   }
 
@@ -490,6 +539,101 @@ static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG& DAG,
   return CreateCMovFP(DAG, Cond, True, False, N->getDebugLoc());
 }
 
+static SDValue PerformANDCombine(SDNode *N, SelectionDAG& DAG,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 const MipsSubtarget* Subtarget) {
+  // Pattern match EXT.
+  //  $dst = and ((sra or srl) $src , pos), (2**size - 1)
+  //  => ext $dst, $src, size, pos
+  if (DCI.isBeforeLegalizeOps() || !Subtarget->hasMips32r2())
+    return SDValue();
+
+  SDValue ShiftRight = N->getOperand(0), Mask = N->getOperand(1);
+  
+  // Op's first operand must be a shift right.
+  if (ShiftRight.getOpcode() != ISD::SRA && ShiftRight.getOpcode() != ISD::SRL)
+    return SDValue();
+
+  // The second operand of the shift must be an immediate.
+  uint64_t Pos;
+  ConstantSDNode *CN;
+  if (!(CN = dyn_cast<ConstantSDNode>(ShiftRight.getOperand(1))))
+    return SDValue();
+  
+  Pos = CN->getZExtValue();
+
+  uint64_t SMPos, SMSize;
+  // Op's second operand must be a shifted mask.
+  if (!(CN = dyn_cast<ConstantSDNode>(Mask)) ||
+      !IsShiftedMask(CN->getZExtValue(), SMPos, SMSize))
+    return SDValue();
+
+  // Return if the shifted mask does not start at bit 0 or the sum of its size
+  // and Pos exceeds the word's size.
+  if (SMPos != 0 || Pos + SMSize > 32)
+    return SDValue();
+
+  return DAG.getNode(MipsISD::Ext, N->getDebugLoc(), MVT::i32,
+                     ShiftRight.getOperand(0),
+                     DAG.getConstant(Pos, MVT::i32),
+                     DAG.getConstant(SMSize, MVT::i32));
+}
+  
+static SDValue PerformORCombine(SDNode *N, SelectionDAG& DAG,
+                                TargetLowering::DAGCombinerInfo &DCI,
+                                const MipsSubtarget* Subtarget) {
+  // Pattern match INS.
+  //  $dst = or (and $src1 , mask0), (and (shl $src, pos), mask1),
+  //  where mask1 = (2**size - 1) << pos, mask0 = ~mask1 
+  //  => ins $dst, $src, size, pos, $src1
+  if (DCI.isBeforeLegalizeOps() || !Subtarget->hasMips32r2())
+    return SDValue();
+
+  SDValue And0 = N->getOperand(0), And1 = N->getOperand(1);
+  uint64_t SMPos0, SMSize0, SMPos1, SMSize1;
+  ConstantSDNode *CN;
+
+  // See if Op's first operand matches (and $src1 , mask0).
+  if (And0.getOpcode() != ISD::AND)
+    return SDValue();
+
+  if (!(CN = dyn_cast<ConstantSDNode>(And0.getOperand(1))) ||
+      !IsShiftedMask(~CN->getSExtValue(), SMPos0, SMSize0))
+    return SDValue();
+
+  // See if Op's second operand matches (and (shl $src, pos), mask1).
+  if (And1.getOpcode() != ISD::AND)
+    return SDValue();
+  
+  if (!(CN = dyn_cast<ConstantSDNode>(And1.getOperand(1))) ||
+      !IsShiftedMask(CN->getZExtValue(), SMPos1, SMSize1))
+    return SDValue();
+
+  // The shift masks must have the same position and size.
+  if (SMPos0 != SMPos1 || SMSize0 != SMSize1)
+    return SDValue();
+
+  SDValue Shl = And1.getOperand(0);
+  if (Shl.getOpcode() != ISD::SHL)
+    return SDValue();
+
+  if (!(CN = dyn_cast<ConstantSDNode>(Shl.getOperand(1))))
+    return SDValue();
+
+  unsigned Shamt = CN->getZExtValue();
+
+  // Return if the shift amount and the first bit position of mask are not the
+  // same.  
+  if (Shamt != SMPos0)
+    return SDValue();
+  
+  return DAG.getNode(MipsISD::Ins, N->getDebugLoc(), MVT::i32,
+                     Shl.getOperand(0),
+                     DAG.getConstant(SMPos0, MVT::i32),
+                     DAG.getConstant(SMSize0, MVT::i32),
+                     And0.getOperand(0));  
+}
+  
 SDValue  MipsTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
   const {
   SelectionDAG &DAG = DCI.DAG;
@@ -506,6 +650,10 @@ SDValue  MipsTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
     return PerformDivRemCombine(N, DAG, DCI, Subtarget);
   case ISD::SETCC:
     return PerformSETCCCombine(N, DAG, DCI, Subtarget);
+  case ISD::AND:
+    return PerformANDCombine(N, DAG, DCI, Subtarget);
+  case ISD::OR:
+    return PerformORCombine(N, DAG, DCI, Subtarget);
   }
 
   return SDValue();
@@ -527,6 +675,8 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const
     case ISD::VASTART:            return LowerVASTART(Op, DAG);
     case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
     case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
+    case ISD::MEMBARRIER:         return LowerMEMBARRIER(Op, DAG);
+    case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op, DAG);
   }
   return SDValue();
 }
@@ -733,13 +883,13 @@ MipsTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
   const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
   DebugLoc dl = MI->getDebugLoc();
 
-  unsigned Dest = MI->getOperand(0).getReg();
+  unsigned OldVal = MI->getOperand(0).getReg();
   unsigned Ptr = MI->getOperand(1).getReg();
   unsigned Incr = MI->getOperand(2).getReg();
 
-  unsigned Oldval = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp1 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp2 = RegInfo.createVirtualRegister(RC);
+  unsigned StoreVal = RegInfo.createVirtualRegister(RC);
+  unsigned AndRes = RegInfo.createVirtualRegister(RC);
+  unsigned Success = RegInfo.createVirtualRegister(RC);
 
   // insert new blocks after the current block
   const BasicBlock *LLVM_BB = BB->getBasicBlock();
@@ -758,61 +908,38 @@ MipsTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
 
   //  thisMBB:
   //    ...
-  //    sw incr, fi(sp)           // store incr to stack (when BinOpcode == 0)
   //    fallthrough --> loopMBB
-
-  // Note: for atomic.swap (when BinOpcode == 0), storing incr to stack before
-  // the loop and then loading it from stack in block loopMBB is necessary to
-  // prevent MachineLICM pass to hoist "or" instruction out of the block
-  // loopMBB.
-
-  int fi = 0;
-  if (BinOpcode == 0 && !Nand) {
-    // Get or create a temporary stack location.
-    MipsFunctionInfo *MipsFI = MF->getInfo<MipsFunctionInfo>();
-    fi = MipsFI->getAtomicFrameIndex();
-    if (fi == -1) {
-      fi = MF->getFrameInfo()->CreateStackObject(Size, Size, false);
-      MipsFI->setAtomicFrameIndex(fi);
-    }
-
-    BuildMI(BB, dl, TII->get(Mips::SW))
-        .addReg(Incr).addFrameIndex(fi).addImm(0);
-  }
   BB->addSuccessor(loopMBB);
+  loopMBB->addSuccessor(loopMBB);
+  loopMBB->addSuccessor(exitMBB);
 
   //  loopMBB:
   //    ll oldval, 0(ptr)
-  //    or dest, $0, oldval
-  //    <binop> tmp1, oldval, incr
-  //    sc tmp1, 0(ptr)
-  //    beq tmp1, $0, loopMBB
+  //    <binop> storeval, oldval, incr
+  //    sc success, storeval, 0(ptr)
+  //    beq success, $0, loopMBB
   BB = loopMBB;
-  BuildMI(BB, dl, TII->get(Mips::LL), Oldval).addReg(Ptr).addImm(0);
-  BuildMI(BB, dl, TII->get(Mips::OR), Dest).addReg(Mips::ZERO).addReg(Oldval);
+  BuildMI(BB, dl, TII->get(Mips::LL), OldVal).addReg(Ptr).addImm(0);
   if (Nand) {
-    //  and tmp2, oldval, incr
-    //  nor tmp1, $0, tmp2
-    BuildMI(BB, dl, TII->get(Mips::AND), Tmp2).addReg(Oldval).addReg(Incr);
-    BuildMI(BB, dl, TII->get(Mips::NOR), Tmp1).addReg(Mips::ZERO).addReg(Tmp2);
+    //  and andres, oldval, incr
+    //  nor storeval, $0, andres
+    BuildMI(BB, dl, TII->get(Mips::AND), AndRes).addReg(OldVal).addReg(Incr);
+    BuildMI(BB, dl, TII->get(Mips::NOR), StoreVal)
+      .addReg(Mips::ZERO).addReg(AndRes);
   } else if (BinOpcode) {
-    //  <binop> tmp1, oldval, incr
-    BuildMI(BB, dl, TII->get(BinOpcode), Tmp1).addReg(Oldval).addReg(Incr);
+    //  <binop> storeval, oldval, incr
+    BuildMI(BB, dl, TII->get(BinOpcode), StoreVal).addReg(OldVal).addReg(Incr);
   } else {
-    //  lw tmp2, fi(sp)              // load incr from stack
-    //  or tmp1, $zero, tmp2
-    BuildMI(BB, dl, TII->get(Mips::LW), Tmp2).addFrameIndex(fi).addImm(0);
-    BuildMI(BB, dl, TII->get(Mips::OR), Tmp1).addReg(Mips::ZERO).addReg(Tmp2);
+    StoreVal = Incr;
   }
-  BuildMI(BB, dl, TII->get(Mips::SC), Tmp1).addReg(Tmp1).addReg(Ptr).addImm(0);
+  BuildMI(BB, dl, TII->get(Mips::SC), Success)
+    .addReg(StoreVal).addReg(Ptr).addImm(0);
   BuildMI(BB, dl, TII->get(Mips::BEQ))
-    .addReg(Tmp1).addReg(Mips::ZERO).addMBB(loopMBB);
-  BB->addSuccessor(loopMBB);
-  BB->addSuccessor(exitMBB);
+    .addReg(Success).addReg(Mips::ZERO).addMBB(loopMBB);
 
   MI->eraseFromParent();   // The instruction is gone now.
 
-  return BB;
+  return exitMBB;
 }
 
 MachineBasicBlock *
@@ -833,33 +960,34 @@ MipsTargetLowering::EmitAtomicBinaryPartword(MachineInstr *MI,
   unsigned Ptr = MI->getOperand(1).getReg();
   unsigned Incr = MI->getOperand(2).getReg();
 
-  unsigned Addr = RegInfo.createVirtualRegister(RC);
-  unsigned Shift = RegInfo.createVirtualRegister(RC);
+  unsigned AlignedAddr = RegInfo.createVirtualRegister(RC);
+  unsigned ShiftAmt = RegInfo.createVirtualRegister(RC);
   unsigned Mask = RegInfo.createVirtualRegister(RC);
   unsigned Mask2 = RegInfo.createVirtualRegister(RC);
-  unsigned Newval = RegInfo.createVirtualRegister(RC);
-  unsigned Oldval = RegInfo.createVirtualRegister(RC);
+  unsigned NewVal = RegInfo.createVirtualRegister(RC);
+  unsigned OldVal = RegInfo.createVirtualRegister(RC);
   unsigned Incr2 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp1 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp2 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp3 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp4 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp5 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp6 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp7 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp8 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp9 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp10 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp11 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp12 = RegInfo.createVirtualRegister(RC);
+  unsigned MaskLSB2 = RegInfo.createVirtualRegister(RC);
+  unsigned PtrLSB2 = RegInfo.createVirtualRegister(RC);
+  unsigned MaskUpper = RegInfo.createVirtualRegister(RC);
+  unsigned AndRes = RegInfo.createVirtualRegister(RC);
+  unsigned BinOpRes = RegInfo.createVirtualRegister(RC);
+  unsigned MaskedOldVal0 = RegInfo.createVirtualRegister(RC);
+  unsigned StoreVal = RegInfo.createVirtualRegister(RC);
+  unsigned MaskedOldVal1 = RegInfo.createVirtualRegister(RC);
+  unsigned SrlRes = RegInfo.createVirtualRegister(RC);
+  unsigned SllRes = RegInfo.createVirtualRegister(RC);
+  unsigned Success = RegInfo.createVirtualRegister(RC);
 
   // insert new blocks after the current block
   const BasicBlock *LLVM_BB = BB->getBasicBlock();
   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
   MachineFunction::iterator It = BB;
   ++It;
   MF->insert(It, loopMBB);
+  MF->insert(It, sinkMBB);
   MF->insert(It, exitMBB);
 
   // Transfer the remainder of BB and its successor edges to exitMBB.
@@ -868,111 +996,104 @@ MipsTargetLowering::EmitAtomicBinaryPartword(MachineInstr *MI,
                   BB->end());
   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
-  //  thisMBB:
-  //    addiu   tmp1,$0,-4                # 0xfffffffc
-  //    and     addr,ptr,tmp1
-  //    andi    tmp2,ptr,3
-  //    sll     shift,tmp2,3
-  //    ori     tmp3,$0,255               # 0xff
-  //    sll     mask,tmp3,shift
-  //    nor     mask2,$0,mask
-  //    andi    tmp4,incr,255
-  //    sll     incr2,tmp4,shift
-  //    sw      incr2, fi(sp)      // store incr2 to stack (when BinOpcode == 0)
+  BB->addSuccessor(loopMBB);
+  loopMBB->addSuccessor(loopMBB);
+  loopMBB->addSuccessor(sinkMBB);
+  sinkMBB->addSuccessor(exitMBB);
 
-  // Note: for atomic.swap (when BinOpcode == 0), storing incr2 to stack before
-  // the loop and then loading it from stack in block loopMBB is necessary to
-  // prevent MachineLICM pass to hoist "or" instruction out of the block
-  // loopMBB.
+  //  thisMBB:
+  //    addiu   masklsb2,$0,-4                # 0xfffffffc
+  //    and     alignedaddr,ptr,masklsb2
+  //    andi    ptrlsb2,ptr,3
+  //    sll     shiftamt,ptrlsb2,3
+  //    ori     maskupper,$0,255               # 0xff
+  //    sll     mask,maskupper,shiftamt
+  //    nor     mask2,$0,mask
+  //    sll     incr2,incr,shiftamt
 
   int64_t MaskImm = (Size == 1) ? 255 : 65535;
-  BuildMI(BB, dl, TII->get(Mips::ADDiu), Tmp1).addReg(Mips::ZERO).addImm(-4);
-  BuildMI(BB, dl, TII->get(Mips::AND), Addr).addReg(Ptr).addReg(Tmp1);
-  BuildMI(BB, dl, TII->get(Mips::ANDi), Tmp2).addReg(Ptr).addImm(3);
-  BuildMI(BB, dl, TII->get(Mips::SLL), Shift).addReg(Tmp2).addImm(3);
-  BuildMI(BB, dl, TII->get(Mips::ORi), Tmp3).addReg(Mips::ZERO).addImm(MaskImm);
-  BuildMI(BB, dl, TII->get(Mips::SLL), Mask).addReg(Tmp3).addReg(Shift);
+  BuildMI(BB, dl, TII->get(Mips::ADDiu), MaskLSB2)
+    .addReg(Mips::ZERO).addImm(-4);
+  BuildMI(BB, dl, TII->get(Mips::AND), AlignedAddr)
+    .addReg(Ptr).addReg(MaskLSB2);
+  BuildMI(BB, dl, TII->get(Mips::ANDi), PtrLSB2).addReg(Ptr).addImm(3);
+  BuildMI(BB, dl, TII->get(Mips::SLL), ShiftAmt).addReg(PtrLSB2).addImm(3);
+  BuildMI(BB, dl, TII->get(Mips::ORi), MaskUpper)
+    .addReg(Mips::ZERO).addImm(MaskImm);
+  BuildMI(BB, dl, TII->get(Mips::SLLV), Mask)
+    .addReg(ShiftAmt).addReg(MaskUpper);
   BuildMI(BB, dl, TII->get(Mips::NOR), Mask2).addReg(Mips::ZERO).addReg(Mask);
-  if (BinOpcode != Mips::SUBu) {
-    BuildMI(BB, dl, TII->get(Mips::ANDi), Tmp4).addReg(Incr).addImm(MaskImm);
-    BuildMI(BB, dl, TII->get(Mips::SLL), Incr2).addReg(Tmp4).addReg(Shift);
-  } else {
-    BuildMI(BB, dl, TII->get(Mips::SUBu), Tmp4).addReg(Mips::ZERO).addReg(Incr);
-    BuildMI(BB, dl, TII->get(Mips::ANDi), Tmp5).addReg(Tmp4).addImm(MaskImm);
-    BuildMI(BB, dl, TII->get(Mips::SLL), Incr2).addReg(Tmp5).addReg(Shift);
-  }
+  BuildMI(BB, dl, TII->get(Mips::SLLV), Incr2).addReg(ShiftAmt).addReg(Incr);
 
-  int fi = 0;
-  if (BinOpcode == 0 && !Nand) {
-    // Get or create a temporary stack location.
-    MipsFunctionInfo *MipsFI = MF->getInfo<MipsFunctionInfo>();
-    fi = MipsFI->getAtomicFrameIndex();
-    if (fi == -1) {
-      fi = MF->getFrameInfo()->CreateStackObject(Size, Size, false);
-      MipsFI->setAtomicFrameIndex(fi);
-    }
-
-    BuildMI(BB, dl, TII->get(Mips::SW))
-        .addReg(Incr2).addFrameIndex(fi).addImm(0);
-  }
-  BB->addSuccessor(loopMBB);
 
+  // atomic.load.binop
   // loopMBB:
-  //   ll      oldval,0(addr)
-  //   binop   tmp7,oldval,incr2
-  //   and     newval,tmp7,mask
-  //   and     tmp8,oldval,mask2
-  //   or      tmp9,tmp8,newval
-  //   sc      tmp9,0(addr)
-  //   beq     tmp9,$0,loopMBB
-  BB = loopMBB;
-  BuildMI(BB, dl, TII->get(Mips::LL), Oldval).addReg(Addr).addImm(0);
-  if (Nand) {
-    //  and tmp6, oldval, incr2
-    //  nor tmp7, $0, tmp6
-    BuildMI(BB, dl, TII->get(Mips::AND), Tmp6).addReg(Oldval).addReg(Incr2);
-    BuildMI(BB, dl, TII->get(Mips::NOR), Tmp7).addReg(Mips::ZERO).addReg(Tmp6);
-  } else if (BinOpcode == Mips::SUBu) {
-    //  addu tmp7, oldval, incr2
-    BuildMI(BB, dl, TII->get(Mips::ADDu), Tmp7).addReg(Oldval).addReg(Incr2);
-  } else if (BinOpcode) {
-    //  <binop> tmp7, oldval, incr2
-    BuildMI(BB, dl, TII->get(BinOpcode), Tmp7).addReg(Oldval).addReg(Incr2);
-  } else {
-    //  lw tmp6, fi(sp)              // load incr2 from stack
-    //  or tmp7, $zero, tmp6
-    BuildMI(BB, dl, TII->get(Mips::LW), Tmp6).addFrameIndex(fi).addImm(0);
-    BuildMI(BB, dl, TII->get(Mips::OR), Tmp7).addReg(Mips::ZERO).addReg(Tmp6);
-  }
-  BuildMI(BB, dl, TII->get(Mips::AND), Newval).addReg(Tmp7).addReg(Mask);
-  BuildMI(BB, dl, TII->get(Mips::AND), Tmp8).addReg(Oldval).addReg(Mask2);
-  BuildMI(BB, dl, TII->get(Mips::OR), Tmp9).addReg(Tmp8).addReg(Newval);
-  BuildMI(BB, dl, TII->get(Mips::SC), Tmp9).addReg(Tmp9).addReg(Addr).addImm(0);
-  BuildMI(BB, dl, TII->get(Mips::BEQ))
-      .addReg(Tmp9).addReg(Mips::ZERO).addMBB(loopMBB);
-  BB->addSuccessor(loopMBB);
-  BB->addSuccessor(exitMBB);
+  //   ll      oldval,0(alignedaddr)
+  //   binop   binopres,oldval,incr2
+  //   and     newval,binopres,mask
+  //   and     maskedoldval0,oldval,mask2
+  //   or      storeval,maskedoldval0,newval
+  //   sc      success,storeval,0(alignedaddr)
+  //   beq     success,$0,loopMBB
 
-  //  exitMBB:
-  //    and     tmp10,oldval,mask
-  //    srl     tmp11,tmp10,shift
-  //    sll     tmp12,tmp11,24
-  //    sra     dest,tmp12,24
-  BB = exitMBB;
+  // atomic.swap
+  // loopMBB:
+  //   ll      oldval,0(alignedaddr)
+  //   and     newval,incr2,mask
+  //   and     maskedoldval0,oldval,mask2
+  //   or      storeval,maskedoldval0,newval
+  //   sc      success,storeval,0(alignedaddr)
+  //   beq     success,$0,loopMBB
+
+  BB = loopMBB;
+  BuildMI(BB, dl, TII->get(Mips::LL), OldVal).addReg(AlignedAddr).addImm(0);
+  if (Nand) {
+    //  and andres, oldval, incr2
+    //  nor binopres, $0, andres
+    //  and newval, binopres, mask
+    BuildMI(BB, dl, TII->get(Mips::AND), AndRes).addReg(OldVal).addReg(Incr2);
+    BuildMI(BB, dl, TII->get(Mips::NOR), BinOpRes)
+      .addReg(Mips::ZERO).addReg(AndRes);
+    BuildMI(BB, dl, TII->get(Mips::AND), NewVal).addReg(BinOpRes).addReg(Mask);
+  } else if (BinOpcode) {
+    //  <binop> binopres, oldval, incr2
+    //  and newval, binopres, mask
+    BuildMI(BB, dl, TII->get(BinOpcode), BinOpRes).addReg(OldVal).addReg(Incr2);
+    BuildMI(BB, dl, TII->get(Mips::AND), NewVal).addReg(BinOpRes).addReg(Mask);
+  } else {// atomic.swap
+    //  and newval, incr2, mask
+    BuildMI(BB, dl, TII->get(Mips::AND), NewVal).addReg(Incr2).addReg(Mask);
+  }
+    
+  BuildMI(BB, dl, TII->get(Mips::AND), MaskedOldVal0)
+    .addReg(OldVal).addReg(Mask2);
+  BuildMI(BB, dl, TII->get(Mips::OR), StoreVal)
+    .addReg(MaskedOldVal0).addReg(NewVal);
+  BuildMI(BB, dl, TII->get(Mips::SC), Success)
+    .addReg(StoreVal).addReg(AlignedAddr).addImm(0);
+  BuildMI(BB, dl, TII->get(Mips::BEQ))
+    .addReg(Success).addReg(Mips::ZERO).addMBB(loopMBB);
+
+  //  sinkMBB:
+  //    and     maskedoldval1,oldval,mask
+  //    srl     srlres,maskedoldval1,shiftamt
+  //    sll     sllres,srlres,24
+  //    sra     dest,sllres,24
+  BB = sinkMBB;
   int64_t ShiftImm = (Size == 1) ? 24 : 16;
-  // reverse order
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::SRA), Dest)
-      .addReg(Tmp12).addImm(ShiftImm);
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::SLL), Tmp12)
-      .addReg(Tmp11).addImm(ShiftImm);
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::SRL), Tmp11)
-      .addReg(Tmp10).addReg(Shift);
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::AND), Tmp10)
-    .addReg(Oldval).addReg(Mask);
+
+  BuildMI(BB, dl, TII->get(Mips::AND), MaskedOldVal1)
+    .addReg(OldVal).addReg(Mask);
+  BuildMI(BB, dl, TII->get(Mips::SRLV), SrlRes)
+      .addReg(ShiftAmt).addReg(MaskedOldVal1);
+  BuildMI(BB, dl, TII->get(Mips::SLL), SllRes)
+      .addReg(SrlRes).addImm(ShiftImm);
+  BuildMI(BB, dl, TII->get(Mips::SRA), Dest)
+      .addReg(SllRes).addImm(ShiftImm);
 
   MI->eraseFromParent();   // The instruction is gone now.
 
-  return BB;
+  return exitMBB;
 }
 
 MachineBasicBlock *
@@ -989,11 +1110,10 @@ MipsTargetLowering::EmitAtomicCmpSwap(MachineInstr *MI,
 
   unsigned Dest    = MI->getOperand(0).getReg();
   unsigned Ptr     = MI->getOperand(1).getReg();
-  unsigned Oldval  = MI->getOperand(2).getReg();
-  unsigned Newval  = MI->getOperand(3).getReg();
+  unsigned OldVal  = MI->getOperand(2).getReg();
+  unsigned NewVal  = MI->getOperand(3).getReg();
 
-  unsigned Tmp1 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp2 = RegInfo.createVirtualRegister(RC);
+  unsigned Success = RegInfo.createVirtualRegister(RC);
 
   // insert new blocks after the current block
   const BasicBlock *LLVM_BB = BB->getBasicBlock();
@@ -1012,26 +1132,14 @@ MipsTargetLowering::EmitAtomicCmpSwap(MachineInstr *MI,
                   BB->end());
   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
-  // Get or create a temporary stack location.
-  MipsFunctionInfo *MipsFI = MF->getInfo<MipsFunctionInfo>();
-  int fi = MipsFI->getAtomicFrameIndex();
-  if (fi == -1) {
-    fi = MF->getFrameInfo()->CreateStackObject(Size, Size, false);
-    MipsFI->setAtomicFrameIndex(fi);
-  }
-
   //  thisMBB:
   //    ...
-  //    sw newval, fi(sp)           // store newval to stack
   //    fallthrough --> loop1MBB
-
-  // Note: storing newval to stack before the loop and then loading it from
-  // stack in block loop2MBB is necessary to prevent MachineLICM pass to
-  // hoist "or" instruction out of the block loop2MBB.
-
-  BuildMI(BB, dl, TII->get(Mips::SW))
-      .addReg(Newval).addFrameIndex(fi).addImm(0);
   BB->addSuccessor(loop1MBB);
+  loop1MBB->addSuccessor(exitMBB);
+  loop1MBB->addSuccessor(loop2MBB);
+  loop2MBB->addSuccessor(loop1MBB);
+  loop2MBB->addSuccessor(exitMBB);
 
   // loop1MBB:
   //   ll dest, 0(ptr)
@@ -1039,27 +1147,20 @@ MipsTargetLowering::EmitAtomicCmpSwap(MachineInstr *MI,
   BB = loop1MBB;
   BuildMI(BB, dl, TII->get(Mips::LL), Dest).addReg(Ptr).addImm(0);
   BuildMI(BB, dl, TII->get(Mips::BNE))
-    .addReg(Dest).addReg(Oldval).addMBB(exitMBB);
-  BB->addSuccessor(exitMBB);
-  BB->addSuccessor(loop2MBB);
+    .addReg(Dest).addReg(OldVal).addMBB(exitMBB);
 
   // loop2MBB:
-  //   lw tmp2, fi(sp)              // load newval from stack
-  //   or tmp1, $0, tmp2
-  //   sc tmp1, 0(ptr)
-  //   beq tmp1, $0, loop1MBB
+  //   sc success, newval, 0(ptr)
+  //   beq success, $0, loop1MBB
   BB = loop2MBB;
-  BuildMI(BB, dl, TII->get(Mips::LW), Tmp2).addFrameIndex(fi).addImm(0);
-  BuildMI(BB, dl, TII->get(Mips::OR), Tmp1).addReg(Mips::ZERO).addReg(Tmp2);
-  BuildMI(BB, dl, TII->get(Mips::SC), Tmp1).addReg(Tmp1).addReg(Ptr).addImm(0);
+  BuildMI(BB, dl, TII->get(Mips::SC), Success)
+    .addReg(NewVal).addReg(Ptr).addImm(0);
   BuildMI(BB, dl, TII->get(Mips::BEQ))
-    .addReg(Tmp1).addReg(Mips::ZERO).addMBB(loop1MBB);
-  BB->addSuccessor(loop1MBB);
-  BB->addSuccessor(exitMBB);
+    .addReg(Success).addReg(Mips::ZERO).addMBB(loop1MBB);
 
   MI->eraseFromParent();   // The instruction is gone now.
 
-  return BB;
+  return exitMBB;
 }
 
 MachineBasicBlock *
@@ -1077,36 +1178,39 @@ MipsTargetLowering::EmitAtomicCmpSwapPartword(MachineInstr *MI,
 
   unsigned Dest    = MI->getOperand(0).getReg();
   unsigned Ptr     = MI->getOperand(1).getReg();
-  unsigned Oldval  = MI->getOperand(2).getReg();
-  unsigned Newval  = MI->getOperand(3).getReg();
+  unsigned CmpVal  = MI->getOperand(2).getReg();
+  unsigned NewVal  = MI->getOperand(3).getReg();
 
-  unsigned Addr = RegInfo.createVirtualRegister(RC);
-  unsigned Shift = RegInfo.createVirtualRegister(RC);
+  unsigned AlignedAddr = RegInfo.createVirtualRegister(RC);
+  unsigned ShiftAmt = RegInfo.createVirtualRegister(RC);
   unsigned Mask = RegInfo.createVirtualRegister(RC);
   unsigned Mask2 = RegInfo.createVirtualRegister(RC);
-  unsigned Oldval2 = RegInfo.createVirtualRegister(RC);
-  unsigned Oldval3 = RegInfo.createVirtualRegister(RC);
-  unsigned Oldval4 = RegInfo.createVirtualRegister(RC);
-  unsigned Newval2 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp1 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp2 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp3 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp4 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp5 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp6 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp7 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp8 = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp9 = RegInfo.createVirtualRegister(RC);
+  unsigned ShiftedCmpVal = RegInfo.createVirtualRegister(RC);
+  unsigned OldVal = RegInfo.createVirtualRegister(RC);
+  unsigned MaskedOldVal0 = RegInfo.createVirtualRegister(RC);
+  unsigned ShiftedNewVal = RegInfo.createVirtualRegister(RC);
+  unsigned MaskLSB2 = RegInfo.createVirtualRegister(RC);
+  unsigned PtrLSB2 = RegInfo.createVirtualRegister(RC);
+  unsigned MaskUpper = RegInfo.createVirtualRegister(RC);
+  unsigned MaskedCmpVal = RegInfo.createVirtualRegister(RC);
+  unsigned MaskedNewVal = RegInfo.createVirtualRegister(RC);
+  unsigned MaskedOldVal1 = RegInfo.createVirtualRegister(RC);
+  unsigned StoreVal = RegInfo.createVirtualRegister(RC);
+  unsigned SrlRes = RegInfo.createVirtualRegister(RC);
+  unsigned SllRes = RegInfo.createVirtualRegister(RC);
+  unsigned Success = RegInfo.createVirtualRegister(RC);
 
   // insert new blocks after the current block
   const BasicBlock *LLVM_BB = BB->getBasicBlock();
   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
   MachineFunction::iterator It = BB;
   ++It;
   MF->insert(It, loop1MBB);
   MF->insert(It, loop2MBB);
+  MF->insert(It, sinkMBB);
   MF->insert(It, exitMBB);
 
   // Transfer the remainder of BB and its successor edges to exitMBB.
@@ -1115,76 +1219,90 @@ MipsTargetLowering::EmitAtomicCmpSwapPartword(MachineInstr *MI,
                   BB->end());
   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
-  //  thisMBB:
-  //    addiu   tmp1,$0,-4                # 0xfffffffc
-  //    and     addr,ptr,tmp1
-  //    andi    tmp2,ptr,3
-  //    sll     shift,tmp2,3
-  //    ori     tmp3,$0,255               # 0xff
-  //    sll     mask,tmp3,shift
-  //    nor     mask2,$0,mask
-  //    andi    tmp4,oldval,255
-  //    sll     oldval2,tmp4,shift
-  //    andi    tmp5,newval,255
-  //    sll     newval2,tmp5,shift
-  int64_t MaskImm = (Size == 1) ? 255 : 65535;
-  BuildMI(BB, dl, TII->get(Mips::ADDiu), Tmp1).addReg(Mips::ZERO).addImm(-4);
-  BuildMI(BB, dl, TII->get(Mips::AND), Addr).addReg(Ptr).addReg(Tmp1);
-  BuildMI(BB, dl, TII->get(Mips::ANDi), Tmp2).addReg(Ptr).addImm(3);
-  BuildMI(BB, dl, TII->get(Mips::SLL), Shift).addReg(Tmp2).addImm(3);
-  BuildMI(BB, dl, TII->get(Mips::ORi), Tmp3).addReg(Mips::ZERO).addImm(MaskImm);
-  BuildMI(BB, dl, TII->get(Mips::SLL), Mask).addReg(Tmp3).addReg(Shift);
-  BuildMI(BB, dl, TII->get(Mips::NOR), Mask2).addReg(Mips::ZERO).addReg(Mask);
-  BuildMI(BB, dl, TII->get(Mips::ANDi), Tmp4).addReg(Oldval).addImm(MaskImm);
-  BuildMI(BB, dl, TII->get(Mips::SLL), Oldval2).addReg(Tmp4).addReg(Shift);
-  BuildMI(BB, dl, TII->get(Mips::ANDi), Tmp5).addReg(Newval).addImm(MaskImm);
-  BuildMI(BB, dl, TII->get(Mips::SLL), Newval2).addReg(Tmp5).addReg(Shift);
   BB->addSuccessor(loop1MBB);
+  loop1MBB->addSuccessor(sinkMBB);
+  loop1MBB->addSuccessor(loop2MBB);
+  loop2MBB->addSuccessor(loop1MBB);
+  loop2MBB->addSuccessor(sinkMBB);
+  sinkMBB->addSuccessor(exitMBB);
+
+  // FIXME: computation of newval2 can be moved to loop2MBB.
+  //  thisMBB:
+  //    addiu   masklsb2,$0,-4                # 0xfffffffc
+  //    and     alignedaddr,ptr,masklsb2
+  //    andi    ptrlsb2,ptr,3
+  //    sll     shiftamt,ptrlsb2,3
+  //    ori     maskupper,$0,255               # 0xff
+  //    sll     mask,maskupper,shiftamt
+  //    nor     mask2,$0,mask
+  //    andi    maskedcmpval,cmpval,255
+  //    sll     shiftedcmpval,maskedcmpval,shiftamt
+  //    andi    maskednewval,newval,255
+  //    sll     shiftednewval,maskednewval,shiftamt
+  int64_t MaskImm = (Size == 1) ? 255 : 65535;
+  BuildMI(BB, dl, TII->get(Mips::ADDiu), MaskLSB2)
+    .addReg(Mips::ZERO).addImm(-4);
+  BuildMI(BB, dl, TII->get(Mips::AND), AlignedAddr)
+    .addReg(Ptr).addReg(MaskLSB2);
+  BuildMI(BB, dl, TII->get(Mips::ANDi), PtrLSB2).addReg(Ptr).addImm(3);
+  BuildMI(BB, dl, TII->get(Mips::SLL), ShiftAmt).addReg(PtrLSB2).addImm(3);
+  BuildMI(BB, dl, TII->get(Mips::ORi), MaskUpper)
+    .addReg(Mips::ZERO).addImm(MaskImm);
+  BuildMI(BB, dl, TII->get(Mips::SLLV), Mask)
+    .addReg(ShiftAmt).addReg(MaskUpper);
+  BuildMI(BB, dl, TII->get(Mips::NOR), Mask2).addReg(Mips::ZERO).addReg(Mask);
+  BuildMI(BB, dl, TII->get(Mips::ANDi), MaskedCmpVal)
+    .addReg(CmpVal).addImm(MaskImm);
+  BuildMI(BB, dl, TII->get(Mips::SLLV), ShiftedCmpVal)
+    .addReg(ShiftAmt).addReg(MaskedCmpVal);
+  BuildMI(BB, dl, TII->get(Mips::ANDi), MaskedNewVal)
+    .addReg(NewVal).addImm(MaskImm);
+  BuildMI(BB, dl, TII->get(Mips::SLLV), ShiftedNewVal)
+    .addReg(ShiftAmt).addReg(MaskedNewVal);
 
   //  loop1MBB:
-  //    ll      oldval3,0(addr)
-  //    and     oldval4,oldval3,mask
-  //    bne     oldval4,oldval2,exitMBB
+  //    ll      oldval,0(alginedaddr)
+  //    and     maskedoldval0,oldval,mask
+  //    bne     maskedoldval0,shiftedcmpval,sinkMBB
   BB = loop1MBB;
-  BuildMI(BB, dl, TII->get(Mips::LL), Oldval3).addReg(Addr).addImm(0);
-  BuildMI(BB, dl, TII->get(Mips::AND), Oldval4).addReg(Oldval3).addReg(Mask);
+  BuildMI(BB, dl, TII->get(Mips::LL), OldVal).addReg(AlignedAddr).addImm(0);
+  BuildMI(BB, dl, TII->get(Mips::AND), MaskedOldVal0)
+    .addReg(OldVal).addReg(Mask);
   BuildMI(BB, dl, TII->get(Mips::BNE))
-      .addReg(Oldval4).addReg(Oldval2).addMBB(exitMBB);
-  BB->addSuccessor(exitMBB);
-  BB->addSuccessor(loop2MBB);
+    .addReg(MaskedOldVal0).addReg(ShiftedCmpVal).addMBB(sinkMBB);
 
   //  loop2MBB:
-  //    and     tmp6,oldval3,mask2
-  //    or      tmp7,tmp6,newval2
-  //    sc      tmp7,0(addr)
-  //    beq     tmp7,$0,loop1MBB
+  //    and     maskedoldval1,oldval,mask2
+  //    or      storeval,maskedoldval1,shiftednewval
+  //    sc      success,storeval,0(alignedaddr)
+  //    beq     success,$0,loop1MBB
   BB = loop2MBB;
-  BuildMI(BB, dl, TII->get(Mips::AND), Tmp6).addReg(Oldval3).addReg(Mask2);
-  BuildMI(BB, dl, TII->get(Mips::OR), Tmp7).addReg(Tmp6).addReg(Newval2);
-  BuildMI(BB, dl, TII->get(Mips::SC), Tmp7)
-      .addReg(Tmp7).addReg(Addr).addImm(0);
+  BuildMI(BB, dl, TII->get(Mips::AND), MaskedOldVal1)
+    .addReg(OldVal).addReg(Mask2);
+  BuildMI(BB, dl, TII->get(Mips::OR), StoreVal)
+    .addReg(MaskedOldVal1).addReg(ShiftedNewVal);
+  BuildMI(BB, dl, TII->get(Mips::SC), Success)
+      .addReg(StoreVal).addReg(AlignedAddr).addImm(0);
   BuildMI(BB, dl, TII->get(Mips::BEQ))
-      .addReg(Tmp7).addReg(Mips::ZERO).addMBB(loop1MBB);
-  BB->addSuccessor(loop1MBB);
-  BB->addSuccessor(exitMBB);
+      .addReg(Success).addReg(Mips::ZERO).addMBB(loop1MBB);
 
-  //  exitMBB:
-  //    srl     tmp8,oldval4,shift
-  //    sll     tmp9,tmp8,24
-  //    sra     dest,tmp9,24
-  BB = exitMBB;
+  //  sinkMBB:
+  //    srl     srlres,maskedoldval0,shiftamt
+  //    sll     sllres,srlres,24
+  //    sra     dest,sllres,24
+  BB = sinkMBB;
   int64_t ShiftImm = (Size == 1) ? 24 : 16;
-  // reverse order
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::SRA), Dest)
-      .addReg(Tmp9).addImm(ShiftImm);
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::SLL), Tmp9)
-      .addReg(Tmp8).addImm(ShiftImm);
-  BuildMI(*BB, BB->begin(), dl, TII->get(Mips::SRL), Tmp8)
-      .addReg(Oldval4).addReg(Shift);
+
+  BuildMI(BB, dl, TII->get(Mips::SRLV), SrlRes)
+      .addReg(ShiftAmt).addReg(MaskedOldVal0);
+  BuildMI(BB, dl, TII->get(Mips::SLL), SllRes)
+      .addReg(SrlRes).addImm(ShiftImm);
+  BuildMI(BB, dl, TII->get(Mips::SRA), Dest)
+      .addReg(SllRes).addImm(ShiftImm);
 
   MI->eraseFromParent();   // The instruction is gone now.
 
-  return BB;
+  return exitMBB;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1267,9 +1385,9 @@ SDValue MipsTargetLowering::LowerGlobalAddress(SDValue Op,
                                                SelectionDAG &DAG) const {
   // FIXME there isn't actually debug info here
   DebugLoc dl = Op.getDebugLoc();
-  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
+  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 	
 
-  if (getTargetMachine().getRelocationModel() != Reloc::PIC_) {
+  if (getTargetMachine().getRelocationModel() != Reloc::PIC_ && !IsN64) {
     SDVTList VTs = DAG.getVTList(MVT::i32);
 
     MipsTargetObjectFile &TLOF = (MipsTargetObjectFile&)getObjFileLowering();
@@ -1292,21 +1410,26 @@ SDValue MipsTargetLowering::LowerGlobalAddress(SDValue Op,
     return DAG.getNode(ISD::ADD, dl, MVT::i32, HiPart, Lo);
   }
 
-  SDValue GA = DAG.getTargetGlobalAddress(GV, dl, MVT::i32, 0,
-                                          MipsII::MO_GOT);
-  GA = DAG.getNode(MipsISD::WrapperPIC, dl, MVT::i32, GA);
-  SDValue ResNode = DAG.getLoad(MVT::i32, dl,
+  EVT ValTy = Op.getValueType();
+  bool HasGotOfst = (GV->hasInternalLinkage() ||
+                     (GV->hasLocalLinkage() && !isa<Function>(GV)));
+  unsigned GotFlag = IsN64 ?
+                     (HasGotOfst ? MipsII::MO_GOT_PAGE : MipsII::MO_GOT_DISP) :
+                     MipsII::MO_GOT;
+  SDValue GA = DAG.getTargetGlobalAddress(GV, dl, ValTy, 0, GotFlag);
+  GA = DAG.getNode(MipsISD::WrapperPIC, dl, ValTy, GA);
+  SDValue ResNode = DAG.getLoad(ValTy, dl,
                                 DAG.getEntryNode(), GA, MachinePointerInfo(),
                                 false, false, 0);
   // On functions and global targets not internal linked only
   // a load from got/GP is necessary for PIC to work.
-  if (!GV->hasInternalLinkage() &&
-      (!GV->hasLocalLinkage() || isa<Function>(GV)))
+  if (!HasGotOfst)
     return ResNode;
-  SDValue GALo = DAG.getTargetGlobalAddress(GV, dl, MVT::i32, 0,
-                                            MipsII::MO_ABS_LO);
-  SDValue Lo = DAG.getNode(MipsISD::Lo, dl, MVT::i32, GALo);
-  return DAG.getNode(ISD::ADD, dl, MVT::i32, ResNode, Lo);
+  SDValue GALo = DAG.getTargetGlobalAddress(GV, dl, ValTy, 0,
+                                            IsN64 ? MipsII::MO_GOT_OFST :
+                                                    MipsII::MO_ABS_LO);
+  SDValue Lo = DAG.getNode(MipsISD::Lo, dl, ValTy, GALo);
+  return DAG.getNode(ISD::ADD, dl, ValTy, ResNode, Lo);
 }
 
 SDValue MipsTargetLowering::LowerBlockAddress(SDValue Op,
@@ -1361,11 +1484,11 @@ LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
     ArgListTy Args;
     ArgListEntry Entry;
     Entry.Node = Argument;
-    Entry.Ty = (const Type *) Type::getInt32Ty(*DAG.getContext());
+    Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
     Args.push_back(Entry);
     std::pair<SDValue, SDValue> CallResult =
         LowerCallTo(DAG.getEntryNode(),
-                    (const Type *) Type::getInt32Ty(*DAG.getContext()),
+                    (Type *) Type::getInt32Ty(*DAG.getContext()),
                     false, false, false, false, 0, CallingConv::C, false, true,
                     DAG.getExternalSymbol("__tls_get_addr", PtrVT), Args, DAG,
                     dl);
@@ -1557,6 +1680,25 @@ LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
   return FrameAddr;
 }
 
+// TODO: set SType according to the desired memory barrier behavior.
+SDValue MipsTargetLowering::LowerMEMBARRIER(SDValue Op,
+                                            SelectionDAG& DAG) const {
+  unsigned SType = 0;
+  DebugLoc dl = Op.getDebugLoc();
+  return DAG.getNode(MipsISD::Sync, dl, MVT::Other, Op.getOperand(0),
+                     DAG.getConstant(SType, MVT::i32));
+}
+
+SDValue MipsTargetLowering::LowerATOMIC_FENCE(SDValue Op,
+                                              SelectionDAG& DAG) const {
+  // FIXME: Need pseudo-fence for 'singlethread' fences
+  // FIXME: Set SType for weaker fences where supported/appropriate.
+  unsigned SType = 0;
+  DebugLoc dl = Op.getDebugLoc();
+  return DAG.getNode(MipsISD::Sync, dl, MVT::Other, Op.getOperand(0),
+                     DAG.getConstant(SType, MVT::i32));
+}
+
 //===----------------------------------------------------------------------===//
 //                      Calling Convention Implementation
 //===----------------------------------------------------------------------===//
@@ -1679,55 +1821,109 @@ static const unsigned O32IntRegs[] = {
   Mips::A0, Mips::A1, Mips::A2, Mips::A3
 };
 
+// Return next O32 integer argument register.
+static unsigned getNextIntArgReg(unsigned Reg) {
+  assert((Reg == Mips::A0) || (Reg == Mips::A2));
+  return (Reg == Mips::A0) ? Mips::A1 : Mips::A3;
+}
+
 // Write ByVal Arg to arg registers and stack.
 static void
-WriteByValArg(SDValue& Chain, DebugLoc dl,
+WriteByValArg(SDValue& ByValChain, SDValue Chain, DebugLoc dl,
               SmallVector<std::pair<unsigned, SDValue>, 16>& RegsToPass,
               SmallVector<SDValue, 8>& MemOpChains, int& LastFI,
               MachineFrameInfo *MFI, SelectionDAG &DAG, SDValue Arg,
               const CCValAssign &VA, const ISD::ArgFlagsTy& Flags,
-              MVT PtrType) {
-  unsigned FirstWord = VA.getLocMemOffset() / 4;
-  unsigned NumWords = (Flags.getByValSize() + 3) / 4;
-  unsigned LastWord = FirstWord + NumWords;
-  unsigned CurWord;
+              MVT PtrType, bool isLittle) {
+  unsigned LocMemOffset = VA.getLocMemOffset();
+  unsigned Offset = 0;
+  uint32_t RemainingSize = Flags.getByValSize();
+  unsigned ByValAlign = Flags.getByValAlign();
 
-  // copy the first 4 words of byval arg to registers A0 - A3
-  for (CurWord = FirstWord; CurWord < std::min(LastWord, O32IntRegsSize);
-       ++CurWord) {
+  // Copy the first 4 words of byval arg to registers A0 - A3.
+  // FIXME: Use a stricter alignment if it enables better optimization in passes
+  //        run later.
+  for (; RemainingSize >= 4 && LocMemOffset < 4 * 4;
+       Offset += 4, RemainingSize -= 4, LocMemOffset += 4) {
     SDValue LoadPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
-                                  DAG.getConstant((CurWord - FirstWord) * 4,
-                                                  MVT::i32));
+                                  DAG.getConstant(Offset, MVT::i32));
     SDValue LoadVal = DAG.getLoad(MVT::i32, dl, Chain, LoadPtr,
                                   MachinePointerInfo(),
-                                  false, false, 0);
+                                  false, false, std::min(ByValAlign,
+                                                         (unsigned )4));
     MemOpChains.push_back(LoadVal.getValue(1));
-    unsigned DstReg = O32IntRegs[CurWord];
+    unsigned DstReg = O32IntRegs[LocMemOffset / 4];
     RegsToPass.push_back(std::make_pair(DstReg, LoadVal));
   }
 
-  // copy remaining part of byval arg to stack.
-  if (CurWord < LastWord) {
-    unsigned SizeInBytes = (LastWord - CurWord) * 4;
-    SDValue Src = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
-                              DAG.getConstant((CurWord - FirstWord) * 4,
-                                              MVT::i32));
-    LastFI = MFI->CreateFixedObject(SizeInBytes, CurWord * 4, true);
-    SDValue Dst = DAG.getFrameIndex(LastFI, PtrType);
-    Chain = DAG.getMemcpy(Chain, dl, Dst, Src,
-                          DAG.getConstant(SizeInBytes, MVT::i32),
-                          /*Align*/4,
-                          /*isVolatile=*/false, /*AlwaysInline=*/false,
-                          MachinePointerInfo(0), MachinePointerInfo(0));
-    MemOpChains.push_back(Chain);
+  if (RemainingSize == 0)
+    return;
+
+  // If there still is a register available for argument passing, write the
+  // remaining part of the structure to it using subword loads and shifts.
+  if (LocMemOffset < 4 * 4) {
+    assert(RemainingSize <= 3 && RemainingSize >= 1 &&
+           "There must be one to three bytes remaining.");
+    unsigned LoadSize = (RemainingSize == 3 ? 2 : RemainingSize);
+    SDValue LoadPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
+                                  DAG.getConstant(Offset, MVT::i32));
+    unsigned Alignment = std::min(ByValAlign, (unsigned )4);
+    SDValue LoadVal = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, Chain,
+                                     LoadPtr, MachinePointerInfo(),
+                                     MVT::getIntegerVT(LoadSize * 8), false,
+                                     false, Alignment);
+    MemOpChains.push_back(LoadVal.getValue(1));
+
+    // If target is big endian, shift it to the most significant half-word or
+    // byte.
+    if (!isLittle)
+      LoadVal = DAG.getNode(ISD::SHL, dl, MVT::i32, LoadVal,
+                            DAG.getConstant(32 - LoadSize * 8, MVT::i32));
+
+    Offset += LoadSize;
+    RemainingSize -= LoadSize;
+
+    // Read second subword if necessary.
+    if (RemainingSize != 0)  {
+      assert(RemainingSize == 1 && "There must be one byte remaining.");
+      LoadPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg, 
+                            DAG.getConstant(Offset, MVT::i32));
+      unsigned Alignment = std::min(ByValAlign, (unsigned )2);
+      SDValue Subword = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, Chain,
+                                       LoadPtr, MachinePointerInfo(),
+                                       MVT::i8, false, false, Alignment);
+      MemOpChains.push_back(Subword.getValue(1));
+      // Insert the loaded byte to LoadVal.
+      // FIXME: Use INS if supported by target.
+      unsigned ShiftAmt = isLittle ? 16 : 8;
+      SDValue Shift = DAG.getNode(ISD::SHL, dl, MVT::i32, Subword,
+                                  DAG.getConstant(ShiftAmt, MVT::i32));
+      LoadVal = DAG.getNode(ISD::OR, dl, MVT::i32, LoadVal, Shift);
+    }
+
+    unsigned DstReg = O32IntRegs[LocMemOffset / 4];
+    RegsToPass.push_back(std::make_pair(DstReg, LoadVal));
+    return;
   }
+
+  // Create a fixed object on stack at offset LocMemOffset and copy
+  // remaining part of byval arg to it using memcpy.
+  SDValue Src = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
+                            DAG.getConstant(Offset, MVT::i32));
+  LastFI = MFI->CreateFixedObject(RemainingSize, LocMemOffset, true);
+  SDValue Dst = DAG.getFrameIndex(LastFI, PtrType);
+  ByValChain = DAG.getMemcpy(ByValChain, dl, Dst, Src,
+                             DAG.getConstant(RemainingSize, MVT::i32),
+                             std::min(ByValAlign, (unsigned)4),
+                             /*isVolatile=*/false, /*AlwaysInline=*/false,
+                             MachinePointerInfo(0), MachinePointerInfo(0));
 }
 
 /// LowerCall - functions arguments are copied from virtual regs to
 /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
 /// TODO: isTailCall.
 SDValue
-MipsTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
+MipsTargetLowering::LowerCall(SDValue InChain, SDValue Callee,
                               CallingConv::ID CallConv, bool isVarArg,
                               bool &isTailCall,
                               const SmallVectorImpl<ISD::OutputArg> &Outs,
@@ -1757,8 +1953,13 @@ MipsTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NextStackOffset = CCInfo.getNextStackOffset();
 
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NextStackOffset,
-                                                            true));
+  // Chain is the output chain of the last Load/Store or CopyToReg node.
+  // ByValChain is the output chain of the last Memcpy node created for copying
+  // byval arguments to the stack.
+  SDValue Chain, CallSeqStart, ByValChain;
+  SDValue NextStackOffsetVal = DAG.getIntPtrConstant(NextStackOffset, true);
+  Chain = CallSeqStart = DAG.getCALLSEQ_START(InChain, NextStackOffsetVal);
+  ByValChain = InChain;
 
   // If this is the first call, create a stack frame object that points to
   // a location to which .cprestore saves $gp.
@@ -1818,8 +2019,10 @@ MipsTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
                                    Arg, DAG.getConstant(1, MVT::i32));
           if (!Subtarget->isLittle())
             std::swap(Lo, Hi);
-          RegsToPass.push_back(std::make_pair(VA.getLocReg(), Lo));
-          RegsToPass.push_back(std::make_pair(VA.getLocReg()+1, Hi));
+          unsigned LocRegLo = VA.getLocReg(); 
+          unsigned LocRegHigh = getNextIntArgReg(LocRegLo);
+          RegsToPass.push_back(std::make_pair(LocRegLo, Lo));
+          RegsToPass.push_back(std::make_pair(LocRegHigh, Hi));
           continue;
         }
       }
@@ -1852,8 +2055,8 @@ MipsTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
              "No support for ByVal args by ABIs other than O32 yet.");
       assert(Flags.getByValSize() &&
              "ByVal args of size 0 should have been ignored by front-end.");
-      WriteByValArg(Chain, dl, RegsToPass, MemOpChains, LastFI, MFI, DAG, Arg,
-                    VA, Flags, getPointerTy());
+      WriteByValArg(ByValChain, Chain, dl, RegsToPass, MemOpChains, LastFI, MFI,
+                    DAG, Arg, VA, Flags, getPointerTy(), Subtarget->isLittle());
       continue;
     }
 
@@ -1875,6 +2078,12 @@ MipsTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
   if (LastFI)
     MipsFI->extendOutArgFIRange(FirstFI, LastFI);
 
+  // If a memcpy has been created to copy a byval arg to a stack, replace the
+  // chain input of CallSeqStart with ByValChain.
+  if (InChain != ByValChain)
+    DAG.UpdateNodeOperands(CallSeqStart.getNode(), ByValChain,
+                           NextStackOffsetVal);
+
   // Transform all store nodes into one single node because all store
   // nodes are independent of each other.
   if (!MemOpChains.empty())
@@ -2071,12 +2280,13 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
 
       if (RegVT == MVT::i32)
         RC = Mips::CPURegsRegisterClass;
+      else if (RegVT == MVT::i64)
+        RC = Mips::CPU64RegsRegisterClass;
       else if (RegVT == MVT::f32)
         RC = Mips::FGR32RegisterClass;
-      else if (RegVT == MVT::f64) {
-        if (!Subtarget->isSingleFloat())
-          RC = Mips::AFGR64RegisterClass;
-      } else
+      else if (RegVT == MVT::f64)
+        RC = HasMips64 ? Mips::FGR64RegisterClass : Mips::AFGR64RegisterClass;
+      else
         llvm_unreachable("RegVT not supported by FormalArguments Lowering");
 
       // Transform the arguments stored on
@@ -2105,7 +2315,7 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
           ArgValue = DAG.getNode(ISD::BITCAST, dl, MVT::f32, ArgValue);
         if (RegVT == MVT::i32 && VA.getValVT() == MVT::f64) {
           unsigned Reg2 = AddLiveIn(DAG.getMachineFunction(),
-                                    VA.getLocReg()+1, RC);
+                                    getNextIntArgReg(ArgReg), RC);
           SDValue ArgValue2 = DAG.getCopyFromReg(Chain, dl, Reg2, RegVT);
           if (!Subtarget->isLittle())
             std::swap(ArgValue, ArgValue2);
@@ -2313,7 +2523,7 @@ MipsTargetLowering::getSingleConstraintMatchWeight(
     // but allow it at the lowest weight.
   if (CallOperandVal == NULL)
     return CW_Default;
-  const Type *type = CallOperandVal->getType();
+  Type *type = CallOperandVal->getType();
   // Look at the constraint type.
   switch (*constraint) {
   default:
diff --git a/contrib/llvm/lib/Target/Mips/MipsISelLowering.h b/contrib/llvm/lib/Target/Mips/MipsISelLowering.h
index bda26a229e72..4be3fed59fc0 100644
--- a/contrib/llvm/lib/Target/Mips/MipsISelLowering.h
+++ b/contrib/llvm/lib/Target/Mips/MipsISelLowering.h
@@ -81,7 +81,12 @@ namespace llvm {
 
       WrapperPIC,
 
-      DynAlloc
+      DynAlloc,
+
+      Sync,
+
+      Ext,
+      Ins
     };
   }
 
@@ -93,6 +98,8 @@ namespace llvm {
   public:
     explicit MipsTargetLowering(MipsTargetMachine &TM);
 
+    virtual bool allowsUnalignedMemoryAccesses (EVT VT) const;
+
     /// LowerOperation - Provide custom lowering hooks for some operations.
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
 
@@ -101,13 +108,14 @@ namespace llvm {
     virtual const char *getTargetNodeName(unsigned Opcode) const;
 
     /// getSetCCResultType - get the ISD::SETCC result ValueType
-    MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    EVT getSetCCResultType(EVT VT) const;
 
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   private:
     // Subtarget Info
     const MipsSubtarget *Subtarget;
-
+    
+    bool HasMips64, IsN64;
 
     // Lower Operand helpers
     SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
@@ -128,6 +136,8 @@ namespace llvm {
     SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerMEMBARRIER(SDValue Op, SelectionDAG& DAG) const;
+    SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const;
 
     virtual SDValue
       LowerFormalArguments(SDValue Chain,
diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrFPU.td b/contrib/llvm/lib/Target/Mips/MipsInstrFPU.td
index 021c16799779..2fb9d184b2b1 100644
--- a/contrib/llvm/lib/Target/Mips/MipsInstrFPU.td
+++ b/contrib/llvm/lib/Target/Mips/MipsInstrFPU.td
@@ -27,7 +27,7 @@
 def SDT_MipsFPBrcond : SDTypeProfile<0, 2, [SDTCisInt<0>,
                                             SDTCisVT<1, OtherVT>]>;
 def SDT_MipsFPCmp : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>, SDTCisFP<1>,
-                                         SDTCisInt<2>]>;
+                                         SDTCisVT<2, i32>]>;
 def SDT_MipsCMovFP : SDTypeProfile<1, 2, [SDTCisSameAs<0, 1>,
                                           SDTCisSameAs<1, 2>]>;
 def SDT_MipsBuildPairF64 : SDTypeProfile<1, 2, [SDTCisVT<0, f64>,
@@ -35,12 +35,11 @@ def SDT_MipsBuildPairF64 : SDTypeProfile<1, 2, [SDTCisVT<0, f64>,
                                                 SDTCisSameAs<1, 2>]>;
 def SDT_MipsExtractElementF64 : SDTypeProfile<1, 2, [SDTCisVT<0, i32>,
                                                      SDTCisVT<1, f64>,
-                                                     SDTCisVT<0, i32>]>;
+                                                     SDTCisVT<2, i32>]>;
 
 def MipsFPCmp : SDNode<"MipsISD::FPCmp", SDT_MipsFPCmp, [SDNPOutGlue]>;
 def MipsCMovFP_T : SDNode<"MipsISD::CMovFP_T", SDT_MipsCMovFP, [SDNPInGlue]>;
 def MipsCMovFP_F : SDNode<"MipsISD::CMovFP_F", SDT_MipsCMovFP, [SDNPInGlue]>;
-def MipsFPRound : SDNode<"MipsISD::FPRound", SDTFPRoundOp, [SDNPOptInGlue]>;
 def MipsFPBrcond : SDNode<"MipsISD::FPBrcond", SDT_MipsFPBrcond,
                           [SDNPHasChain, SDNPOptInGlue]>;
 def MipsBuildPairF64 : SDNode<"MipsISD::BuildPairF64", SDT_MipsBuildPairF64>;
@@ -55,10 +54,10 @@ let PrintMethod = "printFCCOperand" in
 // Feature predicates.
 //===----------------------------------------------------------------------===//
 
-def In32BitMode      : Predicate<"!Subtarget.isFP64bit()">;
+def IsFP64bit        : Predicate<"Subtarget.isFP64bit()">;
+def NotFP64bit       : Predicate<"!Subtarget.isFP64bit()">;
 def IsSingleFloat    : Predicate<"Subtarget.isSingleFloat()">;
 def IsNotSingleFloat : Predicate<"!Subtarget.isSingleFloat()">;
-def IsNotMipsI       : Predicate<"!Subtarget.isMips1()">;
 
 //===----------------------------------------------------------------------===//
 // Instruction Class Templates
@@ -74,97 +73,87 @@ def IsNotMipsI       : Predicate<"!Subtarget.isMips1()">;
 // Only S32 and D32 are supported right now.
 //===----------------------------------------------------------------------===//
 
-multiclass FFR1_1<bits<6> funct, string asmstr>
-{
-  def _S32 : FFR<0x11, funct, 0x0, (outs FGR32:$fd), (ins FGR32:$fs),
-      !strconcat(asmstr, ".s\t$fd, $fs"), []>;
+// FP load.
+class FPLoad<bits<6> op, string opstr, PatFrag FOp, RegisterClass RC,
+             Operand MemOpnd>:
+  FFI<op, (outs RC:$ft), (ins MemOpnd:$base),
+      !strconcat(opstr, "\t$ft, $base"), [(set RC:$ft, (FOp addr:$base))]>;
 
-  def _D32  : FFR<0x11, funct, 0x1, (outs FGR32:$fd), (ins AFGR64:$fs),
-      !strconcat(asmstr, ".d\t$fd, $fs"), []>, Requires<[In32BitMode]>;
+// FP store.
+class FPStore<bits<6> op, string opstr, PatFrag FOp, RegisterClass RC,
+              Operand MemOpnd>:
+  FFI<op, (outs), (ins RC:$ft, MemOpnd:$base),
+      !strconcat(opstr, "\t$ft, $base"), [(store RC:$ft, addr:$base)]>;
+
+// Instructions that convert an FP value to 32-bit fixed point.
+multiclass FFR1_W_M<bits<6> funct, string opstr> {
+  def _S   : FFR1<funct, 16, opstr, "w.s", FGR32, FGR32>;
+  def _D32 : FFR1<funct, 17, opstr, "w.d", FGR32, AFGR64>,
+             Requires<[NotFP64bit]>;
+  def _D64 : FFR1<funct, 17, opstr, "w.d", FGR32, FGR64>,
+             Requires<[IsFP64bit]>;
 }
 
-multiclass FFR1_2<bits<6> funct, string asmstr, SDNode FOp>
-{
-  def _S32 : FFR<0x11, funct, 0x0, (outs FGR32:$fd), (ins FGR32:$fs),
-                 !strconcat(asmstr, ".s\t$fd, $fs"),
-                 [(set FGR32:$fd, (FOp FGR32:$fs))]>;
-
-  def _D32  : FFR<0x11, funct, 0x1, (outs AFGR64:$fd), (ins AFGR64:$fs),
-                 !strconcat(asmstr, ".d\t$fd, $fs"),
-                 [(set AFGR64:$fd, (FOp AFGR64:$fs))]>, Requires<[In32BitMode]>;
+// Instructions that convert an FP value to 64-bit fixed point.
+let Predicates = [IsFP64bit] in
+multiclass FFR1_L_M<bits<6> funct, string opstr> {
+  def _S   : FFR1<funct, 16, opstr, "l.s", FGR64, FGR32>;
+  def _D64 : FFR1<funct, 17, opstr, "l.d", FGR64, FGR64>;
 }
 
-class FFR1_3<bits<6> funct, bits<5> fmt, RegisterClass RcSrc,
-              RegisterClass RcDst, string asmstr>:
-  FFR<0x11, funct, fmt, (outs RcSrc:$fd), (ins RcDst:$fs),
-      !strconcat(asmstr, "\t$fd, $fs"), []>;
+// FP-to-FP conversion instructions.
+multiclass FFR1P_M<bits<6> funct, string opstr, SDNode OpNode> {
+  def _S   : FFR1P<funct, 16, opstr, "s", FGR32, FGR32, OpNode>;
+  def _D32 : FFR1P<funct, 17, opstr, "d", AFGR64, AFGR64, OpNode>,
+             Requires<[NotFP64bit]>;
+  def _D64 : FFR1P<funct, 17, opstr, "d", FGR64, FGR64, OpNode>,
+             Requires<[IsFP64bit]>;
+}
 
-
-multiclass FFR1_4<bits<6> funct, string asmstr, SDNode FOp, bit isComm = 0> {
+multiclass FFR2P_M<bits<6> funct, string opstr, SDNode OpNode, bit isComm = 0> {
   let isCommutable = isComm in {
-  def _S32 : FFR<0x11, funct, 0x0, (outs FGR32:$fd),
-                 (ins FGR32:$fs, FGR32:$ft),
-                 !strconcat(asmstr, ".s\t$fd, $fs, $ft"),
-                 [(set FGR32:$fd, (FOp FGR32:$fs, FGR32:$ft))]>;
-
-  def _D32 : FFR<0x11, funct, 0x1, (outs AFGR64:$fd),
-                 (ins AFGR64:$fs, AFGR64:$ft),
-                 !strconcat(asmstr, ".d\t$fd, $fs, $ft"),
-                 [(set AFGR64:$fd, (FOp AFGR64:$fs, AFGR64:$ft))]>,
-                 Requires<[In32BitMode]>;
+  def _S   : FFR2P<funct, 16, opstr, "s", FGR32, OpNode>;
+  def _D32 : FFR2P<funct, 17, opstr, "d", AFGR64, OpNode>,
+             Requires<[NotFP64bit]>;
+  def _D64 : FFR2P<funct, 17, opstr, "d", FGR64, OpNode>,
+             Requires<[IsFP64bit]>;
   }
 }
 
 //===----------------------------------------------------------------------===//
 // Floating Point Instructions
 //===----------------------------------------------------------------------===//
+defm ROUND_W : FFR1_W_M<0xc, "round">;
+defm ROUND_L : FFR1_L_M<0x8, "round">;
+defm TRUNC_W : FFR1_W_M<0xd, "trunc">;
+defm TRUNC_L : FFR1_L_M<0x9, "trunc">;
+defm CEIL_W  : FFR1_W_M<0xe, "ceil">;
+defm CEIL_L  : FFR1_L_M<0xa, "ceil">;
+defm FLOOR_W : FFR1_W_M<0xf, "floor">;
+defm FLOOR_L : FFR1_L_M<0xb, "floor">;
+defm CVT_W   : FFR1_W_M<0x24, "cvt">;
+defm CVT_L   : FFR1_L_M<0x25, "cvt">;
 
-let ft = 0 in {
-  defm FLOOR_W : FFR1_1<0b001111, "floor.w">;
-  defm CEIL_W  : FFR1_1<0b001110, "ceil.w">;
-  defm ROUND_W : FFR1_1<0b001100, "round.w">;
-  defm TRUNC_W : FFR1_1<0b001101, "trunc.w">;
-  defm CVTW    : FFR1_1<0b100100, "cvt.w">;
+def CVT_S_W : FFR1<0x20, 20, "cvt", "s.w", FGR32, FGR32>;
 
-  defm FABS    : FFR1_2<0b000101, "abs",  fabs>;
-  defm FNEG    : FFR1_2<0b000111, "neg",  fneg>;
-  defm FSQRT   : FFR1_2<0b000100, "sqrt", fsqrt>;
-
-  /// Convert to Single Precison
-  def CVTS_W32 : FFR1_3<0b100000, 0x2, FGR32,  FGR32,  "cvt.s.w">;
-
-  let Predicates = [IsNotSingleFloat] in {
-    /// Ceil to long signed integer
-    def CEIL_LS   : FFR1_3<0b001010, 0x0, FGR32, FGR32, "ceil.l">;
-    def CEIL_LD   : FFR1_3<0b001010, 0x1, AFGR64, AFGR64, "ceil.l">;
-
-    /// Round to long signed integer
-    def ROUND_LS  : FFR1_3<0b001000, 0x0, FGR32, FGR32, "round.l">;
-    def ROUND_LD  : FFR1_3<0b001000, 0x1, AFGR64, AFGR64, "round.l">;
-
-    /// Floor to long signed integer
-    def FLOOR_LS  : FFR1_3<0b001011, 0x0, FGR32, FGR32, "floor.l">;
-    def FLOOR_LD  : FFR1_3<0b001011, 0x1, AFGR64, AFGR64, "floor.l">;
-
-    /// Trunc to long signed integer
-    def TRUNC_LS  : FFR1_3<0b001001, 0x0, FGR32, FGR32, "trunc.l">;
-    def TRUNC_LD  : FFR1_3<0b001001, 0x1, AFGR64, AFGR64, "trunc.l">;
-
-    /// Convert to long signed integer
-    def CVTL_S    : FFR1_3<0b100101, 0x0, FGR32, FGR32, "cvt.l">;
-    def CVTL_D    : FFR1_3<0b100101, 0x1, AFGR64, AFGR64, "cvt.l">;
-
-    /// Convert to Double Precison
-    def CVTD_S32 : FFR1_3<0b100001, 0x0, AFGR64, FGR32, "cvt.d.s">;
-    def CVTD_W32 : FFR1_3<0b100001, 0x2, AFGR64, FGR32, "cvt.d.w">;
-    def CVTD_L32 : FFR1_3<0b100001, 0x3, AFGR64, AFGR64, "cvt.d.l">;
-
-    /// Convert to Single Precison
-    def CVTS_D32 : FFR1_3<0b100000, 0x1, FGR32, AFGR64, "cvt.s.d">;
-    def CVTS_L32 : FFR1_3<0b100000, 0x3, FGR32, AFGR64, "cvt.s.l">;
-  }
+let Predicates = [NotFP64bit] in {
+  def CVT_S_D32 : FFR1<0x20, 17, "cvt", "s.d", FGR32, AFGR64>;
+  def CVT_D32_W : FFR1<0x21, 20, "cvt", "d.w", AFGR64, FGR32>;
+  def CVT_D32_S : FFR1<0x21, 16, "cvt", "d.s", AFGR64, FGR32>;
 }
 
+let Predicates = [IsFP64bit] in {
+ def CVT_S_D64 : FFR1<0x20, 17, "cvt", "s.d", FGR32, FGR64>;
+ def CVT_S_L   : FFR1<0x20, 21, "cvt", "s.l", FGR32, FGR64>;
+ def CVT_D64_W : FFR1<0x21, 20, "cvt", "d.w", FGR64, FGR32>;
+ def CVT_D64_S : FFR1<0x21, 16, "cvt", "d.s", FGR64, FGR32>;
+ def CVT_D64_L : FFR1<0x21, 21, "cvt", "d.l", FGR64, FGR64>;
+}
+
+defm FABS    : FFR1P_M<0x5, "abs",  fabs>;
+defm FNEG    : FFR1P_M<0x7, "neg",  fneg>;
+defm FSQRT   : FFR1P_M<0x4, "sqrt", fsqrt>;
+
 // The odd-numbered registers are only referenced when doing loads,
 // stores, and moves between floating-point and integer registers.
 // When defining instructions, we reference all 32-bit registers,
@@ -178,37 +167,46 @@ let fd = 0 in {
                   "ctc1\t$fs, $rt", []>;
 
   def MFC1  : FFR<0x11, 0x00, 0x00, (outs CPURegs:$rt), (ins FGR32:$fs),
-                  "mfc1\t$rt, $fs", []>;
+                  "mfc1\t$rt, $fs",
+                  [(set CPURegs:$rt, (bitconvert FGR32:$fs))]>;
 
   def MTC1  : FFR<0x11, 0x00, 0x04, (outs FGR32:$fs), (ins CPURegs:$rt),
-                  "mtc1\t$rt, $fs", []>;
+                  "mtc1\t$rt, $fs",
+                  [(set FGR32:$fs, (bitconvert CPURegs:$rt))]>;
 }
 
-def FMOV_S32 : FFR<0x11, 0b000110, 0x0, (outs FGR32:$fd), (ins FGR32:$fs),
-                   "mov.s\t$fd, $fs", []>;
-def FMOV_D32 : FFR<0x11, 0b000110, 0x1, (outs AFGR64:$fd), (ins AFGR64:$fs),
-                   "mov.d\t$fd, $fs", []>;
+def FMOV_S   : FFR1<0x6, 16, "mov", "s", FGR32, FGR32>;
+def FMOV_D32 : FFR1<0x6, 17, "mov", "d", AFGR64, AFGR64>,
+               Requires<[NotFP64bit]>;
+def FMOV_D64 : FFR1<0x6, 17, "mov", "d", FGR64, FGR64>,
+               Requires<[IsFP64bit]>;
 
 /// Floating Point Memory Instructions
-let Predicates = [IsNotSingleFloat, IsNotMipsI] in {
-  def LDC1 : FFI<0b110101, (outs AFGR64:$ft), (ins mem:$addr),
-                 "ldc1\t$ft, $addr", [(set AFGR64:$ft, (load addr:$addr))]>;
-
-  def SDC1 : FFI<0b111101, (outs), (ins AFGR64:$ft, mem:$addr),
-                 "sdc1\t$ft, $addr", [(store AFGR64:$ft, addr:$addr)]>;
+let Predicates = [IsN64] in {
+  def LWC1_P8   : FPLoad<0x31, "lwc1", load, FGR32, mem64>;
+  def SWC1_P8   : FPStore<0x39, "swc1", store, FGR32, mem64>;
+  def LDC164_P8 : FPLoad<0x35, "ldc1", load, FGR64, mem64>;
+  def SDC164_P8 : FPStore<0x3d, "sdc1", store, FGR64, mem64>;
 }
 
-// LWC1 and SWC1 can always be emitted with odd registers.
-def LWC1  : FFI<0b110001, (outs FGR32:$ft), (ins mem:$addr), "lwc1\t$ft, $addr",
-               [(set FGR32:$ft, (load addr:$addr))]>;
-def SWC1  : FFI<0b111001, (outs), (ins FGR32:$ft, mem:$addr),
-               "swc1\t$ft, $addr", [(store FGR32:$ft, addr:$addr)]>;
+let Predicates = [NotN64] in {
+  def LWC1   : FPLoad<0x31, "lwc1", load, FGR32, mem>;
+  def SWC1   : FPStore<0x39, "swc1", store, FGR32, mem>;
+  let Predicates = [HasMips64] in {
+    def LDC164 : FPLoad<0x35, "ldc1", load, FGR64, mem>;
+    def SDC164 : FPStore<0x3d, "sdc1", store, FGR64, mem>;
+  }
+  let Predicates = [NotMips64] in {
+    def LDC1   : FPLoad<0x35, "ldc1", load, AFGR64, mem>;
+    def SDC1   : FPStore<0x3d, "sdc1", store, AFGR64, mem>;
+  }
+}
 
 /// Floating-point Aritmetic
-defm FADD : FFR1_4<0x10, "add", fadd, 1>;
-defm FDIV : FFR1_4<0x03, "div", fdiv>;
-defm FMUL : FFR1_4<0x02, "mul", fmul, 1>;
-defm FSUB : FFR1_4<0x01, "sub", fsub>;
+defm FADD : FFR2P_M<0x10, "add", fadd, 1>;
+defm FDIV : FFR2P_M<0x03, "div", fdiv>;
+defm FMUL : FFR2P_M<0x02, "mul", fmul, 1>;
+defm FSUB : FFR2P_M<0x01, "sub", fsub>;
 
 //===----------------------------------------------------------------------===//
 // Floating Point Branch Codes
@@ -217,8 +215,6 @@ defm FSUB : FFR1_4<0x01, "sub", fsub>;
 // They must be kept in synch.
 def MIPS_BRANCH_F  : PatLeaf<(i32 0)>;
 def MIPS_BRANCH_T  : PatLeaf<(i32 1)>;
-def MIPS_BRANCH_FL : PatLeaf<(i32 2)>;
-def MIPS_BRANCH_TL : PatLeaf<(i32 3)>;
 
 /// Floating Point Branch of False/True (Likely)
 let isBranch=1, isTerminator=1, hasDelaySlot=1, base=0x8, Uses=[FCR31] in
@@ -228,8 +224,6 @@ let isBranch=1, isTerminator=1, hasDelaySlot=1, base=0x8, Uses=[FCR31] in
 
 def BC1F  : FBRANCH<MIPS_BRANCH_F,  "bc1f">;
 def BC1T  : FBRANCH<MIPS_BRANCH_T,  "bc1t">;
-def BC1FL : FBRANCH<MIPS_BRANCH_FL, "bc1fl">;
-def BC1TL : FBRANCH<MIPS_BRANCH_TL, "bc1tl">;
 
 //===----------------------------------------------------------------------===//
 // Floating Point Flag Conditions
@@ -254,7 +248,7 @@ def MIPS_FCOND_LE   : PatLeaf<(i32 14)>;
 def MIPS_FCOND_NGT  : PatLeaf<(i32 15)>;
 
 /// Floating Point Compare
-let hasDelaySlot = 1, Defs=[FCR31] in {
+let Defs=[FCR31] in {
   def FCMP_S32 : FCC<0x0, (outs), (ins FGR32:$fs, FGR32:$ft, condcode:$cc),
                      "c.$cc.s\t$fs, $ft",
                      [(MipsFPCmp FGR32:$fs, FGR32:$ft, imm:$cc)]>;
@@ -262,7 +256,7 @@ let hasDelaySlot = 1, Defs=[FCR31] in {
   def FCMP_D32 : FCC<0x1, (outs), (ins AFGR64:$fs, AFGR64:$ft, condcode:$cc),
                      "c.$cc.d\t$fs, $ft",
                      [(MipsFPCmp AFGR64:$fs, AFGR64:$ft, imm:$cc)]>,
-                     Requires<[In32BitMode]>;
+                     Requires<[NotFP64bit]>;
 }
 
 
@@ -280,7 +274,7 @@ class CondMovIntFP<RegisterClass RC, bits<5> fmt, bits<6> func,
 def MOVZ_S : CondMovIntFP<FGR32, 16, 18, "movz.s">;
 def MOVN_S : CondMovIntFP<FGR32, 16, 19, "movn.s">;
 
-let Predicates = [In32BitMode] in {
+let Predicates = [NotFP64bit] in {
   def MOVZ_D : CondMovIntFP<AFGR64, 17, 18, "movz.d">;
   def MOVN_D : CondMovIntFP<AFGR64, 17, 19, "movn.d">;
 }
@@ -288,7 +282,7 @@ let Predicates = [In32BitMode] in {
 defm : MovzPats<FGR32, MOVZ_S>;
 defm : MovnPats<FGR32, MOVN_S>;
 
-let Predicates = [In32BitMode] in {
+let Predicates = [NotFP64bit] in {
   defm : MovzPats<AFGR64, MOVZ_D>;
   defm : MovnPats<AFGR64, MOVN_D>;
 }
@@ -313,7 +307,7 @@ def MOVF : CondMovFPInt<MipsCMovFP_F, 0, "movf">;
 def MOVT_S : CondMovFPFP<FGR32, MipsCMovFP_T, 16, 1, "movt.s">;
 def MOVF_S : CondMovFPFP<FGR32, MipsCMovFP_F, 16, 0, "movf.s">;
 
-let Predicates = [In32BitMode] in {
+let Predicates = [NotFP64bit] in {
   def MOVT_D : CondMovFPFP<AFGR64, MipsCMovFP_T, 17, 1, "movt.d">;
   def MOVF_D : CondMovFPFP<AFGR64, MipsCMovFP_F, 17, 0, "movf.d">;
 }
@@ -353,22 +347,16 @@ def fpimm0neg : PatLeaf<(fpimm), [{
 }]>;
 
 def : Pat<(f32 fpimm0), (MTC1 ZERO)>;
-def : Pat<(f32 fpimm0neg), (FNEG_S32 (MTC1 ZERO))>;
+def : Pat<(f32 fpimm0neg), (FNEG_S (MTC1 ZERO))>;
 
-def : Pat<(f32 (sint_to_fp CPURegs:$src)), (CVTS_W32 (MTC1 CPURegs:$src))>;
-def : Pat<(f64 (sint_to_fp CPURegs:$src)), (CVTD_W32 (MTC1 CPURegs:$src))>;
+def : Pat<(f32 (sint_to_fp CPURegs:$src)), (CVT_S_W (MTC1 CPURegs:$src))>;
+def : Pat<(f64 (sint_to_fp CPURegs:$src)), (CVT_D32_W (MTC1 CPURegs:$src))>;
 
-def : Pat<(i32 (fp_to_sint FGR32:$src)), (MFC1 (TRUNC_W_S32 FGR32:$src))>;
+def : Pat<(i32 (fp_to_sint FGR32:$src)), (MFC1 (TRUNC_W_S FGR32:$src))>;
 def : Pat<(i32 (fp_to_sint AFGR64:$src)), (MFC1 (TRUNC_W_D32 AFGR64:$src))>;
 
-def : Pat<(i32 (bitconvert FGR32:$src)),  (MFC1 FGR32:$src)>;
-def : Pat<(f32 (bitconvert CPURegs:$src)), (MTC1 CPURegs:$src)>;
-
-let Predicates = [In32BitMode] in {
-  def : Pat<(f32 (fround AFGR64:$src)), (CVTS_D32 AFGR64:$src)>;
-  def : Pat<(f64 (fextend FGR32:$src)), (CVTD_S32 FGR32:$src)>;
+let Predicates = [NotFP64bit] in {
+  def : Pat<(f32 (fround AFGR64:$src)), (CVT_S_D32 AFGR64:$src)>;
+  def : Pat<(f64 (fextend FGR32:$src)), (CVT_D32_S FGR32:$src)>;
 }
 
-// MipsFPRound is only emitted for MipsI targets.
-def : Pat<(f32 (MipsFPRound AFGR64:$src)), (CVTW_D32 AFGR64:$src)>;
-
diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrFormats.td b/contrib/llvm/lib/Target/Mips/MipsInstrFormats.td
index 9f55fb38b959..d246a26eb233 100644
--- a/contrib/llvm/lib/Target/Mips/MipsInstrFormats.td
+++ b/contrib/llvm/lib/Target/Mips/MipsInstrFormats.td
@@ -44,7 +44,9 @@ class MipsInst<dag outs, dag ins, string asmstr, list<dag> pattern,
 
 // Mips Pseudo Instructions Format
 class MipsPseudo<dag outs, dag ins, string asmstr, list<dag> pattern>:
-      MipsInst<outs, ins, asmstr, pattern, IIPseudo>;
+      MipsInst<outs, ins, asmstr, pattern, IIPseudo> {
+  let isPseudo = 1;
+}
 
 //===----------------------------------------------------------------------===//
 // Format R instruction class in Mips : <|opcode|rs|rt|rd|shamt|funct|>
@@ -88,6 +90,21 @@ class FI<bits<6> op, dag outs, dag ins, string asmstr, list<dag> pattern,
   let Inst{15-0}  = imm16;
 }
 
+class CBranchBase<bits<6> op, dag outs, dag ins, string asmstr,
+                  list<dag> pattern, InstrItinClass itin>:
+  MipsInst<outs, ins, asmstr, pattern, itin>
+{
+  bits<5>  rs;
+  bits<5>  rt;
+  bits<16> imm16;
+
+  let opcode = op;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-0}  = imm16;
+}
+
 //===----------------------------------------------------------------------===//
 // Format J instruction class in Mips : <|opcode|address|>
 //===----------------------------------------------------------------------===//
@@ -224,4 +241,27 @@ class FFCMOV<bits<5> _fmt, bits<1> _tf, dag outs, dag ins, string asmstr,
   let Inst{15-11} = fs;
   let Inst{10-6}  = fd;
   let Inst{5-0}   = 17;
-}
\ No newline at end of file
+}
+
+// FP unary instructions without patterns.
+class FFR1<bits<6> funct, bits<5> fmt, string opstr, string fmtstr,
+           RegisterClass DstRC, RegisterClass SrcRC> :
+  FFR<0x11, funct, fmt, (outs DstRC:$fd), (ins SrcRC:$fs),
+      !strconcat(opstr, ".", fmtstr, "\t$fd, $fs"), []> {
+  let ft = 0;
+}
+
+// FP unary instructions with patterns.
+class FFR1P<bits<6> funct, bits<5> fmt, string opstr, string fmtstr,
+            RegisterClass DstRC, RegisterClass SrcRC, SDNode OpNode> :
+  FFR<0x11, funct, fmt, (outs DstRC:$fd), (ins SrcRC:$fs),
+      !strconcat(opstr, ".", fmtstr, "\t$fd, $fs"),
+      [(set DstRC:$fd, (OpNode SrcRC:$fs))]> {
+  let ft = 0;
+}
+
+class FFR2P<bits<6> funct, bits<5> fmt, string opstr,
+            string fmtstr, RegisterClass RC, SDNode OpNode> :
+  FFR<0x11, funct, fmt, (outs RC:$fd), (ins RC:$fs, RC:$ft),
+      !strconcat(opstr, ".", fmtstr, "\t$fd, $fs, $ft"),
+      [(set RC:$fd, (OpNode RC:$fs, RC:$ft))]>;
diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp
index 0a7a7f2dfe4e..559943a8dbae 100644
--- a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp
@@ -17,8 +17,8 @@
 #include "InstPrinter/MipsInstPrinter.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 
 #define GET_INSTRINFO_CTOR
@@ -28,7 +28,8 @@ using namespace llvm;
 
 MipsInstrInfo::MipsInstrInfo(MipsTargetMachine &tm)
   : MipsGenInstrInfo(Mips::ADJCALLSTACKDOWN, Mips::ADJCALLSTACKUP),
-    TM(tm), RI(*TM.getSubtargetImpl(), *this) {}
+    TM(tm), IsN64(TM.getSubtarget<MipsSubtarget>().isABI_N64()),
+    RI(*TM.getSubtargetImpl(), *this) {}
 
 
 const MipsRegisterInfo &MipsInstrInfo::getRegisterInfo() const { 
@@ -47,8 +48,12 @@ static bool isZeroImm(const MachineOperand &op) {
 unsigned MipsInstrInfo::
 isLoadFromStackSlot(const MachineInstr *MI, int &FrameIndex) const
 {
-  if ((MI->getOpcode() == Mips::LW) || (MI->getOpcode() == Mips::LWC1) ||
-      (MI->getOpcode() == Mips::LDC1)) {
+  unsigned Opc = MI->getOpcode();
+
+  if ((Opc == Mips::LW)    || (Opc == Mips::LW_P8)  || (Opc == Mips::LD) ||
+      (Opc == Mips::LD_P8) || (Opc == Mips::LWC1)   || (Opc == Mips::LWC1_P8) ||
+      (Opc == Mips::LDC1)  || (Opc == Mips::LDC164) ||
+      (Opc == Mips::LDC164_P8)) {
     if ((MI->getOperand(1).isFI()) && // is a stack slot
         (MI->getOperand(2).isImm()) &&  // the imm is zero
         (isZeroImm(MI->getOperand(2)))) {
@@ -68,8 +73,12 @@ isLoadFromStackSlot(const MachineInstr *MI, int &FrameIndex) const
 unsigned MipsInstrInfo::
 isStoreToStackSlot(const MachineInstr *MI, int &FrameIndex) const
 {
-  if ((MI->getOpcode() == Mips::SW) || (MI->getOpcode() == Mips::SWC1) ||
-      (MI->getOpcode() == Mips::SDC1)) {
+  unsigned Opc = MI->getOpcode();
+
+  if ((Opc == Mips::SW)    || (Opc == Mips::SW_P8)  || (Opc == Mips::SD) ||
+      (Opc == Mips::SD_P8) || (Opc == Mips::SWC1)   || (Opc == Mips::SWC1_P8) ||
+      (Opc == Mips::SDC1)  || (Opc == Mips::SDC164) ||
+      (Opc == Mips::SDC164_P8)) {
     if ((MI->getOperand(1).isFI()) && // is a stack slot
         (MI->getOperand(2).isImm()) &&  // the imm is zero
         (isZeroImm(MI->getOperand(2)))) {
@@ -94,70 +103,63 @@ copyPhysReg(MachineBasicBlock &MBB,
             MachineBasicBlock::iterator I, DebugLoc DL,
             unsigned DestReg, unsigned SrcReg,
             bool KillSrc) const {
-  bool DestCPU = Mips::CPURegsRegClass.contains(DestReg);
-  bool SrcCPU  = Mips::CPURegsRegClass.contains(SrcReg);
+  unsigned Opc = 0, ZeroReg = 0;
 
-  // CPU-CPU is the most common.
-  if (DestCPU && SrcCPU) {
-    BuildMI(MBB, I, DL, get(Mips::ADDu), DestReg).addReg(Mips::ZERO)
-      .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-
-  // Copy to CPU from other registers.
-  if (DestCPU) {
-    if (Mips::CCRRegClass.contains(SrcReg))
-      BuildMI(MBB, I, DL, get(Mips::CFC1), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+  if (Mips::CPURegsRegClass.contains(DestReg)) { // Copy to CPU Reg.
+    if (Mips::CPURegsRegClass.contains(SrcReg))
+      Opc = Mips::ADDu, ZeroReg = Mips::ZERO;
+    else if (Mips::CCRRegClass.contains(SrcReg))
+      Opc = Mips::CFC1;
     else if (Mips::FGR32RegClass.contains(SrcReg))
-      BuildMI(MBB, I, DL, get(Mips::MFC1), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+      Opc = Mips::MFC1;
     else if (SrcReg == Mips::HI)
-      BuildMI(MBB, I, DL, get(Mips::MFHI), DestReg);
+      Opc = Mips::MFHI, SrcReg = 0;
     else if (SrcReg == Mips::LO)
-      BuildMI(MBB, I, DL, get(Mips::MFLO), DestReg);
-    else
-      llvm_unreachable("Copy to CPU from invalid register");
-    return;
+      Opc = Mips::MFLO, SrcReg = 0;
   }
-
-  // Copy to other registers from CPU.
-  if (SrcCPU) {
+  else if (Mips::CPURegsRegClass.contains(SrcReg)) { // Copy from CPU Reg.
     if (Mips::CCRRegClass.contains(DestReg))
-      BuildMI(MBB, I, DL, get(Mips::CTC1), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+      Opc = Mips::CTC1;
     else if (Mips::FGR32RegClass.contains(DestReg))
-      BuildMI(MBB, I, DL, get(Mips::MTC1), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+      Opc = Mips::MTC1;
     else if (DestReg == Mips::HI)
-      BuildMI(MBB, I, DL, get(Mips::MTHI))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+      Opc = Mips::MTHI, DestReg = 0;
     else if (DestReg == Mips::LO)
-      BuildMI(MBB, I, DL, get(Mips::MTLO))
-        .addReg(SrcReg, getKillRegState(KillSrc));
-    else
-      llvm_unreachable("Copy from CPU to invalid register");
-    return;
+      Opc = Mips::MTLO, DestReg = 0;
+  }
+  else if (Mips::FGR32RegClass.contains(DestReg, SrcReg))
+    Opc = Mips::FMOV_S;
+  else if (Mips::AFGR64RegClass.contains(DestReg, SrcReg))
+    Opc = Mips::FMOV_D32;
+  else if (Mips::CCRRegClass.contains(DestReg, SrcReg))
+    Opc = Mips::MOVCCRToCCR;
+  else if (Mips::CPU64RegsRegClass.contains(DestReg)) { // Copy to CPU64 Reg.
+    if (Mips::CPU64RegsRegClass.contains(SrcReg))
+      Opc = Mips::DADDu, ZeroReg = Mips::ZERO_64;
+    else if (SrcReg == Mips::HI64)
+      Opc = Mips::MFHI64, SrcReg = 0;
+    else if (SrcReg == Mips::LO64)
+      Opc = Mips::MFLO64, SrcReg = 0;
+  }
+  else if (Mips::CPU64RegsRegClass.contains(SrcReg)) { // Copy from CPU64 Reg.
+    if (DestReg == Mips::HI64)
+      Opc = Mips::MTHI64, DestReg = 0;
+    else if (DestReg == Mips::LO64)
+      Opc = Mips::MTLO64, DestReg = 0;
   }
 
-  if (Mips::FGR32RegClass.contains(DestReg, SrcReg)) {
-    BuildMI(MBB, I, DL, get(Mips::FMOV_S32), DestReg)
-      .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
+  assert(Opc && "Cannot copy registers");
 
-  if (Mips::AFGR64RegClass.contains(DestReg, SrcReg)) {
-    BuildMI(MBB, I, DL, get(Mips::FMOV_D32), DestReg)
-      .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
+  MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc));
+  
+  if (DestReg)
+    MIB.addReg(DestReg, RegState::Define);
 
-  if (Mips::CCRRegClass.contains(DestReg, SrcReg)) {
-    BuildMI(MBB, I, DL, get(Mips::MOVCCRToCCR), DestReg)
-      .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-  llvm_unreachable("Cannot copy registers");
+  if (ZeroReg)
+    MIB.addReg(ZeroReg);
+
+  if (SrcReg)
+    MIB.addReg(SrcReg, getKillRegState(KillSrc));
 }
 
 void MipsInstrInfo::
@@ -167,31 +169,22 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     const TargetRegisterInfo *TRI) const {
   DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
+  unsigned Opc = 0;
 
   if (RC == Mips::CPURegsRegisterClass)
-    BuildMI(MBB, I, DL, get(Mips::SW)).addReg(SrcReg, getKillRegState(isKill))
-                                      .addFrameIndex(FI).addImm(0);
+    Opc = IsN64 ? Mips::SW_P8 : Mips::SW;
+  else if (RC == Mips::CPU64RegsRegisterClass)
+    Opc = IsN64 ? Mips::SD_P8 : Mips::SD;
   else if (RC == Mips::FGR32RegisterClass)
-    BuildMI(MBB, I, DL, get(Mips::SWC1)).addReg(SrcReg, getKillRegState(isKill))
-                                        .addFrameIndex(FI).addImm(0);
-  else if (RC == Mips::AFGR64RegisterClass) {
-    if (!TM.getSubtarget<MipsSubtarget>().isMips1()) {
-      BuildMI(MBB, I, DL, get(Mips::SDC1))
-        .addReg(SrcReg, getKillRegState(isKill))
-        .addFrameIndex(FI).addImm(0);
-    } else {
-      const TargetRegisterInfo *TRI =
-        MBB.getParent()->getTarget().getRegisterInfo();
-      const unsigned *SubSet = TRI->getSubRegisters(SrcReg);
-      BuildMI(MBB, I, DL, get(Mips::SWC1))
-        .addReg(SubSet[0], getKillRegState(isKill))
-        .addFrameIndex(FI).addImm(0);
-      BuildMI(MBB, I, DL, get(Mips::SWC1))
-        .addReg(SubSet[1], getKillRegState(isKill))
-        .addFrameIndex(FI).addImm(4);
-    }
-  } else
-    llvm_unreachable("Register class not handled!");
+    Opc = IsN64 ? Mips::SWC1_P8 : Mips::SWC1;
+  else if (RC == Mips::AFGR64RegisterClass)
+    Opc = Mips::SDC1;
+  else if (RC == Mips::FGR64RegisterClass)
+    Opc = IsN64 ? Mips::SDC164_P8 : Mips::SDC164;
+
+  assert(Opc && "Register class not handled!");
+  BuildMI(MBB, I, DL, get(Opc)).addReg(SrcReg, getKillRegState(isKill))
+    .addFrameIndex(FI).addImm(0);
 }
 
 void MipsInstrInfo::
@@ -202,25 +195,21 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 {
   DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
+  unsigned Opc = 0;
 
   if (RC == Mips::CPURegsRegisterClass)
-    BuildMI(MBB, I, DL, get(Mips::LW), DestReg).addFrameIndex(FI).addImm(0);
+    Opc = IsN64 ? Mips::LW_P8 : Mips::LW;
+  else if (RC == Mips::CPU64RegsRegisterClass)
+    Opc = IsN64 ? Mips::LD_P8 : Mips::LD;
   else if (RC == Mips::FGR32RegisterClass)
-    BuildMI(MBB, I, DL, get(Mips::LWC1), DestReg).addFrameIndex(FI).addImm(0);
-  else if (RC == Mips::AFGR64RegisterClass) {
-    if (!TM.getSubtarget<MipsSubtarget>().isMips1()) {
-      BuildMI(MBB, I, DL, get(Mips::LDC1), DestReg).addFrameIndex(FI).addImm(0);
-    } else {
-      const TargetRegisterInfo *TRI =
-        MBB.getParent()->getTarget().getRegisterInfo();
-      const unsigned *SubSet = TRI->getSubRegisters(DestReg);
-      BuildMI(MBB, I, DL, get(Mips::LWC1), SubSet[0])
-        .addFrameIndex(FI).addImm(0);
-      BuildMI(MBB, I, DL, get(Mips::LWC1), SubSet[1])
-        .addFrameIndex(FI).addImm(4);
-    }
-  } else
-    llvm_unreachable("Register class not handled!");
+    Opc = IsN64 ? Mips::LWC1_P8 : Mips::LWC1;
+  else if (RC == Mips::AFGR64RegisterClass)
+    Opc = Mips::LDC1;
+  else if (RC == Mips::FGR64RegisterClass)
+    Opc = IsN64 ? Mips::LDC164_P8 : Mips::LDC164;
+
+  assert(Opc && "Register class not handled!");
+  BuildMI(MBB, I, DL, get(Opc), DestReg).addFrameIndex(FI).addImm(0);
 }
 
 MachineInstr*
@@ -237,9 +226,12 @@ MipsInstrInfo::emitFrameIndexDebugValue(MachineFunction &MF, int FrameIx,
 //===----------------------------------------------------------------------===//
 
 static unsigned GetAnalyzableBrOpc(unsigned Opc) {
-  return (Opc == Mips::BEQ  || Opc == Mips::BNE  || Opc == Mips::BGTZ ||
-          Opc == Mips::BGEZ || Opc == Mips::BLTZ || Opc == Mips::BLEZ ||
-          Opc == Mips::BC1T || Opc == Mips::BC1F || Opc == Mips::J) ? Opc : 0;
+  return (Opc == Mips::BEQ    || Opc == Mips::BNE    || Opc == Mips::BGTZ   ||
+          Opc == Mips::BGEZ   || Opc == Mips::BLTZ   || Opc == Mips::BLEZ   ||
+          Opc == Mips::BEQ64  || Opc == Mips::BNE64  || Opc == Mips::BGTZ64 ||
+          Opc == Mips::BGEZ64 || Opc == Mips::BLTZ64 || Opc == Mips::BLEZ64 ||
+          Opc == Mips::BC1T   || Opc == Mips::BC1F   || Opc == Mips::J) ?
+         Opc : 0;
 }
 
 /// GetOppositeBranchOpc - Return the inverse of the specified
@@ -248,14 +240,20 @@ unsigned Mips::GetOppositeBranchOpc(unsigned Opc)
 {
   switch (Opc) {
   default: llvm_unreachable("Illegal opcode!");
-  case Mips::BEQ  : return Mips::BNE;
-  case Mips::BNE  : return Mips::BEQ;
-  case Mips::BGTZ : return Mips::BLEZ;
-  case Mips::BGEZ : return Mips::BLTZ;
-  case Mips::BLTZ : return Mips::BGEZ;
-  case Mips::BLEZ : return Mips::BGTZ;
-  case Mips::BC1T : return Mips::BC1F;
-  case Mips::BC1F : return Mips::BC1T;
+  case Mips::BEQ    : return Mips::BNE;
+  case Mips::BNE    : return Mips::BEQ;
+  case Mips::BGTZ   : return Mips::BLEZ;
+  case Mips::BGEZ   : return Mips::BLTZ;
+  case Mips::BLTZ   : return Mips::BGEZ;
+  case Mips::BLEZ   : return Mips::BGTZ;
+  case Mips::BEQ64  : return Mips::BNE64;
+  case Mips::BNE64  : return Mips::BEQ64;
+  case Mips::BGTZ64 : return Mips::BLEZ64;
+  case Mips::BGEZ64 : return Mips::BLTZ64;
+  case Mips::BLTZ64 : return Mips::BGEZ64;
+  case Mips::BLEZ64 : return Mips::BGTZ64;
+  case Mips::BC1T   : return Mips::BC1F;
+  case Mips::BC1F   : return Mips::BC1T;
   }
 }
 
diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.h b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.h
index 4421c4862fa0..271d2487ecfa 100644
--- a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.h
+++ b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.h
@@ -72,12 +72,47 @@ namespace MipsII {
     /// MO_TPREL_HI/LO - Represents the hi and low part of the offset from
     // the thread pointer (Local Exec TLS).
     MO_TPREL_HI,
-    MO_TPREL_LO
+    MO_TPREL_LO,
+
+    // N32/64 Flags.
+    MO_GPOFF_HI,
+    MO_GPOFF_LO,
+    MO_GOT_DISP,
+    MO_GOT_PAGE,
+    MO_GOT_OFST
+  };
+
+  enum {
+    //===------------------------------------------------------------------===//
+    // Instruction encodings.  These are the standard/most common forms for
+    // Mips instructions.
+    //
+
+    // Pseudo - This represents an instruction that is a pseudo instruction
+    // or one that has not been implemented yet.  It is illegal to code generate
+    // it, but tolerated for intermediate implementation stages.
+    Pseudo   = 0,
+
+    /// FrmR - This form is for instructions of the format R.
+    FrmR  = 1,
+    /// FrmI - This form is for instructions of the format I.
+    FrmI  = 2,
+    /// FrmJ - This form is for instructions of the format J.
+    FrmJ  = 3,
+    /// FrmFR - This form is for instructions of the format FR.
+    FrmFR = 4,
+    /// FrmFI - This form is for instructions of the format FI.
+    FrmFI = 5,
+    /// FrmOther - This form is for instructions that have no specific format.
+    FrmOther = 6,
+
+    FormMask = 15
   };
 }
 
 class MipsInstrInfo : public MipsGenInstrInfo {
   MipsTargetMachine &TM;
+  bool IsN64;
   const MipsRegisterInfo RI;
 public:
   explicit MipsInstrInfo(MipsTargetMachine &TM);
diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td
index d1a058712459..06b7de7e8e32 100644
--- a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td
+++ b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td
@@ -34,13 +34,20 @@ def SDT_MipsMAddMSub     : SDTypeProfile<0, 4,
                                           SDTCisSameAs<1, 2>,
                                           SDTCisSameAs<2, 3>]>;
 def SDT_MipsDivRem       : SDTypeProfile<0, 2,
-                                         [SDTCisVT<0, i32>,
+                                         [SDTCisInt<0>,
                                           SDTCisSameAs<0, 1>]>;
 
 def SDT_MipsThreadPointer : SDTypeProfile<1, 0, [SDTCisPtrTy<0>]>;
 
 def SDT_MipsDynAlloc    : SDTypeProfile<1, 1, [SDTCisVT<0, i32>,
                                                SDTCisVT<1, iPTR>]>;
+def SDT_Sync             : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
+
+def SDT_Ext : SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<0, 1>,
+                                   SDTCisVT<2, i32>, SDTCisSameAs<2, 3>]>;
+def SDT_Ins : SDTypeProfile<1, 4, [SDTCisInt<0>, SDTCisSameAs<0, 1>,
+                                   SDTCisVT<2, i32>, SDTCisSameAs<2, 3>,
+                                   SDTCisSameAs<0, 4>]>;
 
 // Call
 def MipsJmpLink : SDNode<"MipsISD::JmpLink",SDT_MipsJmpLink,
@@ -106,6 +113,11 @@ def MipsWrapperPIC    : SDNode<"MipsISD::WrapperPIC",  SDTIntUnaryOp>;
 def MipsDynAlloc  : SDNode<"MipsISD::DynAlloc", SDT_MipsDynAlloc,
                            [SDNPHasChain, SDNPInGlue]>;
 
+def MipsSync : SDNode<"MipsISD::Sync", SDT_Sync, [SDNPHasChain]>;
+
+def MipsExt :  SDNode<"MipsISD::Ext", SDT_Ext>;
+def MipsIns :  SDNode<"MipsISD::Ins", SDT_Ins>;
+
 //===----------------------------------------------------------------------===//
 // Mips Instruction Predicate Definitions.
 //===----------------------------------------------------------------------===//
@@ -113,8 +125,13 @@ def HasSEInReg  : Predicate<"Subtarget.hasSEInReg()">;
 def HasBitCount : Predicate<"Subtarget.hasBitCount()">;
 def HasSwap     : Predicate<"Subtarget.hasSwap()">;
 def HasCondMov  : Predicate<"Subtarget.hasCondMov()">;
-def IsMips32    : Predicate<"Subtarget.isMips32()">;
-def IsMips32r2  : Predicate<"Subtarget.isMips32r2()">;
+def HasMips32    : Predicate<"Subtarget.hasMips32()">;
+def HasMips32r2  : Predicate<"Subtarget.hasMips32r2()">;
+def HasMips64    : Predicate<"Subtarget.hasMips64()">;
+def NotMips64    : Predicate<"!Subtarget.hasMips64()">;
+def HasMips64r2  : Predicate<"Subtarget.hasMips64r2()">;
+def IsN64       : Predicate<"Subtarget.isABI_N64()">;
+def NotN64      : Predicate<"!Subtarget.isABI_N64()">;
 
 //===----------------------------------------------------------------------===//
 // Mips Operand, Complex Patterns and Transformations Definitions.
@@ -124,6 +141,7 @@ def IsMips32r2  : Predicate<"Subtarget.isMips32r2()">;
 def brtarget    : Operand<OtherVT>;
 def calltarget  : Operand<i32>;
 def simm16      : Operand<i32>;
+def simm16_64   : Operand<i64>;
 def shamt       : Operand<i32>;
 
 // Unsigned Operand
@@ -137,6 +155,11 @@ def mem : Operand<i32> {
   let MIOperandInfo = (ops CPURegs, simm16);
 }
 
+def mem64 : Operand<i64> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops CPU64Regs, simm16_64);
+}
+
 def mem_ea : Operand<i32> {
   let PrintMethod = "printMemOperandEA";
   let MIOperandInfo = (ops CPURegs, simm16);
@@ -176,37 +199,86 @@ def immZExt5 : PatLeaf<(imm), [{
 // since load and store instructions from stack used it.
 def addr : ComplexPattern<iPTR, 2, "SelectAddr", [frameindex], []>;
 
+//===----------------------------------------------------------------------===//
+// Pattern fragment for load/store
+//===----------------------------------------------------------------------===//
+class UnalignedLoad<PatFrag Node> : PatFrag<(ops node:$ptr), (Node node:$ptr), [{
+  LoadSDNode *LD = cast<LoadSDNode>(N);
+  return LD->getMemoryVT().getSizeInBits()/8 > LD->getAlignment();
+}]>;
+
+class AlignedLoad<PatFrag Node> : PatFrag<(ops node:$ptr), (Node node:$ptr), [{
+  LoadSDNode *LD = cast<LoadSDNode>(N);
+  return LD->getMemoryVT().getSizeInBits()/8 <= LD->getAlignment();
+}]>;
+
+class UnalignedStore<PatFrag Node> : PatFrag<(ops node:$val, node:$ptr),
+                                             (Node node:$val, node:$ptr), [{
+  StoreSDNode *SD = cast<StoreSDNode>(N);
+  return SD->getMemoryVT().getSizeInBits()/8 > SD->getAlignment();
+}]>;
+
+class AlignedStore<PatFrag Node> : PatFrag<(ops node:$val, node:$ptr),
+                                           (Node node:$val, node:$ptr), [{
+  StoreSDNode *SD = cast<StoreSDNode>(N);
+  return SD->getMemoryVT().getSizeInBits()/8 <= SD->getAlignment();
+}]>;
+
+// Load/Store PatFrags.
+def sextloadi16_a   : AlignedLoad<sextloadi16>;
+def zextloadi16_a   : AlignedLoad<zextloadi16>;
+def extloadi16_a    : AlignedLoad<extloadi16>;
+def load_a          : AlignedLoad<load>;
+def sextloadi32_a   : AlignedLoad<sextloadi32>;
+def zextloadi32_a   : AlignedLoad<zextloadi32>;
+def extloadi32_a    : AlignedLoad<extloadi32>;
+def truncstorei16_a : AlignedStore<truncstorei16>;
+def store_a         : AlignedStore<store>;
+def truncstorei32_a : AlignedStore<truncstorei32>;
+def sextloadi16_u   : UnalignedLoad<sextloadi16>;
+def zextloadi16_u   : UnalignedLoad<zextloadi16>;
+def extloadi16_u    : UnalignedLoad<extloadi16>;
+def load_u          : UnalignedLoad<load>;
+def sextloadi32_u   : UnalignedLoad<sextloadi32>;
+def zextloadi32_u   : UnalignedLoad<zextloadi32>;
+def extloadi32_u    : UnalignedLoad<extloadi32>;
+def truncstorei16_u : UnalignedStore<truncstorei16>;
+def store_u         : UnalignedStore<store>;
+def truncstorei32_u : UnalignedStore<truncstorei32>;
+
 //===----------------------------------------------------------------------===//
 // Instructions specific format
 //===----------------------------------------------------------------------===//
 
-// Arithmetic 3 register operands
-class ArithR<bits<6> op, bits<6> func, string instr_asm, SDNode OpNode,
-             InstrItinClass itin, bit isComm = 0>:
-  FR<op, func, (outs CPURegs:$dst), (ins CPURegs:$b, CPURegs:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (OpNode CPURegs:$b, CPURegs:$c))], itin> {
+// Arithmetic and logical instructions with 3 register operands.
+class ArithLogicR<bits<6> op, bits<6> func, string instr_asm, SDNode OpNode,
+                  InstrItinClass itin, RegisterClass RC, bit isComm = 0>:
+  FR<op, func, (outs RC:$rd), (ins RC:$rs, RC:$rt),
+     !strconcat(instr_asm, "\t$rd, $rs, $rt"),
+     [(set RC:$rd, (OpNode RC:$rs, RC:$rt))], itin> {
+  let shamt = 0;
   let isCommutable = isComm;
 }
 
 class ArithOverflowR<bits<6> op, bits<6> func, string instr_asm,
-                     bit isComm = 0>:
-  FR<op, func, (outs CPURegs:$dst), (ins CPURegs:$b, CPURegs:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"), [], IIAlu> {
+                    InstrItinClass itin, RegisterClass RC, bit isComm = 0>:
+  FR<op, func, (outs RC:$rd), (ins RC:$rs, RC:$rt),
+     !strconcat(instr_asm, "\t$rd, $rs, $rt"), [], itin> {
+  let shamt = 0;
   let isCommutable = isComm;
 }
 
-// Arithmetic 2 register operands
-class ArithI<bits<6> op, string instr_asm, SDNode OpNode,
-             Operand Od, PatLeaf imm_type> :
-  FI<op, (outs CPURegs:$dst), (ins CPURegs:$b, Od:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (OpNode CPURegs:$b, imm_type:$c))], IIAlu>;
+// Arithmetic and logical instructions with 2 register operands.
+class ArithLogicI<bits<6> op, string instr_asm, SDNode OpNode,
+                  Operand Od, PatLeaf imm_type, RegisterClass RC> :
+  FI<op, (outs RC:$rt), (ins RC:$rs, Od:$i),
+     !strconcat(instr_asm, "\t$rt, $rs, $i"),
+     [(set RC:$rt, (OpNode RC:$rs, imm_type:$i))], IIAlu>;
 
 class ArithOverflowI<bits<6> op, string instr_asm, SDNode OpNode,
-             Operand Od, PatLeaf imm_type> :
-  FI<op, (outs CPURegs:$dst), (ins CPURegs:$b, Od:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"), [], IIAlu>;
+                     Operand Od, PatLeaf imm_type, RegisterClass RC> :
+  FI<op, (outs RC:$rt), (ins RC:$rs, Od:$i),
+     !strconcat(instr_asm, "\t$rt, $rs, $i"), [], IIAlu>;
 
 // Arithmetic Multiply ADD/SUB
 let rd = 0, shamt = 0, Defs = [HI, LO], Uses = [HI, LO] in
@@ -214,92 +286,134 @@ class MArithR<bits<6> func, string instr_asm, SDNode op, bit isComm = 0> :
   FR<0x1c, func, (outs), (ins CPURegs:$rs, CPURegs:$rt),
      !strconcat(instr_asm, "\t$rs, $rt"),
      [(op CPURegs:$rs, CPURegs:$rt, LO, HI)], IIImul> {
+  let rd = 0;
+  let shamt = 0;
   let isCommutable = isComm;
 }
 
 //  Logical
-let isCommutable = 1 in
-class LogicR<bits<6> func, string instr_asm, SDNode OpNode>:
-  FR<0x00, func, (outs CPURegs:$dst), (ins CPURegs:$b, CPURegs:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (OpNode CPURegs:$b, CPURegs:$c))], IIAlu>;
-
-class LogicI<bits<6> op, string instr_asm, SDNode OpNode>:
-  FI<op, (outs CPURegs:$dst), (ins CPURegs:$b, uimm16:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (OpNode CPURegs:$b, immZExt16:$c))], IIAlu>;
-
-let isCommutable = 1 in
-class LogicNOR<bits<6> op, bits<6> func, string instr_asm>:
-  FR<op, func, (outs CPURegs:$dst), (ins CPURegs:$b, CPURegs:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (not (or CPURegs:$b, CPURegs:$c)))], IIAlu>;
+class LogicNOR<bits<6> op, bits<6> func, string instr_asm, RegisterClass RC>:
+  FR<op, func, (outs RC:$rd), (ins RC:$rs, RC:$rt),
+     !strconcat(instr_asm, "\t$rd, $rs, $rt"),
+     [(set RC:$rd, (not (or RC:$rs, RC:$rt)))], IIAlu> {
+  let shamt = 0;
+  let isCommutable = 1;
+}
 
 // Shifts
 class LogicR_shift_rotate_imm<bits<6> func, bits<5> _rs, string instr_asm,
                               SDNode OpNode>:
-  FR<0x00, func, (outs CPURegs:$dst), (ins CPURegs:$b, shamt:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (OpNode CPURegs:$b, immZExt5:$c))], IIAlu> {
+  FR<0x00, func, (outs CPURegs:$rd), (ins CPURegs:$rt, shamt:$shamt),
+     !strconcat(instr_asm, "\t$rd, $rt, $shamt"),
+     [(set CPURegs:$rd, (OpNode CPURegs:$rt, (i32 immZExt5:$shamt)))], IIAlu> {
   let rs = _rs;
 }
 
-class LogicR_shift_rotate_reg<bits<6> func, bits<5> _shamt, string instr_asm,
+class LogicR_shift_rotate_reg<bits<6> func, bits<5> isRotate, string instr_asm,
                               SDNode OpNode>:
-  FR<0x00, func, (outs CPURegs:$dst), (ins CPURegs:$c, CPURegs:$b),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (OpNode CPURegs:$b, CPURegs:$c))], IIAlu> {
-  let shamt = _shamt;
+  FR<0x00, func, (outs CPURegs:$rd), (ins CPURegs:$rs, CPURegs:$rt),
+     !strconcat(instr_asm, "\t$rd, $rt, $rs"),
+     [(set CPURegs:$rd, (OpNode CPURegs:$rt, CPURegs:$rs))], IIAlu> {
+  let shamt = isRotate;
 }
 
 // Load Upper Imediate
 class LoadUpper<bits<6> op, string instr_asm>:
-  FI< op,
-      (outs CPURegs:$dst),
-      (ins uimm16:$imm),
-      !strconcat(instr_asm, "\t$dst, $imm"),
-      [], IIAlu>;
+  FI<op, (outs CPURegs:$rt), (ins uimm16:$imm),
+     !strconcat(instr_asm, "\t$rt, $imm"), [], IIAlu> {
+  let rs = 0;
+}
 
 // Memory Load/Store
-let canFoldAsLoad = 1, hasDelaySlot = 1 in
-class LoadM<bits<6> op, string instr_asm, PatFrag OpNode>:
-  FI<op, (outs CPURegs:$dst), (ins mem:$addr),
-     !strconcat(instr_asm, "\t$dst, $addr"),
-     [(set CPURegs:$dst, (OpNode addr:$addr))], IILoad>;
+let canFoldAsLoad = 1 in
+class LoadM<bits<6> op, string instr_asm, PatFrag OpNode, RegisterClass RC,
+            Operand MemOpnd, bit Pseudo>:
+  FI<op, (outs RC:$rt), (ins MemOpnd:$addr),
+     !strconcat(instr_asm, "\t$rt, $addr"),
+     [(set RC:$rt, (OpNode addr:$addr))], IILoad> {
+  let isPseudo = Pseudo;
+}
 
-class StoreM<bits<6> op, string instr_asm, PatFrag OpNode>:
-  FI<op, (outs), (ins CPURegs:$dst, mem:$addr),
-     !strconcat(instr_asm, "\t$dst, $addr"),
-     [(OpNode CPURegs:$dst, addr:$addr)], IIStore>;
+class StoreM<bits<6> op, string instr_asm, PatFrag OpNode, RegisterClass RC,
+             Operand MemOpnd, bit Pseudo>:
+  FI<op, (outs), (ins RC:$rt, MemOpnd:$addr),
+     !strconcat(instr_asm, "\t$rt, $addr"),
+     [(OpNode RC:$rt, addr:$addr)], IIStore> {
+  let isPseudo = Pseudo;
+}
+
+// 32-bit load.
+multiclass LoadM32<bits<6> op, string instr_asm, PatFrag OpNode,
+                   bit Pseudo = 0> {
+  def #NAME# : LoadM<op, instr_asm, OpNode, CPURegs, mem, Pseudo>,
+               Requires<[NotN64]>;
+  def _P8    : LoadM<op, instr_asm, OpNode, CPURegs, mem64, Pseudo>,
+               Requires<[IsN64]>;
+} 
+
+// 64-bit load.
+multiclass LoadM64<bits<6> op, string instr_asm, PatFrag OpNode,
+                   bit Pseudo = 0> {
+  def #NAME# : LoadM<op, instr_asm, OpNode, CPU64Regs, mem, Pseudo>,
+               Requires<[NotN64]>;
+  def _P8    : LoadM<op, instr_asm, OpNode, CPU64Regs, mem64, Pseudo>,
+               Requires<[IsN64]>;
+} 
+
+// 32-bit store.
+multiclass StoreM32<bits<6> op, string instr_asm, PatFrag OpNode,
+                    bit Pseudo = 0> {
+  def #NAME# : StoreM<op, instr_asm, OpNode, CPURegs, mem, Pseudo>,
+               Requires<[NotN64]>;
+  def _P8    : StoreM<op, instr_asm, OpNode, CPURegs, mem64, Pseudo>,
+               Requires<[IsN64]>;
+}
+
+// 64-bit store.
+multiclass StoreM64<bits<6> op, string instr_asm, PatFrag OpNode,
+                    bit Pseudo = 0> {
+  def #NAME# : StoreM<op, instr_asm, OpNode, CPU64Regs, mem, Pseudo>,
+               Requires<[NotN64]>;
+  def _P8    : StoreM<op, instr_asm, OpNode, CPU64Regs, mem64, Pseudo>,
+               Requires<[IsN64]>;
+}
 
 // Conditional Branch
-let isBranch = 1, isTerminator=1, hasDelaySlot = 1 in {
-class CBranch<bits<6> op, string instr_asm, PatFrag cond_op>:
-  FI<op, (outs), (ins CPURegs:$a, CPURegs:$b, brtarget:$offset),
-     !strconcat(instr_asm, "\t$a, $b, $offset"),
-     [(brcond (cond_op CPURegs:$a, CPURegs:$b), bb:$offset)],
-     IIBranch>;
+class CBranch<bits<6> op, string instr_asm, PatFrag cond_op, RegisterClass RC>:
+  CBranchBase<op, (outs), (ins RC:$rs, RC:$rt, brtarget:$offset),
+              !strconcat(instr_asm, "\t$rs, $rt, $offset"),
+              [(brcond (i32 (cond_op RC:$rs, RC:$rt)), bb:$offset)], IIBranch> {
+  let isBranch = 1;
+  let isTerminator = 1;
+  let hasDelaySlot = 1;
+}
 
-class CBranchZero<bits<6> op, string instr_asm, PatFrag cond_op>:
-  FI<op, (outs), (ins CPURegs:$src, brtarget:$offset),
-     !strconcat(instr_asm, "\t$src, $offset"),
-     [(brcond (cond_op CPURegs:$src, 0), bb:$offset)],
-     IIBranch>;
+class CBranchZero<bits<6> op, bits<5> _rt, string instr_asm, PatFrag cond_op,
+                  RegisterClass RC>:
+  CBranchBase<op, (outs), (ins RC:$rs, brtarget:$offset),
+              !strconcat(instr_asm, "\t$rs, $offset"),
+              [(brcond (i32 (cond_op RC:$rs, 0)), bb:$offset)], IIBranch> {
+  let rt = _rt;
+  let isBranch = 1;
+  let isTerminator = 1;
+  let hasDelaySlot = 1;
 }
 
 // SetCC
-class SetCC_R<bits<6> op, bits<6> func, string instr_asm,
-      PatFrag cond_op>:
-  FR<op, func, (outs CPURegs:$dst), (ins CPURegs:$b, CPURegs:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (cond_op CPURegs:$b, CPURegs:$c))],
-     IIAlu>;
+class SetCC_R<bits<6> op, bits<6> func, string instr_asm, PatFrag cond_op,
+              RegisterClass RC>:
+  FR<op, func, (outs CPURegs:$rd), (ins RC:$rs, RC:$rt),
+     !strconcat(instr_asm, "\t$rd, $rs, $rt"),
+     [(set CPURegs:$rd, (cond_op RC:$rs, RC:$rt))],
+     IIAlu> {
+  let shamt = 0;
+}
 
-class SetCC_I<bits<6> op, string instr_asm, PatFrag cond_op,
-      Operand Od, PatLeaf imm_type>:
-  FI<op, (outs CPURegs:$dst), (ins CPURegs:$b, Od:$c),
-     !strconcat(instr_asm, "\t$dst, $b, $c"),
-     [(set CPURegs:$dst, (cond_op CPURegs:$b, imm_type:$c))],
+class SetCC_I<bits<6> op, string instr_asm, PatFrag cond_op, Operand Od,
+              PatLeaf imm_type, RegisterClass RC>:
+  FI<op, (outs CPURegs:$rd), (ins RC:$rs, Od:$i),
+     !strconcat(instr_asm, "\t$rd, $rs, $i"),
+     [(set CPURegs:$rd, (cond_op RC:$rs, imm_type:$i))],
      IIAlu>;
 
 // Unconditional branch
@@ -310,8 +424,12 @@ class JumpFJ<bits<6> op, string instr_asm>:
 
 let isBranch=1, isTerminator=1, isBarrier=1, rd=0, hasDelaySlot = 1 in
 class JumpFR<bits<6> op, bits<6> func, string instr_asm>:
-  FR<op, func, (outs), (ins CPURegs:$target),
-     !strconcat(instr_asm, "\t$target"), [(brind CPURegs:$target)], IIBranch>;
+  FR<op, func, (outs), (ins CPURegs:$rs),
+     !strconcat(instr_asm, "\t$rs"), [(brind CPURegs:$rs)], IIBranch> {
+  let rt = 0;
+  let rd = 0;
+  let shamt = 0;
+}
 
 // Jump and Link (Call)
 let isCall=1, hasDelaySlot=1,
@@ -323,76 +441,124 @@ let isCall=1, hasDelaySlot=1,
        !strconcat(instr_asm, "\t$target"), [(MipsJmpLink imm:$target)],
        IIBranch>;
 
-  let rd=31 in
   class JumpLinkReg<bits<6> op, bits<6> func, string instr_asm>:
     FR<op, func, (outs), (ins CPURegs:$rs, variable_ops),
-       !strconcat(instr_asm, "\t$rs"), [(MipsJmpLink CPURegs:$rs)], IIBranch>;
+       !strconcat(instr_asm, "\t$rs"), [(MipsJmpLink CPURegs:$rs)], IIBranch> {
+    let rt = 0;
+    let rd = 31;
+    let shamt = 0;
+  }
 
   class BranchLink<string instr_asm>:
     FI<0x1, (outs), (ins CPURegs:$rs, brtarget:$target, variable_ops),
-       !strconcat(instr_asm, "\t$rs, $target"), [], IIBranch>;
+       !strconcat(instr_asm, "\t$rs, $target"), [], IIBranch> {
+    let rt = 0;
+  }
 }
 
 // Mul, Div
-let Defs = [HI, LO] in {
-  let isCommutable = 1 in
-  class Mul<bits<6> func, string instr_asm, InstrItinClass itin>:
-    FR<0x00, func, (outs), (ins CPURegs:$a, CPURegs:$b),
-       !strconcat(instr_asm, "\t$a, $b"), [], itin>;
+class Mul<bits<6> func, string instr_asm, InstrItinClass itin>:
+  FR<0x00, func, (outs), (ins CPURegs:$rs, CPURegs:$rt),
+     !strconcat(instr_asm, "\t$rs, $rt"), [], itin> {
+  let rd = 0;
+  let shamt = 0;
+  let isCommutable = 1;
+  let Defs = [HI, LO];
+}
 
-  class Div<SDNode op, bits<6> func, string instr_asm, InstrItinClass itin>:
-            FR<0x00, func, (outs), (ins CPURegs:$a, CPURegs:$b),
-            !strconcat(instr_asm, "\t$$zero, $a, $b"),
-            [(op CPURegs:$a, CPURegs:$b)], itin>;
+class Div<SDNode op, bits<6> func, string instr_asm, InstrItinClass itin>:
+          FR<0x00, func, (outs), (ins CPURegs:$rs, CPURegs:$rt),
+          !strconcat(instr_asm, "\t$$zero, $rs, $rt"),
+          [(op CPURegs:$rs, CPURegs:$rt)], itin> {
+  let rd = 0;
+  let shamt = 0;
+  let Defs = [HI, LO];
 }
 
 // Move from Hi/Lo
 class MoveFromLOHI<bits<6> func, string instr_asm>:
-  FR<0x00, func, (outs CPURegs:$dst), (ins),
-     !strconcat(instr_asm, "\t$dst"), [], IIHiLo>;
+  FR<0x00, func, (outs CPURegs:$rd), (ins),
+     !strconcat(instr_asm, "\t$rd"), [], IIHiLo> {
+  let rs = 0;
+  let rt = 0;
+  let shamt = 0;
+}
 
 class MoveToLOHI<bits<6> func, string instr_asm>:
-  FR<0x00, func, (outs), (ins CPURegs:$src),
-     !strconcat(instr_asm, "\t$src"), [], IIHiLo>;
+  FR<0x00, func, (outs), (ins CPURegs:$rs),
+     !strconcat(instr_asm, "\t$rs"), [], IIHiLo> {
+  let rt = 0;
+  let rd = 0;
+  let shamt = 0;
+}
 
 class EffectiveAddress<string instr_asm> :
-  FI<0x09, (outs CPURegs:$dst), (ins mem_ea:$addr),
-     instr_asm, [(set CPURegs:$dst, addr:$addr)], IIAlu>;
+  FI<0x09, (outs CPURegs:$rt), (ins mem_ea:$addr),
+     instr_asm, [(set CPURegs:$rt, addr:$addr)], IIAlu>;
 
 // Count Leading Ones/Zeros in Word
 class CountLeading<bits<6> func, string instr_asm, list<dag> pattern>:
-  FR<0x1c, func, (outs CPURegs:$dst), (ins CPURegs:$src),
-     !strconcat(instr_asm, "\t$dst, $src"), pattern, IIAlu>,
+  FR<0x1c, func, (outs CPURegs:$rd), (ins CPURegs:$rs),
+     !strconcat(instr_asm, "\t$rd, $rs"), pattern, IIAlu>,
      Requires<[HasBitCount]> {
   let shamt = 0;
   let rt = rd;
 }
 
 // Sign Extend in Register.
-class SignExtInReg<bits<6> func, string instr_asm, ValueType vt>:
-  FR<0x3f, func, (outs CPURegs:$dst), (ins CPURegs:$src),
-     !strconcat(instr_asm, "\t$dst, $src"),
-     [(set CPURegs:$dst, (sext_inreg CPURegs:$src, vt))], NoItinerary>;
+class SignExtInReg<bits<5> sa, string instr_asm, ValueType vt>:
+  FR<0x3f, 0x20, (outs CPURegs:$rd), (ins CPURegs:$rt),
+     !strconcat(instr_asm, "\t$rd, $rt"),
+     [(set CPURegs:$rd, (sext_inreg CPURegs:$rt, vt))], NoItinerary> {
+  let rs = 0;
+  let shamt = sa;
+  let Predicates = [HasSEInReg];
+}
 
 // Byte Swap
-class ByteSwap<bits<6> func, string instr_asm>:
-  FR<0x1f, func, (outs CPURegs:$dst), (ins CPURegs:$src),
-     !strconcat(instr_asm, "\t$dst, $src"),
-     [(set CPURegs:$dst, (bswap CPURegs:$src))], NoItinerary>;
-
-// Conditional Move
-class CondMov<bits<6> func, string instr_asm, PatLeaf MovCode>:
-  FR<0x00, func, (outs CPURegs:$dst), (ins CPURegs:$F, CPURegs:$T,
-     CPURegs:$cond), !strconcat(instr_asm, "\t$dst, $T, $cond"),
-     [], NoItinerary>;
+class ByteSwap<bits<6> func, bits<5> sa, string instr_asm>:
+  FR<0x1f, func, (outs CPURegs:$rd), (ins CPURegs:$rt),
+     !strconcat(instr_asm, "\t$rd, $rt"),
+     [(set CPURegs:$rd, (bswap CPURegs:$rt))], NoItinerary> {
+  let rs = 0;
+  let shamt = sa;
+  let Predicates = [HasSwap];
+}
 
 // Read Hardware
-class ReadHardware: FR<0x1f, 0x3b, (outs CPURegs:$dst), (ins HWRegs:$src),
-    "rdhwr\t$dst, $src", [], IIAlu> {
+class ReadHardware: FR<0x1f, 0x3b, (outs CPURegs:$rt), (ins HWRegs:$rd),
+    "rdhwr\t$rt, $rd", [], IIAlu> {
   let rs = 0;
   let shamt = 0;
 }
 
+// Ext and Ins
+class ExtIns<bits<6> _funct, string instr_asm, dag outs, dag ins,
+             list<dag> pattern, InstrItinClass itin>:
+  FR<0x1f, _funct, outs, ins, !strconcat(instr_asm, " $rt, $rs, $pos, $sz"),
+     pattern, itin>, Requires<[HasMips32r2]> {
+  bits<5> pos;
+  bits<5> sz;
+  let rd = sz;
+  let shamt = pos;
+}
+
+// Atomic instructions with 2 source operands (ATOMIC_SWAP & ATOMIC_LOAD_*).
+class Atomic2Ops<PatFrag Op, string Opstr> :
+  MipsPseudo<(outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
+             !strconcat("atomic_", Opstr, "\t$dst, $ptr, $incr"),
+             [(set CPURegs:$dst,
+              (Op CPURegs:$ptr, CPURegs:$incr))]>;
+
+// Atomic Compare & Swap.
+class AtomicCmpSwap<PatFrag Op, string Width> :
+  MipsPseudo<(outs CPURegs:$dst), 
+             (ins CPURegs:$ptr, CPURegs:$cmp, CPURegs:$swap),
+             !strconcat("atomic_cmp_swap_", Width, 
+                        "\t$dst, $ptr, $cmp, $swap"),
+             [(set CPURegs:$dst,
+              (Op CPURegs:$ptr, CPURegs:$cmp, CPURegs:$swap))]>;
+
 //===----------------------------------------------------------------------===//
 // Pseudo instructions
 //===----------------------------------------------------------------------===//
@@ -427,112 +593,32 @@ def CPLOAD : MipsPseudo<(outs), (ins CPURegs:$picreg), ".cpload\t$picreg", []>;
 def CPRESTORE : MipsPseudo<(outs), (ins i32imm:$loc), ".cprestore\t$loc", []>;
 
 let usesCustomInserter = 1 in {
-  def ATOMIC_LOAD_ADD_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_add_8\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_add_8 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_ADD_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_add_16\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_add_16 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_ADD_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_add_32\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_add_32 CPURegs:$ptr, CPURegs:$incr))]>;
+  def ATOMIC_LOAD_ADD_I8   : Atomic2Ops<atomic_load_add_8, "load_add_8">;
+  def ATOMIC_LOAD_ADD_I16  : Atomic2Ops<atomic_load_add_16, "load_add_16">;
+  def ATOMIC_LOAD_ADD_I32  : Atomic2Ops<atomic_load_add_32, "load_add_32">;
+  def ATOMIC_LOAD_SUB_I8   : Atomic2Ops<atomic_load_sub_8, "load_sub_8">;
+  def ATOMIC_LOAD_SUB_I16  : Atomic2Ops<atomic_load_sub_16, "load_sub_16">;
+  def ATOMIC_LOAD_SUB_I32  : Atomic2Ops<atomic_load_sub_32, "load_sub_32">;
+  def ATOMIC_LOAD_AND_I8   : Atomic2Ops<atomic_load_and_8, "load_and_8">;
+  def ATOMIC_LOAD_AND_I16  : Atomic2Ops<atomic_load_and_16, "load_and_16">;
+  def ATOMIC_LOAD_AND_I32  : Atomic2Ops<atomic_load_and_32, "load_and_32">;
+  def ATOMIC_LOAD_OR_I8    : Atomic2Ops<atomic_load_or_8, "load_or_8">;
+  def ATOMIC_LOAD_OR_I16   : Atomic2Ops<atomic_load_or_16, "load_or_16">;
+  def ATOMIC_LOAD_OR_I32   : Atomic2Ops<atomic_load_or_32, "load_or_32">;
+  def ATOMIC_LOAD_XOR_I8   : Atomic2Ops<atomic_load_xor_8, "load_xor_8">;
+  def ATOMIC_LOAD_XOR_I16  : Atomic2Ops<atomic_load_xor_16, "load_xor_16">;
+  def ATOMIC_LOAD_XOR_I32  : Atomic2Ops<atomic_load_xor_32, "load_xor_32">;
+  def ATOMIC_LOAD_NAND_I8  : Atomic2Ops<atomic_load_nand_8, "load_nand_8">;
+  def ATOMIC_LOAD_NAND_I16 : Atomic2Ops<atomic_load_nand_16, "load_nand_16">;
+  def ATOMIC_LOAD_NAND_I32 : Atomic2Ops<atomic_load_nand_32, "load_nand_32">;
 
-  def ATOMIC_LOAD_SUB_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_sub_8\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_sub_8 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_SUB_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_sub_16\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_sub_16 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_SUB_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_sub_32\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_sub_32 CPURegs:$ptr, CPURegs:$incr))]>;
+  def ATOMIC_SWAP_I8       : Atomic2Ops<atomic_swap_8, "swap_8">;
+  def ATOMIC_SWAP_I16      : Atomic2Ops<atomic_swap_16, "swap_16">;
+  def ATOMIC_SWAP_I32      : Atomic2Ops<atomic_swap_32, "swap_32">;
 
-  def ATOMIC_LOAD_AND_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_and_8\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_and_8 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_AND_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_and_16\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_and_16 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_AND_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_and_32\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_and_32 CPURegs:$ptr, CPURegs:$incr))]>;
-
-  def ATOMIC_LOAD_OR_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_or_8\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_or_8 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_OR_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_or_16\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_or_16 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_OR_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_or_32\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_or_32 CPURegs:$ptr, CPURegs:$incr))]>;
-
-  def ATOMIC_LOAD_XOR_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_xor_8\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_xor_8 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_XOR_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_xor_16\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_xor_16 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_XOR_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_xor_32\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_xor_32 CPURegs:$ptr, CPURegs:$incr))]>;
-
-  def ATOMIC_LOAD_NAND_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_nand_8\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_nand_8 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_NAND_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_nand_16\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_nand_16 CPURegs:$ptr, CPURegs:$incr))]>;
-  def ATOMIC_LOAD_NAND_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$incr),
-    "atomic_load_nand_32\t$dst, $ptr, $incr",
-    [(set CPURegs:$dst, (atomic_load_nand_32 CPURegs:$ptr, CPURegs:$incr))]>;
-
-  def ATOMIC_SWAP_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$val),
-    "atomic_swap_8\t$dst, $ptr, $val",
-    [(set CPURegs:$dst, (atomic_swap_8 CPURegs:$ptr, CPURegs:$val))]>;
-  def ATOMIC_SWAP_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$val),
-    "atomic_swap_16\t$dst, $ptr, $val",
-    [(set CPURegs:$dst, (atomic_swap_16 CPURegs:$ptr, CPURegs:$val))]>;
-  def ATOMIC_SWAP_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$val),
-    "atomic_swap_32\t$dst, $ptr, $val",
-    [(set CPURegs:$dst, (atomic_swap_32 CPURegs:$ptr, CPURegs:$val))]>;
-
-  def ATOMIC_CMP_SWAP_I8 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$oldval, CPURegs:$newval),
-    "atomic_cmp_swap_8\t$dst, $ptr, $oldval, $newval",
-    [(set CPURegs:$dst,
-         (atomic_cmp_swap_8 CPURegs:$ptr, CPURegs:$oldval, CPURegs:$newval))]>;
-  def ATOMIC_CMP_SWAP_I16 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$oldval, CPURegs:$newval),
-    "atomic_cmp_swap_16\t$dst, $ptr, $oldval, $newval",
-    [(set CPURegs:$dst,
-         (atomic_cmp_swap_16 CPURegs:$ptr, CPURegs:$oldval, CPURegs:$newval))]>;
-  def ATOMIC_CMP_SWAP_I32 : MipsPseudo<
-    (outs CPURegs:$dst), (ins CPURegs:$ptr, CPURegs:$oldval, CPURegs:$newval),
-    "atomic_cmp_swap_32\t$dst, $ptr, $oldval, $newval",
-    [(set CPURegs:$dst,
-         (atomic_cmp_swap_32 CPURegs:$ptr, CPURegs:$oldval, CPURegs:$newval))]>;
+  def ATOMIC_CMP_SWAP_I8   : AtomicCmpSwap<atomic_cmp_swap_8, "8">;
+  def ATOMIC_CMP_SWAP_I16  : AtomicCmpSwap<atomic_cmp_swap_16, "16">;
+  def ATOMIC_CMP_SWAP_I32  : AtomicCmpSwap<atomic_cmp_swap_32, "32">;
 }
 
 //===----------------------------------------------------------------------===//
@@ -544,26 +630,26 @@ let usesCustomInserter = 1 in {
 //===----------------------------------------------------------------------===//
 
 /// Arithmetic Instructions (ALU Immediate)
-def ADDiu   : ArithI<0x09, "addiu", add, simm16, immSExt16>;
-def ADDi    : ArithOverflowI<0x08, "addi",  add, simm16, immSExt16>;
-def SLTi    : SetCC_I<0x0a, "slti", setlt, simm16, immSExt16>;
-def SLTiu   : SetCC_I<0x0b, "sltiu", setult, simm16, immSExt16>;
-def ANDi    : LogicI<0x0c, "andi", and>;
-def ORi     : LogicI<0x0d, "ori",  or>;
-def XORi    : LogicI<0x0e, "xori",  xor>;
+def ADDiu   : ArithLogicI<0x09, "addiu", add, simm16, immSExt16, CPURegs>;
+def ADDi    : ArithOverflowI<0x08, "addi", add, simm16, immSExt16, CPURegs>;
+def SLTi    : SetCC_I<0x0a, "slti", setlt, simm16, immSExt16, CPURegs>;
+def SLTiu   : SetCC_I<0x0b, "sltiu", setult, simm16, immSExt16, CPURegs>;
+def ANDi    : ArithLogicI<0x0c, "andi", and, uimm16, immZExt16, CPURegs>;
+def ORi     : ArithLogicI<0x0d, "ori", or, uimm16, immZExt16, CPURegs>;
+def XORi    : ArithLogicI<0x0e, "xori", xor, uimm16, immZExt16, CPURegs>;
 def LUi     : LoadUpper<0x0f, "lui">;
 
 /// Arithmetic Instructions (3-Operand, R-Type)
-def ADDu    : ArithR<0x00, 0x21, "addu", add, IIAlu, 1>;
-def SUBu    : ArithR<0x00, 0x23, "subu", sub, IIAlu>;
-def ADD     : ArithOverflowR<0x00, 0x20, "add", 1>;
-def SUB     : ArithOverflowR<0x00, 0x22, "sub">;
-def SLT     : SetCC_R<0x00, 0x2a, "slt", setlt>;
-def SLTu    : SetCC_R<0x00, 0x2b, "sltu", setult>;
-def AND     : LogicR<0x24, "and", and>;
-def OR      : LogicR<0x25, "or",  or>;
-def XOR     : LogicR<0x26, "xor", xor>;
-def NOR     : LogicNOR<0x00, 0x27, "nor">;
+def ADDu    : ArithLogicR<0x00, 0x21, "addu", add, IIAlu, CPURegs, 1>;
+def SUBu    : ArithLogicR<0x00, 0x23, "subu", sub, IIAlu, CPURegs>;
+def ADD     : ArithOverflowR<0x00, 0x20, "add", IIAlu, CPURegs, 1>;
+def SUB     : ArithOverflowR<0x00, 0x22, "sub", IIAlu, CPURegs>;
+def SLT     : SetCC_R<0x00, 0x2a, "slt", setlt, CPURegs>;
+def SLTu    : SetCC_R<0x00, 0x2b, "sltu", setult, CPURegs>;
+def AND     : ArithLogicR<0x00, 0x24, "and", and, IIAlu, CPURegs, 1>;
+def OR      : ArithLogicR<0x00, 0x25, "or",  or, IIAlu, CPURegs, 1>;
+def XOR     : ArithLogicR<0x00, 0x26, "xor", xor, IIAlu, CPURegs, 1>;
+def NOR     : LogicNOR<0x00, 0x27, "nor", CPURegs>;
 
 /// Shift Instructions
 def SLL     : LogicR_shift_rotate_imm<0x00, 0x00, "sll", shl>;
@@ -574,45 +660,58 @@ def SRLV    : LogicR_shift_rotate_reg<0x06, 0x00, "srlv", srl>;
 def SRAV    : LogicR_shift_rotate_reg<0x07, 0x00, "srav", sra>;
 
 // Rotate Instructions
-let Predicates = [IsMips32r2] in {
+let Predicates = [HasMips32r2] in {
     def ROTR    : LogicR_shift_rotate_imm<0x02, 0x01, "rotr", rotr>;
     def ROTRV   : LogicR_shift_rotate_reg<0x06, 0x01, "rotrv", rotr>;
 }
 
 /// Load and Store Instructions
-def LB      : LoadM<0x20, "lb",  sextloadi8>;
-def LBu     : LoadM<0x24, "lbu", zextloadi8>;
-def LH      : LoadM<0x21, "lh",  sextloadi16>;
-def LHu     : LoadM<0x25, "lhu", zextloadi16>;
-def LW      : LoadM<0x23, "lw",  load>;
-def SB      : StoreM<0x28, "sb", truncstorei8>;
-def SH      : StoreM<0x29, "sh", truncstorei16>;
-def SW      : StoreM<0x2b, "sw", store>;
+///  aligned
+defm LB      : LoadM32<0x20, "lb",  sextloadi8>;
+defm LBu     : LoadM32<0x24, "lbu", zextloadi8>;
+defm LH      : LoadM32<0x21, "lh",  sextloadi16_a>;
+defm LHu     : LoadM32<0x25, "lhu", zextloadi16_a>;
+defm LW      : LoadM32<0x23, "lw",  load_a>;
+defm SB      : StoreM32<0x28, "sb", truncstorei8>;
+defm SH      : StoreM32<0x29, "sh", truncstorei16_a>;
+defm SW      : StoreM32<0x2b, "sw", store_a>;
+
+///  unaligned
+defm ULH     : LoadM32<0x21, "ulh",  sextloadi16_u, 1>;
+defm ULHu    : LoadM32<0x25, "ulhu", zextloadi16_u, 1>;
+defm ULW     : LoadM32<0x23, "ulw",  load_u, 1>;
+defm USH     : StoreM32<0x29, "ush", truncstorei16_u, 1>;
+defm USW     : StoreM32<0x2b, "usw", store_u, 1>;
+
+let hasSideEffects = 1 in
+def SYNC : MipsInst<(outs), (ins i32imm:$stype), "sync $stype",
+                    [(MipsSync imm:$stype)], NoItinerary>
+{
+  let opcode = 0;
+  let Inst{25-11} = 0;
+  let Inst{5-0} = 15;
+}
 
 /// Load-linked, Store-conditional
-let hasDelaySlot = 1 in
+let mayLoad = 1 in
   def LL    : FI<0x30, (outs CPURegs:$dst), (ins mem:$addr),
               "ll\t$dst, $addr", [], IILoad>;
-let Constraints = "$src = $dst" in
+let mayStore = 1, Constraints = "$src = $dst" in
   def SC    : FI<0x38, (outs CPURegs:$dst), (ins CPURegs:$src, mem:$addr),
               "sc\t$src, $addr", [], IIStore>;
 
 /// Jump and Branch Instructions
 def J       : JumpFJ<0x02, "j">;
-def JR      : JumpFR<0x00, 0x08, "jr">;
+let isIndirectBranch = 1 in
+  def JR      : JumpFR<0x00, 0x08, "jr">;
 def JAL     : JumpLink<0x03, "jal">;
 def JALR    : JumpLinkReg<0x00, 0x09, "jalr">;
-def BEQ     : CBranch<0x04, "beq", seteq>;
-def BNE     : CBranch<0x05, "bne", setne>;
-
-let rt=1 in
-  def BGEZ  : CBranchZero<0x01, "bgez", setge>;
-
-let rt=0 in {
-  def BGTZ  : CBranchZero<0x07, "bgtz", setgt>;
-  def BLEZ  : CBranchZero<0x07, "blez", setle>;
-  def BLTZ  : CBranchZero<0x01, "bltz", setlt>;
-}
+def BEQ     : CBranch<0x04, "beq", seteq, CPURegs>;
+def BNE     : CBranch<0x05, "bne", setne, CPURegs>;
+def BGEZ    : CBranchZero<0x01, 1, "bgez", setge, CPURegs>;
+def BGTZ    : CBranchZero<0x07, 0, "bgtz", setgt, CPURegs>;
+def BLEZ    : CBranchZero<0x07, 0, "blez", setle, CPURegs>;
+def BLTZ    : CBranchZero<0x01, 0, "bltz", setlt, CPURegs>;
 
 def BGEZAL  : BranchLink<"bgezal">;
 def BLTZAL  : BranchLink<"bltzal">;
@@ -639,40 +738,31 @@ let Uses = [LO] in
   def MFLO  : MoveFromLOHI<0x12, "mflo">;
 
 /// Sign Ext In Register Instructions.
-let Predicates = [HasSEInReg] in {
-  let shamt = 0x10, rs = 0 in
-    def SEB : SignExtInReg<0x21, "seb", i8>;
-
-  let shamt = 0x18, rs = 0 in
-    def SEH : SignExtInReg<0x20, "seh", i16>;
-}
+def SEB : SignExtInReg<0x10, "seb", i8>;
+def SEH : SignExtInReg<0x18, "seh", i16>;
 
 /// Count Leading
-def CLZ : CountLeading<0b100000, "clz",
-                       [(set CPURegs:$dst, (ctlz CPURegs:$src))]>;
-def CLO : CountLeading<0b100001, "clo",
-                       [(set CPURegs:$dst, (ctlz (not CPURegs:$src)))]>;
+def CLZ : CountLeading<0x20, "clz",
+                       [(set CPURegs:$rd, (ctlz CPURegs:$rs))]>;
+def CLO : CountLeading<0x21, "clo",
+                       [(set CPURegs:$rd, (ctlz (not CPURegs:$rs)))]>;
 
 /// Byte Swap
-let Predicates = [HasSwap] in {
-  let shamt = 0x3, rs = 0 in
-    def WSBW : ByteSwap<0x20, "wsbw">;
-}
-
-/// Conditional Move
-def MIPS_CMOV_ZERO  : PatLeaf<(i32 0)>;
-def MIPS_CMOV_NZERO : PatLeaf<(i32 1)>;
+def WSBW : ByteSwap<0x20, 0x2, "wsbw">;
 
 // Conditional moves:
 // These instructions are expanded in
 // MipsISelLowering::EmitInstrWithCustomInserter if target does not have
 // conditional move instructions.
 // flag:int, data:int
-let usesCustomInserter = 1, shamt = 0, Constraints = "$F = $dst" in
-  class CondMovIntInt<bits<6> funct, string instr_asm> :
-    FR<0, funct, (outs CPURegs:$dst),
-       (ins CPURegs:$T, CPURegs:$cond, CPURegs:$F),
-       !strconcat(instr_asm, "\t$dst, $T, $cond"), [], NoItinerary>;
+class CondMovIntInt<bits<6> funct, string instr_asm> :
+  FR<0, funct, (outs CPURegs:$rd),
+     (ins CPURegs:$rs, CPURegs:$rt, CPURegs:$F),
+     !strconcat(instr_asm, "\t$rd, $rs, $rt"), [], NoItinerary> {
+  let shamt = 0;
+  let usesCustomInserter = 1;
+  let Constraints = "$F = $rd";
+}
 
 def MOVZ_I : CondMovIntInt<0x0a, "movz">;
 def MOVN_I : CondMovIntInt<0x0b, "movn">;
@@ -685,13 +775,13 @@ let addr=0 in
 // instructions. The same not happens for stack address copies, so an
 // add op with mem ComplexPattern is used and the stack address copy
 // can be matched. It's similar to Sparc LEA_ADDRi
-def LEA_ADDiu : EffectiveAddress<"addiu\t$dst, $addr">;
+def LEA_ADDiu : EffectiveAddress<"addiu\t$rt, $addr">;
 
 // DynAlloc node points to dynamically allocated stack space.
 // $sp is added to the list of implicitly used registers to prevent dead code
 // elimination from removing instructions that modify $sp.
 let Uses = [SP] in
-def DynAlloc : EffectiveAddress<"addiu\t$dst, $addr">;
+def DynAlloc : EffectiveAddress<"addiu\t$rt, $addr">;
 
 // MADD*/MSUB*
 def MADD  : MArithR<0, "madd", MipsMAdd, 1>;
@@ -701,10 +791,25 @@ def MSUBU : MArithR<5, "msubu", MipsMSubu>;
 
 // MUL is a assembly macro in the current used ISAs. In recent ISA's
 // it is a real instruction.
-def MUL   : ArithR<0x1c, 0x02, "mul", mul, IIImul, 1>, Requires<[IsMips32]>;
+def MUL   : ArithLogicR<0x1c, 0x02, "mul", mul, IIImul, CPURegs, 1>,
+            Requires<[HasMips32]>;
 
 def RDHWR : ReadHardware;
 
+def EXT : ExtIns<0, "ext", (outs CPURegs:$rt),
+                 (ins CPURegs:$rs, uimm16:$pos, uimm16:$sz),
+                 [(set CPURegs:$rt,
+                   (MipsExt CPURegs:$rs, immZExt5:$pos, immZExt5:$sz))],
+                 NoItinerary>;
+
+let Constraints = "$src = $rt" in
+def INS : ExtIns<4, "ins", (outs CPURegs:$rt),
+                 (ins CPURegs:$rs, uimm16:$pos, uimm16:$sz, CPURegs:$src),
+                 [(set CPURegs:$rt,
+                   (MipsIns CPURegs:$rs, immZExt5:$pos, immZExt5:$sz,
+                    CPURegs:$src))],
+                 NoItinerary>;
+
 //===----------------------------------------------------------------------===//
 //  Arbitrary patterns that map to one or more instructions
 //===----------------------------------------------------------------------===//
@@ -738,16 +843,20 @@ def : Pat<(MipsJmpLink (i32 texternalsym:$dst)),
 // hi/lo relocs
 def : Pat<(MipsHi tglobaladdr:$in), (LUi tglobaladdr:$in)>;
 def : Pat<(MipsHi tblockaddress:$in), (LUi tblockaddress:$in)>;
+def : Pat<(MipsLo tglobaladdr:$in), (ADDiu ZERO, tglobaladdr:$in)>;
+def : Pat<(MipsLo tblockaddress:$in), (ADDiu ZERO, tblockaddress:$in)>;
 def : Pat<(add CPURegs:$hi, (MipsLo tglobaladdr:$lo)),
           (ADDiu CPURegs:$hi, tglobaladdr:$lo)>;
 def : Pat<(add CPURegs:$hi, (MipsLo tblockaddress:$lo)),
           (ADDiu CPURegs:$hi, tblockaddress:$lo)>;
 
 def : Pat<(MipsHi tjumptable:$in), (LUi tjumptable:$in)>;
+def : Pat<(MipsLo tjumptable:$in), (ADDiu ZERO, tjumptable:$in)>;
 def : Pat<(add CPURegs:$hi, (MipsLo tjumptable:$lo)),
           (ADDiu CPURegs:$hi, tjumptable:$lo)>;
 
 def : Pat<(MipsHi tconstpool:$in), (LUi tconstpool:$in)>;
+def : Pat<(MipsLo tconstpool:$in), (ADDiu ZERO, tconstpool:$in)>;
 def : Pat<(add CPURegs:$hi, (MipsLo tconstpool:$lo)),
           (ADDiu CPURegs:$hi, tconstpool:$lo)>;
 
@@ -763,6 +872,7 @@ def : Pat<(add CPURegs:$gp, (MipsTlsGd tglobaltlsaddr:$in)),
 
 // tprel hi/lo
 def : Pat<(MipsTprelHi tglobaltlsaddr:$in), (LUi tglobaltlsaddr:$in)>;
+def : Pat<(MipsTprelLo tglobaltlsaddr:$in), (ADDiu ZERO, tglobaltlsaddr:$in)>;
 def : Pat<(add CPURegs:$hi, (MipsTprelLo tglobaltlsaddr:$lo)),
           (ADDiu CPURegs:$hi, tglobaltlsaddr:$lo)>;
 
@@ -784,60 +894,67 @@ def : Pat<(not CPURegs:$in),
 // extended load and stores
 def : Pat<(extloadi1  addr:$src), (LBu addr:$src)>;
 def : Pat<(extloadi8  addr:$src), (LBu addr:$src)>;
-def : Pat<(extloadi16 addr:$src), (LHu addr:$src)>;
+def : Pat<(extloadi16_a addr:$src), (LHu addr:$src)>;
+def : Pat<(extloadi16_u addr:$src), (ULHu addr:$src)>;
 
 // peepholes
 def : Pat<(store (i32 0), addr:$dst), (SW ZERO, addr:$dst)>;
 
 // brcond patterns
-def : Pat<(brcond (setne CPURegs:$lhs, 0), bb:$dst),
-          (BNE CPURegs:$lhs, ZERO, bb:$dst)>;
-def : Pat<(brcond (seteq CPURegs:$lhs, 0), bb:$dst),
-          (BEQ CPURegs:$lhs, ZERO, bb:$dst)>;
+multiclass BrcondPats<RegisterClass RC, Instruction BEQOp, Instruction BNEOp,
+                      Instruction SLTOp, Instruction SLTuOp, Instruction SLTiOp,
+                      Instruction SLTiuOp, Register ZEROReg> {
+def : Pat<(brcond (i32 (setne RC:$lhs, 0)), bb:$dst),
+          (BNEOp RC:$lhs, ZEROReg, bb:$dst)>;
+def : Pat<(brcond (i32 (seteq RC:$lhs, 0)), bb:$dst),
+          (BEQOp RC:$lhs, ZEROReg, bb:$dst)>;
 
-def : Pat<(brcond (setge CPURegs:$lhs, CPURegs:$rhs), bb:$dst),
-          (BEQ (SLT CPURegs:$lhs, CPURegs:$rhs), ZERO, bb:$dst)>;
-def : Pat<(brcond (setuge CPURegs:$lhs, CPURegs:$rhs), bb:$dst),
-          (BEQ (SLTu CPURegs:$lhs, CPURegs:$rhs), ZERO, bb:$dst)>;
-def : Pat<(brcond (setge CPURegs:$lhs, immSExt16:$rhs), bb:$dst),
-          (BEQ (SLTi CPURegs:$lhs, immSExt16:$rhs), ZERO, bb:$dst)>;
-def : Pat<(brcond (setuge CPURegs:$lhs, immSExt16:$rhs), bb:$dst),
-          (BEQ (SLTiu CPURegs:$lhs, immSExt16:$rhs), ZERO, bb:$dst)>;
+def : Pat<(brcond (i32 (setge RC:$lhs, RC:$rhs)), bb:$dst),
+          (BEQ (SLTOp RC:$lhs, RC:$rhs), ZERO, bb:$dst)>;
+def : Pat<(brcond (i32 (setuge RC:$lhs, RC:$rhs)), bb:$dst),
+          (BEQ (SLTuOp RC:$lhs, RC:$rhs), ZERO, bb:$dst)>;
+def : Pat<(brcond (i32 (setge RC:$lhs, immSExt16:$rhs)), bb:$dst),
+          (BEQ (SLTiOp RC:$lhs, immSExt16:$rhs), ZERO, bb:$dst)>;
+def : Pat<(brcond (i32 (setuge RC:$lhs, immSExt16:$rhs)), bb:$dst),
+          (BEQ (SLTiuOp RC:$lhs, immSExt16:$rhs), ZERO, bb:$dst)>;
 
-def : Pat<(brcond (setle CPURegs:$lhs, CPURegs:$rhs), bb:$dst),
-          (BEQ (SLT CPURegs:$rhs, CPURegs:$lhs), ZERO, bb:$dst)>;
-def : Pat<(brcond (setule CPURegs:$lhs, CPURegs:$rhs), bb:$dst),
-          (BEQ (SLTu CPURegs:$rhs, CPURegs:$lhs), ZERO, bb:$dst)>;
+def : Pat<(brcond (i32 (setle RC:$lhs, RC:$rhs)), bb:$dst),
+          (BEQ (SLTOp RC:$rhs, RC:$lhs), ZERO, bb:$dst)>;
+def : Pat<(brcond (i32 (setule RC:$lhs, RC:$rhs)), bb:$dst),
+          (BEQ (SLTuOp RC:$rhs, RC:$lhs), ZERO, bb:$dst)>;
 
-def : Pat<(brcond CPURegs:$cond, bb:$dst),
-          (BNE CPURegs:$cond, ZERO, bb:$dst)>;
+def : Pat<(brcond RC:$cond, bb:$dst),
+          (BNEOp RC:$cond, ZEROReg, bb:$dst)>;
+}
+
+defm : BrcondPats<CPURegs, BEQ, BNE, SLT, SLTu, SLTi, SLTiu, ZERO>;
 
 // select patterns
 multiclass MovzPats<RegisterClass RC, Instruction MOVZInst> {
-  def : Pat<(select (setge CPURegs:$lhs, CPURegs:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setge CPURegs:$lhs, CPURegs:$rhs)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (SLT CPURegs:$lhs, CPURegs:$rhs), RC:$F)>;
-  def : Pat<(select (setuge CPURegs:$lhs, CPURegs:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setuge CPURegs:$lhs, CPURegs:$rhs)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (SLTu CPURegs:$lhs, CPURegs:$rhs), RC:$F)>;
-  def : Pat<(select (setge CPURegs:$lhs, immSExt16:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setge CPURegs:$lhs, immSExt16:$rhs)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (SLTi CPURegs:$lhs, immSExt16:$rhs), RC:$F)>;
-  def : Pat<(select (setuge CPURegs:$lh, immSExt16:$rh), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setuge CPURegs:$lh, immSExt16:$rh)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (SLTiu CPURegs:$lh, immSExt16:$rh), RC:$F)>;
-  def : Pat<(select (setle CPURegs:$lhs, CPURegs:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setle CPURegs:$lhs, CPURegs:$rhs)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (SLT CPURegs:$rhs, CPURegs:$lhs), RC:$F)>;
-  def : Pat<(select (setule CPURegs:$lhs, CPURegs:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setule CPURegs:$lhs, CPURegs:$rhs)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (SLTu CPURegs:$rhs, CPURegs:$lhs), RC:$F)>;
-  def : Pat<(select (seteq CPURegs:$lhs, CPURegs:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (seteq CPURegs:$lhs, CPURegs:$rhs)), RC:$T, RC:$F),
             (MOVZInst RC:$T, (XOR CPURegs:$lhs, CPURegs:$rhs), RC:$F)>;
-  def : Pat<(select (seteq CPURegs:$lhs, 0), RC:$T, RC:$F),
+  def : Pat<(select (i32 (seteq CPURegs:$lhs, 0)), RC:$T, RC:$F),
             (MOVZInst RC:$T, CPURegs:$lhs, RC:$F)>;
 }
 
 multiclass MovnPats<RegisterClass RC, Instruction MOVNInst> {
-  def : Pat<(select (setne CPURegs:$lhs, CPURegs:$rhs), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setne CPURegs:$lhs, CPURegs:$rhs)), RC:$T, RC:$F),
             (MOVNInst RC:$T, (XOR CPURegs:$lhs, CPURegs:$rhs), RC:$F)>;
   def : Pat<(select CPURegs:$cond, RC:$T, RC:$F),
             (MOVNInst RC:$T, CPURegs:$cond, RC:$F)>;
-  def : Pat<(select (setne CPURegs:$lhs, 0), RC:$T, RC:$F),
+  def : Pat<(select (i32 (setne CPURegs:$lhs, 0)), RC:$T, RC:$F),
             (MOVNInst RC:$T, CPURegs:$lhs, RC:$F)>;
 }
 
@@ -845,30 +962,48 @@ defm : MovzPats<CPURegs, MOVZ_I>;
 defm : MovnPats<CPURegs, MOVN_I>;
 
 // setcc patterns
-def : Pat<(seteq CPURegs:$lhs, CPURegs:$rhs),
-          (SLTu (XOR CPURegs:$lhs, CPURegs:$rhs), 1)>;
-def : Pat<(setne CPURegs:$lhs, CPURegs:$rhs),
-          (SLTu ZERO, (XOR CPURegs:$lhs, CPURegs:$rhs))>;
+multiclass SeteqPats<RegisterClass RC, Instruction SLTiuOp, Instruction XOROp,
+                     Instruction SLTuOp, Register ZEROReg> {
+  def : Pat<(seteq RC:$lhs, RC:$rhs),
+            (SLTiuOp (XOROp RC:$lhs, RC:$rhs), 1)>;
+  def : Pat<(setne RC:$lhs, RC:$rhs),
+            (SLTuOp ZEROReg, (XOROp RC:$lhs, RC:$rhs))>;
+}
 
-def : Pat<(setle CPURegs:$lhs, CPURegs:$rhs),
-          (XORi (SLT CPURegs:$rhs, CPURegs:$lhs), 1)>;
-def : Pat<(setule CPURegs:$lhs, CPURegs:$rhs),
-          (XORi (SLTu CPURegs:$rhs, CPURegs:$lhs), 1)>;
+multiclass SetlePats<RegisterClass RC, Instruction SLTOp, Instruction SLTuOp> {
+  def : Pat<(setle RC:$lhs, RC:$rhs),
+            (XORi (SLTOp RC:$rhs, RC:$lhs), 1)>;
+  def : Pat<(setule RC:$lhs, RC:$rhs),
+            (XORi (SLTuOp RC:$rhs, RC:$lhs), 1)>;
+}
 
-def : Pat<(setgt CPURegs:$lhs, CPURegs:$rhs),
-          (SLT CPURegs:$rhs, CPURegs:$lhs)>;
-def : Pat<(setugt CPURegs:$lhs, CPURegs:$rhs),
-          (SLTu CPURegs:$rhs, CPURegs:$lhs)>;
+multiclass SetgtPats<RegisterClass RC, Instruction SLTOp, Instruction SLTuOp> {
+  def : Pat<(setgt RC:$lhs, RC:$rhs),
+            (SLTOp RC:$rhs, RC:$lhs)>;
+  def : Pat<(setugt RC:$lhs, RC:$rhs),
+            (SLTuOp RC:$rhs, RC:$lhs)>;
+}
 
-def : Pat<(setge CPURegs:$lhs, CPURegs:$rhs),
-          (XORi (SLT CPURegs:$lhs, CPURegs:$rhs), 1)>;
-def : Pat<(setuge CPURegs:$lhs, CPURegs:$rhs),
-          (XORi (SLTu CPURegs:$lhs, CPURegs:$rhs), 1)>;
+multiclass SetgePats<RegisterClass RC, Instruction SLTOp, Instruction SLTuOp> {
+  def : Pat<(setge RC:$lhs, RC:$rhs),
+            (XORi (SLTOp RC:$lhs, RC:$rhs), 1)>;
+  def : Pat<(setuge RC:$lhs, RC:$rhs),
+            (XORi (SLTuOp RC:$lhs, RC:$rhs), 1)>;
+}
 
-def : Pat<(setge CPURegs:$lhs, immSExt16:$rhs),
-          (XORi (SLTi CPURegs:$lhs, immSExt16:$rhs), 1)>;
-def : Pat<(setuge CPURegs:$lhs, immSExt16:$rhs),
-          (XORi (SLTiu CPURegs:$lhs, immSExt16:$rhs), 1)>;
+multiclass SetgeImmPats<RegisterClass RC, Instruction SLTiOp,
+                        Instruction SLTiuOp> {
+  def : Pat<(setge RC:$lhs, immSExt16:$rhs),
+            (XORi (SLTiOp RC:$lhs, immSExt16:$rhs), 1)>;
+  def : Pat<(setuge RC:$lhs, immSExt16:$rhs),
+            (XORi (SLTiuOp RC:$lhs, immSExt16:$rhs), 1)>;
+}
+
+defm : SeteqPats<CPURegs, SLTiu, XOR, SLTu, ZERO>;
+defm : SetlePats<CPURegs, SLT, SLTu>;
+defm : SetgtPats<CPURegs, SLT, SLTu>;
+defm : SetgePats<CPURegs, SLT, SLTu>;
+defm : SetgeImmPats<CPURegs, SLTi, SLTiu>;
 
 // select MipsDynAlloc
 def : Pat<(MipsDynAlloc addr:$f), (DynAlloc addr:$f)>;
@@ -878,4 +1013,5 @@ def : Pat<(MipsDynAlloc addr:$f), (DynAlloc addr:$f)>;
 //===----------------------------------------------------------------------===//
 
 include "MipsInstrFPU.td"
+include "Mips64InstrInfo.td"
 
diff --git a/contrib/llvm/lib/Target/Mips/MipsJITInfo.cpp b/contrib/llvm/lib/Target/Mips/MipsJITInfo.cpp
new file mode 100644
index 000000000000..28c2b48b2e19
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MipsJITInfo.cpp
@@ -0,0 +1,230 @@
+//===- MipsJITInfo.cpp - Implement the JIT interfaces for the Mips target -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the JIT interfaces for the Mips target.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "jit"
+#include "MipsJITInfo.h"
+#include "MipsInstrInfo.h"
+#include "MipsRelocations.h"
+#include "MipsSubtarget.h"
+#include "llvm/Function.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Memory.h"
+#include <cstdlib>
+using namespace llvm;
+
+
+void MipsJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
+  report_fatal_error("MipsJITInfo::replaceMachineCodeForFunction");
+}
+
+/// JITCompilerFunction - This contains the address of the JIT function used to
+/// compile a function lazily.
+static TargetJITInfo::JITCompilerFn JITCompilerFunction;
+
+// Get the ASMPREFIX for the current host.  This is often '_'.
+#ifndef __USER_LABEL_PREFIX__
+#define __USER_LABEL_PREFIX__
+#endif
+#define GETASMPREFIX2(X) #X
+#define GETASMPREFIX(X) GETASMPREFIX2(X)
+#define ASMPREFIX GETASMPREFIX(__USER_LABEL_PREFIX__)
+
+// CompilationCallback stub - We can't use a C function with inline assembly in
+// it, because the prolog/epilog inserted by GCC won't work for us. Instead,
+// write our own wrapper, which does things our way, so we have complete control
+// over register saving and restoring. This code saves registers, calls
+// MipsCompilationCallbackC and restores registers.
+extern "C" {
+#if defined (__mips__)
+void MipsCompilationCallback();
+
+  asm(
+    ".text\n"
+    ".align 2\n"
+    ".globl " ASMPREFIX "MipsCompilationCallback\n"
+    ASMPREFIX "MipsCompilationCallback:\n"
+    ".ent " ASMPREFIX "MipsCompilationCallback\n"
+    ".frame  $29, 32, $31\n"
+    ".set  noreorder\n"
+    ".cpload $t9\n"
+
+    "addiu $sp, $sp, -60\n"
+    ".cprestore 16\n"
+
+    // Save argument registers a0, a1, a2, a3, f12, f14 since they may contain
+    // stuff for the real target function right now. We have to act as if this
+    // whole compilation callback doesn't exist as far as the caller is
+    // concerned. We also need to save the ra register since it contains the
+    // original return address, and t8 register since it contains the address
+    // of the end of function stub.
+    "sw $a0, 20($sp)\n"
+    "sw $a1, 24($sp)\n"
+    "sw $a2, 28($sp)\n"
+    "sw $a3, 32($sp)\n"
+    "sw $ra, 36($sp)\n"
+    "sw $t8, 40($sp)\n"
+    "sdc1 $f12, 44($sp)\n"
+    "sdc1 $f14, 52($sp)\n"
+
+    // t8 points at the end of function stub. Pass the beginning of the stub
+    // to the MipsCompilationCallbackC.
+    "addiu $a0, $t8, -16\n"
+    "jal " ASMPREFIX "MipsCompilationCallbackC\n"
+    "nop\n"
+
+    // Restore registers.
+    "lw $a0, 20($sp)\n"
+    "lw $a1, 24($sp)\n"
+    "lw $a2, 28($sp)\n"
+    "lw $a3, 32($sp)\n"
+    "lw $ra, 36($sp)\n"
+    "lw $t8, 40($sp)\n"
+    "ldc1 $f12, 44($sp)\n"
+    "ldc1 $f14, 52($sp)\n"
+    "addiu $sp, $sp, 60\n"
+
+    // Jump to the (newly modified) stub to invoke the real function.
+    "addiu $t8, $t8, -16\n"
+    "jr $t8\n"
+    "nop\n"
+
+    ".set  reorder\n"
+    ".end " ASMPREFIX "MipsCompilationCallback\n"
+      );
+#else  // host != Mips
+  void MipsCompilationCallback() {
+    llvm_unreachable(
+      "Cannot call MipsCompilationCallback() on a non-Mips arch!");
+  }
+#endif
+}
+
+/// MipsCompilationCallbackC - This is the target-specific function invoked
+/// by the function stub when we did not know the real target of a call.
+/// This function must locate the start of the stub or call site and pass
+/// it into the JIT compiler function.
+extern "C" void MipsCompilationCallbackC(intptr_t StubAddr) {
+  // Get the address of the compiled code for this function.
+  intptr_t NewVal = (intptr_t) JITCompilerFunction((void*) StubAddr);
+
+  // Rewrite the function stub so that we don't end up here every time we
+  // execute the call. We're replacing the first four instructions of the
+  // stub with code that jumps to the compiled function:
+  //   lui $t9, %hi(NewVal)
+  //   addiu $t9, $t9, %lo(NewVal)
+  //   jr $t9
+  //   nop
+
+  int Hi = ((unsigned)NewVal & 0xffff0000) >> 16;
+  if ((NewVal & 0x8000) != 0)
+    Hi++;
+  int Lo = (int)(NewVal & 0xffff);
+
+  *(intptr_t *)(StubAddr) = 0xf << 26 | 25 << 16 | Hi;
+  *(intptr_t *)(StubAddr + 4) = 9 << 26 | 25 << 21 | 25 << 16 | Lo;
+  *(intptr_t *)(StubAddr + 8) = 25 << 21 | 8;
+  *(intptr_t *)(StubAddr + 12) = 0;
+
+  sys::Memory::InvalidateInstructionCache((void*) StubAddr, 16);
+}
+
+TargetJITInfo::LazyResolverFn MipsJITInfo::getLazyResolverFunction(
+    JITCompilerFn F) {
+  JITCompilerFunction = F;
+  return MipsCompilationCallback;
+}
+
+TargetJITInfo::StubLayout MipsJITInfo::getStubLayout() {
+  // The stub contains 4 4-byte instructions, aligned at 4 bytes. See
+  // emitFunctionStub for details.
+  StubLayout Result = { 4*4, 4 };
+  return Result;
+}
+
+void *MipsJITInfo::emitFunctionStub(const Function* F, void *Fn,
+    JITCodeEmitter &JCE) {
+  JCE.emitAlignment(4);
+  void *Addr = (void*) (JCE.getCurrentPCValue());
+  if (!sys::Memory::setRangeWritable(Addr, 16))
+    llvm_unreachable("ERROR: Unable to mark stub writable.");
+
+  intptr_t EmittedAddr;
+  if (Fn != (void*)(intptr_t)MipsCompilationCallback)
+    EmittedAddr = (intptr_t)Fn;
+  else
+    EmittedAddr = (intptr_t)MipsCompilationCallback;
+
+
+  int Hi = ((unsigned)EmittedAddr & 0xffff0000) >> 16;
+  if ((EmittedAddr & 0x8000) != 0)
+    Hi++;
+  int Lo = (int)(EmittedAddr & 0xffff);
+
+  // lui t9, %hi(EmittedAddr)
+  // addiu t9, t9, %lo(EmittedAddr)
+  // jalr t8, t9
+  // nop
+  JCE.emitWordLE(0xf << 26 | 25 << 16 | Hi);
+  JCE.emitWordLE(9 << 26 | 25 << 21 | 25 << 16 | Lo);
+  JCE.emitWordLE(25 << 21 | 24 << 11 | 9);
+  JCE.emitWordLE(0);
+
+  sys::Memory::InvalidateInstructionCache(Addr, 16);
+  if (!sys::Memory::setRangeExecutable(Addr, 16))
+    llvm_unreachable("ERROR: Unable to mark stub executable.");
+
+  return Addr;
+}
+
+/// relocate - Before the JIT can run a block of code that has been emitted,
+/// it must rewrite the code to contain the actual addresses of any
+/// referenced global symbols.
+void MipsJITInfo::relocate(void *Function, MachineRelocation *MR,
+    unsigned NumRelocs, unsigned char* GOTBase) {
+  for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
+
+    void *RelocPos = (char*) Function + MR->getMachineCodeOffset();
+    intptr_t ResultPtr = (intptr_t) MR->getResultPointer();
+
+    switch ((Mips::RelocationType) MR->getRelocationType()) {
+    case Mips::reloc_mips_branch:
+      ResultPtr = (((ResultPtr - (intptr_t) RelocPos) - 4) >> 2) & 0xffff;
+      *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+      break;
+
+    case Mips::reloc_mips_26:
+      ResultPtr = (ResultPtr & 0x0fffffff) >> 2;
+      *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+      break;
+
+    case Mips::reloc_mips_hi:
+      ResultPtr = ResultPtr >> 16;
+      if ((((intptr_t) (MR->getResultPointer()) & 0xffff) >> 15) == 1) {
+        ResultPtr += 1;
+      }
+      *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+      break;
+
+    case Mips::reloc_mips_lo:
+      ResultPtr = ResultPtr & 0xffff;
+      *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+      break;
+
+    default:
+      llvm_unreachable("ERROR: Unknown Mips relocation.");
+    }
+  }
+}
diff --git a/contrib/llvm/lib/Target/Mips/MipsJITInfo.h b/contrib/llvm/lib/Target/Mips/MipsJITInfo.h
new file mode 100644
index 000000000000..41f32a35f1b0
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MipsJITInfo.h
@@ -0,0 +1,70 @@
+//===- MipsJITInfo.h - Mips implementation of the JIT interface -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the MipsJITInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MIPSJITINFO_H
+#define MIPSJITINFO_H
+
+#include "MipsMachineFunction.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Target/TargetJITInfo.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+class MipsTargetMachine;
+
+class MipsJITInfo : public TargetJITInfo {
+
+  bool IsPIC;
+
+  public:
+    explicit MipsJITInfo() :
+      IsPIC(false) {}
+
+    /// replaceMachineCodeForFunction - Make it so that calling the function
+    /// whose machine code is at OLD turns into a call to NEW, perhaps by
+    /// overwriting OLD with a branch to NEW.  This is used for self-modifying
+    /// code.
+    ///
+    virtual void replaceMachineCodeForFunction(void *Old, void *New);
+
+    // getStubLayout - Returns the size and alignment of the largest call stub
+    // on Mips.
+    virtual StubLayout getStubLayout();
+
+    /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+    /// small native function that simply calls the function at the specified
+    /// address.
+    virtual void *emitFunctionStub(const Function* F, void *Fn,
+        JITCodeEmitter &JCE);
+
+    /// getLazyResolverFunction - Expose the lazy resolver to the JIT.
+    virtual LazyResolverFn getLazyResolverFunction(JITCompilerFn);
+
+    /// relocate - Before the JIT can run a block of code that has been emitted,
+    /// it must rewrite the code to contain the actual addresses of any
+    /// referenced global symbols.
+    virtual void relocate(void *Function, MachineRelocation *MR,
+        unsigned NumRelocs, unsigned char* GOTBase);
+
+    /// Initialize - Initialize internal stage for the function being JITted.
+    void Initialize(const MachineFunction &MF, bool isPIC) {
+      IsPIC = isPIC;
+    }
+
+};
+}
+
+#endif
diff --git a/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp b/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp
index f5cc3aa25f1b..608a7d21a4f9 100644
--- a/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp
@@ -29,10 +29,10 @@ MipsMCInstLower::MipsMCInstLower(Mangler *mang, const MachineFunction &mf,
   : Ctx(mf.getContext()), Mang(mang), AsmPrinter(asmprinter) {}
 
 MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
-                                              MachineOperandType MOTy) const {
+                                              MachineOperandType MOTy,
+                                              unsigned Offset) const {
   MipsMCSymbolRefExpr::VariantKind Kind;
   const MCSymbol *Symbol;
-  int Offset = 0;
 
   switch(MO.getTargetFlags()) {
   default:                  assert(0 && "Invalid target flag!");
@@ -46,6 +46,11 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   case MipsII::MO_GOTTPREL: Kind = MipsMCSymbolRefExpr::VK_Mips_GOTTPREL; break;
   case MipsII::MO_TPREL_HI: Kind = MipsMCSymbolRefExpr::VK_Mips_TPREL_HI; break;
   case MipsII::MO_TPREL_LO: Kind = MipsMCSymbolRefExpr::VK_Mips_TPREL_LO; break;
+  case MipsII::MO_GPOFF_HI: Kind = MipsMCSymbolRefExpr::VK_Mips_GPOFF_HI; break;
+  case MipsII::MO_GPOFF_LO: Kind = MipsMCSymbolRefExpr::VK_Mips_GPOFF_LO; break;
+  case MipsII::MO_GOT_DISP: Kind = MipsMCSymbolRefExpr::VK_Mips_GOT_DISP; break;
+  case MipsII::MO_GOT_PAGE: Kind = MipsMCSymbolRefExpr::VK_Mips_GOT_PAGE; break;
+  case MipsII::MO_GOT_OFST: Kind = MipsMCSymbolRefExpr::VK_Mips_GOT_OFST; break;
   }
 
   switch (MOTy) {
@@ -72,7 +77,7 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
     case MachineOperand::MO_ConstantPoolIndex:
       Symbol = AsmPrinter.GetCPISymbol(MO.getIndex());
       if (MO.getOffset())
-        Offset = MO.getOffset();  
+        Offset += MO.getOffset();  
       break;
 
     default:
@@ -83,36 +88,39 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
                                                            Ctx));
 }
 
+MCOperand MipsMCInstLower::LowerOperand(const MachineOperand& MO) const {
+  MachineOperandType MOTy = MO.getType();
+  
+  switch (MOTy) {
+  default:
+    assert(0 && "unknown operand type");
+    break;
+  case MachineOperand::MO_Register:
+    // Ignore all implicit register operands.
+    if (MO.isImplicit()) break;
+    return MCOperand::CreateReg(MO.getReg());
+  case MachineOperand::MO_Immediate:
+    return MCOperand::CreateImm(MO.getImm());
+  case MachineOperand::MO_MachineBasicBlock:
+  case MachineOperand::MO_GlobalAddress:
+  case MachineOperand::MO_ExternalSymbol:
+  case MachineOperand::MO_JumpTableIndex:
+  case MachineOperand::MO_ConstantPoolIndex:
+  case MachineOperand::MO_BlockAddress:
+    return LowerSymbolOperand(MO, MOTy, 0);
+ }
+
+  return MCOperand();
+}
+
 void MipsMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
   OutMI.setOpcode(MI->getOpcode());
   
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
-    MCOperand MCOp;
-    MachineOperandType MOTy = MO.getType();
+    MCOperand MCOp = LowerOperand(MO);
 
-    switch (MOTy) {
-    default:
-      MI->dump();
-      llvm_unreachable("unknown operand type");
-    case MachineOperand::MO_Register:
-      // Ignore all implicit register operands.
-      if (MO.isImplicit()) continue;
-      MCOp = MCOperand::CreateReg(MO.getReg());
-      break;
-    case MachineOperand::MO_Immediate:
-      MCOp = MCOperand::CreateImm(MO.getImm());
-      break;
-    case MachineOperand::MO_MachineBasicBlock:
-    case MachineOperand::MO_GlobalAddress:
-    case MachineOperand::MO_ExternalSymbol:
-    case MachineOperand::MO_JumpTableIndex:
-    case MachineOperand::MO_ConstantPoolIndex:
-    case MachineOperand::MO_BlockAddress:
-      MCOp = LowerSymbolOperand(MO, MOTy);
-      break;
-    }
-    
-    OutMI.addOperand(MCOp);
+    if (MCOp.isValid())
+      OutMI.addOperand(MCOp);
   }
 }
diff --git a/contrib/llvm/lib/Target/Mips/MipsMCInstLower.h b/contrib/llvm/lib/Target/Mips/MipsMCInstLower.h
index ec5201be7f6d..223f23aed286 100644
--- a/contrib/llvm/lib/Target/Mips/MipsMCInstLower.h
+++ b/contrib/llvm/lib/Target/Mips/MipsMCInstLower.h
@@ -1,4 +1,4 @@
-//===-- MipsMCInstLower.h - Lower MachineInstr to MCInst -------------------===//
+//===-- MipsMCInstLower.h - Lower MachineInstr to MCInst -------------------==//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -36,7 +36,8 @@ class LLVM_LIBRARY_VISIBILITY MipsMCInstLower {
   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
 private:
   MCOperand LowerSymbolOperand(const MachineOperand &MO,
-                               MachineOperandType MOTy) const;
+                               MachineOperandType MOTy, unsigned Offset) const;
+  MCOperand LowerOperand(const MachineOperand& MO) const;
 };
 }
 
diff --git a/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.cpp b/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.cpp
index 9a2bdae0e339..a0a242c8c443 100644
--- a/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.cpp
@@ -33,6 +33,11 @@ void MipsMCSymbolRefExpr::PrintImpl(raw_ostream &OS) const {
   case VK_Mips_GOTTPREL: OS << "%gottprel("; break;
   case VK_Mips_TPREL_HI: OS << "%tprel_hi("; break;
   case VK_Mips_TPREL_LO: OS << "%tprel_lo("; break;
+  case VK_Mips_GPOFF_HI: OS << "%hi(%neg(%gp_rel("; break;
+  case VK_Mips_GPOFF_LO: OS << "%lo(%neg(%gp_rel("; break;
+  case VK_Mips_GOT_DISP: OS << "%got_disp("; break;
+  case VK_Mips_GOT_PAGE: OS << "%got_page("; break;
+  case VK_Mips_GOT_OFST: OS << "%got_ofst("; break;
   }
 
   OS << *Symbol;
@@ -43,7 +48,9 @@ void MipsMCSymbolRefExpr::PrintImpl(raw_ostream &OS) const {
     OS << Offset;
   }
 
-  if (Kind != VK_Mips_None)
+  if (Kind == VK_Mips_GPOFF_HI || Kind == VK_Mips_GPOFF_LO)
+    OS << ")))";
+  else if (Kind != VK_Mips_None)
     OS << ')';
 }
 
diff --git a/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.h b/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.h
index 3e695963709e..55e85a79c1c8 100644
--- a/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.h
+++ b/contrib/llvm/lib/Target/Mips/MipsMCSymbolRefExpr.h
@@ -25,7 +25,12 @@ class MipsMCSymbolRefExpr : public MCTargetExpr {
     VK_Mips_TLSGD,
     VK_Mips_GOTTPREL,
     VK_Mips_TPREL_HI,
-    VK_Mips_TPREL_LO
+    VK_Mips_TPREL_LO,
+    VK_Mips_GPOFF_HI,
+    VK_Mips_GPOFF_LO,
+    VK_Mips_GOT_DISP,
+    VK_Mips_GOT_PAGE,
+    VK_Mips_GOT_OFST
   };
 
 private:
diff --git a/contrib/llvm/lib/Target/Mips/MipsMachineFunction.h b/contrib/llvm/lib/Target/Mips/MipsMachineFunction.h
index dbb7a6744224..bc30b6b2425b 100644
--- a/contrib/llvm/lib/Target/Mips/MipsMachineFunction.h
+++ b/contrib/llvm/lib/Target/Mips/MipsMachineFunction.h
@@ -51,16 +51,12 @@ class MipsFunctionInfo : public MachineFunctionInfo {
   mutable int DynAllocFI; // Frame index of dynamically allocated stack area.   
   unsigned MaxCallFrameSize;
 
-  /// AtomicFrameIndex - To implement atomic.swap and atomic.cmp.swap
-  /// intrinsics, it is necessary to use a temporary stack location.
-  /// This field holds the frame index of this location.
-  int AtomicFrameIndex;
 public:
   MipsFunctionInfo(MachineFunction& MF)
   : MF(MF), SRetReturnReg(0), GlobalBaseReg(0),
     VarArgsFrameIndex(0), InArgFIRange(std::make_pair(-1, 0)),
     OutArgFIRange(std::make_pair(-1, 0)), GPFI(0), DynAllocFI(0),
-    MaxCallFrameSize(0), AtomicFrameIndex(-1)
+    MaxCallFrameSize(0)
   {}
 
   bool isInArgFI(int FI) const {
@@ -104,9 +100,6 @@ class MipsFunctionInfo : public MachineFunctionInfo {
 
   unsigned getMaxCallFrameSize() const { return MaxCallFrameSize; }
   void setMaxCallFrameSize(unsigned S) { MaxCallFrameSize = S; }
-
-  int getAtomicFrameIndex() const { return AtomicFrameIndex; }
-  void setAtomicFrameIndex(int Index) { AtomicFrameIndex = Index; }
 };
 
 } // end of namespace llvm
diff --git a/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.cpp b/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.cpp
index 24390daff75c..f8c0fdac8cf0 100644
--- a/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.cpp
@@ -24,7 +24,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -44,7 +43,7 @@ using namespace llvm;
 
 MipsRegisterInfo::MipsRegisterInfo(const MipsSubtarget &ST,
                                    const TargetInstrInfo &tii)
-  : MipsGenRegisterInfo(), Subtarget(ST), TII(tii) {}
+  : MipsGenRegisterInfo(Mips::RA), Subtarget(ST), TII(tii) {}
 
 /// getRegisterNumbering - Given the enum value for some register, e.g.
 /// Mips::RA, return the number that it corresponds to (e.g. 31).
@@ -52,39 +51,87 @@ unsigned MipsRegisterInfo::
 getRegisterNumbering(unsigned RegEnum)
 {
   switch (RegEnum) {
-    case Mips::ZERO : case Mips::F0 : case Mips::D0 : return 0;
-    case Mips::AT   : case Mips::F1 : return 1;
-    case Mips::V0   : case Mips::F2 : case Mips::D1 : return 2;
-    case Mips::V1   : case Mips::F3 : return 3;
-    case Mips::A0   : case Mips::F4 : case Mips::D2 : return 4;
-    case Mips::A1   : case Mips::F5 : return 5;
-    case Mips::A2   : case Mips::F6 : case Mips::D3 : return 6;
-    case Mips::A3   : case Mips::F7 : return 7;
-    case Mips::T0   : case Mips::F8 : case Mips::D4 : return 8;
-    case Mips::T1   : case Mips::F9 : return 9;
-    case Mips::T2   : case Mips::F10: case Mips::D5: return 10;
-    case Mips::T3   : case Mips::F11: return 11;
-    case Mips::T4   : case Mips::F12: case Mips::D6: return 12;
-    case Mips::T5   : case Mips::F13: return 13;
-    case Mips::T6   : case Mips::F14: case Mips::D7: return 14;
-    case Mips::T7   : case Mips::F15: return 15;
-    case Mips::S0   : case Mips::F16: case Mips::D8: return 16;
-    case Mips::S1   : case Mips::F17: return 17;
-    case Mips::S2   : case Mips::F18: case Mips::D9: return 18;
-    case Mips::S3   : case Mips::F19: return 19;
-    case Mips::S4   : case Mips::F20: case Mips::D10: return 20;
-    case Mips::S5   : case Mips::F21: return 21;
-    case Mips::S6   : case Mips::F22: case Mips::D11: return 22;
-    case Mips::S7   : case Mips::F23: return 23;
-    case Mips::T8   : case Mips::F24: case Mips::D12: return 24;
-    case Mips::T9   : case Mips::F25: return 25;
-    case Mips::K0   : case Mips::F26: case Mips::D13: return 26;
-    case Mips::K1   : case Mips::F27: return 27;
-    case Mips::GP   : case Mips::F28: case Mips::D14: return 28;
-    case Mips::SP   : case Mips::F29: return 29;
-    case Mips::FP   : case Mips::F30: case Mips::D15: return 30;
-    case Mips::RA   : case Mips::F31: return 31;
-    default: llvm_unreachable("Unknown register number!");
+  case Mips::ZERO: case Mips::ZERO_64: case Mips::F0: case Mips::D0_64:
+  case Mips::D0:
+    return 0;
+  case Mips::AT: case Mips::AT_64: case Mips::F1: case Mips::D1_64:
+    return 1;
+  case Mips::V0: case Mips::V0_64: case Mips::F2: case Mips::D2_64:
+  case Mips::D1:
+    return 2;
+  case Mips::V1: case Mips::V1_64: case Mips::F3: case Mips::D3_64:
+    return 3;
+  case Mips::A0: case Mips::A0_64: case Mips::F4: case Mips::D4_64:
+  case Mips::D2:
+    return 4;
+  case Mips::A1: case Mips::A1_64: case Mips::F5: case Mips::D5_64:
+    return 5;
+  case Mips::A2: case Mips::A2_64: case Mips::F6: case Mips::D6_64:
+  case Mips::D3:
+    return 6;
+  case Mips::A3: case Mips::A3_64: case Mips::F7: case Mips::D7_64:
+    return 7;
+  case Mips::T0: case Mips::T0_64: case Mips::F8: case Mips::D8_64:
+  case Mips::D4:
+    return 8;
+  case Mips::T1: case Mips::T1_64: case Mips::F9: case Mips::D9_64:
+    return 9;
+  case Mips::T2: case Mips::T2_64: case Mips::F10: case Mips::D10_64:
+  case Mips::D5:
+    return 10;
+  case Mips::T3: case Mips::T3_64: case Mips::F11: case Mips::D11_64:
+    return 11;
+  case Mips::T4: case Mips::T4_64: case Mips::F12: case Mips::D12_64:
+  case Mips::D6:
+    return 12;
+  case Mips::T5: case Mips::T5_64: case Mips::F13: case Mips::D13_64:
+    return 13;
+  case Mips::T6: case Mips::T6_64: case Mips::F14: case Mips::D14_64:
+  case Mips::D7:
+    return 14;
+  case Mips::T7: case Mips::T7_64: case Mips::F15: case Mips::D15_64:
+    return 15;
+  case Mips::S0: case Mips::S0_64: case Mips::F16: case Mips::D16_64:
+  case Mips::D8:
+    return 16;
+  case Mips::S1: case Mips::S1_64: case Mips::F17: case Mips::D17_64:
+    return 17;
+  case Mips::S2: case Mips::S2_64: case Mips::F18: case Mips::D18_64:
+  case Mips::D9:
+    return 18;
+  case Mips::S3: case Mips::S3_64: case Mips::F19: case Mips::D19_64:
+    return 19;
+  case Mips::S4: case Mips::S4_64: case Mips::F20: case Mips::D20_64:
+  case Mips::D10:
+    return 20;
+  case Mips::S5: case Mips::S5_64: case Mips::F21: case Mips::D21_64:
+    return 21;
+  case Mips::S6: case Mips::S6_64: case Mips::F22: case Mips::D22_64:
+  case Mips::D11:
+    return 22;
+  case Mips::S7: case Mips::S7_64: case Mips::F23: case Mips::D23_64:
+    return 23;
+  case Mips::T8: case Mips::T8_64: case Mips::F24: case Mips::D24_64:
+  case Mips::D12:
+    return 24;
+  case Mips::T9: case Mips::T9_64: case Mips::F25: case Mips::D25_64:
+    return 25;
+  case Mips::K0: case Mips::K0_64: case Mips::F26: case Mips::D26_64:
+  case Mips::D13:
+    return 26;
+  case Mips::K1: case Mips::K1_64: case Mips::F27: case Mips::D27_64:
+    return 27;
+  case Mips::GP: case Mips::GP_64: case Mips::F28: case Mips::D28_64:
+  case Mips::D14:
+    return 28;
+  case Mips::SP: case Mips::SP_64: case Mips::F29: case Mips::D29_64:
+    return 29;
+  case Mips::FP: case Mips::FP_64: case Mips::F30: case Mips::D30_64:
+  case Mips::D15: 
+    return 30;
+  case Mips::RA: case Mips::RA_64: case Mips::F31: case Mips::D31_64:
+    return 31;
+  default: llvm_unreachable("Unknown register number!");
   }
   return 0; // Not reached
 }
@@ -101,7 +148,7 @@ getCalleeSavedRegs(const MachineFunction *MF) const
 {
   // Mips callee-save register range is $16-$23, $f20-$f30
   static const unsigned SingleFloatOnlyCalleeSavedRegs[] = {
-    Mips::F30, Mips::F29, Mips::F28, Mips::F27, Mips::F26,
+    Mips::F31, Mips::F30, Mips::F29, Mips::F28, Mips::F27, Mips::F26,
     Mips::F25, Mips::F24, Mips::F23, Mips::F22, Mips::F21, Mips::F20,
     Mips::RA, Mips::FP, Mips::S7, Mips::S6, Mips::S5, Mips::S4,
     Mips::S3, Mips::S2, Mips::S1, Mips::S0, 0
@@ -113,31 +160,71 @@ getCalleeSavedRegs(const MachineFunction *MF) const
     Mips::S3, Mips::S2, Mips::S1, Mips::S0, 0
   };
 
+  static const unsigned N32CalleeSavedRegs[] = {
+    Mips::D31_64, Mips::D29_64, Mips::D27_64, Mips::D25_64, Mips::D23_64,
+    Mips::D21_64,
+    Mips::RA_64, Mips::FP_64, Mips::GP_64, Mips::S7_64, Mips::S6_64,
+    Mips::S5_64, Mips::S4_64, Mips::S3_64, Mips::S2_64, Mips::S1_64,
+    Mips::S0_64, 0
+  };
+
+  static const unsigned N64CalleeSavedRegs[] = {
+    Mips::D31_64, Mips::D30_64, Mips::D29_64, Mips::D28_64, Mips::D27_64,
+    Mips::D26_64, Mips::D25_64, Mips::D24_64,
+    Mips::RA_64, Mips::FP_64, Mips::GP_64, Mips::S7_64, Mips::S6_64,
+    Mips::S5_64, Mips::S4_64, Mips::S3_64, Mips::S2_64, Mips::S1_64,
+    Mips::S0_64, 0
+  };
+
   if (Subtarget.isSingleFloat())
     return SingleFloatOnlyCalleeSavedRegs;
-  else
+  else if (!Subtarget.hasMips64())
     return Mips32CalleeSavedRegs;
+  else if (Subtarget.isABI_N32())
+    return N32CalleeSavedRegs;
+  
+  assert(Subtarget.isABI_N64());
+  return N64CalleeSavedRegs;  
 }
 
 BitVector MipsRegisterInfo::
 getReservedRegs(const MachineFunction &MF) const {
+  static const unsigned ReservedCPURegs[] = {
+    Mips::ZERO, Mips::AT, Mips::K0, Mips::K1, 
+    Mips::GP, Mips::SP, Mips::FP, Mips::RA, 0
+  };
+
+  static const unsigned ReservedCPU64Regs[] = {
+    Mips::ZERO_64, Mips::AT_64, Mips::K0_64, Mips::K1_64, 
+    Mips::GP_64, Mips::SP_64, Mips::FP_64, Mips::RA_64, 0
+  };
+
   BitVector Reserved(getNumRegs());
-  Reserved.set(Mips::ZERO);
-  Reserved.set(Mips::AT);
-  Reserved.set(Mips::K0);
-  Reserved.set(Mips::K1);
-  Reserved.set(Mips::GP);
-  Reserved.set(Mips::SP);
-  Reserved.set(Mips::FP);
-  Reserved.set(Mips::RA);
-  Reserved.set(Mips::F31);
-  Reserved.set(Mips::D15);
+  typedef TargetRegisterClass::iterator RegIter;
 
-  // SRV4 requires that odd register can't be used.
-  if (!Subtarget.isSingleFloat() && !Subtarget.isMips32())
-    for (unsigned FReg=(Mips::F0)+1; FReg < Mips::F30; FReg+=2)
-      Reserved.set(FReg);
+  for (const unsigned *Reg = ReservedCPURegs; *Reg; ++Reg)
+    Reserved.set(*Reg);
 
+  if (Subtarget.hasMips64()) {
+    for (const unsigned *Reg = ReservedCPU64Regs; *Reg; ++Reg)
+      Reserved.set(*Reg);
+
+    // Reserve all registers in AFGR64.
+    for (RegIter Reg = Mips::AFGR64RegisterClass->begin();
+         Reg != Mips::AFGR64RegisterClass->end(); ++Reg)
+      Reserved.set(*Reg);
+  }
+  else {
+    // Reserve all registers in CPU64Regs & FGR64.
+    for (RegIter Reg = Mips::CPU64RegsRegisterClass->begin();
+         Reg != Mips::CPU64RegsRegisterClass->end(); ++Reg)
+      Reserved.set(*Reg);
+
+    for (RegIter Reg = Mips::FGR64RegisterClass->begin();
+         Reg != Mips::FGR64RegisterClass->end(); ++Reg)
+      Reserved.set(*Reg);
+  }
+  
   return Reserved;
 }
 
@@ -244,11 +331,6 @@ eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
   MI.getOperand(i+1).ChangeToImmediate(Offset);
 }
 
-unsigned MipsRegisterInfo::
-getRARegister() const {
-  return Mips::RA;
-}
-
 unsigned MipsRegisterInfo::
 getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
@@ -267,12 +349,3 @@ getEHHandlerRegister() const {
   llvm_unreachable("What is the exception handler register");
   return 0;
 }
-
-int MipsRegisterInfo::
-getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  return MipsGenRegisterInfo::getDwarfRegNumFull(RegNum, 0);
-}
-
-int MipsRegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  return MipsGenRegisterInfo::getLLVMRegNumFull(DwarfRegNo,0);
-}
diff --git a/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.h b/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.h
index 646369b5966f..67e57dd71bdd 100644
--- a/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.h
+++ b/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.h
@@ -57,15 +57,11 @@ struct MipsRegisterInfo : public MipsGenRegisterInfo {
   void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
 
   /// Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   /// Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
-
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.td b/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.td
index f0db518b754b..925ad9e70ab6 100644
--- a/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.td
+++ b/contrib/llvm/lib/Target/Mips/MipsRegisterInfo.td
@@ -10,6 +10,11 @@
 //===----------------------------------------------------------------------===//
 //  Declarations that describe the MIPS register file
 //===----------------------------------------------------------------------===//
+let Namespace = "Mips" in {
+def sub_fpeven : SubRegIndex;
+def sub_fpodd  : SubRegIndex;
+def sub_32     : SubRegIndex;
+}
 
 // We have banks of 32 registers each.
 class MipsReg<string n> : Register<n> {
@@ -28,22 +33,31 @@ class MipsGPRReg<bits<5> num, string n> : MipsReg<n> {
   let Num = num;
 }
 
+// Mips 64-bit CPU Registers
+class Mips64GPRReg<bits<5> num, string n, list<Register> subregs>
+  : MipsRegWithSubRegs<n, subregs> {
+  let Num = num;
+  let SubRegIndices = [sub_32];
+}
+
 // Mips 32-bit FPU Registers
 class FPR<bits<5> num, string n> : MipsReg<n> {
   let Num = num;
 }
 
 // Mips 64-bit (aliased) FPU Registers
-let Namespace = "Mips" in {
-def sub_fpeven : SubRegIndex;
-def sub_fpodd  : SubRegIndex;
-}
 class AFPR<bits<5> num, string n, list<Register> subregs>
   : MipsRegWithSubRegs<n, subregs> {
   let Num = num;
   let SubRegIndices = [sub_fpeven, sub_fpodd];
 }
 
+class AFPR64<bits<5> num, string n, list<Register> subregs>
+  : MipsRegWithSubRegs<n, subregs> {
+  let Num = num;
+  let SubRegIndices = [sub_32];
+}
+
 // Mips Hardware Registers
 class HWR<bits<5> num, string n> : MipsReg<n> {
   let Num = num;
@@ -54,6 +68,7 @@ class HWR<bits<5> num, string n> : MipsReg<n> {
 //===----------------------------------------------------------------------===//
 
 let Namespace = "Mips" in {
+  // FIXME: Fix DwarfRegNum.
 
   // General Purpose Registers
   def ZERO : MipsGPRReg< 0, "ZERO">, DwarfRegNum<[0]>;
@@ -89,6 +104,40 @@ let Namespace = "Mips" in {
   def FP   : MipsGPRReg< 30, "FP">,  DwarfRegNum<[30]>;
   def RA   : MipsGPRReg< 31, "RA">,  DwarfRegNum<[31]>;
 
+  // General Purpose 64-bit Registers
+  def ZERO_64 : Mips64GPRReg< 0, "ZERO", [ZERO]>;
+  def AT_64   : Mips64GPRReg< 1, "AT",   [AT]>;
+  def V0_64   : Mips64GPRReg< 2, "2",    [V0]>;
+  def V1_64   : Mips64GPRReg< 3, "3",    [V1]>;
+  def A0_64   : Mips64GPRReg< 4, "4",    [A0]>;
+  def A1_64   : Mips64GPRReg< 5, "5",    [A1]>;
+  def A2_64   : Mips64GPRReg< 6, "6",    [A2]>;
+  def A3_64   : Mips64GPRReg< 7, "7",    [A3]>;
+  def T0_64   : Mips64GPRReg< 8, "8",    [T0]>;
+  def T1_64   : Mips64GPRReg< 9, "9",    [T1]>;
+  def T2_64   : Mips64GPRReg< 10, "10",  [T2]>;   
+  def T3_64   : Mips64GPRReg< 11, "11",  [T3]>;   
+  def T4_64   : Mips64GPRReg< 12, "12",  [T4]>;   
+  def T5_64   : Mips64GPRReg< 13, "13",  [T5]>;   
+  def T6_64   : Mips64GPRReg< 14, "14",  [T6]>;   
+  def T7_64   : Mips64GPRReg< 15, "15",  [T7]>;   
+  def S0_64   : Mips64GPRReg< 16, "16",  [S0]>;   
+  def S1_64   : Mips64GPRReg< 17, "17",  [S1]>;   
+  def S2_64   : Mips64GPRReg< 18, "18",  [S2]>;   
+  def S3_64   : Mips64GPRReg< 19, "19",  [S3]>;   
+  def S4_64   : Mips64GPRReg< 20, "20",  [S4]>;   
+  def S5_64   : Mips64GPRReg< 21, "21",  [S5]>;   
+  def S6_64   : Mips64GPRReg< 22, "22",  [S6]>;   
+  def S7_64   : Mips64GPRReg< 23, "23",  [S7]>;
+  def T8_64   : Mips64GPRReg< 24, "24",  [T8]>;
+  def T9_64   : Mips64GPRReg< 25, "25",  [T9]>;
+  def K0_64   : Mips64GPRReg< 26, "26",  [K0]>;
+  def K1_64   : Mips64GPRReg< 27, "27",  [K1]>;
+  def GP_64   : Mips64GPRReg< 28, "GP",  [GP]>;
+  def SP_64   : Mips64GPRReg< 29, "SP",  [SP]>;
+  def FP_64   : Mips64GPRReg< 30, "FP",  [FP]>;
+  def RA_64   : Mips64GPRReg< 31, "RA",  [RA]>;
+
   /// Mips Single point precision FPU Registers
   def F0  : FPR< 0,  "F0">, DwarfRegNum<[32]>;
   def F1  : FPR< 1,  "F1">, DwarfRegNum<[33]>;
@@ -142,10 +191,49 @@ let Namespace = "Mips" in {
   def D14 : AFPR<28, "F28", [F28, F29]>;
   def D15 : AFPR<30, "F30", [F30, F31]>;
 
+  /// Mips Double point precision FPU Registers in MFP64 mode.
+  def D0_64  : AFPR64<0, "F0", [F0]>;
+  def D1_64  : AFPR64<1, "F1", [F1]>;
+  def D2_64  : AFPR64<2, "F2", [F2]>;
+  def D3_64  : AFPR64<3, "F3", [F3]>;
+  def D4_64  : AFPR64<4, "F4", [F4]>;
+  def D5_64  : AFPR64<5, "F5", [F5]>;
+  def D6_64  : AFPR64<6, "F6", [F6]>;
+  def D7_64  : AFPR64<7, "F7", [F7]>;
+  def D8_64  : AFPR64<8, "F8", [F8]>;
+  def D9_64  : AFPR64<9, "F9", [F9]>;
+  def D10_64  : AFPR64<10, "F10", [F10]>;
+  def D11_64  : AFPR64<11, "F11", [F11]>;
+  def D12_64  : AFPR64<12, "F12", [F12]>;
+  def D13_64  : AFPR64<13, "F13", [F13]>;
+  def D14_64  : AFPR64<14, "F14", [F14]>;
+  def D15_64  : AFPR64<15, "F15", [F15]>;
+  def D16_64  : AFPR64<16, "F16", [F16]>;
+  def D17_64  : AFPR64<17, "F17", [F17]>;
+  def D18_64  : AFPR64<18, "F18", [F18]>;
+  def D19_64  : AFPR64<19, "F19", [F19]>;
+  def D20_64  : AFPR64<20, "F20", [F20]>;
+  def D21_64  : AFPR64<21, "F21", [F21]>;
+  def D22_64  : AFPR64<22, "F22", [F22]>;
+  def D23_64  : AFPR64<23, "F23", [F23]>;
+  def D24_64  : AFPR64<24, "F24", [F24]>;
+  def D25_64  : AFPR64<25, "F25", [F25]>;
+  def D26_64  : AFPR64<26, "F26", [F26]>;
+  def D27_64  : AFPR64<27, "F27", [F27]>;
+  def D28_64  : AFPR64<28, "F28", [F28]>;
+  def D29_64  : AFPR64<29, "F29", [F29]>;
+  def D30_64  : AFPR64<30, "F30", [F30]>;
+  def D31_64  : AFPR64<31, "F31", [F31]>;
+
   // Hi/Lo registers
   def HI  : Register<"hi">, DwarfRegNum<[64]>;
   def LO  : Register<"lo">, DwarfRegNum<[65]>;
 
+  let SubRegIndices = [sub_32] in {
+  def HI64  : RegisterWithSubRegs<"hi", [HI]>;
+  def LO64  : RegisterWithSubRegs<"lo", [LO]>;
+  }
+
   // Status flags register
   def FCR31 : Register<"31">;
 
@@ -167,6 +255,18 @@ def CPURegs : RegisterClass<"Mips", [i32], 32, (add
   // Reserved
   ZERO, AT, K0, K1, GP, SP, FP, RA)>;
 
+def CPU64Regs : RegisterClass<"Mips", [i64], 64, (add
+  // Return Values and Arguments
+  V0_64, V1_64, A0_64, A1_64, A2_64, A3_64,
+  // Not preserved across procedure calls
+  T0_64, T1_64, T2_64, T3_64, T4_64, T5_64, T6_64, T7_64, T8_64, T9_64,
+  // Callee save
+  S0_64, S1_64, S2_64, S3_64, S4_64, S5_64, S6_64, S7_64,
+  // Reserved
+  ZERO_64, AT_64, K0_64, K1_64, GP_64, SP_64, FP_64, RA_64)> {
+  let SubRegClasses = [(CPURegs sub_32)];
+}
+
 // 64bit fp:
 // * FGR64  - 32 64-bit registers
 // * AFGR64 - 16 32-bit even registers (32-bit FP Mode)
@@ -182,17 +282,22 @@ def AFGR64 : RegisterClass<"Mips", [f64], 64, (add
   // Not preserved across procedure calls
   D2, D3, D4, D5, D8, D9,
   // Callee save
-  D10, D11, D12, D13, D14,
-  // Reserved
-  D15)> {
+  D10, D11, D12, D13, D14, D15)> {
   let SubRegClasses = [(FGR32 sub_fpeven, sub_fpodd)];
 }
 
+def FGR64 : RegisterClass<"Mips", [f64], 64, (sequence "D%u_64", 0, 31)> {
+  let SubRegClasses = [(FGR32 sub_32)];
+}
+
 // Condition Register for floating point operations
 def CCR  : RegisterClass<"Mips", [i32], 32, (add FCR31)>;
 
 // Hi/Lo Registers
 def HILO : RegisterClass<"Mips", [i32], 32, (add HI, LO)>;
+def HILO64 : RegisterClass<"Mips", [i64], 64, (add HI64, LO64)> {
+  let SubRegClasses = [(HILO sub_32)];
+}
 
 // Hardware registers
 def HWRegs : RegisterClass<"Mips", [i32], 32, (add HWR29)>;
diff --git a/contrib/llvm/lib/Target/Mips/MipsRelocations.h b/contrib/llvm/lib/Target/Mips/MipsRelocations.h
new file mode 100644
index 000000000000..66d1bfd993f5
--- /dev/null
+++ b/contrib/llvm/lib/Target/Mips/MipsRelocations.h
@@ -0,0 +1,41 @@
+//===- MipsRelocations.h - Mips Code Relocations ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file defines the Mips target-specific relocation types
+// (for relocation-model=static).
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef MIPSRELOCATIONS_H_
+#define MIPSRELOCATIONS_H_
+
+#include "llvm/CodeGen/MachineRelocation.h"
+
+namespace llvm {
+  namespace Mips{
+    enum RelocationType {
+      // reloc_mips_branch - pc relative relocation for branches. The lower 18
+      // bits of the difference between the branch target and the branch
+      // instruction, shifted right by 2.
+      reloc_mips_branch = 1,
+
+      // reloc_mips_hi - upper 16 bits of the address (modified by +1 if the
+      // lower 16 bits of the address is negative).
+      reloc_mips_hi = 2,
+
+      // reloc_mips_lo - lower 16 bits of the address.
+      reloc_mips_lo = 3,
+
+      // reloc_mips_26 - lower 28 bits of the address, shifted right by 2.
+      reloc_mips_26 = 4
+    };
+  }
+}
+
+#endif /* MIPSRELOCATIONS_H_ */
diff --git a/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp b/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp
index 6eee3333d584..016d449a1067 100644
--- a/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp
@@ -13,7 +13,7 @@
 
 #include "MipsSubtarget.h"
 #include "Mips.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
@@ -24,15 +24,14 @@ using namespace llvm;
 MipsSubtarget::MipsSubtarget(const std::string &TT, const std::string &CPU,
                              const std::string &FS, bool little) :
   MipsGenSubtargetInfo(TT, CPU, FS),
-  MipsArchVersion(Mips1), MipsABI(O32), IsLittle(little), IsSingleFloat(false),
-  IsFP64bit(false), IsGP64bit(false), HasVFPU(false), IsLinux(true),
-  HasSEInReg(false), HasCondMov(false), HasMulDivAdd(false), HasMinMax(false),
-  HasSwap(false), HasBitCount(false)
+  MipsArchVersion(Mips32), MipsABI(UnknownABI), IsLittle(little), 
+  IsSingleFloat(false), IsFP64bit(false), IsGP64bit(false), HasVFPU(false),
+  IsLinux(true), HasSEInReg(false), HasCondMov(false), HasMulDivAdd(false),
+  HasMinMax(false), HasSwap(false), HasBitCount(false)
 {
   std::string CPUName = CPU;
   if (CPUName.empty())
-    CPUName = "mips1";
-  MipsArchVersion = Mips1;
+    CPUName = "mips32r1";
 
   // Parse features string.
   ParseSubtargetFeatures(CPUName, FS);
@@ -40,23 +39,16 @@ MipsSubtarget::MipsSubtarget(const std::string &TT, const std::string &CPU,
   // Initialize scheduling itinerary for the specified CPU.
   InstrItins = getInstrItineraryForCPU(CPUName);
 
+  // Set MipsABI if it hasn't been set yet.
+  if (MipsABI == UnknownABI)
+    MipsABI = hasMips64() ? N64 : O32; 
+
+  // Check if Architecture and ABI are compatible.
+  assert(((!hasMips64() && (isABI_O32() || isABI_EABI())) ||
+          (hasMips64() && (isABI_N32() || isABI_N64()))) &&
+         "Invalid  Arch & ABI pair.");
+
   // Is the target system Linux ?
   if (TT.find("linux") == std::string::npos)
     IsLinux = false;
-
-  // When only the target triple is specified and is
-  // a allegrex target, set the features. We also match
-  // big and little endian allegrex cores (dont really
-  // know if a big one exists)
-  if (TT.find("mipsallegrex") != std::string::npos ||
-      TT.find("psp") != std::string::npos) {
-    MipsABI = EABI;
-    IsSingleFloat = true;
-    MipsArchVersion = Mips2;
-    HasVFPU = true; // Enables Allegrex Vector FPU (not supported yet)
-    HasSEInReg = true;
-    HasBitCount = true;
-    HasSwap = true;
-    HasCondMov = true;
-  }
 }
diff --git a/contrib/llvm/lib/Target/Mips/MipsSubtarget.h b/contrib/llvm/lib/Target/Mips/MipsSubtarget.h
index 533d4afe073e..d9dddad23a48 100644
--- a/contrib/llvm/lib/Target/Mips/MipsSubtarget.h
+++ b/contrib/llvm/lib/Target/Mips/MipsSubtarget.h
@@ -27,14 +27,15 @@ class StringRef;
 class MipsSubtarget : public MipsGenSubtargetInfo {
 
 public:
+  // NOTE: O64 will not be supported.
   enum MipsABIEnum {
-    O32, O64, N32, N64, EABI
+    UnknownABI, O32, N32, N64, EABI
   };
 
 protected:
 
   enum MipsArchEnum {
-    Mips1, Mips2, Mips3, Mips4, Mips32, Mips32r2
+    Mips32, Mips32r2, Mips64, Mips64r2
   };
 
   // Mips architecture version
@@ -90,6 +91,8 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
 
   /// Only O32 and EABI supported right now.
   bool isABI_EABI() const { return MipsABI == EABI; }
+  bool isABI_N64() const { return MipsABI == N64; }
+  bool isABI_N32() const { return MipsABI == N32; }
   bool isABI_O32() const { return MipsABI == O32; }
   unsigned getTargetABI() const { return MipsABI; }
 
@@ -102,9 +105,11 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
 
-  bool isMips1() const { return MipsArchVersion == Mips1; }
-  bool isMips32() const { return MipsArchVersion >= Mips32; }
-  bool isMips32r2() const { return MipsArchVersion == Mips32r2; }
+  bool hasMips32() const { return MipsArchVersion >= Mips32; }
+  bool hasMips32r2() const { return MipsArchVersion == Mips32r2 ||
+                                   MipsArchVersion == Mips64r2; }
+  bool hasMips64() const { return MipsArchVersion >= Mips64; }
+  bool hasMips64r2() const { return MipsArchVersion == Mips64r2; }
 
   bool isLittle() const { return IsLittle; }
   bool isFP64bit() const { return IsFP64bit; }
diff --git a/contrib/llvm/lib/Target/Mips/MipsTargetMachine.cpp b/contrib/llvm/lib/Target/Mips/MipsTargetMachine.cpp
index 20b9f4ea3853..6480da3e6dfc 100644
--- a/contrib/llvm/lib/Target/Mips/MipsTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsTargetMachine.cpp
@@ -14,13 +14,15 @@
 #include "Mips.h"
 #include "MipsTargetMachine.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 extern "C" void LLVMInitializeMipsTarget() {
   // Register the target.
-  RegisterTargetMachine<MipsTargetMachine> X(TheMipsTarget);
+  RegisterTargetMachine<MipsebTargetMachine> X(TheMipsTarget);
   RegisterTargetMachine<MipselTargetMachine> Y(TheMipselTarget);
+  RegisterTargetMachine<Mips64ebTargetMachine> A(TheMips64Target);
+  RegisterTargetMachine<Mips64elTargetMachine> B(TheMips64elTarget);
 }
 
 // DataLayout --> Big-endian, 32-bit pointer/ABI/alignment
@@ -31,30 +33,47 @@ extern "C" void LLVMInitializeMipsTarget() {
 // an easier handling.
 // Using CodeModel::Large enables different CALL behavior.
 MipsTargetMachine::
-MipsTargetMachine(const Target &T, const std::string &TT,
-                  const std::string &CPU, const std::string &FS,
-                  bool isLittle=false):
-  LLVMTargetMachine(T, TT, CPU, FS),
+MipsTargetMachine(const Target &T, StringRef TT,
+                  StringRef CPU, StringRef FS,
+                  Reloc::Model RM, CodeModel::Model CM,
+                  bool isLittle):
+  LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
   Subtarget(TT, CPU, FS, isLittle),
-  DataLayout(isLittle ? 
-             std::string("e-p:32:32:32-i8:8:32-i16:16:32-i64:64:64-n32") :
-             std::string("E-p:32:32:32-i8:8:32-i16:16:32-i64:64:64-n32")),
+  DataLayout(isLittle ?
+             (Subtarget.isABI_N64() ?
+              "e-p:64:64:64-i8:8:32-i16:16:32-i64:64:64-f128:128:128-n32" :
+              "e-p:32:32:32-i8:8:32-i16:16:32-i64:64:64-n32") :
+             (Subtarget.isABI_N64() ?
+              "E-p:64:64:64-i8:8:32-i16:16:32-i64:64:64-f128:128:128-n32" :
+              "E-p:32:32:32-i8:8:32-i16:16:32-i64:64:64-n32")),
   InstrInfo(*this),
   FrameLowering(Subtarget),
-  TLInfo(*this), TSInfo(*this) {
-  // Abicall enables PIC by default
-  if (getRelocationModel() == Reloc::Default) {
-    if (Subtarget.isABI_O32())
-      setRelocationModel(Reloc::PIC_);
-    else
-      setRelocationModel(Reloc::Static);
-  }
+  TLInfo(*this), TSInfo(*this), JITInfo() {
 }
 
+MipsebTargetMachine::
+MipsebTargetMachine(const Target &T, StringRef TT,
+                    StringRef CPU, StringRef FS,
+                    Reloc::Model RM, CodeModel::Model CM) :
+  MipsTargetMachine(T, TT, CPU, FS, RM, CM, false) {}
+
 MipselTargetMachine::
-MipselTargetMachine(const Target &T, const std::string &TT,
-                    const std::string &CPU, const std::string &FS) :
-  MipsTargetMachine(T, TT, CPU, FS, true) {}
+MipselTargetMachine(const Target &T, StringRef TT,
+                    StringRef CPU, StringRef FS,
+                    Reloc::Model RM, CodeModel::Model CM) :
+  MipsTargetMachine(T, TT, CPU, FS, RM, CM, true) {}
+
+Mips64ebTargetMachine::
+Mips64ebTargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM) :
+  MipsTargetMachine(T, TT, CPU, FS, RM, CM, false) {}
+
+Mips64elTargetMachine::
+Mips64elTargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM) :
+  MipsTargetMachine(T, TT, CPU, FS, RM, CM, true) {}
 
 // Install an instruction selector pass using
 // the ISelDag to gen Mips code.
@@ -77,7 +96,10 @@ addPreEmitPass(PassManagerBase &PM, CodeGenOpt::Level OptLevel)
 
 bool MipsTargetMachine::
 addPreRegAlloc(PassManagerBase &PM, CodeGenOpt::Level OptLevel) {
-  PM.add(createMipsEmitGPRestorePass(*this));
+  // Do not restore $gp if target is Mips64.
+  // In N32/64, $gp is a callee-saved register.
+  if (!Subtarget.hasMips64())
+    PM.add(createMipsEmitGPRestorePass(*this));
   return true;
 }
 
@@ -86,3 +108,12 @@ addPostRegAlloc(PassManagerBase &PM, CodeGenOpt::Level OptLevel) {
   PM.add(createMipsExpandPseudoPass(*this));
   return true;
 }
+
+bool MipsTargetMachine::addCodeEmitter(PassManagerBase &PM,
+                                          CodeGenOpt::Level OptLevel,
+                                          JITCodeEmitter &JCE) {
+  // Machine code emitter pass for Mips.
+  PM.add(createMipsJITCodeEmitterPass(*this, JCE));
+  return false;
+}
+
diff --git a/contrib/llvm/lib/Target/Mips/MipsTargetMachine.h b/contrib/llvm/lib/Target/Mips/MipsTargetMachine.h
index a021af2ff16d..118ed107c514 100644
--- a/contrib/llvm/lib/Target/Mips/MipsTargetMachine.h
+++ b/contrib/llvm/lib/Target/Mips/MipsTargetMachine.h
@@ -22,6 +22,7 @@
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetFrameLowering.h"
+#include "MipsJITInfo.h"
 
 namespace llvm {
   class formatted_raw_ostream;
@@ -33,9 +34,12 @@ namespace llvm {
     MipsFrameLowering   FrameLowering;
     MipsTargetLowering  TLInfo;
     MipsSelectionDAGInfo TSInfo;
+    MipsJITInfo JITInfo;
+
   public:
-    MipsTargetMachine(const Target &T, const std::string &TT,
-                      const std::string &CPU, const std::string &FS,
+    MipsTargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM,
                       bool isLittle);
 
     virtual const MipsInstrInfo   *getInstrInfo()     const
@@ -46,6 +50,9 @@ namespace llvm {
     { return &Subtarget; }
     virtual const TargetData      *getTargetData()    const
     { return &DataLayout;}
+    virtual MipsJITInfo *getJITInfo()
+    { return &JITInfo; }
+
 
     virtual const MipsRegisterInfo *getRegisterInfo()  const {
       return &InstrInfo.getRegisterInfo();
@@ -67,16 +74,47 @@ namespace llvm {
     virtual bool addPreRegAlloc(PassManagerBase &PM,
                                 CodeGenOpt::Level OptLevel);
     virtual bool addPostRegAlloc(PassManagerBase &, CodeGenOpt::Level);
+    virtual bool addCodeEmitter(PassManagerBase &PM,
+				 CodeGenOpt::Level OptLevel,
+				 JITCodeEmitter &JCE);
+
   };
 
-/// MipselTargetMachine - Mipsel target machine.
+/// MipsebTargetMachine - Mips32 big endian target machine.
+///
+class MipsebTargetMachine : public MipsTargetMachine {
+public:
+  MipsebTargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM);
+};
+
+/// MipselTargetMachine - Mips32 little endian target machine.
 ///
 class MipselTargetMachine : public MipsTargetMachine {
 public:
-  MipselTargetMachine(const Target &T, const std::string &TT,
-                      const std::string &CPU, const std::string &FS);
+  MipselTargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM);
 };
 
+/// Mips64ebTargetMachine - Mips64 big endian target machine.
+///
+class Mips64ebTargetMachine : public MipsTargetMachine {
+public:
+  Mips64ebTargetMachine(const Target &T, StringRef TT,
+                        StringRef CPU, StringRef FS,
+                        Reloc::Model RM, CodeModel::Model CM);
+};
+
+/// Mips64elTargetMachine - Mips64 little endian target machine.
+///
+class Mips64elTargetMachine : public MipsTargetMachine {
+public:
+  Mips64elTargetMachine(const Target &T, StringRef TT,
+                        StringRef CPU, StringRef FS,
+                        Reloc::Model RM, CodeModel::Model CM);
+};
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp b/contrib/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp
index cf5d1b58addd..05c46f5c97a5 100644
--- a/contrib/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp
+++ b/contrib/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp
@@ -79,7 +79,7 @@ IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
   if (Kind.isMergeable1ByteCString())
     return false;
 
-  const Type *Ty = GV->getType()->getElementType();
+  Type *Ty = GV->getType()->getElementType();
   return IsInSmallSection(TM.getTargetData()->getTypeAllocSize(Ty));
 }
 
diff --git a/contrib/llvm/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp b/contrib/llvm/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
index a8d6fe94b1ad..243632b20aac 100644
--- a/contrib/llvm/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
@@ -9,13 +9,23 @@
 
 #include "Mips.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheMipsTarget, llvm::TheMipselTarget;
+Target llvm::TheMips64Target, llvm::TheMips64elTarget;
 
 extern "C" void LLVMInitializeMipsTargetInfo() {
-  RegisterTarget<Triple::mips> X(TheMipsTarget, "mips", "Mips");
+  RegisterTarget<Triple::mips,
+        /*HasJIT=*/true> X(TheMipsTarget, "mips", "Mips");
 
-  RegisterTarget<Triple::mipsel> Y(TheMipselTarget, "mipsel", "Mipsel");
+  RegisterTarget<Triple::mipsel,
+        /*HasJIT=*/true> Y(TheMipselTarget, "mipsel", "Mipsel");
+
+  RegisterTarget<Triple::mips64,
+        /*HasJIT=*/false> A(TheMips64Target, "mips64", "Mips64 [experimental]");
+
+  RegisterTarget<Triple::mips64el,
+        /*HasJIT=*/false> B(TheMips64elTarget,
+                            "mips64el", "Mips64el [experimental]");
 }
diff --git a/contrib/llvm/lib/Target/PTX/CMakeLists.txt b/contrib/llvm/lib/Target/PTX/CMakeLists.txt
deleted file mode 100644
index 331266da30b3..000000000000
--- a/contrib/llvm/lib/Target/PTX/CMakeLists.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-set(LLVM_TARGET_DEFINITIONS PTX.td)
-
-tablegen(PTXGenAsmWriter.inc -gen-asm-writer)
-tablegen(PTXGenDAGISel.inc -gen-dag-isel)
-tablegen(PTXGenInstrInfo.inc -gen-instr-desc)
-tablegen(PTXGenInstrNames.inc -gen-instr-enums)
-tablegen(PTXGenRegisterInfo.inc -gen-register-desc)
-tablegen(PTXGenRegisterInfo.h.inc -gen-register-desc-header)
-tablegen(PTXGenRegisterNames.inc -gen-register-enums)
-tablegen(PTXGenSubtarget.inc -gen-subtarget)
-
-add_llvm_target(PTXCodeGen
-  PTXAsmPrinter.cpp
-  PTXISelDAGToDAG.cpp
-  PTXISelLowering.cpp
-  PTXInstrInfo.cpp
-  PTXFrameLowering.cpp
-  PTXMCAsmInfo.cpp
-  PTXMCAsmStreamer.cpp
-  PTXMFInfoExtract.cpp
-  PTXRegisterInfo.cpp
-  PTXSubtarget.cpp
-  PTXTargetMachine.cpp
-  )
-
-add_subdirectory(TargetInfo)
diff --git a/contrib/llvm/lib/Target/PTX/InstPrinter/PTXInstPrinter.cpp b/contrib/llvm/lib/Target/PTX/InstPrinter/PTXInstPrinter.cpp
new file mode 100644
index 000000000000..aabb404dad68
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/InstPrinter/PTXInstPrinter.cpp
@@ -0,0 +1,192 @@
+//===-- PTXInstPrinter.cpp - Convert PTX MCInst to assembly syntax --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This class prints a PTX MCInst to a .ptx file.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "asm-printer"
+#include "PTXInstPrinter.h"
+#include "MCTargetDesc/PTXBaseInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+#define GET_INSTRUCTION_NAME
+#include "PTXGenAsmWriter.inc"
+
+PTXInstPrinter::PTXInstPrinter(const MCAsmInfo &MAI,
+                               const MCSubtargetInfo &STI) :
+  MCInstPrinter(MAI) {
+  // Initialize the set of available features.
+  setAvailableFeatures(STI.getFeatureBits());
+}
+
+StringRef PTXInstPrinter::getOpcodeName(unsigned Opcode) const {
+  return getInstructionName(Opcode);
+}
+
+void PTXInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
+  OS << getRegisterName(RegNo);
+}
+
+void PTXInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
+                               StringRef Annot) {
+  printPredicate(MI, O);
+  switch (MI->getOpcode()) {
+  default:
+    printInstruction(MI, O);
+    break;
+  case PTX::CALL:
+    printCall(MI, O);
+  }
+  O << ";";
+  printAnnotation(O, Annot);
+}
+
+void PTXInstPrinter::printPredicate(const MCInst *MI, raw_ostream &O) {
+  // The last two operands are the predicate operands
+  int RegIndex;
+  int OpIndex;
+
+  if (MI->getOpcode() == PTX::CALL) {
+    RegIndex = 0;
+    OpIndex  = 1;
+  } else {
+    RegIndex = MI->getNumOperands()-2;
+    OpIndex = MI->getNumOperands()-1;
+  }
+
+  int PredOp = MI->getOperand(OpIndex).getImm();
+  if (PredOp == PTXPredicate::None)
+    return;
+
+  if (PredOp == PTXPredicate::Negate)
+    O << '!';
+  else
+    O << '@';
+
+  printOperand(MI, RegIndex, O);
+}
+
+void PTXInstPrinter::printCall(const MCInst *MI, raw_ostream &O) {
+  O << "\tcall.uni\t";
+  // The first two operands are the predicate slot
+  unsigned Index = 2;
+  unsigned NumRets = MI->getOperand(Index++).getImm();
+
+  if (NumRets > 0) {
+    O << "(";
+    printOperand(MI, Index++, O);
+    for (unsigned i = 1; i < NumRets; ++i) {
+      O << ", ";
+      printOperand(MI, Index++, O);
+    }
+    O << "), ";
+  }
+
+  O << *(MI->getOperand(Index++).getExpr()) << ", (";
+
+  unsigned NumArgs = MI->getOperand(Index++).getImm();
+  if (NumArgs > 0) {
+    printOperand(MI, Index++, O);
+    for (unsigned i = 1; i < NumArgs; ++i) {
+      O << ", ";
+      printOperand(MI, Index++, O);
+    }
+  }
+  O << ")";
+}
+
+void PTXInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
+                                  raw_ostream &O) {
+  const MCOperand &Op = MI->getOperand(OpNo);
+  if (Op.isImm()) {
+    O << Op.getImm();
+  } else if (Op.isFPImm()) {
+    double Imm = Op.getFPImm();
+    APFloat FPImm(Imm);
+    APInt FPIntImm = FPImm.bitcastToAPInt();
+    O << "0D";
+    // PTX requires us to output the full 64 bits, even if the number is zero
+    if (FPIntImm.getZExtValue() > 0) {
+      O << FPIntImm.toString(16, false);
+    } else {
+      O << "0000000000000000";
+    }
+  } else {
+    assert(Op.isExpr() && "unknown operand kind in printOperand");
+    const MCExpr *Expr = Op.getExpr();
+    if (const MCSymbolRefExpr *SymRefExpr = dyn_cast<MCSymbolRefExpr>(Expr)) {
+      const MCSymbol &Sym = SymRefExpr->getSymbol();
+      O << Sym.getName();
+    } else {
+      O << *Op.getExpr();
+    }
+  }
+}
+
+void PTXInstPrinter::printMemOperand(const MCInst *MI, unsigned OpNo,
+                                     raw_ostream &O) {
+  // By definition, operand OpNo+1 is an i32imm
+  const MCOperand &Op2 = MI->getOperand(OpNo+1);
+  printOperand(MI, OpNo, O);
+  if (Op2.getImm() == 0)
+    return; // don't print "+0"
+  O << "+" << Op2.getImm();
+}
+
+void PTXInstPrinter::printRoundingMode(const MCInst *MI, unsigned OpNo,
+                                       raw_ostream &O) {
+  const MCOperand &Op = MI->getOperand(OpNo);
+  assert (Op.isImm() && "Rounding modes must be immediate values");
+  switch (Op.getImm()) {
+  default:
+    llvm_unreachable("Unknown rounding mode!");
+  case PTXRoundingMode::RndDefault:
+    llvm_unreachable("FP rounding-mode pass did not handle instruction!");
+    break;
+  case PTXRoundingMode::RndNone:
+    // Do not print anything.
+    break;
+  case PTXRoundingMode::RndNearestEven:
+    O << ".rn";
+    break;
+  case PTXRoundingMode::RndTowardsZero:
+    O << ".rz";
+    break;
+  case PTXRoundingMode::RndNegInf:
+    O << ".rm";
+    break;
+  case PTXRoundingMode::RndPosInf:
+    O << ".rp";
+    break;
+  case PTXRoundingMode::RndApprox:
+    O << ".approx";
+    break;
+  case PTXRoundingMode::RndNearestEvenInt:
+    O << ".rni";
+    break;
+  case PTXRoundingMode::RndTowardsZeroInt:
+    O << ".rzi";
+    break;
+  case PTXRoundingMode::RndNegInfInt:
+    O << ".rmi";
+    break;
+  case PTXRoundingMode::RndPosInfInt:
+    O << ".rpi";
+    break;
+  }
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/InstPrinter/PTXInstPrinter.h b/contrib/llvm/lib/Target/PTX/InstPrinter/PTXInstPrinter.h
new file mode 100644
index 000000000000..86dfd482885b
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/InstPrinter/PTXInstPrinter.h
@@ -0,0 +1,47 @@
+//===-- PTXInstPrinter.h - Convert PTX MCInst to assembly syntax ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This class prints n PTX MCInst to a .ptx file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PTXINSTPRINTER_H
+#define PTXINSTPRINTER_H
+
+#include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+
+namespace llvm {
+
+class MCOperand;
+
+class PTXInstPrinter : public MCInstPrinter {
+public:
+  PTXInstPrinter(const MCAsmInfo &MAI, const MCSubtargetInfo &STI);
+
+  virtual void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot);
+  virtual StringRef getOpcodeName(unsigned Opcode) const;
+  virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
+
+  static const char *getInstructionName(unsigned Opcode);
+
+  // Autogenerated by tblgen.
+  void printInstruction(const MCInst *MI, raw_ostream &O);
+  static const char *getRegisterName(unsigned RegNo);
+
+  void printPredicate(const MCInst *MI, raw_ostream &O);
+  void printCall(const MCInst *MI, raw_ostream &O);
+  void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printMemOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printRoundingMode(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+};
+}
+
+#endif
+
diff --git a/contrib/llvm/lib/Target/PTX/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/PTX/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index df0f63fdba60..000000000000
--- a/contrib/llvm/lib/Target/PTX/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMPTXDesc
-  PTXMCTargetDesc.cpp
-  PTXMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/PTX/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/PTX/MCTargetDesc/Makefile
deleted file mode 100644
index 35f5a7b2e6ad..000000000000
--- a/contrib/llvm/lib/Target/PTX/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/PTX/TargetDesc/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMPTXDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXBaseInfo.h b/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXBaseInfo.h
new file mode 100644
index 000000000000..c6094be4d15b
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXBaseInfo.h
@@ -0,0 +1,63 @@
+//===-- PTXBaseInfo.h - Top level definitions for PTX -------- --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains small standalone helper functions and enum definitions for
+// the PTX target useful for the compiler back-end and the MC libraries.
+// As such, it deliberately does not include references to LLVM core
+// code gen types, passes, etc..
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PTXBASEINFO_H
+#define PTXBASEINFO_H
+
+#include "PTXMCTargetDesc.h"
+
+namespace llvm {
+  namespace PTXStateSpace {
+    enum {
+      Global    = 0, // default to global state space
+      Constant  = 1,
+      Local     = 2,
+      Parameter = 3,
+      Shared    = 4
+    };
+  } // namespace PTXStateSpace
+
+  namespace PTXPredicate {
+    enum {
+      Normal = 0,
+      Negate = 1,
+      None   = 2
+    };
+  } // namespace PTXPredicate
+
+  /// Namespace to hold all target-specific flags.
+  namespace PTXRoundingMode {
+    // Instruction Flags
+    enum {
+      // Rounding Mode Flags
+      RndMask             = 15,
+      RndDefault          =  0, // ---
+      RndNone             =  1, // <NONE>
+      RndNearestEven      =  2, // .rn
+      RndTowardsZero      =  3, // .rz
+      RndNegInf           =  4, // .rm
+      RndPosInf           =  5, // .rp
+      RndApprox           =  6, // .approx
+      RndNearestEvenInt   =  7, // .rni
+      RndTowardsZeroInt   =  8, // .rzi
+      RndNegInfInt        =  9, // .rmi
+      RndPosInfInt        = 10  // .rpi
+    };
+  } // namespace PTXII
+} // namespace llvm
+
+#endif
+
diff --git a/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXMCTargetDesc.cpp b/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXMCTargetDesc.cpp
index 23f70bd13787..a5af3b880135 100644
--- a/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/PTX/MCTargetDesc/PTXMCTargetDesc.cpp
@@ -13,10 +13,12 @@
 
 #include "PTXMCTargetDesc.h"
 #include "PTXMCAsmInfo.h"
+#include "InstPrinter/PTXInstPrinter.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "PTXGenInstrInfo.inc"
@@ -35,9 +37,11 @@ static MCInstrInfo *createPTXMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializePTXMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(ThePTX32Target, createPTXMCInstrInfo);
-  TargetRegistry::RegisterMCInstrInfo(ThePTX64Target, createPTXMCInstrInfo);
+static MCRegisterInfo *createPTXMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  // PTX does not have a return address register.
+  InitPTXMCRegisterInfo(X, 0);
+  return X;
 }
 
 static MCSubtargetInfo *createPTXMCSubtargetInfo(StringRef TT, StringRef CPU,
@@ -47,14 +51,45 @@ static MCSubtargetInfo *createPTXMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializePTXMCSubtargetInfo() {
+static MCCodeGenInfo *createPTXMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                             CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+static MCInstPrinter *createPTXMCInstPrinter(const Target &T,
+                                             unsigned SyntaxVariant,
+                                             const MCAsmInfo &MAI,
+                                             const MCSubtargetInfo &STI) {
+  assert(SyntaxVariant == 0 && "We only have one syntax variant");
+  return new PTXInstPrinter(MAI, STI);
+}
+
+extern "C" void LLVMInitializePTXTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfo<PTXMCAsmInfo> X(ThePTX32Target);
+  RegisterMCAsmInfo<PTXMCAsmInfo> Y(ThePTX64Target);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(ThePTX32Target, createPTXMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(ThePTX64Target, createPTXMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(ThePTX32Target, createPTXMCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(ThePTX64Target, createPTXMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(ThePTX32Target, createPTXMCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(ThePTX64Target, createPTXMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(ThePTX32Target,
                                           createPTXMCSubtargetInfo);
   TargetRegistry::RegisterMCSubtargetInfo(ThePTX64Target,
                                           createPTXMCSubtargetInfo);
-}
 
-extern "C" void LLVMInitializePTXMCAsmInfo() {
-  RegisterMCAsmInfo<PTXMCAsmInfo> X(ThePTX32Target);
-  RegisterMCAsmInfo<PTXMCAsmInfo> Y(ThePTX64Target);
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(ThePTX32Target, createPTXMCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(ThePTX64Target, createPTXMCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/PTX/PTX.h b/contrib/llvm/lib/Target/PTX/PTX.h
index 28cab2429c81..7d46cce4aeca 100644
--- a/contrib/llvm/lib/Target/PTX/PTX.h
+++ b/contrib/llvm/lib/Target/PTX/PTX.h
@@ -15,34 +15,30 @@
 #ifndef PTX_H
 #define PTX_H
 
-#include "MCTargetDesc/PTXMCTargetDesc.h"
+#include "MCTargetDesc/PTXBaseInfo.h"
 #include "llvm/Target/TargetMachine.h"
 
 namespace llvm {
+  class MachineInstr;
+  class MCInst;
+  class PTXAsmPrinter;
   class PTXTargetMachine;
   class FunctionPass;
 
-  namespace PTX {
-    enum StateSpace {
-      GLOBAL = 0, // default to global state space
-      CONSTANT = 1,
-      LOCAL = 2,
-      PARAMETER = 3,
-      SHARED = 4
-    };
-
-    enum Predicate {
-      PRED_NORMAL = 0,
-      PRED_NEGATE = 1
-    };
-  } // namespace PTX
-
   FunctionPass *createPTXISelDag(PTXTargetMachine &TM,
                                  CodeGenOpt::Level OptLevel);
 
   FunctionPass *createPTXMFInfoExtract(PTXTargetMachine &TM,
                                        CodeGenOpt::Level OptLevel);
 
+  FunctionPass *createPTXFPRoundingModePass(PTXTargetMachine &TM,
+                                            CodeGenOpt::Level OptLevel);
+
+  FunctionPass *createPTXRegisterAllocator();
+
+  void LowerPTXMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
+                                    PTXAsmPrinter &AP);
+
 } // namespace llvm;
 
 #endif // PTX_H
diff --git a/contrib/llvm/lib/Target/PTX/PTX.td b/contrib/llvm/lib/Target/PTX/PTX.td
index f6fbe9fffc6f..693bb9c48344 100644
--- a/contrib/llvm/lib/Target/PTX/PTX.td
+++ b/contrib/llvm/lib/Target/PTX/PTX.td
@@ -52,13 +52,13 @@ def FeatureSM12 : SubtargetFeature<"sm12", "PTXTarget", "PTX_SM_1_2",
 def FeatureSM13 : SubtargetFeature<"sm13", "PTXTarget", "PTX_SM_1_3",
                                    "Use Shader Model 1.3">;
 def FeatureSM20 : SubtargetFeature<"sm20", "PTXTarget", "PTX_SM_2_0",
-                                   "Use Shader Model 2.0">;
+                                   "Use Shader Model 2.0", [FeatureDouble]>;
 def FeatureSM21 : SubtargetFeature<"sm21", "PTXTarget", "PTX_SM_2_1",
-                                   "Use Shader Model 2.1">;
+                                   "Use Shader Model 2.1", [FeatureDouble]>;
 def FeatureSM22 : SubtargetFeature<"sm22", "PTXTarget", "PTX_SM_2_2",
-                                   "Use Shader Model 2.2">;
+                                   "Use Shader Model 2.2", [FeatureDouble]>;
 def FeatureSM23 : SubtargetFeature<"sm23", "PTXTarget", "PTX_SM_2_3",
-                                   "Use Shader Model 2.3">;
+                                   "Use Shader Model 2.3", [FeatureDouble]>;
 
 def FeatureCOMPUTE10 : SubtargetFeature<"compute10", "PTXTarget",
                                         "PTX_COMPUTE_1_0",
@@ -74,7 +74,8 @@ def FeatureCOMPUTE13 : SubtargetFeature<"compute13", "PTXTarget",
                                         "Use Compute Compatibility 1.3">;
 def FeatureCOMPUTE20 : SubtargetFeature<"compute20", "PTXTarget",
                                         "PTX_COMPUTE_2_0",
-                                        "Use Compute Compatibility 2.0">;
+                                        "Use Compute Compatibility 2.0",
+                                        [FeatureDouble]>;
 
 //===----------------------------------------------------------------------===//
 // PTX supported processors
@@ -112,12 +113,6 @@ def : Proc<"fermi",      [FeatureSM20, FeatureDouble]>;
 
 include "PTXRegisterInfo.td"
 
-//===----------------------------------------------------------------------===//
-// Calling Conventions
-//===----------------------------------------------------------------------===//
-
-include "PTXCallingConv.td"
-
 //===----------------------------------------------------------------------===//
 // Instruction Descriptions
 //===----------------------------------------------------------------------===//
@@ -126,10 +121,21 @@ include "PTXInstrInfo.td"
 
 def PTXInstrInfo : InstrInfo;
 
+//===----------------------------------------------------------------------===//
+// Assembly printer
+//===----------------------------------------------------------------------===//
+// PTX uses the MC printer for asm output, so make sure the TableGen
+// AsmWriter bits get associated with the correct class.
+def PTXAsmWriter : AsmWriter {
+  string AsmWriterClassName  = "InstPrinter";
+  bit isMCAsmWriter = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Target Declaration
 //===----------------------------------------------------------------------===//
 
 def PTX : Target {
   let InstructionSet = PTXInstrInfo;
+  let AssemblyWriters = [PTXAsmWriter];
 }
diff --git a/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.cpp b/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.cpp
index 2848d5460eee..733744bbd08b 100644
--- a/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.cpp
@@ -15,9 +15,14 @@
 #define DEBUG_TYPE "ptx-asm-printer"
 
 #include "PTX.h"
+#include "PTXAsmPrinter.h"
 #include "PTXMachineFunctionInfo.h"
+#include "PTXParamManager.h"
+#include "PTXRegisterInfo.h"
 #include "PTXTargetMachine.h"
+#include "llvm/Argument.h"
 #include "llvm/DerivedTypes.h"
+#include "llvm/Function.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -28,69 +33,32 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
-namespace {
-class PTXAsmPrinter : public AsmPrinter {
-public:
-  explicit PTXAsmPrinter(TargetMachine &TM, MCStreamer &Streamer)
-    : AsmPrinter(TM, Streamer) {}
-
-  const char *getPassName() const { return "PTX Assembly Printer"; }
-
-  bool doFinalization(Module &M);
-
-  virtual void EmitStartOfAsmFile(Module &M);
-
-  virtual bool runOnMachineFunction(MachineFunction &MF);
-
-  virtual void EmitFunctionBodyStart();
-  virtual void EmitFunctionBodyEnd() { OutStreamer.EmitRawText(Twine("}")); }
-
-  virtual void EmitInstruction(const MachineInstr *MI);
-
-  void printOperand(const MachineInstr *MI, int opNum, raw_ostream &OS);
-  void printMemOperand(const MachineInstr *MI, int opNum, raw_ostream &OS,
-                       const char *Modifier = 0);
-  void printParamOperand(const MachineInstr *MI, int opNum, raw_ostream &OS,
-                         const char *Modifier = 0);
-  void printReturnOperand(const MachineInstr *MI, int opNum, raw_ostream &OS,
-                          const char *Modifier = 0); 
-  void printPredicateOperand(const MachineInstr *MI, raw_ostream &O);
-
-  unsigned GetOrCreateSourceID(StringRef FileName,
-                               StringRef DirName);
-
-  // autogen'd.
-  void printInstruction(const MachineInstr *MI, raw_ostream &OS);
-  static const char *getRegisterName(unsigned RegNo);
-
-private:
-  void EmitVariableDeclaration(const GlobalVariable *gv);
-  void EmitFunctionDeclaration();
-
-  StringMap<unsigned> SourceIdMap;
-}; // class PTXAsmPrinter
-} // namespace
-
 static const char PARAM_PREFIX[] = "__param_";
 static const char RETURN_PREFIX[] = "__ret_";
 
-static const char *getRegisterTypeName(unsigned RegNo) {
-#define TEST_REGCLS(cls, clsstr)                \
-  if (PTX::cls ## RegisterClass->contains(RegNo)) return # clsstr;
+static const char *getRegisterTypeName(unsigned RegNo,
+                                       const MachineRegisterInfo& MRI) {
+  const TargetRegisterClass *TRC = MRI.getRegClass(RegNo);
+
+#define TEST_REGCLS(cls, clsstr) \
+  if (PTX::cls ## RegisterClass == TRC) return # clsstr;
+
   TEST_REGCLS(RegPred, pred);
   TEST_REGCLS(RegI16, b16);
   TEST_REGCLS(RegI32, b32);
@@ -106,16 +74,16 @@ static const char *getRegisterTypeName(unsigned RegNo) {
 static const char *getStateSpaceName(unsigned addressSpace) {
   switch (addressSpace) {
   default: llvm_unreachable("Unknown state space");
-  case PTX::GLOBAL:    return "global";
-  case PTX::CONSTANT:  return "const";
-  case PTX::LOCAL:     return "local";
-  case PTX::PARAMETER: return "param";
-  case PTX::SHARED:    return "shared";
+  case PTXStateSpace::Global:    return "global";
+  case PTXStateSpace::Constant:  return "const";
+  case PTXStateSpace::Local:     return "local";
+  case PTXStateSpace::Parameter: return "param";
+  case PTXStateSpace::Shared:    return "shared";
   }
   return NULL;
 }
 
-static const char *getTypeName(const Type* type) {
+static const char *getTypeName(Type* type) {
   while (true) {
     switch (type->getTypeID()) {
       default: llvm_unreachable("Unknown type");
@@ -130,7 +98,7 @@ static const char *getTypeName(const Type* type) {
         }
       case Type::ArrayTyID:
       case Type::PointerTyID:
-        type = dyn_cast<const SequentialType>(type)->getElementType();
+        type = dyn_cast<SequentialType>(type)->getElementType();
         break;
     }
   }
@@ -170,6 +138,7 @@ void PTXAsmPrinter::EmitStartOfAsmFile(Module &M)
 {
   const PTXSubtarget& ST = TM.getSubtarget<PTXSubtarget>();
 
+  // Emit the PTX .version and .target attributes
   OutStreamer.EmitRawText(Twine("\t.version " + ST.getPTXVersionString()));
   OutStreamer.EmitRawText(Twine("\t.target " + ST.getTargetString() +
                                 (ST.supportsDouble() ? ""
@@ -203,177 +172,118 @@ void PTXAsmPrinter::EmitStartOfAsmFile(Module &M)
     EmitVariableDeclaration(i);
 }
 
-bool PTXAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
-  SetupMachineFunction(MF);
-  EmitFunctionDeclaration();
-  EmitFunctionBody();
-  return false;
-}
-
 void PTXAsmPrinter::EmitFunctionBodyStart() {
   OutStreamer.EmitRawText(Twine("{"));
 
   const PTXMachineFunctionInfo *MFI = MF->getInfo<PTXMachineFunctionInfo>();
+  const PTXParamManager &PM = MFI->getParamManager();
 
-  // Print local variable definition
-  for (PTXMachineFunctionInfo::reg_iterator
-       i = MFI->localVarRegBegin(), e = MFI->localVarRegEnd(); i != e; ++ i) {
-    unsigned reg = *i;
+  // Print register definitions
+  std::string regDefs;
+  unsigned numRegs;
 
-    std::string def = "\t.reg .";
-    def += getRegisterTypeName(reg);
-    def += ' ';
-    def += getRegisterName(reg);
-    def += ';';
-    OutStreamer.EmitRawText(Twine(def));
+  // pred
+  numRegs = MFI->getNumRegistersForClass(PTX::RegPredRegisterClass);
+  if(numRegs > 0) {
+    regDefs += "\t.reg .pred %p<";
+    regDefs += utostr(numRegs);
+    regDefs += ">;\n";
   }
 
+  // i16
+  numRegs = MFI->getNumRegistersForClass(PTX::RegI16RegisterClass);
+  if(numRegs > 0) {
+    regDefs += "\t.reg .b16 %rh<";
+    regDefs += utostr(numRegs);
+    regDefs += ">;\n";
+  }
+
+  // i32
+  numRegs = MFI->getNumRegistersForClass(PTX::RegI32RegisterClass);
+  if(numRegs > 0) {
+    regDefs += "\t.reg .b32 %r<";
+    regDefs += utostr(numRegs);
+    regDefs += ">;\n";
+  }
+
+  // i64
+  numRegs = MFI->getNumRegistersForClass(PTX::RegI64RegisterClass);
+  if(numRegs > 0) {
+    regDefs += "\t.reg .b64 %rd<";
+    regDefs += utostr(numRegs);
+    regDefs += ">;\n";
+  }
+
+  // f32
+  numRegs = MFI->getNumRegistersForClass(PTX::RegF32RegisterClass);
+  if(numRegs > 0) {
+    regDefs += "\t.reg .f32 %f<";
+    regDefs += utostr(numRegs);
+    regDefs += ">;\n";
+  }
+
+  // f64
+  numRegs = MFI->getNumRegistersForClass(PTX::RegF64RegisterClass);
+  if(numRegs > 0) {
+    regDefs += "\t.reg .f64 %fd<";
+    regDefs += utostr(numRegs);
+    regDefs += ">;\n";
+  }
+
+  // Local params
+  for (PTXParamManager::param_iterator i = PM.local_begin(), e = PM.local_end();
+       i != e; ++i) {
+    regDefs += "\t.param .b";
+    regDefs += utostr(PM.getParamSize(*i));
+    regDefs += " ";
+    regDefs += PM.getParamName(*i);
+    regDefs += ";\n";
+  }
+
+  OutStreamer.EmitRawText(Twine(regDefs));
+
+
   const MachineFrameInfo* FrameInfo = MF->getFrameInfo();
   DEBUG(dbgs() << "Have " << FrameInfo->getNumObjects()
                << " frame object(s)\n");
   for (unsigned i = 0, e = FrameInfo->getNumObjects(); i != e; ++i) {
     DEBUG(dbgs() << "Size of object: " << FrameInfo->getObjectSize(i) << "\n");
     if (FrameInfo->getObjectSize(i) > 0) {
-      std::string def = "\t.reg .b";
-      def += utostr(FrameInfo->getObjectSize(i)*8); // Convert to bits
-      def += " s";
+      std::string def = "\t.local .align ";
+      def += utostr(FrameInfo->getObjectAlignment(i));
+      def += " .b8";
+      def += " __local";
       def += utostr(i);
+      def += "[";
+      def += utostr(FrameInfo->getObjectSize(i)); // Convert to bits
+      def += "]";
       def += ";";
       OutStreamer.EmitRawText(Twine(def));
     }
   }
+
+  //unsigned Index = 1;
+  // Print parameter passing params
+  //for (PTXMachineFunctionInfo::param_iterator
+  //     i = MFI->paramBegin(), e = MFI->paramEnd(); i != e; ++i) {
+  //  std::string def = "\t.param .b";
+  //  def += utostr(*i);
+  //  def += " __ret_";
+  //  def += utostr(Index);
+  //  Index++;
+  //  def += ";";
+  //  OutStreamer.EmitRawText(Twine(def));
+  //}
+}
+
+void PTXAsmPrinter::EmitFunctionBodyEnd() {
+  OutStreamer.EmitRawText(Twine("}"));
 }
 
 void PTXAsmPrinter::EmitInstruction(const MachineInstr *MI) {
-  std::string str;
-  str.reserve(64);
-
-  raw_string_ostream OS(str);
-
-  DebugLoc DL = MI->getDebugLoc();
-  if (!DL.isUnknown()) {
-
-    const MDNode *S = DL.getScope(MF->getFunction()->getContext());
-
-    // This is taken from DwarfDebug.cpp, which is conveniently not a public
-    // LLVM class.
-    StringRef Fn;
-    StringRef Dir;
-    unsigned Src = 1;
-    if (S) {
-      DIDescriptor Scope(S);
-      if (Scope.isCompileUnit()) {
-        DICompileUnit CU(S);
-        Fn = CU.getFilename();
-        Dir = CU.getDirectory();
-      } else if (Scope.isFile()) {
-        DIFile F(S);
-        Fn = F.getFilename();
-        Dir = F.getDirectory();
-      } else if (Scope.isSubprogram()) {
-        DISubprogram SP(S);
-        Fn = SP.getFilename();
-        Dir = SP.getDirectory();
-      } else if (Scope.isLexicalBlock()) {
-        DILexicalBlock DB(S);
-        Fn = DB.getFilename();
-        Dir = DB.getDirectory();
-      } else
-        assert(0 && "Unexpected scope info");
-
-      Src = GetOrCreateSourceID(Fn, Dir);
-    }
-    OutStreamer.EmitDwarfLocDirective(Src, DL.getLine(), DL.getCol(),
-                                     0, 0, 0, Fn);
-
-    const MCDwarfLoc& MDL = OutContext.getCurrentDwarfLoc();
-
-    OS << "\t.loc ";
-    OS << utostr(MDL.getFileNum());
-    OS << " ";
-    OS << utostr(MDL.getLine());
-    OS << " ";
-    OS << utostr(MDL.getColumn());
-    OS << "\n";
-  }
-
-
-  // Emit predicate
-  printPredicateOperand(MI, OS);
-
-  // Write instruction to str
-  printInstruction(MI, OS);
-  OS << ';';
-  OS.flush();
-
-  StringRef strref = StringRef(str);
-  OutStreamer.EmitRawText(strref);
-}
-
-void PTXAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
-                                 raw_ostream &OS) {
-  const MachineOperand &MO = MI->getOperand(opNum);
-
-  switch (MO.getType()) {
-    default:
-      llvm_unreachable("<unknown operand type>");
-      break;
-    case MachineOperand::MO_GlobalAddress:
-      OS << *Mang->getSymbol(MO.getGlobal());
-      break;
-    case MachineOperand::MO_Immediate:
-      OS << (long) MO.getImm();
-      break;
-    case MachineOperand::MO_MachineBasicBlock:
-      OS << *MO.getMBB()->getSymbol();
-      break;
-    case MachineOperand::MO_Register:
-      OS << getRegisterName(MO.getReg());
-      break;
-    case MachineOperand::MO_FPImmediate:
-      APInt constFP = MO.getFPImm()->getValueAPF().bitcastToAPInt();
-      bool  isFloat = MO.getFPImm()->getType()->getTypeID() == Type::FloatTyID;
-      // Emit 0F for 32-bit floats and 0D for 64-bit doubles.
-      if (isFloat) {
-        OS << "0F";
-      }
-      else {
-        OS << "0D";
-      }
-      // Emit the encoded floating-point value.
-      if (constFP.getZExtValue() > 0) {
-        OS << constFP.toString(16, false);
-      }
-      else {
-        OS << "00000000";
-        // If We have a double-precision zero, pad to 8-bytes.
-        if (!isFloat) {
-          OS << "00000000";
-        }
-      }
-      break;
-  }
-}
-
-void PTXAsmPrinter::printMemOperand(const MachineInstr *MI, int opNum,
-                                    raw_ostream &OS, const char *Modifier) {
-  printOperand(MI, opNum, OS);
-
-  if (MI->getOperand(opNum+1).isImm() && MI->getOperand(opNum+1).getImm() == 0)
-    return; // don't print "+0"
-
-  OS << "+";
-  printOperand(MI, opNum+1, OS);
-}
-
-void PTXAsmPrinter::printParamOperand(const MachineInstr *MI, int opNum,
-                                      raw_ostream &OS, const char *Modifier) {
-  OS << PARAM_PREFIX << (int) MI->getOperand(opNum).getImm() + 1;
-}
-
-void PTXAsmPrinter::printReturnOperand(const MachineInstr *MI, int opNum,
-                                       raw_ostream &OS, const char *Modifier) {
-  OS << RETURN_PREFIX << (int) MI->getOperand(opNum).getImm() + 1;
+  MCInst TmpInst;
+  LowerPTXMachineInstrToMCInst(MI, TmpInst, *this);
+  OutStreamer.EmitInstruction(TmpInst);
 }
 
 void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
@@ -400,14 +310,14 @@ void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
   unsigned alignment = gv->getAlignment();
   if (alignment != 0) {
     decl += ".align ";
-    decl += utostr(Log2_32(gv->getAlignment()));
+    decl += utostr(gv->getAlignment());
     decl += " ";
   }
 
 
   if (PointerType::classof(gv->getType())) {
-    const PointerType* pointerTy = dyn_cast<const PointerType>(gv->getType());
-    const Type* elementTy = pointerTy->getElementType();
+    PointerType* pointerTy = dyn_cast<PointerType>(gv->getType());
+    Type* elementTy = pointerTy->getElementType();
 
     decl += ".b8 ";
     decl += gvsym->getName();
@@ -417,14 +327,14 @@ void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
     {
       assert(elementTy->isArrayTy() && "Only pointers to arrays are supported");
 
-      const ArrayType* arrayTy = dyn_cast<const ArrayType>(elementTy);
+      ArrayType* arrayTy = dyn_cast<ArrayType>(elementTy);
       elementTy = arrayTy->getElementType();
 
       unsigned numElements = arrayTy->getNumElements();
 
       while (elementTy->isArrayTy()) {
 
-        arrayTy = dyn_cast<const ArrayType>(elementTy);
+        arrayTy = dyn_cast<ArrayType>(elementTy);
         elementTy = arrayTy->getElementType();
 
         numElements *= arrayTy->getNumElements();
@@ -447,7 +357,7 @@ void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
 
     if (gv->hasInitializer())
     {
-      const Constant *C = gv->getInitializer();  
+      const Constant *C = gv->getInitializer();
       if (const ConstantArray *CA = dyn_cast<ConstantArray>(C))
       {
         decl += " = {";
@@ -484,7 +394,7 @@ void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
   OutStreamer.AddBlankLine();
 }
 
-void PTXAsmPrinter::EmitFunctionDeclaration() {
+void PTXAsmPrinter::EmitFunctionEntryLabel() {
   // The function label could have already been emitted if two symbols end up
   // conflicting due to asm renaming.  Detect this and emit an error.
   if (!CurrentFnSym->isUndefined()) {
@@ -494,25 +404,39 @@ void PTXAsmPrinter::EmitFunctionDeclaration() {
   }
 
   const PTXMachineFunctionInfo *MFI = MF->getInfo<PTXMachineFunctionInfo>();
+  const PTXParamManager &PM = MFI->getParamManager();
   const bool isKernel = MFI->isKernel();
   const PTXSubtarget& ST = TM.getSubtarget<PTXSubtarget>();
+  const MachineRegisterInfo& MRI = MF->getRegInfo();
 
   std::string decl = isKernel ? ".entry" : ".func";
 
-  unsigned cnt = 0;
-
   if (!isKernel) {
     decl += " (";
-    for (PTXMachineFunctionInfo::ret_iterator
-         i = MFI->retRegBegin(), e = MFI->retRegEnd(), b = i;
-         i != e; ++i) {
-      if (i != b) {
-        decl += ", ";
+    if (ST.useParamSpaceForDeviceArgs()) {
+      for (PTXParamManager::param_iterator i = PM.ret_begin(), e = PM.ret_end(),
+           b = i; i != e; ++i) {
+        if (i != b) {
+          decl += ", ";
+        }
+
+        decl += ".param .b";
+        decl += utostr(PM.getParamSize(*i));
+        decl += " ";
+        decl += PM.getParamName(*i);
+      }
+    } else {
+      for (PTXMachineFunctionInfo::reg_iterator
+           i = MFI->retreg_begin(), e = MFI->retreg_end(), b = i;
+           i != e; ++i) {
+        if (i != b) {
+          decl += ", ";
+        }
+        decl += ".reg .";
+        decl += getRegisterTypeName(*i, MRI);
+        decl += " ";
+        decl += MFI->getRegisterName(*i);
       }
-      decl += ".reg .";
-      decl += getRegisterTypeName(*i);
-      decl += " ";
-      decl += getRegisterName(*i);
     }
     decl += ")";
   }
@@ -523,26 +447,65 @@ void PTXAsmPrinter::EmitFunctionDeclaration() {
 
   decl += " (";
 
-  cnt = 0;
+  const Function *F = MF->getFunction();
 
   // Print parameters
-  for (PTXMachineFunctionInfo::reg_iterator
-       i = MFI->argRegBegin(), e = MFI->argRegEnd(), b = i;
-       i != e; ++i) {
-    if (i != b) {
-      decl += ", ";
-    }
-    if (isKernel || ST.useParamSpaceForDeviceArgs()) {
+  if (isKernel || ST.useParamSpaceForDeviceArgs()) {
+    /*for (PTXParamManager::param_iterator i = PM.arg_begin(), e = PM.arg_end(),
+         b = i; i != e; ++i) {
+      if (i != b) {
+        decl += ", ";
+      }
+
       decl += ".param .b";
-      decl += utostr(*i);
+      decl += utostr(PM.getParamSize(*i));
       decl += " ";
-      decl += PARAM_PREFIX;
-      decl += utostr(++cnt);
-    } else {
+      decl += PM.getParamName(*i);
+    }*/
+    int Counter = 1;
+    for (Function::const_arg_iterator i = F->arg_begin(), e = F->arg_end(),
+         b = i; i != e; ++i) {
+      if (i != b)
+        decl += ", ";
+      const Type *ArgType = (*i).getType();
+      decl += ".param .b";
+      if (ArgType->isPointerTy()) {
+        if (ST.is64Bit())
+          decl += "64";
+        else
+          decl += "32";
+      } else {
+        decl += utostr(ArgType->getPrimitiveSizeInBits());
+      }
+      if (ArgType->isPointerTy() && ST.emitPtrAttribute()) {
+        const PointerType *PtrType = dyn_cast<const PointerType>(ArgType);
+        decl += " .ptr";
+        switch (PtrType->getAddressSpace()) {
+        default:
+          llvm_unreachable("Unknown address space in argument");
+        case PTXStateSpace::Global:
+          decl += " .global";
+          break;
+        case PTXStateSpace::Shared:
+          decl += " .shared";
+          break;
+        }
+      }
+      decl += " __param_";
+      decl += utostr(Counter++);
+    }
+  } else {
+    for (PTXMachineFunctionInfo::reg_iterator
+         i = MFI->argreg_begin(), e = MFI->argreg_end(), b = i;
+         i != e; ++i) {
+      if (i != b) {
+        decl += ", ";
+      }
+
       decl += ".reg .";
-      decl += getRegisterTypeName(*i);
+      decl += getRegisterTypeName(*i, MRI);
       decl += " ";
-      decl += getRegisterName(*i);
+      decl += MFI->getRegisterName(*i);
     }
   }
   decl += ")";
@@ -550,25 +513,6 @@ void PTXAsmPrinter::EmitFunctionDeclaration() {
   OutStreamer.EmitRawText(Twine(decl));
 }
 
-void PTXAsmPrinter::
-printPredicateOperand(const MachineInstr *MI, raw_ostream &O) {
-  int i = MI->findFirstPredOperandIdx();
-  if (i == -1)
-    llvm_unreachable("missing predicate operand");
-
-  unsigned reg = MI->getOperand(i).getReg();
-  int predOp = MI->getOperand(i+1).getImm();
-
-  DEBUG(dbgs() << "predicate: (" << reg << ", " << predOp << ")\n");
-
-  if (reg != PTX::NoRegister) {
-    O << '@';
-    if (predOp == PTX::PRED_NEGATE)
-      O << '!';
-    O << getRegisterName(reg);
-  }
-}
-
 unsigned PTXAsmPrinter::GetOrCreateSourceID(StringRef FileName,
                                             StringRef DirName) {
   // If FE did not provide a file name, then assume stdin.
@@ -596,10 +540,58 @@ unsigned PTXAsmPrinter::GetOrCreateSourceID(StringRef FileName,
   return SrcId;
 }
 
-#include "PTXGenAsmWriter.inc"
+MCOperand PTXAsmPrinter::GetSymbolRef(const MachineOperand &MO,
+                                      const MCSymbol *Symbol) {
+  const MCExpr *Expr;
+  Expr = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_None, OutContext);
+  return MCOperand::CreateExpr(Expr);
+}
+
+MCOperand PTXAsmPrinter::lowerOperand(const MachineOperand &MO) {
+  MCOperand MCOp;
+  const PTXMachineFunctionInfo *MFI = MF->getInfo<PTXMachineFunctionInfo>();
+  const MCExpr *Expr;
+  const char *RegSymbolName;
+  switch (MO.getType()) {
+  default:
+    llvm_unreachable("Unknown operand type");
+  case MachineOperand::MO_Register:
+    // We create register operands as symbols, since the PTXInstPrinter class
+    // has no way to map virtual registers back to a name without some ugly
+    // hacks.
+    // FIXME: Figure out a better way to handle virtual register naming.
+    RegSymbolName = MFI->getRegisterName(MO.getReg());
+    Expr = MCSymbolRefExpr::Create(RegSymbolName, MCSymbolRefExpr::VK_None,
+                                   OutContext);
+    MCOp = MCOperand::CreateExpr(Expr);
+    break;
+  case MachineOperand::MO_Immediate:
+    MCOp = MCOperand::CreateImm(MO.getImm());
+    break;
+  case MachineOperand::MO_MachineBasicBlock:
+    MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
+                                 MO.getMBB()->getSymbol(), OutContext));
+    break;
+  case MachineOperand::MO_GlobalAddress:
+    MCOp = GetSymbolRef(MO, Mang->getSymbol(MO.getGlobal()));
+    break;
+  case MachineOperand::MO_ExternalSymbol:
+    MCOp = GetSymbolRef(MO, GetExternalSymbolSymbol(MO.getSymbolName()));
+    break;
+  case MachineOperand::MO_FPImmediate:
+    APFloat Val = MO.getFPImm()->getValueAPF();
+    bool ignored;
+    Val.convert(APFloat::IEEEdouble, APFloat::rmTowardZero, &ignored);
+    MCOp = MCOperand::CreateFPImm(Val.convertToDouble());
+    break;
+  }
+
+  return MCOp;
+}
 
 // Force static initialization.
 extern "C" void LLVMInitializePTXAsmPrinter() {
   RegisterAsmPrinter<PTXAsmPrinter> X(ThePTX32Target);
   RegisterAsmPrinter<PTXAsmPrinter> Y(ThePTX64Target);
 }
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.h b/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.h
new file mode 100644
index 000000000000..538c0802a27e
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXAsmPrinter.h
@@ -0,0 +1,57 @@
+//===-- PTXAsmPrinter.h - Print machine code to a PTX file ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// PTX Assembly printer class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PTXASMPRINTER_H
+#define PTXASMPRINTER_H
+
+#include "PTX.h"
+#include "PTXTargetMachine.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+
+class MCOperand;
+
+class LLVM_LIBRARY_VISIBILITY PTXAsmPrinter : public AsmPrinter {
+public:
+  explicit PTXAsmPrinter(TargetMachine &TM, MCStreamer &Streamer)
+    : AsmPrinter(TM, Streamer) {}
+
+  const char *getPassName() const { return "PTX Assembly Printer"; }
+
+  bool doFinalization(Module &M);
+
+  virtual void EmitStartOfAsmFile(Module &M);
+  virtual void EmitFunctionBodyStart();
+  virtual void EmitFunctionBodyEnd();
+  virtual void EmitFunctionEntryLabel();
+  virtual void EmitInstruction(const MachineInstr *MI);
+
+  unsigned GetOrCreateSourceID(StringRef FileName,
+                               StringRef DirName);
+
+  MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol);
+  MCOperand lowerOperand(const MachineOperand &MO);
+
+private:
+  void EmitVariableDeclaration(const GlobalVariable *gv);
+  void EmitFunctionDeclaration();
+
+  StringMap<unsigned> SourceIdMap;
+}; // class PTXAsmPrinter
+} // namespace llvm
+
+#endif
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXCallingConv.td b/contrib/llvm/lib/Target/PTX/PTXCallingConv.td
deleted file mode 100644
index 3e3ff4896621..000000000000
--- a/contrib/llvm/lib/Target/PTX/PTXCallingConv.td
+++ /dev/null
@@ -1,29 +0,0 @@
-
-//===--- PTXCallingConv.td - Calling Conventions -----------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This describes the calling conventions for the PTX architecture.
-//
-//===----------------------------------------------------------------------===//
-
-// PTX Formal Parameter Calling Convention
-def CC_PTX : CallingConv<[
-  CCIfType<[i1],      CCAssignToReg<[P12, P13, P14, P15, P16, P17, P18, P19, P20, P21, P22, P23, P24, P25, P26, P27, P28, P29, P30, P31, P32, P33, P34, P35, P36, P37, P38, P39, P40, P41, P42, P43, P44, P45, P46, P47, P48, P49, P50, P51, P52, P53, P54, P55, P56, P57, P58, P59, P60, P61, P62, P63, P64, P65, P66, P67, P68, P69, P70, P71, P72, P73, P74, P75, P76, P77, P78, P79, P80, P81, P82, P83, P84, P85, P86, P87, P88, P89, P90, P91, P92, P93, P94, P95, P96, P97, P98, P99, P100, P101, P102, P103, P104, P105, P106, P107, P108, P109, P110, P111, P112, P113, P114, P115, P116, P117, P118, P119, P120, P121, P122, P123, P124, P125, P126, P127]>>,
-  CCIfType<[i16],     CCAssignToReg<[RH12, RH13, RH14, RH15, RH16, RH17, RH18, RH19, RH20, RH21, RH22, RH23, RH24, RH25, RH26, RH27, RH28, RH29, RH30, RH31, RH32, RH33, RH34, RH35, RH36, RH37, RH38, RH39, RH40, RH41, RH42, RH43, RH44, RH45, RH46, RH47, RH48, RH49, RH50, RH51, RH52, RH53, RH54, RH55, RH56, RH57, RH58, RH59, RH60, RH61, RH62, RH63, RH64, RH65, RH66, RH67, RH68, RH69, RH70, RH71, RH72, RH73, RH74, RH75, RH76, RH77, RH78, RH79, RH80, RH81, RH82, RH83, RH84, RH85, RH86, RH87, RH88, RH89, RH90, RH91, RH92, RH93, RH94, RH95, RH96, RH97, RH98, RH99, RH100, RH101, RH102, RH103, RH104, RH105, RH106, RH107, RH108, RH109, RH110, RH111, RH112, RH113, RH114, RH115, RH116, RH117, RH118, RH119, RH120, RH121, RH122, RH123, RH124, RH125, RH126, RH127]>>,
-  CCIfType<[i32,f32], CCAssignToReg<[R12, R13, R14, R15, R16, R17, R18, R19, R20, R21, R22, R23, R24, R25, R26, R27, R28, R29, R30, R31, R32, R33, R34, R35, R36, R37, R38, R39, R40, R41, R42, R43, R44, R45, R46, R47, R48, R49, R50, R51, R52, R53, R54, R55, R56, R57, R58, R59, R60, R61, R62, R63, R64, R65, R66, R67, R68, R69, R70, R71, R72, R73, R74, R75, R76, R77, R78, R79, R80, R81, R82, R83, R84, R85, R86, R87, R88, R89, R90, R91, R92, R93, R94, R95, R96, R97, R98, R99, R100, R101, R102, R103, R104, R105, R106, R107, R108, R109, R110, R111, R112, R113, R114, R115, R116, R117, R118, R119, R120, R121, R122, R123, R124, R125, R126, R127]>>,
-  CCIfType<[i64,f64], CCAssignToReg<[RD12, RD13, RD14, RD15, RD16, RD17, RD18, RD19, RD20, RD21, RD22, RD23, RD24, RD25, RD26, RD27, RD28, RD29, RD30, RD31, RD32, RD33, RD34, RD35, RD36, RD37, RD38, RD39, RD40, RD41, RD42, RD43, RD44, RD45, RD46, RD47, RD48, RD49, RD50, RD51, RD52, RD53, RD54, RD55, RD56, RD57, RD58, RD59, RD60, RD61, RD62, RD63, RD64, RD65, RD66, RD67, RD68, RD69, RD70, RD71, RD72, RD73, RD74, RD75, RD76, RD77, RD78, RD79, RD80, RD81, RD82, RD83, RD84, RD85, RD86, RD87, RD88, RD89, RD90, RD91, RD92, RD93, RD94, RD95, RD96, RD97, RD98, RD99, RD100, RD101, RD102, RD103, RD104, RD105, RD106, RD107, RD108, RD109, RD110, RD111, RD112, RD113, RD114, RD115, RD116, RD117, RD118, RD119, RD120, RD121, RD122, RD123, RD124, RD125, RD126, RD127]>>
-]>;
-
-// PTX Return Value Calling Convention
-def RetCC_PTX : CallingConv<[
-  CCIfType<[i1],      CCAssignToReg<[P0, P1, P2, P3, P4, P5, P6, P7, P8, P9, P10, P11]>>,
-  CCIfType<[i16],     CCAssignToReg<[RH0, RH1, RH2, RH3, RH4, RH5, RH6, RH7, RH8, RH9, RH10, RH11]>>,
-  CCIfType<[i32,f32], CCAssignToReg<[R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11]>>,
-  CCIfType<[i64,f64], CCAssignToReg<[RD0, RD1, RD2, RD3, RD4, RD5, RD6, RD7, RD8, RD9, RD10, RD11]>>
-]>;
diff --git a/contrib/llvm/lib/Target/PTX/PTXFPRoundingModePass.cpp b/contrib/llvm/lib/Target/PTX/PTXFPRoundingModePass.cpp
new file mode 100644
index 000000000000..0b653e04b3bb
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXFPRoundingModePass.cpp
@@ -0,0 +1,179 @@
+//===-- PTXFPRoundingModePass.cpp - Assign rounding modes pass ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a machine function pass that sets appropriate FP rounding
+// modes for all relevant instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "ptx-fp-rounding-mode"
+
+#include "PTX.h"
+#include "PTXTargetMachine.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+
+// NOTE: PTXFPRoundingModePass should be executed just before emission.
+
+namespace llvm {
+  /// PTXFPRoundingModePass - Pass to assign appropriate FP rounding modes to
+  /// all FP instructions. Essentially, this pass just looks for all FP
+  /// instructions that have a rounding mode set to RndDefault, and sets an
+  /// appropriate rounding mode based on the target device.
+  ///
+  class PTXFPRoundingModePass : public MachineFunctionPass {
+    private:
+      static char ID;
+
+      typedef std::pair<unsigned, unsigned> RndModeDesc;
+
+      PTXTargetMachine& TargetMachine;
+      DenseMap<unsigned, RndModeDesc> Instrs;
+
+    public:
+      PTXFPRoundingModePass(PTXTargetMachine &TM, CodeGenOpt::Level OptLevel)
+        : MachineFunctionPass(ID),
+          TargetMachine(TM) {
+        initializeMap();
+      }
+
+      virtual bool runOnMachineFunction(MachineFunction &MF);
+
+      virtual const char *getPassName() const {
+        return "PTX FP Rounding Mode Pass";
+      }
+
+    private:
+
+      void initializeMap();
+      void processInstruction(MachineInstr &MI);
+  }; // class PTXFPRoundingModePass
+} // namespace llvm
+
+using namespace llvm;
+
+char PTXFPRoundingModePass::ID = 0;
+
+bool PTXFPRoundingModePass::runOnMachineFunction(MachineFunction &MF) {
+  // Look at each basic block
+  for (MachineFunction::iterator bbi = MF.begin(), bbe = MF.end(); bbi != bbe;
+       ++bbi) {
+    MachineBasicBlock &MBB = *bbi;
+    // Look at each instruction
+    for (MachineBasicBlock::iterator ii = MBB.begin(), ie = MBB.end();
+         ii != ie; ++ii) {
+      MachineInstr &MI = *ii;
+      processInstruction(MI);
+    }
+  }
+  return false;
+}
+
+void PTXFPRoundingModePass::initializeMap() {
+  using namespace PTXRoundingMode;
+  const PTXSubtarget& ST = TargetMachine.getSubtarget<PTXSubtarget>();
+
+  // Build a map of default rounding mode for all instructions that need a
+  // rounding mode.
+  Instrs[PTX::FADDrr32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FADDri32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FADDrr64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FADDri64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSUBrr32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSUBri32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSUBrr64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSUBri64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FMULrr32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FMULri32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FMULrr64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FMULri64] = std::make_pair(1U, (unsigned)RndNearestEven);
+
+  Instrs[PTX::FNEGrr32] = std::make_pair(1U, (unsigned)RndNone);
+  Instrs[PTX::FNEGri32] = std::make_pair(1U, (unsigned)RndNone);
+  Instrs[PTX::FNEGrr64] = std::make_pair(1U, (unsigned)RndNone);
+  Instrs[PTX::FNEGri64] = std::make_pair(1U, (unsigned)RndNone);
+
+  unsigned FDivRndMode = ST.fdivNeedsRoundingMode() ? RndNearestEven : RndNone;
+  Instrs[PTX::FDIVrr32] = std::make_pair(1U, FDivRndMode);
+  Instrs[PTX::FDIVri32] = std::make_pair(1U, FDivRndMode);
+  Instrs[PTX::FDIVrr64] = std::make_pair(1U, FDivRndMode);
+  Instrs[PTX::FDIVri64] = std::make_pair(1U, FDivRndMode);
+
+  unsigned FMADRndMode = ST.fmadNeedsRoundingMode() ? RndNearestEven : RndNone;
+  Instrs[PTX::FMADrrr32] = std::make_pair(1U, FMADRndMode);
+  Instrs[PTX::FMADrri32] = std::make_pair(1U, FMADRndMode);
+  Instrs[PTX::FMADrii32] = std::make_pair(1U, FMADRndMode);
+  Instrs[PTX::FMADrrr64] = std::make_pair(1U, FMADRndMode);
+  Instrs[PTX::FMADrri64] = std::make_pair(1U, FMADRndMode);
+  Instrs[PTX::FMADrii64] = std::make_pair(1U, FMADRndMode);
+
+  Instrs[PTX::FSQRTrr32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSQRTri32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSQRTrr64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::FSQRTri64] = std::make_pair(1U, (unsigned)RndNearestEven);
+
+  Instrs[PTX::FSINrr32] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FSINri32] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FSINrr64] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FSINri64] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FCOSrr32] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FCOSri32] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FCOSrr64] = std::make_pair(1U, (unsigned)RndApprox);
+  Instrs[PTX::FCOSri64] = std::make_pair(1U, (unsigned)RndApprox);
+
+  Instrs[PTX::CVTu16f32] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTs16f32] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTu16f64] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTs16f64] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTu32f32] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTs32f32] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTu32f64] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTs32f64] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTu64f32] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTs64f32] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTu64f64] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+  Instrs[PTX::CVTs64f64] = std::make_pair(1U, (unsigned)RndTowardsZeroInt);
+
+  Instrs[PTX::CVTf32u16] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf32s16] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf32u32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf32s32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf32u64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf32s64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf32f64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf64u16] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf64s16] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf64u32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf64s32] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf64u64] = std::make_pair(1U, (unsigned)RndNearestEven);
+  Instrs[PTX::CVTf64s64] = std::make_pair(1U, (unsigned)RndNearestEven);
+}
+
+void PTXFPRoundingModePass::processInstruction(MachineInstr &MI) {
+  // Is this an instruction that needs a rounding mode?
+  if (Instrs.count(MI.getOpcode())) {
+    const RndModeDesc &Desc = Instrs[MI.getOpcode()];
+    // Get the rounding mode operand
+    MachineOperand &Op = MI.getOperand(Desc.first);
+    // Update the rounding mode if needed
+    if (Op.getImm() == PTXRoundingMode::RndDefault) {
+      Op.setImm(Desc.second);
+    }
+  }
+}
+
+FunctionPass *llvm::createPTXFPRoundingModePass(PTXTargetMachine &TM,
+                                                CodeGenOpt::Level OptLevel) {
+  return new PTXFPRoundingModePass(TM, OptLevel);
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXISelDAGToDAG.cpp b/contrib/llvm/lib/Target/PTX/PTXISelDAGToDAG.cpp
index 9adfa624b29e..5c7ee298f31a 100644
--- a/contrib/llvm/lib/Target/PTX/PTXISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXISelDAGToDAG.cpp
@@ -12,7 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "PTX.h"
+#include "PTXMachineFunctionInfo.h"
 #include "PTXTargetMachine.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Support/Debug.h"
@@ -37,6 +39,7 @@ class PTXDAGToDAGISel : public SelectionDAGISel {
     bool SelectADDRrr(SDValue &Addr, SDValue &R1, SDValue &R2);
     bool SelectADDRri(SDValue &Addr, SDValue &Base, SDValue &Offset);
     bool SelectADDRii(SDValue &Addr, SDValue &Base, SDValue &Offset);
+    bool SelectADDRlocal(SDValue &Addr, SDValue &Base, SDValue &Offset);
 
     // Include the pieces auto'gened from the target description
 #include "PTXGenDAGISel.inc"
@@ -46,6 +49,10 @@ class PTXDAGToDAGISel : public SelectionDAGISel {
     // pattern (PTXbrcond bb:$d, ...) in PTXInstrInfo.td
     SDNode *SelectBRCOND(SDNode *Node);
 
+    SDNode *SelectREADPARAM(SDNode *Node);
+    SDNode *SelectWRITEPARAM(SDNode *Node);
+    SDNode *SelectFrameIndex(SDNode *Node);
+
     bool isImm(const SDValue &operand);
     bool SelectImm(const SDValue &operand, SDValue &imm);
 
@@ -68,6 +75,12 @@ SDNode *PTXDAGToDAGISel::Select(SDNode *Node) {
   switch (Node->getOpcode()) {
     case ISD::BRCOND:
       return SelectBRCOND(Node);
+    case PTXISD::READ_PARAM:
+      return SelectREADPARAM(Node);
+    case PTXISD::WRITE_PARAM:
+      return SelectWRITEPARAM(Node);
+    case ISD::FrameIndex:
+      return SelectFrameIndex(Node);
     default:
       return SelectCode(Node);
   }
@@ -79,7 +92,7 @@ SDNode *PTXDAGToDAGISel::SelectBRCOND(SDNode *Node) {
   SDValue Chain  = Node->getOperand(0);
   SDValue Pred   = Node->getOperand(1);
   SDValue Target = Node->getOperand(2); // branch target
-  SDValue PredOp = CurDAG->getTargetConstant(PTX::PRED_NORMAL, MVT::i32);
+  SDValue PredOp = CurDAG->getTargetConstant(PTXPredicate::Normal, MVT::i32);
   DebugLoc dl = Node->getDebugLoc();
 
   assert(Target.getOpcode()  == ISD::BasicBlock);
@@ -90,6 +103,97 @@ SDNode *PTXDAGToDAGISel::SelectBRCOND(SDNode *Node) {
   return CurDAG->getMachineNode(PTX::BRAdp, dl, MVT::Other, Ops, 4);
 }
 
+SDNode *PTXDAGToDAGISel::SelectREADPARAM(SDNode *Node) {
+  SDValue Chain = Node->getOperand(0);
+  SDValue Index = Node->getOperand(1);
+
+  int OpCode;
+
+  // Get the type of parameter we are reading
+  EVT VT = Node->getValueType(0);
+  assert(VT.isSimple() && "READ_PARAM only implemented for MVT types");
+
+  MVT Type = VT.getSimpleVT();
+
+  if (Type == MVT::i1)
+    OpCode = PTX::READPARAMPRED;
+  else if (Type == MVT::i16)
+    OpCode = PTX::READPARAMI16;
+  else if (Type == MVT::i32)
+    OpCode = PTX::READPARAMI32;
+  else if (Type == MVT::i64)
+    OpCode = PTX::READPARAMI64;
+  else if (Type == MVT::f32)
+    OpCode = PTX::READPARAMF32;
+  else {
+    assert(Type == MVT::f64 && "Unexpected type!");
+    OpCode = PTX::READPARAMF64;
+  }
+
+  SDValue Pred = CurDAG->getRegister(PTX::NoRegister, MVT::i1);
+  SDValue PredOp = CurDAG->getTargetConstant(PTXPredicate::None, MVT::i32);
+  DebugLoc dl = Node->getDebugLoc();
+
+  SDValue Ops[] = { Index, Pred, PredOp, Chain };
+  return CurDAG->getMachineNode(OpCode, dl, VT, Ops, 4);
+}
+
+SDNode *PTXDAGToDAGISel::SelectWRITEPARAM(SDNode *Node) {
+
+  SDValue Chain = Node->getOperand(0);
+  SDValue Value = Node->getOperand(1);
+
+  int OpCode;
+
+  //Node->dumpr(CurDAG);
+
+  // Get the type of parameter we are writing
+  EVT VT = Value->getValueType(0);
+  assert(VT.isSimple() && "WRITE_PARAM only implemented for MVT types");
+
+  MVT Type = VT.getSimpleVT();
+
+  if (Type == MVT::i1)
+    OpCode = PTX::WRITEPARAMPRED;
+  else if (Type == MVT::i16)
+    OpCode = PTX::WRITEPARAMI16;
+  else if (Type == MVT::i32)
+    OpCode = PTX::WRITEPARAMI32;
+  else if (Type == MVT::i64)
+    OpCode = PTX::WRITEPARAMI64;
+  else if (Type == MVT::f32)
+    OpCode = PTX::WRITEPARAMF32;
+  else if (Type == MVT::f64)
+    OpCode = PTX::WRITEPARAMF64;
+  else
+    llvm_unreachable("Invalid type in SelectWRITEPARAM");
+
+  SDValue Pred = CurDAG->getRegister(PTX::NoRegister, MVT::i1);
+  SDValue PredOp = CurDAG->getTargetConstant(PTXPredicate::None, MVT::i32);
+  DebugLoc dl = Node->getDebugLoc();
+
+  SDValue Ops[] = { Value, Pred, PredOp, Chain };
+  SDNode* Ret = CurDAG->getMachineNode(OpCode, dl, MVT::Other, Ops, 4);
+
+  //dbgs() << "SelectWRITEPARAM produced:\n\t";
+  //Ret->dumpr(CurDAG);
+
+  return Ret;
+}
+
+SDNode *PTXDAGToDAGISel::SelectFrameIndex(SDNode *Node) {
+  int FI = cast<FrameIndexSDNode>(Node)->getIndex();
+  //dbgs() << "Selecting FrameIndex at index " << FI << "\n";
+  //SDValue TFI = CurDAG->getTargetFrameIndex(FI, Node->getValueType(0));
+
+  PTXMachineFunctionInfo *MFI = MF->getInfo<PTXMachineFunctionInfo>();
+
+  SDValue FrameSymbol = CurDAG->getTargetExternalSymbol(MFI->getFrameSymbol(FI),
+                                                        Node->getValueType(0));
+
+  return FrameSymbol.getNode();
+}
+
 // Match memory operand of the form [reg+reg]
 bool PTXDAGToDAGISel::SelectADDRrr(SDValue &Addr, SDValue &R1, SDValue &R2) {
   if (Addr.getOpcode() != ISD::ADD || Addr.getNumOperands() < 2 ||
@@ -107,14 +211,54 @@ bool PTXDAGToDAGISel::SelectADDRrr(SDValue &Addr, SDValue &R1, SDValue &R2) {
 // Match memory operand of the form [reg], [imm+reg], and [reg+imm]
 bool PTXDAGToDAGISel::SelectADDRri(SDValue &Addr, SDValue &Base,
                                    SDValue &Offset) {
-  if (Addr.getOpcode() != ISD::ADD) {
+  // FrameIndex addresses are handled separately
+  //errs() << "SelectADDRri: ";
+  //Addr.getNode()->dumpr();
+  if (isa<FrameIndexSDNode>(Addr)) {
+    //errs() << "Failure\n";
+    return false;
+  }
+
+  if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    Base = Addr.getOperand(0);
+    if (isa<FrameIndexSDNode>(Base)) {
+      //errs() << "Failure\n";
+      return false;
+    }
+    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
+    Offset = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
+    //errs() << "Success\n";
+    return true;
+  }
+
+  /*if (Addr.getNumOperands() == 1) {
+    Base = Addr;
+    Offset = CurDAG->getTargetConstant(0, Addr.getValueType().getSimpleVT());
+    errs() << "Success\n";
+    return true;
+  }*/
+
+  //errs() << "SelectADDRri fails on: ";
+  //Addr.getNode()->dumpr();
+
+  if (isImm(Addr)) {
+    //errs() << "Failure\n";
+    return false;
+  }
+
+  Base = Addr;
+  Offset = CurDAG->getTargetConstant(0, Addr.getValueType().getSimpleVT());
+
+  //errs() << "Success\n";
+  return true;
+
+  /*if (Addr.getOpcode() != ISD::ADD) {
     // let SelectADDRii handle the [imm] case
     if (isImm(Addr))
       return false;
     // it is [reg]
 
     assert(Addr.getValueType().isSimple() && "Type must be simple");
-
     Base = Addr;
     Offset = CurDAG->getTargetConstant(0, Addr.getValueType().getSimpleVT());
 
@@ -136,7 +280,7 @@ bool PTXDAGToDAGISel::SelectADDRri(SDValue &Addr, SDValue &Base,
     }
 
   // neither [reg+imm] nor [imm+reg]
-  return false;
+  return false;*/
 }
 
 // Match memory operand of the form [imm+imm] and [imm]
@@ -160,6 +304,36 @@ bool PTXDAGToDAGISel::SelectADDRii(SDValue &Addr, SDValue &Base,
   return false;
 }
 
+// Match memory operand of the form [reg], [imm+reg], and [reg+imm]
+bool PTXDAGToDAGISel::SelectADDRlocal(SDValue &Addr, SDValue &Base,
+                                      SDValue &Offset) {
+  //errs() << "SelectADDRlocal: ";
+  //Addr.getNode()->dumpr();
+  if (isa<FrameIndexSDNode>(Addr)) {
+    Base = Addr;
+    Offset = CurDAG->getTargetConstant(0, Addr.getValueType().getSimpleVT());
+    //errs() << "Success\n";
+    return true;
+  }
+
+  if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    Base = Addr.getOperand(0);
+    if (!isa<FrameIndexSDNode>(Base)) {
+      //errs() << "Failure\n";
+      return false;
+    }
+    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
+    Offset = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
+    //errs() << "Offset: ";
+    //Offset.getNode()->dumpr();
+    //errs() << "Success\n";
+    return true;
+  }
+
+  //errs() << "Failure\n";
+  return false;
+}
+
 bool PTXDAGToDAGISel::isImm(const SDValue &operand) {
   return ConstantSDNode::classof(operand.getNode());
 }
diff --git a/contrib/llvm/lib/Target/PTX/PTXISelLowering.cpp b/contrib/llvm/lib/Target/PTX/PTXISelLowering.cpp
index 6fcf710e3f1f..3307d91a6188 100644
--- a/contrib/llvm/lib/Target/PTX/PTXISelLowering.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXISelLowering.cpp
@@ -16,22 +16,18 @@
 #include "PTXMachineFunctionInfo.h"
 #include "PTXRegisterInfo.h"
 #include "PTXSubtarget.h"
+#include "llvm/Function.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
-//===----------------------------------------------------------------------===//
-// Calling Convention Implementation
-//===----------------------------------------------------------------------===//
-
-#include "PTXGenCallingConv.inc"
-
 //===----------------------------------------------------------------------===//
 // TargetLowering Implementation
 //===----------------------------------------------------------------------===//
@@ -47,57 +43,58 @@ PTXTargetLowering::PTXTargetLowering(TargetMachine &TM)
   addRegisterClass(MVT::f64, PTX::RegF64RegisterClass);
 
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
   setMinFunctionAlignment(2);
-  
+
   ////////////////////////////////////
   /////////// Expansion //////////////
   ////////////////////////////////////
-  
+
   // (any/zero/sign) extload => load + (any/zero/sign) extend
-  
+
   setLoadExtAction(ISD::EXTLOAD, MVT::i16, Expand);
   setLoadExtAction(ISD::ZEXTLOAD, MVT::i16, Expand);
   setLoadExtAction(ISD::SEXTLOAD, MVT::i16, Expand);
-  
+
   // f32 extload => load + fextend
-  
-  setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);  
-  
+
+  setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
+
   // f64 truncstore => trunc + store
-  
-  setTruncStoreAction(MVT::f64, MVT::f32, Expand); 
-  
+
+  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
+
   // sign_extend_inreg => sign_extend
-  
+
   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
-  
+
   // br_cc => brcond
-  
+
   setOperationAction(ISD::BR_CC, MVT::Other, Expand);
 
   // select_cc => setcc
-  
+
   setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
   setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
   setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
-  
+
   ////////////////////////////////////
   //////////// Legal /////////////////
   ////////////////////////////////////
-  
+
   setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
   setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
-  
+
   ////////////////////////////////////
   //////////// Custom ////////////////
   ////////////////////////////////////
-  
+
   // customise setcc to use bitwise logic if possible
-  
+
   setOperationAction(ISD::SETCC, MVT::i1, Custom);
 
   // customize translation of memory addresses
-  
+
   setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
   setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
 
@@ -105,7 +102,7 @@ PTXTargetLowering::PTXTargetLowering(TargetMachine &TM)
   computeRegisterProperties();
 }
 
-MVT::SimpleValueType PTXTargetLowering::getSetCCResultType(EVT VT) const {
+EVT PTXTargetLowering::getSetCCResultType(EVT VT) const {
   return MVT::i1;
 }
 
@@ -130,10 +127,16 @@ const char *PTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
       return "PTXISD::LOAD_PARAM";
     case PTXISD::STORE_PARAM:
       return "PTXISD::STORE_PARAM";
+    case PTXISD::READ_PARAM:
+      return "PTXISD::READ_PARAM";
+    case PTXISD::WRITE_PARAM:
+      return "PTXISD::WRITE_PARAM";
     case PTXISD::EXIT:
       return "PTXISD::EXIT";
     case PTXISD::RET:
       return "PTXISD::RET";
+    case PTXISD::CALL:
+      return "PTXISD::CALL";
   }
 }
 
@@ -149,7 +152,7 @@ SDValue PTXTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   DebugLoc dl = Op.getDebugLoc();
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
 
-  // Look for X == 0, X == 1, X != 0, or X != 1  
+  // Look for X == 0, X == 1, X != 0, or X != 1
   // We can simplify these to bitwise logic
 
   if (Op1.getOpcode() == ISD::Constant &&
@@ -197,6 +200,7 @@ SDValue PTXTargetLowering::
   MachineFunction &MF = DAG.getMachineFunction();
   const PTXSubtarget& ST = getTargetMachine().getSubtarget<PTXSubtarget>();
   PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
+  PTXParamManager &PM = MFI->getParamManager();
 
   switch (CallConv) {
     default:
@@ -216,37 +220,84 @@ SDValue PTXTargetLowering::
   if (MFI->isKernel() || ST.useParamSpaceForDeviceArgs()) {
     // We just need to emit the proper LOAD_PARAM ISDs
     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
-
       assert((!MFI->isKernel() || Ins[i].VT != MVT::i1) &&
              "Kernels cannot take pred operands");
 
+      unsigned ParamSize = Ins[i].VT.getStoreSizeInBits();
+      unsigned Param = PM.addArgumentParam(ParamSize);
+      const std::string &ParamName = PM.getParamName(Param);
+      SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
+                                                       MVT::Other);
       SDValue ArgValue = DAG.getNode(PTXISD::LOAD_PARAM, dl, Ins[i].VT, Chain,
-                                     DAG.getTargetConstant(i, MVT::i32));
+                                     ParamValue);
       InVals.push_back(ArgValue);
-
-      // Instead of storing a physical register in our argument list, we just
-      // store the total size of the parameter, in bits.  The ASM printer
-      // knows how to process this.
-      MFI->addArgReg(Ins[i].VT.getStoreSizeInBits());
     }
   }
   else {
-    // For device functions, we use the PTX calling convention to do register
-    // assignments then create CopyFromReg ISDs for the allocated registers
+    for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
+      EVT                  RegVT = Ins[i].VT;
+      TargetRegisterClass* TRC   = getRegClassFor(RegVT);
 
-    SmallVector<CCValAssign, 16> ArgLocs;
-    CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), ArgLocs,
-                   *DAG.getContext());
+      // Use a unique index in the instruction to prevent instruction folding.
+      // Yes, this is a hack.
+      SDValue Index = DAG.getTargetConstant(i, MVT::i32);
+      unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
+      SDValue ArgValue = DAG.getNode(PTXISD::READ_PARAM, dl, RegVT, Chain,
+                                     Index);
 
-    CCInfo.AnalyzeFormalArguments(Ins, CC_PTX);
+      InVals.push_back(ArgValue);
 
-    for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+      MFI->addArgReg(Reg);
+    }
+  }
 
-      CCValAssign&         VA    = ArgLocs[i];
-      EVT                  RegVT = VA.getLocVT();
-      TargetRegisterClass* TRC   = 0;
+  return Chain;
+}
 
-      assert(VA.isRegLoc() && "CCValAssign must be RegLoc");
+SDValue PTXTargetLowering::
+  LowerReturn(SDValue Chain,
+              CallingConv::ID CallConv,
+              bool isVarArg,
+              const SmallVectorImpl<ISD::OutputArg> &Outs,
+              const SmallVectorImpl<SDValue> &OutVals,
+              DebugLoc dl,
+              SelectionDAG &DAG) const {
+  if (isVarArg) llvm_unreachable("PTX does not support varargs");
+
+  switch (CallConv) {
+    default:
+      llvm_unreachable("Unsupported calling convention.");
+    case CallingConv::PTX_Kernel:
+      assert(Outs.size() == 0 && "Kernel must return void.");
+      return DAG.getNode(PTXISD::EXIT, dl, MVT::Other, Chain);
+    case CallingConv::PTX_Device:
+      assert(Outs.size() <= 1 && "Can at most return one value.");
+      break;
+  }
+
+  MachineFunction& MF = DAG.getMachineFunction();
+  PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
+  PTXParamManager &PM = MFI->getParamManager();
+
+  SDValue Flag;
+  const PTXSubtarget& ST = getTargetMachine().getSubtarget<PTXSubtarget>();
+
+  if (ST.useParamSpaceForDeviceArgs()) {
+    assert(Outs.size() < 2 && "Device functions can return at most one value");
+
+    if (Outs.size() == 1) {
+      unsigned ParamSize = OutVals[0].getValueType().getSizeInBits();
+      unsigned Param = PM.addReturnParam(ParamSize);
+      const std::string &ParamName = PM.getParamName(Param);
+      SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
+                                                       MVT::Other);
+      Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
+                          ParamValue, OutVals[0]);
+    }
+  } else {
+    for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
+      EVT                  RegVT = Outs[i].VT;
+      TargetRegisterClass* TRC = 0;
 
       // Determine which register class we need
       if (RegVT == MVT::i1) {
@@ -272,72 +323,16 @@ SDValue PTXTargetLowering::
       }
 
       unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
-      MF.getRegInfo().addLiveIn(VA.getLocReg(), Reg);
 
-      SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
-      InVals.push_back(ArgValue);
+      SDValue Copy = DAG.getCopyToReg(Chain, dl, Reg, OutVals[i]/*, Flag*/);
+      SDValue OutReg = DAG.getRegister(Reg, RegVT);
 
-      MFI->addArgReg(VA.getLocReg());
+      Chain = DAG.getNode(PTXISD::WRITE_PARAM, dl, MVT::Other, Copy, OutReg);
+
+      MFI->addRetReg(Reg);
     }
   }
 
-  return Chain;
-}
-
-SDValue PTXTargetLowering::
-  LowerReturn(SDValue Chain,
-              CallingConv::ID CallConv,
-              bool isVarArg,
-              const SmallVectorImpl<ISD::OutputArg> &Outs,
-              const SmallVectorImpl<SDValue> &OutVals,
-              DebugLoc dl,
-              SelectionDAG &DAG) const {
-  if (isVarArg) llvm_unreachable("PTX does not support varargs");
-
-  switch (CallConv) {
-    default:
-      llvm_unreachable("Unsupported calling convention.");
-    case CallingConv::PTX_Kernel:
-      assert(Outs.size() == 0 && "Kernel must return void.");
-      return DAG.getNode(PTXISD::EXIT, dl, MVT::Other, Chain);
-    case CallingConv::PTX_Device:
-      //assert(Outs.size() <= 1 && "Can at most return one value.");
-      break;
-  }
-
-  MachineFunction& MF = DAG.getMachineFunction();
-  PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
-
-  SDValue Flag;
-
-  // Even though we could use the .param space for return arguments for
-  // device functions if SM >= 2.0 and the number of return arguments is
-  // only 1, we just always use registers since this makes the codegen
-  // easier.
-  SmallVector<CCValAssign, 16> RVLocs;
-  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
-  getTargetMachine(), RVLocs, *DAG.getContext());
-
-  CCInfo.AnalyzeReturn(Outs, RetCC_PTX);
-
-  for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
-    CCValAssign& VA  = RVLocs[i];
-
-    assert(VA.isRegLoc() && "CCValAssign must be RegLoc");
-
-    unsigned Reg = VA.getLocReg();
-
-    DAG.getMachineFunction().getRegInfo().addLiveOut(Reg);
-
-    Chain = DAG.getCopyToReg(Chain, dl, Reg, OutVals[i], Flag);
-
-    // Guarantee that all emitted copies are stuck together,
-    // avoiding something bad
-    Flag = Chain.getValue(1);
-
-    MFI->addRetReg(Reg);
-  }
-
   if (Flag.getNode() == 0) {
     return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain);
   }
@@ -345,3 +340,83 @@ SDValue PTXTargetLowering::
     return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain, Flag);
   }
 }
+
+SDValue
+PTXTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
+                             CallingConv::ID CallConv, bool isVarArg,
+                             bool &isTailCall,
+                             const SmallVectorImpl<ISD::OutputArg> &Outs,
+                             const SmallVectorImpl<SDValue> &OutVals,
+                             const SmallVectorImpl<ISD::InputArg> &Ins,
+                             DebugLoc dl, SelectionDAG &DAG,
+                             SmallVectorImpl<SDValue> &InVals) const {
+
+  MachineFunction& MF = DAG.getMachineFunction();
+  PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
+  PTXParamManager &PM = MFI->getParamManager();
+
+  assert(getTargetMachine().getSubtarget<PTXSubtarget>().callsAreHandled() &&
+         "Calls are not handled for the target device");
+
+  std::vector<SDValue> Ops;
+  // The layout of the ops will be [Chain, #Ins, Ins, Callee, #Outs, Outs]
+  Ops.resize(Outs.size() + Ins.size() + 4);
+
+  Ops[0] = Chain;
+
+  // Identify the callee function
+  const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
+  assert(cast<Function>(GV)->getCallingConv() == CallingConv::PTX_Device &&
+         "PTX function calls must be to PTX device functions");
+  Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
+  Ops[Ins.size()+2] = Callee;
+
+  // Generate STORE_PARAM nodes for each function argument.  In PTX, function
+  // arguments are explicitly stored into .param variables and passed as
+  // arguments. There is no register/stack-based calling convention in PTX.
+  Ops[Ins.size()+3] = DAG.getTargetConstant(OutVals.size(), MVT::i32);
+  for (unsigned i = 0; i != OutVals.size(); ++i) {
+    unsigned Size = OutVals[i].getValueType().getSizeInBits();
+    unsigned Param = PM.addLocalParam(Size);
+    const std::string &ParamName = PM.getParamName(Param);
+    SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
+                                                     MVT::Other);
+    Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
+                        ParamValue, OutVals[i]);
+    Ops[i+Ins.size()+4] = ParamValue;
+  }
+
+  std::vector<SDValue> InParams;
+
+  // Generate list of .param variables to hold the return value(s).
+  Ops[1] = DAG.getTargetConstant(Ins.size(), MVT::i32);
+  for (unsigned i = 0; i < Ins.size(); ++i) {
+    unsigned Size = Ins[i].VT.getStoreSizeInBits();
+    unsigned Param = PM.addLocalParam(Size);
+    const std::string &ParamName = PM.getParamName(Param);
+    SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
+                                                     MVT::Other);
+    Ops[i+2] = ParamValue;
+    InParams.push_back(ParamValue);
+  }
+
+  Ops[0] = Chain;
+
+  // Create the CALL node.
+  Chain = DAG.getNode(PTXISD::CALL, dl, MVT::Other, &Ops[0], Ops.size());
+
+  // Create the LOAD_PARAM nodes that retrieve the function return value(s).
+  for (unsigned i = 0; i < Ins.size(); ++i) {
+    SDValue Load = DAG.getNode(PTXISD::LOAD_PARAM, dl, Ins[i].VT, Chain,
+                               InParams[i]);
+    InVals.push_back(Load);
+  }
+
+  return Chain;
+}
+
+unsigned PTXTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT) {
+  // All arguments consist of one "register," regardless of the type.
+  return 1;
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXISelLowering.h b/contrib/llvm/lib/Target/PTX/PTXISelLowering.h
index 43185416e1fc..4d2566540af2 100644
--- a/contrib/llvm/lib/Target/PTX/PTXISelLowering.h
+++ b/contrib/llvm/lib/Target/PTX/PTXISelLowering.h
@@ -26,9 +26,12 @@ namespace PTXISD {
     FIRST_NUMBER = ISD::BUILTIN_OP_END,
     LOAD_PARAM,
     STORE_PARAM,
+    READ_PARAM,
+    WRITE_PARAM,
     EXIT,
     RET,
-    COPY_ADDRESS
+    COPY_ADDRESS,
+    CALL
   };
 }                               // namespace PTXISD
 
@@ -60,7 +63,19 @@ class PTXTargetLowering : public TargetLowering {
                   DebugLoc dl,
                   SelectionDAG &DAG) const;
 
-    virtual MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    virtual SDValue
+      LowerCall(SDValue Chain, SDValue Callee,
+                CallingConv::ID CallConv, bool isVarArg,
+                bool &isTailCall,
+                const SmallVectorImpl<ISD::OutputArg> &Outs,
+                const SmallVectorImpl<SDValue> &OutVals,
+                const SmallVectorImpl<ISD::InputArg> &Ins,
+                DebugLoc dl, SelectionDAG &DAG,
+                SmallVectorImpl<SDValue> &InVals) const;
+
+    virtual EVT getSetCCResultType(EVT VT) const;
+
+    virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT);
 
   private:
     SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
diff --git a/contrib/llvm/lib/Target/PTX/PTXInstrFormats.td b/contrib/llvm/lib/Target/PTX/PTXInstrFormats.td
index 8cee351ee0df..397fdc319a84 100644
--- a/contrib/llvm/lib/Target/PTX/PTXInstrFormats.td
+++ b/contrib/llvm/lib/Target/PTX/PTXInstrFormats.td
@@ -7,12 +7,39 @@
 //
 //===----------------------------------------------------------------------===//
 
-// PTX Predicate operand, default to (0, 0) = (zero-reg, always).
+
+// Rounding Mode Specifier
+/*class RoundingMode<bits<3> val> {
+  bits<3> Value = val;
+}
+
+def RndDefault     : RoundingMode<0>;
+def RndNearestEven : RoundingMode<1>;
+def RndNearestZero : RoundingMode<2>;
+def RndNegInf      : RoundingMode<3>;
+def RndPosInf      : RoundingMode<4>;
+def RndApprox      : RoundingMode<5>;*/
+
+
+// Rounding Mode Operand
+def RndMode : Operand<i32> {
+  let PrintMethod = "printRoundingMode";
+}
+
+def RndDefault : PatLeaf<(i32 0)>;
+
+// PTX Predicate operand, default to (0, 0) = (zero-reg, none).
 // Leave PrintMethod empty; predicate printing is defined elsewhere.
 def pred : PredicateOperand<OtherVT, (ops RegPred, i32imm),
-                                     (ops (i1 zero_reg), (i32 0))>;
+                                     (ops (i1 zero_reg), (i32 2))>;
 
+def RndModeOperand : Operand<OtherVT> {
+  let MIOperandInfo = (ops i32imm);
+}
+
+// Instruction Types
 let Namespace = "PTX" in {
+
   class InstPTX<dag oops, dag iops, string asmstr, list<dag> pattern>
     : Instruction {
       dag OutOperandList = oops;
diff --git a/contrib/llvm/lib/Target/PTX/PTXInstrInfo.cpp b/contrib/llvm/lib/Target/PTX/PTXInstrInfo.cpp
index 425265a2fdb7..1b947a5400f4 100644
--- a/contrib/llvm/lib/Target/PTX/PTXInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXInstrInfo.cpp
@@ -16,10 +16,11 @@
 #include "PTX.h"
 #include "PTXInstrInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define GET_INSTRINFO_CTOR
@@ -47,8 +48,13 @@ void PTXInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator I, DebugLoc DL,
                                unsigned DstReg, unsigned SrcReg,
                                bool KillSrc) const {
-  for (int i = 0, e = sizeof(map)/sizeof(map[0]); i != e; ++ i) {
-    if (map[i].cls->contains(DstReg, SrcReg)) {
+
+  const MachineRegisterInfo& MRI = MBB.getParent()->getRegInfo();
+  //assert(MRI.getRegClass(SrcReg) == MRI.getRegClass(DstReg) &&
+  //  "Invalid register copy between two register classes");
+
+  for (int i = 0, e = sizeof(map)/sizeof(map[0]); i != e; ++i) {
+    if (map[i].cls == MRI.getRegClass(DstReg)) {
       const MCInstrDesc &MCID = get(map[i].opcode);
       MachineInstr *MI = BuildMI(MBB, I, DL, MCID, DstReg).
         addReg(SrcReg, getKillRegState(KillSrc));
@@ -161,7 +167,7 @@ DefinesPredicate(MachineInstr *MI,
     return false;
 
   Pred.push_back(MO);
-  Pred.push_back(MachineOperand::CreateImm(PTX::PRED_NORMAL));
+  Pred.push_back(MachineOperand::CreateImm(PTXPredicate::None));
   return true;
 }
 
@@ -277,7 +283,7 @@ InsertBranch(MachineBasicBlock &MBB,
     BuildMI(&MBB, DL, get(PTX::BRAdp))
       .addMBB(TBB).addReg(Cond[0].getReg()).addImm(Cond[1].getImm());
     BuildMI(&MBB, DL, get(PTX::BRAd))
-      .addMBB(FBB).addReg(PTX::NoRegister).addImm(PTX::PRED_NORMAL);
+      .addMBB(FBB).addReg(PTX::NoRegister).addImm(PTXPredicate::None);
     return 2;
   } else if (Cond.size()) {
     BuildMI(&MBB, DL, get(PTX::BRAdp))
@@ -285,7 +291,7 @@ InsertBranch(MachineBasicBlock &MBB,
     return 1;
   } else {
     BuildMI(&MBB, DL, get(PTX::BRAd))
-      .addMBB(TBB).addReg(PTX::NoRegister).addImm(PTX::PRED_NORMAL);
+      .addMBB(TBB).addReg(PTX::NoRegister).addImm(PTXPredicate::None);
     return 1;
   }
 }
@@ -296,34 +302,7 @@ void PTXInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                      unsigned SrcReg, bool isKill, int FrameIdx,
                                        const TargetRegisterClass *RC,
                                        const TargetRegisterInfo *TRI) const {
-  MachineInstr& MI = *MII;
-  DebugLoc DL = MI.getDebugLoc();
-
-  DEBUG(dbgs() << "storeRegToStackSlot: " << MI);
-
-  int OpCode;
-
-  // Select the appropriate opcode based on the register class
-  if (RC == PTX::RegI16RegisterClass) {
-    OpCode = PTX::STACKSTOREI16;
-  }  else if (RC == PTX::RegI32RegisterClass) {
-    OpCode = PTX::STACKSTOREI32;
-  }  else if (RC == PTX::RegI64RegisterClass) {
-    OpCode = PTX::STACKSTOREI32;
-  }  else if (RC == PTX::RegF32RegisterClass) {
-    OpCode = PTX::STACKSTOREF32;
-  }  else if (RC == PTX::RegF64RegisterClass) {
-    OpCode = PTX::STACKSTOREF64;
-  } else {
-    llvm_unreachable("Unknown PTX register class!");
-  }
-
-  // Build the store instruction (really a mov)
-  MachineInstrBuilder MIB = BuildMI(MBB, MII, DL, get(OpCode));
-  MIB.addFrameIndex(FrameIdx);
-  MIB.addReg(SrcReg);
-
-  AddDefaultPredicate(MIB);
+  assert(false && "storeRegToStackSlot should not be called for PTX");
 }
 
 void PTXInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
@@ -331,34 +310,7 @@ void PTXInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                         unsigned DestReg, int FrameIdx,
                                         const TargetRegisterClass *RC,
                                         const TargetRegisterInfo *TRI) const {
-  MachineInstr& MI = *MII;
-  DebugLoc DL = MI.getDebugLoc();
-
-  DEBUG(dbgs() << "loadRegToStackSlot: " << MI);
-
-  int OpCode;
-
-  // Select the appropriate opcode based on the register class
-  if (RC == PTX::RegI16RegisterClass) {
-    OpCode = PTX::STACKLOADI16;
-  } else if (RC == PTX::RegI32RegisterClass) {
-    OpCode = PTX::STACKLOADI32;
-  } else if (RC == PTX::RegI64RegisterClass) {
-    OpCode = PTX::STACKLOADI32;
-  } else if (RC == PTX::RegF32RegisterClass) {
-    OpCode = PTX::STACKLOADF32;
-  } else if (RC == PTX::RegF64RegisterClass) {
-    OpCode = PTX::STACKLOADF64;
-  } else {
-    llvm_unreachable("Unknown PTX register class!");
-  }
-
-  // Build the load instruction (really a mov)
-  MachineInstrBuilder MIB = BuildMI(MBB, MII, DL, get(OpCode));
-  MIB.addReg(DestReg);
-  MIB.addFrameIndex(FrameIdx);
-
-  AddDefaultPredicate(MIB);
+  assert(false && "loadRegFromStackSlot should not be called for PTX");
 }
 
 // static helper routines
@@ -367,7 +319,7 @@ MachineSDNode *PTXInstrInfo::
 GetPTXMachineNode(SelectionDAG *DAG, unsigned Opcode,
                   DebugLoc dl, EVT VT, SDValue Op1) {
   SDValue predReg = DAG->getRegister(PTX::NoRegister, MVT::i1);
-  SDValue predOp = DAG->getTargetConstant(PTX::PRED_NORMAL, MVT::i32);
+  SDValue predOp = DAG->getTargetConstant(PTXPredicate::None, MVT::i32);
   SDValue ops[] = { Op1, predReg, predOp };
   return DAG->getMachineNode(Opcode, dl, VT, ops, array_lengthof(ops));
 }
@@ -376,7 +328,7 @@ MachineSDNode *PTXInstrInfo::
 GetPTXMachineNode(SelectionDAG *DAG, unsigned Opcode,
                   DebugLoc dl, EVT VT, SDValue Op1, SDValue Op2) {
   SDValue predReg = DAG->getRegister(PTX::NoRegister, MVT::i1);
-  SDValue predOp = DAG->getTargetConstant(PTX::PRED_NORMAL, MVT::i32);
+  SDValue predOp = DAG->getTargetConstant(PTXPredicate::None, MVT::i32);
   SDValue ops[] = { Op1, Op2, predReg, predOp };
   return DAG->getMachineNode(Opcode, dl, VT, ops, array_lengthof(ops));
 }
@@ -384,7 +336,7 @@ GetPTXMachineNode(SelectionDAG *DAG, unsigned Opcode,
 void PTXInstrInfo::AddDefaultPredicate(MachineInstr *MI) {
   if (MI->findFirstPredOperandIdx() == -1) {
     MI->addOperand(MachineOperand::CreateReg(PTX::NoRegister, /*IsDef=*/false));
-    MI->addOperand(MachineOperand::CreateImm(PTX::PRED_NORMAL));
+    MI->addOperand(MachineOperand::CreateImm(PTXPredicate::None));
   }
 }
 
diff --git a/contrib/llvm/lib/Target/PTX/PTXInstrInfo.td b/contrib/llvm/lib/Target/PTX/PTXInstrInfo.td
index 6bfe906d40ab..a3fcea9038cf 100644
--- a/contrib/llvm/lib/Target/PTX/PTXInstrInfo.td
+++ b/contrib/llvm/lib/Target/PTX/PTXInstrInfo.td
@@ -21,10 +21,6 @@ include "PTXInstrFormats.td"
 // Code Generation Predicates
 //===----------------------------------------------------------------------===//
 
-// Addressing
-def Use32BitAddresses : Predicate<"!getSubtarget().is64Bit()">;
-def Use64BitAddresses : Predicate<"getSubtarget().is64Bit()">;
-
 // Shader Model Support
 def FDivNeedsRoundingMode : Predicate<"getSubtarget().fdivNeedsRoundingMode()">;
 def FDivNoRoundingMode : Predicate<"!getSubtarget().fdivNeedsRoundingMode()">;
@@ -43,130 +39,19 @@ def DoesNotSupportPTX23 : Predicate<"!getSubtarget().supportsPTX23()">;
 def SupportsFMA         : Predicate<"getSubtarget().supportsFMA()">;
 def DoesNotSupportFMA   : Predicate<"!getSubtarget().supportsFMA()">;
 
-//===----------------------------------------------------------------------===//
-// Instruction Pattern Stuff
-//===----------------------------------------------------------------------===//
 
-def load_global : PatFrag<(ops node:$ptr), (load node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::GLOBAL;
-  return false;
-}]>;
 
-def load_constant : PatFrag<(ops node:$ptr), (load node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::CONSTANT;
-  return false;
-}]>;
+// def SDT_PTXCallSeqStart : SDCallSeqStart<[SDTCisVT<0, i32>]>;
+// def SDT_PTXCallSeqEnd   : SDCallSeqEnd<[SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
 
-def load_local : PatFrag<(ops node:$ptr), (load node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::LOCAL;
-  return false;
-}]>;
+// def PTXcallseq_start : SDNode<"ISD::CALLSEQ_START", SDT_PTXCallSeqStart,
+//                               [SDNPHasChain, SDNPOutGlue]>;
+// def PTXcallseq_end   : SDNode<"ISD::CALLSEQ_END", SDT_PTXCallSeqEnd,
+//                               [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
-def load_parameter : PatFrag<(ops node:$ptr), (load node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::PARAMETER;
-  return false;
-}]>;
+def PTXcall : SDNode<"PTXISD::CALL", SDTNone,
+                     [SDNPHasChain, SDNPVariadic, SDNPOptInGlue, SDNPOutGlue]>;
 
-def load_shared : PatFrag<(ops node:$ptr), (load node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::SHARED;
-  return false;
-}]>;
-
-def store_global
-  : PatFrag<(ops node:$d, node:$ptr), (store node:$d, node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<StoreSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::GLOBAL;
-  return false;
-}]>;
-
-def store_local
-  : PatFrag<(ops node:$d, node:$ptr), (store node:$d, node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<StoreSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::LOCAL;
-  return false;
-}]>;
-
-def store_parameter
-  : PatFrag<(ops node:$d, node:$ptr), (store node:$d, node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<StoreSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::PARAMETER;
-  return false;
-}]>;
-
-def store_shared
-  : PatFrag<(ops node:$d, node:$ptr), (store node:$d, node:$ptr), [{
-  const Value *Src;
-  const PointerType *PT;
-  if ((Src = cast<StoreSDNode>(N)->getSrcValue()) &&
-      (PT = dyn_cast<PointerType>(Src->getType())))
-    return PT->getAddressSpace() == PTX::SHARED;
-  return false;
-}]>;
-
-// Addressing modes.
-def ADDRrr32 : ComplexPattern<i32, 2, "SelectADDRrr", [], []>;
-def ADDRrr64 : ComplexPattern<i64, 2, "SelectADDRrr", [], []>;
-def ADDRri32 : ComplexPattern<i32, 2, "SelectADDRri", [], []>;
-def ADDRri64 : ComplexPattern<i64, 2, "SelectADDRri", [], []>;
-def ADDRii32 : ComplexPattern<i32, 2, "SelectADDRii", [], []>;
-def ADDRii64 : ComplexPattern<i64, 2, "SelectADDRii", [], []>;
-
-// Address operands
-def MEMri32 : Operand<i32> {
-  let PrintMethod = "printMemOperand";
-  let MIOperandInfo = (ops RegI32, i32imm);
-}
-def MEMri64 : Operand<i64> {
-  let PrintMethod = "printMemOperand";
-  let MIOperandInfo = (ops RegI64, i64imm);
-}
-def MEMii32 : Operand<i32> {
-  let PrintMethod = "printMemOperand";
-  let MIOperandInfo = (ops i32imm, i32imm);
-}
-def MEMii64 : Operand<i64> {
-  let PrintMethod = "printMemOperand";
-  let MIOperandInfo = (ops i64imm, i64imm);
-}
-// The operand here does not correspond to an actual address, so we
-// can use i32 in 64-bit address modes.
-def MEMpi : Operand<i32> {
-  let PrintMethod = "printParamOperand";
-  let MIOperandInfo = (ops i32imm);
-}
-def MEMret : Operand<i32> {
-  let PrintMethod = "printReturnOperand";
-  let MIOperandInfo = (ops i32imm);
-}
 
 // Branch & call targets have OtherVT type.
 def brtarget   : Operand<OtherVT>;
@@ -189,87 +74,73 @@ def PTXret
 def PTXcopyaddress
   : SDNode<"PTXISD::COPY_ADDRESS", SDTypeProfile<1, 1, []>, []>;
 
-// Load/store .param space
-def PTXloadparam
-  : SDNode<"PTXISD::LOAD_PARAM", SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>,
-           [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue]>;
-def PTXstoreparam
-  : SDNode<"PTXISD::STORE_PARAM", SDTypeProfile<0, 2, [SDTCisVT<0, i32>]>,
-           [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue]>;
+
 
 //===----------------------------------------------------------------------===//
 // Instruction Class Templates
 //===----------------------------------------------------------------------===//
 
+// For floating-point instructions, we cannot just embed the pattern into the
+// instruction definition since we need to muck around with the rounding mode,
+// and I do not know how to insert constants into instructions directly from
+// pattern matches.
+
 //===- Floating-Point Instructions - 2 Operand Form -----------------------===//
-multiclass PTX_FLOAT_2OP<string opcstr, SDNode opnode> {
+multiclass PTX_FLOAT_2OP<string opcstr> {
   def rr32 : InstPTX<(outs RegF32:$d),
-                     (ins RegF32:$a),
-                     !strconcat(opcstr, ".f32\t$d, $a"),
-                     [(set RegF32:$d, (opnode RegF32:$a))]>;
+                     (ins RndMode:$r, RegF32:$a),
+                     !strconcat(opcstr, "$r.f32\t$d, $a"), []>;
   def ri32 : InstPTX<(outs RegF32:$d),
-                     (ins f32imm:$a),
-                     !strconcat(opcstr, ".f32\t$d, $a"),
-                     [(set RegF32:$d, (opnode fpimm:$a))]>;
+                     (ins RndMode:$r, f32imm:$a),
+                     !strconcat(opcstr, "$r.f32\t$d, $a"), []>;
   def rr64 : InstPTX<(outs RegF64:$d),
-                     (ins RegF64:$a),
-                     !strconcat(opcstr, ".f64\t$d, $a"),
-                     [(set RegF64:$d, (opnode RegF64:$a))]>;
+                     (ins RndMode:$r, RegF64:$a),
+                     !strconcat(opcstr, "$r.f64\t$d, $a"), []>;
   def ri64 : InstPTX<(outs RegF64:$d),
-                     (ins f64imm:$a),
-                     !strconcat(opcstr, ".f64\t$d, $a"),
-                     [(set RegF64:$d, (opnode fpimm:$a))]>;
+                     (ins RndMode:$r, f64imm:$a),
+                     !strconcat(opcstr, "$r.f64\t$d, $a"), []>;
 }
 
 //===- Floating-Point Instructions - 3 Operand Form -----------------------===//
-multiclass PTX_FLOAT_3OP<string opcstr, SDNode opnode> {
+multiclass PTX_FLOAT_3OP<string opcstr> {
   def rr32 : InstPTX<(outs RegF32:$d),
-                     (ins RegF32:$a, RegF32:$b),
-                     !strconcat(opcstr, ".f32\t$d, $a, $b"),
-                     [(set RegF32:$d, (opnode RegF32:$a, RegF32:$b))]>;
+                     (ins RndMode:$r, RegF32:$a, RegF32:$b),
+                     !strconcat(opcstr, "$r.f32\t$d, $a, $b"), []>;
   def ri32 : InstPTX<(outs RegF32:$d),
-                     (ins RegF32:$a, f32imm:$b),
-                     !strconcat(opcstr, ".f32\t$d, $a, $b"),
-                     [(set RegF32:$d, (opnode RegF32:$a, fpimm:$b))]>;
+                     (ins RndMode:$r, RegF32:$a, f32imm:$b),
+                     !strconcat(opcstr, "$r.f32\t$d, $a, $b"), []>;
   def rr64 : InstPTX<(outs RegF64:$d),
-                     (ins RegF64:$a, RegF64:$b),
-                     !strconcat(opcstr, ".f64\t$d, $a, $b"),
-                     [(set RegF64:$d, (opnode RegF64:$a, RegF64:$b))]>;
+                     (ins RndMode:$r, RegF64:$a, RegF64:$b),
+                     !strconcat(opcstr, "$r.f64\t$d, $a, $b"), []>;
   def ri64 : InstPTX<(outs RegF64:$d),
-                     (ins RegF64:$a, f64imm:$b),
-                     !strconcat(opcstr, ".f64\t$d, $a, $b"),
-                     [(set RegF64:$d, (opnode RegF64:$a, fpimm:$b))]>;
+                     (ins RndMode:$r, RegF64:$a, f64imm:$b),
+                     !strconcat(opcstr, "$r.f64\t$d, $a, $b"), []>;
 }
 
 //===- Floating-Point Instructions - 4 Operand Form -----------------------===//
-multiclass PTX_FLOAT_4OP<string opcstr, SDNode opnode1, SDNode opnode2> {
+multiclass PTX_FLOAT_4OP<string opcstr> {
   def rrr32 : InstPTX<(outs RegF32:$d),
-                      (ins RegF32:$a, RegF32:$b, RegF32:$c),
-                      !strconcat(opcstr, ".f32\t$d, $a, $b, $c"),
-                      [(set RegF32:$d, (opnode2 (opnode1 RegF32:$a,
-                                                          RegF32:$b),
-                                                 RegF32:$c))]>;
+                      (ins RndMode:$r, RegF32:$a, RegF32:$b, RegF32:$c),
+                      !strconcat(opcstr, "$r.f32\t$d, $a, $b, $c"), []>;
   def rri32 : InstPTX<(outs RegF32:$d),
-                      (ins RegF32:$a, RegF32:$b, f32imm:$c),
-                      !strconcat(opcstr, ".f32\t$d, $a, $b, $c"),
-                      [(set RegF32:$d, (opnode2 (opnode1 RegF32:$a,
-                                                          RegF32:$b),
-                                                 fpimm:$c))]>;
+                      (ins RndMode:$r, RegF32:$a, RegF32:$b, f32imm:$c),
+                      !strconcat(opcstr, "$r.f32\t$d, $a, $b, $c"), []>;
+  def rii32 : InstPTX<(outs RegF32:$d),
+                      (ins RndMode:$r, RegF32:$a, f32imm:$b, f32imm:$c),
+                      !strconcat(opcstr, "$r.f32\t$d, $a, $b, $c"), []>;
   def rrr64 : InstPTX<(outs RegF64:$d),
-                      (ins RegF64:$a, RegF64:$b, RegF64:$c),
-                      !strconcat(opcstr, ".f64\t$d, $a, $b, $c"),
-                      [(set RegF64:$d, (opnode2 (opnode1 RegF64:$a,
-                                                          RegF64:$b),
-                                                 RegF64:$c))]>;
+                      (ins RndMode:$r, RegF64:$a, RegF64:$b, RegF64:$c),
+                      !strconcat(opcstr, "$r.f64\t$d, $a, $b, $c"), []>;
   def rri64 : InstPTX<(outs RegF64:$d),
-                      (ins RegF64:$a, RegF64:$b, f64imm:$c),
-                      !strconcat(opcstr, ".f64\t$d, $a, $b, $c"),
-                      [(set RegF64:$d, (opnode2 (opnode1 RegF64:$a,
-                                                          RegF64:$b),
-                                                 fpimm:$c))]>;
+                      (ins RndMode:$r, RegF64:$a, RegF64:$b, f64imm:$c),
+                      !strconcat(opcstr, "$r.f64\t$d, $a, $b, $c"), []>;
+  def rii64 : InstPTX<(outs RegF64:$d),
+                      (ins RndMode:$r, RegF64:$a, f64imm:$b, f64imm:$c),
+                      !strconcat(opcstr, "$r.f64\t$d, $a, $b, $c"), []>;
 }
 
-multiclass INT3<string opcstr, SDNode opnode> {
+//===- Integer Instructions - 3 Operand Form ------------------------------===//
+multiclass PTX_INT3<string opcstr, SDNode opnode> {
   def rr16 : InstPTX<(outs RegI16:$d),
                      (ins RegI16:$a, RegI16:$b),
                      !strconcat(opcstr, ".u16\t$d, $a, $b"),
@@ -296,6 +167,35 @@ multiclass INT3<string opcstr, SDNode opnode> {
                      [(set RegI64:$d, (opnode RegI64:$a, imm:$b))]>;
 }
 
+//===- Integer Instructions - 3 Operand Form (Signed) ---------------------===//
+multiclass PTX_INT3_SIGNED<string opcstr, SDNode opnode> {
+  def rr16 : InstPTX<(outs RegI16:$d),
+                     (ins RegI16:$a, RegI16:$b),
+                     !strconcat(opcstr, ".s16\t$d, $a, $b"),
+                     [(set RegI16:$d, (opnode RegI16:$a, RegI16:$b))]>;
+  def ri16 : InstPTX<(outs RegI16:$d),
+                     (ins RegI16:$a, i16imm:$b),
+                     !strconcat(opcstr, ".s16\t$d, $a, $b"),
+                     [(set RegI16:$d, (opnode RegI16:$a, imm:$b))]>;
+  def rr32 : InstPTX<(outs RegI32:$d),
+                     (ins RegI32:$a, RegI32:$b),
+                     !strconcat(opcstr, ".s32\t$d, $a, $b"),
+                     [(set RegI32:$d, (opnode RegI32:$a, RegI32:$b))]>;
+  def ri32 : InstPTX<(outs RegI32:$d),
+                     (ins RegI32:$a, i32imm:$b),
+                     !strconcat(opcstr, ".s32\t$d, $a, $b"),
+                     [(set RegI32:$d, (opnode RegI32:$a, imm:$b))]>;
+  def rr64 : InstPTX<(outs RegI64:$d),
+                     (ins RegI64:$a, RegI64:$b),
+                     !strconcat(opcstr, ".s64\t$d, $a, $b"),
+                     [(set RegI64:$d, (opnode RegI64:$a, RegI64:$b))]>;
+  def ri64 : InstPTX<(outs RegI64:$d),
+                     (ins RegI64:$a, i64imm:$b),
+                     !strconcat(opcstr, ".s64\t$d, $a, $b"),
+                     [(set RegI64:$d, (opnode RegI64:$a, imm:$b))]>;
+}
+
+//===- Bitwise Logic Instructions - 3 Operand Form ------------------------===//
 multiclass PTX_LOGIC<string opcstr, SDNode opnode> {
   def ripreds : InstPTX<(outs RegPred:$d),
                      (ins RegPred:$a, i1imm:$b),
@@ -331,7 +231,8 @@ multiclass PTX_LOGIC<string opcstr, SDNode opnode> {
                      [(set RegI64:$d, (opnode RegI64:$a, imm:$b))]>;
 }
 
-multiclass INT3ntnc<string opcstr, SDNode opnode> {
+//===- Integer Shift Instructions - 3 Operand Form ------------------------===//
+multiclass PTX_INT3ntnc<string opcstr, SDNode opnode> {
   def rr16 : InstPTX<(outs RegI16:$d),
                      (ins RegI16:$a, RegI16:$b),
                      !strconcat(opcstr, "16\t$d, $a, $b"),
@@ -370,6 +271,7 @@ multiclass INT3ntnc<string opcstr, SDNode opnode> {
                      [(set RegI64:$d, (opnode imm:$a, RegI64:$b))]>;
 }
 
+//===- Set Predicate Instructions (Int) - 3/4 Operand Forms ---------------===//
 multiclass PTX_SETP_I<RegisterClass RC, string regclsname, Operand immcls,
                         CondCode cmp, string cmpstr> {
   // TODO support 5-operand format: p|q, a, b, c
@@ -385,56 +287,77 @@ multiclass PTX_SETP_I<RegisterClass RC, string regclsname, Operand immcls,
 
   def rr_and_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".and.", regclsname, "\t$p, $a, $b, $c"),
+              !strconcat("setp.", cmpstr, ".and.", regclsname,
+                         "\t$p, $a, $b, $c"),
               [(set RegPred:$p, (and (setcc RC:$a, RC:$b, cmp), RegPred:$c))]>;
   def ri_and_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".and.", regclsname, "\t$p, $a, $b, $c"),
-              [(set RegPred:$p, (and (setcc RC:$a, imm:$b, cmp), RegPred:$c))]>;
+              !strconcat("setp.", cmpstr, ".and.", regclsname,
+                         "\t$p, $a, $b, $c"),
+              [(set RegPred:$p, (and (setcc RC:$a, imm:$b, cmp),
+                                     RegPred:$c))]>;
   def rr_or_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".or.", regclsname, "\t$p, $a, $b, $c"),
+              !strconcat("setp.", cmpstr, ".or.", regclsname,
+                         "\t$p, $a, $b, $c"),
               [(set RegPred:$p, (or (setcc RC:$a, RC:$b, cmp), RegPred:$c))]>;
   def ri_or_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".or.", regclsname, "\t$p, $a, $b, $c"),
+              !strconcat("setp.", cmpstr, ".or.", regclsname,
+                         "\t$p, $a, $b, $c"),
               [(set RegPred:$p, (or (setcc RC:$a, imm:$b, cmp), RegPred:$c))]>;
   def rr_xor_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".xor.", regclsname, "\t$p, $a, $b, $c"),
+              !strconcat("setp.", cmpstr, ".xor.", regclsname,
+                         "\t$p, $a, $b, $c"),
               [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, cmp), RegPred:$c))]>;
   def ri_xor_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".xor.", regclsname, "\t$p, $a, $b, $c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, imm:$b, cmp), RegPred:$c))]>;
+              !strconcat("setp.", cmpstr, ".xor.", regclsname,
+                         "\t$p, $a, $b, $c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, imm:$b, cmp),
+                                     RegPred:$c))]>;
 
   def rr_and_not_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".and.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, cmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".and.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, cmp),
+                                     (not RegPred:$c)))]>;
   def ri_and_not_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".and.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (and (setcc RC:$a, imm:$b, cmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".and.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (and (setcc RC:$a, imm:$b, cmp),
+                                     (not RegPred:$c)))]>;
   def rr_or_not_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".or.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (or (setcc RC:$a, RC:$b, cmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".or.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (or (setcc RC:$a, RC:$b, cmp),
+                                    (not RegPred:$c)))]>;
   def ri_or_not_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".or.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (or (setcc RC:$a, imm:$b, cmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".or.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (or (setcc RC:$a, imm:$b, cmp),
+                                    (not RegPred:$c)))]>;
   def rr_xor_not_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".xor.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, cmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".xor.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, cmp),
+                                     (not RegPred:$c)))]>;
   def ri_xor_not_r
     : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".xor.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, imm:$b, cmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".xor.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, imm:$b, cmp),
+                                     (not RegPred:$c)))]>;
 }
 
-multiclass PTX_SETP_FP<RegisterClass RC, string regclsname,
+//===- Set Predicate Instructions (FP) - 3/4 Operand Form -----------------===//
+multiclass PTX_SETP_FP<RegisterClass RC, string regclsname, Operand immcls,
                         CondCode ucmp, CondCode ocmp, string cmpstr> {
   // TODO support 5-operand format: p|q, a, b, c
 
@@ -447,137 +370,110 @@ multiclass PTX_SETP_FP<RegisterClass RC, string regclsname,
               !strconcat("setp.", cmpstr, ".", regclsname, "\t$p, $a, $b"),
               [(set RegPred:$p, (setcc RC:$a, RC:$b, ocmp))]>;
 
+  def ri_u
+    : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b),
+              !strconcat("setp.", cmpstr, "u.", regclsname, "\t$p, $a, $b"),
+              [(set RegPred:$p, (setcc RC:$a, fpimm:$b, ucmp))]>;
+  def ri_o
+    : InstPTX<(outs RegPred:$p), (ins RC:$a, immcls:$b),
+              !strconcat("setp.", cmpstr, ".", regclsname, "\t$p, $a, $b"),
+              [(set RegPred:$p, (setcc RC:$a, fpimm:$b, ocmp))]>;
+
   def rr_and_r_u
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, "u.and.", regclsname, "\t$p, $a, $b, $c"),
-              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ucmp), RegPred:$c))]>;
+              !strconcat("setp.", cmpstr, "u.and.", regclsname,
+                         "\t$p, $a, $b, $c"),
+              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ucmp),
+                                     RegPred:$c))]>;
   def rr_and_r_o
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".and.", regclsname, "\t$p, $a, $b, $c"),
-              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ocmp), RegPred:$c))]>;
+              !strconcat("setp.", cmpstr, ".and.", regclsname,
+                         "\t$p, $a, $b, $c"),
+              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ocmp),
+                                     RegPred:$c))]>;
 
   def rr_or_r_u
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, "u.or.", regclsname, "\t$p, $a, $b, $c"),
+              !strconcat("setp.", cmpstr, "u.or.", regclsname,
+                         "\t$p, $a, $b, $c"),
               [(set RegPred:$p, (or (setcc RC:$a, RC:$b, ucmp), RegPred:$c))]>;
   def rr_or_r_o
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".or.", regclsname, "\t$p, $a, $b, $c"),
+              !strconcat("setp.", cmpstr, ".or.", regclsname,
+                         "\t$p, $a, $b, $c"),
               [(set RegPred:$p, (or (setcc RC:$a, RC:$b, ocmp), RegPred:$c))]>;
 
   def rr_xor_r_u
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, "u.xor.", regclsname, "\t$p, $a, $b, $c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ucmp), RegPred:$c))]>;
+              !strconcat("setp.", cmpstr, "u.xor.", regclsname,
+                         "\t$p, $a, $b, $c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ucmp),
+                                     RegPred:$c))]>;
   def rr_xor_r_o
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".xor.", regclsname, "\t$p, $a, $b, $c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ocmp), RegPred:$c))]>;
+              !strconcat("setp.", cmpstr, ".xor.", regclsname,
+                         "\t$p, $a, $b, $c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ocmp),
+                                     RegPred:$c))]>;
 
   def rr_and_not_r_u
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, "u.and.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ucmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, "u.and.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ucmp),
+                                     (not RegPred:$c)))]>;
   def rr_and_not_r_o
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".and.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ocmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".and.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (and (setcc RC:$a, RC:$b, ocmp),
+                                     (not RegPred:$c)))]>;
 
   def rr_or_not_r_u
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, "u.or.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (or (setcc RC:$a, RC:$b, ucmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, "u.or.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (or (setcc RC:$a, RC:$b, ucmp),
+                                    (not RegPred:$c)))]>;
   def rr_or_not_r_o
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".or.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (or (setcc RC:$a, RC:$b, ocmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".or.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (or (setcc RC:$a, RC:$b, ocmp),
+                                    (not RegPred:$c)))]>;
 
   def rr_xor_not_r_u
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, "u.xor.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ucmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, "u.xor.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ucmp),
+                                     (not RegPred:$c)))]>;
   def rr_xor_not_r_o
     : InstPTX<(outs RegPred:$p), (ins RC:$a, RC:$b, RegPred:$c),
-              !strconcat("setp.", cmpstr, ".xor.", regclsname, "\t$p, $a, $b, !$c"),
-              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ocmp), (not RegPred:$c)))]>;
+              !strconcat("setp.", cmpstr, ".xor.", regclsname,
+                         "\t$p, $a, $b, !$c"),
+              [(set RegPred:$p, (xor (setcc RC:$a, RC:$b, ocmp),
+                                     (not RegPred:$c)))]>;
 }
 
-multiclass PTX_SELP<RegisterClass RC, string regclsname> {
+//===- Select Predicate Instructions - 4 Operand Form ---------------------===//
+multiclass PTX_SELP<RegisterClass RC, string regclsname, Operand immcls,
+                    SDNode immnode> {
   def rr
     : InstPTX<(outs RC:$r), (ins RegPred:$a, RC:$b, RC:$c),
               !strconcat("selp.", regclsname, "\t$r, $b, $c, $a"),
               [(set RC:$r, (select RegPred:$a, RC:$b, RC:$c))]>;
+  def ri
+    : InstPTX<(outs RC:$r), (ins RegPred:$a, RC:$b, immcls:$c),
+              !strconcat("selp.", regclsname, "\t$r, $b, $c, $a"),
+              [(set RC:$r, (select RegPred:$a, RC:$b, immnode:$c))]>;
+  def ii
+    : InstPTX<(outs RC:$r), (ins RegPred:$a, immcls:$b, immcls:$c),
+              !strconcat("selp.", regclsname, "\t$r, $b, $c, $a"),
+              [(set RC:$r, (select RegPred:$a, immnode:$b, immnode:$c))]>;
 }
 
-multiclass PTX_LD<string opstr, string typestr, RegisterClass RC, PatFrag pat_load> {
-  def rr32 : InstPTX<(outs RC:$d),
-                     (ins MEMri32:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
-                     [(set RC:$d, (pat_load ADDRrr32:$a))]>, Requires<[Use32BitAddresses]>;
-  def rr64 : InstPTX<(outs RC:$d),
-                     (ins MEMri64:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
-                     [(set RC:$d, (pat_load ADDRrr64:$a))]>, Requires<[Use64BitAddresses]>;
-  def ri32 : InstPTX<(outs RC:$d),
-                     (ins MEMri32:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
-                     [(set RC:$d, (pat_load ADDRri32:$a))]>, Requires<[Use32BitAddresses]>;
-  def ri64 : InstPTX<(outs RC:$d),
-                     (ins MEMri64:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
-                     [(set RC:$d, (pat_load ADDRri64:$a))]>, Requires<[Use64BitAddresses]>;
-  def ii32 : InstPTX<(outs RC:$d),
-                     (ins MEMii32:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
-                     [(set RC:$d, (pat_load ADDRii32:$a))]>, Requires<[Use32BitAddresses]>;
-  def ii64 : InstPTX<(outs RC:$d),
-                     (ins MEMii64:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
-                     [(set RC:$d, (pat_load ADDRii64:$a))]>, Requires<[Use64BitAddresses]>;
-}
 
-multiclass PTX_LD_ALL<string opstr, PatFrag pat_load> {
-  defm u16 : PTX_LD<opstr, ".u16", RegI16, pat_load>;
-  defm u32 : PTX_LD<opstr, ".u32", RegI32, pat_load>;
-  defm u64 : PTX_LD<opstr, ".u64", RegI64, pat_load>;
-  defm f32 : PTX_LD<opstr, ".f32", RegF32, pat_load>;
-  defm f64 : PTX_LD<opstr, ".f64", RegF64, pat_load>;
-}
-
-multiclass PTX_ST<string opstr, string typestr, RegisterClass RC, PatFrag pat_store> {
-  def rr32 : InstPTX<(outs),
-                     (ins RC:$d, MEMri32:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
-                     [(pat_store RC:$d, ADDRrr32:$a)]>, Requires<[Use32BitAddresses]>;
-  def rr64 : InstPTX<(outs),
-                     (ins RC:$d, MEMri64:$a),
-                     !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
-                     [(pat_store RC:$d, ADDRrr64:$a)]>, Requires<[Use64BitAddresses]>;
-  def ri32 : InstPTX<(outs),
-                   (ins RC:$d, MEMri32:$a),
-                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
-                   [(pat_store RC:$d, ADDRri32:$a)]>, Requires<[Use32BitAddresses]>;
-  def ri64 : InstPTX<(outs),
-                   (ins RC:$d, MEMri64:$a),
-                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
-                   [(pat_store RC:$d, ADDRri64:$a)]>, Requires<[Use64BitAddresses]>;
-  def ii32 : InstPTX<(outs),
-                   (ins RC:$d, MEMii32:$a),
-                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
-                   [(pat_store RC:$d, ADDRii32:$a)]>, Requires<[Use32BitAddresses]>;
-  def ii64 : InstPTX<(outs),
-                   (ins RC:$d, MEMii64:$a),
-                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
-                   [(pat_store RC:$d, ADDRii64:$a)]>, Requires<[Use64BitAddresses]>;
-}
-
-multiclass PTX_ST_ALL<string opstr, PatFrag pat_store> {
-  defm u16 : PTX_ST<opstr, ".u16", RegI16, pat_store>;
-  defm u32 : PTX_ST<opstr, ".u32", RegI32, pat_store>;
-  defm u64 : PTX_ST<opstr, ".u64", RegI64, pat_store>;
-  defm f32 : PTX_ST<opstr, ".f32", RegF32, pat_store>;
-  defm f64 : PTX_ST<opstr, ".f64", RegF64, pat_store>;
-}
 
 //===----------------------------------------------------------------------===//
 // Instructions
@@ -585,118 +481,61 @@ multiclass PTX_ST_ALL<string opstr, PatFrag pat_store> {
 
 ///===- Integer Arithmetic Instructions -----------------------------------===//
 
-defm ADD : INT3<"add", add>;
-defm SUB : INT3<"sub", sub>;
-defm MUL : INT3<"mul.lo", mul>; // FIXME: Allow 32x32 -> 64 multiplies
-defm DIV : INT3<"div", udiv>;
-defm REM : INT3<"rem", urem>;
+defm ADD  : PTX_INT3<"add", add>;
+defm SUB  : PTX_INT3<"sub", sub>;
+defm MUL  : PTX_INT3<"mul.lo", mul>; // FIXME: Allow 32x32 -> 64 multiplies
+defm DIV  : PTX_INT3<"div", udiv>;
+defm SDIV : PTX_INT3_SIGNED<"div", sdiv>;
+defm REM  : PTX_INT3<"rem", urem>;
 
 ///===- Floating-Point Arithmetic Instructions ----------------------------===//
 
-// Standard Unary Operations
-defm FNEG : PTX_FLOAT_2OP<"neg", fneg>;
+// FNEG
+defm FNEG : PTX_FLOAT_2OP<"neg">;
 
 // Standard Binary Operations
-defm FADD : PTX_FLOAT_3OP<"add.rn", fadd>;
-defm FSUB : PTX_FLOAT_3OP<"sub.rn", fsub>;
-defm FMUL : PTX_FLOAT_3OP<"mul.rn", fmul>;
-
-// For floating-point division:
-// SM_13+ defaults to .rn for f32 and f64,
-// SM10 must *not* provide a rounding
-
-// TODO: 
-//     - Allow user selection of rounding modes for fdiv
-//     - Add support for -prec-div=false (.approx)
-
-def FDIVrr32SM13 : InstPTX<(outs RegF32:$d),
-                       (ins RegF32:$a, RegF32:$b),
-                       "div.rn.f32\t$d, $a, $b",
-                       [(set RegF32:$d, (fdiv RegF32:$a, RegF32:$b))]>,
-                   Requires<[FDivNeedsRoundingMode]>;
-def FDIVri32SM13 : InstPTX<(outs RegF32:$d),
-                       (ins RegF32:$a, f32imm:$b),
-                       "div.rn.f32\t$d, $a, $b",
-                       [(set RegF32:$d, (fdiv RegF32:$a, fpimm:$b))]>,
-                   Requires<[FDivNeedsRoundingMode]>;
-def FDIVrr32SM10 : InstPTX<(outs RegF32:$d),
-                       (ins RegF32:$a, RegF32:$b),
-                       "div.f32\t$d, $a, $b",
-                       [(set RegF32:$d, (fdiv RegF32:$a, RegF32:$b))]>,
-                   Requires<[FDivNoRoundingMode]>;
-def FDIVri32SM10 : InstPTX<(outs RegF32:$d),
-                       (ins RegF32:$a, f32imm:$b),
-                       "div.f32\t$d, $a, $b",
-                       [(set RegF32:$d, (fdiv RegF32:$a, fpimm:$b))]>,
-                   Requires<[FDivNoRoundingMode]>;
-
-def FDIVrr64SM13 : InstPTX<(outs RegF64:$d),
-                           (ins RegF64:$a, RegF64:$b),
-                           "div.rn.f64\t$d, $a, $b",
-                           [(set RegF64:$d, (fdiv RegF64:$a, RegF64:$b))]>,
-                   Requires<[FDivNeedsRoundingMode]>;
-def FDIVri64SM13 : InstPTX<(outs RegF64:$d),
-                           (ins RegF64:$a, f64imm:$b),
-                           "div.rn.f64\t$d, $a, $b",
-                           [(set RegF64:$d, (fdiv RegF64:$a, fpimm:$b))]>,
-                   Requires<[FDivNeedsRoundingMode]>;
-def FDIVrr64SM10 : InstPTX<(outs RegF64:$d),
-                           (ins RegF64:$a, RegF64:$b),
-                           "div.f64\t$d, $a, $b",
-                           [(set RegF64:$d, (fdiv RegF64:$a, RegF64:$b))]>,
-                   Requires<[FDivNoRoundingMode]>;
-def FDIVri64SM10 : InstPTX<(outs RegF64:$d),
-                           (ins RegF64:$a, f64imm:$b),
-                           "div.f64\t$d, $a, $b",
-                           [(set RegF64:$d, (fdiv RegF64:$a, fpimm:$b))]>,
-                   Requires<[FDivNoRoundingMode]>;
-
-
+defm FADD : PTX_FLOAT_3OP<"add">;
+defm FSUB : PTX_FLOAT_3OP<"sub">;
+defm FMUL : PTX_FLOAT_3OP<"mul">;
+defm FDIV : PTX_FLOAT_3OP<"div">;
 
 // Multi-operation hybrid instructions
+defm FMAD : PTX_FLOAT_4OP<"mad">, Requires<[SupportsFMA]>;
 
-// The selection of mad/fma is tricky.  In some cases, they are the *same*
-// instruction, but in other cases we may prefer one or the other.  Also,
-// different PTX versions differ on whether rounding mode flags are required.
-// In the short term, mad is supported on all PTX versions and we use a
-// default rounding mode no matter what shader model or PTX version.
-// TODO: Allow the rounding mode to be selectable through llc.
-defm FMADSM13 : PTX_FLOAT_4OP<"mad.rn", fmul, fadd>,
-                Requires<[FMadNeedsRoundingMode, SupportsFMA]>;
-defm FMAD : PTX_FLOAT_4OP<"mad", fmul, fadd>,
-            Requires<[FMadNoRoundingMode, SupportsFMA]>;
 
 ///===- Floating-Point Intrinsic Instructions -----------------------------===//
 
-def FSQRT32 : InstPTX<(outs RegF32:$d),
-                      (ins RegF32:$a),
-                      "sqrt.rn.f32\t$d, $a",
-                      [(set RegF32:$d, (fsqrt RegF32:$a))]>;
+// SQRT
+def FSQRTrr32 : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegF32:$a),
+                        "sqrt$r.f32\t$d, $a", []>;
+def FSQRTri32 : InstPTX<(outs RegF32:$d), (ins RndMode:$r, f32imm:$a),
+                        "sqrt$r.f32\t$d, $a", []>;
+def FSQRTrr64 : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegF64:$a),
+                        "sqrt$r.f64\t$d, $a", []>;
+def FSQRTri64 : InstPTX<(outs RegF64:$d), (ins RndMode:$r, f64imm:$a),
+                        "sqrt$r.f64\t$d, $a", []>;
 
-def FSQRT64 : InstPTX<(outs RegF64:$d),
-                      (ins RegF64:$a),
-                      "sqrt.rn.f64\t$d, $a",
-                      [(set RegF64:$d, (fsqrt RegF64:$a))]>;
+// SIN
+def FSINrr32 : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegF32:$a),
+                       "sin$r.f32\t$d, $a", []>;
+def FSINri32 : InstPTX<(outs RegF32:$d), (ins RndMode:$r, f32imm:$a),
+                       "sin$r.f32\t$d, $a", []>;
+def FSINrr64 : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegF64:$a),
+                       "sin$r.f64\t$d, $a", []>;
+def FSINri64 : InstPTX<(outs RegF64:$d), (ins RndMode:$r, f64imm:$a),
+                       "sin$r.f64\t$d, $a", []>;
 
-def FSIN32 : InstPTX<(outs RegF32:$d),
-                     (ins RegF32:$a),
-                     "sin.approx.f32\t$d, $a",
-                     [(set RegF32:$d, (fsin RegF32:$a))]>;
+// COS
+def FCOSrr32 : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegF32:$a),
+                       "cos$r.f32\t$d, $a", []>;
+def FCOSri32 : InstPTX<(outs RegF32:$d), (ins RndMode:$r, f32imm:$a),
+                       "cos$r.f32\t$d, $a", []>;
+def FCOSrr64 : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegF64:$a),
+                       "cos$r.f64\t$d, $a", []>;
+def FCOSri64 : InstPTX<(outs RegF64:$d), (ins RndMode:$r, f64imm:$a),
+                       "cos$r.f64\t$d, $a", []>;
 
-def FSIN64 : InstPTX<(outs RegF64:$d),
-                     (ins RegF64:$a),
-                     "sin.approx.f64\t$d, $a",
-                     [(set RegF64:$d, (fsin RegF64:$a))]>;
 
-def FCOS32 : InstPTX<(outs RegF32:$d),
-                     (ins RegF32:$a),
-                     "cos.approx.f32\t$d, $a",
-                     [(set RegF32:$d, (fcos RegF32:$a))]>;
-
-def FCOS64 : InstPTX<(outs RegF64:$d),
-                     (ins RegF64:$a),
-                     "cos.approx.f64\t$d, $a",
-                     [(set RegF64:$d, (fcos RegF64:$a))]>;
 
 
 ///===- Comparison and Selection Instructions -----------------------------===//
@@ -744,35 +583,35 @@ defm SETPGEs64 : PTX_SETP_I<RegI64, "s64", i64imm, SETGE,  "ge">;
 
 // Compare f32
 
-defm SETPEQf32 : PTX_SETP_FP<RegF32, "f32", SETUEQ, SETOEQ, "eq">;
-defm SETPNEf32 : PTX_SETP_FP<RegF32, "f32", SETUNE, SETONE, "ne">;
-defm SETPLTf32 : PTX_SETP_FP<RegF32, "f32", SETULT, SETOLT, "lt">;
-defm SETPLEf32 : PTX_SETP_FP<RegF32, "f32", SETULE, SETOLE, "le">;
-defm SETPGTf32 : PTX_SETP_FP<RegF32, "f32", SETUGT, SETOGT, "gt">;
-defm SETPGEf32 : PTX_SETP_FP<RegF32, "f32", SETUGE, SETOGE, "ge">;
+defm SETPEQf32 : PTX_SETP_FP<RegF32, "f32", f32imm, SETUEQ, SETOEQ, "eq">;
+defm SETPNEf32 : PTX_SETP_FP<RegF32, "f32", f32imm, SETUNE, SETONE, "ne">;
+defm SETPLTf32 : PTX_SETP_FP<RegF32, "f32", f32imm, SETULT, SETOLT, "lt">;
+defm SETPLEf32 : PTX_SETP_FP<RegF32, "f32", f32imm, SETULE, SETOLE, "le">;
+defm SETPGTf32 : PTX_SETP_FP<RegF32, "f32", f32imm, SETUGT, SETOGT, "gt">;
+defm SETPGEf32 : PTX_SETP_FP<RegF32, "f32", f32imm, SETUGE, SETOGE, "ge">;
 
 // Compare f64
 
-defm SETPEQf64 : PTX_SETP_FP<RegF64, "f64", SETUEQ, SETOEQ, "eq">;
-defm SETPNEf64 : PTX_SETP_FP<RegF64, "f64", SETUNE, SETONE, "ne">;
-defm SETPLTf64 : PTX_SETP_FP<RegF64, "f64", SETULT, SETOLT, "lt">;
-defm SETPLEf64 : PTX_SETP_FP<RegF64, "f64", SETULE, SETOLE, "le">;
-defm SETPGTf64 : PTX_SETP_FP<RegF64, "f64", SETUGT, SETOGT, "gt">;
-defm SETPGEf64 : PTX_SETP_FP<RegF64, "f64", SETUGE, SETOGE, "ge">;
+defm SETPEQf64 : PTX_SETP_FP<RegF64, "f64", f64imm, SETUEQ, SETOEQ, "eq">;
+defm SETPNEf64 : PTX_SETP_FP<RegF64, "f64", f64imm, SETUNE, SETONE, "ne">;
+defm SETPLTf64 : PTX_SETP_FP<RegF64, "f64", f64imm, SETULT, SETOLT, "lt">;
+defm SETPLEf64 : PTX_SETP_FP<RegF64, "f64", f64imm, SETULE, SETOLE, "le">;
+defm SETPGTf64 : PTX_SETP_FP<RegF64, "f64", f64imm, SETUGT, SETOGT, "gt">;
+defm SETPGEf64 : PTX_SETP_FP<RegF64, "f64", f64imm, SETUGE, SETOGE, "ge">;
 
 // .selp
 
-defm PTX_SELPu16 : PTX_SELP<RegI16, "u16">;
-defm PTX_SELPu32 : PTX_SELP<RegI32, "u32">;
-defm PTX_SELPu64 : PTX_SELP<RegI64, "u64">;
-defm PTX_SELPf32 : PTX_SELP<RegF32, "f32">;
-defm PTX_SELPf64 : PTX_SELP<RegF64, "f64">;
+defm SELPi16 : PTX_SELP<RegI16, "u16", i16imm, imm>;
+defm SELPi32 : PTX_SELP<RegI32, "u32", i32imm, imm>;
+defm SELPi64 : PTX_SELP<RegI64, "u64", i64imm, imm>;
+defm SELPf32 : PTX_SELP<RegF32, "f32", f32imm, fpimm>;
+defm SELPf64 : PTX_SELP<RegF64, "f64", f64imm, fpimm>;
 
 ///===- Logic and Shift Instructions --------------------------------------===//
 
-defm SHL : INT3ntnc<"shl.b", PTXshl>;
-defm SRL : INT3ntnc<"shr.u", PTXsrl>;
-defm SRA : INT3ntnc<"shr.s", PTXsra>;
+defm SHL : PTX_INT3ntnc<"shl.b", PTXshl>;
+defm SRL : PTX_INT3ntnc<"shr.u", PTXsrl>;
+defm SRA : PTX_INT3ntnc<"shr.s", PTXsra>;
 
 defm AND : PTX_LOGIC<"and", and>;
 defm OR  : PTX_LOGIC<"or",  or>;
@@ -780,6 +619,24 @@ defm XOR : PTX_LOGIC<"xor", xor>;
 
 ///===- Data Movement and Conversion Instructions -------------------------===//
 
+// any_extend
+// Implement the anyext instruction in terms of the PTX cvt instructions.
+//def : Pat<(i32 (anyext RegI16:$a)), (CVT_u32_u16 RegI16:$a)>;
+//def : Pat<(i64 (anyext RegI16:$a)), (CVT_u64_u16 RegI16:$a)>;
+//def : Pat<(i64 (anyext RegI32:$a)), (CVT_u64_u32 RegI32:$a)>;
+
+// bitconvert
+// These instructions implement the bit-wise conversion between integer and
+// floating-point types.
+def MOVi32f32
+  : InstPTX<(outs RegI32:$d), (ins RegF32:$a), "mov.b32\t$d, $a", []>;
+def MOVf32i32
+  : InstPTX<(outs RegF32:$d), (ins RegI32:$a), "mov.b32\t$d, $a", []>;
+def MOVi64f64
+  : InstPTX<(outs RegI64:$d), (ins RegF64:$a), "mov.b64\t$d, $a", []>;
+def MOVf64i64
+  : InstPTX<(outs RegF64:$d), (ins RegI64:$a), "mov.b64\t$d, $a", []>;
+
 let neverHasSideEffects = 1 in {
   def MOVPREDrr
     : InstPTX<(outs RegPred:$d), (ins RegPred:$a), "mov.pred\t$d, $a", []>;
@@ -825,235 +682,117 @@ let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
               [(set RegI64:$d, (PTXcopyaddress tglobaladdr:$a))]>;
 }
 
-// Loads
-defm LDg : PTX_LD_ALL<"ld.global", load_global>;
-defm LDc : PTX_LD_ALL<"ld.const",  load_constant>;
-defm LDl : PTX_LD_ALL<"ld.local",  load_local>;
-defm LDs : PTX_LD_ALL<"ld.shared", load_shared>;
+// PTX cvt instructions
+// Note all of these may actually be used, we just define all possible patterns
+// here (that make sense).
+// FIXME: Can we collapse this somehow into a multiclass def?
 
-// These instructions are used to load/store from the .param space for
-// device and kernel parameters
+// To i16
+def CVTu16u32
+  : InstPTX<(outs RegI16:$d), (ins RegI32:$a), "cvt.u16.u32\t$d, $a", []>;
+def CVTu16u64
+  : InstPTX<(outs RegI16:$d), (ins RegI64:$a), "cvt.u16.u64\t$d, $a", []>;
+def CVTu16f32
+  : InstPTX<(outs RegI16:$d), (ins RndMode:$r, RegF32:$a),
+            "cvt$r.u16.f32\t$d, $a", []>;
+def CVTs16f32
+  : InstPTX<(outs RegI16:$d), (ins RndMode:$r, RegF32:$a),
+            "cvt$r.s16.f32\t$d, $a", []>;
+def CVTu16f64
+  : InstPTX<(outs RegI16:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.u16.f64\t$d, $a", []>;
+def CVTs16f64
+  : InstPTX<(outs RegI16:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.s16.f64\t$d, $a", []>;
 
-let hasSideEffects = 1 in {
-  def LDpiPred : InstPTX<(outs RegPred:$d), (ins MEMpi:$a),
-                         "ld.param.pred\t$d, [$a]",
-                         [(set RegPred:$d, (PTXloadparam timm:$a))]>;
-  def LDpiU16  : InstPTX<(outs RegI16:$d), (ins MEMpi:$a),
-                         "ld.param.u16\t$d, [$a]",
-                         [(set RegI16:$d, (PTXloadparam timm:$a))]>;
-  def LDpiU32  : InstPTX<(outs RegI32:$d), (ins MEMpi:$a),
-                         "ld.param.u32\t$d, [$a]",
-                         [(set RegI32:$d, (PTXloadparam timm:$a))]>;
-  def LDpiU64  : InstPTX<(outs RegI64:$d), (ins MEMpi:$a),
-                         "ld.param.u64\t$d, [$a]",
-                         [(set RegI64:$d, (PTXloadparam timm:$a))]>;
-  def LDpiF32  : InstPTX<(outs RegF32:$d), (ins MEMpi:$a),
-                         "ld.param.f32\t$d, [$a]",
-                         [(set RegF32:$d, (PTXloadparam timm:$a))]>;
-  def LDpiF64  : InstPTX<(outs RegF64:$d), (ins MEMpi:$a),
-                         "ld.param.f64\t$d, [$a]",
-                         [(set RegF64:$d, (PTXloadparam timm:$a))]>;
+// To i32
+def CVTu32u16
+  : InstPTX<(outs RegI32:$d), (ins RegI16:$a), "cvt.u32.u16\t$d, $a", []>;
+def CVTs32s16
+  : InstPTX<(outs RegI32:$d), (ins RegI16:$a), "cvt.s32.s16\t$d, $a", []>;
+def CVTu32u64
+  : InstPTX<(outs RegI32:$d), (ins RegI64:$a), "cvt.u32.u64\t$d, $a", []>;
+def CVTu32f32
+  : InstPTX<(outs RegI32:$d), (ins RndMode:$r, RegF32:$a),
+            "cvt$r.u32.f32\t$d, $a", []>;
+def CVTs32f32
+  : InstPTX<(outs RegI32:$d), (ins RndMode:$r, RegF32:$a),
+            "cvt$r.s32.f32\t$d, $a", []>;
+def CVTu32f64
+  : InstPTX<(outs RegI32:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.u32.f64\t$d, $a", []>;
+def CVTs32f64
+  : InstPTX<(outs RegI32:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.s32.f64\t$d, $a", []>;
 
-  def STpiPred : InstPTX<(outs), (ins MEMret:$d, RegPred:$a),
-                         "st.param.pred\t[$d], $a",
-                         [(PTXstoreparam timm:$d, RegPred:$a)]>;
-  def STpiU16  : InstPTX<(outs), (ins MEMret:$d, RegI16:$a),
-                         "st.param.u16\t[$d], $a",
-                         [(PTXstoreparam timm:$d, RegI16:$a)]>;
-  def STpiU32  : InstPTX<(outs), (ins MEMret:$d, RegI32:$a),
-                         "st.param.u32\t[$d], $a",
-                         [(PTXstoreparam timm:$d, RegI32:$a)]>;
-  def STpiU64  : InstPTX<(outs), (ins MEMret:$d, RegI64:$a),
-                         "st.param.u64\t[$d], $a",
-                         [(PTXstoreparam timm:$d, RegI64:$a)]>;
-  def STpiF32  : InstPTX<(outs), (ins MEMret:$d, RegF32:$a),
-                         "st.param.f32\t[$d], $a",
-                         [(PTXstoreparam timm:$d, RegF32:$a)]>;
-  def STpiF64  : InstPTX<(outs), (ins MEMret:$d, RegF64:$a),
-                         "st.param.f64\t[$d], $a",
-                         [(PTXstoreparam timm:$d, RegF64:$a)]>;
-}
+// To i64
+def CVTu64u16
+  : InstPTX<(outs RegI64:$d), (ins RegI16:$a), "cvt.u64.u16\t$d, $a", []>;
+def CVTs64s16
+  : InstPTX<(outs RegI64:$d), (ins RegI16:$a), "cvt.s64.s16\t$d, $a", []>;
+def CVTu64u32
+  : InstPTX<(outs RegI64:$d), (ins RegI32:$a), "cvt.u64.u32\t$d, $a", []>;
+def CVTs64s32
+  : InstPTX<(outs RegI64:$d), (ins RegI32:$a), "cvt.s64.s32\t$d, $a", []>;
+def CVTu64f32
+  : InstPTX<(outs RegI64:$d), (ins RndMode:$r, RegF32:$a),
+            "cvt$r.u64.f32\t$d, $a", []>;
+def CVTs64f32
+  : InstPTX<(outs RegI64:$d), (ins RndMode:$r, RegF32:$a),
+            "cvt$r.s64.f32\t$d, $a", []>;
+def CVTu64f64
+  : InstPTX<(outs RegI64:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.u64.f64\t$d, $a", []>;
+def CVTs64f64
+  : InstPTX<(outs RegI64:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.s64.f64\t$d, $a", []>;
 
-// Stores
-defm STg : PTX_ST_ALL<"st.global", store_global>;
-defm STl : PTX_ST_ALL<"st.local",  store_local>;
-defm STs : PTX_ST_ALL<"st.shared", store_shared>;
+// To f32
+def CVTf32u16
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegI16:$a),
+            "cvt$r.f32.u16\t$d, $a", []>;
+def CVTf32s16
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegI16:$a),
+            "cvt$r.f32.s16\t$d, $a", []>;
+def CVTf32u32
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegI32:$a),
+            "cvt$r.f32.u32\t$d, $a", []>;
+def CVTf32s32
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegI32:$a),
+            "cvt$r.f32.s32\t$d, $a", []>;
+def CVTf32u64
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegI64:$a),
+            "cvt$r.f32.u64\t$d, $a", []>;
+def CVTf32s64
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegI64:$a),
+            "cvt$r.f32.s64\t$d, $a", []>;
+def CVTf32f64
+  : InstPTX<(outs RegF32:$d), (ins RndMode:$r, RegF64:$a),
+            "cvt$r.f32.f64\t$d, $a", []>;
 
-// defm STp : PTX_ST_ALL<"st.param",  store_parameter>;
-// defm LDp : PTX_LD_ALL<"ld.param",  load_parameter>;
-// TODO: Do something with st.param if/when it is needed.
+// To f64
+def CVTf64u16
+  : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegI16:$a),
+            "cvt$r.f64.u16\t$d, $a", []>;
+def CVTf64s16
+  : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegI16:$a),
+            "cvt$r.f64.s16\t$d, $a", []>;
+def CVTf64u32
+  : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegI32:$a),
+            "cvt$r.f64.u32\t$d, $a", []>;
+def CVTf64s32
+  : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegI32:$a),
+            "cvt$r.f64.s32\t$d, $a", []>;
+def CVTf64u64
+  : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegI64:$a),
+            "cvt$r.f64.u64\t$d, $a", []>;
+def CVTf64s64
+  : InstPTX<(outs RegF64:$d), (ins RndMode:$r, RegI64:$a),
+            "cvt$r.f64.s64\t$d, $a", []>;
+def CVTf64f32
+  : InstPTX<(outs RegF64:$d), (ins RegF32:$a), "cvt.f64.f32\t$d, $a", []>;
 
-// Conversion to pred
-// PTX does not directly support converting to a predicate type, so we fake it
-// by performing a greater-than test between the value and zero.  This follows
-// the C convention that any non-zero value is equivalent to 'true'.
-def CVT_pred_u16
-  : InstPTX<(outs RegPred:$d), (ins RegI16:$a), "setp.gt.u16\t$d, $a, 0",
-            [(set RegPred:$d, (trunc RegI16:$a))]>;
-
-def CVT_pred_u32
-  : InstPTX<(outs RegPred:$d), (ins RegI32:$a), "setp.gt.u32\t$d, $a, 0",
-            [(set RegPred:$d, (trunc RegI32:$a))]>;
-
-def CVT_pred_u64
-  : InstPTX<(outs RegPred:$d), (ins RegI64:$a), "setp.gt.u64\t$d, $a, 0",
-            [(set RegPred:$d, (trunc RegI64:$a))]>;
-
-def CVT_pred_f32
-  : InstPTX<(outs RegPred:$d), (ins RegF32:$a), "setp.gt.f32\t$d, $a, 0",
-            [(set RegPred:$d, (fp_to_uint RegF32:$a))]>;
-
-def CVT_pred_f64
-  : InstPTX<(outs RegPred:$d), (ins RegF64:$a), "setp.gt.f64\t$d, $a, 0",
-            [(set RegPred:$d, (fp_to_uint RegF64:$a))]>;
-
-// Conversion to u16
-// PTX does not directly support converting a predicate to a value, so we
-// use a select instruction to select either 0 or 1 (integer or fp) based
-// on the truth value of the predicate.
-def CVT_u16_preda
-  : InstPTX<(outs RegI16:$d), (ins RegPred:$a), "selp.u16\t$d, 1, 0, $a",
-            [(set RegI16:$d, (anyext RegPred:$a))]>;
-
-def CVT_u16_pred
-  : InstPTX<(outs RegI16:$d), (ins RegPred:$a), "selp.u16\t$d, 1, 0, $a",
-            [(set RegI16:$d, (zext RegPred:$a))]>;
-
-def CVT_u16_preds
-  : InstPTX<(outs RegI16:$d), (ins RegPred:$a), "selp.u16\t$d, 1, 0, $a",
-            [(set RegI16:$d, (sext RegPred:$a))]>;
-
-def CVT_u16_u32
-  : InstPTX<(outs RegI16:$d), (ins RegI32:$a), "cvt.u16.u32\t$d, $a",
-            [(set RegI16:$d, (trunc RegI32:$a))]>;
-
-def CVT_u16_u64
-  : InstPTX<(outs RegI16:$d), (ins RegI64:$a), "cvt.u16.u64\t$d, $a",
-            [(set RegI16:$d, (trunc RegI64:$a))]>;
-
-def CVT_u16_f32
-  : InstPTX<(outs RegI16:$d), (ins RegF32:$a), "cvt.rzi.u16.f32\t$d, $a",
-            [(set RegI16:$d, (fp_to_uint RegF32:$a))]>;
-
-def CVT_u16_f64
-  : InstPTX<(outs RegI16:$d), (ins RegF64:$a), "cvt.rzi.u16.f64\t$d, $a",
-            [(set RegI16:$d, (fp_to_uint RegF64:$a))]>;
-
-// Conversion to u32
-
-def CVT_u32_pred
-  : InstPTX<(outs RegI32:$d), (ins RegPred:$a), "selp.u32\t$d, 1, 0, $a",
-            [(set RegI32:$d, (zext RegPred:$a))]>;
-
-def CVT_u32_b16
-  : InstPTX<(outs RegI32:$d), (ins RegI16:$a), "cvt.u32.u16\t$d, $a",
-            [(set RegI32:$d, (anyext RegI16:$a))]>;
-
-def CVT_u32_u16
-  : InstPTX<(outs RegI32:$d), (ins RegI16:$a), "cvt.u32.u16\t$d, $a",
-            [(set RegI32:$d, (zext RegI16:$a))]>;
-
-def CVT_u32_preds
-  : InstPTX<(outs RegI32:$d), (ins RegPred:$a), "selp.u32\t$d, 1, 0, $a",
-            [(set RegI32:$d, (sext RegPred:$a))]>;
-
-def CVT_u32_s16
-  : InstPTX<(outs RegI32:$d), (ins RegI16:$a), "cvt.u32.s16\t$d, $a",
-            [(set RegI32:$d, (sext RegI16:$a))]>;
-
-def CVT_u32_u64
-  : InstPTX<(outs RegI32:$d), (ins RegI64:$a), "cvt.u32.u64\t$d, $a",
-            [(set RegI32:$d, (trunc RegI64:$a))]>;
-
-def CVT_u32_f32
-  : InstPTX<(outs RegI32:$d), (ins RegF32:$a), "cvt.rzi.u32.f32\t$d, $a",
-            [(set RegI32:$d, (fp_to_uint RegF32:$a))]>;
-
-def CVT_u32_f64
-  : InstPTX<(outs RegI32:$d), (ins RegF64:$a), "cvt.rzi.u32.f64\t$d, $a",
-            [(set RegI32:$d, (fp_to_uint RegF64:$a))]>;
-
-// Conversion to u64
-
-def CVT_u64_pred
-  : InstPTX<(outs RegI64:$d), (ins RegPred:$a), "selp.u64\t$d, 1, 0, $a",
-            [(set RegI64:$d, (zext RegPred:$a))]>;
-
-def CVT_u64_preds
-  : InstPTX<(outs RegI64:$d), (ins RegPred:$a), "selp.u64\t$d, 1, 0, $a",
-            [(set RegI64:$d, (sext RegPred:$a))]>;
-
-def CVT_u64_u16
-  : InstPTX<(outs RegI64:$d), (ins RegI16:$a), "cvt.u64.u16\t$d, $a",
-            [(set RegI64:$d, (zext RegI16:$a))]>;
-
-def CVT_u64_s16
-  : InstPTX<(outs RegI64:$d), (ins RegI16:$a), "cvt.u64.s16\t$d, $a",
-            [(set RegI64:$d, (sext RegI16:$a))]>;
-
-def CVT_u64_u32
-  : InstPTX<(outs RegI64:$d), (ins RegI32:$a), "cvt.u64.u32\t$d, $a",
-            [(set RegI64:$d, (zext RegI32:$a))]>;
-
-def CVT_u64_s32
-  : InstPTX<(outs RegI64:$d), (ins RegI32:$a), "cvt.u64.s32\t$d, $a",
-            [(set RegI64:$d, (sext RegI32:$a))]>;
-
-def CVT_u64_f32
-  : InstPTX<(outs RegI64:$d), (ins RegF32:$a), "cvt.rzi.u64.f32\t$d, $a",
-            [(set RegI64:$d, (fp_to_uint RegF32:$a))]>;
-
-def CVT_u64_f64
-  : InstPTX<(outs RegI64:$d), (ins RegF64:$a), "cvt.rzi.u64.f64\t$d, $a",
-            [(set RegI64:$d, (fp_to_uint RegF64:$a))]>;
-
-// Conversion to f32
-
-def CVT_f32_pred
-  : InstPTX<(outs RegF32:$d), (ins RegPred:$a),
-            "selp.f32\t$d, 0F3F800000, 0F00000000, $a",  // 1.0
-            [(set RegF32:$d, (uint_to_fp RegPred:$a))]>;
-
-def CVT_f32_u16
-  : InstPTX<(outs RegF32:$d), (ins RegI16:$a), "cvt.rn.f32.u16\t$d, $a",
-            [(set RegF32:$d, (uint_to_fp RegI16:$a))]>;
-
-def CVT_f32_u32
-  : InstPTX<(outs RegF32:$d), (ins RegI32:$a), "cvt.rn.f32.u32\t$d, $a",
-            [(set RegF32:$d, (uint_to_fp RegI32:$a))]>;
-
-def CVT_f32_u64
-  : InstPTX<(outs RegF32:$d), (ins RegI64:$a), "cvt.rn.f32.u64\t$d, $a",
-            [(set RegF32:$d, (uint_to_fp RegI64:$a))]>;
-
-def CVT_f32_f64
-  : InstPTX<(outs RegF32:$d), (ins RegF64:$a), "cvt.rn.f32.f64\t$d, $a",
-            [(set RegF32:$d, (fround RegF64:$a))]>;
-
-// Conversion to f64
-
-def CVT_f64_pred
-  : InstPTX<(outs RegF64:$d), (ins RegPred:$a), 
-            "selp.f64\t$d, 0D3F80000000000000, 0D0000000000000000, $a",  // 1.0
-            [(set RegF64:$d, (uint_to_fp RegPred:$a))]>;
-
-def CVT_f64_u16
-  : InstPTX<(outs RegF64:$d), (ins RegI16:$a), "cvt.rn.f64.u16\t$d, $a",
-            [(set RegF64:$d, (uint_to_fp RegI16:$a))]>;
-
-def CVT_f64_u32
-  : InstPTX<(outs RegF64:$d), (ins RegI32:$a), "cvt.rn.f64.u32\t$d, $a",
-            [(set RegF64:$d, (uint_to_fp RegI32:$a))]>;
-
-def CVT_f64_u64
-  : InstPTX<(outs RegF64:$d), (ins RegI64:$a), "cvt.rn.f64.u64\t$d, $a",
-            [(set RegF64:$d, (uint_to_fp RegI64:$a))]>;
-
-def CVT_f64_f32
-  : InstPTX<(outs RegF64:$d), (ins RegF32:$a), "cvt.f64.f32\t$d, $a",
-            [(set RegF64:$d, (fextend RegF32:$a))]>;
-
-///===- Control Flow Instructions -----------------------------------------===//
+  ///===- Control Flow Instructions -----------------------------------------===//
 
 let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
   def BRAd
@@ -1073,30 +812,202 @@ let isReturn = 1, isTerminator = 1, isBarrier = 1 in {
   def RET  : InstPTX<(outs), (ins), "ret",  [(PTXret)]>;
 }
 
-///===- Spill Instructions ------------------------------------------------===//
-// Special instructions used for stack spilling
-def STACKSTOREI16 : InstPTX<(outs), (ins i32imm:$d, RegI16:$a),
-                            "mov.u16\ts$d, $a", []>;
-def STACKSTOREI32 : InstPTX<(outs), (ins i32imm:$d, RegI32:$a),
-                            "mov.u32\ts$d, $a", []>;
-def STACKSTOREI64 : InstPTX<(outs), (ins i32imm:$d, RegI64:$a),
-                            "mov.u64\ts$d, $a", []>;
-def STACKSTOREF32 : InstPTX<(outs), (ins i32imm:$d, RegF32:$a),
-                            "mov.f32\ts$d, $a", []>;
-def STACKSTOREF64 : InstPTX<(outs), (ins i32imm:$d, RegF64:$a),
-                            "mov.f64\ts$d, $a", []>;
+let hasSideEffects = 1 in {
+  def CALL : InstPTX<(outs), (ins), "call", [(PTXcall)]>;
+}
+
+///===- Parameter Passing Pseudo-Instructions -----------------------------===//
+
+def READPARAMPRED : InstPTX<(outs RegPred:$a), (ins i32imm:$b),
+                            "mov.pred\t$a, %param$b", []>;
+def READPARAMI16  : InstPTX<(outs RegI16:$a), (ins i32imm:$b),
+                            "mov.b16\t$a, %param$b", []>;
+def READPARAMI32  : InstPTX<(outs RegI32:$a), (ins i32imm:$b),
+                            "mov.b32\t$a, %param$b", []>;
+def READPARAMI64  : InstPTX<(outs RegI64:$a), (ins i32imm:$b),
+                            "mov.b64\t$a, %param$b", []>;
+def READPARAMF32  : InstPTX<(outs RegF32:$a), (ins i32imm:$b),
+                            "mov.f32\t$a, %param$b", []>;
+def READPARAMF64  : InstPTX<(outs RegF64:$a), (ins i32imm:$b),
+                            "mov.f64\t$a, %param$b", []>;
+
+def WRITEPARAMPRED : InstPTX<(outs), (ins RegPred:$a), "//w", []>;
+def WRITEPARAMI16  : InstPTX<(outs), (ins RegI16:$a), "//w", []>;
+def WRITEPARAMI32  : InstPTX<(outs), (ins RegI32:$a), "//w", []>;
+def WRITEPARAMI64  : InstPTX<(outs), (ins RegI64:$a), "//w", []>;
+def WRITEPARAMF32  : InstPTX<(outs), (ins RegF32:$a), "//w", []>;
+def WRITEPARAMF64  : InstPTX<(outs), (ins RegF64:$a), "//w", []>;
+
+
+//===----------------------------------------------------------------------===//
+// Instruction Selection Patterns
+//===----------------------------------------------------------------------===//
+
+// FADD
+def : Pat<(f32 (fadd RegF32:$a, RegF32:$b)),
+          (FADDrr32 RndDefault, RegF32:$a, RegF32:$b)>;
+def : Pat<(f32 (fadd RegF32:$a, fpimm:$b)),
+          (FADDri32 RndDefault, RegF32:$a, fpimm:$b)>;
+def : Pat<(f64 (fadd RegF64:$a, RegF64:$b)),
+          (FADDrr64 RndDefault, RegF64:$a, RegF64:$b)>;
+def : Pat<(f64 (fadd RegF64:$a, fpimm:$b)),
+          (FADDri64 RndDefault, RegF64:$a, fpimm:$b)>;
+
+// FSUB
+def : Pat<(f32 (fsub RegF32:$a, RegF32:$b)),
+          (FSUBrr32 RndDefault, RegF32:$a, RegF32:$b)>;
+def : Pat<(f32 (fsub RegF32:$a, fpimm:$b)),
+          (FSUBri32 RndDefault, RegF32:$a, fpimm:$b)>;
+def : Pat<(f64 (fsub RegF64:$a, RegF64:$b)),
+          (FSUBrr64 RndDefault, RegF64:$a, RegF64:$b)>;
+def : Pat<(f64 (fsub RegF64:$a, fpimm:$b)),
+          (FSUBri64 RndDefault, RegF64:$a, fpimm:$b)>;
+
+// FMUL
+def : Pat<(f32 (fmul RegF32:$a, RegF32:$b)),
+          (FMULrr32 RndDefault, RegF32:$a, RegF32:$b)>;
+def : Pat<(f32 (fmul RegF32:$a, fpimm:$b)),
+          (FMULri32 RndDefault, RegF32:$a, fpimm:$b)>;
+def : Pat<(f64 (fmul RegF64:$a, RegF64:$b)),
+          (FMULrr64 RndDefault, RegF64:$a, RegF64:$b)>;
+def : Pat<(f64 (fmul RegF64:$a, fpimm:$b)),
+          (FMULri64 RndDefault, RegF64:$a, fpimm:$b)>;
+
+// FDIV
+def : Pat<(f32 (fdiv RegF32:$a, RegF32:$b)),
+          (FDIVrr32 RndDefault, RegF32:$a, RegF32:$b)>;
+def : Pat<(f32 (fdiv RegF32:$a, fpimm:$b)),
+          (FDIVri32 RndDefault, RegF32:$a, fpimm:$b)>;
+def : Pat<(f64 (fdiv RegF64:$a, RegF64:$b)),
+          (FDIVrr64 RndDefault, RegF64:$a, RegF64:$b)>;
+def : Pat<(f64 (fdiv RegF64:$a, fpimm:$b)),
+          (FDIVri64 RndDefault, RegF64:$a, fpimm:$b)>;
+
+// FMUL+FADD
+def : Pat<(f32 (fadd (fmul RegF32:$a, RegF32:$b), RegF32:$c)),
+          (FMADrrr32 RndDefault, RegF32:$a, RegF32:$b, RegF32:$c)>;
+def : Pat<(f32 (fadd (fmul RegF32:$a, RegF32:$b), fpimm:$c)),
+          (FMADrri32 RndDefault, RegF32:$a, RegF32:$b, fpimm:$c)>;
+def : Pat<(f32 (fadd (fmul RegF32:$a, fpimm:$b), fpimm:$c)),
+          (FMADrrr32 RndDefault, RegF32:$a, fpimm:$b, fpimm:$c)>;
+def : Pat<(f32 (fadd (fmul RegF32:$a, RegF32:$b), fpimm:$c)),
+          (FMADrri32 RndDefault, RegF32:$a, RegF32:$b, fpimm:$c)>;
+def : Pat<(f64 (fadd (fmul RegF64:$a, RegF64:$b), RegF64:$c)),
+          (FMADrrr64 RndDefault, RegF64:$a, RegF64:$b, RegF64:$c)>;
+def : Pat<(f64 (fadd (fmul RegF64:$a, RegF64:$b), fpimm:$c)),
+          (FMADrri64 RndDefault, RegF64:$a, RegF64:$b, fpimm:$c)>;
+def : Pat<(f64 (fadd (fmul RegF64:$a, fpimm:$b), fpimm:$c)),
+          (FMADrri64 RndDefault, RegF64:$a, fpimm:$b, fpimm:$c)>;
+
+// FNEG
+def : Pat<(f32 (fneg RegF32:$a)), (FNEGrr32 RndDefault, RegF32:$a)>;
+def : Pat<(f32 (fneg fpimm:$a)), (FNEGri32 RndDefault, fpimm:$a)>;
+def : Pat<(f64 (fneg RegF64:$a)), (FNEGrr64 RndDefault, RegF64:$a)>;
+def : Pat<(f64 (fneg fpimm:$a)), (FNEGri64 RndDefault, fpimm:$a)>;
+
+// FSQRT
+def : Pat<(f32 (fsqrt RegF32:$a)), (FSQRTrr32 RndDefault, RegF32:$a)>;
+def : Pat<(f32 (fsqrt fpimm:$a)), (FSQRTri32 RndDefault, fpimm:$a)>;
+def : Pat<(f64 (fsqrt RegF64:$a)), (FSQRTrr64 RndDefault, RegF64:$a)>;
+def : Pat<(f64 (fsqrt fpimm:$a)), (FSQRTri64 RndDefault, fpimm:$a)>;
+
+// FSIN
+def : Pat<(f32 (fsin RegF32:$a)), (FSINrr32 RndDefault, RegF32:$a)>;
+def : Pat<(f32 (fsin fpimm:$a)), (FSINri32 RndDefault, fpimm:$a)>;
+def : Pat<(f64 (fsin RegF64:$a)), (FSINrr64 RndDefault, RegF64:$a)>;
+def : Pat<(f64 (fsin fpimm:$a)), (FSINri64 RndDefault, fpimm:$a)>;
+
+// FCOS
+def : Pat<(f32 (fcos RegF32:$a)), (FCOSrr32 RndDefault, RegF32:$a)>;
+def : Pat<(f32 (fcos fpimm:$a)), (FCOSri32 RndDefault, fpimm:$a)>;
+def : Pat<(f64 (fcos RegF64:$a)), (FCOSrr64 RndDefault, RegF64:$a)>;
+def : Pat<(f64 (fcos fpimm:$a)), (FCOSri64 RndDefault, fpimm:$a)>;
+
+// Type conversion notes:
+// - PTX does not directly support converting a predicate to a value, so we
+//   use a select instruction to select either 0 or 1 (integer or fp) based
+//   on the truth value of the predicate.
+// - PTX does not directly support converting to a predicate type, so we fake it
+//   by performing a greater-than test between the value and zero.  This follows
+//   the C convention that any non-zero value is equivalent to 'true'.
+
+// Conversion to pred
+def : Pat<(i1 (trunc RegI16:$a)),      (SETPGTu16ri RegI16:$a, 0)>;
+def : Pat<(i1 (trunc RegI32:$a)),      (SETPGTu32ri RegI32:$a, 0)>;
+def : Pat<(i1 (trunc RegI64:$a)),      (SETPGTu64ri RegI64:$a, 0)>;
+def : Pat<(i1 (fp_to_uint RegF32:$a)), (SETPGTu32ri (MOVi32f32 RegF32:$a), 0)>;
+def : Pat<(i1 (fp_to_uint RegF64:$a)), (SETPGTu64ri (MOVi64f64 RegF64:$a), 0)>;
+
+// Conversion to u16
+def : Pat<(i16 (anyext RegPred:$a)),    (SELPi16ii RegPred:$a, 1, 0)>;
+def : Pat<(i16 (sext RegPred:$a)),      (SELPi16ii RegPred:$a, 0xFFFF, 0)>;
+def : Pat<(i16 (zext RegPred:$a)),      (SELPi16ii RegPred:$a, 1, 0)>;
+def : Pat<(i16 (trunc RegI32:$a)),      (CVTu16u32 RegI32:$a)>;
+def : Pat<(i16 (trunc RegI64:$a)),      (CVTu16u64 RegI64:$a)>;
+def : Pat<(i16 (fp_to_uint RegF32:$a)), (CVTu16f32 RndDefault, RegF32:$a)>;
+def : Pat<(i16 (fp_to_sint RegF32:$a)), (CVTs16f32 RndDefault, RegF32:$a)>;
+def : Pat<(i16 (fp_to_uint RegF64:$a)), (CVTu16f64 RndDefault, RegF64:$a)>;
+def : Pat<(i16 (fp_to_sint RegF64:$a)), (CVTs16f64 RndDefault, RegF64:$a)>;
+
+// Conversion to u32
+def : Pat<(i32 (anyext RegPred:$a)),    (SELPi32ii RegPred:$a, 1, 0)>;
+def : Pat<(i32 (sext RegPred:$a)),      (SELPi32ii RegPred:$a, 0xFFFFFFFF, 0)>;
+def : Pat<(i32 (zext RegPred:$a)),      (SELPi32ii RegPred:$a, 1, 0)>;
+def : Pat<(i32 (anyext RegI16:$a)),     (CVTu32u16 RegI16:$a)>;
+def : Pat<(i32 (sext RegI16:$a)),       (CVTs32s16 RegI16:$a)>;
+def : Pat<(i32 (zext RegI16:$a)),       (CVTu32u16 RegI16:$a)>;
+def : Pat<(i32 (trunc RegI64:$a)),      (CVTu32u64 RegI64:$a)>;
+def : Pat<(i32 (fp_to_uint RegF32:$a)), (CVTu32f32 RndDefault, RegF32:$a)>;
+def : Pat<(i32 (fp_to_sint RegF32:$a)), (CVTs32f32 RndDefault, RegF32:$a)>;
+def : Pat<(i32 (fp_to_uint RegF64:$a)), (CVTu32f64 RndDefault, RegF64:$a)>;
+def : Pat<(i32 (fp_to_sint RegF64:$a)), (CVTs32f64 RndDefault, RegF64:$a)>;
+def : Pat<(i32 (bitconvert RegF32:$a)), (MOVi32f32 RegF32:$a)>;
+
+// Conversion to u64
+def : Pat<(i64 (anyext RegPred:$a)),    (SELPi64ii RegPred:$a, 1, 0)>;
+def : Pat<(i64 (sext RegPred:$a)),      (SELPi64ii RegPred:$a,
+                                         0xFFFFFFFFFFFFFFFF, 0)>;
+def : Pat<(i64 (zext RegPred:$a)),      (SELPi64ii RegPred:$a, 1, 0)>;
+def : Pat<(i64 (anyext RegI16:$a)),     (CVTu64u16 RegI16:$a)>;
+def : Pat<(i64 (sext RegI16:$a)),       (CVTs64s16 RegI16:$a)>;
+def : Pat<(i64 (zext RegI16:$a)),       (CVTu64u16 RegI16:$a)>;
+def : Pat<(i64 (anyext RegI32:$a)),     (CVTu64u32 RegI32:$a)>;
+def : Pat<(i64 (sext RegI32:$a)),       (CVTs64s32 RegI32:$a)>;
+def : Pat<(i64 (zext RegI32:$a)),       (CVTu64u32 RegI32:$a)>;
+def : Pat<(i64 (fp_to_uint RegF32:$a)), (CVTu64f32 RndDefault, RegF32:$a)>;
+def : Pat<(i64 (fp_to_sint RegF32:$a)), (CVTs64f32 RndDefault, RegF32:$a)>;
+def : Pat<(i64 (fp_to_uint RegF64:$a)), (CVTu64f64 RndDefault, RegF64:$a)>;
+def : Pat<(i64 (fp_to_sint RegF64:$a)), (CVTs64f64 RndDefault, RegF64:$a)>;
+def : Pat<(i64 (bitconvert RegF64:$a)), (MOVi64f64 RegF64:$a)>;
+
+// Conversion to f32
+def : Pat<(f32 (uint_to_fp RegPred:$a)), (SELPf32rr RegPred:$a,
+                                        (MOVf32i32 0x3F800000), (MOVf32i32 0))>;
+def : Pat<(f32 (uint_to_fp RegI16:$a)),  (CVTf32u16 RndDefault, RegI16:$a)>;
+def : Pat<(f32 (sint_to_fp RegI16:$a)),  (CVTf32s16 RndDefault, RegI16:$a)>;
+def : Pat<(f32 (uint_to_fp RegI32:$a)),  (CVTf32u32 RndDefault, RegI32:$a)>;
+def : Pat<(f32 (sint_to_fp RegI32:$a)),  (CVTf32s32 RndDefault, RegI32:$a)>;
+def : Pat<(f32 (uint_to_fp RegI64:$a)),  (CVTf32u64 RndDefault, RegI64:$a)>;
+def : Pat<(f32 (sint_to_fp RegI64:$a)),  (CVTf32s64 RndDefault, RegI64:$a)>;
+def : Pat<(f32 (fround RegF64:$a)),      (CVTf32f64 RndDefault, RegF64:$a)>;
+def : Pat<(f32 (bitconvert RegI32:$a)),  (MOVf32i32 RegI32:$a)>;
+
+// Conversion to f64
+def : Pat<(f64 (uint_to_fp RegPred:$a)), (SELPf64rr RegPred:$a,
+                                (MOVf64i64 0x3F80000000000000), (MOVf64i64 0))>;
+def : Pat<(f64 (uint_to_fp RegI16:$a)), (CVTf64u16 RndDefault, RegI16:$a)>;
+def : Pat<(f64 (sint_to_fp RegI16:$a)), (CVTf64s16 RndDefault, RegI16:$a)>;
+def : Pat<(f64 (uint_to_fp RegI32:$a)), (CVTf64u32 RndDefault, RegI32:$a)>;
+def : Pat<(f64 (sint_to_fp RegI32:$a)), (CVTf64s32 RndDefault, RegI32:$a)>;
+def : Pat<(f64 (uint_to_fp RegI64:$a)), (CVTf64u64 RndDefault, RegI64:$a)>;
+def : Pat<(f64 (sint_to_fp RegI64:$a)), (CVTf64s64 RndDefault, RegI64:$a)>;
+def : Pat<(f64 (fextend RegF32:$a)),    (CVTf64f32 RegF32:$a)>;
+def : Pat<(f64 (bitconvert RegI64:$a)), (MOVf64i64 RegI64:$a)>;
 
-def STACKLOADI16 : InstPTX<(outs), (ins RegI16:$d, i32imm:$a),
-                           "mov.u16\t$d, s$a", []>;
-def STACKLOADI32 : InstPTX<(outs), (ins RegI32:$d, i32imm:$a),
-                           "mov.u32\t$d, s$a", []>;
-def STACKLOADI64 : InstPTX<(outs), (ins RegI64:$d, i32imm:$a),
-                           "mov.u64\t$d, s$a", []>;
-def STACKLOADF32 : InstPTX<(outs), (ins RegF32:$d, i32imm:$a),
-                           "mov.f32\t$d, s$a", []>;
-def STACKLOADF64 : InstPTX<(outs), (ins RegF64:$d, i32imm:$a),
-                           "mov.f64\t$d, s$a", []>;
 
 ///===- Intrinsic Instructions --------------------------------------------===//
-
 include "PTXIntrinsicInstrInfo.td"
+
+///===- Load/Store Instructions -------------------------------------------===//
+include "PTXInstrLoadStore.td"
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXInstrLoadStore.td b/contrib/llvm/lib/Target/PTX/PTXInstrLoadStore.td
new file mode 100644
index 000000000000..9b4f56cf25c6
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXInstrLoadStore.td
@@ -0,0 +1,278 @@
+//===- PTXInstrLoadStore.td - PTX Load/Store Instruction Defs -*- tblgen-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the PTX load/store instructions in TableGen format.
+//
+//===----------------------------------------------------------------------===//
+
+
+// Addressing Predicates
+// We have to differentiate between 32- and 64-bit pointer types
+def Use32BitAddresses : Predicate<"!getSubtarget().is64Bit()">;
+def Use64BitAddresses : Predicate<"getSubtarget().is64Bit()">;
+
+//===----------------------------------------------------------------------===//
+// Pattern Fragments for Loads/Stores
+//===----------------------------------------------------------------------===//
+
+def load_global : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  const Value *Src;
+  const PointerType *PT;
+  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
+      (PT = dyn_cast<PointerType>(Src->getType())))
+    return PT->getAddressSpace() == PTXStateSpace::Global;
+  return false;
+}]>;
+
+def load_constant : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  const Value *Src;
+  const PointerType *PT;
+  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
+      (PT = dyn_cast<PointerType>(Src->getType())))
+    return PT->getAddressSpace() == PTXStateSpace::Constant;
+  return false;
+}]>;
+
+def load_shared : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  const Value *Src;
+  const PointerType *PT;
+  if ((Src = cast<LoadSDNode>(N)->getSrcValue()) &&
+      (PT = dyn_cast<PointerType>(Src->getType())))
+    return PT->getAddressSpace() == PTXStateSpace::Shared;
+  return false;
+}]>;
+
+def store_global
+  : PatFrag<(ops node:$d, node:$ptr), (store node:$d, node:$ptr), [{
+  const Value *Src;
+  const PointerType *PT;
+  if ((Src = cast<StoreSDNode>(N)->getSrcValue()) &&
+      (PT = dyn_cast<PointerType>(Src->getType())))
+    return PT->getAddressSpace() == PTXStateSpace::Global;
+  return false;
+}]>;
+
+def store_shared
+  : PatFrag<(ops node:$d, node:$ptr), (store node:$d, node:$ptr), [{
+  const Value *Src;
+  const PointerType *PT;
+  if ((Src = cast<StoreSDNode>(N)->getSrcValue()) &&
+      (PT = dyn_cast<PointerType>(Src->getType())))
+    return PT->getAddressSpace() == PTXStateSpace::Shared;
+  return false;
+}]>;
+
+// Addressing modes.
+def ADDRrr32    : ComplexPattern<i32, 2, "SelectADDRrr", [], []>;
+def ADDRrr64    : ComplexPattern<i64, 2, "SelectADDRrr", [], []>;
+def ADDRri32    : ComplexPattern<i32, 2, "SelectADDRri", [], []>;
+def ADDRri64    : ComplexPattern<i64, 2, "SelectADDRri", [], []>;
+def ADDRii32    : ComplexPattern<i32, 2, "SelectADDRii", [], []>;
+def ADDRii64    : ComplexPattern<i64, 2, "SelectADDRii", [], []>;
+def ADDRlocal32 : ComplexPattern<i32, 2, "SelectADDRlocal", [], []>;
+def ADDRlocal64 : ComplexPattern<i64, 2, "SelectADDRlocal", [], []>;
+
+// Address operands
+def MEMri32 : Operand<i32> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops RegI32, i32imm);
+}
+def MEMri64 : Operand<i64> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops RegI64, i64imm);
+}
+def LOCALri32 : Operand<i32> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops i32imm, i32imm);
+}
+def LOCALri64 : Operand<i64> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops i64imm, i64imm);
+}
+def MEMii32 : Operand<i32> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops i32imm, i32imm);
+}
+def MEMii64 : Operand<i64> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops i64imm, i64imm);
+}
+// The operand here does not correspond to an actual address, so we
+// can use i32 in 64-bit address modes.
+def MEMpi : Operand<i32> {
+  let PrintMethod = "printParamOperand";
+  let MIOperandInfo = (ops i32imm);
+}
+def MEMret : Operand<i32> {
+  let PrintMethod = "printReturnOperand";
+  let MIOperandInfo = (ops i32imm);
+}
+
+
+// Load/store .param space
+def PTXloadparam
+  : SDNode<"PTXISD::LOAD_PARAM", SDTypeProfile<1, 1, [SDTCisPtrTy<1>]>,
+           [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue]>;
+def PTXstoreparam
+  : SDNode<"PTXISD::STORE_PARAM", SDTypeProfile<0, 2, [SDTCisVT<0, i32>]>,
+           [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue]>;
+
+def PTXreadparam
+  : SDNode<"PTXISD::READ_PARAM", SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>,
+      [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue]>;
+def PTXwriteparam
+  : SDNode<"PTXISD::WRITE_PARAM", SDTypeProfile<0, 1, []>,
+      [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue]>;
+
+
+
+//===----------------------------------------------------------------------===//
+// Classes for loads/stores
+//===----------------------------------------------------------------------===//
+multiclass PTX_LD<string opstr, string typestr,
+           RegisterClass RC, PatFrag pat_load> {
+  def rr32 : InstPTX<(outs RC:$d),
+                     (ins MEMri32:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (pat_load ADDRrr32:$a))]>,
+                     Requires<[Use32BitAddresses]>;
+  def rr64 : InstPTX<(outs RC:$d),
+                     (ins MEMri64:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (pat_load ADDRrr64:$a))]>,
+                     Requires<[Use64BitAddresses]>;
+  def ri32 : InstPTX<(outs RC:$d),
+                     (ins MEMri32:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (pat_load ADDRri32:$a))]>,
+                     Requires<[Use32BitAddresses]>;
+  def ri64 : InstPTX<(outs RC:$d),
+                     (ins MEMri64:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (pat_load ADDRri64:$a))]>,
+                     Requires<[Use64BitAddresses]>;
+  def ii32 : InstPTX<(outs RC:$d),
+                     (ins MEMii32:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (pat_load ADDRii32:$a))]>,
+                     Requires<[Use32BitAddresses]>;
+  def ii64 : InstPTX<(outs RC:$d),
+                     (ins MEMii64:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (pat_load ADDRii64:$a))]>,
+                     Requires<[Use64BitAddresses]>;
+}
+
+multiclass PTX_ST<string opstr, string typestr, RegisterClass RC,
+                  PatFrag pat_store> {
+  def rr32 : InstPTX<(outs),
+                     (ins RC:$d, MEMri32:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
+                     [(pat_store RC:$d, ADDRrr32:$a)]>,
+                     Requires<[Use32BitAddresses]>;
+  def rr64 : InstPTX<(outs),
+                     (ins RC:$d, MEMri64:$a),
+                     !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
+                     [(pat_store RC:$d, ADDRrr64:$a)]>,
+                     Requires<[Use64BitAddresses]>;
+  def ri32 : InstPTX<(outs),
+                   (ins RC:$d, MEMri32:$a),
+                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
+                   [(pat_store RC:$d, ADDRri32:$a)]>,
+                   Requires<[Use32BitAddresses]>;
+  def ri64 : InstPTX<(outs),
+                   (ins RC:$d, MEMri64:$a),
+                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
+                   [(pat_store RC:$d, ADDRri64:$a)]>,
+                   Requires<[Use64BitAddresses]>;
+  def ii32 : InstPTX<(outs),
+                   (ins RC:$d, MEMii32:$a),
+                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
+                   [(pat_store RC:$d, ADDRii32:$a)]>,
+                   Requires<[Use32BitAddresses]>;
+  def ii64 : InstPTX<(outs),
+                   (ins RC:$d, MEMii64:$a),
+                   !strconcat(opstr, !strconcat(typestr, "\t[$a], $d")),
+                   [(pat_store RC:$d, ADDRii64:$a)]>,
+                   Requires<[Use64BitAddresses]>;
+}
+
+multiclass PTX_LOCAL_LD_ST<string typestr, RegisterClass RC> {
+  def LDri32 : InstPTX<(outs RC:$d), (ins LOCALri32:$a),
+                      !strconcat("ld.local", !strconcat(typestr, "\t$d, [$a]")),
+                       [(set RC:$d, (load_global ADDRlocal32:$a))]>;
+  def LDri64 : InstPTX<(outs RC:$d), (ins LOCALri64:$a),
+                      !strconcat("ld.local", !strconcat(typestr, "\t$d, [$a]")),
+                       [(set RC:$d, (load_global ADDRlocal64:$a))]>;
+  def STri32 : InstPTX<(outs), (ins RC:$d, LOCALri32:$a),
+                      !strconcat("st.local", !strconcat(typestr, "\t[$a], $d")),
+                       [(store_global RC:$d, ADDRlocal32:$a)]>;
+  def STri64 : InstPTX<(outs), (ins RC:$d, LOCALri64:$a),
+                      !strconcat("st.local", !strconcat(typestr, "\t[$a], $d")),
+                       [(store_global RC:$d, ADDRlocal64:$a)]>;
+}
+
+multiclass PTX_PARAM_LD_ST<string typestr, RegisterClass RC> {
+  let hasSideEffects = 1 in {
+  def LDpi : InstPTX<(outs RC:$d), (ins i32imm:$a),
+                     !strconcat("ld.param", !strconcat(typestr, "\t$d, [$a]")),
+                     [(set RC:$d, (PTXloadparam texternalsym:$a))]>;
+  def STpi : InstPTX<(outs), (ins i32imm:$d, RC:$a),
+                     !strconcat("st.param", !strconcat(typestr, "\t[$d], $a")),
+                     [(PTXstoreparam texternalsym:$d, RC:$a)]>;
+  }
+}
+
+multiclass PTX_LD_ALL<string opstr, PatFrag pat_load> {
+  defm u16 : PTX_LD<opstr, ".u16", RegI16, pat_load>;
+  defm u32 : PTX_LD<opstr, ".u32", RegI32, pat_load>;
+  defm u64 : PTX_LD<opstr, ".u64", RegI64, pat_load>;
+  defm f32 : PTX_LD<opstr, ".f32", RegF32, pat_load>;
+  defm f64 : PTX_LD<opstr, ".f64", RegF64, pat_load>;
+}
+
+multiclass PTX_ST_ALL<string opstr, PatFrag pat_store> {
+  defm u16 : PTX_ST<opstr, ".u16", RegI16, pat_store>;
+  defm u32 : PTX_ST<opstr, ".u32", RegI32, pat_store>;
+  defm u64 : PTX_ST<opstr, ".u64", RegI64, pat_store>;
+  defm f32 : PTX_ST<opstr, ".f32", RegF32, pat_store>;
+  defm f64 : PTX_ST<opstr, ".f64", RegF64, pat_store>;
+}
+
+
+
+//===----------------------------------------------------------------------===//
+// Instruction definitions for loads/stores
+//===----------------------------------------------------------------------===//
+
+// Global/shared stores
+defm STg : PTX_ST_ALL<"st.global", store_global>;
+defm STs : PTX_ST_ALL<"st.shared", store_shared>;
+
+// Global/shared/constant loads
+defm LDg : PTX_LD_ALL<"ld.global", load_global>;
+defm LDc : PTX_LD_ALL<"ld.const",  load_constant>;
+defm LDs : PTX_LD_ALL<"ld.shared", load_shared>;
+
+// Param loads/stores
+defm PARAMPRED : PTX_PARAM_LD_ST<".pred", RegPred>;
+defm PARAMU16  : PTX_PARAM_LD_ST<".u16", RegI16>;
+defm PARAMU32  : PTX_PARAM_LD_ST<".u32", RegI32>;
+defm PARAMU64  : PTX_PARAM_LD_ST<".u64", RegI64>;
+defm PARAMF32  : PTX_PARAM_LD_ST<".f32", RegF32>;
+defm PARAMF64  : PTX_PARAM_LD_ST<".f64", RegF64>;
+
+// Local loads/stores
+defm LOCALPRED : PTX_LOCAL_LD_ST<".pred", RegPred>;
+defm LOCALU16  : PTX_LOCAL_LD_ST<".u16", RegI16>;
+defm LOCALU32  : PTX_LOCAL_LD_ST<".u32", RegI32>;
+defm LOCALU64  : PTX_LOCAL_LD_ST<".u64", RegI64>;
+defm LOCALF32  : PTX_LOCAL_LD_ST<".f32", RegF32>;
+defm LOCALF64  : PTX_LOCAL_LD_ST<".f64", RegF64>;
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXIntrinsicInstrInfo.td b/contrib/llvm/lib/Target/PTX/PTXIntrinsicInstrInfo.td
index 8d97909d339a..9de1cb62719a 100644
--- a/contrib/llvm/lib/Target/PTX/PTXIntrinsicInstrInfo.td
+++ b/contrib/llvm/lib/Target/PTX/PTXIntrinsicInstrInfo.td
@@ -25,37 +25,63 @@ class PTX_READ_SPECIAL_REGISTER_R32<string regname, Intrinsic intop>
 
 // TODO Add read vector-version of special registers
 
-//def PTX_READ_TID_R64 : PTX_READ_SPECIAL_REGISTER_R64<"tid", int_ptx_read_tid_r64>;
-def PTX_READ_TID_X   : PTX_READ_SPECIAL_REGISTER_R32<"tid.x", int_ptx_read_tid_x>;
-def PTX_READ_TID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"tid.y", int_ptx_read_tid_y>;
-def PTX_READ_TID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"tid.z", int_ptx_read_tid_z>;
-def PTX_READ_TID_W   : PTX_READ_SPECIAL_REGISTER_R32<"tid.w", int_ptx_read_tid_w>;
+//def PTX_READ_TID_R64 : PTX_READ_SPECIAL_REGISTER_R64<"tid",
+//                                                     int_ptx_read_tid_r64>;
+def PTX_READ_TID_X   : PTX_READ_SPECIAL_REGISTER_R32<"tid.x",
+                                                     int_ptx_read_tid_x>;
+def PTX_READ_TID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"tid.y",
+                                                     int_ptx_read_tid_y>;
+def PTX_READ_TID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"tid.z",
+                                                     int_ptx_read_tid_z>;
+def PTX_READ_TID_W   : PTX_READ_SPECIAL_REGISTER_R32<"tid.w",
+                                                     int_ptx_read_tid_w>;
 
-//def PTX_READ_NTID_R64 : PTX_READ_SPECIAL_REGISTER_R64<"ntid", int_ptx_read_ntid_r64>;
-def PTX_READ_NTID_X   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.x", int_ptx_read_ntid_x>;
-def PTX_READ_NTID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.y", int_ptx_read_ntid_y>;
-def PTX_READ_NTID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.z", int_ptx_read_ntid_z>;
-def PTX_READ_NTID_W   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.w", int_ptx_read_ntid_w>;
+//def PTX_READ_NTID_R64 : PTX_READ_SPECIAL_REGISTER_R64<"ntid",
+//                                                      int_ptx_read_ntid_r64>;
+def PTX_READ_NTID_X   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.x",
+                                                      int_ptx_read_ntid_x>;
+def PTX_READ_NTID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.y",
+                                                      int_ptx_read_ntid_y>;
+def PTX_READ_NTID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.z",
+                                                      int_ptx_read_ntid_z>;
+def PTX_READ_NTID_W   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.w",
+                                                      int_ptx_read_ntid_w>;
 
-def PTX_READ_LANEID  : PTX_READ_SPECIAL_REGISTER_R32<"laneid", int_ptx_read_laneid>;
-def PTX_READ_WARPID  : PTX_READ_SPECIAL_REGISTER_R32<"warpid", int_ptx_read_warpid>;
-def PTX_READ_NWARPID : PTX_READ_SPECIAL_REGISTER_R32<"nwarpid", int_ptx_read_nwarpid>;
+def PTX_READ_LANEID  : PTX_READ_SPECIAL_REGISTER_R32<"laneid",
+                                                     int_ptx_read_laneid>;
+def PTX_READ_WARPID  : PTX_READ_SPECIAL_REGISTER_R32<"warpid",
+                                                     int_ptx_read_warpid>;
+def PTX_READ_NWARPID : PTX_READ_SPECIAL_REGISTER_R32<"nwarpid",
+                                                     int_ptx_read_nwarpid>;
 
-//def PTX_READ_CTAID_R64 : PTX_READ_SPECIAL_REGISTER_R64<"ctaid", int_ptx_read_ctaid_r64>;
-def PTX_READ_CTAID_X   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.x", int_ptx_read_ctaid_x>;
-def PTX_READ_CTAID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.y", int_ptx_read_ctaid_y>;
-def PTX_READ_CTAID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.z", int_ptx_read_ctaid_z>;
-def PTX_READ_CTAID_W   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.w", int_ptx_read_ctaid_w>;
+//def PTX_READ_CTAID_R64 :
+//PTX_READ_SPECIAL_REGISTER_R64<"ctaid", int_ptx_read_ctaid_r64>;
+def PTX_READ_CTAID_X   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.x",
+                                                       int_ptx_read_ctaid_x>;
+def PTX_READ_CTAID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.y",
+                                                       int_ptx_read_ctaid_y>;
+def PTX_READ_CTAID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.z",
+                                                       int_ptx_read_ctaid_z>;
+def PTX_READ_CTAID_W   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.w",
+                                                       int_ptx_read_ctaid_w>;
 
-//def PTX_READ_NCTAID_R64 : PTX_READ_SPECIAL_REGISTER_R64<"nctaid", int_ptx_read_nctaid_r64>;
-def PTX_READ_NCTAID_X   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.x", int_ptx_read_nctaid_x>;
-def PTX_READ_NCTAID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.y", int_ptx_read_nctaid_y>;
-def PTX_READ_NCTAID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.z", int_ptx_read_nctaid_z>;
-def PTX_READ_NCTAID_W   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.w", int_ptx_read_nctaid_w>;
+//def PTX_READ_NCTAID_R64 :
+//PTX_READ_SPECIAL_REGISTER_R64<"nctaid", int_ptx_read_nctaid_r64>;
+def PTX_READ_NCTAID_X   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.x",
+                                                        int_ptx_read_nctaid_x>;
+def PTX_READ_NCTAID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.y",
+                                                        int_ptx_read_nctaid_y>;
+def PTX_READ_NCTAID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.z",
+                                                        int_ptx_read_nctaid_z>;
+def PTX_READ_NCTAID_W   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.w",
+                                                        int_ptx_read_nctaid_w>;
 
-def PTX_READ_SMID  : PTX_READ_SPECIAL_REGISTER_R32<"smid", int_ptx_read_smid>;
-def PTX_READ_NSMID  : PTX_READ_SPECIAL_REGISTER_R32<"nsmid", int_ptx_read_nsmid>;
-def PTX_READ_GRIDID  : PTX_READ_SPECIAL_REGISTER_R32<"gridid", int_ptx_read_gridid>;
+def PTX_READ_SMID  : PTX_READ_SPECIAL_REGISTER_R32<"smid",
+                                                   int_ptx_read_smid>;
+def PTX_READ_NSMID  : PTX_READ_SPECIAL_REGISTER_R32<"nsmid",
+                                                    int_ptx_read_nsmid>;
+def PTX_READ_GRIDID  : PTX_READ_SPECIAL_REGISTER_R32<"gridid",
+                                                     int_ptx_read_gridid>;
 
 def PTX_READ_LANEMASK_EQ
   : PTX_READ_SPECIAL_REGISTER_R32<"lanemask_eq", int_ptx_read_lanemask_eq>;
diff --git a/contrib/llvm/lib/Target/PTX/PTXMCAsmStreamer.cpp b/contrib/llvm/lib/Target/PTX/PTXMCAsmStreamer.cpp
index b13a3dace130..468ce9301de4 100644
--- a/contrib/llvm/lib/Target/PTX/PTXMCAsmStreamer.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXMCAsmStreamer.cpp
@@ -100,7 +100,7 @@ class PTXMCAsmStreamer : public MCStreamer {
   /// @{
 
   virtual void ChangeSection(const MCSection *Section);
-  virtual void InitSections() {}
+  virtual void InitSections() { /* PTX does not use sections */ }
 
   virtual void EmitLabel(MCSymbol *Symbol);
 
@@ -132,7 +132,9 @@ class PTXMCAsmStreamer : public MCStreamer {
   ///
   /// @param Symbol - The common symbol to emit.
   /// @param Size - The size of the common symbol.
-  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size);
+  /// @param ByteAlignment - The alignment of the common symbol in bytes.
+  virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                     unsigned ByteAlignment);
 
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
                             unsigned Size = 0, unsigned ByteAlignment = 0);
@@ -233,7 +235,7 @@ void PTXMCAsmStreamer::ChangeSection(const MCSection *Section) {
 void PTXMCAsmStreamer::EmitLabel(MCSymbol *Symbol) {
   assert(Symbol->isUndefined() && "Cannot define a symbol twice!");
   assert(!Symbol->isVariable() && "Cannot emit a variable symbol!");
-  //assert(getCurrentSection() && "Cannot emit before setting section!");
+  assert(getCurrentSection() && "Cannot emit before setting section!");
 
   OS << *Symbol << MAI.getLabelSuffix();
   EmitEOL();
@@ -283,7 +285,8 @@ void PTXMCAsmStreamer::EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {}
 void PTXMCAsmStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                         unsigned ByteAlignment) {}
 
-void PTXMCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size) {}
+void PTXMCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                             unsigned ByteAlignment) {}
 
 void PTXMCAsmStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
                                     unsigned Size, unsigned ByteAlignment) {}
@@ -510,7 +513,7 @@ void PTXMCAsmStreamer::EmitInstruction(const MCInst &Inst) {
 
   // If we have an AsmPrinter, use that to print, otherwise print the MCInst.
   if (InstPrinter)
-    InstPrinter->printInst(&Inst, OS);
+    InstPrinter->printInst(&Inst, OS, "");
   else
     Inst.print(OS, &MAI);
   EmitEOL();
@@ -533,7 +536,7 @@ namespace llvm {
                                    formatted_raw_ostream &OS,
                                    bool isVerboseAsm, bool useLoc, bool useCFI,
                                    MCInstPrinter *IP,
-                                   MCCodeEmitter *CE, TargetAsmBackend *TAB,
+                                   MCCodeEmitter *CE, MCAsmBackend *MAB,
                                    bool ShowInst) {
     return new PTXMCAsmStreamer(Context, OS, isVerboseAsm, useLoc,
                                 IP, CE, ShowInst);
diff --git a/contrib/llvm/lib/Target/PTX/PTXMCInstLower.cpp b/contrib/llvm/lib/Target/PTX/PTXMCInstLower.cpp
new file mode 100644
index 000000000000..142e639b3ef7
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXMCInstLower.cpp
@@ -0,0 +1,32 @@
+//===-- PTXMCInstLower.cpp - Convert PTX MachineInstr to an MCInst --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code to lower PTX MachineInstrs to their corresponding
+// MCInst records.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PTX.h"
+#include "PTXAsmPrinter.h"
+#include "llvm/Constants.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/Target/Mangler.h"
+
+void llvm::LowerPTXMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
+                                        PTXAsmPrinter &AP) {
+  OutMI.setOpcode(MI->getOpcode());
+  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    MCOperand MCOp;
+    OutMI.addOperand(AP.lowerOperand(MO));
+  }
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXMFInfoExtract.cpp b/contrib/llvm/lib/Target/PTX/PTXMFInfoExtract.cpp
index 6fe9e6c3f657..b33a273dc93d 100644
--- a/contrib/llvm/lib/Target/PTX/PTXMFInfoExtract.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXMFInfoExtract.cpp
@@ -52,36 +52,12 @@ bool PTXMFInfoExtract::runOnMachineFunction(MachineFunction &MF) {
   PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
-  DEBUG(dbgs() << "******** PTX FUNCTION LOCAL VAR REG DEF ********\n");
-
-  DEBUG(dbgs()
-        << "PTX::NoRegister == " << PTX::NoRegister << "\n"
-        << "PTX::NUM_TARGET_REGS == " << PTX::NUM_TARGET_REGS << "\n");
-
-  DEBUG(for (unsigned reg = PTX::NoRegister + 1;
-             reg < PTX::NUM_TARGET_REGS; ++reg)
-          if (MRI.isPhysRegUsed(reg))
-            dbgs() << "Used Reg: " << reg << "\n";);
-
-  // FIXME: This is a slow linear scanning
-  for (unsigned reg = PTX::NoRegister + 1; reg < PTX::NUM_TARGET_REGS; ++reg)
-    if (MRI.isPhysRegUsed(reg) &&
-        !MFI->isRetReg(reg) &&
-        (MFI->isKernel() || !MFI->isArgReg(reg)))
-      MFI->addLocalVarReg(reg);
-
-  // Notify MachineFunctionInfo that I've done adding local var reg
-  MFI->doneAddLocalVar();
-
-  DEBUG(for (PTXMachineFunctionInfo::reg_iterator
-             i = MFI->argRegBegin(), e = MFI->argRegEnd();
-             i != e; ++i)
-        dbgs() << "Arg Reg: " << *i << "\n";);
-
-  DEBUG(for (PTXMachineFunctionInfo::reg_iterator
-             i = MFI->localVarRegBegin(), e = MFI->localVarRegEnd();
-             i != e; ++i)
-        dbgs() << "Local Var Reg: " << *i << "\n";);
+  // Generate list of all virtual registers used in this function
+  for (unsigned i = 0; i < MRI.getNumVirtRegs(); ++i) {
+    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
+    const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
+    MFI->addVirtualRegister(TRC, Reg);
+  }
 
   return false;
 }
diff --git a/contrib/llvm/lib/Target/PTX/PTXMachineFunctionInfo.h b/contrib/llvm/lib/Target/PTX/PTXMachineFunctionInfo.h
index 9d65f5bd1ade..3b985f7dd6b0 100644
--- a/contrib/llvm/lib/Target/PTX/PTXMachineFunctionInfo.h
+++ b/contrib/llvm/lib/Target/PTX/PTXMachineFunctionInfo.h
@@ -15,75 +15,148 @@
 #define PTX_MACHINE_FUNCTION_INFO_H
 
 #include "PTX.h"
+#include "PTXParamManager.h"
+#include "PTXRegisterInfo.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
+
 /// PTXMachineFunctionInfo - This class is derived from MachineFunction and
 /// contains private PTX target-specific information for each MachineFunction.
 ///
 class PTXMachineFunctionInfo : public MachineFunctionInfo {
 private:
-  bool is_kernel;
-  std::vector<unsigned> reg_arg, reg_local_var;
-  std::vector<unsigned> reg_ret;
-  bool _isDoneAddArg;
+  bool IsKernel;
+  DenseSet<unsigned> RegArgs;
+  DenseSet<unsigned> RegRets;
+
+  typedef std::vector<unsigned> RegisterList;
+  typedef DenseMap<const TargetRegisterClass*, RegisterList> RegisterMap;
+  typedef DenseMap<unsigned, std::string> RegisterNameMap;
+  typedef DenseMap<int, std::string> FrameMap;
+
+  RegisterMap UsedRegs;
+  RegisterNameMap RegNames;
+  FrameMap FrameSymbols;
+
+  PTXParamManager ParamManager;
 
 public:
+  typedef DenseSet<unsigned>::const_iterator reg_iterator;
+
   PTXMachineFunctionInfo(MachineFunction &MF)
-    : is_kernel(false), reg_ret(PTX::NoRegister), _isDoneAddArg(false) {
-      reg_arg.reserve(8);
-      reg_local_var.reserve(32);
+    : IsKernel(false) {
+      UsedRegs[PTX::RegPredRegisterClass] = RegisterList();
+      UsedRegs[PTX::RegI16RegisterClass] = RegisterList();
+      UsedRegs[PTX::RegI32RegisterClass] = RegisterList();
+      UsedRegs[PTX::RegI64RegisterClass] = RegisterList();
+      UsedRegs[PTX::RegF32RegisterClass] = RegisterList();
+      UsedRegs[PTX::RegF64RegisterClass] = RegisterList();
     }
 
-  void setKernel(bool _is_kernel=true) { is_kernel = _is_kernel; }
+  /// getParamManager - Returns the PTXParamManager instance for this function.
+  PTXParamManager& getParamManager() { return ParamManager; }
+  const PTXParamManager& getParamManager() const { return ParamManager; }
 
-  void addArgReg(unsigned reg) { reg_arg.push_back(reg); }
-  void addLocalVarReg(unsigned reg) { reg_local_var.push_back(reg); }
-  void addRetReg(unsigned reg) {
-    if (!isRetReg(reg)) {
-      reg_ret.push_back(reg);
+  /// setKernel/isKernel - Gets/sets a flag that indicates if this function is
+  /// a PTX kernel function.
+  void setKernel(bool _IsKernel=true) { IsKernel = _IsKernel; }
+  bool isKernel() const { return IsKernel; }
+
+  /// argreg_begin/argreg_end - Returns iterators to the set of registers
+  /// containing function arguments.
+  reg_iterator argreg_begin() const { return RegArgs.begin(); }
+  reg_iterator argreg_end()   const { return RegArgs.end(); }
+
+  /// retreg_begin/retreg_end - Returns iterators to the set of registers
+  /// containing the function return values.
+  reg_iterator retreg_begin() const { return RegRets.begin(); }
+  reg_iterator retreg_end()   const { return RegRets.end(); }
+
+  /// addRetReg - Adds a register to the set of return-value registers.
+  void addRetReg(unsigned Reg) {
+    if (!RegRets.count(Reg)) {
+      RegRets.insert(Reg);
+      std::string name;
+      name = "%ret";
+      name += utostr(RegRets.size() - 1);
+      RegNames[Reg] = name;
     }
   }
 
-  void doneAddArg(void) {
-    _isDoneAddArg = true;
-  }
-  void doneAddLocalVar(void) {}
-
-  bool isKernel() const { return is_kernel; }
-
-  typedef std::vector<unsigned>::const_iterator         reg_iterator;
-  typedef std::vector<unsigned>::const_reverse_iterator reg_reverse_iterator;
-  typedef std::vector<unsigned>::const_iterator         ret_iterator;
-
-  bool         argRegEmpty() const { return reg_arg.empty(); }
-  int          getNumArg() const { return reg_arg.size(); }
-  reg_iterator argRegBegin() const { return reg_arg.begin(); }
-  reg_iterator argRegEnd()   const { return reg_arg.end(); }
-  reg_reverse_iterator argRegReverseBegin() const { return reg_arg.rbegin(); }
-  reg_reverse_iterator argRegReverseEnd() const { return reg_arg.rend(); }
-
-  bool         localVarRegEmpty() const { return reg_local_var.empty(); }
-  reg_iterator localVarRegBegin() const { return reg_local_var.begin(); }
-  reg_iterator localVarRegEnd()   const { return reg_local_var.end(); }
-
-  bool         retRegEmpty() const { return reg_ret.empty(); }
-  int          getNumRet() const { return reg_ret.size(); }
-  ret_iterator retRegBegin() const { return reg_ret.begin(); }
-  ret_iterator retRegEnd()   const { return reg_ret.end(); }
-
-  bool isArgReg(unsigned reg) const {
-    return std::find(reg_arg.begin(), reg_arg.end(), reg) != reg_arg.end();
+  /// addArgReg - Adds a register to the set of function argument registers.
+  void addArgReg(unsigned Reg) {
+    RegArgs.insert(Reg);
+    std::string name;
+    name = "%param";
+    name += utostr(RegArgs.size() - 1);
+    RegNames[Reg] = name;
   }
 
-  bool isRetReg(unsigned reg) const {
-    return std::find(reg_ret.begin(), reg_ret.end(), reg) != reg_ret.end();
+  /// addVirtualRegister - Adds a virtual register to the set of all used
+  /// registers in the function.
+  void addVirtualRegister(const TargetRegisterClass *TRC, unsigned Reg) {
+    std::string name;
+
+    // Do not count registers that are argument/return registers.
+    if (!RegRets.count(Reg) && !RegArgs.count(Reg)) {
+      UsedRegs[TRC].push_back(Reg);
+      if (TRC == PTX::RegPredRegisterClass)
+        name = "%p";
+      else if (TRC == PTX::RegI16RegisterClass)
+        name = "%rh";
+      else if (TRC == PTX::RegI32RegisterClass)
+        name = "%r";
+      else if (TRC == PTX::RegI64RegisterClass)
+        name = "%rd";
+      else if (TRC == PTX::RegF32RegisterClass)
+        name = "%f";
+      else if (TRC == PTX::RegF64RegisterClass)
+        name = "%fd";
+      else
+        llvm_unreachable("Invalid register class");
+
+      name += utostr(UsedRegs[TRC].size() - 1);
+      RegNames[Reg] = name;
+    }
   }
 
-  bool isLocalVarReg(unsigned reg) const {
-    return std::find(reg_local_var.begin(), reg_local_var.end(), reg)
-      != reg_local_var.end();
+  /// getRegisterName - Returns the name of the specified virtual register. This
+  /// name is used during PTX emission.
+  const char *getRegisterName(unsigned Reg) const {
+    if (RegNames.count(Reg))
+      return RegNames.find(Reg)->second.c_str();
+    else if (Reg == PTX::NoRegister)
+      return "%noreg";
+    else
+      llvm_unreachable("Register not in register name map");
+  }
+
+  /// getNumRegistersForClass - Returns the number of virtual registers that are
+  /// used for the specified register class.
+  unsigned getNumRegistersForClass(const TargetRegisterClass *TRC) const {
+    return UsedRegs.lookup(TRC).size();
+  }
+
+  /// getFrameSymbol - Returns the symbol name for the given FrameIndex.
+  const char* getFrameSymbol(int FrameIndex) {
+    if (FrameSymbols.count(FrameIndex)) {
+      return FrameSymbols.lookup(FrameIndex).c_str();
+    } else {
+      std::string Name = "__local";
+      Name += utostr(FrameIndex);
+      // The whole point of caching this name is to ensure the pointer we pass
+      // to any getExternalSymbol() calls will remain valid for the lifetime of
+      // the back-end instance. This is to work around an issue in SelectionDAG
+      // where symbol names are expected to be life-long strings.
+      FrameSymbols[FrameIndex] = Name;
+      return FrameSymbols[FrameIndex].c_str();
+    }
   }
 }; // class PTXMachineFunctionInfo
 } // namespace llvm
diff --git a/contrib/llvm/lib/Target/PTX/PTXParamManager.cpp b/contrib/llvm/lib/Target/PTX/PTXParamManager.cpp
new file mode 100644
index 000000000000..7753787ebc51
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXParamManager.cpp
@@ -0,0 +1,73 @@
+//===- PTXParamManager.cpp - Manager for .param variables -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the PTXParamManager class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PTX.h"
+#include "PTXParamManager.h"
+#include "llvm/ADT/StringExtras.h"
+
+using namespace llvm;
+
+PTXParamManager::PTXParamManager() {
+}
+
+unsigned PTXParamManager::addArgumentParam(unsigned Size) {
+  PTXParam Param;
+  Param.Type = PTX_PARAM_TYPE_ARGUMENT;
+  Param.Size = Size;
+
+  std::string Name;
+  Name = "__param_";
+  Name += utostr(ArgumentParams.size()+1);
+  Param.Name = Name;
+
+  unsigned Index = AllParams.size();
+  AllParams[Index] = Param;
+  ArgumentParams.push_back(Index);
+
+  return Index;
+}
+
+unsigned PTXParamManager::addReturnParam(unsigned Size) {
+  PTXParam Param;
+  Param.Type = PTX_PARAM_TYPE_RETURN;
+  Param.Size = Size;
+
+  std::string Name;
+  Name = "__ret_";
+  Name += utostr(ReturnParams.size()+1);
+  Param.Name = Name;
+
+  unsigned Index = AllParams.size();
+  AllParams[Index] = Param;
+  ReturnParams.push_back(Index);
+
+  return Index;
+}
+
+unsigned PTXParamManager::addLocalParam(unsigned Size) {
+  PTXParam Param;
+  Param.Type = PTX_PARAM_TYPE_LOCAL;
+  Param.Size = Size;
+
+  std::string Name;
+  Name = "__localparam_";
+  Name += utostr(LocalParams.size()+1);
+  Param.Name = Name;
+
+  unsigned Index = AllParams.size();
+  AllParams[Index] = Param;
+  LocalParams.push_back(Index);
+
+  return Index;
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXParamManager.h b/contrib/llvm/lib/Target/PTX/PTXParamManager.h
new file mode 100644
index 000000000000..9fd2de52f7f2
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXParamManager.h
@@ -0,0 +1,86 @@
+//===- PTXParamManager.h - Manager for .param variables ----------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PTXParamManager class, which manages all defined .param
+// variables for a particular function.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PTX_PARAM_MANAGER_H
+#define PTX_PARAM_MANAGER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+/// PTXParamManager - This class manages all .param variables defined for a
+/// particular function.
+class PTXParamManager {
+private:
+
+  /// PTXParamType - Type of a .param variable
+  enum PTXParamType {
+    PTX_PARAM_TYPE_ARGUMENT,
+    PTX_PARAM_TYPE_RETURN,
+    PTX_PARAM_TYPE_LOCAL
+  };
+
+  /// PTXParam - Definition of a PTX .param variable
+  struct PTXParam {
+    PTXParamType  Type;
+    unsigned      Size;
+    std::string   Name;
+  };
+
+  DenseMap<unsigned, PTXParam> AllParams;
+  SmallVector<unsigned, 4> ArgumentParams;
+  SmallVector<unsigned, 4> ReturnParams;
+  SmallVector<unsigned, 4> LocalParams;
+
+public:
+
+  typedef SmallVector<unsigned, 4>::const_iterator param_iterator;
+
+  PTXParamManager();
+
+  param_iterator arg_begin() const { return ArgumentParams.begin(); }
+  param_iterator arg_end() const { return ArgumentParams.end(); }
+  param_iterator ret_begin() const { return ReturnParams.begin(); }
+  param_iterator ret_end() const { return ReturnParams.end(); }
+  param_iterator local_begin() const { return LocalParams.begin(); }
+  param_iterator local_end() const { return LocalParams.end(); }
+
+  /// addArgumentParam - Returns a new .param used as an argument.
+  unsigned addArgumentParam(unsigned Size);
+
+  /// addReturnParam - Returns a new .param used as a return argument.
+  unsigned addReturnParam(unsigned Size);
+
+  /// addLocalParam - Returns a new .param used as a local .param variable.
+  unsigned addLocalParam(unsigned Size);
+
+  /// getParamName - Returns the name of the parameter as a string.
+  const std::string &getParamName(unsigned Param) const {
+    assert(AllParams.count(Param) == 1 && "Param has not been defined!");
+    return AllParams.find(Param)->second.Name;
+  }
+
+  /// getParamSize - Returns the size of the parameter in bits.
+  unsigned getParamSize(unsigned Param) const {
+    assert(AllParams.count(Param) == 1 && "Param has not been defined!");
+    return AllParams.find(Param)->second.Size;
+  }
+
+};
+
+}
+
+#endif
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXRegAlloc.cpp b/contrib/llvm/lib/Target/PTX/PTXRegAlloc.cpp
new file mode 100644
index 000000000000..2d2d5c30c8ca
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXRegAlloc.cpp
@@ -0,0 +1,58 @@
+//===-- PTXRegAlloc.cpp - PTX Register Allocator --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a register allocator for PTX code.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "ptx-reg-alloc"
+
+#include "PTX.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
+
+using namespace llvm;
+
+namespace {
+  // Special register allocator for PTX.
+  class PTXRegAlloc : public MachineFunctionPass {
+  public:
+    static char ID;
+    PTXRegAlloc() : MachineFunctionPass(ID) {
+      initializePHIEliminationPass(*PassRegistry::getPassRegistry());
+      initializeTwoAddressInstructionPassPass(*PassRegistry::getPassRegistry());
+    }
+
+    virtual const char* getPassName() const {
+      return "PTX Register Allocator";
+    }
+
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.setPreservesCFG();
+      AU.addRequiredID(PHIEliminationID);
+      AU.addRequiredID(TwoAddressInstructionPassID);
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+
+    virtual bool runOnMachineFunction(MachineFunction &MF) {
+      // We do not actually do anything (at least not yet).
+      return false;
+    }
+  };
+
+  char PTXRegAlloc::ID = 0;
+
+  static RegisterRegAlloc
+    ptxRegAlloc("ptx", "PTX register allocator", createPTXRegisterAllocator);
+}
+
+FunctionPass *llvm::createPTXRegisterAllocator() {
+  return new PTXRegAlloc();
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.cpp b/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.cpp
index cb56ea98a2b8..c8062664a93b 100644
--- a/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.cpp
@@ -14,6 +14,9 @@
 #include "PTX.h"
 #include "PTXRegisterInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -23,15 +26,23 @@
 using namespace llvm;
 
 PTXRegisterInfo::PTXRegisterInfo(PTXTargetMachine &TM,
-                                 const TargetInstrInfo &TII)
-  : PTXGenRegisterInfo() {
+                                 const TargetInstrInfo &tii)
+  // PTX does not have a return address register.
+  : PTXGenRegisterInfo(0), TII(tii) {
 }
 
 void PTXRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                           int SPAdj,
                                           RegScavenger *RS) const {
   unsigned Index;
-  MachineInstr& MI = *II;
+  MachineInstr &MI = *II;
+  //MachineBasicBlock &MBB = *MI.getParent();
+  //DebugLoc dl = MI.getDebugLoc();
+  //MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+
+  //unsigned Reg = MRI.createVirtualRegister(PTX::RegF32RegisterClass);
+
+  llvm_unreachable("FrameIndex should have been previously eliminated!");
 
   Index = 0;
   while (!MI.getOperand(Index).isFI()) {
@@ -46,6 +57,18 @@ void PTXRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   DEBUG(dbgs() << "- SPAdj: " << SPAdj << "\n");
   DEBUG(dbgs() << "- FrameIndex: " << FrameIndex << "\n");
 
+  //MachineInstr* MI2 = BuildMI(MBB, II, dl, TII.get(PTX::LOAD_LOCAL_F32))
+  //.addReg(Reg, RegState::Define).addImm(FrameIndex);
+  //if (MI2->findFirstPredOperandIdx() == -1) {
+  //  MI2->addOperand(MachineOperand::CreateReg(PTX::NoRegister, /*IsDef=*/false));
+  //  MI2->addOperand(MachineOperand::CreateImm(PTX::PRED_NORMAL));
+  //}
+  //MI2->dump();
+
+  //MachineOperand ESOp = MachineOperand::CreateES("__local__");
+
   // This frame index is post stack slot re-use assignments
+  //MI.getOperand(Index).ChangeToRegister(Reg, false);
   MI.getOperand(Index).ChangeToImmediate(FrameIndex);
+  //MI.getOperand(Index) = ESOp;
 }
diff --git a/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.h b/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.h
index 0b63cb6d458e..55fafe47bf35 100644
--- a/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.h
+++ b/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.h
@@ -25,8 +25,12 @@ class PTXTargetMachine;
 class MachineFunction;
 
 struct PTXRegisterInfo : public PTXGenRegisterInfo {
+private:
+  const TargetInstrInfo &TII;
+
+public:
   PTXRegisterInfo(PTXTargetMachine &TM,
-                  const TargetInstrInfo &TII);
+                  const TargetInstrInfo &tii);
 
   virtual const unsigned
     *getCalleeSavedRegs(const MachineFunction *MF = 0) const {
@@ -47,18 +51,6 @@ struct PTXRegisterInfo : public PTXGenRegisterInfo {
     llvm_unreachable("PTX does not have a frame register");
     return 0;
   }
-
-  virtual unsigned getRARegister() const {
-    llvm_unreachable("PTX does not have a return address register");
-    return 0;
-  }
-
-  virtual int getDwarfRegNum(unsigned RegNum, bool isEH) const {
-    return PTXGenRegisterInfo::getDwarfRegNumFull(RegNum, 0);
-  }
-  virtual int getLLVMRegNum(unsigned RegNum, bool isEH) const {
-    return PTXGenRegisterInfo::getLLVMRegNumFull(RegNum, 0);
-  }
 }; // struct PTXRegisterInfo
 } // namespace llvm
 
diff --git a/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.td b/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.td
index 1313d248325e..6ed6d3fe385f 100644
--- a/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.td
+++ b/contrib/llvm/lib/Target/PTX/PTXRegisterInfo.td
@@ -20,536 +20,18 @@ class PTXReg<string n> : Register<n> {
 //  Registers
 //===----------------------------------------------------------------------===//
 
-///===- Predicate Registers -----------------------------------------------===//
-
-def P0 : PTXReg<"p0">;
-def P1 : PTXReg<"p1">;
-def P2 : PTXReg<"p2">;
-def P3 : PTXReg<"p3">;
-def P4 : PTXReg<"p4">;
-def P5 : PTXReg<"p5">;
-def P6 : PTXReg<"p6">;
-def P7 : PTXReg<"p7">;
-def P8 : PTXReg<"p8">;
-def P9 : PTXReg<"p9">;
-def P10 : PTXReg<"p10">;
-def P11 : PTXReg<"p11">;
-def P12 : PTXReg<"p12">;
-def P13 : PTXReg<"p13">;
-def P14 : PTXReg<"p14">;
-def P15 : PTXReg<"p15">;
-def P16 : PTXReg<"p16">;
-def P17 : PTXReg<"p17">;
-def P18 : PTXReg<"p18">;
-def P19 : PTXReg<"p19">;
-def P20 : PTXReg<"p20">;
-def P21 : PTXReg<"p21">;
-def P22 : PTXReg<"p22">;
-def P23 : PTXReg<"p23">;
-def P24 : PTXReg<"p24">;
-def P25 : PTXReg<"p25">;
-def P26 : PTXReg<"p26">;
-def P27 : PTXReg<"p27">;
-def P28 : PTXReg<"p28">;
-def P29 : PTXReg<"p29">;
-def P30 : PTXReg<"p30">;
-def P31 : PTXReg<"p31">;
-def P32 : PTXReg<"p32">;
-def P33 : PTXReg<"p33">;
-def P34 : PTXReg<"p34">;
-def P35 : PTXReg<"p35">;
-def P36 : PTXReg<"p36">;
-def P37 : PTXReg<"p37">;
-def P38 : PTXReg<"p38">;
-def P39 : PTXReg<"p39">;
-def P40 : PTXReg<"p40">;
-def P41 : PTXReg<"p41">;
-def P42 : PTXReg<"p42">;
-def P43 : PTXReg<"p43">;
-def P44 : PTXReg<"p44">;
-def P45 : PTXReg<"p45">;
-def P46 : PTXReg<"p46">;
-def P47 : PTXReg<"p47">;
-def P48 : PTXReg<"p48">;
-def P49 : PTXReg<"p49">;
-def P50 : PTXReg<"p50">;
-def P51 : PTXReg<"p51">;
-def P52 : PTXReg<"p52">;
-def P53 : PTXReg<"p53">;
-def P54 : PTXReg<"p54">;
-def P55 : PTXReg<"p55">;
-def P56 : PTXReg<"p56">;
-def P57 : PTXReg<"p57">;
-def P58 : PTXReg<"p58">;
-def P59 : PTXReg<"p59">;
-def P60 : PTXReg<"p60">;
-def P61 : PTXReg<"p61">;
-def P62 : PTXReg<"p62">;
-def P63 : PTXReg<"p63">;
-def P64 : PTXReg<"p64">;
-def P65 : PTXReg<"p65">;
-def P66 : PTXReg<"p66">;
-def P67 : PTXReg<"p67">;
-def P68 : PTXReg<"p68">;
-def P69 : PTXReg<"p69">;
-def P70 : PTXReg<"p70">;
-def P71 : PTXReg<"p71">;
-def P72 : PTXReg<"p72">;
-def P73 : PTXReg<"p73">;
-def P74 : PTXReg<"p74">;
-def P75 : PTXReg<"p75">;
-def P76 : PTXReg<"p76">;
-def P77 : PTXReg<"p77">;
-def P78 : PTXReg<"p78">;
-def P79 : PTXReg<"p79">;
-def P80 : PTXReg<"p80">;
-def P81 : PTXReg<"p81">;
-def P82 : PTXReg<"p82">;
-def P83 : PTXReg<"p83">;
-def P84 : PTXReg<"p84">;
-def P85 : PTXReg<"p85">;
-def P86 : PTXReg<"p86">;
-def P87 : PTXReg<"p87">;
-def P88 : PTXReg<"p88">;
-def P89 : PTXReg<"p89">;
-def P90 : PTXReg<"p90">;
-def P91 : PTXReg<"p91">;
-def P92 : PTXReg<"p92">;
-def P93 : PTXReg<"p93">;
-def P94 : PTXReg<"p94">;
-def P95 : PTXReg<"p95">;
-def P96 : PTXReg<"p96">;
-def P97 : PTXReg<"p97">;
-def P98 : PTXReg<"p98">;
-def P99 : PTXReg<"p99">;
-def P100 : PTXReg<"p100">;
-def P101 : PTXReg<"p101">;
-def P102 : PTXReg<"p102">;
-def P103 : PTXReg<"p103">;
-def P104 : PTXReg<"p104">;
-def P105 : PTXReg<"p105">;
-def P106 : PTXReg<"p106">;
-def P107 : PTXReg<"p107">;
-def P108 : PTXReg<"p108">;
-def P109 : PTXReg<"p109">;
-def P110 : PTXReg<"p110">;
-def P111 : PTXReg<"p111">;
-def P112 : PTXReg<"p112">;
-def P113 : PTXReg<"p113">;
-def P114 : PTXReg<"p114">;
-def P115 : PTXReg<"p115">;
-def P116 : PTXReg<"p116">;
-def P117 : PTXReg<"p117">;
-def P118 : PTXReg<"p118">;
-def P119 : PTXReg<"p119">;
-def P120 : PTXReg<"p120">;
-def P121 : PTXReg<"p121">;
-def P122 : PTXReg<"p122">;
-def P123 : PTXReg<"p123">;
-def P124 : PTXReg<"p124">;
-def P125 : PTXReg<"p125">;
-def P126 : PTXReg<"p126">;
-def P127 : PTXReg<"p127">;
-
-///===- 16-Bit Registers --------------------------------------------------===//
-
-def RH0 : PTXReg<"rh0">;
-def RH1 : PTXReg<"rh1">;
-def RH2 : PTXReg<"rh2">;
-def RH3 : PTXReg<"rh3">;
-def RH4 : PTXReg<"rh4">;
-def RH5 : PTXReg<"rh5">;
-def RH6 : PTXReg<"rh6">;
-def RH7 : PTXReg<"rh7">;
-def RH8 : PTXReg<"rh8">;
-def RH9 : PTXReg<"rh9">;
-def RH10 : PTXReg<"rh10">;
-def RH11 : PTXReg<"rh11">;
-def RH12 : PTXReg<"rh12">;
-def RH13 : PTXReg<"rh13">;
-def RH14 : PTXReg<"rh14">;
-def RH15 : PTXReg<"rh15">;
-def RH16 : PTXReg<"rh16">;
-def RH17 : PTXReg<"rh17">;
-def RH18 : PTXReg<"rh18">;
-def RH19 : PTXReg<"rh19">;
-def RH20 : PTXReg<"rh20">;
-def RH21 : PTXReg<"rh21">;
-def RH22 : PTXReg<"rh22">;
-def RH23 : PTXReg<"rh23">;
-def RH24 : PTXReg<"rh24">;
-def RH25 : PTXReg<"rh25">;
-def RH26 : PTXReg<"rh26">;
-def RH27 : PTXReg<"rh27">;
-def RH28 : PTXReg<"rh28">;
-def RH29 : PTXReg<"rh29">;
-def RH30 : PTXReg<"rh30">;
-def RH31 : PTXReg<"rh31">;
-def RH32 : PTXReg<"rh32">;
-def RH33 : PTXReg<"rh33">;
-def RH34 : PTXReg<"rh34">;
-def RH35 : PTXReg<"rh35">;
-def RH36 : PTXReg<"rh36">;
-def RH37 : PTXReg<"rh37">;
-def RH38 : PTXReg<"rh38">;
-def RH39 : PTXReg<"rh39">;
-def RH40 : PTXReg<"rh40">;
-def RH41 : PTXReg<"rh41">;
-def RH42 : PTXReg<"rh42">;
-def RH43 : PTXReg<"rh43">;
-def RH44 : PTXReg<"rh44">;
-def RH45 : PTXReg<"rh45">;
-def RH46 : PTXReg<"rh46">;
-def RH47 : PTXReg<"rh47">;
-def RH48 : PTXReg<"rh48">;
-def RH49 : PTXReg<"rh49">;
-def RH50 : PTXReg<"rh50">;
-def RH51 : PTXReg<"rh51">;
-def RH52 : PTXReg<"rh52">;
-def RH53 : PTXReg<"rh53">;
-def RH54 : PTXReg<"rh54">;
-def RH55 : PTXReg<"rh55">;
-def RH56 : PTXReg<"rh56">;
-def RH57 : PTXReg<"rh57">;
-def RH58 : PTXReg<"rh58">;
-def RH59 : PTXReg<"rh59">;
-def RH60 : PTXReg<"rh60">;
-def RH61 : PTXReg<"rh61">;
-def RH62 : PTXReg<"rh62">;
-def RH63 : PTXReg<"rh63">;
-def RH64 : PTXReg<"rh64">;
-def RH65 : PTXReg<"rh65">;
-def RH66 : PTXReg<"rh66">;
-def RH67 : PTXReg<"rh67">;
-def RH68 : PTXReg<"rh68">;
-def RH69 : PTXReg<"rh69">;
-def RH70 : PTXReg<"rh70">;
-def RH71 : PTXReg<"rh71">;
-def RH72 : PTXReg<"rh72">;
-def RH73 : PTXReg<"rh73">;
-def RH74 : PTXReg<"rh74">;
-def RH75 : PTXReg<"rh75">;
-def RH76 : PTXReg<"rh76">;
-def RH77 : PTXReg<"rh77">;
-def RH78 : PTXReg<"rh78">;
-def RH79 : PTXReg<"rh79">;
-def RH80 : PTXReg<"rh80">;
-def RH81 : PTXReg<"rh81">;
-def RH82 : PTXReg<"rh82">;
-def RH83 : PTXReg<"rh83">;
-def RH84 : PTXReg<"rh84">;
-def RH85 : PTXReg<"rh85">;
-def RH86 : PTXReg<"rh86">;
-def RH87 : PTXReg<"rh87">;
-def RH88 : PTXReg<"rh88">;
-def RH89 : PTXReg<"rh89">;
-def RH90 : PTXReg<"rh90">;
-def RH91 : PTXReg<"rh91">;
-def RH92 : PTXReg<"rh92">;
-def RH93 : PTXReg<"rh93">;
-def RH94 : PTXReg<"rh94">;
-def RH95 : PTXReg<"rh95">;
-def RH96 : PTXReg<"rh96">;
-def RH97 : PTXReg<"rh97">;
-def RH98 : PTXReg<"rh98">;
-def RH99 : PTXReg<"rh99">;
-def RH100 : PTXReg<"rh100">;
-def RH101 : PTXReg<"rh101">;
-def RH102 : PTXReg<"rh102">;
-def RH103 : PTXReg<"rh103">;
-def RH104 : PTXReg<"rh104">;
-def RH105 : PTXReg<"rh105">;
-def RH106 : PTXReg<"rh106">;
-def RH107 : PTXReg<"rh107">;
-def RH108 : PTXReg<"rh108">;
-def RH109 : PTXReg<"rh109">;
-def RH110 : PTXReg<"rh110">;
-def RH111 : PTXReg<"rh111">;
-def RH112 : PTXReg<"rh112">;
-def RH113 : PTXReg<"rh113">;
-def RH114 : PTXReg<"rh114">;
-def RH115 : PTXReg<"rh115">;
-def RH116 : PTXReg<"rh116">;
-def RH117 : PTXReg<"rh117">;
-def RH118 : PTXReg<"rh118">;
-def RH119 : PTXReg<"rh119">;
-def RH120 : PTXReg<"rh120">;
-def RH121 : PTXReg<"rh121">;
-def RH122 : PTXReg<"rh122">;
-def RH123 : PTXReg<"rh123">;
-def RH124 : PTXReg<"rh124">;
-def RH125 : PTXReg<"rh125">;
-def RH126 : PTXReg<"rh126">;
-def RH127 : PTXReg<"rh127">;
-
-///===- 32-Bit Registers --------------------------------------------------===//
-
-def R0 : PTXReg<"r0">;
-def R1 : PTXReg<"r1">;
-def R2 : PTXReg<"r2">;
-def R3 : PTXReg<"r3">;
-def R4 : PTXReg<"r4">;
-def R5 : PTXReg<"r5">;
-def R6 : PTXReg<"r6">;
-def R7 : PTXReg<"r7">;
-def R8 : PTXReg<"r8">;
-def R9 : PTXReg<"r9">;
-def R10 : PTXReg<"r10">;
-def R11 : PTXReg<"r11">;
-def R12 : PTXReg<"r12">;
-def R13 : PTXReg<"r13">;
-def R14 : PTXReg<"r14">;
-def R15 : PTXReg<"r15">;
-def R16 : PTXReg<"r16">;
-def R17 : PTXReg<"r17">;
-def R18 : PTXReg<"r18">;
-def R19 : PTXReg<"r19">;
-def R20 : PTXReg<"r20">;
-def R21 : PTXReg<"r21">;
-def R22 : PTXReg<"r22">;
-def R23 : PTXReg<"r23">;
-def R24 : PTXReg<"r24">;
-def R25 : PTXReg<"r25">;
-def R26 : PTXReg<"r26">;
-def R27 : PTXReg<"r27">;
-def R28 : PTXReg<"r28">;
-def R29 : PTXReg<"r29">;
-def R30 : PTXReg<"r30">;
-def R31 : PTXReg<"r31">;
-def R32 : PTXReg<"r32">;
-def R33 : PTXReg<"r33">;
-def R34 : PTXReg<"r34">;
-def R35 : PTXReg<"r35">;
-def R36 : PTXReg<"r36">;
-def R37 : PTXReg<"r37">;
-def R38 : PTXReg<"r38">;
-def R39 : PTXReg<"r39">;
-def R40 : PTXReg<"r40">;
-def R41 : PTXReg<"r41">;
-def R42 : PTXReg<"r42">;
-def R43 : PTXReg<"r43">;
-def R44 : PTXReg<"r44">;
-def R45 : PTXReg<"r45">;
-def R46 : PTXReg<"r46">;
-def R47 : PTXReg<"r47">;
-def R48 : PTXReg<"r48">;
-def R49 : PTXReg<"r49">;
-def R50 : PTXReg<"r50">;
-def R51 : PTXReg<"r51">;
-def R52 : PTXReg<"r52">;
-def R53 : PTXReg<"r53">;
-def R54 : PTXReg<"r54">;
-def R55 : PTXReg<"r55">;
-def R56 : PTXReg<"r56">;
-def R57 : PTXReg<"r57">;
-def R58 : PTXReg<"r58">;
-def R59 : PTXReg<"r59">;
-def R60 : PTXReg<"r60">;
-def R61 : PTXReg<"r61">;
-def R62 : PTXReg<"r62">;
-def R63 : PTXReg<"r63">;
-def R64 : PTXReg<"r64">;
-def R65 : PTXReg<"r65">;
-def R66 : PTXReg<"r66">;
-def R67 : PTXReg<"r67">;
-def R68 : PTXReg<"r68">;
-def R69 : PTXReg<"r69">;
-def R70 : PTXReg<"r70">;
-def R71 : PTXReg<"r71">;
-def R72 : PTXReg<"r72">;
-def R73 : PTXReg<"r73">;
-def R74 : PTXReg<"r74">;
-def R75 : PTXReg<"r75">;
-def R76 : PTXReg<"r76">;
-def R77 : PTXReg<"r77">;
-def R78 : PTXReg<"r78">;
-def R79 : PTXReg<"r79">;
-def R80 : PTXReg<"r80">;
-def R81 : PTXReg<"r81">;
-def R82 : PTXReg<"r82">;
-def R83 : PTXReg<"r83">;
-def R84 : PTXReg<"r84">;
-def R85 : PTXReg<"r85">;
-def R86 : PTXReg<"r86">;
-def R87 : PTXReg<"r87">;
-def R88 : PTXReg<"r88">;
-def R89 : PTXReg<"r89">;
-def R90 : PTXReg<"r90">;
-def R91 : PTXReg<"r91">;
-def R92 : PTXReg<"r92">;
-def R93 : PTXReg<"r93">;
-def R94 : PTXReg<"r94">;
-def R95 : PTXReg<"r95">;
-def R96 : PTXReg<"r96">;
-def R97 : PTXReg<"r97">;
-def R98 : PTXReg<"r98">;
-def R99 : PTXReg<"r99">;
-def R100 : PTXReg<"r100">;
-def R101 : PTXReg<"r101">;
-def R102 : PTXReg<"r102">;
-def R103 : PTXReg<"r103">;
-def R104 : PTXReg<"r104">;
-def R105 : PTXReg<"r105">;
-def R106 : PTXReg<"r106">;
-def R107 : PTXReg<"r107">;
-def R108 : PTXReg<"r108">;
-def R109 : PTXReg<"r109">;
-def R110 : PTXReg<"r110">;
-def R111 : PTXReg<"r111">;
-def R112 : PTXReg<"r112">;
-def R113 : PTXReg<"r113">;
-def R114 : PTXReg<"r114">;
-def R115 : PTXReg<"r115">;
-def R116 : PTXReg<"r116">;
-def R117 : PTXReg<"r117">;
-def R118 : PTXReg<"r118">;
-def R119 : PTXReg<"r119">;
-def R120 : PTXReg<"r120">;
-def R121 : PTXReg<"r121">;
-def R122 : PTXReg<"r122">;
-def R123 : PTXReg<"r123">;
-def R124 : PTXReg<"r124">;
-def R125 : PTXReg<"r125">;
-def R126 : PTXReg<"r126">;
-def R127 : PTXReg<"r127">;
-
-///===- 64-Bit Registers --------------------------------------------------===//
-
-def RD0 : PTXReg<"rd0">;
-def RD1 : PTXReg<"rd1">;
-def RD2 : PTXReg<"rd2">;
-def RD3 : PTXReg<"rd3">;
-def RD4 : PTXReg<"rd4">;
-def RD5 : PTXReg<"rd5">;
-def RD6 : PTXReg<"rd6">;
-def RD7 : PTXReg<"rd7">;
-def RD8 : PTXReg<"rd8">;
-def RD9 : PTXReg<"rd9">;
-def RD10 : PTXReg<"rd10">;
-def RD11 : PTXReg<"rd11">;
-def RD12 : PTXReg<"rd12">;
-def RD13 : PTXReg<"rd13">;
-def RD14 : PTXReg<"rd14">;
-def RD15 : PTXReg<"rd15">;
-def RD16 : PTXReg<"rd16">;
-def RD17 : PTXReg<"rd17">;
-def RD18 : PTXReg<"rd18">;
-def RD19 : PTXReg<"rd19">;
-def RD20 : PTXReg<"rd20">;
-def RD21 : PTXReg<"rd21">;
-def RD22 : PTXReg<"rd22">;
-def RD23 : PTXReg<"rd23">;
-def RD24 : PTXReg<"rd24">;
-def RD25 : PTXReg<"rd25">;
-def RD26 : PTXReg<"rd26">;
-def RD27 : PTXReg<"rd27">;
-def RD28 : PTXReg<"rd28">;
-def RD29 : PTXReg<"rd29">;
-def RD30 : PTXReg<"rd30">;
-def RD31 : PTXReg<"rd31">;
-def RD32 : PTXReg<"rd32">;
-def RD33 : PTXReg<"rd33">;
-def RD34 : PTXReg<"rd34">;
-def RD35 : PTXReg<"rd35">;
-def RD36 : PTXReg<"rd36">;
-def RD37 : PTXReg<"rd37">;
-def RD38 : PTXReg<"rd38">;
-def RD39 : PTXReg<"rd39">;
-def RD40 : PTXReg<"rd40">;
-def RD41 : PTXReg<"rd41">;
-def RD42 : PTXReg<"rd42">;
-def RD43 : PTXReg<"rd43">;
-def RD44 : PTXReg<"rd44">;
-def RD45 : PTXReg<"rd45">;
-def RD46 : PTXReg<"rd46">;
-def RD47 : PTXReg<"rd47">;
-def RD48 : PTXReg<"rd48">;
-def RD49 : PTXReg<"rd49">;
-def RD50 : PTXReg<"rd50">;
-def RD51 : PTXReg<"rd51">;
-def RD52 : PTXReg<"rd52">;
-def RD53 : PTXReg<"rd53">;
-def RD54 : PTXReg<"rd54">;
-def RD55 : PTXReg<"rd55">;
-def RD56 : PTXReg<"rd56">;
-def RD57 : PTXReg<"rd57">;
-def RD58 : PTXReg<"rd58">;
-def RD59 : PTXReg<"rd59">;
-def RD60 : PTXReg<"rd60">;
-def RD61 : PTXReg<"rd61">;
-def RD62 : PTXReg<"rd62">;
-def RD63 : PTXReg<"rd63">;
-def RD64 : PTXReg<"rd64">;
-def RD65 : PTXReg<"rd65">;
-def RD66 : PTXReg<"rd66">;
-def RD67 : PTXReg<"rd67">;
-def RD68 : PTXReg<"rd68">;
-def RD69 : PTXReg<"rd69">;
-def RD70 : PTXReg<"rd70">;
-def RD71 : PTXReg<"rd71">;
-def RD72 : PTXReg<"rd72">;
-def RD73 : PTXReg<"rd73">;
-def RD74 : PTXReg<"rd74">;
-def RD75 : PTXReg<"rd75">;
-def RD76 : PTXReg<"rd76">;
-def RD77 : PTXReg<"rd77">;
-def RD78 : PTXReg<"rd78">;
-def RD79 : PTXReg<"rd79">;
-def RD80 : PTXReg<"rd80">;
-def RD81 : PTXReg<"rd81">;
-def RD82 : PTXReg<"rd82">;
-def RD83 : PTXReg<"rd83">;
-def RD84 : PTXReg<"rd84">;
-def RD85 : PTXReg<"rd85">;
-def RD86 : PTXReg<"rd86">;
-def RD87 : PTXReg<"rd87">;
-def RD88 : PTXReg<"rd88">;
-def RD89 : PTXReg<"rd89">;
-def RD90 : PTXReg<"rd90">;
-def RD91 : PTXReg<"rd91">;
-def RD92 : PTXReg<"rd92">;
-def RD93 : PTXReg<"rd93">;
-def RD94 : PTXReg<"rd94">;
-def RD95 : PTXReg<"rd95">;
-def RD96 : PTXReg<"rd96">;
-def RD97 : PTXReg<"rd97">;
-def RD98 : PTXReg<"rd98">;
-def RD99 : PTXReg<"rd99">;
-def RD100 : PTXReg<"rd100">;
-def RD101 : PTXReg<"rd101">;
-def RD102 : PTXReg<"rd102">;
-def RD103 : PTXReg<"rd103">;
-def RD104 : PTXReg<"rd104">;
-def RD105 : PTXReg<"rd105">;
-def RD106 : PTXReg<"rd106">;
-def RD107 : PTXReg<"rd107">;
-def RD108 : PTXReg<"rd108">;
-def RD109 : PTXReg<"rd109">;
-def RD110 : PTXReg<"rd110">;
-def RD111 : PTXReg<"rd111">;
-def RD112 : PTXReg<"rd112">;
-def RD113 : PTXReg<"rd113">;
-def RD114 : PTXReg<"rd114">;
-def RD115 : PTXReg<"rd115">;
-def RD116 : PTXReg<"rd116">;
-def RD117 : PTXReg<"rd117">;
-def RD118 : PTXReg<"rd118">;
-def RD119 : PTXReg<"rd119">;
-def RD120 : PTXReg<"rd120">;
-def RD121 : PTXReg<"rd121">;
-def RD122 : PTXReg<"rd122">;
-def RD123 : PTXReg<"rd123">;
-def RD124 : PTXReg<"rd124">;
-def RD125 : PTXReg<"rd125">;
-def RD126 : PTXReg<"rd126">;
-def RD127 : PTXReg<"rd127">;
+// The generated register info code throws warnings for empty register classes
+// (e.g. zero-length arrays), so we use a dummy register here just to prevent
+// these warnings.
+def DUMMY_REG : PTXReg<"R0">;
 
 //===----------------------------------------------------------------------===//
 //  Register classes
 //===----------------------------------------------------------------------===//
-def RegPred : RegisterClass<"PTX", [i1], 8, (sequence "P%u", 0, 127)>;
-def RegI16 : RegisterClass<"PTX", [i16], 16, (sequence "RH%u", 0, 127)>;
-def RegI32 : RegisterClass<"PTX", [i32], 32, (sequence "R%u", 0, 127)>;
-def RegI64 : RegisterClass<"PTX", [i64], 64, (sequence "RD%u", 0, 127)>;
-def RegF32 : RegisterClass<"PTX", [f32], 32, (sequence "R%u", 0, 127)>;
-def RegF64 : RegisterClass<"PTX", [f64], 64, (sequence "RD%u", 0, 127)>;
+def RegPred : RegisterClass<"PTX", [i1], 8, (add DUMMY_REG)>;
+def RegI16 : RegisterClass<"PTX", [i16], 16, (add DUMMY_REG)>;
+def RegI32 : RegisterClass<"PTX", [i32], 32, (add DUMMY_REG)>;
+def RegI64 : RegisterClass<"PTX", [i64], 64, (add DUMMY_REG)>;
+def RegF32 : RegisterClass<"PTX", [f32], 32, (add DUMMY_REG)>;
+def RegF64 : RegisterClass<"PTX", [f64], 64, (add DUMMY_REG)>;
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXSelectionDAGInfo.cpp b/contrib/llvm/lib/Target/PTX/PTXSelectionDAGInfo.cpp
new file mode 100644
index 000000000000..50ef14a13d94
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXSelectionDAGInfo.cpp
@@ -0,0 +1,149 @@
+//===-- PTXSelectionDAGInfo.cpp - PTX SelectionDAG Info -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the PTXSelectionDAGInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "ptx-selectiondag-info"
+#include "PTXTargetMachine.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+using namespace llvm;
+
+PTXSelectionDAGInfo::PTXSelectionDAGInfo(const TargetMachine &TM)
+  : TargetSelectionDAGInfo(TM),
+    Subtarget(&TM.getSubtarget<PTXSubtarget>()) {
+}
+
+PTXSelectionDAGInfo::~PTXSelectionDAGInfo() {
+}
+
+SDValue
+PTXSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
+                                             SDValue Chain,
+                                             SDValue Dst, SDValue Src,
+                                             SDValue Size, unsigned Align,
+                                             bool isVolatile, bool AlwaysInline,
+                                             MachinePointerInfo DstPtrInfo,
+                                          MachinePointerInfo SrcPtrInfo) const {
+  // Do repeated 4-byte loads and stores. To be improved.
+  // This requires 4-byte alignment.
+  if ((Align & 3) != 0)
+    return SDValue();
+  // This requires the copy size to be a constant, preferably
+  // within a subtarget-specific limit.
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (!ConstantSize)
+    return SDValue();
+  uint64_t SizeVal = ConstantSize->getZExtValue();
+  // Always inline memcpys. In PTX, we do not have a C library that provides
+  // a memcpy function.
+  //if (!AlwaysInline)
+  //  return SDValue();
+
+  unsigned BytesLeft = SizeVal & 3;
+  unsigned NumMemOps = SizeVal >> 2;
+  unsigned EmittedNumMemOps = 0;
+  EVT VT = MVT::i32;
+  unsigned VTSize = 4;
+  unsigned i = 0;
+  const unsigned MAX_LOADS_IN_LDM = 6;
+  SDValue TFOps[MAX_LOADS_IN_LDM];
+  SDValue Loads[MAX_LOADS_IN_LDM];
+  uint64_t SrcOff = 0, DstOff = 0;
+  EVT PointerType = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
+
+  // Emit up to MAX_LOADS_IN_LDM loads, then a TokenFactor barrier, then the
+  // same number of stores.  The loads and stores will get combined into
+  // ldm/stm later on.
+  while (EmittedNumMemOps < NumMemOps) {
+    for (i = 0;
+         i < MAX_LOADS_IN_LDM && EmittedNumMemOps + i < NumMemOps; ++i) {
+      Loads[i] = DAG.getLoad(VT, dl, Chain,
+                             DAG.getNode(ISD::ADD, dl, PointerType, Src,
+                                         DAG.getConstant(SrcOff, PointerType)),
+                             SrcPtrInfo.getWithOffset(SrcOff), isVolatile,
+                             false, 0);
+      TFOps[i] = Loads[i].getValue(1);
+      SrcOff += VTSize;
+    }
+    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
+
+    for (i = 0;
+         i < MAX_LOADS_IN_LDM && EmittedNumMemOps + i < NumMemOps; ++i) {
+      TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
+                              DAG.getNode(ISD::ADD, dl, PointerType, Dst,
+                                          DAG.getConstant(DstOff, PointerType)),
+                              DstPtrInfo.getWithOffset(DstOff),
+                              isVolatile, false, 0);
+      DstOff += VTSize;
+    }
+    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
+
+    EmittedNumMemOps += i;
+  }
+
+  if (BytesLeft == 0)
+    return Chain;
+
+  // Issue loads / stores for the trailing (1 - 3) bytes.
+  unsigned BytesLeftSave = BytesLeft;
+  i = 0;
+  while (BytesLeft) {
+    if (BytesLeft >= 2) {
+      VT = MVT::i16;
+      VTSize = 2;
+    } else {
+      VT = MVT::i8;
+      VTSize = 1;
+    }
+
+    Loads[i] = DAG.getLoad(VT, dl, Chain,
+                           DAG.getNode(ISD::ADD, dl, PointerType, Src,
+                                       DAG.getConstant(SrcOff, PointerType)),
+                           SrcPtrInfo.getWithOffset(SrcOff), false, false, 0);
+    TFOps[i] = Loads[i].getValue(1);
+    ++i;
+    SrcOff += VTSize;
+    BytesLeft -= VTSize;
+  }
+  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
+
+  i = 0;
+  BytesLeft = BytesLeftSave;
+  while (BytesLeft) {
+    if (BytesLeft >= 2) {
+      VT = MVT::i16;
+      VTSize = 2;
+    } else {
+      VT = MVT::i8;
+      VTSize = 1;
+    }
+
+    TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
+                            DAG.getNode(ISD::ADD, dl, PointerType, Dst,
+                                        DAG.getConstant(DstOff, PointerType)),
+                            DstPtrInfo.getWithOffset(DstOff), false, false, 0);
+    ++i;
+    DstOff += VTSize;
+    BytesLeft -= VTSize;
+  }
+  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
+}
+
+SDValue PTXSelectionDAGInfo::
+EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
+                        SDValue Chain, SDValue Dst,
+                        SDValue Src, SDValue Size,
+                        unsigned Align, bool isVolatile,
+                        MachinePointerInfo DstPtrInfo) const {
+  llvm_unreachable("memset lowering not implemented for PTX yet");
+}
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXSelectionDAGInfo.h b/contrib/llvm/lib/Target/PTX/PTXSelectionDAGInfo.h
new file mode 100644
index 000000000000..e0c716718f08
--- /dev/null
+++ b/contrib/llvm/lib/Target/PTX/PTXSelectionDAGInfo.h
@@ -0,0 +1,53 @@
+//===-- PTXSelectionDAGInfo.h - PTX SelectionDAG Info -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PTX subclass for TargetSelectionDAGInfo.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PTXSELECTIONDAGINFO_H
+#define PTXSELECTIONDAGINFO_H
+
+#include "llvm/Target/TargetSelectionDAGInfo.h"
+
+namespace llvm {
+
+/// PTXSelectionDAGInfo - TargetSelectionDAGInfo sub-class for the PTX target.
+/// At the moment, this is mostly just a copy of ARMSelectionDAGInfo.
+class PTXSelectionDAGInfo : public TargetSelectionDAGInfo {
+  /// Subtarget - Keep a pointer to the PTXSubtarget around so that we can
+  /// make the right decision when generating code for different targets.
+  const PTXSubtarget *Subtarget;
+
+public:
+  explicit PTXSelectionDAGInfo(const TargetMachine &TM);
+  ~PTXSelectionDAGInfo();
+
+  virtual
+  SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
+                                  SDValue Chain,
+                                  SDValue Dst, SDValue Src,
+                                  SDValue Size, unsigned Align,
+                                  bool isVolatile, bool AlwaysInline,
+                                  MachinePointerInfo DstPtrInfo,
+                                  MachinePointerInfo SrcPtrInfo) const;
+
+  virtual
+  SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
+                                  SDValue Chain,
+                                  SDValue Op1, SDValue Op2,
+                                  SDValue Op3, unsigned Align,
+                                  bool isVolatile,
+                                  MachinePointerInfo DstPtrInfo) const;
+};
+
+}
+
+#endif
+
diff --git a/contrib/llvm/lib/Target/PTX/PTXSubtarget.cpp b/contrib/llvm/lib/Target/PTX/PTXSubtarget.cpp
index 8ec646e46f68..1eb57d2f1702 100644
--- a/contrib/llvm/lib/Target/PTX/PTXSubtarget.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXSubtarget.cpp
@@ -14,7 +14,7 @@
 #include "PTXSubtarget.h"
 #include "PTX.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/PTX/PTXSubtarget.h b/contrib/llvm/lib/Target/PTX/PTXSubtarget.h
index 0921f1f22c49..b946d7c11cef 100644
--- a/contrib/llvm/lib/Target/PTX/PTXSubtarget.h
+++ b/contrib/llvm/lib/Target/PTX/PTXSubtarget.h
@@ -114,7 +114,16 @@ class StringRef;
                (PTXTarget >= PTX_COMPUTE_2_0 && PTXTarget < PTX_LAST_COMPUTE);
       }
 
-    void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
+      bool callsAreHandled() const {
+        return (PTXTarget >= PTX_SM_2_0 && PTXTarget < PTX_LAST_SM) ||
+               (PTXTarget >= PTX_COMPUTE_2_0 && PTXTarget < PTX_LAST_COMPUTE);
+      }
+
+      bool emitPtrAttribute() const {
+        return PTXVersion >= PTX_VERSION_2_2;
+      }
+
+      void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
   }; // class PTXSubtarget
 } // namespace llvm
 
diff --git a/contrib/llvm/lib/Target/PTX/PTXTargetMachine.cpp b/contrib/llvm/lib/Target/PTX/PTXTargetMachine.cpp
index ab926e02d66f..449a3d9fc8d4 100644
--- a/contrib/llvm/lib/Target/PTX/PTXTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/PTX/PTXTargetMachine.cpp
@@ -14,8 +14,32 @@
 #include "PTX.h"
 #include "PTXTargetMachine.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Analysis/Passes.h"
+#include "llvm/Analysis/Verifier.h"
+#include "llvm/Assembly/PrintModulePass.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/TargetRegistry.h"
+
 
 using namespace llvm;
 
@@ -25,7 +49,7 @@ namespace llvm {
                                    bool useCFI,
                                    MCInstPrinter *InstPrint,
                                    MCCodeEmitter *CE,
-                                   TargetAsmBackend *TAB,
+                                   MCAsmBackend *MAB,
                                    bool ShowInst);
 }
 
@@ -43,34 +67,47 @@ namespace {
     "e-p:32:32-i64:32:32-f64:32:32-v128:32:128-v64:32:64-n32:64";
   const char* DataLayout64 =
     "e-p:64:64-i64:32:32-f64:32:32-v128:32:128-v64:32:64-n32:64";
+
+  // Copied from LLVMTargetMachine.cpp
+  void printNoVerify(PassManagerBase &PM, const char *Banner) {
+    if (PrintMachineCode)
+      PM.add(createMachineFunctionPrinterPass(dbgs(), Banner));
+  }
+
+  void printAndVerify(PassManagerBase &PM,
+                      const char *Banner) {
+    if (PrintMachineCode)
+      PM.add(createMachineFunctionPrinterPass(dbgs(), Banner));
+
+    //if (VerifyMachineCode)
+    //  PM.add(createMachineVerifierPass(Banner));
+  }
 }
 
 // DataLayout and FrameLowering are filled with dummy data
 PTXTargetMachine::PTXTargetMachine(const Target &T,
-                                   const std::string &TT,
-                                   const std::string &CPU,
-                                   const std::string &FS,
+                                   StringRef TT, StringRef CPU, StringRef FS,
+                                   Reloc::Model RM, CodeModel::Model CM,
                                    bool is64Bit)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     DataLayout(is64Bit ? DataLayout64 : DataLayout32),
     Subtarget(TT, CPU, FS, is64Bit),
     FrameLowering(Subtarget),
     InstrInfo(*this),
+    TSInfo(*this),
     TLInfo(*this) {
 }
 
-PTX32TargetMachine::PTX32TargetMachine(const Target &T,
-                                       const std::string& TT,
-                                       const std::string& CPU,
-                                       const std::string& FS)
-  : PTXTargetMachine(T, TT, CPU, FS, false) {
+PTX32TargetMachine::PTX32TargetMachine(const Target &T, StringRef TT,
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM)
+  : PTXTargetMachine(T, TT, CPU, FS, RM, CM, false) {
 }
 
-PTX64TargetMachine::PTX64TargetMachine(const Target &T,
-                                       const std::string& TT,
-                                       const std::string& CPU,
-                                       const std::string& FS)
-  : PTXTargetMachine(T, TT, CPU, FS, true) {
+PTX64TargetMachine::PTX64TargetMachine(const Target &T, StringRef TT,
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM)
+  : PTXTargetMachine(T, TT, CPU, FS, RM, CM, true) {
 }
 
 bool PTXTargetMachine::addInstSelector(PassManagerBase &PM,
@@ -82,6 +119,255 @@ bool PTXTargetMachine::addInstSelector(PassManagerBase &PM,
 bool PTXTargetMachine::addPostRegAlloc(PassManagerBase &PM,
                                        CodeGenOpt::Level OptLevel) {
   // PTXMFInfoExtract must after register allocation!
-  PM.add(createPTXMFInfoExtract(*this, OptLevel));
+  //PM.add(createPTXMFInfoExtract(*this, OptLevel));
+  return false;
+}
+
+bool PTXTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
+                                           formatted_raw_ostream &Out,
+                                           CodeGenFileType FileType,
+                                           CodeGenOpt::Level OptLevel,
+                                           bool DisableVerify) {
+  // This is mostly based on LLVMTargetMachine::addPassesToEmitFile
+
+  // Add common CodeGen passes.
+  MCContext *Context = 0;
+  if (addCommonCodeGenPasses(PM, OptLevel, DisableVerify, Context))
+    return true;
+  assert(Context != 0 && "Failed to get MCContext");
+
+  if (hasMCSaveTempLabels())
+    Context->setAllowTemporaryLabels(false);
+
+  const MCAsmInfo &MAI = *getMCAsmInfo();
+  const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
+  OwningPtr<MCStreamer> AsmStreamer;
+
+  switch (FileType) {
+  default: return true;
+  case CGFT_AssemblyFile: {
+    MCInstPrinter *InstPrinter =
+      getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI, STI);
+
+    // Create a code emitter if asked to show the encoding.
+    MCCodeEmitter *MCE = 0;
+    MCAsmBackend *MAB = 0;
+
+    MCStreamer *S = getTarget().createAsmStreamer(*Context, Out,
+                                                  true, /* verbose asm */
+                                                  hasMCUseLoc(),
+                                                  hasMCUseCFI(),
+                                                  InstPrinter,
+                                                  MCE, MAB,
+                                                  false /* show MC encoding */);
+    AsmStreamer.reset(S);
+    break;
+  }
+  case CGFT_ObjectFile: {
+    llvm_unreachable("Object file emission is not supported with PTX");
+  }
+  case CGFT_Null:
+    // The Null output is intended for use for performance analysis and testing,
+    // not real users.
+    AsmStreamer.reset(createNullStreamer(*Context));
+    break;
+  }
+
+  // MC Logging
+  //AsmStreamer.reset(createLoggingStreamer(AsmStreamer.take(), errs()));
+
+  // Create the AsmPrinter, which takes ownership of AsmStreamer if successful.
+  FunctionPass *Printer = getTarget().createAsmPrinter(*this, *AsmStreamer);
+  if (Printer == 0)
+    return true;
+
+  // If successful, createAsmPrinter took ownership of AsmStreamer.
+  AsmStreamer.take();
+
+  PM.add(Printer);
+
+  PM.add(createGCInfoDeleter());
+  return false;
+}
+
+bool PTXTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM,
+                                              CodeGenOpt::Level OptLevel,
+                                              bool DisableVerify,
+                                              MCContext *&OutContext) {
+  // Add standard LLVM codegen passes.
+  // This is derived from LLVMTargetMachine::addCommonCodeGenPasses, with some
+  // modifications for the PTX target.
+
+  // Standard LLVM-Level Passes.
+
+  // Basic AliasAnalysis support.
+  // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
+  // BasicAliasAnalysis wins if they disagree. This is intended to help
+  // support "obvious" type-punning idioms.
+  PM.add(createTypeBasedAliasAnalysisPass());
+  PM.add(createBasicAliasAnalysisPass());
+
+  // Before running any passes, run the verifier to determine if the input
+  // coming from the front-end and/or optimizer is valid.
+  if (!DisableVerify)
+    PM.add(createVerifierPass());
+
+  // Run loop strength reduction before anything else.
+  if (OptLevel != CodeGenOpt::None) {
+    PM.add(createLoopStrengthReducePass(getTargetLowering()));
+    //PM.add(createPrintFunctionPass("\n\n*** Code after LSR ***\n", &dbgs()));
+  }
+
+  PM.add(createGCLoweringPass());
+
+  // Make sure that no unreachable blocks are instruction selected.
+  PM.add(createUnreachableBlockEliminationPass());
+
+  PM.add(createLowerInvokePass(getTargetLowering()));
+  // The lower invoke pass may create unreachable code. Remove it.
+  PM.add(createUnreachableBlockEliminationPass());
+
+  if (OptLevel != CodeGenOpt::None)
+    PM.add(createCodeGenPreparePass(getTargetLowering()));
+
+  PM.add(createStackProtectorPass(getTargetLowering()));
+
+  addPreISel(PM, OptLevel);
+
+  //PM.add(createPrintFunctionPass("\n\n"
+  //                               "*** Final LLVM Code input to ISel ***\n",
+  //                               &dbgs()));
+
+  // All passes which modify the LLVM IR are now complete; run the verifier
+  // to ensure that the IR is valid.
+  if (!DisableVerify)
+    PM.add(createVerifierPass());
+
+  // Standard Lower-Level Passes.
+
+  // Install a MachineModuleInfo class, which is an immutable pass that holds
+  // all the per-module stuff we're generating, including MCContext.
+  MachineModuleInfo *MMI = new MachineModuleInfo(*getMCAsmInfo(),
+                                                 *getRegisterInfo(),
+                                    &getTargetLowering()->getObjFileLowering());
+  PM.add(MMI);
+  OutContext = &MMI->getContext(); // Return the MCContext specifically by-ref.
+
+  // Set up a MachineFunction for the rest of CodeGen to work on.
+  PM.add(new MachineFunctionAnalysis(*this, OptLevel));
+
+  // Ask the target for an isel.
+  if (addInstSelector(PM, OptLevel))
+    return true;
+
+  // Print the instruction selected machine code...
+  printAndVerify(PM, "After Instruction Selection");
+
+  // Expand pseudo-instructions emitted by ISel.
+  PM.add(createExpandISelPseudosPass());
+
+  // Pre-ra tail duplication.
+  if (OptLevel != CodeGenOpt::None) {
+    PM.add(createTailDuplicatePass(true));
+    printAndVerify(PM, "After Pre-RegAlloc TailDuplicate");
+  }
+
+  // Optimize PHIs before DCE: removing dead PHI cycles may make more
+  // instructions dead.
+  if (OptLevel != CodeGenOpt::None)
+    PM.add(createOptimizePHIsPass());
+
+  // If the target requests it, assign local variables to stack slots relative
+  // to one another and simplify frame index references where possible.
+  PM.add(createLocalStackSlotAllocationPass());
+
+  if (OptLevel != CodeGenOpt::None) {
+    // With optimization, dead code should already be eliminated. However
+    // there is one known exception: lowered code for arguments that are only
+    // used by tail calls, where the tail calls reuse the incoming stack
+    // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
+    PM.add(createDeadMachineInstructionElimPass());
+    printAndVerify(PM, "After codegen DCE pass");
+
+    PM.add(createMachineLICMPass());
+    PM.add(createMachineCSEPass());
+    PM.add(createMachineSinkingPass());
+    printAndVerify(PM, "After Machine LICM, CSE and Sinking passes");
+
+    PM.add(createPeepholeOptimizerPass());
+    printAndVerify(PM, "After codegen peephole optimization pass");
+  }
+
+  // Run pre-ra passes.
+  if (addPreRegAlloc(PM, OptLevel))
+    printAndVerify(PM, "After PreRegAlloc passes");
+
+  // Perform register allocation.
+  PM.add(createPTXRegisterAllocator());
+  printAndVerify(PM, "After Register Allocation");
+
+  // Perform stack slot coloring and post-ra machine LICM.
+  if (OptLevel != CodeGenOpt::None) {
+    // FIXME: Re-enable coloring with register when it's capable of adding
+    // kill markers.
+    PM.add(createStackSlotColoringPass(false));
+
+    // FIXME: Post-RA LICM has asserts that fire on virtual registers.
+    // Run post-ra machine LICM to hoist reloads / remats.
+    //if (!DisablePostRAMachineLICM)
+    //  PM.add(createMachineLICMPass(false));
+
+    printAndVerify(PM, "After StackSlotColoring and postra Machine LICM");
+  }
+
+  // Run post-ra passes.
+  if (addPostRegAlloc(PM, OptLevel))
+    printAndVerify(PM, "After PostRegAlloc passes");
+
+  PM.add(createExpandPostRAPseudosPass());
+  printAndVerify(PM, "After ExpandPostRAPseudos");
+
+  // Insert prolog/epilog code.  Eliminate abstract frame index references...
+  PM.add(createPrologEpilogCodeInserter());
+  printAndVerify(PM, "After PrologEpilogCodeInserter");
+
+  // Run pre-sched2 passes.
+  if (addPreSched2(PM, OptLevel))
+    printAndVerify(PM, "After PreSched2 passes");
+
+  // Second pass scheduler.
+  if (OptLevel != CodeGenOpt::None) {
+    PM.add(createPostRAScheduler(OptLevel));
+    printAndVerify(PM, "After PostRAScheduler");
+  }
+
+  // Branch folding must be run after regalloc and prolog/epilog insertion.
+  if (OptLevel != CodeGenOpt::None) {
+    PM.add(createBranchFoldingPass(getEnableTailMergeDefault()));
+    printNoVerify(PM, "After BranchFolding");
+  }
+
+  // Tail duplication.
+  if (OptLevel != CodeGenOpt::None) {
+    PM.add(createTailDuplicatePass(false));
+    printNoVerify(PM, "After TailDuplicate");
+  }
+
+  PM.add(createGCMachineCodeAnalysisPass());
+
+  //if (PrintGCInfo)
+  //  PM.add(createGCInfoPrinter(dbgs()));
+
+  if (OptLevel != CodeGenOpt::None) {
+    PM.add(createCodePlacementOptPass());
+    printNoVerify(PM, "After CodePlacementOpt");
+  }
+
+  if (addPreEmitPass(PM, OptLevel))
+    printNoVerify(PM, "After PreEmit passes");
+
+  PM.add(createPTXMFInfoExtract(*this, OptLevel));
+  PM.add(createPTXFPRoundingModePass(*this, OptLevel));
+
   return false;
 }
diff --git a/contrib/llvm/lib/Target/PTX/PTXTargetMachine.h b/contrib/llvm/lib/Target/PTX/PTXTargetMachine.h
index ae4215325211..5b7c82b1f4f4 100644
--- a/contrib/llvm/lib/Target/PTX/PTXTargetMachine.h
+++ b/contrib/llvm/lib/Target/PTX/PTXTargetMachine.h
@@ -17,6 +17,7 @@
 #include "PTXISelLowering.h"
 #include "PTXInstrInfo.h"
 #include "PTXFrameLowering.h"
+#include "PTXSelectionDAGInfo.h"
 #include "PTXSubtarget.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetFrameLowering.h"
@@ -25,15 +26,17 @@
 namespace llvm {
 class PTXTargetMachine : public LLVMTargetMachine {
   private:
-    const TargetData  DataLayout;
-    PTXSubtarget      Subtarget; // has to be initialized before FrameLowering
-    PTXFrameLowering  FrameLowering;
-    PTXInstrInfo      InstrInfo;
-    PTXTargetLowering TLInfo;
+    const TargetData    DataLayout;
+    PTXSubtarget        Subtarget; // has to be initialized before FrameLowering
+    PTXFrameLowering    FrameLowering;
+    PTXInstrInfo        InstrInfo;
+    PTXSelectionDAGInfo TSInfo;
+    PTXTargetLowering   TLInfo;
 
   public:
-    PTXTargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS,
+    PTXTargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM,
                      bool is64Bit);
 
     virtual const TargetData *getTargetData() const { return &DataLayout; }
@@ -49,27 +52,62 @@ class PTXTargetMachine : public LLVMTargetMachine {
     virtual const PTXTargetLowering *getTargetLowering() const {
       return &TLInfo; }
 
+    virtual const PTXSelectionDAGInfo* getSelectionDAGInfo() const {
+      return &TSInfo;
+    }
+
     virtual const PTXSubtarget *getSubtargetImpl() const { return &Subtarget; }
 
     virtual bool addInstSelector(PassManagerBase &PM,
                                  CodeGenOpt::Level OptLevel);
     virtual bool addPostRegAlloc(PassManagerBase &PM,
                                  CodeGenOpt::Level OptLevel);
+
+    // We override this method to supply our own set of codegen passes.
+    virtual bool addPassesToEmitFile(PassManagerBase &,
+                                     formatted_raw_ostream &,
+                                     CodeGenFileType,
+                                     CodeGenOpt::Level,
+                                     bool = true);
+
+    // Emission of machine code through JITCodeEmitter is not supported.
+    virtual bool addPassesToEmitMachineCode(PassManagerBase &,
+                                            JITCodeEmitter &,
+                                            CodeGenOpt::Level,
+                                            bool = true) {
+      return true;
+    }
+
+    // Emission of machine code through MCJIT is not supported.
+    virtual bool addPassesToEmitMC(PassManagerBase &,
+                                   MCContext *&,
+                                   raw_ostream &,
+                                   CodeGenOpt::Level,
+                                   bool = true) {
+      return true;
+    }
+
+  private:
+
+    bool addCommonCodeGenPasses(PassManagerBase &, CodeGenOpt::Level,
+                                bool DisableVerify, MCContext *&OutCtx);
 }; // class PTXTargetMachine
 
 
 class PTX32TargetMachine : public PTXTargetMachine {
 public:
 
-  PTX32TargetMachine(const Target &T, const std::string &TT,
-                     const std::string& CPU, const std::string& FS);
+  PTX32TargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 }; // class PTX32TargetMachine
 
 class PTX64TargetMachine : public PTXTargetMachine {
 public:
 
-  PTX64TargetMachine(const Target &T, const std::string &TT,
-                     const std::string& CPU, const std::string& FS);
+  PTX64TargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 }; // class PTX32TargetMachine
 
 } // namespace llvm
diff --git a/contrib/llvm/lib/Target/PTX/TargetInfo/CMakeLists.txt b/contrib/llvm/lib/Target/PTX/TargetInfo/CMakeLists.txt
deleted file mode 100644
index 4b09cf5ce099..000000000000
--- a/contrib/llvm/lib/Target/PTX/TargetInfo/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMPTXInfo
-  PTXTargetInfo.cpp
-  )
-
-add_dependencies(LLVMPTXInfo PTXCodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/PTX/TargetInfo/Makefile b/contrib/llvm/lib/Target/PTX/TargetInfo/Makefile
deleted file mode 100644
index 8619785889aa..000000000000
--- a/contrib/llvm/lib/Target/PTX/TargetInfo/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/PTX/TargetInfo/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMPTXInfo
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/PTX/TargetInfo/PTXTargetInfo.cpp b/contrib/llvm/lib/Target/PTX/TargetInfo/PTXTargetInfo.cpp
index 9df6c7567bd1..09a27358da9f 100644
--- a/contrib/llvm/lib/Target/PTX/TargetInfo/PTXTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/PTX/TargetInfo/PTXTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "PTX.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/PTX/generate-register-td.py b/contrib/llvm/lib/Target/PTX/generate-register-td.py
deleted file mode 100755
index 15286908961d..000000000000
--- a/contrib/llvm/lib/Target/PTX/generate-register-td.py
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env python
-##===- generate-register-td.py --------------------------------*-python-*--===##
-##
-##                     The LLVM Compiler Infrastructure
-##
-## This file is distributed under the University of Illinois Open Source
-## License. See LICENSE.TXT for details.
-##
-##===----------------------------------------------------------------------===##
-##
-## This file describes the PTX register file generator.
-##
-##===----------------------------------------------------------------------===##
-
-from sys import argv, exit, stdout
-
-
-if len(argv) != 5:
-    print('Usage: generate-register-td.py <num_preds> <num_16> <num_32> <num_64>')
-    exit(1)
-
-try:
-    num_pred  = int(argv[1])
-    num_16bit = int(argv[2])
-    num_32bit = int(argv[3])
-    num_64bit = int(argv[4])
-except:
-    print('ERROR: Invalid integer parameter')
-    exit(1)
-
-## Print the register definition file
-td_file = open('PTXRegisterInfo.td', 'w')
-
-td_file.write('''
-//===- PTXRegisterInfo.td - PTX Register defs ----------------*- tblgen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-//  Declarations that describe the PTX register file
-//===----------------------------------------------------------------------===//
-
-class PTXReg<string n> : Register<n> {
-  let Namespace = "PTX";
-}
-
-//===----------------------------------------------------------------------===//
-//  Registers
-//===----------------------------------------------------------------------===//
-''')
-
-
-# Print predicate registers
-td_file.write('\n///===- Predicate Registers -----------------------------------------------===//\n\n')
-for r in range(0, num_pred):
-    td_file.write('def P%d : PTXReg<"p%d">;\n' % (r, r))
-
-# Print 16-bit registers
-td_file.write('\n///===- 16-Bit Registers --------------------------------------------------===//\n\n')
-for r in range(0, num_16bit):
-    td_file.write('def RH%d : PTXReg<"rh%d">;\n' % (r, r))
-
-# Print 32-bit registers
-td_file.write('\n///===- 32-Bit Registers --------------------------------------------------===//\n\n')
-for r in range(0, num_32bit):
-    td_file.write('def R%d : PTXReg<"r%d">;\n' % (r, r))
-
-# Print 64-bit registers
-td_file.write('\n///===- 64-Bit Registers --------------------------------------------------===//\n\n')
-for r in range(0, num_64bit):
-    td_file.write('def RD%d : PTXReg<"rd%d">;\n' % (r, r))
-
-
-td_file.write('''
-//===----------------------------------------------------------------------===//
-//  Register classes
-//===----------------------------------------------------------------------===//
-''')
-
-
-# Print register classes
-
-td_file.write('def RegPred : RegisterClass<"PTX", [i1], 8, (sequence "P%%u", 0, %d)>;\n' % (num_pred-1))
-td_file.write('def RegI16 : RegisterClass<"PTX", [i16], 16, (sequence "RH%%u", 0, %d)>;\n' % (num_16bit-1))
-td_file.write('def RegI32 : RegisterClass<"PTX", [i32], 32, (sequence "R%%u", 0, %d)>;\n' % (num_32bit-1))
-td_file.write('def RegI64 : RegisterClass<"PTX", [i64], 64, (sequence "RD%%u", 0, %d)>;\n' % (num_64bit-1))
-td_file.write('def RegF32 : RegisterClass<"PTX", [f32], 32, (sequence "R%%u", 0, %d)>;\n' % (num_32bit-1))
-td_file.write('def RegF64 : RegisterClass<"PTX", [f64], 64, (sequence "RD%%u", 0, %d)>;\n' % (num_64bit-1))
-
-
-td_file.close()
-
-## Now write the PTXCallingConv.td file
-td_file = open('PTXCallingConv.td', 'w')
-
-# Reserve 10% of the available registers for return values, and the other 90%
-# for parameters
-num_ret_pred    = int(0.1 * num_pred)
-num_ret_16bit   = int(0.1 * num_16bit)
-num_ret_32bit   = int(0.1 * num_32bit)
-num_ret_64bit   = int(0.1 * num_64bit)
-num_param_pred  = num_pred - num_ret_pred
-num_param_16bit = num_16bit - num_ret_16bit
-num_param_32bit = num_32bit - num_ret_32bit
-num_param_64bit = num_64bit - num_ret_64bit
-
-param_regs_pred  = [('P%d' % (i+num_ret_pred)) for i in range(0, num_param_pred)]
-ret_regs_pred    = ['P%d' % i for i in range(0, num_ret_pred)]
-param_regs_16bit = [('RH%d' % (i+num_ret_16bit)) for i in range(0, num_param_16bit)]
-ret_regs_16bit   = ['RH%d' % i for i in range(0, num_ret_16bit)]
-param_regs_32bit = [('R%d' % (i+num_ret_32bit)) for i in range(0, num_param_32bit)]
-ret_regs_32bit   = ['R%d' % i for i in range(0, num_ret_32bit)]
-param_regs_64bit = [('RD%d' % (i+num_ret_64bit)) for i in range(0, num_param_64bit)]
-ret_regs_64bit   = ['RD%d' % i for i in range(0, num_ret_64bit)]
-
-param_list_pred  = reduce(lambda x, y: '%s, %s' % (x, y), param_regs_pred)
-ret_list_pred    = reduce(lambda x, y: '%s, %s' % (x, y), ret_regs_pred)
-param_list_16bit = reduce(lambda x, y: '%s, %s' % (x, y), param_regs_16bit)
-ret_list_16bit   = reduce(lambda x, y: '%s, %s' % (x, y), ret_regs_16bit)
-param_list_32bit = reduce(lambda x, y: '%s, %s' % (x, y), param_regs_32bit)
-ret_list_32bit   = reduce(lambda x, y: '%s, %s' % (x, y), ret_regs_32bit)
-param_list_64bit = reduce(lambda x, y: '%s, %s' % (x, y), param_regs_64bit)
-ret_list_64bit   = reduce(lambda x, y: '%s, %s' % (x, y), ret_regs_64bit)
-
-td_file.write('''
-//===--- PTXCallingConv.td - Calling Conventions -----------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This describes the calling conventions for the PTX architecture.
-//
-//===----------------------------------------------------------------------===//
-
-// PTX Formal Parameter Calling Convention
-def CC_PTX : CallingConv<[
-  CCIfType<[i1],      CCAssignToReg<[%s]>>,
-  CCIfType<[i16],     CCAssignToReg<[%s]>>,
-  CCIfType<[i32,f32], CCAssignToReg<[%s]>>,
-  CCIfType<[i64,f64], CCAssignToReg<[%s]>>
-]>;
-
-// PTX Return Value Calling Convention
-def RetCC_PTX : CallingConv<[
-  CCIfType<[i1],      CCAssignToReg<[%s]>>,
-  CCIfType<[i16],     CCAssignToReg<[%s]>>,
-  CCIfType<[i32,f32], CCAssignToReg<[%s]>>,
-  CCIfType<[i64,f64], CCAssignToReg<[%s]>>
-]>;
-''' % (param_list_pred, param_list_16bit, param_list_32bit, param_list_64bit,
-       ret_list_pred, ret_list_16bit, ret_list_32bit, ret_list_64bit))
-
-
-td_file.close()
diff --git a/contrib/llvm/lib/Target/PowerPC/InstPrinter/CMakeLists.txt b/contrib/llvm/lib/Target/PowerPC/InstPrinter/CMakeLists.txt
deleted file mode 100644
index 389ea7742b06..000000000000
--- a/contrib/llvm/lib/Target/PowerPC/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMPowerPCAsmPrinter
-  PPCInstPrinter.cpp
-  )
-add_dependencies(LLVMPowerPCAsmPrinter PowerPCCodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/PowerPC/InstPrinter/Makefile b/contrib/llvm/lib/Target/PowerPC/InstPrinter/Makefile
deleted file mode 100644
index f097e84248ff..000000000000
--- a/contrib/llvm/lib/Target/PowerPC/InstPrinter/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/PowerPC/AsmPrinter/Makefile --------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMPowerPCAsmPrinter
-
-# Hack: we need to include 'main' powerpc target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp b/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
index 1a9bd761359c..b6a08354a210 100644
--- a/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
@@ -13,7 +13,8 @@
 
 #define DEBUG_TYPE "asm-printer"
 #include "PPCInstPrinter.h"
-#include "PPCPredicates.h"
+#include "MCTargetDesc/PPCBaseInfo.h"
+#include "MCTargetDesc/PPCPredicates.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/raw_ostream.h"
@@ -30,7 +31,8 @@ void PPCInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   OS << getRegisterName(RegNo);
 }
 
-void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
+void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
+                               StringRef Annot) {
   // Check for slwi/srwi mnemonics.
   if (MI->getOpcode() == PPC::RLWINM) {
     unsigned char SH = MI->getOperand(2).getImm();
@@ -49,6 +51,8 @@ void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
       O << ", ";
       printOperand(MI, 1, O);
       O << ", " << (unsigned int)SH;
+
+      printAnnotation(O, Annot);
       return;
     }
   }
@@ -59,6 +63,7 @@ void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
     printOperand(MI, 0, O);
     O << ", ";
     printOperand(MI, 1, O);
+    printAnnotation(O, Annot);
     return;
   }
   
@@ -72,11 +77,13 @@ void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O) {
       O << ", ";
       printOperand(MI, 1, O);
       O << ", " << (unsigned int)SH;
+      printAnnotation(O, Annot);
       return;
     }
   }
   
   printInstruction(MI, O);
+  printAnnotation(O, Annot);
 }
 
 
diff --git a/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h b/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
index d022a4496e84..4ed4b765c1c7 100644
--- a/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
+++ b/contrib/llvm/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
@@ -32,7 +32,7 @@ class PPCInstPrinter : public MCInstPrinter {
   }
   
   virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
-  virtual void printInst(const MCInst *MI, raw_ostream &O);
+  virtual void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot);
   virtual StringRef getOpcodeName(unsigned Opcode) const;
   
   static const char *getInstructionName(unsigned Opcode);
diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index a1b81662115a..000000000000
--- a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMPowerPCDesc
-  PPCMCTargetDesc.cpp
-  PPCMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/Makefile
deleted file mode 100644
index 9db66622cced..000000000000
--- a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/PowerPC/TargetDesc/Makefile --------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMPowerPCDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCAsmBackend.cpp b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
similarity index 56%
rename from contrib/llvm/lib/Target/PowerPC/PPCAsmBackend.cpp
rename to contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 4b8cbb711833..9f2fd6d01b8e 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCAsmBackend.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -7,17 +7,43 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmBackend.h"
-#include "PPC.h"
-#include "PPCFixupKinds.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "MCTargetDesc/PPCMCTargetDesc.h"
+#include "MCTargetDesc/PPCFixupKinds.h"
+#include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Object/MachOFormat.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/ELF.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
+static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
+  switch (Kind) {
+  default:
+    llvm_unreachable("Unknown fixup kind!");
+  case FK_Data_1:
+  case FK_Data_2:
+  case FK_Data_4:
+    return Value;
+  case PPC::fixup_ppc_brcond14:
+    return Value & 0x3ffc;
+  case PPC::fixup_ppc_br24:
+    return Value & 0x3fffffc;
+#if 0
+  case PPC::fixup_ppc_hi16:
+    return (Value >> 16) & 0xffff;
+#endif
+  case PPC::fixup_ppc_ha16:
+    return ((Value >> 16) + ((Value & 0x8000) ? 1 : 0)) & 0xffff;
+  case PPC::fixup_ppc_lo16:
+    return Value & 0xffff;
+  }
+}
+
 namespace {
 class PPCMachObjectWriter : public MCMachObjectTargetWriter {
 public:
@@ -31,10 +57,17 @@ class PPCMachObjectWriter : public MCMachObjectTargetWriter {
                         MCValue Target, uint64_t &FixedValue) {}
 };
 
-class PPCAsmBackend : public TargetAsmBackend {
+class PPCELFObjectWriter : public MCELFObjectTargetWriter {
+public:
+  PPCELFObjectWriter(bool Is64Bit, Triple::OSType OSType, uint16_t EMachine,
+                     bool HasRelocationAddend, bool isLittleEndian)
+    : MCELFObjectTargetWriter(Is64Bit, OSType, EMachine, HasRelocationAddend) {}
+};
+
+class PPCAsmBackend : public MCAsmBackend {
 const Target &TheTarget;
 public:
-  PPCAsmBackend(const Target &T) : TargetAsmBackend(), TheTarget(T) {}
+  PPCAsmBackend(const Target &T) : MCAsmBackend(), TheTarget(T) {}
 
   unsigned getNumFixupKinds() const { return PPC::NumTargetFixupKinds; }
 
@@ -49,7 +82,7 @@ const Target &TheTarget;
     };
   
     if (Kind < FirstTargetFixupKind)
-      return TargetAsmBackend::getFixupKindInfo(Kind);
+      return MCAsmBackend::getFixupKindInfo(Kind);
   
     assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
            "Invalid kind!");
@@ -109,15 +142,50 @@ namespace {
       return false;
     }
   };
+
+  class ELFPPCAsmBackend : public PPCAsmBackend {
+    Triple::OSType OSType;
+  public:
+    ELFPPCAsmBackend(const Target &T, Triple::OSType OSType) :
+      PPCAsmBackend(T), OSType(OSType) { }
+    
+    void ApplyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
+                    uint64_t Value) const {
+      Value = adjustFixupValue(Fixup.getKind(), Value);
+      if (!Value) return;           // Doesn't change encoding.
+
+      unsigned Offset = Fixup.getOffset();
+
+      // For each byte of the fragment that the fixup touches, mask in the bits from
+      // the fixup value. The Value has been "split up" into the appropriate
+      // bitfields above.
+      for (unsigned i = 0; i != 4; ++i)
+        Data[Offset + i] |= uint8_t((Value >> ((4 - i - 1)*8)) & 0xff);
+    }
+    
+    MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
+      bool is64 = getPointerSize() == 8;
+      return createELFObjectWriter(new PPCELFObjectWriter(
+                                      /*Is64Bit=*/is64,
+                                      OSType,
+                                      is64 ? ELF::EM_PPC64 : ELF::EM_PPC,                                      
+                                      /*addend*/ true, /*isLittleEndian*/ false),
+                                   OS, /*IsLittleEndian=*/false);
+    }
+    
+    virtual bool doesSectionRequireSymbols(const MCSection &Section) const {
+      return false;
+    }
+  };
+
 } // end anonymous namespace
 
 
 
 
-TargetAsmBackend *llvm::createPPCAsmBackend(const Target &T,
-                                            const std::string &TT) {
+MCAsmBackend *llvm::createPPCAsmBackend(const Target &T, StringRef TT) {
   if (Triple(TT).isOSDarwin())
     return new DarwinPPCAsmBackend(T);
 
-  return 0;
+  return new ELFPPCAsmBackend(T, Triple(TT).getOS());
 }
diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCBaseInfo.h b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCBaseInfo.h
new file mode 100644
index 000000000000..369bbdce11f5
--- /dev/null
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCBaseInfo.h
@@ -0,0 +1,70 @@
+//===-- PPCBaseInfo.h - Top level definitions for PPC -------- --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains small standalone helper functions and enum definitions for
+// the PPC target useful for the compiler back-end and the MC libraries.
+// As such, it deliberately does not include references to LLVM core
+// code gen types, passes, etc..
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PPCBASEINFO_H
+#define PPCBASEINFO_H
+
+#include "PPCMCTargetDesc.h"
+#include "llvm/Support/ErrorHandling.h"
+
+namespace llvm {
+
+/// getPPCRegisterNumbering - Given the enum value for some register, e.g.
+/// PPC::F14, return the number that it corresponds to (e.g. 14).
+inline static unsigned getPPCRegisterNumbering(unsigned RegEnum) {
+  using namespace PPC;
+  switch (RegEnum) {
+  case 0: return 0;
+  case R0 :  case X0 :  case F0 :  case V0 : case CR0:  case CR0LT: return  0;
+  case R1 :  case X1 :  case F1 :  case V1 : case CR1:  case CR0GT: return  1;
+  case R2 :  case X2 :  case F2 :  case V2 : case CR2:  case CR0EQ: return  2;
+  case R3 :  case X3 :  case F3 :  case V3 : case CR3:  case CR0UN: return  3;
+  case R4 :  case X4 :  case F4 :  case V4 : case CR4:  case CR1LT: return  4;
+  case R5 :  case X5 :  case F5 :  case V5 : case CR5:  case CR1GT: return  5;
+  case R6 :  case X6 :  case F6 :  case V6 : case CR6:  case CR1EQ: return  6;
+  case R7 :  case X7 :  case F7 :  case V7 : case CR7:  case CR1UN: return  7;
+  case R8 :  case X8 :  case F8 :  case V8 : case CR2LT: return  8;
+  case R9 :  case X9 :  case F9 :  case V9 : case CR2GT: return  9;
+  case R10:  case X10:  case F10:  case V10: case CR2EQ: return 10;
+  case R11:  case X11:  case F11:  case V11: case CR2UN: return 11;
+  case R12:  case X12:  case F12:  case V12: case CR3LT: return 12;
+  case R13:  case X13:  case F13:  case V13: case CR3GT: return 13;
+  case R14:  case X14:  case F14:  case V14: case CR3EQ: return 14;
+  case R15:  case X15:  case F15:  case V15: case CR3UN: return 15;
+  case R16:  case X16:  case F16:  case V16: case CR4LT: return 16;
+  case R17:  case X17:  case F17:  case V17: case CR4GT: return 17;
+  case R18:  case X18:  case F18:  case V18: case CR4EQ: return 18;
+  case R19:  case X19:  case F19:  case V19: case CR4UN: return 19;
+  case R20:  case X20:  case F20:  case V20: case CR5LT: return 20;
+  case R21:  case X21:  case F21:  case V21: case CR5GT: return 21;
+  case R22:  case X22:  case F22:  case V22: case CR5EQ: return 22;
+  case R23:  case X23:  case F23:  case V23: case CR5UN: return 23;
+  case R24:  case X24:  case F24:  case V24: case CR6LT: return 24;
+  case R25:  case X25:  case F25:  case V25: case CR6GT: return 25;
+  case R26:  case X26:  case F26:  case V26: case CR6EQ: return 26;
+  case R27:  case X27:  case F27:  case V27: case CR6UN: return 27;
+  case R28:  case X28:  case F28:  case V28: case CR7LT: return 28;
+  case R29:  case X29:  case F29:  case V29: case CR7GT: return 29;
+  case R30:  case X30:  case F30:  case V30: case CR7EQ: return 30;
+  case R31:  case X31:  case F31:  case V31: case CR7UN: return 31;
+  default:
+    llvm_unreachable("Unhandled reg in PPCRegisterInfo::getRegisterNumbering!");
+  }
+}
+
+} // end namespace llvm;
+
+#endif
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCFixupKinds.h b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
similarity index 100%
rename from contrib/llvm/lib/Target/PowerPC/PPCFixupKinds.h
rename to contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
index b6dca835b18d..e9424d8415f6 100644
--- a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
@@ -31,6 +31,10 @@ PPCMCAsmInfoDarwin::PPCMCAsmInfoDarwin(bool is64Bit) {
 }
 
 PPCLinuxMCAsmInfo::PPCLinuxMCAsmInfo(bool is64Bit) {
+  if (is64Bit)
+    PointerSize = 8;
+  IsLittleEndian = false;
+
   // ".comm align is in bytes but .align is pow-2."
   AlignmentIsInBytes = false;
 
@@ -56,7 +60,7 @@ PPCLinuxMCAsmInfo::PPCLinuxMCAsmInfo(bool is64Bit) {
     
   ZeroDirective = "\t.space\t";
   Data64bitsDirective = is64Bit ? "\t.quad\t" : 0;
-  HasLCOMMDirective = true;
+  LCOMMDirectiveType = LCOMM::NoAlignment;
   AssemblerDialect = 0;           // Old-Style mnemonics.
 }
 
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCMCCodeEmitter.cpp b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
similarity index 97%
rename from contrib/llvm/lib/Target/PowerPC/PPCMCCodeEmitter.cpp
rename to contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index cf73d861fa4d..262f97c36a93 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCMCCodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -12,9 +12,8 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "mccodeemitter"
-#include "PPC.h"
-#include "PPCRegisterInfo.h"
-#include "PPCFixupKinds.h"
+#include "MCTargetDesc/PPCBaseInfo.h"
+#include "MCTargetDesc/PPCFixupKinds.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/ADT/Statistic.h"
@@ -170,7 +169,7 @@ get_crbitm_encoding(const MCInst &MI, unsigned OpNo,
   const MCOperand &MO = MI.getOperand(OpNo);
   assert((MI.getOpcode() == PPC::MTCRF || MI.getOpcode() == PPC::MFOCRF) &&
          (MO.getReg() >= PPC::CR0 && MO.getReg() <= PPC::CR7));
-  return 0x80 >> PPCRegisterInfo::getRegisterNumbering(MO.getReg());
+  return 0x80 >> getPPCRegisterNumbering(MO.getReg());
 }
 
 
@@ -182,7 +181,7 @@ getMachineOpValue(const MCInst &MI, const MCOperand &MO,
     // The GPR operand should come through here though.
     assert((MI.getOpcode() != PPC::MTCRF && MI.getOpcode() != PPC::MFOCRF) ||
            MO.getReg() < PPC::CR0 || MO.getReg() > PPC::CR7);
-    return PPCRegisterInfo::getRegisterNumbering(MO.getReg());
+    return getPPCRegisterNumbering(MO.getReg());
   }
   
   assert(MO.isImm() &&
diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 02b887f4d5dc..d5c8a9e72c67 100644
--- a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -13,10 +13,14 @@
 
 #include "PPCMCTargetDesc.h"
 #include "PPCMCAsmInfo.h"
+#include "InstPrinter/PPCInstPrinter.h"
+#include "llvm/MC/MachineLocation.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "PPCGenInstrInfo.inc"
@@ -35,11 +39,16 @@ static MCInstrInfo *createPPCMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializePowerPCMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(ThePPC32Target, createPPCMCInstrInfo);
-  TargetRegistry::RegisterMCInstrInfo(ThePPC64Target, createPPCMCInstrInfo);
-}
+static MCRegisterInfo *createPPCMCRegisterInfo(StringRef TT) {
+  Triple TheTriple(TT);
+  bool isPPC64 = (TheTriple.getArch() == Triple::ppc64);
+  unsigned Flavour = isPPC64 ? 0 : 1;
+  unsigned RA = isPPC64 ? PPC::LR8 : PPC::LR;
 
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitPPCMCRegisterInfo(X, RA, Flavour, Flavour);
+  return X;
+}
 
 static MCSubtargetInfo *createPPCMCSubtargetInfo(StringRef TT, StringRef CPU,
                                                  StringRef FS) {
@@ -48,23 +57,95 @@ static MCSubtargetInfo *createPPCMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializePowerPCMCSubtargetInfo() {
+static MCAsmInfo *createPPCMCAsmInfo(const Target &T, StringRef TT) {
+  Triple TheTriple(TT);
+  bool isPPC64 = TheTriple.getArch() == Triple::ppc64;
+
+  MCAsmInfo *MAI;
+  if (TheTriple.isOSDarwin())
+    MAI = new PPCMCAsmInfoDarwin(isPPC64);
+  else
+    MAI = new PPCLinuxMCAsmInfo(isPPC64);
+
+  // Initial state of the frame pointer is R1.
+  MachineLocation Dst(MachineLocation::VirtualFP);
+  MachineLocation Src(PPC::R1, 0);
+  MAI->addInitialFrameState(0, Dst, Src);
+
+  return MAI;
+}
+
+static MCCodeGenInfo *createPPCMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                             CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+
+  if (RM == Reloc::Default) {
+    Triple T(TT);
+    if (T.isOSDarwin())
+      RM = Reloc::DynamicNoPIC;
+    else
+      RM = Reloc::Static;
+  }
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+// This is duplicated code. Refactor this.
+static MCStreamer *createMCStreamer(const Target &T, StringRef TT,
+                                    MCContext &Ctx, MCAsmBackend &MAB,
+                                    raw_ostream &OS,
+                                    MCCodeEmitter *Emitter,
+                                    bool RelaxAll,
+                                    bool NoExecStack) {
+  if (Triple(TT).isOSDarwin())
+    return createMachOStreamer(Ctx, MAB, OS, Emitter, RelaxAll);
+
+  return createELFStreamer(Ctx, MAB, OS, Emitter, RelaxAll, NoExecStack);
+}
+
+static MCInstPrinter *createPPCMCInstPrinter(const Target &T,
+                                             unsigned SyntaxVariant,
+                                             const MCAsmInfo &MAI,
+                                             const MCSubtargetInfo &STI) {
+  return new PPCInstPrinter(MAI, SyntaxVariant);
+}
+
+extern "C" void LLVMInitializePowerPCTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfoFn C(ThePPC32Target, createPPCMCAsmInfo);
+  RegisterMCAsmInfoFn D(ThePPC64Target, createPPCMCAsmInfo);  
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(ThePPC32Target, createPPCMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(ThePPC64Target, createPPCMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(ThePPC32Target, createPPCMCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(ThePPC64Target, createPPCMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(ThePPC32Target, createPPCMCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(ThePPC64Target, createPPCMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(ThePPC32Target,
                                           createPPCMCSubtargetInfo);
   TargetRegistry::RegisterMCSubtargetInfo(ThePPC64Target,
                                           createPPCMCSubtargetInfo);
-}
 
-static MCAsmInfo *createMCAsmInfo(const Target &T, StringRef TT) {
-  Triple TheTriple(TT);
-  bool isPPC64 = TheTriple.getArch() == Triple::ppc64;
-  if (TheTriple.isOSDarwin())
-    return new PPCMCAsmInfoDarwin(isPPC64);
-  return new PPCLinuxMCAsmInfo(isPPC64);
+  // Register the MC Code Emitter
+  TargetRegistry::RegisterMCCodeEmitter(ThePPC32Target, createPPCMCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(ThePPC64Target, createPPCMCCodeEmitter);
   
-}
+    // Register the asm backend.
+  TargetRegistry::RegisterMCAsmBackend(ThePPC32Target, createPPCAsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(ThePPC64Target, createPPCAsmBackend);
+  
+  // Register the object streamer.
+  TargetRegistry::RegisterMCObjectStreamer(ThePPC32Target, createMCStreamer);
+  TargetRegistry::RegisterMCObjectStreamer(ThePPC64Target, createMCStreamer);
 
-extern "C" void LLVMInitializePowerPCMCAsmInfo() {
-  RegisterMCAsmInfoFn C(ThePPC32Target, createMCAsmInfo);
-  RegisterMCAsmInfoFn D(ThePPC64Target, createMCAsmInfo);  
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(ThePPC32Target, createPPCMCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(ThePPC64Target, createPPCMCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index cee235097a0a..e5bf2a9dd92f 100644
--- a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -15,6 +15,10 @@
 #define PPCMCTARGETDESC_H
 
 namespace llvm {
+class MCAsmBackend;
+class MCCodeEmitter;
+class MCContext;
+class MCInstrInfo;
 class MCSubtargetInfo;
 class Target;
 class StringRef;
@@ -22,6 +26,12 @@ class StringRef;
 extern Target ThePPC32Target;
 extern Target ThePPC64Target;
   
+MCCodeEmitter *createPPCMCCodeEmitter(const MCInstrInfo &MCII,
+                                      const MCSubtargetInfo &STI,
+                                      MCContext &Ctx);
+
+MCAsmBackend *createPPCAsmBackend(const Target &T, StringRef TT);
+  
 } // End llvm namespace
 
 // Defines symbolic names for PowerPC registers.  This defines a mapping from
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCPredicates.cpp b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp
similarity index 100%
rename from contrib/llvm/lib/Target/PowerPC/PPCPredicates.cpp
rename to contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCPredicates.h b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h
similarity index 98%
rename from contrib/llvm/lib/Target/PowerPC/PPCPredicates.h
rename to contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h
index b2c831579f79..f872e861bfa7 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCPredicates.h
+++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h
@@ -14,8 +14,6 @@
 #ifndef LLVM_TARGET_POWERPC_PPCPREDICATES_H
 #define LLVM_TARGET_POWERPC_PPCPREDICATES_H
 
-#include "PPC.h"
-
 namespace llvm {
 namespace PPC {
   /// Predicate - These are "(BI << 5) | BO"  for various predicates.
diff --git a/contrib/llvm/lib/Target/PowerPC/PPC.h b/contrib/llvm/lib/Target/PowerPC/PPC.h
index 7191dd105f3c..5dc1863a0b2d 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPC.h
+++ b/contrib/llvm/lib/Target/PowerPC/PPC.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_TARGET_POWERPC_H
 #define LLVM_TARGET_POWERPC_H
 
+#include "MCTargetDesc/PPCBaseInfo.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include <string>
 
@@ -30,22 +31,12 @@ namespace llvm {
   class MachineInstr;
   class AsmPrinter;
   class MCInst;
-  class MCCodeEmitter;
-  class MCContext;
-  class MCInstrInfo;
-  class MCSubtargetInfo;
   class TargetMachine;
-  class TargetAsmBackend;
   
   FunctionPass *createPPCBranchSelectionPass();
   FunctionPass *createPPCISelDag(PPCTargetMachine &TM);
   FunctionPass *createPPCJITCodeEmitterPass(PPCTargetMachine &TM,
                                             JITCodeEmitter &MCE);
-  MCCodeEmitter *createPPCMCCodeEmitter(const MCInstrInfo &MCII,
-                                        const MCSubtargetInfo &STI,
-                                        MCContext &Ctx);
-  TargetAsmBackend *createPPCAsmBackend(const Target &, const std::string &);
-  
   void LowerPPCMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                     AsmPrinter &AP, bool isDarwin);
   
diff --git a/contrib/llvm/lib/Target/PowerPC/PPC.td b/contrib/llvm/lib/Target/PowerPC/PPC.td
index aabf494012e2..2d5d302728f7 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPC.td
+++ b/contrib/llvm/lib/Target/PowerPC/PPC.td
@@ -43,9 +43,9 @@ def FeatureAltivec   : SubtargetFeature<"altivec","HasAltivec", "true",
 def FeatureGPUL      : SubtargetFeature<"gpul","IsGigaProcessor", "true",
                                         "Enable GPUL instructions">;
 def FeatureFSqrt     : SubtargetFeature<"fsqrt","HasFSQRT", "true",
-                                        "Enable the fsqrt instruction">; 
+                                        "Enable the fsqrt instruction">;
 def FeatureSTFIWX    : SubtargetFeature<"stfiwx","HasSTFIWX", "true",
-                                        "Enable the stfiwx instruction">; 
+                                        "Enable the stfiwx instruction">;
 
 //===----------------------------------------------------------------------===//
 // Register File Description
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/contrib/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 9de2200296e8..952845943179 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -18,9 +18,9 @@
 
 #define DEBUG_TYPE "asmprinter"
 #include "PPC.h"
-#include "PPCPredicates.h"
 #include "PPCTargetMachine.h"
 #include "PPCSubtarget.h"
+#include "MCTargetDesc/PPCPredicates.h"
 #include "llvm/Analysis/DebugInfo.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
@@ -43,11 +43,11 @@
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
@@ -679,18 +679,8 @@ static AsmPrinter *createPPCAsmPrinterPass(TargetMachine &tm,
   return new PPCLinuxAsmPrinter(tm, Streamer);
 }
 
-static MCInstPrinter *createPPCMCInstPrinter(const Target &T,
-                                             unsigned SyntaxVariant,
-                                             const MCAsmInfo &MAI) {
-  return new PPCInstPrinter(MAI, SyntaxVariant);
-}
-
-
 // Force static initialization.
 extern "C" void LLVMInitializePowerPCAsmPrinter() { 
   TargetRegistry::RegisterAsmPrinter(ThePPC32Target, createPPCAsmPrinterPass);
   TargetRegistry::RegisterAsmPrinter(ThePPC64Target, createPPCAsmPrinterPass);
-  
-  TargetRegistry::RegisterMCInstPrinter(ThePPC32Target, createPPCMCInstPrinter);
-  TargetRegistry::RegisterMCInstPrinter(ThePPC64Target, createPPCMCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp b/contrib/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp
index e161d23600e2..475edf309c0c 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp
@@ -19,7 +19,7 @@
 #include "PPC.h"
 #include "PPCInstrBuilder.h"
 #include "PPCInstrInfo.h"
-#include "PPCPredicates.h"
+#include "MCTargetDesc/PPCPredicates.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/ADT/Statistic.h"
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCCodeEmitter.cpp b/contrib/llvm/lib/Target/PowerPC/PPCCodeEmitter.cpp
index 42232a07535b..4a1f1822afdf 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCCodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCCodeEmitter.cpp
@@ -140,7 +140,7 @@ unsigned PPCCodeEmitter::get_crbitm_encoding(const MachineInstr &MI,
   const MachineOperand &MO = MI.getOperand(OpNo);
   assert((MI.getOpcode() == PPC::MTCRF || MI.getOpcode() == PPC::MFOCRF) &&
          (MO.getReg() >= PPC::CR0 && MO.getReg() <= PPC::CR7));
-  return 0x80 >> PPCRegisterInfo::getRegisterNumbering(MO.getReg());
+  return 0x80 >> getPPCRegisterNumbering(MO.getReg());
 }
 
 MachineRelocation PPCCodeEmitter::GetRelocation(const MachineOperand &MO, 
@@ -250,7 +250,7 @@ unsigned PPCCodeEmitter::getMachineOpValue(const MachineInstr &MI,
     // The GPR operand should come through here though.
     assert((MI.getOpcode() != PPC::MTCRF && MI.getOpcode() != PPC::MFOCRF) ||
            MO.getReg() < PPC::CR0 || MO.getReg() > PPC::CR7);
-    return PPCRegisterInfo::getRegisterNumbering(MO.getReg());
+    return getPPCRegisterNumbering(MO.getReg());
   }
   
   assert(MO.isImm() &&
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 375e000fe401..7dead10c7c2a 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -109,14 +109,14 @@ static void HandleVRSaveUpdate(MachineInstr *MI, const TargetInstrInfo &TII) {
   for (MachineRegisterInfo::livein_iterator
        I = MF->getRegInfo().livein_begin(),
        E = MF->getRegInfo().livein_end(); I != E; ++I) {
-    unsigned RegNo = PPCRegisterInfo::getRegisterNumbering(I->first);
+    unsigned RegNo = getPPCRegisterNumbering(I->first);
     if (VRRegNo[RegNo] == I->first)        // If this really is a vector reg.
       UsedRegMask &= ~(1 << (31-RegNo));   // Doesn't need to be marked.
   }
   for (MachineRegisterInfo::liveout_iterator
        I = MF->getRegInfo().liveout_begin(),
        E = MF->getRegInfo().liveout_end(); I != E; ++I) {
-    unsigned RegNo = PPCRegisterInfo::getRegisterNumbering(*I);
+    unsigned RegNo = getPPCRegisterNumbering(*I);
     if (VRRegNo[RegNo] == *I)              // If this really is a vector reg.
       UsedRegMask &= ~(1 << (31-RegNo));   // Doesn't need to be marked.
   }
@@ -712,13 +712,6 @@ void PPCFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-void PPCFrameLowering::getInitialFrameState(std::vector<MachineMove> &Moves) const {
-  // Initial state of the frame pointer is R1.
-  MachineLocation Dst(MachineLocation::VirtualFP);
-  MachineLocation Src(PPC::R1, 0);
-  Moves.push_back(MachineMove(0, Dst, Src));
-}
-
 static bool spillsCR(const MachineFunction &MF) {
   const PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
   return FuncInfo->isCRSpilled();
@@ -885,7 +878,7 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
       FFI->setObjectOffset(FI, LowerBound + FFI->getObjectOffset(FI));
     }
 
-    LowerBound -= (31 - PPCRegisterInfo::getRegisterNumbering(MinFPR) + 1) * 8;
+    LowerBound -= (31 - getPPCRegisterNumbering(MinFPR) + 1) * 8;
   }
 
   // Check whether the frame pointer register is allocated. If so, make sure it
@@ -919,8 +912,8 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
     }
 
     unsigned MinReg =
-      std::min<unsigned>(PPCRegisterInfo::getRegisterNumbering(MinGPR),
-                         PPCRegisterInfo::getRegisterNumbering(MinG8R));
+      std::min<unsigned>(getPPCRegisterNumbering(MinGPR),
+                         getPPCRegisterNumbering(MinG8R));
 
     if (Subtarget.isPPC64()) {
       LowerBound -= (31 - MinReg + 1) * 8;
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.h b/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.h
index 0c18de1e2e26..20faa71d4148 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.h
+++ b/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.h
@@ -40,7 +40,6 @@ class PPCFrameLowering: public TargetFrameLowering {
 
   bool hasFP(const MachineFunction &MF) const;
   bool needsFP(const MachineFunction &MF) const;
-  void getInitialFrameState(std::vector<MachineMove> &Moves) const;
 
   void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                             RegScavenger *RS = NULL) const;
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 2176c02c8503..6f204cc58636 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -14,8 +14,8 @@
 
 #define DEBUG_TYPE "ppc-codegen"
 #include "PPC.h"
-#include "PPCPredicates.h"
 #include "PPCTargetMachine.h"
+#include "MCTargetDesc/PPCPredicates.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 9741a3902af7..d6b8a9ee93c7 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -14,8 +14,8 @@
 #include "PPCISelLowering.h"
 #include "PPCMachineFunctionInfo.h"
 #include "PPCPerfectShuffle.h"
-#include "PPCPredicates.h"
 #include "PPCTargetMachine.h"
+#include "MCTargetDesc/PPCPredicates.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/VectorExtras.h"
 #include "llvm/CodeGen/CallingConvLower.h"
@@ -211,7 +211,8 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // TRAMPOLINE is custom lowered.
-  setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom);
+  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
+  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
   // VASTART needs to be custom lowered to use the VarArgsFrameIndex
   setOperationAction(ISD::VASTART           , MVT::Other, Custom);
@@ -365,7 +366,11 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
     setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
   }
 
+  setOperationAction(ISD::ATOMIC_LOAD,  MVT::i32, Expand);
+  setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Expand);
+
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
 
   if (TM.getSubtarget<PPCSubtarget>().isPPC64()) {
     setStackPointerRegisterToSaveRestore(PPC::X1);
@@ -401,12 +406,14 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
   if (PPCSubTarget.isDarwin())
     setPrefFunctionAlignment(4);
 
+  setInsertFencesForAtomic(true);
+
   computeRegisterProperties();
 }
 
 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
 /// function arguments in the caller parameter area.
-unsigned PPCTargetLowering::getByValTypeAlignment(const Type *Ty) const {
+unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty) const {
   const TargetMachine &TM = getTargetMachine();
   // Darwin passes everything on 4 byte boundary.
   if (TM.getSubtarget<PPCSubtarget>().isDarwin())
@@ -463,7 +470,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   }
 }
 
-MVT::SimpleValueType PPCTargetLowering::getSetCCResultType(EVT VT) const {
+EVT PPCTargetLowering::getSetCCResultType(EVT VT) const {
   return MVT::i32;
 }
 
@@ -1368,8 +1375,13 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
   return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo(), false, false, 0);
 }
 
-SDValue PPCTargetLowering::LowerTRAMPOLINE(SDValue Op,
-                                           SelectionDAG &DAG) const {
+SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  return Op.getOperand(0);
+}
+
+SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
+                                                SelectionDAG &DAG) const {
   SDValue Chain = Op.getOperand(0);
   SDValue Trmp = Op.getOperand(1); // trampoline
   SDValue FPtr = Op.getOperand(2); // nested function
@@ -1378,7 +1390,7 @@ SDValue PPCTargetLowering::LowerTRAMPOLINE(SDValue Op,
 
   EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
   bool isPPC64 = (PtrVT == MVT::i64);
-  const Type *IntPtrTy =
+  Type *IntPtrTy =
     DAG.getTargetLoweringInfo().getTargetData()->getIntPtrType(
                                                              *DAG.getContext());
 
@@ -1398,16 +1410,13 @@ SDValue PPCTargetLowering::LowerTRAMPOLINE(SDValue Op,
 
   // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
   std::pair<SDValue, SDValue> CallResult =
-    LowerCallTo(Chain, Op.getValueType().getTypeForEVT(*DAG.getContext()),
+    LowerCallTo(Chain, Type::getVoidTy(*DAG.getContext()),
                 false, false, false, false, 0, CallingConv::C, false,
                 /*isReturnValueUsed=*/true,
                 DAG.getExternalSymbol("__trampoline_setup", PtrVT),
                 Args, DAG, dl);
 
-  SDValue Ops[] =
-    { CallResult.first, CallResult.second };
-
-  return DAG.getMergeValues(Ops, 2, dl);
+  return CallResult.second;
 }
 
 SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG,
@@ -2550,7 +2559,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
     if (!DAG.getTarget().getSubtarget<PPCSubtarget>().isJITCodeModel()) {
       unsigned OpFlags = 0;
       if (DAG.getTarget().getRelocationModel() != Reloc::Static &&
-          (!PPCSubTarget.getTargetTriple().isMacOSX() ||
+          (PPCSubTarget.getTargetTriple().isMacOSX() &&
            PPCSubTarget.getTargetTriple().isMacOSXVersionLT(10, 5)) &&
           (G->getGlobal()->isDeclaration() ||
            G->getGlobal()->isWeakForLinker())) {
@@ -2574,7 +2583,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
     unsigned char OpFlags = 0;
 
     if (DAG.getTarget().getRelocationModel() != Reloc::Static &&
-        (!PPCSubTarget.getTargetTriple().isMacOSX() ||
+        (PPCSubTarget.getTargetTriple().isMacOSX() &&
          PPCSubTarget.getTargetTriple().isMacOSXVersionLT(10, 5))) {
       // PC-relative references to external symbols should go through $stub,
       // unless we're building with the leopard linker or later, which
@@ -2941,6 +2950,7 @@ PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
   SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
   SmallVector<SDValue, 8> MemOpChains;
 
+  bool seenFloatArg = false;
   // Walk the register/memloc assignments, inserting copies/loads.
   for (unsigned i = 0, j = 0, e = ArgLocs.size();
        i != e;
@@ -2985,6 +2995,7 @@ PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
     }
 
     if (VA.isRegLoc()) {
+      seenFloatArg |= VA.getLocVT().isFloatingPoint();
       // Put argument in a physical register.
       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
     } else {
@@ -3011,9 +3022,11 @@ PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                         &MemOpChains[0], MemOpChains.size());
 
-  // Set CR6 to true if this is a vararg call.
+  // Set CR6 to true if this is a vararg call with floating args passed in
+  // registers.
   if (isVarArg) {
-    SDValue SetCR(DAG.getMachineNode(PPC::CRSET, dl, MVT::i32), 0);
+    SDValue SetCR(DAG.getMachineNode(seenFloatArg ? PPC::CRSET : PPC::CRUNSET,
+                                     dl, MVT::i32), 0);
     RegsToPass.push_back(std::make_pair(unsigned(PPC::CR1EQ), SetCR));
   }
 
@@ -3403,6 +3416,17 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
                     Ins, InVals);
 }
 
+bool
+PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
+                                  MachineFunction &MF, bool isVarArg,
+                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
+                                  LLVMContext &Context) const {
+  SmallVector<CCValAssign, 16> RVLocs;
+  CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(),
+                 RVLocs, Context);
+  return CCInfo.CheckReturn(Outs, RetCC_PPC);
+}
+
 SDValue
 PPCTargetLowering::LowerReturn(SDValue Chain,
                                CallingConv::ID CallConv, bool isVarArg,
@@ -4490,7 +4514,8 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::GlobalTLSAddress:   llvm_unreachable("TLS not implemented for PPC");
   case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
   case ISD::SETCC:              return LowerSETCC(Op, DAG);
-  case ISD::TRAMPOLINE:         return LowerTRAMPOLINE(Op, DAG);
+  case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
+  case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
   case ISD::VASTART:
     return LowerVASTART(Op, DAG, PPCSubTarget);
 
@@ -5504,7 +5529,7 @@ PPCTargetLowering::getSingleConstraintMatchWeight(
     // but allow it at the lowest weight.
   if (CallOperandVal == NULL)
     return CW_Default;
-  const Type *type = CallOperandVal->getType();
+  Type *type = CallOperandVal->getType();
   // Look at the constraint type.
   switch (*constraint) {
   default:
@@ -5634,7 +5659,7 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 // isLegalAddressingMode - Return true if the addressing mode represented
 // by AM is legal for this target, for a load/store of the specified type.
 bool PPCTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              const Type *Ty) const {
+                                              Type *Ty) const {
   // FIXME: PPC does not allow r+i addressing modes for vectors!
 
   // PPC allows a sign-extended 16-bit immediate field.
@@ -5670,7 +5695,7 @@ bool PPCTargetLowering::isLegalAddressingMode(const AddrMode &AM,
 /// isLegalAddressImmediate - Return true if the integer value can be used
 /// as the offset of the target addressing mode for load / store of the
 /// given type.
-bool PPCTargetLowering::isLegalAddressImmediate(int64_t V,const Type *Ty) const{
+bool PPCTargetLowering::isLegalAddressImmediate(int64_t V,Type *Ty) const{
   // PPC allows a sign-extended 16-bit immediate field.
   return (V > -(1 << 16) && V < (1 << 16)-1);
 }
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 986b4e73fbd3..430e45e80493 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -246,7 +246,7 @@ namespace llvm {
     virtual MVT getShiftAmountTy(EVT LHSTy) const { return MVT::i32; }
 
     /// getSetCCResultType - Return the ISD::SETCC ValueType
-    virtual MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    virtual EVT getSetCCResultType(EVT VT) const;
 
     /// getPreIndexedAddressParts - returns true by value, base pointer and
     /// offset pointer and addressing mode by reference if the node's address
@@ -323,7 +323,7 @@ namespace llvm {
     /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
     /// function arguments in the caller parameter area.  This is the actual
     /// alignment, not its logarithm.
-    unsigned getByValTypeAlignment(const Type *Ty) const;
+    unsigned getByValTypeAlignment(Type *Ty) const;
 
     /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
     /// vector.  If it is invalid, don't add anything to Ops.
@@ -334,12 +334,12 @@ namespace llvm {
 
     /// isLegalAddressingMode - Return true if the addressing mode represented
     /// by AM is legal for this target, for a load/store of the specified type.
-    virtual bool isLegalAddressingMode(const AddrMode &AM, const Type *Ty)const;
+    virtual bool isLegalAddressingMode(const AddrMode &AM, Type *Ty)const;
 
     /// isLegalAddressImmediate - Return true if the integer value can be used
     /// as the offset of the target addressing mode for load / store of the
     /// given type.
-    virtual bool isLegalAddressImmediate(int64_t V, const Type *Ty) const;
+    virtual bool isLegalAddressImmediate(int64_t V, Type *Ty) const;
 
     /// isLegalAddressImmediate - Return true if the GlobalValue can be used as
     /// the offset of the target addressing mode.
@@ -390,7 +390,8 @@ namespace llvm {
     SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerTRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG,
                          const PPCSubtarget &Subtarget) const;
     SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG,
@@ -444,6 +445,12 @@ namespace llvm {
                 DebugLoc dl, SelectionDAG &DAG,
                 SmallVectorImpl<SDValue> &InVals) const;
 
+    virtual bool
+      CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
+                   bool isVarArg,
+                   const SmallVectorImpl<ISD::OutputArg> &Outs,
+                   LLVMContext &Context) const;
+
     virtual SDValue
       LowerReturn(SDValue Chain,
                   CallingConv::ID CallConv, bool isVarArg,
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 143444fdc22b..2bc109c8785a 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -15,18 +15,18 @@
 #include "PPC.h"
 #include "PPCInstrBuilder.h"
 #include "PPCMachineFunctionInfo.h"
-#include "PPCPredicates.h"
 #include "PPCTargetMachine.h"
 #include "PPCHazardRecognizers.h"
+#include "MCTargetDesc/PPCPredicates.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/MC/MCAsmInfo.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/STLExtras.h"
 
@@ -334,7 +334,7 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
                                   const TargetRegisterClass *RC,
                                   SmallVectorImpl<MachineInstr*> &NewMIs) const{
   DebugLoc DL;
-  if (RC == PPC::GPRCRegisterClass) {
+  if (PPC::GPRCRegisterClass->hasSubClassEq(RC)) {
     if (SrcReg != PPC::LR) {
       NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::STW))
                                          .addReg(SrcReg,
@@ -350,7 +350,7 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
                                                  getKillRegState(isKill)),
                                          FrameIdx));
     }
-  } else if (RC == PPC::G8RCRegisterClass) {
+  } else if (PPC::G8RCRegisterClass->hasSubClassEq(RC)) {
     if (SrcReg != PPC::LR8) {
       NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::STD))
                                          .addReg(SrcReg,
@@ -366,17 +366,17 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
                                                  getKillRegState(isKill)),
                                          FrameIdx));
     }
-  } else if (RC == PPC::F8RCRegisterClass) {
+  } else if (PPC::F8RCRegisterClass->hasSubClassEq(RC)) {
     NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::STFD))
                                        .addReg(SrcReg,
                                                getKillRegState(isKill)),
                                        FrameIdx));
-  } else if (RC == PPC::F4RCRegisterClass) {
+  } else if (PPC::F4RCRegisterClass->hasSubClassEq(RC)) {
     NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::STFS))
                                        .addReg(SrcReg,
                                                getKillRegState(isKill)),
                                        FrameIdx));
-  } else if (RC == PPC::CRRCRegisterClass) {
+  } else if (PPC::CRRCRegisterClass->hasSubClassEq(RC)) {
     if ((EnablePPC32RS && !TM.getSubtargetImpl()->isPPC64()) ||
         (EnablePPC64RS && TM.getSubtargetImpl()->isPPC64())) {
       // FIXME (64-bit): Enable
@@ -402,7 +402,7 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
       // If the saved register wasn't CR0, shift the bits left so that they are
       // in CR0's slot.
       if (SrcReg != PPC::CR0) {
-        unsigned ShiftBits = PPCRegisterInfo::getRegisterNumbering(SrcReg)*4;
+        unsigned ShiftBits = getPPCRegisterNumbering(SrcReg)*4;
         // rlwinm scratch, scratch, ShiftBits, 0, 31.
         NewMIs.push_back(BuildMI(MF, DL, get(PPC::RLWINM), ScratchReg)
                        .addReg(ScratchReg).addImm(ShiftBits)
@@ -414,7 +414,7 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
                                                  getKillRegState(isKill)),
                                          FrameIdx));
     }
-  } else if (RC == PPC::CRBITRCRegisterClass) {
+  } else if (PPC::CRBITRCRegisterClass->hasSubClassEq(RC)) {
     // FIXME: We use CRi here because there is no mtcrf on a bit. Since the
     // backend currently only uses CR1EQ as an individual bit, this should
     // not cause any bug. If we need other uses of CR bits, the following
@@ -448,7 +448,7 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
     return StoreRegToStackSlot(MF, Reg, isKill, FrameIdx,
                                PPC::CRRCRegisterClass, NewMIs);
 
-  } else if (RC == PPC::VRRCRegisterClass) {
+  } else if (PPC::VRRCRegisterClass->hasSubClassEq(RC)) {
     // We don't have indexed addressing for vector loads.  Emit:
     // R0 = ADDI FI#
     // STVX VAL, 0, R0
@@ -499,7 +499,7 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
                                    unsigned DestReg, int FrameIdx,
                                    const TargetRegisterClass *RC,
                                    SmallVectorImpl<MachineInstr*> &NewMIs)const{
-  if (RC == PPC::GPRCRegisterClass) {
+  if (PPC::GPRCRegisterClass->hasSubClassEq(RC)) {
     if (DestReg != PPC::LR) {
       NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::LWZ),
                                                  DestReg), FrameIdx));
@@ -508,7 +508,7 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
                                                  PPC::R11), FrameIdx));
       NewMIs.push_back(BuildMI(MF, DL, get(PPC::MTLR)).addReg(PPC::R11));
     }
-  } else if (RC == PPC::G8RCRegisterClass) {
+  } else if (PPC::G8RCRegisterClass->hasSubClassEq(RC)) {
     if (DestReg != PPC::LR8) {
       NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::LD), DestReg),
                                          FrameIdx));
@@ -517,13 +517,13 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
                                                  PPC::R11), FrameIdx));
       NewMIs.push_back(BuildMI(MF, DL, get(PPC::MTLR8)).addReg(PPC::R11));
     }
-  } else if (RC == PPC::F8RCRegisterClass) {
+  } else if (PPC::F8RCRegisterClass->hasSubClassEq(RC)) {
     NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::LFD), DestReg),
                                        FrameIdx));
-  } else if (RC == PPC::F4RCRegisterClass) {
+  } else if (PPC::F4RCRegisterClass->hasSubClassEq(RC)) {
     NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::LFS), DestReg),
                                        FrameIdx));
-  } else if (RC == PPC::CRRCRegisterClass) {
+  } else if (PPC::CRRCRegisterClass->hasSubClassEq(RC)) {
     // FIXME: We need a scatch reg here.  The trouble with using R0 is that
     // it's possible for the stack frame to be so big the save location is
     // out of range of immediate offsets, necessitating another register.
@@ -537,7 +537,7 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
     // If the reloaded register isn't CR0, shift the bits right so that they are
     // in the right CR's slot.
     if (DestReg != PPC::CR0) {
-      unsigned ShiftBits = PPCRegisterInfo::getRegisterNumbering(DestReg)*4;
+      unsigned ShiftBits = getPPCRegisterNumbering(DestReg)*4;
       // rlwinm r11, r11, 32-ShiftBits, 0, 31.
       NewMIs.push_back(BuildMI(MF, DL, get(PPC::RLWINM), ScratchReg)
                     .addReg(ScratchReg).addImm(32-ShiftBits).addImm(0)
@@ -546,7 +546,7 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
 
     NewMIs.push_back(BuildMI(MF, DL, get(PPC::MTCRF), DestReg)
                      .addReg(ScratchReg));
-  } else if (RC == PPC::CRBITRCRegisterClass) {
+  } else if (PPC::CRBITRCRegisterClass->hasSubClassEq(RC)) {
 
     unsigned Reg = 0;
     if (DestReg == PPC::CR0LT || DestReg == PPC::CR0GT ||
@@ -577,7 +577,7 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
     return LoadRegFromStackSlot(MF, DL, Reg, FrameIdx,
                                 PPC::CRRCRegisterClass, NewMIs);
 
-  } else if (RC == PPC::VRRCRegisterClass) {
+  } else if (PPC::VRRCRegisterClass->hasSubClassEq(RC)) {
     // We don't have indexed addressing for vector loads.  Emit:
     // R0 = ADDI FI#
     // Dest = LVX 0, R0
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 773578c6ea81..f248b5ba8c48 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -1053,6 +1053,10 @@ def CRSET  : XLForm_1_ext<19, 289, (outs CRBITRC:$dst), (ins),
               "creqv $dst, $dst, $dst", BrCR,
               []>;
 
+def CRUNSET: XLForm_1_ext<19, 193, (outs CRBITRC:$dst), (ins),
+              "crxor $dst, $dst, $dst", BrCR,
+              []>;
+
 // XFX-Form instructions.  Instructions that deal with SPRs.
 //
 let Uses = [CTR] in {
@@ -1472,5 +1476,7 @@ def : Pat<(membarrier (i32 imm /*ll*/),
                       (i32 imm /*device*/)),
            (SYNC)>;
 
+def : Pat<(atomic_fence (imm), (imm)), (SYNC)>;
+
 include "PPCInstrAltivec.td"
 include "PPCInstr64Bit.td"
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 9c2428b92e65..2e90b7a4086b 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -28,7 +28,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/Target/TargetFrameLowering.h"
@@ -68,52 +67,12 @@ PPCRegisterInfo::requiresRegisterScavenging(const MachineFunction &) const {
           (EnablePPC64RS && Subtarget.isPPC64()));
 }
 
-/// getRegisterNumbering - Given the enum value for some register, e.g.
-/// PPC::F14, return the number that it corresponds to (e.g. 14).
-unsigned PPCRegisterInfo::getRegisterNumbering(unsigned RegEnum) {
-  using namespace PPC;
-  switch (RegEnum) {
-  case 0: return 0;
-  case R0 :  case X0 :  case F0 :  case V0 : case CR0:  case CR0LT: return  0;
-  case R1 :  case X1 :  case F1 :  case V1 : case CR1:  case CR0GT: return  1;
-  case R2 :  case X2 :  case F2 :  case V2 : case CR2:  case CR0EQ: return  2;
-  case R3 :  case X3 :  case F3 :  case V3 : case CR3:  case CR0UN: return  3;
-  case R4 :  case X4 :  case F4 :  case V4 : case CR4:  case CR1LT: return  4;
-  case R5 :  case X5 :  case F5 :  case V5 : case CR5:  case CR1GT: return  5;
-  case R6 :  case X6 :  case F6 :  case V6 : case CR6:  case CR1EQ: return  6;
-  case R7 :  case X7 :  case F7 :  case V7 : case CR7:  case CR1UN: return  7;
-  case R8 :  case X8 :  case F8 :  case V8 : case CR2LT: return  8;
-  case R9 :  case X9 :  case F9 :  case V9 : case CR2GT: return  9;
-  case R10:  case X10:  case F10:  case V10: case CR2EQ: return 10;
-  case R11:  case X11:  case F11:  case V11: case CR2UN: return 11;
-  case R12:  case X12:  case F12:  case V12: case CR3LT: return 12;
-  case R13:  case X13:  case F13:  case V13: case CR3GT: return 13;
-  case R14:  case X14:  case F14:  case V14: case CR3EQ: return 14;
-  case R15:  case X15:  case F15:  case V15: case CR3UN: return 15;
-  case R16:  case X16:  case F16:  case V16: case CR4LT: return 16;
-  case R17:  case X17:  case F17:  case V17: case CR4GT: return 17;
-  case R18:  case X18:  case F18:  case V18: case CR4EQ: return 18;
-  case R19:  case X19:  case F19:  case V19: case CR4UN: return 19;
-  case R20:  case X20:  case F20:  case V20: case CR5LT: return 20;
-  case R21:  case X21:  case F21:  case V21: case CR5GT: return 21;
-  case R22:  case X22:  case F22:  case V22: case CR5EQ: return 22;
-  case R23:  case X23:  case F23:  case V23: case CR5UN: return 23;
-  case R24:  case X24:  case F24:  case V24: case CR6LT: return 24;
-  case R25:  case X25:  case F25:  case V25: case CR6GT: return 25;
-  case R26:  case X26:  case F26:  case V26: case CR6EQ: return 26;
-  case R27:  case X27:  case F27:  case V27: case CR6UN: return 27;
-  case R28:  case X28:  case F28:  case V28: case CR7LT: return 28;
-  case R29:  case X29:  case F29:  case V29: case CR7GT: return 29;
-  case R30:  case X30:  case F30:  case V30: case CR7EQ: return 30;
-  case R31:  case X31:  case F31:  case V31: case CR7UN: return 31;
-  default:
-    llvm_unreachable("Unhandled reg in PPCRegisterInfo::getRegisterNumbering!");
-  }
-}
-
 PPCRegisterInfo::PPCRegisterInfo(const PPCSubtarget &ST,
                                  const TargetInstrInfo &tii)
-  : PPCGenRegisterInfo(), Subtarget(ST), TII(tii) {
+  : PPCGenRegisterInfo(ST.isPPC64() ? PPC::LR8 : PPC::LR,
+                       ST.isPPC64() ? 0 : 1,
+                       ST.isPPC64() ? 0 : 1),
+    Subtarget(ST), TII(tii) {
   ImmToIdxMap[PPC::LD]   = PPC::LDX;    ImmToIdxMap[PPC::STD]  = PPC::STDX;
   ImmToIdxMap[PPC::LBZ]  = PPC::LBZX;   ImmToIdxMap[PPC::STB]  = PPC::STBX;
   ImmToIdxMap[PPC::LHZ]  = PPC::LHZX;   ImmToIdxMap[PPC::LHA]  = PPC::LHAX;
@@ -519,7 +478,7 @@ void PPCRegisterInfo::lowerCRSpilling(MachineBasicBlock::iterator II,
     // rlwinm rA, rA, ShiftBits, 0, 31.
     BuildMI(MBB, II, dl, TII.get(PPC::RLWINM), Reg)
       .addReg(Reg, RegState::Kill)
-      .addImm(PPCRegisterInfo::getRegisterNumbering(SrcReg) * 4)
+      .addImm(getPPCRegisterNumbering(SrcReg) * 4)
       .addImm(0)
       .addImm(31);
 
@@ -668,10 +627,6 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   MI.getOperand(OperandBase + 1).ChangeToRegister(SReg, false);
 }
 
-unsigned PPCRegisterInfo::getRARegister() const {
-  return !Subtarget.isPPC64() ? PPC::LR : PPC::LR8;
-}
-
 unsigned PPCRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
 
@@ -688,27 +643,3 @@ unsigned PPCRegisterInfo::getEHExceptionRegister() const {
 unsigned PPCRegisterInfo::getEHHandlerRegister() const {
   return !Subtarget.isPPC64() ? PPC::R4 : PPC::X4;
 }
-
-/// DWARFFlavour - Flavour of dwarf regnumbers
-///
-namespace DWARFFlavour {
-  enum {
-    PPC64 = 0, PPC32 = 1
-  };
-}
-
-int PPCRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  // FIXME: Most probably dwarf numbers differs for Linux and Darwin
-  unsigned Flavour = Subtarget.isPPC64() ?
-    DWARFFlavour::PPC64 : DWARFFlavour::PPC32;
-
-  return PPCGenRegisterInfo::getDwarfRegNumFull(RegNum, Flavour);
-}
-
-int PPCRegisterInfo::getLLVMRegNum(unsigned RegNum, bool isEH) const {
-  // FIXME: Most probably dwarf numbers differs for Linux and Darwin
-  unsigned Flavour = Subtarget.isPPC64() ?
-    DWARFFlavour::PPC64 : DWARFFlavour::PPC32;
-
-  return PPCGenRegisterInfo::getLLVMRegNumFull(RegNum, Flavour);
-}
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 33fe5ebcf4cd..1cc7213417d1 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/contrib/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -33,10 +33,6 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
 public:
   PPCRegisterInfo(const PPCSubtarget &SubTarget, const TargetInstrInfo &tii);
   
-  /// getRegisterNumbering - Given the enum value for some register, e.g.
-  /// PPC::F14, return the number that it corresponds to (e.g. 14).
-  static unsigned getRegisterNumbering(unsigned RegEnum);
-
   /// getPointerRegClass - Return the register class to use to hold pointers.
   /// This is used for addressing modes.
   virtual const TargetRegisterClass *getPointerRegClass(unsigned Kind=0) const;  
@@ -62,15 +58,11 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
                            int SPAdj, RegScavenger *RS = NULL) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   // Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
-
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 5ea9b0f6596c..cf194de42e8f 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -15,7 +15,7 @@
 #include "PPC.h"
 #include "llvm/GlobalValue.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 #include <cstdlib>
 
 #define GET_SUBTARGETINFO_TARGET_DESC
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index e0ea5adba751..f5744b830489 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -16,76 +16,43 @@
 #include "llvm/PassManager.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
-// This is duplicated code. Refactor this.
-static MCStreamer *createMCStreamer(const Target &T, const std::string &TT,
-                                    MCContext &Ctx, TargetAsmBackend &TAB,
-                                    raw_ostream &OS,
-                                    MCCodeEmitter *Emitter,
-                                    bool RelaxAll,
-                                    bool NoExecStack) {
-  if (Triple(TT).isOSDarwin())
-    return createMachOStreamer(Ctx, TAB, OS, Emitter, RelaxAll);
-
-  return NULL;
-}
-
 extern "C" void LLVMInitializePowerPCTarget() {
   // Register the targets
   RegisterTargetMachine<PPC32TargetMachine> A(ThePPC32Target);  
   RegisterTargetMachine<PPC64TargetMachine> B(ThePPC64Target);
-  
-  // Register the MC Code Emitter
-  TargetRegistry::RegisterCodeEmitter(ThePPC32Target, createPPCMCCodeEmitter);
-  TargetRegistry::RegisterCodeEmitter(ThePPC64Target, createPPCMCCodeEmitter);
-  
-  
-  // Register the asm backend.
-  TargetRegistry::RegisterAsmBackend(ThePPC32Target, createPPCAsmBackend);
-  TargetRegistry::RegisterAsmBackend(ThePPC64Target, createPPCAsmBackend);
-  
-  // Register the object streamer.
-  TargetRegistry::RegisterObjectStreamer(ThePPC32Target, createMCStreamer);
-  TargetRegistry::RegisterObjectStreamer(ThePPC64Target, createMCStreamer);
 }
 
-
-PPCTargetMachine::PPCTargetMachine(const Target &T, const std::string &TT,
-                                   const std::string &CPU,
-                                   const std::string &FS, bool is64Bit)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+PPCTargetMachine::PPCTargetMachine(const Target &T, StringRef TT,
+                                   StringRef CPU, StringRef FS,
+                                   Reloc::Model RM, CodeModel::Model CM,
+                                   bool is64Bit)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS, is64Bit),
     DataLayout(Subtarget.getTargetDataString()), InstrInfo(*this),
     FrameLowering(Subtarget), JITInfo(*this, is64Bit),
     TLInfo(*this), TSInfo(*this),
     InstrItins(Subtarget.getInstrItineraryData()) {
-
-  if (getRelocationModel() == Reloc::Default) {
-    if (Subtarget.isDarwin())
-      setRelocationModel(Reloc::DynamicNoPIC);
-    else
-      setRelocationModel(Reloc::Static);
-  }
 }
 
 /// Override this for PowerPC.  Tail merging happily breaks up instruction issue
 /// groups, which typically degrades performance.
 bool PPCTargetMachine::getEnableTailMergeDefault() const { return false; }
 
-PPC32TargetMachine::PPC32TargetMachine(const Target &T, const std::string &TT, 
-                                       const std::string &CPU,
-                                       const std::string &FS) 
-  : PPCTargetMachine(T, TT, CPU, FS, false) {
+PPC32TargetMachine::PPC32TargetMachine(const Target &T, StringRef TT, 
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM) 
+  : PPCTargetMachine(T, TT, CPU, FS, RM, CM, false) {
 }
 
 
-PPC64TargetMachine::PPC64TargetMachine(const Target &T, const std::string &TT, 
-                                       const std::string &CPU, 
-                                       const std::string &FS)
-  : PPCTargetMachine(T, TT, CPU, FS, true) {
+PPC64TargetMachine::PPC64TargetMachine(const Target &T, StringRef TT, 
+                                       StringRef CPU,  StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM)
+  : PPCTargetMachine(T, TT, CPU, FS, RM, CM, true) {
 }
 
 
@@ -110,19 +77,11 @@ bool PPCTargetMachine::addPreEmitPass(PassManagerBase &PM,
 bool PPCTargetMachine::addCodeEmitter(PassManagerBase &PM,
                                       CodeGenOpt::Level OptLevel,
                                       JITCodeEmitter &JCE) {
-  // The JIT should use the static relocation model in ppc32 mode, PIC in ppc64.
   // FIXME: This should be moved to TargetJITInfo!!
-  if (Subtarget.isPPC64()) {
-    // We use PIC codegen in ppc64 mode, because otherwise we'd have to use many
-    // instructions to materialize arbitrary global variable + function +
-    // constant pool addresses.
-    setRelocationModel(Reloc::PIC_);
+  if (Subtarget.isPPC64())
     // Temporary workaround for the inability of PPC64 JIT to handle jump
     // tables.
     DisableJumpTables = true;      
-  } else {
-    setRelocationModel(Reloc::Static);
-  }
   
   // Inform the subtarget that we are in JIT mode.  FIXME: does this break macho
   // writing?
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.h b/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.h
index baf07e3498f8..d06f0843bd6d 100644
--- a/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/contrib/llvm/lib/Target/PowerPC/PPCTargetMachine.h
@@ -40,9 +40,9 @@ class PPCTargetMachine : public LLVMTargetMachine {
   InstrItineraryData  InstrItins;
 
 public:
-  PPCTargetMachine(const Target &T, const std::string &TT,
-                   const std::string &CPU, const std::string &FS,
-                   bool is64Bit);
+  PPCTargetMachine(const Target &T, StringRef TT,
+                   StringRef CPU, StringRef FS,
+                   Reloc::Model RM, CodeModel::Model CM, bool is64Bit);
 
   virtual const PPCInstrInfo      *getInstrInfo() const { return &InstrInfo; }
   virtual const PPCFrameLowering  *getFrameLowering() const {
@@ -77,16 +77,18 @@ class PPCTargetMachine : public LLVMTargetMachine {
 ///
 class PPC32TargetMachine : public PPCTargetMachine {
 public:
-  PPC32TargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS);
+  PPC32TargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 };
 
 /// PPC64TargetMachine - PowerPC 64-bit target machine.
 ///
 class PPC64TargetMachine : public PPCTargetMachine {
 public:
-  PPC64TargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS);
+  PPC64TargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp b/contrib/llvm/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
index ad607d0ade6a..5dc8568d83f2 100644
--- a/contrib/llvm/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "PPC.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::ThePPC32Target, llvm::ThePPC64Target;
diff --git a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/Sparc/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 1e8c02979887..000000000000
--- a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-add_llvm_library(LLVMSparcDesc
-  SparcMCTargetDesc.cpp
-  SparcMCAsmInfo.cpp
-  )
diff --git a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/Sparc/MCTargetDesc/Makefile
deleted file mode 100644
index abcbe2da18ec..000000000000
--- a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/Sparc/TargetDesc/Makefile ----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMSparcDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp b/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
index cb92a2bfd417..cb2a7dfe6160 100644
--- a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
@@ -13,10 +13,11 @@
 
 #include "SparcMCTargetDesc.h"
 #include "SparcMCAsmInfo.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "SparcGenInstrInfo.inc"
@@ -35,8 +36,10 @@ static MCInstrInfo *createSparcMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeSparcMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheSparcTarget, createSparcMCInstrInfo);
+static MCRegisterInfo *createSparcMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitSparcMCRegisterInfo(X, SP::I7);
+  return X;
 }
 
 static MCSubtargetInfo *createSparcMCSubtargetInfo(StringRef TT, StringRef CPU,
@@ -46,12 +49,31 @@ static MCSubtargetInfo *createSparcMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeSparcMCSubtargetInfo() {
+static MCCodeGenInfo *createSparcMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                               CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+extern "C" void LLVMInitializeSparcTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfo<SparcELFMCAsmInfo> X(TheSparcTarget);
+  RegisterMCAsmInfo<SparcELFMCAsmInfo> Y(TheSparcV9Target);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheSparcTarget,
+                                       createSparcMCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(TheSparcV9Target,
+                                       createSparcMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheSparcTarget, createSparcMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheSparcTarget, createSparcMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(TheSparcTarget,
                                           createSparcMCSubtargetInfo);
 }
-
-extern "C" void LLVMInitializeSparcMCAsmInfo() {
-  RegisterMCAsmInfo<SparcELFMCAsmInfo> X(TheSparcTarget);
-  RegisterMCAsmInfo<SparcELFMCAsmInfo> Y(TheSparcV9Target);
-}
diff --git a/contrib/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp b/contrib/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
index edde8427aa89..345e1bca54c6 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -22,9 +22,9 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/contrib/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 6f30d3fd6c35..d70b16375e95 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/contrib/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -631,8 +631,8 @@ SparcTargetLowering::getSRetArgSize(SelectionDAG &DAG, SDValue Callee) const
   assert(CalleeFn->hasStructRetAttr() &&
          "Callee does not have the StructRet attribute.");
 
-  const PointerType *Ty = cast<PointerType>(CalleeFn->arg_begin()->getType());
-  const Type *ElementTy = Ty->getElementType();
+  PointerType *Ty = cast<PointerType>(CalleeFn->arg_begin()->getType());
+  Type *ElementTy = Ty->getElementType();
   return getTargetData()->getTypeAllocSize(ElementTy);
 }
 
@@ -748,8 +748,10 @@ SparcTargetLowering::SparcTargetLowering(TargetMachine &TM)
   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
   setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
 
-  // SPARC has no intrinsics for these particular operations.
+  // FIXME: There are instructions available for ATOMIC_FENCE
+  // on SparcV8 and later.
   setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
 
   setOperationAction(ISD::FSIN , MVT::f64, Expand);
   setOperationAction(ISD::FCOS , MVT::f64, Expand);
diff --git a/contrib/llvm/lib/Target/Sparc/SparcInstrInfo.cpp b/contrib/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
index 4e3ddf839985..7a6bf50fa7d4 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -17,8 +17,8 @@
 #include "SparcSubtarget.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 
diff --git a/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.cpp b/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.cpp
index 0acdd2c55d6b..8c1625148c8c 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.cpp
@@ -17,7 +17,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Type.h"
@@ -31,7 +30,7 @@ using namespace llvm;
 
 SparcRegisterInfo::SparcRegisterInfo(SparcSubtarget &st,
                                      const TargetInstrInfo &tii)
-  : SparcGenRegisterInfo(), Subtarget(st), TII(tii) {
+  : SparcGenRegisterInfo(SP::I7), Subtarget(st), TII(tii) {
 }
 
 const unsigned* SparcRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF)
@@ -113,10 +112,6 @@ SparcRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 void SparcRegisterInfo::
 processFunctionBeforeFrameFinalized(MachineFunction &MF) const {}
 
-unsigned SparcRegisterInfo::getRARegister() const {
-  return SP::I7;
-}
-
 unsigned SparcRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   return SP::I6;
 }
@@ -130,11 +125,3 @@ unsigned SparcRegisterInfo::getEHHandlerRegister() const {
   llvm_unreachable("What is the exception handler register");
   return 0;
 }
-
-int SparcRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  return SparcGenRegisterInfo::getDwarfRegNumFull(RegNum, 0);
-}
-
-int SparcRegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  return SparcGenRegisterInfo::getLLVMRegNumFull(DwarfRegNo,0);
-}
diff --git a/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.h b/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.h
index ec9e63a686bc..f845667b4d9c 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.h
+++ b/contrib/llvm/lib/Target/Sparc/SparcRegisterInfo.h
@@ -46,15 +46,11 @@ struct SparcRegisterInfo : public SparcGenRegisterInfo {
   void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   // Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
-
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/Sparc/SparcSubtarget.cpp b/contrib/llvm/lib/Target/Sparc/SparcSubtarget.cpp
index de647e8221a2..6c501cff6a3a 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcSubtarget.cpp
+++ b/contrib/llvm/lib/Target/Sparc/SparcSubtarget.cpp
@@ -13,7 +13,7 @@
 
 #include "SparcSubtarget.h"
 #include "Sparc.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.cpp b/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
index cbe6d8754efd..3d7b4a47d1a8 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
@@ -13,7 +13,7 @@
 #include "Sparc.h"
 #include "SparcTargetMachine.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 extern "C" void LLVMInitializeSparcTarget() {
@@ -24,10 +24,11 @@ extern "C" void LLVMInitializeSparcTarget() {
 
 /// SparcTargetMachine ctor - Create an ILP32 architecture model
 ///
-SparcTargetMachine::SparcTargetMachine(const Target &T, const std::string &TT, 
-                                       const std::string &CPU,
-                                       const std::string &FS, bool is64bit)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+SparcTargetMachine::SparcTargetMachine(const Target &T, StringRef TT, 
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM,
+                                       bool is64bit)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS, is64bit),
     DataLayout(Subtarget.getDataLayout()),
     TLInfo(*this), TSInfo(*this), InstrInfo(Subtarget),
@@ -51,15 +52,15 @@ bool SparcTargetMachine::addPreEmitPass(PassManagerBase &PM,
 }
 
 SparcV8TargetMachine::SparcV8TargetMachine(const Target &T,
-                                           const std::string &TT, 
-                                           const std::string &CPU,
-                                           const std::string &FS)
-  : SparcTargetMachine(T, TT, CPU, FS, false) {
+                                           StringRef TT, StringRef CPU,
+                                           StringRef FS, Reloc::Model RM,
+                                           CodeModel::Model CM)
+  : SparcTargetMachine(T, TT, CPU, FS, RM, CM, false) {
 }
 
 SparcV9TargetMachine::SparcV9TargetMachine(const Target &T, 
-                                           const std::string &TT, 
-                                           const std::string &CPU,
-                                           const std::string &FS)
-  : SparcTargetMachine(T, TT, CPU, FS, true) {
+                                           StringRef TT,  StringRef CPU,
+                                           StringRef FS, Reloc::Model RM,
+                                           CodeModel::Model CM)
+  : SparcTargetMachine(T, TT, CPU, FS, RM, CM, true) {
 }
diff --git a/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.h b/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.h
index 799fc497f4ae..3c907dd44de1 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.h
+++ b/contrib/llvm/lib/Target/Sparc/SparcTargetMachine.h
@@ -33,9 +33,9 @@ class SparcTargetMachine : public LLVMTargetMachine {
   SparcInstrInfo InstrInfo;
   SparcFrameLowering FrameLowering;
 public:
-  SparcTargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS,
-                     bool is64bit);
+  SparcTargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM, bool is64bit);
 
   virtual const SparcInstrInfo *getInstrInfo() const { return &InstrInfo; }
   virtual const TargetFrameLowering  *getFrameLowering() const {
@@ -62,16 +62,18 @@ class SparcTargetMachine : public LLVMTargetMachine {
 ///
 class SparcV8TargetMachine : public SparcTargetMachine {
 public:
-  SparcV8TargetMachine(const Target &T, const std::string &TT,
-                       const std::string &CPU, const std::string &FS);
+  SparcV8TargetMachine(const Target &T, StringRef TT,
+                       StringRef CPU, StringRef FS,
+                       Reloc::Model RM, CodeModel::Model CM);
 };
 
 /// SparcV9TargetMachine - Sparc 64-bit target machine
 ///
 class SparcV9TargetMachine : public SparcTargetMachine {
 public:
-  SparcV9TargetMachine(const Target &T, const std::string &TT,
-                       const std::string &CPU, const std::string &FS);
+  SparcV9TargetMachine(const Target &T, StringRef TT,
+                       StringRef CPU, StringRef FS,
+                       Reloc::Model RM, CodeModel::Model CM);
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp b/contrib/llvm/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
index 5c06f0727e96..c9d5b7bdfb3d 100644
--- a/contrib/llvm/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "Sparc.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheSparcTarget;
diff --git a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 2ac90164721f..000000000000
--- a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-add_llvm_library(LLVMSystemZDesc
-  SystemZMCTargetDesc.cpp
-  SystemZMCAsmInfo.cpp
-  )
-
-# Hack: we need to include 'main' target directory to grab private headers
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/..)
diff --git a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/Makefile
deleted file mode 100644
index 08f1a9d51fb5..000000000000
--- a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/SystemZ/TargetDesc/Makefile --------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMSystemZDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 5a826a6ef887..23fb1e068e70 100644
--- a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -13,10 +13,11 @@
 
 #include "SystemZMCTargetDesc.h"
 #include "SystemZMCAsmInfo.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "SystemZGenInstrInfo.inc"
@@ -35,9 +36,10 @@ static MCInstrInfo *createSystemZMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeSystemZMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheSystemZTarget,
-                                      createSystemZMCInstrInfo);
+static MCRegisterInfo *createSystemZMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitSystemZMCRegisterInfo(X, 0);
+  return X;
 }
 
 static MCSubtargetInfo *createSystemZMCSubtargetInfo(StringRef TT,
@@ -48,11 +50,32 @@ static MCSubtargetInfo *createSystemZMCSubtargetInfo(StringRef TT,
   return X;
 }
 
-extern "C" void LLVMInitializeSystemZMCSubtargetInfo() {
+static MCCodeGenInfo *createSystemZMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                                 CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  if (RM == Reloc::Default)
+    RM = Reloc::Static;
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+extern "C" void LLVMInitializeSystemZTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfo<SystemZMCAsmInfo> X(TheSystemZTarget);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheSystemZTarget,
+                                        createSystemZMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheSystemZTarget,
+                                      createSystemZMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheSystemZTarget,
+                                    createSystemZMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(TheSystemZTarget,
                                           createSystemZMCSubtargetInfo);
 }
-
-extern "C" void LLVMInitializeSystemZMCAsmInfo() {
-  RegisterMCAsmInfo<SystemZMCAsmInfo> X(TheSystemZTarget);
-}
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index fd4d8b70c75e..43dcdfc3936b 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -28,10 +28,8 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 871c2972a8c4..48ca99ff9ea2 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -81,6 +81,7 @@ SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) :
   setSchedulingPreference(Sched::RegPressure);
 
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
 
   setOperationAction(ISD::BR_JT,            MVT::Other, Expand);
   setOperationAction(ISD::BRCOND,           MVT::Other, Expand);
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 99e2730609e8..5f3dd80f2d90 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -21,8 +21,8 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_CTOR
 #include "SystemZGenInstrInfo.inc"
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index 11a39fcd023a..580d65b27f8a 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -478,7 +478,7 @@ def MOV64rmm  : RSYI<0x04EB,
                      "lmg\t{$from, $to, $dst}",
                      []>;
 
-let isReMaterializable = 1, isAsCheapAsAMove = 1,
+let isReMaterializable = 1, neverHasSideEffects = 1, isAsCheapAsAMove = 1,
     Constraints = "$src = $dst" in {
 def MOV64Pr0_even : Pseudo<(outs GR64P:$dst), (ins GR64P:$src),
                            "lhi\t${dst:subreg_even}, 0",
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index 59692e883366..b1050d46e550 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -33,7 +33,7 @@ using namespace llvm;
 
 SystemZRegisterInfo::SystemZRegisterInfo(SystemZTargetMachine &tm,
                                          const SystemZInstrInfo &tii)
-  : SystemZGenRegisterInfo(), TM(tm), TII(tii) {
+  : SystemZGenRegisterInfo(0), TM(tm), TII(tii) {
 }
 
 const unsigned*
@@ -126,11 +126,6 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   MI.getOperand(i+1).ChangeToImmediate(Offset);
 }
 
-unsigned SystemZRegisterInfo::getRARegister() const {
-  assert(0 && "What is the return address register");
-  return 0;
-}
-
 unsigned
 SystemZRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   assert(0 && "What is the frame register");
@@ -146,13 +141,3 @@ unsigned SystemZRegisterInfo::getEHHandlerRegister() const {
   assert(0 && "What is the exception handler register");
   return 0;
 }
-
-int SystemZRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  assert(0 && "What is the dwarf register number");
-  return -1;
-}
-
-int SystemZRegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  assert(0 && "What is the dwarf register number");
-  return -1;
-}
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h b/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
index 2e262e1acc30..03935b2bec10 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
@@ -48,15 +48,11 @@ struct SystemZRegisterInfo : public SystemZGenRegisterInfo {
                            int SPAdj, RegScavenger *RS = NULL) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   // Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
-
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
index b3ed06639758..0845510761c2 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
@@ -15,7 +15,7 @@
 #include "SystemZ.h"
 #include "llvm/GlobalValue.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 48298cc744e7..e390f060c9a9 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -10,7 +10,7 @@
 #include "SystemZTargetMachine.h"
 #include "SystemZ.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 extern "C" void LLVMInitializeSystemZTarget() {
@@ -21,18 +21,15 @@ extern "C" void LLVMInitializeSystemZTarget() {
 /// SystemZTargetMachine ctor - Create an ILP64 architecture model
 ///
 SystemZTargetMachine::SystemZTargetMachine(const Target &T,
-                                           const std::string &TT,
-                                           const std::string &CPU,
-                                           const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+                                           StringRef TT, StringRef CPU,
+                                           StringRef FS, Reloc::Model RM,
+                                           CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS),
     DataLayout("E-p:64:64:64-i8:8:16-i16:16:16-i32:32:32-i64:64:64-f32:32:32"
                "-f64:64:64-f128:128:128-a0:16:16-n32:64"),
     InstrInfo(*this), TLInfo(*this), TSInfo(*this),
     FrameLowering(Subtarget) {
-
-  if (getRelocationModel() == Reloc::Default)
-    setRelocationModel(Reloc::Static);
 }
 
 bool SystemZTargetMachine::addInstSelector(PassManagerBase &PM,
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.h b/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.h
index e40b556c0c3c..43dce4bd148e 100644
--- a/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.h
+++ b/contrib/llvm/lib/Target/SystemZ/SystemZTargetMachine.h
@@ -37,8 +37,9 @@ class SystemZTargetMachine : public LLVMTargetMachine {
   SystemZSelectionDAGInfo TSInfo;
   SystemZFrameLowering    FrameLowering;
 public:
-  SystemZTargetMachine(const Target &T, const std::string &TT,
-                       const std::string &CPU, const std::string &FS);
+  SystemZTargetMachine(const Target &T, StringRef TT,
+                       StringRef CPU, StringRef FS,
+                       Reloc::Model RM, CodeModel::Model CM);
 
   virtual const TargetFrameLowering *getFrameLowering() const {
     return &FrameLowering;
diff --git a/contrib/llvm/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp b/contrib/llvm/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
index 8272b1188201..da99282ecb04 100644
--- a/contrib/llvm/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "SystemZ.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheSystemZTarget;
diff --git a/contrib/llvm/lib/Target/Target.cpp b/contrib/llvm/lib/Target/Target.cpp
index a42ce548c895..a2b83bcce46e 100644
--- a/contrib/llvm/lib/Target/Target.cpp
+++ b/contrib/llvm/lib/Target/Target.cpp
@@ -17,6 +17,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/PassManager.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/LLVMContext.h"
 #include <cstring>
 
@@ -39,6 +40,11 @@ void LLVMAddTargetData(LLVMTargetDataRef TD, LLVMPassManagerRef PM) {
   unwrap(PM)->add(new TargetData(*unwrap(TD)));
 }
 
+void LLVMAddTargetLibraryInfo(LLVMTargetLibraryInfoRef TLI,
+                              LLVMPassManagerRef PM) {
+  unwrap(PM)->add(new TargetLibraryInfo(*unwrap(TLI)));
+}
+
 char *LLVMCopyStringRepOfTargetData(LLVMTargetDataRef TD) {
   std::string StringRep = unwrap(TD)->getStringRepresentation();
   return strdup(StringRep.c_str());
@@ -87,13 +93,13 @@ unsigned LLVMPreferredAlignmentOfGlobal(LLVMTargetDataRef TD,
 
 unsigned LLVMElementAtOffset(LLVMTargetDataRef TD, LLVMTypeRef StructTy,
                              unsigned long long Offset) {
-  const StructType *STy = unwrap<StructType>(StructTy);
+  StructType *STy = unwrap<StructType>(StructTy);
   return unwrap(TD)->getStructLayout(STy)->getElementContainingOffset(Offset);
 }
 
 unsigned long long LLVMOffsetOfElement(LLVMTargetDataRef TD, LLVMTypeRef StructTy,
                                        unsigned Element) {
-  const StructType *STy = unwrap<StructType>(StructTy);
+  StructType *STy = unwrap<StructType>(StructTy);
   return unwrap(TD)->getStructLayout(STy)->getElementOffset(Element);
 }
 
diff --git a/contrib/llvm/lib/Target/TargetAsmInfo.cpp b/contrib/llvm/lib/Target/TargetAsmInfo.cpp
deleted file mode 100644
index a97b0e868989..000000000000
--- a/contrib/llvm/lib/Target/TargetAsmInfo.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-//===-- llvm/Target/TargetAsmInfo.cpp - Target Assembly Info --------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetAsmInfo.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetLowering.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegisterInfo.h"
-using namespace llvm;
-
-TargetAsmInfo::TargetAsmInfo(const TargetMachine &TM) {
-  TLOF = &TM.getTargetLowering()->getObjFileLowering();
-  TFI = TM.getFrameLowering();
-  TRI = TM.getRegisterInfo();
-  TFI->getInitialFrameState(InitialFrameState);
-}
diff --git a/contrib/llvm/lib/Target/TargetData.cpp b/contrib/llvm/lib/Target/TargetData.cpp
index 17d022a339e6..bd6a6b67beb9 100644
--- a/contrib/llvm/lib/Target/TargetData.cpp
+++ b/contrib/llvm/lib/Target/TargetData.cpp
@@ -41,7 +41,7 @@ char TargetData::ID = 0;
 // Support for StructLayout
 //===----------------------------------------------------------------------===//
 
-StructLayout::StructLayout(const StructType *ST, const TargetData &TD) {
+StructLayout::StructLayout(StructType *ST, const TargetData &TD) {
   assert(!ST->isOpaque() && "Cannot get layout of opaque structs");
   StructAlignment = 0;
   StructSize = 0;
@@ -49,7 +49,7 @@ StructLayout::StructLayout(const StructType *ST, const TargetData &TD) {
 
   // Loop over each of the elements, placing them in memory.
   for (unsigned i = 0, e = NumElements; i != e; ++i) {
-    const Type *Ty = ST->getElementType(i);
+    Type *Ty = ST->getElementType(i);
     unsigned TyAlign = ST->isPacked() ? 1 : TD.getABITypeAlignment(Ty);
 
     // Add padding if necessary to align the data element properly.
@@ -139,6 +139,7 @@ void TargetData::init(StringRef Desc) {
   PointerMemSize = 8;
   PointerABIAlign = 8;
   PointerPrefAlign = PointerABIAlign;
+  StackNaturalAlign = 0;
 
   // Default alignments
   setAlignment(INTEGER_ALIGN,   1,  1, 1);   // i1
@@ -218,7 +219,12 @@ void TargetData::init(StringRef Desc) {
         Token = Split.second;
       } while (!Specifier.empty() || !Token.empty());
       break;
-
+    case 'S': // Stack natural alignment.
+      StackNaturalAlign = getInt(Specifier.substr(1));
+      StackNaturalAlign /= 8;
+      // FIXME: Should we really be truncating these alingments and
+      // sizes silently?
+      break;
     default:
       break;
     }
@@ -261,7 +267,7 @@ TargetData::setAlignment(AlignTypeEnum align_type, unsigned abi_align,
 /// preferred if ABIInfo = false) the target wants for the specified datatype.
 unsigned TargetData::getAlignmentInfo(AlignTypeEnum AlignType,
                                       uint32_t BitWidth, bool ABIInfo,
-                                      const Type *Ty) const {
+                                      Type *Ty) const {
   // Check to see if we have an exact match and remember the best match we see.
   int BestMatchIdx = -1;
   int LargestInt = -1;
@@ -315,7 +321,7 @@ unsigned TargetData::getAlignmentInfo(AlignTypeEnum AlignType,
 namespace {
 
 class StructLayoutMap {
-  typedef DenseMap<const StructType*, StructLayout*> LayoutInfoTy;
+  typedef DenseMap<StructType*, StructLayout*> LayoutInfoTy;
   LayoutInfoTy LayoutInfo;
 
 public:
@@ -329,7 +335,7 @@ class StructLayoutMap {
     }
   }
 
-  StructLayout *&operator[](const StructType *STy) {
+  StructLayout *&operator[](StructType *STy) {
     return LayoutInfo[STy];
   }
 
@@ -343,7 +349,7 @@ TargetData::~TargetData() {
   delete static_cast<StructLayoutMap*>(LayoutMap);
 }
 
-const StructLayout *TargetData::getStructLayout(const StructType *Ty) const {
+const StructLayout *TargetData::getStructLayout(StructType *Ty) const {
   if (!LayoutMap)
     LayoutMap = new StructLayoutMap();
 
@@ -372,7 +378,9 @@ std::string TargetData::getStringRepresentation() const {
 
   OS << (LittleEndian ? "e" : "E")
      << "-p:" << PointerMemSize*8 << ':' << PointerABIAlign*8
-     << ':' << PointerPrefAlign*8;
+     << ':' << PointerPrefAlign*8
+     << "-S" << StackNaturalAlign*8;
+
   for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
     const TargetAlignElem &AI = Alignments[i];
     OS << '-' << (char)AI.AlignType << AI.TypeBitWidth << ':'
@@ -389,14 +397,14 @@ std::string TargetData::getStringRepresentation() const {
 }
 
 
-uint64_t TargetData::getTypeSizeInBits(const Type *Ty) const {
+uint64_t TargetData::getTypeSizeInBits(Type *Ty) const {
   assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
   switch (Ty->getTypeID()) {
   case Type::LabelTyID:
   case Type::PointerTyID:
     return getPointerSizeInBits();
   case Type::ArrayTyID: {
-    const ArrayType *ATy = cast<ArrayType>(Ty);
+    ArrayType *ATy = cast<ArrayType>(Ty);
     return getTypeAllocSizeInBits(ATy->getElementType())*ATy->getNumElements();
   }
   case Type::StructTyID:
@@ -435,7 +443,7 @@ uint64_t TargetData::getTypeSizeInBits(const Type *Ty) const {
   Get the ABI (\a abi_or_pref == true) or preferred alignment (\a abi_or_pref
   == false) for the requested type \a Ty.
  */
-unsigned TargetData::getAlignment(const Type *Ty, bool abi_or_pref) const {
+unsigned TargetData::getAlignment(Type *Ty, bool abi_or_pref) const {
   int AlignType = -1;
 
   assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
@@ -485,7 +493,7 @@ unsigned TargetData::getAlignment(const Type *Ty, bool abi_or_pref) const {
                           abi_or_pref, Ty);
 }
 
-unsigned TargetData::getABITypeAlignment(const Type *Ty) const {
+unsigned TargetData::getABITypeAlignment(Type *Ty) const {
   return getAlignment(Ty, true);
 }
 
@@ -496,7 +504,7 @@ unsigned TargetData::getABIIntegerTypeAlignment(unsigned BitWidth) const {
 }
 
 
-unsigned TargetData::getCallFrameTypeAlignment(const Type *Ty) const {
+unsigned TargetData::getCallFrameTypeAlignment(Type *Ty) const {
   for (unsigned i = 0, e = Alignments.size(); i != e; ++i)
     if (Alignments[i].AlignType == STACK_ALIGN)
       return Alignments[i].ABIAlign;
@@ -504,11 +512,11 @@ unsigned TargetData::getCallFrameTypeAlignment(const Type *Ty) const {
   return getABITypeAlignment(Ty);
 }
 
-unsigned TargetData::getPrefTypeAlignment(const Type *Ty) const {
+unsigned TargetData::getPrefTypeAlignment(Type *Ty) const {
   return getAlignment(Ty, false);
 }
 
-unsigned TargetData::getPreferredTypeAlignmentShift(const Type *Ty) const {
+unsigned TargetData::getPreferredTypeAlignmentShift(Type *Ty) const {
   unsigned Align = getPrefTypeAlignment(Ty);
   assert(!(Align & (Align-1)) && "Alignment is not a power of two!");
   return Log2_32(Align);
@@ -521,16 +529,17 @@ IntegerType *TargetData::getIntPtrType(LLVMContext &C) const {
 }
 
 
-uint64_t TargetData::getIndexedOffset(const Type *ptrTy, Value* const* Indices,
-                                      unsigned NumIndices) const {
-  const Type *Ty = ptrTy;
+uint64_t TargetData::getIndexedOffset(Type *ptrTy,
+                                      ArrayRef<Value *> Indices) const {
+  Type *Ty = ptrTy;
   assert(Ty->isPointerTy() && "Illegal argument for getIndexedOffset()");
   uint64_t Result = 0;
 
   generic_gep_type_iterator<Value* const*>
-    TI = gep_type_begin(ptrTy, Indices, Indices+NumIndices);
-  for (unsigned CurIDX = 0; CurIDX != NumIndices; ++CurIDX, ++TI) {
-    if (const StructType *STy = dyn_cast<StructType>(*TI)) {
+    TI = gep_type_begin(ptrTy, Indices);
+  for (unsigned CurIDX = 0, EndIDX = Indices.size(); CurIDX != EndIDX;
+       ++CurIDX, ++TI) {
+    if (StructType *STy = dyn_cast<StructType>(*TI)) {
       assert(Indices[CurIDX]->getType() ==
              Type::getInt32Ty(ptrTy->getContext()) &&
              "Illegal struct idx");
@@ -561,7 +570,7 @@ uint64_t TargetData::getIndexedOffset(const Type *ptrTy, Value* const* Indices,
 /// global.  This includes an explicitly requested alignment (if the global
 /// has one).
 unsigned TargetData::getPreferredAlignment(const GlobalVariable *GV) const {
-  const Type *ElemType = GV->getType()->getElementType();
+  Type *ElemType = GV->getType()->getElementType();
   unsigned Alignment = getPrefTypeAlignment(ElemType);
   unsigned GVAlignment = GV->getAlignment();
   if (GVAlignment >= Alignment) {
diff --git a/contrib/llvm/lib/Target/TargetFrameLowering.cpp b/contrib/llvm/lib/Target/TargetFrameLowering.cpp
index 19fd581c7dd5..122f8696e2ce 100644
--- a/contrib/llvm/lib/Target/TargetFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/TargetFrameLowering.cpp
@@ -23,14 +23,6 @@ using namespace llvm;
 TargetFrameLowering::~TargetFrameLowering() {
 }
 
-/// getInitialFrameState - Returns a list of machine moves that are assumed
-/// on entry to a function.
-void
-TargetFrameLowering::getInitialFrameState(std::vector<MachineMove> &Moves)
-                                                                         const {
-  // Default is to do nothing.
-}
-
 /// getFrameIndexOffset - Returns the displacement from the frame register to
 /// the stack frame of the specified index. This is the default implementation
 /// which is overridden for some targets.
diff --git a/contrib/llvm/lib/Target/TargetLoweringObjectFile.cpp b/contrib/llvm/lib/Target/TargetLoweringObjectFile.cpp
index 703431b3806e..56b7b69de0bd 100644
--- a/contrib/llvm/lib/Target/TargetLoweringObjectFile.cpp
+++ b/contrib/llvm/lib/Target/TargetLoweringObjectFile.cpp
@@ -35,34 +35,16 @@ using namespace llvm;
 //                              Generic Code
 //===----------------------------------------------------------------------===//
 
-TargetLoweringObjectFile::TargetLoweringObjectFile() :
-  Ctx(0),
-  TextSection(0),
-  DataSection(0),
-  BSSSection(0),
-  ReadOnlySection(0),
-  StaticCtorSection(0),
-  StaticDtorSection(0),
-  LSDASection(0),
-  CompactUnwindSection(0),
-  DwarfAbbrevSection(0),
-  DwarfInfoSection(0),
-  DwarfLineSection(0),
-  DwarfFrameSection(0),
-  DwarfPubNamesSection(0),
-  DwarfPubTypesSection(0),
-  DwarfDebugInlineSection(0),
-  DwarfStrSection(0),
-  DwarfLocSection(0),
-  DwarfARangesSection(0),
-  DwarfRangesSection(0),
-  DwarfMacroInfoSection(0),
-  TLSExtraDataSection(0),
-  CommDirectiveSupportsAlignment(true),
-  SupportsWeakOmittedEHFrame(true), 
-  IsFunctionEHFrameSymbolPrivate(true) {
+/// Initialize - this method must be called before any actual lowering is
+/// done.  This specifies the current context for codegen, and gives the
+/// lowering implementations a chance to set up their default sections.
+void TargetLoweringObjectFile::Initialize(MCContext &ctx,
+                                          const TargetMachine &TM) {
+  Ctx = &ctx;
+  InitMCObjectFileInfo(TM.getTargetTriple(),
+                       TM.getRelocationModel(), TM.getCodeModel(), *Ctx);
 }
-
+  
 TargetLoweringObjectFile::~TargetLoweringObjectFile() {
 }
 
@@ -93,7 +75,7 @@ static bool isSuitableForBSS(const GlobalVariable *GV) {
 /// known to have a type that is an array of 1/2/4 byte elements) ends with a
 /// nul value and contains no other nuls in it.
 static bool IsNullTerminatedString(const Constant *C) {
-  const ArrayType *ATy = cast<ArrayType>(C->getType());
+  ArrayType *ATy = cast<ArrayType>(C->getType());
 
   // First check: is we have constant array of i8 terminated with zero
   if (const ConstantArray *CVA = dyn_cast<ConstantArray>(C)) {
@@ -188,8 +170,8 @@ SectionKind TargetLoweringObjectFile::getKindForGlobal(const GlobalValue *GV,
         
       // If initializer is a null-terminated string, put it in a "cstring"
       // section of the right width.
-      if (const ArrayType *ATy = dyn_cast<ArrayType>(C->getType())) {
-        if (const IntegerType *ITy =
+      if (ArrayType *ATy = dyn_cast<ArrayType>(C->getType())) {
+        if (IntegerType *ITy =
               dyn_cast<IntegerType>(ATy->getElementType())) {
           if ((ITy->getBitWidth() == 8 || ITy->getBitWidth() == 16 ||
                ITy->getBitWidth() == 32) &&
@@ -341,20 +323,3 @@ getExprForDwarfReference(const MCSymbol *Sym, unsigned Encoding,
   }
   }
 }
-
-unsigned TargetLoweringObjectFile::getPersonalityEncoding() const {
-  return dwarf::DW_EH_PE_absptr;
-}
-
-unsigned TargetLoweringObjectFile::getLSDAEncoding() const {
-  return dwarf::DW_EH_PE_absptr;
-}
-
-unsigned TargetLoweringObjectFile::getFDEEncoding(bool CFI) const {
-  return dwarf::DW_EH_PE_absptr;
-}
-
-unsigned TargetLoweringObjectFile::getTTypeEncoding() const {
-  return dwarf::DW_EH_PE_absptr;
-}
-
diff --git a/contrib/llvm/lib/Target/TargetMachine.cpp b/contrib/llvm/lib/Target/TargetMachine.cpp
index 74a1f4e8da56..fe8a7cebd0a0 100644
--- a/contrib/llvm/lib/Target/TargetMachine.cpp
+++ b/contrib/llvm/lib/Target/TargetMachine.cpp
@@ -40,8 +40,6 @@ namespace llvm {
   bool JITExceptionHandling;
   bool JITEmitDebugInfo;
   bool JITEmitDebugInfoToDisk;
-  Reloc::Model RelocationModel;
-  CodeModel::Model CMModel;
   bool GuaranteedTailCallOpt;
   unsigned StackAlignmentOverride;
   bool RealignStack;
@@ -49,6 +47,7 @@ namespace llvm {
   bool StrongPHIElim;
   bool HasDivModLibcall;
   bool AsmVerbosityDefault(false);
+  bool EnableSegmentedStacks;
 }
 
 static cl::opt<bool, true>
@@ -143,38 +142,6 @@ EmitJitDebugInfoToDisk("jit-emit-debug-to-disk",
   cl::location(JITEmitDebugInfoToDisk),
   cl::init(false));
 
-static cl::opt<llvm::Reloc::Model, true>
-DefRelocationModel("relocation-model",
-  cl::desc("Choose relocation model"),
-  cl::location(RelocationModel),
-  cl::init(Reloc::Default),
-  cl::values(
-    clEnumValN(Reloc::Default, "default",
-               "Target default relocation model"),
-    clEnumValN(Reloc::Static, "static",
-               "Non-relocatable code"),
-    clEnumValN(Reloc::PIC_, "pic",
-               "Fully relocatable, position independent code"),
-    clEnumValN(Reloc::DynamicNoPIC, "dynamic-no-pic",
-               "Relocatable external references, non-relocatable code"),
-    clEnumValEnd));
-static cl::opt<llvm::CodeModel::Model, true>
-DefCodeModel("code-model",
-  cl::desc("Choose code model"),
-  cl::location(CMModel),
-  cl::init(CodeModel::Default),
-  cl::values(
-    clEnumValN(CodeModel::Default, "default",
-               "Target default code model"),
-    clEnumValN(CodeModel::Small, "small",
-               "Small code model"),
-    clEnumValN(CodeModel::Kernel, "kernel",
-               "Kernel code model"),
-    clEnumValN(CodeModel::Medium, "medium",
-               "Medium code model"),
-    clEnumValN(CodeModel::Large, "large",
-               "Large code model"),
-    clEnumValEnd));
 static cl::opt<bool, true>
 EnableGuaranteedTailCallOpt("tailcallopt",
   cl::desc("Turn fastcc calls into tail calls by (potentially) changing ABI."),
@@ -212,13 +179,20 @@ static cl::opt<bool>
 FunctionSections("ffunction-sections",
   cl::desc("Emit functions into separate sections"),
   cl::init(false));
+static cl::opt<bool, true>
+SegmentedStacks("segmented-stacks",
+  cl::desc("Use segmented stacks if possible."),
+  cl::location(EnableSegmentedStacks),
+  cl::init(false));
+                         
 //---------------------------------------------------------------------------
 // TargetMachine Class
 //
 
 TargetMachine::TargetMachine(const Target &T,
                              StringRef TT, StringRef CPU, StringRef FS)
-  : TheTarget(T), TargetTriple(TT), TargetCPU(CPU), TargetFS(FS), AsmInfo(0),
+  : TheTarget(T), TargetTriple(TT), TargetCPU(CPU), TargetFS(FS),
+    CodeGenInfo(0), AsmInfo(0),
     MCRelaxAll(false),
     MCNoExecStack(false),
     MCSaveTempLabels(false),
@@ -231,29 +205,24 @@ TargetMachine::TargetMachine(const Target &T,
 }
 
 TargetMachine::~TargetMachine() {
+  delete CodeGenInfo;
   delete AsmInfo;
 }
 
 /// getRelocationModel - Returns the code generation relocation model. The
 /// choices are static, PIC, and dynamic-no-pic, and target default.
-Reloc::Model TargetMachine::getRelocationModel() {
-  return RelocationModel;
-}
-
-/// setRelocationModel - Sets the code generation relocation model.
-void TargetMachine::setRelocationModel(Reloc::Model Model) {
-  RelocationModel = Model;
+Reloc::Model TargetMachine::getRelocationModel() const {
+  if (!CodeGenInfo)
+    return Reloc::Default;
+  return CodeGenInfo->getRelocationModel();
 }
 
 /// getCodeModel - Returns the code model. The choices are small, kernel,
 /// medium, large, and target default.
-CodeModel::Model TargetMachine::getCodeModel() {
-  return CMModel;
-}
-
-/// setCodeModel - Sets the code model.
-void TargetMachine::setCodeModel(CodeModel::Model Model) {
-  CMModel = Model;
+CodeModel::Model TargetMachine::getCodeModel() const {
+  if (!CodeGenInfo)
+    return CodeModel::Default;
+  return CodeGenInfo->getCodeModel();
 }
 
 bool TargetMachine::getAsmVerbosityDefault() {
diff --git a/contrib/llvm/lib/Target/TargetRegisterInfo.cpp b/contrib/llvm/lib/Target/TargetRegisterInfo.cpp
index 90a8f8d8fdcc..67239b830eb5 100644
--- a/contrib/llvm/lib/Target/TargetRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/TargetRegisterInfo.cpp
@@ -98,44 +98,25 @@ BitVector TargetRegisterInfo::getAllocatableSet(const MachineFunction &MF,
 }
 
 const TargetRegisterClass *
-llvm::getCommonSubClass(const TargetRegisterClass *A,
-                        const TargetRegisterClass *B) {
-  // First take care of the trivial cases
+TargetRegisterInfo::getCommonSubClass(const TargetRegisterClass *A,
+                                      const TargetRegisterClass *B) const {
+  // First take care of the trivial cases.
   if (A == B)
     return A;
   if (!A || !B)
     return 0;
 
-  // If B is a subclass of A, it will be handled in the loop below
-  if (B->hasSubClass(A))
-    return A;
+  // Register classes are ordered topologically, so the largest common
+  // sub-class it the common sub-class with the smallest ID.
+  const unsigned *SubA = A->getSubClassMask();
+  const unsigned *SubB = B->getSubClassMask();
 
-  const TargetRegisterClass *Best = 0;
-  for (TargetRegisterClass::sc_iterator I = A->subclasses_begin();
-       const TargetRegisterClass *X = *I; ++I) {
-    if (X == B)
-      return B;                 // B is a subclass of A
+  // We could start the search from max(A.ID, B.ID), but we are only going to
+  // execute 2-3 iterations anyway.
+  for (unsigned Base = 0, BaseE = getNumRegClasses(); Base < BaseE; Base += 32)
+    if (unsigned Common = *SubA++ & *SubB++)
+      return getRegClass(Base + CountTrailingZeros_32(Common));
 
-    // X must be a common subclass of A and B
-    if (!B->hasSubClass(X))
-      continue;
-
-    // A superclass is definitely better.
-    if (!Best || Best->hasSuperClass(X)) {
-      Best = X;
-      continue;
-    }
-
-    // A subclass is definitely worse
-    if (Best->hasSubClass(X))
-      continue;
-
-    // Best and *I have no super/sub class relation - pick the larger class, or
-    // the smaller spill size.
-    int nb = std::distance(Best->begin(), Best->end());
-    int ni = std::distance(X->begin(), X->end());
-    if (ni>nb || (ni==nb && X->getSize() < Best->getSize()))
-      Best = X;
-  }
-  return Best;
+  // No common sub-class exists.
+  return NULL;
 }
diff --git a/contrib/llvm/lib/Target/X86/AsmParser/X86AsmLexer.cpp b/contrib/llvm/lib/Target/X86/AsmParser/X86AsmLexer.cpp
index ec73087a3305..1eaccff58a9d 100644
--- a/contrib/llvm/lib/Target/X86/AsmParser/X86AsmLexer.cpp
+++ b/contrib/llvm/lib/Target/X86/AsmParser/X86AsmLexer.cpp
@@ -7,20 +7,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Target/TargetAsmLexer.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "MCTargetDesc/X86BaseInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
-#include "X86.h"
+#include "llvm/MC/MCTargetAsmLexer.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace llvm;
 
 namespace {
   
-class X86AsmLexer : public TargetAsmLexer {
+class X86AsmLexer : public MCTargetAsmLexer {
   const MCAsmInfo &AsmInfo;
   
   bool tentativeIsValid;
@@ -60,8 +60,8 @@ class X86AsmLexer : public TargetAsmLexer {
     }
   }
 public:
-  X86AsmLexer(const Target &T, const MCAsmInfo &MAI)
-    : TargetAsmLexer(T), AsmInfo(MAI), tentativeIsValid(false) {
+  X86AsmLexer(const Target &T, const MCRegisterInfo &MRI, const MCAsmInfo &MAI)
+    : MCTargetAsmLexer(T), AsmInfo(MAI), tentativeIsValid(false) {
   }
 };
 
@@ -160,6 +160,6 @@ AsmToken X86AsmLexer::LexTokenIntel() {
 }
 
 extern "C" void LLVMInitializeX86AsmLexer() {
-  RegisterAsmLexer<X86AsmLexer> X(TheX86_32Target);
-  RegisterAsmLexer<X86AsmLexer> Y(TheX86_64Target);
+  RegisterMCAsmLexer<X86AsmLexer> X(TheX86_32Target);
+  RegisterMCAsmLexer<X86AsmLexer> Y(TheX86_64Target);
 }
diff --git a/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index d45dd352fbc4..cb4f15ffed3e 100644
--- a/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -7,14 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmParser.h"
-#include "X86.h"
-#include "X86Subtarget.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmParser.h"
+#include "MCTargetDesc/X86BaseInfo.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
@@ -26,6 +24,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -33,7 +32,7 @@ using namespace llvm;
 namespace {
 struct X86Operand;
 
-class X86ATTAsmParser : public TargetAsmParser {
+class X86ATTAsmParser : public MCTargetAsmParser {
   MCSubtargetInfo &STI;
   MCAsmParser &Parser;
 
@@ -48,6 +47,7 @@ class X86ATTAsmParser : public TargetAsmParser {
   X86Operand *ParseMemOperand(unsigned SegReg, SMLoc StartLoc);
 
   bool ParseDirectiveWord(unsigned Size, SMLoc L);
+  bool ParseDirectiveCode(StringRef IDVal, SMLoc L);
 
   bool MatchAndEmitInstruction(SMLoc IDLoc,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands,
@@ -65,6 +65,10 @@ class X86ATTAsmParser : public TargetAsmParser {
     // FIXME: Can tablegen auto-generate this?
     return (STI.getFeatureBits() & X86::Mode64Bit) != 0;
   }
+  void SwitchMode() {
+    unsigned FB = ComputeAvailableFeatures(STI.ToggleFeature(X86::Mode64Bit));
+    setAvailableFeatures(FB);
+  }
 
   /// @name Auto-generated Matcher Functions
   /// {
@@ -76,7 +80,7 @@ class X86ATTAsmParser : public TargetAsmParser {
 
 public:
   X86ATTAsmParser(MCSubtargetInfo &sti, MCAsmParser &parser)
-    : TargetAsmParser(), STI(sti), Parser(parser) {
+    : MCTargetAsmParser(), STI(sti), Parser(parser) {
 
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
@@ -223,6 +227,21 @@ struct X86Operand : public MCParsedAsmOperand {
             (0x00000000FFFFFF80ULL <= Value && Value <= 0x00000000FFFFFFFFULL)||
             (0xFFFFFFFFFFFFFF80ULL <= Value && Value <= 0xFFFFFFFFFFFFFFFFULL));
   }
+  bool isImmZExtu32u8() const {
+    if (!isImm())
+      return false;
+
+    // If this isn't a constant expr, just assume it fits and let relaxation
+    // handle it.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE)
+      return true;
+
+    // Otherwise, check the value is in a range that makes sense for this
+    // extension.
+    uint64_t Value = CE->getValue();
+    return (Value <= 0x00000000000000FFULL);
+  }
   bool isImmSExti64i8() const {
     if (!isImm())
       return false;
@@ -382,19 +401,25 @@ bool X86ATTAsmParser::ParseRegister(unsigned &RegNo,
   if (Tok.isNot(AsmToken::Identifier))
     return Error(Tok.getLoc(), "invalid register name");
 
-  // FIXME: Validate register for the current architecture; we have to do
-  // validation later, so maybe there is no need for this here.
   RegNo = MatchRegisterName(Tok.getString());
 
   // If the match failed, try the register name as lowercase.
   if (RegNo == 0)
     RegNo = MatchRegisterName(LowercaseString(Tok.getString()));
 
-  // FIXME: This should be done using Requires<In32BitMode> and
-  // Requires<In64BitMode> so "eiz" usage in 64-bit instructions
-  // can be also checked.
-  if (RegNo == X86::RIZ && !is64BitMode())
-    return Error(Tok.getLoc(), "riz register in 64-bit mode only");
+  if (!is64BitMode()) {
+    // FIXME: This should be done using Requires<In32BitMode> and
+    // Requires<In64BitMode> so "eiz" usage in 64-bit instructions can be also
+    // checked.
+    // FIXME: Check AH, CH, DH, BH cannot be used in an instruction requiring a
+    // REX prefix.
+    if (RegNo == X86::RIZ ||
+        X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo) ||
+        X86II::isX86_64NonExtLowByteReg(RegNo) ||
+        X86II::isX86_64ExtendedReg(RegNo))
+      return Error(Tok.getLoc(), "register %"
+                   + Tok.getString() + " is only available in 64-bit mode");
+  }
 
   // Parse "%st" as "%st(0)" and "%st(1)", which is multiple tokens.
   if (RegNo == 0 && (Tok.getString() == "st" || Tok.getString() == "ST")) {
@@ -472,7 +497,7 @@ X86Operand *X86ATTAsmParser::ParseOperand() {
     SMLoc Start, End;
     if (ParseRegister(RegNo, Start, End)) return 0;
     if (RegNo == X86::EIZ || RegNo == X86::RIZ) {
-      Error(Start, "eiz and riz can only be used as index registers");
+      Error(Start, "%eiz and %riz can only be used as index registers");
       return 0;
     }
 
@@ -956,6 +981,7 @@ MatchAndEmitInstruction(SMLoc IDLoc,
 
   // First, try a direct match.
   switch (MatchInstructionImpl(Operands, Inst, OrigErrorInfo)) {
+  default: break;
   case Match_Success:
     Out.EmitInstruction(Inst);
     return false;
@@ -994,7 +1020,7 @@ MatchAndEmitInstruction(SMLoc IDLoc,
   // Check for the various suffix matches.
   Tmp[Base.size()] = Suffixes[0];
   unsigned ErrorInfoIgnore;
-  MatchResultTy Match1, Match2, Match3, Match4;
+  unsigned Match1, Match2, Match3, Match4;
   
   Match1 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore);
   Tmp[Base.size()] = Suffixes[1];
@@ -1096,6 +1122,8 @@ bool X86ATTAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getIdentifier();
   if (IDVal == ".word")
     return ParseDirectiveWord(2, DirectiveID.getLoc());
+  else if (IDVal.startswith(".code"))
+    return ParseDirectiveCode(IDVal, DirectiveID.getLoc());
   return true;
 }
 
@@ -1124,15 +1152,35 @@ bool X86ATTAsmParser::ParseDirectiveWord(unsigned Size, SMLoc L) {
   return false;
 }
 
+/// ParseDirectiveCode
+///  ::= .code32 | .code64
+bool X86ATTAsmParser::ParseDirectiveCode(StringRef IDVal, SMLoc L) {
+  if (IDVal == ".code32") {
+    Parser.Lex();
+    if (is64BitMode()) {
+      SwitchMode();
+      getParser().getStreamer().EmitAssemblerFlag(MCAF_Code32);
+    }
+  } else if (IDVal == ".code64") {
+    Parser.Lex();
+    if (!is64BitMode()) {
+      SwitchMode();
+      getParser().getStreamer().EmitAssemblerFlag(MCAF_Code64);
+    }
+  } else {
+    return Error(L, "unexpected directive " + IDVal);
+  }
 
+  return false;
+}
 
 
 extern "C" void LLVMInitializeX86AsmLexer();
 
 // Force static initialization.
 extern "C" void LLVMInitializeX86AsmParser() {
-  RegisterAsmParser<X86ATTAsmParser> X(TheX86_32Target);
-  RegisterAsmParser<X86ATTAsmParser> Y(TheX86_64Target);
+  RegisterMCAsmParser<X86ATTAsmParser> X(TheX86_32Target);
+  RegisterMCAsmParser<X86ATTAsmParser> Y(TheX86_64Target);
   LLVMInitializeX86AsmLexer();
 }
 
diff --git a/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
index 4a0d2ec727a9..3aacb20e73df 100644
--- a/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -21,13 +21,16 @@
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MemoryObject.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define GET_REGINFO_ENUM
 #include "X86GenRegisterInfo.inc"
+#define GET_INSTRINFO_ENUM
+#include "X86GenInstrInfo.inc"
 #include "X86GenEDInfo.inc"
 
 using namespace llvm;
@@ -64,8 +67,8 @@ extern Target TheX86_32Target, TheX86_64Target;
 static bool translateInstruction(MCInst &target,
                                 InternalInstruction &source);
 
-X86GenericDisassembler::X86GenericDisassembler(DisassemblerMode mode) :
-    MCDisassembler(),
+X86GenericDisassembler::X86GenericDisassembler(const MCSubtargetInfo &STI, DisassemblerMode mode) :
+    MCDisassembler(STI),
     fMode(mode) {
 }
 
@@ -106,28 +109,34 @@ static void logger(void* arg, const char* log) {
 // Public interface for the disassembler
 //
 
-bool X86GenericDisassembler::getInstruction(MCInst &instr,
-                                            uint64_t &size,
-                                            const MemoryObject &region,
-                                            uint64_t address,
-                                            raw_ostream &vStream) const {
+MCDisassembler::DecodeStatus
+X86GenericDisassembler::getInstruction(MCInst &instr,
+                                       uint64_t &size,
+                                       const MemoryObject &region,
+                                       uint64_t address,
+                                       raw_ostream &vStream,
+                                       raw_ostream &cStream) const {
   InternalInstruction internalInstr;
+
+  dlog_t loggerFn = logger;
+  if (&vStream == &nulls())
+    loggerFn = 0; // Disable logging completely if it's going to nulls().
   
   int ret = decodeInstruction(&internalInstr,
                               regionReader,
                               (void*)&region,
-                              logger,
+                              loggerFn,
                               (void*)&vStream,
                               address,
                               fMode);
 
   if (ret) {
     size = internalInstr.readerCursor - address;
-    return false;
+    return Fail;
   }
   else {
     size = internalInstr.length;
-    return !translateInstruction(instr, internalInstr);
+    return (!translateInstruction(instr, internalInstr)) ? Success : Fail;
   }
 }
 
@@ -183,8 +192,46 @@ static void translateImmediate(MCInst &mcInst, uint64_t immediate,
       break;
     }
   }
+  // By default sign-extend all X86 immediates based on their encoding.
+  else if (type == TYPE_IMM8 || type == TYPE_IMM16 || type == TYPE_IMM32 ||
+           type == TYPE_IMM64) {
+    uint32_t Opcode = mcInst.getOpcode();
+    switch (operand.encoding) {
+    default:
+      break;
+    case ENCODING_IB:
+      // Special case those X86 instructions that use the imm8 as a set of
+      // bits, bit count, etc. and are not sign-extend.
+      if (Opcode != X86::BLENDPSrri && Opcode != X86::BLENDPDrri &&
+	  Opcode != X86::PBLENDWrri && Opcode != X86::MPSADBWrri &&
+	  Opcode != X86::DPPSrri && Opcode != X86::DPPDrri &&
+	  Opcode != X86::INSERTPSrr && Opcode != X86::VBLENDPSYrri &&
+	  Opcode != X86::VBLENDPSYrmi && Opcode != X86::VBLENDPDYrri &&
+	  Opcode != X86::VBLENDPDYrmi && Opcode != X86::VPBLENDWrri &&
+	  Opcode != X86::VMPSADBWrri && Opcode != X86::VDPPSYrri &&
+	  Opcode != X86::VDPPSYrmi && Opcode != X86::VDPPDrri &&
+	  Opcode != X86::VINSERTPSrr)
+	type = TYPE_MOFFS8;
+      break;
+    case ENCODING_IW:
+      type = TYPE_MOFFS16;
+      break;
+    case ENCODING_ID:
+      type = TYPE_MOFFS32;
+      break;
+    case ENCODING_IO:
+      type = TYPE_MOFFS64;
+      break;
+    }
+  }
 
   switch (type) {
+  case TYPE_XMM128:
+    mcInst.addOperand(MCOperand::CreateReg(X86::XMM0 + (immediate >> 4)));
+    return;
+  case TYPE_XMM256:
+    mcInst.addOperand(MCOperand::CreateReg(X86::YMM0 + (immediate >> 4)));
+    return;
   case TYPE_MOFFS8:
   case TYPE_REL8:
     if(immediate & 0x80)
@@ -543,12 +590,12 @@ static bool translateInstruction(MCInst &mcInst,
   return false;
 }
 
-static MCDisassembler *createX86_32Disassembler(const Target &T) {
-  return new X86Disassembler::X86_32Disassembler;
+static MCDisassembler *createX86_32Disassembler(const Target &T, const MCSubtargetInfo &STI) {
+  return new X86Disassembler::X86_32Disassembler(STI);
 }
 
-static MCDisassembler *createX86_64Disassembler(const Target &T) {
-  return new X86Disassembler::X86_64Disassembler;
+static MCDisassembler *createX86_64Disassembler(const Target &T, const MCSubtargetInfo &STI) {
+  return new X86Disassembler::X86_64Disassembler(STI);
 }
 
 extern "C" void LLVMInitializeX86Disassembler() { 
diff --git a/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.h b/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.h
index 550cf9d40de2..6ac9a0ff1019 100644
--- a/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.h
+++ b/contrib/llvm/lib/Target/X86/Disassembler/X86Disassembler.h
@@ -92,6 +92,7 @@ struct InternalInstruction;
 namespace llvm {
   
 class MCInst;
+class MCSubtargetInfo;
 class MemoryObject;
 class raw_ostream;
 
@@ -107,16 +108,17 @@ class X86GenericDisassembler : public MCDisassembler {
   /// Constructor     - Initializes the disassembler.
   ///
   /// @param mode     - The X86 architecture mode to decode for.
-  X86GenericDisassembler(DisassemblerMode mode);
+  X86GenericDisassembler(const MCSubtargetInfo &STI, DisassemblerMode mode);
 public:
   ~X86GenericDisassembler();
 
   /// getInstruction - See MCDisassembler.
-  bool getInstruction(MCInst &instr,
-                      uint64_t &size,
-                      const MemoryObject &region,
-                      uint64_t address,
-                      raw_ostream &vStream) const;
+  DecodeStatus getInstruction(MCInst &instr,
+                              uint64_t &size,
+                              const MemoryObject &region,
+                              uint64_t address,
+                              raw_ostream &vStream,
+                              raw_ostream &cStream) const;
 
   /// getEDInfo - See MCDisassembler.
   EDInstInfo *getEDInfo() const;
@@ -127,24 +129,24 @@ class X86GenericDisassembler : public MCDisassembler {
 /// X86_16Disassembler - 16-bit X86 disassembler.
 class X86_16Disassembler : public X86GenericDisassembler {
 public:
-  X86_16Disassembler() :
-    X86GenericDisassembler(MODE_16BIT) {
+  X86_16Disassembler(const MCSubtargetInfo &STI) :
+    X86GenericDisassembler(STI, MODE_16BIT) {
   }
 };  
 
 /// X86_16Disassembler - 32-bit X86 disassembler.
 class X86_32Disassembler : public X86GenericDisassembler {
 public:
-  X86_32Disassembler() :
-    X86GenericDisassembler(MODE_32BIT) {
+  X86_32Disassembler(const MCSubtargetInfo &STI) :
+    X86GenericDisassembler(STI, MODE_32BIT) {
   }
 };
 
 /// X86_16Disassembler - 64-bit X86 disassembler.
 class X86_64Disassembler : public X86GenericDisassembler {
 public:
-  X86_64Disassembler() :
-    X86GenericDisassembler(MODE_64BIT) {
+  X86_64Disassembler(const MCSubtargetInfo &STI) :
+    X86GenericDisassembler(STI, MODE_64BIT) {
   }
 };
 
diff --git a/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c b/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
index de1610ba3d66..f9b0fe5d51b9 100644
--- a/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
+++ b/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
@@ -58,8 +58,8 @@ static InstructionContext contextForAttrs(uint8_t attrMask) {
  * @return            - TRUE if the ModR/M byte is required, FALSE otherwise.
  */
 static int modRMRequired(OpcodeType type,
-                                InstructionContext insnContext,
-                                uint8_t opcode) {
+                         InstructionContext insnContext,
+                         uint8_t opcode) {
   const struct ContextDecision* decision = 0;
   
   switch (type) {
@@ -391,7 +391,7 @@ static int readPrefixes(struct InternalInstruction* insn) {
       return -1;
     }
     
-    if (insn->mode == MODE_64BIT || byte1 & 0x8) {
+    if (insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) {
       insn->vexSize = 3;
       insn->necessaryPrefixLocation = insn->readerCursor - 1;
     }
@@ -406,12 +406,14 @@ static int readPrefixes(struct InternalInstruction* insn) {
       consumeByte(insn, &insn->vexPrefix[2]);
 
       /* We simulate the REX prefix for simplicity's sake */
-    
-      insn->rexPrefix = 0x40 
-                      | (wFromVEX3of3(insn->vexPrefix[2]) << 3)
-                      | (rFromVEX2of3(insn->vexPrefix[1]) << 2)
-                      | (xFromVEX2of3(insn->vexPrefix[1]) << 1)
-                      | (bFromVEX2of3(insn->vexPrefix[1]) << 0);
+   
+      if (insn->mode == MODE_64BIT) {
+        insn->rexPrefix = 0x40 
+                        | (wFromVEX3of3(insn->vexPrefix[2]) << 3)
+                        | (rFromVEX2of3(insn->vexPrefix[1]) << 2)
+                        | (xFromVEX2of3(insn->vexPrefix[1]) << 1)
+                        | (bFromVEX2of3(insn->vexPrefix[1]) << 0);
+      }
     
       switch (ppFromVEX3of3(insn->vexPrefix[2]))
       {
@@ -433,7 +435,7 @@ static int readPrefixes(struct InternalInstruction* insn) {
       return -1;
     }
       
-    if (insn->mode == MODE_64BIT || byte1 & 0x8) {
+    if (insn->mode == MODE_64BIT || (byte1 & 0xc0) == 0xc0) {
       insn->vexSize = 2;
     }
     else {
@@ -444,8 +446,10 @@ static int readPrefixes(struct InternalInstruction* insn) {
       insn->vexPrefix[0] = byte;
       consumeByte(insn, &insn->vexPrefix[1]);
         
-      insn->rexPrefix = 0x40 
-                      | (rFromVEX2of2(insn->vexPrefix[1]) << 2);
+      if (insn->mode == MODE_64BIT) {
+        insn->rexPrefix = 0x40 
+                        | (rFromVEX2of2(insn->vexPrefix[1]) << 2);
+      }
         
       switch (ppFromVEX2of2(insn->vexPrefix[1]))
       {
@@ -699,34 +703,6 @@ static BOOL is16BitEquvalent(const char* orig, const char* equiv) {
   }
 }
 
-/*
- * is64BitEquivalent - Determines whether two instruction names refer to
- * equivalent instructions but one is 64-bit whereas the other is not.
- *
- * @param orig  - The instruction that is not 64-bit
- * @param equiv - The instruction that is 64-bit
- */
-static BOOL is64BitEquivalent(const char* orig, const char* equiv) {
-  off_t i;
-  
-  for (i = 0;; i++) {
-    if (orig[i] == '\0' && equiv[i] == '\0')
-      return TRUE;
-    if (orig[i] == '\0' || equiv[i] == '\0')
-      return FALSE;
-    if (orig[i] != equiv[i]) {
-      if ((orig[i] == 'W' || orig[i] == 'L') && equiv[i] == 'Q')
-        continue;
-      if ((orig[i] == '1' || orig[i] == '3') && equiv[i] == '6')
-        continue;
-      if ((orig[i] == '6' || orig[i] == '2') && equiv[i] == '4')
-        continue;
-      return FALSE;
-    }
-  }
-}
-
-
 /*
  * getID - Determines the ID of an instruction, consuming the ModR/M byte as 
  *   appropriate for extended and escape opcodes.  Determines the attributes and 
@@ -763,8 +739,6 @@ static int getID(struct InternalInstruction* insn) {
         break;
       }
     
-      if (wFromVEX3of3(insn->vexPrefix[2]))
-        attrMask |= ATTR_REXW;
       if (lFromVEX3of3(insn->vexPrefix[2]))
         attrMask |= ATTR_VEXL;
     }
@@ -789,63 +763,55 @@ static int getID(struct InternalInstruction* insn) {
     }
   }
   else {
-    if (insn->rexPrefix & 0x08)
-      attrMask |= ATTR_REXW;
-  
     if (isPrefixAtLocation(insn, 0x66, insn->necessaryPrefixLocation))
       attrMask |= ATTR_OPSIZE;
     else if (isPrefixAtLocation(insn, 0xf3, insn->necessaryPrefixLocation))
       attrMask |= ATTR_XS;
     else if (isPrefixAtLocation(insn, 0xf2, insn->necessaryPrefixLocation))
       attrMask |= ATTR_XD;
-    
   }
 
+  if (insn->rexPrefix & 0x08)
+    attrMask |= ATTR_REXW;
+  
   if (getIDWithAttrMask(&instructionID, insn, attrMask))
     return -1;
   
   /* The following clauses compensate for limitations of the tables. */
   
-  if ((attrMask & ATTR_XD) && (attrMask & ATTR_REXW)) {
+  if ((attrMask & ATTR_VEXL) && (attrMask & ATTR_REXW)) {
     /*
-     * Although for SSE instructions it is usually necessary to treat REX.W+F2
-     * as F2 for decode (in the absence of a 64BIT_REXW_XD category) there is
-     * an occasional instruction where F2 is incidental and REX.W is the more
-     * significant.  If the decoded instruction is 32-bit and adding REX.W
-     * instead of F2 changes a 32 to a 64, we adopt the new encoding.
+     * Some VEX instructions ignore the L-bit, but use the W-bit. Normally L-bit
+     * has precedence since there are no L-bit with W-bit entries in the tables.
+     * So if the L-bit isn't significant we should use the W-bit instead.
      */
-    
+
     const struct InstructionSpecifier *spec;
-    uint16_t instructionIDWithREXw;
-    const struct InstructionSpecifier *specWithREXw;
-    
+    uint16_t instructionIDWithWBit;
+    const struct InstructionSpecifier *specWithWBit;
+
     spec = specifierForUID(instructionID);
-    
-    if (getIDWithAttrMask(&instructionIDWithREXw,
+
+    if (getIDWithAttrMask(&instructionIDWithWBit,
                           insn,
-                          attrMask & (~ATTR_XD))) {
-      /*
-       * Decoding with REX.w would yield nothing; give up and return original
-       * decode.
-       */
-      
+                          (attrMask & (~ATTR_VEXL)) | ATTR_REXW)) {
       insn->instructionID = instructionID;
       insn->spec = spec;
       return 0;
     }
-    
-    specWithREXw = specifierForUID(instructionIDWithREXw);
-    
-    if (is64BitEquivalent(spec->name, specWithREXw->name)) {
-      insn->instructionID = instructionIDWithREXw;
-      insn->spec = specWithREXw;
+
+    specWithWBit = specifierForUID(instructionIDWithWBit);
+
+    if (instructionID != instructionIDWithWBit) {
+      insn->instructionID = instructionIDWithWBit;
+      insn->spec = specWithWBit;
     } else {
       insn->instructionID = instructionID;
       insn->spec = spec;
     }
     return 0;
   }
-  
+
   if (insn->prefixPresent[0x66] && !(attrMask & ATTR_OPSIZE)) {
     /*
      * The instruction tables make no distinction between instructions that
@@ -885,6 +851,43 @@ static int getID(struct InternalInstruction* insn) {
     }
     return 0;
   }
+
+  if (insn->opcodeType == ONEBYTE && insn->opcode == 0x90 &&
+      insn->rexPrefix & 0x01) {
+    /*
+     * NOOP shouldn't decode as NOOP if REX.b is set. Instead
+     * it should decode as XCHG %r8, %eax.
+     */
+
+    const struct InstructionSpecifier *spec;
+    uint16_t instructionIDWithNewOpcode;
+    const struct InstructionSpecifier *specWithNewOpcode;
+
+    spec = specifierForUID(instructionID);
+    
+    /* Borrow opcode from one of the other XCHGar opcodes */
+    insn->opcode = 0x91;
+   
+    if (getIDWithAttrMask(&instructionIDWithNewOpcode,
+                          insn,
+                          attrMask)) {
+      insn->opcode = 0x90;
+
+      insn->instructionID = instructionID;
+      insn->spec = spec;
+      return 0;
+    }
+
+    specWithNewOpcode = specifierForUID(instructionIDWithNewOpcode);
+
+    /* Change back */
+    insn->opcode = 0x90;
+
+    insn->instructionID = instructionIDWithNewOpcode;
+    insn->spec = specWithNewOpcode;
+
+    return 0;
+  }
   
   insn->instructionID = instructionID;
   insn->spec = specifierForUID(insn->instructionID);
@@ -1434,11 +1437,10 @@ static int readImmediate(struct InternalInstruction* insn, uint8_t size) {
 }
 
 /*
- * readVVVV - Consumes an immediate operand from an instruction, given the
- *   desired operand size.
+ * readVVVV - Consumes vvvv from an instruction if it has a VEX prefix.
  *
  * @param insn  - The instruction whose operand is to be read.
- * @return      - 0 if the immediate was successfully consumed; nonzero
+ * @return      - 0 if the vvvv was successfully consumed; nonzero
  *                otherwise.
  */
 static int readVVVV(struct InternalInstruction* insn) {
@@ -1451,6 +1453,9 @@ static int readVVVV(struct InternalInstruction* insn) {
   else
     return -1;
 
+  if (insn->mode != MODE_64BIT)
+    insn->vvvv &= 0x7;
+
   return 0;
 }
 
@@ -1463,8 +1468,14 @@ static int readVVVV(struct InternalInstruction* insn) {
  */
 static int readOperands(struct InternalInstruction* insn) {
   int index;
+  int hasVVVV, needVVVV;
   
   dbgprintf(insn, "readOperands()");
+
+  /* If non-zero vvvv specified, need to make sure one of the operands
+     uses it. */
+  hasVVVV = !readVVVV(insn);
+  needVVVV = hasVVVV && (insn->vvvv != 0);
   
   for (index = 0; index < X86_MAX_OPERANDS; ++index) {
     switch (insn->spec->operands[index].encoding) {
@@ -1537,7 +1548,8 @@ static int readOperands(struct InternalInstruction* insn) {
         return -1;
       break;
     case ENCODING_VVVV:
-      if (readVVVV(insn))
+      needVVVV = 0; /* Mark that we have found a VVVV operand. */
+      if (!hasVVVV)
         return -1;
       if (fixupReg(insn, &insn->spec->operands[index]))
         return -1;
@@ -1549,6 +1561,9 @@ static int readOperands(struct InternalInstruction* insn) {
       return -1;
     }
   }
+
+  /* If we didn't find ENCODING_VVVV operand, but non-zero vvvv present, fail */
+  if (needVVVV) return -1;
   
   return 0;
 }
diff --git a/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h b/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
index 70315ed572b4..8b7933545a56 100644
--- a/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
+++ b/contrib/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
@@ -81,12 +81,18 @@ enum attributeBits {
                                         "but not the operands")                \
   ENUM_ENTRY(IC_XS,                 2,  "may say something about the opcode "  \
                                         "but not the operands")                \
+  ENUM_ENTRY(IC_XD_OPSIZE,          3,  "requires an OPSIZE prefix, so "       \
+                                        "operands change width")               \
+  ENUM_ENTRY(IC_XS_OPSIZE,          3,  "requires an OPSIZE prefix, so "       \
+                                        "operands change width")               \
   ENUM_ENTRY(IC_64BIT_REXW,         4,  "requires a REX.W prefix, so operands "\
                                         "change width; overrides IC_OPSIZE")   \
   ENUM_ENTRY(IC_64BIT_OPSIZE,       3,  "Just as meaningful as IC_OPSIZE")     \
   ENUM_ENTRY(IC_64BIT_XD,           5,  "XD instructions are SSE; REX.W is "   \
                                         "secondary")                           \
   ENUM_ENTRY(IC_64BIT_XS,           5,  "Just as meaningful as IC_64BIT_XD")   \
+  ENUM_ENTRY(IC_64BIT_XD_OPSIZE,    3,  "Just as meaningful as IC_XD_OPSIZE")  \
+  ENUM_ENTRY(IC_64BIT_XS_OPSIZE,    3,  "Just as meaningful as IC_XS_OPSIZE")  \
   ENUM_ENTRY(IC_64BIT_REXW_XS,      6,  "OPSIZE could mean a different "       \
                                         "opcode")                              \
   ENUM_ENTRY(IC_64BIT_REXW_XD,      6,  "Just as meaningful as "               \
@@ -104,7 +110,7 @@ enum attributeBits {
   ENUM_ENTRY(IC_VEX_W_OPSIZE,       4,  "requires VEX, W, and OpSize")         \
   ENUM_ENTRY(IC_VEX_L,              3,  "requires VEX and the L prefix")       \
   ENUM_ENTRY(IC_VEX_L_XS,           4,  "requires VEX and the L and XS prefix")\
-  ENUM_ENTRY(IC_VEX_L_XD,           4,  "requires VEX and the L and XS prefix")\
+  ENUM_ENTRY(IC_VEX_L_XD,           4,  "requires VEX and the L and XD prefix")\
   ENUM_ENTRY(IC_VEX_L_OPSIZE,       4,  "requires VEX, L, and OpSize")
 
 
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/CMakeLists.txt b/contrib/llvm/lib/Target/X86/InstPrinter/CMakeLists.txt
deleted file mode 100644
index 033973eeeff9..000000000000
--- a/contrib/llvm/lib/Target/X86/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMX86AsmPrinter
-  X86ATTInstPrinter.cpp
-  X86IntelInstPrinter.cpp
-  X86InstComments.cpp
-  )
-add_dependencies(LLVMX86AsmPrinter X86CodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/Makefile b/contrib/llvm/lib/Target/X86/InstPrinter/Makefile
deleted file mode 100644
index c82aa330a20c..000000000000
--- a/contrib/llvm/lib/Target/X86/InstPrinter/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/X86/AsmPrinter/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMX86AsmPrinter
-
-# Hack: we need to include 'main' x86 target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp b/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
index c37d8797b39c..029d491260f6 100644
--- a/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
+++ b/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
@@ -39,14 +39,17 @@ void X86ATTInstPrinter::printRegName(raw_ostream &OS,
   OS << '%' << getRegisterName(RegNo);
 }
 
-void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS) {
+void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
+                                  StringRef Annot) {
   // Try to print any aliases first.
   if (!printAliasInstr(MI, OS))
     printInstruction(MI, OS);
   
   // If verbose assembly is enabled, we can print some informative comments.
-  if (CommentStream)
+  if (CommentStream) {
+    printAnnotation(OS, Annot);
     EmitAnyX86InstComments(MI, *CommentStream, getRegisterName);
+  }
 }
 
 StringRef X86ATTInstPrinter::getOpcodeName(unsigned Opcode) const {
@@ -90,7 +93,8 @@ void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   if (Op.isReg()) {
     O << '%' << getRegisterName(Op.getReg());
   } else if (Op.isImm()) {
-    O << '$' << Op.getImm();
+    // Print X86 immediates as signed values.
+    O << '$' << (int64_t)Op.getImm();
     
     if (CommentStream && (Op.getImm() > 255 || Op.getImm() < -256))
       *CommentStream << format("imm = 0x%llX\n", (long long)Op.getImm());
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h b/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
index 5426e5cf38d9..0293869b0a9b 100644
--- a/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
+++ b/contrib/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
@@ -25,7 +25,7 @@ class X86ATTInstPrinter : public MCInstPrinter {
   X86ATTInstPrinter(const MCAsmInfo &MAI);
   
   virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
-  virtual void printInst(const MCInst *MI, raw_ostream &OS);
+  virtual void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot);
   virtual StringRef getOpcodeName(unsigned Opcode) const;
 
   // Autogenerated by tblgen, returns true if we successfully printed an
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp b/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp
index 4e28dfe7fa81..8d85b95fe81d 100644
--- a/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp
+++ b/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp
@@ -14,9 +14,9 @@
 
 #include "X86InstComments.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
+#include "Utils/X86ShuffleDecode.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/raw_ostream.h"
-#include "../Utils/X86ShuffleDecode.h"
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -136,9 +136,11 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
     break;
 
   case X86::SHUFPDrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::SHUFPDrmi:
     DecodeSHUFPSMask(2, MI->getOperand(3).getImm(), ShuffleMask);
     Src1Name = getRegName(MI->getOperand(0).getReg());
-    Src2Name = getRegName(MI->getOperand(2).getReg());
     break;
 
   case X86::SHUFPSrri:
@@ -205,6 +207,31 @@ void llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
     DecodeUNPCKHPMask(4, ShuffleMask);
     Src1Name = getRegName(MI->getOperand(0).getReg());
     break;
+  case X86::VPERMILPSri:
+    DecodeVPERMILPSMask(4, MI->getOperand(2).getImm(),
+                        ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPSYri:
+    DecodeVPERMILPSMask(8, MI->getOperand(2).getImm(),
+                        ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPDri:
+    DecodeVPERMILPDMask(2, MI->getOperand(2).getImm(),
+                        ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPDYri:
+    DecodeVPERMILPDMask(4, MI->getOperand(2).getImm(),
+                        ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERM2F128rr:
+    DecodeVPERM2F128Mask(MI->getOperand(3).getImm(), ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    break;
   }
 
 
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp b/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
index 506e26cbf7cd..f9ab5aeee122 100644
--- a/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
+++ b/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
@@ -32,12 +32,15 @@ void X86IntelInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   OS << getRegisterName(RegNo);
 }
 
-void X86IntelInstPrinter::printInst(const MCInst *MI, raw_ostream &OS) {
+void X86IntelInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
+                                    StringRef Annot) {
   printInstruction(MI, OS);
   
   // If verbose assembly is enabled, we can print some informative comments.
-  if (CommentStream)
+  if (CommentStream) {
+    printAnnotation(OS, Annot);
     EmitAnyX86InstComments(MI, *CommentStream, getRegisterName);
+  }
 }
 StringRef X86IntelInstPrinter::getOpcodeName(unsigned Opcode) const {
   return getInstructionName(Opcode);
diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h b/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
index e84a1940017d..6d5ec6226a9e 100644
--- a/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
+++ b/contrib/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
@@ -27,7 +27,7 @@ class X86IntelInstPrinter : public MCInstPrinter {
     : MCInstPrinter(MAI) {}
 
   virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
-  virtual void printInst(const MCInst *MI, raw_ostream &OS);
+  virtual void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot);
   virtual StringRef getOpcodeName(unsigned Opcode) const;
   
   // Autogenerated by tblgen.
diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index ca88f8ffd08c..000000000000
--- a/contrib/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-add_llvm_library(LLVMX86Desc
-  X86MCTargetDesc.cpp
-  X86MCAsmInfo.cpp
-  )
-
-# Hack: we need to include 'main' target directory to grab private headers
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/..)
diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/X86/MCTargetDesc/Makefile
deleted file mode 100644
index b19774ee379e..000000000000
--- a/contrib/llvm/lib/Target/X86/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/X86/TargetDesc/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMX86Desc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/X86/X86AsmBackend.cpp b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
similarity index 94%
rename from contrib/llvm/lib/Target/X86/X86AsmBackend.cpp
rename to contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 9b556a55efd9..69ad7d7b6b32 100644
--- a/contrib/llvm/lib/Target/X86/X86AsmBackend.cpp
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -7,9 +7,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetAsmBackend.h"
-#include "X86.h"
-#include "X86FixupKinds.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "MCTargetDesc/X86BaseInfo.h"
+#include "MCTargetDesc/X86FixupKinds.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
@@ -24,9 +24,8 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetAsmBackend.h"
 using namespace llvm;
 
 // Option to allow disabling arithmetic relaxation to workaround PR9807, which
@@ -63,10 +62,10 @@ class X86ELFObjectWriter : public MCELFObjectTargetWriter {
     : MCELFObjectTargetWriter(is64Bit, OSType, EMachine, HasRelocationAddend) {}
 };
 
-class X86AsmBackend : public TargetAsmBackend {
+class X86AsmBackend : public MCAsmBackend {
 public:
   X86AsmBackend(const Target &T)
-    : TargetAsmBackend() {}
+    : MCAsmBackend() {}
 
   unsigned getNumFixupKinds() const {
     return X86::NumTargetFixupKinds;
@@ -81,7 +80,7 @@ class X86AsmBackend : public TargetAsmBackend {
     };
 
     if (Kind < FirstTargetFixupKind)
-      return TargetAsmBackend::getFixupKindInfo(Kind);
+      return MCAsmBackend::getFixupKindInfo(Kind);
 
     assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
            "Invalid kind!");
@@ -94,6 +93,14 @@ class X86AsmBackend : public TargetAsmBackend {
 
     assert(Fixup.getOffset() + Size <= DataSize &&
            "Invalid fixup offset!");
+
+    // Check that uppper bits are either all zeros or all ones.
+    // Specifically ignore overflow/underflow as long as the leakage is
+    // limited to the lower bits. This is to remain compatible with
+    // other assemblers.
+    assert(isIntN(Size * 8 + 1, Value) &&
+           "Value does not fit in the Fixup field");
+
     for (unsigned i = 0; i != Size; ++i)
       Data[Fixup.getOffset() + i] = uint8_t(Value >> (i * 8));
   }
@@ -426,8 +433,7 @@ class DarwinX86_64AsmBackend : public DarwinX86AsmBackend {
 
 } // end anonymous namespace
 
-TargetAsmBackend *llvm::createX86_32AsmBackend(const Target &T,
-                                               const std::string &TT) {
+MCAsmBackend *llvm::createX86_32AsmBackend(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
@@ -439,8 +445,7 @@ TargetAsmBackend *llvm::createX86_32AsmBackend(const Target &T,
   return new ELFX86_32AsmBackend(T, TheTriple.getOS());
 }
 
-TargetAsmBackend *llvm::createX86_64AsmBackend(const Target &T,
-                                               const std::string &TT) {
+MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
new file mode 100644
index 000000000000..e6ba705d4d87
--- /dev/null
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -0,0 +1,548 @@
+//===-- X86BaseInfo.h - Top level definitions for X86 -------- --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains small standalone helper functions and enum definitions for
+// the X86 target useful for the compiler back-end and the MC libraries.
+// As such, it deliberately does not include references to LLVM core
+// code gen types, passes, etc..
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef X86BASEINFO_H
+#define X86BASEINFO_H
+
+#include "X86MCTargetDesc.h"
+#include "llvm/Support/DataTypes.h"
+#include <cassert>
+
+namespace llvm {
+
+namespace X86 {
+  // Enums for memory operand decoding.  Each memory operand is represented with
+  // a 5 operand sequence in the form:
+  //   [BaseReg, ScaleAmt, IndexReg, Disp, Segment]
+  // These enums help decode this.
+  enum {
+    AddrBaseReg = 0,
+    AddrScaleAmt = 1,
+    AddrIndexReg = 2,
+    AddrDisp = 3,
+
+    /// AddrSegmentReg - The operand # of the segment in the memory operand.
+    AddrSegmentReg = 4,
+
+    /// AddrNumOperands - Total number of operands in a memory reference.
+    AddrNumOperands = 5
+  };
+} // end namespace X86;
+ 
+
+/// X86II - This namespace holds all of the target specific flags that
+/// instruction info tracks.
+///
+namespace X86II {
+  /// Target Operand Flag enum.
+  enum TOF {
+    //===------------------------------------------------------------------===//
+    // X86 Specific MachineOperand flags.
+
+    MO_NO_FLAG,
+
+    /// MO_GOT_ABSOLUTE_ADDRESS - On a symbol operand, this represents a
+    /// relocation of:
+    ///    SYMBOL_LABEL + [. - PICBASELABEL]
+    MO_GOT_ABSOLUTE_ADDRESS,
+
+    /// MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the
+    /// immediate should get the value of the symbol minus the PIC base label:
+    ///    SYMBOL_LABEL - PICBASELABEL
+    MO_PIC_BASE_OFFSET,
+
+    /// MO_GOT - On a symbol operand this indicates that the immediate is the
+    /// offset to the GOT entry for the symbol name from the base of the GOT.
+    ///
+    /// See the X86-64 ELF ABI supplement for more details.
+    ///    SYMBOL_LABEL @GOT
+    MO_GOT,
+
+    /// MO_GOTOFF - On a symbol operand this indicates that the immediate is
+    /// the offset to the location of the symbol name from the base of the GOT.
+    ///
+    /// See the X86-64 ELF ABI supplement for more details.
+    ///    SYMBOL_LABEL @GOTOFF
+    MO_GOTOFF,
+
+    /// MO_GOTPCREL - On a symbol operand this indicates that the immediate is
+    /// offset to the GOT entry for the symbol name from the current code
+    /// location.
+    ///
+    /// See the X86-64 ELF ABI supplement for more details.
+    ///    SYMBOL_LABEL @GOTPCREL
+    MO_GOTPCREL,
+
+    /// MO_PLT - On a symbol operand this indicates that the immediate is
+    /// offset to the PLT entry of symbol name from the current code location.
+    ///
+    /// See the X86-64 ELF ABI supplement for more details.
+    ///    SYMBOL_LABEL @PLT
+    MO_PLT,
+
+    /// MO_TLSGD - On a symbol operand this indicates that the immediate is
+    /// some TLS offset.
+    ///
+    /// See 'ELF Handling for Thread-Local Storage' for more details.
+    ///    SYMBOL_LABEL @TLSGD
+    MO_TLSGD,
+
+    /// MO_GOTTPOFF - On a symbol operand this indicates that the immediate is
+    /// some TLS offset.
+    ///
+    /// See 'ELF Handling for Thread-Local Storage' for more details.
+    ///    SYMBOL_LABEL @GOTTPOFF
+    MO_GOTTPOFF,
+
+    /// MO_INDNTPOFF - On a symbol operand this indicates that the immediate is
+    /// some TLS offset.
+    ///
+    /// See 'ELF Handling for Thread-Local Storage' for more details.
+    ///    SYMBOL_LABEL @INDNTPOFF
+    MO_INDNTPOFF,
+
+    /// MO_TPOFF - On a symbol operand this indicates that the immediate is
+    /// some TLS offset.
+    ///
+    /// See 'ELF Handling for Thread-Local Storage' for more details.
+    ///    SYMBOL_LABEL @TPOFF
+    MO_TPOFF,
+
+    /// MO_NTPOFF - On a symbol operand this indicates that the immediate is
+    /// some TLS offset.
+    ///
+    /// See 'ELF Handling for Thread-Local Storage' for more details.
+    ///    SYMBOL_LABEL @NTPOFF
+    MO_NTPOFF,
+
+    /// MO_DLLIMPORT - On a symbol operand "FOO", this indicates that the
+    /// reference is actually to the "__imp_FOO" symbol.  This is used for
+    /// dllimport linkage on windows.
+    MO_DLLIMPORT,
+
+    /// MO_DARWIN_STUB - On a symbol operand "FOO", this indicates that the
+    /// reference is actually to the "FOO$stub" symbol.  This is used for calls
+    /// and jumps to external functions on Tiger and earlier.
+    MO_DARWIN_STUB,
+
+    /// MO_DARWIN_NONLAZY - On a symbol operand "FOO", this indicates that the
+    /// reference is actually to the "FOO$non_lazy_ptr" symbol, which is a
+    /// non-PIC-base-relative reference to a non-hidden dyld lazy pointer stub.
+    MO_DARWIN_NONLAZY,
+
+    /// MO_DARWIN_NONLAZY_PIC_BASE - On a symbol operand "FOO", this indicates
+    /// that the reference is actually to "FOO$non_lazy_ptr - PICBASE", which is
+    /// a PIC-base-relative reference to a non-hidden dyld lazy pointer stub.
+    MO_DARWIN_NONLAZY_PIC_BASE,
+
+    /// MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE - On a symbol operand "FOO", this
+    /// indicates that the reference is actually to "FOO$non_lazy_ptr -PICBASE",
+    /// which is a PIC-base-relative reference to a hidden dyld lazy pointer
+    /// stub.
+    MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE,
+
+    /// MO_TLVP - On a symbol operand this indicates that the immediate is
+    /// some TLS offset.
+    ///
+    /// This is the TLS offset for the Darwin TLS mechanism.
+    MO_TLVP,
+
+    /// MO_TLVP_PIC_BASE - On a symbol operand this indicates that the immediate
+    /// is some TLS offset from the picbase.
+    ///
+    /// This is the 32-bit TLS offset for Darwin TLS in PIC mode.
+    MO_TLVP_PIC_BASE
+  };
+
+  enum {
+    //===------------------------------------------------------------------===//
+    // Instruction encodings.  These are the standard/most common forms for X86
+    // instructions.
+    //
+
+    // PseudoFrm - This represents an instruction that is a pseudo instruction
+    // or one that has not been implemented yet.  It is illegal to code generate
+    // it, but tolerated for intermediate implementation stages.
+    Pseudo         = 0,
+
+    /// Raw - This form is for instructions that don't have any operands, so
+    /// they are just a fixed opcode value, like 'leave'.
+    RawFrm         = 1,
+
+    /// AddRegFrm - This form is used for instructions like 'push r32' that have
+    /// their one register operand added to their opcode.
+    AddRegFrm      = 2,
+
+    /// MRMDestReg - This form is used for instructions that use the Mod/RM byte
+    /// to specify a destination, which in this case is a register.
+    ///
+    MRMDestReg     = 3,
+
+    /// MRMDestMem - This form is used for instructions that use the Mod/RM byte
+    /// to specify a destination, which in this case is memory.
+    ///
+    MRMDestMem     = 4,
+
+    /// MRMSrcReg - This form is used for instructions that use the Mod/RM byte
+    /// to specify a source, which in this case is a register.
+    ///
+    MRMSrcReg      = 5,
+
+    /// MRMSrcMem - This form is used for instructions that use the Mod/RM byte
+    /// to specify a source, which in this case is memory.
+    ///
+    MRMSrcMem      = 6,
+
+    /// MRM[0-7][rm] - These forms are used to represent instructions that use
+    /// a Mod/RM byte, and use the middle field to hold extended opcode
+    /// information.  In the intel manual these are represented as /0, /1, ...
+    ///
+
+    // First, instructions that operate on a register r/m operand...
+    MRM0r = 16,  MRM1r = 17,  MRM2r = 18,  MRM3r = 19, // Format /0 /1 /2 /3
+    MRM4r = 20,  MRM5r = 21,  MRM6r = 22,  MRM7r = 23, // Format /4 /5 /6 /7
+
+    // Next, instructions that operate on a memory r/m operand...
+    MRM0m = 24,  MRM1m = 25,  MRM2m = 26,  MRM3m = 27, // Format /0 /1 /2 /3
+    MRM4m = 28,  MRM5m = 29,  MRM6m = 30,  MRM7m = 31, // Format /4 /5 /6 /7
+
+    // MRMInitReg - This form is used for instructions whose source and
+    // destinations are the same register.
+    MRMInitReg = 32,
+
+    //// MRM_C1 - A mod/rm byte of exactly 0xC1.
+    MRM_C1 = 33,
+    MRM_C2 = 34,
+    MRM_C3 = 35,
+    MRM_C4 = 36,
+    MRM_C8 = 37,
+    MRM_C9 = 38,
+    MRM_E8 = 39,
+    MRM_F0 = 40,
+    MRM_F8 = 41,
+    MRM_F9 = 42,
+    MRM_D0 = 45,
+    MRM_D1 = 46,
+
+    /// RawFrmImm8 - This is used for the ENTER instruction, which has two
+    /// immediates, the first of which is a 16-bit immediate (specified by
+    /// the imm encoding) and the second is a 8-bit fixed value.
+    RawFrmImm8 = 43,
+
+    /// RawFrmImm16 - This is used for CALL FAR instructions, which have two
+    /// immediates, the first of which is a 16 or 32-bit immediate (specified by
+    /// the imm encoding) and the second is a 16-bit fixed value.  In the AMD
+    /// manual, this operand is described as pntr16:32 and pntr16:16
+    RawFrmImm16 = 44,
+
+    FormMask       = 63,
+
+    //===------------------------------------------------------------------===//
+    // Actual flags...
+
+    // OpSize - Set if this instruction requires an operand size prefix (0x66),
+    // which most often indicates that the instruction operates on 16 bit data
+    // instead of 32 bit data.
+    OpSize      = 1 << 6,
+
+    // AsSize - Set if this instruction requires an operand size prefix (0x67),
+    // which most often indicates that the instruction address 16 bit address
+    // instead of 32 bit address (or 32 bit address in 64 bit mode).
+    AdSize      = 1 << 7,
+
+    //===------------------------------------------------------------------===//
+    // Op0Mask - There are several prefix bytes that are used to form two byte
+    // opcodes.  These are currently 0x0F, 0xF3, and 0xD8-0xDF.  This mask is
+    // used to obtain the setting of this field.  If no bits in this field is
+    // set, there is no prefix byte for obtaining a multibyte opcode.
+    //
+    Op0Shift    = 8,
+    Op0Mask     = 0x1F << Op0Shift,
+
+    // TB - TwoByte - Set if this instruction has a two byte opcode, which
+    // starts with a 0x0F byte before the real opcode.
+    TB          = 1 << Op0Shift,
+
+    // REP - The 0xF3 prefix byte indicating repetition of the following
+    // instruction.
+    REP         = 2 << Op0Shift,
+
+    // D8-DF - These escape opcodes are used by the floating point unit.  These
+    // values must remain sequential.
+    D8 = 3 << Op0Shift,   D9 = 4 << Op0Shift,
+    DA = 5 << Op0Shift,   DB = 6 << Op0Shift,
+    DC = 7 << Op0Shift,   DD = 8 << Op0Shift,
+    DE = 9 << Op0Shift,   DF = 10 << Op0Shift,
+
+    // XS, XD - These prefix codes are for single and double precision scalar
+    // floating point operations performed in the SSE registers.
+    XD = 11 << Op0Shift,  XS = 12 << Op0Shift,
+
+    // T8, TA, A6, A7 - Prefix after the 0x0F prefix.
+    T8 = 13 << Op0Shift,  TA = 14 << Op0Shift,
+    A6 = 15 << Op0Shift,  A7 = 16 << Op0Shift,
+
+    // TF - Prefix before and after 0x0F
+    TF = 17 << Op0Shift,
+
+    //===------------------------------------------------------------------===//
+    // REX_W - REX prefixes are instruction prefixes used in 64-bit mode.
+    // They are used to specify GPRs and SSE registers, 64-bit operand size,
+    // etc. We only cares about REX.W and REX.R bits and only the former is
+    // statically determined.
+    //
+    REXShift    = Op0Shift + 5,
+    REX_W       = 1 << REXShift,
+
+    //===------------------------------------------------------------------===//
+    // This three-bit field describes the size of an immediate operand.  Zero is
+    // unused so that we can tell if we forgot to set a value.
+    ImmShift = REXShift + 1,
+    ImmMask    = 7 << ImmShift,
+    Imm8       = 1 << ImmShift,
+    Imm8PCRel  = 2 << ImmShift,
+    Imm16      = 3 << ImmShift,
+    Imm16PCRel = 4 << ImmShift,
+    Imm32      = 5 << ImmShift,
+    Imm32PCRel = 6 << ImmShift,
+    Imm64      = 7 << ImmShift,
+
+    //===------------------------------------------------------------------===//
+    // FP Instruction Classification...  Zero is non-fp instruction.
+
+    // FPTypeMask - Mask for all of the FP types...
+    FPTypeShift = ImmShift + 3,
+    FPTypeMask  = 7 << FPTypeShift,
+
+    // NotFP - The default, set for instructions that do not use FP registers.
+    NotFP      = 0 << FPTypeShift,
+
+    // ZeroArgFP - 0 arg FP instruction which implicitly pushes ST(0), f.e. fld0
+    ZeroArgFP  = 1 << FPTypeShift,
+
+    // OneArgFP - 1 arg FP instructions which implicitly read ST(0), such as fst
+    OneArgFP   = 2 << FPTypeShift,
+
+    // OneArgFPRW - 1 arg FP instruction which implicitly read ST(0) and write a
+    // result back to ST(0).  For example, fcos, fsqrt, etc.
+    //
+    OneArgFPRW = 3 << FPTypeShift,
+
+    // TwoArgFP - 2 arg FP instructions which implicitly read ST(0), and an
+    // explicit argument, storing the result to either ST(0) or the implicit
+    // argument.  For example: fadd, fsub, fmul, etc...
+    TwoArgFP   = 4 << FPTypeShift,
+
+    // CompareFP - 2 arg FP instructions which implicitly read ST(0) and an
+    // explicit argument, but have no destination.  Example: fucom, fucomi, ...
+    CompareFP  = 5 << FPTypeShift,
+
+    // CondMovFP - "2 operand" floating point conditional move instructions.
+    CondMovFP  = 6 << FPTypeShift,
+
+    // SpecialFP - Special instruction forms.  Dispatch by opcode explicitly.
+    SpecialFP  = 7 << FPTypeShift,
+
+    // Lock prefix
+    LOCKShift = FPTypeShift + 3,
+    LOCK = 1 << LOCKShift,
+
+    // Segment override prefixes. Currently we just need ability to address
+    // stuff in gs and fs segments.
+    SegOvrShift = LOCKShift + 1,
+    SegOvrMask  = 3 << SegOvrShift,
+    FS          = 1 << SegOvrShift,
+    GS          = 2 << SegOvrShift,
+
+    // Execution domain for SSE instructions in bits 23, 24.
+    // 0 in bits 23-24 means normal, non-SSE instruction.
+    SSEDomainShift = SegOvrShift + 2,
+
+    OpcodeShift   = SSEDomainShift + 2,
+
+    //===------------------------------------------------------------------===//
+    /// VEX - The opcode prefix used by AVX instructions
+    VEXShift = OpcodeShift + 8,
+    VEX         = 1U << 0,
+
+    /// VEX_W - Has a opcode specific functionality, but is used in the same
+    /// way as REX_W is for regular SSE instructions.
+    VEX_W       = 1U << 1,
+
+    /// VEX_4V - Used to specify an additional AVX/SSE register. Several 2
+    /// address instructions in SSE are represented as 3 address ones in AVX
+    /// and the additional register is encoded in VEX_VVVV prefix.
+    VEX_4V      = 1U << 2,
+
+    /// VEX_I8IMM - Specifies that the last register used in a AVX instruction,
+    /// must be encoded in the i8 immediate field. This usually happens in
+    /// instructions with 4 operands.
+    VEX_I8IMM   = 1U << 3,
+
+    /// VEX_L - Stands for a bit in the VEX opcode prefix meaning the current
+    /// instruction uses 256-bit wide registers. This is usually auto detected
+    /// if a VR256 register is used, but some AVX instructions also have this
+    /// field marked when using a f256 memory references.
+    VEX_L       = 1U << 4,
+
+    // VEX_LIG - Specifies that this instruction ignores the L-bit in the VEX
+    // prefix. Usually used for scalar instructions. Needed by disassembler.
+    VEX_LIG     = 1U << 5,
+
+    /// Has3DNow0F0FOpcode - This flag indicates that the instruction uses the
+    /// wacky 0x0F 0x0F prefix for 3DNow! instructions.  The manual documents
+    /// this as having a 0x0F prefix with a 0x0F opcode, and each instruction
+    /// storing a classifier in the imm8 field.  To simplify our implementation,
+    /// we handle this by storeing the classifier in the opcode field and using
+    /// this flag to indicate that the encoder should do the wacky 3DNow! thing.
+    Has3DNow0F0FOpcode = 1U << 6
+  };
+
+  // getBaseOpcodeFor - This function returns the "base" X86 opcode for the
+  // specified machine instruction.
+  //
+  static inline unsigned char getBaseOpcodeFor(uint64_t TSFlags) {
+    return TSFlags >> X86II::OpcodeShift;
+  }
+
+  static inline bool hasImm(uint64_t TSFlags) {
+    return (TSFlags & X86II::ImmMask) != 0;
+  }
+
+  /// getSizeOfImm - Decode the "size of immediate" field from the TSFlags field
+  /// of the specified instruction.
+  static inline unsigned getSizeOfImm(uint64_t TSFlags) {
+    switch (TSFlags & X86II::ImmMask) {
+    default: assert(0 && "Unknown immediate size");
+    case X86II::Imm8:
+    case X86II::Imm8PCRel:  return 1;
+    case X86II::Imm16:
+    case X86II::Imm16PCRel: return 2;
+    case X86II::Imm32:
+    case X86II::Imm32PCRel: return 4;
+    case X86II::Imm64:      return 8;
+    }
+  }
+
+  /// isImmPCRel - Return true if the immediate of the specified instruction's
+  /// TSFlags indicates that it is pc relative.
+  static inline unsigned isImmPCRel(uint64_t TSFlags) {
+    switch (TSFlags & X86II::ImmMask) {
+    default: assert(0 && "Unknown immediate size");
+    case X86II::Imm8PCRel:
+    case X86II::Imm16PCRel:
+    case X86II::Imm32PCRel:
+      return true;
+    case X86II::Imm8:
+    case X86II::Imm16:
+    case X86II::Imm32:
+    case X86II::Imm64:
+      return false;
+    }
+  }
+
+  /// getMemoryOperandNo - The function returns the MCInst operand # for the
+  /// first field of the memory operand.  If the instruction doesn't have a
+  /// memory operand, this returns -1.
+  ///
+  /// Note that this ignores tied operands.  If there is a tied register which
+  /// is duplicated in the MCInst (e.g. "EAX = addl EAX, [mem]") it is only
+  /// counted as one operand.
+  ///
+  static inline int getMemoryOperandNo(uint64_t TSFlags) {
+    switch (TSFlags & X86II::FormMask) {
+    case X86II::MRMInitReg:  assert(0 && "FIXME: Remove this form");
+    default: assert(0 && "Unknown FormMask value in getMemoryOperandNo!");
+    case X86II::Pseudo:
+    case X86II::RawFrm:
+    case X86II::AddRegFrm:
+    case X86II::MRMDestReg:
+    case X86II::MRMSrcReg:
+    case X86II::RawFrmImm8:
+    case X86II::RawFrmImm16:
+       return -1;
+    case X86II::MRMDestMem:
+      return 0;
+    case X86II::MRMSrcMem: {
+      bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V;
+      unsigned FirstMemOp = 1;
+      if (HasVEX_4V)
+        ++FirstMemOp;// Skip the register source (which is encoded in VEX_VVVV).
+
+      // FIXME: Maybe lea should have its own form?  This is a horrible hack.
+      //if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r ||
+      //    Opcode == X86::LEA16r || Opcode == X86::LEA32r)
+      return FirstMemOp;
+    }
+    case X86II::MRM0r: case X86II::MRM1r:
+    case X86II::MRM2r: case X86II::MRM3r:
+    case X86II::MRM4r: case X86II::MRM5r:
+    case X86II::MRM6r: case X86II::MRM7r:
+      return -1;
+    case X86II::MRM0m: case X86II::MRM1m:
+    case X86II::MRM2m: case X86II::MRM3m:
+    case X86II::MRM4m: case X86II::MRM5m:
+    case X86II::MRM6m: case X86II::MRM7m:
+      return 0;
+    case X86II::MRM_C1:
+    case X86II::MRM_C2:
+    case X86II::MRM_C3:
+    case X86II::MRM_C4:
+    case X86II::MRM_C8:
+    case X86II::MRM_C9:
+    case X86II::MRM_E8:
+    case X86II::MRM_F0:
+    case X86II::MRM_F8:
+    case X86II::MRM_F9:
+    case X86II::MRM_D0:
+    case X86II::MRM_D1:
+      return -1;
+    }
+  }
+
+  /// isX86_64ExtendedReg - Is the MachineOperand a x86-64 extended (r8 or
+  /// higher) register?  e.g. r8, xmm8, xmm13, etc.
+  static inline bool isX86_64ExtendedReg(unsigned RegNo) {
+    switch (RegNo) {
+    default: break;
+    case X86::R8:    case X86::R9:    case X86::R10:   case X86::R11:
+    case X86::R12:   case X86::R13:   case X86::R14:   case X86::R15:
+    case X86::R8D:   case X86::R9D:   case X86::R10D:  case X86::R11D:
+    case X86::R12D:  case X86::R13D:  case X86::R14D:  case X86::R15D:
+    case X86::R8W:   case X86::R9W:   case X86::R10W:  case X86::R11W:
+    case X86::R12W:  case X86::R13W:  case X86::R14W:  case X86::R15W:
+    case X86::R8B:   case X86::R9B:   case X86::R10B:  case X86::R11B:
+    case X86::R12B:  case X86::R13B:  case X86::R14B:  case X86::R15B:
+    case X86::XMM8:  case X86::XMM9:  case X86::XMM10: case X86::XMM11:
+    case X86::XMM12: case X86::XMM13: case X86::XMM14: case X86::XMM15:
+    case X86::YMM8:  case X86::YMM9:  case X86::YMM10: case X86::YMM11:
+    case X86::YMM12: case X86::YMM13: case X86::YMM14: case X86::YMM15:
+    case X86::CR8:   case X86::CR9:   case X86::CR10:  case X86::CR11:
+    case X86::CR12:  case X86::CR13:  case X86::CR14:  case X86::CR15:
+        return true;
+    }
+    return false;
+  }
+  
+  static inline bool isX86_64NonExtLowByteReg(unsigned reg) {
+    return (reg == X86::SPL || reg == X86::BPL ||
+            reg == X86::SIL || reg == X86::DIL);
+  }
+}
+
+} // end namespace llvm;
+
+#endif
diff --git a/contrib/llvm/lib/Target/X86/X86FixupKinds.h b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86FixupKinds.h
similarity index 100%
rename from contrib/llvm/lib/Target/X86/X86FixupKinds.h
rename to contrib/llvm/lib/Target/X86/MCTargetDesc/X86FixupKinds.h
diff --git a/contrib/llvm/lib/Target/X86/X86MCCodeEmitter.cpp b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
similarity index 89%
rename from contrib/llvm/lib/Target/X86/X86MCCodeEmitter.cpp
rename to contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index ce8ef495c001..2eee1128119e 100644
--- a/contrib/llvm/lib/Target/X86/X86MCCodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -12,12 +12,14 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "mccodeemitter"
-#include "X86.h"
-#include "X86InstrInfo.h"
-#include "X86FixupKinds.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
+#include "MCTargetDesc/X86BaseInfo.h"
+#include "MCTargetDesc/X86FixupKinds.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/raw_ostream.h"
@@ -45,7 +47,7 @@ class X86MCCodeEmitter : public MCCodeEmitter {
   }
 
   static unsigned GetX86RegNum(const MCOperand &MO) {
-    return X86RegisterInfo::getX86RegNum(MO.getReg());
+    return X86_MC::getX86RegNum(MO.getReg());
   }
 
   // On regular x86, both XMM0-XMM7 and XMM8-XMM15 are encoded in the range
@@ -159,9 +161,11 @@ static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
 static bool Is32BitMemOperand(const MCInst &MI, unsigned Op) {
   const MCOperand &BaseReg  = MI.getOperand(Op+X86::AddrBaseReg);
   const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
-  
-  if ((BaseReg.getReg() != 0 && X86::GR32RegClass.contains(BaseReg.getReg())) ||
-      (IndexReg.getReg() != 0 && X86::GR32RegClass.contains(IndexReg.getReg())))
+
+  if ((BaseReg.getReg() != 0 &&
+       X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
+      (IndexReg.getReg() != 0 &&
+       X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
     return true;
   return false;
 }
@@ -191,11 +195,11 @@ EmitImmediate(const MCOperand &DispOp, unsigned Size, MCFixupKind FixupKind,
               SmallVectorImpl<MCFixup> &Fixups, int ImmOffset) const {
   const MCExpr *Expr = NULL;
   if (DispOp.isImm()) {
-    // If this is a simple integer displacement that doesn't require a relocation,
-    // emit it now.
+    // If this is a simple integer displacement that doesn't require a
+    // relocation, emit it now.
     if (FixupKind != FK_PCRel_1 &&
-	FixupKind != FK_PCRel_2 &&
-	FixupKind != FK_PCRel_4) {
+        FixupKind != FK_PCRel_2 &&
+        FixupKind != FK_PCRel_4) {
       EmitConstant(DispOp.getImm()+ImmOffset, Size, CurByte, OS);
       return;
     }
@@ -205,7 +209,9 @@ EmitImmediate(const MCOperand &DispOp, unsigned Size, MCFixupKind FixupKind,
   }
 
   // If we have an immoffset, add it to the expression.
-  if (FixupKind == FK_Data_4 && StartsWithGlobalOffsetTable(Expr)) {
+  if ((FixupKind == FK_Data_4 ||
+       FixupKind == MCFixupKind(X86::reloc_signed_4byte)) &&
+      StartsWithGlobalOffsetTable(Expr)) {
     assert(ImmOffset == 0);
 
     FixupKind = MCFixupKind(X86::reloc_global_offset_table);
@@ -346,7 +352,7 @@ void X86MCCodeEmitter::EmitMemModRMByte(const MCInst &MI, unsigned Op,
   }
 
   // Calculate what the SS field value should be...
-  static const unsigned SSTable[] = { ~0, 0, 1, ~0, 2, ~0, ~0, ~0, 3 };
+  static const unsigned SSTable[] = { ~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3 };
   unsigned SS = SSTable[Scale.getImm()];
 
   if (BaseReg == 0) {
@@ -486,71 +492,100 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
       VEX_L = 1;
   }
 
-  unsigned NumOps = MI.getNumOperands();
+  // Classify VEX_B, VEX_4V, VEX_R, VEX_X
   unsigned CurOp = 0;
-  bool IsDestMem = false;
-
   switch (TSFlags & X86II::FormMask) {
   case X86II::MRMInitReg: assert(0 && "FIXME: Remove this!");
-  case X86II::MRMDestMem:
-    IsDestMem = true;
-    // The important info for the VEX prefix is never beyond the address
-    // registers. Don't check beyond that.
-    NumOps = CurOp = X86::AddrNumOperands;
+  case X86II::MRMDestMem: {
+    // MRMDestMem instructions forms:
+    //  MemAddr, src1(ModR/M)
+    //  MemAddr, src1(VEX_4V), src2(ModR/M)
+    //  MemAddr, src1(ModR/M), imm8
+    //
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(X86::AddrBaseReg).getReg()))
+      VEX_B = 0x0;
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(X86::AddrIndexReg).getReg()))
+      VEX_X = 0x0;
+
+    CurOp = X86::AddrNumOperands;
+    if (HasVEX_4V)
+      VEX_4V = getVEXRegisterEncoding(MI, CurOp++);
+
+    const MCOperand &MO = MI.getOperand(CurOp);
+    if (MO.isReg() && X86II::isX86_64ExtendedReg(MO.getReg()))
+      VEX_R = 0x0;
+    break;
+  }
+  case X86II::MRMSrcMem: {
+    // MRMSrcMem instructions forms:
+    //  src1(ModR/M), MemAddr
+    //  src1(ModR/M), src2(VEX_4V), MemAddr
+    //  src1(ModR/M), MemAddr, imm8
+    //  src1(ModR/M), MemAddr, src2(VEX_I8IMM)
+    //
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
+      VEX_R = 0x0;
+
+    unsigned MemAddrOffset = 1;
+    if (HasVEX_4V) {
+      VEX_4V = getVEXRegisterEncoding(MI, 1);
+      MemAddrOffset++;
+    }
+
+    if (X86II::isX86_64ExtendedReg(
+               MI.getOperand(MemAddrOffset+X86::AddrBaseReg).getReg()))
+      VEX_B = 0x0;
+    if (X86II::isX86_64ExtendedReg(
+               MI.getOperand(MemAddrOffset+X86::AddrIndexReg).getReg()))
+      VEX_X = 0x0;
+    break;
+  }
   case X86II::MRM0m: case X86II::MRM1m:
   case X86II::MRM2m: case X86II::MRM3m:
   case X86II::MRM4m: case X86II::MRM5m:
   case X86II::MRM6m: case X86II::MRM7m:
-  case X86II::MRMSrcMem:
+    // MRM[0-9]m instructions forms:
+    //  MemAddr
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(X86::AddrBaseReg).getReg()))
+      VEX_B = 0x0;
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(X86::AddrIndexReg).getReg()))
+      VEX_X = 0x0;
+    break;
   case X86II::MRMSrcReg:
-    if (MI.getNumOperands() > CurOp && MI.getOperand(CurOp).isReg() &&
-        X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+    // MRMSrcReg instructions forms:
+    //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(VEX_I8IMM)
+    //  dst(ModR/M), src1(ModR/M)
+    //  dst(ModR/M), src1(ModR/M), imm8
+    //
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
       VEX_R = 0x0;
     CurOp++;
 
-    if (HasVEX_4V) {
-      VEX_4V = getVEXRegisterEncoding(MI, IsDestMem ? CurOp-1 : CurOp);
-      CurOp++;
-    }
-
-    // To only check operands before the memory address ones, start
-    // the search from the beginning
-    if (IsDestMem)
-      CurOp = 0;
-
-    // If the last register should be encoded in the immediate field
-    // do not use any bit from VEX prefix to this register, ignore it
-    if ((TSFlags >> X86II::VEXShift) & X86II::VEX_I8IMM)
-      NumOps--;
-
-    for (; CurOp != NumOps; ++CurOp) {
-      const MCOperand &MO = MI.getOperand(CurOp);
-      if (MO.isReg() && X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
-        VEX_B = 0x0;
-      if (!VEX_B && MO.isReg() &&
-          ((TSFlags & X86II::FormMask) == X86II::MRMSrcMem) &&
-          X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
-        VEX_X = 0x0;
-    }
-    break;
-  default: // MRMDestReg, MRM0r-MRM7r, RawFrm
-    if (!MI.getNumOperands())
-      break;
-
-    if (MI.getOperand(CurOp).isReg() &&
-        X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
-      VEX_B = 0;
-
     if (HasVEX_4V)
-      VEX_4V = getVEXRegisterEncoding(MI, CurOp);
-
-    CurOp++;
-    for (; CurOp != NumOps; ++CurOp) {
-      const MCOperand &MO = MI.getOperand(CurOp);
-      if (MO.isReg() && !HasVEX_4V &&
-          X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
-        VEX_R = 0x0;
-    }
+      VEX_4V = getVEXRegisterEncoding(MI, CurOp++);
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+      VEX_B = 0x0;
+    break;
+  case X86II::MRMDestReg:
+    // MRMDestReg instructions forms:
+    //  dst(ModR/M), src(ModR/M)
+    //  dst(ModR/M), src(ModR/M), imm8
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
+      VEX_B = 0x0;
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(1).getReg()))
+      VEX_R = 0x0;
+    break;
+  case X86II::MRM0r: case X86II::MRM1r:
+  case X86II::MRM2r: case X86II::MRM3r:
+  case X86II::MRM4r: case X86II::MRM5r:
+  case X86II::MRM6r: case X86II::MRM7r:
+    // MRM0r-MRM7r instructions forms:
+    //  dst(VEX_4V), src(ModR/M), imm8
+    VEX_4V = getVEXRegisterEncoding(MI, 0);
+    if (X86II::isX86_64ExtendedReg(MI.getOperand(1).getReg()))
+      VEX_B = 0x0;
+    break;
+  default: // RawFrm
     break;
   }
 
@@ -604,7 +639,7 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
     const MCOperand &MO = MI.getOperand(i);
     if (!MO.isReg()) continue;
     unsigned Reg = MO.getReg();
-    if (!X86InstrInfo::isX86_64NonExtLowByteReg(Reg)) continue;
+    if (!X86II::isX86_64NonExtLowByteReg(Reg)) continue;
     // FIXME: The caller of DetermineREXPrefix slaps this prefix onto anything
     // that returns non-zero.
     REX |= 0x40; // REX fixed encoding prefix
@@ -615,25 +650,25 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
   case X86II::MRMInitReg: assert(0 && "FIXME: Remove this!");
   case X86II::MRMSrcReg:
     if (MI.getOperand(0).isReg() &&
-        X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
+        X86II::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
       REX |= 1 << 2; // set REX.R
     i = isTwoAddr ? 2 : 1;
     for (; i != NumOps; ++i) {
       const MCOperand &MO = MI.getOperand(i);
-      if (MO.isReg() && X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
+      if (MO.isReg() && X86II::isX86_64ExtendedReg(MO.getReg()))
         REX |= 1 << 0; // set REX.B
     }
     break;
   case X86II::MRMSrcMem: {
     if (MI.getOperand(0).isReg() &&
-        X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
+        X86II::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
       REX |= 1 << 2; // set REX.R
     unsigned Bit = 0;
     i = isTwoAddr ? 2 : 1;
     for (; i != NumOps; ++i) {
       const MCOperand &MO = MI.getOperand(i);
       if (MO.isReg()) {
-        if (X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
+        if (X86II::isX86_64ExtendedReg(MO.getReg()))
           REX |= 1 << Bit; // set REX.B (Bit=0) and REX.X (Bit=1)
         Bit++;
       }
@@ -648,13 +683,13 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
     unsigned e = (isTwoAddr ? X86::AddrNumOperands+1 : X86::AddrNumOperands);
     i = isTwoAddr ? 1 : 0;
     if (NumOps > e && MI.getOperand(e).isReg() &&
-        X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(e).getReg()))
+        X86II::isX86_64ExtendedReg(MI.getOperand(e).getReg()))
       REX |= 1 << 2; // set REX.R
     unsigned Bit = 0;
     for (; i != e; ++i) {
       const MCOperand &MO = MI.getOperand(i);
       if (MO.isReg()) {
-        if (X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
+        if (X86II::isX86_64ExtendedReg(MO.getReg()))
           REX |= 1 << Bit; // REX.B (Bit=0) and REX.X (Bit=1)
         Bit++;
       }
@@ -663,12 +698,12 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
   }
   default:
     if (MI.getOperand(0).isReg() &&
-        X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
+        X86II::isX86_64ExtendedReg(MI.getOperand(0).getReg()))
       REX |= 1 << 0; // set REX.B
     i = isTwoAddr ? 2 : 1;
     for (unsigned e = NumOps; i != e; ++i) {
       const MCOperand &MO = MI.getOperand(i);
-      if (MO.isReg() && X86InstrInfo::isX86_64ExtendedReg(MO.getReg()))
+      if (MO.isReg() && X86II::isX86_64ExtendedReg(MO.getReg()))
         REX |= 1 << 2; // set REX.R
     }
     break;
@@ -731,7 +766,7 @@ void X86MCCodeEmitter::EmitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
   if ((TSFlags & X86II::AdSize) ||
       (MemOperand != -1 && is64BitMode() && Is32BitMemOperand(MI, MemOperand)))
     EmitByte(0x67, CurByte, OS);
-  
+
   // Emit the operand size opcode prefix as needed.
   if (TSFlags & X86II::OpSize)
     EmitByte(0x66, CurByte, OS);
@@ -834,7 +869,6 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   if ((TSFlags >> X86II::VEXShift) & X86II::VEX_4V)
     HasVEX_4V = true;
 
-  
   // Determine where the memory operand starts, if present.
   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
   if (MemoryOperand != -1) MemoryOperand += CurOp;
@@ -844,12 +878,11 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   else
     EmitVEXOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, OS);
 
-  
   unsigned char BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
-  
+
   if ((TSFlags >> X86II::VEXShift) & X86II::Has3DNow0F0FOpcode)
     BaseOpcode = 0x0F;   // Weird 3DNow! encoding.
-  
+
   unsigned SrcRegNum = 0;
   switch (TSFlags & X86II::FormMask) {
   case X86II::MRMInitReg:
@@ -861,7 +894,6 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   case X86II::RawFrm:
     EmitByte(BaseOpcode, CurByte, OS);
     break;
-      
   case X86II::RawFrmImm8:
     EmitByte(BaseOpcode, CurByte, OS);
     EmitImmediate(MI.getOperand(CurOp++),
@@ -1006,8 +1038,7 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
     // in bits[7:4] of a immediate byte, and bits[3:0] are ignored.
     if ((TSFlags >> X86II::VEXShift) & X86II::VEX_I8IMM) {
       const MCOperand &MO = MI.getOperand(CurOp++);
-      bool IsExtReg =
-        X86InstrInfo::isX86_64ExtendedReg(MO.getReg());
+      bool IsExtReg = X86II::isX86_64ExtendedReg(MO.getReg());
       unsigned RegNum = (IsExtReg ? (1 << 7) : 0);
       RegNum |= GetX86RegNum(MO) << 4;
       EmitImmediate(MCOperand::CreateImm(RegNum), 1, FK_Data_1, CurByte, OS,
@@ -1030,7 +1061,6 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
 
   if ((TSFlags >> X86II::VEXShift) & X86II::Has3DNow0F0FOpcode)
     EmitByte(X86II::getBaseOpcodeFor(TSFlags), CurByte, OS);
-  
 
 #ifndef NDEBUG
   // FIXME: Verify.
diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index b77f37b03f19..f98d5e331fef 100644
--- a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -13,12 +13,18 @@
 
 #include "X86MCTargetDesc.h"
 #include "X86MCAsmInfo.h"
+#include "InstPrinter/X86ATTInstPrinter.h"
+#include "InstPrinter/X86IntelInstPrinter.h"
+#include "llvm/MC/MachineLocation.h"
+#include "llvm/MC/MCCodeGenInfo.h"
+#include "llvm/MC/MCInstrAnalysis.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/Host.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_REGINFO_MC_DESC
 #include "X86GenRegisterInfo.inc"
@@ -34,9 +40,16 @@ using namespace llvm;
 
 std::string X86_MC::ParseX86Triple(StringRef TT) {
   Triple TheTriple(TT);
+  std::string FS;
   if (TheTriple.getArch() == Triple::x86_64)
-    return "+64bit-mode";
-  return "-64bit-mode";
+    FS = "+64bit-mode";
+  else
+    FS = "-64bit-mode";
+  if (TheTriple.getOS() == Triple::NativeClient)
+    FS += ",+nacl-mode";
+  else
+    FS += ",-nacl-mode";
+  return FS;
 }
 
 /// GetCpuIDAndInfo - Execute the specified cpuid and return the 4 values in the
@@ -107,6 +120,135 @@ void X86_MC::DetectFamilyModel(unsigned EAX, unsigned &Family,
   }
 }
 
+unsigned X86_MC::getDwarfRegFlavour(StringRef TT, bool isEH) {
+  Triple TheTriple(TT);
+  if (TheTriple.getArch() == Triple::x86_64)
+    return DWARFFlavour::X86_64;
+
+  if (TheTriple.isOSDarwin())
+    return isEH ? DWARFFlavour::X86_32_DarwinEH : DWARFFlavour::X86_32_Generic;
+  if (TheTriple.getOS() == Triple::MinGW32 ||
+      TheTriple.getOS() == Triple::Cygwin)
+    // Unsupported by now, just quick fallback
+    return DWARFFlavour::X86_32_Generic;
+  return DWARFFlavour::X86_32_Generic;
+}
+
+/// getX86RegNum - This function maps LLVM register identifiers to their X86
+/// specific numbering, which is used in various places encoding instructions.
+unsigned X86_MC::getX86RegNum(unsigned RegNo) {
+  switch(RegNo) {
+  case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: return N86::EAX;
+  case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: return N86::ECX;
+  case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: return N86::EDX;
+  case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: return N86::EBX;
+  case X86::RSP: case X86::ESP: case X86::SP: case X86::SPL: case X86::AH:
+    return N86::ESP;
+  case X86::RBP: case X86::EBP: case X86::BP: case X86::BPL: case X86::CH:
+    return N86::EBP;
+  case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL: case X86::DH:
+    return N86::ESI;
+  case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL: case X86::BH:
+    return N86::EDI;
+
+  case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
+    return N86::EAX;
+  case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
+    return N86::ECX;
+  case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
+    return N86::EDX;
+  case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
+    return N86::EBX;
+  case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
+    return N86::ESP;
+  case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
+    return N86::EBP;
+  case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
+    return N86::ESI;
+  case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
+    return N86::EDI;
+
+  case X86::ST0: case X86::ST1: case X86::ST2: case X86::ST3:
+  case X86::ST4: case X86::ST5: case X86::ST6: case X86::ST7:
+    return RegNo-X86::ST0;
+
+  case X86::XMM0: case X86::XMM8:
+  case X86::YMM0: case X86::YMM8: case X86::MM0:
+    return 0;
+  case X86::XMM1: case X86::XMM9:
+  case X86::YMM1: case X86::YMM9: case X86::MM1:
+    return 1;
+  case X86::XMM2: case X86::XMM10:
+  case X86::YMM2: case X86::YMM10: case X86::MM2:
+    return 2;
+  case X86::XMM3: case X86::XMM11:
+  case X86::YMM3: case X86::YMM11: case X86::MM3:
+    return 3;
+  case X86::XMM4: case X86::XMM12:
+  case X86::YMM4: case X86::YMM12: case X86::MM4:
+    return 4;
+  case X86::XMM5: case X86::XMM13:
+  case X86::YMM5: case X86::YMM13: case X86::MM5:
+    return 5;
+  case X86::XMM6: case X86::XMM14:
+  case X86::YMM6: case X86::YMM14: case X86::MM6:
+    return 6;
+  case X86::XMM7: case X86::XMM15:
+  case X86::YMM7: case X86::YMM15: case X86::MM7:
+    return 7;
+
+  case X86::ES: return 0;
+  case X86::CS: return 1;
+  case X86::SS: return 2;
+  case X86::DS: return 3;
+  case X86::FS: return 4;
+  case X86::GS: return 5;
+
+  case X86::CR0: case X86::CR8 : case X86::DR0: return 0;
+  case X86::CR1: case X86::CR9 : case X86::DR1: return 1;
+  case X86::CR2: case X86::CR10: case X86::DR2: return 2;
+  case X86::CR3: case X86::CR11: case X86::DR3: return 3;
+  case X86::CR4: case X86::CR12: case X86::DR4: return 4;
+  case X86::CR5: case X86::CR13: case X86::DR5: return 5;
+  case X86::CR6: case X86::CR14: case X86::DR6: return 6;
+  case X86::CR7: case X86::CR15: case X86::DR7: return 7;
+
+  // Pseudo index registers are equivalent to a "none"
+  // scaled index (See Intel Manual 2A, table 2-3)
+  case X86::EIZ:
+  case X86::RIZ:
+    return 4;
+
+  default:
+    assert((int(RegNo) > 0) && "Unknown physical register!");
+    return 0;
+  }
+}
+
+void X86_MC::InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI) {
+  // FIXME: TableGen these.
+  for (unsigned Reg = X86::NoRegister+1; Reg < X86::NUM_TARGET_REGS; ++Reg) {
+    int SEH = X86_MC::getX86RegNum(Reg);
+    switch (Reg) {
+    case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
+    case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
+    case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
+    case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
+    case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
+    case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
+    case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
+    case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
+    case X86::XMM8:  case X86::XMM9:  case X86::XMM10: case X86::XMM11:
+    case X86::XMM12: case X86::XMM13: case X86::XMM14: case X86::XMM15:
+    case X86::YMM8:  case X86::YMM9:  case X86::YMM10: case X86::YMM11:
+    case X86::YMM12: case X86::YMM13: case X86::YMM14: case X86::YMM15:
+      SEH += 8;
+      break;
+    }
+    MRI->mapLLVMRegToSEHReg(Reg, SEH);
+  }
+}
+
 MCSubtargetInfo *X86_MC::createX86MCSubtargetInfo(StringRef TT, StringRef CPU,
                                                   StringRef FS) {
   std::string ArchFS = X86_MC::ParseX86Triple(TT);
@@ -131,55 +273,191 @@ MCSubtargetInfo *X86_MC::createX86MCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-// Force static initialization.
-extern "C" void LLVMInitializeX86MCSubtargetInfo() {
-  TargetRegistry::RegisterMCSubtargetInfo(TheX86_32Target,
-                                          X86_MC::createX86MCSubtargetInfo);
-  TargetRegistry::RegisterMCSubtargetInfo(TheX86_64Target,
-                                          X86_MC::createX86MCSubtargetInfo);
-}
-
 static MCInstrInfo *createX86MCInstrInfo() {
   MCInstrInfo *X = new MCInstrInfo();
   InitX86MCInstrInfo(X);
   return X;
 }
 
-extern "C" void LLVMInitializeX86MCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheX86_32Target, createX86MCInstrInfo);
-  TargetRegistry::RegisterMCInstrInfo(TheX86_64Target, createX86MCInstrInfo);
-}
+static MCRegisterInfo *createX86MCRegisterInfo(StringRef TT) {
+  Triple TheTriple(TT);
+  unsigned RA = (TheTriple.getArch() == Triple::x86_64)
+    ? X86::RIP     // Should have dwarf #16.
+    : X86::EIP;    // Should have dwarf #8.
 
-static MCRegisterInfo *createX86MCRegisterInfo() {
   MCRegisterInfo *X = new MCRegisterInfo();
-  InitX86MCRegisterInfo(X);
+  InitX86MCRegisterInfo(X, RA,
+                        X86_MC::getDwarfRegFlavour(TT, false),
+                        X86_MC::getDwarfRegFlavour(TT, true));
+  X86_MC::InitLLVM2SEHRegisterMapping(X);
   return X;
 }
 
-extern "C" void LLVMInitializeX86MCRegInfo() {
-  TargetRegistry::RegisterMCRegInfo(TheX86_32Target, createX86MCRegisterInfo);
-  TargetRegistry::RegisterMCRegInfo(TheX86_64Target, createX86MCRegisterInfo);
-}
-
-
 static MCAsmInfo *createX86MCAsmInfo(const Target &T, StringRef TT) {
   Triple TheTriple(TT);
+  bool is64Bit = TheTriple.getArch() == Triple::x86_64;
 
+  MCAsmInfo *MAI;
   if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO) {
-    if (TheTriple.getArch() == Triple::x86_64)
-      return new X86_64MCAsmInfoDarwin(TheTriple);
+    if (is64Bit)
+      MAI = new X86_64MCAsmInfoDarwin(TheTriple);
     else
-      return new X86MCAsmInfoDarwin(TheTriple);
+      MAI = new X86MCAsmInfoDarwin(TheTriple);
+  } else if (TheTriple.isOSWindows()) {
+    MAI = new X86MCAsmInfoCOFF(TheTriple);
+  } else {
+    MAI = new X86ELFMCAsmInfo(TheTriple);
   }
 
-  if (TheTriple.isOSWindows())
-    return new X86MCAsmInfoCOFF(TheTriple);
+  // Initialize initial frame state.
+  // Calculate amount of bytes used for return address storing
+  int stackGrowth = is64Bit ? -8 : -4;
 
-  return new X86ELFMCAsmInfo(TheTriple);
+  // Initial state of the frame pointer is esp+stackGrowth.
+  MachineLocation Dst(MachineLocation::VirtualFP);
+  MachineLocation Src(is64Bit ? X86::RSP : X86::ESP, stackGrowth);
+  MAI->addInitialFrameState(0, Dst, Src);
+
+  // Add return address to move list
+  MachineLocation CSDst(is64Bit ? X86::RSP : X86::ESP, stackGrowth);
+  MachineLocation CSSrc(is64Bit ? X86::RIP : X86::EIP);
+  MAI->addInitialFrameState(0, CSDst, CSSrc);
+
+  return MAI;
 }
 
-extern "C" void LLVMInitializeX86MCAsmInfo() {
-  // Register the target asm info.
+static MCCodeGenInfo *createX86MCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                             CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+
+  Triple T(TT);
+  bool is64Bit = T.getArch() == Triple::x86_64;
+
+  if (RM == Reloc::Default) {
+    // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
+    // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
+    // use static relocation model by default.
+    if (T.isOSDarwin()) {
+      if (is64Bit)
+        RM = Reloc::PIC_;
+      else
+        RM = Reloc::DynamicNoPIC;
+    } else if (T.isOSWindows() && is64Bit)
+      RM = Reloc::PIC_;
+    else
+      RM = Reloc::Static;
+  }
+
+  // ELF and X86-64 don't have a distinct DynamicNoPIC model.  DynamicNoPIC
+  // is defined as a model for code which may be used in static or dynamic
+  // executables but not necessarily a shared library. On X86-32 we just
+  // compile in -static mode, in x86-64 we use PIC.
+  if (RM == Reloc::DynamicNoPIC) {
+    if (is64Bit)
+      RM = Reloc::PIC_;
+    else if (!T.isOSDarwin())
+      RM = Reloc::Static;
+  }
+
+  // If we are on Darwin, disallow static relocation model in X86-64 mode, since
+  // the Mach-O file format doesn't support it.
+  if (RM == Reloc::Static && T.isOSDarwin() && is64Bit)
+    RM = Reloc::PIC_;
+
+  // For static codegen, if we're not already set, use Small codegen.
+  if (CM == CodeModel::Default)
+    CM = CodeModel::Small;
+  else if (CM == CodeModel::JITDefault)
+    // 64-bit JIT places everything in the same buffer except external funcs.
+    CM = is64Bit ? CodeModel::Large : CodeModel::Small;
+
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+static MCStreamer *createMCStreamer(const Target &T, StringRef TT,
+                                    MCContext &Ctx, MCAsmBackend &MAB,
+                                    raw_ostream &_OS,
+                                    MCCodeEmitter *_Emitter,
+                                    bool RelaxAll,
+                                    bool NoExecStack) {
+  Triple TheTriple(TT);
+
+  if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
+    return createMachOStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll);
+
+  if (TheTriple.isOSWindows())
+    return createWinCOFFStreamer(Ctx, MAB, *_Emitter, _OS, RelaxAll);
+
+  return createELFStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll, NoExecStack);
+}
+
+static MCInstPrinter *createX86MCInstPrinter(const Target &T,
+                                             unsigned SyntaxVariant,
+                                             const MCAsmInfo &MAI,
+                                             const MCSubtargetInfo &STI) {
+  if (SyntaxVariant == 0)
+    return new X86ATTInstPrinter(MAI);
+  if (SyntaxVariant == 1)
+    return new X86IntelInstPrinter(MAI);
+  return 0;
+}
+
+static MCInstrAnalysis *createX86MCInstrAnalysis(const MCInstrInfo *Info) {
+  return new MCInstrAnalysis(Info);
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeX86TargetMC() {
+  // Register the MC asm info.
   RegisterMCAsmInfoFn A(TheX86_32Target, createX86MCAsmInfo);
   RegisterMCAsmInfoFn B(TheX86_64Target, createX86MCAsmInfo);
+
+  // Register the MC codegen info.
+  RegisterMCCodeGenInfoFn C(TheX86_32Target, createX86MCCodeGenInfo);
+  RegisterMCCodeGenInfoFn D(TheX86_64Target, createX86MCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheX86_32Target, createX86MCInstrInfo);
+  TargetRegistry::RegisterMCInstrInfo(TheX86_64Target, createX86MCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheX86_32Target, createX86MCRegisterInfo);
+  TargetRegistry::RegisterMCRegInfo(TheX86_64Target, createX86MCRegisterInfo);
+
+  // Register the MC subtarget info.
+  TargetRegistry::RegisterMCSubtargetInfo(TheX86_32Target,
+                                          X86_MC::createX86MCSubtargetInfo);
+  TargetRegistry::RegisterMCSubtargetInfo(TheX86_64Target,
+                                          X86_MC::createX86MCSubtargetInfo);
+
+  // Register the MC instruction analyzer.
+  TargetRegistry::RegisterMCInstrAnalysis(TheX86_32Target,
+                                          createX86MCInstrAnalysis);
+  TargetRegistry::RegisterMCInstrAnalysis(TheX86_64Target,
+                                          createX86MCInstrAnalysis);
+
+  // Register the code emitter.
+  TargetRegistry::RegisterMCCodeEmitter(TheX86_32Target,
+                                        createX86MCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(TheX86_64Target,
+                                        createX86MCCodeEmitter);
+
+  // Register the asm backend.
+  TargetRegistry::RegisterMCAsmBackend(TheX86_32Target,
+                                       createX86_32AsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(TheX86_64Target,
+                                       createX86_64AsmBackend);
+
+  // Register the object streamer.
+  TargetRegistry::RegisterMCObjectStreamer(TheX86_32Target,
+                                           createMCStreamer);
+  TargetRegistry::RegisterMCObjectStreamer(TheX86_64Target,
+                                           createMCStreamer);
+
+  // Register the MCInstPrinter.
+  TargetRegistry::RegisterMCInstPrinter(TheX86_32Target,
+                                        createX86MCInstPrinter);
+  TargetRegistry::RegisterMCInstPrinter(TheX86_64Target,
+                                        createX86MCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
index 89ea22b31be2..c144c513de15 100644
--- a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
@@ -14,15 +14,39 @@
 #ifndef X86MCTARGETDESC_H
 #define X86MCTARGETDESC_H
 
+#include "llvm/Support/DataTypes.h"
 #include <string>
 
 namespace llvm {
+class MCAsmBackend;
+class MCCodeEmitter;
+class MCContext;
+class MCInstrInfo;
+class MCObjectWriter;
+class MCRegisterInfo;
 class MCSubtargetInfo;
 class Target;
 class StringRef;
+class raw_ostream;
 
 extern Target TheX86_32Target, TheX86_64Target;
 
+/// DWARFFlavour - Flavour of dwarf regnumbers
+///
+namespace DWARFFlavour {
+  enum {
+    X86_64 = 0, X86_32_DarwinEH = 1, X86_32_Generic = 2
+  };
+} 
+  
+/// N86 namespace - Native X86 register numbers
+///
+namespace N86 {
+  enum {
+    EAX = 0, ECX = 1, EDX = 2, EBX = 3, ESP = 4, EBP = 5, ESI = 6, EDI = 7
+  };
+}
+
 namespace X86_MC {
   std::string ParseX86Triple(StringRef TT);
 
@@ -33,13 +57,32 @@ namespace X86_MC {
 
   void DetectFamilyModel(unsigned EAX, unsigned &Family, unsigned &Model);
 
-  /// createARMMCSubtargetInfo - Create a X86 MCSubtargetInfo instance.
+  unsigned getDwarfRegFlavour(StringRef TT, bool isEH);
+
+  unsigned getX86RegNum(unsigned RegNo);
+
+  void InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI);
+
+  /// createX86MCSubtargetInfo - Create a X86 MCSubtargetInfo instance.
   /// This is exposed so Asm parser, etc. do not need to go through
   /// TargetRegistry.
   MCSubtargetInfo *createX86MCSubtargetInfo(StringRef TT, StringRef CPU,
                                             StringRef FS);
 }
 
+MCCodeEmitter *createX86MCCodeEmitter(const MCInstrInfo &MCII,
+                                      const MCSubtargetInfo &STI,
+                                      MCContext &Ctx);
+
+MCAsmBackend *createX86_32AsmBackend(const Target &T, StringRef TT);
+MCAsmBackend *createX86_64AsmBackend(const Target &T, StringRef TT);
+
+/// createX86MachObjectWriter - Construct an X86 Mach-O object writer.
+MCObjectWriter *createX86MachObjectWriter(raw_ostream &OS,
+                                          bool Is64Bit,
+                                          uint32_t CPUType,
+                                          uint32_t CPUSubtype);
+
 } // End llvm namespace
 
 
diff --git a/contrib/llvm/lib/Target/X86/X86MachObjectWriter.cpp b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
similarity index 99%
rename from contrib/llvm/lib/Target/X86/X86MachObjectWriter.cpp
rename to contrib/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index 37110382379e..f0f1982d57f2 100644
--- a/contrib/llvm/lib/Target/X86/X86MachObjectWriter.cpp
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -7,14 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "X86.h"
-#include "X86FixupKinds.h"
-#include "llvm/ADT/Twine.h"
+#include "MCTargetDesc/X86FixupKinds.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCValue.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Object/MachOFormat.h"
 
diff --git a/contrib/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp b/contrib/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
index 08d4d84f8a8a..52a67f763b0a 100644
--- a/contrib/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
+++ b/contrib/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "X86.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheX86_32Target, llvm::TheX86_64Target;
diff --git a/contrib/llvm/lib/Target/X86/Utils/CMakeLists.txt b/contrib/llvm/lib/Target/X86/Utils/CMakeLists.txt
deleted file mode 100644
index 3ad5f991c865..000000000000
--- a/contrib/llvm/lib/Target/X86/Utils/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-add_llvm_library(LLVMX86Utils
-  X86ShuffleDecode.cpp
-  )
-add_dependencies(LLVMX86Utils X86CodeGenTable_gen)
diff --git a/contrib/llvm/lib/Target/X86/Utils/Makefile b/contrib/llvm/lib/Target/X86/Utils/Makefile
deleted file mode 100644
index 1df6f0f561d4..000000000000
--- a/contrib/llvm/lib/Target/X86/Utils/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/X86/Utils/Makefile -----------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMX86Utils
-
-# Hack: we need to include 'main' x86 target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp
index cd06060748b7..aeb3309d09aa 100644
--- a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp
+++ b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp
@@ -167,24 +167,77 @@ void DecodeUNPCKLPMask(EVT VT,
                        SmallVectorImpl<unsigned> &ShuffleMask) {
   unsigned NumElts = VT.getVectorNumElements();
 
-  // Handle vector lengths > 128 bits.  Define a "section" as a set of
-  // 128 bits.  AVX defines UNPCK* to operate independently on 128-bit
-  // sections.
-  unsigned NumSections = VT.getSizeInBits() / 128;
-  if (NumSections == 0 ) NumSections = 1;  // Handle MMX
-  unsigned NumSectionElts = NumElts / NumSections;
+  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
+  // independently on 128-bit lanes.
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  if (NumLanes == 0 ) NumLanes = 1;  // Handle MMX
+  unsigned NumLaneElts = NumElts / NumLanes;
 
   unsigned Start = 0;
-  unsigned End = NumSectionElts / 2;
-  for (unsigned s = 0; s < NumSections; ++s) {
+  unsigned End = NumLaneElts / 2;
+  for (unsigned s = 0; s < NumLanes; ++s) {
     for (unsigned i = Start; i != End; ++i) {
       ShuffleMask.push_back(i);                 // Reads from dest/src1
-      ShuffleMask.push_back(i+NumSectionElts);  // Reads from src/src2
+      ShuffleMask.push_back(i+NumLaneElts);  // Reads from src/src2
     }
     // Process the next 128 bits.
-    Start += NumSectionElts;
-    End += NumSectionElts;
+    Start += NumLaneElts;
+    End += NumLaneElts;
   }
 }
 
+// DecodeVPERMILPSMask - Decodes VPERMILPS permutes for any 128-bit 32-bit
+// elements. For 256-bit vectors, it's considered as two 128 lanes, the
+// referenced elements can't cross lanes and the mask of the first lane must
+// be the same of the second.
+void DecodeVPERMILPSMask(unsigned NumElts, unsigned Imm,
+                         SmallVectorImpl<unsigned> &ShuffleMask) {
+  unsigned NumLanes = (NumElts*32)/128;
+  unsigned LaneSize = NumElts/NumLanes;
+
+  for (unsigned l = 0; l != NumLanes; ++l) {
+    for (unsigned i = 0; i != LaneSize; ++i) {
+      unsigned Idx = (Imm >> (i*2)) & 0x3 ;
+      ShuffleMask.push_back(Idx+(l*LaneSize));
+    }
+  }
+}
+
+// DecodeVPERMILPDMask - Decodes VPERMILPD permutes for any 128-bit 64-bit
+// elements. For 256-bit vectors, it's considered as two 128 lanes, the
+// referenced elements can't cross lanes but the mask of the first lane can
+// be the different of the second (not like VPERMILPS).
+void DecodeVPERMILPDMask(unsigned NumElts, unsigned Imm,
+                         SmallVectorImpl<unsigned> &ShuffleMask) {
+  unsigned NumLanes = (NumElts*64)/128;
+  unsigned LaneSize = NumElts/NumLanes;
+
+  for (unsigned l = 0; l < NumLanes; ++l) {
+    for (unsigned i = l*LaneSize; i < LaneSize*(l+1); ++i) {
+      unsigned Idx = (Imm >> i) & 0x1;
+      ShuffleMask.push_back(Idx+(l*LaneSize));
+    }
+  }
+}
+
+void DecodeVPERM2F128Mask(EVT VT, unsigned Imm,
+                          SmallVectorImpl<unsigned> &ShuffleMask) {
+  unsigned HalfSize = VT.getVectorNumElements()/2;
+  unsigned FstHalfBegin = (Imm & 0x3) * HalfSize;
+  unsigned SndHalfBegin = ((Imm >> 4) & 0x3) * HalfSize;
+
+  for (int i = FstHalfBegin, e = FstHalfBegin+HalfSize; i != e; ++i)
+    ShuffleMask.push_back(i);
+  for (int i = SndHalfBegin, e = SndHalfBegin+HalfSize; i != e; ++i)
+    ShuffleMask.push_back(i);
+}
+
+void DecodeVPERM2F128Mask(unsigned Imm,
+                          SmallVectorImpl<unsigned> &ShuffleMask) {
+  // VPERM2F128 is used by any 256-bit EVT, but X86InstComments only
+  // has information about the instruction and not the types. So for
+  // instruction comments purpose, assume the 256-bit vector is v4i64.
+  return DecodeVPERM2F128Mask(MVT::v4i64, Imm, ShuffleMask);
+}
+
 } // llvm namespace
diff --git a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h
index b18f67033096..58193e6a4688 100644
--- a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h
+++ b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h
@@ -82,6 +82,26 @@ void DecodeUNPCKLPDMask(unsigned NElts,
 void DecodeUNPCKLPMask(EVT VT,
                        SmallVectorImpl<unsigned> &ShuffleMask);
 
+
+// DecodeVPERMILPSMask - Decodes VPERMILPS permutes for any 128-bit 32-bit
+// elements. For 256-bit vectors, it's considered as two 128 lanes, the
+// referenced elements can't cross lanes and the mask of the first lane must
+// be the same of the second.
+void DecodeVPERMILPSMask(unsigned NElts, unsigned Imm,
+                        SmallVectorImpl<unsigned> &ShuffleMask);
+
+// DecodeVPERMILPDMask - Decodes VPERMILPD permutes for any 128-bit 64-bit
+// elements. For 256-bit vectors, it's considered as two 128 lanes, the
+// referenced elements can't cross lanes but the mask of the first lane can
+// be the different of the second (not like VPERMILPS).
+void DecodeVPERMILPDMask(unsigned NElts, unsigned Imm,
+                        SmallVectorImpl<unsigned> &ShuffleMask);
+
+void DecodeVPERM2F128Mask(unsigned Imm,
+                          SmallVectorImpl<unsigned> &ShuffleMask);
+void DecodeVPERM2F128Mask(EVT VT, unsigned Imm,
+                          SmallVectorImpl<unsigned> &ShuffleMask);
+
 } // llvm namespace
 
 #endif
diff --git a/contrib/llvm/lib/Target/X86/X86.h b/contrib/llvm/lib/Target/X86/X86.h
index ec52dfb3e7d1..81e94227fca6 100644
--- a/contrib/llvm/lib/Target/X86/X86.h
+++ b/contrib/llvm/lib/Target/X86/X86.h
@@ -15,6 +15,7 @@
 #ifndef TARGET_X86_H
 #define TARGET_X86_H
 
+#include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Target/TargetMachine.h"
@@ -24,16 +25,8 @@ namespace llvm {
 class FunctionPass;
 class JITCodeEmitter;
 class MachineCodeEmitter;
-class MCCodeEmitter;
-class MCContext;
-class MCInstrInfo;
-class MCObjectWriter;
-class MCSubtargetInfo;
 class Target;
-class TargetAsmBackend;
 class X86TargetMachine;
-class formatted_raw_ostream;
-class raw_ostream;
 
 /// createX86ISelDag - This pass converts a legalized DAG into a 
 /// X86-specific DAG, ready for instruction scheduling.
@@ -51,22 +44,16 @@ FunctionPass* createGlobalBaseRegPass();
 ///
 FunctionPass *createX86FloatingPointStackifierPass();
 
-/// createSSEDomainFixPass - This pass twiddles SSE opcodes to prevent domain
-/// crossings.
-FunctionPass *createSSEDomainFixPass();
+/// createX86IssueVZeroUpperPass - This pass inserts AVX vzeroupper instructions
+/// before each call to avoid transition penalty between functions encoded with
+/// AVX and SSE.
+FunctionPass *createX86IssueVZeroUpperPass();
 
 /// createX86CodeEmitterPass - Return a pass that emits the collected X86 code
 /// to the specified MCE object.
 FunctionPass *createX86JITCodeEmitterPass(X86TargetMachine &TM,
                                           JITCodeEmitter &JCE);
 
-MCCodeEmitter *createX86MCCodeEmitter(const MCInstrInfo &MCII,
-                                      const MCSubtargetInfo &STI,
-                                      MCContext &Ctx);
-
-TargetAsmBackend *createX86_32AsmBackend(const Target &, const std::string &);
-TargetAsmBackend *createX86_64AsmBackend(const Target &, const std::string &);
-
 /// createX86EmitCodeToMemory - Returns a pass that converts a register
 /// allocated function into raw machine code in a dynamically
 /// allocated chunk of memory.
@@ -79,13 +66,6 @@ FunctionPass *createEmitX86CodeToMemory();
 ///
 FunctionPass *createX86MaxStackAlignmentHeuristicPass();
 
-
-/// createX86MachObjectWriter - Construct an X86 Mach-O object writer.
-MCObjectWriter *createX86MachObjectWriter(raw_ostream &OS,
-                                          bool Is64Bit,
-                                          uint32_t CPUType,
-                                          uint32_t CPUSubtype);
-
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/lib/Target/X86/X86.td b/contrib/llvm/lib/Target/X86/X86.td
index 4ccb43fe18cc..104b91fd3534 100644
--- a/contrib/llvm/lib/Target/X86/X86.td
+++ b/contrib/llvm/lib/Target/X86/X86.td
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This is a target description file for the Intel i386 architecture, referred to
-// here as the "X86" architecture.
+// This is a target description file for the Intel i386 architecture, referred
+// to here as the "X86" architecture.
 //
 //===----------------------------------------------------------------------===//
 
@@ -23,6 +23,9 @@ include "llvm/Target/Target.td"
 def Mode64Bit : SubtargetFeature<"64bit-mode", "In64BitMode", "true",
                                   "64-bit mode (x86_64)">;
 
+def ModeNaCl  : SubtargetFeature<"nacl-mode", "InNaClMode", "true",
+                                 "Native Client mode">;
+
 //===----------------------------------------------------------------------===//
 // X86 Subtarget features.
 //===----------------------------------------------------------------------===//
@@ -68,6 +71,9 @@ def Feature3DNowA  : SubtargetFeature<"3dnowa", "X863DNowLevel", "ThreeDNowA",
 def Feature64Bit   : SubtargetFeature<"64bit", "HasX86_64", "true",
                                       "Support 64-bit instructions",
                                       [FeatureCMOV]>;
+def FeatureCMPXCHG16B : SubtargetFeature<"cmpxchg16b", "HasCmpxchg16b", "true",
+                                      "64-bit with cmpxchg16b",
+                                      [Feature64Bit]>;
 def FeatureSlowBTMem : SubtargetFeature<"slow-bt-mem", "IsBTMemSlow", "true",
                                        "Bit testing of memory is slow">;
 def FeatureFastUAMem : SubtargetFeature<"fast-unaligned-mem",
@@ -90,6 +96,16 @@ def FeatureVectorUAMem : SubtargetFeature<"vector-unaligned-mem",
                  "Allow unaligned memory operands on vector/SIMD instructions">;
 def FeatureAES     : SubtargetFeature<"aes", "HasAES", "true",
                                       "Enable AES instructions">;
+def FeatureMOVBE   : SubtargetFeature<"movbe", "HasMOVBE", "true",
+                                      "Support MOVBE instruction">;
+def FeatureRDRAND  : SubtargetFeature<"rdrand", "HasRDRAND", "true",
+                                      "Support RDRAND instruction">;
+def FeatureF16C    : SubtargetFeature<"f16c", "HasF16C", "true",
+                       "Support 16-bit floating point conversion instructions">;
+def FeatureLZCNT   : SubtargetFeature<"lzcnt", "HasLZCNT", "true",
+                                      "Support LZCNT instruction">;
+def FeatureBMI     : SubtargetFeature<"bmi", "HasBMI", "true",
+                                      "Support BMI instructions">;
 
 //===----------------------------------------------------------------------===//
 // X86 processors supported.
@@ -112,27 +128,43 @@ def : Proc<"pentium3m",       [FeatureSSE1, FeatureSlowBTMem]>;
 def : Proc<"pentium-m",       [FeatureSSE2, FeatureSlowBTMem]>;
 def : Proc<"pentium4",        [FeatureSSE2]>;
 def : Proc<"pentium4m",       [FeatureSSE2, FeatureSlowBTMem]>;
-def : Proc<"x86-64",          [FeatureSSE2,   Feature64Bit, FeatureSlowBTMem]>;
+def : Proc<"x86-64",          [FeatureSSE2, Feature64Bit, FeatureSlowBTMem]>;
 def : Proc<"yonah",           [FeatureSSE3, FeatureSlowBTMem]>;
 def : Proc<"prescott",        [FeatureSSE3, FeatureSlowBTMem]>;
-def : Proc<"nocona",          [FeatureSSE3,   Feature64Bit, FeatureSlowBTMem]>;
-def : Proc<"core2",           [FeatureSSSE3,  Feature64Bit, FeatureSlowBTMem]>;
-def : Proc<"penryn",          [FeatureSSE41,  Feature64Bit, FeatureSlowBTMem]>;
-def : Proc<"atom",            [FeatureSSE3,   Feature64Bit, FeatureSlowBTMem]>;
+def : Proc<"nocona",          [FeatureSSE3, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem]>;
+def : Proc<"core2",           [FeatureSSSE3, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem]>;
+def : Proc<"penryn",          [FeatureSSE41, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem]>;
+def : Proc<"atom",            [FeatureSSE3, FeatureCMPXCHG16B, FeatureMOVBE,
+                               FeatureSlowBTMem]>;
 // "Arrandale" along with corei3 and corei5
-def : Proc<"corei7",          [FeatureSSE42,  Feature64Bit, FeatureSlowBTMem,
-                               FeatureFastUAMem, FeatureAES]>;
-def : Proc<"nehalem",         [FeatureSSE42,  Feature64Bit, FeatureSlowBTMem,
-                               FeatureFastUAMem]>;
+def : Proc<"corei7",          [FeatureSSE42, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem, FeatureFastUAMem, FeatureAES]>;
+def : Proc<"nehalem",         [FeatureSSE42,  FeatureCMPXCHG16B,
+                               FeatureSlowBTMem, FeatureFastUAMem]>;
 // Westmere is a similar machine to nehalem with some additional features.
 // Westmere is the corei3/i5/i7 path from nehalem to sandybridge
-def : Proc<"westmere",        [FeatureSSE42,  Feature64Bit, FeatureSlowBTMem,
-                               FeatureFastUAMem, FeatureAES, FeatureCLMUL]>;
+def : Proc<"westmere",        [FeatureSSE42, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem, FeatureFastUAMem, FeatureAES,
+                               FeatureCLMUL]>;
+// Sandy Bridge
 // SSE is not listed here since llvm treats AVX as a reimplementation of SSE,
 // rather than a superset.
 // FIXME: Disabling AVX for now since it's not ready.
-def : Proc<"corei7-avx",      [FeatureSSE42, Feature64Bit,
+def : Proc<"corei7-avx",      [FeatureSSE42, FeatureCMPXCHG16B,
                                FeatureAES, FeatureCLMUL]>;
+// Ivy Bridge
+def : Proc<"core-avx-i",      [FeatureSSE42, FeatureCMPXCHG16B,
+                               FeatureAES, FeatureCLMUL,
+                               FeatureRDRAND, FeatureF16C]>;
+
+// Haswell
+def : Proc<"core-avx2",       [FeatureSSE42, FeatureCMPXCHG16B, FeatureAES,
+                               FeatureCLMUL, FeatureRDRAND, FeatureF16C,
+                               FeatureFMA3, FeatureMOVBE, FeatureLZCNT,
+                               FeatureBMI]>;
 
 def : Proc<"k6",              [FeatureMMX]>;
 def : Proc<"k6-2",            [Feature3DNow]>;
@@ -150,19 +182,21 @@ def : Proc<"athlon64",        [FeatureSSE2,   Feature3DNowA, Feature64Bit,
                                FeatureSlowBTMem]>;
 def : Proc<"athlon-fx",       [FeatureSSE2,   Feature3DNowA, Feature64Bit,
                                FeatureSlowBTMem]>;
-def : Proc<"k8-sse3",         [FeatureSSE3,   Feature3DNowA, Feature64Bit,
+def : Proc<"k8-sse3",         [FeatureSSE3,   Feature3DNowA, FeatureCMPXCHG16B,
                                FeatureSlowBTMem]>;
-def : Proc<"opteron-sse3",    [FeatureSSE3,   Feature3DNowA, Feature64Bit,
+def : Proc<"opteron-sse3",    [FeatureSSE3,   Feature3DNowA, FeatureCMPXCHG16B,
                                FeatureSlowBTMem]>;
-def : Proc<"athlon64-sse3",   [FeatureSSE3,   Feature3DNowA, Feature64Bit,
+def : Proc<"athlon64-sse3",   [FeatureSSE3,   Feature3DNowA, FeatureCMPXCHG16B,
                                FeatureSlowBTMem]>;
 def : Proc<"amdfam10",        [FeatureSSE3,   FeatureSSE4A,
-                               Feature3DNowA, Feature64Bit, FeatureSlowBTMem]>;
+                               Feature3DNowA, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem]>;
 def : Proc<"barcelona",       [FeatureSSE3,   FeatureSSE4A,
-                               Feature3DNowA, Feature64Bit, FeatureSlowBTMem]>;
-def : Proc<"istanbul",        [Feature3DNowA, Feature64Bit, FeatureSSE4A,
-                               Feature3DNowA]>;
-def : Proc<"shanghai",        [Feature3DNowA, Feature64Bit, FeatureSSE4A,
+                               Feature3DNowA, FeatureCMPXCHG16B,
+                               FeatureSlowBTMem]>;
+def : Proc<"istanbul",        [Feature3DNowA, FeatureCMPXCHG16B,
+                               FeatureSSE4A, Feature3DNowA]>;
+def : Proc<"shanghai",        [Feature3DNowA, FeatureCMPXCHG16B, FeatureSSE4A,
                                Feature3DNowA]>;
 
 def : Proc<"winchip-c6",      [FeatureMMX]>;
diff --git a/contrib/llvm/lib/Target/X86/X86AsmPrinter.cpp b/contrib/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 99b4479a9fc9..4c3ff02826b0 100644
--- a/contrib/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -35,12 +35,12 @@
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
+#include "llvm/Target/Mangler.h"
+#include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/COFF.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 using namespace llvm;
 
@@ -504,8 +504,8 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) {
         //   .indirect_symbol _foo
         OutStreamer.EmitSymbolAttribute(Stubs[i].second.getPointer(),
                                         MCSA_IndirectSymbol);
-        // hlt; hlt; hlt; hlt; hlt     hlt = 0xf4 = -12.
-        const char HltInsts[] = { -12, -12, -12, -12, -12 };
+        // hlt; hlt; hlt; hlt; hlt     hlt = 0xf4.
+        const char HltInsts[] = "\xf4\xf4\xf4\xf4\xf4";
         OutStreamer.EmitBytes(StringRef(HltInsts, 5), 0/*addrspace*/);
       }
 
@@ -708,21 +708,8 @@ void X86AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
 // Target Registry Stuff
 //===----------------------------------------------------------------------===//
 
-static MCInstPrinter *createX86MCInstPrinter(const Target &T,
-                                             unsigned SyntaxVariant,
-                                             const MCAsmInfo &MAI) {
-  if (SyntaxVariant == 0)
-    return new X86ATTInstPrinter(MAI);
-  if (SyntaxVariant == 1)
-    return new X86IntelInstPrinter(MAI);
-  return 0;
-}
-
 // Force static initialization.
 extern "C" void LLVMInitializeX86AsmPrinter() {
   RegisterAsmPrinter<X86AsmPrinter> X(TheX86_32Target);
   RegisterAsmPrinter<X86AsmPrinter> Y(TheX86_64Target);
-
-  TargetRegistry::RegisterMCInstPrinter(TheX86_32Target,createX86MCInstPrinter);
-  TargetRegistry::RegisterMCInstPrinter(TheX86_64Target,createX86MCInstPrinter);
 }
diff --git a/contrib/llvm/lib/Target/X86/X86CodeEmitter.cpp b/contrib/llvm/lib/Target/X86/X86CodeEmitter.cpp
index 4b11db7c0331..aeff03a89ec9 100644
--- a/contrib/llvm/lib/Target/X86/X86CodeEmitter.cpp
+++ b/contrib/llvm/lib/Target/X86/X86CodeEmitter.cpp
@@ -98,8 +98,6 @@ namespace {
     void emitMemModRMByte(const MachineInstr &MI,
                           unsigned Op, unsigned RegOpcodeField,
                           intptr_t PCAdj = 0);
-
-    unsigned getX86RegNum(unsigned RegNo) const;
   };
 
 template<class CodeEmitter>
@@ -169,7 +167,7 @@ static unsigned determineREX(const MachineInstr &MI) {
       const MachineOperand& MO = MI.getOperand(i);
       if (MO.isReg()) {
         unsigned Reg = MO.getReg();
-        if (X86InstrInfo::isX86_64NonExtLowByteReg(Reg))
+        if (X86II::isX86_64NonExtLowByteReg(Reg))
           REX |= 0x40;
       }
     }
@@ -346,11 +344,6 @@ void Emitter<CodeEmitter>::emitJumpTableAddress(unsigned JTI, unsigned Reloc,
     MCE.emitWordLE(0);
 }
 
-template<class CodeEmitter>
-unsigned Emitter<CodeEmitter>::getX86RegNum(unsigned RegNo) const {
-  return X86RegisterInfo::getX86RegNum(RegNo);
-}
-
 inline static unsigned char ModRMByte(unsigned Mod, unsigned RegOpcode,
                                       unsigned RM) {
   assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
@@ -360,7 +353,7 @@ inline static unsigned char ModRMByte(unsigned Mod, unsigned RegOpcode,
 template<class CodeEmitter>
 void Emitter<CodeEmitter>::emitRegModRMByte(unsigned ModRMReg,
                                             unsigned RegOpcodeFld){
-  MCE.emitByte(ModRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)));
+  MCE.emitByte(ModRMByte(3, RegOpcodeFld, X86_MC::getX86RegNum(ModRMReg)));
 }
 
 template<class CodeEmitter>
@@ -498,7 +491,7 @@ void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
   // 2-7) and absolute references.
   unsigned BaseRegNo = -1U;
   if (BaseReg != 0 && BaseReg != X86::RIP)
-    BaseRegNo = getX86RegNum(BaseReg);
+    BaseRegNo = X86_MC::getX86RegNum(BaseReg);
 
   if (// The SIB byte must be used if there is an index register.
       IndexReg.getReg() == 0 && 
@@ -566,7 +559,7 @@ void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
   }
 
   // Calculate what the SS field value should be...
-  static const unsigned SSTable[] = { ~0, 0, 1, ~0, 2, ~0, ~0, ~0, 3 };
+  static const unsigned SSTable[] = { ~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3 };
   unsigned SS = SSTable[Scale.getImm()];
 
   if (BaseReg == 0) {
@@ -574,15 +567,15 @@ void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
     // Manual 2A, table 2-7. The displacement has already been output.
     unsigned IndexRegNo;
     if (IndexReg.getReg())
-      IndexRegNo = getX86RegNum(IndexReg.getReg());
+      IndexRegNo = X86_MC::getX86RegNum(IndexReg.getReg());
     else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
       IndexRegNo = 4;
     emitSIBByte(SS, IndexRegNo, 5);
   } else {
-    unsigned BaseRegNo = getX86RegNum(BaseReg);
+    unsigned BaseRegNo = X86_MC::getX86RegNum(BaseReg);
     unsigned IndexRegNo;
     if (IndexReg.getReg())
-      IndexRegNo = getX86RegNum(IndexReg.getReg());
+      IndexRegNo = X86_MC::getX86RegNum(IndexReg.getReg());
     else
       IndexRegNo = 4;   // For example [ESP+1*<noreg>+4]
     emitSIBByte(SS, IndexRegNo, BaseRegNo);
@@ -809,7 +802,8 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
   }
       
   case X86II::AddRegFrm: {
-    MCE.emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++).getReg()));
+    MCE.emitByte(BaseOpcode +
+                 X86_MC::getX86RegNum(MI.getOperand(CurOp++).getReg()));
     
     if (CurOp == NumOps)
       break;
@@ -844,7 +838,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
   case X86II::MRMDestReg: {
     MCE.emitByte(BaseOpcode);
     emitRegModRMByte(MI.getOperand(CurOp).getReg(),
-                     getX86RegNum(MI.getOperand(CurOp+1).getReg()));
+                     X86_MC::getX86RegNum(MI.getOperand(CurOp+1).getReg()));
     CurOp += 2;
     if (CurOp != NumOps)
       emitConstant(MI.getOperand(CurOp++).getImm(),
@@ -854,7 +848,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
   case X86II::MRMDestMem: {
     MCE.emitByte(BaseOpcode);
     emitMemModRMByte(MI, CurOp,
-                     getX86RegNum(MI.getOperand(CurOp + X86::AddrNumOperands)
+                X86_MC::getX86RegNum(MI.getOperand(CurOp + X86::AddrNumOperands)
                                   .getReg()));
     CurOp +=  X86::AddrNumOperands + 1;
     if (CurOp != NumOps)
@@ -866,7 +860,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
   case X86II::MRMSrcReg:
     MCE.emitByte(BaseOpcode);
     emitRegModRMByte(MI.getOperand(CurOp+1).getReg(),
-                     getX86RegNum(MI.getOperand(CurOp).getReg()));
+                     X86_MC::getX86RegNum(MI.getOperand(CurOp).getReg()));
     CurOp += 2;
     if (CurOp != NumOps)
       emitConstant(MI.getOperand(CurOp++).getImm(),
@@ -880,8 +874,8 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
       X86II::getSizeOfImm(Desc->TSFlags) : 0;
 
     MCE.emitByte(BaseOpcode);
-    emitMemModRMByte(MI, CurOp+1, getX86RegNum(MI.getOperand(CurOp).getReg()),
-                     PCAdj);
+    emitMemModRMByte(MI, CurOp+1,
+                     X86_MC::getX86RegNum(MI.getOperand(CurOp).getReg()),PCAdj);
     CurOp += AddrOperands + 1;
     if (CurOp != NumOps)
       emitConstant(MI.getOperand(CurOp++).getImm(),
@@ -968,7 +962,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
     MCE.emitByte(BaseOpcode);
     // Duplicate register, used by things like MOV8r0 (aka xor reg,reg).
     emitRegModRMByte(MI.getOperand(CurOp).getReg(),
-                     getX86RegNum(MI.getOperand(CurOp).getReg()));
+                     X86_MC::getX86RegNum(MI.getOperand(CurOp).getReg()));
     ++CurOp;
     break;
       
diff --git a/contrib/llvm/lib/Target/X86/X86ELFWriterInfo.cpp b/contrib/llvm/lib/Target/X86/X86ELFWriterInfo.cpp
index f1d7edea7210..4a72d154c335 100644
--- a/contrib/llvm/lib/Target/X86/X86ELFWriterInfo.cpp
+++ b/contrib/llvm/lib/Target/X86/X86ELFWriterInfo.cpp
@@ -147,7 +147,7 @@ long int X86ELFWriterInfo::computeRelocation(unsigned SymOffset,
   if (RelTy == ELF::R_X86_64_PC32 || RelTy == ELF::R_386_PC32)
     return SymOffset - (RelOffset + 4);
   else
-    assert("computeRelocation unknown for this relocation type");
+    assert(0 && "computeRelocation unknown for this relocation type");
 
   return 0;
 }
diff --git a/contrib/llvm/lib/Target/X86/X86FastISel.cpp b/contrib/llvm/lib/Target/X86/X86FastISel.cpp
index 21e163a30054..f912b28eb47b 100644
--- a/contrib/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/contrib/llvm/lib/Target/X86/X86FastISel.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CallingConv.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/GlobalVariable.h"
+#include "llvm/GlobalAlias.h"
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Operator.h"
@@ -59,8 +60,8 @@ class X86FastISel : public FastISel {
   explicit X86FastISel(FunctionLoweringInfo &funcInfo) : FastISel(funcInfo) {
     Subtarget = &TM.getSubtarget<X86Subtarget>();
     StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
-    X86ScalarSSEf64 = Subtarget->hasSSE2();
-    X86ScalarSSEf32 = Subtarget->hasSSE1();
+    X86ScalarSSEf64 = Subtarget->hasSSE2() || Subtarget->hasAVX();
+    X86ScalarSSEf32 = Subtarget->hasSSE1() || Subtarget->hasAVX();
   }
 
   virtual bool TargetSelectInstruction(const Instruction *I);
@@ -134,7 +135,7 @@ class X86FastISel : public FastISel {
       (VT == MVT::f32 && X86ScalarSSEf32);   // f32 is when SSE1
   }
 
-  bool isTypeLegal(const Type *Ty, MVT &VT, bool AllowI1 = false);
+  bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
 
   bool IsMemcpySmall(uint64_t Len);
 
@@ -144,7 +145,7 @@ class X86FastISel : public FastISel {
 
 } // end anonymous namespace.
 
-bool X86FastISel::isTypeLegal(const Type *Ty, MVT &VT, bool AllowI1) {
+bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
   EVT evt = TLI.getValueType(Ty, /*HandleUnknown=*/true);
   if (evt == MVT::Other || !evt.isSimple())
     // Unhandled type. Halt "fast" selection and bail.
@@ -198,8 +199,8 @@ bool X86FastISel::X86FastEmitLoad(EVT VT, const X86AddressMode &AM,
     RC  = X86::GR64RegisterClass;
     break;
   case MVT::f32:
-    if (Subtarget->hasSSE1()) {
-      Opc = X86::MOVSSrm;
+    if (X86ScalarSSEf32) {
+      Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
       RC  = X86::FR32RegisterClass;
     } else {
       Opc = X86::LD_Fp32m;
@@ -207,8 +208,8 @@ bool X86FastISel::X86FastEmitLoad(EVT VT, const X86AddressMode &AM,
     }
     break;
   case MVT::f64:
-    if (Subtarget->hasSSE2()) {
-      Opc = X86::MOVSDrm;
+    if (X86ScalarSSEf64) {
+      Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
       RC  = X86::FR64RegisterClass;
     } else {
       Opc = X86::LD_Fp64m;
@@ -250,10 +251,12 @@ X86FastISel::X86FastEmitStore(EVT VT, unsigned Val, const X86AddressMode &AM) {
   case MVT::i32: Opc = X86::MOV32mr; break;
   case MVT::i64: Opc = X86::MOV64mr; break; // Must be in x86-64 mode.
   case MVT::f32:
-    Opc = Subtarget->hasSSE1() ? X86::MOVSSmr : X86::ST_Fp32m;
+    Opc = X86ScalarSSEf32 ?
+          (Subtarget->hasAVX() ? X86::VMOVSSmr : X86::MOVSSmr) : X86::ST_Fp32m;
     break;
   case MVT::f64:
-    Opc = Subtarget->hasSSE2() ? X86::MOVSDmr : X86::ST_Fp64m;
+    Opc = X86ScalarSSEf64 ?
+          (Subtarget->hasAVX() ? X86::VMOVSDmr : X86::MOVSDmr) : X86::ST_Fp64m;
     break;
   }
 
@@ -336,7 +339,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
     U = C;
   }
 
-  if (const PointerType *Ty = dyn_cast<PointerType>(V->getType()))
+  if (PointerType *Ty = dyn_cast<PointerType>(V->getType()))
     if (Ty->getAddressSpace() > 255)
       // Fast instruction selection doesn't support the special
       // address spaces.
@@ -399,7 +402,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
     for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
          i != e; ++i, ++GTI) {
       const Value *Op = *i;
-      if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
         const StructLayout *SL = TD.getStructLayout(STy);
         Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue());
         continue;
@@ -465,14 +468,23 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
 
   // Handle constant address.
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
-    // Can't handle alternate code models or TLS yet.
+    // Can't handle alternate code models yet.
     if (TM.getCodeModel() != CodeModel::Small)
       return false;
 
+    // Can't handle TLS yet.
     if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
       if (GVar->isThreadLocal())
         return false;
 
+    // Can't handle TLS yet, part 2 (this is slightly crazy, but this is how
+    // it works...).
+    if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
+      if (const GlobalVariable *GVar =
+            dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)))
+        if (GVar->isThreadLocal())
+          return false;
+
     // RIP-relative addresses can't have additional register operands, so if
     // we've already folded stuff into the addressing mode, just force the
     // global value into its own register, which we can use as the basereg.
@@ -658,6 +670,10 @@ bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
 
 /// X86SelectStore - Select and emit code to implement store instructions.
 bool X86FastISel::X86SelectStore(const Instruction *I) {
+  // Atomic stores need special handling.
+  if (cast<StoreInst>(I)->isAtomic())
+    return false;
+
   MVT VT;
   if (!isTypeLegal(I->getOperand(0)->getType(), VT, /*AllowI1=*/true))
     return false;
@@ -780,6 +796,10 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
 /// X86SelectLoad - Select and emit code to implement load instructions.
 ///
 bool X86FastISel::X86SelectLoad(const Instruction *I)  {
+  // Atomic loads need special handling.
+  if (cast<LoadInst>(I)->isAtomic())
+    return false;
+
   MVT VT;
   if (!isTypeLegal(I->getType(), VT, /*AllowI1=*/true))
     return false;
@@ -797,14 +817,20 @@ bool X86FastISel::X86SelectLoad(const Instruction *I)  {
 }
 
 static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {
+  bool HasAVX = Subtarget->hasAVX();
+  bool X86ScalarSSEf32 = HasAVX || Subtarget->hasSSE1();
+  bool X86ScalarSSEf64 = HasAVX || Subtarget->hasSSE2();
+
   switch (VT.getSimpleVT().SimpleTy) {
   default:       return 0;
   case MVT::i8:  return X86::CMP8rr;
   case MVT::i16: return X86::CMP16rr;
   case MVT::i32: return X86::CMP32rr;
   case MVT::i64: return X86::CMP64rr;
-  case MVT::f32: return Subtarget->hasSSE1() ? X86::UCOMISSrr : 0;
-  case MVT::f64: return Subtarget->hasSSE2() ? X86::UCOMISDrr : 0;
+  case MVT::f32:
+    return X86ScalarSSEf32 ? (HasAVX ? X86::VUCOMISSrr : X86::UCOMISSrr) : 0;
+  case MVT::f64:
+    return X86ScalarSSEf64 ? (HasAVX ? X86::VUCOMISDrr : X86::UCOMISDrr) : 0;
   }
 }
 
@@ -1207,7 +1233,7 @@ bool X86FastISel::X86SelectSelect(const Instruction *I) {
 
 bool X86FastISel::X86SelectFPExt(const Instruction *I) {
   // fpext from float to double.
-  if (Subtarget->hasSSE2() &&
+  if (X86ScalarSSEf64 &&
       I->getType()->isDoubleTy()) {
     const Value *V = I->getOperand(0);
     if (V->getType()->isFloatTy()) {
@@ -1226,7 +1252,7 @@ bool X86FastISel::X86SelectFPExt(const Instruction *I) {
 }
 
 bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
-  if (Subtarget->hasSSE2()) {
+  if (X86ScalarSSEf64) {
     if (I->getType()->isFloatTy()) {
       const Value *V = I->getOperand(0);
       if (V->getType()->isDoubleTy()) {
@@ -1365,6 +1391,9 @@ bool X86FastISel::X86VisitIntrinsicCall(const IntrinsicInst &I) {
   case Intrinsic::memset: {
     const MemSetInst &MSI = cast<MemSetInst>(I);
 
+    if (MSI.isVolatile())
+      return false;
+
     unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
     if (!MSI.getLength()->getType()->isIntegerTy(SizeWidth))
       return false;
@@ -1411,7 +1440,7 @@ bool X86FastISel::X86VisitIntrinsicCall(const IntrinsicInst &I) {
     // Replace "add with overflow" intrinsics with an "add" instruction followed
     // by a seto/setc instruction.
     const Function *Callee = I.getCalledFunction();
-    const Type *RetTy =
+    Type *RetTy =
       cast<StructType>(Callee->getReturnType())->getTypeAtIndex(unsigned(0));
 
     MVT VT;
@@ -1484,8 +1513,8 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
   if (CC == CallingConv::Fast && GuaranteedTailCallOpt)
     return false;
 
-  const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
-  const FunctionType *FTy = cast<FunctionType>(PT->getElementType());
+  PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
+  FunctionType *FTy = cast<FunctionType>(PT->getElementType());
   bool isVarArg = FTy->isVarArg();
 
   // Don't know how to handle Win64 varargs yet.  Nothing special needed for
@@ -1547,8 +1576,8 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
       Flags.setZExt();
 
     if (CS.paramHasAttr(AttrInd, Attribute::ByVal)) {
-      const PointerType *Ty = cast<PointerType>(ArgVal->getType());
-      const Type *ElementTy = Ty->getElementType();
+      PointerType *Ty = cast<PointerType>(ArgVal->getType());
+      Type *ElementTy = Ty->getElementType();
       unsigned FrameSize = TD.getTypeAllocSize(ElementTy);
       unsigned FrameAlign = CS.getParamAlignment(AttrInd);
       if (!FrameAlign)
@@ -1600,7 +1629,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
 
     if (ArgReg == 0) return false;
 
-    const Type *ArgTy = ArgVal->getType();
+    Type *ArgTy = ArgVal->getType();
     MVT ArgVT;
     if (!isTypeLegal(ArgTy, ArgVT))
       return false;
@@ -1709,7 +1738,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
         assert(Res && "memcpy length already checked!"); (void)Res;
       } else if (isa<ConstantInt>(ArgVal) || isa<ConstantPointerNull>(ArgVal)) {
         // If this is a really simple value, emit this with the Value* version
-        //of X86FastEmitStore.  If it isn't simple, we don't want to do this,
+        // of X86FastEmitStore.  If it isn't simple, we don't want to do this,
         // as it can cause us to reevaluate the argument.
         X86FastEmitStore(ArgVT, ArgVal, AM);
       } else {
@@ -1965,8 +1994,8 @@ unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) {
     RC  = X86::GR64RegisterClass;
     break;
   case MVT::f32:
-    if (Subtarget->hasSSE1()) {
-      Opc = X86::MOVSSrm;
+    if (X86ScalarSSEf32) {
+      Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
       RC  = X86::FR32RegisterClass;
     } else {
       Opc = X86::LD_Fp32m;
@@ -1974,8 +2003,8 @@ unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) {
     }
     break;
   case MVT::f64:
-    if (Subtarget->hasSSE2()) {
-      Opc = X86::MOVSDrm;
+    if (X86ScalarSSEf64) {
+      Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
       RC  = X86::FR64RegisterClass;
     } else {
       Opc = X86::LD_Fp64m;
@@ -2070,8 +2099,8 @@ unsigned X86FastISel::TargetMaterializeFloatZero(const ConstantFP *CF) {
   switch (VT.SimpleTy) {
     default: return false;
     case MVT::f32:
-      if (Subtarget->hasSSE1()) {
-        Opc = X86::FsFLD0SS;
+      if (X86ScalarSSEf32) {
+        Opc = Subtarget->hasAVX() ? X86::VFsFLD0SS : X86::FsFLD0SS;
         RC  = X86::FR32RegisterClass;
       } else {
         Opc = X86::LD_Fp032;
@@ -2079,8 +2108,8 @@ unsigned X86FastISel::TargetMaterializeFloatZero(const ConstantFP *CF) {
       }
       break;
     case MVT::f64:
-      if (Subtarget->hasSSE2()) {
-        Opc = X86::FsFLD0SD;
+      if (X86ScalarSSEf64) {
+        Opc = Subtarget->hasAVX() ? X86::VFsFLD0SD : X86::FsFLD0SD;
         RC  = X86::FR64RegisterClass;
       } else {
         Opc = X86::LD_Fp064;
diff --git a/contrib/llvm/lib/Target/X86/X86FloatingPoint.cpp b/contrib/llvm/lib/Target/X86/X86FloatingPoint.cpp
index 6eed6abd43e2..e3461c82c7a6 100644
--- a/contrib/llvm/lib/Target/X86/X86FloatingPoint.cpp
+++ b/contrib/llvm/lib/Target/X86/X86FloatingPoint.cpp
@@ -260,6 +260,21 @@ namespace {
       BuildMI(*MBB, I, dl, TII->get(X86::LD_Frr)).addReg(STReg);
     }
 
+    /// duplicatePendingSTBeforeKill - The instruction at I is about to kill
+    /// RegNo. If any PendingST registers still need the RegNo value, duplicate
+    /// them to new scratch registers.
+    void duplicatePendingSTBeforeKill(unsigned RegNo, MachineInstr *I) {
+      for (unsigned i = 0; i != NumPendingSTs; ++i) {
+        if (PendingST[i] != RegNo)
+          continue;
+        unsigned SR = getScratchReg();
+        DEBUG(dbgs() << "Duplicating pending ST" << i
+                     << " in FP" << RegNo << " to FP" << SR << '\n');
+        duplicateToTop(RegNo, SR, I);
+        PendingST[i] = SR;
+      }
+    }
+
     /// popStackAfter - Pop the current value off of the top of the FP stack
     /// after the specified instruction.
     void popStackAfter(MachineBasicBlock::iterator &I);
@@ -406,6 +421,10 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) {
     if (MI->isCopy() && isFPCopy(MI))
       FPInstClass = X86II::SpecialFP;
 
+    if (MI->isImplicitDef() &&
+        X86::RFP80RegClass.contains(MI->getOperand(0).getReg()))
+      FPInstClass = X86II::SpecialFP;
+
     if (FPInstClass == X86II::NotFP)
       continue;  // Efficiently ignore non-fp insts!
 
@@ -461,6 +480,7 @@ bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) {
       }
       dumpStack();
     );
+    (void)PrevMI;
 
     Changed = true;
   }
@@ -969,6 +989,9 @@ void FPS::handleOneArgFP(MachineBasicBlock::iterator &I) {
   unsigned Reg = getFPReg(MI->getOperand(NumOps-1));
   bool KillsSrc = MI->killsRegister(X86::FP0+Reg);
 
+  if (KillsSrc)
+    duplicatePendingSTBeforeKill(Reg, I);
+
   // FISTP64m is strange because there isn't a non-popping versions.
   // If we have one _and_ we don't want to pop the operand, duplicate the value
   // on the stack instead of moving it.  This ensure that popping the value is
@@ -1032,6 +1055,7 @@ void FPS::handleOneArgFPRW(MachineBasicBlock::iterator &I) {
   bool KillsSrc = MI->killsRegister(X86::FP0+Reg);
 
   if (KillsSrc) {
+    duplicatePendingSTBeforeKill(Reg, I);
     // If this is the last use of the source register, just make sure it's on
     // the top of the stack.
     moveToTop(Reg, I);
@@ -1318,6 +1342,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) {
 
       // When the source is killed, allocate a scratch FP register.
       if (KillsSrc) {
+        duplicatePendingSTBeforeKill(SrcFP, I);
         unsigned Slot = getSlot(SrcFP);
         unsigned SR = getScratchReg();
         PendingST[DstST] = SR;
@@ -1369,6 +1394,15 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &I) {
     break;
   }
 
+  case TargetOpcode::IMPLICIT_DEF: {
+    // All FP registers must be explicitly defined, so load a 0 instead.
+    unsigned Reg = MI->getOperand(0).getReg() - X86::FP0;
+    DEBUG(dbgs() << "Emitting LD_F0 for implicit FP" << Reg << '\n');
+    BuildMI(*MBB, I, MI->getDebugLoc(), TII->get(X86::LD_F0));
+    pushReg(Reg);
+    break;
+  }
+
   case X86::FpPOP_RETVAL: {
     // The FpPOP_RETVAL instruction is used after calls that return a value on
     // the floating point stack. We cannot model this with ST defs since CALL
diff --git a/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp b/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp
index ed45a9a4c1c0..d54f4ae2a2c8 100644
--- a/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -15,6 +15,7 @@
 #include "X86InstrBuilder.h"
 #include "X86InstrInfo.h"
 #include "X86MachineFunctionInfo.h"
+#include "X86Subtarget.h"
 #include "X86TargetMachine.h"
 #include "llvm/Function.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -91,12 +92,12 @@ static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
     return 0;
 
   static const unsigned CallerSavedRegs32Bit[] = {
-    X86::EAX, X86::EDX, X86::ECX
+    X86::EAX, X86::EDX, X86::ECX, 0
   };
 
   static const unsigned CallerSavedRegs64Bit[] = {
     X86::RAX, X86::RDX, X86::RCX, X86::RSI, X86::RDI,
-    X86::R8,  X86::R9,  X86::R10, X86::R11
+    X86::R8,  X86::R9,  X86::R10, X86::R11, 0
   };
 
   unsigned Opc = MBBI->getOpcode();
@@ -283,8 +284,8 @@ static bool isEAXLiveIn(MachineFunction &MF) {
 }
 
 void X86FrameLowering::emitCalleeSavedFrameMoves(MachineFunction &MF,
-                                             MCSymbol *Label,
-                                             unsigned FramePtr) const {
+                                                 MCSymbol *Label,
+                                                 unsigned FramePtr) const {
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo &MMI = MF.getMMI();
 
@@ -346,6 +347,247 @@ void X86FrameLowering::emitCalleeSavedFrameMoves(MachineFunction &MF,
   }
 }
 
+/// getCompactUnwindRegNum - Get the compact unwind number for a given
+/// register. The number corresponds to the enum lists in
+/// compact_unwind_encoding.h.
+static int getCompactUnwindRegNum(const unsigned *CURegs, unsigned Reg) {
+  int Idx = 1;
+  for (; *CURegs; ++CURegs, ++Idx)
+    if (*CURegs == Reg)
+      return Idx;
+
+  return -1;
+}
+
+/// encodeCompactUnwindRegistersWithoutFrame - Create the permutation encoding
+/// used with frameless stacks. It is passed the number of registers to be saved
+/// and an array of the registers saved.
+static uint32_t encodeCompactUnwindRegistersWithoutFrame(unsigned SavedRegs[6],
+                                                         unsigned RegCount,
+                                                         bool Is64Bit) {
+  // The saved registers are numbered from 1 to 6. In order to encode the order
+  // in which they were saved, we re-number them according to their place in the
+  // register order. The re-numbering is relative to the last re-numbered
+  // register. E.g., if we have registers {6, 2, 4, 5} saved in that order:
+  //
+  //    Orig  Re-Num
+  //    ----  ------
+  //     6       6
+  //     2       2
+  //     4       3
+  //     5       3
+  //
+  static const unsigned CU32BitRegs[] = {
+    X86::EBX, X86::ECX, X86::EDX, X86::EDI, X86::ESI, X86::EBP, 0
+  };
+  static const unsigned CU64BitRegs[] = {
+    X86::RBX, X86::R12, X86::R13, X86::R14, X86::R15, X86::RBP, 0
+  };
+  const unsigned *CURegs = (Is64Bit ? CU64BitRegs : CU32BitRegs);
+
+  uint32_t RenumRegs[6];
+  for (unsigned i = 6 - RegCount; i < 6; ++i) {
+    int CUReg = getCompactUnwindRegNum(CURegs, SavedRegs[i]);
+    if (CUReg == -1) return ~0U;
+    SavedRegs[i] = CUReg;
+
+    unsigned Countless = 0;
+    for (unsigned j = 6 - RegCount; j < i; ++j)
+      if (SavedRegs[j] < SavedRegs[i])
+        ++Countless;
+
+    RenumRegs[i] = SavedRegs[i] - Countless - 1;
+  }
+
+  // Take the renumbered values and encode them into a 10-bit number.
+  uint32_t permutationEncoding = 0;
+  switch (RegCount) {
+  case 6:
+    permutationEncoding |= 120 * RenumRegs[0] + 24 * RenumRegs[1]
+                           + 6 * RenumRegs[2] +  2 * RenumRegs[3]
+                           +     RenumRegs[4];
+    break;
+  case 5:
+    permutationEncoding |= 120 * RenumRegs[1] + 24 * RenumRegs[2]
+                           + 6 * RenumRegs[3] +  2 * RenumRegs[4]
+                           +     RenumRegs[5];
+    break;
+  case 4:
+    permutationEncoding |=  60 * RenumRegs[2] + 12 * RenumRegs[3]
+                           + 3 * RenumRegs[4] +      RenumRegs[5];
+    break;
+  case 3:
+    permutationEncoding |=  20 * RenumRegs[3] +  4 * RenumRegs[4]
+                           +     RenumRegs[5];
+    break;
+  case 2:
+    permutationEncoding |=   5 * RenumRegs[4] +      RenumRegs[5];
+    break;
+  case 1:
+    permutationEncoding |=       RenumRegs[5];
+    break;
+  }
+
+  assert((permutationEncoding & 0x3FF) == permutationEncoding &&
+         "Invalid compact register encoding!");
+  return permutationEncoding;
+}
+
+/// encodeCompactUnwindRegistersWithFrame - Return the registers encoded for a
+/// compact encoding with a frame pointer.
+static uint32_t encodeCompactUnwindRegistersWithFrame(unsigned SavedRegs[6],
+                                                      bool Is64Bit) {
+  static const unsigned CU32BitRegs[] = {
+    X86::EBX, X86::ECX, X86::EDX, X86::EDI, X86::ESI, X86::EBP, 0
+  };
+  static const unsigned CU64BitRegs[] = {
+    X86::RBX, X86::R12, X86::R13, X86::R14, X86::R15, X86::RBP, 0
+  };
+  const unsigned *CURegs = (Is64Bit ? CU64BitRegs : CU32BitRegs);
+
+  // Encode the registers in the order they were saved, 3-bits per register. The
+  // registers are numbered from 1 to 6.
+  uint32_t RegEnc = 0;
+  for (int I = 5; I >= 0; --I) {
+    unsigned Reg = SavedRegs[I];
+    if (Reg == 0) break;
+    int CURegNum = getCompactUnwindRegNum(CURegs, Reg);
+    if (CURegNum == -1)
+      return ~0U;
+    RegEnc |= (CURegNum & 0x7) << (5 - I);
+  }
+
+  assert((RegEnc & 0x7FFF) == RegEnc && "Invalid compact register encoding!");
+  return RegEnc;
+}
+
+uint32_t X86FrameLowering::getCompactUnwindEncoding(MachineFunction &MF) const {
+  const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
+  unsigned FramePtr = RegInfo->getFrameRegister(MF);
+  unsigned StackPtr = RegInfo->getStackRegister();
+
+  X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
+  int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
+
+  bool Is64Bit = STI.is64Bit();
+  bool HasFP = hasFP(MF);
+
+  unsigned SavedRegs[6] = { 0, 0, 0, 0, 0, 0 };
+  int SavedRegIdx = 6;
+
+  unsigned OffsetSize = (Is64Bit ? 8 : 4);
+
+  unsigned PushInstr = (Is64Bit ? X86::PUSH64r : X86::PUSH32r);
+  unsigned PushInstrSize = 1;
+  unsigned MoveInstr = (Is64Bit ? X86::MOV64rr : X86::MOV32rr);
+  unsigned MoveInstrSize = (Is64Bit ? 3 : 2);
+  unsigned SubtractInstr = getSUBriOpcode(Is64Bit, -TailCallReturnAddrDelta);
+  unsigned SubtractInstrIdx = (Is64Bit ? 3 : 2);
+
+  unsigned StackDivide = (Is64Bit ? 8 : 4);
+
+  unsigned InstrOffset = 0;
+  unsigned CFAOffset = 0;
+  unsigned StackAdjust = 0;
+
+  MachineBasicBlock &MBB = MF.front(); // Prologue is in entry BB.
+  bool ExpectEnd = false;
+  for (MachineBasicBlock::iterator
+         MBBI = MBB.begin(), MBBE = MBB.end(); MBBI != MBBE; ++MBBI) {
+    MachineInstr &MI = *MBBI;
+    unsigned Opc = MI.getOpcode();
+    if (Opc == X86::PROLOG_LABEL) continue;
+    if (!MI.getFlag(MachineInstr::FrameSetup)) break;
+
+    // We don't exect any more prolog instructions.
+    if (ExpectEnd) return 0;
+
+    if (Opc == PushInstr) {
+      // If there are too many saved registers, we cannot use compact encoding.
+      if (--SavedRegIdx < 0) return 0;
+
+      SavedRegs[SavedRegIdx] = MI.getOperand(0).getReg();
+      CFAOffset += OffsetSize;
+      InstrOffset += PushInstrSize;
+    } else if (Opc == MoveInstr) {
+      unsigned SrcReg = MI.getOperand(1).getReg();
+      unsigned DstReg = MI.getOperand(0).getReg();
+
+      if (DstReg != FramePtr || SrcReg != StackPtr)
+        return 0;
+
+      CFAOffset = 0;
+      memset(SavedRegs, 0, sizeof(SavedRegs));
+      InstrOffset += MoveInstrSize;
+    } else if (Opc == SubtractInstr) {
+      if (StackAdjust)
+        // We all ready have a stack pointer adjustment.
+        return 0;
+
+      if (!MI.getOperand(0).isReg() ||
+          MI.getOperand(0).getReg() != MI.getOperand(1).getReg() ||
+          MI.getOperand(0).getReg() != StackPtr || !MI.getOperand(2).isImm())
+        // We need this to be a stack adjustment pointer. Something like:
+        //
+        //   %RSP<def> = SUB64ri8 %RSP, 48
+        return 0;
+
+      StackAdjust = MI.getOperand(2).getImm() / StackDivide;
+      SubtractInstrIdx += InstrOffset;
+      ExpectEnd = true;
+    }
+  }
+
+  // Encode that we are using EBP/RBP as the frame pointer.
+  uint32_t CompactUnwindEncoding = 0;
+  CFAOffset /= StackDivide;
+  if (HasFP) {
+    if ((CFAOffset & 0xFF) != CFAOffset)
+      // Offset was too big for compact encoding.
+      return 0;
+
+    // Get the encoding of the saved registers when we have a frame pointer.
+    uint32_t RegEnc = encodeCompactUnwindRegistersWithFrame(SavedRegs, Is64Bit);
+    if (RegEnc == ~0U)
+      return 0;
+
+    CompactUnwindEncoding |= 0x01000000;
+    CompactUnwindEncoding |= (CFAOffset & 0xFF) << 16;
+    CompactUnwindEncoding |= RegEnc & 0x7FFF;
+  } else {
+    unsigned FullOffset = CFAOffset + StackAdjust;
+    if ((FullOffset & 0xFF) == FullOffset) {
+      // Frameless stack.
+      CompactUnwindEncoding |= 0x02000000;
+      CompactUnwindEncoding |= (FullOffset & 0xFF) << 16;
+    } else {
+      if ((CFAOffset & 0x7) != CFAOffset)
+        // The extra stack adjustments are too big for us to handle.
+        return 0;
+
+      // Frameless stack with an offset too large for us to encode compactly.
+      CompactUnwindEncoding |= 0x03000000;
+
+      // Encode the offset to the nnnnnn value in the 'subl $nnnnnn, ESP'
+      // instruction.
+      CompactUnwindEncoding |= (SubtractInstrIdx & 0xFF) << 16;
+
+      // Encode any extra stack stack changes (done via push instructions).
+      CompactUnwindEncoding |= (CFAOffset & 0x7) << 13;
+    }
+
+    // Get the encoding of the saved registers when we don't have a frame
+    // pointer.
+    uint32_t RegEnc = encodeCompactUnwindRegistersWithoutFrame(SavedRegs,
+                                                               6 - SavedRegIdx,
+                                                               Is64Bit);
+    if (RegEnc == ~0U) return 0;
+    CompactUnwindEncoding |= RegEnc & 0x3FF;
+  }
+
+  return CompactUnwindEncoding;
+}
+
 /// emitPrologue - Push callee-saved registers onto the stack, which
 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate
 /// space for local variables. Also emit labels used by the exception handler to
@@ -370,7 +612,6 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   unsigned SlotSize = RegInfo->getSlotSize();
   unsigned FramePtr = RegInfo->getFrameRegister(MF);
   unsigned StackPtr = RegInfo->getStackRegister();
-
   DebugLoc DL;
 
   // If we're forcing a stack realignment we can't rely on just the frame
@@ -398,7 +639,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
       !RegInfo->needsStackRealignment(MF) &&
       !MFI->hasVarSizedObjects() &&                // No dynamic alloca.
       !MFI->adjustsStack() &&                      // No calls.
-      !IsWin64) {                                  // Win64 has no Red Zone
+      !IsWin64 &&                                  // Win64 has no Red Zone
+      !EnableSegmentedStacks) {                    // Regular stack
     uint64_t MinSize = X86FI->getCalleeSavedFrameSize();
     if (HasFP) MinSize += SlotSize;
     StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
@@ -459,7 +701,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     if (needsFrameMoves) {
       // Mark the place where EBP/RBP was saved.
       MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol();
-      BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)).addSym(FrameLabel);
+      BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL))
+        .addSym(FrameLabel);
 
       // Define the current CFA rule to use the provided offset.
       if (StackSize) {
@@ -478,7 +721,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
       Moves.push_back(MachineMove(FrameLabel, FPDst, FPSrc));
     }
 
-    // Update EBP with the new base value...
+    // Update EBP with the new base value.
     BuildMI(MBB, MBBI, DL,
             TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr), FramePtr)
         .addReg(StackPtr)
@@ -487,7 +730,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     if (needsFrameMoves) {
       // Mark effective beginning of when frame pointer becomes valid.
       MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol();
-      BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)).addSym(FrameLabel);
+      BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL))
+        .addSym(FrameLabel);
 
       // Define the current CFA to use the EBP/RBP register.
       MachineLocation FPDst(FramePtr);
@@ -504,8 +748,10 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     if (RegInfo->needsStackRealignment(MF)) {
       MachineInstr *MI =
         BuildMI(MBB, MBBI, DL,
-                TII.get(Is64Bit ? X86::AND64ri32 : X86::AND32ri),
-                StackPtr).addReg(StackPtr).addImm(-MaxAlign);
+                TII.get(Is64Bit ? X86::AND64ri32 : X86::AND32ri), StackPtr)
+        .addReg(StackPtr)
+        .addImm(-MaxAlign)
+        .setMIFlag(MachineInstr::FrameSetup);
 
       // The EFLAGS implicit def is dead.
       MI->getOperand(3).setIsDead();
@@ -522,6 +768,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
          (MBBI->getOpcode() == X86::PUSH32r ||
           MBBI->getOpcode() == X86::PUSH64r)) {
     PushedRegs = true;
+    MBBI->setFlag(MachineInstr::FrameSetup);
     ++MBBI;
 
     if (!HasFP && needsFrameMoves) {
@@ -530,8 +777,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
       BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)).addSym(Label);
 
       // Define the current CFA rule to use the provided offset.
-      unsigned Ptr = StackSize ?
-        MachineLocation::VirtualFP : StackPtr;
+      unsigned Ptr = StackSize ? MachineLocation::VirtualFP : StackPtr;
       MachineLocation SPDst(Ptr);
       MachineLocation SPSrc(Ptr, StackOffset);
       Moves.push_back(MachineMove(Label, SPDst, SPSrc));
@@ -586,26 +832,30 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
 
       // Save EAX
       BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
-        .addReg(X86::EAX, RegState::Kill);
+        .addReg(X86::EAX, RegState::Kill)
+        .setMIFlag(MachineInstr::FrameSetup);
     }
 
     if (Is64Bit) {
       // Handle the 64-bit Windows ABI case where we need to call __chkstk.
       // Function prologue is responsible for adjusting the stack pointer.
       BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX)
-        .addImm(NumBytes);
+        .addImm(NumBytes)
+        .setMIFlag(MachineInstr::FrameSetup);
     } else {
       // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
       // We'll also use 4 already allocated bytes for EAX.
       BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
-        .addImm(isEAXAlive ? NumBytes - 4 : NumBytes);
+        .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
+        .setMIFlag(MachineInstr::FrameSetup);
     }
 
     BuildMI(MBB, MBBI, DL,
             TII.get(Is64Bit ? X86::W64ALLOCA : X86::CALLpcrel32))
       .addExternalSymbol(StackProbeSymbol)
       .addReg(StackPtr,    RegState::Define | RegState::Implicit)
-      .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit);
+      .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit)
+      .setMIFlag(MachineInstr::FrameSetup);
 
     // MSVC x64's __chkstk needs to adjust %rsp.
     // FIXME: %rax preserves the offset and should be available.
@@ -618,6 +868,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
         MachineInstr *MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm),
                                                 X86::EAX),
                                         StackPtr, false, NumBytes - 4);
+        MI->setFlag(MachineInstr::FrameSetup);
         MBB.insert(MBBI, MI);
     }
   } else if (NumBytes)
@@ -627,7 +878,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   if (( (!HasFP && NumBytes) || PushedRegs) && needsFrameMoves) {
     // Mark end of stack pointer adjustment.
     MCSymbol *Label = MMI.getContext().CreateTempSymbol();
-    BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL)).addSym(Label);
+    BuildMI(MBB, MBBI, DL, TII.get(X86::PROLOG_LABEL))
+      .addSym(Label);
 
     if (!HasFP && NumBytes) {
       // Define the current CFA rule to use the provided offset.
@@ -647,6 +899,11 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     if (PushedRegs)
       emitCalleeSavedFrameMoves(MF, Label, HasFP ? FramePtr : StackPtr);
   }
+
+  // Darwin 10.7 and greater has support for compact unwind encoding.
+  if (STI.getTargetTriple().isMacOSX() &&
+      !STI.getTargetTriple().isMacOSXVersionLT(10, 7))
+    MMI.setCompactUnwindEncoding(getCompactUnwindEncoding(MF));
 }
 
 void X86FrameLowering::emitEpilogue(MachineFunction &MF,
@@ -844,23 +1101,6 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-void
-X86FrameLowering::getInitialFrameState(std::vector<MachineMove> &Moves) const {
-  // Calculate amount of bytes used for return address storing
-  int stackGrowth = (STI.is64Bit() ? -8 : -4);
-  const X86RegisterInfo *RI = TM.getRegisterInfo();
-
-  // Initial state of the frame pointer is esp+stackGrowth.
-  MachineLocation Dst(MachineLocation::VirtualFP);
-  MachineLocation Src(RI->getStackRegister(), stackGrowth);
-  Moves.push_back(MachineMove(0, Dst, Src));
-
-  // Add return address to move list
-  MachineLocation CSDst(RI->getStackRegister(), stackGrowth);
-  MachineLocation CSSrc(RI->getRARegister());
-  Moves.push_back(MachineMove(0, CSDst, CSSrc));
-}
-
 int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF, int FI) const {
   const X86RegisterInfo *RI =
     static_cast<const X86RegisterInfo*>(MF.getTarget().getRegisterInfo());
@@ -873,9 +1113,7 @@ int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF, int FI) con
       // Skip the saved EBP.
       Offset += RI->getSlotSize();
     } else {
-      unsigned Align = MFI->getObjectAlignment(FI);
-      assert((-(Offset + StackSize)) % Align == 0);
-      Align = 0;
+      assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0);
       return Offset + StackSize;
     }
     // FIXME: Support tail calls
@@ -1027,184 +1265,183 @@ X86FrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                           true);
     assert(FrameIdx == MFI->getObjectIndexBegin() &&
            "Slot for EBP register must be last in order to be found!");
-    FrameIdx = 0;
+    (void)FrameIdx;
   }
 }
 
-/// permuteEncode - Create the permutation encoding used with frameless
-/// stacks. It is passed the number of registers to be saved and an array of the
-/// registers saved.
-static uint32_t permuteEncode(unsigned SavedCount, unsigned Registers[6]) {
-  // The saved registers are numbered from 1 to 6. In order to encode the order
-  // in which they were saved, we re-number them according to their place in the
-  // register order. The re-numbering is relative to the last re-numbered
-  // register. E.g., if we have registers {6, 2, 4, 5} saved in that order:
-  //
-  //    Orig  Re-Num
-  //    ----  ------
-  //     6       6
-  //     2       2
-  //     4       3
-  //     5       3
-  //
-  bool Used[7] = { false, false, false, false, false, false, false };
-  uint32_t RenumRegs[6];
-  for (unsigned I = 0; I < SavedCount; ++I) {
-    uint32_t Renum = 0;
-    for (unsigned U = 1; U < 7; ++U) {
-      if (U == Registers[I])
-        break;
-      if (!Used[U])
-        ++Renum;
-    }
-
-    Used[Registers[I]] = true;
-    RenumRegs[I] = Renum;
+static bool
+HasNestArgument(const MachineFunction *MF) {
+  const Function *F = MF->getFunction();
+  for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
+       I != E; I++) {
+    if (I->hasNestAttr())
+      return true;
   }
-
-  // Take the renumbered values and encode them into a 10-bit number.
-  uint32_t permutationEncoding = 0;
-  switch (SavedCount) {
-  case 6:
-    permutationEncoding |= 120 * RenumRegs[0] + 24 * RenumRegs[1]
-                           + 6 * RenumRegs[2] +  2 * RenumRegs[3]
-                           +     RenumRegs[4];
-    break;
-  case 5:
-    permutationEncoding |= 120 * RenumRegs[0] + 24 * RenumRegs[1]
-                           + 6 * RenumRegs[2] +  2 * RenumRegs[3]
-                           +     RenumRegs[4];
-    break;
-  case 4:
-    permutationEncoding |= 60 * RenumRegs[0] + 12 * RenumRegs[1]
-                          + 3 * RenumRegs[2] +      RenumRegs[3];
-    break;
-  case 3:
-    permutationEncoding |= 20 * RenumRegs[0] + 4 * RenumRegs[1]
-                              + RenumRegs[2];
-    break;
-  case 2:
-    permutationEncoding |=  5 * RenumRegs[0] +     RenumRegs[1];
-    break;
-  case 1:
-    permutationEncoding |=      RenumRegs[0];
-    break;
-  }
-
-  return permutationEncoding;
+  return false;
 }
 
-uint32_t X86FrameLowering::
-getCompactUnwindEncoding(ArrayRef<MCCFIInstruction> Instrs,
-                         int DataAlignmentFactor, bool IsEH) const {
-  uint32_t Encoding = 0;
-  int CFAOffset = 0;
-  const TargetRegisterInfo *TRI = TM.getRegisterInfo();
-  unsigned SavedRegs[6] = { 0, 0, 0, 0, 0, 0 };
-  unsigned SavedRegIdx = 0;
-  int FramePointerReg = -1;
-
-  for (ArrayRef<MCCFIInstruction>::const_iterator
-         I = Instrs.begin(), E = Instrs.end(); I != E; ++I) {
-    const MCCFIInstruction &Inst = *I;
-    MCSymbol *Label = Inst.getLabel();
-
-    // Ignore invalid labels.
-    if (Label && !Label->isDefined()) continue;
-
-    unsigned Operation = Inst.getOperation();
-    if (Operation != MCCFIInstruction::Move &&
-        Operation != MCCFIInstruction::RelMove)
-      // FIXME: We can't handle this frame just yet.
-      return 0;
-
-    const MachineLocation &Dst = Inst.getDestination();
-    const MachineLocation &Src = Inst.getSource();
-    const bool IsRelative = (Operation == MCCFIInstruction::RelMove);
-
-    if (Dst.isReg() && Dst.getReg() == MachineLocation::VirtualFP) {
-      if (Src.getReg() != MachineLocation::VirtualFP) {
-        // DW_CFA_def_cfa
-        assert(FramePointerReg == -1 &&"Defining more than one frame pointer?");
-        if (TRI->getLLVMRegNum(Src.getReg(), IsEH) != X86::EBP &&
-            TRI->getLLVMRegNum(Src.getReg(), IsEH) != X86::RBP)
-          // The frame pointer isn't EBP/RBP. Cannot make unwind information
-          // compact.
-          return 0;
-        FramePointerReg = TRI->getCompactUnwindRegNum(Src.getReg(), IsEH);
-      } // else DW_CFA_def_cfa_offset
-
-      if (IsRelative)
-        CFAOffset += Src.getOffset();
-      else
-        CFAOffset -= Src.getOffset();
-
-      continue;
-    }
-
-    if (Src.isReg() && Src.getReg() == MachineLocation::VirtualFP) {
-      // DW_CFA_def_cfa_register
-      assert(FramePointerReg == -1 && "Defining more than one frame pointer?");
-
-      if (TRI->getLLVMRegNum(Dst.getReg(), IsEH) != X86::EBP &&
-          TRI->getLLVMRegNum(Dst.getReg(), IsEH) != X86::RBP)
-        // The frame pointer isn't EBP/RBP. Cannot make unwind information
-        // compact.
-        return 0;
-
-      FramePointerReg = TRI->getCompactUnwindRegNum(Dst.getReg(), IsEH);
-      if (SavedRegIdx != 1 || SavedRegs[0] != unsigned(FramePointerReg))
-        return 0;
-
-      SavedRegs[0] = 0;
-      SavedRegIdx = 0;
-      continue;
-    }
-
-    unsigned Reg = Src.getReg();
-    int Offset = Dst.getOffset();
-    if (IsRelative)
-      Offset -= CFAOffset;
-    Offset /= DataAlignmentFactor;
-
-    if (Offset < 0) {
-      // FIXME: Handle?
-      // DW_CFA_offset_extended_sf
-      return 0;
-    } else if (Reg < 64) {
-      // DW_CFA_offset + Reg
-      if (SavedRegIdx >= 6) return 0;
-      int CURegNum = TRI->getCompactUnwindRegNum(Reg, IsEH);
-      if (CURegNum == -1) return 0;
-      SavedRegs[SavedRegIdx++] = CURegNum;
-    } else {
-      // FIXME: Handle?
-      // DW_CFA_offset_extended
-      return 0;
-    }
-  }
-
-  // Bail if there are too many registers to encode.
-  if (SavedRegIdx > 6) return 0;
-
-  // Check if the offset is too big.
-  CFAOffset /= 4;
-  if ((CFAOffset & 0xFF) != CFAOffset)
-    return 0;
-  Encoding |= (CFAOffset & 0xFF) << 16; // Size encoding.
-
-  if (FramePointerReg != -1) {
-    Encoding |= 0x01000000;     // EBP/RBP Unwind Frame
-    for (unsigned I = 0; I != SavedRegIdx; ++I) {
-      unsigned Reg = SavedRegs[I];
-      if (Reg == unsigned(FramePointerReg)) continue;
-      Encoding |= (Reg & 0x7) << (I * 3); // Register encoding
-    }
+static unsigned
+GetScratchRegister(bool Is64Bit, const MachineFunction &MF) {
+  if (Is64Bit) {
+    return X86::R11;
   } else {
-    Encoding |= 0x02000000;     // Frameless unwind with small stack
-    Encoding |= (SavedRegIdx & 0x7) << 10;
-    Encoding |= permuteEncode(SavedRegIdx, SavedRegs);
+    CallingConv::ID CallingConvention = MF.getFunction()->getCallingConv();
+    bool IsNested = HasNestArgument(&MF);
+
+    if (CallingConvention == CallingConv::X86_FastCall) {
+      if (IsNested) {
+        report_fatal_error("Segmented stacks does not support fastcall with "
+                           "nested function.");
+        return -1;
+      } else {
+        return X86::EAX;
+      }
+    } else {
+      if (IsNested)
+        return X86::EDX;
+      else
+        return X86::ECX;
+    }
+  }
+}
+
+void
+X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
+  MachineBasicBlock &prologueMBB = MF.front();
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  const X86InstrInfo &TII = *TM.getInstrInfo();
+  uint64_t StackSize;
+  bool Is64Bit = STI.is64Bit();
+  unsigned TlsReg, TlsOffset;
+  DebugLoc DL;
+  const X86Subtarget *ST = &MF.getTarget().getSubtarget<X86Subtarget>();
+
+  unsigned ScratchReg = GetScratchRegister(Is64Bit, MF);
+  assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
+         "Scratch register is live-in");
+
+  if (MF.getFunction()->isVarArg())
+    report_fatal_error("Segmented stacks do not support vararg functions.");
+  if (!ST->isTargetLinux())
+    report_fatal_error("Segmented stacks supported only on linux.");
+
+  MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
+  MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
+  X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
+  bool IsNested = false;
+
+  // We need to know if the function has a nest argument only in 64 bit mode.
+  if (Is64Bit)
+    IsNested = HasNestArgument(&MF);
+
+  // The MOV R10, RAX needs to be in a different block, since the RET we emit in
+  // allocMBB needs to be last (terminating) instruction.
+  MachineBasicBlock *restoreR10MBB = NULL;
+  if (IsNested)
+    restoreR10MBB = MF.CreateMachineBasicBlock();
+
+  for (MachineBasicBlock::livein_iterator i = prologueMBB.livein_begin(),
+         e = prologueMBB.livein_end(); i != e; i++) {
+    allocMBB->addLiveIn(*i);
+    checkMBB->addLiveIn(*i);
+
+    if (IsNested)
+      restoreR10MBB->addLiveIn(*i);
   }
 
-  return Encoding;
+  if (IsNested) {
+    allocMBB->addLiveIn(X86::R10);
+    restoreR10MBB->addLiveIn(X86::RAX);
+  }
+
+  if (IsNested)
+    MF.push_front(restoreR10MBB);
+  MF.push_front(allocMBB);
+  MF.push_front(checkMBB);
+
+  // Eventually StackSize will be calculated by a link-time pass; which will
+  // also decide whether checking code needs to be injected into this particular
+  // prologue.
+  StackSize = MFI->getStackSize();
+
+  // Read the limit off the current stacklet off the stack_guard location.
+  if (Is64Bit) {
+    TlsReg = X86::FS;
+    TlsOffset = 0x70;
+
+    BuildMI(checkMBB, DL, TII.get(X86::LEA64r), ScratchReg).addReg(X86::RSP)
+      .addImm(0).addReg(0).addImm(-StackSize).addReg(0);
+    BuildMI(checkMBB, DL, TII.get(X86::CMP64rm)).addReg(ScratchReg)
+      .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg);
+  } else {
+    TlsReg = X86::GS;
+    TlsOffset = 0x30;
+
+    BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP)
+      .addImm(0).addReg(0).addImm(-StackSize).addReg(0);
+    BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg)
+      .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg);
+  }
+
+  // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
+  // It jumps to normal execution of the function body.
+  BuildMI(checkMBB, DL, TII.get(X86::JG_4)).addMBB(&prologueMBB);
+
+  // On 32 bit we first push the arguments size and then the frame size. On 64
+  // bit, we pass the stack frame size in r10 and the argument size in r11.
+  if (Is64Bit) {
+    // Functions with nested arguments use R10, so it needs to be saved across
+    // the call to _morestack
+
+    if (IsNested)
+      BuildMI(allocMBB, DL, TII.get(X86::MOV64rr), X86::RAX).addReg(X86::R10);
+
+    BuildMI(allocMBB, DL, TII.get(X86::MOV64ri), X86::R10)
+      .addImm(StackSize);
+    BuildMI(allocMBB, DL, TII.get(X86::MOV64ri), X86::R11)
+      .addImm(X86FI->getArgumentStackSize());
+    MF.getRegInfo().setPhysRegUsed(X86::R10);
+    MF.getRegInfo().setPhysRegUsed(X86::R11);
+  } else {
+    // Since we'll call __morestack, stack alignment needs to be preserved.
+    BuildMI(allocMBB, DL, TII.get(X86::SUB32ri), X86::ESP).addReg(X86::ESP)
+      .addImm(8);
+    BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
+      .addImm(X86FI->getArgumentStackSize());
+    BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
+      .addImm(StackSize);
+  }
+
+  // __morestack is in libgcc
+  if (Is64Bit)
+    BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32))
+      .addExternalSymbol("__morestack");
+  else
+    BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32))
+      .addExternalSymbol("__morestack");
+
+  // __morestack only seems to remove 8 bytes off the stack. Add back the
+  // additional 8 bytes we added before pushing the arguments.
+  if (!Is64Bit)
+    BuildMI(allocMBB, DL, TII.get(X86::ADD32ri), X86::ESP).addReg(X86::ESP)
+      .addImm(8);
+  BuildMI(allocMBB, DL, TII.get(X86::RET));
+
+  if (IsNested)
+    BuildMI(restoreR10MBB, DL, TII.get(X86::MOV64rr), X86::R10)
+      .addReg(X86::RAX);
+
+  if (IsNested) {
+    allocMBB->addSuccessor(restoreR10MBB);
+    restoreR10MBB->addSuccessor(&prologueMBB);
+  } else {
+    allocMBB->addSuccessor(&prologueMBB);
+  }
+
+  checkMBB->addSuccessor(allocMBB);
+  checkMBB->addSuccessor(&prologueMBB);
+
+#ifdef XDEBUG
+  MF.verify();
+#endif
 }
diff --git a/contrib/llvm/lib/Target/X86/X86FrameLowering.h b/contrib/llvm/lib/Target/X86/X86FrameLowering.h
index 14c31ed47cf1..6f490640b4ed 100644
--- a/contrib/llvm/lib/Target/X86/X86FrameLowering.h
+++ b/contrib/llvm/lib/Target/X86/X86FrameLowering.h
@@ -41,6 +41,8 @@ class X86FrameLowering : public TargetFrameLowering {
   void emitPrologue(MachineFunction &MF) const;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const;
 
+  void adjustForSegmentedStacks(MachineFunction &MF) const;
+
   void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                             RegScavenger *RS = NULL) const;
 
@@ -57,11 +59,8 @@ class X86FrameLowering : public TargetFrameLowering {
   bool hasFP(const MachineFunction &MF) const;
   bool hasReservedCallFrame(const MachineFunction &MF) const;
 
-  void getInitialFrameState(std::vector<MachineMove> &Moves) const;
   int getFrameIndexOffset(const MachineFunction &MF, int FI) const;
-
-  uint32_t getCompactUnwindEncoding(ArrayRef<MCCFIInstruction> Instrs,
-                                    int DataAlignmentFactor, bool IsEH) const;
+  uint32_t getCompactUnwindEncoding(MachineFunction &MF) const;
 };
 
 } // End llvm namespace
diff --git a/contrib/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/contrib/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 2b0f283bec75..02b0ff26032b 100644
--- a/contrib/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -474,10 +474,15 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
     if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND)
       continue;
     
-    // If the source and destination are SSE registers, then this is a legal
-    // conversion that should not be lowered.
     EVT SrcVT = N->getOperand(0).getValueType();
     EVT DstVT = N->getValueType(0);
+
+    // If any of the sources are vectors, no fp stack involved.
+    if (SrcVT.isVector() || DstVT.isVector())
+      continue;
+
+    // If the source and destination are SSE registers, then this is a legal
+    // conversion that should not be lowered.
     bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT);
     bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT);
     if (SrcIsSSE && DstIsSSE)
@@ -2168,9 +2173,10 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
         SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl,
                                                         MVT::i8, Reg);
 
-        // Emit a testb. No special NOREX tricks are needed since there's
-        // only one GPR operand!
-        return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32,
+        // Emit a testb.  The EXTRACT_SUBREG becomes a COPY that can only
+        // target GR8_NOREX registers, so make sure the register class is
+        // forced.
+        return CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl, MVT::i32,
                                       Subreg, ShiftedImm);
       }
 
diff --git a/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp b/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5096d9ae2edf..7c8ce177ecd3 100644
--- a/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -51,6 +51,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetOptions.h"
 using namespace llvm;
 using namespace dwarf;
 
@@ -71,9 +72,6 @@ static SDValue Extract128BitVector(SDValue Vec,
                                    SelectionDAG &DAG,
                                    DebugLoc dl);
 
-static SDValue ConcatVectors(SDValue Lower, SDValue Upper, SelectionDAG &DAG);
-
-
 /// Generate a DAG to grab 128-bits from a vector > 128 bits.  This
 /// sets things up to match to an AVX VEXTRACTF128 instruction or a
 /// simple subregister reference.  Idx is an index in the 128 bits we
@@ -85,14 +83,10 @@ static SDValue Extract128BitVector(SDValue Vec,
                                    DebugLoc dl) {
   EVT VT = Vec.getValueType();
   assert(VT.getSizeInBits() == 256 && "Unexpected vector size!");
-
   EVT ElVT = VT.getVectorElementType();
-
-  int Factor = VT.getSizeInBits() / 128;
-
-  EVT ResultVT = EVT::getVectorVT(*DAG.getContext(),
-                                  ElVT,
-                                  VT.getVectorNumElements() / Factor);
+  int Factor = VT.getSizeInBits()/128;
+  EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
+                                  VT.getVectorNumElements()/Factor);
 
   // Extract from UNDEF is UNDEF.
   if (Vec.getOpcode() == ISD::UNDEF)
@@ -111,7 +105,6 @@ static SDValue Extract128BitVector(SDValue Vec,
                                  * ElemsPerChunk);
 
     SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32);
-
     SDValue Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec,
                                  VecIdx);
 
@@ -136,21 +129,18 @@ static SDValue Insert128BitVector(SDValue Result,
     assert(VT.getSizeInBits() == 128 && "Unexpected vector size!");
 
     EVT ElVT = VT.getVectorElementType();
-
     unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
-
     EVT ResultVT = Result.getValueType();
 
     // Insert the relevant 128 bits.
-    unsigned ElemsPerChunk = 128 / ElVT.getSizeInBits();
+    unsigned ElemsPerChunk = 128/ElVT.getSizeInBits();
 
     // This is the index of the first element of the 128-bit chunk
     // we want.
-    unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / 128)
+    unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/128)
                                  * ElemsPerChunk);
 
     SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32);
-
     Result = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec,
                          VecIdx);
     return Result;
@@ -159,34 +149,6 @@ static SDValue Insert128BitVector(SDValue Result,
   return SDValue();
 }
 
-/// Given two vectors, concat them.
-static SDValue ConcatVectors(SDValue Lower, SDValue Upper, SelectionDAG &DAG) {
-  DebugLoc dl = Lower.getDebugLoc();
-
-  assert(Lower.getValueType() == Upper.getValueType() && "Mismatched vectors!");
-
-  EVT VT = EVT::getVectorVT(*DAG.getContext(),
-                            Lower.getValueType().getVectorElementType(),
-                            Lower.getValueType().getVectorNumElements() * 2);
-
-  // TODO: Generalize to arbitrary vector length (this assumes 256-bit vectors).
-  assert(VT.getSizeInBits() == 256 && "Unsupported vector concat!");
-
-  // Insert the upper subvector.
-  SDValue Vec = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT), Upper,
-                                   DAG.getConstant(
-                                     // This is half the length of the result
-                                     // vector.  Start inserting the upper 128
-                                     // bits here.
-                                     Lower.getValueType().getVectorNumElements(),
-                                     MVT::i32),
-                                   DAG, dl);
-
-  // Insert the lower subvector.
-  Vec = Insert128BitVector(Vec, Lower, DAG.getConstant(0, MVT::i32), DAG, dl);
-  return Vec;
-}
-
 static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) {
   const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>();
   bool is64Bit = Subtarget->is64Bit();
@@ -197,11 +159,8 @@ static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) {
     return new TargetLoweringObjectFileMachO();
   }
 
-  if (Subtarget->isTargetELF()) {
-    if (is64Bit)
-      return new X8664_ELFTargetObjectFile(TM);
-    return new X8632_ELFTargetObjectFile(TM);
-  }
+  if (Subtarget->isTargetELF())
+    return new TargetLoweringObjectFileELF();
   if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho())
     return new TargetLoweringObjectFileCOFF();
   llvm_unreachable("unknown subtarget type");
@@ -222,6 +181,8 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
 
   // X86 is weird, it always uses i8 for shift amounts and setcc results.
   setBooleanContents(ZeroOrOneBooleanContent);
+  // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
+  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
   // For 64-bit since we have so many registers use the ILP scheduler, for
   // 32-bit code use the register pressure specific scheduling.
@@ -354,7 +315,8 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FP_TO_UINT     , MVT::i64  , Expand);
     setOperationAction(ISD::FP_TO_UINT     , MVT::i32  , Promote);
   } else if (!UseSoftFloat) {
-    if (X86ScalarSSEf32 && !Subtarget->hasSSE3())
+    // Since AVX is a superset of SSE3, only check for SSE here.
+    if (Subtarget->hasSSE1() && !Subtarget->hasSSE3())
       // Expand FP_TO_UINT into a select.
       // FIXME: We would like to use a Custom expander here eventually to do
       // the optimal thing for SSE vs. the default expansion in the legalizer.
@@ -417,15 +379,24 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   setOperationAction(ISD::FREM             , MVT::f80  , Expand);
   setOperationAction(ISD::FLT_ROUNDS_      , MVT::i32  , Custom);
 
-  setOperationAction(ISD::CTTZ             , MVT::i8   , Custom);
-  setOperationAction(ISD::CTLZ             , MVT::i8   , Custom);
-  setOperationAction(ISD::CTTZ             , MVT::i16  , Custom);
-  setOperationAction(ISD::CTLZ             , MVT::i16  , Custom);
-  setOperationAction(ISD::CTTZ             , MVT::i32  , Custom);
-  setOperationAction(ISD::CTLZ             , MVT::i32  , Custom);
-  if (Subtarget->is64Bit()) {
-    setOperationAction(ISD::CTTZ           , MVT::i64  , Custom);
-    setOperationAction(ISD::CTLZ           , MVT::i64  , Custom);
+  if (Subtarget->hasBMI()) {
+    setOperationAction(ISD::CTTZ           , MVT::i8   , Promote);
+  } else {
+    setOperationAction(ISD::CTTZ           , MVT::i8   , Custom);
+    setOperationAction(ISD::CTTZ           , MVT::i16  , Custom);
+    setOperationAction(ISD::CTTZ           , MVT::i32  , Custom);
+    if (Subtarget->is64Bit())
+      setOperationAction(ISD::CTTZ         , MVT::i64  , Custom);
+  }
+
+  if (Subtarget->hasLZCNT()) {
+    setOperationAction(ISD::CTLZ           , MVT::i8   , Promote);
+  } else {
+    setOperationAction(ISD::CTLZ           , MVT::i8   , Custom);
+    setOperationAction(ISD::CTLZ           , MVT::i16  , Custom);
+    setOperationAction(ISD::CTLZ           , MVT::i32  , Custom);
+    if (Subtarget->is64Bit())
+      setOperationAction(ISD::CTLZ         , MVT::i64  , Custom);
   }
 
   if (Subtarget->hasPOPCNT()) {
@@ -491,8 +462,8 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   if (Subtarget->hasXMM())
     setOperationAction(ISD::PREFETCH      , MVT::Other, Legal);
 
-  // We may not have a libcall for MEMBARRIER so we should lower this.
   setOperationAction(ISD::MEMBARRIER    , MVT::Other, Custom);
+  setOperationAction(ISD::ATOMIC_FENCE  , MVT::Other, Custom);
 
   // On X86 and X86-64, atomic operations are lowered to locked instructions.
   // Locked instructions, in turn, have implicit fence semantics (all memory
@@ -506,9 +477,11 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     MVT VT = IntVTs[i];
     setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom);
     setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
+    setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
   }
 
   if (!Subtarget->is64Bit()) {
+    setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom);
     setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
     setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
@@ -518,6 +491,10 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom);
   }
 
+  if (Subtarget->hasCmpxchg16b()) {
+    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
+  }
+
   // FIXME - use subtarget debug flags
   if (!Subtarget->isTargetDarwin() &&
       !Subtarget->isTargetELF() &&
@@ -539,7 +516,8 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
 
-  setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom);
+  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
+  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
@@ -556,11 +534,16 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
 
   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
-  setOperationAction(ISD::DYNAMIC_STACKALLOC,
-                     (Subtarget->is64Bit() ? MVT::i64 : MVT::i32),
-                     (Subtarget->isTargetCOFF()
-                      && !Subtarget->isTargetEnvMacho()
-                      ? Custom : Expand));
+
+  if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho())
+    setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ?
+                       MVT::i64 : MVT::i32, Custom);
+  else if (EnableSegmentedStacks)
+    setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ?
+                       MVT::i64 : MVT::i32, Custom);
+  else
+    setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ?
+                       MVT::i64 : MVT::i32, Expand);
 
   if (!UseSoftFloat && X86ScalarSSEf64) {
     // f32 and f64 use SSE.
@@ -739,7 +722,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand);
-    setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand);
+    setOperationAction(ISD::SETCC, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand);
@@ -754,6 +737,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::SIGN_EXTEND,  (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::ZERO_EXTEND,  (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::ANY_EXTEND,  (MVT::SimpleValueType)VT, Expand);
+    setOperationAction(ISD::VSELECT,  (MVT::SimpleValueType)VT, Expand);
     for (unsigned InnerVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
          InnerVT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++InnerVT)
       setTruncStoreAction((MVT::SimpleValueType)VT,
@@ -816,7 +800,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);
     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
     setOperationAction(ISD::SELECT,             MVT::v4f32, Custom);
-    setOperationAction(ISD::VSETCC,             MVT::v4f32, Custom);
+    setOperationAction(ISD::SETCC,              MVT::v4f32, Custom);
   }
 
   if (!UseSoftFloat && Subtarget->hasXMMInt()) {
@@ -846,10 +830,10 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FSQRT,              MVT::v2f64, Legal);
     setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
 
-    setOperationAction(ISD::VSETCC,             MVT::v2f64, Custom);
-    setOperationAction(ISD::VSETCC,             MVT::v16i8, Custom);
-    setOperationAction(ISD::VSETCC,             MVT::v8i16, Custom);
-    setOperationAction(ISD::VSETCC,             MVT::v4i32, Custom);
+    setOperationAction(ISD::SETCC,              MVT::v2i64, Custom);
+    setOperationAction(ISD::SETCC,              MVT::v16i8, Custom);
+    setOperationAction(ISD::SETCC,              MVT::v8i16, Custom);
+    setOperationAction(ISD::SETCC,              MVT::v4i32, Custom);
 
     setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16i8, Custom);
     setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8i16, Custom);
@@ -925,7 +909,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Legal);
   }
 
-  if (Subtarget->hasSSE41()) {
+  if (Subtarget->hasSSE41() || Subtarget->hasAVX()) {
     setOperationAction(ISD::FFLOOR,             MVT::f32,   Legal);
     setOperationAction(ISD::FCEIL,              MVT::f32,   Legal);
     setOperationAction(ISD::FTRUNC,             MVT::f32,   Legal);
@@ -944,6 +928,12 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::SHL,                MVT::v4i32, Custom);
     setOperationAction(ISD::SHL,                MVT::v16i8, Custom);
 
+    setOperationAction(ISD::VSELECT,            MVT::v2f64, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v2i64, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v16i8, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v4i32, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v4f32, Legal);
+
     // i8 and i16 vectors are custom , because the source register and source
     // source memory operand types are not the same width.  f32 vectors are
     // custom since the immediate controlling the insert encodes additional
@@ -964,10 +954,11 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     }
   }
 
-  if (Subtarget->hasSSE2()) {
+  if (Subtarget->hasXMMInt()) {
     setOperationAction(ISD::SRL,               MVT::v2i64, Custom);
     setOperationAction(ISD::SRL,               MVT::v4i32, Custom);
     setOperationAction(ISD::SRL,               MVT::v16i8, Custom);
+    setOperationAction(ISD::SRL,               MVT::v8i16, Custom);
 
     setOperationAction(ISD::SHL,               MVT::v2i64, Custom);
     setOperationAction(ISD::SHL,               MVT::v4i32, Custom);
@@ -977,15 +968,16 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::SRA,               MVT::v8i16, Custom);
   }
 
-  if (Subtarget->hasSSE42())
-    setOperationAction(ISD::VSETCC,             MVT::v2i64, Custom);
+  if (Subtarget->hasSSE42() || Subtarget->hasAVX())
+    setOperationAction(ISD::SETCC,             MVT::v2i64, Custom);
 
   if (!UseSoftFloat && Subtarget->hasAVX()) {
-    addRegisterClass(MVT::v8f32, X86::VR256RegisterClass);
-    addRegisterClass(MVT::v4f64, X86::VR256RegisterClass);
-    addRegisterClass(MVT::v8i32, X86::VR256RegisterClass);
-    addRegisterClass(MVT::v4i64, X86::VR256RegisterClass);
-    addRegisterClass(MVT::v32i8, X86::VR256RegisterClass);
+    addRegisterClass(MVT::v32i8,  X86::VR256RegisterClass);
+    addRegisterClass(MVT::v16i16, X86::VR256RegisterClass);
+    addRegisterClass(MVT::v8i32,  X86::VR256RegisterClass);
+    addRegisterClass(MVT::v8f32,  X86::VR256RegisterClass);
+    addRegisterClass(MVT::v4i64,  X86::VR256RegisterClass);
+    addRegisterClass(MVT::v4f64,  X86::VR256RegisterClass);
 
     setOperationAction(ISD::LOAD,               MVT::v8f32, Legal);
     setOperationAction(ISD::LOAD,               MVT::v4f64, Legal);
@@ -1005,6 +997,59 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FSQRT,              MVT::v4f64, Legal);
     setOperationAction(ISD::FNEG,               MVT::v4f64, Custom);
 
+    setOperationAction(ISD::FP_TO_SINT,         MVT::v8i32, Legal);
+    setOperationAction(ISD::SINT_TO_FP,         MVT::v8i32, Legal);
+    setOperationAction(ISD::FP_ROUND,           MVT::v4f32, Legal);
+
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v4f64,  Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v4i64,  Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v8f32,  Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v8i32,  Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v32i8,  Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v16i16, Custom);
+
+    setOperationAction(ISD::SRL,               MVT::v4i64, Custom);
+    setOperationAction(ISD::SRL,               MVT::v8i32, Custom);
+    setOperationAction(ISD::SRL,               MVT::v16i16, Custom);
+    setOperationAction(ISD::SRL,               MVT::v32i8, Custom);
+
+    setOperationAction(ISD::SHL,               MVT::v4i64, Custom);
+    setOperationAction(ISD::SHL,               MVT::v8i32, Custom);
+    setOperationAction(ISD::SHL,               MVT::v16i16, Custom);
+    setOperationAction(ISD::SHL,               MVT::v32i8, Custom);
+
+    setOperationAction(ISD::SRA,               MVT::v8i32, Custom);
+    setOperationAction(ISD::SRA,               MVT::v16i16, Custom);
+
+    setOperationAction(ISD::SETCC,             MVT::v32i8, Custom);
+    setOperationAction(ISD::SETCC,             MVT::v16i16, Custom);
+    setOperationAction(ISD::SETCC,             MVT::v8i32, Custom);
+    setOperationAction(ISD::SETCC,             MVT::v4i64, Custom);
+
+    setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);
+    setOperationAction(ISD::SELECT,            MVT::v4i64, Custom);
+    setOperationAction(ISD::SELECT,            MVT::v8f32, Custom);
+
+    setOperationAction(ISD::VSELECT,            MVT::v4f64, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v4i64, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v8i32, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v8f32, Legal);
+
+    setOperationAction(ISD::ADD,               MVT::v4i64, Custom);
+    setOperationAction(ISD::ADD,               MVT::v8i32, Custom);
+    setOperationAction(ISD::ADD,               MVT::v16i16, Custom);
+    setOperationAction(ISD::ADD,               MVT::v32i8, Custom);
+
+    setOperationAction(ISD::SUB,               MVT::v4i64, Custom);
+    setOperationAction(ISD::SUB,               MVT::v8i32, Custom);
+    setOperationAction(ISD::SUB,               MVT::v16i16, Custom);
+    setOperationAction(ISD::SUB,               MVT::v32i8, Custom);
+
+    setOperationAction(ISD::MUL,               MVT::v4i64, Custom);
+    setOperationAction(ISD::MUL,               MVT::v8i32, Custom);
+    setOperationAction(ISD::MUL,               MVT::v16i16, Custom);
+    // Don't lower v32i8 because there is no 128-bit byte mul
+
     // Custom lower several nodes for 256-bit types.
     for (unsigned i = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
                   i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
@@ -1093,6 +1138,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
   setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
   setTargetDAGCombine(ISD::BUILD_VECTOR);
+  setTargetDAGCombine(ISD::VSELECT);
   setTargetDAGCombine(ISD::SELECT);
   setTargetDAGCombine(ISD::SHL);
   setTargetDAGCombine(ISD::SRA);
@@ -1100,7 +1146,10 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   setTargetDAGCombine(ISD::OR);
   setTargetDAGCombine(ISD::AND);
   setTargetDAGCombine(ISD::ADD);
+  setTargetDAGCombine(ISD::FADD);
+  setTargetDAGCombine(ISD::FSUB);
   setTargetDAGCombine(ISD::SUB);
+  setTargetDAGCombine(ISD::LOAD);
   setTargetDAGCombine(ISD::STORE);
   setTargetDAGCombine(ISD::ZERO_EXTEND);
   setTargetDAGCombine(ISD::SINT_TO_FP);
@@ -1124,25 +1173,26 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
 }
 
 
-MVT::SimpleValueType X86TargetLowering::getSetCCResultType(EVT VT) const {
-  return MVT::i8;
+EVT X86TargetLowering::getSetCCResultType(EVT VT) const {
+  if (!VT.isVector()) return MVT::i8;
+  return VT.changeVectorElementTypeToInteger();
 }
 
 
 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine
 /// the desired ByVal argument alignment.
-static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) {
+static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
   if (MaxAlign == 16)
     return;
-  if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) {
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
     if (VTy->getBitWidth() == 128)
       MaxAlign = 16;
-  } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+  } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     unsigned EltAlign = 0;
     getMaxByValAlign(ATy->getElementType(), EltAlign);
     if (EltAlign > MaxAlign)
       MaxAlign = EltAlign;
-  } else if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+  } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
       unsigned EltAlign = 0;
       getMaxByValAlign(STy->getElementType(i), EltAlign);
@@ -1159,7 +1209,7 @@ static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) {
 /// function arguments in the caller parameter area. For X86, aggregates
 /// that contain SSE vectors are placed at 16-byte boundaries while the rest
 /// are at 4-byte boundaries.
-unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const {
+unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty) const {
   if (Subtarget->is64Bit()) {
     // Max of 8 and alignment of type.
     unsigned TyAlign = TD->getABITypeAlignment(Ty);
@@ -1203,9 +1253,12 @@ X86TargetLowering::getOptimalMemOpType(uint64_t Size,
          ((DstAlign == 0 || DstAlign >= 16) &&
           (SrcAlign == 0 || SrcAlign >= 16))) &&
         Subtarget->getStackAlignment() >= 16) {
-      if (Subtarget->hasSSE2())
+      if (Subtarget->hasAVX() &&
+          Subtarget->getStackAlignment() >= 32)
+        return MVT::v8f32;
+      if (Subtarget->hasXMMInt())
         return MVT::v4i32;
-      if (Subtarget->hasSSE1())
+      if (Subtarget->hasXMM())
         return MVT::v4f32;
     } else if (!MemcpyStrSrc && Size >= 8 &&
                !Subtarget->is64Bit() &&
@@ -1408,7 +1461,7 @@ X86TargetLowering::LowerReturn(SDValue Chain,
                                   ValToCopy);
           // If we don't have SSE2 available, convert to v4f32 so the generated
           // register is legal.
-          if (!Subtarget->hasSSE2())
+          if (!Subtarget->hasXMMInt())
             ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy);
         }
       }
@@ -1700,6 +1753,7 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
     // places.
     assert(VA.getValNo() != LastVal &&
            "Don't support value assigned to multiple locs yet");
+    (void)LastVal;
     LastVal = VA.getValNo();
 
     if (VA.isRegLoc()) {
@@ -1917,6 +1971,8 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
       FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
   }
 
+  FuncInfo->setArgumentStackSize(StackSize);
+
   return Chain;
 }
 
@@ -2744,8 +2800,6 @@ static bool isTargetShuffle(unsigned Opcode) {
   case X86ISD::MOVSD:
   case X86ISD::UNPCKLPS:
   case X86ISD::UNPCKLPD:
-  case X86ISD::VUNPCKLPS:
-  case X86ISD::VUNPCKLPD:
   case X86ISD::VUNPCKLPSY:
   case X86ISD::VUNPCKLPDY:
   case X86ISD::PUNPCKLWD:
@@ -2754,10 +2808,17 @@ static bool isTargetShuffle(unsigned Opcode) {
   case X86ISD::PUNPCKLQDQ:
   case X86ISD::UNPCKHPS:
   case X86ISD::UNPCKHPD:
+  case X86ISD::VUNPCKHPSY:
+  case X86ISD::VUNPCKHPDY:
   case X86ISD::PUNPCKHWD:
   case X86ISD::PUNPCKHBW:
   case X86ISD::PUNPCKHDQ:
   case X86ISD::PUNPCKHQDQ:
+  case X86ISD::VPERMILPS:
+  case X86ISD::VPERMILPSY:
+  case X86ISD::VPERMILPD:
+  case X86ISD::VPERMILPDY:
+  case X86ISD::VPERM2F128:
     return true;
   }
   return false;
@@ -2783,6 +2844,10 @@ static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
   case X86ISD::PSHUFD:
   case X86ISD::PSHUFHW:
   case X86ISD::PSHUFLW:
+  case X86ISD::VPERMILPS:
+  case X86ISD::VPERMILPSY:
+  case X86ISD::VPERMILPD:
+  case X86ISD::VPERMILPDY:
     return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8));
   }
 
@@ -2796,6 +2861,7 @@ static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
   case X86ISD::PALIGN:
   case X86ISD::SHUFPD:
   case X86ISD::SHUFPS:
+  case X86ISD::VPERM2F128:
     return DAG.getNode(Opc, dl, VT, V1, V2,
                        DAG.getConstant(TargetMask, MVT::i8));
   }
@@ -2815,8 +2881,6 @@ static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
   case X86ISD::MOVSD:
   case X86ISD::UNPCKLPS:
   case X86ISD::UNPCKLPD:
-  case X86ISD::VUNPCKLPS:
-  case X86ISD::VUNPCKLPD:
   case X86ISD::VUNPCKLPSY:
   case X86ISD::VUNPCKLPDY:
   case X86ISD::PUNPCKLWD:
@@ -2825,6 +2889,8 @@ static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
   case X86ISD::PUNPCKLQDQ:
   case X86ISD::UNPCKHPS:
   case X86ISD::UNPCKHPD:
+  case X86ISD::VUNPCKHPSY:
+  case X86ISD::VUNPCKHPDY:
   case X86ISD::PUNPCKHWD:
   case X86ISD::PUNPCKHBW:
   case X86ISD::PUNPCKHDQ:
@@ -3026,6 +3092,17 @@ static bool isUndefOrInRange(int Val, int Low, int Hi) {
   return (Val < 0) || (Val >= Low && Val < Hi);
 }
 
+/// isUndefOrInRange - Return true if every element in Mask, begining
+/// from position Pos and ending in Pos+Size, falls within the specified
+/// range (L, L+Pos]. or is undef.
+static bool isUndefOrInRange(const SmallVectorImpl<int> &Mask,
+                             int Pos, int Size, int Low, int Hi) {
+  for (int i = Pos, e = Pos+Size; i != e; ++i)
+    if (!isUndefOrInRange(Mask[i], Low, Hi))
+      return false;
+  return true;
+}
+
 /// isUndefOrEqual - Val is either less than zero (undef) or equal to the
 /// specified value.
 static bool isUndefOrEqual(int Val, int CmpVal) {
@@ -3034,6 +3111,17 @@ static bool isUndefOrEqual(int Val, int CmpVal) {
   return false;
 }
 
+/// isSequentialOrUndefInRange - Return true if every element in Mask, begining
+/// from position Pos and ending in Pos+Size, falls within the specified
+/// sequential range (L, L+Pos]. or is undef.
+static bool isSequentialOrUndefInRange(const SmallVectorImpl<int> &Mask,
+                                       int Pos, int Size, int Low) {
+  for (int i = Pos, e = Pos+Size; i != e; ++i, ++Low)
+    if (!isUndefOrEqual(Mask[i], Low))
+      return false;
+  return true;
+}
+
 /// isPSHUFDMask - Return true if the node specifies a shuffle of elements that
 /// is suitable for input to PSHUFD or PSHUFW.  That is, it doesn't reference
 /// the second operand.
@@ -3104,11 +3192,13 @@ bool X86::isPSHUFLWMask(ShuffleVectorSDNode *N) {
 /// isPALIGNRMask - Return true if the node specifies a shuffle of elements that
 /// is suitable for input to PALIGNR.
 static bool isPALIGNRMask(const SmallVectorImpl<int> &Mask, EVT VT,
-                          bool hasSSSE3) {
+                          bool hasSSSE3OrAVX) {
   int i, e = VT.getVectorNumElements();
+  if (VT.getSizeInBits() != 128 && VT.getSizeInBits() != 64)
+    return false;
 
   // Do not handle v2i64 / v2f64 shuffles with palignr.
-  if (e < 4 || !hasSSSE3)
+  if (e < 4 || !hasSSSE3OrAVX)
     return false;
 
   for (i = 0; i != e; ++i)
@@ -3119,42 +3209,176 @@ static bool isPALIGNRMask(const SmallVectorImpl<int> &Mask, EVT VT,
   if (i == e)
     return false;
 
-  // Determine if it's ok to perform a palignr with only the LHS, since we
-  // don't have access to the actual shuffle elements to see if RHS is undef.
-  bool Unary = Mask[i] < (int)e;
-  bool NeedsUnary = false;
+  // Make sure we're shifting in the right direction.
+  if (Mask[i] <= i)
+    return false;
 
   int s = Mask[i] - i;
 
   // Check the rest of the elements to see if they are consecutive.
   for (++i; i != e; ++i) {
     int m = Mask[i];
-    if (m < 0)
-      continue;
-
-    Unary = Unary && (m < (int)e);
-    NeedsUnary = NeedsUnary || (m < s);
-
-    if (NeedsUnary && !Unary)
-      return false;
-    if (Unary && m != ((s+i) & (e-1)))
-      return false;
-    if (!Unary && m != (s+i))
+    if (m >= 0 && m != s+i)
       return false;
   }
   return true;
 }
 
-bool X86::isPALIGNRMask(ShuffleVectorSDNode *N) {
-  SmallVector<int, 8> M;
-  N->getMask(M);
-  return ::isPALIGNRMask(M, N->getValueType(0), true);
+/// isVSHUFPSYMask - Return true if the specified VECTOR_SHUFFLE operand
+/// specifies a shuffle of elements that is suitable for input to 256-bit
+/// VSHUFPSY.
+static bool isVSHUFPSYMask(const SmallVectorImpl<int> &Mask, EVT VT,
+                          const X86Subtarget *Subtarget) {
+  int NumElems = VT.getVectorNumElements();
+
+  if (!Subtarget->hasAVX() || VT.getSizeInBits() != 256)
+    return false;
+
+  if (NumElems != 8)
+    return false;
+
+  // VSHUFPSY divides the resulting vector into 4 chunks.
+  // The sources are also splitted into 4 chunks, and each destination
+  // chunk must come from a different source chunk.
+  //
+  //  SRC1 =>   X7    X6    X5    X4    X3    X2    X1    X0
+  //  SRC2 =>   Y7    Y6    Y5    Y4    Y3    Y2    Y1    Y9
+  //
+  //  DST  =>  Y7..Y4,   Y7..Y4,   X7..X4,   X7..X4,
+  //           Y3..Y0,   Y3..Y0,   X3..X0,   X3..X0
+  //
+  int QuarterSize = NumElems/4;
+  int HalfSize = QuarterSize*2;
+  for (int i = 0; i < QuarterSize; ++i)
+    if (!isUndefOrInRange(Mask[i], 0, HalfSize))
+      return false;
+  for (int i = QuarterSize; i < QuarterSize*2; ++i)
+    if (!isUndefOrInRange(Mask[i], NumElems, NumElems+HalfSize))
+      return false;
+
+  // The mask of the second half must be the same as the first but with
+  // the appropriate offsets. This works in the same way as VPERMILPS
+  // works with masks.
+  for (int i = QuarterSize*2; i < QuarterSize*3; ++i) {
+    if (!isUndefOrInRange(Mask[i], HalfSize, NumElems))
+      return false;
+    int FstHalfIdx = i-HalfSize;
+    if (Mask[FstHalfIdx] < 0)
+      continue;
+    if (!isUndefOrEqual(Mask[i], Mask[FstHalfIdx]+HalfSize))
+      return false;
+  }
+  for (int i = QuarterSize*3; i < NumElems; ++i) {
+    if (!isUndefOrInRange(Mask[i], NumElems+HalfSize, NumElems*2))
+      return false;
+    int FstHalfIdx = i-HalfSize;
+    if (Mask[FstHalfIdx] < 0)
+      continue;
+    if (!isUndefOrEqual(Mask[i], Mask[FstHalfIdx]+HalfSize))
+      return false;
+
+  }
+
+  return true;
+}
+
+/// getShuffleVSHUFPSYImmediate - Return the appropriate immediate to shuffle
+/// the specified VECTOR_MASK mask with VSHUFPSY instruction.
+static unsigned getShuffleVSHUFPSYImmediate(SDNode *N) {
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+  EVT VT = SVOp->getValueType(0);
+  int NumElems = VT.getVectorNumElements();
+
+  assert(NumElems == 8 && VT.getSizeInBits() == 256 &&
+         "Only supports v8i32 and v8f32 types");
+
+  int HalfSize = NumElems/2;
+  unsigned Mask = 0;
+  for (int i = 0; i != NumElems ; ++i) {
+    if (SVOp->getMaskElt(i) < 0)
+      continue;
+    // The mask of the first half must be equal to the second one.
+    unsigned Shamt = (i%HalfSize)*2;
+    unsigned Elt = SVOp->getMaskElt(i) % HalfSize;
+    Mask |= Elt << Shamt;
+  }
+
+  return Mask;
+}
+
+/// isVSHUFPDYMask - Return true if the specified VECTOR_SHUFFLE operand
+/// specifies a shuffle of elements that is suitable for input to 256-bit
+/// VSHUFPDY. This shuffle doesn't have the same restriction as the PS
+/// version and the mask of the second half isn't binded with the first
+/// one.
+static bool isVSHUFPDYMask(const SmallVectorImpl<int> &Mask, EVT VT,
+                           const X86Subtarget *Subtarget) {
+  int NumElems = VT.getVectorNumElements();
+
+  if (!Subtarget->hasAVX() || VT.getSizeInBits() != 256)
+    return false;
+
+  if (NumElems != 4)
+    return false;
+
+  // VSHUFPSY divides the resulting vector into 4 chunks.
+  // The sources are also splitted into 4 chunks, and each destination
+  // chunk must come from a different source chunk.
+  //
+  //  SRC1 =>      X3       X2       X1       X0
+  //  SRC2 =>      Y3       Y2       Y1       Y0
+  //
+  //  DST  =>  Y2..Y3,  X2..X3,  Y1..Y0,  X1..X0
+  //
+  int QuarterSize = NumElems/4;
+  int HalfSize = QuarterSize*2;
+  for (int i = 0; i < QuarterSize; ++i)
+    if (!isUndefOrInRange(Mask[i], 0, HalfSize))
+      return false;
+  for (int i = QuarterSize; i < QuarterSize*2; ++i)
+    if (!isUndefOrInRange(Mask[i], NumElems, NumElems+HalfSize))
+      return false;
+  for (int i = QuarterSize*2; i < QuarterSize*3; ++i)
+    if (!isUndefOrInRange(Mask[i], HalfSize, NumElems))
+      return false;
+  for (int i = QuarterSize*3; i < NumElems; ++i)
+    if (!isUndefOrInRange(Mask[i], NumElems+HalfSize, NumElems*2))
+      return false;
+
+  return true;
+}
+
+/// getShuffleVSHUFPDYImmediate - Return the appropriate immediate to shuffle
+/// the specified VECTOR_MASK mask with VSHUFPDY instruction.
+static unsigned getShuffleVSHUFPDYImmediate(SDNode *N) {
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+  EVT VT = SVOp->getValueType(0);
+  int NumElems = VT.getVectorNumElements();
+
+  assert(NumElems == 4 && VT.getSizeInBits() == 256 &&
+         "Only supports v4i64 and v4f64 types");
+
+  int HalfSize = NumElems/2;
+  unsigned Mask = 0;
+  for (int i = 0; i != NumElems ; ++i) {
+    if (SVOp->getMaskElt(i) < 0)
+      continue;
+    int Elt = SVOp->getMaskElt(i) % HalfSize;
+    Mask |= Elt << i;
+  }
+
+  return Mask;
 }
 
 /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
-/// specifies a shuffle of elements that is suitable for input to SHUFP*.
+/// specifies a shuffle of elements that is suitable for input to 128-bit
+/// SHUFPS and SHUFPD.
 static bool isSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) {
   int NumElems = VT.getVectorNumElements();
+
+  if (VT.getSizeInBits() != 128)
+    return false;
+
   if (NumElems != 2 && NumElems != 4)
     return false;
 
@@ -3204,7 +3428,13 @@ static bool isCommutedSHUFP(ShuffleVectorSDNode *N) {
 /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
 /// specifies a shuffle of elements that is suitable for input to MOVHLPS.
 bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) {
-  if (N->getValueType(0).getVectorNumElements() != 4)
+  EVT VT = N->getValueType(0);
+  unsigned NumElems = VT.getVectorNumElements();
+
+  if (VT.getSizeInBits() != 128)
+    return false;
+
+  if (NumElems != 4)
     return false;
 
   // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
@@ -3218,15 +3448,19 @@ bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) {
 /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
 /// <2, 3, 2, 3>
 bool X86::isMOVHLPS_v_undef_Mask(ShuffleVectorSDNode *N) {
-  unsigned NumElems = N->getValueType(0).getVectorNumElements();
+  EVT VT = N->getValueType(0);
+  unsigned NumElems = VT.getVectorNumElements();
+
+  if (VT.getSizeInBits() != 128)
+    return false;
 
   if (NumElems != 4)
     return false;
 
   return isUndefOrEqual(N->getMaskElt(0), 2) &&
-  isUndefOrEqual(N->getMaskElt(1), 3) &&
-  isUndefOrEqual(N->getMaskElt(2), 2) &&
-  isUndefOrEqual(N->getMaskElt(3), 3);
+         isUndefOrEqual(N->getMaskElt(1), 3) &&
+         isUndefOrEqual(N->getMaskElt(2), 2) &&
+         isUndefOrEqual(N->getMaskElt(3), 3);
 }
 
 /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
@@ -3273,20 +3507,22 @@ bool X86::isMOVLHPSMask(ShuffleVectorSDNode *N) {
 static bool isUNPCKLMask(const SmallVectorImpl<int> &Mask, EVT VT,
                          bool V2IsSplat = false) {
   int NumElts = VT.getVectorNumElements();
-  if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
+
+  assert((VT.is128BitVector() || VT.is256BitVector()) &&
+         "Unsupported vector type for unpckh");
+
+  if (VT.getSizeInBits() == 256 && NumElts != 4 && NumElts != 8)
     return false;
 
-  // Handle vector lengths > 128 bits.  Define a "section" as a set of
-  // 128 bits.  AVX defines UNPCK* to operate independently on 128-bit
-  // sections.
-  unsigned NumSections = VT.getSizeInBits() / 128;
-  if (NumSections == 0 ) NumSections = 1;  // Handle MMX
-  unsigned NumSectionElts = NumElts / NumSections;
+  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
+  // independently on 128-bit lanes.
+  unsigned NumLanes = VT.getSizeInBits()/128;
+  unsigned NumLaneElts = NumElts/NumLanes;
 
   unsigned Start = 0;
-  unsigned End = NumSectionElts;
-  for (unsigned s = 0; s < NumSections; ++s) {
-    for (unsigned i = Start, j = s * NumSectionElts;
+  unsigned End = NumLaneElts;
+  for (unsigned s = 0; s < NumLanes; ++s) {
+    for (unsigned i = Start, j = s * NumLaneElts;
          i != End;
          i += 2, ++j) {
       int BitI  = Mask[i];
@@ -3302,8 +3538,8 @@ static bool isUNPCKLMask(const SmallVectorImpl<int> &Mask, EVT VT,
       }
     }
     // Process the next 128 bits.
-    Start += NumSectionElts;
-    End += NumSectionElts;
+    Start += NumLaneElts;
+    End += NumLaneElts;
   }
 
   return true;
@@ -3320,21 +3556,38 @@ bool X86::isUNPCKLMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
 static bool isUNPCKHMask(const SmallVectorImpl<int> &Mask, EVT VT,
                          bool V2IsSplat = false) {
   int NumElts = VT.getVectorNumElements();
-  if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
+
+  assert((VT.is128BitVector() || VT.is256BitVector()) &&
+         "Unsupported vector type for unpckh");
+
+  if (VT.getSizeInBits() == 256 && NumElts != 4 && NumElts != 8)
     return false;
 
-  for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
-    int BitI  = Mask[i];
-    int BitI1 = Mask[i+1];
-    if (!isUndefOrEqual(BitI, j + NumElts/2))
-      return false;
-    if (V2IsSplat) {
-      if (isUndefOrEqual(BitI1, NumElts))
-        return false;
-    } else {
-      if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts))
+  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
+  // independently on 128-bit lanes.
+  unsigned NumLanes = VT.getSizeInBits()/128;
+  unsigned NumLaneElts = NumElts/NumLanes;
+
+  unsigned Start = 0;
+  unsigned End = NumLaneElts;
+  for (unsigned l = 0; l != NumLanes; ++l) {
+    for (unsigned i = Start, j = (l*NumLaneElts)+NumLaneElts/2;
+                             i != End; i += 2, ++j) {
+      int BitI  = Mask[i];
+      int BitI1 = Mask[i+1];
+      if (!isUndefOrEqual(BitI, j))
         return false;
+      if (V2IsSplat) {
+        if (isUndefOrEqual(BitI1, NumElts))
+          return false;
+      } else {
+        if (!isUndefOrEqual(BitI1, j+NumElts))
+          return false;
+      }
     }
+    // Process the next 128 bits.
+    Start += NumLaneElts;
+    End += NumLaneElts;
   }
   return true;
 }
@@ -3353,16 +3606,21 @@ static bool isUNPCKL_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) {
   if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
     return false;
 
-  // Handle vector lengths > 128 bits.  Define a "section" as a set of
-  // 128 bits.  AVX defines UNPCK* to operate independently on 128-bit
-  // sections.
-  unsigned NumSections = VT.getSizeInBits() / 128;
-  if (NumSections == 0 ) NumSections = 1;  // Handle MMX
-  unsigned NumSectionElts = NumElems / NumSections;
+  // For 256-bit i64/f64, use MOVDDUPY instead, so reject the matching pattern
+  // FIXME: Need a better way to get rid of this, there's no latency difference
+  // between UNPCKLPD and MOVDDUP, the later should always be checked first and
+  // the former later. We should also remove the "_undef" special mask.
+  if (NumElems == 4 && VT.getSizeInBits() == 256)
+    return false;
 
-  for (unsigned s = 0; s < NumSections; ++s) {
-    for (unsigned i = s * NumSectionElts, j = s * NumSectionElts;
-         i != NumSectionElts * (s + 1);
+  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
+  // independently on 128-bit lanes.
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  unsigned NumLaneElts = NumElems / NumLanes;
+
+  for (unsigned s = 0; s < NumLanes; ++s) {
+    for (unsigned i = s * NumLaneElts, j = s * NumLaneElts;
+         i != NumLaneElts * (s + 1);
          i += 2, ++j) {
       int BitI  = Mask[i];
       int BitI1 = Mask[i+1];
@@ -3433,6 +3691,189 @@ bool X86::isMOVLMask(ShuffleVectorSDNode *N) {
   return ::isMOVLMask(M, N->getValueType(0));
 }
 
+/// isVPERM2F128Mask - Match 256-bit shuffles where the elements are considered
+/// as permutations between 128-bit chunks or halves. As an example: this
+/// shuffle bellow:
+///   vector_shuffle <4, 5, 6, 7, 12, 13, 14, 15>
+/// The first half comes from the second half of V1 and the second half from the
+/// the second half of V2.
+static bool isVPERM2F128Mask(const SmallVectorImpl<int> &Mask, EVT VT,
+                             const X86Subtarget *Subtarget) {
+  if (!Subtarget->hasAVX() || VT.getSizeInBits() != 256)
+    return false;
+
+  // The shuffle result is divided into half A and half B. In total the two
+  // sources have 4 halves, namely: C, D, E, F. The final values of A and
+  // B must come from C, D, E or F.
+  int HalfSize = VT.getVectorNumElements()/2;
+  bool MatchA = false, MatchB = false;
+
+  // Check if A comes from one of C, D, E, F.
+  for (int Half = 0; Half < 4; ++Half) {
+    if (isSequentialOrUndefInRange(Mask, 0, HalfSize, Half*HalfSize)) {
+      MatchA = true;
+      break;
+    }
+  }
+
+  // Check if B comes from one of C, D, E, F.
+  for (int Half = 0; Half < 4; ++Half) {
+    if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, Half*HalfSize)) {
+      MatchB = true;
+      break;
+    }
+  }
+
+  return MatchA && MatchB;
+}
+
+/// getShuffleVPERM2F128Immediate - Return the appropriate immediate to shuffle
+/// the specified VECTOR_MASK mask with VPERM2F128 instructions.
+static unsigned getShuffleVPERM2F128Immediate(SDNode *N) {
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+  EVT VT = SVOp->getValueType(0);
+
+  int HalfSize = VT.getVectorNumElements()/2;
+
+  int FstHalf = 0, SndHalf = 0;
+  for (int i = 0; i < HalfSize; ++i) {
+    if (SVOp->getMaskElt(i) > 0) {
+      FstHalf = SVOp->getMaskElt(i)/HalfSize;
+      break;
+    }
+  }
+  for (int i = HalfSize; i < HalfSize*2; ++i) {
+    if (SVOp->getMaskElt(i) > 0) {
+      SndHalf = SVOp->getMaskElt(i)/HalfSize;
+      break;
+    }
+  }
+
+  return (FstHalf | (SndHalf << 4));
+}
+
+/// isVPERMILPDMask - Return true if the specified VECTOR_SHUFFLE operand
+/// specifies a shuffle of elements that is suitable for input to VPERMILPD*.
+/// Note that VPERMIL mask matching is different depending whether theunderlying
+/// type is 32 or 64. In the VPERMILPS the high half of the mask should point
+/// to the same elements of the low, but to the higher half of the source.
+/// In VPERMILPD the two lanes could be shuffled independently of each other
+/// with the same restriction that lanes can't be crossed.
+static bool isVPERMILPDMask(const SmallVectorImpl<int> &Mask, EVT VT,
+                            const X86Subtarget *Subtarget) {
+  int NumElts = VT.getVectorNumElements();
+  int NumLanes = VT.getSizeInBits()/128;
+
+  if (!Subtarget->hasAVX())
+    return false;
+
+  // Only match 256-bit with 64-bit types
+  if (VT.getSizeInBits() != 256 || NumElts != 4)
+    return false;
+
+  // The mask on the high lane is independent of the low. Both can match
+  // any element in inside its own lane, but can't cross.
+  int LaneSize = NumElts/NumLanes;
+  for (int l = 0; l < NumLanes; ++l)
+    for (int i = l*LaneSize; i < LaneSize*(l+1); ++i) {
+      int LaneStart = l*LaneSize;
+      if (!isUndefOrInRange(Mask[i], LaneStart, LaneStart+LaneSize))
+        return false;
+    }
+
+  return true;
+}
+
+/// isVPERMILPSMask - Return true if the specified VECTOR_SHUFFLE operand
+/// specifies a shuffle of elements that is suitable for input to VPERMILPS*.
+/// Note that VPERMIL mask matching is different depending whether theunderlying
+/// type is 32 or 64. In the VPERMILPS the high half of the mask should point
+/// to the same elements of the low, but to the higher half of the source.
+/// In VPERMILPD the two lanes could be shuffled independently of each other
+/// with the same restriction that lanes can't be crossed.
+static bool isVPERMILPSMask(const SmallVectorImpl<int> &Mask, EVT VT,
+                            const X86Subtarget *Subtarget) {
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned NumLanes = VT.getSizeInBits()/128;
+
+  if (!Subtarget->hasAVX())
+    return false;
+
+  // Only match 256-bit with 32-bit types
+  if (VT.getSizeInBits() != 256 || NumElts != 8)
+    return false;
+
+  // The mask on the high lane should be the same as the low. Actually,
+  // they can differ if any of the corresponding index in a lane is undef
+  // and the other stays in range.
+  int LaneSize = NumElts/NumLanes;
+  for (int i = 0; i < LaneSize; ++i) {
+    int HighElt = i+LaneSize;
+    bool HighValid = isUndefOrInRange(Mask[HighElt], LaneSize, NumElts);
+    bool LowValid = isUndefOrInRange(Mask[i], 0, LaneSize);
+
+    if (!HighValid || !LowValid)
+      return false;
+    if (Mask[i] < 0 || Mask[HighElt] < 0)
+      continue;
+    if (Mask[HighElt]-Mask[i] != LaneSize)
+      return false;
+  }
+
+  return true;
+}
+
+/// getShuffleVPERMILPSImmediate - Return the appropriate immediate to shuffle
+/// the specified VECTOR_MASK mask with VPERMILPS* instructions.
+static unsigned getShuffleVPERMILPSImmediate(SDNode *N) {
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+  EVT VT = SVOp->getValueType(0);
+
+  int NumElts = VT.getVectorNumElements();
+  int NumLanes = VT.getSizeInBits()/128;
+  int LaneSize = NumElts/NumLanes;
+
+  // Although the mask is equal for both lanes do it twice to get the cases
+  // where a mask will match because the same mask element is undef on the
+  // first half but valid on the second. This would get pathological cases
+  // such as: shuffle <u, 0, 1, 2, 4, 4, 5, 6>, which is completely valid.
+  unsigned Mask = 0;
+  for (int l = 0; l < NumLanes; ++l) {
+    for (int i = 0; i < LaneSize; ++i) {
+      int MaskElt = SVOp->getMaskElt(i+(l*LaneSize));
+      if (MaskElt < 0)
+        continue;
+      if (MaskElt >= LaneSize)
+        MaskElt -= LaneSize;
+      Mask |= MaskElt << (i*2);
+    }
+  }
+
+  return Mask;
+}
+
+/// getShuffleVPERMILPDImmediate - Return the appropriate immediate to shuffle
+/// the specified VECTOR_MASK mask with VPERMILPD* instructions.
+static unsigned getShuffleVPERMILPDImmediate(SDNode *N) {
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+  EVT VT = SVOp->getValueType(0);
+
+  int NumElts = VT.getVectorNumElements();
+  int NumLanes = VT.getSizeInBits()/128;
+
+  unsigned Mask = 0;
+  int LaneSize = NumElts/NumLanes;
+  for (int l = 0; l < NumLanes; ++l)
+    for (int i = l*LaneSize; i < LaneSize*(l+1); ++i) {
+      int MaskElt = SVOp->getMaskElt(i);
+      if (MaskElt < 0)
+        continue;
+      Mask |= (MaskElt-l*LaneSize) << i;
+    }
+
+  return Mask;
+}
+
 /// isCommutedMOVL - Returns true if the shuffle mask is except the reverse
 /// of what x86 movss want. X86 movs requires the lowest  element to be lowest
 /// element of vector 2 and the other elements to come from vector 1 in order.
@@ -3463,58 +3904,92 @@ static bool isCommutedMOVL(ShuffleVectorSDNode *N, bool V2IsSplat = false,
 
 /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
 /// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
-bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N) {
-  if (N->getValueType(0).getVectorNumElements() != 4)
+/// Masks to match: <1, 1, 3, 3> or <1, 1, 3, 3, 5, 5, 7, 7>
+bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N,
+                         const X86Subtarget *Subtarget) {
+  if (!Subtarget->hasSSE3() && !Subtarget->hasAVX())
     return false;
 
-  // Expect 1, 1, 3, 3
-  for (unsigned i = 0; i < 2; ++i) {
-    int Elt = N->getMaskElt(i);
-    if (Elt >= 0 && Elt != 1)
-      return false;
-  }
+  // The second vector must be undef
+  if (N->getOperand(1).getOpcode() != ISD::UNDEF)
+    return false;
 
-  bool HasHi = false;
-  for (unsigned i = 2; i < 4; ++i) {
-    int Elt = N->getMaskElt(i);
-    if (Elt >= 0 && Elt != 3)
+  EVT VT = N->getValueType(0);
+  unsigned NumElems = VT.getVectorNumElements();
+
+  if ((VT.getSizeInBits() == 128 && NumElems != 4) ||
+      (VT.getSizeInBits() == 256 && NumElems != 8))
+    return false;
+
+  // "i+1" is the value the indexed mask element must have
+  for (unsigned i = 0; i < NumElems; i += 2)
+    if (!isUndefOrEqual(N->getMaskElt(i), i+1) ||
+        !isUndefOrEqual(N->getMaskElt(i+1), i+1))
       return false;
-    if (Elt == 3)
-      HasHi = true;
-  }
-  // Don't use movshdup if it can be done with a shufps.
-  // FIXME: verify that matching u, u, 3, 3 is what we want.
-  return HasHi;
+
+  return true;
 }
 
 /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
 /// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
-bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N) {
-  if (N->getValueType(0).getVectorNumElements() != 4)
+/// Masks to match: <0, 0, 2, 2> or <0, 0, 2, 2, 4, 4, 6, 6>
+bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N,
+                         const X86Subtarget *Subtarget) {
+  if (!Subtarget->hasSSE3() && !Subtarget->hasAVX())
     return false;
 
-  // Expect 0, 0, 2, 2
-  for (unsigned i = 0; i < 2; ++i)
-    if (N->getMaskElt(i) > 0)
+  // The second vector must be undef
+  if (N->getOperand(1).getOpcode() != ISD::UNDEF)
+    return false;
+
+  EVT VT = N->getValueType(0);
+  unsigned NumElems = VT.getVectorNumElements();
+
+  if ((VT.getSizeInBits() == 128 && NumElems != 4) ||
+      (VT.getSizeInBits() == 256 && NumElems != 8))
+    return false;
+
+  // "i" is the value the indexed mask element must have
+  for (unsigned i = 0; i < NumElems; i += 2)
+    if (!isUndefOrEqual(N->getMaskElt(i), i) ||
+        !isUndefOrEqual(N->getMaskElt(i+1), i))
       return false;
 
-  bool HasHi = false;
-  for (unsigned i = 2; i < 4; ++i) {
-    int Elt = N->getMaskElt(i);
-    if (Elt >= 0 && Elt != 2)
+  return true;
+}
+
+/// isMOVDDUPYMask - Return true if the specified VECTOR_SHUFFLE operand
+/// specifies a shuffle of elements that is suitable for input to 256-bit
+/// version of MOVDDUP.
+static bool isMOVDDUPYMask(ShuffleVectorSDNode *N,
+                           const X86Subtarget *Subtarget) {
+  EVT VT = N->getValueType(0);
+  int NumElts = VT.getVectorNumElements();
+  bool V2IsUndef = N->getOperand(1).getOpcode() == ISD::UNDEF;
+
+  if (!Subtarget->hasAVX() || VT.getSizeInBits() != 256 ||
+      !V2IsUndef || NumElts != 4)
+    return false;
+
+  for (int i = 0; i != NumElts/2; ++i)
+    if (!isUndefOrEqual(N->getMaskElt(i), 0))
       return false;
-    if (Elt == 2)
-      HasHi = true;
-  }
-  // Don't use movsldup if it can be done with a shufps.
-  return HasHi;
+  for (int i = NumElts/2; i != NumElts; ++i)
+    if (!isUndefOrEqual(N->getMaskElt(i), NumElts/2))
+      return false;
+  return true;
 }
 
 /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
-/// specifies a shuffle of elements that is suitable for input to MOVDDUP.
+/// specifies a shuffle of elements that is suitable for input to 128-bit
+/// version of MOVDDUP.
 bool X86::isMOVDDUPMask(ShuffleVectorSDNode *N) {
-  int e = N->getValueType(0).getVectorNumElements() / 2;
+  EVT VT = N->getValueType(0);
 
+  if (VT.getSizeInBits() != 128)
+    return false;
+
+  int e = VT.getVectorNumElements() / 2;
   for (int i = 0; i < e; ++i)
     if (!isUndefOrEqual(N->getMaskElt(i), i))
       return false;
@@ -3627,6 +4102,7 @@ unsigned X86::getShufflePALIGNRImmediate(SDNode *N) {
     if (Val >= 0)
       break;
   }
+  assert(Val - i > 0 && "PALIGNR imm should be positive");
   return (Val - i) * EltSize;
 }
 
@@ -3644,7 +4120,6 @@ unsigned X86::getExtractVEXTRACTF128Immediate(SDNode *N) {
   EVT ElVT = VecVT.getVectorElementType();
 
   unsigned NumElemsPerChunk = 128 / ElVT.getSizeInBits();
-
   return Index / NumElemsPerChunk;
 }
 
@@ -3662,7 +4137,6 @@ unsigned X86::getInsertVINSERTF128Immediate(SDNode *N) {
   EVT ElVT = VecVT.getVectorElementType();
 
   unsigned NumElemsPerChunk = 128 / ElVT.getSizeInBits();
-
   return Index / NumElemsPerChunk;
 }
 
@@ -3716,7 +4190,10 @@ static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask, EVT VT) {
 /// V1 (and in order), and the upper half elements should come from the upper
 /// half of V2 (and in order).
 static bool ShouldXformToMOVHLPS(ShuffleVectorSDNode *Op) {
-  if (Op->getValueType(0).getVectorNumElements() != 4)
+  EVT VT = Op->getValueType(0);
+  if (VT.getSizeInBits() != 128)
+    return false;
+  if (VT.getVectorNumElements() != 4)
     return false;
   for (unsigned i = 0, e = 2; i != e; ++i)
     if (!isUndefOrEqual(Op->getMaskElt(i), i+2))
@@ -3748,6 +4225,10 @@ static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) {
 /// MOVLP, it must be either a vector load or a scalar load to vector.
 static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
                                ShuffleVectorSDNode *Op) {
+  EVT VT = Op->getValueType(0);
+  if (VT.getSizeInBits() != 128)
+    return false;
+
   if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
     return false;
   // Is V2 is a vector load, don't do this transformation. We will try to use
@@ -3755,7 +4236,7 @@ static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
   if (ISD::isNON_EXTLoad(V2))
     return false;
 
-  unsigned NumElems = Op->getValueType(0).getVectorNumElements();
+  unsigned NumElems = VT.getVectorNumElements();
 
   if (NumElems != 2 && NumElems != 4)
     return false;
@@ -3811,7 +4292,7 @@ static bool isZeroShuffle(ShuffleVectorSDNode *N) {
 
 /// getZeroVector - Returns a vector of specified type with all zero elements.
 ///
-static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG,
+static SDValue getZeroVector(EVT VT, bool HasXMMInt, SelectionDAG &DAG,
                              DebugLoc dl) {
   assert(VT.isVector() && "Expected a vector type");
 
@@ -3819,7 +4300,7 @@ static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG,
   // to their dest type. This ensures they get CSE'd.
   SDValue Vec;
   if (VT.getSizeInBits() == 128) {  // SSE
-    if (HasSSE2) {  // SSE2
+    if (HasXMMInt) {  // SSE2
       SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
     } else { // SSE1
@@ -3838,21 +4319,25 @@ static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG,
 }
 
 /// getOnesVector - Returns a vector of specified type with all bits set.
-/// Always build ones vectors as <4 x i32> or <8 x i32> bitcasted to
-/// their original type, ensuring they get CSE'd.
+/// Always build ones vectors as <4 x i32>. For 256-bit types, use two
+/// <4 x i32> inserted in a <8 x i32> appropriately. Then bitcast to their
+/// original type, ensuring they get CSE'd.
 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) {
   assert(VT.isVector() && "Expected a vector type");
   assert((VT.is128BitVector() || VT.is256BitVector())
          && "Expected a 128-bit or 256-bit vector type");
 
   SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32);
+  SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
+                            Cst, Cst, Cst, Cst);
 
-  SDValue Vec;
   if (VT.is256BitVector()) {
-    SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
-    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops, 8);
-  } else
-    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
+    SDValue InsV = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, MVT::v8i32),
+                              Vec, DAG.getConstant(0, MVT::i32), DAG, dl);
+    Vec = Insert128BitVector(InsV, Vec,
+                  DAG.getConstant(4 /* NumElems/2 */, MVT::i32), DAG, dl);
+  }
+
   return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
 }
 
@@ -3902,7 +4387,7 @@ static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }
 
-/// getUnpackhMask - Returns a vector_shuffle node for an unpackh operation.
+/// getUnpackh - Returns a vector_shuffle node for an unpackh operation.
 static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
                           SDValue V2) {
   unsigned NumElems = VT.getVectorNumElements();
@@ -3915,31 +4400,95 @@ static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }
 
-/// PromoteSplat - Promote a splat of v4i32, v8i16 or v16i8 to v4f32.
-static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) {
-  EVT PVT = MVT::v4f32;
-  EVT VT = SV->getValueType(0);
-  DebugLoc dl = SV->getDebugLoc();
-  SDValue V1 = SV->getOperand(0);
+// PromoteSplati8i16 - All i16 and i8 vector types can't be used directly by
+// a generic shuffle instruction because the target has no such instructions.
+// Generate shuffles which repeat i16 and i8 several times until they can be
+// represented by v4f32 and then be manipulated by target suported shuffles.
+static SDValue PromoteSplati8i16(SDValue V, SelectionDAG &DAG, int &EltNo) {
+  EVT VT = V.getValueType();
   int NumElems = VT.getVectorNumElements();
-  int EltNo = SV->getSplatIndex();
+  DebugLoc dl = V.getDebugLoc();
 
-  // unpack elements to the correct location
   while (NumElems > 4) {
     if (EltNo < NumElems/2) {
-      V1 = getUnpackl(DAG, dl, VT, V1, V1);
+      V = getUnpackl(DAG, dl, VT, V, V);
     } else {
-      V1 = getUnpackh(DAG, dl, VT, V1, V1);
+      V = getUnpackh(DAG, dl, VT, V, V);
       EltNo -= NumElems/2;
     }
     NumElems >>= 1;
   }
+  return V;
+}
 
-  // Perform the splat.
-  int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
-  V1 = DAG.getNode(ISD::BITCAST, dl, PVT, V1);
-  V1 = DAG.getVectorShuffle(PVT, dl, V1, DAG.getUNDEF(PVT), &SplatMask[0]);
-  return DAG.getNode(ISD::BITCAST, dl, VT, V1);
+/// getLegalSplat - Generate a legal splat with supported x86 shuffles
+static SDValue getLegalSplat(SelectionDAG &DAG, SDValue V, int EltNo) {
+  EVT VT = V.getValueType();
+  DebugLoc dl = V.getDebugLoc();
+  assert((VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256)
+         && "Vector size not supported");
+
+  if (VT.getSizeInBits() == 128) {
+    V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V);
+    int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
+    V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32),
+                             &SplatMask[0]);
+  } else {
+    // To use VPERMILPS to splat scalars, the second half of indicies must
+    // refer to the higher part, which is a duplication of the lower one,
+    // because VPERMILPS can only handle in-lane permutations.
+    int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo,
+                         EltNo+4, EltNo+4, EltNo+4, EltNo+4 };
+
+    V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V);
+    V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32),
+                             &SplatMask[0]);
+  }
+
+  return DAG.getNode(ISD::BITCAST, dl, VT, V);
+}
+
+/// PromoteSplat - Splat is promoted to target supported vector shuffles.
+static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) {
+  EVT SrcVT = SV->getValueType(0);
+  SDValue V1 = SV->getOperand(0);
+  DebugLoc dl = SV->getDebugLoc();
+
+  int EltNo = SV->getSplatIndex();
+  int NumElems = SrcVT.getVectorNumElements();
+  unsigned Size = SrcVT.getSizeInBits();
+
+  assert(((Size == 128 && NumElems > 4) || Size == 256) &&
+          "Unknown how to promote splat for type");
+
+  // Extract the 128-bit part containing the splat element and update
+  // the splat element index when it refers to the higher register.
+  if (Size == 256) {
+    unsigned Idx = (EltNo > NumElems/2) ? NumElems/2 : 0;
+    V1 = Extract128BitVector(V1, DAG.getConstant(Idx, MVT::i32), DAG, dl);
+    if (Idx > 0)
+      EltNo -= NumElems/2;
+  }
+
+  // All i16 and i8 vector types can't be used directly by a generic shuffle
+  // instruction because the target has no such instruction. Generate shuffles
+  // which repeat i16 and i8 several times until they fit in i32, and then can
+  // be manipulated by target suported shuffles.
+  EVT EltVT = SrcVT.getVectorElementType();
+  if (EltVT == MVT::i8 || EltVT == MVT::i16)
+    V1 = PromoteSplati8i16(V1, DAG, EltNo);
+
+  // Recreate the 256-bit vector and place the same 128-bit vector
+  // into the low and high part. This is necessary because we want
+  // to use VPERM* to shuffle the vectors
+  if (Size == 256) {
+    SDValue InsV = Insert128BitVector(DAG.getUNDEF(SrcVT), V1,
+                         DAG.getConstant(0, MVT::i32), DAG, dl);
+    V1 = Insert128BitVector(InsV, V1,
+               DAG.getConstant(NumElems/2, MVT::i32), DAG, dl);
+  }
+
+  return getLegalSplat(DAG, V1, EltNo);
 }
 
 /// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
@@ -3947,11 +4496,11 @@ static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) {
 /// element of V2 is swizzled into the zero/undef vector, landing at element
 /// Idx.  This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
-                                             bool isZero, bool HasSSE2,
-                                             SelectionDAG &DAG) {
+                                           bool isZero, bool HasXMMInt,
+                                           SelectionDAG &DAG) {
   EVT VT = V2.getValueType();
   SDValue V1 = isZero
-    ? getZeroVector(VT, HasSSE2, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT);
+    ? getZeroVector(VT, HasXMMInt, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT);
   unsigned NumElems = VT.getVectorNumElements();
   SmallVector<int, 16> MaskVec;
   for (unsigned i = 0; i != NumElems; ++i)
@@ -4005,6 +4554,8 @@ static SDValue getShuffleScalarElt(SDNode *N, int Index, SelectionDAG &DAG,
       break;
     case X86ISD::UNPCKHPS:
     case X86ISD::UNPCKHPD:
+    case X86ISD::VUNPCKHPSY:
+    case X86ISD::VUNPCKHPDY:
       DecodeUNPCKHPMask(NumElems, ShuffleMask);
       break;
     case X86ISD::PUNPCKLBW:
@@ -4015,8 +4566,6 @@ static SDValue getShuffleScalarElt(SDNode *N, int Index, SelectionDAG &DAG,
       break;
     case X86ISD::UNPCKLPS:
     case X86ISD::UNPCKLPD:
-    case X86ISD::VUNPCKLPS:
-    case X86ISD::VUNPCKLPD:
     case X86ISD::VUNPCKLPSY:
     case X86ISD::VUNPCKLPDY:
       DecodeUNPCKLPMask(VT, ShuffleMask);
@@ -4052,8 +4601,41 @@ static SDValue getShuffleScalarElt(SDNode *N, int Index, SelectionDAG &DAG,
       return getShuffleScalarElt(V.getOperand(OpNum).getNode(), Index, DAG,
                                  Depth+1);
     }
+    case X86ISD::VPERMILPS:
+      ImmN = N->getOperand(N->getNumOperands()-1);
+      DecodeVPERMILPSMask(4, cast<ConstantSDNode>(ImmN)->getZExtValue(),
+                        ShuffleMask);
+      break;
+    case X86ISD::VPERMILPSY:
+      ImmN = N->getOperand(N->getNumOperands()-1);
+      DecodeVPERMILPSMask(8, cast<ConstantSDNode>(ImmN)->getZExtValue(),
+                        ShuffleMask);
+      break;
+    case X86ISD::VPERMILPD:
+      ImmN = N->getOperand(N->getNumOperands()-1);
+      DecodeVPERMILPDMask(2, cast<ConstantSDNode>(ImmN)->getZExtValue(),
+                        ShuffleMask);
+      break;
+    case X86ISD::VPERMILPDY:
+      ImmN = N->getOperand(N->getNumOperands()-1);
+      DecodeVPERMILPDMask(4, cast<ConstantSDNode>(ImmN)->getZExtValue(),
+                        ShuffleMask);
+      break;
+    case X86ISD::VPERM2F128:
+      ImmN = N->getOperand(N->getNumOperands()-1);
+      DecodeVPERM2F128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(),
+                           ShuffleMask);
+      break;
+    case X86ISD::MOVDDUP:
+    case X86ISD::MOVLHPD:
+    case X86ISD::MOVLPD:
+    case X86ISD::MOVLPS:
+    case X86ISD::MOVSHDUP:
+    case X86ISD::MOVSLDUP:
+    case X86ISD::PALIGN:
+      return SDValue(); // Not yet implemented.
     default:
-      assert("not implemented for target shuffle node");
+      assert(0 && "unknown target shuffle node");
       return SDValue();
     }
 
@@ -4205,6 +4787,11 @@ static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
 /// logical left or right shift of a vector.
 static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
                           bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
+  // Although the logic below support any bitwidth size, there are no
+  // shift instructions which handle more than 128-bit vectors.
+  if (SVOp->getValueType(0).getSizeInBits() > 128)
+    return false;
+
   if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) ||
       isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt))
     return true;
@@ -4295,6 +4882,7 @@ static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
                          unsigned NumBits, SelectionDAG &DAG,
                          const TargetLowering &TLI, DebugLoc dl) {
+  assert(VT.getSizeInBits() == 128 && "Unknown type for VShift");
   EVT ShVT = MVT::v2i64;
   unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL;
   SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp);
@@ -4333,42 +4921,52 @@ X86TargetLowering::LowerAsSplatVectorLoad(SDValue SrcOp, EVT VT, DebugLoc dl,
       return SDValue();
     }
 
+    // FIXME: 256-bit vector instructions don't require a strict alignment,
+    // improve this code to support it better.
+    unsigned RequiredAlign = VT.getSizeInBits()/8;
     SDValue Chain = LD->getChain();
-    // Make sure the stack object alignment is at least 16.
+    // Make sure the stack object alignment is at least 16 or 32.
     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
-    if (DAG.InferPtrAlignment(Ptr) < 16) {
+    if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) {
       if (MFI->isFixedObjectIndex(FI)) {
         // Can't change the alignment. FIXME: It's possible to compute
         // the exact stack offset and reference FI + adjust offset instead.
         // If someone *really* cares about this. That's the way to implement it.
         return SDValue();
       } else {
-        MFI->setObjectAlignment(FI, 16);
+        MFI->setObjectAlignment(FI, RequiredAlign);
       }
     }
 
-    // (Offset % 16) must be multiple of 4. Then address is then
+    // (Offset % 16 or 32) must be multiple of 4. Then address is then
     // Ptr + (Offset & ~15).
     if (Offset < 0)
       return SDValue();
-    if ((Offset % 16) & 3)
+    if ((Offset % RequiredAlign) & 3)
       return SDValue();
-    int64_t StartOffset = Offset & ~15;
+    int64_t StartOffset = Offset & ~(RequiredAlign-1);
     if (StartOffset)
       Ptr = DAG.getNode(ISD::ADD, Ptr.getDebugLoc(), Ptr.getValueType(),
                         Ptr,DAG.getConstant(StartOffset, Ptr.getValueType()));
 
     int EltNo = (Offset - StartOffset) >> 2;
-    int Mask[4] = { EltNo, EltNo, EltNo, EltNo };
-    EVT VT = (PVT == MVT::i32) ? MVT::v4i32 : MVT::v4f32;
-    SDValue V1 = DAG.getLoad(VT, dl, Chain, Ptr,
+    int NumElems = VT.getVectorNumElements();
+
+    EVT CanonVT = VT.getSizeInBits() == 128 ? MVT::v4i32 : MVT::v8i32;
+    EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
+    SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
                              LD->getPointerInfo().getWithOffset(StartOffset),
                              false, false, 0);
-    // Canonicalize it to a v4i32 shuffle.
-    V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
-    return DAG.getNode(ISD::BITCAST, dl, VT,
-                       DAG.getVectorShuffle(MVT::v4i32, dl, V1,
-                                            DAG.getUNDEF(MVT::v4i32),&Mask[0]));
+
+    // Canonicalize it to a v4i32 or v8i32 shuffle.
+    SmallVector<int, 8> Mask;
+    for (int i = 0; i < NumElems; ++i)
+      Mask.push_back(EltNo);
+
+    V1 = DAG.getNode(ISD::BITCAST, dl, CanonVT, V1);
+    return DAG.getNode(ISD::BITCAST, dl, NVT,
+                       DAG.getVectorShuffle(CanonVT, dl, V1,
+                                            DAG.getUNDEF(CanonVT),&Mask[0]));
   }
 
   return SDValue();
@@ -4428,12 +5026,16 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts,
                        LDBase->getPointerInfo(),
                        LDBase->isVolatile(), LDBase->isNonTemporal(),
                        LDBase->getAlignment());
-  } else if (NumElems == 4 && LastLoadedElt == 1) {
+  } else if (NumElems == 4 && LastLoadedElt == 1 &&
+             DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) {
     SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
     SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
-    SDValue ResNode = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys,
-                                              Ops, 2, MVT::i32,
-                                              LDBase->getMemOperand());
+    SDValue ResNode =
+        DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, 2, MVT::i64,
+                                LDBase->getPointerInfo(),
+                                LDBase->getAlignment(),
+                                false/*isVolatile*/, true/*ReadMem*/,
+                                false/*WriteMem*/);
     return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
   }
   return SDValue();
@@ -4445,47 +5047,26 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
 
   EVT VT = Op.getValueType();
   EVT ExtVT = VT.getVectorElementType();
-
   unsigned NumElems = Op.getNumOperands();
 
-  // For AVX-length vectors, build the individual 128-bit pieces and
-  // use shuffles to put them in place.
-  if (VT.getSizeInBits() > 256 &&
-      Subtarget->hasAVX() &&
-      !ISD::isBuildVectorAllZeros(Op.getNode())) {
-    SmallVector<SDValue, 8> V;
-    V.resize(NumElems);
-    for (unsigned i = 0; i < NumElems; ++i) {
-      V[i] = Op.getOperand(i);
-    }
-
-    EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2);
-
-    // Build the lower subvector.
-    SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[0], NumElems/2);
-    // Build the upper subvector.
-    SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[NumElems / 2],
-                                NumElems/2);
-
-    return ConcatVectors(Lower, Upper, DAG);
-  }
-
-  // All zero's:
-  //  - pxor (SSE2), xorps (SSE1), vpxor (128 AVX), xorp[s|d] (256 AVX)
-  // All one's:
-  //  - pcmpeqd (SSE2 and 128 AVX), fallback to constant pools (256 AVX)
-  if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
-      ISD::isBuildVectorAllOnes(Op.getNode())) {
-    // Canonicalize this to <4 x i32> or <8 x 32> (SSE) to
-    // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are
-    // eliminated on x86-32 hosts.
+  // Vectors containing all zeros can be matched by pxor and xorps later
+  if (ISD::isBuildVectorAllZeros(Op.getNode())) {
+    // Canonicalize this to <4 x i32> to 1) ensure the zero vectors are CSE'd
+    // and 2) ensure that i64 scalars are eliminated on x86-32 hosts.
     if (Op.getValueType() == MVT::v4i32 ||
         Op.getValueType() == MVT::v8i32)
       return Op;
 
-    if (ISD::isBuildVectorAllOnes(Op.getNode()))
-      return getOnesVector(Op.getValueType(), DAG, dl);
-    return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG, dl);
+    return getZeroVector(Op.getValueType(), Subtarget->hasXMMInt(), DAG, dl);
+  }
+
+  // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
+  // vectors or broken into v4i32 operations on 256-bit vectors.
+  if (ISD::isBuildVectorAllOnes(Op.getNode())) {
+    if (Op.getValueType() == MVT::v4i32)
+      return Op;
+
+    return getOnesVector(Op.getValueType(), DAG, dl);
   }
 
   unsigned EVTBits = ExtVT.getSizeInBits();
@@ -4538,7 +5119,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
         Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item);
         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item);
         Item = getShuffleVectorZeroOrUndef(Item, 0, true,
-                                           Subtarget->hasSSE2(), DAG);
+                                           Subtarget->hasXMMInt(), DAG);
 
         // Now we have our 32-bit value zero extended in the low element of
         // a vector.  If Idx != 0, swizzle it into place.
@@ -4566,7 +5147,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
           (ExtVT == MVT::i64 && Subtarget->is64Bit())) {
         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
         // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
-        return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget->hasSSE2(),
+        return getShuffleVectorZeroOrUndef(Item, 0, true,Subtarget->hasXMMInt(),
                                            DAG);
       } else if (ExtVT == MVT::i16 || ExtVT == MVT::i8) {
         Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
@@ -4574,7 +5155,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
         EVT MiddleVT = MVT::v4i32;
         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MiddleVT, Item);
         Item = getShuffleVectorZeroOrUndef(Item, 0, true,
-                                           Subtarget->hasSSE2(), DAG);
+                                           Subtarget->hasXMMInt(), DAG);
         return DAG.getNode(ISD::BITCAST, dl, VT, Item);
       }
     }
@@ -4603,7 +5184,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
 
       // Turn it into a shuffle of zero and zero-extended scalar to vector.
       Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
-                                         Subtarget->hasSSE2(), DAG);
+                                         Subtarget->hasXMMInt(), DAG);
       SmallVector<int, 8> MaskVec;
       for (unsigned i = 0; i < NumElems; i++)
         MaskVec.push_back(i == Idx ? 0 : 1);
@@ -4631,6 +5212,27 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   if (IsAllConstants)
     return SDValue();
 
+  // For AVX-length vectors, build the individual 128-bit pieces and use
+  // shuffles to put them in place.
+  if (VT.getSizeInBits() == 256 && !ISD::isBuildVectorAllZeros(Op.getNode())) {
+    SmallVector<SDValue, 32> V;
+    for (unsigned i = 0; i < NumElems; ++i)
+      V.push_back(Op.getOperand(i));
+
+    EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2);
+
+    // Build both the lower and upper subvector.
+    SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[0], NumElems/2);
+    SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[NumElems / 2],
+                                NumElems/2);
+
+    // Recreate the wider vector with the lower and upper part.
+    SDValue Vec = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT), Lower,
+                                DAG.getConstant(0, MVT::i32), DAG, dl);
+    return Insert128BitVector(Vec, Upper, DAG.getConstant(NumElems/2, MVT::i32),
+                              DAG, dl);
+  }
+
   // Let legalizer expand 2-wide build_vectors.
   if (EVTBits == 64) {
     if (NumNonZero == 1) {
@@ -4639,7 +5241,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
       SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
                                  Op.getOperand(Idx));
       return getShuffleVectorZeroOrUndef(V2, Idx, true,
-                                         Subtarget->hasSSE2(), DAG);
+                                         Subtarget->hasXMMInt(), DAG);
     }
     return SDValue();
   }
@@ -4664,7 +5266,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
     for (unsigned i = 0; i < 4; ++i) {
       bool isZero = !(NonZeros & (1 << i));
       if (isZero)
-        V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl);
+        V[i] = getZeroVector(VT, Subtarget->hasXMMInt(), DAG, dl);
       else
         V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
     }
@@ -4708,7 +5310,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
       return LD;
 
     // For SSE 4.1, use insertps to put the high elements into the low element.
-    if (getSubtarget()->hasSSE41()) {
+    if (getSubtarget()->hasSSE41() || getSubtarget()->hasAVX()) {
       SDValue Result;
       if (Op.getOperand(0).getOpcode() != ISD::UNDEF)
         Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
@@ -4758,13 +5360,12 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
-SDValue
-X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
-  // We support concatenate two MMX registers and place them in a MMX
-  // register.  This is better than doing a stack convert.
+// LowerMMXCONCAT_VECTORS - We support concatenate two MMX registers and place
+// them in a MMX register.  This is better than doing a stack convert.
+static SDValue LowerMMXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
   DebugLoc dl = Op.getDebugLoc();
   EVT ResVT = Op.getValueType();
-  assert(Op.getNumOperands() == 2);
+
   assert(ResVT == MVT::v2i64 || ResVT == MVT::v4i32 ||
          ResVT == MVT::v8i16 || ResVT == MVT::v16i8);
   int Mask[2];
@@ -4785,6 +5386,42 @@ X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp);
 }
 
+// LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction
+// to create 256-bit vectors from two other 128-bit ones.
+static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
+  DebugLoc dl = Op.getDebugLoc();
+  EVT ResVT = Op.getValueType();
+
+  assert(ResVT.getSizeInBits() == 256 && "Value type must be 256-bit wide");
+
+  SDValue V1 = Op.getOperand(0);
+  SDValue V2 = Op.getOperand(1);
+  unsigned NumElems = ResVT.getVectorNumElements();
+
+  SDValue V = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, ResVT), V1,
+                                 DAG.getConstant(0, MVT::i32), DAG, dl);
+  return Insert128BitVector(V, V2, DAG.getConstant(NumElems/2, MVT::i32),
+                            DAG, dl);
+}
+
+SDValue
+X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
+  EVT ResVT = Op.getValueType();
+
+  assert(Op.getNumOperands() == 2);
+  assert((ResVT.getSizeInBits() == 128 || ResVT.getSizeInBits() == 256) &&
+         "Unsupported CONCAT_VECTORS for value type");
+
+  // We support concatenate two MMX registers and place them in a MMX register.
+  // This is better than doing a stack convert.
+  if (ResVT.is128BitVector())
+    return LowerMMXCONCAT_VECTORS(Op, DAG);
+
+  // 256-bit AVX can use the vinsertf128 instruction to create 256-bit vectors
+  // from two other 128-bit ones.
+  return LowerAVXCONCAT_VECTORS(Op, DAG);
+}
+
 // v8i16 shuffles - Prefer shuffles in the following order:
 // 1. [all]   pshuflw, pshufhw, optional move
 // 2. [ssse3] 1 x pshufb
@@ -4844,7 +5481,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op,
   // quads, disable the next transformation since it does not help SSSE3.
   bool V1Used = InputQuads[0] || InputQuads[1];
   bool V2Used = InputQuads[2] || InputQuads[3];
-  if (Subtarget->hasSSSE3()) {
+  if (Subtarget->hasSSSE3() || Subtarget->hasAVX()) {
     if (InputQuads.count() == 2 && V1Used && V2Used) {
       BestLoQuad = InputQuads.find_first();
       BestHiQuad = InputQuads.find_next(BestLoQuad);
@@ -4917,7 +5554,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op,
   // If we have SSSE3, and all words of the result are from 1 input vector,
   // case 2 is generated, otherwise case 3 is generated.  If no SSSE3
   // is present, fall back to case 4.
-  if (Subtarget->hasSSSE3()) {
+  if (Subtarget->hasSSSE3() || Subtarget->hasAVX()) {
     SmallVector<SDValue,16> pshufbMask;
 
     // If we have elements from both input vectors, set the high bit of the
@@ -4985,7 +5622,8 @@ X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op,
     NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
                                 &MaskV[0]);
 
-    if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSSE3())
+    if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE &&
+        (Subtarget->hasSSSE3() || Subtarget->hasAVX()))
       NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16,
                                NewV.getOperand(0),
                                X86::getShufflePSHUFLWImmediate(NewV.getNode()),
@@ -5013,7 +5651,8 @@ X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op,
     NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
                                 &MaskV[0]);
 
-    if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSSE3())
+    if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE &&
+        (Subtarget->hasSSSE3() || Subtarget->hasAVX()))
       NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16,
                               NewV.getOperand(0),
                               X86::getShufflePSHUFHWImmediate(NewV.getNode()),
@@ -5079,7 +5718,7 @@ SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
   }
 
   // If SSSE3, use 1 pshufb instruction per vector with elements in the result.
-  if (TLI.getSubtarget()->hasSSSE3()) {
+  if (TLI.getSubtarget()->hasSSSE3() || TLI.getSubtarget()->hasAVX()) {
     SmallVector<SDValue,16> pshufbMask;
 
     // If all result elements are from one input vector, then only translate
@@ -5276,15 +5915,109 @@ static SDValue getVZextMovL(EVT VT, EVT OpVT,
                                              OpVT, SrcOp)));
 }
 
-/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of
-/// shuffles.
+/// areShuffleHalvesWithinDisjointLanes - Check whether each half of a vector
+/// shuffle node referes to only one lane in the sources.
+static bool areShuffleHalvesWithinDisjointLanes(ShuffleVectorSDNode *SVOp) {
+  EVT VT = SVOp->getValueType(0);
+  int NumElems = VT.getVectorNumElements();
+  int HalfSize = NumElems/2;
+  SmallVector<int, 16> M;
+  SVOp->getMask(M);
+  bool MatchA = false, MatchB = false;
+
+  for (int l = 0; l < NumElems*2; l += HalfSize) {
+    if (isUndefOrInRange(M, 0, HalfSize, l, l+HalfSize)) {
+      MatchA = true;
+      break;
+    }
+  }
+
+  for (int l = 0; l < NumElems*2; l += HalfSize) {
+    if (isUndefOrInRange(M, HalfSize, HalfSize, l, l+HalfSize)) {
+      MatchB = true;
+      break;
+    }
+  }
+
+  return MatchA && MatchB;
+}
+
+/// LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles
+/// which could not be matched by any known target speficic shuffle
 static SDValue
-LowerVECTOR_SHUFFLE_4wide(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
+LowerVECTOR_SHUFFLE_256(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
+  if (areShuffleHalvesWithinDisjointLanes(SVOp)) {
+    // If each half of a vector shuffle node referes to only one lane in the
+    // source vectors, extract each used 128-bit lane and shuffle them using
+    // 128-bit shuffles. Then, concatenate the results. Otherwise leave
+    // the work to the legalizer.
+    DebugLoc dl = SVOp->getDebugLoc();
+    EVT VT = SVOp->getValueType(0);
+    int NumElems = VT.getVectorNumElements();
+    int HalfSize = NumElems/2;
+
+    // Extract the reference for each half
+    int FstVecExtractIdx = 0, SndVecExtractIdx = 0;
+    int FstVecOpNum = 0, SndVecOpNum = 0;
+    for (int i = 0; i < HalfSize; ++i) {
+      int Elt = SVOp->getMaskElt(i);
+      if (SVOp->getMaskElt(i) < 0)
+        continue;
+      FstVecOpNum = Elt/NumElems;
+      FstVecExtractIdx = Elt % NumElems < HalfSize ? 0 : HalfSize;
+      break;
+    }
+    for (int i = HalfSize; i < NumElems; ++i) {
+      int Elt = SVOp->getMaskElt(i);
+      if (SVOp->getMaskElt(i) < 0)
+        continue;
+      SndVecOpNum = Elt/NumElems;
+      SndVecExtractIdx = Elt % NumElems < HalfSize ? 0 : HalfSize;
+      break;
+    }
+
+    // Extract the subvectors
+    SDValue V1 = Extract128BitVector(SVOp->getOperand(FstVecOpNum),
+                      DAG.getConstant(FstVecExtractIdx, MVT::i32), DAG, dl);
+    SDValue V2 = Extract128BitVector(SVOp->getOperand(SndVecOpNum),
+                      DAG.getConstant(SndVecExtractIdx, MVT::i32), DAG, dl);
+
+    // Generate 128-bit shuffles
+    SmallVector<int, 16> MaskV1, MaskV2;
+    for (int i = 0; i < HalfSize; ++i) {
+      int Elt = SVOp->getMaskElt(i);
+      MaskV1.push_back(Elt < 0 ? Elt : Elt % HalfSize);
+    }
+    for (int i = HalfSize; i < NumElems; ++i) {
+      int Elt = SVOp->getMaskElt(i);
+      MaskV2.push_back(Elt < 0 ? Elt : Elt % HalfSize);
+    }
+
+    EVT NVT = V1.getValueType();
+    V1 = DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &MaskV1[0]);
+    V2 = DAG.getVectorShuffle(NVT, dl, V2, DAG.getUNDEF(NVT), &MaskV2[0]);
+
+    // Concatenate the result back
+    SDValue V = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT), V1,
+                                   DAG.getConstant(0, MVT::i32), DAG, dl);
+    return Insert128BitVector(V, V2, DAG.getConstant(NumElems/2, MVT::i32),
+                              DAG, dl);
+  }
+
+  return SDValue();
+}
+
+/// LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with
+/// 4 elements, and match them with several different shuffle types.
+static SDValue
+LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   DebugLoc dl = SVOp->getDebugLoc();
   EVT VT = SVOp->getValueType(0);
 
+  assert(VT.getSizeInBits() == 128 && "Unsupported vector size");
+
   SmallVector<std::pair<int, int>, 8> Locs;
   Locs.resize(4);
   SmallVector<int, 8> Mask1(4U, -1);
@@ -5542,18 +6275,21 @@ SDValue getMOVDDup(SDValue &Op, DebugLoc &dl, SDValue V1, SelectionDAG &DAG) {
 
 static
 SDValue getMOVLowToHigh(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG,
-                        bool HasSSE2) {
+                        bool HasXMMInt) {
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   EVT VT = Op.getValueType();
 
   assert(VT != MVT::v2i64 && "unsupported shuffle type");
 
-  if (HasSSE2 && VT == MVT::v2f64)
+  if (HasXMMInt && VT == MVT::v2f64)
     return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
 
-  // v4f32 or v4i32
-  return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V2, DAG);
+  // v4f32 or v4i32: canonizalized to v4f32 (which is legal for SSE1)
+  return DAG.getNode(ISD::BITCAST, dl, VT,
+                     getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32,
+                           DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1),
+                           DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG));
 }
 
 static
@@ -5572,8 +6308,24 @@ SDValue getMOVHighToLow(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG) {
   return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
 }
 
+static inline unsigned getSHUFPOpcode(EVT VT) {
+  switch(VT.getSimpleVT().SimpleTy) {
+  case MVT::v8i32: // Use fp unit for int unpack.
+  case MVT::v8f32:
+  case MVT::v4i32: // Use fp unit for int unpack.
+  case MVT::v4f32: return X86ISD::SHUFPS;
+  case MVT::v4i64: // Use fp unit for int unpack.
+  case MVT::v4f64:
+  case MVT::v2i64: // Use fp unit for int unpack.
+  case MVT::v2f64: return X86ISD::SHUFPD;
+  default:
+    llvm_unreachable("Unknown type for shufp*");
+  }
+  return 0;
+}
+
 static
-SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
+SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasXMMInt) {
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   EVT VT = Op.getValueType();
@@ -5602,7 +6354,7 @@ SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
     CanFoldLoad = false;
 
   if (CanFoldLoad) {
-    if (HasSSE2 && NumElems == 2)
+    if (HasXMMInt && NumElems == 2)
       return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
 
     if (NumElems == 4)
@@ -5616,28 +6368,30 @@ SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
   // this is horrible, but will stay like this until we move all shuffle
   // matching to x86 specific nodes. Note that for the 1st condition all
   // types are matched with movsd.
-  if ((HasSSE2 && NumElems == 2) || !X86::isMOVLMask(SVOp))
-    return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
-  else if (HasSSE2)
+  if (HasXMMInt) {
+    // FIXME: isMOVLMask should be checked and matched before getMOVLP,
+    // as to remove this logic from here, as much as possible
+    if (NumElems == 2 || !X86::isMOVLMask(SVOp))
+      return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
     return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
-
+  }
 
   assert(VT != MVT::v4i32 && "unsupported shuffle type");
 
   // Invert the operand order and use SHUFPS to match it.
-  return getTargetShuffleNode(X86ISD::SHUFPS, dl, VT, V2, V1,
+  return getTargetShuffleNode(getSHUFPOpcode(VT), dl, VT, V2, V1,
                               X86::getShuffleSHUFImmediate(SVOp), DAG);
 }
 
-static inline unsigned getUNPCKLOpcode(EVT VT, const X86Subtarget *Subtarget) {
+static inline unsigned getUNPCKLOpcode(EVT VT) {
   switch(VT.getSimpleVT().SimpleTy) {
   case MVT::v4i32: return X86ISD::PUNPCKLDQ;
   case MVT::v2i64: return X86ISD::PUNPCKLQDQ;
-  case MVT::v4f32:
-    return Subtarget->hasAVX() ? X86ISD::VUNPCKLPS : X86ISD::UNPCKLPS;
-  case MVT::v2f64:
-    return Subtarget->hasAVX() ? X86ISD::VUNPCKLPD : X86ISD::UNPCKLPD;
+  case MVT::v4f32: return X86ISD::UNPCKLPS;
+  case MVT::v2f64: return X86ISD::UNPCKLPD;
+  case MVT::v8i32: // Use fp unit for int unpack.
   case MVT::v8f32: return X86ISD::VUNPCKLPSY;
+  case MVT::v4i64: // Use fp unit for int unpack.
   case MVT::v4f64: return X86ISD::VUNPCKLPDY;
   case MVT::v16i8: return X86ISD::PUNPCKLBW;
   case MVT::v8i16: return X86ISD::PUNPCKLWD;
@@ -5653,6 +6407,10 @@ static inline unsigned getUNPCKHOpcode(EVT VT) {
   case MVT::v2i64: return X86ISD::PUNPCKHQDQ;
   case MVT::v4f32: return X86ISD::UNPCKHPS;
   case MVT::v2f64: return X86ISD::UNPCKHPD;
+  case MVT::v8i32: // Use fp unit for int unpack.
+  case MVT::v8f32: return X86ISD::VUNPCKHPSY;
+  case MVT::v4i64: // Use fp unit for int unpack.
+  case MVT::v4f64: return X86ISD::VUNPCKHPDY;
   case MVT::v16i8: return X86ISD::PUNPCKHBW;
   case MVT::v8i16: return X86ISD::PUNPCKHWD;
   default:
@@ -5661,6 +6419,68 @@ static inline unsigned getUNPCKHOpcode(EVT VT) {
   return 0;
 }
 
+static inline unsigned getVPERMILOpcode(EVT VT) {
+  switch(VT.getSimpleVT().SimpleTy) {
+  case MVT::v4i32:
+  case MVT::v4f32: return X86ISD::VPERMILPS;
+  case MVT::v2i64:
+  case MVT::v2f64: return X86ISD::VPERMILPD;
+  case MVT::v8i32:
+  case MVT::v8f32: return X86ISD::VPERMILPSY;
+  case MVT::v4i64:
+  case MVT::v4f64: return X86ISD::VPERMILPDY;
+  default:
+    llvm_unreachable("Unknown type for vpermil");
+  }
+  return 0;
+}
+
+/// isVectorBroadcast - Check if the node chain is suitable to be xformed to
+/// a vbroadcast node. The nodes are suitable whenever we can fold a load coming
+/// from a 32 or 64 bit scalar. Update Op to the desired load to be folded.
+static bool isVectorBroadcast(SDValue &Op) {
+  EVT VT = Op.getValueType();
+  bool Is256 = VT.getSizeInBits() == 256;
+
+  assert((VT.getSizeInBits() == 128 || Is256) &&
+         "Unsupported type for vbroadcast node");
+
+  SDValue V = Op;
+  if (V.hasOneUse() && V.getOpcode() == ISD::BITCAST)
+    V = V.getOperand(0);
+
+  if (Is256 && !(V.hasOneUse() &&
+                 V.getOpcode() == ISD::INSERT_SUBVECTOR &&
+                 V.getOperand(0).getOpcode() == ISD::UNDEF))
+    return false;
+
+  if (Is256)
+    V = V.getOperand(1);
+
+  if (!V.hasOneUse())
+    return false;
+
+  // Check the source scalar_to_vector type. 256-bit broadcasts are
+  // supported for 32/64-bit sizes, while 128-bit ones are only supported
+  // for 32-bit scalars.
+  if (V.getOpcode() != ISD::SCALAR_TO_VECTOR)
+    return false;
+
+  unsigned ScalarSize = V.getOperand(0).getValueType().getSizeInBits();
+  if (ScalarSize != 32 && ScalarSize != 64)
+    return false;
+  if (!Is256 && ScalarSize == 64)
+    return false;
+
+  V = V.getOperand(0);
+  if (!MayFoldLoad(V))
+    return false;
+
+  // Return the load node
+  Op = V;
+  return true;
+}
+
 static
 SDValue NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG,
                                const TargetLowering &TLI,
@@ -5672,23 +6492,29 @@ SDValue NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG,
   SDValue V2 = Op.getOperand(1);
 
   if (isZeroShuffle(SVOp))
-    return getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl);
+    return getZeroVector(VT, Subtarget->hasXMMInt(), DAG, dl);
 
   // Handle splat operations
   if (SVOp->isSplat()) {
-    // Special case, this is the only place now where it's
-    // allowed to return a vector_shuffle operation without
-    // using a target specific node, because *hopefully* it
-    // will be optimized away by the dag combiner.
-    if (VT.getVectorNumElements() <= 4 &&
-        CanXFormVExtractWithShuffleIntoLoad(Op, DAG, TLI))
+    unsigned NumElem = VT.getVectorNumElements();
+    int Size = VT.getSizeInBits();
+    // Special case, this is the only place now where it's allowed to return
+    // a vector_shuffle operation without using a target specific node, because
+    // *hopefully* it will be optimized away by the dag combiner. FIXME: should
+    // this be moved to DAGCombine instead?
+    if (NumElem <= 4 && CanXFormVExtractWithShuffleIntoLoad(Op, DAG, TLI))
       return Op;
 
-    // Handle splats by matching through known masks
-    if (VT.getVectorNumElements() <= 4)
+    // Use vbroadcast whenever the splat comes from a foldable load
+    if (Subtarget->hasAVX() && isVectorBroadcast(V1))
+      return DAG.getNode(X86ISD::VBROADCAST, dl, VT, V1);
+
+    // Handle splats by matching through known shuffle masks
+    if ((Size == 128 && NumElem <= 4) ||
+        (Size == 256 && NumElem < 8))
       return SDValue();
 
-    // Canonicalize all of the remaining to v4f32.
+    // All remaning splats are promoted to target supported vector shuffles.
     return PromoteSplat(SVOp, DAG);
   }
 
@@ -5698,7 +6524,8 @@ SDValue NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG,
     SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl);
     if (NewOp.getNode())
       return DAG.getNode(ISD::BITCAST, dl, VT, NewOp);
-  } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) {
+  } else if ((VT == MVT::v4i32 ||
+             (VT == MVT::v4f32 && Subtarget->hasXMMInt()))) {
     // FIXME: Figure out a cleaner way to do this.
     // Try to make use of movq to zero out the top part.
     if (ISD::isBuildVectorAllZeros(V2.getNode())) {
@@ -5731,9 +6558,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
   bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
   bool V1IsSplat = false;
   bool V2IsSplat = false;
-  bool HasSSE2 = Subtarget->hasSSE2() || Subtarget->hasAVX();
-  bool HasSSE3 = Subtarget->hasSSE3() || Subtarget->hasAVX();
-  bool HasSSSE3 = Subtarget->hasSSSE3() || Subtarget->hasAVX();
+  bool HasXMMInt = Subtarget->hasXMMInt();
   MachineFunction &MF = DAG.getMachineFunction();
   bool OptForSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize);
 
@@ -5765,21 +6590,20 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
   // NOTE: isPSHUFDMask can also match both masks below (unpckl_undef and
   // unpckh_undef). Only use pshufd if speed is more important than size.
   if (OptForSize && X86::isUNPCKL_v_undef_Mask(SVOp))
-    if (VT != MVT::v2i64 && VT != MVT::v2f64)
-      return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()), dl, VT, V1, V1, DAG);
+    return getTargetShuffleNode(getUNPCKLOpcode(VT), dl, VT, V1, V1, DAG);
   if (OptForSize && X86::isUNPCKH_v_undef_Mask(SVOp))
-    if (VT != MVT::v2i64 && VT != MVT::v2f64)
-      return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG);
+    return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG);
 
-  if (X86::isMOVDDUPMask(SVOp) && HasSSE3 && V2IsUndef &&
-      RelaxedMayFoldVectorLoad(V1))
+  if (X86::isMOVDDUPMask(SVOp) &&
+      (Subtarget->hasSSE3() || Subtarget->hasAVX()) &&
+      V2IsUndef && RelaxedMayFoldVectorLoad(V1))
     return getMOVDDup(Op, dl, V1, DAG);
 
   if (X86::isMOVHLPS_v_undef_Mask(SVOp))
     return getMOVHighToLow(Op, dl, DAG);
 
   // Use to match splats
-  if (HasSSE2 && X86::isUNPCKHMask(SVOp) && V2IsUndef &&
+  if (HasXMMInt && X86::isUNPCKHMask(SVOp) && V2IsUndef &&
       (VT == MVT::v2f64 || VT == MVT::v2i64))
     return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG);
 
@@ -5792,24 +6616,19 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
 
     unsigned TargetMask = X86::getShuffleSHUFImmediate(SVOp);
 
-    if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32))
+    if (HasXMMInt && (VT == MVT::v4f32 || VT == MVT::v4i32))
       return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG);
 
-    if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64))
-      return getTargetShuffleNode(X86ISD::SHUFPD, dl, VT, V1, V1,
-                                  TargetMask, DAG);
-
-    if (VT == MVT::v4f32)
-      return getTargetShuffleNode(X86ISD::SHUFPS, dl, VT, V1, V1,
-                                  TargetMask, DAG);
+    return getTargetShuffleNode(getSHUFPOpcode(VT), dl, VT, V1, V1,
+                                TargetMask, DAG);
   }
 
   // Check if this can be converted into a logical shift.
   bool isLeft = false;
   unsigned ShAmt = 0;
   SDValue ShVal;
-  bool isShift = getSubtarget()->hasSSE2() &&
-    isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt);
+  bool isShift = getSubtarget()->hasXMMInt() &&
+                 isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt);
   if (isShift && ShVal.hasOneUse()) {
     // If the shifted value has multiple uses, it may be cheaper to use
     // v_set0 + movlhps or movhlps, etc.
@@ -5824,7 +6643,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
     if (ISD::isBuildVectorAllZeros(V1.getNode()))
       return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl);
     if (!X86::isMOVLPMask(SVOp)) {
-      if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64))
+      if (HasXMMInt && (VT == MVT::v2i64 || VT == MVT::v2f64))
         return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
 
       if (VT == MVT::v4i32 || VT == MVT::v4f32)
@@ -5834,19 +6653,19 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
 
   // FIXME: fold these into legal mask.
   if (X86::isMOVLHPSMask(SVOp) && !X86::isUNPCKLMask(SVOp))
-    return getMOVLowToHigh(Op, dl, DAG, HasSSE2);
+    return getMOVLowToHigh(Op, dl, DAG, HasXMMInt);
 
   if (X86::isMOVHLPSMask(SVOp))
     return getMOVHighToLow(Op, dl, DAG);
 
-  if (X86::isMOVSHDUPMask(SVOp) && HasSSE3 && V2IsUndef && NumElems == 4)
+  if (X86::isMOVSHDUPMask(SVOp, Subtarget))
     return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG);
 
-  if (X86::isMOVSLDUPMask(SVOp) && HasSSE3 && V2IsUndef && NumElems == 4)
+  if (X86::isMOVSLDUPMask(SVOp, Subtarget))
     return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG);
 
   if (X86::isMOVLPMask(SVOp))
-    return getMOVLP(Op, dl, DAG, HasSSE2);
+    return getMOVLP(Op, dl, DAG, HasXMMInt);
 
   if (ShouldXformToMOVHLPS(SVOp) ||
       ShouldXformToMOVLP(V1.getNode(), V2.getNode(), SVOp))
@@ -5887,8 +6706,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (X86::isUNPCKLMask(SVOp))
-    return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()),
-                                dl, VT, V1, V2, DAG);
+    return getTargetShuffleNode(getUNPCKLOpcode(VT), dl, VT, V1, V2, DAG);
 
   if (X86::isUNPCKHMask(SVOp))
     return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V2, DAG);
@@ -5915,8 +6733,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
     ShuffleVectorSDNode *NewSVOp = cast<ShuffleVectorSDNode>(NewOp);
 
     if (X86::isUNPCKLMask(NewSVOp))
-      return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()),
-                                  dl, VT, V2, V1, DAG);
+      return getTargetShuffleNode(getUNPCKLOpcode(VT), dl, VT, V2, V1, DAG);
 
     if (X86::isUNPCKHMask(NewSVOp))
       return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V2, V1, DAG);
@@ -5932,18 +6749,15 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
   SmallVector<int, 16> M;
   SVOp->getMask(M);
 
-  if (isPALIGNRMask(M, VT, HasSSSE3))
+  if (isPALIGNRMask(M, VT, Subtarget->hasSSSE3() || Subtarget->hasAVX()))
     return getTargetShuffleNode(X86ISD::PALIGN, dl, VT, V1, V2,
                                 X86::getShufflePALIGNRImmediate(SVOp),
                                 DAG);
 
   if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) &&
       SVOp->getSplatIndex() == 0 && V2IsUndef) {
-    if (VT == MVT::v2f64) {
-      X86ISD::NodeType Opcode =
-        getSubtarget()->hasAVX() ? X86ISD::VUNPCKLPD : X86ISD::UNPCKLPD;
-      return getTargetShuffleNode(Opcode, dl, VT, V1, V1, DAG);
-    }
+    if (VT == MVT::v2f64)
+      return getTargetShuffleNode(X86ISD::UNPCKLPD, dl, VT, V1, V1, DAG);
     if (VT == MVT::v2i64)
       return getTargetShuffleNode(X86ISD::PUNPCKLQDQ, dl, VT, V1, V1, DAG);
   }
@@ -5958,23 +6772,54 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
                                 X86::getShufflePSHUFLWImmediate(SVOp),
                                 DAG);
 
-  if (isSHUFPMask(M, VT)) {
-    unsigned TargetMask = X86::getShuffleSHUFImmediate(SVOp);
-    if (VT == MVT::v4f32 || VT == MVT::v4i32)
-      return getTargetShuffleNode(X86ISD::SHUFPS, dl, VT, V1, V2,
-                                  TargetMask, DAG);
-    if (VT == MVT::v2f64 || VT == MVT::v2i64)
-      return getTargetShuffleNode(X86ISD::SHUFPD, dl, VT, V1, V2,
-                                  TargetMask, DAG);
-  }
+  if (isSHUFPMask(M, VT))
+    return getTargetShuffleNode(getSHUFPOpcode(VT), dl, VT, V1, V2,
+                                X86::getShuffleSHUFImmediate(SVOp), DAG);
 
   if (X86::isUNPCKL_v_undef_Mask(SVOp))
-    if (VT != MVT::v2i64 && VT != MVT::v2f64)
-      return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()),
-                                  dl, VT, V1, V1, DAG);
+    return getTargetShuffleNode(getUNPCKLOpcode(VT), dl, VT, V1, V1, DAG);
   if (X86::isUNPCKH_v_undef_Mask(SVOp))
-    if (VT != MVT::v2i64 && VT != MVT::v2f64)
-      return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG);
+    return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG);
+
+  //===--------------------------------------------------------------------===//
+  // Generate target specific nodes for 128 or 256-bit shuffles only
+  // supported in the AVX instruction set.
+  //
+
+  // Handle VMOVDDUPY permutations
+  if (isMOVDDUPYMask(SVOp, Subtarget))
+    return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG);
+
+  // Handle VPERMILPS* permutations
+  if (isVPERMILPSMask(M, VT, Subtarget))
+    return getTargetShuffleNode(getVPERMILOpcode(VT), dl, VT, V1,
+                                getShuffleVPERMILPSImmediate(SVOp), DAG);
+
+  // Handle VPERMILPD* permutations
+  if (isVPERMILPDMask(M, VT, Subtarget))
+    return getTargetShuffleNode(getVPERMILOpcode(VT), dl, VT, V1,
+                                getShuffleVPERMILPDImmediate(SVOp), DAG);
+
+  // Handle VPERM2F128 permutations
+  if (isVPERM2F128Mask(M, VT, Subtarget))
+    return getTargetShuffleNode(X86ISD::VPERM2F128, dl, VT, V1, V2,
+                                getShuffleVPERM2F128Immediate(SVOp), DAG);
+
+  // Handle VSHUFPSY permutations
+  if (isVSHUFPSYMask(M, VT, Subtarget))
+    return getTargetShuffleNode(getSHUFPOpcode(VT), dl, VT, V1, V2,
+                                getShuffleVSHUFPSYImmediate(SVOp), DAG);
+
+  // Handle VSHUFPDY permutations
+  if (isVSHUFPDYMask(M, VT, Subtarget))
+    return getTargetShuffleNode(getSHUFPOpcode(VT), dl, VT, V1, V2,
+                                getShuffleVSHUFPDYImmediate(SVOp), DAG);
+
+  //===--------------------------------------------------------------------===//
+  // Since no target specific shuffle was selected for this generic one,
+  // lower it into other known shuffles. FIXME: this isn't true yet, but
+  // this is the plan.
+  //
 
   // Handle v8i16 specifically since SSE can do byte extraction and insertion.
   if (VT == MVT::v8i16) {
@@ -5989,9 +6834,14 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
       return NewOp;
   }
 
-  // Handle all 4 wide cases with a number of shuffles.
-  if (NumElems == 4)
-    return LowerVECTOR_SHUFFLE_4wide(SVOp, DAG);
+  // Handle all 128-bit wide vectors with 4 elements, and match them with
+  // several different shuffle types.
+  if (NumElems == 4 && VT.getSizeInBits() == 128)
+    return LowerVECTOR_SHUFFLE_128v4(SVOp, DAG);
+
+  // Handle general 256-bit shuffles
+  if (VT.is256BitVector())
+    return LowerVECTOR_SHUFFLE_256(SVOp, DAG);
 
   return SDValue();
 }
@@ -6001,6 +6851,10 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op,
                                                 SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   DebugLoc dl = Op.getDebugLoc();
+
+  if (Op.getOperand(0).getValueType().getSizeInBits() != 128)
+    return SDValue();
+
   if (VT.getSizeInBits() == 8) {
     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
                                     Op.getOperand(0), Op.getOperand(1));
@@ -6060,36 +6914,26 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
   SDValue Vec = Op.getOperand(0);
   EVT VecVT = Vec.getValueType();
 
-  // If this is a 256-bit vector result, first extract the 128-bit
-  // vector and then extract from the 128-bit vector.
-  if (VecVT.getSizeInBits() > 128) {
+  // If this is a 256-bit vector result, first extract the 128-bit vector and
+  // then extract the element from the 128-bit vector.
+  if (VecVT.getSizeInBits() == 256) {
     DebugLoc dl = Op.getNode()->getDebugLoc();
     unsigned NumElems = VecVT.getVectorNumElements();
     SDValue Idx = Op.getOperand(1);
-
-    if (!isa<ConstantSDNode>(Idx))
-      return SDValue();
-
-    unsigned ExtractNumElems = NumElems / (VecVT.getSizeInBits() / 128);
     unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
 
     // Get the 128-bit vector.
-    bool Upper = IdxVal >= ExtractNumElems;
-    Vec = Extract128BitVector(Vec, Idx, DAG, dl);
+    bool Upper = IdxVal >= NumElems/2;
+    Vec = Extract128BitVector(Vec,
+                    DAG.getConstant(Upper ? NumElems/2 : 0, MVT::i32), DAG, dl);
 
-    // Extract from it.
-    SDValue ScaledIdx = Idx;
-    if (Upper)
-      ScaledIdx = DAG.getNode(ISD::SUB, dl, Idx.getValueType(), Idx,
-                              DAG.getConstant(ExtractNumElems,
-                                              Idx.getValueType()));
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
-                       ScaledIdx);
+                    Upper ? DAG.getConstant(IdxVal-NumElems/2, MVT::i32) : Idx);
   }
 
   assert(Vec.getValueSizeInBits() <= 128 && "Unexpected vector length");
 
-  if (Subtarget->hasSSE41()) {
+  if (Subtarget->hasSSE41() || Subtarget->hasAVX()) {
     SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
     if (Res.getNode())
       return Res;
@@ -6120,7 +6964,7 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
       return Op;
 
     // SHUFPS the element to the lowest double word, then movss.
-    int Mask[4] = { Idx, -1, -1, -1 };
+    int Mask[4] = { static_cast<int>(Idx), -1, -1, -1 };
     EVT VVT = Op.getOperand(0).getValueType();
     SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
                                        DAG.getUNDEF(VVT), Mask);
@@ -6159,6 +7003,9 @@ X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op,
   SDValue N1 = Op.getOperand(1);
   SDValue N2 = Op.getOperand(2);
 
+  if (VT.getSizeInBits() == 256)
+    return SDValue();
+
   if ((EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) &&
       isa<ConstantSDNode>(N2)) {
     unsigned Opc;
@@ -6206,35 +7053,28 @@ X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
   SDValue N1 = Op.getOperand(1);
   SDValue N2 = Op.getOperand(2);
 
-  // If this is a 256-bit vector result, first insert into a 128-bit
-  // vector and then insert into the 256-bit vector.
-  if (VT.getSizeInBits() > 128) {
+  // If this is a 256-bit vector result, first extract the 128-bit vector,
+  // insert the element into the extracted half and then place it back.
+  if (VT.getSizeInBits() == 256) {
     if (!isa<ConstantSDNode>(N2))
       return SDValue();
 
-    // Get the 128-bit vector.
+    // Get the desired 128-bit vector half.
     unsigned NumElems = VT.getVectorNumElements();
     unsigned IdxVal = cast<ConstantSDNode>(N2)->getZExtValue();
-    bool Upper = IdxVal >= NumElems / 2;
+    bool Upper = IdxVal >= NumElems/2;
+    SDValue Ins128Idx = DAG.getConstant(Upper ? NumElems/2 : 0, MVT::i32);
+    SDValue V = Extract128BitVector(N0, Ins128Idx, DAG, dl);
 
-    SDValue SubN0 = Extract128BitVector(N0, N2, DAG, dl);
+    // Insert the element into the desired half.
+    V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V,
+                 N1, Upper ? DAG.getConstant(IdxVal-NumElems/2, MVT::i32) : N2);
 
-    // Insert into it.
-    SDValue ScaledN2 = N2;
-    if (Upper)
-      ScaledN2 = DAG.getNode(ISD::SUB, dl, N2.getValueType(), N2,
-                             DAG.getConstant(NumElems /
-                                             (VT.getSizeInBits() / 128),
-                                             N2.getValueType()));
-    Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubN0.getValueType(), SubN0,
-                     N1, ScaledN2);
-
-    // Insert the 128-bit vector
-    // FIXME: Why UNDEF?
-    return Insert128BitVector(N0, Op, N2, DAG, dl);
+    // Insert the changed part back to the 256-bit vector
+    return Insert128BitVector(N0, V, Ins128Idx, DAG, dl);
   }
 
-  if (Subtarget->hasSSE41())
+  if (Subtarget->hasSSE41() || Subtarget->hasAVX())
     return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG);
 
   if (EltVT == MVT::i8)
@@ -6405,12 +7245,17 @@ X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
   CodeModel::Model M = getTargetMachine().getCodeModel();
 
   if (Subtarget->isPICStyleRIPRel() &&
-      (M == CodeModel::Small || M == CodeModel::Kernel))
+      (M == CodeModel::Small || M == CodeModel::Kernel)) {
+    if (Subtarget->isTargetDarwin() || Subtarget->isTargetELF())
+      OpFlag = X86II::MO_GOTPCREL;
     WrapperKind = X86ISD::WrapperRIP;
-  else if (Subtarget->isPICStyleGOT())
-    OpFlag = X86II::MO_GOTOFF;
-  else if (Subtarget->isPICStyleStubPIC())
-    OpFlag = X86II::MO_PIC_BASE_OFFSET;
+  } else if (Subtarget->isPICStyleGOT()) {
+    OpFlag = X86II::MO_GOT;
+  } else if (Subtarget->isPICStyleStubPIC()) {
+    OpFlag = X86II::MO_DARWIN_NONLAZY_PIC_BASE;
+  } else if (Subtarget->isPICStyleStubNoDynamic()) {
+    OpFlag = X86II::MO_DARWIN_NONLAZY;
+  }
 
   SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag);
 
@@ -6427,6 +7272,12 @@ X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
                          Result);
   }
 
+  // For symbols that require a load from a stub to get the address, emit the
+  // load.
+  if (isGlobalStubReference(OpFlag))
+    Result = DAG.getLoad(getPointerTy(), DL, DAG.getEntryNode(), Result,
+                         MachinePointerInfo::getGOT(), false, false, 0);
+
   return Result;
 }
 
@@ -6676,7 +7527,8 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
     // And our return value (tls address) is in the standard call return value
     // location.
     unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
-    return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy());
+    return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(),
+                              Chain.getValue(1));
   }
 
   assert(false &&
@@ -6922,9 +7774,11 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
 
   // Load the 32-bit value into an XMM register.
   SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
-                             DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                                         Op.getOperand(0),
-                                         DAG.getIntPtrConstant(0)));
+                             Op.getOperand(0));
+
+  // Zero out the upper parts of the register.
+  Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget->hasXMMInt(),
+                                     DAG);
 
   Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
                      DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load),
@@ -7513,6 +8367,9 @@ SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
 }
 
 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
+
+  if (Op.getValueType().isVector()) return LowerVSETCC(Op, DAG);
+
   assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
@@ -7563,6 +8420,39 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
                      DAG.getConstant(X86CC, MVT::i8), EFLAGS);
 }
 
+// Lower256IntVSETCC - Break a VSETCC 256-bit integer VSETCC into two new 128
+// ones, and then concatenate the result back.
+static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
+  EVT VT = Op.getValueType();
+
+  assert(VT.getSizeInBits() == 256 && Op.getOpcode() == ISD::SETCC &&
+         "Unsupported value type for operation");
+
+  int NumElems = VT.getVectorNumElements();
+  DebugLoc dl = Op.getDebugLoc();
+  SDValue CC = Op.getOperand(2);
+  SDValue Idx0 = DAG.getConstant(0, MVT::i32);
+  SDValue Idx1 = DAG.getConstant(NumElems/2, MVT::i32);
+
+  // Extract the LHS vectors
+  SDValue LHS = Op.getOperand(0);
+  SDValue LHS1 = Extract128BitVector(LHS, Idx0, DAG, dl);
+  SDValue LHS2 = Extract128BitVector(LHS, Idx1, DAG, dl);
+
+  // Extract the RHS vectors
+  SDValue RHS = Op.getOperand(1);
+  SDValue RHS1 = Extract128BitVector(RHS, Idx0, DAG, dl);
+  SDValue RHS2 = Extract128BitVector(RHS, Idx1, DAG, dl);
+
+  // Issue the operation on the smaller types and concatenate the result back
+  MVT EltVT = VT.getVectorElementType().getSimpleVT();
+  EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
+  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
+                     DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
+                     DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
+}
+
+
 SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const {
   SDValue Cond;
   SDValue Op0 = Op.getOperand(0);
@@ -7575,11 +8465,21 @@ SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const {
 
   if (isFP) {
     unsigned SSECC = 8;
-    EVT VT0 = Op0.getValueType();
-    assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64);
-    unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD;
+    EVT EltVT = Op0.getValueType().getVectorElementType();
+    assert(EltVT == MVT::f32 || EltVT == MVT::f64);
+
+    unsigned Opc = EltVT == MVT::f32 ? X86ISD::CMPPS : X86ISD::CMPPD;
     bool Swap = false;
 
+    // SSE Condition code mapping:
+    //  0 - EQ
+    //  1 - LT
+    //  2 - LE
+    //  3 - UNORD
+    //  4 - NEQ
+    //  5 - NLT
+    //  6 - NLE
+    //  7 - ORD
     switch (SetCCOpcode) {
     default: break;
     case ISD::SETOEQ:
@@ -7624,6 +8524,10 @@ SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const {
     return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8));
   }
 
+  // Break 256-bit integer vector compare into smaller ones.
+  if (!isFP && VT.getSizeInBits() == 256)
+    return Lower256IntVSETCC(Op, DAG);
+
   // We are handling one of the integer comparisons here.  Since SSE only has
   // GT and EQ comparisons for integer, swapping operands and multiple
   // operations may be required for some comparisons.
@@ -7654,6 +8558,13 @@ SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const {
   if (Swap)
     std::swap(Op0, Op1);
 
+  // Check that the operation in question is available (most are plain SSE2,
+  // but PCMPGTQ and PCMPEQQ have different requirements).
+  if (Opc == X86ISD::PCMPGTQ && !Subtarget->hasSSE42() && !Subtarget->hasAVX())
+    return SDValue();
+  if (Opc == X86ISD::PCMPEQQ && !Subtarget->hasSSE41() && !Subtarget->hasAVX())
+    return SDValue();
+
   // Since SSE has no unsigned integer comparisons, we need to flip  the sign
   // bits of the inputs before performing those operations.
   if (FlipSigns) {
@@ -8014,9 +8925,11 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 SDValue
 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                            SelectionDAG &DAG) const {
-  assert((Subtarget->isTargetCygMing() || Subtarget->isTargetWindows()) &&
-         "This should be used only on Windows targets");
-  assert(!Subtarget->isTargetEnvMacho());
+  assert((Subtarget->isTargetCygMing() || Subtarget->isTargetWindows() ||
+          EnableSegmentedStacks) &&
+         "This should be used only on Windows targets or when segmented stacks "
+         "are being used");
+  assert(!Subtarget->isTargetEnvMacho() && "Not implemented");
   DebugLoc dl = Op.getDebugLoc();
 
   // Get the inputs.
@@ -8024,23 +8937,49 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   SDValue Size  = Op.getOperand(1);
   // FIXME: Ensure alignment here
 
-  SDValue Flag;
+  bool Is64Bit = Subtarget->is64Bit();
+  EVT SPTy = Is64Bit ? MVT::i64 : MVT::i32;
 
-  EVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
-  unsigned Reg = (Subtarget->is64Bit() ? X86::RAX : X86::EAX);
+  if (EnableSegmentedStacks) {
+    MachineFunction &MF = DAG.getMachineFunction();
+    MachineRegisterInfo &MRI = MF.getRegInfo();
 
-  Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag);
-  Flag = Chain.getValue(1);
+    if (Is64Bit) {
+      // The 64 bit implementation of segmented stacks needs to clobber both r10
+      // r11. This makes it impossible to use it along with nested parameters.
+      const Function *F = MF.getFunction();
 
-  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
+      for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
+           I != E; I++)
+        if (I->hasNestAttr())
+          report_fatal_error("Cannot use segmented stacks with functions that "
+                             "have nested arguments.");
+    }
 
-  Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
-  Flag = Chain.getValue(1);
+    const TargetRegisterClass *AddrRegClass =
+      getRegClassFor(Subtarget->is64Bit() ? MVT::i64:MVT::i32);
+    unsigned Vreg = MRI.createVirtualRegister(AddrRegClass);
+    Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
+    SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
+                                DAG.getRegister(Vreg, SPTy));
+    SDValue Ops1[2] = { Value, Chain };
+    return DAG.getMergeValues(Ops1, 2, dl);
+  } else {
+    SDValue Flag;
+    unsigned Reg = (Subtarget->is64Bit() ? X86::RAX : X86::EAX);
 
-  Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1);
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag);
+    Flag = Chain.getValue(1);
+    SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
-  SDValue Ops1[2] = { Chain.getValue(0), Chain };
-  return DAG.getMergeValues(Ops1, 2, dl);
+    Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
+    Flag = Chain.getValue(1);
+
+    Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1);
+
+    SDValue Ops1[2] = { Chain.getValue(0), Chain };
+    return DAG.getMergeValues(Ops1, 2, dl);
+  }
 }
 
 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
@@ -8118,7 +9057,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   DebugLoc dl = Op.getDebugLoc();
 
   EVT ArgVT = Op.getNode()->getValueType(0);
-  const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
   uint32_t ArgSize = getTargetData()->getTypeAllocSize(ArgTy);
   uint8_t ArgMode;
 
@@ -8292,6 +9231,19 @@ X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const
                                 DAG.getConstant(X86CC, MVT::i8), Cond);
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
+  // Arithmetic intrinsics.
+  case Intrinsic::x86_sse3_hadd_ps:
+  case Intrinsic::x86_sse3_hadd_pd:
+  case Intrinsic::x86_avx_hadd_ps_256:
+  case Intrinsic::x86_avx_hadd_pd_256:
+    return DAG.getNode(X86ISD::FHADD, dl, Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2));
+  case Intrinsic::x86_sse3_hsub_ps:
+  case Intrinsic::x86_sse3_hsub_pd:
+  case Intrinsic::x86_avx_hsub_ps_256:
+  case Intrinsic::x86_avx_hsub_pd_256:
+    return DAG.getNode(X86ISD::FHSUB, dl, Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2));
   // ptest and testp intrinsics. The intrinsic these come from are designed to
   // return an integer value, not just an instruction so lower it to the ptest
   // or testp pattern and a setcc for the result.
@@ -8535,8 +9487,13 @@ SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
                      Chain, DAG.getRegister(StoreAddrReg, getPointerTy()));
 }
 
-SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
-                                             SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  return Op.getOperand(0);
+}
+
+SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
+                                                SelectionDAG &DAG) const {
   SDValue Root = Op.getOperand(0);
   SDValue Trmp = Op.getOperand(1); // trampoline
   SDValue FPtr = Op.getOperand(2); // nested function
@@ -8552,8 +9509,8 @@ SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
     const unsigned char JMP64r  = 0xFF; // 64-bit jmp through register opcode.
     const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
 
-    const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10);
-    const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11);
+    const unsigned char N86R10 = X86_MC::getX86RegNum(X86::R10);
+    const unsigned char N86R11 = X86_MC::getX86RegNum(X86::R11);
 
     const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
 
@@ -8600,9 +9557,7 @@ SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
                                 MachinePointerInfo(TrmpAddr, 22),
                                 false, false, 0);
 
-    SDValue Ops[] =
-      { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6) };
-    return DAG.getMergeValues(Ops, 2, dl);
+    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6);
   } else {
     const Function *Func =
       cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
@@ -8619,7 +9574,7 @@ SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
       NestReg = X86::ECX;
 
       // Check that ECX wasn't needed by an 'inreg' parameter.
-      const FunctionType *FTy = Func->getFunctionType();
+      FunctionType *FTy = Func->getFunctionType();
       const AttrListPtr &Attrs = Func->getAttributes();
 
       if (!Attrs.isEmpty() && !Func->isVarArg()) {
@@ -8657,7 +9612,7 @@ SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
 
     // This is storing the opcode for MOV32ri.
     const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
-    const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg);
+    const unsigned char N86Reg = X86_MC::getX86RegNum(NestReg);
     OutChains[0] = DAG.getStore(Root, dl,
                                 DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
                                 Trmp, MachinePointerInfo(TrmpAddr),
@@ -8682,9 +9637,7 @@ SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
                                 MachinePointerInfo(TrmpAddr, 6),
                                 false, false, 1);
 
-    SDValue Ops[] =
-      { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4) };
-    return DAG.getMergeValues(Ops, 2, dl);
+    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4);
   }
 }
 
@@ -8822,8 +9775,58 @@ SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
   return Op;
 }
 
-SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) const {
+// Lower256IntArith - Break a 256-bit integer operation into two new 128-bit
+// ones, and then concatenate the result back.
+static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
+
+  assert(VT.getSizeInBits() == 256 && VT.isInteger() &&
+         "Unsupported value type for operation");
+
+  int NumElems = VT.getVectorNumElements();
+  DebugLoc dl = Op.getDebugLoc();
+  SDValue Idx0 = DAG.getConstant(0, MVT::i32);
+  SDValue Idx1 = DAG.getConstant(NumElems/2, MVT::i32);
+
+  // Extract the LHS vectors
+  SDValue LHS = Op.getOperand(0);
+  SDValue LHS1 = Extract128BitVector(LHS, Idx0, DAG, dl);
+  SDValue LHS2 = Extract128BitVector(LHS, Idx1, DAG, dl);
+
+  // Extract the RHS vectors
+  SDValue RHS = Op.getOperand(1);
+  SDValue RHS1 = Extract128BitVector(RHS, Idx0, DAG, dl);
+  SDValue RHS2 = Extract128BitVector(RHS, Idx1, DAG, dl);
+
+  MVT EltVT = VT.getVectorElementType().getSimpleVT();
+  EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
+
+  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
+                     DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
+                     DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
+}
+
+SDValue X86TargetLowering::LowerADD(SDValue Op, SelectionDAG &DAG) const {
+  assert(Op.getValueType().getSizeInBits() == 256 &&
+         Op.getValueType().isInteger() &&
+         "Only handle AVX 256-bit vector integer operation");
+  return Lower256IntArith(Op, DAG);
+}
+
+SDValue X86TargetLowering::LowerSUB(SDValue Op, SelectionDAG &DAG) const {
+  assert(Op.getValueType().getSizeInBits() == 256 &&
+         Op.getValueType().isInteger() &&
+         "Only handle AVX 256-bit vector integer operation");
+  return Lower256IntArith(Op, DAG);
+}
+
+SDValue X86TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+
+  // Decompose 256-bit ops into smaller 128-bit ops.
+  if (VT.getSizeInBits() == 256)
+    return Lower256IntArith(Op, DAG);
+
   assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply");
   DebugLoc dl = Op.getDebugLoc();
 
@@ -8872,11 +9875,51 @@ SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
   DebugLoc dl = Op.getDebugLoc();
   SDValue R = Op.getOperand(0);
   SDValue Amt = Op.getOperand(1);
-
   LLVMContext *Context = DAG.getContext();
 
-  // Must have SSE2.
-  if (!Subtarget->hasSSE2()) return SDValue();
+  if (!Subtarget->hasXMMInt())
+    return SDValue();
+
+  // Decompose 256-bit shifts into smaller 128-bit shifts.
+  if (VT.getSizeInBits() == 256) {
+    int NumElems = VT.getVectorNumElements();
+    MVT EltVT = VT.getVectorElementType().getSimpleVT();
+    EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
+
+    // Extract the two vectors
+    SDValue V1 = Extract128BitVector(R, DAG.getConstant(0, MVT::i32), DAG, dl);
+    SDValue V2 = Extract128BitVector(R, DAG.getConstant(NumElems/2, MVT::i32),
+                                     DAG, dl);
+
+    // Recreate the shift amount vectors
+    SDValue Amt1, Amt2;
+    if (Amt.getOpcode() == ISD::BUILD_VECTOR) {
+      // Constant shift amount
+      SmallVector<SDValue, 4> Amt1Csts;
+      SmallVector<SDValue, 4> Amt2Csts;
+      for (int i = 0; i < NumElems/2; ++i)
+        Amt1Csts.push_back(Amt->getOperand(i));
+      for (int i = NumElems/2; i < NumElems; ++i)
+        Amt2Csts.push_back(Amt->getOperand(i));
+
+      Amt1 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT,
+                                 &Amt1Csts[0], NumElems/2);
+      Amt2 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT,
+                                 &Amt2Csts[0], NumElems/2);
+    } else {
+      // Variable shift amount
+      Amt1 = Extract128BitVector(Amt, DAG.getConstant(0, MVT::i32), DAG, dl);
+      Amt2 = Extract128BitVector(Amt, DAG.getConstant(NumElems/2, MVT::i32),
+                                 DAG, dl);
+    }
+
+    // Issue new vector shifts for the smaller types
+    V1 = DAG.getNode(Op.getOpcode(), dl, NewVT, V1, Amt1);
+    V2 = DAG.getNode(Op.getOpcode(), dl, NewVT, V2, Amt2);
+
+    // Concatenate the result back
+    return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2);
+  }
 
   // Optimize shl/srl/sra with constant shift amount.
   if (isSplatVector(Amt.getNode())) {
@@ -8927,9 +9970,6 @@ SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
   }
 
   // Lower SHL with variable shift amount.
-  // Cannot lower SHL without SSE2 or later.
-  if (!Subtarget->hasSSE2()) return SDValue();
-
   if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
     Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                      DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32),
@@ -8971,7 +10011,7 @@ SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
     M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M,
                     DAG.getConstant(4, MVT::i32));
-    R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op);
+    R = DAG.getNode(ISD::VSELECT, dl, VT, Op, R, M);
     // a += a
     Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
 
@@ -8986,13 +10026,13 @@ SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
     M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M,
                     DAG.getConstant(2, MVT::i32));
-    R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op);
+    R = DAG.getNode(ISD::VSELECT, dl, VT, Op, R, M);
     // a += a
     Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
 
     // return pblendv(r, r+r, a);
-    R = DAG.getNode(X86ISD::PBLENDVB, dl, VT,
-                    R, DAG.getNode(ISD::ADD, dl, VT, R, R), Op);
+    R = DAG.getNode(ISD::VSELECT, dl, VT, Op,
+                    R, DAG.getNode(ISD::ADD, dl, VT, R, R));
     return R;
   }
   return SDValue();
@@ -9057,8 +10097,7 @@ SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
                   DAG.getConstant(X86::COND_O, MVT::i32),
                   SDValue(Sum.getNode(), 2));
 
-    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC);
-    return Sum;
+    return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
   }
   }
 
@@ -9071,8 +10110,7 @@ SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
                 DAG.getConstant(Cond, MVT::i32),
                 SDValue(Sum.getNode(), 1));
 
-  DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC);
-  return Sum;
+  return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
 }
 
 SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const{
@@ -9080,8 +10118,7 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG)
   SDNode* Node = Op.getNode();
   EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
   EVT VT = Node->getValueType(0);
-
-  if (Subtarget->hasSSE2() && VT.isVector()) {
+  if (Subtarget->hasXMMInt() && VT.isVector()) {
     unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
                         ExtraVT.getScalarType().getSizeInBits();
     SDValue ShAmt = DAG.getConstant(BitsDiff, MVT::i32);
@@ -9091,11 +10128,6 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG)
     switch (VT.getSimpleVT().SimpleTy) {
       default:
         return SDValue();
-      case MVT::v2i64: {
-        SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_q;
-        SRAIntrinsicsID = 0;
-        break;
-      }
       case MVT::v4i32: {
         SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_d;
         SRAIntrinsicsID = Intrinsic::x86_sse2_psrai_d;
@@ -9115,12 +10147,9 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG)
     // In case of 1 bit sext, no need to shr
     if (ExtraVT.getScalarType().getSizeInBits() == 1) return Tmp1;
 
-    if (SRAIntrinsicsID) {
-      Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
-                         DAG.getConstant(SRAIntrinsicsID, MVT::i32),
-                         Tmp1, ShAmt);
-    }
-    return Tmp1;
+    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
+                       DAG.getConstant(SRAIntrinsicsID, MVT::i32),
+                       Tmp1, ShAmt);
   }
 
   return SDValue();
@@ -9132,7 +10161,7 @@ SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{
 
   // Go ahead and emit the fence on x86-64 even if we asked for no-sse2.
   // There isn't any reason to disable it if the target processor supports it.
-  if (!Subtarget->hasSSE2() && !Subtarget->is64Bit()) {
+  if (!Subtarget->hasXMMInt() && !Subtarget->is64Bit()) {
     SDValue Chain = Op.getOperand(0);
     SDValue Zero = DAG.getConstant(0, MVT::i32);
     SDValue Ops[] = {
@@ -9172,6 +10201,45 @@ SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{
   return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 }
 
+SDValue X86TargetLowering::LowerATOMIC_FENCE(SDValue Op,
+                                             SelectionDAG &DAG) const {
+  DebugLoc dl = Op.getDebugLoc();
+  AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
+    cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
+  SynchronizationScope FenceScope = static_cast<SynchronizationScope>(
+    cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
+
+  // The only fence that needs an instruction is a sequentially-consistent
+  // cross-thread fence.
+  if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
+    // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
+    // no-sse2). There isn't any reason to disable it if the target processor
+    // supports it.
+    if (Subtarget->hasXMMInt() || Subtarget->is64Bit())
+      return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
+
+    SDValue Chain = Op.getOperand(0);
+    SDValue Zero = DAG.getConstant(0, MVT::i32);
+    SDValue Ops[] = {
+      DAG.getRegister(X86::ESP, MVT::i32), // Base
+      DAG.getTargetConstant(1, MVT::i8),   // Scale
+      DAG.getRegister(0, MVT::i32),        // Index
+      DAG.getTargetConstant(0, MVT::i32),  // Disp
+      DAG.getRegister(0, MVT::i32),        // Segment.
+      Zero,
+      Chain
+    };
+    SDNode *Res =
+      DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops,
+                         array_lengthof(Ops));
+    return SDValue(Res, 0);
+  }
+
+  // MEMBARRIER is a compiler barrier; it codegens to a no-op.
+  return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
+}
+
+
 SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
   EVT T = Op.getValueType();
   DebugLoc DL = Op.getDebugLoc();
@@ -9227,7 +10295,7 @@ SDValue X86TargetLowering::LowerBITCAST(SDValue Op,
                                             SelectionDAG &DAG) const {
   EVT SrcVT = Op.getOperand(0).getValueType();
   EVT DstVT = Op.getValueType();
-  assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
+  assert(Subtarget->is64Bit() && !Subtarget->hasXMMInt() &&
          Subtarget->hasMMX() && "Unexpected custom BITCAST");
   assert((DstVT == MVT::i64 ||
           (DstVT.isVector() && DstVT.getSizeInBits()==64)) &&
@@ -9255,7 +10323,34 @@ SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const {
                        Node->getOperand(0),
                        Node->getOperand(1), negOp,
                        cast<AtomicSDNode>(Node)->getSrcValue(),
-                       cast<AtomicSDNode>(Node)->getAlignment());
+                       cast<AtomicSDNode>(Node)->getAlignment(),
+                       cast<AtomicSDNode>(Node)->getOrdering(),
+                       cast<AtomicSDNode>(Node)->getSynchScope());
+}
+
+static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
+  SDNode *Node = Op.getNode();
+  DebugLoc dl = Node->getDebugLoc();
+  EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
+
+  // Convert seq_cst store -> xchg
+  // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
+  // FIXME: On 32-bit, store -> fist or movq would be more efficient
+  //        (The only way to get a 16-byte store is cmpxchg16b)
+  // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
+  if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent ||
+      !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
+    SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
+                                 cast<AtomicSDNode>(Node)->getMemoryVT(),
+                                 Node->getOperand(0),
+                                 Node->getOperand(1), Node->getOperand(2),
+                                 cast<AtomicSDNode>(Node)->getMemOperand(),
+                                 cast<AtomicSDNode>(Node)->getOrdering(),
+                                 cast<AtomicSDNode>(Node)->getSynchScope());
+    return Swap.getValue(1);
+  }
+  // Other atomic stores have a simple pattern.
+  return Op;
 }
 
 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
@@ -9291,8 +10386,10 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   default: llvm_unreachable("Should not custom lower this!");
   case ISD::SIGN_EXTEND_INREG:  return LowerSIGN_EXTEND_INREG(Op,DAG);
   case ISD::MEMBARRIER:         return LowerMEMBARRIER(Op,DAG);
+  case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op,DAG);
   case ISD::ATOMIC_CMP_SWAP:    return LowerCMP_SWAP(Op,DAG);
   case ISD::ATOMIC_LOAD_SUB:    return LowerLOAD_SUB(Op,DAG);
+  case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op,DAG);
   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
   case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, DAG);
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
@@ -9318,7 +10415,6 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
   case ISD::FGETSIGN:           return LowerFGETSIGN(Op, DAG);
   case ISD::SETCC:              return LowerSETCC(Op, DAG);
-  case ISD::VSETCC:             return LowerVSETCC(Op, DAG);
   case ISD::SELECT:             return LowerSELECT(Op, DAG);
   case ISD::BRCOND:             return LowerBRCOND(Op, DAG);
   case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
@@ -9332,11 +10428,12 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
                                 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
-  case ISD::TRAMPOLINE:         return LowerTRAMPOLINE(Op, DAG);
+  case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
+  case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
   case ISD::FLT_ROUNDS_:        return LowerFLT_ROUNDS_(Op, DAG);
   case ISD::CTLZ:               return LowerCTLZ(Op, DAG);
   case ISD::CTTZ:               return LowerCTTZ(Op, DAG);
-  case ISD::MUL:                return LowerMUL_V2I64(Op, DAG);
+  case ISD::MUL:                return LowerMUL(Op, DAG);
   case ISD::SRA:
   case ISD::SRL:
   case ISD::SHL:                return LowerShift(Op, DAG);
@@ -9352,15 +10449,38 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ADDE:
   case ISD::SUBC:
   case ISD::SUBE:               return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
+  case ISD::ADD:                return LowerADD(Op, DAG);
+  case ISD::SUB:                return LowerSUB(Op, DAG);
   }
 }
 
+static void ReplaceATOMIC_LOAD(SDNode *Node,
+                                  SmallVectorImpl<SDValue> &Results,
+                                  SelectionDAG &DAG) {
+  DebugLoc dl = Node->getDebugLoc();
+  EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
+
+  // Convert wide load -> cmpxchg8b/cmpxchg16b
+  // FIXME: On 32-bit, load -> fild or movq would be more efficient
+  //        (The only way to get a 16-byte load is cmpxchg16b)
+  // FIXME: 16-byte ATOMIC_CMP_SWAP isn't actually hooked up at the moment.
+  SDValue Zero = DAG.getConstant(0, VT);
+  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT,
+                               Node->getOperand(0),
+                               Node->getOperand(1), Zero, Zero,
+                               cast<AtomicSDNode>(Node)->getMemOperand(),
+                               cast<AtomicSDNode>(Node)->getOrdering(),
+                               cast<AtomicSDNode>(Node)->getSynchScope());
+  Results.push_back(Swap.getValue(0));
+  Results.push_back(Swap.getValue(1));
+}
+
 void X86TargetLowering::
 ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results,
                         SelectionDAG &DAG, unsigned NewOp) const {
-  EVT T = Node->getValueType(0);
   DebugLoc dl = Node->getDebugLoc();
-  assert (T == MVT::i64 && "Only know how to expand i64 atomics");
+  assert (Node->getValueType(0) == MVT::i64 &&
+          "Only know how to expand i64 atomics");
 
   SDValue Chain = Node->getOperand(0);
   SDValue In1 = Node->getOperand(1);
@@ -9423,37 +10543,48 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   }
   case ISD::ATOMIC_CMP_SWAP: {
     EVT T = N->getValueType(0);
-    assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
+    assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");
+    bool Regs64bit = T == MVT::i128;
+    EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
     SDValue cpInL, cpInH;
-    cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
-                        DAG.getConstant(0, MVT::i32));
-    cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
-                        DAG.getConstant(1, MVT::i32));
-    cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue());
-    cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH,
-                             cpInL.getValue(1));
+    cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
+                        DAG.getConstant(0, HalfT));
+    cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
+                        DAG.getConstant(1, HalfT));
+    cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
+                             Regs64bit ? X86::RAX : X86::EAX,
+                             cpInL, SDValue());
+    cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl,
+                             Regs64bit ? X86::RDX : X86::EDX,
+                             cpInH, cpInL.getValue(1));
     SDValue swapInL, swapInH;
-    swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
-                          DAG.getConstant(0, MVT::i32));
-    swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
-                          DAG.getConstant(1, MVT::i32));
-    swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL,
-                               cpInH.getValue(1));
-    swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH,
-                               swapInL.getValue(1));
+    swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
+                          DAG.getConstant(0, HalfT));
+    swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
+                          DAG.getConstant(1, HalfT));
+    swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl,
+                               Regs64bit ? X86::RBX : X86::EBX,
+                               swapInL, cpInH.getValue(1));
+    swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl,
+                               Regs64bit ? X86::RCX : X86::ECX, 
+                               swapInH, swapInL.getValue(1));
     SDValue Ops[] = { swapInH.getValue(0),
                       N->getOperand(1),
                       swapInH.getValue(1) };
     SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
-    SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys,
+    unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG :
+                                  X86ISD::LCMPXCHG8_DAG;
+    SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys,
                                              Ops, 3, T, MMO);
-    SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX,
-                                        MVT::i32, Result.getValue(1));
-    SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX,
-                                        MVT::i32, cpOutL.getValue(2));
+    SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
+                                        Regs64bit ? X86::RAX : X86::EAX,
+                                        HalfT, Result.getValue(1));
+    SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
+                                        Regs64bit ? X86::RDX : X86::EDX,
+                                        HalfT, cpOutL.getValue(2));
     SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
-    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
+    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF, 2));
     Results.push_back(cpOutH.getValue(1));
     return;
   }
@@ -9478,6 +10609,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::ATOMIC_SWAP:
     ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG);
     return;
+  case ISD::ATOMIC_LOAD:
+    ReplaceATOMIC_LOAD(N, Results, DAG);
   }
 }
 
@@ -9527,11 +10660,12 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::PSIGNB:             return "X86ISD::PSIGNB";
   case X86ISD::PSIGNW:             return "X86ISD::PSIGNW";
   case X86ISD::PSIGND:             return "X86ISD::PSIGND";
-  case X86ISD::PBLENDVB:           return "X86ISD::PBLENDVB";
   case X86ISD::FMAX:               return "X86ISD::FMAX";
   case X86ISD::FMIN:               return "X86ISD::FMIN";
   case X86ISD::FRSQRT:             return "X86ISD::FRSQRT";
   case X86ISD::FRCP:               return "X86ISD::FRCP";
+  case X86ISD::FHADD:              return "X86ISD::FHADD";
+  case X86ISD::FHSUB:              return "X86ISD::FHSUB";
   case X86ISD::TLSADDR:            return "X86ISD::TLSADDR";
   case X86ISD::TLSCALL:            return "X86ISD::TLSCALL";
   case X86ISD::EH_RETURN:          return "X86ISD::EH_RETURN";
@@ -9570,6 +10704,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::OR:                 return "X86ISD::OR";
   case X86ISD::XOR:                return "X86ISD::XOR";
   case X86ISD::AND:                return "X86ISD::AND";
+  case X86ISD::ANDN:               return "X86ISD::ANDN";
   case X86ISD::MUL_IMM:            return "X86ISD::MUL_IMM";
   case X86ISD::PTEST:              return "X86ISD::PTEST";
   case X86ISD::TESTP:              return "X86ISD::TESTP";
@@ -9596,9 +10731,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::MOVSS:              return "X86ISD::MOVSS";
   case X86ISD::UNPCKLPS:           return "X86ISD::UNPCKLPS";
   case X86ISD::UNPCKLPD:           return "X86ISD::UNPCKLPD";
-  case X86ISD::VUNPCKLPS:          return "X86ISD::VUNPCKLPS";
-  case X86ISD::VUNPCKLPD:          return "X86ISD::VUNPCKLPD";
-  case X86ISD::VUNPCKLPSY:         return "X86ISD::VUNPCKLPSY";
   case X86ISD::VUNPCKLPDY:         return "X86ISD::VUNPCKLPDY";
   case X86ISD::UNPCKHPS:           return "X86ISD::UNPCKHPS";
   case X86ISD::UNPCKHPD:           return "X86ISD::UNPCKHPD";
@@ -9610,16 +10742,24 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::PUNPCKHWD:          return "X86ISD::PUNPCKHWD";
   case X86ISD::PUNPCKHDQ:          return "X86ISD::PUNPCKHDQ";
   case X86ISD::PUNPCKHQDQ:         return "X86ISD::PUNPCKHQDQ";
+  case X86ISD::VBROADCAST:         return "X86ISD::VBROADCAST";
+  case X86ISD::VPERMILPS:          return "X86ISD::VPERMILPS";
+  case X86ISD::VPERMILPSY:         return "X86ISD::VPERMILPSY";
+  case X86ISD::VPERMILPD:          return "X86ISD::VPERMILPD";
+  case X86ISD::VPERMILPDY:         return "X86ISD::VPERMILPDY";
+  case X86ISD::VPERM2F128:         return "X86ISD::VPERM2F128";
   case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
   case X86ISD::VAARG_64:           return "X86ISD::VAARG_64";
   case X86ISD::WIN_ALLOCA:         return "X86ISD::WIN_ALLOCA";
+  case X86ISD::MEMBARRIER:         return "X86ISD::MEMBARRIER";
+  case X86ISD::SEG_ALLOCA:         return "X86ISD::SEG_ALLOCA";
   }
 }
 
 // isLegalAddressingMode - Return true if the addressing mode represented
 // by AM is legal for this target, for a load/store of the specified type.
 bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              const Type *Ty) const {
+                                              Type *Ty) const {
   // X86 supports extremely general addressing modes.
   CodeModel::Model M = getTargetMachine().getCodeModel();
   Reloc::Model R = getTargetMachine().getRelocationModel();
@@ -9671,7 +10811,7 @@ bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
 }
 
 
-bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const {
+bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
     return false;
   unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
@@ -9691,7 +10831,7 @@ bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
   return true;
 }
 
-bool X86TargetLowering::isZExtFree(const Type *Ty1, const Type *Ty2) const {
+bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
   // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
   return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit();
 }
@@ -9715,7 +10855,7 @@ X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
                                       EVT VT) const {
   // Very little shuffling can be done for 64-bit vectors right now.
   if (VT.getSizeInBits() == 64)
-    return isPALIGNRMask(M, VT, Subtarget->hasSSSE3());
+    return isPALIGNRMask(M, VT, Subtarget->hasSSSE3() || Subtarget->hasAVX());
 
   // FIXME: pshufb, blends, shifts.
   return (VT.getVectorNumElements() == 2 ||
@@ -9725,7 +10865,7 @@ X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
           isPSHUFDMask(M, VT) ||
           isPSHUFHWMask(M, VT) ||
           isPSHUFLWMask(M, VT) ||
-          isPALIGNRMask(M, VT, Subtarget->hasSSSE3()) ||
+          isPALIGNRMask(M, VT, Subtarget->hasSSSE3() || Subtarget->hasAVX()) ||
           isUNPCKLMask(M, VT) ||
           isUNPCKHMask(M, VT) ||
           isUNPCKL_v_undef_Mask(M, VT) ||
@@ -10158,7 +11298,9 @@ X86TargetLowering::EmitPCMP(MachineInstr *MI, MachineBasicBlock *BB,
     if (!(Op.isReg() && Op.isImplicit()))
       MIB.addOperand(Op);
   }
-  BuildMI(*BB, MI, dl, TII->get(X86::MOVAPSrr), MI->getOperand(0).getReg())
+  BuildMI(*BB, MI, dl,
+    TII->get(Subtarget->hasAVX() ? X86::VMOVAPSrr : X86::MOVAPSrr),
+             MI->getOperand(0).getReg())
     .addReg(X86::XMM0);
 
   MI->eraseFromParent();
@@ -10513,6 +11655,7 @@ X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
     MBB->addSuccessor(EndMBB);
   }
 
+  unsigned MOVOpc = Subtarget->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
   // In the XMM save block, save all the XMM argument registers.
   for (int i = 3, e = MI->getNumOperands(); i != e; ++i) {
     int64_t Offset = (i - 3) * 16 + VarArgsFPOffset;
@@ -10521,7 +11664,7 @@ X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
           MachinePointerInfo::getFixedStack(RegSaveFrameIndex, Offset),
         MachineMemOperand::MOStore,
         /*Size=*/16, /*Align=*/16);
-    BuildMI(XMMSaveMBB, DL, TII->get(X86::MOVAPSmr))
+    BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc))
       .addFrameIndex(RegSaveFrameIndex)
       .addImm(/*Scale=*/1)
       .addReg(/*IndexReg=*/0)
@@ -10565,17 +11708,9 @@ X86TargetLowering::EmitLoweredSelect(MachineInstr *MI,
 
   // If the EFLAGS register isn't dead in the terminator, then claim that it's
   // live into the sink and copy blocks.
-  const MachineFunction *MF = BB->getParent();
-  const TargetRegisterInfo *TRI = MF->getTarget().getRegisterInfo();
-  BitVector ReservedRegs = TRI->getReservedRegs(*MF);
-
-  for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) {
-    const MachineOperand &MO = MI->getOperand(I);
-    if (!MO.isReg() || !MO.isUse() || MO.isKill()) continue;
-    unsigned Reg = MO.getReg();
-    if (Reg != X86::EFLAGS) continue;
-    copy0MBB->addLiveIn(Reg);
-    sinkMBB->addLiveIn(Reg);
+  if (!MI->killsRegister(X86::EFLAGS)) {
+    copy0MBB->addLiveIn(X86::EFLAGS);
+    sinkMBB->addLiveIn(X86::EFLAGS);
   }
 
   // Transfer the remainder of BB and its successor edges to sinkMBB.
@@ -10610,6 +11745,119 @@ X86TargetLowering::EmitLoweredSelect(MachineInstr *MI,
   return sinkMBB;
 }
 
+MachineBasicBlock *
+X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI, MachineBasicBlock *BB,
+                                        bool Is64Bit) const {
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+  DebugLoc DL = MI->getDebugLoc();
+  MachineFunction *MF = BB->getParent();
+  const BasicBlock *LLVM_BB = BB->getBasicBlock();
+
+  assert(EnableSegmentedStacks);
+
+  unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
+  unsigned TlsOffset = Is64Bit ? 0x70 : 0x30;
+
+  // BB:
+  //  ... [Till the alloca]
+  // If stacklet is not large enough, jump to mallocMBB
+  //
+  // bumpMBB:
+  //  Allocate by subtracting from RSP
+  //  Jump to continueMBB
+  //
+  // mallocMBB:
+  //  Allocate by call to runtime
+  //
+  // continueMBB:
+  //  ...
+  //  [rest of original BB]
+  //
+
+  MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
+
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  const TargetRegisterClass *AddrRegClass =
+    getRegClassFor(Is64Bit ? MVT::i64:MVT::i32);
+
+  unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
+    bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
+    tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
+    sizeVReg = MI->getOperand(1).getReg(),
+    physSPReg = Is64Bit ? X86::RSP : X86::ESP;
+
+  MachineFunction::iterator MBBIter = BB;
+  ++MBBIter;
+
+  MF->insert(MBBIter, bumpMBB);
+  MF->insert(MBBIter, mallocMBB);
+  MF->insert(MBBIter, continueMBB);
+
+  continueMBB->splice(continueMBB->begin(), BB, llvm::next
+                      (MachineBasicBlock::iterator(MI)), BB->end());
+  continueMBB->transferSuccessorsAndUpdatePHIs(BB);
+
+  // Add code to the main basic block to check if the stack limit has been hit,
+  // and if so, jump to mallocMBB otherwise to bumpMBB.
+  BuildMI(BB, DL, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
+  BuildMI(BB, DL, TII->get(Is64Bit ? X86::SUB64rr:X86::SUB32rr), tmpSPVReg)
+    .addReg(tmpSPVReg).addReg(sizeVReg);
+  BuildMI(BB, DL, TII->get(Is64Bit ? X86::CMP64mr:X86::CMP32mr))
+    .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg)
+    .addReg(tmpSPVReg);
+  BuildMI(BB, DL, TII->get(X86::JG_4)).addMBB(mallocMBB);
+
+  // bumpMBB simply decreases the stack pointer, since we know the current
+  // stacklet has enough space.
+  BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), physSPReg)
+    .addReg(tmpSPVReg);
+  BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
+    .addReg(tmpSPVReg);
+  BuildMI(bumpMBB, DL, TII->get(X86::JMP_4)).addMBB(continueMBB);
+
+  // Calls into a routine in libgcc to allocate more space from the heap.
+  if (Is64Bit) {
+    BuildMI(mallocMBB, DL, TII->get(X86::MOV64rr), X86::RDI)
+      .addReg(sizeVReg);
+    BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
+    .addExternalSymbol("__morestack_allocate_stack_space").addReg(X86::RDI);
+  } else {
+    BuildMI(mallocMBB, DL, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
+      .addImm(12);
+    BuildMI(mallocMBB, DL, TII->get(X86::PUSH32r)).addReg(sizeVReg);
+    BuildMI(mallocMBB, DL, TII->get(X86::CALLpcrel32))
+      .addExternalSymbol("__morestack_allocate_stack_space");
+  }
+
+  if (!Is64Bit)
+    BuildMI(mallocMBB, DL, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
+      .addImm(16);
+
+  BuildMI(mallocMBB, DL, TII->get(TargetOpcode::COPY), mallocPtrVReg)
+    .addReg(Is64Bit ? X86::RAX : X86::EAX);
+  BuildMI(mallocMBB, DL, TII->get(X86::JMP_4)).addMBB(continueMBB);
+
+  // Set up the CFG correctly.
+  BB->addSuccessor(bumpMBB);
+  BB->addSuccessor(mallocMBB);
+  mallocMBB->addSuccessor(continueMBB);
+  bumpMBB->addSuccessor(continueMBB);
+
+  // Take care of the PHI nodes.
+  BuildMI(*continueMBB, continueMBB->begin(), DL, TII->get(X86::PHI),
+          MI->getOperand(0).getReg())
+    .addReg(mallocPtrVReg).addMBB(mallocMBB)
+    .addReg(bumpSPPtrVReg).addMBB(bumpMBB);
+
+  // Delete the original pseudo instruction.
+  MI->eraseFromParent();
+
+  // And we're done.
+  return continueMBB;
+}
+
 MachineBasicBlock *
 X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI,
                                           MachineBasicBlock *BB) const {
@@ -10718,11 +11966,11 @@ MachineBasicBlock *
 X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
                                                MachineBasicBlock *BB) const {
   switch (MI->getOpcode()) {
-  default: assert(false && "Unexpected instr type to insert");
+  default: assert(0 && "Unexpected instr type to insert");
   case X86::TAILJMPd64:
   case X86::TAILJMPr64:
   case X86::TAILJMPm64:
-    assert(!"TAILJMP64 would not be touched here.");
+    assert(0 && "TAILJMP64 would not be touched here.");
   case X86::TCRETURNdi64:
   case X86::TCRETURNri64:
   case X86::TCRETURNmi64:
@@ -10745,6 +11993,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
     return BB;
   case X86::WIN_ALLOCA:
     return EmitLoweredWinAlloca(MI, BB);
+  case X86::SEG_ALLOCA_32:
+    return EmitLoweredSegAlloca(MI, BB, false);
+  case X86::SEG_ALLOCA_64:
+    return EmitLoweredSegAlloca(MI, BB, true);
   case X86::TLSCall_32:
   case X86::TLSCall_64:
     return EmitLoweredTLSCall(MI, BB);
@@ -10754,6 +12006,9 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case X86::CMOV_V4F32:
   case X86::CMOV_V2F64:
   case X86::CMOV_V2I64:
+  case X86::CMOV_V8F32:
+  case X86::CMOV_V4F64:
+  case X86::CMOV_V4I64:
   case X86::CMOV_GR16:
   case X86::CMOV_GR32:
   case X86::CMOV_RFP32:
@@ -11074,6 +12329,33 @@ void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op,
     KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(),
                                        Mask.getBitWidth() - 1);
     break;
+  case ISD::INTRINSIC_WO_CHAIN: {
+    unsigned IntId = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+    unsigned NumLoBits = 0;
+    switch (IntId) {
+    default: break;
+    case Intrinsic::x86_sse_movmsk_ps:
+    case Intrinsic::x86_avx_movmsk_ps_256:
+    case Intrinsic::x86_sse2_movmsk_pd:
+    case Intrinsic::x86_avx_movmsk_pd_256:
+    case Intrinsic::x86_mmx_pmovmskb:
+    case Intrinsic::x86_sse2_pmovmskb_128: {
+      // High bits of movmskp{s|d}, pmovmskb are known zero.
+      switch (IntId) {
+        case Intrinsic::x86_sse_movmsk_ps:      NumLoBits = 4; break;
+        case Intrinsic::x86_avx_movmsk_ps_256:  NumLoBits = 8; break;
+        case Intrinsic::x86_sse2_movmsk_pd:     NumLoBits = 2; break;
+        case Intrinsic::x86_avx_movmsk_pd_256:  NumLoBits = 4; break;
+        case Intrinsic::x86_mmx_pmovmskb:       NumLoBits = 8; break;
+        case Intrinsic::x86_sse2_pmovmskb_128:  NumLoBits = 16; break;
+      }
+      KnownZero = APInt::getHighBitsSet(Mask.getBitWidth(),
+                                        Mask.getBitWidth() - NumLoBits);
+      break;
+    }
+    }
+    break;
+  }
   }
 }
 
@@ -11102,23 +12384,132 @@ bool X86TargetLowering::isGAPlusOffset(SDNode *N,
   return TargetLowering::isGAPlusOffset(N, GA, Offset);
 }
 
-/// PerformShuffleCombine - Combine a vector_shuffle that is equal to
-/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load
-/// if the load addresses are consecutive, non-overlapping, and in the right
-/// order.
+/// isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the
+/// same as extracting the high 128-bit part of 256-bit vector and then
+/// inserting the result into the low part of a new 256-bit vector
+static bool isShuffleHigh128VectorInsertLow(ShuffleVectorSDNode *SVOp) {
+  EVT VT = SVOp->getValueType(0);
+  int NumElems = VT.getVectorNumElements();
+
+  // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
+  for (int i = 0, j = NumElems/2; i < NumElems/2; ++i, ++j)
+    if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
+        SVOp->getMaskElt(j) >= 0)
+      return false;
+
+  return true;
+}
+
+/// isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the
+/// same as extracting the low 128-bit part of 256-bit vector and then
+/// inserting the result into the high part of a new 256-bit vector
+static bool isShuffleLow128VectorInsertHigh(ShuffleVectorSDNode *SVOp) {
+  EVT VT = SVOp->getValueType(0);
+  int NumElems = VT.getVectorNumElements();
+
+  // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
+  for (int i = NumElems/2, j = 0; i < NumElems; ++i, ++j)
+    if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
+        SVOp->getMaskElt(j) >= 0)
+      return false;
+
+  return true;
+}
+
+/// PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors.
+static SDValue PerformShuffleCombine256(SDNode *N, SelectionDAG &DAG,
+                                        TargetLowering::DAGCombinerInfo &DCI) {
+  DebugLoc dl = N->getDebugLoc();
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+  SDValue V1 = SVOp->getOperand(0);
+  SDValue V2 = SVOp->getOperand(1);
+  EVT VT = SVOp->getValueType(0);
+  int NumElems = VT.getVectorNumElements();
+
+  if (V1.getOpcode() == ISD::CONCAT_VECTORS &&
+      V2.getOpcode() == ISD::CONCAT_VECTORS) {
+    //
+    //                   0,0,0,...
+    //                      |
+    //    V      UNDEF    BUILD_VECTOR    UNDEF
+    //     \      /           \           /
+    //  CONCAT_VECTOR         CONCAT_VECTOR
+    //         \                  /
+    //          \                /
+    //          RESULT: V + zero extended
+    //
+    if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR ||
+        V2.getOperand(1).getOpcode() != ISD::UNDEF ||
+        V1.getOperand(1).getOpcode() != ISD::UNDEF)
+      return SDValue();
+
+    if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()))
+      return SDValue();
+
+    // To match the shuffle mask, the first half of the mask should
+    // be exactly the first vector, and all the rest a splat with the
+    // first element of the second one.
+    for (int i = 0; i < NumElems/2; ++i)
+      if (!isUndefOrEqual(SVOp->getMaskElt(i), i) ||
+          !isUndefOrEqual(SVOp->getMaskElt(i+NumElems/2), NumElems))
+        return SDValue();
+
+    // Emit a zeroed vector and insert the desired subvector on its
+    // first half.
+    SDValue Zeros = getZeroVector(VT, true /* HasXMMInt */, DAG, dl);
+    SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0),
+                         DAG.getConstant(0, MVT::i32), DAG, dl);
+    return DCI.CombineTo(N, InsV);
+  }
+
+  //===--------------------------------------------------------------------===//
+  // Combine some shuffles into subvector extracts and inserts:
+  //
+
+  // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
+  if (isShuffleHigh128VectorInsertLow(SVOp)) {
+    SDValue V = Extract128BitVector(V1, DAG.getConstant(NumElems/2, MVT::i32),
+                                    DAG, dl);
+    SDValue InsV = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT),
+                                      V, DAG.getConstant(0, MVT::i32), DAG, dl);
+    return DCI.CombineTo(N, InsV);
+  }
+
+  // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
+  if (isShuffleLow128VectorInsertHigh(SVOp)) {
+    SDValue V = Extract128BitVector(V1, DAG.getConstant(0, MVT::i32), DAG, dl);
+    SDValue InsV = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT),
+                             V, DAG.getConstant(NumElems/2, MVT::i32), DAG, dl);
+    return DCI.CombineTo(N, InsV);
+  }
+
+  return SDValue();
+}
+
+/// PerformShuffleCombine - Performs several different shuffle combines.
 static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
-                                     TargetLowering::DAGCombinerInfo &DCI) {
+                                     TargetLowering::DAGCombinerInfo &DCI,
+                                     const X86Subtarget *Subtarget) {
   DebugLoc dl = N->getDebugLoc();
   EVT VT = N->getValueType(0);
 
-  if (VT.getSizeInBits() != 128)
-    return SDValue();
-
   // Don't create instructions with illegal types after legalize types has run.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType()))
     return SDValue();
 
+  // Combine 256-bit vector shuffles. This is only profitable when in AVX mode
+  if (Subtarget->hasAVX() && VT.getSizeInBits() == 256 &&
+      N->getOpcode() == ISD::VECTOR_SHUFFLE)
+    return PerformShuffleCombine256(N, DAG, DCI);
+
+  // Only handle 128 wide vector from here on.
+  if (VT.getSizeInBits() != 128)
+    return SDValue();
+
+  // Combine a vector_shuffle that is equal to build_vector load1, load2, load3,
+  // load4, <0, 1, 2, 3> into a 128-bit load if the load addresses are
+  // consecutive, non-overlapping, and in the right order.
   SmallVector<SDValue, 16> Elts;
   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
     Elts.push_back(getShuffleScalarElt(N, i, DAG, 0));
@@ -11209,7 +12600,8 @@ static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
-/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes.
+/// PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT
+/// nodes.
 static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
                                     const X86Subtarget *Subtarget) {
   DebugLoc DL = N->getDebugLoc();
@@ -11217,14 +12609,16 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
   // Get the LHS/RHS of the select.
   SDValue LHS = N->getOperand(1);
   SDValue RHS = N->getOperand(2);
+  EVT VT = LHS.getValueType();
 
   // If we have SSE[12] support, try to form min/max nodes. SSE min/max
   // instructions match the semantics of the common C idiom x<y?x:y but not
   // x<=y?x:y, because of how they handle negative zero (which can be
   // ignored in unsafe-math mode).
-  if (Subtarget->hasSSE2() &&
-      (LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64) &&
-      Cond.getOpcode() == ISD::SETCC) {
+  if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
+      VT != MVT::f80 && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+      (Subtarget->hasXMMInt() ||
+       (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) {
     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
 
     unsigned Opcode = 0;
@@ -11267,7 +12661,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
         // Converting this to a max would handle comparisons between positive
         // and negative zero incorrectly.
         if (!UnsafeFPMath &&
-            !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(LHS))
+            !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
           break;
         Opcode = X86ISD::FMAX;
         break;
@@ -11680,7 +13074,7 @@ static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG,
   // all elements are shifted by the same amount.  We can't do this in legalize
   // because the a constant vector is typically transformed to a constant pool
   // so we have no knowledge of the shift amount.
-  if (!Subtarget->hasSSE2())
+  if (!Subtarget->hasXMMInt())
     return SDValue();
 
   if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16)
@@ -11796,7 +13190,7 @@ static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
 
   // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
   // we're requiring SSE2 for both.
-  if (Subtarget->hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
+  if (Subtarget->hasXMMInt() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
     SDValue N0 = N->getOperand(0);
     SDValue N1 = N->getOperand(1);
     SDValue CMP0 = N0->getOperand(1);
@@ -11864,6 +13258,36 @@ static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+/// CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector
+/// so it can be folded inside ANDNP.
+static bool CanFoldXORWithAllOnes(const SDNode *N) {
+  EVT VT = N->getValueType(0);
+
+  // Match direct AllOnes for 128 and 256-bit vectors
+  if (ISD::isBuildVectorAllOnes(N))
+    return true;
+
+  // Look through a bit convert.
+  if (N->getOpcode() == ISD::BITCAST)
+    N = N->getOperand(0).getNode();
+
+  // Sometimes the operand may come from a insert_subvector building a 256-bit
+  // allones vector
+  if (VT.getSizeInBits() == 256 &&
+      N->getOpcode() == ISD::INSERT_SUBVECTOR) {
+    SDValue V1 = N->getOperand(0);
+    SDValue V2 = N->getOperand(1);
+
+    if (V1.getOpcode() == ISD::INSERT_SUBVECTOR &&
+        V1.getOperand(0).getOpcode() == ISD::UNDEF &&
+        ISD::isBuildVectorAllOnes(V1.getOperand(1).getNode()) &&
+        ISD::isBuildVectorAllOnes(V2.getNode()))
+      return true;
+  }
+
+  return false;
+}
+
 static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const X86Subtarget *Subtarget) {
@@ -11874,11 +13298,28 @@ static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
   if (R.getNode())
     return R;
 
+  EVT VT = N->getValueType(0);
+
+  // Create ANDN instructions
+  if (Subtarget->hasBMI() && (VT == MVT::i32 || VT == MVT::i64)) {
+    SDValue N0 = N->getOperand(0);
+    SDValue N1 = N->getOperand(1);
+    DebugLoc DL = N->getDebugLoc();
+
+    // Check LHS for not
+    if (N0.getOpcode() == ISD::XOR && isAllOnes(N0.getOperand(1)))
+      return DAG.getNode(X86ISD::ANDN, DL, VT, N0.getOperand(0), N1);
+    // Check RHS for not
+    if (N1.getOpcode() == ISD::XOR && isAllOnes(N1.getOperand(1)))
+      return DAG.getNode(X86ISD::ANDN, DL, VT, N1.getOperand(0), N0);
+
+    return SDValue();
+  }
+
   // Want to form ANDNP nodes:
   // 1) In the hopes of then easily combining them with OR and AND nodes
   //    to form PBLEND/PSIGN.
   // 2) To match ANDN packed intrinsics
-  EVT VT = N->getValueType(0);
   if (VT != MVT::v2i64 && VT != MVT::v4i64)
     return SDValue();
 
@@ -11888,12 +13329,14 @@ static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
 
   // Check LHS for vnot
   if (N0.getOpcode() == ISD::XOR &&
-      ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode()))
+      //ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode()))
+      CanFoldXORWithAllOnes(N0.getOperand(1).getNode()))
     return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
 
   // Check RHS for vnot
   if (N1.getOpcode() == ISD::XOR &&
-      ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode()))
+      //ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode()))
+      CanFoldXORWithAllOnes(N1.getOperand(1).getNode()))
     return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
 
   return SDValue();
@@ -11917,7 +13360,7 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
   SDValue N1 = N->getOperand(1);
 
   // look for psign/blend
-  if (Subtarget->hasSSSE3()) {
+  if (Subtarget->hasSSSE3() || Subtarget->hasAVX()) {
     if (VT == MVT::v2i64) {
       // Canonicalize pandn to RHS
       if (N0.getOpcode() == X86ISD::ANDNP)
@@ -11990,13 +13433,13 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
           }
         }
         // PBLENDVB only available on SSE 4.1
-        if (!Subtarget->hasSSE41())
+        if (!(Subtarget->hasSSE41() || Subtarget->hasAVX()))
           return SDValue();
 
         X = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, X);
         Y = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Y);
         Mask = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Mask);
-        Mask = DAG.getNode(X86ISD::PBLENDVB, DL, MVT::v16i8, X, Y, Mask);
+        Mask = DAG.getNode(ISD::VSELECT, DL, MVT::v16i8, Mask, X, Y);
         return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Mask);
       }
     }
@@ -12057,24 +13500,211 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+/// PerformLOADCombine - Do target-specific dag combines on LOAD nodes.
+static SDValue PerformLOADCombine(SDNode *N, SelectionDAG &DAG,
+                                   const X86Subtarget *Subtarget) {
+  LoadSDNode *Ld = cast<LoadSDNode>(N);
+  EVT RegVT = Ld->getValueType(0);
+  EVT MemVT = Ld->getMemoryVT();
+  DebugLoc dl = Ld->getDebugLoc();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  ISD::LoadExtType Ext = Ld->getExtensionType();
+
+  // If this is a vector EXT Load then attempt to optimize it using a
+  // shuffle. We need SSE4 for the shuffles.
+  // TODO: It is possible to support ZExt by zeroing the undef values
+  // during the shuffle phase or after the shuffle.
+  if (RegVT.isVector() && Ext == ISD::EXTLOAD && Subtarget->hasSSE41()) {
+    assert(MemVT != RegVT && "Cannot extend to the same type");
+    assert(MemVT.isVector() && "Must load a vector from memory");
+
+    unsigned NumElems = RegVT.getVectorNumElements();
+    unsigned RegSz = RegVT.getSizeInBits();
+    unsigned MemSz = MemVT.getSizeInBits();
+    assert(RegSz > MemSz && "Register size must be greater than the mem size");
+    // All sizes must be a power of two
+    if (!isPowerOf2_32(RegSz * MemSz * NumElems)) return SDValue();
+
+    // Attempt to load the original value using a single load op.
+    // Find a scalar type which is equal to the loaded word size.
+    MVT SclrLoadTy = MVT::i8;
+    for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE;
+         tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) {
+      MVT Tp = (MVT::SimpleValueType)tp;
+      if (TLI.isTypeLegal(Tp) &&  Tp.getSizeInBits() == MemSz) {
+        SclrLoadTy = Tp;
+        break;
+      }
+    }
+
+    // Proceed if a load word is found.
+    if (SclrLoadTy.getSizeInBits() != MemSz) return SDValue();
+
+    EVT LoadUnitVecVT = EVT::getVectorVT(*DAG.getContext(), SclrLoadTy,
+      RegSz/SclrLoadTy.getSizeInBits());
+
+    EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
+                                  RegSz/MemVT.getScalarType().getSizeInBits());
+    // Can't shuffle using an illegal type.
+    if (!TLI.isTypeLegal(WideVecVT)) return SDValue();
+
+    // Perform a single load.
+    SDValue ScalarLoad = DAG.getLoad(SclrLoadTy, dl, Ld->getChain(),
+                                  Ld->getBasePtr(),
+                                  Ld->getPointerInfo(), Ld->isVolatile(),
+                                  Ld->isNonTemporal(), Ld->getAlignment());
+
+    // Insert the word loaded into a vector.
+    SDValue ScalarInVector = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
+      LoadUnitVecVT, ScalarLoad);
+
+    // Bitcast the loaded value to a vector of the original element type, in
+    // the size of the target vector type.
+    SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, ScalarInVector);
+    unsigned SizeRatio = RegSz/MemSz;
+
+    // Redistribute the loaded elements into the different locations.
+    SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
+    for (unsigned i = 0; i < NumElems; i++) ShuffleVec[i*SizeRatio] = i;
+
+    SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
+                                DAG.getUNDEF(SlicedVec.getValueType()),
+                                ShuffleVec.data());
+
+    // Bitcast to the requested type.
+    Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
+    // Replace the original load with the new sequence
+    // and return the new chain.
+    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Shuff);
+    return SDValue(ScalarLoad.getNode(), 1);
+  }
+
+  return SDValue();
+}
+
 /// PerformSTORECombine - Do target-specific dag combines on STORE nodes.
 static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
                                    const X86Subtarget *Subtarget) {
+  StoreSDNode *St = cast<StoreSDNode>(N);
+  EVT VT = St->getValue().getValueType();
+  EVT StVT = St->getMemoryVT();
+  DebugLoc dl = St->getDebugLoc();
+  SDValue StoredVal = St->getOperand(1);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // If we are saving a concatination of two XMM registers, perform two stores.
+  // This is better in Sandy Bridge cause one 256-bit mem op is done via two
+  // 128-bit ones. If in the future the cost becomes only one memory access the
+  // first version would be better.
+  if (VT.getSizeInBits() == 256 &&
+    StoredVal.getNode()->getOpcode() == ISD::CONCAT_VECTORS &&
+    StoredVal.getNumOperands() == 2) {
+
+    SDValue Value0 = StoredVal.getOperand(0);
+    SDValue Value1 = StoredVal.getOperand(1);
+
+    SDValue Stride = DAG.getConstant(16, TLI.getPointerTy());
+    SDValue Ptr0 = St->getBasePtr();
+    SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride);
+
+    SDValue Ch0 = DAG.getStore(St->getChain(), dl, Value0, Ptr0,
+                                St->getPointerInfo(), St->isVolatile(),
+                                St->isNonTemporal(), St->getAlignment());
+    SDValue Ch1 = DAG.getStore(St->getChain(), dl, Value1, Ptr1,
+                                St->getPointerInfo(), St->isVolatile(),
+                                St->isNonTemporal(), St->getAlignment());
+    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
+  }
+
+  // Optimize trunc store (of multiple scalars) to shuffle and store.
+  // First, pack all of the elements in one place. Next, store to memory
+  // in fewer chunks.
+  if (St->isTruncatingStore() && VT.isVector()) {
+    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+    unsigned NumElems = VT.getVectorNumElements();
+    assert(StVT != VT && "Cannot truncate to the same type");
+    unsigned FromSz = VT.getVectorElementType().getSizeInBits();
+    unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
+
+    // From, To sizes and ElemCount must be pow of two
+    if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue();
+    // We are going to use the original vector elt for storing.
+    // Accumulated smaller vector elements must be a multiple of the store size.
+    if (0 != (NumElems * FromSz) % ToSz) return SDValue();
+
+    unsigned SizeRatio  = FromSz / ToSz;
+
+    assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
+
+    // Create a type on which we perform the shuffle
+    EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
+            StVT.getScalarType(), NumElems*SizeRatio);
+
+    assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
+
+    SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue());
+    SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
+    for (unsigned i = 0; i < NumElems; i++ ) ShuffleVec[i] = i * SizeRatio;
+
+    // Can't shuffle using an illegal type
+    if (!TLI.isTypeLegal(WideVecVT)) return SDValue();
+
+    SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
+                                DAG.getUNDEF(WideVec.getValueType()),
+                                ShuffleVec.data());
+    // At this point all of the data is stored at the bottom of the
+    // register. We now need to save it to mem.
+
+    // Find the largest store unit
+    MVT StoreType = MVT::i8;
+    for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE;
+         tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) {
+      MVT Tp = (MVT::SimpleValueType)tp;
+      if (TLI.isTypeLegal(Tp) && StoreType.getSizeInBits() < NumElems * ToSz)
+        StoreType = Tp;
+    }
+
+    // Bitcast the original vector into a vector of store-size units
+    EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
+            StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits());
+    assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
+    SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff);
+    SmallVector<SDValue, 8> Chains;
+    SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8,
+                                        TLI.getPointerTy());
+    SDValue Ptr = St->getBasePtr();
+
+    // Perform one or more big stores into memory.
+    for (unsigned i = 0; i < (ToSz*NumElems)/StoreType.getSizeInBits() ; i++) {
+      SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                                   StoreType, ShuffWide,
+                                   DAG.getIntPtrConstant(i));
+      SDValue Ch = DAG.getStore(St->getChain(), dl, SubVec, Ptr,
+                                St->getPointerInfo(), St->isVolatile(),
+                                St->isNonTemporal(), St->getAlignment());
+      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
+      Chains.push_back(Ch);
+    }
+
+    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0],
+                               Chains.size());
+  }
+
+
   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
   // the FP state in cases where an emms may be missing.
   // A preferable solution to the general problem is to figure out the right
   // places to insert EMMS.  This qualifies as a quick hack.
 
   // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
-  StoreSDNode *St = cast<StoreSDNode>(N);
-  EVT VT = St->getValue().getValueType();
   if (VT.getSizeInBits() != 64)
     return SDValue();
 
   const Function *F = DAG.getMachineFunction().getFunction();
   bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat);
   bool F64IsLegal = !UseSoftFloat && !NoImplicitFloatOps
-    && Subtarget->hasSSE2();
+                     && Subtarget->hasXMMInt();
   if ((VT.isVector() ||
        (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
       isa<LoadSDNode>(St->getValue()) &&
@@ -12172,6 +13802,150 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+/// isHorizontalBinOp - Return 'true' if this vector operation is "horizontal"
+/// and return the operands for the horizontal operation in LHS and RHS.  A
+/// horizontal operation performs the binary operation on successive elements
+/// of its first operand, then on successive elements of its second operand,
+/// returning the resulting values in a vector.  For example, if
+///   A = < float a0, float a1, float a2, float a3 >
+/// and
+///   B = < float b0, float b1, float b2, float b3 >
+/// then the result of doing a horizontal operation on A and B is
+///   A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
+/// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
+/// A horizontal-op B, for some already available A and B, and if so then LHS is
+/// set to A, RHS to B, and the routine returns 'true'.
+/// Note that the binary operation should have the property that if one of the
+/// operands is UNDEF then the result is UNDEF.
+static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, bool isCommutative) {
+  // Look for the following pattern: if
+  //   A = < float a0, float a1, float a2, float a3 >
+  //   B = < float b0, float b1, float b2, float b3 >
+  // and
+  //   LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
+  //   RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
+  // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
+  // which is A horizontal-op B.
+
+  // At least one of the operands should be a vector shuffle.
+  if (LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
+      RHS.getOpcode() != ISD::VECTOR_SHUFFLE)
+    return false;
+
+  EVT VT = LHS.getValueType();
+  unsigned N = VT.getVectorNumElements();
+
+  // View LHS in the form
+  //   LHS = VECTOR_SHUFFLE A, B, LMask
+  // If LHS is not a shuffle then pretend it is the shuffle
+  //   LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
+  // NOTE: in what follows a default initialized SDValue represents an UNDEF of
+  // type VT.
+  SDValue A, B;
+  SmallVector<int, 8> LMask(N);
+  if (LHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
+    if (LHS.getOperand(0).getOpcode() != ISD::UNDEF)
+      A = LHS.getOperand(0);
+    if (LHS.getOperand(1).getOpcode() != ISD::UNDEF)
+      B = LHS.getOperand(1);
+    cast<ShuffleVectorSDNode>(LHS.getNode())->getMask(LMask);
+  } else {
+    if (LHS.getOpcode() != ISD::UNDEF)
+      A = LHS;
+    for (unsigned i = 0; i != N; ++i)
+      LMask[i] = i;
+  }
+
+  // Likewise, view RHS in the form
+  //   RHS = VECTOR_SHUFFLE C, D, RMask
+  SDValue C, D;
+  SmallVector<int, 8> RMask(N);
+  if (RHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
+    if (RHS.getOperand(0).getOpcode() != ISD::UNDEF)
+      C = RHS.getOperand(0);
+    if (RHS.getOperand(1).getOpcode() != ISD::UNDEF)
+      D = RHS.getOperand(1);
+    cast<ShuffleVectorSDNode>(RHS.getNode())->getMask(RMask);
+  } else {
+    if (RHS.getOpcode() != ISD::UNDEF)
+      C = RHS;
+    for (unsigned i = 0; i != N; ++i)
+      RMask[i] = i;
+  }
+
+  // Check that the shuffles are both shuffling the same vectors.
+  if (!(A == C && B == D) && !(A == D && B == C))
+    return false;
+
+  // If everything is UNDEF then bail out: it would be better to fold to UNDEF.
+  if (!A.getNode() && !B.getNode())
+    return false;
+
+  // If A and B occur in reverse order in RHS, then "swap" them (which means
+  // rewriting the mask).
+  if (A != C)
+    for (unsigned i = 0; i != N; ++i) {
+      unsigned Idx = RMask[i];
+      if (Idx < N)
+        RMask[i] += N;
+      else if (Idx < 2*N)
+        RMask[i] -= N;
+    }
+
+  // At this point LHS and RHS are equivalent to
+  //   LHS = VECTOR_SHUFFLE A, B, LMask
+  //   RHS = VECTOR_SHUFFLE A, B, RMask
+  // Check that the masks correspond to performing a horizontal operation.
+  for (unsigned i = 0; i != N; ++i) {
+    unsigned LIdx = LMask[i], RIdx = RMask[i];
+
+    // Ignore any UNDEF components.
+    if (LIdx >= 2*N || RIdx >= 2*N || (!A.getNode() && (LIdx < N || RIdx < N))
+        || (!B.getNode() && (LIdx >= N || RIdx >= N)))
+      continue;
+
+    // Check that successive elements are being operated on.  If not, this is
+    // not a horizontal operation.
+    if (!(LIdx == 2*i && RIdx == 2*i + 1) &&
+        !(isCommutative && LIdx == 2*i + 1 && RIdx == 2*i))
+      return false;
+  }
+
+  LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
+  RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
+  return true;
+}
+
+/// PerformFADDCombine - Do target-specific dag combines on floating point adds.
+static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
+                                  const X86Subtarget *Subtarget) {
+  EVT VT = N->getValueType(0);
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+
+  // Try to synthesize horizontal adds from adds of shuffles.
+  if ((Subtarget->hasSSE3() || Subtarget->hasAVX()) &&
+      (VT == MVT::v4f32 || VT == MVT::v2f64) &&
+      isHorizontalBinOp(LHS, RHS, true))
+    return DAG.getNode(X86ISD::FHADD, N->getDebugLoc(), VT, LHS, RHS);
+  return SDValue();
+}
+
+/// PerformFSUBCombine - Do target-specific dag combines on floating point subs.
+static SDValue PerformFSUBCombine(SDNode *N, SelectionDAG &DAG,
+                                  const X86Subtarget *Subtarget) {
+  EVT VT = N->getValueType(0);
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+
+  // Try to synthesize horizontal subs from subs of shuffles.
+  if ((Subtarget->hasSSE3() || Subtarget->hasAVX()) &&
+      (VT == MVT::v4f32 || VT == MVT::v2f64) &&
+      isHorizontalBinOp(LHS, RHS, false))
+    return DAG.getNode(X86ISD::FHSUB, N->getDebugLoc(), VT, LHS, RHS);
+  return SDValue();
+}
+
 /// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and
 /// X86ISD::FXOR nodes.
 static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) {
@@ -12326,7 +14100,7 @@ static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG,
 //      (add Y, (setne X, 0)) -> sbb -1, Y
 //      (sub (sete  X, 0), Y) -> sbb  0, Y
 //      (sub (setne X, 0), Y) -> adc -1, Y
-static SDValue OptimizeConditonalInDecrement(SDNode *N, SelectionDAG &DAG) {
+static SDValue OptimizeConditionalInDecrement(SDNode *N, SelectionDAG &DAG) {
   DebugLoc DL = N->getDebugLoc();
 
   // Look through ZExts.
@@ -12362,6 +14136,31 @@ static SDValue OptimizeConditonalInDecrement(SDNode *N, SelectionDAG &DAG) {
                      DAG.getConstant(0, OtherVal.getValueType()), NewCmp);
 }
 
+static SDValue PerformSubCombine(SDNode *N, SelectionDAG &DAG) {
+  SDValue Op0 = N->getOperand(0);
+  SDValue Op1 = N->getOperand(1);
+
+  // X86 can't encode an immediate LHS of a sub. See if we can push the
+  // negation into a preceding instruction.
+  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op0)) {
+    // If the RHS of the sub is a XOR with one use and a constant, invert the
+    // immediate. Then add one to the LHS of the sub so we can turn
+    // X-Y -> X+~Y+1, saving one register.
+    if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR &&
+        isa<ConstantSDNode>(Op1.getOperand(1))) {
+      APInt XorC = cast<ConstantSDNode>(Op1.getOperand(1))->getAPIntValue();
+      EVT VT = Op0.getValueType();
+      SDValue NewXor = DAG.getNode(ISD::XOR, Op1.getDebugLoc(), VT,
+                                   Op1.getOperand(0),
+                                   DAG.getConstant(~XorC, VT));
+      return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, NewXor,
+                         DAG.getConstant(C->getAPIntValue()+1, VT));
+    }
+  }
+
+  return OptimizeConditionalInDecrement(N, DAG);
+}
+
 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -12369,10 +14168,11 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   default: break;
   case ISD::EXTRACT_VECTOR_ELT:
     return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, *this);
+  case ISD::VSELECT:
   case ISD::SELECT:         return PerformSELECTCombine(N, DAG, Subtarget);
   case X86ISD::CMOV:        return PerformCMOVCombine(N, DAG, DCI);
-  case ISD::ADD:
-  case ISD::SUB:            return OptimizeConditonalInDecrement(N, DAG);
+  case ISD::ADD:            return OptimizeConditionalInDecrement(N, DAG);
+  case ISD::SUB:            return PerformSubCombine(N, DAG);
   case X86ISD::ADC:         return PerformADCCombine(N, DAG, DCI);
   case ISD::MUL:            return PerformMulCombine(N, DAG, DCI);
   case ISD::SHL:
@@ -12380,8 +14180,11 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::SRL:            return PerformShiftCombine(N, DAG, Subtarget);
   case ISD::AND:            return PerformAndCombine(N, DAG, DCI, Subtarget);
   case ISD::OR:             return PerformOrCombine(N, DAG, DCI, Subtarget);
+  case ISD::LOAD:           return PerformLOADCombine(N, DAG, Subtarget);
   case ISD::STORE:          return PerformSTORECombine(N, DAG, Subtarget);
   case ISD::SINT_TO_FP:     return PerformSINT_TO_FPCombine(N, DAG, this);
+  case ISD::FADD:           return PerformFADDCombine(N, DAG, Subtarget);
+  case ISD::FSUB:           return PerformFSUBCombine(N, DAG, Subtarget);
   case X86ISD::FXOR:
   case X86ISD::FOR:         return PerformFORCombine(N, DAG);
   case X86ISD::FAND:        return PerformFANDCombine(N, DAG);
@@ -12398,14 +14201,14 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::PUNPCKHQDQ:
   case X86ISD::UNPCKHPS:
   case X86ISD::UNPCKHPD:
+  case X86ISD::VUNPCKHPSY:
+  case X86ISD::VUNPCKHPDY:
   case X86ISD::PUNPCKLBW:
   case X86ISD::PUNPCKLWD:
   case X86ISD::PUNPCKLDQ:
   case X86ISD::PUNPCKLQDQ:
   case X86ISD::UNPCKLPS:
   case X86ISD::UNPCKLPD:
-  case X86ISD::VUNPCKLPS:
-  case X86ISD::VUNPCKLPD:
   case X86ISD::VUNPCKLPSY:
   case X86ISD::VUNPCKLPDY:
   case X86ISD::MOVHLPS:
@@ -12415,7 +14218,12 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::PSHUFLW:
   case X86ISD::MOVSS:
   case X86ISD::MOVSD:
-  case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI);
+  case X86ISD::VPERMILPS:
+  case X86ISD::VPERMILPSY:
+  case X86ISD::VPERMILPD:
+  case X86ISD::VPERMILPDY:
+  case X86ISD::VPERM2F128:
+  case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI,Subtarget);
   }
 
   return SDValue();
@@ -12551,7 +14359,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
          AsmPieces[1] == "${0:q}")) {
       // No need to check constraints, nothing other than the equivalent of
       // "=r,0" would be valid here.
-      const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
+      IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
       if (!Ty || Ty->getBitWidth() % 16 != 0)
         return false;
       return IntrinsicLowering::LowerToByteSwap(CI);
@@ -12572,7 +14380,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
           AsmPieces[1] == "~{dirflag}" &&
           AsmPieces[2] == "~{flags}" &&
           AsmPieces[3] == "~{fpsr}") {
-        const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
+        IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
         if (!Ty || Ty->getBitWidth() % 16 != 0)
           return false;
         return IntrinsicLowering::LowerToByteSwap(CI);
@@ -12603,7 +14411,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
                 AsmPieces[1] == "~{dirflag}" &&
                 AsmPieces[2] == "~{flags}" &&
                 AsmPieces[3] == "~{fpsr}") {
-              const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
+              IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
               if (!Ty || Ty->getBitWidth() % 16 != 0)
                 return false;
               return IntrinsicLowering::LowerToByteSwap(CI);
@@ -12629,7 +14437,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
             SplitString(AsmPieces[2], Words, " \t,");
             if (Words.size() == 3 && Words[0] == "xchgl" && Words[1] == "%eax" &&
                 Words[2] == "%edx") {
-              const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
+              IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
               if (!Ty || Ty->getBitWidth() % 16 != 0)
                 return false;
               return IntrinsicLowering::LowerToByteSwap(CI);
@@ -12700,7 +14508,7 @@ TargetLowering::ConstraintWeight
     // but allow it at the lowest weight.
   if (CallOperandVal == NULL)
     return CW_Default;
-  const Type *type = CallOperandVal->getType();
+  Type *type = CallOperandVal->getType();
   // Look at the constraint type.
   switch (*constraint) {
   default:
diff --git a/contrib/llvm/lib/Target/X86/X86ISelLowering.h b/contrib/llvm/lib/Target/X86/X86ISelLowering.h
index b6036782b865..342a5e617545 100644
--- a/contrib/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/contrib/llvm/lib/Target/X86/X86ISelLowering.h
@@ -175,8 +175,14 @@ namespace llvm {
       /// PSIGNB/W/D - Copy integer sign.
       PSIGNB, PSIGNW, PSIGND,
 
-      /// PBLENDVB - Variable blend
-      PBLENDVB,
+      /// BLEND family of opcodes
+      BLENDV,
+
+      /// FHADD - Floating point horizontal add.
+      FHADD,
+
+      /// FHSUB - Floating point horizontal sub.
+      FHSUB,
 
       /// FMAX, FMIN - Floating point max and min.
       ///
@@ -222,6 +228,8 @@ namespace llvm {
       ADD, SUB, ADC, SBB, SMUL,
       INC, DEC, OR, XOR, AND,
 
+      ANDN, // ANDN - Bitwise AND NOT with FLAGS results.
+
       UMUL, // LOW, HI, FLAGS = umul LHS, RHS
 
       // MUL_IMM - X86 specific multiply by immediate.
@@ -257,12 +265,12 @@ namespace llvm {
       MOVSS,
       UNPCKLPS,
       UNPCKLPD,
-      VUNPCKLPS,
-      VUNPCKLPD,
       VUNPCKLPSY,
       VUNPCKLPDY,
       UNPCKHPS,
       UNPCKHPD,
+      VUNPCKHPSY,
+      VUNPCKHPDY,
       PUNPCKLBW,
       PUNPCKLWD,
       PUNPCKLDQ,
@@ -271,6 +279,12 @@ namespace llvm {
       PUNPCKHWD,
       PUNPCKHDQ,
       PUNPCKHQDQ,
+      VPERMILPS,
+      VPERMILPSY,
+      VPERMILPD,
+      VPERMILPDY,
+      VPERM2F128,
+      VBROADCAST,
 
       // VASTART_SAVE_XMM_REGS - Save xmm argument registers to the stack,
       // according to %al. An operator is needed so that this can be expanded
@@ -280,6 +294,11 @@ namespace llvm {
       // WIN_ALLOCA - Windows's _chkstk call to do stack probing.
       WIN_ALLOCA,
 
+      // SEG_ALLOCA - For allocating variable amounts of stack space when using
+      // segmented stacks. Check if the current stacklet has enough space, and
+      // falls back to heap allocation if not.
+      SEG_ALLOCA,
+
       // Memory barrier
       MEMBARRIER,
       MFENCE,
@@ -297,9 +316,10 @@ namespace llvm {
       ATOMNAND64_DAG,
       ATOMSWAP64_DAG,
 
-      // LCMPXCHG_DAG, LCMPXCHG8_DAG - Compare and swap.
+      // LCMPXCHG_DAG, LCMPXCHG8_DAG, LCMPXCHG16_DAG - Compare and swap.
       LCMPXCHG_DAG,
       LCMPXCHG8_DAG,
+      LCMPXCHG16_DAG,
 
       // VZEXT_LOAD - Load, scalar_to_vector, and zero extend.
       VZEXT_LOAD,
@@ -407,20 +427,16 @@ namespace llvm {
 
     /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
     /// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
-    bool isMOVSHDUPMask(ShuffleVectorSDNode *N);
+    bool isMOVSHDUPMask(ShuffleVectorSDNode *N, const X86Subtarget *Subtarget);
 
     /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
     /// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
-    bool isMOVSLDUPMask(ShuffleVectorSDNode *N);
+    bool isMOVSLDUPMask(ShuffleVectorSDNode *N, const X86Subtarget *Subtarget);
 
     /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
     /// specifies a shuffle of elements that is suitable for input to MOVDDUP.
     bool isMOVDDUPMask(ShuffleVectorSDNode *N);
 
-    /// isPALIGNRMask - Return true if the specified VECTOR_SHUFFLE operand
-    /// specifies a shuffle of elements that is suitable for input to PALIGNR.
-    bool isPALIGNRMask(ShuffleVectorSDNode *N);
-
     /// isVEXTRACTF128Index - Return true if the specified
     /// EXTRACT_SUBVECTOR operand specifies a vector extract that is
     /// suitable for input to VEXTRACTF128.
@@ -505,7 +521,7 @@ namespace llvm {
     /// function arguments in the caller parameter area. For X86, aggregates
     /// that contains are placed at 16-byte boundaries while the rest are at
     /// 4-byte boundaries.
-    virtual unsigned getByValTypeAlignment(const Type *Ty) const;
+    virtual unsigned getByValTypeAlignment(Type *Ty) const;
 
     /// getOptimalMemOpType - Returns the target specific optimal type for load
     /// and store operations as a result of memset, memcpy, and memmove
@@ -564,8 +580,8 @@ namespace llvm {
     /// DAG node.
     virtual const char *getTargetNodeName(unsigned Opcode) const;
 
-    /// getSetCCResultType - Return the ISD::SETCC ValueType
-    virtual MVT::SimpleValueType getSetCCResultType(EVT VT) const;
+    /// getSetCCResultType - Return the value type to use for ISD::SETCC.
+    virtual EVT getSetCCResultType(EVT VT) const;
 
     /// computeMaskedBitsForTargetNode - Determine which of the bits specified
     /// in Mask are known to be either zero or one and return them in the
@@ -617,12 +633,12 @@ namespace llvm {
 
     /// isLegalAddressingMode - Return true if the addressing mode represented
     /// by AM is legal for this target, for a load/store of the specified type.
-    virtual bool isLegalAddressingMode(const AddrMode &AM, const Type *Ty)const;
+    virtual bool isLegalAddressingMode(const AddrMode &AM, Type *Ty)const;
 
     /// isTruncateFree - Return true if it's free to truncate a value of
     /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
     /// register EAX to i16 by referencing its sub-register AX.
-    virtual bool isTruncateFree(const Type *Ty1, const Type *Ty2) const;
+    virtual bool isTruncateFree(Type *Ty1, Type *Ty2) const;
     virtual bool isTruncateFree(EVT VT1, EVT VT2) const;
 
     /// isZExtFree - Return true if any actual instruction that defines a
@@ -633,7 +649,7 @@ namespace llvm {
     /// does not necessarily apply to truncate instructions. e.g. on x86-64,
     /// all instructions that define 32-bit values implicit zero-extend the
     /// result out to 64 bits.
-    virtual bool isZExtFree(const Type *Ty1, const Type *Ty2) const;
+    virtual bool isZExtFree(Type *Ty1, Type *Ty2) const;
     virtual bool isZExtFree(EVT VT1, EVT VT2) const;
 
     /// isNarrowingProfitable - Return true if it's profitable to narrow
@@ -813,11 +829,14 @@ namespace llvm {
     SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerTRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerADD(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerShift(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) const;
 
@@ -825,6 +844,7 @@ namespace llvm {
     SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
 
     // Utility functions to help LowerVECTOR_SHUFFLE
@@ -931,6 +951,10 @@ namespace llvm {
     MachineBasicBlock *EmitLoweredWinAlloca(MachineInstr *MI,
                                               MachineBasicBlock *BB) const;
 
+    MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr *MI,
+                                            MachineBasicBlock *BB,
+                                            bool Is64Bit) const;
+
     MachineBasicBlock *EmitLoweredTLSCall(MachineInstr *MI,
                                           MachineBasicBlock *BB) const;
 
diff --git a/contrib/llvm/lib/Target/X86/X86InstrArithmetic.td b/contrib/llvm/lib/Target/X86/X86InstrArithmetic.td
index 9f7a4b06dc6f..74b647a4f6b1 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrArithmetic.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrArithmetic.td
@@ -650,6 +650,15 @@ class BinOpRR_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo>
   let isCodeGenOnly = 1;
 }
 
+// BinOpRR_F_Rev - Instructions like "cmp reg, reg" (reversed encoding).
+class BinOpRR_F_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo>
+  : ITy<opcode, MRMSrcReg, typeinfo, (outs),
+        (ins typeinfo.RegClass:$src1, typeinfo.RegClass:$src2),
+        mnemonic, "{$src2, $src1|$src1, $src2}", []> {
+  // The disassembler should know about this, but not the asmparser.
+  let isCodeGenOnly = 1;
+}
+
 // BinOpRM - Instructions like "add reg, reg, [mem]".
 class BinOpRM<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo,
               dag outlist, list<dag> pattern>
@@ -857,11 +866,10 @@ class BinOpMI8_F<string mnemonic, X86TypeInfo typeinfo,
 
 // BinOpAI - Instructions like "add %eax, %eax, imm".
 class BinOpAI<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo,
-              Register areg>
+              Register areg, string operands>
   : ITy<opcode, RawFrm, typeinfo,
         (outs), (ins typeinfo.ImmOperand:$src),
-        mnemonic, !strconcat("{$src, %", areg.AsmName, "|%",
-                               areg.AsmName, ", $src}"), []> {
+        mnemonic, operands, []> {
   let ImmT = typeinfo.ImmEncoding;
   let Uses = [areg];
   let Defs = [areg];
@@ -926,10 +934,14 @@ multiclass ArithBinOp_RF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4,
     def #NAME#32mi   : BinOpMI_RMW<mnemonic, Xi32, opnode, MemMRM>;
     def #NAME#64mi32 : BinOpMI_RMW<mnemonic, Xi64, opnode, MemMRM>;
 
-    def #NAME#8i8   : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL>;
-    def #NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX>;
-    def #NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX>;
-    def #NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX>;
+    def #NAME#8i8   : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL,
+                              "{$src, %al|AL, $src}">;
+    def #NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX,
+                              "{$src, %ax|AX, $src}">;
+    def #NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX,
+                              "{$src, %eax|EAX, $src}">;
+    def #NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX,
+                              "{$src, %rax|RAX, $src}">;
   }                          
 }
 
@@ -993,10 +1005,14 @@ multiclass ArithBinOp_RFF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4,
     def #NAME#32mi   : BinOpMI_RMW_FF<mnemonic, Xi32, opnode, MemMRM>;
     def #NAME#64mi32 : BinOpMI_RMW_FF<mnemonic, Xi64, opnode, MemMRM>;
 
-    def #NAME#8i8   : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL>;
-    def #NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX>;
-    def #NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX>;
-    def #NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX>;
+    def #NAME#8i8   : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL,
+                              "{$src, %al|AL, $src}">;
+    def #NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX,
+                              "{$src, %ax|AX, $src}">;
+    def #NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX,
+                              "{$src, %eax|EAX, $src}">;
+    def #NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX, 
+                              "{$src, %rax|RAX, $src}">;
   }                          
 }
 
@@ -1017,10 +1033,10 @@ multiclass ArithBinOp_F<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4,
       def #NAME#64rr : BinOpRR_F<BaseOpc, mnemonic, Xi64, opnode>;
     } // isCommutable
 
-    def #NAME#8rr_REV  : BinOpRR_Rev<BaseOpc2, mnemonic, Xi8>;
-    def #NAME#16rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi16>;
-    def #NAME#32rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi32>;
-    def #NAME#64rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi64>;
+    def #NAME#8rr_REV  : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi8>;
+    def #NAME#16rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi16>;
+    def #NAME#32rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi32>;
+    def #NAME#64rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi64>;
 
     def #NAME#8rm   : BinOpRM_F<BaseOpc2, mnemonic, Xi8 , opnode>;
     def #NAME#16rm  : BinOpRM_F<BaseOpc2, mnemonic, Xi16, opnode>;
@@ -1056,10 +1072,14 @@ multiclass ArithBinOp_F<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4,
     def #NAME#32mi   : BinOpMI_F<mnemonic, Xi32, opnode, MemMRM>;
     def #NAME#64mi32 : BinOpMI_F<mnemonic, Xi64, opnode, MemMRM>;
 
-    def #NAME#8i8   : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL>;
-    def #NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX>;
-    def #NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX>;
-    def #NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX>;
+    def #NAME#8i8   : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL,
+                              "{$src, %al|AL, $src}">;
+    def #NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX,
+                              "{$src, %ax|AX, $src}">;
+    def #NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX,
+                              "{$src, %eax|EAX, $src}">;
+    def #NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX,
+                              "{$src, %rax|RAX, $src}">;
   }                          
 }
 
@@ -1117,9 +1137,37 @@ let Defs = [EFLAGS] in {
   def TEST32mi   : BinOpMI_F<"test", Xi32, X86testpat, MRM0m, 0xF6>;
   def TEST64mi32 : BinOpMI_F<"test", Xi64, X86testpat, MRM0m, 0xF6>;
                      
-  def TEST8i8    : BinOpAI<0xA8, "test", Xi8 , AL>;
-  def TEST16i16  : BinOpAI<0xA8, "test", Xi16, AX>;
-  def TEST32i32  : BinOpAI<0xA8, "test", Xi32, EAX>;
-  def TEST64i32  : BinOpAI<0xA8, "test", Xi64, RAX>;
-}                          
+  def TEST8i8    : BinOpAI<0xA8, "test", Xi8 , AL,
+                           "{$src, %al|AL, $src}">;
+  def TEST16i16  : BinOpAI<0xA8, "test", Xi16, AX,
+                           "{$src, %ax|AX, $src}">;
+  def TEST32i32  : BinOpAI<0xA8, "test", Xi32, EAX,
+                           "{$src, %eax|EAX, $src}">;
+  def TEST64i32  : BinOpAI<0xA8, "test", Xi64, RAX,
+                           "{$src, %rax|RAX, $src}">;
 
+  // When testing the result of EXTRACT_SUBREG sub_8bit_hi, make sure the
+  // register class is constrained to GR8_NOREX.
+  let isPseudo = 1 in
+  def TEST8ri_NOREX : I<0, Pseudo, (outs), (ins GR8_NOREX:$src, i8imm:$mask),
+                        "", []>;
+}
+
+//===----------------------------------------------------------------------===//
+// ANDN Instruction
+//
+multiclass bmi_andn<string mnemonic, RegisterClass RC, X86MemOperand x86memop,
+                    PatFrag ld_frag> {
+  def rr : I<0xF2, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
+            !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
+            [(set RC:$dst, EFLAGS, (X86andn_flag RC:$src1, RC:$src2))]>;
+  def rm : I<0xF2, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
+            !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
+            [(set RC:$dst, EFLAGS,
+             (X86andn_flag RC:$src1, (ld_frag addr:$src2)))]>;
+}
+
+let Predicates = [HasBMI], Defs = [EFLAGS] in {
+  defm ANDN32 : bmi_andn<"andn{l}", GR32, i32mem, loadi32>, T8, VEX_4V;
+  defm ANDN64 : bmi_andn<"andn{q}", GR64, i64mem, loadi64>, T8, VEX_4V, VEX_W;
+}
diff --git a/contrib/llvm/lib/Target/X86/X86InstrCompiler.td b/contrib/llvm/lib/Target/X86/X86InstrCompiler.td
index adcc747eb4b8..da28690672a6 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -106,6 +106,26 @@ let Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in
   def WIN_ALLOCA : I<0, Pseudo, (outs), (ins),
                      "# dynamic stack allocation",
                      [(X86WinAlloca)]>;
+
+// When using segmented stacks these are lowered into instructions which first
+// check if the current stacklet has enough free memory. If it does, memory is
+// allocated by bumping the stack pointer. Otherwise memory is allocated from 
+// the heap.
+
+let Defs = [EAX, ESP, EFLAGS], Uses = [ESP, EAX] in
+def SEG_ALLOCA_32 : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$size),
+                      "# variable sized alloca for segmented stacks",
+                      [(set GR32:$dst,
+                         (X86SegAlloca GR32:$size))]>,
+                    Requires<[In32BitMode]>;
+
+let Defs = [RAX, RSP, EFLAGS], Uses = [RSP, RAX] in
+def SEG_ALLOCA_64 : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$size),
+                      "# variable sized alloca for segmented stacks",
+                      [(set GR64:$dst,
+                         (X86SegAlloca GR64:$size))]>,
+                    Requires<[In64BitMode]>;
+
 }
 
 
@@ -329,18 +349,11 @@ def TLSCall_64 : I<0, Pseudo, (outs), (ins i64mem:$sym),
 //===----------------------------------------------------------------------===//
 // Conditional Move Pseudo Instructions
 
-let Constraints = "$src1 = $dst" in {
-
-// Conditional moves
-let Uses = [EFLAGS] in {
-
 // X86 doesn't have 8-bit conditional moves. Use a customInserter to
 // emit control flow. An alternative to this is to mark i8 SELECT as Promote,
 // however that requires promoting the operands, and can induce additional
-// i8 register pressure. Note that CMOV_GR8 is conservatively considered to
-// clobber EFLAGS, because if one of the operands is zero, the expansion
-// could involve an xor.
-let usesCustomInserter = 1, Constraints = "", Defs = [EFLAGS] in {
+// i8 register pressure.
+let usesCustomInserter = 1, Uses = [EFLAGS] in {
 def CMOV_GR8 : I<0, Pseudo,
                  (outs GR8:$dst), (ins GR8:$src1, GR8:$src2, i8imm:$cond),
                  "#CMOV_GR8 PSEUDO!",
@@ -380,10 +393,7 @@ def CMOV_RFP80 : I<0, Pseudo,
                       (X86cmov RFP80:$src1, RFP80:$src2, imm:$cond,
                                                   EFLAGS))]>;
 } // Predicates = [NoCMov]
-} // UsesCustomInserter = 1, Constraints = "", Defs = [EFLAGS]
-} // Uses = [EFLAGS]
-
-} // Constraints = "$src1 = $dst" in
+} // UsesCustomInserter = 1, Uses = [EFLAGS]
 
 
 //===----------------------------------------------------------------------===//
@@ -532,7 +542,7 @@ def OR32mrLocked  : I<0x09, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$zero),
 let hasSideEffects = 1 in
 def Int_MemBarrier : I<0, Pseudo, (outs), (ins),
                      "#MEMBARRIER",
-                     [(X86MemBarrier)]>, Requires<[HasSSE2]>;
+                     [(X86MemBarrier)]>;
 
 // TODO: Get this to fold the constant into the instruction.
 let hasSideEffects = 1, Defs = [ESP], isCodeGenOnly = 1 in
@@ -630,8 +640,8 @@ def #NAME#64mi8 : RIi8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4},
 defm LOCK_ADD : LOCK_ArithBinOp<0x00, 0x80, 0x83, MRM0m, "add">;
 defm LOCK_SUB : LOCK_ArithBinOp<0x28, 0x80, 0x83, MRM5m, "sub">;
 defm LOCK_OR  : LOCK_ArithBinOp<0x08, 0x80, 0x83, MRM1m, "or">;
-defm LOCK_AND : LOCK_ArithBinOp<0x08, 0x80, 0x83, MRM4m, "and">;
-defm LOCK_XOR : LOCK_ArithBinOp<0x08, 0x80, 0x83, MRM6m, "xor">;
+defm LOCK_AND : LOCK_ArithBinOp<0x20, 0x80, 0x83, MRM4m, "and">;
+defm LOCK_XOR : LOCK_ArithBinOp<0x30, 0x80, 0x83, MRM6m, "xor">;
 
 // Optimized codegen when the non-memory output is not used.
 let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1 in {
@@ -665,12 +675,20 @@ def LOCK_DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst),
 
 // Atomic compare and swap.
 let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX],
-    isCodeGenOnly = 1 in {
+    isCodeGenOnly = 1 in
 def LCMPXCHG8B : I<0xC7, MRM1m, (outs), (ins i64mem:$ptr),
                "lock\n\t"
                "cmpxchg8b\t$ptr",
                [(X86cas8 addr:$ptr)]>, TB, LOCK;
-}
+
+let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RBX, RCX, RDX],
+    isCodeGenOnly = 1 in
+def LCMPXCHG16B : RI<0xC7, MRM1m, (outs), (ins i128mem:$ptr),
+                    "lock\n\t"
+                    "cmpxchg16b\t$ptr",
+                    [(X86cas16 addr:$ptr)]>, TB, LOCK,
+                    Requires<[HasCmpxchg16b]>;
+
 let Defs = [AL, EFLAGS], Uses = [AL], isCodeGenOnly = 1 in {
 def LCMPXCHG8 : I<0xB0, MRMDestMem, (outs), (ins i8mem:$ptr, GR8:$swap),
                "lock\n\t"
@@ -695,7 +713,7 @@ def LCMPXCHG32 : I<0xB1, MRMDestMem, (outs), (ins i32mem:$ptr, GR32:$swap),
 let Defs = [RAX, EFLAGS], Uses = [RAX], isCodeGenOnly = 1 in {
 def LCMPXCHG64 : RI<0xB1, MRMDestMem, (outs), (ins i64mem:$ptr, GR64:$swap),
                "lock\n\t"
-               "cmpxchgq\t$swap,$ptr",
+               "cmpxchg{q}\t{$swap, $ptr|$ptr, $swap}",
                [(X86cas addr:$ptr, GR64:$swap, 8)]>, TB, LOCK;
 }
 
@@ -718,11 +736,37 @@ def LXADD32 : I<0xC1, MRMSrcMem, (outs GR32:$dst), (ins GR32:$val, i32mem:$ptr),
                 TB, LOCK;
 def LXADD64 : RI<0xC1, MRMSrcMem, (outs GR64:$dst), (ins GR64:$val,i64mem:$ptr),
                "lock\n\t"
-               "xadd\t$val, $ptr",
+               "xadd{q}\t{$val, $ptr|$ptr, $val}",
                [(set GR64:$dst, (atomic_load_add_64 addr:$ptr, GR64:$val))]>,
                 TB, LOCK;
 }
 
+def ACQUIRE_MOV8rm  : I<0, Pseudo, (outs GR8 :$dst), (ins i8mem :$src),
+                      "#ACQUIRE_MOV PSEUDO!",
+                      [(set GR8:$dst,  (atomic_load_8  addr:$src))]>;
+def ACQUIRE_MOV16rm : I<0, Pseudo, (outs GR16:$dst), (ins i16mem:$src),
+                      "#ACQUIRE_MOV PSEUDO!",
+                      [(set GR16:$dst, (atomic_load_16 addr:$src))]>;
+def ACQUIRE_MOV32rm : I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$src),
+                      "#ACQUIRE_MOV PSEUDO!",
+                      [(set GR32:$dst, (atomic_load_32 addr:$src))]>;
+def ACQUIRE_MOV64rm : I<0, Pseudo, (outs GR64:$dst), (ins i64mem:$src),
+                      "#ACQUIRE_MOV PSEUDO!",
+                      [(set GR64:$dst, (atomic_load_64 addr:$src))]>;
+
+def RELEASE_MOV8mr  : I<0, Pseudo, (outs), (ins i8mem :$dst, GR8 :$src),
+                        "#RELEASE_MOV PSEUDO!",
+                        [(atomic_store_8  addr:$dst, GR8 :$src)]>;
+def RELEASE_MOV16mr : I<0, Pseudo, (outs), (ins i16mem:$dst, GR16:$src),
+                        "#RELEASE_MOV PSEUDO!",
+                        [(atomic_store_16 addr:$dst, GR16:$src)]>;
+def RELEASE_MOV32mr : I<0, Pseudo, (outs), (ins i32mem:$dst, GR32:$src),
+                        "#RELEASE_MOV PSEUDO!",
+                        [(atomic_store_32 addr:$dst, GR32:$src)]>;
+def RELEASE_MOV64mr : I<0, Pseudo, (outs), (ins i64mem:$dst, GR64:$src),
+                        "#RELEASE_MOV PSEUDO!",
+                        [(atomic_store_64 addr:$dst, GR64:$src)]>;
+
 //===----------------------------------------------------------------------===//
 // Conditional Move Pseudo Instructions.
 //===----------------------------------------------------------------------===//
@@ -759,6 +803,24 @@ let Uses = [EFLAGS], usesCustomInserter = 1 in {
                     [(set VR128:$dst,
                       (v2i64 (X86cmov VR128:$t, VR128:$f, imm:$cond,
                                           EFLAGS)))]>;
+  def CMOV_V8F32 : I<0, Pseudo,
+                    (outs VR256:$dst), (ins VR256:$t, VR256:$f, i8imm:$cond),
+                    "#CMOV_V8F32 PSEUDO!",
+                    [(set VR256:$dst,
+                      (v8f32 (X86cmov VR256:$t, VR256:$f, imm:$cond,
+                                          EFLAGS)))]>;
+  def CMOV_V4F64 : I<0, Pseudo,
+                    (outs VR256:$dst), (ins VR256:$t, VR256:$f, i8imm:$cond),
+                    "#CMOV_V4F64 PSEUDO!",
+                    [(set VR256:$dst,
+                      (v4f64 (X86cmov VR256:$t, VR256:$f, imm:$cond,
+                                          EFLAGS)))]>;
+  def CMOV_V4I64 : I<0, Pseudo,
+                    (outs VR256:$dst), (ins VR256:$t, VR256:$f, i8imm:$cond),
+                    "#CMOV_V4I64 PSEUDO!",
+                    [(set VR256:$dst,
+                      (v4i64 (X86cmov VR256:$t, VR256:$f, imm:$cond,
+                                          EFLAGS)))]>;
 }
 
 
diff --git a/contrib/llvm/lib/Target/X86/X86InstrExtension.td b/contrib/llvm/lib/Target/X86/X86InstrExtension.td
index 2e1d523ea16d..e62e6b701f46 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrExtension.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrExtension.td
@@ -76,12 +76,12 @@ def MOVZX32rm16: I<0xB7, MRMSrcMem, (outs GR32:$dst), (ins i16mem:$src),
 // except that they use GR32_NOREX for the output operand register class
 // instead of GR32. This allows them to operate on h registers on x86-64.
 def MOVZX32_NOREXrr8 : I<0xB6, MRMSrcReg,
-                         (outs GR32_NOREX:$dst), (ins GR8:$src),
+                         (outs GR32_NOREX:$dst), (ins GR8_NOREX:$src),
                          "movz{bl|x}\t{$src, $dst|$dst, $src}",
                          []>, TB;
 let mayLoad = 1 in
 def MOVZX32_NOREXrm8 : I<0xB6, MRMSrcMem,
-                         (outs GR32_NOREX:$dst), (ins i8mem:$src),
+                         (outs GR32_NOREX:$dst), (ins i8mem_NOREX:$src),
                          "movz{bl|x}\t{$src, $dst|$dst, $src}",
                          []>, TB;
 
diff --git a/contrib/llvm/lib/Target/X86/X86InstrFormats.td b/contrib/llvm/lib/Target/X86/X86InstrFormats.td
index 6d89bcc29e7b..0a1590b3e0a2 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrFormats.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrFormats.td
@@ -113,6 +113,7 @@ class VEX_W  { bit hasVEX_WPrefix = 1; }
 class VEX_4V : VEX { bit hasVEX_4VPrefix = 1; }
 class VEX_I8IMM { bit hasVEX_i8ImmReg = 1; }
 class VEX_L  { bit hasVEX_L = 1; }
+class VEX_LIG { bit ignoresVEX_L = 1; }
 class Has3DNow0F0FOpcode  { bit has3DNow0F0FOpcode = 1; }
 
 class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins,
@@ -150,6 +151,7 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins,
   bit hasVEX_i8ImmReg = 0;  // Does this inst require the last source register
                             // to be encoded in a immediate field?
   bit hasVEX_L = 0;         // Does this inst use large (256-bit) registers?
+  bit ignoresVEX_L = 0;     // Does this instruction ignore the L-bit
   bit has3DNow0F0FOpcode =0;// Wacky 3dNow! encoding?
 
   // TSFlags layout should be kept in sync with X86InstrInfo.h.
@@ -169,7 +171,8 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins,
   let TSFlags{35}    = hasVEX_4VPrefix;
   let TSFlags{36}    = hasVEX_i8ImmReg;
   let TSFlags{37}    = hasVEX_L;
-  let TSFlags{38}    = has3DNow0F0FOpcode;
+  let TSFlags{38}    = ignoresVEX_L;
+  let TSFlags{39}    = has3DNow0F0FOpcode;
 }
 
 class PseudoI<dag oops, dag iops, list<dag> pattern>
@@ -501,6 +504,9 @@ class RSDI<bits<8> o, Format F, dag outs, dag ins, string asm,
 class RPDI<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern>
       : PDI<o, F, outs, ins, asm, pattern>, REX_W;
+class VRPDI<bits<8> o, Format F, dag outs, dag ins, string asm,
+           list<dag> pattern>
+      : VPDI<o, F, outs, ins, asm, pattern>, VEX_W;
 
 // MMX Instruction templates
 //
diff --git a/contrib/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/contrib/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index b00109c9fa4d..af919fba8ee4 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -39,6 +39,8 @@ def X86frsqrt  : SDNode<"X86ISD::FRSQRT",    SDTFPUnaryOp>;
 def X86frcp    : SDNode<"X86ISD::FRCP",      SDTFPUnaryOp>;
 def X86fsrl    : SDNode<"X86ISD::FSRL",      SDTX86FPShiftOp>;
 def X86fgetsign: SDNode<"X86ISD::FGETSIGNx86",SDTFPToIntOp>;
+def X86fhadd   : SDNode<"X86ISD::FHADD",     SDTFPBinOp>;
+def X86fhsub   : SDNode<"X86ISD::FHSUB",     SDTFPBinOp>;
 def X86comi    : SDNode<"X86ISD::COMI",      SDTX86CmpTest>;
 def X86ucomi   : SDNode<"X86ISD::UCOMI",     SDTX86CmpTest>;
 def X86cmpss   : SDNode<"X86ISD::FSETCCss",    SDTX86Cmpss>;
@@ -49,18 +51,15 @@ def X86pshufb  : SDNode<"X86ISD::PSHUFB",
 def X86andnp   : SDNode<"X86ISD::ANDNP",
                  SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
-def X86psignb  : SDNode<"X86ISD::PSIGNB", 
+def X86psignb  : SDNode<"X86ISD::PSIGNB",
                  SDTypeProfile<1, 2, [SDTCisVT<0, v16i8>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
-def X86psignw  : SDNode<"X86ISD::PSIGNW", 
+def X86psignw  : SDNode<"X86ISD::PSIGNW",
                  SDTypeProfile<1, 2, [SDTCisVT<0, v8i16>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
-def X86psignd  : SDNode<"X86ISD::PSIGND", 
+def X86psignd  : SDNode<"X86ISD::PSIGND",
                  SDTypeProfile<1, 2, [SDTCisVT<0, v4i32>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
-def X86pblendv : SDNode<"X86ISD::PBLENDVB", 
-                 SDTypeProfile<1, 3, [SDTCisVT<0, v16i8>, SDTCisSameAs<0,1>,
-                                      SDTCisSameAs<0,2>, SDTCisSameAs<0,3>]>>;
 def X86pextrb  : SDNode<"X86ISD::PEXTRB",
                  SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<2>]>>;
 def X86pextrw  : SDNode<"X86ISD::PEXTRW",
@@ -109,6 +108,8 @@ def SDTShuff2OpI : SDTypeProfile<1, 2, [SDTCisVec<0>,
 def SDTShuff3OpI : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                  SDTCisSameAs<0,2>, SDTCisInt<3>]>;
 
+def SDTVBroadcast : SDTypeProfile<1, 1, [SDTCisVec<0>]>;
+
 def X86PAlign : SDNode<"X86ISD::PALIGN", SDTShuff3OpI>;
 
 def X86PShufd  : SDNode<"X86ISD::PSHUFD", SDTShuff2OpI>;
@@ -133,12 +134,15 @@ def X86Movhlpd : SDNode<"X86ISD::MOVHLPD", SDTShuff2Op>;
 def X86Movlps : SDNode<"X86ISD::MOVLPS", SDTShuff2Op>;
 def X86Movlpd : SDNode<"X86ISD::MOVLPD", SDTShuff2Op>;
 
-def X86Unpcklps : SDNode<"X86ISD::UNPCKLPS", SDTShuff2Op>;
-def X86Unpcklpd : SDNode<"X86ISD::UNPCKLPD", SDTShuff2Op>;
+def X86Unpcklps  : SDNode<"X86ISD::UNPCKLPS", SDTShuff2Op>;
+def X86Unpcklpd  : SDNode<"X86ISD::UNPCKLPD", SDTShuff2Op>;
 def X86Unpcklpsy : SDNode<"X86ISD::VUNPCKLPSY", SDTShuff2Op>;
 def X86Unpcklpdy : SDNode<"X86ISD::VUNPCKLPDY", SDTShuff2Op>;
-def X86Unpckhps : SDNode<"X86ISD::UNPCKHPS", SDTShuff2Op>;
-def X86Unpckhpd : SDNode<"X86ISD::UNPCKHPD", SDTShuff2Op>;
+
+def X86Unpckhps  : SDNode<"X86ISD::UNPCKHPS", SDTShuff2Op>;
+def X86Unpckhpd  : SDNode<"X86ISD::UNPCKHPD", SDTShuff2Op>;
+def X86Unpckhpsy : SDNode<"X86ISD::VUNPCKHPSY", SDTShuff2Op>;
+def X86Unpckhpdy : SDNode<"X86ISD::VUNPCKHPDY", SDTShuff2Op>;
 
 def X86Punpcklbw  : SDNode<"X86ISD::PUNPCKLBW", SDTShuff2Op>;
 def X86Punpcklwd  : SDNode<"X86ISD::PUNPCKLWD", SDTShuff2Op>;
@@ -150,6 +154,15 @@ def X86Punpckhwd  : SDNode<"X86ISD::PUNPCKHWD", SDTShuff2Op>;
 def X86Punpckhdq  : SDNode<"X86ISD::PUNPCKHDQ", SDTShuff2Op>;
 def X86Punpckhqdq : SDNode<"X86ISD::PUNPCKHQDQ", SDTShuff2Op>;
 
+def X86VPermilps  : SDNode<"X86ISD::VPERMILPS", SDTShuff2OpI>;
+def X86VPermilpsy : SDNode<"X86ISD::VPERMILPSY", SDTShuff2OpI>;
+def X86VPermilpd  : SDNode<"X86ISD::VPERMILPD", SDTShuff2OpI>;
+def X86VPermilpdy : SDNode<"X86ISD::VPERMILPDY", SDTShuff2OpI>;
+
+def X86VPerm2f128 : SDNode<"X86ISD::VPERM2F128", SDTShuff3OpI>;
+
+def X86VBroadcast : SDNode<"X86ISD::VBROADCAST", SDTVBroadcast>;
+
 //===----------------------------------------------------------------------===//
 // SSE Complex Patterns
 //===----------------------------------------------------------------------===//
@@ -193,17 +206,28 @@ def loadv4f64    : PatFrag<(ops node:$ptr), (v4f64 (load node:$ptr))>;
 def loadv8i32    : PatFrag<(ops node:$ptr), (v8i32 (load node:$ptr))>;
 def loadv4i64    : PatFrag<(ops node:$ptr), (v4i64 (load node:$ptr))>;
 
-// Like 'store', but always requires vector alignment.
+// Like 'store', but always requires 128-bit vector alignment.
 def alignedstore : PatFrag<(ops node:$val, node:$ptr),
                            (store node:$val, node:$ptr), [{
   return cast<StoreSDNode>(N)->getAlignment() >= 16;
 }]>;
 
-// Like 'load', but always requires vector alignment.
+// Like 'store', but always requires 256-bit vector alignment.
+def alignedstore256 : PatFrag<(ops node:$val, node:$ptr),
+                              (store node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getAlignment() >= 32;
+}]>;
+
+// Like 'load', but always requires 128-bit vector alignment.
 def alignedload : PatFrag<(ops node:$ptr), (load node:$ptr), [{
   return cast<LoadSDNode>(N)->getAlignment() >= 16;
 }]>;
 
+// Like 'load', but always requires 256-bit vector alignment.
+def alignedload256 : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  return cast<LoadSDNode>(N)->getAlignment() >= 32;
+}]>;
+
 def alignedloadfsf32 : PatFrag<(ops node:$ptr),
                                (f32 (alignedload node:$ptr))>;
 def alignedloadfsf64 : PatFrag<(ops node:$ptr),
@@ -221,13 +245,13 @@ def alignedloadv2i64 : PatFrag<(ops node:$ptr),
 
 // 256-bit aligned load pattern fragments
 def alignedloadv8f32 : PatFrag<(ops node:$ptr),
-                               (v8f32 (alignedload node:$ptr))>;
+                               (v8f32 (alignedload256 node:$ptr))>;
 def alignedloadv4f64 : PatFrag<(ops node:$ptr),
-                               (v4f64 (alignedload node:$ptr))>;
+                               (v4f64 (alignedload256 node:$ptr))>;
 def alignedloadv8i32 : PatFrag<(ops node:$ptr),
-                               (v8i32 (alignedload node:$ptr))>;
+                               (v8i32 (alignedload256 node:$ptr))>;
 def alignedloadv4i64 : PatFrag<(ops node:$ptr),
-                               (v4i64 (alignedload node:$ptr))>;
+                               (v4i64 (alignedload256 node:$ptr))>;
 
 // Like 'load', but uses special alignment checks suitable for use in
 // memory operands in most SSE instructions, which are required to
@@ -356,7 +380,7 @@ def EXTRACT_get_vextractf128_imm : SDNodeXForm<extract_subvector, [{
   return getI8Imm(X86::getExtractVEXTRACTF128Immediate(N));
 }]>;
 
-// INSERT_get_vinsertf128_imm xform function: convert insert_subvector index to 
+// INSERT_get_vinsertf128_imm xform function: convert insert_subvector index to
 // VINSERTF128 imm.
 def INSERT_get_vinsertf128_imm : SDNodeXForm<insert_subvector, [{
   return getI8Imm(X86::getInsertVINSERTF128Immediate(N));
@@ -398,16 +422,6 @@ def movl : PatFrag<(ops node:$lhs, node:$rhs),
   return X86::isMOVLMask(cast<ShuffleVectorSDNode>(N));
 }]>;
 
-def movshdup : PatFrag<(ops node:$lhs, node:$rhs),
-                       (vector_shuffle node:$lhs, node:$rhs), [{
-  return X86::isMOVSHDUPMask(cast<ShuffleVectorSDNode>(N));
-}]>;
-
-def movsldup : PatFrag<(ops node:$lhs, node:$rhs),
-                       (vector_shuffle node:$lhs, node:$rhs), [{
-  return X86::isMOVSLDUPMask(cast<ShuffleVectorSDNode>(N));
-}]>;
-
 def unpckl : PatFrag<(ops node:$lhs, node:$rhs),
                      (vector_shuffle node:$lhs, node:$rhs), [{
   return X86::isUNPCKLMask(cast<ShuffleVectorSDNode>(N));
@@ -418,16 +432,6 @@ def unpckh : PatFrag<(ops node:$lhs, node:$rhs),
   return X86::isUNPCKHMask(cast<ShuffleVectorSDNode>(N));
 }]>;
 
-def unpckl_undef : PatFrag<(ops node:$lhs, node:$rhs),
-                           (vector_shuffle node:$lhs, node:$rhs), [{
-  return X86::isUNPCKL_v_undef_Mask(cast<ShuffleVectorSDNode>(N));
-}]>;
-
-def unpckh_undef : PatFrag<(ops node:$lhs, node:$rhs),
-                           (vector_shuffle node:$lhs, node:$rhs), [{
-  return X86::isUNPCKH_v_undef_Mask(cast<ShuffleVectorSDNode>(N));
-}]>;
-
 def pshufd : PatFrag<(ops node:$lhs, node:$rhs),
                      (vector_shuffle node:$lhs, node:$rhs), [{
   return X86::isPSHUFDMask(cast<ShuffleVectorSDNode>(N));
@@ -448,11 +452,6 @@ def pshuflw : PatFrag<(ops node:$lhs, node:$rhs),
   return X86::isPSHUFLWMask(cast<ShuffleVectorSDNode>(N));
 }], SHUFFLE_get_pshuflw_imm>;
 
-def palign : PatFrag<(ops node:$lhs, node:$rhs),
-                     (vector_shuffle node:$lhs, node:$rhs), [{
-  return X86::isPALIGNRMask(cast<ShuffleVectorSDNode>(N));
-}], SHUFFLE_get_palign_imm>;
-
 def vextractf128_extract : PatFrag<(ops node:$bigvec, node:$index),
                                    (extract_subvector node:$bigvec,
                                                       node:$index), [{
@@ -465,3 +464,4 @@ def vinsertf128_insert : PatFrag<(ops node:$bigvec, node:$smallvec,
                                                    node:$index), [{
   return X86::isVINSERTF128Index(N);
 }], INSERT_get_vinsertf128_imm>;
+
diff --git a/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp b/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp
index 55b5835f52a7..3a02de0aa01b 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -53,6 +53,36 @@ ReMatPICStubLoad("remat-pic-stub-load",
                  cl::desc("Re-materialize load from stub in PIC mode"),
                  cl::init(false), cl::Hidden);
 
+enum {
+  // Select which memory operand is being unfolded.
+  // (stored in bits 0 - 7)
+  TB_INDEX_0    = 0,
+  TB_INDEX_1    = 1,
+  TB_INDEX_2    = 2,
+  TB_INDEX_MASK = 0xff,
+
+  // Minimum alignment required for load/store.
+  // Used for RegOp->MemOp conversion.
+  // (stored in bits 8 - 15)
+  TB_ALIGN_SHIFT = 8,
+  TB_ALIGN_NONE  =    0 << TB_ALIGN_SHIFT,
+  TB_ALIGN_16    =   16 << TB_ALIGN_SHIFT,
+  TB_ALIGN_32    =   32 << TB_ALIGN_SHIFT,
+  TB_ALIGN_MASK  = 0xff << TB_ALIGN_SHIFT,
+
+  // Do not insert the reverse map (MemOp -> RegOp) into the table.
+  // This may be needed because there is a many -> one mapping.
+  TB_NO_REVERSE   = 1 << 16,
+
+  // Do not insert the forward map (RegOp -> MemOp) into the table.
+  // This is needed for Native Client, which prohibits branch
+  // instructions from using a memory operand.
+  TB_NO_FORWARD   = 1 << 17,
+
+  TB_FOLDED_LOAD  = 1 << 18,
+  TB_FOLDED_STORE = 1 << 19
+};
+
 X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
   : X86GenInstrInfo((tm.getSubtarget<X86Subtarget>().is64Bit()
                      ? X86::ADJCALLSTACKDOWN64
@@ -61,657 +91,831 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
                      ? X86::ADJCALLSTACKUP64
                      : X86::ADJCALLSTACKUP32)),
     TM(tm), RI(tm, *this) {
-  enum {
-    TB_NOT_REVERSABLE = 1U << 31,
-    TB_FLAGS = TB_NOT_REVERSABLE
-  };
 
-  static const unsigned OpTbl2Addr[][2] = {
-    { X86::ADC32ri,     X86::ADC32mi },
-    { X86::ADC32ri8,    X86::ADC32mi8 },
-    { X86::ADC32rr,     X86::ADC32mr },
-    { X86::ADC64ri32,   X86::ADC64mi32 },
-    { X86::ADC64ri8,    X86::ADC64mi8 },
-    { X86::ADC64rr,     X86::ADC64mr },
-    { X86::ADD16ri,     X86::ADD16mi },
-    { X86::ADD16ri8,    X86::ADD16mi8 },
-    { X86::ADD16ri_DB,  X86::ADD16mi  | TB_NOT_REVERSABLE },
-    { X86::ADD16ri8_DB, X86::ADD16mi8 | TB_NOT_REVERSABLE },
-    { X86::ADD16rr,     X86::ADD16mr },
-    { X86::ADD16rr_DB,  X86::ADD16mr | TB_NOT_REVERSABLE },
-    { X86::ADD32ri,     X86::ADD32mi },
-    { X86::ADD32ri8,    X86::ADD32mi8 },
-    { X86::ADD32ri_DB,  X86::ADD32mi | TB_NOT_REVERSABLE },
-    { X86::ADD32ri8_DB, X86::ADD32mi8 | TB_NOT_REVERSABLE },
-    { X86::ADD32rr,     X86::ADD32mr },
-    { X86::ADD32rr_DB,  X86::ADD32mr | TB_NOT_REVERSABLE },
-    { X86::ADD64ri32,   X86::ADD64mi32 },
-    { X86::ADD64ri8,    X86::ADD64mi8 },
-    { X86::ADD64ri32_DB,X86::ADD64mi32 | TB_NOT_REVERSABLE },
-    { X86::ADD64ri8_DB, X86::ADD64mi8 | TB_NOT_REVERSABLE },
-    { X86::ADD64rr,     X86::ADD64mr },
-    { X86::ADD64rr_DB,  X86::ADD64mr | TB_NOT_REVERSABLE },
-    { X86::ADD8ri,      X86::ADD8mi },
-    { X86::ADD8rr,      X86::ADD8mr },
-    { X86::AND16ri,     X86::AND16mi },
-    { X86::AND16ri8,    X86::AND16mi8 },
-    { X86::AND16rr,     X86::AND16mr },
-    { X86::AND32ri,     X86::AND32mi },
-    { X86::AND32ri8,    X86::AND32mi8 },
-    { X86::AND32rr,     X86::AND32mr },
-    { X86::AND64ri32,   X86::AND64mi32 },
-    { X86::AND64ri8,    X86::AND64mi8 },
-    { X86::AND64rr,     X86::AND64mr },
-    { X86::AND8ri,      X86::AND8mi },
-    { X86::AND8rr,      X86::AND8mr },
-    { X86::DEC16r,      X86::DEC16m },
-    { X86::DEC32r,      X86::DEC32m },
-    { X86::DEC64_16r,   X86::DEC64_16m },
-    { X86::DEC64_32r,   X86::DEC64_32m },
-    { X86::DEC64r,      X86::DEC64m },
-    { X86::DEC8r,       X86::DEC8m },
-    { X86::INC16r,      X86::INC16m },
-    { X86::INC32r,      X86::INC32m },
-    { X86::INC64_16r,   X86::INC64_16m },
-    { X86::INC64_32r,   X86::INC64_32m },
-    { X86::INC64r,      X86::INC64m },
-    { X86::INC8r,       X86::INC8m },
-    { X86::NEG16r,      X86::NEG16m },
-    { X86::NEG32r,      X86::NEG32m },
-    { X86::NEG64r,      X86::NEG64m },
-    { X86::NEG8r,       X86::NEG8m },
-    { X86::NOT16r,      X86::NOT16m },
-    { X86::NOT32r,      X86::NOT32m },
-    { X86::NOT64r,      X86::NOT64m },
-    { X86::NOT8r,       X86::NOT8m },
-    { X86::OR16ri,      X86::OR16mi },
-    { X86::OR16ri8,     X86::OR16mi8 },
-    { X86::OR16rr,      X86::OR16mr },
-    { X86::OR32ri,      X86::OR32mi },
-    { X86::OR32ri8,     X86::OR32mi8 },
-    { X86::OR32rr,      X86::OR32mr },
-    { X86::OR64ri32,    X86::OR64mi32 },
-    { X86::OR64ri8,     X86::OR64mi8 },
-    { X86::OR64rr,      X86::OR64mr },
-    { X86::OR8ri,       X86::OR8mi },
-    { X86::OR8rr,       X86::OR8mr },
-    { X86::ROL16r1,     X86::ROL16m1 },
-    { X86::ROL16rCL,    X86::ROL16mCL },
-    { X86::ROL16ri,     X86::ROL16mi },
-    { X86::ROL32r1,     X86::ROL32m1 },
-    { X86::ROL32rCL,    X86::ROL32mCL },
-    { X86::ROL32ri,     X86::ROL32mi },
-    { X86::ROL64r1,     X86::ROL64m1 },
-    { X86::ROL64rCL,    X86::ROL64mCL },
-    { X86::ROL64ri,     X86::ROL64mi },
-    { X86::ROL8r1,      X86::ROL8m1 },
-    { X86::ROL8rCL,     X86::ROL8mCL },
-    { X86::ROL8ri,      X86::ROL8mi },
-    { X86::ROR16r1,     X86::ROR16m1 },
-    { X86::ROR16rCL,    X86::ROR16mCL },
-    { X86::ROR16ri,     X86::ROR16mi },
-    { X86::ROR32r1,     X86::ROR32m1 },
-    { X86::ROR32rCL,    X86::ROR32mCL },
-    { X86::ROR32ri,     X86::ROR32mi },
-    { X86::ROR64r1,     X86::ROR64m1 },
-    { X86::ROR64rCL,    X86::ROR64mCL },
-    { X86::ROR64ri,     X86::ROR64mi },
-    { X86::ROR8r1,      X86::ROR8m1 },
-    { X86::ROR8rCL,     X86::ROR8mCL },
-    { X86::ROR8ri,      X86::ROR8mi },
-    { X86::SAR16r1,     X86::SAR16m1 },
-    { X86::SAR16rCL,    X86::SAR16mCL },
-    { X86::SAR16ri,     X86::SAR16mi },
-    { X86::SAR32r1,     X86::SAR32m1 },
-    { X86::SAR32rCL,    X86::SAR32mCL },
-    { X86::SAR32ri,     X86::SAR32mi },
-    { X86::SAR64r1,     X86::SAR64m1 },
-    { X86::SAR64rCL,    X86::SAR64mCL },
-    { X86::SAR64ri,     X86::SAR64mi },
-    { X86::SAR8r1,      X86::SAR8m1 },
-    { X86::SAR8rCL,     X86::SAR8mCL },
-    { X86::SAR8ri,      X86::SAR8mi },
-    { X86::SBB32ri,     X86::SBB32mi },
-    { X86::SBB32ri8,    X86::SBB32mi8 },
-    { X86::SBB32rr,     X86::SBB32mr },
-    { X86::SBB64ri32,   X86::SBB64mi32 },
-    { X86::SBB64ri8,    X86::SBB64mi8 },
-    { X86::SBB64rr,     X86::SBB64mr },
-    { X86::SHL16rCL,    X86::SHL16mCL },
-    { X86::SHL16ri,     X86::SHL16mi },
-    { X86::SHL32rCL,    X86::SHL32mCL },
-    { X86::SHL32ri,     X86::SHL32mi },
-    { X86::SHL64rCL,    X86::SHL64mCL },
-    { X86::SHL64ri,     X86::SHL64mi },
-    { X86::SHL8rCL,     X86::SHL8mCL },
-    { X86::SHL8ri,      X86::SHL8mi },
-    { X86::SHLD16rrCL,  X86::SHLD16mrCL },
-    { X86::SHLD16rri8,  X86::SHLD16mri8 },
-    { X86::SHLD32rrCL,  X86::SHLD32mrCL },
-    { X86::SHLD32rri8,  X86::SHLD32mri8 },
-    { X86::SHLD64rrCL,  X86::SHLD64mrCL },
-    { X86::SHLD64rri8,  X86::SHLD64mri8 },
-    { X86::SHR16r1,     X86::SHR16m1 },
-    { X86::SHR16rCL,    X86::SHR16mCL },
-    { X86::SHR16ri,     X86::SHR16mi },
-    { X86::SHR32r1,     X86::SHR32m1 },
-    { X86::SHR32rCL,    X86::SHR32mCL },
-    { X86::SHR32ri,     X86::SHR32mi },
-    { X86::SHR64r1,     X86::SHR64m1 },
-    { X86::SHR64rCL,    X86::SHR64mCL },
-    { X86::SHR64ri,     X86::SHR64mi },
-    { X86::SHR8r1,      X86::SHR8m1 },
-    { X86::SHR8rCL,     X86::SHR8mCL },
-    { X86::SHR8ri,      X86::SHR8mi },
-    { X86::SHRD16rrCL,  X86::SHRD16mrCL },
-    { X86::SHRD16rri8,  X86::SHRD16mri8 },
-    { X86::SHRD32rrCL,  X86::SHRD32mrCL },
-    { X86::SHRD32rri8,  X86::SHRD32mri8 },
-    { X86::SHRD64rrCL,  X86::SHRD64mrCL },
-    { X86::SHRD64rri8,  X86::SHRD64mri8 },
-    { X86::SUB16ri,     X86::SUB16mi },
-    { X86::SUB16ri8,    X86::SUB16mi8 },
-    { X86::SUB16rr,     X86::SUB16mr },
-    { X86::SUB32ri,     X86::SUB32mi },
-    { X86::SUB32ri8,    X86::SUB32mi8 },
-    { X86::SUB32rr,     X86::SUB32mr },
-    { X86::SUB64ri32,   X86::SUB64mi32 },
-    { X86::SUB64ri8,    X86::SUB64mi8 },
-    { X86::SUB64rr,     X86::SUB64mr },
-    { X86::SUB8ri,      X86::SUB8mi },
-    { X86::SUB8rr,      X86::SUB8mr },
-    { X86::XOR16ri,     X86::XOR16mi },
-    { X86::XOR16ri8,    X86::XOR16mi8 },
-    { X86::XOR16rr,     X86::XOR16mr },
-    { X86::XOR32ri,     X86::XOR32mi },
-    { X86::XOR32ri8,    X86::XOR32mi8 },
-    { X86::XOR32rr,     X86::XOR32mr },
-    { X86::XOR64ri32,   X86::XOR64mi32 },
-    { X86::XOR64ri8,    X86::XOR64mi8 },
-    { X86::XOR64rr,     X86::XOR64mr },
-    { X86::XOR8ri,      X86::XOR8mi },
-    { X86::XOR8rr,      X86::XOR8mr }
+  static const unsigned OpTbl2Addr[][3] = {
+    { X86::ADC32ri,     X86::ADC32mi,    0 },
+    { X86::ADC32ri8,    X86::ADC32mi8,   0 },
+    { X86::ADC32rr,     X86::ADC32mr,    0 },
+    { X86::ADC64ri32,   X86::ADC64mi32,  0 },
+    { X86::ADC64ri8,    X86::ADC64mi8,   0 },
+    { X86::ADC64rr,     X86::ADC64mr,    0 },
+    { X86::ADD16ri,     X86::ADD16mi,    0 },
+    { X86::ADD16ri8,    X86::ADD16mi8,   0 },
+    { X86::ADD16ri_DB,  X86::ADD16mi,    TB_NO_REVERSE },
+    { X86::ADD16ri8_DB, X86::ADD16mi8,   TB_NO_REVERSE },
+    { X86::ADD16rr,     X86::ADD16mr,    0 },
+    { X86::ADD16rr_DB,  X86::ADD16mr,    TB_NO_REVERSE },
+    { X86::ADD32ri,     X86::ADD32mi,    0 },
+    { X86::ADD32ri8,    X86::ADD32mi8,   0 },
+    { X86::ADD32ri_DB,  X86::ADD32mi,    TB_NO_REVERSE },
+    { X86::ADD32ri8_DB, X86::ADD32mi8,   TB_NO_REVERSE },
+    { X86::ADD32rr,     X86::ADD32mr,    0 },
+    { X86::ADD32rr_DB,  X86::ADD32mr,    TB_NO_REVERSE },
+    { X86::ADD64ri32,   X86::ADD64mi32,  0 },
+    { X86::ADD64ri8,    X86::ADD64mi8,   0 },
+    { X86::ADD64ri32_DB,X86::ADD64mi32,  TB_NO_REVERSE },
+    { X86::ADD64ri8_DB, X86::ADD64mi8,   TB_NO_REVERSE },
+    { X86::ADD64rr,     X86::ADD64mr,    0 },
+    { X86::ADD64rr_DB,  X86::ADD64mr,    TB_NO_REVERSE },
+    { X86::ADD8ri,      X86::ADD8mi,     0 },
+    { X86::ADD8rr,      X86::ADD8mr,     0 },
+    { X86::AND16ri,     X86::AND16mi,    0 },
+    { X86::AND16ri8,    X86::AND16mi8,   0 },
+    { X86::AND16rr,     X86::AND16mr,    0 },
+    { X86::AND32ri,     X86::AND32mi,    0 },
+    { X86::AND32ri8,    X86::AND32mi8,   0 },
+    { X86::AND32rr,     X86::AND32mr,    0 },
+    { X86::AND64ri32,   X86::AND64mi32,  0 },
+    { X86::AND64ri8,    X86::AND64mi8,   0 },
+    { X86::AND64rr,     X86::AND64mr,    0 },
+    { X86::AND8ri,      X86::AND8mi,     0 },
+    { X86::AND8rr,      X86::AND8mr,     0 },
+    { X86::DEC16r,      X86::DEC16m,     0 },
+    { X86::DEC32r,      X86::DEC32m,     0 },
+    { X86::DEC64_16r,   X86::DEC64_16m,  0 },
+    { X86::DEC64_32r,   X86::DEC64_32m,  0 },
+    { X86::DEC64r,      X86::DEC64m,     0 },
+    { X86::DEC8r,       X86::DEC8m,      0 },
+    { X86::INC16r,      X86::INC16m,     0 },
+    { X86::INC32r,      X86::INC32m,     0 },
+    { X86::INC64_16r,   X86::INC64_16m,  0 },
+    { X86::INC64_32r,   X86::INC64_32m,  0 },
+    { X86::INC64r,      X86::INC64m,     0 },
+    { X86::INC8r,       X86::INC8m,      0 },
+    { X86::NEG16r,      X86::NEG16m,     0 },
+    { X86::NEG32r,      X86::NEG32m,     0 },
+    { X86::NEG64r,      X86::NEG64m,     0 },
+    { X86::NEG8r,       X86::NEG8m,      0 },
+    { X86::NOT16r,      X86::NOT16m,     0 },
+    { X86::NOT32r,      X86::NOT32m,     0 },
+    { X86::NOT64r,      X86::NOT64m,     0 },
+    { X86::NOT8r,       X86::NOT8m,      0 },
+    { X86::OR16ri,      X86::OR16mi,     0 },
+    { X86::OR16ri8,     X86::OR16mi8,    0 },
+    { X86::OR16rr,      X86::OR16mr,     0 },
+    { X86::OR32ri,      X86::OR32mi,     0 },
+    { X86::OR32ri8,     X86::OR32mi8,    0 },
+    { X86::OR32rr,      X86::OR32mr,     0 },
+    { X86::OR64ri32,    X86::OR64mi32,   0 },
+    { X86::OR64ri8,     X86::OR64mi8,    0 },
+    { X86::OR64rr,      X86::OR64mr,     0 },
+    { X86::OR8ri,       X86::OR8mi,      0 },
+    { X86::OR8rr,       X86::OR8mr,      0 },
+    { X86::ROL16r1,     X86::ROL16m1,    0 },
+    { X86::ROL16rCL,    X86::ROL16mCL,   0 },
+    { X86::ROL16ri,     X86::ROL16mi,    0 },
+    { X86::ROL32r1,     X86::ROL32m1,    0 },
+    { X86::ROL32rCL,    X86::ROL32mCL,   0 },
+    { X86::ROL32ri,     X86::ROL32mi,    0 },
+    { X86::ROL64r1,     X86::ROL64m1,    0 },
+    { X86::ROL64rCL,    X86::ROL64mCL,   0 },
+    { X86::ROL64ri,     X86::ROL64mi,    0 },
+    { X86::ROL8r1,      X86::ROL8m1,     0 },
+    { X86::ROL8rCL,     X86::ROL8mCL,    0 },
+    { X86::ROL8ri,      X86::ROL8mi,     0 },
+    { X86::ROR16r1,     X86::ROR16m1,    0 },
+    { X86::ROR16rCL,    X86::ROR16mCL,   0 },
+    { X86::ROR16ri,     X86::ROR16mi,    0 },
+    { X86::ROR32r1,     X86::ROR32m1,    0 },
+    { X86::ROR32rCL,    X86::ROR32mCL,   0 },
+    { X86::ROR32ri,     X86::ROR32mi,    0 },
+    { X86::ROR64r1,     X86::ROR64m1,    0 },
+    { X86::ROR64rCL,    X86::ROR64mCL,   0 },
+    { X86::ROR64ri,     X86::ROR64mi,    0 },
+    { X86::ROR8r1,      X86::ROR8m1,     0 },
+    { X86::ROR8rCL,     X86::ROR8mCL,    0 },
+    { X86::ROR8ri,      X86::ROR8mi,     0 },
+    { X86::SAR16r1,     X86::SAR16m1,    0 },
+    { X86::SAR16rCL,    X86::SAR16mCL,   0 },
+    { X86::SAR16ri,     X86::SAR16mi,    0 },
+    { X86::SAR32r1,     X86::SAR32m1,    0 },
+    { X86::SAR32rCL,    X86::SAR32mCL,   0 },
+    { X86::SAR32ri,     X86::SAR32mi,    0 },
+    { X86::SAR64r1,     X86::SAR64m1,    0 },
+    { X86::SAR64rCL,    X86::SAR64mCL,   0 },
+    { X86::SAR64ri,     X86::SAR64mi,    0 },
+    { X86::SAR8r1,      X86::SAR8m1,     0 },
+    { X86::SAR8rCL,     X86::SAR8mCL,    0 },
+    { X86::SAR8ri,      X86::SAR8mi,     0 },
+    { X86::SBB32ri,     X86::SBB32mi,    0 },
+    { X86::SBB32ri8,    X86::SBB32mi8,   0 },
+    { X86::SBB32rr,     X86::SBB32mr,    0 },
+    { X86::SBB64ri32,   X86::SBB64mi32,  0 },
+    { X86::SBB64ri8,    X86::SBB64mi8,   0 },
+    { X86::SBB64rr,     X86::SBB64mr,    0 },
+    { X86::SHL16rCL,    X86::SHL16mCL,   0 },
+    { X86::SHL16ri,     X86::SHL16mi,    0 },
+    { X86::SHL32rCL,    X86::SHL32mCL,   0 },
+    { X86::SHL32ri,     X86::SHL32mi,    0 },
+    { X86::SHL64rCL,    X86::SHL64mCL,   0 },
+    { X86::SHL64ri,     X86::SHL64mi,    0 },
+    { X86::SHL8rCL,     X86::SHL8mCL,    0 },
+    { X86::SHL8ri,      X86::SHL8mi,     0 },
+    { X86::SHLD16rrCL,  X86::SHLD16mrCL, 0 },
+    { X86::SHLD16rri8,  X86::SHLD16mri8, 0 },
+    { X86::SHLD32rrCL,  X86::SHLD32mrCL, 0 },
+    { X86::SHLD32rri8,  X86::SHLD32mri8, 0 },
+    { X86::SHLD64rrCL,  X86::SHLD64mrCL, 0 },
+    { X86::SHLD64rri8,  X86::SHLD64mri8, 0 },
+    { X86::SHR16r1,     X86::SHR16m1,    0 },
+    { X86::SHR16rCL,    X86::SHR16mCL,   0 },
+    { X86::SHR16ri,     X86::SHR16mi,    0 },
+    { X86::SHR32r1,     X86::SHR32m1,    0 },
+    { X86::SHR32rCL,    X86::SHR32mCL,   0 },
+    { X86::SHR32ri,     X86::SHR32mi,    0 },
+    { X86::SHR64r1,     X86::SHR64m1,    0 },
+    { X86::SHR64rCL,    X86::SHR64mCL,   0 },
+    { X86::SHR64ri,     X86::SHR64mi,    0 },
+    { X86::SHR8r1,      X86::SHR8m1,     0 },
+    { X86::SHR8rCL,     X86::SHR8mCL,    0 },
+    { X86::SHR8ri,      X86::SHR8mi,     0 },
+    { X86::SHRD16rrCL,  X86::SHRD16mrCL, 0 },
+    { X86::SHRD16rri8,  X86::SHRD16mri8, 0 },
+    { X86::SHRD32rrCL,  X86::SHRD32mrCL, 0 },
+    { X86::SHRD32rri8,  X86::SHRD32mri8, 0 },
+    { X86::SHRD64rrCL,  X86::SHRD64mrCL, 0 },
+    { X86::SHRD64rri8,  X86::SHRD64mri8, 0 },
+    { X86::SUB16ri,     X86::SUB16mi,    0 },
+    { X86::SUB16ri8,    X86::SUB16mi8,   0 },
+    { X86::SUB16rr,     X86::SUB16mr,    0 },
+    { X86::SUB32ri,     X86::SUB32mi,    0 },
+    { X86::SUB32ri8,    X86::SUB32mi8,   0 },
+    { X86::SUB32rr,     X86::SUB32mr,    0 },
+    { X86::SUB64ri32,   X86::SUB64mi32,  0 },
+    { X86::SUB64ri8,    X86::SUB64mi8,   0 },
+    { X86::SUB64rr,     X86::SUB64mr,    0 },
+    { X86::SUB8ri,      X86::SUB8mi,     0 },
+    { X86::SUB8rr,      X86::SUB8mr,     0 },
+    { X86::XOR16ri,     X86::XOR16mi,    0 },
+    { X86::XOR16ri8,    X86::XOR16mi8,   0 },
+    { X86::XOR16rr,     X86::XOR16mr,    0 },
+    { X86::XOR32ri,     X86::XOR32mi,    0 },
+    { X86::XOR32ri8,    X86::XOR32mi8,   0 },
+    { X86::XOR32rr,     X86::XOR32mr,    0 },
+    { X86::XOR64ri32,   X86::XOR64mi32,  0 },
+    { X86::XOR64ri8,    X86::XOR64mi8,   0 },
+    { X86::XOR64rr,     X86::XOR64mr,    0 },
+    { X86::XOR8ri,      X86::XOR8mi,     0 },
+    { X86::XOR8rr,      X86::XOR8mr,     0 }
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl2Addr); i != e; ++i) {
     unsigned RegOp = OpTbl2Addr[i][0];
-    unsigned MemOp = OpTbl2Addr[i][1] & ~TB_FLAGS;
-    assert(!RegOp2MemOpTable2Addr.count(RegOp) && "Duplicated entries?");
-    RegOp2MemOpTable2Addr[RegOp] = std::make_pair(MemOp, 0U);
-
-    // If this is not a reversible operation (because there is a many->one)
-    // mapping, don't insert the reverse of the operation into MemOp2RegOpTable.
-    if (OpTbl2Addr[i][1] & TB_NOT_REVERSABLE)
-      continue;
-
-    // Index 0, folded load and store, no alignment requirement.
-    unsigned AuxInfo = 0 | (1 << 4) | (1 << 5);
-
-    assert(!MemOp2RegOpTable.count(MemOp) &&
-            "Duplicated entries in unfolding maps?");
-    MemOp2RegOpTable[MemOp] = std::make_pair(RegOp, AuxInfo);
+    unsigned MemOp = OpTbl2Addr[i][1];
+    unsigned Flags = OpTbl2Addr[i][2];
+    AddTableEntry(RegOp2MemOpTable2Addr, MemOp2RegOpTable,
+                  RegOp, MemOp,
+                  // Index 0, folded load and store, no alignment requirement.
+                  Flags | TB_INDEX_0 | TB_FOLDED_LOAD | TB_FOLDED_STORE);
   }
 
-  // If the third value is 1, then it's folding either a load or a store.
-  static const unsigned OpTbl0[][4] = {
-    { X86::BT16ri8,     X86::BT16mi8, 1, 0 },
-    { X86::BT32ri8,     X86::BT32mi8, 1, 0 },
-    { X86::BT64ri8,     X86::BT64mi8, 1, 0 },
-    { X86::CALL32r,     X86::CALL32m, 1, 0 },
-    { X86::CALL64r,     X86::CALL64m, 1, 0 },
-    { X86::WINCALL64r,  X86::WINCALL64m, 1, 0 },
-    { X86::CMP16ri,     X86::CMP16mi, 1, 0 },
-    { X86::CMP16ri8,    X86::CMP16mi8, 1, 0 },
-    { X86::CMP16rr,     X86::CMP16mr, 1, 0 },
-    { X86::CMP32ri,     X86::CMP32mi, 1, 0 },
-    { X86::CMP32ri8,    X86::CMP32mi8, 1, 0 },
-    { X86::CMP32rr,     X86::CMP32mr, 1, 0 },
-    { X86::CMP64ri32,   X86::CMP64mi32, 1, 0 },
-    { X86::CMP64ri8,    X86::CMP64mi8, 1, 0 },
-    { X86::CMP64rr,     X86::CMP64mr, 1, 0 },
-    { X86::CMP8ri,      X86::CMP8mi, 1, 0 },
-    { X86::CMP8rr,      X86::CMP8mr, 1, 0 },
-    { X86::DIV16r,      X86::DIV16m, 1, 0 },
-    { X86::DIV32r,      X86::DIV32m, 1, 0 },
-    { X86::DIV64r,      X86::DIV64m, 1, 0 },
-    { X86::DIV8r,       X86::DIV8m, 1, 0 },
-    { X86::EXTRACTPSrr, X86::EXTRACTPSmr, 0, 16 },
-    { X86::FsMOVAPDrr,  X86::MOVSDmr | TB_NOT_REVERSABLE , 0, 0 },
-    { X86::FsMOVAPSrr,  X86::MOVSSmr | TB_NOT_REVERSABLE , 0, 0 },
-    { X86::IDIV16r,     X86::IDIV16m, 1, 0 },
-    { X86::IDIV32r,     X86::IDIV32m, 1, 0 },
-    { X86::IDIV64r,     X86::IDIV64m, 1, 0 },
-    { X86::IDIV8r,      X86::IDIV8m, 1, 0 },
-    { X86::IMUL16r,     X86::IMUL16m, 1, 0 },
-    { X86::IMUL32r,     X86::IMUL32m, 1, 0 },
-    { X86::IMUL64r,     X86::IMUL64m, 1, 0 },
-    { X86::IMUL8r,      X86::IMUL8m, 1, 0 },
-    { X86::JMP32r,      X86::JMP32m, 1, 0 },
-    { X86::JMP64r,      X86::JMP64m, 1, 0 },
-    { X86::MOV16ri,     X86::MOV16mi, 0, 0 },
-    { X86::MOV16rr,     X86::MOV16mr, 0, 0 },
-    { X86::MOV32ri,     X86::MOV32mi, 0, 0 },
-    { X86::MOV32rr,     X86::MOV32mr, 0, 0 },
-    { X86::MOV64ri32,   X86::MOV64mi32, 0, 0 },
-    { X86::MOV64rr,     X86::MOV64mr, 0, 0 },
-    { X86::MOV8ri,      X86::MOV8mi, 0, 0 },
-    { X86::MOV8rr,      X86::MOV8mr, 0, 0 },
-    { X86::MOV8rr_NOREX, X86::MOV8mr_NOREX, 0, 0 },
-    { X86::MOVAPDrr,    X86::MOVAPDmr, 0, 16 },
-    { X86::MOVAPSrr,    X86::MOVAPSmr, 0, 16 },
-    { X86::MOVDQArr,    X86::MOVDQAmr, 0, 16 },
-    { X86::VMOVAPDYrr,  X86::VMOVAPDYmr, 0, 32 },
-    { X86::VMOVAPSYrr,  X86::VMOVAPSYmr, 0, 32 },
-    { X86::VMOVDQAYrr,  X86::VMOVDQAYmr, 0, 32 },
-    { X86::MOVPDI2DIrr, X86::MOVPDI2DImr, 0, 0 },
-    { X86::MOVPQIto64rr,X86::MOVPQI2QImr, 0, 0 },
-    { X86::MOVSDto64rr, X86::MOVSDto64mr, 0, 0 },
-    { X86::MOVSS2DIrr,  X86::MOVSS2DImr, 0, 0 },
-    { X86::MOVUPDrr,    X86::MOVUPDmr, 0, 0 },
-    { X86::MOVUPSrr,    X86::MOVUPSmr, 0, 0 },
-    { X86::VMOVUPDYrr,  X86::VMOVUPDYmr, 0, 0 },
-    { X86::VMOVUPSYrr,  X86::VMOVUPSYmr, 0, 0 },
-    { X86::MUL16r,      X86::MUL16m, 1, 0 },
-    { X86::MUL32r,      X86::MUL32m, 1, 0 },
-    { X86::MUL64r,      X86::MUL64m, 1, 0 },
-    { X86::MUL8r,       X86::MUL8m, 1, 0 },
-    { X86::SETAEr,      X86::SETAEm, 0, 0 },
-    { X86::SETAr,       X86::SETAm, 0, 0 },
-    { X86::SETBEr,      X86::SETBEm, 0, 0 },
-    { X86::SETBr,       X86::SETBm, 0, 0 },
-    { X86::SETEr,       X86::SETEm, 0, 0 },
-    { X86::SETGEr,      X86::SETGEm, 0, 0 },
-    { X86::SETGr,       X86::SETGm, 0, 0 },
-    { X86::SETLEr,      X86::SETLEm, 0, 0 },
-    { X86::SETLr,       X86::SETLm, 0, 0 },
-    { X86::SETNEr,      X86::SETNEm, 0, 0 },
-    { X86::SETNOr,      X86::SETNOm, 0, 0 },
-    { X86::SETNPr,      X86::SETNPm, 0, 0 },
-    { X86::SETNSr,      X86::SETNSm, 0, 0 },
-    { X86::SETOr,       X86::SETOm, 0, 0 },
-    { X86::SETPr,       X86::SETPm, 0, 0 },
-    { X86::SETSr,       X86::SETSm, 0, 0 },
-    { X86::TAILJMPr,    X86::TAILJMPm, 1, 0 },
-    { X86::TAILJMPr64,  X86::TAILJMPm64, 1, 0 },
-    { X86::TEST16ri,    X86::TEST16mi, 1, 0 },
-    { X86::TEST32ri,    X86::TEST32mi, 1, 0 },
-    { X86::TEST64ri32,  X86::TEST64mi32, 1, 0 },
-    { X86::TEST8ri,     X86::TEST8mi, 1, 0 }
+  static const unsigned OpTbl0[][3] = {
+    { X86::BT16ri8,     X86::BT16mi8,       TB_FOLDED_LOAD },
+    { X86::BT32ri8,     X86::BT32mi8,       TB_FOLDED_LOAD },
+    { X86::BT64ri8,     X86::BT64mi8,       TB_FOLDED_LOAD },
+    { X86::CALL32r,     X86::CALL32m,       TB_FOLDED_LOAD },
+    { X86::CALL64r,     X86::CALL64m,       TB_FOLDED_LOAD },
+    { X86::WINCALL64r,  X86::WINCALL64m,    TB_FOLDED_LOAD },
+    { X86::CMP16ri,     X86::CMP16mi,       TB_FOLDED_LOAD },
+    { X86::CMP16ri8,    X86::CMP16mi8,      TB_FOLDED_LOAD },
+    { X86::CMP16rr,     X86::CMP16mr,       TB_FOLDED_LOAD },
+    { X86::CMP32ri,     X86::CMP32mi,       TB_FOLDED_LOAD },
+    { X86::CMP32ri8,    X86::CMP32mi8,      TB_FOLDED_LOAD },
+    { X86::CMP32rr,     X86::CMP32mr,       TB_FOLDED_LOAD },
+    { X86::CMP64ri32,   X86::CMP64mi32,     TB_FOLDED_LOAD },
+    { X86::CMP64ri8,    X86::CMP64mi8,      TB_FOLDED_LOAD },
+    { X86::CMP64rr,     X86::CMP64mr,       TB_FOLDED_LOAD },
+    { X86::CMP8ri,      X86::CMP8mi,        TB_FOLDED_LOAD },
+    { X86::CMP8rr,      X86::CMP8mr,        TB_FOLDED_LOAD },
+    { X86::DIV16r,      X86::DIV16m,        TB_FOLDED_LOAD },
+    { X86::DIV32r,      X86::DIV32m,        TB_FOLDED_LOAD },
+    { X86::DIV64r,      X86::DIV64m,        TB_FOLDED_LOAD },
+    { X86::DIV8r,       X86::DIV8m,         TB_FOLDED_LOAD },
+    { X86::EXTRACTPSrr, X86::EXTRACTPSmr,   TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::FsMOVAPDrr,  X86::MOVSDmr,       TB_FOLDED_STORE | TB_NO_REVERSE },
+    { X86::FsMOVAPSrr,  X86::MOVSSmr,       TB_FOLDED_STORE | TB_NO_REVERSE },
+    { X86::IDIV16r,     X86::IDIV16m,       TB_FOLDED_LOAD },
+    { X86::IDIV32r,     X86::IDIV32m,       TB_FOLDED_LOAD },
+    { X86::IDIV64r,     X86::IDIV64m,       TB_FOLDED_LOAD },
+    { X86::IDIV8r,      X86::IDIV8m,        TB_FOLDED_LOAD },
+    { X86::IMUL16r,     X86::IMUL16m,       TB_FOLDED_LOAD },
+    { X86::IMUL32r,     X86::IMUL32m,       TB_FOLDED_LOAD },
+    { X86::IMUL64r,     X86::IMUL64m,       TB_FOLDED_LOAD },
+    { X86::IMUL8r,      X86::IMUL8m,        TB_FOLDED_LOAD },
+    { X86::JMP32r,      X86::JMP32m,        TB_FOLDED_LOAD },
+    { X86::JMP64r,      X86::JMP64m,        TB_FOLDED_LOAD },
+    { X86::MOV16ri,     X86::MOV16mi,       TB_FOLDED_STORE },
+    { X86::MOV16rr,     X86::MOV16mr,       TB_FOLDED_STORE },
+    { X86::MOV32ri,     X86::MOV32mi,       TB_FOLDED_STORE },
+    { X86::MOV32rr,     X86::MOV32mr,       TB_FOLDED_STORE },
+    { X86::MOV64ri32,   X86::MOV64mi32,     TB_FOLDED_STORE },
+    { X86::MOV64rr,     X86::MOV64mr,       TB_FOLDED_STORE },
+    { X86::MOV8ri,      X86::MOV8mi,        TB_FOLDED_STORE },
+    { X86::MOV8rr,      X86::MOV8mr,        TB_FOLDED_STORE },
+    { X86::MOV8rr_NOREX, X86::MOV8mr_NOREX, TB_FOLDED_STORE },
+    { X86::MOVAPDrr,    X86::MOVAPDmr,      TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::MOVAPSrr,    X86::MOVAPSmr,      TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::MOVDQArr,    X86::MOVDQAmr,      TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::MOVPDI2DIrr, X86::MOVPDI2DImr,   TB_FOLDED_STORE },
+    { X86::MOVPQIto64rr,X86::MOVPQI2QImr,   TB_FOLDED_STORE },
+    { X86::MOVSDto64rr, X86::MOVSDto64mr,   TB_FOLDED_STORE },
+    { X86::MOVSS2DIrr,  X86::MOVSS2DImr,    TB_FOLDED_STORE },
+    { X86::MOVUPDrr,    X86::MOVUPDmr,      TB_FOLDED_STORE },
+    { X86::MOVUPSrr,    X86::MOVUPSmr,      TB_FOLDED_STORE },
+    { X86::MUL16r,      X86::MUL16m,        TB_FOLDED_LOAD },
+    { X86::MUL32r,      X86::MUL32m,        TB_FOLDED_LOAD },
+    { X86::MUL64r,      X86::MUL64m,        TB_FOLDED_LOAD },
+    { X86::MUL8r,       X86::MUL8m,         TB_FOLDED_LOAD },
+    { X86::SETAEr,      X86::SETAEm,        TB_FOLDED_STORE },
+    { X86::SETAr,       X86::SETAm,         TB_FOLDED_STORE },
+    { X86::SETBEr,      X86::SETBEm,        TB_FOLDED_STORE },
+    { X86::SETBr,       X86::SETBm,         TB_FOLDED_STORE },
+    { X86::SETEr,       X86::SETEm,         TB_FOLDED_STORE },
+    { X86::SETGEr,      X86::SETGEm,        TB_FOLDED_STORE },
+    { X86::SETGr,       X86::SETGm,         TB_FOLDED_STORE },
+    { X86::SETLEr,      X86::SETLEm,        TB_FOLDED_STORE },
+    { X86::SETLr,       X86::SETLm,         TB_FOLDED_STORE },
+    { X86::SETNEr,      X86::SETNEm,        TB_FOLDED_STORE },
+    { X86::SETNOr,      X86::SETNOm,        TB_FOLDED_STORE },
+    { X86::SETNPr,      X86::SETNPm,        TB_FOLDED_STORE },
+    { X86::SETNSr,      X86::SETNSm,        TB_FOLDED_STORE },
+    { X86::SETOr,       X86::SETOm,         TB_FOLDED_STORE },
+    { X86::SETPr,       X86::SETPm,         TB_FOLDED_STORE },
+    { X86::SETSr,       X86::SETSm,         TB_FOLDED_STORE },
+    { X86::TAILJMPr,    X86::TAILJMPm,      TB_FOLDED_LOAD },
+    { X86::TAILJMPr64,  X86::TAILJMPm64,    TB_FOLDED_LOAD },
+    { X86::TEST16ri,    X86::TEST16mi,      TB_FOLDED_LOAD },
+    { X86::TEST32ri,    X86::TEST32mi,      TB_FOLDED_LOAD },
+    { X86::TEST64ri32,  X86::TEST64mi32,    TB_FOLDED_LOAD },
+    { X86::TEST8ri,     X86::TEST8mi,       TB_FOLDED_LOAD },
+    // AVX 128-bit versions of foldable instructions
+    { X86::VEXTRACTPSrr,X86::VEXTRACTPSmr,  TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::FsVMOVAPDrr, X86::VMOVSDmr,      TB_FOLDED_STORE | TB_NO_REVERSE },
+    { X86::FsVMOVAPSrr, X86::VMOVSSmr,      TB_FOLDED_STORE | TB_NO_REVERSE },
+    { X86::VMOVAPDrr,   X86::VMOVAPDmr,     TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::VMOVAPSrr,   X86::VMOVAPSmr,     TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::VMOVDQArr,   X86::VMOVDQAmr,     TB_FOLDED_STORE | TB_ALIGN_16 },
+    { X86::VMOVPDI2DIrr,X86::VMOVPDI2DImr,  TB_FOLDED_STORE },
+    { X86::VMOVPQIto64rr, X86::VMOVPQI2QImr,TB_FOLDED_STORE },
+    { X86::VMOVSDto64rr,X86::VMOVSDto64mr,  TB_FOLDED_STORE },
+    { X86::VMOVSS2DIrr, X86::VMOVSS2DImr,   TB_FOLDED_STORE },
+    { X86::VMOVUPDrr,   X86::VMOVUPDmr,     TB_FOLDED_STORE },
+    { X86::VMOVUPSrr,   X86::VMOVUPSmr,     TB_FOLDED_STORE },
+    // AVX 256-bit foldable instructions
+    { X86::VMOVAPDYrr,  X86::VMOVAPDYmr,    TB_FOLDED_STORE | TB_ALIGN_32 },
+    { X86::VMOVAPSYrr,  X86::VMOVAPSYmr,    TB_FOLDED_STORE | TB_ALIGN_32 },
+    { X86::VMOVDQAYrr,  X86::VMOVDQAYmr,    TB_FOLDED_STORE | TB_ALIGN_32 },
+    { X86::VMOVUPDYrr,  X86::VMOVUPDYmr,    TB_FOLDED_STORE },
+    { X86::VMOVUPSYrr,  X86::VMOVUPSYmr,    TB_FOLDED_STORE }
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl0); i != e; ++i) {
     unsigned RegOp      = OpTbl0[i][0];
-    unsigned MemOp      = OpTbl0[i][1] & ~TB_FLAGS;
-    unsigned FoldedLoad = OpTbl0[i][2];
-    unsigned Align      = OpTbl0[i][3];
-    assert(!RegOp2MemOpTable0.count(RegOp) && "Duplicated entries?");
-    RegOp2MemOpTable0[RegOp] = std::make_pair(MemOp, Align);
-
-    // If this is not a reversible operation (because there is a many->one)
-    // mapping, don't insert the reverse of the operation into MemOp2RegOpTable.
-    if (OpTbl0[i][1] & TB_NOT_REVERSABLE)
-      continue;
-
-    // Index 0, folded load or store.
-    unsigned AuxInfo = 0 | (FoldedLoad << 4) | ((FoldedLoad^1) << 5);
-    assert(!MemOp2RegOpTable.count(MemOp) && "Duplicated entries?");
-    MemOp2RegOpTable[MemOp] = std::make_pair(RegOp, AuxInfo);
+    unsigned MemOp      = OpTbl0[i][1];
+    unsigned Flags      = OpTbl0[i][2];
+    AddTableEntry(RegOp2MemOpTable0, MemOp2RegOpTable,
+                  RegOp, MemOp, TB_INDEX_0 | Flags);
   }
 
   static const unsigned OpTbl1[][3] = {
-    { X86::CMP16rr,         X86::CMP16rm, 0 },
-    { X86::CMP32rr,         X86::CMP32rm, 0 },
-    { X86::CMP64rr,         X86::CMP64rm, 0 },
-    { X86::CMP8rr,          X86::CMP8rm, 0 },
-    { X86::CVTSD2SSrr,      X86::CVTSD2SSrm, 0 },
-    { X86::CVTSI2SD64rr,    X86::CVTSI2SD64rm, 0 },
-    { X86::CVTSI2SDrr,      X86::CVTSI2SDrm, 0 },
-    { X86::CVTSI2SS64rr,    X86::CVTSI2SS64rm, 0 },
-    { X86::CVTSI2SSrr,      X86::CVTSI2SSrm, 0 },
-    { X86::CVTSS2SDrr,      X86::CVTSS2SDrm, 0 },
-    { X86::CVTTSD2SI64rr,   X86::CVTTSD2SI64rm, 0 },
-    { X86::CVTTSD2SIrr,     X86::CVTTSD2SIrm, 0 },
-    { X86::CVTTSS2SI64rr,   X86::CVTTSS2SI64rm, 0 },
-    { X86::CVTTSS2SIrr,     X86::CVTTSS2SIrm, 0 },
-    { X86::FsMOVAPDrr,      X86::MOVSDrm | TB_NOT_REVERSABLE , 0 },
-    { X86::FsMOVAPSrr,      X86::MOVSSrm | TB_NOT_REVERSABLE , 0 },
-    { X86::IMUL16rri,       X86::IMUL16rmi, 0 },
-    { X86::IMUL16rri8,      X86::IMUL16rmi8, 0 },
-    { X86::IMUL32rri,       X86::IMUL32rmi, 0 },
-    { X86::IMUL32rri8,      X86::IMUL32rmi8, 0 },
-    { X86::IMUL64rri32,     X86::IMUL64rmi32, 0 },
-    { X86::IMUL64rri8,      X86::IMUL64rmi8, 0 },
-    { X86::Int_COMISDrr,    X86::Int_COMISDrm, 0 },
-    { X86::Int_COMISSrr,    X86::Int_COMISSrm, 0 },
-    { X86::Int_CVTDQ2PDrr,  X86::Int_CVTDQ2PDrm, 16 },
-    { X86::Int_CVTDQ2PSrr,  X86::Int_CVTDQ2PSrm, 16 },
-    { X86::Int_CVTPD2DQrr,  X86::Int_CVTPD2DQrm, 16 },
-    { X86::Int_CVTPD2PSrr,  X86::Int_CVTPD2PSrm, 16 },
-    { X86::Int_CVTPS2DQrr,  X86::Int_CVTPS2DQrm, 16 },
-    { X86::Int_CVTPS2PDrr,  X86::Int_CVTPS2PDrm, 0 },
-    { X86::CVTSD2SI64rr,    X86::CVTSD2SI64rm, 0 },
-    { X86::CVTSD2SIrr,      X86::CVTSD2SIrm, 0 },
-    { X86::Int_CVTSD2SSrr,  X86::Int_CVTSD2SSrm, 0 },
-    { X86::Int_CVTSI2SD64rr,X86::Int_CVTSI2SD64rm, 0 },
-    { X86::Int_CVTSI2SDrr,  X86::Int_CVTSI2SDrm, 0 },
-    { X86::Int_CVTSI2SS64rr,X86::Int_CVTSI2SS64rm, 0 },
-    { X86::Int_CVTSI2SSrr,  X86::Int_CVTSI2SSrm, 0 },
-    { X86::Int_CVTSS2SDrr,  X86::Int_CVTSS2SDrm, 0 },
-    { X86::Int_CVTSS2SI64rr,X86::Int_CVTSS2SI64rm, 0 },
-    { X86::Int_CVTSS2SIrr,  X86::Int_CVTSS2SIrm, 0 },
-    { X86::CVTTPD2DQrr,     X86::CVTTPD2DQrm, 16 },
-    { X86::CVTTPS2DQrr,     X86::CVTTPS2DQrm, 16 },
-    { X86::Int_CVTTSD2SI64rr,X86::Int_CVTTSD2SI64rm, 0 },
-    { X86::Int_CVTTSD2SIrr, X86::Int_CVTTSD2SIrm, 0 },
-    { X86::Int_CVTTSS2SI64rr,X86::Int_CVTTSS2SI64rm, 0 },
-    { X86::Int_CVTTSS2SIrr, X86::Int_CVTTSS2SIrm, 0 },
-    { X86::Int_UCOMISDrr,   X86::Int_UCOMISDrm, 0 },
-    { X86::Int_UCOMISSrr,   X86::Int_UCOMISSrm, 0 },
-    { X86::MOV16rr,         X86::MOV16rm, 0 },
-    { X86::MOV32rr,         X86::MOV32rm, 0 },
-    { X86::MOV64rr,         X86::MOV64rm, 0 },
-    { X86::MOV64toPQIrr,    X86::MOVQI2PQIrm, 0 },
-    { X86::MOV64toSDrr,     X86::MOV64toSDrm, 0 },
-    { X86::MOV8rr,          X86::MOV8rm, 0 },
-    { X86::MOVAPDrr,        X86::MOVAPDrm, 16 },
-    { X86::MOVAPSrr,        X86::MOVAPSrm, 16 },
-    { X86::VMOVAPDYrr,      X86::VMOVAPDYrm, 32 },
-    { X86::VMOVAPSYrr,      X86::VMOVAPSYrm, 32 },
-    { X86::MOVDDUPrr,       X86::MOVDDUPrm, 0 },
-    { X86::MOVDI2PDIrr,     X86::MOVDI2PDIrm, 0 },
-    { X86::MOVDI2SSrr,      X86::MOVDI2SSrm, 0 },
-    { X86::MOVDQArr,        X86::MOVDQArm, 16 },
-    { X86::VMOVDQAYrr,      X86::VMOVDQAYrm, 16 },
-    { X86::MOVSHDUPrr,      X86::MOVSHDUPrm, 16 },
-    { X86::MOVSLDUPrr,      X86::MOVSLDUPrm, 16 },
-    { X86::MOVSX16rr8,      X86::MOVSX16rm8, 0 },
-    { X86::MOVSX32rr16,     X86::MOVSX32rm16, 0 },
-    { X86::MOVSX32rr8,      X86::MOVSX32rm8, 0 },
-    { X86::MOVSX64rr16,     X86::MOVSX64rm16, 0 },
-    { X86::MOVSX64rr32,     X86::MOVSX64rm32, 0 },
-    { X86::MOVSX64rr8,      X86::MOVSX64rm8, 0 },
-    { X86::MOVUPDrr,        X86::MOVUPDrm, 16 },
-    { X86::MOVUPSrr,        X86::MOVUPSrm, 0 },
-    { X86::VMOVUPDYrr,      X86::VMOVUPDYrm, 0 },
-    { X86::VMOVUPSYrr,      X86::VMOVUPSYrm, 0 },
-    { X86::MOVZDI2PDIrr,    X86::MOVZDI2PDIrm, 0 },
-    { X86::MOVZQI2PQIrr,    X86::MOVZQI2PQIrm, 0 },
-    { X86::MOVZPQILo2PQIrr, X86::MOVZPQILo2PQIrm, 16 },
-    { X86::MOVZX16rr8,      X86::MOVZX16rm8, 0 },
-    { X86::MOVZX32rr16,     X86::MOVZX32rm16, 0 },
-    { X86::MOVZX32_NOREXrr8, X86::MOVZX32_NOREXrm8, 0 },
-    { X86::MOVZX32rr8,      X86::MOVZX32rm8, 0 },
-    { X86::MOVZX64rr16,     X86::MOVZX64rm16, 0 },
-    { X86::MOVZX64rr32,     X86::MOVZX64rm32, 0 },
-    { X86::MOVZX64rr8,      X86::MOVZX64rm8, 0 },
-    { X86::PSHUFDri,        X86::PSHUFDmi, 16 },
-    { X86::PSHUFHWri,       X86::PSHUFHWmi, 16 },
-    { X86::PSHUFLWri,       X86::PSHUFLWmi, 16 },
-    { X86::RCPPSr,          X86::RCPPSm, 16 },
-    { X86::RCPPSr_Int,      X86::RCPPSm_Int, 16 },
-    { X86::RSQRTPSr,        X86::RSQRTPSm, 16 },
-    { X86::RSQRTPSr_Int,    X86::RSQRTPSm_Int, 16 },
-    { X86::RSQRTSSr,        X86::RSQRTSSm, 0 },
-    { X86::RSQRTSSr_Int,    X86::RSQRTSSm_Int, 0 },
-    { X86::SQRTPDr,         X86::SQRTPDm, 16 },
-    { X86::SQRTPDr_Int,     X86::SQRTPDm_Int, 16 },
-    { X86::SQRTPSr,         X86::SQRTPSm, 16 },
-    { X86::SQRTPSr_Int,     X86::SQRTPSm_Int, 16 },
-    { X86::SQRTSDr,         X86::SQRTSDm, 0 },
-    { X86::SQRTSDr_Int,     X86::SQRTSDm_Int, 0 },
-    { X86::SQRTSSr,         X86::SQRTSSm, 0 },
-    { X86::SQRTSSr_Int,     X86::SQRTSSm_Int, 0 },
-    { X86::TEST16rr,        X86::TEST16rm, 0 },
-    { X86::TEST32rr,        X86::TEST32rm, 0 },
-    { X86::TEST64rr,        X86::TEST64rm, 0 },
-    { X86::TEST8rr,         X86::TEST8rm, 0 },
+    { X86::CMP16rr,         X86::CMP16rm,             0 },
+    { X86::CMP32rr,         X86::CMP32rm,             0 },
+    { X86::CMP64rr,         X86::CMP64rm,             0 },
+    { X86::CMP8rr,          X86::CMP8rm,              0 },
+    { X86::CVTSD2SSrr,      X86::CVTSD2SSrm,          0 },
+    { X86::CVTSI2SD64rr,    X86::CVTSI2SD64rm,        0 },
+    { X86::CVTSI2SDrr,      X86::CVTSI2SDrm,          0 },
+    { X86::CVTSI2SS64rr,    X86::CVTSI2SS64rm,        0 },
+    { X86::CVTSI2SSrr,      X86::CVTSI2SSrm,          0 },
+    { X86::CVTSS2SDrr,      X86::CVTSS2SDrm,          0 },
+    { X86::CVTTSD2SI64rr,   X86::CVTTSD2SI64rm,       0 },
+    { X86::CVTTSD2SIrr,     X86::CVTTSD2SIrm,         0 },
+    { X86::CVTTSS2SI64rr,   X86::CVTTSS2SI64rm,       0 },
+    { X86::CVTTSS2SIrr,     X86::CVTTSS2SIrm,         0 },
+    { X86::FsMOVAPDrr,      X86::MOVSDrm,             TB_NO_REVERSE },
+    { X86::FsMOVAPSrr,      X86::MOVSSrm,             TB_NO_REVERSE },
+    { X86::IMUL16rri,       X86::IMUL16rmi,           0 },
+    { X86::IMUL16rri8,      X86::IMUL16rmi8,          0 },
+    { X86::IMUL32rri,       X86::IMUL32rmi,           0 },
+    { X86::IMUL32rri8,      X86::IMUL32rmi8,          0 },
+    { X86::IMUL64rri32,     X86::IMUL64rmi32,         0 },
+    { X86::IMUL64rri8,      X86::IMUL64rmi8,          0 },
+    { X86::Int_COMISDrr,    X86::Int_COMISDrm,        0 },
+    { X86::Int_COMISSrr,    X86::Int_COMISSrm,        0 },
+    { X86::Int_CVTDQ2PDrr,  X86::Int_CVTDQ2PDrm,      TB_ALIGN_16 },
+    { X86::Int_CVTDQ2PSrr,  X86::Int_CVTDQ2PSrm,      TB_ALIGN_16 },
+    { X86::Int_CVTPD2DQrr,  X86::Int_CVTPD2DQrm,      TB_ALIGN_16 },
+    { X86::Int_CVTPD2PSrr,  X86::Int_CVTPD2PSrm,      TB_ALIGN_16 },
+    { X86::Int_CVTPS2DQrr,  X86::Int_CVTPS2DQrm,      TB_ALIGN_16 },
+    { X86::Int_CVTPS2PDrr,  X86::Int_CVTPS2PDrm,      0 },
+    { X86::CVTSD2SI64rr,    X86::CVTSD2SI64rm,        0 },
+    { X86::CVTSD2SIrr,      X86::CVTSD2SIrm,          0 },
+    { X86::Int_CVTSD2SSrr,  X86::Int_CVTSD2SSrm,      0 },
+    { X86::Int_CVTSI2SD64rr,X86::Int_CVTSI2SD64rm,    0 },
+    { X86::Int_CVTSI2SDrr,  X86::Int_CVTSI2SDrm,      0 },
+    { X86::Int_CVTSI2SS64rr,X86::Int_CVTSI2SS64rm,    0 },
+    { X86::Int_CVTSI2SSrr,  X86::Int_CVTSI2SSrm,      0 },
+    { X86::Int_CVTSS2SDrr,  X86::Int_CVTSS2SDrm,      0 },
+    { X86::CVTTPD2DQrr,     X86::CVTTPD2DQrm,         TB_ALIGN_16 },
+    { X86::CVTTPS2DQrr,     X86::CVTTPS2DQrm,         TB_ALIGN_16 },
+    { X86::Int_CVTTSD2SI64rr,X86::Int_CVTTSD2SI64rm,  0 },
+    { X86::Int_CVTTSD2SIrr, X86::Int_CVTTSD2SIrm,     0 },
+    { X86::Int_CVTTSS2SI64rr,X86::Int_CVTTSS2SI64rm,  0 },
+    { X86::Int_CVTTSS2SIrr, X86::Int_CVTTSS2SIrm,     0 },
+    { X86::Int_UCOMISDrr,   X86::Int_UCOMISDrm,       0 },
+    { X86::Int_UCOMISSrr,   X86::Int_UCOMISSrm,       0 },
+    { X86::MOV16rr,         X86::MOV16rm,             0 },
+    { X86::MOV32rr,         X86::MOV32rm,             0 },
+    { X86::MOV64rr,         X86::MOV64rm,             0 },
+    { X86::MOV64toPQIrr,    X86::MOVQI2PQIrm,         0 },
+    { X86::MOV64toSDrr,     X86::MOV64toSDrm,         0 },
+    { X86::MOV8rr,          X86::MOV8rm,              0 },
+    { X86::MOVAPDrr,        X86::MOVAPDrm,            TB_ALIGN_16 },
+    { X86::MOVAPSrr,        X86::MOVAPSrm,            TB_ALIGN_16 },
+    { X86::MOVDDUPrr,       X86::MOVDDUPrm,           0 },
+    { X86::MOVDI2PDIrr,     X86::MOVDI2PDIrm,         0 },
+    { X86::MOVDI2SSrr,      X86::MOVDI2SSrm,          0 },
+    { X86::MOVDQArr,        X86::MOVDQArm,            TB_ALIGN_16 },
+    { X86::MOVSHDUPrr,      X86::MOVSHDUPrm,          TB_ALIGN_16 },
+    { X86::MOVSLDUPrr,      X86::MOVSLDUPrm,          TB_ALIGN_16 },
+    { X86::MOVSX16rr8,      X86::MOVSX16rm8,          0 },
+    { X86::MOVSX32rr16,     X86::MOVSX32rm16,         0 },
+    { X86::MOVSX32rr8,      X86::MOVSX32rm8,          0 },
+    { X86::MOVSX64rr16,     X86::MOVSX64rm16,         0 },
+    { X86::MOVSX64rr32,     X86::MOVSX64rm32,         0 },
+    { X86::MOVSX64rr8,      X86::MOVSX64rm8,          0 },
+    { X86::MOVUPDrr,        X86::MOVUPDrm,            TB_ALIGN_16 },
+    { X86::MOVUPSrr,        X86::MOVUPSrm,            0 },
+    { X86::MOVZDI2PDIrr,    X86::MOVZDI2PDIrm,        0 },
+    { X86::MOVZQI2PQIrr,    X86::MOVZQI2PQIrm,        0 },
+    { X86::MOVZPQILo2PQIrr, X86::MOVZPQILo2PQIrm,     TB_ALIGN_16 },
+    { X86::MOVZX16rr8,      X86::MOVZX16rm8,          0 },
+    { X86::MOVZX32rr16,     X86::MOVZX32rm16,         0 },
+    { X86::MOVZX32_NOREXrr8, X86::MOVZX32_NOREXrm8,   0 },
+    { X86::MOVZX32rr8,      X86::MOVZX32rm8,          0 },
+    { X86::MOVZX64rr16,     X86::MOVZX64rm16,         0 },
+    { X86::MOVZX64rr32,     X86::MOVZX64rm32,         0 },
+    { X86::MOVZX64rr8,      X86::MOVZX64rm8,          0 },
+    { X86::PSHUFDri,        X86::PSHUFDmi,            TB_ALIGN_16 },
+    { X86::PSHUFHWri,       X86::PSHUFHWmi,           TB_ALIGN_16 },
+    { X86::PSHUFLWri,       X86::PSHUFLWmi,           TB_ALIGN_16 },
+    { X86::RCPPSr,          X86::RCPPSm,              TB_ALIGN_16 },
+    { X86::RCPPSr_Int,      X86::RCPPSm_Int,          TB_ALIGN_16 },
+    { X86::RSQRTPSr,        X86::RSQRTPSm,            TB_ALIGN_16 },
+    { X86::RSQRTPSr_Int,    X86::RSQRTPSm_Int,        TB_ALIGN_16 },
+    { X86::RSQRTSSr,        X86::RSQRTSSm,            0 },
+    { X86::RSQRTSSr_Int,    X86::RSQRTSSm_Int,        0 },
+    { X86::SQRTPDr,         X86::SQRTPDm,             TB_ALIGN_16 },
+    { X86::SQRTPDr_Int,     X86::SQRTPDm_Int,         TB_ALIGN_16 },
+    { X86::SQRTPSr,         X86::SQRTPSm,             TB_ALIGN_16 },
+    { X86::SQRTPSr_Int,     X86::SQRTPSm_Int,         TB_ALIGN_16 },
+    { X86::SQRTSDr,         X86::SQRTSDm,             0 },
+    { X86::SQRTSDr_Int,     X86::SQRTSDm_Int,         0 },
+    { X86::SQRTSSr,         X86::SQRTSSm,             0 },
+    { X86::SQRTSSr_Int,     X86::SQRTSSm_Int,         0 },
+    { X86::TEST16rr,        X86::TEST16rm,            0 },
+    { X86::TEST32rr,        X86::TEST32rm,            0 },
+    { X86::TEST64rr,        X86::TEST64rm,            0 },
+    { X86::TEST8rr,         X86::TEST8rm,             0 },
     // FIXME: TEST*rr EAX,EAX ---> CMP [mem], 0
-    { X86::UCOMISDrr,       X86::UCOMISDrm, 0 },
-    { X86::UCOMISSrr,       X86::UCOMISSrm, 0 }
+    { X86::UCOMISDrr,       X86::UCOMISDrm,           0 },
+    { X86::UCOMISSrr,       X86::UCOMISSrm,           0 },
+    // AVX 128-bit versions of foldable instructions
+    { X86::Int_VCOMISDrr,   X86::Int_VCOMISDrm,       0 },
+    { X86::Int_VCOMISSrr,   X86::Int_VCOMISSrm,       0 },
+    { X86::Int_VCVTDQ2PDrr, X86::Int_VCVTDQ2PDrm,     TB_ALIGN_16 },
+    { X86::Int_VCVTDQ2PSrr, X86::Int_VCVTDQ2PSrm,     TB_ALIGN_16 },
+    { X86::Int_VCVTPD2DQrr, X86::Int_VCVTPD2DQrm,     TB_ALIGN_16 },
+    { X86::Int_VCVTPD2PSrr, X86::Int_VCVTPD2PSrm,     TB_ALIGN_16 },
+    { X86::Int_VCVTPS2DQrr, X86::Int_VCVTPS2DQrm,     TB_ALIGN_16 },
+    { X86::Int_VCVTPS2PDrr, X86::Int_VCVTPS2PDrm,     0 },
+    { X86::Int_VUCOMISDrr,  X86::Int_VUCOMISDrm,      0 },
+    { X86::Int_VUCOMISSrr,  X86::Int_VUCOMISSrm,      0 },
+    { X86::FsVMOVAPDrr,     X86::VMOVSDrm,            TB_NO_REVERSE },
+    { X86::FsVMOVAPSrr,     X86::VMOVSSrm,            TB_NO_REVERSE },
+    { X86::VMOV64toPQIrr,   X86::VMOVQI2PQIrm,        0 },
+    { X86::VMOV64toSDrr,    X86::VMOV64toSDrm,        0 },
+    { X86::VMOVAPDrr,       X86::VMOVAPDrm,           TB_ALIGN_16 },
+    { X86::VMOVAPSrr,       X86::VMOVAPSrm,           TB_ALIGN_16 },
+    { X86::VMOVDDUPrr,      X86::VMOVDDUPrm,          0 },
+    { X86::VMOVDI2PDIrr,    X86::VMOVDI2PDIrm,        0 },
+    { X86::VMOVDI2SSrr,     X86::VMOVDI2SSrm,         0 },
+    { X86::VMOVDQArr,       X86::VMOVDQArm,           TB_ALIGN_16 },
+    { X86::VMOVSLDUPrr,     X86::VMOVSLDUPrm,         TB_ALIGN_16 },
+    { X86::VMOVSHDUPrr,     X86::VMOVSHDUPrm,         TB_ALIGN_16 },
+    { X86::VMOVUPDrr,       X86::VMOVUPDrm,           TB_ALIGN_16 },
+    { X86::VMOVUPSrr,       X86::VMOVUPSrm,           0 },
+    { X86::VMOVZDI2PDIrr,   X86::VMOVZDI2PDIrm,       0 },
+    { X86::VMOVZQI2PQIrr,   X86::VMOVZQI2PQIrm,       0 },
+    { X86::VMOVZPQILo2PQIrr,X86::VMOVZPQILo2PQIrm,    TB_ALIGN_16 },
+    { X86::VPSHUFDri,       X86::VPSHUFDmi,           TB_ALIGN_16 },
+    { X86::VPSHUFHWri,      X86::VPSHUFHWmi,          TB_ALIGN_16 },
+    { X86::VPSHUFLWri,      X86::VPSHUFLWmi,          TB_ALIGN_16 },
+    { X86::VRCPPSr,         X86::VRCPPSm,             TB_ALIGN_16 },
+    { X86::VRCPPSr_Int,     X86::VRCPPSm_Int,         TB_ALIGN_16 },
+    { X86::VRSQRTPSr,       X86::VRSQRTPSm,           TB_ALIGN_16 },
+    { X86::VRSQRTPSr_Int,   X86::VRSQRTPSm_Int,       TB_ALIGN_16 },
+    { X86::VSQRTPDr,        X86::VSQRTPDm,            TB_ALIGN_16 },
+    { X86::VSQRTPDr_Int,    X86::VSQRTPDm_Int,        TB_ALIGN_16 },
+    { X86::VSQRTPSr,        X86::VSQRTPSm,            TB_ALIGN_16 },
+    { X86::VSQRTPSr_Int,    X86::VSQRTPSm_Int,        TB_ALIGN_16 },
+    { X86::VUCOMISDrr,      X86::VUCOMISDrm,          0 },
+    { X86::VUCOMISSrr,      X86::VUCOMISSrm,          0 },
+    // AVX 256-bit foldable instructions
+    { X86::VMOVAPDYrr,      X86::VMOVAPDYrm,          TB_ALIGN_32 },
+    { X86::VMOVAPSYrr,      X86::VMOVAPSYrm,          TB_ALIGN_32 },
+    { X86::VMOVDQAYrr,      X86::VMOVDQAYrm,          TB_ALIGN_16 },
+    { X86::VMOVUPDYrr,      X86::VMOVUPDYrm,          0 },
+    { X86::VMOVUPSYrr,      X86::VMOVUPSYrm,          0 }
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl1); i != e; ++i) {
     unsigned RegOp = OpTbl1[i][0];
-    unsigned MemOp = OpTbl1[i][1] & ~TB_FLAGS;
-    unsigned Align = OpTbl1[i][2];
-    assert(!RegOp2MemOpTable1.count(RegOp) && "Duplicate entries");
-    RegOp2MemOpTable1[RegOp] = std::make_pair(MemOp, Align);
-
-    // If this is not a reversible operation (because there is a many->one)
-    // mapping, don't insert the reverse of the operation into MemOp2RegOpTable.
-    if (OpTbl1[i][1] & TB_NOT_REVERSABLE)
-      continue;
-
-    // Index 1, folded load
-    unsigned AuxInfo = 1 | (1 << 4);
-    assert(!MemOp2RegOpTable.count(MemOp) && "Duplicate entries");
-    MemOp2RegOpTable[MemOp] = std::make_pair(RegOp, AuxInfo);
+    unsigned MemOp = OpTbl1[i][1];
+    unsigned Flags = OpTbl1[i][2];
+    AddTableEntry(RegOp2MemOpTable1, MemOp2RegOpTable,
+                  RegOp, MemOp,
+                  // Index 1, folded load
+                  Flags | TB_INDEX_1 | TB_FOLDED_LOAD);
   }
 
   static const unsigned OpTbl2[][3] = {
-    { X86::ADC32rr,         X86::ADC32rm, 0 },
-    { X86::ADC64rr,         X86::ADC64rm, 0 },
-    { X86::ADD16rr,         X86::ADD16rm, 0 },
-    { X86::ADD16rr_DB,      X86::ADD16rm | TB_NOT_REVERSABLE, 0 },
-    { X86::ADD32rr,         X86::ADD32rm, 0 },
-    { X86::ADD32rr_DB,      X86::ADD32rm | TB_NOT_REVERSABLE, 0 },
-    { X86::ADD64rr,         X86::ADD64rm, 0 },
-    { X86::ADD64rr_DB,      X86::ADD64rm | TB_NOT_REVERSABLE, 0 },
-    { X86::ADD8rr,          X86::ADD8rm, 0 },
-    { X86::ADDPDrr,         X86::ADDPDrm, 16 },
-    { X86::ADDPSrr,         X86::ADDPSrm, 16 },
-    { X86::ADDSDrr,         X86::ADDSDrm, 0 },
-    { X86::ADDSSrr,         X86::ADDSSrm, 0 },
-    { X86::ADDSUBPDrr,      X86::ADDSUBPDrm, 16 },
-    { X86::ADDSUBPSrr,      X86::ADDSUBPSrm, 16 },
-    { X86::AND16rr,         X86::AND16rm, 0 },
-    { X86::AND32rr,         X86::AND32rm, 0 },
-    { X86::AND64rr,         X86::AND64rm, 0 },
-    { X86::AND8rr,          X86::AND8rm, 0 },
-    { X86::ANDNPDrr,        X86::ANDNPDrm, 16 },
-    { X86::ANDNPSrr,        X86::ANDNPSrm, 16 },
-    { X86::ANDPDrr,         X86::ANDPDrm, 16 },
-    { X86::ANDPSrr,         X86::ANDPSrm, 16 },
-    { X86::CMOVA16rr,       X86::CMOVA16rm, 0 },
-    { X86::CMOVA32rr,       X86::CMOVA32rm, 0 },
-    { X86::CMOVA64rr,       X86::CMOVA64rm, 0 },
-    { X86::CMOVAE16rr,      X86::CMOVAE16rm, 0 },
-    { X86::CMOVAE32rr,      X86::CMOVAE32rm, 0 },
-    { X86::CMOVAE64rr,      X86::CMOVAE64rm, 0 },
-    { X86::CMOVB16rr,       X86::CMOVB16rm, 0 },
-    { X86::CMOVB32rr,       X86::CMOVB32rm, 0 },
-    { X86::CMOVB64rr,       X86::CMOVB64rm, 0 },
-    { X86::CMOVBE16rr,      X86::CMOVBE16rm, 0 },
-    { X86::CMOVBE32rr,      X86::CMOVBE32rm, 0 },
-    { X86::CMOVBE64rr,      X86::CMOVBE64rm, 0 },
-    { X86::CMOVE16rr,       X86::CMOVE16rm, 0 },
-    { X86::CMOVE32rr,       X86::CMOVE32rm, 0 },
-    { X86::CMOVE64rr,       X86::CMOVE64rm, 0 },
-    { X86::CMOVG16rr,       X86::CMOVG16rm, 0 },
-    { X86::CMOVG32rr,       X86::CMOVG32rm, 0 },
-    { X86::CMOVG64rr,       X86::CMOVG64rm, 0 },
-    { X86::CMOVGE16rr,      X86::CMOVGE16rm, 0 },
-    { X86::CMOVGE32rr,      X86::CMOVGE32rm, 0 },
-    { X86::CMOVGE64rr,      X86::CMOVGE64rm, 0 },
-    { X86::CMOVL16rr,       X86::CMOVL16rm, 0 },
-    { X86::CMOVL32rr,       X86::CMOVL32rm, 0 },
-    { X86::CMOVL64rr,       X86::CMOVL64rm, 0 },
-    { X86::CMOVLE16rr,      X86::CMOVLE16rm, 0 },
-    { X86::CMOVLE32rr,      X86::CMOVLE32rm, 0 },
-    { X86::CMOVLE64rr,      X86::CMOVLE64rm, 0 },
-    { X86::CMOVNE16rr,      X86::CMOVNE16rm, 0 },
-    { X86::CMOVNE32rr,      X86::CMOVNE32rm, 0 },
-    { X86::CMOVNE64rr,      X86::CMOVNE64rm, 0 },
-    { X86::CMOVNO16rr,      X86::CMOVNO16rm, 0 },
-    { X86::CMOVNO32rr,      X86::CMOVNO32rm, 0 },
-    { X86::CMOVNO64rr,      X86::CMOVNO64rm, 0 },
-    { X86::CMOVNP16rr,      X86::CMOVNP16rm, 0 },
-    { X86::CMOVNP32rr,      X86::CMOVNP32rm, 0 },
-    { X86::CMOVNP64rr,      X86::CMOVNP64rm, 0 },
-    { X86::CMOVNS16rr,      X86::CMOVNS16rm, 0 },
-    { X86::CMOVNS32rr,      X86::CMOVNS32rm, 0 },
-    { X86::CMOVNS64rr,      X86::CMOVNS64rm, 0 },
-    { X86::CMOVO16rr,       X86::CMOVO16rm, 0 },
-    { X86::CMOVO32rr,       X86::CMOVO32rm, 0 },
-    { X86::CMOVO64rr,       X86::CMOVO64rm, 0 },
-    { X86::CMOVP16rr,       X86::CMOVP16rm, 0 },
-    { X86::CMOVP32rr,       X86::CMOVP32rm, 0 },
-    { X86::CMOVP64rr,       X86::CMOVP64rm, 0 },
-    { X86::CMOVS16rr,       X86::CMOVS16rm, 0 },
-    { X86::CMOVS32rr,       X86::CMOVS32rm, 0 },
-    { X86::CMOVS64rr,       X86::CMOVS64rm, 0 },
-    { X86::CMPPDrri,        X86::CMPPDrmi, 16 },
-    { X86::CMPPSrri,        X86::CMPPSrmi, 16 },
-    { X86::CMPSDrr,         X86::CMPSDrm, 0 },
-    { X86::CMPSSrr,         X86::CMPSSrm, 0 },
-    { X86::DIVPDrr,         X86::DIVPDrm, 16 },
-    { X86::DIVPSrr,         X86::DIVPSrm, 16 },
-    { X86::DIVSDrr,         X86::DIVSDrm, 0 },
-    { X86::DIVSSrr,         X86::DIVSSrm, 0 },
-    { X86::FsANDNPDrr,      X86::FsANDNPDrm, 16 },
-    { X86::FsANDNPSrr,      X86::FsANDNPSrm, 16 },
-    { X86::FsANDPDrr,       X86::FsANDPDrm, 16 },
-    { X86::FsANDPSrr,       X86::FsANDPSrm, 16 },
-    { X86::FsORPDrr,        X86::FsORPDrm, 16 },
-    { X86::FsORPSrr,        X86::FsORPSrm, 16 },
-    { X86::FsXORPDrr,       X86::FsXORPDrm, 16 },
-    { X86::FsXORPSrr,       X86::FsXORPSrm, 16 },
-    { X86::HADDPDrr,        X86::HADDPDrm, 16 },
-    { X86::HADDPSrr,        X86::HADDPSrm, 16 },
-    { X86::HSUBPDrr,        X86::HSUBPDrm, 16 },
-    { X86::HSUBPSrr,        X86::HSUBPSrm, 16 },
-    { X86::IMUL16rr,        X86::IMUL16rm, 0 },
-    { X86::IMUL32rr,        X86::IMUL32rm, 0 },
-    { X86::IMUL64rr,        X86::IMUL64rm, 0 },
-    { X86::Int_CMPSDrr,     X86::Int_CMPSDrm, 0 },
-    { X86::Int_CMPSSrr,     X86::Int_CMPSSrm, 0 },
-    { X86::MAXPDrr,         X86::MAXPDrm, 16 },
-    { X86::MAXPDrr_Int,     X86::MAXPDrm_Int, 16 },
-    { X86::MAXPSrr,         X86::MAXPSrm, 16 },
-    { X86::MAXPSrr_Int,     X86::MAXPSrm_Int, 16 },
-    { X86::MAXSDrr,         X86::MAXSDrm, 0 },
-    { X86::MAXSDrr_Int,     X86::MAXSDrm_Int, 0 },
-    { X86::MAXSSrr,         X86::MAXSSrm, 0 },
-    { X86::MAXSSrr_Int,     X86::MAXSSrm_Int, 0 },
-    { X86::MINPDrr,         X86::MINPDrm, 16 },
-    { X86::MINPDrr_Int,     X86::MINPDrm_Int, 16 },
-    { X86::MINPSrr,         X86::MINPSrm, 16 },
-    { X86::MINPSrr_Int,     X86::MINPSrm_Int, 16 },
-    { X86::MINSDrr,         X86::MINSDrm, 0 },
-    { X86::MINSDrr_Int,     X86::MINSDrm_Int, 0 },
-    { X86::MINSSrr,         X86::MINSSrm, 0 },
-    { X86::MINSSrr_Int,     X86::MINSSrm_Int, 0 },
-    { X86::MULPDrr,         X86::MULPDrm, 16 },
-    { X86::MULPSrr,         X86::MULPSrm, 16 },
-    { X86::MULSDrr,         X86::MULSDrm, 0 },
-    { X86::MULSSrr,         X86::MULSSrm, 0 },
-    { X86::OR16rr,          X86::OR16rm, 0 },
-    { X86::OR32rr,          X86::OR32rm, 0 },
-    { X86::OR64rr,          X86::OR64rm, 0 },
-    { X86::OR8rr,           X86::OR8rm, 0 },
-    { X86::ORPDrr,          X86::ORPDrm, 16 },
-    { X86::ORPSrr,          X86::ORPSrm, 16 },
-    { X86::PACKSSDWrr,      X86::PACKSSDWrm, 16 },
-    { X86::PACKSSWBrr,      X86::PACKSSWBrm, 16 },
-    { X86::PACKUSWBrr,      X86::PACKUSWBrm, 16 },
-    { X86::PADDBrr,         X86::PADDBrm, 16 },
-    { X86::PADDDrr,         X86::PADDDrm, 16 },
-    { X86::PADDQrr,         X86::PADDQrm, 16 },
-    { X86::PADDSBrr,        X86::PADDSBrm, 16 },
-    { X86::PADDSWrr,        X86::PADDSWrm, 16 },
-    { X86::PADDWrr,         X86::PADDWrm, 16 },
-    { X86::PANDNrr,         X86::PANDNrm, 16 },
-    { X86::PANDrr,          X86::PANDrm, 16 },
-    { X86::PAVGBrr,         X86::PAVGBrm, 16 },
-    { X86::PAVGWrr,         X86::PAVGWrm, 16 },
-    { X86::PCMPEQBrr,       X86::PCMPEQBrm, 16 },
-    { X86::PCMPEQDrr,       X86::PCMPEQDrm, 16 },
-    { X86::PCMPEQWrr,       X86::PCMPEQWrm, 16 },
-    { X86::PCMPGTBrr,       X86::PCMPGTBrm, 16 },
-    { X86::PCMPGTDrr,       X86::PCMPGTDrm, 16 },
-    { X86::PCMPGTWrr,       X86::PCMPGTWrm, 16 },
-    { X86::PINSRWrri,       X86::PINSRWrmi, 16 },
-    { X86::PMADDWDrr,       X86::PMADDWDrm, 16 },
-    { X86::PMAXSWrr,        X86::PMAXSWrm, 16 },
-    { X86::PMAXUBrr,        X86::PMAXUBrm, 16 },
-    { X86::PMINSWrr,        X86::PMINSWrm, 16 },
-    { X86::PMINUBrr,        X86::PMINUBrm, 16 },
-    { X86::PMULDQrr,        X86::PMULDQrm, 16 },
-    { X86::PMULHUWrr,       X86::PMULHUWrm, 16 },
-    { X86::PMULHWrr,        X86::PMULHWrm, 16 },
-    { X86::PMULLDrr,        X86::PMULLDrm, 16 },
-    { X86::PMULLWrr,        X86::PMULLWrm, 16 },
-    { X86::PMULUDQrr,       X86::PMULUDQrm, 16 },
-    { X86::PORrr,           X86::PORrm, 16 },
-    { X86::PSADBWrr,        X86::PSADBWrm, 16 },
-    { X86::PSLLDrr,         X86::PSLLDrm, 16 },
-    { X86::PSLLQrr,         X86::PSLLQrm, 16 },
-    { X86::PSLLWrr,         X86::PSLLWrm, 16 },
-    { X86::PSRADrr,         X86::PSRADrm, 16 },
-    { X86::PSRAWrr,         X86::PSRAWrm, 16 },
-    { X86::PSRLDrr,         X86::PSRLDrm, 16 },
-    { X86::PSRLQrr,         X86::PSRLQrm, 16 },
-    { X86::PSRLWrr,         X86::PSRLWrm, 16 },
-    { X86::PSUBBrr,         X86::PSUBBrm, 16 },
-    { X86::PSUBDrr,         X86::PSUBDrm, 16 },
-    { X86::PSUBSBrr,        X86::PSUBSBrm, 16 },
-    { X86::PSUBSWrr,        X86::PSUBSWrm, 16 },
-    { X86::PSUBWrr,         X86::PSUBWrm, 16 },
-    { X86::PUNPCKHBWrr,     X86::PUNPCKHBWrm, 16 },
-    { X86::PUNPCKHDQrr,     X86::PUNPCKHDQrm, 16 },
-    { X86::PUNPCKHQDQrr,    X86::PUNPCKHQDQrm, 16 },
-    { X86::PUNPCKHWDrr,     X86::PUNPCKHWDrm, 16 },
-    { X86::PUNPCKLBWrr,     X86::PUNPCKLBWrm, 16 },
-    { X86::PUNPCKLDQrr,     X86::PUNPCKLDQrm, 16 },
-    { X86::PUNPCKLQDQrr,    X86::PUNPCKLQDQrm, 16 },
-    { X86::PUNPCKLWDrr,     X86::PUNPCKLWDrm, 16 },
-    { X86::PXORrr,          X86::PXORrm, 16 },
-    { X86::SBB32rr,         X86::SBB32rm, 0 },
-    { X86::SBB64rr,         X86::SBB64rm, 0 },
-    { X86::SHUFPDrri,       X86::SHUFPDrmi, 16 },
-    { X86::SHUFPSrri,       X86::SHUFPSrmi, 16 },
-    { X86::SUB16rr,         X86::SUB16rm, 0 },
-    { X86::SUB32rr,         X86::SUB32rm, 0 },
-    { X86::SUB64rr,         X86::SUB64rm, 0 },
-    { X86::SUB8rr,          X86::SUB8rm, 0 },
-    { X86::SUBPDrr,         X86::SUBPDrm, 16 },
-    { X86::SUBPSrr,         X86::SUBPSrm, 16 },
-    { X86::SUBSDrr,         X86::SUBSDrm, 0 },
-    { X86::SUBSSrr,         X86::SUBSSrm, 0 },
+    { X86::ADC32rr,         X86::ADC32rm,       0 },
+    { X86::ADC64rr,         X86::ADC64rm,       0 },
+    { X86::ADD16rr,         X86::ADD16rm,       0 },
+    { X86::ADD16rr_DB,      X86::ADD16rm,       TB_NO_REVERSE },
+    { X86::ADD32rr,         X86::ADD32rm,       0 },
+    { X86::ADD32rr_DB,      X86::ADD32rm,       TB_NO_REVERSE },
+    { X86::ADD64rr,         X86::ADD64rm,       0 },
+    { X86::ADD64rr_DB,      X86::ADD64rm,       TB_NO_REVERSE },
+    { X86::ADD8rr,          X86::ADD8rm,        0 },
+    { X86::ADDPDrr,         X86::ADDPDrm,       TB_ALIGN_16 },
+    { X86::ADDPSrr,         X86::ADDPSrm,       TB_ALIGN_16 },
+    { X86::ADDSDrr,         X86::ADDSDrm,       0 },
+    { X86::ADDSSrr,         X86::ADDSSrm,       0 },
+    { X86::ADDSUBPDrr,      X86::ADDSUBPDrm,    TB_ALIGN_16 },
+    { X86::ADDSUBPSrr,      X86::ADDSUBPSrm,    TB_ALIGN_16 },
+    { X86::AND16rr,         X86::AND16rm,       0 },
+    { X86::AND32rr,         X86::AND32rm,       0 },
+    { X86::AND64rr,         X86::AND64rm,       0 },
+    { X86::AND8rr,          X86::AND8rm,        0 },
+    { X86::ANDNPDrr,        X86::ANDNPDrm,      TB_ALIGN_16 },
+    { X86::ANDNPSrr,        X86::ANDNPSrm,      TB_ALIGN_16 },
+    { X86::ANDPDrr,         X86::ANDPDrm,       TB_ALIGN_16 },
+    { X86::ANDPSrr,         X86::ANDPSrm,       TB_ALIGN_16 },
+    { X86::CMOVA16rr,       X86::CMOVA16rm,     0 },
+    { X86::CMOVA32rr,       X86::CMOVA32rm,     0 },
+    { X86::CMOVA64rr,       X86::CMOVA64rm,     0 },
+    { X86::CMOVAE16rr,      X86::CMOVAE16rm,    0 },
+    { X86::CMOVAE32rr,      X86::CMOVAE32rm,    0 },
+    { X86::CMOVAE64rr,      X86::CMOVAE64rm,    0 },
+    { X86::CMOVB16rr,       X86::CMOVB16rm,     0 },
+    { X86::CMOVB32rr,       X86::CMOVB32rm,     0 },
+    { X86::CMOVB64rr,       X86::CMOVB64rm,     0 },
+    { X86::CMOVBE16rr,      X86::CMOVBE16rm,    0 },
+    { X86::CMOVBE32rr,      X86::CMOVBE32rm,    0 },
+    { X86::CMOVBE64rr,      X86::CMOVBE64rm,    0 },
+    { X86::CMOVE16rr,       X86::CMOVE16rm,     0 },
+    { X86::CMOVE32rr,       X86::CMOVE32rm,     0 },
+    { X86::CMOVE64rr,       X86::CMOVE64rm,     0 },
+    { X86::CMOVG16rr,       X86::CMOVG16rm,     0 },
+    { X86::CMOVG32rr,       X86::CMOVG32rm,     0 },
+    { X86::CMOVG64rr,       X86::CMOVG64rm,     0 },
+    { X86::CMOVGE16rr,      X86::CMOVGE16rm,    0 },
+    { X86::CMOVGE32rr,      X86::CMOVGE32rm,    0 },
+    { X86::CMOVGE64rr,      X86::CMOVGE64rm,    0 },
+    { X86::CMOVL16rr,       X86::CMOVL16rm,     0 },
+    { X86::CMOVL32rr,       X86::CMOVL32rm,     0 },
+    { X86::CMOVL64rr,       X86::CMOVL64rm,     0 },
+    { X86::CMOVLE16rr,      X86::CMOVLE16rm,    0 },
+    { X86::CMOVLE32rr,      X86::CMOVLE32rm,    0 },
+    { X86::CMOVLE64rr,      X86::CMOVLE64rm,    0 },
+    { X86::CMOVNE16rr,      X86::CMOVNE16rm,    0 },
+    { X86::CMOVNE32rr,      X86::CMOVNE32rm,    0 },
+    { X86::CMOVNE64rr,      X86::CMOVNE64rm,    0 },
+    { X86::CMOVNO16rr,      X86::CMOVNO16rm,    0 },
+    { X86::CMOVNO32rr,      X86::CMOVNO32rm,    0 },
+    { X86::CMOVNO64rr,      X86::CMOVNO64rm,    0 },
+    { X86::CMOVNP16rr,      X86::CMOVNP16rm,    0 },
+    { X86::CMOVNP32rr,      X86::CMOVNP32rm,    0 },
+    { X86::CMOVNP64rr,      X86::CMOVNP64rm,    0 },
+    { X86::CMOVNS16rr,      X86::CMOVNS16rm,    0 },
+    { X86::CMOVNS32rr,      X86::CMOVNS32rm,    0 },
+    { X86::CMOVNS64rr,      X86::CMOVNS64rm,    0 },
+    { X86::CMOVO16rr,       X86::CMOVO16rm,     0 },
+    { X86::CMOVO32rr,       X86::CMOVO32rm,     0 },
+    { X86::CMOVO64rr,       X86::CMOVO64rm,     0 },
+    { X86::CMOVP16rr,       X86::CMOVP16rm,     0 },
+    { X86::CMOVP32rr,       X86::CMOVP32rm,     0 },
+    { X86::CMOVP64rr,       X86::CMOVP64rm,     0 },
+    { X86::CMOVS16rr,       X86::CMOVS16rm,     0 },
+    { X86::CMOVS32rr,       X86::CMOVS32rm,     0 },
+    { X86::CMOVS64rr,       X86::CMOVS64rm,     0 },
+    { X86::CMPPDrri,        X86::CMPPDrmi,      TB_ALIGN_16 },
+    { X86::CMPPSrri,        X86::CMPPSrmi,      TB_ALIGN_16 },
+    { X86::CMPSDrr,         X86::CMPSDrm,       0 },
+    { X86::CMPSSrr,         X86::CMPSSrm,       0 },
+    { X86::DIVPDrr,         X86::DIVPDrm,       TB_ALIGN_16 },
+    { X86::DIVPSrr,         X86::DIVPSrm,       TB_ALIGN_16 },
+    { X86::DIVSDrr,         X86::DIVSDrm,       0 },
+    { X86::DIVSSrr,         X86::DIVSSrm,       0 },
+    { X86::FsANDNPDrr,      X86::FsANDNPDrm,    TB_ALIGN_16 },
+    { X86::FsANDNPSrr,      X86::FsANDNPSrm,    TB_ALIGN_16 },
+    { X86::FsANDPDrr,       X86::FsANDPDrm,     TB_ALIGN_16 },
+    { X86::FsANDPSrr,       X86::FsANDPSrm,     TB_ALIGN_16 },
+    { X86::FsORPDrr,        X86::FsORPDrm,      TB_ALIGN_16 },
+    { X86::FsORPSrr,        X86::FsORPSrm,      TB_ALIGN_16 },
+    { X86::FsXORPDrr,       X86::FsXORPDrm,     TB_ALIGN_16 },
+    { X86::FsXORPSrr,       X86::FsXORPSrm,     TB_ALIGN_16 },
+    { X86::HADDPDrr,        X86::HADDPDrm,      TB_ALIGN_16 },
+    { X86::HADDPSrr,        X86::HADDPSrm,      TB_ALIGN_16 },
+    { X86::HSUBPDrr,        X86::HSUBPDrm,      TB_ALIGN_16 },
+    { X86::HSUBPSrr,        X86::HSUBPSrm,      TB_ALIGN_16 },
+    { X86::IMUL16rr,        X86::IMUL16rm,      0 },
+    { X86::IMUL32rr,        X86::IMUL32rm,      0 },
+    { X86::IMUL64rr,        X86::IMUL64rm,      0 },
+    { X86::Int_CMPSDrr,     X86::Int_CMPSDrm,   0 },
+    { X86::Int_CMPSSrr,     X86::Int_CMPSSrm,   0 },
+    { X86::MAXPDrr,         X86::MAXPDrm,       TB_ALIGN_16 },
+    { X86::MAXPDrr_Int,     X86::MAXPDrm_Int,   TB_ALIGN_16 },
+    { X86::MAXPSrr,         X86::MAXPSrm,       TB_ALIGN_16 },
+    { X86::MAXPSrr_Int,     X86::MAXPSrm_Int,   TB_ALIGN_16 },
+    { X86::MAXSDrr,         X86::MAXSDrm,       0 },
+    { X86::MAXSDrr_Int,     X86::MAXSDrm_Int,   0 },
+    { X86::MAXSSrr,         X86::MAXSSrm,       0 },
+    { X86::MAXSSrr_Int,     X86::MAXSSrm_Int,   0 },
+    { X86::MINPDrr,         X86::MINPDrm,       TB_ALIGN_16 },
+    { X86::MINPDrr_Int,     X86::MINPDrm_Int,   TB_ALIGN_16 },
+    { X86::MINPSrr,         X86::MINPSrm,       TB_ALIGN_16 },
+    { X86::MINPSrr_Int,     X86::MINPSrm_Int,   TB_ALIGN_16 },
+    { X86::MINSDrr,         X86::MINSDrm,       0 },
+    { X86::MINSDrr_Int,     X86::MINSDrm_Int,   0 },
+    { X86::MINSSrr,         X86::MINSSrm,       0 },
+    { X86::MINSSrr_Int,     X86::MINSSrm_Int,   0 },
+    { X86::MULPDrr,         X86::MULPDrm,       TB_ALIGN_16 },
+    { X86::MULPSrr,         X86::MULPSrm,       TB_ALIGN_16 },
+    { X86::MULSDrr,         X86::MULSDrm,       0 },
+    { X86::MULSSrr,         X86::MULSSrm,       0 },
+    { X86::OR16rr,          X86::OR16rm,        0 },
+    { X86::OR32rr,          X86::OR32rm,        0 },
+    { X86::OR64rr,          X86::OR64rm,        0 },
+    { X86::OR8rr,           X86::OR8rm,         0 },
+    { X86::ORPDrr,          X86::ORPDrm,        TB_ALIGN_16 },
+    { X86::ORPSrr,          X86::ORPSrm,        TB_ALIGN_16 },
+    { X86::PACKSSDWrr,      X86::PACKSSDWrm,    TB_ALIGN_16 },
+    { X86::PACKSSWBrr,      X86::PACKSSWBrm,    TB_ALIGN_16 },
+    { X86::PACKUSWBrr,      X86::PACKUSWBrm,    TB_ALIGN_16 },
+    { X86::PADDBrr,         X86::PADDBrm,       TB_ALIGN_16 },
+    { X86::PADDDrr,         X86::PADDDrm,       TB_ALIGN_16 },
+    { X86::PADDQrr,         X86::PADDQrm,       TB_ALIGN_16 },
+    { X86::PADDSBrr,        X86::PADDSBrm,      TB_ALIGN_16 },
+    { X86::PADDSWrr,        X86::PADDSWrm,      TB_ALIGN_16 },
+    { X86::PADDWrr,         X86::PADDWrm,       TB_ALIGN_16 },
+    { X86::PANDNrr,         X86::PANDNrm,       TB_ALIGN_16 },
+    { X86::PANDrr,          X86::PANDrm,        TB_ALIGN_16 },
+    { X86::PAVGBrr,         X86::PAVGBrm,       TB_ALIGN_16 },
+    { X86::PAVGWrr,         X86::PAVGWrm,       TB_ALIGN_16 },
+    { X86::PCMPEQBrr,       X86::PCMPEQBrm,     TB_ALIGN_16 },
+    { X86::PCMPEQDrr,       X86::PCMPEQDrm,     TB_ALIGN_16 },
+    { X86::PCMPEQWrr,       X86::PCMPEQWrm,     TB_ALIGN_16 },
+    { X86::PCMPGTBrr,       X86::PCMPGTBrm,     TB_ALIGN_16 },
+    { X86::PCMPGTDrr,       X86::PCMPGTDrm,     TB_ALIGN_16 },
+    { X86::PCMPGTWrr,       X86::PCMPGTWrm,     TB_ALIGN_16 },
+    { X86::PINSRWrri,       X86::PINSRWrmi,     TB_ALIGN_16 },
+    { X86::PMADDWDrr,       X86::PMADDWDrm,     TB_ALIGN_16 },
+    { X86::PMAXSWrr,        X86::PMAXSWrm,      TB_ALIGN_16 },
+    { X86::PMAXUBrr,        X86::PMAXUBrm,      TB_ALIGN_16 },
+    { X86::PMINSWrr,        X86::PMINSWrm,      TB_ALIGN_16 },
+    { X86::PMINUBrr,        X86::PMINUBrm,      TB_ALIGN_16 },
+    { X86::PMULDQrr,        X86::PMULDQrm,      TB_ALIGN_16 },
+    { X86::PMULHUWrr,       X86::PMULHUWrm,     TB_ALIGN_16 },
+    { X86::PMULHWrr,        X86::PMULHWrm,      TB_ALIGN_16 },
+    { X86::PMULLDrr,        X86::PMULLDrm,      TB_ALIGN_16 },
+    { X86::PMULLWrr,        X86::PMULLWrm,      TB_ALIGN_16 },
+    { X86::PMULUDQrr,       X86::PMULUDQrm,     TB_ALIGN_16 },
+    { X86::PORrr,           X86::PORrm,         TB_ALIGN_16 },
+    { X86::PSADBWrr,        X86::PSADBWrm,      TB_ALIGN_16 },
+    { X86::PSLLDrr,         X86::PSLLDrm,       TB_ALIGN_16 },
+    { X86::PSLLQrr,         X86::PSLLQrm,       TB_ALIGN_16 },
+    { X86::PSLLWrr,         X86::PSLLWrm,       TB_ALIGN_16 },
+    { X86::PSRADrr,         X86::PSRADrm,       TB_ALIGN_16 },
+    { X86::PSRAWrr,         X86::PSRAWrm,       TB_ALIGN_16 },
+    { X86::PSRLDrr,         X86::PSRLDrm,       TB_ALIGN_16 },
+    { X86::PSRLQrr,         X86::PSRLQrm,       TB_ALIGN_16 },
+    { X86::PSRLWrr,         X86::PSRLWrm,       TB_ALIGN_16 },
+    { X86::PSUBBrr,         X86::PSUBBrm,       TB_ALIGN_16 },
+    { X86::PSUBDrr,         X86::PSUBDrm,       TB_ALIGN_16 },
+    { X86::PSUBSBrr,        X86::PSUBSBrm,      TB_ALIGN_16 },
+    { X86::PSUBSWrr,        X86::PSUBSWrm,      TB_ALIGN_16 },
+    { X86::PSUBWrr,         X86::PSUBWrm,       TB_ALIGN_16 },
+    { X86::PUNPCKHBWrr,     X86::PUNPCKHBWrm,   TB_ALIGN_16 },
+    { X86::PUNPCKHDQrr,     X86::PUNPCKHDQrm,   TB_ALIGN_16 },
+    { X86::PUNPCKHQDQrr,    X86::PUNPCKHQDQrm,  TB_ALIGN_16 },
+    { X86::PUNPCKHWDrr,     X86::PUNPCKHWDrm,   TB_ALIGN_16 },
+    { X86::PUNPCKLBWrr,     X86::PUNPCKLBWrm,   TB_ALIGN_16 },
+    { X86::PUNPCKLDQrr,     X86::PUNPCKLDQrm,   TB_ALIGN_16 },
+    { X86::PUNPCKLQDQrr,    X86::PUNPCKLQDQrm,  TB_ALIGN_16 },
+    { X86::PUNPCKLWDrr,     X86::PUNPCKLWDrm,   TB_ALIGN_16 },
+    { X86::PXORrr,          X86::PXORrm,        TB_ALIGN_16 },
+    { X86::SBB32rr,         X86::SBB32rm,       0 },
+    { X86::SBB64rr,         X86::SBB64rm,       0 },
+    { X86::SHUFPDrri,       X86::SHUFPDrmi,     TB_ALIGN_16 },
+    { X86::SHUFPSrri,       X86::SHUFPSrmi,     TB_ALIGN_16 },
+    { X86::SUB16rr,         X86::SUB16rm,       0 },
+    { X86::SUB32rr,         X86::SUB32rm,       0 },
+    { X86::SUB64rr,         X86::SUB64rm,       0 },
+    { X86::SUB8rr,          X86::SUB8rm,        0 },
+    { X86::SUBPDrr,         X86::SUBPDrm,       TB_ALIGN_16 },
+    { X86::SUBPSrr,         X86::SUBPSrm,       TB_ALIGN_16 },
+    { X86::SUBSDrr,         X86::SUBSDrm,       0 },
+    { X86::SUBSSrr,         X86::SUBSSrm,       0 },
     // FIXME: TEST*rr -> swapped operand of TEST*mr.
-    { X86::UNPCKHPDrr,      X86::UNPCKHPDrm, 16 },
-    { X86::UNPCKHPSrr,      X86::UNPCKHPSrm, 16 },
-    { X86::UNPCKLPDrr,      X86::UNPCKLPDrm, 16 },
-    { X86::UNPCKLPSrr,      X86::UNPCKLPSrm, 16 },
-    { X86::XOR16rr,         X86::XOR16rm, 0 },
-    { X86::XOR32rr,         X86::XOR32rm, 0 },
-    { X86::XOR64rr,         X86::XOR64rm, 0 },
-    { X86::XOR8rr,          X86::XOR8rm, 0 },
-    { X86::XORPDrr,         X86::XORPDrm, 16 },
-    { X86::XORPSrr,         X86::XORPSrm, 16 }
+    { X86::UNPCKHPDrr,      X86::UNPCKHPDrm,    TB_ALIGN_16 },
+    { X86::UNPCKHPSrr,      X86::UNPCKHPSrm,    TB_ALIGN_16 },
+    { X86::UNPCKLPDrr,      X86::UNPCKLPDrm,    TB_ALIGN_16 },
+    { X86::UNPCKLPSrr,      X86::UNPCKLPSrm,    TB_ALIGN_16 },
+    { X86::XOR16rr,         X86::XOR16rm,       0 },
+    { X86::XOR32rr,         X86::XOR32rm,       0 },
+    { X86::XOR64rr,         X86::XOR64rm,       0 },
+    { X86::XOR8rr,          X86::XOR8rm,        0 },
+    { X86::XORPDrr,         X86::XORPDrm,       TB_ALIGN_16 },
+    { X86::XORPSrr,         X86::XORPSrm,       TB_ALIGN_16 },
+    // AVX 128-bit versions of foldable instructions
+    { X86::VCVTSD2SSrr,       X86::VCVTSD2SSrm,        0 },
+    { X86::Int_VCVTSD2SSrr,   X86::Int_VCVTSD2SSrm,    0 },
+    { X86::VCVTSI2SD64rr,     X86::VCVTSI2SD64rm,      0 },
+    { X86::Int_VCVTSI2SD64rr, X86::Int_VCVTSI2SD64rm,  0 },
+    { X86::VCVTSI2SDrr,       X86::VCVTSI2SDrm,        0 },
+    { X86::Int_VCVTSI2SDrr,   X86::Int_VCVTSI2SDrm,    0 },
+    { X86::VCVTSI2SS64rr,     X86::VCVTSI2SS64rm,      0 },
+    { X86::Int_VCVTSI2SS64rr, X86::Int_VCVTSI2SS64rm,  0 },
+    { X86::VCVTSI2SSrr,       X86::VCVTSI2SSrm,        0 },
+    { X86::Int_VCVTSI2SSrr,   X86::Int_VCVTSI2SSrm,    0 },
+    { X86::VCVTSS2SDrr,       X86::VCVTSS2SDrm,        0 },
+    { X86::Int_VCVTSS2SDrr,   X86::Int_VCVTSS2SDrm,    0 },
+    { X86::VCVTTSD2SI64rr,    X86::VCVTTSD2SI64rm,     0 },
+    { X86::Int_VCVTTSD2SI64rr,X86::Int_VCVTTSD2SI64rm, 0 },
+    { X86::VCVTTSD2SIrr,      X86::VCVTTSD2SIrm,       0 },
+    { X86::Int_VCVTTSD2SIrr,  X86::Int_VCVTTSD2SIrm,   0 },
+    { X86::VCVTTSS2SI64rr,    X86::VCVTTSS2SI64rm,     0 },
+    { X86::Int_VCVTTSS2SI64rr,X86::Int_VCVTTSS2SI64rm, 0 },
+    { X86::VCVTTSS2SIrr,      X86::VCVTTSS2SIrm,       0 },
+    { X86::Int_VCVTTSS2SIrr,  X86::Int_VCVTTSS2SIrm,   0 },
+    { X86::VCVTSD2SI64rr,     X86::VCVTSD2SI64rm,      0 },
+    { X86::VCVTSD2SIrr,       X86::VCVTSD2SIrm,        0 },
+    { X86::VCVTTPD2DQrr,      X86::VCVTTPD2DQrm,       TB_ALIGN_16 },
+    { X86::VCVTTPS2DQrr,      X86::VCVTTPS2DQrm,       TB_ALIGN_16 },
+    { X86::VRSQRTSSr,         X86::VRSQRTSSm,          0 },
+    { X86::VSQRTSDr,          X86::VSQRTSDm,           0 },
+    { X86::VSQRTSSr,          X86::VSQRTSSm,           0 },
+    { X86::VADDPDrr,          X86::VADDPDrm,           TB_ALIGN_16 },
+    { X86::VADDPSrr,          X86::VADDPSrm,           TB_ALIGN_16 },
+    { X86::VADDSDrr,          X86::VADDSDrm,           0 },
+    { X86::VADDSSrr,          X86::VADDSSrm,           0 },
+    { X86::VADDSUBPDrr,       X86::VADDSUBPDrm,        TB_ALIGN_16 },
+    { X86::VADDSUBPSrr,       X86::VADDSUBPSrm,        TB_ALIGN_16 },
+    { X86::VANDNPDrr,         X86::VANDNPDrm,          TB_ALIGN_16 },
+    { X86::VANDNPSrr,         X86::VANDNPSrm,          TB_ALIGN_16 },
+    { X86::VANDPDrr,          X86::VANDPDrm,           TB_ALIGN_16 },
+    { X86::VANDPSrr,          X86::VANDPSrm,           TB_ALIGN_16 },
+    { X86::VCMPPDrri,         X86::VCMPPDrmi,          TB_ALIGN_16 },
+    { X86::VCMPPSrri,         X86::VCMPPSrmi,          TB_ALIGN_16 },
+    { X86::VCMPSDrr,          X86::VCMPSDrm,           0 },
+    { X86::VCMPSSrr,          X86::VCMPSSrm,           0 },
+    { X86::VDIVPDrr,          X86::VDIVPDrm,           TB_ALIGN_16 },
+    { X86::VDIVPSrr,          X86::VDIVPSrm,           TB_ALIGN_16 },
+    { X86::VDIVSDrr,          X86::VDIVSDrm,           0 },
+    { X86::VDIVSSrr,          X86::VDIVSSrm,           0 },
+    { X86::VFsANDNPDrr,       X86::VFsANDNPDrm,        TB_ALIGN_16 },
+    { X86::VFsANDNPSrr,       X86::VFsANDNPSrm,        TB_ALIGN_16 },
+    { X86::VFsANDPDrr,        X86::VFsANDPDrm,         TB_ALIGN_16 },
+    { X86::VFsANDPSrr,        X86::VFsANDPSrm,         TB_ALIGN_16 },
+    { X86::VFsORPDrr,         X86::VFsORPDrm,          TB_ALIGN_16 },
+    { X86::VFsORPSrr,         X86::VFsORPSrm,          TB_ALIGN_16 },
+    { X86::VFsXORPDrr,        X86::VFsXORPDrm,         TB_ALIGN_16 },
+    { X86::VFsXORPSrr,        X86::VFsXORPSrm,         TB_ALIGN_16 },
+    { X86::VHADDPDrr,         X86::VHADDPDrm,          TB_ALIGN_16 },
+    { X86::VHADDPSrr,         X86::VHADDPSrm,          TB_ALIGN_16 },
+    { X86::VHSUBPDrr,         X86::VHSUBPDrm,          TB_ALIGN_16 },
+    { X86::VHSUBPSrr,         X86::VHSUBPSrm,          TB_ALIGN_16 },
+    { X86::Int_VCMPSDrr,      X86::Int_VCMPSDrm,       0 },
+    { X86::Int_VCMPSSrr,      X86::Int_VCMPSSrm,       0 },
+    { X86::VMAXPDrr,          X86::VMAXPDrm,           TB_ALIGN_16 },
+    { X86::VMAXPDrr_Int,      X86::VMAXPDrm_Int,       TB_ALIGN_16 },
+    { X86::VMAXPSrr,          X86::VMAXPSrm,           TB_ALIGN_16 },
+    { X86::VMAXPSrr_Int,      X86::VMAXPSrm_Int,       TB_ALIGN_16 },
+    { X86::VMAXSDrr,          X86::VMAXSDrm,           0 },
+    { X86::VMAXSDrr_Int,      X86::VMAXSDrm_Int,       0 },
+    { X86::VMAXSSrr,          X86::VMAXSSrm,           0 },
+    { X86::VMAXSSrr_Int,      X86::VMAXSSrm_Int,       0 },
+    { X86::VMINPDrr,          X86::VMINPDrm,           TB_ALIGN_16 },
+    { X86::VMINPDrr_Int,      X86::VMINPDrm_Int,       TB_ALIGN_16 },
+    { X86::VMINPSrr,          X86::VMINPSrm,           TB_ALIGN_16 },
+    { X86::VMINPSrr_Int,      X86::VMINPSrm_Int,       TB_ALIGN_16 },
+    { X86::VMINSDrr,          X86::VMINSDrm,           0 },
+    { X86::VMINSDrr_Int,      X86::VMINSDrm_Int,       0 },
+    { X86::VMINSSrr,          X86::VMINSSrm,           0 },
+    { X86::VMINSSrr_Int,      X86::VMINSSrm_Int,       0 },
+    { X86::VMULPDrr,          X86::VMULPDrm,           TB_ALIGN_16 },
+    { X86::VMULPSrr,          X86::VMULPSrm,           TB_ALIGN_16 },
+    { X86::VMULSDrr,          X86::VMULSDrm,           0 },
+    { X86::VMULSSrr,          X86::VMULSSrm,           0 },
+    { X86::VORPDrr,           X86::VORPDrm,            TB_ALIGN_16 },
+    { X86::VORPSrr,           X86::VORPSrm,            TB_ALIGN_16 },
+    { X86::VPACKSSDWrr,       X86::VPACKSSDWrm,        TB_ALIGN_16 },
+    { X86::VPACKSSWBrr,       X86::VPACKSSWBrm,        TB_ALIGN_16 },
+    { X86::VPACKUSWBrr,       X86::VPACKUSWBrm,        TB_ALIGN_16 },
+    { X86::VPADDBrr,          X86::VPADDBrm,           TB_ALIGN_16 },
+    { X86::VPADDDrr,          X86::VPADDDrm,           TB_ALIGN_16 },
+    { X86::VPADDQrr,          X86::VPADDQrm,           TB_ALIGN_16 },
+    { X86::VPADDSBrr,         X86::VPADDSBrm,          TB_ALIGN_16 },
+    { X86::VPADDSWrr,         X86::VPADDSWrm,          TB_ALIGN_16 },
+    { X86::VPADDWrr,          X86::VPADDWrm,           TB_ALIGN_16 },
+    { X86::VPANDNrr,          X86::VPANDNrm,           TB_ALIGN_16 },
+    { X86::VPANDrr,           X86::VPANDrm,            TB_ALIGN_16 },
+    { X86::VPCMPEQBrr,        X86::VPCMPEQBrm,         TB_ALIGN_16 },
+    { X86::VPCMPEQDrr,        X86::VPCMPEQDrm,         TB_ALIGN_16 },
+    { X86::VPCMPEQWrr,        X86::VPCMPEQWrm,         TB_ALIGN_16 },
+    { X86::VPCMPGTBrr,        X86::VPCMPGTBrm,         TB_ALIGN_16 },
+    { X86::VPCMPGTDrr,        X86::VPCMPGTDrm,         TB_ALIGN_16 },
+    { X86::VPCMPGTWrr,        X86::VPCMPGTWrm,         TB_ALIGN_16 },
+    { X86::VPINSRWrri,        X86::VPINSRWrmi,         TB_ALIGN_16 },
+    { X86::VPMADDWDrr,        X86::VPMADDWDrm,         TB_ALIGN_16 },
+    { X86::VPMAXSWrr,         X86::VPMAXSWrm,          TB_ALIGN_16 },
+    { X86::VPMAXUBrr,         X86::VPMAXUBrm,          TB_ALIGN_16 },
+    { X86::VPMINSWrr,         X86::VPMINSWrm,          TB_ALIGN_16 },
+    { X86::VPMINUBrr,         X86::VPMINUBrm,          TB_ALIGN_16 },
+    { X86::VPMULDQrr,         X86::VPMULDQrm,          TB_ALIGN_16 },
+    { X86::VPMULHUWrr,        X86::VPMULHUWrm,         TB_ALIGN_16 },
+    { X86::VPMULHWrr,         X86::VPMULHWrm,          TB_ALIGN_16 },
+    { X86::VPMULLDrr,         X86::VPMULLDrm,          TB_ALIGN_16 },
+    { X86::VPMULLWrr,         X86::VPMULLWrm,          TB_ALIGN_16 },
+    { X86::VPMULUDQrr,        X86::VPMULUDQrm,         TB_ALIGN_16 },
+    { X86::VPORrr,            X86::VPORrm,             TB_ALIGN_16 },
+    { X86::VPSADBWrr,         X86::VPSADBWrm,          TB_ALIGN_16 },
+    { X86::VPSLLDrr,          X86::VPSLLDrm,           TB_ALIGN_16 },
+    { X86::VPSLLQrr,          X86::VPSLLQrm,           TB_ALIGN_16 },
+    { X86::VPSLLWrr,          X86::VPSLLWrm,           TB_ALIGN_16 },
+    { X86::VPSRADrr,          X86::VPSRADrm,           TB_ALIGN_16 },
+    { X86::VPSRAWrr,          X86::VPSRAWrm,           TB_ALIGN_16 },
+    { X86::VPSRLDrr,          X86::VPSRLDrm,           TB_ALIGN_16 },
+    { X86::VPSRLQrr,          X86::VPSRLQrm,           TB_ALIGN_16 },
+    { X86::VPSRLWrr,          X86::VPSRLWrm,           TB_ALIGN_16 },
+    { X86::VPSUBBrr,          X86::VPSUBBrm,           TB_ALIGN_16 },
+    { X86::VPSUBDrr,          X86::VPSUBDrm,           TB_ALIGN_16 },
+    { X86::VPSUBSBrr,         X86::VPSUBSBrm,          TB_ALIGN_16 },
+    { X86::VPSUBSWrr,         X86::VPSUBSWrm,          TB_ALIGN_16 },
+    { X86::VPSUBWrr,          X86::VPSUBWrm,           TB_ALIGN_16 },
+    { X86::VPUNPCKHBWrr,      X86::VPUNPCKHBWrm,       TB_ALIGN_16 },
+    { X86::VPUNPCKHDQrr,      X86::VPUNPCKHDQrm,       TB_ALIGN_16 },
+    { X86::VPUNPCKHQDQrr,     X86::VPUNPCKHQDQrm,      TB_ALIGN_16 },
+    { X86::VPUNPCKHWDrr,      X86::VPUNPCKHWDrm,       TB_ALIGN_16 },
+    { X86::VPUNPCKLBWrr,      X86::VPUNPCKLBWrm,       TB_ALIGN_16 },
+    { X86::VPUNPCKLDQrr,      X86::VPUNPCKLDQrm,       TB_ALIGN_16 },
+    { X86::VPUNPCKLQDQrr,     X86::VPUNPCKLQDQrm,      TB_ALIGN_16 },
+    { X86::VPUNPCKLWDrr,      X86::VPUNPCKLWDrm,       TB_ALIGN_16 },
+    { X86::VPXORrr,           X86::VPXORrm,            TB_ALIGN_16 },
+    { X86::VSHUFPDrri,        X86::VSHUFPDrmi,         TB_ALIGN_16 },
+    { X86::VSHUFPSrri,        X86::VSHUFPSrmi,         TB_ALIGN_16 },
+    { X86::VSUBPDrr,          X86::VSUBPDrm,           TB_ALIGN_16 },
+    { X86::VSUBPSrr,          X86::VSUBPSrm,           TB_ALIGN_16 },
+    { X86::VSUBSDrr,          X86::VSUBSDrm,           0 },
+    { X86::VSUBSSrr,          X86::VSUBSSrm,           0 },
+    { X86::VUNPCKHPDrr,       X86::VUNPCKHPDrm,        TB_ALIGN_16 },
+    { X86::VUNPCKHPSrr,       X86::VUNPCKHPSrm,        TB_ALIGN_16 },
+    { X86::VUNPCKLPDrr,       X86::VUNPCKLPDrm,        TB_ALIGN_16 },
+    { X86::VUNPCKLPSrr,       X86::VUNPCKLPSrm,        TB_ALIGN_16 },
+    { X86::VXORPDrr,          X86::VXORPDrm,           TB_ALIGN_16 },
+    { X86::VXORPSrr,          X86::VXORPSrm,           TB_ALIGN_16 }
+    // FIXME: add AVX 256-bit foldable instructions
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl2); i != e; ++i) {
     unsigned RegOp = OpTbl2[i][0];
-    unsigned MemOp = OpTbl2[i][1] & ~TB_FLAGS;
-    unsigned Align = OpTbl2[i][2];
-
-    assert(!RegOp2MemOpTable2.count(RegOp) && "Duplicate entry!");
-    RegOp2MemOpTable2[RegOp] = std::make_pair(MemOp, Align);
-
-    // If this is not a reversible operation (because there is a many->one)
-    // mapping, don't insert the reverse of the operation into MemOp2RegOpTable.
-    if (OpTbl2[i][1] & TB_NOT_REVERSABLE)
-      continue;
-
-    // Index 2, folded load
-    unsigned AuxInfo = 2 | (1 << 4);
-    assert(!MemOp2RegOpTable.count(MemOp) &&
-           "Duplicated entries in unfolding maps?");
-    MemOp2RegOpTable[MemOp] = std::make_pair(RegOp, AuxInfo);
+    unsigned MemOp = OpTbl2[i][1];
+    unsigned Flags = OpTbl2[i][2];
+    AddTableEntry(RegOp2MemOpTable2, MemOp2RegOpTable,
+                  RegOp, MemOp,
+                  // Index 2, folded load
+                  Flags | TB_INDEX_2 | TB_FOLDED_LOAD);
   }
 }
 
+void
+X86InstrInfo::AddTableEntry(RegOp2MemOpTableType &R2MTable,
+                            MemOp2RegOpTableType &M2RTable,
+                            unsigned RegOp, unsigned MemOp, unsigned Flags) {
+    if ((Flags & TB_NO_FORWARD) == 0) {
+      assert(!R2MTable.count(RegOp) && "Duplicate entry!");
+      R2MTable[RegOp] = std::make_pair(MemOp, Flags);
+    }
+    if ((Flags & TB_NO_REVERSE) == 0) {
+      assert(!M2RTable.count(MemOp) &&
+           "Duplicated entries in unfolding maps?");
+      M2RTable[MemOp] = std::make_pair(RegOp, Flags);
+    }
+}
+
 bool
 X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
                                     unsigned &SrcReg, unsigned &DstReg,
@@ -796,6 +1000,11 @@ static bool isFrameLoadOpcode(int Opcode) {
   case X86::MOVAPSrm:
   case X86::MOVAPDrm:
   case X86::MOVDQArm:
+  case X86::VMOVSSrm:
+  case X86::VMOVSDrm:
+  case X86::VMOVAPSrm:
+  case X86::VMOVAPDrm:
+  case X86::VMOVDQArm:
   case X86::VMOVAPSYrm:
   case X86::VMOVAPDYrm:
   case X86::VMOVDQAYrm:
@@ -820,6 +1029,11 @@ static bool isFrameStoreOpcode(int Opcode) {
   case X86::MOVAPSmr:
   case X86::MOVAPDmr:
   case X86::MOVDQAmr:
+  case X86::VMOVSSmr:
+  case X86::VMOVSDmr:
+  case X86::VMOVAPSmr:
+  case X86::VMOVAPDmr:
+  case X86::VMOVDQAmr:
   case X86::VMOVAPSYmr:
   case X86::VMOVAPDYmr:
   case X86::VMOVDQAYmr:
@@ -852,24 +1066,6 @@ unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr *MI,
   return 0;
 }
 
-bool X86InstrInfo::hasLoadFromStackSlot(const MachineInstr *MI,
-                                        const MachineMemOperand *&MMO,
-                                        int &FrameIndex) const {
-  for (MachineInstr::mmo_iterator o = MI->memoperands_begin(),
-         oe = MI->memoperands_end();
-       o != oe;
-       ++o) {
-    if ((*o)->isLoad() && (*o)->getValue())
-      if (const FixedStackPseudoSourceValue *Value =
-          dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) {
-        FrameIndex = Value->getFrameIndex();
-        MMO = *o;
-        return true;
-      }
-  }
-  return false;
-}
-
 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr *MI,
                                           int &FrameIndex) const {
   if (isFrameStoreOpcode(MI->getOpcode()))
@@ -892,24 +1088,6 @@ unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr *MI,
   return 0;
 }
 
-bool X86InstrInfo::hasStoreToStackSlot(const MachineInstr *MI,
-                                       const MachineMemOperand *&MMO,
-                                       int &FrameIndex) const {
-  for (MachineInstr::mmo_iterator o = MI->memoperands_begin(),
-         oe = MI->memoperands_end();
-       o != oe;
-       ++o) {
-    if ((*o)->isStore() && (*o)->getValue())
-      if (const FixedStackPseudoSourceValue *Value =
-          dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) {
-        FrameIndex = Value->getFrameIndex();
-        MMO = *o;
-        return true;
-      }
-  }
-  return false;
-}
-
 /// regIsPICBase - Return true if register is PIC base (i.e.g defined by
 /// X86::MOVPC32r.
 static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI) {
@@ -941,12 +1119,20 @@ X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr *MI,
     case X86::MOVUPSrm:
     case X86::MOVAPDrm:
     case X86::MOVDQArm:
+    case X86::VMOVSSrm:
+    case X86::VMOVSDrm:
+    case X86::VMOVAPSrm:
+    case X86::VMOVUPSrm:
+    case X86::VMOVAPDrm:
+    case X86::VMOVDQArm:
     case X86::VMOVAPSYrm:
     case X86::VMOVUPSYrm:
     case X86::VMOVAPDYrm:
     case X86::VMOVDQAYrm:
     case X86::MMX_MOVD64rm:
     case X86::MMX_MOVQ64rm:
+    case X86::FsVMOVAPSrm:
+    case X86::FsVMOVAPDrm:
     case X86::FsMOVAPSrm:
     case X86::FsMOVAPDrm: {
       // Loads from constant pools are trivially rematerializable.
@@ -1009,15 +1195,11 @@ static bool isSafeToClobberEFLAGS(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator I) {
   MachineBasicBlock::iterator E = MBB.end();
 
-  // It's always safe to clobber EFLAGS at the end of a block.
-  if (I == E)
-    return true;
-
   // For compile time consideration, if we are not able to determine the
   // safety after visiting 4 instructions in each direction, we will assume
   // it's not safe.
   MachineBasicBlock::iterator Iter = I;
-  for (unsigned i = 0; i < 4; ++i) {
+  for (unsigned i = 0; Iter != E && i < 4; ++i) {
     bool SeenDef = false;
     for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) {
       MachineOperand &MO = Iter->getOperand(j);
@@ -1037,10 +1219,16 @@ static bool isSafeToClobberEFLAGS(MachineBasicBlock &MBB,
     // Skip over DBG_VALUE.
     while (Iter != E && Iter->isDebugValue())
       ++Iter;
+  }
 
-    // If we make it to the end of the block, it's safe to clobber EFLAGS.
-    if (Iter == E)
-      return true;
+  // It is safe to clobber EFLAGS at the end of a block of no successor has it
+  // live in.
+  if (Iter == E) {
+    for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
+           SE = MBB.succ_end(); SI != SE; ++SI)
+      if ((*SI)->isLiveIn(X86::EFLAGS))
+        return false;
+    return true;
   }
 
   MachineBasicBlock::iterator B = MBB.begin();
@@ -1946,7 +2134,8 @@ static bool isHReg(unsigned Reg) {
 }
 
 // Try and copy between VR128/VR64 and GR64 registers.
-static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg) {
+static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
+                                        bool HasAVX) {
   // SrcReg(VR128) -> DestReg(GR64)
   // SrcReg(VR64)  -> DestReg(GR64)
   // SrcReg(GR64)  -> DestReg(VR128)
@@ -1955,7 +2144,7 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg) {
   if (X86::GR64RegClass.contains(DestReg)) {
     if (X86::VR128RegClass.contains(SrcReg)) {
       // Copy from a VR128 register to a GR64 register.
-      return X86::MOVPQIto64rr;
+      return HasAVX ? X86::VMOVPQIto64rr : X86::MOVPQIto64rr;
     } else if (X86::VR64RegClass.contains(SrcReg)) {
       // Copy from a VR64 register to a GR64 register.
       return X86::MOVSDto64rr;
@@ -1963,12 +2152,23 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg) {
   } else if (X86::GR64RegClass.contains(SrcReg)) {
     // Copy from a GR64 register to a VR128 register.
     if (X86::VR128RegClass.contains(DestReg))
-      return X86::MOV64toPQIrr;
+      return HasAVX ? X86::VMOV64toPQIrr : X86::MOV64toPQIrr;
     // Copy from a GR64 register to a VR64 register.
     else if (X86::VR64RegClass.contains(DestReg))
       return X86::MOV64toSDrr;
   }
 
+  // SrcReg(FR32) -> DestReg(GR32)
+  // SrcReg(GR32) -> DestReg(FR32)
+
+  if (X86::GR32RegClass.contains(DestReg) && X86::FR32RegClass.contains(SrcReg))
+      // Copy from a FR32 register to a GR32 register.
+      return HasAVX ? X86::VMOVSS2DIrr : X86::MOVSS2DIrr;
+
+  if (X86::FR32RegClass.contains(DestReg) && X86::GR32RegClass.contains(SrcReg))
+      // Copy from a GR32 register to a FR32 register.
+      return HasAVX ? X86::VMOVDI2SSrr : X86::MOVDI2SSrr;
+
   return 0;
 }
 
@@ -1977,6 +2177,7 @@ void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                unsigned DestReg, unsigned SrcReg,
                                bool KillSrc) const {
   // First deal with the normal symmetric copies.
+  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
   unsigned Opc = 0;
   if (X86::GR64RegClass.contains(DestReg, SrcReg))
     Opc = X86::MOV64rr;
@@ -1988,18 +2189,21 @@ void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     // Copying to or from a physical H register on x86-64 requires a NOREX
     // move.  Otherwise use a normal move.
     if ((isHReg(DestReg) || isHReg(SrcReg)) &&
-        TM.getSubtarget<X86Subtarget>().is64Bit())
+        TM.getSubtarget<X86Subtarget>().is64Bit()) {
       Opc = X86::MOV8rr_NOREX;
-    else
+      // Both operands must be encodable without an REX prefix.
+      assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
+             "8-bit H register can not be copied outside GR8_NOREX");
+    } else
       Opc = X86::MOV8rr;
   } else if (X86::VR128RegClass.contains(DestReg, SrcReg))
-    Opc = X86::MOVAPSrr;
+    Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
   else if (X86::VR256RegClass.contains(DestReg, SrcReg))
     Opc = X86::VMOVAPSYrr;
   else if (X86::VR64RegClass.contains(DestReg, SrcReg))
     Opc = X86::MMX_MOVQ64rr;
   else
-    Opc = CopyToFromAsymmetricReg(DestReg, SrcReg);
+    Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, HasAVX);
 
   if (Opc) {
     BuildMI(MBB, MI, DL, get(Opc), DestReg)
@@ -2043,6 +2247,7 @@ static unsigned getLoadStoreRegOpcode(unsigned Reg,
                                       bool isStackAligned,
                                       const TargetMachine &TM,
                                       bool load) {
+  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
   switch (RC->getSize()) {
   default:
     llvm_unreachable("Unknown spill size");
@@ -2061,7 +2266,9 @@ static unsigned getLoadStoreRegOpcode(unsigned Reg,
     if (X86::GR32RegClass.hasSubClassEq(RC))
       return load ? X86::MOV32rm : X86::MOV32mr;
     if (X86::FR32RegClass.hasSubClassEq(RC))
-      return load ? X86::MOVSSrm : X86::MOVSSmr;
+      return load ?
+        (HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) :
+        (HasAVX ? X86::VMOVSSmr : X86::MOVSSmr);
     if (X86::RFP32RegClass.hasSubClassEq(RC))
       return load ? X86::LD_Fp32m : X86::ST_Fp32m;
     llvm_unreachable("Unknown 4-byte regclass");
@@ -2069,7 +2276,9 @@ static unsigned getLoadStoreRegOpcode(unsigned Reg,
     if (X86::GR64RegClass.hasSubClassEq(RC))
       return load ? X86::MOV64rm : X86::MOV64mr;
     if (X86::FR64RegClass.hasSubClassEq(RC))
-      return load ? X86::MOVSDrm : X86::MOVSDmr;
+      return load ?
+        (HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) :
+        (HasAVX ? X86::VMOVSDmr : X86::MOVSDmr);
     if (X86::VR64RegClass.hasSubClassEq(RC))
       return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
     if (X86::RFP64RegClass.hasSubClassEq(RC))
@@ -2078,13 +2287,18 @@ static unsigned getLoadStoreRegOpcode(unsigned Reg,
   case 10:
     assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
     return load ? X86::LD_Fp80m : X86::ST_FpP80m;
-  case 16:
+  case 16: {
     assert(X86::VR128RegClass.hasSubClassEq(RC) && "Unknown 16-byte regclass");
     // If stack is realigned we can use aligned stores.
     if (isStackAligned)
-      return load ? X86::MOVAPSrm : X86::MOVAPSmr;
+      return load ?
+        (HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) :
+        (HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
     else
-      return load ? X86::MOVUPSrm : X86::MOVUPSmr;
+      return load ?
+        (HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) :
+        (HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
+  }
   case 32:
     assert(X86::VR256RegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
     // If stack is realigned we can use aligned stores.
@@ -2118,7 +2332,8 @@ void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   const MachineFunction &MF = *MBB.getParent();
   assert(MF.getFrameInfo()->getObjectSize(FrameIdx) >= RC->getSize() &&
          "Stack slot too small for store");
-  bool isAligned = (TM.getFrameLowering()->getStackAlignment() >= 16) ||
+  unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
+  bool isAligned = (TM.getFrameLowering()->getStackAlignment() >= Alignment) ||
     RI.canRealignStack(MF);
   unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, TM);
   DebugLoc DL = MBB.findDebugLoc(MI);
@@ -2133,7 +2348,9 @@ void X86InstrInfo::storeRegToAddr(MachineFunction &MF, unsigned SrcReg,
                                   MachineInstr::mmo_iterator MMOBegin,
                                   MachineInstr::mmo_iterator MMOEnd,
                                   SmallVectorImpl<MachineInstr*> &NewMIs) const {
-  bool isAligned = MMOBegin != MMOEnd && (*MMOBegin)->getAlignment() >= 16;
+  unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
+  bool isAligned = MMOBegin != MMOEnd &&
+                   (*MMOBegin)->getAlignment() >= Alignment;
   unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, TM);
   DebugLoc DL;
   MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc));
@@ -2151,7 +2368,8 @@ void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                         const TargetRegisterClass *RC,
                                         const TargetRegisterInfo *TRI) const {
   const MachineFunction &MF = *MBB.getParent();
-  bool isAligned = (TM.getFrameLowering()->getStackAlignment() >= 16) ||
+  unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
+  bool isAligned = (TM.getFrameLowering()->getStackAlignment() >= Alignment) ||
     RI.canRealignStack(MF);
   unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, TM);
   DebugLoc DL = MBB.findDebugLoc(MI);
@@ -2164,7 +2382,9 @@ void X86InstrInfo::loadRegFromAddr(MachineFunction &MF, unsigned DestReg,
                                  MachineInstr::mmo_iterator MMOBegin,
                                  MachineInstr::mmo_iterator MMOEnd,
                                  SmallVectorImpl<MachineInstr*> &NewMIs) const {
-  bool isAligned = MMOBegin != MMOEnd && (*MMOBegin)->getAlignment() >= 16;
+  unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
+  bool isAligned = MMOBegin != MMOEnd &&
+                   (*MMOBegin)->getAlignment() >= Alignment;
   unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, TM);
   DebugLoc DL;
   MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), DestReg);
@@ -2174,6 +2394,40 @@ void X86InstrInfo::loadRegFromAddr(MachineFunction &MF, unsigned DestReg,
   NewMIs.push_back(MIB);
 }
 
+/// Expand2AddrUndef - Expand a single-def pseudo instruction to a two-addr
+/// instruction with two undef reads of the register being defined.  This is
+/// used for mapping:
+///   %xmm4 = V_SET0
+/// to:
+///   %xmm4 = PXORrr %xmm4<undef>, %xmm4<undef>
+///
+static bool Expand2AddrUndef(MachineInstr *MI, const MCInstrDesc &Desc) {
+  assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
+  unsigned Reg = MI->getOperand(0).getReg();
+  MI->setDesc(Desc);
+
+  // MachineInstr::addOperand() will insert explicit operands before any
+  // implicit operands.
+  MachineInstrBuilder(MI).addReg(Reg, RegState::Undef)
+                         .addReg(Reg, RegState::Undef);
+  // But we don't trust that.
+  assert(MI->getOperand(1).getReg() == Reg &&
+         MI->getOperand(2).getReg() == Reg && "Misplaced operand");
+  return true;
+}
+
+bool X86InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
+  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
+  switch (MI->getOpcode()) {
+  case X86::V_SET0:
+    return Expand2AddrUndef(MI, get(HasAVX ? X86::VPXORrr : X86::PXORrr));
+  case X86::TEST8ri_NOREX:
+    MI->setDesc(get(X86::TEST8ri));
+    return true;
+  }
+  return false;
+}
+
 MachineInstr*
 X86InstrInfo::emitFrameIndexDebugValue(MachineFunction &MF,
                                        int FrameIx, uint64_t Offset,
@@ -2305,7 +2559,7 @@ X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
       OpcodeTablePtr->find(MI->getOpcode());
     if (I != OpcodeTablePtr->end()) {
       unsigned Opcode = I->second.first;
-      unsigned MinAlign = I->second.second;
+      unsigned MinAlign = (I->second.second & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT;
       if (Align < MinAlign)
         return NULL;
       bool NarrowToMOV32rm = false;
@@ -2352,6 +2606,51 @@ X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
   return NULL;
 }
 
+/// hasPartialRegUpdate - Return true for all instructions that only update
+/// the first 32 or 64-bits of the destination register and leave the rest
+/// unmodified. This can be used to avoid folding loads if the instructions
+/// only update part of the destination register, and the non-updated part is
+/// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these
+/// instructions breaks the partial register dependency and it can improve
+/// performance. e.g.:
+///
+///   movss (%rdi), %xmm0
+///   cvtss2sd %xmm0, %xmm0
+///
+/// Instead of
+///   cvtss2sd (%rdi), %xmm0
+///
+/// FIXME: This should be turned into a TSFlags.
+///
+static bool hasPartialRegUpdate(unsigned Opcode) {
+  switch (Opcode) {
+  case X86::CVTSD2SSrr:
+  case X86::Int_CVTSD2SSrr:
+  case X86::CVTSS2SDrr:
+  case X86::Int_CVTSS2SDrr:
+  case X86::RCPSSr:
+  case X86::RCPSSr_Int:
+  case X86::ROUNDSDr:
+  case X86::ROUNDSSr:
+  case X86::RSQRTSSr:
+  case X86::RSQRTSSr_Int:
+  case X86::SQRTSSr:
+  case X86::SQRTSSr_Int:
+  // AVX encoded versions
+  case X86::VCVTSD2SSrr:
+  case X86::Int_VCVTSD2SSrr:
+  case X86::VCVTSS2SDrr:
+  case X86::Int_VCVTSS2SDrr:
+  case X86::VRCPSSr:
+  case X86::VROUNDSDr:
+  case X86::VROUNDSSr:
+  case X86::VRSQRTSSr:
+  case X86::VSQRTSSr:
+    return true;
+  }
+
+  return false;
+}
 
 MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
                                                   MachineInstr *MI,
@@ -2360,22 +2659,11 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
   // Check switch flag
   if (NoFusing) return NULL;
 
-  if (!MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
-    switch (MI->getOpcode()) {
-    case X86::CVTSD2SSrr:
-    case X86::Int_CVTSD2SSrr:
-    case X86::CVTSS2SDrr:
-    case X86::Int_CVTSS2SDrr:
-    case X86::RCPSSr:
-    case X86::RCPSSr_Int:
-    case X86::ROUNDSDr:
-    case X86::ROUNDSSr:
-    case X86::RSQRTSSr:
-    case X86::RSQRTSSr_Int:
-    case X86::SQRTSSr:
-    case X86::SQRTSSr_Int:
-      return 0;
-    }
+  // Unless optimizing for size, don't fold to avoid partial
+  // register update stalls
+  if (!MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize) &&
+      hasPartialRegUpdate(MI->getOpcode()))
+    return 0;
 
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   unsigned Size = MFI->getObjectSize(FrameIndex);
@@ -2412,22 +2700,11 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
   // Check switch flag
   if (NoFusing) return NULL;
 
-  if (!MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
-    switch (MI->getOpcode()) {
-    case X86::CVTSD2SSrr:
-    case X86::Int_CVTSD2SSrr:
-    case X86::CVTSS2SDrr:
-    case X86::Int_CVTSS2SDrr:
-    case X86::RCPSSr:
-    case X86::RCPSSr_Int:
-    case X86::ROUNDSDr:
-    case X86::ROUNDSSr:
-    case X86::RSQRTSSr:
-    case X86::RSQRTSSr_Int:
-    case X86::SQRTSSr:
-    case X86::SQRTSSr_Int:
-      return 0;
-    }
+  // Unless optimizing for size, don't fold to avoid partial
+  // register update stalls
+  if (!MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize) &&
+      hasPartialRegUpdate(MI->getOpcode()))
+    return 0;
 
   // Determine the alignment of the load.
   unsigned Alignment = 0;
@@ -2439,13 +2716,9 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     case X86::AVX_SET0PDY:
       Alignment = 32;
       break;
-    case X86::V_SET0PS:
-    case X86::V_SET0PD:
-    case X86::V_SET0PI:
+    case X86::V_SET0:
     case X86::V_SETALLONES:
-    case X86::AVX_SET0PS:
-    case X86::AVX_SET0PD:
-    case X86::AVX_SET0PI:
+    case X86::AVX_SETALLONES:
       Alignment = 16;
       break;
     case X86::FsFLD0SD:
@@ -2481,18 +2754,16 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
 
   SmallVector<MachineOperand,X86::AddrNumOperands> MOs;
   switch (LoadMI->getOpcode()) {
-  case X86::V_SET0PS:
-  case X86::V_SET0PD:
-  case X86::V_SET0PI:
+  case X86::V_SET0:
   case X86::V_SETALLONES:
-  case X86::AVX_SET0PS:
-  case X86::AVX_SET0PD:
-  case X86::AVX_SET0PI:
   case X86::AVX_SET0PSY:
   case X86::AVX_SET0PDY:
+  case X86::AVX_SETALLONES:
   case X86::FsFLD0SD:
-  case X86::FsFLD0SS: {
-    // Folding a V_SET0P? or V_SETALLONES as a load, to ease register pressure.
+  case X86::FsFLD0SS:
+  case X86::VFsFLD0SD:
+  case X86::VFsFLD0SS: {
+    // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure.
     // Create a constant-pool entry and operands to load from it.
 
     // Medium and large mode can't fold loads this way.
@@ -2515,7 +2786,7 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
 
     // Create a constant-pool entry.
     MachineConstantPool &MCP = *MF.getConstantPool();
-    const Type *Ty;
+    Type *Ty;
     unsigned Opc = LoadMI->getOpcode();
     if (Opc == X86::FsFLD0SS || Opc == X86::VFsFLD0SS)
       Ty = Type::getFloatTy(MF.getFunction()->getContext());
@@ -2525,9 +2796,10 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
       Ty = VectorType::get(Type::getFloatTy(MF.getFunction()->getContext()), 8);
     else
       Ty = VectorType::get(Type::getInt32Ty(MF.getFunction()->getContext()), 4);
-    const Constant *C = LoadMI->getOpcode() == X86::V_SETALLONES ?
-                    Constant::getAllOnesValue(Ty) :
-                    Constant::getNullValue(Ty);
+
+    bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX_SETALLONES);
+    const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) :
+                                    Constant::getNullValue(Ty);
     unsigned CPI = MCP.getConstantPoolIndex(C, Alignment);
 
     // Create operands to load from the constant pool entry.
@@ -2615,9 +2887,9 @@ bool X86InstrInfo::unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
   if (I == MemOp2RegOpTable.end())
     return false;
   unsigned Opc = I->second.first;
-  unsigned Index = I->second.second & 0xf;
-  bool FoldedLoad = I->second.second & (1 << 4);
-  bool FoldedStore = I->second.second & (1 << 5);
+  unsigned Index = I->second.second & TB_INDEX_MASK;
+  bool FoldedLoad = I->second.second & TB_FOLDED_LOAD;
+  bool FoldedStore = I->second.second & TB_FOLDED_STORE;
   if (UnfoldLoad && !FoldedLoad)
     return false;
   UnfoldLoad &= FoldedLoad;
@@ -2743,9 +3015,9 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
   if (I == MemOp2RegOpTable.end())
     return false;
   unsigned Opc = I->second.first;
-  unsigned Index = I->second.second & 0xf;
-  bool FoldedLoad = I->second.second & (1 << 4);
-  bool FoldedStore = I->second.second & (1 << 5);
+  unsigned Index = I->second.second & TB_INDEX_MASK;
+  bool FoldedLoad = I->second.second & TB_FOLDED_LOAD;
+  bool FoldedStore = I->second.second & TB_FOLDED_STORE;
   const MCInstrDesc &MCID = get(Opc);
   const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI);
   unsigned NumDefs = MCID.NumDefs;
@@ -2780,7 +3052,9 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
         !TM.getSubtarget<X86Subtarget>().isUnalignedMemAccessFast())
       // Do not introduce a slow unaligned load.
       return false;
-    bool isAligned = (*MMOs.first) && (*MMOs.first)->getAlignment() >= 16;
+    unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
+    bool isAligned = (*MMOs.first) &&
+                     (*MMOs.first)->getAlignment() >= Alignment;
     Load = DAG.getMachineNode(getLoadRegOpcode(0, RC, isAligned, TM), dl,
                               VT, MVT::Other, &AddrOps[0], AddrOps.size());
     NewNodes.push_back(Load);
@@ -2822,7 +3096,9 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
         !TM.getSubtarget<X86Subtarget>().isUnalignedMemAccessFast())
       // Do not introduce a slow unaligned store.
       return false;
-    bool isAligned = (*MMOs.first) && (*MMOs.first)->getAlignment() >= 16;
+    unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
+    bool isAligned = (*MMOs.first) &&
+                     (*MMOs.first)->getAlignment() >= Alignment;
     SDNode *Store = DAG.getMachineNode(getStoreRegOpcode(0, DstRC,
                                                          isAligned, TM),
                                        dl, MVT::Other,
@@ -2843,14 +3119,14 @@ unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc,
     MemOp2RegOpTable.find(Opc);
   if (I == MemOp2RegOpTable.end())
     return 0;
-  bool FoldedLoad = I->second.second & (1 << 4);
-  bool FoldedStore = I->second.second & (1 << 5);
+  bool FoldedLoad = I->second.second & TB_FOLDED_LOAD;
+  bool FoldedStore = I->second.second & TB_FOLDED_STORE;
   if (UnfoldLoad && !FoldedLoad)
     return 0;
   if (UnfoldStore && !FoldedStore)
     return 0;
   if (LoadRegIndex)
-    *LoadRegIndex = I->second.second & 0xf;
+    *LoadRegIndex = I->second.second & TB_INDEX_MASK;
   return I->second.first;
 }
 
@@ -2881,6 +3157,16 @@ X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
   case X86::MOVAPDrm:
   case X86::MOVDQArm:
   case X86::MOVDQUrm:
+  // AVX load instructions
+  case X86::VMOVSSrm:
+  case X86::VMOVSDrm:
+  case X86::FsVMOVAPSrm:
+  case X86::FsVMOVAPDrm:
+  case X86::VMOVAPSrm:
+  case X86::VMOVUPSrm:
+  case X86::VMOVAPDrm:
+  case X86::VMOVDQArm:
+  case X86::VMOVDQUrm:
   case X86::VMOVAPSYrm:
   case X86::VMOVUPSYrm:
   case X86::VMOVAPDYrm:
@@ -2908,6 +3194,16 @@ X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
   case X86::MOVAPDrm:
   case X86::MOVDQArm:
   case X86::MOVDQUrm:
+  // AVX load instructions
+  case X86::VMOVSSrm:
+  case X86::VMOVSDrm:
+  case X86::FsVMOVAPSrm:
+  case X86::FsVMOVAPDrm:
+  case X86::VMOVAPSrm:
+  case X86::VMOVUPSrm:
+  case X86::VMOVAPDrm:
+  case X86::VMOVDQArm:
+  case X86::VMOVDQUrm:
   case X86::VMOVAPSYrm:
   case X86::VMOVUPSYrm:
   case X86::VMOVAPDYrm:
@@ -3007,31 +3303,6 @@ isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const {
            RC == &X86::RFP64RegClass || RC == &X86::RFP80RegClass);
 }
 
-
-/// isX86_64ExtendedReg - Is the MachineOperand a x86-64 extended (r8 or higher)
-/// register?  e.g. r8, xmm8, xmm13, etc.
-bool X86InstrInfo::isX86_64ExtendedReg(unsigned RegNo) {
-  switch (RegNo) {
-  default: break;
-  case X86::R8:    case X86::R9:    case X86::R10:   case X86::R11:
-  case X86::R12:   case X86::R13:   case X86::R14:   case X86::R15:
-  case X86::R8D:   case X86::R9D:   case X86::R10D:  case X86::R11D:
-  case X86::R12D:  case X86::R13D:  case X86::R14D:  case X86::R15D:
-  case X86::R8W:   case X86::R9W:   case X86::R10W:  case X86::R11W:
-  case X86::R12W:  case X86::R13W:  case X86::R14W:  case X86::R15W:
-  case X86::R8B:   case X86::R9B:   case X86::R10B:  case X86::R11B:
-  case X86::R12B:  case X86::R13B:  case X86::R14B:  case X86::R15B:
-  case X86::XMM8:  case X86::XMM9:  case X86::XMM10: case X86::XMM11:
-  case X86::XMM12: case X86::XMM13: case X86::XMM14: case X86::XMM15:
-  case X86::YMM8:  case X86::YMM9:  case X86::YMM10: case X86::YMM11:
-  case X86::YMM12: case X86::YMM13: case X86::YMM14: case X86::YMM15:
-  case X86::CR8:   case X86::CR9:   case X86::CR10:  case X86::CR11:
-  case X86::CR12:  case X86::CR13:  case X86::CR14:  case X86::CR15:
-    return true;
-  }
-  return false;
-}
-
 /// getGlobalBaseReg - Return a virtual register initialized with the
 /// the global base register value. Output instructions required to
 /// initialize the register in the function entry block, if necessary.
@@ -3072,7 +3343,6 @@ static const unsigned ReplaceableInstrs[][3] = {
   { X86::ANDPSrr,    X86::ANDPDrr,   X86::PANDrr    },
   { X86::ORPSrm,     X86::ORPDrm,    X86::PORrm     },
   { X86::ORPSrr,     X86::ORPDrr,    X86::PORrr     },
-  { X86::V_SET0PS,   X86::V_SET0PD,  X86::V_SET0PI  },
   { X86::XORPSrm,    X86::XORPDrm,   X86::PXORrm    },
   { X86::XORPSrr,    X86::XORPDrr,   X86::PXORrr    },
   // AVX 128-bit support
@@ -3088,7 +3358,6 @@ static const unsigned ReplaceableInstrs[][3] = {
   { X86::VANDPSrr,   X86::VANDPDrr,   X86::VPANDrr    },
   { X86::VORPSrm,    X86::VORPDrm,    X86::VPORrm     },
   { X86::VORPSrr,    X86::VORPDrr,    X86::VPORrr     },
-  { X86::AVX_SET0PS, X86::AVX_SET0PD, X86::AVX_SET0PI },
   { X86::VXORPSrm,   X86::VXORPDrm,   X86::VPXORrm    },
   { X86::VXORPSrr,   X86::VXORPDrr,   X86::VPXORrr    },
   // AVX 256-bit support
@@ -3111,13 +3380,13 @@ static const unsigned *lookup(unsigned opcode, unsigned domain) {
 }
 
 std::pair<uint16_t, uint16_t>
-X86InstrInfo::GetSSEDomain(const MachineInstr *MI) const {
+X86InstrInfo::getExecutionDomain(const MachineInstr *MI) const {
   uint16_t domain = (MI->getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
   return std::make_pair(domain,
                         domain && lookup(MI->getOpcode(), domain) ? 0xe : 0);
 }
 
-void X86InstrInfo::SetSSEDomain(MachineInstr *MI, unsigned Domain) const {
+void X86InstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const {
   assert(Domain>0 && Domain<4 && "Invalid execution domain");
   uint16_t dom = (MI->getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
   assert(dom && "Not an SSE instruction");
@@ -3158,6 +3427,29 @@ bool X86InstrInfo::isHighLatencyDef(int opc) const {
   case X86::SQRTSSm_Int:
   case X86::SQRTSSr:
   case X86::SQRTSSr_Int:
+  // AVX instructions with high latency
+  case X86::VDIVSDrm:
+  case X86::VDIVSDrm_Int:
+  case X86::VDIVSDrr:
+  case X86::VDIVSDrr_Int:
+  case X86::VDIVSSrm:
+  case X86::VDIVSSrm_Int:
+  case X86::VDIVSSrr:
+  case X86::VDIVSSrr_Int:
+  case X86::VSQRTPDm:
+  case X86::VSQRTPDm_Int:
+  case X86::VSQRTPDr:
+  case X86::VSQRTPDr_Int:
+  case X86::VSQRTPSm:
+  case X86::VSQRTPSm_Int:
+  case X86::VSQRTPSr:
+  case X86::VSQRTPSr_Int:
+  case X86::VSQRTSDm:
+  case X86::VSQRTSDm_Int:
+  case X86::VSQRTSDr:
+  case X86::VSQRTSSm:
+  case X86::VSQRTSSm_Int:
+  case X86::VSQRTSSr:
     return true;
   }
 }
diff --git a/contrib/llvm/lib/Target/X86/X86InstrInfo.h b/contrib/llvm/lib/Target/X86/X86InstrInfo.h
index 5f2eba34ac45..97009dbdbe50 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/contrib/llvm/lib/Target/X86/X86InstrInfo.h
@@ -27,24 +27,6 @@ namespace llvm {
   class X86TargetMachine;
 
 namespace X86 {
-  // Enums for memory operand decoding.  Each memory operand is represented with
-  // a 5 operand sequence in the form:
-  //   [BaseReg, ScaleAmt, IndexReg, Disp, Segment]
-  // These enums help decode this.
-  enum {
-    AddrBaseReg = 0,
-    AddrScaleAmt = 1,
-    AddrIndexReg = 2,
-    AddrDisp = 3,
-
-    /// AddrSegmentReg - The operand # of the segment in the memory operand.
-    AddrSegmentReg = 4,
-
-    /// AddrNumOperands - Total number of operands in a memory reference.
-    AddrNumOperands = 5
-  };
-
-
   // X86 specific condition code. These correspond to X86_*_COND in
   // X86InstrInfo.td. They must be kept in synch.
   enum CondCode {
@@ -82,133 +64,8 @@ namespace X86 {
   /// GetOppositeBranchCondition - Return the inverse of the specified cond,
   /// e.g. turning COND_E to COND_NE.
   CondCode GetOppositeBranchCondition(X86::CondCode CC);
+}  // end namespace X86;
 
-}
-
-/// X86II - This namespace holds all of the target specific flags that
-/// instruction info tracks.
-///
-namespace X86II {
-  /// Target Operand Flag enum.
-  enum TOF {
-    //===------------------------------------------------------------------===//
-    // X86 Specific MachineOperand flags.
-
-    MO_NO_FLAG,
-
-    /// MO_GOT_ABSOLUTE_ADDRESS - On a symbol operand, this represents a
-    /// relocation of:
-    ///    SYMBOL_LABEL + [. - PICBASELABEL]
-    MO_GOT_ABSOLUTE_ADDRESS,
-
-    /// MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the
-    /// immediate should get the value of the symbol minus the PIC base label:
-    ///    SYMBOL_LABEL - PICBASELABEL
-    MO_PIC_BASE_OFFSET,
-
-    /// MO_GOT - On a symbol operand this indicates that the immediate is the
-    /// offset to the GOT entry for the symbol name from the base of the GOT.
-    ///
-    /// See the X86-64 ELF ABI supplement for more details.
-    ///    SYMBOL_LABEL @GOT
-    MO_GOT,
-
-    /// MO_GOTOFF - On a symbol operand this indicates that the immediate is
-    /// the offset to the location of the symbol name from the base of the GOT.
-    ///
-    /// See the X86-64 ELF ABI supplement for more details.
-    ///    SYMBOL_LABEL @GOTOFF
-    MO_GOTOFF,
-
-    /// MO_GOTPCREL - On a symbol operand this indicates that the immediate is
-    /// offset to the GOT entry for the symbol name from the current code
-    /// location.
-    ///
-    /// See the X86-64 ELF ABI supplement for more details.
-    ///    SYMBOL_LABEL @GOTPCREL
-    MO_GOTPCREL,
-
-    /// MO_PLT - On a symbol operand this indicates that the immediate is
-    /// offset to the PLT entry of symbol name from the current code location.
-    ///
-    /// See the X86-64 ELF ABI supplement for more details.
-    ///    SYMBOL_LABEL @PLT
-    MO_PLT,
-
-    /// MO_TLSGD - On a symbol operand this indicates that the immediate is
-    /// some TLS offset.
-    ///
-    /// See 'ELF Handling for Thread-Local Storage' for more details.
-    ///    SYMBOL_LABEL @TLSGD
-    MO_TLSGD,
-
-    /// MO_GOTTPOFF - On a symbol operand this indicates that the immediate is
-    /// some TLS offset.
-    ///
-    /// See 'ELF Handling for Thread-Local Storage' for more details.
-    ///    SYMBOL_LABEL @GOTTPOFF
-    MO_GOTTPOFF,
-
-    /// MO_INDNTPOFF - On a symbol operand this indicates that the immediate is
-    /// some TLS offset.
-    ///
-    /// See 'ELF Handling for Thread-Local Storage' for more details.
-    ///    SYMBOL_LABEL @INDNTPOFF
-    MO_INDNTPOFF,
-
-    /// MO_TPOFF - On a symbol operand this indicates that the immediate is
-    /// some TLS offset.
-    ///
-    /// See 'ELF Handling for Thread-Local Storage' for more details.
-    ///    SYMBOL_LABEL @TPOFF
-    MO_TPOFF,
-
-    /// MO_NTPOFF - On a symbol operand this indicates that the immediate is
-    /// some TLS offset.
-    ///
-    /// See 'ELF Handling for Thread-Local Storage' for more details.
-    ///    SYMBOL_LABEL @NTPOFF
-    MO_NTPOFF,
-
-    /// MO_DLLIMPORT - On a symbol operand "FOO", this indicates that the
-    /// reference is actually to the "__imp_FOO" symbol.  This is used for
-    /// dllimport linkage on windows.
-    MO_DLLIMPORT,
-
-    /// MO_DARWIN_STUB - On a symbol operand "FOO", this indicates that the
-    /// reference is actually to the "FOO$stub" symbol.  This is used for calls
-    /// and jumps to external functions on Tiger and earlier.
-    MO_DARWIN_STUB,
-
-    /// MO_DARWIN_NONLAZY - On a symbol operand "FOO", this indicates that the
-    /// reference is actually to the "FOO$non_lazy_ptr" symbol, which is a
-    /// non-PIC-base-relative reference to a non-hidden dyld lazy pointer stub.
-    MO_DARWIN_NONLAZY,
-
-    /// MO_DARWIN_NONLAZY_PIC_BASE - On a symbol operand "FOO", this indicates
-    /// that the reference is actually to "FOO$non_lazy_ptr - PICBASE", which is
-    /// a PIC-base-relative reference to a non-hidden dyld lazy pointer stub.
-    MO_DARWIN_NONLAZY_PIC_BASE,
-
-    /// MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE - On a symbol operand "FOO", this
-    /// indicates that the reference is actually to "FOO$non_lazy_ptr -PICBASE",
-    /// which is a PIC-base-relative reference to a hidden dyld lazy pointer
-    /// stub.
-    MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE,
-
-    /// MO_TLVP - On a symbol operand this indicates that the immediate is
-    /// some TLS offset.
-    ///
-    /// This is the TLS offset for the Darwin TLS mechanism.
-    MO_TLVP,
-
-    /// MO_TLVP_PIC_BASE - On a symbol operand this indicates that the immediate
-    /// is some TLS offset from the picbase.
-    ///
-    /// This is the 32-bit TLS offset for Darwin TLS in PIC mode.
-    MO_TLVP_PIC_BASE
-  };
-}
 
 /// isGlobalStubReference - Return true if the specified TargetFlag operand is
 /// a reference to a stub for a global, not the global itself.
@@ -243,353 +100,6 @@ inline static bool isGlobalRelativeToPICBase(unsigned char TargetFlag) {
   }
 }
 
-/// X86II - This namespace holds all of the target specific flags that
-/// instruction info tracks.
-///
-namespace X86II {
-  enum {
-    //===------------------------------------------------------------------===//
-    // Instruction encodings.  These are the standard/most common forms for X86
-    // instructions.
-    //
-
-    // PseudoFrm - This represents an instruction that is a pseudo instruction
-    // or one that has not been implemented yet.  It is illegal to code generate
-    // it, but tolerated for intermediate implementation stages.
-    Pseudo         = 0,
-
-    /// Raw - This form is for instructions that don't have any operands, so
-    /// they are just a fixed opcode value, like 'leave'.
-    RawFrm         = 1,
-
-    /// AddRegFrm - This form is used for instructions like 'push r32' that have
-    /// their one register operand added to their opcode.
-    AddRegFrm      = 2,
-
-    /// MRMDestReg - This form is used for instructions that use the Mod/RM byte
-    /// to specify a destination, which in this case is a register.
-    ///
-    MRMDestReg     = 3,
-
-    /// MRMDestMem - This form is used for instructions that use the Mod/RM byte
-    /// to specify a destination, which in this case is memory.
-    ///
-    MRMDestMem     = 4,
-
-    /// MRMSrcReg - This form is used for instructions that use the Mod/RM byte
-    /// to specify a source, which in this case is a register.
-    ///
-    MRMSrcReg      = 5,
-
-    /// MRMSrcMem - This form is used for instructions that use the Mod/RM byte
-    /// to specify a source, which in this case is memory.
-    ///
-    MRMSrcMem      = 6,
-
-    /// MRM[0-7][rm] - These forms are used to represent instructions that use
-    /// a Mod/RM byte, and use the middle field to hold extended opcode
-    /// information.  In the intel manual these are represented as /0, /1, ...
-    ///
-
-    // First, instructions that operate on a register r/m operand...
-    MRM0r = 16,  MRM1r = 17,  MRM2r = 18,  MRM3r = 19, // Format /0 /1 /2 /3
-    MRM4r = 20,  MRM5r = 21,  MRM6r = 22,  MRM7r = 23, // Format /4 /5 /6 /7
-
-    // Next, instructions that operate on a memory r/m operand...
-    MRM0m = 24,  MRM1m = 25,  MRM2m = 26,  MRM3m = 27, // Format /0 /1 /2 /3
-    MRM4m = 28,  MRM5m = 29,  MRM6m = 30,  MRM7m = 31, // Format /4 /5 /6 /7
-
-    // MRMInitReg - This form is used for instructions whose source and
-    // destinations are the same register.
-    MRMInitReg = 32,
-
-    //// MRM_C1 - A mod/rm byte of exactly 0xC1.
-    MRM_C1 = 33,
-    MRM_C2 = 34,
-    MRM_C3 = 35,
-    MRM_C4 = 36,
-    MRM_C8 = 37,
-    MRM_C9 = 38,
-    MRM_E8 = 39,
-    MRM_F0 = 40,
-    MRM_F8 = 41,
-    MRM_F9 = 42,
-    MRM_D0 = 45,
-    MRM_D1 = 46,
-
-    /// RawFrmImm8 - This is used for the ENTER instruction, which has two
-    /// immediates, the first of which is a 16-bit immediate (specified by
-    /// the imm encoding) and the second is a 8-bit fixed value.
-    RawFrmImm8 = 43,
-
-    /// RawFrmImm16 - This is used for CALL FAR instructions, which have two
-    /// immediates, the first of which is a 16 or 32-bit immediate (specified by
-    /// the imm encoding) and the second is a 16-bit fixed value.  In the AMD
-    /// manual, this operand is described as pntr16:32 and pntr16:16
-    RawFrmImm16 = 44,
-
-    FormMask       = 63,
-
-    //===------------------------------------------------------------------===//
-    // Actual flags...
-
-    // OpSize - Set if this instruction requires an operand size prefix (0x66),
-    // which most often indicates that the instruction operates on 16 bit data
-    // instead of 32 bit data.
-    OpSize      = 1 << 6,
-
-    // AsSize - Set if this instruction requires an operand size prefix (0x67),
-    // which most often indicates that the instruction address 16 bit address
-    // instead of 32 bit address (or 32 bit address in 64 bit mode).
-    AdSize      = 1 << 7,
-
-    //===------------------------------------------------------------------===//
-    // Op0Mask - There are several prefix bytes that are used to form two byte
-    // opcodes.  These are currently 0x0F, 0xF3, and 0xD8-0xDF.  This mask is
-    // used to obtain the setting of this field.  If no bits in this field is
-    // set, there is no prefix byte for obtaining a multibyte opcode.
-    //
-    Op0Shift    = 8,
-    Op0Mask     = 0x1F << Op0Shift,
-
-    // TB - TwoByte - Set if this instruction has a two byte opcode, which
-    // starts with a 0x0F byte before the real opcode.
-    TB          = 1 << Op0Shift,
-
-    // REP - The 0xF3 prefix byte indicating repetition of the following
-    // instruction.
-    REP         = 2 << Op0Shift,
-
-    // D8-DF - These escape opcodes are used by the floating point unit.  These
-    // values must remain sequential.
-    D8 = 3 << Op0Shift,   D9 = 4 << Op0Shift,
-    DA = 5 << Op0Shift,   DB = 6 << Op0Shift,
-    DC = 7 << Op0Shift,   DD = 8 << Op0Shift,
-    DE = 9 << Op0Shift,   DF = 10 << Op0Shift,
-
-    // XS, XD - These prefix codes are for single and double precision scalar
-    // floating point operations performed in the SSE registers.
-    XD = 11 << Op0Shift,  XS = 12 << Op0Shift,
-
-    // T8, TA, A6, A7 - Prefix after the 0x0F prefix.
-    T8 = 13 << Op0Shift,  TA = 14 << Op0Shift,
-    A6 = 15 << Op0Shift,  A7 = 16 << Op0Shift,
-
-    // TF - Prefix before and after 0x0F
-    TF = 17 << Op0Shift,
-
-    //===------------------------------------------------------------------===//
-    // REX_W - REX prefixes are instruction prefixes used in 64-bit mode.
-    // They are used to specify GPRs and SSE registers, 64-bit operand size,
-    // etc. We only cares about REX.W and REX.R bits and only the former is
-    // statically determined.
-    //
-    REXShift    = Op0Shift + 5,
-    REX_W       = 1 << REXShift,
-
-    //===------------------------------------------------------------------===//
-    // This three-bit field describes the size of an immediate operand.  Zero is
-    // unused so that we can tell if we forgot to set a value.
-    ImmShift = REXShift + 1,
-    ImmMask    = 7 << ImmShift,
-    Imm8       = 1 << ImmShift,
-    Imm8PCRel  = 2 << ImmShift,
-    Imm16      = 3 << ImmShift,
-    Imm16PCRel = 4 << ImmShift,
-    Imm32      = 5 << ImmShift,
-    Imm32PCRel = 6 << ImmShift,
-    Imm64      = 7 << ImmShift,
-
-    //===------------------------------------------------------------------===//
-    // FP Instruction Classification...  Zero is non-fp instruction.
-
-    // FPTypeMask - Mask for all of the FP types...
-    FPTypeShift = ImmShift + 3,
-    FPTypeMask  = 7 << FPTypeShift,
-
-    // NotFP - The default, set for instructions that do not use FP registers.
-    NotFP      = 0 << FPTypeShift,
-
-    // ZeroArgFP - 0 arg FP instruction which implicitly pushes ST(0), f.e. fld0
-    ZeroArgFP  = 1 << FPTypeShift,
-
-    // OneArgFP - 1 arg FP instructions which implicitly read ST(0), such as fst
-    OneArgFP   = 2 << FPTypeShift,
-
-    // OneArgFPRW - 1 arg FP instruction which implicitly read ST(0) and write a
-    // result back to ST(0).  For example, fcos, fsqrt, etc.
-    //
-    OneArgFPRW = 3 << FPTypeShift,
-
-    // TwoArgFP - 2 arg FP instructions which implicitly read ST(0), and an
-    // explicit argument, storing the result to either ST(0) or the implicit
-    // argument.  For example: fadd, fsub, fmul, etc...
-    TwoArgFP   = 4 << FPTypeShift,
-
-    // CompareFP - 2 arg FP instructions which implicitly read ST(0) and an
-    // explicit argument, but have no destination.  Example: fucom, fucomi, ...
-    CompareFP  = 5 << FPTypeShift,
-
-    // CondMovFP - "2 operand" floating point conditional move instructions.
-    CondMovFP  = 6 << FPTypeShift,
-
-    // SpecialFP - Special instruction forms.  Dispatch by opcode explicitly.
-    SpecialFP  = 7 << FPTypeShift,
-
-    // Lock prefix
-    LOCKShift = FPTypeShift + 3,
-    LOCK = 1 << LOCKShift,
-
-    // Segment override prefixes. Currently we just need ability to address
-    // stuff in gs and fs segments.
-    SegOvrShift = LOCKShift + 1,
-    SegOvrMask  = 3 << SegOvrShift,
-    FS          = 1 << SegOvrShift,
-    GS          = 2 << SegOvrShift,
-
-    // Execution domain for SSE instructions in bits 23, 24.
-    // 0 in bits 23-24 means normal, non-SSE instruction.
-    SSEDomainShift = SegOvrShift + 2,
-
-    OpcodeShift   = SSEDomainShift + 2,
-
-    //===------------------------------------------------------------------===//
-    /// VEX - The opcode prefix used by AVX instructions
-    VEXShift = OpcodeShift + 8,
-    VEX         = 1U << 0,
-
-    /// VEX_W - Has a opcode specific functionality, but is used in the same
-    /// way as REX_W is for regular SSE instructions.
-    VEX_W       = 1U << 1,
-
-    /// VEX_4V - Used to specify an additional AVX/SSE register. Several 2
-    /// address instructions in SSE are represented as 3 address ones in AVX
-    /// and the additional register is encoded in VEX_VVVV prefix.
-    VEX_4V      = 1U << 2,
-
-    /// VEX_I8IMM - Specifies that the last register used in a AVX instruction,
-    /// must be encoded in the i8 immediate field. This usually happens in
-    /// instructions with 4 operands.
-    VEX_I8IMM   = 1U << 3,
-
-    /// VEX_L - Stands for a bit in the VEX opcode prefix meaning the current
-    /// instruction uses 256-bit wide registers. This is usually auto detected
-    /// if a VR256 register is used, but some AVX instructions also have this
-    /// field marked when using a f256 memory references.
-    VEX_L       = 1U << 4,
-
-    /// Has3DNow0F0FOpcode - This flag indicates that the instruction uses the
-    /// wacky 0x0F 0x0F prefix for 3DNow! instructions.  The manual documents
-    /// this as having a 0x0F prefix with a 0x0F opcode, and each instruction
-    /// storing a classifier in the imm8 field.  To simplify our implementation,
-    /// we handle this by storeing the classifier in the opcode field and using
-    /// this flag to indicate that the encoder should do the wacky 3DNow! thing.
-    Has3DNow0F0FOpcode = 1U << 5
-  };
-
-  // getBaseOpcodeFor - This function returns the "base" X86 opcode for the
-  // specified machine instruction.
-  //
-  static inline unsigned char getBaseOpcodeFor(uint64_t TSFlags) {
-    return TSFlags >> X86II::OpcodeShift;
-  }
-
-  static inline bool hasImm(uint64_t TSFlags) {
-    return (TSFlags & X86II::ImmMask) != 0;
-  }
-
-  /// getSizeOfImm - Decode the "size of immediate" field from the TSFlags field
-  /// of the specified instruction.
-  static inline unsigned getSizeOfImm(uint64_t TSFlags) {
-    switch (TSFlags & X86II::ImmMask) {
-    default: assert(0 && "Unknown immediate size");
-    case X86II::Imm8:
-    case X86II::Imm8PCRel:  return 1;
-    case X86II::Imm16:
-    case X86II::Imm16PCRel: return 2;
-    case X86II::Imm32:
-    case X86II::Imm32PCRel: return 4;
-    case X86II::Imm64:      return 8;
-    }
-  }
-
-  /// isImmPCRel - Return true if the immediate of the specified instruction's
-  /// TSFlags indicates that it is pc relative.
-  static inline unsigned isImmPCRel(uint64_t TSFlags) {
-    switch (TSFlags & X86II::ImmMask) {
-    default: assert(0 && "Unknown immediate size");
-    case X86II::Imm8PCRel:
-    case X86II::Imm16PCRel:
-    case X86II::Imm32PCRel:
-      return true;
-    case X86II::Imm8:
-    case X86II::Imm16:
-    case X86II::Imm32:
-    case X86II::Imm64:
-      return false;
-    }
-  }
-
-  /// getMemoryOperandNo - The function returns the MCInst operand # for the
-  /// first field of the memory operand.  If the instruction doesn't have a
-  /// memory operand, this returns -1.
-  ///
-  /// Note that this ignores tied operands.  If there is a tied register which
-  /// is duplicated in the MCInst (e.g. "EAX = addl EAX, [mem]") it is only
-  /// counted as one operand.
-  ///
-  static inline int getMemoryOperandNo(uint64_t TSFlags) {
-    switch (TSFlags & X86II::FormMask) {
-    case X86II::MRMInitReg:  assert(0 && "FIXME: Remove this form");
-    default: assert(0 && "Unknown FormMask value in getMemoryOperandNo!");
-    case X86II::Pseudo:
-    case X86II::RawFrm:
-    case X86II::AddRegFrm:
-    case X86II::MRMDestReg:
-    case X86II::MRMSrcReg:
-    case X86II::RawFrmImm8:
-    case X86II::RawFrmImm16:
-       return -1;
-    case X86II::MRMDestMem:
-      return 0;
-    case X86II::MRMSrcMem: {
-      bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V;
-      unsigned FirstMemOp = 1;
-      if (HasVEX_4V)
-        ++FirstMemOp;// Skip the register source (which is encoded in VEX_VVVV).
-
-      // FIXME: Maybe lea should have its own form?  This is a horrible hack.
-      //if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r ||
-      //    Opcode == X86::LEA16r || Opcode == X86::LEA32r)
-      return FirstMemOp;
-    }
-    case X86II::MRM0r: case X86II::MRM1r:
-    case X86II::MRM2r: case X86II::MRM3r:
-    case X86II::MRM4r: case X86II::MRM5r:
-    case X86II::MRM6r: case X86II::MRM7r:
-      return -1;
-    case X86II::MRM0m: case X86II::MRM1m:
-    case X86II::MRM2m: case X86II::MRM3m:
-    case X86II::MRM4m: case X86II::MRM5m:
-    case X86II::MRM6m: case X86II::MRM7m:
-      return 0;
-    case X86II::MRM_C1:
-    case X86II::MRM_C2:
-    case X86II::MRM_C3:
-    case X86II::MRM_C4:
-    case X86II::MRM_C8:
-    case X86II::MRM_C9:
-    case X86II::MRM_E8:
-    case X86II::MRM_F0:
-    case X86II::MRM_F8:
-    case X86II::MRM_F9:
-    case X86II::MRM_D0:
-    case X86II::MRM_D1:
-      return -1;
-    }
-  }
-}
-
 inline static bool isScale(const MachineOperand &MO) {
   return MO.isImm() &&
     (MO.getImm() == 1 || MO.getImm() == 2 ||
@@ -621,14 +131,22 @@ class X86InstrInfo : public X86GenInstrInfo {
   /// RegOp2MemOpTable2Addr, RegOp2MemOpTable0, RegOp2MemOpTable1,
   /// RegOp2MemOpTable2 - Load / store folding opcode maps.
   ///
-  DenseMap<unsigned, std::pair<unsigned,unsigned> > RegOp2MemOpTable2Addr;
-  DenseMap<unsigned, std::pair<unsigned,unsigned> > RegOp2MemOpTable0;
-  DenseMap<unsigned, std::pair<unsigned,unsigned> > RegOp2MemOpTable1;
-  DenseMap<unsigned, std::pair<unsigned,unsigned> > RegOp2MemOpTable2;
+  typedef DenseMap<unsigned,
+                   std::pair<unsigned, unsigned> > RegOp2MemOpTableType;
+  RegOp2MemOpTableType RegOp2MemOpTable2Addr;
+  RegOp2MemOpTableType RegOp2MemOpTable0;
+  RegOp2MemOpTableType RegOp2MemOpTable1;
+  RegOp2MemOpTableType RegOp2MemOpTable2;
 
   /// MemOp2RegOpTable - Load / store unfolding opcode map.
   ///
-  DenseMap<unsigned, std::pair<unsigned, unsigned> > MemOp2RegOpTable;
+  typedef DenseMap<unsigned,
+                   std::pair<unsigned, unsigned> > MemOp2RegOpTableType;
+  MemOp2RegOpTableType MemOp2RegOpTable;
+
+  void AddTableEntry(RegOp2MemOpTableType &R2MTable,
+                     MemOp2RegOpTableType &M2RTable,
+                     unsigned RegOp, unsigned MemOp, unsigned Flags);
 
 public:
   explicit X86InstrInfo(X86TargetMachine &tm);
@@ -656,17 +174,6 @@ class X86InstrInfo : public X86GenInstrInfo {
   unsigned isLoadFromStackSlotPostFE(const MachineInstr *MI,
                                      int &FrameIndex) const;
 
-  /// hasLoadFromStackSlot - If the specified machine instruction has
-  /// a load from a stack slot, return true along with the FrameIndex
-  /// of the loaded stack slot and the machine mem operand containing
-  /// the reference.  If not, return false.  Unlike
-  /// isLoadFromStackSlot, this returns true for any instructions that
-  /// loads from the stack.  This is a hint only and may not catch all
-  /// cases.
-  bool hasLoadFromStackSlot(const MachineInstr *MI,
-                            const MachineMemOperand *&MMO,
-                            int &FrameIndex) const;
-
   unsigned isStoreToStackSlot(const MachineInstr *MI, int &FrameIndex) const;
   /// isStoreToStackSlotPostFE - Check for post-frame ptr elimination
   /// stack locations as well.  This uses a heuristic so it isn't
@@ -674,16 +181,6 @@ class X86InstrInfo : public X86GenInstrInfo {
   unsigned isStoreToStackSlotPostFE(const MachineInstr *MI,
                                     int &FrameIndex) const;
 
-  /// hasStoreToStackSlot - If the specified machine instruction has a
-  /// store to a stack slot, return true along with the FrameIndex of
-  /// the loaded stack slot and the machine mem operand containing the
-  /// reference.  If not, return false.  Unlike isStoreToStackSlot,
-  /// this returns true for any instructions that loads from the
-  /// stack.  This is a hint only and may not catch all cases.
-  bool hasStoreToStackSlot(const MachineInstr *MI,
-                           const MachineMemOperand *&MMO,
-                           int &FrameIndex) const;
-
   bool isReallyTriviallyReMaterializable(const MachineInstr *MI,
                                          AliasAnalysis *AA) const;
   void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
@@ -750,6 +247,9 @@ class X86InstrInfo : public X86GenInstrInfo {
                                MachineInstr::mmo_iterator MMOBegin,
                                MachineInstr::mmo_iterator MMOEnd,
                                SmallVectorImpl<MachineInstr*> &NewMIs) const;
+
+  virtual bool expandPostRAPseudo(MachineBasicBlock::iterator MI) const;
+
   virtual
   MachineInstr *emitFrameIndexDebugValue(MachineFunction &MF,
                                          int FrameIx, uint64_t Offset,
@@ -829,32 +329,21 @@ class X86InstrInfo : public X86GenInstrInfo {
   /// instruction that defines the specified register class.
   bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const;
 
-  static bool isX86_64NonExtLowByteReg(unsigned reg) {
-    return (reg == X86::SPL || reg == X86::BPL ||
-          reg == X86::SIL || reg == X86::DIL);
-  }
-
   static bool isX86_64ExtendedReg(const MachineOperand &MO) {
     if (!MO.isReg()) return false;
-    return isX86_64ExtendedReg(MO.getReg());
+    return X86II::isX86_64ExtendedReg(MO.getReg());
   }
 
-  /// isX86_64ExtendedReg - Is the MachineOperand a x86-64 extended (r8 or
-  /// higher) register?  e.g. r8, xmm8, xmm13, etc.
-  static bool isX86_64ExtendedReg(unsigned RegNo);
-
   /// getGlobalBaseReg - Return a virtual register initialized with the
   /// the global base register value. Output instructions required to
   /// initialize the register in the function entry block, if necessary.
   ///
   unsigned getGlobalBaseReg(MachineFunction *MF) const;
 
-  /// GetSSEDomain - Return the SSE execution domain of MI as the first element,
-  /// and a bitmask of possible arguments to SetSSEDomain ase the second.
-  std::pair<uint16_t, uint16_t> GetSSEDomain(const MachineInstr *MI) const;
+  std::pair<uint16_t, uint16_t>
+  getExecutionDomain(const MachineInstr *MI) const;
 
-  /// SetSSEDomain - Set the SSEDomain of MI.
-  void SetSSEDomain(MachineInstr *MI, unsigned Domain) const;
+  void setExecutionDomain(MachineInstr *MI, unsigned Domain) const;
 
   MachineInstr* foldMemoryOperandImpl(MachineFunction &MF,
                                       MachineInstr* MI,
diff --git a/contrib/llvm/lib/Target/X86/X86InstrInfo.td b/contrib/llvm/lib/Target/X86/X86InstrInfo.td
index 7eb07b0a97bd..d54bf275c04f 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrInfo.td
@@ -65,7 +65,7 @@ def SDTX86SetCC_C : SDTypeProfile<1, 2,
 
 def SDTX86cas : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisInt<1>,
                                      SDTCisVT<2, i8>]>;
-def SDTX86cas8 : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
+def SDTX86caspair : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
 
 def SDTX86atomicBinary : SDTypeProfile<2, 3, [SDTCisInt<0>, SDTCisInt<1>,
                                 SDTCisPtrTy<2>, SDTCisInt<3>,SDTCisInt<4>]>;
@@ -97,6 +97,8 @@ def SDT_X86TLSADDR : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
 def SDT_X86TLSCALL : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
+def SDT_X86SEG_ALLOCA : SDTypeProfile<1, 1, [SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>;
+
 def SDT_X86EHRET : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
 def SDT_X86TCRET : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>;
@@ -133,9 +135,13 @@ def X86setcc_c : SDNode<"X86ISD::SETCC_CARRY", SDTX86SetCC_C>;
 def X86cas : SDNode<"X86ISD::LCMPXCHG_DAG", SDTX86cas,
                         [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore,
                          SDNPMayLoad, SDNPMemOperand]>;
-def X86cas8 : SDNode<"X86ISD::LCMPXCHG8_DAG", SDTX86cas8,
+def X86cas8 : SDNode<"X86ISD::LCMPXCHG8_DAG", SDTX86caspair,
                         [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore,
                          SDNPMayLoad, SDNPMemOperand]>;
+def X86cas16 : SDNode<"X86ISD::LCMPXCHG16_DAG", SDTX86caspair,
+                        [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore,
+                         SDNPMayLoad, SDNPMemOperand]>;
+
 def X86AtomAdd64 : SDNode<"X86ISD::ATOMADD64_DAG", SDTX86atomicBinary,
                         [SDNPHasChain, SDNPMayStore,
                          SDNPMayLoad, SDNPMemOperand]>;
@@ -218,12 +224,16 @@ def X86xor_flag  : SDNode<"X86ISD::XOR",  SDTBinaryArithWithFlags,
                           [SDNPCommutative]>;
 def X86and_flag  : SDNode<"X86ISD::AND",  SDTBinaryArithWithFlags,
                           [SDNPCommutative]>;
+def X86andn_flag : SDNode<"X86ISD::ANDN", SDTBinaryArithWithFlags>;
 
 def X86mul_imm : SDNode<"X86ISD::MUL_IMM", SDTIntBinOp>;
 
 def X86WinAlloca : SDNode<"X86ISD::WIN_ALLOCA", SDTX86Void,
                           [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
 
+def X86SegAlloca : SDNode<"X86ISD::SEG_ALLOCA", SDT_X86SEG_ALLOCA,
+                          [SDNPHasChain]>;
+
 def X86TLSCall : SDNode<"X86ISD::TLSCALL", SDT_X86TLSCALL,
                         [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
@@ -331,6 +341,11 @@ class ImmSExtAsmOperandClass : AsmOperandClass {
   let RenderMethod = "addImmOperands";
 }
 
+class ImmZExtAsmOperandClass : AsmOperandClass {
+  let SuperClasses = [ImmAsmOperand];
+  let RenderMethod = "addImmOperands";
+}
+
 // Sign-extended immediate classes. We don't need to define the full lattice
 // here because there is no instruction with an ambiguity between ImmSExti64i32
 // and ImmSExti32i8.
@@ -358,6 +373,12 @@ def ImmSExti32i8AsmOperand : ImmSExtAsmOperandClass {
   let Name = "ImmSExti32i8";
 }
 
+// [0, 0x000000FF]
+def ImmZExtu32u8AsmOperand : ImmZExtAsmOperandClass {
+  let Name = "ImmZExtu32u8";
+}
+
+
 // [0, 0x0000007F]                                            |
 //   [0xFFFFFFFFFFFFFF80, 0xFFFFFFFFFFFFFFFF]
 def ImmSExti64i8AsmOperand : ImmSExtAsmOperandClass {
@@ -377,6 +398,11 @@ def i32i8imm  : Operand<i32> {
   let ParserMatchClass = ImmSExti32i8AsmOperand;
   let OperandType = "OPERAND_IMMEDIATE";
 }
+// 32-bits but only 8 bits are significant, and those 8 bits are unsigned.
+def u32u8imm  : Operand<i32> {
+  let ParserMatchClass = ImmZExtu32u8AsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
+}
 
 // 64-bits but only 32 bits are significant.
 def i64i32imm  : Operand<i64> {
@@ -389,11 +415,13 @@ def i64i32imm  : Operand<i64> {
 def i64i32imm_pcrel : Operand<i64> {
   let PrintMethod = "print_pcrel_imm";
   let ParserMatchClass = X86AbsMemAsmOperand;
+  let OperandType = "OPERAND_PCREL";
 }
 
 // 64-bits but only 8 bits are significant.
 def i64i8imm   : Operand<i64> {
   let ParserMatchClass = ImmSExti64i8AsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def lea64_32mem : Operand<i32> {
@@ -442,18 +470,33 @@ def HasSSE4A     : Predicate<"Subtarget->hasSSE4A()">;
 def HasAVX       : Predicate<"Subtarget->hasAVX()">;
 def HasXMMInt    : Predicate<"Subtarget->hasXMMInt()">;
 
+def HasPOPCNT    : Predicate<"Subtarget->hasPOPCNT()">;
 def HasAES       : Predicate<"Subtarget->hasAES()">;
 def HasCLMUL     : Predicate<"Subtarget->hasCLMUL()">;
 def HasFMA3      : Predicate<"Subtarget->hasFMA3()">;
 def HasFMA4      : Predicate<"Subtarget->hasFMA4()">;
+def HasMOVBE     : Predicate<"Subtarget->hasMOVBE()">;
+def HasRDRAND    : Predicate<"Subtarget->hasRDRAND()">;
+def HasF16C      : Predicate<"Subtarget->hasF16C()">;
+def HasLZCNT     : Predicate<"Subtarget->hasLZCNT()">;
+def HasBMI       : Predicate<"Subtarget->hasBMI()">;
 def FPStackf32   : Predicate<"!Subtarget->hasXMM()">;
 def FPStackf64   : Predicate<"!Subtarget->hasXMMInt()">;
+def HasCmpxchg16b: Predicate<"Subtarget->hasCmpxchg16b()">;
 def In32BitMode  : Predicate<"!Subtarget->is64Bit()">,
                              AssemblerPredicate<"!Mode64Bit">;
 def In64BitMode  : Predicate<"Subtarget->is64Bit()">,
                              AssemblerPredicate<"Mode64Bit">;
 def IsWin64      : Predicate<"Subtarget->isTargetWin64()">;
 def NotWin64     : Predicate<"!Subtarget->isTargetWin64()">;
+def IsNaCl       : Predicate<"Subtarget->isTargetNaCl()">,
+                             AssemblerPredicate<"ModeNaCl">;
+def IsNaCl32     : Predicate<"Subtarget->isTargetNaCl32()">,
+                             AssemblerPredicate<"ModeNaCl,!Mode64Bit">;
+def IsNaCl64     : Predicate<"Subtarget->isTargetNaCl64()">,
+                             AssemblerPredicate<"ModeNaCl,Mode64Bit">;
+def NotNaCl      : Predicate<"!Subtarget->isTargetNaCl()">,
+                             AssemblerPredicate<"!ModeNaCl">;
 def SmallCode    : Predicate<"TM.getCodeModel() == CodeModel::Small">;
 def KernelCode   : Predicate<"TM.getCodeModel() == CodeModel::Kernel">;
 def FarData      : Predicate<"TM.getCodeModel() != CodeModel::Small &&"
@@ -766,30 +809,30 @@ def BSR64rm  : RI<0xBD, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
 
 // These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI
 let Defs = [EDI,ESI], Uses = [EDI,ESI,EFLAGS] in {
-def MOVSB : I<0xA4, RawFrm, (outs), (ins), "{movsb}", []>;
-def MOVSW : I<0xA5, RawFrm, (outs), (ins), "{movsw}", []>, OpSize;
-def MOVSD : I<0xA5, RawFrm, (outs), (ins), "{movsl|movsd}", []>;
+def MOVSB : I<0xA4, RawFrm, (outs), (ins), "movsb", []>;
+def MOVSW : I<0xA5, RawFrm, (outs), (ins), "movsw", []>, OpSize;
+def MOVSD : I<0xA5, RawFrm, (outs), (ins), "movs{l|d}", []>;
 def MOVSQ : RI<0xA5, RawFrm, (outs), (ins), "movsq", []>;
 }
 
 // These uses the DF flag in the EFLAGS register to inc or dec EDI and ESI
 let Defs = [EDI], Uses = [AL,EDI,EFLAGS] in
-def STOSB : I<0xAA, RawFrm, (outs), (ins), "{stosb}", []>;
+def STOSB : I<0xAA, RawFrm, (outs), (ins), "stosb", []>;
 let Defs = [EDI], Uses = [AX,EDI,EFLAGS] in
-def STOSW : I<0xAB, RawFrm, (outs), (ins), "{stosw}", []>, OpSize;
+def STOSW : I<0xAB, RawFrm, (outs), (ins), "stosw", []>, OpSize;
 let Defs = [EDI], Uses = [EAX,EDI,EFLAGS] in
-def STOSD : I<0xAB, RawFrm, (outs), (ins), "{stosl|stosd}", []>;
+def STOSD : I<0xAB, RawFrm, (outs), (ins), "stos{l|d}", []>;
 let Defs = [RCX,RDI], Uses = [RAX,RCX,RDI,EFLAGS] in
 def STOSQ : RI<0xAB, RawFrm, (outs), (ins), "stosq", []>;
 
-def SCAS8 : I<0xAE, RawFrm, (outs), (ins), "scas{b}", []>;
-def SCAS16 : I<0xAF, RawFrm, (outs), (ins), "scas{w}", []>, OpSize;
-def SCAS32 : I<0xAF, RawFrm, (outs), (ins), "scas{l}", []>;
+def SCAS8 : I<0xAE, RawFrm, (outs), (ins), "scasb", []>;
+def SCAS16 : I<0xAF, RawFrm, (outs), (ins), "scasw", []>, OpSize;
+def SCAS32 : I<0xAF, RawFrm, (outs), (ins), "scas{l|d}", []>;
 def SCAS64 : RI<0xAF, RawFrm, (outs), (ins), "scasq", []>;
 
-def CMPS8 : I<0xA6, RawFrm, (outs), (ins), "cmps{b}", []>;
-def CMPS16 : I<0xA7, RawFrm, (outs), (ins), "cmps{w}", []>, OpSize;
-def CMPS32 : I<0xA7, RawFrm, (outs), (ins), "cmps{l}", []>;
+def CMPS8 : I<0xA6, RawFrm, (outs), (ins), "cmpsb", []>;
+def CMPS16 : I<0xA7, RawFrm, (outs), (ins), "cmpsw", []>, OpSize;
+def CMPS32 : I<0xA7, RawFrm, (outs), (ins), "cmps{l|d}", []>;
 def CMPS64 : RI<0xA7, RawFrm, (outs), (ins), "cmpsq", []>;
 
 
@@ -841,22 +884,22 @@ def MOV64mi32 : RIi32<0xC7, MRM0m, (outs), (ins i64mem:$dst, i64i32imm:$src),
 /// moffs8, moffs16 and moffs32 versions of moves.  The immediate is a
 /// 32-bit offset from the PC.  These are only valid in x86-32 mode.
 def MOV8o8a : Ii32 <0xA0, RawFrm, (outs), (ins offset8:$src),
-                   "mov{b}\t{$src, %al|%al, $src}", []>,
+                   "mov{b}\t{$src, %al|AL, $src}", []>,
                    Requires<[In32BitMode]>;
 def MOV16o16a : Ii32 <0xA1, RawFrm, (outs), (ins offset16:$src),
-                      "mov{w}\t{$src, %ax|%ax, $src}", []>, OpSize,
+                      "mov{w}\t{$src, %ax|AL, $src}", []>, OpSize,
                      Requires<[In32BitMode]>;
 def MOV32o32a : Ii32 <0xA1, RawFrm, (outs), (ins offset32:$src),
-                      "mov{l}\t{$src, %eax|%eax, $src}", []>,
+                      "mov{l}\t{$src, %eax|EAX, $src}", []>,
                      Requires<[In32BitMode]>;
 def MOV8ao8 : Ii32 <0xA2, RawFrm, (outs offset8:$dst), (ins),
-                   "mov{b}\t{%al, $dst|$dst, %al}", []>,
+                   "mov{b}\t{%al, $dst|$dst, AL}", []>,
                   Requires<[In32BitMode]>;
 def MOV16ao16 : Ii32 <0xA3, RawFrm, (outs offset16:$dst), (ins),
-                      "mov{w}\t{%ax, $dst|$dst, %ax}", []>, OpSize,
+                      "mov{w}\t{%ax, $dst|$dst, AL}", []>, OpSize,
                      Requires<[In32BitMode]>;
 def MOV32ao32 : Ii32 <0xA3, RawFrm, (outs offset32:$dst), (ins),
-                      "mov{l}\t{%eax, $dst|$dst, %eax}", []>,
+                      "mov{l}\t{%eax, $dst|$dst, EAX}", []>,
                      Requires<[In32BitMode]>;
 
 // FIXME: These definitions are utterly broken
@@ -865,13 +908,13 @@ def MOV32ao32 : Ii32 <0xA3, RawFrm, (outs offset32:$dst), (ins),
 // in question.
 /*
 def MOV64o8a : RIi8<0xA0, RawFrm, (outs), (ins offset8:$src),
-                      "mov{q}\t{$src, %rax|%rax, $src}", []>;
+                      "mov{q}\t{$src, %rax|RAX, $src}", []>;
 def MOV64o64a : RIi32<0xA1, RawFrm, (outs), (ins offset64:$src),
-                       "mov{q}\t{$src, %rax|%rax, $src}", []>;
+                       "mov{q}\t{$src, %rax|RAX, $src}", []>;
 def MOV64ao8 : RIi8<0xA2, RawFrm, (outs offset8:$dst), (ins),
-                       "mov{q}\t{%rax, $dst|$dst, %rax}", []>;
+                       "mov{q}\t{%rax, $dst|$dst, RAX}", []>;
 def MOV64ao64 : RIi32<0xA3, RawFrm, (outs offset64:$dst), (ins),
-                       "mov{q}\t{%rax, $dst|$dst, %rax}", []>;
+                       "mov{q}\t{%rax, $dst|$dst, RAX}", []>;
 */
 
 
@@ -926,7 +969,7 @@ let mayStore = 1 in
 def MOV8mr_NOREX : I<0x88, MRMDestMem,
                      (outs), (ins i8mem_NOREX:$dst, GR8_NOREX:$src),
                      "mov{b}\t{$src, $dst|$dst, $src}  # NOREX", []>;
-let mayLoad = 1,
+let mayLoad = 1, neverHasSideEffects = 1,
     canFoldAsLoad = 1, isReMaterializable = 1 in
 def MOV8rm_NOREX : I<0x8A, MRMSrcMem,
                      (outs GR8_NOREX:$dst), (ins i8mem_NOREX:$src),
@@ -1117,11 +1160,15 @@ def XCHG64rr : RI<0x87, MRMSrcReg, (outs GR64:$dst), (ins GR64:$val,GR64:$src),
 }
 
 def XCHG16ar : I<0x90, AddRegFrm, (outs), (ins GR16:$src),
-                  "xchg{w}\t{$src, %ax|%ax, $src}", []>, OpSize;
+                  "xchg{w}\t{$src, %ax|AX, $src}", []>, OpSize;
 def XCHG32ar : I<0x90, AddRegFrm, (outs), (ins GR32:$src),
-                  "xchg{l}\t{$src, %eax|%eax, $src}", []>;
+                  "xchg{l}\t{$src, %eax|EAX, $src}", []>, Requires<[In32BitMode]>;
+// Uses GR32_NOAX in 64-bit mode to prevent encoding using the 0x90 NOP encoding.
+// xchg %eax, %eax needs to clear upper 32-bits of RAX so is not a NOP.
+def XCHG32ar64 : I<0x90, AddRegFrm, (outs), (ins GR32_NOAX:$src),
+                   "xchg{l}\t{$src, %eax|EAX, $src}", []>, Requires<[In64BitMode]>;
 def XCHG64ar : RI<0x90, AddRegFrm, (outs), (ins GR64:$src),
-                  "xchg{q}\t{$src, %rax|%rax, $src}", []>;
+                  "xchg{q}\t{$src, %rax|RAX, $src}", []>;
 
 
 
@@ -1172,7 +1219,7 @@ def CMPXCHG8B : I<0xC7, MRM1m, (outs), (ins i64mem:$dst),
 
 let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RBX, RCX, RDX] in
 def CMPXCHG16B : RI<0xC7, MRM1m, (outs), (ins i128mem:$dst),
-                    "cmpxchg16b\t$dst", []>, TB;
+                    "cmpxchg16b\t$dst", []>, TB, Requires<[HasCmpxchg16b]>;
 
 
 
@@ -1260,6 +1307,104 @@ def ARPL16rr : I<0x63, MRMDestReg, (outs GR16:$src), (ins GR16:$dst),
 def ARPL16mr : I<0x63, MRMSrcMem, (outs GR16:$src), (ins i16mem:$dst),
                  "arpl\t{$src, $dst|$dst, $src}", []>, Requires<[In32BitMode]>;
 
+//===----------------------------------------------------------------------===//
+// MOVBE Instructions
+//
+let Predicates = [HasMOVBE] in {
+  def MOVBE16rm : I<0xF0, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
+                    "movbe{w}\t{$src, $dst|$dst, $src}",
+                    [(set GR16:$dst, (bswap (loadi16 addr:$src)))]>, OpSize, T8;
+  def MOVBE32rm : I<0xF0, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
+                    "movbe{l}\t{$src, $dst|$dst, $src}",
+                    [(set GR32:$dst, (bswap (loadi32 addr:$src)))]>, T8;
+  def MOVBE64rm : RI<0xF0, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
+                     "movbe{q}\t{$src, $dst|$dst, $src}",
+                     [(set GR64:$dst, (bswap (loadi64 addr:$src)))]>, T8;
+  def MOVBE16mr : I<0xF1, MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src),
+                    "movbe{w}\t{$src, $dst|$dst, $src}",
+                    [(store (bswap GR16:$src), addr:$dst)]>, OpSize, T8;
+  def MOVBE32mr : I<0xF1, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src),
+                    "movbe{l}\t{$src, $dst|$dst, $src}",
+                    [(store (bswap GR32:$src), addr:$dst)]>, T8;
+  def MOVBE64mr : RI<0xF1, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src),
+                     "movbe{q}\t{$src, $dst|$dst, $src}",
+                     [(store (bswap GR64:$src), addr:$dst)]>, T8;
+}
+
+//===----------------------------------------------------------------------===//
+// RDRAND Instruction
+//
+let Predicates = [HasRDRAND], Defs = [EFLAGS] in {
+  def RDRAND16r : I<0xC7, MRM6r, (outs GR16:$dst), (ins),
+                    "rdrand{w}\t$dst", []>, OpSize, TB;
+  def RDRAND32r : I<0xC7, MRM6r, (outs GR32:$dst), (ins),
+                    "rdrand{l}\t$dst", []>, TB;
+  def RDRAND64r : RI<0xC7, MRM6r, (outs GR64:$dst), (ins),
+                     "rdrand{q}\t$dst", []>, TB;
+}
+
+//===----------------------------------------------------------------------===//
+// LZCNT Instruction
+//
+let Predicates = [HasLZCNT], Defs = [EFLAGS] in {
+  def LZCNT16rr : I<0xBD, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src),
+                    "lzcnt{w}\t{$src, $dst|$dst, $src}",
+                    [(set GR16:$dst, (ctlz GR16:$src)), (implicit EFLAGS)]>, XS,
+                    OpSize;
+  def LZCNT16rm : I<0xBD, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
+                    "lzcnt{w}\t{$src, $dst|$dst, $src}",
+                    [(set GR16:$dst, (ctlz (loadi16 addr:$src))),
+                     (implicit EFLAGS)]>, XS, OpSize;
+
+  def LZCNT32rr : I<0xBD, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
+                    "lzcnt{l}\t{$src, $dst|$dst, $src}",
+                    [(set GR32:$dst, (ctlz GR32:$src)), (implicit EFLAGS)]>, XS;
+  def LZCNT32rm : I<0xBD, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
+                    "lzcnt{l}\t{$src, $dst|$dst, $src}",
+                    [(set GR32:$dst, (ctlz (loadi32 addr:$src))),
+                     (implicit EFLAGS)]>, XS;
+
+  def LZCNT64rr : RI<0xBD, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
+                     "lzcnt{q}\t{$src, $dst|$dst, $src}",
+                     [(set GR64:$dst, (ctlz GR64:$src)), (implicit EFLAGS)]>,
+                     XS;
+  def LZCNT64rm : RI<0xBD, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
+                     "lzcnt{q}\t{$src, $dst|$dst, $src}",
+                     [(set GR64:$dst, (ctlz (loadi64 addr:$src))),
+                      (implicit EFLAGS)]>, XS;
+}
+
+//===----------------------------------------------------------------------===//
+// TZCNT Instruction
+//
+let Predicates = [HasBMI], Defs = [EFLAGS] in {
+  def TZCNT16rr : I<0xBC, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src),
+                    "tzcnt{w}\t{$src, $dst|$dst, $src}",
+                    [(set GR16:$dst, (cttz GR16:$src)), (implicit EFLAGS)]>, XS,
+                    OpSize;
+  def TZCNT16rm : I<0xBC, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
+                    "tzcnt{w}\t{$src, $dst|$dst, $src}",
+                    [(set GR16:$dst, (cttz (loadi16 addr:$src))),
+                     (implicit EFLAGS)]>, XS, OpSize;
+
+  def TZCNT32rr : I<0xBC, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
+                    "tzcnt{l}\t{$src, $dst|$dst, $src}",
+                    [(set GR32:$dst, (cttz GR32:$src)), (implicit EFLAGS)]>, XS;
+  def TZCNT32rm : I<0xBC, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
+                    "tzcnt{l}\t{$src, $dst|$dst, $src}",
+                    [(set GR32:$dst, (cttz (loadi32 addr:$src))),
+                     (implicit EFLAGS)]>, XS;
+
+  def TZCNT64rr : RI<0xBC, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
+                     "tzcnt{q}\t{$src, $dst|$dst, $src}",
+                     [(set GR64:$dst, (cttz GR64:$src)), (implicit EFLAGS)]>,
+                     XS;
+  def TZCNT64rm : RI<0xBC, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
+                     "tzcnt{q}\t{$src, $dst|$dst, $src}",
+                     [(set GR64:$dst, (cttz (loadi64 addr:$src))),
+                      (implicit EFLAGS)]>, XS;
+}
+
 //===----------------------------------------------------------------------===//
 // Subsystems.
 //===----------------------------------------------------------------------===//
@@ -1646,3 +1791,9 @@ def : InstAlias<"xchgb $mem, $val", (XCHG8rm  GR8 :$val, i8mem :$mem)>;
 def : InstAlias<"xchgw $mem, $val", (XCHG16rm GR16:$val, i16mem:$mem)>;
 def : InstAlias<"xchgl $mem, $val", (XCHG32rm GR32:$val, i32mem:$mem)>;
 def : InstAlias<"xchgq $mem, $val", (XCHG64rm GR64:$val, i64mem:$mem)>;
+
+// xchg: We accept "xchgX <reg>, %eax" and "xchgX %eax, <reg>" as synonyms.
+def : InstAlias<"xchgw %ax, $src", (XCHG16ar GR16:$src)>;
+def : InstAlias<"xchgl %eax, $src", (XCHG32ar GR32:$src)>, Requires<[In32BitMode]>;
+def : InstAlias<"xchgl %eax, $src", (XCHG32ar64 GR32_NOAX:$src)>, Requires<[In64BitMode]>;
+def : InstAlias<"xchgq %rax, $src", (XCHG64ar GR64:$src)>;
diff --git a/contrib/llvm/lib/Target/X86/X86InstrSSE.td b/contrib/llvm/lib/Target/X86/X86InstrSSE.td
index fe11d776804c..d3ced23450fe 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrSSE.td
@@ -116,7 +116,217 @@ multiclass sse12_fp_packed_int<bits<8> opc, string OpcodeStr, RegisterClass RC,
 }
 
 //===----------------------------------------------------------------------===//
-// SSE 1 & 2 - Move Instructions
+//  Non-instruction patterns
+//===----------------------------------------------------------------------===//
+
+// A vector extract of the first f32/f64 position is a subregister copy
+def : Pat<(f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))),
+          (f32 (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+def : Pat<(f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))),
+          (f64 (EXTRACT_SUBREG (v2f64 VR128:$src), sub_sd))>;
+
+// A 128-bit subvector extract from the first 256-bit vector position
+// is a subregister copy that needs no instruction.
+def : Pat<(v4i32 (extract_subvector (v8i32 VR256:$src), (i32 0))),
+          (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), sub_xmm))>;
+def : Pat<(v4f32 (extract_subvector (v8f32 VR256:$src), (i32 0))),
+          (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), sub_xmm))>;
+
+def : Pat<(v2i64 (extract_subvector (v4i64 VR256:$src), (i32 0))),
+          (v2i64 (EXTRACT_SUBREG (v4i64 VR256:$src), sub_xmm))>;
+def : Pat<(v2f64 (extract_subvector (v4f64 VR256:$src), (i32 0))),
+          (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), sub_xmm))>;
+
+def : Pat<(v8i16 (extract_subvector (v16i16 VR256:$src), (i32 0))),
+          (v8i16 (EXTRACT_SUBREG (v16i16 VR256:$src), sub_xmm))>;
+def : Pat<(v16i8 (extract_subvector (v32i8 VR256:$src), (i32 0))),
+          (v16i8 (EXTRACT_SUBREG (v32i8 VR256:$src), sub_xmm))>;
+
+// A 128-bit subvector insert to the first 256-bit vector position
+// is a subregister copy that needs no instruction.
+def : Pat<(insert_subvector undef, (v2i64 VR128:$src), (i32 0)),
+          (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+def : Pat<(insert_subvector undef, (v2f64 VR128:$src), (i32 0)),
+          (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+def : Pat<(insert_subvector undef, (v4i32 VR128:$src), (i32 0)),
+          (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+def : Pat<(insert_subvector undef, (v4f32 VR128:$src), (i32 0)),
+          (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+def : Pat<(insert_subvector undef, (v8i16 VR128:$src), (i32 0)),
+          (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+def : Pat<(insert_subvector undef, (v16i8 VR128:$src), (i32 0)),
+          (INSERT_SUBREG (v32i8 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+
+// Implicitly promote a 32-bit scalar to a vector.
+def : Pat<(v4f32 (scalar_to_vector FR32:$src)),
+          (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FR32:$src, sub_ss)>;
+def : Pat<(v8f32 (scalar_to_vector FR32:$src)),
+          (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), FR32:$src, sub_ss)>;
+// Implicitly promote a 64-bit scalar to a vector.
+def : Pat<(v2f64 (scalar_to_vector FR64:$src)),
+          (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FR64:$src, sub_sd)>;
+def : Pat<(v4f64 (scalar_to_vector FR64:$src)),
+          (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), FR64:$src, sub_sd)>;
+
+// Bitcasts between 128-bit vector types. Return the original type since
+// no instruction is needed for the conversion
+let Predicates = [HasXMMInt] in {
+  def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>;
+  def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>;
+  def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>;
+  def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>;
+  def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>;
+  def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>;
+  def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>;
+  def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>;
+  def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>;
+  def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>;
+  def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>;
+  def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>;
+  def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>;
+  def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>;
+  def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>;
+  def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>;
+  def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>;
+  def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>;
+  def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>;
+  def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>;
+  def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>;
+  def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>;
+  def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>;
+  def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>;
+  def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>;
+  def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>;
+  def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>;
+  def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>;
+  def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>;
+  def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>;
+}
+
+// Bitcasts between 256-bit vector types. Return the original type since
+// no instruction is needed for the conversion
+let Predicates = [HasAVX] in {
+  def : Pat<(v4f64  (bitconvert (v8f32 VR256:$src))),  (v4f64 VR256:$src)>;
+  def : Pat<(v4f64  (bitconvert (v8i32 VR256:$src))),  (v4f64 VR256:$src)>;
+  def : Pat<(v4f64  (bitconvert (v4i64 VR256:$src))),  (v4f64 VR256:$src)>;
+  def : Pat<(v4f64  (bitconvert (v16i16 VR256:$src))), (v4f64 VR256:$src)>;
+  def : Pat<(v4f64  (bitconvert (v32i8 VR256:$src))),  (v4f64 VR256:$src)>;
+  def : Pat<(v8f32  (bitconvert (v8i32 VR256:$src))),  (v8f32 VR256:$src)>;
+  def : Pat<(v8f32  (bitconvert (v4i64 VR256:$src))),  (v8f32 VR256:$src)>;
+  def : Pat<(v8f32  (bitconvert (v4f64 VR256:$src))),  (v8f32 VR256:$src)>;
+  def : Pat<(v8f32  (bitconvert (v32i8 VR256:$src))),  (v8f32 VR256:$src)>;
+  def : Pat<(v8f32  (bitconvert (v16i16 VR256:$src))), (v8f32 VR256:$src)>;
+  def : Pat<(v4i64  (bitconvert (v8f32 VR256:$src))),  (v4i64 VR256:$src)>;
+  def : Pat<(v4i64  (bitconvert (v8i32 VR256:$src))),  (v4i64 VR256:$src)>;
+  def : Pat<(v4i64  (bitconvert (v4f64 VR256:$src))),  (v4i64 VR256:$src)>;
+  def : Pat<(v4i64  (bitconvert (v32i8 VR256:$src))),  (v4i64 VR256:$src)>;
+  def : Pat<(v4i64  (bitconvert (v16i16 VR256:$src))), (v4i64 VR256:$src)>;
+  def : Pat<(v32i8  (bitconvert (v4f64 VR256:$src))),  (v32i8 VR256:$src)>;
+  def : Pat<(v32i8  (bitconvert (v4i64 VR256:$src))),  (v32i8 VR256:$src)>;
+  def : Pat<(v32i8  (bitconvert (v8f32 VR256:$src))),  (v32i8 VR256:$src)>;
+  def : Pat<(v32i8  (bitconvert (v8i32 VR256:$src))),  (v32i8 VR256:$src)>;
+  def : Pat<(v32i8  (bitconvert (v16i16 VR256:$src))), (v32i8 VR256:$src)>;
+  def : Pat<(v8i32  (bitconvert (v32i8 VR256:$src))),  (v8i32 VR256:$src)>;
+  def : Pat<(v8i32  (bitconvert (v16i16 VR256:$src))), (v8i32 VR256:$src)>;
+  def : Pat<(v8i32  (bitconvert (v8f32 VR256:$src))),  (v8i32 VR256:$src)>;
+  def : Pat<(v8i32  (bitconvert (v4i64 VR256:$src))),  (v8i32 VR256:$src)>;
+  def : Pat<(v8i32  (bitconvert (v4f64 VR256:$src))),  (v8i32 VR256:$src)>;
+  def : Pat<(v16i16 (bitconvert (v8f32 VR256:$src))),  (v16i16 VR256:$src)>;
+  def : Pat<(v16i16 (bitconvert (v8i32 VR256:$src))),  (v16i16 VR256:$src)>;
+  def : Pat<(v16i16 (bitconvert (v4i64 VR256:$src))),  (v16i16 VR256:$src)>;
+  def : Pat<(v16i16 (bitconvert (v4f64 VR256:$src))),  (v16i16 VR256:$src)>;
+  def : Pat<(v16i16 (bitconvert (v32i8 VR256:$src))),  (v16i16 VR256:$src)>;
+}
+
+// Alias instructions that map fld0 to pxor for sse.
+// FIXME: Set encoding to pseudo!
+let isReMaterializable = 1, isAsCheapAsAMove = 1, isCodeGenOnly = 1,
+    canFoldAsLoad = 1 in {
+  def FsFLD0SS : I<0xEF, MRMInitReg, (outs FR32:$dst), (ins), "",
+                   [(set FR32:$dst, fp32imm0)]>,
+                   Requires<[HasSSE1]>, TB, OpSize;
+  def FsFLD0SD : I<0xEF, MRMInitReg, (outs FR64:$dst), (ins), "",
+                   [(set FR64:$dst, fpimm0)]>,
+                 Requires<[HasSSE2]>, TB, OpSize;
+  def VFsFLD0SS : I<0xEF, MRMInitReg, (outs FR32:$dst), (ins), "",
+                    [(set FR32:$dst, fp32imm0)]>,
+                    Requires<[HasAVX]>, TB, OpSize, VEX_4V;
+  def VFsFLD0SD : I<0xEF, MRMInitReg, (outs FR64:$dst), (ins), "",
+                    [(set FR64:$dst, fpimm0)]>,
+                    Requires<[HasAVX]>, TB, OpSize, VEX_4V;
+}
+
+//===----------------------------------------------------------------------===//
+// AVX & SSE - Zero/One Vectors
+//===----------------------------------------------------------------------===//
+
+// Alias instruction that maps zero vector to pxor / xorp* for sse.
+// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then
+// swizzled by ExecutionDepsFix to pxor.
+// We set canFoldAsLoad because this can be converted to a constant-pool
+// load of an all-zeros value if folding it would be beneficial.
+let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
+    isPseudo = 1, neverHasSideEffects = 1 in {
+def V_SET0 : I<0, Pseudo, (outs VR128:$dst), (ins), "", []>;
+}
+
+def : Pat<(v4f32 immAllZerosV), (V_SET0)>;
+def : Pat<(v2f64 immAllZerosV), (V_SET0)>;
+def : Pat<(v4i32 immAllZerosV), (V_SET0)>;
+def : Pat<(v2i64 immAllZerosV), (V_SET0)>;
+def : Pat<(v8i16 immAllZerosV), (V_SET0)>;
+def : Pat<(v16i8 immAllZerosV), (V_SET0)>;
+
+
+// The same as done above but for AVX.  The 256-bit ISA does not support PI,
+// and doesn't need it because on sandy bridge the register is set to zero
+// at the rename stage without using any execution unit, so SET0PSY
+// and SET0PDY can be used for vector int instructions without penalty
+// FIXME: Change encoding to pseudo! This is blocked right now by the x86
+// JIT implementatioan, it does not expand the instructions below like
+// X86MCInstLower does.
+let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
+    isCodeGenOnly = 1, Predicates = [HasAVX] in {
+def AVX_SET0PSY : PSI<0x57, MRMInitReg, (outs VR256:$dst), (ins), "",
+                   [(set VR256:$dst, (v8f32 immAllZerosV))]>, VEX_4V;
+def AVX_SET0PDY : PDI<0x57, MRMInitReg, (outs VR256:$dst), (ins), "",
+                   [(set VR256:$dst, (v4f64 immAllZerosV))]>, VEX_4V;
+}
+
+
+// AVX has no support for 256-bit integer instructions, but since the 128-bit
+// VPXOR instruction writes zero to its upper part, it's safe build zeros.
+def : Pat<(v8i32 immAllZerosV), (SUBREG_TO_REG (i32 0), (V_SET0), sub_xmm)>;
+def : Pat<(bc_v8i32 (v8f32 immAllZerosV)),
+          (SUBREG_TO_REG (i32 0), (V_SET0), sub_xmm)>;
+
+def : Pat<(v4i64 immAllZerosV), (SUBREG_TO_REG (i64 0), (V_SET0), sub_xmm)>;
+def : Pat<(bc_v4i64 (v8f32 immAllZerosV)),
+          (SUBREG_TO_REG (i64 0), (V_SET0), sub_xmm)>;
+
+// We set canFoldAsLoad because this can be converted to a constant-pool
+// load of an all-ones value if folding it would be beneficial.
+// FIXME: Change encoding to pseudo! This is blocked right now by the x86
+// JIT implementation, it does not expand the instructions below like
+// X86MCInstLower does.
+let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
+    isCodeGenOnly = 1, ExeDomain = SSEPackedInt in
+  def V_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins), "",
+                         [(set VR128:$dst, (v4i32 immAllOnesV))]>;
+let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
+    isCodeGenOnly = 1, ExeDomain = SSEPackedInt, Predicates = [HasAVX] in
+  def AVX_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins), "",
+                         [(set VR128:$dst, (v4i32 immAllOnesV))]>, VEX_4V;
+
+
+//===----------------------------------------------------------------------===//
+// SSE 1 & 2 - Move FP Scalar Instructions
+//
+// Move Instructions. Register-to-register movss/movsd is not used for FR32/64
+// register copies because it's a partial register update; FsMOVAPSrr/FsMOVAPDrr
+// is used instead. Register-to-register movss/movsd is not modeled as an
+// INSERT_SUBREG because INSERT_SUBREG requires that the insert be implementable
+// in terms of a copy, and just mentioned, we don't use movss/movsd for copies.
 //===----------------------------------------------------------------------===//
 
 class sse12_move_rr<RegisterClass RC, ValueType vt, string asm> :
@@ -130,28 +340,57 @@ class sse12_move_rm<RegisterClass RC, X86MemOperand x86memop,
          !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                         [(set RC:$dst, (mem_pat addr:$src))]>;
 
-// Move Instructions. Register-to-register movss/movsd is not used for FR32/64
-// register copies because it's a partial register update; FsMOVAPSrr/FsMOVAPDrr
-// is used instead. Register-to-register movss/movsd is not modeled as an
-// INSERT_SUBREG because INSERT_SUBREG requires that the insert be implementable
-// in terms of a copy, and just mentioned, we don't use movss/movsd for copies.
+// AVX
 def VMOVSSrr : sse12_move_rr<FR32, v4f32,
-                "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}">, XS, VEX_4V;
+                "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}">, XS, VEX_4V,
+                VEX_LIG;
 def VMOVSDrr : sse12_move_rr<FR64, v2f64,
-                "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}">, XD, VEX_4V;
+                "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}">, XD, VEX_4V,
+                VEX_LIG;
 
-let canFoldAsLoad = 1, isReMaterializable = 1 in {
-  def VMOVSSrm : sse12_move_rm<FR32, f32mem, loadf32, "movss">, XS, VEX;
-
-  let AddedComplexity = 20 in
-    def VMOVSDrm : sse12_move_rm<FR64, f64mem, loadf64, "movsd">, XD, VEX;
+// For the disassembler
+let isCodeGenOnly = 1 in {
+  def VMOVSSrr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst),
+                        (ins VR128:$src1, FR32:$src2),
+                        "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+                        XS, VEX_4V, VEX_LIG;
+  def VMOVSDrr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst),
+                        (ins VR128:$src1, FR64:$src2),
+                        "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+                        XD, VEX_4V, VEX_LIG;
 }
 
+let canFoldAsLoad = 1, isReMaterializable = 1 in {
+  def VMOVSSrm : sse12_move_rm<FR32, f32mem, loadf32, "movss">, XS, VEX,
+                 VEX_LIG;
+  let AddedComplexity = 20 in
+    def VMOVSDrm : sse12_move_rm<FR64, f64mem, loadf64, "movsd">, XD, VEX,
+                   VEX_LIG;
+}
+
+def VMOVSSmr : SI<0x11, MRMDestMem, (outs), (ins f32mem:$dst, FR32:$src),
+                  "movss\t{$src, $dst|$dst, $src}",
+                  [(store FR32:$src, addr:$dst)]>, XS, VEX, VEX_LIG;
+def VMOVSDmr : SI<0x11, MRMDestMem, (outs), (ins f64mem:$dst, FR64:$src),
+                  "movsd\t{$src, $dst|$dst, $src}",
+                  [(store FR64:$src, addr:$dst)]>, XD, VEX, VEX_LIG;
+
+// SSE1 & 2
 let Constraints = "$src1 = $dst" in {
   def MOVSSrr : sse12_move_rr<FR32, v4f32,
                           "movss\t{$src2, $dst|$dst, $src2}">, XS;
   def MOVSDrr : sse12_move_rr<FR64, v2f64,
                           "movsd\t{$src2, $dst|$dst, $src2}">, XD;
+
+  // For the disassembler
+  let isCodeGenOnly = 1 in {
+    def MOVSSrr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst),
+                         (ins VR128:$src1, FR32:$src2),
+                         "movss\t{$src2, $dst|$dst, $src2}", []>, XS;
+    def MOVSDrr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst),
+                         (ins VR128:$src1, FR64:$src2),
+                         "movsd\t{$src2, $dst|$dst, $src2}", []>, XD;
+  }
 }
 
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
@@ -161,54 +400,6 @@ let canFoldAsLoad = 1, isReMaterializable = 1 in {
     def MOVSDrm : sse12_move_rm<FR64, f64mem, loadf64, "movsd">, XD;
 }
 
-let AddedComplexity = 15 in {
-// Extract the low 32-bit value from one vector and insert it into another.
-def : Pat<(v4f32 (movl VR128:$src1, VR128:$src2)),
-          (MOVSSrr (v4f32 VR128:$src1),
-                   (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_ss))>;
-// Extract the low 64-bit value from one vector and insert it into another.
-def : Pat<(v2f64 (movl VR128:$src1, VR128:$src2)),
-          (MOVSDrr (v2f64 VR128:$src1),
-                   (EXTRACT_SUBREG (v2f64 VR128:$src2), sub_sd))>;
-}
-
-// Implicitly promote a 32-bit scalar to a vector.
-def : Pat<(v4f32 (scalar_to_vector FR32:$src)),
-          (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FR32:$src, sub_ss)>;
-// Implicitly promote a 64-bit scalar to a vector.
-def : Pat<(v2f64 (scalar_to_vector FR64:$src)),
-          (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FR64:$src, sub_sd)>;
-// Implicitly promote a 32-bit scalar to a vector.
-def : Pat<(v8f32 (scalar_to_vector FR32:$src)),
-          (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), FR32:$src, sub_ss)>;
-// Implicitly promote a 64-bit scalar to a vector.
-def : Pat<(v4f64 (scalar_to_vector FR64:$src)),
-          (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), FR64:$src, sub_sd)>;
-
-let AddedComplexity = 20 in {
-// MOVSSrm zeros the high parts of the register; represent this
-// with SUBREG_TO_REG.
-def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
-          (SUBREG_TO_REG (i32 0), (MOVSSrm addr:$src), sub_ss)>;
-def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
-          (SUBREG_TO_REG (i32 0), (MOVSSrm addr:$src), sub_ss)>;
-def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
-          (SUBREG_TO_REG (i32 0), (MOVSSrm addr:$src), sub_ss)>;
-// MOVSDrm zeros the high parts of the register; represent this
-// with SUBREG_TO_REG.
-def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
-          (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
-def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
-          (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
-def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
-          (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
-def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
-          (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
-def : Pat<(v2f64 (X86vzload addr:$src)),
-          (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
-}
-
-// Store scalar value to memory.
 def MOVSSmr : SSI<0x11, MRMDestMem, (outs), (ins f32mem:$dst, FR32:$src),
                   "movss\t{$src, $dst|$dst, $src}",
                   [(store FR32:$src, addr:$dst)]>;
@@ -216,24 +407,257 @@ def MOVSDmr : SDI<0x11, MRMDestMem, (outs), (ins f64mem:$dst, FR64:$src),
                   "movsd\t{$src, $dst|$dst, $src}",
                   [(store FR64:$src, addr:$dst)]>;
 
-def VMOVSSmr : SI<0x11, MRMDestMem, (outs), (ins f32mem:$dst, FR32:$src),
-                  "movss\t{$src, $dst|$dst, $src}",
-                  [(store FR32:$src, addr:$dst)]>, XS, VEX;
-def VMOVSDmr : SI<0x11, MRMDestMem, (outs), (ins f64mem:$dst, FR64:$src),
-                  "movsd\t{$src, $dst|$dst, $src}",
-                  [(store FR64:$src, addr:$dst)]>, XD, VEX;
+// Patterns
+let Predicates = [HasSSE1] in {
+  let AddedComplexity = 15 in {
+  // Extract the low 32-bit value from one vector and insert it into another.
+  def : Pat<(v4f32 (movl VR128:$src1, VR128:$src2)),
+            (MOVSSrr (v4f32 VR128:$src1),
+                     (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_ss))>;
+  def : Pat<(v4i32 (movl VR128:$src1, VR128:$src2)),
+            (MOVSSrr (v4i32 VR128:$src1),
+                     (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_ss))>;
 
-// Extract and store.
-def : Pat<(store (f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))),
-                 addr:$dst),
-          (MOVSSmr addr:$dst,
-                   (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
-def : Pat<(store (f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))),
-                 addr:$dst),
-          (MOVSDmr addr:$dst,
-                   (EXTRACT_SUBREG (v2f64 VR128:$src), sub_sd))>;
+  // Move scalar to XMM zero-extended, zeroing a VR128 then do a
+  // MOVSS to the lower bits.
+  def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))),
+            (MOVSSrr (v4f32 (V_SET0)), FR32:$src)>;
+  def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),
+            (MOVSSrr (v4f32 (V_SET0)),
+                     (f32 (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss)))>;
+  def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),
+            (MOVSSrr (v4i32 (V_SET0)),
+                     (EXTRACT_SUBREG (v4i32 VR128:$src), sub_ss))>;
+  }
+
+  let AddedComplexity = 20 in {
+  // MOVSSrm zeros the high parts of the register; represent this
+  // with SUBREG_TO_REG.
+  def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
+            (SUBREG_TO_REG (i32 0), (MOVSSrm addr:$src), sub_ss)>;
+  def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
+            (SUBREG_TO_REG (i32 0), (MOVSSrm addr:$src), sub_ss)>;
+  def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
+            (SUBREG_TO_REG (i32 0), (MOVSSrm addr:$src), sub_ss)>;
+  }
+
+  // Extract and store.
+  def : Pat<(store (f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))),
+                   addr:$dst),
+            (MOVSSmr addr:$dst,
+                     (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+
+  // Shuffle with MOVSS
+  def : Pat<(v4f32 (X86Movss VR128:$src1, (scalar_to_vector FR32:$src2))),
+            (MOVSSrr VR128:$src1, FR32:$src2)>;
+  def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)),
+            (MOVSSrr (v4i32 VR128:$src1),
+                     (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_ss))>;
+  def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)),
+            (MOVSSrr (v4f32 VR128:$src1),
+                     (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_ss))>;
+}
+
+let Predicates = [HasSSE2] in {
+  let AddedComplexity = 15 in {
+  // Extract the low 64-bit value from one vector and insert it into another.
+  def : Pat<(v2f64 (movl VR128:$src1, VR128:$src2)),
+            (MOVSDrr (v2f64 VR128:$src1),
+                     (EXTRACT_SUBREG (v2f64 VR128:$src2), sub_sd))>;
+  def : Pat<(v2i64 (movl VR128:$src1, VR128:$src2)),
+            (MOVSDrr (v2i64 VR128:$src1),
+                     (EXTRACT_SUBREG (v2i64 VR128:$src2), sub_sd))>;
+
+  // vector_shuffle v1, v2 <4, 5, 2, 3> using movsd
+  def : Pat<(v4f32 (movlp VR128:$src1, VR128:$src2)),
+            (MOVSDrr VR128:$src1, (EXTRACT_SUBREG VR128:$src2, sub_sd))>;
+  def : Pat<(v4i32 (movlp VR128:$src1, VR128:$src2)),
+            (MOVSDrr VR128:$src1, (EXTRACT_SUBREG VR128:$src2, sub_sd))>;
+
+  // Move scalar to XMM zero-extended, zeroing a VR128 then do a
+  // MOVSD to the lower bits.
+  def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))),
+            (MOVSDrr (v2f64 (V_SET0)), FR64:$src)>;
+  }
+
+  let AddedComplexity = 20 in {
+  // MOVSDrm zeros the high parts of the register; represent this
+  // with SUBREG_TO_REG.
+  def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
+            (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
+            (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
+            (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
+            (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (X86vzload addr:$src)),
+            (SUBREG_TO_REG (i64 0), (MOVSDrm addr:$src), sub_sd)>;
+  }
+
+  // Extract and store.
+  def : Pat<(store (f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))),
+                   addr:$dst),
+            (MOVSDmr addr:$dst,
+                     (EXTRACT_SUBREG (v2f64 VR128:$src), sub_sd))>;
+
+  // Shuffle with MOVSD
+  def : Pat<(v2f64 (X86Movsd VR128:$src1, (scalar_to_vector FR64:$src2))),
+            (MOVSDrr VR128:$src1, FR64:$src2)>;
+  def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)),
+            (MOVSDrr (v2i64 VR128:$src1),
+                     (EXTRACT_SUBREG (v2i64 VR128:$src2), sub_sd))>;
+  def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)),
+            (MOVSDrr (v2f64 VR128:$src1),
+                     (EXTRACT_SUBREG (v2f64 VR128:$src2), sub_sd))>;
+  def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)),
+            (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4f32 VR128:$src2),sub_sd))>;
+  def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)),
+            (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4i32 VR128:$src2),sub_sd))>;
+
+  // FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem
+  // is during lowering, where it's not possible to recognize the fold cause
+  // it has two uses through a bitcast. One use disappears at isel time and the
+  // fold opportunity reappears.
+  def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)),
+            (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4f32 VR128:$src2),sub_sd))>;
+  def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)),
+            (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4i32 VR128:$src2),sub_sd))>;
+}
+
+let Predicates = [HasAVX] in {
+  let AddedComplexity = 15 in {
+  // Extract the low 32-bit value from one vector and insert it into another.
+  def : Pat<(v4f32 (movl VR128:$src1, VR128:$src2)),
+            (VMOVSSrr (v4f32 VR128:$src1),
+                      (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_ss))>;
+  def : Pat<(v4i32 (movl VR128:$src1, VR128:$src2)),
+            (VMOVSSrr (v4i32 VR128:$src1),
+                      (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_ss))>;
+
+  // Extract the low 64-bit value from one vector and insert it into another.
+  def : Pat<(v2f64 (movl VR128:$src1, VR128:$src2)),
+            (VMOVSDrr (v2f64 VR128:$src1),
+                      (EXTRACT_SUBREG (v2f64 VR128:$src2), sub_sd))>;
+  def : Pat<(v2i64 (movl VR128:$src1, VR128:$src2)),
+            (VMOVSDrr (v2i64 VR128:$src1),
+                      (EXTRACT_SUBREG (v2i64 VR128:$src2), sub_sd))>;
+
+  // vector_shuffle v1, v2 <4, 5, 2, 3> using movsd
+  def : Pat<(v4f32 (movlp VR128:$src1, VR128:$src2)),
+            (VMOVSDrr VR128:$src1, (EXTRACT_SUBREG VR128:$src2, sub_sd))>;
+  def : Pat<(v4i32 (movlp VR128:$src1, VR128:$src2)),
+            (VMOVSDrr VR128:$src1, (EXTRACT_SUBREG VR128:$src2, sub_sd))>;
+
+  // Move scalar to XMM zero-extended, zeroing a VR128 then do a
+  // MOVS{S,D} to the lower bits.
+  def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))),
+            (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)>;
+  def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),
+            (VMOVSSrr (v4f32 (V_SET0)),
+                      (f32 (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss)))>;
+  def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),
+            (VMOVSSrr (v4i32 (V_SET0)),
+                      (EXTRACT_SUBREG (v4i32 VR128:$src), sub_ss))>;
+  def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))),
+            (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)>;
+  }
+
+  let AddedComplexity = 20 in {
+  // MOVSSrm zeros the high parts of the register; represent this
+  // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
+  def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
+            (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_ss)>;
+  def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
+            (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_ss)>;
+  def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
+            (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_ss)>;
+
+  // MOVSDrm zeros the high parts of the register; represent this
+  // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
+  def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
+            (SUBREG_TO_REG (i64 0), (VMOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
+            (SUBREG_TO_REG (i64 0), (VMOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
+            (SUBREG_TO_REG (i64 0), (VMOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
+            (SUBREG_TO_REG (i64 0), (VMOVSDrm addr:$src), sub_sd)>;
+  def : Pat<(v2f64 (X86vzload addr:$src)),
+            (SUBREG_TO_REG (i64 0), (VMOVSDrm addr:$src), sub_sd)>;
+
+  // Represent the same patterns above but in the form they appear for
+  // 256-bit types
+  def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
+                   (v4f32 (scalar_to_vector (loadf32 addr:$src))), (i32 0)))),
+            (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_ss)>;
+  def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
+                   (v2f64 (scalar_to_vector (loadf64 addr:$src))), (i32 0)))),
+            (SUBREG_TO_REG (i32 0), (VMOVSDrm addr:$src), sub_sd)>;
+  }
+  def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
+                   (v4f32 (scalar_to_vector FR32:$src)), (i32 0)))),
+            (SUBREG_TO_REG (i32 0),
+                           (v4f32 (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)),
+                           sub_xmm)>;
+  def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
+                   (v2f64 (scalar_to_vector FR64:$src)), (i32 0)))),
+            (SUBREG_TO_REG (i64 0),
+                           (v2f64 (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)),
+                           sub_xmm)>;
+
+  // Extract and store.
+  def : Pat<(store (f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))),
+                   addr:$dst),
+            (VMOVSSmr addr:$dst,
+                     (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+  def : Pat<(store (f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))),
+                   addr:$dst),
+            (VMOVSDmr addr:$dst,
+                     (EXTRACT_SUBREG (v2f64 VR128:$src), sub_sd))>;
+
+  // Shuffle with VMOVSS
+  def : Pat<(v4f32 (X86Movss VR128:$src1, (scalar_to_vector FR32:$src2))),
+            (VMOVSSrr VR128:$src1, FR32:$src2)>;
+  def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)),
+            (VMOVSSrr (v4i32 VR128:$src1),
+                      (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_ss))>;
+  def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)),
+            (VMOVSSrr (v4f32 VR128:$src1),
+                      (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_ss))>;
+
+  // Shuffle with VMOVSD
+  def : Pat<(v2f64 (X86Movsd VR128:$src1, (scalar_to_vector FR64:$src2))),
+            (VMOVSDrr VR128:$src1, FR64:$src2)>;
+  def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)),
+            (VMOVSDrr (v2i64 VR128:$src1),
+                     (EXTRACT_SUBREG (v2i64 VR128:$src2), sub_sd))>;
+  def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)),
+            (VMOVSDrr (v2f64 VR128:$src1),
+                     (EXTRACT_SUBREG (v2f64 VR128:$src2), sub_sd))>;
+  def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)),
+            (VMOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4f32 VR128:$src2),
+                                                   sub_sd))>;
+  def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)),
+            (VMOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4i32 VR128:$src2),
+                                                   sub_sd))>;
+
+  // FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem
+  // is during lowering, where it's not possible to recognize the fold cause
+  // it has two uses through a bitcast. One use disappears at isel time and the
+  // fold opportunity reappears.
+  def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)),
+            (VMOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4f32 VR128:$src2),
+                                                   sub_sd))>;
+  def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)),
+            (VMOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4i32 VR128:$src2),
+                                                   sub_sd))>;
+}
+
+//===----------------------------------------------------------------------===//
+// SSE 1 & 2 - Move Aligned/Unaligned FP Instructions
+//===----------------------------------------------------------------------===//
 
-// Move Aligned/Unaligned floating point values
 multiclass sse12_mov_packed<bits<8> opc, RegisterClass RC,
                             X86MemOperand x86memop, PatFrag ld_frag,
                             string asm, Domain d,
@@ -248,22 +672,22 @@ let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in
 }
 
 defm VMOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32,
-                              "movaps", SSEPackedSingle>, VEX;
+                              "movaps", SSEPackedSingle>, TB, VEX;
 defm VMOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64,
-                              "movapd", SSEPackedDouble>, OpSize, VEX;
+                              "movapd", SSEPackedDouble>, TB, OpSize, VEX;
 defm VMOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32,
-                              "movups", SSEPackedSingle>, VEX;
+                              "movups", SSEPackedSingle>, TB, VEX;
 defm VMOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64,
-                              "movupd", SSEPackedDouble, 0>, OpSize, VEX;
+                              "movupd", SSEPackedDouble, 0>, TB, OpSize, VEX;
 
 defm VMOVAPSY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv8f32,
-                              "movaps", SSEPackedSingle>, VEX;
+                              "movaps", SSEPackedSingle>, TB, VEX;
 defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64,
-                              "movapd", SSEPackedDouble>, OpSize, VEX;
+                              "movapd", SSEPackedDouble>, TB, OpSize, VEX;
 defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32,
-                              "movups", SSEPackedSingle>, VEX;
+                              "movups", SSEPackedSingle>, TB, VEX;
 defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64,
-                              "movupd", SSEPackedDouble, 0>, OpSize, VEX;
+                              "movupd", SSEPackedDouble, 0>, TB, OpSize, VEX;
 defm MOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32,
                               "movaps", SSEPackedSingle>, TB;
 defm MOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64,
@@ -287,10 +711,10 @@ def VMOVUPDmr : VPDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
                    [(store (v2f64 VR128:$src), addr:$dst)]>, VEX;
 def VMOVAPSYmr : VPSI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movaps\t{$src, $dst|$dst, $src}",
-                   [(alignedstore (v8f32 VR256:$src), addr:$dst)]>, VEX;
+                   [(alignedstore256 (v8f32 VR256:$src), addr:$dst)]>, VEX;
 def VMOVAPDYmr : VPDI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movapd\t{$src, $dst|$dst, $src}",
-                   [(alignedstore (v4f64 VR256:$src), addr:$dst)]>, VEX;
+                   [(alignedstore256 (v4f64 VR256:$src), addr:$dst)]>, VEX;
 def VMOVUPSYmr : VPSI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movups\t{$src, $dst|$dst, $src}",
                    [(store (v8f32 VR256:$src), addr:$dst)]>, VEX;
@@ -298,6 +722,34 @@ def VMOVUPDYmr : VPDI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movupd\t{$src, $dst|$dst, $src}",
                    [(store (v4f64 VR256:$src), addr:$dst)]>, VEX;
 
+// For disassembler
+let isCodeGenOnly = 1 in {
+  def VMOVAPSrr_REV : VPSI<0x29, MRMDestReg, (outs VR128:$dst),
+                          (ins VR128:$src),
+                          "movaps\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVAPDrr_REV : VPDI<0x29, MRMDestReg, (outs VR128:$dst),
+                           (ins VR128:$src),
+                           "movapd\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVUPSrr_REV : VPSI<0x11, MRMDestReg, (outs VR128:$dst),
+                           (ins VR128:$src),
+                           "movups\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVUPDrr_REV : VPDI<0x11, MRMDestReg, (outs VR128:$dst),
+                           (ins VR128:$src),
+                           "movupd\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVAPSYrr_REV : VPSI<0x29, MRMDestReg, (outs VR256:$dst),
+                            (ins VR256:$src),
+                            "movaps\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVAPDYrr_REV : VPDI<0x29, MRMDestReg, (outs VR256:$dst),
+                            (ins VR256:$src),
+                            "movapd\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVUPSYrr_REV : VPSI<0x11, MRMDestReg, (outs VR256:$dst),
+                            (ins VR256:$src),
+                            "movups\t{$src, $dst|$dst, $src}", []>, VEX;
+  def VMOVUPDYrr_REV : VPDI<0x11, MRMDestReg, (outs VR256:$dst),
+                            (ins VR256:$src),
+                            "movupd\t{$src, $dst|$dst, $src}", []>, VEX;
+}
+
 def : Pat<(int_x86_avx_loadu_ps_256 addr:$src), (VMOVUPSYrm addr:$src)>;
 def : Pat<(int_x86_avx_storeu_ps_256 addr:$dst, VR256:$src),
           (VMOVUPSYmr addr:$dst, VR256:$src)>;
@@ -319,24 +771,155 @@ def MOVUPDmr : PDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
                    "movupd\t{$src, $dst|$dst, $src}",
                    [(store (v2f64 VR128:$src), addr:$dst)]>;
 
-// Intrinsic forms of MOVUPS/D load and store
-def VMOVUPSmr_Int : VPSI<0x11, MRMDestMem, (outs),
-           (ins f128mem:$dst, VR128:$src),
-           "movups\t{$src, $dst|$dst, $src}",
-           [(int_x86_sse_storeu_ps addr:$dst, VR128:$src)]>, VEX;
-def VMOVUPDmr_Int : VPDI<0x11, MRMDestMem, (outs),
-           (ins f128mem:$dst, VR128:$src),
-           "movupd\t{$src, $dst|$dst, $src}",
-           [(int_x86_sse2_storeu_pd addr:$dst, VR128:$src)]>, VEX;
+// For disassembler
+let isCodeGenOnly = 1 in {
+  def MOVAPSrr_REV : PSI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                         "movaps\t{$src, $dst|$dst, $src}", []>;
+  def MOVAPDrr_REV : PDI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                         "movapd\t{$src, $dst|$dst, $src}", []>;
+  def MOVUPSrr_REV : PSI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                         "movups\t{$src, $dst|$dst, $src}", []>;
+  def MOVUPDrr_REV : PDI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                         "movupd\t{$src, $dst|$dst, $src}", []>;
+}
 
-def MOVUPSmr_Int : PSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
-                       "movups\t{$src, $dst|$dst, $src}",
-                       [(int_x86_sse_storeu_ps addr:$dst, VR128:$src)]>;
-def MOVUPDmr_Int : PDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
-                       "movupd\t{$src, $dst|$dst, $src}",
-                       [(int_x86_sse2_storeu_pd addr:$dst, VR128:$src)]>;
+let Predicates = [HasAVX] in {
+  def : Pat<(int_x86_sse_storeu_ps addr:$dst, VR128:$src),
+            (VMOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(int_x86_sse2_storeu_pd addr:$dst, VR128:$src),
+            (VMOVUPDmr addr:$dst, VR128:$src)>;
+}
+
+let Predicates = [HasSSE1] in
+  def : Pat<(int_x86_sse_storeu_ps addr:$dst, VR128:$src),
+            (MOVUPSmr addr:$dst, VR128:$src)>;
+let Predicates = [HasSSE2] in
+  def : Pat<(int_x86_sse2_storeu_pd addr:$dst, VR128:$src),
+            (MOVUPDmr addr:$dst, VR128:$src)>;
+
+// Use movaps / movups for SSE integer load / store (one byte shorter).
+// The instructions selected below are then converted to MOVDQA/MOVDQU
+// during the SSE domain pass.
+let Predicates = [HasSSE1] in {
+  def : Pat<(alignedloadv4i32 addr:$src),
+            (MOVAPSrm addr:$src)>;
+  def : Pat<(loadv4i32 addr:$src),
+            (MOVUPSrm addr:$src)>;
+  def : Pat<(alignedloadv2i64 addr:$src),
+            (MOVAPSrm addr:$src)>;
+  def : Pat<(loadv2i64 addr:$src),
+            (MOVUPSrm addr:$src)>;
+
+  def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst),
+            (MOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst),
+            (MOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst),
+            (MOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst),
+            (MOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v2i64 VR128:$src), addr:$dst),
+            (MOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v4i32 VR128:$src), addr:$dst),
+            (MOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v8i16 VR128:$src), addr:$dst),
+            (MOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v16i8 VR128:$src), addr:$dst),
+            (MOVUPSmr addr:$dst, VR128:$src)>;
+}
+
+// Use vmovaps/vmovups for AVX integer load/store.
+let Predicates = [HasAVX] in {
+  // 128-bit load/store
+  def : Pat<(alignedloadv4i32 addr:$src),
+            (VMOVAPSrm addr:$src)>;
+  def : Pat<(loadv4i32 addr:$src),
+            (VMOVUPSrm addr:$src)>;
+  def : Pat<(alignedloadv2i64 addr:$src),
+            (VMOVAPSrm addr:$src)>;
+  def : Pat<(loadv2i64 addr:$src),
+            (VMOVUPSrm addr:$src)>;
+
+  def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst),
+            (VMOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst),
+            (VMOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst),
+            (VMOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst),
+            (VMOVAPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v2i64 VR128:$src), addr:$dst),
+            (VMOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v4i32 VR128:$src), addr:$dst),
+            (VMOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v8i16 VR128:$src), addr:$dst),
+            (VMOVUPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (v16i8 VR128:$src), addr:$dst),
+            (VMOVUPSmr addr:$dst, VR128:$src)>;
+
+  // 256-bit load/store
+  def : Pat<(alignedloadv4i64 addr:$src),
+            (VMOVAPSYrm addr:$src)>;
+  def : Pat<(loadv4i64 addr:$src),
+            (VMOVUPSYrm addr:$src)>;
+  def : Pat<(alignedloadv8i32 addr:$src),
+            (VMOVAPSYrm addr:$src)>;
+  def : Pat<(loadv8i32 addr:$src),
+            (VMOVUPSYrm addr:$src)>;
+  def : Pat<(alignedstore256 (v4i64 VR256:$src), addr:$dst),
+            (VMOVAPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(alignedstore256 (v8i32 VR256:$src), addr:$dst),
+            (VMOVAPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(alignedstore256 (v16i16 VR256:$src), addr:$dst),
+            (VMOVAPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(alignedstore256 (v32i8 VR256:$src), addr:$dst),
+            (VMOVAPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(store (v4i64 VR256:$src), addr:$dst),
+            (VMOVUPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(store (v8i32 VR256:$src), addr:$dst),
+            (VMOVUPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(store (v16i16 VR256:$src), addr:$dst),
+            (VMOVUPSYmr addr:$dst, VR256:$src)>;
+  def : Pat<(store (v32i8 VR256:$src), addr:$dst),
+            (VMOVUPSYmr addr:$dst, VR256:$src)>;
+}
+
+// Alias instruction to do FR32 or FR64 reg-to-reg copy using movaps. Upper
+// bits are disregarded. FIXME: Set encoding to pseudo!
+let neverHasSideEffects = 1 in {
+def FsMOVAPSrr : PSI<0x28, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src),
+                     "movaps\t{$src, $dst|$dst, $src}", []>;
+def FsMOVAPDrr : PDI<0x28, MRMSrcReg, (outs FR64:$dst), (ins FR64:$src),
+                     "movapd\t{$src, $dst|$dst, $src}", []>;
+def FsVMOVAPSrr : VPSI<0x28, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src),
+                       "movaps\t{$src, $dst|$dst, $src}", []>, VEX;
+def FsVMOVAPDrr : VPDI<0x28, MRMSrcReg, (outs FR64:$dst), (ins FR64:$src),
+                       "movapd\t{$src, $dst|$dst, $src}", []>, VEX;
+}
+
+// Alias instruction to load FR32 or FR64 from f128mem using movaps. Upper
+// bits are disregarded. FIXME: Set encoding to pseudo!
+let canFoldAsLoad = 1, isReMaterializable = 1 in {
+def FsMOVAPSrm : PSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src),
+                     "movaps\t{$src, $dst|$dst, $src}",
+                     [(set FR32:$dst, (alignedloadfsf32 addr:$src))]>;
+def FsMOVAPDrm : PDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src),
+                     "movapd\t{$src, $dst|$dst, $src}",
+                     [(set FR64:$dst, (alignedloadfsf64 addr:$src))]>;
+let isCodeGenOnly = 1 in {
+  def FsVMOVAPSrm : VPSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src),
+                         "movaps\t{$src, $dst|$dst, $src}",
+                         [(set FR32:$dst, (alignedloadfsf32 addr:$src))]>, VEX;
+  def FsVMOVAPDrm : VPDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src),
+                         "movapd\t{$src, $dst|$dst, $src}",
+                         [(set FR64:$dst, (alignedloadfsf64 addr:$src))]>, VEX;
+}
+}
+
+//===----------------------------------------------------------------------===//
+// SSE 1 & 2 - Move Low packed FP Instructions
+//===----------------------------------------------------------------------===//
 
-// Move Low/High packed floating point values
 multiclass sse12_mov_hilo_packed<bits<8>opc, RegisterClass RC,
                                  PatFrag mov_frag, string base_opc,
                                  string asm_opr> {
@@ -359,14 +942,10 @@ multiclass sse12_mov_hilo_packed<bits<8>opc, RegisterClass RC,
 let AddedComplexity = 20 in {
   defm VMOVL : sse12_mov_hilo_packed<0x12, VR128, movlp, "movlp",
                      "\t{$src2, $src1, $dst|$dst, $src1, $src2}">, VEX_4V;
-  defm VMOVH : sse12_mov_hilo_packed<0x16, VR128, movlhps, "movhp",
-                     "\t{$src2, $src1, $dst|$dst, $src1, $src2}">, VEX_4V;
 }
 let Constraints = "$src1 = $dst", AddedComplexity = 20 in {
   defm MOVL : sse12_mov_hilo_packed<0x12, VR128, movlp, "movlp",
                                    "\t{$src2, $dst|$dst, $src2}">;
-  defm MOVH : sse12_mov_hilo_packed<0x16, VR128, movlhps, "movhp",
-                                   "\t{$src2, $dst|$dst, $src2}">;
 }
 
 def VMOVLPSmr : VPSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),
@@ -386,6 +965,147 @@ def MOVLPDmr : PDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),
                    [(store (f64 (vector_extract (v2f64 VR128:$src),
                                  (iPTR 0))), addr:$dst)]>;
 
+let Predicates = [HasAVX] in {
+  let AddedComplexity = 20 in {
+    // vector_shuffle v1, (load v2) <4, 5, 2, 3> using MOVLPS
+    def : Pat<(v4f32 (movlp VR128:$src1, (load addr:$src2))),
+              (VMOVLPSrm VR128:$src1, addr:$src2)>;
+    def : Pat<(v4i32 (movlp VR128:$src1, (load addr:$src2))),
+              (VMOVLPSrm VR128:$src1, addr:$src2)>;
+    // vector_shuffle v1, (load v2) <2, 1> using MOVLPS
+    def : Pat<(v2f64 (movlp VR128:$src1, (load addr:$src2))),
+              (VMOVLPDrm VR128:$src1, addr:$src2)>;
+    def : Pat<(v2i64 (movlp VR128:$src1, (load addr:$src2))),
+              (VMOVLPDrm VR128:$src1, addr:$src2)>;
+  }
+
+  // (store (vector_shuffle (load addr), v2, <4, 5, 2, 3>), addr) using MOVLPS
+  def : Pat<(store (v4f32 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
+            (VMOVLPSmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v4i32 (movlp (bc_v4i32 (loadv2i64 addr:$src1)),
+                                 VR128:$src2)), addr:$src1),
+            (VMOVLPSmr addr:$src1, VR128:$src2)>;
+
+  // (store (vector_shuffle (load addr), v2, <2, 1>), addr) using MOVLPS
+  def : Pat<(store (v2f64 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
+            (VMOVLPDmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v2i64 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
+            (VMOVLPDmr addr:$src1, VR128:$src2)>;
+
+  // Shuffle with VMOVLPS
+  def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))),
+            (VMOVLPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))),
+            (VMOVLPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(X86Movlps VR128:$src1,
+                      (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
+            (VMOVLPSrm VR128:$src1, addr:$src2)>;
+
+  // Shuffle with VMOVLPD
+  def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))),
+            (VMOVLPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))),
+            (VMOVLPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2f64 (X86Movlpd VR128:$src1,
+                              (scalar_to_vector (loadf64 addr:$src2)))),
+            (VMOVLPDrm VR128:$src1, addr:$src2)>;
+
+  // Store patterns
+  def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)),
+                   addr:$src1),
+            (VMOVLPSmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v4i32 (X86Movlps
+                   (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)), addr:$src1),
+            (VMOVLPSmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)),
+                   addr:$src1),
+            (VMOVLPDmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)),
+                   addr:$src1),
+            (VMOVLPDmr addr:$src1, VR128:$src2)>;
+}
+
+let Predicates = [HasSSE1] in {
+  let AddedComplexity = 20 in {
+    // vector_shuffle v1, (load v2) <4, 5, 2, 3> using MOVLPS
+    def : Pat<(v4f32 (movlp VR128:$src1, (load addr:$src2))),
+              (MOVLPSrm VR128:$src1, addr:$src2)>;
+    def : Pat<(v4i32 (movlp VR128:$src1, (load addr:$src2))),
+              (MOVLPSrm VR128:$src1, addr:$src2)>;
+  }
+
+  // (store (vector_shuffle (load addr), v2, <4, 5, 2, 3>), addr) using MOVLPS
+  def : Pat<(store (v4f32 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
+            (MOVLPSmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v4i32 (movlp (bc_v4i32 (loadv2i64 addr:$src1)),
+                                 VR128:$src2)), addr:$src1),
+            (MOVLPSmr addr:$src1, VR128:$src2)>;
+
+  // Shuffle with MOVLPS
+  def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))),
+            (MOVLPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))),
+            (MOVLPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(X86Movlps VR128:$src1,
+                      (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
+            (MOVLPSrm VR128:$src1, addr:$src2)>;
+
+  // Store patterns
+  def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)),
+                                      addr:$src1),
+            (MOVLPSmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v4i32 (X86Movlps
+                   (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)),
+                              addr:$src1),
+            (MOVLPSmr addr:$src1, VR128:$src2)>;
+}
+
+let Predicates = [HasSSE2] in {
+  let AddedComplexity = 20 in {
+    // vector_shuffle v1, (load v2) <2, 1> using MOVLPS
+    def : Pat<(v2f64 (movlp VR128:$src1, (load addr:$src2))),
+              (MOVLPDrm VR128:$src1, addr:$src2)>;
+    def : Pat<(v2i64 (movlp VR128:$src1, (load addr:$src2))),
+              (MOVLPDrm VR128:$src1, addr:$src2)>;
+  }
+
+  // (store (vector_shuffle (load addr), v2, <2, 1>), addr) using MOVLPS
+  def : Pat<(store (v2f64 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
+            (MOVLPDmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v2i64 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
+            (MOVLPDmr addr:$src1, VR128:$src2)>;
+
+  // Shuffle with MOVLPD
+  def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))),
+            (MOVLPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))),
+            (MOVLPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2f64 (X86Movlpd VR128:$src1,
+                              (scalar_to_vector (loadf64 addr:$src2)))),
+            (MOVLPDrm VR128:$src1, addr:$src2)>;
+
+  // Store patterns
+  def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)),
+                           addr:$src1),
+            (MOVLPDmr addr:$src1, VR128:$src2)>;
+  def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)),
+                           addr:$src1),
+            (MOVLPDmr addr:$src1, VR128:$src2)>;
+}
+
+//===----------------------------------------------------------------------===//
+// SSE 1 & 2 - Move Hi packed FP Instructions
+//===----------------------------------------------------------------------===//
+
+let AddedComplexity = 20 in {
+  defm VMOVH : sse12_mov_hilo_packed<0x16, VR128, movlhps, "movhp",
+                     "\t{$src2, $src1, $dst|$dst, $src1, $src2}">, VEX_4V;
+}
+let Constraints = "$src1 = $dst", AddedComplexity = 20 in {
+  defm MOVH : sse12_mov_hilo_packed<0x16, VR128, movlhps, "movhp",
+                                   "\t{$src2, $dst|$dst, $src2}">;
+}
+
 // v2f64 extract element 1 is always custom lowered to unpack high to low
 // and extract element 0 so the non-store version isn't too horrible.
 def VMOVHPSmr : VPSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),
@@ -411,6 +1131,80 @@ def MOVHPDmr : PDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),
                                  (v2f64 (unpckh VR128:$src, (undef))),
                                  (iPTR 0))), addr:$dst)]>;
 
+let Predicates = [HasAVX] in {
+  // VMOVHPS patterns
+  def : Pat<(movlhps VR128:$src1, (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
+            (VMOVHPSrm (v4i32 VR128:$src1), addr:$src2)>;
+  def : Pat<(X86Movlhps VR128:$src1,
+                 (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
+            (VMOVHPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(X86Movlhps VR128:$src1,
+                 (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
+            (VMOVHPSrm VR128:$src1, addr:$src2)>;
+
+  // FIXME: Instead of X86Unpcklpd, there should be a X86Movlhpd here, the problem
+  // is during lowering, where it's not possible to recognize the load fold cause
+  // it has two uses through a bitcast. One use disappears at isel time and the
+  // fold opportunity reappears.
+  def : Pat<(v2f64 (X86Unpcklpd VR128:$src1,
+                      (scalar_to_vector (loadf64 addr:$src2)))),
+            (VMOVHPDrm VR128:$src1, addr:$src2)>;
+
+  // FIXME: This should be matched by a X86Movhpd instead. Same as above
+  def : Pat<(v2f64 (X86Movlhpd VR128:$src1,
+                      (scalar_to_vector (loadf64 addr:$src2)))),
+            (VMOVHPDrm VR128:$src1, addr:$src2)>;
+
+  // Store patterns
+  def : Pat<(store (f64 (vector_extract
+            (v2f64 (X86Unpckhps VR128:$src, (undef))), (iPTR 0))), addr:$dst),
+            (VMOVHPSmr addr:$dst, VR128:$src)>;
+  def : Pat<(store (f64 (vector_extract
+            (v2f64 (X86Unpckhpd VR128:$src, (undef))), (iPTR 0))), addr:$dst),
+            (VMOVHPDmr addr:$dst, VR128:$src)>;
+}
+
+let Predicates = [HasSSE1] in {
+  // MOVHPS patterns
+  def : Pat<(movlhps VR128:$src1, (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
+            (MOVHPSrm (v4i32 VR128:$src1), addr:$src2)>;
+  def : Pat<(X86Movlhps VR128:$src1,
+                 (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
+            (MOVHPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(X86Movlhps VR128:$src1,
+                 (bc_v4f32 (v2i64 (X86vzload addr:$src2)))),
+            (MOVHPSrm VR128:$src1, addr:$src2)>;
+
+  // Store patterns
+  def : Pat<(store (f64 (vector_extract
+            (v2f64 (X86Unpckhps VR128:$src, (undef))), (iPTR 0))), addr:$dst),
+            (MOVHPSmr addr:$dst, VR128:$src)>;
+}
+
+let Predicates = [HasSSE2] in {
+  // FIXME: Instead of X86Unpcklpd, there should be a X86Movlhpd here, the problem
+  // is during lowering, where it's not possible to recognize the load fold cause
+  // it has two uses through a bitcast. One use disappears at isel time and the
+  // fold opportunity reappears.
+  def : Pat<(v2f64 (X86Unpcklpd VR128:$src1,
+                      (scalar_to_vector (loadf64 addr:$src2)))),
+            (MOVHPDrm VR128:$src1, addr:$src2)>;
+
+  // FIXME: This should be matched by a X86Movhpd instead. Same as above
+  def : Pat<(v2f64 (X86Movlhpd VR128:$src1,
+                      (scalar_to_vector (loadf64 addr:$src2)))),
+            (MOVHPDrm VR128:$src1, addr:$src2)>;
+
+  // Store patterns
+  def : Pat<(store (f64 (vector_extract
+            (v2f64 (X86Unpckhpd VR128:$src, (undef))), (iPTR 0))),addr:$dst),
+            (MOVHPDmr addr:$dst, VR128:$src)>;
+}
+
+//===----------------------------------------------------------------------===//
+// SSE 1 & 2 - Move Low to High and High to Low packed FP Instructions
+//===----------------------------------------------------------------------===//
+
 let AddedComplexity = 20 in {
   def VMOVLHPSrr : VPSI<0x16, MRMSrcReg, (outs VR128:$dst),
                                        (ins VR128:$src1, VR128:$src2),
@@ -438,13 +1232,80 @@ let Constraints = "$src1 = $dst", AddedComplexity = 20 in {
                         (v4f32 (movhlps VR128:$src1, VR128:$src2)))]>;
 }
 
-def : Pat<(movlhps VR128:$src1, (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
-          (MOVHPSrm (v4i32 VR128:$src1), addr:$src2)>;
-let AddedComplexity = 20 in {
-  def : Pat<(v4f32 (movddup VR128:$src, (undef))),
-            (MOVLHPSrr (v4f32 VR128:$src), (v4f32 VR128:$src))>;
-  def : Pat<(v2i64 (movddup VR128:$src, (undef))),
-            (MOVLHPSrr (v2i64 VR128:$src), (v2i64 VR128:$src))>;
+let Predicates = [HasAVX] in {
+  // MOVLHPS patterns
+  let AddedComplexity = 20 in {
+    def : Pat<(v4f32 (movddup VR128:$src, (undef))),
+              (VMOVLHPSrr (v4f32 VR128:$src), (v4f32 VR128:$src))>;
+    def : Pat<(v2i64 (movddup VR128:$src, (undef))),
+              (VMOVLHPSrr (v2i64 VR128:$src), (v2i64 VR128:$src))>;
+
+    // vector_shuffle v1, v2 <0, 1, 4, 5> using MOVLHPS
+    def : Pat<(v4i32 (movlhps VR128:$src1, VR128:$src2)),
+              (VMOVLHPSrr VR128:$src1, VR128:$src2)>;
+  }
+  def : Pat<(v4f32 (X86Movlhps VR128:$src1, VR128:$src2)),
+            (VMOVLHPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)),
+            (VMOVLHPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)),
+            (VMOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>;
+
+  // MOVHLPS patterns
+  let AddedComplexity = 20 in {
+    // vector_shuffle v1, v2 <6, 7, 2, 3> using MOVHLPS
+    def : Pat<(v4i32 (movhlps VR128:$src1, VR128:$src2)),
+              (VMOVHLPSrr VR128:$src1, VR128:$src2)>;
+
+    // vector_shuffle v1, undef <2, ?, ?, ?> using MOVHLPS
+    def : Pat<(v4f32 (movhlps_undef VR128:$src1, (undef))),
+              (VMOVHLPSrr VR128:$src1, VR128:$src1)>;
+    def : Pat<(v4i32 (movhlps_undef VR128:$src1, (undef))),
+              (VMOVHLPSrr VR128:$src1, VR128:$src1)>;
+  }
+
+  def : Pat<(v4f32 (X86Movhlps VR128:$src1, VR128:$src2)),
+            (VMOVHLPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)),
+            (VMOVHLPSrr VR128:$src1, VR128:$src2)>;
+}
+
+let Predicates = [HasSSE1] in {
+  // MOVLHPS patterns
+  let AddedComplexity = 20 in {
+    def : Pat<(v4f32 (movddup VR128:$src, (undef))),
+              (MOVLHPSrr (v4f32 VR128:$src), (v4f32 VR128:$src))>;
+    def : Pat<(v2i64 (movddup VR128:$src, (undef))),
+              (MOVLHPSrr (v2i64 VR128:$src), (v2i64 VR128:$src))>;
+
+    // vector_shuffle v1, v2 <0, 1, 4, 5> using MOVLHPS
+    def : Pat<(v4i32 (movlhps VR128:$src1, VR128:$src2)),
+              (MOVLHPSrr VR128:$src1, VR128:$src2)>;
+  }
+  def : Pat<(v4f32 (X86Movlhps VR128:$src1, VR128:$src2)),
+            (MOVLHPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)),
+            (MOVLHPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)),
+            (MOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>;
+
+  // MOVHLPS patterns
+  let AddedComplexity = 20 in {
+    // vector_shuffle v1, v2 <6, 7, 2, 3> using MOVHLPS
+    def : Pat<(v4i32 (movhlps VR128:$src1, VR128:$src2)),
+              (MOVHLPSrr VR128:$src1, VR128:$src2)>;
+
+    // vector_shuffle v1, undef <2, ?, ?, ?> using MOVHLPS
+    def : Pat<(v4f32 (movhlps_undef VR128:$src1, (undef))),
+              (MOVHLPSrr VR128:$src1, VR128:$src1)>;
+    def : Pat<(v4i32 (movhlps_undef VR128:$src1, (undef))),
+              (MOVHLPSrr VR128:$src1, VR128:$src1)>;
+  }
+
+  def : Pat<(v4f32 (X86Movhlps VR128:$src1, VR128:$src2)),
+            (MOVHLPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)),
+            (MOVHLPSrr VR128:$src1, VR128:$src2)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -462,10 +1323,9 @@ multiclass sse12_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
 
 multiclass sse12_cvt_s_np<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
                           X86MemOperand x86memop, string asm> {
-  def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm,
-                        []>;
-  def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm,
-                        []>;
+  def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, []>;
+  let mayLoad = 1 in
+  def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, []>;
 }
 
 multiclass sse12_cvt_p<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
@@ -481,36 +1341,39 @@ multiclass sse12_vcvt_avx<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
                           X86MemOperand x86memop, string asm> {
   def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src),
               !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>;
+  let mayLoad = 1 in
   def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst),
               (ins DstRC:$src1, x86memop:$src),
               !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>;
 }
 
 defm VCVTTSS2SI   : sse12_cvt_s<0x2C, FR32, GR32, fp_to_sint, f32mem, loadf32,
-                                "cvttss2si\t{$src, $dst|$dst, $src}">, XS, VEX;
+                                "cvttss2si\t{$src, $dst|$dst, $src}">, XS, VEX,
+                                VEX_LIG;
 defm VCVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, fp_to_sint, f32mem, loadf32,
                                 "cvttss2si\t{$src, $dst|$dst, $src}">, XS, VEX,
-                                VEX_W;
+                                VEX_W, VEX_LIG;
 defm VCVTTSD2SI   : sse12_cvt_s<0x2C, FR64, GR32, fp_to_sint, f64mem, loadf64,
-                                "cvttsd2si\t{$src, $dst|$dst, $src}">, XD, VEX;
+                                "cvttsd2si\t{$src, $dst|$dst, $src}">, XD, VEX,
+                                VEX_LIG;
 defm VCVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, fp_to_sint, f64mem, loadf64,
                                 "cvttsd2si\t{$src, $dst|$dst, $src}">, XD,
-                                VEX, VEX_W;
+                                VEX, VEX_W, VEX_LIG;
 
 // The assembler can recognize rr 64-bit instructions by seeing a rxx
 // register, but the same isn't true when only using memory operands,
 // provide other assembly "l" and "q" forms to address this explicitly
 // where appropriate to do so.
 defm VCVTSI2SS   : sse12_vcvt_avx<0x2A, GR32, FR32, i32mem, "cvtsi2ss">, XS,
-                                  VEX_4V;
+                                  VEX_4V, VEX_LIG;
 defm VCVTSI2SS64 : sse12_vcvt_avx<0x2A, GR64, FR32, i64mem, "cvtsi2ss{q}">, XS,
-                                  VEX_4V, VEX_W;
+                                  VEX_4V, VEX_W, VEX_LIG;
 defm VCVTSI2SD   : sse12_vcvt_avx<0x2A, GR32, FR64, i32mem, "cvtsi2sd">, XD,
-                                  VEX_4V;
+                                  VEX_4V, VEX_LIG;
 defm VCVTSI2SDL  : sse12_vcvt_avx<0x2A, GR32, FR64, i32mem, "cvtsi2sd{l}">, XD,
-                                  VEX_4V;
+                                  VEX_4V, VEX_LIG;
 defm VCVTSI2SD64 : sse12_vcvt_avx<0x2A, GR64, FR64, i64mem, "cvtsi2sd{q}">, XD,
-                                  VEX_4V, VEX_W;
+                                  VEX_4V, VEX_W, VEX_LIG;
 
 let Predicates = [HasAVX] in {
   def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
@@ -579,11 +1442,6 @@ multiclass sse12_cvt_sint_3addr<bits<8> opc, RegisterClass SrcRC,
               [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))]>;
 }
 
-defm Int_VCVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si,
-                      f32mem, load, "cvtss2si">, XS, VEX;
-defm Int_VCVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64,
-                        int_x86_sse_cvtss2si64, f32mem, load, "cvtss2si">,
-                        XS, VEX, VEX_W;
 defm Int_VCVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse2_cvtsd2si,
                       f128mem, load, "cvtsd2si">, XD, VEX;
 defm Int_VCVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64,
@@ -594,14 +1452,12 @@ defm Int_VCVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64,
 // Get rid of this hack or rename the intrinsics, there are several
 // intructions that only match with the intrinsic form, why create duplicates
 // to let them be recognized by the assembler?
-defm VCVTSD2SI_alt : sse12_cvt_s_np<0x2D, FR64, GR32, f64mem,
-                      "cvtsd2si\t{$src, $dst|$dst, $src}">, XD, VEX;
+defm VCVTSD2SI     : sse12_cvt_s_np<0x2D, FR64, GR32, f64mem,
+                      "cvtsd2si\t{$src, $dst|$dst, $src}">, XD, VEX, VEX_LIG;
 defm VCVTSD2SI64   : sse12_cvt_s_np<0x2D, FR64, GR64, f64mem,
-                      "cvtsd2si\t{$src, $dst|$dst, $src}">, XD, VEX, VEX_W;
-defm Int_CVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si,
-                      f32mem, load, "cvtss2si">, XS;
-defm Int_CVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse_cvtss2si64,
-                      f32mem, load, "cvtss2si{q}">, XS, REX_W;
+                      "cvtsd2si\t{$src, $dst|$dst, $src}">, XD, VEX, VEX_W,
+                      VEX_LIG;
+
 defm CVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse2_cvtsd2si,
                 f128mem, load, "cvtsd2si{l}">, XD;
 defm CVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse2_cvtsd2si64,
@@ -660,10 +1516,11 @@ defm Int_CVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64,
 
 let Pattern = []<dag> in {
 defm VCVTSS2SI   : sse12_cvt_s<0x2D, FR32, GR32, undef, f32mem, load,
-                               "cvtss2si{l}\t{$src, $dst|$dst, $src}">, XS, VEX;
+                               "cvtss2si{l}\t{$src, $dst|$dst, $src}">, XS,
+                               VEX, VEX_LIG;
 defm VCVTSS2SI64 : sse12_cvt_s<0x2D, FR32, GR64, undef, f32mem, load,
                                "cvtss2si\t{$src, $dst|$dst, $src}">, XS, VEX,
-                               VEX_W;
+                               VEX_W, VEX_LIG;
 defm VCVTDQ2PS   : sse12_cvt_p<0x5B, VR128, VR128, undef, i128mem, load,
                                "cvtdq2ps\t{$src, $dst|$dst, $src}",
                                SSEPackedSingle>, TB, VEX;
@@ -671,6 +1528,7 @@ defm VCVTDQ2PSY  : sse12_cvt_p<0x5B, VR256, VR256, undef, i256mem, load,
                                "cvtdq2ps\t{$src, $dst|$dst, $src}",
                                SSEPackedSingle>, TB, VEX;
 }
+
 let Pattern = []<dag> in {
 defm CVTSS2SI : sse12_cvt_s<0x2D, FR32, GR32, undef, f32mem, load /*dummy*/,
                           "cvtss2si{l}\t{$src, $dst|$dst, $src}">, XS;
@@ -681,19 +1539,43 @@ defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, undef, i128mem, load /*dummy*/,
                             SSEPackedSingle>, TB; /* PD SSE3 form is avaiable */
 }
 
+let Predicates = [HasSSE1] in {
+  def : Pat<(int_x86_sse_cvtss2si VR128:$src),
+            (CVTSS2SIrr (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+  def : Pat<(int_x86_sse_cvtss2si (load addr:$src)),
+            (CVTSS2SIrm addr:$src)>;
+  def : Pat<(int_x86_sse_cvtss2si64 VR128:$src),
+            (CVTSS2SI64rr (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+  def : Pat<(int_x86_sse_cvtss2si64 (load addr:$src)),
+            (CVTSS2SI64rm addr:$src)>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(int_x86_sse_cvtss2si VR128:$src),
+            (VCVTSS2SIrr (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+  def : Pat<(int_x86_sse_cvtss2si (load addr:$src)),
+            (VCVTSS2SIrm addr:$src)>;
+  def : Pat<(int_x86_sse_cvtss2si64 VR128:$src),
+            (VCVTSS2SI64rr (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
+  def : Pat<(int_x86_sse_cvtss2si64 (load addr:$src)),
+            (VCVTSS2SI64rm addr:$src)>;
+}
+
 /// SSE 2 Only
 
 // Convert scalar double to scalar single
 def VCVTSD2SSrr  : VSDI<0x5A, MRMSrcReg, (outs FR32:$dst),
                        (ins FR64:$src1, FR64:$src2),
                       "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
-                      VEX_4V;
+                      VEX_4V, VEX_LIG;
+let mayLoad = 1 in
 def VCVTSD2SSrm  : I<0x5A, MRMSrcMem, (outs FR32:$dst),
                        (ins FR64:$src1, f64mem:$src2),
                       "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                      []>, XD, Requires<[HasAVX, OptForSize]>, VEX_4V;
+                      []>, XD, Requires<[HasAVX, OptForSize]>, VEX_4V, VEX_LIG;
+
 def : Pat<(f32 (fround FR64:$src)), (VCVTSD2SSrr FR64:$src, FR64:$src)>,
-        Requires<[HasAVX]>;
+          Requires<[HasAVX]>;
 
 def CVTSD2SSrr  : SDI<0x5A, MRMSrcReg, (outs FR32:$dst), (ins FR64:$src),
                       "cvtsd2ss\t{$src, $dst|$dst, $src}",
@@ -715,13 +1597,25 @@ defm Int_CVTSD2SS: sse12_cvt_sint_3addr<0x5A, VR128, VR128,
 def VCVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst),
                     (ins FR32:$src1, FR32:$src2),
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                    []>, XS, Requires<[HasAVX]>, VEX_4V;
+                    []>, XS, Requires<[HasAVX]>, VEX_4V, VEX_LIG;
+let mayLoad = 1 in
 def VCVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst),
                     (ins FR32:$src1, f32mem:$src2),
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                    []>, XS, VEX_4V, Requires<[HasAVX, OptForSize]>;
-def : Pat<(f64 (fextend FR32:$src)), (VCVTSS2SDrr FR32:$src, FR32:$src)>,
-        Requires<[HasAVX]>;
+                    []>, XS, VEX_4V, VEX_LIG, Requires<[HasAVX, OptForSize]>;
+
+let Predicates = [HasAVX] in {
+  def : Pat<(f64 (fextend FR32:$src)),
+            (VCVTSS2SDrr FR32:$src, FR32:$src)>;
+  def : Pat<(fextend (loadf32 addr:$src)),
+            (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>;
+  def : Pat<(extloadf32 addr:$src),
+            (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>;
+}
+
+def : Pat<(extloadf32 addr:$src),
+          (VCVTSS2SDrr (f32 (IMPLICIT_DEF)), (MOVSSrm addr:$src))>,
+          Requires<[HasAVX, OptForSpeed]>;
 
 def CVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), (ins FR32:$src),
                    "cvtss2sd\t{$src, $dst|$dst, $src}",
@@ -732,6 +1626,16 @@ def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src),
                    [(set FR64:$dst, (extloadf32 addr:$src))]>, XS,
                  Requires<[HasSSE2, OptForSize]>;
 
+// extload f32 -> f64.  This matches load+fextend because we have a hack in
+// the isel (PreprocessForFPConvert) that can introduce loads after dag
+// combine.
+// Since these loads aren't folded into the fextend, we have to match it
+// explicitly here.
+def : Pat<(fextend (loadf32 addr:$src)),
+          (CVTSS2SDrm addr:$src)>, Requires<[HasSSE2]>;
+def : Pat<(extloadf32 addr:$src),
+          (CVTSS2SDrr (MOVSSrm addr:$src))>, Requires<[HasSSE2, OptForSpeed]>;
+
 def Int_VCVTSS2SDrr: I<0x5A, MRMSrcReg,
                       (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
@@ -759,10 +1663,6 @@ def Int_CVTSS2SDrm: I<0x5A, MRMSrcMem,
                     Requires<[HasSSE2]>;
 }
 
-def : Pat<(extloadf32 addr:$src),
-          (CVTSS2SDrr (MOVSSrm addr:$src))>,
-      Requires<[HasSSE2, OptForSpeed]>;
-
 // Convert doubleword to packed single/double fp
 // SSE2 instructions without OpSize prefix
 def Int_VCVTDQ2PSrr : I<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
@@ -862,10 +1762,12 @@ def Int_CVTPD2DQrm : I<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
 // SSE2 packed instructions with XS prefix
 def VCVTTPS2DQrr : VSSI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                       "cvttps2dq\t{$src, $dst|$dst, $src}", []>, VEX;
+let mayLoad = 1 in
 def VCVTTPS2DQrm : VSSI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                       "cvttps2dq\t{$src, $dst|$dst, $src}", []>, VEX;
 def VCVTTPS2DQYrr : VSSI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                       "cvttps2dq\t{$src, $dst|$dst, $src}", []>, VEX;
+let mayLoad = 1 in
 def VCVTTPS2DQYrm : VSSI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                       "cvttps2dq\t{$src, $dst|$dst, $src}", []>, VEX;
 def CVTTPS2DQrr : SSI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
@@ -877,7 +1779,6 @@ def CVTTPS2DQrm : SSI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                       [(set VR128:$dst,
                             (int_x86_sse2_cvttps2dq (memop addr:$src)))]>;
 
-
 def Int_VCVTTPS2DQrr : I<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                         "vcvttps2dq\t{$src, $dst|$dst, $src}",
                         [(set VR128:$dst,
@@ -889,16 +1790,33 @@ def Int_VCVTTPS2DQrm : I<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                                            (memop addr:$src)))]>,
                       XS, VEX, Requires<[HasAVX]>;
 
-def Int_VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst),
-                            (ins VR128:$src),
-                          "cvttpd2dq\t{$src, $dst|$dst, $src}",
-                       [(set VR128:$dst, (int_x86_sse2_cvttpd2dq VR128:$src))]>,
-                       VEX;
-def Int_VCVTTPD2DQrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst),
-                          (ins f128mem:$src),
-                          "cvttpd2dq\t{$src, $dst|$dst, $src}",
-                          [(set VR128:$dst, (int_x86_sse2_cvttpd2dq
-                                             (memop addr:$src)))]>, VEX;
+let Predicates = [HasSSE2] in {
+  def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))),
+            (Int_CVTDQ2PSrr VR128:$src)>;
+  def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))),
+            (CVTTPS2DQrr VR128:$src)>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))),
+            (Int_VCVTDQ2PSrr VR128:$src)>;
+  def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))),
+            (VCVTTPS2DQrr VR128:$src)>;
+  def : Pat<(v8f32 (sint_to_fp (v8i32 VR256:$src))),
+            (VCVTDQ2PSYrr VR256:$src)>;
+  def : Pat<(v8i32 (fp_to_sint (v8f32 VR256:$src))),
+            (VCVTTPS2DQYrr VR256:$src)>;
+}
+
+def VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
+                        "cvttpd2dq\t{$src, $dst|$dst, $src}",
+                        [(set VR128:$dst,
+                              (int_x86_sse2_cvttpd2dq VR128:$src))]>, VEX;
+let isCodeGenOnly = 1 in
+def VCVTTPD2DQrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
+                        "cvttpd2dq\t{$src, $dst|$dst, $src}",
+                        [(set VR128:$dst, (int_x86_sse2_cvttpd2dq
+                                               (memop addr:$src)))]>, VEX;
 def CVTTPD2DQrr : PDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                       "cvttpd2dq\t{$src, $dst|$dst, $src}",
                       [(set VR128:$dst, (int_x86_sse2_cvttpd2dq VR128:$src))]>;
@@ -910,8 +1828,6 @@ def CVTTPD2DQrm : PDI<0xE6, MRMSrcMem, (outs VR128:$dst),(ins f128mem:$src),
 // The assembler can recognize rr 256-bit instructions by seeing a ymm
 // register, but the same isn't true when using memory operands instead.
 // Provide other assembly rr and rm forms to address this explicitly.
-def VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
-                        "cvttpd2dq\t{$src, $dst|$dst, $src}", []>, VEX;
 def VCVTTPD2DQXrYr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
                           "cvttpd2dq\t{$src, $dst|$dst, $src}", []>, VEX;
 
@@ -931,13 +1847,13 @@ def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
 let Predicates = [HasAVX] in {
                   // SSE2 instructions without OpSize prefix
 def VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
-                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, VEX;
+                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, TB, VEX;
 def VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
-                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, VEX;
+                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, TB, VEX;
 def VCVTPS2PDYrr : I<0x5A, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
-                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, VEX;
+                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, TB, VEX;
 def VCVTPS2PDYrm : I<0x5A, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src),
-                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, VEX;
+                     "vcvtps2pd\t{$src, $dst|$dst, $src}", []>, TB, VEX;
 }
 def CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "cvtps2pd\t{$src, $dst|$dst, $src}", []>, TB;
@@ -947,12 +1863,12 @@ def CVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
 def Int_VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "vcvtps2pd\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))]>,
-                     VEX, Requires<[HasAVX]>;
+                     TB, VEX, Requires<[HasAVX]>;
 def Int_VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
                        "vcvtps2pd\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (int_x86_sse2_cvtps2pd
                                           (load addr:$src)))]>,
-                     VEX, Requires<[HasAVX]>;
+                     TB, VEX, Requires<[HasAVX]>;
 def Int_CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "cvtps2pd\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))]>,
@@ -1038,75 +1954,61 @@ def : Pat<(int_x86_avx_cvtt_ps2dq_256 VR256:$src),
 def : Pat<(int_x86_avx_cvtt_ps2dq_256 (memopv8f32 addr:$src)),
           (VCVTTPS2DQYrm addr:$src)>;
 
+// Match fround and fextend for 128/256-bit conversions
+def : Pat<(v4f32 (fround (v4f64 VR256:$src))),
+          (VCVTPD2PSYrr VR256:$src)>;
+def : Pat<(v4f32 (fround (loadv4f64 addr:$src))),
+          (VCVTPD2PSYrm addr:$src)>;
+
+def : Pat<(v4f64 (fextend (v4f32 VR128:$src))),
+          (VCVTPS2PDYrr VR128:$src)>;
+def : Pat<(v4f64 (fextend (loadv4f32 addr:$src))),
+          (VCVTPS2PDYrm addr:$src)>;
+
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Compare Instructions
 //===----------------------------------------------------------------------===//
 
 // sse12_cmp_scalar - sse 1 & 2 compare scalar instructions
 multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop,
+                            SDNode OpNode, ValueType VT, PatFrag ld_frag,
                             string asm, string asm_alt> {
-  let isAsmParserOnly = 1 in {
-    def rr : SIi8<0xC2, MRMSrcReg,
-                  (outs RC:$dst), (ins RC:$src1, RC:$src, SSECC:$cc),
-                  asm, []>;
-    let mayLoad = 1 in
-    def rm : SIi8<0xC2, MRMSrcMem,
-                  (outs RC:$dst), (ins RC:$src1, x86memop:$src, SSECC:$cc),
-                  asm, []>;
-  }
+  def rr : SIi8<0xC2, MRMSrcReg,
+                (outs RC:$dst), (ins RC:$src1, RC:$src2, SSECC:$cc), asm,
+                [(set RC:$dst, (OpNode (VT RC:$src1), RC:$src2, imm:$cc))]>;
+  def rm : SIi8<0xC2, MRMSrcMem,
+                (outs RC:$dst), (ins RC:$src1, x86memop:$src2, SSECC:$cc), asm,
+                [(set RC:$dst, (OpNode (VT RC:$src1),
+                                         (ld_frag addr:$src2), imm:$cc))]>;
 
   // Accept explicit immediate argument form instead of comparison code.
-  def rr_alt : SIi8<0xC2, MRMSrcReg,
-                (outs RC:$dst), (ins RC:$src1, RC:$src, i8imm:$src2),
-                asm_alt, []>;
-  let mayLoad = 1 in
-  def rm_alt : SIi8<0xC2, MRMSrcMem,
-                (outs RC:$dst), (ins RC:$src1, x86memop:$src, i8imm:$src2),
-                asm_alt, []>;
+  let neverHasSideEffects = 1 in {
+    def rr_alt : SIi8<0xC2, MRMSrcReg, (outs RC:$dst),
+                      (ins RC:$src1, RC:$src2, i8imm:$cc), asm_alt, []>;
+    let mayLoad = 1 in
+    def rm_alt : SIi8<0xC2, MRMSrcMem, (outs RC:$dst),
+                      (ins RC:$src1, x86memop:$src2, i8imm:$cc), asm_alt, []>;
+  }
 }
 
-let neverHasSideEffects = 1 in {
-  defm VCMPSS  : sse12_cmp_scalar<FR32, f32mem,
-                  "cmp${cc}ss\t{$src, $src1, $dst|$dst, $src1, $src}",
-                  "cmpss\t{$src2, $src, $src1, $dst|$dst, $src1, $src, $src2}">,
-                  XS, VEX_4V;
-  defm VCMPSD  : sse12_cmp_scalar<FR64, f64mem,
-                  "cmp${cc}sd\t{$src, $src1, $dst|$dst, $src1, $src}",
-                  "cmpsd\t{$src2, $src, $src1, $dst|$dst, $src1, $src, $src2}">,
-                  XD, VEX_4V;
-}
+defm VCMPSS : sse12_cmp_scalar<FR32, f32mem, X86cmpss, f32, loadf32,
+                 "cmp${cc}ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+                 "cmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}">,
+                 XS, VEX_4V, VEX_LIG;
+defm VCMPSD : sse12_cmp_scalar<FR64, f64mem, X86cmpsd, f64, loadf64,
+                 "cmp${cc}sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+                 "cmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}">,
+                 XD, VEX_4V, VEX_LIG;
 
 let Constraints = "$src1 = $dst" in {
-def CMPSSrr : SIi8<0xC2, MRMSrcReg,
-                  (outs FR32:$dst), (ins FR32:$src1, FR32:$src2, SSECC:$cc),
+  defm CMPSS : sse12_cmp_scalar<FR32, f32mem, X86cmpss, f32, loadf32,
                   "cmp${cc}ss\t{$src2, $dst|$dst, $src2}",
-                  [(set FR32:$dst, (X86cmpss (f32 FR32:$src1), FR32:$src2, imm:$cc))]>, XS;
-def CMPSSrm : SIi8<0xC2, MRMSrcMem,
-                  (outs FR32:$dst), (ins FR32:$src1, f32mem:$src2, SSECC:$cc),
-                  "cmp${cc}ss\t{$src2, $dst|$dst, $src2}",
-                  [(set FR32:$dst, (X86cmpss (f32 FR32:$src1), (loadf32 addr:$src2), imm:$cc))]>, XS;
-def CMPSDrr : SIi8<0xC2, MRMSrcReg,
-                  (outs FR64:$dst), (ins FR64:$src1, FR64:$src2, SSECC:$cc),
+                  "cmpss\t{$cc, $src2, $dst|$dst, $src2, $cc}">,
+                  XS;
+  defm CMPSD : sse12_cmp_scalar<FR64, f64mem, X86cmpsd, f64, loadf64,
                   "cmp${cc}sd\t{$src2, $dst|$dst, $src2}",
-                  [(set FR64:$dst, (X86cmpsd (f64 FR64:$src1), FR64:$src2, imm:$cc))]>, XD;
-def CMPSDrm : SIi8<0xC2, MRMSrcMem,
-                  (outs FR64:$dst), (ins FR64:$src1, f64mem:$src2, SSECC:$cc),
-                  "cmp${cc}sd\t{$src2, $dst|$dst, $src2}",
-                  [(set FR64:$dst, (X86cmpsd (f64 FR64:$src1), (loadf64 addr:$src2), imm:$cc))]>, XD;
-}
-let Constraints = "$src1 = $dst", neverHasSideEffects = 1 in {
-def CMPSSrr_alt : SIi8<0xC2, MRMSrcReg,
-                  (outs FR32:$dst), (ins FR32:$src1, FR32:$src, i8imm:$src2),
-                  "cmpss\t{$src2, $src, $dst|$dst, $src, $src2}", []>, XS;
-def CMPSSrm_alt : SIi8<0xC2, MRMSrcMem,
-                  (outs FR32:$dst), (ins FR32:$src1, f32mem:$src, i8imm:$src2),
-                  "cmpss\t{$src2, $src, $dst|$dst, $src, $src2}", []>, XS;
-def CMPSDrr_alt : SIi8<0xC2, MRMSrcReg,
-                  (outs FR64:$dst), (ins FR64:$src1, FR64:$src, i8imm:$src2),
-                  "cmpsd\t{$src2, $src, $dst|$dst, $src, $src2}", []>, XD;
-def CMPSDrm_alt : SIi8<0xC2, MRMSrcMem,
-                  (outs FR64:$dst), (ins FR64:$src1, f64mem:$src, i8imm:$src2),
-                  "cmpsd\t{$src2, $src, $dst|$dst, $src, $src2}", []>, XD;
+                  "cmpsd\t{$cc, $src2, $dst|$dst, $src2, $cc}">,
+                  XD;
 }
 
 multiclass sse12_cmp_scalar_int<RegisterClass RC, X86MemOperand x86memop,
@@ -1151,25 +2053,28 @@ multiclass sse12_ord_cmp<bits<8> opc, RegisterClass RC, SDNode OpNode,
 
 let Defs = [EFLAGS] in {
   defm VUCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32,
-                                  "ucomiss", SSEPackedSingle>, VEX;
+                                  "ucomiss", SSEPackedSingle>, TB, VEX, VEX_LIG;
   defm VUCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64,
-                                  "ucomisd", SSEPackedDouble>, OpSize, VEX;
+                                  "ucomisd", SSEPackedDouble>, TB, OpSize, VEX,
+                                  VEX_LIG;
   let Pattern = []<dag> in {
     defm VCOMISS  : sse12_ord_cmp<0x2F, VR128, undef, v4f32, f128mem, load,
-                                    "comiss", SSEPackedSingle>, VEX;
+                                    "comiss", SSEPackedSingle>, TB, VEX,
+                                    VEX_LIG;
     defm VCOMISD  : sse12_ord_cmp<0x2F, VR128, undef, v2f64, f128mem, load,
-                                    "comisd", SSEPackedDouble>, OpSize, VEX;
+                                    "comisd", SSEPackedDouble>, TB, OpSize, VEX,
+                                    VEX_LIG;
   }
 
   defm Int_VUCOMISS  : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem,
-                            load, "ucomiss", SSEPackedSingle>, VEX;
+                            load, "ucomiss", SSEPackedSingle>, TB, VEX;
   defm Int_VUCOMISD  : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem,
-                            load, "ucomisd", SSEPackedDouble>, OpSize, VEX;
+                            load, "ucomisd", SSEPackedDouble>, TB, OpSize, VEX;
 
   defm Int_VCOMISS  : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem,
-                            load, "comiss", SSEPackedSingle>, VEX;
+                            load, "comiss", SSEPackedSingle>, TB, VEX;
   defm Int_VCOMISD  : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem,
-                            load, "comisd", SSEPackedDouble>, OpSize, VEX;
+                            load, "comisd", SSEPackedDouble>, TB, OpSize, VEX;
   defm UCOMISS  : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32,
                                   "ucomiss", SSEPackedSingle>, TB;
   defm UCOMISD  : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64,
@@ -1199,57 +2104,82 @@ multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop,
                             Domain d> {
   let isAsmParserOnly = 1 in {
     def rri : PIi8<0xC2, MRMSrcReg,
-               (outs RC:$dst), (ins RC:$src1, RC:$src, SSECC:$cc), asm,
-               [(set RC:$dst, (Int RC:$src1, RC:$src, imm:$cc))], d>;
+               (outs RC:$dst), (ins RC:$src1, RC:$src2, SSECC:$cc), asm,
+               [(set RC:$dst, (Int RC:$src1, RC:$src2, imm:$cc))], d>;
     def rmi : PIi8<0xC2, MRMSrcMem,
-               (outs RC:$dst), (ins RC:$src1, f128mem:$src, SSECC:$cc), asm,
-               [(set RC:$dst, (Int RC:$src1, (memop addr:$src), imm:$cc))], d>;
+               (outs RC:$dst), (ins RC:$src1, f128mem:$src2, SSECC:$cc), asm,
+               [(set RC:$dst, (Int RC:$src1, (memop addr:$src2), imm:$cc))], d>;
   }
 
   // Accept explicit immediate argument form instead of comparison code.
   def rri_alt : PIi8<0xC2, MRMSrcReg,
-             (outs RC:$dst), (ins RC:$src1, RC:$src, i8imm:$src2),
+             (outs RC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc),
              asm_alt, [], d>;
   def rmi_alt : PIi8<0xC2, MRMSrcMem,
-             (outs RC:$dst), (ins RC:$src1, f128mem:$src, i8imm:$src2),
+             (outs RC:$dst), (ins RC:$src1, f128mem:$src2, i8imm:$cc),
              asm_alt, [], d>;
 }
 
 defm VCMPPS : sse12_cmp_packed<VR128, f128mem, int_x86_sse_cmp_ps,
-               "cmp${cc}ps\t{$src, $src1, $dst|$dst, $src1, $src}",
-               "cmpps\t{$src2, $src, $src1, $dst|$dst, $src1, $src, $src2}",
-               SSEPackedSingle>, VEX_4V;
+               "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+               "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",
+               SSEPackedSingle>, TB, VEX_4V;
 defm VCMPPD : sse12_cmp_packed<VR128, f128mem, int_x86_sse2_cmp_pd,
-               "cmp${cc}pd\t{$src, $src1, $dst|$dst, $src1, $src}",
-               "cmppd\t{$src2, $src, $src1, $dst|$dst, $src1, $src, $src2}",
-               SSEPackedDouble>, OpSize, VEX_4V;
+               "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+               "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",
+               SSEPackedDouble>, TB, OpSize, VEX_4V;
 defm VCMPPSY : sse12_cmp_packed<VR256, f256mem, int_x86_avx_cmp_ps_256,
-               "cmp${cc}ps\t{$src, $src1, $dst|$dst, $src1, $src}",
-               "cmpps\t{$src2, $src, $src1, $dst|$dst, $src1, $src, $src2}",
-               SSEPackedSingle>, VEX_4V;
+               "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+               "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",
+               SSEPackedSingle>, TB, VEX_4V;
 defm VCMPPDY : sse12_cmp_packed<VR256, f256mem, int_x86_avx_cmp_pd_256,
-               "cmp${cc}pd\t{$src, $src1, $dst|$dst, $src1, $src}",
-               "cmppd\t{$src2, $src, $src1, $dst|$dst, $src1, $src, $src2}",
-               SSEPackedDouble>, OpSize, VEX_4V;
+               "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+               "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",
+               SSEPackedDouble>, TB, OpSize, VEX_4V;
 let Constraints = "$src1 = $dst" in {
   defm CMPPS : sse12_cmp_packed<VR128, f128mem, int_x86_sse_cmp_ps,
-                 "cmp${cc}ps\t{$src, $dst|$dst, $src}",
-                 "cmpps\t{$src2, $src, $dst|$dst, $src, $src2}",
+                 "cmp${cc}ps\t{$src2, $dst|$dst, $src2}",
+                 "cmpps\t{$cc, $src2, $dst|$dst, $src2, $cc}",
                  SSEPackedSingle>, TB;
   defm CMPPD : sse12_cmp_packed<VR128, f128mem, int_x86_sse2_cmp_pd,
-                 "cmp${cc}pd\t{$src, $dst|$dst, $src}",
-                 "cmppd\t{$src2, $src, $dst|$dst, $src, $src2}",
+                 "cmp${cc}pd\t{$src2, $dst|$dst, $src2}",
+                 "cmppd\t{$cc, $src2, $dst|$dst, $src2, $cc}",
                  SSEPackedDouble>, TB, OpSize;
 }
 
+let Predicates = [HasSSE1] in {
 def : Pat<(v4i32 (X86cmpps (v4f32 VR128:$src1), VR128:$src2, imm:$cc)),
           (CMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>;
 def : Pat<(v4i32 (X86cmpps (v4f32 VR128:$src1), (memop addr:$src2), imm:$cc)),
           (CMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>;
+}
+
+let Predicates = [HasSSE2] in {
 def : Pat<(v2i64 (X86cmppd (v2f64 VR128:$src1), VR128:$src2, imm:$cc)),
           (CMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>;
 def : Pat<(v2i64 (X86cmppd (v2f64 VR128:$src1), (memop addr:$src2), imm:$cc)),
           (CMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>;
+}
+
+let Predicates = [HasAVX] in {
+def : Pat<(v4i32 (X86cmpps (v4f32 VR128:$src1), VR128:$src2, imm:$cc)),
+          (VCMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>;
+def : Pat<(v4i32 (X86cmpps (v4f32 VR128:$src1), (memop addr:$src2), imm:$cc)),
+          (VCMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>;
+def : Pat<(v2i64 (X86cmppd (v2f64 VR128:$src1), VR128:$src2, imm:$cc)),
+          (VCMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>;
+def : Pat<(v2i64 (X86cmppd (v2f64 VR128:$src1), (memop addr:$src2), imm:$cc)),
+          (VCMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>;
+
+def : Pat<(v8i32 (X86cmpps (v8f32 VR256:$src1), VR256:$src2, imm:$cc)),
+          (VCMPPSYrri (v8f32 VR256:$src1), (v8f32 VR256:$src2), imm:$cc)>;
+def : Pat<(v8i32 (X86cmpps (v8f32 VR256:$src1), (memop addr:$src2), imm:$cc)),
+          (VCMPPSYrmi (v8f32 VR256:$src1), addr:$src2, imm:$cc)>;
+def : Pat<(v4i64 (X86cmppd (v4f64 VR256:$src1), VR256:$src2, imm:$cc)),
+          (VCMPPDYrri VR256:$src1, VR256:$src2, imm:$cc)>;
+def : Pat<(v4i64 (X86cmppd (v4f64 VR256:$src1), (memop addr:$src2), imm:$cc)),
+          (VCMPPDYrmi VR256:$src1, addr:$src2, imm:$cc)>;
+}
 
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Shuffle Instructions
@@ -1293,6 +2223,132 @@ let Constraints = "$src1 = $dst" in {
                     memopv2f64, SSEPackedDouble>, TB, OpSize;
 }
 
+let Predicates = [HasSSE1] in {
+  def : Pat<(v4f32 (X86Shufps VR128:$src1,
+                       (memopv4f32 addr:$src2), (i8 imm:$imm))),
+            (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
+  def : Pat<(v4f32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
+  def : Pat<(v4i32 (X86Shufps VR128:$src1,
+                       (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))),
+            (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
+  def : Pat<(v4i32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
+  // vector_shuffle v1, v2 <4, 5, 2, 3> using SHUFPSrri (we prefer movsd, but
+  // fall back to this for SSE1)
+  def : Pat<(v4f32 (movlp:$src3 VR128:$src1, (v4f32 VR128:$src2))),
+            (SHUFPSrri VR128:$src2, VR128:$src1,
+                       (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special unary SHUFPSrri case.
+  def : Pat<(v4f32 (pshufd:$src3 VR128:$src1, (undef))),
+            (SHUFPSrri VR128:$src1, VR128:$src1,
+                       (SHUFFLE_get_shuf_imm VR128:$src3))>;
+}
+
+let Predicates = [HasSSE2] in {
+  // Special binary v4i32 shuffle cases with SHUFPS.
+  def : Pat<(v4i32 (shufp:$src3 VR128:$src1, (v4i32 VR128:$src2))),
+            (SHUFPSrri VR128:$src1, VR128:$src2,
+                       (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  def : Pat<(v4i32 (shufp:$src3 VR128:$src1,
+                                (bc_v4i32 (memopv2i64 addr:$src2)))),
+            (SHUFPSrmi VR128:$src1, addr:$src2,
+                      (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special unary SHUFPDrri cases.
+  def : Pat<(v2i64 (pshufd:$src3 VR128:$src1, (undef))),
+            (SHUFPDrri VR128:$src1, VR128:$src1,
+                       (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  def : Pat<(v2f64 (pshufd:$src3 VR128:$src1, (undef))),
+            (SHUFPDrri VR128:$src1, VR128:$src1,
+                       (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special binary v2i64 shuffle cases using SHUFPDrri.
+  def : Pat<(v2i64 (shufp:$src3 VR128:$src1, VR128:$src2)),
+            (SHUFPDrri VR128:$src1, VR128:$src2,
+                       (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Generic SHUFPD patterns
+  def : Pat<(v2f64 (X86Shufps VR128:$src1,
+                       (memopv2f64 addr:$src2), (i8 imm:$imm))),
+            (SHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>;
+  def : Pat<(v2i64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (SHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>;
+  def : Pat<(v2f64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (SHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(v4f32 (X86Shufps VR128:$src1,
+                       (memopv4f32 addr:$src2), (i8 imm:$imm))),
+            (VSHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
+  def : Pat<(v4f32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (VSHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
+  def : Pat<(v4i32 (X86Shufps VR128:$src1,
+                       (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))),
+            (VSHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
+  def : Pat<(v4i32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (VSHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
+  // vector_shuffle v1, v2 <4, 5, 2, 3> using SHUFPSrri (we prefer movsd, but
+  // fall back to this for SSE1)
+  def : Pat<(v4f32 (movlp:$src3 VR128:$src1, (v4f32 VR128:$src2))),
+            (VSHUFPSrri VR128:$src2, VR128:$src1,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special unary SHUFPSrri case.
+  def : Pat<(v4f32 (pshufd:$src3 VR128:$src1, (undef))),
+            (VSHUFPSrri VR128:$src1, VR128:$src1,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special binary v4i32 shuffle cases with SHUFPS.
+  def : Pat<(v4i32 (shufp:$src3 VR128:$src1, (v4i32 VR128:$src2))),
+            (VSHUFPSrri VR128:$src1, VR128:$src2,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  def : Pat<(v4i32 (shufp:$src3 VR128:$src1,
+                                (bc_v4i32 (memopv2i64 addr:$src2)))),
+            (VSHUFPSrmi VR128:$src1, addr:$src2,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special unary SHUFPDrri cases.
+  def : Pat<(v2i64 (pshufd:$src3 VR128:$src1, (undef))),
+            (VSHUFPDrri VR128:$src1, VR128:$src1,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  def : Pat<(v2f64 (pshufd:$src3 VR128:$src1, (undef))),
+            (VSHUFPDrri VR128:$src1, VR128:$src1,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+  // Special binary v2i64 shuffle cases using SHUFPDrri.
+  def : Pat<(v2i64 (shufp:$src3 VR128:$src1, VR128:$src2)),
+            (VSHUFPDrri VR128:$src1, VR128:$src2,
+                        (SHUFFLE_get_shuf_imm VR128:$src3))>;
+
+  def : Pat<(v2f64 (X86Shufps VR128:$src1,
+                       (memopv2f64 addr:$src2), (i8 imm:$imm))),
+            (VSHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>;
+  def : Pat<(v2i64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (VSHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>;
+  def : Pat<(v2f64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+            (VSHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>;
+
+  // 256-bit patterns
+  def : Pat<(v8i32 (X86Shufps VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+            (VSHUFPSYrri VR256:$src1, VR256:$src2, imm:$imm)>;
+  def : Pat<(v8i32 (X86Shufps VR256:$src1,
+                      (bc_v8i32 (memopv4i64 addr:$src2)), (i8 imm:$imm))),
+            (VSHUFPSYrmi VR256:$src1, addr:$src2, imm:$imm)>;
+
+  def : Pat<(v8f32 (X86Shufps VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+            (VSHUFPSYrri VR256:$src1, VR256:$src2, imm:$imm)>;
+  def : Pat<(v8f32 (X86Shufps VR256:$src1,
+                              (memopv8f32 addr:$src2), (i8 imm:$imm))),
+            (VSHUFPSYrmi VR256:$src1, addr:$src2, imm:$imm)>;
+
+  def : Pat<(v4i64 (X86Shufpd VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+            (VSHUFPDYrri VR256:$src1, VR256:$src2, imm:$imm)>;
+  def : Pat<(v4i64 (X86Shufpd VR256:$src1,
+                              (memopv4i64 addr:$src2), (i8 imm:$imm))),
+            (VSHUFPDYrmi VR256:$src1, addr:$src2, imm:$imm)>;
+
+  def : Pat<(v4f64 (X86Shufpd VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+            (VSHUFPDYrri VR256:$src1, VR256:$src2, imm:$imm)>;
+  def : Pat<(v4f64 (X86Shufpd VR256:$src1,
+                              (memopv4f64 addr:$src2), (i8 imm:$imm))),
+            (VSHUFPDYrmi VR256:$src1, addr:$src2, imm:$imm)>;
+}
+
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Unpack Instructions
 //===----------------------------------------------------------------------===//
@@ -1316,29 +2372,29 @@ multiclass sse12_unpack_interleave<bits<8> opc, PatFrag OpNode, ValueType vt,
 let AddedComplexity = 10 in {
   defm VUNPCKHPS: sse12_unpack_interleave<0x15, unpckh, v4f32, memopv4f32,
         VR128, f128mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedSingle>, VEX_4V;
+                       SSEPackedSingle>, TB, VEX_4V;
   defm VUNPCKHPD: sse12_unpack_interleave<0x15, unpckh, v2f64, memopv2f64,
         VR128, f128mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedDouble>, OpSize, VEX_4V;
+                       SSEPackedDouble>, TB, OpSize, VEX_4V;
   defm VUNPCKLPS: sse12_unpack_interleave<0x14, unpckl, v4f32, memopv4f32,
         VR128, f128mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedSingle>, VEX_4V;
+                       SSEPackedSingle>, TB, VEX_4V;
   defm VUNPCKLPD: sse12_unpack_interleave<0x14, unpckl, v2f64, memopv2f64,
         VR128, f128mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedDouble>, OpSize, VEX_4V;
+                       SSEPackedDouble>, TB, OpSize, VEX_4V;
 
   defm VUNPCKHPSY: sse12_unpack_interleave<0x15, unpckh, v8f32, memopv8f32,
         VR256, f256mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedSingle>, VEX_4V;
+                       SSEPackedSingle>, TB, VEX_4V;
   defm VUNPCKHPDY: sse12_unpack_interleave<0x15, unpckh, v4f64, memopv4f64,
         VR256, f256mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedDouble>, OpSize, VEX_4V;
+                       SSEPackedDouble>, TB, OpSize, VEX_4V;
   defm VUNPCKLPSY: sse12_unpack_interleave<0x14, unpckl, v8f32, memopv8f32,
         VR256, f256mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedSingle>, VEX_4V;
+                       SSEPackedSingle>, TB, VEX_4V;
   defm VUNPCKLPDY: sse12_unpack_interleave<0x14, unpckl, v4f64, memopv4f64,
         VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                       SSEPackedDouble>, OpSize, VEX_4V;
+                       SSEPackedDouble>, TB, OpSize, VEX_4V;
 
   let Constraints = "$src1 = $dst" in {
     defm UNPCKHPS: sse12_unpack_interleave<0x15, unpckh, v4f32, memopv4f32,
@@ -1356,6 +2412,103 @@ let AddedComplexity = 10 in {
   } // Constraints = "$src1 = $dst"
 } // AddedComplexity
 
+let Predicates = [HasSSE1] in {
+  def : Pat<(v4f32 (X86Unpcklps VR128:$src1, (memopv4f32 addr:$src2))),
+            (UNPCKLPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4f32 (X86Unpcklps VR128:$src1, VR128:$src2)),
+            (UNPCKLPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4f32 (X86Unpckhps VR128:$src1, (memopv4f32 addr:$src2))),
+            (UNPCKHPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4f32 (X86Unpckhps VR128:$src1, VR128:$src2)),
+            (UNPCKHPSrr VR128:$src1, VR128:$src2)>;
+}
+
+let Predicates = [HasSSE2] in {
+  def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, (memopv2f64 addr:$src2))),
+            (UNPCKLPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, VR128:$src2)),
+            (UNPCKLPDrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, (memopv2f64 addr:$src2))),
+            (UNPCKHPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, VR128:$src2)),
+            (UNPCKHPDrr VR128:$src1, VR128:$src2)>;
+
+  // FIXME: Instead of X86Movddup, there should be a X86Unpcklpd here, the
+  // problem is during lowering, where it's not possible to recognize the load
+  // fold cause it has two uses through a bitcast. One use disappears at isel
+  // time and the fold opportunity reappears.
+  def : Pat<(v2f64 (X86Movddup VR128:$src)),
+            (UNPCKLPDrr VR128:$src, VR128:$src)>;
+
+  let AddedComplexity = 10 in
+  def : Pat<(splat_lo (v2f64 VR128:$src), (undef)),
+            (UNPCKLPDrr VR128:$src, VR128:$src)>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(v4f32 (X86Unpcklps VR128:$src1, (memopv4f32 addr:$src2))),
+            (VUNPCKLPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4f32 (X86Unpcklps VR128:$src1, VR128:$src2)),
+            (VUNPCKLPSrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4f32 (X86Unpckhps VR128:$src1, (memopv4f32 addr:$src2))),
+            (VUNPCKHPSrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4f32 (X86Unpckhps VR128:$src1, VR128:$src2)),
+            (VUNPCKHPSrr VR128:$src1, VR128:$src2)>;
+
+  def : Pat<(v8f32 (X86Unpcklpsy VR256:$src1, (memopv8f32 addr:$src2))),
+            (VUNPCKLPSYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v8f32 (X86Unpcklpsy VR256:$src1, VR256:$src2)),
+            (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v8i32 (X86Unpcklpsy VR256:$src1, VR256:$src2)),
+            (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v8i32 (X86Unpcklpsy VR256:$src1, (memopv8i32 addr:$src2))),
+            (VUNPCKLPSYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v8f32 (X86Unpckhpsy VR256:$src1, (memopv8f32 addr:$src2))),
+            (VUNPCKHPSYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v8f32 (X86Unpckhpsy VR256:$src1, VR256:$src2)),
+            (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v8i32 (X86Unpckhpsy VR256:$src1, (memopv8i32 addr:$src2))),
+            (VUNPCKHPSYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v8i32 (X86Unpckhpsy VR256:$src1, VR256:$src2)),
+            (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>;
+
+  def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, (memopv2f64 addr:$src2))),
+            (VUNPCKLPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, VR128:$src2)),
+            (VUNPCKLPDrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, (memopv2f64 addr:$src2))),
+            (VUNPCKHPDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, VR128:$src2)),
+            (VUNPCKHPDrr VR128:$src1, VR128:$src2)>;
+
+  def : Pat<(v4f64 (X86Unpcklpdy VR256:$src1, (memopv4f64 addr:$src2))),
+            (VUNPCKLPDYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v4f64 (X86Unpcklpdy VR256:$src1, VR256:$src2)),
+            (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v4i64 (X86Unpcklpdy VR256:$src1, (memopv4i64 addr:$src2))),
+            (VUNPCKLPDYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v4i64 (X86Unpcklpdy VR256:$src1, VR256:$src2)),
+            (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v4f64 (X86Unpckhpdy VR256:$src1, (memopv4f64 addr:$src2))),
+            (VUNPCKHPDYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v4f64 (X86Unpckhpdy VR256:$src1, VR256:$src2)),
+            (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v4i64 (X86Unpckhpdy VR256:$src1, (memopv4i64 addr:$src2))),
+            (VUNPCKHPDYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v4i64 (X86Unpckhpdy VR256:$src1, VR256:$src2)),
+            (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>;
+
+  // FIXME: Instead of X86Movddup, there should be a X86Unpcklpd here, the
+  // problem is during lowering, where it's not possible to recognize the load
+  // fold cause it has two uses through a bitcast. One use disappears at isel
+  // time and the fold opportunity reappears.
+  def : Pat<(v2f64 (X86Movddup VR128:$src)),
+            (VUNPCKLPDrr VR128:$src, VR128:$src)>;
+  let AddedComplexity = 10 in
+  def : Pat<(splat_lo (v2f64 VR128:$src), (undef)),
+            (VUNPCKLPDrr VR128:$src, VR128:$src)>;
+}
+
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Extract Floating-Point Sign mask
 //===----------------------------------------------------------------------===//
@@ -1370,91 +2523,60 @@ multiclass sse12_extr_sign_mask<RegisterClass RC, Intrinsic Int, string asm,
                 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [], d>, REX_W;
 }
 
-// Mask creation
-defm VMOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps,
-                                      "movmskps", SSEPackedSingle>, VEX;
-defm VMOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd,
-                                      "movmskpd", SSEPackedDouble>, OpSize,
-                                      VEX;
-defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_ps_256,
-                                      "movmskps", SSEPackedSingle>, VEX;
-defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_pd_256,
-                                      "movmskpd", SSEPackedDouble>, OpSize,
-                                      VEX;
 defm MOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps, "movmskps",
                                      SSEPackedSingle>, TB;
 defm MOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd, "movmskpd",
                                      SSEPackedDouble>, TB, OpSize;
 
-// X86fgetsign
-def MOVMSKPDrr32_alt : PI<0x50, MRMSrcReg, (outs GR32:$dst), (ins FR64:$src),
-                    "movmskpd\t{$src, $dst|$dst, $src}",
-                    [(set GR32:$dst, (X86fgetsign FR64:$src))], SSEPackedDouble>, TB, OpSize;
-def MOVMSKPDrr64_alt : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins FR64:$src),
-                    "movmskpd\t{$src, $dst|$dst, $src}",
-                    [(set GR64:$dst, (X86fgetsign FR64:$src))], SSEPackedDouble>, TB, OpSize;
-def MOVMSKPSrr32_alt : PI<0x50, MRMSrcReg, (outs GR32:$dst), (ins FR32:$src),
-                    "movmskps\t{$src, $dst|$dst, $src}",
-                    [(set GR32:$dst, (X86fgetsign FR32:$src))], SSEPackedSingle>, TB;
-def MOVMSKPSrr64_alt : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins FR32:$src),
-                    "movmskps\t{$src, $dst|$dst, $src}",
-                    [(set GR64:$dst, (X86fgetsign FR32:$src))], SSEPackedSingle>, TB;
+def : Pat<(i32 (X86fgetsign FR32:$src)),
+          (MOVMSKPSrr32 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FR32:$src,
+                                       sub_ss))>, Requires<[HasSSE1]>;
+def : Pat<(i64 (X86fgetsign FR32:$src)),
+          (MOVMSKPSrr64 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FR32:$src,
+                                       sub_ss))>, Requires<[HasSSE1]>;
+def : Pat<(i32 (X86fgetsign FR64:$src)),
+          (MOVMSKPDrr32 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FR64:$src,
+                                       sub_sd))>, Requires<[HasSSE2]>;
+def : Pat<(i64 (X86fgetsign FR64:$src)),
+          (MOVMSKPDrr64 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FR64:$src,
+                                       sub_sd))>, Requires<[HasSSE2]>;
 
-// Assembler Only
-def VMOVMSKPSr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR128:$src),
-           "movmskps\t{$src, $dst|$dst, $src}", [], SSEPackedSingle>, VEX;
-def VMOVMSKPDr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR128:$src),
-           "movmskpd\t{$src, $dst|$dst, $src}", [], SSEPackedDouble>, OpSize,
-           VEX;
-def VMOVMSKPSYr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
-           "movmskps\t{$src, $dst|$dst, $src}", [], SSEPackedSingle>, VEX;
-def VMOVMSKPDYr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
-           "movmskpd\t{$src, $dst|$dst, $src}", [], SSEPackedDouble>, OpSize,
-           VEX;
+let Predicates = [HasAVX] in {
+  defm VMOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps,
+                                        "movmskps", SSEPackedSingle>, TB, VEX;
+  defm VMOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd,
+                                        "movmskpd", SSEPackedDouble>, TB,
+                                        OpSize, VEX;
+  defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_ps_256,
+                                        "movmskps", SSEPackedSingle>, TB, VEX;
+  defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_pd_256,
+                                        "movmskpd", SSEPackedDouble>, TB,
+                                        OpSize, VEX;
 
-//===----------------------------------------------------------------------===//
-// SSE 1 & 2 - Misc aliasing of packed SSE 1 & 2 instructions
-//===----------------------------------------------------------------------===//
+  def : Pat<(i32 (X86fgetsign FR32:$src)),
+            (VMOVMSKPSrr32 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FR32:$src,
+                                          sub_ss))>;
+  def : Pat<(i64 (X86fgetsign FR32:$src)),
+            (VMOVMSKPSrr64 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FR32:$src,
+                                          sub_ss))>;
+  def : Pat<(i32 (X86fgetsign FR64:$src)),
+            (VMOVMSKPDrr32 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FR64:$src,
+                                          sub_sd))>;
+  def : Pat<(i64 (X86fgetsign FR64:$src)),
+            (VMOVMSKPDrr64 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FR64:$src,
+                                          sub_sd))>;
 
-// Aliases of packed SSE1 & SSE2 instructions for scalar use. These all have
-// names that start with 'Fs'.
-
-// Alias instructions that map fld0 to pxor for sse.
-let isReMaterializable = 1, isAsCheapAsAMove = 1, isCodeGenOnly = 1,
-    canFoldAsLoad = 1 in {
-  // FIXME: Set encoding to pseudo!
-def FsFLD0SS : I<0xEF, MRMInitReg, (outs FR32:$dst), (ins), "",
-                 [(set FR32:$dst, fp32imm0)]>,
-                 Requires<[HasSSE1]>, TB, OpSize;
-def FsFLD0SD : I<0xEF, MRMInitReg, (outs FR64:$dst), (ins), "",
-                 [(set FR64:$dst, fpimm0)]>,
-               Requires<[HasSSE2]>, TB, OpSize;
-def VFsFLD0SS : I<0xEF, MRMInitReg, (outs FR32:$dst), (ins), "",
-                  [(set FR32:$dst, fp32imm0)]>,
-                  Requires<[HasAVX]>, TB, OpSize, VEX_4V;
-def VFsFLD0SD : I<0xEF, MRMInitReg, (outs FR64:$dst), (ins), "",
-                  [(set FR64:$dst, fpimm0)]>,
-                  Requires<[HasAVX]>, TB, OpSize, VEX_4V;
-}
-
-// Alias instruction to do FR32 or FR64 reg-to-reg copy using movaps. Upper
-// bits are disregarded.
-let neverHasSideEffects = 1 in {
-def FsMOVAPSrr : PSI<0x28, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src),
-                     "movaps\t{$src, $dst|$dst, $src}", []>;
-def FsMOVAPDrr : PDI<0x28, MRMSrcReg, (outs FR64:$dst), (ins FR64:$src),
-                     "movapd\t{$src, $dst|$dst, $src}", []>;
-}
-
-// Alias instruction to load FR32 or FR64 from f128mem using movaps. Upper
-// bits are disregarded.
-let canFoldAsLoad = 1, isReMaterializable = 1 in {
-def FsMOVAPSrm : PSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src),
-                     "movaps\t{$src, $dst|$dst, $src}",
-                     [(set FR32:$dst, (alignedloadfsf32 addr:$src))]>;
-def FsMOVAPDrm : PDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src),
-                     "movapd\t{$src, $dst|$dst, $src}",
-                     [(set FR64:$dst, (alignedloadfsf64 addr:$src))]>;
+  // Assembler Only
+  def VMOVMSKPSr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR128:$src),
+             "movmskps\t{$src, $dst|$dst, $src}", [], SSEPackedSingle>, TB, VEX;
+  def VMOVMSKPDr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR128:$src),
+             "movmskpd\t{$src, $dst|$dst, $src}", [], SSEPackedDouble>, TB,
+             OpSize, VEX;
+  def VMOVMSKPSYr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
+             "movmskps\t{$src, $dst|$dst, $src}", [], SSEPackedSingle>, TB, VEX;
+  def VMOVMSKPDYr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
+             "movmskpd\t{$src, $dst|$dst, $src}", [], SSEPackedDouble>, TB,
+             OpSize, VEX;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1466,10 +2588,10 @@ def FsMOVAPDrm : PDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src),
 multiclass sse12_fp_alias_pack_logical<bits<8> opc, string OpcodeStr,
                                        SDNode OpNode> {
   defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,
-              FR32, f32, f128mem, memopfsf32, SSEPackedSingle, 0>, VEX_4V;
+              FR32, f32, f128mem, memopfsf32, SSEPackedSingle, 0>, TB, VEX_4V;
 
   defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
-        FR64, f64, f128mem, memopfsf64, SSEPackedDouble, 0>, OpSize, VEX_4V;
+        FR64, f64, f128mem, memopfsf64, SSEPackedDouble, 0>, TB, OpSize, VEX_4V;
 
   let Constraints = "$src1 = $dst" in {
     defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, FR32,
@@ -1494,21 +2616,22 @@ let neverHasSideEffects = 1, Pattern = []<dag>, isCommutable = 0 in
 ///
 multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr,
                                    SDNode OpNode> {
-  let Pattern = []<dag> in {
-    defm V#NAME#PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle,
-         !strconcat(OpcodeStr, "ps"), f128mem,
-         [(set VR128:$dst, (v2i64 (OpNode VR128:$src1, VR128:$src2)))],
-         [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)),
-                                   (memopv2i64 addr:$src2)))], 0>, VEX_4V;
+  // In AVX no need to add a pattern for 128-bit logical rr ps, because they
+  // are all promoted to v2i64, and the patterns are covered by the int
+  // version. This is needed in SSE only, because v2i64 isn't supported on
+  // SSE1, but only on SSE2.
+  defm V#NAME#PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle,
+       !strconcat(OpcodeStr, "ps"), f128mem, [],
+       [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)),
+                                 (memopv2i64 addr:$src2)))], 0>, TB, VEX_4V;
 
-    defm V#NAME#PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble,
-         !strconcat(OpcodeStr, "pd"), f128mem,
-         [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)),
-                                   (bc_v2i64 (v2f64 VR128:$src2))))],
-         [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)),
-                                   (memopv2i64 addr:$src2)))], 0>,
-                                                   OpSize, VEX_4V;
-  }
+  defm V#NAME#PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble,
+       !strconcat(OpcodeStr, "pd"), f128mem,
+       [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)),
+                                 (bc_v2i64 (v2f64 VR128:$src2))))],
+       [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)),
+                                 (memopv2i64 addr:$src2)))], 0>,
+                                                 TB, OpSize, VEX_4V;
   let Constraints = "$src1 = $dst" in {
     defm PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle,
          !strconcat(OpcodeStr, "ps"), f128mem,
@@ -1533,7 +2656,7 @@ multiclass sse12_fp_packed_logical_y<bits<8> opc, string OpcodeStr,
           !strconcat(OpcodeStr, "ps"), f256mem,
           [(set VR256:$dst, (v4i64 (OpNode VR256:$src1, VR256:$src2)))],
           [(set VR256:$dst, (OpNode (bc_v4i64 (v8f32 VR256:$src1)),
-                                    (memopv4i64 addr:$src2)))], 0>, VEX_4V;
+                                    (memopv4i64 addr:$src2)))], 0>, TB, VEX_4V;
 
     defm PDY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedDouble,
           !strconcat(OpcodeStr, "pd"), f256mem,
@@ -1541,7 +2664,7 @@ multiclass sse12_fp_packed_logical_y<bits<8> opc, string OpcodeStr,
                                     (bc_v4i64 (v4f64 VR256:$src2))))],
           [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)),
                                     (memopv4i64 addr:$src2)))], 0>,
-                                    OpSize, VEX_4V;
+                                    TB, OpSize, VEX_4V;
 }
 
 // AVX 256-bit packed logical ops forms
@@ -1632,32 +2755,32 @@ multiclass basic_sse12_fp_binop_p_y_int<bits<8> opc, string OpcodeStr> {
 
 // Binary Arithmetic instructions
 defm VADD : basic_sse12_fp_binop_s<0x58, "add", fadd, 0>,
-            basic_sse12_fp_binop_s_int<0x58, "add", 0>,
-            basic_sse12_fp_binop_p<0x58, "add", fadd, 0>,
+            basic_sse12_fp_binop_s_int<0x58, "add", 0>, VEX_4V, VEX_LIG;
+defm VADD : basic_sse12_fp_binop_p<0x58, "add", fadd, 0>,
             basic_sse12_fp_binop_p_y<0x58, "add", fadd>, VEX_4V;
 defm VMUL : basic_sse12_fp_binop_s<0x59, "mul", fmul, 0>,
-            basic_sse12_fp_binop_s_int<0x59, "mul", 0>,
-            basic_sse12_fp_binop_p<0x59, "mul", fmul, 0>,
+            basic_sse12_fp_binop_s_int<0x59, "mul", 0>, VEX_4V, VEX_LIG;
+defm VMUL : basic_sse12_fp_binop_p<0x59, "mul", fmul, 0>,
             basic_sse12_fp_binop_p_y<0x59, "mul", fmul>, VEX_4V;
 
 let isCommutable = 0 in {
   defm VSUB : basic_sse12_fp_binop_s<0x5C, "sub", fsub, 0>,
-              basic_sse12_fp_binop_s_int<0x5C, "sub", 0>,
-              basic_sse12_fp_binop_p<0x5C, "sub", fsub, 0>,
+              basic_sse12_fp_binop_s_int<0x5C, "sub", 0>, VEX_4V, VEX_LIG;
+  defm VSUB : basic_sse12_fp_binop_p<0x5C, "sub", fsub, 0>,
               basic_sse12_fp_binop_p_y<0x5C, "sub", fsub>, VEX_4V;
   defm VDIV : basic_sse12_fp_binop_s<0x5E, "div", fdiv, 0>,
-              basic_sse12_fp_binop_s_int<0x5E, "div", 0>,
-              basic_sse12_fp_binop_p<0x5E, "div", fdiv, 0>,
+              basic_sse12_fp_binop_s_int<0x5E, "div", 0>, VEX_4V, VEX_LIG;
+  defm VDIV : basic_sse12_fp_binop_p<0x5E, "div", fdiv, 0>,
               basic_sse12_fp_binop_p_y<0x5E, "div", fdiv>, VEX_4V;
   defm VMAX : basic_sse12_fp_binop_s<0x5F, "max", X86fmax, 0>,
-              basic_sse12_fp_binop_s_int<0x5F, "max", 0>,
-              basic_sse12_fp_binop_p<0x5F, "max", X86fmax, 0>,
+              basic_sse12_fp_binop_s_int<0x5F, "max", 0>, VEX_4V, VEX_LIG;
+  defm VMAX : basic_sse12_fp_binop_p<0x5F, "max", X86fmax, 0>,
               basic_sse12_fp_binop_p_int<0x5F, "max", 0>,
               basic_sse12_fp_binop_p_y<0x5F, "max", X86fmax>,
               basic_sse12_fp_binop_p_y_int<0x5F, "max">, VEX_4V;
   defm VMIN : basic_sse12_fp_binop_s<0x5D, "min", X86fmin, 0>,
-              basic_sse12_fp_binop_s_int<0x5D, "min", 0>,
-              basic_sse12_fp_binop_p<0x5D, "min", X86fmin, 0>,
+              basic_sse12_fp_binop_s_int<0x5D, "min", 0>, VEX_4V, VEX_LIG;
+  defm VMIN : basic_sse12_fp_binop_p<0x5D, "min", X86fmin, 0>,
               basic_sse12_fp_binop_p_int<0x5D, "min", 0>,
               basic_sse12_fp_binop_p_y_int<0x5D, "min">,
               basic_sse12_fp_binop_p_y<0x5D, "min", X86fmin>, VEX_4V;
@@ -1720,23 +2843,18 @@ multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr,
 }
 
 /// sse1_fp_unop_s_avx - AVX SSE1 unops in scalar form.
-multiclass sse1_fp_unop_s_avx<bits<8> opc, string OpcodeStr,
-                              SDNode OpNode, Intrinsic F32Int> {
+multiclass sse1_fp_unop_s_avx<bits<8> opc, string OpcodeStr> {
   def SSr : SSI<opc, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src1, FR32:$src2),
                 !strconcat(OpcodeStr,
                            "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-  def SSm : I<opc, MRMSrcMem, (outs FR32:$dst), (ins FR32:$src1, f32mem:$src2),
+  let mayLoad = 1 in
+  def SSm : SSI<opc, MRMSrcMem, (outs FR32:$dst), (ins FR32:$src1,f32mem:$src2),
                 !strconcat(OpcodeStr,
-                           "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                []>, XS, Requires<[HasAVX, OptForSize]>;
-  def SSr_Int : SSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
+                           "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
+  def SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst),
+                (ins ssmem:$src1, VR128:$src2),
                 !strconcat(OpcodeStr,
-                           "ss\t{$src, $dst, $dst|$dst, $dst, $src}"),
-                [(set VR128:$dst, (F32Int VR128:$src))]>;
-  def SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst), (ins ssmem:$src),
-                !strconcat(OpcodeStr,
-                           "ss\t{$src, $dst, $dst|$dst, $dst, $src}"),
-                [(set VR128:$dst, (F32Int sse_load_f32:$src))]>;
+                           "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
 }
 
 /// sse1_fp_unop_p - SSE1 unops in packed form.
@@ -1801,21 +2919,17 @@ multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr,
 }
 
 /// sse2_fp_unop_s_avx - AVX SSE2 unops in scalar form.
-multiclass sse2_fp_unop_s_avx<bits<8> opc, string OpcodeStr,
-                              SDNode OpNode, Intrinsic F64Int> {
+multiclass sse2_fp_unop_s_avx<bits<8> opc, string OpcodeStr> {
   def SDr : SDI<opc, MRMSrcReg, (outs FR64:$dst), (ins FR64:$src1, FR64:$src2),
                !strconcat(OpcodeStr,
                           "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-  def SDm : SDI<opc, MRMSrcMem, (outs FR64:$dst),
-               (ins FR64:$src1, f64mem:$src2),
+  def SDm : SDI<opc, MRMSrcMem, (outs FR64:$dst), (ins FR64:$src1,f64mem:$src2),
+               !strconcat(OpcodeStr,
+                          "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
+  def SDm_Int : SDI<opc, MRMSrcMem, (outs VR128:$dst),
+               (ins VR128:$src1, sdmem:$src2),
                !strconcat(OpcodeStr,
                           "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-  def SDr_Int : SDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
-           !strconcat(OpcodeStr, "sd\t{$src, $dst, $dst|$dst, $dst, $src}"),
-           [(set VR128:$dst, (F64Int VR128:$src))]>;
-  def SDm_Int : SDI<opc, MRMSrcMem, (outs VR128:$dst), (ins sdmem:$src),
-           !strconcat(OpcodeStr, "sd\t{$src, $dst, $dst|$dst, $dst, $src}"),
-           [(set VR128:$dst, (F64Int sse_load_f64:$src))]>;
 }
 
 /// sse2_fp_unop_p - SSE2 unops in vector forms.
@@ -1863,9 +2977,8 @@ multiclass sse2_fp_unop_p_y_int<bits<8> opc, string OpcodeStr,
 
 let Predicates = [HasAVX] in {
   // Square root.
-  defm VSQRT  : sse1_fp_unop_s_avx<0x51, "vsqrt", fsqrt, int_x86_sse_sqrt_ss>,
-                sse2_fp_unop_s_avx<0x51, "vsqrt", fsqrt, int_x86_sse2_sqrt_sd>,
-                VEX_4V;
+  defm VSQRT  : sse1_fp_unop_s_avx<0x51, "vsqrt">,
+                sse2_fp_unop_s_avx<0x51, "vsqrt">, VEX_4V, VEX_LIG;
 
   defm VSQRT  : sse1_fp_unop_p<0x51, "vsqrt", fsqrt>,
                 sse2_fp_unop_p<0x51, "vsqrt", fsqrt>,
@@ -1879,21 +2992,76 @@ let Predicates = [HasAVX] in {
 
   // Reciprocal approximations. Note that these typically require refinement
   // in order to obtain suitable precision.
-  defm VRSQRT : sse1_fp_unop_s_avx<0x52, "vrsqrt", X86frsqrt,
-                                   int_x86_sse_rsqrt_ss>, VEX_4V;
+  defm VRSQRT : sse1_fp_unop_s_avx<0x52, "vrsqrt">, VEX_4V, VEX_LIG;
   defm VRSQRT : sse1_fp_unop_p<0x52, "vrsqrt", X86frsqrt>,
                 sse1_fp_unop_p_y<0x52, "vrsqrt", X86frsqrt>,
                 sse1_fp_unop_p_y_int<0x52, "vrsqrt", int_x86_avx_rsqrt_ps_256>,
                 sse1_fp_unop_p_int<0x52, "vrsqrt", int_x86_sse_rsqrt_ps>, VEX;
 
-  defm VRCP   : sse1_fp_unop_s_avx<0x53, "vrcp", X86frcp, int_x86_sse_rcp_ss>,
-                                   VEX_4V;
+  defm VRCP   : sse1_fp_unop_s_avx<0x53, "vrcp">, VEX_4V, VEX_LIG;
   defm VRCP   : sse1_fp_unop_p<0x53, "vrcp", X86frcp>,
                 sse1_fp_unop_p_y<0x53, "vrcp", X86frcp>,
                 sse1_fp_unop_p_y_int<0x53, "vrcp", int_x86_avx_rcp_ps_256>,
                 sse1_fp_unop_p_int<0x53, "vrcp", int_x86_sse_rcp_ps>, VEX;
 }
 
+def : Pat<(f32 (fsqrt FR32:$src)),
+          (VSQRTSSr (f32 (IMPLICIT_DEF)), FR32:$src)>, Requires<[HasAVX]>;
+def : Pat<(f32 (fsqrt (load addr:$src))),
+          (VSQRTSSm (f32 (IMPLICIT_DEF)), addr:$src)>,
+          Requires<[HasAVX, OptForSize]>;
+def : Pat<(f64 (fsqrt FR64:$src)),
+          (VSQRTSDr (f64 (IMPLICIT_DEF)), FR64:$src)>, Requires<[HasAVX]>;
+def : Pat<(f64 (fsqrt (load addr:$src))),
+          (VSQRTSDm (f64 (IMPLICIT_DEF)), addr:$src)>,
+          Requires<[HasAVX, OptForSize]>;
+
+def : Pat<(f32 (X86frsqrt FR32:$src)),
+          (VRSQRTSSr (f32 (IMPLICIT_DEF)), FR32:$src)>, Requires<[HasAVX]>;
+def : Pat<(f32 (X86frsqrt (load addr:$src))),
+          (VRSQRTSSm (f32 (IMPLICIT_DEF)), addr:$src)>,
+          Requires<[HasAVX, OptForSize]>;
+
+def : Pat<(f32 (X86frcp FR32:$src)),
+          (VRCPSSr (f32 (IMPLICIT_DEF)), FR32:$src)>, Requires<[HasAVX]>;
+def : Pat<(f32 (X86frcp (load addr:$src))),
+          (VRCPSSm (f32 (IMPLICIT_DEF)), addr:$src)>,
+          Requires<[HasAVX, OptForSize]>;
+
+let Predicates = [HasAVX] in {
+  def : Pat<(int_x86_sse_sqrt_ss VR128:$src),
+            (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)),
+                (VSQRTSSr (f32 (IMPLICIT_DEF)),
+                          (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss)),
+                sub_ss)>;
+  def : Pat<(int_x86_sse_sqrt_ss sse_load_f32:$src),
+            (VSQRTSSm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>;
+
+  def : Pat<(int_x86_sse2_sqrt_sd VR128:$src),
+            (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)),
+                (VSQRTSDr (f64 (IMPLICIT_DEF)),
+                          (EXTRACT_SUBREG (v2f64 VR128:$src), sub_sd)),
+                sub_sd)>;
+  def : Pat<(int_x86_sse2_sqrt_sd sse_load_f64:$src),
+            (VSQRTSDm_Int (v2f64 (IMPLICIT_DEF)), sse_load_f64:$src)>;
+
+  def : Pat<(int_x86_sse_rsqrt_ss VR128:$src),
+            (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)),
+                (VRSQRTSSr (f32 (IMPLICIT_DEF)),
+                          (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss)),
+                sub_ss)>;
+  def : Pat<(int_x86_sse_rsqrt_ss sse_load_f32:$src),
+            (VRSQRTSSm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>;
+
+  def : Pat<(int_x86_sse_rcp_ss VR128:$src),
+            (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)),
+                (VRCPSSr (f32 (IMPLICIT_DEF)),
+                         (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss)),
+                sub_ss)>;
+  def : Pat<(int_x86_sse_rcp_ss sse_load_f32:$src),
+            (VRCPSSm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>;
+}
+
 // Square root.
 defm SQRT  : sse1_fp_unop_s<0x51, "sqrt",  fsqrt, int_x86_sse_sqrt_ss>,
              sse1_fp_unop_p<0x51, "sqrt",  fsqrt>,
@@ -1992,7 +3160,7 @@ def MOVNTDQmr : PDI<0xE7, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
                     [(alignednontemporalstore (v4f32 VR128:$src), addr:$dst)]>;
 
 def : Pat<(alignednontemporalstore (v2i64 VR128:$src), addr:$dst),
-          (MOVNTDQmr addr:$dst, VR128:$src)>;
+          (MOVNTDQmr addr:$dst, VR128:$src)>, Requires<[HasSSE2]>;
 
 // There is no AVX form for instructions below this point
 def MOVNTImr : I<0xC3, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src),
@@ -2006,7 +3174,7 @@ def MOVNTI_64mr : RI<0xC3, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src),
 }
 
 //===----------------------------------------------------------------------===//
-// SSE 1 & 2 - Misc Instructions (No AVX form)
+// SSE 1 & 2 - Prefetch and memory fence
 //===----------------------------------------------------------------------===//
 
 // Prefetch intrinsic.
@@ -2019,66 +3187,26 @@ def PREFETCHT2   : PSI<0x18, MRM3m, (outs), (ins i8mem:$src),
 def PREFETCHNTA  : PSI<0x18, MRM0m, (outs), (ins i8mem:$src),
     "prefetchnta\t$src", [(prefetch addr:$src, imm, (i32 0), (i32 1))]>;
 
+// Flush cache
+def CLFLUSH : I<0xAE, MRM7m, (outs), (ins i8mem:$src),
+               "clflush\t$src", [(int_x86_sse2_clflush addr:$src)]>,
+              TB, Requires<[HasSSE2]>;
+
+// Pause. This "instruction" is encoded as "rep; nop", so even though it
+// was introduced with SSE2, it's backward compatible.
+def PAUSE : I<0x90, RawFrm, (outs), (ins), "pause", []>, REP;
+
 // Load, store, and memory fence
-def SFENCE : I<0xAE, MRM_F8, (outs), (ins), "sfence", [(int_x86_sse_sfence)]>,
-             TB, Requires<[HasSSE1]>;
+def SFENCE : I<0xAE, MRM_F8, (outs), (ins),
+               "sfence", [(int_x86_sse_sfence)]>, TB, Requires<[HasSSE1]>;
+def LFENCE : I<0xAE, MRM_E8, (outs), (ins),
+               "lfence", [(int_x86_sse2_lfence)]>, TB, Requires<[HasSSE2]>;
+def MFENCE : I<0xAE, MRM_F0, (outs), (ins),
+               "mfence", [(int_x86_sse2_mfence)]>, TB, Requires<[HasSSE2]>;
+
 def : Pat<(X86SFence), (SFENCE)>;
-
-// Alias instructions that map zero vector to pxor / xorp* for sse.
-// We set canFoldAsLoad because this can be converted to a constant-pool
-// load of an all-zeros value if folding it would be beneficial.
-// FIXME: Change encoding to pseudo! This is blocked right now by the x86
-// JIT implementation, it does not expand the instructions below like
-// X86MCInstLower does.
-let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
-    isCodeGenOnly = 1 in {
-def V_SET0PS : PSI<0x57, MRMInitReg, (outs VR128:$dst), (ins), "",
-                 [(set VR128:$dst, (v4f32 immAllZerosV))]>;
-def V_SET0PD : PDI<0x57, MRMInitReg, (outs VR128:$dst), (ins), "",
-                 [(set VR128:$dst, (v2f64 immAllZerosV))]>;
-let ExeDomain = SSEPackedInt in
-def V_SET0PI : PDI<0xEF, MRMInitReg, (outs VR128:$dst), (ins), "",
-                 [(set VR128:$dst, (v4i32 immAllZerosV))]>;
-}
-
-// The same as done above but for AVX. The 128-bit versions are the
-// same, but re-encoded. The 256-bit does not support PI version, and
-// doesn't need it because on sandy bridge the register is set to zero
-// at the rename stage without using any execution unit, so SET0PSY
-// and SET0PDY can be used for vector int instructions without penalty
-// FIXME: Change encoding to pseudo! This is blocked right now by the x86
-// JIT implementatioan, it does not expand the instructions below like
-// X86MCInstLower does.
-let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
-    isCodeGenOnly = 1, Predicates = [HasAVX] in {
-def AVX_SET0PS  : PSI<0x57, MRMInitReg, (outs VR128:$dst), (ins), "",
-                   [(set VR128:$dst, (v4f32 immAllZerosV))]>, VEX_4V;
-def AVX_SET0PD  : PDI<0x57, MRMInitReg, (outs VR128:$dst), (ins), "",
-                   [(set VR128:$dst, (v2f64 immAllZerosV))]>, VEX_4V;
-def AVX_SET0PSY : PSI<0x57, MRMInitReg, (outs VR256:$dst), (ins), "",
-                   [(set VR256:$dst, (v8f32 immAllZerosV))]>, VEX_4V;
-def AVX_SET0PDY : PDI<0x57, MRMInitReg, (outs VR256:$dst), (ins), "",
-                   [(set VR256:$dst, (v4f64 immAllZerosV))]>, VEX_4V;
-let ExeDomain = SSEPackedInt in
-def AVX_SET0PI  : PDI<0xEF, MRMInitReg, (outs VR128:$dst), (ins), "",
-                   [(set VR128:$dst, (v4i32 immAllZerosV))]>;
-}
-
-def : Pat<(v2i64 immAllZerosV), (V_SET0PI)>;
-def : Pat<(v8i16 immAllZerosV), (V_SET0PI)>;
-def : Pat<(v16i8 immAllZerosV), (V_SET0PI)>;
-
-def : Pat<(f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))),
-          (f32 (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss))>;
-
-// FIXME: According to the intel manual, DEST[127:64] <- SRC1[127:64], while
-// in the non-AVX version bits 127:64 aren't touched. Find a better way to
-// represent this instead of always zeroing SRC1. One possible solution is
-// to represent the instruction w/ something similar as the "$src1 = $dst"
-// constraint but without the tied operands.
-def : Pat<(extloadf32 addr:$src),
-          (VCVTSS2SDrm (f32 (EXTRACT_SUBREG (AVX_SET0PS), sub_ss)), addr:$src)>,
-      Requires<[HasAVX, OptForSpeed]>;
+def : Pat<(X86LFence), (LFENCE)>;
+def : Pat<(X86MFence), (MFENCE)>;
 
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Load/Store XCSR register
@@ -2106,10 +3234,22 @@ def VMOVDQArr  : VPDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
 def VMOVDQAYrr : VPDI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                     "movdqa\t{$src, $dst|$dst, $src}", []>, VEX;
 }
-def VMOVDQUrr  : VPDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
-                    "movdqu\t{$src, $dst|$dst, $src}", []>, XS, VEX;
-def VMOVDQUYrr : VPDI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
-                    "movdqu\t{$src, $dst|$dst, $src}", []>, XS, VEX;
+def VMOVDQUrr  : VSSI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
+                    "movdqu\t{$src, $dst|$dst, $src}", []>, VEX;
+def VMOVDQUYrr : VSSI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
+                    "movdqu\t{$src, $dst|$dst, $src}", []>, VEX;
+
+// For Disassembler
+let isCodeGenOnly = 1 in {
+def VMOVDQArr_REV  : VPDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                        "movdqa\t{$src, $dst|$dst, $src}", []>, VEX;
+def VMOVDQAYrr_REV : VPDI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src),
+                        "movdqa\t{$src, $dst|$dst, $src}", []>, VEX;
+def VMOVDQUrr_REV  : VSSI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                        "movdqu\t{$src, $dst|$dst, $src}", []>, VEX;
+def VMOVDQUYrr_REV : VSSI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src),
+                        "movdqu\t{$src, $dst|$dst, $src}", []>, VEX;
+}
 
 let canFoldAsLoad = 1, mayLoad = 1 in {
 def VMOVDQArm  : VPDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
@@ -2147,6 +3287,16 @@ def MOVDQUrr :   I<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                    "movdqu\t{$src, $dst|$dst, $src}",
                    []>, XS, Requires<[HasSSE2]>;
 
+// For Disassembler
+let isCodeGenOnly = 1 in {
+def MOVDQArr_REV : PDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                       "movdqa\t{$src, $dst|$dst, $src}", []>;
+
+def MOVDQUrr_REV :   I<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
+                       "movdqu\t{$src, $dst|$dst, $src}",
+                       []>, XS, Requires<[HasSSE2]>;
+}
+
 let canFoldAsLoad = 1, mayLoad = 1 in {
 def MOVDQArm : PDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                    "movdqa\t{$src, $dst|$dst, $src}",
@@ -2180,9 +3330,11 @@ def MOVDQUmr_Int :   I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),
 
 } // ExeDomain = SSEPackedInt
 
-def : Pat<(int_x86_avx_loadu_dq_256 addr:$src), (VMOVDQUYrm addr:$src)>;
-def : Pat<(int_x86_avx_storeu_dq_256 addr:$dst, VR256:$src),
-          (VMOVDQUYmr addr:$dst, VR256:$src)>;
+let Predicates = [HasAVX] in {
+  def : Pat<(int_x86_avx_loadu_dq_256 addr:$src), (VMOVDQUYrm addr:$src)>;
+  def : Pat<(int_x86_avx_storeu_dq_256 addr:$dst, VR256:$src),
+            (VMOVDQUYmr addr:$dst, VR256:$src)>;
+}
 
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Arithmetic Instructions
@@ -2415,15 +3567,14 @@ let ExeDomain = SSEPackedInt in {
   def VPANDNrr : PDI<0xDF, MRMSrcReg,
                     (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
                     "vpandn\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                    [(set VR128:$dst, (v2i64 (and (vnot VR128:$src1),
-                                              VR128:$src2)))]>, VEX_4V;
+                    [(set VR128:$dst,
+                          (v2i64 (X86andnp VR128:$src1, VR128:$src2)))]>,VEX_4V;
 
   def VPANDNrm : PDI<0xDF, MRMSrcMem,
                     (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
                     "vpandn\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                    [(set VR128:$dst, (v2i64 (and (vnot VR128:$src1),
-                                              (memopv2i64 addr:$src2))))]>,
-                                              VEX_4V;
+                    [(set VR128:$dst, (X86andnp VR128:$src1,
+                                            (memopv2i64 addr:$src2)))]>, VEX_4V;
 }
 }
 
@@ -2527,6 +3678,32 @@ let Predicates = [HasAVX] in {
                                     0>, VEX_4V;
   defm VPCMPGTD  : PDI_binop_rm_int<0x66, "vpcmpgtd", int_x86_sse2_pcmpgt_d, 0,
                                     0>, VEX_4V;
+
+  def : Pat<(v16i8 (X86pcmpeqb VR128:$src1, VR128:$src2)),
+            (VPCMPEQBrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v16i8 (X86pcmpeqb VR128:$src1, (memop addr:$src2))),
+            (VPCMPEQBrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v8i16 (X86pcmpeqw VR128:$src1, VR128:$src2)),
+            (VPCMPEQWrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v8i16 (X86pcmpeqw VR128:$src1, (memop addr:$src2))),
+            (VPCMPEQWrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4i32 (X86pcmpeqd VR128:$src1, VR128:$src2)),
+            (VPCMPEQDrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86pcmpeqd VR128:$src1, (memop addr:$src2))),
+            (VPCMPEQDrm VR128:$src1, addr:$src2)>;
+
+  def : Pat<(v16i8 (X86pcmpgtb VR128:$src1, VR128:$src2)),
+            (VPCMPGTBrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v16i8 (X86pcmpgtb VR128:$src1, (memop addr:$src2))),
+            (VPCMPGTBrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v8i16 (X86pcmpgtw VR128:$src1, VR128:$src2)),
+            (VPCMPGTWrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v8i16 (X86pcmpgtw VR128:$src1, (memop addr:$src2))),
+            (VPCMPGTWrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4i32 (X86pcmpgtd VR128:$src1, VR128:$src2)),
+            (VPCMPGTDrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86pcmpgtd VR128:$src1, (memop addr:$src2))),
+            (VPCMPGTDrm VR128:$src1, addr:$src2)>;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -2538,31 +3715,33 @@ let Constraints = "$src1 = $dst" in {
   defm PCMPGTD  : PDI_binop_rm_int<0x66, "pcmpgtd", int_x86_sse2_pcmpgt_d>;
 } // Constraints = "$src1 = $dst"
 
-def : Pat<(v16i8 (X86pcmpeqb VR128:$src1, VR128:$src2)),
-          (PCMPEQBrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v16i8 (X86pcmpeqb VR128:$src1, (memop addr:$src2))),
-          (PCMPEQBrm VR128:$src1, addr:$src2)>;
-def : Pat<(v8i16 (X86pcmpeqw VR128:$src1, VR128:$src2)),
-          (PCMPEQWrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v8i16 (X86pcmpeqw VR128:$src1, (memop addr:$src2))),
-          (PCMPEQWrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4i32 (X86pcmpeqd VR128:$src1, VR128:$src2)),
-          (PCMPEQDrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v4i32 (X86pcmpeqd VR128:$src1, (memop addr:$src2))),
-          (PCMPEQDrm VR128:$src1, addr:$src2)>;
+let Predicates = [HasSSE2] in {
+  def : Pat<(v16i8 (X86pcmpeqb VR128:$src1, VR128:$src2)),
+            (PCMPEQBrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v16i8 (X86pcmpeqb VR128:$src1, (memop addr:$src2))),
+            (PCMPEQBrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v8i16 (X86pcmpeqw VR128:$src1, VR128:$src2)),
+            (PCMPEQWrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v8i16 (X86pcmpeqw VR128:$src1, (memop addr:$src2))),
+            (PCMPEQWrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4i32 (X86pcmpeqd VR128:$src1, VR128:$src2)),
+            (PCMPEQDrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86pcmpeqd VR128:$src1, (memop addr:$src2))),
+            (PCMPEQDrm VR128:$src1, addr:$src2)>;
 
-def : Pat<(v16i8 (X86pcmpgtb VR128:$src1, VR128:$src2)),
-          (PCMPGTBrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v16i8 (X86pcmpgtb VR128:$src1, (memop addr:$src2))),
-          (PCMPGTBrm VR128:$src1, addr:$src2)>;
-def : Pat<(v8i16 (X86pcmpgtw VR128:$src1, VR128:$src2)),
-          (PCMPGTWrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v8i16 (X86pcmpgtw VR128:$src1, (memop addr:$src2))),
-          (PCMPGTWrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4i32 (X86pcmpgtd VR128:$src1, VR128:$src2)),
-          (PCMPGTDrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v4i32 (X86pcmpgtd VR128:$src1, (memop addr:$src2))),
-          (PCMPGTDrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v16i8 (X86pcmpgtb VR128:$src1, VR128:$src2)),
+            (PCMPGTBrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v16i8 (X86pcmpgtb VR128:$src1, (memop addr:$src2))),
+            (PCMPGTBrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v8i16 (X86pcmpgtw VR128:$src1, VR128:$src2)),
+            (PCMPGTWrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v8i16 (X86pcmpgtw VR128:$src1, (memop addr:$src2))),
+            (PCMPGTWrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v4i32 (X86pcmpgtd VR128:$src1, VR128:$src2)),
+            (PCMPGTDrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v4i32 (X86pcmpgtd VR128:$src1, (memop addr:$src2))),
+            (PCMPGTDrm VR128:$src1, addr:$src2)>;
+}
 
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Pack Instructions
@@ -2608,7 +3787,7 @@ def mi : Ii8<0x70, MRMSrcMem,
 
 let Predicates = [HasAVX] in {
   let AddedComplexity = 5 in
-  defm VPSHUFD : sse2_pshuffle<"vpshufd", v4i32, pshufd, bc_v4i32>, OpSize,
+  defm VPSHUFD : sse2_pshuffle<"vpshufd", v4i32, pshufd, bc_v4i32>, TB, OpSize,
                                VEX;
 
   // SSE2 with ImmT == Imm8 and XS prefix.
@@ -2618,6 +3797,34 @@ let Predicates = [HasAVX] in {
   // SSE2 with ImmT == Imm8 and XD prefix.
   defm VPSHUFLW : sse2_pshuffle<"vpshuflw", v8i16, pshuflw, bc_v8i16>, XD,
                                VEX;
+
+  let AddedComplexity = 5 in
+  def : Pat<(v4f32 (pshufd:$src2 VR128:$src1, (undef))),
+            (VPSHUFDri VR128:$src1, (SHUFFLE_get_shuf_imm VR128:$src2))>;
+  // Unary v4f32 shuffle with VPSHUF* in order to fold a load.
+  def : Pat<(pshufd:$src2 (bc_v4i32 (memopv4f32 addr:$src1)), (undef)),
+            (VPSHUFDmi addr:$src1, (SHUFFLE_get_shuf_imm VR128:$src2))>;
+
+  def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv2i64 addr:$src1)),
+                                   (i8 imm:$imm))),
+            (VPSHUFDmi addr:$src1, imm:$imm)>;
+  def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv4f32 addr:$src1)),
+                                   (i8 imm:$imm))),
+            (VPSHUFDmi addr:$src1, imm:$imm)>;
+  def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
+            (VPSHUFDri VR128:$src1, imm:$imm)>;
+  def : Pat<(v4i32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
+            (VPSHUFDri VR128:$src1, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShufhw VR128:$src, (i8 imm:$imm))),
+            (VPSHUFHWri VR128:$src, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShufhw (bc_v8i16 (memopv2i64 addr:$src)),
+                               (i8 imm:$imm))),
+            (VPSHUFHWmi addr:$src, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShuflw VR128:$src, (i8 imm:$imm))),
+            (VPSHUFLWri VR128:$src, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShuflw (bc_v8i16 (memopv2i64 addr:$src)),
+                               (i8 imm:$imm))),
+            (VPSHUFLWmi addr:$src, imm:$imm)>;
 }
 
 let Predicates = [HasSSE2] in {
@@ -2629,6 +3836,34 @@ let Predicates = [HasSSE2] in {
 
   // SSE2 with ImmT == Imm8 and XD prefix.
   defm PSHUFLW : sse2_pshuffle<"pshuflw", v8i16, pshuflw, bc_v8i16>, XD;
+
+  let AddedComplexity = 5 in
+  def : Pat<(v4f32 (pshufd:$src2 VR128:$src1, (undef))),
+            (PSHUFDri VR128:$src1, (SHUFFLE_get_shuf_imm VR128:$src2))>;
+  // Unary v4f32 shuffle with PSHUF* in order to fold a load.
+  def : Pat<(pshufd:$src2 (bc_v4i32 (memopv4f32 addr:$src1)), (undef)),
+            (PSHUFDmi addr:$src1, (SHUFFLE_get_shuf_imm VR128:$src2))>;
+
+  def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv2i64 addr:$src1)),
+                                   (i8 imm:$imm))),
+            (PSHUFDmi addr:$src1, imm:$imm)>;
+  def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv4f32 addr:$src1)),
+                                   (i8 imm:$imm))),
+            (PSHUFDmi addr:$src1, imm:$imm)>;
+  def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
+            (PSHUFDri VR128:$src1, imm:$imm)>;
+  def : Pat<(v4i32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
+            (PSHUFDri VR128:$src1, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShufhw VR128:$src, (i8 imm:$imm))),
+            (PSHUFHWri VR128:$src, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShufhw (bc_v8i16 (memopv2i64 addr:$src)),
+                               (i8 imm:$imm))),
+            (PSHUFHWmi addr:$src, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShuflw VR128:$src, (i8 imm:$imm))),
+            (PSHUFLWri VR128:$src, imm:$imm)>;
+  def : Pat<(v8i16 (X86PShuflw (bc_v8i16 (memopv2i64 addr:$src)),
+                               (i8 imm:$imm))),
+            (PSHUFLWmi addr:$src, imm:$imm)>;
 }
 
 //===---------------------------------------------------------------------===//
@@ -2637,71 +3872,69 @@ let Predicates = [HasSSE2] in {
 
 let ExeDomain = SSEPackedInt in {
 multiclass sse2_unpack<bits<8> opc, string OpcodeStr, ValueType vt,
-                       PatFrag unp_frag, PatFrag bc_frag, bit Is2Addr = 1> {
+                       SDNode OpNode, PatFrag bc_frag, bit Is2Addr = 1> {
   def rr : PDI<opc, MRMSrcReg,
       (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
       !if(Is2Addr,
           !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"),
           !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-      [(set VR128:$dst, (vt (unp_frag VR128:$src1, VR128:$src2)))]>;
+      [(set VR128:$dst, (vt (OpNode VR128:$src1, VR128:$src2)))]>;
   def rm : PDI<opc, MRMSrcMem,
       (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
       !if(Is2Addr,
           !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"),
           !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-      [(set VR128:$dst, (unp_frag VR128:$src1,
+      [(set VR128:$dst, (OpNode VR128:$src1,
                                   (bc_frag (memopv2i64
                                                addr:$src2))))]>;
 }
 
 let Predicates = [HasAVX] in {
-  defm VPUNPCKLBW  : sse2_unpack<0x60, "vpunpcklbw", v16i8, unpckl, bc_v16i8,
-                                 0>, VEX_4V;
-  defm VPUNPCKLWD  : sse2_unpack<0x61, "vpunpcklwd", v8i16, unpckl, bc_v8i16,
-                                 0>, VEX_4V;
-  defm VPUNPCKLDQ  : sse2_unpack<0x62, "vpunpckldq", v4i32, unpckl, bc_v4i32,
-                                 0>, VEX_4V;
+  defm VPUNPCKLBW  : sse2_unpack<0x60, "vpunpcklbw", v16i8, X86Punpcklbw,
+                                 bc_v16i8, 0>, VEX_4V;
+  defm VPUNPCKLWD  : sse2_unpack<0x61, "vpunpcklwd", v8i16, X86Punpcklwd,
+                                 bc_v8i16, 0>, VEX_4V;
+  defm VPUNPCKLDQ  : sse2_unpack<0x62, "vpunpckldq", v4i32, X86Punpckldq,
+                                 bc_v4i32, 0>, VEX_4V;
 
   /// FIXME: we could eliminate this and use sse2_unpack instead if tblgen
   /// knew to collapse (bitconvert VT to VT) into its operand.
   def VPUNPCKLQDQrr : PDI<0x6C, MRMSrcReg,
-                         (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
-                         "vpunpcklqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                        [(set VR128:$dst,
-                          (v2i64 (unpckl VR128:$src1, VR128:$src2)))]>, VEX_4V;
+            (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
+            "vpunpcklqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+            [(set VR128:$dst, (v2i64 (X86Punpcklqdq VR128:$src1,
+                                                    VR128:$src2)))]>, VEX_4V;
   def VPUNPCKLQDQrm : PDI<0x6C, MRMSrcMem,
-                         (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
-                         "vpunpcklqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                        [(set VR128:$dst,
-                          (v2i64 (unpckl VR128:$src1,
-                                         (memopv2i64 addr:$src2))))]>, VEX_4V;
+            (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
+            "vpunpcklqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+            [(set VR128:$dst, (v2i64 (X86Punpcklqdq VR128:$src1,
+                                        (memopv2i64 addr:$src2))))]>, VEX_4V;
 
-  defm VPUNPCKHBW  : sse2_unpack<0x68, "vpunpckhbw", v16i8, unpckh, bc_v16i8,
-                                 0>, VEX_4V;
-  defm VPUNPCKHWD  : sse2_unpack<0x69, "vpunpckhwd", v8i16, unpckh, bc_v8i16,
-                                 0>, VEX_4V;
-  defm VPUNPCKHDQ  : sse2_unpack<0x6A, "vpunpckhdq", v4i32, unpckh, bc_v4i32,
-                                 0>, VEX_4V;
+  defm VPUNPCKHBW  : sse2_unpack<0x68, "vpunpckhbw", v16i8, X86Punpckhbw,
+                                 bc_v16i8, 0>, VEX_4V;
+  defm VPUNPCKHWD  : sse2_unpack<0x69, "vpunpckhwd", v8i16, X86Punpckhwd,
+                                 bc_v8i16, 0>, VEX_4V;
+  defm VPUNPCKHDQ  : sse2_unpack<0x6A, "vpunpckhdq", v4i32, X86Punpckhdq,
+                                 bc_v4i32, 0>, VEX_4V;
 
   /// FIXME: we could eliminate this and use sse2_unpack instead if tblgen
   /// knew to collapse (bitconvert VT to VT) into its operand.
   def VPUNPCKHQDQrr : PDI<0x6D, MRMSrcReg,
-                         (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
-                         "vpunpckhqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                        [(set VR128:$dst,
-                          (v2i64 (unpckh VR128:$src1, VR128:$src2)))]>, VEX_4V;
+             (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
+             "vpunpckhqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+             [(set VR128:$dst, (v2i64 (X86Punpckhqdq VR128:$src1,
+                                                     VR128:$src2)))]>, VEX_4V;
   def VPUNPCKHQDQrm : PDI<0x6D, MRMSrcMem,
-                        (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
-                        "vpunpckhqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                        [(set VR128:$dst,
-                          (v2i64 (unpckh VR128:$src1,
-                                         (memopv2i64 addr:$src2))))]>, VEX_4V;
+             (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
+             "vpunpckhqdq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+             [(set VR128:$dst, (v2i64 (X86Punpckhqdq VR128:$src1,
+                                        (memopv2i64 addr:$src2))))]>, VEX_4V;
 }
 
 let Constraints = "$src1 = $dst" in {
-  defm PUNPCKLBW  : sse2_unpack<0x60, "punpcklbw", v16i8, unpckl, bc_v16i8>;
-  defm PUNPCKLWD  : sse2_unpack<0x61, "punpcklwd", v8i16, unpckl, bc_v8i16>;
-  defm PUNPCKLDQ  : sse2_unpack<0x62, "punpckldq", v4i32, unpckl, bc_v4i32>;
+  defm PUNPCKLBW  : sse2_unpack<0x60, "punpcklbw", v16i8, X86Punpcklbw, bc_v16i8>;
+  defm PUNPCKLWD  : sse2_unpack<0x61, "punpcklwd", v8i16, X86Punpcklwd, bc_v8i16>;
+  defm PUNPCKLDQ  : sse2_unpack<0x62, "punpckldq", v4i32, X86Punpckldq, bc_v4i32>;
 
   /// FIXME: we could eliminate this and use sse2_unpack instead if tblgen
   /// knew to collapse (bitconvert VT to VT) into its operand.
@@ -2709,17 +3942,17 @@ let Constraints = "$src1 = $dst" in {
                          (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
                          "punpcklqdq\t{$src2, $dst|$dst, $src2}",
                         [(set VR128:$dst,
-                          (v2i64 (unpckl VR128:$src1, VR128:$src2)))]>;
+                          (v2i64 (X86Punpcklqdq VR128:$src1, VR128:$src2)))]>;
   def PUNPCKLQDQrm : PDI<0x6C, MRMSrcMem,
                          (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
                          "punpcklqdq\t{$src2, $dst|$dst, $src2}",
                         [(set VR128:$dst,
-                          (v2i64 (unpckl VR128:$src1,
+                          (v2i64 (X86Punpcklqdq VR128:$src1,
                                          (memopv2i64 addr:$src2))))]>;
 
-  defm PUNPCKHBW  : sse2_unpack<0x68, "punpckhbw", v16i8, unpckh, bc_v16i8>;
-  defm PUNPCKHWD  : sse2_unpack<0x69, "punpckhwd", v8i16, unpckh, bc_v8i16>;
-  defm PUNPCKHDQ  : sse2_unpack<0x6A, "punpckhdq", v4i32, unpckh, bc_v4i32>;
+  defm PUNPCKHBW  : sse2_unpack<0x68, "punpckhbw", v16i8, X86Punpckhbw, bc_v16i8>;
+  defm PUNPCKHWD  : sse2_unpack<0x69, "punpckhwd", v8i16, X86Punpckhwd, bc_v8i16>;
+  defm PUNPCKHDQ  : sse2_unpack<0x6A, "punpckhdq", v4i32, X86Punpckhdq, bc_v4i32>;
 
   /// FIXME: we could eliminate this and use sse2_unpack instead if tblgen
   /// knew to collapse (bitconvert VT to VT) into its operand.
@@ -2727,17 +3960,24 @@ let Constraints = "$src1 = $dst" in {
                          (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
                          "punpckhqdq\t{$src2, $dst|$dst, $src2}",
                         [(set VR128:$dst,
-                          (v2i64 (unpckh VR128:$src1, VR128:$src2)))]>;
+                          (v2i64 (X86Punpckhqdq VR128:$src1, VR128:$src2)))]>;
   def PUNPCKHQDQrm : PDI<0x6D, MRMSrcMem,
                         (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2),
                         "punpckhqdq\t{$src2, $dst|$dst, $src2}",
                         [(set VR128:$dst,
-                          (v2i64 (unpckh VR128:$src1,
+                          (v2i64 (X86Punpckhqdq VR128:$src1,
                                          (memopv2i64 addr:$src2))))]>;
 }
-
 } // ExeDomain = SSEPackedInt
 
+// Splat v2f64 / v2i64
+let AddedComplexity = 10 in {
+  def : Pat<(splat_lo (v2i64 VR128:$src), (undef)),
+            (PUNPCKLQDQrr VR128:$src, VR128:$src)>, Requires<[HasSSE2]>;
+  def : Pat<(splat_lo (v2i64 VR128:$src), (undef)),
+            (VPUNPCKLQDQrr VR128:$src, VR128:$src)>, Requires<[HasAVX]>;
+}
+
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Extract and Insert
 //===---------------------------------------------------------------------===//
@@ -2769,7 +4009,7 @@ def VPEXTRWri : Ii8<0xC5, MRMSrcReg,
                     (outs GR32:$dst), (ins VR128:$src1, i32i8imm:$src2),
                     "vpextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set GR32:$dst, (X86pextrw (v8i16 VR128:$src1),
-                                                imm:$src2))]>, OpSize, VEX;
+                                                imm:$src2))]>, TB, OpSize, VEX;
 def PEXTRWri : PDIi8<0xC5, MRMSrcReg,
                     (outs GR32:$dst), (ins VR128:$src1, i32i8imm:$src2),
                     "pextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}",
@@ -2778,11 +4018,11 @@ def PEXTRWri : PDIi8<0xC5, MRMSrcReg,
 
 // Insert
 let Predicates = [HasAVX] in {
-  defm VPINSRW : sse2_pinsrw<0>, OpSize, VEX_4V;
+  defm VPINSRW : sse2_pinsrw<0>, TB, OpSize, VEX_4V;
   def  VPINSRWrr64i : Ii8<0xC4, MRMSrcReg, (outs VR128:$dst),
        (ins VR128:$src1, GR64:$src2, i32i8imm:$src3),
        "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-       []>, OpSize, VEX_4V;
+       []>, TB, OpSize, VEX_4V;
 }
 
 let Constraints = "$src1 = $dst" in
@@ -2839,7 +4079,9 @@ def MASKMOVDQU64 : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask),
 // SSE2 - Move Doubleword
 //===---------------------------------------------------------------------===//
 
+//===---------------------------------------------------------------------===//
 // Move Int Doubleword to Packed Double Int
+//
 def VMOVDI2PDIrr : VPDI<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src),
                       "movd\t{$src, $dst|$dst, $src}",
                       [(set VR128:$dst,
@@ -2849,6 +4091,14 @@ def VMOVDI2PDIrm : VPDI<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src),
                       [(set VR128:$dst,
                         (v4i32 (scalar_to_vector (loadi32 addr:$src))))]>,
                       VEX;
+def VMOV64toPQIrr : VRPDI<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src),
+                        "mov{d|q}\t{$src, $dst|$dst, $src}",
+                        [(set VR128:$dst,
+                          (v2i64 (scalar_to_vector GR64:$src)))]>, VEX;
+def VMOV64toSDrr : VRPDI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src),
+                       "mov{d|q}\t{$src, $dst|$dst, $src}",
+                       [(set FR64:$dst, (bitconvert GR64:$src))]>, VEX;
+
 def MOVDI2PDIrr : PDI<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src),
                       "movd\t{$src, $dst|$dst, $src}",
                       [(set VR128:$dst,
@@ -2865,8 +4115,9 @@ def MOV64toSDrr : RPDI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src),
                        "mov{d|q}\t{$src, $dst|$dst, $src}",
                        [(set FR64:$dst, (bitconvert GR64:$src))]>;
 
-
+//===---------------------------------------------------------------------===//
 // Move Int Doubleword to Single Scalar
+//
 def VMOVDI2SSrr  : VPDI<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src),
                       "movd\t{$src, $dst|$dst, $src}",
                       [(set FR32:$dst, (bitconvert GR32:$src))]>, VEX;
@@ -2883,7 +4134,9 @@ def MOVDI2SSrm  : PDI<0x6E, MRMSrcMem, (outs FR32:$dst), (ins i32mem:$src),
                       "movd\t{$src, $dst|$dst, $src}",
                       [(set FR32:$dst, (bitconvert (loadi32 addr:$src)))]>;
 
+//===---------------------------------------------------------------------===//
 // Move Packed Doubleword Int to Packed Double Int
+//
 def VMOVPDI2DIrr  : VPDI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
                        "movd\t{$src, $dst|$dst, $src}",
                        [(set GR32:$dst, (vector_extract (v4i32 VR128:$src),
@@ -2902,22 +4155,48 @@ def MOVPDI2DImr  : PDI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src),
                        [(store (i32 (vector_extract (v4i32 VR128:$src),
                                      (iPTR 0))), addr:$dst)]>;
 
-def MOVPQIto64rr  : RPDI<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src),
+//===---------------------------------------------------------------------===//
+// Move Packed Doubleword Int first element to Doubleword Int
+//
+def VMOVPQIto64rr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src),
+                          "mov{d|q}\t{$src, $dst|$dst, $src}",
+                          [(set GR64:$dst, (vector_extract (v2i64 VR128:$src),
+                                                           (iPTR 0)))]>,
+                      TB, OpSize, VEX, VEX_W, Requires<[HasAVX, In64BitMode]>;
+
+def MOVPQIto64rr : RPDI<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src),
+                        "mov{d|q}\t{$src, $dst|$dst, $src}",
+                        [(set GR64:$dst, (vector_extract (v2i64 VR128:$src),
+                                                         (iPTR 0)))]>;
+
+//===---------------------------------------------------------------------===//
+// Bitcast FR64 <-> GR64
+//
+let Predicates = [HasAVX] in
+def VMOV64toSDrm : S3SI<0x7E, MRMSrcMem, (outs FR64:$dst), (ins i64mem:$src),
+                        "vmovq\t{$src, $dst|$dst, $src}",
+                        [(set FR64:$dst, (bitconvert (loadi64 addr:$src)))]>,
+                        VEX;
+def VMOVSDto64rr : VRPDI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src),
                          "mov{d|q}\t{$src, $dst|$dst, $src}",
-                         [(set GR64:$dst, (vector_extract (v2i64 VR128:$src),
-                                           (iPTR 0)))]>;
+                         [(set GR64:$dst, (bitconvert FR64:$src))]>;
+def VMOVSDto64mr : VRPDI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src),
+                         "movq\t{$src, $dst|$dst, $src}",
+                         [(store (i64 (bitconvert FR64:$src)), addr:$dst)]>;
+
 def MOV64toSDrm : S3SI<0x7E, MRMSrcMem, (outs FR64:$dst), (ins i64mem:$src),
                        "movq\t{$src, $dst|$dst, $src}",
                        [(set FR64:$dst, (bitconvert (loadi64 addr:$src)))]>;
+def MOVSDto64rr : RPDI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src),
+                       "mov{d|q}\t{$src, $dst|$dst, $src}",
+                       [(set GR64:$dst, (bitconvert FR64:$src))]>;
+def MOVSDto64mr : RPDI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src),
+                       "movq\t{$src, $dst|$dst, $src}",
+                       [(store (i64 (bitconvert FR64:$src)), addr:$dst)]>;
 
-def MOVSDto64rr  : RPDI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src),
-                        "mov{d|q}\t{$src, $dst|$dst, $src}",
-                        [(set GR64:$dst, (bitconvert FR64:$src))]>;
-def MOVSDto64mr  : RPDI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src),
-                        "movq\t{$src, $dst|$dst, $src}",
-                        [(store (i64 (bitconvert FR64:$src)), addr:$dst)]>;
-
+//===---------------------------------------------------------------------===//
 // Move Scalar Single to Double Int
+//
 def VMOVSS2DIrr  : VPDI<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src),
                       "movd\t{$src, $dst|$dst, $src}",
                       [(set GR32:$dst, (bitconvert FR32:$src))]>, VEX;
@@ -2931,7 +4210,9 @@ def MOVSS2DImr  : PDI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32:$src),
                       "movd\t{$src, $dst|$dst, $src}",
                       [(store (i32 (bitconvert FR32:$src)), addr:$dst)]>;
 
-// movd / movq to XMM register zero-extends
+//===---------------------------------------------------------------------===//
+// Patterns and instructions to describe movd/movq to XMM register zero-extends
+//
 let AddedComplexity = 15 in {
 def VMOVZDI2PDIrr : VPDI<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src),
                        "movd\t{$src, $dst|$dst, $src}",
@@ -2967,15 +4248,36 @@ def MOVZDI2PDIrm : PDI<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src),
                        [(set VR128:$dst,
                          (v4i32 (X86vzmovl (v4i32 (scalar_to_vector
                                                    (loadi32 addr:$src))))))]>;
+}
 
-def : Pat<(v4i32 (X86vzmovl (loadv4i32 addr:$src))),
+let Predicates = [HasSSE2], AddedComplexity = 20 in {
+  def : Pat<(v4i32 (X86vzmovl (loadv4i32 addr:$src))),
             (MOVZDI2PDIrm addr:$src)>;
-def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
+  def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
             (MOVZDI2PDIrm addr:$src)>;
-def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
+  def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
             (MOVZDI2PDIrm addr:$src)>;
 }
 
+let Predicates = [HasAVX] in {
+  // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part.
+  let AddedComplexity = 20 in {
+    def : Pat<(v4i32 (X86vzmovl (loadv4i32 addr:$src))),
+              (VMOVZDI2PDIrm addr:$src)>;
+    def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
+              (VMOVZDI2PDIrm addr:$src)>;
+    def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
+              (VMOVZDI2PDIrm addr:$src)>;
+  }
+  // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext.
+  def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
+                                (v4i32 (scalar_to_vector GR32:$src)),(i32 0)))),
+            (SUBREG_TO_REG (i32 0), (VMOVZDI2PDIrr GR32:$src), sub_xmm)>;
+  def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
+                                (v2i64 (scalar_to_vector GR64:$src)),(i32 0)))),
+            (SUBREG_TO_REG (i64 0), (VMOVZQI2PQIrr GR64:$src), sub_xmm)>;
+}
+
 // These are the correct encodings of the instructions so that we know how to
 // read correct assembly, even though we continue to emit the wrong ones for
 // compatibility with Darwin's buggy assembler.
@@ -2996,7 +4298,9 @@ def : InstAlias<"movq\t{$src, $dst|$dst, $src}",
 // SSE2 - Move Quadword
 //===---------------------------------------------------------------------===//
 
+//===---------------------------------------------------------------------===//
 // Move Quadword Int to Packed Quadword Int
+//
 def VMOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                     "vmovq\t{$src, $dst|$dst, $src}",
                     [(set VR128:$dst,
@@ -3008,7 +4312,9 @@ def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                       (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, XS,
                     Requires<[HasSSE2]>; // SSE2 instruction with XS Prefix
 
+//===---------------------------------------------------------------------===//
 // Move Packed Quadword Int to Quadword Int
+//
 def VMOVPQI2QImr : VPDI<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
                       "movq\t{$src, $dst|$dst, $src}",
                       [(store (i64 (vector_extract (v2i64 VR128:$src),
@@ -3018,10 +4324,9 @@ def MOVPQI2QImr : PDI<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
                       [(store (i64 (vector_extract (v2i64 VR128:$src),
                                     (iPTR 0))), addr:$dst)]>;
 
-def : Pat<(f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))),
-          (f64 (EXTRACT_SUBREG (v2f64 VR128:$src), sub_sd))>;
-
+//===---------------------------------------------------------------------===//
 // Store / copy lower 64-bits of a XMM register.
+//
 def VMOVLQ128mr : VPDI<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
                      "movq\t{$src, $dst|$dst, $src}",
                      [(int_x86_sse2_storel_dq addr:$dst, VR128:$src)]>, VEX;
@@ -3037,7 +4342,7 @@ def VMOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                                                  (loadi64 addr:$src))))))]>,
                      XS, VEX, Requires<[HasAVX]>;
 
-let AddedComplexity = 20 in {
+let AddedComplexity = 20 in
 def MOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                      "movq\t{$src, $dst|$dst, $src}",
                      [(set VR128:$dst,
@@ -3045,15 +4350,27 @@ def MOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                                                  (loadi64 addr:$src))))))]>,
                      XS, Requires<[HasSSE2]>;
 
-def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
+let Predicates = [HasSSE2], AddedComplexity = 20 in {
+  def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
             (MOVZQI2PQIrm addr:$src)>;
-def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))),
+  def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))),
             (MOVZQI2PQIrm addr:$src)>;
-def : Pat<(v2i64 (X86vzload addr:$src)), (MOVZQI2PQIrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzload addr:$src)), (MOVZQI2PQIrm addr:$src)>;
 }
 
+let Predicates = [HasAVX], AddedComplexity = 20 in {
+  def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
+            (VMOVZQI2PQIrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))),
+            (VMOVZQI2PQIrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzload addr:$src)),
+            (VMOVZQI2PQIrm addr:$src)>;
+}
+
+//===---------------------------------------------------------------------===//
 // Moving from XMM to XMM and clear upper 64 bits. Note, there is a bug in
 // IA32 document. movq xmm1, xmm2 does clear the high bits.
+//
 let AddedComplexity = 15 in
 def VMOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                         "vmovq\t{$src, $dst|$dst, $src}",
@@ -3077,9 +4394,21 @@ def MOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                     [(set VR128:$dst, (v2i64 (X86vzmovl
                                              (loadv2i64 addr:$src))))]>,
                       XS, Requires<[HasSSE2]>;
+}
 
-def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4i32 addr:$src)))),
-            (MOVZPQILo2PQIrm addr:$src)>;
+let AddedComplexity = 20 in {
+  let Predicates = [HasSSE2] in {
+    def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4i32 addr:$src)))),
+              (MOVZPQILo2PQIrm addr:$src)>;
+    def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),
+              (MOVZPQILo2PQIrr VR128:$src)>;
+  }
+  let Predicates = [HasAVX] in {
+    def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4i32 addr:$src)))),
+              (VMOVZPQILo2PQIrm addr:$src)>;
+    def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),
+              (VMOVZPQILo2PQIrr VR128:$src)>;
+  }
 }
 
 // Instructions to match in the assembler
@@ -3101,37 +4430,6 @@ def VMOVQxrxr: I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
 def MOVQxrxr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                  "movq\t{$src, $dst|$dst, $src}", []>, XS;
 
-//===---------------------------------------------------------------------===//
-// SSE2 - Misc Instructions
-//===---------------------------------------------------------------------===//
-
-// Flush cache
-def CLFLUSH : I<0xAE, MRM7m, (outs), (ins i8mem:$src),
-               "clflush\t$src", [(int_x86_sse2_clflush addr:$src)]>,
-              TB, Requires<[HasSSE2]>;
-
-// Load, store, and memory fence
-def LFENCE : I<0xAE, MRM_E8, (outs), (ins),
-               "lfence", [(int_x86_sse2_lfence)]>, TB, Requires<[HasSSE2]>;
-def MFENCE : I<0xAE, MRM_F0, (outs), (ins),
-               "mfence", [(int_x86_sse2_mfence)]>, TB, Requires<[HasSSE2]>;
-def : Pat<(X86LFence), (LFENCE)>;
-def : Pat<(X86MFence), (MFENCE)>;
-
-
-// Pause. This "instruction" is encoded as "rep; nop", so even though it
-// was introduced with SSE2, it's backward compatible.
-def PAUSE : I<0x90, RawFrm, (outs), (ins), "pause", []>, REP;
-
-// Alias instructions that map zero vector to pxor / xorp* for sse.
-// We set canFoldAsLoad because this can be converted to a constant-pool
-// load of an all-ones value if folding it would be beneficial.
-let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
-    isCodeGenOnly = 1, ExeDomain = SSEPackedInt in
-  // FIXME: Change encoding to pseudo.
-  def V_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins), "",
-                         [(set VR128:$dst, (v4i32 immAllOnesV))]>;
-
 //===---------------------------------------------------------------------===//
 // SSE3 - Conversion Instructions
 //===---------------------------------------------------------------------===//
@@ -3164,6 +4462,11 @@ def CVTPD2DQrm  : S3DI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
 def CVTPD2DQrr  : S3DI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "cvtpd2dq\t{$src, $dst|$dst, $src}", []>;
 
+def : Pat<(v4i32 (fp_to_sint (v4f64 VR256:$src))),
+          (VCVTPD2DQYrr VR256:$src)>;
+def : Pat<(v4i32 (fp_to_sint (memopv4f64 addr:$src))),
+          (VCVTPD2DQYrm addr:$src)>;
+
 // Convert Packed DW Integers to Packed Double FP
 let Predicates = [HasAVX] in {
 def VCVTDQ2PDrm  : S3SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
@@ -3192,41 +4495,74 @@ def : Pat<(int_x86_avx_cvt_pd2dq_256 VR256:$src),
 def : Pat<(int_x86_avx_cvt_pd2dq_256 (memopv4f64 addr:$src)),
           (VCVTPD2DQYrm addr:$src)>;
 
-//===---------------------------------------------------------------------===//
-// SSE3 - Move Instructions
-//===---------------------------------------------------------------------===//
+def : Pat<(v4f64 (sint_to_fp (v4i32 VR128:$src))),
+          (VCVTDQ2PDYrr VR128:$src)>;
+def : Pat<(v4f64 (sint_to_fp (memopv4i32 addr:$src))),
+          (VCVTDQ2PDYrm addr:$src)>;
 
-// Replicate Single FP
-multiclass sse3_replicate_sfp<bits<8> op, PatFrag rep_frag, string OpcodeStr> {
-def rr : S3SI<op, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
+//===---------------------------------------------------------------------===//
+// SSE3 - Replicate Single FP - MOVSHDUP and MOVSLDUP
+//===---------------------------------------------------------------------===//
+multiclass sse3_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr,
+                              ValueType vt, RegisterClass RC, PatFrag mem_frag,
+                              X86MemOperand x86memop> {
+def rr : S3SI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
                     !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                      [(set VR128:$dst, (v4f32 (rep_frag
-                                                VR128:$src, (undef))))]>;
-def rm : S3SI<op, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
+                      [(set RC:$dst, (vt (OpNode RC:$src)))]>;
+def rm : S3SI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
                     !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                      [(set VR128:$dst, (rep_frag
-                                         (memopv4f32 addr:$src), (undef)))]>;
-}
-
-multiclass sse3_replicate_sfp_y<bits<8> op, PatFrag rep_frag,
-                                string OpcodeStr> {
-def rr : S3SI<op, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
-              !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
-def rm : S3SI<op, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
-              !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
+                      [(set RC:$dst, (OpNode (mem_frag addr:$src)))]>;
 }
 
 let Predicates = [HasAVX] in {
-  // FIXME: Merge above classes when we have patterns for the ymm version
-  defm VMOVSHDUP  : sse3_replicate_sfp<0x16, movshdup, "vmovshdup">, VEX;
-  defm VMOVSLDUP  : sse3_replicate_sfp<0x12, movsldup, "vmovsldup">, VEX;
-  defm VMOVSHDUPY : sse3_replicate_sfp_y<0x16, movshdup, "vmovshdup">, VEX;
-  defm VMOVSLDUPY : sse3_replicate_sfp_y<0x12, movsldup, "vmovsldup">, VEX;
+  defm VMOVSHDUP  : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup",
+                                       v4f32, VR128, memopv4f32, f128mem>, VEX;
+  defm VMOVSLDUP  : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup",
+                                       v4f32, VR128, memopv4f32, f128mem>, VEX;
+  defm VMOVSHDUPY : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup",
+                                       v8f32, VR256, memopv8f32, f256mem>, VEX;
+  defm VMOVSLDUPY : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup",
+                                       v8f32, VR256, memopv8f32, f256mem>, VEX;
 }
-defm MOVSHDUP : sse3_replicate_sfp<0x16, movshdup, "movshdup">;
-defm MOVSLDUP : sse3_replicate_sfp<0x12, movsldup, "movsldup">;
+defm MOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "movshdup", v4f32, VR128,
+                                   memopv4f32, f128mem>;
+defm MOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "movsldup", v4f32, VR128,
+                                   memopv4f32, f128mem>;
+
+let Predicates = [HasSSE3] in {
+  def : Pat<(v4i32 (X86Movshdup VR128:$src)),
+            (MOVSHDUPrr VR128:$src)>;
+  def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (memopv2i64 addr:$src)))),
+            (MOVSHDUPrm addr:$src)>;
+  def : Pat<(v4i32 (X86Movsldup VR128:$src)),
+            (MOVSLDUPrr VR128:$src)>;
+  def : Pat<(v4i32 (X86Movsldup (bc_v4i32 (memopv2i64 addr:$src)))),
+            (MOVSLDUPrm addr:$src)>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(v4i32 (X86Movshdup VR128:$src)),
+            (VMOVSHDUPrr VR128:$src)>;
+  def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (memopv2i64 addr:$src)))),
+            (VMOVSHDUPrm addr:$src)>;
+  def : Pat<(v4i32 (X86Movsldup VR128:$src)),
+            (VMOVSLDUPrr VR128:$src)>;
+  def : Pat<(v4i32 (X86Movsldup (bc_v4i32 (memopv2i64 addr:$src)))),
+            (VMOVSLDUPrm addr:$src)>;
+  def : Pat<(v8i32 (X86Movshdup VR256:$src)),
+            (VMOVSHDUPYrr VR256:$src)>;
+  def : Pat<(v8i32 (X86Movshdup (bc_v8i32 (memopv4i64 addr:$src)))),
+            (VMOVSHDUPYrm addr:$src)>;
+  def : Pat<(v8i32 (X86Movsldup VR256:$src)),
+            (VMOVSLDUPYrr VR256:$src)>;
+  def : Pat<(v8i32 (X86Movsldup (bc_v8i32 (memopv4i64 addr:$src)))),
+            (VMOVSLDUPYrm addr:$src)>;
+}
+
+//===---------------------------------------------------------------------===//
+// SSE3 - Replicate Double FP - MOVDDUP
+//===---------------------------------------------------------------------===//
 
-// Replicate Double FP
 multiclass sse3_replicate_dfp<string OpcodeStr> {
 def rr  : S3DI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                     !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
@@ -3238,23 +4574,90 @@ def rm  : S3DI<0x12, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
                                       (undef))))]>;
 }
 
+// FIXME: Merge with above classe when there're patterns for the ymm version
 multiclass sse3_replicate_dfp_y<string OpcodeStr> {
-def rr  : S3DI<0x12, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
-                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                    []>;
-def rm  : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
-                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                    []>;
+let Predicates = [HasAVX] in {
+  def rr  : S3DI<0x12, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
+                      !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
+                      []>;
+  def rm  : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
+                      !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
+                      []>;
+  }
+}
+
+defm MOVDDUP : sse3_replicate_dfp<"movddup">;
+defm VMOVDDUP  : sse3_replicate_dfp<"vmovddup">, VEX;
+defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup">, VEX;
+
+let Predicates = [HasSSE3] in {
+  def : Pat<(movddup (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src)))),
+                   (undef)),
+            (MOVDDUPrm addr:$src)>;
+  let AddedComplexity = 5 in {
+  def : Pat<(movddup (memopv2f64 addr:$src), (undef)), (MOVDDUPrm addr:$src)>;
+  def : Pat<(movddup (bc_v4f32 (memopv2f64 addr:$src)), (undef)),
+            (MOVDDUPrm addr:$src)>;
+  def : Pat<(movddup (memopv2i64 addr:$src), (undef)), (MOVDDUPrm addr:$src)>;
+  def : Pat<(movddup (bc_v4i32 (memopv2i64 addr:$src)), (undef)),
+            (MOVDDUPrm addr:$src)>;
+  }
+  def : Pat<(X86Movddup (memopv2f64 addr:$src)),
+            (MOVDDUPrm addr:$src)>;
+  def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))),
+            (MOVDDUPrm addr:$src)>;
+  def : Pat<(X86Movddup (bc_v2f64 (memopv2i64 addr:$src))),
+            (MOVDDUPrm addr:$src)>;
+  def : Pat<(X86Movddup (v2f64 (scalar_to_vector (loadf64 addr:$src)))),
+            (MOVDDUPrm addr:$src)>;
+  def : Pat<(X86Movddup (bc_v2f64
+                             (v2i64 (scalar_to_vector (loadi64 addr:$src))))),
+            (MOVDDUPrm addr:$src)>;
 }
 
 let Predicates = [HasAVX] in {
-  // FIXME: Merge above classes when we have patterns for the ymm version
-  defm VMOVDDUP  : sse3_replicate_dfp<"vmovddup">, VEX;
-  defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup">, VEX;
-}
-defm MOVDDUP : sse3_replicate_dfp<"movddup">;
+  def : Pat<(movddup (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src)))),
+                   (undef)),
+            (VMOVDDUPrm addr:$src)>;
+  let AddedComplexity = 5 in {
+  def : Pat<(movddup (memopv2f64 addr:$src), (undef)), (VMOVDDUPrm addr:$src)>;
+  def : Pat<(movddup (bc_v4f32 (memopv2f64 addr:$src)), (undef)),
+            (VMOVDDUPrm addr:$src)>;
+  def : Pat<(movddup (memopv2i64 addr:$src), (undef)), (VMOVDDUPrm addr:$src)>;
+  def : Pat<(movddup (bc_v4i32 (memopv2i64 addr:$src)), (undef)),
+            (VMOVDDUPrm addr:$src)>;
+  }
+  def : Pat<(X86Movddup (memopv2f64 addr:$src)),
+            (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
+  def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))),
+            (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
+  def : Pat<(X86Movddup (bc_v2f64 (memopv2i64 addr:$src))),
+            (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
+  def : Pat<(X86Movddup (v2f64 (scalar_to_vector (loadf64 addr:$src)))),
+            (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
+  def : Pat<(X86Movddup (bc_v2f64
+                             (v2i64 (scalar_to_vector (loadi64 addr:$src))))),
+            (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
+
+  // 256-bit version
+  def : Pat<(X86Movddup (memopv4f64 addr:$src)),
+            (VMOVDDUPYrm addr:$src)>;
+  def : Pat<(X86Movddup (memopv4i64 addr:$src)),
+            (VMOVDDUPYrm addr:$src)>;
+  def : Pat<(X86Movddup (v4f64 (scalar_to_vector (loadf64 addr:$src)))),
+            (VMOVDDUPYrm addr:$src)>;
+  def : Pat<(X86Movddup (v4i64 (scalar_to_vector (loadi64 addr:$src)))),
+            (VMOVDDUPYrm addr:$src)>;
+  def : Pat<(X86Movddup (v4f64 VR256:$src)),
+            (VMOVDDUPYrr VR256:$src)>;
+  def : Pat<(X86Movddup (v4i64 VR256:$src)),
+            (VMOVDDUPYrr VR256:$src)>;
+}
+
+//===---------------------------------------------------------------------===//
+// SSE3 - Move Unaligned Integer
+//===---------------------------------------------------------------------===//
 
-// Move Unaligned Integer
 let Predicates = [HasAVX] in {
   def VLDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                    "vlddqu\t{$src, $dst|$dst, $src}",
@@ -3267,38 +4670,6 @@ def LDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                    "lddqu\t{$src, $dst|$dst, $src}",
                    [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))]>;
 
-def : Pat<(movddup (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src)))),
-                   (undef)),
-          (MOVDDUPrm addr:$src)>, Requires<[HasSSE3]>;
-
-// Several Move patterns
-let AddedComplexity = 5 in {
-def : Pat<(movddup (memopv2f64 addr:$src), (undef)),
-          (MOVDDUPrm addr:$src)>, Requires<[HasSSE3]>;
-def : Pat<(movddup (bc_v4f32 (memopv2f64 addr:$src)), (undef)),
-          (MOVDDUPrm addr:$src)>, Requires<[HasSSE3]>;
-def : Pat<(movddup (memopv2i64 addr:$src), (undef)),
-          (MOVDDUPrm addr:$src)>, Requires<[HasSSE3]>;
-def : Pat<(movddup (bc_v4i32 (memopv2i64 addr:$src)), (undef)),
-          (MOVDDUPrm addr:$src)>, Requires<[HasSSE3]>;
-}
-
-// vector_shuffle v1, <undef> <1, 1, 3, 3>
-let AddedComplexity = 15 in
-def : Pat<(v4i32 (movshdup VR128:$src, (undef))),
-          (MOVSHDUPrr VR128:$src)>, Requires<[HasSSE3]>;
-let AddedComplexity = 20 in
-def : Pat<(v4i32 (movshdup (bc_v4i32 (memopv2i64 addr:$src)), (undef))),
-          (MOVSHDUPrm addr:$src)>, Requires<[HasSSE3]>;
-
-// vector_shuffle v1, <undef> <0, 0, 2, 2>
-let AddedComplexity = 15 in
-  def : Pat<(v4i32 (movsldup VR128:$src, (undef))),
-            (MOVSLDUPrr VR128:$src)>, Requires<[HasSSE3]>;
-let AddedComplexity = 20 in
-  def : Pat<(v4i32 (movsldup (bc_v4i32 (memopv2i64 addr:$src)), (undef))),
-            (MOVSLDUPrm addr:$src)>, Requires<[HasSSE3]>;
-
 //===---------------------------------------------------------------------===//
 // SSE3 - Arithmetic
 //===---------------------------------------------------------------------===//
@@ -3344,62 +4715,58 @@ let Constraints = "$src1 = $dst", Predicates = [HasSSE3],
 
 // Horizontal ops
 multiclass S3D_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC,
-                   X86MemOperand x86memop, Intrinsic IntId, bit Is2Addr = 1> {
+                   X86MemOperand x86memop, SDNode OpNode, bit Is2Addr = 1> {
   def rr : S3DI<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-      [(set RC:$dst, (vt (IntId RC:$src1, RC:$src2)))]>;
+      [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))]>;
 
   def rm : S3DI<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-      [(set RC:$dst, (vt (IntId RC:$src1, (memop addr:$src2))))]>;
+      [(set RC:$dst, (vt (OpNode RC:$src1, (memop addr:$src2))))]>;
 }
 multiclass S3_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC,
-                  X86MemOperand x86memop, Intrinsic IntId, bit Is2Addr = 1> {
+                  X86MemOperand x86memop, SDNode OpNode, bit Is2Addr = 1> {
   def rr : S3I<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-      [(set RC:$dst, (vt (IntId RC:$src1, RC:$src2)))]>;
+      [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))]>;
 
   def rm : S3I<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-      [(set RC:$dst, (vt (IntId RC:$src1, (memop addr:$src2))))]>;
+      [(set RC:$dst, (vt (OpNode RC:$src1, (memop addr:$src2))))]>;
 }
 
 let Predicates = [HasAVX] in {
   defm VHADDPS  : S3D_Int<0x7C, "vhaddps", v4f32, VR128, f128mem,
-                          int_x86_sse3_hadd_ps, 0>, VEX_4V;
+                          X86fhadd, 0>, VEX_4V;
   defm VHADDPD  : S3_Int <0x7C, "vhaddpd", v2f64, VR128, f128mem,
-                          int_x86_sse3_hadd_pd, 0>, VEX_4V;
+                          X86fhadd, 0>, VEX_4V;
   defm VHSUBPS  : S3D_Int<0x7D, "vhsubps", v4f32, VR128, f128mem,
-                          int_x86_sse3_hsub_ps, 0>, VEX_4V;
+                          X86fhsub, 0>, VEX_4V;
   defm VHSUBPD  : S3_Int <0x7D, "vhsubpd", v2f64, VR128, f128mem,
-                          int_x86_sse3_hsub_pd, 0>, VEX_4V;
+                          X86fhsub, 0>, VEX_4V;
   defm VHADDPSY : S3D_Int<0x7C, "vhaddps", v8f32, VR256, f256mem,
-                          int_x86_avx_hadd_ps_256, 0>, VEX_4V;
+                          X86fhadd, 0>, VEX_4V;
   defm VHADDPDY : S3_Int <0x7C, "vhaddpd", v4f64, VR256, f256mem,
-                          int_x86_avx_hadd_pd_256, 0>, VEX_4V;
+                          X86fhadd, 0>, VEX_4V;
   defm VHSUBPSY : S3D_Int<0x7D, "vhsubps", v8f32, VR256, f256mem,
-                          int_x86_avx_hsub_ps_256, 0>, VEX_4V;
+                          X86fhsub, 0>, VEX_4V;
   defm VHSUBPDY : S3_Int <0x7D, "vhsubpd", v4f64, VR256, f256mem,
-                          int_x86_avx_hsub_pd_256, 0>, VEX_4V;
+                          X86fhsub, 0>, VEX_4V;
 }
 
 let Constraints = "$src1 = $dst" in {
-  defm HADDPS : S3D_Int<0x7C, "haddps", v4f32, VR128, f128mem,
-                        int_x86_sse3_hadd_ps>;
-  defm HADDPD : S3_Int<0x7C, "haddpd", v2f64, VR128, f128mem,
-                       int_x86_sse3_hadd_pd>;
-  defm HSUBPS : S3D_Int<0x7D, "hsubps", v4f32, VR128, f128mem,
-                        int_x86_sse3_hsub_ps>;
-  defm HSUBPD : S3_Int<0x7D, "hsubpd", v2f64, VR128, f128mem,
-                       int_x86_sse3_hsub_pd>;
+  defm HADDPS : S3D_Int<0x7C, "haddps", v4f32, VR128, f128mem, X86fhadd>;
+  defm HADDPD : S3_Int<0x7C, "haddpd", v2f64, VR128, f128mem, X86fhadd>;
+  defm HSUBPS : S3D_Int<0x7D, "hsubps", v4f32, VR128, f128mem, X86fhsub>;
+  defm HSUBPD : S3_Int<0x7D, "hsubpd", v2f64, VR128, f128mem, X86fhsub>;
 }
 
 //===---------------------------------------------------------------------===//
@@ -3466,7 +4833,7 @@ multiclass SS3I_binop_rm_int<bits<8> opc, string OpcodeStr,
           (bitconvert (memopv16i8 addr:$src2))))]>, OpSize;
 }
 
-let Predicates = [HasAVX] in {
+let ImmT = NoImm, Predicates = [HasAVX] in {
 let isCommutable = 0 in {
   defm VPHADDW    : SS3I_binop_rm_int<0x01, "vphaddw", memopv8i16,
                                       int_x86_ssse3_phadd_w_128, 0>, VEX_4V;
@@ -3525,17 +4892,33 @@ defm PMULHRSW    : SS3I_binop_rm_int<0x0B, "pmulhrsw", memopv8i16,
                                      int_x86_ssse3_pmul_hr_sw_128>;
 }
 
-def : Pat<(X86pshufb VR128:$src, VR128:$mask),
-          (PSHUFBrr128 VR128:$src, VR128:$mask)>, Requires<[HasSSSE3]>;
-def : Pat<(X86pshufb VR128:$src, (bc_v16i8 (memopv2i64 addr:$mask))),
-          (PSHUFBrm128 VR128:$src, addr:$mask)>, Requires<[HasSSSE3]>;
+let Predicates = [HasSSSE3] in {
+  def : Pat<(X86pshufb VR128:$src, VR128:$mask),
+            (PSHUFBrr128 VR128:$src, VR128:$mask)>;
+  def : Pat<(X86pshufb VR128:$src, (bc_v16i8 (memopv2i64 addr:$mask))),
+            (PSHUFBrm128 VR128:$src, addr:$mask)>;
 
-def : Pat<(X86psignb VR128:$src1, VR128:$src2),
-          (PSIGNBrr128 VR128:$src1, VR128:$src2)>, Requires<[HasSSSE3]>;
-def : Pat<(X86psignw VR128:$src1, VR128:$src2),
-          (PSIGNWrr128 VR128:$src1, VR128:$src2)>, Requires<[HasSSSE3]>;
-def : Pat<(X86psignd VR128:$src1, VR128:$src2),
-          (PSIGNDrr128 VR128:$src1, VR128:$src2)>, Requires<[HasSSSE3]>;
+  def : Pat<(X86psignb VR128:$src1, VR128:$src2),
+            (PSIGNBrr128 VR128:$src1, VR128:$src2)>;
+  def : Pat<(X86psignw VR128:$src1, VR128:$src2),
+            (PSIGNWrr128 VR128:$src1, VR128:$src2)>;
+  def : Pat<(X86psignd VR128:$src1, VR128:$src2),
+            (PSIGNDrr128 VR128:$src1, VR128:$src2)>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(X86pshufb VR128:$src, VR128:$mask),
+            (VPSHUFBrr128 VR128:$src, VR128:$mask)>;
+  def : Pat<(X86pshufb VR128:$src, (bc_v16i8 (memopv2i64 addr:$mask))),
+            (VPSHUFBrm128 VR128:$src, addr:$mask)>;
+
+  def : Pat<(X86psignb VR128:$src1, VR128:$src2),
+            (VPSIGNBrr128 VR128:$src1, VR128:$src2)>;
+  def : Pat<(X86psignw VR128:$src1, VR128:$src2),
+            (VPSIGNWrr128 VR128:$src1, VR128:$src2)>;
+  def : Pat<(X86psignd VR128:$src1, VR128:$src2),
+            (VPSIGNDrr128 VR128:$src1, VR128:$src2)>;
+}
 
 //===---------------------------------------------------------------------===//
 // SSSE3 - Packed Align Instruction Patterns
@@ -3560,33 +4943,35 @@ multiclass ssse3_palign<string asm, bit Is2Addr = 1> {
 
 let Predicates = [HasAVX] in
   defm VPALIGN : ssse3_palign<"vpalignr", 0>, VEX_4V;
-let Constraints = "$src1 = $dst" in
+let Constraints = "$src1 = $dst", Predicates = [HasSSSE3] in
   defm PALIGN : ssse3_palign<"palignr">;
 
-let AddedComplexity = 5 in {
-def : Pat<(v4i32 (palign:$src3 VR128:$src1, VR128:$src2)),
-          (PALIGNR128rr VR128:$src2, VR128:$src1,
-                        (SHUFFLE_get_palign_imm VR128:$src3))>,
-      Requires<[HasSSSE3]>;
-def : Pat<(v4f32 (palign:$src3 VR128:$src1, VR128:$src2)),
-          (PALIGNR128rr VR128:$src2, VR128:$src1,
-                        (SHUFFLE_get_palign_imm VR128:$src3))>,
-      Requires<[HasSSSE3]>;
-def : Pat<(v8i16 (palign:$src3 VR128:$src1, VR128:$src2)),
-          (PALIGNR128rr VR128:$src2, VR128:$src1,
-                        (SHUFFLE_get_palign_imm VR128:$src3))>,
-      Requires<[HasSSSE3]>;
-def : Pat<(v16i8 (palign:$src3 VR128:$src1, VR128:$src2)),
-          (PALIGNR128rr VR128:$src2, VR128:$src1,
-                        (SHUFFLE_get_palign_imm VR128:$src3))>,
-      Requires<[HasSSSE3]>;
+let Predicates = [HasSSSE3] in {
+def : Pat<(v4i32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+def : Pat<(v4f32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+def : Pat<(v8i16 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+def : Pat<(v16i8 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+}
+
+let Predicates = [HasAVX] in {
+def : Pat<(v4i32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+def : Pat<(v4f32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+def : Pat<(v8i16 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
+def : Pat<(v16i8 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
+          (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
 }
 
 //===---------------------------------------------------------------------===//
-// SSSE3 Misc Instructions
+// SSSE3 - Thread synchronization
 //===---------------------------------------------------------------------===//
 
-// Thread synchronization
 let usesCustomInserter = 1 in {
 def MONITOR : PseudoI<(outs), (ins i32mem:$src1, GR32:$src2, GR32:$src3),
                 [(int_x86_sse3_monitor addr:$src1, GR32:$src2, GR32:$src3)]>;
@@ -3609,338 +4994,6 @@ def : InstAlias<"monitor %eax, %ecx, %edx", (MONITORrrr)>,
 def : InstAlias<"monitor %rax, %rcx, %rdx", (MONITORrrr)>,
       Requires<[In64BitMode]>;
 
-//===---------------------------------------------------------------------===//
-// Non-Instruction Patterns
-//===---------------------------------------------------------------------===//
-
-// extload f32 -> f64.  This matches load+fextend because we have a hack in
-// the isel (PreprocessForFPConvert) that can introduce loads after dag
-// combine.
-// Since these loads aren't folded into the fextend, we have to match it
-// explicitly here.
-let Predicates = [HasSSE2] in
- def : Pat<(fextend (loadf32 addr:$src)),
-           (CVTSS2SDrm addr:$src)>;
-
-// FIXME: According to the intel manual, DEST[127:64] <- SRC1[127:64], while
-// in the non-AVX version bits 127:64 aren't touched. Find a better way to
-// represent this instead of always zeroing SRC1. One possible solution is
-// to represent the instruction w/ something similar as the "$src1 = $dst"
-// constraint but without the tied operands.
-let Predicates = [HasAVX] in
- def : Pat<(fextend (loadf32 addr:$src)),
-           (VCVTSS2SDrm (f32 (EXTRACT_SUBREG (AVX_SET0PS), sub_ss)),
-                        addr:$src)>;
-
-// bit_convert
-let Predicates = [HasXMMInt] in {
-  def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>;
-  def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>;
-  def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>;
-  def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>;
-  def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>;
-  def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>;
-  def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>;
-  def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>;
-  def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>;
-  def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>;
-  def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>;
-  def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>;
-  def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>;
-  def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>;
-  def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>;
-  def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>;
-  def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>;
-  def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>;
-  def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>;
-  def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>;
-  def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>;
-  def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>;
-  def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>;
-  def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>;
-  def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>;
-  def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>;
-  def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>;
-  def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>;
-  def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>;
-  def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>;
-}
-
-let Predicates = [HasAVX] in {
-  def : Pat<(v4f64 (bitconvert (v8f32 VR256:$src))), (v4f64 VR256:$src)>;
-  def : Pat<(v4f64 (bitconvert (v4i64 VR256:$src))), (v4f64 VR256:$src)>;
-  def : Pat<(v4f64 (bitconvert (v32i8 VR256:$src))), (v4f64 VR256:$src)>;
-  def : Pat<(v8f32 (bitconvert (v4i64 VR256:$src))), (v8f32 VR256:$src)>;
-  def : Pat<(v8f32 (bitconvert (v4f64 VR256:$src))), (v8f32 VR256:$src)>;
-  def : Pat<(v8f32 (bitconvert (v32i8 VR256:$src))), (v8f32 VR256:$src)>;
-  def : Pat<(v4i64 (bitconvert (v8f32 VR256:$src))), (v4i64 VR256:$src)>;
-  def : Pat<(v4i64 (bitconvert (v4f64 VR256:$src))), (v4i64 VR256:$src)>;
-  def : Pat<(v4i64 (bitconvert (v32i8 VR256:$src))), (v4i64 VR256:$src)>;
-  def : Pat<(v32i8 (bitconvert (v4f64 VR256:$src))), (v32i8 VR256:$src)>;
-  def : Pat<(v32i8 (bitconvert (v4i64 VR256:$src))), (v32i8 VR256:$src)>;
-  def : Pat<(v32i8 (bitconvert (v8f32 VR256:$src))), (v32i8 VR256:$src)>;
-  def : Pat<(v32i8 (bitconvert (v8i32 VR256:$src))), (v32i8 VR256:$src)>;
-  def : Pat<(v8i32 (bitconvert (v32i8 VR256:$src))), (v8i32 VR256:$src)>;
-}
-
-// Move scalar to XMM zero-extended
-// movd to XMM register zero-extends
-let AddedComplexity = 15 in {
-// Zeroing a VR128 then do a MOVS{S|D} to the lower bits.
-def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))),
-          (MOVSDrr (v2f64 (V_SET0PS)), FR64:$src)>;
-def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))),
-          (MOVSSrr (v4f32 (V_SET0PS)), FR32:$src)>;
-def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),
-          (MOVSSrr (v4f32 (V_SET0PS)),
-                   (f32 (EXTRACT_SUBREG (v4f32 VR128:$src), sub_ss)))>;
-def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),
-          (MOVSSrr (v4i32 (V_SET0PI)),
-                   (EXTRACT_SUBREG (v4i32 VR128:$src), sub_ss))>;
-}
-
-// Splat v2f64 / v2i64
-let AddedComplexity = 10 in {
-def : Pat<(splat_lo (v2f64 VR128:$src), (undef)),
-          (UNPCKLPDrr VR128:$src, VR128:$src)>,   Requires<[HasSSE2]>;
-def : Pat<(unpckh (v2f64 VR128:$src), (undef)),
-          (UNPCKHPDrr VR128:$src, VR128:$src)>,   Requires<[HasSSE2]>;
-def : Pat<(splat_lo (v2i64 VR128:$src), (undef)),
-          (PUNPCKLQDQrr VR128:$src, VR128:$src)>, Requires<[HasSSE2]>;
-def : Pat<(unpckh (v2i64 VR128:$src), (undef)),
-          (PUNPCKHQDQrr VR128:$src, VR128:$src)>, Requires<[HasSSE2]>;
-}
-
-// Special unary SHUFPSrri case.
-def : Pat<(v4f32 (pshufd:$src3 VR128:$src1, (undef))),
-          (SHUFPSrri VR128:$src1, VR128:$src1,
-                     (SHUFFLE_get_shuf_imm VR128:$src3))>;
-let AddedComplexity = 5 in
-def : Pat<(v4f32 (pshufd:$src2 VR128:$src1, (undef))),
-          (PSHUFDri VR128:$src1, (SHUFFLE_get_shuf_imm VR128:$src2))>,
-      Requires<[HasSSE2]>;
-// Special unary SHUFPDrri case.
-def : Pat<(v2i64 (pshufd:$src3 VR128:$src1, (undef))),
-          (SHUFPDrri VR128:$src1, VR128:$src1,
-                     (SHUFFLE_get_shuf_imm VR128:$src3))>,
-      Requires<[HasSSE2]>;
-// Special unary SHUFPDrri case.
-def : Pat<(v2f64 (pshufd:$src3 VR128:$src1, (undef))),
-          (SHUFPDrri VR128:$src1, VR128:$src1,
-                     (SHUFFLE_get_shuf_imm VR128:$src3))>,
-      Requires<[HasSSE2]>;
-// Unary v4f32 shuffle with PSHUF* in order to fold a load.
-def : Pat<(pshufd:$src2 (bc_v4i32 (memopv4f32 addr:$src1)), (undef)),
-          (PSHUFDmi addr:$src1, (SHUFFLE_get_shuf_imm VR128:$src2))>,
-      Requires<[HasSSE2]>;
-
-// Special binary v4i32 shuffle cases with SHUFPS.
-def : Pat<(v4i32 (shufp:$src3 VR128:$src1, (v4i32 VR128:$src2))),
-          (SHUFPSrri VR128:$src1, VR128:$src2,
-                     (SHUFFLE_get_shuf_imm VR128:$src3))>,
-           Requires<[HasSSE2]>;
-def : Pat<(v4i32 (shufp:$src3 VR128:$src1, (bc_v4i32 (memopv2i64 addr:$src2)))),
-          (SHUFPSrmi VR128:$src1, addr:$src2,
-                    (SHUFFLE_get_shuf_imm VR128:$src3))>,
-           Requires<[HasSSE2]>;
-// Special binary v2i64 shuffle cases using SHUFPDrri.
-def : Pat<(v2i64 (shufp:$src3 VR128:$src1, VR128:$src2)),
-          (SHUFPDrri VR128:$src1, VR128:$src2,
-                     (SHUFFLE_get_shuf_imm VR128:$src3))>,
-          Requires<[HasSSE2]>;
-
-// vector_shuffle v1, <undef>, <0, 0, 1, 1, ...>
-let AddedComplexity = 15 in {
-def : Pat<(v4i32 (unpckl_undef:$src2 VR128:$src, (undef))),
-          (PSHUFDri VR128:$src, (SHUFFLE_get_shuf_imm VR128:$src2))>,
-          Requires<[OptForSpeed, HasSSE2]>;
-def : Pat<(v4f32 (unpckl_undef:$src2 VR128:$src, (undef))),
-          (PSHUFDri VR128:$src, (SHUFFLE_get_shuf_imm VR128:$src2))>,
-          Requires<[OptForSpeed, HasSSE2]>;
-}
-let AddedComplexity = 10 in {
-def : Pat<(v4f32 (unpckl_undef VR128:$src, (undef))),
-          (UNPCKLPSrr VR128:$src, VR128:$src)>;
-def : Pat<(v16i8 (unpckl_undef VR128:$src, (undef))),
-          (PUNPCKLBWrr VR128:$src, VR128:$src)>;
-def : Pat<(v8i16 (unpckl_undef VR128:$src, (undef))),
-          (PUNPCKLWDrr VR128:$src, VR128:$src)>;
-def : Pat<(v4i32 (unpckl_undef VR128:$src, (undef))),
-          (PUNPCKLDQrr VR128:$src, VR128:$src)>;
-}
-
-// vector_shuffle v1, <undef>, <2, 2, 3, 3, ...>
-let AddedComplexity = 15 in {
-def : Pat<(v4i32 (unpckh_undef:$src2 VR128:$src, (undef))),
-          (PSHUFDri VR128:$src, (SHUFFLE_get_shuf_imm VR128:$src2))>,
-          Requires<[OptForSpeed, HasSSE2]>;
-def : Pat<(v4f32 (unpckh_undef:$src2 VR128:$src, (undef))),
-          (PSHUFDri VR128:$src, (SHUFFLE_get_shuf_imm VR128:$src2))>,
-          Requires<[OptForSpeed, HasSSE2]>;
-}
-let AddedComplexity = 10 in {
-def : Pat<(v4f32 (unpckh_undef VR128:$src, (undef))),
-          (UNPCKHPSrr VR128:$src, VR128:$src)>;
-def : Pat<(v16i8 (unpckh_undef VR128:$src, (undef))),
-          (PUNPCKHBWrr VR128:$src, VR128:$src)>;
-def : Pat<(v8i16 (unpckh_undef VR128:$src, (undef))),
-          (PUNPCKHWDrr VR128:$src, VR128:$src)>;
-def : Pat<(v4i32 (unpckh_undef VR128:$src, (undef))),
-          (PUNPCKHDQrr VR128:$src, VR128:$src)>;
-}
-
-let AddedComplexity = 20 in {
-// vector_shuffle v1, v2 <0, 1, 4, 5> using MOVLHPS
-def : Pat<(v4i32 (movlhps VR128:$src1, VR128:$src2)),
-          (MOVLHPSrr VR128:$src1, VR128:$src2)>;
-
-// vector_shuffle v1, v2 <6, 7, 2, 3> using MOVHLPS
-def : Pat<(v4i32 (movhlps VR128:$src1, VR128:$src2)),
-          (MOVHLPSrr VR128:$src1, VR128:$src2)>;
-
-// vector_shuffle v1, undef <2, ?, ?, ?> using MOVHLPS
-def : Pat<(v4f32 (movhlps_undef VR128:$src1, (undef))),
-          (MOVHLPSrr VR128:$src1, VR128:$src1)>;
-def : Pat<(v4i32 (movhlps_undef VR128:$src1, (undef))),
-          (MOVHLPSrr VR128:$src1, VR128:$src1)>;
-}
-
-let AddedComplexity = 20 in {
-// vector_shuffle v1, (load v2) <4, 5, 2, 3> using MOVLPS
-def : Pat<(v4f32 (movlp VR128:$src1, (load addr:$src2))),
-          (MOVLPSrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2f64 (movlp VR128:$src1, (load addr:$src2))),
-          (MOVLPDrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4i32 (movlp VR128:$src1, (load addr:$src2))),
-          (MOVLPSrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (movlp VR128:$src1, (load addr:$src2))),
-          (MOVLPDrm VR128:$src1, addr:$src2)>;
-}
-
-// (store (vector_shuffle (load addr), v2, <4, 5, 2, 3>), addr) using MOVLPS
-def : Pat<(store (v4f32 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
-          (MOVLPSmr addr:$src1, VR128:$src2)>;
-def : Pat<(store (v2f64 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
-          (MOVLPDmr addr:$src1, VR128:$src2)>;
-def : Pat<(store (v4i32 (movlp (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)),
-                 addr:$src1),
-          (MOVLPSmr addr:$src1, VR128:$src2)>;
-def : Pat<(store (v2i64 (movlp (load addr:$src1), VR128:$src2)), addr:$src1),
-          (MOVLPDmr addr:$src1, VR128:$src2)>;
-
-let AddedComplexity = 15 in {
-// Setting the lowest element in the vector.
-def : Pat<(v4i32 (movl VR128:$src1, VR128:$src2)),
-          (MOVSSrr (v4i32 VR128:$src1),
-                   (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_ss))>;
-def : Pat<(v2i64 (movl VR128:$src1, VR128:$src2)),
-          (MOVSDrr (v2i64 VR128:$src1),
-                   (EXTRACT_SUBREG (v2i64 VR128:$src2), sub_sd))>;
-
-// vector_shuffle v1, v2 <4, 5, 2, 3> using movsd
-def : Pat<(v4f32 (movlp VR128:$src1, VR128:$src2)),
-          (MOVSDrr VR128:$src1, (EXTRACT_SUBREG VR128:$src2, sub_sd))>,
-      Requires<[HasSSE2]>;
-def : Pat<(v4i32 (movlp VR128:$src1, VR128:$src2)),
-          (MOVSDrr VR128:$src1, (EXTRACT_SUBREG VR128:$src2, sub_sd))>,
-      Requires<[HasSSE2]>;
-}
-
-// vector_shuffle v1, v2 <4, 5, 2, 3> using SHUFPSrri (we prefer movsd, but
-// fall back to this for SSE1)
-def : Pat<(v4f32 (movlp:$src3 VR128:$src1, (v4f32 VR128:$src2))),
-          (SHUFPSrri VR128:$src2, VR128:$src1,
-                     (SHUFFLE_get_shuf_imm VR128:$src3))>;
-
-// Set lowest element and zero upper elements.
-def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),
-          (MOVZPQILo2PQIrr VR128:$src)>, Requires<[HasSSE2]>;
-
-// vector -> vector casts
-def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))),
-          (Int_CVTDQ2PSrr VR128:$src)>, Requires<[HasSSE2]>;
-def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))),
-          (CVTTPS2DQrr VR128:$src)>, Requires<[HasSSE2]>;
-
-// Use movaps / movups for SSE integer load / store (one byte shorter).
-// The instructions selected below are then converted to MOVDQA/MOVDQU
-// during the SSE domain pass.
-let Predicates = [HasSSE1] in {
-  def : Pat<(alignedloadv4i32 addr:$src),
-            (MOVAPSrm addr:$src)>;
-  def : Pat<(loadv4i32 addr:$src),
-            (MOVUPSrm addr:$src)>;
-  def : Pat<(alignedloadv2i64 addr:$src),
-            (MOVAPSrm addr:$src)>;
-  def : Pat<(loadv2i64 addr:$src),
-            (MOVUPSrm addr:$src)>;
-
-  def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst),
-            (MOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst),
-            (MOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst),
-            (MOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst),
-            (MOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v2i64 VR128:$src), addr:$dst),
-            (MOVUPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v4i32 VR128:$src), addr:$dst),
-            (MOVUPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v8i16 VR128:$src), addr:$dst),
-            (MOVUPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v16i8 VR128:$src), addr:$dst),
-            (MOVUPSmr addr:$dst, VR128:$src)>;
-}
-
-// Use vmovaps/vmovups for AVX integer load/store.
-let Predicates = [HasAVX] in {
-  // 128-bit load/store
-  def : Pat<(alignedloadv4i32 addr:$src),
-            (VMOVAPSrm addr:$src)>;
-  def : Pat<(loadv4i32 addr:$src),
-            (VMOVUPSrm addr:$src)>;
-  def : Pat<(alignedloadv2i64 addr:$src),
-            (VMOVAPSrm addr:$src)>;
-  def : Pat<(loadv2i64 addr:$src),
-            (VMOVUPSrm addr:$src)>;
-
-  def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst),
-            (VMOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst),
-            (VMOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst),
-            (VMOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst),
-            (VMOVAPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v2i64 VR128:$src), addr:$dst),
-            (VMOVUPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v4i32 VR128:$src), addr:$dst),
-            (VMOVUPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v8i16 VR128:$src), addr:$dst),
-            (VMOVUPSmr addr:$dst, VR128:$src)>;
-  def : Pat<(store (v16i8 VR128:$src), addr:$dst),
-            (VMOVUPSmr addr:$dst, VR128:$src)>;
-
-  // 256-bit load/store
-  def : Pat<(alignedloadv4i64 addr:$src),
-            (VMOVAPSYrm addr:$src)>;
-  def : Pat<(loadv4i64 addr:$src),
-            (VMOVUPSYrm addr:$src)>;
-  def : Pat<(alignedloadv8i32 addr:$src),
-            (VMOVAPSYrm addr:$src)>;
-  def : Pat<(loadv8i32 addr:$src),
-            (VMOVUPSYrm addr:$src)>;
-  def : Pat<(alignedstore (v4i64 VR256:$src), addr:$dst),
-            (VMOVAPSYmr addr:$dst, VR256:$src)>;
-  def : Pat<(alignedstore (v8i32 VR256:$src), addr:$dst),
-            (VMOVAPSYmr addr:$dst, VR256:$src)>;
-  def : Pat<(store (v4i64 VR256:$src), addr:$dst),
-            (VMOVUPSYmr addr:$dst, VR256:$src)>;
-  def : Pat<(store (v8i32 VR256:$src), addr:$dst),
-            (VMOVUPSYmr addr:$dst, VR256:$src)>;
-}
-
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Packed Move with Sign/Zero Extend
 //===----------------------------------------------------------------------===//
@@ -3979,36 +5032,71 @@ defm PMOVZXBW   : SS41I_binop_rm_int8<0x30, "pmovzxbw", int_x86_sse41_pmovzxbw>;
 defm PMOVZXWD   : SS41I_binop_rm_int8<0x33, "pmovzxwd", int_x86_sse41_pmovzxwd>;
 defm PMOVZXDQ   : SS41I_binop_rm_int8<0x35, "pmovzxdq", int_x86_sse41_pmovzxdq>;
 
-// Common patterns involving scalar load.
-def : Pat<(int_x86_sse41_pmovsxbw (vzmovl_v2i64 addr:$src)),
-          (PMOVSXBWrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovsxbw (vzload_v2i64 addr:$src)),
-          (PMOVSXBWrm addr:$src)>, Requires<[HasSSE41]>;
+let Predicates = [HasSSE41] in {
+  // Common patterns involving scalar load.
+  def : Pat<(int_x86_sse41_pmovsxbw (vzmovl_v2i64 addr:$src)),
+            (PMOVSXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxbw (vzload_v2i64 addr:$src)),
+            (PMOVSXBWrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovsxwd (vzmovl_v2i64 addr:$src)),
-          (PMOVSXWDrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovsxwd (vzload_v2i64 addr:$src)),
-          (PMOVSXWDrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovsxwd (vzmovl_v2i64 addr:$src)),
+            (PMOVSXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxwd (vzload_v2i64 addr:$src)),
+            (PMOVSXWDrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovsxdq (vzmovl_v2i64 addr:$src)),
-          (PMOVSXDQrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovsxdq (vzload_v2i64 addr:$src)),
-          (PMOVSXDQrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovsxdq (vzmovl_v2i64 addr:$src)),
+            (PMOVSXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxdq (vzload_v2i64 addr:$src)),
+            (PMOVSXDQrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovzxbw (vzmovl_v2i64 addr:$src)),
-          (PMOVZXBWrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovzxbw (vzload_v2i64 addr:$src)),
-          (PMOVZXBWrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovzxbw (vzmovl_v2i64 addr:$src)),
+            (PMOVZXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxbw (vzload_v2i64 addr:$src)),
+            (PMOVZXBWrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovzxwd (vzmovl_v2i64 addr:$src)),
-          (PMOVZXWDrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovzxwd (vzload_v2i64 addr:$src)),
-          (PMOVZXWDrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovzxwd (vzmovl_v2i64 addr:$src)),
+            (PMOVZXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxwd (vzload_v2i64 addr:$src)),
+            (PMOVZXWDrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovzxdq (vzmovl_v2i64 addr:$src)),
-          (PMOVZXDQrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovzxdq (vzload_v2i64 addr:$src)),
-          (PMOVZXDQrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovzxdq (vzmovl_v2i64 addr:$src)),
+            (PMOVZXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxdq (vzload_v2i64 addr:$src)),
+            (PMOVZXDQrm addr:$src)>;
+}
+
+let Predicates = [HasAVX] in {
+  // Common patterns involving scalar load.
+  def : Pat<(int_x86_sse41_pmovsxbw (vzmovl_v2i64 addr:$src)),
+            (VPMOVSXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxbw (vzload_v2i64 addr:$src)),
+            (VPMOVSXBWrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovsxwd (vzmovl_v2i64 addr:$src)),
+            (VPMOVSXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxwd (vzload_v2i64 addr:$src)),
+            (VPMOVSXWDrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovsxdq (vzmovl_v2i64 addr:$src)),
+            (VPMOVSXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxdq (vzload_v2i64 addr:$src)),
+            (VPMOVSXDQrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovzxbw (vzmovl_v2i64 addr:$src)),
+            (VPMOVZXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxbw (vzload_v2i64 addr:$src)),
+            (VPMOVZXBWrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovzxwd (vzmovl_v2i64 addr:$src)),
+            (VPMOVZXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxwd (vzload_v2i64 addr:$src)),
+            (VPMOVZXWDrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovzxdq (vzmovl_v2i64 addr:$src)),
+            (VPMOVZXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxdq (vzload_v2i64 addr:$src)),
+            (VPMOVZXDQrm addr:$src)>;
+}
 
 
 multiclass SS41I_binop_rm_int4<bits<8> opc, string OpcodeStr, Intrinsic IntId> {
@@ -4039,17 +5127,31 @@ defm PMOVSXWQ   : SS41I_binop_rm_int4<0x24, "pmovsxwq", int_x86_sse41_pmovsxwq>;
 defm PMOVZXBD   : SS41I_binop_rm_int4<0x31, "pmovzxbd", int_x86_sse41_pmovzxbd>;
 defm PMOVZXWQ   : SS41I_binop_rm_int4<0x34, "pmovzxwq", int_x86_sse41_pmovzxwq>;
 
-// Common patterns involving scalar load
-def : Pat<(int_x86_sse41_pmovsxbd (vzmovl_v4i32 addr:$src)),
-          (PMOVSXBDrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovsxwq (vzmovl_v4i32 addr:$src)),
-          (PMOVSXWQrm addr:$src)>, Requires<[HasSSE41]>;
+let Predicates = [HasSSE41] in {
+  // Common patterns involving scalar load
+  def : Pat<(int_x86_sse41_pmovsxbd (vzmovl_v4i32 addr:$src)),
+            (PMOVSXBDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxwq (vzmovl_v4i32 addr:$src)),
+            (PMOVSXWQrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovzxbd (vzmovl_v4i32 addr:$src)),
-          (PMOVZXBDrm addr:$src)>, Requires<[HasSSE41]>;
-def : Pat<(int_x86_sse41_pmovzxwq (vzmovl_v4i32 addr:$src)),
-          (PMOVZXWQrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovzxbd (vzmovl_v4i32 addr:$src)),
+            (PMOVZXBDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxwq (vzmovl_v4i32 addr:$src)),
+            (PMOVZXWQrm addr:$src)>;
+}
 
+let Predicates = [HasAVX] in {
+  // Common patterns involving scalar load
+  def : Pat<(int_x86_sse41_pmovsxbd (vzmovl_v4i32 addr:$src)),
+            (VPMOVSXBDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxwq (vzmovl_v4i32 addr:$src)),
+            (VPMOVSXWQrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovzxbd (vzmovl_v4i32 addr:$src)),
+            (VPMOVZXBDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxwq (vzmovl_v4i32 addr:$src)),
+            (VPMOVZXWQrm addr:$src)>;
+}
 
 multiclass SS41I_binop_rm_int2<bits<8> opc, string OpcodeStr, Intrinsic IntId> {
   def rr : SS48I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
@@ -4073,16 +5175,31 @@ defm VPMOVZXBQ : SS41I_binop_rm_int2<0x32, "vpmovzxbq", int_x86_sse41_pmovzxbq>,
 defm PMOVSXBQ   : SS41I_binop_rm_int2<0x22, "pmovsxbq", int_x86_sse41_pmovsxbq>;
 defm PMOVZXBQ   : SS41I_binop_rm_int2<0x32, "pmovzxbq", int_x86_sse41_pmovzxbq>;
 
-// Common patterns involving scalar load
-def : Pat<(int_x86_sse41_pmovsxbq
-            (bitconvert (v4i32 (X86vzmovl
-                             (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
-          (PMOVSXBQrm addr:$src)>, Requires<[HasSSE41]>;
+let Predicates = [HasSSE41] in {
+  // Common patterns involving scalar load
+  def : Pat<(int_x86_sse41_pmovsxbq
+              (bitconvert (v4i32 (X86vzmovl
+                            (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (PMOVSXBQrm addr:$src)>;
 
-def : Pat<(int_x86_sse41_pmovzxbq
-            (bitconvert (v4i32 (X86vzmovl
-                             (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
-          (PMOVZXBQrm addr:$src)>, Requires<[HasSSE41]>;
+  def : Pat<(int_x86_sse41_pmovzxbq
+              (bitconvert (v4i32 (X86vzmovl
+                            (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (PMOVZXBQrm addr:$src)>;
+}
+
+let Predicates = [HasAVX] in {
+  // Common patterns involving scalar load
+  def : Pat<(int_x86_sse41_pmovsxbq
+              (bitconvert (v4i32 (X86vzmovl
+                            (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (VPMOVSXBQrm addr:$src)>;
+
+  def : Pat<(int_x86_sse41_pmovzxbq
+              (bitconvert (v4i32 (X86vzmovl
+                            (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (VPMOVZXBQrm addr:$src)>;
+}
 
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Extract Instructions
@@ -4208,7 +5325,12 @@ def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)),
                                               imm:$src2))),
                  addr:$dst),
           (EXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>,
-         Requires<[HasSSE41]>;
+          Requires<[HasSSE41]>;
+def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)),
+                                              imm:$src2))),
+                 addr:$dst),
+          (VEXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>,
+          Requires<[HasAVX]>;
 
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Insert Instructions
@@ -4297,7 +5419,7 @@ let Constraints = "$src1 = $dst" in
 // in the target vector.
 multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1> {
   def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst),
-      (ins VR128:$src1, VR128:$src2, i32i8imm:$src3),
+      (ins VR128:$src1, VR128:$src2, u32u8imm:$src3),
       !if(Is2Addr,
         !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
         !strconcat(asm,
@@ -4306,7 +5428,7 @@ multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1> {
         (X86insrtps VR128:$src1, VR128:$src2, imm:$src3))]>,
       OpSize;
   def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),
-      (ins VR128:$src1, f32mem:$src2, i32i8imm:$src3),
+      (ins VR128:$src1, f32mem:$src2, u32u8imm:$src3),
       !if(Is2Addr,
         !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
         !strconcat(asm,
@@ -4348,7 +5470,7 @@ multiclass sse41_fp_unop_rm<bits<8> opcps, bits<8> opcpd, string OpcodeStr,
 
   // Vector intrinsic operation, mem
   def PSm : Ii8<opcps, MRMSrcMem,
-                    (outs RC:$dst), (ins f256mem:$src1, i32i8imm:$src2),
+                    (outs RC:$dst), (ins x86memop:$src1, i32i8imm:$src2),
                     !strconcat(OpcodeStr,
                     "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                     [(set RC:$dst,
@@ -4366,7 +5488,7 @@ multiclass sse41_fp_unop_rm<bits<8> opcps, bits<8> opcpd, string OpcodeStr,
 
   // Vector intrinsic operation, mem
   def PDm : SS4AIi8<opcpd, MRMSrcMem,
-                    (outs RC:$dst), (ins f256mem:$src1, i32i8imm:$src2),
+                    (outs RC:$dst), (ins x86memop:$src1, i32i8imm:$src2),
                     !strconcat(OpcodeStr,
                     "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                     [(set RC:$dst,
@@ -4501,14 +5623,14 @@ let Predicates = [HasAVX] in {
                                   int_x86_avx_round_pd_256>, VEX;
   defm VROUND  : sse41_fp_binop_rm<0x0A, 0x0B, "vround",
                                   int_x86_sse41_round_ss,
-                                  int_x86_sse41_round_sd, 0>, VEX_4V;
+                                  int_x86_sse41_round_sd, 0>, VEX_4V, VEX_LIG;
 
   // Instructions for the assembler
   defm VROUND  : sse41_fp_unop_rm_avx_p<0x08, 0x09, VR128, f128mem, "vround">,
                                         VEX;
   defm VROUNDY : sse41_fp_unop_rm_avx_p<0x08, 0x09, VR256, f256mem, "vround">,
                                         VEX;
-  defm VROUND  : sse41_fp_binop_rm_avx_s<0x0A, 0x0B, "vround">, VEX_4V;
+  defm VROUND  : sse41_fp_binop_rm_avx_s<0x0A, 0x0B, "vround">, VEX_4V, VEX_LIG;
 }
 
 defm ROUND  : sse41_fp_unop_rm<0x08, 0x09, "round", f128mem, VR128,
@@ -4578,26 +5700,34 @@ defm VTESTPDY : avx_bittest<0x0F, "vtestpd", VR256, f256mem, memopv4f64, v4f64>;
 // SSE4.1 - Misc Instructions
 //===----------------------------------------------------------------------===//
 
-def POPCNT16rr : I<0xB8, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src),
-                   "popcnt{w}\t{$src, $dst|$dst, $src}",
-                   [(set GR16:$dst, (ctpop GR16:$src))]>, OpSize, XS;
-def POPCNT16rm : I<0xB8, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
-                   "popcnt{w}\t{$src, $dst|$dst, $src}",
-                   [(set GR16:$dst, (ctpop (loadi16 addr:$src)))]>, OpSize, XS;
+let Defs = [EFLAGS], Predicates = [HasPOPCNT] in {
+  def POPCNT16rr : I<0xB8, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src),
+                     "popcnt{w}\t{$src, $dst|$dst, $src}",
+                     [(set GR16:$dst, (ctpop GR16:$src)), (implicit EFLAGS)]>,
+                     OpSize, XS;
+  def POPCNT16rm : I<0xB8, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
+                     "popcnt{w}\t{$src, $dst|$dst, $src}",
+                     [(set GR16:$dst, (ctpop (loadi16 addr:$src))),
+                      (implicit EFLAGS)]>, OpSize, XS;
 
-def POPCNT32rr : I<0xB8, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
-                   "popcnt{l}\t{$src, $dst|$dst, $src}",
-                   [(set GR32:$dst, (ctpop GR32:$src))]>, XS;
-def POPCNT32rm : I<0xB8, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
-                   "popcnt{l}\t{$src, $dst|$dst, $src}",
-                   [(set GR32:$dst, (ctpop (loadi32 addr:$src)))]>, XS;
+  def POPCNT32rr : I<0xB8, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
+                     "popcnt{l}\t{$src, $dst|$dst, $src}",
+                     [(set GR32:$dst, (ctpop GR32:$src)), (implicit EFLAGS)]>,
+                     XS;
+  def POPCNT32rm : I<0xB8, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
+                     "popcnt{l}\t{$src, $dst|$dst, $src}",
+                     [(set GR32:$dst, (ctpop (loadi32 addr:$src))),
+                      (implicit EFLAGS)]>, XS;
 
-def POPCNT64rr : RI<0xB8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
-                    "popcnt{q}\t{$src, $dst|$dst, $src}",
-                    [(set GR64:$dst, (ctpop GR64:$src))]>, XS;
-def POPCNT64rm : RI<0xB8, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
-                    "popcnt{q}\t{$src, $dst|$dst, $src}",
-                    [(set GR64:$dst, (ctpop (loadi64 addr:$src)))]>, XS;
+  def POPCNT64rr : RI<0xB8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
+                      "popcnt{q}\t{$src, $dst|$dst, $src}",
+                      [(set GR64:$dst, (ctpop GR64:$src)), (implicit EFLAGS)]>,
+                      XS;
+  def POPCNT64rm : RI<0xB8, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
+                      "popcnt{q}\t{$src, $dst|$dst, $src}",
+                      [(set GR64:$dst, (ctpop (loadi64 addr:$src))),
+                       (implicit EFLAGS)]>, XS;
+}
 
 
 
@@ -4666,6 +5796,11 @@ let Predicates = [HasAVX] in {
                                                          0>, VEX_4V;
   defm VPMULDQ   : SS41I_binop_rm_int<0x28, "vpmuldq",   int_x86_sse41_pmuldq,
                                                          0>, VEX_4V;
+
+  def : Pat<(v2i64 (X86pcmpeqq VR128:$src1, VR128:$src2)),
+            (VPCMPEQQrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (X86pcmpeqq VR128:$src1, (memop addr:$src2))),
+            (VPCMPEQQrm VR128:$src1, addr:$src2)>;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -4720,7 +5855,7 @@ multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr,
                  X86MemOperand x86memop, bit Is2Addr = 1> {
   let isCommutable = 1 in
   def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst),
-        (ins RC:$src1, RC:$src2, i32i8imm:$src3),
+        (ins RC:$src1, RC:$src2, u32u8imm:$src3),
         !if(Is2Addr,
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
@@ -4729,7 +5864,7 @@ multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr,
         [(set RC:$dst, (IntId RC:$src1, RC:$src2, imm:$src3))]>,
         OpSize;
   def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),
-        (ins RC:$src1, x86memop:$src2, i32i8imm:$src3),
+        (ins RC:$src1, x86memop:$src2, u32u8imm:$src3),
         !if(Is2Addr,
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
@@ -4815,6 +5950,36 @@ defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, i256mem,
 defm VBLENDVPSY : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR256, i256mem,
                                          memopv32i8, int_x86_avx_blendv_ps_256>;
 
+let Predicates = [HasAVX] in {
+  def : Pat<(v16i8 (vselect (v16i8 VR128:$mask), (v16i8 VR128:$src1),
+                            (v16i8 VR128:$src2))),
+            (VPBLENDVBrr VR128:$src2, VR128:$src1, VR128:$mask)>;
+  def : Pat<(v4i32 (vselect (v4i32 VR128:$mask), (v4i32 VR128:$src1),
+                            (v4i32 VR128:$src2))),
+            (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>;
+  def : Pat<(v4f32 (vselect (v4i32 VR128:$mask), (v4f32 VR128:$src1),
+                            (v4f32 VR128:$src2))),
+            (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>;
+  def : Pat<(v2i64 (vselect (v2i64 VR128:$mask), (v2i64 VR128:$src1),
+                            (v2i64 VR128:$src2))),
+            (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>;
+  def : Pat<(v2f64 (vselect (v2i64 VR128:$mask), (v2f64 VR128:$src1),
+                            (v2f64 VR128:$src2))),
+            (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>;
+  def : Pat<(v8i32 (vselect (v8i32 VR256:$mask), (v8i32 VR256:$src1),
+                            (v8i32 VR256:$src2))),
+            (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
+  def : Pat<(v8f32 (vselect (v8i32 VR256:$mask), (v8f32 VR256:$src1),
+                            (v8f32 VR256:$src2))),
+            (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
+  def : Pat<(v4i64 (vselect (v4i64 VR256:$mask), (v4i64 VR256:$src1),
+                            (v4i64 VR256:$src2))),
+            (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
+  def : Pat<(v4f64 (vselect (v4i64 VR256:$mask), (v4f64 VR256:$src1),
+                            (v4f64 VR256:$src2))),
+            (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
+}
+
 /// SS41I_ternary_int - SSE 4.1 ternary operator
 let Uses = [XMM0], Constraints = "$src1 = $dst" in {
   multiclass SS41I_ternary_int<bits<8> opc, string OpcodeStr, Intrinsic IntId> {
@@ -4835,12 +6000,27 @@ let Uses = [XMM0], Constraints = "$src1 = $dst" in {
   }
 }
 
-defm BLENDVPD     : SS41I_ternary_int<0x15, "blendvpd", int_x86_sse41_blendvpd>;
-defm BLENDVPS     : SS41I_ternary_int<0x14, "blendvps", int_x86_sse41_blendvps>;
-defm PBLENDVB     : SS41I_ternary_int<0x10, "pblendvb", int_x86_sse41_pblendvb>;
+defm BLENDVPD : SS41I_ternary_int<0x15, "blendvpd", int_x86_sse41_blendvpd>;
+defm BLENDVPS : SS41I_ternary_int<0x14, "blendvps", int_x86_sse41_blendvps>;
+defm PBLENDVB : SS41I_ternary_int<0x10, "pblendvb", int_x86_sse41_pblendvb>;
 
-def : Pat<(X86pblendv VR128:$src1, VR128:$src2, XMM0),
-          (PBLENDVBrr0 VR128:$src1, VR128:$src2)>;
+let Predicates = [HasSSE41] in {
+  def : Pat<(v16i8 (vselect (v16i8 XMM0), (v16i8 VR128:$src1),
+                            (v16i8 VR128:$src2))),
+            (PBLENDVBrr0 VR128:$src2, VR128:$src1)>;
+  def : Pat<(v4i32 (vselect (v4i32 XMM0), (v4i32 VR128:$src1),
+                            (v4i32 VR128:$src2))),
+            (BLENDVPSrr0 VR128:$src2, VR128:$src1)>;
+  def : Pat<(v4f32 (vselect (v4i32 XMM0), (v4f32 VR128:$src1),
+                            (v4f32 VR128:$src2))),
+            (BLENDVPSrr0 VR128:$src2, VR128:$src1)>;
+  def : Pat<(v2i64 (vselect (v2i64 XMM0), (v2i64 VR128:$src1),
+                            (v2i64 VR128:$src2))),
+            (BLENDVPDrr0 VR128:$src2, VR128:$src1)>;
+  def : Pat<(v2f64 (vselect (v2i64 XMM0), (v2f64 VR128:$src1),
+                            (v2f64 VR128:$src2))),
+            (BLENDVPDrr0 VR128:$src2, VR128:$src1)>;
+}
 
 let Predicates = [HasAVX] in
 def VMOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
@@ -4876,9 +6056,16 @@ multiclass SS42I_binop_rm_int<bits<8> opc, string OpcodeStr,
           (bitconvert (memopv16i8 addr:$src2))))]>, OpSize;
 }
 
-let Predicates = [HasAVX] in
+let Predicates = [HasAVX] in {
   defm VPCMPGTQ : SS42I_binop_rm_int<0x37, "vpcmpgtq", int_x86_sse42_pcmpgtq,
                                      0>, VEX_4V;
+
+  def : Pat<(v2i64 (X86pcmpgtq VR128:$src1, VR128:$src2)),
+            (VPCMPGTQrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (X86pcmpgtq VR128:$src1, (memop addr:$src2))),
+            (VPCMPGTQrm VR128:$src1, addr:$src2)>;
+}
+
 let Constraints = "$src1 = $dst" in
   defm PCMPGTQ : SS42I_binop_rm_int<0x37, "pcmpgtq", int_x86_sse42_pcmpgtq>;
 
@@ -5158,22 +6345,43 @@ let Constraints = "$src1 = $dst" in {
                          int_x86_aesni_aesdeclast>;
 }
 
-def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, VR128:$src2)),
-          (AESENCrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, (memop addr:$src2))),
-          (AESENCrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, VR128:$src2)),
-          (AESENCLASTrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, (memop addr:$src2))),
-          (AESENCLASTrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, VR128:$src2)),
-          (AESDECrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, (memop addr:$src2))),
-          (AESDECrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, VR128:$src2)),
-          (AESDECLASTrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, (memop addr:$src2))),
-          (AESDECLASTrm VR128:$src1, addr:$src2)>;
+let Predicates = [HasAES] in {
+  def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, VR128:$src2)),
+            (AESENCrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, (memop addr:$src2))),
+            (AESENCrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, VR128:$src2)),
+            (AESENCLASTrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, (memop addr:$src2))),
+            (AESENCLASTrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, VR128:$src2)),
+            (AESDECrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, (memop addr:$src2))),
+            (AESDECrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, VR128:$src2)),
+            (AESDECLASTrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, (memop addr:$src2))),
+            (AESDECLASTrm VR128:$src1, addr:$src2)>;
+}
+
+let Predicates = [HasAVX, HasAES], AddedComplexity = 20 in {
+  def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, VR128:$src2)),
+            (VAESENCrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, (memop addr:$src2))),
+            (VAESENCrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, VR128:$src2)),
+            (VAESENCLASTrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, (memop addr:$src2))),
+            (VAESENCLASTrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, VR128:$src2)),
+            (VAESDECrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, (memop addr:$src2))),
+            (VAESDECrm VR128:$src1, addr:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, VR128:$src2)),
+            (VAESDECLASTrr VR128:$src1, VR128:$src2)>;
+  def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, (memop addr:$src2))),
+            (VAESDECLASTrm VR128:$src1, addr:$src2)>;
+}
 
 // Perform the AES InvMixColumn Transformation
 let Predicates = [HasAVX, HasAES] in {
@@ -5288,8 +6496,10 @@ defm : pclmul_alias<"lqlq", 0x00>;
 // AVX Instructions
 //===----------------------------------------------------------------------===//
 
-
-// Load from memory and broadcast to all elements of the destination operand
+//===----------------------------------------------------------------------===//
+// VBROADCAST - Load from memory and broadcast to all elements of the
+//              destination operand
+//
 class avx_broadcast<bits<8> opc, string OpcodeStr, RegisterClass RC,
                     X86MemOperand x86memop, Intrinsic Int> :
   AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
@@ -5305,7 +6515,26 @@ def VBROADCASTSD   : avx_broadcast<0x19, "vbroadcastsd", VR256, f64mem,
 def VBROADCASTF128 : avx_broadcast<0x1A, "vbroadcastf128", VR256, f128mem,
                                    int_x86_avx_vbroadcastf128_pd_256>;
 
-// Insert packed floating-point values
+def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src),
+          (VBROADCASTF128 addr:$src)>;
+
+def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))),
+          (VBROADCASTSSY addr:$src)>;
+def : Pat<(v4i64 (X86VBroadcast (loadi64 addr:$src))),
+          (VBROADCASTSD addr:$src)>;
+def : Pat<(v8f32 (X86VBroadcast (loadf32 addr:$src))),
+          (VBROADCASTSSY addr:$src)>;
+def : Pat<(v4f64 (X86VBroadcast (loadf64 addr:$src))),
+          (VBROADCASTSD addr:$src)>;
+
+def : Pat<(v4f32 (X86VBroadcast (loadf32 addr:$src))),
+          (VBROADCASTSS addr:$src)>;
+def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
+          (VBROADCASTSS addr:$src)>;
+
+//===----------------------------------------------------------------------===//
+// VINSERTF128 - Insert packed floating-point values
+//
 def VINSERTF128rr : AVXAIi8<0x18, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR128:$src2, i8imm:$src3),
           "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
@@ -5315,7 +6544,41 @@ def VINSERTF128rm : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst),
           "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           []>, VEX_4V;
 
-// Extract packed floating-point values
+def : Pat<(int_x86_avx_vinsertf128_pd_256 VR256:$src1, VR128:$src2, imm:$src3),
+          (VINSERTF128rr VR256:$src1, VR128:$src2, imm:$src3)>;
+def : Pat<(int_x86_avx_vinsertf128_ps_256 VR256:$src1, VR128:$src2, imm:$src3),
+          (VINSERTF128rr VR256:$src1, VR128:$src2, imm:$src3)>;
+def : Pat<(int_x86_avx_vinsertf128_si_256 VR256:$src1, VR128:$src2, imm:$src3),
+          (VINSERTF128rr VR256:$src1, VR128:$src2, imm:$src3)>;
+
+def : Pat<(vinsertf128_insert:$ins (v8f32 VR256:$src1), (v4f32 VR128:$src2),
+                                   (i32 imm)),
+          (VINSERTF128rr VR256:$src1, VR128:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v4f64 VR256:$src1), (v2f64 VR128:$src2),
+                                   (i32 imm)),
+          (VINSERTF128rr VR256:$src1, VR128:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2),
+                                   (i32 imm)),
+          (VINSERTF128rr VR256:$src1, VR128:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2),
+                                   (i32 imm)),
+          (VINSERTF128rr VR256:$src1, VR128:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2),
+                                   (i32 imm)),
+          (VINSERTF128rr VR256:$src1, VR128:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2),
+                                   (i32 imm)),
+          (VINSERTF128rr VR256:$src1, VR128:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+
+//===----------------------------------------------------------------------===//
+// VEXTRACTF128 - Extract packed floating-point values
+//
 def VEXTRACTF128rr : AVXAIi8<0x19, MRMDestReg, (outs VR128:$dst),
           (ins VR256:$src1, i8imm:$src2),
           "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}",
@@ -5325,7 +6588,41 @@ def VEXTRACTF128mr : AVXAIi8<0x19, MRMDestMem, (outs),
           "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}",
           []>, VEX;
 
-// Conditional SIMD Packed Loads and Stores
+def : Pat<(int_x86_avx_vextractf128_pd_256 VR256:$src1, imm:$src2),
+          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
+def : Pat<(int_x86_avx_vextractf128_ps_256 VR256:$src1, imm:$src2),
+          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
+def : Pat<(int_x86_avx_vextractf128_si_256 VR256:$src1, imm:$src2),
+          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
+
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+          (v4f32 (VEXTRACTF128rr
+                    (v8f32 VR256:$src1),
+                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+          (v2f64 (VEXTRACTF128rr
+                    (v4f64 VR256:$src1),
+                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+          (v4i32 (VEXTRACTF128rr
+                    (v8i32 VR256:$src1),
+                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+          (v2i64 (VEXTRACTF128rr
+                    (v4i64 VR256:$src1),
+                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+          (v8i16 (VEXTRACTF128rr
+                    (v16i16 VR256:$src1),
+                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+          (v16i8 (VEXTRACTF128rr
+                    (v32i8 VR256:$src1),
+                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+
+//===----------------------------------------------------------------------===//
+// VMASKMOV - Conditional SIMD Packed Loads and Stores
+//
 multiclass avx_movmask_rm<bits<8> opc_rm, bits<8> opc_mr, string OpcodeStr,
                           Intrinsic IntLd, Intrinsic IntLd256,
                           Intrinsic IntSt, Intrinsic IntSt256,
@@ -5363,7 +6660,9 @@ defm VMASKMOVPD : avx_movmask_rm<0x2D, 0x2F, "vmaskmovpd",
                                  int_x86_avx_maskstore_pd_256,
                                  memopv2f64, memopv4f64>;
 
-// Permute Floating-Point Values
+//===----------------------------------------------------------------------===//
+// VPERMIL - Permute Single and Double Floating-Point Values
+//
 multiclass avx_permil<bits<8> opc_rm, bits<8> opc_rmi, string OpcodeStr,
                       RegisterClass RC, X86MemOperand x86memop_f,
                       X86MemOperand x86memop_i, PatFrag f_frag, PatFrag i_frag,
@@ -5404,6 +6703,18 @@ defm VPERMILPDY : avx_permil<0x0D, 0x05, "vpermilpd", VR256, f256mem, i256mem,
                              int_x86_avx_vpermilvar_pd_256,
                              int_x86_avx_vpermil_pd_256>;
 
+def : Pat<(v8f32 (X86VPermilpsy VR256:$src1, (i8 imm:$imm))),
+          (VPERMILPSYri VR256:$src1, imm:$imm)>;
+def : Pat<(v4f64 (X86VPermilpdy VR256:$src1, (i8 imm:$imm))),
+          (VPERMILPDYri VR256:$src1, imm:$imm)>;
+def : Pat<(v8i32 (X86VPermilpsy VR256:$src1, (i8 imm:$imm))),
+          (VPERMILPSYri VR256:$src1, imm:$imm)>;
+def : Pat<(v4i64 (X86VPermilpdy VR256:$src1, (i8 imm:$imm))),
+          (VPERMILPDYri VR256:$src1, imm:$imm)>;
+
+//===----------------------------------------------------------------------===//
+// VPERM2F128 - Permute Floating-Point Values in 128-bit chunks
+//
 def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, i8imm:$src3),
           "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
@@ -5413,65 +6724,6 @@ def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst),
           "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           []>, VEX_4V;
 
-// Zero All YMM registers
-def VZEROALL : I<0x77, RawFrm, (outs), (ins), "vzeroall",
-                 [(int_x86_avx_vzeroall)]>, VEX, VEX_L, Requires<[HasAVX]>;
-
-// Zero Upper bits of YMM registers
-def VZEROUPPER : I<0x77, RawFrm, (outs), (ins), "vzeroupper",
-                   [(int_x86_avx_vzeroupper)]>, VEX, Requires<[HasAVX]>;
-
-def : Pat<(int_x86_avx_vinsertf128_pd_256 VR256:$src1, VR128:$src2, imm:$src3),
-          (VINSERTF128rr VR256:$src1, VR128:$src2, imm:$src3)>;
-def : Pat<(int_x86_avx_vinsertf128_ps_256 VR256:$src1, VR128:$src2, imm:$src3),
-          (VINSERTF128rr VR256:$src1, VR128:$src2, imm:$src3)>;
-def : Pat<(int_x86_avx_vinsertf128_si_256 VR256:$src1, VR128:$src2, imm:$src3),
-          (VINSERTF128rr VR256:$src1, VR128:$src2, imm:$src3)>;
-
-def : Pat<(vinsertf128_insert:$ins (v8f32 VR256:$src1), (v4f32 VR128:$src2),
-                                   (i32 imm)),
-          (VINSERTF128rr VR256:$src1, VR128:$src2,
-                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
-def : Pat<(vinsertf128_insert:$ins (v4f64 VR256:$src1), (v2f64 VR128:$src2),
-                                   (i32 imm)),
-          (VINSERTF128rr VR256:$src1, VR128:$src2,
-                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
-def : Pat<(vinsertf128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2),
-                                   (i32 imm)),
-          (VINSERTF128rr VR256:$src1, VR128:$src2,
-                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
-def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2),
-                                   (i32 imm)),
-          (VINSERTF128rr VR256:$src1, VR128:$src2,
-                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
-
-def : Pat<(int_x86_avx_vextractf128_pd_256 VR256:$src1, imm:$src2),
-          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
-def : Pat<(int_x86_avx_vextractf128_ps_256 VR256:$src1, imm:$src2),
-          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
-def : Pat<(int_x86_avx_vextractf128_si_256 VR256:$src1, imm:$src2),
-          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
-
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
-          (v4f32 (VEXTRACTF128rr
-                    (v8f32 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
-          (v2f64 (VEXTRACTF128rr
-                    (v4f64 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
-          (v4i32 (VEXTRACTF128rr
-                    (v8i32 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
-          (v2i64 (VEXTRACTF128rr
-                    (v4i64 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-
-def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src),
-          (VBROADCASTF128 addr:$src)>;
-
 def : Pat<(int_x86_avx_vperm2f128_ps_256 VR256:$src1, VR256:$src2, imm:$src3),
           (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$src3)>;
 def : Pat<(int_x86_avx_vperm2f128_pd_256 VR256:$src1, VR256:$src2, imm:$src3),
@@ -5489,377 +6741,59 @@ def : Pat<(int_x86_avx_vperm2f128_si_256
                   VR256:$src1, (memopv8i32 addr:$src2), imm:$src3),
           (VPERM2F128rm VR256:$src1, addr:$src2, imm:$src3)>;
 
+def : Pat<(v8f32 (X86VPerm2f128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v8i32 (X86VPerm2f128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v4i64 (X86VPerm2f128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v4f64 (X86VPerm2f128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v32i8 (X86VPerm2f128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v16i16 (X86VPerm2f128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+
 //===----------------------------------------------------------------------===//
-// SSE Shuffle pattern fragments
+// VZERO - Zero YMM registers
+//
+let Defs = [YMM0, YMM1, YMM2, YMM3, YMM4, YMM5, YMM6, YMM7,
+            YMM8, YMM9, YMM10, YMM11, YMM12, YMM13, YMM14, YMM15] in {
+  // Zero All YMM registers
+  def VZEROALL : I<0x77, RawFrm, (outs), (ins), "vzeroall",
+                  [(int_x86_avx_vzeroall)]>, TB, VEX, VEX_L, Requires<[HasAVX]>;
+
+  // Zero Upper bits of YMM registers
+  def VZEROUPPER : I<0x77, RawFrm, (outs), (ins), "vzeroupper",
+                     [(int_x86_avx_vzeroupper)]>, TB, VEX, Requires<[HasAVX]>;
+}
+
 //===----------------------------------------------------------------------===//
-
-// This is part of a "work in progress" refactoring. The idea is that all
-// vector shuffles are going to be translated into target specific nodes and
-// directly matched by the patterns below (which can be changed along the way)
-// The AVX version of some but not all of them are described here, and more
-// should come in a near future.
-
-// Shuffle with PSHUFD instruction folding loads. The first two patterns match
-// SSE2 loads, which are always promoted to v2i64. The last one should match
-// the SSE1 case, where the only legal load is v4f32, but there is no PSHUFD
-// in SSE2, how does it ever worked? Anyway, the pattern will remain here until
-// we investigate further.
-def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv2i64 addr:$src1)),
-                                 (i8 imm:$imm))),
-          (VPSHUFDmi addr:$src1, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv2i64 addr:$src1)),
-                                 (i8 imm:$imm))),
-          (PSHUFDmi addr:$src1, imm:$imm)>;
-def : Pat<(v4i32 (X86PShufd (bc_v4i32 (memopv4f32 addr:$src1)),
-                                 (i8 imm:$imm))),
-          (PSHUFDmi addr:$src1, imm:$imm)>; // FIXME: has this ever worked?
-
-// Shuffle with PSHUFD instruction.
-def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
-          (VPSHUFDri VR128:$src1, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
-          (PSHUFDri VR128:$src1, imm:$imm)>;
-
-def : Pat<(v4i32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
-          (VPSHUFDri VR128:$src1, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4i32 (X86PShufd VR128:$src1, (i8 imm:$imm))),
-          (PSHUFDri VR128:$src1, imm:$imm)>;
-
-// Shuffle with SHUFPD instruction.
-def : Pat<(v2f64 (X86Shufps VR128:$src1,
-                     (memopv2f64 addr:$src2), (i8 imm:$imm))),
-          (VSHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v2f64 (X86Shufps VR128:$src1,
-                     (memopv2f64 addr:$src2), (i8 imm:$imm))),
-          (SHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>;
-
-def : Pat<(v2i64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (VSHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v2i64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (SHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>;
-
-def : Pat<(v2f64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (VSHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v2f64 (X86Shufpd VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (SHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>;
-
-// Shuffle with SHUFPS instruction.
-def : Pat<(v4f32 (X86Shufps VR128:$src1,
-                     (memopv4f32 addr:$src2), (i8 imm:$imm))),
-          (VSHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86Shufps VR128:$src1,
-                     (memopv4f32 addr:$src2), (i8 imm:$imm))),
-          (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
-
-def : Pat<(v4f32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (VSHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
-
-def : Pat<(v4i32 (X86Shufps VR128:$src1,
-                     (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))),
-          (VSHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4i32 (X86Shufps VR128:$src1,
-                     (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))),
-          (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
-
-def : Pat<(v4i32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (VSHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>, Requires<[HasAVX]>;
-def : Pat<(v4i32 (X86Shufps VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
-
-// Shuffle with MOVHLPS instruction
-def : Pat<(v4f32 (X86Movhlps VR128:$src1, VR128:$src2)),
-          (MOVHLPSrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)),
-          (MOVHLPSrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with MOVDDUP instruction
-def : Pat<(X86Movddup (memopv2f64 addr:$src)),
-          (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
-def : Pat<(X86Movddup (memopv2f64 addr:$src)),
-          (MOVDDUPrm addr:$src)>;
-
-def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))),
-          (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
-def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))),
-          (MOVDDUPrm addr:$src)>;
-
-def : Pat<(X86Movddup (bc_v2f64 (memopv2i64 addr:$src))),
-          (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
-def : Pat<(X86Movddup (bc_v2f64 (memopv2i64 addr:$src))),
-          (MOVDDUPrm addr:$src)>;
-
-def : Pat<(X86Movddup (v2f64 (scalar_to_vector (loadf64 addr:$src)))),
-          (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
-def : Pat<(X86Movddup (v2f64 (scalar_to_vector (loadf64 addr:$src)))),
-          (MOVDDUPrm addr:$src)>;
-
-def : Pat<(X86Movddup (bc_v2f64
-                           (v2i64 (scalar_to_vector (loadi64 addr:$src))))),
-          (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
-def : Pat<(X86Movddup (bc_v2f64
-                           (v2i64 (scalar_to_vector (loadi64 addr:$src))))),
-          (MOVDDUPrm addr:$src)>;
-
-
-// Shuffle with UNPCKLPS
-def : Pat<(v4f32 (X86Unpcklps VR128:$src1, (memopv4f32 addr:$src2))),
-          (VUNPCKLPSrm VR128:$src1, addr:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v8f32 (X86Unpcklpsy VR256:$src1, (memopv8f32 addr:$src2))),
-          (VUNPCKLPSYrm VR256:$src1, addr:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86Unpcklps VR128:$src1, (memopv4f32 addr:$src2))),
-          (UNPCKLPSrm VR128:$src1, addr:$src2)>;
-
-def : Pat<(v4f32 (X86Unpcklps VR128:$src1, VR128:$src2)),
-          (VUNPCKLPSrr VR128:$src1, VR128:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v8f32 (X86Unpcklpsy VR256:$src1, VR256:$src2)),
-          (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86Unpcklps VR128:$src1, VR128:$src2)),
-          (UNPCKLPSrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with UNPCKHPS
-def : Pat<(v4f32 (X86Unpckhps VR128:$src1, (memopv4f32 addr:$src2))),
-          (VUNPCKHPSrm VR128:$src1, addr:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86Unpckhps VR128:$src1, (memopv4f32 addr:$src2))),
-          (UNPCKHPSrm VR128:$src1, addr:$src2)>;
-
-def : Pat<(v4f32 (X86Unpckhps VR128:$src1, VR128:$src2)),
-          (VUNPCKHPSrr VR128:$src1, VR128:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v4f32 (X86Unpckhps VR128:$src1, VR128:$src2)),
-          (UNPCKHPSrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with UNPCKLPD
-def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, (memopv2f64 addr:$src2))),
-          (VUNPCKLPDrm VR128:$src1, addr:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v4f64 (X86Unpcklpdy VR256:$src1, (memopv4f64 addr:$src2))),
-          (VUNPCKLPDYrm VR256:$src1, addr:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, (memopv2f64 addr:$src2))),
-          (UNPCKLPDrm VR128:$src1, addr:$src2)>;
-
-def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, VR128:$src2)),
-          (VUNPCKLPDrr VR128:$src1, VR128:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v4f64 (X86Unpcklpdy VR256:$src1, VR256:$src2)),
-          (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v2f64 (X86Unpcklpd VR128:$src1, VR128:$src2)),
-          (UNPCKLPDrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with UNPCKHPD
-def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, (memopv2f64 addr:$src2))),
-          (VUNPCKHPDrm VR128:$src1, addr:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, (memopv2f64 addr:$src2))),
-          (UNPCKHPDrm VR128:$src1, addr:$src2)>;
-
-def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, VR128:$src2)),
-          (VUNPCKHPDrr VR128:$src1, VR128:$src2)>, Requires<[HasAVX]>;
-def : Pat<(v2f64 (X86Unpckhpd VR128:$src1, VR128:$src2)),
-          (UNPCKHPDrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKLBW
-def : Pat<(v16i8 (X86Punpcklbw VR128:$src1,
-                                   (bc_v16i8 (memopv2i64 addr:$src2)))),
-          (PUNPCKLBWrm VR128:$src1, addr:$src2)>;
-def : Pat<(v16i8 (X86Punpcklbw VR128:$src1, VR128:$src2)),
-          (PUNPCKLBWrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKLWD
-def : Pat<(v8i16 (X86Punpcklwd VR128:$src1,
-                                   (bc_v8i16 (memopv2i64 addr:$src2)))),
-          (PUNPCKLWDrm VR128:$src1, addr:$src2)>;
-def : Pat<(v8i16 (X86Punpcklwd VR128:$src1, VR128:$src2)),
-          (PUNPCKLWDrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKLDQ
-def : Pat<(v4i32 (X86Punpckldq VR128:$src1,
-                                   (bc_v4i32 (memopv2i64 addr:$src2)))),
-          (PUNPCKLDQrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4i32 (X86Punpckldq VR128:$src1, VR128:$src2)),
-          (PUNPCKLDQrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKLQDQ
-def : Pat<(v2i64 (X86Punpcklqdq VR128:$src1, (memopv2i64 addr:$src2))),
-          (PUNPCKLQDQrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (X86Punpcklqdq VR128:$src1, VR128:$src2)),
-          (PUNPCKLQDQrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKHBW
-def : Pat<(v16i8 (X86Punpckhbw VR128:$src1,
-                                   (bc_v16i8 (memopv2i64 addr:$src2)))),
-          (PUNPCKHBWrm VR128:$src1, addr:$src2)>;
-def : Pat<(v16i8 (X86Punpckhbw VR128:$src1, VR128:$src2)),
-          (PUNPCKHBWrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKHWD
-def : Pat<(v8i16 (X86Punpckhwd VR128:$src1,
-                                   (bc_v8i16 (memopv2i64 addr:$src2)))),
-          (PUNPCKHWDrm VR128:$src1, addr:$src2)>;
-def : Pat<(v8i16 (X86Punpckhwd VR128:$src1, VR128:$src2)),
-          (PUNPCKHWDrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKHDQ
-def : Pat<(v4i32 (X86Punpckhdq VR128:$src1,
-                                   (bc_v4i32 (memopv2i64 addr:$src2)))),
-          (PUNPCKHDQrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4i32 (X86Punpckhdq VR128:$src1, VR128:$src2)),
-          (PUNPCKHDQrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with PUNPCKHQDQ
-def : Pat<(v2i64 (X86Punpckhqdq VR128:$src1, (memopv2i64 addr:$src2))),
-          (PUNPCKHQDQrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (X86Punpckhqdq VR128:$src1, VR128:$src2)),
-          (PUNPCKHQDQrr VR128:$src1, VR128:$src2)>;
-
-// Shuffle with MOVLHPS
-def : Pat<(X86Movlhps VR128:$src1,
-                    (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
-          (MOVHPSrm VR128:$src1, addr:$src2)>;
-def : Pat<(X86Movlhps VR128:$src1,
-                    (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
-          (MOVHPSrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4f32 (X86Movlhps VR128:$src1, VR128:$src2)),
-          (MOVLHPSrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)),
-          (MOVLHPSrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)),
-          (MOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>;
-
-// FIXME: Instead of X86Movddup, there should be a X86Unpcklpd here, the problem
-// is during lowering, where it's not possible to recognize the load fold cause
-// it has two uses through a bitcast. One use disappears at isel time and the
-// fold opportunity reappears.
-def : Pat<(v2f64 (X86Movddup VR128:$src)),
-          (UNPCKLPDrr VR128:$src, VR128:$src)>;
-
-// Shuffle with MOVLHPD
-def : Pat<(v2f64 (X86Movlhpd VR128:$src1,
-                    (scalar_to_vector (loadf64 addr:$src2)))),
-          (MOVHPDrm VR128:$src1, addr:$src2)>;
-
-// FIXME: Instead of X86Unpcklpd, there should be a X86Movlhpd here, the problem
-// is during lowering, where it's not possible to recognize the load fold cause
-// it has two uses through a bitcast. One use disappears at isel time and the
-// fold opportunity reappears.
-def : Pat<(v2f64 (X86Unpcklpd VR128:$src1,
-                    (scalar_to_vector (loadf64 addr:$src2)))),
-          (MOVHPDrm VR128:$src1, addr:$src2)>;
-
-// Shuffle with MOVSS
-def : Pat<(v4f32 (X86Movss VR128:$src1, (scalar_to_vector FR32:$src2))),
-          (MOVSSrr VR128:$src1, FR32:$src2)>;
-def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)),
-          (MOVSSrr (v4i32 VR128:$src1),
-                   (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_ss))>;
-def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)),
-          (MOVSSrr (v4f32 VR128:$src1),
-                   (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_ss))>;
-// FIXME: Instead of a X86Movss there should be a X86Movlps here, the problem
-// is during lowering, where it's not possible to recognize the load fold cause
-// it has two uses through a bitcast. One use disappears at isel time and the
-// fold opportunity reappears.
-def : Pat<(X86Movss VR128:$src1,
-                    (bc_v4i32 (v2i64 (load addr:$src2)))),
-          (MOVLPSrm VR128:$src1, addr:$src2)>;
-
-// Shuffle with MOVSD
-def : Pat<(v2f64 (X86Movsd VR128:$src1, (scalar_to_vector FR64:$src2))),
-          (MOVSDrr VR128:$src1, FR64:$src2)>;
-def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)),
-          (MOVSDrr (v2i64 VR128:$src1),
-                   (EXTRACT_SUBREG (v2i64 VR128:$src2), sub_sd))>;
-def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)),
-          (MOVSDrr (v2f64 VR128:$src1),
-                   (EXTRACT_SUBREG (v2f64 VR128:$src2), sub_sd))>;
-def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)),
-          (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_sd))>;
-def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)),
-          (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_sd))>;
-
-// Shuffle with MOVSHDUP
-def : Pat<(v4i32 (X86Movshdup VR128:$src)),
-          (MOVSHDUPrr VR128:$src)>;
-def : Pat<(X86Movshdup (bc_v4i32 (memopv2i64 addr:$src))),
-          (MOVSHDUPrm addr:$src)>;
-
-def : Pat<(v4f32 (X86Movshdup VR128:$src)),
-          (MOVSHDUPrr VR128:$src)>;
-def : Pat<(X86Movshdup (memopv4f32 addr:$src)),
-          (MOVSHDUPrm addr:$src)>;
-
-// Shuffle with MOVSLDUP
-def : Pat<(v4i32 (X86Movsldup VR128:$src)),
-          (MOVSLDUPrr VR128:$src)>;
-def : Pat<(X86Movsldup (bc_v4i32 (memopv2i64 addr:$src))),
-          (MOVSLDUPrm addr:$src)>;
-
-def : Pat<(v4f32 (X86Movsldup VR128:$src)),
-          (MOVSLDUPrr VR128:$src)>;
-def : Pat<(X86Movsldup (memopv4f32 addr:$src)),
-          (MOVSLDUPrm addr:$src)>;
-
-// Shuffle with PSHUFHW
-def : Pat<(v8i16 (X86PShufhw VR128:$src, (i8 imm:$imm))),
-          (PSHUFHWri VR128:$src, imm:$imm)>;
-def : Pat<(v8i16 (X86PShufhw (bc_v8i16 (memopv2i64 addr:$src)), (i8 imm:$imm))),
-          (PSHUFHWmi addr:$src, imm:$imm)>;
-
-// Shuffle with PSHUFLW
-def : Pat<(v8i16 (X86PShuflw VR128:$src, (i8 imm:$imm))),
-          (PSHUFLWri VR128:$src, imm:$imm)>;
-def : Pat<(v8i16 (X86PShuflw (bc_v8i16 (memopv2i64 addr:$src)), (i8 imm:$imm))),
-          (PSHUFLWmi addr:$src, imm:$imm)>;
-
-// Shuffle with PALIGN
-def : Pat<(v4i32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
-def : Pat<(v4f32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
-def : Pat<(v8i16 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
-def : Pat<(v16i8 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
-          (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
-
-// Shuffle with MOVLPS
-def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))),
-          (MOVLPSrm VR128:$src1, addr:$src2)>;
-def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))),
-          (MOVLPSrm VR128:$src1, addr:$src2)>;
-def : Pat<(X86Movlps VR128:$src1,
-                    (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
-          (MOVLPSrm VR128:$src1, addr:$src2)>;
-// FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem
-// is during lowering, where it's not possible to recognize the load fold cause
-// it has two uses through a bitcast. One use disappears at isel time and the
-// fold opportunity reappears.
-def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)),
-          (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4f32 VR128:$src2), sub_sd))>;
-
-def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)), 
-          (MOVSDrr VR128:$src1, (EXTRACT_SUBREG (v4i32 VR128:$src2), sub_sd))>; 
-
-// Shuffle with MOVLPD
-def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))),
-          (MOVLPDrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))),
-          (MOVLPDrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2f64 (X86Movlpd VR128:$src1,
-                            (scalar_to_vector (loadf64 addr:$src2)))),
-          (MOVLPDrm VR128:$src1, addr:$src2)>;
-
-// Extra patterns to match stores with MOVHPS/PD and MOVLPS/PD
-def : Pat<(store (f64 (vector_extract
-          (v2f64 (X86Unpckhps VR128:$src, (undef))), (iPTR 0))),addr:$dst),
-          (MOVHPSmr addr:$dst, VR128:$src)>;
-def : Pat<(store (f64 (vector_extract
-          (v2f64 (X86Unpckhpd VR128:$src, (undef))), (iPTR 0))),addr:$dst),
-          (MOVHPDmr addr:$dst, VR128:$src)>;
-
-def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)),addr:$src1),
-          (MOVLPSmr addr:$src1, VR128:$src2)>;
-def : Pat<(store (v4i32 (X86Movlps
-                 (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)), addr:$src1),
-          (MOVLPSmr addr:$src1, VR128:$src2)>;
-
-def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)),addr:$src1),
-          (MOVLPDmr addr:$src1, VR128:$src2)>;
-def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)),addr:$src1),
-          (MOVLPDmr addr:$src1, VR128:$src2)>;
+// Half precision conversion instructions
+//
+let Predicates = [HasAVX, HasF16C] in {
+  def VCVTPH2PSrm : I<0x13, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
+                     "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8, OpSize, VEX;
+  def VCVTPH2PSrr : I<0x13, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
+                     "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8, OpSize, VEX;
+  def VCVTPH2PSYrm : I<0x13, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src),
+                     "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8, OpSize, VEX;
+  def VCVTPH2PSYrr : I<0x13, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
+                     "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8, OpSize, VEX;
+  def VCVTPS2PHmr : Ii8<0x1D, MRMDestMem, (outs f64mem:$dst),
+                      (ins VR128:$src1, i32i8imm:$src2),
+                      "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+                      TA, OpSize, VEX;
+  def VCVTPS2PHrr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst),
+                      (ins VR128:$src1, i32i8imm:$src2),
+                      "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+                      TA, OpSize, VEX;
+  def VCVTPS2PHYmr : Ii8<0x1D, MRMDestMem, (outs f128mem:$dst),
+                      (ins VR256:$src1, i32i8imm:$src2),
+                      "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+                      TA, OpSize, VEX;
+  def VCVTPS2PHYrr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst),
+                      (ins VR256:$src1, i32i8imm:$src2),
+                      "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+                      TA, OpSize, VEX;
+}
diff --git a/contrib/llvm/lib/Target/X86/X86InstrSystem.td b/contrib/llvm/lib/Target/X86/X86InstrSystem.td
index 31de878343ef..05a5b36b95ed 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrSystem.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrSystem.td
@@ -67,43 +67,43 @@ def IRET64 : RI<0xcf, RawFrm, (outs), (ins), "iretq", []>,
 //
 let Defs = [AL], Uses = [DX] in
 def IN8rr  : I<0xEC, RawFrm, (outs), (ins),
-               "in{b}\t{%dx, %al|%AL, %DX}", []>;
+               "in{b}\t{%dx, %al|AL, DX}", []>;
 let Defs = [AX], Uses = [DX] in
 def IN16rr : I<0xED, RawFrm, (outs), (ins),
-               "in{w}\t{%dx, %ax|%AX, %DX}", []>,  OpSize;
+               "in{w}\t{%dx, %ax|AX, DX}", []>,  OpSize;
 let Defs = [EAX], Uses = [DX] in
 def IN32rr : I<0xED, RawFrm, (outs), (ins),
-               "in{l}\t{%dx, %eax|%EAX, %DX}", []>;
+               "in{l}\t{%dx, %eax|EAX, DX}", []>;
 
 let Defs = [AL] in
 def IN8ri  : Ii8<0xE4, RawFrm, (outs), (ins i8imm:$port),
-                  "in{b}\t{$port, %al|%AL, $port}", []>;
+                  "in{b}\t{$port, %al|AL, $port}", []>;
 let Defs = [AX] in
 def IN16ri : Ii8<0xE5, RawFrm, (outs), (ins i8imm:$port),
-                  "in{w}\t{$port, %ax|%AX, $port}", []>, OpSize;
+                  "in{w}\t{$port, %ax|AX, $port}", []>, OpSize;
 let Defs = [EAX] in
 def IN32ri : Ii8<0xE5, RawFrm, (outs), (ins i8imm:$port),
-                  "in{l}\t{$port, %eax|%EAX, $port}", []>;
+                  "in{l}\t{$port, %eax|EAX, $port}", []>;
 
 let Uses = [DX, AL] in
 def OUT8rr  : I<0xEE, RawFrm, (outs), (ins),
-                "out{b}\t{%al, %dx|%DX, %AL}", []>;
+                "out{b}\t{%al, %dx|DX, AL}", []>;
 let Uses = [DX, AX] in
 def OUT16rr : I<0xEF, RawFrm, (outs), (ins),
-                "out{w}\t{%ax, %dx|%DX, %AX}", []>, OpSize;
+                "out{w}\t{%ax, %dx|DX, AX}", []>, OpSize;
 let Uses = [DX, EAX] in
 def OUT32rr : I<0xEF, RawFrm, (outs), (ins),
-                "out{l}\t{%eax, %dx|%DX, %EAX}", []>;
+                "out{l}\t{%eax, %dx|DX, EAX}", []>;
 
 let Uses = [AL] in
 def OUT8ir  : Ii8<0xE6, RawFrm, (outs), (ins i8imm:$port),
-                   "out{b}\t{%al, $port|$port, %AL}", []>;
+                   "out{b}\t{%al, $port|$port, AL}", []>;
 let Uses = [AX] in
 def OUT16ir : Ii8<0xE7, RawFrm, (outs), (ins i8imm:$port),
-                   "out{w}\t{%ax, $port|$port, %AX}", []>, OpSize;
+                   "out{w}\t{%ax, $port|$port, AX}", []>, OpSize;
 let Uses = [EAX] in
 def OUT32ir : Ii8<0xE7, RawFrm, (outs), (ins i8imm:$port),
-                   "out{l}\t{%eax, $port|$port, %EAX}", []>;
+                   "out{l}\t{%eax, $port|$port, EAX}", []>;
 
 def IN8  : I<0x6C, RawFrm, (outs), (ins), "ins{b}", []>;
 def IN16 : I<0x6D, RawFrm, (outs), (ins), "ins{w}", []>,  OpSize;
@@ -229,65 +229,65 @@ def LTRm : I<0x00, MRM3m, (outs), (ins i16mem:$src),
              "ltr{w}\t{$src}", []>, TB;
              
 def PUSHCS16 : I<0x0E, RawFrm, (outs), (ins),
-                 "push{w}\t%cs", []>, Requires<[In32BitMode]>, OpSize;
+                 "push{w}\t{%cs|CS}", []>, Requires<[In32BitMode]>, OpSize;
 def PUSHCS32 : I<0x0E, RawFrm, (outs), (ins),
-                 "push{l}\t%cs", []>, Requires<[In32BitMode]>;
+                 "push{l}\t{%cs|CS}", []>, Requires<[In32BitMode]>;
 def PUSHSS16 : I<0x16, RawFrm, (outs), (ins),
-                 "push{w}\t%ss", []>, Requires<[In32BitMode]>, OpSize;
+                 "push{w}\t{%ss|SS}", []>, Requires<[In32BitMode]>, OpSize;
 def PUSHSS32 : I<0x16, RawFrm, (outs), (ins),
-                 "push{l}\t%ss", []>, Requires<[In32BitMode]>;
+                 "push{l}\t{%ss|SS}", []>, Requires<[In32BitMode]>;
 def PUSHDS16 : I<0x1E, RawFrm, (outs), (ins),
-                 "push{w}\t%ds", []>, Requires<[In32BitMode]>, OpSize;
+                 "push{w}\t{%ds|DS}", []>, Requires<[In32BitMode]>, OpSize;
 def PUSHDS32 : I<0x1E, RawFrm, (outs), (ins),
-                 "push{l}\t%ds", []>, Requires<[In32BitMode]>;
+                 "push{l}\t{%ds|DS}", []>, Requires<[In32BitMode]>;
 def PUSHES16 : I<0x06, RawFrm, (outs), (ins),
-                 "push{w}\t%es", []>, Requires<[In32BitMode]>, OpSize;
+                 "push{w}\t{%es|ES}", []>, Requires<[In32BitMode]>, OpSize;
 def PUSHES32 : I<0x06, RawFrm, (outs), (ins),
-                 "push{l}\t%es", []>, Requires<[In32BitMode]>;
+                 "push{l}\t{%es|ES}", []>, Requires<[In32BitMode]>;
                  
 def PUSHFS16 : I<0xa0, RawFrm, (outs), (ins),
-                 "push{w}\t%fs", []>, OpSize, TB;
+                 "push{w}\t{%fs|FS}", []>, OpSize, TB;
 def PUSHFS32 : I<0xa0, RawFrm, (outs), (ins),
-                 "push{l}\t%fs", []>, TB, Requires<[In32BitMode]>;
+                 "push{l}\t{%fs|FS}", []>, TB, Requires<[In32BitMode]>;
 def PUSHGS16 : I<0xa8, RawFrm, (outs), (ins),
-                 "push{w}\t%gs", []>, OpSize, TB;
+                 "push{w}\t{%gs|GS}", []>, OpSize, TB;
 def PUSHGS32 : I<0xa8, RawFrm, (outs), (ins),
-                 "push{l}\t%gs", []>, TB, Requires<[In32BitMode]>;
+                 "push{l}\t{%gs|GS}", []>, TB, Requires<[In32BitMode]>;
 
 def PUSHFS64 : I<0xa0, RawFrm, (outs), (ins),
-                 "push{q}\t%fs", []>, TB;
+                 "push{q}\t{%fs|FS}", []>, TB;
 def PUSHGS64 : I<0xa8, RawFrm, (outs), (ins),
-                 "push{q}\t%gs", []>, TB;
+                 "push{q}\t{%gs|GS}", []>, TB;
 
 // No "pop cs" instruction.
 def POPSS16 : I<0x17, RawFrm, (outs), (ins),
-                "pop{w}\t%ss", []>, OpSize, Requires<[In32BitMode]>;
+                "pop{w}\t{%ss|SS}", []>, OpSize, Requires<[In32BitMode]>;
 def POPSS32 : I<0x17, RawFrm, (outs), (ins),
-                "pop{l}\t%ss", []>        , Requires<[In32BitMode]>;
+                "pop{l}\t{%ss|SS}", []>        , Requires<[In32BitMode]>;
                 
 def POPDS16 : I<0x1F, RawFrm, (outs), (ins),
-                "pop{w}\t%ds", []>, OpSize, Requires<[In32BitMode]>;
+                "pop{w}\t{%ds|DS}", []>, OpSize, Requires<[In32BitMode]>;
 def POPDS32 : I<0x1F, RawFrm, (outs), (ins),
-                "pop{l}\t%ds", []>        , Requires<[In32BitMode]>;
+                "pop{l}\t{%ds|DS}", []>        , Requires<[In32BitMode]>;
                 
 def POPES16 : I<0x07, RawFrm, (outs), (ins),
-                "pop{w}\t%es", []>, OpSize, Requires<[In32BitMode]>;
+                "pop{w}\t{%es|ES}", []>, OpSize, Requires<[In32BitMode]>;
 def POPES32 : I<0x07, RawFrm, (outs), (ins),
-                "pop{l}\t%es", []>        , Requires<[In32BitMode]>;
+                "pop{l}\t{%es|ES}", []>        , Requires<[In32BitMode]>;
                 
 def POPFS16 : I<0xa1, RawFrm, (outs), (ins),
-                "pop{w}\t%fs", []>, OpSize, TB;
+                "pop{w}\t{%fs|FS}", []>, OpSize, TB;
 def POPFS32 : I<0xa1, RawFrm, (outs), (ins),
-                "pop{l}\t%fs", []>, TB    , Requires<[In32BitMode]>;
+                "pop{l}\t{%fs|FS}", []>, TB    , Requires<[In32BitMode]>;
 def POPFS64 : I<0xa1, RawFrm, (outs), (ins),
-                "pop{q}\t%fs", []>, TB;
+                "pop{q}\t{%fs|FS}", []>, TB;
                 
 def POPGS16 : I<0xa9, RawFrm, (outs), (ins),
-                "pop{w}\t%gs", []>, OpSize, TB;
+                "pop{w}\t{%gs|GS}", []>, OpSize, TB;
 def POPGS32 : I<0xa9, RawFrm, (outs), (ins),
-                "pop{l}\t%gs", []>, TB    , Requires<[In32BitMode]>;
+                "pop{l}\t{%gs|GS}", []>, TB    , Requires<[In32BitMode]>;
 def POPGS64 : I<0xa9, RawFrm, (outs), (ins),
-                "pop{q}\t%gs", []>, TB;
+                "pop{q}\t{%gs|GS}", []>, TB;
                  
 
 def LDS16rm : I<0xc5, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src),
@@ -400,12 +400,29 @@ def CPUID : I<0xA2, RawFrm, (outs), (ins), "cpuid", []>, TB;
 def INVD : I<0x08, RawFrm, (outs), (ins), "invd", []>, TB;
 def WBINVD : I<0x09, RawFrm, (outs), (ins), "wbinvd", []>, TB;
 
+//===----------------------------------------------------------------------===//
+// XSAVE instructions
 let Defs = [RDX, RAX], Uses = [RCX] in
   def XGETBV : I<0x01, MRM_D0, (outs), (ins), "xgetbv", []>, TB;
 
 let Uses = [RDX, RAX, RCX] in
   def XSETBV : I<0x01, MRM_D1, (outs), (ins), "xsetbv", []>, TB;
 
+let Uses = [RDX, RAX] in {
+  def XSAVE : I<0xAE, MRM4m, (outs opaque512mem:$dst), (ins),
+               "xsave\t$dst", []>, TB;
+  def XSAVE64 : I<0xAE, MRM4m, (outs opaque512mem:$dst), (ins),
+                 "xsaveq\t$dst", []>, TB, REX_W, Requires<[In64BitMode]>;
+  def XRSTOR : I<0xAE, MRM5m, (outs), (ins opaque512mem:$dst),
+               "xrstor\t$dst", []>, TB;
+  def XRSTOR64 : I<0xAE, MRM5m, (outs), (ins opaque512mem:$dst),
+                 "xrstorq\t$dst", []>, TB, REX_W, Requires<[In64BitMode]>;
+  def XSAVEOPT : I<0xAE, MRM6m, (outs opaque512mem:$dst), (ins),
+                  "xsaveopt\t$dst", []>, TB;
+  def XSAVEOPT64 : I<0xAE, MRM6m, (outs opaque512mem:$dst), (ins),
+                    "xsaveoptq\t$dst", []>, TB, REX_W, Requires<[In64BitMode]>;
+}
+
 //===----------------------------------------------------------------------===//
 // VIA PadLock crypto instructions
 let Defs = [RAX, RDI], Uses = [RDX, RDI] in
@@ -427,3 +444,24 @@ let Defs = [RAX, RSI, RDI], Uses = [RAX, RSI, RDI] in {
 }
 let Defs = [RAX, RDX, RSI], Uses = [RAX, RSI] in
   def MONTMUL : I<0xc0, RawFrm, (outs), (ins), "montmul", []>, A6;
+
+//===----------------------------------------------------------------------===//
+// FS/GS Base Instructions
+let Predicates = [In64BitMode] in {
+  def RDFSBASE : I<0xAE, MRM0r, (outs GR32:$dst), (ins),
+                   "rdfsbase{l}\t$dst", []>, TB, XS;
+  def RDFSBASE64 : RI<0xAE, MRM0r, (outs GR64:$dst), (ins),
+                     "rdfsbase{q}\t$dst", []>, TB, XS;
+  def RDGSBASE : I<0xAE, MRM1r, (outs GR32:$dst), (ins),
+                   "rdgsbase{l}\t$dst", []>, TB, XS;
+  def RDGSBASE64 : RI<0xAE, MRM1r, (outs GR64:$dst), (ins),
+                     "rdgsbase{q}\t$dst", []>, TB, XS;
+  def WRFSBASE : I<0xAE, MRM2r, (outs), (ins GR32:$dst),
+                   "wrfsbase{l}\t$dst", []>, TB, XS;
+  def WRFSBASE64 : RI<0xAE, MRM2r, (outs), (ins GR64:$dst),
+                   "wrfsbase{q}\t$dst", []>, TB, XS;
+  def WRGSBASE : I<0xAE, MRM3r, (outs), (ins GR32:$dst),
+                   "wrgsbase{l}\t$dst", []>, TB, XS;
+  def WRGSBASE64 : RI<0xAE, MRM3r, (outs), (ins GR64:$dst),
+                    "wrgsbase{q}\t$dst", []>, TB, XS;
+}
diff --git a/contrib/llvm/lib/Target/X86/X86InstrVMX.td b/contrib/llvm/lib/Target/X86/X86InstrVMX.td
index daf61e4625d4..09a7a7d0c4d0 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrVMX.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrVMX.td
@@ -16,9 +16,15 @@
 // VMX instructions
 
 // 66 0F 38 80
-def INVEPT : I<0x80, RawFrm, (outs), (ins), "invept", []>, OpSize, T8;
+def INVEPT32 : I<0x80, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2),
+               "invept {$src2, $src1|$src1, $src2}", []>, OpSize, T8;
+def INVEPT64 : I<0x80, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2),
+               "invept {$src2, $src1|$src1, $src2}", []>, OpSize, T8;
 // 66 0F 38 81
-def INVVPID : I<0x81, RawFrm, (outs), (ins), "invvpid", []>, OpSize, T8;
+def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2),
+                "invvpid {$src2, $src1|$src1, $src2}", []>, OpSize, T8;
+def INVVPID64 : I<0x81, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2),
+                "invvpid {$src2, $src1|$src1, $src2}", []>, OpSize, T8;
 // 0F 01 C1
 def VMCALL : I<0x01, MRM_C1, (outs), (ins), "vmcall", []>, TB;
 def VMCLEARm : I<0xC7, MRM6m, (outs), (ins i64mem:$vmcs),
diff --git a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
index e38533555534..50bc14d357f8 100644
--- a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -372,15 +372,10 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
   case X86::FsFLD0SD:      LowerUnaryToTwoAddr(OutMI, X86::PXORrr); break;
   case X86::VFsFLD0SS:     LowerUnaryToTwoAddr(OutMI, X86::VPXORrr); break;
   case X86::VFsFLD0SD:     LowerUnaryToTwoAddr(OutMI, X86::VPXORrr); break;
-  case X86::V_SET0PS:      LowerUnaryToTwoAddr(OutMI, X86::XORPSrr); break;
-  case X86::V_SET0PD:      LowerUnaryToTwoAddr(OutMI, X86::XORPDrr); break;
-  case X86::V_SET0PI:      LowerUnaryToTwoAddr(OutMI, X86::PXORrr); break;
   case X86::V_SETALLONES:  LowerUnaryToTwoAddr(OutMI, X86::PCMPEQDrr); break;
-  case X86::AVX_SET0PS:    LowerUnaryToTwoAddr(OutMI, X86::VXORPSrr); break;
   case X86::AVX_SET0PSY:   LowerUnaryToTwoAddr(OutMI, X86::VXORPSYrr); break;
-  case X86::AVX_SET0PD:    LowerUnaryToTwoAddr(OutMI, X86::VXORPDrr); break;
   case X86::AVX_SET0PDY:   LowerUnaryToTwoAddr(OutMI, X86::VXORPDYrr); break;
-  case X86::AVX_SET0PI:    LowerUnaryToTwoAddr(OutMI, X86::VPXORrr); break;
+  case X86::AVX_SETALLONES:  LowerUnaryToTwoAddr(OutMI, X86::VPCMPEQDrr); break;
 
   case X86::MOV16r0:
     LowerSubReg32_Op0(OutMI, X86::MOV32r0);   // MOV16r0 -> MOV32r0
@@ -468,6 +463,18 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
   case X86::JLE_4: OutMI.setOpcode(X86::JLE_1); break;
   case X86::JG_4:  OutMI.setOpcode(X86::JG_1); break;
 
+  // Atomic load and store require a separate pseudo-inst because Acquire
+  // implies mayStore and Release implies mayLoad; fix these to regular MOV
+  // instructions here
+  case X86::ACQUIRE_MOV8rm:  OutMI.setOpcode(X86::MOV8rm); goto ReSimplify;
+  case X86::ACQUIRE_MOV16rm: OutMI.setOpcode(X86::MOV16rm); goto ReSimplify;
+  case X86::ACQUIRE_MOV32rm: OutMI.setOpcode(X86::MOV32rm); goto ReSimplify;
+  case X86::ACQUIRE_MOV64rm: OutMI.setOpcode(X86::MOV64rm); goto ReSimplify;
+  case X86::RELEASE_MOV8mr:  OutMI.setOpcode(X86::MOV8mr); goto ReSimplify;
+  case X86::RELEASE_MOV16mr: OutMI.setOpcode(X86::MOV16mr); goto ReSimplify;
+  case X86::RELEASE_MOV32mr: OutMI.setOpcode(X86::MOV32mr); goto ReSimplify;
+  case X86::RELEASE_MOV64mr: OutMI.setOpcode(X86::MOV64mr); goto ReSimplify;
+
   // We don't currently select the correct instruction form for instructions
   // which have a short %eax, etc. form. Handle this by custom lowering, for
   // now.
@@ -585,6 +592,8 @@ static void LowerTlsAddr(MCStreamer &OutStreamer,
 }
 
 void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
+  OutStreamer.EmitCodeRegion();
+
   X86MCInstLower MCInstLowering(Mang, *MF, *this);
   switch (MI->getOpcode()) {
   case TargetOpcode::DBG_VALUE:
@@ -601,7 +610,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     if (OutStreamer.hasRawTextSupport())
       OutStreamer.EmitRawText(StringRef("\t#MEMBARRIER"));
     return;
-        
+
 
   case X86::EH_RETURN:
   case X86::EH_RETURN64: {
diff --git a/contrib/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/contrib/llvm/lib/Target/X86/X86MachineFunctionInfo.h
index 06043ecd3f30..b0bb313ec639 100644
--- a/contrib/llvm/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/contrib/llvm/lib/Target/X86/X86MachineFunctionInfo.h
@@ -53,10 +53,6 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   /// relocation models.
   unsigned GlobalBaseReg;
 
-  /// ReserveFP - whether the function should reserve the frame pointer
-  /// when allocating, even if there may not actually be a frame pointer used.
-  bool ReserveFP;
-
   /// VarArgsFrameIndex - FrameIndex for start of varargs area.
   int VarArgsFrameIndex;
   /// RegSaveFrameIndex - X86-64 vararg func register save area.
@@ -65,6 +61,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   unsigned VarArgsGPOffset;
   /// VarArgsFPOffset - X86-64 vararg func fp reg offset.
   unsigned VarArgsFPOffset;
+  /// ArgumentStackSize - The number of bytes on stack consumed by the arguments
+  /// being passed on the stack.
+  unsigned ArgumentStackSize;
 
 public:
   X86MachineFunctionInfo() : ForceFramePointer(false),
@@ -77,7 +76,8 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
                              VarArgsFrameIndex(0),
                              RegSaveFrameIndex(0),
                              VarArgsGPOffset(0),
-                             VarArgsFPOffset(0) {}
+                             VarArgsFPOffset(0),
+                             ArgumentStackSize(0) {}
   
   explicit X86MachineFunctionInfo(MachineFunction &MF)
     : ForceFramePointer(false),
@@ -87,11 +87,11 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
       TailCallReturnAddrDelta(0),
       SRetReturnReg(0),
       GlobalBaseReg(0),
-      ReserveFP(false),
       VarArgsFrameIndex(0),
       RegSaveFrameIndex(0),
       VarArgsGPOffset(0),
-      VarArgsFPOffset(0) {}
+      VarArgsFPOffset(0),
+      ArgumentStackSize(0) {}
   
   bool getForceFramePointer() const { return ForceFramePointer;} 
   void setForceFramePointer(bool forceFP) { ForceFramePointer = forceFP; }
@@ -114,9 +114,6 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   unsigned getGlobalBaseReg() const { return GlobalBaseReg; }
   void setGlobalBaseReg(unsigned Reg) { GlobalBaseReg = Reg; }
 
-  bool getReserveFP() const { return ReserveFP; }
-  void setReserveFP(bool reserveFP) { ReserveFP = reserveFP; }
-
   int getVarArgsFrameIndex() const { return VarArgsFrameIndex; }
   void setVarArgsFrameIndex(int Idx) { VarArgsFrameIndex = Idx; }
 
@@ -128,6 +125,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
 
   unsigned getVarArgsFPOffset() const { return VarArgsFPOffset; }
   void setVarArgsFPOffset(unsigned Offset) { VarArgsFPOffset = Offset; }
+
+  unsigned getArgumentStackSize() const { return ArgumentStackSize; }
+  void setArgumentStackSize(unsigned size) { ArgumentStackSize = size; }
 };
 
 } // End llvm namespace
diff --git a/contrib/llvm/lib/Target/X86/X86RegisterInfo.cpp b/contrib/llvm/lib/Target/X86/X86RegisterInfo.cpp
index f2faf59367a1..c1ac9f343116 100644
--- a/contrib/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -27,7 +27,6 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -53,7 +52,13 @@ ForceStackAlign("force-align-stack",
 
 X86RegisterInfo::X86RegisterInfo(X86TargetMachine &tm,
                                  const TargetInstrInfo &tii)
-  : X86GenRegisterInfo(), TM(tm), TII(tii) {
+  : X86GenRegisterInfo(tm.getSubtarget<X86Subtarget>().is64Bit()
+                         ? X86::RIP : X86::EIP,
+                       X86_MC::getDwarfRegFlavour(tm.getTargetTriple(), false),
+                       X86_MC::getDwarfRegFlavour(tm.getTargetTriple(), true)),
+                       TM(tm), TII(tii) {
+  X86_MC::InitLLVM2SEHRegisterMapping(this);
+
   // Cache some information.
   const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>();
   Is64Bit = Subtarget->is64Bit();
@@ -70,40 +75,6 @@ X86RegisterInfo::X86RegisterInfo(X86TargetMachine &tm,
   }
 }
 
-static unsigned getFlavour(const X86Subtarget *Subtarget, bool isEH) {
-  if (!Subtarget->is64Bit()) {
-    if (Subtarget->isTargetDarwin()) {
-      if (isEH)
-        return DWARFFlavour::X86_32_DarwinEH;
-      else
-        return DWARFFlavour::X86_32_Generic;
-    } else if (Subtarget->isTargetCygMing()) {
-      // Unsupported by now, just quick fallback
-      return DWARFFlavour::X86_32_Generic;
-    } else {
-      return DWARFFlavour::X86_32_Generic;
-    }
-  }
-  return DWARFFlavour::X86_64;
-}
-
-/// getDwarfRegNum - This function maps LLVM register identifiers to the DWARF
-/// specific numbering, used in debug info and exception tables.
-int X86RegisterInfo::getDwarfRegNum(unsigned RegNo, bool isEH) const {
-  const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>();
-  unsigned Flavour = getFlavour(Subtarget, isEH);
-
-  return X86GenRegisterInfo::getDwarfRegNumFull(RegNo, Flavour);
-}
-
-/// getLLVMRegNum - This function maps DWARF register numbers to LLVM register.
-int X86RegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>();
-  unsigned Flavour = getFlavour(Subtarget, isEH);
-
-  return X86GenRegisterInfo::getLLVMRegNumFull(DwarfRegNo, Flavour);
-}
-
 /// getCompactUnwindRegNum - This function maps the register to the number for
 /// compact unwind encoding. Return -1 if the register isn't valid.
 int X86RegisterInfo::getCompactUnwindRegNum(unsigned RegNum, bool isEH) const {
@@ -121,7 +92,7 @@ int X86RegisterInfo::getCompactUnwindRegNum(unsigned RegNum, bool isEH) const {
 
 int
 X86RegisterInfo::getSEHRegNum(unsigned i) const {
-  int reg = getX86RegNum(i);
+  int reg = X86_MC::getX86RegNum(i);
   switch (i) {
   case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
   case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
@@ -140,96 +111,16 @@ X86RegisterInfo::getSEHRegNum(unsigned i) const {
   return reg;
 }
 
-/// getX86RegNum - This function maps LLVM register identifiers to their X86
-/// specific numbering, which is used in various places encoding instructions.
-unsigned X86RegisterInfo::getX86RegNum(unsigned RegNo) {
-  switch(RegNo) {
-  case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: return N86::EAX;
-  case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: return N86::ECX;
-  case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: return N86::EDX;
-  case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: return N86::EBX;
-  case X86::RSP: case X86::ESP: case X86::SP: case X86::SPL: case X86::AH:
-    return N86::ESP;
-  case X86::RBP: case X86::EBP: case X86::BP: case X86::BPL: case X86::CH:
-    return N86::EBP;
-  case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL: case X86::DH:
-    return N86::ESI;
-  case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL: case X86::BH:
-    return N86::EDI;
+const TargetRegisterClass *
+X86RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
+                                       unsigned Idx) const {
+  // The sub_8bit sub-register index is more constrained in 32-bit mode.
+  // It behaves just like the sub_8bit_hi index.
+  if (!Is64Bit && Idx == X86::sub_8bit)
+    Idx = X86::sub_8bit_hi;
 
-  case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
-    return N86::EAX;
-  case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
-    return N86::ECX;
-  case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
-    return N86::EDX;
-  case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
-    return N86::EBX;
-  case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
-    return N86::ESP;
-  case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
-    return N86::EBP;
-  case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
-    return N86::ESI;
-  case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
-    return N86::EDI;
-
-  case X86::ST0: case X86::ST1: case X86::ST2: case X86::ST3:
-  case X86::ST4: case X86::ST5: case X86::ST6: case X86::ST7:
-    return RegNo-X86::ST0;
-
-  case X86::XMM0: case X86::XMM8:
-  case X86::YMM0: case X86::YMM8: case X86::MM0:
-    return 0;
-  case X86::XMM1: case X86::XMM9:
-  case X86::YMM1: case X86::YMM9: case X86::MM1:
-    return 1;
-  case X86::XMM2: case X86::XMM10:
-  case X86::YMM2: case X86::YMM10: case X86::MM2:
-    return 2;
-  case X86::XMM3: case X86::XMM11:
-  case X86::YMM3: case X86::YMM11: case X86::MM3:
-    return 3;
-  case X86::XMM4: case X86::XMM12:
-  case X86::YMM4: case X86::YMM12: case X86::MM4:
-    return 4;
-  case X86::XMM5: case X86::XMM13:
-  case X86::YMM5: case X86::YMM13: case X86::MM5:
-    return 5;
-  case X86::XMM6: case X86::XMM14:
-  case X86::YMM6: case X86::YMM14: case X86::MM6:
-    return 6;
-  case X86::XMM7: case X86::XMM15:
-  case X86::YMM7: case X86::YMM15: case X86::MM7:
-    return 7;
-
-  case X86::ES: return 0;
-  case X86::CS: return 1;
-  case X86::SS: return 2;
-  case X86::DS: return 3;
-  case X86::FS: return 4;
-  case X86::GS: return 5;
-
-  case X86::CR0: case X86::CR8 : case X86::DR0: return 0;
-  case X86::CR1: case X86::CR9 : case X86::DR1: return 1;
-  case X86::CR2: case X86::CR10: case X86::DR2: return 2;
-  case X86::CR3: case X86::CR11: case X86::DR3: return 3;
-  case X86::CR4: case X86::CR12: case X86::DR4: return 4;
-  case X86::CR5: case X86::CR13: case X86::DR5: return 5;
-  case X86::CR6: case X86::CR14: case X86::DR6: return 6;
-  case X86::CR7: case X86::CR15: case X86::DR7: return 7;
-
-  // Pseudo index registers are equivalent to a "none"
-  // scaled index (See Intel Manual 2A, table 2-3)
-  case X86::EIZ:
-  case X86::RIZ:
-    return 4;
-
-  default:
-    assert(isVirtualRegister(RegNo) && "Unknown physical register!");
-    llvm_unreachable("Register allocator hasn't allocated reg correctly yet!");
-    return 0;
-  }
+  // Forward to TableGen's default version.
+  return X86GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
 }
 
 const TargetRegisterClass *
@@ -355,8 +246,19 @@ X86RegisterInfo::getMatchingSuperRegClass(const TargetRegisterClass *A,
 
 const TargetRegisterClass*
 X86RegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC) const{
+  // Don't allow super-classes of GR8_NOREX.  This class is only used after
+  // extrating sub_8bit_hi sub-registers.  The H sub-registers cannot be copied
+  // to the full GR8 register class in 64-bit mode, so we cannot allow the
+  // reigster class inflation.
+  //
+  // The GR8_NOREX class is always used in a way that won't be constrained to a
+  // sub-class, so sub-classes like GR8_ABCD_L are allowed to expand to the
+  // full GR8 class.
+  if (RC == X86::GR8_NOREXRegisterClass)
+    return RC;
+
   const TargetRegisterClass *Super = RC;
-  TargetRegisterClass::sc_iterator I = RC->superclasses_begin();
+  TargetRegisterClass::sc_iterator I = RC->getSuperClasses();
   do {
     switch (Super->getID()) {
     case X86::GR8RegClassID:
@@ -741,11 +643,6 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 }
 
-unsigned X86RegisterInfo::getRARegister() const {
-  return Is64Bit ? X86::RIP     // Should have dwarf #16.
-                 : X86::EIP;    // Should have dwarf #8.
-}
-
 unsigned X86RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
   return TFI->hasFP(MF) ? FramePtr : StackPtr;
@@ -948,7 +845,7 @@ namespace {
       for (unsigned i = 0, e = RI.getNumVirtRegs(); i != e; ++i) {
         unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
         if (RI.getRegClass(Reg)->getAlignment() > StackAlignment) {
-          FuncInfo->setReserveFP(true);
+          FuncInfo->setForceFramePointer(true);
           return true;
         }
       }
diff --git a/contrib/llvm/lib/Target/X86/X86RegisterInfo.h b/contrib/llvm/lib/Target/X86/X86RegisterInfo.h
index a12eb1297f7e..7d39c6853597 100644
--- a/contrib/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/contrib/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -24,22 +24,6 @@ namespace llvm {
   class TargetInstrInfo;
   class X86TargetMachine;
 
-/// N86 namespace - Native X86 register numbers
-///
-namespace N86 {
-  enum {
-    EAX = 0, ECX = 1, EDX = 2, EBX = 3, ESP = 4, EBP = 5, ESI = 6, EDI = 7
-  };
-}
-
-/// DWARFFlavour - Flavour of dwarf regnumbers
-///
-namespace DWARFFlavour {
-  enum {
-    X86_64 = 0, X86_32_DarwinEH = 1, X86_32_Generic = 2
-  };
-} 
-  
 class X86RegisterInfo : public X86GenRegisterInfo {
 public:
   X86TargetMachine &TM;
@@ -73,11 +57,6 @@ class X86RegisterInfo : public X86GenRegisterInfo {
   /// register identifier.
   static unsigned getX86RegNum(unsigned RegNo);
 
-  /// getDwarfRegNum - allows modification of X86GenRegisterInfo::getDwarfRegNum
-  /// (created by TableGen) for target dependencies.
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
-
   // FIXME: This should be tablegen'd like getDwarfRegNum is
   int getSEHRegNum(unsigned i) const;
 
@@ -95,6 +74,9 @@ class X86RegisterInfo : public X86GenRegisterInfo {
   getMatchingSuperRegClass(const TargetRegisterClass *A,
                            const TargetRegisterClass *B, unsigned Idx) const;
 
+  virtual const TargetRegisterClass *
+  getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const;
+
   const TargetRegisterClass*
   getLargestLegalSuperClass(const TargetRegisterClass *RC) const;
 
@@ -136,7 +118,6 @@ class X86RegisterInfo : public X86GenRegisterInfo {
                            int SPAdj, RegScavenger *RS = NULL) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
   unsigned getStackRegister() const { return StackPtr; }
   // FIXME: Move to FrameInfok
diff --git a/contrib/llvm/lib/Target/X86/X86RegisterInfo.td b/contrib/llvm/lib/Target/X86/X86RegisterInfo.td
index 203722a66162..9a7db36e0871 100644
--- a/contrib/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/contrib/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -390,6 +390,13 @@ def GR64_NOREX : RegisterClass<"X86", [i64], 64,
                        (GR32_NOREX sub_32bit)];
 }
 
+// GR32_NOAX - GR32 registers except EAX. Used by AddRegFrm of XCHG32 in 64-bit
+// mode to prevent encoding using the 0x90 NOP encoding. xchg %eax, %eax needs
+// to clear upper 32-bits of RAX so is not a NOP.
+def GR32_NOAX : RegisterClass<"X86", [i32], 32, (sub GR32, EAX)> {
+  let SubRegClasses = [(GR8 sub_8bit, sub_8bit_hi), (GR16 sub_16bit)];
+}
+
 // GR32_NOSP - GR32 registers except ESP.
 def GR32_NOSP : RegisterClass<"X86", [i32], 32, (sub GR32, ESP)> {
   let SubRegClasses = [(GR8 sub_8bit, sub_8bit_hi), (GR16 sub_16bit)];
@@ -455,8 +462,8 @@ def VR128 : RegisterClass<"X86", [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
   let SubRegClasses = [(FR32 sub_ss), (FR64 sub_sd)];
 }
 
-def VR256 : RegisterClass<"X86", [v32i8, v8i32, v4i64, v8f32, v4f64], 256,
-                          (sequence "YMM%u", 0, 15)> {
+def VR256 : RegisterClass<"X86", [v32i8, v16i16, v8i32, v4i64, v8f32, v4f64],
+                          256, (sequence "YMM%u", 0, 15)> {
   let SubRegClasses = [(FR32 sub_ss), (FR64 sub_sd), (VR128 sub_xmm)];
 }
 
diff --git a/contrib/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/contrib/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index 02754f9ae503..6406bce31187 100644
--- a/contrib/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/contrib/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -54,7 +54,7 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
     if (const char *bzeroEntry =  V &&
         V->isNullValue() ? Subtarget->getBZeroEntry() : 0) {
       EVT IntPtr = TLI.getPointerTy();
-      const Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
+      Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
       TargetLowering::ArgListTy Args;
       TargetLowering::ArgListEntry Entry;
       Entry.Node = Dst;
diff --git a/contrib/llvm/lib/Target/X86/X86Subtarget.cpp b/contrib/llvm/lib/Target/X86/X86Subtarget.cpp
index 5e6c659e5393..7064dd06fa30 100644
--- a/contrib/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/contrib/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -16,9 +16,11 @@
 #include "X86InstrInfo.h"
 #include "llvm/GlobalValue.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
 #include "llvm/ADT/SmallVector.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
@@ -185,24 +187,53 @@ void X86Subtarget::AutoDetectSubtargetFeatures() {
 
   X86_MC::GetCpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX);
   
-  if ((EDX >> 15) & 1) HasCMov = true;      ToggleFeature(X86::FeatureCMOV);
-  if ((EDX >> 23) & 1) X86SSELevel = MMX;   ToggleFeature(X86::FeatureMMX);
-  if ((EDX >> 25) & 1) X86SSELevel = SSE1;  ToggleFeature(X86::FeatureSSE1);
-  if ((EDX >> 26) & 1) X86SSELevel = SSE2;  ToggleFeature(X86::FeatureSSE2);
-  if (ECX & 0x1)       X86SSELevel = SSE3;  ToggleFeature(X86::FeatureSSE3);
-  if ((ECX >> 9)  & 1) X86SSELevel = SSSE3; ToggleFeature(X86::FeatureSSSE3);
-  if ((ECX >> 19) & 1) X86SSELevel = SSE41; ToggleFeature(X86::FeatureSSE41);
-  if ((ECX >> 20) & 1) X86SSELevel = SSE42; ToggleFeature(X86::FeatureSSE42);
+  if ((EDX >> 15) & 1) { HasCMov = true;      ToggleFeature(X86::FeatureCMOV); }
+  if ((EDX >> 23) & 1) { X86SSELevel = MMX;   ToggleFeature(X86::FeatureMMX);  }
+  if ((EDX >> 25) & 1) { X86SSELevel = SSE1;  ToggleFeature(X86::FeatureSSE1); }
+  if ((EDX >> 26) & 1) { X86SSELevel = SSE2;  ToggleFeature(X86::FeatureSSE2); }
+  if (ECX & 0x1)       { X86SSELevel = SSE3;  ToggleFeature(X86::FeatureSSE3); }
+  if ((ECX >> 9)  & 1) { X86SSELevel = SSSE3; ToggleFeature(X86::FeatureSSSE3);}
+  if ((ECX >> 19) & 1) { X86SSELevel = SSE41; ToggleFeature(X86::FeatureSSE41);}
+  if ((ECX >> 20) & 1) { X86SSELevel = SSE42; ToggleFeature(X86::FeatureSSE42);}
   // FIXME: AVX codegen support is not ready.
-  //if ((ECX >> 28) & 1) { HasAVX = true; } ToggleFeature(X86::FeatureAVX);
+  //if ((ECX >> 28) & 1) { HasAVX = true;  ToggleFeature(X86::FeatureAVX); }
 
   bool IsIntel = memcmp(text.c, "GenuineIntel", 12) == 0;
   bool IsAMD   = !IsIntel && memcmp(text.c, "AuthenticAMD", 12) == 0;
 
-  HasCLMUL = IsIntel && ((ECX >> 1) & 0x1);   ToggleFeature(X86::FeatureCLMUL);
-  HasFMA3  = IsIntel && ((ECX >> 12) & 0x1);  ToggleFeature(X86::FeatureFMA3);
-  HasPOPCNT = IsIntel && ((ECX >> 23) & 0x1); ToggleFeature(X86::FeaturePOPCNT);
-  HasAES   = IsIntel && ((ECX >> 25) & 0x1);  ToggleFeature(X86::FeatureAES);
+  if (IsIntel && ((ECX >> 1) & 0x1)) {
+    HasCLMUL = true;
+    ToggleFeature(X86::FeatureCLMUL);
+  }
+  if (IsIntel && ((ECX >> 12) & 0x1)) {
+    HasFMA3 = true;
+    ToggleFeature(X86::FeatureFMA3);
+  }
+  if (IsIntel && ((ECX >> 22) & 0x1)) {
+    HasMOVBE = true;
+    ToggleFeature(X86::FeatureMOVBE);
+  }
+  if (IsIntel && ((ECX >> 23) & 0x1)) {
+    HasPOPCNT = true;
+    ToggleFeature(X86::FeaturePOPCNT);
+  }
+  if (IsIntel && ((ECX >> 25) & 0x1)) {
+    HasAES = true;
+    ToggleFeature(X86::FeatureAES);
+  }
+  if (IsIntel && ((ECX >> 29) & 0x1)) {
+    HasF16C = true;
+    ToggleFeature(X86::FeatureF16C);
+  }
+  if (IsIntel && ((ECX >> 30) & 0x1)) {
+    HasRDRAND = true;
+    ToggleFeature(X86::FeatureRDRAND);
+  }
+
+  if ((ECX >> 13) & 0x1) {
+    HasCmpxchg16b = true;
+    ToggleFeature(X86::FeatureCMPXCHG16B);
+  }
 
   if (IsIntel || IsAMD) {
     // Determine if bit test memory instructions are slow.
@@ -224,6 +255,10 @@ void X86Subtarget::AutoDetectSubtargetFeatures() {
       HasX86_64 = true;
       ToggleFeature(X86::Feature64Bit);
     }
+    if ((ECX >> 5) & 0x1) {
+      HasLZCNT = true;
+      ToggleFeature(X86::FeatureLZCNT);
+    }
     if (IsAMD && ((ECX >> 6) & 0x1)) {
       HasSSE4A = true;
       ToggleFeature(X86::FeatureSSE4A);
@@ -251,14 +286,21 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
   , HasCLMUL(false)
   , HasFMA3(false)
   , HasFMA4(false)
+  , HasMOVBE(false)
+  , HasRDRAND(false)
+  , HasF16C(false)
+  , HasLZCNT(false)
+  , HasBMI(false)
   , IsBTMemSlow(false)
   , IsUAMemFast(false)
   , HasVectorUAMem(false)
+  , HasCmpxchg16b(false)
   , stackAlignment(8)
   // FIXME: this is a known good value for Yonah. How about others?
   , MaxInlineSizeThreshold(128)
   , TargetTriple(TT)
-  , In64BitMode(is64Bit) {
+  , In64BitMode(is64Bit)
+  , InNaClMode(false) {
   // Determine default and user specified characteristics
   if (!FS.empty() || !CPU.empty()) {
     std::string CPUName = CPU;
@@ -304,6 +346,11 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
   if (In64BitMode)
     ToggleFeature(X86::Mode64Bit);
 
+  if (isTargetNaCl()) {
+    InNaClMode = true;
+    ToggleFeature(X86::ModeNaCl);
+  }
+
   if (HasAVX)
     X86SSELevel = NoMMXSSE;
     
@@ -313,6 +360,9 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
   assert((!In64BitMode || HasX86_64) &&
          "64-bit code requested on a subtarget that doesn't support it!");
 
+  if(EnableSegmentedStacks && !isTargetELF())
+    report_fatal_error("Segmented stacks are only implemented on ELF.");
+
   // Stack alignment is 16 bytes on Darwin, FreeBSD, Linux and Solaris (both
   // 32 and 64 bit) and for all 64-bit targets.
   if (StackAlignOverride)
diff --git a/contrib/llvm/lib/Target/X86/X86Subtarget.h b/contrib/llvm/lib/Target/X86/X86Subtarget.h
index 6d22027b7aa8..3258d3d0ada3 100644
--- a/contrib/llvm/lib/Target/X86/X86Subtarget.h
+++ b/contrib/llvm/lib/Target/X86/X86Subtarget.h
@@ -90,6 +90,21 @@ class X86Subtarget : public X86GenSubtargetInfo {
   /// HasFMA4 - Target has 4-operand fused multiply-add
   bool HasFMA4;
 
+  /// HasMOVBE - True if the processor has the MOVBE instruction.
+  bool HasMOVBE;
+
+  /// HasRDRAND - True if the processor has the RDRAND instruction.
+  bool HasRDRAND;
+
+  /// HasF16C - Processor has 16-bit floating point conversion instructions.
+  bool HasF16C;
+
+  /// HasLZCNT - Processor has LZCNT instruction.
+  bool HasLZCNT;
+
+  /// HasBMI - Processor has BMI1 instructions.
+  bool HasBMI;
+
   /// IsBTMemSlow - True if BT (bit test) of memory instructions are slow.
   bool IsBTMemSlow;
 
@@ -100,6 +115,10 @@ class X86Subtarget : public X86GenSubtargetInfo {
   /// operands. This may require setting a feature bit in the processor.
   bool HasVectorUAMem;
 
+  /// HasCmpxchg16b - True if this processor has the CMPXCHG16B instruction;
+  /// this is true for most x86-64 chips, but not the first AMD chips.
+  bool HasCmpxchg16b;
+
   /// stackAlignment - The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   unsigned stackAlignment;
@@ -115,6 +134,9 @@ class X86Subtarget : public X86GenSubtargetInfo {
   /// In64BitMode - True if compiling for 64-bit, false for 32-bit.
   bool In64BitMode;
 
+  /// InNaClMode - True if compiling for Native Client target.
+  bool InNaClMode;
+
 public:
 
   /// This constructor initializes the data members to match that
@@ -165,9 +187,15 @@ class X86Subtarget : public X86GenSubtargetInfo {
   bool hasCLMUL() const { return HasCLMUL; }
   bool hasFMA3() const { return HasFMA3; }
   bool hasFMA4() const { return HasFMA4; }
+  bool hasMOVBE() const { return HasMOVBE; }
+  bool hasRDRAND() const { return HasRDRAND; }
+  bool hasF16C() const { return HasF16C; }
+  bool hasLZCNT() const { return HasLZCNT; }
+  bool hasBMI() const { return HasBMI; }
   bool isBTMemSlow() const { return IsBTMemSlow; }
   bool isUnalignedMemAccessFast() const { return IsUAMemFast; }
   bool hasVectorUAMem() const { return HasVectorUAMem; }
+  bool hasCmpxchg16b() const { return HasCmpxchg16b; }
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
@@ -185,6 +213,11 @@ class X86Subtarget : public X86GenSubtargetInfo {
     return !isTargetDarwin() && !isTargetWindows() && !isTargetCygMing();
   }
   bool isTargetLinux() const { return TargetTriple.getOS() == Triple::Linux; }
+  bool isTargetNaCl() const {
+    return TargetTriple.getOS() == Triple::NativeClient;
+  }
+  bool isTargetNaCl32() const { return isTargetNaCl() && !is64Bit(); }
+  bool isTargetNaCl64() const { return isTargetNaCl() && is64Bit(); }
 
   bool isTargetWindows() const { return TargetTriple.getOS() == Triple::Win32; }
   bool isTargetMingw() const { return TargetTriple.getOS() == Triple::MinGW32; }
@@ -199,7 +232,8 @@ class X86Subtarget : public X86GenSubtargetInfo {
   }
 
   bool isTargetWin64() const {
-    return In64BitMode && (isTargetMingw() || isTargetWindows());
+    // FIXME: x86_64-cygwin has not been released yet.
+    return In64BitMode && (isTargetCygMing() || isTargetWindows());
   }
 
   bool isTargetEnvMacho() const {
diff --git a/contrib/llvm/lib/Target/X86/X86TargetMachine.cpp b/contrib/llvm/lib/Target/X86/X86TargetMachine.cpp
index 9cab0e089098..15c6c4e7a7d2 100644
--- a/contrib/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/contrib/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -16,65 +16,32 @@
 #include "llvm/PassManager.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/MC/MCCodeEmitter.h"
-#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
-static MCStreamer *createMCStreamer(const Target &T, const std::string &TT,
-                                    MCContext &Ctx, TargetAsmBackend &TAB,
-                                    raw_ostream &_OS,
-                                    MCCodeEmitter *_Emitter,
-                                    bool RelaxAll,
-                                    bool NoExecStack) {
-  Triple TheTriple(TT);
-
-  if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
-    return createMachOStreamer(Ctx, TAB, _OS, _Emitter, RelaxAll);
-
-  if (TheTriple.isOSWindows())
-    return createWinCOFFStreamer(Ctx, TAB, *_Emitter, _OS, RelaxAll);
-
-  return createELFStreamer(Ctx, TAB, _OS, _Emitter, RelaxAll, NoExecStack);
-}
-
 extern "C" void LLVMInitializeX86Target() {
   // Register the target.
   RegisterTargetMachine<X86_32TargetMachine> X(TheX86_32Target);
   RegisterTargetMachine<X86_64TargetMachine> Y(TheX86_64Target);
-
-  // Register the code emitter.
-  TargetRegistry::RegisterCodeEmitter(TheX86_32Target,
-                                      createX86MCCodeEmitter);
-  TargetRegistry::RegisterCodeEmitter(TheX86_64Target,
-                                      createX86MCCodeEmitter);
-
-  // Register the asm backend.
-  TargetRegistry::RegisterAsmBackend(TheX86_32Target,
-                                     createX86_32AsmBackend);
-  TargetRegistry::RegisterAsmBackend(TheX86_64Target,
-                                     createX86_64AsmBackend);
-
-  // Register the object streamer.
-  TargetRegistry::RegisterObjectStreamer(TheX86_32Target,
-                                         createMCStreamer);
-  TargetRegistry::RegisterObjectStreamer(TheX86_64Target,
-                                         createMCStreamer);
 }
 
 
-X86_32TargetMachine::X86_32TargetMachine(const Target &T, const std::string &TT,
-                                         const std::string &CPU,
-                                         const std::string &FS)
-  : X86TargetMachine(T, TT, CPU, FS, false),
+X86_32TargetMachine::X86_32TargetMachine(const Target &T, StringRef TT,
+                                         StringRef CPU, StringRef FS,
+                                         Reloc::Model RM, CodeModel::Model CM)
+  : X86TargetMachine(T, TT, CPU, FS, RM, CM, false),
     DataLayout(getSubtargetImpl()->isTargetDarwin() ?
-               "e-p:32:32-f64:32:64-i64:32:64-f80:128:128-f128:128:128-n8:16:32" :
+               "e-p:32:32-f64:32:64-i64:32:64-f80:128:128-f128:128:128-"
+               "n8:16:32-S128" :
                (getSubtargetImpl()->isTargetCygMing() ||
                 getSubtargetImpl()->isTargetWindows()) ?
-               "e-p:32:32-f64:64:64-i64:64:64-f80:32:32-f128:128:128-n8:16:32" :
-               "e-p:32:32-f64:32:64-i64:32:64-f80:32:32-f128:128:128-n8:16:32"),
+               "e-p:32:32-f64:64:64-i64:64:64-f80:32:32-f128:128:128-"
+               "n8:16:32-S32" :
+               "e-p:32:32-f64:32:64-i64:32:64-f80:32:32-f128:128:128-"
+               "n8:16:32-S128"),
     InstrInfo(*this),
     TSInfo(*this),
     TLInfo(*this),
@@ -82,11 +49,12 @@ X86_32TargetMachine::X86_32TargetMachine(const Target &T, const std::string &TT,
 }
 
 
-X86_64TargetMachine::X86_64TargetMachine(const Target &T, const std::string &TT,
-                                         const std::string &CPU, 
-                                         const std::string &FS)
-  : X86TargetMachine(T, TT, CPU, FS, true),
-    DataLayout("e-p:64:64-s:64-f64:64:64-i64:64:64-f80:128:128-f128:128:128-n8:16:32:64"),
+X86_64TargetMachine::X86_64TargetMachine(const Target &T, StringRef TT,
+                                         StringRef CPU, StringRef FS,
+                                         Reloc::Model RM, CodeModel::Model CM)
+  : X86TargetMachine(T, TT, CPU, FS, RM, CM, true),
+    DataLayout("e-p:64:64-s:64-f64:64:64-i64:64:64-f80:128:128-f128:128:128-"
+               "n8:16:32:64-S128"),
     InstrInfo(*this),
     TSInfo(*this),
     TLInfo(*this),
@@ -95,52 +63,14 @@ X86_64TargetMachine::X86_64TargetMachine(const Target &T, const std::string &TT,
 
 /// X86TargetMachine ctor - Create an X86 target.
 ///
-X86TargetMachine::X86TargetMachine(const Target &T, const std::string &TT,
-                                   const std::string &CPU,
-                                   const std::string &FS, bool is64Bit)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+X86TargetMachine::X86TargetMachine(const Target &T, StringRef TT,
+                                   StringRef CPU, StringRef FS,
+                                   Reloc::Model RM, CodeModel::Model CM,
+                                   bool is64Bit)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS, StackAlignmentOverride, is64Bit),
     FrameLowering(*this, Subtarget),
     ELFWriterInfo(is64Bit, true) {
-  DefRelocModel = getRelocationModel();
-
-  // If no relocation model was picked, default as appropriate for the target.
-  if (getRelocationModel() == Reloc::Default) {
-    // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
-    // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
-    // use static relocation model by default.
-    if (Subtarget.isTargetDarwin()) {
-      if (Subtarget.is64Bit())
-        setRelocationModel(Reloc::PIC_);
-      else
-        setRelocationModel(Reloc::DynamicNoPIC);
-    } else if (Subtarget.isTargetWin64())
-      setRelocationModel(Reloc::PIC_);
-    else
-      setRelocationModel(Reloc::Static);
-  }
-
-  assert(getRelocationModel() != Reloc::Default &&
-         "Relocation mode not picked");
-
-  // ELF and X86-64 don't have a distinct DynamicNoPIC model.  DynamicNoPIC
-  // is defined as a model for code which may be used in static or dynamic
-  // executables but not necessarily a shared library. On X86-32 we just
-  // compile in -static mode, in x86-64 we use PIC.
-  if (getRelocationModel() == Reloc::DynamicNoPIC) {
-    if (is64Bit)
-      setRelocationModel(Reloc::PIC_);
-    else if (!Subtarget.isTargetDarwin())
-      setRelocationModel(Reloc::Static);
-  }
-
-  // If we are on Darwin, disallow static relocation model in X86-64 mode, since
-  // the Mach-O file format doesn't support it.
-  if (getRelocationModel() == Reloc::Static &&
-      Subtarget.isTargetDarwin() &&
-      is64Bit)
-    setRelocationModel(Reloc::PIC_);
-
   // Determine the PICStyle based on the target selected.
   if (getRelocationModel() == Reloc::Static) {
     // Unless we're in PIC or DynamicNoPIC mode, set the PIC style to None.
@@ -161,15 +91,19 @@ X86TargetMachine::X86TargetMachine(const Target &T, const std::string &TT,
     Subtarget.setPICStyle(PICStyles::GOT);
   }
 
-  // Finally, if we have "none" as our PIC style, force to static mode.
-  if (Subtarget.getPICStyle() == PICStyles::None)
-    setRelocationModel(Reloc::Static);
-
   // default to hard float ABI
   if (FloatABIType == FloatABI::Default)
     FloatABIType = FloatABI::Hard;    
 }
 
+//===----------------------------------------------------------------------===//
+// Command line options for x86
+//===----------------------------------------------------------------------===//
+static cl::opt<bool>
+UseVZeroUpper("x86-use-vzeroupper",
+  cl::desc("Minimize AVX to SSE transition penalty"),
+  cl::init(false));
+
 //===----------------------------------------------------------------------===//
 // Pass Pipeline Configuration
 //===----------------------------------------------------------------------===//
@@ -200,46 +134,25 @@ bool X86TargetMachine::addPostRegAlloc(PassManagerBase &PM,
 
 bool X86TargetMachine::addPreEmitPass(PassManagerBase &PM,
                                       CodeGenOpt::Level OptLevel) {
-  if (OptLevel != CodeGenOpt::None && Subtarget.hasSSE2()) {
-    PM.add(createSSEDomainFixPass());
-    return true;
+  bool ShouldPrint = false;
+  if (OptLevel != CodeGenOpt::None &&
+      (Subtarget.hasSSE2() || Subtarget.hasAVX())) {
+    PM.add(createExecutionDependencyFixPass(&X86::VR128RegClass));
+    ShouldPrint = true;
   }
-  return false;
+
+  if (Subtarget.hasAVX() && UseVZeroUpper) {
+    PM.add(createX86IssueVZeroUpperPass());
+    ShouldPrint = true;
+  }
+
+  return ShouldPrint;
 }
 
 bool X86TargetMachine::addCodeEmitter(PassManagerBase &PM,
                                       CodeGenOpt::Level OptLevel,
                                       JITCodeEmitter &JCE) {
-  // FIXME: Move this to TargetJITInfo!
-  // On Darwin, do not override 64-bit setting made in X86TargetMachine().
-  if (DefRelocModel == Reloc::Default &&
-      (!Subtarget.isTargetDarwin() || !Subtarget.is64Bit())) {
-    setRelocationModel(Reloc::Static);
-    Subtarget.setPICStyle(PICStyles::None);
-  }
-
-
   PM.add(createX86JITCodeEmitterPass(*this, JCE));
 
   return false;
 }
-
-void X86TargetMachine::setCodeModelForStatic() {
-
-    if (getCodeModel() != CodeModel::Default) return;
-
-    // For static codegen, if we're not already set, use Small codegen.
-    setCodeModel(CodeModel::Small);
-}
-
-
-void X86TargetMachine::setCodeModelForJIT() {
-
-  if (getCodeModel() != CodeModel::Default) return;
-
-  // 64-bit JIT places everything in the same buffer except external functions.
-  if (Subtarget.is64Bit())
-    setCodeModel(CodeModel::Large);
-  else
-    setCodeModel(CodeModel::Small);
-}
diff --git a/contrib/llvm/lib/Target/X86/X86TargetMachine.h b/contrib/llvm/lib/Target/X86/X86TargetMachine.h
index 885334a365fe..d1569aa9d751 100644
--- a/contrib/llvm/lib/Target/X86/X86TargetMachine.h
+++ b/contrib/llvm/lib/Target/X86/X86TargetMachine.h
@@ -29,21 +29,17 @@
 namespace llvm {
   
 class formatted_raw_ostream;
+class StringRef;
 
 class X86TargetMachine : public LLVMTargetMachine {
   X86Subtarget      Subtarget;
   X86FrameLowering  FrameLowering;
   X86ELFWriterInfo  ELFWriterInfo;
-  Reloc::Model      DefRelocModel; // Reloc model before it's overridden.
 
-private:
-  // We have specific defaults for X86.
-  virtual void setCodeModelForJIT();
-  virtual void setCodeModelForStatic();
-  
 public:
-  X86TargetMachine(const Target &T, const std::string &TT, 
-                   const std::string &CPU, const std::string &FS,
+  X86TargetMachine(const Target &T, StringRef TT, 
+                   StringRef CPU, StringRef FS,
+                   Reloc::Model RM, CodeModel::Model CM,
                    bool is64Bit);
 
   virtual const X86InstrInfo     *getInstrInfo() const {
@@ -87,8 +83,9 @@ class X86_32TargetMachine : public X86TargetMachine {
   X86TargetLowering TLInfo;
   X86JITInfo        JITInfo;
 public:
-  X86_32TargetMachine(const Target &T, const std::string &M,
-                      const std::string &CPU, const std::string &FS);
+  X86_32TargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM);
   virtual const TargetData *getTargetData() const { return &DataLayout; }
   virtual const X86TargetLowering *getTargetLowering() const {
     return &TLInfo;
@@ -113,8 +110,9 @@ class X86_64TargetMachine : public X86TargetMachine {
   X86TargetLowering TLInfo;
   X86JITInfo        JITInfo;
 public:
-  X86_64TargetMachine(const Target &T, const std::string &TT,
-                      const std::string &CPU, const std::string &FS);
+  X86_64TargetMachine(const Target &T, StringRef TT,
+                      StringRef CPU, StringRef FS,
+                      Reloc::Model RM, CodeModel::Model CM);
   virtual const TargetData *getTargetData() const { return &DataLayout; }
   virtual const X86TargetLowering *getTargetLowering() const {
     return &TLInfo;
diff --git a/contrib/llvm/lib/Target/X86/X86TargetObjectFile.cpp b/contrib/llvm/lib/Target/X86/X86TargetObjectFile.cpp
index 1231798297ac..991f322a6346 100644
--- a/contrib/llvm/lib/Target/X86/X86TargetObjectFile.cpp
+++ b/contrib/llvm/lib/Target/X86/X86TargetObjectFile.cpp
@@ -43,79 +43,3 @@ getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
                         MachineModuleInfo *MMI) const {
   return Mang->getSymbol(GV);
 }
-
-unsigned X8632_ELFTargetObjectFile::getPersonalityEncoding() const {
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-  else
-    return DW_EH_PE_absptr;
-}
-
-unsigned X8632_ELFTargetObjectFile::getLSDAEncoding() const {
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-  else
-    return DW_EH_PE_absptr;
-}
-
-unsigned X8632_ELFTargetObjectFile::getFDEEncoding(bool FDE) const {
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-  else
-    return DW_EH_PE_absptr;
-}
-
-unsigned X8632_ELFTargetObjectFile::getTTypeEncoding() const {
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-  else
-    return DW_EH_PE_absptr;
-}
-
-unsigned X8664_ELFTargetObjectFile::getPersonalityEncoding() const {
-  CodeModel::Model Model = TM.getCodeModel();
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_indirect | DW_EH_PE_pcrel | (Model == CodeModel::Small ||
-                                                 Model == CodeModel::Medium ?
-                                            DW_EH_PE_sdata4 : DW_EH_PE_sdata8);
-
-  if (Model == CodeModel::Small || Model == CodeModel::Medium)
-    return DW_EH_PE_udata4;
-
-  return DW_EH_PE_absptr;
-}
-
-unsigned X8664_ELFTargetObjectFile::getLSDAEncoding() const {
-  CodeModel::Model Model = TM.getCodeModel();
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_pcrel | (Model == CodeModel::Small ?
-                             DW_EH_PE_sdata4 : DW_EH_PE_sdata8);
-
-  if (Model == CodeModel::Small)
-    return DW_EH_PE_udata4;
-
-  return DW_EH_PE_absptr;
-}
-
-unsigned X8664_ELFTargetObjectFile::getFDEEncoding(bool CFI) const {
-  if (CFI)
-    return DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-
-  return DW_EH_PE_udata4;
-}
-
-unsigned X8664_ELFTargetObjectFile::getTTypeEncoding() const {
-  CodeModel::Model Model = TM.getCodeModel();
-  if (TM.getRelocationModel() == Reloc::PIC_)
-    return DW_EH_PE_indirect | DW_EH_PE_pcrel | (Model == CodeModel::Small ||
-                                                 Model == CodeModel::Medium ?
-                                            DW_EH_PE_sdata4 : DW_EH_PE_sdata8);
-
-  if (Model == CodeModel::Small)
-    return DW_EH_PE_udata4;
-
-  return DW_EH_PE_absptr;
-}
diff --git a/contrib/llvm/lib/Target/X86/X86TargetObjectFile.h b/contrib/llvm/lib/Target/X86/X86TargetObjectFile.h
index e21b5bffd059..d7adf27ecaef 100644
--- a/contrib/llvm/lib/Target/X86/X86TargetObjectFile.h
+++ b/contrib/llvm/lib/Target/X86/X86TargetObjectFile.h
@@ -33,28 +33,6 @@ namespace llvm {
                             MachineModuleInfo *MMI) const;
   };
 
-  class X8632_ELFTargetObjectFile : public TargetLoweringObjectFileELF {
-    const X86TargetMachine &TM;
-  public:
-    X8632_ELFTargetObjectFile(const X86TargetMachine &tm)
-      :TM(tm) { }
-    virtual unsigned getPersonalityEncoding() const;
-    virtual unsigned getLSDAEncoding() const;
-    virtual unsigned getFDEEncoding(bool CFI) const;
-    virtual unsigned getTTypeEncoding() const;
-  };
-
-  class X8664_ELFTargetObjectFile : public TargetLoweringObjectFileELF {
-    const X86TargetMachine &TM;
-  public:
-    X8664_ELFTargetObjectFile(const X86TargetMachine &tm)
-      :TM(tm) { }
-    virtual unsigned getPersonalityEncoding() const;
-    virtual unsigned getLSDAEncoding() const;
-    virtual unsigned getFDEEncoding(bool CFI) const;
-    virtual unsigned getTTypeEncoding() const;
-  };
-
 } // end namespace llvm
 
 #endif
diff --git a/contrib/llvm/lib/Target/X86/X86VZeroUpper.cpp b/contrib/llvm/lib/Target/X86/X86VZeroUpper.cpp
new file mode 100644
index 000000000000..39584942468d
--- /dev/null
+++ b/contrib/llvm/lib/Target/X86/X86VZeroUpper.cpp
@@ -0,0 +1,105 @@
+//===-- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the pass which inserts x86 AVX vzeroupper instructions
+// before calls to SSE encoded functions. This avoids transition latency
+// penalty when tranfering control between AVX encoded instructions and old
+// SSE encoding mode.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "x86-codegen"
+#include "X86.h"
+#include "X86InstrInfo.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/GlobalValue.h"
+#include "llvm/Target/TargetInstrInfo.h"
+using namespace llvm;
+
+STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
+
+namespace {
+  struct VZeroUpperInserter : public MachineFunctionPass {
+    static char ID;
+    VZeroUpperInserter() : MachineFunctionPass(ID) {}
+
+    virtual bool runOnMachineFunction(MachineFunction &MF);
+
+    bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB);
+
+    virtual const char *getPassName() const { return "X86 vzeroupper inserter";}
+
+  private:
+    const TargetInstrInfo *TII; // Machine instruction info.
+    MachineBasicBlock *MBB;     // Current basic block
+  };
+  char VZeroUpperInserter::ID = 0;
+}
+
+FunctionPass *llvm::createX86IssueVZeroUpperPass() {
+  return new VZeroUpperInserter();
+}
+
+/// runOnMachineFunction - Loop over all of the basic blocks, inserting
+/// vzero upper instructions before function calls.
+bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
+  TII = MF.getTarget().getInstrInfo();
+  bool Changed = false;
+
+  // Process any unreachable blocks in arbitrary order now.
+  for (MachineFunction::iterator BB = MF.begin(), E = MF.end(); BB != E; ++BB)
+    Changed |= processBasicBlock(MF, *BB);
+
+  return Changed;
+}
+
+static bool isCallToModuleFn(const MachineInstr *MI) {
+  assert(MI->getDesc().isCall() && "Isn't a call instruction");
+
+  for (int i = 0, e = MI->getNumOperands(); i != e; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+
+    if (!MO.isGlobal())
+      continue;
+
+    const GlobalValue *GV = MO.getGlobal();
+    GlobalValue::LinkageTypes LT = GV->getLinkage();
+    if (GV->isInternalLinkage(LT) || GV->isPrivateLinkage(LT) ||
+        (GV->isExternalLinkage(LT) && !GV->isDeclaration()))
+      return true;
+
+    return false;
+  }
+  return false;
+}
+
+/// processBasicBlock - Loop over all of the instructions in the basic block,
+/// inserting vzero upper instructions before function calls.
+bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
+                                           MachineBasicBlock &BB) {
+  bool Changed = false;
+  MBB = &BB;
+
+  for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) {
+    MachineInstr *MI = I;
+    DebugLoc dl = I->getDebugLoc();
+
+    // Insert a vzeroupper instruction before each control transfer
+    // to functions outside this module
+    if (MI->getDesc().isCall() && !isCallToModuleFn(MI)) {
+      BuildMI(*MBB, I, dl, TII->get(X86::VZEROUPPER));
+      ++NumVZU;
+    }
+  }
+
+  return Changed;
+}
diff --git a/contrib/llvm/lib/Target/XCore/MCTargetDesc/CMakeLists.txt b/contrib/llvm/lib/Target/XCore/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index c3b3dc9e647d..000000000000
--- a/contrib/llvm/lib/Target/XCore/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-add_llvm_library(LLVMXCoreDesc
-  XCoreMCTargetDesc.cpp
-  XCoreMCAsmInfo.cpp
-  )
-
-# Hack: we need to include 'main' target directory to grab private headers
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/..)
diff --git a/contrib/llvm/lib/Target/XCore/MCTargetDesc/Makefile b/contrib/llvm/lib/Target/XCore/MCTargetDesc/Makefile
deleted file mode 100644
index de61543bfe9c..000000000000
--- a/contrib/llvm/lib/Target/XCore/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/XCore/TargetDesc/Makefile ----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMXCoreDesc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp b/contrib/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
index 939d97c9d87c..276e841e6acc 100644
--- a/contrib/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
+++ b/contrib/llvm/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
@@ -13,10 +13,11 @@
 
 #include "XCoreMCTargetDesc.h"
 #include "XCoreMCAsmInfo.h"
+#include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_MC_DESC
 #include "XCoreGenInstrInfo.inc"
@@ -35,8 +36,10 @@ static MCInstrInfo *createXCoreMCInstrInfo() {
   return X;
 }
 
-extern "C" void LLVMInitializeXCoreMCInstrInfo() {
-  TargetRegistry::RegisterMCInstrInfo(TheXCoreTarget, createXCoreMCInstrInfo);
+static MCRegisterInfo *createXCoreMCRegisterInfo(StringRef TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitXCoreMCRegisterInfo(X, XCore::LR);
+  return X;
 }
 
 static MCSubtargetInfo *createXCoreMCSubtargetInfo(StringRef TT, StringRef CPU,
@@ -46,11 +49,40 @@ static MCSubtargetInfo *createXCoreMCSubtargetInfo(StringRef TT, StringRef CPU,
   return X;
 }
 
-extern "C" void LLVMInitializeXCoreMCSubtargetInfo() {
+static MCAsmInfo *createXCoreMCAsmInfo(const Target &T, StringRef TT) {
+  MCAsmInfo *MAI = new XCoreMCAsmInfo(T, TT);
+
+  // Initial state of the frame pointer is SP.
+  MachineLocation Dst(MachineLocation::VirtualFP);
+  MachineLocation Src(XCore::SP, 0);
+  MAI->addInitialFrameState(0, Dst, Src);
+
+  return MAI;
+}
+
+static MCCodeGenInfo *createXCoreMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                               CodeModel::Model CM) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->InitMCCodeGenInfo(RM, CM);
+  return X;
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeXCoreTargetMC() {
+  // Register the MC asm info.
+  RegisterMCAsmInfoFn X(TheXCoreTarget, createXCoreMCAsmInfo);
+
+  // Register the MC codegen info.
+  TargetRegistry::RegisterMCCodeGenInfo(TheXCoreTarget,
+                                        createXCoreMCCodeGenInfo);
+
+  // Register the MC instruction info.
+  TargetRegistry::RegisterMCInstrInfo(TheXCoreTarget, createXCoreMCInstrInfo);
+
+  // Register the MC register info.
+  TargetRegistry::RegisterMCRegInfo(TheXCoreTarget, createXCoreMCRegisterInfo);
+
+  // Register the MC subtarget info.
   TargetRegistry::RegisterMCSubtargetInfo(TheXCoreTarget,
                                           createXCoreMCSubtargetInfo);
 }
-
-extern "C" void LLVMInitializeXCoreMCAsmInfo() {
-  RegisterMCAsmInfo<XCoreMCAsmInfo> X(TheXCoreTarget);
-}
diff --git a/contrib/llvm/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp b/contrib/llvm/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
index 7aa8965c4ac6..9a0971d1e45f 100644
--- a/contrib/llvm/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
+++ b/contrib/llvm/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
@@ -9,7 +9,7 @@
 
 #include "XCore.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 Target llvm::TheXCoreTarget;
diff --git a/contrib/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp b/contrib/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp
index 1a43714d63b9..8906b2459e0a 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Module.h"
+#include "llvm/Analysis/DebugInfo.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -32,11 +33,11 @@
 #include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cctype>
@@ -51,6 +52,7 @@ static cl::opt<unsigned> MaxThreads("xcore-max-threads", cl::Optional,
 namespace {
   class XCoreAsmPrinter : public AsmPrinter {
     const XCoreSubtarget &Subtarget;
+    void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
   public:
     explicit XCoreAsmPrinter(TargetMachine &TM, MCStreamer &Streamer)
       : AsmPrinter(TM, Streamer), Subtarget(TM.getSubtarget<XCoreSubtarget>()){}
@@ -79,6 +81,7 @@ namespace {
     void EmitFunctionEntryLabel();
     void EmitInstruction(const MachineInstr *MI);
     void EmitFunctionBodyEnd();
+    virtual MachineLocation getDebugValueLocation(const MachineInstr *MI) const;
   };
 } // end of anonymous namespace
 
@@ -88,7 +91,7 @@ void XCoreAsmPrinter::emitArrayBound(MCSymbol *Sym, const GlobalVariable *GV) {
   assert(((GV->hasExternalLinkage() ||
     GV->hasWeakLinkage()) ||
     GV->hasLinkOnceLinkage()) && "Unexpected linkage");
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(
+  if (ArrayType *ATy = dyn_cast<ArrayType>(
     cast<PointerType>(GV->getType())->getElementType())) {
     OutStreamer.EmitSymbolAttribute(Sym, MCSA_Global);
     // FIXME: MCStreamerize.
@@ -261,16 +264,57 @@ bool XCoreAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
   return false;
 }
 
+void XCoreAsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
+                                             raw_ostream &OS) {
+  unsigned NOps = MI->getNumOperands();
+  assert(NOps == 4);
+  OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
+  // cast away const; DIetc do not take const operands for some reason.
+  DIVariable V(const_cast<MDNode *>(MI->getOperand(NOps-1).getMetadata()));
+  OS << V.getName();
+  OS << " <- ";
+  // Frame address.  Currently handles register +- offset only.
+  assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
+  OS << '['; printOperand(MI, 0, OS); OS << '+'; printOperand(MI, 1, OS);
+  OS << ']';
+  OS << "+";
+  printOperand(MI, NOps-2, OS);
+}
+
+MachineLocation XCoreAsmPrinter::
+getDebugValueLocation(const MachineInstr *MI) const {
+  // Handles frame addresses emitted in XCoreInstrInfo::emitFrameIndexDebugValue.
+  assert(MI->getNumOperands() == 4 && "Invalid no. of machine operands!");
+  assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm() &&
+         "Unexpected MachineOperand types");
+  return MachineLocation(MI->getOperand(0).getReg(),
+                         MI->getOperand(1).getImm());
+}
+
 void XCoreAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   SmallString<128> Str;
   raw_svector_ostream O(Str);
 
-  // Check for mov mnemonic
-  if (MI->getOpcode() == XCore::ADD_2rus && !MI->getOperand(2).getImm())
-    O << "\tmov " << getRegisterName(MI->getOperand(0).getReg()) << ", "
-      << getRegisterName(MI->getOperand(1).getReg());
-  else
-    printInstruction(MI, O);
+  switch (MI->getOpcode()) {
+  case XCore::DBG_VALUE: {
+    if (isVerbose() && OutStreamer.hasRawTextSupport()) {
+      SmallString<128> TmpStr;
+      raw_svector_ostream OS(TmpStr);
+      PrintDebugValueComment(MI, OS);
+      OutStreamer.EmitRawText(StringRef(OS.str()));
+    }
+    return;
+  }
+  case XCore::ADD_2rus:
+    if (MI->getOperand(2).getImm() == 0) {
+      O << "\tmov " << getRegisterName(MI->getOperand(0).getReg()) << ", "
+        << getRegisterName(MI->getOperand(1).getReg());
+      OutStreamer.EmitRawText(O.str());
+      return;
+    }
+    break;
+  }
+  printInstruction(MI, O);
   OutStreamer.EmitRawText(O.str());
 }
 
diff --git a/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.cpp b/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
index 057822074e54..7f8b169819a7 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -100,7 +100,8 @@ void XCoreFrameLowering::emitPrologue(MachineFunction &MF) const {
   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   bool FP = hasFP(MF);
-  bool Nested = MF.getFunction()->getAttributes().hasAttrSomewhere(Attribute::Nest);
+  bool Nested = MF.getFunction()->
+                getAttributes().hasAttrSomewhere(Attribute::Nest);
 
   if (Nested) {
     loadFromStack(MBB, MBBI, XCore::R11, 0, dl, TII);
@@ -270,14 +271,6 @@ void XCoreFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-void XCoreFrameLowering::getInitialFrameState(std::vector<MachineMove> &Moves)
-                                                                        const {
-  // Initial state of the frame pointer is SP.
-  MachineLocation Dst(MachineLocation::VirtualFP);
-  MachineLocation Src(XCore::SP, 0);
-  Moves.push_back(MachineMove(0, Dst, Src));
-}
-
 bool XCoreFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
                                                MachineBasicBlock::iterator MI,
                                         const std::vector<CalleeSavedInfo> &CSI,
diff --git a/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.h b/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.h
index 7da19f0deb1b..c591e93780aa 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.h
+++ b/contrib/llvm/lib/Target/XCore/XCoreFrameLowering.h
@@ -42,8 +42,6 @@ namespace llvm {
 
     bool hasFP(const MachineFunction &MF) const;
 
-    void getInitialFrameState(std::vector<MachineMove> &Moves) const;
-
     void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                               RegScavenger *RS = NULL) const;
 
diff --git a/contrib/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp b/contrib/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp
index a8dd8479d45f..4dac1cee9827 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp
@@ -169,9 +169,14 @@ SDNode *XCoreDAGToDAGISel::Select(SDNode *N) {
         CurDAG->getTargetConstantPool(ConstantInt::get(
                               Type::getInt32Ty(*CurDAG->getContext()), Val),
                                       TLI.getPointerTy());
-      return CurDAG->getMachineNode(XCore::LDWCP_lru6, dl, MVT::i32, 
-                                    MVT::Other, CPIdx, 
-                                    CurDAG->getEntryNode());
+      SDNode *node = CurDAG->getMachineNode(XCore::LDWCP_lru6, dl, MVT::i32,
+                                            MVT::Other, CPIdx,
+                                            CurDAG->getEntryNode());
+      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
+      MemOp[0] = MF->getMachineMemOperand(
+        MachinePointerInfo::getConstantPool(), MachineMemOperand::MOLoad, 4, 4);      
+      cast<MachineSDNode>(node)->setMemRefs(MemOp, MemOp + 1);
+      return node;
     }
     break;
   }
diff --git a/contrib/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/contrib/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index 6d040e052659..2afe0e35afb1 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -81,6 +81,7 @@ XCoreTargetLowering::XCoreTargetLowering(XCoreTargetMachine &XTM)
 
   // Use i32 for setcc operations results (slt, sgt, ...).
   setBooleanContents(ZeroOrOneBooleanContent);
+  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
 
   // XCore does not have the NodeTypes below.
   setOperationAction(ISD::BR_CC,     MVT::Other, Expand);
@@ -147,7 +148,8 @@ XCoreTargetLowering::XCoreTargetLowering(XCoreTargetMachine &XTM)
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
 
   // TRAMPOLINE is custom lowered.
-  setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom);
+  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
+  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
   maxStoresPerMemset = maxStoresPerMemsetOptSize = 4;
   maxStoresPerMemmove = maxStoresPerMemmoveOptSize
@@ -180,7 +182,8 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ADD:
   case ISD::SUB:              return ExpandADDSUB(Op.getNode(), DAG);
   case ISD::FRAMEADDR:        return LowerFRAMEADDR(Op, DAG);
-  case ISD::TRAMPOLINE:       return LowerTRAMPOLINE(Op, DAG);
+  case ISD::INIT_TRAMPOLINE:  return LowerINIT_TRAMPOLINE(Op, DAG);
+  case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
   default:
     llvm_unreachable("unimplemented operand");
     return SDValue();
@@ -252,8 +255,8 @@ static inline SDValue BuildGetId(SelectionDAG &DAG, DebugLoc dl) {
                      DAG.getConstant(Intrinsic::xcore_getid, MVT::i32));
 }
 
-static inline bool isZeroLengthArray(const Type *Ty) {
-  const ArrayType *AT = dyn_cast_or_null<ArrayType>(Ty);
+static inline bool isZeroLengthArray(Type *Ty) {
+  ArrayType *AT = dyn_cast_or_null<ArrayType>(Ty);
   return AT && (AT->getNumElements() == 0);
 }
 
@@ -275,7 +278,7 @@ LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
     llvm_unreachable("Thread local object not a GlobalVariable?");
     return SDValue();
   }
-  const Type *Ty = cast<PointerType>(GV->getType())->getElementType();
+  Type *Ty = cast<PointerType>(GV->getType())->getElementType();
   if (!Ty->isSized() || isZeroLengthArray(Ty)) {
 #ifndef NDEBUG
     errs() << "Size of thread local object " << GVar->getName()
@@ -465,7 +468,7 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   }
 
   // Lower to a call to __misaligned_load(BasePtr).
-  const Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
@@ -524,7 +527,7 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
   }
 
   // Lower to a call to __misaligned_store(BasePtr, Value).
-  const Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
@@ -789,7 +792,12 @@ SDValue XCoreTargetLowering::LowerFRAMEADDR(SDValue Op,
 }
 
 SDValue XCoreTargetLowering::
-LowerTRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
+LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
+  return Op.getOperand(0);
+}
+
+SDValue XCoreTargetLowering::
+LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
   SDValue Chain = Op.getOperand(0);
   SDValue Trmp = Op.getOperand(1); // trampoline
   SDValue FPtr = Op.getOperand(2); // nested function
@@ -841,9 +849,7 @@ LowerTRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
                               MachinePointerInfo(TrmpAddr, 16), false, false,
                               0);
 
-  SDValue Ops[] =
-    { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 5) };
-  return DAG.getMergeValues(Ops, 2, dl);
+  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 5);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1148,10 +1154,10 @@ XCoreTargetLowering::LowerCCCArguments(SDValue Chain,
       int offset = 0;
       // Save remaining registers, storing higher register numbers at a higher
       // address
-      for (unsigned i = array_lengthof(ArgRegs) - 1; i >= FirstVAReg; --i) {
+      for (int i = array_lengthof(ArgRegs) - 1; i >= (int)FirstVAReg; --i) {
         // Create a stack slot
         int FI = MFI->CreateFixedObject(4, offset, true);
-        if (i == FirstVAReg) {
+        if (i == (int)FirstVAReg) {
           XFI->setVarArgsFrameIndex(FI);
         }
         offset -= StackSlotSize;
@@ -1409,7 +1415,8 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
     // operands are constant canonicalize smallest to RHS.
     if ((N0C && !N1C) ||
         (N0C && N1C && N0C->getZExtValue() < N1C->getZExtValue()))
-      return DAG.getNode(XCoreISD::LMUL, dl, DAG.getVTList(VT, VT), N1, N0, N2, N3);
+      return DAG.getNode(XCoreISD::LMUL, dl, DAG.getVTList(VT, VT),
+                         N1, N0, N2, N3);
 
     // lmul(x, 0, a, b)
     if (N1C && N1C->isNullValue()) {
@@ -1548,7 +1555,7 @@ static inline bool isImmUs4(int64_t val)
 /// by AM is legal for this target, for a load/store of the specified type.
 bool
 XCoreTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              const Type *Ty) const {
+                                              Type *Ty) const {
   if (Ty->getTypeID() == Type::VoidTyID)
     return AM.Scale == 0 && isImmUs(AM.BaseOffs) && isImmUs4(AM.BaseOffs);
 
diff --git a/contrib/llvm/lib/Target/XCore/XCoreISelLowering.h b/contrib/llvm/lib/Target/XCore/XCoreISelLowering.h
index 9c803bef6dd2..d6c5b329a0a0 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreISelLowering.h
+++ b/contrib/llvm/lib/Target/XCore/XCoreISelLowering.h
@@ -101,7 +101,7 @@ namespace llvm {
                                   MachineBasicBlock *MBB) const;
 
     virtual bool isLegalAddressingMode(const AddrMode &AM,
-                                       const Type *Ty) const;
+                                       Type *Ty) const;
 
   private:
     const XCoreTargetMachine &TM;
@@ -145,7 +145,8 @@ namespace llvm {
     SDValue LowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerTRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
 
     // Inline asm support
     std::pair<unsigned, const TargetRegisterClass*>
diff --git a/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.cpp b/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
index f90481f3fbc9..a0946a197a1a 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
@@ -17,11 +17,10 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_INSTRINFO_CTOR
 #include "XCoreGenInstrInfo.inc"
@@ -387,6 +386,15 @@ void XCoreInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
     .addImm(0);
 }
 
+MachineInstr*
+XCoreInstrInfo::emitFrameIndexDebugValue(MachineFunction &MF, int FrameIx,
+                                         uint64_t Offset, const MDNode *MDPtr,
+                                         DebugLoc DL) const {
+  MachineInstrBuilder MIB = BuildMI(MF, DL, get(XCore::DBG_VALUE))
+    .addFrameIndex(FrameIx).addImm(0).addImm(Offset).addMetadata(MDPtr);
+  return &*MIB;
+}
+
 /// ReverseBranchCondition - Return the inverse opcode of the 
 /// specified Branch instruction.
 bool XCoreInstrInfo::
diff --git a/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.h b/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.h
index 840b1e163652..d354802ee03f 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.h
+++ b/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.h
@@ -78,6 +78,11 @@ class XCoreInstrInfo : public XCoreGenInstrInfo {
                                     const TargetRegisterClass *RC,
                                     const TargetRegisterInfo *TRI) const;
 
+  virtual MachineInstr *emitFrameIndexDebugValue(MachineFunction &MF,
+                                                 int FrameIx,
+                                                 uint64_t Offset,
+                                                 const MDNode *MDPtr,
+                                                 DebugLoc DL) const;
 
   virtual bool ReverseBranchCondition(
                             SmallVectorImpl<MachineOperand> &Cond) const;
diff --git a/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.td b/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.td
index 55c7527f4e83..4d2e93bc7a04 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.td
+++ b/contrib/llvm/lib/Target/XCore/XCoreInstrInfo.td
@@ -572,7 +572,7 @@ def STWDP_lru6 : _FLRU6<(outs), (ins GRRegs:$val, MEMii:$addr),
                   [(store GRRegs:$val, ADDRdpii:$addr)]>;
 
 //let Uses = [CP] in ..
-let mayLoad = 1, isReMaterializable = 1 in
+let mayLoad = 1, isReMaterializable = 1, neverHasSideEffects = 1 in
 defm LDWCP : FRU6_LRU6_cp<"ldw">;
 
 let Uses = [SP] in {
@@ -739,7 +739,7 @@ def LDAP_lu10_ba : _FLU10<(outs),
 
 let isCall=1,
 // All calls clobber the link register and the non-callee-saved registers:
-Defs = [R0, R1, R2, R3, R11, LR] in {
+Defs = [R0, R1, R2, R3, R11, LR], Uses = [SP] in {
 def BL_u10 : _FU10<
                   (outs),
                   (ins calltarget:$target, variable_ops),
@@ -754,7 +754,7 @@ def BL_lu10 : _FLU10<
 }
 
 // Two operand short
-// TODO eet, eef, testwct, tsetmr, sext (reg), zext (reg)
+// TODO eet, eef, tsetmr
 def NOT : _F2R<(outs GRRegs:$dst), (ins GRRegs:$b),
                  "not $dst, $b",
                  [(set GRRegs:$dst, (not GRRegs:$b))]>;
@@ -764,15 +764,25 @@ def NEG : _F2R<(outs GRRegs:$dst), (ins GRRegs:$b),
                  [(set GRRegs:$dst, (ineg GRRegs:$b))]>;
 
 let Constraints = "$src1 = $dst" in {
-let neverHasSideEffects = 1 in
 def SEXT_rus : _FRUS<(outs GRRegs:$dst), (ins GRRegs:$src1, i32imm:$src2),
-                 "sext $dst, $src2",
-                 []>;
+                      "sext $dst, $src2",
+                      [(set GRRegs:$dst, (int_xcore_sext GRRegs:$src1,
+                                                         immBitp:$src2))]>;
+
+def SEXT_2r : _FRUS<(outs GRRegs:$dst), (ins GRRegs:$src1, GRRegs:$src2),
+                     "sext $dst, $src2",
+                     [(set GRRegs:$dst, (int_xcore_sext GRRegs:$src1,
+                                                        GRRegs:$src2))]>;
 
-let neverHasSideEffects = 1 in
 def ZEXT_rus : _FRUS<(outs GRRegs:$dst), (ins GRRegs:$src1, i32imm:$src2),
-                 "zext $dst, $src2",
-                 []>;
+                      "zext $dst, $src2",
+                      [(set GRRegs:$dst, (int_xcore_zext GRRegs:$src1,
+                                                         immBitp:$src2))]>;
+
+def ZEXT_2r : _FRUS<(outs GRRegs:$dst), (ins GRRegs:$src1, GRRegs:$src2),
+                     "zext $dst, $src2",
+                     [(set GRRegs:$dst, (int_xcore_zext GRRegs:$src1,
+                                                        GRRegs:$src2))]>;
 
 def ANDNOT_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$src1, GRRegs:$src2),
                  "andnot $dst, $src2",
@@ -819,7 +829,8 @@ def OUT_2r : _F2R<(outs), (ins GRRegs:$r, GRRegs:$val),
 let Constraints = "$src = $dst" in
 def OUTSHR_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$r, GRRegs:$src),
                  "outshr res[$r], $src",
-                 [(set GRRegs:$dst, (int_xcore_outshr GRRegs:$r, GRRegs:$src))]>;
+                 [(set GRRegs:$dst, (int_xcore_outshr GRRegs:$r,
+                                                      GRRegs:$src))]>;
 
 def INCT_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$r),
                  "inct $dst, res[$r]",
@@ -836,7 +847,8 @@ def IN_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$r),
 let Constraints = "$src = $dst" in
 def INSHR_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$r, GRRegs:$src),
                  "inshr $dst, res[$r]",
-                 [(set GRRegs:$dst, (int_xcore_inshr GRRegs:$r, GRRegs:$src))]>;
+                 [(set GRRegs:$dst, (int_xcore_inshr GRRegs:$r,
+                                                     GRRegs:$src))]>;
 
 def CHKCT_2r : _F2R<(outs), (ins GRRegs:$r, GRRegs:$val),
                  "chkct res[$r], $val",
@@ -846,6 +858,14 @@ def CHKCT_rus : _F2R<(outs), (ins GRRegs:$r, i32imm:$val),
                  "chkct res[$r], $val",
                  [(int_xcore_chkct GRRegs:$r, immUs:$val)]>;
 
+def TESTCT_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$src),
+                     "testct $dst, res[$src]",
+                     [(set GRRegs:$dst, (int_xcore_testct GRRegs:$src))]>;
+
+def TESTWCT_2r : _F2R<(outs GRRegs:$dst), (ins GRRegs:$src),
+                      "testwct $dst, res[$src]",
+                      [(set GRRegs:$dst, (int_xcore_testwct GRRegs:$src))]>;
+
 def SETD_2r : _F2R<(outs), (ins GRRegs:$r, GRRegs:$val),
                  "setd res[$r], $val",
                  [(int_xcore_setd GRRegs:$r, GRRegs:$val)]>;
@@ -871,7 +891,6 @@ def INITDP_2r : _F2R<(outs), (ins GRRegs:$t, GRRegs:$src),
                      [(int_xcore_initdp GRRegs:$t, GRRegs:$src)]>;
 
 // Two operand long
-// TODO endin, peek,
 // getd, testlcl
 def BITREV_l2r : _FL2R<(outs GRRegs:$dst), (ins GRRegs:$src),
                  "bitrev $dst, $src",
@@ -917,6 +936,14 @@ def SETPSC_l2r : _FL2R<(outs), (ins GRRegs:$src1, GRRegs:$src2),
                        "setpsc res[$src1], $src2",
                        [(int_xcore_setpsc GRRegs:$src1, GRRegs:$src2)]>;
 
+def PEEK_l2r : _FL2R<(outs GRRegs:$dst), (ins GRRegs:$src),
+                      "peek $dst, res[$src]",
+                      [(set GRRegs:$dst, (int_xcore_peek GRRegs:$src))]>;
+
+def ENDIN_l2r : _FL2R<(outs GRRegs:$dst), (ins GRRegs:$src),
+                       "endin $dst, res[$src]",
+                       [(set GRRegs:$dst, (int_xcore_endin GRRegs:$src))]>;
+
 // One operand short
 // TODO edu, eeu, waitet, waitef, tstart, clrtp
 // setdp, setcp, setev, kcall
@@ -960,7 +987,7 @@ def ECALLF_1r : _F1R<(outs), (ins GRRegs:$src),
 
 let isCall=1, 
 // All calls clobber the link register and the non-callee-saved registers:
-Defs = [R0, R1, R2, R3, R11, LR] in {
+Defs = [R0, R1, R2, R3, R11, LR], Uses = [SP] in {
 def BLA_1r : _F1R<(outs), (ins GRRegs:$addr, variable_ops),
                  "bla $addr",
                  [(XCoreBranchLink GRRegs:$addr)]>;
@@ -974,10 +1001,15 @@ def FREER_1r : _F1R<(outs), (ins GRRegs:$r),
                "freer res[$r]",
                [(int_xcore_freer GRRegs:$r)]>;
 
-let Uses=[R11] in
+let Uses=[R11] in {
 def SETV_1r : _F1R<(outs), (ins GRRegs:$r),
-               "setv res[$r], r11",
-               [(int_xcore_setv GRRegs:$r, R11)]>;
+                   "setv res[$r], r11",
+                   [(int_xcore_setv GRRegs:$r, R11)]>;
+
+def SETEV_1r : _F1R<(outs), (ins GRRegs:$r),
+                    "setev res[$r], r11",
+                    [(int_xcore_setev GRRegs:$r, R11)]>;
+}
 
 def EEU_1r : _F1R<(outs), (ins GRRegs:$r),
                "eeu res[$r]",
@@ -985,15 +1017,24 @@ def EEU_1r : _F1R<(outs), (ins GRRegs:$r),
 
 // Zero operand short
 // TODO freet, ldspc, stspc, ldssr, stssr, ldsed, stsed,
-// stet, geted, getet, getkep, getksp, setkep, getid, kret, dcall, dret,
+// stet, getkep, getksp, setkep, getid, kret, dcall, dret,
 // dentsp, drestsp
 
 def CLRE_0R : _F0R<(outs), (ins), "clre", [(int_xcore_clre)]>;
 
-let Defs = [R11] in
+let Defs = [R11] in {
 def GETID_0R : _F0R<(outs), (ins),
-                 "get r11, id",
-                 [(set R11, (int_xcore_getid))]>;
+                    "get r11, id",
+                    [(set R11, (int_xcore_getid))]>;
+
+def GETED_0R : _F0R<(outs), (ins),
+                    "get r11, ed",
+                    [(set R11, (int_xcore_geted))]>;
+
+def GETET_0R : _F0R<(outs), (ins),
+                    "get r11, et",
+                    [(set R11, (int_xcore_getet))]>;
+}
 
 def SSYNC_0r : _F0R<(outs), (ins),
                     "ssync",
diff --git a/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.cpp b/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.cpp
index 357a4a083582..1b78b373fffa 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.cpp
@@ -17,7 +17,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineLocation.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
@@ -39,7 +38,7 @@
 using namespace llvm;
 
 XCoreRegisterInfo::XCoreRegisterInfo(const TargetInstrInfo &tii)
-  : XCoreGenRegisterInfo(), TII(tii) {
+  : XCoreGenRegisterInfo(XCore::LR), TII(tii) {
 }
 
 // helper functions
@@ -321,20 +320,8 @@ loadConstant(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   BuildMI(MBB, I, dl, TII.get(Opcode), DstReg).addImm(Value);
 }
 
-int XCoreRegisterInfo::getDwarfRegNum(unsigned RegNum, bool isEH) const {
-  return XCoreGenRegisterInfo::getDwarfRegNumFull(RegNum, 0);
-}
-
-int XCoreRegisterInfo::getLLVMRegNum(unsigned DwarfRegNo, bool isEH) const {
-  return XCoreGenRegisterInfo::getLLVMRegNumFull(DwarfRegNo,0);
-}
-
 unsigned XCoreRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
 
   return TFI->hasFP(MF) ? XCore::R10 : XCore::SP;
 }
-
-unsigned XCoreRegisterInfo::getRARegister() const {
-  return XCore::LR;
-}
diff --git a/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.h b/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.h
index 801d9eba2171..5c28f39d8788 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.h
+++ b/contrib/llvm/lib/Target/XCore/XCoreRegisterInfo.h
@@ -60,7 +60,6 @@ struct XCoreRegisterInfo : public XCoreGenRegisterInfo {
                            int SPAdj, RegScavenger *RS = NULL) const;
 
   // Debug information queries.
-  unsigned getRARegister() const;
   unsigned getFrameRegister(const MachineFunction &MF) const;
 
   //! Return the array of argument passing registers
@@ -74,10 +73,6 @@ struct XCoreRegisterInfo : public XCoreGenRegisterInfo {
   
   //! Return whether to emit frame moves
   static bool needsFrameMoves(const MachineFunction &MF);
-
-  //! Get DWARF debugging register number
-  int getDwarfRegNum(unsigned RegNum, bool isEH) const;
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
 };
 
 } // end namespace llvm
diff --git a/contrib/llvm/lib/Target/XCore/XCoreSubtarget.cpp b/contrib/llvm/lib/Target/XCore/XCoreSubtarget.cpp
index ad069bf138a7..b4e992710419 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreSubtarget.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreSubtarget.cpp
@@ -13,7 +13,7 @@
 
 #include "XCoreSubtarget.h"
 #include "XCore.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
 #define GET_SUBTARGETINFO_CTOR
diff --git a/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.cpp b/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.cpp
index 342966ae5c86..fdc5d35036bb 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.cpp
+++ b/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.cpp
@@ -14,15 +14,15 @@
 #include "XCore.h"
 #include "llvm/Module.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
 /// XCoreTargetMachine ctor - Create an ILP32 architecture model
 ///
-XCoreTargetMachine::XCoreTargetMachine(const Target &T, const std::string &TT,
-                                       const std::string &CPU,
-                                       const std::string &FS)
-  : LLVMTargetMachine(T, TT, CPU, FS),
+XCoreTargetMachine::XCoreTargetMachine(const Target &T, StringRef TT,
+                                       StringRef CPU, StringRef FS,
+                                       Reloc::Model RM, CodeModel::Model CM)
+  : LLVMTargetMachine(T, TT, CPU, FS, RM, CM),
     Subtarget(TT, CPU, FS),
     DataLayout("e-p:32:32:32-a0:0:32-f32:32:32-f64:32:32-i1:8:32-i8:8:32-"
                "i16:16:32-i32:32:32-i64:32:32-n32"),
diff --git a/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.h b/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.h
index 6235ac3a6a1a..83d09d6df49d 100644
--- a/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.h
+++ b/contrib/llvm/lib/Target/XCore/XCoreTargetMachine.h
@@ -32,8 +32,9 @@ class XCoreTargetMachine : public LLVMTargetMachine {
   XCoreTargetLowering TLInfo;
   XCoreSelectionDAGInfo TSInfo;
 public:
-  XCoreTargetMachine(const Target &T, const std::string &TT,
-                     const std::string &CPU, const std::string &FS);
+  XCoreTargetMachine(const Target &T, StringRef TT,
+                     StringRef CPU, StringRef FS,
+                     Reloc::Model RM, CodeModel::Model CM);
 
   virtual const XCoreInstrInfo *getInstrInfo() const { return &InstrInfo; }
   virtual const XCoreFrameLowering *getFrameLowering() const {
diff --git a/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index fa007cfc6513..e160f63ae358 100644
--- a/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -155,12 +155,12 @@ CallGraphNode *ArgPromotion::PromoteArguments(CallGraphNode *CGN) {
   for (unsigned i = 0; i != PointerArgs.size(); ++i) {
     bool isByVal = F->paramHasAttr(PointerArgs[i].second+1, Attribute::ByVal);
     Argument *PtrArg = PointerArgs[i].first;
-    const Type *AgTy = cast<PointerType>(PtrArg->getType())->getElementType();
+    Type *AgTy = cast<PointerType>(PtrArg->getType())->getElementType();
 
     // If this is a byval argument, and if the aggregate type is small, just
     // pass the elements, which is always safe.
     if (isByVal) {
-      if (const StructType *STy = dyn_cast<StructType>(AgTy)) {
+      if (StructType *STy = dyn_cast<StructType>(AgTy)) {
         if (maxElements > 0 && STy->getNumElements() > maxElements) {
           DEBUG(dbgs() << "argpromotion disable promoting argument '"
                 << PtrArg->getName() << "' because it would require adding more"
@@ -190,7 +190,7 @@ CallGraphNode *ArgPromotion::PromoteArguments(CallGraphNode *CGN) {
     // If the argument is a recursive type and we're in a recursive
     // function, we could end up infinitely peeling the function argument.
     if (isSelfRecursive) {
-      if (const StructType *STy = dyn_cast<StructType>(AgTy)) {
+      if (StructType *STy = dyn_cast<StructType>(AgTy)) {
         bool RecursiveType = false;
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           if (STy->getElementType(i) == PtrArg->getType()) {
@@ -382,7 +382,8 @@ bool ArgPromotion::isSafeToPromoteArgument(Argument *Arg, bool isByVal) const {
     User *U = *UI;
     Operands.clear();
     if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
-      if (LI->isVolatile()) return false;  // Don't hack volatile loads
+      // Don't hack volatile/atomic loads
+      if (!LI->isSimple()) return false;
       Loads.push_back(LI);
       // Direct loads are equivalent to a GEP with a zero index and then a load.
       Operands.push_back(0);
@@ -410,7 +411,8 @@ bool ArgPromotion::isSafeToPromoteArgument(Argument *Arg, bool isByVal) const {
       for (Value::use_iterator UI = GEP->use_begin(), E = GEP->use_end();
            UI != E; ++UI)
         if (LoadInst *LI = dyn_cast<LoadInst>(*UI)) {
-          if (LI->isVolatile()) return false;  // Don't hack volatile loads
+          // Don't hack volatile/atomic loads
+          if (!LI->isSimple()) return false;
           Loads.push_back(LI);
         } else {
           // Other uses than load?
@@ -492,7 +494,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
 
   // Start by computing a new prototype for the function, which is the same as
   // the old function, but has modified arguments.
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
   std::vector<Type*> Params;
 
   typedef std::set<IndicesVector> ScalarizeTable;
@@ -527,8 +529,8 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
        ++I, ++ArgIndex) {
     if (ByValArgsToTransform.count(I)) {
       // Simple byval argument? Just add all the struct element types.
-      const Type *AgTy = cast<PointerType>(I->getType())->getElementType();
-      const StructType *STy = cast<StructType>(AgTy);
+      Type *AgTy = cast<PointerType>(I->getType())->getElementType();
+      StructType *STy = cast<StructType>(AgTy);
       for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
         Params.push_back(STy->getElementType(i));
       ++NumByValArgsPromoted;
@@ -576,9 +578,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
       for (ScalarizeTable::iterator SI = ArgIndices.begin(),
              E = ArgIndices.end(); SI != E; ++SI) {
         // not allowed to dereference ->begin() if size() is 0
-        Params.push_back(GetElementPtrInst::getIndexedType(I->getType(),
-                                                           SI->begin(),
-                                                           SI->end()));
+        Params.push_back(GetElementPtrInst::getIndexedType(I->getType(), *SI));
         assert(Params.back());
       }
 
@@ -593,7 +593,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
   if (Attributes attrs = PAL.getFnAttributes())
     AttributesVec.push_back(AttributeWithIndex::get(~0, attrs));
 
-  const Type *RetTy = FTy->getReturnType();
+  Type *RetTy = FTy->getReturnType();
 
   // Work around LLVM bug PR56: the CWriter cannot emit varargs functions which
   // have zero fixed arguments.
@@ -662,13 +662,13 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
 
       } else if (ByValArgsToTransform.count(I)) {
         // Emit a GEP and load for each element of the struct.
-        const Type *AgTy = cast<PointerType>(I->getType())->getElementType();
-        const StructType *STy = cast<StructType>(AgTy);
+        Type *AgTy = cast<PointerType>(I->getType())->getElementType();
+        StructType *STy = cast<StructType>(AgTy);
         Value *Idxs[2] = {
               ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), 0 };
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i);
-          Value *Idx = GetElementPtrInst::Create(*AI, Idxs, Idxs+2,
+          Value *Idx = GetElementPtrInst::Create(*AI, Idxs,
                                                  (*AI)->getName()+"."+utostr(i),
                                                  Call);
           // TODO: Tell AA about the new values?
@@ -686,12 +686,12 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
           LoadInst *OrigLoad = OriginalLoads[*SI];
           if (!SI->empty()) {
             Ops.reserve(SI->size());
-            const Type *ElTy = V->getType();
+            Type *ElTy = V->getType();
             for (IndicesVector::const_iterator II = SI->begin(),
                  IE = SI->end(); II != IE; ++II) {
               // Use i32 to index structs, and i64 for others (pointers/arrays).
               // This satisfies GEP constraints.
-              const Type *IdxTy = (ElTy->isStructTy() ?
+              Type *IdxTy = (ElTy->isStructTy() ?
                     Type::getInt32Ty(F->getContext()) : 
                     Type::getInt64Ty(F->getContext()));
               Ops.push_back(ConstantInt::get(IdxTy, *II));
@@ -699,8 +699,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
               ElTy = cast<CompositeType>(ElTy)->getTypeAtIndex(*II);
             }
             // And create a GEP to extract those indices.
-            V = GetElementPtrInst::Create(V, Ops.begin(), Ops.end(),
-                                          V->getName()+".idx", Call);
+            V = GetElementPtrInst::Create(V, Ops, V->getName()+".idx", Call);
             Ops.clear();
             AA.copyValue(OrigLoad->getOperand(0), V);
           }
@@ -792,16 +791,16 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
       Instruction *InsertPt = NF->begin()->begin();
 
       // Just add all the struct element types.
-      const Type *AgTy = cast<PointerType>(I->getType())->getElementType();
+      Type *AgTy = cast<PointerType>(I->getType())->getElementType();
       Value *TheAlloca = new AllocaInst(AgTy, 0, "", InsertPt);
-      const StructType *STy = cast<StructType>(AgTy);
+      StructType *STy = cast<StructType>(AgTy);
       Value *Idxs[2] = {
             ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), 0 };
 
       for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
         Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i);
         Value *Idx = 
-          GetElementPtrInst::Create(TheAlloca, Idxs, Idxs+2,
+          GetElementPtrInst::Create(TheAlloca, Idxs,
                                     TheAlloca->getName()+"."+Twine(i), 
                                     InsertPt);
         I2->setName(I->getName()+"."+Twine(i));
diff --git a/contrib/llvm/lib/Transforms/IPO/ConstantMerge.cpp b/contrib/llvm/lib/Transforms/IPO/ConstantMerge.cpp
index a21efced73b9..c3ecb7afff7a 100644
--- a/contrib/llvm/lib/Transforms/IPO/ConstantMerge.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/ConstantMerge.cpp
@@ -23,7 +23,9 @@
 #include "llvm/DerivedTypes.h"
 #include "llvm/Module.h"
 #include "llvm/Pass.h"
+#include "llvm/Target/TargetData.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
 using namespace llvm;
@@ -37,10 +39,18 @@ namespace {
       initializeConstantMergePass(*PassRegistry::getPassRegistry());
     }
 
-    // run - For this pass, process all of the globals in the module,
-    // eliminating duplicate constants.
-    //
+    // For this pass, process all of the globals in the module, eliminating
+    // duplicate constants.
     bool runOnModule(Module &M);
+
+    // Return true iff we can determine the alignment of this global variable.
+    bool hasKnownAlignment(GlobalVariable *GV) const;
+
+    // Return the alignment of the global, including converting the default
+    // alignment to a concrete value.
+    unsigned getAlignment(GlobalVariable *GV) const;
+
+    const TargetData *TD;
   };
 }
 
@@ -77,15 +87,28 @@ static bool IsBetterCannonical(const GlobalVariable &A,
   return A.hasUnnamedAddr();
 }
 
+bool ConstantMerge::hasKnownAlignment(GlobalVariable *GV) const {
+  return TD || GV->getAlignment() != 0;
+}
+
+unsigned ConstantMerge::getAlignment(GlobalVariable *GV) const {
+  if (TD)
+    return TD->getPreferredAlignment(GV);
+  return GV->getAlignment();
+}
+
 bool ConstantMerge::runOnModule(Module &M) {
+  TD = getAnalysisIfAvailable<TargetData>();
+
   // Find all the globals that are marked "used".  These cannot be merged.
   SmallPtrSet<const GlobalValue*, 8> UsedGlobals;
   FindUsedValues(M.getGlobalVariable("llvm.used"), UsedGlobals);
   FindUsedValues(M.getGlobalVariable("llvm.compiler.used"), UsedGlobals);
   
-  // Map unique constant/section pairs to globals.  We don't want to merge
-  // globals in different sections.
-  DenseMap<Constant*, GlobalVariable*> CMap;
+  // Map unique <constants, has-unknown-alignment> pairs to globals.  We don't
+  // want to merge globals of unknown alignment with those of explicit
+  // alignment.  If we have TargetData, we always know the alignment.
+  DenseMap<PointerIntPair<Constant*, 1, bool>, GlobalVariable*> CMap;
 
   // Replacements - This vector contains a list of replacements to perform.
   SmallVector<std::pair<GlobalVariable*, GlobalVariable*>, 32> Replacements;
@@ -120,7 +143,8 @@ bool ConstantMerge::runOnModule(Module &M) {
       Constant *Init = GV->getInitializer();
 
       // Check to see if the initializer is already known.
-      GlobalVariable *&Slot = CMap[Init];
+      PointerIntPair<Constant*, 1, bool> Pair(Init, hasKnownAlignment(GV));
+      GlobalVariable *&Slot = CMap[Pair];
 
       // If this is the first constant we find or if the old on is local,
       // replace with the current one. It the current is externally visible
@@ -152,7 +176,8 @@ bool ConstantMerge::runOnModule(Module &M) {
       Constant *Init = GV->getInitializer();
 
       // Check to see if the initializer is already known.
-      GlobalVariable *Slot = CMap[Init];
+      PointerIntPair<Constant*, 1, bool> Pair(Init, hasKnownAlignment(GV));
+      GlobalVariable *Slot = CMap[Pair];
 
       if (!Slot || Slot == GV)
         continue;
@@ -175,6 +200,14 @@ bool ConstantMerge::runOnModule(Module &M) {
     // now.  This avoid invalidating the pointers in CMap, which are unneeded
     // now.
     for (unsigned i = 0, e = Replacements.size(); i != e; ++i) {
+      // Bump the alignment if necessary.
+      if (Replacements[i].first->getAlignment() ||
+          Replacements[i].second->getAlignment()) {
+        Replacements[i].second->setAlignment(std::max(
+            Replacements[i].first->getAlignment(),
+            Replacements[i].second->getAlignment()));
+      }
+
       // Eliminate any uses of the dead global.
       Replacements[i].first->replaceAllUsesWith(Replacements[i].second);
 
diff --git a/contrib/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp b/contrib/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
index 15177650f4e5..4bb6f7a90e93 100644
--- a/contrib/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -206,7 +206,7 @@ bool DAE::DeleteDeadVarargs(Function &Fn) {
 
   // Start by computing a new prototype for the function, which is the same as
   // the old function, but doesn't have isVarArg set.
-  const FunctionType *FTy = Fn.getFunctionType();
+  FunctionType *FTy = Fn.getFunctionType();
 
   std::vector<Type*> Params(FTy->param_begin(), FTy->param_end());
   FunctionType *NFTy = FunctionType::get(FTy->getReturnType(),
@@ -344,7 +344,7 @@ bool DAE::RemoveDeadArgumentsFromCallers(Function &Fn)
 static unsigned NumRetVals(const Function *F) {
   if (F->getReturnType()->isVoidTy())
     return 0;
-  else if (const StructType *STy = dyn_cast<StructType>(F->getReturnType()))
+  else if (StructType *STy = dyn_cast<StructType>(F->getReturnType()))
     return STy->getNumElements();
   else
     return 1;
@@ -491,7 +491,7 @@ void DAE::SurveyFunction(const Function &F) {
   // Keep track of the number of live retvals, so we can skip checks once all
   // of them turn out to be live.
   unsigned NumLiveRetVals = 0;
-  const Type *STy = dyn_cast<StructType>(F.getReturnType());
+  Type *STy = dyn_cast<StructType>(F.getReturnType());
   // Loop all uses of the function.
   for (Value::const_use_iterator I = F.use_begin(), E = F.use_end();
        I != E; ++I) {
@@ -646,7 +646,7 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
 
   // Start by computing a new prototype for the function, which is the same as
   // the old function, but has fewer arguments and a different return type.
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
   std::vector<Type*> Params;
 
   // Set up to build a new list of parameter attributes.
@@ -660,7 +660,7 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
   // Find out the new return value.
 
   Type *RetTy = FTy->getReturnType();
-  const Type *NRetTy = NULL;
+  Type *NRetTy = NULL;
   unsigned RetCount = NumRetVals(F);
 
   // -1 means unused, other numbers are the new index
@@ -669,7 +669,7 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
   if (RetTy->isVoidTy()) {
     NRetTy = RetTy;
   } else {
-    const StructType *STy = dyn_cast<StructType>(RetTy);
+    StructType *STy = dyn_cast<StructType>(RetTy);
     if (STy)
       // Look at each of the original return values individually.
       for (unsigned i = 0; i != RetCount; ++i) {
diff --git a/contrib/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/contrib/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 95decec0f874..0edf3427507b 100644
--- a/contrib/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -163,14 +163,14 @@ bool FunctionAttrs::AddReadAttrs(const CallGraphSCC &SCC) {
           ReadsMemory = true;
         continue;
       } else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-        // Ignore non-volatile loads from local memory.
+        // Ignore non-volatile loads from local memory. (Atomic is okay here.)
         if (!LI->isVolatile()) {
           AliasAnalysis::Location Loc = AA->getLocation(LI);
           if (AA->pointsToConstantMemory(Loc, /*OrLocal=*/true))
             continue;
         }
       } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
-        // Ignore non-volatile stores to local memory.
+        // Ignore non-volatile stores to local memory. (Atomic is okay here.)
         if (!SI->isVolatile()) {
           AliasAnalysis::Location Loc = AA->getLocation(SI);
           if (AA->pointsToConstantMemory(Loc, /*OrLocal=*/true))
diff --git a/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 4ac721dd0600..3552d03919ba 100644
--- a/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -195,12 +195,14 @@ static bool AnalyzeGlobal(const Value *V, GlobalStatus &GS,
       }
       if (const LoadInst *LI = dyn_cast<LoadInst>(I)) {
         GS.isLoaded = true;
-        if (LI->isVolatile()) return true;  // Don't hack on volatile loads.
+        // Don't hack on volatile/atomic loads.
+        if (!LI->isSimple()) return true;
       } else if (const StoreInst *SI = dyn_cast<StoreInst>(I)) {
         // Don't allow a store OF the address, only stores TO the address.
         if (SI->getOperand(0) == V) return true;
 
-        if (SI->isVolatile()) return true;  // Don't hack on volatile stores.
+        // Don't hack on volatile/atomic stores.
+        if (!SI->isSimple()) return true;
 
         // If this is a direct store to the global (i.e., the global is a scalar
         // value, not an aggregate), keep more specific information about
@@ -281,18 +283,18 @@ static Constant *getAggregateConstantElement(Constant *Agg, Constant *Idx) {
   } else if (ConstantVector *CP = dyn_cast<ConstantVector>(Agg)) {
     if (IdxV < CP->getNumOperands()) return CP->getOperand(IdxV);
   } else if (isa<ConstantAggregateZero>(Agg)) {
-    if (const StructType *STy = dyn_cast<StructType>(Agg->getType())) {
+    if (StructType *STy = dyn_cast<StructType>(Agg->getType())) {
       if (IdxV < STy->getNumElements())
         return Constant::getNullValue(STy->getElementType(IdxV));
-    } else if (const SequentialType *STy =
+    } else if (SequentialType *STy =
                dyn_cast<SequentialType>(Agg->getType())) {
       return Constant::getNullValue(STy->getElementType());
     }
   } else if (isa<UndefValue>(Agg)) {
-    if (const StructType *STy = dyn_cast<StructType>(Agg->getType())) {
+    if (StructType *STy = dyn_cast<StructType>(Agg->getType())) {
       if (IdxV < STy->getNumElements())
         return UndefValue::get(STy->getElementType(IdxV));
-    } else if (const SequentialType *STy =
+    } else if (SequentialType *STy =
                dyn_cast<SequentialType>(Agg->getType())) {
       return UndefValue::get(STy->getElementType());
     }
@@ -430,7 +432,7 @@ static bool IsUserOfGlobalSafeForSRA(User *U, GlobalValue *GV) {
   ++GEPI;  // Skip over the pointer index.
 
   // If this is a use of an array allocation, do a bit more checking for sanity.
-  if (const ArrayType *AT = dyn_cast<ArrayType>(*GEPI)) {
+  if (ArrayType *AT = dyn_cast<ArrayType>(*GEPI)) {
     uint64_t NumElements = AT->getNumElements();
     ConstantInt *Idx = cast<ConstantInt>(U->getOperand(2));
 
@@ -451,9 +453,9 @@ static bool IsUserOfGlobalSafeForSRA(User *U, GlobalValue *GV) {
          GEPI != E;
          ++GEPI) {
       uint64_t NumElements;
-      if (const ArrayType *SubArrayTy = dyn_cast<ArrayType>(*GEPI))
+      if (ArrayType *SubArrayTy = dyn_cast<ArrayType>(*GEPI))
         NumElements = SubArrayTy->getNumElements();
-      else if (const VectorType *SubVectorTy = dyn_cast<VectorType>(*GEPI))
+      else if (VectorType *SubVectorTy = dyn_cast<VectorType>(*GEPI))
         NumElements = SubVectorTy->getNumElements();
       else {
         assert((*GEPI)->isStructTy() &&
@@ -498,7 +500,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const TargetData &TD) {
 
   assert(GV->hasLocalLinkage() && !GV->isConstant());
   Constant *Init = GV->getInitializer();
-  const Type *Ty = Init->getType();
+  Type *Ty = Init->getType();
 
   std::vector<GlobalVariable*> NewGlobals;
   Module::GlobalListType &Globals = GV->getParent()->getGlobalList();
@@ -508,7 +510,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const TargetData &TD) {
   if (StartAlignment == 0)
     StartAlignment = TD.getABITypeAlignment(GV->getType());
 
-  if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+  if (StructType *STy = dyn_cast<StructType>(Ty)) {
     NewGlobals.reserve(STy->getNumElements());
     const StructLayout &Layout = *TD.getStructLayout(STy);
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
@@ -531,9 +533,9 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const TargetData &TD) {
       if (NewAlign > TD.getABITypeAlignment(STy->getElementType(i)))
         NGV->setAlignment(NewAlign);
     }
-  } else if (const SequentialType *STy = dyn_cast<SequentialType>(Ty)) {
+  } else if (SequentialType *STy = dyn_cast<SequentialType>(Ty)) {
     unsigned NumElements = 0;
-    if (const ArrayType *ATy = dyn_cast<ArrayType>(STy))
+    if (ArrayType *ATy = dyn_cast<ArrayType>(STy))
       NumElements = ATy->getNumElements();
     else
       NumElements = cast<VectorType>(STy)->getNumElements();
@@ -596,15 +598,14 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const TargetData &TD) {
         Idxs.push_back(NullInt);
         for (unsigned i = 3, e = CE->getNumOperands(); i != e; ++i)
           Idxs.push_back(CE->getOperand(i));
-        NewPtr = ConstantExpr::getGetElementPtr(cast<Constant>(NewPtr),
-                                                &Idxs[0], Idxs.size());
+        NewPtr = ConstantExpr::getGetElementPtr(cast<Constant>(NewPtr), Idxs);
       } else {
         GetElementPtrInst *GEPI = cast<GetElementPtrInst>(GEP);
         SmallVector<Value*, 8> Idxs;
         Idxs.push_back(NullInt);
         for (unsigned i = 3, e = GEPI->getNumOperands(); i != e; ++i)
           Idxs.push_back(GEPI->getOperand(i));
-        NewPtr = GetElementPtrInst::Create(NewPtr, Idxs.begin(), Idxs.end(),
+        NewPtr = GetElementPtrInst::Create(NewPtr, Idxs,
                                            GEPI->getName()+"."+Twine(Val),GEPI);
       }
     }
@@ -753,8 +754,7 @@ static bool OptimizeAwayTrappingUsesOfValue(Value *V, Constant *NewV) {
           break;
       if (Idxs.size() == GEPI->getNumOperands()-1)
         Changed |= OptimizeAwayTrappingUsesOfValue(GEPI,
-                          ConstantExpr::getGetElementPtr(NewV, &Idxs[0],
-                                                        Idxs.size()));
+                          ConstantExpr::getGetElementPtr(NewV, Idxs));
       if (GEPI->use_empty()) {
         Changed = true;
         GEPI->eraseFromParent();
@@ -846,12 +846,12 @@ static void ConstantPropUsersOf(Value *V) {
 /// malloc into a global, and any loads of GV as uses of the new global.
 static GlobalVariable *OptimizeGlobalAddressOfMalloc(GlobalVariable *GV,
                                                      CallInst *CI,
-                                                     const Type *AllocTy,
+                                                     Type *AllocTy,
                                                      ConstantInt *NElements,
                                                      TargetData* TD) {
   DEBUG(errs() << "PROMOTING GLOBAL: " << *GV << "  CALL = " << *CI << '\n');
 
-  const Type *GlobalType;
+  Type *GlobalType;
   if (NElements->getZExtValue() == 1)
     GlobalType = AllocTy;
   else
@@ -1192,7 +1192,7 @@ static Value *GetHeapSROAValue(Value *V, unsigned FieldNo,
   } else if (PHINode *PN = dyn_cast<PHINode>(V)) {
     // PN's type is pointer to struct.  Make a new PHI of pointer to struct
     // field.
-    const StructType *ST =
+    StructType *ST =
       cast<StructType>(cast<PointerType>(PN->getType())->getElementType());
 
     PHINode *NewPN =
@@ -1245,8 +1245,7 @@ static void RewriteHeapSROALoadUser(Instruction *LoadUser,
     GEPIdx.push_back(GEPI->getOperand(1));
     GEPIdx.append(GEPI->op_begin()+3, GEPI->op_end());
 
-    Value *NGEPI = GetElementPtrInst::Create(NewPtr,
-                                             GEPIdx.begin(), GEPIdx.end(),
+    Value *NGEPI = GetElementPtrInst::Create(NewPtr, GEPIdx,
                                              GEPI->getName(), GEPI);
     GEPI->replaceAllUsesWith(NGEPI);
     GEPI->eraseFromParent();
@@ -1260,11 +1259,9 @@ static void RewriteHeapSROALoadUser(Instruction *LoadUser,
   // already been seen first by another load, so its uses have already been
   // processed.
   PHINode *PN = cast<PHINode>(LoadUser);
-  bool Inserted;
-  DenseMap<Value*, std::vector<Value*> >::iterator InsertPos;
-  tie(InsertPos, Inserted) =
-    InsertedScalarizedValues.insert(std::make_pair(PN, std::vector<Value*>()));
-  if (!Inserted) return;
+  if (!InsertedScalarizedValues.insert(std::make_pair(PN,
+                                              std::vector<Value*>())).second)
+    return;
 
   // If this is the first time we've seen this PHI, recursively process all
   // users.
@@ -1298,8 +1295,8 @@ static void RewriteUsesOfLoadForHeapSRoA(LoadInst *Load,
 static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
                                             Value* NElems, TargetData *TD) {
   DEBUG(dbgs() << "SROA HEAP ALLOC: " << *GV << "  MALLOC = " << *CI << '\n');
-  const Type* MAT = getMallocAllocatedType(CI);
-  const StructType *STy = cast<StructType>(MAT);
+  Type* MAT = getMallocAllocatedType(CI);
+  StructType *STy = cast<StructType>(MAT);
 
   // There is guaranteed to be at least one use of the malloc (storing
   // it into GV).  If there are other uses, change them to be uses of
@@ -1313,8 +1310,8 @@ static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
   std::vector<Value*> FieldMallocs;
 
   for (unsigned FieldNo = 0, e = STy->getNumElements(); FieldNo != e;++FieldNo){
-    const Type *FieldTy = STy->getElementType(FieldNo);
-    const PointerType *PFieldTy = PointerType::getUnqual(FieldTy);
+    Type *FieldTy = STy->getElementType(FieldNo);
+    PointerType *PFieldTy = PointerType::getUnqual(FieldTy);
 
     GlobalVariable *NGV =
       new GlobalVariable(*GV->getParent(),
@@ -1325,9 +1322,9 @@ static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
     FieldGlobals.push_back(NGV);
 
     unsigned TypeSize = TD->getTypeAllocSize(FieldTy);
-    if (const StructType *ST = dyn_cast<StructType>(FieldTy))
+    if (StructType *ST = dyn_cast<StructType>(FieldTy))
       TypeSize = TD->getStructLayout(ST)->getSizeInBytes();
-    const Type *IntPtrTy = TD->getIntPtrType(CI->getContext());
+    Type *IntPtrTy = TD->getIntPtrType(CI->getContext());
     Value *NMI = CallInst::CreateMalloc(CI, IntPtrTy, FieldTy,
                                         ConstantInt::get(IntPtrTy, TypeSize),
                                         NElems, 0,
@@ -1379,8 +1376,7 @@ static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
   for (unsigned i = 0, e = FieldGlobals.size(); i != e; ++i) {
     Value *GVVal = new LoadInst(FieldGlobals[i], "tmp", NullPtrBlock);
     Value *Cmp = new ICmpInst(*NullPtrBlock, ICmpInst::ICMP_NE, GVVal,
-                              Constant::getNullValue(GVVal->getType()),
-                              "tmp");
+                              Constant::getNullValue(GVVal->getType()));
     BasicBlock *FreeBlock = BasicBlock::Create(Cmp->getContext(), "free_it",
                                                OrigBB->getParent());
     BasicBlock *NextBlock = BasicBlock::Create(Cmp->getContext(), "next",
@@ -1428,7 +1424,7 @@ static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
 
     // Insert a store of null into each global.
     for (unsigned i = 0, e = FieldGlobals.size(); i != e; ++i) {
-      const PointerType *PT = cast<PointerType>(FieldGlobals[i]->getType());
+      PointerType *PT = cast<PointerType>(FieldGlobals[i]->getType());
       Constant *Null = Constant::getNullValue(PT->getElementType());
       new StoreInst(Null, FieldGlobals[i], SI);
     }
@@ -1485,7 +1481,7 @@ static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
 /// cast of malloc.
 static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
                                                CallInst *CI,
-                                               const Type *AllocTy,
+                                               Type *AllocTy,
                                                Module::global_iterator &GVI,
                                                TargetData *TD) {
   if (!TD)
@@ -1538,10 +1534,10 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
   // If this is an allocation of a fixed size array of structs, analyze as a
   // variable size array.  malloc [100 x struct],1 -> malloc struct, 100
   if (NElems == ConstantInt::get(CI->getArgOperand(0)->getType(), 1))
-    if (const ArrayType *AT = dyn_cast<ArrayType>(AllocTy))
+    if (ArrayType *AT = dyn_cast<ArrayType>(AllocTy))
       AllocTy = AT->getElementType();
 
-  const StructType *AllocSTy = dyn_cast<StructType>(AllocTy);
+  StructType *AllocSTy = dyn_cast<StructType>(AllocTy);
   if (!AllocSTy)
     return false;
 
@@ -1552,8 +1548,8 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
 
     // If this is a fixed size array, transform the Malloc to be an alloc of
     // structs.  malloc [100 x struct],1 -> malloc struct, 100
-    if (const ArrayType *AT = dyn_cast<ArrayType>(getMallocAllocatedType(CI))) {
-      const Type *IntPtrTy = TD->getIntPtrType(CI->getContext());
+    if (ArrayType *AT = dyn_cast<ArrayType>(getMallocAllocatedType(CI))) {
+      Type *IntPtrTy = TD->getIntPtrType(CI->getContext());
       unsigned TypeSize = TD->getStructLayout(AllocSTy)->getSizeInBytes();
       Value *AllocSize = ConstantInt::get(IntPtrTy, TypeSize);
       Value *NumElements = ConstantInt::get(IntPtrTy, AT->getNumElements());
@@ -1596,7 +1592,7 @@ static bool OptimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,
       if (OptimizeAwayTrappingUsesOfLoads(GV, SOVC))
         return true;
     } else if (CallInst *CI = extractMallocCall(StoredOnceVal)) {
-      const Type* MallocType = getMallocAllocatedType(CI);
+      Type* MallocType = getMallocAllocatedType(CI);
       if (MallocType && TryToOptimizeStoreOfMallocToGlobal(GV, CI, MallocType,
                                                            GVI, TD))
         return true;
@@ -1611,7 +1607,7 @@ static bool OptimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,
 /// can shrink the global into a boolean and select between the two values
 /// whenever it is used.  This exposes the values to other scalar optimizations.
 static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) {
-  const Type *GVElType = GV->getType()->getElementType();
+  Type *GVElType = GV->getType()->getElementType();
 
   // If GVElType is already i1, it is already shrunk.  If the type of the GV is
   // an FP value, pointer or vector, don't do this optimization because a select
@@ -1761,7 +1757,7 @@ bool GlobalOpt::ProcessInternalGlobal(GlobalVariable *GV,
     DEBUG(dbgs() << "LOCALIZING GLOBAL: " << *GV);
     Instruction& FirstI = const_cast<Instruction&>(*GS.AccessingFunction
                                                    ->getEntryBlock().begin());
-    const Type* ElemTy = GV->getType()->getElementType();
+    Type* ElemTy = GV->getType()->getElementType();
     // FIXME: Pass Global's alignment when globals have alignment
     AllocaInst* Alloca = new AllocaInst(ElemTy, NULL, GV->getName(), &FirstI);
     if (!isa<UndefValue>(GV->getInitializer()))
@@ -2003,7 +1999,7 @@ static GlobalVariable *InstallGlobalCtors(GlobalVariable *GCL,
   CSVals[0] = ConstantInt::get(Type::getInt32Ty(GCL->getContext()), 65535);
   CSVals[1] = 0;
 
-  const StructType *StructTy =
+  StructType *StructTy =
     cast <StructType>(
     cast<ArrayType>(GCL->getType()->getElementType())->getElementType());
 
@@ -2013,9 +2009,9 @@ static GlobalVariable *InstallGlobalCtors(GlobalVariable *GCL,
     if (Ctors[i]) {
       CSVals[1] = Ctors[i];
     } else {
-      const Type *FTy = FunctionType::get(Type::getVoidTy(GCL->getContext()),
+      Type *FTy = FunctionType::get(Type::getVoidTy(GCL->getContext()),
                                           false);
-      const PointerType *PFTy = PointerType::getUnqual(FTy);
+      PointerType *PFTy = PointerType::getUnqual(FTy);
       CSVals[1] = Constant::getNullValue(PFTy);
       CSVals[0] = ConstantInt::get(Type::getInt32Ty(GCL->getContext()),
                                    0x7fffffff);
@@ -2196,7 +2192,7 @@ static Constant *EvaluateStoreInto(Constant *Init, Constant *Val,
   }
 
   std::vector<Constant*> Elts;
-  if (const StructType *STy = dyn_cast<StructType>(Init->getType())) {
+  if (StructType *STy = dyn_cast<StructType>(Init->getType())) {
 
     // Break up the constant into its elements.
     if (ConstantStruct *CS = dyn_cast<ConstantStruct>(Init)) {
@@ -2224,10 +2220,10 @@ static Constant *EvaluateStoreInto(Constant *Init, Constant *Val,
   }
   
   ConstantInt *CI = cast<ConstantInt>(Addr->getOperand(OpNo));
-  const SequentialType *InitTy = cast<SequentialType>(Init->getType());
+  SequentialType *InitTy = cast<SequentialType>(Init->getType());
 
   uint64_t NumElts;
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(InitTy))
+  if (ArrayType *ATy = dyn_cast<ArrayType>(InitTy))
     NumElts = ATy->getNumElements();
   else
     NumElts = cast<VectorType>(InitTy)->getNumElements();
@@ -2338,7 +2334,7 @@ static bool EvaluateFunction(Function *F, Constant *&RetVal,
     Constant *InstResult = 0;
 
     if (StoreInst *SI = dyn_cast<StoreInst>(CurInst)) {
-      if (SI->isVolatile()) return false;  // no volatile accesses.
+      if (!SI->isSimple()) return false;  // no volatile/atomic accesses.
       Constant *Ptr = getVal(Values, SI->getOperand(1));
       if (!isSimpleEnoughPointerToCommit(Ptr))
         // If this is too complex for us to commit, reject it.
@@ -2358,7 +2354,7 @@ static bool EvaluateFunction(Function *F, Constant *&RetVal,
           // stored value.
           Ptr = CE->getOperand(0);
           
-          const Type *NewTy=cast<PointerType>(Ptr->getType())->getElementType();
+          Type *NewTy=cast<PointerType>(Ptr->getType())->getElementType();
           
           // In order to push the bitcast onto the stored value, a bitcast
           // from NewTy to Val's type must be legal.  If it's not, we can try
@@ -2367,14 +2363,14 @@ static bool EvaluateFunction(Function *F, Constant *&RetVal,
             // If NewTy is a struct, we can convert the pointer to the struct
             // into a pointer to its first member.
             // FIXME: This could be extended to support arrays as well.
-            if (const StructType *STy = dyn_cast<StructType>(NewTy)) {
+            if (StructType *STy = dyn_cast<StructType>(NewTy)) {
               NewTy = STy->getTypeAtIndex(0U);
 
-              const IntegerType *IdxTy =IntegerType::get(NewTy->getContext(), 32);
+              IntegerType *IdxTy =IntegerType::get(NewTy->getContext(), 32);
               Constant *IdxZero = ConstantInt::get(IdxTy, 0, false);
               Constant * const IdxList[] = {IdxZero, IdxZero};
 
-              Ptr = ConstantExpr::getGetElementPtr(Ptr, IdxList, 2);
+              Ptr = ConstantExpr::getGetElementPtr(Ptr, IdxList);
             
             // If we can't improve the situation by introspecting NewTy,
             // we have to give up.
@@ -2411,17 +2407,17 @@ static bool EvaluateFunction(Function *F, Constant *&RetVal,
       for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end();
            i != e; ++i)
         GEPOps.push_back(getVal(Values, *i));
-      InstResult = cast<GEPOperator>(GEP)->isInBounds() ?
-          ConstantExpr::getInBoundsGetElementPtr(P, &GEPOps[0], GEPOps.size()) :
-          ConstantExpr::getGetElementPtr(P, &GEPOps[0], GEPOps.size());
+      InstResult =
+        ConstantExpr::getGetElementPtr(P, GEPOps,
+                                       cast<GEPOperator>(GEP)->isInBounds());
     } else if (LoadInst *LI = dyn_cast<LoadInst>(CurInst)) {
-      if (LI->isVolatile()) return false;  // no volatile accesses.
+      if (!LI->isSimple()) return false;  // no volatile/atomic accesses.
       InstResult = ComputeLoadResult(getVal(Values, LI->getOperand(0)),
                                      MutatedMemory);
       if (InstResult == 0) return false; // Could not evaluate load.
     } else if (AllocaInst *AI = dyn_cast<AllocaInst>(CurInst)) {
       if (AI->isArrayAllocation()) return false;  // Cannot handle array allocs.
-      const Type *Ty = AI->getType()->getElementType();
+      Type *Ty = AI->getType()->getElementType();
       AllocaTmps.push_back(new GlobalVariable(Ty, false,
                                               GlobalValue::InternalLinkage,
                                               UndefValue::get(Ty),
@@ -2465,8 +2461,7 @@ static bool EvaluateFunction(Function *F, Constant *&RetVal,
 
       if (Callee->isDeclaration()) {
         // If this is a function we can constant fold, do it.
-        if (Constant *C = ConstantFoldCall(Callee, Formals.data(),
-                                           Formals.size())) {
+        if (Constant *C = ConstantFoldCall(Callee, Formals)) {
           InstResult = C;
         } else {
           return false;
@@ -2512,7 +2507,7 @@ static bool EvaluateFunction(Function *F, Constant *&RetVal,
         CallStack.pop_back();  // return from fn.
         return true;  // We succeeded at evaluating this ctor!
       } else {
-        // invoke, unwind, unreachable.
+        // invoke, unwind, resume, unreachable.
         return false;  // Cannot handle this terminator.
       }
 
@@ -2711,7 +2706,7 @@ static Function *FindCXAAtExit(Module &M) {
   if (!Fn)
     return 0;
   
-  const FunctionType *FTy = Fn->getFunctionType();
+  FunctionType *FTy = Fn->getFunctionType();
   
   // Checking that the function has the right return type, the right number of 
   // parameters and that they all have pointer types should be enough.
diff --git a/contrib/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp b/contrib/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
index 25c01346642b..d757e1fdb1da 100644
--- a/contrib/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
@@ -167,7 +167,7 @@ bool IPCP::PropagateConstantReturn(Function &F) {
     
   // Check to see if this function returns a constant.
   SmallVector<Value *,4> RetVals;
-  const StructType *STy = dyn_cast<StructType>(F.getReturnType());
+  StructType *STy = dyn_cast<StructType>(F.getReturnType());
   if (STy)
     for (unsigned i = 0, e = STy->getNumElements(); i < e; ++i) 
       RetVals.push_back(UndefValue::get(STy->getElementType(i)));
diff --git a/contrib/llvm/lib/Transforms/IPO/IPO.cpp b/contrib/llvm/lib/Transforms/IPO/IPO.cpp
index 31ce95f53d33..6233922db927 100644
--- a/contrib/llvm/lib/Transforms/IPO/IPO.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/IPO.cpp
@@ -13,6 +13,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm-c/Initialization.h"
 #include "llvm-c/Transforms/IPO.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/PassManager.h"
@@ -35,7 +36,6 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeLoopExtractorPass(Registry);
   initializeBlockExtractorPassPass(Registry);
   initializeSingleLoopExtractorPass(Registry);
-  initializeLowerSetJmpPass(Registry);
   initializeMergeFunctionsPass(Registry);
   initializePartialInlinerPass(Registry);
   initializePruneEHPass(Registry);
@@ -70,6 +70,10 @@ void LLVMAddFunctionInliningPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createFunctionInliningPass());
 }
 
+void LLVMAddAlwaysInlinerPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(llvm::createAlwaysInlinerPass());
+}
+
 void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createGlobalDCEPass());
 }
@@ -82,10 +86,6 @@ void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createIPConstantPropagationPass());
 }
 
-void LLVMAddLowerSetJmpPass(LLVMPassManagerRef PM) {
-  unwrap(PM)->add(createLowerSetJmpPass());
-}
-
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createPruneEHPass());
 }
@@ -98,11 +98,6 @@ void LLVMAddInternalizePass(LLVMPassManagerRef PM, unsigned AllButMain) {
   unwrap(PM)->add(createInternalizePass(AllButMain != 0));
 }
 
-
-void LLVMAddRaiseAllocationsPass(LLVMPassManagerRef PM) {
-  // FIXME: Remove in LLVM 3.0.
-}
-
 void LLVMAddStripDeadPrototypesPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createStripDeadPrototypesPass());
 }
diff --git a/contrib/llvm/lib/Transforms/IPO/InlineAlways.cpp b/contrib/llvm/lib/Transforms/IPO/InlineAlways.cpp
index ce795b72438d..c0426da2c687 100644
--- a/contrib/llvm/lib/Transforms/IPO/InlineAlways.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/InlineAlways.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/InlinerPass.h"
+#include "llvm/Target/TargetData.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace llvm;
@@ -32,10 +33,10 @@ namespace {
   // AlwaysInliner only inlines functions that are mark as "always inline".
   class AlwaysInliner : public Inliner {
     // Functions that are never inlined
-    SmallPtrSet<const Function*, 16> NeverInline; 
+    SmallPtrSet<const Function*, 16> NeverInline;
     InlineCostAnalyzer CA;
   public:
-    // Use extremely low threshold. 
+    // Use extremely low threshold.
     AlwaysInliner() : Inliner(ID, -2000000000) {
       initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry());
     }
@@ -52,8 +53,8 @@ namespace {
     void growCachedCostInfo(Function* Caller, Function* Callee) {
       CA.growCachedCostInfo(Caller, Callee);
     }
-    virtual bool doFinalization(CallGraph &CG) { 
-      return removeDeadFunctions(CG, &NeverInline); 
+    virtual bool doFinalization(CallGraph &CG) {
+      return removeDeadFunctions(CG, &NeverInline);
     }
     virtual bool doInitialization(CallGraph &CG);
     void releaseMemory() {
@@ -71,11 +72,13 @@ INITIALIZE_PASS_END(AlwaysInliner, "always-inline",
 
 Pass *llvm::createAlwaysInlinerPass() { return new AlwaysInliner(); }
 
-// doInitialization - Initializes the vector of functions that have not 
+// doInitialization - Initializes the vector of functions that have not
 // been annotated with the "always inline" attribute.
 bool AlwaysInliner::doInitialization(CallGraph &CG) {
+  CA.setTargetData(getAnalysisIfAvailable<TargetData>());
+
   Module &M = CG.getModule();
-  
+
   for (Module::iterator I = M.begin(), E = M.end();
        I != E; ++I)
     if (!I->isDeclaration() && !I->hasFnAttr(Attribute::AlwaysInline))
diff --git a/contrib/llvm/lib/Transforms/IPO/InlineSimple.cpp b/contrib/llvm/lib/Transforms/IPO/InlineSimple.cpp
index 0c5b3be8f983..84dd4fdd9887 100644
--- a/contrib/llvm/lib/Transforms/IPO/InlineSimple.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/InlineSimple.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/InlinerPass.h"
+#include "llvm/Target/TargetData.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace llvm;
@@ -30,7 +31,7 @@ namespace {
 
   class SimpleInliner : public Inliner {
     // Functions that are never inlined
-    SmallPtrSet<const Function*, 16> NeverInline; 
+    SmallPtrSet<const Function*, 16> NeverInline;
     InlineCostAnalyzer CA;
   public:
     SimpleInliner() : Inliner(ID) {
@@ -68,16 +69,17 @@ INITIALIZE_PASS_END(SimpleInliner, "inline",
 
 Pass *llvm::createFunctionInliningPass() { return new SimpleInliner(); }
 
-Pass *llvm::createFunctionInliningPass(int Threshold) { 
+Pass *llvm::createFunctionInliningPass(int Threshold) {
   return new SimpleInliner(Threshold);
 }
 
 // doInitialization - Initializes the vector of functions that have been
 // annotated with the noinline attribute.
 bool SimpleInliner::doInitialization(CallGraph &CG) {
-  
+  CA.setTargetData(getAnalysisIfAvailable<TargetData>());
+
   Module &M = CG.getModule();
-  
+
   for (Module::iterator I = M.begin(), E = M.end();
        I != E; ++I)
     if (!I->isDeclaration() && I->hasFnAttr(Attribute::NoInline))
@@ -85,34 +87,34 @@ bool SimpleInliner::doInitialization(CallGraph &CG) {
 
   // Get llvm.noinline
   GlobalVariable *GV = M.getNamedGlobal("llvm.noinline");
-  
+
   if (GV == 0)
     return false;
 
   // Don't crash on invalid code
   if (!GV->hasDefinitiveInitializer())
     return false;
-  
+
   const ConstantArray *InitList = dyn_cast<ConstantArray>(GV->getInitializer());
-  
+
   if (InitList == 0)
     return false;
 
   // Iterate over each element and add to the NeverInline set
   for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) {
-        
+
     // Get Source
     const Constant *Elt = InitList->getOperand(i);
-        
+
     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Elt))
-      if (CE->getOpcode() == Instruction::BitCast) 
+      if (CE->getOpcode() == Instruction::BitCast)
         Elt = CE->getOperand(0);
-    
+
     // Insert into set of functions to never inline
     if (const Function *F = dyn_cast<Function>(Elt))
       NeverInline.insert(F);
   }
-  
+
   return false;
 }
 
diff --git a/contrib/llvm/lib/Transforms/IPO/Inliner.cpp b/contrib/llvm/lib/Transforms/IPO/Inliner.cpp
index 57f3e772b569..f00935b08887 100644
--- a/contrib/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -62,7 +62,7 @@ void Inliner::getAnalysisUsage(AnalysisUsage &Info) const {
 }
 
 
-typedef DenseMap<const ArrayType*, std::vector<AllocaInst*> >
+typedef DenseMap<ArrayType*, std::vector<AllocaInst*> >
 InlinedArrayAllocasTy;
 
 /// InlineCallIfPossible - If it is possible to inline the specified call site,
@@ -139,7 +139,7 @@ static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
     // Don't bother trying to merge array allocations (they will usually be
     // canonicalized to be an allocation *of* an array), or allocations whose
     // type is not itself an array (because we're afraid of pessimizing SRoA).
-    const ArrayType *ATy = dyn_cast<ArrayType>(AI->getAllocatedType());
+    ArrayType *ATy = dyn_cast<ArrayType>(AI->getAllocatedType());
     if (ATy == 0 || AI->isArrayAllocation())
       continue;
     
diff --git a/contrib/llvm/lib/Transforms/IPO/LoopExtractor.cpp b/contrib/llvm/lib/Transforms/IPO/LoopExtractor.cpp
index 848944dc9381..4f96afe44c0f 100644
--- a/contrib/llvm/lib/Transforms/IPO/LoopExtractor.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/LoopExtractor.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/FunctionUtils.h"
 #include "llvm/ADT/Statistic.h"
 #include <fstream>
@@ -53,12 +54,12 @@ namespace {
 
 char LoopExtractor::ID = 0;
 INITIALIZE_PASS_BEGIN(LoopExtractor, "loop-extract",
-                "Extract loops into new functions", false, false)
+                      "Extract loops into new functions", false, false)
 INITIALIZE_PASS_DEPENDENCY(BreakCriticalEdges)
 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
 INITIALIZE_PASS_DEPENDENCY(DominatorTree)
 INITIALIZE_PASS_END(LoopExtractor, "loop-extract",
-                "Extract loops into new functions", false, false)
+                    "Extract loops into new functions", false, false)
 
 namespace {
   /// SingleLoopExtractor - For bugpoint.
@@ -100,9 +101,9 @@ bool LoopExtractor::runOnLoop(Loop *L, LPPassManager &LPM) {
     L->getHeader()->getParent()->getEntryBlock().getTerminator();
   if (!isa<BranchInst>(EntryTI) ||
       !cast<BranchInst>(EntryTI)->isUnconditional() ||
-      EntryTI->getSuccessor(0) != L->getHeader())
+      EntryTI->getSuccessor(0) != L->getHeader()) {
     ShouldExtractLoop = true;
-  else {
+  } else {
     // Check to see if any exits from the loop are more than just return
     // blocks.
     SmallVector<BasicBlock*, 8> ExitBlocks;
@@ -113,6 +114,21 @@ bool LoopExtractor::runOnLoop(Loop *L, LPPassManager &LPM) {
         break;
       }
   }
+
+  if (ShouldExtractLoop) {
+    // We must omit landing pads. Landing pads must accompany the invoke
+    // instruction. But this would result in a loop in the extracted
+    // function. An infinite cycle occurs when it tries to extract that loop as
+    // well.
+    SmallVector<BasicBlock*, 8> ExitBlocks;
+    L->getExitBlocks(ExitBlocks);
+    for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i)
+      if (ExitBlocks[i]->isLandingPad()) {
+        ShouldExtractLoop = false;
+        break;
+      }
+  }
+
   if (ShouldExtractLoop) {
     if (NumLoops == 0) return Changed;
     --NumLoops;
@@ -149,6 +165,7 @@ namespace {
   /// BlocksToNotExtract list.
   class BlockExtractorPass : public ModulePass {
     void LoadFile(const char *Filename);
+    void SplitLandingPadPreds(Function *F);
 
     std::vector<BasicBlock*> BlocksToNotExtract;
     std::vector<std::pair<std::string, std::string> > BlocksToNotExtractByName;
@@ -171,8 +188,7 @@ INITIALIZE_PASS(BlockExtractorPass, "extract-blocks",
 // createBlockExtractorPass - This pass extracts all blocks (except those
 // specified in the argument list) from the functions in the module.
 //
-ModulePass *llvm::createBlockExtractorPass()
-{
+ModulePass *llvm::createBlockExtractorPass() {
   return new BlockExtractorPass();
 }
 
@@ -194,6 +210,37 @@ void BlockExtractorPass::LoadFile(const char *Filename) {
   }
 }
 
+/// SplitLandingPadPreds - The landing pad needs to be extracted with the invoke
+/// instruction. The critical edge breaker will refuse to break critical edges
+/// to a landing pad. So do them here. After this method runs, all landing pads
+/// should have only one predecessor.
+void BlockExtractorPass::SplitLandingPadPreds(Function *F) {
+  for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) {
+    InvokeInst *II = dyn_cast<InvokeInst>(I);
+    if (!II) continue;
+    BasicBlock *Parent = II->getParent();
+    BasicBlock *LPad = II->getUnwindDest();
+
+    // Look through the landing pad's predecessors. If one of them ends in an
+    // 'invoke', then we want to split the landing pad.
+    bool Split = false;
+    for (pred_iterator
+           PI = pred_begin(LPad), PE = pred_end(LPad); PI != PE; ++PI) {
+      BasicBlock *BB = *PI;
+      if (BB->isLandingPad() && BB != Parent &&
+          isa<InvokeInst>(Parent->getTerminator())) {
+        Split = true;
+        break;
+      }
+    }
+
+    if (!Split) continue;
+
+    SmallVector<BasicBlock*, 2> NewBBs;
+    SplitLandingPadPredecessors(LPad, Parent, ".1", ".2", 0, NewBBs);
+  }
+}
+
 bool BlockExtractorPass::runOnModule(Module &M) {
   std::set<BasicBlock*> TranslatedBlocksToNotExtract;
   for (unsigned i = 0, e = BlocksToNotExtract.size(); i != e; ++i) {
@@ -236,13 +283,21 @@ bool BlockExtractorPass::runOnModule(Module &M) {
   // Now that we know which blocks to not extract, figure out which ones we WANT
   // to extract.
   std::vector<BasicBlock*> BlocksToExtract;
-  for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F)
+  for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F) {
+    SplitLandingPadPreds(&*F);
     for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
       if (!TranslatedBlocksToNotExtract.count(BB))
         BlocksToExtract.push_back(BB);
+  }
 
-  for (unsigned i = 0, e = BlocksToExtract.size(); i != e; ++i)
-    ExtractBasicBlock(BlocksToExtract[i]);
+  for (unsigned i = 0, e = BlocksToExtract.size(); i != e; ++i) {
+    SmallVector<BasicBlock*, 2> BlocksToExtractVec;
+    BlocksToExtractVec.push_back(BlocksToExtract[i]);
+    if (const InvokeInst *II =
+        dyn_cast<InvokeInst>(BlocksToExtract[i]->getTerminator()))
+      BlocksToExtractVec.push_back(II->getUnwindDest());
+    ExtractBasicBlock(BlocksToExtractVec);
+  }
 
   return !BlocksToExtract.empty();
 }
diff --git a/contrib/llvm/lib/Transforms/IPO/LowerSetJmp.cpp b/contrib/llvm/lib/Transforms/IPO/LowerSetJmp.cpp
deleted file mode 100644
index 659476b139e4..000000000000
--- a/contrib/llvm/lib/Transforms/IPO/LowerSetJmp.cpp
+++ /dev/null
@@ -1,547 +0,0 @@
-//===- LowerSetJmp.cpp - Code pertaining to lowering set/long jumps -------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file implements the lowering of setjmp and longjmp to use the
-//  LLVM invoke and unwind instructions as necessary.
-//
-//  Lowering of longjmp is fairly trivial. We replace the call with a
-//  call to the LLVM library function "__llvm_sjljeh_throw_longjmp()".
-//  This unwinds the stack for us calling all of the destructors for
-//  objects allocated on the stack.
-//
-//  At a setjmp call, the basic block is split and the setjmp removed.
-//  The calls in a function that have a setjmp are converted to invoke
-//  where the except part checks to see if it's a longjmp exception and,
-//  if so, if it's handled in the function. If it is, then it gets the
-//  value returned by the longjmp and goes to where the basic block was
-//  split. Invoke instructions are handled in a similar fashion with the
-//  original except block being executed if it isn't a longjmp except
-//  that is handled by that function.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// FIXME: This pass doesn't deal with PHI statements just yet. That is,
-// we expect this to occur before SSAification is done. This would seem
-// to make sense, but in general, it might be a good idea to make this
-// pass invokable via the "opt" command at will.
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "lowersetjmp"
-#include "llvm/Transforms/IPO.h"
-#include "llvm/Constants.h"
-#include "llvm/DerivedTypes.h"
-#include "llvm/Instructions.h"
-#include "llvm/Intrinsics.h"
-#include "llvm/LLVMContext.h"
-#include "llvm/Module.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/CallSite.h"
-#include "llvm/Support/CFG.h"
-#include "llvm/Support/InstVisitor.h"
-#include "llvm/Transforms/Utils/Local.h"
-#include "llvm/ADT/DepthFirstIterator.h"
-#include "llvm/ADT/Statistic.h"
-#include <map>
-using namespace llvm;
-
-STATISTIC(LongJmpsTransformed, "Number of longjmps transformed");
-STATISTIC(SetJmpsTransformed , "Number of setjmps transformed");
-STATISTIC(CallsTransformed   , "Number of calls invokified");
-STATISTIC(InvokesTransformed , "Number of invokes modified");
-
-namespace {
-  //===--------------------------------------------------------------------===//
-  // LowerSetJmp pass implementation.
-  class LowerSetJmp : public ModulePass, public InstVisitor<LowerSetJmp> {
-    // LLVM library functions...
-    Constant *InitSJMap;        // __llvm_sjljeh_init_setjmpmap
-    Constant *DestroySJMap;     // __llvm_sjljeh_destroy_setjmpmap
-    Constant *AddSJToMap;       // __llvm_sjljeh_add_setjmp_to_map
-    Constant *ThrowLongJmp;     // __llvm_sjljeh_throw_longjmp
-    Constant *TryCatchLJ;       // __llvm_sjljeh_try_catching_longjmp_exception
-    Constant *IsLJException;    // __llvm_sjljeh_is_longjmp_exception
-    Constant *GetLJValue;       // __llvm_sjljeh_get_longjmp_value
-
-    typedef std::pair<SwitchInst*, CallInst*> SwitchValuePair;
-
-    // Keep track of those basic blocks reachable via a depth-first search of
-    // the CFG from a setjmp call. We only need to transform those "call" and
-    // "invoke" instructions that are reachable from the setjmp call site.
-    std::set<BasicBlock*> DFSBlocks;
-
-    // The setjmp map is going to hold information about which setjmps
-    // were called (each setjmp gets its own number) and with which
-    // buffer it was called.
-    std::map<Function*, AllocaInst*>            SJMap;
-
-    // The rethrow basic block map holds the basic block to branch to if
-    // the exception isn't handled in the current function and needs to
-    // be rethrown.
-    std::map<const Function*, BasicBlock*>      RethrowBBMap;
-
-    // The preliminary basic block map holds a basic block that grabs the
-    // exception and determines if it's handled by the current function.
-    std::map<const Function*, BasicBlock*>      PrelimBBMap;
-
-    // The switch/value map holds a switch inst/call inst pair. The
-    // switch inst controls which handler (if any) gets called and the
-    // value is the value returned to that handler by the call to
-    // __llvm_sjljeh_get_longjmp_value.
-    std::map<const Function*, SwitchValuePair>  SwitchValMap;
-
-    // A map of which setjmps we've seen so far in a function.
-    std::map<const Function*, unsigned>         SetJmpIDMap;
-
-    AllocaInst*     GetSetJmpMap(Function* Func);
-    BasicBlock*     GetRethrowBB(Function* Func);
-    SwitchValuePair GetSJSwitch(Function* Func, BasicBlock* Rethrow);
-
-    void TransformLongJmpCall(CallInst* Inst);
-    void TransformSetJmpCall(CallInst* Inst);
-
-    bool IsTransformableFunction(StringRef Name);
-  public:
-    static char ID; // Pass identification, replacement for typeid
-    LowerSetJmp() : ModulePass(ID) {
-      initializeLowerSetJmpPass(*PassRegistry::getPassRegistry());
-    }
-
-    void visitCallInst(CallInst& CI);
-    void visitInvokeInst(InvokeInst& II);
-    void visitReturnInst(ReturnInst& RI);
-    void visitUnwindInst(UnwindInst& UI);
-
-    bool runOnModule(Module& M);
-    bool doInitialization(Module& M);
-  };
-} // end anonymous namespace
-
-char LowerSetJmp::ID = 0;
-INITIALIZE_PASS(LowerSetJmp, "lowersetjmp", "Lower Set Jump", false, false)
-
-// run - Run the transformation on the program. We grab the function
-// prototypes for longjmp and setjmp. If they are used in the program,
-// then we can go directly to the places they're at and transform them.
-bool LowerSetJmp::runOnModule(Module& M) {
-  bool Changed = false;
-
-  // These are what the functions are called.
-  Function* SetJmp = M.getFunction("llvm.setjmp");
-  Function* LongJmp = M.getFunction("llvm.longjmp");
-
-  // This program doesn't have longjmp and setjmp calls.
-  if ((!LongJmp || LongJmp->use_empty()) &&
-        (!SetJmp || SetJmp->use_empty())) return false;
-
-  // Initialize some values and functions we'll need to transform the
-  // setjmp/longjmp functions.
-  doInitialization(M);
-
-  if (SetJmp) {
-    for (Value::use_iterator B = SetJmp->use_begin(), E = SetJmp->use_end();
-         B != E; ++B) {
-      BasicBlock* BB = cast<Instruction>(*B)->getParent();
-      for (df_ext_iterator<BasicBlock*> I = df_ext_begin(BB, DFSBlocks),
-             E = df_ext_end(BB, DFSBlocks); I != E; ++I)
-        /* empty */;
-    }
-
-    while (!SetJmp->use_empty()) {
-      assert(isa<CallInst>(SetJmp->use_back()) &&
-             "User of setjmp intrinsic not a call?");
-      TransformSetJmpCall(cast<CallInst>(SetJmp->use_back()));
-      Changed = true;
-    }
-  }
-
-  if (LongJmp)
-    while (!LongJmp->use_empty()) {
-      assert(isa<CallInst>(LongJmp->use_back()) &&
-             "User of longjmp intrinsic not a call?");
-      TransformLongJmpCall(cast<CallInst>(LongJmp->use_back()));
-      Changed = true;
-    }
-
-  // Now go through the affected functions and convert calls and invokes
-  // to new invokes...
-  for (std::map<Function*, AllocaInst*>::iterator
-      B = SJMap.begin(), E = SJMap.end(); B != E; ++B) {
-    Function* F = B->first;
-    for (Function::iterator BB = F->begin(), BE = F->end(); BB != BE; ++BB)
-      for (BasicBlock::iterator IB = BB->begin(), IE = BB->end(); IB != IE; ) {
-        visit(*IB++);
-        if (IB != BB->end() && IB->getParent() != BB)
-          break;  // The next instruction got moved to a different block!
-      }
-  }
-
-  DFSBlocks.clear();
-  SJMap.clear();
-  RethrowBBMap.clear();
-  PrelimBBMap.clear();
-  SwitchValMap.clear();
-  SetJmpIDMap.clear();
-
-  return Changed;
-}
-
-// doInitialization - For the lower long/setjmp pass, this ensures that a
-// module contains a declaration for the intrisic functions we are going
-// to call to convert longjmp and setjmp calls.
-//
-// This function is always successful, unless it isn't.
-bool LowerSetJmp::doInitialization(Module& M)
-{
-  const Type *SBPTy = Type::getInt8PtrTy(M.getContext());
-  const Type *SBPPTy = PointerType::getUnqual(SBPTy);
-
-  // N.B. See llvm/runtime/GCCLibraries/libexception/SJLJ-Exception.h for
-  // a description of the following library functions.
-
-  // void __llvm_sjljeh_init_setjmpmap(void**)
-  InitSJMap = M.getOrInsertFunction("__llvm_sjljeh_init_setjmpmap",
-                                    Type::getVoidTy(M.getContext()),
-                                    SBPPTy, (Type *)0);
-  // void __llvm_sjljeh_destroy_setjmpmap(void**)
-  DestroySJMap = M.getOrInsertFunction("__llvm_sjljeh_destroy_setjmpmap",
-                                       Type::getVoidTy(M.getContext()),
-                                       SBPPTy, (Type *)0);
-
-  // void __llvm_sjljeh_add_setjmp_to_map(void**, void*, unsigned)
-  AddSJToMap = M.getOrInsertFunction("__llvm_sjljeh_add_setjmp_to_map",
-                                     Type::getVoidTy(M.getContext()),
-                                     SBPPTy, SBPTy,
-                                     Type::getInt32Ty(M.getContext()),
-                                     (Type *)0);
-
-  // void __llvm_sjljeh_throw_longjmp(int*, int)
-  ThrowLongJmp = M.getOrInsertFunction("__llvm_sjljeh_throw_longjmp",
-                                       Type::getVoidTy(M.getContext()), SBPTy, 
-                                       Type::getInt32Ty(M.getContext()),
-                                       (Type *)0);
-
-  // unsigned __llvm_sjljeh_try_catching_longjmp_exception(void **)
-  TryCatchLJ =
-    M.getOrInsertFunction("__llvm_sjljeh_try_catching_longjmp_exception",
-                          Type::getInt32Ty(M.getContext()), SBPPTy, (Type *)0);
-
-  // bool __llvm_sjljeh_is_longjmp_exception()
-  IsLJException = M.getOrInsertFunction("__llvm_sjljeh_is_longjmp_exception",
-                                        Type::getInt1Ty(M.getContext()),
-                                        (Type *)0);
-
-  // int __llvm_sjljeh_get_longjmp_value()
-  GetLJValue = M.getOrInsertFunction("__llvm_sjljeh_get_longjmp_value",
-                                     Type::getInt32Ty(M.getContext()),
-                                     (Type *)0);
-  return true;
-}
-
-// IsTransformableFunction - Return true if the function name isn't one
-// of the ones we don't want transformed. Currently, don't transform any
-// "llvm.{setjmp,longjmp}" functions and none of the setjmp/longjmp error
-// handling functions (beginning with __llvm_sjljeh_...they don't throw
-// exceptions).
-bool LowerSetJmp::IsTransformableFunction(StringRef Name) {
-  return !Name.startswith("__llvm_sjljeh_");
-}
-
-// TransformLongJmpCall - Transform a longjmp call into a call to the
-// internal __llvm_sjljeh_throw_longjmp function. It then takes care of
-// throwing the exception for us.
-void LowerSetJmp::TransformLongJmpCall(CallInst* Inst)
-{
-  const Type* SBPTy = Type::getInt8PtrTy(Inst->getContext());
-
-  // Create the call to "__llvm_sjljeh_throw_longjmp". This takes the
-  // same parameters as "longjmp", except that the buffer is cast to a
-  // char*. It returns "void", so it doesn't need to replace any of
-  // Inst's uses and doesn't get a name.
-  CastInst* CI = 
-    new BitCastInst(Inst->getArgOperand(0), SBPTy, "LJBuf", Inst);
-  Value *Args[] = { CI, Inst->getArgOperand(1) };
-  CallInst::Create(ThrowLongJmp, Args, "", Inst);
-
-  SwitchValuePair& SVP = SwitchValMap[Inst->getParent()->getParent()];
-
-  // If the function has a setjmp call in it (they are transformed first)
-  // we should branch to the basic block that determines if this longjmp
-  // is applicable here. Otherwise, issue an unwind.
-  if (SVP.first)
-    BranchInst::Create(SVP.first->getParent(), Inst);
-  else
-    new UnwindInst(Inst->getContext(), Inst);
-
-  // Remove all insts after the branch/unwind inst.  Go from back to front to
-  // avoid replaceAllUsesWith if possible.
-  BasicBlock *BB = Inst->getParent();
-  Instruction *Removed;
-  do {
-    Removed = &BB->back();
-    // If the removed instructions have any users, replace them now.
-    if (!Removed->use_empty())
-      Removed->replaceAllUsesWith(UndefValue::get(Removed->getType()));
-    Removed->eraseFromParent();
-  } while (Removed != Inst);
-
-  ++LongJmpsTransformed;
-}
-
-// GetSetJmpMap - Retrieve (create and initialize, if necessary) the
-// setjmp map. This map is going to hold information about which setjmps
-// were called (each setjmp gets its own number) and with which buffer it
-// was called. There can be only one!
-AllocaInst* LowerSetJmp::GetSetJmpMap(Function* Func)
-{
-  if (SJMap[Func]) return SJMap[Func];
-
-  // Insert the setjmp map initialization before the first instruction in
-  // the function.
-  Instruction* Inst = Func->getEntryBlock().begin();
-  assert(Inst && "Couldn't find even ONE instruction in entry block!");
-
-  // Fill in the alloca and call to initialize the SJ map.
-  const Type *SBPTy =
-        Type::getInt8PtrTy(Func->getContext());
-  AllocaInst* Map = new AllocaInst(SBPTy, 0, "SJMap", Inst);
-  CallInst::Create(InitSJMap, Map, "", Inst);
-  return SJMap[Func] = Map;
-}
-
-// GetRethrowBB - Only one rethrow basic block is needed per function.
-// If this is a longjmp exception but not handled in this block, this BB
-// performs the rethrow.
-BasicBlock* LowerSetJmp::GetRethrowBB(Function* Func)
-{
-  if (RethrowBBMap[Func]) return RethrowBBMap[Func];
-
-  // The basic block we're going to jump to if we need to rethrow the
-  // exception.
-  BasicBlock* Rethrow =
-        BasicBlock::Create(Func->getContext(), "RethrowExcept", Func);
-
-  // Fill in the "Rethrow" BB with a call to rethrow the exception. This
-  // is the last instruction in the BB since at this point the runtime
-  // should exit this function and go to the next function.
-  new UnwindInst(Func->getContext(), Rethrow);
-  return RethrowBBMap[Func] = Rethrow;
-}
-
-// GetSJSwitch - Return the switch statement that controls which handler
-// (if any) gets called and the value returned to that handler.
-LowerSetJmp::SwitchValuePair LowerSetJmp::GetSJSwitch(Function* Func,
-                                                      BasicBlock* Rethrow)
-{
-  if (SwitchValMap[Func].first) return SwitchValMap[Func];
-
-  BasicBlock* LongJmpPre =
-        BasicBlock::Create(Func->getContext(), "LongJmpBlkPre", Func);
-
-  // Keep track of the preliminary basic block for some of the other
-  // transformations.
-  PrelimBBMap[Func] = LongJmpPre;
-
-  // Grab the exception.
-  CallInst* Cond = CallInst::Create(IsLJException, "IsLJExcept", LongJmpPre);
-
-  // The "decision basic block" gets the number associated with the
-  // setjmp call returning to switch on and the value returned by
-  // longjmp.
-  BasicBlock* DecisionBB =
-        BasicBlock::Create(Func->getContext(), "LJDecisionBB", Func);
-
-  BranchInst::Create(DecisionBB, Rethrow, Cond, LongJmpPre);
-
-  // Fill in the "decision" basic block.
-  CallInst* LJVal = CallInst::Create(GetLJValue, "LJVal", DecisionBB);
-  CallInst* SJNum = CallInst::Create(TryCatchLJ, GetSetJmpMap(Func), "SJNum",
-                                     DecisionBB);
-
-  SwitchInst* SI = SwitchInst::Create(SJNum, Rethrow, 0, DecisionBB);
-  return SwitchValMap[Func] = SwitchValuePair(SI, LJVal);
-}
-
-// TransformSetJmpCall - The setjmp call is a bit trickier to transform.
-// We're going to convert all setjmp calls to nops. Then all "call" and
-// "invoke" instructions in the function are converted to "invoke" where
-// the "except" branch is used when returning from a longjmp call.
-void LowerSetJmp::TransformSetJmpCall(CallInst* Inst)
-{
-  BasicBlock* ABlock = Inst->getParent();
-  Function* Func = ABlock->getParent();
-
-  // Add this setjmp to the setjmp map.
-  const Type* SBPTy =
-          Type::getInt8PtrTy(Inst->getContext());
-  CastInst* BufPtr = 
-    new BitCastInst(Inst->getArgOperand(0), SBPTy, "SBJmpBuf", Inst);
-  Value *Args[] = {
-    GetSetJmpMap(Func), BufPtr,
-    ConstantInt::get(Type::getInt32Ty(Inst->getContext()), SetJmpIDMap[Func]++)
-  };
-  CallInst::Create(AddSJToMap, Args, "", Inst);
-
-  // We are guaranteed that there are no values live across basic blocks
-  // (because we are "not in SSA form" yet), but there can still be values live
-  // in basic blocks.  Because of this, splitting the setjmp block can cause
-  // values above the setjmp to not dominate uses which are after the setjmp
-  // call.  For all of these occasions, we must spill the value to the stack.
-  //
-  std::set<Instruction*> InstrsAfterCall;
-
-  // The call is probably very close to the end of the basic block, for the
-  // common usage pattern of: 'if (setjmp(...))', so keep track of the
-  // instructions after the call.
-  for (BasicBlock::iterator I = ++BasicBlock::iterator(Inst), E = ABlock->end();
-       I != E; ++I)
-    InstrsAfterCall.insert(I);
-
-  for (BasicBlock::iterator II = ABlock->begin();
-       II != BasicBlock::iterator(Inst); ++II)
-    // Loop over all of the uses of instruction.  If any of them are after the
-    // call, "spill" the value to the stack.
-    for (Value::use_iterator UI = II->use_begin(), E = II->use_end();
-         UI != E; ++UI) {
-      User *U = *UI;
-      if (cast<Instruction>(U)->getParent() != ABlock ||
-          InstrsAfterCall.count(cast<Instruction>(U))) {
-        DemoteRegToStack(*II);
-        break;
-      }
-    }
-  InstrsAfterCall.clear();
-
-  // Change the setjmp call into a branch statement. We'll remove the
-  // setjmp call in a little bit. No worries.
-  BasicBlock* SetJmpContBlock = ABlock->splitBasicBlock(Inst);
-  assert(SetJmpContBlock && "Couldn't split setjmp BB!!");
-
-  SetJmpContBlock->setName(ABlock->getName()+"SetJmpCont");
-
-  // Add the SetJmpContBlock to the set of blocks reachable from a setjmp.
-  DFSBlocks.insert(SetJmpContBlock);
-
-  // This PHI node will be in the new block created from the
-  // splitBasicBlock call.
-  PHINode* PHI = PHINode::Create(Type::getInt32Ty(Inst->getContext()), 2,
-                                 "SetJmpReturn", Inst);
-
-  // Coming from a call to setjmp, the return is 0.
-  PHI->addIncoming(Constant::getNullValue(Type::getInt32Ty(Inst->getContext())),
-                   ABlock);
-
-  // Add the case for this setjmp's number...
-  SwitchValuePair SVP = GetSJSwitch(Func, GetRethrowBB(Func));
-  SVP.first->addCase(ConstantInt::get(Type::getInt32Ty(Inst->getContext()),
-                                      SetJmpIDMap[Func] - 1),
-                     SetJmpContBlock);
-
-  // Value coming from the handling of the exception.
-  PHI->addIncoming(SVP.second, SVP.second->getParent());
-
-  // Replace all uses of this instruction with the PHI node created by
-  // the eradication of setjmp.
-  Inst->replaceAllUsesWith(PHI);
-  Inst->eraseFromParent();
-
-  ++SetJmpsTransformed;
-}
-
-// visitCallInst - This converts all LLVM call instructions into invoke
-// instructions. The except part of the invoke goes to the "LongJmpBlkPre"
-// that grabs the exception and proceeds to determine if it's a longjmp
-// exception or not.
-void LowerSetJmp::visitCallInst(CallInst& CI)
-{
-  if (CI.getCalledFunction())
-    if (!IsTransformableFunction(CI.getCalledFunction()->getName()) ||
-        CI.getCalledFunction()->isIntrinsic()) return;
-
-  BasicBlock* OldBB = CI.getParent();
-
-  // If not reachable from a setjmp call, don't transform.
-  if (!DFSBlocks.count(OldBB)) return;
-
-  BasicBlock* NewBB = OldBB->splitBasicBlock(CI);
-  assert(NewBB && "Couldn't split BB of \"call\" instruction!!");
-  DFSBlocks.insert(NewBB);
-  NewBB->setName("Call2Invoke");
-
-  Function* Func = OldBB->getParent();
-
-  // Construct the new "invoke" instruction.
-  TerminatorInst* Term = OldBB->getTerminator();
-  CallSite CS(&CI);
-  std::vector<Value*> Params(CS.arg_begin(), CS.arg_end());
-  InvokeInst* II =
-    InvokeInst::Create(CI.getCalledValue(), NewBB, PrelimBBMap[Func],
-                       Params, CI.getName(), Term);
-  II->setCallingConv(CI.getCallingConv());
-  II->setAttributes(CI.getAttributes());
-
-  // Replace the old call inst with the invoke inst and remove the call.
-  CI.replaceAllUsesWith(II);
-  CI.eraseFromParent();
-
-  // The old terminator is useless now that we have the invoke inst.
-  Term->eraseFromParent();
-  ++CallsTransformed;
-}
-
-// visitInvokeInst - Converting the "invoke" instruction is fairly
-// straight-forward. The old exception part is replaced by a query asking
-// if this is a longjmp exception. If it is, then it goes to the longjmp
-// exception blocks. Otherwise, control is passed the old exception.
-void LowerSetJmp::visitInvokeInst(InvokeInst& II)
-{
-  if (II.getCalledFunction())
-    if (!IsTransformableFunction(II.getCalledFunction()->getName()) ||
-        II.getCalledFunction()->isIntrinsic()) return;
-
-  BasicBlock* BB = II.getParent();
-
-  // If not reachable from a setjmp call, don't transform.
-  if (!DFSBlocks.count(BB)) return;
-
-  BasicBlock* ExceptBB = II.getUnwindDest();
-
-  Function* Func = BB->getParent();
-  BasicBlock* NewExceptBB = BasicBlock::Create(II.getContext(), 
-                                               "InvokeExcept", Func);
-
-  // If this is a longjmp exception, then branch to the preliminary BB of
-  // the longjmp exception handling. Otherwise, go to the old exception.
-  CallInst* IsLJExcept = CallInst::Create(IsLJException, "IsLJExcept",
-                                          NewExceptBB);
-
-  BranchInst::Create(PrelimBBMap[Func], ExceptBB, IsLJExcept, NewExceptBB);
-
-  II.setUnwindDest(NewExceptBB);
-  ++InvokesTransformed;
-}
-
-// visitReturnInst - We want to destroy the setjmp map upon exit from the
-// function.
-void LowerSetJmp::visitReturnInst(ReturnInst &RI) {
-  Function* Func = RI.getParent()->getParent();
-  CallInst::Create(DestroySJMap, GetSetJmpMap(Func), "", &RI);
-}
-
-// visitUnwindInst - We want to destroy the setjmp map upon exit from the
-// function.
-void LowerSetJmp::visitUnwindInst(UnwindInst &UI) {
-  Function* Func = UI.getParent()->getParent();
-  CallInst::Create(DestroySJMap, GetSetJmpMap(Func), "", &UI);
-}
-
-ModulePass *llvm::createLowerSetJmpPass() {
-  return new LowerSetJmp();
-}
-
diff --git a/contrib/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/contrib/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index 7796d05b7bc6..0b01c3822f28 100644
--- a/contrib/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -76,7 +76,7 @@ STATISTIC(NumDoubleWeak, "Number of new functions created");
 /// functions that will compare equal, without looking at the instructions
 /// inside the function.
 static unsigned profileFunction(const Function *F) {
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
 
   FoldingSetNodeID ID;
   ID.AddInteger(F->size());
@@ -185,7 +185,7 @@ class FunctionComparator {
   }
 
   /// Compare two Types, treating all pointer types as equal.
-  bool isEquivalentType(const Type *Ty1, const Type *Ty2) const;
+  bool isEquivalentType(Type *Ty1, Type *Ty2) const;
 
   // The two functions undergoing comparison.
   const Function *F1, *F2;
@@ -200,8 +200,8 @@ class FunctionComparator {
 
 // Any two pointers in the same address space are equivalent, intptr_t and
 // pointers are equivalent. Otherwise, standard type equivalence rules apply.
-bool FunctionComparator::isEquivalentType(const Type *Ty1,
-                                          const Type *Ty2) const {
+bool FunctionComparator::isEquivalentType(Type *Ty1,
+                                          Type *Ty2) const {
   if (Ty1 == Ty2)
     return true;
   if (Ty1->getTypeID() != Ty2->getTypeID()) {
@@ -233,14 +233,14 @@ bool FunctionComparator::isEquivalentType(const Type *Ty1,
     return true;
 
   case Type::PointerTyID: {
-    const PointerType *PTy1 = cast<PointerType>(Ty1);
-    const PointerType *PTy2 = cast<PointerType>(Ty2);
+    PointerType *PTy1 = cast<PointerType>(Ty1);
+    PointerType *PTy2 = cast<PointerType>(Ty2);
     return PTy1->getAddressSpace() == PTy2->getAddressSpace();
   }
 
   case Type::StructTyID: {
-    const StructType *STy1 = cast<StructType>(Ty1);
-    const StructType *STy2 = cast<StructType>(Ty2);
+    StructType *STy1 = cast<StructType>(Ty1);
+    StructType *STy2 = cast<StructType>(Ty2);
     if (STy1->getNumElements() != STy2->getNumElements())
       return false;
 
@@ -255,8 +255,8 @@ bool FunctionComparator::isEquivalentType(const Type *Ty1,
   }
 
   case Type::FunctionTyID: {
-    const FunctionType *FTy1 = cast<FunctionType>(Ty1);
-    const FunctionType *FTy2 = cast<FunctionType>(Ty2);
+    FunctionType *FTy1 = cast<FunctionType>(Ty1);
+    FunctionType *FTy2 = cast<FunctionType>(Ty2);
     if (FTy1->getNumParams() != FTy2->getNumParams() ||
         FTy1->isVarArg() != FTy2->isVarArg())
       return false;
@@ -272,8 +272,8 @@ bool FunctionComparator::isEquivalentType(const Type *Ty1,
   }
 
   case Type::ArrayTyID: {
-    const ArrayType *ATy1 = cast<ArrayType>(Ty1);
-    const ArrayType *ATy2 = cast<ArrayType>(Ty2);
+    ArrayType *ATy1 = cast<ArrayType>(Ty1);
+    ArrayType *ATy2 = cast<ArrayType>(Ty2);
     return ATy1->getNumElements() == ATy2->getNumElements() &&
            isEquivalentType(ATy1->getElementType(), ATy2->getElementType());
   }
@@ -305,10 +305,14 @@ bool FunctionComparator::isEquivalentOperation(const Instruction *I1,
   // Check special state that is a part of some instructions.
   if (const LoadInst *LI = dyn_cast<LoadInst>(I1))
     return LI->isVolatile() == cast<LoadInst>(I2)->isVolatile() &&
-           LI->getAlignment() == cast<LoadInst>(I2)->getAlignment();
+           LI->getAlignment() == cast<LoadInst>(I2)->getAlignment() &&
+           LI->getOrdering() == cast<LoadInst>(I2)->getOrdering() &&
+           LI->getSynchScope() == cast<LoadInst>(I2)->getSynchScope();
   if (const StoreInst *SI = dyn_cast<StoreInst>(I1))
     return SI->isVolatile() == cast<StoreInst>(I2)->isVolatile() &&
-           SI->getAlignment() == cast<StoreInst>(I2)->getAlignment();
+           SI->getAlignment() == cast<StoreInst>(I2)->getAlignment() &&
+           SI->getOrdering() == cast<StoreInst>(I2)->getOrdering() &&
+           SI->getSynchScope() == cast<StoreInst>(I2)->getSynchScope();
   if (const CmpInst *CI = dyn_cast<CmpInst>(I1))
     return CI->getPredicate() == cast<CmpInst>(I2)->getPredicate();
   if (const CallInst *CI = dyn_cast<CallInst>(I1))
@@ -317,22 +321,22 @@ bool FunctionComparator::isEquivalentOperation(const Instruction *I1,
   if (const InvokeInst *CI = dyn_cast<InvokeInst>(I1))
     return CI->getCallingConv() == cast<InvokeInst>(I2)->getCallingConv() &&
            CI->getAttributes() == cast<InvokeInst>(I2)->getAttributes();
-  if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1)) {
-    if (IVI->getNumIndices() != cast<InsertValueInst>(I2)->getNumIndices())
-      return false;
-    for (unsigned i = 0, e = IVI->getNumIndices(); i != e; ++i)
-      if (IVI->idx_begin()[i] != cast<InsertValueInst>(I2)->idx_begin()[i])
-        return false;
-    return true;
-  }
-  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1)) {
-    if (EVI->getNumIndices() != cast<ExtractValueInst>(I2)->getNumIndices())
-      return false;
-    for (unsigned i = 0, e = EVI->getNumIndices(); i != e; ++i)
-      if (EVI->idx_begin()[i] != cast<ExtractValueInst>(I2)->idx_begin()[i])
-        return false;
-    return true;
-  }
+  if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1))
+    return IVI->getIndices() == cast<InsertValueInst>(I2)->getIndices();
+  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1))
+    return EVI->getIndices() == cast<ExtractValueInst>(I2)->getIndices();
+  if (const FenceInst *FI = dyn_cast<FenceInst>(I1))
+    return FI->getOrdering() == cast<FenceInst>(I2)->getOrdering() &&
+           FI->getSynchScope() == cast<FenceInst>(I2)->getSynchScope();
+  if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I1))
+    return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I2)->isVolatile() &&
+           CXI->getOrdering() == cast<AtomicCmpXchgInst>(I2)->getOrdering() &&
+           CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I2)->getSynchScope();
+  if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I1))
+    return RMWI->getOperation() == cast<AtomicRMWInst>(I2)->getOperation() &&
+           RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
+           RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
+           RMWI->getSynchScope() == cast<AtomicRMWInst>(I2)->getSynchScope();
 
   return true;
 }
@@ -346,9 +350,9 @@ bool FunctionComparator::isEquivalentGEP(const GEPOperator *GEP1,
     SmallVector<Value *, 8> Indices1(GEP1->idx_begin(), GEP1->idx_end());
     SmallVector<Value *, 8> Indices2(GEP2->idx_begin(), GEP2->idx_end());
     uint64_t Offset1 = TD->getIndexedOffset(GEP1->getPointerOperandType(),
-                                            Indices1.data(), Indices1.size());
+                                            Indices1);
     uint64_t Offset2 = TD->getIndexedOffset(GEP2->getPointerOperandType(),
-                                            Indices2.data(), Indices2.size());
+                                            Indices2);
     return Offset1 == Offset2;
   }
 
@@ -725,7 +729,7 @@ void MergeFunctions::writeThunk(Function *F, Function *G) {
 
   SmallVector<Value *, 16> Args;
   unsigned i = 0;
-  const FunctionType *FFTy = F->getFunctionType();
+  FunctionType *FFTy = F->getFunctionType();
   for (Function::arg_iterator AI = NewG->arg_begin(), AE = NewG->arg_end();
        AI != AE; ++AI) {
     Args.push_back(Builder.CreateBitCast(AI, FFTy->getParamType(i)));
diff --git a/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
new file mode 100644
index 000000000000..8fdfd72237f5
--- /dev/null
+++ b/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -0,0 +1,343 @@
+//===- PassManagerBuilder.cpp - Build Standard Pass -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PassManagerBuilder class, which is used to set up a
+// "standard" optimization sequence suitable for languages like C and C++.
+//
+//===----------------------------------------------------------------------===//
+
+
+#include "llvm/Transforms/IPO/PassManagerBuilder.h"
+
+#include "llvm-c/Transforms/PassManagerBuilder.h"
+
+#include "llvm/PassManager.h"
+#include "llvm/DefaultPasses.h"
+#include "llvm/PassManager.h"
+#include "llvm/Analysis/Passes.h"
+#include "llvm/Analysis/Verifier.h"
+#include "llvm/Target/TargetLibraryInfo.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/IPO.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/ManagedStatic.h"
+
+using namespace llvm;
+
+PassManagerBuilder::PassManagerBuilder() {
+    OptLevel = 2;
+    SizeLevel = 0;
+    LibraryInfo = 0;
+    Inliner = 0;
+    DisableSimplifyLibCalls = false;
+    DisableUnitAtATime = false;
+    DisableUnrollLoops = false;
+}
+
+PassManagerBuilder::~PassManagerBuilder() {
+  delete LibraryInfo;
+  delete Inliner;
+}
+
+/// Set of global extensions, automatically added as part of the standard set.
+static ManagedStatic<SmallVector<std::pair<PassManagerBuilder::ExtensionPointTy,
+   PassManagerBuilder::ExtensionFn>, 8> > GlobalExtensions;
+
+void PassManagerBuilder::addGlobalExtension(
+    PassManagerBuilder::ExtensionPointTy Ty,
+    PassManagerBuilder::ExtensionFn Fn) {
+  GlobalExtensions->push_back(std::make_pair(Ty, Fn));
+}
+
+void PassManagerBuilder::addExtension(ExtensionPointTy Ty, ExtensionFn Fn) {
+  Extensions.push_back(std::make_pair(Ty, Fn));
+}
+
+void PassManagerBuilder::addExtensionsToPM(ExtensionPointTy ETy,
+                                           PassManagerBase &PM) const {
+  for (unsigned i = 0, e = GlobalExtensions->size(); i != e; ++i)
+    if ((*GlobalExtensions)[i].first == ETy)
+      (*GlobalExtensions)[i].second(*this, PM);
+  for (unsigned i = 0, e = Extensions.size(); i != e; ++i)
+    if (Extensions[i].first == ETy)
+      Extensions[i].second(*this, PM);
+}
+
+void
+PassManagerBuilder::addInitialAliasAnalysisPasses(PassManagerBase &PM) const {
+  // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
+  // BasicAliasAnalysis wins if they disagree. This is intended to help
+  // support "obvious" type-punning idioms.
+  PM.add(createTypeBasedAliasAnalysisPass());
+  PM.add(createBasicAliasAnalysisPass());
+}
+
+void PassManagerBuilder::populateFunctionPassManager(FunctionPassManager &FPM) {
+  addExtensionsToPM(EP_EarlyAsPossible, FPM);
+
+  // Add LibraryInfo if we have some.
+  if (LibraryInfo) FPM.add(new TargetLibraryInfo(*LibraryInfo));
+
+  if (OptLevel == 0) return;
+
+  addInitialAliasAnalysisPasses(FPM);
+
+  FPM.add(createCFGSimplificationPass());
+  FPM.add(createScalarReplAggregatesPass());
+  FPM.add(createEarlyCSEPass());
+  FPM.add(createLowerExpectIntrinsicPass());
+}
+
+void PassManagerBuilder::populateModulePassManager(PassManagerBase &MPM) {
+  // If all optimizations are disabled, just run the always-inline pass.
+  if (OptLevel == 0) {
+    if (Inliner) {
+      MPM.add(Inliner);
+      Inliner = 0;
+    }
+    return;
+  }
+
+  // Add LibraryInfo if we have some.
+  if (LibraryInfo) MPM.add(new TargetLibraryInfo(*LibraryInfo));
+
+  addInitialAliasAnalysisPasses(MPM);
+
+  if (!DisableUnitAtATime) {
+    MPM.add(createGlobalOptimizerPass());     // Optimize out global vars
+
+    MPM.add(createIPSCCPPass());              // IP SCCP
+    MPM.add(createDeadArgEliminationPass());  // Dead argument elimination
+
+    MPM.add(createInstructionCombiningPass());// Clean up after IPCP & DAE
+    MPM.add(createCFGSimplificationPass());   // Clean up after IPCP & DAE
+  }
+
+  // Start of CallGraph SCC passes.
+  if (!DisableUnitAtATime)
+    MPM.add(createPruneEHPass());             // Remove dead EH info
+  if (Inliner) {
+    MPM.add(Inliner);
+    Inliner = 0;
+  }
+  if (!DisableUnitAtATime)
+    MPM.add(createFunctionAttrsPass());       // Set readonly/readnone attrs
+  if (OptLevel > 2)
+    MPM.add(createArgumentPromotionPass());   // Scalarize uninlined fn args
+
+  // Start of function pass.
+  // Break up aggregate allocas, using SSAUpdater.
+  MPM.add(createScalarReplAggregatesPass(-1, false));
+  MPM.add(createEarlyCSEPass());              // Catch trivial redundancies
+  if (!DisableSimplifyLibCalls)
+    MPM.add(createSimplifyLibCallsPass());    // Library Call Optimizations
+  MPM.add(createJumpThreadingPass());         // Thread jumps.
+  MPM.add(createCorrelatedValuePropagationPass()); // Propagate conditionals
+  MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
+  MPM.add(createInstructionCombiningPass());  // Combine silly seq's
+
+  MPM.add(createTailCallEliminationPass());   // Eliminate tail calls
+  MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
+  MPM.add(createReassociatePass());           // Reassociate expressions
+  MPM.add(createLoopRotatePass());            // Rotate Loop
+  MPM.add(createLICMPass());                  // Hoist loop invariants
+  MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3));
+  MPM.add(createInstructionCombiningPass());
+  MPM.add(createIndVarSimplifyPass());        // Canonicalize indvars
+  MPM.add(createLoopIdiomPass());             // Recognize idioms like memset.
+  MPM.add(createLoopDeletionPass());          // Delete dead loops
+  if (!DisableUnrollLoops)
+    MPM.add(createLoopUnrollPass());          // Unroll small loops
+  addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
+
+  if (OptLevel > 1)
+    MPM.add(createGVNPass());                 // Remove redundancies
+  MPM.add(createMemCpyOptPass());             // Remove memcpy / form memset
+  MPM.add(createSCCPPass());                  // Constant prop with SCCP
+
+  // Run instcombine after redundancy elimination to exploit opportunities
+  // opened up by them.
+  MPM.add(createInstructionCombiningPass());
+  MPM.add(createJumpThreadingPass());         // Thread jumps
+  MPM.add(createCorrelatedValuePropagationPass());
+  MPM.add(createDeadStoreEliminationPass());  // Delete dead stores
+
+  addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
+
+  MPM.add(createAggressiveDCEPass());         // Delete dead instructions
+  MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
+  MPM.add(createInstructionCombiningPass());  // Clean up after everything.
+
+  if (!DisableUnitAtATime) {
+    // FIXME: We shouldn't bother with this anymore.
+    MPM.add(createStripDeadPrototypesPass()); // Get rid of dead prototypes
+
+    // GlobalOpt already deletes dead functions and globals, at -O3 try a
+    // late pass of GlobalDCE.  It is capable of deleting dead cycles.
+    if (OptLevel > 2)
+      MPM.add(createGlobalDCEPass());         // Remove dead fns and globals.
+
+    if (OptLevel > 1)
+      MPM.add(createConstantMergePass());     // Merge dup global constants
+  }
+}
+
+void PassManagerBuilder::populateLTOPassManager(PassManagerBase &PM,
+                                                bool Internalize,
+                                                bool RunInliner) {
+  // Provide AliasAnalysis services for optimizations.
+  addInitialAliasAnalysisPasses(PM);
+
+  // Now that composite has been compiled, scan through the module, looking
+  // for a main function.  If main is defined, mark all other functions
+  // internal.
+  if (Internalize)
+    PM.add(createInternalizePass(true));
+
+  // Propagate constants at call sites into the functions they call.  This
+  // opens opportunities for globalopt (and inlining) by substituting function
+  // pointers passed as arguments to direct uses of functions.
+  PM.add(createIPSCCPPass());
+
+  // Now that we internalized some globals, see if we can hack on them!
+  PM.add(createGlobalOptimizerPass());
+
+  // Linking modules together can lead to duplicated global constants, only
+  // keep one copy of each constant.
+  PM.add(createConstantMergePass());
+
+  // Remove unused arguments from functions.
+  PM.add(createDeadArgEliminationPass());
+
+  // Reduce the code after globalopt and ipsccp.  Both can open up significant
+  // simplification opportunities, and both can propagate functions through
+  // function pointers.  When this happens, we often have to resolve varargs
+  // calls, etc, so let instcombine do this.
+  PM.add(createInstructionCombiningPass());
+
+  // Inline small functions
+  if (RunInliner)
+    PM.add(createFunctionInliningPass());
+
+  PM.add(createPruneEHPass());   // Remove dead EH info.
+
+  // Optimize globals again if we ran the inliner.
+  if (RunInliner)
+    PM.add(createGlobalOptimizerPass());
+  PM.add(createGlobalDCEPass()); // Remove dead functions.
+
+  // If we didn't decide to inline a function, check to see if we can
+  // transform it to pass arguments by value instead of by reference.
+  PM.add(createArgumentPromotionPass());
+
+  // The IPO passes may leave cruft around.  Clean up after them.
+  PM.add(createInstructionCombiningPass());
+  PM.add(createJumpThreadingPass());
+  // Break up allocas
+  PM.add(createScalarReplAggregatesPass());
+
+  // Run a few AA driven optimizations here and now, to cleanup the code.
+  PM.add(createFunctionAttrsPass()); // Add nocapture.
+  PM.add(createGlobalsModRefPass()); // IP alias analysis.
+
+  PM.add(createLICMPass());      // Hoist loop invariants.
+  PM.add(createGVNPass());       // Remove redundancies.
+  PM.add(createMemCpyOptPass()); // Remove dead memcpys.
+  // Nuke dead stores.
+  PM.add(createDeadStoreEliminationPass());
+
+  // Cleanup and simplify the code after the scalar optimizations.
+  PM.add(createInstructionCombiningPass());
+
+  PM.add(createJumpThreadingPass());
+
+  // Delete basic blocks, which optimization passes may have killed.
+  PM.add(createCFGSimplificationPass());
+
+  // Now that we have optimized the program, discard unreachable functions.
+  PM.add(createGlobalDCEPass());
+}
+
+LLVMPassManagerBuilderRef LLVMPassManagerBuilderCreate(void) {
+  PassManagerBuilder *PMB = new PassManagerBuilder();
+  return wrap(PMB);
+}
+
+void LLVMPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  delete Builder;
+}
+
+void
+LLVMPassManagerBuilderSetOptLevel(LLVMPassManagerBuilderRef PMB,
+                                  unsigned OptLevel) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  Builder->OptLevel = OptLevel;
+}
+
+void
+LLVMPassManagerBuilderSetSizeLevel(LLVMPassManagerBuilderRef PMB,
+                                   unsigned SizeLevel) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  Builder->SizeLevel = SizeLevel;
+}
+
+void
+LLVMPassManagerBuilderSetDisableUnitAtATime(LLVMPassManagerBuilderRef PMB,
+                                            LLVMBool Value) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  Builder->DisableUnitAtATime = Value;
+}
+
+void
+LLVMPassManagerBuilderSetDisableUnrollLoops(LLVMPassManagerBuilderRef PMB,
+                                            LLVMBool Value) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  Builder->DisableUnrollLoops = Value;
+}
+
+void
+LLVMPassManagerBuilderSetDisableSimplifyLibCalls(LLVMPassManagerBuilderRef PMB,
+                                                 LLVMBool Value) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  Builder->DisableSimplifyLibCalls = Value;
+}
+
+void
+LLVMPassManagerBuilderUseInlinerWithThreshold(LLVMPassManagerBuilderRef PMB,
+                                              unsigned Threshold) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  Builder->Inliner = createFunctionInliningPass(Threshold);
+}
+
+void
+LLVMPassManagerBuilderPopulateFunctionPassManager(LLVMPassManagerBuilderRef PMB,
+                                                  LLVMPassManagerRef PM) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  FunctionPassManager *FPM = unwrap<FunctionPassManager>(PM);
+  Builder->populateFunctionPassManager(*FPM);
+}
+
+void
+LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB,
+                                                LLVMPassManagerRef PM) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  PassManagerBase *MPM = unwrap(PM);
+  Builder->populateModulePassManager(*MPM);
+}
+
+void LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB,
+                                                  LLVMPassManagerRef PM,
+                                                  bool Internalize,
+                                                  bool RunInliner) {
+  PassManagerBuilder *Builder = unwrap(PMB);
+  PassManagerBase *LPM = unwrap(PM);
+  Builder->populateLTOPassManager(*LPM, Internalize, RunInliner);
+}
+
diff --git a/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp b/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp
index b7e63dc4484c..cbb80f075087 100644
--- a/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp
@@ -101,8 +101,9 @@ bool PruneEH::runOnSCC(CallGraphSCC &SCC) {
       // Check to see if this function performs an unwind or calls an
       // unwinding function.
       for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
-        if (CheckUnwind && isa<UnwindInst>(BB->getTerminator())) {
-          // Uses unwind!
+        if (CheckUnwind && (isa<UnwindInst>(BB->getTerminator()) ||
+                            isa<ResumeInst>(BB->getTerminator()))) {
+          // Uses unwind / resume!
           SCCMightUnwind = true;
         } else if (CheckReturn && isa<ReturnInst>(BB->getTerminator())) {
           SCCMightReturn = true;
diff --git a/contrib/llvm/lib/Transforms/IPO/StripSymbols.cpp b/contrib/llvm/lib/Transforms/IPO/StripSymbols.cpp
index 0fbaff1509a7..b5caa9a557c9 100644
--- a/contrib/llvm/lib/Transforms/IPO/StripSymbols.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/StripSymbols.cpp
@@ -180,7 +180,7 @@ static void StripTypeNames(Module &M, bool PreserveDbgInfo) {
 
   for (unsigned i = 0, e = StructTypes.size(); i != e; ++i) {
     StructType *STy = StructTypes[i];
-    if (STy->isAnonymous() || STy->getName().empty()) continue;
+    if (STy->isLiteral() || STy->getName().empty()) continue;
     
     if (PreserveDbgInfo && STy->getName().startswith("llvm.dbg"))
       continue;
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombine.h b/contrib/llvm/lib/Transforms/InstCombine/InstCombine.h
index 8257d6b89ddc..38082787ce4d 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombine.h
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombine.h
@@ -11,6 +11,7 @@
 #define INSTCOMBINE_INSTCOMBINE_H
 
 #include "InstCombineWorklist.h"
+#include "llvm/IntrinsicInst.h"
 #include "llvm/Operator.h"
 #include "llvm/Pass.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -103,7 +104,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   //
   Instruction *visitAdd(BinaryOperator &I);
   Instruction *visitFAdd(BinaryOperator &I);
-  Value *OptimizePointerDifference(Value *LHS, Value *RHS, const Type *Ty);
+  Value *OptimizePointerDifference(Value *LHS, Value *RHS, Type *Ty);
   Instruction *visitSub(BinaryOperator &I);
   Instruction *visitFSub(BinaryOperator &I);
   Instruction *visitMul(BinaryOperator &I);
@@ -192,15 +193,16 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   Instruction *visitExtractElementInst(ExtractElementInst &EI);
   Instruction *visitShuffleVectorInst(ShuffleVectorInst &SVI);
   Instruction *visitExtractValueInst(ExtractValueInst &EV);
+  Instruction *visitLandingPadInst(LandingPadInst &LI);
 
   // visitInstruction - Specify what to return for unhandled instructions...
   Instruction *visitInstruction(Instruction &I) { return 0; }
 
 private:
-  bool ShouldChangeType(const Type *From, const Type *To) const;
+  bool ShouldChangeType(Type *From, Type *To) const;
   Value *dyn_castNegVal(Value *V) const;
   Value *dyn_castFNegVal(Value *V) const;
-  const Type *FindElementAtOffset(const Type *Ty, int64_t Offset, 
+  Type *FindElementAtOffset(Type *Ty, int64_t Offset, 
                                   SmallVectorImpl<Value*> &NewIndices);
   Instruction *FoldOpIntoSelect(Instruction &Op, SelectInst *SI);
                                  
@@ -209,12 +211,13 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   /// the cast can be eliminated by some other simple transformation, we prefer
   /// to do the simplification first.
   bool ShouldOptimizeCast(Instruction::CastOps opcode,const Value *V,
-                          const Type *Ty);
+                          Type *Ty);
 
   Instruction *visitCallSite(CallSite CS);
   Instruction *tryOptimizeCall(CallInst *CI, const TargetData *TD);
   bool transformConstExprCastCall(CallSite CS);
-  Instruction *transformCallThroughTrampoline(CallSite CS);
+  Instruction *transformCallThroughTrampoline(CallSite CS,
+                                              IntrinsicInst *Tramp);
   Instruction *transformZExtICmp(ICmpInst *ICI, Instruction &CI,
                                  bool DoXform = true);
   Instruction *transformSExtICmp(ICmpInst *ICI, Instruction &CI);
@@ -357,7 +360,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   Instruction *SimplifyMemSet(MemSetInst *MI);
 
 
-  Value *EvaluateInDifferentType(Value *V, const Type *Ty, bool isSigned);
+  Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
 };
 
       
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index c36a9552e7a3..d10046c10baf 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -188,7 +188,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) {
     return BinaryOperator::CreateMul(LHS, AddOne(C2));
 
   // A+B --> A|B iff A and B have no bits set in common.
-  if (const IntegerType *IT = dyn_cast<IntegerType>(I.getType())) {
+  if (IntegerType *IT = dyn_cast<IntegerType>(I.getType())) {
     APInt Mask = APInt::getAllOnesValue(IT->getBitWidth());
     APInt LHSKnownOne(IT->getBitWidth(), 0);
     APInt LHSKnownZero(IT->getBitWidth(), 0);
@@ -401,7 +401,7 @@ Instruction *InstCombiner::visitFAdd(BinaryOperator &I) {
 Value *InstCombiner::EmitGEPOffset(User *GEP) {
   TargetData &TD = *getTargetData();
   gep_type_iterator GTI = gep_type_begin(GEP);
-  const Type *IntPtrTy = TD.getIntPtrType(GEP->getContext());
+  Type *IntPtrTy = TD.getIntPtrType(GEP->getContext());
   Value *Result = Constant::getNullValue(IntPtrTy);
 
   // If the GEP is inbounds, we know that none of the addressing operations will
@@ -420,7 +420,7 @@ Value *InstCombiner::EmitGEPOffset(User *GEP) {
       if (OpC->isZero()) continue;
       
       // Handle a struct index, which adds its field offset to the pointer.
-      if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
         Size = TD.getStructLayout(STy)->getElementOffset(OpC->getZExtValue());
         
         if (Size)
@@ -460,7 +460,7 @@ Value *InstCombiner::EmitGEPOffset(User *GEP) {
 /// operands to the ptrtoint instructions for the LHS/RHS of the subtract.
 ///
 Value *InstCombiner::OptimizePointerDifference(Value *LHS, Value *RHS,
-                                               const Type *Ty) {
+                                               Type *Ty) {
   assert(TD && "Must have target data info for this");
   
   // If LHS is a gep based on RHS or RHS is a gep based on LHS, we can optimize
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 64ea36fb1e9d..5e0bfe8e26d2 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1174,30 +1174,31 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) {
         ((A == C && B == D) || (A == D && B == C)))
       return BinaryOperator::CreateXor(A, B);
     
-    if (Op0->hasOneUse() &&
-        match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
-      if (A == Op1) {                                // (A^B)&A -> A&(A^B)
-        I.swapOperands();     // Simplify below
-        std::swap(Op0, Op1);
-      } else if (B == Op1) {                         // (A^B)&B -> B&(B^A)
-        cast<BinaryOperator>(Op0)->swapOperands();
-        I.swapOperands();     // Simplify below
-        std::swap(Op0, Op1);
+    // A&(A^B) => A & ~B
+    {
+      Value *tmpOp0 = Op0;
+      Value *tmpOp1 = Op1;
+      if (Op0->hasOneUse() &&
+          match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
+        if (A == Op1 || B == Op1 ) {
+          tmpOp1 = Op0;
+          tmpOp0 = Op1;
+          // Simplify below
+        }
       }
-    }
 
-    if (Op1->hasOneUse() &&
-        match(Op1, m_Xor(m_Value(A), m_Value(B)))) {
-      if (B == Op0) {                                // B&(A^B) -> B&(B^A)
-        cast<BinaryOperator>(Op1)->swapOperands();
-        std::swap(A, B);
+      if (tmpOp1->hasOneUse() &&
+          match(tmpOp1, m_Xor(m_Value(A), m_Value(B)))) {
+        if (B == tmpOp0) {
+          std::swap(A, B);
+        }
+        // Notice that the patten (A&(~B)) is actually (A&(-1^B)), so if
+        // A is originally -1 (or a vector of -1 and undefs), then we enter
+        // an endless loop. By checking that A is non-constant we ensure that
+        // we will never get to the loop.
+        if (A == tmpOp0 && !isa<Constant>(A)) // A&(A^B) -> A & ~B
+          return BinaryOperator::CreateAnd(A, Builder->CreateNot(B));
       }
-      // Notice that the patten (A&(~B)) is actually (A&(-1^B)), so if
-      // A is originally -1 (or a vector of -1 and undefs), then we enter
-      // an endless loop. By checking that A is non-constant we ensure that
-      // we will never get to the loop.
-      if (A == Op0 && !isa<Constant>(A)) // A&(A^B) -> A & ~B
-        return BinaryOperator::CreateAnd(A, Builder->CreateNot(B, "tmp"));
     }
 
     // (A&((~A)|B)) -> A&B
@@ -1224,7 +1225,7 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) {
   // fold (and (cast A), (cast B)) -> (cast (and A, B))
   if (CastInst *Op0C = dyn_cast<CastInst>(Op0))
     if (CastInst *Op1C = dyn_cast<CastInst>(Op1)) {
-      const Type *SrcTy = Op0C->getOperand(0)->getType();
+      Type *SrcTy = Op0C->getOperand(0)->getType();
       if (Op0C->getOpcode() == Op1C->getOpcode() && // same cast kind ?
           SrcTy == Op1C->getOperand(0)->getType() &&
           SrcTy->isIntOrIntVectorTy()) {
@@ -2008,7 +2009,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) {
   if (CastInst *Op0C = dyn_cast<CastInst>(Op0)) {
     CastInst *Op1C = dyn_cast<CastInst>(Op1);
     if (Op1C && Op0C->getOpcode() == Op1C->getOpcode()) {// same cast kind ?
-      const Type *SrcTy = Op0C->getOperand(0)->getType();
+      Type *SrcTy = Op0C->getOperand(0)->getType();
       if (SrcTy == Op1C->getOperand(0)->getType() &&
           SrcTy->isIntOrIntVectorTy()) {
         Value *Op0COp = Op0C->getOperand(0), *Op1COp = Op1C->getOperand(0);
@@ -2227,14 +2228,14 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) {
       if (A == Op1)                                  // (B|A)^B == (A|B)^B
         std::swap(A, B);
       if (B == Op1)                                  // (A|B)^B == A & ~B
-        return BinaryOperator::CreateAnd(A, Builder->CreateNot(Op1, "tmp"));
+        return BinaryOperator::CreateAnd(A, Builder->CreateNot(Op1));
     } else if (match(Op0I, m_And(m_Value(A), m_Value(B))) && 
                Op0I->hasOneUse()){
       if (A == Op1)                                        // (A&B)^A -> (B&A)^A
         std::swap(A, B);
       if (B == Op1 &&                                      // (B&A)^A == ~B & A
           !isa<ConstantInt>(Op1)) {  // Canonical form is (B&C)^C
-        return BinaryOperator::CreateAnd(Builder->CreateNot(A, "tmp"), Op1);
+        return BinaryOperator::CreateAnd(Builder->CreateNot(A), Op1);
       }
     }
   }
@@ -2288,7 +2289,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) {
   if (CastInst *Op0C = dyn_cast<CastInst>(Op0)) {
     if (CastInst *Op1C = dyn_cast<CastInst>(Op1))
       if (Op0C->getOpcode() == Op1C->getOpcode()) { // same cast kind?
-        const Type *SrcTy = Op0C->getOperand(0)->getType();
+        Type *SrcTy = Op0C->getOperand(0)->getType();
         if (SrcTy == Op1C->getOperand(0)->getType() && SrcTy->isIntegerTy() &&
             // Only do this if the casts both really cause code to be generated.
             ShouldOptimizeCast(Op0C->getOpcode(), Op0C->getOperand(0), 
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 537f2b318aa9..c7b3ff8504ac 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "InstCombine.h"
-#include "llvm/IntrinsicInst.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
@@ -22,8 +21,8 @@ using namespace llvm;
 
 /// getPromotedType - Return the specified type promoted as it would be to pass
 /// though a va_arg area.
-static const Type *getPromotedType(const Type *Ty) {
-  if (const IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
+static Type *getPromotedType(Type *Ty) {
+  if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
     if (ITy->getBitWidth() < 32)
       return Type::getInt32Ty(Ty->getContext());
   }
@@ -64,7 +63,7 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
   unsigned DstAddrSp =
     cast<PointerType>(MI->getArgOperand(0)->getType())->getAddressSpace();
 
-  const IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
+  IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
   Type *NewSrcPtrTy = PointerType::get(IntType, SrcAddrSp);
   Type *NewDstPtrTy = PointerType::get(IntType, DstAddrSp);
   
@@ -76,18 +75,18 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
   // integer datatype.
   Value *StrippedDest = MI->getArgOperand(0)->stripPointerCasts();
   if (StrippedDest != MI->getArgOperand(0)) {
-    const Type *SrcETy = cast<PointerType>(StrippedDest->getType())
+    Type *SrcETy = cast<PointerType>(StrippedDest->getType())
                                     ->getElementType();
     if (TD && SrcETy->isSized() && TD->getTypeStoreSize(SrcETy) == Size) {
       // The SrcETy might be something like {{{double}}} or [1 x double].  Rip
       // down through these levels if so.
       while (!SrcETy->isSingleValueType()) {
-        if (const StructType *STy = dyn_cast<StructType>(SrcETy)) {
+        if (StructType *STy = dyn_cast<StructType>(SrcETy)) {
           if (STy->getNumElements() == 1)
             SrcETy = STy->getElementType(0);
           else
             break;
-        } else if (const ArrayType *ATy = dyn_cast<ArrayType>(SrcETy)) {
+        } else if (ArrayType *ATy = dyn_cast<ArrayType>(SrcETy)) {
           if (ATy->getNumElements() == 1)
             SrcETy = ATy->getElementType();
           else
@@ -142,7 +141,7 @@ Instruction *InstCombiner::SimplifyMemSet(MemSetInst *MI) {
   
   // memset(s,c,n) -> store s, c (for n=1,2,4,8)
   if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
-    const Type *ITy = IntegerType::get(MI->getContext(), Len*8);  // n=1 -> i8.
+    Type *ITy = IntegerType::get(MI->getContext(), Len*8);  // n=1 -> i8.
     
     Value *Dest = MI->getDest();
     unsigned DstAddrSp = cast<PointerType>(Dest->getType())->getAddressSpace();
@@ -250,7 +249,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // We need target data for just about everything so depend on it.
     if (!TD) break;
     
-    const Type *ReturnTy = CI.getType();
+    Type *ReturnTy = CI.getType();
     uint64_t DontKnow = II->getArgOperand(1) == Builder->getTrue() ? 0 : -1ULL;
 
     // Get to the real allocated thing and offset as fast as possible.
@@ -266,8 +265,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       // Get the current byte offset into the thing. Use the original
       // operand in case we're looking through a bitcast.
       SmallVector<Value*, 8> Ops(GEP->idx_begin(), GEP->idx_end());
-      Offset = TD->getIndexedOffset(GEP->getPointerOperandType(),
-                                    Ops.data(), Ops.size());
+      Offset = TD->getIndexedOffset(GEP->getPointerOperandType(), Ops);
 
       Op1 = GEP->getPointerOperand()->stripPointerCasts();
 
@@ -300,7 +298,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       }
     } else if (CallInst *MI = extractMallocCall(Op1)) {
       // Get allocation size.
-      const Type* MallocType = getMallocAllocatedType(MI);
+      Type* MallocType = getMallocAllocatedType(MI);
       if (MallocType && MallocType->isSized())
         if (Value *NElems = getMallocArraySize(MI, TD, true))
           if (ConstantInt *NElements = dyn_cast<ConstantInt>(NElems))
@@ -355,7 +353,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   case Intrinsic::cttz: {
     // If all bits below the first known one are known zero,
     // this value is constant.
-    const IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType());
+    IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType());
     // FIXME: Try to simplify vectors of integers.
     if (!IT) break;
     uint32_t BitWidth = IT->getBitWidth();
@@ -374,7 +372,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   case Intrinsic::ctlz: {
     // If all bits above the first known one are known zero,
     // this value is constant.
-    const IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType());
+    IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType());
     // FIXME: Try to simplify vectors of integers.
     if (!IT) break;
     uint32_t BitWidth = IT->getBitWidth();
@@ -392,7 +390,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     break;
   case Intrinsic::uadd_with_overflow: {
     Value *LHS = II->getArgOperand(0), *RHS = II->getArgOperand(1);
-    const IntegerType *IT = cast<IntegerType>(II->getArgOperand(0)->getType());
+    IntegerType *IT = cast<IntegerType>(II->getArgOperand(0)->getType());
     uint32_t BitWidth = IT->getBitWidth();
     APInt Mask = APInt::getSignBit(BitWidth);
     APInt LHSKnownZero(BitWidth, 0);
@@ -416,7 +414,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
           UndefValue::get(LHS->getType()),
           ConstantInt::getTrue(II->getContext())
         };
-        const StructType *ST = cast<StructType>(II->getType());
+        StructType *ST = cast<StructType>(II->getType());
         Constant *Struct = ConstantStruct::get(ST, V);
         return InsertValueInst::Create(Struct, Add, 0);
       }
@@ -430,7 +428,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
           UndefValue::get(LHS->getType()),
           ConstantInt::getFalse(II->getContext())
         };
-        const StructType *ST = cast<StructType>(II->getType());
+        StructType *ST = cast<StructType>(II->getType());
         Constant *Struct = ConstantStruct::get(ST, V);
         return InsertValueInst::Create(Struct, Add, 0);
       }
@@ -559,7 +557,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   case Intrinsic::ppc_altivec_stvxl:
     // Turn stvx -> store if the pointer is known aligned.
     if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, TD) >= 16) {
-      const Type *OpPtrTy = 
+      Type *OpPtrTy = 
         PointerType::getUnqual(II->getArgOperand(0)->getType());
       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy);
       return new StoreInst(II->getArgOperand(0), Ptr);
@@ -570,7 +568,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   case Intrinsic::x86_sse2_storeu_dq:
     // Turn X86 storeu -> store if the pointer is known aligned.
     if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, TD) >= 16) {
-      const Type *OpPtrTy = 
+      Type *OpPtrTy = 
         PointerType::getUnqual(II->getArgOperand(1)->getType());
       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), OpPtrTy);
       return new StoreInst(II->getArgOperand(1), Ptr);
@@ -656,15 +654,13 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
           
           if (ExtractedElts[Idx] == 0) {
             ExtractedElts[Idx] = 
-              Builder->CreateExtractElement(Idx < 16 ? Op0 : Op1, 
-                  ConstantInt::get(Type::getInt32Ty(II->getContext()),
-                                   Idx&15, false), "tmp");
+              Builder->CreateExtractElement(Idx < 16 ? Op0 : Op1,
+                                            Builder->getInt32(Idx&15));
           }
         
           // Insert this value into the result vector.
           Result = Builder->CreateInsertElement(Result, ExtractedElts[Idx],
-                         ConstantInt::get(Type::getInt32Ty(II->getContext()),
-                                          i, false), "tmp");
+                                                Builder->getInt32(i));
         }
         return CastInst::Create(Instruction::BitCast, Result, CI.getType());
       }
@@ -733,9 +729,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       }
     }
     
-    // If the stack restore is in a return/unwind block and if there are no
-    // allocas or calls between the restore and the return, nuke the restore.
-    if (!CannotRemove && (isa<ReturnInst>(TI) || isa<UnwindInst>(TI)))
+    // If the stack restore is in a return, resume, or unwind block and if there
+    // are no allocas or calls between the restore and the return, nuke the
+    // restore.
+    if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI) ||
+                          isa<UnwindInst>(TI)))
       return EraseInstFromFunction(CI);
     break;
   }
@@ -765,9 +763,9 @@ static bool isSafeToEliminateVarargsCast(const CallSite CS,
   if (!CS.paramHasAttr(ix, Attribute::ByVal))
     return true;
 
-  const Type* SrcTy = 
+  Type* SrcTy = 
             cast<PointerType>(CI->getOperand(0)->getType())->getElementType();
-  const Type* DstTy = cast<PointerType>(CI->getType())->getElementType();
+  Type* DstTy = cast<PointerType>(CI->getType())->getElementType();
   if (!SrcTy->isSized() || !DstTy->isSized())
     return false;
   if (!TD || TD->getTypeAllocSize(SrcTy) != TD->getTypeAllocSize(DstTy))
@@ -820,6 +818,83 @@ Instruction *InstCombiner::tryOptimizeCall(CallInst *CI, const TargetData *TD) {
   return Simplifier.NewInstruction;
 }
 
+static IntrinsicInst *FindInitTrampolineFromAlloca(Value *TrampMem) {
+  // Strip off at most one level of pointer casts, looking for an alloca.  This
+  // is good enough in practice and simpler than handling any number of casts.
+  Value *Underlying = TrampMem->stripPointerCasts();
+  if (Underlying != TrampMem &&
+      (!Underlying->hasOneUse() || *Underlying->use_begin() != TrampMem))
+    return 0;
+  if (!isa<AllocaInst>(Underlying))
+    return 0;
+
+  IntrinsicInst *InitTrampoline = 0;
+  for (Value::use_iterator I = TrampMem->use_begin(), E = TrampMem->use_end();
+       I != E; I++) {
+    IntrinsicInst *II = dyn_cast<IntrinsicInst>(*I);
+    if (!II)
+      return 0;
+    if (II->getIntrinsicID() == Intrinsic::init_trampoline) {
+      if (InitTrampoline)
+        // More than one init_trampoline writes to this value.  Give up.
+        return 0;
+      InitTrampoline = II;
+      continue;
+    }
+    if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)
+      // Allow any number of calls to adjust.trampoline.
+      continue;
+    return 0;
+  }
+
+  // No call to init.trampoline found.
+  if (!InitTrampoline)
+    return 0;
+
+  // Check that the alloca is being used in the expected way.
+  if (InitTrampoline->getOperand(0) != TrampMem)
+    return 0;
+
+  return InitTrampoline;
+}
+
+static IntrinsicInst *FindInitTrampolineFromBB(IntrinsicInst *AdjustTramp,
+                                               Value *TrampMem) {
+  // Visit all the previous instructions in the basic block, and try to find a
+  // init.trampoline which has a direct path to the adjust.trampoline.
+  for (BasicBlock::iterator I = AdjustTramp,
+       E = AdjustTramp->getParent()->begin(); I != E; ) {
+    Instruction *Inst = --I;
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
+      if (II->getIntrinsicID() == Intrinsic::init_trampoline &&
+          II->getOperand(0) == TrampMem)
+        return II;
+    if (Inst->mayWriteToMemory())
+      return 0;
+  }
+  return 0;
+}
+
+// Given a call to llvm.adjust.trampoline, find and return the corresponding
+// call to llvm.init.trampoline if the call to the trampoline can be optimized
+// to a direct call to a function.  Otherwise return NULL.
+//
+static IntrinsicInst *FindInitTrampoline(Value *Callee) {
+  Callee = Callee->stripPointerCasts();
+  IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);
+  if (!AdjustTramp ||
+      AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)
+    return 0;
+
+  Value *TrampMem = AdjustTramp->getOperand(0);
+
+  if (IntrinsicInst *IT = FindInitTrampolineFromAlloca(TrampMem))
+    return IT;
+  if (IntrinsicInst *IT = FindInitTrampolineFromBB(AdjustTramp, TrampMem))
+    return IT;
+  return 0;
+}
+
 // visitCallSite - Improvements for call and invoke instructions.
 //
 Instruction *InstCombiner::visitCallSite(CallSite CS) {
@@ -879,13 +954,11 @@ Instruction *InstCombiner::visitCallSite(CallSite CS) {
     return EraseInstFromFunction(*CS.getInstruction());
   }
 
-  if (BitCastInst *BC = dyn_cast<BitCastInst>(Callee))
-    if (IntrinsicInst *In = dyn_cast<IntrinsicInst>(BC->getOperand(0)))
-      if (In->getIntrinsicID() == Intrinsic::init_trampoline)
-        return transformCallThroughTrampoline(CS);
+  if (IntrinsicInst *II = FindInitTrampoline(Callee))
+    return transformCallThroughTrampoline(CS, II);
 
-  const PointerType *PTy = cast<PointerType>(Callee->getType());
-  const FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+  PointerType *PTy = cast<PointerType>(Callee->getType());
+  FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
   if (FTy->isVarArg()) {
     int ix = FTy->getNumParams() + (isa<InvokeInst>(Callee) ? 3 : 1);
     // See if we can optimize any arguments passed through the varargs area of
@@ -934,9 +1007,9 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
   // would cause a type conversion of one of our arguments, change this call to
   // be a direct call with arguments casted to the appropriate types.
   //
-  const FunctionType *FT = Callee->getFunctionType();
-  const Type *OldRetTy = Caller->getType();
-  const Type *NewRetTy = FT->getReturnType();
+  FunctionType *FT = Callee->getFunctionType();
+  Type *OldRetTy = Caller->getType();
+  Type *NewRetTy = FT->getReturnType();
 
   if (NewRetTy->isStructTy())
     return false; // TODO: Handle multiple return values.
@@ -982,8 +1055,8 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
 
   CallSite::arg_iterator AI = CS.arg_begin();
   for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
-    const Type *ParamTy = FT->getParamType(i);
-    const Type *ActTy = (*AI)->getType();
+    Type *ParamTy = FT->getParamType(i);
+    Type *ActTy = (*AI)->getType();
 
     if (!CastInst::isCastable(ActTy, ParamTy))
       return false;   // Cannot transform this parameter value.
@@ -995,11 +1068,11 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
     // If the parameter is passed as a byval argument, then we have to have a
     // sized type and the sized type has to have the same size as the old type.
     if (ParamTy != ActTy && (Attrs & Attribute::ByVal)) {
-      const PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
+      PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
       if (ParamPTy == 0 || !ParamPTy->getElementType()->isSized() || TD == 0)
         return false;
       
-      const Type *CurElTy = cast<PointerType>(ActTy)->getElementType();
+      Type *CurElTy = cast<PointerType>(ActTy)->getElementType();
       if (TD->getTypeAllocSize(CurElTy) !=
           TD->getTypeAllocSize(ParamPTy->getElementType()))
         return false;
@@ -1023,7 +1096,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
     // If the callee is just a declaration, don't change the varargsness of the
     // call.  We don't want to introduce a varargs call where one doesn't
     // already exist.
-    const PointerType *APTy = cast<PointerType>(CS.getCalledValue()->getType());
+    PointerType *APTy = cast<PointerType>(CS.getCalledValue()->getType());
     if (FT->isVarArg()!=cast<FunctionType>(APTy->getElementType())->isVarArg())
       return false;
   }
@@ -1062,13 +1135,13 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
 
   AI = CS.arg_begin();
   for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
-    const Type *ParamTy = FT->getParamType(i);
+    Type *ParamTy = FT->getParamType(i);
     if ((*AI)->getType() == ParamTy) {
       Args.push_back(*AI);
     } else {
       Instruction::CastOps opcode = CastInst::getCastOpcode(*AI,
           false, ParamTy, false);
-      Args.push_back(Builder->CreateCast(opcode, *AI, ParamTy, "tmp"));
+      Args.push_back(Builder->CreateCast(opcode, *AI, ParamTy));
     }
 
     // Add any parameter attributes.
@@ -1089,12 +1162,12 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
     } else {
       // Add all of the arguments in their promoted form to the arg list.
       for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
-        const Type *PTy = getPromotedType((*AI)->getType());
+        Type *PTy = getPromotedType((*AI)->getType());
         if (PTy != (*AI)->getType()) {
           // Must promote to pass through va_arg area!
           Instruction::CastOps opcode =
             CastInst::getCastOpcode(*AI, false, PTy, false);
-          Args.push_back(Builder->CreateCast(opcode, *AI, PTy, "tmp"));
+          Args.push_back(Builder->CreateCast(opcode, *AI, PTy));
         } else {
           Args.push_back(*AI);
         }
@@ -1138,13 +1211,13 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
     if (!NV->getType()->isVoidTy()) {
       Instruction::CastOps opcode =
         CastInst::getCastOpcode(NC, false, OldRetTy, false);
-      NV = NC = CastInst::Create(opcode, NC, OldRetTy, "tmp");
+      NV = NC = CastInst::Create(opcode, NC, OldRetTy);
       NC->setDebugLoc(Caller->getDebugLoc());
 
       // If this is an invoke instruction, we should insert it after the first
       // non-phi, instruction in the normal successor block.
       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
-        BasicBlock::iterator I = II->getNormalDest()->getFirstNonPHI();
+        BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt();
         InsertNewInstBefore(NC, *I);
       } else {
         // Otherwise, it's a call, just insert cast right after the call.
@@ -1163,13 +1236,16 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
   return true;
 }
 
-// transformCallThroughTrampoline - Turn a call to a function created by the
-// init_trampoline intrinsic into a direct call to the underlying function.
+// transformCallThroughTrampoline - Turn a call to a function created by
+// init_trampoline / adjust_trampoline intrinsic pair into a direct call to the
+// underlying function.
 //
-Instruction *InstCombiner::transformCallThroughTrampoline(CallSite CS) {
+Instruction *
+InstCombiner::transformCallThroughTrampoline(CallSite CS,
+                                             IntrinsicInst *Tramp) {
   Value *Callee = CS.getCalledValue();
-  const PointerType *PTy = cast<PointerType>(Callee->getType());
-  const FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+  PointerType *PTy = cast<PointerType>(Callee->getType());
+  FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
   const AttrListPtr &Attrs = CS.getAttributes();
 
   // If the call already has the 'nest' attribute somewhere then give up -
@@ -1177,12 +1253,12 @@ Instruction *InstCombiner::transformCallThroughTrampoline(CallSite CS) {
   if (Attrs.hasAttrSomewhere(Attribute::Nest))
     return 0;
 
-  IntrinsicInst *Tramp =
-    cast<IntrinsicInst>(cast<BitCastInst>(Callee)->getOperand(0));
+  assert(Tramp &&
+         "transformCallThroughTrampoline called with incorrect CallSite.");
 
   Function *NestF =cast<Function>(Tramp->getArgOperand(1)->stripPointerCasts());
-  const PointerType *NestFPTy = cast<PointerType>(NestF->getType());
-  const FunctionType *NestFTy = cast<FunctionType>(NestFPTy->getElementType());
+  PointerType *NestFPTy = cast<PointerType>(NestF->getType());
+  FunctionType *NestFTy = cast<FunctionType>(NestFPTy->getElementType());
 
   const AttrListPtr &NestAttrs = NestF->getAttributes();
   if (!NestAttrs.isEmpty()) {
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 82c734e0b829..f10e48abf108 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "InstCombine.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Support/PatternMatch.h"
 using namespace llvm;
@@ -79,14 +80,14 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
   // This requires TargetData to get the alloca alignment and size information.
   if (!TD) return 0;
 
-  const PointerType *PTy = cast<PointerType>(CI.getType());
+  PointerType *PTy = cast<PointerType>(CI.getType());
   
   BuilderTy AllocaBuilder(*Builder);
   AllocaBuilder.SetInsertPoint(AI.getParent(), &AI);
 
   // Get the type really allocated and the type casted to.
-  const Type *AllocElTy = AI.getAllocatedType();
-  const Type *CastElTy = PTy->getElementType();
+  Type *AllocElTy = AI.getAllocatedType();
+  Type *CastElTy = PTy->getElementType();
   if (!AllocElTy->isSized() || !CastElTy->isSized()) return 0;
 
   unsigned AllocElTyAlign = TD->getABITypeAlignment(AllocElTy);
@@ -121,13 +122,13 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
   } else {
     Amt = ConstantInt::get(AI.getArraySize()->getType(), Scale);
     // Insert before the alloca, not before the cast.
-    Amt = AllocaBuilder.CreateMul(Amt, NumElements, "tmp");
+    Amt = AllocaBuilder.CreateMul(Amt, NumElements);
   }
   
   if (uint64_t Offset = (AllocElTySize*ArrayOffset)/CastElTySize) {
     Value *Off = ConstantInt::get(AI.getArraySize()->getType(),
                                   Offset, true);
-    Amt = AllocaBuilder.CreateAdd(Amt, Off, "tmp");
+    Amt = AllocaBuilder.CreateAdd(Amt, Off);
   }
   
   AllocaInst *New = AllocaBuilder.CreateAlloca(CastElTy, Amt);
@@ -151,7 +152,7 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
 /// EvaluateInDifferentType - Given an expression that 
 /// CanEvaluateTruncated or CanEvaluateSExtd returns true for, actually
 /// insert the code to evaluate the expression.
-Value *InstCombiner::EvaluateInDifferentType(Value *V, const Type *Ty, 
+Value *InstCombiner::EvaluateInDifferentType(Value *V, Type *Ty, 
                                              bool isSigned) {
   if (Constant *C = dyn_cast<Constant>(V)) {
     C = ConstantExpr::getIntegerCast(C, Ty, isSigned /*Sext or ZExt*/);
@@ -229,12 +230,12 @@ static Instruction::CastOps
 isEliminableCastPair(
   const CastInst *CI, ///< The first cast instruction
   unsigned opcode,       ///< The opcode of the second cast instruction
-  const Type *DstTy,     ///< The target type for the second cast instruction
+  Type *DstTy,     ///< The target type for the second cast instruction
   TargetData *TD         ///< The target data for pointer size
 ) {
 
-  const Type *SrcTy = CI->getOperand(0)->getType();   // A from above
-  const Type *MidTy = CI->getType();                  // B from above
+  Type *SrcTy = CI->getOperand(0)->getType();   // A from above
+  Type *MidTy = CI->getType();                  // B from above
 
   // Get the opcodes of the two Cast instructions
   Instruction::CastOps firstOp = Instruction::CastOps(CI->getOpcode());
@@ -260,7 +261,7 @@ isEliminableCastPair(
 /// the cast can be eliminated by some other simple transformation, we prefer
 /// to do the simplification first.
 bool InstCombiner::ShouldOptimizeCast(Instruction::CastOps opc, const Value *V,
-                                      const Type *Ty) {
+                                      Type *Ty) {
   // Noop casts and casts of constants should be eliminated trivially.
   if (V->getType() == Ty || isa<Constant>(V)) return false;
   
@@ -324,7 +325,7 @@ Instruction *InstCombiner::commonCastTransforms(CastInst &CI) {
 ///
 /// This function works on both vectors and scalars.
 ///
-static bool CanEvaluateTruncated(Value *V, const Type *Ty) {
+static bool CanEvaluateTruncated(Value *V, Type *Ty) {
   // We can always evaluate constants in another type.
   if (isa<Constant>(V))
     return true;
@@ -332,7 +333,7 @@ static bool CanEvaluateTruncated(Value *V, const Type *Ty) {
   Instruction *I = dyn_cast<Instruction>(V);
   if (!I) return false;
   
-  const Type *OrigTy = V->getType();
+  Type *OrigTy = V->getType();
   
   // If this is an extension from the dest type, we can eliminate it, even if it
   // has multiple uses.
@@ -435,7 +436,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) {
     return &CI;
   
   Value *Src = CI.getOperand(0);
-  const Type *DestTy = CI.getType(), *SrcTy = Src->getType();
+  Type *DestTy = CI.getType(), *SrcTy = Src->getType();
   
   // Attempt to truncate the entire input expression tree to the destination
   // type.   Only do this if the dest type is a simple type, don't convert the
@@ -456,7 +457,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) {
   // Canonicalize trunc x to i1 -> (icmp ne (and x, 1), 0), likewise for vector.
   if (DestTy->getScalarSizeInBits() == 1) {
     Constant *One = ConstantInt::get(Src->getType(), 1);
-    Src = Builder->CreateAnd(Src, One, "tmp");
+    Src = Builder->CreateAnd(Src, One);
     Value *Zero = Constant::getNullValue(Src->getType());
     return new ICmpInst(ICmpInst::ICMP_NE, Src, Zero);
   }
@@ -518,7 +519,7 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, Instruction &CI,
                                    In->getType()->getScalarSizeInBits()-1);
       In = Builder->CreateLShr(In, Sh, In->getName()+".lobit");
       if (In->getType() != CI.getType())
-        In = Builder->CreateIntCast(In, CI.getType(), false/*ZExt*/, "tmp");
+        In = Builder->CreateIntCast(In, CI.getType(), false/*ZExt*/);
 
       if (ICI->getPredicate() == ICmpInst::ICMP_SGT) {
         Constant *One = ConstantInt::get(In->getType(), 1);
@@ -572,7 +573,7 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, Instruction &CI,
           
         if ((Op1CV != 0) == isNE) { // Toggle the low bit.
           Constant *One = ConstantInt::get(In->getType(), 1);
-          In = Builder->CreateXor(In, One, "tmp");
+          In = Builder->CreateXor(In, One);
         }
           
         if (CI.getType() == In->getType())
@@ -586,7 +587,7 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, Instruction &CI,
   // It is also profitable to transform icmp eq into not(xor(A, B)) because that
   // may lead to additional simplifications.
   if (ICI->isEquality() && CI.getType() == ICI->getOperand(0)->getType()) {
-    if (const IntegerType *ITy = dyn_cast<IntegerType>(CI.getType())) {
+    if (IntegerType *ITy = dyn_cast<IntegerType>(CI.getType())) {
       uint32_t BitWidth = ITy->getBitWidth();
       Value *LHS = ICI->getOperand(0);
       Value *RHS = ICI->getOperand(1);
@@ -644,7 +645,7 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, Instruction &CI,
 /// clear the top bits anyway, doing this has no extra cost.
 ///
 /// This function works on both vectors and scalars.
-static bool CanEvaluateZExtd(Value *V, const Type *Ty, unsigned &BitsToClear) {
+static bool CanEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear) {
   BitsToClear = 0;
   if (isa<Constant>(V))
     return true;
@@ -758,7 +759,7 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) {
     return &CI;
   
   Value *Src = CI.getOperand(0);
-  const Type *SrcTy = Src->getType(), *DestTy = CI.getType();
+  Type *SrcTy = Src->getType(), *DestTy = CI.getType();
   
   // Attempt to extend the entire input expression tree to the destination
   // type.   Only do this if the dest type is a simple type, don't convert the
@@ -820,7 +821,7 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) {
                                                            AndValue));
     }
     if (SrcSize > DstSize) {
-      Value *Trunc = Builder->CreateTrunc(A, CI.getType(), "tmp");
+      Value *Trunc = Builder->CreateTrunc(A, CI.getType());
       APInt AndValue(APInt::getLowBitsSet(DstSize, MidSize));
       return BinaryOperator::CreateAnd(Trunc, 
                                        ConstantInt::get(Trunc->getType(),
@@ -867,7 +868,7 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) {
             Value *TI0 = TI->getOperand(0);
             if (TI0->getType() == CI.getType()) {
               Constant *ZC = ConstantExpr::getZExt(C, CI.getType());
-              Value *NewAnd = Builder->CreateAnd(TI0, ZC, "tmp");
+              Value *NewAnd = Builder->CreateAnd(TI0, ZC);
               return BinaryOperator::CreateXor(NewAnd, ZC);
             }
           }
@@ -900,7 +901,7 @@ Instruction *InstCombiner::transformSExtICmp(ICmpInst *ICI, Instruction &CI) {
                                    Op0->getType()->getScalarSizeInBits()-1);
       Value *In = Builder->CreateAShr(Op0, Sh, Op0->getName()+".lobit");
       if (In->getType() != CI.getType())
-        In = Builder->CreateIntCast(In, CI.getType(), true/*SExt*/, "tmp");
+        In = Builder->CreateIntCast(In, CI.getType(), true/*SExt*/);
 
       if (Pred == ICmpInst::ICMP_SGT)
         In = Builder->CreateNot(In, In->getName()+".not");
@@ -965,10 +966,10 @@ Instruction *InstCombiner::transformSExtICmp(ICmpInst *ICI, Instruction &CI) {
   }
 
   // vector (x <s 0) ? -1 : 0 -> ashr x, 31   -> all ones if signed.
-  if (const VectorType *VTy = dyn_cast<VectorType>(CI.getType())) {
+  if (VectorType *VTy = dyn_cast<VectorType>(CI.getType())) {
     if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_Zero()) &&
         Op0->getType() == CI.getType()) {
-      const Type *EltTy = VTy->getElementType();
+      Type *EltTy = VTy->getElementType();
 
       // splat the shift constant to a constant vector.
       Constant *VSh = ConstantInt::get(VTy, EltTy->getScalarSizeInBits()-1);
@@ -988,7 +989,7 @@ Instruction *InstCombiner::transformSExtICmp(ICmpInst *ICI, Instruction &CI) {
 ///
 /// This function works on both vectors and scalars.
 ///
-static bool CanEvaluateSExtd(Value *V, const Type *Ty) {
+static bool CanEvaluateSExtd(Value *V, Type *Ty) {
   assert(V->getType()->getScalarSizeInBits() < Ty->getScalarSizeInBits() &&
          "Can't sign extend type to a smaller type");
   // If this is a constant, it can be trivially promoted.
@@ -1063,7 +1064,7 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) {
     return &CI;
   
   Value *Src = CI.getOperand(0);
-  const Type *SrcTy = Src->getType(), *DestTy = CI.getType();
+  Type *SrcTy = Src->getType(), *DestTy = CI.getType();
 
   // Attempt to extend the entire input expression tree to the destination
   // type.   Only do this if the dest type is a simple type, don't convert the
@@ -1192,7 +1193,7 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) {
     case Instruction::FMul:
     case Instruction::FDiv:
     case Instruction::FRem:
-      const Type *SrcTy = OpI->getType();
+      Type *SrcTy = OpI->getType();
       Value *LHSTrunc = LookThroughFPExtensions(OpI->getOperand(0));
       Value *RHSTrunc = LookThroughFPExtensions(OpI->getOperand(1));
       if (LHSTrunc->getType() != SrcTy && 
@@ -1306,13 +1307,13 @@ Instruction *InstCombiner::visitIntToPtr(IntToPtrInst &CI) {
     if (CI.getOperand(0)->getType()->getScalarSizeInBits() >
         TD->getPointerSizeInBits()) {
       Value *P = Builder->CreateTrunc(CI.getOperand(0),
-                                      TD->getIntPtrType(CI.getContext()), "tmp");
+                                      TD->getIntPtrType(CI.getContext()));
       return new IntToPtrInst(P, CI.getType());
     }
     if (CI.getOperand(0)->getType()->getScalarSizeInBits() <
         TD->getPointerSizeInBits()) {
       Value *P = Builder->CreateZExt(CI.getOperand(0),
-                                     TD->getIntPtrType(CI.getContext()), "tmp");
+                                     TD->getIntPtrType(CI.getContext()));
       return new IntToPtrInst(P, CI.getType());
     }
   }
@@ -1351,7 +1352,7 @@ Instruction *InstCombiner::commonPointerCastTransforms(CastInst &CI) {
       
       // Get the base pointer input of the bitcast, and the type it points to.
       Value *OrigBase = cast<BitCastInst>(GEP->getOperand(0))->getOperand(0);
-      const Type *GEPIdxTy =
+      Type *GEPIdxTy =
       cast<PointerType>(OrigBase->getType())->getElementType();
       SmallVector<Value*, 8> NewIndices;
       if (FindElementAtOffset(GEPIdxTy, Offset, NewIndices)) {
@@ -1359,9 +1360,8 @@ Instruction *InstCombiner::commonPointerCastTransforms(CastInst &CI) {
         // and bitcast the result.  This eliminates one bitcast, potentially
         // two.
         Value *NGEP = cast<GEPOperator>(GEP)->isInBounds() ?
-        Builder->CreateInBoundsGEP(OrigBase,
-                                   NewIndices.begin(), NewIndices.end()) :
-        Builder->CreateGEP(OrigBase, NewIndices.begin(), NewIndices.end());
+        Builder->CreateInBoundsGEP(OrigBase, NewIndices) :
+        Builder->CreateGEP(OrigBase, NewIndices);
         NGEP->takeName(GEP);
         
         if (isa<BitCastInst>(CI))
@@ -1382,14 +1382,12 @@ Instruction *InstCombiner::visitPtrToInt(PtrToIntInst &CI) {
   if (TD) {
     if (CI.getType()->getScalarSizeInBits() < TD->getPointerSizeInBits()) {
       Value *P = Builder->CreatePtrToInt(CI.getOperand(0),
-                                         TD->getIntPtrType(CI.getContext()),
-                                         "tmp");
+                                         TD->getIntPtrType(CI.getContext()));
       return new TruncInst(P, CI.getType());
     }
     if (CI.getType()->getScalarSizeInBits() > TD->getPointerSizeInBits()) {
       Value *P = Builder->CreatePtrToInt(CI.getOperand(0),
-                                         TD->getIntPtrType(CI.getContext()),
-                                         "tmp");
+                                         TD->getIntPtrType(CI.getContext()));
       return new ZExtInst(P, CI.getType());
     }
   }
@@ -1402,12 +1400,12 @@ Instruction *InstCombiner::visitPtrToInt(PtrToIntInst &CI) {
 /// replace it with a shuffle (and vector/vector bitcast) if possible.
 ///
 /// The source and destination vector types may have different element types.
-static Instruction *OptimizeVectorResize(Value *InVal, const VectorType *DestTy,
+static Instruction *OptimizeVectorResize(Value *InVal, VectorType *DestTy,
                                          InstCombiner &IC) {
   // We can only do this optimization if the output is a multiple of the input
   // element size, or the input is a multiple of the output element size.
   // Convert the input type to have the same element type as the output.
-  const VectorType *SrcTy = cast<VectorType>(InVal->getType());
+  VectorType *SrcTy = cast<VectorType>(InVal->getType());
   
   if (SrcTy->getElementType() != DestTy->getElementType()) {
     // The input types don't need to be identical, but for now they must be the
@@ -1427,7 +1425,7 @@ static Instruction *OptimizeVectorResize(Value *InVal, const VectorType *DestTy,
   // size of the input.
   SmallVector<Constant*, 16> ShuffleMask;
   Value *V2;
-  const IntegerType *Int32Ty = Type::getInt32Ty(SrcTy->getContext());
+  IntegerType *Int32Ty = Type::getInt32Ty(SrcTy->getContext());
   
   if (SrcTy->getNumElements() > DestTy->getNumElements()) {
     // If we're shrinking the number of elements, just shuffle in the low
@@ -1453,11 +1451,11 @@ static Instruction *OptimizeVectorResize(Value *InVal, const VectorType *DestTy,
   return new ShuffleVectorInst(InVal, V2, ConstantVector::get(ShuffleMask));
 }
 
-static bool isMultipleOfTypeSize(unsigned Value, const Type *Ty) {
+static bool isMultipleOfTypeSize(unsigned Value, Type *Ty) {
   return Value % Ty->getPrimitiveSizeInBits() == 0;
 }
 
-static unsigned getTypeSizeIndex(unsigned Value, const Type *Ty) {
+static unsigned getTypeSizeIndex(unsigned Value, Type *Ty) {
   return Value / Ty->getPrimitiveSizeInBits();
 }
 
@@ -1471,7 +1469,7 @@ static unsigned getTypeSizeIndex(unsigned Value, const Type *Ty) {
 /// filling in Elements with the elements found here.
 static bool CollectInsertionElements(Value *V, unsigned ElementIndex,
                                      SmallVectorImpl<Value*> &Elements,
-                                     const Type *VecEltTy) {
+                                     Type *VecEltTy) {
   // Undef values never contribute useful bits to the result.
   if (isa<UndefValue>(V)) return true;
   
@@ -1508,7 +1506,7 @@ static bool CollectInsertionElements(Value *V, unsigned ElementIndex,
       C = ConstantExpr::getBitCast(C, IntegerType::get(V->getContext(),
                                        C->getType()->getPrimitiveSizeInBits()));
     unsigned ElementSize = VecEltTy->getPrimitiveSizeInBits();
-    const Type *ElementIntTy = IntegerType::get(C->getContext(), ElementSize);
+    Type *ElementIntTy = IntegerType::get(C->getContext(), ElementSize);
     
     for (unsigned i = 0; i != NumElts; ++i) {
       Constant *Piece = ConstantExpr::getLShr(C, ConstantInt::get(C->getType(),
@@ -1572,7 +1570,7 @@ static bool CollectInsertionElements(Value *V, unsigned ElementIndex,
 /// Into two insertelements that do "buildvector{%inc, %inc5}".
 static Value *OptimizeIntegerToVectorInsertions(BitCastInst &CI,
                                                 InstCombiner &IC) {
-  const VectorType *DestVecTy = cast<VectorType>(CI.getType());
+  VectorType *DestVecTy = cast<VectorType>(CI.getType());
   Value *IntInput = CI.getOperand(0);
 
   SmallVector<Value*, 8> Elements(DestVecTy->getNumElements());
@@ -1599,7 +1597,7 @@ static Value *OptimizeIntegerToVectorInsertions(BitCastInst &CI,
 /// bitcast.  The various long double bitcasts can't get in here.
 static Instruction *OptimizeIntToFloatBitCast(BitCastInst &CI,InstCombiner &IC){
   Value *Src = CI.getOperand(0);
-  const Type *DestTy = CI.getType();
+  Type *DestTy = CI.getType();
 
   // If this is a bitcast from int to float, check to see if the int is an
   // extraction from a vector.
@@ -1607,7 +1605,7 @@ static Instruction *OptimizeIntToFloatBitCast(BitCastInst &CI,InstCombiner &IC){
   // bitcast(trunc(bitcast(somevector)))
   if (match(Src, m_Trunc(m_BitCast(m_Value(VecInput)))) &&
       isa<VectorType>(VecInput->getType())) {
-    const VectorType *VecTy = cast<VectorType>(VecInput->getType());
+    VectorType *VecTy = cast<VectorType>(VecInput->getType());
     unsigned DestWidth = DestTy->getPrimitiveSizeInBits();
 
     if (VecTy->getPrimitiveSizeInBits() % DestWidth == 0) {
@@ -1628,7 +1626,7 @@ static Instruction *OptimizeIntToFloatBitCast(BitCastInst &CI,InstCombiner &IC){
   if (match(Src, m_Trunc(m_LShr(m_BitCast(m_Value(VecInput)),
                                 m_ConstantInt(ShAmt)))) &&
       isa<VectorType>(VecInput->getType())) {
-    const VectorType *VecTy = cast<VectorType>(VecInput->getType());
+    VectorType *VecTy = cast<VectorType>(VecInput->getType());
     unsigned DestWidth = DestTy->getPrimitiveSizeInBits();
     if (VecTy->getPrimitiveSizeInBits() % DestWidth == 0 &&
         ShAmt->getZExtValue() % DestWidth == 0) {
@@ -1651,18 +1649,18 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
   // If the operands are integer typed then apply the integer transforms,
   // otherwise just apply the common ones.
   Value *Src = CI.getOperand(0);
-  const Type *SrcTy = Src->getType();
-  const Type *DestTy = CI.getType();
+  Type *SrcTy = Src->getType();
+  Type *DestTy = CI.getType();
 
   // Get rid of casts from one type to the same type. These are useless and can
   // be replaced by the operand.
   if (DestTy == Src->getType())
     return ReplaceInstUsesWith(CI, Src);
 
-  if (const PointerType *DstPTy = dyn_cast<PointerType>(DestTy)) {
-    const PointerType *SrcPTy = cast<PointerType>(SrcTy);
-    const Type *DstElTy = DstPTy->getElementType();
-    const Type *SrcElTy = SrcPTy->getElementType();
+  if (PointerType *DstPTy = dyn_cast<PointerType>(DestTy)) {
+    PointerType *SrcPTy = cast<PointerType>(SrcTy);
+    Type *DstElTy = DstPTy->getElementType();
+    Type *SrcElTy = SrcPTy->getElementType();
     
     // If the address spaces don't match, don't eliminate the bitcast, which is
     // required for changing types.
@@ -1693,7 +1691,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
     // If we found a path from the src to dest, create the getelementptr now.
     if (SrcElTy == DstElTy) {
       SmallVector<Value*, 8> Idxs(NumZeros+1, ZeroUInt);
-      return GetElementPtrInst::CreateInBounds(Src, Idxs.begin(), Idxs.end());
+      return GetElementPtrInst::CreateInBounds(Src, Idxs);
     }
   }
   
@@ -1702,7 +1700,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
     if (Instruction *I = OptimizeIntToFloatBitCast(CI, *this))
       return I;
 
-  if (const VectorType *DestVTy = dyn_cast<VectorType>(DestTy)) {
+  if (VectorType *DestVTy = dyn_cast<VectorType>(DestTy)) {
     if (DestVTy->getNumElements() == 1 && !SrcTy->isVectorTy()) {
       Value *Elem = Builder->CreateBitCast(Src, DestVTy->getElementType());
       return InsertElementInst::Create(UndefValue::get(DestTy), Elem,
@@ -1731,7 +1729,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
     }
   }
 
-  if (const VectorType *SrcVTy = dyn_cast<VectorType>(SrcTy)) {
+  if (VectorType *SrcVTy = dyn_cast<VectorType>(SrcTy)) {
     if (SrcVTy->getNumElements() == 1 && !DestTy->isVectorTy()) {
       Value *Elem = 
         Builder->CreateExtractElement(Src,
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index c78760b20692..bb1cbfade34d 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -13,6 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/IntrinsicInst.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Target/TargetData.h"
@@ -56,7 +57,7 @@ static bool AddWithOverflow(Constant *&Result, Constant *In1,
                             Constant *In2, bool IsSigned = false) {
   Result = ConstantExpr::getAdd(In1, In2);
 
-  if (const VectorType *VTy = dyn_cast<VectorType>(In1->getType())) {
+  if (VectorType *VTy = dyn_cast<VectorType>(In1->getType())) {
     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) {
       Constant *Idx = ConstantInt::get(Type::getInt32Ty(In1->getContext()), i);
       if (HasAddOverflow(ExtractElement(Result, Idx),
@@ -78,7 +79,7 @@ static bool HasSubOverflow(ConstantInt *Result,
                            bool IsSigned) {
   if (!IsSigned)
     return Result->getValue().ugt(In1->getValue());
-  
+
   if (In2->isNegative())
     return Result->getValue().slt(In1->getValue());
 
@@ -91,7 +92,7 @@ static bool SubWithOverflow(Constant *&Result, Constant *In1,
                             Constant *In2, bool IsSigned = false) {
   Result = ConstantExpr::getSub(In1, In2);
 
-  if (const VectorType *VTy = dyn_cast<VectorType>(In1->getType())) {
+  if (VectorType *VTy = dyn_cast<VectorType>(In1->getType())) {
     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) {
       Constant *Idx = ConstantInt::get(Type::getInt32Ty(In1->getContext()), i);
       if (HasSubOverflow(ExtractElement(Result, Idx),
@@ -128,7 +129,7 @@ static bool isSignBitCheck(ICmpInst::Predicate pred, ConstantInt *RHS,
     // True if LHS u> RHS and RHS == high-bit-mask - 1
     TrueIfSigned = true;
     return RHS->isMaxValue(true);
-  case ICmpInst::ICMP_UGE: 
+  case ICmpInst::ICMP_UGE:
     // True if LHS u>= RHS and RHS == high-bit-mask (2^7, 2^15, 2^31, etc)
     TrueIfSigned = true;
     return RHS->getValue().isSignBit();
@@ -143,7 +144,7 @@ static bool isHighOnes(const ConstantInt *CI) {
   return (~CI->getValue() + 1).isPowerOf2();
 }
 
-/// ComputeSignedMinMaxValuesFromKnownBits - Given a signed integer type and a 
+/// ComputeSignedMinMaxValuesFromKnownBits - Given a signed integer type and a
 /// set of known zero and one bits, compute the maximum and minimum values that
 /// could have the specified known zero and known one bits, returning them in
 /// min/max.
@@ -160,7 +161,7 @@ static void ComputeSignedMinMaxValuesFromKnownBits(const APInt& KnownZero,
   // bit if it is unknown.
   Min = KnownOne;
   Max = KnownOne|UnknownBits;
-  
+
   if (UnknownBits.isNegative()) { // Sign bit is unknown
     Min.setBit(Min.getBitWidth()-1);
     Max.clearBit(Max.getBitWidth()-1);
@@ -179,7 +180,7 @@ static void ComputeUnsignedMinMaxValuesFromKnownBits(const APInt &KnownZero,
          KnownZero.getBitWidth() == Max.getBitWidth() &&
          "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.");
   APInt UnknownBits = ~(KnownZero|KnownOne);
-  
+
   // The minimum value is when the unknown bits are all zeros.
   Min = KnownOne;
   // The maximum value is when the unknown bits are all ones.
@@ -201,10 +202,10 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
                              CmpInst &ICI, ConstantInt *AndCst) {
   // We need TD information to know the pointer size unless this is inbounds.
   if (!GEP->isInBounds() && TD == 0) return 0;
-  
+
   ConstantArray *Init = dyn_cast<ConstantArray>(GV->getInitializer());
   if (Init == 0 || Init->getNumOperands() > 1024) return 0;
-  
+
   // There are many forms of this optimization we can handle, for now, just do
   // the simple index into a single-dimensional array.
   //
@@ -219,31 +220,31 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
   // type they index.  Collect the indices.  This is typically for arrays of
   // structs.
   SmallVector<unsigned, 4> LaterIndices;
-  
-  const Type *EltTy = cast<ArrayType>(Init->getType())->getElementType();
+
+  Type *EltTy = cast<ArrayType>(Init->getType())->getElementType();
   for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) {
     ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i));
     if (Idx == 0) return 0;  // Variable index.
-    
+
     uint64_t IdxVal = Idx->getZExtValue();
     if ((unsigned)IdxVal != IdxVal) return 0; // Too large array index.
-    
-    if (const StructType *STy = dyn_cast<StructType>(EltTy))
+
+    if (StructType *STy = dyn_cast<StructType>(EltTy))
       EltTy = STy->getElementType(IdxVal);
-    else if (const ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) {
+    else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) {
       if (IdxVal >= ATy->getNumElements()) return 0;
       EltTy = ATy->getElementType();
     } else {
       return 0; // Unknown type.
     }
-    
+
     LaterIndices.push_back(IdxVal);
   }
-  
+
   enum { Overdefined = -3, Undefined = -2 };
 
   // Variables for our state machines.
-  
+
   // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
   // "i == 47 | i == 87", where 47 is the first index the condition is true for,
   // and 87 is the second (and last) index.  FirstTrueElement is -2 when
@@ -254,7 +255,7 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
   // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
   // form "i != 47 & i != 87".  Same state transitions as for true elements.
   int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
-  
+
   /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
   /// define a state machine that triggers for ranges of values that the index
   /// is true or false for.  This triggers on things like "abbbbc"[i] == 'b'.
@@ -262,25 +263,25 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
   /// index in the range (inclusive).  We use -2 for undefined here because we
   /// use relative comparisons and don't want 0-1 to match -1.
   int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
-  
+
   // MagicBitvector - This is a magic bitvector where we set a bit if the
   // comparison is true for element 'i'.  If there are 64 elements or less in
   // the array, this will fully represent all the comparison results.
   uint64_t MagicBitvector = 0;
-  
-  
+
+
   // Scan the array and see if one of our patterns matches.
   Constant *CompareRHS = cast<Constant>(ICI.getOperand(1));
   for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
     Constant *Elt = Init->getOperand(i);
-    
+
     // If this is indexing an array of structures, get the structure element.
     if (!LaterIndices.empty())
       Elt = ConstantExpr::getExtractValue(Elt, LaterIndices);
-    
+
     // If the element is masked, handle it.
     if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst);
-    
+
     // Find out if the comparison would be true or false for the i'th element.
     Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt,
                                                   CompareRHS, TD);
@@ -294,15 +295,15 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
         FalseRangeEnd = i;
       continue;
     }
-    
+
     // If we can't compute the result for any of the elements, we have to give
     // up evaluating the entire conditional.
     if (!isa<ConstantInt>(C)) return 0;
-    
+
     // Otherwise, we know if the comparison is true or false for this element,
     // update our state machines.
     bool IsTrueForElt = !cast<ConstantInt>(C)->isZero();
-    
+
     // State machine for single/double/range index comparison.
     if (IsTrueForElt) {
       // Update the TrueElement state machine.
@@ -314,7 +315,7 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
           SecondTrueElement = i;
         else
           SecondTrueElement = Overdefined;
-        
+
         // Update range state machine.
         if (TrueRangeEnd == (int)i-1)
           TrueRangeEnd = i;
@@ -331,7 +332,7 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
           SecondFalseElement = i;
         else
           SecondFalseElement = Overdefined;
-        
+
         // Update range state machine.
         if (FalseRangeEnd == (int)i-1)
           FalseRangeEnd = i;
@@ -339,12 +340,12 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
           FalseRangeEnd = Overdefined;
       }
     }
-    
-    
+
+
     // If this element is in range, update our magic bitvector.
     if (i < 64 && IsTrueForElt)
       MagicBitvector |= 1ULL << i;
-    
+
     // If all of our states become overdefined, bail out early.  Since the
     // predicate is expensive, only check it every 8 elements.  This is only
     // really useful for really huge arrays.
@@ -364,20 +365,20 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
   if (!GEP->isInBounds() &&
       Idx->getType()->getPrimitiveSizeInBits() > TD->getPointerSizeInBits())
     Idx = Builder->CreateTrunc(Idx, TD->getIntPtrType(Idx->getContext()));
-  
+
   // If the comparison is only true for one or two elements, emit direct
   // comparisons.
   if (SecondTrueElement != Overdefined) {
     // None true -> false.
     if (FirstTrueElement == Undefined)
       return ReplaceInstUsesWith(ICI, ConstantInt::getFalse(GEP->getContext()));
-    
+
     Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement);
-    
+
     // True for one element -> 'i == 47'.
     if (SecondTrueElement == Undefined)
       return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
-    
+
     // True for two elements -> 'i == 47 | i == 72'.
     Value *C1 = Builder->CreateICmpEQ(Idx, FirstTrueIdx);
     Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement);
@@ -391,36 +392,36 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
     // None false -> true.
     if (FirstFalseElement == Undefined)
       return ReplaceInstUsesWith(ICI, ConstantInt::getTrue(GEP->getContext()));
-    
+
     Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement);
 
     // False for one element -> 'i != 47'.
     if (SecondFalseElement == Undefined)
       return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
-     
+
     // False for two elements -> 'i != 47 & i != 72'.
     Value *C1 = Builder->CreateICmpNE(Idx, FirstFalseIdx);
     Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement);
     Value *C2 = Builder->CreateICmpNE(Idx, SecondFalseIdx);
     return BinaryOperator::CreateAnd(C1, C2);
   }
-  
+
   // If the comparison can be replaced with a range comparison for the elements
   // where it is true, emit the range check.
   if (TrueRangeEnd != Overdefined) {
     assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
-    
+
     // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
     if (FirstTrueElement) {
       Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement);
       Idx = Builder->CreateAdd(Idx, Offs);
     }
-    
+
     Value *End = ConstantInt::get(Idx->getType(),
                                   TrueRangeEnd-FirstTrueElement+1);
     return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
   }
-  
+
   // False range check.
   if (FalseRangeEnd != Overdefined) {
     assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
@@ -429,19 +430,19 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
       Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement);
       Idx = Builder->CreateAdd(Idx, Offs);
     }
-    
+
     Value *End = ConstantInt::get(Idx->getType(),
                                   FalseRangeEnd-FirstFalseElement);
     return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
   }
-  
-  
+
+
   // If a 32-bit or 64-bit magic bitvector captures the entire comparison state
   // of this load, replace it with computation that does:
   //   ((magic_cst >> i) & 1) != 0
   if (Init->getNumOperands() <= 32 ||
       (TD && Init->getNumOperands() <= 64 && TD->isLegalInteger(64))) {
-    const Type *Ty;
+    Type *Ty;
     if (Init->getNumOperands() <= 32)
       Ty = Type::getInt32Ty(Init->getContext());
     else
@@ -451,7 +452,7 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
     V = Builder->CreateAnd(ConstantInt::get(Ty, 1), V);
     return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0));
   }
-  
+
   return 0;
 }
 
@@ -465,11 +466,11 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
 /// to generate the first by knowing that pointer arithmetic doesn't overflow.
 ///
 /// If we can't emit an optimized form for this expression, this returns null.
-/// 
+///
 static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC) {
   TargetData &TD = *IC.getTargetData();
   gep_type_iterator GTI = gep_type_begin(GEP);
-  
+
   // Check to see if this gep only has a single variable index.  If so, and if
   // any constant indices are a multiple of its scale, then we can compute this
   // in terms of the scale of the variable index.  For example, if the GEP
@@ -481,9 +482,9 @@ static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC) {
     if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) {
       // Compute the aggregate offset of constant indices.
       if (CI->isZero()) continue;
-      
+
       // Handle a struct index, which adds its field offset to the pointer.
-      if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
         Offset += TD.getStructLayout(STy)->getElementOffset(CI->getZExtValue());
       } else {
         uint64_t Size = TD.getTypeAllocSize(GTI.getIndexedType());
@@ -494,33 +495,33 @@ static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC) {
       break;
     }
   }
-  
+
   // If there are no variable indices, we must have a constant offset, just
   // evaluate it the general way.
   if (i == e) return 0;
-  
+
   Value *VariableIdx = GEP->getOperand(i);
   // Determine the scale factor of the variable element.  For example, this is
   // 4 if the variable index is into an array of i32.
   uint64_t VariableScale = TD.getTypeAllocSize(GTI.getIndexedType());
-  
+
   // Verify that there are no other variable indices.  If so, emit the hard way.
   for (++i, ++GTI; i != e; ++i, ++GTI) {
     ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i));
     if (!CI) return 0;
-    
+
     // Compute the aggregate offset of constant indices.
     if (CI->isZero()) continue;
-    
+
     // Handle a struct index, which adds its field offset to the pointer.
-    if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+    if (StructType *STy = dyn_cast<StructType>(*GTI)) {
       Offset += TD.getStructLayout(STy)->getElementOffset(CI->getZExtValue());
     } else {
       uint64_t Size = TD.getTypeAllocSize(GTI.getIndexedType());
       Offset += Size*CI->getSExtValue();
     }
   }
-  
+
   // Okay, we know we have a single variable index, which must be a
   // pointer/array/vector index.  If there is no offset, life is simple, return
   // the index.
@@ -530,19 +531,19 @@ static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC) {
     // we don't need to bother extending: the extension won't affect where the
     // computation crosses zero.
     if (VariableIdx->getType()->getPrimitiveSizeInBits() > IntPtrWidth) {
-      const Type *IntPtrTy = TD.getIntPtrType(VariableIdx->getContext());
+      Type *IntPtrTy = TD.getIntPtrType(VariableIdx->getContext());
       VariableIdx = IC.Builder->CreateTrunc(VariableIdx, IntPtrTy);
     }
     return VariableIdx;
   }
-  
+
   // Otherwise, there is an index.  The computation we will do will be modulo
   // the pointer size, so get it.
   uint64_t PtrSizeMask = ~0ULL >> (64-IntPtrWidth);
-  
+
   Offset &= PtrSizeMask;
   VariableScale &= PtrSizeMask;
-  
+
   // To do this transformation, any constant index must be a multiple of the
   // variable scale factor.  For example, we can evaluate "12 + 4*i" as "3 + i",
   // but we can't evaluate "10 + 3*i" in terms of i.  Check that the offset is a
@@ -550,9 +551,9 @@ static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC) {
   int64_t NewOffs = Offset / (int64_t)VariableScale;
   if (Offset != NewOffs*(int64_t)VariableScale)
     return 0;
-  
+
   // Okay, we can do this evaluation.  Start by converting the index to intptr.
-  const Type *IntPtrTy = TD.getIntPtrType(VariableIdx->getContext());
+  Type *IntPtrTy = TD.getIntPtrType(VariableIdx->getContext());
   if (VariableIdx->getType() != IntPtrTy)
     VariableIdx = IC.Builder->CreateIntCast(VariableIdx, IntPtrTy,
                                             true /*Signed*/);
@@ -576,7 +577,7 @@ Instruction *InstCombiner::FoldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
     // know pointers can't overflow since the gep is inbounds.  See if we can
     // output an optimized form.
     Value *Offset = EvaluateGEPOffsetExpression(GEPLHS, *this);
-    
+
     // If not, synthesize the offset the hard way.
     if (Offset == 0)
       Offset = EmitGEPOffset(GEPLHS);
@@ -686,7 +687,7 @@ Instruction *InstCombiner::FoldICmpAddOpCst(ICmpInst &ICI,
     bool isTrue = ICmpInst::isTrueWhenEqual(Pred);
     return ReplaceInstUsesWith(ICI, ConstantInt::get(ICI.getType(), isTrue));
   }
-  
+
   // (X+4) == X -> false.
   if (Pred == ICmpInst::ICMP_EQ)
     return ReplaceInstUsesWith(ICI, ConstantInt::getFalse(X->getContext()));
@@ -698,22 +699,22 @@ Instruction *InstCombiner::FoldICmpAddOpCst(ICmpInst &ICI,
   // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
   // so the values can never be equal.  Similarly for all other "or equals"
   // operators.
-  
+
   // (X+1) <u X        --> X >u (MAXUINT-1)        --> X == 255
   // (X+2) <u X        --> X >u (MAXUINT-2)        --> X > 253
   // (X+MAXUINT) <u X  --> X >u (MAXUINT-MAXUINT)  --> X != 0
   if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
-    Value *R = 
+    Value *R =
       ConstantExpr::getSub(ConstantInt::getAllOnesValue(CI->getType()), CI);
     return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
   }
-  
+
   // (X+1) >u X        --> X <u (0-1)        --> X != 255
   // (X+2) >u X        --> X <u (0-2)        --> X <u 254
   // (X+MAXUINT) >u X  --> X <u (0-MAXUINT)  --> X <u 1  --> X == 0
   if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
     return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantExpr::getNeg(CI));
-  
+
   unsigned BitWidth = CI->getType()->getPrimitiveSizeInBits();
   ConstantInt *SMax = ConstantInt::get(X->getContext(),
                                        APInt::getSignedMaxValue(BitWidth));
@@ -726,14 +727,14 @@ Instruction *InstCombiner::FoldICmpAddOpCst(ICmpInst &ICI,
   // (X+ -1) <s X      --> X >s (MAXSINT- -1)        --> X != 127
   if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
     return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantExpr::getSub(SMax, CI));
-  
+
   // (X+ 1) >s X       --> X <s (MAXSINT-(1-1))       --> X != 127
   // (X+ 2) >s X       --> X <s (MAXSINT-(2-1))       --> X <s 126
   // (X+MAXSINT) >s X  --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
   // (X+MINSINT) >s X  --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
   // (X+ -2) >s X      --> X <s (MAXSINT-(-2-1))      --> X <s -126
   // (X+ -1) >s X      --> X <s (MAXSINT-(-1-1))      --> X == -128
-  
+
   assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
   Constant *C = ConstantInt::get(X->getContext(), CI->getValue()-1);
   return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantExpr::getSub(SMax, C));
@@ -745,14 +746,14 @@ Instruction *InstCombiner::FoldICmpDivCst(ICmpInst &ICI, BinaryOperator *DivI,
                                           ConstantInt *DivRHS) {
   ConstantInt *CmpRHS = cast<ConstantInt>(ICI.getOperand(1));
   const APInt &CmpRHSV = CmpRHS->getValue();
-  
-  // FIXME: If the operand types don't match the type of the divide 
+
+  // FIXME: If the operand types don't match the type of the divide
   // then don't attempt this transform. The code below doesn't have the
   // logic to deal with a signed divide and an unsigned compare (and
-  // vice versa). This is because (x /s C1) <s C2  produces different 
+  // vice versa). This is because (x /s C1) <s C2  produces different
   // results than (x /s C1) <u C2 or (x /u C1) <s C2 or even
-  // (x /u C1) <u C2.  Simply casting the operands and result won't 
-  // work. :(  The if statement below tests that condition and bails 
+  // (x /u C1) <u C2.  Simply casting the operands and result won't
+  // work. :(  The if statement below tests that condition and bails
   // if it finds it.
   bool DivIsSigned = DivI->getOpcode() == Instruction::SDiv;
   if (!ICI.isEquality() && DivIsSigned != ICI.isSigned())
@@ -768,14 +769,14 @@ Instruction *InstCombiner::FoldICmpDivCst(ICmpInst &ICI, BinaryOperator *DivI,
   }
 
   // Compute Prod = CI * DivRHS. We are essentially solving an equation
-  // of form X/C1=C2. We solve for X by multiplying C1 (DivRHS) and 
-  // C2 (CI). By solving for X we can turn this into a range check 
-  // instead of computing a divide. 
+  // of form X/C1=C2. We solve for X by multiplying C1 (DivRHS) and
+  // C2 (CI). By solving for X we can turn this into a range check
+  // instead of computing a divide.
   Constant *Prod = ConstantExpr::getMul(CmpRHS, DivRHS);
 
   // Determine if the product overflows by seeing if the product is
   // not equal to the divide. Make sure we do the same kind of divide
-  // as in the LHS instruction that we're folding. 
+  // as in the LHS instruction that we're folding.
   bool ProdOV = (DivIsSigned ? ConstantExpr::getSDiv(Prod, DivRHS) :
                  ConstantExpr::getUDiv(Prod, DivRHS)) != CmpRHS;
 
@@ -785,9 +786,9 @@ Instruction *InstCombiner::FoldICmpDivCst(ICmpInst &ICI, BinaryOperator *DivI,
   /// If the division is known to be exact, then there is no remainder from the
   /// divide, so the covered range size is unit, otherwise it is the divisor.
   ConstantInt *RangeSize = DivI->isExact() ? getOne(Prod) : DivRHS;
-  
+
   // Figure out the interval that is being checked.  For example, a comparison
-  // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). 
+  // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
   // Compute this interval based on the constants involved and the signedness of
   // the compare/divide.  This computes a half-open interval, keeping track of
   // whether either value in the interval overflows.  After analysis each
@@ -805,7 +806,7 @@ Instruction *InstCombiner::FoldICmpDivCst(ICmpInst &ICI, BinaryOperator *DivI,
       // to the same result value.
       HiOverflow = AddWithOverflow(HiBound, LoBound, RangeSize, false);
     }
-    
+
   } else if (DivRHS->getValue().isStrictlyPositive()) { // Divisor is > 0.
     if (CmpRHSV == 0) {       // (X / pos) op 0
       // Can't overflow.  e.g.  X/2 op 0 --> [-1, 2)
@@ -848,7 +849,7 @@ Instruction *InstCombiner::FoldICmpDivCst(ICmpInst &ICI, BinaryOperator *DivI,
       if (!HiOverflow)
         HiOverflow = SubWithOverflow(HiBound, Prod, RangeSize, true);
     }
-    
+
     // Dividing by a negative swaps the condition.  LT <-> GT
     Pred = ICmpInst::getSwappedPredicate(Pred);
   }
@@ -901,7 +902,7 @@ Instruction *InstCombiner::FoldICmpDivCst(ICmpInst &ICI, BinaryOperator *DivI,
 Instruction *InstCombiner::FoldICmpShrCst(ICmpInst &ICI, BinaryOperator *Shr,
                                           ConstantInt *ShAmt) {
   const APInt &CmpRHSV = cast<ConstantInt>(ICI.getOperand(1))->getValue();
-  
+
   // Check that the shift amount is in range.  If not, don't perform
   // undefined shifts.  When the shift is visited it will be
   // simplified.
@@ -909,48 +910,48 @@ Instruction *InstCombiner::FoldICmpShrCst(ICmpInst &ICI, BinaryOperator *Shr,
   uint32_t ShAmtVal = (uint32_t)ShAmt->getLimitedValue(TypeBits);
   if (ShAmtVal >= TypeBits || ShAmtVal == 0)
     return 0;
-  
+
   if (!ICI.isEquality()) {
     // If we have an unsigned comparison and an ashr, we can't simplify this.
     // Similarly for signed comparisons with lshr.
     if (ICI.isSigned() != (Shr->getOpcode() == Instruction::AShr))
       return 0;
-    
+
     // Otherwise, all lshr and most exact ashr's are equivalent to a udiv/sdiv
     // by a power of 2.  Since we already have logic to simplify these,
     // transform to div and then simplify the resultant comparison.
     if (Shr->getOpcode() == Instruction::AShr &&
         (!Shr->isExact() || ShAmtVal == TypeBits - 1))
       return 0;
-    
+
     // Revisit the shift (to delete it).
     Worklist.Add(Shr);
-    
+
     Constant *DivCst =
       ConstantInt::get(Shr->getType(), APInt::getOneBitSet(TypeBits, ShAmtVal));
-    
+
     Value *Tmp =
       Shr->getOpcode() == Instruction::AShr ?
       Builder->CreateSDiv(Shr->getOperand(0), DivCst, "", Shr->isExact()) :
       Builder->CreateUDiv(Shr->getOperand(0), DivCst, "", Shr->isExact());
-    
+
     ICI.setOperand(0, Tmp);
-    
+
     // If the builder folded the binop, just return it.
     BinaryOperator *TheDiv = dyn_cast<BinaryOperator>(Tmp);
     if (TheDiv == 0)
       return &ICI;
-    
+
     // Otherwise, fold this div/compare.
     assert(TheDiv->getOpcode() == Instruction::SDiv ||
            TheDiv->getOpcode() == Instruction::UDiv);
-    
+
     Instruction *Res = FoldICmpDivCst(ICI, TheDiv, cast<ConstantInt>(DivCst));
     assert(Res && "This div/cst should have folded!");
     return Res;
   }
-  
-  
+
+
   // If we are comparing against bits always shifted out, the
   // comparison cannot succeed.
   APInt Comp = CmpRHSV << ShAmtVal;
@@ -959,25 +960,25 @@ Instruction *InstCombiner::FoldICmpShrCst(ICmpInst &ICI, BinaryOperator *Shr,
     Comp = Comp.lshr(ShAmtVal);
   else
     Comp = Comp.ashr(ShAmtVal);
-  
+
   if (Comp != CmpRHSV) { // Comparing against a bit that we know is zero.
     bool IsICMP_NE = ICI.getPredicate() == ICmpInst::ICMP_NE;
     Constant *Cst = ConstantInt::get(Type::getInt1Ty(ICI.getContext()),
                                      IsICMP_NE);
     return ReplaceInstUsesWith(ICI, Cst);
   }
-  
+
   // Otherwise, check to see if the bits shifted out are known to be zero.
   // If so, we can compare against the unshifted value:
   //  (X & 4) >> 1 == 2  --> (X & 4) == 4.
   if (Shr->hasOneUse() && Shr->isExact())
     return new ICmpInst(ICI.getPredicate(), Shr->getOperand(0), ShiftedCmpRHS);
-  
+
   if (Shr->hasOneUse()) {
     // Otherwise strength reduce the shift into an and.
     APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal));
     Constant *Mask = ConstantInt::get(ICI.getContext(), Val);
-    
+
     Value *And = Builder->CreateAnd(Shr->getOperand(0),
                                     Mask, Shr->getName()+".mask");
     return new ICmpInst(ICI.getPredicate(), And, ShiftedCmpRHS);
@@ -992,7 +993,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
                                                           Instruction *LHSI,
                                                           ConstantInt *RHS) {
   const APInt &RHSV = RHS->getValue();
-  
+
   switch (LHSI->getOpcode()) {
   case Instruction::Trunc:
     if (ICI.isEquality() && LHSI->hasOneUse()) {
@@ -1003,7 +1004,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
       APInt Mask(APInt::getHighBitsSet(SrcBits, SrcBits-DstBits));
       APInt KnownZero(SrcBits, 0), KnownOne(SrcBits, 0);
       ComputeMaskedBits(LHSI->getOperand(0), Mask, KnownZero, KnownOne);
-      
+
       // If all the high bits are known, we can do this xform.
       if ((KnownZero|KnownOne).countLeadingOnes() >= SrcBits-DstBits) {
         // Pull in the high bits from known-ones set.
@@ -1014,7 +1015,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
       }
     }
     break;
-      
+
   case Instruction::Xor:         // (icmp pred (xor X, XorCST), CI)
     if (ConstantInt *XorCST = dyn_cast<ConstantInt>(LHSI->getOperand(1))) {
       // If this is a comparison that tests the signbit (X < 0) or (x > -1),
@@ -1022,7 +1023,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
       if ((ICI.getPredicate() == ICmpInst::ICMP_SLT && RHSV == 0) ||
           (ICI.getPredicate() == ICmpInst::ICMP_SGT && RHSV.isAllOnesValue())) {
         Value *CompareVal = LHSI->getOperand(0);
-        
+
         // If the sign bit of the XorCST is not set, there is no change to
         // the operation, just stop using the Xor.
         if (!XorCST->isNegative()) {
@@ -1030,13 +1031,13 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           Worklist.Add(LHSI);
           return &ICI;
         }
-        
+
         // Was the old condition true if the operand is positive?
         bool isTrueIfPositive = ICI.getPredicate() == ICmpInst::ICMP_SGT;
-        
+
         // If so, the new one isn't.
         isTrueIfPositive ^= true;
-        
+
         if (isTrueIfPositive)
           return new ICmpInst(ICmpInst::ICMP_SGT, CompareVal,
                               SubOne(RHS));
@@ -1075,13 +1076,13 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
     if (LHSI->hasOneUse() && isa<ConstantInt>(LHSI->getOperand(1)) &&
         LHSI->getOperand(0)->hasOneUse()) {
       ConstantInt *AndCST = cast<ConstantInt>(LHSI->getOperand(1));
-      
+
       // If the LHS is an AND of a truncating cast, we can widen the
       // and/compare to be the input width without changing the value
       // produced, eliminating a cast.
       if (TruncInst *Cast = dyn_cast<TruncInst>(LHSI->getOperand(0))) {
         // We can do this transformation if either the AND constant does not
-        // have its sign bit set or if it is an equality comparison. 
+        // have its sign bit set or if it is an equality comparison.
         // Extending a relational comparison when we're checking the sign
         // bit would not work.
         if (ICI.isEquality() ||
@@ -1098,7 +1099,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
       // If the LHS is an AND of a zext, and we have an equality compare, we can
       // shrink the and/compare to the smaller type, eliminating the cast.
       if (ZExtInst *Cast = dyn_cast<ZExtInst>(LHSI->getOperand(0))) {
-        const IntegerType *Ty = cast<IntegerType>(Cast->getSrcTy());
+        IntegerType *Ty = cast<IntegerType>(Cast->getSrcTy());
         // Make sure we don't compare the upper bits, SimplifyDemandedBits
         // should fold the icmp to true/false in that case.
         if (ICI.isEquality() && RHSV.getActiveBits() <= Ty->getBitWidth()) {
@@ -1118,12 +1119,12 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
       BinaryOperator *Shift = dyn_cast<BinaryOperator>(LHSI->getOperand(0));
       if (Shift && !Shift->isShift())
         Shift = 0;
-      
+
       ConstantInt *ShAmt;
       ShAmt = Shift ? dyn_cast<ConstantInt>(Shift->getOperand(1)) : 0;
-      const Type *Ty = Shift ? Shift->getType() : 0;  // Type of the shift.
-      const Type *AndTy = AndCST->getType();          // Type of the and.
-      
+      Type *Ty = Shift ? Shift->getType() : 0;  // Type of the shift.
+      Type *AndTy = AndCST->getType();          // Type of the and.
+
       // We can fold this as long as we can't shift unknown bits
       // into the mask.  This can only happen with signed shift
       // rights, as they sign-extend.
@@ -1134,20 +1135,20 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           // of the bits shifted in could be tested after the mask.
           uint32_t TyBits = Ty->getPrimitiveSizeInBits();
           int ShAmtVal = TyBits - ShAmt->getLimitedValue(TyBits);
-          
+
           uint32_t BitWidth = AndTy->getPrimitiveSizeInBits();
-          if ((APInt::getHighBitsSet(BitWidth, BitWidth-ShAmtVal) & 
+          if ((APInt::getHighBitsSet(BitWidth, BitWidth-ShAmtVal) &
                AndCST->getValue()) == 0)
             CanFold = true;
         }
-        
+
         if (CanFold) {
           Constant *NewCst;
           if (Shift->getOpcode() == Instruction::Shl)
             NewCst = ConstantExpr::getLShr(RHS, ShAmt);
           else
             NewCst = ConstantExpr::getShl(RHS, ShAmt);
-          
+
           // Check to see if we are shifting out any of the bits being
           // compared.
           if (ConstantExpr::get(Shift->getOpcode(),
@@ -1175,7 +1176,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           }
         }
       }
-      
+
       // Turn ((X >> Y) & C) == 0  into  (X & (C << Y)) == 0.  The later is
       // preferable because it allows the C<<Y expression to be hoisted out
       // of a loop if Y is invariant and X is not.
@@ -1185,21 +1186,21 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
         // Compute C << Y.
         Value *NS;
         if (Shift->getOpcode() == Instruction::LShr) {
-          NS = Builder->CreateShl(AndCST, Shift->getOperand(1), "tmp");
+          NS = Builder->CreateShl(AndCST, Shift->getOperand(1));
         } else {
           // Insert a logical shift.
-          NS = Builder->CreateLShr(AndCST, Shift->getOperand(1), "tmp");
+          NS = Builder->CreateLShr(AndCST, Shift->getOperand(1));
         }
-        
+
         // Compute X & (C << Y).
-        Value *NewAnd = 
+        Value *NewAnd =
           Builder->CreateAnd(Shift->getOperand(0), NS, LHSI->getName());
-        
+
         ICI.setOperand(0, NewAnd);
         return &ICI;
       }
     }
-      
+
     // Try to optimize things like "A[i]&42 == 0" to index computations.
     if (LoadInst *LI = dyn_cast<LoadInst>(LHSI->getOperand(0))) {
       if (GetElementPtrInst *GEP =
@@ -1234,19 +1235,19 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
     }
     break;
   }
-    
+
   case Instruction::Shl: {       // (icmp pred (shl X, ShAmt), CI)
     ConstantInt *ShAmt = dyn_cast<ConstantInt>(LHSI->getOperand(1));
     if (!ShAmt) break;
-    
+
     uint32_t TypeBits = RHSV.getBitWidth();
-    
+
     // Check that the shift amount is in range.  If not, don't perform
     // undefined shifts.  When the shift is visited it will be
     // simplified.
     if (ShAmt->uge(TypeBits))
       break;
-    
+
     if (ICI.isEquality()) {
       // If we are comparing against bits always shifted out, the
       // comparison cannot succeed.
@@ -1259,34 +1260,34 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           ConstantInt::get(Type::getInt1Ty(ICI.getContext()), IsICMP_NE);
         return ReplaceInstUsesWith(ICI, Cst);
       }
-      
+
       // If the shift is NUW, then it is just shifting out zeros, no need for an
       // AND.
       if (cast<BinaryOperator>(LHSI)->hasNoUnsignedWrap())
         return new ICmpInst(ICI.getPredicate(), LHSI->getOperand(0),
                             ConstantExpr::getLShr(RHS, ShAmt));
-      
+
       if (LHSI->hasOneUse()) {
         // Otherwise strength reduce the shift into an and.
         uint32_t ShAmtVal = (uint32_t)ShAmt->getLimitedValue(TypeBits);
         Constant *Mask =
-          ConstantInt::get(ICI.getContext(), APInt::getLowBitsSet(TypeBits, 
+          ConstantInt::get(ICI.getContext(), APInt::getLowBitsSet(TypeBits,
                                                        TypeBits-ShAmtVal));
-        
+
         Value *And =
           Builder->CreateAnd(LHSI->getOperand(0),Mask, LHSI->getName()+".mask");
         return new ICmpInst(ICI.getPredicate(), And,
                             ConstantExpr::getLShr(RHS, ShAmt));
       }
     }
-    
+
     // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
     bool TrueIfSigned = false;
     if (LHSI->hasOneUse() &&
         isSignBitCheck(ICI.getPredicate(), RHS, TrueIfSigned)) {
       // (X << 31) <s 0  --> (X&1) != 0
       Constant *Mask = ConstantInt::get(LHSI->getOperand(0)->getType(),
-                                        APInt::getOneBitSet(TypeBits, 
+                                        APInt::getOneBitSet(TypeBits,
                                             TypeBits-ShAmt->getZExtValue()-1));
       Value *And =
         Builder->CreateAnd(LHSI->getOperand(0), Mask, LHSI->getName()+".mask");
@@ -1295,7 +1296,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
     }
     break;
   }
-    
+
   case Instruction::LShr:         // (icmp pred (shr X, ShAmt), CI)
   case Instruction::AShr: {
     // Handle equality comparisons of shift-by-constant.
@@ -1312,13 +1313,13 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
     }
     break;
   }
-    
+
   case Instruction::SDiv:
   case Instruction::UDiv:
     // Fold: icmp pred ([us]div X, C1), C2 -> range test
-    // Fold this div into the comparison, producing a range check. 
-    // Determine, based on the divide type, what the range is being 
-    // checked.  If there is an overflow on the low or high side, remember 
+    // Fold this div into the comparison, producing a range check.
+    // Determine, based on the divide type, what the range is being
+    // checked.  If there is an overflow on the low or high side, remember
     // it, otherwise compute the range [low, hi) bounding the new value.
     // See: InsertRangeTest above for the kinds of replacements possible.
     if (ConstantInt *DivRHS = dyn_cast<ConstantInt>(LHSI->getOperand(1)))
@@ -1357,12 +1358,12 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
     }
     break;
   }
-  
+
   // Simplify icmp_eq and icmp_ne instructions with integer constant RHS.
   if (ICI.isEquality()) {
     bool isICMP_NE = ICI.getPredicate() == ICmpInst::ICMP_NE;
-    
-    // If the first operand is (add|sub|and|or|xor|rem) with a constant, and 
+
+    // If the first operand is (add|sub|and|or|xor|rem) with a constant, and
     // the second operand is a constant, simplify a bit.
     if (BinaryOperator *BO = dyn_cast<BinaryOperator>(LHSI)) {
       switch (BO->getOpcode()) {
@@ -1389,7 +1390,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           // Replace ((add A, B) != 0) with (A != -B) if A or B is
           // efficiently invertible, or if the add has just this one use.
           Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
-          
+
           if (Value *NegVal = dyn_castNegVal(BOp1))
             return new ICmpInst(ICI.getPredicate(), BOp0, NegVal);
           if (Value *NegVal = dyn_castNegVal(BOp0))
@@ -1432,11 +1433,11 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           Constant *NotCI = ConstantExpr::getNot(RHS);
           if (!ConstantExpr::getAnd(BOC, NotCI)->isNullValue())
             return ReplaceInstUsesWith(ICI,
-                             ConstantInt::get(Type::getInt1Ty(ICI.getContext()), 
+                             ConstantInt::get(Type::getInt1Ty(ICI.getContext()),
                                        isICMP_NE));
         }
         break;
-        
+
       case Instruction::And:
         if (ConstantInt *BOC = dyn_cast<ConstantInt>(BO->getOperand(1))) {
           // If bits are being compared against that are and'd out, then the
@@ -1445,7 +1446,7 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
             return ReplaceInstUsesWith(ICI,
                              ConstantInt::get(Type::getInt1Ty(ICI.getContext()),
                                        isICMP_NE));
-          
+
           // If we have ((X & C) == C), turn it into ((X & C) != 0).
           if (RHS == BOC && RHSV.isPowerOf2())
             return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ :
@@ -1460,16 +1461,16 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
           if (BOC->getValue().isSignBit()) {
             Value *X = BO->getOperand(0);
             Constant *Zero = Constant::getNullValue(X->getType());
-            ICmpInst::Predicate pred = isICMP_NE ? 
+            ICmpInst::Predicate pred = isICMP_NE ?
               ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
             return new ICmpInst(pred, X, Zero);
           }
-          
+
           // ((X & ~7) == 0) --> X < 8
           if (RHSV == 0 && isHighOnes(BOC)) {
             Value *X = BO->getOperand(0);
             Constant *NegX = ConstantExpr::getNeg(BOC);
-            ICmpInst::Predicate pred = isICMP_NE ? 
+            ICmpInst::Predicate pred = isICMP_NE ?
               ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
             return new ICmpInst(pred, X, NegX);
           }
@@ -1517,11 +1518,11 @@ Instruction *InstCombiner::visitICmpInstWithInstAndIntCst(ICmpInst &ICI,
 Instruction *InstCombiner::visitICmpInstWithCastAndCast(ICmpInst &ICI) {
   const CastInst *LHSCI = cast<CastInst>(ICI.getOperand(0));
   Value *LHSCIOp        = LHSCI->getOperand(0);
-  const Type *SrcTy     = LHSCIOp->getType();
-  const Type *DestTy    = LHSCI->getType();
+  Type *SrcTy     = LHSCIOp->getType();
+  Type *DestTy    = LHSCI->getType();
   Value *RHSCIOp;
 
-  // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the 
+  // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
   // integer type is the same size as the pointer type.
   if (TD && LHSCI->getOpcode() == Instruction::PtrToInt &&
       TD->getPointerSizeInBits() ==
@@ -1539,7 +1540,7 @@ Instruction *InstCombiner::visitICmpInstWithCastAndCast(ICmpInst &ICI) {
     if (RHSOp)
       return new ICmpInst(ICI.getPredicate(), LHSCIOp, RHSOp);
   }
-  
+
   // The code below only handles extension cast instructions, so far.
   // Enforce this.
   if (LHSCI->getOpcode() != Instruction::ZExt &&
@@ -1552,9 +1553,9 @@ Instruction *InstCombiner::visitICmpInstWithCastAndCast(ICmpInst &ICI) {
   if (CastInst *CI = dyn_cast<CastInst>(ICI.getOperand(1))) {
     // Not an extension from the same type?
     RHSCIOp = CI->getOperand(0);
-    if (RHSCIOp->getType() != LHSCIOp->getType()) 
+    if (RHSCIOp->getType() != LHSCIOp->getType())
       return 0;
-    
+
     // If the signedness of the two casts doesn't agree (i.e. one is a sext
     // and the other is a zext), then we can't handle this.
     if (CI->getOpcode() != LHSCI->getOpcode())
@@ -1599,7 +1600,7 @@ Instruction *InstCombiner::visitICmpInstWithCastAndCast(ICmpInst &ICI) {
     return new ICmpInst(ICI.getUnsignedPredicate(), LHSCIOp, Res1);
   }
 
-  // The re-extended constant changed so the constant cannot be represented 
+  // The re-extended constant changed so the constant cannot be represented
   // in the shorter type. Consequently, we cannot emit a simple comparison.
   // All the cases that fold to true or false will have already been handled
   // by SimplifyICmpInst, so only deal with the tricky case.
@@ -1637,26 +1638,26 @@ static Instruction *ProcessUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
   // llvm.sadd.with.overflow.  To do this, we have to replace the original add
   // with a narrower add, and discard the add-with-constant that is part of the
   // range check (if we can't eliminate it, this isn't profitable).
-  
+
   // In order to eliminate the add-with-constant, the compare can be its only
   // use.
   Instruction *AddWithCst = cast<Instruction>(I.getOperand(0));
   if (!AddWithCst->hasOneUse()) return 0;
-  
+
   // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
   if (!CI2->getValue().isPowerOf2()) return 0;
   unsigned NewWidth = CI2->getValue().countTrailingZeros();
   if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) return 0;
-    
+
   // The width of the new add formed is 1 more than the bias.
   ++NewWidth;
-  
+
   // Check to see that CI1 is an all-ones value with NewWidth bits.
   if (CI1->getBitWidth() == NewWidth ||
       CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth))
     return 0;
-  
-  // In order to replace the original add with a narrower 
+
+  // In order to replace the original add with a narrower
   // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
   // and truncates that discard the high bits of the add.  Verify that this is
   // the case.
@@ -1664,7 +1665,7 @@ static Instruction *ProcessUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
   for (Value::use_iterator UI = OrigAdd->use_begin(), E = OrigAdd->use_end();
        UI != E; ++UI) {
     if (*UI == AddWithCst) continue;
-    
+
     // Only accept truncates for now.  We would really like a nice recursive
     // predicate like SimplifyDemandedBits, but which goes downwards the use-def
     // chain to see which bits of a value are actually demanded.  If the
@@ -1674,32 +1675,32 @@ static Instruction *ProcessUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
     if (TI == 0 ||
         TI->getType()->getPrimitiveSizeInBits() > NewWidth) return 0;
   }
-  
+
   // If the pattern matches, truncate the inputs to the narrower type and
   // use the sadd_with_overflow intrinsic to efficiently compute both the
   // result and the overflow bit.
   Module *M = I.getParent()->getParent()->getParent();
-  
+
   Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
   Value *F = Intrinsic::getDeclaration(M, Intrinsic::sadd_with_overflow,
                                        NewType);
 
   InstCombiner::BuilderTy *Builder = IC.Builder;
-  
+
   // Put the new code above the original add, in case there are any uses of the
   // add between the add and the compare.
   Builder->SetInsertPoint(OrigAdd);
-  
+
   Value *TruncA = Builder->CreateTrunc(A, NewType, A->getName()+".trunc");
   Value *TruncB = Builder->CreateTrunc(B, NewType, B->getName()+".trunc");
   CallInst *Call = Builder->CreateCall2(F, TruncA, TruncB, "sadd");
   Value *Add = Builder->CreateExtractValue(Call, 0, "sadd.result");
   Value *ZExt = Builder->CreateZExt(Add, OrigAdd->getType());
-  
+
   // The inner add was the result of the narrow add, zero extended to the
   // wider type.  Replace it with the result computed by the intrinsic.
   IC.ReplaceInstUsesWith(*OrigAdd, ZExt);
-  
+
   // The original icmp gets replaced with the overflow value.
   return ExtractValueInst::Create(Call, 1, "sadd.overflow");
 }
@@ -1709,13 +1710,13 @@ static Instruction *ProcessUAddIdiom(Instruction &I, Value *OrigAddV,
   // Don't bother doing this transformation for pointers, don't do it for
   // vectors.
   if (!isa<IntegerType>(OrigAddV->getType())) return 0;
-  
+
   // If the add is a constant expr, then we don't bother transforming it.
   Instruction *OrigAdd = dyn_cast<Instruction>(OrigAddV);
   if (OrigAdd == 0) return 0;
-  
+
   Value *LHS = OrigAdd->getOperand(0), *RHS = OrigAdd->getOperand(1);
-  
+
   // Put the new code above the original add, in case there are any uses of the
   // add between the add and the compare.
   InstCombiner::BuilderTy *Builder = IC.Builder;
@@ -1740,13 +1741,13 @@ static APInt DemandedBitsLHSMask(ICmpInst &I,
                                  unsigned BitWidth, bool isSignCheck) {
   if (isSignCheck)
     return APInt::getSignBit(BitWidth);
-  
+
   ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(1));
   if (!CI) return APInt::getAllOnesValue(BitWidth);
   const APInt &RHS = CI->getValue();
-  
+
   switch (I.getPredicate()) {
-  // For a UGT comparison, we don't care about any bits that 
+  // For a UGT comparison, we don't care about any bits that
   // correspond to the trailing ones of the comparand.  The value of these
   // bits doesn't impact the outcome of the comparison, because any value
   // greater than the RHS must differ in a bit higher than these due to carry.
@@ -1755,7 +1756,7 @@ static APInt DemandedBitsLHSMask(ICmpInst &I,
     APInt lowBitsSet = APInt::getLowBitsSet(BitWidth, trailingOnes);
     return ~lowBitsSet;
   }
-  
+
   // Similarly, for a ULT comparison, we don't care about the trailing zeros.
   // Any value less than the RHS must differ in a higher bit because of carries.
   case ICmpInst::ICMP_ULT: {
@@ -1763,17 +1764,17 @@ static APInt DemandedBitsLHSMask(ICmpInst &I,
     APInt lowBitsSet = APInt::getLowBitsSet(BitWidth, trailingZeros);
     return ~lowBitsSet;
   }
-  
+
   default:
     return APInt::getAllOnesValue(BitWidth);
   }
-  
+
 }
 
 Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
-  
+
   /// Orders the operands of the compare so that they are listed from most
   /// complex to least complex.  This puts constants before unary operators,
   /// before binary operators.
@@ -1782,11 +1783,11 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     std::swap(Op0, Op1);
     Changed = true;
   }
-  
+
   if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, TD))
     return ReplaceInstUsesWith(I, V);
-  
-  const Type *Ty = Op0->getType();
+
+  Type *Ty = Op0->getType();
 
   // icmp's with boolean values can always be turned into bitwise operations
   if (Ty->isIntegerTy(1)) {
@@ -1835,13 +1836,13 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     BitWidth = Ty->getScalarSizeInBits();
   else if (TD)  // Pointers require TD info to get their size.
     BitWidth = TD->getTypeSizeInBits(Ty->getScalarType());
-  
+
   bool isSignBit = false;
 
   // See if we are doing a comparison with a constant.
   if (ConstantInt *CI = dyn_cast<ConstantInt>(Op1)) {
     Value *A = 0, *B = 0;
-    
+
     // Match the following pattern, which is a common idiom when writing
     // overflow-safe integer arithmetic function.  The source performs an
     // addition in wider type, and explicitly checks for overflow using
@@ -1849,9 +1850,9 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     // sadd_with_overflow intrinsic.
     //
     // TODO: This could probably be generalized to handle other overflow-safe
-    // operations if we worked out the formulas to compute the appropriate 
+    // operations if we worked out the formulas to compute the appropriate
     // magic constants.
-    // 
+    //
     // sum = a + b
     // if (sum+128 >u 255)  ...  -> llvm.sadd.with.overflow.i8
     {
@@ -1861,14 +1862,14 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       if (Instruction *Res = ProcessUGT_ADDCST_ADD(I, A, B, CI2, CI, *this))
         return Res;
     }
-    
+
     // (icmp ne/eq (sub A B) 0) -> (icmp ne/eq A, B)
     if (I.isEquality() && CI->isZero() &&
         match(Op0, m_Sub(m_Value(A), m_Value(B)))) {
       // (icmp cond A B) if cond is equality
       return new ICmpInst(I.getPredicate(), A, B);
     }
-    
+
     // If we have an icmp le or icmp ge instruction, turn it into the
     // appropriate icmp lt or icmp gt instruction.  This allows us to rely on
     // them being folded in the code below.  The SimplifyICmpInst code has
@@ -1892,7 +1893,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       return new ICmpInst(ICmpInst::ICMP_SGT, Op0,
                           ConstantInt::get(CI->getContext(), CI->getValue()-1));
     }
-    
+
     // If this comparison is a normal comparison, it demands all
     // bits, if it is a sign bit comparison, it only demands the sign bit.
     bool UnusedBit;
@@ -1948,7 +1949,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     case ICmpInst::ICMP_EQ: {
       if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max))
         return ReplaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
-        
+
       // If all bits are known zero except for one, then we know at most one
       // bit is set.   If the comparison is against zero, then this is a check
       // to see if *that* bit is set.
@@ -1960,7 +1961,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
         if (!match(Op0, m_And(m_Value(LHS), m_ConstantInt(LHSC))) ||
             LHSC->getValue() != Op0KnownZeroInverted)
           LHS = Op0;
-        
+
         // If the LHS is 1 << x, and we know the result is a power of 2 like 8,
         // then turn "((1 << x)&8) == 0" into "x != 3".
         Value *X = 0;
@@ -1969,7 +1970,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
           return new ICmpInst(ICmpInst::ICMP_NE, X,
                               ConstantInt::get(X->getType(), CmpVal));
         }
-        
+
         // If the LHS is 8 >>u x, and we know the result is a power of 2 like 1,
         // then turn "((8 >>u x)&1) == 0" into "x != 3".
         const APInt *CI;
@@ -1979,13 +1980,13 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
                               ConstantInt::get(X->getType(),
                                                CI->countTrailingZeros()));
       }
-        
+
       break;
     }
     case ICmpInst::ICMP_NE: {
       if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max))
         return ReplaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
-      
+
       // If all bits are known zero except for one, then we know at most one
       // bit is set.   If the comparison is against zero, then this is a check
       // to see if *that* bit is set.
@@ -1997,7 +1998,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
         if (!match(Op0, m_And(m_Value(LHS), m_ConstantInt(LHSC))) ||
             LHSC->getValue() != Op0KnownZeroInverted)
           LHS = Op0;
-        
+
         // If the LHS is 1 << x, and we know the result is a power of 2 like 8,
         // then turn "((1 << x)&8) != 0" into "x == 3".
         Value *X = 0;
@@ -2006,7 +2007,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
           return new ICmpInst(ICmpInst::ICMP_EQ, X,
                               ConstantInt::get(X->getType(), CmpVal));
         }
-        
+
         // If the LHS is 8 >>u x, and we know the result is a power of 2 like 1,
         // then turn "((8 >>u x)&1) != 0" into "x == 3".
         const APInt *CI;
@@ -2016,7 +2017,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
                               ConstantInt::get(X->getType(),
                                                CI->countTrailingZeros()));
       }
-      
+
       break;
     }
     case ICmpInst::ICMP_ULT:
@@ -2137,9 +2138,9 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   // See if we are doing a comparison between a constant and an instruction that
   // can be folded into the comparison.
   if (ConstantInt *CI = dyn_cast<ConstantInt>(Op1)) {
-    // Since the RHS is a ConstantInt (CI), if the left hand side is an 
-    // instruction, see if that instruction also has constants so that the 
-    // instruction can be folded into the icmp 
+    // Since the RHS is a ConstantInt (CI), if the left hand side is an
+    // instruction, see if that instruction also has constants so that the
+    // instruction can be folded into the icmp
     if (Instruction *LHSI = dyn_cast<Instruction>(Op0))
       if (Instruction *Res = visitICmpInstWithInstAndIntCst(I, LHSI, CI))
         return Res;
@@ -2194,7 +2195,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       case Instruction::IntToPtr:
         // icmp pred inttoptr(X), null -> icmp pred X, 0
         if (RHSC->isNullValue() && TD &&
-            TD->getIntPtrType(RHSC->getContext()) == 
+            TD->getIntPtrType(RHSC->getContext()) ==
                LHSI->getOperand(0)->getType())
           return new ICmpInst(I.getPredicate(), LHSI->getOperand(0),
                         Constant::getNullValue(LHSI->getOperand(0)->getType()));
@@ -2227,8 +2228,8 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   // values.  If the ptr->ptr cast can be stripped off both arguments, we do so
   // now.
   if (BitCastInst *CI = dyn_cast<BitCastInst>(Op0)) {
-    if (Op0->getType()->isPointerTy() && 
-        (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { 
+    if (Op0->getType()->isPointerTy() &&
+        (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) {
       // We keep moving the cast from the left operand over to the right
       // operand, where it can often be eliminated completely.
       Op0 = CI->getOperand(0);
@@ -2250,7 +2251,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       return new ICmpInst(I.getPredicate(), Op0, Op1);
     }
   }
-  
+
   if (isa<CastInst>(Op0)) {
     // Handle the special case of: icmp (cast bool to X), <cst>
     // This comes up when you have code like
@@ -2384,7 +2385,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
             return new ICmpInst(Pred, BO0->getOperand(0),
                                 BO1->getOperand(0));
           }
-          
+
           if (CI->isMaxValue(true)) {
             ICmpInst::Predicate Pred = I.isSigned()
                                            ? I.getUnsignedPredicate()
@@ -2404,7 +2405,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
           // Mask = -1 >> count-trailing-zeros(Cst).
           if (!CI->isZero() && !CI->isOne()) {
             const APInt &AP = CI->getValue();
-            ConstantInt *Mask = ConstantInt::get(I.getContext(), 
+            ConstantInt *Mask = ConstantInt::get(I.getContext(),
                                     APInt::getLowBitsSet(AP.getBitWidth(),
                                                          AP.getBitWidth() -
                                                     AP.countTrailingZeros()));
@@ -2438,7 +2439,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       }
     }
   }
-  
+
   { Value *A, *B;
     // ~x < ~y --> y < x
     // ~x < cst --> ~cst < x
@@ -2452,11 +2453,11 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     // (a+b) <u a  --> llvm.uadd.with.overflow.
     // (a+b) <u b  --> llvm.uadd.with.overflow.
     if (I.getPredicate() == ICmpInst::ICMP_ULT &&
-        match(Op0, m_Add(m_Value(A), m_Value(B))) && 
+        match(Op0, m_Add(m_Value(A), m_Value(B))) &&
         (Op1 == A || Op1 == B))
       if (Instruction *R = ProcessUAddIdiom(I, Op0, *this))
         return R;
-                                 
+
     // a >u (a+b)  --> llvm.uadd.with.overflow.
     // b >u (a+b)  --> llvm.uadd.with.overflow.
     if (I.getPredicate() == ICmpInst::ICMP_UGT &&
@@ -2465,7 +2466,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       if (Instruction *R = ProcessUAddIdiom(I, Op1, *this))
         return R;
   }
-  
+
   if (I.isEquality()) {
     Value *A, *B, *C, *D;
 
@@ -2483,10 +2484,10 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
             match(D, m_ConstantInt(C2)) && Op1->hasOneUse()) {
           Constant *NC = ConstantInt::get(I.getContext(),
                                           C1->getValue() ^ C2->getValue());
-          Value *Xor = Builder->CreateXor(C, NC, "tmp");
+          Value *Xor = Builder->CreateXor(C, NC);
           return new ICmpInst(I.getPredicate(), A, Xor);
         }
-        
+
         // A^B == A^D -> B == D
         if (A == C) return new ICmpInst(I.getPredicate(), B, D);
         if (A == D) return new ICmpInst(I.getPredicate(), B, C);
@@ -2494,7 +2495,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
         if (B == D) return new ICmpInst(I.getPredicate(), A, C);
       }
     }
-    
+
     if (match(Op1, m_Xor(m_Value(A), m_Value(B))) &&
         (A == Op0 || B == Op0)) {
       // A == (A^B)  ->  B == 0
@@ -2504,10 +2505,10 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     }
 
     // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
-    if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && 
+    if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) &&
         match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) {
       Value *X = 0, *Y = 0, *Z = 0;
-      
+
       if (A == C) {
         X = B; Y = D; Z = A;
       } else if (A == D) {
@@ -2517,16 +2518,16 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
       } else if (B == D) {
         X = A; Y = C; Z = B;
       }
-      
+
       if (X) {   // Build (X^Y) & Z
-        Op1 = Builder->CreateXor(X, Y, "tmp");
-        Op1 = Builder->CreateAnd(Op1, Z, "tmp");
+        Op1 = Builder->CreateXor(X, Y);
+        Op1 = Builder->CreateAnd(Op1, Z);
         I.setOperand(0, Op1);
         I.setOperand(1, Constant::getNullValue(Op1->getType()));
         return &I;
       }
     }
-    
+
     // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
     // "icmp (and X, mask), cst"
     uint64_t ShAmt = 0;
@@ -2539,21 +2540,21 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
         // when it exposes other optimizations.
         !A->hasOneUse()) {
       unsigned ASize =cast<IntegerType>(A->getType())->getPrimitiveSizeInBits();
-      
+
       if (ShAmt < ASize) {
         APInt MaskV =
           APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits());
         MaskV <<= ShAmt;
-        
+
         APInt CmpV = Cst1->getValue().zext(ASize);
         CmpV <<= ShAmt;
-        
+
         Value *Mask = Builder->CreateAnd(A, Builder->getInt(MaskV));
         return new ICmpInst(I.getPredicate(), Mask, Builder->getInt(CmpV));
       }
     }
   }
-  
+
   {
     Value *X; ConstantInt *Cst;
     // icmp X+Cst, X
@@ -2579,31 +2580,31 @@ Instruction *InstCombiner::FoldFCmp_IntToFP_Cst(FCmpInst &I,
                                                 Constant *RHSC) {
   if (!isa<ConstantFP>(RHSC)) return 0;
   const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF();
-  
+
   // Get the width of the mantissa.  We don't want to hack on conversions that
   // might lose information from the integer, e.g. "i64 -> float"
   int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
   if (MantissaWidth == -1) return 0;  // Unknown.
-  
+
   // Check to see that the input is converted from an integer type that is small
   // enough that preserves all bits.  TODO: check here for "known" sign bits.
   // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
   unsigned InputSize = LHSI->getOperand(0)->getType()->getScalarSizeInBits();
-  
+
   // If this is a uitofp instruction, we need an extra bit to hold the sign.
   bool LHSUnsigned = isa<UIToFPInst>(LHSI);
   if (LHSUnsigned)
     ++InputSize;
-  
+
   // If the conversion would lose info, don't hack on this.
   if ((int)InputSize > MantissaWidth)
     return 0;
-  
+
   // Otherwise, we can potentially simplify the comparison.  We know that it
   // will always come through as an integer value and we know the constant is
   // not a NAN (it would have been previously simplified).
   assert(!RHS.isNaN() && "NaN comparison not already folded!");
-  
+
   ICmpInst::Predicate Pred;
   switch (I.getPredicate()) {
   default: llvm_unreachable("Unexpected predicate!");
@@ -2636,15 +2637,15 @@ Instruction *InstCombiner::FoldFCmp_IntToFP_Cst(FCmpInst &I,
   case FCmpInst::FCMP_UNO:
     return ReplaceInstUsesWith(I, ConstantInt::getFalse(I.getContext()));
   }
-  
-  const IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType());
-  
+
+  IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType());
+
   // Now we know that the APFloat is a normal number, zero or inf.
-  
+
   // See if the FP constant is too large for the integer.  For example,
   // comparing an i8 to 300.0.
   unsigned IntWidth = IntTy->getScalarSizeInBits();
-  
+
   if (!LHSUnsigned) {
     // If the RHS value is > SignedMax, fold the comparison.  This handles +INF
     // and large values.
@@ -2670,7 +2671,7 @@ Instruction *InstCombiner::FoldFCmp_IntToFP_Cst(FCmpInst &I,
       return ReplaceInstUsesWith(I, ConstantInt::getFalse(I.getContext()));
     }
   }
-  
+
   if (!LHSUnsigned) {
     // See if the RHS value is < SignedMin.
     APFloat SMin(RHS.getSemantics(), APFloat::fcZero, false);
@@ -2766,7 +2767,7 @@ Instruction *InstCombiner::FoldFCmp_IntToFP_Cst(FCmpInst &I,
 
 Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
   bool Changed = false;
-  
+
   /// Orders the operands of the compare so that they are listed from most
   /// complex to least complex.  This puts constants before unary operators,
   /// before binary operators.
@@ -2776,7 +2777,7 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
   }
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
-  
+
   if (Value *V = SimplifyFCmpInst(I.getPredicate(), Op0, Op1, TD))
     return ReplaceInstUsesWith(I, V);
 
@@ -2792,7 +2793,7 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
       I.setPredicate(FCmpInst::FCMP_UNO);
       I.setOperand(1, Constant::getNullValue(Op0->getType()));
       return &I;
-      
+
     case FCmpInst::FCMP_ORD:    // True if ordered (no nans)
     case FCmpInst::FCMP_OEQ:    // True if ordered and equal
     case FCmpInst::FCMP_OGE:    // True if ordered and greater than or equal
@@ -2803,7 +2804,7 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
       return &I;
     }
   }
-    
+
   // Handle fcmp with constant RHS
   if (Constant *RHSC = dyn_cast<Constant>(Op1)) {
     if (Instruction *LHSI = dyn_cast<Instruction>(Op0))
@@ -2836,10 +2837,14 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
         APFloat F = RHSF->getValueAPF();
         F.convert(*Sem, APFloat::rmNearestTiesToEven, &Lossy);
 
-        // Avoid lossy conversions and denormals.
+        // Avoid lossy conversions and denormals. Zero is a special case
+        // that's OK to convert.
+        APFloat Fabs = F;
+        Fabs.clearSign();
         if (!Lossy &&
-            F.compare(APFloat::getSmallestNormalized(*Sem)) !=
-                                                           APFloat::cmpLessThan)
+            ((Fabs.compare(APFloat::getSmallestNormalized(*Sem)) !=
+                 APFloat::cmpLessThan) || Fabs.isZero()))
+
           return new FCmpInst(I.getPredicate(), LHSExt->getOperand(0),
                               ConstantFP::get(RHSC->getContext(), F));
         break;
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index f499290fe876..7446a51a4db1 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -26,7 +26,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
   // Ensure that the alloca array size argument has type intptr_t, so that
   // any casting is exposed early.
   if (TD) {
-    const Type *IntPtrTy = TD->getIntPtrType(AI.getContext());
+    Type *IntPtrTy = TD->getIntPtrType(AI.getContext());
     if (AI.getArraySize()->getType() != IntPtrTy) {
       Value *V = Builder->CreateIntCast(AI.getArraySize(),
                                         IntPtrTy, false);
@@ -38,7 +38,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
   // Convert: alloca Ty, C - where C is a constant != 1 into: alloca [C x Ty], 1
   if (AI.isArrayAllocation()) {  // Check C != 1
     if (const ConstantInt *C = dyn_cast<ConstantInt>(AI.getArraySize())) {
-      const Type *NewTy = 
+      Type *NewTy = 
         ArrayType::get(AI.getAllocatedType(), C->getZExtValue());
       assert(isa<AllocaInst>(AI) && "Unknown type of allocation inst!");
       AllocaInst *New = Builder->CreateAlloca(NewTy, 0, AI.getName());
@@ -58,8 +58,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
       Idx[0] = NullIdx;
       Idx[1] = NullIdx;
       Instruction *GEP =
-           GetElementPtrInst::CreateInBounds(New, Idx, Idx + 2,
-                                             New->getName()+".sub");
+           GetElementPtrInst::CreateInBounds(New, Idx, New->getName()+".sub");
       InsertNewInstBefore(GEP, *It);
 
       // Now make everything use the getelementptr instead of the original
@@ -92,28 +91,28 @@ static Instruction *InstCombineLoadCast(InstCombiner &IC, LoadInst &LI,
   User *CI = cast<User>(LI.getOperand(0));
   Value *CastOp = CI->getOperand(0);
 
-  const PointerType *DestTy = cast<PointerType>(CI->getType());
-  const Type *DestPTy = DestTy->getElementType();
-  if (const PointerType *SrcTy = dyn_cast<PointerType>(CastOp->getType())) {
+  PointerType *DestTy = cast<PointerType>(CI->getType());
+  Type *DestPTy = DestTy->getElementType();
+  if (PointerType *SrcTy = dyn_cast<PointerType>(CastOp->getType())) {
 
     // If the address spaces don't match, don't eliminate the cast.
     if (DestTy->getAddressSpace() != SrcTy->getAddressSpace())
       return 0;
 
-    const Type *SrcPTy = SrcTy->getElementType();
+    Type *SrcPTy = SrcTy->getElementType();
 
     if (DestPTy->isIntegerTy() || DestPTy->isPointerTy() || 
          DestPTy->isVectorTy()) {
       // If the source is an array, the code below will not succeed.  Check to
       // see if a trivial 'gep P, 0, 0' will help matters.  Only do this for
       // constants.
-      if (const ArrayType *ASrcTy = dyn_cast<ArrayType>(SrcPTy))
+      if (ArrayType *ASrcTy = dyn_cast<ArrayType>(SrcPTy))
         if (Constant *CSrc = dyn_cast<Constant>(CastOp))
           if (ASrcTy->getNumElements() != 0) {
             Value *Idxs[2];
             Idxs[0] = Constant::getNullValue(Type::getInt32Ty(LI.getContext()));
             Idxs[1] = Idxs[0];
-            CastOp = ConstantExpr::getGetElementPtr(CSrc, Idxs, 2);
+            CastOp = ConstantExpr::getGetElementPtr(CSrc, Idxs);
             SrcTy = cast<PointerType>(CastOp->getType());
             SrcPTy = SrcTy->getElementType();
           }
@@ -133,6 +132,7 @@ static Instruction *InstCombineLoadCast(InstCombiner &IC, LoadInst &LI,
         LoadInst *NewLoad = 
           IC.Builder->CreateLoad(CastOp, LI.isVolatile(), CI->getName());
         NewLoad->setAlignment(LI.getAlignment());
+        NewLoad->setAtomic(LI.getOrdering(), LI.getSynchScope());
         // Now cast the result of the load.
         return new BitCastInst(NewLoad, LI.getType());
       }
@@ -163,8 +163,9 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
     if (Instruction *Res = InstCombineLoadCast(*this, LI, TD))
       return Res;
 
-  // None of the following transforms are legal for volatile loads.
-  if (LI.isVolatile()) return 0;
+  // None of the following transforms are legal for volatile/atomic loads.
+  // FIXME: Some of it is okay for atomic loads; needs refactoring.
+  if (!LI.isSimple()) return 0;
   
   // Do really simple store-to-load forwarding and load CSE, to catch cases
   // where there are several consecutive memory accesses to the same location,
@@ -256,11 +257,11 @@ static Instruction *InstCombineStoreToCast(InstCombiner &IC, StoreInst &SI) {
   User *CI = cast<User>(SI.getOperand(1));
   Value *CastOp = CI->getOperand(0);
 
-  const Type *DestPTy = cast<PointerType>(CI->getType())->getElementType();
-  const PointerType *SrcTy = dyn_cast<PointerType>(CastOp->getType());
+  Type *DestPTy = cast<PointerType>(CI->getType())->getElementType();
+  PointerType *SrcTy = dyn_cast<PointerType>(CastOp->getType());
   if (SrcTy == 0) return 0;
   
-  const Type *SrcPTy = SrcTy->getElementType();
+  Type *SrcPTy = SrcTy->getElementType();
 
   if (!DestPTy->isIntegerTy() && !DestPTy->isPointerTy())
     return 0;
@@ -280,12 +281,12 @@ static Instruction *InstCombineStoreToCast(InstCombiner &IC, StoreInst &SI) {
     NewGEPIndices.push_back(Zero);
     
     while (1) {
-      if (const StructType *STy = dyn_cast<StructType>(SrcPTy)) {
+      if (StructType *STy = dyn_cast<StructType>(SrcPTy)) {
         if (!STy->getNumElements()) /* Struct can be empty {} */
           break;
         NewGEPIndices.push_back(Zero);
         SrcPTy = STy->getElementType(0);
-      } else if (const ArrayType *ATy = dyn_cast<ArrayType>(SrcPTy)) {
+      } else if (ArrayType *ATy = dyn_cast<ArrayType>(SrcPTy)) {
         NewGEPIndices.push_back(Zero);
         SrcPTy = ATy->getElementType();
       } else {
@@ -314,8 +315,8 @@ static Instruction *InstCombineStoreToCast(InstCombiner &IC, StoreInst &SI) {
   Value *NewCast;
   Value *SIOp0 = SI.getOperand(0);
   Instruction::CastOps opcode = Instruction::BitCast;
-  const Type* CastSrcTy = SIOp0->getType();
-  const Type* CastDstTy = SrcPTy;
+  Type* CastSrcTy = SIOp0->getType();
+  Type* CastDstTy = SrcPTy;
   if (CastDstTy->isPointerTy()) {
     if (CastSrcTy->isIntegerTy())
       opcode = Instruction::IntToPtr;
@@ -327,8 +328,7 @@ static Instruction *InstCombineStoreToCast(InstCombiner &IC, StoreInst &SI) {
   // SIOp0 is a pointer to aggregate and this is a store to the first field,
   // emit a GEP to index into its first field.
   if (!NewGEPIndices.empty())
-    CastOp = IC.Builder->CreateInBoundsGEP(CastOp, NewGEPIndices.begin(),
-                                           NewGEPIndices.end());
+    CastOp = IC.Builder->CreateInBoundsGEP(CastOp, NewGEPIndices);
   
   NewCast = IC.Builder->CreateCast(opcode, SIOp0, CastDstTy,
                                    SIOp0->getName()+".c");
@@ -370,21 +370,6 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
   Value *Val = SI.getOperand(0);
   Value *Ptr = SI.getOperand(1);
 
-  // If the RHS is an alloca with a single use, zapify the store, making the
-  // alloca dead.
-  if (!SI.isVolatile()) {
-    if (Ptr->hasOneUse()) {
-      if (isa<AllocaInst>(Ptr)) 
-        return EraseInstFromFunction(SI);
-      if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
-        if (isa<AllocaInst>(GEP->getOperand(0))) {
-          if (GEP->getOperand(0)->hasOneUse())
-            return EraseInstFromFunction(SI);
-        }
-      }
-    }
-  }
-
   // Attempt to improve the alignment.
   if (TD) {
     unsigned KnownAlign =
@@ -400,6 +385,23 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
       SI.setAlignment(EffectiveStoreAlign);
   }
 
+  // Don't hack volatile/atomic stores.
+  // FIXME: Some bits are legal for atomic stores; needs refactoring.
+  if (!SI.isSimple()) return 0;
+
+  // If the RHS is an alloca with a single use, zapify the store, making the
+  // alloca dead.
+  if (Ptr->hasOneUse()) {
+    if (isa<AllocaInst>(Ptr)) 
+      return EraseInstFromFunction(SI);
+    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
+      if (isa<AllocaInst>(GEP->getOperand(0))) {
+        if (GEP->getOperand(0)->hasOneUse())
+          return EraseInstFromFunction(SI);
+      }
+    }
+  }
+
   // Do really simple DSE, to catch cases where there are several consecutive
   // stores to the same location, separated by a few arithmetic operations. This
   // situation often occurs with bitfield accesses.
@@ -417,8 +419,8 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
     
     if (StoreInst *PrevSI = dyn_cast<StoreInst>(BBI)) {
       // Prev store isn't volatile, and stores to the same location?
-      if (!PrevSI->isVolatile() &&equivalentAddressValues(PrevSI->getOperand(1),
-                                                          SI.getOperand(1))) {
+      if (PrevSI->isSimple() && equivalentAddressValues(PrevSI->getOperand(1),
+                                                        SI.getOperand(1))) {
         ++NumDeadStore;
         ++BBI;
         EraseInstFromFunction(*PrevSI);
@@ -432,7 +434,7 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
     // then *this* store is dead (X = load P; store X -> P).
     if (LoadInst *LI = dyn_cast<LoadInst>(BBI)) {
       if (LI == Val && equivalentAddressValues(LI->getOperand(0), Ptr) &&
-          !SI.isVolatile())
+          LI->isSimple())
         return EraseInstFromFunction(SI);
       
       // Otherwise, this is a load from some other location.  Stores before it
@@ -444,9 +446,6 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
     if (BBI->mayWriteToMemory() || BBI->mayReadFromMemory())
       break;
   }
-  
-  
-  if (SI.isVolatile()) return 0;  // Don't hack volatile stores.
 
   // store X, null    -> turns into 'unreachable' in SimplifyCFG
   if (isa<ConstantPointerNull>(Ptr) && SI.getPointerAddressSpace() == 0) {
@@ -549,11 +548,11 @@ bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) {
         return false;
       --BBI;
     }
-    // If this isn't a store, isn't a store to the same location, or if the
-    // alignments differ, bail out.
+    // If this isn't a store, isn't a store to the same location, or is not the
+    // right kind of store, bail out.
     OtherStore = dyn_cast<StoreInst>(BBI);
     if (!OtherStore || OtherStore->getOperand(1) != SI.getOperand(1) ||
-        OtherStore->getAlignment() != SI.getAlignment())
+        !SI.isSameOperationAs(OtherStore))
       return false;
   } else {
     // Otherwise, the other block ended with a conditional branch. If one of the
@@ -569,7 +568,7 @@ bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) {
       // Check to see if we find the matching store.
       if ((OtherStore = dyn_cast<StoreInst>(BBI))) {
         if (OtherStore->getOperand(1) != SI.getOperand(1) ||
-            OtherStore->getAlignment() != SI.getAlignment())
+            !SI.isSameOperationAs(OtherStore))
           return false;
         break;
       }
@@ -601,10 +600,12 @@ bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) {
   
   // Advance to a place where it is safe to insert the new store and
   // insert it.
-  BBI = DestBB->getFirstNonPHI();
+  BBI = DestBB->getFirstInsertionPt();
   StoreInst *NewSI = new StoreInst(MergedVal, SI.getOperand(1),
-                                   OtherStore->isVolatile(),
-                                   SI.getAlignment());
+                                   SI.isVolatile(),
+                                   SI.getAlignment(),
+                                   SI.getOrdering(),
+                                   SI.getSynchScope());
   InsertNewInstBefore(NewSI, *BBI);
   NewSI->setDebugLoc(OtherStore->getDebugLoc()); 
 
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 630a6fee3990..7f48125a97ab 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -38,7 +38,7 @@ static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC) {
                       m_Value(B))) &&
       // The "1" can be any value known to be a power of 2.
       isPowerOfTwo(PowerOf2, IC.getTargetData())) {
-    A = IC.Builder->CreateSub(A, B, "tmp");
+    A = IC.Builder->CreateSub(A, B);
     return IC.Builder->CreateShl(PowerOf2, A);
   }
   
@@ -131,7 +131,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
     { Value *X; ConstantInt *C1;
       if (Op0->hasOneUse() &&
           match(Op0, m_Add(m_Value(X), m_ConstantInt(C1)))) {
-        Value *Add = Builder->CreateMul(X, CI, "tmp");
+        Value *Add = Builder->CreateMul(X, CI);
         return BinaryOperator::CreateAdd(Add, Builder->CreateMul(C1, CI));
       }
     }
@@ -244,7 +244,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
 
     if (BoolCast) {
       Value *V = Builder->CreateSub(Constant::getNullValue(I.getType()),
-                                    BoolCast, "tmp");
+                                    BoolCast);
       return BinaryOperator::CreateAnd(V, OtherOp);
     }
   }
@@ -421,7 +421,7 @@ Instruction *InstCombiner::commonIDivTransforms(BinaryOperator &I) {
 
 /// dyn_castZExtVal - Checks if V is a zext or constant that can
 /// be truncated to Ty without losing bits.
-static Value *dyn_castZExtVal(Value *V, const Type *Ty) {
+static Value *dyn_castZExtVal(Value *V, Type *Ty) {
   if (ZExtInst *Z = dyn_cast<ZExtInst>(V)) {
     if (Z->getSrcTy() == Ty)
       return Z->getOperand(0);
@@ -466,8 +466,7 @@ Instruction *InstCombiner::visitUDiv(BinaryOperator &I) {
   { const APInt *CI; Value *N;
     if (match(Op1, m_Shl(m_Power2(CI), m_Value(N)))) {
       if (*CI != 1)
-        N = Builder->CreateAdd(N, ConstantInt::get(I.getType(), CI->logBase2()),
-                               "tmp");
+        N = Builder->CreateAdd(N, ConstantInt::get(I.getType(),CI->logBase2()));
       if (I.isExact())
         return BinaryOperator::CreateExactLShr(Op0, N);
       return BinaryOperator::CreateLShr(Op0, N);
@@ -630,7 +629,7 @@ Instruction *InstCombiner::visitURem(BinaryOperator &I) {
   // Turn A % (C << N), where C is 2^k, into A & ((C << N)-1)  
   if (match(Op1, m_Shl(m_Power2(), m_Value()))) {
     Constant *N1 = Constant::getAllOnesValue(I.getType());
-    Value *Add = Builder->CreateAdd(Op1, N1, "tmp");
+    Value *Add = Builder->CreateAdd(Op1, N1);
     return BinaryOperator::CreateAnd(Op0, Add);
   }
 
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 37773403490c..664546c16551 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -28,8 +28,8 @@ Instruction *InstCombiner::FoldPHIArgBinOpIntoPHI(PHINode &PN) {
   Value *LHSVal = FirstInst->getOperand(0);
   Value *RHSVal = FirstInst->getOperand(1);
     
-  const Type *LHSType = LHSVal->getType();
-  const Type *RHSType = RHSVal->getType();
+  Type *LHSType = LHSVal->getType();
+  Type *RHSType = RHSVal->getType();
   
   bool isNUW = false, isNSW = false, isExact = false;
   if (OverflowingBinaryOperator *BO =
@@ -229,8 +229,7 @@ Instruction *InstCombiner::FoldPHIArgGEPIntoPHI(PHINode &PN) {
   
   Value *Base = FixedOperands[0];
   GetElementPtrInst *NewGEP = 
-    GetElementPtrInst::Create(Base, FixedOperands.begin()+1,
-                              FixedOperands.end());
+    GetElementPtrInst::Create(Base, makeArrayRef(FixedOperands).slice(1));
   if (AllInBounds) NewGEP->setIsInBounds();
   NewGEP->setDebugLoc(FirstInst->getDebugLoc());
   return NewGEP;
@@ -287,7 +286,12 @@ static bool isSafeAndProfitableToSinkLoad(LoadInst *L) {
 
 Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
   LoadInst *FirstLI = cast<LoadInst>(PN.getIncomingValue(0));
-  
+
+  // FIXME: This is overconservative; this transform is allowed in some cases
+  // for atomic operations.
+  if (FirstLI->isAtomic())
+    return 0;
+
   // When processing loads, we need to propagate two bits of information to the
   // sunk load: whether it is volatile, and what its alignment is.  We currently
   // don't sink loads when some have their alignment specified and some don't.
@@ -397,7 +401,7 @@ Instruction *InstCombiner::FoldPHIArgOpIntoPHI(PHINode &PN) {
   // the same type or "+42") we can pull the operation through the PHI, reducing
   // code size and simplifying code.
   Constant *ConstantOp = 0;
-  const Type *CastSrcTy = 0;
+  Type *CastSrcTy = 0;
   bool isNUW = false, isNSW = false, isExact = false;
   
   if (isa<CastInst>(FirstInst)) {
@@ -572,7 +576,7 @@ struct LoweredPHIRecord {
   unsigned Shift;     // The amount shifted.
   unsigned Width;     // The width extracted.
   
-  LoweredPHIRecord(PHINode *pn, unsigned Sh, const Type *Ty)
+  LoweredPHIRecord(PHINode *pn, unsigned Sh, Type *Ty)
     : PN(pn), Shift(Sh), Width(Ty->getPrimitiveSizeInBits()) {}
   
   // Ctor form used by DenseMap.
@@ -701,7 +705,7 @@ Instruction *InstCombiner::SliceUpIllegalIntegerPHI(PHINode &FirstPhi) {
     unsigned PHIId = PHIUsers[UserI].PHIId;
     PHINode *PN = PHIsToSlice[PHIId];
     unsigned Offset = PHIUsers[UserI].Shift;
-    const Type *Ty = PHIUsers[UserI].Inst->getType();
+    Type *Ty = PHIUsers[UserI].Inst->getType();
     
     PHINode *EltPHI;
     
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 5733c20828c6..91e60a4fb244 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -13,6 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/Support/PatternMatch.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 using namespace llvm;
 using namespace PatternMatch;
@@ -323,9 +324,14 @@ static Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
     }
 
     // All operands were constants, fold it.
-    if (ConstOps.size() == I->getNumOperands())
+    if (ConstOps.size() == I->getNumOperands()) {
+      if (LoadInst *LI = dyn_cast<LoadInst>(I))
+        if (!LI->isVolatile())
+          return ConstantFoldLoadFromConstPtr(ConstOps[0], TD);
+
       return ConstantFoldInstOperands(I->getOpcode(), I->getType(),
-                                      ConstOps.data(), ConstOps.size(), TD);
+                                      ConstOps, TD);
+    }
   }
 
   return 0;
@@ -363,7 +369,7 @@ Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
       case ICmpInst::ICMP_UGT:
       case ICmpInst::ICMP_SGT: {
         // These transformations only work for selects over integers.
-        const IntegerType *SelectTy = dyn_cast<IntegerType>(SI.getType());
+        IntegerType *SelectTy = dyn_cast<IntegerType>(SI.getType());
         if (!SelectTy)
           break;
 
@@ -443,7 +449,7 @@ Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
   // FIXME: Type and constness constraints could be lifted, but we have to
   //        watch code size carefully. We should consider xor instead of
   //        sub/add when we decide to do that.
-  if (const IntegerType *Ty = dyn_cast<IntegerType>(CmpLHS->getType())) {
+  if (IntegerType *Ty = dyn_cast<IntegerType>(CmpLHS->getType())) {
     if (TrueVal->getType() == Ty) {
       if (ConstantInt *Cmp = dyn_cast<ConstantInt>(CmpRHS)) {
         ConstantInt *C1 = NULL, *C2 = NULL;
@@ -476,10 +482,16 @@ Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
     if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, TD) == TrueVal ||
         SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, TD) == TrueVal)
       return ReplaceInstUsesWith(SI, FalseVal);
+    if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, TD) == FalseVal ||
+        SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, TD) == FalseVal)
+      return ReplaceInstUsesWith(SI, FalseVal);
   } else if (Pred == ICmpInst::ICMP_NE) {
     if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, TD) == FalseVal ||
         SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, TD) == FalseVal)
       return ReplaceInstUsesWith(SI, TrueVal);
+    if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, TD) == TrueVal ||
+        SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, TD) == TrueVal)
+      return ReplaceInstUsesWith(SI, TrueVal);
   }
 
   // NOTE: if we wanted to, this is where to detect integer MIN/MAX
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 811f94976f68..6d85adde9b85 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -13,6 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/IntrinsicInst.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Support/PatternMatch.h"
 using namespace llvm;
@@ -207,11 +208,12 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
     return I;
     
   case Instruction::Shl: {
-    unsigned TypeWidth = I->getType()->getScalarSizeInBits();
+    BinaryOperator *BO = cast<BinaryOperator>(I);
+    unsigned TypeWidth = BO->getType()->getScalarSizeInBits();
 
     // We only accept shifts-by-a-constant in CanEvaluateShifted.
-    ConstantInt *CI = cast<ConstantInt>(I->getOperand(1));
-    
+    ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
+
     // We can always fold shl(c1)+shl(c2) -> shl(c1+c2).
     if (isLeftShift) {
       // If this is oversized composite shift, then unsigned shifts get 0.
@@ -219,7 +221,9 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
       if (NewShAmt >= TypeWidth)
         return Constant::getNullValue(I->getType());
 
-      I->setOperand(1, ConstantInt::get(I->getType(), NewShAmt));
+      BO->setOperand(1, ConstantInt::get(BO->getType(), NewShAmt));
+      BO->setHasNoUnsignedWrap(false);
+      BO->setHasNoSignedWrap(false);
       return I;
     }
     
@@ -227,11 +231,11 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
     // zeros.
     if (CI->getValue() == NumBits) {
       APInt Mask(APInt::getLowBitsSet(TypeWidth, TypeWidth - NumBits));
-      V = IC.Builder->CreateAnd(I->getOperand(0),
-                                ConstantInt::get(I->getContext(), Mask));
+      V = IC.Builder->CreateAnd(BO->getOperand(0),
+                                ConstantInt::get(BO->getContext(), Mask));
       if (Instruction *VI = dyn_cast<Instruction>(V)) {
-        VI->moveBefore(I);
-        VI->takeName(I);
+        VI->moveBefore(BO);
+        VI->takeName(BO);
       }
       return V;
     }
@@ -239,23 +243,27 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
     // We turn shl(c1)+shr(c2) -> shl(c3)+and(c4), but only when we know that
     // the and won't be needed.
     assert(CI->getZExtValue() > NumBits);
-    I->setOperand(1, ConstantInt::get(I->getType(),
-                                      CI->getZExtValue() - NumBits));
-    return I;
+    BO->setOperand(1, ConstantInt::get(BO->getType(),
+                                       CI->getZExtValue() - NumBits));
+    BO->setHasNoUnsignedWrap(false);
+    BO->setHasNoSignedWrap(false);
+    return BO;
   }
   case Instruction::LShr: {
-    unsigned TypeWidth = I->getType()->getScalarSizeInBits();
+    BinaryOperator *BO = cast<BinaryOperator>(I);
+    unsigned TypeWidth = BO->getType()->getScalarSizeInBits();
     // We only accept shifts-by-a-constant in CanEvaluateShifted.
-    ConstantInt *CI = cast<ConstantInt>(I->getOperand(1));
+    ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
     
     // We can always fold lshr(c1)+lshr(c2) -> lshr(c1+c2).
     if (!isLeftShift) {
       // If this is oversized composite shift, then unsigned shifts get 0.
       unsigned NewShAmt = NumBits+CI->getZExtValue();
       if (NewShAmt >= TypeWidth)
-        return Constant::getNullValue(I->getType());
+        return Constant::getNullValue(BO->getType());
       
-      I->setOperand(1, ConstantInt::get(I->getType(), NewShAmt));
+      BO->setOperand(1, ConstantInt::get(BO->getType(), NewShAmt));
+      BO->setIsExact(false);
       return I;
     }
     
@@ -264,7 +272,7 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
     if (CI->getValue() == NumBits) {
       APInt Mask(APInt::getHighBitsSet(TypeWidth, TypeWidth - NumBits));
       V = IC.Builder->CreateAnd(I->getOperand(0),
-                                ConstantInt::get(I->getContext(), Mask));
+                                ConstantInt::get(BO->getContext(), Mask));
       if (Instruction *VI = dyn_cast<Instruction>(V)) {
         VI->moveBefore(I);
         VI->takeName(I);
@@ -275,9 +283,10 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
     // We turn lshr(c1)+shl(c2) -> lshr(c3)+and(c4), but only when we know that
     // the and won't be needed.
     assert(CI->getZExtValue() > NumBits);
-    I->setOperand(1, ConstantInt::get(I->getType(),
-                                      CI->getZExtValue() - NumBits));
-    return I;
+    BO->setOperand(1, ConstantInt::get(BO->getType(),
+                                       CI->getZExtValue() - NumBits));
+    BO->setIsExact(false);
+    return BO;
   }
     
   case Instruction::Select:
@@ -528,7 +537,7 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, ConstantInt *Op1,
     
     uint32_t AmtSum = ShiftAmt1+ShiftAmt2;   // Fold into one big shift.
     
-    const IntegerType *Ty = cast<IntegerType>(I.getType());
+    IntegerType *Ty = cast<IntegerType>(I.getType());
     
     // Check for (X << c1) << c2  and  (X >> c1) >> c2
     if (I.getOpcode() == ShiftOp->getOpcode()) {
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 8fea8eb7efb5..5cd9a4b7954c 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -103,7 +103,7 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
   assert(V != 0 && "Null pointer of Value???");
   assert(Depth <= 6 && "Limit Search Depth");
   uint32_t BitWidth = DemandedMask.getBitWidth();
-  const Type *VTy = V->getType();
+  Type *VTy = V->getType();
   assert((TD || !VTy->isPointerTy()) &&
          "SimplifyDemandedBits needs to know bit widths!");
   assert((!TD || TD->getTypeSizeInBits(VTy->getScalarType()) == BitWidth) &&
@@ -325,8 +325,7 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
       if ((RHSKnownOne & LHSKnownOne) == RHSKnownOne) {
         Constant *AndC = Constant::getIntegerValue(VTy,
                                                    ~RHSKnownOne & DemandedMask);
-        Instruction *And = 
-          BinaryOperator::CreateAnd(I->getOperand(0), AndC, "tmp");
+        Instruction *And = BinaryOperator::CreateAnd(I->getOperand(0), AndC);
         return InsertNewInstWith(And, *I);
       }
     }
@@ -351,14 +350,12 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         
         Constant *AndC =
           ConstantInt::get(I->getType(), NewMask & AndRHS->getValue());
-        Instruction *NewAnd = 
-          BinaryOperator::CreateAnd(I->getOperand(0), AndC, "tmp");
+        Instruction *NewAnd = BinaryOperator::CreateAnd(I->getOperand(0), AndC);
         InsertNewInstWith(NewAnd, *I);
         
         Constant *XorC =
           ConstantInt::get(I->getType(), NewMask & XorRHS->getValue());
-        Instruction *NewXor =
-          BinaryOperator::CreateXor(NewAnd, XorC, "tmp");
+        Instruction *NewXor = BinaryOperator::CreateXor(NewAnd, XorC);
         return InsertNewInstWith(NewXor, *I);
       }
 
@@ -404,8 +401,8 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     if (!I->getOperand(0)->getType()->isIntOrIntVectorTy())
       return 0;  // vector->int or fp->int?
 
-    if (const VectorType *DstVTy = dyn_cast<VectorType>(I->getType())) {
-      if (const VectorType *SrcVTy =
+    if (VectorType *DstVTy = dyn_cast<VectorType>(I->getType())) {
+      if (VectorType *SrcVTy =
             dyn_cast<VectorType>(I->getOperand(0)->getType())) {
         if (DstVTy->getNumElements() != SrcVTy->getNumElements())
           // Don't touch a bitcast between vectors of different element counts.
@@ -826,7 +823,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
 
   UndefElts = 0;
   if (ConstantVector *CV = dyn_cast<ConstantVector>(V)) {
-    const Type *EltTy = cast<VectorType>(V->getType())->getElementType();
+    Type *EltTy = cast<VectorType>(V->getType())->getElementType();
     Constant *Undef = UndefValue::get(EltTy);
 
     std::vector<Constant*> Elts;
@@ -855,7 +852,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
     if (DemandedElts.isAllOnesValue())
       return 0;
     
-    const Type *EltTy = cast<VectorType>(V->getType())->getElementType();
+    Type *EltTy = cast<VectorType>(V->getType())->getElementType();
     Constant *Zero = Constant::getNullValue(EltTy);
     Constant *Undef = UndefValue::get(EltTy);
     std::vector<Constant*> Elts;
@@ -962,6 +959,9 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
       unsigned MaskVal = Shuffle->getMaskValue(i);
       if (MaskVal == -1u) {
         UndefElts.setBit(i);
+      } else if (!DemandedElts[i]) {
+        NewUndefElts = true;
+        UndefElts.setBit(i);
       } else if (MaskVal < LHSVWidth) {
         if (UndefElts4[MaskVal]) {
           NewUndefElts = true;
@@ -992,7 +992,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
   }
   case Instruction::BitCast: {
     // Vector->vector casts only.
-    const VectorType *VTy = dyn_cast<VectorType>(I->getOperand(0)->getType());
+    VectorType *VTy = dyn_cast<VectorType>(I->getOperand(0)->getType());
     if (!VTy) break;
     unsigned InVWidth = VTy->getNumElements();
     APInt InputDemandedElts(InVWidth, 0);
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index ad6a8d054ee7..154267c03465 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -77,7 +77,7 @@ static std::vector<int> getShuffleMask(const ShuffleVectorInst *SVI) {
 /// extracted from the vector.
 static Value *FindScalarElement(Value *V, unsigned EltNo) {
   assert(V->getType()->isVectorTy() && "Not looking at a vector?");
-  const VectorType *PTy = cast<VectorType>(V->getType());
+  VectorType *PTy = cast<VectorType>(V->getType());
   unsigned Width = PTy->getNumElements();
   if (EltNo >= Width)  // Out of range access.
     return UndefValue::get(PTy->getElementType());
@@ -175,7 +175,7 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
     // the same number of elements, see if we can find the source element from
     // it.  In this case, we will end up needing to bitcast the scalars.
     if (BitCastInst *BCI = dyn_cast<BitCastInst>(EI.getOperand(0))) {
-      if (const VectorType *VT =
+      if (VectorType *VT =
           dyn_cast<VectorType>(BCI->getOperand(0)->getType()))
         if (VT->getNumElements() == VectorWidth)
           if (Value *Elt = FindScalarElement(BCI->getOperand(0), IndexVal))
@@ -225,7 +225,7 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
           SrcIdx -= LHSWidth;
           Src = SVI->getOperand(1);
         }
-        const Type *Int32Ty = Type::getInt32Ty(EI.getContext());
+        Type *Int32Ty = Type::getInt32Ty(EI.getContext());
         return ExtractElementInst::Create(Src,
                                           ConstantInt::get(Int32Ty,
                                                            SrcIdx, false));
@@ -555,7 +555,7 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
         // shuffle mask, do the replacement.
         if (isSplat || NewMask == LHSMask || NewMask == Mask) {
           std::vector<Constant*> Elts;
-          const Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
+          Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
           for (unsigned i = 0, e = NewMask.size(); i != e; ++i) {
             if (NewMask[i] < 0) {
               Elts.push_back(UndefValue::get(Int32Ty));
diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index ab98ef9fccf8..92874b95234d 100644
--- a/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -46,8 +46,10 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/PatternMatch.h"
+#include "llvm/Support/ValueHandle.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm-c/Initialization.h"
 #include <algorithm>
 #include <climits>
@@ -83,7 +85,7 @@ void InstCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
 /// ShouldChangeType - Return true if it is desirable to convert a computation
 /// from 'From' to 'To'.  We don't want to convert from a legal to an illegal
 /// type for example, or from a smaller to a larger illegal type.
-bool InstCombiner::ShouldChangeType(const Type *From, const Type *To) const {
+bool InstCombiner::ShouldChangeType(Type *From, Type *To) const {
   assert(From->isIntegerTy() && To->isIntegerTy());
   
   // If we don't have TD, we don't know if the source/dest are legal.
@@ -107,6 +109,43 @@ bool InstCombiner::ShouldChangeType(const Type *From, const Type *To) const {
   return true;
 }
 
+// Return true, if No Signed Wrap should be maintained for I.
+// The No Signed Wrap flag can be kept if the operation "B (I.getOpcode) C",
+// where both B and C should be ConstantInts, results in a constant that does
+// not overflow. This function only handles the Add and Sub opcodes. For
+// all other opcodes, the function conservatively returns false.
+static bool MaintainNoSignedWrap(BinaryOperator &I, Value *B, Value *C) {
+  OverflowingBinaryOperator *OBO = dyn_cast<OverflowingBinaryOperator>(&I);
+  if (!OBO || !OBO->hasNoSignedWrap()) {
+    return false;
+  }
+
+  // We reason about Add and Sub Only.
+  Instruction::BinaryOps Opcode = I.getOpcode();
+  if (Opcode != Instruction::Add && 
+      Opcode != Instruction::Sub) {
+    return false;
+  }
+
+  ConstantInt *CB = dyn_cast<ConstantInt>(B);
+  ConstantInt *CC = dyn_cast<ConstantInt>(C);
+
+  if (!CB || !CC) {
+    return false;
+  }
+
+  const APInt &BVal = CB->getValue();
+  const APInt &CVal = CC->getValue();
+  bool Overflow = false;
+
+  if (Opcode == Instruction::Add) {
+    BVal.sadd_ov(CVal, Overflow);
+  } else {
+    BVal.ssub_ov(CVal, Overflow);
+  }
+
+  return !Overflow;
+}
 
 /// SimplifyAssociativeOrCommutative - This performs a few simplifications for
 /// operators which are associative or commutative:
@@ -158,7 +197,16 @@ bool InstCombiner::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
           I.setOperand(1, V);
           // Conservatively clear the optional flags, since they may not be
           // preserved by the reassociation.
-          I.clearSubclassOptionalData();
+          if (MaintainNoSignedWrap(I, B, C) &&
+	      (!Op0 || (isa<BinaryOperator>(Op0) && Op0->hasNoSignedWrap()))) {
+            // Note: this is only valid because SimplifyBinOp doesn't look at
+            // the operands to Op0.
+            I.clearSubclassOptionalData();
+            I.setHasNoSignedWrap(true);
+          } else {
+            I.clearSubclassOptionalData();
+          }
+            
           Changed = true;
           ++NumReassoc;
           continue;
@@ -240,7 +288,7 @@ bool InstCombiner::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
         Constant *C2 = cast<Constant>(Op1->getOperand(1));
 
         Constant *Folded = ConstantExpr::get(Opcode, C1, C2);
-        Instruction *New = BinaryOperator::Create(Opcode, A, B);
+        BinaryOperator *New = BinaryOperator::Create(Opcode, A, B);
         InsertNewInstWith(New, I);
         New->takeName(Op1);
         I.setOperand(0, New);
@@ -248,6 +296,7 @@ bool InstCombiner::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
         // Conservatively clear the optional flags, since they may not be
         // preserved by the reassociation.
         I.clearSubclassOptionalData();
+
         Changed = true;
         continue;
       }
@@ -516,8 +565,8 @@ Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) {
     // If it's a bitcast involving vectors, make sure it has the same number of
     // elements on both sides.
     if (BitCastInst *BC = dyn_cast<BitCastInst>(&Op)) {
-      const VectorType *DestTy = dyn_cast<VectorType>(BC->getDestTy());
-      const VectorType *SrcTy = dyn_cast<VectorType>(BC->getSrcTy());
+      VectorType *DestTy = dyn_cast<VectorType>(BC->getDestTy());
+      VectorType *SrcTy = dyn_cast<VectorType>(BC->getSrcTy());
 
       // Verify that either both or neither are vectors.
       if ((SrcTy == NULL) != (DestTy == NULL)) return 0;
@@ -654,7 +703,7 @@ Instruction *InstCombiner::FoldOpIntoPhi(Instruction &I) {
     }
   } else { 
     CastInst *CI = cast<CastInst>(&I);
-    const Type *RetTy = CI->getType();
+    Type *RetTy = CI->getType();
     for (unsigned i = 0; i != NumPHIValues; ++i) {
       Value *InV;
       if (Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i)))
@@ -680,7 +729,7 @@ Instruction *InstCombiner::FoldOpIntoPhi(Instruction &I) {
 /// or not there is a sequence of GEP indices into the type that will land us at
 /// the specified offset.  If so, fill them into NewIndices and return the
 /// resultant element type, otherwise return null.
-const Type *InstCombiner::FindElementAtOffset(const Type *Ty, int64_t Offset, 
+Type *InstCombiner::FindElementAtOffset(Type *Ty, int64_t Offset, 
                                           SmallVectorImpl<Value*> &NewIndices) {
   if (!TD) return 0;
   if (!Ty->isSized()) return 0;
@@ -688,7 +737,7 @@ const Type *InstCombiner::FindElementAtOffset(const Type *Ty, int64_t Offset,
   // Start with the index over the outer type.  Note that the type size
   // might be zero (even if the offset isn't zero) if the indexed type
   // is something like [0 x {int, int}]
-  const Type *IntPtrTy = TD->getIntPtrType(Ty->getContext());
+  Type *IntPtrTy = TD->getIntPtrType(Ty->getContext());
   int64_t FirstIdx = 0;
   if (int64_t TySize = TD->getTypeAllocSize(Ty)) {
     FirstIdx = Offset/TySize;
@@ -711,7 +760,7 @@ const Type *InstCombiner::FindElementAtOffset(const Type *Ty, int64_t Offset,
     if (uint64_t(Offset*8) >= TD->getTypeSizeInBits(Ty))
       return 0;
     
-    if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+    if (StructType *STy = dyn_cast<StructType>(Ty)) {
       const StructLayout *SL = TD->getStructLayout(STy);
       assert(Offset < (int64_t)SL->getSizeInBytes() &&
              "Offset must stay within the indexed type");
@@ -722,7 +771,7 @@ const Type *InstCombiner::FindElementAtOffset(const Type *Ty, int64_t Offset,
       
       Offset -= SL->getElementOffset(Elt);
       Ty = STy->getElementType(Elt);
-    } else if (const ArrayType *AT = dyn_cast<ArrayType>(Ty)) {
+    } else if (ArrayType *AT = dyn_cast<ArrayType>(Ty)) {
       uint64_t EltSize = TD->getTypeAllocSize(AT->getElementType());
       assert(EltSize && "Cannot index into a zero-sized array");
       NewIndices.push_back(ConstantInt::get(IntPtrTy,Offset/EltSize));
@@ -737,12 +786,20 @@ const Type *InstCombiner::FindElementAtOffset(const Type *Ty, int64_t Offset,
   return Ty;
 }
 
-
+static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src) {
+  // If this GEP has only 0 indices, it is the same pointer as
+  // Src. If Src is not a trivial GEP too, don't combine
+  // the indices.
+  if (GEP.hasAllZeroIndices() && !Src.hasAllZeroIndices() &&
+      !Src.hasOneUse())
+    return false;
+  return true;
+}
 
 Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   SmallVector<Value*, 8> Ops(GEP.op_begin(), GEP.op_end());
 
-  if (Value *V = SimplifyGEPInst(&Ops[0], Ops.size(), TD))
+  if (Value *V = SimplifyGEPInst(Ops, TD))
     return ReplaceInstUsesWith(GEP, V);
 
   Value *PtrOp = GEP.getOperand(0);
@@ -751,13 +808,13 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   // by multiples of a zero size type with zero.
   if (TD) {
     bool MadeChange = false;
-    const Type *IntPtrTy = TD->getIntPtrType(GEP.getContext());
+    Type *IntPtrTy = TD->getIntPtrType(GEP.getContext());
 
     gep_type_iterator GTI = gep_type_begin(GEP);
     for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end();
          I != E; ++I, ++GTI) {
       // Skip indices into struct types.
-      const SequentialType *SeqTy = dyn_cast<SequentialType>(*GTI);
+      SequentialType *SeqTy = dyn_cast<SequentialType>(*GTI);
       if (!SeqTy) continue;
 
       // If the element type has zero size then any index over it is equivalent
@@ -785,21 +842,15 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   // getelementptr instructions into a single instruction.
   //
   if (GEPOperator *Src = dyn_cast<GEPOperator>(PtrOp)) {
-
-    // If this GEP has only 0 indices, it is the same pointer as
-    // Src. If Src is not a trivial GEP too, don't combine
-    // the indices.
-    if (GEP.hasAllZeroIndices() && !Src->hasAllZeroIndices() &&
-        !Src->hasOneUse())
+    if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
       return 0;
 
     // Note that if our source is a gep chain itself that we wait for that
     // chain to be resolved before we perform this transformation.  This
     // avoids us creating a TON of code in some cases.
-    //
-    if (GetElementPtrInst *SrcGEP =
-          dyn_cast<GetElementPtrInst>(Src->getOperand(0)))
-      if (SrcGEP->getNumOperands() == 2)
+    if (GEPOperator *SrcGEP =
+          dyn_cast<GEPOperator>(Src->getOperand(0)))
+      if (SrcGEP->getNumOperands() == 2 && shouldMergeGEPs(*Src, *SrcGEP))
         return 0;   // Wait until our source is folded to completion.
 
     SmallVector<Value*, 8> Indices;
@@ -851,15 +902,14 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 
     if (!Indices.empty())
       return (GEP.isInBounds() && Src->isInBounds()) ?
-        GetElementPtrInst::CreateInBounds(Src->getOperand(0), Indices.begin(),
-                                          Indices.end(), GEP.getName()) :
-        GetElementPtrInst::Create(Src->getOperand(0), Indices.begin(),
-                                  Indices.end(), GEP.getName());
+        GetElementPtrInst::CreateInBounds(Src->getOperand(0), Indices,
+                                          GEP.getName()) :
+        GetElementPtrInst::Create(Src->getOperand(0), Indices, GEP.getName());
   }
 
   // Handle gep(bitcast x) and gep(gep x, 0, 0, 0).
   Value *StrippedPtr = PtrOp->stripPointerCasts();
-  const PointerType *StrippedPtrTy =cast<PointerType>(StrippedPtr->getType());
+  PointerType *StrippedPtrTy =cast<PointerType>(StrippedPtr->getType());
   if (StrippedPtr != PtrOp &&
     StrippedPtrTy->getAddressSpace() == GEP.getPointerAddressSpace()) {
 
@@ -875,21 +925,20 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     //
     // This occurs when the program declares an array extern like "int X[];"
     if (HasZeroPointerIndex) {
-      const PointerType *CPTy = cast<PointerType>(PtrOp->getType());
-      if (const ArrayType *CATy =
+      PointerType *CPTy = cast<PointerType>(PtrOp->getType());
+      if (ArrayType *CATy =
           dyn_cast<ArrayType>(CPTy->getElementType())) {
         // GEP (bitcast i8* X to [0 x i8]*), i32 0, ... ?
         if (CATy->getElementType() == StrippedPtrTy->getElementType()) {
           // -> GEP i8* X, ...
           SmallVector<Value*, 8> Idx(GEP.idx_begin()+1, GEP.idx_end());
           GetElementPtrInst *Res =
-            GetElementPtrInst::Create(StrippedPtr, Idx.begin(),
-                                      Idx.end(), GEP.getName());
+            GetElementPtrInst::Create(StrippedPtr, Idx, GEP.getName());
           Res->setIsInBounds(GEP.isInBounds());
           return Res;
         }
         
-        if (const ArrayType *XATy =
+        if (ArrayType *XATy =
               dyn_cast<ArrayType>(StrippedPtrTy->getElementType())){
           // GEP (bitcast [10 x i8]* X to [0 x i8]*), i32 0, ... ?
           if (CATy->getElementType() == XATy->getElementType()) {
@@ -907,8 +956,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // Transform things like:
       // %t = getelementptr i32* bitcast ([2 x i32]* %str to i32*), i32 %V
       // into:  %t1 = getelementptr [2 x i32]* %str, i32 0, i32 %V; bitcast
-      const Type *SrcElTy = StrippedPtrTy->getElementType();
-      const Type *ResElTy=cast<PointerType>(PtrOp->getType())->getElementType();
+      Type *SrcElTy = StrippedPtrTy->getElementType();
+      Type *ResElTy=cast<PointerType>(PtrOp->getType())->getElementType();
       if (TD && SrcElTy->isArrayTy() &&
           TD->getTypeAllocSize(cast<ArrayType>(SrcElTy)->getElementType()) ==
           TD->getTypeAllocSize(ResElTy)) {
@@ -916,8 +965,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
         Idx[0] = Constant::getNullValue(Type::getInt32Ty(GEP.getContext()));
         Idx[1] = GEP.getOperand(1);
         Value *NewGEP = GEP.isInBounds() ?
-          Builder->CreateInBoundsGEP(StrippedPtr, Idx, Idx + 2, GEP.getName()) :
-          Builder->CreateGEP(StrippedPtr, Idx, Idx + 2, GEP.getName());
+          Builder->CreateInBoundsGEP(StrippedPtr, Idx, GEP.getName()) :
+          Builder->CreateGEP(StrippedPtr, Idx, GEP.getName());
         // V and GEP are both pointer types --> BitCast
         return new BitCastInst(NewGEP, GEP.getType());
       }
@@ -975,8 +1024,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
           Idx[0] = Constant::getNullValue(Type::getInt32Ty(GEP.getContext()));
           Idx[1] = NewIdx;
           Value *NewGEP = GEP.isInBounds() ?
-            Builder->CreateInBoundsGEP(StrippedPtr, Idx, Idx + 2,GEP.getName()):
-            Builder->CreateGEP(StrippedPtr, Idx, Idx + 2, GEP.getName());
+            Builder->CreateInBoundsGEP(StrippedPtr, Idx, GEP.getName()):
+            Builder->CreateGEP(StrippedPtr, Idx, GEP.getName());
           // The NewGEP must be pointer typed, so must the old one -> BitCast
           return new BitCastInst(NewGEP, GEP.getType());
         }
@@ -1023,14 +1072,12 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // field at Offset in 'A's type.  If so, we can pull the cast through the
       // GEP.
       SmallVector<Value*, 8> NewIndices;
-      const Type *InTy =
+      Type *InTy =
         cast<PointerType>(BCI->getOperand(0)->getType())->getElementType();
       if (FindElementAtOffset(InTy, Offset, NewIndices)) {
         Value *NGEP = GEP.isInBounds() ?
-          Builder->CreateInBoundsGEP(BCI->getOperand(0), NewIndices.begin(),
-                                     NewIndices.end()) :
-          Builder->CreateGEP(BCI->getOperand(0), NewIndices.begin(),
-                             NewIndices.end());
+          Builder->CreateInBoundsGEP(BCI->getOperand(0), NewIndices) :
+          Builder->CreateGEP(BCI->getOperand(0), NewIndices);
         
         if (NGEP->getType() == GEP.getType())
           return ReplaceInstUsesWith(GEP, NGEP);
@@ -1045,15 +1092,43 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 
 
 
-static bool IsOnlyNullComparedAndFreed(const Value &V) {
-  for (Value::const_use_iterator UI = V.use_begin(), UE = V.use_end();
+static bool IsOnlyNullComparedAndFreed(Value *V, SmallVectorImpl<WeakVH> &Users,
+                                       int Depth = 0) {
+  if (Depth == 8)
+    return false;
+
+  for (Value::use_iterator UI = V->use_begin(), UE = V->use_end();
        UI != UE; ++UI) {
-    const User *U = *UI;
-    if (isFreeCall(U))
+    User *U = *UI;
+    if (isFreeCall(U)) {
+      Users.push_back(U);
       continue;
-    if (const ICmpInst *ICI = dyn_cast<ICmpInst>(U))
-      if (ICI->isEquality() && isa<ConstantPointerNull>(ICI->getOperand(1)))
+    }
+    if (ICmpInst *ICI = dyn_cast<ICmpInst>(U)) {
+      if (ICI->isEquality() && isa<ConstantPointerNull>(ICI->getOperand(1))) {
+        Users.push_back(ICI);
         continue;
+      }
+    }
+    if (BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
+      if (IsOnlyNullComparedAndFreed(BCI, Users, Depth+1)) {
+        Users.push_back(BCI);
+        continue;
+      }
+    }
+    if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U)) {
+      if (IsOnlyNullComparedAndFreed(GEPI, Users, Depth+1)) {
+        Users.push_back(GEPI);
+        continue;
+      }
+    }
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+          II->getIntrinsicID() == Intrinsic::lifetime_end) {
+        Users.push_back(II);
+        continue;
+      }
+    }
     return false;
   }
   return true;
@@ -1063,25 +1138,20 @@ Instruction *InstCombiner::visitMalloc(Instruction &MI) {
   // If we have a malloc call which is only used in any amount of comparisons
   // to null and free calls, delete the calls and replace the comparisons with
   // true or false as appropriate.
-  if (IsOnlyNullComparedAndFreed(MI)) {
-    for (Value::use_iterator UI = MI.use_begin(), UE = MI.use_end();
-         UI != UE;) {
-      // We can assume that every remaining use is a free call or an icmp eq/ne
-      // to null, so the cast is safe.
-      Instruction *I = cast<Instruction>(*UI);
+  SmallVector<WeakVH, 64> Users;
+  if (IsOnlyNullComparedAndFreed(&MI, Users)) {
+    for (unsigned i = 0, e = Users.size(); i != e; ++i) {
+      Instruction *I = cast_or_null<Instruction>(&*Users[i]);
+      if (!I) continue;
 
-      // Early increment here, as we're about to get rid of the user.
-      ++UI;
-
-      if (isFreeCall(I)) {
-        EraseInstFromFunction(*cast<CallInst>(I));
-        continue;
+      if (ICmpInst *C = dyn_cast<ICmpInst>(I)) {
+        ReplaceInstUsesWith(*C,
+                            ConstantInt::get(Type::getInt1Ty(C->getContext()),
+                                             C->isFalseWhenEqual()));
+      } else if (isa<BitCastInst>(I) || isa<GetElementPtrInst>(I)) {
+        ReplaceInstUsesWith(*I, UndefValue::get(I->getType()));
       }
-      // Again, the cast is safe.
-      ICmpInst *C = cast<ICmpInst>(I);
-      ReplaceInstUsesWith(*C, ConstantInt::get(Type::getInt1Ty(C->getContext()),
-                                               C->isFalseWhenEqual()));
-      EraseInstFromFunction(*C);
+      EraseInstFromFunction(*I);
     }
     return EraseInstFromFunction(MI);
   }
@@ -1120,8 +1190,7 @@ Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
       !isa<Constant>(X)) {
     // Swap Destinations and condition...
     BI.setCondition(X);
-    BI.setSuccessor(0, FalseDest);
-    BI.setSuccessor(1, TrueDest);
+    BI.swapSuccessors();
     return &BI;
   }
 
@@ -1136,8 +1205,7 @@ Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
       Cond->setPredicate(FCmpInst::getInversePredicate(FPred));
       
       // Swap Destinations and condition.
-      BI.setSuccessor(0, FalseDest);
-      BI.setSuccessor(1, TrueDest);
+      BI.swapSuccessors();
       Worklist.Add(Cond);
       return &BI;
     }
@@ -1153,8 +1221,7 @@ Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
       ICmpInst *Cond = cast<ICmpInst>(BI.getCondition());
       Cond->setPredicate(ICmpInst::getInversePredicate(IPred));
       // Swap Destinations and condition.
-      BI.setSuccessor(0, FalseDest);
-      BI.setSuccessor(1, TrueDest);
+      BI.swapSuccessors();
       Worklist.Add(Cond);
       return &BI;
     }
@@ -1168,11 +1235,17 @@ Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) {
     if (I->getOpcode() == Instruction::Add)
       if (ConstantInt *AddRHS = dyn_cast<ConstantInt>(I->getOperand(1))) {
         // change 'switch (X+4) case 1:' into 'switch (X) case -3'
-        for (unsigned i = 2, e = SI.getNumOperands(); i != e; i += 2)
-          SI.setOperand(i,
-                   ConstantExpr::getSub(cast<Constant>(SI.getOperand(i)),
-                                                AddRHS));
-        SI.setOperand(0, I->getOperand(0));
+        unsigned NumCases = SI.getNumCases();
+        // Skip the first item since that's the default case.
+        for (unsigned i = 1; i < NumCases; ++i) {
+          ConstantInt* CaseVal = SI.getCaseValue(i);
+          Constant* NewCaseVal = ConstantExpr::getSub(cast<Constant>(CaseVal),
+                                                      AddRHS);
+          assert(isa<ConstantInt>(NewCaseVal) &&
+                 "Result of expression should be constant");
+          SI.setSuccessorValue(i, cast<ConstantInt>(NewCaseVal));
+        }
+        SI.setCondition(I->getOperand(0));
         Worklist.Add(I);
         return &SI;
       }
@@ -1242,7 +1315,7 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
       Value *NewEV = Builder->CreateExtractValue(IV->getAggregateOperand(),
                                                  EV.getIndices());
       return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(),
-                                     ArrayRef<unsigned>(insi, inse));
+                                     makeArrayRef(insi, inse));
     }
     if (insi == inse)
       // The insert list is a prefix of the extract list
@@ -1254,7 +1327,7 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
       // with
       // %E extractvalue { i32 } { i32 42 }, 0
       return ExtractValueInst::Create(IV->getInsertedValueOperand(), 
-                                      ArrayRef<unsigned>(exti, exte));
+                                      makeArrayRef(exti, exte));
   }
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Agg)) {
     // We're extracting from an intrinsic, see if we're the only user, which
@@ -1310,7 +1383,7 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
     // load from a GEP. This reduces the size of the load.
     // FIXME: If a load is used only by extractvalue instructions then this
     //        could be done regardless of having multiple uses.
-    if (!L->isVolatile() && L->hasOneUse()) {
+    if (L->isSimple() && L->hasOneUse()) {
       // extractvalue has integer indices, getelementptr has Value*s. Convert.
       SmallVector<Value*, 4> Indices;
       // Prefix an i32 0 since we need the first element.
@@ -1322,8 +1395,7 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
       // We need to insert these at the location of the old load, not at that of
       // the extractvalue.
       Builder->SetInsertPoint(L->getParent(), L);
-      Value *GEP = Builder->CreateInBoundsGEP(L->getPointerOperand(),
-                                              Indices.begin(), Indices.end());
+      Value *GEP = Builder->CreateInBoundsGEP(L->getPointerOperand(), Indices);
       // Returning the load directly will cause the main loop to insert it in
       // the wrong spot, so use ReplaceInstUsesWith().
       return ReplaceInstUsesWith(EV, Builder->CreateLoad(GEP));
@@ -1339,6 +1411,342 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
   return 0;
 }
 
+enum Personality_Type {
+  Unknown_Personality,
+  GNU_Ada_Personality,
+  GNU_CXX_Personality
+};
+
+/// RecognizePersonality - See if the given exception handling personality
+/// function is one that we understand.  If so, return a description of it;
+/// otherwise return Unknown_Personality.
+static Personality_Type RecognizePersonality(Value *Pers) {
+  Function *F = dyn_cast<Function>(Pers->stripPointerCasts());
+  if (!F)
+    return Unknown_Personality;
+  return StringSwitch<Personality_Type>(F->getName())
+    .Case("__gnat_eh_personality", GNU_Ada_Personality)
+    .Case("__gxx_personality_v0", GNU_CXX_Personality)
+    .Default(Unknown_Personality);
+}
+
+/// isCatchAll - Return 'true' if the given typeinfo will match anything.
+static bool isCatchAll(Personality_Type Personality, Constant *TypeInfo) {
+  switch (Personality) {
+  case Unknown_Personality:
+    return false;
+  case GNU_Ada_Personality:
+    // While __gnat_all_others_value will match any Ada exception, it doesn't
+    // match foreign exceptions (or didn't, before gcc-4.7).
+    return false;
+  case GNU_CXX_Personality:
+    return TypeInfo->isNullValue();
+  }
+  llvm_unreachable("Unknown personality!");
+}
+
+static bool shorter_filter(const Value *LHS, const Value *RHS) {
+  return
+    cast<ArrayType>(LHS->getType())->getNumElements()
+  <
+    cast<ArrayType>(RHS->getType())->getNumElements();
+}
+
+Instruction *InstCombiner::visitLandingPadInst(LandingPadInst &LI) {
+  // The logic here should be correct for any real-world personality function.
+  // However if that turns out not to be true, the offending logic can always
+  // be conditioned on the personality function, like the catch-all logic is.
+  Personality_Type Personality = RecognizePersonality(LI.getPersonalityFn());
+
+  // Simplify the list of clauses, eg by removing repeated catch clauses
+  // (these are often created by inlining).
+  bool MakeNewInstruction = false; // If true, recreate using the following:
+  SmallVector<Value *, 16> NewClauses; // - Clauses for the new instruction;
+  bool CleanupFlag = LI.isCleanup();   // - The new instruction is a cleanup.
+
+  SmallPtrSet<Value *, 16> AlreadyCaught; // Typeinfos known caught already.
+  for (unsigned i = 0, e = LI.getNumClauses(); i != e; ++i) {
+    bool isLastClause = i + 1 == e;
+    if (LI.isCatch(i)) {
+      // A catch clause.
+      Value *CatchClause = LI.getClause(i);
+      Constant *TypeInfo = cast<Constant>(CatchClause->stripPointerCasts());
+
+      // If we already saw this clause, there is no point in having a second
+      // copy of it.
+      if (AlreadyCaught.insert(TypeInfo)) {
+        // This catch clause was not already seen.
+        NewClauses.push_back(CatchClause);
+      } else {
+        // Repeated catch clause - drop the redundant copy.
+        MakeNewInstruction = true;
+      }
+
+      // If this is a catch-all then there is no point in keeping any following
+      // clauses or marking the landingpad as having a cleanup.
+      if (isCatchAll(Personality, TypeInfo)) {
+        if (!isLastClause)
+          MakeNewInstruction = true;
+        CleanupFlag = false;
+        break;
+      }
+    } else {
+      // A filter clause.  If any of the filter elements were already caught
+      // then they can be dropped from the filter.  It is tempting to try to
+      // exploit the filter further by saying that any typeinfo that does not
+      // occur in the filter can't be caught later (and thus can be dropped).
+      // However this would be wrong, since typeinfos can match without being
+      // equal (for example if one represents a C++ class, and the other some
+      // class derived from it).
+      assert(LI.isFilter(i) && "Unsupported landingpad clause!");
+      Value *FilterClause = LI.getClause(i);
+      ArrayType *FilterType = cast<ArrayType>(FilterClause->getType());
+      unsigned NumTypeInfos = FilterType->getNumElements();
+
+      // An empty filter catches everything, so there is no point in keeping any
+      // following clauses or marking the landingpad as having a cleanup.  By
+      // dealing with this case here the following code is made a bit simpler.
+      if (!NumTypeInfos) {
+        NewClauses.push_back(FilterClause);
+        if (!isLastClause)
+          MakeNewInstruction = true;
+        CleanupFlag = false;
+        break;
+      }
+
+      bool MakeNewFilter = false; // If true, make a new filter.
+      SmallVector<Constant *, 16> NewFilterElts; // New elements.
+      if (isa<ConstantAggregateZero>(FilterClause)) {
+        // Not an empty filter - it contains at least one null typeinfo.
+        assert(NumTypeInfos > 0 && "Should have handled empty filter already!");
+        Constant *TypeInfo =
+          Constant::getNullValue(FilterType->getElementType());
+        // If this typeinfo is a catch-all then the filter can never match.
+        if (isCatchAll(Personality, TypeInfo)) {
+          // Throw the filter away.
+          MakeNewInstruction = true;
+          continue;
+        }
+
+        // There is no point in having multiple copies of this typeinfo, so
+        // discard all but the first copy if there is more than one.
+        NewFilterElts.push_back(TypeInfo);
+        if (NumTypeInfos > 1)
+          MakeNewFilter = true;
+      } else {
+        ConstantArray *Filter = cast<ConstantArray>(FilterClause);
+        SmallPtrSet<Value *, 16> SeenInFilter; // For uniquing the elements.
+        NewFilterElts.reserve(NumTypeInfos);
+
+        // Remove any filter elements that were already caught or that already
+        // occurred in the filter.  While there, see if any of the elements are
+        // catch-alls.  If so, the filter can be discarded.
+        bool SawCatchAll = false;
+        for (unsigned j = 0; j != NumTypeInfos; ++j) {
+          Value *Elt = Filter->getOperand(j);
+          Constant *TypeInfo = cast<Constant>(Elt->stripPointerCasts());
+          if (isCatchAll(Personality, TypeInfo)) {
+            // This element is a catch-all.  Bail out, noting this fact.
+            SawCatchAll = true;
+            break;
+          }
+          if (AlreadyCaught.count(TypeInfo))
+            // Already caught by an earlier clause, so having it in the filter
+            // is pointless.
+            continue;
+          // There is no point in having multiple copies of the same typeinfo in
+          // a filter, so only add it if we didn't already.
+          if (SeenInFilter.insert(TypeInfo))
+            NewFilterElts.push_back(cast<Constant>(Elt));
+        }
+        // A filter containing a catch-all cannot match anything by definition.
+        if (SawCatchAll) {
+          // Throw the filter away.
+          MakeNewInstruction = true;
+          continue;
+        }
+
+        // If we dropped something from the filter, make a new one.
+        if (NewFilterElts.size() < NumTypeInfos)
+          MakeNewFilter = true;
+      }
+      if (MakeNewFilter) {
+        FilterType = ArrayType::get(FilterType->getElementType(),
+                                    NewFilterElts.size());
+        FilterClause = ConstantArray::get(FilterType, NewFilterElts);
+        MakeNewInstruction = true;
+      }
+
+      NewClauses.push_back(FilterClause);
+
+      // If the new filter is empty then it will catch everything so there is
+      // no point in keeping any following clauses or marking the landingpad
+      // as having a cleanup.  The case of the original filter being empty was
+      // already handled above.
+      if (MakeNewFilter && !NewFilterElts.size()) {
+        assert(MakeNewInstruction && "New filter but not a new instruction!");
+        CleanupFlag = false;
+        break;
+      }
+    }
+  }
+
+  // If several filters occur in a row then reorder them so that the shortest
+  // filters come first (those with the smallest number of elements).  This is
+  // advantageous because shorter filters are more likely to match, speeding up
+  // unwinding, but mostly because it increases the effectiveness of the other
+  // filter optimizations below.
+  for (unsigned i = 0, e = NewClauses.size(); i + 1 < e; ) {
+    unsigned j;
+    // Find the maximal 'j' s.t. the range [i, j) consists entirely of filters.
+    for (j = i; j != e; ++j)
+      if (!isa<ArrayType>(NewClauses[j]->getType()))
+        break;
+
+    // Check whether the filters are already sorted by length.  We need to know
+    // if sorting them is actually going to do anything so that we only make a
+    // new landingpad instruction if it does.
+    for (unsigned k = i; k + 1 < j; ++k)
+      if (shorter_filter(NewClauses[k+1], NewClauses[k])) {
+        // Not sorted, so sort the filters now.  Doing an unstable sort would be
+        // correct too but reordering filters pointlessly might confuse users.
+        std::stable_sort(NewClauses.begin() + i, NewClauses.begin() + j,
+                         shorter_filter);
+        MakeNewInstruction = true;
+        break;
+      }
+
+    // Look for the next batch of filters.
+    i = j + 1;
+  }
+
+  // If typeinfos matched if and only if equal, then the elements of a filter L
+  // that occurs later than a filter F could be replaced by the intersection of
+  // the elements of F and L.  In reality two typeinfos can match without being
+  // equal (for example if one represents a C++ class, and the other some class
+  // derived from it) so it would be wrong to perform this transform in general.
+  // However the transform is correct and useful if F is a subset of L.  In that
+  // case L can be replaced by F, and thus removed altogether since repeating a
+  // filter is pointless.  So here we look at all pairs of filters F and L where
+  // L follows F in the list of clauses, and remove L if every element of F is
+  // an element of L.  This can occur when inlining C++ functions with exception
+  // specifications.
+  for (unsigned i = 0; i + 1 < NewClauses.size(); ++i) {
+    // Examine each filter in turn.
+    Value *Filter = NewClauses[i];
+    ArrayType *FTy = dyn_cast<ArrayType>(Filter->getType());
+    if (!FTy)
+      // Not a filter - skip it.
+      continue;
+    unsigned FElts = FTy->getNumElements();
+    // Examine each filter following this one.  Doing this backwards means that
+    // we don't have to worry about filters disappearing under us when removed.
+    for (unsigned j = NewClauses.size() - 1; j != i; --j) {
+      Value *LFilter = NewClauses[j];
+      ArrayType *LTy = dyn_cast<ArrayType>(LFilter->getType());
+      if (!LTy)
+        // Not a filter - skip it.
+        continue;
+      // If Filter is a subset of LFilter, i.e. every element of Filter is also
+      // an element of LFilter, then discard LFilter.
+      SmallVector<Value *, 16>::iterator J = NewClauses.begin() + j;
+      // If Filter is empty then it is a subset of LFilter.
+      if (!FElts) {
+        // Discard LFilter.
+        NewClauses.erase(J);
+        MakeNewInstruction = true;
+        // Move on to the next filter.
+        continue;
+      }
+      unsigned LElts = LTy->getNumElements();
+      // If Filter is longer than LFilter then it cannot be a subset of it.
+      if (FElts > LElts)
+        // Move on to the next filter.
+        continue;
+      // At this point we know that LFilter has at least one element.
+      if (isa<ConstantAggregateZero>(LFilter)) { // LFilter only contains zeros.
+        // Filter is a subset of LFilter iff Filter contains only zeros (as we
+        // already know that Filter is not longer than LFilter).
+        if (isa<ConstantAggregateZero>(Filter)) {
+          assert(FElts <= LElts && "Should have handled this case earlier!");
+          // Discard LFilter.
+          NewClauses.erase(J);
+          MakeNewInstruction = true;
+        }
+        // Move on to the next filter.
+        continue;
+      }
+      ConstantArray *LArray = cast<ConstantArray>(LFilter);
+      if (isa<ConstantAggregateZero>(Filter)) { // Filter only contains zeros.
+        // Since Filter is non-empty and contains only zeros, it is a subset of
+        // LFilter iff LFilter contains a zero.
+        assert(FElts > 0 && "Should have eliminated the empty filter earlier!");
+        for (unsigned l = 0; l != LElts; ++l)
+          if (LArray->getOperand(l)->isNullValue()) {
+            // LFilter contains a zero - discard it.
+            NewClauses.erase(J);
+            MakeNewInstruction = true;
+            break;
+          }
+        // Move on to the next filter.
+        continue;
+      }
+      // At this point we know that both filters are ConstantArrays.  Loop over
+      // operands to see whether every element of Filter is also an element of
+      // LFilter.  Since filters tend to be short this is probably faster than
+      // using a method that scales nicely.
+      ConstantArray *FArray = cast<ConstantArray>(Filter);
+      bool AllFound = true;
+      for (unsigned f = 0; f != FElts; ++f) {
+        Value *FTypeInfo = FArray->getOperand(f)->stripPointerCasts();
+        AllFound = false;
+        for (unsigned l = 0; l != LElts; ++l) {
+          Value *LTypeInfo = LArray->getOperand(l)->stripPointerCasts();
+          if (LTypeInfo == FTypeInfo) {
+            AllFound = true;
+            break;
+          }
+        }
+        if (!AllFound)
+          break;
+      }
+      if (AllFound) {
+        // Discard LFilter.
+        NewClauses.erase(J);
+        MakeNewInstruction = true;
+      }
+      // Move on to the next filter.
+    }
+  }
+
+  // If we changed any of the clauses, replace the old landingpad instruction
+  // with a new one.
+  if (MakeNewInstruction) {
+    LandingPadInst *NLI = LandingPadInst::Create(LI.getType(),
+                                                 LI.getPersonalityFn(),
+                                                 NewClauses.size());
+    for (unsigned i = 0, e = NewClauses.size(); i != e; ++i)
+      NLI->addClause(NewClauses[i]);
+    // A landing pad with no clauses must have the cleanup flag set.  It is
+    // theoretically possible, though highly unlikely, that we eliminated all
+    // clauses.  If so, force the cleanup flag to true.
+    if (NewClauses.empty())
+      CleanupFlag = true;
+    NLI->setCleanup(CleanupFlag);
+    return NLI;
+  }
+
+  // Even if none of the clauses changed, we may nonetheless have understood
+  // that the cleanup flag is pointless.  Clear it if so.
+  if (LI.isCleanup() != CleanupFlag) {
+    assert(!CleanupFlag && "Adding a cleanup, not removing one?!");
+    LI.setCleanup(CleanupFlag);
+    return &LI;
+  }
+
+  return 0;
+}
+
 
 
 
@@ -1350,7 +1758,8 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
   assert(I->hasOneUse() && "Invariants didn't hold!");
 
   // Cannot move control-flow-involving, volatile loads, vaarg, etc.
-  if (isa<PHINode>(I) || I->mayHaveSideEffects() || isa<TerminatorInst>(I))
+  if (isa<PHINode>(I) || isa<LandingPadInst>(I) || I->mayHaveSideEffects() ||
+      isa<TerminatorInst>(I))
     return false;
 
   // Do not sink alloca instructions out of the entry block.
@@ -1367,8 +1776,7 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
         return false;
   }
 
-  BasicBlock::iterator InsertPos = DestBlock->getFirstNonPHI();
-
+  BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
   I->moveBefore(InsertPos);
   ++NumSunkInst;
   return true;
@@ -1503,27 +1911,29 @@ bool InstCombiner::DoOneIteration(Function &F, unsigned Iteration) {
     // Do a quick scan over the function.  If we find any blocks that are
     // unreachable, remove any instructions inside of them.  This prevents
     // the instcombine code from having to deal with some bad special cases.
-    for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
-      if (!Visited.count(BB)) {
-        Instruction *Term = BB->getTerminator();
-        while (Term != BB->begin()) {   // Remove instrs bottom-up
-          BasicBlock::iterator I = Term; --I;
+    for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
+      if (Visited.count(BB)) continue;
 
-          DEBUG(errs() << "IC: DCE: " << *I << '\n');
-          // A debug intrinsic shouldn't force another iteration if we weren't
-          // going to do one without it.
-          if (!isa<DbgInfoIntrinsic>(I)) {
-            ++NumDeadInst;
-            MadeIRChange = true;
-          }
-
-          // If I is not void type then replaceAllUsesWith undef.
-          // This allows ValueHandlers and custom metadata to adjust itself.
-          if (!I->getType()->isVoidTy())
-            I->replaceAllUsesWith(UndefValue::get(I->getType()));
-          I->eraseFromParent();
+      // Delete the instructions backwards, as it has a reduced likelihood of
+      // having to update as many def-use and use-def chains.
+      Instruction *EndInst = BB->getTerminator(); // Last not to be deleted.
+      while (EndInst != BB->begin()) {
+        // Delete the next to last instruction.
+        BasicBlock::iterator I = EndInst;
+        Instruction *Inst = --I;
+        if (!Inst->use_empty())
+          Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
+        if (isa<LandingPadInst>(Inst)) {
+          EndInst = Inst;
+          continue;
         }
+        if (!isa<DbgInfoIntrinsic>(Inst)) {
+          ++NumDeadInst;
+          MadeIRChange = true;
+        }
+        Inst->eraseFromParent();
       }
+    }
   }
 
   while (!Worklist.isEmpty()) {
@@ -1604,13 +2014,13 @@ bool InstCombiner::DoOneIteration(Function &F, unsigned Iteration) {
         // Everything uses the new instruction now.
         I->replaceAllUsesWith(Result);
 
+        // Move the name to the new instruction first.
+        Result->takeName(I);
+
         // Push the new instruction and any users onto the worklist.
         Worklist.Add(Result);
         Worklist.AddUsersToWorkList(*Result);
 
-        // Move the name to the new instruction first.
-        Result->takeName(I);
-
         // Insert the new instruction into the basic block...
         BasicBlock *InstParent = I->getParent();
         BasicBlock::iterator InsertPos = I;
diff --git a/contrib/llvm/lib/Transforms/Instrumentation/EdgeProfiling.cpp b/contrib/llvm/lib/Transforms/Instrumentation/EdgeProfiling.cpp
index 1d31fcc4df3f..e8ef2654d256 100644
--- a/contrib/llvm/lib/Transforms/Instrumentation/EdgeProfiling.cpp
+++ b/contrib/llvm/lib/Transforms/Instrumentation/EdgeProfiling.cpp
@@ -74,7 +74,7 @@ bool EdgeProfiler::runOnModule(Module &M) {
     }
   }
 
-  const Type *ATy = ArrayType::get(Type::getInt32Ty(M.getContext()), NumEdges);
+  Type *ATy = ArrayType::get(Type::getInt32Ty(M.getContext()), NumEdges);
   GlobalVariable *Counters =
     new GlobalVariable(M, ATy, false, GlobalValue::InternalLinkage,
                        Constant::getNullValue(ATy), "EdgeProfCounters");
diff --git a/contrib/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/contrib/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 3f2c4123882d..ccf7e1109cd9 100644
--- a/contrib/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/contrib/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -60,11 +60,11 @@ namespace {
     bool runOnModule(Module &M);
 
     // Create the GCNO files for the Module based on DebugInfo.
-    void emitGCNO(DebugInfoFinder &DIF);
+    void emitGCNO();
 
     // Modify the program to track transitions along edges and call into the
     // profiling runtime to emit .gcda files when run.
-    bool emitProfileArcs(DebugInfoFinder &DIF);
+    bool emitProfileArcs();
 
     // Get pointers to the functions in the runtime library.
     Constant *getStartFileFunc();
@@ -86,8 +86,7 @@ namespace {
 
     // Add the function to write out all our counters to the global destructor
     // list.
-    void insertCounterWriteout(DebugInfoFinder &,
-                               SmallVector<std::pair<GlobalVariable *,
+    void insertCounterWriteout(SmallVector<std::pair<GlobalVariable *,
                                                      MDNode *>, 8> &);
 
     std::string mangleName(DICompileUnit CU, std::string NewStem);
@@ -110,15 +109,6 @@ ModulePass *llvm::createGCOVProfilerPass(bool EmitNotes, bool EmitData,
   return new GCOVProfiler(EmitNotes, EmitData, Use402Format);
 }
 
-static DISubprogram findSubprogram(DIScope Scope) {
-  while (!Scope.isSubprogram()) {
-    assert(Scope.isLexicalBlock() &&
-           "Debug location not lexical block or subprogram");
-    Scope = DILexicalBlock(Scope).getContext();
-  }
-  return DISubprogram(Scope);
-}
-
 namespace {
   class GCOVRecord {
    protected:
@@ -177,18 +167,24 @@ namespace {
     }
 
     uint32_t length() {
+      // Here 2 = 1 for string lenght + 1 for '0' id#.
       return lengthOfGCOVString(Filename) + 2 + Lines.size();
     }
 
-   private:
-    friend class GCOVBlock;
+    void writeOut() {
+      write(0);
+      writeGCOVString(Filename);
+      for (int i = 0, e = Lines.size(); i != e; ++i)
+        write(Lines[i]);
+    }
 
-    GCOVLines(std::string Filename, raw_ostream *os)
-        : Filename(Filename) {
+    GCOVLines(StringRef F, raw_ostream *os) 
+      : Filename(F) {
       this->os = os;
     }
 
-    std::string Filename;
+   private:
+    StringRef Filename;
     SmallVector<uint32_t, 32> Lines;
   };
 
@@ -197,7 +193,7 @@ namespace {
   // other blocks.
   class GCOVBlock : public GCOVRecord {
    public:
-    GCOVLines &getFile(std::string Filename) {
+    GCOVLines &getFile(StringRef Filename) {
       GCOVLines *&Lines = LinesByFile[Filename];
       if (!Lines) {
         Lines = new GCOVLines(Filename, os);
@@ -220,13 +216,8 @@ namespace {
       write(Len);
       write(Number);
       for (StringMap<GCOVLines *>::iterator I = LinesByFile.begin(),
-               E = LinesByFile.end(); I != E; ++I) {
-        write(0);
-        writeGCOVString(I->second->Filename);
-        for (int i = 0, e = I->second->Lines.size(); i != e; ++i) {
-          write(I->second->Lines[i]);
-        }
-      }
+               E = LinesByFile.end(); I != E; ++I) 
+        I->second->writeOut();
       write(0);
       write(0);
     }
@@ -353,66 +344,66 @@ bool GCOVProfiler::runOnModule(Module &M) {
   this->M = &M;
   Ctx = &M.getContext();
 
-  DebugInfoFinder DIF;
-  DIF.processModule(M);
-
-  if (EmitNotes) emitGCNO(DIF);
-  if (EmitData) return emitProfileArcs(DIF);
+  if (EmitNotes) emitGCNO();
+  if (EmitData) return emitProfileArcs();
   return false;
 }
 
-void GCOVProfiler::emitGCNO(DebugInfoFinder &DIF) {
+void GCOVProfiler::emitGCNO() {
   DenseMap<const MDNode *, raw_fd_ostream *> GcnoFiles;
-  for (DebugInfoFinder::iterator I = DIF.compile_unit_begin(),
-           E = DIF.compile_unit_end(); I != E; ++I) {
-    // Each compile unit gets its own .gcno file. This means that whether we run
-    // this pass over the original .o's as they're produced, or run it after
-    // LTO, we'll generate the same .gcno files.
-
-    DICompileUnit CU(*I);
-    raw_fd_ostream *&out = GcnoFiles[CU];
-    std::string ErrorInfo;
-    out = new raw_fd_ostream(mangleName(CU, "gcno").c_str(), ErrorInfo,
-                             raw_fd_ostream::F_Binary);
-    if (!Use402Format)
-      out->write("oncg*404MVLL", 12);
-    else
-      out->write("oncg*402MVLL", 12);
-  }
-
-  for (DebugInfoFinder::iterator SPI = DIF.subprogram_begin(),
-           SPE = DIF.subprogram_end(); SPI != SPE; ++SPI) {
-    DISubprogram SP(*SPI);
-    raw_fd_ostream *&os = GcnoFiles[SP.getCompileUnit()];
-
-    Function *F = SP.getFunction();
-    if (!F) continue;
-    GCOVFunction Func(SP, os, Use402Format);
-
-    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
-      GCOVBlock &Block = Func.getBlock(BB);
-      TerminatorInst *TI = BB->getTerminator();
-      if (int successors = TI->getNumSuccessors()) {
-        for (int i = 0; i != successors; ++i) {
-          Block.addEdge(Func.getBlock(TI->getSuccessor(i)));
+  NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
+  if (CU_Nodes) {
+    for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
+      // Each compile unit gets its own .gcno file. This means that whether we run
+      // this pass over the original .o's as they're produced, or run it after
+      // LTO, we'll generate the same .gcno files.
+      
+      DICompileUnit CU(CU_Nodes->getOperand(i));
+      raw_fd_ostream *&out = GcnoFiles[CU];
+      std::string ErrorInfo;
+      out = new raw_fd_ostream(mangleName(CU, "gcno").c_str(), ErrorInfo,
+                               raw_fd_ostream::F_Binary);
+      if (!Use402Format)
+        out->write("oncg*404MVLL", 12);
+      else
+        out->write("oncg*204MVLL", 12);
+  
+      DIArray SPs = CU.getSubprograms();
+      for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) {
+        DISubprogram SP(SPs.getElement(i));
+        if (!SP.Verify()) continue;
+        raw_fd_ostream *&os = GcnoFiles[CU];
+        
+        Function *F = SP.getFunction();
+        if (!F) continue;
+        GCOVFunction Func(SP, os, Use402Format);
+        
+        for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
+          GCOVBlock &Block = Func.getBlock(BB);
+          TerminatorInst *TI = BB->getTerminator();
+          if (int successors = TI->getNumSuccessors()) {
+            for (int i = 0; i != successors; ++i) {
+              Block.addEdge(Func.getBlock(TI->getSuccessor(i)));
+            }
+          } else if (isa<ReturnInst>(TI)) {
+            Block.addEdge(Func.getReturnBlock());
+          }
+          
+          uint32_t Line = 0;
+          for (BasicBlock::iterator I = BB->begin(), IE = BB->end(); I != IE; ++I) {
+            const DebugLoc &Loc = I->getDebugLoc();
+            if (Loc.isUnknown()) continue;
+            if (Line == Loc.getLine()) continue;
+            Line = Loc.getLine();
+            if (SP != getDISubprogram(Loc.getScope(*Ctx))) continue;
+            
+            GCOVLines &Lines = Block.getFile(SP.getFilename());
+            Lines.addLine(Loc.getLine());
+          }
         }
-      } else if (isa<ReturnInst>(TI)) {
-        Block.addEdge(Func.getReturnBlock());
-      }
-
-      uint32_t Line = 0;
-      for (BasicBlock::iterator I = BB->begin(), IE = BB->end(); I != IE; ++I) {
-        const DebugLoc &Loc = I->getDebugLoc();
-        if (Loc.isUnknown()) continue;
-        if (Line == Loc.getLine()) continue;
-        Line = Loc.getLine();
-        if (SP != findSubprogram(DIScope(Loc.getScope(*Ctx)))) continue;
-
-        GCOVLines &Lines = Block.getFile(SP.getFilename());
-        Lines.addLine(Loc.getLine());
+        Func.writeOut();
       }
     }
-    Func.writeOut();
   }
 
   for (DenseMap<const MDNode *, raw_fd_ostream *>::iterator
@@ -424,103 +415,107 @@ void GCOVProfiler::emitGCNO(DebugInfoFinder &DIF) {
   }
 }
 
-bool GCOVProfiler::emitProfileArcs(DebugInfoFinder &DIF) {
-  if (DIF.subprogram_begin() == DIF.subprogram_end())
-    return false;
+bool GCOVProfiler::emitProfileArcs() {
+  NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
+  if (!CU_Nodes) return false;
 
-  SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
-  for (DebugInfoFinder::iterator SPI = DIF.subprogram_begin(),
-           SPE = DIF.subprogram_end(); SPI != SPE; ++SPI) {
-    DISubprogram SP(*SPI);
-    Function *F = SP.getFunction();
-    if (!F) continue;
-
-    unsigned Edges = 0;
-    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
-      TerminatorInst *TI = BB->getTerminator();
-      if (isa<ReturnInst>(TI))
-        ++Edges;
-      else
-        Edges += TI->getNumSuccessors();
-    }
-
-    const ArrayType *CounterTy =
+  bool Result = false;  
+  for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
+    DICompileUnit CU(CU_Nodes->getOperand(i));
+    DIArray SPs = CU.getSubprograms();
+    SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
+    for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) {
+      DISubprogram SP(SPs.getElement(i));
+      if (!SP.Verify()) continue;
+      Function *F = SP.getFunction();
+      if (!F) continue;
+      if (!Result) Result = true;
+      unsigned Edges = 0;
+      for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
+        TerminatorInst *TI = BB->getTerminator();
+        if (isa<ReturnInst>(TI))
+          ++Edges;
+        else
+          Edges += TI->getNumSuccessors();
+      }
+      
+      ArrayType *CounterTy =
         ArrayType::get(Type::getInt64Ty(*Ctx), Edges);
-    GlobalVariable *Counters =
+      GlobalVariable *Counters =
         new GlobalVariable(*M, CounterTy, false,
                            GlobalValue::InternalLinkage,
                            Constant::getNullValue(CounterTy),
                            "__llvm_gcov_ctr", 0, false, 0);
-    CountersBySP.push_back(std::make_pair(Counters, (MDNode*)SP));
-
-    UniqueVector<BasicBlock *> ComplexEdgePreds;
-    UniqueVector<BasicBlock *> ComplexEdgeSuccs;
-
-    unsigned Edge = 0;
-    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
-      TerminatorInst *TI = BB->getTerminator();
-      int Successors = isa<ReturnInst>(TI) ? 1 : TI->getNumSuccessors();
-      if (Successors) {
-        IRBuilder<> Builder(TI);
-
-        if (Successors == 1) {
-          Value *Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0,
-                                                              Edge);
-          Value *Count = Builder.CreateLoad(Counter);
-          Count = Builder.CreateAdd(Count,
-                                    ConstantInt::get(Type::getInt64Ty(*Ctx),1));
-          Builder.CreateStore(Count, Counter);
-        } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
-          Value *Sel = Builder.CreateSelect(
+      CountersBySP.push_back(std::make_pair(Counters, (MDNode*)SP));
+      
+      UniqueVector<BasicBlock *> ComplexEdgePreds;
+      UniqueVector<BasicBlock *> ComplexEdgeSuccs;
+      
+      unsigned Edge = 0;
+      for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
+        TerminatorInst *TI = BB->getTerminator();
+        int Successors = isa<ReturnInst>(TI) ? 1 : TI->getNumSuccessors();
+        if (Successors) {
+          IRBuilder<> Builder(TI);
+          
+          if (Successors == 1) {
+            Value *Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0,
+                                                                Edge);
+            Value *Count = Builder.CreateLoad(Counter);
+            Count = Builder.CreateAdd(Count,
+                                      ConstantInt::get(Type::getInt64Ty(*Ctx),1));
+            Builder.CreateStore(Count, Counter);
+          } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
+            Value *Sel = Builder.CreateSelect(
               BI->getCondition(),
               ConstantInt::get(Type::getInt64Ty(*Ctx), Edge),
               ConstantInt::get(Type::getInt64Ty(*Ctx), Edge + 1));
-          SmallVector<Value *, 2> Idx;
-          Idx.push_back(Constant::getNullValue(Type::getInt64Ty(*Ctx)));
-          Idx.push_back(Sel);
-          Value *Counter = Builder.CreateInBoundsGEP(Counters,
-                                                     Idx.begin(), Idx.end());
-          Value *Count = Builder.CreateLoad(Counter);
-          Count = Builder.CreateAdd(Count,
-                                    ConstantInt::get(Type::getInt64Ty(*Ctx),1));
-          Builder.CreateStore(Count, Counter);
-        } else {
-          ComplexEdgePreds.insert(BB);
-          for (int i = 0; i != Successors; ++i)
-            ComplexEdgeSuccs.insert(TI->getSuccessor(i));
+            SmallVector<Value *, 2> Idx;
+            Idx.push_back(Constant::getNullValue(Type::getInt64Ty(*Ctx)));
+            Idx.push_back(Sel);
+            Value *Counter = Builder.CreateInBoundsGEP(Counters, Idx);
+            Value *Count = Builder.CreateLoad(Counter);
+            Count = Builder.CreateAdd(Count,
+                                      ConstantInt::get(Type::getInt64Ty(*Ctx),1));
+            Builder.CreateStore(Count, Counter);
+          } else {
+            ComplexEdgePreds.insert(BB);
+            for (int i = 0; i != Successors; ++i)
+              ComplexEdgeSuccs.insert(TI->getSuccessor(i));
+          }
+          Edge += Successors;
         }
-        Edge += Successors;
       }
-    }
-
-    if (!ComplexEdgePreds.empty()) {
-      GlobalVariable *EdgeTable =
+      
+      if (!ComplexEdgePreds.empty()) {
+        GlobalVariable *EdgeTable =
           buildEdgeLookupTable(F, Counters,
                                ComplexEdgePreds, ComplexEdgeSuccs);
-      GlobalVariable *EdgeState = getEdgeStateValue();
-
-      const Type *Int32Ty = Type::getInt32Ty(*Ctx);
-      for (int i = 0, e = ComplexEdgePreds.size(); i != e; ++i) {
-        IRBuilder<> Builder(ComplexEdgePreds[i+1]->getTerminator());
-        Builder.CreateStore(ConstantInt::get(Int32Ty, i), EdgeState);
-      }
-      for (int i = 0, e = ComplexEdgeSuccs.size(); i != e; ++i) {
-        // call runtime to perform increment
-        IRBuilder<> Builder(ComplexEdgeSuccs[i+1]->getFirstNonPHI());
-        Value *CounterPtrArray =
+        GlobalVariable *EdgeState = getEdgeStateValue();
+        
+        Type *Int32Ty = Type::getInt32Ty(*Ctx);
+        for (int i = 0, e = ComplexEdgePreds.size(); i != e; ++i) {
+          IRBuilder<> Builder(ComplexEdgePreds[i+1]->getTerminator());
+          Builder.CreateStore(ConstantInt::get(Int32Ty, i), EdgeState);
+        }
+        for (int i = 0, e = ComplexEdgeSuccs.size(); i != e; ++i) {
+          // call runtime to perform increment
+          BasicBlock::iterator InsertPt =
+            ComplexEdgeSuccs[i+1]->getFirstInsertionPt();
+          IRBuilder<> Builder(InsertPt);
+          Value *CounterPtrArray =
             Builder.CreateConstInBoundsGEP2_64(EdgeTable, 0,
                                                i * ComplexEdgePreds.size());
-        Builder.CreateCall2(getIncrementIndirectCounterFunc(),
-                            EdgeState, CounterPtrArray);
-        // clear the predecessor number
-        Builder.CreateStore(ConstantInt::get(Int32Ty, 0xffffffff), EdgeState);
+          Builder.CreateCall2(getIncrementIndirectCounterFunc(),
+                              EdgeState, CounterPtrArray);
+          // clear the predecessor number
+          Builder.CreateStore(ConstantInt::get(Int32Ty, 0xffffffff), EdgeState);
+        }
       }
     }
+    insertCounterWriteout(CountersBySP);
   }
-
-  insertCounterWriteout(DIF, CountersBySP);
-
-  return true;
+  return Result;
 }
 
 // All edges with successors that aren't branches are "complex", because it
@@ -535,8 +530,8 @@ GlobalVariable *GCOVProfiler::buildEdgeLookupTable(
   // read it. Threads and invoke make this untrue.
 
   // emit [(succs * preds) x i64*], logically [succ x [pred x i64*]].
-  const Type *Int64PtrTy = Type::getInt64PtrTy(*Ctx);
-  const ArrayType *EdgeTableTy = ArrayType::get(
+  Type *Int64PtrTy = Type::getInt64PtrTy(*Ctx);
+  ArrayType *EdgeTableTy = ArrayType::get(
       Int64PtrTy, Succs.size() * Preds.size());
 
   Constant **EdgeTable = new Constant*[Succs.size() * Preds.size()];
@@ -572,7 +567,7 @@ GlobalVariable *GCOVProfiler::buildEdgeLookupTable(
 }
 
 Constant *GCOVProfiler::getStartFileFunc() {
-  const FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
                                               Type::getInt8PtrTy(*Ctx), false);
   return M->getOrInsertFunction("llvm_gcda_start_file", FTy);
 }
@@ -582,7 +577,7 @@ Constant *GCOVProfiler::getIncrementIndirectCounterFunc() {
     Type::getInt32PtrTy(*Ctx),                  // uint32_t *predecessor
     Type::getInt64PtrTy(*Ctx)->getPointerTo(),  // uint64_t **state_table_row
   };
-  const FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
                                               Args, false);
   return M->getOrInsertFunction("llvm_gcda_increment_indirect_counter", FTy);
 }
@@ -592,7 +587,7 @@ Constant *GCOVProfiler::getEmitFunctionFunc() {
     Type::getInt32Ty(*Ctx),    // uint32_t ident
     Type::getInt8PtrTy(*Ctx),  // const char *function_name
   };
-  const FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
                                               Args, false);
   return M->getOrInsertFunction("llvm_gcda_emit_function", FTy);
 }
@@ -602,13 +597,13 @@ Constant *GCOVProfiler::getEmitArcsFunc() {
     Type::getInt32Ty(*Ctx),     // uint32_t num_counters
     Type::getInt64PtrTy(*Ctx),  // uint64_t *counters
   };
-  const FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx),
                                               Args, false);
   return M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy);
 }
 
 Constant *GCOVProfiler::getEndFileFunc() {
-  const FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
   return M->getOrInsertFunction("llvm_gcda_end_file", FTy);
 }
 
@@ -626,9 +621,8 @@ GlobalVariable *GCOVProfiler::getEdgeStateValue() {
 }
 
 void GCOVProfiler::insertCounterWriteout(
-    DebugInfoFinder &DIF,
     SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> &CountersBySP) {
-  const FunctionType *WriteoutFTy =
+  FunctionType *WriteoutFTy =
       FunctionType::get(Type::getVoidTy(*Ctx), false);
   Function *WriteoutF = Function::Create(WriteoutFTy,
                                          GlobalValue::InternalLinkage,
@@ -642,29 +636,31 @@ void GCOVProfiler::insertCounterWriteout(
   Constant *EmitArcs = getEmitArcsFunc();
   Constant *EndFile = getEndFileFunc();
 
-  for (DebugInfoFinder::iterator CUI = DIF.compile_unit_begin(),
-           CUE = DIF.compile_unit_end(); CUI != CUE; ++CUI) {
-    DICompileUnit compile_unit(*CUI);
-    std::string FilenameGcda = mangleName(compile_unit, "gcda");
-    Builder.CreateCall(StartFile,
-                       Builder.CreateGlobalStringPtr(FilenameGcda));
-    for (SmallVector<std::pair<GlobalVariable *, MDNode *>, 8>::iterator
+  NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
+  if (CU_Nodes) {
+    for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
+      DICompileUnit compile_unit(CU_Nodes->getOperand(i));
+      std::string FilenameGcda = mangleName(compile_unit, "gcda");
+      Builder.CreateCall(StartFile,
+                         Builder.CreateGlobalStringPtr(FilenameGcda));
+      for (SmallVector<std::pair<GlobalVariable *, MDNode *>, 8>::iterator
              I = CountersBySP.begin(), E = CountersBySP.end();
-         I != E; ++I) {
-      DISubprogram SP(I->second);
-      intptr_t ident = reinterpret_cast<intptr_t>(I->second);
-      Builder.CreateCall2(EmitFunction,
-                          ConstantInt::get(Type::getInt32Ty(*Ctx), ident),
-                          Builder.CreateGlobalStringPtr(SP.getName()));
-                                                        
-      GlobalVariable *GV = I->first;
-      unsigned Arcs =
+           I != E; ++I) {
+        DISubprogram SP(I->second);
+        intptr_t ident = reinterpret_cast<intptr_t>(I->second);
+        Builder.CreateCall2(EmitFunction,
+                            ConstantInt::get(Type::getInt32Ty(*Ctx), ident),
+                            Builder.CreateGlobalStringPtr(SP.getName()));
+        
+        GlobalVariable *GV = I->first;
+        unsigned Arcs =
           cast<ArrayType>(GV->getType()->getElementType())->getNumElements();
-      Builder.CreateCall2(EmitArcs,
-                          ConstantInt::get(Type::getInt32Ty(*Ctx), Arcs),
-                          Builder.CreateConstGEP2_64(GV, 0, 0));
+        Builder.CreateCall2(EmitArcs,
+                            ConstantInt::get(Type::getInt32Ty(*Ctx), Arcs),
+                            Builder.CreateConstGEP2_64(GV, 0, 0));
+      }
+      Builder.CreateCall(EndFile);
     }
-    Builder.CreateCall(EndFile);
   }
   Builder.CreateRetVoid();
 
diff --git a/contrib/llvm/lib/Transforms/Instrumentation/OptimalEdgeProfiling.cpp b/contrib/llvm/lib/Transforms/Instrumentation/OptimalEdgeProfiling.cpp
index e09f882aa323..62c21b8e9c59 100644
--- a/contrib/llvm/lib/Transforms/Instrumentation/OptimalEdgeProfiling.cpp
+++ b/contrib/llvm/lib/Transforms/Instrumentation/OptimalEdgeProfiling.cpp
@@ -112,8 +112,8 @@ bool OptimalEdgeProfiler::runOnModule(Module &M) {
   // be calculated from other edge counters on reading the profile info back
   // in.
 
-  const Type *Int32 = Type::getInt32Ty(M.getContext());
-  const ArrayType *ATy = ArrayType::get(Int32, NumEdges);
+  Type *Int32 = Type::getInt32Ty(M.getContext());
+  ArrayType *ATy = ArrayType::get(Int32, NumEdges);
   GlobalVariable *Counters =
     new GlobalVariable(M, ATy, false, GlobalValue::InternalLinkage,
                        Constant::getNullValue(ATy), "OptEdgeProfCounters");
diff --git a/contrib/llvm/lib/Transforms/Instrumentation/PathProfiling.cpp b/contrib/llvm/lib/Transforms/Instrumentation/PathProfiling.cpp
index 75416637db4f..23915d39f214 100644
--- a/contrib/llvm/lib/Transforms/Instrumentation/PathProfiling.cpp
+++ b/contrib/llvm/lib/Transforms/Instrumentation/PathProfiling.cpp
@@ -374,7 +374,7 @@ namespace llvm {
   template<bool xcompile> class TypeBuilder<PathProfilingFunctionTable,
                                             xcompile> {
   public:
-    static const StructType *get(LLVMContext& C) {
+    static StructType *get(LLVMContext& C) {
       return( StructType::get(
                 TypeBuilder<types::i<32>, xcompile>::get(C), // type
                 TypeBuilder<types::i<32>, xcompile>::get(C), // array size
@@ -909,7 +909,7 @@ BasicBlock::iterator PathProfiler::getInsertionPoint(BasicBlock* block, Value*
                                                      pathNumber) {
   if(pathNumber == NULL || isa<ConstantInt>(pathNumber)
      || (((Instruction*)(pathNumber))->getParent()) != block) {
-    return(block->getFirstNonPHI());
+    return(block->getFirstInsertionPt());
   } else {
     Instruction* pathNumberInst = (Instruction*) (pathNumber);
     BasicBlock::iterator insertPoint;
@@ -930,7 +930,7 @@ BasicBlock::iterator PathProfiler::getInsertionPoint(BasicBlock* block, Value*
 // A PHINode is created in the node, and its values initialized to -1U.
 void PathProfiler::preparePHI(BLInstrumentationNode* node) {
   BasicBlock* block = node->getBlock();
-  BasicBlock::iterator insertPoint = block->getFirstNonPHI();
+  BasicBlock::iterator insertPoint = block->getFirstInsertionPt();
   pred_iterator PB = pred_begin(node->getBlock()),
           PE = pred_end(node->getBlock());
   PHINode* phi = PHINode::Create(Type::getInt32Ty(*Context),
@@ -999,7 +999,7 @@ void PathProfiler::insertNumberIncrement(BLInstrumentationNode* node,
   BasicBlock::iterator insertPoint;
 
   if( atBeginning )
-    insertPoint = block->getFirstNonPHI();
+    insertPoint = block->getFirstInsertionPt();
   else
     insertPoint = block->getTerminator();
 
@@ -1029,8 +1029,7 @@ void PathProfiler::insertCounterIncrement(Value* incValue,
     gepIndices[1] = incValue;
 
     GetElementPtrInst* pcPointer =
-      GetElementPtrInst::Create(dag->getCounterArray(),
-                                gepIndices.begin(), gepIndices.end(),
+      GetElementPtrInst::Create(dag->getCounterArray(), gepIndices,
                                 "counterInc", insertPoint);
 
     // Load from the array - call it oldPC
@@ -1140,7 +1139,7 @@ void PathProfiler::insertInstrumentationStartingAt(BLInstrumentationEdge* edge,
     }
 
     BasicBlock::iterator insertPoint = atBeginning ?
-      instrumentNode->getBlock()->getFirstNonPHI() :
+      instrumentNode->getBlock()->getFirstInsertionPt() :
       instrumentNode->getBlock()->getTerminator();
 
     // add information from the bottom edge, if it exists
@@ -1172,7 +1171,7 @@ void PathProfiler::insertInstrumentationStartingAt(BLInstrumentationEdge* edge,
   // Insert instrumentation if this is a normal edge
   else {
     BasicBlock::iterator insertPoint = atBeginning ?
-      instrumentNode->getBlock()->getFirstNonPHI() :
+      instrumentNode->getBlock()->getFirstInsertionPt() :
       instrumentNode->getBlock()->getTerminator();
 
     if( edge->isInitialization() ) { // initialize path number
@@ -1233,7 +1232,7 @@ void PathProfiler::insertInstrumentation(
          end = callEdges.end(); edge != end; edge++ ) {
     BLInstrumentationNode* node =
       (BLInstrumentationNode*)(*edge)->getSource();
-    BasicBlock::iterator insertPoint = node->getBlock()->getFirstNonPHI();
+    BasicBlock::iterator insertPoint = node->getBlock()->getFirstInsertionPt();
 
     // Find the first function call
     while( ((Instruction&)(*insertPoint)).getOpcode() != Instruction::Call )
@@ -1289,7 +1288,7 @@ void PathProfiler::runOnFunction(std::vector<Constant*> &ftInit,
 
   // Should we store the information in an array or hash
   if( dag.getNumberOfPaths() <= HASH_THRESHHOLD ) {
-    const Type* t = ArrayType::get(Type::getInt32Ty(*Context),
+    Type* t = ArrayType::get(Type::getInt32Ty(*Context),
                                    dag.getNumberOfPaths());
 
     dag.setCounterArray(new GlobalVariable(M, t, false,
@@ -1301,7 +1300,7 @@ void PathProfiler::runOnFunction(std::vector<Constant*> &ftInit,
 
   // Add to global function reference table
   unsigned type;
-  const Type* voidPtr = TypeBuilder<types::i<8>*, true>::get(*Context);
+  Type* voidPtr = TypeBuilder<types::i<8>*, true>::get(*Context);
 
   if( dag.getNumberOfPaths() <= HASH_THRESHHOLD )
     type = ProfilingArray;
@@ -1315,7 +1314,7 @@ void PathProfiler::runOnFunction(std::vector<Constant*> &ftInit,
     ConstantExpr::getBitCast(dag.getCounterArray(), voidPtr) :
     Constant::getNullValue(voidPtr);
 
-  const StructType* at = ftEntryTypeBuilder::get(*Context);
+  StructType* at = ftEntryTypeBuilder::get(*Context);
   ConstantStruct* functionEntry =
     (ConstantStruct*)ConstantStruct::get(at, entryArray);
   ftInit.push_back(functionEntry);
@@ -1379,8 +1378,8 @@ bool PathProfiler::runOnModule(Module &M) {
     runOnFunction(ftInit, *F, M);
   }
 
-  const Type *t = ftEntryTypeBuilder::get(*Context);
-  const ArrayType* ftArrayType = ArrayType::get(t, ftInit.size());
+  Type *t = ftEntryTypeBuilder::get(*Context);
+  ArrayType* ftArrayType = ArrayType::get(t, ftInit.size());
   Constant* ftInitConstant = ConstantArray::get(ftArrayType, ftInit);
 
   DEBUG(dbgs() << " ftArrayType:" << *ftArrayType << "\n");
@@ -1388,7 +1387,7 @@ bool PathProfiler::runOnModule(Module &M) {
   GlobalVariable* functionTable =
     new GlobalVariable(M, ftArrayType, false, GlobalValue::InternalLinkage,
                        ftInitConstant, "functionPathTable");
-  const Type *eltType = ftArrayType->getTypeAtIndex((unsigned)0);
+  Type *eltType = ftArrayType->getTypeAtIndex((unsigned)0);
   InsertProfilingInitCall(Main, "llvm_start_path_profiling", functionTable,
                           PointerType::getUnqual(eltType));
 
diff --git a/contrib/llvm/lib/Transforms/Instrumentation/ProfilingUtils.cpp b/contrib/llvm/lib/Transforms/Instrumentation/ProfilingUtils.cpp
index 445a5b6f6074..de57cd173483 100644
--- a/contrib/llvm/lib/Transforms/Instrumentation/ProfilingUtils.cpp
+++ b/contrib/llvm/lib/Transforms/Instrumentation/ProfilingUtils.cpp
@@ -25,9 +25,9 @@ void llvm::InsertProfilingInitCall(Function *MainFn, const char *FnName,
                                    GlobalValue *Array,
                                    PointerType *arrayType) {
   LLVMContext &Context = MainFn->getContext();
-  const Type *ArgVTy =
+  Type *ArgVTy =
     PointerType::getUnqual(Type::getInt8PtrTy(Context));
-  const PointerType *UIntPtr = arrayType ? arrayType :
+  PointerType *UIntPtr = arrayType ? arrayType :
     Type::getInt32PtrTy(Context);
   Module &M = *MainFn->getParent();
   Constant *InitFn = M.getOrInsertFunction(FnName, Type::getInt32Ty(Context),
@@ -51,8 +51,7 @@ void llvm::InsertProfilingInitCall(Function *MainFn, const char *FnName,
                              Constant::getNullValue(Type::getInt32Ty(Context)));
   unsigned NumElements = 0;
   if (Array) {
-    Args[2] = ConstantExpr::getGetElementPtr(Array, &GEPIndices[0],
-                                             GEPIndices.size());
+    Args[2] = ConstantExpr::getGetElementPtr(Array, GEPIndices);
     NumElements =
       cast<ArrayType>(Array->getType()->getElementType())->getNumElements();
   } else {
@@ -108,7 +107,7 @@ void llvm::InsertProfilingInitCall(Function *MainFn, const char *FnName,
 void llvm::IncrementCounterInBlock(BasicBlock *BB, unsigned CounterNum,
                                    GlobalValue *CounterArray, bool beginning) {
   // Insert the increment after any alloca or PHI instructions...
-  BasicBlock::iterator InsertPos = beginning ? BB->getFirstNonPHI() :
+  BasicBlock::iterator InsertPos = beginning ? BB->getFirstInsertionPt() :
                                    BB->getTerminator();
   while (isa<AllocaInst>(InsertPos))
     ++InsertPos;
@@ -120,7 +119,7 @@ void llvm::IncrementCounterInBlock(BasicBlock *BB, unsigned CounterNum,
   Indices[0] = Constant::getNullValue(Type::getInt32Ty(Context));
   Indices[1] = ConstantInt::get(Type::getInt32Ty(Context), CounterNum);
   Constant *ElementPtr =
-    ConstantExpr::getGetElementPtr(CounterArray, &Indices[0], Indices.size());
+    ConstantExpr::getGetElementPtr(CounterArray, Indices);
 
   // Load, increment and store the value back.
   Value *OldVal = new LoadInst(ElementPtr, "OldFuncCounter", InsertPos);
@@ -137,7 +136,7 @@ void llvm::InsertProfilingShutdownCall(Function *Callee, Module *Mod) {
     Type::getInt32Ty(Mod->getContext()),
     FunctionType::get(Type::getVoidTy(Mod->getContext()), false)->getPointerTo()
   };
-  const StructType *GlobalDtorElemTy =
+  StructType *GlobalDtorElemTy =
       StructType::get(Mod->getContext(), GlobalDtorElems, false);
 
   // Construct the new element we'll be adding.
diff --git a/contrib/llvm/lib/Transforms/Scalar/ADCE.cpp b/contrib/llvm/lib/Transforms/Scalar/ADCE.cpp
index a5adb5e7cefe..ba214d1a332e 100644
--- a/contrib/llvm/lib/Transforms/Scalar/ADCE.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/ADCE.cpp
@@ -57,6 +57,7 @@ bool ADCE::runOnFunction(Function& F) {
   for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I)
     if (isa<TerminatorInst>(I.getInstructionIterator()) ||
         isa<DbgInfoIntrinsic>(I.getInstructionIterator()) ||
+        isa<LandingPadInst>(I.getInstructionIterator()) ||
         I->mayHaveSideEffects()) {
       alive.insert(I.getInstructionIterator());
       worklist.push_back(I.getInstructionIterator());
@@ -65,7 +66,6 @@ bool ADCE::runOnFunction(Function& F) {
   // Propagate liveness backwards to operands.
   while (!worklist.empty()) {
     Instruction* curr = worklist.pop_back_val();
-    
     for (Instruction::op_iterator OI = curr->op_begin(), OE = curr->op_end();
          OI != OE; ++OI)
       if (Instruction* Inst = dyn_cast<Instruction>(OI))
diff --git a/contrib/llvm/lib/Transforms/Scalar/CodeGenPrepare.cpp b/contrib/llvm/lib/Transforms/Scalar/CodeGenPrepare.cpp
index 0af14ed17bf6..f8f18b217355 100644
--- a/contrib/llvm/lib/Transforms/Scalar/CodeGenPrepare.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/CodeGenPrepare.cpp
@@ -58,6 +58,7 @@ STATISTIC(NumMemoryInsts, "Number of memory instructions whose address "
 STATISTIC(NumExtsMoved,  "Number of [s|z]ext instructions combined with loads");
 STATISTIC(NumExtUses,    "Number of uses of [s|z]ext instructions optimized");
 STATISTIC(NumRetsDup,    "Number of return instructions duplicated");
+STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
 
 static cl::opt<bool> DisableBranchOpts(
   "disable-cgp-branch-opts", cl::Hidden, cl::init(false),
@@ -104,12 +105,13 @@ namespace {
     void EliminateMostlyEmptyBlock(BasicBlock *BB);
     bool OptimizeBlock(BasicBlock &BB);
     bool OptimizeInst(Instruction *I);
-    bool OptimizeMemoryInst(Instruction *I, Value *Addr, const Type *AccessTy);
+    bool OptimizeMemoryInst(Instruction *I, Value *Addr, Type *AccessTy);
     bool OptimizeInlineAsmInst(CallInst *CS);
     bool OptimizeCallInst(CallInst *CI);
     bool MoveExtToFormExtLoad(Instruction *I);
     bool OptimizeExtUses(Instruction *I);
     bool DupRetToEnableTailCallOpts(ReturnInst *RI);
+    bool PlaceDbgValues(Function &F);
   };
 }
 
@@ -132,6 +134,11 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
   // unconditional branch.
   EverMadeChange |= EliminateMostlyEmptyBlocks(F);
 
+  // llvm.dbg.value is far away from the value then iSel may not be able
+  // handle it properly. iSel will drop llvm.dbg.value if it can not 
+  // find a node corresponding to the value.
+  EverMadeChange |= PlaceDbgValues(F);
+
   bool MadeChange = true;
   while (MadeChange) {
     MadeChange = false;
@@ -410,8 +417,7 @@ static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI){
     CastInst *&InsertedCast = InsertedCasts[UserBB];
 
     if (!InsertedCast) {
-      BasicBlock::iterator InsertPt = UserBB->getFirstNonPHI();
-
+      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       InsertedCast =
         CastInst::Create(CI->getOpcode(), CI->getOperand(0), CI->getType(), "",
                          InsertPt);
@@ -467,8 +473,7 @@ static bool OptimizeCmpExpression(CmpInst *CI) {
     CmpInst *&InsertedCmp = InsertedCmps[UserBB];
 
     if (!InsertedCmp) {
-      BasicBlock::iterator InsertPt = UserBB->getFirstNonPHI();
-
+      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       InsertedCmp =
         CmpInst::Create(CI->getOpcode(),
                         CI->getPredicate(),  CI->getOperand(0),
@@ -528,7 +533,7 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
   if (II && II->getIntrinsicID() == Intrinsic::objectsize) {
     bool Min = (cast<ConstantInt>(II->getArgOperand(1))->getZExtValue() == 1);
-    const Type *ReturnTy = CI->getType();
+    Type *ReturnTy = CI->getType();
     Constant *RetVal = ConstantInt::get(ReturnTy, Min ? 0 : -1ULL);    
     
     // Substituting this can cause recursive simplifications, which can
@@ -551,22 +556,6 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
   // From here on out we're working with named functions.
   if (CI->getCalledFunction() == 0) return false;
 
-  // llvm.dbg.value is far away from the value then iSel may not be able
-  // handle it properly. iSel will drop llvm.dbg.value if it can not 
-  // find a node corresponding to the value.
-  if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(CI))
-    if (Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue()))
-      if (!VI->isTerminator() &&
-          (DVI->getParent() != VI->getParent() || DT->dominates(DVI, VI))) {
-        DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI);
-        DVI->removeFromParent();
-        if (isa<PHINode>(VI))
-          DVI->insertBefore(VI->getParent()->getFirstNonPHI());
-        else
-          DVI->insertAfter(VI);
-        return true;
-      }
-
   // We'll need TargetData from here on out.
   const TargetData *TD = TLI ? TLI->getTargetData() : 0;
   if (!TD) return false;
@@ -724,7 +713,7 @@ static bool IsNonLocalValue(Value *V, BasicBlock *BB) {
 /// This method is used to optimize both load/store and inline asms with memory
 /// operands.
 bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
-                                        const Type *AccessTy) {
+                                        Type *AccessTy) {
   Value *Repl = Addr;
   
   // Try to collapse single-value PHI nodes.  This is necessary to undo 
@@ -746,13 +735,11 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     worklist.pop_back();
     
     // Break use-def graph loops.
-    if (Visited.count(V)) {
+    if (!Visited.insert(V)) {
       Consensus = 0;
       break;
     }
     
-    Visited.insert(V);
-    
     // For a PHI node, push all of its incoming values.
     if (PHINode *P = dyn_cast<PHINode>(V)) {
       for (unsigned i = 0, e = P->getNumIncomingValues(); i != e; ++i)
@@ -763,7 +750,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     // For non-PHIs, determine the addressing mode being computed.
     SmallVector<Instruction*, 16> NewAddrModeInsts;
     ExtAddrMode NewAddrMode =
-      AddressingModeMatcher::Match(V, AccessTy,MemoryInst,
+      AddressingModeMatcher::Match(V, AccessTy, MemoryInst,
                                    NewAddrModeInsts, *TLI);
 
     // This check is broken into two cases with very similar code to avoid using
@@ -822,7 +809,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
   // Insert this computation right after this user.  Since our caller is
   // scanning from the top of the BB to the bottom, reuse of the expr are
   // guaranteed to happen later.
-  BasicBlock::iterator InsertPt = MemoryInst;
+  IRBuilder<> Builder(MemoryInst);
 
   // Now that we determined the addressing expression we want to use and know
   // that we have to sink it into this block.  Check to see if we have already
@@ -833,11 +820,11 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for "
                  << *MemoryInst);
     if (SunkAddr->getType() != Addr->getType())
-      SunkAddr = new BitCastInst(SunkAddr, Addr->getType(), "tmp", InsertPt);
+      SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType());
   } else {
     DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
                  << *MemoryInst);
-    const Type *IntPtrTy =
+    Type *IntPtrTy =
           TLI->getTargetData()->getIntPtrType(AccessTy->getContext());
 
     Value *Result = 0;
@@ -850,10 +837,9 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     if (AddrMode.BaseReg) {
       Value *V = AddrMode.BaseReg;
       if (V->getType()->isPointerTy())
-        V = new PtrToIntInst(V, IntPtrTy, "sunkaddr", InsertPt);
+        V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
       if (V->getType() != IntPtrTy)
-        V = CastInst::CreateIntegerCast(V, IntPtrTy, /*isSigned=*/true,
-                                        "sunkaddr", InsertPt);
+        V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
       Result = V;
     }
 
@@ -863,29 +849,27 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
       if (V->getType() == IntPtrTy) {
         // done.
       } else if (V->getType()->isPointerTy()) {
-        V = new PtrToIntInst(V, IntPtrTy, "sunkaddr", InsertPt);
+        V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
       } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
                  cast<IntegerType>(V->getType())->getBitWidth()) {
-        V = new TruncInst(V, IntPtrTy, "sunkaddr", InsertPt);
+        V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
       } else {
-        V = new SExtInst(V, IntPtrTy, "sunkaddr", InsertPt);
+        V = Builder.CreateSExt(V, IntPtrTy, "sunkaddr");
       }
       if (AddrMode.Scale != 1)
-        V = BinaryOperator::CreateMul(V, ConstantInt::get(IntPtrTy,
-                                                                AddrMode.Scale),
-                                      "sunkaddr", InsertPt);
+        V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
+                              "sunkaddr");
       if (Result)
-        Result = BinaryOperator::CreateAdd(Result, V, "sunkaddr", InsertPt);
+        Result = Builder.CreateAdd(Result, V, "sunkaddr");
       else
         Result = V;
     }
 
     // Add in the BaseGV if present.
     if (AddrMode.BaseGV) {
-      Value *V = new PtrToIntInst(AddrMode.BaseGV, IntPtrTy, "sunkaddr",
-                                  InsertPt);
+      Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr");
       if (Result)
-        Result = BinaryOperator::CreateAdd(Result, V, "sunkaddr", InsertPt);
+        Result = Builder.CreateAdd(Result, V, "sunkaddr");
       else
         Result = V;
     }
@@ -894,7 +878,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     if (AddrMode.BaseOffs) {
       Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
       if (Result)
-        Result = BinaryOperator::CreateAdd(Result, V, "sunkaddr", InsertPt);
+        Result = Builder.CreateAdd(Result, V, "sunkaddr");
       else
         Result = V;
     }
@@ -902,7 +886,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     if (Result == 0)
       SunkAddr = Constant::getNullValue(Addr->getType());
     else
-      SunkAddr = new IntToPtrInst(Result, Addr->getType(), "sunkaddr",InsertPt);
+      SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr");
   }
 
   MemoryInst->replaceUsesOfWith(Repl, SunkAddr);
@@ -1059,8 +1043,7 @@ bool CodeGenPrepare::OptimizeExtUses(Instruction *I) {
     Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
 
     if (!InsertedTrunc) {
-      BasicBlock::iterator InsertPt = UserBB->getFirstNonPHI();
-
+      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       InsertedTrunc = new TruncInst(I, Src->getType(), "", InsertPt);
     }
 
@@ -1159,3 +1142,34 @@ bool CodeGenPrepare::OptimizeBlock(BasicBlock &BB) {
 
   return MadeChange;
 }
+
+// llvm.dbg.value is far away from the value then iSel may not be able
+// handle it properly. iSel will drop llvm.dbg.value if it can not 
+// find a node corresponding to the value.
+bool CodeGenPrepare::PlaceDbgValues(Function &F) {
+  bool MadeChange = false;
+  for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
+    Instruction *PrevNonDbgInst = NULL;
+    for (BasicBlock::iterator BI = I->begin(), BE = I->end(); BI != BE;) {
+      Instruction *Insn = BI; ++BI;
+      DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn);
+      if (!DVI) {
+        PrevNonDbgInst = Insn;
+        continue;
+      }
+
+      Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue());
+      if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) {
+        DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI);
+        DVI->removeFromParent();
+        if (isa<PHINode>(VI))
+          DVI->insertBefore(VI->getParent()->getFirstInsertionPt());
+        else
+          DVI->insertAfter(VI);
+        MadeChange = true;
+        ++NumDbgValueMoved;
+      }
+    }
+  }
+  return MadeChange;
+}
diff --git a/contrib/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/contrib/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index cb9b5bebc5c7..a593d0f44633 100644
--- a/contrib/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -52,18 +52,18 @@ namespace {
       AA = &getAnalysis<AliasAnalysis>();
       MD = &getAnalysis<MemoryDependenceAnalysis>();
       DominatorTree &DT = getAnalysis<DominatorTree>();
-      
+
       bool Changed = false;
       for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
         // Only check non-dead blocks.  Dead blocks may have strange pointer
         // cycles that will confuse alias analysis.
         if (DT.isReachableFromEntry(I))
           Changed |= runOnBasicBlock(*I);
-      
+
       AA = 0; MD = 0;
       return Changed;
     }
-    
+
     bool runOnBasicBlock(BasicBlock &BB);
     bool HandleFree(CallInst *F);
     bool handleEndBlock(BasicBlock &BB);
@@ -105,34 +105,34 @@ static void DeleteDeadInstruction(Instruction *I,
                                   MemoryDependenceAnalysis &MD,
                                   SmallPtrSet<Value*, 16> *ValueSet = 0) {
   SmallVector<Instruction*, 32> NowDeadInsts;
-  
+
   NowDeadInsts.push_back(I);
   --NumFastOther;
-  
+
   // Before we touch this instruction, remove it from memdep!
   do {
     Instruction *DeadInst = NowDeadInsts.pop_back_val();
     ++NumFastOther;
-    
+
     // This instruction is dead, zap it, in stages.  Start by removing it from
     // MemDep, which needs to know the operands and needs it to be in the
     // function.
     MD.removeInstruction(DeadInst);
-    
+
     for (unsigned op = 0, e = DeadInst->getNumOperands(); op != e; ++op) {
       Value *Op = DeadInst->getOperand(op);
       DeadInst->setOperand(op, 0);
-      
+
       // If this operand just became dead, add it to the NowDeadInsts list.
       if (!Op->use_empty()) continue;
-      
+
       if (Instruction *OpI = dyn_cast<Instruction>(Op))
         if (isInstructionTriviallyDead(OpI))
           NowDeadInsts.push_back(OpI);
     }
-    
+
     DeadInst->eraseFromParent();
-    
+
     if (ValueSet) ValueSet->erase(DeadInst);
   } while (!NowDeadInsts.empty());
 }
@@ -159,11 +159,13 @@ static bool hasMemoryWrite(Instruction *I) {
 }
 
 /// getLocForWrite - Return a Location stored to by the specified instruction.
+/// If isRemovable returns true, this function and getLocForRead completely
+/// describe the memory operations for this instruction.
 static AliasAnalysis::Location
 getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
   if (StoreInst *SI = dyn_cast<StoreInst>(Inst))
     return AA.getLocation(SI);
-  
+
   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Inst)) {
     // memcpy/memmove/memset.
     AliasAnalysis::Location Loc = AA.getLocationForDest(MI);
@@ -174,10 +176,10 @@ getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
       return AliasAnalysis::Location();
     return Loc;
   }
-  
+
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst);
   if (II == 0) return AliasAnalysis::Location();
-  
+
   switch (II->getIntrinsicID()) {
   default: return AliasAnalysis::Location(); // Unhandled intrinsic.
   case Intrinsic::init_trampoline:
@@ -185,7 +187,7 @@ getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
     // that we should use the size of the pointee type.  This isn't valid for
     // init.trampoline, which writes more than an i8.
     if (AA.getTargetData() == 0) return AliasAnalysis::Location();
-      
+
     // FIXME: We don't know the size of the trampoline, so we can't really
     // handle it here.
     return AliasAnalysis::Location(II->getArgOperand(0));
@@ -198,10 +200,10 @@ getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
 
 /// getLocForRead - Return the location read by the specified "hasMemoryWrite"
 /// instruction if any.
-static AliasAnalysis::Location 
+static AliasAnalysis::Location
 getLocForRead(Instruction *Inst, AliasAnalysis &AA) {
   assert(hasMemoryWrite(Inst) && "Unknown instruction case");
-  
+
   // The only instructions that both read and write are the mem transfer
   // instructions (memcpy/memmove).
   if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(Inst))
@@ -213,10 +215,10 @@ getLocForRead(Instruction *Inst, AliasAnalysis &AA) {
 /// isRemovable - If the value of this instruction and the memory it writes to
 /// is unused, may we delete this instruction?
 static bool isRemovable(Instruction *I) {
-  // Don't remove volatile stores.
+  // Don't remove volatile/atomic stores.
   if (StoreInst *SI = dyn_cast<StoreInst>(I))
-    return !SI->isVolatile();
-  
+    return SI->isUnordered();
+
   IntrinsicInst *II = cast<IntrinsicInst>(I);
   switch (II->getIntrinsicID()) {
   default: assert(0 && "doesn't pass 'hasMemoryWrite' predicate");
@@ -227,7 +229,7 @@ static bool isRemovable(Instruction *I) {
   case Intrinsic::init_trampoline:
     // Always safe to remove init_trampoline.
     return true;
-    
+
   case Intrinsic::memset:
   case Intrinsic::memmove:
   case Intrinsic::memcpy:
@@ -255,16 +257,16 @@ static uint64_t getPointerSize(Value *V, AliasAnalysis &AA) {
   const TargetData *TD = AA.getTargetData();
   if (TD == 0)
     return AliasAnalysis::UnknownSize;
-  
+
   if (AllocaInst *A = dyn_cast<AllocaInst>(V)) {
     // Get size information for the alloca
     if (ConstantInt *C = dyn_cast<ConstantInt>(A->getArraySize()))
       return C->getZExtValue() * TD->getTypeAllocSize(A->getAllocatedType());
     return AliasAnalysis::UnknownSize;
   }
-  
+
   assert(isa<Argument>(V) && "Expected AllocaInst or Argument!");
-  const PointerType *PT = cast<PointerType>(V->getType());
+  PointerType *PT = cast<PointerType>(V->getType());
   return TD->getTypeAllocSize(PT->getElementType());
 }
 
@@ -287,7 +289,7 @@ static bool isCompleteOverwrite(const AliasAnalysis::Location &Later,
                                 AliasAnalysis &AA) {
   const Value *P1 = Earlier.Ptr->stripPointerCasts();
   const Value *P2 = Later.Ptr->stripPointerCasts();
-  
+
   // If the start pointers are the same, we just have to compare sizes to see if
   // the later store was larger than the earlier store.
   if (P1 == P2) {
@@ -302,33 +304,33 @@ static bool isCompleteOverwrite(const AliasAnalysis::Location &Later,
         return Later.Ptr->getType() == Earlier.Ptr->getType();
       return false;
     }
-    
+
     // Make sure that the Later size is >= the Earlier size.
     if (Later.Size < Earlier.Size)
       return false;
     return true;
   }
-  
+
   // Otherwise, we have to have size information, and the later store has to be
   // larger than the earlier one.
   if (Later.Size == AliasAnalysis::UnknownSize ||
       Earlier.Size == AliasAnalysis::UnknownSize ||
       Later.Size <= Earlier.Size || AA.getTargetData() == 0)
     return false;
-  
+
   // Check to see if the later store is to the entire object (either a global,
   // an alloca, or a byval argument).  If so, then it clearly overwrites any
   // other store to the same object.
   const TargetData &TD = *AA.getTargetData();
-  
+
   const Value *UO1 = GetUnderlyingObject(P1, &TD),
               *UO2 = GetUnderlyingObject(P2, &TD);
-  
+
   // If we can't resolve the same pointers to the same object, then we can't
   // analyze them at all.
   if (UO1 != UO2)
     return false;
-  
+
   // If the "Later" store is to a recognizable object, get its size.
   if (isObjectPointerWithTrustworthySize(UO2)) {
     uint64_t ObjectSize =
@@ -336,26 +338,26 @@ static bool isCompleteOverwrite(const AliasAnalysis::Location &Later,
     if (ObjectSize == Later.Size)
       return true;
   }
-  
+
   // Okay, we have stores to two completely different pointers.  Try to
   // decompose the pointer into a "base + constant_offset" form.  If the base
   // pointers are equal, then we can reason about the two stores.
   int64_t EarlierOff = 0, LaterOff = 0;
   const Value *BP1 = GetPointerBaseWithConstantOffset(P1, EarlierOff, TD);
   const Value *BP2 = GetPointerBaseWithConstantOffset(P2, LaterOff, TD);
-  
+
   // If the base pointers still differ, we have two completely different stores.
   if (BP1 != BP2)
     return false;
 
   // The later store completely overlaps the earlier store if:
-  // 
+  //
   // 1. Both start at the same offset and the later one's size is greater than
   //    or equal to the earlier one's, or
   //
   //      |--earlier--|
   //      |--   later   --|
-  //      
+  //
   // 2. The earlier store has an offset greater than the later offset, but which
   //    still lies completely within the later store.
   //
@@ -373,7 +375,7 @@ static bool isCompleteOverwrite(const AliasAnalysis::Location &Later,
 
 /// isPossibleSelfRead - If 'Inst' might be a self read (i.e. a noop copy of a
 /// memory region into an identical pointer) then it doesn't actually make its
-/// input dead in the traditional sense.  Consider this case: 
+/// input dead in the traditional sense.  Consider this case:
 ///
 ///   memcpy(A <- B)
 ///   memcpy(A <- A)
@@ -391,10 +393,10 @@ static bool isPossibleSelfRead(Instruction *Inst,
   // location read.
   AliasAnalysis::Location InstReadLoc = getLocForRead(Inst, AA);
   if (InstReadLoc.Ptr == 0) return false;  // Not a reading instruction.
-  
+
   // If the read and written loc obviously don't alias, it isn't a read.
   if (AA.isNoAlias(InstReadLoc, InstStoreLoc)) return false;
-  
+
   // Okay, 'Inst' may copy over itself.  However, we can still remove a the
   // DepWrite instruction if we can prove that it reads from the same location
   // as Inst.  This handles useful cases like:
@@ -404,10 +406,10 @@ static bool isPossibleSelfRead(Instruction *Inst,
   // aliases, so removing the first memcpy is safe (assuming it writes <= #
   // bytes as the second one.
   AliasAnalysis::Location DepReadLoc = getLocForRead(DepWrite, AA);
-  
+
   if (DepReadLoc.Ptr && AA.isMustAlias(InstReadLoc.Ptr, DepReadLoc.Ptr))
     return false;
-  
+
   // If DepWrite doesn't read memory or if we can't prove it is a must alias,
   // then it can't be considered dead.
   return true;
@@ -420,43 +422,43 @@ static bool isPossibleSelfRead(Instruction *Inst,
 
 bool DSE::runOnBasicBlock(BasicBlock &BB) {
   bool MadeChange = false;
-  
+
   // Do a top-down walk on the BB.
   for (BasicBlock::iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE; ) {
     Instruction *Inst = BBI++;
-    
+
     // Handle 'free' calls specially.
     if (CallInst *F = isFreeCall(Inst)) {
       MadeChange |= HandleFree(F);
       continue;
     }
-    
+
     // If we find something that writes memory, get its memory dependence.
     if (!hasMemoryWrite(Inst))
       continue;
 
     MemDepResult InstDep = MD->getDependency(Inst);
-    
+
     // Ignore any store where we can't find a local dependence.
     // FIXME: cross-block DSE would be fun. :)
-    if (InstDep.isNonLocal() || InstDep.isUnknown())
+    if (!InstDep.isDef() && !InstDep.isClobber())
       continue;
-     
+
     // If we're storing the same value back to a pointer that we just
     // loaded from, then the store can be removed.
     if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
       if (LoadInst *DepLoad = dyn_cast<LoadInst>(InstDep.getInst())) {
         if (SI->getPointerOperand() == DepLoad->getPointerOperand() &&
-            SI->getOperand(0) == DepLoad && !SI->isVolatile()) {
+            SI->getOperand(0) == DepLoad && isRemovable(SI)) {
           DEBUG(dbgs() << "DSE: Remove Store Of Load from same pointer:\n  "
                        << "LOAD: " << *DepLoad << "\n  STORE: " << *SI << '\n');
-          
+
           // DeleteDeadInstruction can delete the current instruction.  Save BBI
           // in case we need it.
           WeakVH NextInst(BBI);
-          
+
           DeleteDeadInstruction(SI, *MD);
-          
+
           if (NextInst == 0)  // Next instruction deleted.
             BBI = BB.begin();
           else if (BBI != BB.begin())  // Revisit this instruction if possible.
@@ -467,15 +469,15 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
         }
       }
     }
-    
+
     // Figure out what location is being stored to.
     AliasAnalysis::Location Loc = getLocForWrite(Inst, *AA);
 
     // If we didn't get a useful location, fail.
     if (Loc.Ptr == 0)
       continue;
-    
-    while (!InstDep.isNonLocal() && !InstDep.isUnknown()) {
+
+    while (InstDep.isDef() || InstDep.isClobber()) {
       // Get the memory clobbered by the instruction we depend on.  MemDep will
       // skip any instructions that 'Loc' clearly doesn't interact with.  If we
       // end up depending on a may- or must-aliased load, then we can't optimize
@@ -496,12 +498,12 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
           !isPossibleSelfRead(Inst, Loc, DepWrite, *AA)) {
         DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: "
               << *DepWrite << "\n  KILLER: " << *Inst << '\n');
-        
+
         // Delete the store and now-dead instructions that feed it.
         DeleteDeadInstruction(DepWrite, *MD);
         ++NumFastStores;
         MadeChange = true;
-        
+
         // DeleteDeadInstruction can delete the current instruction in loop
         // cases, reset BBI.
         BBI = Inst;
@@ -509,7 +511,7 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
           --BBI;
         break;
       }
-      
+
       // If this is a may-aliased store that is clobbering the store value, we
       // can keep searching past it for another must-aliased pointer that stores
       // to the same location.  For example, in:
@@ -519,20 +521,20 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
       // we can remove the first store to P even though we don't know if P and Q
       // alias.
       if (DepWrite == &BB.front()) break;
-      
+
       // Can't look past this instruction if it might read 'Loc'.
       if (AA->getModRefInfo(DepWrite, Loc) & AliasAnalysis::Ref)
         break;
-        
+
       InstDep = MD->getPointerDependencyFrom(Loc, false, DepWrite, &BB);
     }
   }
-  
+
   // If this block ends in a return, unwind, or unreachable, all allocas are
   // dead at its end, which means stores to them are also dead.
   if (BB.getTerminator()->getNumSuccessors() == 0)
     MadeChange |= handleEndBlock(BB);
-  
+
   return MadeChange;
 }
 
@@ -543,18 +545,18 @@ bool DSE::HandleFree(CallInst *F) {
 
   MemDepResult Dep = MD->getDependency(F);
 
-  while (!Dep.isNonLocal() && !Dep.isUnknown()) {
+  while (Dep.isDef() || Dep.isClobber()) {
     Instruction *Dependency = Dep.getInst();
     if (!hasMemoryWrite(Dependency) || !isRemovable(Dependency))
       return MadeChange;
-  
+
     Value *DepPointer =
       GetUnderlyingObject(getStoredPointerOperand(Dependency));
 
     // Check for aliasing.
     if (!AA->isMustAlias(F->getArgOperand(0), DepPointer))
       return MadeChange;
-  
+
     // DCE instructions only used to calculate that store
     DeleteDeadInstruction(Dependency, *MD);
     ++NumFastStores;
@@ -567,7 +569,7 @@ bool DSE::HandleFree(CallInst *F) {
     //    free(s);
     Dep = MD->getDependency(F);
   };
-  
+
   return MadeChange;
 }
 
@@ -579,28 +581,28 @@ bool DSE::HandleFree(CallInst *F) {
 /// ret void
 bool DSE::handleEndBlock(BasicBlock &BB) {
   bool MadeChange = false;
-  
+
   // Keep track of all of the stack objects that are dead at the end of the
   // function.
   SmallPtrSet<Value*, 16> DeadStackObjects;
-  
+
   // Find all of the alloca'd pointers in the entry block.
   BasicBlock *Entry = BB.getParent()->begin();
   for (BasicBlock::iterator I = Entry->begin(), E = Entry->end(); I != E; ++I)
     if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
       DeadStackObjects.insert(AI);
-  
+
   // Treat byval arguments the same, stores to them are dead at the end of the
   // function.
   for (Function::arg_iterator AI = BB.getParent()->arg_begin(),
        AE = BB.getParent()->arg_end(); AI != AE; ++AI)
     if (AI->hasByValAttr())
       DeadStackObjects.insert(AI);
-  
+
   // Scan the basic block backwards
   for (BasicBlock::iterator BBI = BB.end(); BBI != BB.begin(); ){
     --BBI;
-    
+
     // If we find a store, check to see if it points into a dead stack value.
     if (hasMemoryWrite(BBI) && isRemovable(BBI)) {
       // See through pointer-to-pointer bitcasts
@@ -609,10 +611,10 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
       // Stores to stack values are valid candidates for removal.
       if (DeadStackObjects.count(Pointer)) {
         Instruction *Dead = BBI++;
-        
+
         DEBUG(dbgs() << "DSE: Dead Store at End of Block:\n  DEAD: "
                      << *Dead << "\n  Object: " << *Pointer << '\n');
-        
+
         // DCE instructions only used to calculate that store.
         DeleteDeadInstruction(Dead, *MD, &DeadStackObjects);
         ++NumFastStores;
@@ -620,7 +622,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
         continue;
       }
     }
-    
+
     // Remove any dead non-memory-mutating instructions.
     if (isInstructionTriviallyDead(BBI)) {
       Instruction *Inst = BBI++;
@@ -629,55 +631,61 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
       MadeChange = true;
       continue;
     }
-    
+
     if (AllocaInst *A = dyn_cast<AllocaInst>(BBI)) {
       DeadStackObjects.erase(A);
       continue;
     }
-    
+
     if (CallSite CS = cast<Value>(BBI)) {
       // If this call does not access memory, it can't be loading any of our
       // pointers.
       if (AA->doesNotAccessMemory(CS))
         continue;
-      
+
       // If the call might load from any of our allocas, then any store above
       // the call is live.
       SmallVector<Value*, 8> LiveAllocas;
       for (SmallPtrSet<Value*, 16>::iterator I = DeadStackObjects.begin(),
            E = DeadStackObjects.end(); I != E; ++I) {
         // See if the call site touches it.
-        AliasAnalysis::ModRefResult A = 
+        AliasAnalysis::ModRefResult A =
           AA->getModRefInfo(CS, *I, getPointerSize(*I, *AA));
-        
+
         if (A == AliasAnalysis::ModRef || A == AliasAnalysis::Ref)
           LiveAllocas.push_back(*I);
       }
-      
+
       for (SmallVector<Value*, 8>::iterator I = LiveAllocas.begin(),
            E = LiveAllocas.end(); I != E; ++I)
         DeadStackObjects.erase(*I);
-      
+
       // If all of the allocas were clobbered by the call then we're not going
       // to find anything else to process.
       if (DeadStackObjects.empty())
         return MadeChange;
-      
+
       continue;
     }
-    
+
     AliasAnalysis::Location LoadedLoc;
-    
+
     // If we encounter a use of the pointer, it is no longer considered dead
     if (LoadInst *L = dyn_cast<LoadInst>(BBI)) {
+      if (!L->isUnordered()) // Be conservative with atomic/volatile load
+        break;
       LoadedLoc = AA->getLocation(L);
     } else if (VAArgInst *V = dyn_cast<VAArgInst>(BBI)) {
       LoadedLoc = AA->getLocation(V);
     } else if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(BBI)) {
       LoadedLoc = AA->getLocationForSource(MTI);
-    } else {
-      // Not a loading instruction.
+    } else if (!BBI->mayReadFromMemory()) {
+      // Instruction doesn't read memory.  Note that stores that weren't removed
+      // above will hit this case.
       continue;
+    } else {
+      // Unknown inst; assume it clobbers everything.
+      break;
     }
 
     // Remove any allocas from the DeadPointer set that are loaded, as this
@@ -689,7 +697,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
     if (DeadStackObjects.empty())
       break;
   }
-  
+
   return MadeChange;
 }
 
@@ -703,14 +711,14 @@ void DSE::RemoveAccessedObjects(const AliasAnalysis::Location &LoadedLoc,
   // A constant can't be in the dead pointer set.
   if (isa<Constant>(UnderlyingPointer))
     return;
-  
+
   // If the kill pointer can be easily reduced to an alloca, don't bother doing
   // extraneous AA queries.
   if (isa<AllocaInst>(UnderlyingPointer) || isa<Argument>(UnderlyingPointer)) {
     DeadStackObjects.erase(const_cast<Value*>(UnderlyingPointer));
     return;
   }
-  
+
   SmallVector<Value*, 16> NowLive;
   for (SmallPtrSet<Value*, 16>::iterator I = DeadStackObjects.begin(),
        E = DeadStackObjects.end(); I != E; ++I) {
diff --git a/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 3d3f17b26fc6..c0223d2bf199 100644
--- a/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -92,7 +92,7 @@ unsigned DenseMapInfo<SimpleValue>::getHashValue(SimpleValue Val) {
   // Hash in all of the operands as pointers.
   unsigned Res = 0;
   for (unsigned i = 0, e = Inst->getNumOperands(); i != e; ++i)
-    Res ^= getHash(Inst->getOperand(i)) << i;
+    Res ^= getHash(Inst->getOperand(i)) << (i & 0xF);
 
   if (CastInst *CI = dyn_cast<CastInst>(Inst))
     Res ^= getHash(CI->getType());
@@ -185,7 +185,7 @@ unsigned DenseMapInfo<CallValue>::getHashValue(CallValue Val) {
   for (unsigned i = 0, e = Inst->getNumOperands(); i != e; ++i) {
     assert(!Inst->getOperand(i)->getType()->isMetadataTy() &&
            "Cannot value number calls with metadata operands");
-    Res ^= getHash(Inst->getOperand(i)) << i;
+    Res ^= getHash(Inst->getOperand(i)) << (i & 0xF);
   }
   
   // Mix in the opcode.
@@ -357,7 +357,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
     // If this is a non-volatile load, process it.
     if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
       // Ignore volatile loads.
-      if (LI->isVolatile()) {
+      if (!LI->isSimple()) {
         LastStore = 0;
         continue;
       }
@@ -437,7 +437,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
          std::pair<Value*, unsigned>(SI->getValueOperand(), CurrentGeneration));
         
         // Remember that this was the last store we saw for DSE.
-        if (!SI->isVolatile())
+        if (SI->isSimple())
           LastStore = SI;
       }
     }
diff --git a/contrib/llvm/lib/Transforms/Scalar/GVN.cpp b/contrib/llvm/lib/Transforms/Scalar/GVN.cpp
index 87b7317ad2dd..cbfdbcddaeca 100644
--- a/contrib/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -41,12 +41,16 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/IRBuilder.h"
+#include "llvm/Support/PatternMatch.h"
 using namespace llvm;
+using namespace PatternMatch;
 
 STATISTIC(NumGVNInstr,  "Number of instructions deleted");
 STATISTIC(NumGVNLoad,   "Number of loads deleted");
 STATISTIC(NumGVNPRE,    "Number of instructions PRE'd");
 STATISTIC(NumGVNBlocks, "Number of blocks merged");
+STATISTIC(NumGVNSimpl,  "Number of instructions simplified");
+STATISTIC(NumGVNEqProp, "Number of equalities propagated");
 STATISTIC(NumPRELoad,   "Number of loads PRE'd");
 
 static cl::opt<bool> EnablePRE("enable-pre",
@@ -63,7 +67,7 @@ static cl::opt<bool> EnableLoadPRE("enable-load-pre", cl::init(true));
 namespace {
   struct Expression {
     uint32_t opcode;
-    const Type *type;
+    Type *type;
     SmallVector<uint32_t, 4> varargs;
 
     Expression(uint32_t o = ~2U) : opcode(o) { }
@@ -548,6 +552,9 @@ namespace {
     void cleanupGlobalSets();
     void verifyRemoved(const Instruction *I) const;
     bool splitCriticalEdges();
+    unsigned replaceAllDominatedUsesWith(Value *From, Value *To,
+                                         BasicBlock *Root);
+    bool propagateEquality(Value *LHS, Value *RHS, BasicBlock *Root);
   };
 
   char GVN::ID = 0;
@@ -655,7 +662,7 @@ static bool IsValueFullyAvailableInBlock(BasicBlock *BB,
 /// CanCoerceMustAliasedValueToLoad - Return true if
 /// CoerceAvailableValueToLoadType will succeed.
 static bool CanCoerceMustAliasedValueToLoad(Value *StoredVal,
-                                            const Type *LoadTy,
+                                            Type *LoadTy,
                                             const TargetData &TD) {
   // If the loaded or stored value is an first class array or struct, don't try
   // to transform them.  We need to be able to bitcast to integer.
@@ -680,17 +687,17 @@ static bool CanCoerceMustAliasedValueToLoad(Value *StoredVal,
 ///
 /// If we can't do it, return null.
 static Value *CoerceAvailableValueToLoadType(Value *StoredVal, 
-                                             const Type *LoadedTy,
+                                             Type *LoadedTy,
                                              Instruction *InsertPt,
                                              const TargetData &TD) {
   if (!CanCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, TD))
     return 0;
   
   // If this is already the right type, just return it.
-  const Type *StoredValTy = StoredVal->getType();
+  Type *StoredValTy = StoredVal->getType();
   
-  uint64_t StoreSize = TD.getTypeStoreSizeInBits(StoredValTy);
-  uint64_t LoadSize = TD.getTypeStoreSizeInBits(LoadedTy);
+  uint64_t StoreSize = TD.getTypeSizeInBits(StoredValTy);
+  uint64_t LoadSize = TD.getTypeSizeInBits(LoadedTy);
   
   // If the store and reload are the same size, we can always reuse it.
   if (StoreSize == LoadSize) {
@@ -704,7 +711,7 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
       StoredVal = new PtrToIntInst(StoredVal, StoredValTy, "", InsertPt);
     }
     
-    const Type *TypeToCastTo = LoadedTy;
+    Type *TypeToCastTo = LoadedTy;
     if (TypeToCastTo->isPointerTy())
       TypeToCastTo = TD.getIntPtrType(StoredValTy->getContext());
     
@@ -743,7 +750,7 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
   }
   
   // Truncate the integer to the right size now.
-  const Type *NewIntTy = IntegerType::get(StoredValTy->getContext(), LoadSize);
+  Type *NewIntTy = IntegerType::get(StoredValTy->getContext(), LoadSize);
   StoredVal = new TruncInst(StoredVal, NewIntTy, "trunc", InsertPt);
   
   if (LoadedTy == NewIntTy)
@@ -765,7 +772,7 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
 /// Check this case to see if there is anything more we can do before we give
 /// up.  This returns -1 if we have to give up, or a byte number in the stored
 /// value of the piece that feeds the load.
-static int AnalyzeLoadFromClobberingWrite(const Type *LoadTy, Value *LoadPtr,
+static int AnalyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr,
                                           Value *WritePtr,
                                           uint64_t WriteSizeInBits,
                                           const TargetData &TD) {
@@ -839,7 +846,7 @@ static int AnalyzeLoadFromClobberingWrite(const Type *LoadTy, Value *LoadPtr,
 
 /// AnalyzeLoadFromClobberingStore - This function is called when we have a
 /// memdep query of a load that ends up being a clobbering store.
-static int AnalyzeLoadFromClobberingStore(const Type *LoadTy, Value *LoadPtr,
+static int AnalyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr,
                                           StoreInst *DepSI,
                                           const TargetData &TD) {
   // Cannot handle reading from store of first-class aggregate yet.
@@ -856,7 +863,7 @@ static int AnalyzeLoadFromClobberingStore(const Type *LoadTy, Value *LoadPtr,
 /// AnalyzeLoadFromClobberingLoad - This function is called when we have a
 /// memdep query of a load that ends up being clobbered by another load.  See if
 /// the other load can feed into the second load.
-static int AnalyzeLoadFromClobberingLoad(const Type *LoadTy, Value *LoadPtr,
+static int AnalyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr,
                                          LoadInst *DepLI, const TargetData &TD){
   // Cannot handle reading from store of first-class aggregate yet.
   if (DepLI->getType()->isStructTy() || DepLI->getType()->isArrayTy())
@@ -883,7 +890,7 @@ static int AnalyzeLoadFromClobberingLoad(const Type *LoadTy, Value *LoadPtr,
 
 
 
-static int AnalyzeLoadFromClobberingMemInst(const Type *LoadTy, Value *LoadPtr,
+static int AnalyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
                                             MemIntrinsic *MI,
                                             const TargetData &TD) {
   // If the mem operation is a non-constant size, we can't handle it.
@@ -920,7 +927,7 @@ static int AnalyzeLoadFromClobberingMemInst(const Type *LoadTy, Value *LoadPtr,
                                  llvm::Type::getInt8PtrTy(Src->getContext()));
   Constant *OffsetCst = 
     ConstantInt::get(Type::getInt64Ty(Src->getContext()), (unsigned)Offset);
-  Src = ConstantExpr::getGetElementPtr(Src, &OffsetCst, 1);
+  Src = ConstantExpr::getGetElementPtr(Src, OffsetCst);
   Src = ConstantExpr::getBitCast(Src, PointerType::getUnqual(LoadTy));
   if (ConstantFoldLoadFromConstPtr(Src, &TD))
     return Offset;
@@ -934,7 +941,7 @@ static int AnalyzeLoadFromClobberingMemInst(const Type *LoadTy, Value *LoadPtr,
 /// mustalias.  Check this case to see if there is anything more we can do
 /// before we give up.
 static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
-                                   const Type *LoadTy,
+                                   Type *LoadTy,
                                    Instruction *InsertPt, const TargetData &TD){
   LLVMContext &Ctx = SrcVal->getType()->getContext();
   
@@ -946,10 +953,9 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
   // Compute which bits of the stored value are being used by the load.  Convert
   // to an integer type to start with.
   if (SrcVal->getType()->isPointerTy())
-    SrcVal = Builder.CreatePtrToInt(SrcVal, TD.getIntPtrType(Ctx), "tmp");
+    SrcVal = Builder.CreatePtrToInt(SrcVal, TD.getIntPtrType(Ctx));
   if (!SrcVal->getType()->isIntegerTy())
-    SrcVal = Builder.CreateBitCast(SrcVal, IntegerType::get(Ctx, StoreSize*8),
-                                   "tmp");
+    SrcVal = Builder.CreateBitCast(SrcVal, IntegerType::get(Ctx, StoreSize*8));
   
   // Shift the bits to the least significant depending on endianness.
   unsigned ShiftAmt;
@@ -959,11 +965,10 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
     ShiftAmt = (StoreSize-LoadSize-Offset)*8;
   
   if (ShiftAmt)
-    SrcVal = Builder.CreateLShr(SrcVal, ShiftAmt, "tmp");
+    SrcVal = Builder.CreateLShr(SrcVal, ShiftAmt);
   
   if (LoadSize != StoreSize)
-    SrcVal = Builder.CreateTrunc(SrcVal, IntegerType::get(Ctx, LoadSize*8),
-                                 "tmp");
+    SrcVal = Builder.CreateTrunc(SrcVal, IntegerType::get(Ctx, LoadSize*8));
   
   return CoerceAvailableValueToLoadType(SrcVal, LoadTy, InsertPt, TD);
 }
@@ -974,7 +979,7 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
 /// because the pointers don't mustalias.  Check this case to see if there is
 /// anything more we can do before we give up.
 static Value *GetLoadValueForLoad(LoadInst *SrcVal, unsigned Offset,
-                                  const Type *LoadTy, Instruction *InsertPt,
+                                  Type *LoadTy, Instruction *InsertPt,
                                   GVN &gvn) {
   const TargetData &TD = *gvn.getTargetData();
   // If Offset+LoadTy exceeds the size of SrcVal, then we must be wanting to
@@ -982,8 +987,8 @@ static Value *GetLoadValueForLoad(LoadInst *SrcVal, unsigned Offset,
   unsigned SrcValSize = TD.getTypeStoreSize(SrcVal->getType());
   unsigned LoadSize = TD.getTypeStoreSize(LoadTy);
   if (Offset+LoadSize > SrcValSize) {
-    assert(!SrcVal->isVolatile() && "Cannot widen volatile load!");
-    assert(isa<IntegerType>(SrcVal->getType())&&"Can't widen non-integer load");
+    assert(SrcVal->isSimple() && "Cannot widen volatile/atomic load!");
+    assert(SrcVal->getType()->isIntegerTy() && "Can't widen non-integer load");
     // If we have a load/load clobber an DepLI can be widened to cover this
     // load, then we should widen it to the next power of 2 size big enough!
     unsigned NewLoadSize = Offset+LoadSize;
@@ -996,7 +1001,7 @@ static Value *GetLoadValueForLoad(LoadInst *SrcVal, unsigned Offset,
     // memdep queries will find the new load.  We can't easily remove the old
     // load completely because it is already in the value numbering table.
     IRBuilder<> Builder(SrcVal->getParent(), ++BasicBlock::iterator(SrcVal));
-    const Type *DestPTy = 
+    Type *DestPTy = 
       IntegerType::get(LoadTy->getContext(), NewLoadSize*8);
     DestPTy = PointerType::get(DestPTy, 
                        cast<PointerType>(PtrVal->getType())->getAddressSpace());
@@ -1034,7 +1039,7 @@ static Value *GetLoadValueForLoad(LoadInst *SrcVal, unsigned Offset,
 /// GetMemInstValueForLoad - This function is called when we have a
 /// memdep query of a load that ends up being a clobbering mem intrinsic.
 static Value *GetMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
-                                     const Type *LoadTy, Instruction *InsertPt,
+                                     Type *LoadTy, Instruction *InsertPt,
                                      const TargetData &TD){
   LLVMContext &Ctx = LoadTy->getContext();
   uint64_t LoadSize = TD.getTypeSizeInBits(LoadTy)/8;
@@ -1081,7 +1086,7 @@ static Value *GetMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
                                  llvm::Type::getInt8PtrTy(Src->getContext()));
   Constant *OffsetCst = 
   ConstantInt::get(Type::getInt64Ty(Src->getContext()), (unsigned)Offset);
-  Src = ConstantExpr::getGetElementPtr(Src, &OffsetCst, 1);
+  Src = ConstantExpr::getGetElementPtr(Src, OffsetCst);
   Src = ConstantExpr::getBitCast(Src, PointerType::getUnqual(LoadTy));
   return ConstantFoldLoadFromConstPtr(Src, &TD);
 }
@@ -1154,7 +1159,7 @@ struct AvailableValueInBlock {
   
   /// MaterializeAdjustedValue - Emit code into this block to adjust the value
   /// defined here to the specified type.  This handles various coercion cases.
-  Value *MaterializeAdjustedValue(const Type *LoadTy, GVN &gvn) const {
+  Value *MaterializeAdjustedValue(Type *LoadTy, GVN &gvn) const {
     Value *Res;
     if (isSimpleValue()) {
       Res = getSimpleValue();
@@ -1213,7 +1218,7 @@ static Value *ConstructSSAForLoadSet(LoadInst *LI,
   SSAUpdater SSAUpdate(&NewPHIs);
   SSAUpdate.Initialize(LI->getType(), LI->getName());
   
-  const Type *LoadTy = LI->getType();
+  Type *LoadTy = LI->getType();
   
   for (unsigned i = 0, e = ValuesPerBlock.size(); i != e; ++i) {
     const AvailableValueInBlock &AV = ValuesPerBlock[i];
@@ -1274,7 +1279,9 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
 
   // If we had a phi translation failure, we'll have a single entry which is a
   // clobber in the current block.  Reject this early.
-  if (Deps.size() == 1 && Deps[0].getResult().isUnknown()) {
+  if (Deps.size() == 1
+      && !Deps[0].getResult().isDef() && !Deps[0].getResult().isClobber())
+  {
     DEBUG(
       dbgs() << "GVN: non-local load ";
       WriteAsOperand(dbgs(), LI);
@@ -1294,7 +1301,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
     BasicBlock *DepBB = Deps[i].getBB();
     MemDepResult DepInfo = Deps[i].getResult();
 
-    if (DepInfo.isUnknown()) {
+    if (!DepInfo.isDef() && !DepInfo.isClobber()) {
       UnavailableBlocks.push_back(DepBB);
       continue;
     }
@@ -1359,7 +1366,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
       continue;
     }
 
-    assert(DepInfo.isDef() && "Expecting def here");
+    // DepInfo.isDef() here
 
     Instruction *DepInst = DepInfo.getInst();
 
@@ -1446,8 +1453,8 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
   for (unsigned i = 0, e = UnavailableBlocks.size(); i != e; ++i)
     Blockers.insert(UnavailableBlocks[i]);
 
-  // Lets find first basic block with more than one predecessor.  Walk backwards
-  // through predecessors if needed.
+  // Let's find the first basic block with more than one predecessor.  Walk
+  // backwards through predecessors if needed.
   BasicBlock *LoadBB = LI->getParent();
   BasicBlock *TmpBB = LoadBB;
 
@@ -1519,10 +1526,19 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
               << Pred->getName() << "': " << *LI << '\n');
         return false;
       }
+
+      if (LoadBB->isLandingPad()) {
+        DEBUG(dbgs()
+              << "COULD NOT PRE LOAD BECAUSE OF LANDING PAD CRITICAL EDGE '"
+              << Pred->getName() << "': " << *LI << '\n');
+        return false;
+      }
+
       unsigned SuccNum = GetSuccessorNumber(Pred, LoadBB);
       NeedToSplit.push_back(std::make_pair(Pred->getTerminator(), SuccNum));
     }
   }
+
   if (!NeedToSplit.empty()) {
     toSplit.append(NeedToSplit.begin(), NeedToSplit.end());
     return false;
@@ -1660,7 +1676,7 @@ bool GVN::processLoad(LoadInst *L) {
   if (!MD)
     return false;
 
-  if (L->isVolatile())
+  if (!L->isSimple())
     return false;
 
   if (L->use_empty()) {
@@ -1747,7 +1763,11 @@ bool GVN::processLoad(LoadInst *L) {
     return false;
   }
 
-  if (Dep.isUnknown()) {
+  // If it is defined in another block, try harder.
+  if (Dep.isNonLocal())
+    return processNonLocalLoad(L);
+
+  if (!Dep.isDef()) {
     DEBUG(
       // fast print dep, using operator<< on instruction is too slow.
       dbgs() << "GVN: load ";
@@ -1757,12 +1777,6 @@ bool GVN::processLoad(LoadInst *L) {
     return false;
   }
 
-  // If it is defined in another block, try harder.
-  if (Dep.isNonLocal())
-    return processNonLocalLoad(L);
-
-  assert(Dep.isDef() && "Expecting def here");
-
   Instruction *DepInst = Dep.getInst();
   if (StoreInst *DepSI = dyn_cast<StoreInst>(DepInst)) {
     Value *StoredVal = DepSI->getValueOperand();
@@ -1874,6 +1888,133 @@ Value *GVN::findLeader(BasicBlock *BB, uint32_t num) {
   return Val;
 }
 
+/// replaceAllDominatedUsesWith - Replace all uses of 'From' with 'To' if the
+/// use is dominated by the given basic block.  Returns the number of uses that
+/// were replaced.
+unsigned GVN::replaceAllDominatedUsesWith(Value *From, Value *To,
+                                          BasicBlock *Root) {
+  unsigned Count = 0;
+  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
+       UI != UE; ) {
+    Instruction *User = cast<Instruction>(*UI);
+    unsigned OpNum = UI.getOperandNo();
+    ++UI;
+
+    if (DT->dominates(Root, User->getParent())) {
+      User->setOperand(OpNum, To);
+      ++Count;
+    }
+  }
+  return Count;
+}
+
+/// propagateEquality - The given values are known to be equal in every block
+/// dominated by 'Root'.  Exploit this, for example by replacing 'LHS' with
+/// 'RHS' everywhere in the scope.  Returns whether a change was made.
+bool GVN::propagateEquality(Value *LHS, Value *RHS, BasicBlock *Root) {
+  if (LHS == RHS) return false;
+  assert(LHS->getType() == RHS->getType() && "Equal but types differ!");
+
+  // Don't try to propagate equalities between constants.
+  if (isa<Constant>(LHS) && isa<Constant>(RHS))
+    return false;
+
+  // Make sure that any constants are on the right-hand side.  In general the
+  // best results are obtained by placing the longest lived value on the RHS.
+  if (isa<Constant>(LHS))
+    std::swap(LHS, RHS);
+
+  // If neither term is constant then bail out.  This is not for correctness,
+  // it's just that the non-constant case is much less useful: it occurs just
+  // as often as the constant case but handling it hardly ever results in an
+  // improvement.
+  if (!isa<Constant>(RHS))
+    return false;
+
+  // If value numbering later deduces that an instruction in the scope is equal
+  // to 'LHS' then ensure it will be turned into 'RHS'.
+  addToLeaderTable(VN.lookup_or_add(LHS), RHS, Root);
+
+  // Replace all occurrences of 'LHS' with 'RHS' everywhere in the scope.
+  unsigned NumReplacements = replaceAllDominatedUsesWith(LHS, RHS, Root);
+  bool Changed = NumReplacements > 0;
+  NumGVNEqProp += NumReplacements;
+
+  // Now try to deduce additional equalities from this one.  For example, if the
+  // known equality was "(A != B)" == "false" then it follows that A and B are
+  // equal in the scope.  Only boolean equalities with an explicit true or false
+  // RHS are currently supported.
+  if (!RHS->getType()->isIntegerTy(1))
+    // Not a boolean equality - bail out.
+    return Changed;
+  ConstantInt *CI = dyn_cast<ConstantInt>(RHS);
+  if (!CI)
+    // RHS neither 'true' nor 'false' - bail out.
+    return Changed;
+  // Whether RHS equals 'true'.  Otherwise it equals 'false'.
+  bool isKnownTrue = CI->isAllOnesValue();
+  bool isKnownFalse = !isKnownTrue;
+
+  // If "A && B" is known true then both A and B are known true.  If "A || B"
+  // is known false then both A and B are known false.
+  Value *A, *B;
+  if ((isKnownTrue && match(LHS, m_And(m_Value(A), m_Value(B)))) ||
+      (isKnownFalse && match(LHS, m_Or(m_Value(A), m_Value(B))))) {
+    Changed |= propagateEquality(A, RHS, Root);
+    Changed |= propagateEquality(B, RHS, Root);
+    return Changed;
+  }
+
+  // If we are propagating an equality like "(A == B)" == "true" then also
+  // propagate the equality A == B.
+  if (ICmpInst *Cmp = dyn_cast<ICmpInst>(LHS)) {
+    // Only equality comparisons are supported.
+    if ((isKnownTrue && Cmp->getPredicate() == CmpInst::ICMP_EQ) ||
+        (isKnownFalse && Cmp->getPredicate() == CmpInst::ICMP_NE)) {
+      Value *Op0 = Cmp->getOperand(0), *Op1 = Cmp->getOperand(1);
+      Changed |= propagateEquality(Op0, Op1, Root);
+    }
+    return Changed;
+  }
+
+  return Changed;
+}
+
+/// isOnlyReachableViaThisEdge - There is an edge from 'Src' to 'Dst'.  Return
+/// true if every path from the entry block to 'Dst' passes via this edge.  In
+/// particular 'Dst' must not be reachable via another edge from 'Src'.
+static bool isOnlyReachableViaThisEdge(BasicBlock *Src, BasicBlock *Dst,
+                                       DominatorTree *DT) {
+  // First off, there must not be more than one edge from Src to Dst, there
+  // should be exactly one.  So keep track of the number of times Src occurs
+  // as a predecessor of Dst and fail if it's more than once.  Secondly, any
+  // other predecessors of Dst should be dominated by Dst (see logic below).
+  bool SawEdgeFromSrc = false;
+  for (pred_iterator PI = pred_begin(Dst), PE = pred_end(Dst); PI != PE; ++PI) {
+    BasicBlock *Pred = *PI;
+    if (Pred == Src) {
+      // An edge from Src to Dst.
+      if (SawEdgeFromSrc)
+        // There are multiple edges from Src to Dst - fail.
+        return false;
+      SawEdgeFromSrc = true;
+      continue;
+    }
+    // If the predecessor is not dominated by Dst, then it must be possible to
+    // reach it either without passing through Src (and thus not via the edge)
+    // or by passing through Src but taking a different edge out of Src.  Either
+    // way it is possible to reach Dst without passing via the edge, so fail.
+    if (!DT->dominates(Dst, *PI))
+      return false;
+  }
+  assert(SawEdgeFromSrc && "No edge between these basic blocks!");
+
+  // Every path from the entry block to Dst must at some point pass to Dst from
+  // a predecessor that is not dominated by Dst.  This predecessor can only be
+  // Src, since all others are dominated by Dst.  As there is only one edge from
+  // Src to Dst, the path passes by this edge.
+  return true;
+}
 
 /// processInstruction - When calculating availability, handle an instruction
 /// by inserting it into the appropriate sets
@@ -1891,6 +2032,7 @@ bool GVN::processInstruction(Instruction *I) {
     if (MD && V->getType()->isPointerTy())
       MD->invalidateCachedPointerInfo(V);
     markInstructionForDeletion(I);
+    ++NumGVNSimpl;
     return true;
   }
 
@@ -1903,30 +2045,45 @@ bool GVN::processInstruction(Instruction *I) {
     return false;
   }
 
-  // For conditions branches, we can perform simple conditional propagation on
+  // For conditional branches, we can perform simple conditional propagation on
   // the condition value itself.
   if (BranchInst *BI = dyn_cast<BranchInst>(I)) {
     if (!BI->isConditional() || isa<Constant>(BI->getCondition()))
       return false;
-    
+
     Value *BranchCond = BI->getCondition();
-    uint32_t CondVN = VN.lookup_or_add(BranchCond);
-  
+
     BasicBlock *TrueSucc = BI->getSuccessor(0);
     BasicBlock *FalseSucc = BI->getSuccessor(1);
-  
-    if (TrueSucc->getSinglePredecessor())
-      addToLeaderTable(CondVN,
-                   ConstantInt::getTrue(TrueSucc->getContext()),
-                   TrueSucc);
-    if (FalseSucc->getSinglePredecessor())
-      addToLeaderTable(CondVN,
-                   ConstantInt::getFalse(TrueSucc->getContext()),
-                   FalseSucc);
-    
-    return false;
+    BasicBlock *Parent = BI->getParent();
+    bool Changed = false;
+
+    if (isOnlyReachableViaThisEdge(Parent, TrueSucc, DT))
+      Changed |= propagateEquality(BranchCond,
+                                   ConstantInt::getTrue(TrueSucc->getContext()),
+                                   TrueSucc);
+
+    if (isOnlyReachableViaThisEdge(Parent, FalseSucc, DT))
+      Changed |= propagateEquality(BranchCond,
+                                   ConstantInt::getFalse(FalseSucc->getContext()),
+                                   FalseSucc);
+
+    return Changed;
   }
-  
+
+  // For switches, propagate the case values into the case destinations.
+  if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
+    Value *SwitchCond = SI->getCondition();
+    BasicBlock *Parent = SI->getParent();
+    bool Changed = false;
+    for (unsigned i = 1, e = SI->getNumCases(); i != e; ++i) {
+      BasicBlock *Dst = SI->getSuccessor(i);
+      if (isOnlyReachableViaThisEdge(Parent, Dst, DT))
+        Changed |= propagateEquality(SwitchCond, SI->getCaseValue(i), Dst);
+    }
+    return Changed;
+  }
+
   // Instructions with void type don't return a value, so there's
   // no point in trying to find redudancies in them.
   if (I->getType()->isVoidTy()) return false;
@@ -2071,6 +2228,9 @@ bool GVN::performPRE(Function &F) {
     // Nothing to PRE in the entry block.
     if (CurrentBlock == &F.getEntryBlock()) continue;
 
+    // Don't perform PRE on a landing pad.
+    if (CurrentBlock->isLandingPad()) continue;
+
     for (BasicBlock::iterator BI = CurrentBlock->begin(),
          BE = CurrentBlock->end(); BI != BE; ) {
       Instruction *CurInst = BI++;
diff --git a/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index dee3d38d72af..75fa011a14b7 100644
--- a/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -11,17 +11,6 @@
 // computations derived from them) into simpler forms suitable for subsequent
 // analysis and transformation.
 //
-// This transformation makes the following changes to each loop with an
-// identifiable induction variable:
-//   1. All loops are transformed to have a SINGLE canonical induction variable
-//      which starts at zero and steps by one.
-//   2. The canonical induction variable is guaranteed to be the first PHI node
-//      in the loop header block.
-//   3. The canonical induction variable is guaranteed to be in a wide enough
-//      type so that IV expressions need not be (directly) zero-extended or
-//      sign-extended.
-//   4. Any pointer arithmetic recurrences are raised to use array subscripts.
-//
 // If the trip count of a loop is computable, this pass also makes the following
 // changes:
 //   1. The exit condition for the loop is canonicalized to compare the
@@ -33,9 +22,6 @@
 //      purpose of the loop is to compute the exit value of some derived
 //      expression, this transformation will make the loop dead.
 //
-// This transformation should be followed by strength reduction after all of the
-// desired loop transformations have been performed.
-//
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "indvars"
@@ -57,11 +43,11 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/SimplifyIndVar.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/STLExtras.h"
 using namespace llvm;
 
 STATISTIC(NumRemoved     , "Number of aux indvars removed");
@@ -69,15 +55,21 @@ STATISTIC(NumWidened     , "Number of indvars widened");
 STATISTIC(NumInserted    , "Number of canonical indvars added");
 STATISTIC(NumReplaced    , "Number of exit values replaced");
 STATISTIC(NumLFTR        , "Number of loop exit tests replaced");
-STATISTIC(NumElimIdentity, "Number of IV identities eliminated");
 STATISTIC(NumElimExt     , "Number of IV sign/zero extends eliminated");
-STATISTIC(NumElimRem     , "Number of IV remainder operations eliminated");
-STATISTIC(NumElimCmp     , "Number of IV comparisons eliminated");
 STATISTIC(NumElimIV      , "Number of congruent IVs eliminated");
 
-static cl::opt<bool> DisableIVRewrite(
-  "disable-iv-rewrite", cl::Hidden,
-  cl::desc("Disable canonical induction variable rewriting"));
+namespace llvm {
+  cl::opt<bool> EnableIVRewrite(
+    "enable-iv-rewrite", cl::Hidden,
+    cl::desc("Enable canonical induction variable rewriting"));
+
+  // Trip count verification can be enabled by default under NDEBUG if we
+  // implement a strong expression equivalence checker in SCEV. Until then, we
+  // use the verify-indvars flag, which may assert in some cases.
+  cl::opt<bool> VerifyIndvars(
+    "verify-indvars", cl::Hidden,
+    cl::desc("Verify the ScalarEvolution result after running indvars"));
+}
 
 namespace {
   class IndVarSimplify : public LoopPass {
@@ -105,12 +97,12 @@ namespace {
       AU.addRequired<ScalarEvolution>();
       AU.addRequiredID(LoopSimplifyID);
       AU.addRequiredID(LCSSAID);
-      if (!DisableIVRewrite)
+      if (EnableIVRewrite)
         AU.addRequired<IVUsers>();
       AU.addPreserved<ScalarEvolution>();
       AU.addPreservedID(LoopSimplifyID);
       AU.addPreservedID(LCSSAID);
-      if (!DisableIVRewrite)
+      if (EnableIVRewrite)
         AU.addPreserved<IVUsers>();
       AU.setPreservesCFG();
     }
@@ -125,24 +117,14 @@ namespace {
     void HandleFloatingPointIV(Loop *L, PHINode *PH);
     void RewriteNonIntegerIVs(Loop *L);
 
+    void SimplifyAndExtend(Loop *L, SCEVExpander &Rewriter, LPPassManager &LPM);
+
     void RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter);
 
-    void SimplifyIVUsers(SCEVExpander &Rewriter);
-    void SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter);
-
-    bool EliminateIVUser(Instruction *UseInst, Instruction *IVOperand);
-    void EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand);
-    void EliminateIVRemainder(BinaryOperator *Rem,
-                              Value *IVOperand,
-                              bool IsSigned);
-
-    void SimplifyCongruentIVs(Loop *L);
-
     void RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter);
 
-    ICmpInst *LinearFunctionTestReplace(Loop *L, const SCEV *BackedgeTakenCount,
-                                        PHINode *IndVar,
-                                        SCEVExpander &Rewriter);
+    Value *LinearFunctionTestReplace(Loop *L, const SCEV *BackedgeTakenCount,
+                                     PHINode *IndVar, SCEVExpander &Rewriter);
 
     void SinkUnusedInvariants(Loop *L);
   };
@@ -211,6 +193,36 @@ bool IndVarSimplify::isValidRewrite(Value *FromVal, Value *ToVal) {
   return true;
 }
 
+/// Determine the insertion point for this user. By default, insert immediately
+/// before the user. SCEVExpander or LICM will hoist loop invariants out of the
+/// loop. For PHI nodes, there may be multiple uses, so compute the nearest
+/// common dominator for the incoming blocks.
+static Instruction *getInsertPointForUses(Instruction *User, Value *Def,
+                                          DominatorTree *DT) {
+  PHINode *PHI = dyn_cast<PHINode>(User);
+  if (!PHI)
+    return User;
+
+  Instruction *InsertPt = 0;
+  for (unsigned i = 0, e = PHI->getNumIncomingValues(); i != e; ++i) {
+    if (PHI->getIncomingValue(i) != Def)
+      continue;
+
+    BasicBlock *InsertBB = PHI->getIncomingBlock(i);
+    if (!InsertPt) {
+      InsertPt = InsertBB->getTerminator();
+      continue;
+    }
+    InsertBB = DT->findNearestCommonDominator(InsertPt->getParent(), InsertBB);
+    InsertPt = InsertBB->getTerminator();
+  }
+  assert(InsertPt && "Missing phi operand");
+  assert((!isa<Instruction>(Def) ||
+          DT->dominates(cast<Instruction>(Def), InsertPt)) &&
+         "def does not dominate all uses");
+  return InsertPt;
+}
+
 //===----------------------------------------------------------------------===//
 // RewriteNonIntegerIVs and helpers. Prefer integer IVs.
 //===----------------------------------------------------------------------===//
@@ -337,14 +349,14 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
   // Positive and negative strides have different safety conditions.
   if (IncValue > 0) {
     // If we have a positive stride, we require the init to be less than the
-    // exit value and an equality or less than comparison.
-    if (InitValue >= ExitValue ||
-        NewPred == CmpInst::ICMP_SGT || NewPred == CmpInst::ICMP_SGE)
+    // exit value.
+    if (InitValue >= ExitValue)
       return;
 
     uint32_t Range = uint32_t(ExitValue-InitValue);
-    if (NewPred == CmpInst::ICMP_SLE) {
-      // Normalize SLE -> SLT, check for infinite loop.
+    // Check for infinite loop, either:
+    // while (i <= Exit) or until (i > Exit)
+    if (NewPred == CmpInst::ICMP_SLE || NewPred == CmpInst::ICMP_SGT) {
       if (++Range == 0) return;  // Range overflows.
     }
 
@@ -364,14 +376,14 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
 
   } else {
     // If we have a negative stride, we require the init to be greater than the
-    // exit value and an equality or greater than comparison.
-    if (InitValue >= ExitValue ||
-        NewPred == CmpInst::ICMP_SLT || NewPred == CmpInst::ICMP_SLE)
+    // exit value.
+    if (InitValue <= ExitValue)
       return;
 
     uint32_t Range = uint32_t(InitValue-ExitValue);
-    if (NewPred == CmpInst::ICMP_SGE) {
-      // Normalize SGE -> SGT, check for infinite loop.
+    // Check for infinite loop, either:
+    // while (i >= Exit) or until (i < Exit)
+    if (NewPred == CmpInst::ICMP_SGE || NewPred == CmpInst::ICMP_SLT) {
       if (++Range == 0) return;  // Range overflows.
     }
 
@@ -390,7 +402,7 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
       return;
   }
 
-  const IntegerType *Int32Ty = Type::getInt32Ty(PN->getContext());
+  IntegerType *Int32Ty = Type::getInt32Ty(PN->getContext());
 
   // Insert new integer induction variable.
   PHINode *NewPHI = PHINode::Create(Int32Ty, 2, PN->getName()+".int", PN);
@@ -429,7 +441,7 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
   // platforms.
   if (WeakPH) {
     Value *Conv = new SIToFPInst(NewPHI, PN->getType(), "indvar.conv",
-                                 PN->getParent()->getFirstNonPHI());
+                                 PN->getParent()->getFirstInsertionPt());
     PN->replaceAllUsesWith(Conv);
     RecursivelyDeleteTriviallyDeadInstructions(PN);
   }
@@ -437,6 +449,8 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
   // Add a new IVUsers entry for the newly-created integer PHI.
   if (IU)
     IU->AddUsersIfInteresting(NewPHI);
+
+  Changed = true;
 }
 
 void IndVarSimplify::RewriteNonIntegerIVs(Loop *L) {
@@ -582,45 +596,15 @@ void IndVarSimplify::RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter) {
 
 //===----------------------------------------------------------------------===//
 //  Rewrite IV users based on a canonical IV.
-//  To be replaced by -disable-iv-rewrite.
+//  Only for use with -enable-iv-rewrite.
 //===----------------------------------------------------------------------===//
 
-/// SimplifyIVUsers - Iteratively perform simplification on IVUsers within this
-/// loop. IVUsers is treated as a worklist. Each successive simplification may
-/// push more users which may themselves be candidates for simplification.
-///
-/// This is the old approach to IV simplification to be replaced by
-/// SimplifyIVUsersNoRewrite.
-///
-void IndVarSimplify::SimplifyIVUsers(SCEVExpander &Rewriter) {
-  // Each round of simplification involves a round of eliminating operations
-  // followed by a round of widening IVs. A single IVUsers worklist is used
-  // across all rounds. The inner loop advances the user. If widening exposes
-  // more uses, then another pass through the outer loop is triggered.
-  for (IVUsers::iterator I = IU->begin(); I != IU->end(); ++I) {
-    Instruction *UseInst = I->getUser();
-    Value *IVOperand = I->getOperandValToReplace();
-
-    if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) {
-      EliminateIVComparison(ICmp, IVOperand);
-      continue;
-    }
-    if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) {
-      bool IsSigned = Rem->getOpcode() == Instruction::SRem;
-      if (IsSigned || Rem->getOpcode() == Instruction::URem) {
-        EliminateIVRemainder(Rem, IVOperand, IsSigned);
-        continue;
-      }
-    }
-  }
-}
-
-// FIXME: It is an extremely bad idea to indvar substitute anything more
-// complex than affine induction variables.  Doing so will put expensive
-// polynomial evaluations inside of the loop, and the str reduction pass
-// currently can only reduce affine polynomials.  For now just disable
-// indvar subst on anything more complex than an affine addrec, unless
-// it can be expanded to a trivial value.
+/// FIXME: It is an extremely bad idea to indvar substitute anything more
+/// complex than affine induction variables.  Doing so will put expensive
+/// polynomial evaluations inside of the loop, and the str reduction pass
+/// currently can only reduce affine polynomials.  For now just disable
+/// indvar subst on anything more complex than an affine addrec, unless
+/// it can be expanded to a trivial value.
 static bool isSafe(const SCEV *S, const Loop *L, ScalarEvolution *SE) {
   // Loop-invariant values are safe.
   if (SE->isLoopInvariant(S, L)) return true;
@@ -631,7 +615,8 @@ static bool isSafe(const SCEV *S, const Loop *L, ScalarEvolution *SE) {
     return AR->isAffine();
 
   // An add is safe it all its operands are safe.
-  if (const SCEVCommutativeExpr *Commutative = dyn_cast<SCEVCommutativeExpr>(S)) {
+  if (const SCEVCommutativeExpr *Commutative
+      = dyn_cast<SCEVCommutativeExpr>(S)) {
     for (SCEVCommutativeExpr::op_iterator I = Commutative->op_begin(),
          E = Commutative->op_end(); I != E; ++I)
       if (!isSafe(*I, L, SE)) return false;
@@ -665,7 +650,7 @@ void IndVarSimplify::RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter) {
   // of different sizes.
   for (IVUsers::iterator UI = IU->begin(), E = IU->end(); UI != E; ++UI) {
     Value *Op = UI->getOperandValToReplace();
-    const Type *UseTy = Op->getType();
+    Type *UseTy = Op->getType();
     Instruction *User = UI->getUser();
 
     // Compute the final addrec to expand into code.
@@ -692,18 +677,7 @@ void IndVarSimplify::RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter) {
     // hoist loop invariants out of the loop. For PHI nodes, there may be
     // multiple uses, so compute the nearest common dominator for the
     // incoming blocks.
-    Instruction *InsertPt = User;
-    if (PHINode *PHI = dyn_cast<PHINode>(InsertPt))
-      for (unsigned i = 0, e = PHI->getNumIncomingValues(); i != e; ++i)
-        if (PHI->getIncomingValue(i) == Op) {
-          if (InsertPt == User)
-            InsertPt = PHI->getIncomingBlock(i)->getTerminator();
-          else
-            InsertPt =
-              DT->findNearestCommonDominator(InsertPt->getParent(),
-                                             PHI->getIncomingBlock(i))
-                    ->getTerminator();
-        }
+    Instruction *InsertPt = getInsertPointForUses(User, Op, DT);
 
     // Now expand it into actual Instructions and patch it into place.
     Value *NewVal = Rewriter.expandCodeFor(AR, UseTy, InsertPt);
@@ -747,19 +721,38 @@ namespace {
   // extend operations. This information is recorded by CollectExtend and
   // provides the input to WidenIV.
   struct WideIVInfo {
-    const Type *WidestNativeType; // Widest integer type created [sz]ext
-    bool IsSigned;                // Was an sext user seen before a zext?
+    PHINode *NarrowIV;
+    Type *WidestNativeType; // Widest integer type created [sz]ext
+    bool IsSigned;          // Was an sext user seen before a zext?
 
-    WideIVInfo() : WidestNativeType(0), IsSigned(false) {}
+    WideIVInfo() : NarrowIV(0), WidestNativeType(0), IsSigned(false) {}
+  };
+
+  class WideIVVisitor : public IVVisitor {
+    ScalarEvolution *SE;
+    const TargetData *TD;
+
+  public:
+    WideIVInfo WI;
+
+    WideIVVisitor(PHINode *NarrowIV, ScalarEvolution *SCEV,
+                  const TargetData *TData) :
+      SE(SCEV), TD(TData) { WI.NarrowIV = NarrowIV; }
+
+    // Implement the interface used by simplifyUsersOfIV.
+    virtual void visitCast(CastInst *Cast);
   };
 }
 
-/// CollectExtend - Update information about the induction variable that is
+/// visitCast - Update information about the induction variable that is
 /// extended by this sign or zero extend operation. This is used to determine
 /// the final width of the IV before actually widening it.
-static void CollectExtend(CastInst *Cast, bool IsSigned, WideIVInfo &WI,
-                          ScalarEvolution *SE, const TargetData *TD) {
-  const Type *Ty = Cast->getType();
+void WideIVVisitor::visitCast(CastInst *Cast) {
+  bool IsSigned = Cast->getOpcode() == Instruction::SExt;
+  if (!IsSigned && Cast->getOpcode() != Instruction::ZExt)
+    return;
+
+  Type *Ty = Cast->getType();
   uint64_t Width = SE->getTypeSizeInBits(Ty);
   if (TD && !TD->isLegalInteger(Width))
     return;
@@ -779,6 +772,21 @@ static void CollectExtend(CastInst *Cast, bool IsSigned, WideIVInfo &WI,
 }
 
 namespace {
+
+/// NarrowIVDefUse - Record a link in the Narrow IV def-use chain along with the
+/// WideIV that computes the same value as the Narrow IV def.  This avoids
+/// caching Use* pointers.
+struct NarrowIVDefUse {
+  Instruction *NarrowDef;
+  Instruction *NarrowUse;
+  Instruction *WideDef;
+
+  NarrowIVDefUse(): NarrowDef(0), NarrowUse(0), WideDef(0) {}
+
+  NarrowIVDefUse(Instruction *ND, Instruction *NU, Instruction *WD):
+    NarrowDef(ND), NarrowUse(NU), WideDef(WD) {}
+};
+
 /// WidenIV - The goal of this transform is to remove sign and zero extends
 /// without creating any new induction variables. To do this, it creates a new
 /// phi of the wider type and redirects all users, either removing extends or
@@ -787,7 +795,7 @@ namespace {
 class WidenIV {
   // Parameters
   PHINode *OrigPhi;
-  const Type *WideType;
+  Type *WideType;
   bool IsSigned;
 
   // Context
@@ -803,13 +811,13 @@ class WidenIV {
   SmallVectorImpl<WeakVH> &DeadInsts;
 
   SmallPtrSet<Instruction*,16> Widened;
-  SmallVector<std::pair<Use *, Instruction *>, 8> NarrowIVUsers;
+  SmallVector<NarrowIVDefUse, 8> NarrowIVUsers;
 
 public:
-  WidenIV(PHINode *PN, const WideIVInfo &WI, LoopInfo *LInfo,
+  WidenIV(const WideIVInfo &WI, LoopInfo *LInfo,
           ScalarEvolution *SEv, DominatorTree *DTree,
           SmallVectorImpl<WeakVH> &DI) :
-    OrigPhi(PN),
+    OrigPhi(WI.NarrowIV),
     WideType(WI.WidestNativeType),
     IsSigned(WI.IsSigned),
     LI(LInfo),
@@ -826,21 +834,42 @@ class WidenIV {
   PHINode *CreateWideIV(SCEVExpander &Rewriter);
 
 protected:
-  Instruction *CloneIVUser(Instruction *NarrowUse,
-                           Instruction *NarrowDef,
-                           Instruction *WideDef);
+  Value *getExtend(Value *NarrowOper, Type *WideType, bool IsSigned,
+                   Instruction *Use);
+
+  Instruction *CloneIVUser(NarrowIVDefUse DU);
 
   const SCEVAddRecExpr *GetWideRecurrence(Instruction *NarrowUse);
 
-  Instruction *WidenIVUse(Use &NarrowDefUse, Instruction *NarrowDef,
-                          Instruction *WideDef);
+  const SCEVAddRecExpr* GetExtendedOperandRecurrence(NarrowIVDefUse DU);
+
+  Instruction *WidenIVUse(NarrowIVDefUse DU);
 
   void pushNarrowIVUsers(Instruction *NarrowDef, Instruction *WideDef);
 };
 } // anonymous namespace
 
-static Value *getExtend( Value *NarrowOper, const Type *WideType,
-                               bool IsSigned, IRBuilder<> &Builder) {
+/// isLoopInvariant - Perform a quick domtree based check for loop invariance
+/// assuming that V is used within the loop. LoopInfo::isLoopInvariant() seems
+/// gratuitous for this purpose.
+static bool isLoopInvariant(Value *V, const Loop *L, const DominatorTree *DT) {
+  Instruction *Inst = dyn_cast<Instruction>(V);
+  if (!Inst)
+    return true;
+
+  return DT->properlyDominates(Inst->getParent(), L->getHeader());
+}
+
+Value *WidenIV::getExtend(Value *NarrowOper, Type *WideType, bool IsSigned,
+                          Instruction *Use) {
+  // Set the debug location and conservative insertion point.
+  IRBuilder<> Builder(Use);
+  // Hoist the insertion point into loop preheaders as far as possible.
+  for (const Loop *L = LI->getLoopFor(Use->getParent());
+       L && L->getLoopPreheader() && isLoopInvariant(NarrowOper, L, DT);
+       L = L->getParentLoop())
+    Builder.SetInsertPoint(L->getLoopPreheader()->getTerminator());
+
   return IsSigned ? Builder.CreateSExt(NarrowOper, WideType) :
                     Builder.CreateZExt(NarrowOper, WideType);
 }
@@ -848,10 +877,8 @@ static Value *getExtend( Value *NarrowOper, const Type *WideType,
 /// CloneIVUser - Instantiate a wide operation to replace a narrow
 /// operation. This only needs to handle operations that can evaluation to
 /// SCEVAddRec. It can safely return 0 for any operation we decide not to clone.
-Instruction *WidenIV::CloneIVUser(Instruction *NarrowUse,
-                                  Instruction *NarrowDef,
-                                  Instruction *WideDef) {
-  unsigned Opcode = NarrowUse->getOpcode();
+Instruction *WidenIV::CloneIVUser(NarrowIVDefUse DU) {
+  unsigned Opcode = DU.NarrowUse->getOpcode();
   switch (Opcode) {
   default:
     return 0;
@@ -865,24 +892,23 @@ Instruction *WidenIV::CloneIVUser(Instruction *NarrowUse,
   case Instruction::Shl:
   case Instruction::LShr:
   case Instruction::AShr:
-    DEBUG(dbgs() << "Cloning IVUser: " << *NarrowUse << "\n");
-
-    IRBuilder<> Builder(NarrowUse);
+    DEBUG(dbgs() << "Cloning IVUser: " << *DU.NarrowUse << "\n");
 
     // Replace NarrowDef operands with WideDef. Otherwise, we don't know
     // anything about the narrow operand yet so must insert a [sz]ext. It is
     // probably loop invariant and will be folded or hoisted. If it actually
     // comes from a widened IV, it should be removed during a future call to
     // WidenIVUse.
-    Value *LHS = (NarrowUse->getOperand(0) == NarrowDef) ? WideDef :
-      getExtend(NarrowUse->getOperand(0), WideType, IsSigned, Builder);
-    Value *RHS = (NarrowUse->getOperand(1) == NarrowDef) ? WideDef :
-      getExtend(NarrowUse->getOperand(1), WideType, IsSigned, Builder);
+    Value *LHS = (DU.NarrowUse->getOperand(0) == DU.NarrowDef) ? DU.WideDef :
+      getExtend(DU.NarrowUse->getOperand(0), WideType, IsSigned, DU.NarrowUse);
+    Value *RHS = (DU.NarrowUse->getOperand(1) == DU.NarrowDef) ? DU.WideDef :
+      getExtend(DU.NarrowUse->getOperand(1), WideType, IsSigned, DU.NarrowUse);
 
-    BinaryOperator *NarrowBO = cast<BinaryOperator>(NarrowUse);
+    BinaryOperator *NarrowBO = cast<BinaryOperator>(DU.NarrowUse);
     BinaryOperator *WideBO = BinaryOperator::Create(NarrowBO->getOpcode(),
                                                     LHS, RHS,
                                                     NarrowBO->getName());
+    IRBuilder<> Builder(DU.NarrowUse);
     Builder.Insert(WideBO);
     if (const OverflowingBinaryOperator *OBO =
         dyn_cast<OverflowingBinaryOperator>(NarrowBO)) {
@@ -894,45 +920,46 @@ Instruction *WidenIV::CloneIVUser(Instruction *NarrowUse,
   llvm_unreachable(0);
 }
 
-/// HoistStep - Attempt to hoist an IV increment above a potential use.
-///
-/// To successfully hoist, two criteria must be met:
-/// - IncV operands dominate InsertPos and
-/// - InsertPos dominates IncV
-///
-/// Meeting the second condition means that we don't need to check all of IncV's
-/// existing uses (it's moving up in the domtree).
-///
-/// This does not yet recursively hoist the operands, although that would
-/// not be difficult.
-static bool HoistStep(Instruction *IncV, Instruction *InsertPos,
-                      const DominatorTree *DT)
-{
-  if (DT->dominates(IncV, InsertPos))
-    return true;
+/// No-wrap operations can transfer sign extension of their result to their
+/// operands. Generate the SCEV value for the widened operation without
+/// actually modifying the IR yet. If the expression after extending the
+/// operands is an AddRec for this loop, return it.
+const SCEVAddRecExpr* WidenIV::GetExtendedOperandRecurrence(NarrowIVDefUse DU) {
+  // Handle the common case of add<nsw/nuw>
+  if (DU.NarrowUse->getOpcode() != Instruction::Add)
+    return 0;
 
-  if (!DT->dominates(InsertPos->getParent(), IncV->getParent()))
-    return false;
+  // One operand (NarrowDef) has already been extended to WideDef. Now determine
+  // if extending the other will lead to a recurrence.
+  unsigned ExtendOperIdx = DU.NarrowUse->getOperand(0) == DU.NarrowDef ? 1 : 0;
+  assert(DU.NarrowUse->getOperand(1-ExtendOperIdx) == DU.NarrowDef && "bad DU");
 
-  if (IncV->mayHaveSideEffects())
-    return false;
+  const SCEV *ExtendOperExpr = 0;
+  const OverflowingBinaryOperator *OBO =
+    cast<OverflowingBinaryOperator>(DU.NarrowUse);
+  if (IsSigned && OBO->hasNoSignedWrap())
+    ExtendOperExpr = SE->getSignExtendExpr(
+      SE->getSCEV(DU.NarrowUse->getOperand(ExtendOperIdx)), WideType);
+  else if(!IsSigned && OBO->hasNoUnsignedWrap())
+    ExtendOperExpr = SE->getZeroExtendExpr(
+      SE->getSCEV(DU.NarrowUse->getOperand(ExtendOperIdx)), WideType);
+  else
+    return 0;
 
-  // Attempt to hoist IncV
-  for (User::op_iterator OI = IncV->op_begin(), OE = IncV->op_end();
-       OI != OE; ++OI) {
-    Instruction *OInst = dyn_cast<Instruction>(OI);
-    if (OInst && !DT->dominates(OInst, InsertPos))
-      return false;
-  }
-  IncV->moveBefore(InsertPos);
-  return true;
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(
+    SE->getAddExpr(SE->getSCEV(DU.WideDef), ExtendOperExpr,
+                   IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW));
+
+  if (!AddRec || AddRec->getLoop() != L)
+    return 0;
+  return AddRec;
 }
 
-// GetWideRecurrence - Is this instruction potentially interesting from IVUsers'
-// perspective after widening it's type? In other words, can the extend be
-// safely hoisted out of the loop with SCEV reducing the value to a recurrence
-// on the same loop. If so, return the sign or zero extended
-// recurrence. Otherwise return NULL.
+/// GetWideRecurrence - Is this instruction potentially interesting from
+/// IVUsers' perspective after widening it's type? In other words, can the
+/// extend be safely hoisted out of the loop with SCEV reducing the value to a
+/// recurrence on the same loop. If so, return the sign or zero extended
+/// recurrence. Otherwise return NULL.
 const SCEVAddRecExpr *WidenIV::GetWideRecurrence(Instruction *NarrowUse) {
   if (!SE->isSCEVable(NarrowUse->getType()))
     return 0;
@@ -951,47 +978,45 @@ const SCEVAddRecExpr *WidenIV::GetWideRecurrence(Instruction *NarrowUse) {
   const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(WideExpr);
   if (!AddRec || AddRec->getLoop() != L)
     return 0;
-
   return AddRec;
 }
 
 /// WidenIVUse - Determine whether an individual user of the narrow IV can be
 /// widened. If so, return the wide clone of the user.
-Instruction *WidenIV::WidenIVUse(Use &NarrowDefUse, Instruction *NarrowDef,
-                                 Instruction *WideDef) {
-  Instruction *NarrowUse = cast<Instruction>(NarrowDefUse.getUser());
+Instruction *WidenIV::WidenIVUse(NarrowIVDefUse DU) {
 
   // Stop traversing the def-use chain at inner-loop phis or post-loop phis.
-  if (isa<PHINode>(NarrowUse) && LI->getLoopFor(NarrowUse->getParent()) != L)
+  if (isa<PHINode>(DU.NarrowUse) &&
+      LI->getLoopFor(DU.NarrowUse->getParent()) != L)
     return 0;
 
   // Our raison d'etre! Eliminate sign and zero extension.
-  if (IsSigned ? isa<SExtInst>(NarrowUse) : isa<ZExtInst>(NarrowUse)) {
-    Value *NewDef = WideDef;
-    if (NarrowUse->getType() != WideType) {
-      unsigned CastWidth = SE->getTypeSizeInBits(NarrowUse->getType());
+  if (IsSigned ? isa<SExtInst>(DU.NarrowUse) : isa<ZExtInst>(DU.NarrowUse)) {
+    Value *NewDef = DU.WideDef;
+    if (DU.NarrowUse->getType() != WideType) {
+      unsigned CastWidth = SE->getTypeSizeInBits(DU.NarrowUse->getType());
       unsigned IVWidth = SE->getTypeSizeInBits(WideType);
       if (CastWidth < IVWidth) {
         // The cast isn't as wide as the IV, so insert a Trunc.
-        IRBuilder<> Builder(NarrowDefUse);
-        NewDef = Builder.CreateTrunc(WideDef, NarrowUse->getType());
+        IRBuilder<> Builder(DU.NarrowUse);
+        NewDef = Builder.CreateTrunc(DU.WideDef, DU.NarrowUse->getType());
       }
       else {
         // A wider extend was hidden behind a narrower one. This may induce
         // another round of IV widening in which the intermediate IV becomes
         // dead. It should be very rare.
         DEBUG(dbgs() << "INDVARS: New IV " << *WidePhi
-              << " not wide enough to subsume " << *NarrowUse << "\n");
-        NarrowUse->replaceUsesOfWith(NarrowDef, WideDef);
-        NewDef = NarrowUse;
+              << " not wide enough to subsume " << *DU.NarrowUse << "\n");
+        DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, DU.WideDef);
+        NewDef = DU.NarrowUse;
       }
     }
-    if (NewDef != NarrowUse) {
-      DEBUG(dbgs() << "INDVARS: eliminating " << *NarrowUse
-            << " replaced by " << *WideDef << "\n");
+    if (NewDef != DU.NarrowUse) {
+      DEBUG(dbgs() << "INDVARS: eliminating " << *DU.NarrowUse
+            << " replaced by " << *DU.WideDef << "\n");
       ++NumElimExt;
-      NarrowUse->replaceAllUsesWith(NewDef);
-      DeadInsts.push_back(NarrowUse);
+      DU.NarrowUse->replaceAllUsesWith(NewDef);
+      DeadInsts.push_back(DU.NarrowUse);
     }
     // Now that the extend is gone, we want to expose it's uses for potential
     // further simplification. We don't need to directly inform SimplifyIVUsers
@@ -1004,29 +1029,32 @@ Instruction *WidenIV::WidenIVUse(Use &NarrowDefUse, Instruction *NarrowDef,
   }
 
   // Does this user itself evaluate to a recurrence after widening?
-  const SCEVAddRecExpr *WideAddRec = GetWideRecurrence(NarrowUse);
+  const SCEVAddRecExpr *WideAddRec = GetWideRecurrence(DU.NarrowUse);
+  if (!WideAddRec) {
+      WideAddRec = GetExtendedOperandRecurrence(DU);
+  }
   if (!WideAddRec) {
     // This user does not evaluate to a recurence after widening, so don't
     // follow it. Instead insert a Trunc to kill off the original use,
     // eventually isolating the original narrow IV so it can be removed.
-    IRBuilder<> Builder(NarrowDefUse);
-    Value *Trunc = Builder.CreateTrunc(WideDef, NarrowDef->getType());
-    NarrowUse->replaceUsesOfWith(NarrowDef, Trunc);
+    IRBuilder<> Builder(getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT));
+    Value *Trunc = Builder.CreateTrunc(DU.WideDef, DU.NarrowDef->getType());
+    DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, Trunc);
     return 0;
   }
-  // We assume that block terminators are not SCEVable. We wouldn't want to
+  // Assume block terminators cannot evaluate to a recurrence. We can't to
   // insert a Trunc after a terminator if there happens to be a critical edge.
-  assert(NarrowUse != NarrowUse->getParent()->getTerminator() &&
+  assert(DU.NarrowUse != DU.NarrowUse->getParent()->getTerminator() &&
          "SCEV is not expected to evaluate a block terminator");
 
   // Reuse the IV increment that SCEVExpander created as long as it dominates
   // NarrowUse.
   Instruction *WideUse = 0;
-  if (WideAddRec == WideIncExpr && HoistStep(WideInc, NarrowUse, DT)) {
+  if (WideAddRec == WideIncExpr
+      && SCEVExpander::hoistStep(WideInc, DU.NarrowUse, DT))
     WideUse = WideInc;
-  }
   else {
-    WideUse = CloneIVUser(NarrowUse, NarrowDef, WideDef);
+    WideUse = CloneIVUser(DU);
     if (!WideUse)
       return 0;
   }
@@ -1051,13 +1079,13 @@ Instruction *WidenIV::WidenIVUse(Use &NarrowDefUse, Instruction *NarrowDef,
 void WidenIV::pushNarrowIVUsers(Instruction *NarrowDef, Instruction *WideDef) {
   for (Value::use_iterator UI = NarrowDef->use_begin(),
          UE = NarrowDef->use_end(); UI != UE; ++UI) {
-    Use &U = UI.getUse();
+    Instruction *NarrowUse = cast<Instruction>(*UI);
 
     // Handle data flow merges and bizarre phi cycles.
-    if (!Widened.insert(cast<Instruction>(U.getUser())))
+    if (!Widened.insert(NarrowUse))
       continue;
 
-    NarrowIVUsers.push_back(std::make_pair(&UI.getUse(), WideDef));
+    NarrowIVUsers.push_back(NarrowIVDefUse(NarrowDef, NarrowUse, WideDef));
   }
 }
 
@@ -1124,23 +1152,19 @@ PHINode *WidenIV::CreateWideIV(SCEVExpander &Rewriter) {
   pushNarrowIVUsers(OrigPhi, WidePhi);
 
   while (!NarrowIVUsers.empty()) {
-    Use *UsePtr;
-    Instruction *WideDef;
-    tie(UsePtr, WideDef) = NarrowIVUsers.pop_back_val();
-    Use &NarrowDefUse = *UsePtr;
+    NarrowIVDefUse DU = NarrowIVUsers.pop_back_val();
 
     // Process a def-use edge. This may replace the use, so don't hold a
     // use_iterator across it.
-    Instruction *NarrowDef = cast<Instruction>(NarrowDefUse.get());
-    Instruction *WideUse = WidenIVUse(NarrowDefUse, NarrowDef, WideDef);
+    Instruction *WideUse = WidenIVUse(DU);
 
     // Follow all def-use edges from the previous narrow use.
     if (WideUse)
-      pushNarrowIVUsers(cast<Instruction>(NarrowDefUse.getUser()), WideUse);
+      pushNarrowIVUsers(DU.NarrowUse, WideUse);
 
     // WidenIVUse may have removed the def-use edge.
-    if (NarrowDef->use_empty())
-      DeadInsts.push_back(NarrowDef);
+    if (DU.NarrowDef->use_empty())
+      DeadInsts.push_back(DU.NarrowDef);
   }
   return WidePhi;
 }
@@ -1149,187 +1173,17 @@ PHINode *WidenIV::CreateWideIV(SCEVExpander &Rewriter) {
 //  Simplification of IV users based on SCEV evaluation.
 //===----------------------------------------------------------------------===//
 
-void IndVarSimplify::EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) {
-  unsigned IVOperIdx = 0;
-  ICmpInst::Predicate Pred = ICmp->getPredicate();
-  if (IVOperand != ICmp->getOperand(0)) {
-    // Swapped
-    assert(IVOperand == ICmp->getOperand(1) && "Can't find IVOperand");
-    IVOperIdx = 1;
-    Pred = ICmpInst::getSwappedPredicate(Pred);
-  }
 
-  // Get the SCEVs for the ICmp operands.
-  const SCEV *S = SE->getSCEV(ICmp->getOperand(IVOperIdx));
-  const SCEV *X = SE->getSCEV(ICmp->getOperand(1 - IVOperIdx));
-
-  // Simplify unnecessary loops away.
-  const Loop *ICmpLoop = LI->getLoopFor(ICmp->getParent());
-  S = SE->getSCEVAtScope(S, ICmpLoop);
-  X = SE->getSCEVAtScope(X, ICmpLoop);
-
-  // If the condition is always true or always false, replace it with
-  // a constant value.
-  if (SE->isKnownPredicate(Pred, S, X))
-    ICmp->replaceAllUsesWith(ConstantInt::getTrue(ICmp->getContext()));
-  else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X))
-    ICmp->replaceAllUsesWith(ConstantInt::getFalse(ICmp->getContext()));
-  else
-    return;
-
-  DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n');
-  ++NumElimCmp;
-  Changed = true;
-  DeadInsts.push_back(ICmp);
-}
-
-void IndVarSimplify::EliminateIVRemainder(BinaryOperator *Rem,
-                                          Value *IVOperand,
-                                          bool IsSigned) {
-  // We're only interested in the case where we know something about
-  // the numerator.
-  if (IVOperand != Rem->getOperand(0))
-    return;
-
-  // Get the SCEVs for the ICmp operands.
-  const SCEV *S = SE->getSCEV(Rem->getOperand(0));
-  const SCEV *X = SE->getSCEV(Rem->getOperand(1));
-
-  // Simplify unnecessary loops away.
-  const Loop *ICmpLoop = LI->getLoopFor(Rem->getParent());
-  S = SE->getSCEVAtScope(S, ICmpLoop);
-  X = SE->getSCEVAtScope(X, ICmpLoop);
-
-  // i % n  -->  i  if i is in [0,n).
-  if ((!IsSigned || SE->isKnownNonNegative(S)) &&
-      SE->isKnownPredicate(IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
-                           S, X))
-    Rem->replaceAllUsesWith(Rem->getOperand(0));
-  else {
-    // (i+1) % n  -->  (i+1)==n?0:(i+1)  if i is in [0,n).
-    const SCEV *LessOne =
-      SE->getMinusSCEV(S, SE->getConstant(S->getType(), 1));
-    if (IsSigned && !SE->isKnownNonNegative(LessOne))
-      return;
-
-    if (!SE->isKnownPredicate(IsSigned ?
-                              ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
-                              LessOne, X))
-      return;
-
-    ICmpInst *ICmp = new ICmpInst(Rem, ICmpInst::ICMP_EQ,
-                                  Rem->getOperand(0), Rem->getOperand(1),
-                                  "tmp");
-    SelectInst *Sel =
-      SelectInst::Create(ICmp,
-                         ConstantInt::get(Rem->getType(), 0),
-                         Rem->getOperand(0), "tmp", Rem);
-    Rem->replaceAllUsesWith(Sel);
-  }
-
-  // Inform IVUsers about the new users.
-  if (IU) {
-    if (Instruction *I = dyn_cast<Instruction>(Rem->getOperand(0)))
-      IU->AddUsersIfInteresting(I);
-  }
-  DEBUG(dbgs() << "INDVARS: Simplified rem: " << *Rem << '\n');
-  ++NumElimRem;
-  Changed = true;
-  DeadInsts.push_back(Rem);
-}
-
-/// EliminateIVUser - Eliminate an operation that consumes a simple IV and has
-/// no observable side-effect given the range of IV values.
-bool IndVarSimplify::EliminateIVUser(Instruction *UseInst,
-                                     Instruction *IVOperand) {
-  if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) {
-    EliminateIVComparison(ICmp, IVOperand);
-    return true;
-  }
-  if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) {
-    bool IsSigned = Rem->getOpcode() == Instruction::SRem;
-    if (IsSigned || Rem->getOpcode() == Instruction::URem) {
-      EliminateIVRemainder(Rem, IVOperand, IsSigned);
-      return true;
-    }
-  }
-
-  // Eliminate any operation that SCEV can prove is an identity function.
-  if (!SE->isSCEVable(UseInst->getType()) ||
-      (UseInst->getType() != IVOperand->getType()) ||
-      (SE->getSCEV(UseInst) != SE->getSCEV(IVOperand)))
-    return false;
-
-  DEBUG(dbgs() << "INDVARS: Eliminated identity: " << *UseInst << '\n');
-
-  UseInst->replaceAllUsesWith(IVOperand);
-  ++NumElimIdentity;
-  Changed = true;
-  DeadInsts.push_back(UseInst);
-  return true;
-}
-
-/// pushIVUsers - Add all uses of Def to the current IV's worklist.
-///
-static void pushIVUsers(
-  Instruction *Def,
-  SmallPtrSet<Instruction*,16> &Simplified,
-  SmallVectorImpl< std::pair<Instruction*,Instruction*> > &SimpleIVUsers) {
-
-  for (Value::use_iterator UI = Def->use_begin(), E = Def->use_end();
-       UI != E; ++UI) {
-    Instruction *User = cast<Instruction>(*UI);
-
-    // Avoid infinite or exponential worklist processing.
-    // Also ensure unique worklist users.
-    // If Def is a LoopPhi, it may not be in the Simplified set, so check for
-    // self edges first.
-    if (User != Def && Simplified.insert(User))
-      SimpleIVUsers.push_back(std::make_pair(User, Def));
-  }
-}
-
-/// isSimpleIVUser - Return true if this instruction generates a simple SCEV
-/// expression in terms of that IV.
-///
-/// This is similar to IVUsers' isInsteresting() but processes each instruction
-/// non-recursively when the operand is already known to be a simpleIVUser.
-///
-static bool isSimpleIVUser(Instruction *I, const Loop *L, ScalarEvolution *SE) {
-  if (!SE->isSCEVable(I->getType()))
-    return false;
-
-  // Get the symbolic expression for this instruction.
-  const SCEV *S = SE->getSCEV(I);
-
-  // We assume that terminators are not SCEVable.
-  assert((!S || I != I->getParent()->getTerminator()) &&
-         "can't fold terminators");
-
-  // Only consider affine recurrences.
-  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S);
-  if (AR && AR->getLoop() == L)
-    return true;
-
-  return false;
-}
-
-/// SimplifyIVUsersNoRewrite - Iteratively perform simplification on a worklist
-/// of IV users. Each successive simplification may push more users which may
+/// SimplifyAndExtend - Iteratively perform simplification on a worklist of IV
+/// users. Each successive simplification may push more users which may
 /// themselves be candidates for simplification.
 ///
-/// The "NoRewrite" algorithm does not require IVUsers analysis. Instead, it
-/// simplifies instructions in-place during analysis. Rather than rewriting
-/// induction variables bottom-up from their users, it transforms a chain of
-/// IVUsers top-down, updating the IR only when it encouters a clear
-/// optimization opportunitiy. A SCEVExpander "Rewriter" instance is still
-/// needed, but only used to generate a new IV (phi) of wider type for sign/zero
-/// extend elimination.
+/// Sign/Zero extend elimination is interleaved with IV simplification.
 ///
-/// Once DisableIVRewrite is default, LSR will be the only client of IVUsers.
-///
-void IndVarSimplify::SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter) {
-  std::map<PHINode *, WideIVInfo> WideIVMap;
+void IndVarSimplify::SimplifyAndExtend(Loop *L,
+                                       SCEVExpander &Rewriter,
+                                       LPPassManager &LPM) {
+  SmallVector<WideIVInfo, 8> WideIVs;
 
   SmallVector<PHINode*, 8> LoopPhis;
   for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
@@ -1345,102 +1199,27 @@ void IndVarSimplify::SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter) {
     // extension. The first time SCEV attempts to normalize sign/zero extension,
     // the result becomes final. So for the most predictable results, we delay
     // evaluation of sign/zero extend evaluation until needed, and avoid running
-    // other SCEV based analysis prior to SimplifyIVUsersNoRewrite.
+    // other SCEV based analysis prior to SimplifyAndExtend.
     do {
       PHINode *CurrIV = LoopPhis.pop_back_val();
 
       // Information about sign/zero extensions of CurrIV.
-      WideIVInfo WI;
+      WideIVVisitor WIV(CurrIV, SE, TD);
 
-      // Instructions processed by SimplifyIVUsers for CurrIV.
-      SmallPtrSet<Instruction*,16> Simplified;
+      Changed |= simplifyUsersOfIV(CurrIV, SE, &LPM, DeadInsts, &WIV);
 
-      // Use-def pairs if IV users waiting to be processed for CurrIV.
-      SmallVector<std::pair<Instruction*, Instruction*>, 8> SimpleIVUsers;
-
-      // Push users of the current LoopPhi. In rare cases, pushIVUsers may be
-      // called multiple times for the same LoopPhi. This is the proper thing to
-      // do for loop header phis that use each other.
-      pushIVUsers(CurrIV, Simplified, SimpleIVUsers);
-
-      while (!SimpleIVUsers.empty()) {
-        Instruction *UseInst, *Operand;
-        tie(UseInst, Operand) = SimpleIVUsers.pop_back_val();
-        // Bypass back edges to avoid extra work.
-        if (UseInst == CurrIV) continue;
-
-        if (EliminateIVUser(UseInst, Operand)) {
-          pushIVUsers(Operand, Simplified, SimpleIVUsers);
-          continue;
-        }
-        if (CastInst *Cast = dyn_cast<CastInst>(UseInst)) {
-          bool IsSigned = Cast->getOpcode() == Instruction::SExt;
-          if (IsSigned || Cast->getOpcode() == Instruction::ZExt) {
-            CollectExtend(Cast, IsSigned, WI, SE, TD);
-          }
-          continue;
-        }
-        if (isSimpleIVUser(UseInst, L, SE)) {
-          pushIVUsers(UseInst, Simplified, SimpleIVUsers);
-        }
-      }
-      if (WI.WidestNativeType) {
-        WideIVMap[CurrIV] = WI;
+      if (WIV.WI.WidestNativeType) {
+        WideIVs.push_back(WIV.WI);
       }
     } while(!LoopPhis.empty());
 
-    for (std::map<PHINode *, WideIVInfo>::const_iterator I = WideIVMap.begin(),
-           E = WideIVMap.end(); I != E; ++I) {
-      WidenIV Widener(I->first, I->second, LI, SE, DT, DeadInsts);
+    for (; !WideIVs.empty(); WideIVs.pop_back()) {
+      WidenIV Widener(WideIVs.back(), LI, SE, DT, DeadInsts);
       if (PHINode *WidePhi = Widener.CreateWideIV(Rewriter)) {
         Changed = true;
         LoopPhis.push_back(WidePhi);
       }
     }
-    WideIVMap.clear();
-  }
-}
-
-/// SimplifyCongruentIVs - Check for congruent phis in this loop header and
-/// populate ExprToIVMap for use later.
-///
-void IndVarSimplify::SimplifyCongruentIVs(Loop *L) {
-  DenseMap<const SCEV *, PHINode *> ExprToIVMap;
-  for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
-    PHINode *Phi = cast<PHINode>(I);
-    if (!SE->isSCEVable(Phi->getType()))
-      continue;
-
-    const SCEV *S = SE->getSCEV(Phi);
-    DenseMap<const SCEV *, PHINode *>::const_iterator Pos;
-    bool Inserted;
-    tie(Pos, Inserted) = ExprToIVMap.insert(std::make_pair(S, Phi));
-    if (Inserted)
-      continue;
-    PHINode *OrigPhi = Pos->second;
-    // Replacing the congruent phi is sufficient because acyclic redundancy
-    // elimination, CSE/GVN, should handle the rest. However, once SCEV proves
-    // that a phi is congruent, it's almost certain to be the head of an IV
-    // user cycle that is isomorphic with the original phi. So it's worth
-    // eagerly cleaning up the common case of a single IV increment.
-    if (BasicBlock *LatchBlock = L->getLoopLatch()) {
-      Instruction *OrigInc =
-        cast<Instruction>(OrigPhi->getIncomingValueForBlock(LatchBlock));
-      Instruction *IsomorphicInc =
-        cast<Instruction>(Phi->getIncomingValueForBlock(LatchBlock));
-      if (OrigInc != IsomorphicInc &&
-          SE->getSCEV(OrigInc) == SE->getSCEV(IsomorphicInc) &&
-          HoistStep(OrigInc, IsomorphicInc, DT)) {
-        DEBUG(dbgs() << "INDVARS: Eliminated congruent iv.inc: "
-              << *IsomorphicInc << '\n');
-        IsomorphicInc->replaceAllUsesWith(OrigInc);
-        DeadInsts.push_back(IsomorphicInc);
-      }
-    }
-    DEBUG(dbgs() << "INDVARS: Eliminated congruent iv: " << *Phi << '\n');
-    ++NumElimIV;
-    Phi->replaceAllUsesWith(OrigPhi);
-    DeadInsts.push_back(Phi);
   }
 }
 
@@ -1448,6 +1227,54 @@ void IndVarSimplify::SimplifyCongruentIVs(Loop *L) {
 //  LinearFunctionTestReplace and its kin. Rewrite the loop exit condition.
 //===----------------------------------------------------------------------===//
 
+/// Check for expressions that ScalarEvolution generates to compute
+/// BackedgeTakenInfo. If these expressions have not been reduced, then
+/// expanding them may incur additional cost (albeit in the loop preheader).
+static bool isHighCostExpansion(const SCEV *S, BranchInst *BI,
+                                ScalarEvolution *SE) {
+  // If the backedge-taken count is a UDiv, it's very likely a UDiv that
+  // ScalarEvolution's HowFarToZero or HowManyLessThans produced to compute a
+  // precise expression, rather than a UDiv from the user's code. If we can't
+  // find a UDiv in the code with some simple searching, assume the former and
+  // forego rewriting the loop.
+  if (isa<SCEVUDivExpr>(S)) {
+    ICmpInst *OrigCond = dyn_cast<ICmpInst>(BI->getCondition());
+    if (!OrigCond) return true;
+    const SCEV *R = SE->getSCEV(OrigCond->getOperand(1));
+    R = SE->getMinusSCEV(R, SE->getConstant(R->getType(), 1));
+    if (R != S) {
+      const SCEV *L = SE->getSCEV(OrigCond->getOperand(0));
+      L = SE->getMinusSCEV(L, SE->getConstant(L->getType(), 1));
+      if (L != S)
+        return true;
+    }
+  }
+
+  if (EnableIVRewrite)
+    return false;
+
+  // Recurse past add expressions, which commonly occur in the
+  // BackedgeTakenCount. They may already exist in program code, and if not,
+  // they are not too expensive rematerialize.
+  if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
+    for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
+         I != E; ++I) {
+      if (isHighCostExpansion(*I, BI, SE))
+        return true;
+    }
+    return false;
+  }
+
+  // HowManyLessThans uses a Max expression whenever the loop is not guarded by
+  // the exit condition.
+  if (isa<SCEVSMaxExpr>(S) || isa<SCEVUMaxExpr>(S))
+    return true;
+
+  // If we haven't recognized an expensive SCEV patter, assume its an expression
+  // produced by program code.
+  return false;
+}
+
 /// canExpandBackedgeTakenCount - Return true if this loop's backedge taken
 /// count expression can be safely and cheaply expanded into an instruction
 /// sequence that can be used by LinearFunctionTestReplace.
@@ -1465,31 +1292,17 @@ static bool canExpandBackedgeTakenCount(Loop *L, ScalarEvolution *SE) {
   if (!BI)
     return false;
 
-  // Special case: If the backedge-taken count is a UDiv, it's very likely a
-  // UDiv that ScalarEvolution produced in order to compute a precise
-  // expression, rather than a UDiv from the user's code. If we can't find a
-  // UDiv in the code with some simple searching, assume the former and forego
-  // rewriting the loop.
-  if (isa<SCEVUDivExpr>(BackedgeTakenCount)) {
-    ICmpInst *OrigCond = dyn_cast<ICmpInst>(BI->getCondition());
-    if (!OrigCond) return false;
-    const SCEV *R = SE->getSCEV(OrigCond->getOperand(1));
-    R = SE->getMinusSCEV(R, SE->getConstant(R->getType(), 1));
-    if (R != BackedgeTakenCount) {
-      const SCEV *L = SE->getSCEV(OrigCond->getOperand(0));
-      L = SE->getMinusSCEV(L, SE->getConstant(L->getType(), 1));
-      if (L != BackedgeTakenCount)
-        return false;
-    }
-  }
+  if (isHighCostExpansion(BackedgeTakenCount, BI, SE))
+    return false;
+
   return true;
 }
 
 /// getBackedgeIVType - Get the widest type used by the loop test after peeking
 /// through Truncs.
 ///
-/// TODO: Unnecessary if LFTR does not force a canonical IV.
-static const Type *getBackedgeIVType(Loop *L) {
+/// TODO: Unnecessary when ForceLFTR is removed.
+static Type *getBackedgeIVType(Loop *L) {
   if (!L->getExitingBlock())
     return 0;
 
@@ -1502,7 +1315,7 @@ static const Type *getBackedgeIVType(Loop *L) {
   if (!Cond)
     return 0;
 
-  const Type *Ty = 0;
+  Type *Ty = 0;
   for(User::op_iterator OI = Cond->op_begin(), OE = Cond->op_end();
       OI != OE; ++OI) {
     assert((!Ty || Ty == (*OI)->getType()) && "bad icmp operand types");
@@ -1515,12 +1328,187 @@ static const Type *getBackedgeIVType(Loop *L) {
   return Ty;
 }
 
+/// getLoopPhiForCounter - Return the loop header phi IFF IncV adds a loop
+/// invariant value to the phi.
+static PHINode *getLoopPhiForCounter(Value *IncV, Loop *L, DominatorTree *DT) {
+  Instruction *IncI = dyn_cast<Instruction>(IncV);
+  if (!IncI)
+    return 0;
+
+  switch (IncI->getOpcode()) {
+  case Instruction::Add:
+  case Instruction::Sub:
+    break;
+  case Instruction::GetElementPtr:
+    // An IV counter must preserve its type.
+    if (IncI->getNumOperands() == 2)
+      break;
+  default:
+    return 0;
+  }
+
+  PHINode *Phi = dyn_cast<PHINode>(IncI->getOperand(0));
+  if (Phi && Phi->getParent() == L->getHeader()) {
+    if (isLoopInvariant(IncI->getOperand(1), L, DT))
+      return Phi;
+    return 0;
+  }
+  if (IncI->getOpcode() == Instruction::GetElementPtr)
+    return 0;
+
+  // Allow add/sub to be commuted.
+  Phi = dyn_cast<PHINode>(IncI->getOperand(1));
+  if (Phi && Phi->getParent() == L->getHeader()) {
+    if (isLoopInvariant(IncI->getOperand(0), L, DT))
+      return Phi;
+  }
+  return 0;
+}
+
+/// needsLFTR - LinearFunctionTestReplace policy. Return true unless we can show
+/// that the current exit test is already sufficiently canonical.
+static bool needsLFTR(Loop *L, DominatorTree *DT) {
+  assert(L->getExitingBlock() && "expected loop exit");
+
+  BasicBlock *LatchBlock = L->getLoopLatch();
+  // Don't bother with LFTR if the loop is not properly simplified.
+  if (!LatchBlock)
+    return false;
+
+  BranchInst *BI = dyn_cast<BranchInst>(L->getExitingBlock()->getTerminator());
+  assert(BI && "expected exit branch");
+
+  // Do LFTR to simplify the exit condition to an ICMP.
+  ICmpInst *Cond = dyn_cast<ICmpInst>(BI->getCondition());
+  if (!Cond)
+    return true;
+
+  // Do LFTR to simplify the exit ICMP to EQ/NE
+  ICmpInst::Predicate Pred = Cond->getPredicate();
+  if (Pred != ICmpInst::ICMP_NE && Pred != ICmpInst::ICMP_EQ)
+    return true;
+
+  // Look for a loop invariant RHS
+  Value *LHS = Cond->getOperand(0);
+  Value *RHS = Cond->getOperand(1);
+  if (!isLoopInvariant(RHS, L, DT)) {
+    if (!isLoopInvariant(LHS, L, DT))
+      return true;
+    std::swap(LHS, RHS);
+  }
+  // Look for a simple IV counter LHS
+  PHINode *Phi = dyn_cast<PHINode>(LHS);
+  if (!Phi)
+    Phi = getLoopPhiForCounter(LHS, L, DT);
+
+  if (!Phi)
+    return true;
+
+  // Do LFTR if the exit condition's IV is *not* a simple counter.
+  Value *IncV = Phi->getIncomingValueForBlock(L->getLoopLatch());
+  return Phi != getLoopPhiForCounter(IncV, L, DT);
+}
+
+/// AlmostDeadIV - Return true if this IV has any uses other than the (soon to
+/// be rewritten) loop exit test.
+static bool AlmostDeadIV(PHINode *Phi, BasicBlock *LatchBlock, Value *Cond) {
+  int LatchIdx = Phi->getBasicBlockIndex(LatchBlock);
+  Value *IncV = Phi->getIncomingValue(LatchIdx);
+
+  for (Value::use_iterator UI = Phi->use_begin(), UE = Phi->use_end();
+       UI != UE; ++UI) {
+    if (*UI != Cond && *UI != IncV) return false;
+  }
+
+  for (Value::use_iterator UI = IncV->use_begin(), UE = IncV->use_end();
+       UI != UE; ++UI) {
+    if (*UI != Cond && *UI != Phi) return false;
+  }
+  return true;
+}
+
+/// FindLoopCounter - Find an affine IV in canonical form.
+///
+/// FIXME: Accept -1 stride and set IVLimit = IVInit - BECount
+///
+/// FIXME: Accept non-unit stride as long as SCEV can reduce BECount * Stride.
+/// This is difficult in general for SCEV because of potential overflow. But we
+/// could at least handle constant BECounts.
+static PHINode *
+FindLoopCounter(Loop *L, const SCEV *BECount,
+                ScalarEvolution *SE, DominatorTree *DT, const TargetData *TD) {
+  // I'm not sure how BECount could be a pointer type, but we definitely don't
+  // want to LFTR that.
+  if (BECount->getType()->isPointerTy())
+    return 0;
+
+  uint64_t BCWidth = SE->getTypeSizeInBits(BECount->getType());
+
+  Value *Cond =
+    cast<BranchInst>(L->getExitingBlock()->getTerminator())->getCondition();
+
+  // Loop over all of the PHI nodes, looking for a simple counter.
+  PHINode *BestPhi = 0;
+  const SCEV *BestInit = 0;
+  BasicBlock *LatchBlock = L->getLoopLatch();
+  assert(LatchBlock && "needsLFTR should guarantee a loop latch");
+
+  for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
+    PHINode *Phi = cast<PHINode>(I);
+    if (!SE->isSCEVable(Phi->getType()))
+      continue;
+
+    const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(Phi));
+    if (!AR || AR->getLoop() != L || !AR->isAffine())
+      continue;
+
+    // AR may be a pointer type, while BECount is an integer type.
+    // AR may be wider than BECount. With eq/ne tests overflow is immaterial.
+    // AR may not be a narrower type, or we may never exit.
+    uint64_t PhiWidth = SE->getTypeSizeInBits(AR->getType());
+    if (PhiWidth < BCWidth || (TD && !TD->isLegalInteger(PhiWidth)))
+      continue;
+
+    const SCEV *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE));
+    if (!Step || !Step->isOne())
+      continue;
+
+    int LatchIdx = Phi->getBasicBlockIndex(LatchBlock);
+    Value *IncV = Phi->getIncomingValue(LatchIdx);
+    if (getLoopPhiForCounter(IncV, L, DT) != Phi)
+      continue;
+
+    const SCEV *Init = AR->getStart();
+
+    if (BestPhi && !AlmostDeadIV(BestPhi, LatchBlock, Cond)) {
+      // Don't force a live loop counter if another IV can be used.
+      if (AlmostDeadIV(Phi, LatchBlock, Cond))
+        continue;
+
+      // Prefer to count-from-zero. This is a more "canonical" counter form. It
+      // also prefers integer to pointer IVs.
+      if (BestInit->isZero() != Init->isZero()) {
+        if (BestInit->isZero())
+          continue;
+      }
+      // If two IVs both count from zero or both count from nonzero then the
+      // narrower is likely a dead phi that has been widened. Use the wider phi
+      // to allow the other to be eliminated.
+      if (PhiWidth <= SE->getTypeSizeInBits(BestPhi->getType()))
+        continue;
+    }
+    BestPhi = Phi;
+    BestInit = Init;
+  }
+  return BestPhi;
+}
+
 /// LinearFunctionTestReplace - This method rewrites the exit condition of the
 /// loop to be a canonical != comparison against the incremented loop induction
 /// variable.  This pass is able to rewrite the exit tests of any loop where the
 /// SCEV analysis can determine a loop-invariant trip count of the loop, which
 /// is actually a much broader range than just linear tests.
-ICmpInst *IndVarSimplify::
+Value *IndVarSimplify::
 LinearFunctionTestReplace(Loop *L,
                           const SCEV *BackedgeTakenCount,
                           PHINode *IndVar,
@@ -1528,62 +1516,117 @@ LinearFunctionTestReplace(Loop *L,
   assert(canExpandBackedgeTakenCount(L, SE) && "precondition");
   BranchInst *BI = cast<BranchInst>(L->getExitingBlock()->getTerminator());
 
+  // LFTR can ignore IV overflow and truncate to the width of
+  // BECount. This avoids materializing the add(zext(add)) expression.
+  Type *CntTy = !EnableIVRewrite ?
+    BackedgeTakenCount->getType() : IndVar->getType();
+
+  const SCEV *IVLimit = BackedgeTakenCount;
+
   // If the exiting block is not the same as the backedge block, we must compare
   // against the preincremented value, otherwise we prefer to compare against
   // the post-incremented value.
   Value *CmpIndVar;
-  const SCEV *RHS = BackedgeTakenCount;
   if (L->getExitingBlock() == L->getLoopLatch()) {
     // Add one to the "backedge-taken" count to get the trip count.
     // If this addition may overflow, we have to be more pessimistic and
     // cast the induction variable before doing the add.
-    const SCEV *Zero = SE->getConstant(BackedgeTakenCount->getType(), 0);
     const SCEV *N =
-      SE->getAddExpr(BackedgeTakenCount,
-                     SE->getConstant(BackedgeTakenCount->getType(), 1));
-    if ((isa<SCEVConstant>(N) && !N->isZero()) ||
-        SE->isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, N, Zero)) {
-      // No overflow. Cast the sum.
-      RHS = SE->getTruncateOrZeroExtend(N, IndVar->getType());
-    } else {
-      // Potential overflow. Cast before doing the add.
-      RHS = SE->getTruncateOrZeroExtend(BackedgeTakenCount,
-                                        IndVar->getType());
-      RHS = SE->getAddExpr(RHS,
-                           SE->getConstant(IndVar->getType(), 1));
+      SE->getAddExpr(IVLimit, SE->getConstant(IVLimit->getType(), 1));
+    if (CntTy == IVLimit->getType())
+      IVLimit = N;
+    else {
+      const SCEV *Zero = SE->getConstant(IVLimit->getType(), 0);
+      if ((isa<SCEVConstant>(N) && !N->isZero()) ||
+          SE->isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, N, Zero)) {
+        // No overflow. Cast the sum.
+        IVLimit = SE->getTruncateOrZeroExtend(N, CntTy);
+      } else {
+        // Potential overflow. Cast before doing the add.
+        IVLimit = SE->getTruncateOrZeroExtend(IVLimit, CntTy);
+        IVLimit = SE->getAddExpr(IVLimit, SE->getConstant(CntTy, 1));
+      }
     }
-
     // The BackedgeTaken expression contains the number of times that the
     // backedge branches to the loop header.  This is one less than the
     // number of times the loop executes, so use the incremented indvar.
     CmpIndVar = IndVar->getIncomingValueForBlock(L->getExitingBlock());
   } else {
     // We have to use the preincremented value...
-    RHS = SE->getTruncateOrZeroExtend(BackedgeTakenCount,
-                                      IndVar->getType());
+    IVLimit = SE->getTruncateOrZeroExtend(IVLimit, CntTy);
     CmpIndVar = IndVar;
   }
 
+  // For unit stride, IVLimit = Start + BECount with 2's complement overflow.
+  // So for, non-zero start compute the IVLimit here.
+  bool isPtrIV = false;
+  Type *CmpTy = CntTy;
+  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(IndVar));
+  assert(AR && AR->getLoop() == L && AR->isAffine() && "bad loop counter");
+  if (!AR->getStart()->isZero()) {
+    assert(AR->getStepRecurrence(*SE)->isOne() && "only handles unit stride");
+    const SCEV *IVInit = AR->getStart();
+
+    // For pointer types, sign extend BECount in order to materialize a GEP.
+    // Note that for without EnableIVRewrite, we never run SCEVExpander on a
+    // pointer type, because we must preserve the existing GEPs. Instead we
+    // directly generate a GEP later.
+    if (IVInit->getType()->isPointerTy()) {
+      isPtrIV = true;
+      CmpTy = SE->getEffectiveSCEVType(IVInit->getType());
+      IVLimit = SE->getTruncateOrSignExtend(IVLimit, CmpTy);
+    }
+    // For integer types, truncate the IV before computing IVInit + BECount.
+    else {
+      if (SE->getTypeSizeInBits(IVInit->getType())
+          > SE->getTypeSizeInBits(CmpTy))
+        IVInit = SE->getTruncateExpr(IVInit, CmpTy);
+
+      IVLimit = SE->getAddExpr(IVInit, IVLimit);
+    }
+  }
   // Expand the code for the iteration count.
-  assert(SE->isLoopInvariant(RHS, L) &&
+  IRBuilder<> Builder(BI);
+
+  assert(SE->isLoopInvariant(IVLimit, L) &&
          "Computed iteration count is not loop invariant!");
-  Value *ExitCnt = Rewriter.expandCodeFor(RHS, IndVar->getType(), BI);
+  Value *ExitCnt = Rewriter.expandCodeFor(IVLimit, CmpTy, BI);
+
+  // Create a gep for IVInit + IVLimit from on an existing pointer base.
+  assert(isPtrIV == IndVar->getType()->isPointerTy() &&
+         "IndVar type must match IVInit type");
+  if (isPtrIV) {
+      Value *IVStart = IndVar->getIncomingValueForBlock(L->getLoopPreheader());
+      assert(AR->getStart() == SE->getSCEV(IVStart) && "bad loop counter");
+      assert(SE->getSizeOfExpr(
+               cast<PointerType>(IVStart->getType())->getElementType())->isOne()
+             && "unit stride pointer IV must be i8*");
+
+      Builder.SetInsertPoint(L->getLoopPreheader()->getTerminator());
+      ExitCnt = Builder.CreateGEP(IVStart, ExitCnt, "lftr.limit");
+      Builder.SetInsertPoint(BI);
+  }
 
   // Insert a new icmp_ne or icmp_eq instruction before the branch.
-  ICmpInst::Predicate Opcode;
+  ICmpInst::Predicate P;
   if (L->contains(BI->getSuccessor(0)))
-    Opcode = ICmpInst::ICMP_NE;
+    P = ICmpInst::ICMP_NE;
   else
-    Opcode = ICmpInst::ICMP_EQ;
+    P = ICmpInst::ICMP_EQ;
 
   DEBUG(dbgs() << "INDVARS: Rewriting loop exit condition to:\n"
                << "      LHS:" << *CmpIndVar << '\n'
                << "       op:\t"
-               << (Opcode == ICmpInst::ICMP_NE ? "!=" : "==") << "\n"
-               << "      RHS:\t" << *RHS << "\n");
+               << (P == ICmpInst::ICMP_NE ? "!=" : "==") << "\n"
+               << "      RHS:\t" << *ExitCnt << "\n"
+               << "     Expr:\t" << *IVLimit << "\n");
 
-  ICmpInst *Cond = new ICmpInst(BI, Opcode, CmpIndVar, ExitCnt, "exitcond");
-  Cond->setDebugLoc(BI->getDebugLoc());
+  if (SE->getTypeSizeInBits(CmpIndVar->getType())
+      > SE->getTypeSizeInBits(CmpTy)) {
+    CmpIndVar = Builder.CreateTrunc(CmpIndVar, CmpTy, "lftr.wideiv");
+  }
+
+  Value *Cond = Builder.CreateICmp(P, CmpIndVar, ExitCnt, "exitcond");
   Value *OrigCond = BI->getCondition();
   // It's tempting to use replaceAllUsesWith here to fully replace the old
   // comparison, but that's not immediately safe, since users of the old
@@ -1612,7 +1655,7 @@ void IndVarSimplify::SinkUnusedInvariants(Loop *L) {
   BasicBlock *Preheader = L->getLoopPreheader();
   if (!Preheader) return;
 
-  Instruction *InsertPt = ExitBlock->getFirstNonPHI();
+  Instruction *InsertPt = ExitBlock->getFirstInsertionPt();
   BasicBlock::iterator I = Preheader->getTerminator();
   while (I != Preheader->begin()) {
     --I;
@@ -1633,6 +1676,10 @@ void IndVarSimplify::SinkUnusedInvariants(Loop *L) {
     if (isa<DbgInfoIntrinsic>(I))
       continue;
 
+    // Skip landingpad instructions.
+    if (isa<LandingPadInst>(I))
+      continue;
+
     // Don't sink static AllocaInsts out of the entry block, which would
     // turn them into dynamic allocas!
     if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
@@ -1699,7 +1746,7 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   if (!L->isLoopSimplifyForm())
     return false;
 
-  if (!DisableIVRewrite)
+  if (EnableIVRewrite)
     IU = &getAnalysis<IVUsers>();
   LI = &getAnalysis<LoopInfo>();
   SE = &getAnalysis<ScalarEvolution>();
@@ -1717,6 +1764,9 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
 
   // Create a rewriter object which we'll use to transform the code with.
   SCEVExpander Rewriter(*SE, "indvars");
+#ifndef NDEBUG
+  Rewriter.setDebugType(DEBUG_TYPE);
+#endif
 
   // Eliminate redundant IV users.
   //
@@ -1724,9 +1774,9 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   // attempt to avoid evaluating SCEVs for sign/zero extend operations until
   // other expressions involving loop IVs have been evaluated. This helps SCEV
   // set no-wrap flags before normalizing sign/zero extension.
-  if (DisableIVRewrite) {
+  if (!EnableIVRewrite) {
     Rewriter.disableCanonicalMode();
-    SimplifyIVUsersNoRewrite(L, Rewriter);
+    SimplifyAndExtend(L, Rewriter, LPM);
   }
 
   // Check to see if this loop has a computable loop-invariant execution count.
@@ -1739,25 +1789,25 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
     RewriteLoopExitValues(L, Rewriter);
 
   // Eliminate redundant IV users.
-  if (!DisableIVRewrite)
-    SimplifyIVUsers(Rewriter);
+  if (EnableIVRewrite)
+    Changed |= simplifyIVUsers(IU, SE, &LPM, DeadInsts);
 
   // Eliminate redundant IV cycles.
-  if (DisableIVRewrite)
-    SimplifyCongruentIVs(L);
+  if (!EnableIVRewrite)
+    NumElimIV += Rewriter.replaceCongruentIVs(L, DT, DeadInsts);
 
   // Compute the type of the largest recurrence expression, and decide whether
   // a canonical induction variable should be inserted.
-  const Type *LargestType = 0;
+  Type *LargestType = 0;
   bool NeedCannIV = false;
   bool ExpandBECount = canExpandBackedgeTakenCount(L, SE);
-  if (ExpandBECount) {
+  if (EnableIVRewrite && ExpandBECount) {
     // If we have a known trip count and a single exit block, we'll be
     // rewriting the loop exit test condition below, which requires a
     // canonical induction variable.
     NeedCannIV = true;
-    const Type *Ty = BackedgeTakenCount->getType();
-    if (DisableIVRewrite) {
+    Type *Ty = BackedgeTakenCount->getType();
+    if (!EnableIVRewrite) {
       // In this mode, SimplifyIVUsers may have already widened the IV used by
       // the backedge test and inserted a Trunc on the compare's operand. Get
       // the wider type to avoid creating a redundant narrow IV only used by the
@@ -1769,10 +1819,10 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
         SE->getTypeSizeInBits(LargestType))
       LargestType = SE->getEffectiveSCEVType(Ty);
   }
-  if (!DisableIVRewrite) {
+  if (EnableIVRewrite) {
     for (IVUsers::const_iterator I = IU->begin(), E = IU->end(); I != E; ++I) {
       NeedCannIV = true;
-      const Type *Ty =
+      Type *Ty =
         SE->getEffectiveSCEVType(I->getOperandValToReplace()->getType());
       if (!LargestType ||
           SE->getTypeSizeInBits(Ty) >
@@ -1811,18 +1861,16 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
     // the end of the pass.
     while (!OldCannIVs.empty()) {
       PHINode *OldCannIV = OldCannIVs.pop_back_val();
-      OldCannIV->insertBefore(L->getHeader()->getFirstNonPHI());
+      OldCannIV->insertBefore(L->getHeader()->getFirstInsertionPt());
     }
   }
-
+  else if (!EnableIVRewrite && ExpandBECount && needsLFTR(L, DT)) {
+    IndVar = FindLoopCounter(L, BackedgeTakenCount, SE, DT, TD);
+  }
   // If we have a trip count expression, rewrite the loop's exit condition
   // using it.  We can currently only handle loops with a single exit.
-  ICmpInst *NewICmp = 0;
-  if (ExpandBECount) {
-    assert(canExpandBackedgeTakenCount(L, SE) &&
-           "canonical IV disrupted BackedgeTaken expansion");
-    assert(NeedCannIV &&
-           "LinearFunctionTestReplace requires a canonical induction variable");
+  Value *NewICmp = 0;
+  if (ExpandBECount && IndVar) {
     // Check preconditions for proper SCEVExpander operation. SCEV does not
     // express SCEVExpander's dependencies, such as LoopSimplify. Instead any
     // pass that uses the SCEVExpander must do it. This does not work well for
@@ -1837,7 +1885,7 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
         LinearFunctionTestReplace(L, BackedgeTakenCount, IndVar, Rewriter);
   }
   // Rewrite IV-derived expressions.
-  if (!DisableIVRewrite)
+  if (EnableIVRewrite)
     RewriteIVExpressions(L, Rewriter);
 
   // Clear the rewriter cache, because values that are in the rewriter's cache
@@ -1860,12 +1908,34 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
 
   // For completeness, inform IVUsers of the IV use in the newly-created
   // loop exit test instruction.
-  if (NewICmp && IU)
-    IU->AddUsersIfInteresting(cast<Instruction>(NewICmp->getOperand(0)));
-
+  if (IU && NewICmp) {
+    ICmpInst *NewICmpInst = dyn_cast<ICmpInst>(NewICmp);
+    if (NewICmpInst)
+      IU->AddUsersIfInteresting(cast<Instruction>(NewICmpInst->getOperand(0)));
+  }
   // Clean up dead instructions.
   Changed |= DeleteDeadPHIs(L->getHeader());
   // Check a post-condition.
-  assert(L->isLCSSAForm(*DT) && "Indvars did not leave the loop in lcssa form!");
+  assert(L->isLCSSAForm(*DT) &&
+         "Indvars did not leave the loop in lcssa form!");
+
+  // Verify that LFTR, and any other change have not interfered with SCEV's
+  // ability to compute trip count.
+#ifndef NDEBUG
+  if (!EnableIVRewrite && VerifyIndvars &&
+      !isa<SCEVCouldNotCompute>(BackedgeTakenCount)) {
+    SE->forgetLoop(L);
+    const SCEV *NewBECount = SE->getBackedgeTakenCount(L);
+    if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) <
+        SE->getTypeSizeInBits(NewBECount->getType()))
+      NewBECount = SE->getTruncateOrNoop(NewBECount,
+                                         BackedgeTakenCount->getType());
+    else
+      BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount,
+                                                 NewBECount->getType());
+    assert(BackedgeTakenCount == NewBECount && "indvars must preserve SCEV");
+  }
+#endif
+
   return Changed;
 }
diff --git a/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index b500d5b4fdff..f410af32759c 100644
--- a/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -811,8 +811,8 @@ bool JumpThreading::ProcessBlock(BasicBlock *BB) {
 /// important optimization that encourages jump threading, and needs to be run
 /// interlaced with other jump threading tasks.
 bool JumpThreading::SimplifyPartiallyRedundantLoad(LoadInst *LI) {
-  // Don't hack volatile loads.
-  if (LI->isVolatile()) return false;
+  // Don't hack volatile/atomic loads.
+  if (!LI->isSimple()) return false;
 
   // If the load is defined in a block with exactly one predecessor, it can't be
   // partially redundant.
diff --git a/contrib/llvm/lib/Transforms/Scalar/LICM.cpp b/contrib/llvm/lib/Transforms/Scalar/LICM.cpp
index 66add6ca01ee..b79bb1300fec 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -151,6 +151,11 @@ namespace {
     ///
     bool isSafeToExecuteUnconditionally(Instruction &I);
 
+    /// isGuaranteedToExecute - Check that the instruction is guaranteed to
+    /// execute.
+    ///
+    bool isGuaranteedToExecute(Instruction &I);
+
     /// pointerInvalidatedByLoop - Return true if the body of this loop may
     /// store into the memory location pointed to by V.
     ///
@@ -357,8 +362,8 @@ void LICM::HoistRegion(DomTreeNode *N) {
 bool LICM::canSinkOrHoistInst(Instruction &I) {
   // Loads have extra constraints we have to verify before we can hoist them.
   if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
-    if (LI->isVolatile())
-      return false;        // Don't hoist volatile loads!
+    if (!LI->isUnordered())
+      return false;        // Don't hoist volatile/atomic loads!
 
     // Loads from constant memory are always safe to move, even if they end up
     // in the same alias set as something that ends up being modified.
@@ -461,7 +466,7 @@ void LICM::sink(Instruction &I) {
     } else {
       // Move the instruction to the start of the exit block, after any PHI
       // nodes in it.
-      I.moveBefore(ExitBlocks[0]->getFirstNonPHI());
+      I.moveBefore(ExitBlocks[0]->getFirstInsertionPt());
 
       // This instruction is no longer in the AST for the current loop, because
       // we just sunk it out of the loop.  If we just sunk it into an outer
@@ -504,7 +509,7 @@ void LICM::sink(Instruction &I) {
       continue;
 
     // Insert the code after the last PHI node.
-    BasicBlock::iterator InsertPt = ExitBlock->getFirstNonPHI();
+    BasicBlock::iterator InsertPt = ExitBlock->getFirstInsertionPt();
 
     // If this is the first exit block processed, just move the original
     // instruction, otherwise clone the original instruction and insert
@@ -577,6 +582,10 @@ bool LICM::isSafeToExecuteUnconditionally(Instruction &Inst) {
   if (Inst.isSafeToSpeculativelyExecute())
     return true;
 
+  return isGuaranteedToExecute(Inst);
+}
+
+bool LICM::isGuaranteedToExecute(Instruction &Inst) {
   // Otherwise we have to check to make sure that the instruction dominates all
   // of the exit blocks.  If it doesn't, then there is a path out of the loop
   // which does not execute this instruction, so we can't hoist it.
@@ -635,7 +644,7 @@ namespace {
       for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) {
         BasicBlock *ExitBlock = LoopExitBlocks[i];
         Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
-        Instruction *InsertPos = ExitBlock->getFirstNonPHI();
+        Instruction *InsertPos = ExitBlock->getFirstInsertionPt();
         StoreInst *NewSI = new StoreInst(LiveInValue, SomePtr, InsertPos);
         NewSI->setAlignment(Alignment);
         NewSI->setDebugLoc(DL);
@@ -713,34 +722,41 @@ void LICM::PromoteAliasSet(AliasSet &AS) {
 
       // If there is an non-load/store instruction in the loop, we can't promote
       // it.
-      unsigned InstAlignment;
       if (LoadInst *load = dyn_cast<LoadInst>(Use)) {
-        assert(!cast<LoadInst>(Use)->isVolatile() && "AST broken");
-        InstAlignment = load->getAlignment();
+        assert(!load->isVolatile() && "AST broken");
+        if (!load->isSimple())
+          return;
       } else if (StoreInst *store = dyn_cast<StoreInst>(Use)) {
         // Stores *of* the pointer are not interesting, only stores *to* the
         // pointer.
         if (Use->getOperand(1) != ASIV)
           continue;
-        InstAlignment = store->getAlignment();
-        assert(!cast<StoreInst>(Use)->isVolatile() && "AST broken");
+        assert(!store->isVolatile() && "AST broken");
+        if (!store->isSimple())
+          return;
+
+        // Note that we only check GuaranteedToExecute inside the store case
+        // so that we do not introduce stores where they did not exist before
+        // (which would break the LLVM concurrency model).
+
+        // If the alignment of this instruction allows us to specify a more
+        // restrictive (and performant) alignment and if we are sure this
+        // instruction will be executed, update the alignment.
+        // Larger is better, with the exception of 0 being the best alignment.
+        unsigned InstAlignment = store->getAlignment();
+        if ((InstAlignment > Alignment || InstAlignment == 0)
+            && (Alignment != 0))
+          if (isGuaranteedToExecute(*Use)) {
+            GuaranteedToExecute = true;
+            Alignment = InstAlignment;
+          }
+
+        if (!GuaranteedToExecute)
+          GuaranteedToExecute = isGuaranteedToExecute(*Use);
+
       } else
         return; // Not a load or store.
 
-      // If the alignment of this instruction allows us to specify a more
-      // restrictive (and performant) alignment and if we are sure this
-      // instruction will be executed, update the alignment.
-      // Larger is better, with the exception of 0 being the best alignment.
-      if ((InstAlignment > Alignment || InstAlignment == 0)
-          && (Alignment != 0))
-        if (isSafeToExecuteUnconditionally(*Use)) {
-          GuaranteedToExecute = true;
-          Alignment = InstAlignment;
-        }
-
-      if (!GuaranteedToExecute)
-        GuaranteedToExecute = isSafeToExecuteUnconditionally(*Use);
-
       LoopUses.push_back(Use);
     }
   }
diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index a0e41d9a9772..ad15cbb9b436 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -267,7 +267,7 @@ bool LoopIdiomRecognize::runOnLoopBlock(BasicBlock *BB, const SCEV *BECount,
 
 /// processLoopStore - See if this store can be promoted to a memset or memcpy.
 bool LoopIdiomRecognize::processLoopStore(StoreInst *SI, const SCEV *BECount) {
-  if (SI->isVolatile()) return false;
+  if (!SI->isSimple()) return false;
 
   Value *StoredVal = SI->getValueOperand();
   Value *StorePtr = SI->getPointerOperand();
@@ -314,7 +314,7 @@ bool LoopIdiomRecognize::processLoopStore(StoreInst *SI, const SCEV *BECount) {
     const SCEVAddRecExpr *LoadEv =
       dyn_cast<SCEVAddRecExpr>(SE->getSCEV(LI->getOperand(0)));
     if (LoadEv && LoadEv->getLoop() == CurLoop && LoadEv->isAffine() &&
-        StoreEv->getOperand(1) == LoadEv->getOperand(1) && !LI->isVolatile())
+        StoreEv->getOperand(1) == LoadEv->getOperand(1) && LI->isSimple())
       if (processLoopStoreOfLoopLoad(SI, StoreSize, StoreEv, LoadEv, BECount))
         return true;
   }
@@ -463,7 +463,7 @@ processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
     SplatValue = 0;
   } else {
     // Otherwise, this isn't an idiom we can transform.  For example, we can't
-    // do anything with a 3-byte store, for example.
+    // do anything with a 3-byte store.
     return false;
   }
 
@@ -498,7 +498,7 @@ processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
 
   // The # stored bytes is (BECount+1)*Size.  Expand the trip count out to
   // pointer size if it isn't already.
-  const Type *IntPtr = TD->getIntPtrType(DestPtr->getContext());
+  Type *IntPtr = TD->getIntPtrType(DestPtr->getContext());
   BECount = SE->getTruncateOrZeroExtend(BECount, IntPtr);
 
   const SCEV *NumBytesS = SE->getAddExpr(BECount, SE->getConstant(IntPtr, 1),
@@ -604,7 +604,7 @@ processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
 
   // The # stored bytes is (BECount+1)*Size.  Expand the trip count out to
   // pointer size if it isn't already.
-  const Type *IntPtr = TD->getIntPtrType(SI->getContext());
+  Type *IntPtr = TD->getIntPtrType(SI->getContext());
   BECount = SE->getTruncateOrZeroExtend(BECount, IntPtr);
 
   const SCEV *NumBytesS = SE->getAddExpr(BECount, SE->getConstant(IntPtr, 1),
diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 509d0264f10b..3e122c2a866e 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -70,12 +70,27 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ValueHandle.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetLowering.h"
 #include <algorithm>
 using namespace llvm;
 
+namespace llvm {
+cl::opt<bool> EnableNested(
+  "enable-lsr-nested", cl::Hidden, cl::desc("Enable LSR on nested loops"));
+
+cl::opt<bool> EnableRetry(
+    "enable-lsr-retry", cl::Hidden, cl::desc("Enable LSR retry"));
+
+// Temporary flag to cleanup congruent phis after LSR phi expansion.
+// It's currently disabled until we can determine whether it's truly useful or
+// not. The flag should be removed after the v3.0 release.
+cl::opt<bool> EnablePhiElim(
+    "enable-lsr-phielim", cl::Hidden, cl::desc("Enable LSR phi elimination"));
+}
+
 namespace {
 
 /// RegSortData - This class holds data which is used to order reuse candidates.
@@ -219,7 +234,7 @@ struct Formula {
   void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
 
   unsigned getNumRegs() const;
-  const Type *getType() const;
+  Type *getType() const;
 
   void DeleteBaseReg(const SCEV *&S);
 
@@ -319,7 +334,7 @@ unsigned Formula::getNumRegs() const {
 
 /// getType - Return the type of this formula, if it has one, or null
 /// otherwise. This type is meaningless except for the bit size.
-const Type *Formula::getType() const {
+Type *Formula::getType() const {
   return !BaseRegs.empty() ? BaseRegs.front()->getType() :
          ScaledReg ? ScaledReg->getType() :
          AM.BaseGV ? AM.BaseGV->getType() :
@@ -397,7 +412,7 @@ void Formula::dump() const {
 /// isAddRecSExtable - Return true if the given addrec can be sign-extended
 /// without changing its value.
 static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
-  const Type *WideTy =
+  Type *WideTy =
     IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1);
   return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
 }
@@ -405,7 +420,7 @@ static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
 /// isAddSExtable - Return true if the given add can be sign-extended
 /// without changing its value.
 static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
-  const Type *WideTy =
+  Type *WideTy =
     IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1);
   return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy));
 }
@@ -413,7 +428,7 @@ static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
 /// isMulSExtable - Return true if the given mul can be sign-extended
 /// without changing its value.
 static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
-  const Type *WideTy =
+  Type *WideTy =
     IntegerType::get(SE.getContext(),
                      SE.getTypeSizeInBits(M->getType()) * M->getNumOperands());
   return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy));
@@ -594,8 +609,8 @@ static bool isAddressUse(Instruction *Inst, Value *OperandVal) {
 }
 
 /// getAccessType - Return the type of the memory being accessed.
-static const Type *getAccessType(const Instruction *Inst) {
-  const Type *AccessTy = Inst->getType();
+static Type *getAccessType(const Instruction *Inst) {
+  Type *AccessTy = Inst->getType();
   if (const StoreInst *SI = dyn_cast<StoreInst>(Inst))
     AccessTy = SI->getOperand(0)->getType();
   else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
@@ -614,7 +629,7 @@ static const Type *getAccessType(const Instruction *Inst) {
 
   // All pointers have the same requirements, so canonicalize them to an
   // arbitrary pointer type to minimize variation.
-  if (const PointerType *PTy = dyn_cast<PointerType>(AccessTy))
+  if (PointerType *PTy = dyn_cast<PointerType>(AccessTy))
     AccessTy = PointerType::get(IntegerType::get(PTy->getContext(), 1),
                                 PTy->getAddressSpace());
 
@@ -670,6 +685,21 @@ class Cost {
 
   void Loose();
 
+#ifndef NDEBUG
+  // Once any of the metrics loses, they must all remain losers.
+  bool isValid() {
+    return ((NumRegs | AddRecCost | NumIVMuls | NumBaseAdds
+             | ImmCost | SetupCost) != ~0u)
+      || ((NumRegs & AddRecCost & NumIVMuls & NumBaseAdds
+           & ImmCost & SetupCost) == ~0u);
+  }
+#endif
+
+  bool isLoser() {
+    assert(isValid() && "invalid cost");
+    return NumRegs == ~0u;
+  }
+
   void RateFormula(const Formula &F,
                    SmallPtrSet<const SCEV *, 16> &Regs,
                    const DenseSet<const SCEV *> &VisitedRegs,
@@ -702,34 +732,48 @@ void Cost::RateRegister(const SCEV *Reg,
     if (AR->getLoop() == L)
       AddRecCost += 1; /// TODO: This should be a function of the stride.
 
-    // If this is an addrec for a loop that's already been visited by LSR,
-    // don't second-guess its addrec phi nodes. LSR isn't currently smart
-    // enough to reason about more than one loop at a time. Consider these
-    // registers free and leave them alone.
-    else if (L->contains(AR->getLoop()) ||
+    // If this is an addrec for another loop, don't second-guess its addrec phi
+    // nodes. LSR isn't currently smart enough to reason about more than one
+    // loop at a time. LSR has either already run on inner loops, will not run
+    // on other loops, and cannot be expected to change sibling loops. If the
+    // AddRec exists, consider it's register free and leave it alone. Otherwise,
+    // do not consider this formula at all.
+    // FIXME: why do we need to generate such fomulae?
+    else if (!EnableNested || L->contains(AR->getLoop()) ||
              (!AR->getLoop()->contains(L) &&
               DT.dominates(L->getHeader(), AR->getLoop()->getHeader()))) {
       for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin();
-           PHINode *PN = dyn_cast<PHINode>(I); ++I)
+           PHINode *PN = dyn_cast<PHINode>(I); ++I) {
         if (SE.isSCEVable(PN->getType()) &&
             (SE.getEffectiveSCEVType(PN->getType()) ==
              SE.getEffectiveSCEVType(AR->getType())) &&
             SE.getSCEV(PN) == AR)
           return;
-
+      }
+      if (!EnableNested) {
+        Loose();
+        return;
+      }
       // If this isn't one of the addrecs that the loop already has, it
       // would require a costly new phi and add. TODO: This isn't
       // precisely modeled right now.
       ++NumBaseAdds;
-      if (!Regs.count(AR->getStart()))
+      if (!Regs.count(AR->getStart())) {
         RateRegister(AR->getStart(), Regs, L, SE, DT);
+        if (isLoser())
+          return;
+      }
     }
 
     // Add the step value register, if it needs one.
     // TODO: The non-affine case isn't precisely modeled here.
-    if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1)))
-      if (!Regs.count(AR->getStart()))
+    if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) {
+      if (!Regs.count(AR->getOperand(1))) {
         RateRegister(AR->getOperand(1), Regs, L, SE, DT);
+        if (isLoser())
+          return;
+      }
+    }
   }
   ++NumRegs;
 
@@ -769,6 +813,8 @@ void Cost::RateFormula(const Formula &F,
       return;
     }
     RatePrimaryRegister(ScaledReg, Regs, L, SE, DT);
+    if (isLoser())
+      return;
   }
   for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
        E = F.BaseRegs.end(); I != E; ++I) {
@@ -778,6 +824,8 @@ void Cost::RateFormula(const Formula &F,
       return;
     }
     RatePrimaryRegister(BaseReg, Regs, L, SE, DT);
+    if (isLoser())
+      return;
   }
 
   // Determine how many (unfolded) adds we'll need inside the loop.
@@ -795,6 +843,7 @@ void Cost::RateFormula(const Formula &F,
     else if (Offset != 0)
       ImmCost += APInt(64, Offset, true).getMinSignedBits();
   }
+  assert(isValid() && "invalid cost");
 }
 
 /// Loose - Set this cost to a losing value.
@@ -980,7 +1029,7 @@ class LSRUse {
   };
 
   KindType Kind;
-  const Type *AccessTy;
+  Type *AccessTy;
 
   SmallVector<int64_t, 8> Offsets;
   int64_t MinOffset;
@@ -995,7 +1044,7 @@ class LSRUse {
   /// this LSRUse. FindUseWithSimilarFormula can't consider uses with different
   /// max fixup widths to be equivalent, because the narrower one may be relying
   /// on the implicit truncation to truncate away bogus bits.
-  const Type *WidestFixupType;
+  Type *WidestFixupType;
 
   /// Formulae - A list of ways to build a value that can satisfy this user.
   /// After the list is populated, one of these is selected heuristically and
@@ -1005,7 +1054,7 @@ class LSRUse {
   /// Regs - The set of register candidates used by all formulae in this LSRUse.
   SmallPtrSet<const SCEV *, 4> Regs;
 
-  LSRUse(KindType K, const Type *T) : Kind(K), AccessTy(T),
+  LSRUse(KindType K, Type *T) : Kind(K), AccessTy(T),
                                       MinOffset(INT64_MAX),
                                       MaxOffset(INT64_MIN),
                                       AllFixupsOutsideLoop(true),
@@ -1127,7 +1176,7 @@ void LSRUse::dump() const {
 /// be completely folded into the user instruction at isel time. This includes
 /// address-mode folding and special icmp tricks.
 static bool isLegalUse(const TargetLowering::AddrMode &AM,
-                       LSRUse::KindType Kind, const Type *AccessTy,
+                       LSRUse::KindType Kind, Type *AccessTy,
                        const TargetLowering *TLI) {
   switch (Kind) {
   case LSRUse::Address:
@@ -1156,7 +1205,7 @@ static bool isLegalUse(const TargetLowering::AddrMode &AM,
     // If we have low-level target information, ask the target if it can fold an
     // integer immediate on an icmp.
     if (AM.BaseOffs != 0) {
-      if (TLI) return TLI->isLegalICmpImmediate(-AM.BaseOffs);
+      if (TLI) return TLI->isLegalICmpImmediate(-(uint64_t)AM.BaseOffs);
       return false;
     }
 
@@ -1176,7 +1225,7 @@ static bool isLegalUse(const TargetLowering::AddrMode &AM,
 
 static bool isLegalUse(TargetLowering::AddrMode AM,
                        int64_t MinOffset, int64_t MaxOffset,
-                       LSRUse::KindType Kind, const Type *AccessTy,
+                       LSRUse::KindType Kind, Type *AccessTy,
                        const TargetLowering *TLI) {
   // Check for overflow.
   if (((int64_t)((uint64_t)AM.BaseOffs + MinOffset) > AM.BaseOffs) !=
@@ -1198,7 +1247,7 @@ static bool isLegalUse(TargetLowering::AddrMode AM,
 static bool isAlwaysFoldable(int64_t BaseOffs,
                              GlobalValue *BaseGV,
                              bool HasBaseReg,
-                             LSRUse::KindType Kind, const Type *AccessTy,
+                             LSRUse::KindType Kind, Type *AccessTy,
                              const TargetLowering *TLI) {
   // Fast-path: zero is always foldable.
   if (BaseOffs == 0 && !BaseGV) return true;
@@ -1224,7 +1273,7 @@ static bool isAlwaysFoldable(int64_t BaseOffs,
 static bool isAlwaysFoldable(const SCEV *S,
                              int64_t MinOffset, int64_t MaxOffset,
                              bool HasBaseReg,
-                             LSRUse::KindType Kind, const Type *AccessTy,
+                             LSRUse::KindType Kind, Type *AccessTy,
                              const TargetLowering *TLI,
                              ScalarEvolution &SE) {
   // Fast-path: zero is always foldable.
@@ -1299,7 +1348,7 @@ class LSRInstance {
   SmallSetVector<int64_t, 8> Factors;
 
   /// Types - Interesting use types, to facilitate truncation reuse.
-  SmallSetVector<const Type *, 4> Types;
+  SmallSetVector<Type *, 4> Types;
 
   /// Fixups - The list of operands which are to be replaced.
   SmallVector<LSRFixup, 16> Fixups;
@@ -1330,11 +1379,11 @@ class LSRInstance {
   UseMapTy UseMap;
 
   bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
-                          LSRUse::KindType Kind, const Type *AccessTy);
+                          LSRUse::KindType Kind, Type *AccessTy);
 
   std::pair<size_t, int64_t> getUse(const SCEV *&Expr,
                                     LSRUse::KindType Kind,
-                                    const Type *AccessTy);
+                                    Type *AccessTy);
 
   void DeleteUse(LSRUse &LU, size_t LUIdx);
 
@@ -1426,7 +1475,8 @@ void LSRInstance::OptimizeShadowIV() {
     IVUsers::const_iterator CandidateUI = UI;
     ++UI;
     Instruction *ShadowUse = CandidateUI->getUser();
-    const Type *DestTy = NULL;
+    Type *DestTy = NULL;
+    bool IsSigned = false;
 
     /* If shadow use is a int->float cast then insert a second IV
        to eliminate this cast.
@@ -1440,10 +1490,14 @@ void LSRInstance::OptimizeShadowIV() {
          for (unsigned i = 0; i < n; ++i, ++d)
            foo(d);
     */
-    if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser()))
+    if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) {
+      IsSigned = false;
       DestTy = UCast->getDestTy();
-    else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser()))
+    }
+    else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) {
+      IsSigned = true;
       DestTy = SCast->getDestTy();
+    }
     if (!DestTy) continue;
 
     if (TLI) {
@@ -1457,7 +1511,7 @@ void LSRInstance::OptimizeShadowIV() {
     if (!PH) continue;
     if (PH->getNumIncomingValues() != 2) continue;
 
-    const Type *SrcTy = PH->getType();
+    Type *SrcTy = PH->getType();
     int Mantissa = DestTy->getFPMantissaWidth();
     if (Mantissa == -1) continue;
     if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa)
@@ -1474,7 +1528,9 @@ void LSRInstance::OptimizeShadowIV() {
 
     ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry));
     if (!Init) continue;
-    Constant *NewInit = ConstantFP::get(DestTy, Init->getZExtValue());
+    Constant *NewInit = ConstantFP::get(DestTy, IsSigned ?
+                                        (double)Init->getSExtValue() :
+                                        (double)Init->getZExtValue());
 
     BinaryOperator *Incr =
       dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch));
@@ -1776,7 +1832,7 @@ LSRInstance::OptimizeLoopTermCond() {
             if (!TLI)
               goto decline_post_inc;
             // Check for possible scaled-address reuse.
-            const Type *AccessTy = getAccessType(UI->getUser());
+            Type *AccessTy = getAccessType(UI->getUser());
             TargetLowering::AddrMode AM;
             AM.Scale = C->getSExtValue();
             if (TLI->isLegalAddressingMode(AM, AccessTy))
@@ -1840,10 +1896,10 @@ LSRInstance::OptimizeLoopTermCond() {
 /// return true.
 bool
 LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
-                                LSRUse::KindType Kind, const Type *AccessTy) {
+                                LSRUse::KindType Kind, Type *AccessTy) {
   int64_t NewMinOffset = LU.MinOffset;
   int64_t NewMaxOffset = LU.MaxOffset;
-  const Type *NewAccessTy = AccessTy;
+  Type *NewAccessTy = AccessTy;
 
   // Check for a mismatched kind. It's tempting to collapse mismatched kinds to
   // something conservative, however this can pessimize in the case that one of
@@ -1882,7 +1938,7 @@ LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
 /// Either reuse an existing use or create a new one, as needed.
 std::pair<size_t, int64_t>
 LSRInstance::getUse(const SCEV *&Expr,
-                    LSRUse::KindType Kind, const Type *AccessTy) {
+                    LSRUse::KindType Kind, Type *AccessTy) {
   const SCEV *Copy = Expr;
   int64_t Offset = ExtractImmediate(Expr, SE);
 
@@ -2044,7 +2100,7 @@ void LSRInstance::CollectFixupsAndInitialFormulae() {
     LF.PostIncLoops = UI->getPostIncLoops();
 
     LSRUse::KindType Kind = LSRUse::Basic;
-    const Type *AccessTy = 0;
+    Type *AccessTy = 0;
     if (isAddressUse(LF.UserInst, LF.OperandValToReplace)) {
       Kind = LSRUse::Address;
       AccessTy = getAccessType(LF.UserInst);
@@ -2464,7 +2520,7 @@ void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
   if (LU.Kind != LSRUse::ICmpZero) return;
 
   // Determine the integer type for the base formula.
-  const Type *IntTy = Base.getType();
+  Type *IntTy = Base.getType();
   if (!IntTy) return;
   if (SE.getTypeSizeInBits(IntTy) > 64) return;
 
@@ -2538,7 +2594,7 @@ void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
 /// scaled-offset address modes, for example.
 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
   // Determine the integer type for the base formula.
-  const Type *IntTy = Base.getType();
+  Type *IntTy = Base.getType();
   if (!IntTy) return;
 
   // If this Formula already has a scaled register, we can't add another one.
@@ -2598,13 +2654,13 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
   if (Base.AM.BaseGV) return;
 
   // Determine the integer type for the base formula.
-  const Type *DstTy = Base.getType();
+  Type *DstTy = Base.getType();
   if (!DstTy) return;
   DstTy = SE.getEffectiveSCEVType(DstTy);
 
-  for (SmallSetVector<const Type *, 4>::const_iterator
+  for (SmallSetVector<Type *, 4>::const_iterator
        I = Types.begin(), E = Types.end(); I != E; ++I) {
-    const Type *SrcTy = *I;
+    Type *SrcTy = *I;
     if (SrcTy != DstTy && TLI->isTruncateFree(SrcTy, DstTy)) {
       Formula F = Base;
 
@@ -2741,7 +2797,7 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
     int64_t Imm = WI.Imm;
     const SCEV *OrigReg = WI.OrigReg;
 
-    const Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
+    Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
     const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm));
     unsigned BitWidth = SE.getTypeSizeInBits(IntTy);
 
@@ -3275,6 +3331,9 @@ void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
   skip:;
   }
 
+  if (!EnableRetry && !AnySatisfiedReqRegs)
+    return;
+
   // If none of the formulae had all of the required registers, relax the
   // constraint so that we don't exclude all formulae.
   if (!AnySatisfiedReqRegs) {
@@ -3298,6 +3357,10 @@ void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
   // SolveRecurse does all the work.
   SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
                CurRegs, VisitedRegs);
+  if (Solution.empty()) {
+    DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
+    return;
+  }
 
   // Ok, we've now made all our decisions.
   DEBUG(dbgs() << "\n"
@@ -3416,6 +3479,9 @@ LSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator IP,
   // Don't insert instructions before PHI nodes.
   while (isa<PHINode>(IP)) ++IP;
 
+  // Ignore landingpad instructions.
+  while (isa<LandingPadInst>(IP)) ++IP;
+
   // Ignore debug intrinsics.
   while (isa<DbgInfoIntrinsic>(IP)) ++IP;
 
@@ -3440,9 +3506,9 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
   Rewriter.setPostInc(LF.PostIncLoops);
 
   // This is the type that the user actually needs.
-  const Type *OpTy = LF.OperandValToReplace->getType();
+  Type *OpTy = LF.OperandValToReplace->getType();
   // This will be the type that we'll initially expand to.
-  const Type *Ty = F.getType();
+  Type *Ty = F.getType();
   if (!Ty)
     // No type known; just expand directly to the ultimate type.
     Ty = OpTy;
@@ -3450,7 +3516,7 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
     // Expand directly to the ultimate type if it's the right size.
     Ty = OpTy;
   // This is the type to do integer arithmetic in.
-  const Type *IntTy = SE.getEffectiveSCEVType(Ty);
+  Type *IntTy = SE.getEffectiveSCEVType(Ty);
 
   // Build up a list of operands to add together to form the full base.
   SmallVector<const SCEV *, 8> Ops;
@@ -3527,7 +3593,7 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
       // The other interesting way of "folding" with an ICmpZero is to use a
       // negated immediate.
       if (!ICmpScaledV)
-        ICmpScaledV = ConstantInt::get(IntTy, -Offset);
+        ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset);
       else {
         Ops.push_back(SE.getUnknown(ICmpScaledV));
         ICmpScaledV = ConstantInt::get(IntTy, Offset);
@@ -3611,10 +3677,20 @@ void LSRInstance::RewriteForPHI(PHINode *PN,
       // users.
       if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
           !isa<IndirectBrInst>(BB->getTerminator())) {
-        Loop *PNLoop = LI.getLoopFor(PN->getParent());
-        if (!PNLoop || PN->getParent() != PNLoop->getHeader()) {
+        BasicBlock *Parent = PN->getParent();
+        Loop *PNLoop = LI.getLoopFor(Parent);
+        if (!PNLoop || Parent != PNLoop->getHeader()) {
           // Split the critical edge.
-          BasicBlock *NewBB = SplitCriticalEdge(BB, PN->getParent(), P);
+          BasicBlock *NewBB = 0;
+          if (!Parent->isLandingPad()) {
+            NewBB = SplitCriticalEdge(BB, Parent, P,
+                                      /*MergeIdenticalEdges=*/true,
+                                      /*DontDeleteUselessPhis=*/true);
+          } else {
+            SmallVector<BasicBlock*, 2> NewBBs;
+            SplitLandingPadPredecessors(Parent, BB, "", "", P, NewBBs);
+            NewBB = NewBBs[0];
+          }
 
           // If PN is outside of the loop and BB is in the loop, we want to
           // move the block to be immediately before the PHI block, not
@@ -3637,7 +3713,7 @@ void LSRInstance::RewriteForPHI(PHINode *PN,
         Value *FullV = Expand(LF, F, BB->getTerminator(), Rewriter, DeadInsts);
 
         // If this is reuse-by-noop-cast, insert the noop cast.
-        const Type *OpTy = LF.OperandValToReplace->getType();
+        Type *OpTy = LF.OperandValToReplace->getType();
         if (FullV->getType() != OpTy)
           FullV =
             CastInst::Create(CastInst::getCastOpcode(FullV, false,
@@ -3667,7 +3743,7 @@ void LSRInstance::Rewrite(const LSRFixup &LF,
     Value *FullV = Expand(LF, F, LF.UserInst, Rewriter, DeadInsts);
 
     // If this is reuse-by-noop-cast, insert the noop cast.
-    const Type *OpTy = LF.OperandValToReplace->getType();
+    Type *OpTy = LF.OperandValToReplace->getType();
     if (FullV->getType() != OpTy) {
       Instruction *Cast =
         CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false),
@@ -3700,6 +3776,7 @@ LSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution,
 
   SCEVExpander Rewriter(SE, "lsr");
   Rewriter.disableCanonicalMode();
+  Rewriter.enableLSRMode();
   Rewriter.setIVIncInsertPos(L, IVIncInsertPos);
 
   // Expand the new value definitions and update the users.
@@ -3740,6 +3817,23 @@ LSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P)
   OptimizeShadowIV();
   OptimizeLoopTermCond();
 
+  // If loop preparation eliminates all interesting IV users, bail.
+  if (IU.empty()) return;
+
+  // Skip nested loops until we can model them better with formulae.
+  if (!EnableNested && !L->empty()) {
+
+    if (EnablePhiElim) {
+      // Remove any extra phis created by processing inner loops.
+      SmallVector<WeakVH, 16> DeadInsts;
+      SCEVExpander Rewriter(SE, "lsr");
+      Changed |= Rewriter.replaceCongruentIVs(L, &DT, DeadInsts);
+      Changed |= DeleteTriviallyDeadInstructions(DeadInsts);
+    }
+    DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
+    return;
+  }
+
   // Start collecting data and preparing for the solver.
   CollectInterestingTypesAndFactors();
   CollectFixupsAndInitialFormulae();
@@ -3763,6 +3857,9 @@ LSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P)
   Types.clear();
   RegUses.clear();
 
+  if (Solution.empty())
+    return;
+
 #ifndef NDEBUG
   // Formulae should be legal.
   for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
@@ -3778,6 +3875,14 @@ LSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P)
 
   // Now that we've decided what we want, make it so.
   ImplementSolution(Solution, P);
+
+  if (EnablePhiElim) {
+    // Remove any extra phis created by processing inner loops.
+    SmallVector<WeakVH, 16> DeadInsts;
+    SCEVExpander Rewriter(SE, "lsr");
+    Changed |= Rewriter.replaceCongruentIVs(L, &DT, DeadInsts);
+    Changed |= DeleteTriviallyDeadInstructions(DeadInsts);
+  }
 }
 
 void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
@@ -3793,7 +3898,7 @@ void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
     OS << '*' << *I;
   }
 
-  for (SmallSetVector<const Type *, 4>::const_iterator
+  for (SmallSetVector<Type *, 4>::const_iterator
        I = Types.begin(), E = Types.end(); I != E; ++I) {
     if (!First) OS << ", ";
     First = false;
diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index fef6bc31c7b6..91395b2af6aa 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
+#include "llvm/Target/TargetData.h"
 #include <climits>
 
 using namespace llvm;
@@ -39,6 +40,11 @@ UnrollAllowPartial("unroll-allow-partial", cl::init(false), cl::Hidden,
   cl::desc("Allows loops to be partially unrolled until "
            "-unroll-threshold loop size is reached."));
 
+// Temporary flag to be removed in 3.0
+static cl::opt<bool>
+NoSCEVUnroll("disable-unroll-scev", cl::init(false), cl::Hidden,
+  cl::desc("Use ScalarEvolution to analyze loop trip counts for unrolling"));
+
 namespace {
   class LoopUnroll : public LoopPass {
   public:
@@ -49,7 +55,7 @@ namespace {
       CurrentAllowPartial = (P == -1) ? UnrollAllowPartial : (bool)P;
 
       UserThreshold = (T != -1) || (UnrollThreshold.getNumOccurrences() > 0);
-     
+
       initializeLoopUnrollPass(*PassRegistry::getPassRegistry());
     }
 
@@ -57,11 +63,11 @@ namespace {
     /// that the loop unroll should be performed regardless of how much
     /// code expansion would result.
     static const unsigned NoThreshold = UINT_MAX;
-    
+
     // Threshold to use when optsize is specified (and there is no
     // explicit -unroll-threshold).
     static const unsigned OptSizeUnrollThreshold = 50;
-    
+
     unsigned CurrentCount;
     unsigned CurrentThreshold;
     bool     CurrentAllowPartial;
@@ -79,6 +85,7 @@ namespace {
       AU.addPreservedID(LoopSimplifyID);
       AU.addRequiredID(LCSSAID);
       AU.addPreservedID(LCSSAID);
+      AU.addRequired<ScalarEvolution>();
       AU.addPreserved<ScalarEvolution>();
       // FIXME: Loop unroll requires LCSSA. And LCSSA requires dom info.
       // If loop unroll does not preserve dom info then LCSSA pass on next
@@ -101,45 +108,62 @@ Pass *llvm::createLoopUnrollPass(int Threshold, int Count, int AllowPartial) {
 }
 
 /// ApproximateLoopSize - Approximate the size of the loop.
-static unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls) {
+static unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls,
+                                    const TargetData *TD) {
   CodeMetrics Metrics;
   for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
        I != E; ++I)
-    Metrics.analyzeBasicBlock(*I);
+    Metrics.analyzeBasicBlock(*I, TD);
   NumCalls = Metrics.NumInlineCandidates;
-  
+
   unsigned LoopSize = Metrics.NumInsts;
-  
+
   // Don't allow an estimate of size zero.  This would allows unrolling of loops
   // with huge iteration counts, which is a compile time problem even if it's
   // not a problem for code quality.
   if (LoopSize == 0) LoopSize = 1;
-  
+
   return LoopSize;
 }
 
 bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
   LoopInfo *LI = &getAnalysis<LoopInfo>();
+  ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
 
   BasicBlock *Header = L->getHeader();
   DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
         << "] Loop %" << Header->getName() << "\n");
   (void)Header;
-  
+
   // Determine the current unrolling threshold.  While this is normally set
   // from UnrollThreshold, it is overridden to a smaller value if the current
   // function is marked as optimize-for-size, and the unroll threshold was
   // not user specified.
   unsigned Threshold = CurrentThreshold;
-  if (!UserThreshold && 
+  if (!UserThreshold &&
       Header->getParent()->hasFnAttr(Attribute::OptimizeForSize))
     Threshold = OptSizeUnrollThreshold;
 
-  // Find trip count
-  unsigned TripCount = L->getSmallConstantTripCount();
-  unsigned Count = CurrentCount;
-
+  // Find trip count and trip multiple if count is not available
+  unsigned TripCount = 0;
+  unsigned TripMultiple = 1;
+  if (!NoSCEVUnroll) {
+    // Find "latch trip count". UnrollLoop assumes that control cannot exit
+    // via the loop latch on any iteration prior to TripCount. The loop may exit
+    // early via an earlier branch.
+    BasicBlock *LatchBlock = L->getLoopLatch();
+    if (LatchBlock) {
+      TripCount = SE->getSmallConstantTripCount(L, LatchBlock);
+      TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock);
+    }
+  }
+  else {
+    TripCount = L->getSmallConstantTripCount();
+    if (TripCount == 0)
+      TripMultiple = L->getSmallConstantTripMultiple();
+  }
   // Automatically select an unroll count.
+  unsigned Count = CurrentCount;
   if (Count == 0) {
     // Conservative heuristic: if we know the trip count, see if we can
     // completely unroll (subject to the threshold, checked below); otherwise
@@ -152,8 +176,9 @@ bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
 
   // Enforce the threshold.
   if (Threshold != NoThreshold) {
+    const TargetData *TD = getAnalysisIfAvailable<TargetData>();
     unsigned NumInlineCandidates;
-    unsigned LoopSize = ApproximateLoopSize(L, NumInlineCandidates);
+    unsigned LoopSize = ApproximateLoopSize(L, NumInlineCandidates, TD);
     DEBUG(dbgs() << "  Loop Size = " << LoopSize << "\n");
     if (NumInlineCandidates != 0) {
       DEBUG(dbgs() << "  Not unrolling loop with inlinable calls.\n");
@@ -182,12 +207,8 @@ bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
   }
 
   // Unroll the loop.
-  Function *F = L->getHeader()->getParent();
-  if (!UnrollLoop(L, Count, LI, &LPM))
+  if (!UnrollLoop(L, Count, TripCount, TripMultiple, LI, &LPM))
     return false;
 
-  // FIXME: Reconstruct dom info, because it is not preserved properly.
-  if (DominatorTree *DT = getAnalysisIfAvailable<DominatorTree>())
-    DT->runOnFunction(*F);
   return true;
 }
diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
index 840c4b69cf06..458949c8444d 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -492,7 +492,7 @@ void LoopUnswitch::EmitPreheaderBranchOnCondition(Value *LIC, Constant *Val,
   Value *BranchVal = LIC;
   if (!isa<ConstantInt>(Val) ||
       Val->getType() != Type::getInt1Ty(LIC->getContext()))
-    BranchVal = new ICmpInst(InsertPt, ICmpInst::ICMP_EQ, LIC, Val, "tmp");
+    BranchVal = new ICmpInst(InsertPt, ICmpInst::ICMP_EQ, LIC, Val);
   else if (Val != ConstantInt::getTrue(Val->getContext()))
     // We want to enter the new loop when the condition is true.
     std::swap(TrueDest, FalseDest);
@@ -561,10 +561,17 @@ void LoopUnswitch::SplitExitEdges(Loop *L,
     BasicBlock *ExitBlock = ExitBlocks[i];
     SmallVector<BasicBlock *, 4> Preds(pred_begin(ExitBlock),
                                        pred_end(ExitBlock));
+
     // Although SplitBlockPredecessors doesn't preserve loop-simplify in
     // general, if we call it on all predecessors of all exits then it does.
-    SplitBlockPredecessors(ExitBlock, Preds.data(), Preds.size(),
-                           ".us-lcssa", this);
+    if (!ExitBlock->isLandingPad()) {
+      SplitBlockPredecessors(ExitBlock, Preds.data(), Preds.size(),
+                             ".us-lcssa", this);
+    } else {
+      SmallVector<BasicBlock*, 2> NewBBs;
+      SplitLandingPadPredecessors(ExitBlock, Preds, ".us-lcssa", ".us-lcssa",
+                                  this, NewBBs);
+    }
   }
 }
 
@@ -632,7 +639,7 @@ void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
     // as well.
     ParentLoop->addBasicBlockToLoop(NewBlocks[0], LI->getBase());
   }
-  
+
   for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
     BasicBlock *NewExit = cast<BasicBlock>(VMap[ExitBlocks[i]]);
     // The new exit block should be in the same loop as the old one.
@@ -653,6 +660,19 @@ void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
       if (It != VMap.end()) V = It->second;
       PN->addIncoming(V, NewExit);
     }
+
+    if (LandingPadInst *LPad = NewExit->getLandingPadInst()) {
+      PN = PHINode::Create(LPad->getType(), 0, "",
+                           ExitSucc->getFirstInsertionPt());
+
+      for (pred_iterator I = pred_begin(ExitSucc), E = pred_end(ExitSucc);
+           I != E; ++I) {
+        BasicBlock *BB = *I;
+        LandingPadInst *LPI = BB->getLandingPadInst();
+        LPI->replaceAllUsesWith(PN);
+        PN->addIncoming(LPI, BB);
+      }
+    }
   }
 
   // Rewrite the code to refer to itself.
diff --git a/contrib/llvm/lib/Transforms/Scalar/LowerAtomic.cpp b/contrib/llvm/lib/Transforms/Scalar/LowerAtomic.cpp
index 9087b46c138b..689bbe9b03c8 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LowerAtomic.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LowerAtomic.cpp
@@ -20,98 +20,88 @@
 #include "llvm/Support/IRBuilder.h"
 using namespace llvm;
 
-static bool LowerAtomicIntrinsic(IntrinsicInst *II) {
-  IRBuilder<> Builder(II->getParent(), II);
-  unsigned IID = II->getIntrinsicID();
-  switch (IID) {
-  case Intrinsic::memory_barrier:
+static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
+  IRBuilder<> Builder(CXI->getParent(), CXI);
+  Value *Ptr = CXI->getPointerOperand();
+  Value *Cmp = CXI->getCompareOperand();
+  Value *Val = CXI->getNewValOperand();
+ 
+  LoadInst *Orig = Builder.CreateLoad(Ptr);
+  Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
+  Value *Res = Builder.CreateSelect(Equal, Val, Orig);
+  Builder.CreateStore(Res, Ptr);
+ 
+  CXI->replaceAllUsesWith(Orig);
+  CXI->eraseFromParent();
+  return true;
+}
+
+static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
+  IRBuilder<> Builder(RMWI->getParent(), RMWI);
+  Value *Ptr = RMWI->getPointerOperand();
+  Value *Val = RMWI->getValOperand();
+
+  LoadInst *Orig = Builder.CreateLoad(Ptr);
+  Value *Res = NULL;
+
+  switch (RMWI->getOperation()) {
+  default: llvm_unreachable("Unexpected RMW operation");
+  case AtomicRMWInst::Xchg:
+    Res = Val;
     break;
-
-  case Intrinsic::atomic_load_add:
-  case Intrinsic::atomic_load_sub:
-  case Intrinsic::atomic_load_and:
-  case Intrinsic::atomic_load_nand:
-  case Intrinsic::atomic_load_or:
-  case Intrinsic::atomic_load_xor:
-  case Intrinsic::atomic_load_max:
-  case Intrinsic::atomic_load_min:
-  case Intrinsic::atomic_load_umax:
-  case Intrinsic::atomic_load_umin: {
-    Value *Ptr = II->getArgOperand(0), *Delta = II->getArgOperand(1);
-
-    LoadInst *Orig = Builder.CreateLoad(Ptr);
-    Value *Res = NULL;
-    switch (IID) {
-    default: assert(0 && "Unrecognized atomic modify operation");
-    case Intrinsic::atomic_load_add:
-      Res = Builder.CreateAdd(Orig, Delta);
-      break;
-    case Intrinsic::atomic_load_sub:
-      Res = Builder.CreateSub(Orig, Delta);
-      break;
-    case Intrinsic::atomic_load_and:
-      Res = Builder.CreateAnd(Orig, Delta);
-      break;
-    case Intrinsic::atomic_load_nand:
-      Res = Builder.CreateNot(Builder.CreateAnd(Orig, Delta));
-      break;
-    case Intrinsic::atomic_load_or:
-      Res = Builder.CreateOr(Orig, Delta);
-      break;
-    case Intrinsic::atomic_load_xor:
-      Res = Builder.CreateXor(Orig, Delta);
-      break;
-    case Intrinsic::atomic_load_max:
-      Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Delta),
-                                 Delta, Orig);
-      break;
-    case Intrinsic::atomic_load_min:
-      Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Delta),
-                                 Orig, Delta);
-      break;
-    case Intrinsic::atomic_load_umax:
-      Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Delta),
-                                 Delta, Orig);
-      break;
-    case Intrinsic::atomic_load_umin:
-      Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Delta),
-                                 Orig, Delta);
-      break;
-    }
-    Builder.CreateStore(Res, Ptr);
-
-    II->replaceAllUsesWith(Orig);
+  case AtomicRMWInst::Add:
+    Res = Builder.CreateAdd(Orig, Val);
+    break;
+  case AtomicRMWInst::Sub:
+    Res = Builder.CreateSub(Orig, Val);
+    break;
+  case AtomicRMWInst::And:
+    Res = Builder.CreateAnd(Orig, Val);
+    break;
+  case AtomicRMWInst::Nand:
+    Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
+    break;
+  case AtomicRMWInst::Or:
+    Res = Builder.CreateOr(Orig, Val);
+    break;
+  case AtomicRMWInst::Xor:
+    Res = Builder.CreateXor(Orig, Val);
+    break;
+  case AtomicRMWInst::Max:
+    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
+                               Val, Orig);
+    break;
+  case AtomicRMWInst::Min:
+    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
+                               Orig, Val);
+    break;
+  case AtomicRMWInst::UMax:
+    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
+                               Val, Orig);
+    break;
+  case AtomicRMWInst::UMin:
+    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
+                               Orig, Val);
     break;
   }
+  Builder.CreateStore(Res, Ptr);
+  RMWI->replaceAllUsesWith(Orig);
+  RMWI->eraseFromParent();
+  return true;
+}
 
-  case Intrinsic::atomic_swap: {
-    Value *Ptr = II->getArgOperand(0), *Val = II->getArgOperand(1);
-    LoadInst *Orig = Builder.CreateLoad(Ptr);
-    Builder.CreateStore(Val, Ptr);
-    II->replaceAllUsesWith(Orig);
-    break;
-  }
+static bool LowerFenceInst(FenceInst *FI) {
+  FI->eraseFromParent();
+  return true;
+}
 
-  case Intrinsic::atomic_cmp_swap: {
-    Value *Ptr = II->getArgOperand(0), *Cmp = II->getArgOperand(1);
-    Value *Val = II->getArgOperand(2);
-
-    LoadInst *Orig = Builder.CreateLoad(Ptr);
-    Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
-    Value *Res = Builder.CreateSelect(Equal, Val, Orig);
-    Builder.CreateStore(Res, Ptr);
-    II->replaceAllUsesWith(Orig);
-    break;
-  }
-
-  default:
-    return false;
-  }
-
-  assert(II->use_empty() &&
-         "Lowering should have eliminated any uses of the intrinsic call!");
-  II->eraseFromParent();
+static bool LowerLoadInst(LoadInst *LI) {
+  LI->setAtomic(NotAtomic);
+  return true;
+}
 
+static bool LowerStoreInst(StoreInst *SI) {
+  SI->setAtomic(NotAtomic);
   return true;
 }
 
@@ -123,9 +113,22 @@ namespace {
     }
     bool runOnBasicBlock(BasicBlock &BB) {
       bool Changed = false;
-      for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE; )
-        if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(DI++))
-          Changed |= LowerAtomicIntrinsic(II);
+      for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE; ) {
+        Instruction *Inst = DI++;
+        if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
+          Changed |= LowerFenceInst(FI);
+        else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
+          Changed |= LowerAtomicCmpXchgInst(CXI);
+        else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
+          Changed |= LowerAtomicRMWInst(RMWI);
+        else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
+          if (LI->isAtomic())
+            LowerLoadInst(LI);
+        } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
+          if (SI->isAtomic())
+            LowerStoreInst(SI);
+        }
+      }
       return Changed;
     }
   };
diff --git a/contrib/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/contrib/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 7ed3db6cc1db..eeb8931446dc 100644
--- a/contrib/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -54,7 +54,7 @@ static int64_t GetOffsetFromIndex(const GetElementPtrInst *GEP, unsigned Idx,
     if (OpC->isZero()) continue;  // No offset.
 
     // Handle struct indices, which add their field offset to the pointer.
-    if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+    if (StructType *STy = dyn_cast<StructType>(*GTI)) {
       Offset += TD.getStructLayout(STy)->getElementOffset(OpC->getZExtValue());
       continue;
     }
@@ -384,7 +384,7 @@ Instruction *MemCpyOpt::tryMergingIntoMemset(Instruction *StartInst,
     
     if (StoreInst *NextStore = dyn_cast<StoreInst>(BI)) {
       // If this is a store, see if we can merge it in.
-      if (NextStore->isVolatile()) break;
+      if (!NextStore->isSimple()) break;
     
       // Check to see if this stored value is of the same byte-splattable value.
       if (ByteVal != isBytewiseValue(NextStore->getOperand(0)))
@@ -448,7 +448,7 @@ Instruction *MemCpyOpt::tryMergingIntoMemset(Instruction *StartInst,
     // Determine alignment
     unsigned Alignment = Range.Alignment;
     if (Alignment == 0) {
-      const Type *EltType = 
+      Type *EltType = 
         cast<PointerType>(StartPtr->getType())->getElementType();
       Alignment = TD->getABITypeAlignment(EltType);
     }
@@ -479,7 +479,7 @@ Instruction *MemCpyOpt::tryMergingIntoMemset(Instruction *StartInst,
 
 
 bool MemCpyOpt::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
-  if (SI->isVolatile()) return false;
+  if (!SI->isSimple()) return false;
   
   if (TD == 0) return false;
 
@@ -487,7 +487,7 @@ bool MemCpyOpt::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
   // happen to be using a load-store pair to implement it, rather than
   // a memcpy.
   if (LoadInst *LI = dyn_cast<LoadInst>(SI->getOperand(0))) {
-    if (!LI->isVolatile() && LI->hasOneUse() &&
+    if (LI->isSimple() && LI->hasOneUse() &&
         LI->getParent() == SI->getParent()) {
       MemDepResult ldep = MD->getDependency(LI);
       CallInst *C = 0;
@@ -616,7 +616,7 @@ bool MemCpyOpt::performCallSlotOptzn(Instruction *cpy,
     if (!A->hasStructRetAttr())
       return false;
 
-    const Type *StructTy = cast<PointerType>(A->getType())->getElementType();
+    Type *StructTy = cast<PointerType>(A->getType())->getElementType();
     uint64_t destSize = TD->getTypeAllocSize(StructTy);
 
     if (destSize < srcSize)
@@ -860,7 +860,7 @@ bool MemCpyOpt::processByValArgument(CallSite CS, unsigned ArgNo) {
 
   // Find out what feeds this byval argument.
   Value *ByValArg = CS.getArgument(ArgNo);
-  const Type *ByValTy =cast<PointerType>(ByValArg->getType())->getElementType();
+  Type *ByValTy = cast<PointerType>(ByValArg->getType())->getElementType();
   uint64_t ByValSize = TD->getTypeAllocSize(ByValTy);
   MemDepResult DepInfo =
     MD->getPointerDependencyFrom(AliasAnalysis::Location(ByValArg, ByValSize),
diff --git a/contrib/llvm/lib/Transforms/Scalar/ObjCARC.cpp b/contrib/llvm/lib/Transforms/Scalar/ObjCARC.cpp
index ee132d3be4f5..da74e9c3ec79 100644
--- a/contrib/llvm/lib/Transforms/Scalar/ObjCARC.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/ObjCARC.cpp
@@ -180,7 +180,7 @@ static bool IsPotentialUse(const Value *Op) {
         Arg->hasStructRetAttr())
       return false;
   // Only consider values with pointer types, and not function pointers.
-  const PointerType *Ty = dyn_cast<PointerType>(Op->getType());
+  PointerType *Ty = dyn_cast<PointerType>(Op->getType());
   if (!Ty || isa<FunctionType>(Ty->getElementType()))
     return false;
   // Conservatively assume anything else is a potential use.
@@ -213,8 +213,8 @@ static InstructionClass GetFunctionClass(const Function *F) {
   const Argument *A0 = AI++;
   if (AI == AE)
     // Argument is a pointer.
-    if (const PointerType *PTy = dyn_cast<PointerType>(A0->getType())) {
-      const Type *ETy = PTy->getElementType();
+    if (PointerType *PTy = dyn_cast<PointerType>(A0->getType())) {
+      Type *ETy = PTy->getElementType();
       // Argument is i8*.
       if (ETy->isIntegerTy(8))
         return StringSwitch<InstructionClass>(F->getName())
@@ -234,7 +234,7 @@ static InstructionClass GetFunctionClass(const Function *F) {
           .Default(IC_CallOrUser);
 
       // Argument is i8**
-      if (const PointerType *Pte = dyn_cast<PointerType>(ETy))
+      if (PointerType *Pte = dyn_cast<PointerType>(ETy))
         if (Pte->getElementType()->isIntegerTy(8))
           return StringSwitch<InstructionClass>(F->getName())
             .Case("objc_loadWeakRetained",      IC_LoadWeakRetained)
@@ -246,11 +246,11 @@ static InstructionClass GetFunctionClass(const Function *F) {
   // Two arguments, first is i8**.
   const Argument *A1 = AI++;
   if (AI == AE)
-    if (const PointerType *PTy = dyn_cast<PointerType>(A0->getType()))
-      if (const PointerType *Pte = dyn_cast<PointerType>(PTy->getElementType()))
+    if (PointerType *PTy = dyn_cast<PointerType>(A0->getType()))
+      if (PointerType *Pte = dyn_cast<PointerType>(PTy->getElementType()))
         if (Pte->getElementType()->isIntegerTy(8))
-          if (const PointerType *PTy1 = dyn_cast<PointerType>(A1->getType())) {
-            const Type *ETy1 = PTy1->getElementType();
+          if (PointerType *PTy1 = dyn_cast<PointerType>(A1->getType())) {
+            Type *ETy1 = PTy1->getElementType();
             // Second argument is i8*
             if (ETy1->isIntegerTy(8))
               return StringSwitch<InstructionClass>(F->getName())
@@ -258,7 +258,7 @@ static InstructionClass GetFunctionClass(const Function *F) {
                      .Case("objc_initWeak",              IC_InitWeak)
                      .Default(IC_CallOrUser);
             // Second argument is i8**.
-            if (const PointerType *Pte1 = dyn_cast<PointerType>(ETy1))
+            if (PointerType *Pte1 = dyn_cast<PointerType>(ETy1))
               if (Pte1->getElementType()->isIntegerTy(8))
                 return StringSwitch<InstructionClass>(F->getName())
                        .Case("objc_moveWeak",              IC_MoveWeak)
@@ -344,6 +344,10 @@ static InstructionClass GetInstructionClass(const Value *V) {
       break;
     default:
       // For anything else, check all the operands.
+      // Note that this includes both operands of a Store: while the first
+      // operand isn't actually being dereferenced, it is being stored to
+      // memory where we can no longer track who might read it and dereference
+      // it, so we have to consider it potentially used.
       for (User::const_op_iterator OI = I->op_begin(), OE = I->op_end();
            OI != OE; ++OI)
         if (IsPotentialUse(*OI))
@@ -421,9 +425,10 @@ static bool IsAlwaysTail(InstructionClass Class) {
 /// IsNoThrow - Test if the given class represents instructions which are always
 /// safe to mark with the nounwind attribute..
 static bool IsNoThrow(InstructionClass Class) {
+  // objc_retainBlock is not nounwind because it calls user copy constructors
+  // which could theoretically throw.
   return Class == IC_Retain ||
          Class == IC_RetainRV ||
-         Class == IC_RetainBlock ||
          Class == IC_Release ||
          Class == IC_Autorelease ||
          Class == IC_AutoreleaseRV ||
@@ -515,6 +520,10 @@ static bool IsObjCIdentifiedObject(const Value *V) {
     const Value *Pointer =
       StripPointerCastsAndObjCCalls(LI->getPointerOperand());
     if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Pointer)) {
+      // A constant pointer can't be pointing to an object on the heap. It may
+      // be reference-counted, but it won't be deleted.
+      if (GV->isConstant())
+        return true;
       StringRef Name = GV->getName();
       // These special variables are known to hold values which are not
       // reference-counted pointers.
@@ -738,7 +747,6 @@ ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS, const Location &Loc) {
   switch (GetBasicInstructionClass(CS.getInstruction())) {
   case IC_Retain:
   case IC_RetainRV:
-  case IC_RetainBlock:
   case IC_Autorelease:
   case IC_AutoreleaseRV:
   case IC_NoopCast:
@@ -746,6 +754,8 @@ ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS, const Location &Loc) {
   case IC_FusedRetainAutorelease:
   case IC_FusedRetainAutoreleaseRV:
     // These functions don't access any memory visible to the compiler.
+    // Note that this doesn't include objc_retainBlock, becuase it updates
+    // pointers when it copies block data.
     return NoModRef;
   default:
     break;
@@ -877,7 +887,9 @@ bool ObjCARCExpand::runOnFunction(Function &F) {
 // usually can't sink them past other calls, which would be the main
 // case where it would be useful.
 
-/// TODO: The pointer returned from objc_loadWeakRetained is retained.
+// TODO: The pointer returned from objc_loadWeakRetained is retained.
+
+// TODO: Delete release+retain pairs (rare).
 
 #include "llvm/GlobalAlias.h"
 #include "llvm/Constants.h"
@@ -1098,16 +1110,16 @@ static Sequence MergeSeqs(Sequence A, Sequence B, bool TopDown) {
   if (A == S_None || B == S_None)
     return S_None;
 
-  // Note that we can't merge S_CanRelease and S_Use.
   if (A > B) std::swap(A, B);
   if (TopDown) {
     // Choose the side which is further along in the sequence.
-    if (A == S_Retain && (B == S_CanRelease || B == S_Use))
+    if ((A == S_Retain || A == S_CanRelease) &&
+        (B == S_CanRelease || B == S_Use))
       return B;
   } else {
     // Choose the side which is further along in the sequence.
     if ((A == S_Use || A == S_CanRelease) &&
-        (B == S_Release || B == S_Stop || B == S_MovableRelease))
+        (B == S_Use || B == S_Release || B == S_Stop || B == S_MovableRelease))
       return A;
     // If both sides are releases, choose the more conservative one.
     if (A == S_Stop && (B == S_Release || B == S_MovableRelease))
@@ -1124,13 +1136,19 @@ namespace {
   /// retain-decrement-use-release sequence or release-use-decrement-retain
   /// reverese sequence.
   struct RRInfo {
-    /// KnownIncremented - After an objc_retain, the reference count of the
-    /// referenced object is known to be positive. Similarly, before an
-    /// objc_release, the reference count of the referenced object is known to
-    /// be positive. If there are retain-release pairs in code regions where the
-    /// retain count is known to be positive, they can be eliminated, regardless
-    /// of any side effects between them.
-    bool KnownIncremented;
+    /// KnownSafe - After an objc_retain, the reference count of the referenced
+    /// object is known to be positive. Similarly, before an objc_release, the
+    /// reference count of the referenced object is known to be positive. If
+    /// there are retain-release pairs in code regions where the retain count
+    /// is known to be positive, they can be eliminated, regardless of any side
+    /// effects between them.
+    ///
+    /// Also, a retain+release pair nested within another retain+release
+    /// pair all on the known same pointer value can be eliminated, regardless
+    /// of any intervening side effects.
+    ///
+    /// KnownSafe is true when either of these conditions is satisfied.
+    bool KnownSafe;
 
     /// IsRetainBlock - True if the Calls are objc_retainBlock calls (as
     /// opposed to objc_retain calls).
@@ -1153,7 +1171,7 @@ namespace {
     SmallPtrSet<Instruction *, 2> ReverseInsertPts;
 
     RRInfo() :
-      KnownIncremented(false), IsRetainBlock(false), IsTailCallRelease(false),
+      KnownSafe(false), IsRetainBlock(false), IsTailCallRelease(false),
       ReleaseMetadata(0) {}
 
     void clear();
@@ -1161,7 +1179,7 @@ namespace {
 }
 
 void RRInfo::clear() {
-  KnownIncremented = false;
+  KnownSafe = false;
   IsRetainBlock = false;
   IsTailCallRelease = false;
   ReleaseMetadata = 0;
@@ -1176,6 +1194,9 @@ namespace {
     /// RefCount - The known minimum number of reference count increments.
     unsigned RefCount;
 
+    /// NestCount - The known minimum level of retain+release nesting.
+    unsigned NestCount;
+
     /// Seq - The current position in the sequence.
     Sequence Seq;
 
@@ -1184,7 +1205,11 @@ namespace {
     /// TODO: Encapsulate this better.
     RRInfo RRI;
 
-    PtrState() : RefCount(0), Seq(S_None) {}
+    PtrState() : RefCount(0), NestCount(0), Seq(S_None) {}
+
+    void SetAtLeastOneRefCount()  {
+      if (RefCount == 0) RefCount = 1;
+    }
 
     void IncrementRefCount() {
       if (RefCount != UINT_MAX) ++RefCount;
@@ -1194,14 +1219,22 @@ namespace {
       if (RefCount != 0) --RefCount;
     }
 
-    void ClearRefCount() {
-      RefCount = 0;
-    }
-
     bool IsKnownIncremented() const {
       return RefCount > 0;
     }
 
+    void IncrementNestCount() {
+      if (NestCount != UINT_MAX) ++NestCount;
+    }
+
+    void DecrementNestCount() {
+      if (NestCount != 0) --NestCount;
+    }
+
+    bool IsKnownNested() const {
+      return NestCount > 0;
+    }
+
     void SetSeq(Sequence NewSeq) {
       Seq = NewSeq;
     }
@@ -1233,6 +1266,7 @@ void
 PtrState::Merge(const PtrState &Other, bool TopDown) {
   Seq = MergeSeqs(Seq, Other.Seq, TopDown);
   RefCount = std::min(RefCount, Other.RefCount);
+  NestCount = std::min(NestCount, Other.NestCount);
 
   // We can't merge a plain objc_retain with an objc_retainBlock.
   if (RRI.IsRetainBlock != Other.RRI.IsRetainBlock)
@@ -1245,7 +1279,7 @@ PtrState::Merge(const PtrState &Other, bool TopDown) {
     if (RRI.ReleaseMetadata != Other.RRI.ReleaseMetadata)
       RRI.ReleaseMetadata = 0;
 
-    RRI.KnownIncremented = RRI.KnownIncremented && Other.RRI.KnownIncremented;
+    RRI.KnownSafe = RRI.KnownSafe && Other.RRI.KnownSafe;
     RRI.IsTailCallRelease = RRI.IsTailCallRelease && Other.RRI.IsTailCallRelease;
     RRI.Calls.insert(Other.RRI.Calls.begin(), Other.RRI.Calls.end());
     RRI.ReverseInsertPts.insert(Other.RRI.ReverseInsertPts.begin(),
@@ -1316,7 +1350,7 @@ namespace {
     }
 
     void clearBottomUpPointers() {
-      PerPtrTopDown.clear();
+      PerPtrBottomUp.clear();
     }
 
     void clearTopDownPointers() {
@@ -1334,6 +1368,12 @@ namespace {
     unsigned GetAllPathCount() const {
       return TopDownPathCount * BottomUpPathCount;
     }
+
+    /// IsVisitedTopDown - Test whether the block for this BBState has been
+    /// visited by the top-down portion of the algorithm.
+    bool isVisitedTopDown() const {
+      return TopDownPathCount != 0;
+    }
   };
 }
 
@@ -1364,7 +1404,7 @@ void BBState::MergePred(const BBState &Other) {
                              /*TopDown=*/true);
   }
 
-  // For each entry in our set, if the other set doens't have an entry with the
+  // For each entry in our set, if the other set doesn't have an entry with the
   // same key, force it to merge with an empty entry.
   for (ptr_iterator MI = top_down_ptr_begin(),
        ME = top_down_ptr_end(); MI != ME; ++MI)
@@ -1389,7 +1429,7 @@ void BBState::MergeSucc(const BBState &Other) {
                              /*TopDown=*/false);
   }
 
-  // For each entry in our set, if the other set doens't have an entry
+  // For each entry in our set, if the other set doesn't have an entry
   // with the same key, force it to merge with an empty entry.
   for (ptr_iterator MI = bottom_up_ptr_begin(),
        ME = bottom_up_ptr_end(); MI != ME; ++MI)
@@ -1406,15 +1446,11 @@ namespace {
     /// Run - A flag indicating whether this optimization pass should run.
     bool Run;
 
-    /// RetainFunc, RelaseFunc - Declarations for objc_retain,
-    /// objc_retainBlock, and objc_release.
-    Function *RetainFunc, *RetainBlockFunc, *RetainRVFunc, *ReleaseFunc;
-
     /// RetainRVCallee, etc. - Declarations for ObjC runtime
     /// functions, for use in creating calls to them. These are initialized
     /// lazily to avoid cluttering up the Module with unused declarations.
     Constant *RetainRVCallee, *AutoreleaseRVCallee, *ReleaseCallee,
-             *RetainCallee, *AutoreleaseCallee;
+             *RetainCallee, *RetainBlockCallee, *AutoreleaseCallee;
 
     /// UsedInThisFunciton - Flags which determine whether each of the
     /// interesting runtine functions is in fact used in the current function.
@@ -1428,6 +1464,7 @@ namespace {
     Constant *getAutoreleaseRVCallee(Module *M);
     Constant *getReleaseCallee(Module *M);
     Constant *getRetainCallee(Module *M);
+    Constant *getRetainBlockCallee(Module *M);
     Constant *getAutoreleaseCallee(Module *M);
 
     void OptimizeRetainCall(Function &F, Instruction *Retain);
@@ -1452,11 +1489,13 @@ namespace {
     void MoveCalls(Value *Arg, RRInfo &RetainsToMove, RRInfo &ReleasesToMove,
                    MapVector<Value *, RRInfo> &Retains,
                    DenseMap<Value *, RRInfo> &Releases,
-                   SmallVectorImpl<Instruction *> &DeadInsts);
+                   SmallVectorImpl<Instruction *> &DeadInsts,
+                   Module *M);
 
     bool PerformCodePlacement(DenseMap<const BasicBlock *, BBState> &BBStates,
                               MapVector<Value *, RRInfo> &Retains,
-                              DenseMap<Value *, RRInfo> &Releases);
+                              DenseMap<Value *, RRInfo> &Releases,
+                              Module *M);
 
     void OptimizeWeakCalls(Function &F);
 
@@ -1501,7 +1540,7 @@ Constant *ObjCARCOpt::getRetainRVCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     std::vector<Type *> Params;
     Params.push_back(I8X);
-    const FunctionType *FTy =
+    FunctionType *FTy =
       FunctionType::get(I8X, Params, /*isVarArg=*/false);
     AttrListPtr Attributes;
     Attributes.addAttr(~0u, Attribute::NoUnwind);
@@ -1518,7 +1557,7 @@ Constant *ObjCARCOpt::getAutoreleaseRVCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     std::vector<Type *> Params;
     Params.push_back(I8X);
-    const FunctionType *FTy =
+    FunctionType *FTy =
       FunctionType::get(I8X, Params, /*isVarArg=*/false);
     AttrListPtr Attributes;
     Attributes.addAttr(~0u, Attribute::NoUnwind);
@@ -1561,6 +1600,23 @@ Constant *ObjCARCOpt::getRetainCallee(Module *M) {
   return RetainCallee;
 }
 
+Constant *ObjCARCOpt::getRetainBlockCallee(Module *M) {
+  if (!RetainBlockCallee) {
+    LLVMContext &C = M->getContext();
+    std::vector<Type *> Params;
+    Params.push_back(PointerType::getUnqual(Type::getInt8Ty(C)));
+    AttrListPtr Attributes;
+    // objc_retainBlock is not nounwind because it calls user copy constructors
+    // which could theoretically throw.
+    RetainBlockCallee =
+      M->getOrInsertFunction(
+        "objc_retainBlock",
+        FunctionType::get(Params[0], Params, /*isVarArg=*/false),
+        Attributes);
+  }
+  return RetainBlockCallee;
+}
+
 Constant *ObjCARCOpt::getAutoreleaseCallee(Module *M) {
   if (!AutoreleaseCallee) {
     LLVMContext &C = M->getContext();
@@ -1904,12 +1960,19 @@ void
 ObjCARCOpt::OptimizeAutoreleaseRVCall(Function &F, Instruction *AutoreleaseRV) {
   // Check for a return of the pointer value.
   const Value *Ptr = GetObjCArg(AutoreleaseRV);
-  for (Value::const_use_iterator UI = Ptr->use_begin(), UE = Ptr->use_end();
-       UI != UE; ++UI) {
-    const User *I = *UI;
-    if (isa<ReturnInst>(I) || GetBasicInstructionClass(I) == IC_RetainRV)
-      return;
-  }
+  SmallVector<const Value *, 2> Users;
+  Users.push_back(Ptr);
+  do {
+    Ptr = Users.pop_back_val();
+    for (Value::const_use_iterator UI = Ptr->use_begin(), UE = Ptr->use_end();
+         UI != UE; ++UI) {
+      const User *I = *UI;
+      if (isa<ReturnInst>(I) || GetBasicInstructionClass(I) == IC_RetainRV)
+        return;
+      if (isa<BitCastInst>(I))
+        Users.push_back(I);
+    }
+  } while (!Users.empty());
 
   Changed = true;
   ++NumPeeps;
@@ -1953,7 +2016,7 @@ void ObjCARCOpt::OptimizeIndividualCalls(Function &F) {
     case IC_DestroyWeak: {
       CallInst *CI = cast<CallInst>(Inst);
       if (isNullOrUndef(CI->getArgOperand(0))) {
-        const Type *Ty = CI->getArgOperand(0)->getType();
+        Type *Ty = CI->getArgOperand(0)->getType();
         new StoreInst(UndefValue::get(cast<PointerType>(Ty)->getElementType()),
                       Constant::getNullValue(Ty),
                       CI);
@@ -1968,7 +2031,7 @@ void ObjCARCOpt::OptimizeIndividualCalls(Function &F) {
       CallInst *CI = cast<CallInst>(Inst);
       if (isNullOrUndef(CI->getArgOperand(0)) ||
           isNullOrUndef(CI->getArgOperand(1))) {
-        const Type *Ty = CI->getArgOperand(0)->getType();
+        Type *Ty = CI->getArgOperand(0)->getType();
         new StoreInst(UndefValue::get(cast<PointerType>(Ty)->getElementType()),
                       Constant::getNullValue(Ty),
                       CI);
@@ -2090,7 +2153,7 @@ void ObjCARCOpt::OptimizeIndividualCalls(Function &F) {
           ++NumPartialNoops;
           // Clone the call into each predecessor that has a non-null value.
           CallInst *CInst = cast<CallInst>(Inst);
-          const Type *ParamTy = CInst->getArgOperand(0)->getType();
+          Type *ParamTy = CInst->getArgOperand(0)->getType();
           for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
             Value *Incoming =
               StripPointerCastsAndObjCCalls(PN->getIncomingValue(i));
@@ -2132,41 +2195,49 @@ ObjCARCOpt::CheckForCFGHazards(const BasicBlock *BB,
       const TerminatorInst *TI = cast<TerminatorInst>(&BB->back());
       bool SomeSuccHasSame = false;
       bool AllSuccsHaveSame = true;
-      for (succ_const_iterator SI(TI), SE(TI, false); SI != SE; ++SI)
-        switch (BBStates[*SI].getPtrBottomUpState(Arg).GetSeq()) {
+      PtrState &S = MyStates.getPtrTopDownState(Arg);
+      for (succ_const_iterator SI(TI), SE(TI, false); SI != SE; ++SI) {
+        PtrState &SuccS = BBStates[*SI].getPtrBottomUpState(Arg);
+        switch (SuccS.GetSeq()) {
         case S_None:
-        case S_CanRelease:
-          MyStates.getPtrTopDownState(Arg).ClearSequenceProgress();
-          SomeSuccHasSame = false;
-          break;
+        case S_CanRelease: {
+          if (!S.RRI.KnownSafe && !SuccS.RRI.KnownSafe)
+            S.ClearSequenceProgress();
+          continue;
+        }
         case S_Use:
           SomeSuccHasSame = true;
           break;
         case S_Stop:
         case S_Release:
         case S_MovableRelease:
-          AllSuccsHaveSame = false;
+          if (!S.RRI.KnownSafe && !SuccS.RRI.KnownSafe)
+            AllSuccsHaveSame = false;
           break;
         case S_Retain:
           llvm_unreachable("bottom-up pointer in retain state!");
         }
+      }
       // If the state at the other end of any of the successor edges
       // matches the current state, require all edges to match. This
       // guards against loops in the middle of a sequence.
       if (SomeSuccHasSame && !AllSuccsHaveSame)
-        MyStates.getPtrTopDownState(Arg).ClearSequenceProgress();
+        S.ClearSequenceProgress();
     }
     case S_CanRelease: {
       const Value *Arg = I->first;
       const TerminatorInst *TI = cast<TerminatorInst>(&BB->back());
       bool SomeSuccHasSame = false;
       bool AllSuccsHaveSame = true;
-      for (succ_const_iterator SI(TI), SE(TI, false); SI != SE; ++SI)
-        switch (BBStates[*SI].getPtrBottomUpState(Arg).GetSeq()) {
-        case S_None:
-          MyStates.getPtrTopDownState(Arg).ClearSequenceProgress();
-          SomeSuccHasSame = false;
-          break;
+      PtrState &S = MyStates.getPtrTopDownState(Arg);
+      for (succ_const_iterator SI(TI), SE(TI, false); SI != SE; ++SI) {
+        PtrState &SuccS = BBStates[*SI].getPtrBottomUpState(Arg);
+        switch (SuccS.GetSeq()) {
+        case S_None: {
+          if (!S.RRI.KnownSafe && !SuccS.RRI.KnownSafe)
+            S.ClearSequenceProgress();
+          continue;
+        }
         case S_CanRelease:
           SomeSuccHasSame = true;
           break;
@@ -2174,16 +2245,18 @@ ObjCARCOpt::CheckForCFGHazards(const BasicBlock *BB,
         case S_Release:
         case S_MovableRelease:
         case S_Use:
-          AllSuccsHaveSame = false;
+          if (!S.RRI.KnownSafe && !SuccS.RRI.KnownSafe)
+            AllSuccsHaveSame = false;
           break;
         case S_Retain:
           llvm_unreachable("bottom-up pointer in retain state!");
         }
+      }
       // If the state at the other end of any of the successor edges
       // matches the current state, require all edges to match. This
       // guards against loops in the middle of a sequence.
       if (SomeSuccHasSame && !AllSuccsHaveSame)
-        MyStates.getPtrTopDownState(Arg).ClearSequenceProgress();
+        S.ClearSequenceProgress();
     }
     }
 }
@@ -2207,6 +2280,8 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
       if (Succ == BB)
         continue;
       DenseMap<const BasicBlock *, BBState>::iterator I = BBStates.find(Succ);
+      // If we haven't seen this node yet, then we've found a CFG cycle.
+      // Be optimistic here; it's CheckForCFGHazards' job detect trouble.
       if (I == BBStates.end())
         continue;
       MyStates.InitFromSucc(I->second);
@@ -2245,11 +2320,12 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
 
       S.SetSeqToRelease(Inst->getMetadata(ImpreciseReleaseMDKind));
       S.RRI.clear();
-      S.RRI.KnownIncremented = S.IsKnownIncremented();
+      S.RRI.KnownSafe = S.IsKnownNested() || S.IsKnownIncremented();
       S.RRI.IsTailCallRelease = cast<CallInst>(Inst)->isTailCall();
       S.RRI.Calls.insert(Inst);
 
       S.IncrementRefCount();
+      S.IncrementNestCount();
       break;
     }
     case IC_RetainBlock:
@@ -2259,6 +2335,13 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
 
       PtrState &S = MyStates.getPtrBottomUpState(Arg);
       S.DecrementRefCount();
+      S.SetAtLeastOneRefCount();
+      S.DecrementNestCount();
+
+      // An objc_retainBlock call with just a use still needs to be kept,
+      // because it may be copying a block from the stack to the heap.
+      if (Class == IC_RetainBlock && S.GetSeq() == S_Use)
+        S.SetSeq(S_CanRelease);
 
       switch (S.GetSeq()) {
       case S_Stop:
@@ -2281,7 +2364,7 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
       case S_Retain:
         llvm_unreachable("bottom-up pointer in retain state!");
       }
-      break;
+      continue;
     }
     case IC_AutoreleasepoolPop:
       // Conservatively, clear MyStates for all known pointers.
@@ -2305,26 +2388,22 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
       PtrState &S = MI->second;
       Sequence Seq = S.GetSeq();
 
-      // Check for possible retains and releases.
+      // Check for possible releases.
       if (CanAlterRefCount(Inst, Ptr, PA, Class)) {
-        // Check for a retain (we're going bottom-up here).
         S.DecrementRefCount();
-
-        // Check for a release.
-        if (!IsRetain(Class) && Class != IC_RetainBlock)
-          switch (Seq) {
-          case S_Use:
-            S.SetSeq(S_CanRelease);
-            continue;
-          case S_CanRelease:
-          case S_Release:
-          case S_MovableRelease:
-          case S_Stop:
-          case S_None:
-            break;
-          case S_Retain:
-            llvm_unreachable("bottom-up pointer in retain state!");
-          }
+        switch (Seq) {
+        case S_Use:
+          S.SetSeq(S_CanRelease);
+          continue;
+        case S_CanRelease:
+        case S_Release:
+        case S_MovableRelease:
+        case S_Stop:
+        case S_None:
+          break;
+        case S_Retain:
+          llvm_unreachable("bottom-up pointer in retain state!");
+        }
       }
 
       // Check for possible direct uses.
@@ -2332,14 +2411,14 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
       case S_Release:
       case S_MovableRelease:
         if (CanUse(Inst, Ptr, PA, Class)) {
-          S.RRI.ReverseInsertPts.clear();
+          assert(S.RRI.ReverseInsertPts.empty());
           S.RRI.ReverseInsertPts.insert(Inst);
           S.SetSeq(S_Use);
         } else if (Seq == S_Release &&
                    (Class == IC_User || Class == IC_CallOrUser)) {
           // Non-movable releases depend on any possible objc pointer use.
           S.SetSeq(S_Stop);
-          S.RRI.ReverseInsertPts.clear();
+          assert(S.RRI.ReverseInsertPts.empty());
           S.RRI.ReverseInsertPts.insert(Inst);
         }
         break;
@@ -2378,14 +2457,18 @@ ObjCARCOpt::VisitTopDown(BasicBlock *BB,
       if (Pred == BB)
         continue;
       DenseMap<const BasicBlock *, BBState>::iterator I = BBStates.find(Pred);
-      if (I == BBStates.end())
+      assert(I != BBStates.end());
+      // If we haven't seen this node yet, then we've found a CFG cycle.
+      // Be optimistic here; it's CheckForCFGHazards' job detect trouble.
+      if (!I->second.isVisitedTopDown())
         continue;
       MyStates.InitFromPred(I->second);
       while (PI != PE) {
         Pred = *PI++;
         if (Pred != BB) {
           I = BBStates.find(Pred);
-          if (I != BBStates.end())
+          assert(I != BBStates.end());
+          if (I->second.isVisitedTopDown())
             MyStates.MergePred(I->second);
         }
       }
@@ -2422,18 +2505,23 @@ ObjCARCOpt::VisitTopDown(BasicBlock *BB,
         S.SetSeq(S_Retain);
         S.RRI.clear();
         S.RRI.IsRetainBlock = Class == IC_RetainBlock;
-        S.RRI.KnownIncremented = S.IsKnownIncremented();
+        // Don't check S.IsKnownIncremented() here because it's not
+        // sufficient.
+        S.RRI.KnownSafe = S.IsKnownNested();
         S.RRI.Calls.insert(Inst);
       }
 
+      S.SetAtLeastOneRefCount();
       S.IncrementRefCount();
-      break;
+      S.IncrementNestCount();
+      continue;
     }
     case IC_Release: {
       Arg = GetObjCArg(Inst);
 
       PtrState &S = MyStates.getPtrTopDownState(Arg);
       S.DecrementRefCount();
+      S.DecrementNestCount();
 
       switch (S.GetSeq()) {
       case S_Retain:
@@ -2478,16 +2566,12 @@ ObjCARCOpt::VisitTopDown(BasicBlock *BB,
       Sequence Seq = S.GetSeq();
 
       // Check for possible releases.
-      if (!IsRetain(Class) && Class != IC_RetainBlock &&
-          CanAlterRefCount(Inst, Ptr, PA, Class)) {
-        // Check for a release.
+      if (CanAlterRefCount(Inst, Ptr, PA, Class)) {
         S.DecrementRefCount();
-
-        // Check for a release.
         switch (Seq) {
         case S_Retain:
           S.SetSeq(S_CanRelease);
-          S.RRI.ReverseInsertPts.clear();
+          assert(S.RRI.ReverseInsertPts.empty());
           S.RRI.ReverseInsertPts.insert(Inst);
 
           // One call can't cause a transition from S_Retain to S_CanRelease
@@ -2511,8 +2595,18 @@ ObjCARCOpt::VisitTopDown(BasicBlock *BB,
         if (CanUse(Inst, Ptr, PA, Class))
           S.SetSeq(S_Use);
         break;
-      case S_Use:
       case S_Retain:
+        // An objc_retainBlock call may be responsible for copying the block
+        // data from the stack to the heap. Model this by moving it straight
+        // from S_Retain to S_Use.
+        if (S.RRI.IsRetainBlock &&
+            CanUse(Inst, Ptr, PA, Class)) {
+          assert(S.RRI.ReverseInsertPts.empty());
+          S.RRI.ReverseInsertPts.insert(Inst);
+          S.SetSeq(S_Use);
+        }
+        break;
+      case S_Use:
       case S_None:
         break;
       case S_Stop:
@@ -2533,28 +2627,43 @@ ObjCARCOpt::Visit(Function &F,
                   DenseMap<const BasicBlock *, BBState> &BBStates,
                   MapVector<Value *, RRInfo> &Retains,
                   DenseMap<Value *, RRInfo> &Releases) {
-  // Use postorder for bottom-up, and reverse-postorder for top-down, because we
+  // Use reverse-postorder on the reverse CFG for bottom-up, because we
   // magically know that loops will be well behaved, i.e. they won't repeatedly
-  // call retain on a single pointer without doing a release.
+  // call retain on a single pointer without doing a release. We can't use
+  // ReversePostOrderTraversal here because we want to walk up from each
+  // function exit point.
+  SmallPtrSet<BasicBlock *, 16> Visited;
+  SmallVector<std::pair<BasicBlock *, pred_iterator>, 16> Stack;
+  SmallVector<BasicBlock *, 16> Order;
+  for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
+    BasicBlock *BB = I;
+    if (BB->getTerminator()->getNumSuccessors() == 0)
+      Stack.push_back(std::make_pair(BB, pred_begin(BB)));
+  }
+  while (!Stack.empty()) {
+    pred_iterator End = pred_end(Stack.back().first);
+    while (Stack.back().second != End) {
+      BasicBlock *BB = *Stack.back().second++;
+      if (Visited.insert(BB))
+        Stack.push_back(std::make_pair(BB, pred_begin(BB)));
+    }
+    Order.push_back(Stack.pop_back_val().first);
+  }
   bool BottomUpNestingDetected = false;
-  SmallVector<BasicBlock *, 8> PostOrder;
-  for (po_iterator<Function *> I = po_begin(&F), E = po_end(&F); I != E; ++I) {
+  for (SmallVectorImpl<BasicBlock *>::const_reverse_iterator I =
+         Order.rbegin(), E = Order.rend(); I != E; ++I) {
     BasicBlock *BB = *I;
-    PostOrder.push_back(BB);
-
     BottomUpNestingDetected |= VisitBottomUp(BB, BBStates, Retains);
   }
 
-  // Iterate through the post-order in reverse order, achieving a
-  // reverse-postorder traversal. We don't use the ReversePostOrderTraversal
-  // class here because it works by computing its own full postorder iteration,
-  // recording the sequence, and playing it back in reverse. Since we're already
-  // doing a full iteration above, we can just record the sequence manually and
-  // avoid the cost of having ReversePostOrderTraversal compute it.
+  // Use regular reverse-postorder for top-down.
   bool TopDownNestingDetected = false;
-  for (SmallVectorImpl<BasicBlock *>::const_reverse_iterator
-       RI = PostOrder.rbegin(), RE = PostOrder.rend(); RI != RE; ++RI)
-    TopDownNestingDetected |= VisitTopDown(*RI, BBStates, Releases);
+  typedef ReversePostOrderTraversal<Function *> RPOTType;
+  RPOTType RPOT(&F);
+  for (RPOTType::rpo_iterator I = RPOT.begin(), E = RPOT.end(); I != E; ++I) {
+    BasicBlock *BB = *I;
+    TopDownNestingDetected |= VisitTopDown(BB, BBStates, Releases);
+  }
 
   return TopDownNestingDetected && BottomUpNestingDetected;
 }
@@ -2565,12 +2674,10 @@ void ObjCARCOpt::MoveCalls(Value *Arg,
                            RRInfo &ReleasesToMove,
                            MapVector<Value *, RRInfo> &Retains,
                            DenseMap<Value *, RRInfo> &Releases,
-                           SmallVectorImpl<Instruction *> &DeadInsts) {
-  const Type *ArgTy = Arg->getType();
-  const Type *ParamTy =
-    (RetainRVFunc ? RetainRVFunc :
-     RetainFunc ? RetainFunc :
-     RetainBlockFunc)->arg_begin()->getType();
+                           SmallVectorImpl<Instruction *> &DeadInsts,
+                           Module *M) {
+  Type *ArgTy = Arg->getType();
+  Type *ParamTy = PointerType::getUnqual(Type::getInt8Ty(ArgTy->getContext()));
 
   // Insert the new retain and release calls.
   for (SmallPtrSet<Instruction *, 2>::const_iterator
@@ -2581,7 +2688,7 @@ void ObjCARCOpt::MoveCalls(Value *Arg,
                    new BitCastInst(Arg, ParamTy, "", InsertPt);
     CallInst *Call =
       CallInst::Create(RetainsToMove.IsRetainBlock ?
-                         RetainBlockFunc : RetainFunc,
+                         getRetainBlockCallee(M) : getRetainCallee(M),
                        MyArg, "", InsertPt);
     Call->setDoesNotThrow();
     if (!RetainsToMove.IsRetainBlock)
@@ -2598,8 +2705,8 @@ void ObjCARCOpt::MoveCalls(Value *Arg,
       // The invoke's return value isn't available in the unwind block,
       // but our releases will never depend on it, because they must be
       // paired with retains from before the invoke.
-      InsertPts[0] = II->getNormalDest()->getFirstNonPHI();
-      InsertPts[1] = II->getUnwindDest()->getFirstNonPHI();
+      InsertPts[0] = II->getNormalDest()->getFirstInsertionPt();
+      InsertPts[1] = II->getUnwindDest()->getFirstInsertionPt();
     } else {
       // Insert code immediately after the last use.
       InsertPts[0] = llvm::next(BasicBlock::iterator(LastUse));
@@ -2609,7 +2716,8 @@ void ObjCARCOpt::MoveCalls(Value *Arg,
       Instruction *InsertPt = *I;
       Value *MyArg = ArgTy == ParamTy ? Arg :
                      new BitCastInst(Arg, ParamTy, "", InsertPt);
-      CallInst *Call = CallInst::Create(ReleaseFunc, MyArg, "", InsertPt);
+      CallInst *Call = CallInst::Create(getReleaseCallee(M), MyArg,
+                                        "", InsertPt);
       // Attach a clang.imprecise_release metadata tag, if appropriate.
       if (MDNode *M = ReleasesToMove.ReleaseMetadata)
         Call->setMetadata(ImpreciseReleaseMDKind, M);
@@ -2640,7 +2748,8 @@ bool
 ObjCARCOpt::PerformCodePlacement(DenseMap<const BasicBlock *, BBState>
                                    &BBStates,
                                  MapVector<Value *, RRInfo> &Retains,
-                                 DenseMap<Value *, RRInfo> &Releases) {
+                                 DenseMap<Value *, RRInfo> &Releases,
+                                 Module *M) {
   bool AnyPairsCompletelyEliminated = false;
   RRInfo RetainsToMove;
   RRInfo ReleasesToMove;
@@ -2649,21 +2758,36 @@ ObjCARCOpt::PerformCodePlacement(DenseMap<const BasicBlock *, BBState>
   SmallVector<Instruction *, 8> DeadInsts;
 
   for (MapVector<Value *, RRInfo>::const_iterator I = Retains.begin(),
-       E = Retains.end(); I != E; ) {
-    Value *V = (I++)->first;
+       E = Retains.end(); I != E; ++I) {
+    Value *V = I->first;
     if (!V) continue; // blotted
 
     Instruction *Retain = cast<Instruction>(V);
     Value *Arg = GetObjCArg(Retain);
 
-    // If the object being released is in static or stack storage, we know it's
+    // If the object being released is in static storage, we know it's
     // not being managed by ObjC reference counting, so we can delete pairs
     // regardless of what possible decrements or uses lie between them.
-    bool KnownSafe = isa<Constant>(Arg) || isa<AllocaInst>(Arg);
+    bool KnownSafe = isa<Constant>(Arg);
+   
+    // Same for stack storage, unless this is an objc_retainBlock call,
+    // which is responsible for copying the block data from the stack to
+    // the heap.
+    if (!I->second.IsRetainBlock && isa<AllocaInst>(Arg))
+      KnownSafe = true;
+
+    // A constant pointer can't be pointing to an object on the heap. It may
+    // be reference-counted, but it won't be deleted.
+    if (const LoadInst *LI = dyn_cast<LoadInst>(Arg))
+      if (const GlobalVariable *GV =
+            dyn_cast<GlobalVariable>(
+              StripPointerCastsAndObjCCalls(LI->getPointerOperand())))
+        if (GV->isConstant())
+          KnownSafe = true;
 
     // If a pair happens in a region where it is known that the reference count
     // is already incremented, we can similarly ignore possible decrements.
-    bool KnownIncrementedTD = true, KnownIncrementedBU = true;
+    bool KnownSafeTD = true, KnownSafeBU = true;
 
     // Connect the dots between the top-down-collected RetainsToMove and
     // bottom-up-collected ReleasesToMove to form sets of related calls.
@@ -2683,7 +2807,7 @@ ObjCARCOpt::PerformCodePlacement(DenseMap<const BasicBlock *, BBState>
         MapVector<Value *, RRInfo>::const_iterator It = Retains.find(NewRetain);
         assert(It != Retains.end());
         const RRInfo &NewRetainRRI = It->second;
-        KnownIncrementedTD &= NewRetainRRI.KnownIncremented;
+        KnownSafeTD &= NewRetainRRI.KnownSafe;
         for (SmallPtrSet<Instruction *, 2>::const_iterator
              LI = NewRetainRRI.Calls.begin(),
              LE = NewRetainRRI.Calls.end(); LI != LE; ++LI) {
@@ -2739,7 +2863,7 @@ ObjCARCOpt::PerformCodePlacement(DenseMap<const BasicBlock *, BBState>
           Releases.find(NewRelease);
         assert(It != Releases.end());
         const RRInfo &NewReleaseRRI = It->second;
-        KnownIncrementedBU &= NewReleaseRRI.KnownIncremented;
+        KnownSafeBU &= NewReleaseRRI.KnownSafe;
         for (SmallPtrSet<Instruction *, 2>::const_iterator
              LI = NewReleaseRRI.Calls.begin(),
              LE = NewReleaseRRI.Calls.end(); LI != LE; ++LI) {
@@ -2787,12 +2911,19 @@ ObjCARCOpt::PerformCodePlacement(DenseMap<const BasicBlock *, BBState>
       if (NewRetains.empty()) break;
     }
 
-    // If the pointer is known incremented, we can safely delete the pair
-    // regardless of what's between them.
-    if (KnownIncrementedTD || KnownIncrementedBU) {
+    // If the pointer is known incremented or nested, we can safely delete the
+    // pair regardless of what's between them.
+    if (KnownSafeTD || KnownSafeBU) {
       RetainsToMove.ReverseInsertPts.clear();
       ReleasesToMove.ReverseInsertPts.clear();
       NewCount = 0;
+    } else {
+      // Determine whether the new insertion points we computed preserve the
+      // balance of retain and release calls through the program.
+      // TODO: If the fully aggressive solution isn't valid, try to find a
+      // less aggressive solution which is.
+      if (NewDelta != 0)
+        goto next_retain;
     }
 
     // Determine whether the original call points are balanced in the retain and
@@ -2803,18 +2934,12 @@ ObjCARCOpt::PerformCodePlacement(DenseMap<const BasicBlock *, BBState>
     if (OldDelta != 0)
       goto next_retain;
 
-    // Determine whether the new insertion points we computed preserve the
-    // balance of retain and release calls through the program.
-    // TODO: If the fully aggressive solution isn't valid, try to find a
-    // less aggressive solution which is.
-    if (NewDelta != 0)
-      goto next_retain;
-
     // Ok, everything checks out and we're all set. Let's move some code!
     Changed = true;
     AnyPairsCompletelyEliminated = NewCount == 0;
     NumRRs += OldCount - NewCount;
-    MoveCalls(Arg, RetainsToMove, ReleasesToMove, Retains, Releases, DeadInsts);
+    MoveCalls(Arg, RetainsToMove, ReleasesToMove,
+              Retains, Releases, DeadInsts, M);
 
   next_retain:
     NewReleases.clear();
@@ -2993,7 +3118,8 @@ bool ObjCARCOpt::OptimizeSequences(Function &F) {
   bool NestingDetected = Visit(F, BBStates, Retains, Releases);
 
   // Transform.
-  return PerformCodePlacement(BBStates, Retains, Releases) && NestingDetected;
+  return PerformCodePlacement(BBStates, Retains, Releases, F.getParent()) &&
+         NestingDetected;
 }
 
 /// OptimizeReturns - Look for this pattern:
@@ -3072,7 +3198,8 @@ void ObjCARCOpt::OptimizeReturns(Function &F) {
 
         // Check that there is nothing that can affect the reference
         // count between the retain and the call.
-        FindDependencies(CanChangeRetainCount, Arg, BB, Retain,
+        // Note that Retain need not be in BB.
+        FindDependencies(CanChangeRetainCount, Arg, Retain->getParent(), Retain,
                          DependingInstructions, Visited, PA);
         if (DependingInstructions.size() != 1)
           goto next_block;
@@ -3117,12 +3244,6 @@ bool ObjCARCOpt::doInitialization(Module &M) {
   ImpreciseReleaseMDKind =
     M.getContext().getMDKindID("clang.imprecise_release");
 
-  // Identify the declarations for objc_retain and friends.
-  RetainFunc = M.getFunction("objc_retain");
-  RetainBlockFunc = M.getFunction("objc_retainBlock");
-  RetainRVFunc = M.getFunction("objc_retainAutoreleasedReturnValue");
-  ReleaseFunc = M.getFunction("objc_release");
-
   // Intuitively, objc_retain and others are nocapture, however in practice
   // they are not, because they return their argument value. And objc_release
   // calls finalizers.
@@ -3132,6 +3253,7 @@ bool ObjCARCOpt::doInitialization(Module &M) {
   AutoreleaseRVCallee = 0;
   ReleaseCallee = 0;
   RetainCallee = 0;
+  RetainBlockCallee = 0;
   AutoreleaseCallee = 0;
 
   return false;
@@ -3294,7 +3416,7 @@ Constant *ObjCARCContract::getRetainAutoreleaseCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     std::vector<Type *> Params;
     Params.push_back(I8X);
-    const FunctionType *FTy =
+    FunctionType *FTy =
       FunctionType::get(I8X, Params, /*isVarArg=*/false);
     AttrListPtr Attributes;
     Attributes.addAttr(~0u, Attribute::NoUnwind);
@@ -3310,7 +3432,7 @@ Constant *ObjCARCContract::getRetainAutoreleaseRVCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     std::vector<Type *> Params;
     Params.push_back(I8X);
-    const FunctionType *FTy =
+    FunctionType *FTy =
       FunctionType::get(I8X, Params, /*isVarArg=*/false);
     AttrListPtr Attributes;
     Attributes.addAttr(~0u, Attribute::NoUnwind);
@@ -3377,7 +3499,7 @@ ObjCARCContract::ContractAutorelease(Function &F, Instruction *Autorelease,
 void ObjCARCContract::ContractRelease(Instruction *Release,
                                       inst_iterator &Iter) {
   LoadInst *Load = dyn_cast<LoadInst>(GetObjCArg(Release));
-  if (!Load || Load->isVolatile()) return;
+  if (!Load || !Load->isSimple()) return;
 
   // For now, require everything to be in one basic block.
   BasicBlock *BB = Release->getParent();
@@ -3393,7 +3515,7 @@ void ObjCARCContract::ContractRelease(Instruction *Release,
           !(AA->getModRefInfo(I, Loc) & AliasAnalysis::Mod)))
     ++I;
   StoreInst *Store = dyn_cast<StoreInst>(I);
-  if (!Store || Store->isVolatile()) return;
+  if (!Store || !Store->isSimple()) return;
   if (Store->getPointerOperand() != Loc.Ptr) return;
 
   Value *New = StripPointerCastsAndObjCCalls(Store->getValueOperand());
@@ -3411,8 +3533,8 @@ void ObjCARCContract::ContractRelease(Instruction *Release,
   ++NumStoreStrongs;
 
   LLVMContext &C = Release->getContext();
-  const Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
-  const Type *I8XX = PointerType::getUnqual(I8X);
+  Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
+  Type *I8XX = PointerType::getUnqual(I8X);
 
   Value *Args[] = { Load->getPointerOperand(), New };
   if (Args[0]->getType() != I8XX)
@@ -3548,7 +3670,7 @@ bool ObjCARCContract::runOnFunction(Function &F) {
           if (Inst != UserInst && DT->dominates(Inst, UserInst)) {
             Changed = true;
             Instruction *Replacement = Inst;
-            const Type *UseTy = U.get()->getType();
+            Type *UseTy = U.get()->getType();
             if (PHINode *PHI = dyn_cast<PHINode>(UserInst)) {
               // For PHI nodes, insert the bitcast in the predecessor block.
               unsigned ValNo =
diff --git a/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp b/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp
index e6341ae3071f..8f98a5b6503e 100644
--- a/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp
@@ -309,7 +309,7 @@ void Reassociate::LinearizeExprTree(BinaryOperator *I,
     std::swap(LHS, RHS);
     bool Success = !I->swapOperands();
     assert(Success && "swapOperands failed");
-    Success = false;
+    (void)Success;
     MadeChange = true;
   } else if (RHSBO) {
     // Turn (A+B)+(C+D) -> (((A+B)+C)+D).  This guarantees the RHS is not
diff --git a/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp b/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp
index 083412ed942d..196a847fc0ea 100644
--- a/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -156,7 +156,7 @@ namespace {
 ///
 class SCCPSolver : public InstVisitor<SCCPSolver> {
   const TargetData *TD;
-  SmallPtrSet<BasicBlock*, 8> BBExecutable;// The BBs that are executable.
+  SmallPtrSet<BasicBlock*, 8> BBExecutable; // The BBs that are executable.
   DenseMap<Value*, LatticeVal> ValueState;  // The state each value is in.
 
   /// StructValueState - This maintains ValueState for values that have
@@ -241,7 +241,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   /// this method must be called.
   void AddTrackedFunction(Function *F) {
     // Add an entry, F -> undef.
-    if (const StructType *STy = dyn_cast<StructType>(F->getReturnType())) {
+    if (StructType *STy = dyn_cast<StructType>(F->getReturnType())) {
       MRVFunctionsTracked.insert(F);
       for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
         TrackedMultipleRetVals.insert(std::make_pair(std::make_pair(F, i),
@@ -302,7 +302,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   /// markAnythingOverdefined - Mark the specified value overdefined.  This
   /// works with both scalars and structs.
   void markAnythingOverdefined(Value *V) {
-    if (const StructType *STy = dyn_cast<StructType>(V->getType()))
+    if (StructType *STy = dyn_cast<StructType>(V->getType()))
       for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
         markOverdefined(getStructValueState(V, i), V);
     else
@@ -417,7 +417,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
       else if (ConstantStruct *CS = dyn_cast<ConstantStruct>(C))
         LV.markConstant(CS->getOperand(i));      // Constants are constant.
       else if (isa<ConstantAggregateZero>(C)) {
-        const Type *FieldTy = cast<StructType>(V->getType())->getElementType(i);
+        Type *FieldTy = cast<StructType>(V->getType())->getElementType(i);
         LV.markConstant(Constant::getNullValue(FieldTy));
       } else
         LV.markOverdefined();      // Unknown sort of constant.
@@ -471,9 +471,9 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   /// UsersOfOverdefinedPHIs map for PN, remove them now.
   void RemoveFromOverdefinedPHIs(Instruction *I, PHINode *PN) {
     if (UsersOfOverdefinedPHIs.empty()) return;
-    std::multimap<PHINode*, Instruction*>::iterator It, E;
-    tie(It, E) = UsersOfOverdefinedPHIs.equal_range(PN);
-    while (It != E) {
+    typedef std::multimap<PHINode*, Instruction*>::iterator ItTy;
+    std::pair<ItTy, ItTy> Range = UsersOfOverdefinedPHIs.equal_range(PN);
+    for (ItTy It = Range.first, E = Range.second; It != E;) {
       if (It->second == I)
         UsersOfOverdefinedPHIs.erase(It++);
       else
@@ -486,9 +486,9 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   /// (Duplicate entries do not break anything directly, but can lead to
   /// exponential growth of the table in rare cases.)
   void InsertInOverdefinedPHIs(Instruction *I, PHINode *PN) {
-    std::multimap<PHINode*, Instruction*>::iterator J, E;
-    tie(J, E) = UsersOfOverdefinedPHIs.equal_range(PN);
-    for (; J != E; ++J)
+    typedef std::multimap<PHINode*, Instruction*>::iterator ItTy;
+    std::pair<ItTy, ItTy> Range = UsersOfOverdefinedPHIs.equal_range(PN);
+    for (ItTy J = Range.first, E = Range.second; J != E; ++J)
       if (J->second == I)
         return;
     UsersOfOverdefinedPHIs.insert(std::make_pair(PN, I));
@@ -515,6 +515,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   void visitShuffleVectorInst(ShuffleVectorInst &I);
   void visitExtractValueInst(ExtractValueInst &EVI);
   void visitInsertValueInst(InsertValueInst &IVI);
+  void visitLandingPadInst(LandingPadInst &I) { markAnythingOverdefined(&I); }
 
   // Instructions that cannot be folded away.
   void visitStoreInst     (StoreInst &I);
@@ -528,8 +529,12 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     visitTerminatorInst(II);
   }
   void visitCallSite      (CallSite CS);
+  void visitResumeInst    (TerminatorInst &I) { /*returns void*/ }
   void visitUnwindInst    (TerminatorInst &I) { /*returns void*/ }
   void visitUnreachableInst(TerminatorInst &I) { /*returns void*/ }
+  void visitFenceInst     (FenceInst &I) { /*returns void*/ }
+  void visitAtomicCmpXchgInst (AtomicCmpXchgInst &I) { markOverdefined(&I); }
+  void visitAtomicRMWInst (AtomicRMWInst &I) { markOverdefined(&I); }
   void visitAllocaInst    (Instruction &I) { markOverdefined(&I); }
   void visitVAArgInst     (Instruction &I) { markAnythingOverdefined(&I); }
 
@@ -577,6 +582,10 @@ void SCCPSolver::getFeasibleSuccessors(TerminatorInst &TI,
   }
   
   if (SwitchInst *SI = dyn_cast<SwitchInst>(&TI)) {
+    if (TI.getNumSuccessors() < 2) {
+      Succs[0] = true;
+      return;
+    }
     LatticeVal SCValue = getValueState(SI->getCondition());
     ConstantInt *CI = SCValue.getConstantInt();
     
@@ -637,6 +646,9 @@ bool SCCPSolver::isEdgeFeasible(BasicBlock *From, BasicBlock *To) {
     return true;
   
   if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
+    if (SI->getNumSuccessors() < 2)
+      return true;
+
     LatticeVal SCValue = getValueState(SI->getCondition());
     ConstantInt *CI = SCValue.getConstantInt();
     
@@ -692,13 +704,14 @@ void SCCPSolver::visitPHINode(PHINode &PN) {
     // There may be instructions using this PHI node that are not overdefined
     // themselves.  If so, make sure that they know that the PHI node operand
     // changed.
-    std::multimap<PHINode*, Instruction*>::iterator I, E;
-    tie(I, E) = UsersOfOverdefinedPHIs.equal_range(&PN);
-    if (I == E)
+    typedef std::multimap<PHINode*, Instruction*>::iterator ItTy;
+    std::pair<ItTy, ItTy> Range = UsersOfOverdefinedPHIs.equal_range(&PN);
+    
+    if (Range.first == Range.second)
       return;
     
     SmallVector<Instruction*, 16> Users;
-    for (; I != E; ++I)
+    for (ItTy I = Range.first, E = Range.second; I != E; ++I)
       Users.push_back(I->second);
     while (!Users.empty())
       visit(Users.pop_back_val());
@@ -772,7 +785,7 @@ void SCCPSolver::visitReturnInst(ReturnInst &I) {
   
   // Handle functions that return multiple values.
   if (!TrackedMultipleRetVals.empty()) {
-    if (const StructType *STy = dyn_cast<StructType>(ResultOp->getType()))
+    if (StructType *STy = dyn_cast<StructType>(ResultOp->getType()))
       if (MRVFunctionsTracked.count(F))
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
           mergeInValue(TrackedMultipleRetVals[std::make_pair(F, i)], F,
@@ -825,7 +838,7 @@ void SCCPSolver::visitExtractValueInst(ExtractValueInst &EVI) {
 }
 
 void SCCPSolver::visitInsertValueInst(InsertValueInst &IVI) {
-  const StructType *STy = dyn_cast<StructType>(IVI.getType());
+  StructType *STy = dyn_cast<StructType>(IVI.getType());
   if (STy == 0)
     return markOverdefined(&IVI);
   
@@ -925,7 +938,7 @@ void SCCPSolver::visitBinaryOperator(Instruction &I) {
         // Could annihilate value.
         if (I.getOpcode() == Instruction::And)
           markConstant(IV, &I, Constant::getNullValue(I.getType()));
-        else if (const VectorType *PT = dyn_cast<VectorType>(I.getType()))
+        else if (VectorType *PT = dyn_cast<VectorType>(I.getType()))
           markConstant(IV, &I, Constant::getAllOnesValue(PT));
         else
           markConstant(IV, &I,
@@ -1179,8 +1192,8 @@ void SCCPSolver::visitGetElementPtrInst(GetElementPtrInst &I) {
   }
 
   Constant *Ptr = Operands[0];
-  markConstant(&I, ConstantExpr::getGetElementPtr(Ptr, &Operands[0]+1,
-                                                  Operands.size()-1));
+  ArrayRef<Constant *> Indices(Operands.begin() + 1, Operands.end());
+  markConstant(&I, ConstantExpr::getGetElementPtr(Ptr, Indices));
 }
 
 void SCCPSolver::visitStoreInst(StoreInst &SI) {
@@ -1278,7 +1291,7 @@ void SCCPSolver::visitCallSite(CallSite CS) {
      
       // If we can constant fold this, mark the result of the call as a
       // constant.
-      if (Constant *C = ConstantFoldCall(F, Operands.data(), Operands.size()))
+      if (Constant *C = ConstantFoldCall(F, Operands))
         return markConstant(I, C);
     }
 
@@ -1303,7 +1316,7 @@ void SCCPSolver::visitCallSite(CallSite CS) {
         continue;
       }
       
-      if (const StructType *STy = dyn_cast<StructType>(AI->getType())) {
+      if (StructType *STy = dyn_cast<StructType>(AI->getType())) {
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           LatticeVal CallArg = getStructValueState(*CAI, i);
           mergeInValue(getStructValueState(AI, i), AI, CallArg);
@@ -1315,7 +1328,7 @@ void SCCPSolver::visitCallSite(CallSite CS) {
   }
   
   // If this is a single/zero retval case, see if we're tracking the function.
-  if (const StructType *STy = dyn_cast<StructType>(F->getReturnType())) {
+  if (StructType *STy = dyn_cast<StructType>(F->getReturnType())) {
     if (!MRVFunctionsTracked.count(F))
       goto CallOverdefined;  // Not tracking this callee.
     
@@ -1419,67 +1432,116 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) {
       // Look for instructions which produce undef values.
       if (I->getType()->isVoidTy()) continue;
       
-      if (const StructType *STy = dyn_cast<StructType>(I->getType())) {
-        // Only a few things that can be structs matter for undef.  Just send
-        // all their results to overdefined.  We could be more precise than this
-        // but it isn't worth bothering.
-        if (isa<CallInst>(I) || isa<SelectInst>(I)) {
-          for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-            LatticeVal &LV = getStructValueState(I, i);
-            if (LV.isUndefined())
-              markOverdefined(LV, I);
-          }
+      if (StructType *STy = dyn_cast<StructType>(I->getType())) {
+        // Only a few things that can be structs matter for undef.
+
+        // Tracked calls must never be marked overdefined in ResolvedUndefsIn.
+        if (CallSite CS = CallSite(I))
+          if (Function *F = CS.getCalledFunction())
+            if (MRVFunctionsTracked.count(F))
+              continue;
+
+        // extractvalue and insertvalue don't need to be marked; they are
+        // tracked as precisely as their operands. 
+        if (isa<ExtractValueInst>(I) || isa<InsertValueInst>(I))
+          continue;
+
+        // Send the results of everything else to overdefined.  We could be
+        // more precise than this but it isn't worth bothering.
+        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
+          LatticeVal &LV = getStructValueState(I, i);
+          if (LV.isUndefined())
+            markOverdefined(LV, I);
         }
         continue;
       }
-      
+
       LatticeVal &LV = getValueState(I);
       if (!LV.isUndefined()) continue;
 
-      // No instructions using structs need disambiguation.
-      if (I->getOperand(0)->getType()->isStructTy())
+      // extractvalue is safe; check here because the argument is a struct.
+      if (isa<ExtractValueInst>(I))
         continue;
 
-      // Get the lattice values of the first two operands for use below.
+      // Compute the operand LatticeVals, for convenience below.
+      // Anything taking a struct is conservatively assumed to require
+      // overdefined markings.
+      if (I->getOperand(0)->getType()->isStructTy()) {
+        markOverdefined(I);
+        return true;
+      }
       LatticeVal Op0LV = getValueState(I->getOperand(0));
       LatticeVal Op1LV;
       if (I->getNumOperands() == 2) {
-        // No instructions using structs need disambiguation.
-        if (I->getOperand(1)->getType()->isStructTy())
-          continue;
-        
-        // If this is a two-operand instruction, and if both operands are
-        // undefs, the result stays undef.
+        if (I->getOperand(1)->getType()->isStructTy()) {
+          markOverdefined(I);
+          return true;
+        }
+
         Op1LV = getValueState(I->getOperand(1));
-        if (Op0LV.isUndefined() && Op1LV.isUndefined())
-          continue;
       }
-      
       // If this is an instructions whose result is defined even if the input is
       // not fully defined, propagate the information.
-      const Type *ITy = I->getType();
+      Type *ITy = I->getType();
       switch (I->getOpcode()) {
-      default: break;          // Leave the instruction as an undef.
+      case Instruction::Add:
+      case Instruction::Sub:
+      case Instruction::Trunc:
+      case Instruction::FPTrunc:
+      case Instruction::BitCast:
+        break; // Any undef -> undef
+      case Instruction::FSub:
+      case Instruction::FAdd:
+      case Instruction::FMul:
+      case Instruction::FDiv:
+      case Instruction::FRem:
+        // Floating-point binary operation: be conservative.
+        if (Op0LV.isUndefined() && Op1LV.isUndefined())
+          markForcedConstant(I, Constant::getNullValue(ITy));
+        else
+          markOverdefined(I);
+        return true;
       case Instruction::ZExt:
-        // After a zero extend, we know the top part is zero.  SExt doesn't have
-        // to be handled here, because we don't know whether the top part is 1's
-        // or 0's.
-      case Instruction::SIToFP:  // some FP values are not possible, just use 0.
-      case Instruction::UIToFP:  // some FP values are not possible, just use 0.
+      case Instruction::SExt:
+      case Instruction::FPToUI:
+      case Instruction::FPToSI:
+      case Instruction::FPExt:
+      case Instruction::PtrToInt:
+      case Instruction::IntToPtr:
+      case Instruction::SIToFP:
+      case Instruction::UIToFP:
+        // undef -> 0; some outputs are impossible
         markForcedConstant(I, Constant::getNullValue(ITy));
         return true;
       case Instruction::Mul:
       case Instruction::And:
+        // Both operands undef -> undef
+        if (Op0LV.isUndefined() && Op1LV.isUndefined())
+          break;
         // undef * X -> 0.   X could be zero.
         // undef & X -> 0.   X could be zero.
         markForcedConstant(I, Constant::getNullValue(ITy));
         return true;
 
       case Instruction::Or:
+        // Both operands undef -> undef
+        if (Op0LV.isUndefined() && Op1LV.isUndefined())
+          break;
         // undef | X -> -1.   X could be -1.
         markForcedConstant(I, Constant::getAllOnesValue(ITy));
         return true;
 
+      case Instruction::Xor:
+        // undef ^ undef -> 0; strictly speaking, this is not strictly
+        // necessary, but we try to be nice to people who expect this
+        // behavior in simple cases
+        if (Op0LV.isUndefined() && Op1LV.isUndefined()) {
+          markForcedConstant(I, Constant::getNullValue(ITy));
+          return true;
+        }
+        // undef ^ X -> undef
+        break;
+
       case Instruction::SDiv:
       case Instruction::UDiv:
       case Instruction::SRem:
@@ -1494,26 +1556,24 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) {
         return true;
         
       case Instruction::AShr:
-        // undef >>s X -> undef.  No change.
-        if (Op0LV.isUndefined()) break;
-        
-        // X >>s undef -> X.  X could be 0, X could have the high-bit known set.
-        if (Op0LV.isConstant())
-          markForcedConstant(I, Op0LV.getConstant());
-        else
-          markOverdefined(I);
+        // X >>a undef -> undef.
+        if (Op1LV.isUndefined()) break;
+
+        // undef >>a X -> all ones
+        markForcedConstant(I, Constant::getAllOnesValue(ITy));
         return true;
       case Instruction::LShr:
       case Instruction::Shl:
-        // undef >> X -> undef.  No change.
-        // undef << X -> undef.  No change.
-        if (Op0LV.isUndefined()) break;
-        
-        // X >> undef -> 0.  X could be 0.
-        // X << undef -> 0.  X could be 0.
+        // X << undef -> undef.
+        // X >> undef -> undef.
+        if (Op1LV.isUndefined()) break;
+
+        // undef << X -> 0
+        // undef >> X -> 0
         markForcedConstant(I, Constant::getNullValue(ITy));
         return true;
       case Instruction::Select:
+        Op1LV = getValueState(I->getOperand(1));
         // undef ? X : Y  -> X or Y.  There could be commonality between X/Y.
         if (Op0LV.isUndefined()) {
           if (!Op1LV.isConstant())  // Pick the constant one if there is any.
@@ -1533,9 +1593,35 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) {
         else
           markOverdefined(I);
         return true;
+      case Instruction::Load:
+        // A load here means one of two things: a load of undef from a global,
+        // a load from an unknown pointer.  Either way, having it return undef
+        // is okay.
+        break;
+      case Instruction::ICmp:
+        // X == undef -> undef.  Other comparisons get more complicated.
+        if (cast<ICmpInst>(I)->isEquality())
+          break;
+        markOverdefined(I);
+        return true;
       case Instruction::Call:
-        // If a call has an undef result, it is because it is constant foldable
-        // but one of the inputs was undef.  Just force the result to
+      case Instruction::Invoke: {
+        // There are two reasons a call can have an undef result
+        // 1. It could be tracked.
+        // 2. It could be constant-foldable.
+        // Because of the way we solve return values, tracked calls must
+        // never be marked overdefined in ResolvedUndefsIn.
+        if (Function *F = CallSite(I).getCalledFunction())
+          if (TrackedRetVals.count(F))
+            break;
+
+        // If the call is constant-foldable, we mark it overdefined because
+        // we do not know what return values are valid.
+        markOverdefined(I);
+        return true;
+      }
+      default:
+        // If we don't know what should happen here, conservatively mark it
         // overdefined.
         markOverdefined(I);
         return true;
@@ -1621,15 +1707,25 @@ FunctionPass *llvm::createSCCPPass() {
 static void DeleteInstructionInBlock(BasicBlock *BB) {
   DEBUG(dbgs() << "  BasicBlock Dead:" << *BB);
   ++NumDeadBlocks;
-  
-  // Delete the instructions backwards, as it has a reduced likelihood of
-  // having to update as many def-use and use-def chains.
-  while (!isa<TerminatorInst>(BB->begin())) {
-    Instruction *I = --BasicBlock::iterator(BB->getTerminator());
-    
-    if (!I->use_empty())
-      I->replaceAllUsesWith(UndefValue::get(I->getType()));
-    BB->getInstList().erase(I);
+
+  // Check to see if there are non-terminating instructions to delete.
+  if (isa<TerminatorInst>(BB->begin()))
+    return;
+
+  // Delete the instructions backwards, as it has a reduced likelihood of having
+  // to update as many def-use and use-def chains.
+  Instruction *EndInst = BB->getTerminator(); // Last not to be deleted.
+  while (EndInst != BB->begin()) {
+    // Delete the next to last instruction.
+    BasicBlock::iterator I = EndInst;
+    Instruction *Inst = --I;
+    if (!Inst->use_empty())
+      Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
+    if (isa<LandingPadInst>(Inst)) {
+      EndInst = Inst;
+      continue;
+    }
+    BB->getInstList().erase(Inst);
     ++NumInstRemoved;
   }
 }
diff --git a/contrib/llvm/lib/Transforms/Scalar/Scalar.cpp b/contrib/llvm/lib/Transforms/Scalar/Scalar.cpp
index 302c287d3cbd..f6918deafebd 100644
--- a/contrib/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -63,7 +63,6 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeCFGSimplifyPassPass(Registry);
   initializeSimplifyLibCallsPass(Registry);
   initializeSinkingPass(Registry);
-  initializeTailDupPass(Registry);
   initializeTailCallElimPass(Registry);
 }
 
@@ -187,3 +186,7 @@ void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM) {
 void LLVMAddBasicAliasAnalysisPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createBasicAliasAnalysisPass());
 }
+
+void LLVMAddLowerExpectIntrinsicPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createLowerExpectIntrinsicPass());
+}
diff --git a/contrib/llvm/lib/Transforms/Scalar/ScalarReplAggregates.cpp b/contrib/llvm/lib/Transforms/Scalar/ScalarReplAggregates.cpp
index 7d6349cf4e77..c6d9123d6611 100644
--- a/contrib/llvm/lib/Transforms/Scalar/ScalarReplAggregates.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/ScalarReplAggregates.cpp
@@ -129,11 +129,11 @@ namespace {
                                          AllocaInfo &Info);
     void isSafeGEP(GetElementPtrInst *GEPI, uint64_t &Offset, AllocaInfo &Info);
     void isSafeMemAccess(uint64_t Offset, uint64_t MemSize,
-                         const Type *MemOpType, bool isStore, AllocaInfo &Info,
+                         Type *MemOpType, bool isStore, AllocaInfo &Info,
                          Instruction *TheAccess, bool AllowWholeAccess);
-    bool TypeHasComponent(const Type *T, uint64_t Offset, uint64_t Size);
-    uint64_t FindElementAndOffset(const Type *&T, uint64_t &Offset,
-                                  const Type *&IdxTy);
+    bool TypeHasComponent(Type *T, uint64_t Offset, uint64_t Size);
+    uint64_t FindElementAndOffset(Type *&T, uint64_t &Offset,
+                                  Type *&IdxTy);
 
     void DoScalarReplacement(AllocaInst *AI,
                              std::vector<AllocaInst*> &WorkList);
@@ -145,6 +145,9 @@ namespace {
                         SmallVector<AllocaInst*, 32> &NewElts);
     void RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
                     SmallVector<AllocaInst*, 32> &NewElts);
+    void RewriteLifetimeIntrinsic(IntrinsicInst *II, AllocaInst *AI,
+                                  uint64_t Offset,
+                                  SmallVector<AllocaInst*, 32> &NewElts);
     void RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
                                       AllocaInst *AI,
                                       SmallVector<AllocaInst*, 32> &NewElts);
@@ -253,7 +256,7 @@ class ConvertToScalarInfo {
   /// VectorTy - This tracks the type that we should promote the vector to if
   /// it is possible to turn it into a vector.  This starts out null, and if it
   /// isn't possible to turn into a vector type, it gets set to VoidTy.
-  const VectorType *VectorTy;
+  VectorType *VectorTy;
 
   /// HadNonMemTransferAccess - True if there is at least one access to the 
   /// alloca that is not a MemTransferInst.  We don't want to turn structs into
@@ -269,11 +272,11 @@ class ConvertToScalarInfo {
 
 private:
   bool CanConvertToScalar(Value *V, uint64_t Offset);
-  void MergeInTypeForLoadOrStore(const Type *In, uint64_t Offset);
-  bool MergeInVectorType(const VectorType *VInTy, uint64_t Offset);
+  void MergeInTypeForLoadOrStore(Type *In, uint64_t Offset);
+  bool MergeInVectorType(VectorType *VInTy, uint64_t Offset);
   void ConvertUsesToScalar(Value *Ptr, AllocaInst *NewAI, uint64_t Offset);
 
-  Value *ConvertScalar_ExtractValue(Value *NV, const Type *ToType,
+  Value *ConvertScalar_ExtractValue(Value *NV, Type *ToType,
                                     uint64_t Offset, IRBuilder<> &Builder);
   Value *ConvertScalar_InsertValue(Value *StoredVal, Value *ExistingVal,
                                    uint64_t Offset, IRBuilder<> &Builder);
@@ -295,8 +298,6 @@ AllocaInst *ConvertToScalarInfo::TryConvert(AllocaInst *AI) {
   if (ScalarKind == Unknown)
     ScalarKind = Integer;
 
-  // FIXME: It should be possible to promote the vector type up to the alloca's
-  // size.
   if (ScalarKind == Vector && VectorTy->getBitWidth() != AllocaSize * 8)
     ScalarKind = Integer;
 
@@ -306,7 +307,7 @@ AllocaInst *ConvertToScalarInfo::TryConvert(AllocaInst *AI) {
   // random stuff that doesn't use vectors (e.g. <9 x double>) because then
   // we just get a lot of insert/extracts.  If at least one vector is
   // involved, then we probably really do have a union of vector/array.
-  const Type *NewTy;
+  Type *NewTy;
   if (ScalarKind == Vector) {
     assert(VectorTy && "Missing type for vector scalar.");
     DEBUG(dbgs() << "CONVERT TO VECTOR: " << *AI << "\n  TYPE = "
@@ -331,20 +332,16 @@ AllocaInst *ConvertToScalarInfo::TryConvert(AllocaInst *AI) {
 /// (VectorTy) so far at the offset specified by Offset (which is specified in
 /// bytes).
 ///
-/// There are three cases we handle here:
+/// There are two cases we handle here:
 ///   1) A union of vector types of the same size and potentially its elements.
 ///      Here we turn element accesses into insert/extract element operations.
 ///      This promotes a <4 x float> with a store of float to the third element
 ///      into a <4 x float> that uses insert element.
-///   2) A union of vector types with power-of-2 size differences, e.g. a float,
-///      <2 x float> and <4 x float>.  Here we turn element accesses into insert
-///      and extract element operations, and <2 x float> accesses into a cast to
-///      <2 x double>, an extract, and a cast back to <2 x float>.
-///   3) A fully general blob of memory, which we turn into some (potentially
+///   2) A fully general blob of memory, which we turn into some (potentially
 ///      large) integer type with extract and insert operations where the loads
 ///      and stores would mutate the memory.  We mark this by setting VectorTy
 ///      to VoidTy.
-void ConvertToScalarInfo::MergeInTypeForLoadOrStore(const Type *In,
+void ConvertToScalarInfo::MergeInTypeForLoadOrStore(Type *In,
                                                     uint64_t Offset) {
   // If we already decided to turn this into a blob of integer memory, there is
   // nothing to be done.
@@ -355,7 +352,7 @@ void ConvertToScalarInfo::MergeInTypeForLoadOrStore(const Type *In,
 
   // If the In type is a vector that is the same size as the alloca, see if it
   // matches the existing VecTy.
-  if (const VectorType *VInTy = dyn_cast<VectorType>(In)) {
+  if (VectorType *VInTy = dyn_cast<VectorType>(In)) {
     if (MergeInVectorType(VInTy, Offset))
       return;
   } else if (In->isFloatTy() || In->isDoubleTy() ||
@@ -371,20 +368,13 @@ void ConvertToScalarInfo::MergeInTypeForLoadOrStore(const Type *In,
     // if the implied vector agrees with what we already have and if Offset is
     // compatible with it.
     if (Offset % EltSize == 0 && AllocaSize % EltSize == 0 &&
-        (!VectorTy || Offset * 8 < VectorTy->getPrimitiveSizeInBits())) {
+        (!VectorTy || EltSize == VectorTy->getElementType()
+                                         ->getPrimitiveSizeInBits()/8)) {
       if (!VectorTy) {
         ScalarKind = ImplicitVector;
         VectorTy = VectorType::get(In, AllocaSize/EltSize);
-        return;
       }
-
-      unsigned CurrentEltSize = VectorTy->getElementType()
-                                ->getPrimitiveSizeInBits()/8;
-      if (EltSize == CurrentEltSize)
-        return;
-
-      if (In->isIntegerTy() && isPowerOf2_32(AllocaSize / EltSize))
-        return;
+      return;
     }
   }
 
@@ -395,74 +385,21 @@ void ConvertToScalarInfo::MergeInTypeForLoadOrStore(const Type *In,
 
 /// MergeInVectorType - Handles the vector case of MergeInTypeForLoadOrStore,
 /// returning true if the type was successfully merged and false otherwise.
-bool ConvertToScalarInfo::MergeInVectorType(const VectorType *VInTy,
+bool ConvertToScalarInfo::MergeInVectorType(VectorType *VInTy,
                                             uint64_t Offset) {
-  // TODO: Support nonzero offsets?
-  if (Offset != 0)
-    return false;
-
-  // Only allow vectors that are a power-of-2 away from the size of the alloca.
-  if (!isPowerOf2_64(AllocaSize / (VInTy->getBitWidth() / 8)))
-    return false;
-
-  // If this the first vector we see, remember the type so that we know the
-  // element size.
-  if (!VectorTy) {
-    ScalarKind = Vector;
-    VectorTy = VInTy;
-    return true;
-  }
-
-  unsigned BitWidth = VectorTy->getBitWidth();
-  unsigned InBitWidth = VInTy->getBitWidth();
-
-  // Vectors of the same size can be converted using a simple bitcast.
-  if (InBitWidth == BitWidth && AllocaSize == (InBitWidth / 8)) {
+  if (VInTy->getBitWidth()/8 == AllocaSize && Offset == 0) {
+    // If we're storing/loading a vector of the right size, allow it as a
+    // vector.  If this the first vector we see, remember the type so that
+    // we know the element size. If this is a subsequent access, ignore it
+    // even if it is a differing type but the same size. Worst case we can
+    // bitcast the resultant vectors.
+    if (!VectorTy)
+      VectorTy = VInTy;
     ScalarKind = Vector;
     return true;
   }
 
-  const Type *ElementTy = VectorTy->getElementType();
-  const Type *InElementTy = VInTy->getElementType();
-
-  // Do not allow mixed integer and floating-point accesses from vectors of
-  // different sizes.
-  if (ElementTy->isFloatingPointTy() != InElementTy->isFloatingPointTy())
-    return false;
-
-  if (ElementTy->isFloatingPointTy()) {
-    // Only allow floating-point vectors of different sizes if they have the
-    // same element type.
-    // TODO: This could be loosened a bit, but would anything benefit?
-    if (ElementTy != InElementTy)
-      return false;
-
-    // There are no arbitrary-precision floating-point types, which limits the
-    // number of legal vector types with larger element types that we can form
-    // to bitcast and extract a subvector.
-    // TODO: We could support some more cases with mixed fp128 and double here.
-    if (!(BitWidth == 64 || BitWidth == 128) ||
-        !(InBitWidth == 64 || InBitWidth == 128))
-      return false;
-  } else {
-    assert(ElementTy->isIntegerTy() && "Vector elements must be either integer "
-                                       "or floating-point.");
-    unsigned BitWidth = ElementTy->getPrimitiveSizeInBits();
-    unsigned InBitWidth = InElementTy->getPrimitiveSizeInBits();
-
-    // Do not allow integer types smaller than a byte or types whose widths are
-    // not a multiple of a byte.
-    if (BitWidth < 8 || InBitWidth < 8 ||
-        BitWidth % 8 != 0 || InBitWidth % 8 != 0)
-      return false;
-  }
-
-  // Pick the largest of the two vector types.
-  ScalarKind = Vector;
-  if (InBitWidth > BitWidth)
-    VectorTy = VInTy;
-
-  return true;
+  return false;
 }
 
 /// CanConvertToScalar - V is a pointer.  If we can convert the pointee and all
@@ -480,7 +417,7 @@ bool ConvertToScalarInfo::CanConvertToScalar(Value *V, uint64_t Offset) {
 
     if (LoadInst *LI = dyn_cast<LoadInst>(User)) {
       // Don't break volatile loads.
-      if (LI->isVolatile())
+      if (!LI->isSimple())
         return false;
       // Don't touch MMX operations.
       if (LI->getType()->isX86_MMXTy())
@@ -492,7 +429,7 @@ bool ConvertToScalarInfo::CanConvertToScalar(Value *V, uint64_t Offset) {
 
     if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
       // Storing the pointer, not into the value?
-      if (SI->getOperand(0) == V || SI->isVolatile()) return false;
+      if (SI->getOperand(0) == V || !SI->isSimple()) return false;
       // Don't touch MMX operations.
       if (SI->getOperand(0)->getType()->isX86_MMXTy())
         return false;
@@ -502,7 +439,8 @@ bool ConvertToScalarInfo::CanConvertToScalar(Value *V, uint64_t Offset) {
     }
 
     if (BitCastInst *BCI = dyn_cast<BitCastInst>(User)) {
-      IsNotTrivial = true;  // Can't be mem2reg'd.
+      if (!onlyUsedByLifetimeMarkers(BCI))
+        IsNotTrivial = true;  // Can't be mem2reg'd.
       if (!CanConvertToScalar(BCI, Offset))
         return false;
       continue;
@@ -516,7 +454,7 @@ bool ConvertToScalarInfo::CanConvertToScalar(Value *V, uint64_t Offset) {
       // Compute the offset that this GEP adds to the pointer.
       SmallVector<Value*, 8> Indices(GEP->op_begin()+1, GEP->op_end());
       uint64_t GEPOffset = TD.getIndexedOffset(GEP->getPointerOperandType(),
-                                               &Indices[0], Indices.size());
+                                               Indices);
       // See if all uses can be converted.
       if (!CanConvertToScalar(GEP, Offset+GEPOffset))
         return false;
@@ -560,6 +498,14 @@ bool ConvertToScalarInfo::CanConvertToScalar(Value *V, uint64_t Offset) {
       continue;
     }
 
+    // If this is a lifetime intrinsic, we can handle it.
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(User)) {
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+          II->getIntrinsicID() == Intrinsic::lifetime_end) {
+        continue;
+      }
+    }
+
     // Otherwise, we cannot handle this!
     return false;
   }
@@ -589,7 +535,7 @@ void ConvertToScalarInfo::ConvertUsesToScalar(Value *Ptr, AllocaInst *NewAI,
       // Compute the offset that this GEP adds to the pointer.
       SmallVector<Value*, 8> Indices(GEP->op_begin()+1, GEP->op_end());
       uint64_t GEPOffset = TD.getIndexedOffset(GEP->getPointerOperandType(),
-                                               &Indices[0], Indices.size());
+                                               Indices);
       ConvertUsesToScalar(GEP, NewAI, Offset+GEPOffset*8);
       GEP->eraseFromParent();
       continue;
@@ -599,7 +545,7 @@ void ConvertToScalarInfo::ConvertUsesToScalar(Value *Ptr, AllocaInst *NewAI,
 
     if (LoadInst *LI = dyn_cast<LoadInst>(User)) {
       // The load is a bit extract from NewAI shifted right by Offset bits.
-      Value *LoadedVal = Builder.CreateLoad(NewAI, "tmp");
+      Value *LoadedVal = Builder.CreateLoad(NewAI);
       Value *NewLoadVal
         = ConvertScalar_ExtractValue(LoadedVal, LI->getType(), Offset, Builder);
       LI->replaceAllUsesWith(NewLoadVal);
@@ -668,8 +614,8 @@ void ConvertToScalarInfo::ConvertUsesToScalar(Value *Ptr, AllocaInst *NewAI,
         // pointer (bitcasted), then a store to our new alloca.
         assert(MTI->getRawDest() == Ptr && "Neither use is of pointer?");
         Value *SrcPtr = MTI->getSource();
-        const PointerType* SPTy = cast<PointerType>(SrcPtr->getType());
-        const PointerType* AIPTy = cast<PointerType>(NewAI->getType());
+        PointerType* SPTy = cast<PointerType>(SrcPtr->getType());
+        PointerType* AIPTy = cast<PointerType>(NewAI->getType());
         if (SPTy->getAddressSpace() != AIPTy->getAddressSpace()) {
           AIPTy = PointerType::get(AIPTy->getElementType(),
                                    SPTy->getAddressSpace());
@@ -685,8 +631,8 @@ void ConvertToScalarInfo::ConvertUsesToScalar(Value *Ptr, AllocaInst *NewAI,
         assert(MTI->getRawSource() == Ptr && "Neither use is of pointer?");
         LoadInst *SrcVal = Builder.CreateLoad(NewAI, "srcval");
 
-        const PointerType* DPTy = cast<PointerType>(MTI->getDest()->getType());
-        const PointerType* AIPTy = cast<PointerType>(NewAI->getType());
+        PointerType* DPTy = cast<PointerType>(MTI->getDest()->getType());
+        PointerType* AIPTy = cast<PointerType>(NewAI->getType());
         if (DPTy->getAddressSpace() != AIPTy->getAddressSpace()) {
           AIPTy = PointerType::get(AIPTy->getElementType(),
                                    DPTy->getAddressSpace());
@@ -703,67 +649,20 @@ void ConvertToScalarInfo::ConvertUsesToScalar(Value *Ptr, AllocaInst *NewAI,
       continue;
     }
 
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(User)) {
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+          II->getIntrinsicID() == Intrinsic::lifetime_end) {
+        // There's no need to preserve these, as the resulting alloca will be
+        // converted to a register anyways.
+        II->eraseFromParent();
+        continue;
+      }
+    }
+
     llvm_unreachable("Unsupported operation!");
   }
 }
 
-/// getScaledElementType - Gets a scaled element type for a partial vector
-/// access of an alloca. The input types must be integer or floating-point
-/// scalar or vector types, and the resulting type is an integer, float or
-/// double.
-static const Type *getScaledElementType(const Type *Ty1, const Type *Ty2,
-                                        unsigned NewBitWidth) {
-  bool IsFP1 = Ty1->isFloatingPointTy() ||
-               (Ty1->isVectorTy() &&
-                cast<VectorType>(Ty1)->getElementType()->isFloatingPointTy());
-  bool IsFP2 = Ty2->isFloatingPointTy() ||
-               (Ty2->isVectorTy() &&
-                cast<VectorType>(Ty2)->getElementType()->isFloatingPointTy());
-
-  LLVMContext &Context = Ty1->getContext();
-
-  // Prefer floating-point types over integer types, as integer types may have
-  // been created by earlier scalar replacement.
-  if (IsFP1 || IsFP2) {
-    if (NewBitWidth == 32)
-      return Type::getFloatTy(Context);
-    if (NewBitWidth == 64)
-      return Type::getDoubleTy(Context);
-  }
-
-  return Type::getIntNTy(Context, NewBitWidth);
-}
-
-/// CreateShuffleVectorCast - Creates a shuffle vector to convert one vector
-/// to another vector of the same element type which has the same allocation
-/// size but different primitive sizes (e.g. <3 x i32> and <4 x i32>).
-static Value *CreateShuffleVectorCast(Value *FromVal, const Type *ToType,
-                                      IRBuilder<> &Builder) {
-  const Type *FromType = FromVal->getType();
-  const VectorType *FromVTy = cast<VectorType>(FromType);
-  const VectorType *ToVTy = cast<VectorType>(ToType);
-  assert((ToVTy->getElementType() == FromVTy->getElementType()) &&
-         "Vectors must have the same element type");
-   Value *UnV = UndefValue::get(FromType);
-   unsigned numEltsFrom = FromVTy->getNumElements();
-   unsigned numEltsTo = ToVTy->getNumElements();
-
-   SmallVector<Constant*, 3> Args;
-   const Type* Int32Ty = Builder.getInt32Ty();
-   unsigned minNumElts = std::min(numEltsFrom, numEltsTo);
-   unsigned i;
-   for (i=0; i != minNumElts; ++i)
-     Args.push_back(ConstantInt::get(Int32Ty, i));
-
-   if (i < numEltsTo) {
-     Constant* UnC = UndefValue::get(Int32Ty);
-     for (; i != numEltsTo; ++i)
-       Args.push_back(UnC);
-   }
-   Constant *Mask = ConstantVector::get(Args);
-   return Builder.CreateShuffleVector(FromVal, UnV, Mask, "tmpV");
-}
-
 /// ConvertScalar_ExtractValue - Extract a value of type ToType from an integer
 /// or vector value FromVal, extracting the bits from the offset specified by
 /// Offset.  This returns the value, which is of type ToType.
@@ -775,50 +674,20 @@ static Value *CreateShuffleVectorCast(Value *FromVal, const Type *ToType,
 /// Offset is an offset from the original alloca, in bits that need to be
 /// shifted to the right.
 Value *ConvertToScalarInfo::
-ConvertScalar_ExtractValue(Value *FromVal, const Type *ToType,
+ConvertScalar_ExtractValue(Value *FromVal, Type *ToType,
                            uint64_t Offset, IRBuilder<> &Builder) {
   // If the load is of the whole new alloca, no conversion is needed.
-  const Type *FromType = FromVal->getType();
+  Type *FromType = FromVal->getType();
   if (FromType == ToType && Offset == 0)
     return FromVal;
 
   // If the result alloca is a vector type, this is either an element
   // access or a bitcast to another vector type of the same size.
-  if (const VectorType *VTy = dyn_cast<VectorType>(FromType)) {
+  if (VectorType *VTy = dyn_cast<VectorType>(FromType)) {
     unsigned FromTypeSize = TD.getTypeAllocSize(FromType);
     unsigned ToTypeSize = TD.getTypeAllocSize(ToType);
-    if (FromTypeSize == ToTypeSize) {
-      // If the two types have the same primitive size, use a bit cast.
-      // Otherwise, it is two vectors with the same element type that has
-      // the same allocation size but different number of elements so use
-      // a shuffle vector.
-      if (FromType->getPrimitiveSizeInBits() ==
-          ToType->getPrimitiveSizeInBits())
-        return Builder.CreateBitCast(FromVal, ToType, "tmp");
-      else
-        return CreateShuffleVectorCast(FromVal, ToType, Builder);
-    }
-
-    if (isPowerOf2_64(FromTypeSize / ToTypeSize)) {
-      assert(!(ToType->isVectorTy() && Offset != 0) && "Can't extract a value "
-             "of a smaller vector type at a nonzero offset.");
-
-      const Type *CastElementTy = getScaledElementType(FromType, ToType,
-                                                       ToTypeSize * 8);
-      unsigned NumCastVectorElements = FromTypeSize / ToTypeSize;
-
-      LLVMContext &Context = FromVal->getContext();
-      const Type *CastTy = VectorType::get(CastElementTy,
-                                           NumCastVectorElements);
-      Value *Cast = Builder.CreateBitCast(FromVal, CastTy, "tmp");
-
-      unsigned EltSize = TD.getTypeAllocSizeInBits(CastElementTy);
-      unsigned Elt = Offset/EltSize;
-      assert(EltSize*Elt == Offset && "Invalid modulus in validity checking");
-      Value *Extract = Builder.CreateExtractElement(Cast, ConstantInt::get(
-                                        Type::getInt32Ty(Context), Elt), "tmp");
-      return Builder.CreateBitCast(Extract, ToType, "tmp");
-    }
+    if (FromTypeSize == ToTypeSize)
+        return Builder.CreateBitCast(FromVal, ToType);
 
     // Otherwise it must be an element access.
     unsigned Elt = 0;
@@ -828,40 +697,39 @@ ConvertScalar_ExtractValue(Value *FromVal, const Type *ToType,
       assert(EltSize*Elt == Offset && "Invalid modulus in validity checking");
     }
     // Return the element extracted out of it.
-    Value *V = Builder.CreateExtractElement(FromVal, ConstantInt::get(
-                    Type::getInt32Ty(FromVal->getContext()), Elt), "tmp");
+    Value *V = Builder.CreateExtractElement(FromVal, Builder.getInt32(Elt));
     if (V->getType() != ToType)
-      V = Builder.CreateBitCast(V, ToType, "tmp");
+      V = Builder.CreateBitCast(V, ToType);
     return V;
   }
 
   // If ToType is a first class aggregate, extract out each of the pieces and
   // use insertvalue's to form the FCA.
-  if (const StructType *ST = dyn_cast<StructType>(ToType)) {
+  if (StructType *ST = dyn_cast<StructType>(ToType)) {
     const StructLayout &Layout = *TD.getStructLayout(ST);
     Value *Res = UndefValue::get(ST);
     for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i) {
       Value *Elt = ConvertScalar_ExtractValue(FromVal, ST->getElementType(i),
                                         Offset+Layout.getElementOffsetInBits(i),
                                               Builder);
-      Res = Builder.CreateInsertValue(Res, Elt, i, "tmp");
+      Res = Builder.CreateInsertValue(Res, Elt, i);
     }
     return Res;
   }
 
-  if (const ArrayType *AT = dyn_cast<ArrayType>(ToType)) {
+  if (ArrayType *AT = dyn_cast<ArrayType>(ToType)) {
     uint64_t EltSize = TD.getTypeAllocSizeInBits(AT->getElementType());
     Value *Res = UndefValue::get(AT);
     for (unsigned i = 0, e = AT->getNumElements(); i != e; ++i) {
       Value *Elt = ConvertScalar_ExtractValue(FromVal, AT->getElementType(),
                                               Offset+i*EltSize, Builder);
-      Res = Builder.CreateInsertValue(Res, Elt, i, "tmp");
+      Res = Builder.CreateInsertValue(Res, Elt, i);
     }
     return Res;
   }
 
   // Otherwise, this must be a union that was converted to an integer value.
-  const IntegerType *NTy = cast<IntegerType>(FromVal->getType());
+  IntegerType *NTy = cast<IntegerType>(FromVal->getType());
 
   // If this is a big-endian system and the load is narrower than the
   // full alloca type, we need to do a shift to get the right bits.
@@ -881,33 +749,31 @@ ConvertScalar_ExtractValue(Value *FromVal, const Type *ToType,
   // only some bits are used.
   if (ShAmt > 0 && (unsigned)ShAmt < NTy->getBitWidth())
     FromVal = Builder.CreateLShr(FromVal,
-                                 ConstantInt::get(FromVal->getType(),
-                                                           ShAmt), "tmp");
+                                 ConstantInt::get(FromVal->getType(), ShAmt));
   else if (ShAmt < 0 && (unsigned)-ShAmt < NTy->getBitWidth())
     FromVal = Builder.CreateShl(FromVal,
-                                ConstantInt::get(FromVal->getType(),
-                                                          -ShAmt), "tmp");
+                                ConstantInt::get(FromVal->getType(), -ShAmt));
 
   // Finally, unconditionally truncate the integer to the right width.
   unsigned LIBitWidth = TD.getTypeSizeInBits(ToType);
   if (LIBitWidth < NTy->getBitWidth())
     FromVal =
       Builder.CreateTrunc(FromVal, IntegerType::get(FromVal->getContext(),
-                                                    LIBitWidth), "tmp");
+                                                    LIBitWidth));
   else if (LIBitWidth > NTy->getBitWidth())
     FromVal =
        Builder.CreateZExt(FromVal, IntegerType::get(FromVal->getContext(),
-                                                    LIBitWidth), "tmp");
+                                                    LIBitWidth));
 
   // If the result is an integer, this is a trunc or bitcast.
   if (ToType->isIntegerTy()) {
     // Should be done.
   } else if (ToType->isFloatingPointTy() || ToType->isVectorTy()) {
     // Just do a bitcast, we know the sizes match up.
-    FromVal = Builder.CreateBitCast(FromVal, ToType, "tmp");
+    FromVal = Builder.CreateBitCast(FromVal, ToType);
   } else {
     // Otherwise must be a pointer.
-    FromVal = Builder.CreateIntToPtr(FromVal, ToType, "tmp");
+    FromVal = Builder.CreateIntToPtr(FromVal, ToType);
   }
   assert(FromVal->getType() == ToType && "Didn't convert right?");
   return FromVal;
@@ -927,65 +793,30 @@ ConvertScalar_InsertValue(Value *SV, Value *Old,
                           uint64_t Offset, IRBuilder<> &Builder) {
   // Convert the stored type to the actual type, shift it left to insert
   // then 'or' into place.
-  const Type *AllocaType = Old->getType();
+  Type *AllocaType = Old->getType();
   LLVMContext &Context = Old->getContext();
 
-  if (const VectorType *VTy = dyn_cast<VectorType>(AllocaType)) {
+  if (VectorType *VTy = dyn_cast<VectorType>(AllocaType)) {
     uint64_t VecSize = TD.getTypeAllocSizeInBits(VTy);
     uint64_t ValSize = TD.getTypeAllocSizeInBits(SV->getType());
 
     // Changing the whole vector with memset or with an access of a different
     // vector type?
-    if (ValSize == VecSize) {
-      // If the two types have the same primitive size, use a bit cast.
-      // Otherwise, it is two vectors with the same element type that has
-      // the same allocation size but different number of elements so use
-      // a shuffle vector.
-      if (VTy->getPrimitiveSizeInBits() ==
-          SV->getType()->getPrimitiveSizeInBits())
-        return Builder.CreateBitCast(SV, AllocaType, "tmp");
-      else
-        return CreateShuffleVectorCast(SV, VTy, Builder);
-    }
-
-    if (isPowerOf2_64(VecSize / ValSize)) {
-      assert(!(SV->getType()->isVectorTy() && Offset != 0) && "Can't insert a "
-             "value of a smaller vector type at a nonzero offset.");
-
-      const Type *CastElementTy = getScaledElementType(VTy, SV->getType(),
-                                                       ValSize);
-      unsigned NumCastVectorElements = VecSize / ValSize;
-
-      LLVMContext &Context = SV->getContext();
-      const Type *OldCastTy = VectorType::get(CastElementTy,
-                                              NumCastVectorElements);
-      Value *OldCast = Builder.CreateBitCast(Old, OldCastTy, "tmp");
-
-      Value *SVCast = Builder.CreateBitCast(SV, CastElementTy, "tmp");
-
-      unsigned EltSize = TD.getTypeAllocSizeInBits(CastElementTy);
-      unsigned Elt = Offset/EltSize;
-      assert(EltSize*Elt == Offset && "Invalid modulus in validity checking");
-      Value *Insert =
-        Builder.CreateInsertElement(OldCast, SVCast, ConstantInt::get(
-                                        Type::getInt32Ty(Context), Elt), "tmp");
-      return Builder.CreateBitCast(Insert, AllocaType, "tmp");
-    }
+    if (ValSize == VecSize)
+        return Builder.CreateBitCast(SV, AllocaType);
 
     // Must be an element insertion.
     assert(SV->getType() == VTy->getElementType());
     uint64_t EltSize = TD.getTypeAllocSizeInBits(VTy->getElementType());
     unsigned Elt = Offset/EltSize;
-    return Builder.CreateInsertElement(Old, SV,
-                     ConstantInt::get(Type::getInt32Ty(SV->getContext()), Elt),
-                                     "tmp");
+    return Builder.CreateInsertElement(Old, SV, Builder.getInt32(Elt));
   }
 
   // If SV is a first-class aggregate value, insert each value recursively.
-  if (const StructType *ST = dyn_cast<StructType>(SV->getType())) {
+  if (StructType *ST = dyn_cast<StructType>(SV->getType())) {
     const StructLayout &Layout = *TD.getStructLayout(ST);
     for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i) {
-      Value *Elt = Builder.CreateExtractValue(SV, i, "tmp");
+      Value *Elt = Builder.CreateExtractValue(SV, i);
       Old = ConvertScalar_InsertValue(Elt, Old,
                                       Offset+Layout.getElementOffsetInBits(i),
                                       Builder);
@@ -993,10 +824,10 @@ ConvertScalar_InsertValue(Value *SV, Value *Old,
     return Old;
   }
 
-  if (const ArrayType *AT = dyn_cast<ArrayType>(SV->getType())) {
+  if (ArrayType *AT = dyn_cast<ArrayType>(SV->getType())) {
     uint64_t EltSize = TD.getTypeAllocSizeInBits(AT->getElementType());
     for (unsigned i = 0, e = AT->getNumElements(); i != e; ++i) {
-      Value *Elt = Builder.CreateExtractValue(SV, i, "tmp");
+      Value *Elt = Builder.CreateExtractValue(SV, i);
       Old = ConvertScalar_InsertValue(Elt, Old, Offset+i*EltSize, Builder);
     }
     return Old;
@@ -1009,20 +840,19 @@ ConvertScalar_InsertValue(Value *SV, Value *Old,
   unsigned SrcStoreWidth = TD.getTypeStoreSizeInBits(SV->getType());
   unsigned DestStoreWidth = TD.getTypeStoreSizeInBits(AllocaType);
   if (SV->getType()->isFloatingPointTy() || SV->getType()->isVectorTy())
-    SV = Builder.CreateBitCast(SV,
-                            IntegerType::get(SV->getContext(),SrcWidth), "tmp");
+    SV = Builder.CreateBitCast(SV, IntegerType::get(SV->getContext(),SrcWidth));
   else if (SV->getType()->isPointerTy())
-    SV = Builder.CreatePtrToInt(SV, TD.getIntPtrType(SV->getContext()), "tmp");
+    SV = Builder.CreatePtrToInt(SV, TD.getIntPtrType(SV->getContext()));
 
   // Zero extend or truncate the value if needed.
   if (SV->getType() != AllocaType) {
     if (SV->getType()->getPrimitiveSizeInBits() <
              AllocaType->getPrimitiveSizeInBits())
-      SV = Builder.CreateZExt(SV, AllocaType, "tmp");
+      SV = Builder.CreateZExt(SV, AllocaType);
     else {
       // Truncation may be needed if storing more than the alloca can hold
       // (undefined behavior).
-      SV = Builder.CreateTrunc(SV, AllocaType, "tmp");
+      SV = Builder.CreateTrunc(SV, AllocaType);
       SrcWidth = DestWidth;
       SrcStoreWidth = DestStoreWidth;
     }
@@ -1045,12 +875,10 @@ ConvertScalar_InsertValue(Value *SV, Value *Old,
   // only some bits in the structure are set.
   APInt Mask(APInt::getLowBitsSet(DestWidth, SrcWidth));
   if (ShAmt > 0 && (unsigned)ShAmt < DestWidth) {
-    SV = Builder.CreateShl(SV, ConstantInt::get(SV->getType(),
-                           ShAmt), "tmp");
+    SV = Builder.CreateShl(SV, ConstantInt::get(SV->getType(), ShAmt));
     Mask <<= ShAmt;
   } else if (ShAmt < 0 && (unsigned)-ShAmt < DestWidth) {
-    SV = Builder.CreateLShr(SV, ConstantInt::get(SV->getType(),
-                            -ShAmt), "tmp");
+    SV = Builder.CreateLShr(SV, ConstantInt::get(SV->getType(), -ShAmt));
     Mask = Mask.lshr(-ShAmt);
   }
 
@@ -1196,7 +1024,7 @@ static bool isSafeSelectToSpeculate(SelectInst *SI, const TargetData *TD) {
   for (Value::use_iterator UI = SI->use_begin(), UE = SI->use_end();
        UI != UE; ++UI) {
     LoadInst *LI = dyn_cast<LoadInst>(*UI);
-    if (LI == 0 || LI->isVolatile()) return false;
+    if (LI == 0 || !LI->isSimple()) return false;
     
     // Both operands to the select need to be dereferencable, either absolutely
     // (e.g. allocas) or at this point because we can see other accesses to it.
@@ -1237,7 +1065,7 @@ static bool isSafePHIToSpeculate(PHINode *PN, const TargetData *TD) {
   for (Value::use_iterator UI = PN->use_begin(), UE = PN->use_end();
        UI != UE; ++UI) {
     LoadInst *LI = dyn_cast<LoadInst>(*UI);
-    if (LI == 0 || LI->isVolatile()) return false;
+    if (LI == 0 || !LI->isSimple()) return false;
     
     // For now we only allow loads in the same block as the PHI.  This is a
     // common case that happens when instcombine merges two loads through a PHI.
@@ -1258,17 +1086,21 @@ static bool isSafePHIToSpeculate(PHINode *PN, const TargetData *TD) {
   // trapping load in the predecessor if it is a critical edge.
   for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
     BasicBlock *Pred = PN->getIncomingBlock(i);
+    Value *InVal = PN->getIncomingValue(i);
+
+    // If the terminator of the predecessor has side-effects (an invoke),
+    // there is no safe place to put a load in the predecessor.
+    if (Pred->getTerminator()->mayHaveSideEffects())
+      return false;
+
+    // If the value is produced by the terminator of the predecessor
+    // (an invoke), there is no valid place to put a load in the predecessor.
+    if (Pred->getTerminator() == InVal)
+      return false;
 
     // If the predecessor has a single successor, then the edge isn't critical.
     if (Pred->getTerminator()->getNumSuccessors() == 1)
       continue;
-    
-    Value *InVal = PN->getIncomingValue(i);
-    
-    // If the InVal is an invoke in the pred, we can't put a load on the edge.
-    if (InvokeInst *II = dyn_cast<InvokeInst>(InVal))
-      if (II->getParent() == Pred)
-        return false;
 
     // If this pointer is always safe to load, or if we can prove that there is
     // already a load in the block, then we can move the load to the pred block.
@@ -1295,13 +1127,13 @@ static bool tryToMakeAllocaBePromotable(AllocaInst *AI, const TargetData *TD) {
        UI != UE; ++UI) {
     User *U = *UI;
     if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
-      if (LI->isVolatile())
+      if (!LI->isSimple())
         return false;
       continue;
     }
     
     if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
-      if (SI->getOperand(0) == AI || SI->isVolatile())
+      if (SI->getOperand(0) == AI || !SI->isSimple())
         return false;   // Don't allow a store OF the AI, only INTO the AI.
       continue;
     }
@@ -1343,6 +1175,13 @@ static bool tryToMakeAllocaBePromotable(AllocaInst *AI, const TargetData *TD) {
       continue;
     }
     
+    if (BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
+      if (onlyUsedByLifetimeMarkers(BCI)) {
+        InstsToRewrite.insert(BCI);
+        continue;
+      }
+    }
+    
     return false;
   }
 
@@ -1354,6 +1193,18 @@ static bool tryToMakeAllocaBePromotable(AllocaInst *AI, const TargetData *TD) {
   // If we have instructions that need to be rewritten for this to be promotable
   // take care of it now.
   for (unsigned i = 0, e = InstsToRewrite.size(); i != e; ++i) {
+    if (BitCastInst *BCI = dyn_cast<BitCastInst>(InstsToRewrite[i])) {
+      // This could only be a bitcast used by nothing but lifetime intrinsics.
+      for (BitCastInst::use_iterator I = BCI->use_begin(), E = BCI->use_end();
+           I != E;) {
+        Use &U = I.getUse();
+        ++I;
+        cast<Instruction>(U.getUser())->eraseFromParent();
+      }
+      BCI->eraseFromParent();
+      continue;
+    }
+
     if (SelectInst *SI = dyn_cast<SelectInst>(InstsToRewrite[i])) {
       // Selects in InstsToRewrite only have load uses.  Rewrite each as two
       // loads with a new select.
@@ -1393,7 +1244,7 @@ static bool tryToMakeAllocaBePromotable(AllocaInst *AI, const TargetData *TD) {
       continue;
     }
     
-    const Type *LoadTy = cast<PointerType>(PN->getType())->getElementType();
+    Type *LoadTy = cast<PointerType>(PN->getType())->getElementType();
     PHINode *NewPN = PHINode::Create(LoadTy, PN->getNumIncomingValues(),
                                      PN->getName()+".ld", PN);
 
@@ -1483,13 +1334,13 @@ bool SROA::performPromotion(Function &F) {
 /// ShouldAttemptScalarRepl - Decide if an alloca is a good candidate for
 /// SROA.  It must be a struct or array type with a small number of elements.
 static bool ShouldAttemptScalarRepl(AllocaInst *AI) {
-  const Type *T = AI->getAllocatedType();
+  Type *T = AI->getAllocatedType();
   // Do not promote any struct into more than 32 separate vars.
-  if (const StructType *ST = dyn_cast<StructType>(T))
+  if (StructType *ST = dyn_cast<StructType>(T))
     return ST->getNumElements() <= 32;
   // Arrays are much less likely to be safe for SROA; only consider
   // them if they are very small.
-  if (const ArrayType *AT = dyn_cast<ArrayType>(T))
+  if (ArrayType *AT = dyn_cast<ArrayType>(T))
     return AT->getNumElements() <= 8;
   return false;
 }
@@ -1594,7 +1445,7 @@ void SROA::DoScalarReplacement(AllocaInst *AI,
                                std::vector<AllocaInst*> &WorkList) {
   DEBUG(dbgs() << "Found inst to SROA: " << *AI << '\n');
   SmallVector<AllocaInst*, 32> ElementAllocas;
-  if (const StructType *ST = dyn_cast<StructType>(AI->getAllocatedType())) {
+  if (StructType *ST = dyn_cast<StructType>(AI->getAllocatedType())) {
     ElementAllocas.reserve(ST->getNumContainedTypes());
     for (unsigned i = 0, e = ST->getNumContainedTypes(); i != e; ++i) {
       AllocaInst *NA = new AllocaInst(ST->getContainedType(i), 0,
@@ -1604,9 +1455,9 @@ void SROA::DoScalarReplacement(AllocaInst *AI,
       WorkList.push_back(NA);  // Add to worklist for recursive processing
     }
   } else {
-    const ArrayType *AT = cast<ArrayType>(AI->getAllocatedType());
+    ArrayType *AT = cast<ArrayType>(AI->getAllocatedType());
     ElementAllocas.reserve(AT->getNumElements());
-    const Type *ElTy = AT->getElementType();
+    Type *ElTy = AT->getElementType();
     for (unsigned i = 0, e = AT->getNumElements(); i != e; ++i) {
       AllocaInst *NA = new AllocaInst(ElTy, 0, AI->getAlignment(),
                                       AI->getName() + "." + Twine(i), AI);
@@ -1670,22 +1521,26 @@ void SROA::isSafeForScalarRepl(Instruction *I, uint64_t Offset,
                       UI.getOperandNo() == 0, Info, MI,
                       true /*AllowWholeAccess*/);
     } else if (LoadInst *LI = dyn_cast<LoadInst>(User)) {
-      if (LI->isVolatile())
+      if (!LI->isSimple())
         return MarkUnsafe(Info, User);
-      const Type *LIType = LI->getType();
+      Type *LIType = LI->getType();
       isSafeMemAccess(Offset, TD->getTypeAllocSize(LIType),
                       LIType, false, Info, LI, true /*AllowWholeAccess*/);
       Info.hasALoadOrStore = true;
         
     } else if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
       // Store is ok if storing INTO the pointer, not storing the pointer
-      if (SI->isVolatile() || SI->getOperand(0) == I)
+      if (!SI->isSimple() || SI->getOperand(0) == I)
         return MarkUnsafe(Info, User);
         
-      const Type *SIType = SI->getOperand(0)->getType();
+      Type *SIType = SI->getOperand(0)->getType();
       isSafeMemAccess(Offset, TD->getTypeAllocSize(SIType),
                       SIType, true, Info, SI, true /*AllowWholeAccess*/);
       Info.hasALoadOrStore = true;
+    } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(User)) {
+      if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
+          II->getIntrinsicID() != Intrinsic::lifetime_end)
+        return MarkUnsafe(Info, User);
     } else if (isa<PHINode>(User) || isa<SelectInst>(User)) {
       isSafePHISelectUseForScalarRepl(User, Offset, Info);
     } else {
@@ -1725,19 +1580,19 @@ void SROA::isSafePHISelectUseForScalarRepl(Instruction *I, uint64_t Offset,
         return MarkUnsafe(Info, User);
       isSafePHISelectUseForScalarRepl(GEPI, Offset, Info);
     } else if (LoadInst *LI = dyn_cast<LoadInst>(User)) {
-      if (LI->isVolatile())
+      if (!LI->isSimple())
         return MarkUnsafe(Info, User);
-      const Type *LIType = LI->getType();
+      Type *LIType = LI->getType();
       isSafeMemAccess(Offset, TD->getTypeAllocSize(LIType),
                       LIType, false, Info, LI, false /*AllowWholeAccess*/);
       Info.hasALoadOrStore = true;
       
     } else if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
       // Store is ok if storing INTO the pointer, not storing the pointer
-      if (SI->isVolatile() || SI->getOperand(0) == I)
+      if (!SI->isSimple() || SI->getOperand(0) == I)
         return MarkUnsafe(Info, User);
       
-      const Type *SIType = SI->getOperand(0)->getType();
+      Type *SIType = SI->getOperand(0)->getType();
       isSafeMemAccess(Offset, TD->getTypeAllocSize(SIType),
                       SIType, true, Info, SI, false /*AllowWholeAccess*/);
       Info.hasALoadOrStore = true;
@@ -1776,8 +1631,7 @@ void SROA::isSafeGEP(GetElementPtrInst *GEPI,
   // Compute the offset due to this GEP and check if the alloca has a
   // component element at that offset.
   SmallVector<Value*, 8> Indices(GEPI->op_begin() + 1, GEPI->op_end());
-  Offset += TD->getIndexedOffset(GEPI->getPointerOperandType(),
-                                 &Indices[0], Indices.size());
+  Offset += TD->getIndexedOffset(GEPI->getPointerOperandType(), Indices);
   if (!TypeHasComponent(Info.AI->getAllocatedType(), Offset, 0))
     MarkUnsafe(Info, GEPI);
 }
@@ -1786,14 +1640,14 @@ void SROA::isSafeGEP(GetElementPtrInst *GEPI,
 /// elements of the same type (which is always true for arrays).  If so,
 /// return true with NumElts and EltTy set to the number of elements and the
 /// element type, respectively.
-static bool isHomogeneousAggregate(const Type *T, unsigned &NumElts,
-                                   const Type *&EltTy) {
-  if (const ArrayType *AT = dyn_cast<ArrayType>(T)) {
+static bool isHomogeneousAggregate(Type *T, unsigned &NumElts,
+                                   Type *&EltTy) {
+  if (ArrayType *AT = dyn_cast<ArrayType>(T)) {
     NumElts = AT->getNumElements();
     EltTy = (NumElts == 0 ? 0 : AT->getElementType());
     return true;
   }
-  if (const StructType *ST = dyn_cast<StructType>(T)) {
+  if (StructType *ST = dyn_cast<StructType>(T)) {
     NumElts = ST->getNumContainedTypes();
     EltTy = (NumElts == 0 ? 0 : ST->getContainedType(0));
     for (unsigned n = 1; n < NumElts; ++n) {
@@ -1807,12 +1661,12 @@ static bool isHomogeneousAggregate(const Type *T, unsigned &NumElts,
 
 /// isCompatibleAggregate - Check if T1 and T2 are either the same type or are
 /// "homogeneous" aggregates with the same element type and number of elements.
-static bool isCompatibleAggregate(const Type *T1, const Type *T2) {
+static bool isCompatibleAggregate(Type *T1, Type *T2) {
   if (T1 == T2)
     return true;
 
   unsigned NumElts1, NumElts2;
-  const Type *EltTy1, *EltTy2;
+  Type *EltTy1, *EltTy2;
   if (isHomogeneousAggregate(T1, NumElts1, EltTy1) &&
       isHomogeneousAggregate(T2, NumElts2, EltTy2) &&
       NumElts1 == NumElts2 &&
@@ -1830,7 +1684,7 @@ static bool isCompatibleAggregate(const Type *T1, const Type *T2) {
 /// If AllowWholeAccess is true, then this allows uses of the entire alloca as a
 /// unit.  If false, it only allows accesses known to be in a single element.
 void SROA::isSafeMemAccess(uint64_t Offset, uint64_t MemSize,
-                           const Type *MemOpType, bool isStore,
+                           Type *MemOpType, bool isStore,
                            AllocaInfo &Info, Instruction *TheAccess,
                            bool AllowWholeAccess) {
   // Check if this is a load/store of the entire alloca.
@@ -1857,7 +1711,7 @@ void SROA::isSafeMemAccess(uint64_t Offset, uint64_t MemSize,
     }
   }
   // Check if the offset/size correspond to a component within the alloca type.
-  const Type *T = Info.AI->getAllocatedType();
+  Type *T = Info.AI->getAllocatedType();
   if (TypeHasComponent(T, Offset, MemSize)) {
     Info.hasSubelementAccess = true;
     return;
@@ -1868,16 +1722,16 @@ void SROA::isSafeMemAccess(uint64_t Offset, uint64_t MemSize,
 
 /// TypeHasComponent - Return true if T has a component type with the
 /// specified offset and size.  If Size is zero, do not check the size.
-bool SROA::TypeHasComponent(const Type *T, uint64_t Offset, uint64_t Size) {
-  const Type *EltTy;
+bool SROA::TypeHasComponent(Type *T, uint64_t Offset, uint64_t Size) {
+  Type *EltTy;
   uint64_t EltSize;
-  if (const StructType *ST = dyn_cast<StructType>(T)) {
+  if (StructType *ST = dyn_cast<StructType>(T)) {
     const StructLayout *Layout = TD->getStructLayout(ST);
     unsigned EltIdx = Layout->getElementContainingOffset(Offset);
     EltTy = ST->getContainedType(EltIdx);
     EltSize = TD->getTypeAllocSize(EltTy);
     Offset -= Layout->getElementOffset(EltIdx);
-  } else if (const ArrayType *AT = dyn_cast<ArrayType>(T)) {
+  } else if (ArrayType *AT = dyn_cast<ArrayType>(T)) {
     EltTy = AT->getElementType();
     EltSize = TD->getTypeAllocSize(EltTy);
     if (Offset >= AT->getNumElements() * EltSize)
@@ -1924,9 +1778,17 @@ void SROA::RewriteForScalarRepl(Instruction *I, AllocaInst *AI, uint64_t Offset,
       // address operand will be updated, so nothing else needs to be done.
       continue;
     }
+
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(User)) {
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+          II->getIntrinsicID() == Intrinsic::lifetime_end) {
+        RewriteLifetimeIntrinsic(II, AI, Offset, NewElts);
+      }
+      continue;
+    }
     
     if (LoadInst *LI = dyn_cast<LoadInst>(User)) {
-      const Type *LIType = LI->getType();
+      Type *LIType = LI->getType();
       
       if (isCompatibleAggregate(LIType, AI->getAllocatedType())) {
         // Replace:
@@ -1956,7 +1818,7 @@ void SROA::RewriteForScalarRepl(Instruction *I, AllocaInst *AI, uint64_t Offset,
     
     if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
       Value *Val = SI->getOperand(0);
-      const Type *SIType = Val->getType();
+      Type *SIType = Val->getType();
       if (isCompatibleAggregate(SIType, AI->getAllocatedType())) {
         // Replace:
         //   store { i32, i32 } %val, { i32, i32 }* %alloc
@@ -2026,10 +1888,10 @@ void SROA::RewriteBitCast(BitCastInst *BC, AllocaInst *AI, uint64_t Offset,
 /// Sets T to the type of the element and Offset to the offset within that
 /// element.  IdxTy is set to the type of the index result to be used in a
 /// GEP instruction.
-uint64_t SROA::FindElementAndOffset(const Type *&T, uint64_t &Offset,
-                                    const Type *&IdxTy) {
+uint64_t SROA::FindElementAndOffset(Type *&T, uint64_t &Offset,
+                                    Type *&IdxTy) {
   uint64_t Idx = 0;
-  if (const StructType *ST = dyn_cast<StructType>(T)) {
+  if (StructType *ST = dyn_cast<StructType>(T)) {
     const StructLayout *Layout = TD->getStructLayout(ST);
     Idx = Layout->getElementContainingOffset(Offset);
     T = ST->getContainedType(Idx);
@@ -2037,7 +1899,7 @@ uint64_t SROA::FindElementAndOffset(const Type *&T, uint64_t &Offset,
     IdxTy = Type::getInt32Ty(T->getContext());
     return Idx;
   }
-  const ArrayType *AT = cast<ArrayType>(T);
+  ArrayType *AT = cast<ArrayType>(T);
   T = AT->getElementType();
   uint64_t EltSize = TD->getTypeAllocSize(T);
   Idx = Offset / EltSize;
@@ -2053,13 +1915,12 @@ void SROA::RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
                       SmallVector<AllocaInst*, 32> &NewElts) {
   uint64_t OldOffset = Offset;
   SmallVector<Value*, 8> Indices(GEPI->op_begin() + 1, GEPI->op_end());
-  Offset += TD->getIndexedOffset(GEPI->getPointerOperandType(),
-                                 &Indices[0], Indices.size());
+  Offset += TD->getIndexedOffset(GEPI->getPointerOperandType(), Indices);
 
   RewriteForScalarRepl(GEPI, AI, Offset, NewElts);
 
-  const Type *T = AI->getAllocatedType();
-  const Type *IdxTy;
+  Type *T = AI->getAllocatedType();
+  Type *IdxTy;
   uint64_t OldIdx = FindElementAndOffset(T, OldOffset, IdxTy);
   if (GEPI->getOperand(0) == AI)
     OldIdx = ~0ULL; // Force the GEP to be rewritten.
@@ -2073,7 +1934,7 @@ void SROA::RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
   if (Idx == OldIdx)
     return;
 
-  const Type *i32Ty = Type::getInt32Ty(AI->getContext());
+  Type *i32Ty = Type::getInt32Ty(AI->getContext());
   SmallVector<Value*, 8> NewArgs;
   NewArgs.push_back(Constant::getNullValue(i32Ty));
   while (EltOffset != 0) {
@@ -2082,8 +1943,7 @@ void SROA::RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
   }
   Instruction *Val = NewElts[Idx];
   if (NewArgs.size() > 1) {
-    Val = GetElementPtrInst::CreateInBounds(Val, NewArgs.begin(),
-                                            NewArgs.end(), "", GEPI);
+    Val = GetElementPtrInst::CreateInBounds(Val, NewArgs, "", GEPI);
     Val->takeName(GEPI);
   }
   if (Val->getType() != GEPI->getType())
@@ -2092,6 +1952,62 @@ void SROA::RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
   DeadInsts.push_back(GEPI);
 }
 
+/// RewriteLifetimeIntrinsic - II is a lifetime.start/lifetime.end. Rewrite it
+/// to mark the lifetime of the scalarized memory.
+void SROA::RewriteLifetimeIntrinsic(IntrinsicInst *II, AllocaInst *AI,
+                                    uint64_t Offset,
+                                    SmallVector<AllocaInst*, 32> &NewElts) {
+  ConstantInt *OldSize = cast<ConstantInt>(II->getArgOperand(0));
+  // Put matching lifetime markers on everything from Offset up to
+  // Offset+OldSize.
+  Type *AIType = AI->getAllocatedType();
+  uint64_t NewOffset = Offset;
+  Type *IdxTy;
+  uint64_t Idx = FindElementAndOffset(AIType, NewOffset, IdxTy);
+
+  IRBuilder<> Builder(II);
+  uint64_t Size = OldSize->getLimitedValue();
+
+  if (NewOffset) {
+    // Splice the first element and index 'NewOffset' bytes in.  SROA will
+    // split the alloca again later.
+    Value *V = Builder.CreateBitCast(NewElts[Idx], Builder.getInt8PtrTy());
+    V = Builder.CreateGEP(V, Builder.getInt64(NewOffset));
+
+    IdxTy = NewElts[Idx]->getAllocatedType();
+    uint64_t EltSize = TD->getTypeAllocSize(IdxTy) - NewOffset;
+    if (EltSize > Size) {
+      EltSize = Size;
+      Size = 0;
+    } else {
+      Size -= EltSize;
+    }
+    if (II->getIntrinsicID() == Intrinsic::lifetime_start)
+      Builder.CreateLifetimeStart(V, Builder.getInt64(EltSize));
+    else
+      Builder.CreateLifetimeEnd(V, Builder.getInt64(EltSize));
+    ++Idx;
+  }
+
+  for (; Idx != NewElts.size() && Size; ++Idx) {
+    IdxTy = NewElts[Idx]->getAllocatedType();
+    uint64_t EltSize = TD->getTypeAllocSize(IdxTy);
+    if (EltSize > Size) {
+      EltSize = Size;
+      Size = 0;
+    } else {
+      Size -= EltSize;
+    }
+    if (II->getIntrinsicID() == Intrinsic::lifetime_start)
+      Builder.CreateLifetimeStart(NewElts[Idx],
+                                  Builder.getInt64(EltSize));
+    else
+      Builder.CreateLifetimeEnd(NewElts[Idx],
+                                Builder.getInt64(EltSize));
+  }
+  DeadInsts.push_back(II);
+}
+
 /// RewriteMemIntrinUserOfAlloca - MI is a memcpy/memset/memmove from or to AI.
 /// Rewrite it to copy or set the elements of the scalarized memory.
 void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
@@ -2139,7 +2055,7 @@ void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
 
     // If the pointer is not the right type, insert a bitcast to the right
     // type.
-    const Type *NewTy =
+    Type *NewTy =
       PointerType::get(AI->getType()->getElementType(), AddrSpace);
 
     if (OtherPtr->getType() != NewTy)
@@ -2159,16 +2075,16 @@ void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
     if (OtherPtr) {
       Value *Idx[2] = { Zero,
                       ConstantInt::get(Type::getInt32Ty(MI->getContext()), i) };
-      OtherElt = GetElementPtrInst::CreateInBounds(OtherPtr, Idx, Idx + 2,
+      OtherElt = GetElementPtrInst::CreateInBounds(OtherPtr, Idx,
                                               OtherPtr->getName()+"."+Twine(i),
                                                    MI);
       uint64_t EltOffset;
-      const PointerType *OtherPtrTy = cast<PointerType>(OtherPtr->getType());
-      const Type *OtherTy = OtherPtrTy->getElementType();
-      if (const StructType *ST = dyn_cast<StructType>(OtherTy)) {
+      PointerType *OtherPtrTy = cast<PointerType>(OtherPtr->getType());
+      Type *OtherTy = OtherPtrTy->getElementType();
+      if (StructType *ST = dyn_cast<StructType>(OtherTy)) {
         EltOffset = TD->getStructLayout(ST)->getElementOffset(i);
       } else {
-        const Type *EltTy = cast<SequentialType>(OtherTy)->getElementType();
+        Type *EltTy = cast<SequentialType>(OtherTy)->getElementType();
         EltOffset = TD->getTypeAllocSize(EltTy)*i;
       }
 
@@ -2181,7 +2097,7 @@ void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
     }
 
     Value *EltPtr = NewElts[i];
-    const Type *EltTy = cast<PointerType>(EltPtr->getType())->getElementType();
+    Type *EltTy = cast<PointerType>(EltPtr->getType())->getElementType();
 
     // If we got down to a scalar, insert a load or store as appropriate.
     if (EltTy->isSingleValueType()) {
@@ -2207,7 +2123,7 @@ void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
           StoreVal = Constant::getNullValue(EltTy);  // 0.0, null, 0, <0,0>
         } else {
           // If EltTy is a vector type, get the element type.
-          const Type *ValTy = EltTy->getScalarType();
+          Type *ValTy = EltTy->getScalarType();
 
           // Construct an integer with the right value.
           unsigned EltSize = TD->getTypeSizeInBits(ValTy);
@@ -2228,8 +2144,8 @@ void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
           assert(StoreVal->getType() == ValTy && "Type mismatch!");
 
           // If the requested value was a vector constant, create it.
-          if (EltTy != ValTy) {
-            unsigned NumElts = cast<VectorType>(ValTy)->getNumElements();
+          if (EltTy->isVectorTy()) {
+            unsigned NumElts = cast<VectorType>(EltTy)->getNumElements();
             SmallVector<Constant*, 16> Elts(NumElts, StoreVal);
             StoreVal = ConstantVector::get(Elts);
           }
@@ -2271,7 +2187,7 @@ void SROA::RewriteStoreUserOfWholeAlloca(StoreInst *SI, AllocaInst *AI,
   // Extract each element out of the integer according to its structure offset
   // and store the element value to the individual alloca.
   Value *SrcVal = SI->getOperand(0);
-  const Type *AllocaEltTy = AI->getAllocatedType();
+  Type *AllocaEltTy = AI->getAllocatedType();
   uint64_t AllocaSizeBits = TD->getTypeAllocSizeInBits(AllocaEltTy);
 
   IRBuilder<> Builder(SI);
@@ -2286,12 +2202,12 @@ void SROA::RewriteStoreUserOfWholeAlloca(StoreInst *SI, AllocaInst *AI,
 
   // There are two forms here: AI could be an array or struct.  Both cases
   // have different ways to compute the element offset.
-  if (const StructType *EltSTy = dyn_cast<StructType>(AllocaEltTy)) {
+  if (StructType *EltSTy = dyn_cast<StructType>(AllocaEltTy)) {
     const StructLayout *Layout = TD->getStructLayout(EltSTy);
 
     for (unsigned i = 0, e = NewElts.size(); i != e; ++i) {
       // Get the number of bits to shift SrcVal to get the value.
-      const Type *FieldTy = EltSTy->getElementType(i);
+      Type *FieldTy = EltSTy->getElementType(i);
       uint64_t Shift = Layout->getElementOffsetInBits(i);
 
       if (TD->isBigEndian())
@@ -2327,8 +2243,8 @@ void SROA::RewriteStoreUserOfWholeAlloca(StoreInst *SI, AllocaInst *AI,
     }
 
   } else {
-    const ArrayType *ATy = cast<ArrayType>(AllocaEltTy);
-    const Type *ArrayEltTy = ATy->getElementType();
+    ArrayType *ATy = cast<ArrayType>(AllocaEltTy);
+    Type *ArrayEltTy = ATy->getElementType();
     uint64_t ElementOffset = TD->getTypeAllocSizeInBits(ArrayEltTy);
     uint64_t ElementSizeBits = TD->getTypeSizeInBits(ArrayEltTy);
 
@@ -2384,7 +2300,7 @@ void SROA::RewriteLoadUserOfWholeAlloca(LoadInst *LI, AllocaInst *AI,
                                         SmallVector<AllocaInst*, 32> &NewElts) {
   // Extract each element out of the NewElts according to its structure offset
   // and form the result value.
-  const Type *AllocaEltTy = AI->getAllocatedType();
+  Type *AllocaEltTy = AI->getAllocatedType();
   uint64_t AllocaSizeBits = TD->getTypeAllocSizeInBits(AllocaEltTy);
 
   DEBUG(dbgs() << "PROMOTING LOAD OF WHOLE ALLOCA: " << *AI << '\n' << *LI
@@ -2394,10 +2310,10 @@ void SROA::RewriteLoadUserOfWholeAlloca(LoadInst *LI, AllocaInst *AI,
   // have different ways to compute the element offset.
   const StructLayout *Layout = 0;
   uint64_t ArrayEltBitOffset = 0;
-  if (const StructType *EltSTy = dyn_cast<StructType>(AllocaEltTy)) {
+  if (StructType *EltSTy = dyn_cast<StructType>(AllocaEltTy)) {
     Layout = TD->getStructLayout(EltSTy);
   } else {
-    const Type *ArrayEltTy = cast<ArrayType>(AllocaEltTy)->getElementType();
+    Type *ArrayEltTy = cast<ArrayType>(AllocaEltTy)->getElementType();
     ArrayEltBitOffset = TD->getTypeAllocSizeInBits(ArrayEltTy);
   }
 
@@ -2408,14 +2324,14 @@ void SROA::RewriteLoadUserOfWholeAlloca(LoadInst *LI, AllocaInst *AI,
     // Load the value from the alloca.  If the NewElt is an aggregate, cast
     // the pointer to an integer of the same size before doing the load.
     Value *SrcField = NewElts[i];
-    const Type *FieldTy =
+    Type *FieldTy =
       cast<PointerType>(SrcField->getType())->getElementType();
     uint64_t FieldSizeBits = TD->getTypeSizeInBits(FieldTy);
 
     // Ignore zero sized fields like {}, they obviously contain no data.
     if (FieldSizeBits == 0) continue;
 
-    const IntegerType *FieldIntTy = IntegerType::get(LI->getContext(),
+    IntegerType *FieldIntTy = IntegerType::get(LI->getContext(),
                                                      FieldSizeBits);
     if (!FieldTy->isIntegerTy() && !FieldTy->isFloatingPointTy() &&
         !FieldTy->isVectorTy())
@@ -2468,14 +2384,14 @@ void SROA::RewriteLoadUserOfWholeAlloca(LoadInst *LI, AllocaInst *AI,
 /// HasPadding - Return true if the specified type has any structure or
 /// alignment padding in between the elements that would be split apart
 /// by SROA; return false otherwise.
-static bool HasPadding(const Type *Ty, const TargetData &TD) {
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+static bool HasPadding(Type *Ty, const TargetData &TD) {
+  if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Ty = ATy->getElementType();
     return TD.getTypeSizeInBits(Ty) != TD.getTypeAllocSizeInBits(Ty);
   }
 
   // SROA currently handles only Arrays and Structs.
-  const StructType *STy = cast<StructType>(Ty);
+  StructType *STy = cast<StructType>(Ty);
   const StructLayout *SL = TD.getStructLayout(STy);
   unsigned PrevFieldBitOffset = 0;
   for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
@@ -2530,7 +2446,7 @@ bool SROA::isSafeAllocaToScalarRepl(AllocaInst *AI) {
   // and fusion code.
   if (!Info.hasSubelementAccess && Info.hasALoadOrStore) {
     // If the struct/array just has one element, use basic SRoA.
-    if (const StructType *ST = dyn_cast<StructType>(AI->getAllocatedType())) {
+    if (StructType *ST = dyn_cast<StructType>(AI->getAllocatedType())) {
       if (ST->getNumElements() > 1) return false;
     } else {
       if (cast<ArrayType>(AI->getAllocatedType())->getNumElements() > 1)
@@ -2576,7 +2492,7 @@ isOnlyCopiedFromConstantGlobal(Value *V, MemTransferInst *&TheCopy,
 
     if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
       // Ignore non-volatile loads, they are always ok.
-      if (LI->isVolatile()) return false;
+      if (!LI->isSimple()) return false;
       continue;
     }
 
diff --git a/contrib/llvm/lib/Transforms/Scalar/SimplifyLibCalls.cpp b/contrib/llvm/lib/Transforms/Scalar/SimplifyLibCalls.cpp
index 7c415e5150dc..fbb9465743ce 100644
--- a/contrib/llvm/lib/Transforms/Scalar/SimplifyLibCalls.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/SimplifyLibCalls.cpp
@@ -134,7 +134,7 @@ namespace {
 struct StrCatOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strcat" function prototype.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         FT->getReturnType() != B.getInt8PtrTy() ||
         FT->getParamType(0) != FT->getReturnType() ||
@@ -184,7 +184,7 @@ struct StrCatOpt : public LibCallOptimization {
 struct StrNCatOpt : public StrCatOpt {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strncat" function prototype.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 ||
         FT->getReturnType() != B.getInt8PtrTy() ||
         FT->getParamType(0) != FT->getReturnType() ||
@@ -232,7 +232,7 @@ struct StrNCatOpt : public StrCatOpt {
 struct StrChrOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strchr" function prototype.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         FT->getReturnType() != B.getInt8PtrTy() ||
         FT->getParamType(0) != FT->getReturnType() ||
@@ -282,7 +282,7 @@ struct StrChrOpt : public LibCallOptimization {
 struct StrRChrOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strrchr" function prototype.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         FT->getReturnType() != B.getInt8PtrTy() ||
         FT->getParamType(0) != FT->getReturnType() ||
@@ -323,7 +323,7 @@ struct StrRChrOpt : public LibCallOptimization {
 struct StrCmpOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strcmp" function prototype.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         !FT->getReturnType()->isIntegerTy(32) ||
         FT->getParamType(0) != FT->getParamType(1) ||
@@ -338,16 +338,17 @@ struct StrCmpOpt : public LibCallOptimization {
     bool HasStr1 = GetConstantStringInfo(Str1P, Str1);
     bool HasStr2 = GetConstantStringInfo(Str2P, Str2);
 
-    if (HasStr1 && Str1.empty()) // strcmp("", x) -> *x
-      return B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"), CI->getType());
-
-    if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
-      return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
-
     // strcmp(x, y)  -> cnst  (if both x and y are constant strings)
     if (HasStr1 && HasStr2)
       return ConstantInt::get(CI->getType(),
-                                     strcmp(Str1.c_str(),Str2.c_str()));
+                              StringRef(Str1).compare(Str2));
+
+    if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
+      return B.CreateNeg(B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"),
+                                      CI->getType()));
+
+    if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
+      return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
 
     // strcmp(P, "x") -> memcmp(P, "x", 2)
     uint64_t Len1 = GetStringLength(Str1P);
@@ -371,7 +372,7 @@ struct StrCmpOpt : public LibCallOptimization {
 struct StrNCmpOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strncmp" function prototype.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 ||
         !FT->getReturnType()->isIntegerTy(32) ||
         FT->getParamType(0) != FT->getParamType(1) ||
@@ -400,16 +401,20 @@ struct StrNCmpOpt : public LibCallOptimization {
     bool HasStr1 = GetConstantStringInfo(Str1P, Str1);
     bool HasStr2 = GetConstantStringInfo(Str2P, Str2);
 
-    if (HasStr1 && Str1.empty())  // strncmp("", x, n) -> *x
-      return B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"), CI->getType());
+    // strncmp(x, y)  -> cnst  (if both x and y are constant strings)
+    if (HasStr1 && HasStr2) {
+      StringRef SubStr1 = StringRef(Str1).substr(0, Length);
+      StringRef SubStr2 = StringRef(Str2).substr(0, Length);
+      return ConstantInt::get(CI->getType(), SubStr1.compare(SubStr2));
+    }
+
+    if (HasStr1 && Str1.empty())  // strncmp("", x, n) -> -*x
+      return B.CreateNeg(B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"),
+                                      CI->getType()));
 
     if (HasStr2 && Str2.empty())  // strncmp(x, "", n) -> *x
       return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
 
-    // strncmp(x, y)  -> cnst  (if both x and y are constant strings)
-    if (HasStr1 && HasStr2)
-      return ConstantInt::get(CI->getType(),
-                              strncmp(Str1.c_str(), Str2.c_str(), Length));
     return 0;
   }
 };
@@ -426,7 +431,7 @@ struct StrCpyOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Verify the "strcpy" function prototype.
     unsigned NumParams = OptChkCall ? 3 : 2;
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != NumParams ||
         FT->getReturnType() != FT->getParamType(0) ||
         FT->getParamType(0) != FT->getParamType(1) ||
@@ -462,7 +467,7 @@ struct StrCpyOpt : public LibCallOptimization {
 
 struct StrNCpyOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
         FT->getParamType(0) != FT->getParamType(1) ||
         FT->getParamType(0) != B.getInt8PtrTy() ||
@@ -511,7 +516,7 @@ struct StrNCpyOpt : public LibCallOptimization {
 
 struct StrLenOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 1 ||
         FT->getParamType(0) != B.getInt8PtrTy() ||
         !FT->getReturnType()->isIntegerTy())
@@ -537,7 +542,7 @@ struct StrLenOpt : public LibCallOptimization {
 
 struct StrPBrkOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         FT->getParamType(0) != B.getInt8PtrTy() ||
         FT->getParamType(1) != FT->getParamType(0) ||
@@ -575,7 +580,7 @@ struct StrPBrkOpt : public LibCallOptimization {
 
 struct StrToOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if ((FT->getNumParams() != 2 && FT->getNumParams() != 3) ||
         !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy())
@@ -597,7 +602,7 @@ struct StrToOpt : public LibCallOptimization {
 
 struct StrSpnOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         FT->getParamType(0) != B.getInt8PtrTy() ||
         FT->getParamType(1) != FT->getParamType(0) ||
@@ -626,7 +631,7 @@ struct StrSpnOpt : public LibCallOptimization {
 
 struct StrCSpnOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         FT->getParamType(0) != B.getInt8PtrTy() ||
         FT->getParamType(1) != FT->getParamType(0) ||
@@ -658,7 +663,7 @@ struct StrCSpnOpt : public LibCallOptimization {
 
 struct StrStrOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 ||
         !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
@@ -722,7 +727,7 @@ struct StrStrOpt : public LibCallOptimization {
 
 struct MemCmpOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 || !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
         !FT->getReturnType()->isIntegerTy(32))
@@ -773,7 +778,7 @@ struct MemCpyOpt : public LibCallOptimization {
     // These optimizations require TargetData.
     if (!TD) return 0;
 
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
         !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
@@ -795,7 +800,7 @@ struct MemMoveOpt : public LibCallOptimization {
     // These optimizations require TargetData.
     if (!TD) return 0;
 
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
         !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
@@ -817,7 +822,7 @@ struct MemSetOpt : public LibCallOptimization {
     // These optimizations require TargetData.
     if (!TD) return 0;
 
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
         !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isIntegerTy() ||
@@ -840,7 +845,7 @@ struct MemSetOpt : public LibCallOptimization {
 
 struct PowOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // Just make sure this has 2 arguments of the same FP type, which match the
     // result type.
     if (FT->getNumParams() != 2 || FT->getReturnType() != FT->getParamType(0) ||
@@ -874,8 +879,8 @@ struct PowOpt : public LibCallOptimization {
                                          Callee->getAttributes());
       Value *FAbs = EmitUnaryFloatFnCall(Sqrt, "fabs", B,
                                          Callee->getAttributes());
-      Value *FCmp = B.CreateFCmpOEQ(Op1, NegInf, "tmp");
-      Value *Sel = B.CreateSelect(FCmp, Inf, FAbs, "tmp");
+      Value *FCmp = B.CreateFCmpOEQ(Op1, NegInf);
+      Value *Sel = B.CreateSelect(FCmp, Inf, FAbs);
       return Sel;
     }
 
@@ -895,7 +900,7 @@ struct PowOpt : public LibCallOptimization {
 
 struct Exp2Opt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // Just make sure this has 1 argument of FP type, which matches the
     // result type.
     if (FT->getNumParams() != 1 || FT->getReturnType() != FT->getParamType(0) ||
@@ -908,10 +913,10 @@ struct Exp2Opt : public LibCallOptimization {
     Value *LdExpArg = 0;
     if (SIToFPInst *OpC = dyn_cast<SIToFPInst>(Op)) {
       if (OpC->getOperand(0)->getType()->getPrimitiveSizeInBits() <= 32)
-        LdExpArg = B.CreateSExt(OpC->getOperand(0), B.getInt32Ty(), "tmp");
+        LdExpArg = B.CreateSExt(OpC->getOperand(0), B.getInt32Ty());
     } else if (UIToFPInst *OpC = dyn_cast<UIToFPInst>(Op)) {
       if (OpC->getOperand(0)->getType()->getPrimitiveSizeInBits() < 32)
-        LdExpArg = B.CreateZExt(OpC->getOperand(0), B.getInt32Ty(), "tmp");
+        LdExpArg = B.CreateZExt(OpC->getOperand(0), B.getInt32Ty());
     }
 
     if (LdExpArg) {
@@ -946,7 +951,7 @@ struct Exp2Opt : public LibCallOptimization {
 
 struct UnaryDoubleFPOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 1 || !FT->getReturnType()->isDoubleTy() ||
         !FT->getParamType(0)->isDoubleTy())
       return 0;
@@ -973,7 +978,7 @@ struct UnaryDoubleFPOpt : public LibCallOptimization {
 
 struct FFSOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // Just make sure this has 2 arguments of the same FP type, which match the
     // result type.
     if (FT->getNumParams() != 1 ||
@@ -996,10 +1001,10 @@ struct FFSOpt : public LibCallOptimization {
     Value *F = Intrinsic::getDeclaration(Callee->getParent(),
                                          Intrinsic::cttz, ArgType);
     Value *V = B.CreateCall(F, Op, "cttz");
-    V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1), "tmp");
-    V = B.CreateIntCast(V, B.getInt32Ty(), false, "tmp");
+    V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
+    V = B.CreateIntCast(V, B.getInt32Ty(), false);
 
-    Value *Cond = B.CreateICmpNE(Op, Constant::getNullValue(ArgType), "tmp");
+    Value *Cond = B.CreateICmpNE(Op, Constant::getNullValue(ArgType));
     return B.CreateSelect(Cond, V, B.getInt32(0));
   }
 };
@@ -1009,7 +1014,7 @@ struct FFSOpt : public LibCallOptimization {
 
 struct IsDigitOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // We require integer(i32)
     if (FT->getNumParams() != 1 || !FT->getReturnType()->isIntegerTy() ||
         !FT->getParamType(0)->isIntegerTy(32))
@@ -1028,7 +1033,7 @@ struct IsDigitOpt : public LibCallOptimization {
 
 struct IsAsciiOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // We require integer(i32)
     if (FT->getNumParams() != 1 || !FT->getReturnType()->isIntegerTy() ||
         !FT->getParamType(0)->isIntegerTy(32))
@@ -1046,7 +1051,7 @@ struct IsAsciiOpt : public LibCallOptimization {
 
 struct AbsOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // We require integer(integer) where the types agree.
     if (FT->getNumParams() != 1 || !FT->getReturnType()->isIntegerTy() ||
         FT->getParamType(0) != FT->getReturnType())
@@ -1067,7 +1072,7 @@ struct AbsOpt : public LibCallOptimization {
 
 struct ToAsciiOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     // We require i32(i32)
     if (FT->getNumParams() != 1 || FT->getReturnType() != FT->getParamType(0) ||
         !FT->getParamType(0)->isIntegerTy(32))
@@ -1147,7 +1152,7 @@ struct PrintFOpt : public LibCallOptimization {
 
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Require one fixed pointer argument and an integer/void result.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() < 1 || !FT->getParamType(0)->isPointerTy() ||
         !(FT->getReturnType()->isIntegerTy() ||
           FT->getReturnType()->isVoidTy()))
@@ -1241,7 +1246,7 @@ struct SPrintFOpt : public LibCallOptimization {
 
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Require two fixed pointer arguments and an integer result.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 || !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
         !FT->getReturnType()->isIntegerTy())
@@ -1272,7 +1277,7 @@ struct SPrintFOpt : public LibCallOptimization {
 struct FWriteOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Require a pointer, an integer, an integer, a pointer, returning integer.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 4 || !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isIntegerTy() ||
         !FT->getParamType(2)->isIntegerTy() ||
@@ -1310,7 +1315,7 @@ struct FPutsOpt : public LibCallOptimization {
     if (!TD) return 0;
 
     // Require two pointers.  Also, we can't optimize if return value is used.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 || !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
         !CI->use_empty())
@@ -1379,7 +1384,7 @@ struct FPrintFOpt : public LibCallOptimization {
 
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Require two fixed paramters as pointers and integer result.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() != 2 || !FT->getParamType(0)->isPointerTy() ||
         !FT->getParamType(1)->isPointerTy() ||
         !FT->getReturnType()->isIntegerTy())
@@ -1410,7 +1415,7 @@ struct FPrintFOpt : public LibCallOptimization {
 struct PutsOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     // Require one fixed pointer argument and an integer/void result.
-    const FunctionType *FT = Callee->getFunctionType();
+    FunctionType *FT = Callee->getFunctionType();
     if (FT->getNumParams() < 1 || !FT->getParamType(0)->isPointerTy() ||
         !(FT->getReturnType()->isIntegerTy() ||
           FT->getReturnType()->isVoidTy()))
@@ -1685,7 +1690,7 @@ void SimplifyLibCalls::setDoesNotAlias(Function &F, unsigned n) {
 
 
 void SimplifyLibCalls::inferPrototypeAttributes(Function &F) {
-  const FunctionType *FTy = F.getFunctionType();
+  FunctionType *FTy = F.getFunctionType();
   
   StringRef Name = F.getName();
   switch (Name[0]) {
diff --git a/contrib/llvm/lib/Transforms/Scalar/Sink.cpp b/contrib/llvm/lib/Transforms/Scalar/Sink.cpp
index 705f44204900..c83f56c4d2d7 100644
--- a/contrib/llvm/lib/Transforms/Scalar/Sink.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/Sink.cpp
@@ -153,9 +153,13 @@ bool Sinking::ProcessBlock(BasicBlock &BB) {
 
 static bool isSafeToMove(Instruction *Inst, AliasAnalysis *AA,
                          SmallPtrSet<Instruction *, 8> &Stores) {
-  if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
-    if (L->isVolatile()) return false;
 
+  if (Inst->mayWriteToMemory()) {
+    Stores.insert(Inst);
+    return false;
+  }
+
+  if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
     AliasAnalysis::Location Loc = AA->getLocation(L);
     for (SmallPtrSet<Instruction *, 8>::iterator I = Stores.begin(),
          E = Stores.end(); I != E; ++I)
@@ -163,11 +167,6 @@ static bool isSafeToMove(Instruction *Inst, AliasAnalysis *AA,
         return false;
   }
 
-  if (Inst->mayWriteToMemory()) {
-    Stores.insert(Inst);
-    return false;
-  }
-
   if (isa<TerminatorInst>(Inst) || isa<PHINode>(Inst))
     return false;
 
diff --git a/contrib/llvm/lib/Transforms/Scalar/TailDuplication.cpp b/contrib/llvm/lib/Transforms/Scalar/TailDuplication.cpp
deleted file mode 100644
index 9dd83c04fa61..000000000000
--- a/contrib/llvm/lib/Transforms/Scalar/TailDuplication.cpp
+++ /dev/null
@@ -1,373 +0,0 @@
-//===- TailDuplication.cpp - Simplify CFG through tail duplication --------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass performs a limited form of tail duplication, intended to simplify
-// CFGs by removing some unconditional branches.  This pass is necessary to
-// straighten out loops created by the C front-end, but also is capable of
-// making other code nicer.  After this pass is run, the CFG simplify pass
-// should be run to clean up the mess.
-//
-// This pass could be enhanced in the future to use profile information to be
-// more aggressive.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "tailduplicate"
-#include "llvm/Transforms/Scalar.h"
-#include "llvm/Constant.h"
-#include "llvm/Function.h"
-#include "llvm/Instructions.h"
-#include "llvm/IntrinsicInst.h"
-#include "llvm/Pass.h"
-#include "llvm/Type.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Support/CFG.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Utils/Local.h"
-#include <map>
-using namespace llvm;
-
-STATISTIC(NumEliminated, "Number of unconditional branches eliminated");
-
-static cl::opt<unsigned>
-TailDupThreshold("taildup-threshold",
-                 cl::desc("Max block size to tail duplicate"),
-                 cl::init(1), cl::Hidden);
-
-namespace {
-  class TailDup : public FunctionPass {
-    bool runOnFunction(Function &F);
-  public:
-    static char ID; // Pass identification, replacement for typeid
-    TailDup() : FunctionPass(ID) {
-      initializeTailDupPass(*PassRegistry::getPassRegistry());
-    }
-
-  private:
-    inline bool shouldEliminateUnconditionalBranch(TerminatorInst *, unsigned);
-    inline void eliminateUnconditionalBranch(BranchInst *BI);
-    SmallPtrSet<BasicBlock*, 4> CycleDetector;
-  };
-}
-
-char TailDup::ID = 0;
-INITIALIZE_PASS(TailDup, "tailduplicate", "Tail Duplication", false, false)
-
-// Public interface to the Tail Duplication pass
-FunctionPass *llvm::createTailDuplicationPass() { return new TailDup(); }
-
-/// runOnFunction - Top level algorithm - Loop over each unconditional branch in
-/// the function, eliminating it if it looks attractive enough.  CycleDetector
-/// prevents infinite loops by checking that we aren't redirecting a branch to
-/// a place it already pointed to earlier; see PR 2323.
-bool TailDup::runOnFunction(Function &F) {
-  bool Changed = false;
-  CycleDetector.clear();
-  for (Function::iterator I = F.begin(), E = F.end(); I != E; ) {
-    if (shouldEliminateUnconditionalBranch(I->getTerminator(),
-                                           TailDupThreshold)) {
-      eliminateUnconditionalBranch(cast<BranchInst>(I->getTerminator()));
-      Changed = true;
-    } else {
-      ++I;
-      CycleDetector.clear();
-    }
-  }
-  return Changed;
-}
-
-/// shouldEliminateUnconditionalBranch - Return true if this branch looks
-/// attractive to eliminate.  We eliminate the branch if the destination basic
-/// block has <= 5 instructions in it, not counting PHI nodes.  In practice,
-/// since one of these is a terminator instruction, this means that we will add
-/// up to 4 instructions to the new block.
-///
-/// We don't count PHI nodes in the count since they will be removed when the
-/// contents of the block are copied over.
-///
-bool TailDup::shouldEliminateUnconditionalBranch(TerminatorInst *TI,
-                                                 unsigned Threshold) {
-  BranchInst *BI = dyn_cast<BranchInst>(TI);
-  if (!BI || !BI->isUnconditional()) return false;  // Not an uncond branch!
-
-  BasicBlock *Dest = BI->getSuccessor(0);
-  if (Dest == BI->getParent()) return false;        // Do not loop infinitely!
-
-  // Do not inline a block if we will just get another branch to the same block!
-  TerminatorInst *DTI = Dest->getTerminator();
-  if (BranchInst *DBI = dyn_cast<BranchInst>(DTI))
-    if (DBI->isUnconditional() && DBI->getSuccessor(0) == Dest)
-      return false;                                 // Do not loop infinitely!
-
-  // FIXME: DemoteRegToStack cannot yet demote invoke instructions to the stack,
-  // because doing so would require breaking critical edges.  This should be
-  // fixed eventually.
-  if (!DTI->use_empty())
-    return false;
-
-  // Do not bother with blocks with only a single predecessor: simplify
-  // CFG will fold these two blocks together!
-  pred_iterator PI = pred_begin(Dest), PE = pred_end(Dest);
-  ++PI;
-  if (PI == PE) return false;  // Exactly one predecessor!
-
-  BasicBlock::iterator I = Dest->getFirstNonPHI();
-
-  for (unsigned Size = 0; I != Dest->end(); ++I) {
-    if (Size == Threshold) return false;  // The block is too large.
-    
-    // Don't tail duplicate call instructions.  They are very large compared to
-    // other instructions.
-    if (isa<CallInst>(I) || isa<InvokeInst>(I)) return false;
-
-    // Also alloca and malloc.
-    if (isa<AllocaInst>(I)) return false;
-
-    // Some vector instructions can expand into a number of instructions.
-    if (isa<ShuffleVectorInst>(I) || isa<ExtractElementInst>(I) ||
-        isa<InsertElementInst>(I)) return false;
-    
-    // Only count instructions that are not debugger intrinsics.
-    if (!isa<DbgInfoIntrinsic>(I)) ++Size;
-  }
-
-  // Do not tail duplicate a block that has thousands of successors into a block
-  // with a single successor if the block has many other predecessors.  This can
-  // cause an N^2 explosion in CFG edges (and PHI node entries), as seen in
-  // cases that have a large number of indirect gotos.
-  unsigned NumSuccs = DTI->getNumSuccessors();
-  if (NumSuccs > 8) {
-    unsigned TooMany = 128;
-    if (NumSuccs >= TooMany) return false;
-    TooMany = TooMany/NumSuccs;
-    for (; PI != PE; ++PI)
-      if (TooMany-- == 0) return false;
-  }
-  
-  // If this unconditional branch is a fall-through, be careful about
-  // tail duplicating it.  In particular, we don't want to taildup it if the
-  // original block will still be there after taildup is completed: doing so
-  // would eliminate the fall-through, requiring unconditional branches.
-  Function::iterator DestI = Dest;
-  if (&*--DestI == BI->getParent()) {
-    // The uncond branch is a fall-through.  Tail duplication of the block is
-    // will eliminate the fall-through-ness and end up cloning the terminator
-    // at the end of the Dest block.  Since the original Dest block will
-    // continue to exist, this means that one or the other will not be able to
-    // fall through.  One typical example that this helps with is code like:
-    // if (a)
-    //   foo();
-    // if (b)
-    //   foo();
-    // Cloning the 'if b' block into the end of the first foo block is messy.
-    
-    // The messy case is when the fall-through block falls through to other
-    // blocks.  This is what we would be preventing if we cloned the block.
-    DestI = Dest;
-    if (++DestI != Dest->getParent()->end()) {
-      BasicBlock *DestSucc = DestI;
-      // If any of Dest's successors are fall-throughs, don't do this xform.
-      for (succ_iterator SI = succ_begin(Dest), SE = succ_end(Dest);
-           SI != SE; ++SI)
-        if (*SI == DestSucc)
-          return false;
-    }
-  }
-
-  // Finally, check that we haven't redirected to this target block earlier;
-  // there are cases where we loop forever if we don't check this (PR 2323).
-  if (!CycleDetector.insert(Dest))
-    return false;
-
-  return true;
-}
-
-/// FindObviousSharedDomOf - We know there is a branch from SrcBlock to
-/// DestBlock, and that SrcBlock is not the only predecessor of DstBlock.  If we
-/// can find a predecessor of SrcBlock that is a dominator of both SrcBlock and
-/// DstBlock, return it.
-static BasicBlock *FindObviousSharedDomOf(BasicBlock *SrcBlock,
-                                          BasicBlock *DstBlock) {
-  // SrcBlock must have a single predecessor.
-  pred_iterator PI = pred_begin(SrcBlock), PE = pred_end(SrcBlock);
-  if (PI == PE || ++PI != PE) return 0;
-
-  BasicBlock *SrcPred = *pred_begin(SrcBlock);
-
-  // Look at the predecessors of DstBlock.  One of them will be SrcBlock.  If
-  // there is only one other pred, get it, otherwise we can't handle it.
-  PI = pred_begin(DstBlock); PE = pred_end(DstBlock);
-  BasicBlock *DstOtherPred = 0;
-  BasicBlock *P = *PI;
-  if (P == SrcBlock) {
-    if (++PI == PE) return 0;
-    DstOtherPred = *PI;
-    if (++PI != PE) return 0;
-  } else {
-    DstOtherPred = P;
-    if (++PI == PE || *PI != SrcBlock || ++PI != PE) return 0;
-  }
-
-  // We can handle two situations here: "if then" and "if then else" blocks.  An
-  // 'if then' situation is just where DstOtherPred == SrcPred.
-  if (DstOtherPred == SrcPred)
-    return SrcPred;
-
-  // Check to see if we have an "if then else" situation, which means that
-  // DstOtherPred will have a single predecessor and it will be SrcPred.
-  PI = pred_begin(DstOtherPred); PE = pred_end(DstOtherPred);
-  if (PI != PE && *PI == SrcPred) {
-    if (++PI != PE) return 0;  // Not a single pred.
-    return SrcPred;  // Otherwise, it's an "if then" situation.  Return the if.
-  }
-
-  // Otherwise, this is something we can't handle.
-  return 0;
-}
-
-
-/// eliminateUnconditionalBranch - Clone the instructions from the destination
-/// block into the source block, eliminating the specified unconditional branch.
-/// If the destination block defines values used by successors of the dest
-/// block, we may need to insert PHI nodes.
-///
-void TailDup::eliminateUnconditionalBranch(BranchInst *Branch) {
-  BasicBlock *SourceBlock = Branch->getParent();
-  BasicBlock *DestBlock = Branch->getSuccessor(0);
-  assert(SourceBlock != DestBlock && "Our predicate is broken!");
-
-  DEBUG(dbgs() << "TailDuplication[" << SourceBlock->getParent()->getName()
-        << "]: Eliminating branch: " << *Branch);
-
-  // See if we can avoid duplicating code by moving it up to a dominator of both
-  // blocks.
-  if (BasicBlock *DomBlock = FindObviousSharedDomOf(SourceBlock, DestBlock)) {
-    DEBUG(dbgs() << "Found shared dominator: " << DomBlock->getName() << "\n");
-
-    // If there are non-phi instructions in DestBlock that have no operands
-    // defined in DestBlock, and if the instruction has no side effects, we can
-    // move the instruction to DomBlock instead of duplicating it.
-    BasicBlock::iterator BBI = DestBlock->getFirstNonPHI();
-    while (!isa<TerminatorInst>(BBI)) {
-      Instruction *I = BBI++;
-
-      bool CanHoist = I->isSafeToSpeculativelyExecute() &&
-                      !I->mayReadFromMemory();
-      if (CanHoist) {
-        for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op)
-          if (Instruction *OpI = dyn_cast<Instruction>(I->getOperand(op)))
-            if (OpI->getParent() == DestBlock ||
-                (isa<InvokeInst>(OpI) && OpI->getParent() == DomBlock)) {
-              CanHoist = false;
-              break;
-            }
-        if (CanHoist) {
-          // Remove from DestBlock, move right before the term in DomBlock.
-          DestBlock->getInstList().remove(I);
-          DomBlock->getInstList().insert(DomBlock->getTerminator(), I);
-          DEBUG(dbgs() << "Hoisted: " << *I);
-        }
-      }
-    }
-  }
-
-  // Tail duplication can not update SSA properties correctly if the values
-  // defined in the duplicated tail are used outside of the tail itself.  For
-  // this reason, we spill all values that are used outside of the tail to the
-  // stack.
-  for (BasicBlock::iterator I = DestBlock->begin(); I != DestBlock->end(); ++I)
-    if (I->isUsedOutsideOfBlock(DestBlock)) {
-      // We found a use outside of the tail.  Create a new stack slot to
-      // break this inter-block usage pattern.
-      DemoteRegToStack(*I);
-    }
-
-  // We are going to have to map operands from the original block B to the new
-  // copy of the block B'.  If there are PHI nodes in the DestBlock, these PHI
-  // nodes also define part of this mapping.  Loop over these PHI nodes, adding
-  // them to our mapping.
-  //
-  std::map<Value*, Value*> ValueMapping;
-
-  BasicBlock::iterator BI = DestBlock->begin();
-  bool HadPHINodes = isa<PHINode>(BI);
-  for (; PHINode *PN = dyn_cast<PHINode>(BI); ++BI)
-    ValueMapping[PN] = PN->getIncomingValueForBlock(SourceBlock);
-
-  // Clone the non-phi instructions of the dest block into the source block,
-  // keeping track of the mapping...
-  //
-  for (; BI != DestBlock->end(); ++BI) {
-    Instruction *New = BI->clone();
-    New->setName(BI->getName());
-    SourceBlock->getInstList().push_back(New);
-    ValueMapping[BI] = New;
-  }
-
-  // Now that we have built the mapping information and cloned all of the
-  // instructions (giving us a new terminator, among other things), walk the new
-  // instructions, rewriting references of old instructions to use new
-  // instructions.
-  //
-  BI = Branch; ++BI;  // Get an iterator to the first new instruction
-  for (; BI != SourceBlock->end(); ++BI)
-    for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
-      std::map<Value*, Value*>::const_iterator I =
-        ValueMapping.find(BI->getOperand(i));
-      if (I != ValueMapping.end())
-        BI->setOperand(i, I->second);
-    }
-
-  // Next we check to see if any of the successors of DestBlock had PHI nodes.
-  // If so, we need to add entries to the PHI nodes for SourceBlock now.
-  for (succ_iterator SI = succ_begin(DestBlock), SE = succ_end(DestBlock);
-       SI != SE; ++SI) {
-    BasicBlock *Succ = *SI;
-    for (BasicBlock::iterator PNI = Succ->begin(); isa<PHINode>(PNI); ++PNI) {
-      PHINode *PN = cast<PHINode>(PNI);
-      // Ok, we have a PHI node.  Figure out what the incoming value was for the
-      // DestBlock.
-      Value *IV = PN->getIncomingValueForBlock(DestBlock);
-
-      // Remap the value if necessary...
-      std::map<Value*, Value*>::const_iterator I = ValueMapping.find(IV);
-      if (I != ValueMapping.end())
-        IV = I->second;
-      PN->addIncoming(IV, SourceBlock);
-    }
-  }
-
-  // Next, remove the old branch instruction, and any PHI node entries that we
-  // had.
-  BI = Branch; ++BI;  // Get an iterator to the first new instruction
-  DestBlock->removePredecessor(SourceBlock); // Remove entries in PHI nodes...
-  SourceBlock->getInstList().erase(Branch);  // Destroy the uncond branch...
-
-  // Final step: now that we have finished everything up, walk the cloned
-  // instructions one last time, constant propagating and DCE'ing them, because
-  // they may not be needed anymore.
-  //
-  if (HadPHINodes) {
-    while (BI != SourceBlock->end()) {
-      Instruction *Inst = BI++;
-      if (isInstructionTriviallyDead(Inst))
-        Inst->eraseFromParent();
-      else if (Value *V = SimplifyInstruction(Inst)) {
-        Inst->replaceAllUsesWith(V);
-        Inst->eraseFromParent();
-      }
-    }
-  }
-
-  ++NumEliminated;  // We just killed a branch!
-}
diff --git a/contrib/llvm/lib/Transforms/Utils/AddrModeMatcher.cpp b/contrib/llvm/lib/Transforms/Utils/AddrModeMatcher.cpp
index be7bed1cecdf..8e5a1eb2c843 100644
--- a/contrib/llvm/lib/Transforms/Utils/AddrModeMatcher.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/AddrModeMatcher.cpp
@@ -222,7 +222,7 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
     const TargetData *TD = TLI.getTargetData();
     gep_type_iterator GTI = gep_type_begin(AddrInst);
     for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
-      if (const StructType *STy = dyn_cast<StructType>(*GTI)) {
+      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
         const StructLayout *SL = TD->getStructLayout(STy);
         unsigned Idx =
           cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
@@ -557,7 +557,7 @@ IsProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore,
     Value *Address = User->getOperand(OpNo);
     if (!Address->getType()->isPointerTy())
       return false;
-    const Type *AddressAccessTy =
+    Type *AddressAccessTy =
       cast<PointerType>(Address->getType())->getElementType();
     
     // Do a match against the root of this address, ignoring profitability. This
diff --git a/contrib/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/contrib/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index b4f74f97e978..a7f9efd562e1 100644
--- a/contrib/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -287,7 +287,7 @@ BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, Pass *P) {
 ///
 BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt, Pass *P) {
   BasicBlock::iterator SplitIt = SplitPt;
-  while (isa<PHINode>(SplitIt))
+  while (isa<PHINode>(SplitIt) || isa<LandingPadInst>(SplitIt))
     ++SplitIt;
   BasicBlock *New = Old->splitBasicBlock(SplitIt, Old->getName()+".split");
 
@@ -299,21 +299,147 @@ BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt, Pass *P) {
 
   if (DominatorTree *DT = P->getAnalysisIfAvailable<DominatorTree>()) {
     // Old dominates New. New node dominates all other nodes dominated by Old.
-    DomTreeNode *OldNode = DT->getNode(Old);
-    std::vector<DomTreeNode *> Children;
-    for (DomTreeNode::iterator I = OldNode->begin(), E = OldNode->end();
-         I != E; ++I) 
-      Children.push_back(*I);
+    if (DomTreeNode *OldNode = DT->getNode(Old)) {
+      std::vector<DomTreeNode *> Children;
+      for (DomTreeNode::iterator I = OldNode->begin(), E = OldNode->end();
+           I != E; ++I) 
+        Children.push_back(*I);
 
       DomTreeNode *NewNode = DT->addNewBlock(New,Old);
       for (std::vector<DomTreeNode *>::iterator I = Children.begin(),
              E = Children.end(); I != E; ++I) 
         DT->changeImmediateDominator(*I, NewNode);
+    }
   }
 
   return New;
 }
 
+/// UpdateAnalysisInformation - Update DominatorTree, LoopInfo, and LCCSA
+/// analysis information.
+static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
+                                      ArrayRef<BasicBlock *> Preds,
+                                      Pass *P, bool &HasLoopExit) {
+  if (!P) return;
+
+  LoopInfo *LI = P->getAnalysisIfAvailable<LoopInfo>();
+  Loop *L = LI ? LI->getLoopFor(OldBB) : 0;
+
+  // If we need to preserve loop analyses, collect some information about how
+  // this split will affect loops.
+  bool IsLoopEntry = !!L;
+  bool SplitMakesNewLoopHeader = false;
+  if (LI) {
+    bool PreserveLCSSA = P->mustPreserveAnalysisID(LCSSAID);
+    for (ArrayRef<BasicBlock*>::iterator
+           i = Preds.begin(), e = Preds.end(); i != e; ++i) {
+      BasicBlock *Pred = *i;
+
+      // If we need to preserve LCSSA, determine if any of the preds is a loop
+      // exit.
+      if (PreserveLCSSA)
+        if (Loop *PL = LI->getLoopFor(Pred))
+          if (!PL->contains(OldBB))
+            HasLoopExit = true;
+
+      // If we need to preserve LoopInfo, note whether any of the preds crosses
+      // an interesting loop boundary.
+      if (!L) continue;
+      if (L->contains(Pred))
+        IsLoopEntry = false;
+      else
+        SplitMakesNewLoopHeader = true;
+    }
+  }
+
+  // Update dominator tree if available.
+  DominatorTree *DT = P->getAnalysisIfAvailable<DominatorTree>();
+  if (DT)
+    DT->splitBlock(NewBB);
+
+  if (!L) return;
+
+  if (IsLoopEntry) {
+    // Add the new block to the nearest enclosing loop (and not an adjacent
+    // loop). To find this, examine each of the predecessors and determine which
+    // loops enclose them, and select the most-nested loop which contains the
+    // loop containing the block being split.
+    Loop *InnermostPredLoop = 0;
+    for (ArrayRef<BasicBlock*>::iterator
+           i = Preds.begin(), e = Preds.end(); i != e; ++i) {
+      BasicBlock *Pred = *i;
+      if (Loop *PredLoop = LI->getLoopFor(Pred)) {
+        // Seek a loop which actually contains the block being split (to avoid
+        // adjacent loops).
+        while (PredLoop && !PredLoop->contains(OldBB))
+          PredLoop = PredLoop->getParentLoop();
+
+        // Select the most-nested of these loops which contains the block.
+        if (PredLoop && PredLoop->contains(OldBB) &&
+            (!InnermostPredLoop ||
+             InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
+          InnermostPredLoop = PredLoop;
+      }
+    }
+
+    if (InnermostPredLoop)
+      InnermostPredLoop->addBasicBlockToLoop(NewBB, LI->getBase());
+  } else {
+    L->addBasicBlockToLoop(NewBB, LI->getBase());
+    if (SplitMakesNewLoopHeader)
+      L->moveToHeader(NewBB);
+  }
+}
+
+/// UpdatePHINodes - Update the PHI nodes in OrigBB to include the values coming
+/// from NewBB. This also updates AliasAnalysis, if available.
+static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
+                           ArrayRef<BasicBlock*> Preds, BranchInst *BI,
+                           Pass *P, bool HasLoopExit) {
+  // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
+  AliasAnalysis *AA = P ? P->getAnalysisIfAvailable<AliasAnalysis>() : 0;
+  for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(I); ) {
+    PHINode *PN = cast<PHINode>(I++);
+
+    // Check to see if all of the values coming in are the same.  If so, we
+    // don't need to create a new PHI node, unless it's needed for LCSSA.
+    Value *InVal = 0;
+    if (!HasLoopExit) {
+      InVal = PN->getIncomingValueForBlock(Preds[0]);
+      for (unsigned i = 1, e = Preds.size(); i != e; ++i)
+        if (InVal != PN->getIncomingValueForBlock(Preds[i])) {
+          InVal = 0;
+          break;
+        }
+    }
+
+    if (InVal) {
+      // If all incoming values for the new PHI would be the same, just don't
+      // make a new PHI.  Instead, just remove the incoming values from the old
+      // PHI.
+      for (unsigned i = 0, e = Preds.size(); i != e; ++i)
+        PN->removeIncomingValue(Preds[i], false);
+    } else {
+      // If the values coming into the block are not the same, we need a PHI.
+      // Create the new PHI node, insert it into NewBB at the end of the block
+      PHINode *NewPHI =
+        PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI);
+      if (AA) AA->copyValue(PN, NewPHI);
+      
+      // Move all of the PHI values for 'Preds' to the new PHI.
+      for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
+        Value *V = PN->removeIncomingValue(Preds[i], false);
+        NewPHI->addIncoming(V, Preds[i]);
+      }
+
+      InVal = NewPHI;
+    }
+
+    // Add an incoming value to the PHI node in the loop for the preheader
+    // edge.
+    PN->addIncoming(InVal, NewBB);
+  }
+}
 
 /// SplitBlockPredecessors - This method transforms BB by introducing a new
 /// basic block into the function, and moving some of the predecessors of BB to
@@ -337,48 +463,16 @@ BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
   // The new block unconditionally branches to the old block.
   BranchInst *BI = BranchInst::Create(BB, NewBB);
   
-  LoopInfo *LI = P ? P->getAnalysisIfAvailable<LoopInfo>() : 0;
-  Loop *L = LI ? LI->getLoopFor(BB) : 0;
-  bool PreserveLCSSA = P->mustPreserveAnalysisID(LCSSAID);
-
   // Move the edges from Preds to point to NewBB instead of BB.
-  // While here, if we need to preserve loop analyses, collect
-  // some information about how this split will affect loops.
-  bool HasLoopExit = false;
-  bool IsLoopEntry = !!L;
-  bool SplitMakesNewLoopHeader = false;
   for (unsigned i = 0; i != NumPreds; ++i) {
     // This is slightly more strict than necessary; the minimum requirement
     // is that there be no more than one indirectbr branching to BB. And
     // all BlockAddress uses would need to be updated.
     assert(!isa<IndirectBrInst>(Preds[i]->getTerminator()) &&
            "Cannot split an edge from an IndirectBrInst");
-
     Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB);
-
-    if (LI) {
-      // If we need to preserve LCSSA, determine if any of
-      // the preds is a loop exit.
-      if (PreserveLCSSA)
-        if (Loop *PL = LI->getLoopFor(Preds[i]))
-          if (!PL->contains(BB))
-            HasLoopExit = true;
-      // If we need to preserve LoopInfo, note whether any of the
-      // preds crosses an interesting loop boundary.
-      if (L) {
-        if (L->contains(Preds[i]))
-          IsLoopEntry = false;
-        else
-          SplitMakesNewLoopHeader = true;
-      }
-    }
   }
 
-  // Update dominator tree if available.
-  DominatorTree *DT = P ? P->getAnalysisIfAvailable<DominatorTree>() : 0;
-  if (DT)
-    DT->splitBlock(NewBB);
-
   // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
   // node becomes an incoming value for BB's phi node.  However, if the Preds
   // list is empty, we need to insert dummy entries into the PHI nodes in BB to
@@ -390,82 +484,124 @@ BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
     return NewBB;
   }
 
-  AliasAnalysis *AA = P ? P->getAnalysisIfAvailable<AliasAnalysis>() : 0;
+  // Update DominatorTree, LoopInfo, and LCCSA analysis information.
+  bool HasLoopExit = false;
+  UpdateAnalysisInformation(BB, NewBB, ArrayRef<BasicBlock*>(Preds, NumPreds),
+                            P, HasLoopExit);
 
-  if (L) {
-    if (IsLoopEntry) {
-      // Add the new block to the nearest enclosing loop (and not an
-      // adjacent loop). To find this, examine each of the predecessors and
-      // determine which loops enclose them, and select the most-nested loop
-      // which contains the loop containing the block being split.
-      Loop *InnermostPredLoop = 0;
-      for (unsigned i = 0; i != NumPreds; ++i)
-        if (Loop *PredLoop = LI->getLoopFor(Preds[i])) {
-          // Seek a loop which actually contains the block being split (to
-          // avoid adjacent loops).
-          while (PredLoop && !PredLoop->contains(BB))
-            PredLoop = PredLoop->getParentLoop();
-          // Select the most-nested of these loops which contains the block.
-          if (PredLoop &&
-              PredLoop->contains(BB) &&
-              (!InnermostPredLoop ||
-               InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
-            InnermostPredLoop = PredLoop;
-        }
-      if (InnermostPredLoop)
-        InnermostPredLoop->addBasicBlockToLoop(NewBB, LI->getBase());
-    } else {
-      L->addBasicBlockToLoop(NewBB, LI->getBase());
-      if (SplitMakesNewLoopHeader)
-        L->moveToHeader(NewBB);
-    }
-  }
-  
-  // Otherwise, create a new PHI node in NewBB for each PHI node in BB.
-  for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ) {
-    PHINode *PN = cast<PHINode>(I++);
-    
-    // Check to see if all of the values coming in are the same.  If so, we
-    // don't need to create a new PHI node, unless it's needed for LCSSA.
-    Value *InVal = 0;
-    if (!HasLoopExit) {
-      InVal = PN->getIncomingValueForBlock(Preds[0]);
-      for (unsigned i = 1; i != NumPreds; ++i)
-        if (InVal != PN->getIncomingValueForBlock(Preds[i])) {
-          InVal = 0;
-          break;
-        }
-    }
-
-    if (InVal) {
-      // If all incoming values for the new PHI would be the same, just don't
-      // make a new PHI.  Instead, just remove the incoming values from the old
-      // PHI.
-      for (unsigned i = 0; i != NumPreds; ++i)
-        PN->removeIncomingValue(Preds[i], false);
-    } else {
-      // If the values coming into the block are not the same, we need a PHI.
-      // Create the new PHI node, insert it into NewBB at the end of the block
-      PHINode *NewPHI =
-        PHINode::Create(PN->getType(), NumPreds, PN->getName()+".ph", BI);
-      if (AA) AA->copyValue(PN, NewPHI);
-      
-      // Move all of the PHI values for 'Preds' to the new PHI.
-      for (unsigned i = 0; i != NumPreds; ++i) {
-        Value *V = PN->removeIncomingValue(Preds[i], false);
-        NewPHI->addIncoming(V, Preds[i]);
-      }
-      InVal = NewPHI;
-    }
-    
-    // Add an incoming value to the PHI node in the loop for the preheader
-    // edge.
-    PN->addIncoming(InVal, NewBB);
-  }
-  
+  // Update the PHI nodes in BB with the values coming from NewBB.
+  UpdatePHINodes(BB, NewBB, ArrayRef<BasicBlock*>(Preds, NumPreds), BI,
+                 P, HasLoopExit);
   return NewBB;
 }
 
+/// SplitLandingPadPredecessors - This method transforms the landing pad,
+/// OrigBB, by introducing two new basic blocks into the function. One of those
+/// new basic blocks gets the predecessors listed in Preds. The other basic
+/// block gets the remaining predecessors of OrigBB. The landingpad instruction
+/// OrigBB is clone into both of the new basic blocks. The new blocks are given
+/// the suffixes 'Suffix1' and 'Suffix2', and are returned in the NewBBs vector.
+/// 
+/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree,
+/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses. In particular,
+/// it does not preserve LoopSimplify (because it's complicated to handle the
+/// case where one of the edges being split is an exit of a loop with other
+/// exits).
+/// 
+void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
+                                       ArrayRef<BasicBlock*> Preds,
+                                       const char *Suffix1, const char *Suffix2,
+                                       Pass *P,
+                                       SmallVectorImpl<BasicBlock*> &NewBBs) {
+  assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
+
+  // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
+  // it right before the original block.
+  BasicBlock *NewBB1 = BasicBlock::Create(OrigBB->getContext(),
+                                          OrigBB->getName() + Suffix1,
+                                          OrigBB->getParent(), OrigBB);
+  NewBBs.push_back(NewBB1);
+
+  // The new block unconditionally branches to the old block.
+  BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
+
+  // Move the edges from Preds to point to NewBB1 instead of OrigBB.
+  for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
+    // This is slightly more strict than necessary; the minimum requirement
+    // is that there be no more than one indirectbr branching to BB. And
+    // all BlockAddress uses would need to be updated.
+    assert(!isa<IndirectBrInst>(Preds[i]->getTerminator()) &&
+           "Cannot split an edge from an IndirectBrInst");
+    Preds[i]->getTerminator()->replaceUsesOfWith(OrigBB, NewBB1);
+  }
+
+  // Update DominatorTree, LoopInfo, and LCCSA analysis information.
+  bool HasLoopExit = false;
+  UpdateAnalysisInformation(OrigBB, NewBB1, Preds, P, HasLoopExit);
+
+  // Update the PHI nodes in OrigBB with the values coming from NewBB1.
+  UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, P, HasLoopExit);
+
+  // Move the remaining edges from OrigBB to point to NewBB2.
+  SmallVector<BasicBlock*, 8> NewBB2Preds;
+  for (pred_iterator i = pred_begin(OrigBB), e = pred_end(OrigBB);
+       i != e; ) {
+    BasicBlock *Pred = *i++;
+    if (Pred == NewBB1) continue;
+    assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
+           "Cannot split an edge from an IndirectBrInst");
+    NewBB2Preds.push_back(Pred);
+    e = pred_end(OrigBB);
+  }
+
+  BasicBlock *NewBB2 = 0;
+  if (!NewBB2Preds.empty()) {
+    // Create another basic block for the rest of OrigBB's predecessors.
+    NewBB2 = BasicBlock::Create(OrigBB->getContext(),
+                                OrigBB->getName() + Suffix2,
+                                OrigBB->getParent(), OrigBB);
+    NewBBs.push_back(NewBB2);
+
+    // The new block unconditionally branches to the old block.
+    BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
+
+    // Move the remaining edges from OrigBB to point to NewBB2.
+    for (SmallVectorImpl<BasicBlock*>::iterator
+           i = NewBB2Preds.begin(), e = NewBB2Preds.end(); i != e; ++i)
+      (*i)->getTerminator()->replaceUsesOfWith(OrigBB, NewBB2);
+
+    // Update DominatorTree, LoopInfo, and LCCSA analysis information.
+    HasLoopExit = false;
+    UpdateAnalysisInformation(OrigBB, NewBB2, NewBB2Preds, P, HasLoopExit);
+
+    // Update the PHI nodes in OrigBB with the values coming from NewBB2.
+    UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, P, HasLoopExit);
+  }
+
+  LandingPadInst *LPad = OrigBB->getLandingPadInst();
+  Instruction *Clone1 = LPad->clone();
+  Clone1->setName(Twine("lpad") + Suffix1);
+  NewBB1->getInstList().insert(NewBB1->getFirstInsertionPt(), Clone1);
+
+  if (NewBB2) {
+    Instruction *Clone2 = LPad->clone();
+    Clone2->setName(Twine("lpad") + Suffix2);
+    NewBB2->getInstList().insert(NewBB2->getFirstInsertionPt(), Clone2);
+
+    // Create a PHI node for the two cloned landingpad instructions.
+    PHINode *PN = PHINode::Create(LPad->getType(), 2, "lpad.phi", LPad);
+    PN->addIncoming(Clone1, NewBB1);
+    PN->addIncoming(Clone2, NewBB2);
+    LPad->replaceAllUsesWith(PN);
+    LPad->eraseFromParent();
+  } else {
+    // There is no second clone. Just replace the landing pad with the first
+    // clone.
+    LPad->replaceAllUsesWith(Clone1);
+    LPad->eraseFromParent();
+  }
+}
+
 /// FindFunctionBackedges - Analyze the specified function to find all of the
 /// loop backedges in the function and return them.  This is a relatively cheap
 /// (compared to computing dominators and loop info) analysis.
diff --git a/contrib/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp b/contrib/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
index 92ce50030a5d..c05291088157 100644
--- a/contrib/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
@@ -102,7 +102,7 @@ bool llvm::isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum,
   ++I;        // Skip one edge due to the incoming arc from TI.
   if (!AllowIdenticalEdges)
     return I != E;
-  
+
   // If AllowIdenticalEdges is true, then we allow this edge to be considered
   // non-critical iff all preds come from TI's block.
   while (I != E) {
@@ -155,10 +155,10 @@ static void CreatePHIsForSplitLoopExit(SmallVectorImpl<BasicBlock *> &Preds,
 /// This returns the new block if the edge was split, null otherwise.
 ///
 /// If MergeIdenticalEdges is true (not the default), *all* edges from TI to the
-/// specified successor will be merged into the same critical edge block.  
-/// This is most commonly interesting with switch instructions, which may 
+/// specified successor will be merged into the same critical edge block.
+/// This is most commonly interesting with switch instructions, which may
 /// have many edges to any one destination.  This ensures that all edges to that
-/// dest go to one block instead of each going to a different block, but isn't 
+/// dest go to one block instead of each going to a different block, but isn't
 /// the standard definition of a "critical edge".
 ///
 /// It is invalid to call this function on a critical edge that starts at an
@@ -167,15 +167,20 @@ static void CreatePHIsForSplitLoopExit(SmallVectorImpl<BasicBlock *> &Preds,
 /// to.
 ///
 BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
-                                    Pass *P, bool MergeIdenticalEdges) {
+                                    Pass *P, bool MergeIdenticalEdges,
+                                    bool DontDeleteUselessPhis) {
   if (!isCriticalEdge(TI, SuccNum, MergeIdenticalEdges)) return 0;
-  
+
   assert(!isa<IndirectBrInst>(TI) &&
          "Cannot split critical edge from IndirectBrInst");
-  
+
   BasicBlock *TIBB = TI->getParent();
   BasicBlock *DestBB = TI->getSuccessor(SuccNum);
 
+  // Splitting the critical edge to a landing pad block is non-trivial. Don't do
+  // it in this generic function.
+  if (DestBB->isLandingPad()) return 0;
+
   // Create a new basic block, linking it into the CFG.
   BasicBlock *NewBB = BasicBlock::Create(TI->getContext(),
                       TIBB->getName() + "." + DestBB->getName() + "_crit_edge");
@@ -190,7 +195,7 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
   Function &F = *TIBB->getParent();
   Function::iterator FBBI = TIBB;
   F.getBasicBlockList().insert(++FBBI, NewBB);
-  
+
   // If there are any PHI nodes in DestBB, we need to update them so that they
   // merge incoming values from NewBB instead of from TIBB.
   {
@@ -207,35 +212,35 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
       // happens because the BB list of PHI nodes are usually in the same
       // order.
       if (PN->getIncomingBlock(BBIdx) != TIBB)
-	BBIdx = PN->getBasicBlockIndex(TIBB);
+        BBIdx = PN->getBasicBlockIndex(TIBB);
       PN->setIncomingBlock(BBIdx, NewBB);
     }
   }
-   
+
   // If there are any other edges from TIBB to DestBB, update those to go
   // through the split block, making those edges non-critical as well (and
   // reducing the number of phi entries in the DestBB if relevant).
   if (MergeIdenticalEdges) {
     for (unsigned i = SuccNum+1, e = TI->getNumSuccessors(); i != e; ++i) {
       if (TI->getSuccessor(i) != DestBB) continue;
-      
+
       // Remove an entry for TIBB from DestBB phi nodes.
-      DestBB->removePredecessor(TIBB);
-      
+      DestBB->removePredecessor(TIBB, DontDeleteUselessPhis);
+
       // We found another edge to DestBB, go to NewBB instead.
       TI->setSuccessor(i, NewBB);
     }
   }
-  
-  
+
+
 
   // If we don't have a pass object, we can't update anything...
   if (P == 0) return NewBB;
-  
+
   DominatorTree *DT = P->getAnalysisIfAvailable<DominatorTree>();
   LoopInfo *LI = P->getAnalysisIfAvailable<LoopInfo>();
   ProfileInfo *PI = P->getAnalysisIfAvailable<ProfileInfo>();
-  
+
   // If we have nothing to update, just return.
   if (DT == 0 && LI == 0 && PI == 0)
     return NewBB;
@@ -263,7 +268,7 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
   }
 
   bool NewBBDominatesDestBB = true;
-  
+
   // Should we update DominatorTree information?
   if (DT) {
     DomTreeNode *TINode = DT->getNode(TIBB);
@@ -274,7 +279,7 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
     if (TINode) {       // Don't break unreachable code!
       DomTreeNode *NewBBNode = DT->addNewBlock(NewBB, TIBB);
       DomTreeNode *DestBBNode = 0;
-     
+
       // If NewBBDominatesDestBB hasn't been computed yet, do so with DT.
       if (!OtherPreds.empty()) {
         DestBBNode = DT->getNode(DestBB);
@@ -285,7 +290,7 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
         }
         OtherPreds.clear();
       }
-      
+
       // If NewBBDominatesDestBB, then NewBB dominates DestBB, otherwise it
       // doesn't dominate anything.
       if (NewBBDominatesDestBB) {
@@ -337,6 +342,8 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
         }
 
         // For each unique exit block...
+        // FIXME: This code is functionally equivalent to the corresponding
+        // loop in LoopSimplify.
         SmallVector<BasicBlock *, 4> ExitBlocks;
         TIL->getExitBlocks(ExitBlocks);
         for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
@@ -348,10 +355,15 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
           for (pred_iterator I = pred_begin(Exit), E = pred_end(Exit);
                I != E; ++I) {
             BasicBlock *P = *I;
-            if (TIL->contains(P))
+            if (TIL->contains(P)) {
+              if (isa<IndirectBrInst>(P->getTerminator())) {
+                Preds.clear();
+                break;
+              }
               Preds.push_back(P);
-            else
+            } else {
               HasPredOutsideOfLoop = true;
+            }
           }
           // If there are any preds not in the loop, we'll need to split
           // the edges. The Preds.empty() check is needed because a block
diff --git a/contrib/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/contrib/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 14bb17fbda3f..4b5f45b31f9b 100644
--- a/contrib/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -58,8 +58,8 @@ Value *llvm::EmitStrChr(Value *Ptr, char C, IRBuilder<> &B,
   AttributeWithIndex AWI =
     AttributeWithIndex::get(~0u, Attribute::ReadOnly | Attribute::NoUnwind);
 
-  const Type *I8Ptr = B.getInt8PtrTy();
-  const Type *I32Ty = B.getInt32Ty();
+  Type *I8Ptr = B.getInt8PtrTy();
+  Type *I32Ty = B.getInt32Ty();
   Constant *StrChr = M->getOrInsertFunction("strchr", AttrListPtr::get(&AWI, 1),
                                             I8Ptr, I8Ptr, I32Ty, NULL);
   CallInst *CI = B.CreateCall2(StrChr, CastToCStr(Ptr, B),
@@ -102,7 +102,7 @@ Value *llvm::EmitStrCpy(Value *Dst, Value *Src, IRBuilder<> &B,
   AttributeWithIndex AWI[2];
   AWI[0] = AttributeWithIndex::get(2, Attribute::NoCapture);
   AWI[1] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
-  const Type *I8Ptr = B.getInt8PtrTy();
+  Type *I8Ptr = B.getInt8PtrTy();
   Value *StrCpy = M->getOrInsertFunction(Name, AttrListPtr::get(AWI, 2),
                                          I8Ptr, I8Ptr, I8Ptr, NULL);
   CallInst *CI = B.CreateCall2(StrCpy, CastToCStr(Dst, B), CastToCStr(Src, B),
@@ -120,7 +120,7 @@ Value *llvm::EmitStrNCpy(Value *Dst, Value *Src, Value *Len,
   AttributeWithIndex AWI[2];
   AWI[0] = AttributeWithIndex::get(2, Attribute::NoCapture);
   AWI[1] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
-  const Type *I8Ptr = B.getInt8PtrTy();
+  Type *I8Ptr = B.getInt8PtrTy();
   Value *StrNCpy = M->getOrInsertFunction(Name, AttrListPtr::get(AWI, 2),
                                           I8Ptr, I8Ptr, I8Ptr,
                                           Len->getType(), NULL);
@@ -361,7 +361,7 @@ bool SimplifyFortifiedLibCalls::fold(CallInst *CI, const TargetData *TD) {
   this->CI = CI;
   Function *Callee = CI->getCalledFunction();
   StringRef Name = Callee->getName();
-  const FunctionType *FT = Callee->getFunctionType();
+  FunctionType *FT = Callee->getFunctionType();
   LLVMContext &Context = CI->getParent()->getContext();
   IRBuilder<> B(CI);
 
diff --git a/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp b/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 6ea831f5345b..cf21f1ed9703 100644
--- a/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -331,15 +331,10 @@ ConstantFoldMappedInstruction(const Instruction *I) {
                                            TD);
 
   if (const LoadInst *LI = dyn_cast<LoadInst>(I))
-    if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Ops[0]))
-      if (!LI->isVolatile() && CE->getOpcode() == Instruction::GetElementPtr)
-        if (GlobalVariable *GV = dyn_cast<GlobalVariable>(CE->getOperand(0)))
-          if (GV->isConstant() && GV->hasDefinitiveInitializer())
-            return ConstantFoldLoadThroughGEPConstantExpr(GV->getInitializer(),
-                                                          CE);
+    if (!LI->isVolatile())
+      return ConstantFoldLoadFromConstPtr(Ops[0], TD);
 
-  return ConstantFoldInstOperands(I->getOpcode(), I->getType(), &Ops[0],
-                                  Ops.size(), TD);
+  return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Ops, TD);
 }
 
 /// CloneAndPruneFunctionInto - This works exactly like CloneFunctionInto,
diff --git a/contrib/llvm/lib/Transforms/Utils/CloneModule.cpp b/contrib/llvm/lib/Transforms/Utils/CloneModule.cpp
index a08fa35065cc..a0e027b5f12e 100644
--- a/contrib/llvm/lib/Transforms/Utils/CloneModule.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/CloneModule.cpp
@@ -50,10 +50,12 @@ Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
        I != E; ++I) {
     GlobalVariable *GV = new GlobalVariable(*New, 
                                             I->getType()->getElementType(),
-                                            false,
-                                            GlobalValue::ExternalLinkage, 0,
-                                            I->getName());
-    GV->setAlignment(I->getAlignment());
+                                            I->isConstant(), I->getLinkage(),
+                                            (Constant*) 0, I->getName(),
+                                            (GlobalVariable*) 0,
+                                            I->isThreadLocal(),
+                                            I->getType()->getAddressSpace());
+    GV->copyAttributesFrom(I);
     VMap[I] = GV;
   }
 
@@ -61,16 +63,19 @@ Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
   for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
     Function *NF =
       Function::Create(cast<FunctionType>(I->getType()->getElementType()),
-                       GlobalValue::ExternalLinkage, I->getName(), New);
+                       I->getLinkage(), I->getName(), New);
     NF->copyAttributesFrom(I);
     VMap[I] = NF;
   }
 
   // Loop over the aliases in the module
   for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
-       I != E; ++I)
-    VMap[I] = new GlobalAlias(I->getType(), GlobalAlias::ExternalLinkage,
-                                  I->getName(), NULL, New);
+       I != E; ++I) {
+    GlobalAlias *GA = new GlobalAlias(I->getType(), I->getLinkage(),
+                                      I->getName(), NULL, New);
+    GA->copyAttributesFrom(I);
+    VMap[I] = GA;
+  }
   
   // Now that all of the things that global variable initializer can refer to
   // have been created, loop through and copy the global variable referrers
@@ -81,9 +86,6 @@ Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
     GlobalVariable *GV = cast<GlobalVariable>(VMap[I]);
     if (I->hasInitializer())
       GV->setInitializer(MapValue(I->getInitializer(), VMap));
-    GV->setLinkage(I->getLinkage());
-    GV->setThreadLocal(I->isThreadLocal());
-    GV->setConstant(I->isConstant());
   }
 
   // Similarly, copy over function bodies now...
@@ -101,15 +103,12 @@ Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
       SmallVector<ReturnInst*, 8> Returns;  // Ignore returns cloned.
       CloneFunctionInto(F, I, VMap, /*ModuleLevelChanges=*/true, Returns);
     }
-
-    F->setLinkage(I->getLinkage());
   }
 
   // And aliases
   for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
        I != E; ++I) {
     GlobalAlias *GA = cast<GlobalAlias>(VMap[I]);
-    GA->setLinkage(I->getLinkage());
     if (const Constant *C = I->getAliasee())
       GA->setAliasee(MapValue(C, VMap));
   }
diff --git a/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 081352358b95..5f47ebb78202 100644
--- a/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -50,14 +50,14 @@ namespace {
     DominatorTree* DT;
     bool AggregateArgs;
     unsigned NumExitBlocks;
-    const Type *RetTy;
+    Type *RetTy;
   public:
     CodeExtractor(DominatorTree* dt = 0, bool AggArgs = false)
       : DT(dt), AggregateArgs(AggArgs||AggregateArgsOpt), NumExitBlocks(~0U) {}
 
-    Function *ExtractCodeRegion(const std::vector<BasicBlock*> &code);
+    Function *ExtractCodeRegion(ArrayRef<BasicBlock*> code);
 
-    bool isEligible(const std::vector<BasicBlock*> &code);
+    bool isEligible(ArrayRef<BasicBlock*> code);
 
   private:
     /// definedInRegion - Return true if the specified value is defined in the
@@ -290,7 +290,7 @@ Function *CodeExtractor::constructFunction(const Values &inputs,
     paramTy.clear();
     paramTy.push_back(StructPtr);
   }
-  const FunctionType *funcType =
+  FunctionType *funcType =
                   FunctionType::get(RetTy, paramTy, false);
 
   // Create the new function
@@ -317,8 +317,7 @@ Function *CodeExtractor::constructFunction(const Values &inputs,
       Idx[1] = ConstantInt::get(Type::getInt32Ty(header->getContext()), i);
       TerminatorInst *TI = newFunction->begin()->getTerminator();
       GetElementPtrInst *GEP = 
-        GetElementPtrInst::Create(AI, Idx, Idx+2, 
-                                  "gep_" + inputs[i]->getName(), TI);
+        GetElementPtrInst::Create(AI, Idx, "gep_" + inputs[i]->getName(), TI);
       RewriteVal = new LoadInst(GEP, "loadgep_" + inputs[i]->getName(), TI);
     } else
       RewriteVal = AI++;
@@ -420,7 +419,7 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
       Idx[0] = Constant::getNullValue(Type::getInt32Ty(Context));
       Idx[1] = ConstantInt::get(Type::getInt32Ty(Context), i);
       GetElementPtrInst *GEP =
-        GetElementPtrInst::Create(Struct, Idx, Idx + 2,
+        GetElementPtrInst::Create(Struct, Idx,
                                   "gep_" + StructValues[i]->getName());
       codeReplacer->getInstList().push_back(GEP);
       StoreInst *SI = new StoreInst(StructValues[i], GEP);
@@ -446,7 +445,7 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
       Idx[0] = Constant::getNullValue(Type::getInt32Ty(Context));
       Idx[1] = ConstantInt::get(Type::getInt32Ty(Context), FirstOut + i);
       GetElementPtrInst *GEP
-        = GetElementPtrInst::Create(Struct, Idx, Idx + 2,
+        = GetElementPtrInst::Create(Struct, Idx,
                                     "gep_reload_" + outputs[i]->getName());
       codeReplacer->getInstList().push_back(GEP);
       Output = GEP;
@@ -561,7 +560,7 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
                 Idx[1] = ConstantInt::get(Type::getInt32Ty(Context),
                                           FirstOut+out);
                 GetElementPtrInst *GEP =
-                  GetElementPtrInst::Create(OAI, Idx, Idx + 2,
+                  GetElementPtrInst::Create(OAI, Idx,
                                             "gep_" + outputs[out]->getName(),
                                             NTRet);
                 new StoreInst(outputs[out], GEP, NTRet);
@@ -580,7 +579,7 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
   }
 
   // Now that we've done the deed, simplify the switch instruction.
-  const Type *OldFnRetTy = TheSwitch->getParent()->getParent()->getReturnType();
+  Type *OldFnRetTy = TheSwitch->getParent()->getParent()->getReturnType();
   switch (NumExitBlocks) {
   case 0:
     // There are no successors (the block containing the switch itself), which
@@ -655,7 +654,7 @@ void CodeExtractor::moveCodeToFunction(Function *newFunction) {
 /// computed result back into memory.
 ///
 Function *CodeExtractor::
-ExtractCodeRegion(const std::vector<BasicBlock*> &code) {
+ExtractCodeRegion(ArrayRef<BasicBlock*> code) {
   if (!isEligible(code))
     return 0;
 
@@ -755,9 +754,13 @@ ExtractCodeRegion(const std::vector<BasicBlock*> &code) {
   return newFunction;
 }
 
-bool CodeExtractor::isEligible(const std::vector<BasicBlock*> &code) {
+bool CodeExtractor::isEligible(ArrayRef<BasicBlock*> code) {
+  // Deny a single basic block that's a landing pad block.
+  if (code.size() == 1 && code[0]->isLandingPad())
+    return false;
+
   // Deny code region if it contains allocas or vastarts.
-  for (std::vector<BasicBlock*>::const_iterator BB = code.begin(), e=code.end();
+  for (ArrayRef<BasicBlock*>::iterator BB = code.begin(), e=code.end();
        BB != e; ++BB)
     for (BasicBlock::const_iterator I = (*BB)->begin(), Ie = (*BB)->end();
          I != Ie; ++I)
@@ -771,25 +774,23 @@ bool CodeExtractor::isEligible(const std::vector<BasicBlock*> &code) {
 }
 
 
-/// ExtractCodeRegion - slurp a sequence of basic blocks into a brand new
-/// function
+/// ExtractCodeRegion - Slurp a sequence of basic blocks into a brand new
+/// function.
 ///
 Function* llvm::ExtractCodeRegion(DominatorTree &DT,
-                                  const std::vector<BasicBlock*> &code,
+                                  ArrayRef<BasicBlock*> code,
                                   bool AggregateArgs) {
   return CodeExtractor(&DT, AggregateArgs).ExtractCodeRegion(code);
 }
 
-/// ExtractBasicBlock - slurp a natural loop into a brand new function
+/// ExtractLoop - Slurp a natural loop into a brand new function.
 ///
 Function* llvm::ExtractLoop(DominatorTree &DT, Loop *L, bool AggregateArgs) {
   return CodeExtractor(&DT, AggregateArgs).ExtractCodeRegion(L->getBlocks());
 }
 
-/// ExtractBasicBlock - slurp a basic block into a brand new function
+/// ExtractBasicBlock - Slurp a basic block into a brand new function.
 ///
-Function* llvm::ExtractBasicBlock(BasicBlock *BB, bool AggregateArgs) {
-  std::vector<BasicBlock*> Blocks;
-  Blocks.push_back(BB);
-  return CodeExtractor(0, AggregateArgs).ExtractCodeRegion(Blocks);
+Function* llvm::ExtractBasicBlock(ArrayRef<BasicBlock*> BBs, bool AggregateArgs){
+  return CodeExtractor(0, AggregateArgs).ExtractCodeRegion(BBs);
 }
diff --git a/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp b/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
index d5b382e55e5c..5464dbc4a830 100644
--- a/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -45,6 +45,9 @@ bool llvm::InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI) {
   return InlineFunction(CallSite(II), IFI);
 }
 
+// FIXME: New EH - Remove the functions marked [LIBUNWIND] when new EH is
+// turned on.
+
 /// [LIBUNWIND] Look for an llvm.eh.exception call in the given block.
 static EHExceptionInst *findExceptionInBlock(BasicBlock *bb) {
   for (BasicBlock::iterator i = bb->begin(), e = bb->end(); i != e; i++) {
@@ -250,20 +253,32 @@ namespace {
     PHINode *InnerSelectorPHI;
     SmallVector<Value*, 8> UnwindDestPHIValues;
 
-  public:
-    InvokeInliningInfo(InvokeInst *II) :
-      OuterUnwindDest(II->getUnwindDest()), OuterSelector(0),
-      InnerUnwindDest(0), InnerExceptionPHI(0), InnerSelectorPHI(0) {
+    // FIXME: New EH - These will replace the analogous ones above.
+    BasicBlock *OuterResumeDest; //< Destination of the invoke's unwind.
+    BasicBlock *InnerResumeDest; //< Destination for the callee's resume.
+    LandingPadInst *CallerLPad;  //< LandingPadInst associated with the invoke.
+    PHINode *InnerEHValuesPHI;   //< PHI for EH values from landingpad insts.
 
-      // If there are PHI nodes in the unwind destination block, we
-      // need to keep track of which values came into them from the
-      // invoke before removing the edge from this block.
-      llvm::BasicBlock *invokeBB = II->getParent();
-      for (BasicBlock::iterator I = OuterUnwindDest->begin();
-             isa<PHINode>(I); ++I) {
+  public:
+    InvokeInliningInfo(InvokeInst *II)
+      : OuterUnwindDest(II->getUnwindDest()), OuterSelector(0),
+        InnerUnwindDest(0), InnerExceptionPHI(0), InnerSelectorPHI(0),
+        OuterResumeDest(II->getUnwindDest()), InnerResumeDest(0),
+        CallerLPad(0), InnerEHValuesPHI(0) {
+      // If there are PHI nodes in the unwind destination block, we need to keep
+      // track of which values came into them from the invoke before removing
+      // the edge from this block.
+      llvm::BasicBlock *InvokeBB = II->getParent();
+      BasicBlock::iterator I = OuterUnwindDest->begin();
+      for (; isa<PHINode>(I); ++I) {
         // Save the value to use for this edge.
-        PHINode *phi = cast<PHINode>(I);
-        UnwindDestPHIValues.push_back(phi->getIncomingValueForBlock(invokeBB));
+        PHINode *PHI = cast<PHINode>(I);
+        UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
+      }
+
+      // FIXME: With the new EH, this if/dyn_cast should be a 'cast'.
+      if (LandingPadInst *LPI = dyn_cast<LandingPadInst>(I)) {
+        CallerLPad = LPI;
       }
     }
 
@@ -281,11 +296,23 @@ namespace {
 
     BasicBlock *getInnerUnwindDest();
 
+    // FIXME: New EH - Rename when new EH is turned on.
+    BasicBlock *getInnerUnwindDestNewEH();
+
+    LandingPadInst *getLandingPadInst() const { return CallerLPad; }
+
     bool forwardEHResume(CallInst *call, BasicBlock *src);
 
-    /// Add incoming-PHI values to the unwind destination block for
-    /// the given basic block, using the values for the original
-    /// invoke's source block.
+    /// forwardResume - Forward the 'resume' instruction to the caller's landing
+    /// pad block. When the landing pad block has only one predecessor, this is
+    /// a simple branch. When there is more than one predecessor, we need to
+    /// split the landing pad block after the landingpad instruction and jump
+    /// to there.
+    void forwardResume(ResumeInst *RI);
+
+    /// addIncomingPHIValuesFor - Add incoming-PHI values to the unwind
+    /// destination block for the given basic block, using the values for the
+    /// original invoke's source block.
     void addIncomingPHIValuesFor(BasicBlock *BB) const {
       addIncomingPHIValuesForInto(BB, OuterUnwindDest);
     }
@@ -300,7 +327,7 @@ namespace {
   };
 }
 
-/// Get or create a target for the branch out of rewritten calls to
+/// [LIBUNWIND] Get or create a target for the branch out of rewritten calls to
 /// llvm.eh.resume.
 BasicBlock *InvokeInliningInfo::getInnerUnwindDest() {
   if (InnerUnwindDest) return InnerUnwindDest;
@@ -404,6 +431,60 @@ bool InvokeInliningInfo::forwardEHResume(CallInst *call, BasicBlock *src) {
   return true;
 }
 
+/// Get or create a target for the branch from ResumeInsts.
+BasicBlock *InvokeInliningInfo::getInnerUnwindDestNewEH() {
+  // FIXME: New EH - rename this function when new EH is turned on.
+  if (InnerResumeDest) return InnerResumeDest;
+
+  // Split the landing pad.
+  BasicBlock::iterator SplitPoint = CallerLPad; ++SplitPoint;
+  InnerResumeDest =
+    OuterResumeDest->splitBasicBlock(SplitPoint,
+                                     OuterResumeDest->getName() + ".body");
+
+  // The number of incoming edges we expect to the inner landing pad.
+  const unsigned PHICapacity = 2;
+
+  // Create corresponding new PHIs for all the PHIs in the outer landing pad.
+  BasicBlock::iterator InsertPoint = InnerResumeDest->begin();
+  BasicBlock::iterator I = OuterResumeDest->begin();
+  for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
+    PHINode *OuterPHI = cast<PHINode>(I);
+    PHINode *InnerPHI = PHINode::Create(OuterPHI->getType(), PHICapacity,
+                                        OuterPHI->getName() + ".lpad-body",
+                                        InsertPoint);
+    OuterPHI->replaceAllUsesWith(InnerPHI);
+    InnerPHI->addIncoming(OuterPHI, OuterResumeDest);
+  }
+
+  // Create a PHI for the exception values.
+  InnerEHValuesPHI = PHINode::Create(CallerLPad->getType(), PHICapacity,
+                                     "eh.lpad-body", InsertPoint);
+  CallerLPad->replaceAllUsesWith(InnerEHValuesPHI);
+  InnerEHValuesPHI->addIncoming(CallerLPad, OuterResumeDest);
+
+  // All done.
+  return InnerResumeDest;
+}
+
+/// forwardResume - Forward the 'resume' instruction to the caller's landing pad
+/// block. When the landing pad block has only one predecessor, this is a simple
+/// branch. When there is more than one predecessor, we need to split the
+/// landing pad block after the landingpad instruction and jump to there.
+void InvokeInliningInfo::forwardResume(ResumeInst *RI) {
+  BasicBlock *Dest = getInnerUnwindDestNewEH();
+  BasicBlock *Src = RI->getParent();
+
+  BranchInst::Create(Dest, Src);
+
+  // Update the PHIs in the destination. They were inserted in an order which
+  // makes this work.
+  addIncomingPHIValuesForInto(Src, Dest);
+
+  InnerEHValuesPHI->addIncoming(RI->getOperand(0), Src);
+  RI->eraseFromParent();
+}
+
 /// [LIBUNWIND] Check whether this selector is "only cleanups":
 ///   call i32 @llvm.eh.selector(blah, blah, i32 0)
 static bool isCleanupOnlySelector(EHSelectorInst *selector) {
@@ -421,9 +502,19 @@ static bool isCleanupOnlySelector(EHSelectorInst *selector) {
 /// Returns true to indicate that the next block should be skipped.
 static bool HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB,
                                                    InvokeInliningInfo &Invoke) {
+  LandingPadInst *LPI = Invoke.getLandingPadInst();
+
   for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E; ) {
     Instruction *I = BBI++;
-    
+
+    if (LPI) // FIXME: New EH - This won't be NULL in the new EH.
+      if (LandingPadInst *L = dyn_cast<LandingPadInst>(I)) {
+        unsigned NumClauses = LPI->getNumClauses();
+        L->reserveClauses(NumClauses);
+        for (unsigned i = 0; i != NumClauses; ++i)
+          L->addClause(LPI->getClause(i));
+      }
+
     // We only need to check for function calls: inlined invoke
     // instructions require no special handling.
     CallInst *CI = dyn_cast<CallInst>(I);
@@ -557,6 +648,10 @@ static void HandleInlinedInvoke(InvokeInst *II, BasicBlock *FirstNewBlock,
       // there is now a new entry in them.
       Invoke.addIncomingPHIValuesFor(BB);
     }
+
+    if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator())) {
+      Invoke.forwardResume(RI);
+    }
   }
 
   // Now that everything is happy, we have one final detail.  The PHI nodes in
@@ -636,7 +731,7 @@ static Value *HandleByValArgument(Value *Arg, Instruction *TheCall,
                                   const Function *CalledFunc,
                                   InlineFunctionInfo &IFI,
                                   unsigned ByValAlignment) {
-  const Type *AggTy = cast<PointerType>(Arg->getType())->getElementType();
+  Type *AggTy = cast<PointerType>(Arg->getType())->getElementType();
 
   // If the called function is readonly, then it could not mutate the caller's
   // copy of the byval'd memory.  In this case, it is safe to elide the copy and
@@ -726,7 +821,7 @@ static bool isUsedByLifetimeMarker(Value *V) {
 // hasLifetimeMarkers - Check whether the given alloca already has
 // lifetime.start or lifetime.end intrinsics.
 static bool hasLifetimeMarkers(AllocaInst *AI) {
-  const Type *Int8PtrTy = Type::getInt8PtrTy(AI->getType()->getContext());
+  Type *Int8PtrTy = Type::getInt8PtrTy(AI->getType()->getContext());
   if (AI->getType() == Int8PtrTy)
     return isUsedByLifetimeMarker(AI);
 
@@ -770,8 +865,15 @@ static void fixupLineNumbers(Function *Fn, Function::iterator FI,
     for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
          BI != BE; ++BI) {
       DebugLoc DL = BI->getDebugLoc();
-      if (!DL.isUnknown())
+      if (!DL.isUnknown()) {
         BI->setDebugLoc(updateInlinedAtInfo(DL, TheCallDL, BI->getContext()));
+        if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(BI)) {
+          LLVMContext &Ctx = BI->getContext();
+          MDNode *InlinedAt = BI->getDebugLoc().getInlinedAt(Ctx);
+          DVI->setOperand(2, createInlinedVariable(DVI->getVariable(), 
+                                                   InlinedAt, Ctx));
+        }
+      }
     }
   }
 }
@@ -822,6 +924,40 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI) {
       return false;
   }
 
+  // Find the personality function used by the landing pads of the caller. If it
+  // exists, then check to see that it matches the personality function used in
+  // the callee.
+  for (Function::const_iterator
+         I = Caller->begin(), E = Caller->end(); I != E; ++I)
+    if (const InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator())) {
+      const BasicBlock *BB = II->getUnwindDest();
+      // FIXME: This 'isa' here should become go away once the new EH system is
+      // in place.
+      if (!isa<LandingPadInst>(BB->getFirstNonPHI()))
+        continue;
+      const LandingPadInst *LP = cast<LandingPadInst>(BB->getFirstNonPHI());
+      const Value *CallerPersFn = LP->getPersonalityFn();
+
+      // If the personality functions match, then we can perform the
+      // inlining. Otherwise, we can't inline.
+      // TODO: This isn't 100% true. Some personality functions are proper
+      //       supersets of others and can be used in place of the other.
+      for (Function::const_iterator
+             I = CalledFunc->begin(), E = CalledFunc->end(); I != E; ++I)
+        if (const InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator())) {
+          const BasicBlock *BB = II->getUnwindDest();
+          // FIXME: This 'if/dyn_cast' here should become a normal 'cast' once
+          // the new EH system is in place.
+          if (const LandingPadInst *LP =
+              dyn_cast<LandingPadInst>(BB->getFirstNonPHI()))
+            if (CallerPersFn != LP->getPersonalityFn())
+              return false;
+          break;
+        }
+
+      break;
+    }
+
   // Get an iterator to the last basic block in the function, which will have
   // the new function inlined after it.
   //
@@ -1090,7 +1226,7 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI) {
 
   // Handle all of the return instructions that we just cloned in, and eliminate
   // any users of the original call/invoke instruction.
-  const Type *RTy = CalledFunc->getReturnType();
+  Type *RTy = CalledFunc->getReturnType();
 
   PHINode *PHI = 0;
   if (Returns.size() > 1) {
diff --git a/contrib/llvm/lib/Transforms/Utils/Local.cpp b/contrib/llvm/lib/Transforms/Utils/Local.cpp
index 0f6d9ae99d66..7034feb227ad 100644
--- a/contrib/llvm/lib/Transforms/Utils/Local.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/Local.cpp
@@ -27,7 +27,6 @@
 #include "llvm/Analysis/DebugInfo.h"
 #include "llvm/Analysis/DIBuilder.h"
 #include "llvm/Analysis/Dominators.h"
-#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/ProfileInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -227,13 +226,17 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
 bool llvm::isInstructionTriviallyDead(Instruction *I) {
   if (!I->use_empty() || isa<TerminatorInst>(I)) return false;
 
+  // We don't want the landingpad instruction removed by anything this general.
+  if (isa<LandingPadInst>(I))
+    return false;
+
   // We don't want debug info removed by anything this general, unless
   // debug info is empty.
   if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
-    if (DDI->getAddress()) 
+    if (DDI->getAddress())
       return false;
     return true;
-  } 
+  }
   if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
     if (DVI->getValue())
       return false;
@@ -244,10 +247,16 @@ bool llvm::isInstructionTriviallyDead(Instruction *I) {
 
   // Special case intrinsics that "may have side effects" but can be deleted
   // when dead.
-  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
     // Safe to delete llvm.stacksave if dead.
     if (II->getIntrinsicID() == Intrinsic::stacksave)
       return true;
+
+    // Lifetime intrinsics are dead when their right-hand is undef.
+    if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+        II->getIntrinsicID() == Intrinsic::lifetime_end)
+      return isa<UndefValue>(II->getArgOperand(1));
+  }
   return false;
 }
 
@@ -712,10 +721,14 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
 /// their preferred alignment from the beginning.
 ///
 static unsigned enforceKnownAlignment(Value *V, unsigned Align,
-                                      unsigned PrefAlign) {
+                                      unsigned PrefAlign, const TargetData *TD) {
   V = V->stripPointerCasts();
 
   if (AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
+    // If the preferred alignment is greater than the natural stack alignment
+    // then don't round up. This avoids dynamic stack realignment.
+    if (TD && TD->exceedsNaturalStackAlignment(PrefAlign))
+      return Align;
     // If there is a requested alignment and if this is an alloca, round up.
     if (AI->getAlignment() >= PrefAlign)
       return AI->getAlignment();
@@ -766,7 +779,7 @@ unsigned llvm::getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign,
   Align = std::min(Align, +Value::MaximumAlignment);
   
   if (PrefAlign > Align)
-    Align = enforceKnownAlignment(V, Align, PrefAlign);
+    Align = enforceKnownAlignment(V, Align, PrefAlign, TD);
     
   // We don't need to make any adjustment.
   return Align;
diff --git a/contrib/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/contrib/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index e79fb5ac21b4..cbd54a8dcbf8 100644
--- a/contrib/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -213,7 +213,7 @@ bool LoopSimplify::ProcessLoop(Loop *L, LPPassManager &LPM) {
   // predecessors from outside of the loop, split the edge now.
   SmallVector<BasicBlock*, 8> ExitBlocks;
   L->getExitBlocks(ExitBlocks);
-    
+
   SmallSetVector<BasicBlock *, 8> ExitBlockSet(ExitBlocks.begin(),
                                                ExitBlocks.end());
   for (SmallSetVector<BasicBlock *, 8>::iterator I = ExitBlockSet.begin(),
@@ -325,6 +325,14 @@ bool LoopSimplify::ProcessLoop(Loop *L, LPPassManager &LPM) {
       DEBUG(dbgs() << "LoopSimplify: Eliminating exiting block "
                    << ExitingBlock->getName() << "\n");
 
+      // If any reachable control flow within this loop has changed, notify
+      // ScalarEvolution. Currently assume the parent loop doesn't change
+      // (spliting edges doesn't count). If blocks, CFG edges, or other values
+      // in the parent loop change, then we need call to forgetLoop() for the
+      // parent instead.
+      if (SE)
+        SE->forgetLoop(L);
+
       assert(pred_begin(ExitingBlock) == pred_end(ExitingBlock));
       Changed = true;
       LI->removeBlock(ExitingBlock);
@@ -402,13 +410,24 @@ BasicBlock *LoopSimplify::RewriteLoopExitBlock(Loop *L, BasicBlock *Exit) {
   }
 
   assert(!LoopBlocks.empty() && "No edges coming in from outside the loop?");
-  BasicBlock *NewBB = SplitBlockPredecessors(Exit, &LoopBlocks[0], 
-                                             LoopBlocks.size(), ".loopexit",
-                                             this);
+  BasicBlock *NewExitBB = 0;
+
+  if (Exit->isLandingPad()) {
+    SmallVector<BasicBlock*, 2> NewBBs;
+    SplitLandingPadPredecessors(Exit, ArrayRef<BasicBlock*>(&LoopBlocks[0],
+                                                            LoopBlocks.size()),
+                                ".loopexit", ".nonloopexit",
+                                this, NewBBs);
+    NewExitBB = NewBBs[0];
+  } else {
+    NewExitBB = SplitBlockPredecessors(Exit, &LoopBlocks[0],
+                                       LoopBlocks.size(), ".loopexit",
+                                       this);
+  }
 
   DEBUG(dbgs() << "LoopSimplify: Creating dedicated exit block "
-               << NewBB->getName() << "\n");
-  return NewBB;
+               << NewExitBB->getName() << "\n");
+  return NewExitBB;
 }
 
 /// AddBlockAndPredsToSet - Add the specified block, and all of its
@@ -467,23 +486,23 @@ void LoopSimplify::PlaceSplitBlockCarefully(BasicBlock *NewBB,
     if (&*BBI == SplitPreds[i])
       return;
   }
-  
+
   // If it isn't already after an outside block, move it after one.  This is
   // always good as it makes the uncond branch from the outside block into a
   // fall-through.
-  
+
   // Figure out *which* outside block to put this after.  Prefer an outside
   // block that neighbors a BB actually in the loop.
   BasicBlock *FoundBB = 0;
   for (unsigned i = 0, e = SplitPreds.size(); i != e; ++i) {
     Function::iterator BBI = SplitPreds[i];
-    if (++BBI != NewBB->getParent()->end() && 
+    if (++BBI != NewBB->getParent()->end() &&
         L->contains(BBI)) {
       FoundBB = SplitPreds[i];
       break;
     }
   }
-  
+
   // If our heuristic for a *good* bb to place this after doesn't find
   // anything, just pick something.  It's likely better than leaving it within
   // the loop.
@@ -544,7 +563,7 @@ Loop *LoopSimplify::SeparateNestedLoop(Loop *L, LPPassManager &LPM) {
   // Make sure that NewBB is put someplace intelligent, which doesn't mess up
   // code layout too horribly.
   PlaceSplitBlockCarefully(NewBB, OuterLoopPreds, L);
-  
+
   // Create the new outer loop.
   Loop *NewOuter = new Loop();
 
@@ -735,6 +754,7 @@ void LoopSimplify::verifyAnalysis() const {
       }
     assert(HasIndBrPred &&
            "LoopSimplify has no excuse for missing loop header info!");
+    (void)HasIndBrPred;
   }
 
   // Indirectbr can interfere with exit block canonicalization.
@@ -742,12 +762,15 @@ void LoopSimplify::verifyAnalysis() const {
     bool HasIndBrExiting = false;
     SmallVector<BasicBlock*, 8> ExitingBlocks;
     L->getExitingBlocks(ExitingBlocks);
-    for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i)
+    for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
       if (isa<IndirectBrInst>((ExitingBlocks[i])->getTerminator())) {
         HasIndBrExiting = true;
         break;
       }
+    }
+
     assert(HasIndBrExiting &&
            "LoopSimplify has no excuse for missing exit block info!");
+    (void)HasIndBrExiting;
   }
 }
diff --git a/contrib/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/contrib/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 6772511b5d5a..62e4fa295378 100644
--- a/contrib/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -11,9 +11,6 @@
 // actual pass or policy, but provides a single function to perform loop
 // unrolling.
 //
-// It works best when loops have been canonicalized by the -indvars pass,
-// allowing it to determine the trip counts of loops easily.
-//
 // The process of unrolling can produce extraneous basic blocks linked with
 // unconditional branches.  This will be corrected in the future.
 //
@@ -24,6 +21,7 @@
 #include "llvm/BasicBlock.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopIterator.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Support/Debug.h"
@@ -31,6 +29,7 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/SimplifyIndVar.h"
 using namespace llvm;
 
 // TODO: Should these be here or in LoopUnroll?
@@ -61,7 +60,8 @@ static inline void RemapInstruction(Instruction *I,
 /// only has one predecessor, and that predecessor only has one successor.
 /// The LoopInfo Analysis that is passed will be kept consistent.
 /// Returns the new combined block.
-static BasicBlock *FoldBlockIntoPredecessor(BasicBlock *BB, LoopInfo* LI) {
+static BasicBlock *FoldBlockIntoPredecessor(BasicBlock *BB, LoopInfo* LI,
+                                            LPPassManager *LPM) {
   // Merge basic blocks into their predecessor if there is only one distinct
   // pred, and if there is only one distinct successor of the predecessor, and
   // if there are no PHI nodes.
@@ -93,6 +93,12 @@ static BasicBlock *FoldBlockIntoPredecessor(BasicBlock *BB, LoopInfo* LI) {
   std::string OldName = BB->getName();
 
   // Erase basic block from the function...
+
+  // ScalarEvolution holds references to loop exit blocks.
+  if (ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>()) {
+    if (Loop *L = LI->getLoopFor(BB))
+      SE->forgetLoop(L);
+  }
   LI->removeBlock(BB);
   BB->eraseFromParent();
 
@@ -109,12 +115,27 @@ static BasicBlock *FoldBlockIntoPredecessor(BasicBlock *BB, LoopInfo* LI) {
 /// branch instruction. However, if the trip count (and multiple) are not known,
 /// loop unrolling will mostly produce more code that is no faster.
 ///
+/// TripCount is generally defined as the number of times the loop header
+/// executes. UnrollLoop relaxes the definition to permit early exits: here
+/// TripCount is the iteration on which control exits LatchBlock if no early
+/// exits were taken. Note that UnrollLoop assumes that the loop counter test
+/// terminates LatchBlock in order to remove unnecesssary instances of the
+/// test. In other words, control may exit the loop prior to TripCount
+/// iterations via an early branch, but control may not exit the loop from the
+/// LatchBlock's terminator prior to TripCount iterations.
+///
+/// Similarly, TripMultiple divides the number of times that the LatchBlock may
+/// execute without exiting the loop.
+///
 /// The LoopInfo Analysis that is passed will be kept consistent.
 ///
 /// If a LoopPassManager is passed in, and the loop is fully removed, it will be
 /// removed from the LoopPassManager as well. LPM can also be NULL.
-bool llvm::UnrollLoop(Loop *L, unsigned Count,
-                      LoopInfo *LI, LPPassManager *LPM) {
+///
+/// This utility preserves LoopInfo. If DominatorTree or ScalarEvolution are
+/// available it must also preserve those analyses.
+bool llvm::UnrollLoop(Loop *L, unsigned Count, unsigned TripCount,
+                      unsigned TripMultiple, LoopInfo *LI, LPPassManager *LPM) {
   BasicBlock *Preheader = L->getLoopPreheader();
   if (!Preheader) {
     DEBUG(dbgs() << "  Can't unroll; loop preheader-insertion failed.\n");
@@ -129,14 +150,14 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
 
   BasicBlock *Header = L->getHeader();
   BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
-  
+
   if (!BI || BI->isUnconditional()) {
     // The loop-rotate pass can be helpful to avoid this in many cases.
     DEBUG(dbgs() <<
              "  Can't unroll; loop not terminated by a conditional branch.\n");
     return false;
   }
-  
+
   if (Header->hasAddressTaken()) {
     // The loop-rotate pass can be helpful to avoid this in many cases.
     DEBUG(dbgs() <<
@@ -146,16 +167,10 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
 
   // Notify ScalarEvolution that the loop will be substantially changed,
   // if not outright eliminated.
-  if (ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>())
+  ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>();
+  if (SE)
     SE->forgetLoop(L);
 
-  // Find trip count
-  unsigned TripCount = L->getSmallConstantTripCount();
-  // Find trip multiple if count is not available
-  unsigned TripMultiple = 1;
-  if (TripCount == 0)
-    TripMultiple = L->getSmallConstantTripMultiple();
-
   if (TripCount != 0)
     DEBUG(dbgs() << "  Trip Count = " << TripCount << "\n");
   if (TripMultiple != 1)
@@ -208,12 +223,7 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
   ValueToValueMapTy LastValueMap;
   std::vector<PHINode*> OrigPHINode;
   for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
-    PHINode *PN = cast<PHINode>(I);
-    OrigPHINode.push_back(PN);
-    if (Instruction *I = 
-                dyn_cast<Instruction>(PN->getIncomingValueForBlock(LatchBlock)))
-      if (L->contains(I))
-        LastValueMap[I] = I;
+    OrigPHINode.push_back(cast<PHINode>(I));
   }
 
   std::vector<BasicBlock*> Headers;
@@ -221,11 +231,20 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
   Headers.push_back(Header);
   Latches.push_back(LatchBlock);
 
+  // The current on-the-fly SSA update requires blocks to be processed in
+  // reverse postorder so that LastValueMap contains the correct value at each
+  // exit.
+  LoopBlocksDFS DFS(L);
+  DFS.perform(LI);
+
+  // Stash the DFS iterators before adding blocks to the loop.
+  LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
+  LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
+
   for (unsigned It = 1; It != Count; ++It) {
     std::vector<BasicBlock*> NewBlocks;
-    
-    for (std::vector<BasicBlock*>::iterator BB = LoopBlocks.begin(),
-         E = LoopBlocks.end(); BB != E; ++BB) {
+
+    for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
       ValueToValueMapTy VMap;
       BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
       Header->getParent()->getBasicBlockList().push_back(New);
@@ -251,75 +270,55 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
 
       L->addBasicBlockToLoop(New, LI->getBase());
 
-      // Add phi entries for newly created values to all exit blocks except
-      // the successor of the latch block.  The successor of the exit block will
-      // be updated specially after unrolling all the way.
-      if (*BB != LatchBlock)
-        for (succ_iterator SI = succ_begin(*BB), SE = succ_end(*BB); SI != SE;
-             ++SI)
-          if (!L->contains(*SI))
-            for (BasicBlock::iterator BBI = (*SI)->begin();
-                 PHINode *phi = dyn_cast<PHINode>(BBI); ++BBI) {
-              Value *Incoming = phi->getIncomingValueForBlock(*BB);
-              phi->addIncoming(Incoming, New);
-            }
-
+      // Add phi entries for newly created values to all exit blocks.
+      for (succ_iterator SI = succ_begin(*BB), SE = succ_end(*BB);
+           SI != SE; ++SI) {
+        if (L->contains(*SI))
+          continue;
+        for (BasicBlock::iterator BBI = (*SI)->begin();
+             PHINode *phi = dyn_cast<PHINode>(BBI); ++BBI) {
+          Value *Incoming = phi->getIncomingValueForBlock(*BB);
+          ValueToValueMapTy::iterator It = LastValueMap.find(Incoming);
+          if (It != LastValueMap.end())
+            Incoming = It->second;
+          phi->addIncoming(Incoming, New);
+        }
+      }
       // Keep track of new headers and latches as we create them, so that
       // we can insert the proper branches later.
       if (*BB == Header)
         Headers.push_back(New);
-      if (*BB == LatchBlock) {
+      if (*BB == LatchBlock)
         Latches.push_back(New);
 
-        // Also, clear out the new latch's back edge so that it doesn't look
-        // like a new loop, so that it's amenable to being merged with adjacent
-        // blocks later on.
-        TerminatorInst *Term = New->getTerminator();
-        assert(L->contains(Term->getSuccessor(!ContinueOnTrue)));
-        assert(Term->getSuccessor(ContinueOnTrue) == LoopExit);
-        Term->setSuccessor(!ContinueOnTrue, NULL);
-      }
-
       NewBlocks.push_back(New);
     }
-    
+
     // Remap all instructions in the most recent iteration
     for (unsigned i = 0; i < NewBlocks.size(); ++i)
       for (BasicBlock::iterator I = NewBlocks[i]->begin(),
            E = NewBlocks[i]->end(); I != E; ++I)
         ::RemapInstruction(I, LastValueMap);
   }
-  
-  // The latch block exits the loop.  If there are any PHI nodes in the
-  // successor blocks, update them to use the appropriate values computed as the
-  // last iteration of the loop.
-  if (Count != 1) {
-    BasicBlock *LastIterationBB = cast<BasicBlock>(LastValueMap[LatchBlock]);
-    for (succ_iterator SI = succ_begin(LatchBlock), SE = succ_end(LatchBlock);
-         SI != SE; ++SI) {
-      for (BasicBlock::iterator BBI = (*SI)->begin();
-           PHINode *PN = dyn_cast<PHINode>(BBI); ++BBI) {
-        Value *InVal = PN->removeIncomingValue(LatchBlock, false);
-        // If this value was defined in the loop, take the value defined by the
-        // last iteration of the loop.
-        if (Instruction *InValI = dyn_cast<Instruction>(InVal)) {
-          if (L->contains(InValI))
-            InVal = LastValueMap[InVal];
-        }
-        PN->addIncoming(InVal, LastIterationBB);
-      }
-    }
-  }
 
-  // Now, if we're doing complete unrolling, loop over the PHI nodes in the
-  // original block, setting them to their incoming values.
-  if (CompletelyUnroll) {
-    BasicBlock *Preheader = L->getLoopPreheader();
-    for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
-      PHINode *PN = OrigPHINode[i];
+  // Loop over the PHI nodes in the original block, setting incoming values.
+  for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
+    PHINode *PN = OrigPHINode[i];
+    if (CompletelyUnroll) {
       PN->replaceAllUsesWith(PN->getIncomingValueForBlock(Preheader));
       Header->getInstList().erase(PN);
     }
+    else if (Count > 1) {
+      Value *InVal = PN->removeIncomingValue(LatchBlock, false);
+      // If this value was defined in the loop, take the value defined by the
+      // last iteration of the loop.
+      if (Instruction *InValI = dyn_cast<Instruction>(InVal)) {
+        if (L->contains(InValI))
+          InVal = LastValueMap[InVal];
+      }
+      assert(Latches.back() == LastValueMap[LatchBlock] && "bad last latch");
+      PN->addIncoming(InVal, Latches.back());
+    }
   }
 
   // Now that all the basic blocks for the unrolled iterations are in place,
@@ -351,6 +350,19 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
       // iteration.
       Term->setSuccessor(!ContinueOnTrue, Dest);
     } else {
+      // Remove phi operands at this loop exit
+      if (Dest != LoopExit) {
+        BasicBlock *BB = Latches[i];
+        for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
+             SI != SE; ++SI) {
+          if (*SI == Headers[i])
+            continue;
+          for (BasicBlock::iterator BBI = (*SI)->begin();
+               PHINode *Phi = dyn_cast<PHINode>(BBI); ++BBI) {
+            Phi->removeIncomingValue(BB, false);
+          }
+        }
+      }
       // Replace the conditional branch with an unconditional one.
       BranchInst::Create(Dest, Term);
       Term->eraseFromParent();
@@ -362,11 +374,29 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count,
     BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
     if (Term->isUnconditional()) {
       BasicBlock *Dest = Term->getSuccessor(0);
-      if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI))
+      if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI, LPM))
         std::replace(Latches.begin(), Latches.end(), Dest, Fold);
     }
   }
-  
+
+  // FIXME: Reconstruct dom info, because it is not preserved properly.
+  // Incrementally updating domtree after loop unrolling would be easy.
+  if (DominatorTree *DT = LPM->getAnalysisIfAvailable<DominatorTree>())
+    DT->runOnFunction(*L->getHeader()->getParent());
+
+  // Simplify any new induction variables in the partially unrolled loop.
+  if (SE && !CompletelyUnroll) {
+    SmallVector<WeakVH, 16> DeadInsts;
+    simplifyLoopIVs(L, SE, LPM, DeadInsts);
+
+    // Aggressively clean up dead instructions that simplifyLoopIVs already
+    // identified. Any remaining should be cleaned up below.
+    while (!DeadInsts.empty())
+      if (Instruction *Inst =
+          dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()))
+        RecursivelyDeleteTriviallyDeadInstructions(Inst);
+  }
+
   // At this point, the code is well formed.  We now do a quick sweep over the
   // inserted code, doing constant propagation and dead code elimination as we
   // go.
diff --git a/contrib/llvm/lib/Transforms/Utils/LowerExpectIntrinsic.cpp b/contrib/llvm/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
index c1213fac7bc7..61ab3f65330a 100644
--- a/contrib/llvm/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/LowerExpectIntrinsic.cpp
@@ -58,7 +58,7 @@ bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
     return false;
 
   LLVMContext &Context = CI->getContext();
-  const Type *Int32Ty = Type::getInt32Ty(Context);
+  Type *Int32Ty = Type::getInt32Ty(Context);
 
   unsigned caseNo = SI->findCaseValue(ExpectedValue);
   std::vector<Value *> Vec;
@@ -105,7 +105,7 @@ bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
     return false;
 
   LLVMContext &Context = CI->getContext();
-  const Type *Int32Ty = Type::getInt32Ty(Context);
+  Type *Int32Ty = Type::getInt32Ty(Context);
   bool Likely = ExpectedValue->isOne();
 
   // If expect value is equal to 1 it means that we are more likely to take
diff --git a/contrib/llvm/lib/Transforms/Utils/LowerInvoke.cpp b/contrib/llvm/lib/Transforms/Utils/LowerInvoke.cpp
index f77d19de900d..c96c8fce7b19 100644
--- a/contrib/llvm/lib/Transforms/Utils/LowerInvoke.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/LowerInvoke.cpp
@@ -120,18 +120,18 @@ FunctionPass *llvm::createLowerInvokePass(const TargetLowering *TLI,
 // doInitialization - Make sure that there is a prototype for abort in the
 // current module.
 bool LowerInvoke::doInitialization(Module &M) {
-  const Type *VoidPtrTy = Type::getInt8PtrTy(M.getContext());
+  Type *VoidPtrTy = Type::getInt8PtrTy(M.getContext());
   if (useExpensiveEHSupport) {
     // Insert a type for the linked list of jump buffers.
     unsigned JBSize = TLI ? TLI->getJumpBufSize() : 0;
     JBSize = JBSize ? JBSize : 200;
     Type *JmpBufTy = ArrayType::get(VoidPtrTy, JBSize);
 
-    JBLinkTy = StructType::createNamed(M.getContext(), "llvm.sjljeh.jmpbufty");
+    JBLinkTy = StructType::create(M.getContext(), "llvm.sjljeh.jmpbufty");
     Type *Elts[] = { JmpBufTy, PointerType::getUnqual(JBLinkTy) };
     JBLinkTy->setBody(Elts);
 
-    const Type *PtrJBList = PointerType::getUnqual(JBLinkTy);
+    Type *PtrJBList = PointerType::getUnqual(JBLinkTy);
 
     // Now that we've done that, insert the jmpbuf list head global, unless it
     // already exists.
@@ -240,14 +240,14 @@ void LowerInvoke::rewriteExpensiveInvoke(InvokeInst *II, unsigned InvokeNo,
   CallInst* StackSaveRet = CallInst::Create(StackSaveFn, "ssret", II);
   new StoreInst(StackSaveRet, StackPtr, true, II); // volatile
 
-  BasicBlock::iterator NI = II->getNormalDest()->getFirstNonPHI();
+  BasicBlock::iterator NI = II->getNormalDest()->getFirstInsertionPt();
   // nonvolatile.
   new StoreInst(Constant::getNullValue(Type::getInt32Ty(II->getContext())), 
                 InvokeNum, false, NI);
 
-  Instruction* StackPtrLoad = new LoadInst(StackPtr, "stackptr.restore", true,
-                                           II->getUnwindDest()->getFirstNonPHI()
-                                           );
+  Instruction* StackPtrLoad =
+    new LoadInst(StackPtr, "stackptr.restore", true,
+                 II->getUnwindDest()->getFirstInsertionPt());
   CallInst::Create(StackRestoreFn, StackPtrLoad, "")->insertAfter(StackPtrLoad);
     
   // Add a switch case to our unwind block.
@@ -305,7 +305,7 @@ splitLiveRangesLiveAcrossInvokes(SmallVectorImpl<InvokeInst*> &Invokes) {
     ++AfterAllocaInsertPt;
   for (Function::arg_iterator AI = F->arg_begin(), E = F->arg_end();
        AI != E; ++AI) {
-    const Type *Ty = AI->getType();
+    Type *Ty = AI->getType();
     // Aggregate types can't be cast, but are legal argument types, so we have
     // to handle them differently. We use an extract/insert pair as a
     // lightweight method to achieve the same goal.
@@ -406,6 +406,7 @@ bool LowerInvoke::insertExpensiveEHSupport(Function &F) {
   SmallVector<ReturnInst*,16> Returns;
   SmallVector<UnwindInst*,16> Unwinds;
   SmallVector<InvokeInst*,16> Invokes;
+  UnreachableInst* UnreachablePlaceholder = 0;
 
   for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
     if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) {
@@ -455,8 +456,7 @@ bool LowerInvoke::insertExpensiveEHSupport(Function &F) {
 
     Value *Idx[] = { Constant::getNullValue(Type::getInt32Ty(F.getContext())),
                      ConstantInt::get(Type::getInt32Ty(F.getContext()), 1) };
-    OldJmpBufPtr = GetElementPtrInst::Create(JmpBuf, &Idx[0], &Idx[2],
-                                             "OldBuf",
+    OldJmpBufPtr = GetElementPtrInst::Create(JmpBuf, Idx, "OldBuf",
                                              EntryBB->getTerminator());
 
     // Copy the JBListHead to the alloca.
@@ -487,9 +487,10 @@ bool LowerInvoke::insertExpensiveEHSupport(Function &F) {
 
     // Insert a load in the Catch block, and a switch on its value.  By default,
     // we go to a block that just does an unwind (which is the correct action
-    // for a standard call).
+    // for a standard call). We insert an unreachable instruction here and
+    // modify the block to jump to the correct unwinding pad later.
     BasicBlock *UnwindBB = BasicBlock::Create(F.getContext(), "unwindbb", &F);
-    Unwinds.push_back(new UnwindInst(F.getContext(), UnwindBB));
+    UnreachablePlaceholder = new UnreachableInst(F.getContext(), UnwindBB);
 
     Value *CatchLoad = new LoadInst(InvokeNum, "invoke.num", true, CatchBB);
     SwitchInst *CatchSwitch =
@@ -502,8 +503,7 @@ bool LowerInvoke::insertExpensiveEHSupport(Function &F) {
                                                      "setjmp.cont");
 
     Idx[1] = ConstantInt::get(Type::getInt32Ty(F.getContext()), 0);
-    Value *JmpBufPtr = GetElementPtrInst::Create(JmpBuf, &Idx[0], &Idx[2],
-                                                 "TheJmpBuf",
+    Value *JmpBufPtr = GetElementPtrInst::Create(JmpBuf, Idx, "TheJmpBuf",
                                                  EntryBB->getTerminator());
     JmpBufPtr = new BitCastInst(JmpBufPtr,
                         Type::getInt8PtrTy(F.getContext()),
@@ -557,8 +557,7 @@ bool LowerInvoke::insertExpensiveEHSupport(Function &F) {
   // Get a pointer to the jmpbuf and longjmp.
   Value *Idx[] = { Constant::getNullValue(Type::getInt32Ty(F.getContext())),
                    ConstantInt::get(Type::getInt32Ty(F.getContext()), 0) };
-  Idx[0] = GetElementPtrInst::Create(BufPtr, &Idx[0], &Idx[2], "JmpBuf",
-                                     UnwindBlock);
+  Idx[0] = GetElementPtrInst::Create(BufPtr, Idx, "JmpBuf", UnwindBlock);
   Idx[0] = new BitCastInst(Idx[0],
              Type::getInt8PtrTy(F.getContext()),
                            "tmp", UnwindBlock);
@@ -580,6 +579,12 @@ bool LowerInvoke::insertExpensiveEHSupport(Function &F) {
     Unwinds[i]->eraseFromParent();
   }
 
+  // Replace the inserted unreachable with a branch to the unwind handler.
+  if (UnreachablePlaceholder) {
+    BranchInst::Create(UnwindHandler, UnreachablePlaceholder);
+    UnreachablePlaceholder->eraseFromParent();
+  }
+
   // Finally, for any returns from this function, if this function contains an
   // invoke, restore the old jmpbuf pointer to its input value.
   if (OldJmpBufPtr) {
diff --git a/contrib/llvm/lib/Transforms/Utils/LowerSwitch.cpp b/contrib/llvm/lib/Transforms/Utils/LowerSwitch.cpp
index ed733d393a11..686178ca01cc 100644
--- a/contrib/llvm/lib/Transforms/Utils/LowerSwitch.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/LowerSwitch.cpp
@@ -277,11 +277,11 @@ void LowerSwitch::processSwitchInst(SwitchInst *SI) {
   BasicBlock *CurBlock = SI->getParent();
   BasicBlock *OrigBlock = CurBlock;
   Function *F = CurBlock->getParent();
-  Value *Val = SI->getOperand(0);  // The value we are switching on...
+  Value *Val = SI->getCondition();  // The value we are switching on...
   BasicBlock* Default = SI->getDefaultDest();
 
   // If there is only the default destination, don't bother with the code below.
-  if (SI->getNumOperands() == 2) {
+  if (SI->getNumCases() == 1) {
     BranchInst::Create(SI->getDefaultDest(), CurBlock);
     CurBlock->getInstList().erase(SI);
     return;
diff --git a/contrib/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/contrib/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index e5a00f4e9774..db3e94251350 100644
--- a/contrib/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -86,11 +86,15 @@ bool llvm::isAllocaPromotable(const AllocaInst *AI) {
        UI != UE; ++UI) {   // Loop over all of the uses of the alloca
     const User *U = *UI;
     if (const LoadInst *LI = dyn_cast<LoadInst>(U)) {
+      // Note that atomic loads can be transformed; atomic semantics do
+      // not have any meaning for a local alloca.
       if (LI->isVolatile())
         return false;
     } else if (const StoreInst *SI = dyn_cast<StoreInst>(U)) {
       if (SI->getOperand(0) == AI)
         return false;   // Don't allow a store OF the AI, only INTO the AI.
+      // Note that atomic stores can be transformed; atomic semantics do
+      // not have any meaning for a local alloca.
       if (SI->isVolatile())
         return false;
     } else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
diff --git a/contrib/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/contrib/llvm/lib/Transforms/Utils/SSAUpdater.cpp
index b47a7ccd80ba..fa8061c2b44c 100644
--- a/contrib/llvm/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/SSAUpdater.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
@@ -43,7 +44,7 @@ SSAUpdater::~SSAUpdater() {
 
 /// Initialize - Reset this object to get ready for a new set of SSA
 /// updates with type 'Ty'.  PHI nodes get a name based on 'Name'.
-void SSAUpdater::Initialize(const Type *Ty, StringRef Name) {
+void SSAUpdater::Initialize(Type *Ty, StringRef Name) {
   if (AV == 0)
     AV = new AvailableValsTy();
   else
@@ -378,8 +379,7 @@ run(const SmallVectorImpl<Instruction*> &Insts) const {
   // First step: bucket up uses of the alloca by the block they occur in.
   // This is important because we have to handle multiple defs/uses in a block
   // ourselves: SSAUpdater is purely for cross-block references.
-  // FIXME: Want a TinyVector<Instruction*> since there is often 0/1 element.
-  DenseMap<BasicBlock*, std::vector<Instruction*> > UsesByBlock;
+  DenseMap<BasicBlock*, TinyPtrVector<Instruction*> > UsesByBlock;
   
   for (unsigned i = 0, e = Insts.size(); i != e; ++i) {
     Instruction *User = Insts[i];
@@ -395,7 +395,7 @@ run(const SmallVectorImpl<Instruction*> &Insts) const {
   for (unsigned i = 0, e = Insts.size(); i != e; ++i) {
     Instruction *User = Insts[i];
     BasicBlock *BB = User->getParent();
-    std::vector<Instruction*> &BlockUses = UsesByBlock[BB];
+    TinyPtrVector<Instruction*> &BlockUses = UsesByBlock[BB];
     
     // If this block has already been processed, ignore this repeat use.
     if (BlockUses.empty()) continue;
diff --git a/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 9d9c324b8468..b8c3ab4c6077 100644
--- a/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -63,6 +63,7 @@ class SimplifyCFGOpt {
   bool FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
                                            IRBuilder<> &Builder);
 
+  bool SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
   bool SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder);
   bool SimplifyUnwind(UnwindInst *UI, IRBuilder<> &Builder);
   bool SimplifyUnreachable(UnreachableInst *UI);
@@ -322,7 +323,7 @@ static ConstantInt *GetConstantInt(Value *V, const TargetData *TD) {
 
   // This is some kind of pointer constant. Turn it into a pointer-sized
   // ConstantInt if possible.
-  const IntegerType *PtrTy = TD->getIntPtrType(V->getContext());
+  IntegerType *PtrTy = TD->getIntPtrType(V->getContext());
 
   // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
   if (isa<ConstantPointerNull>(V))
@@ -2138,6 +2139,52 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
   return true;
 }
 
+bool SimplifyCFGOpt::SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
+  // If this is a trivial landing pad that just continues unwinding the caught
+  // exception then zap the landing pad, turning its invokes into calls.
+  BasicBlock *BB = RI->getParent();
+  LandingPadInst *LPInst = dyn_cast<LandingPadInst>(BB->getFirstNonPHI());
+  if (RI->getValue() != LPInst)
+    // Not a landing pad, or the resume is not unwinding the exception that
+    // caused control to branch here.
+    return false;
+
+  // Check that there are no other instructions except for debug intrinsics.
+  BasicBlock::iterator I = LPInst, E = RI;
+  while (++I != E)
+    if (!isa<DbgInfoIntrinsic>(I))
+      return false;
+
+  // Turn all invokes that unwind here into calls and delete the basic block.
+  for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
+    InvokeInst *II = cast<InvokeInst>((*PI++)->getTerminator());
+    SmallVector<Value*, 8> Args(II->op_begin(), II->op_end() - 3);
+    // Insert a call instruction before the invoke.
+    CallInst *Call = CallInst::Create(II->getCalledValue(), Args, "", II);
+    Call->takeName(II);
+    Call->setCallingConv(II->getCallingConv());
+    Call->setAttributes(II->getAttributes());
+    Call->setDebugLoc(II->getDebugLoc());
+
+    // Anything that used the value produced by the invoke instruction now uses
+    // the value produced by the call instruction.  Note that we do this even
+    // for void functions and calls with no uses so that the callgraph edge is
+    // updated.
+    II->replaceAllUsesWith(Call);
+    BB->removePredecessor(II->getParent());
+
+    // Insert a branch to the normal destination right before the invoke.
+    BranchInst::Create(II->getNormalDest(), II);
+
+    // Finally, delete the invoke instruction!
+    II->eraseFromParent();
+  }
+
+  // The landingpad is now unreachable.  Zap it.
+  BB->eraseFromParent();
+  return true;
+}
+
 bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
   BasicBlock *BB = RI->getParent();
   if (!BB->getFirstNonPHIOrDbg()->isTerminator()) return false;
@@ -2244,18 +2291,34 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
   while (UI != BB->begin()) {
     BasicBlock::iterator BBI = UI;
     --BBI;
-    // Do not delete instructions that can have side effects, like calls
-    // (which may never return) and volatile loads and stores.
+    // Do not delete instructions that can have side effects which might cause
+    // the unreachable to not be reachable; specifically, calls and volatile
+    // operations may have this effect.
     if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI)) break;
-    
-    if (StoreInst *SI = dyn_cast<StoreInst>(BBI))
-      if (SI->isVolatile())
+
+    if (BBI->mayHaveSideEffects()) {
+      if (StoreInst *SI = dyn_cast<StoreInst>(BBI)) {
+        if (SI->isVolatile())
+          break;
+      } else if (LoadInst *LI = dyn_cast<LoadInst>(BBI)) {
+        if (LI->isVolatile())
+          break;
+      } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(BBI)) {
+        if (RMWI->isVolatile())
+          break;
+      } else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) {
+        if (CXI->isVolatile())
+          break;
+      } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) &&
+                 !isa<LandingPadInst>(BBI)) {
         break;
-    
-    if (LoadInst *LI = dyn_cast<LoadInst>(BBI))
-      if (LI->isVolatile())
-        break;
-    
+      }
+      // Note that deleting LandingPad's here is in fact okay, although it
+      // involves a bit of subtle reasoning. If this inst is a LandingPad,
+      // all the predecessors of this block will be the unwind edges of Invokes,
+      // and we can therefore guarantee this block will be erased.
+    }
+
     // Delete this instruction (any uses are guaranteed to be dead)
     if (!BBI->use_empty())
       BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
@@ -2707,6 +2770,71 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
   return false;
 }
 
+/// Check if passing a value to an instruction will cause undefined behavior.
+static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I) {
+  Constant *C = dyn_cast<Constant>(V);
+  if (!C)
+    return false;
+
+  if (!I->hasOneUse()) // Only look at single-use instructions, for compile time
+    return false;
+
+  if (C->isNullValue()) {
+    Instruction *Use = I->use_back();
+
+    // Now make sure that there are no instructions in between that can alter
+    // control flow (eg. calls)
+    for (BasicBlock::iterator i = ++BasicBlock::iterator(I); &*i != Use; ++i)
+      if (i == I->getParent()->end() || i->mayHaveSideEffects())
+        return false;
+
+    // Look through GEPs. A load from a GEP derived from NULL is still undefined
+    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use))
+      if (GEP->getPointerOperand() == I)
+        return passingValueIsAlwaysUndefined(V, GEP);
+
+    // Look through bitcasts.
+    if (BitCastInst *BC = dyn_cast<BitCastInst>(Use))
+      return passingValueIsAlwaysUndefined(V, BC);
+
+    // Load from null is undefined.
+    if (LoadInst *LI = dyn_cast<LoadInst>(Use))
+      return LI->getPointerAddressSpace() == 0;
+
+    // Store to null is undefined.
+    if (StoreInst *SI = dyn_cast<StoreInst>(Use))
+      return SI->getPointerAddressSpace() == 0 && SI->getPointerOperand() == I;
+  }
+  return false;
+}
+
+/// If BB has an incoming value that will always trigger undefined behavior
+/// (eg. null pointer derefence), remove the branch leading here.
+static bool removeUndefIntroducingPredecessor(BasicBlock *BB) {
+  for (BasicBlock::iterator i = BB->begin();
+       PHINode *PHI = dyn_cast<PHINode>(i); ++i)
+    for (unsigned i = 0, e = PHI->getNumIncomingValues(); i != e; ++i)
+      if (passingValueIsAlwaysUndefined(PHI->getIncomingValue(i), PHI)) {
+        TerminatorInst *T = PHI->getIncomingBlock(i)->getTerminator();
+        IRBuilder<> Builder(T);
+        if (BranchInst *BI = dyn_cast<BranchInst>(T)) {
+          BB->removePredecessor(PHI->getIncomingBlock(i));
+          // Turn uncoditional branches into unreachables and remove the dead
+          // destination from conditional branches.
+          if (BI->isUnconditional())
+            Builder.CreateUnreachable();
+          else
+            Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1) :
+                                                         BI->getSuccessor(0));
+          BI->eraseFromParent();
+          return true;
+        }
+        // TODO: SwitchInst.
+      }
+
+  return false;
+}
+
 bool SimplifyCFGOpt::run(BasicBlock *BB) {
   bool Changed = false;
 
@@ -2730,6 +2858,9 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
   // Check for and eliminate duplicate PHI nodes in this block.
   Changed |= EliminateDuplicatePHINodes(BB);
 
+  // Check for and remove branches that will always cause undefined behavior.
+  Changed |= removeUndefIntroducingPredecessor(BB);
+
   // Merge basic blocks into their predecessor if there is only one distinct
   // pred, and if there is only one distinct successor of the predecessor, and
   // if there are no PHI nodes.
@@ -2752,6 +2883,8 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
     } else {
       if (SimplifyCondBranch(BI, Builder)) return true;
     }
+  } else if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator())) {
+    if (SimplifyResume(RI, Builder)) return true;
   } else if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) {
     if (SimplifyReturn(RI, Builder)) return true;
   } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
diff --git a/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
new file mode 100644
index 000000000000..76289c055b90
--- /dev/null
+++ b/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -0,0 +1,432 @@
+//===-- SimplifyIndVar.cpp - Induction variable simplification ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements induction variable simplification. It does
+// not define any actual pass or policy, but provides a single function to
+// simplify a loop's induction variables based on ScalarEvolution.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "indvars"
+
+#include "llvm/Instructions.h"
+#include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/IVUsers.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/SimplifyIndVar.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+
+using namespace llvm;
+
+STATISTIC(NumElimIdentity, "Number of IV identities eliminated");
+STATISTIC(NumElimOperand,  "Number of IV operands folded into a use");
+STATISTIC(NumElimRem     , "Number of IV remainder operations eliminated");
+STATISTIC(NumElimCmp     , "Number of IV comparisons eliminated");
+
+namespace {
+  /// SimplifyIndvar - This is a utility for simplifying induction variables
+  /// based on ScalarEvolution. It is the primary instrument of the
+  /// IndvarSimplify pass, but it may also be directly invoked to cleanup after
+  /// other loop passes that preserve SCEV.
+  class SimplifyIndvar {
+    Loop             *L;
+    LoopInfo         *LI;
+    DominatorTree    *DT;
+    ScalarEvolution  *SE;
+    IVUsers          *IU; // NULL for DisableIVRewrite
+    const TargetData *TD; // May be NULL
+
+    SmallVectorImpl<WeakVH> &DeadInsts;
+
+    bool Changed;
+
+  public:
+    SimplifyIndvar(Loop *Loop, ScalarEvolution *SE, LPPassManager *LPM,
+                   SmallVectorImpl<WeakVH> &Dead, IVUsers *IVU = NULL) :
+      L(Loop),
+      LI(LPM->getAnalysisIfAvailable<LoopInfo>()),
+      SE(SE),
+      IU(IVU),
+      TD(LPM->getAnalysisIfAvailable<TargetData>()),
+      DeadInsts(Dead),
+      Changed(false) {
+      assert(LI && "IV simplification requires LoopInfo");
+    }
+
+    bool hasChanged() const { return Changed; }
+
+    /// Iteratively perform simplification on a worklist of users of the
+    /// specified induction variable. This is the top-level driver that applies
+    /// all simplicitions to users of an IV.
+    void simplifyUsers(PHINode *CurrIV, IVVisitor *V = NULL);
+
+    Value *foldIVUser(Instruction *UseInst, Instruction *IVOperand);
+
+    bool eliminateIVUser(Instruction *UseInst, Instruction *IVOperand);
+    void eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand);
+    void eliminateIVRemainder(BinaryOperator *Rem, Value *IVOperand,
+                              bool IsSigned);
+  };
+}
+
+/// foldIVUser - Fold an IV operand into its use.  This removes increments of an
+/// aligned IV when used by a instruction that ignores the low bits.
+///
+/// IVOperand is guaranteed SCEVable, but UseInst may not be.
+///
+/// Return the operand of IVOperand for this induction variable if IVOperand can
+/// be folded (in case more folding opportunities have been exposed).
+/// Otherwise return null.
+Value *SimplifyIndvar::foldIVUser(Instruction *UseInst, Instruction *IVOperand) {
+  Value *IVSrc = 0;
+  unsigned OperIdx = 0;
+  const SCEV *FoldedExpr = 0;
+  switch (UseInst->getOpcode()) {
+  default:
+    return 0;
+  case Instruction::UDiv:
+  case Instruction::LShr:
+    // We're only interested in the case where we know something about
+    // the numerator and have a constant denominator.
+    if (IVOperand != UseInst->getOperand(OperIdx) ||
+        !isa<ConstantInt>(UseInst->getOperand(1)))
+      return 0;
+
+    // Attempt to fold a binary operator with constant operand.
+    // e.g. ((I + 1) >> 2) => I >> 2
+    if (IVOperand->getNumOperands() != 2 ||
+        !isa<ConstantInt>(IVOperand->getOperand(1)))
+      return 0;
+
+    IVSrc = IVOperand->getOperand(0);
+    // IVSrc must be the (SCEVable) IV, since the other operand is const.
+    assert(SE->isSCEVable(IVSrc->getType()) && "Expect SCEVable IV operand");
+
+    ConstantInt *D = cast<ConstantInt>(UseInst->getOperand(1));
+    if (UseInst->getOpcode() == Instruction::LShr) {
+      // Get a constant for the divisor. See createSCEV.
+      uint32_t BitWidth = cast<IntegerType>(UseInst->getType())->getBitWidth();
+      if (D->getValue().uge(BitWidth))
+        return 0;
+
+      D = ConstantInt::get(UseInst->getContext(),
+                           APInt(BitWidth, 1).shl(D->getZExtValue()));
+    }
+    FoldedExpr = SE->getUDivExpr(SE->getSCEV(IVSrc), SE->getSCEV(D));
+  }
+  // We have something that might fold it's operand. Compare SCEVs.
+  if (!SE->isSCEVable(UseInst->getType()))
+    return 0;
+
+  // Bypass the operand if SCEV can prove it has no effect.
+  if (SE->getSCEV(UseInst) != FoldedExpr)
+    return 0;
+
+  DEBUG(dbgs() << "INDVARS: Eliminated IV operand: " << *IVOperand
+        << " -> " << *UseInst << '\n');
+
+  UseInst->setOperand(OperIdx, IVSrc);
+  assert(SE->getSCEV(UseInst) == FoldedExpr && "bad SCEV with folded oper");
+
+  ++NumElimOperand;
+  Changed = true;
+  if (IVOperand->use_empty())
+    DeadInsts.push_back(IVOperand);
+  return IVSrc;
+}
+
+/// eliminateIVComparison - SimplifyIVUsers helper for eliminating useless
+/// comparisons against an induction variable.
+void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) {
+  unsigned IVOperIdx = 0;
+  ICmpInst::Predicate Pred = ICmp->getPredicate();
+  if (IVOperand != ICmp->getOperand(0)) {
+    // Swapped
+    assert(IVOperand == ICmp->getOperand(1) && "Can't find IVOperand");
+    IVOperIdx = 1;
+    Pred = ICmpInst::getSwappedPredicate(Pred);
+  }
+
+  // Get the SCEVs for the ICmp operands.
+  const SCEV *S = SE->getSCEV(ICmp->getOperand(IVOperIdx));
+  const SCEV *X = SE->getSCEV(ICmp->getOperand(1 - IVOperIdx));
+
+  // Simplify unnecessary loops away.
+  const Loop *ICmpLoop = LI->getLoopFor(ICmp->getParent());
+  S = SE->getSCEVAtScope(S, ICmpLoop);
+  X = SE->getSCEVAtScope(X, ICmpLoop);
+
+  // If the condition is always true or always false, replace it with
+  // a constant value.
+  if (SE->isKnownPredicate(Pred, S, X))
+    ICmp->replaceAllUsesWith(ConstantInt::getTrue(ICmp->getContext()));
+  else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X))
+    ICmp->replaceAllUsesWith(ConstantInt::getFalse(ICmp->getContext()));
+  else
+    return;
+
+  DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n');
+  ++NumElimCmp;
+  Changed = true;
+  DeadInsts.push_back(ICmp);
+}
+
+/// eliminateIVRemainder - SimplifyIVUsers helper for eliminating useless
+/// remainder operations operating on an induction variable.
+void SimplifyIndvar::eliminateIVRemainder(BinaryOperator *Rem,
+                                      Value *IVOperand,
+                                      bool IsSigned) {
+  // We're only interested in the case where we know something about
+  // the numerator.
+  if (IVOperand != Rem->getOperand(0))
+    return;
+
+  // Get the SCEVs for the ICmp operands.
+  const SCEV *S = SE->getSCEV(Rem->getOperand(0));
+  const SCEV *X = SE->getSCEV(Rem->getOperand(1));
+
+  // Simplify unnecessary loops away.
+  const Loop *ICmpLoop = LI->getLoopFor(Rem->getParent());
+  S = SE->getSCEVAtScope(S, ICmpLoop);
+  X = SE->getSCEVAtScope(X, ICmpLoop);
+
+  // i % n  -->  i  if i is in [0,n).
+  if ((!IsSigned || SE->isKnownNonNegative(S)) &&
+      SE->isKnownPredicate(IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
+                           S, X))
+    Rem->replaceAllUsesWith(Rem->getOperand(0));
+  else {
+    // (i+1) % n  -->  (i+1)==n?0:(i+1)  if i is in [0,n).
+    const SCEV *LessOne =
+      SE->getMinusSCEV(S, SE->getConstant(S->getType(), 1));
+    if (IsSigned && !SE->isKnownNonNegative(LessOne))
+      return;
+
+    if (!SE->isKnownPredicate(IsSigned ?
+                              ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
+                              LessOne, X))
+      return;
+
+    ICmpInst *ICmp = new ICmpInst(Rem, ICmpInst::ICMP_EQ,
+                                  Rem->getOperand(0), Rem->getOperand(1));
+    SelectInst *Sel =
+      SelectInst::Create(ICmp,
+                         ConstantInt::get(Rem->getType(), 0),
+                         Rem->getOperand(0), "tmp", Rem);
+    Rem->replaceAllUsesWith(Sel);
+  }
+
+  // Inform IVUsers about the new users.
+  if (IU) {
+    if (Instruction *I = dyn_cast<Instruction>(Rem->getOperand(0)))
+      IU->AddUsersIfInteresting(I);
+  }
+  DEBUG(dbgs() << "INDVARS: Simplified rem: " << *Rem << '\n');
+  ++NumElimRem;
+  Changed = true;
+  DeadInsts.push_back(Rem);
+}
+
+/// eliminateIVUser - Eliminate an operation that consumes a simple IV and has
+/// no observable side-effect given the range of IV values.
+/// IVOperand is guaranteed SCEVable, but UseInst may not be.
+bool SimplifyIndvar::eliminateIVUser(Instruction *UseInst,
+                                     Instruction *IVOperand) {
+  if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) {
+    eliminateIVComparison(ICmp, IVOperand);
+    return true;
+  }
+  if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) {
+    bool IsSigned = Rem->getOpcode() == Instruction::SRem;
+    if (IsSigned || Rem->getOpcode() == Instruction::URem) {
+      eliminateIVRemainder(Rem, IVOperand, IsSigned);
+      return true;
+    }
+  }
+
+  // Eliminate any operation that SCEV can prove is an identity function.
+  if (!SE->isSCEVable(UseInst->getType()) ||
+      (UseInst->getType() != IVOperand->getType()) ||
+      (SE->getSCEV(UseInst) != SE->getSCEV(IVOperand)))
+    return false;
+
+  DEBUG(dbgs() << "INDVARS: Eliminated identity: " << *UseInst << '\n');
+
+  UseInst->replaceAllUsesWith(IVOperand);
+  ++NumElimIdentity;
+  Changed = true;
+  DeadInsts.push_back(UseInst);
+  return true;
+}
+
+/// pushIVUsers - Add all uses of Def to the current IV's worklist.
+///
+static void pushIVUsers(
+  Instruction *Def,
+  SmallPtrSet<Instruction*,16> &Simplified,
+  SmallVectorImpl< std::pair<Instruction*,Instruction*> > &SimpleIVUsers) {
+
+  for (Value::use_iterator UI = Def->use_begin(), E = Def->use_end();
+       UI != E; ++UI) {
+    Instruction *User = cast<Instruction>(*UI);
+
+    // Avoid infinite or exponential worklist processing.
+    // Also ensure unique worklist users.
+    // If Def is a LoopPhi, it may not be in the Simplified set, so check for
+    // self edges first.
+    if (User != Def && Simplified.insert(User))
+      SimpleIVUsers.push_back(std::make_pair(User, Def));
+  }
+}
+
+/// isSimpleIVUser - Return true if this instruction generates a simple SCEV
+/// expression in terms of that IV.
+///
+/// This is similar to IVUsers' isInteresting() but processes each instruction
+/// non-recursively when the operand is already known to be a simpleIVUser.
+///
+static bool isSimpleIVUser(Instruction *I, const Loop *L, ScalarEvolution *SE) {
+  if (!SE->isSCEVable(I->getType()))
+    return false;
+
+  // Get the symbolic expression for this instruction.
+  const SCEV *S = SE->getSCEV(I);
+
+  // Only consider affine recurrences.
+  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S);
+  if (AR && AR->getLoop() == L)
+    return true;
+
+  return false;
+}
+
+/// simplifyUsers - Iteratively perform simplification on a worklist of users
+/// of the specified induction variable. Each successive simplification may push
+/// more users which may themselves be candidates for simplification.
+///
+/// This algorithm does not require IVUsers analysis. Instead, it simplifies
+/// instructions in-place during analysis. Rather than rewriting induction
+/// variables bottom-up from their users, it transforms a chain of IVUsers
+/// top-down, updating the IR only when it encouters a clear optimization
+/// opportunitiy.
+///
+/// Once DisableIVRewrite is default, LSR will be the only client of IVUsers.
+///
+void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) {
+  if (!SE->isSCEVable(CurrIV->getType()))
+    return;
+
+  // Instructions processed by SimplifyIndvar for CurrIV.
+  SmallPtrSet<Instruction*,16> Simplified;
+
+  // Use-def pairs if IV users waiting to be processed for CurrIV.
+  SmallVector<std::pair<Instruction*, Instruction*>, 8> SimpleIVUsers;
+
+  // Push users of the current LoopPhi. In rare cases, pushIVUsers may be
+  // called multiple times for the same LoopPhi. This is the proper thing to
+  // do for loop header phis that use each other.
+  pushIVUsers(CurrIV, Simplified, SimpleIVUsers);
+
+  while (!SimpleIVUsers.empty()) {
+    std::pair<Instruction*, Instruction*> UseOper =
+      SimpleIVUsers.pop_back_val();
+    // Bypass back edges to avoid extra work.
+    if (UseOper.first == CurrIV) continue;
+
+    Instruction *IVOperand = UseOper.second;
+    for (unsigned N = 0; IVOperand; ++N) {
+      assert(N <= Simplified.size() && "runaway iteration");
+
+      Value *NewOper = foldIVUser(UseOper.first, IVOperand);
+      if (!NewOper)
+        break; // done folding
+      IVOperand = dyn_cast<Instruction>(NewOper);
+    }
+    if (!IVOperand)
+      continue;
+
+    if (eliminateIVUser(UseOper.first, IVOperand)) {
+      pushIVUsers(IVOperand, Simplified, SimpleIVUsers);
+      continue;
+    }
+    CastInst *Cast = dyn_cast<CastInst>(UseOper.first);
+    if (V && Cast) {
+      V->visitCast(Cast);
+      continue;
+    }
+    if (isSimpleIVUser(UseOper.first, L, SE)) {
+      pushIVUsers(UseOper.first, Simplified, SimpleIVUsers);
+    }
+  }
+}
+
+namespace llvm {
+
+/// simplifyUsersOfIV - Simplify instructions that use this induction variable
+/// by using ScalarEvolution to analyze the IV's recurrence.
+bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, LPPassManager *LPM,
+                       SmallVectorImpl<WeakVH> &Dead, IVVisitor *V)
+{
+  LoopInfo *LI = &LPM->getAnalysis<LoopInfo>();
+  SimplifyIndvar SIV(LI->getLoopFor(CurrIV->getParent()), SE, LPM, Dead);
+  SIV.simplifyUsers(CurrIV, V);
+  return SIV.hasChanged();
+}
+
+/// simplifyLoopIVs - Simplify users of induction variables within this
+/// loop. This does not actually change or add IVs.
+bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, LPPassManager *LPM,
+                     SmallVectorImpl<WeakVH> &Dead) {
+  bool Changed = false;
+  for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
+    Changed |= simplifyUsersOfIV(cast<PHINode>(I), SE, LPM, Dead);
+  }
+  return Changed;
+}
+
+/// simplifyIVUsers - Perform simplification on instructions recorded by the
+/// IVUsers pass.
+///
+/// This is the old approach to IV simplification to be replaced by
+/// SimplifyLoopIVs.
+bool simplifyIVUsers(IVUsers *IU, ScalarEvolution *SE, LPPassManager *LPM,
+                     SmallVectorImpl<WeakVH> &Dead) {
+  SimplifyIndvar SIV(IU->getLoop(), SE, LPM, Dead);
+
+  // Each round of simplification involves a round of eliminating operations
+  // followed by a round of widening IVs. A single IVUsers worklist is used
+  // across all rounds. The inner loop advances the user. If widening exposes
+  // more uses, then another pass through the outer loop is triggered.
+  for (IVUsers::iterator I = IU->begin(); I != IU->end(); ++I) {
+    Instruction *UseInst = I->getUser();
+    Value *IVOperand = I->getOperandValToReplace();
+
+    if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) {
+      SIV.eliminateIVComparison(ICmp, IVOperand);
+      continue;
+    }
+    if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) {
+      bool IsSigned = Rem->getOpcode() == Instruction::SRem;
+      if (IsSigned || Rem->getOpcode() == Instruction::URem) {
+        SIV.eliminateIVRemainder(Rem, IVOperand, IsSigned);
+        continue;
+      }
+    }
+  }
+  return SIV.hasChanged();
+}
+
+} // namespace llvm
diff --git a/contrib/llvm/lib/Transforms/Utils/ValueMapper.cpp b/contrib/llvm/lib/Transforms/Utils/ValueMapper.cpp
index 973b105a1cbb..fc2538db6482 100644
--- a/contrib/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -183,10 +183,9 @@ void llvm::RemapInstruction(Instruction *I, ValueToValueMapTy &VMap,
     }
   }
 
-  // Remap attached metadata.  Don't bother remapping DebugLoc, it can never
-  // have mappings to do.
+  // Remap attached metadata.
   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
-  I->getAllMetadataOtherThanDebugLoc(MDs);
+  I->getAllMetadata(MDs);
   for (SmallVectorImpl<std::pair<unsigned, MDNode *> >::iterator
        MI = MDs.begin(), ME = MDs.end(); MI != ME; ++MI) {
     MDNode *Old = MI->second;
diff --git a/contrib/llvm/lib/VMCore/AsmWriter.cpp b/contrib/llvm/lib/VMCore/AsmWriter.cpp
index 94794c35fe0b..18308f27cfe4 100644
--- a/contrib/llvm/lib/VMCore/AsmWriter.cpp
+++ b/contrib/llvm/lib/VMCore/AsmWriter.cpp
@@ -58,7 +58,7 @@ static const Module *getModuleFromVal(const Value *V) {
     const Function *M = I->getParent() ? I->getParent()->getParent() : 0;
     return M ? M->getParent() : 0;
   }
-  
+
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V))
     return GV->getParent();
   return 0;
@@ -142,18 +142,18 @@ class TypePrinting {
 
   /// NamedTypes - The named types that are used by the current module.
   std::vector<StructType*> NamedTypes;
-  
+
   /// NumberedTypes - The numbered types, along with their value.
   DenseMap<StructType*, unsigned> NumberedTypes;
-  
+
 
   TypePrinting() {}
   ~TypePrinting() {}
-  
+
   void incorporateTypes(const Module &M);
-  
+
   void print(Type *Ty, raw_ostream &OS);
-  
+
   void printStructBody(StructType *Ty, raw_ostream &OS);
 };
 } // end anonymous namespace.
@@ -161,25 +161,25 @@ class TypePrinting {
 
 void TypePrinting::incorporateTypes(const Module &M) {
   M.findUsedStructTypes(NamedTypes);
-  
+
   // The list of struct types we got back includes all the struct types, split
   // the unnamed ones out to a numbering and remove the anonymous structs.
   unsigned NextNumber = 0;
-  
+
   std::vector<StructType*>::iterator NextToUse = NamedTypes.begin(), I, E;
   for (I = NamedTypes.begin(), E = NamedTypes.end(); I != E; ++I) {
     StructType *STy = *I;
-    
+
     // Ignore anonymous types.
-    if (STy->isAnonymous())
+    if (STy->isLiteral())
       continue;
-    
+
     if (STy->getName().empty())
       NumberedTypes[STy] = NextNumber++;
     else
       *NextToUse++ = STy;
   }
-    
+
   NamedTypes.erase(NextToUse, NamedTypes.end());
 }
 
@@ -220,13 +220,13 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) {
   }
   case Type::StructTyID: {
     StructType *STy = cast<StructType>(Ty);
-    
-    if (STy->isAnonymous())
+
+    if (STy->isLiteral())
       return printStructBody(STy, OS);
 
     if (!STy->getName().empty())
       return PrintLLVMName(OS, STy->getName(), LocalPrefix);
-    
+
     DenseMap<StructType*, unsigned>::iterator I = NumberedTypes.find(STy);
     if (I != NumberedTypes.end())
       OS << '%' << I->second;
@@ -267,10 +267,10 @@ void TypePrinting::printStructBody(StructType *STy, raw_ostream &OS) {
     OS << "opaque";
     return;
   }
-  
+
   if (STy->isPacked())
     OS << '<';
-  
+
   if (STy->getNumElements() == 0) {
     OS << "{}";
   } else {
@@ -281,7 +281,7 @@ void TypePrinting::printStructBody(StructType *STy, raw_ostream &OS) {
       OS << ", ";
       print(*I, OS);
     }
-  
+
     OS << " }";
   }
   if (STy->isPacked())
@@ -386,7 +386,8 @@ static SlotTracker *createSlotTracker(const Value *V) {
     return new SlotTracker(FA->getParent());
 
   if (const Instruction *I = dyn_cast<Instruction>(V))
-    return new SlotTracker(I->getParent()->getParent());
+    if (I->getParent())
+      return new SlotTracker(I->getParent()->getParent());
 
   if (const BasicBlock *BB = dyn_cast<BasicBlock>(V))
     return new SlotTracker(BB->getParent());
@@ -419,7 +420,7 @@ static SlotTracker *createSlotTracker(const Value *V) {
 // Module level constructor. Causes the contents of the Module (sans functions)
 // to be added to the slot table.
 SlotTracker::SlotTracker(const Module *M)
-  : TheModule(M), TheFunction(0), FunctionProcessed(false), 
+  : TheModule(M), TheFunction(0), FunctionProcessed(false),
     mNext(0), fNext(0),  mdnNext(0) {
 }
 
@@ -490,12 +491,12 @@ void SlotTracker::processFunction() {
        E = TheFunction->end(); BB != E; ++BB) {
     if (!BB->hasName())
       CreateFunctionSlot(BB);
-    
+
     for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E;
          ++I) {
       if (!I->getType()->isVoidTy() && !I->hasName())
         CreateFunctionSlot(I);
-      
+
       // Intrinsics can directly use metadata.  We allow direct calls to any
       // llvm.foo function here, because the target may not be linked into the
       // optimizer.
@@ -658,6 +659,23 @@ static const char *getPredicateText(unsigned predicate) {
   return pred;
 }
 
+static void writeAtomicRMWOperation(raw_ostream &Out,
+                                    AtomicRMWInst::BinOp Op) {
+  switch (Op) {
+  default: Out << " <unknown operation " << Op << ">"; break;
+  case AtomicRMWInst::Xchg: Out << " xchg"; break;
+  case AtomicRMWInst::Add:  Out << " add"; break;
+  case AtomicRMWInst::Sub:  Out << " sub"; break;
+  case AtomicRMWInst::And:  Out << " and"; break;
+  case AtomicRMWInst::Nand: Out << " nand"; break;
+  case AtomicRMWInst::Or:   Out << " or"; break;
+  case AtomicRMWInst::Xor:  Out << " xor"; break;
+  case AtomicRMWInst::Max:  Out << " max"; break;
+  case AtomicRMWInst::Min:  Out << " min"; break;
+  case AtomicRMWInst::UMax: Out << " umax"; break;
+  case AtomicRMWInst::UMin: Out << " umin"; break;
+  }
+}
 
 static void WriteOptimizationInfo(raw_ostream &Out, const User *U) {
   if (const OverflowingBinaryOperator *OBO =
@@ -792,7 +810,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
     Out << "zeroinitializer";
     return;
   }
-  
+
   if (const BlockAddress *BA = dyn_cast<BlockAddress>(CV)) {
     Out << "blockaddress(";
     WriteAsOperandInternal(Out, BA->getFunction(), &TypePrinter, Machine,
@@ -939,13 +957,13 @@ static void WriteMDNodeBodyInternal(raw_ostream &Out, const MDNode *Node,
     else {
       TypePrinter->print(V->getType(), Out);
       Out << ' ';
-      WriteAsOperandInternal(Out, Node->getOperand(mi), 
+      WriteAsOperandInternal(Out, Node->getOperand(mi),
                              TypePrinter, Machine, Context);
     }
     if (mi + 1 != me)
       Out << ", ";
   }
-  
+
   Out << "}";
 }
 
@@ -990,7 +1008,7 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V,
       WriteMDNodeBodyInternal(Out, N, TypePrinter, Machine, Context);
       return;
     }
-  
+
     if (!Machine) {
       if (N->isFunctionLocal())
         Machine = new SlotTracker(N->getFunction());
@@ -1020,26 +1038,35 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V,
 
   char Prefix = '%';
   int Slot;
+  // If we have a SlotTracker, use it.
   if (Machine) {
     if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
       Slot = Machine->getGlobalSlot(GV);
       Prefix = '@';
     } else {
       Slot = Machine->getLocalSlot(V);
+
+      // If the local value didn't succeed, then we may be referring to a value
+      // from a different function.  Translate it, as this can happen when using
+      // address of blocks.
+      if (Slot == -1)
+        if ((Machine = createSlotTracker(V))) {
+          Slot = Machine->getLocalSlot(V);
+          delete Machine;
+        }
     }
-  } else {
-    Machine = createSlotTracker(V);
-    if (Machine) {
-      if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
-        Slot = Machine->getGlobalSlot(GV);
-        Prefix = '@';
-      } else {
-        Slot = Machine->getLocalSlot(V);
-      }
-      delete Machine;
+  } else if ((Machine = createSlotTracker(V))) {
+    // Otherwise, create one to get the # and then destroy it.
+    if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
+      Slot = Machine->getGlobalSlot(GV);
+      Prefix = '@';
     } else {
-      Slot = -1;
+      Slot = Machine->getLocalSlot(V);
     }
+    delete Machine;
+    Machine = 0;
+  } else {
+    Slot = -1;
   }
 
   if (Slot != -1)
@@ -1081,7 +1108,7 @@ class AssemblyWriter {
   const Module *TheModule;
   TypePrinting TypePrinter;
   AssemblyAnnotationWriter *AnnotationWriter;
-  
+
 public:
   inline AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac,
                         const Module *M,
@@ -1093,11 +1120,12 @@ class AssemblyWriter {
 
   void printMDNodeBody(const MDNode *MD);
   void printNamedMDNode(const NamedMDNode *NMD);
-  
+
   void printModule(const Module *M);
 
   void writeOperand(const Value *Op, bool PrintType);
   void writeParamOperand(const Value *Operand, Attributes Attrs);
+  void writeAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope);
 
   void writeAllMDNodes();
 
@@ -1128,6 +1156,28 @@ void AssemblyWriter::writeOperand(const Value *Operand, bool PrintType) {
   WriteAsOperandInternal(Out, Operand, &TypePrinter, &Machine, TheModule);
 }
 
+void AssemblyWriter::writeAtomic(AtomicOrdering Ordering,
+                                 SynchronizationScope SynchScope) {
+  if (Ordering == NotAtomic)
+    return;
+
+  switch (SynchScope) {
+  default: Out << " <bad scope " << int(SynchScope) << ">"; break;
+  case SingleThread: Out << " singlethread"; break;
+  case CrossThread: break;
+  }
+
+  switch (Ordering) {
+  default: Out << " <bad ordering " << int(Ordering) << ">"; break;
+  case Unordered: Out << " unordered"; break;
+  case Monotonic: Out << " monotonic"; break;
+  case Acquire: Out << " acquire"; break;
+  case Release: Out << " release"; break;
+  case AcquireRelease: Out << " acq_rel"; break;
+  case SequentiallyConsistent: Out << " seq_cst"; break;
+  }
+}
+
 void AssemblyWriter::writeParamOperand(const Value *Operand,
                                        Attributes Attrs) {
   if (Operand == 0) {
@@ -1216,7 +1266,7 @@ void AssemblyWriter::printModule(const Module *M) {
 
   // Output named metadata.
   if (!M->named_metadata_empty()) Out << '\n';
-  
+
   for (Module::const_named_metadata_iterator I = M->named_metadata_begin(),
        E = M->named_metadata_end(); I != E; ++I)
     printNamedMDNode(I);
@@ -1357,26 +1407,8 @@ void AssemblyWriter::printAlias(const GlobalAlias *GA) {
   if (Aliasee == 0) {
     TypePrinter.print(GA->getType(), Out);
     Out << " <<NULL ALIASEE>>";
-  } else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Aliasee)) {
-    TypePrinter.print(GV->getType(), Out);
-    Out << ' ';
-    PrintLLVMName(Out, GV);
-  } else if (const Function *F = dyn_cast<Function>(Aliasee)) {
-    TypePrinter.print(F->getFunctionType(), Out);
-    Out << "* ";
-
-    WriteAsOperandInternal(Out, F, &TypePrinter, &Machine, F->getParent());
-  } else if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Aliasee)) {
-    TypePrinter.print(GA->getType(), Out);
-    Out << ' ';
-    PrintLLVMName(Out, GA);
   } else {
-    const ConstantExpr *CE = cast<ConstantExpr>(Aliasee);
-    // The only valid GEP is an all zero GEP.
-    assert((CE->getOpcode() == Instruction::BitCast ||
-            CE->getOpcode() == Instruction::GetElementPtr) &&
-           "Unsupported aliasee");
-    writeOperand(CE, false);
+    writeOperand(Aliasee, !isa<ConstantExpr>(Aliasee));
   }
 
   printInfoComment(*GA);
@@ -1387,29 +1419,29 @@ void AssemblyWriter::printTypeIdentities() {
   if (TypePrinter.NumberedTypes.empty() &&
       TypePrinter.NamedTypes.empty())
     return;
-  
+
   Out << '\n';
-  
+
   // We know all the numbers that each type is used and we know that it is a
   // dense assignment.  Convert the map to an index table.
   std::vector<StructType*> NumberedTypes(TypePrinter.NumberedTypes.size());
-  for (DenseMap<StructType*, unsigned>::iterator I = 
+  for (DenseMap<StructType*, unsigned>::iterator I =
        TypePrinter.NumberedTypes.begin(), E = TypePrinter.NumberedTypes.end();
        I != E; ++I) {
     assert(I->second < NumberedTypes.size() && "Didn't get a dense numbering?");
     NumberedTypes[I->second] = I->first;
   }
-           
+
   // Emit all numbered types.
   for (unsigned i = 0, e = NumberedTypes.size(); i != e; ++i) {
     Out << '%' << i << " = type ";
-    
+
     // Make sure we print out at least one level of the type structure, so
     // that we do not get %2 = type %2
     TypePrinter.printStructBody(NumberedTypes[i], Out);
     Out << '\n';
   }
-  
+
   for (unsigned i = 0, e = TypePrinter.NamedTypes.size(); i != e; ++i) {
     PrintLLVMName(Out, TypePrinter.NamedTypes[i]->getName(), LocalPrefix);
     Out << " = type ";
@@ -1457,7 +1489,7 @@ void AssemblyWriter::printFunction(const Function *F) {
   default: Out << "cc" << F->getCallingConv() << " "; break;
   }
 
-  const FunctionType *FT = F->getFunctionType();
+  FunctionType *FT = F->getFunctionType();
   const AttrListPtr &Attrs = F->getAttributes();
   Attributes RetAttrs = Attrs.getRetAttributes();
   if (RetAttrs != Attribute::None)
@@ -1628,18 +1660,24 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
       Out << '%' << SlotNum << " = ";
   }
 
-  // If this is a volatile load or store, print out the volatile marker.
-  if ((isa<LoadInst>(I)  && cast<LoadInst>(I).isVolatile()) ||
-      (isa<StoreInst>(I) && cast<StoreInst>(I).isVolatile())) {
-      Out << "volatile ";
-  } else if (isa<CallInst>(I) && cast<CallInst>(I).isTailCall()) {
-    // If this is a call, check if it's a tail call.
+  if (isa<CallInst>(I) && cast<CallInst>(I).isTailCall())
     Out << "tail ";
-  }
 
   // Print out the opcode...
   Out << I.getOpcodeName();
 
+  // If this is an atomic load or store, print out the atomic marker.
+  if ((isa<LoadInst>(I)  && cast<LoadInst>(I).isAtomic()) ||
+      (isa<StoreInst>(I) && cast<StoreInst>(I).isAtomic()))
+    Out << " atomic";
+
+  // If this is a volatile operation, print out the volatile marker.
+  if ((isa<LoadInst>(I)  && cast<LoadInst>(I).isVolatile()) ||
+      (isa<StoreInst>(I) && cast<StoreInst>(I).isVolatile()) ||
+      (isa<AtomicCmpXchgInst>(I) && cast<AtomicCmpXchgInst>(I).isVolatile()) ||
+      (isa<AtomicRMWInst>(I) && cast<AtomicRMWInst>(I).isVolatile()))
+    Out << " volatile";
+
   // Print out optimization information.
   WriteOptimizationInfo(Out, &I);
 
@@ -1647,6 +1685,10 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
   if (const CmpInst *CI = dyn_cast<CmpInst>(&I))
     Out << ' ' << getPredicateText(CI->getPredicate());
 
+  // Print out the atomicrmw operation
+  if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&I))
+    writeAtomicRMWOperation(Out, RMWI->getOperation());
+
   // Print out the type of the operands...
   const Value *Operand = I.getNumOperands() ? I.getOperand(0) : 0;
 
@@ -1661,18 +1703,20 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     writeOperand(BI.getSuccessor(1), true);
 
   } else if (isa<SwitchInst>(I)) {
+    SwitchInst& SI(cast<SwitchInst>(I));
     // Special case switch instruction to get formatting nice and correct.
     Out << ' ';
-    writeOperand(Operand        , true);
+    writeOperand(SI.getCondition(), true);
     Out << ", ";
-    writeOperand(I.getOperand(1), true);
+    writeOperand(SI.getDefaultDest(), true);
     Out << " [";
-
-    for (unsigned op = 2, Eop = I.getNumOperands(); op < Eop; op += 2) {
+    // Skip the first item since that's the default case.
+    unsigned NumCases = SI.getNumCases();
+    for (unsigned i = 1; i < NumCases; ++i) {
       Out << "\n    ";
-      writeOperand(I.getOperand(op  ), true);
+      writeOperand(SI.getCaseValue(i), true);
       Out << ", ";
-      writeOperand(I.getOperand(op+1), true);
+      writeOperand(SI.getSuccessor(i), true);
     }
     Out << "\n  ]";
   } else if (isa<IndirectBrInst>(I)) {
@@ -1680,7 +1724,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     Out << ' ';
     writeOperand(Operand, true);
     Out << ", [";
-    
+
     for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) {
       if (i != 1)
         Out << ", ";
@@ -1709,6 +1753,24 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     writeOperand(I.getOperand(1), true);
     for (const unsigned *i = IVI->idx_begin(), *e = IVI->idx_end(); i != e; ++i)
       Out << ", " << *i;
+  } else if (const LandingPadInst *LPI = dyn_cast<LandingPadInst>(&I)) {
+    Out << ' ';
+    TypePrinter.print(I.getType(), Out);
+    Out << " personality ";
+    writeOperand(I.getOperand(0), true); Out << '\n';
+
+    if (LPI->isCleanup())
+      Out << "          cleanup";
+
+    for (unsigned i = 0, e = LPI->getNumClauses(); i != e; ++i) {
+      if (i != 0 || LPI->isCleanup()) Out << "\n";
+      if (LPI->isCatch(i))
+        Out << "          catch ";
+      else
+        Out << "          filter ";
+
+      writeOperand(LPI->getClause(i), true);
+    }
   } else if (isa<ReturnInst>(I) && !Operand) {
     Out << " void";
   } else if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
@@ -1878,11 +1940,23 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     }
   }
 
-  // Print post operand alignment for load/store.
-  if (isa<LoadInst>(I) && cast<LoadInst>(I).getAlignment()) {
-    Out << ", align " << cast<LoadInst>(I).getAlignment();
-  } else if (isa<StoreInst>(I) && cast<StoreInst>(I).getAlignment()) {
-    Out << ", align " << cast<StoreInst>(I).getAlignment();
+  // Print atomic ordering/alignment for memory operations
+  if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
+    if (LI->isAtomic())
+      writeAtomic(LI->getOrdering(), LI->getSynchScope());
+    if (LI->getAlignment())
+      Out << ", align " << LI->getAlignment();
+  } else if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
+    if (SI->isAtomic())
+      writeAtomic(SI->getOrdering(), SI->getSynchScope());
+    if (SI->getAlignment())
+      Out << ", align " << SI->getAlignment();
+  } else if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(&I)) {
+    writeAtomic(CXI->getOrdering(), CXI->getSynchScope());
+  } else if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&I)) {
+    writeAtomic(RMWI->getOrdering(), RMWI->getSynchScope());
+  } else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
+    writeAtomic(FI->getOrdering(), FI->getSynchScope());
   }
 
   // Print Metadata info.
@@ -1916,7 +1990,7 @@ static void WriteMDNodeComment(const MDNode *Node,
   APInt Tag = Val & ~APInt(Val.getBitWidth(), LLVMDebugVersionMask);
   if (Val.ult(LLVMDebugVersion))
     return;
-  
+
   Out.PadToColumn(50);
   if (Tag == dwarf::DW_TAG_user_base)
     Out << "; [ DW_TAG_user_base ]";
@@ -1932,7 +2006,7 @@ void AssemblyWriter::writeAllMDNodes() {
   for (SlotTracker::mdn_iterator I = Machine.mdn_begin(), E = Machine.mdn_end();
        I != E; ++I)
     Nodes[I->second] = cast<MDNode>(I->first);
-  
+
   for (unsigned i = 0, e = Nodes.size(); i != e; ++i) {
     Out << '!' << i << " = metadata ";
     printMDNodeBody(Nodes[i]);
@@ -1970,10 +2044,10 @@ void Type::print(raw_ostream &OS) const {
   }
   TypePrinting TP;
   TP.print(const_cast<Type*>(this), OS);
-  
+
   // If the type is a named struct type, print the body as well.
   if (StructType *STy = dyn_cast<StructType>(const_cast<Type*>(this)))
-    if (!STy->isAnonymous()) {
+    if (!STy->isLiteral()) {
       OS << " = type ";
       TP.printStructBody(STy, OS);
     }
diff --git a/contrib/llvm/lib/VMCore/Attributes.cpp b/contrib/llvm/lib/VMCore/Attributes.cpp
index bf6efa1645a2..485be75d1b12 100644
--- a/contrib/llvm/lib/VMCore/Attributes.cpp
+++ b/contrib/llvm/lib/VMCore/Attributes.cpp
@@ -38,6 +38,8 @@ std::string Attribute::getAsString(Attributes Attrs) {
     Result += "nounwind ";
   if (Attrs & Attribute::UWTable)
     Result += "uwtable ";
+  if (Attrs & Attribute::ReturnsTwice)
+    Result += "returns_twice ";
   if (Attrs & Attribute::InReg)
     Result += "inreg ";
   if (Attrs & Attribute::NoAlias)
@@ -72,8 +74,6 @@ std::string Attribute::getAsString(Attributes Attrs) {
     Result += "noimplicitfloat ";
   if (Attrs & Attribute::Naked)
     Result += "naked ";
-  if (Attrs & Attribute::Hotpatch)
-    Result += "hotpatch ";
   if (Attrs & Attribute::NonLazyBind)
     Result += "nonlazybind ";
   if (Attrs & Attribute::StackAlignment) {
@@ -92,7 +92,7 @@ std::string Attribute::getAsString(Attributes Attrs) {
   return Result;
 }
 
-Attributes Attribute::typeIncompatible(const Type *Ty) {
+Attributes Attribute::typeIncompatible(Type *Ty) {
   Attributes Incompatible = None;
   
   if (!Ty->isIntegerTy())
diff --git a/contrib/llvm/lib/VMCore/AutoUpgrade.cpp b/contrib/llvm/lib/VMCore/AutoUpgrade.cpp
index 9e93ff370e25..b849d3ef8dc2 100644
--- a/contrib/llvm/lib/VMCore/AutoUpgrade.cpp
+++ b/contrib/llvm/lib/VMCore/AutoUpgrade.cpp
@@ -14,11 +14,15 @@
 #include "llvm/AutoUpgrade.h"
 #include "llvm/Constants.h"
 #include "llvm/Function.h"
+#include "llvm/Instruction.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/Module.h"
 #include "llvm/IntrinsicInst.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/CallSite.h"
+#include "llvm/Support/CFG.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/IRBuilder.h"
 #include <cstring>
@@ -34,11 +38,48 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
     return false;
   Name = Name.substr(5); // Strip off "llvm."
 
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
   Module *M = F->getParent();
   
   switch (Name[0]) {
   default: break;
+  case 'a':
+    if (Name.startswith("atomic.cmp.swap") ||
+        Name.startswith("atomic.swap") ||
+        Name.startswith("atomic.load.add") ||
+        Name.startswith("atomic.load.sub") ||
+        Name.startswith("atomic.load.and") ||
+        Name.startswith("atomic.load.nand") ||
+        Name.startswith("atomic.load.or") ||
+        Name.startswith("atomic.load.xor") ||
+        Name.startswith("atomic.load.max") ||
+        Name.startswith("atomic.load.min") ||
+        Name.startswith("atomic.load.umax") ||
+        Name.startswith("atomic.load.umin"))
+      return true;
+  case 'i':
+    //  This upgrades the old llvm.init.trampoline to the new
+    //  llvm.init.trampoline and llvm.adjust.trampoline pair.
+    if (Name == "init.trampoline") {
+      // The new llvm.init.trampoline returns nothing.
+      if (FTy->getReturnType()->isVoidTy())
+        break;
+
+      assert(FTy->getNumParams() == 3 && "old init.trampoline takes 3 args!");
+
+      // Change the name of the old intrinsic so that we can play with its type.
+      std::string NameTmp = F->getName();
+      F->setName("");
+      NewFn = cast<Function>(M->getOrInsertFunction(
+                               NameTmp,
+                               Type::getVoidTy(M->getContext()),
+                               FTy->getParamType(0), FTy->getParamType(1),
+                               FTy->getParamType(2), (Type *)0));
+      return true;
+    }
+  case 'm':
+    if (Name == "memory.barrier")
+      return true;
   case 'p':
     //  This upgrades the llvm.prefetch intrinsic to accept one more parameter,
     //  which is a instruction / data cache identifier. The old version only
@@ -139,8 +180,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
         F->getName() == "llvm.x86.sse2.loadu.dq" ||
         F->getName() == "llvm.x86.sse2.loadu.pd") {
       // Convert to a native, unaligned load.
-      const Type *VecTy = CI->getType();
-      const Type *IntTy = IntegerType::get(C, 128);
+      Type *VecTy = CI->getType();
+      Type *IntTy = IntegerType::get(C, 128);
       IRBuilder<> Builder(C);
       Builder.SetInsertPoint(CI->getParent(), CI);
 
@@ -180,6 +221,80 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
       SI->setAlignment(16);
 
+      // Remove intrinsic.
+      CI->eraseFromParent();
+    } else if (F->getName().startswith("llvm.atomic.cmp.swap")) {
+      IRBuilder<> Builder(C);
+      Builder.SetInsertPoint(CI->getParent(), CI);
+      Value *Val = Builder.CreateAtomicCmpXchg(CI->getArgOperand(0),
+                                               CI->getArgOperand(1),
+                                               CI->getArgOperand(2),
+                                               Monotonic);
+
+      // Replace intrinsic.
+      Val->takeName(CI);
+      if (!CI->use_empty())
+        CI->replaceAllUsesWith(Val);
+      CI->eraseFromParent();
+    } else if (F->getName().startswith("llvm.atomic")) {
+      IRBuilder<> Builder(C);
+      Builder.SetInsertPoint(CI->getParent(), CI);
+
+      AtomicRMWInst::BinOp Op;
+      if (F->getName().startswith("llvm.atomic.swap"))
+        Op = AtomicRMWInst::Xchg;
+      else if (F->getName().startswith("llvm.atomic.load.add"))
+        Op = AtomicRMWInst::Add;
+      else if (F->getName().startswith("llvm.atomic.load.sub"))
+        Op = AtomicRMWInst::Sub;
+      else if (F->getName().startswith("llvm.atomic.load.and"))
+        Op = AtomicRMWInst::And;
+      else if (F->getName().startswith("llvm.atomic.load.nand"))
+        Op = AtomicRMWInst::Nand;
+      else if (F->getName().startswith("llvm.atomic.load.or"))
+        Op = AtomicRMWInst::Or;
+      else if (F->getName().startswith("llvm.atomic.load.xor"))
+        Op = AtomicRMWInst::Xor;
+      else if (F->getName().startswith("llvm.atomic.load.max"))
+        Op = AtomicRMWInst::Max;
+      else if (F->getName().startswith("llvm.atomic.load.min"))
+        Op = AtomicRMWInst::Min;
+      else if (F->getName().startswith("llvm.atomic.load.umax"))
+        Op = AtomicRMWInst::UMax;
+      else if (F->getName().startswith("llvm.atomic.load.umin"))
+        Op = AtomicRMWInst::UMin;
+      else
+        llvm_unreachable("Unknown atomic");
+
+      Value *Val = Builder.CreateAtomicRMW(Op, CI->getArgOperand(0),
+                                           CI->getArgOperand(1),
+                                           Monotonic);
+
+      // Replace intrinsic.
+      Val->takeName(CI);
+      if (!CI->use_empty())
+        CI->replaceAllUsesWith(Val);
+      CI->eraseFromParent();
+    } else if (F->getName() == "llvm.memory.barrier") {
+      IRBuilder<> Builder(C);
+      Builder.SetInsertPoint(CI->getParent(), CI);
+
+      // Note that this conversion ignores the "device" bit; it was not really
+      // well-defined, and got abused because nobody paid enough attention to
+      // get it right. In practice, this probably doesn't matter; application
+      // code generally doesn't need anything stronger than
+      // SequentiallyConsistent (and realistically, SequentiallyConsistent
+      // is lowered to a strong enough barrier for almost anything).
+
+      if (cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue())
+        Builder.CreateFence(SequentiallyConsistent);
+      else if (!cast<ConstantInt>(CI->getArgOperand(0))->getZExtValue())
+        Builder.CreateFence(Release);
+      else if (!cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue())
+        Builder.CreateFence(Acquire);
+      else
+        Builder.CreateFence(AcquireRelease);
+
       // Remove intrinsic.
       CI->eraseFromParent();
     } else {
@@ -192,7 +307,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
   case Intrinsic::prefetch: {
     IRBuilder<> Builder(C);
     Builder.SetInsertPoint(CI->getParent(), CI);
-    const llvm::Type *I32Ty = llvm::Type::getInt32Ty(CI->getContext());
+    llvm::Type *I32Ty = llvm::Type::getInt32Ty(CI->getContext());
 
     // Add the extra "data cache" argument
     Value *Operands[4] = { CI->getArgOperand(0), CI->getArgOperand(1),
@@ -212,6 +327,32 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
     CI->eraseFromParent();
     break;
   }
+  case Intrinsic::init_trampoline: {
+
+    //  Transform
+    //    %tramp = call i8* llvm.init.trampoline (i8* x, i8* y, i8* z)
+    //  to
+    //    call void llvm.init.trampoline (i8* %x, i8* %y, i8* %z)
+    //    %tramp = call i8* llvm.adjust.trampoline (i8* %x)
+
+    Function *AdjustTrampolineFn =
+      cast<Function>(Intrinsic::getDeclaration(F->getParent(),
+                                               Intrinsic::adjust_trampoline));
+
+    IRBuilder<> Builder(C);
+    Builder.SetInsertPoint(CI);
+
+    Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
+                        CI->getArgOperand(2));
+
+    CallInst *AdjustCall = Builder.CreateCall(AdjustTrampolineFn,
+                                              CI->getArgOperand(0),
+                                              CI->getName());
+    if (!CI->use_empty())
+      CI->replaceAllUsesWith(AdjustCall);
+    CI->eraseFromParent();
+    break;
+  }
   }
 }
 
@@ -279,3 +420,249 @@ void llvm::CheckDebugInfoIntrinsics(Module *M) {
     }
   }
 }
+
+/// FindExnAndSelIntrinsics - Find the eh_exception and eh_selector intrinsic
+/// calls reachable from the unwind basic block.
+static void FindExnAndSelIntrinsics(BasicBlock *BB, CallInst *&Exn,
+                                    CallInst *&Sel,
+                                    SmallPtrSet<BasicBlock*, 8> &Visited) {
+  if (!Visited.insert(BB)) return;
+
+  for (BasicBlock::iterator
+         I = BB->begin(), E = BB->end(); I != E; ++I) {
+    if (CallInst *CI = dyn_cast<CallInst>(I)) {
+      switch (CI->getCalledFunction()->getIntrinsicID()) {
+      default: break;
+      case Intrinsic::eh_exception:
+        assert(!Exn && "Found more than one eh.exception call!");
+        Exn = CI;
+        break;
+      case Intrinsic::eh_selector:
+        assert(!Sel && "Found more than one eh.selector call!");
+        Sel = CI;
+        break;
+      }
+
+      if (Exn && Sel) return;
+    }
+  }
+
+  if (Exn && Sel) return;
+
+  for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
+    FindExnAndSelIntrinsics(*I, Exn, Sel, Visited);
+    if (Exn && Sel) return;
+  }
+}
+
+/// TransferClausesToLandingPadInst - Transfer the exception handling clauses
+/// from the eh_selector call to the new landingpad instruction.
+static void TransferClausesToLandingPadInst(LandingPadInst *LPI,
+                                            CallInst *EHSel) {
+  LLVMContext &Context = LPI->getContext();
+  unsigned N = EHSel->getNumArgOperands();
+
+  for (unsigned i = N - 1; i > 1; --i) {
+    if (const ConstantInt *CI = dyn_cast<ConstantInt>(EHSel->getArgOperand(i))){
+      unsigned FilterLength = CI->getZExtValue();
+      unsigned FirstCatch = i + FilterLength + !FilterLength;
+      assert(FirstCatch <= N && "Invalid filter length");
+
+      if (FirstCatch < N)
+        for (unsigned j = FirstCatch; j < N; ++j) {
+          Value *Val = EHSel->getArgOperand(j);
+          if (!Val->hasName() || Val->getName() != "llvm.eh.catch.all.value") {
+            LPI->addClause(EHSel->getArgOperand(j));
+          } else {
+            GlobalVariable *GV = cast<GlobalVariable>(Val);
+            LPI->addClause(GV->getInitializer());
+          }
+        }
+
+      if (!FilterLength) {
+        // Cleanup.
+        LPI->setCleanup(true);
+      } else {
+        // Filter.
+        SmallVector<Constant *, 4> TyInfo;
+        TyInfo.reserve(FilterLength - 1);
+        for (unsigned j = i + 1; j < FirstCatch; ++j)
+          TyInfo.push_back(cast<Constant>(EHSel->getArgOperand(j)));
+        ArrayType *AType =
+          ArrayType::get(!TyInfo.empty() ? TyInfo[0]->getType() :
+                         PointerType::getUnqual(Type::getInt8Ty(Context)),
+                         TyInfo.size());
+        LPI->addClause(ConstantArray::get(AType, TyInfo));
+      }
+
+      N = i;
+    }
+  }
+
+  if (N > 2)
+    for (unsigned j = 2; j < N; ++j) {
+      Value *Val = EHSel->getArgOperand(j);
+      if (!Val->hasName() || Val->getName() != "llvm.eh.catch.all.value") {
+        LPI->addClause(EHSel->getArgOperand(j));
+      } else {
+        GlobalVariable *GV = cast<GlobalVariable>(Val);
+        LPI->addClause(GV->getInitializer());
+      }
+    }
+}
+
+/// This function upgrades the old pre-3.0 exception handling system to the new
+/// one. N.B. This will be removed in 3.1.
+void llvm::UpgradeExceptionHandling(Module *M) {
+  Function *EHException = M->getFunction("llvm.eh.exception");
+  Function *EHSelector = M->getFunction("llvm.eh.selector");
+  if (!EHException || !EHSelector)
+    return;
+
+  LLVMContext &Context = M->getContext();
+  Type *ExnTy = PointerType::getUnqual(Type::getInt8Ty(Context));
+  Type *SelTy = Type::getInt32Ty(Context);
+  Type *LPadSlotTy = StructType::get(ExnTy, SelTy, NULL);
+
+  // This map links the invoke instruction with the eh.exception and eh.selector
+  // calls associated with it.
+  DenseMap<InvokeInst*, std::pair<Value*, Value*> > InvokeToIntrinsicsMap;
+  for (Module::iterator
+         I = M->begin(), E = M->end(); I != E; ++I) {
+    Function &F = *I;
+
+    for (Function::iterator
+           II = F.begin(), IE = F.end(); II != IE; ++II) {
+      BasicBlock *BB = &*II;
+      InvokeInst *Inst = dyn_cast<InvokeInst>(BB->getTerminator());
+      if (!Inst) continue;
+      BasicBlock *UnwindDest = Inst->getUnwindDest();
+      if (UnwindDest->isLandingPad()) continue; // Already converted.
+
+      SmallPtrSet<BasicBlock*, 8> Visited;
+      CallInst *Exn = 0;
+      CallInst *Sel = 0;
+      FindExnAndSelIntrinsics(UnwindDest, Exn, Sel, Visited);
+      assert(Exn && Sel && "Cannot find eh.exception and eh.selector calls!");
+      InvokeToIntrinsicsMap[Inst] = std::make_pair(Exn, Sel);
+    }
+  }
+
+  // This map stores the slots where the exception object and selector value are
+  // stored within a function.
+  DenseMap<Function*, std::pair<Value*, Value*> > FnToLPadSlotMap;
+  SmallPtrSet<Instruction*, 32> DeadInsts;
+  for (DenseMap<InvokeInst*, std::pair<Value*, Value*> >::iterator
+         I = InvokeToIntrinsicsMap.begin(), E = InvokeToIntrinsicsMap.end();
+       I != E; ++I) {
+    InvokeInst *Invoke = I->first;
+    BasicBlock *UnwindDest = Invoke->getUnwindDest();
+    Function *F = UnwindDest->getParent();
+    std::pair<Value*, Value*> EHIntrinsics = I->second;
+    CallInst *Exn = cast<CallInst>(EHIntrinsics.first);
+    CallInst *Sel = cast<CallInst>(EHIntrinsics.second);
+
+    // Store the exception object and selector value in the entry block.
+    Value *ExnSlot = 0;
+    Value *SelSlot = 0;
+    if (!FnToLPadSlotMap[F].first) {
+      BasicBlock *Entry = &F->front();
+      ExnSlot = new AllocaInst(ExnTy, "exn", Entry->getTerminator());
+      SelSlot = new AllocaInst(SelTy, "sel", Entry->getTerminator());
+      FnToLPadSlotMap[F] = std::make_pair(ExnSlot, SelSlot);
+    } else {
+      ExnSlot = FnToLPadSlotMap[F].first;
+      SelSlot = FnToLPadSlotMap[F].second;
+    }
+
+    if (!UnwindDest->getSinglePredecessor()) {
+      // The unwind destination doesn't have a single predecessor. Create an
+      // unwind destination which has only one predecessor.
+      BasicBlock *NewBB = BasicBlock::Create(Context, "new.lpad",
+                                             UnwindDest->getParent());
+      BranchInst::Create(UnwindDest, NewBB);
+      Invoke->setUnwindDest(NewBB);
+
+      // Fix up any PHIs in the original unwind destination block.
+      for (BasicBlock::iterator
+             II = UnwindDest->begin(); isa<PHINode>(II); ++II) {
+        PHINode *PN = cast<PHINode>(II);
+        int Idx = PN->getBasicBlockIndex(Invoke->getParent());
+        if (Idx == -1) continue;
+        PN->setIncomingBlock(Idx, NewBB);
+      }
+
+      UnwindDest = NewBB;
+    }
+
+    IRBuilder<> Builder(Context);
+    Builder.SetInsertPoint(UnwindDest, UnwindDest->getFirstInsertionPt());
+
+    Value *PersFn = Sel->getArgOperand(1);
+    LandingPadInst *LPI = Builder.CreateLandingPad(LPadSlotTy, PersFn, 0);
+    Value *LPExn = Builder.CreateExtractValue(LPI, 0);
+    Value *LPSel = Builder.CreateExtractValue(LPI, 1);
+    Builder.CreateStore(LPExn, ExnSlot);
+    Builder.CreateStore(LPSel, SelSlot);
+
+    TransferClausesToLandingPadInst(LPI, Sel);
+
+    DeadInsts.insert(Exn);
+    DeadInsts.insert(Sel);
+  }
+
+  // Replace the old intrinsic calls with the values from the landingpad
+  // instruction(s). These values were stored in allocas for us to use here.
+  for (DenseMap<InvokeInst*, std::pair<Value*, Value*> >::iterator
+         I = InvokeToIntrinsicsMap.begin(), E = InvokeToIntrinsicsMap.end();
+       I != E; ++I) {
+    std::pair<Value*, Value*> EHIntrinsics = I->second;
+    CallInst *Exn = cast<CallInst>(EHIntrinsics.first);
+    CallInst *Sel = cast<CallInst>(EHIntrinsics.second);
+    BasicBlock *Parent = Exn->getParent();
+
+    std::pair<Value*,Value*> ExnSelSlots = FnToLPadSlotMap[Parent->getParent()];
+
+    IRBuilder<> Builder(Context);
+    Builder.SetInsertPoint(Parent, Exn);
+    LoadInst *LPExn = Builder.CreateLoad(ExnSelSlots.first, "exn.load");
+    LoadInst *LPSel = Builder.CreateLoad(ExnSelSlots.second, "sel.load");
+
+    Exn->replaceAllUsesWith(LPExn);
+    Sel->replaceAllUsesWith(LPSel);
+  }
+
+  // Remove the dead instructions.
+  for (SmallPtrSet<Instruction*, 32>::iterator
+         I = DeadInsts.begin(), E = DeadInsts.end(); I != E; ++I) {
+    Instruction *Inst = *I;
+    Inst->eraseFromParent();
+  }
+
+  // Replace calls to "llvm.eh.resume" with the 'resume' instruction. Load the
+  // exception and selector values from the stored place.
+  Function *EHResume = M->getFunction("llvm.eh.resume");
+  if (!EHResume) return;
+
+  while (!EHResume->use_empty()) {
+    CallInst *Resume = cast<CallInst>(EHResume->use_back());
+    BasicBlock *BB = Resume->getParent();
+
+    IRBuilder<> Builder(Context);
+    Builder.SetInsertPoint(BB, Resume);
+
+    Value *LPadVal =
+      Builder.CreateInsertValue(UndefValue::get(LPadSlotTy),
+                                Resume->getArgOperand(0), 0, "lpad.val");
+    LPadVal = Builder.CreateInsertValue(LPadVal, Resume->getArgOperand(1),
+                                        1, "lpad.val");
+    Builder.CreateResume(LPadVal);
+
+    // Remove all instructions after the 'resume.'
+    BasicBlock::iterator I = Resume;
+    while (I != BB->end()) {
+      Instruction *Inst = &*I++;
+      Inst->eraseFromParent();
+    }
+  }
+}
diff --git a/contrib/llvm/lib/VMCore/BasicBlock.cpp b/contrib/llvm/lib/VMCore/BasicBlock.cpp
index 70265c899d7e..d0aa275c8fe2 100644
--- a/contrib/llvm/lib/VMCore/BasicBlock.cpp
+++ b/contrib/llvm/lib/VMCore/BasicBlock.cpp
@@ -53,7 +53,7 @@ BasicBlock::BasicBlock(LLVMContext &C, const Twine &Name, Function *NewParent,
   } else if (NewParent) {
     NewParent->getBasicBlockList().push_back(this);
   }
-  
+
   setName(Name);
 }
 
@@ -76,7 +76,7 @@ BasicBlock::~BasicBlock() {
       BA->destroyConstant();
     }
   }
-  
+
   assert(getParent() == 0 && "BasicBlock still linked into the program!");
   dropAllReferences();
   InstList.clear();
@@ -167,6 +167,12 @@ Instruction* BasicBlock::getFirstNonPHIOrDbgOrLifetime() {
   return &*i;
 }
 
+BasicBlock::iterator BasicBlock::getFirstInsertionPt() {
+  iterator InsertPt = getFirstNonPHI();
+  if (isa<LandingPadInst>(InsertPt)) ++InsertPt;
+  return InsertPt;
+}
+
 void BasicBlock::dropAllReferences() {
   for(iterator I = begin(), E = end(); I != E; ++I)
     I->dropAllReferences();
@@ -184,8 +190,8 @@ BasicBlock *BasicBlock::getSinglePredecessor() {
 
 /// getUniquePredecessor - If this basic block has a unique predecessor block,
 /// return the block, otherwise return a null pointer.
-/// Note that unique predecessor doesn't mean single edge, there can be 
-/// multiple edges from the unique predecessor to this block (for example 
+/// Note that unique predecessor doesn't mean single edge, there can be
+/// multiple edges from the unique predecessor to this block (for example
 /// a switch statement with multiple cases having the same destination).
 BasicBlock *BasicBlock::getUniquePredecessor() {
   pred_iterator PI = pred_begin(this), E = pred_end(this);
@@ -336,11 +342,27 @@ void BasicBlock::replaceSuccessorsPhiUsesWith(BasicBlock *New) {
     return;
   for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
     BasicBlock *Succ = TI->getSuccessor(i);
-    for (iterator II = Succ->begin(); PHINode *PN = dyn_cast<PHINode>(II);
-         ++II) {
+    // N.B. Succ might not be a complete BasicBlock, so don't assume
+    // that it ends with a non-phi instruction.
+    for (iterator II = Succ->begin(), IE = Succ->end(); II != IE; ++II) {
+      PHINode *PN = dyn_cast<PHINode>(II);
+      if (!PN)
+        break;
       int i;
       while ((i = PN->getBasicBlockIndex(this)) >= 0)
         PN->setIncomingBlock(i, New);
     }
   }
 }
+
+/// isLandingPad - Return true if this basic block is a landing pad. I.e., it's
+/// the destination of the 'unwind' edge of an invoke instruction.
+bool BasicBlock::isLandingPad() const {
+  return isa<LandingPadInst>(getFirstNonPHI());
+}
+
+/// getLandingPadInst() - Return the landingpad instruction associated with
+/// the landing pad.
+LandingPadInst *BasicBlock::getLandingPadInst() {
+  return dyn_cast<LandingPadInst>(getFirstNonPHI());
+}
diff --git a/contrib/llvm/lib/VMCore/ConstantFold.cpp b/contrib/llvm/lib/VMCore/ConstantFold.cpp
index 323e2a280999..30bae7162cea 100644
--- a/contrib/llvm/lib/VMCore/ConstantFold.cpp
+++ b/contrib/llvm/lib/VMCore/ConstantFold.cpp
@@ -42,7 +42,7 @@ using namespace llvm;
 /// specified vector type.  At this point, we know that the elements of the
 /// input vector constant are all simple integer or FP values.
 static Constant *BitCastConstantVector(ConstantVector *CV,
-                                       const VectorType *DstTy) {
+                                       VectorType *DstTy) {
 
   if (CV->isAllOnesValue()) return Constant::getAllOnesValue(DstTy);
   if (CV->isNullValue()) return Constant::getNullValue(DstTy);
@@ -63,7 +63,7 @@ static Constant *BitCastConstantVector(ConstantVector *CV,
 
   // Bitcast each element now.
   std::vector<Constant*> Result;
-  const Type *DstEltTy = DstTy->getElementType();
+  Type *DstEltTy = DstTy->getElementType();
   for (unsigned i = 0; i != NumElts; ++i)
     Result.push_back(ConstantExpr::getBitCast(CV->getOperand(i),
                                                     DstEltTy));
@@ -78,15 +78,15 @@ static unsigned
 foldConstantCastPair(
   unsigned opc,          ///< opcode of the second cast constant expression
   ConstantExpr *Op,      ///< the first cast constant expression
-  const Type *DstTy      ///< desintation type of the first cast
+  Type *DstTy      ///< desintation type of the first cast
 ) {
   assert(Op && Op->isCast() && "Can't fold cast of cast without a cast!");
   assert(DstTy && DstTy->isFirstClassType() && "Invalid cast destination type");
   assert(CastInst::isCast(opc) && "Invalid cast opcode");
 
   // The the types and opcodes for the two Cast constant expressions
-  const Type *SrcTy = Op->getOperand(0)->getType();
-  const Type *MidTy = Op->getType();
+  Type *SrcTy = Op->getOperand(0)->getType();
+  Type *MidTy = Op->getType();
   Instruction::CastOps firstOp = Instruction::CastOps(Op->getOpcode());
   Instruction::CastOps secondOp = Instruction::CastOps(opc);
 
@@ -95,27 +95,27 @@ foldConstantCastPair(
                                         Type::getInt64Ty(DstTy->getContext()));
 }
 
-static Constant *FoldBitCast(Constant *V, const Type *DestTy) {
-  const Type *SrcTy = V->getType();
+static Constant *FoldBitCast(Constant *V, Type *DestTy) {
+  Type *SrcTy = V->getType();
   if (SrcTy == DestTy)
     return V; // no-op cast
 
   // Check to see if we are casting a pointer to an aggregate to a pointer to
   // the first element.  If so, return the appropriate GEP instruction.
-  if (const PointerType *PTy = dyn_cast<PointerType>(V->getType()))
-    if (const PointerType *DPTy = dyn_cast<PointerType>(DestTy))
+  if (PointerType *PTy = dyn_cast<PointerType>(V->getType()))
+    if (PointerType *DPTy = dyn_cast<PointerType>(DestTy))
       if (PTy->getAddressSpace() == DPTy->getAddressSpace()) {
         SmallVector<Value*, 8> IdxList;
         Value *Zero =
           Constant::getNullValue(Type::getInt32Ty(DPTy->getContext()));
         IdxList.push_back(Zero);
-        const Type *ElTy = PTy->getElementType();
+        Type *ElTy = PTy->getElementType();
         while (ElTy != DPTy->getElementType()) {
-          if (const StructType *STy = dyn_cast<StructType>(ElTy)) {
+          if (StructType *STy = dyn_cast<StructType>(ElTy)) {
             if (STy->getNumElements() == 0) break;
             ElTy = STy->getElementType(0);
             IdxList.push_back(Zero);
-          } else if (const SequentialType *STy = 
+          } else if (SequentialType *STy = 
                      dyn_cast<SequentialType>(ElTy)) {
             if (ElTy->isPointerTy()) break;  // Can't index into pointers!
             ElTy = STy->getElementType();
@@ -127,14 +127,13 @@ static Constant *FoldBitCast(Constant *V, const Type *DestTy) {
 
         if (ElTy == DPTy->getElementType())
           // This GEP is inbounds because all indices are zero.
-          return ConstantExpr::getInBoundsGetElementPtr(V, &IdxList[0],
-                                                        IdxList.size());
+          return ConstantExpr::getInBoundsGetElementPtr(V, IdxList);
       }
 
   // Handle casts from one vector constant to another.  We know that the src 
   // and dest type have the same size (otherwise its an illegal cast).
-  if (const VectorType *DestPTy = dyn_cast<VectorType>(DestTy)) {
-    if (const VectorType *SrcTy = dyn_cast<VectorType>(V->getType())) {
+  if (VectorType *DestPTy = dyn_cast<VectorType>(DestTy)) {
+    if (VectorType *SrcTy = dyn_cast<VectorType>(V->getType())) {
       assert(DestPTy->getBitWidth() == SrcTy->getBitWidth() &&
              "Not cast between same sized vectors!");
       SrcTy = NULL;
@@ -332,15 +331,15 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart,
 /// return null if no factoring was possible, to avoid endlessly
 /// bouncing an unfoldable expression back into the top-level folder.
 ///
-static Constant *getFoldedSizeOf(const Type *Ty, const Type *DestTy,
+static Constant *getFoldedSizeOf(Type *Ty, Type *DestTy,
                                  bool Folded) {
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+  if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Constant *N = ConstantInt::get(DestTy, ATy->getNumElements());
     Constant *E = getFoldedSizeOf(ATy->getElementType(), DestTy, true);
     return ConstantExpr::getNUWMul(E, N);
   }
 
-  if (const StructType *STy = dyn_cast<StructType>(Ty))
+  if (StructType *STy = dyn_cast<StructType>(Ty))
     if (!STy->isPacked()) {
       unsigned NumElems = STy->getNumElements();
       // An empty struct has size zero.
@@ -364,7 +363,7 @@ static Constant *getFoldedSizeOf(const Type *Ty, const Type *DestTy,
 
   // Pointer size doesn't depend on the pointee type, so canonicalize them
   // to an arbitrary pointee.
-  if (const PointerType *PTy = dyn_cast<PointerType>(Ty))
+  if (PointerType *PTy = dyn_cast<PointerType>(Ty))
     if (!PTy->getElementType()->isIntegerTy(1))
       return
         getFoldedSizeOf(PointerType::get(IntegerType::get(PTy->getContext(), 1),
@@ -389,11 +388,11 @@ static Constant *getFoldedSizeOf(const Type *Ty, const Type *DestTy,
 /// return null if no factoring was possible, to avoid endlessly
 /// bouncing an unfoldable expression back into the top-level folder.
 ///
-static Constant *getFoldedAlignOf(const Type *Ty, const Type *DestTy,
+static Constant *getFoldedAlignOf(Type *Ty, Type *DestTy,
                                   bool Folded) {
   // The alignment of an array is equal to the alignment of the
   // array element. Note that this is not always true for vectors.
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+  if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Constant *C = ConstantExpr::getAlignOf(ATy->getElementType());
     C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
                                                       DestTy,
@@ -402,7 +401,7 @@ static Constant *getFoldedAlignOf(const Type *Ty, const Type *DestTy,
     return C;
   }
 
-  if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+  if (StructType *STy = dyn_cast<StructType>(Ty)) {
     // Packed structs always have an alignment of 1.
     if (STy->isPacked())
       return ConstantInt::get(DestTy, 1);
@@ -429,7 +428,7 @@ static Constant *getFoldedAlignOf(const Type *Ty, const Type *DestTy,
 
   // Pointer alignment doesn't depend on the pointee type, so canonicalize them
   // to an arbitrary pointee.
-  if (const PointerType *PTy = dyn_cast<PointerType>(Ty))
+  if (PointerType *PTy = dyn_cast<PointerType>(Ty))
     if (!PTy->getElementType()->isIntegerTy(1))
       return
         getFoldedAlignOf(PointerType::get(IntegerType::get(PTy->getContext(),
@@ -455,10 +454,10 @@ static Constant *getFoldedAlignOf(const Type *Ty, const Type *DestTy,
 /// return null if no factoring was possible, to avoid endlessly
 /// bouncing an unfoldable expression back into the top-level folder.
 ///
-static Constant *getFoldedOffsetOf(const Type *Ty, Constant *FieldNo,
-                                   const Type *DestTy,
+static Constant *getFoldedOffsetOf(Type *Ty, Constant *FieldNo,
+                                   Type *DestTy,
                                    bool Folded) {
-  if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+  if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Constant *N = ConstantExpr::getCast(CastInst::getCastOpcode(FieldNo, false,
                                                                 DestTy, false),
                                         FieldNo, DestTy);
@@ -466,7 +465,7 @@ static Constant *getFoldedOffsetOf(const Type *Ty, Constant *FieldNo,
     return ConstantExpr::getNUWMul(E, N);
   }
 
-  if (const StructType *STy = dyn_cast<StructType>(Ty))
+  if (StructType *STy = dyn_cast<StructType>(Ty))
     if (!STy->isPacked()) {
       unsigned NumElems = STy->getNumElements();
       // An empty struct has no members.
@@ -506,7 +505,7 @@ static Constant *getFoldedOffsetOf(const Type *Ty, Constant *FieldNo,
 }
 
 Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
-                                            const Type *DestTy) {
+                                            Type *DestTy) {
   if (isa<UndefValue>(V)) {
     // zext(undef) = 0, because the top bits will be zero.
     // sext(undef) = 0, because the top bits will all be the same.
@@ -554,8 +553,8 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
         cast<VectorType>(DestTy)->getNumElements() ==
         CV->getType()->getNumElements()) {
       std::vector<Constant*> res;
-      const VectorType *DestVecTy = cast<VectorType>(DestTy);
-      const Type *DstEltTy = DestVecTy->getElementType();
+      VectorType *DestVecTy = cast<VectorType>(DestTy);
+      Type *DstEltTy = DestVecTy->getElementType();
       for (unsigned i = 0, e = CV->getType()->getNumElements(); i != e; ++i)
         res.push_back(ConstantExpr::getCast(opc,
                                             CV->getOperand(i), DstEltTy));
@@ -590,7 +589,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
       uint32_t DestBitWidth = cast<IntegerType>(DestTy)->getBitWidth();
       (void) V.convertToInteger(x, DestBitWidth, opc==Instruction::FPToSI,
                                 APFloat::rmTowardZero, &ignored);
-      APInt Val(DestBitWidth, 2, x);
+      APInt Val(DestBitWidth, x);
       return ConstantInt::get(FPC->getContext(), Val);
     }
     return 0; // Can't fold.
@@ -608,7 +607,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
       if (CE->getOpcode() == Instruction::GetElementPtr &&
           CE->getOperand(0)->isNullValue()) {
-        const Type *Ty =
+        Type *Ty =
           cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
         if (CE->getNumOperands() == 2) {
           // Handle a sizeof-like expression.
@@ -623,7 +622,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
         } else if (CE->getNumOperands() == 3 &&
                    CE->getOperand(1)->isNullValue()) {
           // Handle an alignof-like expression.
-          if (const StructType *STy = dyn_cast<StructType>(Ty))
+          if (StructType *STy = dyn_cast<StructType>(Ty))
             if (!STy->isPacked()) {
               ConstantInt *CI = cast<ConstantInt>(CE->getOperand(2));
               if (CI->isOne() &&
@@ -701,7 +700,7 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
 
     if (CondV->isAllOnesValue()) return V1;
 
-    const VectorType *VTy = cast<VectorType>(V1->getType());
+    VectorType *VTy = cast<VectorType>(V1->getType());
     ConstantVector *CP1 = dyn_cast<ConstantVector>(V1);
     ConstantVector *CP2 = dyn_cast<ConstantVector>(V2);
 
@@ -709,7 +708,7 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
         (CP2 || isa<ConstantAggregateZero>(V2))) {
 
       // Find the element type of the returned vector
-      const Type *EltTy = VTy->getElementType();
+      Type *EltTy = VTy->getElementType();
       unsigned NumElem = VTy->getNumElements();
       std::vector<Constant*> Res(NumElem);
 
@@ -762,10 +761,14 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val,
 
   if (ConstantVector *CVal = dyn_cast<ConstantVector>(Val)) {
     if (ConstantInt *CIdx = dyn_cast<ConstantInt>(Idx)) {
+      uint64_t Index = CIdx->getZExtValue();
+      if (Index >= CVal->getNumOperands())
+        // ee({w,x,y,z}, wrong_value) -> undef
+        return UndefValue::get(cast<VectorType>(Val->getType())->getElementType());
       return CVal->getOperand(CIdx->getZExtValue());
     } else if (isa<UndefValue>(Idx)) {
-      // ee({w,x,y,z}, undef) -> w (an arbitrary value).
-      return CVal->getOperand(0);
+      // ee({w,x,y,z}, undef) -> undef
+      return UndefValue::get(cast<VectorType>(Val->getType())->getElementType());
     }
   }
   return 0;
@@ -834,7 +837,7 @@ static Constant *GetVectorElement(Constant *C, unsigned EltNo) {
   if (ConstantVector *CV = dyn_cast<ConstantVector>(C))
     return CV->getOperand(EltNo);
 
-  const Type *EltTy = cast<VectorType>(C->getType())->getElementType();
+  Type *EltTy = cast<VectorType>(C->getType())->getElementType();
   if (isa<ConstantAggregateZero>(C))
     return Constant::getNullValue(EltTy);
   if (isa<UndefValue>(C))
@@ -850,7 +853,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
 
   unsigned MaskNumElts = cast<VectorType>(Mask->getType())->getNumElements();
   unsigned SrcNumElts = cast<VectorType>(V1->getType())->getNumElements();
-  const Type *EltTy = cast<VectorType>(V1->getType())->getElementType();
+  Type *EltTy = cast<VectorType>(V1->getType())->getElementType();
 
   // Loop over the shuffle mask, evaluating each element.
   SmallVector<Constant*, 32> Result;
@@ -922,16 +925,16 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg,
     
     // Otherwise break the aggregate undef into multiple undefs and do
     // the insertion.
-    const CompositeType *AggTy = cast<CompositeType>(Agg->getType());
+    CompositeType *AggTy = cast<CompositeType>(Agg->getType());
     unsigned numOps;
-    if (const ArrayType *AR = dyn_cast<ArrayType>(AggTy))
+    if (ArrayType *AR = dyn_cast<ArrayType>(AggTy))
       numOps = AR->getNumElements();
     else
       numOps = cast<StructType>(AggTy)->getNumElements();
     
     std::vector<Constant*> Ops(numOps); 
     for (unsigned i = 0; i < numOps; ++i) {
-      const Type *MemberTy = AggTy->getTypeAtIndex(i);
+      Type *MemberTy = AggTy->getTypeAtIndex(i);
       Constant *Op =
         (Idxs[0] == i) ?
         ConstantFoldInsertValueInstruction(UndefValue::get(MemberTy),
@@ -940,7 +943,7 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg,
       Ops[i] = Op;
     }
     
-    if (const StructType* ST = dyn_cast<StructType>(AggTy))
+    if (StructType* ST = dyn_cast<StructType>(AggTy))
       return ConstantStruct::get(ST, Ops);
     return ConstantArray::get(cast<ArrayType>(AggTy), Ops);
   }
@@ -953,16 +956,16 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg,
     
     // Otherwise break the aggregate zero into multiple zeros and do
     // the insertion.
-    const CompositeType *AggTy = cast<CompositeType>(Agg->getType());
+    CompositeType *AggTy = cast<CompositeType>(Agg->getType());
     unsigned numOps;
-    if (const ArrayType *AR = dyn_cast<ArrayType>(AggTy))
+    if (ArrayType *AR = dyn_cast<ArrayType>(AggTy))
       numOps = AR->getNumElements();
     else
       numOps = cast<StructType>(AggTy)->getNumElements();
     
     std::vector<Constant*> Ops(numOps);
     for (unsigned i = 0; i < numOps; ++i) {
-      const Type *MemberTy = AggTy->getTypeAtIndex(i);
+      Type *MemberTy = AggTy->getTypeAtIndex(i);
       Constant *Op =
         (Idxs[0] == i) ?
         ConstantFoldInsertValueInstruction(Constant::getNullValue(MemberTy),
@@ -971,7 +974,7 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg,
       Ops[i] = Op;
     }
     
-    if (const StructType *ST = dyn_cast<StructType>(AggTy))
+    if (StructType *ST = dyn_cast<StructType>(AggTy))
       return ConstantStruct::get(ST, Ops);
     return ConstantArray::get(cast<ArrayType>(AggTy), Ops);
   }
@@ -986,7 +989,7 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg,
       Ops[i] = Op;
     }
     
-    if (const StructType* ST = dyn_cast<StructType>(Agg->getType()))
+    if (StructType* ST = dyn_cast<StructType>(Agg->getType()))
       return ConstantStruct::get(ST, Ops);
     return ConstantArray::get(cast<ArrayType>(Agg->getType()), Ops);
   }
@@ -1265,13 +1268,13 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode,
         return ConstantFP::get(C1->getContext(), C3V);
       }
     }
-  } else if (const VectorType *VTy = dyn_cast<VectorType>(C1->getType())) {
+  } else if (VectorType *VTy = dyn_cast<VectorType>(C1->getType())) {
     ConstantVector *CP1 = dyn_cast<ConstantVector>(C1);
     ConstantVector *CP2 = dyn_cast<ConstantVector>(C2);
     if ((CP1 != NULL || isa<ConstantAggregateZero>(C1)) &&
         (CP2 != NULL || isa<ConstantAggregateZero>(C2))) {
       std::vector<Constant*> Res;
-      const Type* EltTy = VTy->getElementType();  
+      Type* EltTy = VTy->getElementType();  
       Constant *C1 = 0;
       Constant *C2 = 0;
       switch (Opcode) {
@@ -1461,8 +1464,8 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode,
 
 /// isZeroSizedType - This type is zero sized if its an array or structure of
 /// zero sized types.  The only leaf zero sized type is an empty structure.
-static bool isMaybeZeroSizedType(const Type *Ty) {
-  if (const StructType *STy = dyn_cast<StructType>(Ty)) {
+static bool isMaybeZeroSizedType(Type *Ty) {
+  if (StructType *STy = dyn_cast<StructType>(Ty)) {
     if (STy->isOpaque()) return true;  // Can't say.
 
     // If all of elements have zero size, this does too.
@@ -1470,7 +1473,7 @@ static bool isMaybeZeroSizedType(const Type *Ty) {
       if (!isMaybeZeroSizedType(STy->getElementType(i))) return false;
     return true;
 
-  } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+  } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     return isMaybeZeroSizedType(ATy->getElementType());
   }
   return false;
@@ -1483,7 +1486,7 @@ static bool isMaybeZeroSizedType(const Type *Ty) {
 /// first is less than the second, return -1, if the second is less than the
 /// first, return 1.  If the constants are not integral, return -2.
 ///
-static int IdxCompare(Constant *C1, Constant *C2, const Type *ElTy) {
+static int IdxCompare(Constant *C1, Constant *C2, Type *ElTy) {
   if (C1 == C2) return 0;
 
   // Ok, we found a different index.  If they are not ConstantInt, we can't do
@@ -1832,8 +1835,8 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2,
 
 Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, 
                                                Constant *C1, Constant *C2) {
-  const Type *ResultTy;
-  if (const VectorType *VT = dyn_cast<VectorType>(C1->getType()))
+  Type *ResultTy;
+  if (VectorType *VT = dyn_cast<VectorType>(C1->getType()))
     ResultTy = VectorType::get(Type::getInt1Ty(C1->getContext()),
                                VT->getNumElements());
   else
@@ -2146,9 +2149,9 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
 /// isInBoundsIndices - Test whether the given sequence of *normalized* indices
 /// is "inbounds".
 template<typename IndexTy>
-static bool isInBoundsIndices(IndexTy const *Idxs, size_t NumIdx) {
+static bool isInBoundsIndices(ArrayRef<IndexTy> Idxs) {
   // No indices means nothing that could be out of bounds.
-  if (NumIdx == 0) return true;
+  if (Idxs.empty()) return true;
 
   // If the first index is zero, it's in bounds.
   if (cast<Constant>(Idxs[0])->isNullValue()) return true;
@@ -2157,7 +2160,7 @@ static bool isInBoundsIndices(IndexTy const *Idxs, size_t NumIdx) {
   // by the one-past-the-end rule.
   if (!cast<ConstantInt>(Idxs[0])->isOne())
     return false;
-  for (unsigned i = 1, e = NumIdx; i != e; ++i)
+  for (unsigned i = 1, e = Idxs.size(); i != e; ++i)
     if (!cast<Constant>(Idxs[i])->isNullValue())
       return false;
   return true;
@@ -2166,31 +2169,29 @@ static bool isInBoundsIndices(IndexTy const *Idxs, size_t NumIdx) {
 template<typename IndexTy>
 static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
                                                bool inBounds,
-                                               IndexTy const *Idxs,
-                                               unsigned NumIdx) {
-  if (NumIdx == 0) return C;
+                                               ArrayRef<IndexTy> Idxs) {
+  if (Idxs.empty()) return C;
   Constant *Idx0 = cast<Constant>(Idxs[0]);
-  if ((NumIdx == 1 && Idx0->isNullValue()))
+  if ((Idxs.size() == 1 && Idx0->isNullValue()))
     return C;
 
   if (isa<UndefValue>(C)) {
-    const PointerType *Ptr = cast<PointerType>(C->getType());
-    const Type *Ty = GetElementPtrInst::getIndexedType(Ptr, Idxs, Idxs+NumIdx);
+    PointerType *Ptr = cast<PointerType>(C->getType());
+    Type *Ty = GetElementPtrInst::getIndexedType(Ptr, Idxs);
     assert(Ty != 0 && "Invalid indices for GEP!");
     return UndefValue::get(PointerType::get(Ty, Ptr->getAddressSpace()));
   }
 
   if (C->isNullValue()) {
     bool isNull = true;
-    for (unsigned i = 0, e = NumIdx; i != e; ++i)
+    for (unsigned i = 0, e = Idxs.size(); i != e; ++i)
       if (!cast<Constant>(Idxs[i])->isNullValue()) {
         isNull = false;
         break;
       }
     if (isNull) {
-      const PointerType *Ptr = cast<PointerType>(C->getType());
-      const Type *Ty = GetElementPtrInst::getIndexedType(Ptr, Idxs,
-                                                         Idxs+NumIdx);
+      PointerType *Ptr = cast<PointerType>(C->getType());
+      Type *Ty = GetElementPtrInst::getIndexedType(Ptr, Idxs);
       assert(Ty != 0 && "Invalid indices for GEP!");
       return ConstantPointerNull::get(PointerType::get(Ty,
                                                        Ptr->getAddressSpace()));
@@ -2203,14 +2204,14 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
     // getelementptr instructions into a single instruction.
     //
     if (CE->getOpcode() == Instruction::GetElementPtr) {
-      const Type *LastTy = 0;
+      Type *LastTy = 0;
       for (gep_type_iterator I = gep_type_begin(CE), E = gep_type_end(CE);
            I != E; ++I)
         LastTy = *I;
 
       if ((LastTy && LastTy->isArrayTy()) || Idx0->isNullValue()) {
         SmallVector<Value*, 16> NewIndices;
-        NewIndices.reserve(NumIdx + CE->getNumOperands());
+        NewIndices.reserve(Idxs.size() + CE->getNumOperands());
         for (unsigned i = 1, e = CE->getNumOperands()-1; i != e; ++i)
           NewIndices.push_back(CE->getOperand(i));
 
@@ -2219,9 +2220,9 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
         Constant *Combined = CE->getOperand(CE->getNumOperands()-1);
         // Otherwise it must be an array.
         if (!Idx0->isNullValue()) {
-          const Type *IdxTy = Combined->getType();
+          Type *IdxTy = Combined->getType();
           if (IdxTy != Idx0->getType()) {
-            const Type *Int64Ty = Type::getInt64Ty(IdxTy->getContext());
+            Type *Int64Ty = Type::getInt64Ty(IdxTy->getContext());
             Constant *C1 = ConstantExpr::getSExtOrBitCast(Idx0, Int64Ty);
             Constant *C2 = ConstantExpr::getSExtOrBitCast(Combined, Int64Ty);
             Combined = ConstantExpr::get(Instruction::Add, C1, C2);
@@ -2232,14 +2233,11 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
         }
 
         NewIndices.push_back(Combined);
-        NewIndices.append(Idxs+1, Idxs+NumIdx);
-        return (inBounds && cast<GEPOperator>(CE)->isInBounds()) ?
-          ConstantExpr::getInBoundsGetElementPtr(CE->getOperand(0),
-                                                 &NewIndices[0],
-                                                 NewIndices.size()) :
-          ConstantExpr::getGetElementPtr(CE->getOperand(0),
-                                         &NewIndices[0],
-                                         NewIndices.size());
+        NewIndices.append(Idxs.begin() + 1, Idxs.end());
+        return
+          ConstantExpr::getGetElementPtr(CE->getOperand(0), NewIndices,
+                                         inBounds &&
+                                           cast<GEPOperator>(CE)->isInBounds());
       }
     }
 
@@ -2248,18 +2246,16 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
     //                        i64 0, i64 0)
     // To: i32* getelementptr ([3 x i32]* %X, i64 0, i64 0)
     //
-    if (CE->isCast() && NumIdx > 1 && Idx0->isNullValue()) {
-      if (const PointerType *SPT =
+    if (CE->isCast() && Idxs.size() > 1 && Idx0->isNullValue()) {
+      if (PointerType *SPT =
           dyn_cast<PointerType>(CE->getOperand(0)->getType()))
-        if (const ArrayType *SAT = dyn_cast<ArrayType>(SPT->getElementType()))
-          if (const ArrayType *CAT =
+        if (ArrayType *SAT = dyn_cast<ArrayType>(SPT->getElementType()))
+          if (ArrayType *CAT =
         dyn_cast<ArrayType>(cast<PointerType>(C->getType())->getElementType()))
             if (CAT->getElementType() == SAT->getElementType())
-              return inBounds ?
-                ConstantExpr::getInBoundsGetElementPtr(
-                      (Constant*)CE->getOperand(0), Idxs, NumIdx) :
-                ConstantExpr::getGetElementPtr(
-                      (Constant*)CE->getOperand(0), Idxs, NumIdx);
+              return
+                ConstantExpr::getGetElementPtr((Constant*)CE->getOperand(0),
+                                               Idxs, inBounds);
     }
   }
 
@@ -2268,19 +2264,19 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
   // out into preceding dimensions.
   bool Unknown = false;
   SmallVector<Constant *, 8> NewIdxs;
-  const Type *Ty = C->getType();
-  const Type *Prev = 0;
-  for (unsigned i = 0; i != NumIdx;
+  Type *Ty = C->getType();
+  Type *Prev = 0;
+  for (unsigned i = 0, e = Idxs.size(); i != e;
        Prev = Ty, Ty = cast<CompositeType>(Ty)->getTypeAtIndex(Idxs[i]), ++i) {
     if (ConstantInt *CI = dyn_cast<ConstantInt>(Idxs[i])) {
-      if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty))
+      if (ArrayType *ATy = dyn_cast<ArrayType>(Ty))
         if (ATy->getNumElements() <= INT64_MAX &&
             ATy->getNumElements() != 0 &&
             CI->getSExtValue() >= (int64_t)ATy->getNumElements()) {
           if (isa<SequentialType>(Prev)) {
             // It's out of range, but we can factor it into the prior
             // dimension.
-            NewIdxs.resize(NumIdx);
+            NewIdxs.resize(Idxs.size());
             ConstantInt *Factor = ConstantInt::get(CI->getType(),
                                                    ATy->getNumElements());
             NewIdxs[i] = ConstantExpr::getSRem(CI, Factor);
@@ -2312,33 +2308,28 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
 
   // If we did any factoring, start over with the adjusted indices.
   if (!NewIdxs.empty()) {
-    for (unsigned i = 0; i != NumIdx; ++i)
+    for (unsigned i = 0, e = Idxs.size(); i != e; ++i)
       if (!NewIdxs[i]) NewIdxs[i] = cast<Constant>(Idxs[i]);
-    return inBounds ?
-      ConstantExpr::getInBoundsGetElementPtr(C, NewIdxs.data(),
-                                             NewIdxs.size()) :
-      ConstantExpr::getGetElementPtr(C, NewIdxs.data(), NewIdxs.size());
+    return ConstantExpr::getGetElementPtr(C, NewIdxs, inBounds);
   }
 
   // If all indices are known integers and normalized, we can do a simple
   // check for the "inbounds" property.
   if (!Unknown && !inBounds &&
-      isa<GlobalVariable>(C) && isInBoundsIndices(Idxs, NumIdx))
-    return ConstantExpr::getInBoundsGetElementPtr(C, Idxs, NumIdx);
+      isa<GlobalVariable>(C) && isInBoundsIndices(Idxs))
+    return ConstantExpr::getInBoundsGetElementPtr(C, Idxs);
 
   return 0;
 }
 
 Constant *llvm::ConstantFoldGetElementPtr(Constant *C,
                                           bool inBounds,
-                                          Constant* const *Idxs,
-                                          unsigned NumIdx) {
-  return ConstantFoldGetElementPtrImpl(C, inBounds, Idxs, NumIdx);
+                                          ArrayRef<Constant *> Idxs) {
+  return ConstantFoldGetElementPtrImpl(C, inBounds, Idxs);
 }
 
 Constant *llvm::ConstantFoldGetElementPtr(Constant *C,
                                           bool inBounds,
-                                          Value* const *Idxs,
-                                          unsigned NumIdx) {
-  return ConstantFoldGetElementPtrImpl(C, inBounds, Idxs, NumIdx);
+                                          ArrayRef<Value *> Idxs) {
+  return ConstantFoldGetElementPtrImpl(C, inBounds, Idxs);
 }
diff --git a/contrib/llvm/lib/VMCore/ConstantFold.h b/contrib/llvm/lib/VMCore/ConstantFold.h
index 653a1c3f377d..e12f27a7cb1e 100644
--- a/contrib/llvm/lib/VMCore/ConstantFold.h
+++ b/contrib/llvm/lib/VMCore/ConstantFold.h
@@ -30,7 +30,7 @@ namespace llvm {
   Constant *ConstantFoldCastInstruction(
     unsigned opcode,     ///< The opcode of the cast
     Constant *V,         ///< The source constant
-    const Type *DestTy   ///< The destination type
+    Type *DestTy   ///< The destination type
   );
   Constant *ConstantFoldSelectInstruction(Constant *Cond,
                                           Constant *V1, Constant *V2);
@@ -48,9 +48,9 @@ namespace llvm {
   Constant *ConstantFoldCompareInstruction(unsigned short predicate, 
                                            Constant *C1, Constant *C2);
   Constant *ConstantFoldGetElementPtr(Constant *C, bool inBounds,
-                                      Constant* const *Idxs, unsigned NumIdx);
+                                      ArrayRef<Constant *> Idxs);
   Constant *ConstantFoldGetElementPtr(Constant *C, bool inBounds,
-                                      Value* const *Idxs, unsigned NumIdx);
+                                      ArrayRef<Value *> Idxs);
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm/lib/VMCore/Constants.cpp b/contrib/llvm/lib/VMCore/Constants.cpp
index 316c8846f94f..a84a046bb998 100644
--- a/contrib/llvm/lib/VMCore/Constants.cpp
+++ b/contrib/llvm/lib/VMCore/Constants.cpp
@@ -62,8 +62,23 @@ bool Constant::isNullValue() const {
   return isa<ConstantAggregateZero>(this) || isa<ConstantPointerNull>(this);
 }
 
+bool Constant::isAllOnesValue() const {
+  // Check for -1 integers
+  if (const ConstantInt *CI = dyn_cast<ConstantInt>(this))
+    return CI->isMinusOne();
+
+  // Check for FP which are bitcasted from -1 integers
+  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this))
+    return CFP->getValueAPF().bitcastToAPInt().isAllOnesValue();
+
+  // Check for constant vectors
+  if (const ConstantVector *CV = dyn_cast<ConstantVector>(this))
+    return CV->isAllOnesValue();
+
+  return false;
+}
 // Constructor to create a '0' constant of arbitrary type...
-Constant *Constant::getNullValue(const Type *Ty) {
+Constant *Constant::getNullValue(Type *Ty) {
   switch (Ty->getTypeID()) {
   case Type::IntegerTyID:
     return ConstantInt::get(Ty, 0);
@@ -90,30 +105,30 @@ Constant *Constant::getNullValue(const Type *Ty) {
     return ConstantAggregateZero::get(Ty);
   default:
     // Function, Label, or Opaque type?
-    assert(!"Cannot create a null constant of that type!");
+    assert(0 && "Cannot create a null constant of that type!");
     return 0;
   }
 }
 
-Constant *Constant::getIntegerValue(const Type *Ty, const APInt &V) {
-  const Type *ScalarTy = Ty->getScalarType();
+Constant *Constant::getIntegerValue(Type *Ty, const APInt &V) {
+  Type *ScalarTy = Ty->getScalarType();
 
   // Create the base integer constant.
   Constant *C = ConstantInt::get(Ty->getContext(), V);
 
   // Convert an integer to a pointer, if necessary.
-  if (const PointerType *PTy = dyn_cast<PointerType>(ScalarTy))
+  if (PointerType *PTy = dyn_cast<PointerType>(ScalarTy))
     C = ConstantExpr::getIntToPtr(C, PTy);
 
   // Broadcast a scalar to a vector, if necessary.
-  if (const VectorType *VTy = dyn_cast<VectorType>(Ty))
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
     C = ConstantVector::get(std::vector<Constant *>(VTy->getNumElements(), C));
 
   return C;
 }
 
-Constant *Constant::getAllOnesValue(const Type *Ty) {
-  if (const IntegerType *ITy = dyn_cast<IntegerType>(Ty))
+Constant *Constant::getAllOnesValue(Type *Ty) {
+  if (IntegerType *ITy = dyn_cast<IntegerType>(Ty))
     return ConstantInt::get(Ty->getContext(),
                             APInt::getAllOnesValue(ITy->getBitWidth()));
 
@@ -124,9 +139,9 @@ Constant *Constant::getAllOnesValue(const Type *Ty) {
   }
 
   SmallVector<Constant*, 16> Elts;
-  const VectorType *VTy = cast<VectorType>(Ty);
+  VectorType *VTy = cast<VectorType>(Ty);
   Elts.resize(VTy->getNumElements(), getAllOnesValue(VTy->getElementType()));
-  assert(Elts[0] && "Not a vector integer type!");
+  assert(Elts[0] && "Invalid AllOnes value!");
   return cast<ConstantVector>(ConstantVector::get(Elts));
 }
 
@@ -269,7 +284,7 @@ void Constant::getVectorElements(SmallVectorImpl<Constant*> &Elts) const {
     return;
   }
   
-  const VectorType *VT = cast<VectorType>(getType());
+  VectorType *VT = cast<VectorType>(getType());
   if (isa<ConstantAggregateZero>(this)) {
     Elts.assign(VT->getNumElements(), 
                 Constant::getNullValue(VT->getElementType()));
@@ -343,7 +358,7 @@ void Constant::removeDeadConstantUsers() const {
 //                                ConstantInt
 //===----------------------------------------------------------------------===//
 
-ConstantInt::ConstantInt(const IntegerType *Ty, const APInt& V)
+ConstantInt::ConstantInt(IntegerType *Ty, const APInt& V)
   : Constant(Ty, ConstantIntVal, 0, 0), Val(V) {
   assert(V.getBitWidth() == Ty->getBitWidth() && "Invalid constant for type");
 }
@@ -362,8 +377,8 @@ ConstantInt *ConstantInt::getFalse(LLVMContext &Context) {
   return pImpl->TheFalseVal;
 }
 
-Constant *ConstantInt::getTrue(const Type *Ty) {
-  const VectorType *VTy = dyn_cast<VectorType>(Ty);
+Constant *ConstantInt::getTrue(Type *Ty) {
+  VectorType *VTy = dyn_cast<VectorType>(Ty);
   if (!VTy) {
     assert(Ty->isIntegerTy(1) && "True must be i1 or vector of i1.");
     return ConstantInt::getTrue(Ty->getContext());
@@ -375,8 +390,8 @@ Constant *ConstantInt::getTrue(const Type *Ty) {
   return ConstantVector::get(Splat);
 }
 
-Constant *ConstantInt::getFalse(const Type *Ty) {
-  const VectorType *VTy = dyn_cast<VectorType>(Ty);
+Constant *ConstantInt::getFalse(Type *Ty) {
+  VectorType *VTy = dyn_cast<VectorType>(Ty);
   if (!VTy) {
     assert(Ty->isIntegerTy(1) && "False must be i1 or vector of i1.");
     return ConstantInt::getFalse(Ty->getContext());
@@ -396,7 +411,7 @@ Constant *ConstantInt::getFalse(const Type *Ty) {
 // invariant which generates an assertion.
 ConstantInt *ConstantInt::get(LLVMContext &Context, const APInt &V) {
   // Get the corresponding integer type for the bit width of the value.
-  const IntegerType *ITy = IntegerType::get(Context, V.getBitWidth());
+  IntegerType *ITy = IntegerType::get(Context, V.getBitWidth());
   // get an existing value or the insertion position
   DenseMapAPIntKeyInfo::KeyTy Key(V, ITy);
   ConstantInt *&Slot = Context.pImpl->IntConstants[Key]; 
@@ -404,44 +419,44 @@ ConstantInt *ConstantInt::get(LLVMContext &Context, const APInt &V) {
   return Slot;
 }
 
-Constant *ConstantInt::get(const Type *Ty, uint64_t V, bool isSigned) {
+Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) {
   Constant *C = get(cast<IntegerType>(Ty->getScalarType()), V, isSigned);
 
   // For vectors, broadcast the value.
-  if (const VectorType *VTy = dyn_cast<VectorType>(Ty))
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
     return ConstantVector::get(SmallVector<Constant*,
                                            16>(VTy->getNumElements(), C));
 
   return C;
 }
 
-ConstantInt* ConstantInt::get(const IntegerType* Ty, uint64_t V, 
+ConstantInt* ConstantInt::get(IntegerType* Ty, uint64_t V, 
                               bool isSigned) {
   return get(Ty->getContext(), APInt(Ty->getBitWidth(), V, isSigned));
 }
 
-ConstantInt* ConstantInt::getSigned(const IntegerType* Ty, int64_t V) {
+ConstantInt* ConstantInt::getSigned(IntegerType* Ty, int64_t V) {
   return get(Ty, V, true);
 }
 
-Constant *ConstantInt::getSigned(const Type *Ty, int64_t V) {
+Constant *ConstantInt::getSigned(Type *Ty, int64_t V) {
   return get(Ty, V, true);
 }
 
-Constant *ConstantInt::get(const Type* Ty, const APInt& V) {
+Constant *ConstantInt::get(Type* Ty, const APInt& V) {
   ConstantInt *C = get(Ty->getContext(), V);
   assert(C->getType() == Ty->getScalarType() &&
          "ConstantInt type doesn't match the type implied by its value!");
 
   // For vectors, broadcast the value.
-  if (const VectorType *VTy = dyn_cast<VectorType>(Ty))
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
     return ConstantVector::get(
       SmallVector<Constant *, 16>(VTy->getNumElements(), C));
 
   return C;
 }
 
-ConstantInt* ConstantInt::get(const IntegerType* Ty, StringRef Str,
+ConstantInt* ConstantInt::get(IntegerType* Ty, StringRef Str,
                               uint8_t radix) {
   return get(Ty->getContext(), APInt(Ty->getBitWidth(), Str, radix));
 }
@@ -450,7 +465,7 @@ ConstantInt* ConstantInt::get(const IntegerType* Ty, StringRef Str,
 //                                ConstantFP
 //===----------------------------------------------------------------------===//
 
-static const fltSemantics *TypeToFloatSemantics(const Type *Ty) {
+static const fltSemantics *TypeToFloatSemantics(Type *Ty) {
   if (Ty->isFloatTy())
     return &APFloat::IEEEsingle;
   if (Ty->isDoubleTy())
@@ -467,7 +482,7 @@ static const fltSemantics *TypeToFloatSemantics(const Type *Ty) {
 /// get() - This returns a constant fp for the specified value in the
 /// specified type.  This should only be used for simple constant values like
 /// 2.0/1.0 etc, that are known-valid both as double and as the target format.
-Constant *ConstantFP::get(const Type* Ty, double V) {
+Constant *ConstantFP::get(Type* Ty, double V) {
   LLVMContext &Context = Ty->getContext();
   
   APFloat FV(V);
@@ -477,7 +492,7 @@ Constant *ConstantFP::get(const Type* Ty, double V) {
   Constant *C = get(Context, FV);
 
   // For vectors, broadcast the value.
-  if (const VectorType *VTy = dyn_cast<VectorType>(Ty))
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
     return ConstantVector::get(
       SmallVector<Constant *, 16>(VTy->getNumElements(), C));
 
@@ -485,14 +500,14 @@ Constant *ConstantFP::get(const Type* Ty, double V) {
 }
 
 
-Constant *ConstantFP::get(const Type* Ty, StringRef Str) {
+Constant *ConstantFP::get(Type* Ty, StringRef Str) {
   LLVMContext &Context = Ty->getContext();
 
   APFloat FV(*TypeToFloatSemantics(Ty->getScalarType()), Str);
   Constant *C = get(Context, FV);
 
   // For vectors, broadcast the value.
-  if (const VectorType *VTy = dyn_cast<VectorType>(Ty))
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
     return ConstantVector::get(
       SmallVector<Constant *, 16>(VTy->getNumElements(), C));
 
@@ -500,7 +515,7 @@ Constant *ConstantFP::get(const Type* Ty, StringRef Str) {
 }
 
 
-ConstantFP* ConstantFP::getNegativeZero(const Type* Ty) {
+ConstantFP* ConstantFP::getNegativeZero(Type* Ty) {
   LLVMContext &Context = Ty->getContext();
   APFloat apf = cast <ConstantFP>(Constant::getNullValue(Ty))->getValueAPF();
   apf.changeSign();
@@ -508,8 +523,8 @@ ConstantFP* ConstantFP::getNegativeZero(const Type* Ty) {
 }
 
 
-Constant *ConstantFP::getZeroValueForNegation(const Type* Ty) {
-  if (const VectorType *PTy = dyn_cast<VectorType>(Ty))
+Constant *ConstantFP::getZeroValueForNegation(Type* Ty) {
+  if (VectorType *PTy = dyn_cast<VectorType>(Ty))
     if (PTy->getElementType()->isFloatingPointTy()) {
       SmallVector<Constant*, 16> zeros(PTy->getNumElements(),
                            getNegativeZero(PTy->getElementType()));
@@ -532,7 +547,7 @@ ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) {
   ConstantFP *&Slot = pImpl->FPConstants[Key];
     
   if (!Slot) {
-    const Type *Ty;
+    Type *Ty;
     if (&V.getSemantics() == &APFloat::IEEEsingle)
       Ty = Type::getFloatTy(Context);
     else if (&V.getSemantics() == &APFloat::IEEEdouble)
@@ -552,13 +567,13 @@ ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) {
   return Slot;
 }
 
-ConstantFP *ConstantFP::getInfinity(const Type *Ty, bool Negative) {
+ConstantFP *ConstantFP::getInfinity(Type *Ty, bool Negative) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty);
   return ConstantFP::get(Ty->getContext(),
                          APFloat::getInf(Semantics, Negative));
 }
 
-ConstantFP::ConstantFP(const Type *Ty, const APFloat& V)
+ConstantFP::ConstantFP(Type *Ty, const APFloat& V)
   : Constant(Ty, ConstantFPVal, 0, 0), Val(V) {
   assert(&V.getSemantics() == TypeToFloatSemantics(Ty) &&
          "FP type Mismatch");
@@ -573,24 +588,19 @@ bool ConstantFP::isExactlyValue(const APFloat &V) const {
 //===----------------------------------------------------------------------===//
 
 
-ConstantArray::ConstantArray(const ArrayType *T,
-                             const std::vector<Constant*> &V)
+ConstantArray::ConstantArray(ArrayType *T, ArrayRef<Constant *> V)
   : Constant(T, ConstantArrayVal,
              OperandTraits<ConstantArray>::op_end(this) - V.size(),
              V.size()) {
   assert(V.size() == T->getNumElements() &&
          "Invalid initializer vector for constant array");
-  Use *OL = OperandList;
-  for (std::vector<Constant*>::const_iterator I = V.begin(), E = V.end();
-       I != E; ++I, ++OL) {
-    Constant *C = *I;
-    assert(C->getType() == T->getElementType() &&
+  for (unsigned i = 0, e = V.size(); i != e; ++i)
+    assert(V[i]->getType() == T->getElementType() &&
            "Initializer for array element doesn't match array element type!");
-    *OL = C;
-  }
+  std::copy(V.begin(), V.end(), op_begin());
 }
 
-Constant *ConstantArray::get(const ArrayType *Ty, ArrayRef<Constant*> V) {
+Constant *ConstantArray::get(ArrayType *Ty, ArrayRef<Constant*> V) {
   for (unsigned i = 0, e = V.size(); i != e; ++i) {
     assert(V[i]->getType() == Ty->getElementType() &&
            "Wrong type in array element initializer");
@@ -653,25 +663,20 @@ StructType *ConstantStruct::getTypeForElements(ArrayRef<Constant*> V,
 }
 
 
-ConstantStruct::ConstantStruct(const StructType *T,
-                               const std::vector<Constant*> &V)
+ConstantStruct::ConstantStruct(StructType *T, ArrayRef<Constant *> V)
   : Constant(T, ConstantStructVal,
              OperandTraits<ConstantStruct>::op_end(this) - V.size(),
              V.size()) {
-  assert((T->isOpaque() || V.size() == T->getNumElements()) &&
+  assert(V.size() == T->getNumElements() &&
          "Invalid initializer vector for constant structure");
-  Use *OL = OperandList;
-  for (std::vector<Constant*>::const_iterator I = V.begin(), E = V.end();
-       I != E; ++I, ++OL) {
-    Constant *C = *I;
-    assert((T->isOpaque() || C->getType() == T->getElementType(I-V.begin())) &&
+  for (unsigned i = 0, e = V.size(); i != e; ++i)
+    assert((T->isOpaque() || V[i]->getType() == T->getElementType(i)) &&
            "Initializer for struct element doesn't match struct element type!");
-    *OL = C;
-  }
+  std::copy(V.begin(), V.end(), op_begin());
 }
 
 // ConstantStruct accessors.
-Constant *ConstantStruct::get(const StructType *ST, ArrayRef<Constant*> V) {
+Constant *ConstantStruct::get(StructType *ST, ArrayRef<Constant*> V) {
   // Create a ConstantAggregateZero value if all elements are zeros.
   for (unsigned i = 0, e = V.size(); i != e; ++i)
     if (!V[i]->isNullValue())
@@ -682,7 +687,7 @@ Constant *ConstantStruct::get(const StructType *ST, ArrayRef<Constant*> V) {
   return ConstantAggregateZero::get(ST);
 }
 
-Constant* ConstantStruct::get(const StructType *T, ...) {
+Constant *ConstantStruct::get(StructType *T, ...) {
   va_list ap;
   SmallVector<Constant*, 8> Values;
   va_start(ap, T);
@@ -692,25 +697,20 @@ Constant* ConstantStruct::get(const StructType *T, ...) {
   return get(T, Values);
 }
 
-ConstantVector::ConstantVector(const VectorType *T,
-                               const std::vector<Constant*> &V)
+ConstantVector::ConstantVector(VectorType *T, ArrayRef<Constant *> V)
   : Constant(T, ConstantVectorVal,
              OperandTraits<ConstantVector>::op_end(this) - V.size(),
              V.size()) {
-  Use *OL = OperandList;
-  for (std::vector<Constant*>::const_iterator I = V.begin(), E = V.end();
-       I != E; ++I, ++OL) {
-    Constant *C = *I;
-    assert(C->getType() == T->getElementType() &&
+  for (size_t i = 0, e = V.size(); i != e; i++)
+    assert(V[i]->getType() == T->getElementType() &&
            "Initializer for vector element doesn't match vector element type!");
-    *OL = C;
-  }
+  std::copy(V.begin(), V.end(), op_begin());
 }
 
 // ConstantVector accessors.
 Constant *ConstantVector::get(ArrayRef<Constant*> V) {
   assert(!V.empty() && "Vectors can't be empty");
-  const VectorType *T = VectorType::get(V.front()->getType(), V.size());
+  VectorType *T = VectorType::get(V.front()->getType(), V.size());
   LLVMContextImpl *pImpl = T->getContext().pImpl;
 
   // If this is an all-undef or all-zero vector, return a
@@ -761,7 +761,7 @@ bool ConstantExpr::isGEPWithNoNotionalOverIndexing() const {
   for (; GEPI != E; ++GEPI, ++OI) {
     ConstantInt *CI = dyn_cast<ConstantInt>(*OI);
     if (!CI) return false;
-    if (const ArrayType *ATy = dyn_cast<ArrayType>(*GEPI))
+    if (ArrayType *ATy = dyn_cast<ArrayType>(*GEPI))
       if (CI->getValue().getActiveBits() > 64 ||
           CI->getZExtValue() >= ATy->getNumElements())
         return false;
@@ -839,13 +839,13 @@ ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
     for (unsigned i = 1, e = getNumOperands(); i != e; ++i)
       Ops[i-1] = getOperand(i);
     if (OpNo == 0)
-      return cast<GEPOperator>(this)->isInBounds() ?
-        ConstantExpr::getInBoundsGetElementPtr(Op, &Ops[0], Ops.size()) :
-        ConstantExpr::getGetElementPtr(Op, &Ops[0], Ops.size());
+      return
+        ConstantExpr::getGetElementPtr(Op, Ops,
+                                       cast<GEPOperator>(this)->isInBounds());
     Ops[OpNo-1] = Op;
-    return cast<GEPOperator>(this)->isInBounds() ?
-      ConstantExpr::getInBoundsGetElementPtr(getOperand(0), &Ops[0],Ops.size()):
-      ConstantExpr::getGetElementPtr(getOperand(0), &Ops[0], Ops.size());
+    return
+      ConstantExpr::getGetElementPtr(getOperand(0), Ops,
+                                     cast<GEPOperator>(this)->isInBounds());
   }
   default:
     assert(getNumOperands() == 2 && "Must be binary operator?");
@@ -859,7 +859,7 @@ ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
 /// operands replaced with the specified values.  The specified array must
 /// have the same number of operands as our current one.
 Constant *ConstantExpr::
-getWithOperands(ArrayRef<Constant*> Ops, const Type *Ty) const {
+getWithOperands(ArrayRef<Constant*> Ops, Type *Ty) const {
   assert(Ops.size() == getNumOperands() && "Operand count mismatch!");
   bool AnyChange = Ty != getType();
   for (unsigned i = 0; i != Ops.size(); ++i)
@@ -891,9 +891,9 @@ getWithOperands(ArrayRef<Constant*> Ops, const Type *Ty) const {
   case Instruction::ShuffleVector:
     return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
   case Instruction::GetElementPtr:
-    return cast<GEPOperator>(this)->isInBounds() ?
-      ConstantExpr::getInBoundsGetElementPtr(Ops[0], &Ops[1], Ops.size()-1) :
-      ConstantExpr::getGetElementPtr(Ops[0], &Ops[1], Ops.size()-1);
+    return
+      ConstantExpr::getGetElementPtr(Ops[0], Ops.slice(1),
+                                     cast<GEPOperator>(this)->isInBounds());
   case Instruction::ICmp:
   case Instruction::FCmp:
     return ConstantExpr::getCompare(getPredicate(), Ops[0], Ops[1]);
@@ -907,7 +907,7 @@ getWithOperands(ArrayRef<Constant*> Ops, const Type *Ty) const {
 //===----------------------------------------------------------------------===//
 //                      isValueValidForType implementations
 
-bool ConstantInt::isValueValidForType(const Type *Ty, uint64_t Val) {
+bool ConstantInt::isValueValidForType(Type *Ty, uint64_t Val) {
   unsigned NumBits = cast<IntegerType>(Ty)->getBitWidth(); // assert okay
   if (Ty == Type::getInt1Ty(Ty->getContext()))
     return Val == 0 || Val == 1;
@@ -917,7 +917,7 @@ bool ConstantInt::isValueValidForType(const Type *Ty, uint64_t Val) {
   return Val <= Max;
 }
 
-bool ConstantInt::isValueValidForType(const Type *Ty, int64_t Val) {
+bool ConstantInt::isValueValidForType(Type *Ty, int64_t Val) {
   unsigned NumBits = cast<IntegerType>(Ty)->getBitWidth(); // assert okay
   if (Ty == Type::getInt1Ty(Ty->getContext()))
     return Val == 0 || Val == 1 || Val == -1;
@@ -928,7 +928,7 @@ bool ConstantInt::isValueValidForType(const Type *Ty, int64_t Val) {
   return (Val >= Min && Val <= Max);
 }
 
-bool ConstantFP::isValueValidForType(const Type *Ty, const APFloat& Val) {
+bool ConstantFP::isValueValidForType(Type *Ty, const APFloat& Val) {
   // convert modifies in place, so make a copy.
   APFloat Val2 = APFloat(Val);
   bool losesInfo;
@@ -968,7 +968,7 @@ bool ConstantFP::isValueValidForType(const Type *Ty, const APFloat& Val) {
 //===----------------------------------------------------------------------===//
 //                      Factory Function Implementation
 
-ConstantAggregateZero* ConstantAggregateZero::get(const Type* Ty) {
+ConstantAggregateZero* ConstantAggregateZero::get(Type* Ty) {
   assert((Ty->isStructTy() || Ty->isArrayTy() || Ty->isVectorTy()) &&
          "Cannot create an aggregate zero of non-aggregate type!");
   
@@ -1079,13 +1079,16 @@ bool ConstantVector::isAllOnesValue() const {
   // Check out first element.
   const Constant *Elt = getOperand(0);
   const ConstantInt *CI = dyn_cast<ConstantInt>(Elt);
-  if (!CI || !CI->isAllOnesValue()) return false;
+  const ConstantFP *CF = dyn_cast<ConstantFP>(Elt);
+
   // Then make sure all remaining elements point to the same value.
   for (unsigned I = 1, E = getNumOperands(); I < E; ++I)
     if (getOperand(I) != Elt)
       return false;
   
-  return true;
+  // First value is all-ones.
+  return (CI && CI->isAllOnesValue()) || 
+         (CF && CF->isAllOnesValue());
 }
 
 /// getSplatValue - If this is a splat constant, where all of the
@@ -1103,7 +1106,7 @@ Constant *ConstantVector::getSplatValue() const {
 //---- ConstantPointerNull::get() implementation.
 //
 
-ConstantPointerNull *ConstantPointerNull::get(const PointerType *Ty) {
+ConstantPointerNull *ConstantPointerNull::get(PointerType *Ty) {
   return Ty->getContext().pImpl->NullPtrConstants.getOrCreate(Ty, 0);
 }
 
@@ -1118,7 +1121,7 @@ void ConstantPointerNull::destroyConstant() {
 //---- UndefValue::get() implementation.
 //
 
-UndefValue *UndefValue::get(const Type *Ty) {
+UndefValue *UndefValue::get(Type *Ty) {
   return Ty->getContext().pImpl->UndefValueConstants.getOrCreate(Ty, 0);
 }
 
@@ -1209,7 +1212,7 @@ void BlockAddress::replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) {
 /// This is a utility function to handle folding of casts and lookup of the
 /// cast in the ExprConstants map. It is used by the various get* methods below.
 static inline Constant *getFoldedCast(
-  Instruction::CastOps opc, Constant *C, const Type *Ty) {
+  Instruction::CastOps opc, Constant *C, Type *Ty) {
   assert(Ty->isFirstClassType() && "Cannot cast to an aggregate type!");
   // Fold a few common cases
   if (Constant *FC = ConstantFoldCastInstruction(opc, C, Ty))
@@ -1224,7 +1227,7 @@ static inline Constant *getFoldedCast(
   return pImpl->ExprConstants.getOrCreate(Ty, Key);
 }
  
-Constant *ConstantExpr::getCast(unsigned oc, Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getCast(unsigned oc, Constant *C, Type *Ty) {
   Instruction::CastOps opc = Instruction::CastOps(oc);
   assert(Instruction::isCast(opc) && "opcode out of range");
   assert(C && Ty && "Null arguments to getCast");
@@ -1250,25 +1253,25 @@ Constant *ConstantExpr::getCast(unsigned oc, Constant *C, const Type *Ty) {
   return 0;
 } 
 
-Constant *ConstantExpr::getZExtOrBitCast(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getZExtOrBitCast(Constant *C, Type *Ty) {
   if (C->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
     return getBitCast(C, Ty);
   return getZExt(C, Ty);
 }
 
-Constant *ConstantExpr::getSExtOrBitCast(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getSExtOrBitCast(Constant *C, Type *Ty) {
   if (C->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
     return getBitCast(C, Ty);
   return getSExt(C, Ty);
 }
 
-Constant *ConstantExpr::getTruncOrBitCast(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getTruncOrBitCast(Constant *C, Type *Ty) {
   if (C->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
     return getBitCast(C, Ty);
   return getTrunc(C, Ty);
 }
 
-Constant *ConstantExpr::getPointerCast(Constant *S, const Type *Ty) {
+Constant *ConstantExpr::getPointerCast(Constant *S, Type *Ty) {
   assert(S->getType()->isPointerTy() && "Invalid cast");
   assert((Ty->isIntegerTy() || Ty->isPointerTy()) && "Invalid cast");
 
@@ -1277,7 +1280,7 @@ Constant *ConstantExpr::getPointerCast(Constant *S, const Type *Ty) {
   return getBitCast(S, Ty);
 }
 
-Constant *ConstantExpr::getIntegerCast(Constant *C, const Type *Ty, 
+Constant *ConstantExpr::getIntegerCast(Constant *C, Type *Ty, 
                                        bool isSigned) {
   assert(C->getType()->isIntOrIntVectorTy() &&
          Ty->isIntOrIntVectorTy() && "Invalid cast");
@@ -1290,7 +1293,7 @@ Constant *ConstantExpr::getIntegerCast(Constant *C, const Type *Ty,
   return getCast(opcode, C, Ty);
 }
 
-Constant *ConstantExpr::getFPCast(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getFPCast(Constant *C, Type *Ty) {
   assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() &&
          "Invalid cast");
   unsigned SrcBits = C->getType()->getScalarSizeInBits();
@@ -1302,7 +1305,7 @@ Constant *ConstantExpr::getFPCast(Constant *C, const Type *Ty) {
   return getCast(opcode, C, Ty);
 }
 
-Constant *ConstantExpr::getTrunc(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getTrunc(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1316,7 +1319,7 @@ Constant *ConstantExpr::getTrunc(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::Trunc, C, Ty);
 }
 
-Constant *ConstantExpr::getSExt(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getSExt(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1330,7 +1333,7 @@ Constant *ConstantExpr::getSExt(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::SExt, C, Ty);
 }
 
-Constant *ConstantExpr::getZExt(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getZExt(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1344,7 +1347,7 @@ Constant *ConstantExpr::getZExt(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::ZExt, C, Ty);
 }
 
-Constant *ConstantExpr::getFPTrunc(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getFPTrunc(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1356,7 +1359,7 @@ Constant *ConstantExpr::getFPTrunc(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::FPTrunc, C, Ty);
 }
 
-Constant *ConstantExpr::getFPExtend(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getFPExtend(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1368,7 +1371,7 @@ Constant *ConstantExpr::getFPExtend(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::FPExt, C, Ty);
 }
 
-Constant *ConstantExpr::getUIToFP(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getUIToFP(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1379,7 +1382,7 @@ Constant *ConstantExpr::getUIToFP(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::UIToFP, C, Ty);
 }
 
-Constant *ConstantExpr::getSIToFP(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getSIToFP(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1390,7 +1393,7 @@ Constant *ConstantExpr::getSIToFP(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::SIToFP, C, Ty);
 }
 
-Constant *ConstantExpr::getFPToUI(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getFPToUI(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1401,7 +1404,7 @@ Constant *ConstantExpr::getFPToUI(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::FPToUI, C, Ty);
 }
 
-Constant *ConstantExpr::getFPToSI(Constant *C, const Type *Ty) {
+Constant *ConstantExpr::getFPToSI(Constant *C, Type *Ty) {
 #ifndef NDEBUG
   bool fromVec = C->getType()->getTypeID() == Type::VectorTyID;
   bool toVec = Ty->getTypeID() == Type::VectorTyID;
@@ -1412,19 +1415,19 @@ Constant *ConstantExpr::getFPToSI(Constant *C, const Type *Ty) {
   return getFoldedCast(Instruction::FPToSI, C, Ty);
 }
 
-Constant *ConstantExpr::getPtrToInt(Constant *C, const Type *DstTy) {
+Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy) {
   assert(C->getType()->isPointerTy() && "PtrToInt source must be pointer");
   assert(DstTy->isIntegerTy() && "PtrToInt destination must be integral");
   return getFoldedCast(Instruction::PtrToInt, C, DstTy);
 }
 
-Constant *ConstantExpr::getIntToPtr(Constant *C, const Type *DstTy) {
+Constant *ConstantExpr::getIntToPtr(Constant *C, Type *DstTy) {
   assert(C->getType()->isIntegerTy() && "IntToPtr source must be integral");
   assert(DstTy->isPointerTy() && "IntToPtr destination must be a pointer");
   return getFoldedCast(Instruction::IntToPtr, C, DstTy);
 }
 
-Constant *ConstantExpr::getBitCast(Constant *C, const Type *DstTy) {
+Constant *ConstantExpr::getBitCast(Constant *C, Type *DstTy) {
   assert(CastInst::castIsValid(Instruction::BitCast, C, DstTy) &&
          "Invalid constantexpr bitcast!");
   
@@ -1513,36 +1516,36 @@ Constant *ConstantExpr::get(unsigned Opcode, Constant *C1, Constant *C2,
   return pImpl->ExprConstants.getOrCreate(C1->getType(), Key);
 }
 
-Constant *ConstantExpr::getSizeOf(const Type* Ty) {
+Constant *ConstantExpr::getSizeOf(Type* Ty) {
   // sizeof is implemented as: (i64) gep (Ty*)null, 1
   // Note that a non-inbounds gep is used, as null isn't within any object.
   Constant *GEPIdx = ConstantInt::get(Type::getInt32Ty(Ty->getContext()), 1);
   Constant *GEP = getGetElementPtr(
-                 Constant::getNullValue(PointerType::getUnqual(Ty)), &GEPIdx, 1);
+                 Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx);
   return getPtrToInt(GEP, 
                      Type::getInt64Ty(Ty->getContext()));
 }
 
-Constant *ConstantExpr::getAlignOf(const Type* Ty) {
+Constant *ConstantExpr::getAlignOf(Type* Ty) {
   // alignof is implemented as: (i64) gep ({i1,Ty}*)null, 0, 1
   // Note that a non-inbounds gep is used, as null isn't within any object.
-  const Type *AligningTy = 
+  Type *AligningTy = 
     StructType::get(Type::getInt1Ty(Ty->getContext()), Ty, NULL);
   Constant *NullPtr = Constant::getNullValue(AligningTy->getPointerTo());
   Constant *Zero = ConstantInt::get(Type::getInt64Ty(Ty->getContext()), 0);
   Constant *One = ConstantInt::get(Type::getInt32Ty(Ty->getContext()), 1);
   Constant *Indices[2] = { Zero, One };
-  Constant *GEP = getGetElementPtr(NullPtr, Indices, 2);
+  Constant *GEP = getGetElementPtr(NullPtr, Indices);
   return getPtrToInt(GEP,
                      Type::getInt64Ty(Ty->getContext()));
 }
 
-Constant *ConstantExpr::getOffsetOf(const StructType* STy, unsigned FieldNo) {
+Constant *ConstantExpr::getOffsetOf(StructType* STy, unsigned FieldNo) {
   return getOffsetOf(STy, ConstantInt::get(Type::getInt32Ty(STy->getContext()),
                                            FieldNo));
 }
 
-Constant *ConstantExpr::getOffsetOf(const Type* Ty, Constant *FieldNo) {
+Constant *ConstantExpr::getOffsetOf(Type* Ty, Constant *FieldNo) {
   // offsetof is implemented as: (i64) gep (Ty*)null, 0, FieldNo
   // Note that a non-inbounds gep is used, as null isn't within any object.
   Constant *GEPIdx[] = {
@@ -1550,7 +1553,7 @@ Constant *ConstantExpr::getOffsetOf(const Type* Ty, Constant *FieldNo) {
     FieldNo
   };
   Constant *GEP = getGetElementPtr(
-                Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx, 2);
+                Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx);
   return getPtrToInt(GEP,
                      Type::getInt64Ty(Ty->getContext()));
 }
@@ -1592,14 +1595,13 @@ Constant *ConstantExpr::getSelect(Constant *C, Constant *V1, Constant *V2) {
   return pImpl->ExprConstants.getOrCreate(V1->getType(), Key);
 }
 
-Constant *ConstantExpr::getGetElementPtr(Constant *C, Value* const *Idxs,
-                                         unsigned NumIdx, bool InBounds) {
-  if (Constant *FC = ConstantFoldGetElementPtr(C, InBounds, Idxs, NumIdx))
+Constant *ConstantExpr::getGetElementPtr(Constant *C, ArrayRef<Value *> Idxs,
+                                         bool InBounds) {
+  if (Constant *FC = ConstantFoldGetElementPtr(C, InBounds, Idxs))
     return FC;          // Fold a few common cases.
 
   // Get the result type of the getelementptr!
-  const Type *Ty = 
-    GetElementPtrInst::getIndexedType(C->getType(), Idxs, Idxs+NumIdx);
+  Type *Ty = GetElementPtrInst::getIndexedType(C->getType(), Idxs);
   assert(Ty && "GEP indices invalid!");
   unsigned AS = cast<PointerType>(C->getType())->getAddressSpace();
   Type *ReqTy = Ty->getPointerTo(AS);
@@ -1608,9 +1610,9 @@ Constant *ConstantExpr::getGetElementPtr(Constant *C, Value* const *Idxs,
          "Non-pointer type for constant GetElementPtr expression");
   // Look up the constant in the table first to ensure uniqueness
   std::vector<Constant*> ArgVec;
-  ArgVec.reserve(NumIdx+1);
+  ArgVec.reserve(1 + Idxs.size());
   ArgVec.push_back(C);
-  for (unsigned i = 0; i != NumIdx; ++i)
+  for (unsigned i = 0, e = Idxs.size(); i != e; ++i)
     ArgVec.push_back(cast<Constant>(Idxs[i]));
   const ExprMapKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
                            InBounds ? GEPOperator::IsInBounds : 0);
@@ -1635,8 +1637,8 @@ ConstantExpr::getICmp(unsigned short pred, Constant *LHS, Constant *RHS) {
   // Get the key type with both the opcode and predicate
   const ExprMapKeyType Key(Instruction::ICmp, ArgVec, pred);
 
-  const Type *ResultTy = Type::getInt1Ty(LHS->getContext());
-  if (const VectorType *VT = dyn_cast<VectorType>(LHS->getType()))
+  Type *ResultTy = Type::getInt1Ty(LHS->getContext());
+  if (VectorType *VT = dyn_cast<VectorType>(LHS->getType()))
     ResultTy = VectorType::get(ResultTy, VT->getNumElements());
 
   LLVMContextImpl *pImpl = LHS->getType()->getContext().pImpl;
@@ -1658,8 +1660,8 @@ ConstantExpr::getFCmp(unsigned short pred, Constant *LHS, Constant *RHS) {
   // Get the key type with both the opcode and predicate
   const ExprMapKeyType Key(Instruction::FCmp, ArgVec, pred);
 
-  const Type *ResultTy = Type::getInt1Ty(LHS->getContext());
-  if (const VectorType *VT = dyn_cast<VectorType>(LHS->getType()))
+  Type *ResultTy = Type::getInt1Ty(LHS->getContext());
+  if (VectorType *VT = dyn_cast<VectorType>(LHS->getType()))
     ResultTy = VectorType::get(ResultTy, VT->getNumElements());
 
   LLVMContextImpl *pImpl = LHS->getType()->getContext().pImpl;
@@ -1715,8 +1717,8 @@ Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
     return FC;          // Fold a few common cases.
 
   unsigned NElts = cast<VectorType>(Mask->getType())->getNumElements();
-  const Type *EltTy = cast<VectorType>(V1->getType())->getElementType();
-  const Type *ShufTy = VectorType::get(EltTy, NElts);
+  Type *EltTy = cast<VectorType>(V1->getType())->getElementType();
+  Type *ShufTy = VectorType::get(EltTy, NElts);
 
   // Look up the constant in the table first to ensure uniqueness
   std::vector<Constant*> ArgVec(1, V1);
@@ -1745,7 +1747,7 @@ Constant *ConstantExpr::getExtractValue(Constant *Agg,
   assert(Agg->getType()->isFirstClassType() &&
          "Tried to create extractelement operation on non-first-class type!");
 
-  const Type *ReqTy = ExtractValueInst::getIndexedType(Agg->getType(), Idxs);
+  Type *ReqTy = ExtractValueInst::getIndexedType(Agg->getType(), Idxs);
   (void)ReqTy;
   assert(ReqTy && "extractvalue indices invalid!");
   
@@ -1878,7 +1880,7 @@ const char *ConstantExpr::getOpcodeName() const {
 
 GetElementPtrConstantExpr::
 GetElementPtrConstantExpr(Constant *C, const std::vector<Constant*> &IdxList,
-                          const Type *DestTy)
+                          Type *DestTy)
   : ConstantExpr(DestTy, Instruction::GetElementPtr,
                  OperandTraits<GetElementPtrConstantExpr>::op_end(this)
                  - (IdxList.size()+1), IdxList.size()+1) {
@@ -2091,8 +2093,7 @@ void ConstantExpr::replaceUsesOfWithOnConstant(Value *From, Value *ToV,
       if (Val == From) Val = To;
       Indices.push_back(Val);
     }
-    Replacement = ConstantExpr::getGetElementPtr(Pointer,
-                                                 &Indices[0], Indices.size(),
+    Replacement = ConstantExpr::getGetElementPtr(Pointer, Indices,
                                          cast<GEPOperator>(this)->isInBounds());
   } else if (getOpcode() == Instruction::ExtractValue) {
     Constant *Agg = getOperand(0);
diff --git a/contrib/llvm/lib/VMCore/ConstantsContext.h b/contrib/llvm/lib/VMCore/ConstantsContext.h
index bd134d9b892d..1077004d7c76 100644
--- a/contrib/llvm/lib/VMCore/ConstantsContext.h
+++ b/contrib/llvm/lib/VMCore/ConstantsContext.h
@@ -36,7 +36,7 @@ class UnaryConstantExpr : public ConstantExpr {
   void *operator new(size_t s) {
     return User::operator new(s, 1);
   }
-  UnaryConstantExpr(unsigned Opcode, Constant *C, const Type *Ty)
+  UnaryConstantExpr(unsigned Opcode, Constant *C, Type *Ty)
     : ConstantExpr(Ty, Opcode, &Op<0>(), 1) {
     Op<0>() = C;
   }
@@ -159,7 +159,7 @@ class ExtractValueConstantExpr : public ConstantExpr {
   }
   ExtractValueConstantExpr(Constant *Agg,
                            const SmallVector<unsigned, 4> &IdxList,
-                           const Type *DestTy)
+                           Type *DestTy)
     : ConstantExpr(DestTy, Instruction::ExtractValue, &Op<0>(), 1),
       Indices(IdxList) {
     Op<0>() = Agg;
@@ -184,7 +184,7 @@ class InsertValueConstantExpr : public ConstantExpr {
   }
   InsertValueConstantExpr(Constant *Agg, Constant *Val,
                           const SmallVector<unsigned, 4> &IdxList,
-                          const Type *DestTy)
+                          Type *DestTy)
     : ConstantExpr(DestTy, Instruction::InsertValue, &Op<0>(), 2),
       Indices(IdxList) {
     Op<0>() = Agg;
@@ -203,11 +203,11 @@ class InsertValueConstantExpr : public ConstantExpr {
 /// used behind the scenes to implement getelementpr constant exprs.
 class GetElementPtrConstantExpr : public ConstantExpr {
   GetElementPtrConstantExpr(Constant *C, const std::vector<Constant*> &IdxList,
-                            const Type *DestTy);
+                            Type *DestTy);
 public:
   static GetElementPtrConstantExpr *Create(Constant *C,
                                            const std::vector<Constant*>&IdxList,
-                                           const Type *DestTy,
+                                           Type *DestTy,
                                            unsigned Flags) {
     GetElementPtrConstantExpr *Result =
       new(IdxList.size() + 1) GetElementPtrConstantExpr(C, IdxList, DestTy);
@@ -228,7 +228,7 @@ struct CompareConstantExpr : public ConstantExpr {
     return User::operator new(s, 2);
   }
   unsigned short predicate;
-  CompareConstantExpr(const Type *ty, Instruction::OtherOps opc,
+  CompareConstantExpr(Type *ty, Instruction::OtherOps opc,
                       unsigned short pred,  Constant* LHS, Constant* RHS)
     : ConstantExpr(ty, opc, &Op<0>(), 2), predicate(pred) {
     Op<0>() = LHS;
@@ -392,7 +392,7 @@ struct ConstantTraits<Constant *> {
 
 template<class ConstantClass, class TypeClass, class ValType>
 struct ConstantCreator {
-  static ConstantClass *create(const TypeClass *Ty, const ValType &V) {
+  static ConstantClass *create(TypeClass *Ty, const ValType &V) {
     return new(ConstantTraits<ValType>::uses(V)) ConstantClass(Ty, V);
   }
 };
@@ -407,7 +407,7 @@ struct ConstantKeyData {
 
 template<>
 struct ConstantCreator<ConstantExpr, Type, ExprMapKeyType> {
-  static ConstantExpr *create(const Type *Ty, const ExprMapKeyType &V,
+  static ConstantExpr *create(Type *Ty, const ExprMapKeyType &V,
       unsigned short pred = 0) {
     if (Instruction::isCast(V.opcode))
       return new UnaryConstantExpr(V.opcode, V.operands[0], Ty);
@@ -470,7 +470,7 @@ struct ConstantKeyData<ConstantExpr> {
 // ConstantAggregateZero does not take extra "value" argument...
 template<class ValType>
 struct ConstantCreator<ConstantAggregateZero, Type, ValType> {
-  static ConstantAggregateZero *create(const Type *Ty, const ValType &V){
+  static ConstantAggregateZero *create(Type *Ty, const ValType &V){
     return new ConstantAggregateZero(Ty);
   }
 };
@@ -522,7 +522,7 @@ struct ConstantKeyData<ConstantStruct> {
 // ConstantPointerNull does not take extra "value" argument...
 template<class ValType>
 struct ConstantCreator<ConstantPointerNull, PointerType, ValType> {
-  static ConstantPointerNull *create(const PointerType *Ty, const ValType &V){
+  static ConstantPointerNull *create(PointerType *Ty, const ValType &V){
     return new ConstantPointerNull(Ty);
   }
 };
@@ -538,7 +538,7 @@ struct ConstantKeyData<ConstantPointerNull> {
 // UndefValue does not take extra "value" argument...
 template<class ValType>
 struct ConstantCreator<UndefValue, Type, ValType> {
-  static UndefValue *create(const Type *Ty, const ValType &V) {
+  static UndefValue *create(Type *Ty, const ValType &V) {
     return new UndefValue(Ty);
   }
 };
@@ -553,7 +553,7 @@ struct ConstantKeyData<UndefValue> {
 
 template<>
 struct ConstantCreator<InlineAsm, PointerType, InlineAsmKeyType> {
-  static InlineAsm *create(const PointerType *Ty, const InlineAsmKeyType &Key) {
+  static InlineAsm *create(PointerType *Ty, const InlineAsmKeyType &Key) {
     return new InlineAsm(Ty, Key.asm_string, Key.constraints,
                          Key.has_side_effects, Key.is_align_stack);
   }
@@ -572,7 +572,7 @@ template<class ValType, class ValRefType, class TypeClass, class ConstantClass,
          bool HasLargeKey = false /*true for arrays and structs*/ >
 class ConstantUniqueMap {
 public:
-  typedef std::pair<const TypeClass*, ValType> MapKey;
+  typedef std::pair<TypeClass*, ValType> MapKey;
   typedef std::map<MapKey, ConstantClass *> MapTy;
   typedef std::map<ConstantClass *, typename MapTy::iterator> InverseMapTy;
 private:
@@ -623,7 +623,7 @@ class ConstantUniqueMap {
     }
       
     typename MapTy::iterator I =
-      Map.find(MapKey(static_cast<const TypeClass*>(CP->getType()),
+      Map.find(MapKey(static_cast<TypeClass*>(CP->getType()),
                       ConstantKeyData<ConstantClass>::getValType(CP)));
     if (I == Map.end() || I->second != CP) {
       // FIXME: This should not use a linear scan.  If this gets to be a
@@ -634,7 +634,7 @@ class ConstantUniqueMap {
     return I;
   }
 
-  ConstantClass *Create(const TypeClass *Ty, ValRefType V,
+  ConstantClass *Create(TypeClass *Ty, ValRefType V,
                         typename MapTy::iterator I) {
     ConstantClass* Result =
       ConstantCreator<ConstantClass,TypeClass,ValType>::create(Ty, V);
@@ -651,7 +651,7 @@ class ConstantUniqueMap {
     
   /// getOrCreate - Return the specified constant from the map, creating it if
   /// necessary.
-  ConstantClass *getOrCreate(const TypeClass *Ty, ValRefType V) {
+  ConstantClass *getOrCreate(TypeClass *Ty, ValRefType V) {
     MapKey Lookup(Ty, V);
     ConstantClass* Result = 0;
     
diff --git a/contrib/llvm/lib/VMCore/Core.cpp b/contrib/llvm/lib/VMCore/Core.cpp
index 2a816e123a61..a505e4b4f5e5 100644
--- a/contrib/llvm/lib/VMCore/Core.cpp
+++ b/contrib/llvm/lib/VMCore/Core.cpp
@@ -167,6 +167,11 @@ LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty) {
   }
 }
 
+LLVMBool LLVMTypeIsSized(LLVMTypeRef Ty)
+{
+    return unwrap(Ty)->isSized();
+}
+
 LLVMContextRef LLVMGetTypeContext(LLVMTypeRef Ty) {
   return wrap(&unwrap(Ty)->getContext());
 }
@@ -299,7 +304,15 @@ LLVMTypeRef LLVMStructType(LLVMTypeRef *ElementTypes,
 
 LLVMTypeRef LLVMStructCreateNamed(LLVMContextRef C, const char *Name)
 {
-  return wrap(StructType::createNamed(*unwrap(C), Name));
+  return wrap(StructType::create(*unwrap(C), Name));
+}
+
+const char *LLVMGetStructName(LLVMTypeRef Ty)
+{
+  StructType *Type = unwrap<StructType>(Ty);
+  if (!Type->hasName())
+    return 0;
+  return Type->getName().data();
 }
 
 void LLVMStructSetBody(LLVMTypeRef StructTy, LLVMTypeRef *ElementTypes,
@@ -448,7 +461,10 @@ LLVMValueRef LLVMGetUsedValue(LLVMUseRef U) {
 
 /*--.. Operations on Users .................................................--*/
 LLVMValueRef LLVMGetOperand(LLVMValueRef Val, unsigned Index) {
-  return wrap(unwrap<User>(Val)->getOperand(Index));
+  Value *V = unwrap(Val);
+  if (MDNode *MD = dyn_cast<MDNode>(V))
+      return wrap(MD->getOperand(Index));
+  return wrap(cast<User>(V)->getOperand(Index));
 }
 
 void LLVMSetOperand(LLVMValueRef Val, unsigned Index, LLVMValueRef Op) {
@@ -456,7 +472,10 @@ void LLVMSetOperand(LLVMValueRef Val, unsigned Index, LLVMValueRef Op) {
 }
 
 int LLVMGetNumOperands(LLVMValueRef Val) {
-  return unwrap<User>(Val)->getNumOperands();
+  Value *V = unwrap(Val);
+  if (MDNode *MD = dyn_cast<MDNode>(V))
+      return MD->getNumOperands();
+  return cast<User>(V)->getNumOperands();
 }
 
 /*--.. Operations on constants of any type .................................--*/
@@ -506,13 +525,39 @@ LLVMValueRef LLVMMDString(const char *Str, unsigned SLen) {
 LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals,
                                  unsigned Count) {
   return wrap(MDNode::get(*unwrap(C),
-                          ArrayRef<Value*>(unwrap<Value>(Vals, Count), Count)));
+                          makeArrayRef(unwrap<Value>(Vals, Count), Count)));
 }
 
 LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count) {
   return LLVMMDNodeInContext(LLVMGetGlobalContext(), Vals, Count);
 }
 
+const char *LLVMGetMDString(LLVMValueRef V, unsigned* Len) {
+  if (const MDString *S = dyn_cast<MDString>(unwrap(V))) {
+    *Len = S->getString().size();
+    return S->getString().data();
+  }
+  *Len = 0;
+  return 0;
+}
+
+unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char* name)
+{
+  if (NamedMDNode *N = unwrap(M)->getNamedMetadata(name)) {
+    return N->getNumOperands();
+  }
+  return 0;
+}
+
+void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char* name, LLVMValueRef *Dest)
+{
+  NamedMDNode *N = unwrap(M)->getNamedMetadata(name);
+  if (!N)
+    return;
+  for (unsigned i=0;i<N->getNumOperands();i++)
+    Dest[i] = wrap(N->getOperand(i));
+}
+
 /*--.. Operations on scalar constants ......................................--*/
 
 LLVMValueRef LLVMConstInt(LLVMTypeRef IntTy, unsigned long long N,
@@ -525,7 +570,8 @@ LLVMValueRef LLVMConstIntOfArbitraryPrecision(LLVMTypeRef IntTy,
                                               const uint64_t Words[]) {
     IntegerType *Ty = unwrap<IntegerType>(IntTy);
     return wrap(ConstantInt::get(Ty->getContext(),
-                                 APInt(Ty->getBitWidth(), NumWords, Words)));
+                                 APInt(Ty->getBitWidth(),
+                                       makeArrayRef(Words, NumWords))));
 }
 
 LLVMValueRef LLVMConstIntOfString(LLVMTypeRef IntTy, const char Str[],
@@ -575,8 +621,7 @@ LLVMValueRef LLVMConstStructInContext(LLVMContextRef C,
                                       LLVMValueRef *ConstantVals,
                                       unsigned Count, LLVMBool Packed) {
   Constant **Elements = unwrap<Constant>(ConstantVals, Count);
-  return wrap(ConstantStruct::getAnon(*unwrap(C),
-                                      ArrayRef<Constant*>(Elements, Count),
+  return wrap(ConstantStruct::getAnon(*unwrap(C), makeArrayRef(Elements, Count),
                                       Packed != 0));
 }
 
@@ -600,19 +645,44 @@ LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy,
                                   LLVMValueRef *ConstantVals,
                                   unsigned Count) {
   Constant **Elements = unwrap<Constant>(ConstantVals, Count);
-  const StructType *Ty = cast<StructType>(unwrap(StructTy));
+  StructType *Ty = cast<StructType>(unwrap(StructTy));
 
-  return wrap(ConstantStruct::get(Ty, ArrayRef<Constant*>(Elements, Count)));
+  return wrap(ConstantStruct::get(Ty, makeArrayRef(Elements, Count)));
 }
 
 LLVMValueRef LLVMConstVector(LLVMValueRef *ScalarConstantVals, unsigned Size) {
-  return wrap(ConstantVector::get(ArrayRef<Constant*>(
+  return wrap(ConstantVector::get(makeArrayRef(
                             unwrap<Constant>(ScalarConstantVals, Size), Size)));
 }
+
+/*-- Opcode mapping */
+
+static LLVMOpcode map_to_llvmopcode(int opcode)
+{
+    switch (opcode) {
+      default:
+        assert(0 && "Unhandled Opcode.");
+#define HANDLE_INST(num, opc, clas) case num: return LLVM##opc;
+#include "llvm/Instruction.def"
+#undef HANDLE_INST
+    }
+}
+
+static int map_from_llvmopcode(LLVMOpcode code)
+{
+    switch (code) {
+      default:
+        assert(0 && "Unhandled Opcode.");
+#define HANDLE_INST(num, opc, clas) case LLVM##opc: return num;
+#include "llvm/Instruction.def"
+#undef HANDLE_INST
+    }
+}
+
 /*--.. Constant expressions ................................................--*/
 
 LLVMOpcode LLVMGetConstOpcode(LLVMValueRef ConstantVal) {
-  return (LLVMOpcode)unwrap<ConstantExpr>(ConstantVal)->getOpcode();
+  return map_to_llvmopcode(unwrap<ConstantExpr>(ConstantVal)->getOpcode());
 }
 
 LLVMValueRef LLVMAlignOf(LLVMTypeRef Ty) {
@@ -792,18 +862,19 @@ LLVMValueRef LLVMConstAShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant) {
 
 LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal,
                           LLVMValueRef *ConstantIndices, unsigned NumIndices) {
+  ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
+                               NumIndices);
   return wrap(ConstantExpr::getGetElementPtr(unwrap<Constant>(ConstantVal),
-                                             unwrap<Constant>(ConstantIndices, 
-                                                              NumIndices),
-                                             NumIndices));
+                                             IdxList));
 }
 
 LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
                                   LLVMValueRef *ConstantIndices,
                                   unsigned NumIndices) {
   Constant* Val = unwrap<Constant>(ConstantVal);
-  Constant** Idxs = unwrap<Constant>(ConstantIndices, NumIndices);
-  return wrap(ConstantExpr::getInBoundsGetElementPtr(Val, Idxs, NumIndices));
+  ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
+                               NumIndices);
+  return wrap(ConstantExpr::getInBoundsGetElementPtr(Val, IdxList));
 }
 
 LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType) {
@@ -934,8 +1005,7 @@ LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
 LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList,
                                    unsigned NumIdx) {
   return wrap(ConstantExpr::getExtractValue(unwrap<Constant>(AggConstant),
-                                            ArrayRef<unsigned>(IdxList,
-                                                               NumIdx)));
+                                            makeArrayRef(IdxList, NumIdx)));
 }
 
 LLVMValueRef LLVMConstInsertValue(LLVMValueRef AggConstant,
@@ -943,8 +1013,7 @@ LLVMValueRef LLVMConstInsertValue(LLVMValueRef AggConstant,
                                   unsigned *IdxList, unsigned NumIdx) {
   return wrap(ConstantExpr::getInsertValue(unwrap<Constant>(AggConstant),
                                          unwrap<Constant>(ElementValueConstant),
-                                           ArrayRef<unsigned>(IdxList,
-                                                              NumIdx)));
+                                           makeArrayRef(IdxList, NumIdx)));
 }
 
 LLVMValueRef LLVMConstInlineAsm(LLVMTypeRef Ty, const char *AsmString,
@@ -1383,6 +1452,10 @@ LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB) {
   return wrap(unwrap(BB)->getParent());
 }
 
+LLVMValueRef LLVMGetBasicBlockTerminator(LLVMBasicBlockRef BB) {
+  return wrap(unwrap(BB)->getTerminator());
+}
+
 unsigned LLVMCountBasicBlocks(LLVMValueRef FnRef) {
   return unwrap<Function>(FnRef)->size();
 }
@@ -1455,6 +1528,10 @@ void LLVMDeleteBasicBlock(LLVMBasicBlockRef BBRef) {
   unwrap(BBRef)->eraseFromParent();
 }
 
+void LLVMRemoveBasicBlockFromParent(LLVMBasicBlockRef BBRef) {
+  unwrap(BBRef)->removeFromParent();
+}
+
 void LLVMMoveBasicBlockBefore(LLVMBasicBlockRef BB, LLVMBasicBlockRef MovePos) {
   unwrap(BB)->moveBefore(unwrap(MovePos));
 }
@@ -1501,6 +1578,25 @@ LLVMValueRef LLVMGetPreviousInstruction(LLVMValueRef Inst) {
   return wrap(--I);
 }
 
+void LLVMInstructionEraseFromParent(LLVMValueRef Inst) {
+  unwrap<Instruction>(Inst)->eraseFromParent();
+}
+
+LLVMIntPredicate LLVMGetICmpPredicate(LLVMValueRef Inst) {
+  if (ICmpInst *I = dyn_cast<ICmpInst>(unwrap(Inst)))
+    return (LLVMIntPredicate)I->getPredicate();
+  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(unwrap(Inst)))
+    if (CE->getOpcode() == Instruction::ICmp)
+      return (LLVMIntPredicate)CE->getPredicate();
+  return (LLVMIntPredicate)0;
+}
+
+LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst) {
+  if (Instruction *C = dyn_cast<Instruction>(unwrap(Inst)))
+    return map_to_llvmopcode(C->getOpcode());
+  return (LLVMOpcode)0;
+}
+
 /*--.. Call and invoke instructions ........................................--*/
 
 unsigned LLVMGetInstructionCallConv(LLVMValueRef Instr) {
@@ -1554,6 +1650,12 @@ void LLVMSetTailCall(LLVMValueRef Call, LLVMBool isTailCall) {
   unwrap<CallInst>(Call)->setTailCall(isTailCall);
 }
 
+/*--.. Operations on switch instructions (only) ............................--*/
+
+LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef Switch) {
+  return wrap(unwrap<SwitchInst>(Switch)->getDefaultDest());
+}
+
 /*--.. Operations on phi nodes .............................................--*/
 
 void LLVMAddIncoming(LLVMValueRef PhiNode, LLVMValueRef *IncomingValues,
@@ -1680,12 +1782,20 @@ LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn,
                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
                              const char *Name) {
   return wrap(unwrap(B)->CreateInvoke(unwrap(Fn), unwrap(Then), unwrap(Catch),
-                                      ArrayRef<Value *>(unwrap(Args), NumArgs),
+                                      makeArrayRef(unwrap(Args), NumArgs),
                                       Name));
 }
 
-LLVMValueRef LLVMBuildUnwind(LLVMBuilderRef B) {
-  return wrap(unwrap(B)->CreateUnwind());
+LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                 LLVMValueRef PersFn, unsigned NumClauses,
+                                 const char *Name) {
+  return wrap(unwrap(B)->CreateLandingPad(unwrap(Ty),
+                                          cast<Function>(unwrap(PersFn)),
+                                          NumClauses, Name));
+}
+
+LLVMValueRef LLVMBuildResume(LLVMBuilderRef B, LLVMValueRef Exn) {
+  return wrap(unwrap(B)->CreateResume(unwrap(Exn)));
 }
 
 LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef B) {
@@ -1701,6 +1811,15 @@ void LLVMAddDestination(LLVMValueRef IndirectBr, LLVMBasicBlockRef Dest) {
   unwrap<IndirectBrInst>(IndirectBr)->addDestination(unwrap(Dest));
 }
 
+void LLVMAddClause(LLVMValueRef LandingPad, LLVMValueRef ClauseVal) {
+  unwrap<LandingPadInst>(LandingPad)->
+    addClause(cast<Constant>(unwrap(ClauseVal)));
+}
+
+void LLVMSetCleanup(LLVMValueRef LandingPad, LLVMBool Val) {
+  unwrap<LandingPadInst>(LandingPad)->setCleanup(Val);
+}
+
 /*--.. Arithmetic ..........................................................--*/
 
 LLVMValueRef LLVMBuildAdd(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS,
@@ -1831,7 +1950,7 @@ LLVMValueRef LLVMBuildXor(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS,
 LLVMValueRef LLVMBuildBinOp(LLVMBuilderRef B, LLVMOpcode Op,
                             LLVMValueRef LHS, LLVMValueRef RHS,
                             const char *Name) {
-  return wrap(unwrap(B)->CreateBinOp(Instruction::BinaryOps(Op), unwrap(LHS),
+  return wrap(unwrap(B)->CreateBinOp(Instruction::BinaryOps(map_from_llvmopcode(Op)), unwrap(LHS),
                                      unwrap(RHS), Name));
 }
 
@@ -1861,7 +1980,7 @@ LLVMValueRef LLVMBuildNot(LLVMBuilderRef B, LLVMValueRef V, const char *Name) {
 
 LLVMValueRef LLVMBuildMalloc(LLVMBuilderRef B, LLVMTypeRef Ty,
                              const char *Name) {
-  const Type* ITy = Type::getInt32Ty(unwrap(B)->GetInsertBlock()->getContext());
+  Type* ITy = Type::getInt32Ty(unwrap(B)->GetInsertBlock()->getContext());
   Constant* AllocSize = ConstantExpr::getSizeOf(unwrap(Ty));
   AllocSize = ConstantExpr::getTruncOrBitCast(AllocSize, ITy);
   Instruction* Malloc = CallInst::CreateMalloc(unwrap(B)->GetInsertBlock(), 
@@ -1872,7 +1991,7 @@ LLVMValueRef LLVMBuildMalloc(LLVMBuilderRef B, LLVMTypeRef Ty,
 
 LLVMValueRef LLVMBuildArrayMalloc(LLVMBuilderRef B, LLVMTypeRef Ty,
                                   LLVMValueRef Val, const char *Name) {
-  const Type* ITy = Type::getInt32Ty(unwrap(B)->GetInsertBlock()->getContext());
+  Type* ITy = Type::getInt32Ty(unwrap(B)->GetInsertBlock()->getContext());
   Constant* AllocSize = ConstantExpr::getSizeOf(unwrap(Ty));
   AllocSize = ConstantExpr::getTruncOrBitCast(AllocSize, ITy);
   Instruction* Malloc = CallInst::CreateMalloc(unwrap(B)->GetInsertBlock(), 
@@ -1910,15 +2029,15 @@ LLVMValueRef LLVMBuildStore(LLVMBuilderRef B, LLVMValueRef Val,
 LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                           LLVMValueRef *Indices, unsigned NumIndices,
                           const char *Name) {
-  return wrap(unwrap(B)->CreateGEP(unwrap(Pointer), unwrap(Indices),
-                                   unwrap(Indices) + NumIndices, Name));
+  ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
+  return wrap(unwrap(B)->CreateGEP(unwrap(Pointer), IdxList, Name));
 }
 
 LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                                   LLVMValueRef *Indices, unsigned NumIndices,
                                   const char *Name) {
-  return wrap(unwrap(B)->CreateInBoundsGEP(unwrap(Pointer), unwrap(Indices),
-                                           unwrap(Indices) + NumIndices, Name));
+  ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
+  return wrap(unwrap(B)->CreateInBoundsGEP(unwrap(Pointer), IdxList, Name));
 }
 
 LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
@@ -2018,7 +2137,7 @@ LLVMValueRef LLVMBuildTruncOrBitCast(LLVMBuilderRef B, LLVMValueRef Val,
 
 LLVMValueRef LLVMBuildCast(LLVMBuilderRef B, LLVMOpcode Op, LLVMValueRef Val,
                            LLVMTypeRef DestTy, const char *Name) {
-  return wrap(unwrap(B)->CreateCast(Instruction::CastOps(Op), unwrap(Val),
+  return wrap(unwrap(B)->CreateCast(Instruction::CastOps(map_from_llvmopcode(Op)), unwrap(Val),
                                     unwrap(DestTy), Name));
 }
 
@@ -2064,7 +2183,7 @@ LLVMValueRef LLVMBuildCall(LLVMBuilderRef B, LLVMValueRef Fn,
                            LLVMValueRef *Args, unsigned NumArgs,
                            const char *Name) {
   return wrap(unwrap(B)->CreateCall(unwrap(Fn),
-                                    ArrayRef<Value *>(unwrap(Args), NumArgs),
+                                    makeArrayRef(unwrap(Args), NumArgs),
                                     Name));
 }
 
diff --git a/contrib/llvm/lib/VMCore/DebugLoc.cpp b/contrib/llvm/lib/VMCore/DebugLoc.cpp
index 4ff6b2cd80e8..328244f80673 100644
--- a/contrib/llvm/lib/VMCore/DebugLoc.cpp
+++ b/contrib/llvm/lib/VMCore/DebugLoc.cpp
@@ -104,7 +104,7 @@ MDNode *DebugLoc::getAsMDNode(const LLVMContext &Ctx) const {
   assert(Scope && "If scope is null, this should be isUnknown()");
   
   LLVMContext &Ctx2 = Scope->getContext();
-  const Type *Int32 = Type::getInt32Ty(Ctx2);
+  Type *Int32 = Type::getInt32Ty(Ctx2);
   Value *Elts[] = {
     ConstantInt::get(Int32, getLine()), ConstantInt::get(Int32, getCol()),
     Scope, IA
@@ -240,7 +240,7 @@ int LLVMContextImpl::getOrAddScopeInlinedAtIdxEntry(MDNode *Scope, MDNode *IA,
 /// deleted - The MDNode this is pointing to got deleted, so this pointer needs
 /// to drop to null and we need remove our entry from the DenseMap.
 void DebugRecVH::deleted() {
-  // If this is a  non-canonical reference, just drop the value to null, we know
+  // If this is a non-canonical reference, just drop the value to null, we know
   // it doesn't have a map entry.
   if (Idx == 0) {
     setValPtr(0);
diff --git a/contrib/llvm/lib/VMCore/Function.cpp b/contrib/llvm/lib/VMCore/Function.cpp
index 6536bcd0e2ed..1215e6a57ced 100644
--- a/contrib/llvm/lib/VMCore/Function.cpp
+++ b/contrib/llvm/lib/VMCore/Function.cpp
@@ -17,6 +17,7 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/Support/CallSite.h"
+#include "llvm/Support/InstIterator.h"
 #include "llvm/Support/LeakDetector.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/StringPool.h"
@@ -38,7 +39,7 @@ template class llvm::SymbolTableListTraits<BasicBlock, Function>;
 // Argument Implementation
 //===----------------------------------------------------------------------===//
 
-Argument::Argument(const Type *Ty, const Twine &Name, Function *Par)
+Argument::Argument(Type *Ty, const Twine &Name, Function *Par)
   : Value(Ty, Value::ArgumentVal) {
   Parent = 0;
 
@@ -158,7 +159,7 @@ void Function::eraseFromParent() {
 // Function Implementation
 //===----------------------------------------------------------------------===//
 
-Function::Function(const FunctionType *Ty, LinkageTypes Linkage,
+Function::Function(FunctionType *Ty, LinkageTypes Linkage,
                    const Twine &name, Module *ParentModule)
   : GlobalValue(PointerType::getUnqual(Ty), 
                 Value::FunctionVal, 0, 0, Linkage, name) {
@@ -195,7 +196,7 @@ Function::~Function() {
 
 void Function::BuildLazyArguments() const {
   // Create the arguments vector, all arguments start out unnamed.
-  const FunctionType *FT = getFunctionType();
+  FunctionType *FT = getFunctionType();
   for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i) {
     assert(!FT->getParamType(i)->isVoidTy() &&
            "Cannot have void typed arguments!");
@@ -345,7 +346,7 @@ std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) {
     return Table[id];
   std::string Result(Table[id]);
   for (unsigned i = 0; i < Tys.size(); ++i) {
-    if (const PointerType* PTyp = dyn_cast<PointerType>(Tys[i])) {
+    if (PointerType* PTyp = dyn_cast<PointerType>(Tys[i])) {
       Result += ".p" + llvm::utostr(PTyp->getAddressSpace()) + 
                 EVT::getEVT(PTyp->getElementType()).getEVTString();
     }
@@ -355,9 +356,9 @@ std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) {
   return Result;
 }
 
-const FunctionType *Intrinsic::getType(LLVMContext &Context,
+FunctionType *Intrinsic::getType(LLVMContext &Context,
                                        ID id, ArrayRef<Type*> Tys) {
-  const Type *ResultTy = NULL;
+  Type *ResultTy = NULL;
   std::vector<Type*> ArgTys;
   bool IsVarArg = false;
   
@@ -416,8 +417,7 @@ bool Function::hasAddressTaken(const User* *PutOffender) const {
 /// FIXME: Remove after <rdar://problem/8031714> is fixed.
 /// FIXME: Is the above FIXME valid?
 bool Function::callsFunctionThatReturnsTwice() const {
-  const Module *M = this->getParent();
-  static const char *ReturnsTwiceFns[] = {
+  static const char *const ReturnsTwiceFns[] = {
     "_setjmp",
     "setjmp",
     "sigsetjmp",
@@ -428,16 +428,25 @@ bool Function::callsFunctionThatReturnsTwice() const {
     "getcontext"
   };
 
-  for (unsigned I = 0; I < array_lengthof(ReturnsTwiceFns); ++I)
-    if (const Function *Callee = M->getFunction(ReturnsTwiceFns[I])) {
-      if (!Callee->use_empty())
-        for (Value::const_use_iterator
-               I = Callee->use_begin(), E = Callee->use_end();
-             I != E; ++I)
-          if (const CallInst *CI = dyn_cast<CallInst>(*I))
-            if (CI->getParent()->getParent() == this)
-              return true;
+  for (const_inst_iterator I = inst_begin(this), E = inst_end(this); I != E;
+       ++I) {
+    const CallInst* callInst = dyn_cast<CallInst>(&*I);
+    if (!callInst)
+      continue;
+    if (callInst->canReturnTwice())
+      return true;
+
+    // check for known function names.
+    // FIXME: move this to clang.
+    Function *F = callInst->getCalledFunction();
+    if (!F)
+      continue;
+    StringRef Name = F->getName();
+    for (unsigned J = 0, e = array_lengthof(ReturnsTwiceFns); J != e; ++J) {
+      if (Name == ReturnsTwiceFns[J])
+        return true;
     }
+  }
 
   return false;
 }
diff --git a/contrib/llvm/lib/VMCore/GCOV.cpp b/contrib/llvm/lib/VMCore/GCOV.cpp
new file mode 100644
index 000000000000..fc7f96fccaaa
--- /dev/null
+++ b/contrib/llvm/lib/VMCore/GCOV.cpp
@@ -0,0 +1,281 @@
+//===- GCOVr.cpp - LLVM coverage tool -------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// GCOV implements the interface to read and write coverage files that use 
+// 'gcov' format.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/GCOV.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/MemoryObject.h"
+#include "llvm/Support/system_error.h"
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+// GCOVFile implementation.
+
+/// ~GCOVFile - Delete GCOVFile and its content.
+GCOVFile::~GCOVFile() {
+  DeleteContainerPointers(Functions);
+}
+
+/// isGCDAFile - Return true if Format identifies a .gcda file.
+static bool isGCDAFile(GCOVFormat Format) {
+  return Format == GCDA_402 || Format == GCDA_404;
+}
+
+/// isGCNOFile - Return true if Format identifies a .gcno file.
+static bool isGCNOFile(GCOVFormat Format) {
+  return Format == GCNO_402 || Format == GCNO_404;
+}
+
+/// read - Read GCOV buffer.
+bool GCOVFile::read(GCOVBuffer &Buffer) {
+  GCOVFormat Format = Buffer.readGCOVFormat();
+  if (Format == InvalidGCOV)
+    return false;
+
+  unsigned i = 0;
+  while (1) {
+    GCOVFunction *GFun = NULL;
+    if (isGCDAFile(Format)) {
+      // Use existing function while reading .gcda file.
+      assert (i < Functions.size() && ".gcda data does not match .gcno data");
+      GFun = Functions[i];
+    } else if (isGCNOFile(Format)){
+      GFun = new GCOVFunction();
+      Functions.push_back(GFun);
+    }
+    if (!GFun || !GFun->read(Buffer, Format))
+      break;
+    ++i;
+  }
+  return true;
+}
+
+/// dump - Dump GCOVFile content on standard out for debugging purposes.
+void GCOVFile::dump() {
+  for (SmallVector<GCOVFunction *, 16>::iterator I = Functions.begin(),
+	 E = Functions.end(); I != E; ++I)
+    (*I)->dump();
+}
+
+/// collectLineCounts - Collect line counts. This must be used after
+/// reading .gcno and .gcda files.
+void GCOVFile::collectLineCounts(FileInfo &FI) {
+  for (SmallVector<GCOVFunction *, 16>::iterator I = Functions.begin(),
+	 E = Functions.end(); I != E; ++I) 
+    (*I)->collectLineCounts(FI);
+  FI.print();
+}
+
+//===----------------------------------------------------------------------===//
+// GCOVFunction implementation.
+
+/// ~GCOVFunction - Delete GCOVFunction and its content.
+GCOVFunction::~GCOVFunction() {
+  DeleteContainerPointers(Blocks);
+}
+
+/// read - Read a aunction from the buffer. Return false if buffer cursor
+/// does not point to a function tag.
+bool GCOVFunction::read(GCOVBuffer &Buff, GCOVFormat Format) {
+  if (!Buff.readFunctionTag())
+    return false;
+
+  Buff.readInt(); // Function header length
+  Ident = Buff.readInt(); 
+  Buff.readInt(); // Checksum #1
+  if (Format != GCNO_402)
+    Buff.readInt(); // Checksum #2
+
+  Name = Buff.readString();
+  if (Format == GCNO_402 || Format == GCNO_404)
+    Filename = Buff.readString();
+
+  if (Format == GCDA_402 || Format == GCDA_404) {
+    Buff.readArcTag();
+    uint32_t Count = Buff.readInt() / 2;
+    for (unsigned i = 0, e = Count; i != e; ++i) {
+      Blocks[i]->addCount(Buff.readInt64());
+    }
+    return true;;
+  }
+
+  LineNumber = Buff.readInt();
+
+  // read blocks.
+  assert (Buff.readBlockTag() && "Block Tag not found!");
+  uint32_t BlockCount = Buff.readInt();
+  for (int i = 0, e = BlockCount; i != e; ++i) {
+    Buff.readInt(); // Block flags;
+    Blocks.push_back(new GCOVBlock(i));
+  }
+
+  // read edges.
+  while (Buff.readEdgeTag()) {
+    uint32_t EdgeCount = (Buff.readInt() - 1) / 2;
+    uint32_t BlockNo = Buff.readInt();
+    assert (BlockNo < BlockCount && "Unexpected Block number!");
+    for (int i = 0, e = EdgeCount; i != e; ++i) {
+      Blocks[BlockNo]->addEdge(Buff.readInt());
+      Buff.readInt(); // Edge flag
+    }
+  }
+
+  // read line table.
+  while (Buff.readLineTag()) {
+    uint32_t LineTableLength = Buff.readInt();
+    uint32_t Size = Buff.getCursor() + LineTableLength*4;
+    uint32_t BlockNo = Buff.readInt();
+    assert (BlockNo < BlockCount && "Unexpected Block number!");
+    GCOVBlock *Block = Blocks[BlockNo];
+    Buff.readInt(); // flag
+    while (Buff.getCursor() != (Size - 4)) {
+      StringRef Filename = Buff.readString();
+      if (Buff.getCursor() == (Size - 4)) break;
+      while (uint32_t L = Buff.readInt())
+	Block->addLine(Filename, L);
+    }
+    Buff.readInt(); // flag
+  }
+  return true;
+}
+
+/// dump - Dump GCOVFunction content on standard out for debugging purposes.
+void GCOVFunction::dump() {
+  outs() <<  "===== " << Name << " @ " << Filename << ":" << LineNumber << "\n";
+  for (SmallVector<GCOVBlock *, 16>::iterator I = Blocks.begin(),
+	 E = Blocks.end(); I != E; ++I)
+    (*I)->dump();
+}
+
+/// collectLineCounts - Collect line counts. This must be used after
+/// reading .gcno and .gcda files.
+void GCOVFunction::collectLineCounts(FileInfo &FI) {
+  for (SmallVector<GCOVBlock *, 16>::iterator I = Blocks.begin(),
+	 E = Blocks.end(); I != E; ++I)
+    (*I)->collectLineCounts(FI);
+}
+
+//===----------------------------------------------------------------------===//
+// GCOVBlock implementation.
+
+/// ~GCOVBlock - Delete GCOVBlock and its content.
+GCOVBlock::~GCOVBlock() {
+  Edges.clear();
+  DeleteContainerSeconds(Lines);
+}
+
+void GCOVBlock::addLine(StringRef Filename, uint32_t LineNo) {
+  GCOVLines *&LinesForFile = Lines[Filename];
+  if (!LinesForFile)
+    LinesForFile = new GCOVLines();
+  LinesForFile->add(LineNo);
+}
+
+/// collectLineCounts - Collect line counts. This must be used after
+/// reading .gcno and .gcda files.
+void GCOVBlock::collectLineCounts(FileInfo &FI) {
+  for (StringMap<GCOVLines *>::iterator I = Lines.begin(),
+	 E = Lines.end(); I != E; ++I)
+    I->second->collectLineCounts(FI, I->first(), Counter);
+}
+
+/// dump - Dump GCOVBlock content on standard out for debugging purposes.
+void GCOVBlock::dump() {
+  outs() << "Block : " << Number << " Counter : " << Counter << "\n";
+  if (!Edges.empty()) {
+    outs() << "\tEdges : ";
+    for (SmallVector<uint32_t, 16>::iterator I = Edges.begin(), E = Edges.end();
+	 I != E; ++I)
+      outs() << (*I) << ",";
+    outs() << "\n";
+  }
+  if (!Lines.empty()) {
+    outs() << "\tLines : ";
+    for (StringMap<GCOVLines *>::iterator LI = Lines.begin(),
+	   LE = Lines.end(); LI != LE; ++LI) {
+      outs() << LI->first() << " -> ";
+      LI->second->dump();
+      outs() << "\n";
+    }
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// GCOVLines implementation.
+
+/// collectLineCounts - Collect line counts. This must be used after
+/// reading .gcno and .gcda files.
+void GCOVLines::collectLineCounts(FileInfo &FI, StringRef Filename, 
+				  uint32_t Count) {
+  for (SmallVector<uint32_t, 16>::iterator I = Lines.begin(),
+	 E = Lines.end(); I != E; ++I)
+    FI.addLineCount(Filename, *I, Count);
+}
+
+/// dump - Dump GCOVLines content on standard out for debugging purposes.
+void GCOVLines::dump() {
+  for (SmallVector<uint32_t, 16>::iterator I = Lines.begin(),
+	 E = Lines.end(); I != E; ++I)
+    outs() << (*I) << ",";
+}
+
+//===----------------------------------------------------------------------===//
+// FileInfo implementation.
+
+/// addLineCount - Add line count for the given line number in a file.
+void FileInfo::addLineCount(StringRef Filename, uint32_t Line, uint32_t Count) {
+  if (LineInfo.find(Filename) == LineInfo.end()) {
+    OwningPtr<MemoryBuffer> Buff;
+    if (error_code ec = MemoryBuffer::getFileOrSTDIN(Filename, Buff)) {
+      errs() << Filename << ": " << ec.message() << "\n";
+      return;
+    }
+    StringRef AllLines = Buff.take()->getBuffer();
+    LineCounts L(AllLines.count('\n')+2);
+    L[Line-1] = Count;
+    LineInfo[Filename] = L;
+    return;
+  }
+  LineCounts &L = LineInfo[Filename];
+  L[Line-1] = Count;
+}
+
+/// print -  Print source files with collected line count information.
+void FileInfo::print() {
+  for (StringMap<LineCounts>::iterator I = LineInfo.begin(), E = LineInfo.end();
+       I != E; ++I) {
+    StringRef Filename = I->first();
+    outs() << Filename << "\n";
+    LineCounts &L = LineInfo[Filename];
+    OwningPtr<MemoryBuffer> Buff;
+    if (error_code ec = MemoryBuffer::getFileOrSTDIN(Filename, Buff)) {
+      errs() << Filename << ": " << ec.message() << "\n";
+      return;
+    }
+    StringRef AllLines = Buff.take()->getBuffer();
+    for (unsigned i = 0, e = L.size(); i != e; ++i) {
+      if (L[i])
+	outs() << L[i] << ":\t";
+      else
+	outs() << " :\t";
+      std::pair<StringRef, StringRef> P = AllLines.split('\n');
+      if (AllLines != P.first)
+	outs() << P.first;
+      outs() << "\n";
+      AllLines = P.second;
+    }
+  }
+}
+
+
diff --git a/contrib/llvm/lib/VMCore/Globals.cpp b/contrib/llvm/lib/VMCore/Globals.cpp
index db008e09d1c8..4254fb29e88b 100644
--- a/contrib/llvm/lib/VMCore/Globals.cpp
+++ b/contrib/llvm/lib/VMCore/Globals.cpp
@@ -80,7 +80,7 @@ bool GlobalValue::isDeclaration() const {
 // GlobalVariable Implementation
 //===----------------------------------------------------------------------===//
 
-GlobalVariable::GlobalVariable(const Type *Ty, bool constant, LinkageTypes Link,
+GlobalVariable::GlobalVariable(Type *Ty, bool constant, LinkageTypes Link,
                                Constant *InitVal, const Twine &Name,
                                bool ThreadLocal, unsigned AddressSpace)
   : GlobalValue(PointerType::get(Ty, AddressSpace), 
@@ -97,7 +97,7 @@ GlobalVariable::GlobalVariable(const Type *Ty, bool constant, LinkageTypes Link,
   LeakDetector::addGarbageObject(this);
 }
 
-GlobalVariable::GlobalVariable(Module &M, const Type *Ty, bool constant,
+GlobalVariable::GlobalVariable(Module &M, Type *Ty, bool constant,
                                LinkageTypes Link, Constant *InitVal,
                                const Twine &Name,
                                GlobalVariable *Before, bool ThreadLocal,
@@ -186,7 +186,7 @@ void GlobalVariable::copyAttributesFrom(const GlobalValue *Src) {
 // GlobalAlias Implementation
 //===----------------------------------------------------------------------===//
 
-GlobalAlias::GlobalAlias(const Type *Ty, LinkageTypes Link,
+GlobalAlias::GlobalAlias(Type *Ty, LinkageTypes Link,
                          const Twine &Name, Constant* aliasee,
                          Module *ParentModule)
   : GlobalValue(Ty, Value::GlobalAliasVal, &Op<0>(), 1, Link, Name) {
@@ -235,7 +235,7 @@ const GlobalValue *GlobalAlias::getAliasedGlobal() const {
           CE->getOpcode() == Instruction::GetElementPtr) &&
          "Unsupported aliasee");
   
-  return dyn_cast<GlobalValue>(CE->getOperand(0));
+  return cast<GlobalValue>(CE->getOperand(0));
 }
 
 const GlobalValue *GlobalAlias::resolveAliasedGlobal(bool stopOnWeak) const {
diff --git a/contrib/llvm/lib/VMCore/IRBuilder.cpp b/contrib/llvm/lib/VMCore/IRBuilder.cpp
index ffe961fee7c2..5114e2d498c5 100644
--- a/contrib/llvm/lib/VMCore/IRBuilder.cpp
+++ b/contrib/llvm/lib/VMCore/IRBuilder.cpp
@@ -40,7 +40,7 @@ Type *IRBuilderBase::getCurrentFunctionReturnType() const {
 }
 
 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
-  const PointerType *PT = cast<PointerType>(Ptr->getType());
+  PointerType *PT = cast<PointerType>(Ptr->getType());
   if (PT->getElementType()->isIntegerTy(8))
     return Ptr;
   
diff --git a/contrib/llvm/lib/VMCore/InlineAsm.cpp b/contrib/llvm/lib/VMCore/InlineAsm.cpp
index 4a03b395e98e..736e370a6de6 100644
--- a/contrib/llvm/lib/VMCore/InlineAsm.cpp
+++ b/contrib/llvm/lib/VMCore/InlineAsm.cpp
@@ -25,7 +25,7 @@ InlineAsm::~InlineAsm() {
 }
 
 
-InlineAsm *InlineAsm::get(const FunctionType *Ty, StringRef AsmString,
+InlineAsm *InlineAsm::get(FunctionType *Ty, StringRef AsmString,
                           StringRef Constraints, bool hasSideEffects,
                           bool isAlignStack) {
   InlineAsmKeyType Key(AsmString, Constraints, hasSideEffects, isAlignStack);
@@ -33,7 +33,7 @@ InlineAsm *InlineAsm::get(const FunctionType *Ty, StringRef AsmString,
   return pImpl->InlineAsms.getOrCreate(PointerType::getUnqual(Ty), Key);
 }
 
-InlineAsm::InlineAsm(const PointerType *Ty, const std::string &asmString,
+InlineAsm::InlineAsm(PointerType *Ty, const std::string &asmString,
                      const std::string &constraints, bool hasSideEffects,
                      bool isAlignStack)
   : Value(Ty, Value::InlineAsmVal),
@@ -242,7 +242,7 @@ InlineAsm::ParseConstraints(StringRef Constraints) {
 
 /// Verify - Verify that the specified constraint string is reasonable for the
 /// specified function type, and otherwise validate the constraint string.
-bool InlineAsm::Verify(const FunctionType *Ty, StringRef ConstStr) {
+bool InlineAsm::Verify(FunctionType *Ty, StringRef ConstStr) {
   if (Ty->isVarArg()) return false;
   
   ConstraintInfoVector Constraints = ParseConstraints(ConstStr);
@@ -282,7 +282,7 @@ bool InlineAsm::Verify(const FunctionType *Ty, StringRef ConstStr) {
     if (Ty->getReturnType()->isStructTy()) return false;
     break;
   default:
-    const StructType *STy = dyn_cast<StructType>(Ty->getReturnType());
+    StructType *STy = dyn_cast<StructType>(Ty->getReturnType());
     if (STy == 0 || STy->getNumElements() != NumOutputs)
       return false;
     break;
diff --git a/contrib/llvm/lib/VMCore/Instruction.cpp b/contrib/llvm/lib/VMCore/Instruction.cpp
index 02c075743959..73191c19658c 100644
--- a/contrib/llvm/lib/VMCore/Instruction.cpp
+++ b/contrib/llvm/lib/VMCore/Instruction.cpp
@@ -20,7 +20,7 @@
 #include "llvm/Support/LeakDetector.h"
 using namespace llvm;
 
-Instruction::Instruction(const Type *ty, unsigned it, Use *Ops, unsigned NumOps,
+Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps,
                          Instruction *InsertBefore)
   : User(ty, Value::InstructionVal + it, Ops, NumOps), Parent(0) {
   // Make sure that we get added to a basicblock
@@ -34,7 +34,7 @@ Instruction::Instruction(const Type *ty, unsigned it, Use *Ops, unsigned NumOps,
   }
 }
 
-Instruction::Instruction(const Type *ty, unsigned it, Use *Ops, unsigned NumOps,
+Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps,
                          BasicBlock *InsertAtEnd)
   : User(ty, Value::InstructionVal + it, Ops, NumOps), Parent(0) {
   // Make sure that we get added to a basicblock
@@ -101,6 +101,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case Switch: return "switch";
   case IndirectBr: return "indirectbr";
   case Invoke: return "invoke";
+  case Resume: return "resume";
   case Unwind: return "unwind";
   case Unreachable: return "unreachable";
 
@@ -127,6 +128,9 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case Alloca:        return "alloca";
   case Load:          return "load";
   case Store:         return "store";
+  case AtomicCmpXchg: return "cmpxchg";
+  case AtomicRMW:     return "atomicrmw";
+  case Fence:         return "fence";
   case GetElementPtr: return "getelementptr";
 
   // Convert instructions...
@@ -158,6 +162,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case ShuffleVector:  return "shufflevector";
   case ExtractValue:   return "extractvalue";
   case InsertValue:    return "insertvalue";
+  case LandingPad:     return "landingpad";
 
   default: return "<Invalid operator> ";
   }
@@ -191,10 +196,14 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
   // Check special state that is a part of some instructions.
   if (const LoadInst *LI = dyn_cast<LoadInst>(this))
     return LI->isVolatile() == cast<LoadInst>(I)->isVolatile() &&
-           LI->getAlignment() == cast<LoadInst>(I)->getAlignment();
+           LI->getAlignment() == cast<LoadInst>(I)->getAlignment() &&
+           LI->getOrdering() == cast<LoadInst>(I)->getOrdering() &&
+           LI->getSynchScope() == cast<LoadInst>(I)->getSynchScope();
   if (const StoreInst *SI = dyn_cast<StoreInst>(this))
     return SI->isVolatile() == cast<StoreInst>(I)->isVolatile() &&
-           SI->getAlignment() == cast<StoreInst>(I)->getAlignment();
+           SI->getAlignment() == cast<StoreInst>(I)->getAlignment() &&
+           SI->getOrdering() == cast<StoreInst>(I)->getOrdering() &&
+           SI->getSynchScope() == cast<StoreInst>(I)->getSynchScope();
   if (const CmpInst *CI = dyn_cast<CmpInst>(this))
     return CI->getPredicate() == cast<CmpInst>(I)->getPredicate();
   if (const CallInst *CI = dyn_cast<CallInst>(this))
@@ -208,6 +217,18 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
     return IVI->getIndices() == cast<InsertValueInst>(I)->getIndices();
   if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(this))
     return EVI->getIndices() == cast<ExtractValueInst>(I)->getIndices();
+  if (const FenceInst *FI = dyn_cast<FenceInst>(this))
+    return FI->getOrdering() == cast<FenceInst>(FI)->getOrdering() &&
+           FI->getSynchScope() == cast<FenceInst>(FI)->getSynchScope();
+  if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(this))
+    return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I)->isVolatile() &&
+           CXI->getOrdering() == cast<AtomicCmpXchgInst>(I)->getOrdering() &&
+           CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I)->getSynchScope();
+  if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(this))
+    return RMWI->getOperation() == cast<AtomicRMWInst>(I)->getOperation() &&
+           RMWI->isVolatile() == cast<AtomicRMWInst>(I)->isVolatile() &&
+           RMWI->getOrdering() == cast<AtomicRMWInst>(I)->getOrdering() &&
+           RMWI->getSynchScope() == cast<AtomicRMWInst>(I)->getSynchScope();
 
   return true;
 }
@@ -230,10 +251,14 @@ bool Instruction::isSameOperationAs(const Instruction *I) const {
   // Check special state that is a part of some instructions.
   if (const LoadInst *LI = dyn_cast<LoadInst>(this))
     return LI->isVolatile() == cast<LoadInst>(I)->isVolatile() &&
-           LI->getAlignment() == cast<LoadInst>(I)->getAlignment();
+           LI->getAlignment() == cast<LoadInst>(I)->getAlignment() &&
+           LI->getOrdering() == cast<LoadInst>(I)->getOrdering() &&
+           LI->getSynchScope() == cast<LoadInst>(I)->getSynchScope();
   if (const StoreInst *SI = dyn_cast<StoreInst>(this))
     return SI->isVolatile() == cast<StoreInst>(I)->isVolatile() &&
-           SI->getAlignment() == cast<StoreInst>(I)->getAlignment();
+           SI->getAlignment() == cast<StoreInst>(I)->getAlignment() &&
+           SI->getOrdering() == cast<StoreInst>(I)->getOrdering() &&
+           SI->getSynchScope() == cast<StoreInst>(I)->getSynchScope();
   if (const CmpInst *CI = dyn_cast<CmpInst>(this))
     return CI->getPredicate() == cast<CmpInst>(I)->getPredicate();
   if (const CallInst *CI = dyn_cast<CallInst>(this))
@@ -248,6 +273,18 @@ bool Instruction::isSameOperationAs(const Instruction *I) const {
     return IVI->getIndices() == cast<InsertValueInst>(I)->getIndices();
   if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(this))
     return EVI->getIndices() == cast<ExtractValueInst>(I)->getIndices();
+  if (const FenceInst *FI = dyn_cast<FenceInst>(this))
+    return FI->getOrdering() == cast<FenceInst>(I)->getOrdering() &&
+           FI->getSynchScope() == cast<FenceInst>(I)->getSynchScope();
+  if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(this))
+    return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I)->isVolatile() &&
+           CXI->getOrdering() == cast<AtomicCmpXchgInst>(I)->getOrdering() &&
+           CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I)->getSynchScope();
+  if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(this))
+    return RMWI->getOperation() == cast<AtomicRMWInst>(I)->getOperation() &&
+           RMWI->isVolatile() == cast<AtomicRMWInst>(I)->isVolatile() &&
+           RMWI->getOrdering() == cast<AtomicRMWInst>(I)->getOrdering() &&
+           RMWI->getSynchScope() == cast<AtomicRMWInst>(I)->getSynchScope();
 
   return true;
 }
@@ -280,13 +317,16 @@ bool Instruction::mayReadFromMemory() const {
   default: return false;
   case Instruction::VAArg:
   case Instruction::Load:
+  case Instruction::Fence: // FIXME: refine definition of mayReadFromMemory
+  case Instruction::AtomicCmpXchg:
+  case Instruction::AtomicRMW:
     return true;
   case Instruction::Call:
     return !cast<CallInst>(this)->doesNotAccessMemory();
   case Instruction::Invoke:
     return !cast<InvokeInst>(this)->doesNotAccessMemory();
   case Instruction::Store:
-    return cast<StoreInst>(this)->isVolatile();
+    return !cast<StoreInst>(this)->isUnordered();
   }
 }
 
@@ -295,15 +335,18 @@ bool Instruction::mayReadFromMemory() const {
 bool Instruction::mayWriteToMemory() const {
   switch (getOpcode()) {
   default: return false;
+  case Instruction::Fence: // FIXME: refine definition of mayWriteToMemory
   case Instruction::Store:
   case Instruction::VAArg:
+  case Instruction::AtomicCmpXchg:
+  case Instruction::AtomicRMW:
     return true;
   case Instruction::Call:
     return !cast<CallInst>(this)->onlyReadsMemory();
   case Instruction::Invoke:
     return !cast<InvokeInst>(this)->onlyReadsMemory();
   case Instruction::Load:
-    return cast<LoadInst>(this)->isVolatile();
+    return !cast<LoadInst>(this)->isUnordered();
   }
 }
 
@@ -312,7 +355,7 @@ bool Instruction::mayWriteToMemory() const {
 bool Instruction::mayThrow() const {
   if (const CallInst *CI = dyn_cast<CallInst>(this))
     return !CI->doesNotThrow();
-  return false;
+  return isa<ResumeInst>(this);
 }
 
 /// isAssociative - Return true if the instruction is associative:
@@ -372,7 +415,7 @@ bool Instruction::isSafeToSpeculativelyExecute() const {
   }
   case Load: {
     const LoadInst *LI = cast<LoadInst>(this);
-    if (LI->isVolatile())
+    if (!LI->isUnordered())
       return false;
     return LI->getPointerOperand()->isDereferenceablePointer();
   }
@@ -392,6 +435,11 @@ bool Instruction::isSafeToSpeculativelyExecute() const {
   case Switch:
   case Unwind:
   case Unreachable:
+  case Fence:
+  case LandingPad:
+  case AtomicRMW:
+  case AtomicCmpXchg:
+  case Resume:
     return false; // Misc instructions which have effects
   }
 }
diff --git a/contrib/llvm/lib/VMCore/Instructions.cpp b/contrib/llvm/lib/VMCore/Instructions.cpp
index 9baad09cb272..b3a720527a8e 100644
--- a/contrib/llvm/lib/VMCore/Instructions.cpp
+++ b/contrib/llvm/lib/VMCore/Instructions.cpp
@@ -62,11 +62,11 @@ const char *SelectInst::areInvalidOperands(Value *Op0, Value *Op1, Value *Op2) {
   if (Op1->getType() != Op2->getType())
     return "both values to select must have same type";
   
-  if (const VectorType *VT = dyn_cast<VectorType>(Op0->getType())) {
+  if (VectorType *VT = dyn_cast<VectorType>(Op0->getType())) {
     // Vector select.
     if (VT->getElementType() != Type::getInt1Ty(Op0->getContext()))
       return "vector select condition element type must be i1";
-    const VectorType *ET = dyn_cast<VectorType>(Op1->getType());
+    VectorType *ET = dyn_cast<VectorType>(Op1->getType());
     if (ET == 0)
       return "selected values for vector select must be vectors";
     if (ET->getNumElements() != VT->getNumElements())
@@ -166,6 +166,88 @@ Value *PHINode::hasConstantValue() const {
   return ConstantValue;
 }
 
+//===----------------------------------------------------------------------===//
+//                       LandingPadInst Implementation
+//===----------------------------------------------------------------------===//
+
+LandingPadInst::LandingPadInst(Type *RetTy, Value *PersonalityFn,
+                               unsigned NumReservedValues, const Twine &NameStr,
+                               Instruction *InsertBefore)
+  : Instruction(RetTy, Instruction::LandingPad, 0, 0, InsertBefore) {
+  init(PersonalityFn, 1 + NumReservedValues, NameStr);
+}
+
+LandingPadInst::LandingPadInst(Type *RetTy, Value *PersonalityFn,
+                               unsigned NumReservedValues, const Twine &NameStr,
+                               BasicBlock *InsertAtEnd)
+  : Instruction(RetTy, Instruction::LandingPad, 0, 0, InsertAtEnd) {
+  init(PersonalityFn, 1 + NumReservedValues, NameStr);
+}
+
+LandingPadInst::LandingPadInst(const LandingPadInst &LP)
+  : Instruction(LP.getType(), Instruction::LandingPad,
+                allocHungoffUses(LP.getNumOperands()), LP.getNumOperands()),
+    ReservedSpace(LP.getNumOperands()) {
+  Use *OL = OperandList, *InOL = LP.OperandList;
+  for (unsigned I = 0, E = ReservedSpace; I != E; ++I)
+    OL[I] = InOL[I];
+
+  setCleanup(LP.isCleanup());
+}
+
+LandingPadInst::~LandingPadInst() {
+  dropHungoffUses();
+}
+
+LandingPadInst *LandingPadInst::Create(Type *RetTy, Value *PersonalityFn,
+                                       unsigned NumReservedClauses,
+                                       const Twine &NameStr,
+                                       Instruction *InsertBefore) {
+  return new LandingPadInst(RetTy, PersonalityFn, NumReservedClauses, NameStr,
+                            InsertBefore);
+}
+
+LandingPadInst *LandingPadInst::Create(Type *RetTy, Value *PersonalityFn,
+                                       unsigned NumReservedClauses,
+                                       const Twine &NameStr,
+                                       BasicBlock *InsertAtEnd) {
+  return new LandingPadInst(RetTy, PersonalityFn, NumReservedClauses, NameStr,
+                            InsertAtEnd);
+}
+
+void LandingPadInst::init(Value *PersFn, unsigned NumReservedValues,
+                          const Twine &NameStr) {
+  ReservedSpace = NumReservedValues;
+  NumOperands = 1;
+  OperandList = allocHungoffUses(ReservedSpace);
+  OperandList[0] = PersFn;
+  setName(NameStr);
+  setCleanup(false);
+}
+
+/// growOperands - grow operands - This grows the operand list in response to a
+/// push_back style of operation. This grows the number of ops by 2 times.
+void LandingPadInst::growOperands(unsigned Size) {
+  unsigned e = getNumOperands();
+  if (ReservedSpace >= e + Size) return;
+  ReservedSpace = (e + Size / 2) * 2;
+
+  Use *NewOps = allocHungoffUses(ReservedSpace);
+  Use *OldOps = OperandList;
+  for (unsigned i = 0; i != e; ++i)
+      NewOps[i] = OldOps[i];
+
+  OperandList = NewOps;
+  Use::zap(OldOps, OldOps + e, true);
+}
+
+void LandingPadInst::addClause(Value *Val) {
+  unsigned OpNo = getNumOperands();
+  growOperands(1);
+  assert(OpNo < ReservedSpace && "Growing didn't work!");
+  ++NumOperands;
+  OperandList[OpNo] = Val;
+}
 
 //===----------------------------------------------------------------------===//
 //                        CallInst Implementation
@@ -179,7 +261,7 @@ void CallInst::init(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr) {
   Op<-1>() = Func;
 
 #ifndef NDEBUG
-  const FunctionType *FTy =
+  FunctionType *FTy =
     cast<FunctionType>(cast<PointerType>(Func->getType())->getElementType());
 
   assert((Args.size() == FTy->getNumParams() ||
@@ -201,7 +283,7 @@ void CallInst::init(Value *Func, const Twine &NameStr) {
   Op<-1>() = Func;
 
 #ifndef NDEBUG
-  const FunctionType *FTy =
+  FunctionType *FTy =
     cast<FunctionType>(cast<PointerType>(Func->getType())->getElementType());
 
   assert(FTy->getNumParams() == 0 && "Calling a function with bad signature");
@@ -269,8 +351,8 @@ static bool IsConstantOne(Value *val) {
 }
 
 static Instruction *createMalloc(Instruction *InsertBefore,
-                                 BasicBlock *InsertAtEnd, const Type *IntPtrTy,
-                                 const Type *AllocTy, Value *AllocSize, 
+                                 BasicBlock *InsertAtEnd, Type *IntPtrTy,
+                                 Type *AllocTy, Value *AllocSize, 
                                  Value *ArraySize, Function *MallocF,
                                  const Twine &Name) {
   assert(((!InsertBefore && InsertAtEnd) || (InsertBefore && !InsertAtEnd)) &&
@@ -319,7 +401,7 @@ static Instruction *createMalloc(Instruction *InsertBefore,
   if (!MallocFunc)
     // prototype malloc as "void *malloc(size_t)"
     MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy, NULL);
-  const PointerType *AllocPtrType = PointerType::getUnqual(AllocTy);
+  PointerType *AllocPtrType = PointerType::getUnqual(AllocTy);
   CallInst *MCall = NULL;
   Instruction *Result = NULL;
   if (InsertBefore) {
@@ -354,7 +436,7 @@ static Instruction *createMalloc(Instruction *InsertBefore,
 /// 2. Call malloc with that argument.
 /// 3. Bitcast the result of the malloc call to the specified type.
 Instruction *CallInst::CreateMalloc(Instruction *InsertBefore,
-                                    const Type *IntPtrTy, const Type *AllocTy,
+                                    Type *IntPtrTy, Type *AllocTy,
                                     Value *AllocSize, Value *ArraySize,
                                     Function * MallocF,
                                     const Twine &Name) {
@@ -371,7 +453,7 @@ Instruction *CallInst::CreateMalloc(Instruction *InsertBefore,
 /// Note: This function does not add the bitcast to the basic block, that is the
 /// responsibility of the caller.
 Instruction *CallInst::CreateMalloc(BasicBlock *InsertAtEnd,
-                                    const Type *IntPtrTy, const Type *AllocTy,
+                                    Type *IntPtrTy, Type *AllocTy,
                                     Value *AllocSize, Value *ArraySize, 
                                     Function *MallocF, const Twine &Name) {
   return createMalloc(NULL, InsertAtEnd, IntPtrTy, AllocTy, AllocSize,
@@ -388,8 +470,8 @@ static Instruction* createFree(Value* Source, Instruction *InsertBefore,
   BasicBlock* BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd;
   Module* M = BB->getParent()->getParent();
 
-  const Type *VoidTy = Type::getVoidTy(M->getContext());
-  const Type *IntPtrTy = Type::getInt8PtrTy(M->getContext());
+  Type *VoidTy = Type::getVoidTy(M->getContext());
+  Type *IntPtrTy = Type::getInt8PtrTy(M->getContext());
   // prototype free as "void free(void*)"
   Value *FreeFunc = M->getOrInsertFunction("free", VoidTy, IntPtrTy, NULL);
   CallInst* Result = NULL;
@@ -436,7 +518,7 @@ void InvokeInst::init(Value *Fn, BasicBlock *IfNormal, BasicBlock *IfException,
   Op<-1>() = IfException;
 
 #ifndef NDEBUG
-  const FunctionType *FTy =
+  FunctionType *FTy =
     cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType());
 
   assert(((Args.size() == FTy->getNumParams()) ||
@@ -494,6 +576,9 @@ void InvokeInst::removeAttribute(unsigned i, Attributes attr) {
   setAttributes(PAL);
 }
 
+LandingPadInst *InvokeInst::getLandingPadInst() const {
+  return cast<LandingPadInst>(getUnwindDest()->getFirstNonPHI());
+}
 
 //===----------------------------------------------------------------------===//
 //                        ReturnInst Implementation
@@ -573,6 +658,41 @@ BasicBlock *UnwindInst::getSuccessorV(unsigned idx) const {
   return 0;
 }
 
+//===----------------------------------------------------------------------===//
+//                        ResumeInst Implementation
+//===----------------------------------------------------------------------===//
+
+ResumeInst::ResumeInst(const ResumeInst &RI)
+  : TerminatorInst(Type::getVoidTy(RI.getContext()), Instruction::Resume,
+                   OperandTraits<ResumeInst>::op_begin(this), 1) {
+  Op<0>() = RI.Op<0>();
+}
+
+ResumeInst::ResumeInst(Value *Exn, Instruction *InsertBefore)
+  : TerminatorInst(Type::getVoidTy(Exn->getContext()), Instruction::Resume,
+                   OperandTraits<ResumeInst>::op_begin(this), 1, InsertBefore) {
+  Op<0>() = Exn;
+}
+
+ResumeInst::ResumeInst(Value *Exn, BasicBlock *InsertAtEnd)
+  : TerminatorInst(Type::getVoidTy(Exn->getContext()), Instruction::Resume,
+                   OperandTraits<ResumeInst>::op_begin(this), 1, InsertAtEnd) {
+  Op<0>() = Exn;
+}
+
+unsigned ResumeInst::getNumSuccessorsV() const {
+  return getNumSuccessors();
+}
+
+void ResumeInst::setSuccessorV(unsigned idx, BasicBlock *NewSucc) {
+  llvm_unreachable("ResumeInst has no successors!");
+}
+
+BasicBlock *ResumeInst::getSuccessorV(unsigned idx) const {
+  llvm_unreachable("ResumeInst has no successors!");
+  return 0;
+}
+
 //===----------------------------------------------------------------------===//
 //                      UnreachableInst Implementation
 //===----------------------------------------------------------------------===//
@@ -665,6 +785,27 @@ BranchInst::BranchInst(const BranchInst &BI) :
   SubclassOptionalData = BI.SubclassOptionalData;
 }
 
+void BranchInst::swapSuccessors() {
+  assert(isConditional() &&
+         "Cannot swap successors of an unconditional branch");
+  Op<-1>().swap(Op<-2>());
+
+  // Update profile metadata if present and it matches our structural
+  // expectations.
+  MDNode *ProfileData = getMetadata(LLVMContext::MD_prof);
+  if (!ProfileData || ProfileData->getNumOperands() != 3)
+    return;
+
+  // The first operand is the name. Fetch them backwards and build a new one.
+  Value *Ops[] = {
+    ProfileData->getOperand(0),
+    ProfileData->getOperand(2),
+    ProfileData->getOperand(1)
+  };
+  setMetadata(LLVMContext::MD_prof,
+              MDNode::get(ProfileData->getContext(), Ops));
+}
+
 BasicBlock *BranchInst::getSuccessorV(unsigned idx) const {
   return getSuccessor(idx);
 }
@@ -692,7 +833,7 @@ static Value *getAISize(LLVMContext &Context, Value *Amt) {
   return Amt;
 }
 
-AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize,
+AllocaInst::AllocaInst(Type *Ty, Value *ArraySize,
                        const Twine &Name, Instruction *InsertBefore)
   : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
                      getAISize(Ty->getContext(), ArraySize), InsertBefore) {
@@ -701,7 +842,7 @@ AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize,
   setName(Name);
 }
 
-AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize,
+AllocaInst::AllocaInst(Type *Ty, Value *ArraySize,
                        const Twine &Name, BasicBlock *InsertAtEnd)
   : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
                      getAISize(Ty->getContext(), ArraySize), InsertAtEnd) {
@@ -710,7 +851,7 @@ AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize,
   setName(Name);
 }
 
-AllocaInst::AllocaInst(const Type *Ty, const Twine &Name,
+AllocaInst::AllocaInst(Type *Ty, const Twine &Name,
                        Instruction *InsertBefore)
   : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
                      getAISize(Ty->getContext(), 0), InsertBefore) {
@@ -719,7 +860,7 @@ AllocaInst::AllocaInst(const Type *Ty, const Twine &Name,
   setName(Name);
 }
 
-AllocaInst::AllocaInst(const Type *Ty, const Twine &Name,
+AllocaInst::AllocaInst(Type *Ty, const Twine &Name,
                        BasicBlock *InsertAtEnd)
   : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
                      getAISize(Ty->getContext(), 0), InsertAtEnd) {
@@ -728,7 +869,7 @@ AllocaInst::AllocaInst(const Type *Ty, const Twine &Name,
   setName(Name);
 }
 
-AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize, unsigned Align,
+AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
                        const Twine &Name, Instruction *InsertBefore)
   : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
                      getAISize(Ty->getContext(), ArraySize), InsertBefore) {
@@ -737,7 +878,7 @@ AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize, unsigned Align,
   setName(Name);
 }
 
-AllocaInst::AllocaInst(const Type *Ty, Value *ArraySize, unsigned Align,
+AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
                        const Twine &Name, BasicBlock *InsertAtEnd)
   : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
                      getAISize(Ty->getContext(), ArraySize), InsertAtEnd) {
@@ -787,6 +928,8 @@ bool AllocaInst::isStaticAlloca() const {
 void LoadInst::AssertOK() {
   assert(getOperand(0)->getType()->isPointerTy() &&
          "Ptr must have pointer type.");
+  assert(!(isAtomic() && getAlignment() == 0) &&
+         "Alignment required for atomic load");
 }
 
 LoadInst::LoadInst(Value *Ptr, const Twine &Name, Instruction *InsertBef)
@@ -794,6 +937,7 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, Instruction *InsertBef)
                      Load, Ptr, InsertBef) {
   setVolatile(false);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   setName(Name);
 }
@@ -803,6 +947,7 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, BasicBlock *InsertAE)
                      Load, Ptr, InsertAE) {
   setVolatile(false);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   setName(Name);
 }
@@ -813,6 +958,18 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
                      Load, Ptr, InsertBef) {
   setVolatile(isVolatile);
   setAlignment(0);
+  setAtomic(NotAtomic);
+  AssertOK();
+  setName(Name);
+}
+
+LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
+                   BasicBlock *InsertAE)
+  : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
+                     Load, Ptr, InsertAE) {
+  setVolatile(isVolatile);
+  setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   setName(Name);
 }
@@ -823,6 +980,7 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
                      Load, Ptr, InsertBef) {
   setVolatile(isVolatile);
   setAlignment(Align);
+  setAtomic(NotAtomic);
   AssertOK();
   setName(Name);
 }
@@ -833,27 +991,43 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
                      Load, Ptr, InsertAE) {
   setVolatile(isVolatile);
   setAlignment(Align);
+  setAtomic(NotAtomic);
   AssertOK();
   setName(Name);
 }
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
+LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, 
+                   unsigned Align, AtomicOrdering Order,
+                   SynchronizationScope SynchScope,
+                   Instruction *InsertBef)
+  : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
+                     Load, Ptr, InsertBef) {
+  setVolatile(isVolatile);
+  setAlignment(Align);
+  setAtomic(Order, SynchScope);
+  AssertOK();
+  setName(Name);
+}
+
+LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, 
+                   unsigned Align, AtomicOrdering Order,
+                   SynchronizationScope SynchScope,
                    BasicBlock *InsertAE)
   : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
                      Load, Ptr, InsertAE) {
   setVolatile(isVolatile);
-  setAlignment(0);
+  setAlignment(Align);
+  setAtomic(Order, SynchScope);
   AssertOK();
   setName(Name);
 }
 
-
-
 LoadInst::LoadInst(Value *Ptr, const char *Name, Instruction *InsertBef)
   : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
                      Load, Ptr, InsertBef) {
   setVolatile(false);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   if (Name && Name[0]) setName(Name);
 }
@@ -863,6 +1037,7 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, BasicBlock *InsertAE)
                      Load, Ptr, InsertAE) {
   setVolatile(false);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   if (Name && Name[0]) setName(Name);
 }
@@ -873,6 +1048,7 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, bool isVolatile,
                    Load, Ptr, InsertBef) {
   setVolatile(isVolatile);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   if (Name && Name[0]) setName(Name);
 }
@@ -883,6 +1059,7 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, bool isVolatile,
                      Load, Ptr, InsertAE) {
   setVolatile(isVolatile);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
   if (Name && Name[0]) setName(Name);
 }
@@ -891,7 +1068,7 @@ void LoadInst::setAlignment(unsigned Align) {
   assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
   assert(Align <= MaximumAlignment &&
          "Alignment is greater than MaximumAlignment!");
-  setInstructionSubclassData((getSubclassDataFromInstruction() & 1) |
+  setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) |
                              ((Log2_32(Align)+1)<<1));
   assert(getAlignment() == Align && "Alignment representation error!");
 }
@@ -907,6 +1084,8 @@ void StoreInst::AssertOK() {
   assert(getOperand(0)->getType() ==
                  cast<PointerType>(getOperand(1)->getType())->getElementType()
          && "Ptr must be a pointer to Val type!");
+  assert(!(isAtomic() && getAlignment() == 0) &&
+         "Alignment required for atomic load");
 }
 
 
@@ -919,6 +1098,7 @@ StoreInst::StoreInst(Value *val, Value *addr, Instruction *InsertBefore)
   Op<1>() = addr;
   setVolatile(false);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
 }
 
@@ -931,6 +1111,7 @@ StoreInst::StoreInst(Value *val, Value *addr, BasicBlock *InsertAtEnd)
   Op<1>() = addr;
   setVolatile(false);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
 }
 
@@ -944,6 +1125,7 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
   Op<1>() = addr;
   setVolatile(isVolatile);
   setAlignment(0);
+  setAtomic(NotAtomic);
   AssertOK();
 }
 
@@ -957,19 +1139,23 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
   Op<1>() = addr;
   setVolatile(isVolatile);
   setAlignment(Align);
+  setAtomic(NotAtomic);
   AssertOK();
 }
 
 StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
-                     unsigned Align, BasicBlock *InsertAtEnd)
+                     unsigned Align, AtomicOrdering Order,
+                     SynchronizationScope SynchScope,
+                     Instruction *InsertBefore)
   : Instruction(Type::getVoidTy(val->getContext()), Store,
                 OperandTraits<StoreInst>::op_begin(this),
                 OperandTraits<StoreInst>::operands(this),
-                InsertAtEnd) {
+                InsertBefore) {
   Op<0>() = val;
   Op<1>() = addr;
   setVolatile(isVolatile);
   setAlignment(Align);
+  setAtomic(Order, SynchScope);
   AssertOK();
 }
 
@@ -983,6 +1169,37 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
   Op<1>() = addr;
   setVolatile(isVolatile);
   setAlignment(0);
+  setAtomic(NotAtomic);
+  AssertOK();
+}
+
+StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
+                     unsigned Align, BasicBlock *InsertAtEnd)
+  : Instruction(Type::getVoidTy(val->getContext()), Store,
+                OperandTraits<StoreInst>::op_begin(this),
+                OperandTraits<StoreInst>::operands(this),
+                InsertAtEnd) {
+  Op<0>() = val;
+  Op<1>() = addr;
+  setVolatile(isVolatile);
+  setAlignment(Align);
+  setAtomic(NotAtomic);
+  AssertOK();
+}
+
+StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
+                     unsigned Align, AtomicOrdering Order,
+                     SynchronizationScope SynchScope,
+                     BasicBlock *InsertAtEnd)
+  : Instruction(Type::getVoidTy(val->getContext()), Store,
+                OperandTraits<StoreInst>::op_begin(this),
+                OperandTraits<StoreInst>::operands(this),
+                InsertAtEnd) {
+  Op<0>() = val;
+  Op<1>() = addr;
+  setVolatile(isVolatile);
+  setAlignment(Align);
+  setAtomic(Order, SynchScope);
   AssertOK();
 }
 
@@ -990,37 +1207,135 @@ void StoreInst::setAlignment(unsigned Align) {
   assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
   assert(Align <= MaximumAlignment &&
          "Alignment is greater than MaximumAlignment!");
-  setInstructionSubclassData((getSubclassDataFromInstruction() & 1) |
+  setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) |
                              ((Log2_32(Align)+1) << 1));
   assert(getAlignment() == Align && "Alignment representation error!");
 }
 
+//===----------------------------------------------------------------------===//
+//                       AtomicCmpXchgInst Implementation
+//===----------------------------------------------------------------------===//
+
+void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,
+                             AtomicOrdering Ordering,
+                             SynchronizationScope SynchScope) {
+  Op<0>() = Ptr;
+  Op<1>() = Cmp;
+  Op<2>() = NewVal;
+  setOrdering(Ordering);
+  setSynchScope(SynchScope);
+
+  assert(getOperand(0) && getOperand(1) && getOperand(2) &&
+         "All operands must be non-null!");
+  assert(getOperand(0)->getType()->isPointerTy() &&
+         "Ptr must have pointer type!");
+  assert(getOperand(1)->getType() ==
+                 cast<PointerType>(getOperand(0)->getType())->getElementType()
+         && "Ptr must be a pointer to Cmp type!");
+  assert(getOperand(2)->getType() ==
+                 cast<PointerType>(getOperand(0)->getType())->getElementType()
+         && "Ptr must be a pointer to NewVal type!");
+  assert(Ordering != NotAtomic &&
+         "AtomicCmpXchg instructions must be atomic!");
+}
+
+AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                                     AtomicOrdering Ordering,
+                                     SynchronizationScope SynchScope,
+                                     Instruction *InsertBefore)
+  : Instruction(Cmp->getType(), AtomicCmpXchg,
+                OperandTraits<AtomicCmpXchgInst>::op_begin(this),
+                OperandTraits<AtomicCmpXchgInst>::operands(this),
+                InsertBefore) {
+  Init(Ptr, Cmp, NewVal, Ordering, SynchScope);
+}
+
+AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                                     AtomicOrdering Ordering,
+                                     SynchronizationScope SynchScope,
+                                     BasicBlock *InsertAtEnd)
+  : Instruction(Cmp->getType(), AtomicCmpXchg,
+                OperandTraits<AtomicCmpXchgInst>::op_begin(this),
+                OperandTraits<AtomicCmpXchgInst>::operands(this),
+                InsertAtEnd) {
+  Init(Ptr, Cmp, NewVal, Ordering, SynchScope);
+}
+ 
+//===----------------------------------------------------------------------===//
+//                       AtomicRMWInst Implementation
+//===----------------------------------------------------------------------===//
+
+void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
+                         AtomicOrdering Ordering,
+                         SynchronizationScope SynchScope) {
+  Op<0>() = Ptr;
+  Op<1>() = Val;
+  setOperation(Operation);
+  setOrdering(Ordering);
+  setSynchScope(SynchScope);
+
+  assert(getOperand(0) && getOperand(1) &&
+         "All operands must be non-null!");
+  assert(getOperand(0)->getType()->isPointerTy() &&
+         "Ptr must have pointer type!");
+  assert(getOperand(1)->getType() ==
+         cast<PointerType>(getOperand(0)->getType())->getElementType()
+         && "Ptr must be a pointer to Val type!");
+  assert(Ordering != NotAtomic &&
+         "AtomicRMW instructions must be atomic!");
+}
+
+AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+                             AtomicOrdering Ordering,
+                             SynchronizationScope SynchScope,
+                             Instruction *InsertBefore)
+  : Instruction(Val->getType(), AtomicRMW,
+                OperandTraits<AtomicRMWInst>::op_begin(this),
+                OperandTraits<AtomicRMWInst>::operands(this),
+                InsertBefore) {
+  Init(Operation, Ptr, Val, Ordering, SynchScope);
+}
+
+AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+                             AtomicOrdering Ordering,
+                             SynchronizationScope SynchScope,
+                             BasicBlock *InsertAtEnd)
+  : Instruction(Val->getType(), AtomicRMW,
+                OperandTraits<AtomicRMWInst>::op_begin(this),
+                OperandTraits<AtomicRMWInst>::operands(this),
+                InsertAtEnd) {
+  Init(Operation, Ptr, Val, Ordering, SynchScope);
+}
+
+//===----------------------------------------------------------------------===//
+//                       FenceInst Implementation
+//===----------------------------------------------------------------------===//
+
+FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering, 
+                     SynchronizationScope SynchScope,
+                     Instruction *InsertBefore)
+  : Instruction(Type::getVoidTy(C), Fence, 0, 0, InsertBefore) {
+  setOrdering(Ordering);
+  setSynchScope(SynchScope);
+}
+
+FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering, 
+                     SynchronizationScope SynchScope,
+                     BasicBlock *InsertAtEnd)
+  : Instruction(Type::getVoidTy(C), Fence, 0, 0, InsertAtEnd) {
+  setOrdering(Ordering);
+  setSynchScope(SynchScope);
+}
+
 //===----------------------------------------------------------------------===//
 //                       GetElementPtrInst Implementation
 //===----------------------------------------------------------------------===//
 
-static unsigned retrieveAddrSpace(const Value *Val) {
-  return cast<PointerType>(Val->getType())->getAddressSpace();
-}
-
-void GetElementPtrInst::init(Value *Ptr, Value* const *Idx, unsigned NumIdx,
+void GetElementPtrInst::init(Value *Ptr, ArrayRef<Value *> IdxList,
                              const Twine &Name) {
-  assert(NumOperands == 1+NumIdx && "NumOperands not initialized?");
-  Use *OL = OperandList;
-  OL[0] = Ptr;
-
-  for (unsigned i = 0; i != NumIdx; ++i)
-    OL[i+1] = Idx[i];
-
-  setName(Name);
-}
-
-void GetElementPtrInst::init(Value *Ptr, Value *Idx, const Twine &Name) {
-  assert(NumOperands == 2 && "NumOperands not initialized?");
-  Use *OL = OperandList;
-  OL[0] = Ptr;
-  OL[1] = Idx;
-
+  assert(NumOperands == 1 + IdxList.size() && "NumOperands not initialized?");
+  OperandList[0] = Ptr;
+  std::copy(IdxList.begin(), IdxList.end(), op_begin() + 1);
   setName(Name);
 }
 
@@ -1029,34 +1344,10 @@ GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI)
                 OperandTraits<GetElementPtrInst>::op_end(this)
                 - GEPI.getNumOperands(),
                 GEPI.getNumOperands()) {
-  Use *OL = OperandList;
-  Use *GEPIOL = GEPI.OperandList;
-  for (unsigned i = 0, E = NumOperands; i != E; ++i)
-    OL[i] = GEPIOL[i];
+  std::copy(GEPI.op_begin(), GEPI.op_end(), op_begin());
   SubclassOptionalData = GEPI.SubclassOptionalData;
 }
 
-GetElementPtrInst::GetElementPtrInst(Value *Ptr, Value *Idx,
-                                     const Twine &Name, Instruction *InBe)
-  : Instruction(PointerType::get(
-      checkGEPType(getIndexedType(Ptr->getType(),Idx)), retrieveAddrSpace(Ptr)),
-                GetElementPtr,
-                OperandTraits<GetElementPtrInst>::op_end(this) - 2,
-                2, InBe) {
-  init(Ptr, Idx, Name);
-}
-
-GetElementPtrInst::GetElementPtrInst(Value *Ptr, Value *Idx,
-                                     const Twine &Name, BasicBlock *IAE)
-  : Instruction(PointerType::get(
-            checkGEPType(getIndexedType(Ptr->getType(),Idx)),  
-                retrieveAddrSpace(Ptr)),
-                GetElementPtr,
-                OperandTraits<GetElementPtrInst>::op_end(this) - 2,
-                2, IAE) {
-  init(Ptr, Idx, Name);
-}
-
 /// getIndexedType - Returns the type of the element that would be accessed with
 /// a gep instruction with the specified parameters.
 ///
@@ -1067,14 +1358,13 @@ GetElementPtrInst::GetElementPtrInst(Value *Ptr, Value *Idx,
 /// pointer type.
 ///
 template <typename IndexTy>
-static Type *getIndexedTypeInternal(const Type *Ptr, IndexTy const *Idxs,
-                                    unsigned NumIdx) {
-  const PointerType *PTy = dyn_cast<PointerType>(Ptr);
+static Type *getIndexedTypeInternal(Type *Ptr, ArrayRef<IndexTy> IdxList) {
+  PointerType *PTy = dyn_cast<PointerType>(Ptr);
   if (!PTy) return 0;   // Type isn't a pointer type!
   Type *Agg = PTy->getElementType();
 
   // Handle the special case of the empty set index set, which is always valid.
-  if (NumIdx == 0)
+  if (IdxList.empty())
     return Agg;
   
   // If there is at least one index, the top level type must be sized, otherwise
@@ -1083,44 +1373,29 @@ static Type *getIndexedTypeInternal(const Type *Ptr, IndexTy const *Idxs,
     return 0;
 
   unsigned CurIdx = 1;
-  for (; CurIdx != NumIdx; ++CurIdx) {
+  for (; CurIdx != IdxList.size(); ++CurIdx) {
     CompositeType *CT = dyn_cast<CompositeType>(Agg);
     if (!CT || CT->isPointerTy()) return 0;
-    IndexTy Index = Idxs[CurIdx];
+    IndexTy Index = IdxList[CurIdx];
     if (!CT->indexValid(Index)) return 0;
     Agg = CT->getTypeAtIndex(Index);
   }
-  return CurIdx == NumIdx ? Agg : 0;
+  return CurIdx == IdxList.size() ? Agg : 0;
 }
 
-Type *GetElementPtrInst::getIndexedType(const Type *Ptr, Value* const *Idxs,
-                                        unsigned NumIdx) {
-  return getIndexedTypeInternal(Ptr, Idxs, NumIdx);
+Type *GetElementPtrInst::getIndexedType(Type *Ptr, ArrayRef<Value *> IdxList) {
+  return getIndexedTypeInternal(Ptr, IdxList);
 }
 
-Type *GetElementPtrInst::getIndexedType(const Type *Ptr,
-                                        Constant* const *Idxs,
-                                        unsigned NumIdx) {
-  return getIndexedTypeInternal(Ptr, Idxs, NumIdx);
+Type *GetElementPtrInst::getIndexedType(Type *Ptr,
+                                        ArrayRef<Constant *> IdxList) {
+  return getIndexedTypeInternal(Ptr, IdxList);
 }
 
-Type *GetElementPtrInst::getIndexedType(const Type *Ptr,
-                                        uint64_t const *Idxs,
-                                        unsigned NumIdx) {
-  return getIndexedTypeInternal(Ptr, Idxs, NumIdx);
+Type *GetElementPtrInst::getIndexedType(Type *Ptr, ArrayRef<uint64_t> IdxList) {
+  return getIndexedTypeInternal(Ptr, IdxList);
 }
 
-Type *GetElementPtrInst::getIndexedType(const Type *Ptr, Value *Idx) {
-  const PointerType *PTy = dyn_cast<PointerType>(Ptr);
-  if (!PTy) return 0;   // Type isn't a pointer type!
-
-  // Check the pointer index.
-  if (!PTy->indexValid(Idx)) return 0;
-
-  return PTy->getElementType();
-}
-
-
 /// hasAllZeroIndices - Return true if all of the indices of this GEP are
 /// zeros.  If so, the result pointer and the first operand have the same
 /// value, just potentially different types.
@@ -1286,13 +1561,13 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
   if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
     return false;
   
-  const VectorType *MaskTy = dyn_cast<VectorType>(Mask->getType());
+  VectorType *MaskTy = dyn_cast<VectorType>(Mask->getType());
   if (MaskTy == 0 || !MaskTy->getElementType()->isIntegerTy(32))
     return false;
 
   // Check to see if Mask is valid.
   if (const ConstantVector *MV = dyn_cast<ConstantVector>(Mask)) {
-    const VectorType *VTy = cast<VectorType>(V1->getType());
+    VectorType *VTy = cast<VectorType>(V1->getType());
     for (unsigned i = 0, e = MV->getNumOperands(); i != e; ++i) {
       if (ConstantInt* CI = dyn_cast<ConstantInt>(MV->getOperand(i))) {
         if (CI->uge(VTy->getNumElements()*2))
@@ -1382,7 +1657,7 @@ ExtractValueInst::ExtractValueInst(const ExtractValueInst &EVI)
 // A null type is returned if the indices are invalid for the specified
 // pointer type.
 //
-Type *ExtractValueInst::getIndexedType(const Type *Agg,
+Type *ExtractValueInst::getIndexedType(Type *Agg,
                                        ArrayRef<unsigned> Idxs) {
   for (unsigned CurIdx = 0; CurIdx != Idxs.size(); ++CurIdx) {
     unsigned Index = Idxs[CurIdx];
@@ -1392,10 +1667,10 @@ Type *ExtractValueInst::getIndexedType(const Type *Agg,
     // insertvalue we need to check array indexing manually.
     // Since the only other types we can index into are struct types it's just
     // as easy to check those manually as well.
-    if (const ArrayType *AT = dyn_cast<ArrayType>(Agg)) {
+    if (ArrayType *AT = dyn_cast<ArrayType>(Agg)) {
       if (Index >= AT->getNumElements())
         return 0;
-    } else if (const StructType *ST = dyn_cast<StructType>(Agg)) {
+    } else if (StructType *ST = dyn_cast<StructType>(Agg)) {
       if (Index >= ST->getNumElements())
         return 0;
     } else {
@@ -1413,7 +1688,7 @@ Type *ExtractValueInst::getIndexedType(const Type *Agg,
 //===----------------------------------------------------------------------===//
 
 BinaryOperator::BinaryOperator(BinaryOps iType, Value *S1, Value *S2,
-                               const Type *Ty, const Twine &Name,
+                               Type *Ty, const Twine &Name,
                                Instruction *InsertBefore)
   : Instruction(Ty, iType,
                 OperandTraits<BinaryOperator>::op_begin(this),
@@ -1426,7 +1701,7 @@ BinaryOperator::BinaryOperator(BinaryOps iType, Value *S1, Value *S2,
 }
 
 BinaryOperator::BinaryOperator(BinaryOps iType, Value *S1, Value *S2, 
-                               const Type *Ty, const Twine &Name,
+                               Type *Ty, const Twine &Name,
                                BasicBlock *InsertAtEnd)
   : Instruction(Ty, iType,
                 OperandTraits<BinaryOperator>::op_begin(this),
@@ -1589,7 +1864,7 @@ BinaryOperator *BinaryOperator::CreateFNeg(Value *Op, const Twine &Name,
 BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,
                                           Instruction *InsertBefore) {
   Constant *C;
-  if (const VectorType *PTy = dyn_cast<VectorType>(Op->getType())) {
+  if (VectorType *PTy = dyn_cast<VectorType>(Op->getType())) {
     C = Constant::getAllOnesValue(PTy->getElementType());
     C = ConstantVector::get(
                               std::vector<Constant*>(PTy->getNumElements(), C));
@@ -1604,7 +1879,7 @@ BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,
 BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,
                                           BasicBlock *InsertAtEnd) {
   Constant *AllOnes;
-  if (const VectorType *PTy = dyn_cast<VectorType>(Op->getType())) {
+  if (VectorType *PTy = dyn_cast<VectorType>(Op->getType())) {
     // Create a vector of all ones values.
     Constant *Elt = Constant::getAllOnesValue(PTy->getElementType());
     AllOnes = ConstantVector::get(
@@ -1743,8 +2018,8 @@ bool CastInst::isLosslessCast() const {
     return false;
 
   // Identity cast is always lossless
-  const Type* SrcTy = getOperand(0)->getType();
-  const Type* DstTy = getType();
+  Type* SrcTy = getOperand(0)->getType();
+  Type* DstTy = getType();
   if (SrcTy == DstTy)
     return true;
   
@@ -1763,12 +2038,12 @@ bool CastInst::isLosslessCast() const {
 /// # ptrtoint i32* %x to i32     ; on 32-bit plaforms only
 /// @brief Determine if the described cast is a no-op.
 bool CastInst::isNoopCast(Instruction::CastOps Opcode,
-                          const Type *SrcTy,
-                          const Type *DestTy,
-                          const Type *IntPtrTy) {
+                          Type *SrcTy,
+                          Type *DestTy,
+                          Type *IntPtrTy) {
   switch (Opcode) {
     default:
-      assert(!"Invalid CastOp");
+      assert(0 && "Invalid CastOp");
     case Instruction::Trunc:
     case Instruction::ZExt:
     case Instruction::SExt: 
@@ -1791,7 +2066,7 @@ bool CastInst::isNoopCast(Instruction::CastOps Opcode,
 }
 
 /// @brief Determine if a cast is a no-op.
-bool CastInst::isNoopCast(const Type *IntPtrTy) const {
+bool CastInst::isNoopCast(Type *IntPtrTy) const {
   return isNoopCast(getOpcode(), getOperand(0)->getType(), getType(), IntPtrTy);
 }
 
@@ -1805,8 +2080,7 @@ bool CastInst::isNoopCast(const Type *IntPtrTy) const {
 /// If no such cast is permited, the function returns 0.
 unsigned CastInst::isEliminableCastPair(
   Instruction::CastOps firstOp, Instruction::CastOps secondOp,
-  const Type *SrcTy, const Type *MidTy, const Type *DstTy, const Type *IntPtrTy)
-{
+  Type *SrcTy, Type *MidTy, Type *DstTy, Type *IntPtrTy) {
   // Define the 144 possibilities for these two cast instructions. The values
   // in this matrix determine what to do in a given situation and select the
   // case in the switch below.  The rows correspond to firstOp, the columns 
@@ -1859,12 +2133,16 @@ unsigned CastInst::isEliminableCastPair(
   };
   
   // If either of the casts are a bitcast from scalar to vector, disallow the
-  // merging.
-  if ((firstOp == Instruction::BitCast &&
-       isa<VectorType>(SrcTy) != isa<VectorType>(MidTy)) ||
-      (secondOp == Instruction::BitCast &&
-       isa<VectorType>(MidTy) != isa<VectorType>(DstTy)))
-    return 0; // Disallowed
+  // merging. However, bitcast of A->B->A are allowed.
+  bool isFirstBitcast  = (firstOp == Instruction::BitCast);
+  bool isSecondBitcast = (secondOp == Instruction::BitCast);
+  bool chainedBitcast  = (SrcTy == DstTy && isFirstBitcast && isSecondBitcast);
+
+  // Check if any of the bitcasts convert scalars<->vectors.
+  if ((isFirstBitcast  && isa<VectorType>(SrcTy) != isa<VectorType>(MidTy)) ||
+      (isSecondBitcast && isa<VectorType>(MidTy) != isa<VectorType>(DstTy)))
+    // Unless we are bitcasing to the original type, disallow optimizations.
+    if (!chainedBitcast) return 0;
 
   int ElimCase = CastResults[firstOp-Instruction::CastOpsBegin]
                             [secondOp-Instruction::CastOpsBegin];
@@ -1958,16 +2236,16 @@ unsigned CastInst::isEliminableCastPair(
     case 99: 
       // cast combination can't happen (error in input). This is for all cases
       // where the MidTy is not the same for the two cast instructions.
-      assert(!"Invalid Cast Combination");
+      assert(0 && "Invalid Cast Combination");
       return 0;
     default:
-      assert(!"Error in CastResults table!!!");
+      assert(0 && "Error in CastResults table!!!");
       return 0;
   }
   return 0;
 }
 
-CastInst *CastInst::Create(Instruction::CastOps op, Value *S, const Type *Ty, 
+CastInst *CastInst::Create(Instruction::CastOps op, Value *S, Type *Ty, 
   const Twine &Name, Instruction *InsertBefore) {
   assert(castIsValid(op, S, Ty) && "Invalid cast!");
   // Construct and return the appropriate CastInst subclass
@@ -1985,12 +2263,12 @@ CastInst *CastInst::Create(Instruction::CastOps op, Value *S, const Type *Ty,
     case IntToPtr: return new IntToPtrInst (S, Ty, Name, InsertBefore);
     case BitCast:  return new BitCastInst  (S, Ty, Name, InsertBefore);
     default:
-      assert(!"Invalid opcode provided");
+      assert(0 && "Invalid opcode provided");
   }
   return 0;
 }
 
-CastInst *CastInst::Create(Instruction::CastOps op, Value *S, const Type *Ty,
+CastInst *CastInst::Create(Instruction::CastOps op, Value *S, Type *Ty,
   const Twine &Name, BasicBlock *InsertAtEnd) {
   assert(castIsValid(op, S, Ty) && "Invalid cast!");
   // Construct and return the appropriate CastInst subclass
@@ -2008,12 +2286,12 @@ CastInst *CastInst::Create(Instruction::CastOps op, Value *S, const Type *Ty,
     case IntToPtr: return new IntToPtrInst (S, Ty, Name, InsertAtEnd);
     case BitCast:  return new BitCastInst  (S, Ty, Name, InsertAtEnd);
     default:
-      assert(!"Invalid opcode provided");
+      assert(0 && "Invalid opcode provided");
   }
   return 0;
 }
 
-CastInst *CastInst::CreateZExtOrBitCast(Value *S, const Type *Ty, 
+CastInst *CastInst::CreateZExtOrBitCast(Value *S, Type *Ty, 
                                         const Twine &Name,
                                         Instruction *InsertBefore) {
   if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
@@ -2021,7 +2299,7 @@ CastInst *CastInst::CreateZExtOrBitCast(Value *S, const Type *Ty,
   return Create(Instruction::ZExt, S, Ty, Name, InsertBefore);
 }
 
-CastInst *CastInst::CreateZExtOrBitCast(Value *S, const Type *Ty, 
+CastInst *CastInst::CreateZExtOrBitCast(Value *S, Type *Ty, 
                                         const Twine &Name,
                                         BasicBlock *InsertAtEnd) {
   if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
@@ -2029,7 +2307,7 @@ CastInst *CastInst::CreateZExtOrBitCast(Value *S, const Type *Ty,
   return Create(Instruction::ZExt, S, Ty, Name, InsertAtEnd);
 }
 
-CastInst *CastInst::CreateSExtOrBitCast(Value *S, const Type *Ty, 
+CastInst *CastInst::CreateSExtOrBitCast(Value *S, Type *Ty, 
                                         const Twine &Name,
                                         Instruction *InsertBefore) {
   if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
@@ -2037,7 +2315,7 @@ CastInst *CastInst::CreateSExtOrBitCast(Value *S, const Type *Ty,
   return Create(Instruction::SExt, S, Ty, Name, InsertBefore);
 }
 
-CastInst *CastInst::CreateSExtOrBitCast(Value *S, const Type *Ty, 
+CastInst *CastInst::CreateSExtOrBitCast(Value *S, Type *Ty, 
                                         const Twine &Name,
                                         BasicBlock *InsertAtEnd) {
   if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
@@ -2045,7 +2323,7 @@ CastInst *CastInst::CreateSExtOrBitCast(Value *S, const Type *Ty,
   return Create(Instruction::SExt, S, Ty, Name, InsertAtEnd);
 }
 
-CastInst *CastInst::CreateTruncOrBitCast(Value *S, const Type *Ty,
+CastInst *CastInst::CreateTruncOrBitCast(Value *S, Type *Ty,
                                          const Twine &Name,
                                          Instruction *InsertBefore) {
   if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
@@ -2053,7 +2331,7 @@ CastInst *CastInst::CreateTruncOrBitCast(Value *S, const Type *Ty,
   return Create(Instruction::Trunc, S, Ty, Name, InsertBefore);
 }
 
-CastInst *CastInst::CreateTruncOrBitCast(Value *S, const Type *Ty,
+CastInst *CastInst::CreateTruncOrBitCast(Value *S, Type *Ty,
                                          const Twine &Name, 
                                          BasicBlock *InsertAtEnd) {
   if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
@@ -2061,7 +2339,7 @@ CastInst *CastInst::CreateTruncOrBitCast(Value *S, const Type *Ty,
   return Create(Instruction::Trunc, S, Ty, Name, InsertAtEnd);
 }
 
-CastInst *CastInst::CreatePointerCast(Value *S, const Type *Ty,
+CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty,
                                       const Twine &Name,
                                       BasicBlock *InsertAtEnd) {
   assert(S->getType()->isPointerTy() && "Invalid cast");
@@ -2074,7 +2352,7 @@ CastInst *CastInst::CreatePointerCast(Value *S, const Type *Ty,
 }
 
 /// @brief Create a BitCast or a PtrToInt cast instruction
-CastInst *CastInst::CreatePointerCast(Value *S, const Type *Ty, 
+CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty, 
                                       const Twine &Name, 
                                       Instruction *InsertBefore) {
   assert(S->getType()->isPointerTy() && "Invalid cast");
@@ -2086,7 +2364,7 @@ CastInst *CastInst::CreatePointerCast(Value *S, const Type *Ty,
   return Create(Instruction::BitCast, S, Ty, Name, InsertBefore);
 }
 
-CastInst *CastInst::CreateIntegerCast(Value *C, const Type *Ty, 
+CastInst *CastInst::CreateIntegerCast(Value *C, Type *Ty, 
                                       bool isSigned, const Twine &Name,
                                       Instruction *InsertBefore) {
   assert(C->getType()->isIntOrIntVectorTy() && Ty->isIntOrIntVectorTy() &&
@@ -2100,7 +2378,7 @@ CastInst *CastInst::CreateIntegerCast(Value *C, const Type *Ty,
   return Create(opcode, C, Ty, Name, InsertBefore);
 }
 
-CastInst *CastInst::CreateIntegerCast(Value *C, const Type *Ty, 
+CastInst *CastInst::CreateIntegerCast(Value *C, Type *Ty, 
                                       bool isSigned, const Twine &Name,
                                       BasicBlock *InsertAtEnd) {
   assert(C->getType()->isIntOrIntVectorTy() && Ty->isIntOrIntVectorTy() &&
@@ -2114,7 +2392,7 @@ CastInst *CastInst::CreateIntegerCast(Value *C, const Type *Ty,
   return Create(opcode, C, Ty, Name, InsertAtEnd);
 }
 
-CastInst *CastInst::CreateFPCast(Value *C, const Type *Ty, 
+CastInst *CastInst::CreateFPCast(Value *C, Type *Ty, 
                                  const Twine &Name, 
                                  Instruction *InsertBefore) {
   assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() &&
@@ -2127,7 +2405,7 @@ CastInst *CastInst::CreateFPCast(Value *C, const Type *Ty,
   return Create(opcode, C, Ty, Name, InsertBefore);
 }
 
-CastInst *CastInst::CreateFPCast(Value *C, const Type *Ty, 
+CastInst *CastInst::CreateFPCast(Value *C, Type *Ty, 
                                  const Twine &Name, 
                                  BasicBlock *InsertAtEnd) {
   assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() &&
@@ -2142,15 +2420,15 @@ CastInst *CastInst::CreateFPCast(Value *C, const Type *Ty,
 
 // Check whether it is valid to call getCastOpcode for these types.
 // This routine must be kept in sync with getCastOpcode.
-bool CastInst::isCastable(const Type *SrcTy, const Type *DestTy) {
+bool CastInst::isCastable(Type *SrcTy, Type *DestTy) {
   if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())
     return false;
 
   if (SrcTy == DestTy)
     return true;
 
-  if (const VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
-    if (const VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
+  if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
+    if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
       if (SrcVecTy->getNumElements() == DestVecTy->getNumElements()) {
         // An element by element cast.  Valid if casting the elements is valid.
         SrcTy = SrcVecTy->getElementType();
@@ -2212,8 +2490,8 @@ bool CastInst::isCastable(const Type *SrcTy, const Type *DestTy) {
 // This routine must be kept in sync with isCastable.
 Instruction::CastOps
 CastInst::getCastOpcode(
-  const Value *Src, bool SrcIsSigned, const Type *DestTy, bool DestIsSigned) {
-  const Type *SrcTy = Src->getType();
+  const Value *Src, bool SrcIsSigned, Type *DestTy, bool DestIsSigned) {
+  Type *SrcTy = Src->getType();
 
   assert(SrcTy->isFirstClassType() && DestTy->isFirstClassType() &&
          "Only first class types are castable!");
@@ -2221,8 +2499,8 @@ CastInst::getCastOpcode(
   if (SrcTy == DestTy)
     return BitCast;
 
-  if (const VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
-    if (const VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
+  if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
+    if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
       if (SrcVecTy->getNumElements() == DestVecTy->getNumElements()) {
         // An element by element cast.  Find the appropriate opcode based on the
         // element types.
@@ -2292,17 +2570,17 @@ CastInst::getCastOpcode(
     } else if (SrcTy->isIntegerTy()) {
       return IntToPtr;                              // int -> ptr
     } else {
-      assert(!"Casting pointer to other than pointer or int");
+      assert(0 && "Casting pointer to other than pointer or int");
     }
   } else if (DestTy->isX86_MMXTy()) {
     if (SrcTy->isVectorTy()) {
       assert(DestBits == SrcBits && "Casting vector of wrong width to X86_MMX");
       return BitCast;                               // 64-bit vector to MMX
     } else {
-      assert(!"Illegal cast to X86_MMX");
+      assert(0 && "Illegal cast to X86_MMX");
     }
   } else {
-    assert(!"Casting to type that is not first-class");
+    assert(0 && "Casting to type that is not first-class");
   }
 
   // If we fall through to here we probably hit an assertion cast above
@@ -2320,10 +2598,10 @@ CastInst::getCastOpcode(
 /// it in one place and to eliminate the redundant code for getting the sizes
 /// of the types involved.
 bool 
-CastInst::castIsValid(Instruction::CastOps op, Value *S, const Type *DstTy) {
+CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
 
   // Check for type sanity on the arguments
-  const Type *SrcTy = S->getType();
+  Type *SrcTy = S->getType();
   if (!SrcTy->isFirstClassType() || !DstTy->isFirstClassType() ||
       SrcTy->isAggregateType() || DstTy->isAggregateType())
     return false;
@@ -2384,144 +2662,144 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, const Type *DstTy) {
 }
 
 TruncInst::TruncInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, Trunc, S, Name, InsertBefore) {
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal Trunc");
 }
 
 TruncInst::TruncInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, Trunc, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal Trunc");
 }
 
 ZExtInst::ZExtInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 )  : CastInst(Ty, ZExt, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal ZExt");
 }
 
 ZExtInst::ZExtInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 )  : CastInst(Ty, ZExt, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal ZExt");
 }
 SExtInst::SExtInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, SExt, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal SExt");
 }
 
 SExtInst::SExtInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 )  : CastInst(Ty, SExt, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal SExt");
 }
 
 FPTruncInst::FPTruncInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, FPTrunc, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPTrunc");
 }
 
 FPTruncInst::FPTruncInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, FPTrunc, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPTrunc");
 }
 
 FPExtInst::FPExtInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, FPExt, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPExt");
 }
 
 FPExtInst::FPExtInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, FPExt, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPExt");
 }
 
 UIToFPInst::UIToFPInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, UIToFP, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal UIToFP");
 }
 
 UIToFPInst::UIToFPInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, UIToFP, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal UIToFP");
 }
 
 SIToFPInst::SIToFPInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, SIToFP, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal SIToFP");
 }
 
 SIToFPInst::SIToFPInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, SIToFP, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal SIToFP");
 }
 
 FPToUIInst::FPToUIInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, FPToUI, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToUI");
 }
 
 FPToUIInst::FPToUIInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, FPToUI, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToUI");
 }
 
 FPToSIInst::FPToSIInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, FPToSI, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToSI");
 }
 
 FPToSIInst::FPToSIInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, FPToSI, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToSI");
 }
 
 PtrToIntInst::PtrToIntInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, PtrToInt, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToInt");
 }
 
 PtrToIntInst::PtrToIntInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, PtrToInt, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToInt");
 }
 
 IntToPtrInst::IntToPtrInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, IntToPtr, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal IntToPtr");
 }
 
 IntToPtrInst::IntToPtrInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, IntToPtr, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal IntToPtr");
 }
 
 BitCastInst::BitCastInst(
-  Value *S, const Type *Ty, const Twine &Name, Instruction *InsertBefore
+  Value *S, Type *Ty, const Twine &Name, Instruction *InsertBefore
 ) : CastInst(Ty, BitCast, S, Name, InsertBefore) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal BitCast");
 }
 
 BitCastInst::BitCastInst(
-  Value *S, const Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
+  Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd
 ) : CastInst(Ty, BitCast, S, Name, InsertAtEnd) { 
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal BitCast");
 }
@@ -2532,7 +2810,7 @@ BitCastInst::BitCastInst(
 
 void CmpInst::Anchor() const {}
 
-CmpInst::CmpInst(const Type *ty, OtherOps op, unsigned short predicate,
+CmpInst::CmpInst(Type *ty, OtherOps op, unsigned short predicate,
                  Value *LHS, Value *RHS, const Twine &Name,
                  Instruction *InsertBefore)
   : Instruction(ty, op,
@@ -2545,7 +2823,7 @@ CmpInst::CmpInst(const Type *ty, OtherOps op, unsigned short predicate,
   setName(Name);
 }
 
-CmpInst::CmpInst(const Type *ty, OtherOps op, unsigned short predicate,
+CmpInst::CmpInst(Type *ty, OtherOps op, unsigned short predicate,
                  Value *LHS, Value *RHS, const Twine &Name,
                  BasicBlock *InsertAtEnd)
   : Instruction(ty, op,
@@ -2612,7 +2890,7 @@ bool CmpInst::isEquality() const {
 
 CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) {
   switch (pred) {
-    default: assert(!"Unknown cmp predicate!");
+    default: assert(0 && "Unknown cmp predicate!");
     case ICMP_EQ: return ICMP_NE;
     case ICMP_NE: return ICMP_EQ;
     case ICMP_UGT: return ICMP_ULE;
@@ -2645,7 +2923,7 @@ CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) {
 
 ICmpInst::Predicate ICmpInst::getSignedPredicate(Predicate pred) {
   switch (pred) {
-    default: assert(! "Unknown icmp predicate!");
+    default: assert(0 && "Unknown icmp predicate!");
     case ICMP_EQ: case ICMP_NE: 
     case ICMP_SGT: case ICMP_SLT: case ICMP_SGE: case ICMP_SLE: 
        return pred;
@@ -2658,7 +2936,7 @@ ICmpInst::Predicate ICmpInst::getSignedPredicate(Predicate pred) {
 
 ICmpInst::Predicate ICmpInst::getUnsignedPredicate(Predicate pred) {
   switch (pred) {
-    default: assert(! "Unknown icmp predicate!");
+    default: assert(0 && "Unknown icmp predicate!");
     case ICMP_EQ: case ICMP_NE: 
     case ICMP_UGT: case ICMP_ULT: case ICMP_UGE: case ICMP_ULE: 
        return pred;
@@ -2734,7 +3012,7 @@ ICmpInst::makeConstantRange(Predicate pred, const APInt &C) {
 
 CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {
   switch (pred) {
-    default: assert(!"Unknown cmp predicate!");
+    default: assert(0 && "Unknown cmp predicate!");
     case ICMP_EQ: case ICMP_NE:
       return pred;
     case ICMP_SGT: return ICMP_SLT;
@@ -3065,14 +3343,34 @@ AllocaInst *AllocaInst::clone_impl() const {
 }
 
 LoadInst *LoadInst::clone_impl() const {
-  return new LoadInst(getOperand(0),
-                      Twine(), isVolatile(),
-                      getAlignment());
+  return new LoadInst(getOperand(0), Twine(), isVolatile(),
+                      getAlignment(), getOrdering(), getSynchScope());
 }
 
 StoreInst *StoreInst::clone_impl() const {
-  return new StoreInst(getOperand(0), getOperand(1),
-                       isVolatile(), getAlignment());
+  return new StoreInst(getOperand(0), getOperand(1), isVolatile(),
+                       getAlignment(), getOrdering(), getSynchScope());
+  
+}
+
+AtomicCmpXchgInst *AtomicCmpXchgInst::clone_impl() const {
+  AtomicCmpXchgInst *Result =
+    new AtomicCmpXchgInst(getOperand(0), getOperand(1), getOperand(2),
+                          getOrdering(), getSynchScope());
+  Result->setVolatile(isVolatile());
+  return Result;
+}
+
+AtomicRMWInst *AtomicRMWInst::clone_impl() const {
+  AtomicRMWInst *Result =
+    new AtomicRMWInst(getOperation(),getOperand(0), getOperand(1),
+                      getOrdering(), getSynchScope());
+  Result->setVolatile(isVolatile());
+  return Result;
+}
+
+FenceInst *FenceInst::clone_impl() const {
+  return new FenceInst(getContext(), getOrdering(), getSynchScope());
 }
 
 TruncInst *TruncInst::clone_impl() const {
@@ -3155,6 +3453,10 @@ PHINode *PHINode::clone_impl() const {
   return new PHINode(*this);
 }
 
+LandingPadInst *LandingPadInst::clone_impl() const {
+  return new LandingPadInst(*this);
+}
+
 ReturnInst *ReturnInst::clone_impl() const {
   return new(getNumOperands()) ReturnInst(*this);
 }
@@ -3176,6 +3478,10 @@ InvokeInst *InvokeInst::clone_impl() const {
   return new(getNumOperands()) InvokeInst(*this);
 }
 
+ResumeInst *ResumeInst::clone_impl() const {
+  return new(1) ResumeInst(*this);
+}
+
 UnwindInst *UnwindInst::clone_impl() const {
   LLVMContext &Context = getContext();
   return new UnwindInst(Context);
diff --git a/contrib/llvm/lib/VMCore/LLVMContextImpl.h b/contrib/llvm/lib/VMCore/LLVMContextImpl.h
index 06a6f2a25a38..a3f68fecbbf6 100644
--- a/contrib/llvm/lib/VMCore/LLVMContextImpl.h
+++ b/contrib/llvm/lib/VMCore/LLVMContextImpl.h
@@ -42,8 +42,8 @@ class Value;
 struct DenseMapAPIntKeyInfo {
   struct KeyTy {
     APInt val;
-    const Type* type;
-    KeyTy(const APInt& V, const Type* Ty) : val(V), type(Ty) {}
+    Type* type;
+    KeyTy(const APInt& V, Type* Ty) : val(V), type(Ty) {}
     KeyTy(const KeyTy& that) : val(that.val), type(that.type) {}
     bool operator==(const KeyTy& that) const {
       return type == that.type && this->val == that.val;
diff --git a/contrib/llvm/lib/VMCore/Module.cpp b/contrib/llvm/lib/VMCore/Module.cpp
index be2fcb8ac6c0..c29029bf6c06 100644
--- a/contrib/llvm/lib/VMCore/Module.cpp
+++ b/contrib/llvm/lib/VMCore/Module.cpp
@@ -32,25 +32,10 @@ using namespace llvm;
 // Methods to implement the globals and functions lists.
 //
 
-GlobalVariable *ilist_traits<GlobalVariable>::createSentinel() {
-  GlobalVariable *Ret = new GlobalVariable(Type::getInt32Ty(getGlobalContext()),
-                                           false, GlobalValue::ExternalLinkage);
-  // This should not be garbage monitored.
-  LeakDetector::removeGarbageObject(Ret);
-  return Ret;
-}
-GlobalAlias *ilist_traits<GlobalAlias>::createSentinel() {
-  GlobalAlias *Ret = new GlobalAlias(Type::getInt32Ty(getGlobalContext()),
-                                     GlobalValue::ExternalLinkage);
-  // This should not be garbage monitored.
-  LeakDetector::removeGarbageObject(Ret);
-  return Ret;
-}
-
 // Explicit instantiations of SymbolTableListTraits since some of the methods
 // are not in the public header file.
-template class llvm::SymbolTableListTraits<GlobalVariable, Module>;
 template class llvm::SymbolTableListTraits<Function, Module>;
+template class llvm::SymbolTableListTraits<GlobalVariable, Module>;
 template class llvm::SymbolTableListTraits<GlobalAlias, Module>;
 
 //===----------------------------------------------------------------------===//
@@ -82,8 +67,10 @@ Module::Endianness Module::getEndianness() const {
   Module::Endianness ret = AnyEndianness;
   
   while (!temp.empty()) {
-    StringRef token = DataLayout;
-    tie(token, temp) = getToken(temp, "-");
+    std::pair<StringRef, StringRef> P = getToken(temp, "-");
+    
+    StringRef token = P.first;
+    temp = P.second;
     
     if (token[0] == 'e') {
       ret = LittleEndian;
@@ -95,15 +82,16 @@ Module::Endianness Module::getEndianness() const {
   return ret;
 }
 
-/// Target Pointer Size information...
+/// Target Pointer Size information.
 Module::PointerSize Module::getPointerSize() const {
   StringRef temp = DataLayout;
   Module::PointerSize ret = AnyPointerSize;
   
   while (!temp.empty()) {
-    StringRef token, signalToken;
-    tie(token, temp) = getToken(temp, "-");
-    tie(signalToken, token) = getToken(token, ":");
+    std::pair<StringRef, StringRef> TmpP = getToken(temp, "-");
+    temp = TmpP.second;
+    TmpP = getToken(TmpP.first, ":");
+    StringRef token = TmpP.second, signalToken = TmpP.first;
     
     if (signalToken[0] == 'p') {
       int size = 0;
@@ -149,7 +137,7 @@ void Module::getMDKindNames(SmallVectorImpl<StringRef> &Result) const {
 // the symbol table directly for this common task.
 //
 Constant *Module::getOrInsertFunction(StringRef Name,
-                                      const FunctionType *Ty,
+                                      FunctionType *Ty,
                                       AttrListPtr AttributeList) {
   // See if we have a definition for the specified function already.
   GlobalValue *F = getNamedValue(Name);
@@ -182,7 +170,7 @@ Constant *Module::getOrInsertFunction(StringRef Name,
 }
 
 Constant *Module::getOrInsertTargetIntrinsic(StringRef Name,
-                                             const FunctionType *Ty,
+                                             FunctionType *Ty,
                                              AttrListPtr AttributeList) {
   // See if we have a definition for the specified function already.
   GlobalValue *F = getNamedValue(Name);
@@ -199,7 +187,7 @@ Constant *Module::getOrInsertTargetIntrinsic(StringRef Name,
 }
 
 Constant *Module::getOrInsertFunction(StringRef Name,
-                                      const FunctionType *Ty) {
+                                      FunctionType *Ty) {
   AttrListPtr AttributeList = AttrListPtr::get((AttributeWithIndex *)0, 0);
   return getOrInsertFunction(Name, Ty, AttributeList);
 }
@@ -211,7 +199,7 @@ Constant *Module::getOrInsertFunction(StringRef Name,
 //
 Constant *Module::getOrInsertFunction(StringRef Name,
                                       AttrListPtr AttributeList,
-                                      const Type *RetTy, ...) {
+                                      Type *RetTy, ...) {
   va_list Args;
   va_start(Args, RetTy);
 
@@ -229,7 +217,7 @@ Constant *Module::getOrInsertFunction(StringRef Name,
 }
 
 Constant *Module::getOrInsertFunction(StringRef Name,
-                                      const Type *RetTy, ...) {
+                                      Type *RetTy, ...) {
   va_list Args;
   va_start(Args, RetTy);
 
@@ -279,7 +267,7 @@ GlobalVariable *Module::getGlobalVariable(StringRef Name,
 ///      with a constantexpr cast to the right type.
 ///   3. Finally, if the existing global is the correct delclaration, return the
 ///      existing global.
-Constant *Module::getOrInsertGlobal(StringRef Name, const Type *Ty) {
+Constant *Module::getOrInsertGlobal(StringRef Name, Type *Ty) {
   // See if we have a definition for the specified global already.
   GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(getNamedValue(Name));
   if (GV == 0) {
@@ -436,7 +424,7 @@ namespace {
     // To avoid walking constant expressions multiple times and other IR
     // objects, we keep several helper maps.
     DenseSet<const Value*> VisitedConstants;
-    DenseSet<const Type*> VisitedTypes;
+    DenseSet<Type*> VisitedTypes;
     
     std::vector<StructType*> &StructTypes;
   public:
@@ -549,5 +537,3 @@ namespace {
 void Module::findUsedStructTypes(std::vector<StructType*> &StructTypes) const {
   TypeFinder(StructTypes).run(*this);
 }
-
-
diff --git a/contrib/llvm/lib/VMCore/PassManager.cpp b/contrib/llvm/lib/VMCore/PassManager.cpp
index 5cf2905733ac..ecedb1db66b0 100644
--- a/contrib/llvm/lib/VMCore/PassManager.cpp
+++ b/contrib/llvm/lib/VMCore/PassManager.cpp
@@ -28,7 +28,6 @@
 #include "llvm/Support/Mutex.h"
 #include "llvm/ADT/StringMap.h"
 #include <algorithm>
-#include <cstdio>
 #include <map>
 using namespace llvm;
 
@@ -167,8 +166,8 @@ class BBPassManager : public PMDataManager, public FunctionPass {
 
 public:
   static char ID;
-  explicit BBPassManager(int Depth)
-    : PMDataManager(Depth), FunctionPass(ID) {}
+  explicit BBPassManager()
+    : PMDataManager(), FunctionPass(ID) {}
 
   /// Execute all of the passes scheduled for execution.  Keep track of
   /// whether any of the passes modifies the function, and if so, return true.
@@ -193,7 +192,7 @@ class BBPassManager : public PMDataManager, public FunctionPass {
 
   // Print passes managed by this manager
   void dumpPassStructure(unsigned Offset) {
-    llvm::dbgs() << std::string(Offset*2, ' ') << "BasicBlockPass Manager\n";
+    llvm::dbgs().indent(Offset*2) << "BasicBlockPass Manager\n";
     for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
       BasicBlockPass *BP = getContainedPass(Index);
       BP->dumpPassStructure(Offset + 1);
@@ -228,9 +227,9 @@ class FunctionPassManagerImpl : public Pass,
   bool wasRun;
 public:
   static char ID;
-  explicit FunctionPassManagerImpl(int Depth) :
-    Pass(PT_PassManager, ID), PMDataManager(Depth),
-    PMTopLevelManager(new FPPassManager(1)), wasRun(false) {}
+  explicit FunctionPassManagerImpl() :
+    Pass(PT_PassManager, ID), PMDataManager(),
+    PMTopLevelManager(new FPPassManager()), wasRun(false) {}
 
   /// add - Add a pass to the queue of passes to run.  This passes ownership of
   /// the Pass to the PassManager.  When the PassManager is destroyed, the pass
@@ -303,8 +302,8 @@ char FunctionPassManagerImpl::ID = 0;
 class MPPassManager : public Pass, public PMDataManager {
 public:
   static char ID;
-  explicit MPPassManager(int Depth) :
-    Pass(PT_PassManager, ID), PMDataManager(Depth) { }
+  explicit MPPassManager() :
+    Pass(PT_PassManager, ID), PMDataManager() { }
 
   // Delete on the fly managers.
   virtual ~MPPassManager() {
@@ -349,7 +348,7 @@ class MPPassManager : public Pass, public PMDataManager {
 
   // Print passes managed by this manager
   void dumpPassStructure(unsigned Offset) {
-    llvm::dbgs() << std::string(Offset*2, ' ') << "ModulePass Manager\n";
+    llvm::dbgs().indent(Offset*2) << "ModulePass Manager\n";
     for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
       ModulePass *MP = getContainedPass(Index);
       MP->dumpPassStructure(Offset + 1);
@@ -388,9 +387,9 @@ class PassManagerImpl : public Pass,
 
 public:
   static char ID;
-  explicit PassManagerImpl(int Depth) :
-    Pass(PT_PassManager, ID), PMDataManager(Depth),
-                              PMTopLevelManager(new MPPassManager(1)) {}
+  explicit PassManagerImpl() :
+    Pass(PT_PassManager, ID), PMDataManager(),
+                              PMTopLevelManager(new MPPassManager()) {}
 
   /// add - Add a pass to the queue of passes to run.  This passes ownership of
   /// the Pass to the PassManager.  When the PassManager is destroyed, the pass
@@ -1340,7 +1339,7 @@ bool BBPassManager::doFinalization(Function &F) {
 
 /// Create new Function pass manager
 FunctionPassManager::FunctionPassManager(Module *m) : M(m) {
-  FPM = new FunctionPassManagerImpl(0);
+  FPM = new FunctionPassManagerImpl();
   // FPM is the top level manager.
   FPM->setTopLevelManager(FPM);
 
@@ -1532,7 +1531,7 @@ bool FPPassManager::runOnModule(Module &M) {
   bool Changed = doInitialization(M);
 
   for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
-    runOnFunction(*I);
+    Changed |= runOnFunction(*I);
 
   return doFinalization(M) || Changed;
 }
@@ -1626,7 +1625,7 @@ void MPPassManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) {
 
   FunctionPassManagerImpl *FPP = OnTheFlyManagers[P];
   if (!FPP) {
-    FPP = new FunctionPassManagerImpl(0);
+    FPP = new FunctionPassManagerImpl();
     // FPP is the top level manager.
     FPP->setTopLevelManager(FPP);
 
@@ -1635,9 +1634,11 @@ void MPPassManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) {
   FPP->add(RequiredPass);
 
   // Register P as the last user of RequiredPass.
-  SmallVector<Pass *, 1> LU;
-  LU.push_back(RequiredPass);
-  FPP->setLastUser(LU,  P);
+  if (RequiredPass) {
+    SmallVector<Pass *, 1> LU;
+    LU.push_back(RequiredPass);
+    FPP->setLastUser(LU,  P);
+  }
 }
 
 /// Return function pass corresponding to PassInfo PI, that is
@@ -1677,7 +1678,7 @@ bool PassManagerImpl::run(Module &M) {
 
 /// Create new pass manager
 PassManager::PassManager() {
-  PM = new PassManagerImpl(0);
+  PM = new PassManagerImpl();
   // PM is the top level manager
   PM->setTopLevelManager(PM);
 }
@@ -1761,13 +1762,23 @@ void PMStack::pop() {
 // Push PM on the stack and set its top level manager.
 void PMStack::push(PMDataManager *PM) {
   assert(PM && "Unable to push. Pass Manager expected");
+  assert(PM->getDepth()==0 && "Pass Manager depth set too early");
 
   if (!this->empty()) {
+    assert(PM->getPassManagerType() > this->top()->getPassManagerType()
+           && "pushing bad pass manager to PMStack");
     PMTopLevelManager *TPM = this->top()->getTopLevelManager();
 
     assert(TPM && "Unable to find top level manager");
     TPM->addIndirectPassManager(PM);
     PM->setTopLevelManager(TPM);
+    PM->setDepth(this->top()->getDepth()+1);
+  }
+  else {
+    assert((PM->getPassManagerType() == PMT_ModulePassManager
+           || PM->getPassManagerType() == PMT_FunctionPassManager)
+           && "pushing bad pass manager to PMStack");
+    PM->setDepth(1);
   }
 
   S.push_back(PM);
@@ -1777,10 +1788,10 @@ void PMStack::push(PMDataManager *PM) {
 void PMStack::dump() const {
   for (std::vector<PMDataManager *>::const_iterator I = S.begin(),
          E = S.end(); I != E; ++I)
-    printf("%s ", (*I)->getAsPass()->getPassName());
+    dbgs() << (*I)->getAsPass()->getPassName() << ' ';
 
   if (!S.empty())
-    printf("\n");
+    dbgs() << '\n';
 }
 
 /// Find appropriate Module Pass Manager in the PM Stack and
@@ -1823,7 +1834,7 @@ void FunctionPass::assignPassManager(PMStack &PMS,
     PMDataManager *PMD = PMS.top();
 
     // [1] Create new Function Pass Manager
-    FPP = new FPPassManager(PMD->getDepth() + 1);
+    FPP = new FPPassManager();
     FPP->populateInheritedAnalysis(PMS);
 
     // [2] Set up new manager's top level manager
@@ -1860,7 +1871,7 @@ void BasicBlockPass::assignPassManager(PMStack &PMS,
     PMDataManager *PMD = PMS.top();
 
     // [1] Create new Basic Block Manager
-    BBP = new BBPassManager(PMD->getDepth() + 1);
+    BBP = new BBPassManager();
 
     // [2] Set up new manager's top level manager
     // Basic Block Pass Manager does not live by itself
diff --git a/contrib/llvm/lib/VMCore/PassRegistry.cpp b/contrib/llvm/lib/VMCore/PassRegistry.cpp
index fa92620b288e..2df65572c592 100644
--- a/contrib/llvm/lib/VMCore/PassRegistry.cpp
+++ b/contrib/llvm/lib/VMCore/PassRegistry.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Function.h"
 #include <vector>
 
 using namespace llvm;
diff --git a/contrib/llvm/lib/VMCore/Type.cpp b/contrib/llvm/lib/VMCore/Type.cpp
index f874d1b28302..10184bc6f0e3 100644
--- a/contrib/llvm/lib/VMCore/Type.cpp
+++ b/contrib/llvm/lib/VMCore/Type.cpp
@@ -40,8 +40,8 @@ Type *Type::getPrimitiveType(LLVMContext &C, TypeID IDNumber) {
 
 /// getScalarType - If this is a vector type, return the element type,
 /// otherwise return this.
-const Type *Type::getScalarType() const {
-  if (const VectorType *VTy = dyn_cast<VectorType>(this))
+Type *Type::getScalarType() {
+  if (VectorType *VTy = dyn_cast<VectorType>(this))
     return VTy->getElementType();
   return this;
 }
@@ -77,7 +77,7 @@ bool Type::isFPOrFPVectorTy() const {
 // canLosslesslyBitCastTo - Return true if this type can be converted to
 // 'Ty' without any reinterpretation of bits.  For example, i8* to i32*.
 //
-bool Type::canLosslesslyBitCastTo(const Type *Ty) const {
+bool Type::canLosslesslyBitCastTo(Type *Ty) const {
   // Identity cast means no change so return true
   if (this == Ty) 
     return true;
@@ -146,7 +146,7 @@ unsigned Type::getPrimitiveSizeInBits() const {
 /// getScalarSizeInBits - If this is a vector type, return the
 /// getPrimitiveSizeInBits value for the element type. Otherwise return the
 /// getPrimitiveSizeInBits value for this type.
-unsigned Type::getScalarSizeInBits() const {
+unsigned Type::getScalarSizeInBits() {
   return getScalarType()->getPrimitiveSizeInBits();
 }
 
@@ -306,7 +306,7 @@ APInt IntegerType::getMask() const {
 //                       FunctionType Implementation
 //===----------------------------------------------------------------------===//
 
-FunctionType::FunctionType(const Type *Result, ArrayRef<Type*> Params,
+FunctionType::FunctionType(Type *Result, ArrayRef<Type*> Params,
                            bool IsVarArgs)
   : Type(Result->getContext(), FunctionTyID) {
   Type **SubTys = reinterpret_cast<Type**>(this+1);
@@ -326,7 +326,7 @@ FunctionType::FunctionType(const Type *Result, ArrayRef<Type*> Params,
 }
 
 // FunctionType::get - The factory function for the FunctionType class.
-FunctionType *FunctionType::get(const Type *ReturnType,
+FunctionType *FunctionType::get(Type *ReturnType,
                                 ArrayRef<Type*> Params, bool isVarArg) {
   // TODO: This is brutally slow.
   std::vector<Type*> Key;
@@ -351,21 +351,21 @@ FunctionType *FunctionType::get(const Type *ReturnType,
 }
 
 
-FunctionType *FunctionType::get(const Type *Result, bool isVarArg) {
+FunctionType *FunctionType::get(Type *Result, bool isVarArg) {
   return get(Result, ArrayRef<Type *>(), isVarArg);
 }
 
 
 /// isValidReturnType - Return true if the specified type is valid as a return
 /// type.
-bool FunctionType::isValidReturnType(const Type *RetTy) {
+bool FunctionType::isValidReturnType(Type *RetTy) {
   return !RetTy->isFunctionTy() && !RetTy->isLabelTy() &&
   !RetTy->isMetadataTy();
 }
 
 /// isValidArgumentType - Return true if the specified type is valid as an
 /// argument type.
-bool FunctionType::isValidArgumentType(const Type *ArgTy) {
+bool FunctionType::isValidArgumentType(Type *ArgTy) {
   return ArgTy->isFirstClassType();
 }
 
@@ -392,7 +392,7 @@ StructType *StructType::get(LLVMContext &Context, ArrayRef<Type*> ETypes,
   
   // Value not found.  Create a new type!
   ST = new (Context.pImpl->TypeAllocator) StructType(Context);
-  ST->setSubclassData(SCDB_IsAnonymous);  // Anonymous struct.
+  ST->setSubclassData(SCDB_IsLiteral);  // Literal struct.
   ST->setBody(ETypes, isPacked);
   return ST;
 }
@@ -412,13 +412,6 @@ void StructType::setBody(ArrayRef<Type*> Elements, bool isPacked) {
   NumContainedTys = Elements.size();
 }
 
-StructType *StructType::createNamed(LLVMContext &Context, StringRef Name) {
-  StructType *ST = new (Context.pImpl->TypeAllocator) StructType(Context);
-  if (!Name.empty())
-    ST->setName(Name);
-  return ST;
-}
-
 void StructType::setName(StringRef Name) {
   if (Name == getName()) return;
 
@@ -461,6 +454,13 @@ void StructType::setName(StringRef Name) {
 //===----------------------------------------------------------------------===//
 // StructType Helper functions.
 
+StructType *StructType::create(LLVMContext &Context, StringRef Name) {
+  StructType *ST = new (Context.pImpl->TypeAllocator) StructType(Context);
+  if (!Name.empty())
+    ST->setName(Name);
+  return ST;
+}
+
 StructType *StructType::get(LLVMContext &Context, bool isPacked) {
   return get(Context, llvm::ArrayRef<Type*>(), isPacked);
 }
@@ -478,21 +478,36 @@ StructType *StructType::get(Type *type, ...) {
   return llvm::StructType::get(Ctx, StructFields);
 }
 
-StructType *StructType::createNamed(LLVMContext &Context, StringRef Name,
-                                    ArrayRef<Type*> Elements, bool isPacked) {
-  StructType *ST = createNamed(Context, Name);
+StructType *StructType::create(LLVMContext &Context, ArrayRef<Type*> Elements,
+                               StringRef Name, bool isPacked) {
+  StructType *ST = create(Context, Name);
   ST->setBody(Elements, isPacked);
   return ST;
 }
 
-StructType *StructType::createNamed(StringRef Name, ArrayRef<Type*> Elements,
-                                    bool isPacked) {
-  assert(!Elements.empty() &&
-         "This method may not be invoked with an empty list");
-  return createNamed(Elements[0]->getContext(), Name, Elements, isPacked);
+StructType *StructType::create(LLVMContext &Context, ArrayRef<Type*> Elements) {
+  return create(Context, Elements, StringRef());
 }
 
-StructType *StructType::createNamed(StringRef Name, Type *type, ...) {
+StructType *StructType::create(LLVMContext &Context) {
+  return create(Context, StringRef());
+}
+
+
+StructType *StructType::create(ArrayRef<Type*> Elements, StringRef Name,
+                               bool isPacked) {
+  assert(!Elements.empty() &&
+         "This method may not be invoked with an empty list");
+  return create(Elements[0]->getContext(), Elements, Name, isPacked);
+}
+
+StructType *StructType::create(ArrayRef<Type*> Elements) {
+  assert(!Elements.empty() &&
+         "This method may not be invoked with an empty list");
+  return create(Elements[0]->getContext(), Elements, StringRef());
+}
+
+StructType *StructType::create(StringRef Name, Type *type, ...) {
   assert(type != 0 && "Cannot create a struct type with no elements with this");
   LLVMContext &Ctx = type->getContext();
   va_list ap;
@@ -502,11 +517,12 @@ StructType *StructType::createNamed(StringRef Name, Type *type, ...) {
     StructFields.push_back(type);
     type = va_arg(ap, llvm::Type*);
   }
-  return llvm::StructType::createNamed(Ctx, Name, StructFields);
+  return llvm::StructType::create(Ctx, StructFields, Name);
 }
 
+
 StringRef StructType::getName() const {
-  assert(!isAnonymous() && "Anonymous structs never have names");
+  assert(!isLiteral() && "Literal structs never have names");
   if (SymbolTableEntry == 0) return StringRef();
   
   return ((StringMapEntry<StructType*> *)SymbolTableEntry)->getKey();
@@ -524,14 +540,14 @@ void StructType::setBody(Type *type, ...) {
   setBody(StructFields);
 }
 
-bool StructType::isValidElementType(const Type *ElemTy) {
+bool StructType::isValidElementType(Type *ElemTy) {
   return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() &&
          !ElemTy->isMetadataTy() && !ElemTy->isFunctionTy();
 }
 
 /// isLayoutIdentical - Return true if this is layout identical to the
 /// specified struct.
-bool StructType::isLayoutIdentical(const StructType *Other) const {
+bool StructType::isLayoutIdentical(StructType *Other) const {
   if (this == Other) return true;
   
   if (isPacked() != Other->isPacked() ||
@@ -557,8 +573,8 @@ StructType *Module::getTypeByName(StringRef Name) const {
 //                       CompositeType Implementation
 //===----------------------------------------------------------------------===//
 
-Type *CompositeType::getTypeAtIndex(const Value *V) const {
-  if (const StructType *STy = dyn_cast<StructType>(this)) {
+Type *CompositeType::getTypeAtIndex(const Value *V) {
+  if (StructType *STy = dyn_cast<StructType>(this)) {
     unsigned Idx = (unsigned)cast<ConstantInt>(V)->getZExtValue();
     assert(indexValid(Idx) && "Invalid structure index!");
     return STy->getElementType(Idx);
@@ -566,8 +582,8 @@ Type *CompositeType::getTypeAtIndex(const Value *V) const {
   
   return cast<SequentialType>(this)->getElementType();
 }
-Type *CompositeType::getTypeAtIndex(unsigned Idx) const {
-  if (const StructType *STy = dyn_cast<StructType>(this)) {
+Type *CompositeType::getTypeAtIndex(unsigned Idx) {
+  if (StructType *STy = dyn_cast<StructType>(this)) {
     assert(indexValid(Idx) && "Invalid structure index!");
     return STy->getElementType(Idx);
   }
@@ -605,7 +621,7 @@ ArrayType::ArrayType(Type *ElType, uint64_t NumEl)
 }
 
 
-ArrayType *ArrayType::get(const Type *elementType, uint64_t NumElements) {
+ArrayType *ArrayType::get(Type *elementType, uint64_t NumElements) {
   Type *ElementType = const_cast<Type*>(elementType);
   assert(isValidElementType(ElementType) && "Invalid type for array element!");
     
@@ -618,7 +634,7 @@ ArrayType *ArrayType::get(const Type *elementType, uint64_t NumElements) {
   return Entry;
 }
 
-bool ArrayType::isValidElementType(const Type *ElemTy) {
+bool ArrayType::isValidElementType(Type *ElemTy) {
   return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() &&
          !ElemTy->isMetadataTy() && !ElemTy->isFunctionTy();
 }
@@ -632,7 +648,7 @@ VectorType::VectorType(Type *ElType, unsigned NumEl)
   NumElements = NumEl;
 }
 
-VectorType *VectorType::get(const Type *elementType, unsigned NumElements) {
+VectorType *VectorType::get(Type *elementType, unsigned NumElements) {
   Type *ElementType = const_cast<Type*>(elementType);
   assert(NumElements > 0 && "#Elements of a VectorType must be greater than 0");
   assert(isValidElementType(ElementType) &&
@@ -647,7 +663,7 @@ VectorType *VectorType::get(const Type *elementType, unsigned NumElements) {
   return Entry;
 }
 
-bool VectorType::isValidElementType(const Type *ElemTy) {
+bool VectorType::isValidElementType(Type *ElemTy) {
   return ElemTy->isIntegerTy() || ElemTy->isFloatingPointTy();
 }
 
@@ -655,8 +671,7 @@ bool VectorType::isValidElementType(const Type *ElemTy) {
 //                         PointerType Implementation
 //===----------------------------------------------------------------------===//
 
-PointerType *PointerType::get(const Type *eltTy, unsigned AddressSpace) {
-  Type *EltTy = const_cast<Type*>(eltTy);
+PointerType *PointerType::get(Type *EltTy, unsigned AddressSpace) {
   assert(EltTy && "Can't get a pointer to <null> type!");
   assert(isValidElementType(EltTy) && "Invalid type for pointer element!");
   
@@ -677,11 +692,11 @@ PointerType::PointerType(Type *E, unsigned AddrSpace)
   setSubclassData(AddrSpace);
 }
 
-PointerType *Type::getPointerTo(unsigned addrs) const {
+PointerType *Type::getPointerTo(unsigned addrs) {
   return PointerType::get(this, addrs);
 }
 
-bool PointerType::isValidElementType(const Type *ElemTy) {
+bool PointerType::isValidElementType(Type *ElemTy) {
   return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() &&
          !ElemTy->isMetadataTy();
 }
diff --git a/contrib/llvm/lib/VMCore/Value.cpp b/contrib/llvm/lib/VMCore/Value.cpp
index f1815e377edc..2fa5f08a3e7f 100644
--- a/contrib/llvm/lib/VMCore/Value.cpp
+++ b/contrib/llvm/lib/VMCore/Value.cpp
@@ -35,12 +35,12 @@ using namespace llvm;
 //                                Value Class
 //===----------------------------------------------------------------------===//
 
-static inline Type *checkType(const Type *Ty) {
+static inline Type *checkType(Type *Ty) {
   assert(Ty && "Value defined with a null type: Error!");
   return const_cast<Type*>(Ty);
 }
 
-Value::Value(const Type *ty, unsigned scid)
+Value::Value(Type *ty, unsigned scid)
   : SubclassID(scid), HasValueHandle(0),
     SubclassOptionalData(0), SubclassData(0), VTy((Type*)checkType(ty)),
     UseList(0), Name(0) {
@@ -369,7 +369,7 @@ bool Value::isDereferenceablePointer() const {
     for (User::const_op_iterator I = GEP->op_begin()+1,
          E = GEP->op_end(); I != E; ++I) {
       Value *Index = *I;
-      const Type *Ty = *GTI++;
+      Type *Ty = *GTI++;
       // Struct indices can't be out of bounds.
       if (isa<StructType>(Ty))
         continue;
@@ -380,7 +380,7 @@ bool Value::isDereferenceablePointer() const {
       if (CI->isZero())
         continue;
       // Check to see that it's within the bounds of an array.
-      const ArrayType *ATy = dyn_cast<ArrayType>(Ty);
+      ArrayType *ATy = dyn_cast<ArrayType>(Ty);
       if (!ATy)
         return false;
       if (CI->getValue().getActiveBits() > 64)
diff --git a/contrib/llvm/lib/VMCore/ValueTypes.cpp b/contrib/llvm/lib/VMCore/ValueTypes.cpp
index 21a1f034446a..e13bd7df7375 100644
--- a/contrib/llvm/lib/VMCore/ValueTypes.cpp
+++ b/contrib/llvm/lib/VMCore/ValueTypes.cpp
@@ -19,6 +19,12 @@
 #include "llvm/Support/ErrorHandling.h"
 using namespace llvm;
 
+EVT EVT::changeExtendedVectorElementTypeToInteger() const {
+  LLVMContext &Context = LLVMTy->getContext();
+  EVT IntTy = getIntegerVT(Context, getVectorElementType().getSizeInBits());
+  return getVectorVT(Context, IntTy, getVectorNumElements());
+}
+
 EVT EVT::getExtendedIntegerVT(LLVMContext &Context, unsigned BitWidth) {
   EVT VT;
   VT.LLVMTy = IntegerType::get(Context, BitWidth);
@@ -77,9 +83,9 @@ unsigned EVT::getExtendedVectorNumElements() const {
 
 unsigned EVT::getExtendedSizeInBits() const {
   assert(isExtended() && "Type is not extended!");
-  if (const IntegerType *ITy = dyn_cast<IntegerType>(LLVMTy))
+  if (IntegerType *ITy = dyn_cast<IntegerType>(LLVMTy))
     return ITy->getBitWidth();
-  if (const VectorType *VTy = dyn_cast<VectorType>(LLVMTy))
+  if (VectorType *VTy = dyn_cast<VectorType>(LLVMTy))
     return VTy->getBitWidth();
   assert(false && "Unrecognized extended type!");
   return 0; // Suppress warnings.
@@ -140,7 +146,7 @@ std::string EVT::getEVTString() const {
 /// getTypeForEVT - This method returns an LLVM type corresponding to the
 /// specified EVT.  For integer types, this returns an unsigned type.  Note
 /// that this will abort for types that cannot be represented.
-const Type *EVT::getTypeForEVT(LLVMContext &Context) const {
+Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   switch (V.SimpleTy) {
   default:
     assert(isExtended() && "Type is not extended!");
@@ -186,7 +192,7 @@ const Type *EVT::getTypeForEVT(LLVMContext &Context) const {
 /// getEVT - Return the value type corresponding to the specified type.  This
 /// returns all pointers as MVT::iPTR.  If HandleUnknown is true, unknown types
 /// are returned as Other, otherwise they are invalid.
-EVT EVT::getEVT(const Type *Ty, bool HandleUnknown){
+EVT EVT::getEVT(Type *Ty, bool HandleUnknown){
   switch (Ty->getTypeID()) {
   default:
     if (HandleUnknown) return MVT(MVT::Other);
@@ -204,7 +210,7 @@ EVT EVT::getEVT(const Type *Ty, bool HandleUnknown){
   case Type::PPC_FP128TyID: return MVT(MVT::ppcf128);
   case Type::PointerTyID:   return MVT(MVT::iPTR);
   case Type::VectorTyID: {
-    const VectorType *VTy = cast<VectorType>(Ty);
+    VectorType *VTy = cast<VectorType>(Ty);
     return getVectorVT(Ty->getContext(), getEVT(VTy->getElementType(), false),
                        VTy->getNumElements());
   }
diff --git a/contrib/llvm/lib/VMCore/Verifier.cpp b/contrib/llvm/lib/VMCore/Verifier.cpp
index b146b896cbfb..9564b7d71f6a 100644
--- a/contrib/llvm/lib/VMCore/Verifier.cpp
+++ b/contrib/llvm/lib/VMCore/Verifier.cpp
@@ -35,6 +35,12 @@
 //  * It is illegal to have a ret instruction that returns a value that does not
 //    agree with the function return value type.
 //  * Function call argument types match the function prototype
+//  * A landing pad is defined by a landingpad instruction, and can be jumped to
+//    only by the unwind edge of an invoke instruction.
+//  * A landingpad instruction must be the first non-PHI instruction in the
+//    block.
+//  * All landingpad instructions must use the same personality function with
+//    the same function.
 //  * All other things that are tested by asserts spread about the code...
 //
 //===----------------------------------------------------------------------===//
@@ -131,18 +137,22 @@ namespace {
     /// already.
     SmallPtrSet<MDNode *, 32> MDNodes;
 
+    /// PersonalityFn - The personality function referenced by the
+    /// LandingPadInsts. All LandingPadInsts within the same function must use
+    /// the same personality function.
+    const Value *PersonalityFn;
+
     Verifier()
-      : FunctionPass(ID), 
-      Broken(false), RealPass(true), action(AbortProcessAction),
-      Mod(0), Context(0), DT(0), MessagesStr(Messages) {
-        initializeVerifierPass(*PassRegistry::getPassRegistry());
-      }
+      : FunctionPass(ID), Broken(false), RealPass(true),
+        action(AbortProcessAction), Mod(0), Context(0), DT(0),
+        MessagesStr(Messages), PersonalityFn(0) {
+      initializeVerifierPass(*PassRegistry::getPassRegistry());
+    }
     explicit Verifier(VerifierFailureAction ctn)
-      : FunctionPass(ID), 
-      Broken(false), RealPass(true), action(ctn), Mod(0), Context(0), DT(0),
-      MessagesStr(Messages) {
-        initializeVerifierPass(*PassRegistry::getPassRegistry());
-      }
+      : FunctionPass(ID), Broken(false), RealPass(true), action(ctn), Mod(0),
+        Context(0), DT(0), MessagesStr(Messages), PersonalityFn(0) {
+      initializeVerifierPass(*PassRegistry::getPassRegistry());
+    }
 
     bool doInitialization(Module &M) {
       Mod = &M;
@@ -165,6 +175,7 @@ namespace {
 
       visit(F);
       InstsInThisBlock.clear();
+      PersonalityFn = 0;
 
       // If this is a real pass, in a pass manager, we must abort before
       // returning back to the pass manager, or else the pass manager may try to
@@ -278,18 +289,22 @@ namespace {
     void visitUserOp1(Instruction &I);
     void visitUserOp2(Instruction &I) { visitUserOp1(I); }
     void visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI);
+    void visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI);
+    void visitAtomicRMWInst(AtomicRMWInst &RMWI);
+    void visitFenceInst(FenceInst &FI);
     void visitAllocaInst(AllocaInst &AI);
     void visitExtractValueInst(ExtractValueInst &EVI);
     void visitInsertValueInst(InsertValueInst &IVI);
+    void visitLandingPadInst(LandingPadInst &LPI);
 
     void VerifyCallSite(CallSite CS);
-    bool PerformTypeCheck(Intrinsic::ID ID, Function *F, const Type *Ty,
+    bool PerformTypeCheck(Intrinsic::ID ID, Function *F, Type *Ty,
                           int VT, unsigned ArgNo, std::string &Suffix);
     void VerifyIntrinsicPrototype(Intrinsic::ID ID, Function *F,
                                   unsigned RetNum, unsigned ParamNum, ...);
-    void VerifyParameterAttrs(Attributes Attrs, const Type *Ty,
+    void VerifyParameterAttrs(Attributes Attrs, Type *Ty,
                               bool isReturnValue, const Value *V);
-    void VerifyFunctionAttrs(const FunctionType *FT, const AttrListPtr &Attrs,
+    void VerifyFunctionAttrs(FunctionType *FT, const AttrListPtr &Attrs,
                              const Value *V);
 
     void WriteValue(const Value *V) {
@@ -302,7 +317,7 @@ namespace {
       }
     }
 
-    void WriteType(const Type *T) {
+    void WriteType(Type *T) {
       if (!T) return;
       MessagesStr << ' ' << *T;
     }
@@ -323,7 +338,7 @@ namespace {
     }
 
     void CheckFailed(const Twine &Message, const Value *V1,
-                     const Type *T2, const Value *V3 = 0) {
+                     Type *T2, const Value *V3 = 0) {
       MessagesStr << Message.str() << "\n";
       WriteValue(V1);
       WriteType(T2);
@@ -331,8 +346,8 @@ namespace {
       Broken = true;
     }
 
-    void CheckFailed(const Twine &Message, const Type *T1,
-                     const Type *T2 = 0, const Type *T3 = 0) {
+    void CheckFailed(const Twine &Message, Type *T1,
+                     Type *T2 = 0, Type *T3 = 0) {
       MessagesStr << Message.str() << "\n";
       WriteType(T1);
       WriteType(T2);
@@ -421,9 +436,9 @@ void Verifier::visitGlobalVariable(GlobalVariable &GV) {
             "invalid linkage for intrinsic global variable", &GV);
     // Don't worry about emitting an error for it not being an array,
     // visitGlobalValue will complain on appending non-array.
-    if (const ArrayType *ATy = dyn_cast<ArrayType>(GV.getType())) {
-      const StructType *STy = dyn_cast<StructType>(ATy->getElementType());
-      const PointerType *FuncPtrTy =
+    if (ArrayType *ATy = dyn_cast<ArrayType>(GV.getType())) {
+      StructType *STy = dyn_cast<StructType>(ATy->getElementType());
+      PointerType *FuncPtrTy =
           FunctionType::get(Type::getVoidTy(*Context), false)->getPointerTo();
       Assert1(STy && STy->getNumElements() == 2 &&
               STy->getTypeAtIndex(0u)->isIntegerTy(32) &&
@@ -514,7 +529,7 @@ void Verifier::visitMDNode(MDNode &MD, Function *F) {
 
 // VerifyParameterAttrs - Check the given attributes for an argument or return
 // value of the specified type.  The value V is printed in error messages.
-void Verifier::VerifyParameterAttrs(Attributes Attrs, const Type *Ty,
+void Verifier::VerifyParameterAttrs(Attributes Attrs, Type *Ty,
                                     bool isReturnValue, const Value *V) {
   if (Attrs == Attribute::None)
     return;
@@ -541,7 +556,7 @@ void Verifier::VerifyParameterAttrs(Attributes Attrs, const Type *Ty,
           Attribute::getAsString(TypeI), V);
 
   Attributes ByValI = Attrs & Attribute::ByVal;
-  if (const PointerType *PTy = dyn_cast<PointerType>(Ty)) {
+  if (PointerType *PTy = dyn_cast<PointerType>(Ty)) {
     Assert1(!ByValI || PTy->getElementType()->isSized(),
             "Attribute " + Attribute::getAsString(ByValI) +
             " does not support unsized types!", V);
@@ -554,7 +569,7 @@ void Verifier::VerifyParameterAttrs(Attributes Attrs, const Type *Ty,
 
 // VerifyFunctionAttrs - Check parameter attributes against a function type.
 // The value V is printed in error messages.
-void Verifier::VerifyFunctionAttrs(const FunctionType *FT,
+void Verifier::VerifyFunctionAttrs(FunctionType *FT,
                                    const AttrListPtr &Attrs,
                                    const Value *V) {
   if (Attrs.isEmpty())
@@ -565,7 +580,7 @@ void Verifier::VerifyFunctionAttrs(const FunctionType *FT,
   for (unsigned i = 0, e = Attrs.getNumSlots(); i != e; ++i) {
     const AttributeWithIndex &Attr = Attrs.getSlot(i);
 
-    const Type *Ty;
+    Type *Ty;
     if (Attr.Index == 0)
       Ty = FT->getReturnType();
     else if (Attr.Index-1 < FT->getNumParams())
@@ -615,7 +630,7 @@ static bool VerifyAttributeCount(const AttrListPtr &Attrs, unsigned Params) {
 //
 void Verifier::visitFunction(Function &F) {
   // Check function arguments.
-  const FunctionType *FT = F.getFunctionType();
+  FunctionType *FT = F.getFunctionType();
   unsigned NumArgs = F.arg_size();
 
   Assert1(Context == &F.getContext(),
@@ -795,7 +810,7 @@ void Verifier::visitReturnInst(ReturnInst &RI) {
 void Verifier::visitSwitchInst(SwitchInst &SI) {
   // Check to make sure that all of the constants in the switch instruction
   // have the same type as the switched-on value.
-  const Type *SwitchTy = SI.getCondition()->getType();
+  Type *SwitchTy = SI.getCondition()->getType();
   SmallPtrSet<ConstantInt*, 32> Constants;
   for (unsigned i = 1, e = SI.getNumCases(); i != e; ++i) {
     Assert1(SI.getCaseValue(i)->getType() == SwitchTy,
@@ -836,8 +851,8 @@ void Verifier::visitUserOp1(Instruction &I) {
 
 void Verifier::visitTruncInst(TruncInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   // Get the size of the types in bits, we'll need this later
   unsigned SrcBitSize = SrcTy->getScalarSizeInBits();
@@ -854,8 +869,8 @@ void Verifier::visitTruncInst(TruncInst &I) {
 
 void Verifier::visitZExtInst(ZExtInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   // Get the size of the types in bits, we'll need this later
   Assert1(SrcTy->isIntOrIntVectorTy(), "ZExt only operates on integer", &I);
@@ -872,8 +887,8 @@ void Verifier::visitZExtInst(ZExtInst &I) {
 
 void Verifier::visitSExtInst(SExtInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   // Get the size of the types in bits, we'll need this later
   unsigned SrcBitSize = SrcTy->getScalarSizeInBits();
@@ -890,8 +905,8 @@ void Verifier::visitSExtInst(SExtInst &I) {
 
 void Verifier::visitFPTruncInst(FPTruncInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
   // Get the size of the types in bits, we'll need this later
   unsigned SrcBitSize = SrcTy->getScalarSizeInBits();
   unsigned DestBitSize = DestTy->getScalarSizeInBits();
@@ -907,8 +922,8 @@ void Verifier::visitFPTruncInst(FPTruncInst &I) {
 
 void Verifier::visitFPExtInst(FPExtInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   // Get the size of the types in bits, we'll need this later
   unsigned SrcBitSize = SrcTy->getScalarSizeInBits();
@@ -925,8 +940,8 @@ void Verifier::visitFPExtInst(FPExtInst &I) {
 
 void Verifier::visitUIToFPInst(UIToFPInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   bool SrcVec = SrcTy->isVectorTy();
   bool DstVec = DestTy->isVectorTy();
@@ -948,8 +963,8 @@ void Verifier::visitUIToFPInst(UIToFPInst &I) {
 
 void Verifier::visitSIToFPInst(SIToFPInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   bool SrcVec = SrcTy->isVectorTy();
   bool DstVec = DestTy->isVectorTy();
@@ -971,8 +986,8 @@ void Verifier::visitSIToFPInst(SIToFPInst &I) {
 
 void Verifier::visitFPToUIInst(FPToUIInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   bool SrcVec = SrcTy->isVectorTy();
   bool DstVec = DestTy->isVectorTy();
@@ -994,8 +1009,8 @@ void Verifier::visitFPToUIInst(FPToUIInst &I) {
 
 void Verifier::visitFPToSIInst(FPToSIInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   bool SrcVec = SrcTy->isVectorTy();
   bool DstVec = DestTy->isVectorTy();
@@ -1017,8 +1032,8 @@ void Verifier::visitFPToSIInst(FPToSIInst &I) {
 
 void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   Assert1(SrcTy->isPointerTy(), "PtrToInt source must be pointer", &I);
   Assert1(DestTy->isIntegerTy(), "PtrToInt result must be integral", &I);
@@ -1028,8 +1043,8 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
 
 void Verifier::visitIntToPtrInst(IntToPtrInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   Assert1(SrcTy->isIntegerTy(), "IntToPtr source must be an integral", &I);
   Assert1(DestTy->isPointerTy(), "IntToPtr result must be a pointer",&I);
@@ -1039,8 +1054,8 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) {
 
 void Verifier::visitBitCastInst(BitCastInst &I) {
   // Get the source and destination types
-  const Type *SrcTy = I.getOperand(0)->getType();
-  const Type *DestTy = I.getType();
+  Type *SrcTy = I.getOperand(0)->getType();
+  Type *DestTy = I.getType();
 
   // Get the size of the types in bits, we'll need this later
   unsigned SrcBitSize = SrcTy->getPrimitiveSizeInBits();
@@ -1090,11 +1105,11 @@ void Verifier::VerifyCallSite(CallSite CS) {
 
   Assert1(CS.getCalledValue()->getType()->isPointerTy(),
           "Called function must be a pointer!", I);
-  const PointerType *FPTy = cast<PointerType>(CS.getCalledValue()->getType());
+  PointerType *FPTy = cast<PointerType>(CS.getCalledValue()->getType());
 
   Assert1(FPTy->getElementType()->isFunctionTy(),
           "Called function is not pointer to function type!", I);
-  const FunctionType *FTy = cast<FunctionType>(FPTy->getElementType());
+  FunctionType *FTy = cast<FunctionType>(FPTy->getElementType());
 
   // Verify that the correct number of arguments are being passed
   if (FTy->isVarArg())
@@ -1152,6 +1167,12 @@ void Verifier::visitCallInst(CallInst &CI) {
 
 void Verifier::visitInvokeInst(InvokeInst &II) {
   VerifyCallSite(&II);
+
+  // Verify that there is a landingpad instruction as the first non-PHI
+  // instruction of the 'unwind' destination.
+  Assert1(II.getUnwindDest()->isLandingPad(),
+          "The unwind destination does not have a landingpad instruction!",&II);
+
   visitTerminatorInst(II);
 }
 
@@ -1219,8 +1240,8 @@ void Verifier::visitBinaryOperator(BinaryOperator &B) {
 
 void Verifier::visitICmpInst(ICmpInst &IC) {
   // Check that the operands are the same type
-  const Type *Op0Ty = IC.getOperand(0)->getType();
-  const Type *Op1Ty = IC.getOperand(1)->getType();
+  Type *Op0Ty = IC.getOperand(0)->getType();
+  Type *Op1Ty = IC.getOperand(1)->getType();
   Assert1(Op0Ty == Op1Ty,
           "Both operands to ICmp instruction are not of the same type!", &IC);
   // Check that the operands are the right type
@@ -1236,8 +1257,8 @@ void Verifier::visitICmpInst(ICmpInst &IC) {
 
 void Verifier::visitFCmpInst(FCmpInst &FC) {
   // Check that the operands are the same type
-  const Type *Op0Ty = FC.getOperand(0)->getType();
-  const Type *Op1Ty = FC.getOperand(1)->getType();
+  Type *Op0Ty = FC.getOperand(0)->getType();
+  Type *Op1Ty = FC.getOperand(1)->getType();
   Assert1(Op0Ty == Op1Ty,
           "Both operands to FCmp instruction are not of the same type!", &FC);
   // Check that the operands are the right type
@@ -1274,10 +1295,13 @@ void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
 }
 
 void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
+  Assert1(cast<PointerType>(GEP.getOperand(0)->getType())
+            ->getElementType()->isSized(),
+          "GEP into unsized type!", &GEP);
+  
   SmallVector<Value*, 16> Idxs(GEP.idx_begin(), GEP.idx_end());
-  const Type *ElTy =
-    GetElementPtrInst::getIndexedType(GEP.getOperand(0)->getType(),
-                                      Idxs.begin(), Idxs.end());
+  Type *ElTy =
+    GetElementPtrInst::getIndexedType(GEP.getOperand(0)->getType(), Idxs);
   Assert1(ElTy, "Invalid indices for GEP pointer type!", &GEP);
   Assert2(GEP.getType()->isPointerTy() &&
           cast<PointerType>(GEP.getType())->getElementType() == ElTy,
@@ -1286,26 +1310,44 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 }
 
 void Verifier::visitLoadInst(LoadInst &LI) {
-  const PointerType *PTy = dyn_cast<PointerType>(LI.getOperand(0)->getType());
+  PointerType *PTy = dyn_cast<PointerType>(LI.getOperand(0)->getType());
   Assert1(PTy, "Load operand must be a pointer.", &LI);
-  const Type *ElTy = PTy->getElementType();
+  Type *ElTy = PTy->getElementType();
   Assert2(ElTy == LI.getType(),
           "Load result type does not match pointer operand type!", &LI, ElTy);
+  if (LI.isAtomic()) {
+    Assert1(LI.getOrdering() != Release && LI.getOrdering() != AcquireRelease,
+            "Load cannot have Release ordering", &LI);
+    Assert1(LI.getAlignment() != 0,
+            "Atomic load must specify explicit alignment", &LI);
+  } else {
+    Assert1(LI.getSynchScope() == CrossThread,
+            "Non-atomic load cannot have SynchronizationScope specified", &LI);
+  }
   visitInstruction(LI);
 }
 
 void Verifier::visitStoreInst(StoreInst &SI) {
-  const PointerType *PTy = dyn_cast<PointerType>(SI.getOperand(1)->getType());
+  PointerType *PTy = dyn_cast<PointerType>(SI.getOperand(1)->getType());
   Assert1(PTy, "Store operand must be a pointer.", &SI);
-  const Type *ElTy = PTy->getElementType();
+  Type *ElTy = PTy->getElementType();
   Assert2(ElTy == SI.getOperand(0)->getType(),
           "Stored value type does not match pointer operand type!",
           &SI, ElTy);
+  if (SI.isAtomic()) {
+    Assert1(SI.getOrdering() != Acquire && SI.getOrdering() != AcquireRelease,
+            "Store cannot have Acquire ordering", &SI);
+    Assert1(SI.getAlignment() != 0,
+            "Atomic store must specify explicit alignment", &SI);
+  } else {
+    Assert1(SI.getSynchScope() == CrossThread,
+            "Non-atomic store cannot have SynchronizationScope specified", &SI);
+  }
   visitInstruction(SI);
 }
 
 void Verifier::visitAllocaInst(AllocaInst &AI) {
-  const PointerType *PTy = AI.getType();
+  PointerType *PTy = AI.getType();
   Assert1(PTy->getAddressSpace() == 0, 
           "Allocation instruction pointer not in the generic address space!",
           &AI);
@@ -1316,6 +1358,49 @@ void Verifier::visitAllocaInst(AllocaInst &AI) {
   visitInstruction(AI);
 }
 
+void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) {
+  Assert1(CXI.getOrdering() != NotAtomic,
+          "cmpxchg instructions must be atomic.", &CXI);
+  Assert1(CXI.getOrdering() != Unordered,
+          "cmpxchg instructions cannot be unordered.", &CXI);
+  PointerType *PTy = dyn_cast<PointerType>(CXI.getOperand(0)->getType());
+  Assert1(PTy, "First cmpxchg operand must be a pointer.", &CXI);
+  Type *ElTy = PTy->getElementType();
+  Assert2(ElTy == CXI.getOperand(1)->getType(),
+          "Expected value type does not match pointer operand type!",
+          &CXI, ElTy);
+  Assert2(ElTy == CXI.getOperand(2)->getType(),
+          "Stored value type does not match pointer operand type!",
+          &CXI, ElTy);
+  visitInstruction(CXI);
+}
+
+void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
+  Assert1(RMWI.getOrdering() != NotAtomic,
+          "atomicrmw instructions must be atomic.", &RMWI);
+  Assert1(RMWI.getOrdering() != Unordered,
+          "atomicrmw instructions cannot be unordered.", &RMWI);
+  PointerType *PTy = dyn_cast<PointerType>(RMWI.getOperand(0)->getType());
+  Assert1(PTy, "First atomicrmw operand must be a pointer.", &RMWI);
+  Type *ElTy = PTy->getElementType();
+  Assert2(ElTy == RMWI.getOperand(1)->getType(),
+          "Argument value type does not match pointer operand type!",
+          &RMWI, ElTy);
+  Assert1(AtomicRMWInst::FIRST_BINOP <= RMWI.getOperation() &&
+          RMWI.getOperation() <= AtomicRMWInst::LAST_BINOP,
+          "Invalid binary operation!", &RMWI);
+  visitInstruction(RMWI);
+}
+
+void Verifier::visitFenceInst(FenceInst &FI) {
+  const AtomicOrdering Ordering = FI.getOrdering();
+  Assert1(Ordering == Acquire || Ordering == Release ||
+          Ordering == AcquireRelease || Ordering == SequentiallyConsistent,
+          "fence instructions may only have "
+          "acquire, release, acq_rel, or seq_cst ordering.", &FI);
+  visitInstruction(FI);
+}
+
 void Verifier::visitExtractValueInst(ExtractValueInst &EVI) {
   Assert1(ExtractValueInst::getIndexedType(EVI.getAggregateOperand()->getType(),
                                            EVI.getIndices()) ==
@@ -1334,6 +1419,55 @@ void Verifier::visitInsertValueInst(InsertValueInst &IVI) {
   visitInstruction(IVI);
 }
 
+void Verifier::visitLandingPadInst(LandingPadInst &LPI) {
+  BasicBlock *BB = LPI.getParent();
+
+  // The landingpad instruction is ill-formed if it doesn't have any clauses and
+  // isn't a cleanup.
+  Assert1(LPI.getNumClauses() > 0 || LPI.isCleanup(),
+          "LandingPadInst needs at least one clause or to be a cleanup.", &LPI);
+
+  // The landingpad instruction defines its parent as a landing pad block. The
+  // landing pad block may be branched to only by the unwind edge of an invoke.
+  for (pred_iterator I = pred_begin(BB), E = pred_end(BB); I != E; ++I) {
+    const InvokeInst *II = dyn_cast<InvokeInst>((*I)->getTerminator());
+    Assert1(II && II->getUnwindDest() == BB,
+            "Block containing LandingPadInst must be jumped to "
+            "only by the unwind edge of an invoke.", &LPI);
+  }
+
+  // The landingpad instruction must be the first non-PHI instruction in the
+  // block.
+  Assert1(LPI.getParent()->getLandingPadInst() == &LPI,
+          "LandingPadInst not the first non-PHI instruction in the block.",
+          &LPI);
+
+  // The personality functions for all landingpad instructions within the same
+  // function should match.
+  if (PersonalityFn)
+    Assert1(LPI.getPersonalityFn() == PersonalityFn,
+            "Personality function doesn't match others in function", &LPI);
+  PersonalityFn = LPI.getPersonalityFn();
+
+  // All operands must be constants.
+  Assert1(isa<Constant>(PersonalityFn), "Personality function is not constant!",
+          &LPI);
+  for (unsigned i = 0, e = LPI.getNumClauses(); i < e; ++i) {
+    Value *Clause = LPI.getClause(i);
+    Assert1(isa<Constant>(Clause), "Clause is not constant!", &LPI);
+    if (LPI.isCatch(i)) {
+      Assert1(isa<PointerType>(Clause->getType()),
+              "Catch operand does not have pointer type!", &LPI);
+    } else {
+      Assert1(LPI.isFilter(i), "Clause is neither catch nor filter!", &LPI);
+      Assert1(isa<ConstantArray>(Clause) || isa<ConstantAggregateZero>(Clause),
+              "Filter operand is not an array of constants!", &LPI);
+    }
+  }
+
+  visitInstruction(LPI);
+}
+
 /// verifyInstruction - Verify that an instruction is well formed.
 ///
 void Verifier::visitInstruction(Instruction &I) {
@@ -1588,20 +1722,20 @@ static std::string IntrinsicParam(unsigned ArgNo, unsigned NumRets) {
   return "Intrinsic result type #" + utostr(ArgNo);
 }
 
-bool Verifier::PerformTypeCheck(Intrinsic::ID ID, Function *F, const Type *Ty,
+bool Verifier::PerformTypeCheck(Intrinsic::ID ID, Function *F, Type *Ty,
                                 int VT, unsigned ArgNo, std::string &Suffix) {
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
 
   unsigned NumElts = 0;
-  const Type *EltTy = Ty;
-  const VectorType *VTy = dyn_cast<VectorType>(Ty);
+  Type *EltTy = Ty;
+  VectorType *VTy = dyn_cast<VectorType>(Ty);
   if (VTy) {
     EltTy = VTy->getElementType();
     NumElts = VTy->getNumElements();
   }
 
-  const Type *RetTy = FTy->getReturnType();
-  const StructType *ST = dyn_cast<StructType>(RetTy);
+  Type *RetTy = FTy->getReturnType();
+  StructType *ST = dyn_cast<StructType>(RetTy);
   unsigned NumRetVals;
   if (RetTy->isVoidTy())
     NumRetVals = 0;
@@ -1618,7 +1752,7 @@ bool Verifier::PerformTypeCheck(Intrinsic::ID ID, Function *F, const Type *Ty,
     // type.
     if ((Match & (ExtendedElementVectorType |
                   TruncatedElementVectorType)) != 0) {
-      const IntegerType *IEltTy = dyn_cast<IntegerType>(EltTy);
+      IntegerType *IEltTy = dyn_cast<IntegerType>(EltTy);
       if (!VTy || !IEltTy) {
         CheckFailed(IntrinsicParam(ArgNo, NumRetVals) + " is not "
                     "an integral vector type.", F);
@@ -1709,7 +1843,7 @@ bool Verifier::PerformTypeCheck(Intrinsic::ID ID, Function *F, const Type *Ty,
     // Outside of TableGen, we don't distinguish iPTRAny (to any address space)
     // and iPTR. In the verifier, we can not distinguish which case we have so
     // allow either case to be legal.
-    if (const PointerType* PTyp = dyn_cast<PointerType>(Ty)) {
+    if (PointerType* PTyp = dyn_cast<PointerType>(Ty)) {
       EVT PointeeVT = EVT::getEVT(PTyp->getElementType(), true);
       if (PointeeVT == MVT::Other) {
         CheckFailed("Intrinsic has pointer to complex type.");
@@ -1757,7 +1891,7 @@ void Verifier::VerifyIntrinsicPrototype(Intrinsic::ID ID, Function *F,
                                         unsigned NumParams, ...) {
   va_list VA;
   va_start(VA, NumParams);
-  const FunctionType *FTy = F->getFunctionType();
+  FunctionType *FTy = F->getFunctionType();
 
   // For overloaded intrinsics, the Suffix of the function name must match the
   // types of the arguments. This variable keeps track of the expected
@@ -1769,8 +1903,8 @@ void Verifier::VerifyIntrinsicPrototype(Intrinsic::ID ID, Function *F,
     return;
   }
 
-  const Type *Ty = FTy->getReturnType();
-  const StructType *ST = dyn_cast<StructType>(Ty);
+  Type *Ty = FTy->getReturnType();
+  StructType *ST = dyn_cast<StructType>(Ty);
 
   if (NumRetVals == 0 && !Ty->isVoidTy()) {
     CheckFailed("Intrinsic should return void", F);
diff --git a/contrib/llvm/tools/clang/LICENSE.TXT b/contrib/llvm/tools/clang/LICENSE.TXT
new file mode 100644
index 000000000000..91895eba029d
--- /dev/null
+++ b/contrib/llvm/tools/clang/LICENSE.TXT
@@ -0,0 +1,63 @@
+==============================================================================
+LLVM Release License
+==============================================================================
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2007-2011 University of Illinois at Urbana-Champaign.
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+
+==============================================================================
+The LLVM software contains code written by third parties.  Such software will
+have its own individual LICENSE.TXT file in the directory in which it appears.
+This file will describe the copyrights, license, and restrictions which apply
+to that code.
+
+The disclaimer of warranty in the University of Illinois Open Source License
+applies to all code in the LLVM Distribution, and nothing in any of the
+other licenses gives permission to use the names of the LLVM Team or the
+University of Illinois to endorse or promote products derived from this
+Software.
+
+The following pieces of software have additional or alternate copyrights,
+licenses, and/or restrictions:
+
+Program             Directory
+-------             ---------
+<none yet>
+
diff --git a/contrib/llvm/tools/clang/include/clang-c/Index.h b/contrib/llvm/tools/clang/include/clang-c/Index.h
index 4852ded7f883..1320145a896f 100644
--- a/contrib/llvm/tools/clang/include/clang-c/Index.h
+++ b/contrib/llvm/tools/clang/include/clang-c/Index.h
@@ -115,7 +115,12 @@ enum CXAvailabilityKind {
   /**
    * \brief The entity is not available; any use of it will be an error.
    */
-  CXAvailability_NotAvailable
+  CXAvailability_NotAvailable,
+  /**
+   * \brief The entity is available, but not accessible; any use of it will be
+   * an error.
+   */
+  CXAvailability_NotAccessible
 };
   
 /**
@@ -263,7 +268,7 @@ CINDEX_LINKAGE CXFile clang_getFile(CXTranslationUnit tu,
  * \brief Identifies a specific source location within a translation
  * unit.
  *
- * Use clang_getInstantiationLocation() or clang_getSpellingLocation()
+ * Use clang_getExpansionLocation() or clang_getSpellingLocation()
  * to map a source location to a particular file, line, and column.
  */
 typedef struct {
@@ -327,12 +332,25 @@ CINDEX_LINKAGE CXSourceRange clang_getNullRange();
 CINDEX_LINKAGE CXSourceRange clang_getRange(CXSourceLocation begin,
                                             CXSourceLocation end);
 
+/**
+ * \brief Determine whether two ranges are equivalent.
+ *
+ * \returns non-zero if the ranges are the same, zero if they differ.
+ */
+CINDEX_LINKAGE unsigned clang_equalRanges(CXSourceRange range1,
+                                          CXSourceRange range2);
+
+/**
+ * \brief Returns non-zero if \arg range is null.
+ */
+CINDEX_LINKAGE int clang_Range_isNull(CXSourceRange range);
+
 /**
  * \brief Retrieve the file, line, column, and offset represented by
  * the given source location.
  *
- * If the location refers into a macro instantiation, retrieves the
- * location of the macro instantiation.
+ * If the location refers into a macro expansion, retrieves the
+ * location of the macro expansion.
  *
  * \param location the location within a source file that will be decomposed
  * into its parts.
@@ -349,6 +367,63 @@ CINDEX_LINKAGE CXSourceRange clang_getRange(CXSourceLocation begin,
  * \param offset [out] if non-NULL, will be set to the offset into the
  * buffer to which the given source location points.
  */
+CINDEX_LINKAGE void clang_getExpansionLocation(CXSourceLocation location,
+                                               CXFile *file,
+                                               unsigned *line,
+                                               unsigned *column,
+                                               unsigned *offset);
+
+/**
+ * \brief Retrieve the file, line, column, and offset represented by
+ * the given source location, as specified in a # line directive.
+ *
+ * Example: given the following source code in a file somefile.c
+ *
+ * #123 "dummy.c" 1
+ *
+ * static int func(void)
+ * {
+ *     return 0;
+ * }
+ *
+ * the location information returned by this function would be
+ *
+ * File: dummy.c Line: 124 Column: 12
+ *
+ * whereas clang_getExpansionLocation would have returned
+ *
+ * File: somefile.c Line: 3 Column: 12
+ *
+ * \param location the location within a source file that will be decomposed
+ * into its parts.
+ *
+ * \param filename [out] if non-NULL, will be set to the filename of the
+ * source location. Note that filenames returned will be for "virtual" files,
+ * which don't necessarily exist on the machine running clang - e.g. when
+ * parsing preprocessed output obtained from a different environment. If
+ * a non-NULL value is passed in, remember to dispose of the returned value
+ * using \c clang_disposeString() once you've finished with it. For an invalid
+ * source location, an empty string is returned.
+ *
+ * \param line [out] if non-NULL, will be set to the line number of the
+ * source location. For an invalid source location, zero is returned.
+ *
+ * \param column [out] if non-NULL, will be set to the column number of the
+ * source location. For an invalid source location, zero is returned.
+ */
+CINDEX_LINKAGE void clang_getPresumedLocation(CXSourceLocation location,
+                                              CXString *filename,
+                                              unsigned *line,
+                                              unsigned *column);
+
+/**
+ * \brief Legacy API to retrieve the file, line, column, and offset represented
+ * by the given source location.
+ *
+ * This interface has been replaced by the newer interface
+ * \see clang_getExpansionLocation(). See that interface's documentation for
+ * details.
+ */
 CINDEX_LINKAGE void clang_getInstantiationLocation(CXSourceLocation location,
                                                    CXFile *file,
                                                    unsigned *line,
@@ -816,18 +891,18 @@ enum CXTranslationUnit_Flags {
    */
   CXTranslationUnit_CacheCompletionResults = 0x08,
   /**
-   * \brief Enable precompiled preambles in C++.
+   * \brief DEPRECATED: Enable precompiled preambles in C++.
    *
    * Note: this is a *temporary* option that is available only while
-   * we are testing C++ precompiled preamble support.
+   * we are testing C++ precompiled preamble support. It is deprecated.
    */
   CXTranslationUnit_CXXPrecompiledPreamble = 0x10,
 
   /**
-   * \brief Enabled chained precompiled preambles in C++.
+   * \brief DEPRECATED: Enabled chained precompiled preambles in C++.
    *
    * Note: this is a *temporary* option that is available only while
-   * we are testing C++ precompiled preamble support.
+   * we are testing C++ precompiled preamble support. It is deprecated.
    */
   CXTranslationUnit_CXXChainedPCH = 0x20,
   
@@ -1094,12 +1169,14 @@ enum CXTUResourceUsageKind {
   CXTUResourceUsage_ExternalASTSource_Membuffer_MMap = 10, 
   CXTUResourceUsage_Preprocessor = 11,
   CXTUResourceUsage_PreprocessingRecord = 12,
+  CXTUResourceUsage_SourceManager_DataStructures = 13,
+  CXTUResourceUsage_Preprocessor_HeaderSearch = 14,
   CXTUResourceUsage_MEMORY_IN_BYTES_BEGIN = CXTUResourceUsage_AST,
   CXTUResourceUsage_MEMORY_IN_BYTES_END =
-    CXTUResourceUsage_PreprocessingRecord,
+    CXTUResourceUsage_Preprocessor_HeaderSearch,
 
   CXTUResourceUsage_First = CXTUResourceUsage_AST,
-  CXTUResourceUsage_Last = CXTUResourceUsage_PreprocessingRecord
+  CXTUResourceUsage_Last = CXTUResourceUsage_Preprocessor_HeaderSearch
 };
 
 /**
@@ -1237,8 +1314,11 @@ enum CXCursorKind {
   CXCursor_ObjCSynthesizeDecl            = 37,
   /** \brief An Objective-C @dynamic definition. */
   CXCursor_ObjCDynamicDecl               = 38,
+  /** \brief An access specifier. */
+  CXCursor_CXXAccessSpecifier            = 39,
+
   CXCursor_FirstDecl                     = CXCursor_UnexposedDecl,
-  CXCursor_LastDecl                      = CXCursor_ObjCDynamicDecl,
+  CXCursor_LastDecl                      = CXCursor_CXXAccessSpecifier,
 
   /* References */
   CXCursor_FirstRef                      = 40, /* Decl references */
@@ -1377,7 +1457,207 @@ enum CXCursorKind {
   /** \brief An expression that represents a block literal. */
   CXCursor_BlockExpr                     = 105,
 
-  CXCursor_LastExpr                      = 105,
+  /** \brief An integer literal.
+   */
+  CXCursor_IntegerLiteral                = 106,
+
+  /** \brief A floating point number literal.
+   */
+  CXCursor_FloatingLiteral               = 107,
+
+  /** \brief An imaginary number literal.
+   */
+  CXCursor_ImaginaryLiteral              = 108,
+
+  /** \brief A string literal.
+   */
+  CXCursor_StringLiteral                 = 109,
+
+  /** \brief A character literal.
+   */
+  CXCursor_CharacterLiteral              = 110,
+
+  /** \brief A parenthesized expression, e.g. "(1)".
+   *
+   * This AST node is only formed if full location information is requested.
+   */
+  CXCursor_ParenExpr                     = 111,
+
+  /** \brief This represents the unary-expression's (except sizeof and
+   * alignof).
+   */
+  CXCursor_UnaryOperator                 = 112,
+
+  /** \brief [C99 6.5.2.1] Array Subscripting.
+   */
+  CXCursor_ArraySubscriptExpr            = 113,
+
+  /** \brief A builtin binary operation expression such as "x + y" or
+   * "x <= y".
+   */
+  CXCursor_BinaryOperator                = 114,
+
+  /** \brief Compound assignment such as "+=".
+   */
+  CXCursor_CompoundAssignOperator        = 115,
+
+  /** \brief The ?: ternary operator.
+   */
+  CXCursor_ConditionalOperator           = 116,
+
+  /** \brief An explicit cast in C (C99 6.5.4) or a C-style cast in C++
+   * (C++ [expr.cast]), which uses the syntax (Type)expr.
+   *
+   * For example: (int)f.
+   */
+  CXCursor_CStyleCastExpr                = 117,
+
+  /** \brief [C99 6.5.2.5]
+   */
+  CXCursor_CompoundLiteralExpr           = 118,
+
+  /** \brief Describes an C or C++ initializer list.
+   */
+  CXCursor_InitListExpr                  = 119,
+
+  /** \brief The GNU address of label extension, representing &&label.
+   */
+  CXCursor_AddrLabelExpr                 = 120,
+
+  /** \brief This is the GNU Statement Expression extension: ({int X=4; X;})
+   */
+  CXCursor_StmtExpr                      = 121,
+
+  /** \brief Represents a C1X generic selection.
+   */
+  CXCursor_GenericSelectionExpr          = 122,
+
+  /** \brief Implements the GNU __null extension, which is a name for a null
+   * pointer constant that has integral type (e.g., int or long) and is the same
+   * size and alignment as a pointer.
+   *
+   * The __null extension is typically only used by system headers, which define
+   * NULL as __null in C++ rather than using 0 (which is an integer that may not
+   * match the size of a pointer).
+   */
+  CXCursor_GNUNullExpr                   = 123,
+
+  /** \brief C++'s static_cast<> expression.
+   */
+  CXCursor_CXXStaticCastExpr             = 124,
+
+  /** \brief C++'s dynamic_cast<> expression.
+   */
+  CXCursor_CXXDynamicCastExpr            = 125,
+
+  /** \brief C++'s reinterpret_cast<> expression.
+   */
+  CXCursor_CXXReinterpretCastExpr        = 126,
+
+  /** \brief C++'s const_cast<> expression.
+   */
+  CXCursor_CXXConstCastExpr              = 127,
+
+  /** \brief Represents an explicit C++ type conversion that uses "functional"
+   * notion (C++ [expr.type.conv]).
+   *
+   * Example:
+   * \code
+   *   x = int(0.5);
+   * \endcode
+   */
+  CXCursor_CXXFunctionalCastExpr         = 128,
+
+  /** \brief A C++ typeid expression (C++ [expr.typeid]).
+   */
+  CXCursor_CXXTypeidExpr                 = 129,
+
+  /** \brief [C++ 2.13.5] C++ Boolean Literal.
+   */
+  CXCursor_CXXBoolLiteralExpr            = 130,
+
+  /** \brief [C++0x 2.14.7] C++ Pointer Literal.
+   */
+  CXCursor_CXXNullPtrLiteralExpr         = 131,
+
+  /** \brief Represents the "this" expression in C++
+   */
+  CXCursor_CXXThisExpr                   = 132,
+
+  /** \brief [C++ 15] C++ Throw Expression.
+   *
+   * This handles 'throw' and 'throw' assignment-expression. When
+   * assignment-expression isn't present, Op will be null.
+   */
+  CXCursor_CXXThrowExpr                  = 133,
+
+  /** \brief A new expression for memory allocation and constructor calls, e.g:
+   * "new CXXNewExpr(foo)".
+   */
+  CXCursor_CXXNewExpr                    = 134,
+
+  /** \brief A delete expression for memory deallocation and destructor calls,
+   * e.g. "delete[] pArray".
+   */
+  CXCursor_CXXDeleteExpr                 = 135,
+
+  /** \brief A unary expression.
+   */
+  CXCursor_UnaryExpr                     = 136,
+
+  /** \brief ObjCStringLiteral, used for Objective-C string literals i.e. "foo".
+   */
+  CXCursor_ObjCStringLiteral             = 137,
+
+  /** \brief ObjCEncodeExpr, used for in Objective-C.
+   */
+  CXCursor_ObjCEncodeExpr                = 138,
+
+  /** \brief ObjCSelectorExpr used for in Objective-C.
+   */
+  CXCursor_ObjCSelectorExpr              = 139,
+
+  /** \brief Objective-C's protocol expression.
+   */
+  CXCursor_ObjCProtocolExpr              = 140,
+
+  /** \brief An Objective-C "bridged" cast expression, which casts between
+   * Objective-C pointers and C pointers, transferring ownership in the process.
+   *
+   * \code
+   *   NSString *str = (__bridge_transfer NSString *)CFCreateString();
+   * \endcode
+   */
+  CXCursor_ObjCBridgedCastExpr           = 141,
+
+  /** \brief Represents a C++0x pack expansion that produces a sequence of
+   * expressions.
+   *
+   * A pack expansion expression contains a pattern (which itself is an
+   * expression) followed by an ellipsis. For example:
+   *
+   * \code
+   * template<typename F, typename ...Types>
+   * void forward(F f, Types &&...args) {
+   *  f(static_cast<Types&&>(args)...);
+   * }
+   * \endcode
+   */
+  CXCursor_PackExpansionExpr             = 142,
+
+  /** \brief Represents an expression that computes the length of a parameter
+   * pack.
+   *
+   * \code
+   * template<typename ...Types>
+   * struct count {
+   *   static const unsigned value = sizeof...(Types);
+   * };
+   * \endcode
+   */
+  CXCursor_SizeOfPackExpr                = 143,
+
+  CXCursor_LastExpr                      = CXCursor_SizeOfPackExpr,
 
   /* Statements */
   CXCursor_FirstStmt                     = 200,
@@ -1404,8 +1684,130 @@ enum CXCursorKind {
    *
    */
   CXCursor_LabelStmt                     = 201,
-  
-  CXCursor_LastStmt                      = CXCursor_LabelStmt,
+
+  /** \brief A group of statements like { stmt stmt }.
+   *
+   * This cursor kind is used to describe compound statements, e.g. function
+   * bodies.
+   */
+  CXCursor_CompoundStmt                  = 202,
+
+  /** \brief A case statment.
+   */
+  CXCursor_CaseStmt                      = 203,
+
+  /** \brief A default statement.
+   */
+  CXCursor_DefaultStmt                   = 204,
+
+  /** \brief An if statement
+   */
+  CXCursor_IfStmt                        = 205,
+
+  /** \brief A switch statement.
+   */
+  CXCursor_SwitchStmt                    = 206,
+
+  /** \brief A while statement.
+   */
+  CXCursor_WhileStmt                     = 207,
+
+  /** \brief A do statement.
+   */
+  CXCursor_DoStmt                        = 208,
+
+  /** \brief A for statement.
+   */
+  CXCursor_ForStmt                       = 209,
+
+  /** \brief A goto statement.
+   */
+  CXCursor_GotoStmt                      = 210,
+
+  /** \brief An indirect goto statement.
+   */
+  CXCursor_IndirectGotoStmt              = 211,
+
+  /** \brief A continue statement.
+   */
+  CXCursor_ContinueStmt                  = 212,
+
+  /** \brief A break statement.
+   */
+  CXCursor_BreakStmt                     = 213,
+
+  /** \brief A return statement.
+   */
+  CXCursor_ReturnStmt                    = 214,
+
+  /** \brief A GNU inline assembly statement extension.
+   */
+  CXCursor_AsmStmt                       = 215,
+
+  /** \brief Objective-C's overall @try-@catc-@finall statement.
+   */
+  CXCursor_ObjCAtTryStmt                 = 216,
+
+  /** \brief Objective-C's @catch statement.
+   */
+  CXCursor_ObjCAtCatchStmt               = 217,
+
+  /** \brief Objective-C's @finally statement.
+   */
+  CXCursor_ObjCAtFinallyStmt             = 218,
+
+  /** \brief Objective-C's @throw statement.
+   */
+  CXCursor_ObjCAtThrowStmt               = 219,
+
+  /** \brief Objective-C's @synchronized statement.
+   */
+  CXCursor_ObjCAtSynchronizedStmt        = 220,
+
+  /** \brief Objective-C's autorelease pool statement.
+   */
+  CXCursor_ObjCAutoreleasePoolStmt       = 221,
+
+  /** \brief Objective-C's collection statement.
+   */
+  CXCursor_ObjCForCollectionStmt         = 222,
+
+  /** \brief C++'s catch statement.
+   */
+  CXCursor_CXXCatchStmt                  = 223,
+
+  /** \brief C++'s try statement.
+   */
+  CXCursor_CXXTryStmt                    = 224,
+
+  /** \brief C++'s for (* : *) statement.
+   */
+  CXCursor_CXXForRangeStmt               = 225,
+
+  /** \brief Windows Structured Exception Handling's try statement.
+   */
+  CXCursor_SEHTryStmt                    = 226,
+
+  /** \brief Windows Structured Exception Handling's except statement.
+   */
+  CXCursor_SEHExceptStmt                 = 227,
+
+  /** \brief Windows Structured Exception Handling's finally statement.
+   */
+  CXCursor_SEHFinallyStmt                = 228,
+
+  /** \brief The null satement ";": C99 6.8.3p3.
+   *
+   * This cursor kind is used to describe the null statement.
+   */
+  CXCursor_NullStmt                      = 230,
+
+  /** \brief Adaptor class for mixing declarations with statements and
+   * expressions.
+   */
+  CXCursor_DeclStmt                      = 231,
+
+  CXCursor_LastStmt                      = CXCursor_DeclStmt,
 
   /**
    * \brief Cursor that represents the translation unit itself.
@@ -1426,7 +1828,10 @@ enum CXCursorKind {
   CXCursor_IBActionAttr                  = 401,
   CXCursor_IBOutletAttr                  = 402,
   CXCursor_IBOutletCollectionAttr        = 403,
-  CXCursor_LastAttr                      = CXCursor_IBOutletCollectionAttr,
+  CXCursor_CXXFinalAttr                  = 404,
+  CXCursor_CXXOverrideAttr               = 405,
+  CXCursor_AnnotateAttr                  = 406,
+  CXCursor_LastAttr                      = CXCursor_AnnotateAttr,
      
   /* Preprocessing */
   CXCursor_PreprocessingDirective        = 500,
@@ -1458,6 +1863,7 @@ enum CXCursorKind {
  */
 typedef struct {
   enum CXCursorKind kind;
+  int xdata;
   void *data[3];
 } CXCursor;
 
@@ -1485,6 +1891,11 @@ CINDEX_LINKAGE CXCursor clang_getTranslationUnitCursor(CXTranslationUnit);
  */
 CINDEX_LINKAGE unsigned clang_equalCursors(CXCursor, CXCursor);
 
+/**
+ * \brief Returns non-zero if \arg cursor is null.
+ */
+int clang_Cursor_isNull(CXCursor);
+
 /**
  * \brief Compute a hash value for the given cursor.
  */
@@ -1600,6 +2011,11 @@ CINDEX_LINKAGE enum CXLanguageKind {
  */
 CINDEX_LINKAGE enum CXLanguageKind clang_getCursorLanguage(CXCursor cursor);
 
+/**
+ * \brief Returns the translation unit that a cursor originated from.
+ */
+CINDEX_LINKAGE CXTranslationUnit clang_Cursor_getTranslationUnit(CXCursor);
+
 
 /**
  * \brief A fast container representing a set of CXCursors.
@@ -1886,7 +2302,8 @@ enum CXTypeKind {
   CXType_ObjCInterface = 108,
   CXType_ObjCObjectPointer = 109,
   CXType_FunctionNoProto = 110,
-  CXType_FunctionProto = 111
+  CXType_FunctionProto = 111,
+  CXType_ConstantArray = 112
 };
 
 /**
@@ -1977,6 +2394,20 @@ CINDEX_LINKAGE CXType clang_getCursorResultType(CXCursor C);
  */
 CINDEX_LINKAGE unsigned clang_isPODType(CXType T);
 
+/**
+ * \brief Return the element type of an array type.
+ *
+ * If a non-array type is passed in, an invalid type is returned.
+ */
+CINDEX_LINKAGE CXType clang_getArrayElementType(CXType T);
+
+/**
+ * \brief Return the the array size of a constant array.
+ *
+ * If a non-array type is passed in, -1 is returned.
+ */
+CINDEX_LINKAGE long long clang_getArraySize(CXType T);
+
 /**
  * \brief Returns 1 if the base class specified by the cursor with kind
  *   CX_CXXBaseSpecifier is virtual.
@@ -1996,7 +2427,8 @@ enum CX_CXXAccessSpecifier {
 
 /**
  * \brief Returns the access control level for the C++ base specifier
- *  represented by a cursor with kind CX_CXXBaseSpecifier.
+ * represented by a cursor with kind CXCursor_CXXBaseSpecifier or
+ * CXCursor_AccessSpecifier.
  */
 CINDEX_LINKAGE enum CX_CXXAccessSpecifier clang_getCXXAccessSpecifier(CXCursor);
 
@@ -2377,6 +2809,54 @@ CINDEX_LINKAGE enum CXCursorKind clang_getTemplateCursorKind(CXCursor C);
  * from which it was instantiated. Otherwise, returns a NULL cursor.
  */
 CINDEX_LINKAGE CXCursor clang_getSpecializedCursorTemplate(CXCursor C);
+
+/**
+ * \brief Given a cursor that references something else, return the source range
+ * covering that reference.
+ *
+ * \param C A cursor pointing to a member reference, a declaration reference, or
+ * an operator call.
+ * \param NameFlags A bitset with three independent flags: 
+ * CXNameRange_WantQualifier, CXNameRange_WantTemplateArgs, and
+ * CXNameRange_WantSinglePiece.
+ * \param PieceIndex For contiguous names or when passing the flag 
+ * CXNameRange_WantSinglePiece, only one piece with index 0 is 
+ * available. When the CXNameRange_WantSinglePiece flag is not passed for a
+ * non-contiguous names, this index can be used to retreive the individual
+ * pieces of the name. See also CXNameRange_WantSinglePiece.
+ *
+ * \returns The piece of the name pointed to by the given cursor. If there is no
+ * name, or if the PieceIndex is out-of-range, a null-cursor will be returned.
+ */
+CINDEX_LINKAGE CXSourceRange clang_getCursorReferenceNameRange(CXCursor C,
+                                                unsigned NameFlags, 
+                                                unsigned PieceIndex);
+
+enum CXNameRefFlags {
+  /**
+   * \brief Include the nested-name-specifier, e.g. Foo:: in x.Foo::y, in the
+   * range.
+   */
+  CXNameRange_WantQualifier = 0x1,
+  
+  /**
+   * \brief Include the explicit template arguments, e.g. <int> in x.f<int>, in 
+   * the range.
+   */
+  CXNameRange_WantTemplateArgs = 0x2,
+
+  /**
+   * \brief If the name is non-contiguous, return the full spanning range.
+   *
+   * Non-contiguous names occur in Objective-C when a selector with two or more
+   * parameters is used, or in C++ when using an operator:
+   * \code
+   * [object doSomething:here withValue:there]; // ObjC
+   * return some_vector[1]; // C++
+   * \endcode
+   */
+  CXNameRange_WantSinglePiece = 0x4
+};
   
 /**
  * @}
@@ -2801,8 +3281,7 @@ clang_getCompletionChunkText(CXCompletionString completion_string,
  * \param chunk_number the 0-based index of the chunk in the completion string.
  *
  * \returns the completion string associated with the chunk at index
- * \c chunk_number, or NULL if that chunk is not represented by a completion
- * string.
+ * \c chunk_number.
  */
 CINDEX_LINKAGE CXCompletionString
 clang_getCompletionChunkCompletionString(CXCompletionString completion_string,
@@ -2840,6 +3319,45 @@ clang_getCompletionPriority(CXCompletionString completion_string);
 CINDEX_LINKAGE enum CXAvailabilityKind 
 clang_getCompletionAvailability(CXCompletionString completion_string);
 
+/**
+ * \brief Retrieve the number of annotations associated with the given
+ * completion string.
+ *
+ * \param completion_string the completion string to query.
+ *
+ * \returns the number of annotations associated with the given completion
+ * string.
+ */
+CINDEX_LINKAGE unsigned
+clang_getCompletionNumAnnotations(CXCompletionString completion_string);
+
+/**
+ * \brief Retrieve the annotation associated with the given completion string.
+ *
+ * \param completion_string the completion string to query.
+ *
+ * \param annotation_number the 0-based index of the annotation of the
+ * completion string.
+ *
+ * \returns annotation string associated with the completion at index
+ * \c annotation_number, or a NULL string if that annotation is not available.
+ */
+CINDEX_LINKAGE CXString
+clang_getCompletionAnnotation(CXCompletionString completion_string,
+                              unsigned annotation_number);
+
+/**
+ * \brief Retrieve a completion string for an arbitrary declaration or macro
+ * definition cursor.
+ *
+ * \param cursor The cursor to query.
+ *
+ * \returns A non-context-sensitive completion string for declaration and macro
+ * definition cursors, or NULL for other kinds of cursors.
+ */
+CINDEX_LINKAGE CXCompletionString
+clang_getCursorCompletionString(CXCursor cursor);
+  
 /**
  * \brief Contains the results of code-completion.
  *
@@ -3144,6 +3662,54 @@ CXDiagnostic clang_codeCompleteGetDiagnostic(CXCodeCompleteResults *Results,
 CINDEX_LINKAGE
 unsigned long long clang_codeCompleteGetContexts(
                                                 CXCodeCompleteResults *Results);
+
+/**
+ * \brief Returns the cursor kind for the container for the current code
+ * completion context. The container is only guaranteed to be set for
+ * contexts where a container exists (i.e. member accesses or Objective-C
+ * message sends); if there is not a container, this function will return
+ * CXCursor_InvalidCode.
+ *
+ * \param Results the code completion results to query
+ *
+ * \param IsIncomplete on return, this value will be false if Clang has complete
+ * information about the container. If Clang does not have complete
+ * information, this value will be true.
+ *
+ * \returns the container kind, or CXCursor_InvalidCode if there is not a
+ * container
+ */
+CINDEX_LINKAGE
+enum CXCursorKind clang_codeCompleteGetContainerKind(
+                                                 CXCodeCompleteResults *Results,
+                                                     unsigned *IsIncomplete);
+
+/**
+ * \brief Returns the USR for the container for the current code completion
+ * context. If there is not a container for the current context, this
+ * function will return the empty string.
+ *
+ * \param Results the code completion results to query
+ *
+ * \returns the USR for the container
+ */
+CINDEX_LINKAGE
+CXString clang_codeCompleteGetContainerUSR(CXCodeCompleteResults *Results);
+  
+  
+/**
+ * \brief Returns the currently-entered selector for an Objective-C message
+ * send, formatted like "initWithFoo:bar:". Only guaranteed to return a
+ * non-empty string for CXCompletionContext_ObjCInstanceMessage and
+ * CXCompletionContext_ObjCClassMessage.
+ *
+ * \param Results the code completion results to query
+ *
+ * \returns the selector (or partial selector) that has been entered thus far
+ * for an Objective-C message send.
+ */
+CINDEX_LINKAGE
+CXString clang_codeCompleteGetObjCSelector(CXCodeCompleteResults *Results);
   
 /**
  * @}
@@ -3242,6 +3808,53 @@ CINDEX_LINKAGE void clang_remap_getFilenames(CXRemapping, unsigned index,
  */
 CINDEX_LINKAGE void clang_remap_dispose(CXRemapping);
 
+/**
+ * @}
+ */
+
+/** \defgroup CINDEX_HIGH Higher level API functions
+ *
+ * @{
+ */
+
+enum CXVisitorResult {
+  CXVisit_Break,
+  CXVisit_Continue
+};
+
+typedef struct {
+  void *context;
+  enum CXVisitorResult (*visit)(void *context, CXCursor, CXSourceRange);
+} CXCursorAndRangeVisitor;
+
+/**
+ * \brief Find references of a declaration in a specific file.
+ * 
+ * \param cursor pointing to a declaration or a reference of one.
+ *
+ * \param file to search for references.
+ *
+ * \param visitor callback that will receive pairs of CXCursor/CXSourceRange for
+ * each reference found.
+ * The CXSourceRange will point inside the file; if the reference is inside
+ * a macro (and not a macro argument) the CXSourceRange will be invalid.
+ */
+CINDEX_LINKAGE void clang_findReferencesInFile(CXCursor cursor, CXFile file,
+                                               CXCursorAndRangeVisitor visitor);
+
+#ifdef __has_feature
+#  if __has_feature(blocks)
+
+typedef enum CXVisitorResult
+    (^CXCursorAndRangeVisitorBlock)(CXCursor, CXSourceRange);
+
+CINDEX_LINKAGE
+void clang_findReferencesInFileWithBlock(CXCursor, CXFile,
+                                         CXCursorAndRangeVisitorBlock);
+
+#  endif
+#endif
+
 /**
  * @}
  */
diff --git a/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMT.h b/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMT.h
index ad5cf4a2c16c..d8dea0b3adb9 100644
--- a/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMT.h
+++ b/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMT.h
@@ -15,7 +15,7 @@
 
 namespace clang {
   class ASTContext;
-  class DiagnosticClient;
+  class DiagnosticConsumer;
 
 namespace arcmt {
   class MigrationPass;
@@ -28,35 +28,53 @@ namespace arcmt {
 /// It then checks the AST and produces errors/warning for ARC migration issues
 /// that the user needs to handle manually.
 ///
+/// \param emitPremigrationARCErrors if true all ARC errors will get emitted
+/// even if the migrator can fix them, but the function will still return false
+/// if all ARC errors can be fixed.
+///
+/// \param plistOut if non-empty, it is the file path to store the plist with
+/// the pre-migration ARC diagnostics.
+///
 /// \returns false if no error is produced, true otherwise.
 bool checkForManualIssues(CompilerInvocation &CI,
-                          llvm::StringRef Filename, InputKind Kind,
-                          DiagnosticClient *DiagClient);
+                          StringRef Filename, InputKind Kind,
+                          DiagnosticConsumer *DiagClient,
+                          bool emitPremigrationARCErrors = false,
+                          StringRef plistOut = StringRef());
 
 /// \brief Works similar to checkForManualIssues but instead of checking, it
 /// applies automatic modifications to source files to conform to ARC.
 ///
 /// \returns false if no error is produced, true otherwise.
 bool applyTransformations(CompilerInvocation &origCI,
-                          llvm::StringRef Filename, InputKind Kind,
-                          DiagnosticClient *DiagClient);
+                          StringRef Filename, InputKind Kind,
+                          DiagnosticConsumer *DiagClient);
 
 /// \brief Applies automatic modifications and produces temporary files
 /// and metadata into the \arg outputDir path.
 ///
+/// \param emitPremigrationARCErrors if true all ARC errors will get emitted
+/// even if the migrator can fix them, but the function will still return false
+/// if all ARC errors can be fixed.
+///
+/// \param plistOut if non-empty, it is the file path to store the plist with
+/// the pre-migration ARC diagnostics.
+///
 /// \returns false if no error is produced, true otherwise.
 bool migrateWithTemporaryFiles(CompilerInvocation &origCI,
-                               llvm::StringRef Filename, InputKind Kind,
-                               DiagnosticClient *DiagClient,
-                               llvm::StringRef outputDir);
+                               StringRef Filename, InputKind Kind,
+                               DiagnosticConsumer *DiagClient,
+                               StringRef outputDir,
+                               bool emitPremigrationARCErrors,
+                               StringRef plistOut);
 
 /// \brief Get the set of file remappings from the \arg outputDir path that
 /// migrateWithTemporaryFiles produced.
 ///
 /// \returns false if no error is produced, true otherwise.
 bool getFileRemappings(std::vector<std::pair<std::string,std::string> > &remap,
-                       llvm::StringRef outputDir,
-                       DiagnosticClient *DiagClient);
+                       StringRef outputDir,
+                       DiagnosticConsumer *DiagClient);
 
 typedef void (*TransformFn)(MigrationPass &pass);
 
@@ -64,12 +82,12 @@ std::vector<TransformFn> getAllTransformations();
 
 class MigrationProcess {
   CompilerInvocation OrigCI;
-  DiagnosticClient *DiagClient;
+  DiagnosticConsumer *DiagClient;
   FileRemapper Remapper;
 
 public:
-  MigrationProcess(const CompilerInvocation &CI, DiagnosticClient *diagClient,
-                   llvm::StringRef outputDir = llvm::StringRef());
+  MigrationProcess(const CompilerInvocation &CI, DiagnosticConsumer *diagClient,
+                   StringRef outputDir = StringRef());
 
   class RewriteListener {
   public:
@@ -78,7 +96,7 @@ class MigrationProcess {
     virtual void start(ASTContext &Ctx) { }
     virtual void finish() { }
 
-    virtual void insert(SourceLocation loc, llvm::StringRef text) { }
+    virtual void insert(SourceLocation loc, StringRef text) { }
     virtual void remove(CharSourceRange range) { }
   };
 
diff --git a/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMTActions.h b/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMTActions.h
index 4c714f55b390..4eac4facdd82 100644
--- a/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMTActions.h
+++ b/contrib/llvm/tools/clang/include/clang/ARCMigrate/ARCMTActions.h
@@ -34,11 +34,15 @@ class ModifyAction : public WrapperFrontendAction {
 
 class MigrateAction : public WrapperFrontendAction {
   std::string MigrateDir;
+  std::string PlistOut;
+  bool EmitPremigrationARCErros;
 protected:
   virtual bool BeginInvocation(CompilerInstance &CI);
 
 public:
-  MigrateAction(FrontendAction *WrappedAction, llvm::StringRef migrateDir);
+  MigrateAction(FrontendAction *WrappedAction, StringRef migrateDir,
+                StringRef plistOut,
+                bool emitPremigrationARCErrors);
 };
 
 }
diff --git a/contrib/llvm/tools/clang/include/clang/ARCMigrate/FileRemapper.h b/contrib/llvm/tools/clang/include/clang/ARCMigrate/FileRemapper.h
index 809f6a5f71be..9a0b690ad691 100644
--- a/contrib/llvm/tools/clang/include/clang/ARCMigrate/FileRemapper.h
+++ b/contrib/llvm/tools/clang/include/clang/ARCMigrate/FileRemapper.h
@@ -10,6 +10,7 @@
 #ifndef LLVM_CLANG_ARCMIGRATE_FILEREMAPPER_H
 #define LLVM_CLANG_ARCMIGRATE_FILEREMAPPER_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/DenseMap.h"
@@ -22,7 +23,7 @@ namespace llvm {
 namespace clang {
   class FileManager;
   class FileEntry;
-  class Diagnostic;
+  class DiagnosticsEngine;
   class CompilerInvocation;
 
 namespace arcmt {
@@ -41,32 +42,32 @@ class FileRemapper {
   FileRemapper();
   ~FileRemapper();
   
-  bool initFromDisk(llvm::StringRef outputDir, Diagnostic &Diag,
+  bool initFromDisk(StringRef outputDir, DiagnosticsEngine &Diag,
                     bool ignoreIfFilesChanged);
-  bool flushToDisk(llvm::StringRef outputDir, Diagnostic &Diag);
+  bool flushToDisk(StringRef outputDir, DiagnosticsEngine &Diag);
 
-  bool overwriteOriginal(Diagnostic &Diag,
-                         llvm::StringRef outputDir = llvm::StringRef());
+  bool overwriteOriginal(DiagnosticsEngine &Diag,
+                         StringRef outputDir = StringRef());
 
-  void remap(llvm::StringRef filePath, llvm::MemoryBuffer *memBuf);
-  void remap(llvm::StringRef filePath, llvm::StringRef newPath);
+  void remap(StringRef filePath, llvm::MemoryBuffer *memBuf);
+  void remap(StringRef filePath, StringRef newPath);
 
   void applyMappings(CompilerInvocation &CI) const;
 
   void transferMappingsAndClear(CompilerInvocation &CI);
 
-  void clear(llvm::StringRef outputDir = llvm::StringRef());
+  void clear(StringRef outputDir = StringRef());
 
 private:
   void remap(const FileEntry *file, llvm::MemoryBuffer *memBuf);
   void remap(const FileEntry *file, const FileEntry *newfile);
 
-  const FileEntry *getOriginalFile(llvm::StringRef filePath);
+  const FileEntry *getOriginalFile(StringRef filePath);
   void resetTarget(Target &targ);
 
-  bool report(const std::string &err, Diagnostic &Diag);
+  bool report(const Twine &err, DiagnosticsEngine &Diag);
 
-  std::string getRemapInfoFile(llvm::StringRef outputDir);
+  std::string getRemapInfoFile(StringRef outputDir);
 };
 
 } // end namespace arcmt
diff --git a/contrib/llvm/tools/clang/include/clang/AST/APValue.h b/contrib/llvm/tools/clang/include/clang/AST/APValue.h
index fec7d29f6d7b..375af28f0c56 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/APValue.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/APValue.h
@@ -14,11 +14,13 @@
 #ifndef LLVM_CLANG_AST_APVALUE_H
 #define LLVM_CLANG_AST_APVALUE_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/APFloat.h"
 
 namespace clang {
   class CharUnits;
+  class DiagnosticBuilder;
   class Expr;
 
 /// APValue - This class implements a discriminated union of [uninitialized]
@@ -103,7 +105,7 @@ class APValue {
   bool isLValue() const { return Kind == LValue; }
   bool isVector() const { return Kind == Vector; }
 
-  void print(llvm::raw_ostream &OS) const;
+  void print(raw_ostream &OS) const;
   void dump() const;
 
   APSInt &getInt() {
@@ -233,11 +235,15 @@ class APValue {
   void MakeLValue();
 };
 
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const APValue &V) {
+inline raw_ostream &operator<<(raw_ostream &OS, const APValue &V) {
   V.print(OS);
   return OS;
 }
 
+// Writes a concise representation of V to DB, in a single << operation.
+const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
+                                    const APValue &V);
+
 } // end namespace clang.
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h b/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h
index 1526f36ba2fb..c4ffac5341e7 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h
@@ -37,7 +37,6 @@
 
 namespace llvm {
   struct fltSemantics;
-  class raw_ostream;
 }
 
 namespace clang {
@@ -45,7 +44,7 @@ namespace clang {
   class ASTRecordLayout;
   class BlockExpr;
   class CharUnits;
-  class Diagnostic;
+  class DiagnosticsEngine;
   class Expr;
   class ExternalASTSource;
   class ASTMutationListener;
@@ -64,6 +63,7 @@ namespace clang {
   class ObjCIvarDecl;
   class ObjCIvarRefExpr;
   class ObjCPropertyDecl;
+  class ParmVarDecl;
   class RecordDecl;
   class StoredDeclsMap;
   class TagDecl;
@@ -120,6 +120,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   mutable llvm::FoldingSet<ObjCObjectTypeImpl> ObjCObjectTypes;
   mutable llvm::FoldingSet<ObjCObjectPointerType> ObjCObjectPointerTypes;
   mutable llvm::FoldingSet<AutoType> AutoTypes;
+  mutable llvm::FoldingSet<AtomicType> AtomicTypes;
   llvm::FoldingSet<AttributedType> AttributedTypes;
 
   mutable llvm::FoldingSet<QualifiedTemplateName> QualifiedTemplateNames;
@@ -149,10 +150,19 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   
   /// \brief Mapping from ObjCContainers to their ObjCImplementations.
   llvm::DenseMap<ObjCContainerDecl*, ObjCImplDecl*> ObjCImpls;
+  
+  /// \brief Mapping from ObjCMethod to its duplicate declaration in the same
+  /// interface.
+  llvm::DenseMap<const ObjCMethodDecl*,const ObjCMethodDecl*> ObjCMethodRedecls;
 
   /// \brief Mapping from __block VarDecls to their copy initialization expr.
   llvm::DenseMap<const VarDecl*, Expr*> BlockVarCopyInits;
     
+  /// \brief Mapping from class scope functions specialization to their
+  ///  template patterns.
+  llvm::DenseMap<const FunctionDecl*, FunctionDecl*>
+    ClassScopeSpecializationPattern;
+
   /// \brief Representation of a "canonical" template template parameter that
   /// is used in canonical template names.
   class CanonicalTemplateTemplateParm : public llvm::FoldingSetNode {
@@ -175,34 +185,41 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   TemplateTemplateParmDecl *
     getCanonicalTemplateTemplateParmDecl(TemplateTemplateParmDecl *TTP) const;
 
-  /// \brief Whether __[u]int128_t identifier is installed.
-  bool IsInt128Installed;
+  /// \brief The typedef for the __int128_t type.
+  mutable TypedefDecl *Int128Decl;
 
+  /// \brief The typedef for the __uint128_t type.
+  mutable TypedefDecl *UInt128Decl;
+  
   /// BuiltinVaListType - built-in va list type.
   /// This is initially null and set by Sema::LazilyCreateBuiltin when
   /// a builtin that takes a valist is encountered.
   QualType BuiltinVaListType;
 
-  /// ObjCIdType - a pseudo built-in typedef type (set by Sema).
-  QualType ObjCIdTypedefType;
+  /// \brief The typedef for the predefined 'id' type.
+  mutable TypedefDecl *ObjCIdDecl;
+  
+  /// \brief The typedef for the predefined 'SEL' type.
+  mutable TypedefDecl *ObjCSelDecl;
 
-  /// ObjCSelType - another pseudo built-in typedef type (set by Sema).
-  QualType ObjCSelTypedefType;
-
-  /// ObjCProtoType - another pseudo built-in typedef type (set by Sema).
   QualType ObjCProtoType;
   const RecordType *ProtoStructType;
 
-  /// ObjCClassType - another pseudo built-in typedef type (set by Sema).
-  QualType ObjCClassTypedefType;
+  /// \brief The typedef for the predefined 'Class' type.
+  mutable TypedefDecl *ObjCClassDecl;
+  
+  // Typedefs which may be provided defining the structure of Objective-C
+  // pseudo-builtins
+  QualType ObjCIdRedefinitionType;
+  QualType ObjCClassRedefinitionType;
+  QualType ObjCSelRedefinitionType;
 
   QualType ObjCConstantStringType;
   mutable RecordDecl *CFConstantStringTypeDecl;
 
-  mutable RecordDecl *NSConstantStringTypeDecl;
-
-  mutable RecordDecl *ObjCFastEnumerationStateTypeDecl;
-
+  /// \brief The typedef declaration for the Objective-C "instancetype" type.
+  TypedefDecl *ObjCInstanceTypeDecl;
+  
   /// \brief The type for the C FILE type.
   TypeDecl *FILEDecl;
 
@@ -213,9 +230,15 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   TypeDecl *sigjmp_bufDecl;
 
   /// \brief Type for the Block descriptor for Blocks CodeGen.
+  ///
+  /// Since this is only used for generation of debug info, it is not
+  /// serialized.
   mutable RecordDecl *BlockDescriptorType;
 
   /// \brief Type for the Block descriptor for Blocks CodeGen.
+  ///
+  /// Since this is only used for generation of debug info, it is not
+  /// serialized.
   mutable RecordDecl *BlockDescriptorExtendedType;
 
   /// \brief Declaration for the CUDA cudaConfigureCall function.
@@ -295,6 +318,12 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   typedef UsuallyTinyPtrVector<const CXXMethodDecl> CXXMethodVector;
   llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector> OverriddenMethods;
 
+  /// \brief Mapping that stores parameterIndex values for ParmVarDecls
+  /// when that value exceeds the bitfield size of
+  /// ParmVarDeclBits.ParameterIndex.
+  typedef llvm::DenseMap<const VarDecl *, unsigned> ParameterIndexTable;
+  ParameterIndexTable ParamIndices;  
+  
   TranslationUnitDecl *TUDecl;
 
   /// SourceMgr - The associated SourceManager object.
@@ -302,7 +331,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
 
   /// LangOpts - The language options used to create the AST associated with
   ///  this ASTContext object.
-  LangOptions LangOpts;
+  LangOptions &LangOpts;
 
   /// \brief The allocator used to create AST objects.
   ///
@@ -318,26 +347,29 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   CXXABI *createCXXABI(const TargetInfo &T);
 
   /// \brief The logical -> physical address space map.
-  const LangAS::Map &AddrSpaceMap;
+  const LangAS::Map *AddrSpaceMap;
 
   friend class ASTDeclReader;
-
+  friend class ASTReader;
+  friend class ASTWriter;
+  
+  const TargetInfo *Target;
+  clang::PrintingPolicy PrintingPolicy;
+  
 public:
-  const TargetInfo &Target;
   IdentifierTable &Idents;
   SelectorTable &Selectors;
   Builtin::Context &BuiltinInfo;
   mutable DeclarationNameTable DeclarationNames;
   llvm::OwningPtr<ExternalASTSource> ExternalSource;
   ASTMutationListener *Listener;
-  clang::PrintingPolicy PrintingPolicy;
 
-  // Typedefs which may be provided defining the structure of Objective-C
-  // pseudo-builtins
-  QualType ObjCIdRedefinitionType;
-  QualType ObjCClassRedefinitionType;
-  QualType ObjCSelRedefinitionType;
+  clang::PrintingPolicy getPrintingPolicy() const { return PrintingPolicy; }
 
+  void setPrintingPolicy(clang::PrintingPolicy Policy) {
+    PrintingPolicy = Policy;
+  }
+  
   SourceManager& getSourceManager() { return SourceMgr; }
   const SourceManager& getSourceManager() const { return SourceMgr; }
   void *Allocate(unsigned Size, unsigned Align = 8) const {
@@ -357,9 +389,11 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
     return DiagAllocator;
   }
 
+  const TargetInfo &getTargetInfo() const { return *Target; }
+  
   const LangOptions& getLangOptions() const { return LangOpts; }
 
-  Diagnostic &getDiagnostics() const;
+  DiagnosticsEngine &getDiagnostics() const;
 
   FullSourceLoc getFullLoc(SourceLocation Loc) const {
     return FullSourceLoc(Loc,SourceMgr);
@@ -377,6 +411,11 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   MemberSpecializationInfo *getInstantiatedFromStaticDataMember(
                                                            const VarDecl *Var);
 
+  FunctionDecl *getClassScopeSpecializationPattern(const FunctionDecl *FD);
+
+  void setClassScopeSpecializationPattern(FunctionDecl *FD,
+                                          FunctionDecl *Pattern);
+
   /// \brief Note that the static data member \p Inst is an instantiation of
   /// the static data member template \p Tmpl of a class template.
   void setInstantiatedFromStaticDataMember(VarDecl *Inst, VarDecl *Tmpl,
@@ -413,17 +452,17 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// BitfieldFollowsBitfield - return 'true" if 'FD' is a
   /// bitfield which follows the bitfield 'LastFD'.
   bool BitfieldFollowsBitfield(const FieldDecl *FD,
-                                   const FieldDecl *LastFD) const;
+                               const FieldDecl *LastFD) const;
   
-  /// NoneBitfieldFollowsBitfield - return 'true" if 'FD' is not a
+  /// NonBitfieldFollowsBitfield - return 'true" if 'FD' is not a
   /// bitfield which follows the bitfield 'LastFD'.
-  bool NoneBitfieldFollowsBitfield(const FieldDecl *FD,
-                                   const FieldDecl *LastFD) const;
+  bool NonBitfieldFollowsBitfield(const FieldDecl *FD,
+                                  const FieldDecl *LastFD) const;
   
-  /// BitfieldFollowsNoneBitfield - return 'true" if 'FD' is a
+  /// BitfieldFollowsNonBitfield - return 'true" if 'FD' is a
   /// bitfield which follows the none bitfield 'LastFD'.
-  bool BitfieldFollowsNoneBitfield(const FieldDecl *FD,
-                                   const FieldDecl *LastFD) const;
+  bool BitfieldFollowsNonBitfield(const FieldDecl *FD,
+                                  const FieldDecl *LastFD) const;
 
   // Access to the set of methods overridden by the given C++ method.
   typedef CXXMethodVector::iterator overridden_cxx_method_iterator;
@@ -454,6 +493,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   CanQualType UnsignedCharTy, UnsignedShortTy, UnsignedIntTy, UnsignedLongTy;
   CanQualType UnsignedLongLongTy, UnsignedInt128Ty;
   CanQualType FloatTy, DoubleTy, LongDoubleTy;
+  CanQualType HalfTy; // [OpenCL 6.1.1.1], ARM NEON
   CanQualType FloatComplexTy, DoubleComplexTy, LongDoubleComplexTy;
   CanQualType VoidPtrTy, NullPtrTy;
   CanQualType DependentTy, OverloadTy, BoundMemberTy, UnknownAnyTy;
@@ -463,10 +503,11 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   mutable QualType AutoDeductTy;     // Deduction against 'auto'.
   mutable QualType AutoRRefDeductTy; // Deduction against 'auto &&'.
 
-  ASTContext(const LangOptions& LOpts, SourceManager &SM, const TargetInfo &t,
+  ASTContext(LangOptions& LOpts, SourceManager &SM, const TargetInfo *t,
              IdentifierTable &idents, SelectorTable &sels,
              Builtin::Context &builtins,
-             unsigned size_reserve);
+             unsigned size_reserve,
+             bool DelayInitialization = false);
 
   ~ASTContext();
 
@@ -497,6 +538,12 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   void PrintStats() const;
   const std::vector<Type*>& getTypes() const { return Types; }
 
+  /// \brief Retrieve the declaration for the 128-bit signed integer type.
+  TypedefDecl *getInt128Decl() const;
+
+  /// \brief Retrieve the declaration for the 128-bit unsigned integer type.
+  TypedefDecl *getUInt128Decl() const;
+  
   //===--------------------------------------------------------------------===//
   //                           Type Constructors
   //===--------------------------------------------------------------------===//
@@ -560,6 +607,10 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
     return CanQualType::CreateUnsafe(getPointerType((QualType) T));
   }
 
+  /// getAtomicType - Return the uniqued reference to the atomic type for
+  /// the specified type.
+  QualType getAtomicType(QualType T) const;
+
   /// getBlockPointerType - Return the uniqued reference to the type for a block
   /// of the specified type.
   QualType getBlockPointerType(QualType T) const;
@@ -568,29 +619,10 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// blocks.
   QualType getBlockDescriptorType() const;
 
-  // Set the type for a Block descriptor type.
-  void setBlockDescriptorType(QualType T);
-  /// Get the BlockDescriptorType type, or NULL if it hasn't yet been built.
-  QualType getRawBlockdescriptorType() {
-    if (BlockDescriptorType)
-      return getTagDeclType(BlockDescriptorType);
-    return QualType();
-  }
-
   /// This gets the struct used to keep track of the extended descriptor for
   /// pointer to blocks.
   QualType getBlockDescriptorExtendedType() const;
 
-  // Set the type for a Block descriptor extended type.
-  void setBlockDescriptorExtendedType(QualType T);
-  /// Get the BlockDescriptorExtendedType type, or NULL if it hasn't yet been
-  /// built.
-  QualType getRawBlockdescriptorExtendedType() const {
-    if (BlockDescriptorExtendedType)
-      return getTagDeclType(BlockDescriptorExtendedType);
-    return QualType();
-  }
-
   void setcudaConfigureCallDecl(FunctionDecl *FD) {
     cudaConfigureCallDecl = FD;
   }
@@ -599,7 +631,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   }
 
   /// This builds the struct used for __block variables.
-  QualType BuildByRefType(llvm::StringRef DeclName, QualType Ty) const;
+  QualType BuildByRefType(StringRef DeclName, QualType Ty) const;
 
   /// Returns true iff we need copy/dispose helpers for the given type.
   bool BlockRequiresCopying(QualType Ty) const;
@@ -824,19 +856,6 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   // constant CFStrings.
   QualType getCFConstantStringType() const;
 
-  // getNSConstantStringType - Return the C structure type used to represent
-  // constant NSStrings.
-  QualType getNSConstantStringType() const;
-  /// Get the structure type used to representation NSStrings, or NULL
-  /// if it hasn't yet been built.
-  QualType getRawNSConstantStringType() const {
-    if (NSConstantStringTypeDecl)
-      return getTagDeclType(NSConstantStringTypeDecl);
-    return QualType();
-  }
-  void setNSConstantStringType(QualType T);
-
-
   /// Get the structure type used to representation CFStrings, or NULL
   /// if it hasn't yet been built.
   QualType getRawCFConstantStringType() const {
@@ -852,19 +871,56 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
     return ObjCConstantStringType;
   }
 
-  //// This gets the struct used to keep track of fast enumerations.
-  QualType getObjCFastEnumerationStateType() const;
-
-  /// Get the ObjCFastEnumerationState type, or NULL if it hasn't yet
-  /// been built.
-  QualType getRawObjCFastEnumerationStateType() const {
-    if (ObjCFastEnumerationStateTypeDecl)
-      return getTagDeclType(ObjCFastEnumerationStateTypeDecl);
-    return QualType();
+  /// \brief Retrieve the type that 'id' has been defined to, which may be
+  /// different from the built-in 'id' if 'id' has been typedef'd.
+  QualType getObjCIdRedefinitionType() const {
+    if (ObjCIdRedefinitionType.isNull())
+      return getObjCIdType();
+    return ObjCIdRedefinitionType;
+  }
+  
+  /// \brief Set the user-written type that redefines 'id'.
+  void setObjCIdRedefinitionType(QualType RedefType) {
+    ObjCIdRedefinitionType = RedefType;
   }
 
-  void setObjCFastEnumerationStateType(QualType T);
+  /// \brief Retrieve the type that 'Class' has been defined to, which may be
+  /// different from the built-in 'Class' if 'Class' has been typedef'd.
+  QualType getObjCClassRedefinitionType() const {
+    if (ObjCClassRedefinitionType.isNull())
+      return getObjCClassType();
+    return ObjCClassRedefinitionType;
+  }
+  
+  /// \brief Set the user-written type that redefines 'SEL'.
+  void setObjCClassRedefinitionType(QualType RedefType) {
+    ObjCClassRedefinitionType = RedefType;
+  }
 
+  /// \brief Retrieve the type that 'SEL' has been defined to, which may be
+  /// different from the built-in 'SEL' if 'SEL' has been typedef'd.
+  QualType getObjCSelRedefinitionType() const {
+    if (ObjCSelRedefinitionType.isNull())
+      return getObjCSelType();
+    return ObjCSelRedefinitionType;
+  }
+
+  
+  /// \brief Set the user-written type that redefines 'SEL'.
+  void setObjCSelRedefinitionType(QualType RedefType) {
+    ObjCSelRedefinitionType = RedefType;
+  }
+
+  /// \brief Retrieve the Objective-C "instancetype" type, if already known;
+  /// otherwise, returns a NULL type;
+  QualType getObjCInstanceType() {
+    return getTypeDeclType(getObjCInstanceTypeDecl());
+  }
+
+  /// \brief Retrieve the typedef declaration corresponding to the Objective-C
+  /// "instancetype" type.
+  TypedefDecl *getObjCInstanceTypeDecl();
+  
   /// \brief Set the type for the C FILE type.
   void setFILEDecl(TypeDecl *FILEDecl) { this->FILEDecl = FILEDecl; }
 
@@ -950,26 +1006,39 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// purpose in characters.
   CharUnits getObjCEncodingTypeSize(QualType t) const;
 
-  /// \brief Whether __[u]int128_t identifier is installed.
-  bool isInt128Installed() const { return IsInt128Installed; }
-  void setInt128Installed() { IsInt128Installed = true; }
-
+  /// \brief Retrieve the typedef corresponding to the predefined 'id' type
+  /// in Objective-C.
+  TypedefDecl *getObjCIdDecl() const;
+  
   /// This setter/getter represents the ObjC 'id' type. It is setup lazily, by
   /// Sema.  id is always a (typedef for a) pointer type, a pointer to a struct.
-  QualType getObjCIdType() const { return ObjCIdTypedefType; }
-  void setObjCIdType(QualType T);
+  QualType getObjCIdType() const {
+    return getTypeDeclType(getObjCIdDecl());
+  }
 
-  void setObjCSelType(QualType T);
-  QualType getObjCSelType() const { return ObjCSelTypedefType; }
+  /// \brief Retrieve the typedef corresponding to the predefined 'SEL' type
+  /// in Objective-C.
+  TypedefDecl *getObjCSelDecl() const;
+  
+  /// \brief Retrieve the type that corresponds to the predefined Objective-C
+  /// 'SEL' type.
+  QualType getObjCSelType() const { 
+    return getTypeDeclType(getObjCSelDecl());
+  }
 
   void setObjCProtoType(QualType QT);
   QualType getObjCProtoType() const { return ObjCProtoType; }
 
+  /// \brief Retrieve the typedef declaration corresponding to the predefined
+  /// Objective-C 'Class' type.
+  TypedefDecl *getObjCClassDecl() const;
+  
   /// This setter/getter repreents the ObjC 'Class' type. It is setup lazily, by
   /// Sema.  'Class' is always a (typedef for a) pointer type, a pointer to a
   /// struct.
-  QualType getObjCClassType() const { return ObjCClassTypedefType; }
-  void setObjCClassType(QualType T);
+  QualType getObjCClassType() const { 
+    return getTypeDeclType(getObjCClassDecl());
+  }
 
   void setBuiltinVaListType(QualType T);
   QualType getBuiltinVaListType() const { return BuiltinVaListType; }
@@ -1145,7 +1214,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   const ASTRecordLayout &getASTObjCInterfaceLayout(const ObjCInterfaceDecl *D)
     const;
 
-  void DumpRecordLayout(const RecordDecl *RD, llvm::raw_ostream &OS) const;
+  void DumpRecordLayout(const RecordDecl *RD, raw_ostream &OS) const;
 
   /// getASTObjCImplementationLayout - Get or compute information about
   /// the layout of the specified Objective-C implementation. This may
@@ -1164,13 +1233,9 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   bool isNearlyEmpty(const CXXRecordDecl *RD) const;
 
   MangleContext *createMangleContext();
-
-  void ShallowCollectObjCIvars(const ObjCInterfaceDecl *OI,
-                               llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars)
-    const;
   
   void DeepCollectObjCIvars(const ObjCInterfaceDecl *OI, bool leafClass,
-                            llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) const;
+                            SmallVectorImpl<const ObjCIvarDecl*> &Ivars) const;
   
   unsigned CountNonClassIvars(const ObjCInterfaceDecl *OI) const;
   void CollectInheritedProtocols(const Decl *CDecl,
@@ -1261,7 +1326,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// \brief Retrieves the canonical representation of the given
   /// calling convention.
   CallingConv getCanonicalCallConv(CallingConv CC) const {
-    if (CC == CC_C)
+    if (!LangOpts.MRTD && CC == CC_C)
       return CC_Default;
     return CC;
   }
@@ -1400,7 +1465,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
     if (AS < LangAS::Offset || AS >= LangAS::Offset + LangAS::Count)
       return AS;
     else
-      return AddrSpaceMap[AS - LangAS::Offset];
+      return (*AddrSpaceMap)[AS - LangAS::Offset];
   }
 
 private:
@@ -1421,13 +1486,13 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   bool typesAreBlockPointerCompatible(QualType, QualType); 
 
   bool isObjCIdType(QualType T) const {
-    return T == ObjCIdTypedefType;
+    return T == getObjCIdType();
   }
   bool isObjCClassType(QualType T) const {
-    return T == ObjCClassTypedefType;
+    return T == getObjCClassType();
   }
   bool isObjCSelType(QualType T) const {
-    return T == ObjCSelTypedefType;
+    return T == getObjCSelType();
   }
   bool QualifiedIdConformsQualifiedId(QualType LHS, QualType RHS);
   bool ObjCQualifiedIdTypesAreCompatible(QualType LHS, QualType RHS,
@@ -1462,6 +1527,10 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
                                      bool Unqualified = false);
   
   QualType mergeObjCGCQualifiers(QualType, QualType);
+    
+  bool FunctionTypesMatchOnNSConsumedAttrs(
+         const FunctionProtoType *FromFunctionType,
+         const FunctionProtoType *ToFunctionType);
 
   void ResetObjCLayout(const ObjCContainerDecl *CD) {
     ObjCLayouts[CD] = 0;
@@ -1521,6 +1590,22 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// \brief Set the implementation of ObjCCategoryDecl.
   void setObjCImplementation(ObjCCategoryDecl *CatD,
                              ObjCCategoryImplDecl *ImplD);
+
+  /// \brief Get the duplicate declaration of a ObjCMethod in the same
+  /// interface, or null if non exists.
+  const ObjCMethodDecl *getObjCMethodRedeclaration(
+                                               const ObjCMethodDecl *MD) const {
+    llvm::DenseMap<const ObjCMethodDecl*, const ObjCMethodDecl*>::const_iterator
+      I = ObjCMethodRedecls.find(MD);
+    if (I == ObjCMethodRedecls.end())
+      return 0;
+    return I->second;
+  }
+
+  void setObjCMethodRedeclaration(const ObjCMethodDecl *MD,
+                                  const ObjCMethodDecl *Redecl) {
+    ObjCMethodRedecls[MD] = Redecl;
+  }
   
   /// \brief Set the copy inialization expression of a block var decl.
   void setBlockVarCopyInits(VarDecl*VD, Expr* Init);
@@ -1570,6 +1655,15 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// it is not used.
   bool DeclMustBeEmitted(const Decl *D);
 
+  
+  /// \brief Used by ParmVarDecl to store on the side the
+  /// index of the parameter when it exceeds the size of the normal bitfield.
+  void setParameterIndex(const ParmVarDecl *D, unsigned index);
+
+  /// \brief Used by ParmVarDecl to retrieve on the side the
+  /// index of the parameter when it exceeds the size of the normal bitfield.
+  unsigned getParameterIndex(const ParmVarDecl *D) const;
+  
   //===--------------------------------------------------------------------===//
   //                    Statistics
   //===--------------------------------------------------------------------===//
@@ -1620,7 +1714,18 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   ASTContext(const ASTContext&); // DO NOT IMPLEMENT
   void operator=(const ASTContext&); // DO NOT IMPLEMENT
 
-  void InitBuiltinTypes();
+public:
+  /// \brief Initialize built-in types.
+  ///
+  /// This routine may only be invoked once for a given ASTContext object.
+  /// It is normally invoked by the ASTContext constructor. However, the
+  /// constructor can be asked to delay initialization, which places the burden
+  /// of calling this function on the user of that object.
+  ///
+  /// \param Target The target 
+  void InitBuiltinTypes(const TargetInfo &Target);
+  
+private:
   void InitBuiltinType(CanQualType &R, BuiltinType::Kind K);
 
   // Return the ObjC type encoding for a given type.
@@ -1644,7 +1749,7 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
 private:
   /// \brief A set of deallocations that should be performed when the 
   /// ASTContext is destroyed.
-  llvm::SmallVector<std::pair<void (*)(void*), void *>, 16> Deallocations;
+  SmallVector<std::pair<void (*)(void*), void *>, 16> Deallocations;
                                        
   // FIXME: This currently contains the set of StoredDeclMaps used
   // by DeclContext objects.  This probably should not be in ASTContext,
@@ -1660,13 +1765,13 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
 };
   
 /// @brief Utility function for constructing a nullary selector.
-static inline Selector GetNullarySelector(llvm::StringRef name, ASTContext& Ctx) {
+static inline Selector GetNullarySelector(StringRef name, ASTContext& Ctx) {
   IdentifierInfo* II = &Ctx.Idents.get(name);
   return Ctx.Selectors.getSelector(0, &II);
 }
 
 /// @brief Utility function for constructing an unary selector.
-static inline Selector GetUnarySelector(llvm::StringRef name, ASTContext& Ctx) {
+static inline Selector GetUnarySelector(StringRef name, ASTContext& Ctx) {
   IdentifierInfo* II = &Ctx.Idents.get(name);
   return Ctx.Selectors.getSelector(1, &II);
 }
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ASTDiagnostic.h b/contrib/llvm/tools/clang/include/clang/AST/ASTDiagnostic.h
index 70a548d4e965..b0057113cf8e 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ASTDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ASTDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define ASTSTART
 #include "clang/Basic/DiagnosticASTKinds.inc"
 #undef DIAG
@@ -24,27 +24,27 @@ namespace clang {
     };
   }  // end namespace diag
   
-  /// \brief Diagnostic argument formatting function for diagnostics that
+  /// \brief DiagnosticsEngine argument formatting function for diagnostics that
   /// involve AST nodes.
   ///
   /// This function formats diagnostic arguments for various AST nodes, 
   /// including types, declaration names, nested name specifiers, and
   /// declaration contexts, into strings that can be printed as part of
   /// diagnostics. It is meant to be used as the argument to
-  /// \c Diagnostic::SetArgToStringFn(), where the cookie is an \c ASTContext
-  /// pointer.
+  /// \c DiagnosticsEngine::SetArgToStringFn(), where the cookie is an \c
+  /// ASTContext pointer.
   void FormatASTNodeDiagnosticArgument(
-      Diagnostic::ArgumentKind Kind,
+      DiagnosticsEngine::ArgumentKind Kind,
       intptr_t Val,
       const char *Modifier,
       unsigned ModLen,
       const char *Argument,
       unsigned ArgLen,
-      const Diagnostic::ArgumentValue *PrevArgs,
+      const DiagnosticsEngine::ArgumentValue *PrevArgs,
       unsigned NumPrevArgs,
-      llvm::SmallVectorImpl<char> &Output,
+      SmallVectorImpl<char> &Output,
       void *Cookie,
-      llvm::SmallVectorImpl<intptr_t> &QualTypeVals);
+      SmallVectorImpl<intptr_t> &QualTypeVals);
 }  // end namespace clang
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ASTImporter.h b/contrib/llvm/tools/clang/include/clang/AST/ASTImporter.h
index e1535965b1e4..b583fbf9061b 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ASTImporter.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ASTImporter.h
@@ -25,7 +25,7 @@ namespace clang {
   class ASTContext;
   class Decl;
   class DeclContext;
-  class Diagnostic;
+  class DiagnosticsEngine;
   class Expr;
   class FileManager;
   class IdentifierInfo;
@@ -67,7 +67,7 @@ namespace clang {
     
     /// \brief Imported, anonymous tag declarations that are missing their 
     /// corresponding typedefs.
-    llvm::SmallVector<TagDecl *, 4> AnonTagsWithPendingTypedefs;
+    SmallVector<TagDecl *, 4> AnonTagsWithPendingTypedefs;
     
     /// \brief Declaration (from, to) pairs that are known not to be equivalent
     /// (which we have already complained about).
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ASTMutationListener.h b/contrib/llvm/tools/clang/include/clang/AST/ASTMutationListener.h
index 470cca8ee76e..793d3ee2b1fb 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ASTMutationListener.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ASTMutationListener.h
@@ -22,6 +22,8 @@ namespace clang {
   class ClassTemplateSpecializationDecl;
   class FunctionDecl;
   class FunctionTemplateDecl;
+  class ObjCCategoryDecl;
+  class ObjCInterfaceDecl;
 
 /// \brief An abstract interface that should be implemented by listeners
 /// that want to be notified when an AST entity gets modified after its
@@ -54,6 +56,10 @@ class ASTMutationListener {
 
   /// \brief A static data member was implicitly instantiated.
   virtual void StaticDataMemberInstantiated(const VarDecl *D) {}
+
+  /// \brief A new objc category class was added for an interface.
+  virtual void AddedObjCCategoryToInterface(const ObjCCategoryDecl *CatD,
+                                            const ObjCInterfaceDecl *IFD) {}
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/AST/Attr.h b/contrib/llvm/tools/clang/include/clang/AST/Attr.h
index 719023926bae..cf2e3c5d25b6 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/Attr.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/Attr.h
@@ -14,18 +14,18 @@
 #ifndef LLVM_CLANG_AST_ATTR_H
 #define LLVM_CLANG_AST_ATTR_H
 
-#include "llvm/Support/Casting.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/AttrKinds.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/VersionTuple.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <cstring>
 #include <algorithm>
-using llvm::dyn_cast;
 
 namespace clang {
   class ASTContext;
@@ -58,7 +58,7 @@ namespace clang {
 /// Attr - This represents one attribute.
 class Attr {
 private:
-  SourceLocation Loc;
+  SourceRange Range;
   unsigned AttrKind : 16;
 
 protected:
@@ -67,11 +67,10 @@ class Attr {
   virtual ~Attr();
   
   void* operator new(size_t bytes) throw() {
-    assert(0 && "Attrs cannot be allocated with regular 'new'.");
-    return 0;
+    llvm_unreachable("Attrs cannot be allocated with regular 'new'.");
   }
   void operator delete(void* data) throw() {
-    assert(0 && "Attrs cannot be released with regular 'delete'.");
+    llvm_unreachable("Attrs cannot be released with regular 'delete'.");
   }
 
 public:
@@ -86,8 +85,8 @@ class Attr {
   }
 
 protected:
-  Attr(attr::Kind AK, SourceLocation L)
-    : Loc(L), AttrKind(AK), Inherited(false) {}
+  Attr(attr::Kind AK, SourceRange R)
+    : Range(R), AttrKind(AK), Inherited(false) {}
 
 public:
 
@@ -95,8 +94,9 @@ class Attr {
     return static_cast<attr::Kind>(AttrKind);
   }
 
-  SourceLocation getLocation() const { return Loc; }
-  void setLocation(SourceLocation L) { Loc = L; }
+  SourceLocation getLocation() const { return Range.getBegin(); }
+  SourceRange getRange() const { return Range; }
+  void setRange(SourceRange R) { Range = R; }
 
   bool isInherited() const { return Inherited; }
 
@@ -109,8 +109,8 @@ class Attr {
 
 class InheritableAttr : public Attr {
 protected:
-  InheritableAttr(attr::Kind AK, SourceLocation L)
-    : Attr(AK, L) {}
+  InheritableAttr(attr::Kind AK, SourceRange R)
+    : Attr(AK, R) {}
 
 public:
   void setInherited(bool I) { Inherited = I; }
@@ -124,8 +124,8 @@ class InheritableAttr : public Attr {
 
 class InheritableParamAttr : public InheritableAttr {
 protected:
-  InheritableParamAttr(attr::Kind AK, SourceLocation L)
-    : InheritableAttr(AK, L) {}
+  InheritableParamAttr(attr::Kind AK, SourceRange R)
+    : InheritableAttr(AK, R) {}
 
 public:
   // Implement isa/cast/dyncast/etc.
@@ -138,8 +138,8 @@ class InheritableParamAttr : public InheritableAttr {
 #include "clang/AST/Attrs.inc"
 
 /// AttrVec - A vector of Attr, which is how they are stored on the AST.
-typedef llvm::SmallVector<Attr*, 2> AttrVec;
-typedef llvm::SmallVector<const Attr*, 2> ConstAttrVec;
+typedef SmallVector<Attr*, 2> AttrVec;
+typedef SmallVector<const Attr*, 2> ConstAttrVec;
 
 /// DestroyAttrs - Destroy the contents of an AttrVec.
 inline void DestroyAttrs (AttrVec& V, ASTContext &C) {
@@ -159,12 +159,12 @@ class specific_attr_iterator {
   mutable AttrVec::const_iterator Current;
 
   void AdvanceToNext() const {
-    while (!llvm::isa<SpecificAttr>(*Current))
+    while (!isa<SpecificAttr>(*Current))
       ++Current;
   }
 
   void AdvanceToNext(AttrVec::const_iterator I) const {
-    while (Current != I && !llvm::isa<SpecificAttr>(*Current))
+    while (Current != I && !isa<SpecificAttr>(*Current))
       ++Current;
   }
 
@@ -180,11 +180,11 @@ class specific_attr_iterator {
 
   reference operator*() const {
     AdvanceToNext();
-    return llvm::cast<SpecificAttr>(*Current);
+    return cast<SpecificAttr>(*Current);
   }
   pointer operator->() const {
     AdvanceToNext();
-    return llvm::cast<SpecificAttr>(*Current);
+    return cast<SpecificAttr>(*Current);
   }
 
   specific_attr_iterator& operator++() {
diff --git a/contrib/llvm/tools/clang/include/clang/AST/BaseSubobject.h b/contrib/llvm/tools/clang/include/clang/AST/BaseSubobject.h
new file mode 100644
index 000000000000..6a036bb62cf4
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/AST/BaseSubobject.h
@@ -0,0 +1,87 @@
+//===--- BaseSubobject.h - BaseSubobject class ----------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides a definition of the BaseSubobject class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_BASESUBOBJECT_H
+#define LLVM_CLANG_AST_BASESUBOBJECT_H
+
+#include "clang/AST/CharUnits.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/type_traits.h"
+
+namespace clang {
+  class CXXRecordDecl;
+
+// BaseSubobject - Uniquely identifies a direct or indirect base class. 
+// Stores both the base class decl and the offset from the most derived class to
+// the base class. Used for vtable and VTT generation.
+class BaseSubobject {
+  /// Base - The base class declaration.
+  const CXXRecordDecl *Base;
+  
+  /// BaseOffset - The offset from the most derived class to the base class.
+  CharUnits BaseOffset;
+  
+public:
+  BaseSubobject() { }
+  BaseSubobject(const CXXRecordDecl *Base, CharUnits BaseOffset)
+    : Base(Base), BaseOffset(BaseOffset) { }
+  
+  /// getBase - Returns the base class declaration.
+  const CXXRecordDecl *getBase() const { return Base; }
+
+  /// getBaseOffset - Returns the base class offset.
+  CharUnits getBaseOffset() const { return BaseOffset; }
+
+  friend bool operator==(const BaseSubobject &LHS, const BaseSubobject &RHS) {
+    return LHS.Base == RHS.Base && LHS.BaseOffset == RHS.BaseOffset;
+ }
+};
+
+} // end namespace clang
+
+namespace llvm {
+
+template<> struct DenseMapInfo<clang::BaseSubobject> {
+  static clang::BaseSubobject getEmptyKey() {
+    return clang::BaseSubobject(
+      DenseMapInfo<const clang::CXXRecordDecl *>::getEmptyKey(),
+      clang::CharUnits::fromQuantity(DenseMapInfo<int64_t>::getEmptyKey()));
+  }
+
+  static clang::BaseSubobject getTombstoneKey() {
+    return clang::BaseSubobject(
+      DenseMapInfo<const clang::CXXRecordDecl *>::getTombstoneKey(),
+      clang::CharUnits::fromQuantity(DenseMapInfo<int64_t>::getTombstoneKey()));
+  }
+
+  static unsigned getHashValue(const clang::BaseSubobject &Base) {
+    return 
+      DenseMapInfo<const clang::CXXRecordDecl *>::getHashValue(Base.getBase()) ^
+      DenseMapInfo<int64_t>::getHashValue(Base.getBaseOffset().getQuantity());
+  }
+
+  static bool isEqual(const clang::BaseSubobject &LHS, 
+                      const clang::BaseSubobject &RHS) {
+    return LHS == RHS;
+  }
+};
+
+// It's OK to treat BaseSubobject as a POD type.
+template <> struct isPodLike<clang::BaseSubobject> {
+  static const bool value = true;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/CXXInheritance.h b/contrib/llvm/tools/clang/include/clang/AST/CXXInheritance.h
index d712e7d0c751..44c554b606c4 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/CXXInheritance.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/CXXInheritance.h
@@ -66,7 +66,7 @@ struct CXXBasePathElement {
 /// structure, which captures both the link from a derived class to one of its
 /// direct bases and identification describing which base class
 /// subobject is being used.
-class CXXBasePath : public llvm::SmallVector<CXXBasePathElement, 4> {
+class CXXBasePath : public SmallVector<CXXBasePathElement, 4> {
 public:
   CXXBasePath() : Access(AS_public) {}
 
@@ -80,7 +80,7 @@ class CXXBasePath : public llvm::SmallVector<CXXBasePathElement, 4> {
   DeclContext::lookup_result Decls;
 
   void clear() {
-    llvm::SmallVectorImpl<CXXBasePathElement>::clear();
+    SmallVectorImpl<CXXBasePathElement>::clear();
     Access = AS_public;
   }
 };
@@ -272,14 +272,14 @@ struct UniqueVirtualMethod {
 /// pair is the virtual method that overrides it (including the
 /// subobject in which that virtual function occurs).
 class OverridingMethods {
-  llvm::DenseMap<unsigned, llvm::SmallVector<UniqueVirtualMethod, 4> > 
+  llvm::DenseMap<unsigned, SmallVector<UniqueVirtualMethod, 4> > 
     Overrides;
 
 public:
   // Iterate over the set of subobjects that have overriding methods.
-  typedef llvm::DenseMap<unsigned, llvm::SmallVector<UniqueVirtualMethod, 4> >
+  typedef llvm::DenseMap<unsigned, SmallVector<UniqueVirtualMethod, 4> >
             ::iterator iterator;
-  typedef llvm::DenseMap<unsigned, llvm::SmallVector<UniqueVirtualMethod, 4> >
+  typedef llvm::DenseMap<unsigned, SmallVector<UniqueVirtualMethod, 4> >
             ::const_iterator const_iterator;
   iterator begin() { return Overrides.begin(); }
   const_iterator begin() const { return Overrides.begin(); }
@@ -289,9 +289,9 @@ class OverridingMethods {
 
   // Iterate over the set of overriding virtual methods in a given
   // subobject.
-  typedef llvm::SmallVector<UniqueVirtualMethod, 4>::iterator 
+  typedef SmallVector<UniqueVirtualMethod, 4>::iterator 
     overriding_iterator;
-  typedef llvm::SmallVector<UniqueVirtualMethod, 4>::const_iterator
+  typedef SmallVector<UniqueVirtualMethod, 4>::const_iterator
     overriding_const_iterator;
 
   // Add a new overriding method for a particular subobject.
diff --git a/contrib/llvm/tools/clang/include/clang/AST/CharUnits.h b/contrib/llvm/tools/clang/include/clang/AST/CharUnits.h
index d7cbd08e6c2e..5be35826a964 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/CharUnits.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/CharUnits.h
@@ -125,6 +125,12 @@ namespace clang {
       /// isNegative - Test whether the quantity is less than zero.
       bool isNegative() const { return Quantity  < 0; }
 
+      /// isPowerOfTwo - Test whether the quantity is a power of two.
+      /// Zero is not a power of two.
+      bool isPowerOfTwo() const {
+        return (Quantity & -Quantity) == Quantity;
+      }
+
       // Arithmetic operators.
       CharUnits operator* (QuantityType N) const {
         return CharUnits(Quantity * N);
diff --git a/contrib/llvm/tools/clang/include/clang/AST/Decl.h b/contrib/llvm/tools/clang/include/clang/AST/Decl.h
index 5691e99e7297..a02a2ce3369f 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/Decl.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/Decl.h
@@ -20,6 +20,7 @@
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/Basic/Linkage.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
 
 namespace clang {
@@ -32,6 +33,7 @@ class StringLiteral;
 class NestedNameSpecifier;
 class TemplateParameterList;
 class TemplateArgumentList;
+struct ASTTemplateArgumentListInfo;
 class MemberSpecializationInfo;
 class FunctionTemplateSpecializationInfo;
 class DependentFunctionTemplateSpecializationInfo;
@@ -115,7 +117,7 @@ class NamedDecl : public Decl {
   /// getName - Get the name of identifier for this declaration as a StringRef.
   /// This requires that the declaration have a name and that it be a simple
   /// identifier.
-  llvm::StringRef getName() const {
+  StringRef getName() const {
     assert(Name.isIdentifier() && "Name is not a simple identifier");
     return getIdentifier() ? getIdentifier()->getName() : "";
   }
@@ -132,7 +134,7 @@ class NamedDecl : public Decl {
   // FIXME: Deprecated, move clients to getName().
   std::string getNameAsString() const { return Name.getAsString(); }
 
-  void printName(llvm::raw_ostream &os) const { return Name.printName(os); }
+  void printName(raw_ostream &os) const { return Name.printName(os); }
 
   /// getDeclName - Get the actual, stored name of the declaration,
   /// which may be a special name.
@@ -180,6 +182,16 @@ class NamedDecl : public Decl {
   /// \brief Determine whether this declaration has linkage.
   bool hasLinkage() const;
 
+  /// \brief Whether this declaration was marked as being private to the
+  /// module in which it was defined.
+  bool isModulePrivate() const { return ModulePrivate; }
+  
+  /// \brief Specify whether this declaration was marked as being private
+  /// to the module in which it was defined.
+  void setModulePrivate(bool MP = true) {
+    ModulePrivate = MP;
+  }
+  
   /// \brief Determine whether this declaration is a C++ class member.
   bool isCXXClassMember() const {
     const DeclContext *DC = getDeclContext();
@@ -294,9 +306,8 @@ class NamedDecl : public Decl {
   static bool classofKind(Kind K) { return K >= firstNamed && K <= lastNamed; }
 };
 
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
-                                     const NamedDecl *ND) {
-  ND->getDeclName().printName(OS);
+inline raw_ostream &operator<<(raw_ostream &OS, const NamedDecl &ND) {
+  ND.printName(OS);
   return OS;
 }
 
@@ -706,13 +717,18 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
     /// \brief Whether this variable is an ARC pseudo-__strong
     /// variable;  see isARCPseudoStrong() for details.
     unsigned ARCPseudoStrong : 1;
+
+    /// \brief Whether this variable is (C++0x) constexpr.
+    unsigned IsConstexpr : 1;
   };
-  enum { NumVarDeclBits = 13 }; // one reserved bit
+  enum { NumVarDeclBits = 13 };
 
   friend class ASTDeclReader;
   friend class StmtIteratorBase;
   
 protected:
+  enum { NumParameterIndexBits = 8 };
+  
   class ParmVarDeclBitfields {
     friend class ParmVarDecl;
     friend class ASTDeclReader;
@@ -737,7 +753,7 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
 
     /// The number of parameters preceding this parameter in the
     /// function parameter scope in which it was declared.
-    unsigned ParameterIndex : 8;
+    unsigned ParameterIndex : NumParameterIndexBits;
   };
 
   union {
@@ -802,7 +818,7 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
       return !isFileVarDecl();
 
     // Return true for:  Auto, Register.
-    // Return false for: Extern, Static, PrivateExtern.
+    // Return false for: Extern, Static, PrivateExtern, OpenCLWorkGroupLocal.
 
     return getStorageClass() >= SC_Auto;
   }
@@ -1128,6 +1144,10 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
   bool isARCPseudoStrong() const { return VarDeclBits.ARCPseudoStrong; }
   void setARCPseudoStrong(bool ps) { VarDeclBits.ARCPseudoStrong = ps; }
   
+  /// Whether this variable is (C++0x) constexpr.
+  bool isConstexpr() const { return VarDeclBits.IsConstexpr; }
+  void setConstexpr(bool IC) { VarDeclBits.IsConstexpr = IC; }
+
   /// \brief If this variable is an instantiated static data member of a
   /// class template specialization, returns the templated static data member
   /// from which it was instantiated.
@@ -1198,11 +1218,11 @@ class ParmVarDecl : public VarDecl {
                              StorageClass S, StorageClass SCAsWritten,
                              Expr *DefArg);
 
+  virtual SourceRange getSourceRange() const;
+  
   void setObjCMethodScopeInfo(unsigned parameterIndex) {
     ParmVarDeclBits.IsObjCMethodParam = true;
-
-    ParmVarDeclBits.ParameterIndex = parameterIndex;
-    assert(ParmVarDeclBits.ParameterIndex == parameterIndex && "truncation!");
+    setParameterIndex(parameterIndex);
   }
 
   void setScopeInfo(unsigned scopeDepth, unsigned parameterIndex) {
@@ -1211,8 +1231,7 @@ class ParmVarDecl : public VarDecl {
     ParmVarDeclBits.ScopeDepthOrObjCQuals = scopeDepth;
     assert(ParmVarDeclBits.ScopeDepthOrObjCQuals == scopeDepth && "truncation!");
 
-    ParmVarDeclBits.ParameterIndex = parameterIndex;
-    assert(ParmVarDeclBits.ParameterIndex == parameterIndex && "truncation!");
+    setParameterIndex(parameterIndex);
   }
 
   bool isObjCMethodParameter() const {
@@ -1226,7 +1245,7 @@ class ParmVarDecl : public VarDecl {
 
   /// Returns the index of this parameter in its prototype or method scope.
   unsigned getFunctionScopeIndex() const {
-    return ParmVarDeclBits.ParameterIndex;
+    return getParameterIndex();
   }
 
   ObjCDeclQualifier getObjCDeclQualifier() const {
@@ -1343,6 +1362,26 @@ class ParmVarDecl : public VarDecl {
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classof(const ParmVarDecl *D) { return true; }
   static bool classofKind(Kind K) { return K == ParmVar; }
+  
+private:
+  enum { ParameterIndexSentinel = (1 << NumParameterIndexBits) - 1 };
+
+  void setParameterIndex(unsigned parameterIndex) {
+    if (parameterIndex >= ParameterIndexSentinel) {
+      setParameterIndexLarge(parameterIndex);
+      return;
+    }
+    
+    ParmVarDeclBits.ParameterIndex = parameterIndex;
+    assert(ParmVarDeclBits.ParameterIndex == parameterIndex && "truncation!");
+  }
+  unsigned getParameterIndex() const {
+    unsigned d = ParmVarDeclBits.ParameterIndex;
+    return d == ParameterIndexSentinel ? getParameterIndexLarge() : d;
+  }
+  
+  void setParameterIndexLarge(unsigned parameterIndex);
+  unsigned getParameterIndexLarge() const;
 };
 
 /// FunctionDecl - An instance of this class is created to represent a
@@ -1394,6 +1433,7 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   bool IsExplicitlyDefaulted : 1; //sunk from CXXMethodDecl
   bool HasImplicitReturnZero : 1;
   bool IsLateTemplateParsed : 1;
+  bool IsConstexpr : 1;
 
   /// \brief End part of this FunctionDecl's source range.
   ///
@@ -1460,13 +1500,14 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   void setInstantiationOfMemberFunction(ASTContext &C, FunctionDecl *FD,
                                         TemplateSpecializationKind TSK);
 
-  void setParams(ASTContext &C, ParmVarDecl **NewParamInfo, unsigned NumParams);
+  void setParams(ASTContext &C, llvm::ArrayRef<ParmVarDecl *> NewParamInfo);
 
 protected:
   FunctionDecl(Kind DK, DeclContext *DC, SourceLocation StartLoc,
                const DeclarationNameInfo &NameInfo,
                QualType T, TypeSourceInfo *TInfo,
-               StorageClass S, StorageClass SCAsWritten, bool isInlineSpecified)
+               StorageClass S, StorageClass SCAsWritten, bool isInlineSpecified,
+               bool isConstexprSpecified)
     : DeclaratorDecl(DK, DC, NameInfo.getLoc(), NameInfo.getName(), T, TInfo,
                      StartLoc),
       DeclContext(DK),
@@ -1477,7 +1518,7 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
       HasWrittenPrototype(true), IsDeleted(false), IsTrivial(false),
       IsDefaulted(false), IsExplicitlyDefaulted(false),
       HasImplicitReturnZero(false), IsLateTemplateParsed(false),
-      EndRangeLoc(NameInfo.getEndLoc()),
+      IsConstexpr(isConstexprSpecified), EndRangeLoc(NameInfo.getEndLoc()),
       TemplateOrSpecialization(),
       DNLoc(NameInfo.getInfo()) {}
 
@@ -1500,11 +1541,13 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
                               StorageClass SC = SC_None,
                               StorageClass SCAsWritten = SC_None,
                               bool isInlineSpecified = false,
-                              bool hasWrittenPrototype = true) {
+                              bool hasWrittenPrototype = true,
+                              bool isConstexprSpecified = false) {
     DeclarationNameInfo NameInfo(N, NLoc);
     return FunctionDecl::Create(C, DC, StartLoc, NameInfo, T, TInfo,
                                 SC, SCAsWritten,
-                                isInlineSpecified, hasWrittenPrototype);
+                                isInlineSpecified, hasWrittenPrototype,
+                                isConstexprSpecified);
   }
 
   static FunctionDecl *Create(ASTContext &C, DeclContext *DC,
@@ -1514,7 +1557,8 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
                               StorageClass SC = SC_None,
                               StorageClass SCAsWritten = SC_None,
                               bool isInlineSpecified = false,
-                              bool hasWrittenPrototype = true);
+                              bool hasWrittenPrototype = true,
+                              bool isConstexprSpecified = false);
 
   DeclarationNameInfo getNameInfo() const {
     return DeclarationNameInfo(getDeclName(), getLocation(), DNLoc);
@@ -1600,10 +1644,6 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   bool isPure() const { return IsPure; }
   void setPure(bool P = true);
 
-  /// Whether this is a constexpr function or constexpr constructor.
-  // FIXME: C++0x: Implement tracking of the constexpr specifier.
-  bool isConstExpr() const { return false; }
-
   /// Whether this templated function will be late parsed.
   bool isLateTemplateParsed() const { return IsLateTemplateParsed; }
   void setLateTemplateParsed(bool ILT = true) { IsLateTemplateParsed = ILT; }
@@ -1646,6 +1686,10 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   bool hasInheritedPrototype() const { return HasInheritedPrototype; }
   void setHasInheritedPrototype(bool P = true) { HasInheritedPrototype = P; }
 
+  /// Whether this is a (C++0x) constexpr function or constexpr constructor.
+  bool isConstexpr() const { return IsConstexpr; }
+  void setConstexpr(bool IC) { IsConstexpr = IC; }
+
   /// \brief Whether this function has been deleted.
   ///
   /// A function that is "deleted" (via the C++0x "= delete" syntax)
@@ -1726,8 +1770,8 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
     assert(i < getNumParams() && "Illegal param #");
     return ParamInfo[i];
   }
-  void setParams(ParmVarDecl **NewParamInfo, unsigned NumParams) {
-    setParams(getASTContext(), NewParamInfo, NumParams);
+  void setParams(llvm::ArrayRef<ParmVarDecl *> NewParamInfo) {
+    setParams(getASTContext(), NewParamInfo);
   }
 
   /// getMinRequiredArguments - Returns the minimum number of arguments
@@ -1768,11 +1812,13 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   }
 
   /// \brief Determine whether this function should be inlined, because it is
-  /// either marked "inline" or is a member function of a C++ class that
-  /// was defined in the class body.
+  /// either marked "inline" or "constexpr" or is a member function of a class
+  /// that was defined in the class body.
   bool isInlined() const;
 
   bool isInlineDefinitionExternallyVisible() const;
+
+  bool doesDeclarationForceExternallyVisibleDefinition() const;
                        
   /// isOverloadedOperator - Whether this function declaration
   /// represents an C++ overloaded operator, e.g., "operator+".
@@ -1847,7 +1893,11 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   bool isFunctionTemplateSpecialization() const {
     return getPrimaryTemplate() != 0;
   }
-       
+
+  /// \brief Retrieve the class scope template pattern that this function
+  ///  template specialization is instantiated from.
+  FunctionDecl *getClassScopeSpecializationPattern() const;
+
   /// \brief If this function is actually a function template specialization,
   /// retrieve information about this function template specialization. 
   /// Otherwise, returns NULL.
@@ -1887,7 +1937,7 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   /// or if it had no explicit template argument list, returns NULL.
   /// Note that it an explicit template argument list may be written empty,
   /// e.g., template<> void foo<>(char* s);
-  const TemplateArgumentListInfo*
+  const ASTTemplateArgumentListInfo*
   getTemplateSpecializationArgsAsWritten() const;
 
   /// \brief Specify that this function declaration is actually a function
@@ -1945,7 +1995,7 @@ class FunctionDecl : public DeclaratorDecl, public DeclContext,
   /// specialization or a member of a class template specialization.
   ///
   /// \returns the first point of instantiation, if this function was 
-  /// instantiated from a template; otherwie, returns an invalid source 
+  /// instantiated from a template; otherwise, returns an invalid source 
   /// location.
   SourceLocation getPointOfInstantiation() const;
                        
@@ -2033,6 +2083,7 @@ class FieldDecl : public DeclaratorDecl {
   Expr *getBitWidth() const {
     return isBitField() ? InitializerOrBitWidth.getPointer() : 0;
   }
+  unsigned getBitWidthValue(const ASTContext &Ctx) const;
   void setBitWidth(Expr *BW) {
     assert(!InitializerOrBitWidth.getPointer() &&
            "bit width or initializer already set");
@@ -2308,9 +2359,10 @@ class TagDecl
   /// TagDeclKind - The TagKind enum.
   unsigned TagDeclKind : 2;
 
-  /// IsDefinition - True if this is a definition ("struct foo {};"), false if
-  /// it is a declaration ("struct foo;").
-  bool IsDefinition : 1;
+  /// IsCompleteDefinition - True if this is a definition ("struct foo
+  /// {};"), false if it is a declaration ("struct foo;").  It is not
+  /// a definition until the definition has been fully processed.
+  bool IsCompleteDefinition : 1;
 
   /// IsBeingDefined - True if this is currently being defined.
   bool IsBeingDefined : 1;
@@ -2320,6 +2372,9 @@ class TagDecl
   /// in the syntax of a declarator.
   bool IsEmbeddedInDeclarator : 1;
 
+  /// /brief True if this tag is free standing, e.g. "struct foo;".
+  bool IsFreeStanding : 1;
+
 protected:
   // These are used by (and only defined for) EnumDecl.
   unsigned NumPositiveBits : 8;
@@ -2367,9 +2422,10 @@ class TagDecl
     assert((DK != Enum || TK == TTK_Enum) &&
            "EnumDecl not matched with TTK_Enum");
     TagDeclKind = TK;
-    IsDefinition = false;
+    IsCompleteDefinition = false;
     IsBeingDefined = false;
     IsEmbeddedInDeclarator = false;
+    IsFreeStanding = false;
     setPreviousDeclaration(PrevDecl);
   }
 
@@ -2408,14 +2464,15 @@ class TagDecl
   }
 
   /// isThisDeclarationADefinition() - Return true if this declaration
-  /// defines the type.  Provided for consistency.
+  /// is a completion definintion of the type.  Provided for consistency.
   bool isThisDeclarationADefinition() const {
-    return isDefinition();
+    return isCompleteDefinition();
   }
 
-  /// isDefinition - Return true if this decl has its body specified.
-  bool isDefinition() const {
-    return IsDefinition;
+  /// isCompleteDefinition - Return true if this decl has its body
+  /// fully specified.
+  bool isCompleteDefinition() const {
+    return IsCompleteDefinition;
   }
 
   /// isBeingDefined - Return true if this decl is currently being defined.
@@ -2430,6 +2487,11 @@ class TagDecl
     IsEmbeddedInDeclarator = isInDeclarator;
   }
 
+  bool isFreeStanding() const { return IsFreeStanding; }
+  void setFreeStanding(bool isFreeStanding = true) {
+    IsFreeStanding = isFreeStanding;
+  }
+
   /// \brief Whether this declaration declares a type that is
   /// dependent, i.e., a type that somehow depends on template
   /// parameters.
@@ -2444,14 +2506,15 @@ class TagDecl
 
   /// getDefinition - Returns the TagDecl that actually defines this
   ///  struct/union/class/enum.  When determining whether or not a
-  ///  struct/union/class/enum is completely defined, one should use this method
-  ///  as opposed to 'isDefinition'.  'isDefinition' indicates whether or not a
-  ///  specific TagDecl is defining declaration, not whether or not the
-  ///  struct/union/class/enum type is defined.  This method returns NULL if
-  ///  there is no TagDecl that defines the struct/union/class/enum.
-  TagDecl* getDefinition() const;
+  ///  struct/union/class/enum has a definition, one should use this
+  ///  method as opposed to 'isDefinition'.  'isDefinition' indicates
+  ///  whether or not a specific TagDecl is defining declaration, not
+  ///  whether or not the struct/union/class/enum type is defined.
+  ///  This method returns NULL if there is no TagDecl that defines
+  ///  the struct/union/class/enum.
+  TagDecl *getDefinition() const;
 
-  void setDefinition(bool V) { IsDefinition = V; }
+  void setCompleteDefinition(bool V) { IsCompleteDefinition = V; }
 
   const char *getKindName() const {
     return TypeWithKeyword::getTagTypeKindName(getTagKind());
@@ -2691,7 +2754,7 @@ class EnumDecl : public TagDecl {
 
   /// \brief Returns true if this can be considered a complete type.
   bool isComplete() const {
-    return isDefinition() || isFixed();
+    return isCompleteDefinition() || isFixed();
   }
 
   /// \brief Returns the enumeration (declared within the template)
@@ -2794,14 +2857,15 @@ class RecordDecl : public TagDecl {
   /// \endcode
   bool isInjectedClassName() const;
 
-  /// getDefinition - Returns the RecordDecl that actually defines this
-  ///  struct/union/class.  When determining whether or not a struct/union/class
-  ///  is completely defined, one should use this method as opposed to
-  ///  'isDefinition'.  'isDefinition' indicates whether or not a specific
-  ///  RecordDecl is defining declaration, not whether or not the record
-  ///  type is defined.  This method returns NULL if there is no RecordDecl
-  ///  that defines the struct/union/tag.
-  RecordDecl* getDefinition() const {
+  /// getDefinition - Returns the RecordDecl that actually defines
+  ///  this struct/union/class.  When determining whether or not a
+  ///  struct/union/class is completely defined, one should use this
+  ///  method as opposed to 'isCompleteDefinition'.
+  ///  'isCompleteDefinition' indicates whether or not a specific
+  ///  RecordDecl is a completed definition, not whether or not the
+  ///  record type is defined.  This method returns NULL if there is
+  ///  no RecordDecl that defines the struct/union/tag.
+  RecordDecl *getDefinition() const {
     return cast_or_null<RecordDecl>(TagDecl::getDefinition());
   }
 
@@ -2967,7 +3031,7 @@ class BlockDecl : public Decl, public DeclContext {
     assert(i < getNumParams() && "Illegal param #");
     return ParamInfo[i];
   }
-  void setParams(ParmVarDecl **NewParamInfo, unsigned NumParams);
+  void setParams(llvm::ArrayRef<ParmVarDecl *> NewParamInfo);
 
   /// hasCaptures - True if this block (or its nested blocks) captures
   /// anything of local storage from its enclosing scopes.
@@ -3010,8 +3074,9 @@ class BlockDecl : public Decl, public DeclContext {
 /// Insertion operator for diagnostics.  This allows sending NamedDecl's
 /// into a diagnostic with <<.
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
-                                           NamedDecl* ND) {
-  DB.AddTaggedVal(reinterpret_cast<intptr_t>(ND), Diagnostic::ak_nameddecl);
+                                           const NamedDecl* ND) {
+  DB.AddTaggedVal(reinterpret_cast<intptr_t>(ND),
+                  DiagnosticsEngine::ak_nameddecl);
   return DB;
 }
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h b/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h
index 8b2ef2a81acb..9f294117682c 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h
@@ -243,19 +243,23 @@ class Decl {
   /// evaluated context or not, e.g. functions used in uninstantiated templates
   /// are regarded as "referenced" but not "used".
   unsigned Referenced : 1;
-
+  
 protected:
   /// Access - Used by C++ decls for the access specifier.
   // NOTE: VC++ treats enums as signed, avoid using the AccessSpecifier enum
   unsigned Access : 2;
   friend class CXXClassMemberWrapper;
 
-  /// PCHLevel - the "level" of AST file from which this declaration was built.
-  unsigned PCHLevel : 2;
-
+  /// \brief Whether this declaration was loaded from an AST file.
+  unsigned FromASTFile : 1;
+  
   /// ChangedAfterLoad - if this declaration has changed since being loaded
   unsigned ChangedAfterLoad : 1;
 
+  /// \brief Whether this declaration is private to the module in which it was
+  /// defined.
+  unsigned ModulePrivate : 1;
+
   /// IdentifierNamespace - This specifies what IDNS_* namespace this lives in.
   unsigned IdentifierNamespace : 12;
 
@@ -269,7 +273,9 @@ class Decl {
   /// This field is only valid for NamedDecls subclasses.
   mutable unsigned CachedLinkage : 2;
   
-  
+  friend class ASTDeclWriter;
+  friend class ASTDeclReader;
+
 private:
   void CheckAccessDeclContext() const;
 
@@ -279,7 +285,8 @@ class Decl {
     : NextDeclInContext(0), DeclCtx(DC),
       Loc(L), DeclKind(DK), InvalidDecl(0),
       HasAttrs(false), Implicit(false), Used(false), Referenced(false),
-      Access(AS_none), PCHLevel(0), ChangedAfterLoad(false),
+      Access(AS_none), FromASTFile(0), ChangedAfterLoad(false),
+      ModulePrivate(0),
       IdentifierNamespace(getIdentifierNamespaceForKind(DK)),
       HasCachedLinkage(0) 
   {
@@ -289,7 +296,8 @@ class Decl {
   Decl(Kind DK, EmptyShell Empty)
     : NextDeclInContext(0), DeclKind(DK), InvalidDecl(0),
       HasAttrs(false), Implicit(false), Used(false), Referenced(false),
-      Access(AS_none), PCHLevel(0), ChangedAfterLoad(false),
+      Access(AS_none), FromASTFile(0), ChangedAfterLoad(false),
+      ModulePrivate(0),
       IdentifierNamespace(getIdentifierNamespaceForKind(DK)),
       HasCachedLinkage(0)
   {
@@ -493,25 +501,10 @@ class Decl {
   /// declaration cannot be weak-imported because it has a definition.
   bool canBeWeakImported(bool &IsDefinition) const;
 
-  /// \brief Retrieve the level of precompiled header from which this
-  /// declaration was generated.
-  ///
-  /// The PCH level of a declaration describes where the declaration originated
-  /// from. A PCH level of 0 indicates that the declaration was parsed from
-  /// source. A PCH level of 1 indicates that the declaration was loaded from
-  /// a top-level AST file. A PCH level 2 indicates that the declaration was
-  /// loaded from a PCH file the AST file depends on, and so on.
-  unsigned getPCHLevel() const { return PCHLevel; }
-
-  /// \brief The maximum PCH level that any declaration may have.
-  static const unsigned MaxPCHLevel = 3;
-
-  /// \brief Set the PCH level of this declaration.
-  void setPCHLevel(unsigned Level) { 
-    assert(Level <= MaxPCHLevel && "PCH level exceeds the maximum");
-    PCHLevel = Level;
-  }
-
+  /// \brief Determine whether this declaration came from an AST file (such as
+  /// a precompiled header or module) rather than having been parsed.
+  bool isFromASTFile() const { return FromASTFile; }
+  
   /// \brief Query whether this declaration was changed in a significant way
   /// since being loaded from an AST file.
   ///
@@ -574,7 +567,17 @@ class Decl {
   /// scoped decl is defined outside the current function or method.  This is
   /// roughly global variables and functions, but also handles enums (which
   /// could be defined inside or outside a function etc).
-  bool isDefinedOutsideFunctionOrMethod() const;
+  bool isDefinedOutsideFunctionOrMethod() const {
+    return getParentFunctionOrMethod() == 0;
+  }
+
+  /// \brief If this decl is defined inside a function/method/block it returns
+  /// the corresponding DeclContext, otherwise it returns null.
+  const DeclContext *getParentFunctionOrMethod() const;
+  DeclContext *getParentFunctionOrMethod() {
+    return const_cast<DeclContext*>(
+                    const_cast<const Decl*>(this)->getParentFunctionOrMethod());
+  }
 
   /// \brief Retrieves the "canonical" declaration of the given declaration.
   virtual Decl *getCanonicalDecl() { return this; }
@@ -671,6 +674,9 @@ class Decl {
   /// \brief Whether this declaration is a parameter pack.
   bool isParameterPack() const;
   
+  /// \brief returns true if this declaration is a template
+  bool isTemplateDecl() const;
+
   /// \brief Whether this declaration is a function or function template.
   bool isFunctionOrFunctionTemplate() const;
 
@@ -734,15 +740,16 @@ class Decl {
   static DeclContext *castToDeclContext(const Decl *);
   static Decl *castFromDeclContext(const DeclContext *);
 
-  void print(llvm::raw_ostream &Out, unsigned Indentation = 0) const;
-  void print(llvm::raw_ostream &Out, const PrintingPolicy &Policy,
-             unsigned Indentation = 0) const;
+  void print(raw_ostream &Out, unsigned Indentation = 0,
+             bool PrintInstantiation = false) const;
+  void print(raw_ostream &Out, const PrintingPolicy &Policy,
+             unsigned Indentation = 0, bool PrintInstantiation = false) const;
   static void printGroup(Decl** Begin, unsigned NumDecls,
-                         llvm::raw_ostream &Out, const PrintingPolicy &Policy,
+                         raw_ostream &Out, const PrintingPolicy &Policy,
                          unsigned Indentation = 0);
   void dump() const;
   void dumpXML() const;
-  void dumpXML(llvm::raw_ostream &OS) const;
+  void dumpXML(raw_ostream &OS) const;
 
 private:
   const Attr *getAttrsImpl() const;
@@ -763,7 +770,7 @@ class PrettyStackTraceDecl : public llvm::PrettyStackTraceEntry {
                        SourceManager &sm, const char *Msg)
   : TheDecl(theDecl), Loc(L), SM(sm), Message(Msg) {}
 
-  virtual void print(llvm::raw_ostream &OS) const;
+  virtual void print(raw_ostream &OS) const;
 };
 
 class DeclContextLookupResult
@@ -839,7 +846,7 @@ class DeclContext {
   ///
   /// \returns the first/last pair of declarations.
   static std::pair<Decl *, Decl *>
-  BuildDeclChain(const llvm::SmallVectorImpl<Decl*> &Decls);
+  BuildDeclChain(const SmallVectorImpl<Decl*> &Decls, bool FieldsAlreadyLoaded);
 
    DeclContext(Decl::Kind K)
      : DeclKind(K), ExternalLexicalStorage(false),
@@ -892,6 +899,18 @@ class DeclContext {
     return DeclKind == Decl::Block;
   }
 
+  bool isObjCContainer() const {
+    switch (DeclKind) {
+        case Decl::ObjCCategory:
+        case Decl::ObjCCategoryImpl:
+        case Decl::ObjCImplementation:
+        case Decl::ObjCInterface:
+        case Decl::ObjCProtocol:
+            return true;
+    }
+    return false;
+  }
+
   bool isFunctionOrMethod() const {
     switch (DeclKind) {
     case Decl::Block:
@@ -1243,6 +1262,15 @@ class DeclContext {
   lookup_result lookup(DeclarationName Name);
   lookup_const_result lookup(DeclarationName Name) const;
 
+  /// \brief A simplistic name lookup mechanism that performs name lookup
+  /// into this declaration context without consulting the external source.
+  ///
+  /// This function should almost never be used, because it subverts the 
+  /// usual relationship between a DeclContext and the external source.
+  /// See the ASTImporter for the (few, but important) use cases.
+  void localUncachedLookup(DeclarationName Name, 
+                           llvm::SmallVectorImpl<NamedDecl *> &Results);
+  
   /// @brief Makes a declaration visible within this context.
   ///
   /// This routine makes the declaration D visible to name lookup
@@ -1263,14 +1291,6 @@ class DeclContext {
   /// the declaration chains.
   void makeDeclVisibleInContext(NamedDecl *D, bool Recoverable = true);
 
-  /// \brief Deserialize all the visible declarations from external storage.
-  ///
-  /// Name lookup deserializes visible declarations lazily, thus a DeclContext
-  /// may not have a complete name lookup table. This function deserializes
-  /// the rest of visible declarations from the external storage and completes
-  /// the name lookup table.
-  void MaterializeVisibleDeclsFromExternalStorage();
-
   /// udir_iterator - Iterates through the using-directives stored
   /// within this context.
   typedef UsingDirectiveDecl * const * udir_iterator;
@@ -1317,6 +1337,12 @@ class DeclContext {
     ExternalVisibleStorage = ES;
   }
 
+  /// \brief Determine whether the given declaration is stored in the list of
+  /// declarations lexically within this context.
+  bool isDeclInLexicalTraversal(const Decl *D) const {
+    return D && (D->NextDeclInContext || D == FirstDecl || D == LastDecl);
+  }
+  
   static bool classof(const Decl *D);
   static bool classof(const DeclContext *D) { return true; }
 #define DECL(NAME, BASE)
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h b/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h
index dd490f48c52c..7e60773e4416 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h
@@ -363,10 +363,10 @@ class CXXRecordDecl : public RecordDecl {
     ///        default constructor.
     bool HasTrivialDefaultConstructor : 1;
 
-    /// HasConstExprNonCopyMoveConstructor - True when this class has at least
+    /// HasConstexprNonCopyMoveConstructor - True when this class has at least
     /// one constexpr constructor which is neither the copy nor move
     /// constructor.
-    bool HasConstExprNonCopyMoveConstructor : 1;
+    bool HasConstexprNonCopyMoveConstructor : 1;
 
     /// HasTrivialCopyConstructor - True when this class has a trivial copy
     /// constructor.
@@ -468,6 +468,14 @@ class CXXRecordDecl : public RecordDecl {
     /// \brief Whether we have already declared a destructor within the class.
     bool DeclaredDestructor : 1;
 
+    /// \brief Whether an implicit move constructor was attempted to be declared
+    /// but would have been deleted.
+    bool FailedImplicitMoveConstructor : 1;
+
+    /// \brief Whether an implicit move assignment operator was attempted to be
+    /// declared but would have been deleted.
+    bool FailedImplicitMoveAssignment : 1;
+
     /// NumBases - The number of base class specifiers in Bases.
     unsigned NumBases;
     
@@ -780,6 +788,33 @@ class CXXRecordDecl : public RecordDecl {
     return data().DeclaredMoveConstructor;
   }
 
+  /// \brief Determine whether implicit move constructor generation for this
+  /// class has failed before.
+  bool hasFailedImplicitMoveConstructor() const {
+    return data().FailedImplicitMoveConstructor;
+  }
+
+  /// \brief Set whether implicit move constructor generation for this class
+  /// has failed before.
+  void setFailedImplicitMoveConstructor(bool Failed = true) {
+    data().FailedImplicitMoveConstructor = Failed;
+  }
+
+  /// \brief Determine whether this class should get an implicit move
+  /// constructor or if any existing special member function inhibits this.
+  ///
+  /// Covers all bullets of C++0x [class.copy]p9 except the last, that the
+  /// constructor wouldn't be deleted, which is only looked up from a cached
+  /// result.
+  bool needsImplicitMoveConstructor() const {
+    return !hasFailedImplicitMoveConstructor() &&
+           !hasDeclaredMoveConstructor() &&
+           !hasUserDeclaredCopyConstructor() &&
+           !hasUserDeclaredCopyAssignment() &&
+           !hasUserDeclaredMoveAssignment() &&
+           !hasUserDeclaredDestructor();
+  }
+
   /// hasUserDeclaredCopyAssignment - Whether this class has a
   /// user-declared copy assignment operator. When false, a copy
   /// assigment operator will be implicitly declared.
@@ -807,6 +842,33 @@ class CXXRecordDecl : public RecordDecl {
     return data().DeclaredMoveAssignment;
   }
 
+  /// \brief Determine whether implicit move assignment generation for this
+  /// class has failed before.
+  bool hasFailedImplicitMoveAssignment() const {
+    return data().FailedImplicitMoveAssignment;
+  }
+
+  /// \brief Set whether implicit move assignment generation for this class
+  /// has failed before.
+  void setFailedImplicitMoveAssignment(bool Failed = true) {
+    data().FailedImplicitMoveAssignment = Failed;
+  }
+
+  /// \brief Determine whether this class should get an implicit move
+  /// assignment operator or if any existing special member function inhibits
+  /// this.
+  ///
+  /// Covers all bullets of C++0x [class.copy]p20 except the last, that the
+  /// constructor wouldn't be deleted.
+  bool needsImplicitMoveAssignment() const {
+    return !hasFailedImplicitMoveAssignment() &&
+           !hasDeclaredMoveAssignment() &&
+           !hasUserDeclaredCopyConstructor() &&
+           !hasUserDeclaredCopyAssignment() &&
+           !hasUserDeclaredMoveConstructor() &&
+           !hasUserDeclaredDestructor();
+  }
+
   /// hasUserDeclaredDestructor - Whether this class has a
   /// user-declared destructor. When false, a destructor will be
   /// implicitly declared.
@@ -889,10 +951,10 @@ class CXXRecordDecl : public RecordDecl {
              data().DeclaredDefaultConstructor);
   }
 
-  // hasConstExprNonCopyMoveConstructor - Whether this class has at least one
-  // constexpr constructor other than the copy or move constructors
-  bool hasConstExprNonCopyMoveConstructor() const {
-    return data().HasConstExprNonCopyMoveConstructor;
+  // hasConstexprNonCopyMoveConstructor - Whether this class has at least one
+  // constexpr constructor other than the copy or move constructors.
+  bool hasConstexprNonCopyMoveConstructor() const {
+    return data().HasConstexprNonCopyMoveConstructor;
   }
 
   // hasTrivialCopyConstructor - Whether this class has a trivial copy
@@ -942,6 +1004,25 @@ class CXXRecordDecl : public RecordDecl {
     return isTriviallyCopyable() && hasTrivialDefaultConstructor();
   }
 
+  // isLiteral - Whether this class is a literal type.
+  //
+  // C++0x [basic.types]p10
+  //   A class type that has all the following properties:
+  //     -- a trivial destructor
+  //     -- every constructor call and full-expression in the
+  //        brace-or-equal-intializers for non-static data members (if any) is
+  //        a constant expression.
+  //     -- it is an aggregate type or has at least one constexpr constructor or
+  //        constructor template that is not a copy or move constructor, and
+  //     -- all non-static data members and base classes of literal types
+  //
+  // We resolve DR1361 by ignoring the second bullet.
+  bool isLiteral() const {
+    return hasTrivialDestructor() &&
+           (isAggregate() || hasConstexprNonCopyMoveConstructor()) &&
+           !hasNonLiteralTypeFieldsOrBases();
+  }
+
   /// \brief If this record is an instantiation of a member class,
   /// retrieves the member class from which it was instantiated.
   ///
@@ -1237,10 +1318,10 @@ class CXXMethodDecl : public FunctionDecl {
                 const DeclarationNameInfo &NameInfo,
                 QualType T, TypeSourceInfo *TInfo,
                 bool isStatic, StorageClass SCAsWritten, bool isInline,
-                SourceLocation EndLocation)
+                bool isConstexpr, SourceLocation EndLocation)
     : FunctionDecl(DK, RD, StartLoc, NameInfo, T, TInfo,
                    (isStatic ? SC_Static : SC_None),
-                   SCAsWritten, isInline) {
+                   SCAsWritten, isInline, isConstexpr) {
       if (EndLocation.isValid())
         setRangeEnd(EndLocation);
     }
@@ -1253,6 +1334,7 @@ class CXXMethodDecl : public FunctionDecl {
                                bool isStatic,
                                StorageClass SCAsWritten,
                                bool isInline,
+                               bool isConstexpr,
                                SourceLocation EndLocation);
 
   bool isStatic() const { return getStorageClass() == SC_Static; }
@@ -1333,6 +1415,7 @@ class CXXMethodDecl : public FunctionDecl {
   ///   void g() &&;
   ///   void h();
   /// };
+  /// \endcode
   RefQualifierKind getRefQualifier() const {
     return getType()->getAs<FunctionProtoType>()->getRefQualifier();
   }
@@ -1630,9 +1713,9 @@ class CXXConstructorDecl : public CXXMethodDecl {
                      const DeclarationNameInfo &NameInfo,
                      QualType T, TypeSourceInfo *TInfo,
                      bool isExplicitSpecified, bool isInline, 
-                     bool isImplicitlyDeclared)
+                     bool isImplicitlyDeclared, bool isConstexpr)
     : CXXMethodDecl(CXXConstructor, RD, StartLoc, NameInfo, T, TInfo, false,
-                    SC_None, isInline, SourceLocation()),
+                    SC_None, isInline, isConstexpr, SourceLocation()),
       IsExplicitSpecified(isExplicitSpecified), ImplicitlyDefined(false),
       CtorInitializers(0), NumCtorInitializers(0) {
     setImplicit(isImplicitlyDeclared);
@@ -1645,7 +1728,8 @@ class CXXConstructorDecl : public CXXMethodDecl {
                                     const DeclarationNameInfo &NameInfo,
                                     QualType T, TypeSourceInfo *TInfo,
                                     bool isExplicit,
-                                    bool isInline, bool isImplicitlyDeclared);
+                                    bool isInline, bool isImplicitlyDeclared,
+                                    bool isConstexpr);
 
   /// isExplicitSpecified - Whether this constructor declaration has the
   /// 'explicit' keyword specified.
@@ -1853,7 +1937,7 @@ class CXXDestructorDecl : public CXXMethodDecl {
                     QualType T, TypeSourceInfo *TInfo,
                     bool isInline, bool isImplicitlyDeclared)
     : CXXMethodDecl(CXXDestructor, RD, StartLoc, NameInfo, T, TInfo, false,
-                    SC_None, isInline, SourceLocation()),
+                    SC_None, isInline, /*isConstexpr=*/false, SourceLocation()),
       ImplicitlyDefined(false), OperatorDelete(0) {
     setImplicit(isImplicitlyDeclared);
   }
@@ -1916,9 +2000,9 @@ class CXXConversionDecl : public CXXMethodDecl {
                     const DeclarationNameInfo &NameInfo,
                     QualType T, TypeSourceInfo *TInfo,
                     bool isInline, bool isExplicitSpecified,
-                    SourceLocation EndLocation)
+                    bool isConstexpr, SourceLocation EndLocation)
     : CXXMethodDecl(CXXConversion, RD, StartLoc, NameInfo, T, TInfo, false,
-                    SC_None, isInline, EndLocation),
+                    SC_None, isInline, isConstexpr, EndLocation),
       IsExplicitSpecified(isExplicitSpecified) { }
 
 public:
@@ -1928,6 +2012,7 @@ class CXXConversionDecl : public CXXMethodDecl {
                                    const DeclarationNameInfo &NameInfo,
                                    QualType T, TypeSourceInfo *TInfo,
                                    bool isInline, bool isExplicit,
+                                   bool isConstexpr,
                                    SourceLocation EndLocation);
 
   /// IsExplicitSpecified - Whether this conversion function declaration is 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclContextInternals.h b/contrib/llvm/tools/clang/include/clang/AST/DeclContextInternals.h
index 97da6ca9a1ef..c5f2aa0b848b 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclContextInternals.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclContextInternals.h
@@ -31,7 +31,7 @@ class DependentDiagnostic;
 struct StoredDeclsList {
 
   /// DeclsTy - When in vector form, this is what the Data pointer points to.
-  typedef llvm::SmallVector<NamedDecl *, 4> DeclsTy;
+  typedef SmallVector<NamedDecl *, 4> DeclsTy;
 
   /// \brief The stored data, which will be either a pointer to a NamedDecl,
   /// or a pointer to a vector.
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclObjC.h b/contrib/llvm/tools/clang/include/clang/AST/DeclObjC.h
index d318fc27a8ba..425c89d290a6 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclObjC.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclObjC.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_AST_DECLOBJC_H
 
 #include "clang/AST/Decl.h"
+#include "clang/AST/SelectorLocationsKind.h"
 #include "llvm/ADT/STLExtras.h"
 
 namespace clang {
@@ -127,6 +128,12 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   // Method has a definition.
   unsigned IsDefined : 1;
 
+  /// \brief Method redeclaration in the same interface.
+  unsigned IsRedeclaration : 1;
+
+  /// \brief Is redeclared in the same interface.
+  mutable unsigned HasRedeclaration : 1;
+
   // NOTE: VC++ treats enums as signed, avoid using ImplementationControl enum
   /// @required/@optional
   unsigned DeclImplementation : 2;
@@ -138,8 +145,9 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   /// \brief Indicates whether this method has a related result type.
   unsigned RelatedResultType : 1;
   
-  // Number of args separated by ':' in a method declaration.
-  unsigned NumSelectorArgs;
+  /// \brief Whether the locations of the selector identifiers are in a
+  /// "standard" position, a enum SelectorLocationsKind.
+  unsigned SelLocsKind : 2;
 
   // Result type of this method.
   QualType MethodDeclType;
@@ -147,9 +155,10 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   // Type source information for the result type.
   TypeSourceInfo *ResultTInfo;
 
-  /// ParamInfo - List of pointers to VarDecls for the formal parameters of this
-  /// Method.
-  ObjCList<ParmVarDecl> ParamInfo;
+  /// \brief Array of ParmVarDecls for the formal parameters of this method
+  /// and optionally followed by selector locations.
+  void *ParamsAndSelLocs;
+  unsigned NumParams;
 
   /// List of attributes for this method declaration.
   SourceLocation EndLoc; // the location of the ';' or '}'.
@@ -165,6 +174,43 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   /// constructed by createImplicitParams.
   ImplicitParamDecl *CmdDecl;
 
+  SelectorLocationsKind getSelLocsKind() const {
+    return (SelectorLocationsKind)SelLocsKind;
+  }
+  bool hasStandardSelLocs() const {
+    return getSelLocsKind() != SelLoc_NonStandard;
+  }
+
+  /// \brief Get a pointer to the stored selector identifiers locations array.
+  /// No locations will be stored if HasStandardSelLocs is true.
+  SourceLocation *getStoredSelLocs() {
+    return reinterpret_cast<SourceLocation*>(getParams() + NumParams);
+  }
+  const SourceLocation *getStoredSelLocs() const {
+    return reinterpret_cast<const SourceLocation*>(getParams() + NumParams);
+  }
+
+  /// \brief Get a pointer to the stored selector identifiers locations array.
+  /// No locations will be stored if HasStandardSelLocs is true.
+  ParmVarDecl **getParams() {
+    return reinterpret_cast<ParmVarDecl **>(ParamsAndSelLocs);
+  }
+  const ParmVarDecl *const *getParams() const {
+    return reinterpret_cast<const ParmVarDecl *const *>(ParamsAndSelLocs);
+  }
+
+  /// \brief Get the number of stored selector identifiers locations.
+  /// No locations will be stored if HasStandardSelLocs is true.
+  unsigned getNumStoredSelLocs() const {
+    if (hasStandardSelLocs())
+      return 0;
+    return getNumSelectorLocs();
+  }
+
+  void setParamsAndSelLocs(ASTContext &C,
+                           ArrayRef<ParmVarDecl*> Params,
+                           ArrayRef<SourceLocation> SelLocs);
+
   ObjCMethodDecl(SourceLocation beginLoc, SourceLocation endLoc,
                  Selector SelInfo, QualType T,
                  TypeSourceInfo *ResultTInfo,
@@ -172,19 +218,23 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
                  bool isInstance = true,
                  bool isVariadic = false,
                  bool isSynthesized = false,
+                 bool isImplicitlyDeclared = false,
                  bool isDefined = false,
                  ImplementationControl impControl = None,
-                 bool HasRelatedResultType = false,
-                 unsigned numSelectorArgs = 0)
+                 bool HasRelatedResultType = false)
   : NamedDecl(ObjCMethod, contextDecl, beginLoc, SelInfo),
     DeclContext(ObjCMethod), Family(InvalidObjCMethodFamily),
     IsInstance(isInstance), IsVariadic(isVariadic),
     IsSynthesized(isSynthesized),
-    IsDefined(isDefined),
+    IsDefined(isDefined), IsRedeclaration(0), HasRedeclaration(0),
     DeclImplementation(impControl), objcDeclQualifier(OBJC_TQ_None),
-    RelatedResultType(HasRelatedResultType), NumSelectorArgs(numSelectorArgs), 
+    RelatedResultType(HasRelatedResultType),
+    SelLocsKind(SelLoc_StandardNoSpace),
     MethodDeclType(T), ResultTInfo(ResultTInfo),
-    EndLoc(endLoc), Body(0), SelfDecl(0), CmdDecl(0) {}
+    ParamsAndSelLocs(0), NumParams(0),
+    EndLoc(endLoc), Body(0), SelfDecl(0), CmdDecl(0) {
+    setImplicit(isImplicitlyDeclared);
+  }
 
   /// \brief A definition will return its interface declaration.
   /// An interface declaration will return its definition.
@@ -194,17 +244,18 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
 public:
   static ObjCMethodDecl *Create(ASTContext &C,
                                 SourceLocation beginLoc,
-                                SourceLocation endLoc, Selector SelInfo,
+                                SourceLocation endLoc,
+                                Selector SelInfo,
                                 QualType T, 
                                 TypeSourceInfo *ResultTInfo,
                                 DeclContext *contextDecl,
                                 bool isInstance = true,
                                 bool isVariadic = false,
                                 bool isSynthesized = false,
+                                bool isImplicitlyDeclared = false,
                                 bool isDefined = false,
                                 ImplementationControl impControl = None,
-                                bool HasRelatedResultType = false,
-                                unsigned numSelectorArgs = 0);
+                                bool HasRelatedResultType = false);
 
   virtual ObjCMethodDecl *getCanonicalDecl();
   const ObjCMethodDecl *getCanonicalDecl() const {
@@ -222,11 +273,10 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   
   /// \brief Note whether this method has a related result type.
   void SetRelatedResultType(bool RRT = true) { RelatedResultType = RRT; }
-  
-  unsigned getNumSelectorArgs() const { return NumSelectorArgs; }
-  void setNumSelectorArgs(unsigned numSelectorArgs) { 
-    NumSelectorArgs = numSelectorArgs; 
-  }
+
+  /// \brief True if this is a method redeclaration in the same interface.
+  bool isRedeclaration() const { return IsRedeclaration; }
+  void setAsRedeclaration(const ObjCMethodDecl *PrevMethod);
   
   // Location information, modeled after the Stmt API.
   SourceLocation getLocStart() const { return getLocation(); }
@@ -236,6 +286,29 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
     return SourceRange(getLocation(), EndLoc);
   }
 
+  SourceLocation getSelectorStartLoc() const { return getSelectorLoc(0); }
+  SourceLocation getSelectorLoc(unsigned Index) const {
+    assert(Index < getNumSelectorLocs() && "Index out of range!");
+    if (hasStandardSelLocs())
+      return getStandardSelectorLoc(Index, getSelector(),
+                                   getSelLocsKind() == SelLoc_StandardWithSpace,
+                      llvm::makeArrayRef(const_cast<ParmVarDecl**>(getParams()),
+                                         NumParams),
+                                   EndLoc);
+    return getStoredSelLocs()[Index];
+  }
+
+  void getSelectorLocs(SmallVectorImpl<SourceLocation> &SelLocs) const;
+
+  unsigned getNumSelectorLocs() const {
+    if (isImplicit())
+      return 0;
+    Selector Sel = getSelector();
+    if (Sel.isUnarySelector())
+      return 1;
+    return Sel.getNumArgs();
+  }
+
   ObjCInterfaceDecl *getClassInterface();
   const ObjCInterfaceDecl *getClassInterface() const {
     return const_cast<ObjCMethodDecl*>(this)->getClassInterface();
@@ -256,25 +329,31 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   void setResultTypeSourceInfo(TypeSourceInfo *TInfo) { ResultTInfo = TInfo; }
 
   // Iterator access to formal parameters.
-  unsigned param_size() const { return ParamInfo.size(); }
-  typedef ObjCList<ParmVarDecl>::iterator param_iterator;
-  param_iterator param_begin() const { return ParamInfo.begin(); }
-  param_iterator param_end() const { return ParamInfo.end(); }
+  unsigned param_size() const { return NumParams; }
+  typedef const ParmVarDecl *const *param_const_iterator;
+  typedef ParmVarDecl *const *param_iterator;
+  param_const_iterator param_begin() const { return getParams(); }
+  param_const_iterator param_end() const { return getParams() + NumParams; }
+  param_iterator param_begin() { return getParams(); }
+  param_iterator param_end() { return getParams() + NumParams; }
   // This method returns and of the parameters which are part of the selector
   // name mangling requirements.
-  param_iterator sel_param_end() const { 
-    return ParamInfo.begin() + NumSelectorArgs; 
+  param_const_iterator sel_param_end() const { 
+    return param_begin() + getSelector().getNumArgs(); 
   }
 
-  void setMethodParams(ASTContext &C, ParmVarDecl *const *List, unsigned Num,
-                       unsigned numSelectorArgs) {
-    ParamInfo.set(List, Num, C);
-    NumSelectorArgs = numSelectorArgs; 
-  }
+  /// \brief Sets the method's parameters and selector source locations.
+  /// If the method is implicit (not coming from source) \arg SelLocs is
+  /// ignored.
+  void setMethodParams(ASTContext &C,
+                       ArrayRef<ParmVarDecl*> Params,
+                       ArrayRef<SourceLocation> SelLocs =
+                           ArrayRef<SourceLocation>());
 
   // Iterator access to parameter types.
   typedef std::const_mem_fun_t<QualType, ParmVarDecl> deref_fun;
-  typedef llvm::mapped_iterator<param_iterator, deref_fun> arg_type_iterator;
+  typedef llvm::mapped_iterator<param_const_iterator, deref_fun>
+      arg_type_iterator;
 
   arg_type_iterator arg_type_begin() const {
     return llvm::map_iterator(param_begin(), deref_fun(&ParmVarDecl::getType));
@@ -337,6 +416,9 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
   static ObjCMethodDecl *castFromDeclContext(const DeclContext *DC) {
     return static_cast<ObjCMethodDecl *>(const_cast<DeclContext*>(DC));
   }
+
+  friend class ASTDeclReader;
+  friend class ASTDeclWriter;
 };
 
 /// ObjCContainerDecl - Represents a container for method declarations.
@@ -344,14 +426,17 @@ class ObjCMethodDecl : public NamedDecl, public DeclContext {
 /// ObjCProtocolDecl, and ObjCImplDecl.
 ///
 class ObjCContainerDecl : public NamedDecl, public DeclContext {
+  SourceLocation AtStart;
+
   // These two locations in the range mark the end of the method container.
   // The first points to the '@' token, and the second to the 'end' token.
   SourceRange AtEnd;
 public:
 
-  ObjCContainerDecl(Kind DK, DeclContext *DC, SourceLocation L,
-                    IdentifierInfo *Id)
-    : NamedDecl(DK, DC, L, Id), DeclContext(DK) {}
+  ObjCContainerDecl(Kind DK, DeclContext *DC,
+                    IdentifierInfo *Id, SourceLocation nameLoc,
+                    SourceLocation atStartLoc)
+    : NamedDecl(DK, DC, nameLoc, Id), DeclContext(DK), AtStart(atStartLoc) {}
 
   // Iterator access to properties.
   typedef specific_decl_iterator<ObjCPropertyDecl> prop_iterator;
@@ -403,6 +488,9 @@ class ObjCContainerDecl : public NamedDecl, public DeclContext {
 
   ObjCPropertyDecl *FindPropertyDeclaration(IdentifierInfo *PropertyId) const;
 
+  SourceLocation getAtStartLoc() const { return AtStart; }
+  void setAtStartLoc(SourceLocation Loc) { AtStart = Loc; }
+
   // Marks the end of the container.
   SourceRange getAtEndRange() const {
     return AtEnd;
@@ -412,7 +500,7 @@ class ObjCContainerDecl : public NamedDecl, public DeclContext {
   }
 
   virtual SourceRange getSourceRange() const {
-    return SourceRange(getLocation(), getAtEndRange().getEnd());
+    return SourceRange(AtStart, getAtEndRange().getEnd());
   }
 
   // Implement isa/cast/dyncast/etc.
@@ -485,7 +573,6 @@ class ObjCInterfaceDecl : public ObjCContainerDecl {
   /// completed by the external AST source when required.
   mutable bool ExternallyCompleted : 1;
   
-  SourceLocation ClassLoc; // location of the class identifier.
   SourceLocation SuperClassLoc; // location of the super class identifier.
   SourceLocation EndLoc; // marks the '>', '}', or identifier.
 
@@ -576,7 +663,7 @@ class ObjCInterfaceDecl : public ObjCContainerDecl {
 
   typedef specific_decl_iterator<ObjCIvarDecl> ivar_iterator;
 
-  ivar_iterator ivar_begin() const { return  ivar_iterator(decls_begin()); }
+  ivar_iterator ivar_begin() const { return ivar_iterator(decls_begin()); }
   ivar_iterator ivar_end() const { return ivar_iterator(decls_end()); }
 
   unsigned ivar_size() const {
@@ -585,7 +672,12 @@ class ObjCInterfaceDecl : public ObjCContainerDecl {
   
   bool ivar_empty() const { return ivar_begin() == ivar_end(); }
   
-  ObjCIvarDecl  *all_declared_ivar_begin();
+  ObjCIvarDecl *all_declared_ivar_begin();
+  const ObjCIvarDecl *all_declared_ivar_begin() const {
+    // Even though this modifies IvarList, it's conceptually const:
+    // the ivar chain is essentially a cached property of ObjCInterfaceDecl.
+    return const_cast<ObjCInterfaceDecl *>(this)->all_declared_ivar_begin();
+  }
   void setIvarList(ObjCIvarDecl *ivar) { IvarList = ivar; }
   
   /// setProtocolList - Set the list of protocols that this interface
@@ -675,12 +767,10 @@ class ObjCInterfaceDecl : public ObjCContainerDecl {
   ObjCMethodDecl *lookupPrivateMethod(const Selector &Sel, bool Instance=true);
 
   // Location information, modeled after the Stmt API.
-  SourceLocation getLocStart() const { return getLocation(); } // '@'interface
+  SourceLocation getLocStart() const { return getAtStartLoc(); } // '@'interface
   SourceLocation getLocEnd() const { return EndLoc; }
   void setLocEnd(SourceLocation LE) { EndLoc = LE; }
 
-  void setClassLoc(SourceLocation Loc) { ClassLoc = Loc; }
-  SourceLocation getClassLoc() const { return ClassLoc; }
   void setSuperClassLoc(SourceLocation Loc) { SuperClassLoc = Loc; }
   SourceLocation getSuperClassLoc() const { return SuperClassLoc; }
 
@@ -754,6 +844,7 @@ class ObjCIvarDecl : public FieldDecl {
   const ObjCInterfaceDecl *getContainingInterface() const;
   
   ObjCIvarDecl *getNextIvar() { return NextIvar; }
+  const ObjCIvarDecl *getNextIvar() const { return NextIvar; }
   void setNextIvar(ObjCIvarDecl *ivar) { NextIvar = ivar; }
 
   void setAccessControl(AccessControl ac) { DeclAccess = ac; }
@@ -837,14 +928,17 @@ class ObjCProtocolDecl : public ObjCContainerDecl {
 
   SourceLocation EndLoc; // marks the '>' or identifier.
 
-  ObjCProtocolDecl(DeclContext *DC, SourceLocation L, IdentifierInfo *Id)
-    : ObjCContainerDecl(ObjCProtocol, DC, L, Id),
+  ObjCProtocolDecl(DeclContext *DC, IdentifierInfo *Id,
+                   SourceLocation nameLoc, SourceLocation atStartLoc)
+    : ObjCContainerDecl(ObjCProtocol, DC, Id, nameLoc, atStartLoc),
       isForwardProtoDecl(true) {
   }
 
 public:
   static ObjCProtocolDecl *Create(ASTContext &C, DeclContext *DC,
-                                  SourceLocation L, IdentifierInfo *Id);
+                                  IdentifierInfo *Id,
+                                  SourceLocation nameLoc,
+                                  SourceLocation atStartLoc);
 
   const ObjCProtocolList &getReferencedProtocols() const {
     return ReferencedProtocols;
@@ -884,7 +978,7 @@ class ObjCProtocolDecl : public ObjCContainerDecl {
   void setForwardDecl(bool val) { isForwardProtoDecl = val; }
 
   // Location information, modeled after the Stmt API.
-  SourceLocation getLocStart() const { return getLocation(); } // '@'protocol
+  SourceLocation getLocStart() const { return getAtStartLoc(); } // '@'protocol
   SourceLocation getLocEnd() const { return EndLoc; }
   void setLocEnd(SourceLocation LE) { EndLoc = LE; }
 
@@ -908,29 +1002,23 @@ class ObjCClassDecl : public Decl {
     ObjCInterfaceDecl *getInterface() const { return ID; }
   };
 private:
-  ObjCClassRef *ForwardDecls;
-  unsigned NumDecls;
+  ObjCClassRef *ForwardDecl;
 
   ObjCClassDecl(DeclContext *DC, SourceLocation L,
-                ObjCInterfaceDecl *const *Elts, const SourceLocation *Locs,                
-                unsigned nElts, ASTContext &C);
+                ObjCInterfaceDecl *const Elt, const SourceLocation Loc,                
+                ASTContext &C);
 public:
   static ObjCClassDecl *Create(ASTContext &C, DeclContext *DC, SourceLocation L,
-                               ObjCInterfaceDecl *const *Elts = 0,
-                               const SourceLocation *Locs = 0,
-                               unsigned nElts = 0);
+                               ObjCInterfaceDecl *const Elt = 0,
+                               const SourceLocation Locs = SourceLocation());
+    
+  ObjCInterfaceDecl *getForwardInterfaceDecl() { return ForwardDecl->getInterface(); }
+  ObjCClassRef *getForwardDecl() { return ForwardDecl; }
+  void setClass(ASTContext &C, ObjCInterfaceDecl*const Cls,
+                const SourceLocation Locs);
   
   virtual SourceRange getSourceRange() const;
 
-  typedef const ObjCClassRef* iterator;
-  iterator begin() const { return ForwardDecls; }
-  iterator end() const { return ForwardDecls + NumDecls; }
-  unsigned size() const { return NumDecls; }
-
-  /// setClassList - Set the list of forward classes.
-  void setClassList(ASTContext &C, ObjCInterfaceDecl*const*List,
-                    const SourceLocation *Locs, unsigned Num);
-
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classof(const ObjCClassDecl *D) { return true; }
   static bool classofKind(Kind K) { return K == ObjCClass; }
@@ -1013,19 +1101,16 @@ class ObjCCategoryDecl : public ObjCContainerDecl {
 
   /// true of class extension has at least one bitfield ivar.
   bool HasSynthBitfield : 1;
-  
-  /// \brief The location of the '@' in '@interface'
-  SourceLocation AtLoc;
 
   /// \brief The location of the category name in this declaration.
   SourceLocation CategoryNameLoc;
 
   ObjCCategoryDecl(DeclContext *DC, SourceLocation AtLoc, 
                    SourceLocation ClassNameLoc, SourceLocation CategoryNameLoc,
-                   IdentifierInfo *Id)
-    : ObjCContainerDecl(ObjCCategory, DC, ClassNameLoc, Id),
-      ClassInterface(0), NextClassCategory(0), HasSynthBitfield(false),
-      AtLoc(AtLoc), CategoryNameLoc(CategoryNameLoc) {
+                   IdentifierInfo *Id, ObjCInterfaceDecl *IDecl)
+    : ObjCContainerDecl(ObjCCategory, DC, Id, ClassNameLoc, AtLoc),
+      ClassInterface(IDecl), NextClassCategory(0), HasSynthBitfield(false),
+      CategoryNameLoc(CategoryNameLoc) {
   }
 public:
 
@@ -1033,11 +1118,12 @@ class ObjCCategoryDecl : public ObjCContainerDecl {
                                   SourceLocation AtLoc, 
                                   SourceLocation ClassNameLoc,
                                   SourceLocation CategoryNameLoc,
-                                  IdentifierInfo *Id);
+                                  IdentifierInfo *Id,
+                                  ObjCInterfaceDecl *IDecl);
+  static ObjCCategoryDecl *Create(ASTContext &C, EmptyShell Empty);
 
   ObjCInterfaceDecl *getClassInterface() { return ClassInterface; }
   const ObjCInterfaceDecl *getClassInterface() const { return ClassInterface; }
-  void setClassInterface(ObjCInterfaceDecl *IDecl) { ClassInterface = IDecl; }
 
   ObjCCategoryImplDecl *getImplementation() const;
   void setImplementation(ObjCCategoryImplDecl *ImplD);
@@ -1066,14 +1152,6 @@ class ObjCCategoryDecl : public ObjCContainerDecl {
   }
 
   ObjCCategoryDecl *getNextClassCategory() const { return NextClassCategory; }
-  void setNextClassCategory(ObjCCategoryDecl *Cat) {
-    NextClassCategory = Cat;
-  }
-  void insertNextClassCategory() {
-    NextClassCategory = ClassInterface->getCategoryList();
-    ClassInterface->setCategoryList(this);
-    ClassInterface->setChangedSinceDeserialization(true);
-  }
 
   bool IsClassExtension() const { return getIdentifier() == 0; }
   const ObjCCategoryDecl *getNextClassExtension() const;
@@ -1094,20 +1172,16 @@ class ObjCCategoryDecl : public ObjCContainerDecl {
   bool ivar_empty() const {
     return ivar_begin() == ivar_end();
   }
-  
-  SourceLocation getAtLoc() const { return AtLoc; }
-  void setAtLoc(SourceLocation At) { AtLoc = At; }
 
   SourceLocation getCategoryNameLoc() const { return CategoryNameLoc; }
   void setCategoryNameLoc(SourceLocation Loc) { CategoryNameLoc = Loc; }
 
-  virtual SourceRange getSourceRange() const {
-    return SourceRange(AtLoc, getAtEndRange().getEnd());
-  }
-
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classof(const ObjCCategoryDecl *D) { return true; }
   static bool classofKind(Kind K) { return K == ObjCCategory; }
+
+  friend class ASTDeclReader;
+  friend class ASTDeclWriter;
 };
 
 class ObjCImplDecl : public ObjCContainerDecl {
@@ -1115,10 +1189,12 @@ class ObjCImplDecl : public ObjCContainerDecl {
   ObjCInterfaceDecl *ClassInterface;
 
 protected:
-  ObjCImplDecl(Kind DK, DeclContext *DC, SourceLocation L,
-               ObjCInterfaceDecl *classInterface)
-    : ObjCContainerDecl(DK, DC, L,
-                        classInterface? classInterface->getIdentifier() : 0),
+  ObjCImplDecl(Kind DK, DeclContext *DC,
+               ObjCInterfaceDecl *classInterface,
+               SourceLocation nameLoc, SourceLocation atStartLoc)
+    : ObjCContainerDecl(DK, DC,
+                        classInterface? classInterface->getIdentifier() : 0,
+                        nameLoc, atStartLoc),
       ClassInterface(classInterface) {}
 
 public:
@@ -1175,13 +1251,17 @@ class ObjCCategoryImplDecl : public ObjCImplDecl {
   // Category name
   IdentifierInfo *Id;
 
-  ObjCCategoryImplDecl(DeclContext *DC, SourceLocation L, IdentifierInfo *Id,
-                       ObjCInterfaceDecl *classInterface)
-    : ObjCImplDecl(ObjCCategoryImpl, DC, L, classInterface), Id(Id) {}
+  ObjCCategoryImplDecl(DeclContext *DC, IdentifierInfo *Id,
+                       ObjCInterfaceDecl *classInterface,
+                       SourceLocation nameLoc, SourceLocation atStartLoc)
+    : ObjCImplDecl(ObjCCategoryImpl, DC, classInterface, nameLoc, atStartLoc),
+      Id(Id) {}
 public:
   static ObjCCategoryImplDecl *Create(ASTContext &C, DeclContext *DC,
-                                      SourceLocation L, IdentifierInfo *Id,
-                                      ObjCInterfaceDecl *classInterface);
+                                      IdentifierInfo *Id,
+                                      ObjCInterfaceDecl *classInterface,
+                                      SourceLocation nameLoc,
+                                      SourceLocation atStartLoc);
 
   /// getIdentifier - Get the identifier that names the category
   /// interface associated with this implementation.
@@ -1203,7 +1283,7 @@ class ObjCCategoryImplDecl : public ObjCImplDecl {
   //
   // FIXME: This is a bad API, we are overriding the NamedDecl::getName, to mean
   // something different.
-  llvm::StringRef getName() const {
+  StringRef getName() const {
     return Id ? Id->getNameStart() : "";
   }
 
@@ -1228,7 +1308,7 @@ class ObjCCategoryImplDecl : public ObjCImplDecl {
   static bool classofKind(Kind K) { return K == ObjCCategoryImpl;}
 };
 
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+raw_ostream &operator<<(raw_ostream &OS,
                               const ObjCCategoryImplDecl *CID);
 
 /// ObjCImplementationDecl - Represents a class definition - this is where
@@ -1259,17 +1339,19 @@ class ObjCImplementationDecl : public ObjCImplDecl {
   /// true of class extension has at least one bitfield ivar.
   bool HasSynthBitfield : 1;
   
-  ObjCImplementationDecl(DeclContext *DC, SourceLocation L,
+  ObjCImplementationDecl(DeclContext *DC,
                          ObjCInterfaceDecl *classInterface,
-                         ObjCInterfaceDecl *superDecl)
-    : ObjCImplDecl(ObjCImplementation, DC, L, classInterface),
+                         ObjCInterfaceDecl *superDecl,
+                         SourceLocation nameLoc, SourceLocation atStartLoc)
+    : ObjCImplDecl(ObjCImplementation, DC, classInterface, nameLoc, atStartLoc),
        SuperClass(superDecl), IvarInitializers(0), NumIvarInitializers(0),
        HasCXXStructors(false), HasSynthBitfield(false) {}
 public:
   static ObjCImplementationDecl *Create(ASTContext &C, DeclContext *DC,
-                                        SourceLocation L,
                                         ObjCInterfaceDecl *classInterface,
-                                        ObjCInterfaceDecl *superDecl);
+                                        ObjCInterfaceDecl *superDecl,
+                                        SourceLocation nameLoc,
+                                        SourceLocation atStartLoc);
   
   /// init_iterator - Iterates through the ivar initializer list.
   typedef CXXCtorInitializer **init_iterator;
@@ -1320,7 +1402,7 @@ class ObjCImplementationDecl : public ObjCImplDecl {
   //
   // FIXME: This is a bad API, we are overriding the NamedDecl::getName, to mean
   // something different.
-  llvm::StringRef getName() const {
+  StringRef getName() const {
     assert(getIdentifier() && "Name is not a simple identifier");
     return getIdentifier()->getName();
   }
@@ -1368,7 +1450,7 @@ class ObjCImplementationDecl : public ObjCImplDecl {
   friend class ASTDeclWriter;
 };
 
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+raw_ostream &operator<<(raw_ostream &OS,
                               const ObjCImplementationDecl *ID);
 
 /// ObjCCompatibleAliasDecl - Represents alias of a class. This alias is
@@ -1423,7 +1505,7 @@ class ObjCPropertyDecl : public NamedDecl {
     NumPropertyAttrsBits = 12
   };
 
-  enum SetterKind { Assign, Retain, Copy };
+  enum SetterKind { Assign, Retain, Copy, Weak };
   enum PropertyControl { None, Required, Optional };
 private:
   SourceLocation AtLoc;   // location of @property
@@ -1495,14 +1577,29 @@ class ObjCPropertyDecl : public NamedDecl {
     return (PropertyAttributes & OBJC_PR_readonly);
   }
 
+  /// isAtomic - Return true if the property is atomic.
+  bool isAtomic() const {
+    return (PropertyAttributes & OBJC_PR_atomic);
+  }
+
+  /// isRetaining - Return true if the property retains its value.
+  bool isRetaining() const {
+    return (PropertyAttributes &
+            (OBJC_PR_retain | OBJC_PR_strong | OBJC_PR_copy));
+  }
+
   /// getSetterKind - Return the method used for doing assignment in
   /// the property setter. This is only valid if the property has been
   /// defined to have a setter.
   SetterKind getSetterKind() const {
-    if (PropertyAttributes & (OBJC_PR_retain|OBJC_PR_strong))
+    if (PropertyAttributes & OBJC_PR_strong)
+      return getType()->isBlockPointerType() ? Copy : Retain;
+    if (PropertyAttributes & OBJC_PR_retain)
       return Retain;
     if (PropertyAttributes & OBJC_PR_copy)
       return Copy;
+    if (PropertyAttributes & OBJC_PR_weak)
+      return Weak;
     return Assign;
   }
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclTemplate.h b/contrib/llvm/tools/clang/include/clang/AST/DeclTemplate.h
index d2b1d3990c66..138e47d1a96d 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclTemplate.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclTemplate.h
@@ -264,7 +264,7 @@ class FunctionTemplateSpecializationInfo : public llvm::FoldingSetNode {
                                      FunctionTemplateDecl *Template,
                                      TemplateSpecializationKind TSK,
                                      const TemplateArgumentList *TemplateArgs,
-                          const TemplateArgumentListInfo *TemplateArgsAsWritten,
+                       const ASTTemplateArgumentListInfo *TemplateArgsAsWritten,
                                      SourceLocation POI)
   : Function(FD),
     Template(Template, TSK - 1),
@@ -278,12 +278,7 @@ class FunctionTemplateSpecializationInfo : public llvm::FoldingSetNode {
          TemplateSpecializationKind TSK,
          const TemplateArgumentList *TemplateArgs,
          const TemplateArgumentListInfo *TemplateArgsAsWritten,
-         SourceLocation POI) {
-    return new (C) FunctionTemplateSpecializationInfo(FD, Template, TSK,
-                                                      TemplateArgs,
-                                                      TemplateArgsAsWritten,
-                                                      POI);
-  }
+         SourceLocation POI);
 
   /// \brief The function template specialization that this structure
   /// describes.
@@ -300,7 +295,7 @@ class FunctionTemplateSpecializationInfo : public llvm::FoldingSetNode {
   const TemplateArgumentList *TemplateArguments;
 
   /// \brief The template arguments as written in the sources, if provided.
-  const TemplateArgumentListInfo *TemplateArgumentsAsWritten;
+  const ASTTemplateArgumentListInfo *TemplateArgumentsAsWritten;
 
   /// \brief The point at which this function template specialization was
   /// first instantiated. 
@@ -913,7 +908,7 @@ class TemplateParmPosition {
   // FIXME: This should probably never be called, but it's here as
   TemplateParmPosition()
     : Depth(0), Position(0)
-  { /* assert(0 && "Cannot create positionless template parameter"); */ }
+  { /* llvm_unreachable("Cannot create positionless template parameter"); */ }
 
   TemplateParmPosition(unsigned D, unsigned P)
     : Depth(D), Position(P)
@@ -1864,7 +1859,7 @@ class ClassTemplateDecl : public RedeclarableTemplateDecl,
 
   /// \brief Retrieve the partial specializations as an ordered list.
   void getPartialSpecializations(
-          llvm::SmallVectorImpl<ClassTemplatePartialSpecializationDecl *> &PS);
+          SmallVectorImpl<ClassTemplatePartialSpecializationDecl *> &PS);
   
   /// \brief Find a class template partial specialization with the given
   /// type T.
@@ -2097,6 +2092,58 @@ class TypeAliasTemplateDecl : public RedeclarableTemplateDecl,
   friend class ASTDeclWriter;
 };
 
+/// Declaration of a function specialization at template class scope.
+/// This is a non standard extension needed to support MSVC.
+/// For example:
+/// template <class T>
+/// class A {
+///    template <class U> void foo(U a) { }
+///    template<> void foo(int a) { }
+/// }
+///
+/// "template<> foo(int a)" will be saved in Specialization as a normal
+/// CXXMethodDecl. Then during an instantiation of class A, it will be
+/// transformed into an actual function specialization.
+class ClassScopeFunctionSpecializationDecl : public Decl {
+private:
+  ClassScopeFunctionSpecializationDecl(DeclContext *DC, SourceLocation Loc,
+                                       CXXMethodDecl *FD)
+    : Decl(Decl::ClassScopeFunctionSpecialization, DC, Loc),
+      Specialization(FD) {}
+
+  ClassScopeFunctionSpecializationDecl(EmptyShell Empty)
+    : Decl(Decl::ClassScopeFunctionSpecialization, Empty) {}
+
+  CXXMethodDecl *Specialization;
+
+public:
+  CXXMethodDecl *getSpecialization() const { return Specialization; }
+
+  static ClassScopeFunctionSpecializationDecl *Create(ASTContext &C,
+                                                      DeclContext *DC,
+                                                      SourceLocation Loc,
+                                                      CXXMethodDecl *FD) {
+    return new (C) ClassScopeFunctionSpecializationDecl(DC , Loc, FD);
+  }
+
+  static ClassScopeFunctionSpecializationDecl *Create(ASTContext &Context,
+                                                      EmptyShell Empty) {
+    return new (Context)ClassScopeFunctionSpecializationDecl(0,
+                                                         SourceLocation(), 0);
+  }
+  // Implement isa/cast/dyncast/etc.
+  static bool classof(const Decl *D) { return classofKind(D->getKind()); }
+  static bool classofKind(Kind K) {
+    return K == Decl::ClassScopeFunctionSpecialization;
+  }
+  static bool classof(const ClassScopeFunctionSpecializationDecl *D) {
+    return true;
+  }
+
+  friend class ASTDeclReader;
+  friend class ASTDeclWriter;
+};
+
 /// Implementation of inline functions that require the template declarations
 inline AnyFunctionDecl::AnyFunctionDecl(FunctionTemplateDecl *FTD)
   : Function(FTD) { }
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclVisitor.h b/contrib/llvm/tools/clang/include/clang/AST/DeclVisitor.h
index aee1998028eb..b5b6bd42ddb1 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclVisitor.h
@@ -30,7 +30,7 @@ class DeclVisitor {
 public:
   RetTy Visit(Decl *D) {
     switch (D->getKind()) {
-      default: assert(false && "Decl that isn't part of DeclNodes.inc!");
+      default: llvm_unreachable("Decl that isn't part of DeclNodes.inc!");
 #define DECL(DERIVED, BASE) \
       case Decl::DERIVED: DISPATCH(DERIVED##Decl, DERIVED##Decl);
 #define ABSTRACT_DECL(DECL)
diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclarationName.h b/contrib/llvm/tools/clang/include/clang/AST/DeclarationName.h
index bb098313ea3a..2170f2b1ef7f 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/DeclarationName.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/DeclarationName.h
@@ -203,7 +203,7 @@ class DeclarationName {
   std::string getAsString() const;
 
   /// printName - Print the human-readable name to a stream.
-  void printName(llvm::raw_ostream &OS) const;
+  void printName(raw_ostream &OS) const;
 
   /// getAsIdentifierInfo - Retrieve the IdentifierInfo * stored in
   /// this declaration name, or NULL if this declaration name isn't a
@@ -503,7 +503,7 @@ struct DeclarationNameInfo {
   std::string getAsString() const;
 
   /// printName - Print the human-readable name to a stream.
-  void printName(llvm::raw_ostream &OS) const;
+  void printName(raw_ostream &OS) const;
 
   /// getBeginLoc - Retrieve the location of the first token.
   SourceLocation getBeginLoc() const { return NameLoc; }
@@ -520,7 +520,7 @@ struct DeclarationNameInfo {
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            DeclarationName N) {
   DB.AddTaggedVal(N.getAsOpaqueInteger(),
-                  Diagnostic::ak_declarationname);
+                  DiagnosticsEngine::ak_declarationname);
   return DB;
 }
 
@@ -529,11 +529,11 @@ inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
 inline const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                            DeclarationName N) {
   PD.AddTaggedVal(N.getAsOpaqueInteger(),
-                  Diagnostic::ak_declarationname);
+                  DiagnosticsEngine::ak_declarationname);
   return PD;
 }
 
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+inline raw_ostream &operator<<(raw_ostream &OS,
                                      DeclarationNameInfo DNInfo) {
   DNInfo.printName(OS);
   return OS;
diff --git a/contrib/llvm/tools/clang/include/clang/AST/Expr.h b/contrib/llvm/tools/clang/include/clang/AST/Expr.h
index c7f870725c42..1242f4e6c702 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/Expr.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/Expr.h
@@ -20,6 +20,7 @@
 #include "clang/AST/DeclAccessPair.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/ASTVector.h"
+#include "clang/AST/TemplateBase.h"
 #include "clang/AST/UsuallyTinyPtrVector.h"
 #include "clang/Basic/TypeTraits.h"
 #include "llvm/ADT/APSInt.h"
@@ -41,12 +42,10 @@ namespace clang {
   class CXXOperatorCallExpr;
   class CXXMemberCallExpr;
   class ObjCPropertyRefExpr;
-  class TemplateArgumentLoc;
-  class TemplateArgumentListInfo;
   class OpaqueValueExpr;
 
 /// \brief A simple array of base specifiers.
-typedef llvm::SmallVector<CXXBaseSpecifier*, 4> CXXCastPath;
+typedef SmallVector<CXXBaseSpecifier*, 4> CXXCastPath;
 
 /// Expr - This represents one expression.  Note that Expr's are subclasses of
 /// Stmt.  This allows an expression to be transparently used any place a Stmt
@@ -454,9 +453,14 @@ class Expr : public Stmt {
 
   /// EvaluateAsBooleanCondition - Return true if this is a constant
   /// which we we can fold and convert to a boolean condition using
-  /// any crazy technique that we want to.
+  /// any crazy technique that we want to, even if the expression has
+  /// side-effects.
   bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx) const;
 
+  /// EvaluateAsInt - Return true if this is a constant which we can fold and
+  /// convert to an integer using any crazy technique that we want to.
+  bool EvaluateAsInt(llvm::APSInt &Result, const ASTContext &Ctx) const;
+
   /// isEvaluatable - Call Evaluate to see if this expression can be constant
   /// folded, but discard the result.
   bool isEvaluatable(const ASTContext &Ctx) const;
@@ -467,9 +471,9 @@ class Expr : public Stmt {
   /// variable read.
   bool HasSideEffects(const ASTContext &Ctx) const;
   
-  /// EvaluateAsInt - Call Evaluate and return the folded integer. This
+  /// EvaluateKnownConstInt - Call Evaluate and return the folded integer. This
   /// must be called on an expression that constant folds to an integer.
-  llvm::APSInt EvaluateAsInt(const ASTContext &Ctx) const;
+  llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const;
 
   /// EvaluateAsLValue - Evaluate an expression to see if it's a lvalue
   /// with link time known address.
@@ -688,39 +692,6 @@ class OpaqueValueExpr : public Expr {
   static bool classof(const OpaqueValueExpr *) { return true; }
 };
 
-/// \brief Represents an explicit template argument list in C++, e.g.,
-/// the "<int>" in "sort<int>".
-struct ExplicitTemplateArgumentList {
-  /// \brief The source location of the left angle bracket ('<');
-  SourceLocation LAngleLoc;
-  
-  /// \brief The source location of the right angle bracket ('>');
-  SourceLocation RAngleLoc;
-  
-  /// \brief The number of template arguments in TemplateArgs.
-  /// The actual template arguments (if any) are stored after the
-  /// ExplicitTemplateArgumentList structure.
-  unsigned NumTemplateArgs;
-  
-  /// \brief Retrieve the template arguments
-  TemplateArgumentLoc *getTemplateArgs() {
-    return reinterpret_cast<TemplateArgumentLoc *> (this + 1);
-  }
-  
-  /// \brief Retrieve the template arguments
-  const TemplateArgumentLoc *getTemplateArgs() const {
-    return reinterpret_cast<const TemplateArgumentLoc *> (this + 1);
-  }
-
-  void initializeFrom(const TemplateArgumentListInfo &List);
-  void initializeFrom(const TemplateArgumentListInfo &List,
-                      bool &Dependent, bool &InstantiationDependent,
-                      bool &ContainsUnexpandedParameterPack);
-  void copyInto(TemplateArgumentListInfo &List) const;
-  static std::size_t sizeFor(unsigned NumTemplateArgs);
-  static std::size_t sizeFor(const TemplateArgumentListInfo &List);
-};
-
 /// \brief A reference to a declared variable, function, enum, etc.
 /// [C99 6.5.1p2]
 ///
@@ -804,6 +775,7 @@ class DeclRefExpr : public Expr {
     DeclRefExprBits.HasQualifier = 0;
     DeclRefExprBits.HasExplicitTemplateArgs = 0;
     DeclRefExprBits.HasFoundDecl = 0;
+    DeclRefExprBits.HadMultipleCandidates = 0;
     computeDependence();
   }
 
@@ -887,29 +859,29 @@ class DeclRefExpr : public Expr {
 
   /// \brief Retrieve the explicit template argument list that followed the
   /// member template name.
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs() {
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs() {
     assert(hasExplicitTemplateArgs());
     if (hasFoundDecl())
-      return *reinterpret_cast<ExplicitTemplateArgumentList *>(
+      return *reinterpret_cast<ASTTemplateArgumentListInfo *>(
         &getInternalFoundDecl() + 1);
 
     if (hasQualifier())
-      return *reinterpret_cast<ExplicitTemplateArgumentList *>(
+      return *reinterpret_cast<ASTTemplateArgumentListInfo *>(
         &getInternalQualifierLoc() + 1);
 
-    return *reinterpret_cast<ExplicitTemplateArgumentList *>(this + 1);
+    return *reinterpret_cast<ASTTemplateArgumentListInfo *>(this + 1);
   }
 
   /// \brief Retrieve the explicit template argument list that followed the
   /// member template name.
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     return const_cast<DeclRefExpr *>(this)->getExplicitTemplateArgs();
   }
 
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getExplicitTemplateArgsOpt() const {
+  const ASTTemplateArgumentListInfo *getExplicitTemplateArgsOpt() const {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -957,6 +929,18 @@ class DeclRefExpr : public Expr {
     return getExplicitTemplateArgs().RAngleLoc;
   }
 
+  /// \brief Returns true if this expression refers to a function that
+  /// was resolved from an overloaded set having size greater than 1.
+  bool hadMultipleCandidates() const {
+    return DeclRefExprBits.HadMultipleCandidates;
+  }
+  /// \brief Sets the flag telling whether this expression refers to
+  /// a function that was resolved from an overloaded set having size
+  /// greater than 1.
+  void setHadMultipleCandidates(bool V = true) {
+    DeclRefExprBits.HadMultipleCandidates = V;
+  }
+
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == DeclRefExprClass;
   }
@@ -1112,29 +1096,39 @@ class IntegerLiteral : public Expr {
 };
 
 class CharacterLiteral : public Expr {
+public:
+  enum CharacterKind {
+    Ascii,
+    Wide,
+    UTF16,
+    UTF32
+  };
+
+private:
   unsigned Value;
   SourceLocation Loc;
-  bool IsWide;
+  unsigned Kind : 2;
 public:
   // type should be IntTy
-  CharacterLiteral(unsigned value, bool iswide, QualType type, SourceLocation l)
+  CharacterLiteral(unsigned value, CharacterKind kind, QualType type,
+                   SourceLocation l)
     : Expr(CharacterLiteralClass, type, VK_RValue, OK_Ordinary, false, false,
            false, false),
-      Value(value), Loc(l), IsWide(iswide) {
+      Value(value), Loc(l), Kind(kind) {
   }
 
   /// \brief Construct an empty character literal.
   CharacterLiteral(EmptyShell Empty) : Expr(CharacterLiteralClass, Empty) { }
 
   SourceLocation getLocation() const { return Loc; }
-  bool isWide() const { return IsWide; }
+  CharacterKind getKind() const { return static_cast<CharacterKind>(Kind); }
 
   SourceRange getSourceRange() const { return SourceRange(Loc); }
 
   unsigned getValue() const { return Value; }
 
   void setLocation(SourceLocation Location) { Loc = Location; }
-  void setWide(bool W) { IsWide = W; }
+  void setKind(CharacterKind kind) { Kind = kind; }
   void setValue(unsigned Val) { Value = Val; }
 
   static bool classof(const Stmt *T) {
@@ -1243,13 +1237,23 @@ class ImaginaryLiteral : public Expr {
 /// In this case, getByteLength() will return 6, but the string literal will
 /// have type "char[2]".
 class StringLiteral : public Expr {
+public:
+  enum StringKind {
+    Ascii,
+    Wide,
+    UTF8,
+    UTF16,
+    UTF32
+  };
+
+private:
   friend class ASTStmtReader;
 
   const char *StrData;
   unsigned ByteLength;
-  bool IsWide;
-  bool IsPascal;
   unsigned NumConcatenated;
+  unsigned Kind : 3;
+  bool IsPascal : 1;
   SourceLocation TokLocs[1];
 
   StringLiteral(QualType Ty) :
@@ -1259,33 +1263,39 @@ class StringLiteral : public Expr {
 public:
   /// This is the "fully general" constructor that allows representation of
   /// strings formed from multiple concatenated tokens.
-  static StringLiteral *Create(ASTContext &C, llvm::StringRef Str, bool Wide,
+  static StringLiteral *Create(ASTContext &C, StringRef Str, StringKind Kind,
                                bool Pascal, QualType Ty,
                                const SourceLocation *Loc, unsigned NumStrs);
 
   /// Simple constructor for string literals made from one token.
-  static StringLiteral *Create(ASTContext &C, llvm::StringRef Str, bool Wide, 
-                               bool Pascal, QualType Ty, SourceLocation Loc) {
-    return Create(C, Str, Wide, Pascal, Ty, &Loc, 1);
+  static StringLiteral *Create(ASTContext &C, StringRef Str, StringKind Kind,
+                               bool Pascal, QualType Ty,
+                               SourceLocation Loc) {
+    return Create(C, Str, Kind, Pascal, Ty, &Loc, 1);
   }
 
   /// \brief Construct an empty string literal.
   static StringLiteral *CreateEmpty(ASTContext &C, unsigned NumStrs);
 
-  llvm::StringRef getString() const {
-    return llvm::StringRef(StrData, ByteLength);
+  StringRef getString() const {
+    return StringRef(StrData, ByteLength);
   }
 
   unsigned getByteLength() const { return ByteLength; }
 
   /// \brief Sets the string data to the given string data.
-  void setString(ASTContext &C, llvm::StringRef Str);
+  void setString(ASTContext &C, StringRef Str);
 
-  bool isWide() const { return IsWide; }
+  StringKind getKind() const { return static_cast<StringKind>(Kind); }
+  bool isAscii() const { return Kind == Ascii; }
+  bool isWide() const { return Kind == Wide; }
+  bool isUTF8() const { return Kind == UTF8; }
+  bool isUTF16() const { return Kind == UTF16; }
+  bool isUTF32() const { return Kind == UTF32; }
   bool isPascal() const { return IsPascal; }
-  
+
   bool containsNonAsciiOrNull() const {
-    llvm::StringRef Str = getString();
+    StringRef Str = getString();
     for (unsigned i = 0, e = Str.size(); i != e; ++i)
       if (!isascii(Str[i]) || !Str[i])
         return true;
@@ -2029,6 +2039,10 @@ class MemberExpr : public Expr {
   /// the MemberNameQualifier structure.
   bool HasExplicitTemplateArgumentList : 1;
 
+  /// \brief True if this member expression refers to a method that
+  /// was resolved from an overloaded set having size greater than 1.
+  bool HadMultipleCandidates : 1;
+
   /// \brief Retrieve the qualifier that preceded the member name, if any.
   MemberNameQualifier *getMemberQualifier() {
     assert(HasQualifierOrFoundDecl);
@@ -2051,7 +2065,8 @@ class MemberExpr : public Expr {
            base->containsUnexpandedParameterPack()),
       Base(base), MemberDecl(memberdecl), MemberLoc(NameInfo.getLoc()),
       MemberDNLoc(NameInfo.getInfo()), IsArrow(isarrow),
-      HasQualifierOrFoundDecl(false), HasExplicitTemplateArgumentList(false) {
+      HasQualifierOrFoundDecl(false), HasExplicitTemplateArgumentList(false),
+      HadMultipleCandidates(false) {
     assert(memberdecl->getDeclName() == NameInfo.getName());
   }
 
@@ -2068,7 +2083,8 @@ class MemberExpr : public Expr {
            base->containsUnexpandedParameterPack()),
       Base(base), MemberDecl(memberdecl), MemberLoc(l), MemberDNLoc(),
       IsArrow(isarrow),
-      HasQualifierOrFoundDecl(false), HasExplicitTemplateArgumentList(false) {}
+      HasQualifierOrFoundDecl(false), HasExplicitTemplateArgumentList(false),
+      HadMultipleCandidates(false) {}
 
   static MemberExpr *Create(ASTContext &C, Expr *base, bool isarrow,
                             NestedNameSpecifierLoc QualifierLoc,
@@ -2136,26 +2152,26 @@ class MemberExpr : public Expr {
   /// \brief Retrieve the explicit template argument list that
   /// follow the member template name.  This must only be called on an
   /// expression with explicit template arguments.
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs() {
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs() {
     assert(HasExplicitTemplateArgumentList);
     if (!HasQualifierOrFoundDecl)
-      return *reinterpret_cast<ExplicitTemplateArgumentList *>(this + 1);
+      return *reinterpret_cast<ASTTemplateArgumentListInfo *>(this + 1);
 
-    return *reinterpret_cast<ExplicitTemplateArgumentList *>(
+    return *reinterpret_cast<ASTTemplateArgumentListInfo *>(
                                                       getMemberQualifier() + 1);
   }
 
   /// \brief Retrieve the explicit template argument list that
   /// followed the member template name.  This must only be called on
   /// an expression with explicit template arguments.
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     return const_cast<MemberExpr *>(this)->getExplicitTemplateArgs();
   }
 
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getOptionalExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo *getOptionalExplicitTemplateArgs() const {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -2218,7 +2234,19 @@ class MemberExpr : public Expr {
   bool isImplicitAccess() const {
     return getBase() && getBase()->isImplicitCXXThis();
   }
-  
+
+  /// \brief Returns true if this member expression refers to a method that
+  /// was resolved from an overloaded set having size greater than 1.
+  bool hadMultipleCandidates() const {
+    return HadMultipleCandidates;
+  }
+  /// \brief Sets the flag telling whether this expression refers to
+  /// a method that was resolved from an overloaded set having size
+  /// greater than 1.
+  void setHadMultipleCandidates(bool V = true) {
+    HadMultipleCandidates = V;
+  }
+
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == MemberExprClass;
   }
@@ -2301,68 +2329,7 @@ class CastExpr : public Expr {
 private:
   Stmt *Op;
 
-  void CheckCastConsistency() const {
-#ifndef NDEBUG
-    switch (getCastKind()) {
-    case CK_DerivedToBase:
-    case CK_UncheckedDerivedToBase:
-    case CK_DerivedToBaseMemberPointer:
-    case CK_BaseToDerived:
-    case CK_BaseToDerivedMemberPointer:
-      assert(!path_empty() && "Cast kind should have a base path!");
-      break;
-
-    // These should not have an inheritance path.
-    case CK_BitCast:
-    case CK_Dynamic:
-    case CK_ToUnion:
-    case CK_ArrayToPointerDecay:
-    case CK_FunctionToPointerDecay:
-    case CK_NullToMemberPointer:
-    case CK_NullToPointer:
-    case CK_ConstructorConversion:
-    case CK_IntegralToPointer:
-    case CK_PointerToIntegral:
-    case CK_ToVoid:
-    case CK_VectorSplat:
-    case CK_IntegralCast:
-    case CK_IntegralToFloating:
-    case CK_FloatingToIntegral:
-    case CK_FloatingCast:
-    case CK_AnyPointerToObjCPointerCast:
-    case CK_AnyPointerToBlockPointerCast:
-    case CK_ObjCObjectLValueCast:
-    case CK_FloatingRealToComplex:
-    case CK_FloatingComplexToReal:
-    case CK_FloatingComplexCast:
-    case CK_FloatingComplexToIntegralComplex:
-    case CK_IntegralRealToComplex:
-    case CK_IntegralComplexToReal:
-    case CK_IntegralComplexCast:
-    case CK_IntegralComplexToFloatingComplex:
-    case CK_ObjCProduceObject:
-    case CK_ObjCConsumeObject:
-    case CK_ObjCReclaimReturnedObject:
-      assert(!getType()->isBooleanType() && "unheralded conversion to bool");
-      // fallthrough to check for null base path
-
-    case CK_Dependent:
-    case CK_LValueToRValue:
-    case CK_GetObjCProperty:
-    case CK_NoOp:
-    case CK_PointerToBoolean:
-    case CK_IntegralToBoolean:
-    case CK_FloatingToBoolean:
-    case CK_MemberPointerToBoolean:
-    case CK_FloatingComplexToBoolean:
-    case CK_IntegralComplexToBoolean:
-    case CK_LValueBitCast:            // -> bool&
-    case CK_UserDefinedConversion:    // operator bool()
-      assert(path_empty() && "Cast kind should not have a base path!");
-      break;
-    }
-#endif
-  }
+  void CheckCastConsistency() const;
 
   const CXXBaseSpecifier * const *path_buffer() const {
     return const_cast<CastExpr*>(this)->path_buffer();
@@ -2393,7 +2360,9 @@ class CastExpr : public Expr {
     assert(kind != CK_Invalid && "creating cast with invalid cast kind");
     CastExprBits.Kind = kind;
     setBasePathSize(BasePathSize);
+#ifndef NDEBUG
     CheckCastConsistency();
+#endif
   }
 
   /// \brief Construct an empty cast.
@@ -2746,7 +2715,7 @@ class BinaryOperator : public Expr {
 
 /// CompoundAssignOperator - For compound assignments (e.g. +=), we keep
 /// track of the type the operation is performed in.  Due to the semantics of
-/// these operators, the operands are promoted, the aritmetic performed, an
+/// these operators, the operands are promoted, the arithmetic performed, an
 /// implicit conversion back to the result type done, then the assignment takes
 /// place.  This captures the intermediate type which the computation is done
 /// in.
@@ -3134,7 +3103,7 @@ class ShuffleVectorExpr : public Expr {
 
   unsigned getShuffleMaskIdx(ASTContext &Ctx, unsigned N) {
     assert((N < NumExprs - 2) && "Shuffle idx out of range!");
-    return getExpr(N+2)->EvaluateAsInt(Ctx).getZExtValue();
+    return getExpr(N+2)->EvaluateKnownConstInt(Ctx).getZExtValue();
   }
 
   // Iterators
@@ -4045,7 +4014,7 @@ class ExtVectorElementExpr : public Expr {
 
   /// getEncodedElementAccess - Encode the elements accessed into an llvm
   /// aggregate Constant of ConstantInt(s).
-  void getEncodedElementAccess(llvm::SmallVectorImpl<unsigned> &Elts) const;
+  void getEncodedElementAccess(SmallVectorImpl<unsigned> &Elts) const;
 
   SourceRange getSourceRange() const {
     return SourceRange(getBase()->getLocStart(), AccessorLoc);
@@ -4193,6 +4162,101 @@ class AsTypeExpr : public Expr { // Should this be an ExplicitCastExpr?
   // Iterators
   child_range children() { return child_range(&SrcExpr, &SrcExpr+1); }
 };
+
+/// AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*,
+/// __atomic_load, __atomic_store, and __atomic_compare_exchange_*, for the
+/// similarly-named C++0x instructions.  All of these instructions take one
+/// primary pointer and at least one memory order.
+class AtomicExpr : public Expr {
+public:
+  enum AtomicOp { Load, Store, CmpXchgStrong, CmpXchgWeak, Xchg,
+                  Add, Sub, And, Or, Xor };
+private:
+  enum { PTR, ORDER, VAL1, ORDER_FAIL, VAL2, END_EXPR };
+  Stmt* SubExprs[END_EXPR];
+  unsigned NumSubExprs;
+  SourceLocation BuiltinLoc, RParenLoc;
+  AtomicOp Op;
+
+public:
+  AtomicExpr(SourceLocation BLoc, Expr **args, unsigned nexpr, QualType t,
+             AtomicOp op, SourceLocation RP);
+
+  /// \brief Build an empty AtomicExpr.
+  explicit AtomicExpr(EmptyShell Empty) : Expr(AtomicExprClass, Empty) { }
+
+  Expr *getPtr() const {
+    return cast<Expr>(SubExprs[PTR]);
+  }
+  void setPtr(Expr *E) {
+    SubExprs[PTR] = E;
+  }
+  Expr *getOrder() const {
+    return cast<Expr>(SubExprs[ORDER]);
+  }
+  void setOrder(Expr *E) {
+    SubExprs[ORDER] = E;
+  }
+  Expr *getVal1() const {
+    assert(NumSubExprs >= 3);
+    return cast<Expr>(SubExprs[VAL1]);
+  }
+  void setVal1(Expr *E) {
+    assert(NumSubExprs >= 3);
+    SubExprs[VAL1] = E;
+  }
+  Expr *getOrderFail() const {
+    assert(NumSubExprs == 5);
+    return cast<Expr>(SubExprs[ORDER_FAIL]);
+  }
+  void setOrderFail(Expr *E) {
+    assert(NumSubExprs == 5);
+    SubExprs[ORDER_FAIL] = E;
+  }
+  Expr *getVal2() const {
+    assert(NumSubExprs == 5);
+    return cast<Expr>(SubExprs[VAL2]);
+  }
+  void setVal2(Expr *E) {
+    assert(NumSubExprs == 5);
+    SubExprs[VAL2] = E;
+  }
+
+  AtomicOp getOp() const { return Op; }
+  void setOp(AtomicOp op) { Op = op; }
+  unsigned getNumSubExprs() { return NumSubExprs; }
+  void setNumSubExprs(unsigned num) { NumSubExprs = num; }
+
+  Expr **getSubExprs() { return reinterpret_cast<Expr **>(SubExprs); }
+
+  bool isVolatile() const {
+    return getPtr()->getType()->getPointeeType().isVolatileQualified();
+  }
+
+  bool isCmpXChg() const {
+    return getOp() == AtomicExpr::CmpXchgStrong ||
+           getOp() == AtomicExpr::CmpXchgWeak;
+  }
+
+  SourceLocation getBuiltinLoc() const { return BuiltinLoc; }
+  void setBuiltinLoc(SourceLocation L) { BuiltinLoc = L; }
+
+  SourceLocation getRParenLoc() const { return RParenLoc; }
+  void setRParenLoc(SourceLocation L) { RParenLoc = L; }
+
+  SourceRange getSourceRange() const {
+    return SourceRange(BuiltinLoc, RParenLoc);
+  }
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == AtomicExprClass;
+  }
+  static bool classof(const AtomicExpr *) { return true; }
+
+  // Iterators
+  child_range children() {
+    return child_range(SubExprs, SubExprs+NumSubExprs);
+  }
+};
 }  // end namespace clang
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ExprCXX.h b/contrib/llvm/tools/clang/include/clang/AST/ExprCXX.h
index 19117040ef96..3cc09cdf5e45 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ExprCXX.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ExprCXX.h
@@ -807,37 +807,41 @@ class CXXConstructExpr : public Expr {
 
   SourceLocation Loc;
   SourceRange ParenRange;
+  unsigned NumArgs : 16;
   bool Elidable : 1;
+  bool HadMultipleCandidates : 1;
   bool ZeroInitialization : 1;
   unsigned ConstructKind : 2;
   Stmt **Args;
-  unsigned NumArgs;
 
 protected:
   CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
                    SourceLocation Loc,
                    CXXConstructorDecl *d, bool elidable,
                    Expr **args, unsigned numargs,
+                   bool HadMultipleCandidates,
                    bool ZeroInitialization = false,
                    ConstructionKind ConstructKind = CK_Complete,
                    SourceRange ParenRange = SourceRange());
 
   /// \brief Construct an empty C++ construction expression.
   CXXConstructExpr(StmtClass SC, EmptyShell Empty)
-    : Expr(SC, Empty), Constructor(0), Elidable(0), ZeroInitialization(0),
-      ConstructKind(0), Args(0), NumArgs(0) { }
+    : Expr(SC, Empty), Constructor(0), NumArgs(0), Elidable(0),
+      HadMultipleCandidates(false), ZeroInitialization(0),
+      ConstructKind(0), Args(0) { }
 
 public:
   /// \brief Construct an empty C++ construction expression.
   explicit CXXConstructExpr(EmptyShell Empty)
     : Expr(CXXConstructExprClass, Empty), Constructor(0),
-      Elidable(0), ZeroInitialization(0),
-      ConstructKind(0), Args(0), NumArgs(0) { }
+      NumArgs(0), Elidable(0), HadMultipleCandidates(false),
+      ZeroInitialization(0), ConstructKind(0), Args(0) { }
 
   static CXXConstructExpr *Create(ASTContext &C, QualType T,
                                   SourceLocation Loc,
                                   CXXConstructorDecl *D, bool Elidable,
                                   Expr **Args, unsigned NumArgs,
+                                  bool HadMultipleCandidates,
                                   bool ZeroInitialization = false,
                                   ConstructionKind ConstructKind = CK_Complete,
                                   SourceRange ParenRange = SourceRange());
@@ -852,7 +856,12 @@ class CXXConstructExpr : public Expr {
   /// \brief Whether this construction is elidable.
   bool isElidable() const { return Elidable; }
   void setElidable(bool E) { Elidable = E; }
-  
+
+  /// \brief Whether the referred constructor was resolved from
+  /// an overloaded set having size greater than 1.
+  bool hadMultipleCandidates() const { return HadMultipleCandidates; }
+  void setHadMultipleCandidates(bool V) { HadMultipleCandidates = V; }
+
   /// \brief Whether this construction first requires
   /// zero-initialization before the initializer is called.
   bool requiresZeroInitialization() const { return ZeroInitialization; }
@@ -980,6 +989,7 @@ class CXXTemporaryObjectExpr : public CXXConstructExpr {
                          TypeSourceInfo *Type,
                          Expr **Args,unsigned NumArgs,
                          SourceRange parenRange,
+                         bool HadMultipleCandidates,
                          bool ZeroInitialization = false);
   explicit CXXTemporaryObjectExpr(EmptyShell Empty)
     : CXXConstructExpr(CXXTemporaryObjectExprClass, Empty), Type() { }
@@ -1049,8 +1059,11 @@ class CXXNewExpr : public Expr {
   // If this is an array allocation, does the usual deallocation
   // function for the allocated type want to know the allocated size?
   bool UsualArrayDeleteWantsSize : 1;
+  // Whether the referred constructor (if any) was resolved from an
+  // overload set having size greater than 1.
+  bool HadMultipleCandidates : 1;
   // The number of placement new arguments.
-  unsigned NumPlacementArgs : 14;
+  unsigned NumPlacementArgs : 13;
   // The number of constructor arguments. This may be 1 even for non-class
   // types; use the pseudo copy constructor.
   unsigned NumConstructorArgs : 14;
@@ -1086,6 +1099,7 @@ class CXXNewExpr : public Expr {
              SourceRange TypeIdParens,
              Expr *arraySize, CXXConstructorDecl *constructor, bool initializer,
              Expr **constructorArgs, unsigned numConsArgs,
+             bool HadMultipleCandidates,
              FunctionDecl *operatorDelete, bool usualArrayDeleteWantsSize,
              QualType ty, TypeSourceInfo *AllocatedTypeInfo,
              SourceLocation startLoc, SourceLocation endLoc,
@@ -1174,6 +1188,11 @@ class CXXNewExpr : public Expr {
     return cast<Expr>(SubExprs[Array + NumPlacementArgs + i]);
   }
 
+  /// \brief Whether the new expression refers a constructor that was
+  /// resolved from an overloaded set having size greater than 1.
+  bool hadMultipleCandidates() const { return HadMultipleCandidates; }
+  void setHadMultipleCandidates(bool V) { HadMultipleCandidates = V; }
+
   typedef ExprIterator arg_iterator;
   typedef ConstExprIterator const_arg_iterator;
 
@@ -1842,16 +1861,16 @@ class OverloadExpr : public Expr {
   /// template argument list, e.g. f<int>.
   bool hasExplicitTemplateArgs() const { return HasExplicitTemplateArgs; }
 
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs(); // defined far below
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs(); // defined far below
 
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     return const_cast<OverloadExpr*>(this)->getExplicitTemplateArgs();
   }
 
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getOptionalExplicitTemplateArgs() {
+  const ASTTemplateArgumentListInfo *getOptionalExplicitTemplateArgs() {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -1969,21 +1988,21 @@ class UnresolvedLookupExpr : public OverloadExpr {
   // nodes, users are *forbidden* from calling these methods on objects
   // without explicit template arguments.
 
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs() {
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs() {
     assert(hasExplicitTemplateArgs());
-    return *reinterpret_cast<ExplicitTemplateArgumentList*>(this + 1);
+    return *reinterpret_cast<ASTTemplateArgumentListInfo*>(this + 1);
   }
 
   /// Gets a reference to the explicit template argument list.
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     assert(hasExplicitTemplateArgs());
-    return *reinterpret_cast<const ExplicitTemplateArgumentList*>(this + 1);
+    return *reinterpret_cast<const ASTTemplateArgumentListInfo*>(this + 1);
   }
 
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getOptionalExplicitTemplateArgs() {
+  const ASTTemplateArgumentListInfo *getOptionalExplicitTemplateArgs() {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -2094,21 +2113,21 @@ class DependentScopeDeclRefExpr : public Expr {
   // nodes, users are *forbidden* from calling these methods on objects
   // without explicit template arguments.
 
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs() {
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs() {
     assert(hasExplicitTemplateArgs());
-    return *reinterpret_cast<ExplicitTemplateArgumentList*>(this + 1);
+    return *reinterpret_cast<ASTTemplateArgumentListInfo*>(this + 1);
   }
 
   /// Gets a reference to the explicit template argument list.
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     assert(hasExplicitTemplateArgs());
-    return *reinterpret_cast<const ExplicitTemplateArgumentList*>(this + 1);
+    return *reinterpret_cast<const ASTTemplateArgumentListInfo*>(this + 1);
   }
 
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getOptionalExplicitTemplateArgs() {
+  const ASTTemplateArgumentListInfo *getOptionalExplicitTemplateArgs() {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -2470,14 +2489,14 @@ class CXXDependentScopeMemberExpr : public Expr {
 
   /// \brief Retrieve the explicit template argument list that followed the
   /// member template name, if any.
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs() {
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs() {
     assert(HasExplicitTemplateArgs);
-    return *reinterpret_cast<ExplicitTemplateArgumentList *>(this + 1);
+    return *reinterpret_cast<ASTTemplateArgumentListInfo *>(this + 1);
   }
 
   /// \brief Retrieve the explicit template argument list that followed the
   /// member template name, if any.
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     return const_cast<CXXDependentScopeMemberExpr *>(this)
              ->getExplicitTemplateArgs();
   }
@@ -2485,7 +2504,7 @@ class CXXDependentScopeMemberExpr : public Expr {
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getOptionalExplicitTemplateArgs() {
+  const ASTTemplateArgumentListInfo *getOptionalExplicitTemplateArgs() {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -2663,22 +2682,22 @@ class UnresolvedMemberExpr : public OverloadExpr {
 
   /// \brief Retrieve the explicit template argument list that followed the
   /// member template name.
-  ExplicitTemplateArgumentList &getExplicitTemplateArgs() {
+  ASTTemplateArgumentListInfo &getExplicitTemplateArgs() {
     assert(hasExplicitTemplateArgs());
-    return *reinterpret_cast<ExplicitTemplateArgumentList *>(this + 1);
+    return *reinterpret_cast<ASTTemplateArgumentListInfo *>(this + 1);
   }
 
   /// \brief Retrieve the explicit template argument list that followed the
   /// member template name, if any.
-  const ExplicitTemplateArgumentList &getExplicitTemplateArgs() const {
+  const ASTTemplateArgumentListInfo &getExplicitTemplateArgs() const {
     assert(hasExplicitTemplateArgs());
-    return *reinterpret_cast<const ExplicitTemplateArgumentList *>(this + 1);
+    return *reinterpret_cast<const ASTTemplateArgumentListInfo *>(this + 1);
   }
 
   /// \brief Retrieves the optional explicit template arguments.
   /// This points to the same data as getExplicitTemplateArgs(), but
   /// returns null if there are no explicit template arguments.
-  const ExplicitTemplateArgumentList *getOptionalExplicitTemplateArgs() {
+  const ASTTemplateArgumentListInfo *getOptionalExplicitTemplateArgs() {
     if (!hasExplicitTemplateArgs()) return 0;
     return &getExplicitTemplateArgs();
   }
@@ -2856,7 +2875,7 @@ class PackExpansionExpr : public Expr {
   }
 };
   
-inline ExplicitTemplateArgumentList &OverloadExpr::getExplicitTemplateArgs() {
+inline ASTTemplateArgumentListInfo &OverloadExpr::getExplicitTemplateArgs() {
   if (isa<UnresolvedLookupExpr>(this))
     return cast<UnresolvedLookupExpr>(this)->getExplicitTemplateArgs();
   else
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ExprObjC.h b/contrib/llvm/tools/clang/include/clang/AST/ExprObjC.h
index 49d4cfe67626..55726eb4ae65 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ExprObjC.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ExprObjC.h
@@ -16,6 +16,7 @@
 
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/SelectorLocationsKind.h"
 #include "clang/Basic/IdentifierTable.h"
 
 namespace clang {
@@ -359,12 +360,12 @@ class ObjCPropertyRefExpr : public Expr {
     QualType ArgType;
     if (isImplicitProperty()) {
       const ObjCMethodDecl *Setter = getImplicitPropertySetter();
-      ObjCMethodDecl::param_iterator P = Setter->param_begin(); 
+      ObjCMethodDecl::param_const_iterator P = Setter->param_begin(); 
       ArgType = (*P)->getType();
     } else {
       if (ObjCPropertyDecl *PDecl = getExplicitProperty())
         if (const ObjCMethodDecl *Setter = PDecl->getSetterMethodDecl()) {
-          ObjCMethodDecl::param_iterator P = Setter->param_begin(); 
+          ObjCMethodDecl::param_const_iterator P = Setter->param_begin(); 
           ArgType = (*P)->getType();
         }
       if (ArgType.isNull())
@@ -444,9 +445,21 @@ class ObjCPropertyRefExpr : public Expr {
 /// class, and can be distinguished via \c getReceiverKind(). Example:
 ///
 class ObjCMessageExpr : public Expr {
+  /// \brief Stores either the selector that this message is sending
+  /// to (when \c HasMethod is zero) or an \c ObjCMethodDecl pointer
+  /// referring to the method that we type-checked against.
+  uintptr_t SelectorOrMethod;
+
+  enum { NumArgsBitWidth = 16 };
+
   /// \brief The number of arguments in the message send, not
   /// including the receiver.
-  unsigned NumArgs : 16;
+  unsigned NumArgs : NumArgsBitWidth;
+  
+  void setNumArgs(unsigned Num) {
+    assert((Num >> NumArgsBitWidth) == 0 && "Num of args is out of range!");
+    NumArgs = Num;
+  }
 
   /// \brief The kind of message send this is, which is one of the
   /// ReceiverKind values.
@@ -464,26 +477,24 @@ class ObjCMessageExpr : public Expr {
   /// \brief Whether this message send is a "delegate init call",
   /// i.e. a call of an init method on self from within an init method.
   unsigned IsDelegateInitCall : 1;
+  
+  /// \brief Whether the locations of the selector identifiers are in a
+  /// "standard" position, a enum SelectorLocationsKind.
+  unsigned SelLocsKind : 2;
 
   /// \brief When the message expression is a send to 'super', this is
   /// the location of the 'super' keyword.
   SourceLocation SuperLoc;
 
-  /// \brief Stores either the selector that this message is sending
-  /// to (when \c HasMethod is zero) or an \c ObjCMethodDecl pointer
-  /// referring to the method that we type-checked against.
-  uintptr_t SelectorOrMethod;
-
-  /// \brief Location of the selector.
-  SourceLocation SelectorLoc;
-
   /// \brief The source locations of the open and close square
   /// brackets ('[' and ']', respectively).
   SourceLocation LBracLoc, RBracLoc;
 
   ObjCMessageExpr(EmptyShell Empty, unsigned NumArgs)
-    : Expr(ObjCMessageExprClass, Empty), NumArgs(NumArgs), Kind(0), 
-      HasMethod(0), IsDelegateInitCall(0), SelectorOrMethod(0) { }
+    : Expr(ObjCMessageExprClass, Empty), SelectorOrMethod(0), Kind(0), 
+      HasMethod(0), IsDelegateInitCall(0) {
+    setNumArgs(NumArgs);
+  }
 
   ObjCMessageExpr(QualType T, ExprValueKind VK,
                   SourceLocation LBracLoc,
@@ -491,27 +502,34 @@ class ObjCMessageExpr : public Expr {
                   bool IsInstanceSuper,
                   QualType SuperType,
                   Selector Sel, 
-                  SourceLocation SelLoc,
+                  ArrayRef<SourceLocation> SelLocs,
+                  SelectorLocationsKind SelLocsK,
                   ObjCMethodDecl *Method,
-                  Expr **Args, unsigned NumArgs,
+                  ArrayRef<Expr *> Args,
                   SourceLocation RBracLoc);
   ObjCMessageExpr(QualType T, ExprValueKind VK,
                   SourceLocation LBracLoc,
                   TypeSourceInfo *Receiver,
                   Selector Sel, 
-                  SourceLocation SelLoc,
+                  ArrayRef<SourceLocation> SelLocs,
+                  SelectorLocationsKind SelLocsK,
                   ObjCMethodDecl *Method,
-                  Expr **Args, unsigned NumArgs,
+                  ArrayRef<Expr *> Args,
                   SourceLocation RBracLoc);
   ObjCMessageExpr(QualType T, ExprValueKind VK,
                   SourceLocation LBracLoc,
                   Expr *Receiver,
                   Selector Sel, 
-                  SourceLocation SelLoc,
+                  ArrayRef<SourceLocation> SelLocs,
+                  SelectorLocationsKind SelLocsK,
                   ObjCMethodDecl *Method,
-                  Expr **Args, unsigned NumArgs,
+                  ArrayRef<Expr *> Args,
                   SourceLocation RBracLoc);
 
+  void initArgsAndSelLocs(ArrayRef<Expr *> Args,
+                          ArrayRef<SourceLocation> SelLocs,
+                          SelectorLocationsKind SelLocsK);
+
   /// \brief Retrieve the pointer value of the message receiver.
   void *getReceiverPointer() const {
     return *const_cast<void **>(
@@ -523,6 +541,40 @@ class ObjCMessageExpr : public Expr {
     *reinterpret_cast<void **>(this + 1) = Value;
   }
 
+  SelectorLocationsKind getSelLocsKind() const {
+    return (SelectorLocationsKind)SelLocsKind;
+  }
+  bool hasStandardSelLocs() const {
+    return getSelLocsKind() != SelLoc_NonStandard;
+  }
+
+  /// \brief Get a pointer to the stored selector identifiers locations array.
+  /// No locations will be stored if HasStandardSelLocs is true.
+  SourceLocation *getStoredSelLocs() {
+    return reinterpret_cast<SourceLocation*>(getArgs() + getNumArgs());
+  }
+  const SourceLocation *getStoredSelLocs() const {
+    return reinterpret_cast<const SourceLocation*>(getArgs() + getNumArgs());
+  }
+
+  /// \brief Get the number of stored selector identifiers locations.
+  /// No locations will be stored if HasStandardSelLocs is true.
+  unsigned getNumStoredSelLocs() const {
+    if (hasStandardSelLocs())
+      return 0;
+    return getNumSelectorLocs();
+  }
+
+  static ObjCMessageExpr *alloc(ASTContext &C,
+                                ArrayRef<Expr *> Args,
+                                SourceLocation RBraceLoc,
+                                ArrayRef<SourceLocation> SelLocs,
+                                Selector Sel,
+                                SelectorLocationsKind &SelLocsK);
+  static ObjCMessageExpr *alloc(ASTContext &C,
+                                unsigned NumArgs,
+                                unsigned NumStoredSelLocs);
+
 public:
   /// \brief The kind of receiver this message is sending to.
   enum ReceiverKind {
@@ -570,9 +622,9 @@ class ObjCMessageExpr : public Expr {
                                  bool IsInstanceSuper,
                                  QualType SuperType,
                                  Selector Sel, 
-                                 SourceLocation SelLoc,
+                                 ArrayRef<SourceLocation> SelLocs,
                                  ObjCMethodDecl *Method,
-                                 Expr **Args, unsigned NumArgs,
+                                 ArrayRef<Expr *> Args,
                                  SourceLocation RBracLoc);
 
   /// \brief Create a class message send.
@@ -605,9 +657,9 @@ class ObjCMessageExpr : public Expr {
                                  SourceLocation LBracLoc,
                                  TypeSourceInfo *Receiver,
                                  Selector Sel, 
-                                 SourceLocation SelLoc,
+                                 ArrayRef<SourceLocation> SelLocs,
                                  ObjCMethodDecl *Method,
-                                 Expr **Args, unsigned NumArgs,
+                                 ArrayRef<Expr *> Args,
                                  SourceLocation RBracLoc);
 
   /// \brief Create an instance message send.
@@ -640,9 +692,9 @@ class ObjCMessageExpr : public Expr {
                                  SourceLocation LBracLoc,
                                  Expr *Receiver,
                                  Selector Sel, 
-                                 SourceLocation SelLoc,
+                                 ArrayRef<SourceLocation> SeLocs,
                                  ObjCMethodDecl *Method,
-                                 Expr **Args, unsigned NumArgs,
+                                 ArrayRef<Expr *> Args,
                                  SourceLocation RBracLoc);
 
   /// \brief Create an empty Objective-C message expression, to be
@@ -652,7 +704,9 @@ class ObjCMessageExpr : public Expr {
   ///
   /// \param NumArgs The number of message arguments, not including
   /// the receiver.
-  static ObjCMessageExpr *CreateEmpty(ASTContext &Context, unsigned NumArgs);
+  static ObjCMessageExpr *CreateEmpty(ASTContext &Context,
+                                      unsigned NumArgs,
+                                      unsigned NumStoredSelLocs);
 
   /// \brief Determine the kind of receiver that this message is being
   /// sent to.
@@ -822,7 +876,27 @@ class ObjCMessageExpr : public Expr {
 
   SourceLocation getLeftLoc() const { return LBracLoc; }
   SourceLocation getRightLoc() const { return RBracLoc; }
-  SourceLocation getSelectorLoc() const { return SelectorLoc; }
+
+  SourceLocation getSelectorStartLoc() const { return getSelectorLoc(0); }
+  SourceLocation getSelectorLoc(unsigned Index) const {
+    assert(Index < getNumSelectorLocs() && "Index out of range!");
+    if (hasStandardSelLocs())
+      return getStandardSelectorLoc(Index, getSelector(),
+                                   getSelLocsKind() == SelLoc_StandardWithSpace,
+                               llvm::makeArrayRef(const_cast<Expr**>(getArgs()),
+                                                  getNumArgs()),
+                                   RBracLoc);
+    return getStoredSelLocs()[Index];
+  }
+
+  void getSelectorLocs(SmallVectorImpl<SourceLocation> &SelLocs) const;
+
+  unsigned getNumSelectorLocs() const {
+    Selector Sel = getSelector();
+    if (Sel.isUnarySelector())
+      return 1;
+    return Sel.getNumArgs();
+  }
 
   void setSourceRange(SourceRange R) {
     LBracLoc = R.getBegin();
@@ -989,10 +1063,10 @@ class ObjCBridgedCastExpr : public ExplicitCastExpr {
   
 public:
   ObjCBridgedCastExpr(SourceLocation LParenLoc, ObjCBridgeCastKind Kind,
-                      SourceLocation BridgeKeywordLoc, TypeSourceInfo *TSInfo,
-                      Expr *Operand)
+                      CastKind CK, SourceLocation BridgeKeywordLoc,
+                      TypeSourceInfo *TSInfo, Expr *Operand)
     : ExplicitCastExpr(ObjCBridgedCastExprClass, TSInfo->getType(), VK_RValue,
-                       CK_BitCast, Operand, 0, TSInfo),
+                       CK, Operand, 0, TSInfo),
       LParenLoc(LParenLoc), BridgeKeywordLoc(BridgeKeywordLoc), Kind(Kind) { }
   
   /// \brief Construct an empty Objective-C bridged cast.
@@ -1007,7 +1081,7 @@ class ObjCBridgedCastExpr : public ExplicitCastExpr {
   }
   
   /// \brief Retrieve the kind of bridge being performed as a string.
-  llvm::StringRef getBridgeKindName() const;
+  StringRef getBridgeKindName() const;
   
   /// \brief The location of the bridge keyword.
   SourceLocation getBridgeKeywordLoc() const { return BridgeKeywordLoc; }
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ExternalASTSource.h b/contrib/llvm/tools/clang/include/clang/AST/ExternalASTSource.h
index ef1f1618ba1d..96d14b29549b 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ExternalASTSource.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ExternalASTSource.h
@@ -15,11 +15,6 @@
 #define LLVM_CLANG_AST_EXTERNAL_AST_SOURCE_H
 
 #include "clang/AST/DeclBase.h"
-#include <cassert>
-
-namespace llvm {
-template <class T> class SmallVectorImpl;
-}
 
 namespace clang {
 
@@ -129,16 +124,6 @@ class ExternalASTSource {
   virtual DeclContextLookupResult
   FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name);
 
-  /// \brief Deserialize all the visible declarations from external storage.
-  ///
-  /// Name lookup deserializes visible declarations lazily, thus a DeclContext
-  /// may not have a complete name lookup table. This function deserializes
-  /// the rest of visible declarations from the external storage and completes
-  /// the name lookup table of the DeclContext.
-  ///
-  /// The default implementation of this method is a no-op.
-  virtual void MaterializeVisibleDecls(const DeclContext *DC);
-
   /// \brief Finds all declarations lexically contained within the given
   /// DeclContext, after applying an optional filter predicate.
   ///
@@ -151,20 +136,20 @@ class ExternalASTSource {
   /// The default implementation of this method is a no-op.
   virtual ExternalLoadResult FindExternalLexicalDecls(const DeclContext *DC,
                                         bool (*isKindWeWant)(Decl::Kind),
-                                        llvm::SmallVectorImpl<Decl*> &Result);
+                                        SmallVectorImpl<Decl*> &Result);
 
   /// \brief Finds all declarations lexically contained within the given
   /// DeclContext.
   ///
   /// \return true if an error occurred
   ExternalLoadResult FindExternalLexicalDecls(const DeclContext *DC,
-                                llvm::SmallVectorImpl<Decl*> &Result) {
+                                SmallVectorImpl<Decl*> &Result) {
     return FindExternalLexicalDecls(DC, 0, Result);
   }
 
   template <typename DeclTy>
   ExternalLoadResult FindExternalLexicalDeclsBy(const DeclContext *DC,
-                                  llvm::SmallVectorImpl<Decl*> &Result) {
+                                  SmallVectorImpl<Decl*> &Result) {
     return FindExternalLexicalDecls(DC, DeclTy::classofKind, Result);
   }
 
@@ -231,15 +216,11 @@ class ExternalASTSource {
   static DeclContextLookupResult
   SetExternalVisibleDeclsForName(const DeclContext *DC,
                                  DeclarationName Name,
-                                 llvm::SmallVectorImpl<NamedDecl*> &Decls);
+                                 ArrayRef<NamedDecl*> Decls);
 
   static DeclContextLookupResult
   SetNoExternalVisibleDeclsForName(const DeclContext *DC,
                                    DeclarationName Name);
-
-  void MaterializeVisibleDeclsForName(const DeclContext *DC,
-                                      DeclarationName Name,
-                                 llvm::SmallVectorImpl<NamedDecl*> &Decls);
 };
 
 /// \brief A lazy pointer to an AST node (of base type T) that resides
@@ -305,6 +286,178 @@ struct LazyOffsetPtr {
   }
 };
 
+/// \brief Represents a lazily-loaded vector of data.
+///
+/// The lazily-loaded vector of data contains data that is partially loaded
+/// from an external source and partially added by local translation. The 
+/// items loaded from the external source are loaded lazily, when needed for
+/// iteration over the complete vector.
+template<typename T, typename Source, 
+         void (Source::*Loader)(SmallVectorImpl<T>&),
+         unsigned LoadedStorage = 2, unsigned LocalStorage = 4>
+class LazyVector {
+  SmallVector<T, LoadedStorage> Loaded;
+  SmallVector<T, LocalStorage> Local;
+
+public:
+  // Iteration over the elements in the vector.
+  class iterator {
+    LazyVector *Self;
+    
+    /// \brief Position within the vector..
+    ///
+    /// In a complete iteration, the Position field walks the range [-M, N),
+    /// where negative values are used to indicate elements
+    /// loaded from the external source while non-negative values are used to
+    /// indicate elements added via \c push_back().
+    /// However, to provide iteration in source order (for, e.g., chained
+    /// precompiled headers), dereferencing the iterator flips the negative
+    /// values (corresponding to loaded entities), so that position -M 
+    /// corresponds to element 0 in the loaded entities vector, position -M+1
+    /// corresponds to element 1 in the loaded entities vector, etc. This
+    /// gives us a reasonably efficient, source-order walk.
+    int Position;
+    
+    friend class LazyVector;
+    
+  public:
+    typedef T                   value_type;
+    typedef value_type&         reference;
+    typedef value_type*         pointer;
+    typedef std::random_access_iterator_tag iterator_category;
+    typedef int                 difference_type;
+    
+    iterator() : Self(0), Position(0) { }
+    
+    iterator(LazyVector *Self, int Position) 
+      : Self(Self), Position(Position) { }
+    
+    reference operator*() const {
+      if (Position < 0)
+        return Self->Loaded.end()[Position];
+      return Self->Local[Position];
+    }
+    
+    pointer operator->() const {
+      if (Position < 0)
+        return &Self->Loaded.end()[Position];
+      
+      return &Self->Local[Position];        
+    }
+    
+    reference operator[](difference_type D) {
+      return *(*this + D);
+    }
+    
+    iterator &operator++() {
+      ++Position;
+      return *this;
+    }
+    
+    iterator operator++(int) {
+      iterator Prev(*this);
+      ++Position;
+      return Prev;
+    }
+    
+    iterator &operator--() {
+      --Position;
+      return *this;
+    }
+    
+    iterator operator--(int) {
+      iterator Prev(*this);
+      --Position;
+      return Prev;
+    }
+    
+    friend bool operator==(const iterator &X, const iterator &Y) {
+      return X.Position == Y.Position;
+    }
+    
+    friend bool operator!=(const iterator &X, const iterator &Y) {
+      return X.Position != Y.Position;
+    }
+    
+    friend bool operator<(const iterator &X, const iterator &Y) {
+      return X.Position < Y.Position;
+    }
+    
+    friend bool operator>(const iterator &X, const iterator &Y) {
+      return X.Position > Y.Position;
+    }
+    
+    friend bool operator<=(const iterator &X, const iterator &Y) {
+      return X.Position < Y.Position;
+    }
+    
+    friend bool operator>=(const iterator &X, const iterator &Y) {
+      return X.Position > Y.Position;
+    }
+    
+    friend iterator& operator+=(iterator &X, difference_type D) {
+      X.Position += D;
+      return X;
+    }
+    
+    friend iterator& operator-=(iterator &X, difference_type D) {
+      X.Position -= D;
+      return X;
+    }
+    
+    friend iterator operator+(iterator X, difference_type D) {
+      X.Position += D;
+      return X;
+    }
+    
+    friend iterator operator+(difference_type D, iterator X) {
+      X.Position += D;
+      return X;
+    }
+    
+    friend difference_type operator-(const iterator &X, const iterator &Y) {
+      return X.Position - Y.Position;
+    }
+    
+    friend iterator operator-(iterator X, difference_type D) {
+      X.Position -= D;
+      return X;
+    }
+  };
+  friend class iterator;
+  
+  iterator begin(Source *source, bool LocalOnly = false) {
+    if (LocalOnly)
+      return iterator(this, 0);
+    
+    if (source)
+      (source->*Loader)(Loaded);
+    return iterator(this, -(int)Loaded.size());
+  }
+  
+  iterator end() {
+    return iterator(this, Local.size());
+  }
+  
+  void push_back(const T& LocalValue) {
+    Local.push_back(LocalValue);
+  }
+  
+  void erase(iterator From, iterator To) {
+    if (From.Position < 0 && To.Position < 0) {
+      Loaded.erase(Loaded.end() + From.Position, Loaded.end() + To.Position);
+      return;
+    }
+    
+    if (From.Position < 0) {
+      Loaded.erase(Loaded.end() + From.Position, Loaded.end());
+      From = begin(0, true);
+    }
+    
+    Local.erase(Local.begin() + From.Position, Local.begin() + To.Position);
+  }
+};
+
 /// \brief A lazy pointer to a statement.
 typedef LazyOffsetPtr<Stmt, uint64_t, &ExternalASTSource::GetExternalDeclStmt>
   LazyDeclStmtPtr;
diff --git a/contrib/llvm/tools/clang/include/clang/AST/Mangle.h b/contrib/llvm/tools/clang/include/clang/AST/Mangle.h
index 7af7702027a4..f58a83b54b6b 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/Mangle.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/Mangle.h
@@ -39,25 +39,25 @@ namespace clang {
 /// external memory ownership.
 class MangleBuffer {
 public:
-  void setString(llvm::StringRef Ref) {
+  void setString(StringRef Ref) {
     String = Ref;
   }
 
-  llvm::SmallVectorImpl<char> &getBuffer() {
+  SmallVectorImpl<char> &getBuffer() {
     return Buffer;
   }
 
-  llvm::StringRef getString() const {
+  StringRef getString() const {
     if (!String.empty()) return String;
     return Buffer.str();
   }
 
-  operator llvm::StringRef() const {
+  operator StringRef() const {
     return getString();
   }
 
 private:
-  llvm::StringRef String;
+  StringRef String;
   llvm::SmallString<256> Buffer;
 };
 
@@ -65,21 +65,21 @@ class MangleBuffer {
 /// calls to the C++ name mangler.
 class MangleContext {
   ASTContext &Context;
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
 
   llvm::DenseMap<const BlockDecl*, unsigned> GlobalBlockIds;
   llvm::DenseMap<const BlockDecl*, unsigned> LocalBlockIds;
   
 public:
   explicit MangleContext(ASTContext &Context,
-                         Diagnostic &Diags)
+                         DiagnosticsEngine &Diags)
     : Context(Context), Diags(Diags) { }
 
   virtual ~MangleContext() { }
 
   ASTContext &getASTContext() const { return Context; }
 
-  Diagnostic &getDiags() const { return Diags; }
+  DiagnosticsEngine &getDiags() const { return Diags; }
 
   virtual void startNewFunction() { LocalBlockIds.clear(); }
   
@@ -95,55 +95,55 @@ class MangleContext {
   /// @{
 
   virtual bool shouldMangleDeclName(const NamedDecl *D) = 0;
-  virtual void mangleName(const NamedDecl *D, llvm::raw_ostream &)=0;
+  virtual void mangleName(const NamedDecl *D, raw_ostream &)=0;
   virtual void mangleThunk(const CXXMethodDecl *MD,
                           const ThunkInfo &Thunk,
-                          llvm::raw_ostream &) = 0;
+                          raw_ostream &) = 0;
   virtual void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
                                   const ThisAdjustment &ThisAdjustment,
-                                  llvm::raw_ostream &) = 0;
+                                  raw_ostream &) = 0;
   virtual void mangleReferenceTemporary(const VarDecl *D,
-                                        llvm::raw_ostream &) = 0;
+                                        raw_ostream &) = 0;
   virtual void mangleCXXVTable(const CXXRecordDecl *RD,
-                               llvm::raw_ostream &) = 0;
+                               raw_ostream &) = 0;
   virtual void mangleCXXVTT(const CXXRecordDecl *RD,
-                            llvm::raw_ostream &) = 0;
+                            raw_ostream &) = 0;
   virtual void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
                                    const CXXRecordDecl *Type,
-                                   llvm::raw_ostream &) = 0;
-  virtual void mangleCXXRTTI(QualType T, llvm::raw_ostream &) = 0;
-  virtual void mangleCXXRTTIName(QualType T, llvm::raw_ostream &) = 0;
+                                   raw_ostream &) = 0;
+  virtual void mangleCXXRTTI(QualType T, raw_ostream &) = 0;
+  virtual void mangleCXXRTTIName(QualType T, raw_ostream &) = 0;
   virtual void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
-                             llvm::raw_ostream &) = 0;
+                             raw_ostream &) = 0;
   virtual void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
-                             llvm::raw_ostream &) = 0;
+                             raw_ostream &) = 0;
 
   void mangleGlobalBlock(const BlockDecl *BD,
-                         llvm::raw_ostream &Out);
+                         raw_ostream &Out);
   void mangleCtorBlock(const CXXConstructorDecl *CD, CXXCtorType CT,
-                       const BlockDecl *BD, llvm::raw_ostream &Out);
+                       const BlockDecl *BD, raw_ostream &Out);
   void mangleDtorBlock(const CXXDestructorDecl *CD, CXXDtorType DT,
-                       const BlockDecl *BD, llvm::raw_ostream &Out);
+                       const BlockDecl *BD, raw_ostream &Out);
   void mangleBlock(const DeclContext *DC, const BlockDecl *BD,
-                   llvm::raw_ostream &Out);
+                   raw_ostream &Out);
   // Do the right thing.
-  void mangleBlock(const BlockDecl *BD, llvm::raw_ostream &Out);
+  void mangleBlock(const BlockDecl *BD, raw_ostream &Out);
 
   void mangleObjCMethodName(const ObjCMethodDecl *MD,
-                            llvm::raw_ostream &);
+                            raw_ostream &);
 
   // This is pretty lame.
   virtual void mangleItaniumGuardVariable(const VarDecl *D,
-                                          llvm::raw_ostream &) {
-    assert(0 && "Target does not support mangling guard variables");
+                                          raw_ostream &) {
+    llvm_unreachable("Target does not support mangling guard variables");
   }
   /// @}
 };
 
 MangleContext *createItaniumMangleContext(ASTContext &Context,
-                                          Diagnostic &Diags);
+                                          DiagnosticsEngine &Diags);
 MangleContext *createMicrosoftMangleContext(ASTContext &Context,
-                                            Diagnostic &Diags);
+                                            DiagnosticsEngine &Diags);
 
 }
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/NestedNameSpecifier.h b/contrib/llvm/tools/clang/include/clang/AST/NestedNameSpecifier.h
index 018041f8ba20..c81c06e332b6 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/NestedNameSpecifier.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/NestedNameSpecifier.h
@@ -18,10 +18,6 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/PointerIntPair.h"
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
 
 class ASTContext;
@@ -196,7 +192,7 @@ class NestedNameSpecifier : public llvm::FoldingSetNode {
 
   /// \brief Print this nested name specifier to the given output
   /// stream.
-  void print(llvm::raw_ostream &OS, const PrintingPolicy &Policy) const;
+  void print(raw_ostream &OS, const PrintingPolicy &Policy) const;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
     ID.AddPointer(Prefix.getOpaqueValue());
@@ -469,7 +465,7 @@ class NestedNameSpecifierLocBuilder {
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            NestedNameSpecifier *NNS) {
   DB.AddTaggedVal(reinterpret_cast<intptr_t>(NNS),
-                  Diagnostic::ak_nestednamespec);
+                  DiagnosticsEngine::ak_nestednamespec);
   return DB;
 }
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/OperationKinds.h b/contrib/llvm/tools/clang/include/clang/AST/OperationKinds.h
index 92ff6041371c..469da99a8cbf 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/OperationKinds.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/OperationKinds.h
@@ -31,9 +31,12 @@ enum CastKind {
   /// to be reinterpreted as a bit pattern of another type.  Generally
   /// the operands must have equivalent size and unrelated types.
   ///
-  /// The pointer conversion char* -> int* is a bitcast.  Many other
-  /// pointer conversions which are "physically" bitcasts are given
-  /// special cast kinds.
+  /// The pointer conversion char* -> int* is a bitcast.  A conversion
+  /// from any pointer type to a C pointer type is a bitcast unless
+  /// it's actually BaseToDerived or DerivedToBase.  A conversion to a
+  /// block pointer or ObjC pointer type is a bitcast only if the
+  /// operand has the same type kind; otherwise, it's one of the
+  /// specialized casts below.
   ///
   /// Vector coercions are bitcasts.
   CK_BitCast,
@@ -186,12 +189,16 @@ enum CastKind {
   ///    (float) ld
   CK_FloatingCast,
     
-  /// CK_AnyPointerToObjCPointerCast - Casting any other pointer kind
-  /// to an Objective-C pointer.
-  CK_AnyPointerToObjCPointerCast,
+  /// CK_CPointerToObjCPointerCast - Casting a C pointer kind to an
+  /// Objective-C pointer.
+  CK_CPointerToObjCPointerCast,
 
-  /// CK_AnyPointerToBlockPointerCast - Casting any other pointer kind
-  /// to a block pointer.
+  /// CK_BlockPointerToObjCPointerCast - Casting a block pointer to an
+  /// ObjC pointer.
+  CK_BlockPointerToObjCPointerCast,
+
+  /// CK_AnyPointerToBlockPointerCast - Casting any non-block pointer
+  /// to a block pointer.  Block-to-block casts are bitcasts.
   CK_AnyPointerToBlockPointerCast,
 
   /// \brief Converting between two Objective-C object types, which
@@ -247,20 +254,27 @@ enum CastKind {
   ///   _Complex unsigned -> _Complex float
   CK_IntegralComplexToFloatingComplex,
 
-  /// \brief Produces a retainable object pointer so that it may be
-  /// consumed, e.g. by being passed to a consuming parameter.  Calls
-  /// objc_retain.
-  CK_ObjCProduceObject,
+  /// \brief [ARC] Produces a retainable object pointer so that it may
+  /// be consumed, e.g. by being passed to a consuming parameter.
+  /// Calls objc_retain.
+  CK_ARCProduceObject,
 
-  /// \brief Consumes a retainable object pointer that has just been
-  /// produced, e.g. as the return value of a retaining call.  Enters
-  /// a cleanup to call objc_release at some indefinite time.
-  CK_ObjCConsumeObject,
+  /// \brief [ARC] Consumes a retainable object pointer that has just
+  /// been produced, e.g. as the return value of a retaining call.
+  /// Enters a cleanup to call objc_release at some indefinite time.
+  CK_ARCConsumeObject,
 
-  /// \brief Reclaim a retainable object pointer object that may have
-  /// been produced and autoreleased as part of a function return
+  /// \brief [ARC] Reclaim a retainable object pointer object that may
+  /// have been produced and autoreleased as part of a function return
   /// sequence.
-  CK_ObjCReclaimReturnedObject
+  CK_ARCReclaimReturnedObject,
+
+  /// \brief [ARC] Causes a value of block type to be copied to the
+  /// heap, if it is not already there.  A number of other operations
+  /// in ARC cause blocks to be copied; this is for cases where that
+  /// would not otherwise be guaranteed, such as when casting to a
+  /// non-block pointer type.
+  CK_ARCExtendBlockObject
 };
 
 #define CK_Invalid ((CastKind) -1)
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ParentMap.h b/contrib/llvm/tools/clang/include/clang/AST/ParentMap.h
index 22c1e7269fac..62eae02c1525 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ParentMap.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ParentMap.h
@@ -32,6 +32,7 @@ class ParentMap {
   Stmt *getParent(Stmt*) const;
   Stmt *getParentIgnoreParens(Stmt *) const;
   Stmt *getParentIgnoreParenCasts(Stmt *) const;
+  Stmt *getParentIgnoreParenImpCasts(Stmt *) const;
   Stmt *getOuterParenParent(Stmt *) const;
 
   const Stmt *getParent(const Stmt* S) const {
diff --git a/contrib/llvm/tools/clang/include/clang/AST/PrettyPrinter.h b/contrib/llvm/tools/clang/include/clang/AST/PrettyPrinter.h
index fc8ac36b3b97..2bdd8d3f4bc4 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/PrettyPrinter.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/PrettyPrinter.h
@@ -15,10 +15,7 @@
 #define LLVM_CLANG_AST_PRETTY_PRINTER_H
 
 #include "clang/Basic/LangOptions.h"
-
-namespace llvm {
-  class raw_ostream;
-}
+#include "clang/Basic/LLVM.h"
 
 namespace clang {
 
@@ -29,7 +26,7 @@ class LangOptions;
 class PrinterHelper {
 public:
   virtual ~PrinterHelper();
-  virtual bool handledStmt(Stmt* E, llvm::raw_ostream& OS) = 0;
+  virtual bool handledStmt(Stmt* E, raw_ostream& OS) = 0;
 };
 
 /// \brief Describes how types, statements, expressions, and
@@ -41,13 +38,14 @@ struct PrintingPolicy {
       SuppressTagKeyword(false), SuppressTag(false), SuppressScope(false),
       SuppressInitializers(false),
       Dump(false), ConstantArraySizeAsWritten(false),
-      AnonymousTagLocations(true), SuppressStrongLifetime(false) { }
+      AnonymousTagLocations(true), SuppressStrongLifetime(false),
+      Bool(LO.Bool) { }
 
   /// \brief The number of spaces to use to indent each line.
   unsigned Indentation : 8;
 
   /// \brief What language we're printing.
-  const LangOptions LangOpts;
+  LangOptions LangOpts;
 
   /// \brief Whether we should suppress printing of the actual specifiers for
   /// the given type or declaration.
@@ -133,6 +131,10 @@ struct PrintingPolicy {
   /// \brief When true, suppress printing of the __strong lifetime qualifier in
   /// ARC.
   unsigned SuppressStrongLifetime : 1;
+  
+  /// \brief Whether we can use 'bool' rather than '_Bool', even if the language
+  /// doesn't actually have 'bool' (because, e.g., it is defined as a macro).
+  unsigned Bool : 1;
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/AST/RecordLayout.h b/contrib/llvm/tools/clang/include/clang/AST/RecordLayout.h
index d7bab80afc5e..b0186cea291c 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/RecordLayout.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/RecordLayout.h
@@ -63,6 +63,9 @@ class ASTRecordLayout {
     /// any empty subobjects.
     CharUnits SizeOfLargestEmptySubobject;
     
+    /// VBPtrOffset - Virtual base table offset.
+    CharUnits VBPtrOffset;
+    
     /// PrimaryBase - The primary base info for this record.
     llvm::PointerIntPair<const CXXRecordDecl *, 1, bool> PrimaryBase;
     
@@ -89,7 +92,8 @@ class ASTRecordLayout {
   // Constructor for C++ records.
   typedef CXXRecordLayoutInfo::BaseOffsetsMapTy BaseOffsetsMapTy;
   ASTRecordLayout(const ASTContext &Ctx,
-                  CharUnits size, CharUnits alignment, CharUnits datasize,
+                  CharUnits size, CharUnits alignment, CharUnits vbptroffset,
+                  CharUnits datasize,
                   const uint64_t *fieldoffsets, unsigned fieldcount,
                   CharUnits nonvirtualsize, CharUnits nonvirtualalign,
                   CharUnits SizeOfLargestEmptySubobject,
@@ -199,6 +203,10 @@ class ASTRecordLayout {
     assert(CXXInfo && "Record layout does not have C++ specific info!");
     return CXXInfo->SizeOfLargestEmptySubobject;
   }
+
+  CharUnits getVBPtrOffset() const {
+    return CXXInfo->VBPtrOffset;
+  }
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h b/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h
index 85c5c08853d1..0ec09c9b09e0 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -813,6 +813,10 @@ DEF_TRAVERSE_TYPE(ObjCObjectPointerType, {
     TRY_TO(TraverseType(T->getPointeeType()));
   })
 
+DEF_TRAVERSE_TYPE(AtomicType, {
+    TRY_TO(TraverseType(T->getValueType()));
+  })
+
 #undef DEF_TRAVERSE_TYPE
 
 // ----------------- TypeLoc traversal -----------------
@@ -1041,6 +1045,10 @@ DEF_TRAVERSE_TYPELOC(ObjCObjectPointerType, {
     TRY_TO(TraverseTypeLoc(TL.getPointeeLoc()));
   })
 
+DEF_TRAVERSE_TYPELOC(AtomicType, {
+    TRY_TO(TraverseTypeLoc(TL.getValueLoc()));
+  })
+
 #undef DEF_TRAVERSE_TYPELOC
 
 // ----------------- Decl traversal -----------------
@@ -1114,6 +1122,10 @@ DEF_TRAVERSE_DECL(FriendTemplateDecl, {
     }
   })
 
+DEF_TRAVERSE_DECL(ClassScopeFunctionSpecializationDecl, {
+  TRY_TO(TraverseDecl(D->getSpecialization()));
+ })
+
 DEF_TRAVERSE_DECL(LinkageSpecDecl, { })
 
 DEF_TRAVERSE_DECL(ObjCClassDecl, {
@@ -1252,7 +1264,7 @@ bool RecursiveASTVisitor<Derived>::TraverseClassInstantiations(
           = (U.get<ClassTemplatePartialSpecializationDecl*>() == Pattern);
 
       if (ShouldVisit)
-        TRY_TO(TraverseClassTemplateSpecializationDecl(SD));
+        TRY_TO(TraverseDecl(SD));
       break;
     }
 
@@ -1280,7 +1292,7 @@ DEF_TRAVERSE_DECL(ClassTemplateDecl, {
     TRY_TO(TraverseTemplateParameterListHelper(D->getTemplateParameters()));
 
     // By default, we do not traverse the instantiations of
-    // class templates since they do not apprear in the user code. The
+    // class templates since they do not appear in the user code. The
     // following code optionally traverses them.
     if (getDerived().shouldVisitTemplateInstantiations()) {
       // If this is the definition of the primary template, visit
@@ -1318,7 +1330,7 @@ bool RecursiveASTVisitor<Derived>::TraverseFunctionInstantiations(
     case TSK_ExplicitSpecialization:
       break;
     default:
-      assert(false && "Unknown specialization kind.");
+      llvm_unreachable("Unknown specialization kind.");
     }
   }
 
@@ -1548,10 +1560,10 @@ bool RecursiveASTVisitor<Derived>::TraverseFunctionHelper(FunctionDecl *D) {
         FTSI->getTemplateSpecializationKind() != TSK_ImplicitInstantiation) {
       // A specialization might not have explicit template arguments if it has
       // a templated return type and concrete arguments.
-      if (const TemplateArgumentListInfo *TALI =
+      if (const ASTTemplateArgumentListInfo *TALI =
           FTSI->TemplateArgumentsAsWritten) {
-        TRY_TO(TraverseTemplateArgumentLocsHelper(TALI->getArgumentArray(),
-                                                  TALI->size()));
+        TRY_TO(TraverseTemplateArgumentLocsHelper(TALI->getTemplateArgs(),
+                                                  TALI->NumTemplateArgs));
       }
     }
   }
@@ -1980,6 +1992,7 @@ DEF_TRAVERSE_STMT(SizeOfPackExpr, { })
 DEF_TRAVERSE_STMT(SubstNonTypeTemplateParmPackExpr, { })
 DEF_TRAVERSE_STMT(SubstNonTypeTemplateParmExpr, { })
 DEF_TRAVERSE_STMT(MaterializeTemporaryExpr, { })
+DEF_TRAVERSE_STMT(AtomicExpr, { })
 
 // These literals (all of them) do not need any action.
 DEF_TRAVERSE_STMT(IntegerLiteral, { })
diff --git a/contrib/llvm/tools/clang/include/clang/AST/SelectorLocationsKind.h b/contrib/llvm/tools/clang/include/clang/AST/SelectorLocationsKind.h
new file mode 100644
index 000000000000..cd43a5c49c55
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/AST/SelectorLocationsKind.h
@@ -0,0 +1,83 @@
+//===--- SelectorLocationsKind.h - Kind of selector locations ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Describes whether the identifier locations for a selector are "standard"
+// or not.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_SELECTORLOCATIONSKIND_H
+#define LLVM_CLANG_AST_SELECTORLOCATIONSKIND_H
+
+#include "clang/Basic/LLVM.h"
+
+namespace clang {
+  class Selector;
+  class SourceLocation;
+  class Expr;
+  class ParmVarDecl;
+
+/// \brief Whether all locations of the selector identifiers are in a
+/// "standard" position.
+enum SelectorLocationsKind {
+  /// \brief Non-standard.
+  SelLoc_NonStandard = 0,
+
+  /// \brief For nullary selectors, immediately before the end:
+  ///    "[foo release]" / "-(void)release;"
+  /// Or immediately before the arguments:
+  ///    "[foo first:1 second:2]" / "-(id)first:(int)x second:(int)y;
+  SelLoc_StandardNoSpace = 1,
+
+  /// \brief For nullary selectors, immediately before the end:
+  ///    "[foo release]" / "-(void)release;"
+  /// Or with a space between the arguments:
+  ///    "[foo first: 1 second: 2]" / "-(id)first: (int)x second: (int)y;
+  SelLoc_StandardWithSpace = 2
+};
+
+/// \brief Returns true if all \arg SelLocs are in a "standard" location.
+SelectorLocationsKind hasStandardSelectorLocs(Selector Sel,
+                                              ArrayRef<SourceLocation> SelLocs,
+                                              ArrayRef<Expr *> Args,
+                                              SourceLocation EndLoc);
+
+/// \brief Get the "standard" location of a selector identifier, e.g:
+/// For nullary selectors, immediately before ']': "[foo release]"
+///
+/// \param WithArgSpace if true the standard location is with a space apart
+/// before arguments: "[foo first: 1 second: 2]"
+/// If false: "[foo first:1 second:2]"
+SourceLocation getStandardSelectorLoc(unsigned Index,
+                                      Selector Sel,
+                                      bool WithArgSpace,
+                                      ArrayRef<Expr *> Args,
+                                      SourceLocation EndLoc);
+
+/// \brief Returns true if all \arg SelLocs are in a "standard" location.
+SelectorLocationsKind hasStandardSelectorLocs(Selector Sel,
+                                              ArrayRef<SourceLocation> SelLocs,
+                                              ArrayRef<ParmVarDecl *> Args,
+                                              SourceLocation EndLoc);
+
+/// \brief Get the "standard" location of a selector identifier, e.g:
+/// For nullary selectors, immediately before ']': "[foo release]"
+///
+/// \param WithArgSpace if true the standard location is with a space apart
+/// before arguments: "-(id)first: (int)x second: (int)y;"
+/// If false: "-(id)first:(int)x second:(int)y;"
+SourceLocation getStandardSelectorLoc(unsigned Index,
+                                      Selector Sel,
+                                      bool WithArgSpace,
+                                      ArrayRef<ParmVarDecl *> Args,
+                                      SourceLocation EndLoc);
+
+} // end namespace clang
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/Stmt.h b/contrib/llvm/tools/clang/include/clang/AST/Stmt.h
index bf5f383be5e0..2a6fd6bc964c 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/Stmt.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/Stmt.h
@@ -14,16 +14,15 @@
 #ifndef LLVM_CLANG_AST_STMT_H
 #define LLVM_CLANG_AST_STMT_H
 
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/raw_ostream.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/StmtIterator.h"
 #include "clang/AST/DeclGroup.h"
-#include "llvm/ADT/SmallVector.h"
 #include "clang/AST/ASTContext.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/SmallVector.h"
 #include <string>
-using llvm::dyn_cast_or_null;
 
 namespace llvm {
   class FoldingSetNodeID;
@@ -108,11 +107,10 @@ class Stmt {
   // Make vanilla 'new' and 'delete' illegal for Stmts.
 protected:
   void* operator new(size_t bytes) throw() {
-    assert(0 && "Stmts cannot be allocated with regular 'new'.");
-    return 0;
+    llvm_unreachable("Stmts cannot be allocated with regular 'new'.");
   }
   void operator delete(void* data) throw() {
-    assert(0 && "Stmts cannot be released with regular 'delete'.");
+    llvm_unreachable("Stmts cannot be released with regular 'delete'.");
   }
 
   class StmtBitfields {
@@ -148,6 +146,7 @@ class Stmt {
     friend class CXXUnresolvedConstructExpr; // ctor
     friend class CXXDependentScopeMemberExpr; // ctor
     friend class OverloadExpr; // ctor
+    friend class AtomicExpr; // ctor
     unsigned : NumStmtBits;
 
     unsigned ValueKind : 2;
@@ -167,6 +166,7 @@ class Stmt {
     unsigned HasQualifier : 1;
     unsigned HasExplicitTemplateArgs : 1;
     unsigned HasFoundDecl : 1;
+    unsigned HadMultipleCandidates : 1;
   };
 
   class CastExprBitfields {
@@ -270,7 +270,7 @@ class Stmt {
   /// This is useful in a debugger.
   void dump() const;
   void dump(SourceManager &SM) const;
-  void dump(llvm::raw_ostream &OS, SourceManager &SM) const;
+  void dump(raw_ostream &OS, SourceManager &SM) const;
 
   /// dumpAll - This does a dump of the specified AST fragment and all subtrees.
   void dumpAll() const;
@@ -279,12 +279,12 @@ class Stmt {
   /// dumpPretty/printPretty - These two methods do a "pretty print" of the AST
   /// back to its original source language syntax.
   void dumpPretty(ASTContext& Context) const;
-  void printPretty(llvm::raw_ostream &OS, PrinterHelper *Helper,
+  void printPretty(raw_ostream &OS, PrinterHelper *Helper,
                    const PrintingPolicy &Policy,
                    unsigned Indentation = 0) const {
     printPretty(OS, *(ASTContext*)0, Helper, Policy, Indentation);
   }
-  void printPretty(llvm::raw_ostream &OS, ASTContext &Context,
+  void printPretty(raw_ostream &OS, ASTContext &Context,
                    PrinterHelper *Helper,
                    const PrintingPolicy &Policy,
                    unsigned Indentation = 0) const;
@@ -297,6 +297,12 @@ class Stmt {
   /// statement, such as ExprWithCleanups or ImplicitCastExpr nodes.
   Stmt *IgnoreImplicit();
 
+  const Stmt *stripLabelLikeStatements() const;
+  Stmt *stripLabelLikeStatements() {
+    return const_cast<Stmt*>(
+      const_cast<const Stmt*>(this)->stripLabelLikeStatements());
+  }
+
   // Implement isa<T> support.
   static bool classof(const Stmt *) { return true; }
 
@@ -407,25 +413,25 @@ class DeclStmt : public Stmt {
 class NullStmt : public Stmt {
   SourceLocation SemiLoc;
 
-  /// \brief If the null statement was preceded by an empty macro this is
-  /// its instantiation source location, e.g:
+  /// \brief True if the null statement was preceded by an empty macro, e.g:
   /// @code
   ///   #define CALL(x)
   ///   CALL(0);
   /// @endcode
-  SourceLocation LeadingEmptyMacro;
+  bool HasLeadingEmptyMacro;
 public:
-  NullStmt(SourceLocation L, SourceLocation LeadingEmptyMacro =SourceLocation())
-    : Stmt(NullStmtClass), SemiLoc(L), LeadingEmptyMacro(LeadingEmptyMacro) {}
+  NullStmt(SourceLocation L, bool hasLeadingEmptyMacro = false)
+    : Stmt(NullStmtClass), SemiLoc(L),
+      HasLeadingEmptyMacro(hasLeadingEmptyMacro) {}
 
   /// \brief Build an empty null statement.
-  explicit NullStmt(EmptyShell Empty) : Stmt(NullStmtClass, Empty) { }
+  explicit NullStmt(EmptyShell Empty) : Stmt(NullStmtClass, Empty),
+      HasLeadingEmptyMacro(false) { }
 
   SourceLocation getSemiLoc() const { return SemiLoc; }
   void setSemiLoc(SourceLocation L) { SemiLoc = L; }
 
-  bool hasLeadingEmptyMacro() const { return LeadingEmptyMacro.isValid(); }
-  SourceLocation getLeadingEmptyMacroLoc() const { return LeadingEmptyMacro; }
+  bool hasLeadingEmptyMacro() const { return HasLeadingEmptyMacro; }
 
   SourceRange getSourceRange() const { return SourceRange(SemiLoc); }
 
@@ -525,6 +531,10 @@ class CompoundStmt : public Stmt {
   child_range children() {
     return child_range(&Body[0], &Body[0]+CompoundStmtBits.NumStmts);
   }
+  
+  const_child_range children() const {
+    return child_range(&Body[0], &Body[0]+CompoundStmtBits.NumStmts);
+  }
 };
 
 // SwitchCase is the base class for CaseStmt and DefaultStmt,
@@ -1308,7 +1318,7 @@ class AsmStmt : public Stmt {
   /// true, otherwise return false.  This handles canonicalization and
   /// translation of strings from GCC syntax to LLVM IR syntax, and handles
   //// flattening of named references like %[foo] to Operand AsmStringPiece's.
-  unsigned AnalyzeAsmString(llvm::SmallVectorImpl<AsmStringPiece> &Pieces,
+  unsigned AnalyzeAsmString(SmallVectorImpl<AsmStringPiece> &Pieces,
                             ASTContext &C, unsigned &DiagOffs) const;
 
 
@@ -1320,17 +1330,17 @@ class AsmStmt : public Stmt {
     return Names[i];
   }
 
-  llvm::StringRef getOutputName(unsigned i) const {
+  StringRef getOutputName(unsigned i) const {
     if (IdentifierInfo *II = getOutputIdentifier(i))
       return II->getName();
     
-    return llvm::StringRef();
+    return StringRef();
   }
 
   /// getOutputConstraint - Return the constraint string for the specified
   /// output operand.  All output constraints are known to be non-empty (either
   /// '=' or '+').
-  llvm::StringRef getOutputConstraint(unsigned i) const;
+  StringRef getOutputConstraint(unsigned i) const;
 
   const StringLiteral *getOutputConstraintLiteral(unsigned i) const {
     return Constraints[i];
@@ -1364,16 +1374,16 @@ class AsmStmt : public Stmt {
     return Names[i + NumOutputs];
   }
 
-  llvm::StringRef getInputName(unsigned i) const {
+  StringRef getInputName(unsigned i) const {
     if (IdentifierInfo *II = getInputIdentifier(i))
       return II->getName();
 
-    return llvm::StringRef();
+    return StringRef();
   }
 
   /// getInputConstraint - Return the specified input constraint.  Unlike output
   /// constraints, these can be empty.
-  llvm::StringRef getInputConstraint(unsigned i) const;
+  StringRef getInputConstraint(unsigned i) const;
 
   const StringLiteral *getInputConstraintLiteral(unsigned i) const {
     return Constraints[i + NumOutputs];
@@ -1403,7 +1413,7 @@ class AsmStmt : public Stmt {
   /// getNamedOperand - Given a symbolic operand reference like %[foo],
   /// translate this into a numeric value needed to reference the same operand.
   /// This returns -1 if the operand name is invalid.
-  int getNamedOperand(llvm::StringRef SymbolicName) const;
+  int getNamedOperand(StringRef SymbolicName) const;
 
   unsigned getNumClobbers() const { return NumClobbers; }
   StringLiteral *getClobber(unsigned i) { return Clobbers[i]; }
diff --git a/contrib/llvm/tools/clang/include/clang/AST/StmtVisitor.h b/contrib/llvm/tools/clang/include/clang/AST/StmtVisitor.h
index 29d234754645..48a0123608c9 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/StmtVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/StmtVisitor.h
@@ -42,7 +42,7 @@ class StmtVisitorBase {
     // below.
     if (PTR(BinaryOperator) BinOp = dyn_cast<BinaryOperator>(S)) {
       switch (BinOp->getOpcode()) {
-      default: assert(0 && "Unknown binary operator!");
+      default: llvm_unreachable("Unknown binary operator!");
       case BO_PtrMemD:   DISPATCH(BinPtrMemD,   BinaryOperator);
       case BO_PtrMemI:   DISPATCH(BinPtrMemI,   BinaryOperator);
       case BO_Mul:       DISPATCH(BinMul,       BinaryOperator);
@@ -80,7 +80,7 @@ class StmtVisitorBase {
       }
     } else if (PTR(UnaryOperator) UnOp = dyn_cast<UnaryOperator>(S)) {
       switch (UnOp->getOpcode()) {
-      default: assert(0 && "Unknown unary operator!");
+      default: llvm_unreachable("Unknown unary operator!");
       case UO_PostInc:   DISPATCH(UnaryPostInc,   UnaryOperator);
       case UO_PostDec:   DISPATCH(UnaryPostDec,   UnaryOperator);
       case UO_PreInc:    DISPATCH(UnaryPreInc,    UnaryOperator);
@@ -99,7 +99,7 @@ class StmtVisitorBase {
 
     // Top switch stmt: dispatch to VisitFooStmt for each FooStmt.
     switch (S->getStmtClass()) {
-    default: assert(0 && "Unknown stmt kind!");
+    default: llvm_unreachable("Unknown stmt kind!");
 #define ABSTRACT_STMT(STMT)
 #define STMT(CLASS, PARENT)                              \
     case Stmt::CLASS ## Class: DISPATCH(CLASS, CLASS);
diff --git a/contrib/llvm/tools/clang/include/clang/AST/TemplateBase.h b/contrib/llvm/tools/clang/include/clang/AST/TemplateBase.h
index 1c693e00c8cc..371c27a0d2b2 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/TemplateBase.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/TemplateBase.h
@@ -23,7 +23,6 @@
 
 namespace llvm {
   class FoldingSetNodeID;
-  class raw_ostream;
 }
 
 namespace clang {
@@ -354,7 +353,7 @@ class TemplateArgument {
   TemplateArgument getPackExpansionPattern() const;
 
   /// \brief Print this template argument to the given output stream.
-  void print(const PrintingPolicy &Policy, llvm::raw_ostream &Out) const;
+  void print(const PrintingPolicy &Policy, raw_ostream &Out) const;
              
   /// \brief Used to insert TemplateArguments into FoldingSets.
   void Profile(llvm::FoldingSetNodeID &ID, const ASTContext &Context) const;
@@ -515,10 +514,14 @@ class TemplateArgumentLoc {
 /// A convenient class for passing around template argument
 /// information.  Designed to be passed by reference.
 class TemplateArgumentListInfo {
-  llvm::SmallVector<TemplateArgumentLoc, 8> Arguments;
+  SmallVector<TemplateArgumentLoc, 8> Arguments;
   SourceLocation LAngleLoc;
   SourceLocation RAngleLoc;
 
+  // This can leak if used in an AST node, use ASTTemplateArgumentListInfo
+  // instead.
+  void* operator new(size_t bytes, ASTContext& C);
+
 public:
   TemplateArgumentListInfo() {}
 
@@ -547,6 +550,48 @@ class TemplateArgumentListInfo {
   }
 };
 
+/// \brief Represents an explicit template argument list in C++, e.g.,
+/// the "<int>" in "sort<int>".
+/// This is safe to be used inside an AST node, in contrast with
+/// TemplateArgumentListInfo.
+struct ASTTemplateArgumentListInfo {
+  /// \brief The source location of the left angle bracket ('<');
+  SourceLocation LAngleLoc;
+  
+  /// \brief The source location of the right angle bracket ('>');
+  SourceLocation RAngleLoc;
+  
+  /// \brief The number of template arguments in TemplateArgs.
+  /// The actual template arguments (if any) are stored after the
+  /// ExplicitTemplateArgumentList structure.
+  unsigned NumTemplateArgs;
+  
+  /// \brief Retrieve the template arguments
+  TemplateArgumentLoc *getTemplateArgs() {
+    return reinterpret_cast<TemplateArgumentLoc *> (this + 1);
+  }
+  
+  /// \brief Retrieve the template arguments
+  const TemplateArgumentLoc *getTemplateArgs() const {
+    return reinterpret_cast<const TemplateArgumentLoc *> (this + 1);
+  }
+
+  const TemplateArgumentLoc &operator[](unsigned I) const {
+    return getTemplateArgs()[I];
+  }
+
+  static const ASTTemplateArgumentListInfo *Create(ASTContext &C,
+                                          const TemplateArgumentListInfo &List);
+
+  void initializeFrom(const TemplateArgumentListInfo &List);
+  void initializeFrom(const TemplateArgumentListInfo &List,
+                      bool &Dependent, bool &InstantiationDependent,
+                      bool &ContainsUnexpandedParameterPack);
+  void copyInto(TemplateArgumentListInfo &List) const;
+  static std::size_t sizeFor(unsigned NumTemplateArgs);
+  static std::size_t sizeFor(const TemplateArgumentListInfo &List);
+};
+
 const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                     const TemplateArgument &Arg);
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/TemplateName.h b/contrib/llvm/tools/clang/include/clang/AST/TemplateName.h
index a180f587eddd..7dc75b19257c 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/TemplateName.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/TemplateName.h
@@ -14,14 +14,11 @@
 #ifndef LLVM_CLANG_AST_TEMPLATENAME_H
 #define LLVM_CLANG_AST_TEMPLATENAME_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "clang/Basic/OperatorKinds.h"
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
   
 class ASTContext;
@@ -308,7 +305,7 @@ class TemplateName {
   /// \param SuppressNNS if true, don't print the
   /// nested-name-specifier that precedes the template name (if it has
   /// one).
-  void print(llvm::raw_ostream &OS, const PrintingPolicy &Policy,
+  void print(raw_ostream &OS, const PrintingPolicy &Policy,
              bool SuppressNNS = false) const;
 
   /// \brief Debugging aid that dumps the template name to standard
diff --git a/contrib/llvm/tools/clang/include/clang/AST/Type.h b/contrib/llvm/tools/clang/include/clang/AST/Type.h
index ef0dbdae037c..dd9aa56ed197 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/Type.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/Type.h
@@ -22,19 +22,15 @@
 #include "clang/Basic/Visibility.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/TemplateName.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/type_traits.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerUnion.h"
+#include "clang/Basic/LLVM.h"
 
-using llvm::isa;
-using llvm::cast;
-using llvm::cast_or_null;
-using llvm::dyn_cast;
-using llvm::dyn_cast_or_null;
 namespace clang {
   enum {
     TypeAlignmentInBits = 4,
@@ -508,6 +504,9 @@ class QualType {
   
   const Type *getTypePtrOrNull() const;
 
+  /// Retrieves a pointer to the name of the base type.
+  const IdentifierInfo *getBaseTypeIdentifier() const;
+
   /// Divides a QualType into its unqualified type and a set of local
   /// qualifiers.
   SplitQualType split() const;
@@ -868,8 +867,9 @@ class QualType {
   ///   type other than void.
   bool isCForbiddenLValueType() const;
 
-  /// \brief Determine whether this type has trivial copy-assignment semantics.
-  bool hasTrivialCopyAssignment(ASTContext &Context) const;
+  /// \brief Determine whether this type has trivial copy/move-assignment
+  ///        semantics.
+  bool hasTrivialAssignment(ASTContext &Context, bool Copying) const;
   
 private:
   // These methods are implemented in a separate translation unit;
@@ -1362,6 +1362,7 @@ class Type : public ExtQualsTypeCommonBase {
   /// various convenient purposes within Clang.  All such types are
   /// BuiltinTypes.
   bool isPlaceholderType() const;
+  const BuiltinType *getAsPlaceholderType() const;
 
   /// isSpecificPlaceholderType - Test for a specific placeholder type.
   bool isSpecificPlaceholderType(unsigned K) const;
@@ -1373,6 +1374,8 @@ class Type : public ExtQualsTypeCommonBase {
   bool isBooleanType() const;
   bool isCharType() const;
   bool isWideCharType() const;
+  bool isChar16Type() const;
+  bool isChar32Type() const;
   bool isAnyCharacterType() const;
   bool isIntegralType(ASTContext &Ctx) const;
   
@@ -1389,6 +1392,7 @@ class Type : public ExtQualsTypeCommonBase {
   bool isComplexType() const;      // C99 6.2.5p11 (complex)
   bool isAnyComplexType() const;   // C99 6.2.5p11 (complex) + Complex Int.
   bool isFloatingType() const;     // C99 6.2.5p11 (real floating + complex)
+  bool isHalfType() const;         // OpenCL 6.1.1.1, NEON (IEEE 754-2008 half)
   bool isRealType() const;         // C99 6.2.5p17 (real floating + integer)
   bool isArithmeticType() const;   // C99 6.2.5p18 (integer + floating)
   bool isVoidType() const;         // C99 6.2.5p19
@@ -1447,6 +1451,7 @@ class Type : public ExtQualsTypeCommonBase {
   bool isCARCBridgableType() const;
   bool isTemplateTypeParmType() const;          // C++ template type parameter
   bool isNullPtrType() const;                   // C++0x nullptr_t
+  bool isAtomicType() const;                    // C1X _Atomic()
 
   /// Determines if this type, which must satisfy
   /// isObjCLifetimeType(), is implicitly __unsafe_unretained rather
@@ -1457,7 +1462,9 @@ class Type : public ExtQualsTypeCommonBase {
   Qualifiers::ObjCLifetime getObjCARCImplicitLifetime() const;
 
   enum ScalarTypeKind {
-    STK_Pointer,
+    STK_CPointer,
+    STK_BlockPointer,
+    STK_ObjCObjectPointer,
     STK_MemberPointer,
     STK_Bool,
     STK_Integral,
@@ -1693,6 +1700,8 @@ class BuiltinType : public Type {
     LongLong,
     Int128,   // __int128_t
 
+    Half,     // This is the 'half' type in OpenCL,
+              // __fp16 in case of ARM NEON.
     Float, Double, LongDouble,
 
     NullPtr,  // This is the type of C++0x 'nullptr'.
@@ -1755,7 +1764,7 @@ class BuiltinType : public Type {
   }
 
   Kind getKind() const { return static_cast<Kind>(BuiltinTypeBits.Kind); }
-  const char *getName(const LangOptions &LO) const;
+  const char *getName(const PrintingPolicy &Policy) const;
 
   bool isSugared() const { return false; }
   QualType desugar() const { return QualType(this, 0); }
@@ -1773,7 +1782,7 @@ class BuiltinType : public Type {
   }
 
   bool isFloatingPoint() const {
-    return getKind() >= Float && getKind() <= LongDouble;
+    return getKind() >= Half && getKind() <= LongDouble;
   }
 
   /// Determines whether this type is a placeholder type, i.e. a type
@@ -2249,7 +2258,7 @@ class VariableArrayType : public ArrayType {
   friend class StmtIteratorBase;
 
   void Profile(llvm::FoldingSetNodeID &ID) {
-    assert(0 && "Cannot unique VariableArrayTypes.");
+    llvm_unreachable("Cannot unique VariableArrayTypes.");
   }
 };
 
@@ -2635,7 +2644,7 @@ class FunctionType : public Type {
     return getResultType().getNonLValueExprType(Context);
   }
 
-  static llvm::StringRef getNameForCallConv(CallingConv CC);
+  static StringRef getNameForCallConv(CallingConv CC);
 
   static bool classof(const Type *T) {
     return T->getTypeClass() == FunctionNoProto ||
@@ -2696,8 +2705,17 @@ class FunctionProtoType : public FunctionType, public llvm::FoldingSetNode {
     unsigned char TypeQuals;
     RefQualifierKind RefQualifier;
     unsigned NumExceptions;
+
+    /// Exceptions - A variable size array after that holds the exception types.
     const QualType *Exceptions;
+
+    /// NoexceptExpr - Instead of Exceptions, there may be a single Expr*
+    /// pointing to the expression in the noexcept() specifier.
     Expr *NoexceptExpr;
+    
+    /// ConsumedArgs - A variable size array, following Exceptions
+    /// and of length NumArgs, holding flags indicating which arguments
+    /// are consumed.  This only appears if HasAnyConsumedArgs is true.
     const bool *ConsumedArguments;
   };
 
@@ -2728,19 +2746,6 @@ class FunctionProtoType : public FunctionType, public llvm::FoldingSetNode {
   /// HasAnyConsumedArgs - Whether this function has any consumed arguments.
   unsigned HasAnyConsumedArgs : 1;
 
-  /// ArgInfo - There is an variable size array after the class in memory that
-  /// holds the argument types.
-
-  /// Exceptions - There is another variable size array after ArgInfo that
-  /// holds the exception types.
-
-  /// NoexceptExpr - Instead of Exceptions, there may be a single Expr* pointing
-  /// to the expression in the noexcept() specifier.
-
-  /// ConsumedArgs - A variable size array, following Exceptions
-  /// and of length NumArgs, holding flags indicating which arguments
-  /// are consumed.  This only appears if HasAnyConsumedArgs is true.
-
   friend class ASTContext;  // ASTContext creates these.
 
   const bool *getConsumedArgsBuffer() const {
@@ -4348,6 +4353,37 @@ class ObjCObjectPointerType : public Type, public llvm::FoldingSetNode {
   static bool classof(const ObjCObjectPointerType *) { return true; }
 };
 
+class AtomicType : public Type, public llvm::FoldingSetNode {
+  QualType ValueType;
+
+  AtomicType(QualType ValTy, QualType Canonical)
+    : Type(Atomic, Canonical, ValTy->isDependentType(),
+           ValTy->isInstantiationDependentType(),
+           ValTy->isVariablyModifiedType(),
+           ValTy->containsUnexpandedParameterPack()),
+      ValueType(ValTy) {}
+  friend class ASTContext;  // ASTContext creates these.
+
+  public:
+  /// getValueType - Gets the type contained by this atomic type, i.e.
+  /// the type returned by performing an atomic load of this atomic type.
+  QualType getValueType() const { return ValueType; }
+
+  bool isSugared() const { return false; }
+  QualType desugar() const { return QualType(this, 0); }
+
+  void Profile(llvm::FoldingSetNodeID &ID) {
+    Profile(ID, getValueType());
+  }
+  static void Profile(llvm::FoldingSetNodeID &ID, QualType T) {
+    ID.AddPointer(T.getAsOpaquePtr());
+  }
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == Atomic;
+  }
+  static bool classof(const AtomicType *) { return true; }
+};
+
 /// A qualifier set is used to build a set of qualifiers.
 class QualifierCollector : public Qualifiers {
 public:
@@ -4673,6 +4709,9 @@ inline bool Type::isObjCObjectOrInterfaceType() const {
   return isa<ObjCInterfaceType>(CanonicalType) || 
     isa<ObjCObjectType>(CanonicalType);
 }
+inline bool Type::isAtomicType() const {
+  return isa<AtomicType>(CanonicalType);
+}
 
 inline bool Type::isObjCQualifiedIdType() const {
   if (const ObjCObjectPointerType *OPT = getAs<ObjCObjectPointerType>())
@@ -4714,11 +4753,18 @@ inline bool Type::isSpecificBuiltinType(unsigned K) const {
 }
 
 inline bool Type::isPlaceholderType() const {
-  if (const BuiltinType *BT = getAs<BuiltinType>())
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(this))
     return BT->isPlaceholderType();
   return false;
 }
 
+inline const BuiltinType *Type::getAsPlaceholderType() const {
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(this))
+    if (BT->isPlaceholderType())
+      return BT;
+  return 0;
+}
+
 inline bool Type::isSpecificPlaceholderType(unsigned K) const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(this))
     return (BT->getKind() == (BuiltinType::Kind) K);
@@ -4757,7 +4803,7 @@ inline const Type *Type::getBaseElementTypeUnsafe() const {
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            QualType T) {
   DB.AddTaggedVal(reinterpret_cast<intptr_t>(T.getAsOpaquePtr()),
-                  Diagnostic::ak_qualtype);
+                  DiagnosticsEngine::ak_qualtype);
   return DB;
 }
 
@@ -4766,7 +4812,7 @@ inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
 inline const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                            QualType T) {
   PD.AddTaggedVal(reinterpret_cast<intptr_t>(T.getAsOpaquePtr()),
-                  Diagnostic::ak_qualtype);
+                  DiagnosticsEngine::ak_qualtype);
   return PD;
 }
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/TypeLoc.h b/contrib/llvm/tools/clang/include/clang/AST/TypeLoc.h
index f5669f7fc05a..20acadab7cf7 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/TypeLoc.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/TypeLoc.h
@@ -538,6 +538,10 @@ class InjectedClassNameTypeLoc :
     public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
                                      InjectedClassNameTypeLoc,
                                      InjectedClassNameType> {
+public:
+  CXXRecordDecl *getDecl() const {
+    return getTypePtr()->getDecl();
+  }
 };
 
 /// \brief Wrapper for source info for unresolved typename using decls.
@@ -561,6 +565,12 @@ class TagTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
                                                     TagType> {
 public:
   TagDecl *getDecl() const { return getTypePtr()->getDecl(); }
+
+  /// \brief True if the tag was defined in this type specifier. 
+  bool isDefinition() const {
+    return getDecl()->isCompleteDefinition() &&
+         (getNameLoc().isInvalid() || getNameLoc() == getDecl()->getLocation());
+  }
 };
 
 /// \brief Wrapper for source info for record types.
@@ -1408,6 +1418,8 @@ class TypeOfTypeLoc
 class DecltypeTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
                                                          DecltypeTypeLoc,
                                                          DecltypeType> {
+public:
+  Expr *getUnderlyingExpr() const { return getTypePtr()->getUnderlyingExpr(); }
 };
 
 struct UnaryTransformTypeLocInfo {
@@ -1718,6 +1730,62 @@ class PackExpansionTypeLoc
   }
 };
 
+struct AtomicTypeLocInfo {
+  SourceLocation KWLoc, LParenLoc, RParenLoc;
+};
+
+class AtomicTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, AtomicTypeLoc,
+                                             AtomicType, AtomicTypeLocInfo> {
+public:  
+  TypeLoc getValueLoc() const {
+    return this->getInnerTypeLoc();
+  }
+
+  SourceRange getLocalSourceRange() const {
+    return SourceRange(getKWLoc(), getRParenLoc());
+  }
+
+  SourceLocation getKWLoc() const {
+    return this->getLocalData()->KWLoc;
+  }
+  void setKWLoc(SourceLocation Loc) {
+    this->getLocalData()->KWLoc = Loc;
+  }
+
+  SourceLocation getLParenLoc() const {
+    return this->getLocalData()->LParenLoc;
+  }
+  void setLParenLoc(SourceLocation Loc) {
+    this->getLocalData()->LParenLoc = Loc;
+  }
+
+  SourceLocation getRParenLoc() const {
+    return this->getLocalData()->RParenLoc;
+  }
+  void setRParenLoc(SourceLocation Loc) {
+    this->getLocalData()->RParenLoc = Loc;
+  }
+
+  SourceRange getParensRange() const {
+    return SourceRange(getLParenLoc(), getRParenLoc());
+  }
+  void setParensRange(SourceRange Range) {
+    setLParenLoc(Range.getBegin());
+    setRParenLoc(Range.getEnd());
+  }
+
+  void initializeLocal(ASTContext &Context, SourceLocation Loc) {
+    setKWLoc(Loc);
+    setLParenLoc(Loc);
+    setRParenLoc(Loc);
+  }
+
+  QualType getInnerType() const {
+    return this->getTypePtr()->getValueType();
+  }
+};
+
+
 }
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/TypeNodes.def b/contrib/llvm/tools/clang/include/clang/AST/TypeNodes.def
index 0792d0d9b30d..d5c485f8a3b9 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/TypeNodes.def
+++ b/contrib/llvm/tools/clang/include/clang/AST/TypeNodes.def
@@ -102,9 +102,10 @@ DEPENDENT_TYPE(PackExpansion, Type)
 TYPE(ObjCObject, Type)
 TYPE(ObjCInterface, ObjCObjectType)
 TYPE(ObjCObjectPointer, Type)
+TYPE(Atomic, Type)
 
 #ifdef LAST_TYPE
-LAST_TYPE(ObjCObjectPointer)
+LAST_TYPE(Atomic)
 #undef LAST_TYPE
 #endif
 
diff --git a/contrib/llvm/tools/clang/include/clang/AST/TypeVisitor.h b/contrib/llvm/tools/clang/include/clang/AST/TypeVisitor.h
index c52926b1fc31..9eebc4b9ea07 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/TypeVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/TypeVisitor.h
@@ -28,7 +28,7 @@ class TypeVisitor {
   RetTy Visit(const Type *T) {
     // Top switch stmt: dispatch to VisitFooType for each FooType.
     switch (T->getTypeClass()) {
-    default: assert(0 && "Unknown type class!");
+    default: llvm_unreachable("Unknown type class!");
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) case Type::CLASS: DISPATCH(CLASS##Type);
 #include "clang/AST/TypeNodes.def"
diff --git a/contrib/llvm/tools/clang/include/clang/AST/UnresolvedSet.h b/contrib/llvm/tools/clang/include/clang/AST/UnresolvedSet.h
index a548b0b704d1..0918dc44aa73 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/UnresolvedSet.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/UnresolvedSet.h
@@ -25,7 +25,7 @@ namespace clang {
 /// non-const iterator.
 class UnresolvedSetIterator {
 private:
-  typedef llvm::SmallVectorImpl<DeclAccessPair> DeclsTy;
+  typedef SmallVectorImpl<DeclAccessPair> DeclsTy;
   typedef DeclsTy::iterator IteratorTy;
 
   IteratorTy ir;
@@ -177,7 +177,7 @@ class UnresolvedSetImpl {
 /// A set of unresolved declarations 
 template <unsigned InlineCapacity> class UnresolvedSet :
     public UnresolvedSetImpl {
-  llvm::SmallVector<DeclAccessPair, InlineCapacity> Decls;
+  SmallVector<DeclAccessPair, InlineCapacity> Decls;
 };
 
   
diff --git a/contrib/llvm/tools/clang/include/clang/AST/VTTBuilder.h b/contrib/llvm/tools/clang/include/clang/AST/VTTBuilder.h
new file mode 100644
index 000000000000..6756dd1e993b
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/AST/VTTBuilder.h
@@ -0,0 +1,176 @@
+//===--- VTTBuilder.h - C++ VTT layout builder --------------------*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with generation of the layout of virtual table
+// tables (VTT).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_VTTBUILDER_H
+#define LLVM_CLANG_AST_VTTBUILDER_H
+
+#include "clang/AST/BaseSubobject.h"
+#include "clang/AST/CXXInheritance.h"
+#include "clang/AST/GlobalDecl.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/Basic/ABI.h"
+#include "llvm/ADT/SetVector.h"
+#include <utility>
+
+namespace clang {
+
+class VTTVTable {
+  llvm::PointerIntPair<const CXXRecordDecl *, 1, bool> BaseAndIsVirtual;
+  CharUnits BaseOffset;
+
+public:
+  VTTVTable() {}
+  VTTVTable(const CXXRecordDecl *Base, CharUnits BaseOffset, bool BaseIsVirtual)
+    : BaseAndIsVirtual(Base, BaseIsVirtual), BaseOffset(BaseOffset) {}
+  VTTVTable(BaseSubobject Base, bool BaseIsVirtual)
+    : BaseAndIsVirtual(Base.getBase(), BaseIsVirtual),
+      BaseOffset(Base.getBaseOffset()) {}
+
+  const CXXRecordDecl *getBase() const {
+    return BaseAndIsVirtual.getPointer();
+  }
+
+  CharUnits getBaseOffset() const {
+    return BaseOffset;
+  }
+
+  bool isVirtual() const {
+    return BaseAndIsVirtual.getInt();
+  }
+
+  BaseSubobject getBaseSubobject() const {
+    return BaseSubobject(getBase(), getBaseOffset());
+  }
+};
+
+struct VTTComponent {
+  uint64_t VTableIndex;
+  BaseSubobject VTableBase;
+
+  VTTComponent() {}
+  VTTComponent(uint64_t VTableIndex, BaseSubobject VTableBase)
+    : VTableIndex(VTableIndex), VTableBase(VTableBase) {}
+};
+
+/// VTT builder - Class for building VTT layout information.
+class VTTBuilder {
+  
+  ASTContext &Ctx;
+
+  /// MostDerivedClass - The most derived class for which we're building this
+  /// vtable.
+  const CXXRecordDecl *MostDerivedClass;
+
+  typedef SmallVector<VTTVTable, 64> VTTVTablesVectorTy;
+  
+  /// VTTVTables - The VTT vtables.
+  VTTVTablesVectorTy VTTVTables;
+  
+  typedef SmallVector<VTTComponent, 64> VTTComponentsVectorTy;
+  
+  /// VTTComponents - The VTT components.
+  VTTComponentsVectorTy VTTComponents;
+  
+  /// MostDerivedClassLayout - the AST record layout of the most derived class.
+  const ASTRecordLayout &MostDerivedClassLayout;
+
+  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
+
+  typedef llvm::DenseMap<BaseSubobject, uint64_t> AddressPointsMapTy;
+
+  /// SubVTTIndicies - The sub-VTT indices for the bases of the most derived
+  /// class.
+  llvm::DenseMap<BaseSubobject, uint64_t> SubVTTIndicies;
+
+  /// SecondaryVirtualPointerIndices - The secondary virtual pointer indices of
+  /// all subobjects of the most derived class.
+  llvm::DenseMap<BaseSubobject, uint64_t> SecondaryVirtualPointerIndices;
+
+  /// GenerateDefinition - Whether the VTT builder should generate LLVM IR for
+  /// the VTT.
+  bool GenerateDefinition;
+
+  /// AddVTablePointer - Add a vtable pointer to the VTT currently being built.
+  ///
+  /// \param AddressPoints - If the vtable is a construction vtable, this has
+  /// the address points for it.
+  void AddVTablePointer(BaseSubobject Base, uint64_t VTableIndex,
+                        const CXXRecordDecl *VTableClass);
+                        
+  /// LayoutSecondaryVTTs - Lay out the secondary VTTs of the given base 
+  /// subobject.
+  void LayoutSecondaryVTTs(BaseSubobject Base);
+  
+  /// LayoutSecondaryVirtualPointers - Lay out the secondary virtual pointers
+  /// for the given base subobject.
+  ///
+  /// \param BaseIsMorallyVirtual whether the base subobject is a virtual base
+  /// or a direct or indirect base of a virtual base.
+  ///
+  /// \param AddressPoints - If the vtable is a construction vtable, this has
+  /// the address points for it.
+  void LayoutSecondaryVirtualPointers(BaseSubobject Base, 
+                                      bool BaseIsMorallyVirtual,
+                                      uint64_t VTableIndex,
+                                      const CXXRecordDecl *VTableClass,
+                                      VisitedVirtualBasesSetTy &VBases);
+  
+  /// LayoutSecondaryVirtualPointers - Lay out the secondary virtual pointers
+  /// for the given base subobject.
+  ///
+  /// \param AddressPoints - If the vtable is a construction vtable, this has
+  /// the address points for it.
+  void LayoutSecondaryVirtualPointers(BaseSubobject Base, 
+                                      uint64_t VTableIndex);
+
+  /// LayoutVirtualVTTs - Lay out the VTTs for the virtual base classes of the
+  /// given record decl.
+  void LayoutVirtualVTTs(const CXXRecordDecl *RD,
+                         VisitedVirtualBasesSetTy &VBases);
+  
+  /// LayoutVTT - Will lay out the VTT for the given subobject, including any
+  /// secondary VTTs, secondary virtual pointers and virtual VTTs.
+  void LayoutVTT(BaseSubobject Base, bool BaseIsVirtual);
+  
+public:
+  VTTBuilder(ASTContext &Ctx, const CXXRecordDecl *MostDerivedClass,
+             bool GenerateDefinition);
+
+  // getVTTComponents - Returns a reference to the VTT components.
+  const VTTComponentsVectorTy &getVTTComponents() const {
+    return VTTComponents;
+  }
+  
+  // getVTTVTables - Returns a reference to the VTT vtables.
+  const VTTVTablesVectorTy &getVTTVTables() const {
+    return VTTVTables;
+  }
+  
+  /// getSubVTTIndicies - Returns a reference to the sub-VTT indices.
+  const llvm::DenseMap<BaseSubobject, uint64_t> &getSubVTTIndicies() const {
+    return SubVTTIndicies;
+  }
+  
+  /// getSecondaryVirtualPointerIndices - Returns a reference to the secondary
+  /// virtual pointer indices.
+  const llvm::DenseMap<BaseSubobject, uint64_t> &
+  getSecondaryVirtualPointerIndices() const {
+    return SecondaryVirtualPointerIndices;
+  }
+
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/AST/VTableBuilder.h b/contrib/llvm/tools/clang/include/clang/AST/VTableBuilder.h
new file mode 100644
index 000000000000..59bab0382604
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/AST/VTableBuilder.h
@@ -0,0 +1,357 @@
+//===--- VTableBuilder.h - C++ vtable layout builder --------------*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with generation of the layout of virtual tables.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_VTABLEBUILDER_H
+#define LLVM_CLANG_AST_VTABLEBUILDER_H
+
+#include "clang/AST/BaseSubobject.h"
+#include "clang/AST/CXXInheritance.h"
+#include "clang/AST/GlobalDecl.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/Basic/ABI.h"
+#include "llvm/ADT/SetVector.h"
+#include <utility>
+
+namespace clang {
+  class CXXRecordDecl;
+
+/// VTableComponent - Represents a single component in a vtable.
+class VTableComponent {
+public:
+  enum Kind {
+    CK_VCallOffset,
+    CK_VBaseOffset,
+    CK_OffsetToTop,
+    CK_RTTI,
+    CK_FunctionPointer,
+    
+    /// CK_CompleteDtorPointer - A pointer to the complete destructor.
+    CK_CompleteDtorPointer,
+    
+    /// CK_DeletingDtorPointer - A pointer to the deleting destructor.
+    CK_DeletingDtorPointer,
+    
+    /// CK_UnusedFunctionPointer - In some cases, a vtable function pointer
+    /// will end up never being called. Such vtable function pointers are
+    /// represented as a CK_UnusedFunctionPointer. 
+    CK_UnusedFunctionPointer
+  };
+
+  VTableComponent() { }
+
+  static VTableComponent MakeVCallOffset(CharUnits Offset) {
+    return VTableComponent(CK_VCallOffset, Offset);
+  }
+
+  static VTableComponent MakeVBaseOffset(CharUnits Offset) {
+    return VTableComponent(CK_VBaseOffset, Offset);
+  }
+
+  static VTableComponent MakeOffsetToTop(CharUnits Offset) {
+    return VTableComponent(CK_OffsetToTop, Offset);
+  }
+  
+  static VTableComponent MakeRTTI(const CXXRecordDecl *RD) {
+    return VTableComponent(CK_RTTI, reinterpret_cast<uintptr_t>(RD));
+  }
+
+  static VTableComponent MakeFunction(const CXXMethodDecl *MD) {
+    assert(!isa<CXXDestructorDecl>(MD) && 
+           "Don't use MakeFunction with destructors!");
+
+    return VTableComponent(CK_FunctionPointer, 
+                           reinterpret_cast<uintptr_t>(MD));
+  }
+  
+  static VTableComponent MakeCompleteDtor(const CXXDestructorDecl *DD) {
+    return VTableComponent(CK_CompleteDtorPointer,
+                           reinterpret_cast<uintptr_t>(DD));
+  }
+
+  static VTableComponent MakeDeletingDtor(const CXXDestructorDecl *DD) {
+    return VTableComponent(CK_DeletingDtorPointer, 
+                           reinterpret_cast<uintptr_t>(DD));
+  }
+
+  static VTableComponent MakeUnusedFunction(const CXXMethodDecl *MD) {
+    assert(!isa<CXXDestructorDecl>(MD) && 
+           "Don't use MakeUnusedFunction with destructors!");
+    return VTableComponent(CK_UnusedFunctionPointer,
+                           reinterpret_cast<uintptr_t>(MD));                           
+  }
+
+  static VTableComponent getFromOpaqueInteger(uint64_t I) {
+    return VTableComponent(I);
+  }
+
+  /// getKind - Get the kind of this vtable component.
+  Kind getKind() const {
+    return (Kind)(Value & 0x7);
+  }
+
+  CharUnits getVCallOffset() const {
+    assert(getKind() == CK_VCallOffset && "Invalid component kind!");
+    
+    return getOffset();
+  }
+
+  CharUnits getVBaseOffset() const {
+    assert(getKind() == CK_VBaseOffset && "Invalid component kind!");
+    
+    return getOffset();
+  }
+
+  CharUnits getOffsetToTop() const {
+    assert(getKind() == CK_OffsetToTop && "Invalid component kind!");
+    
+    return getOffset();
+  }
+  
+  const CXXRecordDecl *getRTTIDecl() const {
+    assert(getKind() == CK_RTTI && "Invalid component kind!");
+    
+    return reinterpret_cast<CXXRecordDecl *>(getPointer());
+  }
+  
+  const CXXMethodDecl *getFunctionDecl() const {
+    assert(getKind() == CK_FunctionPointer);
+    
+    return reinterpret_cast<CXXMethodDecl *>(getPointer());
+  }
+
+  const CXXDestructorDecl *getDestructorDecl() const {
+    assert((getKind() == CK_CompleteDtorPointer ||
+            getKind() == CK_DeletingDtorPointer) && "Invalid component kind!");
+    
+    return reinterpret_cast<CXXDestructorDecl *>(getPointer());
+  }
+
+  const CXXMethodDecl *getUnusedFunctionDecl() const {
+    assert(getKind() == CK_UnusedFunctionPointer);
+    
+    return reinterpret_cast<CXXMethodDecl *>(getPointer());
+  }
+  
+private:
+  VTableComponent(Kind ComponentKind, CharUnits Offset) {
+    assert((ComponentKind == CK_VCallOffset || 
+            ComponentKind == CK_VBaseOffset ||
+            ComponentKind == CK_OffsetToTop) && "Invalid component kind!");
+    assert(Offset.getQuantity() <= ((1LL << 56) - 1) && "Offset is too big!");
+    
+    Value = ((Offset.getQuantity() << 3) | ComponentKind);
+  }
+
+  VTableComponent(Kind ComponentKind, uintptr_t Ptr) {
+    assert((ComponentKind == CK_RTTI || 
+            ComponentKind == CK_FunctionPointer ||
+            ComponentKind == CK_CompleteDtorPointer ||
+            ComponentKind == CK_DeletingDtorPointer ||
+            ComponentKind == CK_UnusedFunctionPointer) &&
+            "Invalid component kind!");
+    
+    assert((Ptr & 7) == 0 && "Pointer not sufficiently aligned!");
+    
+    Value = Ptr | ComponentKind;
+  }
+  
+  CharUnits getOffset() const {
+    assert((getKind() == CK_VCallOffset || getKind() == CK_VBaseOffset ||
+            getKind() == CK_OffsetToTop) && "Invalid component kind!");
+    
+    return CharUnits::fromQuantity(Value >> 3);
+  }
+
+  uintptr_t getPointer() const {
+    assert((getKind() == CK_RTTI || 
+            getKind() == CK_FunctionPointer ||
+            getKind() == CK_CompleteDtorPointer ||
+            getKind() == CK_DeletingDtorPointer ||
+            getKind() == CK_UnusedFunctionPointer) &&
+           "Invalid component kind!");
+    
+    return static_cast<uintptr_t>(Value & ~7ULL);
+  }
+  
+  explicit VTableComponent(uint64_t Value)
+    : Value(Value) { }
+
+  /// The kind is stored in the lower 3 bits of the value. For offsets, we
+  /// make use of the facts that classes can't be larger than 2^55 bytes,
+  /// so we store the offset in the lower part of the 61 bytes that remain.
+  /// (The reason that we're not simply using a PointerIntPair here is that we
+  /// need the offsets to be 64-bit, even when on a 32-bit machine).
+  int64_t Value;
+};
+
+class VTableLayout {
+public:
+  typedef std::pair<uint64_t, ThunkInfo> VTableThunkTy;
+  typedef SmallVector<ThunkInfo, 1> ThunkInfoVectorTy;
+
+  typedef const VTableComponent *vtable_component_iterator;
+  typedef const VTableThunkTy *vtable_thunk_iterator;
+
+  typedef llvm::DenseMap<BaseSubobject, uint64_t> AddressPointsMapTy;
+private:
+  uint64_t NumVTableComponents;
+  VTableComponent *VTableComponents;
+
+  /// VTableThunks - Contains thunks needed by vtables.
+  uint64_t NumVTableThunks;
+  VTableThunkTy *VTableThunks;
+  
+  /// Address points - Address points for all vtables.
+  AddressPointsMapTy AddressPoints;
+
+public:
+  VTableLayout(uint64_t NumVTableComponents,
+               const VTableComponent *VTableComponents,
+               uint64_t NumVTableThunks,
+               const VTableThunkTy *VTableThunks,
+               const AddressPointsMapTy &AddressPoints);
+  ~VTableLayout();
+
+  uint64_t getNumVTableComponents() const {
+    return NumVTableComponents;
+  }
+
+  vtable_component_iterator vtable_component_begin() const {
+   return VTableComponents;
+  }
+
+  vtable_component_iterator vtable_component_end() const {
+   return VTableComponents+NumVTableComponents;
+  }
+
+  uint64_t getNumVTableThunks() const {
+    return NumVTableThunks;
+  }
+
+  vtable_thunk_iterator vtable_thunk_begin() const {
+   return VTableThunks;
+  }
+
+  vtable_thunk_iterator vtable_thunk_end() const {
+   return VTableThunks+NumVTableThunks;
+  }
+
+  uint64_t getAddressPoint(BaseSubobject Base) const {
+    assert(AddressPoints.count(Base) &&
+           "Did not find address point!");
+
+    uint64_t AddressPoint = AddressPoints.lookup(Base);
+    assert(AddressPoint && "Address point must not be zero!");
+
+    return AddressPoint;
+  }
+
+  const AddressPointsMapTy &getAddressPoints() const {
+    return AddressPoints;
+  }
+};
+
+class VTableContext {
+  ASTContext &Context;
+
+public:
+  typedef SmallVector<std::pair<uint64_t, ThunkInfo>, 1> 
+    VTableThunksTy;
+  typedef SmallVector<ThunkInfo, 1> ThunkInfoVectorTy;
+  
+private:
+  /// MethodVTableIndices - Contains the index (relative to the vtable address
+  /// point) where the function pointer for a virtual function is stored.
+  typedef llvm::DenseMap<GlobalDecl, int64_t> MethodVTableIndicesTy;
+  MethodVTableIndicesTy MethodVTableIndices;
+
+  typedef llvm::DenseMap<const CXXRecordDecl *, const VTableLayout *>
+    VTableLayoutMapTy;
+  VTableLayoutMapTy VTableLayouts;
+
+  /// NumVirtualFunctionPointers - Contains the number of virtual function 
+  /// pointers in the vtable for a given record decl.
+  llvm::DenseMap<const CXXRecordDecl *, uint64_t> NumVirtualFunctionPointers;
+
+  typedef std::pair<const CXXRecordDecl *,
+                    const CXXRecordDecl *> ClassPairTy;
+
+  /// VirtualBaseClassOffsetOffsets - Contains the vtable offset (relative to 
+  /// the address point) in chars where the offsets for virtual bases of a class
+  /// are stored.
+  typedef llvm::DenseMap<ClassPairTy, CharUnits> 
+    VirtualBaseClassOffsetOffsetsMapTy;
+  VirtualBaseClassOffsetOffsetsMapTy VirtualBaseClassOffsetOffsets;
+
+  typedef llvm::DenseMap<const CXXMethodDecl *, ThunkInfoVectorTy> ThunksMapTy;
+  
+  /// Thunks - Contains all thunks that a given method decl will need.
+  ThunksMapTy Thunks;
+
+  void ComputeMethodVTableIndices(const CXXRecordDecl *RD);
+
+  /// ComputeVTableRelatedInformation - Compute and store all vtable related
+  /// information (vtable layout, vbase offset offsets, thunks etc) for the
+  /// given record decl.
+  void ComputeVTableRelatedInformation(const CXXRecordDecl *RD);
+
+public:
+  VTableContext(ASTContext &Context) : Context(Context) {}
+  ~VTableContext();
+
+  const VTableLayout &getVTableLayout(const CXXRecordDecl *RD) {
+    ComputeVTableRelatedInformation(RD);
+    assert(VTableLayouts.count(RD) && "No layout for this record decl!");
+    
+    return *VTableLayouts[RD];
+  }
+
+  VTableLayout *
+  createConstructionVTableLayout(const CXXRecordDecl *MostDerivedClass,
+                                 CharUnits MostDerivedClassOffset,
+                                 bool MostDerivedClassIsVirtual,
+                                 const CXXRecordDecl *LayoutClass);
+
+  const ThunkInfoVectorTy *getThunkInfo(const CXXMethodDecl *MD) {
+    ComputeVTableRelatedInformation(MD->getParent());
+
+    ThunksMapTy::const_iterator I = Thunks.find(MD);
+    if (I == Thunks.end()) {
+      // We did not find a thunk for this method.
+      return 0;
+    }
+
+    return &I->second;
+  }
+
+  /// getNumVirtualFunctionPointers - Return the number of virtual function
+  /// pointers in the vtable for a given record decl.
+  uint64_t getNumVirtualFunctionPointers(const CXXRecordDecl *RD);
+  
+  /// getMethodVTableIndex - Return the index (relative to the vtable address
+  /// point) where the function pointer for the given virtual function is
+  /// stored.
+  uint64_t getMethodVTableIndex(GlobalDecl GD);
+
+  /// getVirtualBaseOffsetOffset - Return the offset in chars (relative to the
+  /// vtable address point) where the offset of the virtual base that contains 
+  /// the given base is stored, otherwise, if no virtual base contains the given
+  /// class, return 0.  Base must be a virtual base class or an unambigious
+  /// base.
+  CharUnits getVirtualBaseOffsetOffset(const CXXRecordDecl *RD,
+                                       const CXXRecordDecl *VBase);
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/FormatString.h b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/FormatString.h
index 069ead3f5efc..a3f9c5394af7 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/FormatString.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/FormatString.h
@@ -169,8 +169,8 @@ class ConversionSpecifier {
     return Position;
   }
   
-  llvm::StringRef getCharacters() const {
-    return llvm::StringRef(getStart(), getLength());
+  StringRef getCharacters() const {
+    return StringRef(getStart(), getLength());
   }
   
   bool consumesDataArgument() const {
@@ -276,7 +276,7 @@ class OptionalAmount {
 
   ArgTypeResult getArgType(ASTContext &Ctx) const;
 
-  void toString(llvm::raw_ostream &os) const;
+  void toString(raw_ostream &os) const;
 
   bool usesPositionalArg() const { return (bool) UsesPositionalArg; }
   unsigned getPositionalArgIndex() const {
@@ -470,7 +470,7 @@ class PrintfSpecifier : public analyze_format_string::FormatSpecifier {
     /// was not successful.
   bool fixType(QualType QT);
 
-  void toString(llvm::raw_ostream &os) const;
+  void toString(raw_ostream &os) const;
 
     // Validation methods - to check if any element results in undefined behavior
   bool hasValidPlusPrefix() const;
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/LiveVariables.h b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/LiveVariables.h
index fbbd2613e7c2..302ae1c41f95 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/LiveVariables.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/LiveVariables.h
@@ -1,4 +1,4 @@
-//===- LiveVariables.h - Live Variable Analysis for Source CFGs -*- C++ --*-===//
+//===- LiveVariables.h - Live Variable Analysis for Source CFGs -*- C++ --*-//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -14,110 +14,105 @@
 #ifndef LLVM_CLANG_LIVEVARIABLES_H
 #define LLVM_CLANG_LIVEVARIABLES_H
 
+#include "clang/Analysis/AnalysisContext.h"
 #include "clang/AST/Decl.h"
-#include "clang/Analysis/Support/BlkExprDeclBitVector.h"
-#include "clang/Analysis/FlowSensitive/DataflowValues.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/ImmutableSet.h"
 
 namespace clang {
 
+class CFG;
+class CFGBlock;
 class Stmt;
 class DeclRefExpr;
 class SourceManager;
-class AnalysisContext;
-
-struct LiveVariables_ValueTypes {
-
-  struct ObserverTy;
-
-  // We keep dataflow state for declarations and block-level expressions;
-  typedef StmtDeclBitVector_Types::ValTy ValTy;
-
-  // We need to keep track of both declarations and CFGBlock-level expressions,
-  // (so that we don't explore such expressions twice).  We also want
-  // to compute liveness information for block-level expressions, since these
-  // act as "temporary" values.
-
-  struct AnalysisDataTy : public StmtDeclBitVector_Types::AnalysisDataTy {
-    ObserverTy* Observer;
-    ValTy AlwaysLive;
-    AnalysisContext *AC;
-    bool killAtAssign;
-
-    AnalysisDataTy() : Observer(NULL), AC(NULL), killAtAssign(true) {}
-  };
-
-  //===-----------------------------------------------------===//
-  // ObserverTy - Observer for uninitialized values queries.
-  //===-----------------------------------------------------===//
-
-  struct ObserverTy {
-    virtual ~ObserverTy() {}
-
-    /// ObserveStmt - A callback invoked right before invoking the
-    ///  liveness transfer function on the given statement.
-    virtual void ObserveStmt(Stmt* S, const CFGBlock *currentBlock,
-                             const AnalysisDataTy& AD,
-                             const ValTy& V) {}
-
-    virtual void ObserverKill(DeclRefExpr* DR) {}
-  };
-};
-
-class LiveVariables : public DataflowValues<LiveVariables_ValueTypes,
-                                            dataflow::backward_analysis_tag> {
-
-
+  
+class LiveVariables : public ManagedAnalysis {
 public:
-  typedef LiveVariables_ValueTypes::ObserverTy ObserverTy;
+  class LivenessValues {
+  public:
 
-  LiveVariables(AnalysisContext &AC, bool killAtAssign = true);
+    llvm::ImmutableSet<const Stmt *> liveStmts;
+    llvm::ImmutableSet<const VarDecl *> liveDecls;
+    
+    bool equals(const LivenessValues &V) const;
 
-  /// IsLive - Return true if a variable is live at the end of a
+    LivenessValues()
+      : liveStmts(0), liveDecls(0) {}
+
+    LivenessValues(llvm::ImmutableSet<const Stmt *> LiveStmts,
+                   llvm::ImmutableSet<const VarDecl *> LiveDecls)
+      : liveStmts(LiveStmts), liveDecls(LiveDecls) {}
+
+    ~LivenessValues() {}
+    
+    bool isLive(const Stmt *S) const;
+    bool isLive(const VarDecl *D) const;
+    
+    friend class LiveVariables;    
+  };
+  
+  struct Observer {
+    virtual ~Observer() {}
+    
+    /// A callback invoked right before invoking the
+    ///  liveness transfer function on the given statement.
+    virtual void observeStmt(const Stmt *S,
+                             const CFGBlock *currentBlock,
+                             const LivenessValues& V) {}
+    
+    /// Called when the live variables analysis registers
+    /// that a variable is killed.
+    virtual void observerKill(const DeclRefExpr *DR) {}
+  };    
+
+
+  virtual ~LiveVariables();
+  
+  /// Compute the liveness information for a given CFG.
+  static LiveVariables *computeLiveness(AnalysisContext &analysisContext,
+                                        bool killAtAssign);
+  
+  /// Return true if a variable is live at the end of a
   /// specified block.
-  bool isLive(const CFGBlock* B, const VarDecl* D) const;
-
-  /// IsLive - Returns true if a variable is live at the beginning of the
+  bool isLive(const CFGBlock *B, const VarDecl *D);
+  
+  /// Returns true if a variable is live at the beginning of the
   ///  the statement.  This query only works if liveness information
   ///  has been recorded at the statement level (see runOnAllBlocks), and
   ///  only returns liveness information for block-level expressions.
-  bool isLive(const Stmt* S, const VarDecl* D) const;
-
-  /// IsLive - Returns true the block-level expression "value" is live
+  bool isLive(const Stmt *S, const VarDecl *D);
+  
+  /// Returns true the block-level expression "value" is live
   ///  before the given block-level expression (see runOnAllBlocks).
-  bool isLive(const Stmt* Loc, const Stmt* StmtVal) const;
+  bool isLive(const Stmt *Loc, const Stmt *StmtVal);
+    
+  /// Print to stderr the liveness information associated with
+  /// each basic block.
+  void dumpBlockLiveness(const SourceManager& M);
 
-  /// IsLive - Return true if a variable is live according to the
-  ///  provided livness bitvector.
-  bool isLive(const ValTy& V, const VarDecl* D) const;
-
-  /// dumpLiveness - Print to stderr the liveness information encoded
-  ///  by a specified bitvector.
-  void dumpLiveness(const ValTy& V, const SourceManager& M) const;
-
-  /// dumpBlockLiveness - Print to stderr the liveness information
-  ///  associated with each basic block.
-  void dumpBlockLiveness(const SourceManager& M) const;
-
-  /// getNumDecls - Return the number of variables (declarations) that
-  ///  whose liveness status is being tracked by the dataflow
-  ///  analysis.
-  unsigned getNumDecls() const { return getAnalysisData().getNumDecls(); }
-
-  /// IntializeValues - This routine can perform extra initialization, but
-  ///  for LiveVariables this does nothing since all that logic is in
-  ///  the constructor.
-  void InitializeValues(const CFG& cfg) {}
-
-  void runOnCFG(CFG& cfg);
-
-  /// runOnAllBlocks - Propagate the dataflow values once for each block,
-  ///  starting from the current dataflow values.  'recordStmtValues' indicates
-  ///  whether the method should store dataflow values per each individual
-  ///  block-level expression.
-  void runOnAllBlocks(const CFG& cfg, ObserverTy* Obs,
-                      bool recordStmtValues=false);
+  void runOnAllBlocks(Observer &obs);
+  
+  static LiveVariables *create(AnalysisContext &analysisContext) {
+    return computeLiveness(analysisContext, true);
+  }
+  
+  static const void *getTag();
+  
+private:
+  LiveVariables(void *impl);
+  void *impl;
 };
-
+  
+class RelaxedLiveVariables : public LiveVariables {
+public:
+  static LiveVariables *create(AnalysisContext &analysisContext) {
+    return computeLiveness(analysisContext, false);
+  }
+  
+  static const void *getTag();
+};
+  
 } // end namespace clang
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ReachableCode.h b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ReachableCode.h
index e0c84f97fd33..6cf7fa42ff45 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ReachableCode.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ReachableCode.h
@@ -45,7 +45,7 @@ class Callback {
 
 /// ScanReachableFromBlock - Mark all blocks reachable from Start.
 /// Returns the total number of blocks that were marked reachable.  
-unsigned ScanReachableFromBlock(const CFGBlock &Start,
+unsigned ScanReachableFromBlock(const CFGBlock *Start,
                                 llvm::BitVector &Reachable);
 
 void FindUnreachableCode(AnalysisContext &AC, Callback &CB);
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ThreadSafety.h b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ThreadSafety.h
new file mode 100644
index 000000000000..a32505624a22
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/ThreadSafety.h
@@ -0,0 +1,153 @@
+//===- ThreadSafety.h ------------------------------------------*- C++ --*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//
+// A intra-procedural analysis for thread safety (e.g. deadlocks and race
+// conditions), based off of an annotation system.
+//
+// See http://clang.llvm.org/docs/LanguageExtensions.html#threadsafety for more
+// information.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_THREADSAFETY_H
+#define LLVM_CLANG_THREADSAFETY_H
+
+#include "clang/Analysis/AnalysisContext.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+namespace thread_safety {
+
+/// This enum distinguishes between different kinds of operations that may
+/// need to be protected by locks. We use this enum in error handling.
+enum ProtectedOperationKind {
+  POK_VarDereference, /// Dereferencing a variable (e.g. p in *p = 5;)
+  POK_VarAccess, /// Reading or writing a variable (e.g. x in x = 5;)
+  POK_FunctionCall /// Making a function call (e.g. fool())
+};
+
+/// This enum distinguishes between different kinds of lock actions. For
+/// example, it is an error to write a variable protected by shared version of a
+/// mutex.
+enum LockKind {
+  LK_Shared, /// Shared/reader lock of a mutex
+  LK_Exclusive /// Exclusive/writer lock of a mutex
+};
+
+/// This enum distinguishes between different ways to access (read or write) a
+/// variable.
+enum AccessKind {
+  AK_Read, /// Reading a variable
+  AK_Written /// Writing a variable
+};
+
+/// This enum distinguishes between different situations where we warn due to
+/// inconsistent locking.
+/// \enum SK_LockedSomeLoopIterations -- a mutex is locked for some but not all
+/// loop iterations.
+/// \enum SK_LockedSomePredecessors -- a mutex is locked in some but not all
+/// predecessors of a CFGBlock.
+/// \enum SK_LockedAtEndOfFunction -- a mutex is still locked at the end of a
+/// function.
+enum LockErrorKind {
+  LEK_LockedSomeLoopIterations,
+  LEK_LockedSomePredecessors,
+  LEK_LockedAtEndOfFunction
+};
+
+/// Handler class for thread safety warnings.
+class ThreadSafetyHandler {
+public:
+  typedef llvm::StringRef Name;
+  virtual ~ThreadSafetyHandler() = 0;
+
+  /// Warn about lock expressions which fail to resolve to lockable objects.
+  /// \param Loc -- the SourceLocation of the unresolved expression.
+  virtual void handleInvalidLockExp(SourceLocation Loc) {}
+
+  /// Warn about unlock function calls that do not have a prior matching lock
+  /// expression.
+  /// \param LockName -- A StringRef name for the lock expression, to be printed
+  /// in the error message.
+  /// \param Loc -- The SourceLocation of the Unlock
+  virtual void handleUnmatchedUnlock(Name LockName, SourceLocation Loc) {}
+
+  /// Warn about lock function calls for locks which are already held.
+  /// \param LockName -- A StringRef name for the lock expression, to be printed
+  /// in the error message.
+  /// \param Loc -- The location of the second lock expression.
+  virtual void handleDoubleLock(Name LockName, SourceLocation Loc) {}
+
+  /// Warn about situations where a mutex is sometimes held and sometimes not.
+  /// The three situations are:
+  /// 1. a mutex is locked on an "if" branch but not the "else" branch,
+  /// 2, or a mutex is only held at the start of some loop iterations,
+  /// 3. or when a mutex is locked but not unlocked inside a function.
+  /// \param LockName -- A StringRef name for the lock expression, to be printed
+  /// in the error message.
+  /// \param Loc -- The location of the lock expression where the mutex is locked
+  /// \param LEK -- which of the three above cases we should warn for
+  virtual void handleMutexHeldEndOfScope(Name LockName, SourceLocation Loc,
+                                         LockErrorKind LEK){}
+
+  /// Warn when a mutex is held exclusively and shared at the same point. For
+  /// example, if a mutex is locked exclusively during an if branch and shared
+  /// during the else branch.
+  /// \param LockName -- A StringRef name for the lock expression, to be printed
+  /// in the error message.
+  /// \param Loc1 -- The location of the first lock expression.
+  /// \param Loc2 -- The location of the second lock expression.
+  virtual void handleExclusiveAndShared(Name LockName, SourceLocation Loc1,
+                                        SourceLocation Loc2) {}
+
+  /// Warn when a protected operation occurs while no locks are held.
+  /// \param D -- The decl for the protected variable or function
+  /// \param POK -- The kind of protected operation (e.g. variable access)
+  /// \param AK -- The kind of access (i.e. read or write) that occurred
+  /// \param Loc -- The location of the protected operation.
+  virtual void handleNoMutexHeld(const NamedDecl *D, ProtectedOperationKind POK,
+                                 AccessKind AK, SourceLocation Loc) {}
+
+  /// Warn when a protected operation occurs while the specific mutex protecting
+  /// the operation is not locked.
+  /// \param LockName -- A StringRef name for the lock expression, to be printed
+  /// in the error message.
+  /// \param D -- The decl for the protected variable or function
+  /// \param POK -- The kind of protected operation (e.g. variable access)
+  /// \param AK -- The kind of access (i.e. read or write) that occurred
+  /// \param Loc -- The location of the protected operation.
+  virtual void handleMutexNotHeld(const NamedDecl *D,
+                                  ProtectedOperationKind POK, Name LockName,
+                                  LockKind LK, SourceLocation Loc) {}
+
+  /// Warn when a function is called while an excluded mutex is locked. For
+  /// example, the mutex may be locked inside the function.
+  /// \param FunName -- The name of the function
+  /// \param LockName -- A StringRef name for the lock expression, to be printed
+  /// in the error message.
+  /// \param Loc -- The location of the function call.
+  virtual void handleFunExcludesLock(Name FunName, Name LockName,
+                                     SourceLocation Loc) {}
+};
+
+/// \brief Check a function's CFG for thread-safety violations.
+///
+/// We traverse the blocks in the CFG, compute the set of mutexes that are held
+/// at the end of each block, and issue warnings for thread safety violations.
+/// Each block in the CFG is traversed exactly once.
+void runThreadSafetyAnalysis(AnalysisContext &AC, ThreadSafetyHandler &Handler);
+
+/// \brief Helper function that returns a LockKind required for the given level
+/// of access.
+LockKind getLockKindFromAccessKind(AccessKind AK);
+
+}} // end namespace clang::thread_safety
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/UninitializedValues.h b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/UninitializedValues.h
index badb493a9df4..e2e4f35043ee 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/UninitializedValues.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Analyses/UninitializedValues.h
@@ -27,10 +27,16 @@ class UninitVariablesHandler {
 public:
   UninitVariablesHandler() {}
   virtual ~UninitVariablesHandler();
-  
+
+  /// Called when the uninitialized variable is used at the given expression.
   virtual void handleUseOfUninitVariable(const Expr *ex,
                                          const VarDecl *vd,
                                          bool isAlwaysUninit) {}
+  
+  /// Called when the uninitialized variable analysis detects the
+  /// idiom 'int x = x'.  All other uses of 'x' within the initializer
+  /// are handled by handleUseOfUninitVariable.
+  virtual void handleSelfInit(const VarDecl *vd) {}
 };
 
 struct UninitVariablesAnalysisStats {
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h b/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h
index 6a1876e65900..3d0e88a51d8a 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h
@@ -19,6 +19,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/Analysis/CFG.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/DenseMap.h"
@@ -31,14 +32,35 @@ class Stmt;
 class CFGReverseBlockReachabilityAnalysis;
 class CFGStmtMap;
 class LiveVariables;
+class ManagedAnalysis;
 class ParentMap;
 class PseudoConstantAnalysis;
 class ImplicitParamDecl;
 class LocationContextManager;
 class StackFrameContext;
-
+  
 namespace idx { class TranslationUnit; }
 
+/// The base class of a hierarchy of objects representing analyses tied
+/// to AnalysisContext.
+class ManagedAnalysis {
+protected:
+  ManagedAnalysis() {}
+public:
+  virtual ~ManagedAnalysis();
+  
+  // Subclasses need to implement:
+  //
+  //  static const void *getTag();
+  //
+  // Which returns a fixed pointer address to distinguish classes of
+  // analysis objects.  They also need to implement:
+  //
+  //  static [Derived*] create(AnalysisContext &Ctx);
+  //
+  // which creates the analysis object given an AnalysisContext.
+};
+  
 /// AnalysisContext contains the context data for the function or method under
 /// analysis.
 class AnalysisContext {
@@ -54,7 +76,6 @@ class AnalysisContext {
   CFG::BuildOptions::ForcedBlkExprs *forcedBlkExprs;
   
   bool builtCFG, builtCompleteCFG;
-  const bool useUnoptimizedCFG;
 
   llvm::OwningPtr<LiveVariables> liveness;
   llvm::OwningPtr<LiveVariables> relaxedLiveness;
@@ -67,12 +88,13 @@ class AnalysisContext {
   // FIXME: remove.
   llvm::DenseMap<const BlockDecl*,void*> *ReferencedBlockVars;
 
+  void *ManagedAnalyses;
+
 public:
+  AnalysisContext(const Decl *d, idx::TranslationUnit *tu);
+
   AnalysisContext(const Decl *d, idx::TranslationUnit *tu,
-                  bool useUnoptimizedCFG = false,
-                  bool addehedges = false,
-                  bool addImplicitDtors = false,
-                  bool addInitializers = false);
+                  const CFG::BuildOptions &buildOptions);
 
   ~AnalysisContext();
 
@@ -81,13 +103,22 @@ class AnalysisContext {
 
   idx::TranslationUnit *getTranslationUnit() const { return TU; }
 
+  /// Return the build options used to construct the CFG.
+  CFG::BuildOptions &getCFGBuildOptions() {
+    return cfgBuildOptions;
+  }
+
+  const CFG::BuildOptions &getCFGBuildOptions() const {
+    return cfgBuildOptions;
+  }
+  
   /// getAddEHEdges - Return true iff we are adding exceptional edges from
   /// callExprs.  If this is false, then try/catch statements and blocks
   /// reachable from them can appear to be dead in the CFG, analysis passes must
   /// cope with that.
   bool getAddEHEdges() const { return cfgBuildOptions.AddEHEdges; }  
   bool getUseUnoptimizedCFG() const {
-      return cfgBuildOptions.PruneTriviallyFalseEdges;
+      return !cfgBuildOptions.PruneTriviallyFalseEdges;
   }
   bool getAddImplicitDtors() const { return cfgBuildOptions.AddImplicitDtors; }
   bool getAddInitializers() const { return cfgBuildOptions.AddInitializers; }
@@ -95,7 +126,7 @@ class AnalysisContext {
   void registerForcedBlockExpression(const Stmt *stmt);
   const CFGBlock *getBlockForRegisteredExpression(const Stmt *stmt);
   
-  Stmt *getBody();
+  Stmt *getBody() const;
   CFG *getCFG();
   
   CFGStmtMap *getCFGStmtMap();
@@ -114,8 +145,6 @@ class AnalysisContext {
 
   ParentMap &getParentMap();
   PseudoConstantAnalysis *getPseudoConstantAnalysis();
-  LiveVariables *getLiveVariables();
-  LiveVariables *getRelaxedLiveVariables();
 
   typedef const VarDecl * const * referenced_decls_iterator;
 
@@ -125,29 +154,44 @@ class AnalysisContext {
   /// Return the ImplicitParamDecl* associated with 'self' if this
   /// AnalysisContext wraps an ObjCMethodDecl.  Returns NULL otherwise.
   const ImplicitParamDecl *getSelfDecl() const;
+  
+  /// Return the specified analysis object, lazily running the analysis if
+  /// necessary.  Return NULL if the analysis could not run.
+  template <typename T>
+  T *getAnalysis() {
+    const void *tag = T::getTag();
+    ManagedAnalysis *&data = getAnalysisImpl(tag);
+    if (!data) {
+      data = T::create(*this);
+    }
+    return static_cast<T*>(data);
+  }
+private:
+  ManagedAnalysis *&getAnalysisImpl(const void* tag);
 };
 
 class AnalysisContextManager {
   typedef llvm::DenseMap<const Decl*, AnalysisContext*> ContextMap;
   ContextMap Contexts;
-  bool UseUnoptimizedCFG;
-  bool AddImplicitDtors;
-  bool AddInitializers;
+  CFG::BuildOptions cfgBuildOptions;
 public:
   AnalysisContextManager(bool useUnoptimizedCFG = false,
-      bool addImplicitDtors = false, bool addInitializers = false)
-    : UseUnoptimizedCFG(useUnoptimizedCFG), AddImplicitDtors(addImplicitDtors),
-      AddInitializers(addInitializers) {}
+                         bool addImplicitDtors = false,
+                         bool addInitializers = false);
   
   ~AnalysisContextManager();
 
   AnalysisContext *getContext(const Decl *D, idx::TranslationUnit *TU = 0);
 
-  bool getUseUnoptimizedCFG() const { return UseUnoptimizedCFG; }
-  bool getAddImplicitDtors() const { return AddImplicitDtors; }
-  bool getAddInitializers() const { return AddInitializers; }
+  bool getUseUnoptimizedCFG() const {
+    return !cfgBuildOptions.PruneTriviallyFalseEdges;
+  }
+  
+  CFG::BuildOptions &getCFGBuildOptions() {
+    return cfgBuildOptions;
+  }
 
-  // Discard all previously created AnalysisContexts.
+  /// Discard all previously created AnalysisContexts.
   void clear();
 };
 
@@ -187,8 +231,9 @@ class LocationContext : public llvm::FoldingSetNode {
 
   CFG *getCFG() const { return getAnalysisContext()->getCFG(); }
 
-  LiveVariables *getLiveVariables() const {
-    return getAnalysisContext()->getLiveVariables();
+  template <typename T>
+  T *getAnalysis() const {
+    return getAnalysisContext()->getAnalysis<T>();
   }
 
   ParentMap &getParentMap() const {
@@ -212,7 +257,7 @@ class LocationContext : public llvm::FoldingSetNode {
                             ContextKind ck,
                             AnalysisContext *ctx,
                             const LocationContext *parent,
-                            const void* data);
+                            const void *data);
 };
 
 class StackFrameContext : public LocationContext {
@@ -251,7 +296,7 @@ class StackFrameContext : public LocationContext {
     ID.AddInteger(idx);
   }
 
-  static bool classof(const LocationContext* Ctx) {
+  static bool classof(const LocationContext *Ctx) {
     return Ctx->getKind() == StackFrame;
   }
 };
@@ -274,7 +319,7 @@ class ScopeContext : public LocationContext {
     ProfileCommon(ID, Scope, ctx, parent, s);
   }
 
-  static bool classof(const LocationContext* Ctx) {
+  static bool classof(const LocationContext *Ctx) {
     return Ctx->getKind() == Scope;
   }
 };
@@ -302,7 +347,7 @@ class BlockInvocationContext : public LocationContext {
     ProfileCommon(ID, Block, ctx, parent, bd);
   }
 
-  static bool classof(const LocationContext* Ctx) {
+  static bool classof(const LocationContext *Ctx) {
     return Ctx->getKind() == Block;
   }
 };
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisDiagnostic.h
index dbf4e4c9aefe..16d31b476423 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define ANALYSISSTART
 #include "clang/Basic/DiagnosticAnalysisKinds.inc"
 #undef DIAG
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h b/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h
index ca46459afd9a..f191c802818b 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h
@@ -21,15 +21,13 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/BitVector.h"
+#include "clang/AST/Stmt.h"
 #include "clang/Analysis/Support/BumpVector.h"
 #include "clang/Basic/SourceLocation.h"
 #include <cassert>
 #include <iterator>
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
   class CXXDestructorDecl;
   class Decl;
@@ -98,7 +96,9 @@ class CFGStmt : public CFGElement {
 public:
   CFGStmt(Stmt *S) : CFGElement(Statement, S) {}
 
-  Stmt *getStmt() const { return static_cast<Stmt *>(Data1.getPointer()); }
+  const Stmt *getStmt() const { 
+    return static_cast<const Stmt *>(Data1.getPointer());
+  }
 
   static bool classof(const CFGElement *E) {
     return E->getKind() == Statement;
@@ -280,7 +280,7 @@ class CFGBlock {
   
     void push_back(CFGElement e, BumpVectorContext &C) { Impl.push_back(e, C); }
     reverse_iterator insert(reverse_iterator I, size_t Cnt, CFGElement E,
-        BumpVectorContext& C) {
+        BumpVectorContext &C) {
       return Impl.insert(I, Cnt, E, C);
     }
 
@@ -333,10 +333,21 @@ class CFGBlock {
   AdjacentBlocks Preds;
   AdjacentBlocks Succs;
 
+  /// NoReturn - This bit is set when the basic block contains a function call
+  /// or implicit destructor that is attributed as 'noreturn'. In that case,
+  /// control cannot technically ever proceed past this block. All such blocks
+  /// will have a single immediate successor: the exit block. This allows them
+  /// to be easily reached from the exit block and using this bit quickly
+  /// recognized without scanning the contents of the block.
+  ///
+  /// Optimization Note: This bit could be profitably folded with Terminator's
+  /// storage if the memory usage of CFGBlock becomes an issue.
+  unsigned HasNoReturnElement : 1;
+
 public:
   explicit CFGBlock(unsigned blockid, BumpVectorContext &C)
     : Elements(C), Label(NULL), Terminator(NULL), LoopTarget(NULL),
-      BlockID(blockid), Preds(C, 1), Succs(C, 1) {}
+      BlockID(blockid), Preds(C, 1), Succs(C, 1), HasNoReturnElement(false) {}
   ~CFGBlock() {}
 
   // Statement iterators
@@ -455,42 +466,45 @@ class CFGBlock {
 
   // Manipulation of block contents
 
-  void setTerminator(Stmt* Statement) { Terminator = Statement; }
-  void setLabel(Stmt* Statement) { Label = Statement; }
+  void setTerminator(Stmt *Statement) { Terminator = Statement; }
+  void setLabel(Stmt *Statement) { Label = Statement; }
   void setLoopTarget(const Stmt *loopTarget) { LoopTarget = loopTarget; }
+  void setHasNoReturnElement() { HasNoReturnElement = true; }
 
   CFGTerminator getTerminator() { return Terminator; }
   const CFGTerminator getTerminator() const { return Terminator; }
 
-  Stmt* getTerminatorCondition();
+  Stmt *getTerminatorCondition();
 
-  const Stmt* getTerminatorCondition() const {
+  const Stmt *getTerminatorCondition() const {
     return const_cast<CFGBlock*>(this)->getTerminatorCondition();
   }
 
   const Stmt *getLoopTarget() const { return LoopTarget; }
 
-  Stmt* getLabel() { return Label; }
-  const Stmt* getLabel() const { return Label; }
+  Stmt *getLabel() { return Label; }
+  const Stmt *getLabel() const { return Label; }
+
+  bool hasNoReturnElement() const { return HasNoReturnElement; }
 
   unsigned getBlockID() const { return BlockID; }
 
   void dump(const CFG *cfg, const LangOptions &LO) const;
-  void print(llvm::raw_ostream &OS, const CFG* cfg, const LangOptions &LO) const;
-  void printTerminator(llvm::raw_ostream &OS, const LangOptions &LO) const;
+  void print(raw_ostream &OS, const CFG* cfg, const LangOptions &LO) const;
+  void printTerminator(raw_ostream &OS, const LangOptions &LO) const;
   
-  void addSuccessor(CFGBlock* Block, BumpVectorContext &C) {
+  void addSuccessor(CFGBlock *Block, BumpVectorContext &C) {
     if (Block)
       Block->Preds.push_back(this, C);
     Succs.push_back(Block, C);
   }
   
-  void appendStmt(Stmt* statement, BumpVectorContext &C) {
+  void appendStmt(Stmt *statement, BumpVectorContext &C) {
     Elements.push_back(CFGStmt(statement), C);
   }
 
   void appendInitializer(CXXCtorInitializer *initializer,
-                        BumpVectorContext& C) {
+                        BumpVectorContext &C) {
     Elements.push_back(CFGInitializer(initializer), C);
   }
 
@@ -506,14 +520,18 @@ class CFGBlock {
     Elements.push_back(CFGTemporaryDtor(E), C);
   }
 
+  void appendAutomaticObjDtor(VarDecl *VD, Stmt *S, BumpVectorContext &C) {
+    Elements.push_back(CFGAutomaticObjDtor(VD, S), C);
+  }
+
   // Destructors must be inserted in reversed order. So insertion is in two
   // steps. First we prepare space for some number of elements, then we insert
   // the elements beginning at the last position in prepared space.
   iterator beginAutomaticObjDtorsInsert(iterator I, size_t Cnt,
-      BumpVectorContext& C) {
+      BumpVectorContext &C) {
     return iterator(Elements.insert(I.base(), Cnt, CFGElement(), C));
   }
-  iterator insertAutomaticObjDtor(iterator I, VarDecl* VD, Stmt* S) {
+  iterator insertAutomaticObjDtor(iterator I, VarDecl *VD, Stmt *S) {
     *I = CFGAutomaticObjDtor(VD, S);
     return ++I;
   }
@@ -533,30 +551,46 @@ class CFG {
   //===--------------------------------------------------------------------===//
 
   class BuildOptions {
+    llvm::BitVector alwaysAddMask;
   public:
     typedef llvm::DenseMap<const Stmt *, const CFGBlock*> ForcedBlkExprs;
     ForcedBlkExprs **forcedBlkExprs;    
 
-    bool PruneTriviallyFalseEdges:1;
-    bool AddEHEdges:1;
-    bool AddInitializers:1;
-    bool AddImplicitDtors:1;
+    bool PruneTriviallyFalseEdges;
+    bool AddEHEdges;
+    bool AddInitializers;
+    bool AddImplicitDtors;
+    
+    bool alwaysAdd(const Stmt *stmt) const {
+      return alwaysAddMask[stmt->getStmtClass()];
+    }
+    
+    BuildOptions &setAlwaysAdd(Stmt::StmtClass stmtClass, bool val = true) {
+      alwaysAddMask[stmtClass] = val;
+      return *this;
+    }
+    
+    BuildOptions &setAllAlwaysAdd() {
+      alwaysAddMask.set();
+      return *this;
+    }
 
     BuildOptions()
-        : forcedBlkExprs(0), PruneTriviallyFalseEdges(true)
-        , AddEHEdges(false)
-        , AddInitializers(false)
-        , AddImplicitDtors(false) {}
+    : alwaysAddMask(Stmt::lastStmtConstant, false)
+      ,forcedBlkExprs(0), PruneTriviallyFalseEdges(true)
+      ,AddEHEdges(false)
+      ,AddInitializers(false)
+      ,AddImplicitDtors(false) {}
   };
 
   /// buildCFG - Builds a CFG from an AST.  The responsibility to free the
   ///   constructed CFG belongs to the caller.
-  static CFG* buildCFG(const Decl *D, Stmt* AST, ASTContext *C,
+  static CFG* buildCFG(const Decl *D, Stmt *AST, ASTContext *C,
                        const BuildOptions &BO);
 
   /// createBlock - Create a new block in the CFG.  The CFG owns the block;
   ///  the caller should not directly free it.
-  CFGBlock* createBlock();
+  CFGBlock *createBlock();
 
   /// setEntry - Set the entry block of the CFG.  This is typically used
   ///  only during CFG construction.  Most CFG clients expect that the
@@ -565,7 +599,7 @@ class CFG {
 
   /// setIndirectGotoBlock - Set the block used for indirect goto jumps.
   ///  This is typically used only during CFG construction.
-  void setIndirectGotoBlock(CFGBlock* B) { IndirectGotoBlock = B; }
+  void setIndirectGotoBlock(CFGBlock *B) { IndirectGotoBlock = B; }
 
   //===--------------------------------------------------------------------===//
   // Block Iterators
@@ -577,8 +611,8 @@ class CFG {
   typedef std::reverse_iterator<iterator>          reverse_iterator;
   typedef std::reverse_iterator<const_iterator>    const_reverse_iterator;
 
-  CFGBlock&                 front()                { return *Blocks.front(); }
-  CFGBlock&                 back()                 { return *Blocks.back(); }
+  CFGBlock &                front()                { return *Blocks.front(); }
+  CFGBlock &                back()                 { return *Blocks.back(); }
 
   iterator                  begin()                { return Blocks.begin(); }
   iterator                  end()                  { return Blocks.end(); }
@@ -590,13 +624,25 @@ class CFG {
   const_reverse_iterator    rbegin()      const    { return Blocks.rbegin(); }
   const_reverse_iterator    rend()        const    { return Blocks.rend(); }
 
-  CFGBlock&                 getEntry()             { return *Entry; }
-  const CFGBlock&           getEntry()    const    { return *Entry; }
-  CFGBlock&                 getExit()              { return *Exit; }
-  const CFGBlock&           getExit()     const    { return *Exit; }
+  CFGBlock &                getEntry()             { return *Entry; }
+  const CFGBlock &          getEntry()    const    { return *Entry; }
+  CFGBlock &                getExit()              { return *Exit; }
+  const CFGBlock &          getExit()     const    { return *Exit; }
 
-  CFGBlock*        getIndirectGotoBlock() { return IndirectGotoBlock; }
-  const CFGBlock*  getIndirectGotoBlock() const { return IndirectGotoBlock; }
+  CFGBlock *       getIndirectGotoBlock() { return IndirectGotoBlock; }
+  const CFGBlock * getIndirectGotoBlock() const { return IndirectGotoBlock; }
+  
+  typedef std::vector<const CFGBlock*>::const_iterator try_block_iterator;
+  try_block_iterator try_blocks_begin() const {
+    return TryDispatchBlocks.begin();
+  }
+  try_block_iterator try_blocks_end() const {
+    return TryDispatchBlocks.end();
+  }
+  
+  void addTryDispatchBlock(const CFGBlock *block) {
+    TryDispatchBlocks.push_back(block);
+  }
 
   //===--------------------------------------------------------------------===//
   // Member templates useful for various batch operations over CFGs.
@@ -608,7 +654,7 @@ class CFG {
       for (CFGBlock::const_iterator BI=(*I)->begin(), BE=(*I)->end();
            BI != BE; ++BI) {
         if (const CFGStmt *stmt = BI->getAs<CFGStmt>())
-          O(stmt->getStmt());
+          O(const_cast<Stmt*>(stmt->getStmt()));
       }
   }
 
@@ -624,11 +670,11 @@ class CFG {
     operator unsigned() const { assert(Idx >=0); return (unsigned) Idx; }
   };
 
-  bool isBlkExpr(const Stmt* S) { return getBlkExprNum(S); }
+  bool isBlkExpr(const Stmt *S) { return getBlkExprNum(S); }
   bool isBlkExpr(const Stmt *S) const {
     return const_cast<CFG*>(this)->isBlkExpr(S);
   }
-  BlkExprNumTy  getBlkExprNum(const Stmt* S);
+  BlkExprNumTy  getBlkExprNum(const Stmt *S);
   unsigned      getNumBlkExprs();
 
   /// getNumBlockIDs - Returns the total number of BlockIDs allocated (which
@@ -640,7 +686,7 @@ class CFG {
   //===--------------------------------------------------------------------===//
 
   void viewCFG(const LangOptions &LO) const;
-  void print(llvm::raw_ostream& OS, const LangOptions &LO) const;
+  void print(raw_ostream &OS, const LangOptions &LO) const;
   void dump(const LangOptions &LO) const;
 
   //===--------------------------------------------------------------------===//
@@ -661,8 +707,8 @@ class CFG {
   }
 
 private:
-  CFGBlock* Entry;
-  CFGBlock* Exit;
+  CFGBlock *Entry;
+  CFGBlock *Exit;
   CFGBlock* IndirectGotoBlock;  // Special block to contain collective dispatch
                                 // for indirect gotos
   unsigned  NumBlockIDs;
@@ -670,11 +716,15 @@ class CFG {
   // BlkExprMap - An opaque pointer to prevent inclusion of DenseMap.h.
   //  It represents a map from Expr* to integers to record the set of
   //  block-level expressions and their "statement number" in the CFG.
-  void*     BlkExprMap;
+  void *    BlkExprMap;
   
   BumpVectorContext BlkBVC;
   
   CFGBlockListTy Blocks;
+  
+  /// C++ 'try' statements are modeled with an indirect dispatch block.
+  /// This is the collection of such blocks present in the CFG.
+  std::vector<const CFGBlock *> TryDispatchBlocks;
 
 };
 } // end namespace clang
@@ -703,11 +753,11 @@ template <> struct simplify_type< ::clang::CFGTerminator> {
 
 // Traits for: CFGBlock
 
-template <> struct GraphTraits< ::clang::CFGBlock* > {
+template <> struct GraphTraits< ::clang::CFGBlock *> {
   typedef ::clang::CFGBlock NodeType;
   typedef ::clang::CFGBlock::succ_iterator ChildIteratorType;
 
-  static NodeType* getEntryNode(::clang::CFGBlock* BB)
+  static NodeType* getEntryNode(::clang::CFGBlock *BB)
   { return BB; }
 
   static inline ChildIteratorType child_begin(NodeType* N)
@@ -717,11 +767,11 @@ template <> struct GraphTraits< ::clang::CFGBlock* > {
   { return N->succ_end(); }
 };
 
-template <> struct GraphTraits< const ::clang::CFGBlock* > {
+template <> struct GraphTraits< const ::clang::CFGBlock *> {
   typedef const ::clang::CFGBlock NodeType;
   typedef ::clang::CFGBlock::const_succ_iterator ChildIteratorType;
 
-  static NodeType* getEntryNode(const clang::CFGBlock* BB)
+  static NodeType* getEntryNode(const clang::CFGBlock *BB)
   { return BB; }
 
   static inline ChildIteratorType child_begin(NodeType* N)
@@ -748,7 +798,7 @@ template <> struct GraphTraits<Inverse<const ::clang::CFGBlock*> > {
 // Traits for: CFG
 
 template <> struct GraphTraits< ::clang::CFG* >
-    : public GraphTraits< ::clang::CFGBlock* >  {
+    : public GraphTraits< ::clang::CFGBlock *>  {
 
   typedef ::clang::CFG::iterator nodes_iterator;
 
@@ -758,7 +808,7 @@ template <> struct GraphTraits< ::clang::CFG* >
 };
 
 template <> struct GraphTraits<const ::clang::CFG* >
-    : public GraphTraits<const ::clang::CFGBlock* >  {
+    : public GraphTraits<const ::clang::CFGBlock *>  {
 
   typedef ::clang::CFG::const_iterator nodes_iterator;
 
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/DomainSpecific/CocoaConventions.h b/contrib/llvm/tools/clang/include/clang/Analysis/DomainSpecific/CocoaConventions.h
index 5a4e06ff53e5..fa8afccfcd89 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/DomainSpecific/CocoaConventions.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/DomainSpecific/CocoaConventions.h
@@ -14,12 +14,13 @@
 #ifndef LLVM_CLANG_ANALYSIS_DS_COCOA
 #define LLVM_CLANG_ANALYSIS_DS_COCOA
 
+#include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/StringRef.h"
-#include "clang/AST/Type.h"
 
 namespace clang {
-  
+class FunctionDecl;
 class ObjCMethodDecl;
+class QualType;
   
 namespace ento {
 namespace cocoa {
@@ -33,8 +34,8 @@ namespace cocoa {
     return deriveNamingConvention(S, MD) == CreateRule;
   }
   
-  bool isRefType(QualType RetTy, llvm::StringRef Prefix,
-                 llvm::StringRef Name = llvm::StringRef());
+  bool isRefType(QualType RetTy, StringRef Prefix,
+                 StringRef Name = StringRef());
     
   bool isCocoaObjectRef(QualType T);
 
@@ -43,7 +44,7 @@ namespace cocoa {
 namespace coreFoundation {
   bool isCFObjectRef(QualType T);
   
-  bool followsCreateRule(llvm::StringRef functionName);
+  bool followsCreateRule(const FunctionDecl *FD);
 }
 
 }} // end: "clang:ento"
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowSolver.h b/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowSolver.h
index 9561b964b5f8..017da636ebf6 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowSolver.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowSolver.h
@@ -30,11 +30,11 @@ namespace clang {
 
 class DataflowWorkListTy {
   llvm::DenseMap<const CFGBlock*, unsigned char> BlockSet;
-  llvm::SmallVector<const CFGBlock *, 10> BlockQueue;
+  SmallVector<const CFGBlock *, 10> BlockQueue;
 public:
   /// enqueue - Add a block to the worklist.  Blocks already on the
   ///  worklist are not added a second time.
-  void enqueue(const CFGBlock* B) {
+  void enqueue(const CFGBlock *B) {
     unsigned char &x = BlockSet[B];
     if (x == 1)
       return;
@@ -43,7 +43,7 @@ class DataflowWorkListTy {
   }
 
   /// dequeue - Remove a block from the worklist.
-  const CFGBlock* dequeue() {
+  const CFGBlock *dequeue() {
     assert(!BlockQueue.empty());
     const CFGBlock *B = BlockQueue.back();
     BlockQueue.pop_back();
@@ -69,20 +69,20 @@ template <> struct ItrTraits<forward_analysis_tag> {
   typedef CFGBlock::const_succ_iterator NextBItr;
   typedef CFGBlock::const_iterator      StmtItr;
 
-  static PrevBItr PrevBegin(const CFGBlock* B) { return B->pred_begin(); }
-  static PrevBItr PrevEnd(const CFGBlock* B) { return B->pred_end(); }
+  static PrevBItr PrevBegin(const CFGBlock *B) { return B->pred_begin(); }
+  static PrevBItr PrevEnd(const CFGBlock *B) { return B->pred_end(); }
 
-  static NextBItr NextBegin(const CFGBlock* B) { return B->succ_begin(); }
-  static NextBItr NextEnd(const CFGBlock* B) { return B->succ_end(); }
+  static NextBItr NextBegin(const CFGBlock *B) { return B->succ_begin(); }
+  static NextBItr NextEnd(const CFGBlock *B) { return B->succ_end(); }
 
-  static StmtItr StmtBegin(const CFGBlock* B) { return B->begin(); }
-  static StmtItr StmtEnd(const CFGBlock* B) { return B->end(); }
+  static StmtItr StmtBegin(const CFGBlock *B) { return B->begin(); }
+  static StmtItr StmtEnd(const CFGBlock *B) { return B->end(); }
 
-  static BlockEdge PrevEdge(const CFGBlock* B, const CFGBlock* Prev) {
+  static BlockEdge PrevEdge(const CFGBlock *B, const CFGBlock *Prev) {
     return BlockEdge(Prev, B, 0);
   }
 
-  static BlockEdge NextEdge(const CFGBlock* B, const CFGBlock* Next) {
+  static BlockEdge NextEdge(const CFGBlock *B, const CFGBlock *Next) {
     return BlockEdge(B, Next, 0);
   }
 };
@@ -92,20 +92,20 @@ template <> struct ItrTraits<backward_analysis_tag> {
   typedef CFGBlock::const_pred_iterator    NextBItr;
   typedef CFGBlock::const_reverse_iterator StmtItr;
 
-  static PrevBItr PrevBegin(const CFGBlock* B) { return B->succ_begin(); }
-  static PrevBItr PrevEnd(const CFGBlock* B) { return B->succ_end(); }
+  static PrevBItr PrevBegin(const CFGBlock *B) { return B->succ_begin(); }
+  static PrevBItr PrevEnd(const CFGBlock *B) { return B->succ_end(); }
 
-  static NextBItr NextBegin(const CFGBlock* B) { return B->pred_begin(); }
-  static NextBItr NextEnd(const CFGBlock* B) { return B->pred_end(); }
+  static NextBItr NextBegin(const CFGBlock *B) { return B->pred_begin(); }
+  static NextBItr NextEnd(const CFGBlock *B) { return B->pred_end(); }
 
-  static StmtItr StmtBegin(const CFGBlock* B) { return B->rbegin(); }
-  static StmtItr StmtEnd(const CFGBlock* B) { return B->rend(); }
+  static StmtItr StmtBegin(const CFGBlock *B) { return B->rbegin(); }
+  static StmtItr StmtEnd(const CFGBlock *B) { return B->rend(); }
 
-  static BlockEdge PrevEdge(const CFGBlock* B, const CFGBlock* Prev) {
+  static BlockEdge PrevEdge(const CFGBlock *B, const CFGBlock *Prev) {
     return BlockEdge(B, Prev, 0);
   }
 
-  static BlockEdge NextEdge(const CFGBlock* B, const CFGBlock* Next) {
+  static BlockEdge NextEdge(const CFGBlock *B, const CFGBlock *Next) {
     return BlockEdge(Next, B, 0);
   }
 };
@@ -162,7 +162,7 @@ class DataflowSolver {
   ///  dataflow values using runOnCFG, as runOnBlock is intended to
   ///  only be used for querying the dataflow values within a block
   ///  with and Observer object.
-  void runOnBlock(const CFGBlock* B, bool recordStmtValues) {
+  void runOnBlock(const CFGBlock *B, bool recordStmtValues) {
     BlockDataMapTy& M = D.getBlockDataMap();
     typename BlockDataMapTy::iterator I = M.find(B);
 
@@ -172,13 +172,13 @@ class DataflowSolver {
     }
   }
 
-  void runOnBlock(const CFGBlock& B, bool recordStmtValues) {
+  void runOnBlock(const CFGBlock &B, bool recordStmtValues) {
     runOnBlock(&B, recordStmtValues);
   }
-  void runOnBlock(CFG::iterator& I, bool recordStmtValues) {
+  void runOnBlock(CFG::iterator &I, bool recordStmtValues) {
     runOnBlock(*I, recordStmtValues);
   }
-  void runOnBlock(CFG::const_iterator& I, bool recordStmtValues) {
+  void runOnBlock(CFG::const_iterator &I, bool recordStmtValues) {
     runOnBlock(*I, recordStmtValues);
   }
 
@@ -199,7 +199,7 @@ class DataflowSolver {
     EnqueueBlocksOnWorklist(cfg, AnalysisDirTag());
 
     while (!WorkList.isEmpty()) {
-      const CFGBlock* B = WorkList.dequeue();
+      const CFGBlock *B = WorkList.dequeue();
       ProcessMerge(cfg, B);
       ProcessBlock(B, recordStmtValues, AnalysisDirTag());
       UpdateEdges(cfg, B, TF.getVal());
@@ -222,7 +222,7 @@ class DataflowSolver {
       WorkList.enqueue(&**I);
   }
 
-  void ProcessMerge(CFG& cfg, const CFGBlock* B) {
+  void ProcessMerge(CFG& cfg, const CFGBlock *B) {
     ValTy& V = TF.getVal();
     TF.SetTopValue(V);
 
@@ -270,7 +270,7 @@ class DataflowSolver {
   }
 
   /// ProcessBlock - Process the transfer functions for a given block.
-  void ProcessBlock(const CFGBlock* B, bool recordStmtValues,
+  void ProcessBlock(const CFGBlock *B, bool recordStmtValues,
                     dataflow::forward_analysis_tag) {
 
     TF.setCurrentBlock(B);
@@ -284,7 +284,7 @@ class DataflowSolver {
     TF.VisitTerminator(const_cast<CFGBlock*>(B));
   }
 
-  void ProcessBlock(const CFGBlock* B, bool recordStmtValues,
+  void ProcessBlock(const CFGBlock *B, bool recordStmtValues,
                     dataflow::backward_analysis_tag) {
 
     TF.setCurrentBlock(B);
@@ -298,12 +298,12 @@ class DataflowSolver {
     }
   }
 
-  void ProcessStmt(const Stmt* S, bool record, dataflow::forward_analysis_tag) {
+  void ProcessStmt(const Stmt *S, bool record, dataflow::forward_analysis_tag) {
     if (record) D.getStmtDataMap()[S] = TF.getVal();
     TF.BlockStmt_Visit(const_cast<Stmt*>(S));
   }
 
-  void ProcessStmt(const Stmt* S, bool record, dataflow::backward_analysis_tag){
+  void ProcessStmt(const Stmt *S, bool record, dataflow::backward_analysis_tag){
     TF.BlockStmt_Visit(const_cast<Stmt*>(S));
     if (record) D.getStmtDataMap()[S] = TF.getVal();
   }
@@ -312,14 +312,14 @@ class DataflowSolver {
   ///   block, update the dataflow value associated with the block's
   ///   outgoing/incoming edges (depending on whether we do a
   //    forward/backward analysis respectively)
-  void UpdateEdges(CFG& cfg, const CFGBlock* B, ValTy& V) {
+  void UpdateEdges(CFG& cfg, const CFGBlock *B, ValTy& V) {
     for (NextBItr I=ItrTraits::NextBegin(B), E=ItrTraits::NextEnd(B); I!=E; ++I)
       if (CFGBlock *NextBlk = *I)
         UpdateEdgeValue(ItrTraits::NextEdge(B, NextBlk),V, NextBlk);
   }
 
   /// UpdateEdgeValue - Update the value associated with a given edge.
-  void UpdateEdgeValue(BlockEdge E, ValTy& V, const CFGBlock* TargetBlock) {
+  void UpdateEdgeValue(BlockEdge E, ValTy& V, const CFGBlock *TargetBlock) {
     EdgeDataMapTy& M = D.getEdgeDataMap();
     typename EdgeDataMapTy::iterator I = M.find(E);
 
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowValues.h b/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowValues.h
index 7aa15c5b40ef..f86b2b0bfea1 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowValues.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/FlowSensitive/DataflowValues.h
@@ -84,13 +84,13 @@ class DataflowValues {
 
   /// getEdgeData - Retrieves the dataflow values associated with a
   ///  CFG edge.
-  ValTy& getEdgeData(const BlockEdge& E) {
+  ValTy& getEdgeData(const BlockEdge &E) {
     typename EdgeDataMapTy::iterator I = EdgeDataMap.find(E);
     assert (I != EdgeDataMap.end() && "No data associated with Edge.");
     return I->second;
   }
 
-  const ValTy& getEdgeData(const BlockEdge& E) const {
+  const ValTy& getEdgeData(const BlockEdge &E) const {
     return reinterpret_cast<DataflowValues*>(this)->getEdgeData(E);
   }
 
@@ -98,13 +98,13 @@ class DataflowValues {
   ///  specified CFGBlock.  If the dataflow analysis is a forward analysis,
   ///  this data is associated with the END of the block.  If the analysis
   ///  is a backwards analysis, it is associated with the ENTRY of the block.
-  ValTy& getBlockData(const CFGBlock* B) {
+  ValTy& getBlockData(const CFGBlock *B) {
     typename BlockDataMapTy::iterator I = BlockDataMap.find(B);
     assert (I != BlockDataMap.end() && "No data associated with block.");
     return I->second;
   }
 
-  const ValTy& getBlockData(const CFGBlock* B) const {
+  const ValTy& getBlockData(const CFGBlock *B) const {
     return const_cast<DataflowValues*>(this)->getBlockData(B);
   }
 
@@ -114,14 +114,14 @@ class DataflowValues {
   ///  If the analysis is a backwards analysis, it is associated with
   ///  the point after a Stmt.  This data is only computed for block-level
   ///  expressions, and only when requested when the analysis is executed.
-  ValTy& getStmtData(const Stmt* S) {
+  ValTy& getStmtData(const Stmt *S) {
     assert (StmtDataMap && "Dataflow values were not computed for statements.");
     typename StmtDataMapTy::iterator I = StmtDataMap->find(S);
     assert (I != StmtDataMap->end() && "No data associated with statement.");
     return I->second;
   }
 
-  const ValTy& getStmtData(const Stmt* S) const {
+  const ValTy& getStmtData(const Stmt *S) const {
     return const_cast<DataflowValues*>(this)->getStmtData(S);
   }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/ProgramPoint.h b/contrib/llvm/tools/clang/include/clang/Analysis/ProgramPoint.h
index 07b4dea987de..7ec4ecd41b2b 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/ProgramPoint.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/ProgramPoint.h
@@ -21,15 +21,18 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/ADT/StringRef.h"
 #include <cassert>
 #include <utility>
+#include <string>
 
 namespace clang {
 
-class LocationContext;
 class AnalysisContext;
 class FunctionDecl;
-
+class LocationContext;
+class ProgramPointTag;
+  
 class ProgramPoint {
 public:
   enum Kind { BlockEdgeKind,
@@ -42,7 +45,6 @@ class ProgramPoint {
               PreStoreKind,
               PostStoreKind,
               PostPurgeDeadSymbolsKind,
-              PostStmtCustomKind,
               PostConditionKind,
               PostLValueKind,
               PostInitializerKind,
@@ -58,25 +60,33 @@ class ProgramPoint {
   // The LocationContext could be NULL to allow ProgramPoint to be used in
   // context insensitive analysis.
   const LocationContext *L;
-  const void *Tag;
+  const ProgramPointTag *Tag;
 
+  ProgramPoint();
+  
 protected:
-  ProgramPoint(const void* P, Kind k, const LocationContext *l,
-               const void *tag = 0)
+  ProgramPoint(const void *P, Kind k, const LocationContext *l,
+               const ProgramPointTag *tag = 0)
     : Data(P, static_cast<const void*>(NULL)), K(k), L(l), Tag(tag) {}
 
-  ProgramPoint(const void* P1, const void* P2, Kind k, const LocationContext *l,
-               const void *tag = 0)
+  ProgramPoint(const void *P1, const void *P2, Kind k, const LocationContext *l,
+               const ProgramPointTag *tag = 0)
     : Data(P1, P2), K(k), L(l), Tag(tag) {}
 
 protected:
-  const void* getData1() const { return Data.first; }
-  const void* getData2() const { return Data.second; }
+  const void *getData1() const { return Data.first; }
+  const void *getData2() const { return Data.second; }
 
 public:
+  /// Create a new ProgramPoint object that is the same as the original
+  /// except for using the specified tag value.
+  ProgramPoint withTag(const ProgramPointTag *tag) const {
+    return ProgramPoint(Data.first, Data.second, K, L, tag);
+  }
+
   Kind getKind() const { return K; }
 
-  const void *getTag() const { return Tag; }
+  const ProgramPointTag *getTag() const { return Tag; }
 
   const LocationContext *getLocationContext() const { return L; }
 
@@ -93,7 +103,7 @@ class ProgramPoint {
     return K == RHS.K && Data == RHS.Data && L == RHS.L && Tag == RHS.Tag;
   }
 
-  bool operator!=(const ProgramPoint& RHS) const {
+  bool operator!=(const ProgramPoint &RHS) const {
     return K != RHS.K || Data != RHS.Data || L != RHS.L || Tag != RHS.Tag;
   }
 
@@ -104,29 +114,30 @@ class ProgramPoint {
     ID.AddPointer(L);
     ID.AddPointer(Tag);
   }
+
+  static ProgramPoint getProgramPoint(const Stmt *S, ProgramPoint::Kind K,
+                                      const LocationContext *LC,
+                                      const ProgramPointTag *tag);
+
 };
 
 class BlockEntrance : public ProgramPoint {
 public:
-  BlockEntrance(const CFGBlock* B, const LocationContext *L,
-                const void *tag = 0)
-    : ProgramPoint(B, BlockEntranceKind, L, tag) {}
+  BlockEntrance(const CFGBlock *B, const LocationContext *L,
+                const ProgramPointTag *tag = 0)
+    : ProgramPoint(B, BlockEntranceKind, L, tag) {    
+    assert(B && "BlockEntrance requires non-null block");
+  }
 
-  const CFGBlock* getBlock() const {
+  const CFGBlock *getBlock() const {
     return reinterpret_cast<const CFGBlock*>(getData1());
   }
 
   const CFGElement getFirstElement() const {
-    const CFGBlock* B = getBlock();
+    const CFGBlock *B = getBlock();
     return B->empty() ? CFGElement() : B->front();
   }
   
-  /// Create a new BlockEntrance object that is the same as the original
-  /// except for using the specified tag value.
-  BlockEntrance withTag(const void *tag) {
-    return BlockEntrance(getBlock(), getLocationContext(), tag);
-  }
-  
   static bool classof(const ProgramPoint* Location) {
     return Location->getKind() == BlockEntranceKind;
   }
@@ -134,14 +145,14 @@ class BlockEntrance : public ProgramPoint {
 
 class BlockExit : public ProgramPoint {
 public:
-  BlockExit(const CFGBlock* B, const LocationContext *L)
+  BlockExit(const CFGBlock *B, const LocationContext *L)
     : ProgramPoint(B, BlockExitKind, L) {}
 
-  const CFGBlock* getBlock() const {
+  const CFGBlock *getBlock() const {
     return reinterpret_cast<const CFGBlock*>(getData1());
   }
 
-  const Stmt* getTerminator() const {
+  const Stmt *getTerminator() const {
     return getBlock()->getTerminator();
   }
 
@@ -153,7 +164,7 @@ class BlockExit : public ProgramPoint {
 class StmtPoint : public ProgramPoint {
 public:
   StmtPoint(const Stmt *S, const void *p2, Kind k, const LocationContext *L,
-            const void *tag)
+            const ProgramPointTag *tag)
     : ProgramPoint(S, p2, k, L, tag) {}
 
   const Stmt *getStmt() const { return (const Stmt*) getData1(); }
@@ -170,7 +181,7 @@ class StmtPoint : public ProgramPoint {
 
 class PreStmt : public StmtPoint {
 public:
-  PreStmt(const Stmt *S, const LocationContext *L, const void *tag,
+  PreStmt(const Stmt *S, const LocationContext *L, const ProgramPointTag *tag,
           const Stmt *SubStmt = 0)
     : StmtPoint(S, SubStmt, PreStmtKind, L, tag) {}
 
@@ -183,16 +194,17 @@ class PreStmt : public StmtPoint {
 
 class PostStmt : public StmtPoint {
 protected:
-  PostStmt(const Stmt* S, const void* data, Kind k, const LocationContext *L,
-           const void *tag =0)
+  PostStmt(const Stmt *S, const void *data, Kind k, const LocationContext *L,
+           const ProgramPointTag *tag =0)
     : StmtPoint(S, data, k, L, tag) {}
 
 public:
-  explicit PostStmt(const Stmt* S, Kind k, 
-                    const LocationContext *L, const void *tag = 0)
+  explicit PostStmt(const Stmt *S, Kind k, 
+                    const LocationContext *L, const ProgramPointTag *tag = 0)
     : StmtPoint(S, NULL, k, L, tag) {}
 
-  explicit PostStmt(const Stmt* S, const LocationContext *L,const void *tag = 0)
+  explicit PostStmt(const Stmt *S, const LocationContext *L,
+                    const ProgramPointTag *tag = 0)
     : StmtPoint(S, NULL, PostStmtKind, L, tag) {}
 
   static bool classof(const ProgramPoint* Location) {
@@ -201,31 +213,11 @@ class PostStmt : public StmtPoint {
   }
 };
 
-class PostStmtCustom : public PostStmt {
-public:
-  PostStmtCustom(const Stmt* S,
-                 const std::pair<const void*, const void*>* TaggedData,\
-                 const LocationContext *L)
-    : PostStmt(S, TaggedData, PostStmtCustomKind, L) {}
-
-  const std::pair<const void*, const void*>& getTaggedPair() const {
-    return
-      *reinterpret_cast<const std::pair<const void*, const void*>*>(getData2());
-  }
-
-  const void* getTag() const { return getTaggedPair().first; }
-
-  const void* getTaggedData() const { return getTaggedPair().second; }
-
-  static bool classof(const ProgramPoint* Location) {
-    return Location->getKind() == PostStmtCustomKind;
-  }
-};
-
 // PostCondition represents the post program point of a branch condition.
 class PostCondition : public PostStmt {
 public:
-  PostCondition(const Stmt* S, const LocationContext *L, const void *tag = 0)
+  PostCondition(const Stmt *S, const LocationContext *L,
+                const ProgramPointTag *tag = 0)
     : PostStmt(S, PostConditionKind, L, tag) {}
 
   static bool classof(const ProgramPoint* Location) {
@@ -236,7 +228,7 @@ class PostCondition : public PostStmt {
 class LocationCheck : public StmtPoint {
 protected:
   LocationCheck(const Stmt *S, const LocationContext *L,
-                ProgramPoint::Kind K, const void *tag)
+                ProgramPoint::Kind K, const ProgramPointTag *tag)
     : StmtPoint(S, NULL, K, L, tag) {}
     
   static bool classof(const ProgramPoint *location) {
@@ -247,7 +239,8 @@ class LocationCheck : public StmtPoint {
   
 class PreLoad : public LocationCheck {
 public:
-  PreLoad(const Stmt *S, const LocationContext *L, const void *tag = 0)
+  PreLoad(const Stmt *S, const LocationContext *L,
+          const ProgramPointTag *tag = 0)
     : LocationCheck(S, L, PreLoadKind, tag) {}
   
   static bool classof(const ProgramPoint *location) {
@@ -257,7 +250,8 @@ class PreLoad : public LocationCheck {
 
 class PreStore : public LocationCheck {
 public:
-  PreStore(const Stmt *S, const LocationContext *L, const void *tag = 0)
+  PreStore(const Stmt *S, const LocationContext *L,
+           const ProgramPointTag *tag = 0)
   : LocationCheck(S, L, PreStoreKind, tag) {}
   
   static bool classof(const ProgramPoint *location) {
@@ -267,7 +261,8 @@ class PreStore : public LocationCheck {
 
 class PostLoad : public PostStmt {
 public:
-  PostLoad(const Stmt* S, const LocationContext *L, const void *tag = 0)
+  PostLoad(const Stmt *S, const LocationContext *L,
+           const ProgramPointTag *tag = 0)
     : PostStmt(S, PostLoadKind, L, tag) {}
 
   static bool classof(const ProgramPoint* Location) {
@@ -277,7 +272,8 @@ class PostLoad : public PostStmt {
 
 class PostStore : public PostStmt {
 public:
-  PostStore(const Stmt* S, const LocationContext *L, const void *tag = 0)
+  PostStore(const Stmt *S, const LocationContext *L,
+            const ProgramPointTag *tag = 0)
     : PostStmt(S, PostStoreKind, L, tag) {}
 
   static bool classof(const ProgramPoint* Location) {
@@ -287,7 +283,8 @@ class PostStore : public PostStmt {
 
 class PostLValue : public PostStmt {
 public:
-  PostLValue(const Stmt* S, const LocationContext *L, const void *tag = 0)
+  PostLValue(const Stmt *S, const LocationContext *L,
+             const ProgramPointTag *tag = 0)
     : PostStmt(S, PostLValueKind, L, tag) {}
 
   static bool classof(const ProgramPoint* Location) {
@@ -297,8 +294,8 @@ class PostLValue : public PostStmt {
 
 class PostPurgeDeadSymbols : public PostStmt {
 public:
-  PostPurgeDeadSymbols(const Stmt* S, const LocationContext *L,
-                       const void *tag = 0)
+  PostPurgeDeadSymbols(const Stmt *S, const LocationContext *L,
+                       const ProgramPointTag *tag = 0)
     : PostStmt(S, PostPurgeDeadSymbolsKind, L, tag) {}
 
   static bool classof(const ProgramPoint* Location) {
@@ -308,14 +305,17 @@ class PostPurgeDeadSymbols : public PostStmt {
 
 class BlockEdge : public ProgramPoint {
 public:
-  BlockEdge(const CFGBlock* B1, const CFGBlock* B2, const LocationContext *L)
-    : ProgramPoint(B1, B2, BlockEdgeKind, L) {}
+  BlockEdge(const CFGBlock *B1, const CFGBlock *B2, const LocationContext *L)
+    : ProgramPoint(B1, B2, BlockEdgeKind, L) {
+    assert(B1 && "BlockEdge: source block must be non-null");
+    assert(B2 && "BlockEdge: destination block must be non-null");    
+  }
 
-  const CFGBlock* getSrc() const {
+  const CFGBlock *getSrc() const {
     return static_cast<const CFGBlock*>(getData1());
   }
 
-  const CFGBlock* getDst() const {
+  const CFGBlock *getDst() const {
     return static_cast<const CFGBlock*>(getData2());
   }
 
@@ -365,6 +365,29 @@ class CallExit : public StmtPoint {
   }
 };
 
+/// ProgramPoints can be "tagged" as representing points specific to a given
+/// analysis entity.  Tags are abstract annotations, with an associated
+/// description and potentially other information.
+class ProgramPointTag {
+public:
+  ProgramPointTag(void *tagKind = 0) : TagKind(tagKind) {}
+  virtual ~ProgramPointTag();
+  virtual StringRef getTagDescription() const = 0;    
+
+protected:
+  /// Used to implement 'classof' in subclasses.
+  const void *getTagKind() { return TagKind; }
+  
+private:
+  const void *TagKind;
+};
+  
+class SimpleProgramPointTag : public ProgramPointTag {
+  std::string desc;
+public:
+  SimpleProgramPointTag(StringRef description);
+  StringRef getTagDescription() const;
+};
 
 } // end namespace clang
 
@@ -385,12 +408,12 @@ static inline clang::ProgramPoint getTombstoneKey() {
   return clang::BlockEntrance(reinterpret_cast<clang::CFGBlock*>(x), 0);
 }
 
-static unsigned getHashValue(const clang::ProgramPoint& Loc) {
+static unsigned getHashValue(const clang::ProgramPoint &Loc) {
   return Loc.getHashValue();
 }
 
-static bool isEqual(const clang::ProgramPoint& L,
-                    const clang::ProgramPoint& R) {
+static bool isEqual(const clang::ProgramPoint &L,
+                    const clang::ProgramPoint &R) {
   return L == R;
 }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Support/BlkExprDeclBitVector.h b/contrib/llvm/tools/clang/include/clang/Analysis/Support/BlkExprDeclBitVector.h
index 27ecc66e66e3..d25b84833c54 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Support/BlkExprDeclBitVector.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Support/BlkExprDeclBitVector.h
@@ -62,16 +62,16 @@ struct DeclBitVector_Types {
     AnalysisDataTy() : NDecls(0) {}
     virtual ~AnalysisDataTy() {}
 
-    bool isTracked(const NamedDecl* SD) { return DMap.find(SD) != DMap.end(); }
+    bool isTracked(const NamedDecl *SD) { return DMap.find(SD) != DMap.end(); }
 
-    Idx getIdx(const NamedDecl* SD) const {
+    Idx getIdx(const NamedDecl *SD) const {
       DMapTy::const_iterator I = DMap.find(SD);
       return I == DMap.end() ? Idx() : Idx(I->second);
     }
 
     unsigned getNumDecls() const { return NDecls; }
 
-    void Register(const NamedDecl* SD) {
+    void Register(const NamedDecl *SD) {
       if (!isTracked(SD)) DMap[SD] = NDecls++;
     }
 
@@ -117,11 +117,11 @@ struct DeclBitVector_Types {
     }
 
     llvm::BitVector::reference
-    operator()(const NamedDecl* ND, const AnalysisDataTy& AD) {
+    operator()(const NamedDecl *ND, const AnalysisDataTy& AD) {
       return getBit(AD.getIdx(ND));
     }
 
-    bool operator()(const NamedDecl* ND, const AnalysisDataTy& AD) const {
+    bool operator()(const NamedDecl *ND, const AnalysisDataTy& AD) const {
       return getBit(AD.getIdx(ND));
     }
 
@@ -171,14 +171,14 @@ struct StmtDeclBitVector_Types {
   //===--------------------------------------------------------------------===//
 
   class AnalysisDataTy : public DeclBitVector_Types::AnalysisDataTy {
-    ASTContext* ctx;
+    ASTContext *ctx;
     CFG* cfg;
   public:
     AnalysisDataTy() : ctx(0), cfg(0) {}
     virtual ~AnalysisDataTy() {}
 
-    void setContext(ASTContext& c) { ctx = &c; }
-    ASTContext& getContext() {
+    void setContext(ASTContext &c) { ctx = &c; }
+    ASTContext &getContext() {
       assert(ctx && "ASTContext should not be NULL.");
       return *ctx;
     }
@@ -186,10 +186,10 @@ struct StmtDeclBitVector_Types {
     void setCFG(CFG& c) { cfg = &c; }
     CFG& getCFG() { assert(cfg && "CFG should not be NULL."); return *cfg; }
 
-    bool isTracked(const Stmt* S) { return cfg->isBlkExpr(S); }
+    bool isTracked(const Stmt *S) { return cfg->isBlkExpr(S); }
     using DeclBitVector_Types::AnalysisDataTy::isTracked;
 
-    unsigned getIdx(const Stmt* S) const {
+    unsigned getIdx(const Stmt *S) const {
       CFG::BlkExprNumTy I = cfg->getBlkExprNum(S);
       assert(I && "Stmtession not tracked for bitvector.");
       return I;
@@ -248,11 +248,11 @@ struct StmtDeclBitVector_Types {
     }
 
     llvm::BitVector::reference
-    operator()(const Stmt* S, const AnalysisDataTy& AD) {
+    operator()(const Stmt *S, const AnalysisDataTy& AD) {
       return BlkExprBV[AD.getIdx(S)];
     }
     const llvm::BitVector::reference
-    operator()(const Stmt* S, const AnalysisDataTy& AD) const {
+    operator()(const Stmt *S, const AnalysisDataTy& AD) const {
       return const_cast<ValTy&>(*this)(S,AD);
     }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h b/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h
index 95f4ace76e8c..5c5ec2d7afd8 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h
@@ -28,8 +28,8 @@ static_cast<ImplClass*>(this)->Visit##CLASS##Decl(            \
                                static_cast<CLASS##Decl*>(D)); \
 break;
 
-#define DEFAULT_DISPATCH(CLASS) void Visit##CLASS##Decl(CLASS##Decl* D) {}
-#define DEFAULT_DISPATCH_VARDECL(CLASS) void Visit##CLASS##Decl(CLASS##Decl* D)\
+#define DEFAULT_DISPATCH(CLASS) void Visit##CLASS##Decl(CLASS##Decl *D) {}
+#define DEFAULT_DISPATCH_VARDECL(CLASS) void Visit##CLASS##Decl(CLASS##Decl *D)\
   { static_cast<ImplClass*>(this)->VisitVarDecl(D); }
 
 
@@ -38,23 +38,23 @@ template <typename ImplClass>
 class CFGRecStmtDeclVisitor : public CFGRecStmtVisitor<ImplClass> {
 public:
 
-  void VisitDeclRefExpr(DeclRefExpr* DR) {
+  void VisitDeclRefExpr(DeclRefExpr *DR) {
     static_cast<ImplClass*>(this)->VisitDecl(DR->getDecl());
   }
 
-  void VisitDeclStmt(DeclStmt* DS) {
+  void VisitDeclStmt(DeclStmt *DS) {
     for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end();
         DI != DE; ++DI) {
-      Decl* D = *DI;
+      Decl *D = *DI;
       static_cast<ImplClass*>(this)->VisitDecl(D);
       // Visit the initializer.
-      if (VarDecl* VD = dyn_cast<VarDecl>(D))
-        if (Expr* I = VD->getInit())
+      if (VarDecl *VD = dyn_cast<VarDecl>(D))
+        if (Expr *I = VD->getInit())
           static_cast<ImplClass*>(this)->Visit(I);
     }
   }
 
-  void VisitDecl(Decl* D) {
+  void VisitDecl(Decl *D) {
     switch (D->getKind()) {
         DISPATCH_CASE(Function)
         DISPATCH_CASE(CXXMethod)
@@ -69,7 +69,7 @@ class CFGRecStmtDeclVisitor : public CFGRecStmtVisitor<ImplClass> {
         DISPATCH_CASE(UsingDirective)
         DISPATCH_CASE(Using)
       default:
-        assert(false && "Subtype of ScopedDecl not handled.");
+        llvm_unreachable("Subtype of ScopedDecl not handled.");
     }
   }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtVisitor.h b/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtVisitor.h
index bb7cf0b97e53..4d1cabfc5c0b 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGRecStmtVisitor.h
@@ -22,7 +22,7 @@ template <typename ImplClass>
 class CFGRecStmtVisitor : public CFGStmtVisitor<ImplClass,void> {
 public:
 
-  void VisitStmt(Stmt* S) {
+  void VisitStmt(Stmt *S) {
     static_cast< ImplClass* >(this)->VisitChildren(S);
   }
   
@@ -45,13 +45,13 @@ break;
         CONDVAR_CASE(WhileStmt)
 #undef CONDVAR_CASE
       default:
-        assert(false && "Infeasible");
+        llvm_unreachable("Infeasible");
     }    
     static_cast<ImplClass*>(this)->Visit(CondVar->getInit());
   }
 
   // Defining operator() allows the visitor to be used as a C++ style functor.
-  void operator()(Stmt* S) { static_cast<ImplClass*>(this)->BlockStmt_Visit(S);}
+  void operator()(Stmt *S) { static_cast<ImplClass*>(this)->BlockStmt_Visit(S);}
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGStmtVisitor.h b/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGStmtVisitor.h
index 7fb4ab3ebad9..b354ba7b1680 100644
--- a/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGStmtVisitor.h
+++ b/contrib/llvm/tools/clang/include/clang/Analysis/Visitors/CFGStmtVisitor.h
@@ -33,7 +33,7 @@ static_cast<ImplClass*>(this)->BlockStmt_Visit ## CLASS(static_cast<CLASS*>(S));
 
 template <typename ImplClass, typename RetTy=void>
 class CFGStmtVisitor : public StmtVisitor<ImplClass,RetTy> {
-  Stmt* CurrentBlkStmt;
+  Stmt *CurrentBlkStmt;
 
   struct NullifyStmt {
     Stmt*& S;
@@ -45,9 +45,9 @@ class CFGStmtVisitor : public StmtVisitor<ImplClass,RetTy> {
 public:
   CFGStmtVisitor() : CurrentBlkStmt(NULL) {}
 
-  Stmt* getCurrentBlkStmt() const { return CurrentBlkStmt; }
+  Stmt *getCurrentBlkStmt() const { return CurrentBlkStmt; }
 
-  RetTy Visit(Stmt* S) {
+  RetTy Visit(Stmt *S) {
     if (S == CurrentBlkStmt ||
         !static_cast<ImplClass*>(this)->getCFG().isBlkExpr(S))
       return StmtVisitor<ImplClass,RetTy>::Visit(S);
@@ -67,7 +67,7 @@ class CFGStmtVisitor : public StmtVisitor<ImplClass,RetTy> {
   /// the list of statements in a CFGBlock.  For substatements, or when there
   /// is no implementation provided for a BlockStmt_XXX method, we default
   /// to using StmtVisitor's Visit method.
-  RetTy BlockStmt_Visit(Stmt* S) {
+  RetTy BlockStmt_Visit(Stmt *S) {
     CurrentBlkStmt = S;
     NullifyStmt cleanup(CurrentBlkStmt);
 
@@ -106,23 +106,23 @@ class CFGStmtVisitor : public StmtVisitor<ImplClass,RetTy> {
   DEFAULT_BLOCKSTMT_VISIT(ConditionalOperator)
   DEFAULT_BLOCKSTMT_VISIT(BinaryConditionalOperator)
 
-  RetTy BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
+  RetTy BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
     return static_cast<ImplClass*>(this)->BlockStmt_VisitStmt(S);
   }
 
-  RetTy BlockStmt_VisitCXXForRangeStmt(CXXForRangeStmt* S) {
+  RetTy BlockStmt_VisitCXXForRangeStmt(CXXForRangeStmt *S) {
     return static_cast<ImplClass*>(this)->BlockStmt_VisitStmt(S);
   }
 
-  RetTy BlockStmt_VisitImplicitControlFlowExpr(Expr* E) {
+  RetTy BlockStmt_VisitImplicitControlFlowExpr(Expr *E) {
     return static_cast<ImplClass*>(this)->BlockStmt_VisitExpr(E);
   }
 
-  RetTy BlockStmt_VisitExpr(Expr* E) {
+  RetTy BlockStmt_VisitExpr(Expr *E) {
     return static_cast<ImplClass*>(this)->BlockStmt_VisitStmt(E);
   }
 
-  RetTy BlockStmt_VisitStmt(Stmt* S) {
+  RetTy BlockStmt_VisitStmt(Stmt *S) {
     return static_cast<ImplClass*>(this)->Visit(S);
   }
 
@@ -141,14 +141,14 @@ class CFGStmtVisitor : public StmtVisitor<ImplClass,RetTy> {
   //===--------------------------------------------------------------------===//
 
   /// VisitChildren: Call "Visit" on each child of S.
-  void VisitChildren(Stmt* S) {
+  void VisitChildren(Stmt *S) {
 
     switch (S->getStmtClass()) {
       default:
         break;
 
       case Stmt::StmtExprClass: {
-        CompoundStmt* CS = cast<StmtExpr>(S)->getSubStmt();
+        CompoundStmt *CS = cast<StmtExpr>(S)->getSubStmt();
         if (CS->body_empty()) return;
         static_cast<ImplClass*>(this)->Visit(CS->body_back());
         return;
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Attr.td b/contrib/llvm/tools/clang/include/clang/Basic/Attr.td
index e64dc6a2ade0..2a4ba5c6c73c 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Attr.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Attr.td
@@ -54,7 +54,9 @@ class ExprArgument<string name> : Argument<name>;
 class FunctionArgument<string name> : Argument<name>;
 class TypeArgument<string name> : Argument<name>;
 class UnsignedArgument<string name> : Argument<name>;
+class SourceLocArgument<string name> : Argument<name>;
 class VariadicUnsignedArgument<string name> : Argument<name>;
+class VariadicExprArgument<string name> : Argument<name>;
 
 // A version of the form major.minor[.subminor].
 class VersionArgument<string name> : Argument<name>;
@@ -89,7 +91,9 @@ class Attr {
   // The attribute will not be permitted in C++0x attribute-specifiers if
   // this is empty; the empty string can be used as a namespace.
   list<string> Namespaces = [];
-  // Any additional text that should be included verbatim in the class.
+  // Set to true for attributes with arguments which require delayed parsing. 
+  bit LateParsed = 0;  
+  // Any additional text that should be included verbatim in the class.  
   code AdditionalMembers = [{}];
 }
 
@@ -110,7 +114,7 @@ def Alias : InheritableAttr {
 }
 
 def Aligned : InheritableAttr {
-  let Spellings = ["align", "aligned"];
+  let Spellings = ["aligned"];
   let Subjects = [NonBitField, NormalVar, Tag];
   let Args = [AlignedArgument<"Alignment">];
   let Namespaces = ["", "std"];
@@ -128,7 +132,7 @@ def AnalyzerNoReturn : InheritableAttr {
   let Spellings = ["analyzer_noreturn"];
 }
 
-def Annotate : InheritableAttr {
+def Annotate : InheritableParamAttr {
   let Spellings = ["annotate"];
   let Args = [StringArgument<"Annotation">];
 }
@@ -167,6 +171,23 @@ def CDecl : InheritableAttr {
   let Spellings = ["cdecl", "__cdecl"];
 }
 
+// cf_audited_transfer indicates that the given function has been
+// audited and has been marked with the appropriate cf_consumed and
+// cf_returns_retained attributes.  It is generally applied by
+// '#pragma clang arc_cf_code_audited' rather than explicitly.
+def CFAuditedTransfer : InheritableAttr {
+  let Spellings = ["cf_audited_transfer"];
+  let Subjects = [Function];
+}
+
+// cf_unknown_transfer is an explicit opt-out of cf_audited_transfer.
+// It indicates that the function has unknown or unautomatable
+// transfer semantics.
+def CFUnknownTransfer : InheritableAttr {
+  let Spellings = ["cf_unknown_transfer"];
+  let Subjects = [Function];
+}
+
 def CFReturnsRetained : InheritableAttr {
   let Spellings = ["cf_returns_retained"];
   let Subjects = [ObjCMethod, Function];
@@ -284,7 +305,7 @@ def IBOutlet : InheritableAttr {
 
 def IBOutletCollection : InheritableAttr {
   let Spellings = ["iboutletcollection"];
-  let Args = [TypeArgument<"InterFace">];
+  let Args = [TypeArgument<"Interface">, SourceLocArgument<"InterfaceLoc">];
 }
 
 def Malloc : InheritableAttr {
@@ -317,6 +338,10 @@ def Naked : InheritableAttr {
   let Spellings = ["naked"];
 }
 
+def ReturnsTwice : InheritableAttr {
+  let Spellings = ["returns_twice"];
+}
+
 def NoCommon : InheritableAttr {
   let Spellings = ["nocommon"];
 }
@@ -358,6 +383,12 @@ def NoThrow : InheritableAttr {
   let Spellings = ["nothrow"];
 }
 
+def NSBridged : InheritableAttr {
+  let Spellings = ["ns_bridged"];
+  let Subjects = [Record];
+  let Args = [IdentifierArgument<"BridgedType">];
+}
+
 def NSReturnsRetained : InheritableAttr {
   let Spellings = ["ns_returns_retained"];
   let Subjects = [ObjCMethod, Function];
@@ -405,6 +436,11 @@ def ObjCPreciseLifetime : Attr {
   let Subjects = [Var];
 }
 
+def ObjCReturnsInnerPointer : Attr {
+  let Spellings = ["objc_returns_inner_pointer"];
+  let Subjects = [ObjCMethod];
+}
+
 def Overloadable : Attr {
   let Spellings = ["overloadable"];
 }
@@ -533,3 +569,108 @@ def WeakRef : InheritableAttr {
 def X86ForceAlignArgPointer : InheritableAttr {
   let Spellings = [];
 }
+
+
+// C/C++ Thread safety attributes (e.g. for deadlock, data race checking)
+
+def GuardedVar : InheritableAttr {
+  let Spellings = ["guarded_var"];
+}
+
+def PtGuardedVar : InheritableAttr {
+  let Spellings = ["pt_guarded_var"];
+}
+
+def Lockable : InheritableAttr {
+  let Spellings = ["lockable"];
+}
+
+def ScopedLockable : InheritableAttr {
+  let Spellings = ["scoped_lockable"];
+}
+
+def NoThreadSafetyAnalysis : InheritableAttr {
+  let Spellings = ["no_thread_safety_analysis"];
+}
+
+def GuardedBy : InheritableAttr {
+  let Spellings = ["guarded_by"];
+  let Args = [ExprArgument<"Arg">];
+  let LateParsed = 1;
+}
+
+def PtGuardedBy : InheritableAttr {
+  let Spellings = ["pt_guarded_by"];
+  let Args = [ExprArgument<"Arg">];
+  let LateParsed = 1;
+}
+
+def AcquiredAfter : InheritableAttr {
+  let Spellings = ["acquired_after"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def AcquiredBefore : InheritableAttr {
+  let Spellings = ["acquired_before"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def ExclusiveLockFunction : InheritableAttr {
+  let Spellings = ["exclusive_lock_function"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def SharedLockFunction : InheritableAttr {
+  let Spellings = ["shared_lock_function"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+// The first argument is an integer or boolean value specifying the return value
+// of a successful lock acquisition.
+def ExclusiveTrylockFunction : InheritableAttr {
+  let Spellings = ["exclusive_trylock_function"];
+  let Args = [ExprArgument<"SuccessValue">, VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+// The first argument is an integer or boolean value specifying the return value
+// of a successful lock acquisition.
+def SharedTrylockFunction : InheritableAttr {
+  let Spellings = ["shared_trylock_function"];
+  let Args = [ExprArgument<"SuccessValue">, VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def UnlockFunction : InheritableAttr {
+  let Spellings = ["unlock_function"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def LockReturned : InheritableAttr {
+  let Spellings = ["lock_returned"];
+  let Args = [ExprArgument<"Arg">];
+  let LateParsed = 1;
+}
+
+def LocksExcluded : InheritableAttr {
+  let Spellings = ["locks_excluded"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def ExclusiveLocksRequired : InheritableAttr {
+  let Spellings = ["exclusive_locks_required"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
+
+def SharedLocksRequired : InheritableAttr {
+  let Spellings = ["shared_locks_required"];
+  let Args = [VariadicExprArgument<"Args">];
+  let LateParsed = 1;
+}
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def b/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def
index a3cc6156238e..e06e05b1aa17 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def
@@ -35,6 +35,7 @@
 //  A -> "reference" to __builtin_va_list
 //  V -> Vector, following num elements and a base type.
 //  X -> _Complex, followed by the base type.
+//  Y -> ptrdiff_t
 //  P -> FILE
 //  J -> jmp_buf
 //  SJ -> sigjmp_buf
@@ -76,6 +77,7 @@
 //          in that it accepts its arguments as a va_list rather than
 //          through an ellipsis
 //  e -> const, but only when -fmath-errno=0
+//  j -> returns_twice (like setjmp)
 //  FIXME: gcc has nonnull
 
 #if defined(BUILTIN) && !defined(LIBBUILTIN)
@@ -96,9 +98,9 @@ BUILTIN(__builtin_fabsl, "LdLd", "ncF")
 BUILTIN(__builtin_fmod , "ddd"  , "Fnc")
 BUILTIN(__builtin_fmodf, "fff"  , "Fnc")
 BUILTIN(__builtin_fmodl, "LdLdLd", "Fnc")
-BUILTIN(__builtin_frexp , "ddi*"  , "Fnc")
-BUILTIN(__builtin_frexpf, "ffi*"  , "Fnc")
-BUILTIN(__builtin_frexpl, "LdLdi*", "Fnc")
+BUILTIN(__builtin_frexp , "ddi*"  , "Fn")
+BUILTIN(__builtin_frexpf, "ffi*"  , "Fn")
+BUILTIN(__builtin_frexpl, "LdLdi*", "Fn")
 BUILTIN(__builtin_huge_val, "d", "nc")
 BUILTIN(__builtin_huge_valf, "f", "nc")
 BUILTIN(__builtin_huge_vall, "Ld", "nc")
@@ -108,9 +110,9 @@ BUILTIN(__builtin_infl , "Ld"  , "nc")
 BUILTIN(__builtin_ldexp , "ddi"  , "Fnc")
 BUILTIN(__builtin_ldexpf, "ffi"  , "Fnc")
 BUILTIN(__builtin_ldexpl, "LdLdi", "Fnc")
-BUILTIN(__builtin_modf , "ddd*"  , "Fnc")
-BUILTIN(__builtin_modff, "fff*"  , "Fnc")
-BUILTIN(__builtin_modfl, "LdLdLd*", "Fnc")
+BUILTIN(__builtin_modf , "ddd*"  , "Fn")
+BUILTIN(__builtin_modff, "fff*"  , "Fn")
+BUILTIN(__builtin_modfl, "LdLdLd*", "Fn")
 BUILTIN(__builtin_nan,  "dcC*" , "ncF")
 BUILTIN(__builtin_nanf, "fcC*" , "ncF")
 BUILTIN(__builtin_nanl, "LdcC*", "ncF")
@@ -233,9 +235,9 @@ BUILTIN(__builtin_nexttowardl, "LdLdLd", "Fnc")
 BUILTIN(__builtin_remainder , "ddd", "Fnc")
 BUILTIN(__builtin_remainderf, "fff", "Fnc")
 BUILTIN(__builtin_remainderl, "LdLdLd", "Fnc")
-BUILTIN(__builtin_remquo , "dddi*", "Fnc")
-BUILTIN(__builtin_remquof, "fffi*", "Fnc")
-BUILTIN(__builtin_remquol, "LdLdLdi*", "Fnc")
+BUILTIN(__builtin_remquo , "dddi*", "Fn")
+BUILTIN(__builtin_remquof, "fffi*", "Fn")
+BUILTIN(__builtin_remquol, "LdLdLdi*", "Fn")
 BUILTIN(__builtin_rint , "dd", "Fnc")
 BUILTIN(__builtin_rintf, "ff", "Fnc")
 BUILTIN(__builtin_rintl, "LdLd", "Fnc")
@@ -388,7 +390,7 @@ BUILTIN(__builtin_constant_p, "i.", "nct")
 BUILTIN(__builtin_classify_type, "i.", "nct")
 BUILTIN(__builtin___CFStringMakeConstantString, "FC*cC*", "nc")
 BUILTIN(__builtin___NSStringMakeConstantString, "FC*cC*", "nc")
-BUILTIN(__builtin_va_start, "vA.", "n")
+BUILTIN(__builtin_va_start, "vA.", "nt")
 BUILTIN(__builtin_va_end, "vA", "n")
 BUILTIN(__builtin_va_copy, "vAA", "n")
 BUILTIN(__builtin_stdarg_start, "vA.", "n")
@@ -426,7 +428,7 @@ BUILTIN(__builtin_return_address, "v*IUi", "n")
 BUILTIN(__builtin_extract_return_addr, "v*v*", "n")
 BUILTIN(__builtin_frame_address, "v*IUi", "n")
 BUILTIN(__builtin_flt_rounds, "i", "nc")
-BUILTIN(__builtin_setjmp, "iv**", "")
+BUILTIN(__builtin_setjmp, "iv**", "j")
 BUILTIN(__builtin_longjmp, "vv**i", "r")
 BUILTIN(__builtin_unwind_init, "v", "")
 BUILTIN(__builtin_eh_return_data_regno, "iIi", "nc")
@@ -584,12 +586,21 @@ BUILTIN(__sync_swap_4, "iiD*i.", "n")
 BUILTIN(__sync_swap_8, "LLiLLiD*LLi.", "n")
 BUILTIN(__sync_swap_16, "LLLiLLLiD*LLLi.", "n")
 
-
+BUILTIN(__atomic_load, "v.", "t")
+BUILTIN(__atomic_store, "v.", "t")
+BUILTIN(__atomic_exchange, "v.", "t")
+BUILTIN(__atomic_compare_exchange_strong, "v.", "t")
+BUILTIN(__atomic_compare_exchange_weak, "v.", "t")
+BUILTIN(__atomic_fetch_add, "v.", "t")
+BUILTIN(__atomic_fetch_sub, "v.", "t")
+BUILTIN(__atomic_fetch_and, "v.", "t")
+BUILTIN(__atomic_fetch_or, "v.", "t")
+BUILTIN(__atomic_fetch_xor, "v.", "t")
+BUILTIN(__atomic_thread_fence, "vi", "n")
+BUILTIN(__atomic_signal_fence, "vi", "n")
 
 // Non-overloaded atomic builtins.
 BUILTIN(__sync_synchronize, "v.", "n")
-// LLVM instruction builtin [Clang extension].
-BUILTIN(__builtin_llvm_memory_barrier,"vbbbbb", "n")
 // GCC does not support these, they are a Clang extension.
 BUILTIN(__sync_fetch_and_min, "iiD*i", "n")
 BUILTIN(__sync_fetch_and_max, "iiD*i", "n")
@@ -661,9 +672,26 @@ LIBBUILTIN(rindex, "c*cC*i",      "f",     "strings.h", ALL_LANGUAGES)
 LIBBUILTIN(bzero, "vv*z",         "f",     "strings.h", ALL_LANGUAGES)
 // POSIX unistd.h
 LIBBUILTIN(_exit, "vi",           "fr",    "unistd.h", ALL_LANGUAGES)
+LIBBUILTIN(vfork, "i",            "fj",    "unistd.h", ALL_LANGUAGES)
 // POSIX setjmp.h
+
+// In some systems setjmp is a macro that expands to _setjmp. We undefine
+// it here to avoid having two identical LIBBUILTIN entries.
+#undef setjmp
+LIBBUILTIN(_setjmp, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(__sigsetjmp, "iJi",    "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(setjmp, "iJ",          "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(sigsetjmp, "iJi",      "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(setjmp_syscall, "iJ",  "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(savectx, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(qsetjmp, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(getcontext, "iJ",      "fj",   "setjmp.h", ALL_LANGUAGES)
+
 LIBBUILTIN(_longjmp, "vJi",       "fr",    "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(siglongjmp, "vSJi",    "fr",    "setjmp.h", ALL_LANGUAGES)
+// non-standard but very common
+LIBBUILTIN(strlcpy, "zc*cC*z",    "f",     "string.h", ALL_LANGUAGES)
+LIBBUILTIN(strlcat, "zc*cC*z",    "f",     "string.h", ALL_LANGUAGES)
 //   id objc_msgSend(id, SEL, ...)
 LIBBUILTIN(objc_msgSend, "GGH.",   "f",     "objc/message.h", OBJC_LANG)
 
@@ -687,8 +715,7 @@ LIBBUILTIN(objc_read_weak, "GG*", "f", "/objc/objc-auto.h", OBJC_LANG)
 // id objc_assign_weak(id value, id *location)
 LIBBUILTIN(objc_assign_weak, "GGG*", "f", "/objc/objc-auto.h", OBJC_LANG)
 // id objc_assign_ivar(id value, id dest, ptrdiff_t offset)
-// FIXME. Darwin has ptrdiff_t typedef'ed to int.
-LIBBUILTIN(objc_assign_ivar, "GGGi", "f", "/objc/objc-auto.h", OBJC_LANG)
+LIBBUILTIN(objc_assign_ivar, "GGGY", "f", "/objc/objc-auto.h", OBJC_LANG)
 // id objc_assign_global(id val, id *dest)
 LIBBUILTIN(objc_assign_global, "GGG*", "f", "/objc/objc-auto.h", OBJC_LANG)
 // id objc_assign_strongCast(id val, id *dest
@@ -738,5 +765,8 @@ LIBBUILTIN(_Block_object_assign, "vv*vC*iC", "f", "Blocks.h", ALL_LANGUAGES)
 LIBBUILTIN(_Block_object_dispose, "vvC*iC", "f", "Blocks.h", ALL_LANGUAGES)
 // FIXME: Also declare NSConcreteGlobalBlock and NSConcreteStackBlock.
 
+// Annotation function
+BUILTIN(__builtin_annotation, "UiUicC*", "nc")
+
 #undef BUILTIN
 #undef LIBBUILTIN
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Builtins.h b/contrib/llvm/tools/clang/include/clang/Basic/Builtins.h
index 7469e144c150..5afa02010040 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Builtins.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Builtins.h
@@ -15,16 +15,13 @@
 #ifndef LLVM_CLANG_BASIC_BUILTINS_H
 #define LLVM_CLANG_BASIC_BUILTINS_H
 
+#include "clang/Basic/LLVM.h"
 #include <cstring>
 
 // VC++ defines 'alloca' as an object-like macro, which interferes with our
 // builtins.
 #undef alloca
 
-namespace llvm {
-  template <typename T> class SmallVectorImpl;
-}
-
 namespace clang {
   class TargetInfo;
   class IdentifierTable;
@@ -65,15 +62,18 @@ class Context {
   const Info *TSRecords;
   unsigned NumTSRecords;
 public:
-  Context(const TargetInfo &Target);
+  Context();
 
+  /// \brief Perform target-specific initialization
+  void InitializeTarget(const TargetInfo &Target);
+  
   /// InitializeBuiltins - Mark the identifiers for all the builtins with their
   /// appropriate builtin ID # and mark any non-portable builtin identifiers as
   /// such.
   void InitializeBuiltins(IdentifierTable &Table, const LangOptions& LangOpts);
 
   /// \brief Popular the vector with the names of all of the builtins.
-  void GetBuiltinNames(llvm::SmallVectorImpl<const char *> &Names,
+  void GetBuiltinNames(SmallVectorImpl<const char *> &Names,
                        bool NoBuiltins);
 
   /// Builtin::GetName - Return the identifier name for the specified builtin,
@@ -103,6 +103,11 @@ class Context {
     return strchr(GetRecord(ID).Attributes, 'r') != 0;
   }
 
+  /// isReturnsTwice - Return true if we know this builtin can return twice.
+  bool isReturnsTwice(unsigned ID) const {
+    return strchr(GetRecord(ID).Attributes, 'j') != 0;
+  }
+
   /// isLibFunction - Return true if this is a builtin for a libc/libm function,
   /// with a "__builtin_" prefix (e.g. __builtin_abs).
   bool isLibFunction(unsigned ID) const {
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DeclNodes.td b/contrib/llvm/tools/clang/include/clang/Basic/DeclNodes.td
index ddd08278c37b..a37dc1039891 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DeclNodes.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DeclNodes.td
@@ -74,3 +74,4 @@ def Friend : Decl;
 def FriendTemplate : Decl;
 def StaticAssert : Decl;
 def Block : Decl, DeclContext;
+def ClassScopeFunctionSpecialization : Decl;
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DelayedCleanupPool.h b/contrib/llvm/tools/clang/include/clang/Basic/DelayedCleanupPool.h
index 843205f7b011..8575bc21113f 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DelayedCleanupPool.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DelayedCleanupPool.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CLANG_BASIC_DELAYEDCLEANUPPOOL_H
 #define LLVM_CLANG_BASIC_DELAYEDCLEANUPPOOL_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -66,7 +67,7 @@ class DelayedCleanupPool {
   }
 
   void doCleanup() {
-    for (llvm::SmallVector<std::pair<void *, CleanupFn>, 8>::reverse_iterator
+    for (SmallVector<std::pair<void *, CleanupFn>, 8>::reverse_iterator
            I = Cleanups.rbegin(), E = Cleanups.rend(); I != E; ++I)
       I->second(I->first);
     Cleanups.clear();
@@ -79,7 +80,7 @@ class DelayedCleanupPool {
 
 private:
   llvm::DenseMap<void *, CleanupFn> Ptrs;
-  llvm::SmallVector<std::pair<void *, CleanupFn>, 8> Cleanups;
+  SmallVector<std::pair<void *, CleanupFn>, 8> Cleanups;
 
   template <typename T>
   static void cleanupWithDelete(void *ptr) {
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.h b/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.h
index 6f72976bfcf0..fefc44ce7bc6 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.h
@@ -16,6 +16,7 @@
 
 #include "clang/Basic/DiagnosticIDs.h"
 #include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/OwningPtr.h"
@@ -25,7 +26,7 @@
 #include <list>
 
 namespace clang {
-  class DiagnosticClient;
+  class DiagnosticConsumer;
   class DiagnosticBuilder;
   class IdentifierInfo;
   class DeclContext;
@@ -64,7 +65,7 @@ class FixItHint {
   /// \brief Create a code modification hint that inserts the given
   /// code string at a specific location.
   static FixItHint CreateInsertion(SourceLocation InsertionLoc,
-                                   llvm::StringRef Code) {
+                                   StringRef Code) {
     FixItHint Hint;
     Hint.RemoveRange =
       CharSourceRange(SourceRange(InsertionLoc, InsertionLoc), false);
@@ -86,7 +87,7 @@ class FixItHint {
   /// \brief Create a code modification hint that replaces the given
   /// source range with the given code string.
   static FixItHint CreateReplacement(CharSourceRange RemoveRange,
-                                     llvm::StringRef Code) {
+                                     StringRef Code) {
     FixItHint Hint;
     Hint.RemoveRange = RemoveRange;
     Hint.CodeToInsert = Code;
@@ -94,17 +95,17 @@ class FixItHint {
   }
   
   static FixItHint CreateReplacement(SourceRange RemoveRange,
-                                     llvm::StringRef Code) {
+                                     StringRef Code) {
     return CreateReplacement(CharSourceRange::getTokenRange(RemoveRange), Code);
   }
 };
 
-/// Diagnostic - This concrete class is used by the front-end to report
+/// DiagnosticsEngine - This concrete class is used by the front-end to report
 /// problems and issues.  It massages the diagnostics (e.g. handling things like
-/// "report warnings as errors" and passes them off to the DiagnosticClient for
-/// reporting to the user. Diagnostic is tied to one translation unit and
-/// one SourceManager.
-class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
+/// "report warnings as errors" and passes them off to the DiagnosticConsumer
+/// for reporting to the user. DiagnosticsEngine is tied to one translation unit
+/// and one SourceManager.
+class DiagnosticsEngine : public llvm::RefCountedBase<DiagnosticsEngine> {
 public:
   /// Level - The level of the diagnostic, after it has been through mapping.
   enum Level {
@@ -148,7 +149,8 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
 private:
   unsigned char AllExtensionsSilenced; // Used by __extension__
   bool IgnoreAllWarnings;        // Ignore all warnings: -w
-  bool WarningsAsErrors;         // Treat warnings like errors:
+  bool WarningsAsErrors;         // Treat warnings like errors.
+  bool EnableAllWarnings;        // Enable all warnings.
   bool ErrorsAsFatal;            // Treat errors like fatal errors.
   bool SuppressSystemWarnings;   // Suppress warnings in system headers.
   bool SuppressAllDiagnostics;   // Suppress all diagnostics.
@@ -158,7 +160,7 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
                                    // 0 -> no limit.
   ExtensionHandling ExtBehavior; // Map extensions onto warnings or errors?
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> Diags;
-  DiagnosticClient *Client;
+  DiagnosticConsumer *Client;
   bool OwnsDiagClient;
   SourceManager *SourceMgr;
 
@@ -173,22 +175,22 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   /// the state so that we know what is the diagnostic state at any given
   /// source location.
   class DiagState {
-    llvm::DenseMap<unsigned, unsigned> DiagMap;
+    llvm::DenseMap<unsigned, DiagnosticMappingInfo> DiagMap;
 
   public:
-    typedef llvm::DenseMap<unsigned, unsigned>::const_iterator iterator;
+    typedef llvm::DenseMap<unsigned, DiagnosticMappingInfo>::iterator
+      iterator;
+    typedef llvm::DenseMap<unsigned, DiagnosticMappingInfo>::const_iterator
+      const_iterator;
 
-    void setMapping(diag::kind Diag, unsigned Map) { DiagMap[Diag] = Map; }
-
-    diag::Mapping getMapping(diag::kind Diag) const {
-      iterator I = DiagMap.find(Diag);
-      if (I != DiagMap.end())
-        return (diag::Mapping)I->second;
-      return diag::Mapping();
+    void setMappingInfo(diag::kind Diag, DiagnosticMappingInfo Info) {
+      DiagMap[Diag] = Info;
     }
 
-    iterator begin() const { return DiagMap.begin(); }
-    iterator end() const { return DiagMap.end(); }
+    DiagnosticMappingInfo &getOrAddMappingInfo(diag::kind Diag);
+
+    const_iterator begin() const { return DiagMap.begin(); }
+    const_iterator end() const { return DiagMap.end(); }
   };
 
   /// \brief Keeps and automatically disposes all DiagStates that we create.
@@ -254,10 +256,10 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   /// \brief Indicates that an unrecoverable error has occurred.
   bool UnrecoverableErrorOccurred;
   
-  /// \brief Toggles for DiagnosticErrorTrap to check whether an error occurred
+  /// \brief Counts for DiagnosticErrorTrap to check whether an error occurred
   /// during a parsing section, e.g. during parsing a function.
-  bool TrapErrorOccurred;
-  bool TrapUnrecoverableErrorOccurred;
+  unsigned TrapNumErrorsOccurred;
+  unsigned TrapNumUnrecoverableErrorsOccurred;
 
   /// LastDiagLevel - This is the level of the last diagnostic emitted.  This is
   /// used to emit continuation diagnostics with the same level as the
@@ -283,9 +285,9 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
       const char *Argument, unsigned ArgumentLen,
       const ArgumentValue *PrevArgs,
       unsigned NumPrevArgs,
-      llvm::SmallVectorImpl<char> &Output,
+      SmallVectorImpl<char> &Output,
       void *Cookie,
-      llvm::SmallVectorImpl<intptr_t> &QualTypeVals);
+      SmallVectorImpl<intptr_t> &QualTypeVals);
   void *ArgToStringCookie;
   ArgToStringFnTy ArgToStringFn;
 
@@ -301,21 +303,25 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   std::string DelayedDiagArg2;
 
 public:
-  explicit Diagnostic(const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &Diags,
-                      DiagnosticClient *client = 0,
+  explicit DiagnosticsEngine(
+                      const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &Diags,
+                      DiagnosticConsumer *client = 0,
                       bool ShouldOwnClient = true);
-  ~Diagnostic();
+  ~DiagnosticsEngine();
 
   const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &getDiagnosticIDs() const {
     return Diags;
   }
 
-  DiagnosticClient *getClient() { return Client; }
-  const DiagnosticClient *getClient() const { return Client; }
+  DiagnosticConsumer *getClient() { return Client; }
+  const DiagnosticConsumer *getClient() const { return Client; }
 
+  /// \brief Determine whether this \c DiagnosticsEngine object own its client.
+  bool ownsClient() const { return OwnsDiagClient; }
+  
   /// \brief Return the current diagnostic client along with ownership of that
   /// client.
-  DiagnosticClient *takeClient() {
+  DiagnosticConsumer *takeClient() {
     OwnsDiagClient = false;
     return Client;
   }
@@ -328,8 +334,8 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   void setSourceManager(SourceManager *SrcMgr) { SourceMgr = SrcMgr; }
 
   //===--------------------------------------------------------------------===//
-  //  Diagnostic characterization methods, used by a client to customize how
-  //  diagnostics are emitted.
+  //  DiagnosticsEngine characterization methods, used by a client to customize
+  //  how diagnostics are emitted.
   //
 
   /// pushMappings - Copies the current DiagMappings and pushes the new copy
@@ -346,7 +352,7 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   ///
   /// \param ShouldOwnClient true if the diagnostic object should take
   /// ownership of \c client.
-  void setClient(DiagnosticClient *client, bool ShouldOwnClient = true);
+  void setClient(DiagnosticConsumer *client, bool ShouldOwnClient = true);
 
   /// setErrorLimit - Specify a limit for the number of errors we should
   /// emit before giving up.  Zero disables the limit.
@@ -369,6 +375,12 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   void setIgnoreAllWarnings(bool Val) { IgnoreAllWarnings = Val; }
   bool getIgnoreAllWarnings() const { return IgnoreAllWarnings; }
 
+  /// setEnableAllWarnings - When set to true, any unmapped ignored warnings
+  /// are no longer ignored.  If this and IgnoreAllWarnings are both set,
+  /// then that one wins.
+  void setEnableAllWarnings(bool Val) { EnableAllWarnings = Val; }
+  bool getEnableAllWarnngs() const { return EnableAllWarnings; }
+  
   /// setWarningsAsErrors - When set to true, any warnings reported are issued
   /// as errors.
   void setWarningsAsErrors(bool Val) { WarningsAsErrors = Val; }
@@ -435,10 +447,20 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   ///
   /// 'Loc' is the source location that this change of diagnostic state should
   /// take affect. It can be null if we are setting the state from command-line.
-  bool setDiagnosticGroupMapping(llvm::StringRef Group, diag::Mapping Map,
-                                 SourceLocation Loc = SourceLocation()) {
-    return Diags->setDiagnosticGroupMapping(Group, Map, Loc, *this);
-  }
+  bool setDiagnosticGroupMapping(StringRef Group, diag::Mapping Map,
+                                 SourceLocation Loc = SourceLocation());
+
+  /// \brief Set the warning-as-error flag for the given diagnostic group. This
+  /// function always only operates on the current diagnostic state.
+  ///
+  /// \returns True if the given group is unknown, false otherwise.
+  bool setDiagnosticGroupWarningAsError(StringRef Group, bool Enabled);
+
+  /// \brief Set the error-as-fatal flag for the given diagnostic group. This
+  /// function always only operates on the current diagnostic state.
+  ///
+  /// \returns True if the given group is unknown, false otherwise.
+  bool setDiagnosticGroupErrorAsFatal(StringRef Group, bool Enabled);
 
   bool hasErrorOccurred() const { return ErrorOccurred; }
   bool hasFatalErrorOccurred() const { return FatalErrorOccurred; }
@@ -457,7 +479,7 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   /// getCustomDiagID - Return an ID for a diagnostic with the specified message
   /// and level.  If this is the first request for this diagnosic, it is
   /// registered and created, otherwise the existing ID is returned.
-  unsigned getCustomDiagID(Level L, llvm::StringRef Message) {
+  unsigned getCustomDiagID(Level L, StringRef Message) {
     return Diags->getCustomDiagID((DiagnosticIDs::Level)L, Message);
   }
 
@@ -467,8 +489,8 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
                           const char *Modifier, unsigned ModLen,
                           const char *Argument, unsigned ArgLen,
                           const ArgumentValue *PrevArgs, unsigned NumPrevArgs,
-                          llvm::SmallVectorImpl<char> &Output,
-                          llvm::SmallVectorImpl<intptr_t> &QualTypeVals) const {
+                          SmallVectorImpl<char> &Output,
+                          SmallVectorImpl<intptr_t> &QualTypeVals) const {
     ArgToStringFn(Kind, Val, Modifier, ModLen, Argument, ArgLen,
                   PrevArgs, NumPrevArgs, Output, ArgToStringCookie,
                   QualTypeVals);
@@ -484,18 +506,17 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   void Reset();
   
   //===--------------------------------------------------------------------===//
-  // Diagnostic classification and reporting interfaces.
+  // DiagnosticsEngine classification and reporting interfaces.
   //
 
-  /// \brief Based on the way the client configured the Diagnostic
+  /// \brief Based on the way the client configured the DiagnosticsEngine
   /// object, classify the specified diagnostic ID into a Level, consumable by
-  /// the DiagnosticClient.
+  /// the DiagnosticConsumer.
   ///
   /// \param Loc The source location we are interested in finding out the
   /// diagnostic state. Can be null in order to query the latest state.
-  Level getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
-                           diag::Mapping *mapping = 0) const {
-    return (Level)Diags->getDiagnosticLevel(DiagID, Loc, *this, mapping);
+  Level getDiagnosticLevel(unsigned DiagID, SourceLocation Loc) const {
+    return (Level)Diags->getDiagnosticLevel(DiagID, Loc, *this);
   }
 
   /// Report - Issue the message to the client.  @c DiagID is a member of the
@@ -527,13 +548,13 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   ///
   /// \param Arg1 A string argument that will be provided to the
   /// diagnostic. A copy of this string will be stored in the
-  /// Diagnostic object itself.
+  /// DiagnosticsEngine object itself.
   ///
   /// \param Arg2 A string argument that will be provided to the
   /// diagnostic. A copy of this string will be stored in the
-  /// Diagnostic object itself.
-  void SetDelayedDiagnostic(unsigned DiagID, llvm::StringRef Arg1 = "",
-                            llvm::StringRef Arg2 = "");
+  /// DiagnosticsEngine object itself.
+  void SetDelayedDiagnostic(unsigned DiagID, StringRef Arg1 = "",
+                            StringRef Arg2 = "");
   
   /// \brief Clear out the current diagnostic.
   void Clear() { CurDiagID = ~0U; }
@@ -542,23 +563,6 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   /// \brief Report the delayed diagnostic.
   void ReportDelayed();
 
-
-  /// getDiagnosticMappingInfo - Return the mapping info currently set for the
-  /// specified builtin diagnostic.  This returns the high bit encoding, or zero
-  /// if the field is completely uninitialized.
-  diag::Mapping getDiagnosticMappingInfo(diag::kind Diag,
-                                         DiagState *State) const {
-    return State->getMapping(Diag);
-  }
-
-  void setDiagnosticMappingInternal(unsigned DiagId, unsigned Map,
-                                    DiagState *State,
-                                    bool isUser, bool isPragma) const {
-    if (isUser) Map |= 8;  // Set the high bit for user mappings.
-    if (isPragma) Map |= 0x10;  // Set the bit for diagnostic pragma mappings.
-    State->setMapping((diag::kind)DiagId, Map);
-  }
-
   // This is private state used by DiagnosticBuilder.  We put it here instead of
   // in DiagnosticBuilder in order to keep DiagnosticBuilder a small lightweight
   // object.  This implementation choice means that we can only have one
@@ -567,7 +571,7 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   // diagnostic is in flight at a time.
   friend class DiagnosticIDs;
   friend class DiagnosticBuilder;
-  friend class DiagnosticInfo;
+  friend class Diagnostic;
   friend class PartialDiagnostic;
   friend class DiagnosticErrorTrap;
   
@@ -614,7 +618,7 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
   /// only support 10 ranges, could easily be extended if needed.
   CharSourceRange DiagRanges[10];
 
-  enum { MaxFixItHints = 3 };
+  enum { MaxFixItHints = 6 };
 
   /// FixItHints - If valid, provides a hint with some code
   /// to insert, remove, or modify at a particular position.
@@ -637,28 +641,30 @@ class Diagnostic : public llvm::RefCountedBase<Diagnostic> {
 /// between the time the instance was created and the time it was
 /// queried.
 class DiagnosticErrorTrap {
-  Diagnostic &Diag;
+  DiagnosticsEngine &Diag;
+  unsigned NumErrors;
+  unsigned NumUnrecoverableErrors;
 
 public:
-  explicit DiagnosticErrorTrap(Diagnostic &Diag)
+  explicit DiagnosticErrorTrap(DiagnosticsEngine &Diag)
     : Diag(Diag) { reset(); }
 
   /// \brief Determine whether any errors have occurred since this
   /// object instance was created.
   bool hasErrorOccurred() const {
-    return Diag.TrapErrorOccurred;
+    return Diag.TrapNumErrorsOccurred > NumErrors;
   }
 
   /// \brief Determine whether any unrecoverable errors have occurred since this
   /// object instance was created.
   bool hasUnrecoverableErrorOccurred() const {
-    return Diag.TrapUnrecoverableErrorOccurred;
+    return Diag.TrapNumUnrecoverableErrorsOccurred > NumUnrecoverableErrors;
   }
 
   // Set to initial state of "no errors occurred".
   void reset() {
-    Diag.TrapErrorOccurred = false;
-    Diag.TrapUnrecoverableErrorOccurred = false;
+    NumErrors = Diag.TrapNumErrorsOccurred;
+    NumUnrecoverableErrors = Diag.TrapNumUnrecoverableErrorsOccurred;
   }
 };
 
@@ -667,9 +673,9 @@ class DiagnosticErrorTrap {
 //===----------------------------------------------------------------------===//
 
 /// DiagnosticBuilder - This is a little helper class used to produce
-/// diagnostics.  This is constructed by the Diagnostic::Report method, and
-/// allows insertion of extra information (arguments and source ranges) into the
-/// currently "in flight" diagnostic.  When the temporary for the builder is
+/// diagnostics.  This is constructed by the DiagnosticsEngine::Report method,
+/// and allows insertion of extra information (arguments and source ranges) into
+/// the currently "in flight" diagnostic.  When the temporary for the builder is
 /// destroyed, the diagnostic is issued.
 ///
 /// Note that many of these will be created as temporary objects (many call
@@ -678,12 +684,12 @@ class DiagnosticErrorTrap {
 /// the common fields to registers, eliminating increments of the NumArgs field,
 /// for example.
 class DiagnosticBuilder {
-  mutable Diagnostic *DiagObj;
+  mutable DiagnosticsEngine *DiagObj;
   mutable unsigned NumArgs, NumRanges, NumFixItHints;
 
   void operator=(const DiagnosticBuilder&); // DO NOT IMPLEMENT
-  friend class Diagnostic;
-  explicit DiagnosticBuilder(Diagnostic *diagObj)
+  friend class DiagnosticsEngine;
+  explicit DiagnosticBuilder(DiagnosticsEngine *diagObj)
     : DiagObj(diagObj), NumArgs(0), NumRanges(0), NumFixItHints(0) {}
 
   friend class PartialDiagnostic;
@@ -731,7 +737,7 @@ class DiagnosticBuilder {
   ///
   /// \pre \c isActive()
   unsigned getDiagID() const {
-    assert(isActive() && "Diagnostic is inactive");
+    assert(isActive() && "DiagnosticsEngine is inactive");
     return DiagObj->CurDiagID;
   }
   
@@ -743,17 +749,17 @@ class DiagnosticBuilder {
   /// return Diag(...);
   operator bool() const { return true; }
 
-  void AddString(llvm::StringRef S) const {
-    assert(NumArgs < Diagnostic::MaxArguments &&
+  void AddString(StringRef S) const {
+    assert(NumArgs < DiagnosticsEngine::MaxArguments &&
            "Too many arguments to diagnostic!");
     if (DiagObj) {
-      DiagObj->DiagArgumentsKind[NumArgs] = Diagnostic::ak_std_string;
+      DiagObj->DiagArgumentsKind[NumArgs] = DiagnosticsEngine::ak_std_string;
       DiagObj->DiagArgumentsStr[NumArgs++] = S;
     }
   }
 
-  void AddTaggedVal(intptr_t V, Diagnostic::ArgumentKind Kind) const {
-    assert(NumArgs < Diagnostic::MaxArguments &&
+  void AddTaggedVal(intptr_t V, DiagnosticsEngine::ArgumentKind Kind) const {
+    assert(NumArgs < DiagnosticsEngine::MaxArguments &&
            "Too many arguments to diagnostic!");
     if (DiagObj) {
       DiagObj->DiagArgumentsKind[NumArgs] = Kind;
@@ -770,15 +776,17 @@ class DiagnosticBuilder {
   }
 
   void AddFixItHint(const FixItHint &Hint) const {
-    assert(NumFixItHints < Diagnostic::MaxFixItHints &&
+    assert(NumFixItHints < DiagnosticsEngine::MaxFixItHints &&
            "Too many fix-it hints!");
+    if (NumFixItHints >= DiagnosticsEngine::MaxFixItHints)
+      return;  // Don't crash in release builds
     if (DiagObj)
       DiagObj->FixItHints[NumFixItHints++] = Hint;
   }
 };
 
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
-                                           llvm::StringRef S) {
+                                           StringRef S) {
   DB.AddString(S);
   return DB;
 }
@@ -786,30 +794,30 @@ inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            const char *Str) {
   DB.AddTaggedVal(reinterpret_cast<intptr_t>(Str),
-                  Diagnostic::ak_c_string);
+                  DiagnosticsEngine::ak_c_string);
   return DB;
 }
 
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB, int I) {
-  DB.AddTaggedVal(I, Diagnostic::ak_sint);
+  DB.AddTaggedVal(I, DiagnosticsEngine::ak_sint);
   return DB;
 }
 
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,bool I) {
-  DB.AddTaggedVal(I, Diagnostic::ak_sint);
+  DB.AddTaggedVal(I, DiagnosticsEngine::ak_sint);
   return DB;
 }
 
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            unsigned I) {
-  DB.AddTaggedVal(I, Diagnostic::ak_uint);
+  DB.AddTaggedVal(I, DiagnosticsEngine::ak_uint);
   return DB;
 }
 
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            const IdentifierInfo *II) {
   DB.AddTaggedVal(reinterpret_cast<intptr_t>(II),
-                  Diagnostic::ak_identifierinfo);
+                  DiagnosticsEngine::ak_identifierinfo);
   return DB;
 }
 
@@ -823,7 +831,7 @@ typename llvm::enable_if<llvm::is_same<T, DeclContext>,
                          const DiagnosticBuilder &>::type
 operator<<(const DiagnosticBuilder &DB, T *DC) {
   DB.AddTaggedVal(reinterpret_cast<intptr_t>(DC),
-                  Diagnostic::ak_declcontext);
+                  DiagnosticsEngine::ak_declcontext);
   return DB;
 }
   
@@ -848,33 +856,33 @@ inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
 /// Report - Issue the message to the client.  DiagID is a member of the
 /// diag::kind enum.  This actually returns a new instance of DiagnosticBuilder
 /// which emits the diagnostics (through ProcessDiag) when it is destroyed.
-inline DiagnosticBuilder Diagnostic::Report(SourceLocation Loc,
+inline DiagnosticBuilder DiagnosticsEngine::Report(SourceLocation Loc,
                                             unsigned DiagID){
   assert(CurDiagID == ~0U && "Multiple diagnostics in flight at once!");
   CurDiagLoc = Loc;
   CurDiagID = DiagID;
   return DiagnosticBuilder(this);
 }
-inline DiagnosticBuilder Diagnostic::Report(unsigned DiagID) {
+inline DiagnosticBuilder DiagnosticsEngine::Report(unsigned DiagID) {
   return Report(SourceLocation(), DiagID);
 }
 
 //===----------------------------------------------------------------------===//
-// DiagnosticInfo
+// Diagnostic
 //===----------------------------------------------------------------------===//
 
-/// DiagnosticInfo - This is a little helper class (which is basically a smart
-/// pointer that forward info from Diagnostic) that allows clients to enquire
-/// about the currently in-flight diagnostic.
-class DiagnosticInfo {
-  const Diagnostic *DiagObj;
-  llvm::StringRef StoredDiagMessage;
+/// Diagnostic - This is a little helper class (which is basically a smart
+/// pointer that forward info from DiagnosticsEngine) that allows clients to
+/// enquire about the currently in-flight diagnostic.
+class Diagnostic {
+  const DiagnosticsEngine *DiagObj;
+  StringRef StoredDiagMessage;
 public:
-  explicit DiagnosticInfo(const Diagnostic *DO) : DiagObj(DO) {}
-  DiagnosticInfo(const Diagnostic *DO, llvm::StringRef storedDiagMessage)
+  explicit Diagnostic(const DiagnosticsEngine *DO) : DiagObj(DO) {}
+  Diagnostic(const DiagnosticsEngine *DO, StringRef storedDiagMessage)
     : DiagObj(DO), StoredDiagMessage(storedDiagMessage) {}
 
-  const Diagnostic *getDiags() const { return DiagObj; }
+  const DiagnosticsEngine *getDiags() const { return DiagObj; }
   unsigned getID() const { return DiagObj->CurDiagID; }
   const SourceLocation &getLocation() const { return DiagObj->CurDiagLoc; }
   bool hasSourceManager() const { return DiagObj->hasSourceManager(); }
@@ -884,49 +892,49 @@ class DiagnosticInfo {
 
   /// getArgKind - Return the kind of the specified index.  Based on the kind
   /// of argument, the accessors below can be used to get the value.
-  Diagnostic::ArgumentKind getArgKind(unsigned Idx) const {
+  DiagnosticsEngine::ArgumentKind getArgKind(unsigned Idx) const {
     assert(Idx < getNumArgs() && "Argument index out of range!");
-    return (Diagnostic::ArgumentKind)DiagObj->DiagArgumentsKind[Idx];
+    return (DiagnosticsEngine::ArgumentKind)DiagObj->DiagArgumentsKind[Idx];
   }
 
   /// getArgStdStr - Return the provided argument string specified by Idx.
   const std::string &getArgStdStr(unsigned Idx) const {
-    assert(getArgKind(Idx) == Diagnostic::ak_std_string &&
+    assert(getArgKind(Idx) == DiagnosticsEngine::ak_std_string &&
            "invalid argument accessor!");
     return DiagObj->DiagArgumentsStr[Idx];
   }
 
   /// getArgCStr - Return the specified C string argument.
   const char *getArgCStr(unsigned Idx) const {
-    assert(getArgKind(Idx) == Diagnostic::ak_c_string &&
+    assert(getArgKind(Idx) == DiagnosticsEngine::ak_c_string &&
            "invalid argument accessor!");
     return reinterpret_cast<const char*>(DiagObj->DiagArgumentsVal[Idx]);
   }
 
   /// getArgSInt - Return the specified signed integer argument.
   int getArgSInt(unsigned Idx) const {
-    assert(getArgKind(Idx) == Diagnostic::ak_sint &&
+    assert(getArgKind(Idx) == DiagnosticsEngine::ak_sint &&
            "invalid argument accessor!");
     return (int)DiagObj->DiagArgumentsVal[Idx];
   }
 
   /// getArgUInt - Return the specified unsigned integer argument.
   unsigned getArgUInt(unsigned Idx) const {
-    assert(getArgKind(Idx) == Diagnostic::ak_uint &&
+    assert(getArgKind(Idx) == DiagnosticsEngine::ak_uint &&
            "invalid argument accessor!");
     return (unsigned)DiagObj->DiagArgumentsVal[Idx];
   }
 
   /// getArgIdentifier - Return the specified IdentifierInfo argument.
   const IdentifierInfo *getArgIdentifier(unsigned Idx) const {
-    assert(getArgKind(Idx) == Diagnostic::ak_identifierinfo &&
+    assert(getArgKind(Idx) == DiagnosticsEngine::ak_identifierinfo &&
            "invalid argument accessor!");
     return reinterpret_cast<IdentifierInfo*>(DiagObj->DiagArgumentsVal[Idx]);
   }
 
   /// getRawArg - Return the specified non-string argument in an opaque form.
   intptr_t getRawArg(unsigned Idx) const {
-    assert(getArgKind(Idx) != Diagnostic::ak_std_string &&
+    assert(getArgKind(Idx) != DiagnosticsEngine::ak_std_string &&
            "invalid argument accessor!");
     return DiagObj->DiagArgumentsVal[Idx];
   }
@@ -959,12 +967,12 @@ class DiagnosticInfo {
   /// FormatDiagnostic - Format this diagnostic into a string, substituting the
   /// formal arguments into the %0 slots.  The result is appended onto the Str
   /// array.
-  void FormatDiagnostic(llvm::SmallVectorImpl<char> &OutStr) const;
+  void FormatDiagnostic(SmallVectorImpl<char> &OutStr) const;
 
   /// FormatDiagnostic - Format the given format-string into the
   /// output buffer using the arguments stored in this diagnostic.
   void FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
-                        llvm::SmallVectorImpl<char> &OutStr) const;
+                        SmallVectorImpl<char> &OutStr) const;
 };
 
 /**
@@ -973,7 +981,7 @@ class DiagnosticInfo {
  */
 class StoredDiagnostic {
   unsigned ID;
-  Diagnostic::Level Level;
+  DiagnosticsEngine::Level Level;
   FullSourceLoc Loc;
   std::string Message;
   std::vector<CharSourceRange> Ranges;
@@ -981,18 +989,22 @@ class StoredDiagnostic {
 
 public:
   StoredDiagnostic();
-  StoredDiagnostic(Diagnostic::Level Level, const DiagnosticInfo &Info);
-  StoredDiagnostic(Diagnostic::Level Level, unsigned ID, 
-                   llvm::StringRef Message);
+  StoredDiagnostic(DiagnosticsEngine::Level Level, const Diagnostic &Info);
+  StoredDiagnostic(DiagnosticsEngine::Level Level, unsigned ID, 
+                   StringRef Message);
+  StoredDiagnostic(DiagnosticsEngine::Level Level, unsigned ID, 
+                   StringRef Message, FullSourceLoc Loc,
+                   ArrayRef<CharSourceRange> Ranges,
+                   ArrayRef<FixItHint> Fixits);
   ~StoredDiagnostic();
 
   /// \brief Evaluates true when this object stores a diagnostic.
   operator bool() const { return Message.size() > 0; }
 
   unsigned getID() const { return ID; }
-  Diagnostic::Level getLevel() const { return Level; }
+  DiagnosticsEngine::Level getLevel() const { return Level; }
   const FullSourceLoc &getLocation() const { return Loc; }
-  llvm::StringRef getMessage() const { return Message; }
+  StringRef getMessage() const { return Message; }
 
   void setLocation(FullSourceLoc Loc) { this->Loc = Loc; }
 
@@ -1007,20 +1019,20 @@ class StoredDiagnostic {
   unsigned fixit_size() const { return FixIts.size(); }
 };
 
-/// DiagnosticClient - This is an abstract interface implemented by clients of
+/// DiagnosticConsumer - This is an abstract interface implemented by clients of
 /// the front-end, which formats and prints fully processed diagnostics.
-class DiagnosticClient {
+class DiagnosticConsumer {
 protected:
   unsigned NumWarnings;       // Number of warnings reported
   unsigned NumErrors;         // Number of errors reported
   
 public:
-  DiagnosticClient() : NumWarnings(0), NumErrors(0) { }
+  DiagnosticConsumer() : NumWarnings(0), NumErrors(0) { }
 
   unsigned getNumErrors() const { return NumErrors; }
   unsigned getNumWarnings() const { return NumWarnings; }
 
-  virtual ~DiagnosticClient();
+  virtual ~DiagnosticConsumer();
 
   /// BeginSourceFile - Callback to inform the diagnostic client that processing
   /// of a source file is beginning.
@@ -1043,8 +1055,8 @@ class DiagnosticClient {
 
   /// IncludeInDiagnosticCounts - This method (whose default implementation
   /// returns true) indicates whether the diagnostics handled by this
-  /// DiagnosticClient should be included in the number of diagnostics reported
-  /// by Diagnostic.
+  /// DiagnosticConsumer should be included in the number of diagnostics
+  /// reported by DiagnosticsEngine.
   virtual bool IncludeInDiagnosticCounts() const;
 
   /// HandleDiagnostic - Handle this diagnostic, reporting it to the user or
@@ -1052,8 +1064,24 @@ class DiagnosticClient {
   ///
   /// Default implementation just keeps track of the total number of warnings
   /// and errors.
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info);
+  
+  /// \brief Clone the diagnostic consumer, producing an equivalent consumer
+  /// that can be used in a different context.
+  virtual DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const = 0;
+};
+
+/// IgnoringDiagConsumer - This is a diagnostic client that just ignores all
+/// diags.
+class IgnoringDiagConsumer : public DiagnosticConsumer {
+  void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                        const Diagnostic &Info) {
+    // Just ignore it.
+  }
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
+    return new IgnoringDiagConsumer();
+  }
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.td b/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.td
index 50a22c4a9120..8ae69fef2783 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Diagnostic.td
@@ -18,8 +18,6 @@ def MAP_IGNORE  : DiagMapping;
 def MAP_WARNING : DiagMapping;
 def MAP_ERROR   : DiagMapping;
 def MAP_FATAL   : DiagMapping;
-def MAP_WARNING_NO_WERROR             : DiagMapping;
-def MAP_WARNING_SHOW_IN_SYSTEM_HEADER : DiagMapping;
 
 // Define the diagnostic classes.
 class DiagClass;
@@ -59,6 +57,8 @@ class Diagnostic<string text, DiagClass DC, DiagMapping defaultmapping> {
   DiagClass   Class = DC;
   bit         SFINAE = 1;
   bit         AccessControl = 0;
+  bit         WarningNoWerror = 0;
+  bit         WarningShowInSystemHeader = 0;
   DiagMapping DefaultMapping = defaultmapping;
   DiagGroup   Group;
   string      CategoryName = "";
@@ -77,9 +77,11 @@ class DefaultIgnore { DiagMapping DefaultMapping = MAP_IGNORE; }
 class DefaultWarn   { DiagMapping DefaultMapping = MAP_WARNING; }
 class DefaultError  { DiagMapping DefaultMapping = MAP_ERROR; }
 class DefaultFatal  { DiagMapping DefaultMapping = MAP_FATAL; }
-class DefaultWarnNoWerror { DiagMapping DefaultMapping= MAP_WARNING_NO_WERROR; }
+class DefaultWarnNoWerror {
+  bit WarningNoWerror = 1;
+}
 class DefaultWarnShowInSystemHeader {
-  DiagMapping DefaultMapping = MAP_WARNING_SHOW_IN_SYSTEM_HEADER;
+  bit WarningShowInSystemHeader = 1;
 }
 
 class NoSFINAE { bit SFINAE = 0; }
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticASTKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticASTKinds.td
index 7d45bc58463c..705c95b7fc27 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -57,6 +57,10 @@ def note_odr_number_of_bases : Note<
 def note_odr_enumerator : Note<"enumerator %0 with value %1 here">;
 def note_odr_missing_enumerator : Note<"no corresponding enumerator here">;
 
+def err_odr_field_type_inconsistent : Error<
+  "field %0 declared with incompatible types in different "
+  "translation units (%1 vs. %2)">;
+
 // Importing Objective-C ASTs
 def err_odr_ivar_type_inconsistent : Error<
   "instance variable %0 declared with incompatible types in different "
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticCommonKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticCommonKinds.td
index 4b5de366cbc0..f9a910a1c21e 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -52,18 +52,31 @@ def err_invalid_storage_class_in_func_decl : Error<
   "invalid storage class specifier in function declarator">;
 def err_expected_namespace_name : Error<"expected namespace name">;
 def ext_variadic_templates : ExtWarn<
-  "variadic templates are a C++0x extension">, InGroup<CXX0x>;
+  "variadic templates are a C++11 extension">, InGroup<CXX11>;
+def warn_cxx98_compat_variadic_templates :
+  Warning<"variadic templates are incompatible with C++98">,
+  InGroup<CXX98Compat>, DefaultIgnore;
 def err_default_special_members : Error<
   "only special member functions may be defaulted">;
 def err_friends_define_only_namespace_scope : Error<
   "cannot define a function with non-namespace scope in a friend declaration">;
 def err_deleted_non_function : Error<
   "only functions can have deleted definitions">;
+def err_module_not_found : Error<"module '%0' not found">, DefaultFatal;
+def err_module_not_built : Error<"could not build module '%0'">, DefaultFatal;
+def err_module_cycle : Error<"cyclic dependency in module '%0': %1">, 
+  DefaultFatal;
+def warn_module_build : Warning<"building module '%0' from source">, 
+  InGroup<ModuleBuild>, DefaultIgnore;
+def note_pragma_entered_here : Note<"#pragma entered here">;  
 
 // Sema && Lex
 def ext_longlong : Extension<
   "'long long' is an extension when C99 mode is not enabled">,
   InGroup<LongLong>;
+def warn_cxx98_compat_longlong : Warning<
+  "'long long' is incompatible with C++98">,
+  InGroup<CXX98CompatPedantic>, DefaultIgnore;
 def warn_integer_too_large : Warning<
   "integer constant is too large for its type">;
 def warn_integer_too_large_for_signed : Warning<
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td
index e33b67ef7a07..3c0e4f53d3c1 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -65,6 +65,9 @@ def err_drv_command_signalled : Error<
   "%0 command failed due to signal %1 (use -v to see invocation)">;
 def err_drv_invalid_mfloat_abi : Error<
   "invalid float ABI '%0'">;
+def err_drv_invalid_libcxx_deployment : Error<
+  "invalid deployment target for -stdlib=libc++ (requires %0 or later)">;
+
 def err_drv_I_dash_not_supported : Error<
   "'%0' not supported, please use -iquote instead">;
 def err_drv_unknown_argument : Error<"unknown argument: '%0'">;
@@ -123,4 +126,7 @@ def warn_drv_objc_gc_unsupported : Warning<
 def warn_drv_pch_not_first_include : Warning<
   "precompiled header '%0' was ignored because '%1' is not first '-include'">;
 
+def note_drv_command_failed_diag_msg : Note<
+  "diagnostic msg: %0">;
+
 }
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 120ba67dc1f6..fffa42feb2fb 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -112,139 +112,11 @@ def err_relocatable_without_isysroot : Error<
 def warn_pch_target_triple : Error<
     "PCH file was compiled for the target '%0' but the current translation "
     "unit is being compiled for target '%1'">;
-def warn_pch_c99 : Error<
-    "C99 support was %select{disabled|enabled}0 in PCH file but is "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_c1x : Error<
-    "C1X support was %select{disabled|enabled}0 in PCH file but is "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_cplusplus : Error<
-    "C++ support was %select{disabled|enabled}0 in PCH file but is "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_cplusplus0x : Error<
-    "C++0x support was %select{disabled|enabled}0 in PCH file but is "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_objective_c : Error<
-    "Objective-C support was %select{disabled|enabled}0 in PCH file but is "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_objective_c2 : Error<
-    "Objective-C 2.0 support was %select{disabled|enabled}0 in PCH file but "
-    "is currently %select{disabled|enabled}1">;
-def warn_pch_nonfragile_abi : Error<
-    "PCH file was compiled with the %select{32-bit|non-fragile}0 Objective-C "
-    "ABI but the %select{32-bit|non-fragile}1 Objective-C ABI is selected">;
-def warn_pch_nonfragile_abi2 : Error<
-    "PCH file was compiled with the %select{32-bit|enhanced non-fragile}0 "
-    "Objective-C ABI but the %select{32-bit|enhanced non-fragile}1 "
-    "Objective-C ABI is selected">;
-def warn_pch_auto_ref_count : Error<
-    "PCH file was compiled %select{without|with} automated reference counting,"
-    "which is currently %select{disabled|enabled}">;
-def warn_pch_apple_kext : Error<
-    "PCH file was compiled %select{with|without}0 support for Apple's kernel "
-    "extensions ABI but it is currently %select{disabled|enabled}1">;
-def warn_pch_objc_auto_properties : Error<
-    "PCH file was compiled %select{with|without}0 support for auto-synthesized "
-    "@properties but it is currently %select{disabled|enabled}1">;
-def warn_pch_no_constant_cfstrings : Error<
-    "Objctive-C NSstring generation support was %select{disabled|enabled}0 "
-    "in PCH file but currently %select{disabled|enabled}1">; 
-def warn_pch_extensions : Error<
-    "extensions were %select{enabled|disabled}0 in PCH file but are "
-    "currently %select{enabled|disabled}1">;
-def warn_pch_gnu_extensions : Error<
-    "GNU extensions were %select{disabled|enabled}0 in PCH file but are "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_gnu_keywords : Error<
-    "GNU keywords were %select{disabled|enabled}0 in PCH file but are "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_microsoft_extensions : Error<
-    "Microsoft extensions were %select{disabled|enabled}0 in PCH file but are "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_ms_bitfields : Error<
-    "Microsoft-compatible structure layout was %select{disabled|enabled}0 in "
-    "PCH file but is currently %select{disabled|enabled}1">;
-def warn_pch_heinous_extensions : Error<
-    "heinous extensions were %select{disabled|enabled}0 in PCH file but are "
-    "currently %select{disabled|enabled}1">;
-def warn_pch_lax_vector_conversions : Error<
-    "lax vector conversions were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_altivec : Error<
-    "AltiVec initializers were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_opencl : Error<
-    "OpenCL language extensions were %select{disabled|enabled}0 in PCH file "
-    "but are currently %select{disabled|enabled}1">;
-def warn_pch_cuda : Error<
-    "CUDA language extensions were %select{disabled|enabled}0 in PCH file "
-    "but are currently %select{disabled|enabled}1">;
-def warn_pch_elide_constructors : Error<
-    "Elidable copy constructors were %select{disabled|enabled}0 in PCH file "
-    "but are currently %select{disabled|enabled}1">;
-def warn_pch_exceptions : Error<
-    "exceptions were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_objc_exceptions : Error<
-    "Objective-C exceptions were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_cxx_exceptions : Error<
-    "C++ exceptions were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_sjlj_exceptions : Error<
-    "sjlj-exceptions were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_objc_runtime : Error<
-    "PCH file was compiled with the %select{NeXT|GNU}0 runtime but the "
-    "%select{NeXT|GNU}1 runtime is selected">;
-def warn_pch_freestanding : Error<
-    "PCH file was compiled with a %select{hosted|freestanding}0  "
-    "implementation but a %select{hosted|freestanding}1 implementation "
-    "is selected">;
-def warn_pch_builtins : Error<
-    "PCH file was compiled with builtins %select{enabled|disabled}0 but "
-    "builtins are currently %select{enabled|disabled}1">;
-def warn_pch_thread_safe_statics : Error<
-    "PCH file was compiled %select{without|with}0 thread-safe statics but "
-    "thread-safe statics are currently %select{disabled|enabled}1">;
-def warn_pch_posix_threads : Error<
-    "PCH file was compiled %select{without|with}0 POSIX thread support but "
-    "POSIX threads are currently %select{disabled|enabled}1">;
-def warn_pch_stack_protector : Error<
-    "stack protector was %select{off|on|required}0 in PCH file but "
-    "is currently %select{off|on|required}1">;
-def warn_pch_blocks : Error<
-    "blocks were %select{disabled|enabled}0 in PCH file but "
-    "are currently %select{disabled|enabled}1">;
-def warn_pch_math_errno : Error<
-    "math functions %select{do not respect|respect}0 'errno' in PCH "
-    "file but they are currently set to %select{not respect|respect}1 "
-    "'errno'">;
-def warn_pch_optimize : Error<
-    "the macro '__OPTIMIZE__' was %select{not defined|defined}0 in "
-    "the PCH file but is currently %select{undefined|defined}1">;
-def warn_pch_optimize_size : Error<
-    "the macro '__OPTIMIZE_SIZE__' was %select{not defined|defined}0 in "
-    "the PCH file but is currently %select{undefined|defined}1">;
-def warn_pch_static : Error<
-    "the PCH file was compiled %select{dynamic|static}0 but the "
-    "current translation unit is being compiled as %select{dynamic|static}1">;
-def warn_pch_pic_level : Error<
-    "PCH file was compiled with PIC level %0, but the current translation "
-    "unit will be compiled with PIC level %1">;
-def warn_pch_gnu_inline : Error<
-    "PCH file was compiled with %select{C99|GNU|}0 inline semantics but "
-    "%select{C99|GNU}1 inline semantics are currently selected">;
-def warn_pch_no_inline : Error<
-    "the macro '__NO_INLINE__' was %select{not defined|defined}0 in "
-    "the PCH file but is currently %select{undefined|defined}1">;
-def warn_pch_deprecated : Error<
-    "the macro '__DEPRECATED' was %select{not defined|defined}0 in "
-    "the PCH file but is currently %select{undefined|defined}1">;
-def warn_pch_gc_mode : Error<
-    "the PCH file was built with %select{no||hybrid}0 garbage collection but "
-    "the current translation unit will compiled with %select{no||hybrid}1 "
-    "garbage collection">;
+def err_pch_langopt_mismatch : Error<"%0 was %select{disabled|enabled}1 in "
+    "PCH file but is currently %select{disabled|enabled}2">;
+def err_pch_langopt_value_mismatch : Error<
+  "%0 differs in PCH file vs. current file">;
+  
 def warn_pch_version_too_old : Error<
     "PCH file uses an older PCH format that is no longer supported">;
 def warn_pch_version_too_new : Error<
@@ -266,18 +138,6 @@ def warn_macro_name_used_in_pch : Error<
 def warn_pch_compiler_options_mismatch : Error<
     "compiler options used when building the precompiled header differ from "
     "the options used when using the precompiled header">;
-def warn_pch_access_control : Error<
-    "C++ access control was %select{disabled|enabled}0 in the PCH file but "
-    "is currently %select{disabled|enabled}1">;
-def warn_pch_char_signed : Error<
-    "char was %select{unsigned|signed}0 in the PCH file but "
-    "is currently %select{unsigned|signed}1">;
-def warn_pch_short_wchar : Error<
-    "-fshort-wchar was %select{disabled|enabled}0 in the PCH file but "
-    "is currently %select{disabled|enabled}1">;
-def warn_pch_short_enums : Error<
-    "-fshort-enums was %select{disabled|enabled}0 in the PCH file but "
-    "is currently %select{disabled|enabled}1">;
 
 def err_not_a_pch_file : Error<
     "'%0' does not appear to be a precompiled header file">, DefaultFatal;
@@ -291,6 +151,11 @@ def warn_unknown_warning_specifier : Warning<
     "unknown %0 warning specifier: '%1'">,
     InGroup<DiagGroup<"unknown-warning-option"> >;
 
-def warn_unkwown_analyzer_checker : Warning<
+def warn_unknown_analyzer_checker : Warning<
     "no analyzer checkers are associated with '%0'">;
+def warn_incompatible_analyzer_plugin_api : Warning<
+    "checker plugin '%0' is not compatible with this version of the analyzer">,
+    InGroup<DiagGroup<"analyzer-incompatible-plugin"> >;
+def note_incompatible_analyzer_plugin_api : Note<
+    "current API version is '%0', but plugin was compiled with version '%1'">;
 }
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td
index 8a109149884f..49603eb69e44 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td
@@ -54,9 +54,16 @@ def ExtraTokens : DiagGroup<"extra-tokens">;
 def FormatExtraArgs : DiagGroup<"format-extra-args">;
 def FormatZeroLength : DiagGroup<"format-zero-length">;
 
-def CXXHexFloats : DiagGroup<"c++-hex-floats">;
+def CXX98Compat : DiagGroup<"c++98-compat">;
+// Warnings for C++11 features which are Extensions in C++98 mode.
+def CXX98CompatPedantic : DiagGroup<"c++98-compat-pedantic", [CXX98Compat]>;
+
+def CXX11Narrowing : DiagGroup<"c++11-narrowing">;
+def : DiagGroup<"c++0x-narrowing", [CXX11Narrowing]>;
+
+def CXX11Compat : DiagGroup<"c++11-compat", [CXX11Narrowing]>;
+def : DiagGroup<"c++0x-compat", [CXX11Compat]>;
 
-def : DiagGroup<"c++0x-compat", [CXXHexFloats]>;
 def : DiagGroup<"effc++">;
 def ExitTimeDestructors : DiagGroup<"exit-time-destructors">;
 def FourByteMultiChar : DiagGroup<"four-char-constants">;
@@ -66,13 +73,15 @@ def BitwiseOpParentheses: DiagGroup<"bitwise-op-parentheses">;
 def LogicalOpParentheses: DiagGroup<"logical-op-parentheses">;
 def IgnoredQualifiers : DiagGroup<"ignored-qualifiers">;
 def : DiagGroup<"import">;
+def IncompatiblePointerTypes : DiagGroup<"incompatible-pointer-types">;
 def : DiagGroup<"init-self">;
 def : DiagGroup<"inline">;
 def : DiagGroup<"int-to-pointer-cast">;
 def : DiagGroup<"invalid-pch">;
 def LiteralRange : DiagGroup<"literal-range">;
 def LocalTypeTemplateArgs : DiagGroup<"local-type-template-args">;
-def : DiagGroup<"main">;
+def MalformedWarningCheck : DiagGroup<"malformed-warning-check">;
+def Main : DiagGroup<"main">;
 def MissingBraces : DiagGroup<"missing-braces">;
 def MissingDeclarations: DiagGroup<"missing-declarations">;
 def : DiagGroup<"missing-format-attribute">;
@@ -84,6 +93,7 @@ def : DiagGroup<"newline-eof">;
 def LongLong : DiagGroup<"long-long">;
 def MismatchedTags : DiagGroup<"mismatched-tags">;
 def MissingFieldInitializers : DiagGroup<"missing-field-initializers">;
+def ModuleBuild : DiagGroup<"module-build">;
 def NullDereference : DiagGroup<"null-dereference">;
 def InitializerOverrides : DiagGroup<"initializer-overrides">;
 def NonNull : DiagGroup<"nonnull">;
@@ -95,6 +105,9 @@ def OutOfLineDeclaration : DiagGroup<"out-of-line-declaration">;
 def : DiagGroup<"overflow">;
 def OverlengthStrings : DiagGroup<"overlength-strings">;
 def OverloadedVirtual : DiagGroup<"overloaded-virtual">;
+def ObjCMissingSuperCalls : DiagGroup<"objc-missing-super-calls">;
+def ObjCRetainBlockProperty : DiagGroup<"objc-noncopy-retain-block-property">;
+def ObjCContinuationPropertyType :DiagGroup<"objc-continuation-property-type">;
 def Packed : DiagGroup<"packed">;
 def Padded : DiagGroup<"padded">;
 def PointerArith : DiagGroup<"pointer-arith">;
@@ -108,6 +121,8 @@ def ReturnType : DiagGroup<"return-type">;
 def BindToTemporaryCopy : DiagGroup<"bind-to-temporary-copy">;
 def SelfAssignment : DiagGroup<"self-assign">;
 def SemiBeforeMethodBody : DiagGroup<"semicolon-before-method-body">;
+def Sentinel : DiagGroup<"sentinel">;
+def MissingMethodReturnType : DiagGroup<"missing-method-return-type">;
 def : DiagGroup<"sequence-point">;
 def Shadow : DiagGroup<"shadow">;
 def : DiagGroup<"shorten-64-to-32">;
@@ -152,6 +167,7 @@ def UnknownPragmas : DiagGroup<"unknown-pragmas">;
 def UnknownAttributes : DiagGroup<"attributes">;
 def UnnamedTypeTemplateArgs : DiagGroup<"unnamed-type-template-args">;
 def UnusedArgument : DiagGroup<"unused-argument">;
+def UnusedComparison : DiagGroup<"unused-comparison">;
 def UnusedExceptionParameter : DiagGroup<"unused-exception-parameter">;
 def UnneededInternalDecl : DiagGroup<"unneeded-internal-declaration">;
 def UnneededMemberFunction : DiagGroup<"unneeded-member-function">;
@@ -160,9 +176,11 @@ def UnusedMemberFunction : DiagGroup<"unused-member-function",
                                      [UnneededMemberFunction]>;
 def UnusedLabel : DiagGroup<"unused-label">;
 def UnusedParameter : DiagGroup<"unused-parameter">;
-def UnusedValue    : DiagGroup<"unused-value">;
+def UnusedResult : DiagGroup<"unused-result">;
+def UnusedValue : DiagGroup<"unused-value", [UnusedComparison, UnusedResult]>;
 def UnusedVariable : DiagGroup<"unused-variable">;
 def UsedButMarkedUnused : DiagGroup<"used-but-marked-unused">;
+def UserDefinedLiterals : DiagGroup<"user-defined-literals">;
 def ReadOnlySetterAttrs : DiagGroup<"readonly-setter-attrs">;
 def Reorder : DiagGroup<"reorder">;
 def UndeclaredSelector : DiagGroup<"undeclared-selector">;
@@ -183,6 +201,7 @@ def Selector : DiagGroup<"selector">;
 def NonfragileAbi2 : DiagGroup<"nonfragile-abi2">;
 def Protocol : DiagGroup<"protocol">;
 def SuperSubClassMismatch : DiagGroup<"super-class-method-mismatch">;
+def OverridingMethodMismatch : DiagGroup<"overriding-method-mismatch">;
 def : DiagGroup<"variadic-macros">;
 def VariadicMacros : DiagGroup<"variadic-macros">;
 def VectorConversions : DiagGroup<"vector-conversions">;      // clang specific
@@ -202,9 +221,11 @@ def DuplicateArgDecl : DiagGroup<"duplicate-method-arg">;
 // missing parentheses;  it is off by default.  We do not include it
 // in -Wparentheses because most users who use -Wparentheses explicitly
 // do not want these warnings.
+def ParenthesesOnEquality : DiagGroup<"parentheses-equality">;
 def Parentheses : DiagGroup<"parentheses",
                             [LogicalOpParentheses,
-                             BitwiseOpParentheses]>;
+                             BitwiseOpParentheses,
+                             ParenthesesOnEquality]>;
 
 // -Wconversion has its own warnings, but we split a few out for
 // legacy reasons:
@@ -216,6 +237,7 @@ def Conversion : DiagGroup<"conversion",
                            [DiagGroup<"shorten-64-to-32">,
                             DiagGroup<"constant-conversion">,
                             DiagGroup<"literal-conversion">,
+                            DiagGroup<"string-conversion">,
                             DiagGroup<"sign-conversion">,
                             BoolConversions]>,
                  DiagCategory<"Value Conversion Issue">;
@@ -228,11 +250,12 @@ def Unused : DiagGroup<"unused",
                         DiagCategory<"Unused Entity Issue">;
 
 // Format settings.
+def FormatInvalidSpecifier : DiagGroup<"format-invalid-specifier">;
 def FormatSecurity : DiagGroup<"format-security">;
 def FormatY2K : DiagGroup<"format-y2k">;
 def Format : DiagGroup<"format",
                        [FormatExtraArgs, FormatZeroLength, NonNull,
-                        FormatSecurity, FormatY2K]>,
+                        FormatSecurity, FormatY2K, FormatInvalidSpecifier]>,
              DiagCategory<"Format String Issue">;
 def FormatNonLiteral : DiagGroup<"format-nonliteral", [FormatSecurity]>;
 def Format2 : DiagGroup<"format=2",
@@ -243,6 +266,7 @@ def Extra : DiagGroup<"extra", [
     IgnoredQualifiers,
     InitializerOverrides,
     SemiBeforeMethodBody,
+    MissingMethodReturnType,
     SignCompare,
     UnusedParameter
   ]>;
@@ -265,12 +289,15 @@ def Most : DiagGroup<"most", [
     Uninitialized,
     UnknownPragmas,
     Unused,
-    VectorConversions,
     VolatileRegisterVar,
+    ObjCMissingSuperCalls,
     OverloadedVirtual
  ]>;
 
-// -Wall is -Wmost -Wparentheses
+// Thread Safety warnings 
+def ThreadSafety : DiagGroup<"thread-safety">;
+
+// -Wall is -Wmost -Wparentheses -Wtop-level-comparison
 def : DiagGroup<"all", [Most, Parentheses]>;
 
 // Aliases.
@@ -283,14 +310,16 @@ def : DiagGroup<"comments", [Comment]>;         // -Wcomments = -Wcomment
 def NonGCC : DiagGroup<"non-gcc",
     [SignCompare, Conversion, LiteralRange]>;
 
-// A warning group for warnings about using C++0x features as extensions in
+// A warning group for warnings about using C++11 features as extensions in
 // earlier C++ versions.
-def CXX0xStaticNonIntegralInitializer :
-  DiagGroup<"c++0x-static-nonintegral-init">;
-def CXX0x : DiagGroup<"c++0x-extensions", [CXX0xStaticNonIntegralInitializer]>;
+def CXX11 : DiagGroup<"c++11-extensions">;
+def : DiagGroup<"c++0x-extensions", [CXX11]>;
 def DelegatingCtorCycles :
   DiagGroup<"delegating-ctor-cycles">;
 
+// A warning group for warnings about using C1X features as extensions.
+def C1X : DiagGroup<"c1x-extensions">;
+
 // A warning group for warnings about GCC extensions.
 def GNU : DiagGroup<"gnu", [GNUDesignator, VLA]>;
 
@@ -299,3 +328,4 @@ def Microsoft : DiagGroup<"microsoft">;
 
 def ObjCNonUnifiedException : DiagGroup<"objc-nonunified-exceptions">;
 
+def ObjCProtocolMethodImpl : DiagGroup<"objc-protocol-method-implementation">;
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticIDs.h b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticIDs.h
index ae4ed5bbb13c..16d9b39985ed 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticIDs.h
@@ -16,10 +16,16 @@
 
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringRef.h"
+#include "clang/Basic/LLVM.h"
+
+namespace llvm {
+  template<typename T, unsigned> class SmallVector;
+}
 
 namespace clang {
-  class Diagnostic;
+  class DiagnosticsEngine;
   class SourceLocation;
+  struct WarningOption;
 
   // Import the diagnostic enums themselves.
   namespace diag {
@@ -43,7 +49,7 @@ namespace clang {
     // Get typedefs for common diagnostics.
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,CATEGORY,NOWERROR,SHOWINSYSHEADER,BRIEF,FULL) ENUM,
 #include "clang/Basic/DiagnosticCommonKinds.inc"
       NUM_BUILTIN_COMMON_DIAGNOSTICS
 #undef DIAG
@@ -59,20 +65,47 @@ namespace clang {
       MAP_IGNORE  = 1,     //< Map this diagnostic to nothing, ignore it.
       MAP_WARNING = 2,     //< Map this diagnostic to a warning.
       MAP_ERROR   = 3,     //< Map this diagnostic to an error.
-      MAP_FATAL   = 4,     //< Map this diagnostic to a fatal error.
-
-      /// Map this diagnostic to "warning", but make it immune to -Werror.  This
-      /// happens when you specify -Wno-error=foo.
-      MAP_WARNING_NO_WERROR = 5,
-      /// Map this diagnostic to "warning", but make it immune to
-      /// -Wno-system-headers.
-      MAP_WARNING_SHOW_IN_SYSTEM_HEADER = 6,
-      /// Map this diagnostic to "error", but make it immune to -Wfatal-errors.
-      /// This happens for -Wno-fatal-errors=foo.
-      MAP_ERROR_NO_WFATAL = 7
+      MAP_FATAL   = 4      //< Map this diagnostic to a fatal error.
     };
   }
 
+class DiagnosticMappingInfo {
+  unsigned Mapping : 3;
+  unsigned IsUser : 1;
+  unsigned IsPragma : 1;
+  unsigned HasShowInSystemHeader : 1;
+  unsigned HasNoWarningAsError : 1;
+  unsigned HasNoErrorAsFatal : 1;
+
+public:
+  static DiagnosticMappingInfo Make(diag::Mapping Mapping, bool IsUser,
+                                    bool IsPragma) {
+    DiagnosticMappingInfo Result;
+    Result.Mapping = Mapping;
+    Result.IsUser = IsUser;
+    Result.IsPragma = IsPragma;
+    Result.HasShowInSystemHeader = 0;
+    Result.HasNoWarningAsError = 0;
+    Result.HasNoErrorAsFatal = 0;
+    return Result;
+  }
+
+  diag::Mapping getMapping() const { return diag::Mapping(Mapping); }
+  void setMapping(diag::Mapping Value) { Mapping = Value; }
+
+  bool isUser() const { return IsUser; }
+  bool isPragma() const { return IsPragma; }
+
+  bool hasShowInSystemHeader() const { return HasShowInSystemHeader; }
+  void setShowInSystemHeader(bool Value) { HasShowInSystemHeader = Value; }
+
+  bool hasNoWarningAsError() const { return HasNoWarningAsError; }
+  void setNoWarningAsError(bool Value) { HasNoWarningAsError = Value; }
+
+  bool hasNoErrorAsFatal() const { return HasNoErrorAsFatal; }
+  void setNoErrorAsFatal(bool Value) { HasNoErrorAsFatal = Value; }
+};
+
 /// \brief Used for handling and querying diagnostic IDs. Can be used and shared
 /// by multiple Diagnostics for multiple translation units.
 class DiagnosticIDs : public llvm::RefCountedBase<DiagnosticIDs> {
@@ -93,7 +126,7 @@ class DiagnosticIDs : public llvm::RefCountedBase<DiagnosticIDs> {
   /// getCustomDiagID - Return an ID for a diagnostic with the specified message
   /// and level.  If this is the first request for this diagnosic, it is
   /// registered and created, otherwise the existing ID is returned.
-  unsigned getCustomDiagID(Level L, llvm::StringRef Message);
+  unsigned getCustomDiagID(Level L, StringRef Message);
 
   //===--------------------------------------------------------------------===//
   // Diagnostic classification and reporting interfaces.
@@ -101,14 +134,18 @@ class DiagnosticIDs : public llvm::RefCountedBase<DiagnosticIDs> {
 
   /// getDescription - Given a diagnostic ID, return a description of the
   /// issue.
-  llvm::StringRef getDescription(unsigned DiagID) const;
+  StringRef getDescription(unsigned DiagID) const;
 
-  /// isBuiltinWarningOrExtension - Return true if the unmapped diagnostic
-  /// level of the specified diagnostic ID is a Warning or Extension.
-  /// This only works on builtin diagnostics, not custom ones, and is not legal to
-  /// call on NOTEs.
+  /// isBuiltinWarningOrExtension - Return true if the unmapped diagnostic level
+  /// of the specified diagnostic ID is a Warning or Extension.  This only works
+  /// on builtin diagnostics, not custom ones, and is not legal to call on
+  /// NOTEs.
   static bool isBuiltinWarningOrExtension(unsigned DiagID);
 
+  /// \brief Return true if the specified diagnostic is mapped to errors by
+  /// default.
+  static bool isDefaultMappingAsError(unsigned DiagID);
+
   /// \brief Determine whether the given built-in diagnostic ID is a
   /// Note.
   static bool isBuiltinNote(unsigned DiagID);
@@ -132,7 +169,7 @@ class DiagnosticIDs : public llvm::RefCountedBase<DiagnosticIDs> {
   /// getWarningOptionForDiag - Return the lowest-level warning option that
   /// enables the specified diagnostic.  If there is no -Wfoo flag that controls
   /// the diagnostic, this returns null.
-  static llvm::StringRef getWarningOptionForDiag(unsigned DiagID);
+  static StringRef getWarningOptionForDiag(unsigned DiagID);
   
   /// getCategoryNumberForDiag - Return the category number that a specified
   /// DiagID belongs to, or 0 if no category.
@@ -143,7 +180,7 @@ class DiagnosticIDs : public llvm::RefCountedBase<DiagnosticIDs> {
 
   /// getCategoryNameFromID - Given a category ID, return the name of the
   /// category.
-  static llvm::StringRef getCategoryNameFromID(unsigned CategoryID);
+  static StringRef getCategoryNameFromID(unsigned CategoryID);
   
   /// \brief Enumeration describing how the the emission of a diagnostic should
   /// be treated when it occurs during C++ template argument deduction.
@@ -182,56 +219,80 @@ class DiagnosticIDs : public llvm::RefCountedBase<DiagnosticIDs> {
   static SFINAEResponse getDiagnosticSFINAEResponse(unsigned DiagID);
 
   /// getName - Given a diagnostic ID, return its name
-  static llvm::StringRef getName(unsigned DiagID);
+  static StringRef getName(unsigned DiagID);
   
   /// getIdFromName - Given a diagnostic name, return its ID, or 0
-  static unsigned getIdFromName(llvm::StringRef Name);
+  static unsigned getIdFromName(StringRef Name);
   
   /// getBriefExplanation - Given a diagnostic ID, return a brief explanation
   /// of the issue
-  static llvm::StringRef getBriefExplanation(unsigned DiagID);
+  static StringRef getBriefExplanation(unsigned DiagID);
 
   /// getFullExplanation - Given a diagnostic ID, return a full explanation
   /// of the issue
-  static llvm::StringRef getFullExplanation(unsigned DiagID);
+  static StringRef getFullExplanation(unsigned DiagID);
+  
+  /// Iterator class used for traversing all statically declared
+  /// diagnostics.
+  class diag_iterator {
+    const void *impl;
+
+    friend class DiagnosticIDs;    
+    diag_iterator(const void *im) : impl(im) {}
+  public:
+    diag_iterator &operator++();
+    bool operator==(const diag_iterator &x) const { return impl == x.impl; }
+    bool operator!=(const diag_iterator &x) const { return impl != x.impl; }
+    
+    llvm::StringRef getDiagName() const;
+    unsigned getDiagID() const;    
+  };
+
+  static diag_iterator diags_begin();
+  static diag_iterator diags_end();
+
+  /// \brief Get the set of all diagnostic IDs in the group with the given name.
+  ///
+  /// \param Diags [out] - On return, the diagnostics in the group.
+  /// \returns True if the given group is unknown, false otherwise.
+  bool getDiagnosticsInGroup(StringRef Group,
+                             llvm::SmallVectorImpl<diag::kind> &Diags) const;
 
 private:
-  /// setDiagnosticGroupMapping - Change an entire diagnostic group (e.g.
-  /// "unknown-pragmas" to have the specified mapping.  This returns true and
-  /// ignores the request if "Group" was unknown, false otherwise.
-  bool setDiagnosticGroupMapping(llvm::StringRef Group, diag::Mapping Map,
-                                 SourceLocation Loc, Diagnostic &Diag) const;
-
-  /// \brief Based on the way the client configured the Diagnostic
+  /// \brief Get the set of all diagnostic IDs in the given group.
+  ///
+  /// \param Diags [out] - On return, the diagnostics in the group.
+  void getDiagnosticsInGroup(const WarningOption *Group,
+                             llvm::SmallVectorImpl<diag::kind> &Diags) const;
+ 
+  /// \brief Based on the way the client configured the DiagnosticsEngine
   /// object, classify the specified diagnostic ID into a Level, consumable by
   /// the DiagnosticClient.
   ///
   /// \param Loc The source location we are interested in finding out the
   /// diagnostic state. Can be null in order to query the latest state.
   DiagnosticIDs::Level getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
-                                          const Diagnostic &Diag,
-                                          diag::Mapping *mapping = 0) const;
+                                          const DiagnosticsEngine &Diag) const;
 
   /// getDiagnosticLevel - This is an internal implementation helper used when
   /// DiagClass is already known.
   DiagnosticIDs::Level getDiagnosticLevel(unsigned DiagID,
                                           unsigned DiagClass,
                                           SourceLocation Loc,
-                                          const Diagnostic &Diag,
-                                          diag::Mapping *mapping = 0) const;
+                                          const DiagnosticsEngine &Diag) const;
 
   /// ProcessDiag - This is the method used to report a diagnostic that is
   /// finally fully formed.
   ///
   /// \returns true if the diagnostic was emitted, false if it was
   /// suppressed.
-  bool ProcessDiag(Diagnostic &Diag) const;
+  bool ProcessDiag(DiagnosticsEngine &Diag) const;
 
   /// \brief Whether the diagnostic may leave the AST in a state where some
   /// invariants can break.
   bool isUnrecoverable(unsigned DiagID) const;
 
-  friend class Diagnostic;
+  friend class DiagnosticsEngine;
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticLexKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticLexKinds.td
index 38d6a8001655..9b3a178cac78 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -24,6 +24,11 @@ def escaped_newline_block_comment_end : Warning<
 def backslash_newline_space : Warning<
   "backslash and newline separated by space">;
 
+// Digraphs.
+def warn_cxx98_compat_less_colon_colon : Warning<
+  "'<::' is treated as digraph '<:' (aka '[') followed by ':' in C++98">,
+  InGroup<CXX98Compat>, DefaultIgnore;
+
 // Trigraphs.
 def trigraph_ignored : Warning<"trigraph ignored">, InGroup<Trigraphs>;
 def trigraph_ignored_block_comment : Warning<
@@ -38,12 +43,16 @@ def ext_multi_line_bcpl_comment : Extension<"multi-line // comment">,
 def ext_bcpl_comment : Extension<
   "// comments are not allowed in this language">,
   InGroup<Comment>;
-def ext_no_newline_eof : Extension<"no newline at end of file">;
-def ext_backslash_newline_eof : Extension<"backslash-newline at end of file">;
+def ext_no_newline_eof : Extension<"no newline at end of file">, 
+  InGroup<DiagGroup<"newline-eof">>;
 def ext_dollar_in_identifier : Extension<"'$' in identifier">;
 def charize_microsoft_ext : Extension<"@# is a microsoft extension">;
 
-def ext_token_used : Extension<"extension used">;
+def ext_token_used : Extension<"extension used">,
+  InGroup<DiagGroup<"language-extension-token">>;
+
+def warn_cxx11_keyword : Warning<"'%0' is a keyword in C++11">,
+  InGroup<CXX11Compat>;
 
 def warn_unterminated_string : ExtWarn<"missing terminating '\"' character">;
 def warn_unterminated_char : ExtWarn<"missing terminating ' character">;
@@ -55,6 +64,18 @@ def err_unterminated___pragma : Error<"missing terminating ')' character">;
 
 def err_conflict_marker : Error<"version control conflict marker in file">;
 
+def err_raw_delim_too_long : Error<
+  "raw string delimiter longer than 16 characters"
+  "; use PREFIX( )PREFIX to delimit raw string">;
+def err_invalid_char_raw_delim : Error<
+  "invalid character '%0' character in raw string delimiter"
+  "; use PREFIX( )PREFIX to delimit raw string">;
+def err_unterminated_raw_string : Error<
+  "raw string missing terminating delimiter )%0\"">;
+def warn_cxx98_compat_raw_string_literal : Warning<
+  "raw string literals are incompatible with C++98">,
+  InGroup<CXX98Compat>, DefaultIgnore;
+
 def ext_multichar_character_literal : ExtWarn<
   "multi-character character constant">, InGroup<MultiChar>;
 def ext_four_char_character_literal : Extension<
@@ -77,17 +98,14 @@ def err_invalid_suffix_integer_constant : Error<
   "invalid suffix '%0' on integer constant">;
 def err_invalid_suffix_float_constant : Error<
   "invalid suffix '%0' on floating constant">;
-def warn_extraneous_wide_char_constant : Warning<
-  "extraneous characters in wide character constant ignored">;
+def warn_extraneous_char_constant : Warning<
+  "extraneous characters in character constant ignored">;
 def warn_char_constant_too_large : Warning<
   "character constant too long for its type">;
 def err_exponent_has_no_digits : Error<"exponent has no digits">;
 def ext_imaginary_constant : Extension<"imaginary constants are an extension">;
 def err_hexconstant_requires_exponent : Error<
   "hexadecimal floating constants require an exponent">;
-def ext_hexconstant_cplusplus : Extension<
-  "hexadecimal floating constants are a C99 feature that is incompatible with "
-  "C++0x">, InGroup<CXXHexFloats>;
 def ext_hexconstant_invalid : Extension<
   "hexadecimal floating constants are a C99 feature">;
 def ext_binary_literal : Extension<
@@ -102,6 +120,11 @@ def warn_ucn_escape_too_large : ExtWarn<
   "character unicode escape sequence too long for its type">;
 def warn_ucn_not_valid_in_c89 : ExtWarn<
   "unicode escape sequences are only valid in C99 or C++">;
+def warn_cxx98_compat_unicode_literal : Warning<
+  "unicode literals are incompatible with C++98">,
+  InGroup<CXX98Compat>, DefaultIgnore;
+def err_unsupported_string_concat : Error<
+  "unsupported non-standard concatenation of string literals">;
   
 //===----------------------------------------------------------------------===//
 // PTH Diagnostics
@@ -167,6 +190,9 @@ def ext_pp_bad_vaargs_use : Extension<
 def ext_pp_macro_redef : ExtWarn<"%0 macro redefined">;
 def ext_variadic_macro : Extension<"variadic macros were introduced in C99">,
   InGroup<VariadicMacros>;
+def warn_cxx98_compat_variadic_macro : Warning<
+  "variadic macros are incompatible with C++98">,
+  InGroup<CXX98CompatPedantic>, DefaultIgnore;
 def ext_named_variadic_macro : Extension<
   "named variadic macros are a GNU extension">, InGroup<VariadicMacros>;
 def ext_embedded_directive : Extension<
@@ -175,10 +201,13 @@ def ext_missing_varargs_arg : Extension<
   "varargs argument missing, but tolerated as an extension">;
 def ext_empty_fnmacro_arg : Extension<
   "empty macro arguments were standardized in C99">;
+def warn_cxx98_compat_empty_fnmacro_arg : Warning<
+  "empty macro argument list is incompatible with C++98">,
+  InGroup<CXX98CompatPedantic>, DefaultIgnore;
 
 def err_pp_invalid_directive : Error<"invalid preprocessing directive">;
 def err_pp_hash_error : Error<"#error%0">;
-def warn_pp_file_not_found : Warning<"'%0' file not found">, DefaultFatal;
+def err_pp_file_not_found : Error<"'%0' file not found">, DefaultFatal;
 def err_pp_error_opening_file : Error<
   "error opening file '%0': %1">, DefaultFatal;
 def err_pp_empty_filename : Error<"empty filename">;
@@ -230,6 +259,13 @@ def err_pp_used_poisoned_id : Error<"attempt to use a poisoned identifier">;
 def err_feature_check_malformed : Error<
   "builtin feature check macro requires a parenthesized identifier">;
 
+def err_warning_check_malformed : Error<
+  "builtin warning check macro requires a parenthesized string">,
+  InGroup<MalformedWarningCheck>;
+def warn_has_warning_invalid_option :
+   ExtWarn<"__has_warning expected option name (e.g. \"-Wundef\")">,
+   InGroup<MalformedWarningCheck>;
+
 def err__Pragma_malformed : Error<
   "_Pragma takes a parenthesized string literal">;
 def err_pragma_comment_malformed : Error<
@@ -315,5 +351,20 @@ def err_pp_linemarker_invalid_pop : Error<
   "invalid line marker flag '2': cannot pop empty include stack">;
 def ext_pp_line_too_big : Extension<
   "C requires #line number to be less than %0, allowed as extension">;
+def warn_cxx98_compat_pp_line_too_big : Warning<
+  "#line number greater than 32767 is incompatible with C++98">,
+  InGroup<CXX98CompatPedantic>, DefaultIgnore;
+
+def err_pp_export_non_macro : Error<"no macro named %0 to export">;
+
+def err_pp_arc_cf_code_audited_syntax : Error<"expected 'begin' or 'end'">;
+def err_pp_double_begin_of_arc_cf_code_audited : Error<
+  "already inside '#pragma clang arc_cf_code_audited'">;
+def err_pp_unmatched_end_of_arc_cf_code_audited : Error<
+  "not currently inside '#pragma clang arc_cf_code_audited'">;
+def err_pp_include_in_arc_cf_code_audited : Error<
+  "cannot #include files inside '#pragma clang arc_cf_code_audited'">;
+def err_pp_eof_in_arc_cf_code_audited : Error<
+  "'#pragma clang arc_cf_code_audited' was not ended within this file">;
 
 }
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticParseKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticParseKinds.td
index 3764a4091546..73437b217b18 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -28,10 +28,6 @@ def ext_extra_struct_semi : Extension<
 def ext_extra_ivar_semi : Extension<
   "extra ';' inside instance variable list">;
 
-def auto_storage_class : ExtWarn<
-  "'auto' storage class specifier is redundant and will be "
-  "removed in future releases">;
-
 def ext_duplicate_declspec : Extension<"duplicate '%0' declaration specifier">;
 def ext_plain_complex : ExtWarn<
   "plain '_Complex' requires a type specifier; assuming '_Complex double'">;
@@ -57,7 +53,7 @@ def ext_c99_variable_decl_in_for_loop : Extension<
 def ext_c99_compound_literal : Extension<
   "compound literals are a C99-specific feature">;
 def ext_enumerator_list_comma : Extension<
-  "commas at the end of enumerator lists are a %select{C99|C++0x}0-specific "
+  "commas at the end of enumerator lists are a %select{C99|C++11}0-specific "
   "feature">;
 def err_enumerator_list_missing_comma : Error<
   "missing ',' between enumerators">;
@@ -68,12 +64,15 @@ def ext_ms_enum_fixed_underlying_type : Extension<
   InGroup<Microsoft>;
 
 def ext_c1x_generic_selection : Extension<
-  "generic selections are a C1X-specific feature">;
+  "generic selections are a C1X-specific feature">, InGroup<C1X>;
 def err_duplicate_default_assoc : Error<
   "duplicate default generic association">;
 def note_previous_default_assoc : Note<
   "previous default generic association is here">;
 
+def ext_c1x_alignas : Extension<
+  "_Alignas is a C1X-specific feature">, InGroup<C1X>;
+
 def ext_gnu_indirect_goto : Extension<
   "use of GNU indirect-goto extension">, InGroup<GNU>;
 def ext_gnu_address_of_label : Extension<
@@ -188,16 +187,26 @@ def err_invalid_reference_qualifier_application : Error<
 def err_illegal_decl_reference_to_reference : Error<
   "%0 declared as a reference to a reference">;
 def ext_rvalue_reference : ExtWarn<
-  "rvalue references are a C++0x extension">, InGroup<CXX0x>;
+  "rvalue references are a C++11 extension">, InGroup<CXX11>;
 def ext_ref_qualifier : ExtWarn<
-  "reference qualifiers on functions are a C++0x extension">, InGroup<CXX0x>;
+  "reference qualifiers on functions are a C++11 extension">, InGroup<CXX11>;
 def ext_inline_namespace : ExtWarn<
-  "inline namespaces are a C++0x feature">, InGroup<CXX0x>;
+  "inline namespaces are a C++11 feature">, InGroup<CXX11>;
 def err_generalized_initializer_lists : Error<
-  "generalized initializer lists are a C++0x extension unsupported in Clang">;
+  "generalized initializer lists are a C++11 extension unsupported in Clang">;
 def ext_generalized_initializer_lists : ExtWarn<
-  "generalized initializer lists are a C++0x extension unsupported in Clang">,
-  InGroup<CXX0x>;
+  "generalized initializer lists are a C++11 extension unsupported in Clang">,
+  InGroup<CXX11>;
+def ext_auto_type_specifier : ExtWarn<
+  "'auto' type specifier is a C++11 extension">, InGroup<CXX11>;
+def warn_auto_storage_class : Warning<
+  "'auto' storage class specifier is redundant and incompatible with C++11">,
+  InGroup<CXX11Compat>;
+def ext_auto_storage_class : ExtWarn<
+  "'auto' storage class specifier is not permitted in C++11, and will not "
+  "be supported in future releases">;
+def ext_for_range : ExtWarn<
+  "range-based for loop is a C++11 extension">, InGroup<CXX11>;
 def err_argument_required_after_attribute : Error<
   "argument required after attribute">;
 def err_missing_param : Error<"expected parameter declarator">;
@@ -219,6 +228,9 @@ def err_typename_invalid_storageclass : Error<
   "type name does not allow storage class to be specified">;
 def err_typename_invalid_functionspec : Error<
   "type name does not allow function specifier to be specified">;
+def err_typename_identifiers_only : Error<
+  "typename is allowed for identifiers only">;
+  
 def err_invalid_decl_spec_combination : Error<
   "cannot combine with previous '%0' declaration specifier">;
 def err_invalid_vector_decl_spec_combination : Error<
@@ -250,7 +262,7 @@ def err_unexected_colon_in_nested_name_spec : Error<
 def err_bool_redeclaration : Error<
   "redeclaration of C++ built-in type 'bool'">;
 def ext_c1x_static_assert : Extension<
-  "_Static_assert is a C1X-specific feature">;
+  "_Static_assert is a C1X-specific feature">, InGroup<C1X>;
 
 /// Objective-C parser diagnostics
 def err_expected_minus_or_plus : Error<
@@ -342,6 +354,8 @@ def err_destructor_template_id : Error<
   "destructor name %0 does not refer to a template">;
 def err_default_arg_unparsed : Error<
   "unexpected end of default argument expression">;
+def err_parser_impl_limit_overflow : Error<
+  "parser recursion limit reached, program too complex">, DefaultFatal;
 
 // C++ derived classes
 def err_dup_virtual : Error<"duplicate 'virtual' in base specifier">;
@@ -357,10 +371,15 @@ def err_default_delete_in_multiple_declaration : Error<
   "'= %select{default|delete}0' is a function definition and must occur in a "
   "standalone declaration">;
 
+def warn_cxx98_compat_alignas : Warning<"'alignas' is incompatible with C++98">,
+  InGroup<CXX98Compat>, DefaultIgnore;
+def warn_cxx98_compat_attribute : Warning<
+  "attributes are incompatible with C++98">,
+  InGroup<CXX98Compat>, DefaultIgnore;
 def err_cxx0x_attribute_forbids_arguments : Error<
-  "C++0x attribute '%0' cannot have an argument list">;
+  "C++11 attribute '%0' cannot have an argument list">;
 def err_cxx0x_attribute_requires_arguments : Error<
-  "C++0x attribute '%0' must have an argument list">;
+  "C++1 attribute '%0' must have an argument list">;
 def err_attributes_not_allowed : Error<"an attribute list cannot appear here">;
 
 /// C++ Templates
@@ -385,7 +404,7 @@ def err_two_right_angle_brackets_need_space : Error<
   "a space is required between consecutive right angle brackets (use '> >')">;
 def warn_cxx0x_right_shift_in_template_arg : Warning<
   "use of right-shift operator ('>>') in template argument will require "
-  "parentheses in C++0x">;
+  "parentheses in C++11">;
 def err_multiple_template_declarators : Error<
     "%select{|a template declaration|an explicit template specialization|"
     "an explicit template instantiation}0 can "
@@ -446,31 +465,31 @@ def err_missing_whitespace_digraph : Error<
   " which forms the digraph '<:' (aka '[') and a ':', did you mean '< ::'?">;
 
 def warn_deleted_function_accepted_as_extension: ExtWarn<
-  "deleted function definition accepted as a C++0x extension">, InGroup<CXX0x>;
+  "deleted function definition accepted as a C++11 extension">, InGroup<CXX11>;
 def warn_defaulted_function_accepted_as_extension: ExtWarn<
-  "defaulted function definition accepted as a C++0x extension">,
-  InGroup<CXX0x>;
+  "defaulted function definition accepted as a C++11 extension">,
+  InGroup<CXX11>;
 
-// C++0x in-class member initialization
+// C++11 in-class member initialization
 def warn_nonstatic_member_init_accepted_as_extension: ExtWarn<
-  "in-class initialization of non-static data member accepted as a C++0x extension">,
-  InGroup<CXX0x>;
+  "in-class initialization of non-static data member accepted as a C++11 extension">,
+  InGroup<CXX11>;
 def err_bitfield_member_init: Error<
   "bitfield member cannot have an in-class initializer">;
 def err_incomplete_array_member_init: Error<
   "array bound cannot be deduced from an in-class initializer">;
 
-// C++0x alias-declaration
+// C++11 alias-declaration
 def ext_alias_declaration : ExtWarn<
-  "alias declarations accepted as a C++0x extension">, InGroup<CXX0x>;
+  "alias declarations accepted as a C++11 extension">, InGroup<CXX11>;
 def err_alias_declaration_not_identifier : Error<
   "name defined in alias declaration must be an identifier">;
 def err_alias_declaration_specialization : Error<
   "%select{partial specialization|explicit specialization|explicit instantiation}0 of alias templates is not permitted">;
     
-// C++0x override control
+// C++11 override control
 def ext_override_control_keyword : Extension<
-  "'%0' keyword accepted as a C++0x extension">, InGroup<CXX0x>;
+  "'%0' keyword accepted as a C++11 extension">, InGroup<CXX11>;
 
 def err_duplicate_virt_specifier : Error<
   "class member already marked '%0'">;
@@ -487,6 +506,15 @@ def err_paren_sizeof_parameter_pack : Error<
 def err_sizeof_parameter_pack : Error<
   "expected parenthesized parameter pack name in 'sizeof...' expression">;
 
+// C++11 lambda expressions
+def err_expected_comma_or_rsquare : Error<
+  "expected ',' or ']' in lambda capture list">;
+def err_this_captured_by_reference : Error<
+  "'this' cannot be captured by reference">;
+def err_expected_capture : Error<
+  "expected variable name or 'this' in lambda capture list">;
+def err_expected_lambda_body : Error<"expected body of lambda expression">;
+
 // Availability attribute
 def err_expected_version : Error<
   "expected a version of the form 'major[.minor[.subminor]]'">;
@@ -559,6 +587,14 @@ def err_seh___except_filter : Error<
 
 def err_seh___finally_block : Error<
   "%0 only allowed in __finally block">;
-
+  
 } // end of Parse Issue category.
+
+let CategoryName = "Modules Issue" in {
+def err_module_expected_ident : Error<
+  "expected a module name after '__import_module__'">;
+def err_module_expected_semi : Error<
+  "expected a semicolon name after module name">;
+}
+
 } // end of Parser diagnostics
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 97414f23d796..0fbf0cee0a5d 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -23,7 +23,8 @@ def ext_expr_not_ice : Extension<
 
 // Semantic analysis of constant literals.
 def ext_predef_outside_function : Warning<
-  "predefined identifier is only valid inside function">;
+  "predefined identifier is only valid inside function">,
+  InGroup<DiagGroup<"predefined-identifier-outside-function">>;
 def warn_float_overflow : Warning<
   "magnitude of floating-point constant too large for type %0; maximum is %1">,
    InGroup<LiteralRange>;
@@ -102,7 +103,7 @@ def ext_flexible_array_init : Extension<
 def ext_anon_param_requires_type_specifier : Extension<
   "type specifier required for unnamed parameter, defaults to int">;
 def err_bad_variable_name : Error<
-  "'%0' cannot be the name of a variable or data member">;
+  "%0 cannot be the name of a variable or data member">;
 def err_bad_parameter_name : Error<
   "'%0' cannot be the name of a parameter">;
 def err_parameter_name_omitted : Error<"parameter name omitted">;
@@ -236,10 +237,10 @@ def err_maybe_falloff_nonvoid_block : Error<
 def err_falloff_nonvoid_block : Error<
   "control reaches end of non-void block">;
 def warn_suggest_noreturn_function : Warning<
-  "function could be attribute 'noreturn'">,
+  "%select{function|method}0 %1 could be declared with attribute 'noreturn'">,
   InGroup<DiagGroup<"missing-noreturn">>, DefaultIgnore;
 def warn_suggest_noreturn_block : Warning<
-  "block could be attribute 'noreturn'">,
+  "block could be declared with attribute 'noreturn'">,
   InGroup<DiagGroup<"missing-noreturn">>, DefaultIgnore;
 def warn_unreachable : Warning<"will never be executed">,
   InGroup<DiagGroup<"unreachable-code">>, DefaultIgnore;
@@ -264,8 +265,9 @@ def err_types_compatible_p_in_cplusplus : Error<
   "__builtin_types_compatible_p is not valid in C++">;
 def warn_builtin_unknown : Warning<"use of unknown builtin %0">, DefaultError;
 def warn_dyn_class_memaccess : Warning<
-  "%select{destination for|source of}0 this %1 call is a pointer to dynamic "
-  "class %2; vtable pointer will be overwritten">, 
+  "%select{destination for|source of|first operand of|second operand of}0 this "
+  "%1 call is a pointer to dynamic class %2; vtable pointer will be "
+  "%select{overwritten|copied|moved|compared}3">,
   InGroup<DiagGroup<"dynamic-class-memaccess">>;
 def note_bad_memaccess_silence : Note<
   "explicitly cast the pointer to silence this warning">;
@@ -278,18 +280,25 @@ def warn_sizeof_pointer_type_memaccess : Warning<
   "argument to 'sizeof' in %0 call is the same pointer type %1 as the "
   "%select{destination|source}2; expected %3 or an explicit length">,
   InGroup<DiagGroup<"sizeof-pointer-memaccess">>;
+def warn_strlcpycat_wrong_size : Warning<
+  "size argument in %0 call appears to be size of the source; expected the size of "
+  "the destination">,
+  InGroup<DiagGroup<"strlcpy-strlcat-size">>;
+def note_strlcpycat_wrong_size : Note<
+  "change size argument to be the size of the destination">;
 
 /// main()
 // static/inline main() are not errors in C, just in C++.
-def warn_unusual_main_decl : Warning<"'main' should not be declared "
-    "%select{static|inline|static or inline}0">;
-def err_unusual_main_decl : Error<"'main' is not allowed to be declared "
-    "%select{static|inline|static or inline}0">;
+def warn_static_main : Warning<"'main' should not be declared static">,
+    InGroup<Main>;
+def err_static_main : Error<"'main' is not allowed to be declared static">;
+def err_inline_main : Error<"'main' is not allowed to be declared inline">;
 def err_main_template_decl : Error<"'main' cannot be a template">;
 def err_main_returns_nonint : Error<"'main' must return 'int'">;
 def err_main_surplus_args : Error<"too many parameters (%0) for 'main': "
     "must be 0, 2, or 3">;
-def warn_main_one_arg : Warning<"only one parameter on 'main' declaration">;
+def warn_main_one_arg : Warning<"only one parameter on 'main' declaration">,
+    InGroup<Main>;
 def err_main_arg_wrong : Error<"%select{first|second|third|fourth}0 "
     "parameter of 'main' (%select{argument count|argument array|environment|"
     "platform-specific data}0) must be of type %1">;
@@ -303,6 +312,8 @@ def err_statically_allocated_object : Error<
 def err_object_cannot_be_passed_returned_by_value : Error<
   "interface type %1 cannot be %select{returned|passed}0 by value"
   "; did you forget * in %1">;
+def err_parameters_retval_cannot_have_fp16_type : Error<
+  "%select{parameters|function return value}0 cannot have __fp16 type; did you forget * ?">;
 def warn_enum_value_overflow : Warning<"overflow in enumeration value">;
 def warn_pragma_options_align_unsupported_option : Warning<
   "unsupported alignment option in '#pragma options align'">;
@@ -383,27 +394,75 @@ def note_undef_method_impl : Note<"method definition for %0 not found">;
 def note_required_for_protocol_at : 
   Note<"required for direct or indirect protocol %0">;
 
+def warn_conflicting_overriding_ret_types : Warning<
+  "conflicting return type in "
+  "declaration of %0: %1 vs %2">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
 def warn_conflicting_ret_types : Warning<
-  "conflicting return type in implementation of %0: %1 vs %2">;
+  "conflicting return type in "
+  "implementation of %0: %1 vs %2">;
+
+def warn_conflicting_overriding_ret_type_modifiers : Warning<
+  "conflicting distributed object modifiers on return type "
+  "in declaration of %0">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
 def warn_conflicting_ret_type_modifiers : Warning<
   "conflicting distributed object modifiers on return type "
   "in implementation of %0">,
   InGroup<DiagGroup<"distributed-object-modifiers">>;
+
+def warn_non_covariant_overriding_ret_types : Warning<
+  "conflicting return type in "
+  "declaration of %0: %1 vs %2">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
 def warn_non_covariant_ret_types : Warning<
-  "conflicting return type in implementation of %0: %1 vs %2">,
+  "conflicting return type in "
+  "implementation of %0: %1 vs %2">,
   InGroup<DiagGroup<"method-signatures">>, DefaultIgnore;
 
+def warn_conflicting_overriding_param_types : Warning<
+  "conflicting parameter types in "
+  "declaration of %0: %1 vs %2">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
 def warn_conflicting_param_types : Warning<
-  "conflicting parameter types in implementation of %0: %1 vs %2">;
+  "conflicting parameter types in "
+  "implementation of %0: %1 vs %2">;
 def warn_conflicting_param_modifiers : Warning<
   "conflicting distributed object modifiers on parameter type "
   "in implementation of %0">,
   InGroup<DiagGroup<"distributed-object-modifiers">>;
+
+def warn_conflicting_overriding_param_modifiers : Warning<
+  "conflicting distributed object modifiers on parameter type "
+  "in declaration of %0">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
+def warn_non_contravariant_overriding_param_types : Warning<
+  "conflicting parameter types in "
+  "declaration of %0: %1 vs %2">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
 def warn_non_contravariant_param_types : Warning<
-  "conflicting parameter types in implementation of %0: %1 vs %2">,
+  "conflicting parameter types in "
+  "implementation of %0: %1 vs %2">,
   InGroup<DiagGroup<"method-signatures">>, DefaultIgnore;
+
+def warn_conflicting_overriding_variadic :Warning<
+  "conflicting variadic declaration of method and its "
+  "implementation">,
+  InGroup<OverridingMethodMismatch>, DefaultIgnore;
+
 def warn_conflicting_variadic :Warning<
-  "conflicting variadic declaration of method and its implementation">;
+  "conflicting variadic declaration of method and its "
+  "implementation">;
+
+def warn_category_method_impl_match:Warning<
+  "category is implementing a method which will also be implemented"
+  " by its primary class">, InGroup<ObjCProtocolMethodImpl>;
 
 def warn_implements_nscopying : Warning<
 "default assign attribute on property %0 which implements "
@@ -437,12 +496,22 @@ def warn_property_attr_mismatch : Warning<
 def warn_objc_property_copy_missing_on_block : Warning<
     "'copy' attribute must be specified for the block property "
     "when -fobjc-gc-only is specified">;
+def warn_objc_property_retain_of_block : Warning<
+    "retain'ed block property does not copy the block "
+    "- use copy attribute instead">, InGroup<ObjCRetainBlockProperty>;
 def warn_atomic_property_rule : Warning<
-  "writable atomic property %0 cannot pair a synthesized setter/getter "
-  "with a user defined setter/getter">;
-def warn_ownin_getter_rule : Warning<
+  "writable atomic property %0 cannot pair a synthesized %select{getter|setter}1 "
+  "with a user defined %select{getter|setter}2">,
+  InGroup<DiagGroup<"atomic-property-with-user-defined-accessor">>;
+def note_atomic_property_fixup_suggest : Note<"setter and getter must both be "
+  "synthesized, or both be user defined,or the property must be nonatomic">;
+def warn_atomic_property_nontrivial_assign_op : Warning<
+  "atomic property of type %0 synthesizing setter using non-trivial assignment"
+  " operator">, InGroup<DiagGroup<"objc-property-atomic-setter-synthesis">>;
+def warn_owning_getter_rule : Warning<
   "property's synthesized getter follows Cocoa naming"
-  " convention for returning 'owned' objects">;
+  " convention for returning 'owned' objects">,
+  InGroup<DiagGroup<"objc-property-matches-cocoa-ownership-rule">>;
 def warn_property_getter_owning_mismatch : Warning<
   "property declared as returning non-retained objects"
   "; getter returning retained objects">;
@@ -456,6 +525,9 @@ def warn_default_atomic_custom_getter_setter : Warning<
 def err_use_continuation_class : Error<
   "illegal redeclaration of property in continuation class %0"
   " (attribute must be 'readwrite', while its primary must be 'readonly')">;
+def warn_type_mismatch_continuation_class : Warning<
+  "type of property %0 in continuation class does not match "
+  "property type in primary class">, InGroup<ObjCContinuationPropertyType>;
 def err_use_continuation_class_redeclaration_readwrite : Error<
   "illegal redeclaration of 'readwrite' property in continuation class %0"
   " (perhaps you intended this to be a 'readwrite' redeclaration of a "
@@ -489,7 +561,10 @@ def error_synthesize_weak_non_arc_or_gc : Error<
 def err_arc_perform_selector_retains : Error<
   "performSelector names a selector which retains the object">;
 def warn_arc_perform_selector_leaks : Warning<
-  "performSelector may cause a leak because its selector is unknown">;
+  "performSelector may cause a leak because its selector is unknown">,
+  InGroup<DiagGroup<"arc-performSelector-leaks">>;
+def err_gc_weak_property_strong_type : Error<
+  "weak attribute declared on a __strong type property in GC mode">;
 
 def error_synthesized_ivar_yet_not_supported : Error<
   "instance variable synthesis not yet supported"
@@ -511,6 +586,12 @@ def error_property_implemented : Error<"property %0 is already implemented">;
 def warn_objc_property_attr_mutually_exclusive : Warning<
   "property attributes '%0' and '%1' are mutually exclusive">,
   InGroup<ReadOnlySetterAttrs>, DefaultIgnore;
+def warn_objc_missing_super_dealloc : Warning<
+  "method possibly missing a [super dealloc] call">,
+  InGroup<ObjCMissingSuperCalls>;
+def warn_objc_missing_super_finalize : Warning<
+  "method possibly missing a [super finalize] call">,
+  InGroup<ObjCMissingSuperCalls>;
 def warn_undeclared_selector : Warning<
   "undeclared selector %0">, InGroup<UndeclaredSelector>, DefaultIgnore;
 def warn_implicit_atomic_property : Warning<
@@ -539,12 +620,12 @@ def err_unexpected_friend : Error<
 def ext_enum_friend : ExtWarn<
   "enumeration type %0 cannot be a friend">;
 def ext_nonclass_type_friend : ExtWarn<
-  "non-class friend type %0 is a C++0x extension">, InGroup<CXX0x>;
+  "non-class friend type %0 is a C++11 extension">, InGroup<CXX11>;
 def err_friend_is_member : Error<
   "friends cannot be members of the declaring class">;
 def ext_unelaborated_friend_type : ExtWarn<
   "specify '%select{struct|union|class|enum}0' to befriend %1; accepted "
-  "as a C++0x extension">, InGroup<CXX0x>;
+  "as a C++11 extension">, InGroup<CXX11>;
 def err_qualified_friend_not_found : Error<
   "no function named %0 with type %1 was found in the specified scope">;
 def err_introducing_special_friend : Error<
@@ -554,9 +635,15 @@ def err_tagless_friend_type_template : Error<
   "friend type templates must use an elaborated type">;
 def err_no_matching_local_friend : Error<
   "no matching function found in local scope">;
+def err_no_matching_local_friend_suggest : Error<
+  "no matching function %0 found in local scope; did you mean %2">;
 def err_partial_specialization_friend : Error<
   "partial specialization cannot be declared as a friend">;
-
+def err_qualified_friend_def : Error<
+  "friend function definition cannot be qualified with '%0'">;
+def err_friend_def_in_local_class : Error<
+  "friend function cannot be defined in a local class">;
+  
 def err_abstract_type_in_decl : Error<
   "%select{return|parameter|variable|field}0 type %1 is an abstract class">;
 def err_allocation_of_abstract_type : Error<
@@ -963,7 +1050,7 @@ def warn_maybe_uninit_var_captured_by_block : Warning<
   "variable %0 may be uninitialized when captured by block">,
   InGroup<UninitializedMaybe>, DefaultIgnore;
 def note_var_fixit_add_initialization : Note<
-  "add initialization to silence this warning">;
+  "initialize the variable %0 to silence this warning">;
 def err_init_incomplete_type : Error<"initialization of incomplete type %0">;
 
 def err_temp_copy_no_viable : Error<
@@ -991,11 +1078,11 @@ def err_temp_copy_deleted : Error<
 def err_temp_copy_incomplete : Error<
   "copying a temporary object of incomplete type %0">;
 
-// C++0x decltype
+// C++11 decltype
 def err_cannot_determine_declared_type_of_overloaded_function : Error<
     "cannot determine the type of an overloaded function">;
     
-// C++0x auto
+// C++11 auto
 def err_auto_variable_cannot_appear_in_own_initializer : Error<
   "variable %0 declared with 'auto' type cannot appear in its own initializer">;
 def err_illegal_decl_array_of_auto : Error<
@@ -1028,7 +1115,7 @@ def err_auto_new_deduction_failure : Error<
 def err_auto_different_deductions : Error<
   "'auto' deduced as %0 in declaration of %1 and deduced as %2 in declaration of %3">;
 
-// C++0x override control
+// C++11 override control
 def override_keyword_only_allowed_on_virtual_member_functions : Error<
   "only virtual member functions can be marked '%0'">;
 def err_function_marked_override_not_overriding : Error<
@@ -1036,16 +1123,16 @@ def err_function_marked_override_not_overriding : Error<
 def err_class_marked_final_used_as_base : Error<
   "base %0 is marked 'final'">;
 
-// C++0x attributes
+// C++11 attributes
 def err_repeat_attribute : Error<"'%0' attribute cannot be repeated">;
 
-// C++0x [[final]]
+// C++11 [[final]]
 def err_final_function_overridden : Error<
   "declaration of %0 overrides a 'final' function">;
 def err_final_base : Error<
   "derivation from 'final' %0">;
 
-// C++0x scoped enumerations
+// C++11 scoped enumerations
 def err_enum_invalid_underlying : Error<
   "non-integral type %0 is an invalid underlying type">;
 def err_enumerator_too_large : Error<
@@ -1066,9 +1153,9 @@ def err_only_enums_have_underlying_types : Error<
 def err_incomplete_type_no_underlying_type : Error<
   "an incomplete enumeration type has no underlying type yet">;
 
-// C++0x delegating constructors
+// C++11 delegating constructors
 def err_delegation_0x_only : Error<
-  "delegating constructors are permitted only in C++0x">;
+  "delegating constructors are permitted only in C++11">;
 def err_delegating_initializer_alone : Error<
   "an initializer for a delegating constructor must appear alone">;
 def warn_delegating_ctor_cycle : Warning<
@@ -1081,7 +1168,7 @@ def note_which_delegates_to : Note<
 def err_delegating_codegen_not_implemented : Error<
   "code generation for delegating constructors not implemented">;
 
-// C++0x range-based for loop
+// C++11 range-based for loop
 def err_for_range_decl_must_be_var : Error<
   "for range declaration must declare a variable">;
 def err_for_range_storage_class : Error<
@@ -1102,7 +1189,88 @@ def err_for_range_begin_end_types_differ : Error<
 def note_for_range_type : Note<"range has type %0">;
 def note_for_range_begin_end : Note<
   "selected '%select{begin|end}0' %select{function|template }1%2 with iterator type %3">;
-  
+
+// C++11 constexpr
+def err_invalid_constexpr : Error<
+  "%select{function parameter|typedef|non-static data member}0 "
+  "cannot be constexpr">;
+def err_constexpr_tag : Error<
+  "%select{class|struct|union|enum}0 cannot be marked constexpr">;
+def err_constexpr_dtor : Error<"destructor cannot be marked constexpr">;
+def err_constexpr_no_declarators : Error<
+  "constexpr can only be used in variable and function declarations">;
+def err_invalid_constexpr_var_decl : Error<
+  "constexpr variable declaration must be a definition">;
+def err_constexpr_var_requires_init : Error<
+  "declaration of constexpr variable %0 requires an initializer">;
+def err_constexpr_var_requires_const_init : Error<
+  "constexpr variable %0 must be initialized by a constant expression">;
+def err_constexpr_redecl_mismatch : Error<
+  "%select{non-constexpr declaration of %0 follows constexpr declaration"
+  "|constexpr declaration of %0 follows non-constexpr declaration}1">;
+def note_constexpr_redecl_mismatch : Note<
+  "previous declaration was %select{not |}0marked constexpr">;
+def err_constexpr_virtual : Error<"virtual function cannot be constexpr">;
+def note_constexpr_tmpl_virtual : Note<"function template instantiation is not "
+  "constexpr because it is virtual">;
+def err_constexpr_virtual_base : Error<"constexpr constructor not allowed in "
+  "%select{class|struct}0 with virtual base %plural{1:class|:classes}1">;
+def note_constexpr_tmpl_virtual_base : Note<"constructor template instantiation is "
+  "not constexpr because %select{class|struct}0 has virtual base "
+  "%plural{1:class|:classes}1">;
+def note_non_literal_virtual_base : Note<"%select{class|struct}0 with virtual "
+  "base %plural{1:class|:classes}1 is not a literal type">;
+def note_constexpr_virtual_base_here : Note<"virtual base class declared here">;
+def err_constexpr_non_literal_return : Error<
+  "constexpr function's return type %0 is not a literal type">;
+def note_constexpr_tmpl_non_literal_return : Note<
+  "function template instantiation is not constexpr because return type %0 is "
+  "not a literal type">;
+def err_constexpr_non_literal_param : Error<
+  "constexpr %select{function|constructor}1's %ordinal0 parameter type %2 is "
+  "not a literal type">;
+def note_constexpr_tmpl_non_literal_param : Note<
+  "%select{function|constructor}1 template instantiation is not constexpr "
+  "because %ordinal0 parameter type %2 is not a literal type">;
+def err_constexpr_body_invalid_stmt : Error<
+  "statement not allowed in constexpr %select{function|constructor}0">;
+def err_constexpr_type_definition : Error<
+  "types cannot be defined in a constexpr %select{function|constructor}0">;
+def err_constexpr_vla : Error<
+  "variably-modified type %0 cannot be used in a constexpr "
+  "%select{function|constructor}1">;
+def err_constexpr_var_declaration : Error<
+  "variables cannot be declared in a constexpr %select{function|constructor}0">;
+def err_constexpr_body_no_return : Error<
+  "no return statement in constexpr function">;
+def err_constexpr_body_multiple_return : Error<
+  "multiple return statements in constexpr function">;
+def note_constexpr_body_previous_return : Note<
+  "previous return statement is here">;
+def err_constexpr_function_try_block : Error<
+  "function try block not allowed in constexpr %select{function|constructor}0">;
+def err_constexpr_union_ctor_no_init : Error<
+  "constexpr union constructor does not initialize any member">;
+def err_constexpr_ctor_missing_init : Error<
+  "constexpr constructor must initialize all members">;
+def note_constexpr_ctor_missing_init : Note<
+  "member not initialized by constructor">;
+def err_constexpr_method_non_literal : Error<
+  "non-literal type %0 cannot have constexpr members">;
+def note_non_literal_no_constexpr_ctors : Note<
+  "%0 is not literal because it is not an aggregate and has no constexpr "
+  "constructors other than copy or move constructors">;
+def note_non_literal_base_class : Note<
+  "%0 is not literal because it has base class %1 of non-literal type">;
+def note_non_literal_field : Note<
+  "%0 is not literal because it has data member %1 of non-literal type %2">;
+def note_non_literal_user_provided_dtor : Note<
+  "%0 is not literal because it has a user-provided destructor">;
+def note_non_literal_nontrivial_dtor : Note<
+  "%0 is not literal because it has a non-trivial destructor">;
+def note_non_literal_mutable_field : Note<
+  "%0 is not literal because it has a mutable data member">;
+ 
 // Objective-C++
 def err_objc_decls_may_only_appear_in_global_scope : Error<
   "Objective-C declarations may only appear in global scope">;
@@ -1119,11 +1287,8 @@ def err_attribute_wrong_number_arguments : Error<
   ":requires exactly %0 arguments}0">;
 def err_attribute_too_many_arguments : Error<
   "attribute takes no more than %0 argument%s0">;
-def err_iboutletcollection_type : Error<
-  "invalid type %0 as argument of iboutletcollection attribute">;
-def err_iboutletcollection_object_type : Error<
-  "%select{ivar|property}1 with iboutletcollection attribute must "
-  "have object type (invalid %0)">;
+def err_attribute_too_few_arguments : Error<
+  "attribute takes at least %0 argument%s0">;
 def err_attribute_missing_parameter_name : Error<
   "attribute requires unquoted parameter">;
 def err_attribute_invalid_vector_type : Error<"invalid vector element type %0">;
@@ -1131,6 +1296,10 @@ def err_attribute_bad_neon_vector_size : Error<
   "Neon vector size must be 64 or 128 bits">;
 def err_attribute_argument_not_int : Error<
   "'%0' attribute requires integer constant">;
+def err_attribute_argument_not_class : Error<
+  "%0 attribute requires arguments that are class type or point to class type">;
+def err_attribute_first_argument_not_int_or_bool : Error<
+  "%0 attribute first argument must be of int or bool type">;
 def err_attribute_argument_outof_range : Error<
   "init_priority attribute requires integer constant between "
   "101 and 65535 inclusive">;
@@ -1186,6 +1355,8 @@ def err_attribute_address_space_too_high : Error<
   "address space is larger than the maximum supported (%0)">;
 def err_attribute_address_multiple_qualifiers : Error<
   "multiple address spaces specified for type">;
+def err_attribute_address_function_type : Error<
+  "function type may not be qualified with an address space">;
 def err_as_qualified_auto_decl : Error<
   "automatic variable qualified with an address space">;
 def err_arg_with_address_space : Error<
@@ -1196,6 +1367,8 @@ def err_attr_objc_ownership_redundant : Error<
   "the type %0 already has retainment attributes set on it">;
 def err_attribute_not_string : Error<
   "argument to %0 attribute was not a string literal">;
+def err_only_annotate_after_access_spec : Error<
+  "access specifier can only have annotation attributes">;
 def err_attribute_section_invalid_for_target : Error<
   "argument to 'section' attribute is not valid for this target: %0">;
 def err_attribute_section_local_variable : Error<
@@ -1238,13 +1411,13 @@ def warn_attribute_wrong_decl_type : Warning<
   "parameters and methods|functions, methods and blocks|"
   "classes and virtual methods|functions, methods, and parameters|"
   "classes|virtual methods|class members|variables|methods|"
-  "variables, functions and labels}1">;
+  "variables, functions and labels|fields and global variables}1">;
 def err_attribute_wrong_decl_type : Error<
   "%0 attribute only applies to %select{functions|unions|"
   "variables and functions|functions and methods|parameters|"
   "parameters and methods|functions, methods and blocks|"
   "classes and virtual methods|functions, methods, and parameters|"
-  "classes|virtual methods|class members|variables|methods}1">;
+  "classes|virtual methods|class members|variables|methods|structs}1">;
 def warn_function_attribute_wrong_type : Warning<
   "'%0' only applies to function types; type here is %1">;
 def warn_pointer_attribute_wrong_type : Warning<
@@ -1268,6 +1441,9 @@ def err_cconv_varargs : Error<
 def err_regparm_mismatch : Error<"function declared with with regparm(%0) "
   "attribute was previously declared "
   "%plural{0:without the regparm|:with the regparm(%1)}1 attribute">;
+def err_returns_retained_mismatch : Error<
+  "function declared with the ns_returns_retained attribute "
+  "was previously declared without the ns_returns_retained attribute">;
 def err_objc_precise_lifetime_bad_type : Error<
   "objc_precise_lifetime only applies to retainable types; type here is %0">;
 def warn_objc_precise_lifetime_meaningless : Error<
@@ -1276,6 +1452,11 @@ def warn_objc_precise_lifetime_meaningless : Error<
 def warn_label_attribute_not_unused : Warning<
   "The only valid attribute for labels is 'unused'">;
 def err_invalid_pcs : Error<"Invalid PCS type">;
+def err_attribute_can_be_applied_only_to_value_decl : Error<
+  "%0 attribute can only be applied to value declarations">;
+def warn_attribute_not_on_decl : Error<
+  "%0 attribute ignored when parsing type">;
+
 
 // Availability attribute
 def warn_availability_unknown_platform : Warning<
@@ -1284,6 +1465,68 @@ def warn_availability_version_ordering : Warning<
   "feature cannot be %select{introduced|deprecated|obsoleted}0 in %1 version "
   "%2 before it was %select{introduced|deprecated|obsoleted}3 in version %4; "
   "attribute ignored">;
+  
+// Thread Safety Attributes
+// Errors when parsing the attributes
+def err_attribute_argument_out_of_range : Error<
+  "%0 attribute parameter %1 is out of bounds: "
+  "%plural{0:no parameters to index into|"
+  "1:can only be 1, since there is one parameter|"
+  ":must be between 1 and %2}2">;
+def err_attribute_argument_not_lockable : Error<
+  "%0 attribute requires arguments whose type is annotated "
+  "with 'lockable' attribute">;
+def err_attribute_decl_not_lockable : Error<
+  "%0 attribute can only be applied in a context annotated "
+  "with 'lockable' attribute">;
+def warn_unlock_but_no_lock : Warning<
+  "unlocking '%0' that was not locked">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_double_lock : Warning<
+  "locking '%0' that is already locked">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_no_unlock : Warning<
+  "mutex '%0' is still locked at the end of function">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+// FIXME: improve the error message about locks not in scope
+def warn_lock_at_end_of_scope : Warning<
+  "mutex '%0' is still locked at the end of its scope">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_expecting_lock_held_on_loop : Warning<
+  "expecting mutex '%0' to be locked at start of each loop">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_lock_exclusive_and_shared : Warning<
+  "mutex '%0' is locked exclusively and shared in the same scope">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def note_lock_exclusive_and_shared : Note<
+  "the other lock of mutex '%0' is here">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_variable_requires_lock : Warning<
+  "%select{reading|writing}2 variable '%0' requires locking "
+  "%select{'%1'|'%1' exclusively}2">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_var_deref_requires_lock : Warning<
+  "%select{reading|writing}2 the value pointed to by '%0' requires locking "
+  "%select{'%1'|'%1' exclusively}2">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_variable_requires_any_lock : Warning<
+  "%select{reading|writing}1 variable '%0' requires locking "
+  "%select{any mutex|any mutex exclusively}1">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_var_deref_requires_any_lock : Warning<
+  "%select{reading|writing}1 the value pointed to by '%0' requires locking "
+  "%select{any mutex|any mutex exclusively}1">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_fun_requires_lock : Warning<
+  "calling function '%0' requires %select{shared|exclusive}2 lock on '%1'">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_fun_excludes_mutex : Warning<
+  "cannot call function '%0' while mutex '%1' is locked">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+def warn_cannot_resolve_lock : Warning<
+  "cannot resolve lock expression to a specific lockable object">,
+  InGroup<ThreadSafety>, DefaultIgnore;
+
 
 def warn_impcast_vector_scalar : Warning<
   "implicit conversion turns vector to scalar: %0 to %1">,
@@ -1319,13 +1562,14 @@ def warn_impcast_literal_float_to_integer : Warning<
   "implicit conversion turns literal floating-point number into integer: "
   "%0 to %1">,
   InGroup<DiagGroup<"literal-conversion">>, DefaultIgnore;
-def note_fix_integral_float_as_integer : Note<
-  "this can be rewritten as an integer literal with the exact same value">;
+def warn_impcast_string_literal_to_bool : Warning<
+  "implicit conversion turns string literal into bool: %0 to %1">,
+  InGroup<DiagGroup<"string-conversion">>, DefaultIgnore;
 def warn_impcast_different_enum_types : Warning<
   "implicit conversion from enumeration type %0 to different enumeration type "
   "%1">, InGroup<DiagGroup<"conversion">>;
 def warn_impcast_bool_to_null_pointer : Warning<
-    "initialization of pointer of type %0 to NULL from a constant boolean "
+    "initialization of pointer of type %0 to null from a constant boolean "
     "expression">, InGroup<BoolConversions>;
 def warn_impcast_null_pointer_to_integer : Warning<
     "implicit conversion of NULL constant to integer">,
@@ -1398,6 +1642,11 @@ def warn_attribute_iboutlet : Warning<
   "%0 attribute can only be applied to instance variables or properties">;
 def warn_attribute_ibaction: Warning<
   "ibaction attribute can only be applied to Objective-C instance methods">;
+def err_iboutletcollection_type : Error<
+  "invalid type %0 as argument of iboutletcollection attribute">;
+def err_iboutlet_object_type : Error<
+  "%select{ivar|property}2 with %0 attribute must "
+  "be an object type (invalid %1)">;
 def err_attribute_overloadable_not_function : Error<
   "'overloadable' attribute can only be applied to a function">;
 def err_attribute_overloadable_missing : Error<
@@ -1409,10 +1658,12 @@ def err_attribute_overloadable_no_prototype : Error<
   "'overloadable' function %0 must have a prototype">;
 def warn_ns_attribute_wrong_return_type : Warning<
   "%0 attribute only applies to %select{functions|methods}1 that "
-  "return %select{an Objective-C object|a pointer}2">;
+  "return %select{an Objective-C object|a pointer|a non-retainable pointer}2">;
 def warn_ns_attribute_wrong_parameter_type : Warning<
   "%0 attribute only applies to %select{Objective-C object|pointer}1 "
   "parameters">;
+def err_ns_bridged_not_interface : Error<
+  "parameter of 'ns_bridged' attribute does not name an Objective-C class">;
 
 // Function Parameter Semantic Analysis.
 def err_param_with_void_type : Error<"argument may not have 'void' type">;
@@ -1572,6 +1823,16 @@ def note_ovl_candidate_bad_conv_incomplete : Note<"candidate "
     "function (the implicit move assignment operator)|"
     "constructor (inherited)}0%1 "
     "not viable: cannot convert argument of incomplete type %2 to %3">;
+def note_ovl_candidate_bad_list_argument : Note<"candidate "
+    "%select{function|function|constructor|"
+    "function |function |constructor |"
+    "constructor (the implicit default constructor)|"
+    "constructor (the implicit copy constructor)|"
+    "constructor (the implicit move constructor)|"
+    "function (the implicit copy assignment operator)|"
+    "function (the implicit move assignment operator)|"
+    "constructor (inherited)}0%1 "
+    "not viable: cannot convert initializer list argument to %3">;
 def note_ovl_candidate_bad_overload : Note<"candidate "
     "%select{function|function|constructor|"
     "function |function |constructor |"
@@ -1592,7 +1853,22 @@ def note_ovl_candidate_bad_conv : Note<"candidate "
     "function (the implicit move assignment operator)|"
     "constructor (inherited)}0%1"
     " not viable: no known conversion from %2 to %3 for "
-    "%select{%ordinal5 argument|object argument}4">;
+    "%select{%ordinal5 argument|object argument}4; "
+    "%select{|dereference the argument with *|"
+    "take the address of the argument with &|"
+    "remove *|"
+    "remove &}6">;
+def note_ovl_candidate_bad_arc_conv : Note<"candidate "
+    "%select{function|function|constructor|"
+    "function |function |constructor |"
+    "constructor (the implicit default constructor)|"
+    "constructor (the implicit copy constructor)|"
+    "constructor (the implicit move constructor)|"
+    "function (the implicit copy assignment operator)|"
+    "function (the implicit move assignment operator)|"
+    "constructor (inherited)}0%1"
+    " not viable: cannot implicitly convert argument of type %2 to %3 for "
+    "%select{%ordinal5 argument|object argument}4 under ARC">;
 def note_ovl_candidate_bad_addrspace : Note<"candidate "
     "%select{function|function|constructor|"
     "function |function |constructor |"
@@ -1620,15 +1896,18 @@ def note_ovl_candidate_bad_ownership : Note<"candidate "
     "function |function |constructor |"
     "constructor (the implicit default constructor)|"
     "constructor (the implicit copy constructor)|"
+    "constructor (the implicit move constructor)|"
     "function (the implicit copy assignment operator)|"
+    "function (the implicit move assignment operator)|"
     "constructor (inherited)}0%1 not viable: "
     "%select{%ordinal6|'this'}5 argument (%2) has "
     "%select{no|__unsafe_unretained|__strong|__weak|__autoreleasing}3 ownership,"
     " but parameter has %select{no|__unsafe_unretained|__strong|__weak|"
     "__autoreleasing}4 ownership">;
 def note_ovl_candidate_bad_cvr_this : Note<"candidate "
-    "%select{|function|||function||||"
-    "function (the implicit copy assignment operator)|}0 not viable: "
+    "%select{|function|||function|||||"
+    "function (the implicit copy assignment operator)|"
+    "function (the implicit move assignment operator)|}0 not viable: "
     "'this' argument has type %2, but method is not marked "
     "%select{const|restrict|const or restrict|volatile|const or volatile|"
     "volatile or restrict|const, volatile, or restrict}3">;
@@ -1658,6 +1937,17 @@ def note_ovl_candidate_bad_base_to_derived_conv : Note<"candidate "
     "%select{base class pointer|superclass|base class object of type}2 %3 to "
     "%select{derived class pointer|subclass|derived class reference}2 %4 for "
     "%ordinal5 argument">;
+def note_ovl_candidate_bad_target : Note<
+    "candidate %select{function|function|constructor|"
+    "function |function |constructor |"
+    "constructor (the implicit default constructor)|"
+    "constructor (the implicit copy constructor)|"
+    "constructor (the implicit move constructor)|"
+    "function (the implicit copy assignment operator)|"
+    "function (the implicit move assignment operator)|"
+    "constructor (inherited)}0 not viable: call to "
+    "%select{__device__|__global__|__host__|__host__ __device__}1 function from"
+    " %select{__device__|__global__|__host__|__host__ __device__}2 function">;
 
 def note_ambiguous_type_conversion: Note<
     "because of ambiguity in conversion of %0 to %1">;
@@ -1694,9 +1984,13 @@ def err_ovl_no_viable_subscript :
     Error<"no viable overloaded operator[] for type %0">;
 def err_ovl_no_oper :
     Error<"type %0 does not provide a %select{subscript|call}1 operator">;
-def err_ovl_unresolvable :
-    Error<"cannot resolve overloaded function %0 from context">;
-  
+def err_ovl_unresolvable : Error<
+  "reference to overloaded function could not be resolved; "
+  "did you mean to call it%select{| with no arguments}0?">;
+def err_bound_member_function : Error<
+  "reference to non-static member function must be called"
+  "%select{|; did you mean to call it with no arguments?}0">;
+def note_possible_target_of_call : Note<"possible target for call">;
 
 def err_ovl_no_viable_object_call : Error<
   "no matching function for call to object of type %0">;
@@ -1761,8 +2055,8 @@ def note_template_param_prev_default_arg : Note<
 def err_template_param_default_arg_missing : Error<
   "template parameter missing a default argument">;
 def ext_template_parameter_default_in_function_template : ExtWarn<
-  "default template arguments for a function template are a C++0x extension">,
-  InGroup<CXX0x>;
+  "default template arguments for a function template are a C++11 extension">,
+  InGroup<CXX11>;
 def err_template_parameter_default_template_member : Error<
   "cannot add a default template argument to the definition of a member of a "
   "class template">;
@@ -1774,6 +2068,9 @@ def err_template_template_parm_no_parms : Error<
 def err_template_variable : Error<"variable %0 declared as a template">;
 def err_template_variable_noparams : Error<
   "extraneous 'template<>' in declaration of variable %0">;
+def err_template_member : Error<"member %0 declared as a template">;
+def err_template_member_noparams : Error<
+  "extraneous 'template<>' in declaration of member %0">;
 def err_template_tag_noparams : Error<
   "extraneous 'template<>' in declaration of %0 %1">;
 def err_template_decl_ref : Error<
@@ -1895,8 +2192,8 @@ def err_template_spec_decl_out_of_scope_global : Error<
 def ext_template_spec_decl_out_of_scope_global : ExtWarn<
   "%select{class template|class template partial|function template|member "
   "function|static data member|member class}0 specialization of %1 must "
-  "originally be declared in the global scope; accepted as a C++0x extension">,
-  InGroup<CXX0x>;
+  "originally be declared in the global scope; accepted as a C++11 extension">,
+  InGroup<CXX11>;
 def err_template_spec_decl_out_of_scope : Error<
   "%select{class template|class template partial|function template|member "
   "function|static data member|member class}0 specialization of %1 must "
@@ -1904,8 +2201,8 @@ def err_template_spec_decl_out_of_scope : Error<
 def ext_template_spec_decl_out_of_scope : ExtWarn<
   "%select{class template|class template partial|function template|member "
   "function|static data member|member class}0 specialization of %1 must "
-  "originally be declared in namespace %2; accepted as a C++0x extension">,
-  InGroup<CXX0x>;
+  "originally be declared in namespace %2; accepted as a C++11 extension">,
+  InGroup<CXX11>;
 def err_template_spec_redecl_out_of_scope : Error<
   "%select{class template|class template partial|function template|member "
   "function|static data member|member class}0 specialization of %1 not in a "
@@ -1931,6 +2228,9 @@ def err_not_class_template_specialization : Error<
   "parameter}0">;
 def err_function_specialization_in_class : Error<
   "cannot specialize a function %0 within class scope">;
+def ext_function_specialization_in_class : ExtWarn<
+  "explicit specialization of %0 within class scope is a Microsoft extension">,
+  InGroup<Microsoft>;
 def ext_explicit_specialization_storage_class : ExtWarn<
   "explicit specialization cannot have a storage class">;
 def err_explicit_specialization_inconsistent_storage_class : Error<
@@ -2068,8 +2368,8 @@ def note_previous_explicit_instantiation : Note<
     "previous explicit instantiation is here">;
 def ext_explicit_instantiation_after_specialization : Extension<
     "explicit instantiation of %0 that occurs after an explicit "
-    "specialization will be ignored (C++0x extension)">,
-    InGroup<CXX0x>;
+    "specialization will be ignored (C++11 extension)">,
+    InGroup<CXX11>;
 def note_previous_template_specialization : Note<
     "previous template specialization is here">;
 def err_explicit_instantiation_enum : Error<
@@ -2086,10 +2386,10 @@ def err_explicit_instantiation_must_be_global : Error<
   "explicit instantiation of %0 must occur at global scope">;
 def warn_explicit_instantiation_out_of_scope_0x : Warning<
   "explicit instantiation of %0 not in a namespace enclosing %1">, 
-  InGroup<DiagGroup<"-Wc++0x-compat"> >;
+  InGroup<CXX11Compat>;
 def warn_explicit_instantiation_must_be_global_0x : Warning<
   "explicit instantiation of %0 must occur at global scope">, 
-  InGroup<DiagGroup<"-Wc++0x-compat"> >;
+  InGroup<CXX11Compat>;
   
 def err_explicit_instantiation_requires_name : Error<
   "explicit instantiation declaration requires a name">;
@@ -2114,6 +2414,8 @@ def note_explicit_instantiation_candidate : Note<
   "explicit instantiation candidate function template here %0">;
 def err_explicit_instantiation_inline : Error<
   "explicit instantiation cannot be 'inline'">;
+def err_explicit_instantiation_constexpr : Error<
+  "explicit instantiation cannot be 'constexpr'">;
 def ext_explicit_instantiation_without_qualified_id : Extension<
   "qualifier in explicit instantiation of %q0 requires a template-id "
   "(a typedef is not permitted)">;
@@ -2121,7 +2423,7 @@ def err_explicit_instantiation_unqualified_wrong_namespace : Error<
   "explicit instantiation of %q0 must occur in %1">;
 def warn_explicit_instantiation_unqualified_wrong_namespace_0x : Warning<
   "explicit instantiation of %q0 must occur in %1">, 
-  InGroup<DiagGroup<"c++0x-compat"> >;
+  InGroup<CXX11Compat>;
 def err_explicit_instantiation_undefined_member : Error<
   "explicit instantiation of undefined %select{member class|member function|"
   "static data member}0 %1 of class template %2">;
@@ -2142,9 +2444,10 @@ def note_typename_refers_here : Note<
 def err_typename_missing : Error<
   "missing 'typename' prior to dependent type name '%0%1'">;
 def warn_typename_missing : ExtWarn<
-  "missing 'typename' prior to dependent type name '%0%1'">;
+  "missing 'typename' prior to dependent type name '%0%1'">,
+  InGroup<DiagGroup<"typename-missing">>;
 def ext_typename_outside_of_template : ExtWarn<
-  "'typename' occurs outside of a template">, InGroup<CXX0x>;
+  "'typename' occurs outside of a template">, InGroup<CXX11>;
 def err_typename_refers_to_using_value_decl : Error<
   "typename specifier refers to a dependent using declaration for a value "
   "%0 in %1">;
@@ -2162,7 +2465,7 @@ def note_referenced_class_template : Error<
 def err_template_kw_missing : Error<
   "missing 'template' keyword prior to dependent template name '%0%1'">;
 def ext_template_outside_of_template : ExtWarn<
-  "'template' keyword outside of a template">, InGroup<CXX0x>;
+  "'template' keyword outside of a template">, InGroup<CXX11>;
 
 def err_non_type_template_in_nested_name_specifier : Error<
   "qualified name refers into a specialization of function template '%0'">;
@@ -2172,7 +2475,7 @@ def note_template_declared_here : Note<
   "%select{function template|class template|type alias template|template template parameter}0 "
   "%1 declared here">;
 
-// C++0x Variadic Templates
+// C++11 Variadic Templates
 def err_template_param_pack_default_arg : Error<
   "template parameter pack cannot have a default argument">;
 def err_template_param_pack_must_be_last_template_parameter : Error<
@@ -2240,6 +2543,9 @@ def err_unexpected_typedef : Error<
 def err_unexpected_namespace : Error<
   "unexpected namespace name %0: expected expression">;
 def err_undeclared_var_use : Error<"use of undeclared identifier %0">;
+def warn_found_via_dependent_bases_lookup : ExtWarn<"use of identifier %0 "
+   "found via unqualified lookup into dependent bases of class templates is a "
+   "Microsoft extension">, InGroup<Microsoft>;
 def note_dependent_var_use : Note<"must qualify identifier to find this "
     "declaration in dependent base class">;
 def err_not_found_by_two_phase_lookup : Error<"call to function %0 that is neither "
@@ -2265,9 +2571,11 @@ def note_unavailable_here : Note<
   "%select{declaration|function}0 has been explicitly marked "
   "%select{unavailable|deleted|deprecated}1 here">;
 def warn_not_enough_argument : Warning<
-  "not enough variable arguments in %0 declaration to fit a sentinel">;
+  "not enough variable arguments in %0 declaration to fit a sentinel">,
+  InGroup<Sentinel>;
 def warn_missing_sentinel : Warning <
-  "missing sentinel in %select{function call|method dispatch|block call}0">;
+  "missing sentinel in %select{function call|method dispatch|block call}0">,
+  InGroup<Sentinel>;
 def note_sentinel_here : Note<
   "%select{function|method|block}0 has been explicitly marked sentinel here">;
 def warn_missing_prototype : Warning<
@@ -2410,6 +2718,8 @@ def err_at_least_one_initializer_needed_to_size_array : Error<
 def err_array_size_non_int : Error<"size of array has non-integer type %0">;
 def err_init_element_not_constant : Error<
   "initializer element is not a compile-time constant">;
+def err_local_cant_init : Error<
+  "'__local' variable cannot have an initializer">;
 def err_block_extern_cant_init : Error<
   "'extern' variable cannot have an initializer">;
 def warn_extern_init : Warning<"'extern' variable has an initializer">;
@@ -2436,10 +2746,28 @@ def warn_braces_around_scalar_init : Warning<
   "braces around scalar initializer">;
 def warn_many_braces_around_scalar_init : ExtWarn<
   "too many braces around scalar initializer">;
+def ext_complex_component_init : Extension<
+  "complex initialization specifying real and imaginary components "
+  "is an extension">, InGroup<DiagGroup<"complex-component-init">>;
 def err_empty_scalar_initializer : Error<"scalar initializer cannot be empty">;
 def err_illegal_initializer : Error<
   "illegal initializer (only variables can be initialized)">;
 def err_illegal_initializer_type : Error<"illegal initializer type %0">;
+def err_init_list_variable_narrowing : Error<
+  "non-constant-expression cannot be narrowed from type %0 to %1 in "
+  "initializer list">;
+def err_init_list_constant_narrowing : Error<
+  "constant expression evaluates to %0 which cannot be narrowed to type %1">;
+def warn_init_list_variable_narrowing : Warning<
+  "non-constant-expression cannot be narrowed from type %0 to %1 in "
+  "initializer list in C++11">,
+  InGroup<CXX11Narrowing>, DefaultIgnore;
+def warn_init_list_constant_narrowing : Warning<
+  "constant expression evaluates to %0 which cannot be narrowed to type %1 in "
+  "C++11">,
+  InGroup<CXX11Narrowing>, DefaultIgnore;
+def note_init_list_narrowing_override : Note<
+  "override this message by inserting an explicit cast">;
 def err_init_objc_class : Error<
   "cannot initialize Objective-C class type %0">;
 def err_implicit_empty_initializer : Error<
@@ -2474,6 +2802,8 @@ def warn_unused_label : Warning<"unused label %0">,
   InGroup<UnusedLabel>, DefaultIgnore;
 
 def err_goto_into_protected_scope : Error<"goto into protected scope">;
+def warn_goto_into_protected_scope : ExtWarn<"goto into protected scope">,
+  InGroup<Microsoft>;
 def err_switch_into_protected_scope : Error<
   "switch case is in protected scope">;
 def err_indirect_goto_without_addrlabel : Error<
@@ -2563,8 +2893,8 @@ def ext_flexible_array_in_struct : Extension<
   "%0 may not be nested in a struct due to flexible array member">;
 def ext_flexible_array_in_array : Extension<
   "%0 may not be used as an array element due to flexible array member">;
-def err_flexible_array_init_nonempty : Error<
-  "non-empty initialization of flexible array member inside subobject">;
+def err_flexible_array_init : Error<
+  "initialization of flexible array member is not allowed">;
 def ext_flexible_array_empty_aggregate_ms : Extension<
   "flexible array member %0 in otherwise empty %select{struct|class}1 "
   "is a Microsoft extension">, InGroup<Microsoft>;
@@ -2603,7 +2933,7 @@ def err_arc_objc_object_in_struct : Error<
   "ARC forbids Objective-C objects in structs or unions">;
 def err_arc_objc_property_default_assign_on_object : Error<
   "ARC forbids synthesizing a property of an Objective-C object "
-  "with unspecified storage attribute">;
+  "with unspecified ownership or storage attribute">;
 def err_arc_illegal_selector : Error<
   "ARC forbids use of %0 in a @selector">;
 def err_arc_illegal_method_def : Error<
@@ -2672,10 +3002,11 @@ def err_arc_method_not_found : Error<
 def err_arc_receiver_forward_class : Error<
   "receiver %0 for class message is a forward declaration">;
 def err_arc_may_not_respond : Error<
-  "receiver type %0 for instance message does not declare a method with "
-  "selector %1">;
+  "no visible @interface for %0 declares the selector %1">;
 def err_arc_receiver_forward_instance : Error<
   "receiver type %0 for instance message is a forward declaration">;
+def err_arc_collection_forward : Error<
+  "collection expression type %0 is a forward declaration">;
 def err_arc_multiple_method_decl : Error< 
   "multiple methods named %0 found with mismatched result, "
   "parameter type or attributes">;
@@ -2696,9 +3027,10 @@ def err_arc_strong_property_ownership : Error<
   "existing ivar %1 for strong property %0 may not be "
   "%select{|__unsafe_unretained||__weak}2">;
 def err_arc_assign_property_ownership : Error<
-  "existing ivar %1 for unsafe_unretained property %0 must be __unsafe_unretained">;
+  "existing ivar %1 for property %0 with %select{unsafe_unretained| assign}2 "
+  "attribute must be __unsafe_unretained">;
 def err_arc_inconsistent_property_ownership : Error<
-  "%select{strong|weak|unsafe_unretained}1 property %0 may not also be "
+  "%select{|unsafe_unretained|strong|weak}1 property %0 may not also be "
   "declared %select{|__unsafe_unretained|__strong|__weak|__autoreleasing}2">;
 def err_arc_atomic_ownership : Error<
   "cannot perform atomic operation on a pointer to type %0: type has "
@@ -2759,6 +3091,10 @@ def err_block_with_return_type_requires_args : Error<
   "block with explicit return type requires argument list">;
 def err_func_def_incomplete_result : Error<
   "incomplete result type %0 in function definition">;
+def err_atomic_specifier_bad_type : Error<
+  "_Atomic cannot be applied to "
+  "%select{incomplete |array |function |reference |atomic |qualified |}0type "
+  "%1 %select{||||||which is not trivially copyable}0">;
 
 // Expressions.
 def ext_sizeof_function_type : Extension<
@@ -2793,8 +3129,10 @@ def warn_floatingpoint_eq : Warning<
 
 def warn_division_by_zero : Warning<"division by zero is undefined">;
 def warn_remainder_by_zero : Warning<"remainder by zero is undefined">;
-def warn_shift_negative : Warning<"shift count is negative">;
-def warn_shift_gt_typewidth : Warning<"shift count >= width of type">;
+def warn_shift_negative : Warning<"shift count is negative">,
+  InGroup<DiagGroup<"shift-count-negative">>;
+def warn_shift_gt_typewidth : Warning<"shift count >= width of type">,
+  InGroup<DiagGroup<"shift-count-overflow">>;
 def warn_shift_result_gt_typewidth : Warning<
   "signed shift result (%0) requires %1 bits to represent, but %2 only has "
   "%3 bits">, InGroup<DiagGroup<"shift-overflow">>;
@@ -2820,8 +3158,12 @@ def note_precedence_conditional_silence : Note<
   "place parentheses around the '%0' expression to silence this warning">;
 
 def warn_logical_instead_of_bitwise : Warning<
-  "use of logical %0 with constant operand; switch to bitwise %1 or "
-  "remove constant">, InGroup<DiagGroup<"constant-logical-operand">>;
+  "use of logical '%0' with constant operand">,
+  InGroup<DiagGroup<"constant-logical-operand">>;
+def note_logical_instead_of_bitwise_change_operator : Note<
+  "use '%0' for a bitwise operation">;
+def note_logical_instead_of_bitwise_remove_constant : Note<
+  "remove constant to silence this warning">;
 
 def warn_bitwise_and_in_bitwise_or : Warning<
   "'&' within '|'">, InGroup<BitwiseOpParentheses>;
@@ -2883,10 +3225,8 @@ def err_typecheck_member_reference_type : Error<
 def err_typecheck_member_reference_unknown : Error<
   "cannot refer to member %0 in %1 with '%select{.|->}2'">;
 def err_member_reference_needs_call : Error<
-  "base of member reference is %select{a function|an overloaded function}0; "
-  "perhaps you meant to call it%select{| with no arguments}1?">;
-def note_member_ref_possible_intended_overload : Note<
-  "possibly valid overload here">;
+  "base of member reference is a function; perhaps you meant to call "
+  "it%select{| with no arguments}?">;
 def warn_subscript_is_char : Warning<"array subscript is of type 'char'">,
   InGroup<CharSubscript>, DefaultIgnore;
 
@@ -2904,6 +3244,9 @@ def err_member_def_undefined_record : Error<
   "out-of-line definition of %0 from class %1 without definition">;
 def err_member_def_does_not_match : Error<
   "out-of-line definition of %0 does not match any declaration in %1">;
+def err_member_def_does_not_match_suggest : Error<
+  "out-of-line definition of %0 does not match any declaration in %1; "
+  "did you mean %2">;
 def err_member_def_does_not_match_ret_type : Error<
   "out-of-line definition of %q0 differs from the declaration in the return type">;
 def err_nonstatic_member_out_of_line : Error<
@@ -2922,6 +3265,12 @@ def warn_member_extra_qualification : Warning<
 def err_member_qualification : Error<
   "non-friend class member %0 cannot have a qualified name">;  
 def note_member_def_close_match : Note<"member declaration nearly matches">;
+def note_member_def_close_const_match : Note<
+  "member declaration does not match because "
+  "it %select{is|is not}0 const qualified">;
+def note_member_def_close_param_match : Note<
+  "type of %ordinal0 parameter of member declaration does not match "
+  "definition (%1 vs %2)">;
 def err_typecheck_ivar_variable_size : Error<
   "instance variables must have a constant size">;
 def err_ivar_reference_type : Error<
@@ -2938,7 +3287,8 @@ def err_typecheck_pointer_arith_void_type : Error<
 def err_typecheck_decl_incomplete_type : Error<
   "variable has incomplete type %0">;
 def ext_typecheck_decl_incomplete_type : ExtWarn<
-  "tentative definition of variable with internal linkage has incomplete non-array type %0">;
+  "tentative definition of variable with internal linkage has incomplete non-array type %0">,
+  InGroup<DiagGroup<"tentative-definition-incomplete-type">>;
 def err_tentative_def_incomplete_type : Error<
   "tentative definition has type %0 that is never completed">;
 def err_tentative_def_incomplete_type_arr : Error<
@@ -2969,7 +3319,8 @@ def warn_standalone_specifier : Warning<"'%0' ignored on this declaration">;
 def err_typecheck_sclass_func : Error<"illegal storage class on function">;
 def err_static_block_func : Error<
   "function declared in block scope cannot have 'static' storage class">;
-def err_typecheck_address_of : Error<"address of %0 requested">;
+def err_typecheck_address_of : Error<"address of %select{bit-field"
+  "|vector element|property expression|register variable}0 requested">;
 def ext_typecheck_addrof_void : Extension<
   "ISO C forbids taking the address of an expression of type 'void'">;
 def err_unqualified_pointer_member_function : Error<
@@ -3039,9 +3390,6 @@ def err_stmtexpr_file_scope : Error<
 def warn_mixed_sign_comparison : Warning<
   "comparison of integers of different signs: %0 and %1">,
   InGroup<SignCompare>, DefaultIgnore;
-def warn_mixed_sign_conditional : Warning<
-  "operands of ? are integers of different signs: %0 and %1">,
-  InGroup<SignCompare>, DefaultIgnore;
 def warn_lunsigned_always_true_comparison : Warning<
   "comparison of unsigned%select{| enum}2 expression %0 is always %1">,
   InGroup<TautologicalCompare>;
@@ -3054,6 +3402,10 @@ def warn_comparison_of_mixed_enum_types : Warning<
 def warn_null_in_arithmetic_operation : Warning<
   "use of NULL in arithmetic operation">,
   InGroup<DiagGroup<"null-arithmetic">>;
+def warn_null_in_comparison_operation : Warning<
+  "comparison between NULL and non-pointer "
+  "%select{(%1 and NULL)|(NULL and %1)}0">,
+  InGroup<DiagGroup<"null-arithmetic">>;
 
 def err_invalid_this_use : Error<
   "invalid use of 'this' outside of a nonstatic member function">;
@@ -3140,7 +3492,9 @@ def error_nosetter_property_assignment : Error<
   "setter method is needed to assign to object using property" " assignment syntax">;
 def error_no_subobject_property_setting : Error<
   "expression is not assignable">;
-
+def err_qualified_objc_access : Error<
+  "%select{property|ivar}0 access cannot be qualified with '%1'">;
+  
 def ext_freestanding_complex : Extension<
   "complex numbers are an extension in a freestanding C99 implementation">;
 
@@ -3205,10 +3559,11 @@ def err_incomplete_type_objc_at_encode : Error<
 
 def warn_setter_getter_impl_required : Warning<
   "property %0 requires method %1 to be defined - "
-  "use @synthesize, @dynamic or provide a method implementation">;
+  "use @synthesize, @dynamic or provide a method implementation "
+  "in this class implementation">;
 def warn_setter_getter_impl_required_in_category : Warning<
   "property %0 requires method %1 to be defined - "
-  "use @dynamic or provide a method implementation in category">;
+  "use @dynamic or provide a method implementation in this category">;
 def note_property_impl_required : Note<
   "implementation is here">;
 def note_parameter_named_here : Note<
@@ -3326,8 +3681,8 @@ def err_array_size_ambiguous_conversion : Error<
   "enumeration type">;
 def ext_array_size_conversion : Extension<
   "implicit conversion from array size expression of type %0 to "
-  "%select{integral|enumeration}1 type %2 is a C++0x extension">,
-  InGroup<CXX0x>;
+  "%select{integral|enumeration}1 type %2 is a C++11 extension">,
+  InGroup<CXX11>;
 def err_address_space_qualified_new : Error<
   "'new' cannot allocate objects of type %0 in address space '%1'">;
 def err_address_space_qualified_delete : Error<
@@ -3343,7 +3698,7 @@ def err_ambiguous_delete_operand : Error<"ambiguous conversion of delete "
                                          "expression of type %0 to a pointer">;
 def warn_delete_incomplete : Warning<
   "deleting pointer to incomplete type %0 may cause undefined behaviour">;
-def err_delete_incomplete_class_type : Warning<
+def err_delete_incomplete_class_type : Error<
   "deleting incomplete class type %0; no conversions to pointer type">;
 def warn_delete_array_type : Warning<
   "'delete' applied to a pointer-to-array type %0 treated as delete[]">;
@@ -3385,6 +3740,9 @@ def warn_non_virtual_dtor : Warning<
 def warn_delete_non_virtual_dtor : Warning<
   "delete called on %0 that has virtual functions but non-virtual destructor">,
   InGroup<DeleteNonVirtualDtor>, DefaultIgnore;
+def warn_delete_abstract_non_virtual_dtor : Warning<
+  "delete called on %0 that is abstract but has non-virtual destructor">,
+  InGroup<DeleteNonVirtualDtor>;
 def warn_overloaded_virtual : Warning<
   "%q0 hides overloaded virtual %select{function|functions}1">,
   InGroup<OverloadedVirtual>, DefaultIgnore;
@@ -3458,8 +3816,6 @@ def err_not_tag_in_scope : Error<
 
 def err_cannot_form_pointer_to_member_of_reference_type : Error<
   "cannot form a pointer-to-member to member %0 of reference type %1">;
-def err_invalid_use_of_bound_member_func : Error<
-  "a bound member function may only be called">;
 def err_incomplete_object_call : Error<
   "incomplete type in call to object of type %0">;
 def err_incomplete_pointer_to_member_return : Error<
@@ -3480,7 +3836,7 @@ def note_condition_assign_silence : Note<
   "place parentheses around the assignment to silence this warning">;
 
 def warn_equality_with_extra_parens : Warning<"equality comparison with "
-  "extraneous parentheses">, InGroup<Parentheses>;
+  "extraneous parentheses">, InGroup<ParenthesesOnEquality>;
 def note_equality_comparison_to_assign : Note<
   "use '=' to turn this equality comparison into an assignment">;
 def note_equality_comparison_silence : Note<
@@ -3501,7 +3857,11 @@ def err_typecheck_convert_incompatible : Error<
   "%select{from incompatible type|to parameter of incompatible type|"
   "from a function with incompatible result type|to incompatible type|"
   "with an expression of incompatible type|to parameter of incompatible type|"
-  "to incompatible type}2 %1">;
+  "to incompatible type}2 %1; "
+  "%select{|dereference with *|"
+  "take the address with &|"
+  "remove *|"
+  "remove &}3">;
 def warn_incompatible_qualified_id : Warning<
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
@@ -3514,13 +3874,21 @@ def ext_typecheck_convert_pointer_int : ExtWarn<
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
   "%select{from|to parameter of type|from a function with result type|to type|"
-  "with an expression of type|to parameter of type|to type}2 %1">;
+  "with an expression of type|to parameter of type|to type}2 %1; "
+  "%select{|dereference with *|"
+  "take the address with &|"
+  "remove *|"
+  "remove &}3">;
 def ext_typecheck_convert_int_pointer : ExtWarn<
   "incompatible integer to pointer conversion "
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
   "%select{from|to parameter of type|from a function with result type|to type|"
-  "with an expression of type|to parameter of type|to type}2 %1">;
+  "with an expression of type|to parameter of type|to type}2 %1; "
+  "%select{|dereference with *|"
+  "take the address with &|"
+  "remove *|"
+  "remove &}3">;
 def ext_typecheck_convert_pointer_void_func : Extension<
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
@@ -3539,20 +3907,26 @@ def ext_typecheck_convert_incompatible_pointer : ExtWarn<
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
   "%select{from|to parameter of type|from a function with result type|to type|"
-  "with an expression of type|to parameter of type|to type}2 %1">,
-  InGroup<DiagGroup<"incompatible-pointer-types">>;
+  "with an expression of type|to parameter of type|to type}2 %1"
+  "%select{|dereference with *|"
+  "take the address with &|"
+  "remove *|"
+  "remove &}3">,
+  InGroup<IncompatiblePointerTypes>;
 def ext_typecheck_convert_discards_qualifiers : ExtWarn<
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
   "%select{from|to parameter of type|from a function with result type|to type|"
   "with an expression of type|to parameter of type|to type}2 %1 discards "
-  "qualifiers">;
+  "qualifiers">,
+  InGroup<IncompatiblePointerTypes>;
 def ext_nested_pointer_qualifier_mismatch : ExtWarn<
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
   " %0 "
   "%select{from|to parameter of type|from a function with result type|to type|"
   "with an expression of type|to parameter of type|to type}2 %1 discards "
-  "qualifiers in nested pointer types">;
+  "qualifiers in nested pointer types">,
+  InGroup<IncompatiblePointerTypes>;
 def warn_incompatible_vectors : Warning<
   "incompatible vector types "
   "%select{assigning to|passing|returning|converting|initializing|sending|casting}2"
@@ -3621,19 +3995,23 @@ def note_function_with_incomplete_return_type_declared_here : Note<
 def err_call_incomplete_argument : Error<
   "argument type %0 is incomplete">;
 def err_typecheck_call_too_few_args : Error<
-  "too few arguments to %select{function|block|method}0 call, "
+  "too few %select{|||execution configuration }0arguments to "
+  "%select{function|block|method|kernel function}0 call, "
   "expected %1, have %2">;
 def err_typecheck_call_too_few_args_at_least : Error<
-  "too few arguments to %select{function|block|method}0 call, "
+  "too few %select{|||execution configuration }0arguments to "
+  "%select{function|block|method|kernel function}0 call, "
   "expected at least %1, have %2">;
 def err_typecheck_call_too_many_args : Error<
-  "too many arguments to %select{function|block|method}0 call, "
+  "too many %select{|||execution configuration }0arguments to "
+  "%select{function|block|method|kernel function}0 call, "
   "expected %1, have %2">;
-def note_typecheck_call_too_many_args : Note<
-  "%0 declared here">;
 def err_typecheck_call_too_many_args_at_most : Error<
-  "too many arguments to %select{function|block|method}0 call, "
+  "too many %select{|||execution configuration }0arguments to "
+  "%select{function|block|method|kernel function}0 call, "
   "expected at most %1, have %2">;
+def note_callee_decl : Note<
+  "%0 declared here">;
 def warn_call_wrong_number_of_arguments : Warning<
   "too %select{few|many}0 arguments in call to %1">;
 def err_atomic_builtin_must_be_pointer : Error<
@@ -3644,6 +4022,15 @@ def err_atomic_builtin_must_be_pointer_intptr : Error<
 def err_atomic_builtin_pointer_size : Error<
   "first argument to atomic builtin must be a pointer to 1,2,4,8 or 16 byte "
   "type (%0 invalid)">;
+def err_atomic_op_needs_atomic : Error<
+  "first argument to atomic operation must be a pointer to _Atomic "
+  "type (%0 invalid)">;
+def err_atomic_op_needs_atomic_int_or_ptr : Error<
+  "first argument to atomic operation must be a pointer to atomic "
+  "integer or pointer (%0 invalid)">;
+def err_atomic_op_logical_needs_atomic_int : Error<
+  "first argument to logical atomic operation must be a pointer to atomic "
+  "integer (%0 invalid)">;
 
 def err_deleted_function_use : Error<"attempt to use a deleted function">;
 
@@ -3653,6 +4040,11 @@ def err_config_scalar_return : Error<
   "CUDA special function 'cudaConfigureCall' must have scalar return type">;
 def err_kern_call_not_global_function : Error<
   "kernel call to non-global function %0">;
+def err_global_call_not_config : Error<
+  "call to global function %0 not configured">;
+def err_ref_bad_target : Error<
+  "reference to %select{__device__|__global__|__host__|__host__ __device__}0 "
+  "function %1 in %select{__device__|__global__|__host__|__host__ __device__}2 function">;
 
 
 def err_cannot_pass_objc_interface_to_vararg : Error<
@@ -3696,9 +4088,11 @@ def ext_typecheck_cond_incompatible_operands_nonstandard : ExtWarn<
 def err_cast_selector_expr : Error<
   "cannot type cast @selector expression">;
 def warn_typecheck_cond_incompatible_pointers : ExtWarn<
-  "pointer type mismatch (%0 and %1)">;
+  "pointer type mismatch (%0 and %1)">,
+  InGroup<DiagGroup<"pointer-type-mismatch">>;
 def warn_typecheck_cond_pointer_integer_mismatch : ExtWarn<
-  "pointer/integer type mismatch in conditional expression (%0 and %1)">;
+  "pointer/integer type mismatch in conditional expression (%0 and %1)">,
+  InGroup<DiagGroup<"conditional-type-mismatch">>;
 def err_typecheck_choose_expr_requires_constant : Error<
   "'__builtin_choose_expr' requires a constant expression">;
 def ext_typecheck_expression_not_constant_but_accepted : Extension<
@@ -3715,6 +4109,14 @@ def warn_unused_property_expr : Warning<
 def warn_unused_call : Warning<
   "ignoring return value of function declared with %0 attribute">,
   InGroup<UnusedValue>;
+def warn_unused_result : Warning<
+  "ignoring return value of function declared with warn_unused_result "
+  "attribute">, InGroup<DiagGroup<"unused-result">>;
+def warn_unused_comparison : Warning<
+  "%select{equality|inequality}0 comparison result unused">,
+  InGroup<UnusedComparison>;
+def note_inequality_comparison_to_or_assign : Note<
+  "use '|=' to turn this inequality comparison into an or-assignment">;
 
 def err_incomplete_type_used_in_type_trait_expr : Error<
   "incomplete type %0 used in type trait expression">;
@@ -3798,11 +4200,18 @@ def err_not_direct_base_or_virtual : Error<
 
 def err_in_class_initializer_non_const : Error<
   "non-const static data member must be initialized out of line">;
+def err_in_class_initializer_volatile : Error<
+  "static const volatile data member must be initialized out of line">;
 def err_in_class_initializer_bad_type : Error<
   "static data member of type %0 must be initialized out of line">;
 def ext_in_class_initializer_float_type : ExtWarn<
-  "in-class initializer for static data member of type %0 "
-  "is a C++0x extension">, InGroup<CXX0xStaticNonIntegralInitializer>;
+  "in-class initializer for static data member of type %0 is a GNU extension">,
+  InGroup<GNU>;
+def note_in_class_initializer_float_type_constexpr : Note<
+  "use 'constexpr' specifier to silence this warning">;
+def err_in_class_initializer_literal_type : Error<
+  "in-class initializer for static data member of type %0 requires "
+  "'constexpr' specifier">;
 def err_in_class_initializer_non_constant : Error<
   "in-class initializer is not a constant expression">;
 
@@ -3941,7 +4350,13 @@ def err_literal_operator_outside_namespace : Error<
 // FIXME: This diagnostic sucks
 def err_literal_operator_params : Error<
   "parameter declaration for literal operator %0 is not valid">;
-
+def warn_user_literal_hexfloat : Warning<
+  "user-defined literal with suffix '%0' is preempted by C99 hexfloat "
+  "extension">, InGroup<UserDefinedLiterals>;
+def warn_user_literal_reserved : Warning<
+  "user-defined literals not starting with '_' are reserved by the "
+  "implementation">, InGroup<UserDefinedLiterals>;
+  
 // C++ conversion functions
 def err_conv_function_not_member : Error<
   "conversion function must be a non-static member function">;
@@ -3969,11 +4384,11 @@ def warn_conv_to_void_not_used : Warning<
 def warn_not_compound_assign : Warning<
   "use of unary operator that may be intended as compound assignment (%0=)">;
 
-// C++0x explicit conversion operators
+// C++11 explicit conversion operators
 def warn_explicit_conversion_functions : Warning<
-  "explicit conversion functions are a C++0x extension">, InGroup<CXX0x>;
+  "explicit conversion functions are a C++11 extension">, InGroup<CXX11>;
 
-// C++0x defaulted functions
+// C++11 defaulted functions
 def err_defaulted_default_ctor_params : Error<
   "an explicitly-defaulted default constructor must have no parameters">;
 def err_defaulted_copy_ctor_params : Error<
@@ -4000,8 +4415,34 @@ def err_defaulted_copy_assign_const_param : Error<
   "the parameter for this explicitly-defaulted copy assignment operator is "
   "const, but a member or base requires it to be non-const">;
 def err_defaulted_copy_assign_quals : Error<
-  "an explicitly-defaulted copy assignment operator may not have 'const' "
-  "or 'volatile' qualifiers">;
+  "an explicitly-defaulted copy assignment operator may not have 'const', "
+  "'constexpr' or 'volatile' qualifiers">;
+def err_defaulted_move_ctor_params : Error<
+  "an explicitly-defaulted move constructor must have exactly one parameter">;
+def err_defaulted_move_ctor_volatile_param : Error<
+  "the parameter for an explicitly-defaulted move constructor may not be "
+  "volatile">;
+def err_defaulted_move_ctor_const_param : Error<
+  "the parameter for an explicitly-defaulted move constructor may not be "
+  "const">;
+def err_defaulted_move_assign_params : Error<
+  "an explicitly-defaulted move assignment operator must have exactly one "
+  "parameter">;
+def err_defaulted_move_assign_return_type : Error<
+  "an explicitly-defaulted move assignment operator must return an unqualified "
+  "lvalue reference to its class type">;
+def err_defaulted_move_assign_not_ref : Error<
+  "the parameter for an explicitly-defaulted move assignment operator must be an "
+  "rvalue reference type">;
+def err_defaulted_move_assign_volatile_param : Error<
+  "the parameter for an explicitly-defaulted move assignment operator may not "
+  "be volatile">;
+def err_defaulted_move_assign_const_param : Error<
+  "the parameter for an explicitly-defaulted move assignment operator may not "
+  "be const">;
+def err_defaulted_move_assign_quals : Error<
+  "an explicitly-defaulted move assignment operator may not have 'const', "
+  "'constexpr' or 'volatile' qualifiers">;
 def err_incorrect_defaulted_exception_spec : Error<
   "exception specification of explicitly defaulted %select{default constructor|"
   "copy constructor|move constructor|copy assignment operator|move assignment "
@@ -4011,15 +4452,20 @@ def err_out_of_line_default_deletes : Error<
   "defaulting this %select{default constructor|copy constructor|move "
   "constructor|copy assignment operator|move assignment operator|destructor}0 "
   "would delete it after its first declaration">;
-def err_defaulted_move_unsupported : Error<
-  "defaulting move functions not yet supported">;
 
+def warn_ptr_arith_precedes_bounds : Warning<
+  "the pointer decremented by %0 refers before the beginning of the array">,
+  InGroup<DiagGroup<"array-bounds-pointer-arithmetic">>, DefaultIgnore;
+def warn_ptr_arith_exceeds_bounds : Warning<
+  "the pointer incremented by %0 refers past the end of the array (that "
+  "contains %1 element%s2)">,
+  InGroup<DiagGroup<"array-bounds-pointer-arithmetic">>, DefaultIgnore;
 def warn_array_index_precedes_bounds : Warning<
   "array index of '%0' indexes before the beginning of the array">,
   InGroup<DiagGroup<"array-bounds">>;
 def warn_array_index_exceeds_bounds : Warning<
-  "array index of '%0' indexes past the end of an array (that contains %1 elements)">,
-  InGroup<DiagGroup<"array-bounds">>;
+  "array index of '%0' indexes past the end of an array (that contains %1 "
+  "element%s2)">, InGroup<DiagGroup<"array-bounds">>;
 def note_array_index_out_of_bounds : Note<
   "array %0 declared here">;
 
@@ -4031,7 +4477,7 @@ def warn_printf_insufficient_data_args : Warning<
 def warn_printf_data_arg_not_used : Warning<
   "data argument not used by format string">, InGroup<FormatExtraArgs>;
 def warn_format_invalid_conversion : Warning<
-  "invalid conversion specifier '%0'">, InGroup<Format>;
+  "invalid conversion specifier '%0'">, InGroup<FormatInvalidSpecifier>;
 def warn_printf_incomplete_specifier : Warning<
   "incomplete format specifier">, InGroup<Format>;
 def warn_missing_format_string : Warning<
@@ -4086,20 +4532,38 @@ def warn_scanf_scanlist_incomplete : Warning<
   
 // CHECK: returning address/reference of stack memory
 def warn_ret_stack_addr : Warning<
-  "address of stack memory associated with local variable %0 returned">;
+  "address of stack memory associated with local variable %0 returned">,
+  InGroup<DiagGroup<"return-stack-address">>;
 def warn_ret_stack_ref : Warning<
-  "reference to stack memory associated with local variable %0 returned">;
+  "reference to stack memory associated with local variable %0 returned">,
+  InGroup<DiagGroup<"return-stack-address">>;
 def warn_ret_local_temp_addr : Warning<
-  "returning address of local temporary object">;
+  "returning address of local temporary object">,
+  InGroup<DiagGroup<"return-stack-address">>;
 def warn_ret_local_temp_ref : Warning<
-  "returning reference to local temporary object">;
+  "returning reference to local temporary object">,
+  InGroup<DiagGroup<"return-stack-address">>;
 def warn_ret_addr_label : Warning<
-  "returning address of label, which is local">;
+  "returning address of label, which is local">,
+  InGroup<DiagGroup<"return-stack-address">>;
 def err_ret_local_block : Error<
   "returning block that lives on the local stack">;
 def note_ref_var_local_bind : Note<
   "binding reference variable %0 here">;
 
+// Check for initializing a member variable with the address or a reference to
+// a constructor parameter.
+def warn_bind_ref_member_to_parameter : Warning<
+  "binding reference member %0 to stack allocated parameter %1">,
+  InGroup<DiagGroup<"dangling-field">>;
+def warn_init_ptr_member_to_parameter_addr : Warning<
+  "initializing pointer member %0 with the stack address of parameter %1">,
+  InGroup<DiagGroup<"dangling-field">>;
+def warn_bind_ref_member_to_temporary : Warning<
+  "binding reference member %0 to a temporary value">,
+  InGroup<DiagGroup<"dangling-field">>;
+def note_ref_or_ptr_member_declared_here : Note<
+  "%select{reference|pointer}0 member declared here">;
 
 // For non-floating point, expressions of the form x == x or x != x
 // should result in a warning, since these always evaluate to a constant.
@@ -4110,7 +4574,8 @@ def warn_comparison_always : Warning<
 
 def warn_stringcompare : Warning<
   "result of comparison against %select{a string literal|@encode}0 is "
-  "unspecified (use strncmp instead)">;
+  "unspecified (use strncmp instead)">,
+  InGroup<DiagGroup<"string-compare">>;
 
 // Generic selections.
 def err_assoc_type_incomplete : Error<
@@ -4138,13 +4603,18 @@ def err_return_in_block_expression : Error<
 def err_block_returning_array_function : Error<
   "block cannot return %select{array|function}0 type %1">;
 
+// Builtin annotation string.
+def err_builtin_annotation_not_string_constant : Error<
+  "__builtin_annotation requires a non wide string constant">;
 
 // CFString checking
 def err_cfstring_literal_not_string_constant : Error<
-  "CFString literal is not a string constant">;
+  "CFString literal is not a string constant">,
+  InGroup<DiagGroup<"CFString-literal">>;
 def warn_cfstring_truncated : Warning<
   "input conversion stopped due to an input byte that does not "
-  "belong to the input codeset UTF-8">;
+  "belong to the input codeset UTF-8">,
+  InGroup<DiagGroup<"CFString-literal">>;
 
 // Statements.
 def err_continue_not_in_loop : Error<
@@ -4208,6 +4678,9 @@ def err_second_parameter_to_va_arg_abstract: Error<
 def warn_second_parameter_to_va_arg_not_pod : Warning<
   "second argument to 'va_arg' is of non-POD type %0">,
   InGroup<DiagGroup<"non-pod-varargs">>, DefaultError;
+def warn_second_parameter_to_va_arg_ownership_qualified : Warning<
+  "second argument to 'va_arg' is of ARC ownership-qualified type %0">,
+  InGroup<DiagGroup<"non-pod-varargs">>, DefaultError;
 def warn_second_parameter_to_va_arg_never_compatible : Warning<
   "second argument to 'va_arg' is of promotable type %0; this va_arg has "
   "undefined behavior because arguments will be promoted to %1">;
@@ -4306,13 +4779,20 @@ def err_c99_array_usage_cxx : Error<
   "C99-specific array features are not permitted in C++">;
 def err_double_requires_fp64 : Error<
   "use of type 'double' requires cl_khr_fp64 extension to be enabled">;
+def err_nsconsumed_attribute_mismatch : Error<
+  "overriding method has mismatched ns_consumed attribute on its"
+  " parameter">;
+def err_nsreturns_retained_attribute_mismatch : Error<
+  "overriding method has mismatched ns_returns_%select{not_retained|retained}0"
+  " attributes">;
   
 def note_getter_unavailable : Note<
   "or because setter is declared here, but no getter method %0 is found">;
 def err_invalid_protocol_qualifiers : Error<
   "invalid protocol qualifiers on non-ObjC type">;
 def warn_ivar_use_hidden : Warning<
-  "local declaration of %0 hides instance variable">;
+  "local declaration of %0 hides instance variable">,
+   InGroup<DiagGroup<"shadow-ivar">>;
 def error_ivar_use_in_class_method : Error<
   "instance variable %0 accessed in class method">;
 def error_implicit_ivar_access : Error<
@@ -4327,6 +4807,9 @@ def warn_attribute_method_def : Warning<
 def ext_typecheck_base_super : Warning<
   "method parameter type %0 does not match "
   "super class method parameter type %1">, InGroup<SuperSubClassMismatch>, DefaultIgnore;
+def warn_missing_method_return_type : Warning<
+  "method has no return type specified; defaults to 'id'">,
+  InGroup<MissingMethodReturnType>, DefaultIgnore;
 
 // Spell-checking diagnostics
 def err_unknown_typename_suggest : Error<
@@ -4377,9 +4860,12 @@ def err_sizeof_pack_no_pack_name_suggest : Error<
 def note_parameter_pack_here : Note<"parameter pack %0 declared here">;
 
 def err_uncasted_use_of_unknown_any : Error<
-  "%0 has unknown type;  cast it to its declared type to use it">;
+  "%0 has unknown type; cast it to its declared type to use it">;
 def err_uncasted_call_of_unknown_any : Error<
-  "%0 has unknown return type;  cast the call to its declared return type">;
+  "%0 has unknown return type; cast the call to its declared return type">;
+def err_uncasted_send_to_unknown_any_method : Error<
+  "no known method %select{%objcinstance1|%objcclass1}0; cast the "
+  "message send to the method's return type">;
 def err_unsupported_unknown_any_decl : Error<
   "%0 has unknown type, which is unsupported for this kind of declaration">;
 def err_unsupported_unknown_any_expr : Error<
@@ -4391,6 +4877,8 @@ def err_unknown_any_addrof : Error<
   "can only be cast to a pointer type">;
 def err_unknown_any_var_function_type : Error<
   "variable %0 with unknown type cannot be given a function type">;
+def err_unknown_any_function : Error<
+  "function %0 with unknown type must be given a function type">;
 
 def err_filter_expression_integral : Error<
   "filter expression type should be an integral value not %0">;
@@ -4409,14 +4897,31 @@ def warn_related_result_type_compatibility_class : Warning<
 def warn_related_result_type_compatibility_protocol : Warning<
   "protocol method is expected to return an instance of the implementing "
   "class, but is declared to return %0">;
-def note_related_result_type_overridden : Note<
+def note_related_result_type_overridden_family : Note<
   "overridden method is part of the '%select{|alloc|copy|init|mutableCopy|"
-  "new|autorelease|dealloc|release|retain|retainCount|self}0' method family">;
+  "new|autorelease|dealloc|finalize|release|retain|retainCount|self}0' method "
+  "family">;
+def note_related_result_type_overridden : Note<
+  "overridden method returns an instance of its class type">;
 def note_related_result_type_inferred : Note<
   "%select{class|instance}0 method %1 is assumed to return an instance of "
   "its receiver type (%2)">;
 
 }
 
+let CategoryName = "Modules Issue" in {
+def err_module_private_follows_public : Error<
+  "__module_private__ declaration of %0 follows public declaration">;
+def err_module_private_specialization : Error<
+  "%select{template|partial|member}0 specialization cannot be "
+  "declared __module_private__">;
+def err_module_private_local : Error<
+  "%select{local variable|parameter|typedef}0 %1 cannot be declared "
+  "__module_private__">;
+def err_module_private_local_class : Error<
+  "local %select{struct|union|class|enum}0 cannot be declared "
+  "__module_private__">;
+}
+
 } // end of sema component.
 
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/FileManager.h b/contrib/llvm/tools/clang/include/clang/Basic/FileManager.h
index 1324533fa03e..ea8ed9f02fed 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/FileManager.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/FileManager.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_FILEMANAGER_H
 
 #include "clang/Basic/FileSystemOptions.h"
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -123,9 +124,9 @@ class FileManager : public llvm::RefCountedBase<FileManager> {
   /// \brief The virtual directories that we have allocated.  For each
   /// virtual file (e.g. foo/bar/baz.cpp), we add all of its parent
   /// directories (foo/ and foo/bar/) here.
-  llvm::SmallVector<DirectoryEntry*, 4> VirtualDirectoryEntries;
+  SmallVector<DirectoryEntry*, 4> VirtualDirectoryEntries;
   /// \brief The virtual files that we have allocated.
-  llvm::SmallVector<FileEntry*, 4> VirtualFileEntries;
+  SmallVector<FileEntry*, 4> VirtualFileEntries;
 
   /// SeenDirEntries/SeenFileEntries - This is a cache that maps paths
   /// to directory/file entries (either real or virtual) we have
@@ -153,7 +154,7 @@ class FileManager : public llvm::RefCountedBase<FileManager> {
 
   /// Add all ancestors of the given path (pointing to either a file
   /// or a directory) as virtual directories.
-  void addAncestorsAsVirtualDirs(llvm::StringRef Path);
+  void addAncestorsAsVirtualDirs(StringRef Path);
 
 public:
   FileManager(const FileSystemOptions &FileSystemOpts);
@@ -178,41 +179,51 @@ class FileManager : public llvm::RefCountedBase<FileManager> {
   /// getDirectory - Lookup, cache, and verify the specified directory
   /// (real or virtual).  This returns NULL if the directory doesn't exist.
   ///
-  const DirectoryEntry *getDirectory(llvm::StringRef DirName);
+  /// \param CacheFailure If true and the file does not exist, we'll cache
+  /// the failure to find this file.
+  const DirectoryEntry *getDirectory(StringRef DirName,
+                                     bool CacheFailure = true);
 
   /// \brief Lookup, cache, and verify the specified file (real or
   /// virtual).  This returns NULL if the file doesn't exist.
   ///
-  /// \param openFile if true and the file exists, it will be opened.
-  const FileEntry *getFile(llvm::StringRef Filename, bool openFile = false);
+  /// \param OpenFile if true and the file exists, it will be opened.
+  ///
+  /// \param CacheFailure If true and the file does not exist, we'll cache
+  /// the failure to find this file.
+  const FileEntry *getFile(StringRef Filename, bool OpenFile = false,
+                           bool CacheFailure = true);
+
+  /// \brief Returns the current file system options
+  const FileSystemOptions &getFileSystemOptions() { return FileSystemOpts; }
 
   /// \brief Retrieve a file entry for a "virtual" file that acts as
   /// if there were a file with the given name on disk. The file
   /// itself is not accessed.
-  const FileEntry *getVirtualFile(llvm::StringRef Filename, off_t Size,
+  const FileEntry *getVirtualFile(StringRef Filename, off_t Size,
                                   time_t ModificationTime);
 
   /// \brief Open the specified file as a MemoryBuffer, returning a new
   /// MemoryBuffer if successful, otherwise returning null.
   llvm::MemoryBuffer *getBufferForFile(const FileEntry *Entry,
                                        std::string *ErrorStr = 0);
-  llvm::MemoryBuffer *getBufferForFile(llvm::StringRef Filename,
+  llvm::MemoryBuffer *getBufferForFile(StringRef Filename,
                                        std::string *ErrorStr = 0);
 
   // getNoncachedStatValue - Will get the 'stat' information for the given path.
   // If the path is relative, it will be resolved against the WorkingDir of the
   // FileManager's FileSystemOptions.
-  bool getNoncachedStatValue(llvm::StringRef Path, struct stat &StatBuf);
+  bool getNoncachedStatValue(StringRef Path, struct stat &StatBuf);
 
   /// \brief If path is not absolute and FileSystemOptions set the working
   /// directory, the path is modified to be relative to the given
   /// working directory.
-  void FixupRelativePath(llvm::SmallVectorImpl<char> &path) const;
+  void FixupRelativePath(SmallVectorImpl<char> &path) const;
 
   /// \brief Produce an array mapping from the unique IDs assigned to each
   /// file to the corresponding FileEntry pointer.
   void GetUniqueIDMapping(
-                    llvm::SmallVectorImpl<const FileEntry *> &UIDToFiles) const;
+                    SmallVectorImpl<const FileEntry *> &UIDToFiles) const;
   
   void PrintStats() const;
 };
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/IdentifierTable.h b/contrib/llvm/tools/clang/include/clang/Basic/IdentifierTable.h
index bebcffdddede..5e48a8661974 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/IdentifierTable.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/IdentifierTable.h
@@ -17,6 +17,7 @@
 
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/TokenKinds.h"
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/SmallString.h"
@@ -49,14 +50,15 @@ namespace clang {
 /// set, and all tok::identifier tokens have a pointer to one of these.
 class IdentifierInfo {
   // Note: DON'T make TokenID a 'tok::TokenKind'; MSVC will treat it as a
-  //       signed char and TokenKinds > 127 won't be handled correctly.
-  unsigned TokenID            : 8; // Front-end token ID or tok::identifier.
+  //       signed char and TokenKinds > 255 won't be handled correctly.
+  unsigned TokenID            : 9; // Front-end token ID or tok::identifier.
   // Objective-C keyword ('protocol' in '@protocol') or builtin (__builtin_inf).
   // First NUM_OBJC_KEYWORDS values are for Objective-C, the remaining values
   // are for builtins.
   unsigned ObjCOrBuiltinID    :11;
   bool HasMacro               : 1; // True if there is a #define for this.
   bool IsExtension            : 1; // True if identifier is a lang extension.
+  bool IsCXX11CompatKeyword   : 1; // True if identifier is a keyword in C++11.
   bool IsPoisoned             : 1; // True if identifier is poisoned.
   bool IsCPPOperatorKeyword   : 1; // True if ident is a C++ operator keyword.
   bool NeedsHandleIdentifier  : 1; // See "RecomputeNeedsHandleIdentifier".
@@ -64,7 +66,7 @@ class IdentifierInfo {
                                    // file and wasn't modified since.
   bool RevertedTokenID        : 1; // True if RevertTokenIDToIdentifier was
                                    // called.
-  // 6 bits left in 32-bit word.
+  // 5 bits left in 32-bit word.
   void *FETokenInfo;               // Managed by the language front-end.
   llvm::StringMapEntry<IdentifierInfo*> *Entry;
 
@@ -113,8 +115,8 @@ class IdentifierInfo {
   }
 
   /// getName - Return the actual identifier string.
-  llvm::StringRef getName() const {
-    return llvm::StringRef(getNameStart(), getLength());
+  StringRef getName() const {
+    return StringRef(getNameStart(), getLength());
   }
 
   /// hasMacroDefinition - Return true if this identifier is #defined to some
@@ -198,6 +200,19 @@ class IdentifierInfo {
       RecomputeNeedsHandleIdentifier();
   }
 
+  /// is/setIsCXX11CompatKeyword - Initialize information about whether or not
+  /// this language token is a keyword in C++11. This controls compatibility
+  /// warnings, and is only true when not parsing C++11. Once a compatibility
+  /// problem has been diagnosed with this keyword, the flag will be cleared.
+  bool isCXX11CompatKeyword() const { return IsCXX11CompatKeyword; }
+  void setIsCXX11CompatKeyword(bool Val) {
+    IsCXX11CompatKeyword = Val;
+    if (Val)
+      NeedsHandleIdentifier = 1;
+    else
+      RecomputeNeedsHandleIdentifier();
+  }
+
   /// setIsPoisoned - Mark this identifier as poisoned.  After poisoning, the
   /// Preprocessor will emit an error every time this token is used.
   void setIsPoisoned(bool Value = true) {
@@ -251,7 +266,8 @@ class IdentifierInfo {
   void RecomputeNeedsHandleIdentifier() {
     NeedsHandleIdentifier =
       (isPoisoned() | hasMacroDefinition() | isCPlusPlusOperatorKeyword() |
-       isExtensionToken());
+       isExtensionToken() | isCXX11CompatKeyword() ||
+       (getTokenID() == tok::kw___import_module__));
   }
 };
 
@@ -299,8 +315,8 @@ class IdentifierIterator {
   /// advances the iterator for the following string.
   ///
   /// \returns The next string in the identifier table. If there is
-  /// no such string, returns an empty \c llvm::StringRef.
-  virtual llvm::StringRef Next() = 0;
+  /// no such string, returns an empty \c StringRef.
+  virtual StringRef Next() = 0;
 };
 
 /// IdentifierInfoLookup - An abstract class used by IdentifierTable that
@@ -314,7 +330,7 @@ class IdentifierInfoLookup {
   ///  Unlike the version in IdentifierTable, this returns a pointer instead
   ///  of a reference.  If the pointer is NULL then the IdentifierInfo cannot
   ///  be found.
-  virtual IdentifierInfo* get(llvm::StringRef Name) = 0;
+  virtual IdentifierInfo* get(StringRef Name) = 0;
 
   /// \brief Retrieve an iterator into the set of all identifiers
   /// known to this identifier lookup source.
@@ -376,7 +392,7 @@ class IdentifierTable {
 
   /// get - Return the identifier token info for the specified named identifier.
   ///
-  IdentifierInfo &get(llvm::StringRef Name) {
+  IdentifierInfo &get(StringRef Name) {
     llvm::StringMapEntry<IdentifierInfo*> &Entry =
       HashTable.GetOrCreateValue(Name);
 
@@ -405,9 +421,10 @@ class IdentifierTable {
     return *II;
   }
 
-  IdentifierInfo &get(llvm::StringRef Name, tok::TokenKind TokenCode) {
+  IdentifierInfo &get(StringRef Name, tok::TokenKind TokenCode) {
     IdentifierInfo &II = get(Name);
     II.TokenID = TokenCode;
+    assert(II.TokenID == (unsigned) TokenCode && "TokenCode too large");
     return II;
   }
 
@@ -417,7 +434,7 @@ class IdentifierTable {
   /// This is a version of get() meant for external sources that want to
   /// introduce or modify an identifier. If they called get(), they would
   /// likely end up in a recursion.
-  IdentifierInfo &getOwn(llvm::StringRef Name) {
+  IdentifierInfo &getOwn(StringRef Name) {
     llvm::StringMapEntry<IdentifierInfo*> &Entry =
       HashTable.GetOrCreateValue(Name);
 
@@ -485,6 +502,7 @@ enum ObjCMethodFamily {
   // selector with the given name.
   OMF_autorelease,
   OMF_dealloc,
+  OMF_finalize,
   OMF_release,
   OMF_retain,
   OMF_retainCount,
@@ -507,7 +525,7 @@ enum { InvalidObjCMethodFamily = (1 << ObjCMethodFamilyBitWidth) - 1 };
 /// selectors that take no arguments and selectors that take 1 argument, which
 /// accounts for 78% of all selectors in Cocoa.h.
 class Selector {
-  friend class DiagnosticInfo;
+  friend class Diagnostic;
 
   enum IdentifierInfoFlag {
     // MultiKeywordSelector = 0.
@@ -595,7 +613,7 @@ class Selector {
   ///
   /// \returns the name for this slot, which may be the empty string if no
   /// name was supplied.
-  llvm::StringRef getNameForSlot(unsigned argIndex) const;
+  StringRef getNameForSlot(unsigned argIndex) const;
   
   /// getAsString - Derive the full selector name (e.g. "foo:bar:") and return
   /// it as an std::string.
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/LLVM.h b/contrib/llvm/tools/clang/include/clang/Basic/LLVM.h
new file mode 100644
index 000000000000..27c459dee4e2
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Basic/LLVM.h
@@ -0,0 +1,53 @@
+//===--- LLVM.h - Import various common LLVM datatypes ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file forward declares and imports various common LLVM datatypes that
+// clang wants to use unqualified.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_BASIC_LLVM_H
+#define CLANG_BASIC_LLVM_H
+
+// This should be the only #include, force #includes of all the others on
+// clients.
+#include "llvm/Support/Casting.h"
+
+namespace llvm {
+  // ADT's.
+  class StringRef;
+  class Twine;
+  template<typename T> class ArrayRef;
+  template<typename T, unsigned N> class SmallVector;
+  template<typename T> class SmallVectorImpl;
+
+  class raw_ostream;
+  // TODO: DenseMap, ...
+}
+
+
+namespace clang {
+  // Casting operators.
+  using llvm::isa;
+  using llvm::cast;
+  using llvm::dyn_cast;
+  using llvm::dyn_cast_or_null;
+  using llvm::cast_or_null;
+  
+  // ADT's.
+  using llvm::StringRef;
+  using llvm::Twine;
+  using llvm::ArrayRef;
+  using llvm::SmallVector;
+  using llvm::SmallVectorImpl;
+  
+  using llvm::raw_ostream;
+} // end namespace clang.
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.def b/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.def
new file mode 100644
index 000000000000..c8389ab0e793
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.def
@@ -0,0 +1,159 @@
+//===--- LangOptions.def - Language option database --------------- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the language options. Users of this file must
+// define the LANGOPT macro to make use of this information. 
+// Optionally, the user may also define BENIGN_LANGOPT
+// (for options that don't affect the construction of the AST in an
+// incompatible way), ENUM_LANGOPT (for options that have enumeration,
+// rather than unsigned, type), BENIGN_ENUM_LANGOPT (for benign
+// options that have enumeration type), and VALUE_LANGOPT is a language option
+// that describes a value rather than a flag.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LANGOPT
+#  error Define the LANGOPT macro to handle language options
+#endif
+
+#ifndef VALUE_LANGOPT
+#  define VALUE_LANGOPT(Name, Bits, Default, Description) \
+     LANGOPT(Name, Bits, Default, Description)
+#endif
+
+#ifndef BENIGN_LANGOPT
+#  define BENIGN_LANGOPT(Name, Bits, Default, Description) \
+     LANGOPT(Name, Bits, Default, Description)
+#endif
+
+#ifndef ENUM_LANGOPT
+#  define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+     LANGOPT(Name, Bits, Default, Description)
+#endif
+
+#ifndef BENIGN_ENUM_LANGOPT
+#  define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+     ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+#endif
+
+LANGOPT(C99               , 1, 0, "C99")
+LANGOPT(C1X               , 1, 0, "C1X")
+LANGOPT(MicrosoftExt      , 1, 0, "Microsoft extensions")
+LANGOPT(MicrosoftMode     , 1, 0, "Microsoft compatibility mode")
+LANGOPT(Borland           , 1, 0, "Borland extensions")
+LANGOPT(CPlusPlus         , 1, 0, "C++")
+LANGOPT(CPlusPlus0x       , 1, 0, "C++0x")
+LANGOPT(ObjC1             , 1, 0, "Objective-C 1")
+LANGOPT(ObjC2             , 1, 0, "Objective-C 2")
+LANGOPT(ObjCNonFragileABI , 1, 0, "Objective-C modern abi")
+LANGOPT(ObjCNonFragileABI2 , 1, 0, "Objective-C enhanced modern abi")
+BENIGN_LANGOPT(ObjCDefaultSynthProperties , 1, 0, 
+               "Objective-C auto-synthesized properties")
+BENIGN_LANGOPT(ObjCInferRelatedResultType , 1, 1, 
+               "Objective-C related result type inference")
+LANGOPT(Trigraphs         , 1, 0,"trigraphs")
+LANGOPT(BCPLComment       , 1, 0, "BCPL-style '//' comments")
+LANGOPT(Bool              , 1, 0, "bool, true, and false keywords")
+BENIGN_LANGOPT(DollarIdents   , 1, 1, "'$' in identifiers")
+BENIGN_LANGOPT(AsmPreprocessor, 1, 0, "preprocessor in asm mode")
+BENIGN_LANGOPT(GNUMode        , 1, 1, "GNU extensions")
+LANGOPT(GNUKeywords       , 1, 1, "GNU keywords")
+BENIGN_LANGOPT(ImplicitInt, 1, !C99 && !CPlusPlus, "C89 implicit 'int'")
+LANGOPT(Digraphs          , 1, 0, "digraphs")
+BENIGN_LANGOPT(HexFloats  , 1, C99, "C99 hexadecimal float constants")
+LANGOPT(CXXOperatorNames  , 1, 0, "C++ operator name keywords")
+LANGOPT(AppleKext         , 1, 0, "Apple kext support")
+BENIGN_LANGOPT(PascalStrings, 1, 0, "Pascal string support")
+LANGOPT(WritableStrings   , 1, 0, "writable string support")
+LANGOPT(ConstStrings      , 1, 0, "const-qualified string support")
+LANGOPT(LaxVectorConversions , 1, 1, "lax vector conversions")
+LANGOPT(AltiVec           , 1, 0, "AltiVec-style vector initializers")
+LANGOPT(Exceptions        , 1, 0, "exception handling")
+LANGOPT(ObjCExceptions    , 1, 0, "Objective-C exceptions")
+LANGOPT(CXXExceptions     , 1, 0, "C++ exceptions")
+LANGOPT(SjLjExceptions    , 1, 0, "setjmp-longjump exception handling")
+LANGOPT(TraditionalCPP    , 1, 0, "traditional CPP emulation")
+LANGOPT(RTTI              , 1, 1, "run-time type information")
+LANGOPT(MSBitfields       , 1, 0, "Microsoft-compatible structure layout")
+LANGOPT(NeXTRuntime       , 1, 1, "NeXT Objective-C runtime")
+LANGOPT(Freestanding, 1, 0, "freestanding implementation")
+LANGOPT(FormatExtensions  , 1, 0, "FreeBSD format extensions")
+LANGOPT(NoBuiltin         , 1, 0, "disable builtin functions")
+
+BENIGN_LANGOPT(ThreadsafeStatics , 1, 1, "thread-safe static initializers")
+LANGOPT(POSIXThreads      , 1, 0, "POSIX thread support")
+LANGOPT(Blocks            , 1, 0, "blocks extension to C")
+BENIGN_LANGOPT(EmitAllDecls      , 1, 0, "support for emitting all declarations")
+LANGOPT(MathErrno         , 1, 1, "errno support for math functions")
+BENIGN_LANGOPT(HeinousExtensions , 1, 0, "Extensions that we really don't like and may be ripped out at any time")
+
+LANGOPT(Optimize          , 1, 0, "__OPTIMIZE__ predefined macro")
+LANGOPT(OptimizeSize      , 1, 0, "__OPTIMIZE_SIZE__ predefined macro")
+LANGOPT(Static            , 1, 0, "__STATIC__ predefined macro (as opposed to __DYNAMIC__)")
+VALUE_LANGOPT(PackStruct  , 32, 0, 
+              "default struct packing maximum alignment")
+VALUE_LANGOPT(PICLevel          , 2, 0, "__PIC__ level")
+LANGOPT(GNUInline         , 1, 0, "GNU inline semantics")
+LANGOPT(NoInline          , 1, 0, "__NO_INLINE__ predefined macro")
+LANGOPT(Deprecated        , 1, 0, "__DEPRECATED predefined macro")
+
+BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars")
+
+BENIGN_LANGOPT(AccessControl     , 1, 1, "C++ access control")
+LANGOPT(CharIsSigned      , 1, 1, "signed char")
+LANGOPT(ShortWChar        , 1, 0, "unsigned short wchar_t")
+
+LANGOPT(ShortEnums        , 1, 0, "short enum types")
+
+LANGOPT(OpenCL            , 1, 0, "OpenCL")
+LANGOPT(CUDA              , 1, 0, "CUDA")
+
+LANGOPT(AssumeSaneOperatorNew , 1, 1, "implicit __attribute__((malloc)) for C++'s new operators")
+BENIGN_LANGOPT(ElideConstructors , 1, 1, "C++ copy constructor elision")
+BENIGN_LANGOPT(CatchUndefined    , 1, 0, "catching undefined behavior at run time")
+BENIGN_LANGOPT(DumpRecordLayouts , 1, 0, "dumping the layout of IRgen'd records")
+BENIGN_LANGOPT(DumpVTableLayouts , 1, 0, "dumping the layouts of emitted vtables")
+LANGOPT(NoConstantCFStrings , 1, 0, "no constant CoreFoundation strings")
+BENIGN_LANGOPT(InlineVisibilityHidden , 1, 0, "hidden default visibility for inline C++ methods")
+BENIGN_LANGOPT(ParseUnknownAnytype, 1, 0, "__unknown_anytype")
+BENIGN_LANGOPT(DebuggerSupport , 1, 0, "debugger support")
+
+BENIGN_LANGOPT(SpellChecking , 1, 1, "spell-checking")
+LANGOPT(SinglePrecisionConstants , 1, 0, "treating double-precision floating point constants as single precision constants")
+LANGOPT(FastRelaxedMath , 1, 0, "OpenCL fast relaxed math")
+LANGOPT(DefaultFPContract , 1, 0, "FP_CONTRACT")
+LANGOPT(NoBitFieldTypeAlign , 1, 0, "bit-field type alignment")
+LANGOPT(ObjCAutoRefCount , 1, 0, "Objective-C automated reference counting")
+LANGOPT(ObjCRuntimeHasWeak , 1, 0, "__weak support in the ARC runtime")
+LANGOPT(FakeAddressSpaceMap , 1, 0, "OpenCL fake address space map")
+
+LANGOPT(MRTD , 1, 0, "-mrtd calling convention")
+BENIGN_LANGOPT(DelayedTemplateParsing , 1, 0, "delayed template parsing")
+LANGOPT(BlocksRuntimeOptional , 1, 0, "optional blocks runtime")
+
+ENUM_LANGOPT(GC, GCMode, 2, NonGC, "Objective-C Garbage Collection mode")
+ENUM_LANGOPT(VisibilityMode, Visibility, 3, DefaultVisibility, 
+             "symbol visibility")
+ENUM_LANGOPT(StackProtector, StackProtectorMode, 2, SSPOff, 
+             "stack protector mode")
+ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined,
+             "signed integer overflow handling")
+
+BENIGN_LANGOPT(InstantiationDepth, 32, 1024, 
+               "maximum template instantiation depth")
+BENIGN_LANGOPT(NumLargeByValueCopy, 32, 0, 
+        "if non-zero, warn about parameter or return Warn if parameter/return value is larger in bytes than this setting. 0 is no check.")
+VALUE_LANGOPT(MSCVersion, 32, 0, 
+              "version of Microsoft Visual C/C++")
+
+#undef LANGOPT
+#undef VALUE_LANGOPT
+#undef BENIGN_LANGOPT
+#undef ENUM_LANGOPT
+#undef BENIGN_ENUM_LANGOPT
+
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.h b/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.h
index de2ac6a1bd1c..688047ff5bdf 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/LangOptions.h
@@ -23,255 +23,53 @@ namespace clang {
 /// enabled, which controls the dialect of C that is accepted.
 class LangOptions {
 public:
-  unsigned Trigraphs         : 1;  // Trigraphs in source files.
-  unsigned BCPLComment       : 1;  // BCPL-style '//' comments.
-  unsigned Bool              : 1;  // 'bool', 'true', 'false' keywords.
-  unsigned DollarIdents      : 1;  // '$' allowed in identifiers.
-  unsigned AsmPreprocessor   : 1;  // Preprocessor in asm mode.
-  unsigned GNUMode           : 1;  // True in gnu99 mode false in c99 mode (etc)
-  unsigned GNUKeywords       : 1;  // True if GNU-only keywords are allowed
-  unsigned ImplicitInt       : 1;  // C89 implicit 'int'.
-  unsigned Digraphs          : 1;  // C94, C99 and C++
-  unsigned HexFloats         : 1;  // C99 Hexadecimal float constants.
-  unsigned C99               : 1;  // C99 Support
-  unsigned C1X               : 1;  // C1X Support
-  unsigned Microsoft         : 1;  // Microsoft extensions.
-  unsigned Borland           : 1;  // Borland extensions.
-  unsigned CPlusPlus         : 1;  // C++ Support
-  unsigned CPlusPlus0x       : 1;  // C++0x Support
-  unsigned CXXOperatorNames  : 1;  // Treat C++ operator names as keywords.
-
-  unsigned ObjC1             : 1;  // Objective-C 1 support enabled.
-  unsigned ObjC2             : 1;  // Objective-C 2 support enabled.
-  unsigned ObjCNonFragileABI : 1;  // Objective-C modern abi enabled
-  unsigned ObjCNonFragileABI2 : 1;  // Objective-C enhanced modern abi enabled
-  unsigned ObjCDefaultSynthProperties : 1; // Objective-C auto-synthesized properties.
-  unsigned ObjCInferRelatedResultType : 1; // Infer Objective-C related return
-                                           // types
-  unsigned AppleKext         : 1;  // Allow apple kext features.
-
-  unsigned PascalStrings     : 1;  // Allow Pascal strings
-  unsigned WritableStrings   : 1;  // Allow writable strings
-  unsigned ConstStrings      : 1;  // Add const qualifier to strings (-Wwrite-strings)
-  unsigned LaxVectorConversions : 1;
-  unsigned AltiVec           : 1;  // Support AltiVec-style vector initializers.
-  unsigned Exceptions        : 1;  // Support exception handling.
-  unsigned ObjCExceptions    : 1;  // Support Objective-C exceptions.
-  unsigned CXXExceptions     : 1;  // Support C++ exceptions.
-  unsigned SjLjExceptions    : 1;  // Use setjmp-longjump exception handling.
-  unsigned TraditionalCPP    : 1; /// Enable some traditional CPP emulation.
-  unsigned RTTI              : 1;  // Support RTTI information.
-
-  unsigned MSBitfields       : 1; // MS-compatible structure layout
-  unsigned NeXTRuntime       : 1; // Use NeXT runtime.
-  unsigned Freestanding      : 1; // Freestanding implementation
-  unsigned FormatExtensions  : 1; // FreeBSD format extensions (-fformat-extensions)
-  unsigned NoBuiltin         : 1; // Do not use builtin functions (-fno-builtin)
-
-  unsigned ThreadsafeStatics : 1; // Whether static initializers are protected
-                                  // by locks.
-  unsigned POSIXThreads      : 1; // Compiling with POSIX thread support
-                                  // (-pthread)
-  unsigned Blocks            : 1; // block extension to C
-  unsigned EmitAllDecls      : 1; // Emit all declarations, even if
-                                  // they are unused.
-  unsigned MathErrno         : 1; // Math functions must respect errno
-                                  // (modulo the platform support).
-
-  unsigned HeinousExtensions : 1; // Extensions that we really don't like and
-                                  // may be ripped out at any time.
-
-  unsigned Optimize          : 1; // Whether __OPTIMIZE__ should be defined.
-  unsigned OptimizeSize      : 1; // Whether __OPTIMIZE_SIZE__ should be
-                                  // defined.
-  unsigned Static            : 1; // Should __STATIC__ be defined (as
-                                  // opposed to __DYNAMIC__).
-  unsigned PICLevel          : 2; // The value for __PIC__, if non-zero.
-
-  unsigned GNUInline         : 1; // Should GNU inline semantics be
-                                  // used (instead of C99 semantics).
-  unsigned NoInline          : 1; // Should __NO_INLINE__ be defined.
-
-  unsigned Deprecated        : 1; // Should __DEPRECATED be defined.
-
-  unsigned ObjCGCBitmapPrint : 1; // Enable printing of gc's bitmap layout
-                                  // for __weak/__strong ivars.
-
-  unsigned AccessControl     : 1; // Whether C++ access control should
-                                  // be enabled.
-  unsigned CharIsSigned      : 1; // Whether char is a signed or unsigned type
-  unsigned ShortWChar        : 1; // Force wchar_t to be unsigned short int.
-
-  unsigned ShortEnums        : 1; // The enum type will be equivalent to the
-                                  // smallest integer type with enough room.
-
-  unsigned OpenCL            : 1; // OpenCL C99 language extensions.
-  unsigned CUDA              : 1; // CUDA C++ language extensions.
-
-  unsigned AssumeSaneOperatorNew : 1; // Whether to add __attribute__((malloc))
-                                      // to the declaration of C++'s new
-                                      // operators
-  unsigned ElideConstructors : 1; // Whether C++ copy constructors should be
-                                  // elided if possible.
-  unsigned CatchUndefined    : 1; // Generate code to check for undefined ops.
-  unsigned DumpRecordLayouts : 1; /// Dump the layout of IRgen'd records.
-  unsigned DumpVTableLayouts : 1; /// Dump the layouts of emitted vtables.
-  unsigned NoConstantCFStrings : 1;  // Do not do CF strings
-  unsigned InlineVisibilityHidden : 1; // Whether inline C++ methods have
-                                       // hidden visibility by default.
-  unsigned ParseUnknownAnytype: 1; /// Let the user write __unknown_anytype.
-  unsigned DebuggerSupport : 1;   /// Do things that only make sense when
-                                  /// supporting a debugger
-
-  unsigned SpellChecking : 1; // Whether to perform spell-checking for error
-                              // recovery.
-  unsigned SinglePrecisionConstants : 1; // Whether to treat double-precision
-                                         // floating point constants as
-                                         // single precision constants.
-  unsigned FastRelaxedMath : 1; // OpenCL fast relaxed math (on its own,
-                                // defines __FAST_RELAXED_MATH__).
-  unsigned DefaultFPContract : 1; // Default setting for FP_CONTRACT
-  // FIXME: This is just a temporary option, for testing purposes.
-  unsigned NoBitFieldTypeAlign : 1;
-  unsigned ObjCAutoRefCount : 1; // Objective C automated reference counting
-  unsigned ObjCRuntimeHasWeak : 1; // The ARC runtime supports __weak
-  unsigned ObjCInferRelatedReturnType : 1; // Infer Objective-C related return
-                                           // types
-  unsigned FakeAddressSpaceMap : 1; // Use a fake address space map, for
-                                    // testing languages such as OpenCL.
-
-  unsigned MRTD : 1;            // -mrtd calling convention
-  unsigned DelayedTemplateParsing : 1;  // Delayed template parsing
-
-private:
-  // We declare multibit enums as unsigned because MSVC insists on making enums
-  // signed.  Set/Query these values using accessors.
-  unsigned GC : 2;                // Objective-C Garbage Collection modes.
-  unsigned SymbolVisibility  : 3; // Symbol's visibility.
-  unsigned StackProtector    : 2; // Whether stack protectors are on.
-  unsigned SignedOverflowBehavior : 2; // How to handle signed integer overflow.
-
-public:
-  unsigned InstantiationDepth;    // Maximum template instantiation depth.
-  unsigned NumLargeByValueCopy;   // Warn if parameter/return value is larger
-                                  // in bytes than this setting. 0 is no check.
-
-  // Version of Microsoft Visual C/C++ we are pretending to be. This is
-  // temporary until we support all MS extensions used in Windows SDK and stdlib
-  // headers. Sets _MSC_VER.
-  unsigned MSCVersion;
-
-  std::string ObjCConstantStringClass;
-
+  typedef clang::Visibility Visibility;
+  
   enum GCMode { NonGC, GCOnly, HybridGC };
   enum StackProtectorMode { SSPOff, SSPOn, SSPReq };
-
+  
   enum SignedOverflowBehaviorTy {
     SOB_Undefined,  // Default C standard behavior.
     SOB_Defined,    // -fwrapv
     SOB_Trapping    // -ftrapv
   };
+
+  // Define simple language options (with no accessors).
+#define LANGOPT(Name, Bits, Default, Description) unsigned Name : Bits;
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+#include "clang/Basic/LangOptions.def"
+  
+private:
+  // Define language options of enumeration type. These are private, and will
+  // have accessors (below).
+#define LANGOPT(Name, Bits, Default, Description) 
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  unsigned Name : Bits;
+#include "clang/Basic/LangOptions.def"
+  
+public:
+  std::string ObjCConstantStringClass;
+  
   /// The name of the handler function to be called when -ftrapv is specified.
   /// If none is specified, abort (GCC-compatible behaviour).
   std::string OverflowHandler;
 
-  LangOptions() {
-    Trigraphs = BCPLComment = Bool = DollarIdents = AsmPreprocessor = 0;
-    GNUMode = GNUKeywords = ImplicitInt = Digraphs = 0;
-    HexFloats = 0;
-    ObjCAutoRefCount = 0;
-    ObjCRuntimeHasWeak = 0;
-    ObjCInferRelatedReturnType = 0;
-    GC = ObjC1 = ObjC2 = ObjCNonFragileABI = ObjCNonFragileABI2 = 0;
-    AppleKext = 0;
-    ObjCDefaultSynthProperties = 0;
-    ObjCInferRelatedResultType = 1;
-    NoConstantCFStrings = 0; InlineVisibilityHidden = 0;
-    C99 = C1X = Microsoft = Borland = CPlusPlus = CPlusPlus0x = 0;
-    CXXOperatorNames = PascalStrings = WritableStrings = ConstStrings = 0;
-    Exceptions = ObjCExceptions = CXXExceptions = SjLjExceptions = 0;
-    TraditionalCPP = Freestanding = NoBuiltin = 0;
-    MSBitfields = 0;
-    NeXTRuntime = 1;
-    RTTI = 1;
-    LaxVectorConversions = 1;
-    HeinousExtensions = 0;
-    AltiVec = OpenCL = CUDA = StackProtector = 0;
-
-    SymbolVisibility = (unsigned) DefaultVisibility;
-
-    ThreadsafeStatics = 1;
-    POSIXThreads = 0;
-    Blocks = 0;
-    EmitAllDecls = 0;
-    MathErrno = 1;
-    SignedOverflowBehavior = SOB_Undefined;
-
-    AssumeSaneOperatorNew = 1;
-    AccessControl = 1;
-    ElideConstructors = 1;
-
-    SignedOverflowBehavior = 0;
-    ObjCGCBitmapPrint = 0;
-
-    InstantiationDepth = 1024;
-
-    NumLargeByValueCopy = 0;
-    MSCVersion = 0;
-    
-    Optimize = 0;
-    OptimizeSize = 0;
-
-    Static = 0;
-    PICLevel = 0;
-
-    GNUInline = 0;
-    NoInline = 0;
-
-    Deprecated = 0;
-
-    CharIsSigned = 1;
-    ShortWChar = 0;
-    ShortEnums = 0;
-    CatchUndefined = 0;
-    DumpRecordLayouts = 0;
-    DumpVTableLayouts = 0;
-    SpellChecking = 1;
-    SinglePrecisionConstants = 0;
-    FastRelaxedMath = 0;
-    DefaultFPContract = 0;
-    NoBitFieldTypeAlign = 0;
-    FakeAddressSpaceMap = 0;
-    MRTD = 0;
-    DelayedTemplateParsing = 0;
-    ParseUnknownAnytype = DebuggerSupport = 0;
-  }
-
-  GCMode getGCMode() const { return (GCMode) GC; }
-  void setGCMode(GCMode m) { GC = (unsigned) m; }
-
-  StackProtectorMode getStackProtectorMode() const {
-    return static_cast<StackProtectorMode>(StackProtector);
-  }
-  void setStackProtectorMode(StackProtectorMode m) {
-    StackProtector = static_cast<unsigned>(m);
-  }
-
-  Visibility getVisibilityMode() const {
-    return (Visibility) SymbolVisibility;
-  }
-  void setVisibilityMode(Visibility v) { SymbolVisibility = (unsigned) v; }
-
-  SignedOverflowBehaviorTy getSignedOverflowBehavior() const {
-    return (SignedOverflowBehaviorTy)SignedOverflowBehavior;
-  }
-  void setSignedOverflowBehavior(SignedOverflowBehaviorTy V) {
-    SignedOverflowBehavior = (unsigned)V;
-  }
+  LangOptions();
 
+  // Define accessors/mutators for language options of enumeration type.
+#define LANGOPT(Name, Bits, Default, Description) 
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  Type get##Name() const { return static_cast<Type>(Name); } \
+  void set##Name(Type Value) { Name = static_cast<unsigned>(Value); }  
+#include "clang/Basic/LangOptions.def"
+  
   bool isSignedOverflowDefined() const {
     return getSignedOverflowBehavior() == SOB_Defined;
   }
+
+  /// \brief Reset all of the options that are not considered when building a
+  /// module.
+  void resetNonModularOptions();
 };
 
 /// Floating point control options
@@ -297,6 +95,18 @@ class OpenCLOptions {
   }
 };
 
+/// \brief Describes the kind of translation unit being processed.
+enum TranslationUnitKind {
+  /// \brief The translation unit is a complete translation unit.
+  TU_Complete,
+  /// \brief The translation unit is a prefix to a translation unit, and is
+  /// not complete.
+  TU_Prefix,
+  /// \brief The translation unit is a module.
+  TU_Module
+};
+  
+  /// \brief 
 }  // end namespace clang
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/MacroBuilder.h b/contrib/llvm/tools/clang/include/clang/Basic/MacroBuilder.h
index 3287b304c97f..1d0f1e899c9e 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/MacroBuilder.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/MacroBuilder.h
@@ -20,23 +20,23 @@
 namespace clang {
 
 class MacroBuilder {
-  llvm::raw_ostream &Out;
+  raw_ostream &Out;
 public:
-  MacroBuilder(llvm::raw_ostream &Output) : Out(Output) {}
+  MacroBuilder(raw_ostream &Output) : Out(Output) {}
 
   /// Append a #define line for macro of the form "#define Name Value\n".
-  void defineMacro(const llvm::Twine &Name, const llvm::Twine &Value = "1") {
+  void defineMacro(const Twine &Name, const Twine &Value = "1") {
     Out << "#define " << Name << ' ' << Value << '\n';
   }
 
   /// Append a #undef line for Name.  Name should be of the form XXX
   /// and we emit "#undef XXX".
-  void undefineMacro(const llvm::Twine &Name) {
+  void undefineMacro(const Twine &Name) {
     Out << "#undef " << Name << '\n';
   }
 
   /// Directly append Str and a newline to the underlying buffer.
-  void append(const llvm::Twine &Str) {
+  void append(const Twine &Str) {
     Out << Str << '\n';
   }
 };
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/OnDiskHashTable.h b/contrib/llvm/tools/clang/include/clang/Basic/OnDiskHashTable.h
index 267ecbcd6df4..7328b1a6bbfa 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/OnDiskHashTable.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/OnDiskHashTable.h
@@ -28,31 +28,31 @@ namespace io {
 
 typedef uint32_t Offset;
 
-inline void Emit8(llvm::raw_ostream& Out, uint32_t V) {
+inline void Emit8(raw_ostream& Out, uint32_t V) {
   Out << (unsigned char)(V);
 }
 
-inline void Emit16(llvm::raw_ostream& Out, uint32_t V) {
+inline void Emit16(raw_ostream& Out, uint32_t V) {
   Out << (unsigned char)(V);
   Out << (unsigned char)(V >>  8);
   assert((V >> 16) == 0);
 }
 
-inline void Emit24(llvm::raw_ostream& Out, uint32_t V) {
+inline void Emit24(raw_ostream& Out, uint32_t V) {
   Out << (unsigned char)(V);
   Out << (unsigned char)(V >>  8);
   Out << (unsigned char)(V >> 16);
   assert((V >> 24) == 0);
 }
 
-inline void Emit32(llvm::raw_ostream& Out, uint32_t V) {
+inline void Emit32(raw_ostream& Out, uint32_t V) {
   Out << (unsigned char)(V);
   Out << (unsigned char)(V >>  8);
   Out << (unsigned char)(V >> 16);
   Out << (unsigned char)(V >> 24);
 }
 
-inline void Emit64(llvm::raw_ostream& Out, uint64_t V) {
+inline void Emit64(raw_ostream& Out, uint64_t V) {
   Out << (unsigned char)(V);
   Out << (unsigned char)(V >>  8);
   Out << (unsigned char)(V >> 16);
@@ -63,7 +63,7 @@ inline void Emit64(llvm::raw_ostream& Out, uint64_t V) {
   Out << (unsigned char)(V >> 56);
 }
 
-inline void Pad(llvm::raw_ostream& Out, unsigned A) {
+inline void Pad(raw_ostream& Out, unsigned A) {
   Offset off = (Offset) Out.tell();
   uint32_t n = ((uintptr_t)(off+A-1) & ~(uintptr_t)(A-1)) - off;
   for (; n ; --n)
@@ -182,12 +182,12 @@ class OnDiskChainedHashTableGenerator {
                                                                InfoObj));
   }
 
-  io::Offset Emit(llvm::raw_ostream &out) {
+  io::Offset Emit(raw_ostream &out) {
     Info InfoObj;
     return Emit(out, InfoObj);
   }
 
-  io::Offset Emit(llvm::raw_ostream &out, Info &InfoObj) {
+  io::Offset Emit(raw_ostream &out, Info &InfoObj) {
     using namespace clang::io;
 
     // Emit the payload of the table.
@@ -464,6 +464,8 @@ class OnDiskChainedHashTable {
   }
   item_iterator item_end() { return item_iterator(); }
 
+  Info &getInfoObj() { return InfoObj; }
+  
   static OnDiskChainedHashTable* Create(const unsigned char* buckets,
                                         const unsigned char* const base,
                                         const Info &InfoObj = Info()) {
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/OpenCLExtensions.def b/contrib/llvm/tools/clang/include/clang/Basic/OpenCLExtensions.def
index 95cc1a9a513f..103fa839b8df 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/contrib/llvm/tools/clang/include/clang/Basic/OpenCLExtensions.def
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// OpenCL 1.1.
 OPENCLEXT(cl_khr_fp64)
 OPENCLEXT(cl_khr_int64_base_atomics)
 OPENCLEXT(cl_khr_int64_extended_atomics)
@@ -25,4 +26,7 @@ OPENCLEXT(cl_khr_local_int32_extended_atomics)
 OPENCLEXT(cl_khr_byte_addressable_store)
 OPENCLEXT(cl_khr_3d_image_writes)
 
+// Clang Extensions.
+OPENCLEXT(cl_clang_storage_class_specifiers)
+
 #undef OPENCLEXT
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h
index 7d7c0896f505..c6ca98976555 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h
@@ -33,7 +33,7 @@ class PartialDiagnostic {
         /// currently only support up to 10 arguments (%0-%9).
         /// A single diagnostic with more than that almost certainly has to
         /// be simplified anyway.
-        MaxArguments = 10
+        MaxArguments = DiagnosticsEngine::MaxArguments
     };
   
     /// NumDiagArgs - This contains the number of entries in Arguments.
@@ -65,7 +65,7 @@ class PartialDiagnostic {
     /// only support 10 ranges, could easily be extended if needed.
     CharSourceRange DiagRanges[10];
     
-    enum { MaxFixItHints = 3 };
+    enum { MaxFixItHints = DiagnosticsEngine::MaxFixItHints };
     
     /// FixItHints - If valid, provides a hint with some code
     /// to insert, remove, or modify at a particular position.
@@ -165,6 +165,8 @@ class PartialDiagnostic {
 
     assert(DiagStorage->NumFixItHints < Storage::MaxFixItHints &&
            "Too many code modification hints!");
+    if (DiagStorage->NumFixItHints >= Storage::MaxFixItHints)
+      return;  // Don't crash in release builds
     DiagStorage->FixItHints[DiagStorage->NumFixItHints++]
       = Hint;
   }
@@ -190,12 +192,12 @@ class PartialDiagnostic {
       *this->DiagStorage = *Other.DiagStorage;
   }
   
-  PartialDiagnostic(const DiagnosticInfo &Other, StorageAllocator &Allocator)
+  PartialDiagnostic(const Diagnostic &Other, StorageAllocator &Allocator)
     : DiagID(Other.getID()), DiagStorage(0), Allocator(&Allocator)
   {
     // Copy arguments.
     for (unsigned I = 0, N = Other.getNumArgs(); I != N; ++I) {
-      if (Other.getArgKind(I) == Diagnostic::ak_std_string)
+      if (Other.getArgKind(I) == DiagnosticsEngine::ak_std_string)
         AddString(Other.getArgStdStr(I));
       else
         AddTaggedVal(Other.getRawArg(I), Other.getArgKind(I));
@@ -230,7 +232,7 @@ class PartialDiagnostic {
 
   unsigned getDiagID() const { return DiagID; }
 
-  void AddTaggedVal(intptr_t V, Diagnostic::ArgumentKind Kind) const {
+  void AddTaggedVal(intptr_t V, DiagnosticsEngine::ArgumentKind Kind) const {
     if (!DiagStorage)
       DiagStorage = getStorage();
 
@@ -240,14 +242,14 @@ class PartialDiagnostic {
     DiagStorage->DiagArgumentsVal[DiagStorage->NumDiagArgs++] = V;
   }
 
-  void AddString(llvm::StringRef V) const {
+  void AddString(StringRef V) const {
     if (!DiagStorage)
       DiagStorage = getStorage();
     
     assert(DiagStorage->NumDiagArgs < Storage::MaxArguments &&
            "Too many arguments to diagnostic!");
     DiagStorage->DiagArgumentsKind[DiagStorage->NumDiagArgs]
-      = Diagnostic::ak_std_string;
+      = DiagnosticsEngine::ak_std_string;
     DiagStorage->DiagArgumentsStr[DiagStorage->NumDiagArgs++] = V;
   }
 
@@ -257,12 +259,12 @@ class PartialDiagnostic {
     
     // Add all arguments.
     for (unsigned i = 0, e = DiagStorage->NumDiagArgs; i != e; ++i) {
-      if ((Diagnostic::ArgumentKind)DiagStorage->DiagArgumentsKind[i]
-            == Diagnostic::ak_std_string)
+      if ((DiagnosticsEngine::ArgumentKind)DiagStorage->DiagArgumentsKind[i]
+            == DiagnosticsEngine::ak_std_string)
         DB.AddString(DiagStorage->DiagArgumentsStr[i]);
       else
         DB.AddTaggedVal(DiagStorage->DiagArgumentsVal[i],
-                   (Diagnostic::ArgumentKind)DiagStorage->DiagArgumentsKind[i]);
+            (DiagnosticsEngine::ArgumentKind)DiagStorage->DiagArgumentsKind[i]);
     }
     
     // Add all ranges.
@@ -285,24 +287,25 @@ class PartialDiagnostic {
   
   friend const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                              unsigned I) {
-    PD.AddTaggedVal(I, Diagnostic::ak_uint);
+    PD.AddTaggedVal(I, DiagnosticsEngine::ak_uint);
     return PD;
   }
 
   friend const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                              int I) {
-    PD.AddTaggedVal(I, Diagnostic::ak_sint);
+    PD.AddTaggedVal(I, DiagnosticsEngine::ak_sint);
     return PD;
   }
 
   friend inline const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                                     const char *S) {
-    PD.AddTaggedVal(reinterpret_cast<intptr_t>(S), Diagnostic::ak_c_string);
+    PD.AddTaggedVal(reinterpret_cast<intptr_t>(S),
+                    DiagnosticsEngine::ak_c_string);
     return PD;
   }
 
   friend inline const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
-                                                    llvm::StringRef S) {
+                                                    StringRef S) {
     
     PD.AddString(S);
     return PD;
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/PrettyStackTrace.h b/contrib/llvm/tools/clang/include/clang/Basic/PrettyStackTrace.h
index 5a5d55192b69..06a12644c36e 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/PrettyStackTrace.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/PrettyStackTrace.h
@@ -30,7 +30,7 @@ namespace clang {
   public:
     PrettyStackTraceLoc(SourceManager &sm, SourceLocation L, const char *Msg)
       : SM(sm), Loc(L), Message(Msg) {}
-    virtual void print(llvm::raw_ostream &OS) const;
+    virtual void print(raw_ostream &OS) const;
   };
 }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/SourceLocation.h b/contrib/llvm/tools/clang/include/clang/Basic/SourceLocation.h
index ee5f96fe937a..154148c16b03 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/SourceLocation.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/SourceLocation.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_SOURCELOCATION_H
 #define LLVM_CLANG_SOURCELOCATION_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include <utility>
 #include <functional>
@@ -21,8 +22,6 @@
 
 namespace llvm {
   class MemoryBuffer;
-  class raw_ostream;
-  class StringRef;
   template <typename T> struct DenseMapInfo;
   template <typename T> struct isPodLike;
 }
@@ -35,8 +34,9 @@ class SourceManager;
 /// a source file (MemoryBuffer) along with its #include path and #line data.
 ///
 class FileID {
-  /// ID - Opaque identifier, 0 is "invalid".
-  unsigned ID;
+  /// ID - Opaque identifier, 0 is "invalid". >0 is this module, <-1 is
+  /// something loaded from another module.
+  int ID;
 public:
   FileID() : ID(0) {}
 
@@ -49,49 +49,63 @@ class FileID {
   bool operator>(const FileID &RHS) const { return RHS < *this; }
   bool operator>=(const FileID &RHS) const { return RHS <= *this; }
 
-  static FileID getSentinel() { return get(~0U); }
-  unsigned getHashValue() const { return ID; }
+  static FileID getSentinel() { return get(-1); }
+  unsigned getHashValue() const { return static_cast<unsigned>(ID); }
 
 private:
   friend class SourceManager;
   friend class ASTWriter;
   friend class ASTReader;
   
-  static FileID get(unsigned V) {
+  static FileID get(int V) {
     FileID F;
     F.ID = V;
     return F;
   }
-  unsigned getOpaqueValue() const { return ID; }
+  int getOpaqueValue() const { return ID; }
 };
 
 
-/// SourceLocation - This is a carefully crafted 32-bit identifier that encodes
-/// a full include stack, line and column number information for a position in
-/// an input translation unit.
+/// \brief Encodes a location in the source. The SourceManager can decode this
+/// to get at the full include stack, line and column information.
+///
+/// Technically, a source location is simply an offset into the manager's view
+/// of the input source, which is all input buffers (including macro
+/// expansions) concatenated in an effectively arbitrary order. The manager
+/// actually maintains two blocks of input buffers. One, starting at offset
+/// 0 and growing upwards, contains all buffers from this module. The other,
+/// starting at the highest possible offset and growing downwards, contains
+/// buffers of loaded modules.
+///
+/// In addition, one bit of SourceLocation is used for quick access to the
+/// information whether the location is in a file or a macro expansion.
+///
+/// It is important that this type remains small. It is currently 32 bits wide.
 class SourceLocation {
   unsigned ID;
   friend class SourceManager;
+  friend class ASTReader;
+  friend class ASTWriter;
   enum {
     MacroIDBit = 1U << 31
   };
 public:
 
-  SourceLocation() : ID(0) {}  // 0 is an invalid FileID.
+  SourceLocation() : ID(0) {}
 
   bool isFileID() const  { return (ID & MacroIDBit) == 0; }
   bool isMacroID() const { return (ID & MacroIDBit) != 0; }
 
-  /// isValid - Return true if this is a valid SourceLocation object.  Invalid
-  /// SourceLocations are often used when events have no corresponding location
-  /// in the source (e.g. a diagnostic is required for a command line option).
+  /// \brief Return true if this is a valid SourceLocation object.
   ///
+  /// Invalid SourceLocations are often used when events have no corresponding
+  /// location in the source (e.g. a diagnostic is required for a command line
+  /// option).
   bool isValid() const { return ID != 0; }
   bool isInvalid() const { return ID == 0; }
 
 private:
-  /// getOffset - Return the index for SourceManager's SLocEntryTable table,
-  /// note that this is not an index *into* it though.
+  /// \brief Return the offset into the manager's global input view.
   unsigned getOffset() const {
     return ID & ~MacroIDBit;
   }
@@ -111,10 +125,10 @@ class SourceLocation {
   }
 public:
 
-  /// getFileLocWithOffset - Return a source location with the specified offset
-  /// from this file SourceLocation.
-  SourceLocation getFileLocWithOffset(int Offset) const {
-    assert(((getOffset()+Offset) & MacroIDBit) == 0 && "invalid location");
+  /// \brief Return a source location with the specified offset from this
+  /// SourceLocation.
+  SourceLocation getLocWithOffset(int Offset) const {
+    assert(((getOffset()+Offset) & MacroIDBit) == 0 && "offset overflow");
     SourceLocation L;
     L.ID = ID+Offset;
     return L;
@@ -150,7 +164,7 @@ class SourceLocation {
     return getFromRawEncoding((unsigned)(uintptr_t)Encoding);
   }
 
-  void print(llvm::raw_ostream &OS, const SourceManager &SM) const;
+  void print(raw_ostream &OS, const SourceManager &SM) const;
   void dump(const SourceManager &SM) const;
 };
 
@@ -261,11 +275,11 @@ class FullSourceLoc : public SourceLocation {
 
   FileID getFileID() const;
 
-  FullSourceLoc getInstantiationLoc() const;
+  FullSourceLoc getExpansionLoc() const;
   FullSourceLoc getSpellingLoc() const;
 
-  unsigned getInstantiationLineNumber(bool *Invalid = 0) const;
-  unsigned getInstantiationColumnNumber(bool *Invalid = 0) const;
+  unsigned getExpansionLineNumber(bool *Invalid = 0) const;
+  unsigned getExpansionColumnNumber(bool *Invalid = 0) const;
 
   unsigned getSpellingLineNumber(bool *Invalid = 0) const;
   unsigned getSpellingColumnNumber(bool *Invalid = 0) const;
@@ -276,7 +290,7 @@ class FullSourceLoc : public SourceLocation {
 
   /// getBufferData - Return a StringRef to the source buffer data for the
   /// specified FileID.
-  llvm::StringRef getBufferData(bool *Invalid = 0) const;
+  StringRef getBufferData(bool *Invalid = 0) const;
 
   /// getDecomposedLoc - Decompose the specified location into a raw FileID +
   /// Offset pair.  The first element is the FileID, the second is the
@@ -326,8 +340,8 @@ class FullSourceLoc : public SourceLocation {
 
 /// PresumedLoc - This class represents an unpacked "presumed" location which
 /// can be presented to the user.  A 'presumed' location can be modified by
-/// #line and GNU line marker directives and is always the instantiation point
-/// of a normal location.
+/// #line and GNU line marker directives and is always the expansion point of
+/// a normal location.
 ///
 /// You can get a PresumedLoc from a SourceLocation with SourceManager.
 class PresumedLoc {
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/SourceManager.h b/contrib/llvm/tools/clang/include/clang/Basic/SourceManager.h
index 6301f3197859..985ddd641271 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/SourceManager.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/SourceManager.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_SOURCEMANAGER_H
 #define LLVM_CLANG_SOURCEMANAGER_H
 
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/DataTypes.h"
@@ -22,22 +23,37 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include <map>
 #include <vector>
 #include <cassert>
 
-namespace llvm {
-class StringRef;
-}
-
 namespace clang {
 
-class Diagnostic;
+class DiagnosticsEngine;
 class SourceManager;
 class FileManager;
 class FileEntry;
 class LineTableInfo;
 class LangOptions;
-  
+class ASTWriter;
+class ASTReader;
+
+/// There are three different types of locations in a file: a spelling
+/// location, an expansion location, and a presumed location.
+///
+/// Given an example of:
+/// #define min(x, y) x < y ? x : y
+///
+/// and then later on a use of min:
+/// #line 17
+/// return min(a, b);
+///
+/// The expansion location is the line in the source code where the macro
+/// was expanded (the return statement), the spelling location is the
+/// location in the source where the macro was originally defined,
+/// and the presumed location is where the line directive states that
+/// the line is 17, or any other line.
+
 /// SrcMgr - Public enums and private classes that are part of the
 /// SourceManager implementation.
 ///
@@ -46,7 +62,7 @@ namespace SrcMgr {
   /// holds normal user code, system code, or system code which is implicitly
   /// 'extern "C"' in C++ mode.  Entire directories can be tagged with this
   /// (this is maintained by DirectoryLookup and friends) as can specific
-  /// FileIDInfos when a #pragma system_header is seen or various other cases.
+  /// FileInfos when a #pragma system_header is seen or various other cases.
   ///
   enum CharacteristicKind {
     C_User, C_System, C_ExternCSystem
@@ -61,7 +77,7 @@ namespace SrcMgr {
       /// \brief Whether the buffer should not be freed on destruction.
       DoNotFreeFlag = 0x02
     };
-    
+
     /// Buffer - The actual buffer containing the characters from the input
     /// file.  This is owned by the ContentCache object.
     /// The bits indicate indicates whether the buffer is invalid.
@@ -92,12 +108,12 @@ namespace SrcMgr {
     ///
     /// \param Diag Object through which diagnostics will be emitted if the
     /// buffer cannot be retrieved.
-    /// 
+    ///
     /// \param Loc If specified, is the location that invalid file diagnostics
     ///     will be emitted at.
     ///
     /// \param Invalid If non-NULL, will be set \c true if an error occurred.
-    const llvm::MemoryBuffer *getBuffer(Diagnostic &Diag,
+    const llvm::MemoryBuffer *getBuffer(DiagnosticsEngine &Diag,
                                         const SourceManager &SM,
                                         SourceLocation Loc = SourceLocation(),
                                         bool *Invalid = 0) const;
@@ -109,10 +125,10 @@ namespace SrcMgr {
     unsigned getSize() const;
 
     /// getSizeBytesMapped - Returns the number of bytes actually mapped for
-    ///  this ContentCache.  This can be 0 if the MemBuffer was not actually
-    ///  instantiated.
+    /// this ContentCache. This can be 0 if the MemBuffer was not actually
+    /// expanded.
     unsigned getSizeBytesMapped() const;
-    
+
     /// Returns the kind of memory used to back the memory buffer for
     /// this content cache.  This is used for performance analysis.
     llvm::MemoryBuffer::BufferKind getMemoryBufferKind() const;
@@ -122,7 +138,7 @@ namespace SrcMgr {
       Buffer.setPointer(B);
       Buffer.setInt(false);
     }
-    
+
     /// \brief Get the underlying buffer, returning NULL if the buffer is not
     /// yet available.
     const llvm::MemoryBuffer *getRawBuffer() const {
@@ -137,12 +153,12 @@ namespace SrcMgr {
     bool isBufferInvalid() const {
       return Buffer.getInt() & InvalidFlag;
     }
-    
+
     /// \brief Determine whether the buffer should be freed.
     bool shouldFreeBuffer() const {
       return (Buffer.getInt() & DoNotFreeFlag) == 0;
     }
-    
+
     ContentCache(const FileEntry *Ent = 0)
       : Buffer(0, false), OrigEntry(Ent), ContentsEntry(Ent),
         SourceLineCache(0), NumLines(0) {}
@@ -156,14 +172,14 @@ namespace SrcMgr {
     /// The copy ctor does not allow copies where source object has either
     ///  a non-NULL Buffer or SourceLineCache.  Ownership of allocated memory
     ///  is not transferred, so this is a logical error.
-    ContentCache(const ContentCache &RHS) 
-      : Buffer(0, false), SourceLineCache(0) 
+    ContentCache(const ContentCache &RHS)
+      : Buffer(0, false), SourceLineCache(0)
     {
       OrigEntry = RHS.OrigEntry;
       ContentsEntry = RHS.ContentsEntry;
 
-      assert (RHS.Buffer.getPointer() == 0 && RHS.SourceLineCache == 0
-              && "Passed ContentCache object cannot own a buffer.");
+      assert (RHS.Buffer.getPointer() == 0 && RHS.SourceLineCache == 0 &&
+              "Passed ContentCache object cannot own a buffer.");
 
       NumLines = RHS.NumLines;
     }
@@ -177,8 +193,8 @@ namespace SrcMgr {
   /// that it represents and include stack information.
   ///
   /// Each FileInfo has include stack information, indicating where it came
-  /// from.  This information encodes the #include chain that a token was
-  /// instantiated from.  The main include file has an invalid IncludeLoc.
+  /// from. This information encodes the #include chain that a token was
+  /// expanded from. The main include file has an invalid IncludeLoc.
   ///
   /// FileInfos contain a "ContentCache *", with the contents of the file.
   ///
@@ -187,16 +203,26 @@ namespace SrcMgr {
     /// This is an invalid SLOC for the main file (top of the #include chain).
     unsigned IncludeLoc;  // Really a SourceLocation
 
+    /// \brief Number of FileIDs (files and macros) that were created during
+    /// preprocessing of this #include, including this SLocEntry.
+    /// Zero means the preprocessor didn't provide such info for this SLocEntry.
+    unsigned NumCreatedFIDs;
+
     /// Data - This contains the ContentCache* and the bits indicating the
     /// characteristic of the file and whether it has #line info, all bitmangled
     /// together.
     uintptr_t Data;
+
+    friend class clang::SourceManager;
+    friend class clang::ASTWriter;
+    friend class clang::ASTReader;
   public:
     /// get - Return a FileInfo object.
     static FileInfo get(SourceLocation IL, const ContentCache *Con,
                         CharacteristicKind FileCharacter) {
       FileInfo X;
       X.IncludeLoc = IL.getRawEncoding();
+      X.NumCreatedFIDs = 0;
       X.Data = (uintptr_t)Con;
       assert((X.Data & 7) == 0 &&"ContentCache pointer insufficiently aligned");
       assert((unsigned)FileCharacter < 4 && "invalid file character");
@@ -227,70 +253,68 @@ namespace SrcMgr {
     }
   };
 
-  /// InstantiationInfo - Each InstantiationInfo encodes the Instantiation
-  /// location - where the token was ultimately instantiated, and the
-  /// SpellingLoc - where the actual character data for the token came from.
-  class InstantiationInfo {
-     // Really these are all SourceLocations.
+  /// ExpansionInfo - Each ExpansionInfo encodes the expansion location - where
+  /// the token was ultimately expanded, and the SpellingLoc - where the actual
+  /// character data for the token came from.
+  class ExpansionInfo {
+    // Really these are all SourceLocations.
 
     /// SpellingLoc - Where the spelling for the token can be found.
     unsigned SpellingLoc;
 
-    /// InstantiationLocStart/InstantiationLocEnd - In a macro expansion, these
-    /// indicate the start and end of the instantiation.  In object-like macros,
-    /// these will be the same.  In a function-like macro instantiation, the
-    /// start will be the identifier and the end will be the ')'.  Finally, in
+    /// ExpansionLocStart/ExpansionLocEnd - In a macro expansion, these
+    /// indicate the start and end of the expansion. In object-like macros,
+    /// these will be the same. In a function-like macro expansion, the start
+    /// will be the identifier and the end will be the ')'. Finally, in
     /// macro-argument instantitions, the end will be 'SourceLocation()', an
     /// invalid location.
-    unsigned InstantiationLocStart, InstantiationLocEnd;
+    unsigned ExpansionLocStart, ExpansionLocEnd;
 
   public:
     SourceLocation getSpellingLoc() const {
       return SourceLocation::getFromRawEncoding(SpellingLoc);
     }
-    SourceLocation getInstantiationLocStart() const {
-      return SourceLocation::getFromRawEncoding(InstantiationLocStart);
+    SourceLocation getExpansionLocStart() const {
+      return SourceLocation::getFromRawEncoding(ExpansionLocStart);
     }
-    SourceLocation getInstantiationLocEnd() const {
+    SourceLocation getExpansionLocEnd() const {
       SourceLocation EndLoc =
-        SourceLocation::getFromRawEncoding(InstantiationLocEnd);
-      return EndLoc.isInvalid() ? getInstantiationLocStart() : EndLoc;
+        SourceLocation::getFromRawEncoding(ExpansionLocEnd);
+      return EndLoc.isInvalid() ? getExpansionLocStart() : EndLoc;
     }
 
-    std::pair<SourceLocation,SourceLocation> getInstantiationLocRange() const {
-      return std::make_pair(getInstantiationLocStart(),
-                            getInstantiationLocEnd());
+    std::pair<SourceLocation,SourceLocation> getExpansionLocRange() const {
+      return std::make_pair(getExpansionLocStart(), getExpansionLocEnd());
     }
 
-    bool isMacroArgInstantiation() const {
+    bool isMacroArgExpansion() const {
       // Note that this needs to return false for default constructed objects.
-      return getInstantiationLocStart().isValid() &&
-        SourceLocation::getFromRawEncoding(InstantiationLocEnd).isInvalid();
+      return getExpansionLocStart().isValid() &&
+        SourceLocation::getFromRawEncoding(ExpansionLocEnd).isInvalid();
     }
 
-    /// create - Return a InstantiationInfo for an expansion. ILStart and
-    /// ILEnd specify the instantiation range (where the macro is expanded),
-    /// and SL specifies the spelling location (where the characters from the
-    /// token come from). All three can refer to normal File SLocs or
-    /// instantiation locations.
-    static InstantiationInfo create(SourceLocation SL,
-                                    SourceLocation ILStart,
-                                    SourceLocation ILEnd) {
-      InstantiationInfo X;
-      X.SpellingLoc = SL.getRawEncoding();
-      X.InstantiationLocStart = ILStart.getRawEncoding();
-      X.InstantiationLocEnd = ILEnd.getRawEncoding();
+    /// create - Return a ExpansionInfo for an expansion. Start and End specify
+    /// the expansion range (where the macro is expanded), and SpellingLoc
+    /// specifies the spelling location (where the characters from the token
+    /// come from). All three can refer to normal File SLocs or expansion
+    /// locations.
+    static ExpansionInfo create(SourceLocation SpellingLoc,
+                                SourceLocation Start, SourceLocation End) {
+      ExpansionInfo X;
+      X.SpellingLoc = SpellingLoc.getRawEncoding();
+      X.ExpansionLocStart = Start.getRawEncoding();
+      X.ExpansionLocEnd = End.getRawEncoding();
       return X;
     }
 
-    /// createForMacroArg - Return a special InstantiationInfo for the
-    /// expansion of a macro argument into a function-like macro's body. IL
-    /// specifies the instantiation location (where the macro is expanded).
-    /// This doesn't need to be a range because a macro is always instantiated
-    /// at a macro parameter reference, and macro parameters are always exactly
-    /// one token. SL specifies the spelling location (where the characters
-    /// from the token come from). IL and SL can both refer to normal File
-    /// SLocs or instantiation locations.
+    /// createForMacroArg - Return a special ExpansionInfo for the expansion of
+    /// a macro argument into a function-like macro's body. ExpansionLoc
+    /// specifies the expansion location (where the macro is expanded). This
+    /// doesn't need to be a range because a macro is always expanded at
+    /// a macro parameter reference, and macro parameters are always exactly
+    /// one token. SpellingLoc specifies the spelling location (where the
+    /// characters from the token come from). ExpansionLoc and SpellingLoc can
+    /// both refer to normal File SLocs or expansion locations.
     ///
     /// Given the code:
     /// \code
@@ -298,41 +322,41 @@ namespace SrcMgr {
     ///   F(42);
     /// \endcode
     ///
-    /// When expanding '\c F(42)', the '\c x' would call this with an SL
-    /// pointing at '\c 42' anad an IL pointing at its location in the
-    /// definition of '\c F'.
-    static InstantiationInfo createForMacroArg(SourceLocation SL,
-                                               SourceLocation IL) {
+    /// When expanding '\c F(42)', the '\c x' would call this with an
+    /// SpellingLoc pointing at '\c 42' anad an ExpansionLoc pointing at its
+    /// location in the definition of '\c F'.
+    static ExpansionInfo createForMacroArg(SourceLocation SpellingLoc,
+                                           SourceLocation ExpansionLoc) {
       // We store an intentionally invalid source location for the end of the
-      // instantiation range to mark that this is a macro argument instantation
-      // rather than a normal one.
-      return create(SL, IL, SourceLocation());
+      // expansion range to mark that this is a macro argument ion rather than
+      // a normal one.
+      return create(SpellingLoc, ExpansionLoc, SourceLocation());
     }
   };
 
   /// SLocEntry - This is a discriminated union of FileInfo and
-  /// InstantiationInfo.  SourceManager keeps an array of these objects, and
+  /// ExpansionInfo.  SourceManager keeps an array of these objects, and
   /// they are uniquely identified by the FileID datatype.
   class SLocEntry {
-    unsigned Offset;   // low bit is set for instantiation info.
+    unsigned Offset;   // low bit is set for expansion info.
     union {
       FileInfo File;
-      InstantiationInfo Instantiation;
+      ExpansionInfo Expansion;
     };
   public:
     unsigned getOffset() const { return Offset >> 1; }
 
-    bool isInstantiation() const { return Offset & 1; }
-    bool isFile() const { return !isInstantiation(); }
+    bool isExpansion() const { return Offset & 1; }
+    bool isFile() const { return !isExpansion(); }
 
     const FileInfo &getFile() const {
       assert(isFile() && "Not a file SLocEntry!");
       return File;
     }
 
-    const InstantiationInfo &getInstantiation() const {
-      assert(isInstantiation() && "Not an instantiation SLocEntry!");
-      return Instantiation;
+    const ExpansionInfo &getExpansion() const {
+      assert(isExpansion() && "Not a macro expansion SLocEntry!");
+      return Expansion;
     }
 
     static SLocEntry get(unsigned Offset, const FileInfo &FI) {
@@ -342,10 +366,10 @@ namespace SrcMgr {
       return E;
     }
 
-    static SLocEntry get(unsigned Offset, const InstantiationInfo &II) {
+    static SLocEntry get(unsigned Offset, const ExpansionInfo &Expansion) {
       SLocEntry E;
       E.Offset = (Offset << 1) | 1;
-      E.Instantiation = II;
+      E.Expansion = Expansion;
       return E;
     }
   };
@@ -356,13 +380,14 @@ class ExternalSLocEntrySource {
 public:
   virtual ~ExternalSLocEntrySource();
 
-  /// \brief Read the source location entry with index ID.
+  /// \brief Read the source location entry with index ID, which will always be
+  /// less than -1.
   ///
   /// \returns true if an error occurred that prevented the source-location
   /// entry from being loaded.
-  virtual bool ReadSLocEntry(unsigned ID) = 0;
+  virtual bool ReadSLocEntry(int ID) = 0;
 };
-  
+
 
 /// IsBeforeInTranslationUnitCache - This class holds the cache used by
 /// isBeforeInTranslationUnit.  The cache structure is complex enough to be
@@ -371,24 +396,28 @@ class IsBeforeInTranslationUnitCache {
   /// L/R QueryFID - These are the FID's of the cached query.  If these match up
   /// with a subsequent query, the result can be reused.
   FileID LQueryFID, RQueryFID;
-  
+
+  /// \brief True if LQueryFID was created before RQueryFID. This is used
+  /// to compare macro expansion locations.
+  bool IsLQFIDBeforeRQFID;
+
   /// CommonFID - This is the file found in common between the two #include
   /// traces.  It is the nearest common ancestor of the #include tree.
   FileID CommonFID;
-  
+
   /// L/R CommonOffset - This is the offset of the previous query in CommonFID.
   /// Usually, this represents the location of the #include for QueryFID, but if
   /// LQueryFID is a parent of RQueryFID (or vise versa) then these can be a
   /// random token in the parent.
   unsigned LCommonOffset, RCommonOffset;
 public:
-  
+
   /// isCacheValid - Return true if the currently cached values match up with
   /// the specified LHS/RHS query.  If not, we can't use the cache.
   bool isCacheValid(FileID LHS, FileID RHS) const {
     return LQueryFID == LHS && RQueryFID == RHS;
   }
-  
+
   /// getCachedResult - If the cache is valid, compute the result given the
   /// specified offsets in the LHS/RHS FID's.
   bool getCachedResult(unsigned LOffset, unsigned ROffset) const {
@@ -396,38 +425,56 @@ class IsBeforeInTranslationUnitCache {
     // use the #include loc in the common file.
     if (LQueryFID != CommonFID) LOffset = LCommonOffset;
     if (RQueryFID != CommonFID) ROffset = RCommonOffset;
+
+    // It is common for multiple macro expansions to be "included" from the same
+    // location (expansion location), in which case use the order of the FileIDs
+    // to determine which came first. This will also take care the case where
+    // one of the locations points at the inclusion/expansion point of the other
+    // in which case its FileID will come before the other.
+    if (LOffset == ROffset &&
+        (LQueryFID != CommonFID || RQueryFID != CommonFID))
+      return IsLQFIDBeforeRQFID;
+
     return LOffset < ROffset;
   }
-  
+
   // Set up a new query.
-  void setQueryFIDs(FileID LHS, FileID RHS) {
+  void setQueryFIDs(FileID LHS, FileID RHS, bool isLFIDBeforeRFID) {
+    assert(LHS != RHS);
     LQueryFID = LHS;
     RQueryFID = RHS;
+    IsLQFIDBeforeRQFID = isLFIDBeforeRFID;
   }
-  
+
+  void clear() {
+    LQueryFID = RQueryFID = FileID();
+    IsLQFIDBeforeRQFID = false;
+  }
+
   void setCommonLoc(FileID commonFID, unsigned lCommonOffset,
                     unsigned rCommonOffset) {
     CommonFID = commonFID;
     LCommonOffset = lCommonOffset;
     RCommonOffset = rCommonOffset;
   }
-  
+
 };
 
-/// SourceManager - This file handles loading and caching of source files into
-/// memory.  This object owns the MemoryBuffer objects for all of the loaded
+/// \brief This class handles loading and caching of source files into memory.
+///
+/// This object owns the MemoryBuffer objects for all of the loaded
 /// files and assigns unique FileID's for each unique #include chain.
 ///
 /// The SourceManager can be queried for information about SourceLocation
-/// objects, turning them into either spelling or instantiation locations.
-/// Spelling locations represent where the bytes corresponding to a token came
-/// from and instantiation locations represent where the location is in the
-/// user's view.  In the case of a macro expansion, for example, the spelling
-/// location indicates  where the expanded token came from and the instantiation
-/// location specifies where it was expanded.
+/// objects, turning them into either spelling or expansion locations. Spelling
+/// locations represent where the bytes corresponding to a token came from and
+/// expansion locations represent where the location is in the user's view. In
+/// the case of a macro expansion, for example, the spelling location indicates
+/// where the expanded token came from and the expansion location specifies
+/// where it was expanded.
 class SourceManager : public llvm::RefCountedBase<SourceManager> {
-  /// \brief Diagnostic object.
-  Diagnostic &Diag;
+  /// \brief DiagnosticsEngine object.
+  DiagnosticsEngine &Diag;
 
   FileManager &FileMgr;
 
@@ -451,16 +498,37 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// as they do not refer to a file.
   std::vector<SrcMgr::ContentCache*> MemBufferInfos;
 
-  /// SLocEntryTable - This is an array of SLocEntry's that we have created.
-  /// FileID is an index into this vector.  This array is sorted by the offset.
-  std::vector<SrcMgr::SLocEntry> SLocEntryTable;
-  /// NextOffset - This is the next available offset that a new SLocEntry can
-  /// start at.  It is SLocEntryTable.back().getOffset()+size of back() entry.
-  unsigned NextOffset;
+  /// \brief The table of SLocEntries that are local to this module.
+  ///
+  /// Positive FileIDs are indexes into this table. Entry 0 indicates an invalid
+  /// expansion.
+  std::vector<SrcMgr::SLocEntry> LocalSLocEntryTable;
 
-  /// \brief If source location entries are being lazily loaded from
-  /// an external source, this vector indicates whether the Ith source
-  /// location entry has already been loaded from the external storage.
+  /// \brief The table of SLocEntries that are loaded from other modules.
+  ///
+  /// Negative FileIDs are indexes into this table. To get from ID to an index,
+  /// use (-ID - 2).
+  std::vector<SrcMgr::SLocEntry> LoadedSLocEntryTable;
+
+  /// \brief The starting offset of the next local SLocEntry.
+  ///
+  /// This is LocalSLocEntryTable.back().Offset + the size of that entry.
+  unsigned NextLocalOffset;
+
+  /// \brief The starting offset of the latest batch of loaded SLocEntries.
+  ///
+  /// This is LoadedSLocEntryTable.back().Offset, except that that entry might
+  /// not have been loaded, so that value would be unknown.
+  unsigned CurrentLoadedOffset;
+
+  /// \brief The highest possible offset is 2^31-1, so CurrentLoadedOffset
+  /// starts at 2^31.
+  static const unsigned MaxLoadedOffset = 1U << 31U;
+
+  /// \brief A bitmap that indicates whether the entries of LoadedSLocEntryTable
+  /// have already been loaded from the external source.
+  ///
+  /// Same indexing as LoadedSLocEntryTable.
   std::vector<bool> SLocEntryLoaded;
 
   /// \brief An external source for source location entries.
@@ -485,6 +553,9 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// MainFileID - The file ID for the main source file of the translation unit.
   FileID MainFileID;
 
+  /// \brief The file ID for the precompiled preamble there is one.
+  FileID PreambleFileID;
+
   // Statistics for -print-stats.
   mutable unsigned NumLinearScans, NumBinaryProbes;
 
@@ -493,17 +564,23 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
 
   // Cache for the "fake" buffer used for error-recovery purposes.
   mutable llvm::MemoryBuffer *FakeBufferForRecovery;
-  
+
+  /// \brief Lazily computed map of macro argument chunks to their expanded
+  /// source location.
+  typedef std::map<unsigned, SourceLocation> MacroArgsMap;
+
+  mutable llvm::DenseMap<FileID, MacroArgsMap *> MacroArgsCacheMap; 
+
   // SourceManager doesn't support copy construction.
   explicit SourceManager(const SourceManager&);
   void operator=(const SourceManager&);
 public:
-  SourceManager(Diagnostic &Diag, FileManager &FileMgr);
+  SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr);
   ~SourceManager();
 
   void clearIDTables();
 
-  Diagnostic &getDiagnostics() const { return Diag; }
+  DiagnosticsEngine &getDiagnostics() const { return Diag; }
 
   FileManager &getFileManager() const { return FileMgr; }
 
@@ -513,6 +590,15 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     OverridenFilesKeepOriginalName = value;
   }
 
+  /// createMainFileIDForMembuffer - Create the FileID for a memory buffer
+  ///  that will represent the FileID for the main source.  One example
+  ///  of when this would be used is when the main source is read from STDIN.
+  FileID createMainFileIDForMemBuffer(const llvm::MemoryBuffer *Buffer) {
+    assert(MainFileID.isInvalid() && "MainFileID already set!");
+    MainFileID = createFileIDForMemBuffer(Buffer);
+    return MainFileID;
+  }
+
   //===--------------------------------------------------------------------===//
   // MainFileID creation and querying methods.
   //===--------------------------------------------------------------------===//
@@ -527,67 +613,56 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     return MainFileID;
   }
 
-  /// \brief Set the file ID for the precompiled preamble, which is also the
-  /// main file.
-  void SetPreambleFileID(FileID Preamble) {
-    assert(MainFileID.isInvalid() && "MainFileID already set!");
-    MainFileID = Preamble;
+  /// \brief Set the file ID for the precompiled preamble.
+  void setPreambleFileID(FileID Preamble) {
+    assert(PreambleFileID.isInvalid() && "PreambleFileID already set!");
+    PreambleFileID = Preamble;
   }
-  
+
+  /// \brief Get the file ID for the precompiled preamble if there is one.
+  FileID getPreambleFileID() const { return PreambleFileID; }
+
   //===--------------------------------------------------------------------===//
-  // Methods to create new FileID's and instantiations.
+  // Methods to create new FileID's and macro expansions.
   //===--------------------------------------------------------------------===//
 
   /// createFileID - Create a new FileID that represents the specified file
   /// being #included from the specified IncludePosition.  This translates NULL
   /// into standard input.
-  /// PreallocateID should be non-zero to specify which pre-allocated,
-  /// lazily computed source location is being filled in by this operation.
   FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos,
                       SrcMgr::CharacteristicKind FileCharacter,
-                      unsigned PreallocatedID = 0,
-                      unsigned Offset = 0) {
+                      int LoadedID = 0, unsigned LoadedOffset = 0) {
     const SrcMgr::ContentCache *IR = getOrCreateContentCache(SourceFile);
     assert(IR && "getOrCreateContentCache() cannot return NULL");
-    return createFileID(IR, IncludePos, FileCharacter, PreallocatedID, Offset);
+    return createFileID(IR, IncludePos, FileCharacter, LoadedID, LoadedOffset);
   }
 
   /// createFileIDForMemBuffer - Create a new FileID that represents the
   /// specified memory buffer.  This does no caching of the buffer and takes
   /// ownership of the MemoryBuffer, so only pass a MemoryBuffer to this once.
   FileID createFileIDForMemBuffer(const llvm::MemoryBuffer *Buffer,
-                                  unsigned PreallocatedID = 0,
-                                  unsigned Offset = 0) {
+                                  int LoadedID = 0, unsigned LoadedOffset = 0) {
     return createFileID(createMemBufferContentCache(Buffer), SourceLocation(),
-                        SrcMgr::C_User, PreallocatedID, Offset);
+                        SrcMgr::C_User, LoadedID, LoadedOffset);
   }
 
-  /// createMainFileIDForMembuffer - Create the FileID for a memory buffer
-  ///  that will represent the FileID for the main source.  One example
-  ///  of when this would be used is when the main source is read from STDIN.
-  FileID createMainFileIDForMemBuffer(const llvm::MemoryBuffer *Buffer) {
-    assert(MainFileID.isInvalid() && "MainFileID already set!");
-    MainFileID = createFileIDForMemBuffer(Buffer);
-    return MainFileID;
-  }
+  /// createMacroArgExpansionLoc - Return a new SourceLocation that encodes the
+  /// fact that a token from SpellingLoc should actually be referenced from
+  /// ExpansionLoc, and that it represents the expansion of a macro argument
+  /// into the function-like macro body.
+  SourceLocation createMacroArgExpansionLoc(SourceLocation Loc,
+                                            SourceLocation ExpansionLoc,
+                                            unsigned TokLength);
 
-  /// createMacroArgInstantiationLoc - Return a new SourceLocation that encodes
-  /// the fact that a token from SpellingLoc should actually be referenced from
-  /// InstantiationLoc, and that it represents the instantiation of a macro
-  /// argument into the function-like macro body.
-  SourceLocation createMacroArgInstantiationLoc(SourceLocation Loc,
-                                                SourceLocation InstantiationLoc,
-                                                unsigned TokLength);
-
-  /// createInstantiationLoc - Return a new SourceLocation that encodes the fact
+  /// createExpansionLoc - Return a new SourceLocation that encodes the fact
   /// that a token from SpellingLoc should actually be referenced from
-  /// InstantiationLoc.
-  SourceLocation createInstantiationLoc(SourceLocation Loc,
-                                        SourceLocation InstantiationLocStart,
-                                        SourceLocation InstantiationLocEnd,
-                                        unsigned TokLength,
-                                        unsigned PreallocatedID = 0,
-                                        unsigned Offset = 0);
+  /// ExpansionLoc.
+  SourceLocation createExpansionLoc(SourceLocation Loc,
+                                    SourceLocation ExpansionLocStart,
+                                    SourceLocation ExpansionLocEnd,
+                                    unsigned TokLength,
+                                    int LoadedID = 0,
+                                    unsigned LoadedOffset = 0);
 
   /// \brief Retrieve the memory buffer associated with the given file.
   ///
@@ -633,11 +708,11 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     if (MyInvalid || !Entry.isFile()) {
       if (Invalid)
         *Invalid = true;
-      
+
       return getFakeBufferForRecovery();
     }
-        
-    return Entry.getFile().getContentCache()->getBuffer(Diag, *this, Loc, 
+
+    return Entry.getFile().getContentCache()->getBuffer(Diag, *this, Loc,
                                                         Invalid);
   }
 
@@ -647,22 +722,22 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     if (MyInvalid || !Entry.isFile()) {
       if (Invalid)
         *Invalid = true;
-      
+
       return getFakeBufferForRecovery();
     }
 
-    return Entry.getFile().getContentCache()->getBuffer(Diag, *this, 
-                                                        SourceLocation(), 
+    return Entry.getFile().getContentCache()->getBuffer(Diag, *this,
+                                                        SourceLocation(),
                                                         Invalid);
   }
-  
+
   /// getFileEntryForID - Returns the FileEntry record for the provided FileID.
   const FileEntry *getFileEntryForID(FileID FID) const {
     bool MyInvalid = false;
     const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &MyInvalid);
     if (MyInvalid || !Entry.isFile())
       return 0;
-    
+
     return Entry.getFile().getContentCache()->OrigEntry;
   }
 
@@ -677,8 +752,30 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   ///
   /// \param FID The file ID whose contents will be returned.
   /// \param Invalid If non-NULL, will be set true if an error occurred.
-  llvm::StringRef getBufferData(FileID FID, bool *Invalid = 0) const;
+  StringRef getBufferData(FileID FID, bool *Invalid = 0) const;
 
+  /// \brief Get the number of FileIDs (files and macros) that were created
+  /// during preprocessing of \arg FID, including it.
+  unsigned getNumCreatedFIDsForFileID(FileID FID) const {
+    bool Invalid = false;
+    const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+    if (Invalid || !Entry.isFile())
+      return 0;
+
+    return Entry.getFile().NumCreatedFIDs;
+  }
+
+  /// \brief Set the number of FileIDs (files and macros) that were created
+  /// during preprocessing of \arg FID, including it.
+  void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs) const {
+    bool Invalid = false;
+    const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+    if (Invalid || !Entry.isFile())
+      return;
+
+    assert(Entry.getFile().NumCreatedFIDs == 0 && "Already set!");
+    const_cast<SrcMgr::FileInfo &>(Entry.getFile()).NumCreatedFIDs = NumFIDs;
+  }
 
   //===--------------------------------------------------------------------===//
   // SourceLocation manipulation methods.
@@ -702,34 +799,52 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// getLocForStartOfFile - Return the source location corresponding to the
   /// first byte of the specified file.
   SourceLocation getLocForStartOfFile(FileID FID) const {
-    assert(FID.ID < SLocEntryTable.size() && "FileID out of range");
     bool Invalid = false;
     const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
     if (Invalid || !Entry.isFile())
       return SourceLocation();
-    
+
     unsigned FileOffset = Entry.getOffset();
     return SourceLocation::getFileLoc(FileOffset);
   }
 
-  /// getInstantiationLoc - Given a SourceLocation object, return the
-  /// instantiation location referenced by the ID.
-  SourceLocation getInstantiationLoc(SourceLocation Loc) const {
-    // Handle the non-mapped case inline, defer to out of line code to handle
-    // instantiations.
-    if (Loc.isFileID()) return Loc;
-    return getInstantiationLocSlowCase(Loc);
+  /// \brief Returns the include location if \arg FID is a #include'd file
+  /// otherwise it returns an invalid location.
+  SourceLocation getIncludeLoc(FileID FID) const {
+    bool Invalid = false;
+    const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+    if (Invalid || !Entry.isFile())
+      return SourceLocation();
+
+    return Entry.getFile().getIncludeLoc();
   }
 
-  /// getImmediateInstantiationRange - Loc is required to be an instantiation
-  /// location.  Return the start/end of the instantiation information.
-  std::pair<SourceLocation,SourceLocation>
-  getImmediateInstantiationRange(SourceLocation Loc) const;
+  /// getExpansionLoc - Given a SourceLocation object, return the expansion
+  /// location referenced by the ID.
+  SourceLocation getExpansionLoc(SourceLocation Loc) const {
+    // Handle the non-mapped case inline, defer to out of line code to handle
+    // expansions.
+    if (Loc.isFileID()) return Loc;
+    return getExpansionLocSlowCase(Loc);
+  }
 
-  /// getInstantiationRange - Given a SourceLocation object, return the
-  /// range of tokens covered by the instantiation in the ultimate file.
+  /// \brief Given \arg Loc, if it is a macro location return the expansion
+  /// location or the spelling location, depending on if it comes from a
+  /// macro argument or not.
+  SourceLocation getFileLoc(SourceLocation Loc) const {
+    if (Loc.isFileID()) return Loc;
+    return getFileLocSlowCase(Loc);
+  }
+
+  /// getImmediateExpansionRange - Loc is required to be an expansion location.
+  /// Return the start/end of the expansion information.
   std::pair<SourceLocation,SourceLocation>
-  getInstantiationRange(SourceLocation Loc) const;
+  getImmediateExpansionRange(SourceLocation Loc) const;
+
+  /// getExpansionRange - Given a SourceLocation object, return the range of
+  /// tokens covered by the expansion the ultimate file.
+  std::pair<SourceLocation,SourceLocation>
+  getExpansionRange(SourceLocation Loc) const;
 
 
   /// getSpellingLoc - Given a SourceLocation object, return the spelling
@@ -737,7 +852,7 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// that make up the lexed token can be found.
   SourceLocation getSpellingLoc(SourceLocation Loc) const {
     // Handle the non-mapped case inline, defer to out of line code to handle
-    // instantiations.
+    // expansions.
     if (Loc.isFileID()) return Loc;
     return getSpellingLocSlowCase(Loc);
   }
@@ -756,11 +871,11 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     return std::make_pair(FID, Loc.getOffset()-getSLocEntry(FID).getOffset());
   }
 
-  /// getDecomposedInstantiationLoc - Decompose the specified location into a
-  /// raw FileID + Offset pair.  If the location is an instantiation record,
-  /// walk through it until we find the final location instantiated.
+  /// getDecomposedExpansionLoc - Decompose the specified location into a raw
+  /// FileID + Offset pair. If the location is an expansion record, walk
+  /// through it until we find the final location expanded.
   std::pair<FileID, unsigned>
-  getDecomposedInstantiationLoc(SourceLocation Loc) const {
+  getDecomposedExpansionLoc(SourceLocation Loc) const {
     FileID FID = getFileID(Loc);
     const SrcMgr::SLocEntry *E = &getSLocEntry(FID);
 
@@ -768,11 +883,11 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     if (Loc.isFileID())
       return std::make_pair(FID, Offset);
 
-    return getDecomposedInstantiationLocSlowCase(E);
+    return getDecomposedExpansionLocSlowCase(E);
   }
 
   /// getDecomposedSpellingLoc - Decompose the specified location into a raw
-  /// FileID + Offset pair.  If the location is an instantiation record, walk
+  /// FileID + Offset pair.  If the location is an expansion record, walk
   /// through it until we find its spelling record.
   std::pair<FileID, unsigned>
   getDecomposedSpellingLoc(SourceLocation Loc) const {
@@ -792,12 +907,55 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     return getDecomposedLoc(SpellingLoc).second;
   }
 
-  /// isMacroArgInstantiation - This method tests whether the given source
-  /// location represents a macro argument's instantiation into the
-  /// function-like macro definition. Such source locations only appear inside
-  /// of the instantiation locations representing where a particular
-  /// function-like macro was expanded.
-  bool isMacroArgInstantiation(SourceLocation Loc) const;
+  /// isMacroArgExpansion - This method tests whether the given source location
+  /// represents a macro argument's expansion into the function-like macro
+  /// definition. Such source locations only appear inside of the expansion
+  /// locations representing where a particular function-like macro was
+  /// expanded.
+  bool isMacroArgExpansion(SourceLocation Loc) const;
+
+  /// \brief Returns true if \arg Loc is inside the [\arg Start, +\arg Length)
+  /// chunk of the source location address space.
+  /// If it's true and \arg RelativeOffset is non-null, it will be set to the
+  /// relative offset of \arg Loc inside the chunk.
+  bool isInSLocAddrSpace(SourceLocation Loc,
+                         SourceLocation Start, unsigned Length,
+                         unsigned *RelativeOffset = 0) const {
+    assert(((Start.getOffset() < NextLocalOffset &&
+               Start.getOffset()+Length <= NextLocalOffset) ||
+            (Start.getOffset() >= CurrentLoadedOffset &&
+                Start.getOffset()+Length < MaxLoadedOffset)) &&
+           "Chunk is not valid SLoc address space");
+    unsigned LocOffs = Loc.getOffset();
+    unsigned BeginOffs = Start.getOffset();
+    unsigned EndOffs = BeginOffs + Length;
+    if (LocOffs >= BeginOffs && LocOffs < EndOffs) {
+      if (RelativeOffset)
+        *RelativeOffset = LocOffs - BeginOffs;
+      return true;
+    }
+
+    return false;
+  }
+
+  /// \brief Return true if both \arg LHS and \arg RHS are in the local source
+  /// location address space or the loaded one. If it's true and
+  /// \arg RelativeOffset is non-null, it will be set to the offset of \arg RHS
+  /// relative to \arg LHS.
+  bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS,
+                             int *RelativeOffset) const {
+    unsigned LHSOffs = LHS.getOffset(), RHSOffs = RHS.getOffset();
+    bool LHSLoaded = LHSOffs >= CurrentLoadedOffset;
+    bool RHSLoaded = RHSOffs >= CurrentLoadedOffset;
+
+    if (LHSLoaded == RHSLoaded) {
+      if (RelativeOffset)
+        *RelativeOffset = RHSOffs - LHSOffs;
+      return true;
+    }
+
+    return false;
+  }
 
   //===--------------------------------------------------------------------===//
   // Queries about the code at a SourceLocation.
@@ -811,14 +969,14 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
 
   /// getColumnNumber - Return the column # for the specified file position.
   /// This is significantly cheaper to compute than the line number.  This
-  /// returns zero if the column number isn't known.  This may only be called on
-  /// a file sloc, so you must choose a spelling or instantiation location
+  /// returns zero if the column number isn't known.  This may only be called
+  /// on a file sloc, so you must choose a spelling or expansion location
   /// before calling this method.
-  unsigned getColumnNumber(FileID FID, unsigned FilePos, 
+  unsigned getColumnNumber(FileID FID, unsigned FilePos,
                            bool *Invalid = 0) const;
   unsigned getSpellingColumnNumber(SourceLocation Loc, bool *Invalid = 0) const;
-  unsigned getInstantiationColumnNumber(SourceLocation Loc,
-                                        bool *Invalid = 0) const;
+  unsigned getExpansionColumnNumber(SourceLocation Loc,
+                                    bool *Invalid = 0) const;
   unsigned getPresumedColumnNumber(SourceLocation Loc, bool *Invalid = 0) const;
 
 
@@ -828,8 +986,7 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// about to emit a diagnostic.
   unsigned getLineNumber(FileID FID, unsigned FilePos, bool *Invalid = 0) const;
   unsigned getSpellingLineNumber(SourceLocation Loc, bool *Invalid = 0) const;
-  unsigned getInstantiationLineNumber(SourceLocation Loc, 
-                                      bool *Invalid = 0) const;
+  unsigned getExpansionLineNumber(SourceLocation Loc, bool *Invalid = 0) const;
   unsigned getPresumedLineNumber(SourceLocation Loc, bool *Invalid = 0) const;
 
   /// Return the filename or buffer identifier of the buffer the location is in.
@@ -852,8 +1009,8 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// or GNU line marker directives.  This provides a view on the data that a
   /// user should see in diagnostics, for example.
   ///
-  /// Note that a presumed location is always given as the instantiation point
-  /// of an instantiation location, not at the spelling location.
+  /// Note that a presumed location is always given as the expansion point of
+  /// an expansion location, not at the spelling location.
   ///
   /// \returns The presumed location of the specified SourceLocation. If the
   /// presumed location cannot be calculate (e.g., because \p Loc is invalid
@@ -884,33 +1041,18 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     return getFileCharacteristic(Loc) == SrcMgr::C_ExternCSystem;
   }
 
-  /// \brief Given a specific chunk of a FileID (FileID with offset+length),
-  /// returns true if \arg Loc is inside that chunk and sets relative offset
-  /// (offset of \arg Loc from beginning of chunk) to \arg relativeOffset.
-  bool isInFileID(SourceLocation Loc,
-                  FileID FID, unsigned offset, unsigned length,
-                  unsigned *relativeOffset = 0) const {
-    assert(!FID.isInvalid());
-    if (Loc.isInvalid())
-      return false;
+  /// \brief The size of the SLocEnty that \arg FID represents.
+  unsigned getFileIDSize(FileID FID) const;
 
-    unsigned start = getSLocEntry(FID).getOffset() + offset;
-    unsigned end = start + length;
-
-#ifndef NDEBUG
-    // Make sure offset/length describe a chunk inside the given FileID.
-    unsigned NextOffset;
-    if (FID.ID+1 == SLocEntryTable.size())
-      NextOffset = getNextOffset();
-    else
-      NextOffset = getSLocEntry(FID.ID+1).getOffset();
-    assert(start < NextOffset);
-    assert(end   < NextOffset);
-#endif
-
-    if (Loc.getOffset() >= start && Loc.getOffset() < end) {
-      if (relativeOffset)
-        *relativeOffset = Loc.getOffset() - start;
+  /// \brief Given a specific FileID, returns true if \arg Loc is inside that
+  /// FileID chunk and sets relative offset (offset of \arg Loc from beginning
+  /// of FileID) to \arg relativeOffset.
+  bool isInFileID(SourceLocation Loc, FileID FID,
+                  unsigned *RelativeOffset = 0) const {
+    unsigned Offs = Loc.getOffset();
+    if (isOffsetInFileID(FID, Offs)) {
+      if (RelativeOffset)
+        *RelativeOffset = Offs - getSLocEntry(FID).getOffset();
       return true;
     }
 
@@ -923,7 +1065,7 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
 
   /// getLineTableFilenameID - Return the uniqued ID for the specified filename.
   ///
-  unsigned getLineTableFilenameID(llvm::StringRef Str);
+  unsigned getLineTableFilenameID(StringRef Str);
 
   /// AddLineNote - Add a line note to the line table for the FileID and offset
   /// specified by Loc.  If FilenameID is -1, it is considered to be
@@ -948,11 +1090,11 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   size_t getContentCacheSize() const {
     return ContentCacheAlloc.getTotalMemory();
   }
-  
+
   struct MemoryBufferSizes {
     const size_t malloc_bytes;
     const size_t mmap_bytes;
-    
+
     MemoryBufferSizes(size_t malloc_bytes, size_t mmap_bytes)
       : malloc_bytes(malloc_bytes), mmap_bytes(mmap_bytes) {}
   };
@@ -961,6 +1103,10 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   /// by heap-backed versus mmap'ed memory.
   MemoryBufferSizes getMemoryBufferSizes() const;
 
+  // Return the amount of memory used for various side tables and
+  // data structures in the SourceManager.
+  size_t getDataStructureSizes() const;
+
   //===--------------------------------------------------------------------===//
   // Other miscellaneous methods.
   //===--------------------------------------------------------------------===//
@@ -969,25 +1115,67 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   ///
   /// If the source file is included multiple times, the source location will
   /// be based upon the first inclusion.
-  SourceLocation getLocation(const FileEntry *SourceFile,
-                             unsigned Line, unsigned Col);
+  SourceLocation translateFileLineCol(const FileEntry *SourceFile,
+                                      unsigned Line, unsigned Col) const;
+
+  /// \brief Get the FileID for the given file.
+  ///
+  /// If the source file is included multiple times, the FileID will be the
+  /// first inclusion.
+  FileID translateFile(const FileEntry *SourceFile) const;
+
+  /// \brief Get the source location in \arg FID for the given line:col.
+  /// Returns null location if \arg FID is not a file SLocEntry.
+  SourceLocation translateLineCol(FileID FID,
+                                  unsigned Line, unsigned Col) const;
+
+  /// \brief If \arg Loc points inside a function macro argument, the returned
+  /// location will be the macro location in which the argument was expanded.
+  /// If a macro argument is used multiple times, the expanded location will
+  /// be at the first expansion of the argument.
+  /// e.g.
+  ///   MY_MACRO(foo);
+  ///             ^
+  /// Passing a file location pointing at 'foo', will yield a macro location
+  /// where 'foo' was expanded into.
+  SourceLocation getMacroArgExpandedLocation(SourceLocation Loc) const;
 
   /// \brief Determines the order of 2 source locations in the translation unit.
   ///
   /// \returns true if LHS source location comes before RHS, false otherwise.
   bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const;
 
+  /// \brief Comparison function class.
+  class LocBeforeThanCompare : public std::binary_function<SourceLocation,
+                                                         SourceLocation, bool> {
+    SourceManager &SM;
+
+  public:
+    explicit LocBeforeThanCompare(SourceManager &SM) : SM(SM) { }
+
+    bool operator()(SourceLocation LHS, SourceLocation RHS) const {
+      return SM.isBeforeInTranslationUnit(LHS, RHS);
+    }
+  };
+
   /// \brief Determines the order of 2 source locations in the "source location
   /// address space".
-  static bool isBeforeInSourceLocationOffset(SourceLocation LHS,
-                                             SourceLocation RHS) {
-    return isBeforeInSourceLocationOffset(LHS, RHS.getOffset());
+  bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const {
+    return isBeforeInSLocAddrSpace(LHS, RHS.getOffset());
   }
 
   /// \brief Determines the order of a source location and a source location
   /// offset in the "source location address space".
-  static bool isBeforeInSourceLocationOffset(SourceLocation LHS, unsigned RHS) {
-    return LHS.getOffset() < RHS;
+  ///
+  /// Note that we always consider source locations loaded from
+  bool isBeforeInSLocAddrSpace(SourceLocation LHS, unsigned RHS) const {
+    unsigned LHSOffset = LHS.getOffset();
+    bool LHSLoaded = LHSOffset >= CurrentLoadedOffset;
+    bool RHSLoaded = RHS >= CurrentLoadedOffset;
+    if (LHSLoaded == RHSLoaded)
+      return LHSOffset < RHS;
+
+    return LHSLoaded;
   }
 
   // Iterators over FileInfos.
@@ -1003,53 +1191,80 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   ///
   void PrintStats() const;
 
-  unsigned sloc_entry_size() const { return SLocEntryTable.size(); }
+  /// \brief Get the number of local SLocEntries we have.
+  unsigned local_sloc_entry_size() const { return LocalSLocEntryTable.size(); }
 
-  // FIXME: Exposing this is a little gross; what we want is a good way
-  //  to iterate the entries that were not defined in an AST file (or
-  //  any other external source).
-  unsigned sloc_loaded_entry_size() const { return SLocEntryLoaded.size(); }
+  /// \brief Get a local SLocEntry. This is exposed for indexing.
+  const SrcMgr::SLocEntry &getLocalSLocEntry(unsigned Index,
+                                             bool *Invalid = 0) const {
+    assert(Index < LocalSLocEntryTable.size() && "Invalid index");
+    return LocalSLocEntryTable[Index];
+  }
 
-  const SrcMgr::SLocEntry &getSLocEntry(unsigned ID, bool *Invalid = 0) const {
-    assert(ID < SLocEntryTable.size() && "Invalid id");
-    // If we haven't loaded this source-location entry from the external source
-    // yet, do so now.
-    if (ExternalSLocEntries &&
-        ID < SLocEntryLoaded.size() &&
-        !SLocEntryLoaded[ID] &&
-        ExternalSLocEntries->ReadSLocEntry(ID) &&
-        Invalid)
-      *Invalid = true;
+  /// \brief Get the number of loaded SLocEntries we have.
+  unsigned loaded_sloc_entry_size() const { return LoadedSLocEntryTable.size();}
 
-    return SLocEntryTable[ID];
+  /// \brief Get a loaded SLocEntry. This is exposed for indexing.
+  const SrcMgr::SLocEntry &getLoadedSLocEntry(unsigned Index, bool *Invalid=0) const {
+    assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
+    if (!SLocEntryLoaded[Index])
+      ExternalSLocEntries->ReadSLocEntry(-(static_cast<int>(Index) + 2));
+    return LoadedSLocEntryTable[Index];
   }
 
   const SrcMgr::SLocEntry &getSLocEntry(FileID FID, bool *Invalid = 0) const {
-    return getSLocEntry(FID.ID, Invalid);
+    return getSLocEntryByID(FID.ID);
   }
 
-  unsigned getNextOffset() const { return NextOffset; }
+  unsigned getNextLocalOffset() const { return NextLocalOffset; }
 
-  /// \brief Preallocate some number of source location entries, which
-  /// will be loaded as needed from the given external source.
-  void PreallocateSLocEntries(ExternalSLocEntrySource *Source,
-                              unsigned NumSLocEntries,
-                              unsigned NextOffset);
+  void setExternalSLocEntrySource(ExternalSLocEntrySource *Source) {
+    assert(LoadedSLocEntryTable.empty() &&
+           "Invalidating existing loaded entries");
+    ExternalSLocEntries = Source;
+  }
 
-  /// \brief Clear out any preallocated source location entries that
-  /// haven't already been loaded.
-  void ClearPreallocatedSLocEntries();
+  /// \brief Allocate a number of loaded SLocEntries, which will be actually
+  /// loaded on demand from the external source.
+  ///
+  /// NumSLocEntries will be allocated, which occupy a total of TotalSize space
+  /// in the global source view. The lowest ID and the base offset of the
+  /// entries will be returned.
+  std::pair<int, unsigned>
+  AllocateLoadedSLocEntries(unsigned NumSLocEntries, unsigned TotalSize);
+
+  /// \brief Returns true if \arg Loc came from a PCH/Module.
+  bool isLoadedSourceLocation(SourceLocation Loc) const {
+    return Loc.getOffset() >= CurrentLoadedOffset;
+  }
+
+  /// \brief Returns true if \arg Loc did not come from a PCH/Module.
+  bool isLocalSourceLocation(SourceLocation Loc) const {
+    return Loc.getOffset() < NextLocalOffset;
+  }
 
 private:
   const llvm::MemoryBuffer *getFakeBufferForRecovery() const;
 
-  /// createInstantiationLoc - Implements the common elements of storing an
-  /// instantiation info struct into the SLocEntry table and producing a source
+  /// \brief Get the entry with the given unwrapped FileID.
+  const SrcMgr::SLocEntry &getSLocEntryByID(int ID) const {
+    assert(ID != -1 && "Using FileID sentinel value");
+    if (ID < 0)
+      return getLoadedSLocEntryByID(ID);
+    return getLocalSLocEntry(static_cast<unsigned>(ID));
+  }
+
+  const SrcMgr::SLocEntry &getLoadedSLocEntryByID(int ID) const {
+    return getLoadedSLocEntry(static_cast<unsigned>(-ID - 2));
+  }
+
+  /// createExpansionLoc - Implements the common elements of storing an
+  /// expansion info struct into the SLocEntry table and producing a source
   /// location that refers to it.
-  SourceLocation createInstantiationLocImpl(const SrcMgr::InstantiationInfo &II,
-                                            unsigned TokLength,
-                                            unsigned PreallocatedID = 0,
-                                            unsigned Offset = 0);
+  SourceLocation createExpansionLocImpl(const SrcMgr::ExpansionInfo &Expansion,
+                                        unsigned TokLength,
+                                        int LoadedID = 0,
+                                        unsigned LoadedOffset = 0);
 
   /// isOffsetInFileID - Return true if the specified FileID contains the
   /// specified SourceLocation offset.  This is a very hot method.
@@ -1058,10 +1273,17 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
     // If the entry is after the offset, it can't contain it.
     if (SLocOffset < Entry.getOffset()) return false;
 
-    // If this is the last entry than it does.  Otherwise, the entry after it
-    // has to not include it.
-    if (FID.ID+1 == SLocEntryTable.size()) return true;
+    // If this is the very last entry then it does.
+    if (FID.ID == -2)
+      return true;
 
+    // If it is the last local entry, then it does if the location is local.
+    if (static_cast<unsigned>(FID.ID+1) == LocalSLocEntryTable.size()) {
+      return SLocOffset < NextLocalOffset;
+    }
+
+    // Otherwise, the entry after it has to not include it. This works for both
+    // local and loaded entries.
     return SLocOffset < getSLocEntry(FileID::get(FID.ID+1)).getOffset();
   }
 
@@ -1071,8 +1293,7 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   FileID createFileID(const SrcMgr::ContentCache* File,
                       SourceLocation IncludePos,
                       SrcMgr::CharacteristicKind DirCharacter,
-                      unsigned PreallocatedID = 0,
-                      unsigned Offset = 0);
+                      int LoadedID, unsigned LoadedOffset);
 
   const SrcMgr::ContentCache *
     getOrCreateContentCache(const FileEntry *SourceFile);
@@ -1083,15 +1304,22 @@ class SourceManager : public llvm::RefCountedBase<SourceManager> {
   createMemBufferContentCache(const llvm::MemoryBuffer *Buf);
 
   FileID getFileIDSlow(unsigned SLocOffset) const;
+  FileID getFileIDLocal(unsigned SLocOffset) const;
+  FileID getFileIDLoaded(unsigned SLocOffset) const;
 
-  SourceLocation getInstantiationLocSlowCase(SourceLocation Loc) const;
+  SourceLocation getExpansionLocSlowCase(SourceLocation Loc) const;
   SourceLocation getSpellingLocSlowCase(SourceLocation Loc) const;
+  SourceLocation getFileLocSlowCase(SourceLocation Loc) const;
 
   std::pair<FileID, unsigned>
-  getDecomposedInstantiationLocSlowCase(const SrcMgr::SLocEntry *E) const;
+  getDecomposedExpansionLocSlowCase(const SrcMgr::SLocEntry *E) const;
   std::pair<FileID, unsigned>
   getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
                                    unsigned Offset) const;
+  void computeMacroArgsCache(MacroArgsMap *&MacroArgsCache, FileID FID) const;
+
+  friend class ASTReader;
+  friend class ASTWriter;
 };
 
 
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/SourceManagerInternals.h b/contrib/llvm/tools/clang/include/clang/Basic/SourceManagerInternals.h
index 3f5d1a35e595..1cb16b458f41 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/SourceManagerInternals.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/SourceManagerInternals.h
@@ -84,7 +84,7 @@ class LineTableInfo {
 
   /// LineEntries - This is a map from FileIDs to a list of line entries (sorted
   /// by the offset they occur in the file.
-  std::map<unsigned, std::vector<LineEntry> > LineEntries;
+  std::map<int, std::vector<LineEntry> > LineEntries;
 public:
   LineTableInfo() {
   }
@@ -97,32 +97,32 @@ class LineTableInfo {
 
   ~LineTableInfo() {}
 
-  unsigned getLineTableFilenameID(llvm::StringRef Str);
+  unsigned getLineTableFilenameID(StringRef Str);
   const char *getFilename(unsigned ID) const {
     assert(ID < FilenamesByID.size() && "Invalid FilenameID");
     return FilenamesByID[ID]->getKeyData();
   }
   unsigned getNumFilenames() const { return FilenamesByID.size(); }
 
-  void AddLineNote(unsigned FID, unsigned Offset,
+  void AddLineNote(int FID, unsigned Offset,
                    unsigned LineNo, int FilenameID);
-  void AddLineNote(unsigned FID, unsigned Offset,
+  void AddLineNote(int FID, unsigned Offset,
                    unsigned LineNo, int FilenameID,
                    unsigned EntryExit, SrcMgr::CharacteristicKind FileKind);
 
 
   /// FindNearestLineEntry - Find the line entry nearest to FID that is before
   /// it.  If there is no line entry before Offset in FID, return null.
-  const LineEntry *FindNearestLineEntry(unsigned FID, unsigned Offset);
+  const LineEntry *FindNearestLineEntry(int FID, unsigned Offset);
 
   // Low-level access
-  typedef std::map<unsigned, std::vector<LineEntry> >::iterator iterator;
+  typedef std::map<int, std::vector<LineEntry> >::iterator iterator;
   iterator begin() { return LineEntries.begin(); }
   iterator end() { return LineEntries.end(); }
 
   /// \brief Add a new line entry that has already been encoded into
   /// the internal representation of the line table.
-  void AddEntry(unsigned FID, const std::vector<LineEntry> &Entries);
+  void AddEntry(int FID, const std::vector<LineEntry> &Entries);
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Specifiers.h b/contrib/llvm/tools/clang/include/clang/Basic/Specifiers.h
index cfce0ccbc9dd..2a95d6165c0d 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Specifiers.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Specifiers.h
@@ -40,6 +40,7 @@ namespace clang {
     TST_char16,       // C++0x char16_t
     TST_char32,       // C++0x char32_t
     TST_int,
+    TST_half,         // OpenCL half, ARM NEON __fp16
     TST_float,
     TST_double,
     TST_bool,         // _Bool
@@ -57,6 +58,7 @@ namespace clang {
     TST_underlyingType, // __underlying_type for C++0x
     TST_auto,         // C++0x auto
     TST_unknown_anytype, // __unknown_anytype extension
+    TST_atomic,       // C1X _Atomic
     TST_error         // erroneous type
   };
   
@@ -146,6 +148,7 @@ namespace clang {
     SC_PrivateExtern,
 
     // These are only legal on variables.
+    SC_OpenCLWorkGroupLocal,
     SC_Auto,
     SC_Register
   };
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/StmtNodes.td b/contrib/llvm/tools/clang/include/clang/Basic/StmtNodes.td
index 73996e43d5da..7b3d7762c24a 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/StmtNodes.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/StmtNodes.td
@@ -78,6 +78,9 @@ def ParenListExpr : DStmt<Expr>;
 def VAArgExpr : DStmt<Expr>;
 def GenericSelectionExpr : DStmt<Expr>;
 
+// Atomic expressions
+def AtomicExpr : DStmt<Expr>;
+
 // GNU Extensions.
 def AddrLabelExpr : DStmt<Expr>;
 def StmtExpr : DStmt<Expr>;
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h b/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h
index 4559cf2f64be..a87af2fbbc7a 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_BASIC_TARGETINFO_H
 #define LLVM_CLANG_BASIC_TARGETINFO_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -31,7 +32,7 @@ struct fltSemantics;
 }
 
 namespace clang {
-class Diagnostic;
+class DiagnosticsEngine;
 class LangOptions;
 class MacroBuilder;
 class SourceLocation;
@@ -68,21 +69,24 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   unsigned char PointerWidth, PointerAlign;
   unsigned char BoolWidth, BoolAlign;
   unsigned char IntWidth, IntAlign;
+  unsigned char HalfWidth, HalfAlign;
   unsigned char FloatWidth, FloatAlign;
   unsigned char DoubleWidth, DoubleAlign;
   unsigned char LongDoubleWidth, LongDoubleAlign;
   unsigned char LargeArrayMinWidth, LargeArrayAlign;
   unsigned char LongWidth, LongAlign;
   unsigned char LongLongWidth, LongLongAlign;
+  unsigned char MaxAtomicPromoteWidth, MaxAtomicInlineWidth;
   const char *DescriptionString;
   const char *UserLabelPrefix;
   const char *MCountName;
-  const llvm::fltSemantics *FloatFormat, *DoubleFormat, *LongDoubleFormat;
+  const llvm::fltSemantics *HalfFormat, *FloatFormat, *DoubleFormat,
+    *LongDoubleFormat;
   unsigned char RegParmMax, SSERegParmMax;
   TargetCXXABI CXXABI;
   const LangAS::Map *AddrSpaceMap;
 
-  mutable llvm::StringRef PlatformName;
+  mutable StringRef PlatformName;
   mutable VersionTuple PlatformMinVersion;
 
   unsigned HasAlignMac68kSupport : 1;
@@ -97,7 +101,8 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   /// \param Opts - The options to use to initialize the target. The target may
   /// modify the options to canonicalize the target feature information to match
   /// what the backend expects.
-  static TargetInfo* CreateTargetInfo(Diagnostic &Diags, TargetOptions &Opts);
+  static TargetInfo* CreateTargetInfo(DiagnosticsEngine &Diags,
+                                      TargetOptions &Opts);
 
   virtual ~TargetInfo();
 
@@ -131,6 +136,16 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   /// boundary.
   unsigned UseBitFieldTypeAlignment : 1;
 
+  /// Control whether zero length bitfields (e.g., int : 0;) force alignment of
+  /// the next bitfield.  If the alignment of the zero length bitfield is 
+  /// greater than the member that follows it, `bar', `bar' will be aligned as
+  /// the type of the zero-length bitfield.
+  unsigned UseZeroLengthBitfieldAlignment : 1;
+
+  /// If non-zero, specifies a fixed alignment value for bitfields that follow
+  /// zero length bitfield, regardless of the zero length bitfield type.
+  unsigned ZeroLengthBitfieldBoundary;
+
 public:
   IntType getSizeType() const { return SizeType; }
   IntType getIntMaxType() const { return IntMaxType; }
@@ -211,6 +226,11 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   unsigned getChar32Width() const { return getTypeWidth(Char32Type); }
   unsigned getChar32Align() const { return getTypeAlign(Char32Type); }
 
+  /// getHalfWidth/Align/Format - Return the size/align/format of 'half'.
+  unsigned getHalfWidth() const { return HalfWidth; }
+  unsigned getHalfAlign() const { return HalfAlign; }
+  const llvm::fltSemantics &getHalfFormat() const { return *HalfFormat; }
+
   /// getFloatWidth/Align/Format - Return the size/align/format of 'float'.
   unsigned getFloatWidth() const { return FloatWidth; }
   unsigned getFloatAlign() const { return FloatAlign; }
@@ -234,6 +254,14 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   unsigned getLargeArrayMinWidth() const { return LargeArrayMinWidth; }
   unsigned getLargeArrayAlign() const { return LargeArrayAlign; }
 
+  /// getMaxAtomicPromoteWidth - Return the maximum width lock-free atomic
+  /// operation which will ever be supported for the given target
+  unsigned getMaxAtomicPromoteWidth() const { return MaxAtomicPromoteWidth; }
+  /// getMaxAtomicInlineWidth - Return the maximum width lock-free atomic
+  /// operation which can be inlined given the supported features of the
+  /// given target.
+  unsigned getMaxAtomicInlineWidth() const { return MaxAtomicInlineWidth; }
+
   /// getIntMaxTWidth - Return the size of intmax_t and uintmax_t for this
   /// target, in bits.
   unsigned getIntMaxTWidth() const {
@@ -261,10 +289,24 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
     return MCountName;
   }
 
+  /// useBitFieldTypeAlignment() - Check whether the alignment of bit-field 
+  /// types is respected when laying out structures.
   bool useBitFieldTypeAlignment() const {
     return UseBitFieldTypeAlignment;
   }
 
+  /// useZeroLengthBitfieldAlignment() - Check whether zero length bitfields 
+  /// should force alignment of the next member.
+  bool useZeroLengthBitfieldAlignment() const {
+    return UseZeroLengthBitfieldAlignment;
+  }
+
+  /// getZeroLengthBitfieldBoundary() - Get the fixed alignment value in bits
+  /// for a member that follows a zero length bitfield.
+  unsigned getZeroLengthBitfieldBoundary() const {
+    return ZeroLengthBitfieldBoundary;
+  }
+
   /// hasAlignMac68kSupport - Check whether this target support '#pragma options
   /// align=mac68k'.
   bool hasAlignMac68kSupport() const {
@@ -306,16 +348,16 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   /// isValidClobber - Returns whether the passed in string is
   /// a valid clobber in an inline asm statement. This is used by
   /// Sema.
-  bool isValidClobber(llvm::StringRef Name) const;
+  bool isValidClobber(StringRef Name) const;
 
   /// isValidGCCRegisterName - Returns whether the passed in string
   /// is a valid register name according to GCC. This is used by Sema for
   /// inline asm statements.
-  bool isValidGCCRegisterName(llvm::StringRef Name) const;
+  bool isValidGCCRegisterName(StringRef Name) const;
 
   // getNormalizedGCCRegisterName - Returns the "normalized" GCC register name.
   // For example, on x86 it will return "ax" when "eax" is passed in.
-  llvm::StringRef getNormalizedGCCRegisterName(llvm::StringRef Name) const;
+  StringRef getNormalizedGCCRegisterName(StringRef Name) const;
 
   struct ConstraintInfo {
     enum {
@@ -331,7 +373,7 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
     std::string ConstraintStr;  // constraint: "=rm"
     std::string Name;           // Operand name: [foo] with no []'s.
   public:
-    ConstraintInfo(llvm::StringRef ConstraintStr, llvm::StringRef Name)
+    ConstraintInfo(StringRef ConstraintStr, StringRef Name)
       : Flags(0), TiedOperand(-1), ConstraintStr(ConstraintStr.str()),
       Name(Name.str()) {}
 
@@ -444,7 +486,7 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   /// and give good diagnostics in cases when the assembler or code generator
   /// would otherwise reject the section specifier.
   ///
-  virtual std::string isValidSectionSpecifier(llvm::StringRef SR) const {
+  virtual std::string isValidSectionSpecifier(StringRef SR) const {
     return "";
   }
 
@@ -453,11 +495,9 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   /// language options which change the target configuration.
   virtual void setForcedLangOptions(LangOptions &Opts);
 
-  /// getDefaultFeatures - Get the default set of target features for
-  /// the \args CPU; this should include all legal feature strings on
-  /// the target.
-  virtual void getDefaultFeatures(const std::string &CPU,
-                                  llvm::StringMap<bool> &Features) const {
+  /// getDefaultFeatures - Get the default set of target features for the CPU;
+  /// this should include all legal feature strings on the target.
+  virtual void getDefaultFeatures(llvm::StringMap<bool> &Features) const {
   }
 
   /// getABI - Get the ABI in use.
@@ -473,10 +513,8 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
   /// setCPU - Target the specific CPU.
   ///
   /// \return - False on error (invalid CPU name).
-  //
-  // FIXME: Remove this.
   virtual bool setCPU(const std::string &Name) {
-    return true;
+    return false;
   }
 
   /// setABI - Use the specific ABI.
@@ -565,7 +603,7 @@ class TargetInfo : public llvm::RefCountedBase<TargetInfo> {
 
   /// \brief Retrieve the name of the platform as it is used in the
   /// availability attribute.
-  llvm::StringRef getPlatformName() const { return PlatformName; }
+  StringRef getPlatformName() const { return PlatformName; }
 
   /// \brief Retrieve the minimum desired version of the platform, to
   /// which the program should be compiled.
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/TokenKinds.def b/contrib/llvm/tools/clang/include/clang/Basic/TokenKinds.def
index 86172b83ff42..35a881c660a2 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/TokenKinds.def
+++ b/contrib/llvm/tools/clang/include/clang/Basic/TokenKinds.def
@@ -89,6 +89,9 @@ PPKEYWORD(sccs)
 PPKEYWORD(assert)
 PPKEYWORD(unassert)
 
+// Clang extensions
+PPKEYWORD(__export_macro__)
+
 //===----------------------------------------------------------------------===//
 // Language keywords.
 //===----------------------------------------------------------------------===//
@@ -114,13 +117,23 @@ TOK(raw_identifier)      // Used only in raw lexing mode.
 TOK(numeric_constant)    // 0x123
 
 // C99 6.4.4: Character Constants
-TOK(char_constant)       // 'a'   L'b'
+TOK(char_constant)       // 'a'
+TOK(wide_char_constant)  // L'b'
+
+// C++0x Character Constants
+TOK(utf16_char_constant) // u'a'
+TOK(utf32_char_constant) // U'a'
 
 // C99 6.4.5: String Literals.
 TOK(string_literal)      // "foo"
 TOK(wide_string_literal) // L"foo"
 TOK(angle_string_literal)// <foo>
 
+// C++0x String Literals.
+TOK(utf8_string_literal) // u8"foo"
+TOK(utf16_string_literal)// u"foo"
+TOK(utf32_string_literal)// U"foo"
+
 // C99 6.4.6: Punctuators.
 PUNCTUATOR(l_square,            "[")
 PUNCTUATOR(r_square,            "]")
@@ -236,6 +249,8 @@ KEYWORD(unsigned                    , KEYALL)
 KEYWORD(void                        , KEYALL)
 KEYWORD(volatile                    , KEYALL)
 KEYWORD(while                       , KEYALL)
+KEYWORD(_Alignas                    , KEYALL)
+KEYWORD(_Atomic                     , KEYALL)
 KEYWORD(_Bool                       , KEYNOCXX)
 KEYWORD(_Complex                    , KEYALL)
 KEYWORD(_Generic                    , KEYALL)
@@ -289,6 +304,7 @@ CXX_KEYWORD_OPERATOR(xor     , caret)
 CXX_KEYWORD_OPERATOR(xor_eq  , caretequal)
 
 // C++0x keywords
+KEYWORD(alignas                     , KEYCXX0X)
 KEYWORD(alignof                     , KEYCXX0X)
 KEYWORD(char16_t                    , KEYCXX0X)
 KEYWORD(char32_t                    , KEYCXX0X)
@@ -387,6 +403,8 @@ KEYWORD(__array_extent              , KEYCXX)
 
 // Apple Extension.
 KEYWORD(__private_extern__          , KEYALL)
+KEYWORD(__import_module__           , KEYALL)
+KEYWORD(__module_private__          , KEYALL)
 
 // Microsoft Extension.
 KEYWORD(__declspec                  , KEYALL)
@@ -395,6 +413,7 @@ KEYWORD(__stdcall                   , KEYALL)
 KEYWORD(__fastcall                  , KEYALL)
 KEYWORD(__thiscall                  , KEYALL)
 KEYWORD(__forceinline               , KEYALL)
+KEYWORD(__unaligned                 , KEYMS)
 
 // OpenCL-specific keywords
 KEYWORD(__kernel                    , KEYOPENCL)
@@ -422,6 +441,12 @@ KEYWORD(__pascal                    , KEYALL)
 KEYWORD(__vector                    , KEYALTIVEC)
 KEYWORD(__pixel                     , KEYALTIVEC)
 
+// ARM NEON extensions.
+ALIAS("__fp16", half                , KEYALL)
+
+// OpenCL Extension.
+KEYWORD(half                        , KEYOPENCL)
+
 // Objective-C ARC keywords.
 KEYWORD(__bridge                     , KEYARC)
 KEYWORD(__bridge_transfer            , KEYARC)
@@ -455,6 +480,7 @@ ALIAS("__volatile__" , volatile   , KEYALL)
 
 // Microsoft extensions which should be disabled in strict conformance mode
 KEYWORD(__ptr64                   , KEYMS)
+KEYWORD(__ptr32                   , KEYMS)
 KEYWORD(__w64                     , KEYMS)
 KEYWORD(__uuidof                  , KEYMS | KEYBORLAND)
 KEYWORD(__try                     , KEYMS | KEYBORLAND)
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/VersionTuple.h b/contrib/llvm/tools/clang/include/clang/Basic/VersionTuple.h
index 91eb68eaad9f..30ef6641efc1 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/VersionTuple.h
+++ b/contrib/llvm/tools/clang/include/clang/Basic/VersionTuple.h
@@ -14,13 +14,10 @@
 #ifndef LLVM_CLANG_BASIC_VERSIONTUPLE_H
 #define LLVM_CLANG_BASIC_VERSIONTUPLE_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/Optional.h"
 #include <string>
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
 
 /// \brief Represents a version number in the form major[.minor[.subminor]].
@@ -120,7 +117,7 @@ class VersionTuple {
 };
 
 /// \brief Print a version number.
-llvm::raw_ostream& operator<<(llvm::raw_ostream &Out, const VersionTuple &V);
+raw_ostream& operator<<(raw_ostream &Out, const VersionTuple &V);
 
 } // end namespace clang
 #endif // LLVM_CLANG_BASIC_VERSIONTUPLE_H
diff --git a/contrib/llvm/tools/clang/include/clang/CodeGen/BackendUtil.h b/contrib/llvm/tools/clang/include/clang/CodeGen/BackendUtil.h
index 9636d6e527c2..135b6a927f9a 100644
--- a/contrib/llvm/tools/clang/include/clang/CodeGen/BackendUtil.h
+++ b/contrib/llvm/tools/clang/include/clang/CodeGen/BackendUtil.h
@@ -10,13 +10,14 @@
 #ifndef LLVM_CLANG_CODEGEN_BACKEND_UTIL_H
 #define LLVM_CLANG_CODEGEN_BACKEND_UTIL_H
 
+#include "clang/Basic/LLVM.h"
+
 namespace llvm {
   class Module;
-  class raw_ostream;
 }
 
 namespace clang {
-  class Diagnostic;
+  class DiagnosticsEngine;
   class CodeGenOptions;
   class TargetOptions;
   class LangOptions;
@@ -30,10 +31,10 @@ namespace clang {
     Backend_EmitObj        ///< Emit native object files
   };
   
-  void EmitBackendOutput(Diagnostic &Diags, const CodeGenOptions &CGOpts,
+  void EmitBackendOutput(DiagnosticsEngine &Diags, const CodeGenOptions &CGOpts,
                          const TargetOptions &TOpts, const LangOptions &LOpts,
                          llvm::Module *M,
-                         BackendAction Action, llvm::raw_ostream *OS);
+                         BackendAction Action, raw_ostream *OS);
 }
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenAction.h b/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenAction.h
index 052c6603f5a7..f1a2f6eb455d 100644
--- a/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenAction.h
+++ b/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenAction.h
@@ -37,7 +37,7 @@ class CodeGenAction : public ASTFrontendAction {
   virtual bool hasIRSupport() const;
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
   virtual void ExecuteAction();
 
diff --git a/contrib/llvm/tools/clang/include/clang/CodeGen/ModuleBuilder.h b/contrib/llvm/tools/clang/include/clang/CodeGen/ModuleBuilder.h
index c45ad08716c6..38aba8900243 100644
--- a/contrib/llvm/tools/clang/include/clang/CodeGen/ModuleBuilder.h
+++ b/contrib/llvm/tools/clang/include/clang/CodeGen/ModuleBuilder.h
@@ -23,7 +23,7 @@ namespace llvm {
 }
 
 namespace clang {
-  class Diagnostic;
+  class DiagnosticsEngine;
   class LangOptions;
   class CodeGenOptions;
 
@@ -36,7 +36,7 @@ namespace clang {
   /// CreateLLVMCodeGen - Create a CodeGenerator instance.
   /// It is the responsibility of the caller to call delete on
   /// the allocated CodeGenerator instance.
-  CodeGenerator *CreateLLVMCodeGen(Diagnostic &Diags,
+  CodeGenerator *CreateLLVMCodeGen(DiagnosticsEngine &Diags,
                                    const std::string &ModuleName,
                                    const CodeGenOptions &CGO,
                                    llvm::LLVMContext& C);
diff --git a/contrib/llvm/tools/clang/include/clang/Config/config.h.cmake b/contrib/llvm/tools/clang/include/clang/Config/config.h.cmake
deleted file mode 100644
index 5f13d2faa311..000000000000
--- a/contrib/llvm/tools/clang/include/clang/Config/config.h.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-/* Relative directory for resource files */
-#define CLANG_RESOURCE_DIR "${CLANG_RESOURCE_DIR}"
-
-/* 32 bit multilib directory. */
-#define CXX_INCLUDE_32BIT_DIR "${CXX_INCLUDE_32BIT_DIR}"
-
-/* 64 bit multilib directory. */
-#define CXX_INCLUDE_64BIT_DIR "${CXX_INCLUDE_64BIT_DIR}"
-
-/* Arch the libstdc++ headers. */
-#define CXX_INCLUDE_ARCH "${CXX_INCLUDE_ARCH}"
-
-/* Directory with the libstdc++ headers. */
-#define CXX_INCLUDE_ROOT "${CXX_INCLUDE_ROOT}"
-
-/* Directories clang will search for headers */
-#define C_INCLUDE_DIRS "${C_INCLUDE_DIRS}"
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Action.h b/contrib/llvm/tools/clang/include/clang/Driver/Action.h
index 4b45c98313c4..a33c33bf734c 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Action.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Action.h
@@ -10,17 +10,9 @@
 #ifndef CLANG_DRIVER_ACTION_H_
 #define CLANG_DRIVER_ACTION_H_
 
-#include "llvm/ADT/SmallVector.h"
-
 #include "clang/Driver/Types.h"
 #include "clang/Driver/Util.h"
-
-#include "llvm/Support/Casting.h"
-using llvm::isa;
-using llvm::cast;
-using llvm::cast_or_null;
-using llvm::dyn_cast;
-using llvm::dyn_cast_or_null;
+#include "llvm/ADT/SmallVector.h"
 
 namespace clang {
 namespace driver {
@@ -52,9 +44,10 @@ class Action {
     LinkJobClass,
     LipoJobClass,
     DsymutilJobClass,
+    VerifyJobClass,
 
     JobClassFirst=PreprocessJobClass,
-    JobClassLast=DsymutilJobClass
+    JobClassLast=VerifyJobClass
   };
 
   static const char *getClassName(ActionClass AC);
@@ -222,6 +215,15 @@ class DsymutilJobAction : public JobAction {
   static bool classof(const DsymutilJobAction *) { return true; }
 };
 
+class VerifyJobAction : public JobAction {
+public:
+  VerifyJobAction(ActionList &Inputs, types::ID Type);
+  static bool classof(const Action *A) {
+    return A->getKind() == VerifyJobClass;
+  }
+  static bool classof(const VerifyJobAction *) { return true; }
+};
+
 } // end namespace driver
 } // end namespace clang
 
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Arg.h b/contrib/llvm/tools/clang/include/clang/Driver/Arg.h
index 265d6d871672..e8625bbd5175 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Arg.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Arg.h
@@ -51,7 +51,7 @@ namespace driver {
     mutable unsigned OwnsValues : 1;
 
     /// The argument values, as C strings.
-    llvm::SmallVector<const char *, 2> Values;
+    SmallVector<const char *, 2> Values;
 
   public:
     Arg(const Option *Opt, unsigned Index, const Arg *BaseArg = 0);
@@ -87,11 +87,11 @@ namespace driver {
       return Values[N];
     }
 
-    llvm::SmallVectorImpl<const char*> &getValues() {
+    SmallVectorImpl<const char*> &getValues() {
       return Values;
     }
 
-    bool containsValue(llvm::StringRef Value) const {
+    bool containsValue(StringRef Value) const {
       for (unsigned i = 0, e = getNumValues(); i != e; ++i)
         if (Values[i] == Value)
           return true;
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/ArgList.h b/contrib/llvm/tools/clang/include/clang/Driver/ArgList.h
index 0fcf821c752c..04faf64dc5ec 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/ArgList.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/ArgList.h
@@ -10,6 +10,7 @@
 #ifndef CLANG_DRIVER_ARGLIST_H_
 #define CLANG_DRIVER_ARGLIST_H_
 
+#include "clang/Basic/LLVM.h"
 #include "clang/Driver/OptSpecifier.h"
 #include "clang/Driver/Util.h"
 #include "llvm/ADT/SmallVector.h"
@@ -19,12 +20,8 @@
 #include <string>
 #include <vector>
 
-namespace llvm {
-  class Twine;
-}
-
 namespace clang {
-  class Diagnostic;
+  class DiagnosticsEngine;
 
 namespace driver {
   class Arg;
@@ -34,7 +31,7 @@ namespace driver {
   /// arg_iterator - Iterates through arguments stored inside an ArgList.
   class arg_iterator {
     /// The current argument.
-    llvm::SmallVectorImpl<Arg*>::const_iterator Current;
+    SmallVectorImpl<Arg*>::const_iterator Current;
 
     /// The argument list we are iterating over.
     const ArgList &Args;
@@ -58,7 +55,7 @@ namespace driver {
     typedef std::forward_iterator_tag   iterator_category;
     typedef std::ptrdiff_t              difference_type;
 
-    arg_iterator(llvm::SmallVectorImpl<Arg*>::const_iterator it,
+    arg_iterator(SmallVectorImpl<Arg*>::const_iterator it,
                  const ArgList &_Args, OptSpecifier _Id0 = 0U,
                  OptSpecifier _Id1 = 0U, OptSpecifier _Id2 = 0U)
       : Current(it), Args(_Args), Id0(_Id0), Id1(_Id1), Id2(_Id2) {
@@ -101,7 +98,7 @@ namespace driver {
     void operator=(const ArgList &); // DO NOT IMPLEMENT
 
   public:
-    typedef llvm::SmallVector<Arg*, 16> arglist_type;
+    typedef SmallVector<Arg*, 16> arglist_type;
     typedef arglist_type::iterator iterator;
     typedef arglist_type::const_iterator const_iterator;
     typedef arglist_type::reverse_iterator reverse_iterator;
@@ -152,6 +149,13 @@ namespace driver {
       return arg_iterator(Args.end(), *this);
     }
 
+    /// @}
+    /// @name Arg Removal
+    /// @{
+
+    /// eraseArg - Remove any option matching \arg Id.
+    void eraseArg(OptSpecifier Id);
+
     /// @}
     /// @name Arg Access
     /// @{
@@ -195,13 +199,13 @@ namespace driver {
     /// @{
 
     /// getLastArgValue - Return the value of the last argument, or a default.
-    llvm::StringRef getLastArgValue(OptSpecifier Id,
-                                    llvm::StringRef Default = "") const;
+    StringRef getLastArgValue(OptSpecifier Id,
+                                    StringRef Default = "") const;
 
     /// getLastArgValue - Return the value of the last argument as an integer,
     /// or a default. Emits an error if the argument is given, but non-integral.
     int getLastArgIntValue(OptSpecifier Id, int Default,
-                           Diagnostic &Diags) const;
+                           DiagnosticsEngine &Diags) const;
 
     /// getAllArgValues - Get the values of all instances of the given argument
     /// as strings.
@@ -245,25 +249,29 @@ namespace driver {
     /// option id.
     void ClaimAllArgs(OptSpecifier Id0) const;
 
+    /// ClaimAllArgs - Claim all arguments.
+    ///
+    void ClaimAllArgs() const;
+
     /// @}
     /// @name Arg Synthesis
     /// @{
 
     /// MakeArgString - Construct a constant string pointer whose
     /// lifetime will match that of the ArgList.
-    virtual const char *MakeArgString(llvm::StringRef Str) const = 0;
+    virtual const char *MakeArgString(StringRef Str) const = 0;
     const char *MakeArgString(const char *Str) const {
-      return MakeArgString(llvm::StringRef(Str));
+      return MakeArgString(StringRef(Str));
     }
     const char *MakeArgString(std::string Str) const {
-      return MakeArgString(llvm::StringRef(Str));
+      return MakeArgString(StringRef(Str));
     }
-    const char *MakeArgString(const llvm::Twine &Str) const;
+    const char *MakeArgString(const Twine &Str) const;
 
     /// \brief Create an arg string for (\arg LHS + \arg RHS), reusing the
     /// string at \arg Index if possible.
-    const char *GetOrMakeJoinedArgString(unsigned Index, llvm::StringRef LHS,
-                                         llvm::StringRef RHS) const;
+    const char *GetOrMakeJoinedArgString(unsigned Index, StringRef LHS,
+                                         StringRef RHS) const;
 
     /// @}
   };
@@ -304,10 +312,10 @@ namespace driver {
 
   public:
     /// MakeIndex - Get an index for the given string(s).
-    unsigned MakeIndex(llvm::StringRef String0) const;
-    unsigned MakeIndex(llvm::StringRef String0, llvm::StringRef String1) const;
+    unsigned MakeIndex(StringRef String0) const;
+    unsigned MakeIndex(StringRef String0, StringRef String1) const;
 
-    virtual const char *MakeArgString(llvm::StringRef Str) const;
+    virtual const char *MakeArgString(StringRef Str) const;
 
     /// @}
   };
@@ -346,7 +354,7 @@ namespace driver {
       SynthesizedArgs.push_back(A);
     }
 
-    virtual const char *MakeArgString(llvm::StringRef Str) const;
+    virtual const char *MakeArgString(StringRef Str) const;
 
     /// AddFlagArg - Construct a new FlagArg for the given option \arg Id and
     /// append it to the argument list.
@@ -358,7 +366,7 @@ namespace driver {
     /// \arg Id, with the provided \arg Value and append it to the argument
     /// list.
     void AddPositionalArg(const Arg *BaseArg, const Option *Opt,
-                          llvm::StringRef Value) {
+                          StringRef Value) {
       append(MakePositionalArg(BaseArg, Opt, Value));
     }
 
@@ -367,7 +375,7 @@ namespace driver {
     /// \arg Id, with the provided \arg Value and append it to the argument
     /// list.
     void AddSeparateArg(const Arg *BaseArg, const Option *Opt,
-                        llvm::StringRef Value) {
+                        StringRef Value) {
       append(MakeSeparateArg(BaseArg, Opt, Value));
     }
 
@@ -375,7 +383,7 @@ namespace driver {
     /// AddJoinedArg - Construct a new Positional arg for the given option \arg
     /// Id, with the provided \arg Value and append it to the argument list.
     void AddJoinedArg(const Arg *BaseArg, const Option *Opt,
-                      llvm::StringRef Value) {
+                      StringRef Value) {
       append(MakeJoinedArg(BaseArg, Opt, Value));
     }
 
@@ -387,17 +395,17 @@ namespace driver {
     /// MakePositionalArg - Construct a new Positional arg for the
     /// given option \arg Id, with the provided \arg Value.
     Arg *MakePositionalArg(const Arg *BaseArg, const Option *Opt,
-                           llvm::StringRef Value) const;
+                           StringRef Value) const;
 
     /// MakeSeparateArg - Construct a new Positional arg for the
     /// given option \arg Id, with the provided \arg Value.
     Arg *MakeSeparateArg(const Arg *BaseArg, const Option *Opt,
-                         llvm::StringRef Value) const;
+                         StringRef Value) const;
 
     /// MakeJoinedArg - Construct a new Positional arg for the
     /// given option \arg Id, with the provided \arg Value.
     Arg *MakeJoinedArg(const Arg *BaseArg, const Option *Opt,
-                       llvm::StringRef Value) const;
+                       StringRef Value) const;
 
     /// @}
   };
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/CC1Options.td b/contrib/llvm/tools/clang/include/clang/Driver/CC1Options.td
index c72fb7daaf1f..70ce040360a8 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/CC1Options.td
+++ b/contrib/llvm/tools/clang/include/clang/Driver/CC1Options.td
@@ -57,6 +57,10 @@ def analyzer_output : Separate<"-analyzer-output">,
 def analyzer_output_EQ : Joined<"-analyzer-output=">,
   Alias<analyzer_output>;
 
+def analyzer_purge : Separate<"-analyzer-purge">,
+  HelpText<"Source Code Analysis - Dead Symbol Removal Frequency">;
+def analyzer_purge_EQ : Joined<"-analyzer-purge=">, Alias<analyzer_purge>;
+
 def analyzer_opt_analyze_headers : Flag<"-analyzer-opt-analyze-headers">,
   HelpText<"Force the static analyzer to analyze functions defined in header files">;
 def analyzer_opt_analyze_nested_blocks : Flag<"-analyzer-opt-analyze-nested-blocks">,
@@ -68,8 +72,6 @@ def analyze_function : Separate<"-analyze-function">,
 def analyze_function_EQ : Joined<"-analyze-function=">, Alias<analyze_function>;
 def analyzer_eagerly_assume : Flag<"-analyzer-eagerly-assume">,
   HelpText<"Eagerly assume the truth/falseness of some symbolic constraints">;
-def analyzer_no_purge_dead : Flag<"-analyzer-no-purge-dead">,
-  HelpText<"Don't remove dead symbols, bindings, and constraints before processing a statement">;
 def analyzer_no_eagerly_trim_egraph : Flag<"-analyzer-no-eagerly-trim-egraph">,
   HelpText<"Don't eagerly remove uninteresting ExplodedNodes from the ExplodedGraph">;
 def trim_egraph : Flag<"-trim-egraph">,
@@ -113,9 +115,10 @@ def dwarf_debug_flags : Separate<"-dwarf-debug-flags">,
 def fforbid_guard_variables : Flag<"-fforbid-guard-variables">,
   HelpText<"Emit an error if a C++ static local initializer would need a guard variable">;
 def g : Flag<"-g">, HelpText<"Generate source level debug information">;
-def fno_dwarf2_cfi_asm : Flag<"-fno-dwarf2-cfi-asm">, HelpText<"Don't use the cfi directives">;
+def fno_dwarf2_cfi_asm : Flag<"-fno-dwarf2-cfi-asm">,
+  HelpText<"Don't use the cfi directives">;
 def fcatch_undefined_behavior : Flag<"-fcatch-undefined-behavior">,
-    HelpText<"Generate runtime checks for undefined behavior.">;
+  HelpText<"Generate runtime checks for undefined behavior.">;
 def flimit_debug_info : Flag<"-flimit-debug-info">,
   HelpText<"Limit debug information produced to reduce size of debug binary">;
 def fno_common : Flag<"-fno-common">,
@@ -157,6 +160,8 @@ def mdisable_fp_elim : Flag<"-mdisable-fp-elim">,
   HelpText<"Disable frame pointer elimination optimization">;
 def mfloat_abi : Separate<"-mfloat-abi">,
   HelpText<"The float ABI to use">;
+def mno_global_merge : Flag<"-mno-global-merge">,
+  HelpText<"Disable merging of globals">;
 def mlimit_float_precision : Separate<"-mlimit-float-precision">,
   HelpText<"Limit float precision to the given value">;
 def mno_exec_stack : Flag<"-mnoexecstack">,
@@ -359,6 +364,8 @@ def ast_view : Flag<"-ast-view">,
   HelpText<"Build ASTs and view them with GraphViz">;
 def print_decl_contexts : Flag<"-print-decl-contexts">,
   HelpText<"Print DeclContexts and their Decls">;
+def emit_module : Flag<"-emit-module">,
+  HelpText<"Generate pre-compiled module file">;
 def emit_pth : Flag<"-emit-pth">,
   HelpText<"Generate pre-tokenized header file">;
 def emit_pch : Flag<"-emit-pch">,
@@ -381,9 +388,6 @@ def rewrite_objc : Flag<"-rewrite-objc">,
   HelpText<"Rewrite ObjC into C (code rewriter example)">;
 def rewrite_macros : Flag<"-rewrite-macros">,
   HelpText<"Expand macros without full preprocessing">;
-
-def create_module : Flag<"-create-module">,
-  HelpText<"Create a module definition file">;
 }
 
 def arcmt_check : Flag<"-arcmt-check">,
@@ -394,9 +398,10 @@ def arcmt_migrate : Flag<"-arcmt-migrate">,
   HelpText<"Apply modifications and produces temporary files that conform to ARC">;
 def arcmt_migrate_directory : Separate<"-arcmt-migrate-directory">,
   HelpText<"Directory for temporary files produced during ARC migration">;
-
-def import_module : Separate<"-import-module">,
-  HelpText<"Import a module definition file">;
+def arcmt_migrate_report_output : Separate<"-arcmt-migrate-report-output">,
+  HelpText<"Output path for the plist report">;
+def arcmt_migrate_emit_arc_errors : Flag<"-arcmt-migrate-emit-errors">,
+  HelpText<"Emit ARC errors even if the migrator can fix them">;
 
 def working_directory : JoinedOrSeparate<"-working-directory">,
   HelpText<"Resolve file paths relative to the specified directory">;
@@ -405,8 +410,6 @@ def working_directory_EQ : Joined<"-working-directory=">,
 
 def relocatable_pch : Flag<"-relocatable-pch">,
   HelpText<"Whether to build a relocatable precompiled header">;
-def chained_pch : Flag<"-chained-pch">,
-  HelpText<"Whether to chain the new precompiled header to the old one.">;
 def print_stats : Flag<"-print-stats">,
   HelpText<"Print performance metrics and statistics">;
 def ftime_report : Flag<"-ftime-report">,
@@ -446,7 +449,9 @@ def fno_dollars_in_identifiers : Flag<"-fno-dollars-in-identifiers">,
 def femit_all_decls : Flag<"-femit-all-decls">,
   HelpText<"Emit all declarations, even if unused">;
 def fblocks : Flag<"-fblocks">,
-  HelpText<"enable the 'blocks' language feature">;
+  HelpText<"Enable the 'blocks' language feature">;
+def fblocks_runtime_optional : Flag<"-fblocks-runtime-optional">,
+  HelpText<"Weakly link in the blocks runtime">;
 def fheinous_gnu_extensions : Flag<"-fheinous-gnu-extensions">;
 def fexceptions : Flag<"-fexceptions">,
   HelpText<"Enable support for exception handling">;
@@ -471,7 +476,9 @@ def stdlib_EQ : Joined<"-stdlib=">,
 def fmath_errno : Flag<"-fmath-errno">,
   HelpText<"Require math functions to indicate errors by setting errno">;
 def fms_extensions : Flag<"-fms-extensions">,
-  HelpText<"Accept some non-standard constructs used in Microsoft header files ">;
+  HelpText<"Accept some non-standard constructs supported by the Microsoft compiler">;
+def fms_compatibility : Flag<"-fms-compatibility">,
+  HelpText<"Enable Microsoft compatibility mode">;
 def fmsc_version : Joined<"-fmsc-version=">,
   HelpText<"Version of the Microsoft C/C++ compiler to report in _MSC_VER (0 = don't define it (default))">;
 def fborland_extensions : Flag<"-fborland-extensions">,
@@ -519,8 +526,8 @@ def fobjc_default_synthesize_properties : Flag<"-fobjc-default-synthesize-proper
   HelpText<"enable the default synthesis of Objective-C properties">;
 def print_ivar_layout : Flag<"-print-ivar-layout">,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">;
-def fobjc_nonfragile_abi : Flag<"-fobjc-nonfragile-abi">,
-  HelpText<"enable objective-c's nonfragile abi">;
+def fobjc_fragile_abi : Flag<"-fobjc-fragile-abi">,
+  HelpText<"Use Objective-C's fragile ABI">;
 def fno_objc_infer_related_result_type : Flag<
     "-fno-objc-infer-related-result-type">,
   HelpText<
@@ -536,6 +543,8 @@ def pic_level : Separate<"-pic-level">,
   HelpText<"Value for __PIC__">;
 def pthread : Flag<"-pthread">,
   HelpText<"Support POSIX threads in generated code">;
+def fpack_struct : Separate<"-fpack-struct">,
+  HelpText<"Specify the default maximum struct packing alignment">;
 def fpascal_strings : Flag<"-fpascal-strings">,
   HelpText<"Recognize and construct Pascal-style string literals">;
 def fno_rtti : Flag<"-fno-rtti">,
@@ -595,22 +604,41 @@ def fno_deprecated_macro : Flag<"-fno-deprecated-macro">,
 // Header Search Options
 //===----------------------------------------------------------------------===//
 
-def nostdinc : Flag<"-nostdinc">,
-  HelpText<"Disable standard #include directories">;
+def nostdsysteminc : Flag<"-nostdsysteminc">,
+  HelpText<"Disable standard system #include directories">;
 def nostdincxx : Flag<"-nostdinc++">,
   HelpText<"Disable standard #include directories for the C++ standard library">;
 def nobuiltininc : Flag<"-nobuiltininc">,
   HelpText<"Disable builtin #include directories">;
+def fmodule_cache_path : Separate<"-fmodule-cache-path">, 
+  MetaVarName<"<directory>">,
+  HelpText<"Specify the module cache path">;           
+def fdisable_module_hash : Flag<"-fdisable-module-hash">,
+  HelpText<"Disable the module hash">;
+def fauto_module_import : Flag<"-fauto-module-import">,
+  HelpText<"Automatically translate #include/#import into module imports "
+           "when possible">;
+           
 def F : JoinedOrSeparate<"-F">, MetaVarName<"<directory>">,
   HelpText<"Add directory to framework include search path">;
 def I : JoinedOrSeparate<"-I">, MetaVarName<"<directory>">,
   HelpText<"Add directory to include search path">;
 def idirafter : JoinedOrSeparate<"-idirafter">, MetaVarName<"<directory>">,
   HelpText<"Add directory to AFTER include search path">;
+def index_header_map : Flag<"-index-header-map">, 
+  HelpText<"Make the next included directory (-I or -F) an indexer header map">;
 def iquote : JoinedOrSeparate<"-iquote">, MetaVarName<"<directory>">,
   HelpText<"Add directory to QUOTE include search path">;
+def c_isystem : JoinedOrSeparate<"-c-isystem">, MetaVarName<"<directory>">,
+  HelpText<"Add directory to the C SYSTEM include search path">;
 def cxx_isystem : JoinedOrSeparate<"-cxx-isystem">, MetaVarName<"<directory>">,
   HelpText<"Add directory to the C++ SYSTEM include search path">;
+def objc_isystem : JoinedOrSeparate<"-objc-isystem">,
+  MetaVarName<"<directory>">,
+  HelpText<"Add directory to the ObjC SYSTEM include search path">;
+def objcxx_isystem : JoinedOrSeparate<"-objcxx-isystem">,
+  MetaVarName<"<directory>">,
+  HelpText<"Add directory to the ObjC++ SYSTEM include search path">;
 def isystem : JoinedOrSeparate<"-isystem">, MetaVarName<"<directory>">,
   HelpText<"Add directory to SYSTEM include search path">;
 def iwithsysroot : JoinedOrSeparate<"-iwithsysroot">,MetaVarName<"<directory>">,
@@ -688,3 +716,10 @@ def cl_mad_enable : Flag<"-cl-mad-enable">,
   HelpText<"OpenCL only. Enable less precise MAD instructions to be generated.">;
 def cl_std_EQ : Joined<"-cl-std=">,
   HelpText<"OpenCL language standard to compile for">;
+
+//===----------------------------------------------------------------------===//
+// CUDA Options
+//===----------------------------------------------------------------------===//
+
+def fcuda_is_device : Flag<"-fcuda-is-device">,
+  HelpText<"Generate code for CUDA device">;
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Compilation.h b/contrib/llvm/tools/clang/include/clang/Driver/Compilation.h
index 2db712d9321f..8c9990909ebb 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Compilation.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Compilation.h
@@ -12,12 +12,8 @@
 
 #include "clang/Driver/Job.h"
 #include "clang/Driver/Util.h"
-
 #include "llvm/ADT/DenseMap.h"
-
-namespace llvm {
-  class raw_ostream;
-}
+#include "llvm/Support/Path.h"
 
 namespace clang {
 namespace driver {
@@ -60,6 +56,9 @@ class Compilation {
   /// Result files which should be removed on failure.
   ArgStringList ResultFiles;
 
+  /// Redirection for stdout, stderr, etc.
+  const llvm::sys::Path **Redirects;
+
 public:
   Compilation(const Driver &D, const ToolChain &DefaultToolChain,
               InputArgList *Args, DerivedArgList *TranslatedArgs);
@@ -120,7 +119,7 @@ class Compilation {
   /// \param J - The job to print.
   /// \param Terminator - A string to print at the end of the line.
   /// \param Quote - Should separate arguments be quoted.
-  void PrintJob(llvm::raw_ostream &OS, const Job &J,
+  void PrintJob(raw_ostream &OS, const Job &J,
                 const char *Terminator, bool Quote) const;
 
   /// ExecuteCommand - Execute an actual command.
@@ -136,6 +135,11 @@ class Compilation {
   /// Command which failed.
   /// \return The accumulated result code of the job.
   int ExecuteJob(const Job &J, const Command *&FailingCommand) const;
+
+  /// initCompilationForDiagnostics - Remove stale state and suppress output
+  /// so compilation can be reexecuted to generate additional diagnostic
+  /// information (e.g., preprocessed source(s)).
+  void initCompilationForDiagnostics();
 };
 
 } // end namespace driver
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Driver.h b/contrib/llvm/tools/clang/include/clang/Driver/Driver.h
index b6951663148b..6fdf6fcfad64 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Driver.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Driver.h
@@ -13,6 +13,7 @@
 #include "clang/Basic/Diagnostic.h"
 
 #include "clang/Driver/Phases.h"
+#include "clang/Driver/Types.h"
 #include "clang/Driver/Util.h"
 
 #include "llvm/ADT/StringRef.h"
@@ -24,13 +25,14 @@
 #include <string>
 
 namespace llvm {
-  class raw_ostream;
   template<typename T> class ArrayRef;
 }
 namespace clang {
 namespace driver {
   class Action;
+  class Arg;
   class ArgList;
+  class Command;
   class Compilation;
   class DerivedArgList;
   class HostInfo;
@@ -45,7 +47,7 @@ namespace driver {
 class Driver {
   OptTable *Opts;
 
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
 
 public:
   // Diag - Forwarding function for diagnostics.
@@ -75,7 +77,7 @@ class Driver {
   /// functionality.
   /// FIXME: This type of customization should be removed in favor of the
   /// universal driver when it is ready.
-  typedef llvm::SmallVector<std::string, 4> prefix_list;
+  typedef SmallVector<std::string, 4> prefix_list;
   prefix_list PrefixDirs;
 
   /// sysroot, if present
@@ -109,6 +111,9 @@ class Driver {
   /// The file to log CC_LOG_DIAGNOSTICS output to, if enabled.
   const char *CCLogDiagnosticsFilename;
 
+  /// A list of inputs and their types for the given arguments.
+  typedef SmallVector<std::pair<types::ID, const Arg*>, 16> InputList;
+
   /// Whether the driver should follow g++ like behavior.
   unsigned CCCIsCXX : 1;
 
@@ -134,6 +139,9 @@ class Driver {
   /// format.
   unsigned CCLogDiagnostics : 1;
 
+  /// Whether the driver is generating diagnostics for debugging purposes.
+  unsigned CCGenDiagnostics : 1;
+
 private:
   /// Name to use when invoking gcc/g++.
   std::string CCCGenericGCCName;
@@ -172,12 +180,17 @@ class Driver {
   /// arguments, after applying the standard argument translations.
   DerivedArgList *TranslateInputArgs(const InputArgList &Args) const;
 
+  // getFinalPhase - Determine which compilation mode we are in and record 
+  // which option we used to determine the final phase.
+  phases::ID getFinalPhase(const DerivedArgList &DAL, Arg **FinalPhaseArg = 0)
+    const;
+
 public:
-  Driver(llvm::StringRef _ClangExecutable,
-         llvm::StringRef _DefaultHostTriple,
-         llvm::StringRef _DefaultImageName,
-         bool IsProduction, bool CXXIsProduction,
-         Diagnostic &_Diags);
+  Driver(StringRef _ClangExecutable,
+         StringRef _DefaultHostTriple,
+         StringRef _DefaultImageName,
+         bool IsProduction,
+         DiagnosticsEngine &_Diags);
   ~Driver();
 
   /// @name Accessors
@@ -189,7 +202,7 @@ class Driver {
 
   const OptTable &getOpts() const { return *Opts; }
 
-  const Diagnostic &getDiags() const { return Diags; }
+  const DiagnosticsEngine &getDiags() const { return Diags; }
 
   bool getCheckInputsExist() const { return CheckInputsExist; }
 
@@ -209,7 +222,7 @@ class Driver {
       return InstalledDir.c_str();
     return Dir.c_str();
   }
-  void setInstalledDir(llvm::StringRef Value) {
+  void setInstalledDir(StringRef Value) {
     InstalledDir = Value;
   }
 
@@ -224,14 +237,24 @@ class Driver {
   /// argument vector. A null return value does not necessarily
   /// indicate an error condition, the diagnostics should be queried
   /// to determine if an error occurred.
-  Compilation *BuildCompilation(llvm::ArrayRef<const char *> Args);
+  Compilation *BuildCompilation(ArrayRef<const char *> Args);
 
   /// @name Driver Steps
   /// @{
 
   /// ParseArgStrings - Parse the given list of strings into an
   /// ArgList.
-  InputArgList *ParseArgStrings(llvm::ArrayRef<const char *> Args);
+  InputArgList *ParseArgStrings(ArrayRef<const char *> Args);
+
+  /// BuildInputs - Construct the list of inputs and their types from 
+  /// the given arguments.
+  ///
+  /// \param TC - The default host tool chain.
+  /// \param Args - The input arguments.
+  /// \param Inputs - The list to store the resulting compilation 
+  /// inputs onto.
+  void BuildInputs(const ToolChain &TC, const DerivedArgList &Args,
+                   InputList &Inputs) const;
 
   /// BuildActions - Construct the list of actions to perform for the
   /// given arguments, which are only done for a single architecture.
@@ -240,7 +263,7 @@ class Driver {
   /// \param Args - The input arguments.
   /// \param Actions - The list to store the resulting actions onto.
   void BuildActions(const ToolChain &TC, const DerivedArgList &Args,
-                    ActionList &Actions) const;
+                    const InputList &Inputs, ActionList &Actions) const;
 
   /// BuildUniversalActions - Construct the list of actions to perform
   /// for the given arguments, which may require a universal build.
@@ -249,6 +272,7 @@ class Driver {
   /// \param Args - The input arguments.
   /// \param Actions - The list to store the resulting actions onto.
   void BuildUniversalActions(const ToolChain &TC, const DerivedArgList &Args,
+                             const InputList &BAInputs,
                              ActionList &Actions) const;
 
   /// BuildJobs - Bind actions to concrete tools and translate
@@ -263,7 +287,14 @@ class Driver {
   /// This routine handles additional processing that must be done in addition
   /// to just running the subprocesses, for example reporting errors, removing
   /// temporary files, etc.
-  int ExecuteCompilation(const Compilation &C) const;
+  int ExecuteCompilation(const Compilation &C,
+                         const Command *&FailingCommand) const;
+  
+  /// generateCompilationDiagnostics - Generate diagnostics information 
+  /// including preprocessed source file(s).
+  /// 
+  void generateCompilationDiagnostics(Compilation &C,
+                                      const Command *FailingCommand);
 
   /// @}
   /// @name Helper Methods
@@ -281,7 +312,7 @@ class Driver {
   void PrintOptions(const ArgList &Args) const;
 
   /// PrintVersion - Print the driver version.
-  void PrintVersion(const Compilation &C, llvm::raw_ostream &OS) const;
+  void PrintVersion(const Compilation &C, raw_ostream &OS) const;
 
   /// GetFilePath - Lookup \arg Name in the list of file search paths.
   ///
@@ -342,11 +373,11 @@ class Driver {
                                  const char *BaseInput,
                                  bool AtTopLevel) const;
 
-  /// GetTemporaryPath - Return the pathname of a temporary file to
-  /// use as part of compilation; the file will have the given suffix.
+  /// GetTemporaryPath - Return the pathname of a temporary file to use 
+  /// as part of compilation; the file will have the given prefix and suffix.
   ///
   /// GCC goes to extra lengths here to be a bit more robust.
-  std::string GetTemporaryPath(const char *Suffix) const;
+  std::string GetTemporaryPath(StringRef Prefix, const char *Suffix) const;
 
   /// GetHostInfo - Construct a new host info object for the given
   /// host triple.
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/DriverDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Driver/DriverDiagnostic.h
index 0f9376b8dea1..844f918c1b2e 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/DriverDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/DriverDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define DRIVERSTART
 #include "clang/Basic/DiagnosticDriverKinds.inc"
 #undef DIAG
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Job.h b/contrib/llvm/tools/clang/include/clang/Driver/Job.h
index d2767d1b877b..367955f59f04 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Job.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Job.h
@@ -12,13 +12,7 @@
 
 #include "clang/Driver/Util.h"
 #include "llvm/ADT/SmallVector.h"
-
-#include "llvm/Support/Casting.h"
-using llvm::isa;
-using llvm::cast;
-using llvm::cast_or_null;
-using llvm::dyn_cast;
-using llvm::dyn_cast_or_null;
+#include "clang/Basic/LLVM.h"
 
 namespace clang {
 namespace driver {
@@ -88,7 +82,7 @@ class Command : public Job {
   /// JobList - A sequence of jobs to perform.
 class JobList : public Job {
 public:
-  typedef llvm::SmallVector<Job*, 4> list_type;
+  typedef SmallVector<Job*, 4> list_type;
   typedef list_type::size_type size_type;
   typedef list_type::iterator iterator;
   typedef list_type::const_iterator const_iterator;
@@ -103,6 +97,9 @@ class JobList : public Job {
   /// Add a job to the list (taking ownership).
   void addJob(Job *J) { Jobs.push_back(J); }
 
+  /// Clear the job list.
+  void clear();
+
   const list_type &getJobs() const { return Jobs; }
 
   size_type size() const { return Jobs.size(); }
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/OptTable.h b/contrib/llvm/tools/clang/include/clang/Driver/OptTable.h
index 3befe1defba9..abd224db449d 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/OptTable.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/OptTable.h
@@ -10,12 +10,8 @@
 #ifndef CLANG_DRIVER_OPTTABLE_H
 #define CLANG_DRIVER_OPTTABLE_H
 
+#include "clang/Basic/LLVM.h"
 #include "clang/Driver/OptSpecifier.h"
-#include <cassert>
-
-namespace llvm {
-  class raw_ostream;
-}
 
 namespace clang {
 namespace driver {
@@ -181,7 +177,7 @@ namespace options {
     /// \param Name - The name to use in the usage line.
     /// \param Title - The title to use in the usage line.
     /// \param ShowHidden - Whether help-hidden arguments should be shown.
-    void PrintHelp(llvm::raw_ostream &OS, const char *Name,
+    void PrintHelp(raw_ostream &OS, const char *Name,
                    const char *Title, bool ShowHidden = false) const;
   };
 }
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Option.h b/contrib/llvm/tools/clang/include/clang/Driver/Option.h
index 9dfa4614009f..8243f6d69314 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Option.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Option.h
@@ -12,12 +12,7 @@
 
 #include "clang/Driver/OptSpecifier.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Casting.h"
-using llvm::isa;
-using llvm::cast;
-using llvm::cast_or_null;
-using llvm::dyn_cast;
-using llvm::dyn_cast_or_null;
+#include "clang/Basic/LLVM.h"
 
 namespace clang {
 namespace driver {
@@ -65,7 +60,7 @@ namespace driver {
     OptSpecifier ID;
 
     /// The option name.
-    llvm::StringRef Name;
+    StringRef Name;
 
     /// Group this option is a member of, if any.
     const OptionGroup *Group;
@@ -104,7 +99,7 @@ namespace driver {
 
     unsigned getID() const { return ID.getID(); }
     OptionClass getKind() const { return Kind; }
-    llvm::StringRef getName() const { return Name; }
+    StringRef getName() const { return Name; }
     const OptionGroup *getGroup() const { return Group; }
     const Option *getAlias() const { return Alias; }
 
@@ -144,7 +139,7 @@ namespace driver {
 
     /// getRenderName - Return the name to use when rendering this
     /// option.
-    llvm::StringRef getRenderName() const {
+    StringRef getRenderName() const {
       return getUnaliasedOption()->getName();
     }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Options.td b/contrib/llvm/tools/clang/include/clang/Driver/Options.td
index 14fd9718d717..5b91c196ee27 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Options.td
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Options.td
@@ -122,6 +122,10 @@ def ccc_arcmt_migrate : Separate<"-ccc-arcmt-migrate">, CCCDriverOpt,
   HelpText<"Apply modifications and produces temporary files that conform to ARC">;
 def ccc_arcmt_migrate_EQ : Joined<"-ccc-arcmt-migrate=">, CCCDriverOpt,
   Alias<ccc_arcmt_migrate>;
+def arcmt_migrate_report_output : Separate<"-arcmt-migrate-report-output">,
+  HelpText<"Output path for the plist report">;
+def arcmt_migrate_emit_arc_errors : Flag<"-arcmt-migrate-emit-errors">,
+  HelpText<"Emit ARC errors even if the migrator can fix them">;
 
 // Make sure all other -ccc- options are rejected.
 def ccc_ : Joined<"-ccc-">, Group<ccc_Group>, Flags<[Unsupported]>;
@@ -130,6 +134,9 @@ def ccc_ : Joined<"-ccc-">, Group<ccc_Group>, Flags<[Unsupported]>;
 
 def _HASH_HASH_HASH : Flag<"-###">, Flags<[DriverOption]>,
     HelpText<"Print the commands to run for this compilation">;
+// The '--' option is here for the sake of compatibility with gcc, but is 
+// being ignored by the driver.
+def _DASH_DASH : Flag<"--">, Flags<[DriverOption]>;
 def A : JoinedOrSeparate<"-A">;
 def B : JoinedOrSeparate<"-B">;
 def CC : Flag<"-CC">;
@@ -316,7 +323,7 @@ def fno_gnu89_inline : Flag<"-fno-gnu89-inline">, Group<f_Group>;
 def fgnu_runtime : Flag<"-fgnu-runtime">, Group<f_Group>;
 def fheinous_gnu_extensions : Flag<"-fheinous-gnu-extensions">;
 def filelist : Separate<"-filelist">, Flags<[LinkerInput]>;
-def findirect_virtual_calls : Flag<"-findirect-virtual-calls">, Group<f_Group>;
+def findirect_virtual_calls : Flag<"-findirect-virtual-calls">, Alias<fapple_kext>;
 def finline_functions : Flag<"-finline-functions">, Group<clang_ignored_f_Group>;
 def finline : Flag<"-finline">, Group<clang_ignored_f_Group>;
 def finstrument_functions : Flag<"-finstrument-functions">, Group<f_Group>;
@@ -334,8 +341,14 @@ def fmath_errno : Flag<"-fmath-errno">, Group<f_Group>;
 def fmerge_all_constants : Flag<"-fmerge-all-constants">, Group<f_Group>;
 def fmessage_length_EQ : Joined<"-fmessage-length=">, Group<f_Group>;
 def fms_extensions : Flag<"-fms-extensions">, Group<f_Group>;
+def fms_compatibility : Flag<"-fms-compatibility">, Group<f_Group>;
 def fmsc_version : Joined<"-fmsc-version=">, Group<f_Group>;
 def fdelayed_template_parsing : Flag<"-fdelayed-template-parsing">, Group<f_Group>;
+def fmodule_cache_path : Separate<"-fmodule-cache-path">, Group<i_Group>, 
+  Flags<[NoForward]>;
+def fauto_module_import : Flag <"-fauto-module-import">, Group<f_Group>,
+  Flags<[NoForward]>;
+  
 def fmudflapth : Flag<"-fmudflapth">, Group<f_Group>;
 def fmudflap : Flag<"-fmudflap">, Group<f_Group>;
 def fnested_functions : Flag<"-fnested-functions">, Group<f_Group>;
@@ -371,6 +384,7 @@ def fno_lax_vector_conversions : Flag<"-fno-lax-vector-conversions">, Group<f_Gr
 def fno_math_errno : Flag<"-fno-math-errno">, Group<f_Group>;
 def fno_merge_all_constants : Flag<"-fno-merge-all-constants">, Group<f_Group>;
 def fno_ms_extensions : Flag<"-fno-ms-extensions">, Group<f_Group>;
+def fno_ms_compatibility : Flag<"-fno-ms-compatibility">, Group<f_Group>;
 def fno_delayed_template_parsing : Flag<"-fno-delayed-template-parsing">, Group<f_Group>;
 def fno_objc_default_synthesize_properties 
   : Flag<"-fno-objc-default-synthesize-properties">, Group<f_Group>;
@@ -427,6 +441,9 @@ def force__cpusubtype__ALL : Flag<"-force_cpusubtype_ALL">;
 def force__flat__namespace : Flag<"-force_flat_namespace">;
 def force__load : Separate<"-force_load">;
 def foutput_class_dir_EQ : Joined<"-foutput-class-dir=">, Group<f_Group>;
+def fpack_struct : Flag<"-fpack-struct">, Group<f_Group>;
+def fno_pack_struct : Flag<"-fno-pack-struct">, Group<f_Group>;
+def fpack_struct_EQ : Joined<"-fpack-struct=">, Group<f_Group>;
 def fpascal_strings : Flag<"-fpascal-strings">, Group<f_Group>;
 def fpch_preprocess : Flag<"-fpch-preprocess">, Group<f_Group>;
 def fpic : Flag<"-fpic">, Group<f_Group>;
@@ -467,7 +484,7 @@ def Wlarger_than_ : Joined<"-Wlarger-than-">, Alias<Wlarger_than>;
 def Wframe_larger_than : Separate<"-Wframe-larger-than">, Group<clang_ignored_f_Group>;
 def Wframe_larger_than_EQ : Joined<"-Wframe-larger-than=">, Alias<Wframe_larger_than>;
 
-def fterminated_vtables : Flag<"-fterminated-vtables">, Group<f_Group>;
+def fterminated_vtables : Flag<"-fterminated-vtables">, Alias<fapple_kext>;
 def fthreadsafe_statics : Flag<"-fthreadsafe-statics">, Group<f_Group>;
 def ftime_report : Flag<"-ftime-report">, Group<f_Group>;
 def ftrapv : Flag<"-ftrapv">, Group<f_Group>;
@@ -497,6 +514,7 @@ def gused : Joined<"-gused">, Group<g_Group>;
 def g_Flag : Flag<"-g">, Group<g_Group>;
 def g_Joined : Joined<"-g">, Group<g_Group>;
 def headerpad__max__install__names : Joined<"-headerpad_max_install_names">;
+def index_header_map : Flag<"-index-header-map">;
 def idirafter : JoinedOrSeparate<"-idirafter">, Group<clang_i_Group>;
 def iframework : JoinedOrSeparate<"-iframework">, Group<clang_i_Group>;
 def imacros : JoinedOrSeparate<"-imacros">, Group<clang_i_Group>;
@@ -529,6 +547,7 @@ def mdynamic_no_pic : Joined<"-mdynamic-no-pic">, Group<m_Group>, Flags<[NoArgum
 def mfix_and_continue : Flag<"-mfix-and-continue">, Group<clang_ignored_m_Group>;
 def mfloat_abi_EQ : Joined<"-mfloat-abi=">, Group<m_Group>;
 def mfpu_EQ : Joined<"-mfpu=">, Group<m_Group>;
+def mglobal_merge : Flag<"-mglobal-merge">, Group<m_Group>;
 def mhard_float : Flag<"-mhard-float">, Group<m_Group>;
 def miphoneos_version_min_EQ : Joined<"-miphoneos-version-min=">, Group<m_Group>;
 def mios_version_min_EQ : Joined<"-mios-version-min=">, Alias<miphoneos_version_min_EQ>;
@@ -543,6 +562,7 @@ def mmmx : Flag<"-mmmx">, Group<m_x86_Features_Group>;
 def mno_3dnowa : Flag<"-mno-3dnowa">, Group<m_x86_Features_Group>;
 def mno_3dnow : Flag<"-mno-3dnow">, Group<m_x86_Features_Group>;
 def mno_constant_cfstrings : Flag<"-mno-constant-cfstrings">, Group<m_Group>;
+def mno_global_merge : Flag<"-mno-global-merge">, Group<m_Group>;
 def mno_mmx : Flag<"-mno-mmx">, Group<m_x86_Features_Group>;
 def mno_pascal_strings : Flag<"-mno-pascal-strings">, Group<m_Group>;
 def mno_red_zone : Flag<"-mno-red-zone">, Group<m_Group>;
@@ -605,6 +625,7 @@ def noprebind : Flag<"-noprebind">;
 def noseglinkedit : Flag<"-noseglinkedit">;
 def nostartfiles : Flag<"-nostartfiles">;
 def nostdinc : Flag<"-nostdinc">;
+def nostdlibinc : Flag<"-nostdlibinc">;
 def nostdincxx : Flag<"-nostdinc++">;
 def nostdlib : Flag<"-nostdlib">;
 def object : Flag<"-object">;
@@ -689,6 +710,8 @@ def u : JoinedOrSeparate<"-u">, Group<u_Group>;
 def use_gold_plugin : Flag<"-use-gold-plugin">;
 def v : Flag<"-v">,
   HelpText<"Show commands to run and use verbose output">;
+def verify : Flag<"-verify">, Flags<[DriverOption]>,
+  HelpText<"Verify output using a verifier.">;
 def weak_l : Joined<"-weak-l">, Flags<[LinkerInput]>;
 def weak__framework : Separate<"-weak_framework">, Flags<[LinkerInput]>;
 def weak__library : Separate<"-weak_library">, Flags<[LinkerInput]>;
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Tool.h b/contrib/llvm/tools/clang/include/clang/Driver/Tool.h
index c30fa4c6e731..378b516b39e5 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Tool.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Tool.h
@@ -10,9 +10,7 @@
 #ifndef CLANG_DRIVER_TOOL_H_
 #define CLANG_DRIVER_TOOL_H_
 
-namespace llvm {
-  template<typename T, unsigned N> class SmallVector;
-}
+#include "clang/Basic/LLVM.h"
 
 namespace clang {
 namespace driver {
@@ -23,7 +21,7 @@ namespace driver {
   class JobAction;
   class ToolChain;
 
-  typedef llvm::SmallVector<InputInfo, 4> InputInfoList;
+  typedef SmallVector<InputInfo, 4> InputInfoList;
 
 /// Tool - Information on a specific compilation tool.
 class Tool {
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h b/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h
index 4836d3ffac22..a5d51ca9d7c9 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h
@@ -32,7 +32,7 @@ namespace driver {
 /// ToolChain - Access to tools for a single platform.
 class ToolChain {
 public:
-  typedef llvm::SmallVector<std::string, 4> path_list;
+  typedef SmallVector<std::string, 4> path_list;
 
   enum CXXStdlibType {
     CST_Libcxx,
@@ -63,9 +63,9 @@ class ToolChain {
   const llvm::Triple &getTriple() const { return Triple; }
 
   llvm::Triple::ArchType getArch() const { return Triple.getArch(); }
-  llvm::StringRef getArchName() const { return Triple.getArchName(); }
-  llvm::StringRef getPlatform() const { return Triple.getVendorName(); }
-  llvm::StringRef getOS() const { return Triple.getOSName(); }
+  StringRef getArchName() const { return Triple.getArchName(); }
+  StringRef getPlatform() const { return Triple.getVendorName(); }
+  StringRef getOS() const { return Triple.getOSName(); }
 
   std::string getTripleString() const {
     return Triple.getTriple();
@@ -139,7 +139,9 @@ class ToolChain {
 
   /// GetDefaultStackProtectorLevel - Get the default stack protector level for
   /// this tool chain (0=off, 1=on, 2=all).
-  virtual unsigned GetDefaultStackProtectorLevel() const { return 0; }
+  virtual unsigned GetDefaultStackProtectorLevel(bool KernelOrKext) const {
+    return 0;
+  }
 
   /// IsUnwindTablesDefault - Does this tool chain use -funwind-tables
   /// by default.
@@ -169,21 +171,30 @@ class ToolChain {
 
   /// ComputeLLVMTriple - Return the LLVM target triple to use, after taking
   /// command line arguments into account.
-  virtual std::string ComputeLLVMTriple(const ArgList &Args) const;
+  virtual std::string ComputeLLVMTriple(const ArgList &Args,
+                                 types::ID InputType = types::TY_INVALID) const;
 
   /// ComputeEffectiveClangTriple - Return the Clang triple to use for this
   /// target, which may take into account the command line arguments. For
   /// example, on Darwin the -mmacosx-version-min= command line argument (which
   /// sets the deployment target) determines the version in the triple passed to
   /// Clang.
-  virtual std::string ComputeEffectiveClangTriple(const ArgList &Args) const;
+  virtual std::string ComputeEffectiveClangTriple(const ArgList &Args,
+                                 types::ID InputType = types::TY_INVALID) const;
 
   /// configureObjCRuntime - Configure the known properties of the
   /// Objective-C runtime for this platform.
   ///
-  /// FIXME: this doesn't really belong here.
+  /// FIXME: this really belongs on some sort of DeploymentTarget abstraction
   virtual void configureObjCRuntime(ObjCRuntime &runtime) const;
 
+  /// hasBlocksRuntime - Given that the user is compiling with
+  /// -fblocks, does this tool chain guarantee the existence of a
+  /// blocks runtime?
+  ///
+  /// FIXME: this really belongs on some sort of DeploymentTarget abstraction
+  virtual bool hasBlocksRuntime() const { return true; }
+
   // GetCXXStdlibType - Determine the C++ standard library type to use with the
   // given compilation arguments.
   virtual CXXStdlibType GetCXXStdlibType(const ArgList &Args) const;
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Types.def b/contrib/llvm/tools/clang/include/clang/Driver/Types.def
index f09a1dcaf26f..8449d639e698 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Types.def
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Types.def
@@ -44,15 +44,18 @@ TYPE("c",                        C,            PP_C,            0,       "u")
 TYPE("cl",                       CL,           PP_C,            0,       "u")
 TYPE("cuda",                     CUDA,         PP_CXX,          0,       "u")
 TYPE("objective-c-cpp-output",   PP_ObjC,      INVALID,         "mi",    "u")
+TYPE("objc-cpp-output",          PP_ObjC_Alias, INVALID,        "mi",    "u")
 TYPE("objective-c",              ObjC,         PP_ObjC,         0,       "u")
 TYPE("c++-cpp-output",           PP_CXX,       INVALID,         "ii",    "u")
 TYPE("c++",                      CXX,          PP_CXX,          0,       "u")
 TYPE("objective-c++-cpp-output", PP_ObjCXX,    INVALID,         "mii",   "u")
+TYPE("objc++-cpp-output",        PP_ObjCXX_Alias, INVALID,      "mii",   "u")
 TYPE("objective-c++",            ObjCXX,       PP_ObjCXX,       0,       "u")
 
 // C family input files to precompile.
 TYPE("c-header-cpp-output",      PP_CHeader,   INVALID,         "i",     "p")
 TYPE("c-header",                 CHeader,      PP_CHeader,      0,       "pu")
+TYPE("cl-header",                CLHeader,     PP_CHeader,      0,       "pu")
 TYPE("objective-c-header-cpp-output", PP_ObjCHeader, INVALID,   "mi",    "p")
 TYPE("objective-c-header",       ObjCHeader,   PP_ObjCHeader,   0,       "pu")
 TYPE("c++-header-cpp-output",    PP_CXXHeader, INVALID,         "ii",    "p")
diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Util.h b/contrib/llvm/tools/clang/include/clang/Driver/Util.h
index 52f268d182a8..65aef4b31025 100644
--- a/contrib/llvm/tools/clang/include/clang/Driver/Util.h
+++ b/contrib/llvm/tools/clang/include/clang/Driver/Util.h
@@ -10,19 +10,17 @@
 #ifndef CLANG_DRIVER_UTIL_H_
 #define CLANG_DRIVER_UTIL_H_
 
-namespace llvm {
-  template<typename T, unsigned N> class SmallVector;
-}
+#include "clang/Basic/LLVM.h"
 
 namespace clang {
 namespace driver {
   class Action;
 
   /// ArgStringList - Type used for constructing argv lists for subprocesses.
-  typedef llvm::SmallVector<const char*, 16> ArgStringList;
+  typedef SmallVector<const char*, 16> ArgStringList;
 
   /// ActionList - Type used for lists of actions.
-  typedef llvm::SmallVector<Action*, 3> ActionList;
+  typedef SmallVector<Action*, 3> ActionList;
 
 } // end namespace driver
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/ASTConsumers.h b/contrib/llvm/tools/clang/include/clang/Frontend/ASTConsumers.h
index 3c05834ad6a7..cef9509a9793 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/ASTConsumers.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/ASTConsumers.h
@@ -14,15 +14,16 @@
 #ifndef DRIVER_ASTCONSUMERS_H
 #define DRIVER_ASTCONSUMERS_H
 
+#include "clang/Basic/LLVM.h"
+
 namespace llvm {
-  class raw_ostream;
   namespace sys { class Path; }
 }
 namespace clang {
 
 class ASTConsumer;
 class CodeGenOptions;
-class Diagnostic;
+class DiagnosticsEngine;
 class FileManager;
 class LangOptions;
 class Preprocessor;
@@ -32,7 +33,7 @@ class TargetOptions;
 // original C code.  The output is intended to be in a format such that
 // clang could re-parse the output back into the same AST, but the
 // implementation is still incomplete.
-ASTConsumer *CreateASTPrinter(llvm::raw_ostream *OS);
+ASTConsumer *CreateASTPrinter(raw_ostream *OS);
 
 // AST dumper: dumps the raw AST in human-readable form to stderr; this is
 // intended for debugging.
@@ -40,7 +41,7 @@ ASTConsumer *CreateASTDumper();
 
 // AST XML-dumper: dumps out the AST to stderr in a very detailed XML
 // format; this is intended for particularly intense debugging.
-ASTConsumer *CreateASTDumperXML(llvm::raw_ostream &OS);
+ASTConsumer *CreateASTDumperXML(raw_ostream &OS);
 
 // Graphical AST viewer: for each function definition, creates a graph of
 // the AST and displays it with the graph viewer "dotty".  Also outputs
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/ASTUnit.h b/contrib/llvm/tools/clang/include/clang/Frontend/ASTUnit.h
index 58a60a1f9d87..471476a44477 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/ASTUnit.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/ASTUnit.h
@@ -18,6 +18,7 @@
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
@@ -41,10 +42,11 @@ namespace llvm {
 
 namespace clang {
 class ASTContext;
+class ASTReader;
 class CodeCompleteConsumer;
 class CompilerInvocation;
 class Decl;
-class Diagnostic;
+class DiagnosticsEngine;
 class FileEntry;
 class FileManager;
 class HeaderSearch;
@@ -65,19 +67,15 @@ class GlobalCodeCompletionAllocator
   
 /// \brief Utility class for loading a ASTContext from an AST file.
 ///
-class ASTUnit {
-public:
-  typedef std::map<FileID, std::vector<PreprocessedEntity *> > 
-    PreprocessedEntitiesByFileMap;
-  
+class ASTUnit : public ModuleLoader {
 private:
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diagnostics;
-  llvm::IntrusiveRefCntPtr<FileManager>      FileMgr;
-  llvm::IntrusiveRefCntPtr<SourceManager>    SourceMgr;
-  llvm::OwningPtr<HeaderSearch>     HeaderInfo;
-  llvm::IntrusiveRefCntPtr<TargetInfo>       Target;
-  llvm::IntrusiveRefCntPtr<Preprocessor>     PP;
-  llvm::IntrusiveRefCntPtr<ASTContext>       Ctx;
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
+  llvm::IntrusiveRefCntPtr<FileManager>       FileMgr;
+  llvm::IntrusiveRefCntPtr<SourceManager>     SourceMgr;
+  llvm::OwningPtr<HeaderSearch>               HeaderInfo;
+  llvm::IntrusiveRefCntPtr<TargetInfo>        Target;
+  llvm::IntrusiveRefCntPtr<Preprocessor>      PP;
+  llvm::IntrusiveRefCntPtr<ASTContext>        Ctx;
 
   FileSystemOptions FileSystemOpts;
 
@@ -111,8 +109,8 @@ class ASTUnit {
   /// \brief Track whether the main file was loaded from an AST or not.
   bool MainFileIsAST;
 
-  /// \brief Whether this AST represents a complete translation unit.
-  bool CompleteTranslationUnit;
+  /// \brief What kind of translation unit this AST represents.
+  TranslationUnitKind TUKind;
 
   /// \brief Whether we should time each operation.
   bool WantTiming;
@@ -128,14 +126,6 @@ class ASTUnit {
   // source. In the long term we should make the Index library use efficient and
   // more scalable search mechanisms.
   std::vector<Decl*> TopLevelDecls;
-
-  /// \brief The list of preprocessed entities which appeared when the ASTUnit
-  /// was loaded.
-  ///
-  /// FIXME: This is just an optimization hack to avoid deserializing large
-  /// parts of a PCH file while performing a walk or search. In the long term,
-  /// we should provide more scalable search mechanisms.
-  std::vector<PreprocessedEntity *> PreprocessedEntities;
   
   /// The name of the original source file used to generate this ASTUnit.
   std::string OriginalSourceFile;
@@ -143,9 +133,12 @@ class ASTUnit {
   // Critical optimization when using clang_getCursor().
   ASTLocation LastLoc;
 
+  /// \brief The set of diagnostics produced when creating the preamble.
+  SmallVector<StoredDiagnostic, 4> PreambleDiagnostics;
+
   /// \brief The set of diagnostics produced when creating this
   /// translation unit.
-  llvm::SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
+  SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
 
   /// \brief The number of stored diagnostics that come from the driver
   /// itself.
@@ -156,27 +149,8 @@ class ASTUnit {
   
   /// \brief Temporary files that should be removed when the ASTUnit is 
   /// destroyed.
-  llvm::SmallVector<llvm::sys::Path, 4> TemporaryFiles;
-
-  /// \brief A mapping from file IDs to the set of preprocessed entities
-  /// stored in that file. 
-  ///
-  /// FIXME: This is just an optimization hack to avoid searching through
-  /// many preprocessed entities during cursor traversal in the CIndex library.
-  /// Ideally, we would just be able to perform a binary search within the
-  /// list of preprocessed entities.
-  PreprocessedEntitiesByFileMap PreprocessedEntitiesByFile;
+  SmallVector<llvm::sys::Path, 4> TemporaryFiles;
   
-  /// \brief Simple hack to allow us to assert that ASTUnit is not being
-  /// used concurrently, which is not supported.
-  ///
-  /// Clients should create instances of the ConcurrencyCheck class whenever
-  /// using the ASTUnit in a way that isn't intended to be concurrent, which is
-  /// just about any usage.
-  unsigned int ConcurrencyCheckValue;
-  static const unsigned int CheckLocked = 28573289;
-  static const unsigned int CheckUnlocked = 9803453;
-
   /// \brief Counter that determines when we want to try building a
   /// precompiled preamble.
   ///
@@ -191,9 +165,53 @@ class ASTUnit {
   /// \brief The file in which the precompiled preamble is stored.
   std::string PreambleFile;
   
+public:
+  class PreambleData {
+    const FileEntry *File;
+    std::vector<char> Buffer;
+    mutable unsigned NumLines;
+    
+  public:
+    PreambleData() : File(0), NumLines(0) { }
+    
+    void assign(const FileEntry *F, const char *begin, const char *end) {
+      File = F;
+      Buffer.assign(begin, end);
+      NumLines = 0;
+    }
+
+    void clear() { Buffer.clear(); File = 0; NumLines = 0; }
+
+    size_t size() const { return Buffer.size(); }
+    bool empty() const { return Buffer.empty(); }
+
+    const char *getBufferStart() const { return &Buffer[0]; }
+
+    unsigned getNumLines() const {
+      if (NumLines)
+        return NumLines;
+      countLines();
+      return NumLines;
+    }
+
+    SourceRange getSourceRange(const SourceManager &SM) const {
+      SourceLocation FileLoc = SM.getLocForStartOfFile(SM.getPreambleFileID());
+      return SourceRange(FileLoc, FileLoc.getLocWithOffset(size()-1));
+    }
+
+  private:
+    void countLines() const;
+  };
+
+  const PreambleData &getPreambleData() const {
+    return Preamble;
+  }
+
+private:
+
   /// \brief The contents of the preamble that has been precompiled to
   /// \c PreambleFile.
-  std::vector<char> Preamble;
+  PreambleData Preamble;
 
   /// \brief Whether the preamble ends at the start of a new line.
   /// 
@@ -224,27 +242,15 @@ class ASTUnit {
 
   /// \brief The number of warnings that occurred while parsing the preamble.
   ///
-  /// This value will be used to restore the state of the \c Diagnostic object
-  /// when re-using the precompiled preamble. Note that only the
+  /// This value will be used to restore the state of the \c DiagnosticsEngine
+  /// object when re-using the precompiled preamble. Note that only the
   /// number of warnings matters, since we will not save the preamble
   /// when any errors are present.
   unsigned NumWarningsInPreamble;
 
-  /// \brief The number of diagnostics that were stored when parsing
-  /// the precompiled preamble.
-  ///
-  /// This value is used to determine how many of the stored
-  /// diagnostics should be retained when reparsing in the presence of
-  /// a precompiled preamble.
-  unsigned NumStoredDiagnosticsInPreamble;
-
   /// \brief A list of the serialization ID numbers for each of the top-level
   /// declarations parsed within the precompiled preamble.
   std::vector<serialization::DeclID> TopLevelDeclsInPreamble;
-
-  /// \brief A list of the offsets into the precompiled preamble which
-  /// correspond to preprocessed entities.
-  std::vector<uint64_t> PreprocessedEntitiesInPreamble;
   
   /// \brief Whether we should be caching code-completion results.
   bool ShouldCacheCodeCompletionResults;
@@ -252,11 +258,19 @@ class ASTUnit {
   /// \brief Whether we want to include nested macro expansions in the
   /// detailed preprocessing record.
   bool NestedMacroExpansions;
-  
-  static void ConfigureDiags(llvm::IntrusiveRefCntPtr<Diagnostic> &Diags,
+ 
+  /// \brief The language options used when we load an AST file.
+  LangOptions ASTFileLangOpts;
+
+  static void ConfigureDiags(llvm::IntrusiveRefCntPtr<DiagnosticsEngine> &Diags,
                              const char **ArgBegin, const char **ArgEnd,
                              ASTUnit &AST, bool CaptureDiagnostics);
 
+  void TranslateStoredDiagnostics(ASTReader *MMan, StringRef ModName,
+                                  SourceManager &SrcMan,
+                      const SmallVectorImpl<StoredDiagnostic> &Diags,
+                            SmallVectorImpl<StoredDiagnostic> &Out);
+
 public:
   /// \brief A cached code-completion result, which may be introduced in one of
   /// many different contexts.
@@ -309,10 +323,24 @@ class ASTUnit {
     return CachedCompletionAllocator;
   }
   
+  /// \brief Retrieve the allocator used to cache global code completions.
+  /// Creates the allocator if it doesn't already exist.
+  llvm::IntrusiveRefCntPtr<GlobalCodeCompletionAllocator>
+  getCursorCompletionAllocator() {
+    if (!CursorCompletionAllocator.getPtr()) {
+      CursorCompletionAllocator = new GlobalCodeCompletionAllocator;
+    }
+    return CursorCompletionAllocator;
+  }
+  
 private:
   /// \brief Allocator used to store cached code completions.
   llvm::IntrusiveRefCntPtr<GlobalCodeCompletionAllocator>
     CachedCompletionAllocator;
+  
+  /// \brief Allocator used to store code completions for arbitrary cursors.
+  llvm::IntrusiveRefCntPtr<GlobalCodeCompletionAllocator>
+    CursorCompletionAllocator;
 
   /// \brief The set of cached code-completion results.
   std::vector<CachedCodeCompletionResult> CachedCompletionResults;
@@ -367,23 +395,38 @@ class ASTUnit {
                                                      bool AllowRebuild = true,
                                                         unsigned MaxLines = 0);
   void RealizeTopLevelDeclsFromPreamble();
-  void RealizePreprocessedEntitiesFromPreamble();
   
+  /// \brief Allows us to assert that ASTUnit is not being used concurrently,
+  /// which is not supported.
+  ///
+  /// Clients should create instances of the ConcurrencyCheck class whenever
+  /// using the ASTUnit in a way that isn't intended to be concurrent, which is
+  /// just about any usage.
+  /// Becomes a noop in release mode; only useful for debug mode checking.
+  class ConcurrencyState {
+    void *Mutex; // a llvm::sys::MutexImpl in debug;
+
+  public:
+    ConcurrencyState();
+    ~ConcurrencyState();
+
+    void start();
+    void finish();
+  };
+  ConcurrencyState ConcurrencyCheckValue;
+
 public:
   class ConcurrencyCheck {
-    volatile ASTUnit &Self;
+    ASTUnit &Self;
     
   public:
     explicit ConcurrencyCheck(ASTUnit &Self)
       : Self(Self) 
     { 
-      assert(Self.ConcurrencyCheckValue == CheckUnlocked && 
-             "Concurrent access to ASTUnit!");
-      Self.ConcurrencyCheckValue = CheckLocked;
+      Self.ConcurrencyCheckValue.start();
     }
-    
     ~ConcurrencyCheck() {
-      Self.ConcurrencyCheckValue = CheckUnlocked;
+      Self.ConcurrencyCheckValue.finish();
     }
   };
   friend class ConcurrencyCheck;
@@ -395,8 +438,8 @@ class ASTUnit {
   bool isUnsafeToFree() const { return UnsafeToFree; }
   void setUnsafeToFree(bool Value) { UnsafeToFree = Value; }
 
-  const Diagnostic &getDiagnostics() const { return *Diagnostics; }
-  Diagnostic &getDiagnostics()             { return *Diagnostics; }
+  const DiagnosticsEngine &getDiagnostics() const { return *Diagnostics; }
+  DiagnosticsEngine &getDiagnostics()             { return *Diagnostics; }
   
   const SourceManager &getSourceManager() const { return *SourceMgr; }
         SourceManager &getSourceManager()       { return *SourceMgr; }
@@ -407,6 +450,8 @@ class ASTUnit {
   const ASTContext &getASTContext() const { return *Ctx; }
         ASTContext &getASTContext()       { return *Ctx; }
 
+  void setASTContext(ASTContext *ctx) { Ctx = ctx; }
+
   bool hasSema() const { return TheSema; }
   Sema &getSema() const { 
     assert(TheSema && "ASTUnit does not have a Sema object!");
@@ -419,7 +464,6 @@ class ASTUnit {
   const FileSystemOptions &getFileSystemOpts() const { return FileSystemOpts; }
 
   const std::string &getOriginalSourceFileName();
-  const std::string &getASTFileName();
 
   /// \brief Add a temporary file that the ASTUnit depends on.
   ///
@@ -433,15 +477,11 @@ class ASTUnit {
   bool getOwnsRemappedFileBuffers() const { return OwnsRemappedFileBuffers; }
   void setOwnsRemappedFileBuffers(bool val) { OwnsRemappedFileBuffers = val; }
 
-  /// \brief Retrieve the maximum PCH level of declarations that a
-  /// traversal of the translation unit should consider.
-  unsigned getMaxPCHLevel() const;
-
   void setLastASTLocation(ASTLocation ALoc) { LastLoc = ALoc; }
   ASTLocation getLastASTLocation() const { return LastLoc; }
 
 
-  llvm::StringRef getMainFileName() const;
+  StringRef getMainFileName() const;
 
   typedef std::vector<Decl *>::iterator top_level_iterator;
 
@@ -484,22 +524,38 @@ class ASTUnit {
   ///
   /// Note: This is used internally by the top-level tracking action
   unsigned &getCurrentTopLevelHashValue() { return CurrentTopLevelHashValue; }
-  
-  typedef std::vector<PreprocessedEntity *>::iterator pp_entity_iterator;
-  
-  pp_entity_iterator pp_entity_begin();
-  pp_entity_iterator pp_entity_end();
-  
-  /// \brief Add a new preprocessed entity that's stored at the given offset
-  /// in the precompiled preamble.
-  void addPreprocessedEntityFromPreamble(uint64_t Offset) {
-    PreprocessedEntitiesInPreamble.push_back(Offset);
+
+  /// \brief Get the source location for the given file:line:col triplet.
+  ///
+  /// The difference with SourceManager::getLocation is that this method checks
+  /// whether the requested location points inside the precompiled preamble
+  /// in which case the returned source location will be a "loaded" one.
+  SourceLocation getLocation(const FileEntry *File,
+                             unsigned Line, unsigned Col) const;
+
+  /// \brief Get the source location for the given file:offset pair.
+  SourceLocation getLocation(const FileEntry *File, unsigned Offset) const;
+
+  /// \brief If \arg Loc is a loaded location from the preamble, returns
+  /// the corresponding local location of the main file, otherwise it returns
+  /// \arg Loc.
+  SourceLocation mapLocationFromPreamble(SourceLocation Loc);
+
+  /// \brief If \arg Loc is a local location of the main file but inside the
+  /// preamble chunk, returns the corresponding loaded location from the
+  /// preamble, otherwise it returns \arg Loc.
+  SourceLocation mapLocationToPreamble(SourceLocation Loc);
+
+  /// \brief \see mapLocationFromPreamble.
+  SourceRange mapRangeFromPreamble(SourceRange R) {
+    return SourceRange(mapLocationFromPreamble(R.getBegin()),
+                       mapLocationFromPreamble(R.getEnd()));
   }
-  
-  /// \brief Retrieve the mapping from File IDs to the preprocessed entities
-  /// within that file.
-  PreprocessedEntitiesByFileMap &getPreprocessedEntitiesByFile() {
-    return PreprocessedEntitiesByFile;
+
+  /// \brief \see mapLocationToPreamble.
+  SourceRange mapRangeToPreamble(SourceRange R) {
+    return SourceRange(mapLocationToPreamble(R.getBegin()),
+                       mapLocationToPreamble(R.getEnd()));
   }
   
   // Retrieve the diagnostics associated with this AST
@@ -512,7 +568,7 @@ class ASTUnit {
   }
   unsigned stored_diag_size() const { return StoredDiagnostics.size(); }
   
-  llvm::SmallVector<StoredDiagnostic, 4> &getStoredDiagnostics() { 
+  SmallVector<StoredDiagnostic, 4> &getStoredDiagnostics() { 
     return StoredDiagnostics; 
   }
 
@@ -531,14 +587,11 @@ class ASTUnit {
     return CachedCompletionResults.size(); 
   }
 
-  llvm::MemoryBuffer *getBufferForFile(llvm::StringRef Filename,
+  llvm::MemoryBuffer *getBufferForFile(StringRef Filename,
                                        std::string *ErrorStr = 0);
 
-  /// \brief Whether this AST represents a complete translation unit.
-  ///
-  /// If false, this AST is only a partial translation unit, e.g., one
-  /// that might still be used as a precompiled header or preamble.
-  bool isCompleteTranslationUnit() const { return CompleteTranslationUnit; }
+  /// \brief Determine what kind of translation unit this AST represents.
+  TranslationUnitKind getTranslationUnitKind() const { return TUKind; }
 
   typedef llvm::PointerUnion<const char *, const llvm::MemoryBuffer *>
       FilenameOrMemBuf;
@@ -548,7 +601,7 @@ class ASTUnit {
 
   /// \brief Create a ASTUnit. Gets ownership of the passed CompilerInvocation. 
   static ASTUnit *create(CompilerInvocation *CI,
-                         llvm::IntrusiveRefCntPtr<Diagnostic> Diags);
+                         llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags);
 
   /// \brief Create a ASTUnit from an AST file.
   ///
@@ -559,7 +612,7 @@ class ASTUnit {
   ///
   /// \returns - The initialized ASTUnit or null if the AST failed to load.
   static ASTUnit *LoadFromASTFile(const std::string &Filename,
-                                  llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
                                   const FileSystemOptions &FileSystemOpts,
                                   bool OnlyLocalDecls = false,
                                   RemappedFile *RemappedFiles = 0,
@@ -590,9 +643,13 @@ class ASTUnit {
   ///
   /// \param Action - The ASTFrontendAction to invoke. Its ownership is not
   /// transfered.
+  ///
+  /// \param Unit - optionally an already created ASTUnit. Its ownership is not
+  /// transfered.
   static ASTUnit *LoadFromCompilerInvocationAction(CompilerInvocation *CI,
-                                     llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
-                                             ASTFrontendAction *Action = 0);
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
+                                             ASTFrontendAction *Action = 0,
+                                             ASTUnit *Unit = 0);
 
   /// LoadFromCompilerInvocation - Create an ASTUnit from a source file, via a
   /// CompilerInvocation object.
@@ -606,11 +663,11 @@ class ASTUnit {
   // FIXME: Move OnlyLocalDecls, UseBumpAllocator to setters on the ASTUnit, we
   // shouldn't need to specify them at construction time.
   static ASTUnit *LoadFromCompilerInvocation(CompilerInvocation *CI,
-                                     llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
                                              bool OnlyLocalDecls = false,
                                              bool CaptureDiagnostics = false,
                                              bool PrecompilePreamble = false,
-                                          bool CompleteTranslationUnit = true,
+                                      TranslationUnitKind TUKind = TU_Complete,
                                        bool CacheCodeCompletionResults = false,
                                        bool NestedMacroExpansions = true);
 
@@ -630,18 +687,16 @@ class ASTUnit {
   // shouldn't need to specify them at construction time.
   static ASTUnit *LoadFromCommandLine(const char **ArgBegin,
                                       const char **ArgEnd,
-                                    llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
-                                      llvm::StringRef ResourceFilesPath,
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
+                                      StringRef ResourceFilesPath,
                                       bool OnlyLocalDecls = false,
                                       bool CaptureDiagnostics = false,
                                       RemappedFile *RemappedFiles = 0,
                                       unsigned NumRemappedFiles = 0,
                                       bool RemappedFilesKeepOriginalName = true,
                                       bool PrecompilePreamble = false,
-                                      bool CompleteTranslationUnit = true,
+                                      TranslationUnitKind TUKind = TU_Complete,
                                       bool CacheCodeCompletionResults = false,
-                                      bool CXXPrecompilePreamble = false,
-                                      bool CXXChainedPCH = false,
                                       bool NestedMacroExpansions = true);
   
   /// \brief Reparse the source files using the same command-line options that
@@ -669,24 +724,31 @@ class ASTUnit {
   ///
   /// FIXME: The Diag, LangOpts, SourceMgr, FileMgr, StoredDiagnostics, and
   /// OwnedBuffers parameters are all disgusting hacks. They will go away.
-  void CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
+  void CodeComplete(StringRef File, unsigned Line, unsigned Column,
                     RemappedFile *RemappedFiles, unsigned NumRemappedFiles,
                     bool IncludeMacros, bool IncludeCodePatterns,
                     CodeCompleteConsumer &Consumer,
-                    Diagnostic &Diag, LangOptions &LangOpts,
+                    DiagnosticsEngine &Diag, LangOptions &LangOpts,
                     SourceManager &SourceMgr, FileManager &FileMgr,
-                    llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
-              llvm::SmallVectorImpl<const llvm::MemoryBuffer *> &OwnedBuffers);
+                    SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
+              SmallVectorImpl<const llvm::MemoryBuffer *> &OwnedBuffers);
 
   /// \brief Save this translation unit to a file with the given name.
   ///
   /// \returns An indication of whether the save was successful or not.
-  CXSaveError Save(llvm::StringRef File);
+  CXSaveError Save(StringRef File);
 
   /// \brief Serialize this translation unit with the given output stream.
   ///
   /// \returns True if an error occurred, false otherwise.
-  bool serialize(llvm::raw_ostream &OS);
+  bool serialize(raw_ostream &OS);
+  
+  virtual ModuleKey loadModule(SourceLocation ImportLoc, 
+                               IdentifierInfo &ModuleName,
+                               SourceLocation ModuleNameLoc) {
+    // ASTUnit doesn't know how to load modules (not that this matters).
+    return 0;
+  }
 };
 
 } // namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/Analyses.def b/contrib/llvm/tools/clang/include/clang/Frontend/Analyses.def
index f055549b4e24..010f889c0974 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/Analyses.def
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/Analyses.def
@@ -15,9 +15,7 @@
 #define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATFN)
 #endif
 
-ANALYSIS_STORE(BasicStore, "basic", "Use basic analyzer store", CreateBasicStoreManager)
 ANALYSIS_STORE(RegionStore, "region", "Use region-based analyzer store", CreateRegionStoreManager)
-ANALYSIS_STORE(FlatStore, "flat", "Use flat analyzer store", CreateFlatStoreManager)
 
 #ifndef ANALYSIS_CONSTRAINTS
 #define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATFN)
@@ -30,13 +28,21 @@ ANALYSIS_CONSTRAINTS(RangeConstraints, "range", "Use constraint tracking of conc
 #define ANALYSIS_DIAGNOSTICS(NAME, CMDFLAG, DESC, CREATEFN, AUTOCREATE)
 #endif
 
-ANALYSIS_DIAGNOSTICS(HTML,  "html",  "Output analysis results using HTML",   createHTMLDiagnosticClient, false)
-ANALYSIS_DIAGNOSTICS(PLIST, "plist", "Output analysis results using Plists", createPlistDiagnosticClient, true)
-ANALYSIS_DIAGNOSTICS(PLIST_HTML, "plist-html", "Output analysis results using HTML wrapped with Plists", createPlistHTMLDiagnosticClient, true)
-ANALYSIS_DIAGNOSTICS(TEXT, "text", "Text output of analysis results", createTextPathDiagnosticClient, true)
+ANALYSIS_DIAGNOSTICS(HTML,  "html",  "Output analysis results using HTML",   createHTMLDiagnosticConsumer, false)
+ANALYSIS_DIAGNOSTICS(PLIST, "plist", "Output analysis results using Plists", createPlistDiagnosticConsumer, true)
+ANALYSIS_DIAGNOSTICS(PLIST_HTML, "plist-html", "Output analysis results using HTML wrapped with Plists", createPlistHTMLDiagnosticConsumer, true)
+ANALYSIS_DIAGNOSTICS(TEXT, "text", "Text output of analysis results", createTextPathDiagnosticConsumer, true)
+
+#ifndef ANALYSIS_PURGE
+#define ANALYSIS_PURGE(NAME, CMDFLAG, DESC)
+#endif
+
+ANALYSIS_PURGE(PurgeStmt,  "statement", "Purge symbols, bindings, and constraints before every statement")
+ANALYSIS_PURGE(PurgeBlock, "block", "Purge symbols, bindings, and constraints before every basic block")
+ANALYSIS_PURGE(PurgeNone,  "none", "Do not purge symbols, bindings, or constraints")
 
 #undef ANALYSIS_STORE
 #undef ANALYSIS_CONSTRAINTS
 #undef ANALYSIS_DIAGNOSTICS
-#undef ANALYSIS_STORE
+#undef ANALYSIS_PURGE
 
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/AnalyzerOptions.h b/contrib/llvm/tools/clang/include/clang/Frontend/AnalyzerOptions.h
index ea9f5e38b7b9..3565a51d0751 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/AnalyzerOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/AnalyzerOptions.h
@@ -20,7 +20,7 @@
 
 namespace clang {
 class ASTConsumer;
-class Diagnostic;
+class DiagnosticsEngine;
 class Preprocessor;
 class LangOptions;
 
@@ -53,6 +53,13 @@ enum AnalysisDiagClients {
 NUM_ANALYSIS_DIAG_CLIENTS
 };
 
+/// AnalysisPurgeModes - Set of available strategies for dead symbol removal.
+enum AnalysisPurgeMode {
+#define ANALYSIS_PURGE(NAME, CMDFLAG, DESC) NAME,
+#include "clang/Frontend/Analyses.def"
+NumPurgeModes
+};
+
 class AnalyzerOptions {
 public:
   /// \brief Pair of checker name and enable/disable.
@@ -60,6 +67,7 @@ class AnalyzerOptions {
   AnalysisStores AnalysisStoreOpt;
   AnalysisConstraints AnalysisConstraintsOpt;
   AnalysisDiagClients AnalysisDiagOpt;
+  AnalysisPurgeMode AnalysisPurgeOpt;
   std::string AnalyzeSpecificFunction;
   unsigned MaxNodes;
   unsigned MaxLoop;
@@ -68,7 +76,6 @@ class AnalyzerOptions {
   unsigned AnalyzerDisplayProgress : 1;
   unsigned AnalyzeNestedBlocks : 1;
   unsigned EagerlyAssume : 1;
-  unsigned PurgeDead : 1;
   unsigned TrimGraph : 1;
   unsigned VisualizeEGDot : 1;
   unsigned VisualizeEGUbi : 1;
@@ -80,15 +87,15 @@ class AnalyzerOptions {
 
 public:
   AnalyzerOptions() {
-    AnalysisStoreOpt = BasicStoreModel;
+    AnalysisStoreOpt = RegionStoreModel;
     AnalysisConstraintsOpt = RangeConstraintsModel;
     AnalysisDiagOpt = PD_HTML;
+    AnalysisPurgeOpt = PurgeStmt;
     ShowCheckerHelp = 0;
     AnalyzeAll = 0;
     AnalyzerDisplayProgress = 0;
     AnalyzeNestedBlocks = 0;
     EagerlyAssume = 0;
-    PurgeDead = 1;
     TrimGraph = 0;
     VisualizeEGDot = 0;
     VisualizeEGUbi = 0;
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/ChainedDiagnosticClient.h b/contrib/llvm/tools/clang/include/clang/Frontend/ChainedDiagnosticConsumer.h
similarity index 57%
rename from contrib/llvm/tools/clang/include/clang/Frontend/ChainedDiagnosticClient.h
rename to contrib/llvm/tools/clang/include/clang/Frontend/ChainedDiagnosticConsumer.h
index 70f21901db4f..f20cf6fcf4f2 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/ChainedDiagnosticClient.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/ChainedDiagnosticConsumer.h
@@ -1,4 +1,4 @@
-//===--- ChainedDiagnosticClient.h - Chain Diagnostic Clients ---*- C++ -*-===//
+//===- ChainedDiagnosticConsumer.h - Chain Diagnostic Clients ---*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_FRONTEND_CHAINEDDIAGNOSTICCLIENT_H
-#define LLVM_CLANG_FRONTEND_CHAINEDDIAGNOSTICCLIENT_H
+#ifndef LLVM_CLANG_FRONTEND_CHAINEDDIAGNOSTICCONSUMER_H
+#define LLVM_CLANG_FRONTEND_CHAINEDDIAGNOSTICCONSUMER_H
 
 #include "clang/Basic/Diagnostic.h"
 #include "llvm/ADT/OwningPtr.h"
@@ -16,17 +16,17 @@
 namespace clang {
 class LangOptions;
 
-/// ChainedDiagnosticClient - Chain two diagnostic clients so that diagnostics
+/// ChainedDiagnosticConsumer - Chain two diagnostic clients so that diagnostics
 /// go to the first client and then the second. The first diagnostic client
 /// should be the "primary" client, and will be used for computing whether the
 /// diagnostics should be included in counts.
-class ChainedDiagnosticClient : public DiagnosticClient {
-  llvm::OwningPtr<DiagnosticClient> Primary;
-  llvm::OwningPtr<DiagnosticClient> Secondary;
+class ChainedDiagnosticConsumer : public DiagnosticConsumer {
+  llvm::OwningPtr<DiagnosticConsumer> Primary;
+  llvm::OwningPtr<DiagnosticConsumer> Secondary;
 
 public:
-  ChainedDiagnosticClient(DiagnosticClient *_Primary,
-                          DiagnosticClient *_Secondary) {
+  ChainedDiagnosticConsumer(DiagnosticConsumer *_Primary,
+                          DiagnosticConsumer *_Secondary) {
     Primary.reset(_Primary);
     Secondary.reset(_Secondary);
   }
@@ -46,14 +46,20 @@ class ChainedDiagnosticClient : public DiagnosticClient {
     return Primary->IncludeInDiagnosticCounts();
   }
 
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info) {
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info) {
     // Default implementation (Warnings/errors count).
-    DiagnosticClient::HandleDiagnostic(DiagLevel, Info);
+    DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
 
     Primary->HandleDiagnostic(DiagLevel, Info);
     Secondary->HandleDiagnostic(DiagLevel, Info);
   }
+  
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
+    return new ChainedDiagnosticConsumer(Primary->clone(Diags), 
+                                         Secondary->clone(Diags));
+  }
+
 };
 
 } // end namspace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/CodeGenOptions.h b/contrib/llvm/tools/clang/include/clang/Frontend/CodeGenOptions.h
index 5d040b4f266c..4874c17c7996 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/CodeGenOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/CodeGenOptions.h
@@ -37,6 +37,7 @@ class CodeGenOptions {
 
   unsigned AsmVerbose        : 1; /// -dA, -fverbose-asm.
   unsigned ObjCAutoRefCountExceptions : 1; /// Whether ARC should be EH-safe.
+  unsigned CUDAIsDevice      : 1; /// Set when compiling for CUDA device.
   unsigned CXAAtExit         : 1; /// Use __cxa_atexit for calling destructors.
   unsigned CXXCtorDtorAliases: 1; /// Emit complete ctors/dtors as linker
                                   /// aliases to base ctors when possible.
@@ -71,6 +72,7 @@ class CodeGenOptions {
   unsigned NoCommon          : 1; /// Set when -fno-common or C++ is enabled.
   unsigned NoDwarf2CFIAsm    : 1; /// Set when -fno-dwarf2-cfi-asm is enabled.
   unsigned NoExecStack       : 1; /// Set when -Wa,--noexecstack is enabled.
+  unsigned NoGlobalMerge     : 1; /// Set when -mno-global-merge is enabled.
   unsigned NoImplicitFloat   : 1; /// Set when -mno-implicit-float is enabled.
   unsigned NoInfsFPMath      : 1; /// Assume FP arguments, results not +-Inf.
   unsigned NoNaNsFPMath      : 1; /// Assume FP arguments, results not NaN.
@@ -142,6 +144,7 @@ class CodeGenOptions {
 public:
   CodeGenOptions() {
     AsmVerbose = 0;
+    CUDAIsDevice = 0;
     CXAAtExit = 1;
     CXXCtorDtorAliases = 0;
     DataSections = 0;
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/CommandLineSourceLoc.h b/contrib/llvm/tools/clang/include/clang/Frontend/CommandLineSourceLoc.h
index 8911cfadd530..c01f91d6ec27 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/CommandLineSourceLoc.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/CommandLineSourceLoc.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CLANG_FRONTEND_COMMANDLINESOURCELOC_H
 #define LLVM_CLANG_FRONTEND_COMMANDLINESOURCELOC_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -29,10 +30,10 @@ struct ParsedSourceLocation {
 public:
   /// Construct a parsed source location from a string; the Filename is empty on
   /// error.
-  static ParsedSourceLocation FromString(llvm::StringRef Str) {
+  static ParsedSourceLocation FromString(StringRef Str) {
     ParsedSourceLocation PSL;
-    std::pair<llvm::StringRef, llvm::StringRef> ColSplit = Str.rsplit(':');
-    std::pair<llvm::StringRef, llvm::StringRef> LineSplit =
+    std::pair<StringRef, StringRef> ColSplit = Str.rsplit(':');
+    std::pair<StringRef, StringRef> LineSplit =
       ColSplit.first.rsplit(':');
 
     // If both tail splits were valid integers, return success.
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInstance.h b/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInstance.h
index 004c8896e232..881774022d12 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInstance.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInstance.h
@@ -11,6 +11,7 @@
 #define LLVM_CLANG_FRONTEND_COMPILERINSTANCE_H_
 
 #include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/OwningPtr.h"
@@ -19,7 +20,6 @@
 #include <string>
 
 namespace llvm {
-class raw_ostream;
 class raw_fd_ostream;
 class Timer;
 }
@@ -27,13 +27,13 @@ class Timer;
 namespace clang {
 class ASTContext;
 class ASTConsumer;
+class ASTReader;
 class CodeCompleteConsumer;
-class Diagnostic;
-class DiagnosticClient;
+class DiagnosticsEngine;
+class DiagnosticConsumer;
 class ExternalASTSource;
 class FileManager;
 class FrontendAction;
-class ASTReader;
 class Preprocessor;
 class Sema;
 class SourceManager;
@@ -57,12 +57,12 @@ class TargetInfo;
 /// in to the compiler instance for everything. When possible, utility functions
 /// come in two forms; a short form that reuses the CompilerInstance objects,
 /// and a long form that takes explicit instances of any required objects.
-class CompilerInstance {
+class CompilerInstance : public ModuleLoader {
   /// The options used in this compiler instance.
   llvm::IntrusiveRefCntPtr<CompilerInvocation> Invocation;
 
   /// The diagnostics engine instance.
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diagnostics;
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
 
   /// The target being compiled for.
   llvm::IntrusiveRefCntPtr<TargetInfo> Target;
@@ -88,9 +88,12 @@ class CompilerInstance {
   /// \brief The semantic analysis object.
   llvm::OwningPtr<Sema> TheSema;
   
-  /// The frontend timer
+  /// \brief The frontend timer
   llvm::OwningPtr<llvm::Timer> FrontendTimer;
 
+  /// \brief Non-owning reference to the ASTReader, if one exists.
+  ASTReader *ModuleManager;
+
   /// \brief Holds information about the output file.
   ///
   /// If TempFilename is not empty we must rename it to Filename at the end.
@@ -99,10 +102,10 @@ class CompilerInstance {
   struct OutputFile {
     std::string Filename;
     std::string TempFilename;
-    llvm::raw_ostream *OS;
+    raw_ostream *OS;
 
     OutputFile(const std::string &filename, const std::string &tempFilename,
-               llvm::raw_ostream *os)
+               raw_ostream *os)
       : Filename(filename), TempFilename(tempFilename), OS(os) { }
   };
 
@@ -249,15 +252,15 @@ class CompilerInstance {
   bool hasDiagnostics() const { return Diagnostics != 0; }
 
   /// Get the current diagnostics engine.
-  Diagnostic &getDiagnostics() const {
+  DiagnosticsEngine &getDiagnostics() const {
     assert(Diagnostics && "Compiler instance has no diagnostics!");
     return *Diagnostics;
   }
 
   /// setDiagnostics - Replace the current diagnostics engine.
-  void setDiagnostics(Diagnostic *Value);
+  void setDiagnostics(DiagnosticsEngine *Value);
 
-  DiagnosticClient &getDiagnosticClient() const {
+  DiagnosticConsumer &getDiagnosticClient() const {
     assert(Diagnostics && Diagnostics->getClient() && 
            "Compiler instance has no diagnostic client!");
     return *Diagnostics->getClient();
@@ -387,6 +390,12 @@ class CompilerInstance {
   
   Sema *takeSema() { return TheSema.take(); }
   
+  /// }
+  /// @name Module Management
+  /// {
+
+  ASTReader *getModuleManager() const { return ModuleManager; }
+
   /// }
   /// @name Code Completion
   /// {
@@ -446,38 +455,48 @@ class CompilerInstance {
   /// allocating one if one is not provided.
   ///
   /// \param Client If non-NULL, a diagnostic client that will be
-  /// attached to (and, then, owned by) the Diagnostic inside this AST
+  /// attached to (and, then, owned by) the DiagnosticsEngine inside this AST
   /// unit.
+  ///
+  /// \param ShouldOwnClient If Client is non-NULL, specifies whether 
+  /// the diagnostic object should take ownership of the client.
+  ///
+  /// \param ShouldCloneClient If Client is non-NULL, specifies whether that
+  /// client should be cloned.
   void createDiagnostics(int Argc, const char* const *Argv,
-                         DiagnosticClient *Client = 0);
+                         DiagnosticConsumer *Client = 0,
+                         bool ShouldOwnClient = true,
+                         bool ShouldCloneClient = true);
 
-  /// Create a Diagnostic object with a the TextDiagnosticPrinter.
+  /// Create a DiagnosticsEngine object with a the TextDiagnosticPrinter.
   ///
   /// The \arg Argc and \arg Argv arguments are used only for logging purposes,
   /// when the diagnostic options indicate that the compiler should output
   /// logging information.
   ///
   /// If no diagnostic client is provided, this creates a
-  /// DiagnosticClient that is owned by the returned diagnostic
+  /// DiagnosticConsumer that is owned by the returned diagnostic
   /// object, if using directly the caller is responsible for
-  /// releasing the returned Diagnostic's client eventually.
+  /// releasing the returned DiagnosticsEngine's client eventually.
   ///
   /// \param Opts - The diagnostic options; note that the created text
   /// diagnostic object contains a reference to these options and its lifetime
   /// must extend past that of the diagnostic engine.
   ///
   /// \param Client If non-NULL, a diagnostic client that will be
-  /// attached to (and, then, owned by) the returned Diagnostic
+  /// attached to (and, then, owned by) the returned DiagnosticsEngine
   /// object.
   ///
   /// \param CodeGenOpts If non-NULL, the code gen options in use, which may be
   /// used by some diagnostics printers (for logging purposes only).
   ///
   /// \return The new object on success, or null on failure.
-  static llvm::IntrusiveRefCntPtr<Diagnostic> 
+  static llvm::IntrusiveRefCntPtr<DiagnosticsEngine> 
   createDiagnostics(const DiagnosticOptions &Opts, int Argc,
                     const char* const *Argv,
-                    DiagnosticClient *Client = 0,
+                    DiagnosticConsumer *Client = 0,
+                    bool ShouldOwnClient = true,
+                    bool ShouldCloneClient = true,
                     const CodeGenOptions *CodeGenOpts = 0);
 
   /// Create the file manager and replace any existing one with it.
@@ -490,26 +509,12 @@ class CompilerInstance {
   /// and replace any existing one with it.
   void createPreprocessor();
 
-  /// Create a Preprocessor object.
-  ///
-  /// Note that this also creates a new HeaderSearch object which will be owned
-  /// by the resulting Preprocessor.
-  ///
-  /// \return The new object on success, or null on failure.
-  static Preprocessor *createPreprocessor(Diagnostic &, const LangOptions &,
-                                          const PreprocessorOptions &,
-                                          const HeaderSearchOptions &,
-                                          const DependencyOutputOptions &,
-                                          const TargetInfo &,
-                                          const FrontendOptions &,
-                                          SourceManager &, FileManager &);
-
   /// Create the AST context.
   void createASTContext();
 
   /// Create an external AST source to read a PCH file and attach it to the AST
   /// context.
-  void createPCHExternalASTSource(llvm::StringRef Path,
+  void createPCHExternalASTSource(StringRef Path,
                                   bool DisablePCHValidation,
                                   bool DisableStatCache,
                                   void *DeserializationListener);
@@ -518,7 +523,7 @@ class CompilerInstance {
   ///
   /// \return - The new object on success, or null on failure.
   static ExternalASTSource *
-  createPCHExternalASTSource(llvm::StringRef Path, const std::string &Sysroot,
+  createPCHExternalASTSource(StringRef Path, const std::string &Sysroot,
                              bool DisablePCHValidation,
                              bool DisableStatCache,
                              Preprocessor &PP, ASTContext &Context,
@@ -537,10 +542,10 @@ class CompilerInstance {
                                unsigned Line, unsigned Column,
                                bool ShowMacros,
                                bool ShowCodePatterns, bool ShowGlobals,
-                               llvm::raw_ostream &OS);
+                               raw_ostream &OS);
 
   /// \brief Create the Sema object to be used for parsing.
-  void createSema(bool CompleteTranslationUnit,
+  void createSema(TranslationUnitKind TUKind,
                   CodeCompleteConsumer *CompletionConsumer);
   
   /// Create the frontend timer and replace any existing one with it.
@@ -551,25 +556,27 @@ class CompilerInstance {
   ///
   /// \return - Null on error.
   llvm::raw_fd_ostream *
-  createDefaultOutputFile(bool Binary = true, llvm::StringRef BaseInput = "",
-                          llvm::StringRef Extension = "");
+  createDefaultOutputFile(bool Binary = true, StringRef BaseInput = "",
+                          StringRef Extension = "");
 
   /// Create a new output file and add it to the list of tracked output files,
   /// optionally deriving the output path name.
   ///
   /// \return - Null on error.
   llvm::raw_fd_ostream *
-  createOutputFile(llvm::StringRef OutputPath,
+  createOutputFile(StringRef OutputPath,
                    bool Binary = true, bool RemoveFileOnSignal = true,
-                   llvm::StringRef BaseInput = "",
-                   llvm::StringRef Extension = "");
+                   StringRef BaseInput = "",
+                   StringRef Extension = "",
+                   bool UseTemporary = false);
 
   /// Create a new output file, optionally deriving the output path name.
   ///
   /// If \arg OutputPath is empty, then createOutputFile will derive an output
   /// path location as \arg BaseInput, with any suffix removed, and \arg
-  /// Extension appended. If OutputPath is not stdout createOutputFile will
-  /// create a new temporary file that must be renamed to OutputPath in the end.
+  /// Extension appended. If OutputPath is not stdout and \arg UseTemporary
+  /// is true, createOutputFile will create a new temporary file that must be
+  /// renamed to OutputPath in the end.
   ///
   /// \param OutputPath - If given, the path to the output file.
   /// \param Error [out] - On failure, the error message.
@@ -580,15 +587,18 @@ class CompilerInstance {
   /// \param RemoveFileOnSignal - Whether the file should be registered with
   /// llvm::sys::RemoveFileOnSignal. Note that this is not safe for
   /// multithreaded use, as the underlying signal mechanism is not reentrant
+  /// \param UseTemporary - Create a new temporary file that must be renamed to
+  ///         OutputPath in the end
   /// \param ResultPathName [out] - If given, the result path name will be
   /// stored here on success.
   /// \param TempPathName [out] - If given, the temporary file path name
   /// will be stored here on success.
   static llvm::raw_fd_ostream *
-  createOutputFile(llvm::StringRef OutputPath, std::string &Error,
+  createOutputFile(StringRef OutputPath, std::string &Error,
                    bool Binary = true, bool RemoveFileOnSignal = true,
-                   llvm::StringRef BaseInput = "",
-                   llvm::StringRef Extension = "",
+                   StringRef BaseInput = "",
+                   StringRef Extension = "",
+                   bool UseTemporary = false,
                    std::string *ResultPathName = 0,
                    std::string *TempPathName = 0);
 
@@ -600,19 +610,23 @@ class CompilerInstance {
   /// as the main file.
   ///
   /// \return True on success.
-  bool InitializeSourceManager(llvm::StringRef InputFile);
+  bool InitializeSourceManager(StringRef InputFile);
 
   /// InitializeSourceManager - Initialize the source manager to set InputFile
   /// as the main file.
   ///
   /// \return True on success.
-  static bool InitializeSourceManager(llvm::StringRef InputFile,
-                                      Diagnostic &Diags,
+  static bool InitializeSourceManager(StringRef InputFile,
+                                      DiagnosticsEngine &Diags,
                                       FileManager &FileMgr,
                                       SourceManager &SourceMgr,
                                       const FrontendOptions &Opts);
 
   /// }
+  
+  virtual ModuleKey loadModule(SourceLocation ImportLoc, 
+                               IdentifierInfo &ModuleName,
+                               SourceLocation ModuleNameLoc);
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInvocation.h b/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInvocation.h
index e18f3fe63249..47c70311dda4 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInvocation.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/CompilerInvocation.h
@@ -28,13 +28,9 @@
 #include <string>
 #include <vector>
 
-namespace llvm {
-  template<typename T> class SmallVectorImpl;
-}
-
 namespace clang {
 
-class Diagnostic;
+class DiagnosticsEngine;
 
 /// CompilerInvocation - Helper class for holding the data necessary to invoke
 /// the compiler.
@@ -92,7 +88,7 @@ class CompilerInvocation : public llvm::RefCountedBase<CompilerInvocation> {
   static void CreateFromArgs(CompilerInvocation &Res,
                              const char* const *ArgBegin,
                              const char* const *ArgEnd,
-                             Diagnostic &Diags);
+                             DiagnosticsEngine &Diags);
 
   /// GetBuiltinIncludePath - Get the directory where the compiler headers
   /// reside, relative to the compiler binary (found by the passed in
@@ -127,6 +123,10 @@ class CompilerInvocation : public llvm::RefCountedBase<CompilerInvocation> {
   static void setLangDefaults(LangOptions &Opts, InputKind IK,
                    LangStandard::Kind LangStd = LangStandard::lang_unspecified);
   
+  /// \brief Retrieve a module hash string that is suitable for uniquely 
+  /// identifying the conditions under which the module was built.
+  std::string getModuleHash() const;
+  
   /// @}
   /// @name Option Subgroups
   /// @{
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/DiagnosticOptions.h b/contrib/llvm/tools/clang/include/clang/Frontend/DiagnosticOptions.h
index 5ae8eb367788..319abeb4fbf5 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/DiagnosticOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/DiagnosticOptions.h
@@ -43,7 +43,7 @@ class DiagnosticOptions {
   
   unsigned ShowColors : 1;       /// Show diagnostics with ANSI color sequences.
   unsigned ShowOverloads : 1;    /// Overload candidates to show.  Values from
-                                 /// Diagnostic::OverloadsShown
+                                 /// DiagnosticsEngine::OverloadsShown
   unsigned VerifyDiagnostics: 1; /// Check that diagnostics match the expected
                                  /// diagnostics, indicated by markers in the
                                  /// input source file.
@@ -82,7 +82,7 @@ class DiagnosticOptions {
     PedanticErrors = 0;
     ShowCarets = 1;
     ShowColors = 0;
-    ShowOverloads = Diagnostic::Ovl_All;
+    ShowOverloads = DiagnosticsEngine::Ovl_All;
     ShowColumn = 1;
     ShowFixits = 1;
     ShowLocation = 1;
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendAction.h b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendAction.h
index f335475665fd..f85cc7ec913c 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendAction.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendAction.h
@@ -10,15 +10,13 @@
 #ifndef LLVM_CLANG_FRONTEND_FRONTENDACTION_H
 #define LLVM_CLANG_FRONTEND_FRONTENDACTION_H
 
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/OwningPtr.h"
 #include <string>
 #include <vector>
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
 class ASTConsumer;
 class ASTMergeAction;
@@ -55,7 +53,7 @@ class FrontendAction {
 
 private:
   ASTConsumer* CreateWrappedASTConsumer(CompilerInstance &CI,
-                                        llvm::StringRef InFile);
+                                        StringRef InFile);
 
 protected:
   /// @name Implementation Action Interface
@@ -76,7 +74,7 @@ class FrontendAction {
   ///
   /// \return The new AST consumer, or 0 on failure.
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile) = 0;
+                                         StringRef InFile) = 0;
 
   /// \brief Callback before starting processing a single input, giving the
   /// opportunity to modify the CompilerInvocation or do some other action
@@ -92,7 +90,7 @@ class FrontendAction {
   /// \return True on success; on failure \see ExecutionAction() and
   /// EndSourceFileAction() will not be called.
   virtual bool BeginSourceFileAction(CompilerInstance &CI,
-                                     llvm::StringRef Filename) {
+                                     StringRef Filename) {
     return true;
   }
 
@@ -152,7 +150,7 @@ class FrontendAction {
     return CurrentASTUnit.take();
   }
 
-  void setCurrentFile(llvm::StringRef Value, InputKind Kind, ASTUnit *AST = 0);
+  void setCurrentFile(StringRef Value, InputKind Kind, ASTUnit *AST = 0);
 
   /// @}
   /// @name Supported Modes
@@ -163,9 +161,8 @@ class FrontendAction {
   /// file inputs.
   virtual bool usesPreprocessorOnly() const = 0;
 
-  /// usesCompleteTranslationUnit - For AST based actions, should the
-  /// translation unit be completed?
-  virtual bool usesCompleteTranslationUnit() { return true; }
+  /// \brief For AST-based actions, the kind of translation unit we're handling.
+  virtual TranslationUnitKind getTranslationUnitKind() { return TU_Complete; }
 
   /// hasPCHSupport - Does this action support use with PCH?
   virtual bool hasPCHSupport() const { return !usesPreprocessorOnly(); }
@@ -205,7 +202,7 @@ class FrontendAction {
   ///
   /// \return True on success; the compilation of this file should be aborted
   /// and neither Execute nor EndSourceFile should be called.
-  bool BeginSourceFile(CompilerInstance &CI, llvm::StringRef Filename,
+  bool BeginSourceFile(CompilerInstance &CI, StringRef Filename,
                        InputKind Kind);
 
   /// Execute - Set the source managers main input file, and run the action.
@@ -236,7 +233,7 @@ class ASTFrontendAction : public FrontendAction {
 class PluginASTAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile) = 0;
+                                         StringRef InFile) = 0;
 
 public:
   /// ParseArgs - Parse the given plugin command line arguments.
@@ -256,7 +253,7 @@ class PreprocessorFrontendAction : public FrontendAction {
   /// CreateASTConsumer - Provide a default implementation which returns aborts,
   /// this method should never be called by FrontendAction clients.
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
 public:
   virtual bool usesPreprocessorOnly() const { return true; }
@@ -272,10 +269,10 @@ class WrapperFrontendAction : public FrontendAction {
 
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
   virtual bool BeginInvocation(CompilerInstance &CI);
   virtual bool BeginSourceFileAction(CompilerInstance &CI,
-                                     llvm::StringRef Filename);
+                                     StringRef Filename);
   virtual void ExecuteAction();
   virtual void EndSourceFileAction();
 
@@ -285,7 +282,7 @@ class WrapperFrontendAction : public FrontendAction {
   WrapperFrontendAction(FrontendAction *WrappedAction);
 
   virtual bool usesPreprocessorOnly() const;
-  virtual bool usesCompleteTranslationUnit();
+  virtual TranslationUnitKind getTranslationUnitKind();
   virtual bool hasPCHSupport() const;
   virtual bool hasASTFileSupport() const;
   virtual bool hasIRSupport() const;
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h
index b409ad1e0963..72a3d908476a 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h
@@ -24,7 +24,7 @@ class InitOnlyAction : public FrontendAction {
   virtual void ExecuteAction();
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
 public:
   // Don't claim to only use the preprocessor, we want to follow the AST path,
@@ -39,59 +39,65 @@ class InitOnlyAction : public FrontendAction {
 class ASTPrintAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class ASTDumpAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class ASTDumpXMLAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class ASTViewAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class DeclContextPrintAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class GeneratePCHAction : public ASTFrontendAction {
+  bool MakeModule;
+  
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
-  virtual bool usesCompleteTranslationUnit() { return false; }
+  virtual TranslationUnitKind getTranslationUnitKind() { 
+    return MakeModule? TU_Module : TU_Prefix;
+  }
 
   virtual bool hasASTFileSupport() const { return false; }
 
 public:
+  /// \brief Create a new action
+  explicit GeneratePCHAction(bool MakeModule) : MakeModule(MakeModule) { }
+  
   /// \brief Compute the AST consumer arguments that will be used to
   /// create the PCHGenerator instance returned by CreateASTConsumer.
   ///
   /// \returns true if an error occurred, false otherwise.
   static bool ComputeASTConsumerArguments(CompilerInstance &CI,
-                                          llvm::StringRef InFile,
+                                          StringRef InFile,
                                           std::string &Sysroot,
                                           std::string &OutputFile,
-                                          llvm::raw_ostream *&OS,
-                                          bool &Chaining);
+                                          raw_ostream *&OS);
 };
 
 class SyntaxOnlyAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
 public:
   virtual bool hasCodeCompletionSupport() const { return true; }
@@ -114,10 +120,10 @@ class ASTMergeAction : public FrontendAction {
 
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
   virtual bool BeginSourceFileAction(CompilerInstance &CI,
-                                     llvm::StringRef Filename);
+                                     StringRef Filename);
 
   virtual void ExecuteAction();
   virtual void EndSourceFileAction();
@@ -128,7 +134,7 @@ class ASTMergeAction : public FrontendAction {
   virtual ~ASTMergeAction();
 
   virtual bool usesPreprocessorOnly() const;
-  virtual bool usesCompleteTranslationUnit();
+  virtual TranslationUnitKind getTranslationUnitKind();
   virtual bool hasPCHSupport() const;
   virtual bool hasASTFileSupport() const;
   virtual bool hasCodeCompletionSupport() const;
@@ -137,7 +143,7 @@ class ASTMergeAction : public FrontendAction {
 class PrintPreambleAction : public FrontendAction {
 protected:
   void ExecuteAction();
-  virtual ASTConsumer *CreateASTConsumer(CompilerInstance &, llvm::StringRef) { 
+  virtual ASTConsumer *CreateASTConsumer(CompilerInstance &, StringRef) { 
     return 0; 
   }
   
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendDiagnostic.h
index 3e9508c009c8..21cd2c67910b 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define FRONTENDSTART
 #include "clang/Basic/DiagnosticFrontendKinds.inc"
 #undef DIAG
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendOptions.h b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendOptions.h
index 225a955b4ac6..fa6d044ce04b 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendOptions.h
@@ -24,7 +24,6 @@ namespace frontend {
     ASTDumpXML,             ///< Parse ASTs and dump them in XML.
     ASTPrint,               ///< Parse ASTs and print them.
     ASTView,                ///< Parse ASTs and view them in Graphviz.
-    CreateModule,           ///< Create module definition
     DumpRawTokens,          ///< Dump out raw tokens.
     DumpTokens,             ///< Dump out preprocessed tokens.
     EmitAssembly,           ///< Emit a .s file.
@@ -35,6 +34,7 @@ namespace frontend {
     EmitCodeGenOnly,        ///< Generate machine code, but don't emit anything.
     EmitObj,                ///< Emit a .o file.
     FixIt,                  ///< Parse and apply any fixits to the source.
+    GenerateModule,         ///< Generate pre-compiled module.
     GeneratePCH,            ///< Generate pre-compiled header.
     GeneratePTH,            ///< Generate pre-tokenized header.
     InitOnly,               ///< Only execute frontend initialization.
@@ -58,9 +58,6 @@ class FrontendOptions {
   unsigned RelocatablePCH : 1;             ///< When generating PCH files,
                                            /// instruct the AST writer to create
                                            /// relocatable PCH files.
-  unsigned ChainedPCH : 1;                 ///< When generating PCH files,
-                                           /// instruct the AST writer to create
-                                           /// chained PCH files.
   unsigned ShowHelp : 1;                   ///< Show the -help text.
   unsigned ShowMacrosInCodeCompletion : 1; ///< Show macros in code completion
                                            /// results.
@@ -75,6 +72,8 @@ class FrontendOptions {
   unsigned ShowVersion : 1;                ///< Show the -version text.
   unsigned FixWhatYouCan : 1;              ///< Apply fixes even if there are
                                            /// unfixable errors.
+  unsigned ARCMTMigrateEmitARCErrors : 1;  /// Emit ARC errors even if the
+                                           /// migrator can fix them
 
   enum {
     ARCMT_None,
@@ -84,6 +83,7 @@ class FrontendOptions {
   } ARCMTAction;
 
   std::string ARCMTMigrateDir;
+  std::string ARCMTMigrateReportOut;
 
   /// The input files and their types.
   std::vector<std::pair<InputKind, std::string> > Inputs;
@@ -118,9 +118,6 @@ class FrontendOptions {
   /// \brief The list of AST files to merge.
   std::vector<std::string> ASTMergeFiles;
 
-  /// \brief The list of modules to import.
-  std::vector<std::string> Modules;
-
   /// \brief A list of arguments to forward to LLVM's option processing; this
   /// should only be used for debugging and experimental features.
   std::vector<std::string> LLVMArgs;
@@ -131,7 +128,6 @@ class FrontendOptions {
     ProgramAction = frontend::ParseSyntaxOnly;
     ActionName = "";
     RelocatablePCH = 0;
-    ChainedPCH = 0;
     ShowHelp = 0;
     ShowMacrosInCodeCompletion = 0;
     ShowCodePatternsInCodeCompletion = 0;
@@ -140,6 +136,7 @@ class FrontendOptions {
     ShowTimers = 0;
     ShowVersion = 0;
     ARCMTAction = ARCMT_None;
+    ARCMTMigrateEmitARCErrors = 0;
   }
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
@@ -147,7 +144,7 @@ class FrontendOptions {
   ///
   /// \return The input kind for the extension, or IK_None if the extension is
   /// not recognized.
-  static InputKind getInputKindForExtension(llvm::StringRef Extension);
+  static InputKind getInputKindForExtension(StringRef Extension);
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/HeaderSearchOptions.h b/contrib/llvm/tools/clang/include/clang/Frontend/HeaderSearchOptions.h
index 0347f98fd5ba..92790e99d890 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/HeaderSearchOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/HeaderSearchOptions.h
@@ -23,8 +23,13 @@ namespace frontend {
   enum IncludeDirGroup {
     Quoted = 0,     ///< '#include ""' paths, added by'gcc -iquote'.
     Angled,         ///< Paths for '#include <>' added by '-I'.
+    IndexHeaderMap, ///< Like Angled, but marks header maps used when
+                       ///  building frameworks.
     System,         ///< Like Angled, but marks system directories.
+    CSystem,        ///< Like System, but only used for C.
     CXXSystem,      ///< Like System, but only used for C++.
+    ObjCSystem,     ///< Like System, but only used for ObjC.
+    ObjCXXSystem,   ///< Like System, but only used for ObjC++.
     After           ///< Like System, but searched after the system directories.
   };
 }
@@ -44,7 +49,7 @@ class HeaderSearchOptions {
     /// path.
     unsigned IgnoreSysRoot : 1;
 
-    Entry(llvm::StringRef path, frontend::IncludeDirGroup group,
+    Entry(StringRef path, frontend::IncludeDirGroup group,
           bool isUserSupplied, bool isFramework, bool ignoreSysRoot)
       : Path(path), Group(group), IsUserSupplied(isUserSupplied),
         IsFramework(isFramework), IgnoreSysRoot(ignoreSysRoot) {}
@@ -57,27 +62,24 @@ class HeaderSearchOptions {
   /// User specified include entries.
   std::vector<Entry> UserEntries;
 
-  /// A (system-path) delimited list of include paths to be added from the
-  /// environment following the user specified includes (but prior to builtin
-  /// and standard includes). This is parsed in the same manner as the CPATH
-  /// environment variable for gcc.
-  std::string EnvIncPath;
-
-  /// Per-language environmental include paths, see \see EnvIncPath.
-  std::string CEnvIncPath;
-  std::string ObjCEnvIncPath;
-  std::string CXXEnvIncPath;
-  std::string ObjCXXEnvIncPath;
-
   /// The directory which holds the compiler resource files (builtin includes,
   /// etc.).
   std::string ResourceDir;
 
+  /// \brief The directory used for the module cache.
+  std::string ModuleCachePath;
+  
+  /// \brief Whether we should disable the use of the hash string within the
+  /// module cache.
+  ///
+  /// Note: Only used for testing!
+  unsigned DisableModuleHash : 1;
+  
   /// Include the compiler builtin includes.
   unsigned UseBuiltinIncludes : 1;
 
   /// Include the system standard include search directories.
-  unsigned UseStandardIncludes : 1;
+  unsigned UseStandardSystemIncludes : 1;
 
   /// Include the system standard C++ library include search directories.
   unsigned UseStandardCXXIncludes : 1;
@@ -89,13 +91,13 @@ class HeaderSearchOptions {
   unsigned Verbose : 1;
 
 public:
-  HeaderSearchOptions(llvm::StringRef _Sysroot = "/")
-    : Sysroot(_Sysroot), UseBuiltinIncludes(true),
-      UseStandardIncludes(true), UseStandardCXXIncludes(true), UseLibcxx(false),
-      Verbose(false) {}
+  HeaderSearchOptions(StringRef _Sysroot = "/")
+    : Sysroot(_Sysroot), DisableModuleHash(0), UseBuiltinIncludes(true),
+      UseStandardSystemIncludes(true), UseStandardCXXIncludes(true),
+      UseLibcxx(false), Verbose(false) {}
 
   /// AddPath - Add the \arg Path path to the specified \arg Group list.
-  void AddPath(llvm::StringRef Path, frontend::IncludeDirGroup Group,
+  void AddPath(StringRef Path, frontend::IncludeDirGroup Group,
                bool IsUserSupplied, bool IsFramework, bool IgnoreSysRoot) {
     UserEntries.push_back(Entry(Path, Group, IsUserSupplied, IsFramework,
                                 IgnoreSysRoot));
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/LangStandard.h b/contrib/llvm/tools/clang/include/clang/Frontend/LangStandard.h
index ea37bdd1022b..de2800cd73c9 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/LangStandard.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/LangStandard.h
@@ -10,6 +10,7 @@
 #ifndef LLVM_CLANG_FRONTEND_LANGSTANDARD_H
 #define LLVM_CLANG_FRONTEND_LANGSTANDARD_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
@@ -83,7 +84,7 @@ struct LangStandard {
   bool hasImplicitInt() const { return Flags & frontend::ImplicitInt; }
 
   static const LangStandard &getLangStandardForKind(Kind K);
-  static const LangStandard *getLangStandardForName(llvm::StringRef Name);
+  static const LangStandard *getLangStandardForName(StringRef Name);
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/LangStandards.def b/contrib/llvm/tools/clang/include/clang/Frontend/LangStandards.def
index 6055ad51828e..c82290b20d67 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/LangStandards.def
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/LangStandards.def
@@ -37,6 +37,9 @@ LANGSTANDARD(c94, "iso9899:199409",
 LANGSTANDARD(gnu89, "gnu89",
              "ISO C 1990 with GNU extensions",
              BCPLComment | C89 | Digraphs | GNUMode | ImplicitInt)
+LANGSTANDARD(gnu90, "gnu90",
+             "ISO C 1990 with GNU extensions",
+             BCPLComment | C89 | Digraphs | GNUMode | ImplicitInt)
 
 // C99-ish modes
 LANGSTANDARD(c99, "c99",
@@ -75,15 +78,24 @@ LANGSTANDARD(gnu1x, "gnu1x",
 LANGSTANDARD(cxx98, "c++98",
              "ISO C++ 1998 with amendments",
              BCPLComment | CPlusPlus | Digraphs)
+LANGSTANDARD(cxx03, "c++03",
+             "ISO C++ 1998 with amendments",
+             BCPLComment | CPlusPlus | Digraphs)
 LANGSTANDARD(gnucxx98, "gnu++98",
-             "ISO C++ 1998 with " "amendments and GNU extensions",
+             "ISO C++ 1998 with amendments and GNU extensions",
              BCPLComment | CPlusPlus | Digraphs | GNUMode)
 
 LANGSTANDARD(cxx0x, "c++0x",
-             "Upcoming ISO C++ 200x with amendments",
+             "ISO C++ 2011 with amendments",
+             BCPLComment | CPlusPlus | CPlusPlus0x | Digraphs)
+LANGSTANDARD(cxx11, "c++11",
+             "ISO C++ 2011 with amendments",
              BCPLComment | CPlusPlus | CPlusPlus0x | Digraphs)
 LANGSTANDARD(gnucxx0x, "gnu++0x",
-             "Upcoming ISO C++ 200x with amendments and GNU extensions",
+             "ISO C++ 2011 with amendments and GNU extensions",
+             BCPLComment | CPlusPlus | CPlusPlus0x | Digraphs | GNUMode)
+LANGSTANDARD(gnucxx11, "gnu++11",
+             "ISO C++ 2011 with amendments and GNU extensions",
              BCPLComment | CPlusPlus | CPlusPlus0x | Digraphs | GNUMode)
 
 // OpenCL
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/LogDiagnosticPrinter.h b/contrib/llvm/tools/clang/include/clang/Frontend/LogDiagnosticPrinter.h
index b6fc23ca1f04..4de15f2aed5f 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/LogDiagnosticPrinter.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/LogDiagnosticPrinter.h
@@ -19,7 +19,7 @@ namespace clang {
 class DiagnosticOptions;
 class LangOptions;
 
-class LogDiagnosticPrinter : public DiagnosticClient {
+class LogDiagnosticPrinter : public DiagnosticConsumer {
   struct DiagEntry {
     /// The primary message line of the diagnostic.
     std::string Message;
@@ -37,10 +37,10 @@ class LogDiagnosticPrinter : public DiagnosticClient {
     unsigned DiagnosticID;
   
     /// The level of the diagnostic.
-    Diagnostic::Level DiagnosticLevel;
+    DiagnosticsEngine::Level DiagnosticLevel;
   };
   
-  llvm::raw_ostream &OS;
+  raw_ostream &OS;
   const LangOptions *LangOpts;
   const DiagnosticOptions *DiagOpts;
 
@@ -48,17 +48,17 @@ class LogDiagnosticPrinter : public DiagnosticClient {
   FullSourceLoc LastLoc;
   unsigned OwnsOutputStream : 1;
 
-  llvm::SmallVector<DiagEntry, 8> Entries;
+  SmallVector<DiagEntry, 8> Entries;
 
   std::string MainFilename;
   std::string DwarfDebugFlags;
 
 public:
-  LogDiagnosticPrinter(llvm::raw_ostream &OS, const DiagnosticOptions &Diags,
+  LogDiagnosticPrinter(raw_ostream &OS, const DiagnosticOptions &Diags,
                        bool OwnsOutputStream = false);
   virtual ~LogDiagnosticPrinter();
 
-  void setDwarfDebugFlags(llvm::StringRef Value) {
+  void setDwarfDebugFlags(StringRef Value) {
     DwarfDebugFlags = Value;
   }
 
@@ -68,8 +68,10 @@ class LogDiagnosticPrinter : public DiagnosticClient {
 
   void EndSourceFile();
 
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info);
+  
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const;
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/PreprocessorOptions.h b/contrib/llvm/tools/clang/include/clang/Frontend/PreprocessorOptions.h
index 2e16c97e7d43..0ee8cb38744a 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/PreprocessorOptions.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/PreprocessorOptions.h
@@ -50,6 +50,10 @@ class PreprocessorOptions {
                                /// record of all macro definitions and
                                /// expansions.
   
+  /// \brief Whether we should automatically translate #include or #import
+  /// operations into module imports when possible.
+  unsigned AutoModuleImport : 1;
+
   /// \brief Whether the detailed preprocessing record includes nested macro 
   /// expansions.
   unsigned DetailedRecordIncludesNestedMacroExpansions : 1;
@@ -117,6 +121,14 @@ class PreprocessorOptions {
   /// by providing appropriate definitions to retrofit the standard library
   /// with support for lifetime-qualified pointers.
   ObjCXXARCStandardLibraryKind ObjCXXARCStandardLibrary;
+    
+  /// \brief The path of modules being build, which is used to detect
+  /// cycles in the module dependency graph as modules are being built.
+  ///
+  /// There is no way to set this value from the command line. If we ever need
+  /// to do so (e.g., if on-demand module construction moves out-of-process),
+  /// we can add a cc1-level option to do so.
+  SmallVector<std::string, 2> ModuleBuildPath;
   
   typedef std::vector<std::pair<std::string, std::string> >::iterator
     remapped_file_iterator;
@@ -154,6 +166,7 @@ class PreprocessorOptions {
   
 public:
   PreprocessorOptions() : UsePredefines(true), DetailedRecord(false),
+                          AutoModuleImport(false),
                           DetailedRecordIncludesNestedMacroExpansions(true),
                           DisablePCHValidation(false), DisableStatCache(false),
                           DumpDeserializedPCHDecls(false),
@@ -162,13 +175,13 @@ class PreprocessorOptions {
                           RetainRemappedFileBuffers(false),
                           ObjCXXARCStandardLibrary(ARCXX_nolib) { }
 
-  void addMacroDef(llvm::StringRef Name) {
+  void addMacroDef(StringRef Name) {
     Macros.push_back(std::make_pair(Name, false));
   }
-  void addMacroUndef(llvm::StringRef Name) {
+  void addMacroUndef(StringRef Name) {
     Macros.push_back(std::make_pair(Name, true));
   }
-  void addRemappedFile(llvm::StringRef From, llvm::StringRef To) {
+  void addRemappedFile(StringRef From, StringRef To) {
     RemappedFiles.push_back(std::make_pair(From, To));
   }
   
@@ -176,7 +189,7 @@ class PreprocessorOptions {
     return RemappedFiles.erase(Remapped);
   }
   
-  void addRemappedFile(llvm::StringRef From, const llvm::MemoryBuffer * To) {
+  void addRemappedFile(StringRef From, const llvm::MemoryBuffer * To) {
     RemappedFileBuffers.push_back(std::make_pair(From, To));
   }
   
@@ -189,6 +202,21 @@ class PreprocessorOptions {
     RemappedFiles.clear();
     RemappedFileBuffers.clear();
   }
+  
+  /// \brief Reset any options that are not considered when building a
+  /// module.
+  void resetNonModularOptions() {
+    Includes.clear();
+    MacroIncludes.clear();
+    ChainedIncludes.clear();
+    DumpDeserializedPCHDecls = false;
+    ImplicitPCHInclude.clear();
+    ImplicitPTHInclude.clear();
+    TokenCache.clear();
+    RetainRemappedFileBuffers = true;
+    PrecompiledPreambleBytes.first = 0;
+    PrecompiledPreambleBytes.second = 0;
+  }
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticBuffer.h b/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticBuffer.h
index 380a1dd224af..6f1c0e8aeaf7 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticBuffer.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticBuffer.h
@@ -22,7 +22,7 @@ namespace clang {
 class Preprocessor;
 class SourceManager;
 
-class TextDiagnosticBuffer : public DiagnosticClient {
+class TextDiagnosticBuffer : public DiagnosticConsumer {
 public:
   typedef std::vector<std::pair<SourceLocation, std::string> > DiagList;
   typedef DiagList::iterator iterator;
@@ -39,12 +39,14 @@ class TextDiagnosticBuffer : public DiagnosticClient {
   const_iterator note_begin() const { return Notes.begin(); }
   const_iterator note_end() const   { return Notes.end(); }
 
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info);
 
   /// FlushDiagnostics - Flush the buffered diagnostics to an given
   /// diagnostic engine.
-  void FlushDiagnostics(Diagnostic &Diags) const;
+  void FlushDiagnostics(DiagnosticsEngine &Diags) const;
+  
+  virtual DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const;
 };
 
 } // end namspace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticPrinter.h b/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticPrinter.h
index 79a9916cb435..22fa18b05027 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticPrinter.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/TextDiagnosticPrinter.h
@@ -22,8 +22,8 @@ namespace clang {
 class DiagnosticOptions;
 class LangOptions;
 
-class TextDiagnosticPrinter : public DiagnosticClient {
-  llvm::raw_ostream &OS;
+class TextDiagnosticPrinter : public DiagnosticConsumer {
+  raw_ostream &OS;
   const LangOptions *LangOpts;
   const DiagnosticOptions *DiagOpts;
 
@@ -36,7 +36,7 @@ class TextDiagnosticPrinter : public DiagnosticClient {
   std::string Prefix;
 
 public:
-  TextDiagnosticPrinter(llvm::raw_ostream &os, const DiagnosticOptions &diags,
+  TextDiagnosticPrinter(raw_ostream &os, const DiagnosticOptions &diags,
                         bool OwnsOutputStream = false);
   virtual ~TextDiagnosticPrinter();
 
@@ -53,26 +53,19 @@ class TextDiagnosticPrinter : public DiagnosticClient {
     LangOpts = 0;
   }
 
-  void PrintIncludeStack(Diagnostic::Level Level, SourceLocation Loc,
+  void PrintIncludeStack(DiagnosticsEngine::Level Level, SourceLocation Loc,
                          const SourceManager &SM);
 
-  void HighlightRange(const CharSourceRange &R,
-                      const SourceManager &SrcMgr,
-                      unsigned LineNo, FileID FID,
-                      std::string &CaretLine,
-                      const std::string &SourceLine);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level Level,
+                                const Diagnostic &Info);
 
-  virtual void HandleDiagnostic(Diagnostic::Level Level,
-                                const DiagnosticInfo &Info);
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const;
 
 private:
-  void EmitCaretDiagnostic(SourceLocation Loc, CharSourceRange *Ranges,
-                           unsigned NumRanges, const SourceManager &SM,
-                           const FixItHint *Hints,
-                           unsigned NumHints, unsigned Columns,  
-                           unsigned OnMacroInst, unsigned MacroSkipStart,
-                           unsigned MacroSkipEnd);
-  
+  void EmitDiagnosticLoc(DiagnosticsEngine::Level Level,
+                         const Diagnostic &Info,
+                         const SourceManager &SM,
+                         PresumedLoc PLoc);
 };
 
 } // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/Utils.h b/contrib/llvm/tools/clang/include/clang/Frontend/Utils.h
index 93d2c7d501e7..929beb079efe 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/Utils.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/Utils.h
@@ -31,7 +31,7 @@ class CompilerInstance;
 class CompilerInvocation;
 class Decl;
 class DependencyOutputOptions;
-class Diagnostic;
+class DiagnosticsEngine;
 class DiagnosticOptions;
 class FileManager;
 class HeaderSearch;
@@ -48,7 +48,7 @@ class FrontendOptions;
 
 /// Normalize \arg File for use in a user defined #include directive (in the
 /// predefines buffer).
-std::string NormalizeDashIncludePath(llvm::StringRef File,
+std::string NormalizeDashIncludePath(StringRef File,
                                      FileManager &FileMgr);
 
 /// Apply the header search options to get given HeaderSearch object.
@@ -66,10 +66,10 @@ void InitializePreprocessor(Preprocessor &PP,
 
 /// ProcessWarningOptions - Initialize the diagnostic client and process the
 /// warning options specified on the command line.
-void ProcessWarningOptions(Diagnostic &Diags, const DiagnosticOptions &Opts);
+void ProcessWarningOptions(DiagnosticsEngine &Diags, const DiagnosticOptions &Opts);
 
 /// DoPrintPreprocessedInput - Implement -E mode.
-void DoPrintPreprocessedInput(Preprocessor &PP, llvm::raw_ostream* OS,
+void DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream* OS,
                               const PreprocessorOutputOptions &Opts);
 
 /// AttachDependencyFileGen - Create a dependency file generator, and attach
@@ -87,7 +87,7 @@ void AttachDependencyFileGen(Preprocessor &PP,
 /// \param OutputPath - If non-empty, a path to write the header include
 /// information to, instead of writing to stderr.
 void AttachHeaderIncludeGen(Preprocessor &PP, bool ShowAllHeaders = false,
-                            llvm::StringRef OutputPath = "",
+                            StringRef OutputPath = "",
                             bool ShowDepth = true);
 
 /// CacheTokens - Cache tokens for use with PCH. Note that this requires
@@ -100,9 +100,9 @@ void CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS);
 /// \return A CompilerInvocation, or 0 if none was built for the given
 /// argument vector.
 CompilerInvocation *
-createInvocationFromCommandLine(llvm::ArrayRef<const char *> Args,
-                                llvm::IntrusiveRefCntPtr<Diagnostic> Diags =
-                                    llvm::IntrusiveRefCntPtr<Diagnostic>());
+createInvocationFromCommandLine(ArrayRef<const char *> Args,
+                            llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+                                llvm::IntrusiveRefCntPtr<DiagnosticsEngine>());
 
 }  // end namespace clang
 
diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/VerifyDiagnosticsClient.h b/contrib/llvm/tools/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h
similarity index 74%
rename from contrib/llvm/tools/clang/include/clang/Frontend/VerifyDiagnosticsClient.h
rename to contrib/llvm/tools/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h
index 793cedd8578f..28dc9de03af0 100644
--- a/contrib/llvm/tools/clang/include/clang/Frontend/VerifyDiagnosticsClient.h
+++ b/contrib/llvm/tools/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h
@@ -1,4 +1,4 @@
-//===-- VerifyDiagnosticsClient.h - Verifying Diagnostic Client -*- C++ -*-===//
+//===- VerifyDiagnosticConsumer.h - Verifying Diagnostic Client -*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -15,12 +15,12 @@
 
 namespace clang {
 
-class Diagnostic;
+class DiagnosticsEngine;
 class TextDiagnosticBuffer;
 
-/// VerifyDiagnosticsClient - Create a diagnostic client which will use markers
-/// in the input source to check that all the emitted diagnostics match those
-/// expected.
+/// VerifyDiagnosticConsumer - Create a diagnostic client which will use
+/// markers in the input source to check that all the emitted diagnostics match
+/// those expected.
 ///
 /// USING THE DIAGNOSTIC CHECKER:
 ///
@@ -62,31 +62,34 @@ class TextDiagnosticBuffer;
 ///   // expected-error-re {{variable has has type 'struct (.*)'}}
 ///   // expected-error-re {{variable has has type 'struct[[:space:]](.*)'}}
 ///
-class VerifyDiagnosticsClient : public DiagnosticClient {
+class VerifyDiagnosticConsumer: public DiagnosticConsumer {
 public:
-  Diagnostic &Diags;
-  llvm::OwningPtr<DiagnosticClient> PrimaryClient;
+  DiagnosticsEngine &Diags;
+  DiagnosticConsumer *PrimaryClient;
+  bool OwnsPrimaryClient;
   llvm::OwningPtr<TextDiagnosticBuffer> Buffer;
   Preprocessor *CurrentPreprocessor;
 
 private:
+  FileID FirstErrorFID; // FileID of first diagnostic
   void CheckDiagnostics();
 
 public:
   /// Create a new verifying diagnostic client, which will issue errors to \arg
-  /// PrimaryClient when a diagnostic does not match what is expected (as
-  /// indicated in the source file). The verifying diagnostic client takes
-  /// ownership of \arg PrimaryClient.
-  VerifyDiagnosticsClient(Diagnostic &Diags, DiagnosticClient *PrimaryClient);
-  ~VerifyDiagnosticsClient();
+  /// the currently-attached diagnostic client when a diagnostic does not match 
+  /// what is expected (as indicated in the source file).
+  VerifyDiagnosticConsumer(DiagnosticsEngine &Diags);
+  ~VerifyDiagnosticConsumer();
 
   virtual void BeginSourceFile(const LangOptions &LangOpts,
                                const Preprocessor *PP);
 
   virtual void EndSourceFile();
 
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info);
+  
+  virtual DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const;
 };
 
 } // end namspace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Index/ASTLocation.h b/contrib/llvm/tools/clang/include/clang/Index/ASTLocation.h
index fc18dae1a20c..7b66e7ea4fe3 100644
--- a/contrib/llvm/tools/clang/include/clang/Index/ASTLocation.h
+++ b/contrib/llvm/tools/clang/include/clang/Index/ASTLocation.h
@@ -17,10 +17,6 @@
 #include "clang/AST/TypeLoc.h"
 #include "llvm/ADT/PointerIntPair.h"
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
   class Decl;
   class Stmt;
@@ -150,7 +146,7 @@ class ASTLocation {
 
   SourceRange getSourceRange() const;
 
-  void print(llvm::raw_ostream &OS) const;
+  void print(raw_ostream &OS) const;
 };
 
 /// \brief Like ASTLocation but also contains the TranslationUnit that the
diff --git a/contrib/llvm/tools/clang/include/clang/Index/CallGraph.h b/contrib/llvm/tools/clang/include/clang/Index/CallGraph.h
index 336bf47a2efc..38baf0f7537f 100644
--- a/contrib/llvm/tools/clang/include/clang/Index/CallGraph.h
+++ b/contrib/llvm/tools/clang/include/clang/Index/CallGraph.h
@@ -94,7 +94,7 @@ class CallGraph {
 
   Decl *getDecl(CallGraphNode *Node);
 
-  void print(llvm::raw_ostream &os);
+  void print(raw_ostream &os);
   void dump();
 
   void ViewCallGraph() const;
diff --git a/contrib/llvm/tools/clang/include/clang/Index/Entity.h b/contrib/llvm/tools/clang/include/clang/Index/Entity.h
index 9863963ff217..d104458ec249 100644
--- a/contrib/llvm/tools/clang/include/clang/Index/Entity.h
+++ b/contrib/llvm/tools/clang/include/clang/Index/Entity.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CLANG_INDEX_ENTITY_H
 #define LLVM_CLANG_INDEX_ENTITY_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -73,7 +74,7 @@ class Entity {
   static Entity get(Decl *D, Program &Prog);
 
   /// \brief Get an Entity associated with a name in the global namespace.
-  static Entity get(llvm::StringRef Name, Program &Prog);
+  static Entity get(StringRef Name, Program &Prog);
 
   /// \brief true if the Entity is not visible outside the trasnlation unit.
   bool isInternalToTU() const {
diff --git a/contrib/llvm/tools/clang/include/clang/Index/Handlers.h b/contrib/llvm/tools/clang/include/clang/Index/Handlers.h
index 655aef901cd1..1e017f8a7c83 100644
--- a/contrib/llvm/tools/clang/include/clang/Index/Handlers.h
+++ b/contrib/llvm/tools/clang/include/clang/Index/Handlers.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_INDEX_HANDLERS_H
 #define LLVM_CLANG_INDEX_HANDLERS_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace clang {
@@ -61,7 +62,7 @@ class TULocationHandler {
 template <typename handler_type>
 class Storing : public handler_type {
   typedef typename handler_type::receiving_type receiving_type;
-  typedef llvm::SmallVector<receiving_type, 8> StoreTy;
+  typedef SmallVector<receiving_type, 8> StoreTy;
   StoreTy Store;
 
 public:
diff --git a/contrib/llvm/tools/clang/include/clang/Index/TranslationUnit.h b/contrib/llvm/tools/clang/include/clang/Index/TranslationUnit.h
index 0099d630f164..ba5d48d37321 100644
--- a/contrib/llvm/tools/clang/include/clang/Index/TranslationUnit.h
+++ b/contrib/llvm/tools/clang/include/clang/Index/TranslationUnit.h
@@ -16,7 +16,7 @@
 
 namespace clang {
   class ASTContext;
-  class Diagnostic;
+  class DiagnosticsEngine;
   class Preprocessor;
 
 namespace idx {
@@ -29,7 +29,7 @@ class TranslationUnit {
   virtual ~TranslationUnit();
   virtual ASTContext &getASTContext() = 0;
   virtual Preprocessor &getPreprocessor() = 0;
-  virtual Diagnostic &getDiagnostic() = 0;
+  virtual DiagnosticsEngine &getDiagnostic() = 0;
   virtual DeclReferenceMap &getDeclReferenceMap() = 0;
   virtual SelectorMap &getSelectorMap() = 0;
 };
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/CodeCompletionHandler.h b/contrib/llvm/tools/clang/include/clang/Lex/CodeCompletionHandler.h
index d28a3aa7d630..d876776c927c 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/CodeCompletionHandler.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/CodeCompletionHandler.h
@@ -52,6 +52,10 @@ class CodeCompletionHandler {
   
   /// \brief Callback invoked when performing code completion inside a 
   /// function-like macro argument.
+  ///
+  /// There will be another callback invocation after the macro arguments are
+  /// parsed, so this callback should generally be used to note that the next
+  /// callback is invoked inside a macro argument.
   virtual void CodeCompleteMacroArgument(IdentifierInfo *Macro,
                                          MacroInfo *MacroInfo,
                                          unsigned ArgumentIndex) { }
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/DirectoryLookup.h b/contrib/llvm/tools/clang/include/clang/Lex/DirectoryLookup.h
index 1ee6953a12b2..f7da61b0ed71 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/DirectoryLookup.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/DirectoryLookup.h
@@ -14,12 +14,9 @@
 #ifndef LLVM_CLANG_LEX_DIRECTORYLOOKUP_H
 #define LLVM_CLANG_LEX_DIRECTORYLOOKUP_H
 
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceManager.h"
 
-namespace llvm {
-  class StringRef;
-  template <typename T> class SmallVectorImpl;
-}
 namespace clang {
 class HeaderMap;
 class DirectoryEntry;
@@ -59,21 +56,27 @@ class DirectoryLookup {
   /// LookupType - This indicates whether this DirectoryLookup object is a
   /// normal directory, a framework, or a headermap.
   unsigned LookupType : 2;
+  
+  /// \brief Whether this is a header map used when building a framework.
+  unsigned IsIndexHeaderMap : 1;
+  
 public:
   /// DirectoryLookup ctor - Note that this ctor *does not take ownership* of
   /// 'dir'.
   DirectoryLookup(const DirectoryEntry *dir, SrcMgr::CharacteristicKind DT,
                   bool isUser, bool isFramework)
-    : DirCharacteristic(DT), UserSupplied(isUser),
-     LookupType(isFramework ? LT_Framework : LT_NormalDir) {
+    : DirCharacteristic(DT), UserSupplied(isUser), 
+      LookupType(isFramework ? LT_Framework : LT_NormalDir),
+      IsIndexHeaderMap(false) {
     u.Dir = dir;
   }
 
   /// DirectoryLookup ctor - Note that this ctor *does not take ownership* of
   /// 'map'.
   DirectoryLookup(const HeaderMap *map, SrcMgr::CharacteristicKind DT,
-                  bool isUser)
-    : DirCharacteristic(DT), UserSupplied(isUser), LookupType(LT_HeaderMap) {
+                  bool isUser, bool isIndexHeaderMap)
+    : DirCharacteristic(DT), UserSupplied(isUser), LookupType(LT_HeaderMap),
+      IsIndexHeaderMap(isIndexHeaderMap) {
     u.Map = map;
   }
 
@@ -119,7 +122,11 @@ class DirectoryLookup {
   ///
   bool isUserSupplied() const { return UserSupplied; }
 
-
+  /// \brief Whether this header map is building a framework or not.
+  bool isIndexHeaderMap() const { 
+    return isHeaderMap() && IsIndexHeaderMap; 
+  }
+  
   /// LookupFile - Lookup the specified file in this search path, returning it
   /// if it exists or returning null if not.
   ///
@@ -133,15 +140,25 @@ class DirectoryLookup {
   /// \param RelativePath If not NULL, will be set to the path relative to
   /// SearchPath at which the file was found. This only differs from the
   /// Filename for framework includes.
-  const FileEntry *LookupFile(llvm::StringRef Filename, HeaderSearch &HS,
-                              llvm::SmallVectorImpl<char> *SearchPath,
-                              llvm::SmallVectorImpl<char> *RelativePath) const;
+  ///
+  /// \param BuildingModule The name of the module we're currently building.
+  ///
+  /// \param SuggestedModule If non-null, and the file found is semantically
+  /// part of a known module, this will be set to the name of the module that
+  /// could be imported instead of preprocessing/parsing the file found.
+  const FileEntry *LookupFile(StringRef Filename, HeaderSearch &HS,
+                              SmallVectorImpl<char> *SearchPath,
+                              SmallVectorImpl<char> *RelativePath,
+                              StringRef BuildingModule,
+                              StringRef *SuggestedModule) const;
 
 private:
   const FileEntry *DoFrameworkLookup(
-      llvm::StringRef Filename, HeaderSearch &HS,
-      llvm::SmallVectorImpl<char> *SearchPath,
-      llvm::SmallVectorImpl<char> *RelativePath) const;
+      StringRef Filename, HeaderSearch &HS,
+      SmallVectorImpl<char> *SearchPath,
+      SmallVectorImpl<char> *RelativePath,
+      StringRef BuildingModule,
+      StringRef *SuggestedModule) const;
 
 };
 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/HeaderMap.h b/contrib/llvm/tools/clang/include/clang/Lex/HeaderMap.h
index e333840b6a9d..08bc5b64bc12 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/HeaderMap.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/HeaderMap.h
@@ -14,10 +14,10 @@
 #ifndef LLVM_CLANG_LEX_HEADERMAP_H
 #define LLVM_CLANG_LEX_HEADERMAP_H
 
+#include "clang/Basic/LLVM.h"
+
 namespace llvm {
   class MemoryBuffer;
-  class StringRef;
-  template <typename T> class SmallVectorImpl;
 }
 namespace clang {
   class FileEntry;
@@ -52,7 +52,7 @@ class HeaderMap {
   /// raw path at which the file was found in the file system. For example,
   /// for a search path ".." and a filename "../file.h" this would be
   /// "../../file.h".
-  const FileEntry *LookupFile(llvm::StringRef Filename, FileManager &FM) const;
+  const FileEntry *LookupFile(StringRef Filename, FileManager &FM) const;
 
   /// getFileName - Return the filename of the headermap.
   const char *getFileName() const;
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/HeaderSearch.h b/contrib/llvm/tools/clang/include/clang/Lex/HeaderSearch.h
index 5e36d8e60821..84d59f793bbb 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/HeaderSearch.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/HeaderSearch.h
@@ -16,6 +16,8 @@
 
 #include "clang/Lex/DirectoryLookup.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Allocator.h"
 #include <vector>
 
 namespace clang {
@@ -47,6 +49,15 @@ struct HeaderFileInfo {
   /// "resolved", meaning that it was loaded from the external source.
   unsigned Resolved : 1;
   
+  /// \brief Whether this is a header inside a framework that is currently
+  /// being built. 
+  ///
+  /// When a framework is being built, the headers have not yet been placed
+  /// into the appropriate framework subdirectories, and therefore are
+  /// provided via a header map. This bit indicates when this is one of
+  /// those framework headers.
+  unsigned IndexHeaderMapHeader : 1;
+  
   /// NumIncludes - This is the number of times the file has been included
   /// already.
   unsigned short NumIncludes;
@@ -68,10 +79,14 @@ struct HeaderFileInfo {
   /// external storage.
   const IdentifierInfo *ControllingMacro;
 
+  /// \brief If this header came from a framework include, this is the name
+  /// of the framework.
+  StringRef Framework;
+  
   HeaderFileInfo()
     : isImport(false), isPragmaOnce(false), DirInfo(SrcMgr::C_User), 
-      External(false), Resolved(false), NumIncludes(0), ControllingMacroID(0), 
-      ControllingMacro(0)  {}
+      External(false), Resolved(false), IndexHeaderMapHeader(false),
+      NumIncludes(0), ControllingMacroID(0), ControllingMacro(0)  {}
 
   /// \brief Retrieve the controlling macro for this header file, if
   /// any.
@@ -114,6 +129,12 @@ class HeaderSearch {
   unsigned SystemDirIdx;
   bool NoCurDirSearch;
 
+  /// \brief The path to the module cache.
+  std::string ModuleCachePath;
+  
+  /// \brief The name of the module we're building.
+  std::string BuildingModule;
+  
   /// FileInfo - This contains all of the preprocessor-specific data about files
   /// that are included.  The vector is indexed by the FileEntry's UID.
   ///
@@ -125,17 +146,23 @@ class HeaderSearch {
   /// and this value doesn't match the current query, the cache has to be
   /// ignored.  The second value is the entry in SearchDirs that satisfied the
   /// query.
-  llvm::StringMap<std::pair<unsigned, unsigned> > LookupFileCache;
+  llvm::StringMap<std::pair<unsigned, unsigned>, llvm::BumpPtrAllocator>
+    LookupFileCache;
 
 
   /// FrameworkMap - This is a collection mapping a framework or subframework
   /// name like "Carbon" to the Carbon.framework directory.
-  llvm::StringMap<const DirectoryEntry *> FrameworkMap;
+  llvm::StringMap<const DirectoryEntry *, llvm::BumpPtrAllocator>
+    FrameworkMap;
 
   /// HeaderMaps - This is a mapping from FileEntry -> HeaderMap, uniquing
   /// headermaps.  This vector owns the headermap.
   std::vector<std::pair<const FileEntry*, const HeaderMap*> > HeaderMaps;
 
+  /// \brief Uniqued set of framework names, which is used to track which 
+  /// headers were included as framework headers.
+  llvm::StringSet<llvm::BumpPtrAllocator> FrameworkNames;
+  
   /// \brief Entity used to resolve the identifier IDs of controlling
   /// macros into IdentifierInfo pointers, as needed.
   ExternalIdentifierLookup *ExternalLookup;
@@ -172,6 +199,13 @@ class HeaderSearch {
     //LookupFileCache.clear();
   }
 
+  /// \brief Set the path to the module cache and the name of the module
+  /// we're building
+  void configureModules(StringRef CachePath, StringRef BuildingModule) {
+    ModuleCachePath = CachePath;
+    this->BuildingModule = BuildingModule;
+  }
+  
   /// ClearFileInfo - Forget everything we know about headers so far.
   void ClearFileInfo() {
     FileInfo.clear();
@@ -211,12 +245,17 @@ class HeaderSearch {
   /// \param RelativePath If non-null, will be set to the path relative to
   /// SearchPath at which the file was found. This only differs from the
   /// Filename for framework includes.
-  const FileEntry *LookupFile(llvm::StringRef Filename, bool isAngled,
+  ///
+  /// \param SuggestedModule If non-null, and the file found is semantically
+  /// part of a known module, this will be set to the name of the module that
+  /// could be imported instead of preprocessing/parsing the file found.
+  const FileEntry *LookupFile(StringRef Filename, bool isAngled,
                               const DirectoryLookup *FromDir,
                               const DirectoryLookup *&CurDir,
                               const FileEntry *CurFileEnt,
-                              llvm::SmallVectorImpl<char> *SearchPath,
-                              llvm::SmallVectorImpl<char> *RelativePath);
+                              SmallVectorImpl<char> *SearchPath,
+                              SmallVectorImpl<char> *RelativePath,
+                              StringRef *SuggestedModule);
 
   /// LookupSubframeworkHeader - Look up a subframework for the specified
   /// #include file.  For example, if #include'ing <HIToolbox/HIToolbox.h> from
@@ -224,15 +263,15 @@ class HeaderSearch {
   /// is a subframework within Carbon.framework.  If so, return the FileEntry
   /// for the designated file, otherwise return null.
   const FileEntry *LookupSubframeworkHeader(
-      llvm::StringRef Filename,
+      StringRef Filename,
       const FileEntry *RelativeFileEnt,
-      llvm::SmallVectorImpl<char> *SearchPath,
-      llvm::SmallVectorImpl<char> *RelativePath);
+      SmallVectorImpl<char> *SearchPath,
+      SmallVectorImpl<char> *RelativePath);
 
   /// LookupFrameworkCache - Look up the specified framework name in our
   /// framework cache, returning the DirectoryEntry it is in if we know,
   /// otherwise, return null.
-  const DirectoryEntry *&LookupFrameworkCache(llvm::StringRef FWName) {
+  const DirectoryEntry *&LookupFrameworkCache(StringRef FWName) {
     return FrameworkMap.GetOrCreateValue(FWName).getValue();
   }
 
@@ -287,6 +326,23 @@ class HeaderSearch {
   /// FileEntry, uniquing them through the the 'HeaderMaps' datastructure.
   const HeaderMap *CreateHeaderMap(const FileEntry *FE);
 
+  /// \brief Search in the module cache path for a module with the given
+  /// name.
+  ///
+  /// \param If non-NULL, will be set to the module file name we expected to
+  /// find (regardless of whether it was actually found or not).
+  ///
+  /// \param UmbrellaHeader If non-NULL, and no module was found in the module
+  /// cache, this routine will search in the framework paths to determine
+  /// whether a module can be built from an umbrella header. If so, the pointee
+  /// will be set to the path of the umbrella header.
+  ///
+  /// \returns A file describing the named module, if available, or NULL to
+  /// indicate that the module could not be found.
+  const FileEntry *lookupModule(StringRef ModuleName,
+                                std::string *ModuleFileName = 0,
+                                std::string *UmbrellaHeader = 0);
+  
   void IncrementFrameworkLookupCount() { ++NumFrameworkLookups; }
 
   typedef std::vector<HeaderFileInfo>::const_iterator header_file_iterator;
@@ -322,7 +378,13 @@ class HeaderSearch {
   }
   search_dir_iterator system_dir_end() const { return SearchDirs.end(); }
 
+  /// \brief Retrieve a uniqued framework name.
+  StringRef getUniqueFrameworkName(StringRef Framework);
+  
   void PrintStats();
+  
+  size_t getTotalMemory() const;
+
 private:
 
   /// getFileInfo - Return the HeaderFileInfo structure for the specified
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/LexDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Lex/LexDiagnostic.h
index 7d2eb89c50bc..f454e2309acb 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/LexDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/LexDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define LEXSTART
 #include "clang/Basic/DiagnosticLexKinds.inc"
 #undef DIAG
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/Lexer.h b/contrib/llvm/tools/clang/include/clang/Lex/Lexer.h
index f4297627e86f..e01427f574d2 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/Lexer.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/Lexer.h
@@ -21,11 +21,24 @@
 #include <cassert>
 
 namespace clang {
-class Diagnostic;
+class DiagnosticsEngine;
 class SourceManager;
 class Preprocessor;
 class DiagnosticBuilder;
 
+/// ConflictMarkerKind - Kinds of conflict marker which the lexer might be
+/// recovering from.
+enum ConflictMarkerKind {
+  /// Not within a conflict marker.
+  CMK_None,
+  /// A normal or diff3 conflict marker, initiated by at least 7 <s,
+  /// separated by at least 7 =s or |s, and terminated by at least 7 >s.
+  CMK_Normal,
+  /// A Perforce-style conflict marker, initiated by 4 >s, separated by 4 =s,
+  /// and terminated by 4 <s.
+  CMK_Perforce
+};
+
 /// Lexer - This provides a simple interface that turns a text buffer into a
 /// stream of tokens.  This provides no support for file reading or buffering,
 /// or buffering/seeking of tokens, only forward lexing is supported.  It relies
@@ -37,8 +50,7 @@ class Lexer : public PreprocessorLexer {
   const char *BufferEnd;         // End of the buffer.
   SourceLocation FileLoc;        // Location for start of file.
   LangOptions Features;          // Features enabled by this language (cache).
-  bool Is_PragmaLexer : 1;       // True if lexer for _Pragma handling.
-  bool IsInConflictMarker : 1;   // True if in a VCS conflict marker '<<<<<<<'
+  bool Is_PragmaLexer;           // True if lexer for _Pragma handling.
   
   //===--------------------------------------------------------------------===//
   // Context-specific lexing flags set by the preprocessor.
@@ -66,6 +78,9 @@ class Lexer : public PreprocessorLexer {
   // line" flag set on it.
   bool IsAtStartOfLine;
 
+  // CurrentConflictMarkerState - The kind of conflict marker we are handling.
+  ConflictMarkerKind CurrentConflictMarkerState;
+
   Lexer(const Lexer&);          // DO NOT IMPLEMENT
   void operator=(const Lexer&); // DO NOT IMPLEMENT
   friend class Preprocessor;
@@ -208,7 +223,7 @@ class Lexer : public PreprocessorLexer {
 
   /// Stringify - Convert the specified string into a C string by escaping '\'
   /// and " characters.  This does not add surrounding ""'s to the string.
-  static void Stringify(llvm::SmallVectorImpl<char> &Str);
+  static void Stringify(SmallVectorImpl<char> &Str);
 
   
   /// getSpelling - This method is used to get the spelling of a token into a
@@ -244,8 +259,8 @@ class Lexer : public PreprocessorLexer {
   /// This method lexes at the expansion depth of the given
   /// location and does not jump to the expansion or spelling
   /// location.
-  static llvm::StringRef getSpelling(SourceLocation loc,
-                                     llvm::SmallVectorImpl<char> &buffer,
+  static StringRef getSpelling(SourceLocation loc,
+                                     SmallVectorImpl<char> &buffer,
                                      const SourceManager &SourceMgr,
                                      const LangOptions &Features,
                                      bool *invalid = 0);
@@ -322,7 +337,8 @@ class Lexer : public PreprocessorLexer {
   /// of the file begins along with a boolean value indicating whether 
   /// the preamble ends at the beginning of a new line.
   static std::pair<unsigned, bool>
-  ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines = 0);
+  ComputePreamble(const llvm::MemoryBuffer *Buffer, const LangOptions &Features,
+                  unsigned MaxLines = 0);
                                         
   //===--------------------------------------------------------------------===//
   // Internal implementation interfaces.
@@ -456,6 +472,18 @@ class Lexer : public PreprocessorLexer {
   /// them), skip over them and return the first non-escaped-newline found,
   /// otherwise return P.
   static const char *SkipEscapedNewLines(const char *P);
+
+  /// \brief Checks that the given token is the first token that occurs after
+  /// the given location (this excludes comments and whitespace). Returns the
+  /// location immediately after the specified token. If the token is not found
+  /// or the location is inside a macro, the returned source location will be
+  /// invalid.
+  static SourceLocation findLocationAfterToken(SourceLocation loc,
+                                         tok::TokenKind TKind,
+                                         const SourceManager &SM,
+                                         const LangOptions &LangOpts,
+                                         bool SkipTrailingWhitespaceAndNewLine);
+
 private:
 
   /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a
@@ -471,9 +499,13 @@ class Lexer : public PreprocessorLexer {
   // Helper functions to lex the remainder of a token of the specific type.
   void LexIdentifier         (Token &Result, const char *CurPtr);
   void LexNumericConstant    (Token &Result, const char *CurPtr);
-  void LexStringLiteral      (Token &Result, const char *CurPtr,bool Wide);
+  void LexStringLiteral      (Token &Result, const char *CurPtr,
+                              tok::TokenKind Kind);
+  void LexRawStringLiteral   (Token &Result, const char *CurPtr,
+                              tok::TokenKind Kind);
   void LexAngledStringLiteral(Token &Result, const char *CurPtr);
-  void LexCharConstant       (Token &Result, const char *CurPtr);
+  void LexCharConstant       (Token &Result, const char *CurPtr,
+                              tok::TokenKind Kind);
   bool LexEndOfFile          (Token &Result, const char *CurPtr);
 
   bool SkipWhitespace        (Token &Result, const char *CurPtr);
@@ -483,6 +515,9 @@ class Lexer : public PreprocessorLexer {
   
   bool IsStartOfConflictMarker(const char *CurPtr);
   bool HandleEndOfConflictMarker(const char *CurPtr);
+
+  bool isCodeCompletionPoint(const char *CurPtr) const;
+  void cutOffLexing() { BufferPtr = BufferEnd; }
 };
 
 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/LiteralSupport.h b/contrib/llvm/tools/clang/include/clang/Lex/LiteralSupport.h
index 0dbcd6d72d63..b33092c753a8 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/LiteralSupport.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/LiteralSupport.h
@@ -15,14 +15,16 @@
 #ifndef CLANG_LITERALSUPPORT_H
 #define CLANG_LITERALSUPPORT_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/DataTypes.h"
+#include "clang/Basic/TokenKinds.h"
 #include <cctype>
 
 namespace clang {
 
-class Diagnostic;
+class DiagnosticsEngine;
 class Preprocessor;
 class Token;
 class SourceLocation;
@@ -123,15 +125,19 @@ class NumericLiteralParser {
 /// character literal.
 class CharLiteralParser {
   uint64_t Value;
-  bool IsWide;
+  tok::TokenKind Kind;
   bool IsMultiChar;
   bool HadError;
 public:
   CharLiteralParser(const char *begin, const char *end,
-                    SourceLocation Loc, Preprocessor &PP);
+                    SourceLocation Loc, Preprocessor &PP,
+                    tok::TokenKind kind);
 
   bool hadError() const { return HadError; }
-  bool isWide() const { return IsWide; }
+  bool isAscii() const { return Kind == tok::char_constant; }
+  bool isWide() const { return Kind == tok::wide_char_constant; }
+  bool isUTF16() const { return Kind == tok::utf16_char_constant; }
+  bool isUTF32() const { return Kind == tok::utf32_char_constant; }
   bool isMultiChar() const { return IsMultiChar; }
   uint64_t getValue() const { return Value; }
 };
@@ -143,11 +149,12 @@ class StringLiteralParser {
   const SourceManager &SM;
   const LangOptions &Features;
   const TargetInfo &Target;
-  Diagnostic *Diags;
+  DiagnosticsEngine *Diags;
   
   unsigned MaxTokenLength;
   unsigned SizeBound;
-  unsigned wchar_tByteWidth;
+  unsigned CharByteWidth;
+  tok::TokenKind Kind;
   llvm::SmallString<512> ResultBuf;
   char *ResultPtr; // cursor
 public:
@@ -155,27 +162,24 @@ class StringLiteralParser {
                       Preprocessor &PP, bool Complain = true);
   StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
                       const SourceManager &sm, const LangOptions &features,
-                      const TargetInfo &target, Diagnostic *diags = 0)
+                      const TargetInfo &target, DiagnosticsEngine *diags = 0)
     : SM(sm), Features(features), Target(target), Diags(diags),
-      MaxTokenLength(0), SizeBound(0), wchar_tByteWidth(0),
-      ResultPtr(ResultBuf.data()), hadError(false), AnyWide(false), Pascal(false) {
+      MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
+      ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
     init(StringToks, NumStringToks);
   }
     
 
   bool hadError;
-  bool AnyWide;
   bool Pascal;
 
-  llvm::StringRef GetString() const {
-    return llvm::StringRef(ResultBuf.data(), GetStringLength());
+  StringRef GetString() const {
+    return StringRef(ResultBuf.data(), GetStringLength());
   }
   unsigned GetStringLength() const { return ResultPtr-ResultBuf.data(); }
 
   unsigned GetNumStringChars() const {
-    if (AnyWide)
-      return GetStringLength() / wchar_tByteWidth;
-    return GetStringLength();
+    return GetStringLength() / CharByteWidth;
   }
   /// getOffsetOfStringByte - This function returns the offset of the
   /// specified byte of the string data represented by Token.  This handles
@@ -184,9 +188,16 @@ class StringLiteralParser {
   /// If the Diagnostics pointer is non-null, then this will do semantic
   /// checking of the string literal and emit errors and warnings.
   unsigned getOffsetOfStringByte(const Token &TheTok, unsigned ByteNo) const;
-  
+
+  bool isAscii() { return Kind == tok::string_literal; }
+  bool isWide() { return Kind == tok::wide_string_literal; }
+  bool isUTF8() { return Kind == tok::utf8_string_literal; }
+  bool isUTF16() { return Kind == tok::utf16_string_literal; }
+  bool isUTF32() { return Kind == tok::utf32_string_literal; }
+
 private:
   void init(const Token *StringToks, unsigned NumStringToks);
+  void CopyStringFragment(StringRef Fragment);
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/MacroInfo.h b/contrib/llvm/tools/clang/include/clang/Lex/MacroInfo.h
index 9e9d7cf500a4..b381e0f25f41 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/MacroInfo.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/MacroInfo.h
@@ -39,9 +39,14 @@ class MacroInfo {
   IdentifierInfo **ArgumentList;
   unsigned NumArguments;
 
+  /// \brief The location at which this macro was exported from its module.
+  ///
+  /// If invalid, this macro has not been explicitly exported.
+  SourceLocation ExportLocation;
+  
   /// ReplacementTokens - This is the list of tokens that the macro is defined
   /// to.
-  llvm::SmallVector<Token, 8> ReplacementTokens;
+  SmallVector<Token, 8> ReplacementTokens;
 
   /// \brief Length in characters of the macro definition.
   mutable unsigned DefinitionLength;
@@ -68,6 +73,9 @@ class MacroInfo {
   /// IsFromAST - True if this macro was loaded from an AST file.
   bool IsFromAST : 1;
 
+  /// \brief Whether this macro changed after it was loaded from an AST file.
+  bool ChangedAfterLoad : 1;
+  
 private:
   //===--------------------------------------------------------------------===//
   // State that changes as the macro is used.
@@ -209,6 +217,14 @@ class MacroInfo {
   /// setIsFromAST - Set whether this macro was loaded from an AST file.
   void setIsFromAST(bool FromAST = true) { IsFromAST = FromAST; }
 
+  /// \brief Determine whether this macro has changed since it was loaded from
+  /// an AST file.
+  bool hasChangedAfterLoad() const { return ChangedAfterLoad; }
+  
+  /// \brief Note whether this macro has changed after it was loaded from an
+  /// AST file.
+  void setChangedAfterLoad(bool CAL = true) { ChangedAfterLoad = CAL; }
+  
   /// isUsed - Return false if this macro is defined in the main file and has
   /// not yet been used.
   bool isUsed() const { return IsUsed; }
@@ -235,7 +251,7 @@ class MacroInfo {
     return ReplacementTokens[Tok];
   }
 
-  typedef llvm::SmallVector<Token, 8>::const_iterator tokens_iterator;
+  typedef SmallVector<Token, 8>::const_iterator tokens_iterator;
   tokens_iterator tokens_begin() const { return ReplacementTokens.begin(); }
   tokens_iterator tokens_end() const { return ReplacementTokens.end(); }
   bool tokens_empty() const { return ReplacementTokens.empty(); }
@@ -262,6 +278,19 @@ class MacroInfo {
     IsDisabled = true;
   }
 
+  /// \brief Set the export location for this macro.
+  void setExportLocation(SourceLocation ExportLoc) {
+    ExportLocation = ExportLoc;
+  }
+
+  /// \brief Determine whether this macro was explicitly exported from its
+  /// module.
+  bool isExported() const { return ExportLocation.isValid(); }
+  
+  /// \brief Determine the location where this macro was explicitly exported
+  /// from its module.
+  SourceLocation getExportLocation() { return ExportLocation; }
+  
 private:
   unsigned getDefinitionLengthSlow(SourceManager &SM) const;
 };
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/ModuleLoader.h b/contrib/llvm/tools/clang/include/clang/Lex/ModuleLoader.h
new file mode 100644
index 000000000000..72ec0e3ebc9f
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Lex/ModuleLoader.h
@@ -0,0 +1,55 @@
+//===--- ModuleLoader.h - Module Loader Interface ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ModuleLoader interface, which is responsible for 
+//  loading named modules.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_LEX_MODULE_LOADER_H
+#define LLVM_CLANG_LEX_MODULE_LOADER_H
+
+#include "clang/Basic/SourceLocation.h"
+
+namespace clang {
+
+class IdentifierInfo;
+  
+/// \brief An opaque key that is used to describe the module and can be 
+/// interpreted by the module loader itself.
+typedef void *ModuleKey;
+  
+/// \brief Abstract interface for a module loader.
+///
+/// This abstract interface describes a module loader, which is responsible
+/// for resolving a module name (e.g., "std") to an actual module file, and
+/// then loading that module.
+class ModuleLoader {
+public:
+  virtual ~ModuleLoader();
+  
+  /// \brief Attempt to load the given module.
+  ///
+  /// This routine attempts to load the module described by the given 
+  /// parameters.
+  ///
+  /// \param ImportLoc The location of the 'import' keyword.
+  /// \param ModuleName The name of the module to be loaded.
+  /// \param ModuleNameLoc The location of the module name.
+  ///
+  /// \returns If successful, a non-NULL module key describing this module.
+  /// Otherwise, returns NULL to indicate that the module could not be
+  /// loaded.
+  virtual ModuleKey loadModule(SourceLocation ImportLoc, 
+                               IdentifierInfo &ModuleName,
+                               SourceLocation ModuleNameLoc) = 0;
+};
+  
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/PPCallbacks.h b/contrib/llvm/tools/clang/include/clang/Lex/PPCallbacks.h
index a7948153a728..1fc1a05db646 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/PPCallbacks.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/PPCallbacks.h
@@ -42,8 +42,11 @@ class PPCallbacks {
   /// EnteringFile indicates whether this is because we are entering a new
   /// #include'd file (when true) or whether we're exiting one because we ran
   /// off the end (when false).
+  ///
+  /// \param PrevFID the file that was exited if \arg Reason is ExitFile. 
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType) {
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID = FileID()) {
   }
 
   /// FileSkipped - This callback is invoked whenever a source file is
@@ -90,12 +93,12 @@ class PPCallbacks {
   /// file was found. This is equal to FileName except for framework includes.
   virtual void InclusionDirective(SourceLocation HashLoc,
                                   const Token &IncludeTok,
-                                  llvm::StringRef FileName,
+                                  StringRef FileName,
                                   bool IsAngled,
                                   const FileEntry *File,
                                   SourceLocation EndLoc,
-                                  llvm::StringRef SearchPath,
-                                  llvm::StringRef RelativePath) {
+                                  StringRef SearchPath,
+                                  StringRef RelativePath) {
   }
 
   /// EndOfMainFile - This callback is invoked when the end of the main file is
@@ -122,31 +125,32 @@ class PPCallbacks {
   /// \param Loc The location of the message directive.
   /// \param str The text of the message directive.
   ///
-  virtual void PragmaMessage(SourceLocation Loc, llvm::StringRef Str) {
+  virtual void PragmaMessage(SourceLocation Loc, StringRef Str) {
   }
 
   /// PragmaDiagnosticPush - This callback is invoked when a
   /// #pragma gcc dianostic push directive is read.
   virtual void PragmaDiagnosticPush(SourceLocation Loc,
-                                    llvm::StringRef Namespace) {
+                                    StringRef Namespace) {
   }
 
   /// PragmaDiagnosticPop - This callback is invoked when a
   /// #pragma gcc dianostic pop directive is read.
   virtual void PragmaDiagnosticPop(SourceLocation Loc,
-                                   llvm::StringRef Namespace) {
+                                   StringRef Namespace) {
   }
 
   /// PragmaDiagnostic - This callback is invoked when a
   /// #pragma gcc dianostic directive is read.
-  virtual void PragmaDiagnostic(SourceLocation Loc, llvm::StringRef Namespace,
-                                diag::Mapping mapping, llvm::StringRef Str) {
+  virtual void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
+                                diag::Mapping mapping, StringRef Str) {
   }
 
   /// MacroExpands - This is called by
   /// Preprocessor::HandleMacroExpandedIdentifier when a macro invocation is
   /// found.
-  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI) {
+  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
+                            SourceRange Range) {
   }
 
   /// MacroDefined - This hook is called whenever a macro definition is seen.
@@ -157,6 +161,16 @@ class PPCallbacks {
   /// MI is released immediately following this callback.
   virtual void MacroUndefined(const Token &MacroNameTok, const MacroInfo *MI) {
   }
+  
+  /// Defined - This hook is called whenever the 'defined' operator is seen.
+  virtual void Defined(const Token &MacroNameTok) {
+  }
+  
+  /// SourceRangeSkipped - This hook is called when a source range is skipped.
+  /// \param Range The SourceRange that was skipped. The range begins at the
+  /// #if/#else directive and ends after the #endif/#else directive.
+  virtual void SourceRangeSkipped(SourceRange Range) {
+  }
 
   /// If -- This hook is called whenever an #if is seen.
   /// \param Range The SourceRange of the expression being tested.
@@ -204,9 +218,10 @@ class PPChainedCallbacks : public PPCallbacks {
   }
 
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType) {
-    First->FileChanged(Loc, Reason, FileType);
-    Second->FileChanged(Loc, Reason, FileType);
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID) {
+    First->FileChanged(Loc, Reason, FileType, PrevFID);
+    Second->FileChanged(Loc, Reason, FileType, PrevFID);
   }
 
   virtual void FileSkipped(const FileEntry &ParentFile,
@@ -218,12 +233,12 @@ class PPChainedCallbacks : public PPCallbacks {
 
   virtual void InclusionDirective(SourceLocation HashLoc,
                                   const Token &IncludeTok,
-                                  llvm::StringRef FileName,
+                                  StringRef FileName,
                                   bool IsAngled,
                                   const FileEntry *File,
                                   SourceLocation EndLoc,
-                                  llvm::StringRef SearchPath,
-                                  llvm::StringRef RelativePath) {
+                                  StringRef SearchPath,
+                                  StringRef RelativePath) {
     First->InclusionDirective(HashLoc, IncludeTok, FileName, IsAngled, File,
                               EndLoc, SearchPath, RelativePath);
     Second->InclusionDirective(HashLoc, IncludeTok, FileName, IsAngled, File,
@@ -246,32 +261,33 @@ class PPChainedCallbacks : public PPCallbacks {
     Second->PragmaComment(Loc, Kind, Str);
   }
 
-  virtual void PragmaMessage(SourceLocation Loc, llvm::StringRef Str) {
+  virtual void PragmaMessage(SourceLocation Loc, StringRef Str) {
     First->PragmaMessage(Loc, Str);
     Second->PragmaMessage(Loc, Str);
   }
 
   virtual void PragmaDiagnosticPush(SourceLocation Loc,
-                                    llvm::StringRef Namespace) {
+                                    StringRef Namespace) {
     First->PragmaDiagnosticPush(Loc, Namespace);
     Second->PragmaDiagnosticPush(Loc, Namespace);
   }
 
   virtual void PragmaDiagnosticPop(SourceLocation Loc,
-                                    llvm::StringRef Namespace) {
+                                    StringRef Namespace) {
     First->PragmaDiagnosticPop(Loc, Namespace);
     Second->PragmaDiagnosticPop(Loc, Namespace);
   }
 
-  virtual void PragmaDiagnostic(SourceLocation Loc, llvm::StringRef Namespace,
-                                diag::Mapping mapping, llvm::StringRef Str) {
+  virtual void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
+                                diag::Mapping mapping, StringRef Str) {
     First->PragmaDiagnostic(Loc, Namespace, mapping, Str);
     Second->PragmaDiagnostic(Loc, Namespace, mapping, Str);
   }
 
-  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI) {
-    First->MacroExpands(MacroNameTok, MI);
-    Second->MacroExpands(MacroNameTok, MI);
+  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
+                            SourceRange Range) {
+    First->MacroExpands(MacroNameTok, MI, Range);
+    Second->MacroExpands(MacroNameTok, MI, Range);
   }
 
   virtual void MacroDefined(const Token &MacroNameTok, const MacroInfo *MI) {
@@ -284,6 +300,16 @@ class PPChainedCallbacks : public PPCallbacks {
     Second->MacroUndefined(MacroNameTok, MI);
   }
 
+  virtual void Defined(const Token &MacroNameTok) {
+    First->Defined(MacroNameTok);
+    Second->Defined(MacroNameTok);
+  }
+
+  virtual void SourceRangeSkipped(SourceRange Range) {
+    First->SourceRangeSkipped(Range);
+    Second->SourceRangeSkipped(Range);
+  }
+
   /// If -- This hook is called whenever an #if is seen.
   virtual void If(SourceRange Range) {
     First->If(Range);
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/PTHManager.h b/contrib/llvm/tools/clang/include/clang/Lex/PTHManager.h
index 094b7ef66798..25a49038a863 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/PTHManager.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/PTHManager.h
@@ -30,7 +30,7 @@ namespace clang {
 
 class FileEntry;
 class PTHLexer;
-class Diagnostic;
+class DiagnosticsEngine;
 class FileSystemStatCache;
 
 class PTHManager : public IdentifierInfoLookup {
@@ -115,11 +115,11 @@ class PTHManager : public IdentifierInfoLookup {
   ///  Unlike the version in IdentifierTable, this returns a pointer instead
   ///  of a reference.  If the pointer is NULL then the IdentifierInfo cannot
   ///  be found.
-  IdentifierInfo *get(llvm::StringRef Name);
+  IdentifierInfo *get(StringRef Name);
 
   /// Create - This method creates PTHManager objects.  The 'file' argument
   ///  is the name of the PTH file.  This method returns NULL upon failure.
-  static PTHManager *Create(const std::string& file, Diagnostic &Diags);
+  static PTHManager *Create(const std::string& file, DiagnosticsEngine &Diags);
 
   void setPreprocessor(Preprocessor *pp) { PP = pp; }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/Pragma.h b/contrib/llvm/tools/clang/include/clang/Lex/Pragma.h
index c6ab35c19c1e..4868811e7036 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/Pragma.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/Pragma.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_PRAGMA_H
 #define LLVM_CLANG_PRAGMA_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
@@ -58,11 +59,11 @@ namespace clang {
 class PragmaHandler {
   std::string Name;
 public:
-  explicit PragmaHandler(llvm::StringRef name) : Name(name) {}
+  explicit PragmaHandler(StringRef name) : Name(name) {}
   PragmaHandler() {}
   virtual ~PragmaHandler();
 
-  llvm::StringRef getName() const { return Name; }
+  StringRef getName() const { return Name; }
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken) = 0;
 
@@ -91,14 +92,14 @@ class PragmaNamespace : public PragmaHandler {
   ///
   llvm::StringMap<PragmaHandler*> Handlers;
 public:
-  explicit PragmaNamespace(llvm::StringRef Name) : PragmaHandler(Name) {}
+  explicit PragmaNamespace(StringRef Name) : PragmaHandler(Name) {}
   virtual ~PragmaNamespace();
 
   /// FindHandler - Check to see if there is already a handler for the
   /// specified name.  If not, return the handler for the null name if it
   /// exists, otherwise return null.  If IgnoreNull is true (the default) then
   /// the null handler isn't returned on failure to match.
-  PragmaHandler *FindHandler(llvm::StringRef Name,
+  PragmaHandler *FindHandler(StringRef Name,
                              bool IgnoreNull = true) const;
 
   /// AddPragma - Add a pragma to this namespace.
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/PreprocessingRecord.h b/contrib/llvm/tools/clang/include/clang/Lex/PreprocessingRecord.h
index b38303a2f40b..53da19e6c296 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/PreprocessingRecord.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/PreprocessingRecord.h
@@ -16,6 +16,7 @@
 
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/Allocator.h"
 #include <vector>
@@ -43,14 +44,14 @@ namespace clang {
   public:
     /// \brief The kind of preprocessed entity an object describes.
     enum EntityKind {
+      /// \brief Indicates a problem trying to load the preprocessed entity.
+      InvalidKind,
+
       /// \brief A macro expansion.
       MacroExpansionKind,
       
-      /// \brief A preprocessing directive whose kind is not specified.
-      ///
-      /// This kind will be used for any preprocessing directive that does not
-      /// have a more specific kind within the \c DirectiveKind enumeration.
-      PreprocessingDirectiveKind,
+      /// \defgroup Preprocessing directives
+      /// @{
       
       /// \brief A macro definition.
       MacroDefinitionKind,
@@ -59,7 +60,9 @@ namespace clang {
       /// #import, or \c #include_next.
       InclusionDirectiveKind,
 
-      FirstPreprocessingDirective = PreprocessingDirectiveKind,
+      /// @}
+
+      FirstPreprocessingDirective = MacroDefinitionKind,
       LastPreprocessingDirective = InclusionDirectiveKind
     };
 
@@ -73,7 +76,9 @@ namespace clang {
   protected:
     PreprocessedEntity(EntityKind Kind, SourceRange Range)
       : Kind(Kind), Range(Range) { }
-    
+
+    friend class PreprocessingRecord;
+
   public:
     /// \brief Retrieve the kind of preprocessed entity stored in this object.
     EntityKind getKind() const { return Kind; }
@@ -81,7 +86,11 @@ namespace clang {
     /// \brief Retrieve the source range that covers this entire preprocessed 
     /// entity.
     SourceRange getSourceRange() const { return Range; }
-    
+
+    /// \brief Returns true if there was a problem loading the preprocessed
+    /// entity.
+    bool isInvalid() const { return Kind == InvalidKind; }
+
     // Implement isa/cast/dyncast/etc.
     static bool classof(const PreprocessedEntity *) { return true; }
 
@@ -110,34 +119,6 @@ namespace clang {
     void operator delete(void* data) throw();
   };
   
-  /// \brief Records the location of a macro expansion.
-  class MacroExpansion : public PreprocessedEntity {
-    /// \brief The name of the macro being expanded.
-    IdentifierInfo *Name;
-    
-    /// \brief The definition of this macro.
-    MacroDefinition *Definition;
-    
-  public:
-    MacroExpansion(IdentifierInfo *Name, SourceRange Range,
-                   MacroDefinition *Definition)
-      : PreprocessedEntity(MacroExpansionKind, Range), Name(Name),
-        Definition(Definition) { }
-    
-    /// \brief The name of the macro being expanded.
-    IdentifierInfo *getName() const { return Name; }
-    
-    /// \brief The definition of the macro being expanded.
-    MacroDefinition *getDefinition() const { return Definition; }
-
-    // Implement isa/cast/dyncast/etc.
-    static bool classof(const PreprocessedEntity *PE) {
-      return PE->getKind() == MacroExpansionKind;
-    }
-    static bool classof(const MacroExpansion *) { return true; }
-
-  };
-  
   /// \brief Records the presence of a preprocessor directive.
   class PreprocessingDirective : public PreprocessedEntity {
   public:
@@ -156,21 +137,16 @@ namespace clang {
   class MacroDefinition : public PreprocessingDirective {
     /// \brief The name of the macro being defined.
     const IdentifierInfo *Name;
-    
-    /// \brief The location of the macro name in the macro definition.
-    SourceLocation Location;
 
   public:
-    explicit MacroDefinition(const IdentifierInfo *Name, SourceLocation Location,
-                             SourceRange Range)
-      : PreprocessingDirective(MacroDefinitionKind, Range), Name(Name), 
-        Location(Location) { }
+    explicit MacroDefinition(const IdentifierInfo *Name, SourceRange Range)
+      : PreprocessingDirective(MacroDefinitionKind, Range), Name(Name) { }
     
     /// \brief Retrieve the name of the macro being defined.
     const IdentifierInfo *getName() const { return Name; }
     
     /// \brief Retrieve the location of the macro name in the definition.
-    SourceLocation getLocation() const { return Location; }
+    SourceLocation getLocation() const { return getSourceRange().getBegin(); }
     
     // Implement isa/cast/dyncast/etc.
     static bool classof(const PreprocessedEntity *PE) {
@@ -178,6 +154,44 @@ namespace clang {
     }
     static bool classof(const MacroDefinition *) { return true; }
   };
+  
+  /// \brief Records the location of a macro expansion.
+  class MacroExpansion : public PreprocessedEntity {
+    /// \brief The definition of this macro or the name of the macro if it is
+    /// a builtin macro.
+    llvm::PointerUnion<IdentifierInfo *, MacroDefinition *> NameOrDef; 
+
+  public:
+    MacroExpansion(IdentifierInfo *BuiltinName, SourceRange Range)
+      : PreprocessedEntity(MacroExpansionKind, Range),
+        NameOrDef(BuiltinName) { }
+
+    MacroExpansion(MacroDefinition *Definition, SourceRange Range)
+      : PreprocessedEntity(MacroExpansionKind, Range),
+        NameOrDef(Definition) { }
+
+    /// \brief True if it is a builtin macro.
+    bool isBuiltinMacro() const { return NameOrDef.is<IdentifierInfo *>(); }
+    
+    /// \brief The name of the macro being expanded.
+    const IdentifierInfo *getName() const {
+      if (MacroDefinition *Def = getDefinition())
+        return Def->getName();
+      return NameOrDef.get<IdentifierInfo*>();
+    }
+    
+    /// \brief The definition of the macro being expanded. May return null if
+    /// this is a builtin macro.
+    MacroDefinition *getDefinition() const {
+      return NameOrDef.dyn_cast<MacroDefinition *>();
+    }
+
+    // Implement isa/cast/dyncast/etc.
+    static bool classof(const PreprocessedEntity *PE) {
+      return PE->getKind() == MacroExpansionKind;
+    }
+    static bool classof(const MacroExpansion *) { return true; }
+  };
 
   /// \brief Record the location of an inclusion directive, such as an
   /// \c #include or \c #import statement.
@@ -199,7 +213,7 @@ namespace clang {
   private:
     /// \brief The name of the file that was included, as written in
     /// the source.
-    llvm::StringRef FileName;
+    StringRef FileName;
 
     /// \brief Whether the file name was in quotation marks; otherwise, it was
     /// in angle brackets.
@@ -215,14 +229,14 @@ namespace clang {
 
   public:
     InclusionDirective(PreprocessingRecord &PPRec,
-                       InclusionKind Kind, llvm::StringRef FileName, 
+                       InclusionKind Kind, StringRef FileName, 
                        bool InQuotes, const FileEntry *File, SourceRange Range);
     
     /// \brief Determine what kind of inclusion directive this is.
     InclusionKind getKind() const { return static_cast<InclusionKind>(Kind); }
     
     /// \brief Retrieve the included file name as it was written in the source.
-    llvm::StringRef getFileName() const { return FileName; }
+    StringRef getFileName() const { return FileName; }
     
     /// \brief Determine whether the included file name was written in quotes;
     /// otherwise, it was written in angle brackets.
@@ -245,19 +259,24 @@ namespace clang {
   public:
     virtual ~ExternalPreprocessingRecordSource();
     
-    /// \brief Read any preallocated preprocessed entities from the external
-    /// source.
-    virtual void ReadPreprocessedEntities() = 0;
-    
-    /// \brief Read the preprocessed entity at the given offset.
-    virtual PreprocessedEntity *
-    ReadPreprocessedEntityAtOffset(uint64_t Offset) = 0;
+    /// \brief Read a preallocated preprocessed entity from the external source.
+    ///
+    /// \returns null if an error occurred that prevented the preprocessed
+    /// entity from being loaded.
+    virtual PreprocessedEntity *ReadPreprocessedEntity(unsigned Index) = 0;
+
+    /// \brief Returns a pair of [Begin, End) indices of preallocated
+    /// preprocessed entities that \arg Range encompasses.
+    virtual std::pair<unsigned, unsigned>
+        findPreprocessedEntitiesInRange(SourceRange Range) = 0;
   };
   
   /// \brief A record of the steps taken while preprocessing a source file,
   /// including the various preprocessing directives processed, macros 
   /// expanded, etc.
   class PreprocessingRecord : public PPCallbacks {
+    SourceManager &SourceMgr;
+
     /// \brief Whether we should include nested macro expansions in
     /// the preprocessing record.
     bool IncludeNestedMacroExpansions;
@@ -269,24 +288,64 @@ namespace clang {
     /// were seen.
     std::vector<PreprocessedEntity *> PreprocessedEntities;
     
+    /// \brief The set of preprocessed entities in this record that have been
+    /// loaded from external sources.
+    ///
+    /// The entries in this vector are loaded lazily from the external source,
+    /// and are referenced by the iterator using negative indices.
+    std::vector<PreprocessedEntity *> LoadedPreprocessedEntities;
+
+    /// \brief Global (loaded or local) ID for a preprocessed entity.
+    /// Negative values are used to indicate preprocessed entities
+    /// loaded from the external source while non-negative values are used to
+    /// indicate preprocessed entities introduced by the current preprocessor.
+    /// If M is the number of loaded preprocessed entities, value -M
+    /// corresponds to element 0 in the loaded entities vector, position -M+1
+    /// corresponds to element 1 in the loaded entities vector, etc.
+    typedef int PPEntityID;
+
+    PPEntityID getPPEntityID(unsigned Index, bool isLoaded) const {
+      return isLoaded ? PPEntityID(Index) - LoadedPreprocessedEntities.size()
+                      : Index;
+    }
+
     /// \brief Mapping from MacroInfo structures to their definitions.
-    llvm::DenseMap<const MacroInfo *, MacroDefinition *> MacroDefinitions;
+    llvm::DenseMap<const MacroInfo *, PPEntityID> MacroDefinitions;
 
     /// \brief External source of preprocessed entities.
     ExternalPreprocessingRecordSource *ExternalSource;
+
+    /// \brief Retrieve the preprocessed entity at the given ID.
+    PreprocessedEntity *getPreprocessedEntity(PPEntityID PPID);
+
+    /// \brief Retrieve the loaded preprocessed entity at the given index.
+    PreprocessedEntity *getLoadedPreprocessedEntity(unsigned Index);
     
-    /// \brief The number of preallocated entities (that are known to the
-    /// external source).
-    unsigned NumPreallocatedEntities;
-    
-    /// \brief Whether we have already loaded all of the preallocated entities.
-    mutable bool LoadedPreallocatedEntities;
-    
-    void MaybeLoadPreallocatedEntities() const ;
+    /// \brief Determine the number of preprocessed entities that were
+    /// loaded (or can be loaded) from an external source.
+    unsigned getNumLoadedPreprocessedEntities() const {
+      return LoadedPreprocessedEntities.size();
+    }
+
+    /// \brief Returns a pair of [Begin, End) indices of local preprocessed
+    /// entities that \arg Range encompasses.
+    std::pair<unsigned, unsigned>
+      findLocalPreprocessedEntitiesInRange(SourceRange Range) const;
+    unsigned findBeginLocalPreprocessedEntity(SourceLocation Loc) const;
+    unsigned findEndLocalPreprocessedEntity(SourceLocation Loc) const;
+
+    /// \brief Allocate space for a new set of loaded preprocessed entities.
+    ///
+    /// \returns The index into the set of loaded preprocessed entities, which
+    /// corresponds to the first newly-allocated entity.
+    unsigned allocateLoadedEntities(unsigned NumEntities);
+
+    /// \brief Register a new macro definition.
+    void RegisterMacroDefinition(MacroInfo *Macro, PPEntityID PPID);
     
   public:
-    /// \brief Construct 
-    explicit PreprocessingRecord(bool IncludeNestedMacroExpansions);
+    /// \brief Construct a new preprocessing record.
+    PreprocessingRecord(SourceManager &SM, bool IncludeNestedMacroExpansions);
     
     /// \brief Allocate memory in the preprocessing record.
     void *Allocate(unsigned Size, unsigned Align = 8) {
@@ -295,64 +354,180 @@ namespace clang {
     
     /// \brief Deallocate memory in the preprocessing record.
     void Deallocate(void *Ptr) { }
-    
-    size_t getTotalMemory() const {
-      return BumpAlloc.getTotalMemory();
-    }
-    
+
+    size_t getTotalMemory() const;
+
+    SourceManager &getSourceManager() const { return SourceMgr; }
+
     // Iteration over the preprocessed entities.
-    typedef std::vector<PreprocessedEntity *>::iterator iterator;
-    typedef std::vector<PreprocessedEntity *>::const_iterator const_iterator;
-    iterator begin(bool OnlyLocalEntities = false);
-    iterator end(bool OnlyLocalEntities = false);
-    const_iterator begin(bool OnlyLocalEntities = false) const;
-    const_iterator end(bool OnlyLocalEntities = false) const;
+    class iterator {
+      PreprocessingRecord *Self;
+      
+      /// \brief Position within the preprocessed entity sequence.
+      ///
+      /// In a complete iteration, the Position field walks the range [-M, N),
+      /// where negative values are used to indicate preprocessed entities
+      /// loaded from the external source while non-negative values are used to
+      /// indicate preprocessed entities introduced by the current preprocessor.
+      /// However, to provide iteration in source order (for, e.g., chained
+      /// precompiled headers), dereferencing the iterator flips the negative
+      /// values (corresponding to loaded entities), so that position -M 
+      /// corresponds to element 0 in the loaded entities vector, position -M+1
+      /// corresponds to element 1 in the loaded entities vector, etc. This
+      /// gives us a reasonably efficient, source-order walk.
+      PPEntityID Position;
+      
+    public:
+      typedef PreprocessedEntity *value_type;
+      typedef value_type&         reference;
+      typedef value_type*         pointer;
+      typedef std::random_access_iterator_tag iterator_category;
+      typedef int                 difference_type;
+      
+      iterator() : Self(0), Position(0) { }
+      
+      iterator(PreprocessingRecord *Self, int Position) 
+        : Self(Self), Position(Position) { }
+      
+      value_type operator*() const {
+        return Self->getPreprocessedEntity(Position);
+      }
+      
+      value_type operator[](difference_type D) {
+        return *(*this + D);
+      }
+      
+      iterator &operator++() {
+        ++Position;
+        return *this;
+      }
+      
+      iterator operator++(int) {
+        iterator Prev(*this);
+        ++Position;
+        return Prev;
+      }
+
+      iterator &operator--() {
+        --Position;
+        return *this;
+      }
+      
+      iterator operator--(int) {
+        iterator Prev(*this);
+        --Position;
+        return Prev;
+      }
+
+      friend bool operator==(const iterator &X, const iterator &Y) {
+        return X.Position == Y.Position;
+      }
+
+      friend bool operator!=(const iterator &X, const iterator &Y) {
+        return X.Position != Y.Position;
+      }
+      
+      friend bool operator<(const iterator &X, const iterator &Y) {
+        return X.Position < Y.Position;
+      }
+
+      friend bool operator>(const iterator &X, const iterator &Y) {
+        return X.Position > Y.Position;
+      }
+
+      friend bool operator<=(const iterator &X, const iterator &Y) {
+        return X.Position < Y.Position;
+      }
+      
+      friend bool operator>=(const iterator &X, const iterator &Y) {
+        return X.Position > Y.Position;
+      }
+
+      friend iterator& operator+=(iterator &X, difference_type D) {
+        X.Position += D;
+        return X;
+      }
+
+      friend iterator& operator-=(iterator &X, difference_type D) {
+        X.Position -= D;
+        return X;
+      }
+
+      friend iterator operator+(iterator X, difference_type D) {
+        X.Position += D;
+        return X;
+      }
+
+      friend iterator operator+(difference_type D, iterator X) {
+        X.Position += D;
+        return X;
+      }
+
+      friend difference_type operator-(const iterator &X, const iterator &Y) {
+        return X.Position - Y.Position;
+      }
+
+      friend iterator operator-(iterator X, difference_type D) {
+        X.Position -= D;
+        return X;
+      }
+    };
+    friend class iterator;
+
+    /// \brief Begin iterator for all preprocessed entities.
+    iterator begin() {
+      return iterator(this, -(int)LoadedPreprocessedEntities.size());
+    }
+
+    /// \brief End iterator for all preprocessed entities.
+    iterator end() {
+      return iterator(this, PreprocessedEntities.size());
+    }
+
+    /// \brief Begin iterator for local, non-loaded, preprocessed entities.
+    iterator local_begin() {
+      return iterator(this, 0);
+    }
+
+    /// \brief End iterator for local, non-loaded, preprocessed entities.
+    iterator local_end() {
+      return iterator(this, PreprocessedEntities.size());
+    }
+
+    /// \brief Returns a pair of [Begin, End) iterators of preprocessed entities
+    /// that source range \arg R encompasses.
+    std::pair<iterator, iterator> getPreprocessedEntitiesInRange(SourceRange R);
 
     /// \brief Add a new preprocessed entity to this record.
     void addPreprocessedEntity(PreprocessedEntity *Entity);
     
     /// \brief Set the external source for preprocessed entities.
-    void SetExternalSource(ExternalPreprocessingRecordSource &Source,
-                           unsigned NumPreallocatedEntities);
+    void SetExternalSource(ExternalPreprocessingRecordSource &Source);
 
     /// \brief Retrieve the external source for preprocessed entities.
     ExternalPreprocessingRecordSource *getExternalSource() const {
       return ExternalSource;
     }
     
-    unsigned getNumPreallocatedEntities() const {
-      return NumPreallocatedEntities;
-    }
-    
-    /// \brief Set the preallocated entry at the given index to the given
-    /// preprocessed entity.
-    void SetPreallocatedEntity(unsigned Index, PreprocessedEntity *Entity);
-
-    /// \brief Register a new macro definition.
-    void RegisterMacroDefinition(MacroInfo *Macro, MacroDefinition *MD);
-                           
-    /// \brief Retrieve the preprocessed entity at the given index.
-    PreprocessedEntity *getPreprocessedEntity(unsigned Index) {
-      assert(Index < PreprocessedEntities.size() &&
-             "Out-of-bounds preprocessed entity");
-      return PreprocessedEntities[Index];
-    }
-    
     /// \brief Retrieve the macro definition that corresponds to the given
     /// \c MacroInfo.
     MacroDefinition *findMacroDefinition(const MacroInfo *MI);
-    
-    virtual void MacroExpands(const Token &Id, const MacroInfo* MI);
+        
+    virtual void MacroExpands(const Token &Id, const MacroInfo* MI,
+                              SourceRange Range);
     virtual void MacroDefined(const Token &Id, const MacroInfo *MI);
     virtual void MacroUndefined(const Token &Id, const MacroInfo *MI);
     virtual void InclusionDirective(SourceLocation HashLoc,
                                     const Token &IncludeTok,
-                                    llvm::StringRef FileName,
+                                    StringRef FileName,
                                     bool IsAngled,
                                     const FileEntry *File,
                                     SourceLocation EndLoc,
-                                    llvm::StringRef SearchPath,
-                                    llvm::StringRef RelativePath);
+                                    StringRef SearchPath,
+                                    StringRef RelativePath);
+
+    friend class ASTReader;
+    friend class ASTWriter;
   };
 } // end namespace clang
 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h b/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h
index f6f3205099a0..8b7743316f9c 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h
@@ -49,6 +49,7 @@ class PPCallbacks;
 class CodeCompletionHandler;
 class DirectoryLookup;
 class PreprocessingRecord;
+class ModuleLoader;
   
 /// Preprocessor - This object engages in a tight little dance with the lexer to
 /// efficiently preprocess tokens.  Lexers know only about tokens within a
@@ -56,17 +57,19 @@ class PreprocessingRecord;
 /// like the #include stack, token expansion, etc.
 ///
 class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
-  Diagnostic        *Diags;
-  LangOptions        Features;
-  const TargetInfo  &Target;
+  DiagnosticsEngine        *Diags;
+  LangOptions       &Features;
+  const TargetInfo  *Target;
   FileManager       &FileMgr;
   SourceManager     &SourceMgr;
   ScratchBuffer     *ScratchBuf;
   HeaderSearch      &HeaderInfo;
+  ModuleLoader      &TheModuleLoader;
 
   /// \brief External source of macros.
   ExternalPreprocessorSource *ExternalSource;
 
+  
   /// PTH - An optional PTHManager object used for getting tokens from
   ///  a token cache rather than lexing the original source file.
   llvm::OwningPtr<PTHManager> PTH;
@@ -90,6 +93,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   IdentifierInfo *Ident__has_attribute;            // __has_attribute
   IdentifierInfo *Ident__has_include;              // __has_include
   IdentifierInfo *Ident__has_include_next;         // __has_include_next
+  IdentifierInfo *Ident__has_warning;              // __has_warning
 
   SourceLocation DATELoc, TIMELoc;
   unsigned CounterValue;  // Next __COUNTER__ value.
@@ -102,7 +106,9 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   // State that is set before the preprocessor begins.
   bool KeepComments : 1;
   bool KeepMacroComments : 1;
-
+  bool SuppressIncludeNotFoundError : 1;
+  bool AutoModuleImport : 1;
+  
   // State that changes while the preprocessor runs:
   bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
 
@@ -145,6 +151,29 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// \brief The file that we're performing code-completion for, if any.
   const FileEntry *CodeCompletionFile;
 
+  /// \brief The offset in file for the code-completion point.
+  unsigned CodeCompletionOffset;
+
+  /// \brief The location for the code-completion point. This gets instantiated
+  /// when the CodeCompletionFile gets #include'ed for preprocessing.
+  SourceLocation CodeCompletionLoc;
+
+  /// \brief The start location for the file of the code-completion point.
+  /// This gets instantiated when the CodeCompletionFile gets #include'ed
+  /// for preprocessing.
+  SourceLocation CodeCompletionFileLoc;
+
+  /// \brief The source location of the __import_module__ keyword we just
+  /// lexed, if any.
+  SourceLocation ModuleImportLoc;
+
+  /// \brief The source location of the currently-active
+  /// #pragma clang arc_cf_code_audited begin.
+  SourceLocation PragmaARCCFCodeAuditedLoc;
+  
+  /// \brief True if we hit the code-completion point.
+  bool CodeCompletionReached;
+
   /// \brief The number of bytes that we will initially skip when entering the
   /// main file, which is used when loading a precompiled preamble, along
   /// with a flag that indicates whether skipping this number of bytes will
@@ -165,7 +194,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   ///  if not expanding a macro.  This is an alias for either CurLexer or
   ///  CurPTHLexer.
   PreprocessorLexer *CurPPLexer;
-
+    
   /// CurLookup - The DirectoryLookup structure used to find the current
   /// FileEntry, if CurLexer is non-null and if applicable.  This allows us to
   /// implement #include_next and find directory-specific properties.
@@ -175,20 +204,31 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// expanding a macro.  One of CurLexer and CurTokenLexer must be null.
   llvm::OwningPtr<TokenLexer> CurTokenLexer;
 
+  /// \brief The kind of lexer we're currently working with.
+  enum CurLexerKind { 
+    CLK_Lexer, 
+    CLK_PTHLexer, 
+    CLK_TokenLexer, 
+    CLK_CachingLexer,
+    CLK_LexAfterModuleImport
+  } CurLexerKind;
+
   /// IncludeMacroStack - This keeps track of the stack of files currently
   /// #included, and macros currently being expanded from, not counting
   /// CurLexer/CurTokenLexer.
   struct IncludeStackInfo {
+    enum CurLexerKind     CurLexerKind;
     Lexer                 *TheLexer;
     PTHLexer              *ThePTHLexer;
     PreprocessorLexer     *ThePPLexer;
     TokenLexer            *TheTokenLexer;
     const DirectoryLookup *TheDirLookup;
 
-    IncludeStackInfo(Lexer *L, PTHLexer* P, PreprocessorLexer* PPL,
+    IncludeStackInfo(enum CurLexerKind K, Lexer *L, PTHLexer* P, 
+                     PreprocessorLexer* PPL,
                      TokenLexer* TL, const DirectoryLookup *D)
-      : TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL), TheTokenLexer(TL),
-        TheDirLookup(D) {}
+      : CurLexerKind(K), TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL), 
+        TheTokenLexer(TL), TheDirLookup(D) {}
   };
   std::vector<IncludeStackInfo> IncludeMacroStack;
 
@@ -220,10 +260,6 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// previous macro value.
   llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
 
-  /// \brief Expansion source location for the last macro that expanded
-  /// to no tokens.
-  SourceLocation LastEmptyMacroExpansionLoc;
-
   // Various statistics we track for performance analysis.
   unsigned NumDirectives, NumIncluded, NumDefined, NumUndefined, NumPragma;
   unsigned NumIf, NumElse, NumEndif;
@@ -246,7 +282,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
   /// going to lex in the cache and when it finishes the tokens are removed
   /// from the end of the cache.
-  llvm::SmallVector<Token, 16> MacroExpandedTokens;
+  SmallVector<Token, 16> MacroExpandedTokens;
   std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
 
   /// \brief A record of the macro definitions and expansions that
@@ -257,7 +293,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   PreprocessingRecord *Record;
   
 private:  // Cached tokens state.
-  typedef llvm::SmallVector<Token, 1> CachedTokensTy;
+  typedef SmallVector<Token, 1> CachedTokensTy;
 
   /// CachedTokens - Cached tokens are stored here when we do backtracking or
   /// lookahead. They are "lexed" by the CachingLex() method.
@@ -291,19 +327,27 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   MacroInfo *getInfoForMacro(IdentifierInfo *II) const;
   
 public:
-  Preprocessor(Diagnostic &diags, const LangOptions &opts,
-               const TargetInfo &target,
+  Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
+               const TargetInfo *target,
                SourceManager &SM, HeaderSearch &Headers,
+               ModuleLoader &TheModuleLoader,
                IdentifierInfoLookup *IILookup = 0,
-               bool OwnsHeaderSearch = false);
+               bool OwnsHeaderSearch = false,
+               bool DelayInitialization = false);
 
   ~Preprocessor();
 
-  Diagnostic &getDiagnostics() const { return *Diags; }
-  void setDiagnostics(Diagnostic &D) { Diags = &D; }
+  /// \brief Initialize the preprocessor, if the constructor did not already
+  /// perform the initialization.
+  ///
+  /// \param Target Information about the target.
+  void Initialize(const TargetInfo &Target);
+  
+  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
+  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
 
   const LangOptions &getLangOptions() const { return Features; }
-  const TargetInfo &getTargetInfo() const { return Target; }
+  const TargetInfo &getTargetInfo() const { return *Target; }
   FileManager &getFileManager() const { return FileMgr; }
   SourceManager &getSourceManager() const { return SourceMgr; }
   HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
@@ -325,6 +369,9 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
     return ExternalSource;
   }
 
+  /// \brief Retrieve the module loader associated with this preprocessor.
+  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
+  
   /// SetCommentRetentionState - Control whether or not the preprocessor retains
   /// comments in output.
   void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
@@ -334,6 +381,19 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
 
   bool getCommentRetentionState() const { return KeepComments; }
 
+  void SetSuppressIncludeNotFoundError(bool Suppress) {
+    SuppressIncludeNotFoundError = Suppress;
+  }
+
+  bool GetSuppressIncludeNotFoundError() {
+    return SuppressIncludeNotFoundError;
+  }
+
+  /// \brief Specify whether automatic module imports are enabled.
+  void setAutoModuleImport(bool AutoModuleImport = true) {
+    this->AutoModuleImport = AutoModuleImport;
+  }
+  
   /// isCurrentLexer - Return true if we are lexing directly from the specified
   /// lexer.
   bool isCurrentLexer(const PreprocessorLexer *L) const {
@@ -380,12 +440,6 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
   macro_iterator macro_end(bool IncludeExternalMacros = true) const;
 
-  /// \brief Expansion source location for the last macro that expanded
-  /// to no tokens.
-  SourceLocation getLastEmptyMacroExpansionLoc() const {
-    return LastEmptyMacroExpansionLoc;
-  }
-
   const std::string &getPredefines() const { return Predefines; }
   /// setPredefines - Set the predefines for this Preprocessor.  These
   /// predefines are automatically injected when parsing the main file.
@@ -397,25 +451,25 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// pointers is preferred unless the identifier is already available as a
   /// string (this avoids allocation and copying of memory to construct an
   /// std::string).
-  IdentifierInfo *getIdentifierInfo(llvm::StringRef Name) const {
+  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
     return &Identifiers.get(Name);
   }
 
   /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
   /// If 'Namespace' is non-null, then it is a token required to exist on the
   /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
-  void AddPragmaHandler(llvm::StringRef Namespace, PragmaHandler *Handler);
+  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
   void AddPragmaHandler(PragmaHandler *Handler) {
-    AddPragmaHandler(llvm::StringRef(), Handler);
+    AddPragmaHandler(StringRef(), Handler);
   }
 
   /// RemovePragmaHandler - Remove the specific pragma handler from
   /// the preprocessor. If \arg Namespace is non-null, then it should
   /// be the namespace that \arg Handler was added to. It is an error
   /// to remove a handler that has not been registered.
-  void RemovePragmaHandler(llvm::StringRef Namespace, PragmaHandler *Handler);
+  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
   void RemovePragmaHandler(PragmaHandler *Handler) {
-    RemovePragmaHandler(llvm::StringRef(), Handler);
+    RemovePragmaHandler(StringRef(), Handler);
   }
 
   /// \brief Add the specified comment handler to the preprocessor.
@@ -524,16 +578,17 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// Lex - To lex a token from the preprocessor, just pull a token from the
   /// current lexer or macro object.
   void Lex(Token &Result) {
-    if (CurLexer)
-      CurLexer->Lex(Result);
-    else if (CurPTHLexer)
-      CurPTHLexer->Lex(Result);
-    else if (CurTokenLexer)
-      CurTokenLexer->Lex(Result);
-    else
-      CachingLex(Result);
+    switch (CurLexerKind) {
+    case CLK_Lexer: CurLexer->Lex(Result); break;
+    case CLK_PTHLexer: CurPTHLexer->Lex(Result); break;
+    case CLK_TokenLexer: CurTokenLexer->Lex(Result); break;
+    case CLK_CachingLexer: CachingLex(Result); break;
+    case CLK_LexAfterModuleImport: LexAfterModuleImport(Result); break;
+    }
   }
 
+  void LexAfterModuleImport(Token &Result);
+  
   /// LexNonComment - Lex a token.  If it's a comment, keep lexing until we get
   /// something not a comment.  This is useful in -E -C mode where comments
   /// would foul up preprocessor directive handling.
@@ -556,6 +611,14 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
     DisableMacroExpansion = OldVal;
   }
 
+  /// LexUnexpandedNonComment - Like LexNonComment, but this disables macro
+  /// expansion of identifier tokens.
+  void LexUnexpandedNonComment(Token &Result) {
+    do
+      LexUnexpandedToken(Result);
+    while (Result.getKind() == tok::comment);
+  }
+
   /// LookAhead - This peeks ahead N tokens and returns that token without
   /// consuming any tokens.  LookAhead(0) returns the next token that would be
   /// returned by Lex(), LookAhead(1) returns the token after it, etc.  This
@@ -635,13 +698,46 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   bool SetCodeCompletionPoint(const FileEntry *File,
                               unsigned Line, unsigned Column);
 
-  /// \brief Determine if this source location refers into the file
-  /// for which we are performing code completion.
-  bool isCodeCompletionFile(SourceLocation FileLoc) const;
-
   /// \brief Determine if we are performing code completion.
   bool isCodeCompletionEnabled() const { return CodeCompletionFile != 0; }
 
+  /// \brief Returns the location of the code-completion point.
+  /// Returns an invalid location if code-completion is not enabled or the file
+  /// containing the code-completion point has not been lexed yet.
+  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
+
+  /// \brief Returns the start location of the file of code-completion point.
+  /// Returns an invalid location if code-completion is not enabled or the file
+  /// containing the code-completion point has not been lexed yet.
+  SourceLocation getCodeCompletionFileLoc() const {
+    return CodeCompletionFileLoc;
+  }
+
+  /// \brief Returns true if code-completion is enabled and we have hit the
+  /// code-completion point.
+  bool isCodeCompletionReached() const { return CodeCompletionReached; }
+
+  /// \brief Note that we hit the code-completion point.
+  void setCodeCompletionReached() {
+    assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
+    CodeCompletionReached = true;
+    // Silence any diagnostics that occur after we hit the code-completion.
+    getDiagnostics().setSuppressAllDiagnostics(true);
+  }
+
+  /// \brief The location of the currently-active #pragma clang
+  /// arc_cf_code_audited begin.  Returns an invalid location if there
+  /// is no such pragma active.
+  SourceLocation getPragmaARCCFCodeAuditedLoc() const {
+    return PragmaARCCFCodeAuditedLoc;
+  }
+
+  /// \brief Set the location of the currently-active #pragma clang
+  /// arc_cf_code_audited begin.  An invalid location ends the pragma.
+  void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
+    PragmaARCCFCodeAuditedLoc = Loc;
+  }
+
   /// \brief Instruct the preprocessor to skip part of the main
   /// the main source file.
   ///
@@ -657,11 +753,11 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// Diag - Forwarding function for diagnostics.  This emits a diagnostic at
   /// the specified Token's location, translating the token's start
   /// position in the current buffer into a SourcePosition object for rendering.
-  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) {
+  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
     return Diags->Report(Loc, DiagID);
   }
 
-  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) {
+  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
     return Diags->Report(Tok.getLocation(), DiagID);
   }
 
@@ -672,8 +768,8 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// \param buffer A buffer which will be used only if the token requires
   ///   "cleaning", e.g. if it contains trigraphs or escaped newlines
   /// \param invalid If non-null, will be set \c true if an error occurs.
-  llvm::StringRef getSpelling(SourceLocation loc,
-                              llvm::SmallVectorImpl<char> &buffer,
+  StringRef getSpelling(SourceLocation loc,
+                              SmallVectorImpl<char> &buffer,
                               bool *invalid = 0) const {
     return Lexer::getSpelling(loc, buffer, SourceMgr, Features, invalid);
   }
@@ -707,8 +803,8 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// getSpelling - This method is used to get the spelling of a token into a
   /// SmallVector. Note that the returned StringRef may not point to the
   /// supplied buffer if a copy can be avoided.
-  llvm::StringRef getSpelling(const Token &Tok,
-                              llvm::SmallVectorImpl<char> &Buffer, 
+  StringRef getSpelling(const Token &Tok,
+                              SmallVectorImpl<char> &Buffer, 
                               bool *Invalid = 0) const;
 
   /// getSpellingOfSingleCharacterNumericConstant - Tok is a numeric constant
@@ -731,8 +827,9 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// CreateString - Plop the specified string into a scratch buffer and set the
   /// specified token's location and length to it.  If specified, the source
   /// location provides a location of the expansion point of the token.
-  void CreateString(const char *Buf, unsigned Len,
-                    Token &Tok, SourceLocation SourceLoc = SourceLocation());
+  void CreateString(const char *Buf, unsigned Len, Token &Tok,
+                    SourceLocation ExpansionLocStart = SourceLocation(),
+                    SourceLocation ExpansionLocEnd = SourceLocation());
 
   /// \brief Computes the source location just past the end of the
   /// token at this source location.
@@ -892,16 +989,17 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// caller is expected to provide a buffer that is large enough to hold the
   /// spelling of the filename, but is also expected to handle the case when
   /// this method decides to use a different buffer.
-  bool GetIncludeFilenameSpelling(SourceLocation Loc,llvm::StringRef &Filename);
+  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
 
   /// LookupFile - Given a "foo" or <foo> reference, look up the indicated file,
   /// return null on failure.  isAngled indicates whether the file reference is
   /// for system #include's or not (i.e. using <> instead of "").
-  const FileEntry *LookupFile(llvm::StringRef Filename,
+  const FileEntry *LookupFile(StringRef Filename,
                               bool isAngled, const DirectoryLookup *FromDir,
                               const DirectoryLookup *&CurDir,
-                              llvm::SmallVectorImpl<char> *SearchPath,
-                              llvm::SmallVectorImpl<char> *RelativePath);
+                              SmallVectorImpl<char> *SearchPath,
+                              SmallVectorImpl<char> *RelativePath,
+                              StringRef *SuggestedModule);
 
   /// GetCurLookup - The DirectoryLookup structure used to find the current
   /// FileEntry, if CurLexer is non-null and if applicable.  This allows us to
@@ -932,7 +1030,8 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
 private:
 
   void PushIncludeMacroStack() {
-    IncludeMacroStack.push_back(IncludeStackInfo(CurLexer.take(),
+    IncludeMacroStack.push_back(IncludeStackInfo(CurLexerKind,
+                                                 CurLexer.take(),
                                                  CurPTHLexer.take(),
                                                  CurPPLexer,
                                                  CurTokenLexer.take(),
@@ -946,6 +1045,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
     CurPPLexer = IncludeMacroStack.back().ThePPLexer;
     CurTokenLexer.reset(IncludeMacroStack.back().TheTokenLexer);
     CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
+    CurLexerKind = IncludeMacroStack.back().CurLexerKind;
     IncludeMacroStack.pop_back();
   }
 
@@ -976,7 +1076,8 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// already seen one so a #else directive is a duplicate.  When this returns,
   /// the caller can lex the first valid token.
   void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
-                                    bool FoundNonSkipPortion, bool FoundElse);
+                                    bool FoundNonSkipPortion, bool FoundElse,
+                                    SourceLocation ElseLoc = SourceLocation());
 
   /// PTHSkipExcludedConditionalBlock - A fast PTH version of
   ///  SkipExcludedConditionalBlock.
@@ -1006,7 +1107,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// going to lex in the cache and when it finishes the tokens are removed
   /// from the end of the cache.
   Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
-                                  llvm::ArrayRef<Token> tokens);
+                                  ArrayRef<Token> tokens);
   void removeCachedMacroExpandedTokensOfLastLexer();
   friend void TokenLexer::ExpandFunctionArguments();
 
@@ -1081,7 +1182,8 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   void HandleDigitDirective(Token &Tok);
   void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
   void HandleIdentSCCSDirective(Token &Tok);
-
+  void HandleMacroExportDirective(Token &Tok);
+  
   // File inclusion.
   void HandleIncludeDirective(SourceLocation HashLoc,
                               Token &Tok,
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/PreprocessorLexer.h b/contrib/llvm/tools/clang/include/clang/Lex/PreprocessorLexer.h
index 7bf041df974f..e2e30bf87837 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/PreprocessorLexer.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/PreprocessorLexer.h
@@ -30,6 +30,9 @@ class PreprocessorLexer {
   /// The SourceManager FileID corresponding to the file being lexed.
   const FileID FID;
 
+  /// \brief Number of SLocEntries before lexing the file.
+  unsigned InitialNumSLocEntries;
+
   //===--------------------------------------------------------------------===//
   // Context-specific lexing flags set by the preprocessor.
   //===--------------------------------------------------------------------===//
@@ -61,18 +64,16 @@ class PreprocessorLexer {
 
   /// ConditionalStack - Information about the set of #if/#ifdef/#ifndef blocks
   /// we are currently in.
-  llvm::SmallVector<PPConditionalInfo, 4> ConditionalStack;
+  SmallVector<PPConditionalInfo, 4> ConditionalStack;
 
   PreprocessorLexer(const PreprocessorLexer&);          // DO NOT IMPLEMENT
   void operator=(const PreprocessorLexer&); // DO NOT IMPLEMENT
   friend class Preprocessor;
 
-  PreprocessorLexer(Preprocessor *pp, FileID fid)
-    : PP(pp), FID(fid), ParsingPreprocessorDirective(false),
-      ParsingFilename(false), LexingRawMode(false) {}
+  PreprocessorLexer(Preprocessor *pp, FileID fid);
 
   PreprocessorLexer()
-    : PP(0),
+    : PP(0), InitialNumSLocEntries(0),
       ParsingPreprocessorDirective(false),
       ParsingFilename(false),
       LexingRawMode(false) {}
@@ -151,13 +152,18 @@ class PreprocessorLexer {
     return FID;
   }
 
+  /// \brief Number of SLocEntries before lexing the file.
+  unsigned getInitialNumSLocEntries() const {
+    return InitialNumSLocEntries;
+  }
+
   /// getFileEntry - Return the FileEntry corresponding to this FileID.  Like
   /// getFileID(), this only works for lexers with attached preprocessors.
   const FileEntry *getFileEntry() const;
 
   /// \brief Iterator that traverses the current stack of preprocessor
   /// conditional directives (#if/#ifdef/#ifndef).
-  typedef llvm::SmallVectorImpl<PPConditionalInfo>::const_iterator 
+  typedef SmallVectorImpl<PPConditionalInfo>::const_iterator 
     conditional_iterator;
 
   conditional_iterator conditional_begin() const { 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/Token.h b/contrib/llvm/tools/clang/include/clang/Lex/Token.h
index 9cf11d9a64c4..e6dd1607e88b 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/Token.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/Token.h
@@ -96,7 +96,10 @@ class Token {
   /// constant, string, etc.
   bool isLiteral() const {
     return is(tok::numeric_constant) || is(tok::char_constant) ||
-           is(tok::string_literal) || is(tok::wide_string_literal) ||
+           is(tok::wide_char_constant) || is(tok::utf16_char_constant) ||
+           is(tok::utf32_char_constant) || is(tok::string_literal) ||
+           is(tok::wide_string_literal) || is(tok::utf8_string_literal) ||
+           is(tok::utf16_string_literal) || is(tok::utf32_string_literal) ||
            is(tok::angle_string_literal);
   }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/TokenConcatenation.h b/contrib/llvm/tools/clang/include/clang/Lex/TokenConcatenation.h
index 094990a6e317..551300f402c2 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/TokenConcatenation.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/TokenConcatenation.h
@@ -63,12 +63,9 @@ namespace clang {
                      const Token &Tok) const;
 
   private:
-    /// StartsWithL - Return true if the spelling of this token starts with 'L'.
-    bool StartsWithL(const Token &Tok) const;
-
-    /// IsIdentifierL - Return true if the spelling of this token is literally
-    /// 'L'.
-    bool IsIdentifierL(const Token &Tok) const;
+    /// IsIdentifierStringPrefix - Return true if the spelling of the token
+    /// is literally 'L', 'u', 'U', or 'u8'.
+    bool IsIdentifierStringPrefix(const Token &Tok) const;
   };
   } // end clang namespace
 
diff --git a/contrib/llvm/tools/clang/include/clang/Lex/TokenLexer.h b/contrib/llvm/tools/clang/include/clang/Lex/TokenLexer.h
index 45ff8a03442e..1330ad5f3143 100644
--- a/contrib/llvm/tools/clang/include/clang/Lex/TokenLexer.h
+++ b/contrib/llvm/tools/clang/include/clang/Lex/TokenLexer.h
@@ -71,8 +71,10 @@ class TokenLexer {
   /// "source location address space".
   unsigned MacroStartSLocOffset;
 
-  /// \brief FileID/offset of the start of the macro definition.
-  std::pair<FileID, unsigned> MacroDefStartInfo;
+  /// \brief Location of the macro definition.
+  SourceLocation MacroDefStart;
+  /// \brief Length of the macro definition.
+  unsigned MacroDefLength;
 
   /// Lexical information about the expansion point of the macro: the identifier
   /// that the macro expanded from had these properties.
@@ -169,7 +171,15 @@ class TokenLexer {
   /// \brief If \arg loc is a FileID and points inside the current macro
   /// definition, returns the appropriate source location pointing at the
   /// macro expansion source location entry.
-  SourceLocation getMacroExpansionLocation(SourceLocation loc) const;
+  SourceLocation getExpansionLocForMacroDefLoc(SourceLocation loc) const;
+
+  /// \brief Creates SLocEntries and updates the locations of macro argument
+  /// tokens to their new expanded locations.
+  ///
+  /// \param ArgIdSpellLoc the location of the macro argument id inside the
+  /// macro definition.
+  void updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
+                                  Token *begin_tokens, Token *end_tokens);
 };
 
 }  // end namespace clang
diff --git a/contrib/llvm/tools/clang/include/clang/Parse/ParseAST.h b/contrib/llvm/tools/clang/include/clang/Parse/ParseAST.h
index 0d37e21becd3..725387024e1f 100644
--- a/contrib/llvm/tools/clang/include/clang/Parse/ParseAST.h
+++ b/contrib/llvm/tools/clang/include/clang/Parse/ParseAST.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_CLANG_PARSE_PARSEAST_H
 #define LLVM_CLANG_PARSE_PARSEAST_H
 
+#include "clang/Basic/LangOptions.h"
+
 namespace clang {
   class Preprocessor;
   class ASTConsumer;
@@ -27,15 +29,13 @@ namespace clang {
   /// This operation inserts the parsed decls into the translation
   /// unit held by Ctx.
   ///
-  /// \param CompleteTranslationUnit When true, the parsed file is
-  /// considered to be a complete translation unit, and any
-  /// end-of-translation-unit wrapup will be performed.
+  /// \param TUKind The kind of translation unit being parsed.
   ///
   /// \param CompletionConsumer If given, an object to consume code completion
   /// results.
   void ParseAST(Preprocessor &pp, ASTConsumer *C,
                 ASTContext &Ctx, bool PrintStats = false,
-                bool CompleteTranslationUnit = true,
+                TranslationUnitKind TUKind = TU_Complete,
                 CodeCompleteConsumer *CompletionConsumer = 0);
 
   /// \brief Parse the main file known to the preprocessor, producing an 
diff --git a/contrib/llvm/tools/clang/include/clang/Parse/ParseDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Parse/ParseDiagnostic.h
index c50ac92f6e8d..0e76c614152f 100644
--- a/contrib/llvm/tools/clang/include/clang/Parse/ParseDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Parse/ParseDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define PARSESTART
 #include "clang/Basic/DiagnosticParseKinds.inc"
 #undef DIAG
diff --git a/contrib/llvm/tools/clang/include/clang/Parse/Parser.h b/contrib/llvm/tools/clang/include/clang/Parse/Parser.h
index 8f49ddad2cec..0046f88d260d 100644
--- a/contrib/llvm/tools/clang/include/clang/Parse/Parser.h
+++ b/contrib/llvm/tools/clang/include/clang/Parse/Parser.h
@@ -22,6 +22,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallVector.h"
 #include <stack>
 
 namespace clang {
@@ -42,7 +43,7 @@ class PrettyStackTraceParserEntry : public llvm::PrettyStackTraceEntry {
   const Parser &P;
 public:
   PrettyStackTraceParserEntry(const Parser &p) : P(p) {}
-  virtual void print(llvm::raw_ostream &OS) const;
+  virtual void print(raw_ostream &OS) const;
 };
 
 /// PrecedenceLevels - These are precedences for the binary/ternary
@@ -98,7 +99,7 @@ class Parser : public CodeCompletionHandler {
   /// in the file. 
   Sema &Actions;
 
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
 
   /// ScopeCache - Cache scopes to reduce malloc traffic.
   enum { ScopeCacheSize = 16 };
@@ -120,6 +121,9 @@ class Parser : public CodeCompletionHandler {
   IdentifierInfo *Ident_vector;
   IdentifierInfo *Ident_pixel;
 
+  /// Objective-C contextual keywords.
+  mutable IdentifierInfo *Ident_instancetype;
+  
   /// \brief Identifier for "introduced".
   IdentifierInfo *Ident_introduced;
 
@@ -186,19 +190,15 @@ class Parser : public CodeCompletionHandler {
 
   const Token &getCurToken() const { return Tok; }
   Scope *getCurScope() const { return Actions.getCurScope(); }
+    
+  Decl  *getObjCDeclContext() const { return Actions.getObjCDeclContext(); }
   
   // Type forwarding.  All of these are statically 'void*', but they may all be
   // different actual classes based on the actions in place.
-  typedef Expr ExprTy;
-  typedef Stmt StmtTy;
   typedef OpaquePtr<DeclGroupRef> DeclGroupPtrTy;
-  typedef CXXBaseSpecifier BaseTy;
-  typedef CXXCtorInitializer MemInitTy;
-  typedef NestedNameSpecifier CXXScopeTy;
-  typedef TemplateParameterList TemplateParamsTy;
   typedef OpaquePtr<TemplateName> TemplateTy;
 
-  typedef llvm::SmallVector<TemplateParameterList *, 4> TemplateParameterLists;
+  typedef SmallVector<TemplateParameterList *, 4> TemplateParameterLists;
 
   typedef clang::ExprResult        ExprResult;
   typedef clang::StmtResult        StmtResult;
@@ -265,7 +265,10 @@ class Parser : public CodeCompletionHandler {
   ///
   bool isTokenStringLiteral() const {
     return Tok.getKind() == tok::string_literal ||
-           Tok.getKind() == tok::wide_string_literal;
+           Tok.getKind() == tok::wide_string_literal ||
+           Tok.getKind() == tok::utf8_string_literal ||
+           Tok.getKind() == tok::utf16_string_literal ||
+           Tok.getKind() == tok::utf32_string_literal;
   }
 
   /// \brief Returns true if the current token is a '=' or '==' and
@@ -281,11 +284,10 @@ class Parser : public CodeCompletionHandler {
     assert(!isTokenStringLiteral() && !isTokenParen() && !isTokenBracket() &&
            !isTokenBrace() &&
            "Should consume special tokens with Consume*Token");
-    if (Tok.is(tok::code_completion)) {
-      CodeCompletionRecovery();
-      return ConsumeCodeCompletionToken();
-    }
-    
+
+    if (Tok.is(tok::code_completion))
+      return handleUnexpectedCodeCompletionToken();
+
     PrevTokLocation = Tok.getLocation();
     PP.Lex(Tok);
     return PrevTokLocation;
@@ -371,10 +373,20 @@ class Parser : public CodeCompletionHandler {
     return PrevTokLocation;    
   }
   
-  ///\ brief When we are consuming a code-completion token within having 
+  ///\ brief When we are consuming a code-completion token without having
   /// matched specific position in the grammar, provide code-completion results
   /// based on context.
-  void CodeCompletionRecovery();
+  ///
+  /// \returns the source location of the code-completion token.
+  SourceLocation handleUnexpectedCodeCompletionToken();
+
+  /// \brief Abruptly cut off parsing; mainly used when we have reached the
+  /// code-completion point.
+  void cutOffParsing() {
+    PP.setCodeCompletionReached();
+    // Cut off parsing by acting as if we reached the end-of-file.
+    Tok.setKind(tok::eof);
+  }
 
   /// \brief Handle the annotation token produced for #pragma unused(...)
   void HandlePragmaUnused();
@@ -397,6 +409,84 @@ class Parser : public CodeCompletionHandler {
     return PP.LookAhead(0);
   }
 
+  /// \brief Tracks information about the current nesting depth of 
+  /// opening delimiters of each kind.
+  class DelimiterTracker {
+  private:
+    friend class Parser;
+
+    unsigned Paren, Brace, Square, Less, LLLess;
+    unsigned& get(tok::TokenKind t) {
+      switch (t) {
+      default: llvm_unreachable("Unexpected balanced token");
+      case tok::l_brace:  return Brace;
+      case tok::l_paren:  return Paren;
+      case tok::l_square: return Square;
+      case tok::less:  return Less;
+      case tok::lesslessless:  return LLLess;
+      }
+    }
+
+    void push(tok::TokenKind t) {
+      get(t)++;
+    }
+
+    void pop(tok::TokenKind t) {
+      get(t)--;
+    }
+
+    unsigned getDepth(tok::TokenKind t) {
+      return get(t);
+    }
+
+  public:
+    DelimiterTracker() : Paren(0), Brace(0), Square(0), Less(0), LLLess(0) { }
+  };
+
+  /// \brief RAII class that helps handle the parsing of an open/close delimiter
+  /// pair, such as braces { ... } or parentheses ( ... ).
+  class BalancedDelimiterTracker {
+    tok::TokenKind Kind, Close;
+    Parser& P;
+    bool Cleanup;
+    const unsigned MaxDepth;
+    SourceLocation LOpen, LClose;
+  
+    void assignClosingDelimiter() {
+      switch (Kind) {
+      default: llvm_unreachable("Unexpected balanced token");
+      case tok::l_brace:  Close = tok::r_brace; break;
+      case tok::l_paren:  Close = tok::r_paren; break;
+      case tok::l_square: Close = tok::r_square; break;
+      case tok::less:  Close = tok::greater; break;
+      case tok::lesslessless:  Close = tok::greatergreatergreater; break;
+      }
+    }
+
+  public:
+    BalancedDelimiterTracker(Parser& p, tok::TokenKind k) 
+      : Kind(k), P(p), Cleanup(false), MaxDepth(256) {
+      assignClosingDelimiter();
+    }
+
+    ~BalancedDelimiterTracker() {
+      if (Cleanup)
+        P.QuantityTracker.pop(Kind);
+    }
+
+    SourceLocation getOpenLocation() const { return LOpen; }
+    SourceLocation getCloseLocation() const { return LClose; }
+    SourceRange getRange() const { return SourceRange(LOpen, LClose); }
+
+    bool consumeOpen();
+    bool expectAndConsume(unsigned DiagID, 
+                          const char *Msg = "", 
+                          tok::TokenKind SkipToTok = tok::unknown);
+    bool consumeClose();
+  };
+
+  DelimiterTracker QuantityTracker;
+
   /// getTypeAnnotation - Read a parsed type out of an annotation token.
   static ParsedType getTypeAnnotation(Token &Tok) {
     return ParsedType::getFromOpaquePtr(Tok.getAnnotationValue());
@@ -424,7 +514,10 @@ class Parser : public CodeCompletionHandler {
       Tok.setAnnotationValue(ER.get());
   }
 
-  bool TryAnnotateTypeOrScopeToken(bool EnteringContext = false);
+  // If NeedType is true, then TryAnnotateTypeOrScopeToken will try harder to
+  // find a type name by attempting typo correction.
+  bool TryAnnotateTypeOrScopeToken(bool EnteringContext = false,
+                                   bool NeedType = false);
   bool TryAnnotateCXXScopeToken(bool EnteringContext = false);
 
   /// TryAltiVecToken - Check for context-sensitive AltiVec identifier tokens,
@@ -498,9 +591,23 @@ class Parser : public CodeCompletionHandler {
     }
   };
 
-
-  SourceLocation MatchRHSPunctuation(tok::TokenKind RHSTok,
-                                     SourceLocation LHSLoc);
+  /// ObjCDeclContextSwitch - An object used to switch context from
+  /// an objective-c decl context to its enclosing decl context and
+  /// back.
+  class ObjCDeclContextSwitch {
+    Parser &P;
+    Decl *DC;
+  public:
+    explicit ObjCDeclContextSwitch(Parser &p) : P(p), 
+               DC(p.getObjCDeclContext()) {
+      if (DC)
+        P.Actions.ActOnObjCTemporaryExitContainerContext();
+    }
+    ~ObjCDeclContextSwitch() {
+      if (DC)
+        P.Actions.ActOnObjCReenterContainerContext();
+    }
+  };
 
   /// ExpectAndConsume - The parser expects that 'ExpectedTok' is next in the
   /// input.  If so, it is consumed and false is returned.
@@ -630,6 +737,7 @@ class Parser : public CodeCompletionHandler {
     virtual void ParseLexedMethodDeclarations();
     virtual void ParseLexedMemberInitializers();
     virtual void ParseLexedMethodDefs();
+    virtual void ParseLexedAttributes();
   };
 
   /// Inner node of the LateParsedDeclaration tree that parses
@@ -642,12 +750,39 @@ class Parser : public CodeCompletionHandler {
     virtual void ParseLexedMethodDeclarations();
     virtual void ParseLexedMemberInitializers();
     virtual void ParseLexedMethodDefs();
+    virtual void ParseLexedAttributes();
 
   private:
     Parser *Self;
     ParsingClass *Class;
   };
 
+  /// Contains the lexed tokens of an attribute with arguments that 
+  /// may reference member variables and so need to be parsed at the 
+  /// end of the class declaration after parsing all other member 
+  /// member declarations.
+  /// FIXME: Perhaps we should change the name of LateParsedDeclaration to
+  /// LateParsedTokens.
+  struct LateParsedAttribute : public LateParsedDeclaration {
+    Parser *Self;
+    CachedTokens Toks;
+    IdentifierInfo &AttrName;
+    SourceLocation AttrNameLoc;
+    Decl *D;
+
+    explicit LateParsedAttribute(Parser *P, IdentifierInfo &Name, 
+                                 SourceLocation Loc)
+      : Self(P), AttrName(Name), AttrNameLoc(Loc), D(0)  {}
+
+    virtual void ParseLexedAttributes();
+
+    void setDecl(Decl *Dec) { D = Dec; }
+  };
+
+  /// A list of late parsed attributes.  Used by ParseGNUAttributes.
+  typedef llvm::SmallVector<LateParsedAttribute*, 2> LateParsedAttrList;
+
+
   /// Contains the lexed tokens of a member function definition
   /// which needs to be parsed at the end of the class declaration
   /// after parsing all other member declarations.
@@ -711,7 +846,7 @@ class Parser : public CodeCompletionHandler {
     /// have a default argument, but all of the parameters of the
     /// method will be stored so that they can be reintroduced into
     /// scope at the appropriate times.
-    llvm::SmallVector<LateParsedDefaultArgument, 8> DefaultArgs;
+    SmallVector<LateParsedDefaultArgument, 8> DefaultArgs;
   };
 
   /// LateParsedMemberInitializer - An initializer for a non-static class data
@@ -738,7 +873,7 @@ class Parser : public CodeCompletionHandler {
   /// parsed until after the definition is completed (C++ [class.mem]p2),
   /// the method declarations and possibly attached inline definitions
   /// will be stored here with the tokens that will be parsed to create those entities.
-  typedef llvm::SmallVector<LateParsedDeclaration*, 2> LateParsedDeclarationsContainer;
+  typedef SmallVector<LateParsedDeclaration*, 2> LateParsedDeclarationsContainer;
 
   /// \brief Representation of a class that has been parsed, including
   /// any member function declarations or definitions that need to be
@@ -990,16 +1125,21 @@ class Parser : public CodeCompletionHandler {
   void DeallocateParsedClasses(ParsingClass *Class);
   void PopParsingClass(Sema::ParsingClassState);
 
-  Decl *ParseCXXInlineMethodDef(AccessSpecifier AS, ParsingDeclarator &D,
+  Decl *ParseCXXInlineMethodDef(AccessSpecifier AS, AttributeList *AccessAttrs,
+                                ParsingDeclarator &D,
                                 const ParsedTemplateInfo &TemplateInfo,
                                 const VirtSpecifiers& VS, ExprResult& Init);
   void ParseCXXNonStaticMemberInitializer(Decl *VarD);
+  void ParseLexedAttributes(ParsingClass &Class);
+  void ParseLexedAttribute(LateParsedAttribute &LA);
   void ParseLexedMethodDeclarations(ParsingClass &Class);
   void ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM);
   void ParseLexedMethodDefs(ParsingClass &Class);
   void ParseLexedMethodDef(LexedMethod &LM);
   void ParseLexedMemberInitializers(ParsingClass &Class);
   void ParseLexedMemberInitializer(LateParsedMemberInitializer &MI);
+  Decl *ParseLexedObjCMethodDefs(LexedMethod &LM);
+  bool ConsumeAndStoreFunctionPrologue(CachedTokens &Toks);
   bool ConsumeAndStoreUntil(tok::TokenKind T1,
                             CachedTokens &Toks,
                             bool StopAtSemi = true,
@@ -1038,29 +1178,31 @@ class Parser : public CodeCompletionHandler {
   ExprResult ParseAsmStringLiteral();
 
   // Objective-C External Declarations
-  Decl *ParseObjCAtDirectives();
-  Decl *ParseObjCAtClassDeclaration(SourceLocation atLoc);
+  Parser::DeclGroupPtrTy ParseObjCAtDirectives();
+  Parser::DeclGroupPtrTy ParseObjCAtClassDeclaration(SourceLocation atLoc);
   Decl *ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
                                         ParsedAttributes &prefixAttrs);
   void ParseObjCClassInstanceVariables(Decl *interfaceDecl,
                                        tok::ObjCKeywordKind visibility,
                                        SourceLocation atLoc);
-  bool ParseObjCProtocolReferences(llvm::SmallVectorImpl<Decl *> &P,
-                                   llvm::SmallVectorImpl<SourceLocation> &PLocs,
+  bool ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &P,
+                                   SmallVectorImpl<SourceLocation> &PLocs,
                                    bool WarnOnDeclarations,
                                    SourceLocation &LAngleLoc,
                                    SourceLocation &EndProtoLoc);
   bool ParseObjCProtocolQualifiers(DeclSpec &DS);
-  void ParseObjCInterfaceDeclList(Decl *interfaceDecl,
-                                  tok::ObjCKeywordKind contextKey);
+  void ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
+                                  Decl *CDecl);
   Decl *ParseObjCAtProtocolDeclaration(SourceLocation atLoc,
                                        ParsedAttributes &prefixAttrs);
 
   Decl *ObjCImpDecl;
-  llvm::SmallVector<Decl *, 4> PendingObjCImpDecl;
+  SmallVector<Decl *, 4> PendingObjCImpDecl;
+  typedef SmallVector<LexedMethod*, 2> LateParsedObjCMethodContainer;
+  LateParsedObjCMethodContainer LateParsedObjCMethods;
 
   Decl *ParseObjCAtImplementationDeclaration(SourceLocation atLoc);
-  Decl *ParseObjCAtEndDeclaration(SourceRange atEnd);
+  DeclGroupPtrTy ParseObjCAtEndDeclaration(SourceRange atEnd);
   Decl *ParseObjCAtAliasDeclaration(SourceLocation atLoc);
   Decl *ParseObjCPropertySynthesize(SourceLocation atLoc);
   Decl *ParseObjCPropertyDynamic(SourceLocation atLoc);
@@ -1075,22 +1217,16 @@ class Parser : public CodeCompletionHandler {
 
   bool isTokIdentifier_in() const;
 
-  /// \brief The context in which we are parsing an Objective-C type name.
-  enum ObjCTypeNameContext {
-    OTN_ResultType,
-    OTN_ParameterType
-  };
-  
-  ParsedType ParseObjCTypeName(ObjCDeclSpec &DS, ObjCTypeNameContext Context);
+  ParsedType ParseObjCTypeName(ObjCDeclSpec &DS, Declarator::TheContext Ctx,
+                               ParsedAttributes *ParamAttrs);
   void ParseObjCMethodRequirement();
-  Decl *ParseObjCMethodPrototype(Decl *classOrCat,
+  Decl *ParseObjCMethodPrototype(
             tok::ObjCKeywordKind MethodImplKind = tok::objc_not_keyword,
             bool MethodDefinition = true);
   Decl *ParseObjCMethodDecl(SourceLocation mLoc, tok::TokenKind mType,
-                                Decl *classDecl,
             tok::ObjCKeywordKind MethodImplKind = tok::objc_not_keyword,
             bool MethodDefinition=true);
-  void ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl);
+  void ParseObjCPropertyAttribute(ObjCDeclSpec &DS);
 
   Decl *ParseObjCMethodDefinition();
 
@@ -1134,12 +1270,12 @@ class Parser : public CodeCompletionHandler {
                                                      ParsedType &CastTy,
                                                      SourceRange &CastRange);
 
-  typedef llvm::SmallVector<Expr*, 20> ExprListTy;
-  typedef llvm::SmallVector<SourceLocation, 20> CommaLocsTy;
+  typedef SmallVector<Expr*, 20> ExprListTy;
+  typedef SmallVector<SourceLocation, 20> CommaLocsTy;
 
   /// ParseExpressionList - Used for C/C++ (argument-)expression-list.
-  bool ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
-                           llvm::SmallVectorImpl<SourceLocation> &CommaLocs,
+  bool ParseExpressionList(SmallVectorImpl<Expr*> &Exprs,
+                           SmallVectorImpl<SourceLocation> &CommaLocs,
                            void (Sema::*Completer)(Scope *S,
                                                    Expr *Data,
                                                    Expr **Args,
@@ -1160,10 +1296,8 @@ class Parser : public CodeCompletionHandler {
                                         SourceLocation &RParenLoc);
 
   ExprResult ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
-                                                    ParsedType &CastTy,
-                                                    SourceLocation LParenLoc,
-                                                    SourceLocation &RParenLoc);
-
+                                            ParsedType &CastTy,
+                                            BalancedDelimiterTracker &Tracker);
   ExprResult ParseCompoundLiteralExpression(ParsedType Ty,
                                                   SourceLocation LParenLoc,
                                                   SourceLocation RParenLoc);
@@ -1176,12 +1310,27 @@ class Parser : public CodeCompletionHandler {
   // C++ Expressions
   ExprResult ParseCXXIdExpression(bool isAddressOfOperand = false);
 
+  void CheckForTemplateAndDigraph(Token &Next, ParsedType ObjectTypePtr,
+                                  bool EnteringContext, IdentifierInfo &II,
+                                  CXXScopeSpec &SS);
+
   bool ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
                                       ParsedType ObjectType,
                                       bool EnteringContext,
                                       bool *MayBePseudoDestructor = 0,
                                       bool IsTypename = false);
 
+  //===--------------------------------------------------------------------===//
+  // C++0x 5.1.2: Lambda expressions
+
+  // [...] () -> type {...}
+  ExprResult ParseLambdaExpression();
+  ExprResult TryParseLambdaExpression();
+  llvm::Optional<unsigned> ParseLambdaIntroducer(LambdaIntroducer &Intro);
+  bool TryParseLambdaIntroducer(LambdaIntroducer &Intro);
+  ExprResult ParseLambdaExpressionAfterIntroducer(
+               LambdaIntroducer &Intro);
+
   //===--------------------------------------------------------------------===//
   // C++ 5.2p1: C++ Casts
   ExprResult ParseCXXCasts();
@@ -1211,19 +1360,19 @@ class Parser : public CodeCompletionHandler {
 
   ExceptionSpecificationType MaybeParseExceptionSpecification(
                     SourceRange &SpecificationRange,
-                    llvm::SmallVectorImpl<ParsedType> &DynamicExceptions,
-                    llvm::SmallVectorImpl<SourceRange> &DynamicExceptionRanges,
+                    SmallVectorImpl<ParsedType> &DynamicExceptions,
+                    SmallVectorImpl<SourceRange> &DynamicExceptionRanges,
                     ExprResult &NoexceptExpr);
 
   // EndLoc is filled with the location of the last token of the specification.
   ExceptionSpecificationType ParseDynamicExceptionSpecification(
                                   SourceRange &SpecificationRange,
-                                  llvm::SmallVectorImpl<ParsedType> &Exceptions,
-                                  llvm::SmallVectorImpl<SourceRange> &Ranges);
+                                  SmallVectorImpl<ParsedType> &Exceptions,
+                                  SmallVectorImpl<SourceRange> &Ranges);
 
   //===--------------------------------------------------------------------===//
   // C++0x 8: Function declaration trailing-return-type
-  TypeResult ParseTrailingReturnType();
+  TypeResult ParseTrailingReturnType(SourceRange &Range);
 
   //===--------------------------------------------------------------------===//
   // C++ 2.13.5: C++ Boolean Literals
@@ -1244,7 +1393,7 @@ class Parser : public CodeCompletionHandler {
 
   //===--------------------------------------------------------------------===//
   // C++ 5.3.4 and 5.3.5: C++ new and delete
-  bool ParseExpressionListOrTypeId(llvm::SmallVectorImpl<Expr*> &Exprs,
+  bool ParseExpressionListOrTypeId(SmallVectorImpl<Expr*> &Exprs,
                                    Declarator &D);
   void ParseDirectNewDeclarator(Declarator &D);
   ExprResult ParseCXXNewExpression(bool UseGlobal, SourceLocation Start);
@@ -1332,15 +1481,15 @@ class Parser : public CodeCompletionHandler {
   StmtResult ParseBreakStatement(ParsedAttributes &Attr);
   StmtResult ParseReturnStatement(ParsedAttributes &Attr);
   StmtResult ParseAsmStatement(bool &msAsm);
-  StmtResult FuzzyParseMicrosoftAsmStatement(SourceLocation AsmLoc);
+  StmtResult ParseMicrosoftAsmStatement(SourceLocation AsmLoc);
   bool ParseMicrosoftIfExistsCondition(bool& Result);
   void ParseMicrosoftIfExistsStatement(StmtVector &Stmts);
   void ParseMicrosoftIfExistsExternalDeclaration();
   void ParseMicrosoftIfExistsClassDeclaration(DeclSpec::TST TagType,
                                               AccessSpecifier& CurAS);
-bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
-                           llvm::SmallVectorImpl<ExprTy *> &Constraints,
-                           llvm::SmallVectorImpl<ExprTy *> &Exprs);
+  bool ParseAsmOperandsOpt(SmallVectorImpl<IdentifierInfo *> &Names,
+                           SmallVectorImpl<Expr *> &Constraints,
+                           SmallVectorImpl<Expr *> &Exprs);
 
   //===--------------------------------------------------------------------===//
   // C++ 6: Statements and Blocks
@@ -1431,8 +1580,8 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
 
   void ParseSpecifierQualifierList(DeclSpec &DS, AccessSpecifier AS = AS_none);
 
-  void ParseObjCTypeQualifierList(ObjCDeclSpec &DS, 
-                                  ObjCTypeNameContext Context);
+  void ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
+                                  Declarator::TheContext Context);
 
   void ParseEnumSpecifier(SourceLocation TagLoc, DeclSpec &DS,
                 const ParsedTemplateInfo &TemplateInfo = ParsedTemplateInfo(),
@@ -1607,7 +1756,6 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
   TypeResult ParseTypeName(SourceRange *Range = 0,
                            Declarator::TheContext Context
                              = Declarator::TypeNameContext,
-                           ObjCDeclSpec *objcQuals = 0,
                            AccessSpecifier AS = AS_none,
                            Decl **OwnedType = 0);
   void ParseBlockId();
@@ -1618,21 +1766,28 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
   }
   void DiagnoseProhibitedAttributes(ParsedAttributesWithRange &attrs);
 
-  void MaybeParseGNUAttributes(Declarator &D) {
+  void MaybeParseGNUAttributes(Declarator &D,
+                               LateParsedAttrList *LateAttrs = 0) {
     if (Tok.is(tok::kw___attribute)) {
       ParsedAttributes attrs(AttrFactory);
       SourceLocation endLoc;
-      ParseGNUAttributes(attrs, &endLoc);
+      ParseGNUAttributes(attrs, &endLoc, LateAttrs);
       D.takeAttributes(attrs, endLoc);
     }
   }
   void MaybeParseGNUAttributes(ParsedAttributes &attrs,
-                               SourceLocation *endLoc = 0) {
+                               SourceLocation *endLoc = 0,
+                               LateParsedAttrList *LateAttrs = 0) {
     if (Tok.is(tok::kw___attribute))
-      ParseGNUAttributes(attrs, endLoc);
+      ParseGNUAttributes(attrs, endLoc, LateAttrs);
   }
   void ParseGNUAttributes(ParsedAttributes &attrs,
-                          SourceLocation *endLoc = 0);
+                          SourceLocation *endLoc = 0,
+                          LateParsedAttrList *LateAttrs = 0);
+  void ParseGNUAttributeArgs(IdentifierInfo *AttrName,
+                             SourceLocation AttrNameLoc,
+                             ParsedAttributes &Attrs,
+                             SourceLocation *EndLoc);
 
   void MaybeParseCXX0XAttributes(Declarator &D) {
     if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
@@ -1655,12 +1810,15 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
     if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
       ParseCXX0XAttributes(attrs, endLoc);
   }
+
+  void ParseCXX0XAttributeSpecifier(ParsedAttributes &attrs,
+                                    SourceLocation *EndLoc = 0);
   void ParseCXX0XAttributes(ParsedAttributesWithRange &attrs,
                             SourceLocation *EndLoc = 0);
 
   void MaybeParseMicrosoftAttributes(ParsedAttributes &attrs,
                                      SourceLocation *endLoc = 0) {
-    if (getLang().Microsoft && Tok.is(tok::l_square))
+    if (getLang().MicrosoftExt && Tok.is(tok::l_square))
       ParseMicrosoftAttributes(attrs, endLoc);
   }
   void ParseMicrosoftAttributes(ParsedAttributes &attrs,
@@ -1677,11 +1835,21 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
                                   ParsedAttributes &attrs,
                                   SourceLocation *endLoc);
 
+  bool IsThreadSafetyAttribute(llvm::StringRef AttrName);
+  void ParseThreadSafetyAttribute(IdentifierInfo &AttrName,
+                                  SourceLocation AttrNameLoc,
+                                  ParsedAttributes &Attrs,
+                                  SourceLocation *EndLoc);
+
+
   void ParseTypeofSpecifier(DeclSpec &DS);
   void ParseDecltypeSpecifier(DeclSpec &DS);
   void ParseUnderlyingTypeSpecifier(DeclSpec &DS);
-  
-  ExprResult ParseCXX0XAlignArgument(SourceLocation Start);
+  void ParseAtomicSpecifier(DeclSpec &DS);
+
+  ExprResult ParseAlignArgument(SourceLocation Start);
+  void ParseAlignmentSpecifier(ParsedAttributes &Attrs,
+                               SourceLocation *endLoc = 0);
 
   VirtSpecifiers::Specifier isCXX0XVirtSpecifier() const;
   void ParseOptionalCXX0XVirtSpecifierSeq(VirtSpecifiers &VS);
@@ -1732,17 +1900,18 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
                                  bool CXX0XAttributesAllowed = true);
   void ParseDirectDeclarator(Declarator &D);
   void ParseParenDeclarator(Declarator &D);
-  void ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
+  void ParseFunctionDeclarator(Declarator &D,
                                ParsedAttributes &attrs,
+                               BalancedDelimiterTracker &Tracker,
                                bool RequiresArg = false);
   bool isFunctionDeclaratorIdentifierList();
   void ParseFunctionDeclaratorIdentifierList(
          Declarator &D,
-         llvm::SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo);
+         SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo);
   void ParseParameterDeclarationClause(
          Declarator &D,
          ParsedAttributes &attrs,
-         llvm::SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo,
+         SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo,
          SourceLocation &EllipsisLoc);
   void ParseBracketDeclarator(Declarator &D);
 
@@ -1758,8 +1927,8 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
                            std::vector<IdentifierInfo*>& Ident,
                            std::vector<SourceLocation>& NamespaceLoc,
                            unsigned int index, SourceLocation& InlineLoc,
-                           SourceLocation& LBrace, ParsedAttributes& attrs,
-                           SourceLocation& RBraceLoc);
+                           ParsedAttributes& attrs,
+                           BalancedDelimiterTracker &Tracker);
   Decl *ParseLinkage(ParsingDeclSpec &DS, unsigned Context);
   Decl *ParseUsingDirectiveOrDeclaration(unsigned Context,
                                          const ParsedTemplateInfo &TemplateInfo,
@@ -1793,7 +1962,7 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
                                    Decl *TagDecl);
   ExprResult ParseCXXMemberInitializer(bool IsFunction,
                                        SourceLocation &EqualLoc);
-  void ParseCXXClassMemberDeclaration(AccessSpecifier AS,
+  void ParseCXXClassMemberDeclaration(AccessSpecifier AS, AttributeList *Attr,
                 const ParsedTemplateInfo &TemplateInfo = ParsedTemplateInfo(),
                                  ParsingDeclRAIIObject *DiagsFromTParams = 0);
   void ParseConstructorInitializer(Decl *ConstructorDecl);
@@ -1829,30 +1998,33 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
 
   // C++ 14.1: Template Parameters [temp.param]
   Decl *ParseDeclarationStartingWithTemplate(unsigned Context,
-                                                 SourceLocation &DeclEnd,
-                                                 AccessSpecifier AS = AS_none);
+                                             SourceLocation &DeclEnd,
+                                             AccessSpecifier AS = AS_none,
+                                             AttributeList *AccessAttrs = 0);
   Decl *ParseTemplateDeclarationOrSpecialization(unsigned Context,
-                                                     SourceLocation &DeclEnd,
-                                                     AccessSpecifier AS);
+                                                 SourceLocation &DeclEnd,
+                                                 AccessSpecifier AS,
+                                                 AttributeList *AccessAttrs);
   Decl *ParseSingleDeclarationAfterTemplate(
                                        unsigned Context,
                                        const ParsedTemplateInfo &TemplateInfo,
                                        ParsingDeclRAIIObject &DiagsFromParams,
                                        SourceLocation &DeclEnd,
-                                       AccessSpecifier AS=AS_none);
+                                       AccessSpecifier AS=AS_none,
+                                       AttributeList *AccessAttrs = 0);
   bool ParseTemplateParameters(unsigned Depth,
-                               llvm::SmallVectorImpl<Decl*> &TemplateParams,
+                               SmallVectorImpl<Decl*> &TemplateParams,
                                SourceLocation &LAngleLoc,
                                SourceLocation &RAngleLoc);
   bool ParseTemplateParameterList(unsigned Depth,
-                                  llvm::SmallVectorImpl<Decl*> &TemplateParams);
+                                  SmallVectorImpl<Decl*> &TemplateParams);
   bool isStartOfTemplateTypeParameter();
   Decl *ParseTemplateParameter(unsigned Depth, unsigned Position);
   Decl *ParseTypeParameter(unsigned Depth, unsigned Position);
   Decl *ParseTemplateTemplateParameter(unsigned Depth, unsigned Position);
   Decl *ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position);
   // C++ 14.3: Template arguments [temp.arg]
-  typedef llvm::SmallVector<ParsedTemplateArgument, 16> TemplateArgList;
+  typedef SmallVector<ParsedTemplateArgument, 16> TemplateArgList;
 
   bool ParseTemplateIdAfterTemplateName(TemplateTy Template,
                                         SourceLocation TemplateNameLoc,
@@ -1876,6 +2048,10 @@ bool ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
                                         SourceLocation TemplateLoc,
                                         SourceLocation &DeclEnd);
 
+  //===--------------------------------------------------------------------===//
+  // Modules
+  DeclGroupPtrTy ParseModuleImport();
+  
   //===--------------------------------------------------------------------===//
   // GNU G++: Type Traits [Type-Traits.html in the GCC manual]
   ExprResult ParseUnaryTypeTrait();
diff --git a/contrib/llvm/tools/clang/include/clang/Rewrite/ASTConsumers.h b/contrib/llvm/tools/clang/include/clang/Rewrite/ASTConsumers.h
index b7f642764b22..7a636e536955 100644
--- a/contrib/llvm/tools/clang/include/clang/Rewrite/ASTConsumers.h
+++ b/contrib/llvm/tools/clang/include/clang/Rewrite/ASTConsumers.h
@@ -14,29 +14,27 @@
 #ifndef REWRITE_ASTCONSUMERS_H
 #define REWRITE_ASTCONSUMERS_H
 
+#include "clang/Basic/LLVM.h"
 #include <string>
 
-namespace llvm {
-  class raw_ostream;
-}
 namespace clang {
 
 class ASTConsumer;
-class Diagnostic;
+class DiagnosticsEngine;
 class LangOptions;
 class Preprocessor;
 
 // ObjC rewriter: attempts to rewrite ObjC constructs into pure C code.
 // This is considered experimental, and only works with Apple's ObjC runtime.
 ASTConsumer *CreateObjCRewriter(const std::string &InFile,
-                                llvm::raw_ostream *OS,
-                                Diagnostic &Diags,
+                                raw_ostream *OS,
+                                DiagnosticsEngine &Diags,
                                 const LangOptions &LOpts,
                                 bool SilenceRewriteMacroWarning);
 
 /// CreateHTMLPrinter - Create an AST consumer which rewrites source code to
 /// HTML with syntax highlighting suitable for viewing in a web-browser.
-ASTConsumer *CreateHTMLPrinter(llvm::raw_ostream *OS, Preprocessor &PP,
+ASTConsumer *CreateHTMLPrinter(raw_ostream *OS, Preprocessor &PP,
                                bool SyntaxHighlight = true,
                                bool HighlightMacros = true);
 
diff --git a/contrib/llvm/tools/clang/include/clang/Rewrite/FixItRewriter.h b/contrib/llvm/tools/clang/include/clang/Rewrite/FixItRewriter.h
index bab99624ba4a..bf7e7911c977 100644
--- a/contrib/llvm/tools/clang/include/clang/Rewrite/FixItRewriter.h
+++ b/contrib/llvm/tools/clang/include/clang/Rewrite/FixItRewriter.h
@@ -19,8 +19,6 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Rewrite/Rewriter.h"
 
-namespace llvm { class raw_ostream; }
-
 namespace clang {
 
 class SourceManager;
@@ -38,9 +36,9 @@ class FixItOptions {
   bool FixWhatYouCan;
 };
 
-class FixItRewriter : public DiagnosticClient {
+class FixItRewriter : public DiagnosticConsumer {
   /// \brief The diagnostics machinery.
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
 
   /// \brief The rewriter used to perform the various code
   /// modifications.
@@ -48,7 +46,7 @@ class FixItRewriter : public DiagnosticClient {
 
   /// \brief The diagnostic client that performs the actual formatting
   /// of error messages.
-  DiagnosticClient *Client;
+  DiagnosticConsumer *Client;
 
   /// \brief Turn an input path into an output path. NULL implies overwriting
   /// the original.
@@ -61,7 +59,7 @@ class FixItRewriter : public DiagnosticClient {
   typedef Rewriter::buffer_iterator iterator;
 
   /// \brief Initialize a new fix-it rewriter.
-  FixItRewriter(Diagnostic &Diags, SourceManager &SourceMgr,
+  FixItRewriter(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
                 const LangOptions &LangOpts, FixItOptions *FixItOpts);
 
   /// \brief Destroy the fix-it rewriter.
@@ -79,7 +77,7 @@ class FixItRewriter : public DiagnosticClient {
   /// \brief Write a single modified source file.
   ///
   /// \returns true if there was an error, false otherwise.
-  bool WriteFixedFile(FileID ID, llvm::raw_ostream &OS);
+  bool WriteFixedFile(FileID ID, raw_ostream &OS);
 
   /// \brief Write the modified source files.
   ///
@@ -88,17 +86,19 @@ class FixItRewriter : public DiagnosticClient {
 
   /// IncludeInDiagnosticCounts - This method (whose default implementation
   /// returns true) indicates whether the diagnostics handled by this
-  /// DiagnosticClient should be included in the number of diagnostics
-  /// reported by Diagnostic.
+  /// DiagnosticConsumer should be included in the number of diagnostics
+  /// reported by DiagnosticsEngine.
   virtual bool IncludeInDiagnosticCounts() const;
 
   /// HandleDiagnostic - Handle this diagnostic, reporting it to the user or
   /// capturing it to a log as needed.
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info);
 
   /// \brief Emit a diagnostic via the adapted diagnostic client.
   void Diag(SourceLocation Loc, unsigned DiagID);
+  
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const;
 };
 
 }
diff --git a/contrib/llvm/tools/clang/include/clang/Rewrite/FrontendActions.h b/contrib/llvm/tools/clang/include/clang/Rewrite/FrontendActions.h
index 6b33183166f8..f7aeefae7e58 100644
--- a/contrib/llvm/tools/clang/include/clang/Rewrite/FrontendActions.h
+++ b/contrib/llvm/tools/clang/include/clang/Rewrite/FrontendActions.h
@@ -23,7 +23,7 @@ class FixItOptions;
 class HTMLPrintAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class FixItAction : public ASTFrontendAction {
@@ -32,10 +32,10 @@ class FixItAction : public ASTFrontendAction {
   llvm::OwningPtr<FixItOptions> FixItOpts;
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 
   virtual bool BeginSourceFileAction(CompilerInstance &CI,
-                                     llvm::StringRef Filename);
+                                     StringRef Filename);
 
   virtual void EndSourceFileAction();
 
@@ -49,7 +49,7 @@ class FixItAction : public ASTFrontendAction {
 class RewriteObjCAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
 class RewriteMacrosAction : public PreprocessorFrontendAction {
diff --git a/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriter.h b/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriter.h
index 676744ada36c..f1358a0c8534 100644
--- a/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriter.h
+++ b/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriter.h
@@ -23,8 +23,6 @@
 #include <map>
 #include <string>
 
-namespace llvm { class raw_ostream; }
-
 namespace clang {
   class LangOptions;
   class Rewriter;
@@ -54,7 +52,7 @@ class RewriteBuffer {
   iterator end() const { return Buffer.end(); }
   unsigned size() const { return Buffer.size(); }
 
-  llvm::raw_ostream &write(llvm::raw_ostream &) const;
+  raw_ostream &write(raw_ostream &) const;
 
   /// RemoveText - Remove the specified text.
   void RemoveText(unsigned OrigOffset, unsigned Size,
@@ -64,7 +62,7 @@ class RewriteBuffer {
   /// the buffer is specified relative to the original SourceBuffer.  The
   /// text is inserted after the specified location.
   ///
-  void InsertText(unsigned OrigOffset, llvm::StringRef Str,
+  void InsertText(unsigned OrigOffset, StringRef Str,
                   bool InsertAfter = true);
 
 
@@ -72,14 +70,14 @@ class RewriteBuffer {
   /// offset in the buffer is specified relative to the original
   /// SourceBuffer. The text is inserted before the specified location.  This is
   /// method is the same as InsertText with "InsertAfter == false".
-  void InsertTextBefore(unsigned OrigOffset, llvm::StringRef Str) {
+  void InsertTextBefore(unsigned OrigOffset, StringRef Str) {
     InsertText(OrigOffset, Str, false);
   }
 
   /// InsertTextAfter - Insert some text at the specified point, where the
   /// offset in the buffer is specified relative to the original SourceBuffer.
   /// The text is inserted after the specified location.
-  void InsertTextAfter(unsigned OrigOffset, llvm::StringRef Str) {
+  void InsertTextAfter(unsigned OrigOffset, StringRef Str) {
     InsertText(OrigOffset, Str);
   }
 
@@ -87,7 +85,7 @@ class RewriteBuffer {
   /// buffer with a new string.  This is effectively a combined "remove/insert"
   /// operation.
   void ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                   llvm::StringRef NewStr);
+                   StringRef NewStr);
 
 private:  // Methods only usable by Rewriter.
 
@@ -156,8 +154,8 @@ class Rewriter {
     SourceMgr = &SM;
     LangOpts = &LO;
   }
-  SourceManager &getSourceMgr() { return *SourceMgr; }
-  const LangOptions &getLangOpts() { return *LangOpts; }
+  SourceManager &getSourceMgr() const { return *SourceMgr; }
+  const LangOptions &getLangOpts() const { return *LangOpts; }
 
   /// isRewritable - Return true if this location is a raw file location, which
   /// is rewritable.  Locations from macros, etc are not rewritable.
@@ -186,7 +184,7 @@ class Rewriter {
   ///
   /// \param indentNewLines if true new lines in the string are indented
   /// using the indentation of the source line in position \arg Loc.
-  bool InsertText(SourceLocation Loc, llvm::StringRef Str,
+  bool InsertText(SourceLocation Loc, StringRef Str,
                   bool InsertAfter = true, bool indentNewLines = false);
 
   /// InsertTextAfter - Insert the specified string at the specified location in
@@ -194,20 +192,20 @@ class Rewriter {
   ///  the input location was not rewritable, false otherwise.  Text is
   ///  inserted after any other text that has been previously inserted
   ///  at the some point (the default behavior for InsertText).
-  bool InsertTextAfter(SourceLocation Loc, llvm::StringRef Str) {
+  bool InsertTextAfter(SourceLocation Loc, StringRef Str) {
     return InsertText(Loc, Str);
   }
 
   /// \brief Insert the specified string after the token in the
   /// specified location.
-  bool InsertTextAfterToken(SourceLocation Loc, llvm::StringRef Str);
+  bool InsertTextAfterToken(SourceLocation Loc, StringRef Str);
 
   /// InsertText - Insert the specified string at the specified location in the
   /// original buffer.  This method returns true (and does nothing) if the input
   /// location was not rewritable, false otherwise.  Text is
   /// inserted before any other text that has been previously inserted
   /// at the some point.
-  bool InsertTextBefore(SourceLocation Loc, llvm::StringRef Str) {
+  bool InsertTextBefore(SourceLocation Loc, StringRef Str) {
     return InsertText(Loc, Str, false);
   }
 
@@ -230,12 +228,12 @@ class Rewriter {
   /// buffer with a new string.  This is effectively a combined "remove/insert"
   /// operation.
   bool ReplaceText(SourceLocation Start, unsigned OrigLength,
-                   llvm::StringRef NewStr);
+                   StringRef NewStr);
 
   /// ReplaceText - This method replaces a range of characters in the input
   /// buffer with a new string.  This is effectively a combined "remove/insert"
   /// operation.
-  bool ReplaceText(SourceRange range, llvm::StringRef NewStr) {
+  bool ReplaceText(SourceRange range, StringRef NewStr) {
     return ReplaceText(range.getBegin(), getRangeSize(range), NewStr);
   }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriters.h b/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriters.h
index 669cf8c208ec..203b9bc18b35 100644
--- a/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriters.h
+++ b/contrib/llvm/tools/clang/include/clang/Rewrite/Rewriters.h
@@ -14,17 +14,16 @@
 #ifndef LLVM_CLANG_REWRITE_REWRITERS_H
 #define LLVM_CLANG_REWRITE_REWRITERS_H
 
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/raw_ostream.h"
+#include "clang/Basic/LLVM.h"
 
 namespace clang {
 class Preprocessor;
 
 /// RewriteMacrosInInput - Implement -rewrite-macros mode.
-void RewriteMacrosInInput(Preprocessor &PP, llvm::raw_ostream* OS);
+void RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS);
 
 /// DoRewriteTest - A simple test for the TokenRewriter class.
-void DoRewriteTest(Preprocessor &PP, llvm::raw_ostream* OS);
+void DoRewriteTest(Preprocessor &PP, raw_ostream *OS);
 
 }  // end namespace clang
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/AnalysisBasedWarnings.h b/contrib/llvm/tools/clang/include/clang/Sema/AnalysisBasedWarnings.h
index 8e781cd521c2..eeac97332dfb 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/AnalysisBasedWarnings.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/AnalysisBasedWarnings.h
@@ -37,6 +37,7 @@ class AnalysisBasedWarnings {
     // The warnings to run.
     unsigned enableCheckFallThrough : 1;
     unsigned enableCheckUnreachable : 1;
+    unsigned enableThreadSafetyAnalysis : 1;
   public:
     Policy();
     void disableCheckFallThrough() { enableCheckFallThrough = 0; }
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/AttributeList.h b/contrib/llvm/tools/clang/include/clang/Sema/AttributeList.h
index 5d2d6c2ec691..bcacf7aa142b 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/AttributeList.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/AttributeList.h
@@ -56,7 +56,7 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
   IdentifierInfo *AttrName;
   IdentifierInfo *ScopeName;
   IdentifierInfo *ParmName;
-  SourceLocation AttrLoc;
+  SourceRange AttrRange;
   SourceLocation ScopeLoc;
   SourceLocation ParmLoc;
 
@@ -73,6 +73,9 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
   /// True if already diagnosed as invalid.
   mutable unsigned Invalid : 1;
 
+  /// True if this attribute was used as a type attribute.
+  mutable unsigned UsedAsTypeAttr : 1;
+
   /// True if this has the extra information associated with an
   /// availability attribute.
   unsigned IsAvailability : 1;
@@ -114,21 +117,22 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
 
   size_t allocated_size() const;
 
-  AttributeList(IdentifierInfo *attrName, SourceLocation attrLoc,
+  AttributeList(IdentifierInfo *attrName, SourceRange attrRange,
                 IdentifierInfo *scopeName, SourceLocation scopeLoc,
                 IdentifierInfo *parmName, SourceLocation parmLoc,
                 Expr **args, unsigned numArgs,
                 bool declspec, bool cxx0x)
     : AttrName(attrName), ScopeName(scopeName), ParmName(parmName),
-      AttrLoc(attrLoc), ScopeLoc(scopeLoc), ParmLoc(parmLoc),
+      AttrRange(attrRange), ScopeLoc(scopeLoc), ParmLoc(parmLoc),
       NumArgs(numArgs),
       DeclspecAttribute(declspec), CXX0XAttribute(cxx0x), Invalid(false),
-      IsAvailability(false), NextInPosition(0), NextInPool(0) {
+      UsedAsTypeAttr(false), IsAvailability(false), 
+      NextInPosition(0), NextInPool(0) {
     if (numArgs) memcpy(getArgsBuffer(), args, numArgs * sizeof(Expr*));
     AttrKind = getKind(getName());
   }
 
-  AttributeList(IdentifierInfo *attrName, SourceLocation attrLoc,
+  AttributeList(IdentifierInfo *attrName, SourceRange attrRange,
                 IdentifierInfo *scopeName, SourceLocation scopeLoc,
                 IdentifierInfo *parmName, SourceLocation parmLoc,
                 const AvailabilityChange &introduced,
@@ -137,10 +141,10 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
                 SourceLocation unavailable,
                 bool declspec, bool cxx0x)
     : AttrName(attrName), ScopeName(scopeName), ParmName(parmName),
-      AttrLoc(attrLoc), ScopeLoc(scopeLoc), ParmLoc(parmLoc),
+      AttrRange(attrRange), ScopeLoc(scopeLoc), ParmLoc(parmLoc),
       NumArgs(0), DeclspecAttribute(declspec), CXX0XAttribute(cxx0x),
-      Invalid(false), IsAvailability(true), UnavailableLoc(unavailable),
-      NextInPosition(0), NextInPool(0) {
+      Invalid(false), UsedAsTypeAttr(false), IsAvailability(true),
+      UnavailableLoc(unavailable), NextInPosition(0), NextInPool(0) {
     new (&getAvailabilitySlot(IntroducedSlot)) AvailabilityChange(introduced);
     new (&getAvailabilitySlot(DeprecatedSlot)) AvailabilityChange(deprecated);
     new (&getAvailabilitySlot(ObsoletedSlot)) AvailabilityChange(obsoleted);
@@ -152,6 +156,8 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
 
 public:
   enum Kind {             // Please keep this list alphabetized.
+    AT_acquired_after,
+    AT_acquired_before,
     AT_address_space,
     AT_alias,
     AT_aligned,
@@ -164,10 +170,12 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
     AT_blocks,
     AT_carries_dependency,
     AT_cdecl,
+    AT_cf_audited_transfer,     // Clang-specific.
     AT_cf_consumed,             // Clang-specific.
     AT_cf_returns_autoreleased, // Clang-specific.
     AT_cf_returns_not_retained, // Clang-specific.
     AT_cf_returns_retained,     // Clang-specific.
+    AT_cf_unknown_transfer,     // Clang-specific.
     AT_cleanup,
     AT_common,
     AT_const,
@@ -178,18 +186,26 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
     AT_device,
     AT_dllexport,
     AT_dllimport,
+    AT_exclusive_lock_function,
+    AT_exclusive_locks_required,
+    AT_exclusive_trylock_function,
     AT_ext_vector_type,
     AT_fastcall,
     AT_format,
     AT_format_arg,
     AT_global,
     AT_gnu_inline,
+    AT_guarded_by,
+    AT_guarded_var,
     AT_host,
     AT_IBAction,          // Clang-specific.
     AT_IBOutlet,          // Clang-specific.
     AT_IBOutletCollection, // Clang-specific.
     AT_init_priority,
     AT_launch_bounds,
+    AT_lock_returned,
+    AT_lockable,
+    AT_locks_excluded,
     AT_malloc,
     AT_may_alias,
     AT_mode,
@@ -198,12 +214,14 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
     AT_neon_polyvector_type,    // Clang-specific.
     AT_neon_vector_type,        // Clang-specific.
     AT_no_instrument_function,
+    AT_no_thread_safety_analysis,
     AT_nocommon,
     AT_nodebug,
     AT_noinline,
     AT_nonnull,
     AT_noreturn,
     AT_nothrow,
+    AT_ns_bridged,              // Clang-specific.
     AT_ns_consumed,             // Clang-specific.
     AT_ns_consumes_self,        // Clang-specific.
     AT_ns_returns_autoreleased, // Clang-specific.
@@ -215,6 +233,7 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
     AT_objc_method_family,
     AT_objc_ownership,          // Clang-specific.
     AT_objc_precise_lifetime,   // Clang-specific.
+    AT_objc_returns_inner_pointer, // Clang-specific.
     AT_opencl_image_access,     // OpenCL-specific.
     AT_opencl_kernel_function,  // OpenCL-specific.
     AT_overloadable,       // Clang-specific.
@@ -224,16 +243,23 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
     AT_packed,
     AT_pascal,
     AT_pcs,  // ARM specific
+    AT_pt_guarded_by,
+    AT_pt_guarded_var,
     AT_pure,
     AT_regparm,
     AT_reqd_wg_size,
+    AT_scoped_lockable,
     AT_section,
     AT_sentinel,
     AT_shared,
+    AT_shared_lock_function,
+    AT_shared_locks_required,
+    AT_shared_trylock_function,
     AT_stdcall,
     AT_thiscall,
     AT_transparent_union,
     AT_unavailable,
+    AT_unlock_function,
     AT_unused,
     AT_used,
     AT_uuid,
@@ -244,12 +270,14 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
     AT_weak,
     AT_weak_import,
     AT_weakref,
+    AT_returns_twice,
     IgnoredAttribute,
     UnknownAttribute
   };
 
   IdentifierInfo *getName() const { return AttrName; }
-  SourceLocation getLoc() const { return AttrLoc; }
+  SourceLocation getLoc() const { return AttrRange.getBegin(); }
+  SourceRange getRange() const { return AttrRange; }
   
   bool hasScope() const { return ScopeName; }
   IdentifierInfo *getScopeName() const { return ScopeName; }
@@ -264,6 +292,9 @@ class AttributeList { // TODO: This should really be called ParsedAttribute
   bool isInvalid() const { return Invalid; }
   void setInvalid(bool b = true) const { Invalid = b; }
 
+  bool isUsedAsTypeAttr() const { return UsedAsTypeAttr; }
+  void setUsedAsTypeAttr() { UsedAsTypeAttr = true; }
+
   Kind getKind() const { return Kind(AttrKind); }
   static Kind getKind(const IdentifierInfo *Name);
 
@@ -373,7 +404,7 @@ class AttributeFactory {
 
   /// Free lists.  The index is determined by the following formula:
   ///   (size - sizeof(AttributeList)) / sizeof(void*)
-  llvm::SmallVector<AttributeList*, InlineFreeListsCapacity> FreeLists;
+  SmallVector<AttributeList*, InlineFreeListsCapacity> FreeLists;
 
   // The following are the private interface used by AttributePool.
   friend class AttributePool;
@@ -440,21 +471,21 @@ class AttributePool {
     if (Head) Factory.reclaimPool(Head);
   }
 
-  AttributeList *create(IdentifierInfo *attrName, SourceLocation attrLoc,
+  AttributeList *create(IdentifierInfo *attrName, SourceRange attrRange,
                         IdentifierInfo *scopeName, SourceLocation scopeLoc,
                         IdentifierInfo *parmName, SourceLocation parmLoc,
                         Expr **args, unsigned numArgs,
                         bool declspec = false, bool cxx0x = false) {
     void *memory = allocate(sizeof(AttributeList)
                             + numArgs * sizeof(Expr*));
-    return add(new (memory) AttributeList(attrName, attrLoc,
+    return add(new (memory) AttributeList(attrName, attrRange,
                                           scopeName, scopeLoc,
                                           parmName, parmLoc,
                                           args, numArgs,
                                           declspec, cxx0x));
   }
 
-  AttributeList *create(IdentifierInfo *attrName, SourceLocation attrLoc,
+  AttributeList *create(IdentifierInfo *attrName, SourceRange attrRange,
                         IdentifierInfo *scopeName, SourceLocation scopeLoc,
                         IdentifierInfo *parmName, SourceLocation parmLoc,
                         const AvailabilityChange &introduced,
@@ -463,7 +494,7 @@ class AttributePool {
                         SourceLocation unavailable,
                         bool declspec = false, bool cxx0x = false) {
     void *memory = allocate(AttributeFactory::AvailabilityAllocSize);
-    return add(new (memory) AttributeList(attrName, attrLoc,
+    return add(new (memory) AttributeList(attrName, attrRange,
                                           scopeName, scopeLoc,
                                           parmName, parmLoc,
                                           introduced, deprecated, obsoleted,
@@ -566,19 +597,19 @@ class ParsedAttributes {
   AttributeList *&getListRef() { return list; }
 
 
-  AttributeList *addNew(IdentifierInfo *attrName, SourceLocation attrLoc,
+  AttributeList *addNew(IdentifierInfo *attrName, SourceRange attrRange,
                         IdentifierInfo *scopeName, SourceLocation scopeLoc,
                         IdentifierInfo *parmName, SourceLocation parmLoc,
                         Expr **args, unsigned numArgs,
                         bool declspec = false, bool cxx0x = false) {
     AttributeList *attr =
-      pool.create(attrName, attrLoc, scopeName, scopeLoc, parmName, parmLoc,
+      pool.create(attrName, attrRange, scopeName, scopeLoc, parmName, parmLoc,
                   args, numArgs, declspec, cxx0x);
     add(attr);
     return attr;
   }
 
-  AttributeList *addNew(IdentifierInfo *attrName, SourceLocation attrLoc,
+  AttributeList *addNew(IdentifierInfo *attrName, SourceRange attrRange,
                         IdentifierInfo *scopeName, SourceLocation scopeLoc,
                         IdentifierInfo *parmName, SourceLocation parmLoc,
                         const AvailabilityChange &introduced,
@@ -587,7 +618,7 @@ class ParsedAttributes {
                         SourceLocation unavailable,
                         bool declspec = false, bool cxx0x = false) {
     AttributeList *attr =
-      pool.create(attrName, attrLoc, scopeName, scopeLoc, parmName, parmLoc,
+      pool.create(attrName, attrRange, scopeName, scopeLoc, parmName, parmLoc,
                   introduced, deprecated, obsoleted, unavailable,
                   declspec, cxx0x);
     add(attr);
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/CXXFieldCollector.h b/contrib/llvm/tools/clang/include/clang/Sema/CXXFieldCollector.h
index 63c6ee3f74ba..6f3c0b44b148 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/CXXFieldCollector.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/CXXFieldCollector.h
@@ -26,12 +26,12 @@ class CXXFieldCollector {
   /// Fields - Contains all FieldDecls collected during parsing of a C++
   /// class. When a nested class is entered, its fields are appended to the
   /// fields of its parent class, when it is exited its fields are removed.
-  llvm::SmallVector<FieldDecl*, 32> Fields;
+  SmallVector<FieldDecl*, 32> Fields;
 
   /// FieldCount - Each entry represents the number of fields collected during
   /// the parsing of a C++ class. When a nested class is entered, a new field
   /// count is pushed, when it is exited, the field count is popped.
-  llvm::SmallVector<size_t, 4> FieldCount;
+  SmallVector<size_t, 4> FieldCount;
 
   // Example:
   //
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/CodeCompleteConsumer.h b/contrib/llvm/tools/clang/include/clang/Sema/CodeCompleteConsumer.h
index 74b0105b3310..9e2d60d3e015 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/CodeCompleteConsumer.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/CodeCompleteConsumer.h
@@ -21,11 +21,6 @@
 #include "clang-c/Index.h"
 #include <string>
 
-namespace llvm {
-  class raw_ostream;
-  class Twine;
-}
-
 namespace clang {
 
 class Decl;
@@ -136,7 +131,7 @@ QualType getDeclUsageType(ASTContext &C, NamedDecl *ND);
 ///
 /// \param PreferredTypeIsPointer Whether the preferred type for the context
 /// of this macro is a pointer type.
-unsigned getMacroUsagePriority(llvm::StringRef MacroName, 
+unsigned getMacroUsagePriority(StringRef MacroName, 
                                const LangOptions &LangOpts,
                                bool PreferredTypeIsPointer = false);
 
@@ -253,9 +248,9 @@ class CodeCompletionContext {
     CCC_ObjCInstanceMessage,
     /// \brief Code completion where an Objective-C class message is expected. 
     CCC_ObjCClassMessage,
-    /// \brief Code completion where a superclass of an Objective-C class is
+    /// \brief Code completion where the name of an Objective-C class is
     /// expected.
-    CCC_ObjCSuperclass,
+    CCC_ObjCInterfaceName,
     /// \brief Code completion where an Objective-C category name is expected.
     CCC_ObjCCategoryName,
     /// \brief An unknown context, in which we are recovering from a parsing 
@@ -273,14 +268,26 @@ class CodeCompletionContext {
   /// \brief The type of the base object in a member access expression.
   QualType BaseType;
   
+  /// \brief The identifiers for Objective-C selector parts.
+  IdentifierInfo **SelIdents;
+  
+  /// \brief The number of Objective-C selector parts.
+  unsigned NumSelIdents;
+  
 public:
   /// \brief Construct a new code-completion context of the given kind.
-  CodeCompletionContext(enum Kind Kind) : Kind(Kind) { }
+  CodeCompletionContext(enum Kind Kind) : Kind(Kind), SelIdents(NULL), 
+                                          NumSelIdents(0) { }
   
   /// \brief Construct a new code-completion context of the given kind.
-  CodeCompletionContext(enum Kind Kind, QualType T) : Kind(Kind) { 
+  CodeCompletionContext(enum Kind Kind, QualType T,
+                        IdentifierInfo **SelIdents = NULL,
+                        unsigned NumSelIdents = 0) : Kind(Kind),
+                                                     SelIdents(SelIdents),
+                                                    NumSelIdents(NumSelIdents) { 
     if (Kind == CCC_DotMemberAccess || Kind == CCC_ArrowMemberAccess ||
-        Kind == CCC_ObjCPropertyAccess)
+        Kind == CCC_ObjCPropertyAccess || Kind == CCC_ObjCClassMessage ||
+        Kind == CCC_ObjCInstanceMessage)
       BaseType = T;
     else
       PreferredType = T;
@@ -297,6 +304,12 @@ class CodeCompletionContext {
   /// \brief Retrieve the type of the base object in a member-access 
   /// expression.
   QualType getBaseType() const { return BaseType; }
+  
+  /// \brief Retrieve the Objective-C selector identifiers.
+  IdentifierInfo **getSelIdents() const { return SelIdents; }
+  
+  /// \brief Retrieve the number of Objective-C selector identifiers.
+  unsigned getNumSelIdents() const { return NumSelIdents; }
 
   /// \brief Determines whether we want C++ constructors as results within this
   /// context.
@@ -415,19 +428,23 @@ class CodeCompletionString {
   
 private:
   /// \brief The number of chunks stored in this string.
-  unsigned NumChunks;
+  unsigned NumChunks : 16;
   
+  /// \brief The number of annotations for this code-completion result.
+  unsigned NumAnnotations : 16;
+
   /// \brief The priority of this code-completion string.
   unsigned Priority : 30;
   
   /// \brief The availability of this code-completion result.
   unsigned Availability : 2;
-  
+
   CodeCompletionString(const CodeCompletionString &); // DO NOT IMPLEMENT
   CodeCompletionString &operator=(const CodeCompletionString &); // DITTO
   
   CodeCompletionString(const Chunk *Chunks, unsigned NumChunks,
-                       unsigned Priority, CXAvailabilityKind Availability);
+                       unsigned Priority, CXAvailabilityKind Availability,
+                       const char **Annotations, unsigned NumAnnotations);
   ~CodeCompletionString() { }
   
   friend class CodeCompletionBuilder;
@@ -451,8 +468,14 @@ class CodeCompletionString {
   /// \brief Retrieve the priority of this code completion result.
   unsigned getPriority() const { return Priority; }
   
-  /// \brief Reteirve the availability of this code completion result.
+  /// \brief Retrieve the availability of this code completion result.
   unsigned getAvailability() const { return Availability; }
+
+  /// \brief Retrieve the number of annotations for this code completion result.
+  unsigned getAnnotationCount() const;
+
+  /// \brief Retrieve the annotation string specified by \c AnnotationNr.
+  const char *getAnnotation(unsigned AnnotationNr) const;
   
   /// \brief Retrieve a string representation of the code completion string,
   /// which is mainly useful for debugging.
@@ -463,19 +486,19 @@ class CodeCompletionString {
 class CodeCompletionAllocator : public llvm::BumpPtrAllocator { 
 public:
   /// \brief Copy the given string into this allocator.
-  const char *CopyString(llvm::StringRef String);
+  const char *CopyString(StringRef String);
 
   /// \brief Copy the given string into this allocator.
-  const char *CopyString(llvm::Twine String);
+  const char *CopyString(Twine String);
   
   // \brief Copy the given string into this allocator.
   const char *CopyString(const char *String) {
-    return CopyString(llvm::StringRef(String));
+    return CopyString(StringRef(String));
   }
   
   /// \brief Copy the given string into this allocator.
   const char *CopyString(const std::string &String) {
-    return CopyString(llvm::StringRef(String));
+    return CopyString(StringRef(String));
   }
 };
   
@@ -490,7 +513,9 @@ class CodeCompletionBuilder {
   CXAvailabilityKind Availability;
   
   /// \brief The chunks stored in this string.
-  llvm::SmallVector<Chunk, 4> Chunks;
+  SmallVector<Chunk, 4> Chunks;
+
+  SmallVector<const char *, 2> Annotations;
   
 public:
   CodeCompletionBuilder(CodeCompletionAllocator &Allocator) 
@@ -547,6 +572,8 @@ class CodeCompletionBuilder {
   
   /// \brief Add a new chunk.
   void AddChunk(Chunk C) { Chunks.push_back(C); }
+
+  void AddAnnotation(const char *A) { Annotations.push_back(A); }
 };
   
 /// \brief Captures a result of code completion.
@@ -619,14 +646,15 @@ class CodeCompletionResult {
   /// \brief Build a result that refers to a declaration.
   CodeCompletionResult(NamedDecl *Declaration, 
                        NestedNameSpecifier *Qualifier = 0,
-                       bool QualifierIsInformative = false)
+                       bool QualifierIsInformative = false,
+                       bool Accessible = true)
     : Kind(RK_Declaration), Declaration(Declaration), 
       Priority(getPriorityFromDecl(Declaration)), 
       Availability(CXAvailability_Available), StartParameter(0), 
       Hidden(false), QualifierIsInformative(QualifierIsInformative),
       StartsNestedNameSpecifier(false), AllParametersAreInformative(false),
       DeclaringEntity(false), Qualifier(Qualifier) { 
-    computeCursorKindAndAvailability();
+    computeCursorKindAndAvailability(Accessible);
   }
     
   /// \brief Build a result that refers to a keyword or symbol.
@@ -688,7 +716,7 @@ class CodeCompletionResult {
   static unsigned getPriorityFromDecl(NamedDecl *ND);
     
 private:
-  void computeCursorKindAndAvailability();
+  void computeCursorKindAndAvailability(bool Accessible = true);
 };
   
 bool operator<(const CodeCompletionResult &X, const CodeCompletionResult &Y);
@@ -709,7 +737,7 @@ inline bool operator>=(const CodeCompletionResult &X,
 }
 
   
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, 
+raw_ostream &operator<<(raw_ostream &OS, 
                               const CodeCompletionString &CCS);
 
 /// \brief Abstract interface for a consumer of code-completion 
@@ -850,7 +878,7 @@ class CodeCompleteConsumer {
 /// receives in a simple format.
 class PrintingCodeCompleteConsumer : public CodeCompleteConsumer {
   /// \brief The raw output stream.
-  llvm::raw_ostream &OS;
+  raw_ostream &OS;
     
   CodeCompletionAllocator Allocator;
   
@@ -859,7 +887,7 @@ class PrintingCodeCompleteConsumer : public CodeCompleteConsumer {
   /// results to the given raw output stream.
   PrintingCodeCompleteConsumer(bool IncludeMacros, bool IncludeCodePatterns,
                                bool IncludeGlobals,
-                               llvm::raw_ostream &OS)
+                               raw_ostream &OS)
     : CodeCompleteConsumer(IncludeMacros, IncludeCodePatterns, IncludeGlobals,
                            false), OS(OS) {}
   
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/DeclSpec.h b/contrib/llvm/tools/clang/include/clang/Sema/DeclSpec.h
index a66649955c8c..3260a70ebb4f 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/DeclSpec.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/DeclSpec.h
@@ -34,7 +34,7 @@ namespace clang {
   class ASTContext;
   class TypeLoc;
   class LangOptions;
-  class Diagnostic;
+  class DiagnosticsEngine;
   class IdentifierInfo;
   class NamespaceAliasDecl;
   class NamespaceDecl;
@@ -42,6 +42,7 @@ namespace clang {
   class NestedNameSpecifierLoc;
   class ObjCDeclSpec;
   class Preprocessor;
+  class Sema;
   class Declarator;
   struct TemplateIdAnnotation;
 
@@ -242,6 +243,7 @@ class DeclSpec {
   static const TST TST_char16 = clang::TST_char16;
   static const TST TST_char32 = clang::TST_char32;
   static const TST TST_int = clang::TST_int;
+  static const TST TST_half = clang::TST_half;
   static const TST TST_float = clang::TST_float;
   static const TST TST_double = clang::TST_double;
   static const TST TST_bool = clang::TST_bool;
@@ -259,6 +261,7 @@ class DeclSpec {
   static const TST TST_underlyingType = clang::TST_underlyingType;
   static const TST TST_auto = clang::TST_auto;
   static const TST TST_unknown_anytype = clang::TST_unknown_anytype;
+  static const TST TST_atomic = clang::TST_atomic;
   static const TST TST_error = clang::TST_error;
 
   // type-qualifiers
@@ -345,7 +348,7 @@ class DeclSpec {
   SourceRange TypeofParensRange;
   SourceLocation TQ_constLoc, TQ_restrictLoc, TQ_volatileLoc;
   SourceLocation FS_inlineLoc, FS_virtualLoc, FS_explicitLoc;
-  SourceLocation FriendLoc, ConstexprLoc;
+  SourceLocation FriendLoc, ModulePrivateLoc, ConstexprLoc;
 
   WrittenBuiltinSpecs writtenBS;
   void SaveWrittenBuiltinSpecs();
@@ -355,7 +358,7 @@ class DeclSpec {
 
   static bool isTypeRep(TST T) {
     return (T == TST_typename || T == TST_typeofType ||
-            T == TST_underlyingType);
+            T == TST_underlyingType || T == TST_atomic);
   }
   static bool isExprRep(TST T) {
     return (T == TST_typeofExpr || T == TST_decltype);
@@ -381,7 +384,7 @@ class DeclSpec {
       TypeAltiVecPixel(false),
       TypeAltiVecBool(false),
       TypeSpecOwned(false),
-      TypeQualifiers(TSS_unspecified),
+      TypeQualifiers(TQ_unspecified),
       FS_inline_specified(false),
       FS_virtual_specified(false),
       FS_explicit_specified(false),
@@ -537,8 +540,8 @@ class DeclSpec {
   ///
   /// TODO: use a more general approach that still allows these
   /// diagnostics to be ignored when desired.
-  bool SetStorageClassSpec(SCS S, SourceLocation Loc, const char *&PrevSpec,
-                           unsigned &DiagID, const LangOptions &Lang);
+  bool SetStorageClassSpec(Sema &S, SCS SC, SourceLocation Loc,
+                           const char *&PrevSpec, unsigned &DiagID);
   bool SetStorageClassSpecThread(SourceLocation Loc, const char *&PrevSpec,
                                  unsigned &DiagID);
   bool SetTypeSpecWidth(TSW W, SourceLocation Loc, const char *&PrevSpec,
@@ -592,13 +595,17 @@ class DeclSpec {
 
   bool SetFriendSpec(SourceLocation Loc, const char *&PrevSpec,
                      unsigned &DiagID);
-
+  bool setModulePrivateSpec(SourceLocation Loc, const char *&PrevSpec,
+                            unsigned &DiagID);
   bool SetConstexprSpec(SourceLocation Loc, const char *&PrevSpec,
                         unsigned &DiagID);
 
   bool isFriendSpecified() const { return Friend_specified; }
   SourceLocation getFriendSpecLoc() const { return FriendLoc; }
 
+  bool isModulePrivateSpecified() const { return ModulePrivateLoc.isValid(); }
+  SourceLocation getModulePrivateSpecLoc() const { return ModulePrivateLoc; }
+  
   bool isConstexprSpecified() const { return Constexpr_specified; }
   SourceLocation getConstexprSpecLoc() const { return ConstexprLoc; }
 
@@ -657,7 +664,7 @@ class DeclSpec {
   /// Finish - This does final analysis of the declspec, issuing diagnostics for
   /// things like "_Imaginary" (lacking an FP type).  After calling this method,
   /// DeclSpec is guaranteed self-consistent, even if an error occurred.
-  void Finish(Diagnostic &D, Preprocessor &PP);
+  void Finish(DiagnosticsEngine &D, Preprocessor &PP);
 
   const WrittenBuiltinSpecs& getWrittenBuiltinSpecs() const {
     return writtenBS;
@@ -732,6 +739,7 @@ class ObjCDeclSpec {
   const IdentifierInfo *getSetterName() const { return SetterName; }
   IdentifierInfo *getSetterName() { return SetterName; }
   void setSetterName(IdentifierInfo *name) { SetterName = name; }
+
 private:
   // FIXME: These two are unrelated and mutially exclusive. So perhaps
   // we can put them in a union to reflect their mutual exclusiveness
@@ -961,7 +969,7 @@ class UnqualifiedId {
   
 /// CachedTokens - A set of tokens that has been cached for later
 /// parsing.
-typedef llvm::SmallVector<Token, 4> CachedTokens;
+typedef SmallVector<Token, 4> CachedTokens;
 
 /// DeclaratorChunk - One instance of this struct is used for each type in a
 /// declarator that is parsed.
@@ -1222,7 +1230,7 @@ struct DeclaratorChunk {
 
   void destroy() {
     switch (Kind) {
-    default: assert(0 && "Unknown decl type!");
+    default: llvm_unreachable("Unknown decl type!");
     case DeclaratorChunk::Function:      return Fun.destroy();
     case DeclaratorChunk::Pointer:       return Ptr.destroy();
     case DeclaratorChunk::BlockPointer:  return Cls.destroy();
@@ -1364,7 +1372,8 @@ class Declarator {
   enum TheContext {
     FileContext,         // File scope declaration.
     PrototypeContext,    // Within a function prototype.
-    ObjCPrototypeContext,// Within a method prototype.
+    ObjCResultContext,   // An ObjC method result type.
+    ObjCParameterContext,// An ObjC method parameter type.
     KNRTypeListContext,  // K&R type definition list for formals.
     TypeNameContext,     // Abstract declarator for types.
     MemberContext,       // Struct/Union field.
@@ -1395,7 +1404,7 @@ class Declarator {
   /// parsed.  This is pushed from the identifier out, which means that element
   /// #0 will be the most closely bound to the identifier, and
   /// DeclTypeInfo.back() will be the least closely bound.
-  llvm::SmallVector<DeclaratorChunk, 8> DeclTypeInfo;
+  SmallVector<DeclaratorChunk, 8> DeclTypeInfo;
 
   /// InvalidType - Set by Sema::GetTypeForDeclarator().
   bool InvalidType : 1;
@@ -1403,6 +1412,12 @@ class Declarator {
   /// GroupingParens - Set by Parser::ParseParenDeclarator().
   bool GroupingParens : 1;
 
+  /// FunctionDefinition - Is this Declarator for a function or member defintion
+  bool FunctionDefinition : 1;
+
+  // Redeclaration - Is this Declarator is a redeclaration.
+  bool Redeclaration : 1;
+
   /// Attrs - Attributes.
   ParsedAttributes Attrs;
 
@@ -1428,8 +1443,9 @@ class Declarator {
   Declarator(const DeclSpec &ds, TheContext C)
     : DS(ds), Range(ds.getSourceRange()), Context(C),
       InvalidType(DS.getTypeSpecType() == DeclSpec::TST_error),
-      GroupingParens(false), Attrs(ds.getAttributePool().getFactory()),
-      AsmLabel(0), InlineParamsUsed(false), Extension(false) {
+      GroupingParens(false), FunctionDefinition(false), Redeclaration(false),
+      Attrs(ds.getAttributePool().getFactory()), AsmLabel(0),
+      InlineParamsUsed(false), Extension(false) {
   }
 
   ~Declarator() {
@@ -1462,7 +1478,9 @@ class Declarator {
   TheContext getContext() const { return Context; }
 
   bool isPrototypeContext() const {
-    return (Context == PrototypeContext || Context == ObjCPrototypeContext);
+    return (Context == PrototypeContext ||
+            Context == ObjCParameterContext ||
+            Context == ObjCResultContext);
   }
 
   /// getSourceRange - Get the source range that spans this declarator.
@@ -1522,7 +1540,8 @@ class Declarator {
     case AliasDeclContext:
     case AliasTemplateContext:
     case PrototypeContext:
-    case ObjCPrototypeContext:
+    case ObjCParameterContext:
+    case ObjCResultContext:
     case TemplateParamContext:
     case CXXNewContext:
     case CXXCatchContext:
@@ -1555,7 +1574,8 @@ class Declarator {
     case CXXNewContext:
     case AliasDeclContext:
     case AliasTemplateContext:
-    case ObjCPrototypeContext:
+    case ObjCParameterContext:
+    case ObjCResultContext:
     case BlockLiteralContext:
     case TemplateTypeArgContext:
       return false;
@@ -1578,7 +1598,8 @@ class Declarator {
     case MemberContext:
     case ConditionContext:
     case PrototypeContext:
-    case ObjCPrototypeContext:
+    case ObjCParameterContext:
+    case ObjCResultContext:
     case TemplateParamContext:
     case CXXCatchContext:
     case ObjCCatchContext:
@@ -1783,6 +1804,12 @@ class Declarator {
   bool hasEllipsis() const { return EllipsisLoc.isValid(); }
   SourceLocation getEllipsisLoc() const { return EllipsisLoc; }
   void setEllipsisLoc(SourceLocation EL) { EllipsisLoc = EL; }
+
+  void setFunctionDefinition(bool Val) { FunctionDefinition = Val; }
+  bool isFunctionDefinition() const { return FunctionDefinition; }
+
+  void setRedeclaration(bool Val) { Redeclaration = Val; }
+  bool isRedeclaration() const { return Redeclaration; }
 };
 
 /// FieldDeclarator - This little struct is used to capture information about
@@ -1828,6 +1855,53 @@ class VirtSpecifiers {
   SourceLocation LastLocation;
 };
 
+/// LambdaCaptureDefault - The default, if any, capture method for a
+/// lambda expression.
+enum LambdaCaptureDefault {
+  LCD_None,
+  LCD_ByCopy,
+  LCD_ByRef
+};
+
+/// LambdaCaptureKind - The different capture forms in a lambda
+/// introducer: 'this' or a copied or referenced variable.
+enum LambdaCaptureKind {
+  LCK_This,
+  LCK_ByCopy,
+  LCK_ByRef
+};
+
+/// LambdaCapture - An individual capture in a lambda introducer.
+struct LambdaCapture {
+  LambdaCaptureKind Kind;
+  SourceLocation Loc;
+  IdentifierInfo* Id;
+
+  LambdaCapture(LambdaCaptureKind Kind,
+                     SourceLocation Loc,
+                     IdentifierInfo* Id = 0)
+    : Kind(Kind), Loc(Loc), Id(Id)
+  {}
+};
+
+/// LambdaIntroducer - Represents a complete lambda introducer.
+struct LambdaIntroducer {
+  SourceRange Range;
+  LambdaCaptureDefault Default;
+  llvm::SmallVector<LambdaCapture, 4> Captures;
+
+  LambdaIntroducer()
+    : Default(LCD_None) {}
+
+  /// addCapture - Append a capture in a lambda introducer.
+  void addCapture(LambdaCaptureKind Kind,
+                  SourceLocation Loc,
+                  IdentifierInfo* Id = 0) {
+    Captures.push_back(LambdaCapture(Kind, Loc, Id));
+  }
+
+};
+
 } // end namespace clang
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h
index 8ab938226482..dd2603dbc362 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h
@@ -123,7 +123,7 @@ class DelayedDiagnostic {
 
   static DelayedDiagnostic makeDeprecation(SourceLocation Loc,
                                            const NamedDecl *D,
-                                           llvm::StringRef Msg);
+                                           StringRef Msg);
 
   static DelayedDiagnostic makeAccess(SourceLocation Loc,
                                       const AccessedEntity &Entity) {
@@ -163,9 +163,9 @@ class DelayedDiagnostic {
     return DeprecationData.Decl;
   }
 
-  llvm::StringRef getDeprecationMessage() const {
+  StringRef getDeprecationMessage() const {
     assert(Kind == Deprecation && "Not a deprecation diagnostic.");
-    return llvm::StringRef(DeprecationData.Message,
+    return StringRef(DeprecationData.Message,
                            DeprecationData.MessageLen);
   }
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Designator.h b/contrib/llvm/tools/clang/include/clang/Sema/Designator.h
index 6fe7ab24f0a2..fe01f4d60149 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Designator.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Designator.h
@@ -185,7 +185,7 @@ class Designation {
   unsigned InitIndex;
 
   /// Designators - The actual designators for this initializer.
-  llvm::SmallVector<Designator, 2> Designators;
+  SmallVector<Designator, 2> Designators;
 
   Designation(unsigned Idx) : InitIndex(Idx) {}
 public:
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/ExternalSemaSource.h b/contrib/llvm/tools/clang/include/clang/Sema/ExternalSemaSource.h
index 072e1b58d138..7b83625fa332 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/ExternalSemaSource.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/ExternalSemaSource.h
@@ -14,15 +14,30 @@
 #define LLVM_CLANG_SEMA_EXTERNAL_SEMA_SOURCE_H
 
 #include "clang/AST/ExternalASTSource.h"
+#include "clang/Sema/Weak.h"
 #include <utility>
 
 namespace clang {
 
-struct ObjCMethodList;
-class Sema;
-class Scope;
+class CXXConstructorDecl;
+class CXXRecordDecl;
+class DeclaratorDecl;
 class LookupResult;
-
+struct ObjCMethodList;
+class Scope;
+class Sema;
+class TypedefNameDecl;
+class ValueDecl;
+class VarDecl;
+  
+/// \brief A simple structure that captures a vtable use for the purposes of
+/// the \c ExternalSemaSource.
+struct ExternalVTableUse {
+  CXXRecordDecl *Record;
+  SourceLocation Location;
+  bool DefinitionRequired;
+};
+  
 /// \brief An abstract interface that should be implemented by
 /// external AST sources that also provide information for semantic
 /// analysis.
@@ -52,7 +67,7 @@ class ExternalSemaSource : public ExternalASTSource {
   /// \brief Load the set of namespaces that are known to the external source,
   /// which will be used during typo correction.
   virtual void ReadKnownNamespaces(
-                           llvm::SmallVectorImpl<NamespaceDecl *> &Namespaces);
+                           SmallVectorImpl<NamespaceDecl *> &Namespaces);
   
   /// \brief Do last resort, unqualified lookup on a LookupResult that
   /// Sema cannot find.
@@ -64,12 +79,107 @@ class ExternalSemaSource : public ExternalASTSource {
   /// \return true to tell Sema to recover using the LookupResult.
   virtual bool LookupUnqualified(LookupResult &R, Scope *S) { return false; }
 
+  /// \brief Read the set of tentative definitions known to the external Sema
+  /// source.
+  ///
+  /// The external source should append its own tentative definitions to the
+  /// given vector of tentative definitions. Note that this routine may be
+  /// invoked multiple times; the external source should take care not to
+  /// introduce the same declarations repeatedly.
+  virtual void ReadTentativeDefinitions(
+                                  SmallVectorImpl<VarDecl *> &TentativeDefs) {}
+  
+  /// \brief Read the set of unused file-scope declarations known to the
+  /// external Sema source.
+  ///
+  /// The external source should append its own unused, filed-scope to the
+  /// given vector of declarations. Note that this routine may be
+  /// invoked multiple times; the external source should take care not to
+  /// introduce the same declarations repeatedly.
+  virtual void ReadUnusedFileScopedDecls(
+                 SmallVectorImpl<const DeclaratorDecl *> &Decls) {}
+  
+  /// \brief Read the set of delegating constructors known to the
+  /// external Sema source.
+  ///
+  /// The external source should append its own delegating constructors to the
+  /// given vector of declarations. Note that this routine may be
+  /// invoked multiple times; the external source should take care not to
+  /// introduce the same declarations repeatedly.
+  virtual void ReadDelegatingConstructors(
+                 SmallVectorImpl<CXXConstructorDecl *> &Decls) {}
+
+  /// \brief Read the set of ext_vector type declarations known to the
+  /// external Sema source.
+  ///
+  /// The external source should append its own ext_vector type declarations to
+  /// the given vector of declarations. Note that this routine may be
+  /// invoked multiple times; the external source should take care not to
+  /// introduce the same declarations repeatedly.
+  virtual void ReadExtVectorDecls(SmallVectorImpl<TypedefNameDecl *> &Decls) {}
+
+  /// \brief Read the set of dynamic classes known to the external Sema source.
+  ///
+  /// The external source should append its own dynamic classes to
+  /// the given vector of declarations. Note that this routine may be
+  /// invoked multiple times; the external source should take care not to
+  /// introduce the same declarations repeatedly.
+  virtual void ReadDynamicClasses(SmallVectorImpl<CXXRecordDecl *> &Decls) {}
+
+  /// \brief Read the set of locally-scoped external declarations known to the
+  /// external Sema source.
+  ///
+  /// The external source should append its own locally-scoped external
+  /// declarations to the given vector of declarations. Note that this routine 
+  /// may be invoked multiple times; the external source should take care not 
+  /// to introduce the same declarations repeatedly.
+  virtual void ReadLocallyScopedExternalDecls(
+                 SmallVectorImpl<NamedDecl *> &Decls) {}
+
+  /// \brief Read the set of referenced selectors known to the
+  /// external Sema source.
+  ///
+  /// The external source should append its own referenced selectors to the 
+  /// given vector of selectors. Note that this routine 
+  /// may be invoked multiple times; the external source should take care not 
+  /// to introduce the same selectors repeatedly.
+  virtual void ReadReferencedSelectors(
+                 SmallVectorImpl<std::pair<Selector, SourceLocation> > &Sels) {}
+
+  /// \brief Read the set of weak, undeclared identifiers known to the
+  /// external Sema source.
+  ///
+  /// The external source should append its own weak, undeclared identifiers to
+  /// the given vector. Note that this routine may be invoked multiple times; 
+  /// the external source should take care not to introduce the same identifiers
+  /// repeatedly.
+  virtual void ReadWeakUndeclaredIdentifiers(
+                 SmallVectorImpl<std::pair<IdentifierInfo *, WeakInfo> > &WI) {}
+
+  /// \brief Read the set of used vtables known to the external Sema source.
+  ///
+  /// The external source should append its own used vtables to the given
+  /// vector. Note that this routine may be invoked multiple times; the external
+  /// source should take care not to introduce the same vtables repeatedly.
+  virtual void ReadUsedVTables(SmallVectorImpl<ExternalVTableUse> &VTables) {}
+
+  /// \brief Read the set of pending instantiations known to the external
+  /// Sema source.
+  ///
+  /// The external source should append its own pending instantiations to the
+  /// given vector. Note that this routine may be invoked multiple times; the
+  /// external source should take care not to introduce the same instantiations
+  /// repeatedly.
+  virtual void ReadPendingInstantiations(
+                 SmallVectorImpl<std::pair<ValueDecl *, 
+                                           SourceLocation> > &Pending) {}
+
   // isa/cast/dyn_cast support
   static bool classof(const ExternalASTSource *Source) {
     return Source->SemaSource;
   }
   static bool classof(const ExternalSemaSource *) { return true; }
-};
+}; 
 
 } // end namespace clang
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/IdentifierResolver.h b/contrib/llvm/tools/clang/include/clang/Sema/IdentifierResolver.h
index 8d79fc09f292..85d8ad22bfb0 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/IdentifierResolver.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/IdentifierResolver.h
@@ -37,7 +37,7 @@ class IdentifierResolver {
   /// decl with that declaration name is shadowed in some scope.
   class IdDeclInfo {
   public:
-    typedef llvm::SmallVector<NamedDecl*, 2> DeclsTy;
+    typedef SmallVector<NamedDecl*, 2> DeclsTy;
 
     inline DeclsTy::iterator decls_begin() { return Decls.begin(); }
     inline DeclsTy::iterator decls_end() { return Decls.end(); }
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Initialization.h b/contrib/llvm/tools/clang/include/clang/Sema/Initialization.h
index df6138c7f039..e69bebd56cff 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Initialization.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Initialization.h
@@ -22,10 +22,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include <cassert>
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
   
 class CXXBaseSpecifier;
@@ -71,7 +67,10 @@ class InitializedEntity {
     EK_VectorElement,
     /// \brief The entity being initialized is a field of block descriptor for
     /// the copied-in c++ object.
-    EK_BlockElement
+    EK_BlockElement,
+    /// \brief The entity being initialized is the real or imaginary part of a
+    /// complex number.
+    EK_ComplexElement
   };
   
 private:
@@ -115,8 +114,9 @@ class InitializedEntity {
     /// virtual base.
     uintptr_t Base;
 
-    /// \brief When Kind == EK_ArrayElement or EK_VectorElement, the
-    /// index of the array or vector element being initialized. 
+    /// \brief When Kind == EK_ArrayElement, EK_VectorElement, or
+    /// EK_ComplexElement, the index of the array or vector element being
+    /// initialized. 
     unsigned Index;
   };
 
@@ -313,7 +313,8 @@ class InitializedEntity {
   /// \brief If this is already the initializer for an array or vector
   /// element, sets the element index.
   void setElementIndex(unsigned Index) {
-    assert(getKind() == EK_ArrayElement || getKind() == EK_VectorElement);
+    assert(getKind() == EK_ArrayElement || getKind() == EK_VectorElement ||
+           EK_ComplexElement);
     this->Index = Index;
   }
 };
@@ -525,8 +526,10 @@ class InitializationSequence {
     SK_QualificationConversionLValue,
     /// \brief Perform an implicit conversion sequence.
     SK_ConversionSequence,
-    /// \brief Perform list-initialization
+    /// \brief Perform list-initialization without a constructor
     SK_ListInitialization,
+    /// \brief Perform list-initialization with a constructor.
+    SK_ListConstructorCall,
     /// \brief Perform initialization via a constructor.
     SK_ConstructorInitialization,
     /// \brief Zero-initialize the object
@@ -562,20 +565,24 @@ class InitializationSequence {
       /// \brief When Kind == SK_ResolvedOverloadedFunction or Kind ==
       /// SK_UserConversion, the function that the expression should be 
       /// resolved to or the conversion function to call, respectively.
+      /// When Kind == SK_ConstructorInitialization or SK_ListConstruction,
+      /// the constructor to be called.
       ///
-      /// Always a FunctionDecl.
+      /// Always a FunctionDecl, plus a Boolean flag telling if it was
+      /// selected from an overloaded set having size greater than 1.
       /// For conversion decls, the naming class is the source type.
       /// For construct decls, the naming class is the target type.
       struct {
+        bool HadMultipleCandidates;
         FunctionDecl *Function;
         DeclAccessPair FoundDecl;
       } Function;
-      
+
       /// \brief When Kind = SK_ConversionSequence, the implicit conversion
       /// sequence 
       ImplicitConversionSequence *ICS;
     };
-    
+
     void Destroy();
   };
   
@@ -584,7 +591,7 @@ class InitializationSequence {
   enum SequenceKind SequenceKind;
   
   /// \brief Steps taken by this initialization.
-  llvm::SmallVector<Step, 4> Steps;
+  SmallVector<Step, 4> Steps;
   
 public:
   /// \brief Describes why initialization failed.
@@ -633,11 +640,13 @@ class InitializationSequence {
     /// \brief Default-initialization of a 'const' object.
     FK_DefaultInitOfConst,
     /// \brief Initialization of an incomplete type.
-    FK_Incomplete
+    FK_Incomplete,
+    /// \brief List initialization failed at some point.
+    FK_ListInitializationFailed
   };
   
 private:
-  /// \brief The reason why initialization failued.
+  /// \brief The reason why initialization failed.
   FailureKind Failure;
 
   /// \brief The failed result of overload resolution.
@@ -722,7 +731,7 @@ class InitializationSequence {
   /// \brief Determine whether the initialization sequence is invalid.
   bool Failed() const { return SequenceKind == FailedSequence; }
   
-  typedef llvm::SmallVector<Step, 4>::const_iterator step_iterator;
+  typedef SmallVector<Step, 4>::const_iterator step_iterator;
   step_iterator step_begin() const { return Steps.begin(); }
   step_iterator step_end()   const { return Steps.end(); }
 
@@ -736,7 +745,18 @@ class InitializationSequence {
   /// \brief Determine whether this initialization is direct call to a 
   /// constructor.
   bool isConstructorInitialization() const;
-  
+
+  /// \brief Returns whether the last step in this initialization sequence is a
+  /// narrowing conversion, defined by C++0x [dcl.init.list]p7.
+  ///
+  /// If this function returns true, *isInitializerConstant will be set to
+  /// describe whether *Initializer was a constant expression.  If
+  /// *isInitializerConstant is set to true, *ConstantValue will be set to the
+  /// evaluated value of *Initializer.
+  bool endsWithNarrowing(ASTContext &Ctx, const Expr *Initializer,
+                         bool *isInitializerConstant,
+                         APValue *ConstantValue) const;
+
   /// \brief Add a new step in the initialization that resolves the address
   /// of an overloaded function to a specific function declaration.
   ///
@@ -792,7 +812,7 @@ class InitializationSequence {
   void AddConversionSequenceStep(const ImplicitConversionSequence &ICS,
                                  QualType T);
 
-  /// \brief Add a list-initialiation step  
+  /// \brief Add a list-initialiation step.
   void AddListInitializationStep(QualType T);
 
   /// \brief Add a constructor-initialization step.
@@ -857,7 +877,7 @@ class InitializationSequence {
 
   /// \brief Dump a representation of this initialization sequence to 
   /// the given stream, for debugging purposes.
-  void dump(llvm::raw_ostream &OS) const;
+  void dump(raw_ostream &OS) const;
   
   /// \brief Dump a representation of this initialization sequence to 
   /// standard error, for debugging purposes.
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Lookup.h b/contrib/llvm/tools/clang/include/clang/Sema/Lookup.h
index dceed4efc979..6630bb2981e2 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Lookup.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Lookup.h
@@ -268,7 +268,19 @@ class LookupResult {
 
   /// \brief Tests whether the given declaration is acceptable.
   bool isAcceptableDecl(NamedDecl *D) const {
-    return D->isInIdentifierNamespace(IDNS);
+    if (!D->isInIdentifierNamespace(IDNS))
+      return false;
+    
+    // So long as this declaration is not module-private or was parsed as
+    // part of this translation unit (i.e., in the module), we're allowed to
+    // find it.
+    if (!D->isModulePrivate() || !D->isFromASTFile())
+      return true;
+
+    // FIXME: We should be allowed to refer to a module-private name from 
+    // within the same module, e.g., during template instantiation.
+    // This requires us know which module a particular declaration came from.
+    return false;
   }
 
   /// \brief Returns the identifier namespace mask for this lookup.
@@ -282,18 +294,6 @@ class LookupResult {
     return NamingClass != 0;
   }
 
-  /// \brief Set whether the name lookup is triggered by a 
-  /// using declaration.
-  void setUsingDeclaration(bool U) {
-    UsingDeclaration = U;
-  }
-
-  /// \brief Returns whether the name lookup is triggered by a 
-  /// using declaration.
-  bool isUsingDeclaration() const {
-    return UsingDeclaration;
-  }
-
   /// \brief Returns the 'naming class' for this lookup, i.e. the
   /// class which was looked into to find these results.
   ///
@@ -468,7 +468,7 @@ class LookupResult {
     configure();
   }
 
-  void print(llvm::raw_ostream &);
+  void print(raw_ostream &);
 
   /// Suppress the diagnostics that would normally fire because of this
   /// lookup.  This happens during (e.g.) redeclaration lookups.
@@ -615,10 +615,6 @@ class LookupResult {
   bool HideTags;
 
   bool Diagnose;
-
-  /// \brief True if the lookup is triggered by a using declaration.
-  /// Necessary to handle a MSVC bug.
-  bool UsingDeclaration;
 };
 
   /// \brief Consumes visible declarations found when searching for
@@ -640,9 +636,11 @@ class LookupResult {
     /// \param Hiding a declaration that hides the declaration \p ND,
     /// or NULL if no such declaration exists.
     ///
+    /// \param Ctx the original context from which the lookup started.
+    ///
     /// \param InBaseClass whether this declaration was found in base
     /// class of the context we searched.
-    virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, 
+    virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, DeclContext *Ctx,
                            bool InBaseClass) = 0;
   };
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/MultiInitializer.h b/contrib/llvm/tools/clang/include/clang/Sema/MultiInitializer.h
new file mode 100644
index 000000000000..c44e3934db68
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Sema/MultiInitializer.h
@@ -0,0 +1,72 @@
+//===--- MultiInitializer.h - Initializer expression group ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the MultiInitializer class, which can represent a list
+// initializer or a parentheses-wrapped group of expressions in a C++ member
+// initializer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_MULTIINITIALIZER_H
+#define LLVM_CLANG_SEMA_MULTIINITIALIZER_H
+
+#include "clang/Sema/Ownership.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/PointerUnion.h"
+
+namespace clang {
+  class ASTContext;
+  class Expr;
+  class InitializationKind;
+  class InitializedEntity;
+  class InitListExpr;
+  class Sema;
+
+class MultiInitializer {
+  llvm::PointerUnion<Expr*, Expr**> InitListOrExpressions;
+  unsigned NumInitializers;
+  SourceLocation LParenLoc;
+  SourceLocation RParenLoc;
+
+  InitListExpr *getInitList() const;
+  Expr **getExpressions() const { return InitListOrExpressions.get<Expr**>(); }
+
+public:
+  MultiInitializer(Expr* InitList)
+    : InitListOrExpressions(InitList)
+  {}
+
+  MultiInitializer(SourceLocation LParenLoc, Expr **Exprs, unsigned NumInits,
+                   SourceLocation RParenLoc)
+    : InitListOrExpressions(Exprs), NumInitializers(NumInits),
+    LParenLoc(LParenLoc), RParenLoc(RParenLoc)
+  {}
+
+  bool isInitializerList() const { return InitListOrExpressions.is<Expr*>(); }
+
+  SourceLocation getStartLoc() const;
+  SourceLocation getEndLoc() const;
+
+  typedef Expr **iterator;
+  iterator begin() const;
+  iterator end() const;
+
+  bool isTypeDependent() const;
+
+  bool DiagnoseUnexpandedParameterPack(Sema &SemaRef) const;
+
+  // Return the InitListExpr or create a ParenListExpr.
+  Expr *CreateInitExpr(ASTContext &Ctx, QualType T) const;
+
+  ExprResult PerformInit(Sema &SemaRef, InitializedEntity Entity,
+                         InitializationKind Kind) const;
+};
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Overload.h b/contrib/llvm/tools/clang/include/clang/Sema/Overload.h
index 32d4cbdac15e..dbc0926f03af 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Overload.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Overload.h
@@ -21,6 +21,7 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/UnresolvedSet.h"
+#include "clang/Sema/SemaFixItUtils.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -243,7 +244,12 @@ namespace clang {
     // a gcc code gen. bug which causes a crash in a test. Putting it here seems
     // to work around the crash.
     bool EllipsisConversion : 1;
-    
+
+    /// HadMultipleCandidates - When this is true, it means that the
+    /// conversion function was resolved from an overloaded set having
+    /// size greater than 1.
+    bool HadMultipleCandidates : 1;
+
     /// After - Represents the standard conversion that occurs after
     /// the actual user-defined conversion.
     StandardConversionSequence After;
@@ -255,14 +261,14 @@ namespace clang {
     /// \brief The declaration that we found via name lookup, which might be
     /// the same as \c ConversionFunction or it might be a using declaration
     /// that refers to \c ConversionFunction.
-    NamedDecl *FoundConversionFunction;
+    DeclAccessPair FoundConversionFunction;
     
     void DebugPrint() const;
   };
 
   /// Represents an ambiguous user-defined conversion sequence.
   struct AmbiguousConversionSequence {
-    typedef llvm::SmallVector<FunctionDecl*, 4> ConversionSet;
+    typedef SmallVector<FunctionDecl*, 4> ConversionSet;
 
     void *FromTypePtr;
     void *ToTypePtr;
@@ -463,6 +469,7 @@ namespace clang {
     bool isEllipsis() const { return getKind() == EllipsisConversion; }
     bool isAmbiguous() const { return getKind() == AmbiguousConversion; }
     bool isUserDefined() const { return getKind() == UserDefinedConversion; }
+    bool isFailure() const { return isBad() || isAmbiguous(); }
 
     /// Determines whether this conversion sequence has been
     /// initialized.  Most operations should never need to query
@@ -525,7 +532,12 @@ namespace clang {
     
     /// This conversion function template specialization candidate is not 
     /// viable because the final conversion was not an exact match.
-    ovl_fail_final_conversion_not_exact
+    ovl_fail_final_conversion_not_exact,
+
+    /// (CUDA) This candidate was not viable because the callee
+    /// was not accessible from the caller's target (i.e. host->device,
+    /// global->host, device->host).
+    ovl_fail_bad_target
   };
 
   /// OverloadCandidate - A single candidate in an overload set (C++ 13.3).
@@ -554,7 +566,10 @@ namespace clang {
 
     /// Conversions - The conversion sequences used to convert the
     /// function arguments to the function parameters.
-    llvm::SmallVector<ImplicitConversionSequence, 4> Conversions;
+    SmallVector<ImplicitConversionSequence, 4> Conversions;
+
+    /// The FixIt hints which can be used to fix the Bad candidate.
+    ConversionFixItGenerator Fix;
 
     /// Viable - True to indicate that this overload candidate is viable.
     bool Viable;
@@ -624,19 +639,32 @@ namespace clang {
     /// hasAmbiguousConversion - Returns whether this overload
     /// candidate requires an ambiguous conversion or not.
     bool hasAmbiguousConversion() const {
-      for (llvm::SmallVectorImpl<ImplicitConversionSequence>::const_iterator
+      for (SmallVectorImpl<ImplicitConversionSequence>::const_iterator
              I = Conversions.begin(), E = Conversions.end(); I != E; ++I) {
         if (!I->isInitialized()) return false;
         if (I->isAmbiguous()) return true;
       }
       return false;
     }
+
+    bool TryToFixBadConversion(unsigned Idx, Sema &S) {
+      bool CanFix = Fix.tryToFixConversion(
+                      Conversions[Idx].Bad.FromExpr,
+                      Conversions[Idx].Bad.getFromType(),
+                      Conversions[Idx].Bad.getToType(), S);
+
+      // If at least one conversion fails, the candidate cannot be fixed.
+      if (!CanFix)
+        Fix.clear();
+
+      return CanFix;
+    }
   };
 
   /// OverloadCandidateSet - A set of overload candidates, used in C++
   /// overload resolution (C++ 13.3).
-  class OverloadCandidateSet : public llvm::SmallVector<OverloadCandidate, 16> {
-    typedef llvm::SmallVector<OverloadCandidate, 16> inherited;
+  class OverloadCandidateSet : public SmallVector<OverloadCandidate, 16> {
+    typedef SmallVector<OverloadCandidate, 16> inherited;
     llvm::SmallPtrSet<Decl *, 16> Functions;
 
     SourceLocation Loc;    
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Ownership.h b/contrib/llvm/tools/clang/include/clang/Sema/Ownership.h
index cef93fe5832f..fb9e368dde9f 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Ownership.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Ownership.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_SEMA_OWNERSHIP_H
 #define LLVM_CLANG_SEMA_OWNERSHIP_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 
@@ -368,7 +369,7 @@ namespace clang {
 
   /// \brief A small vector that owns a set of AST nodes.
   template <class PtrTy, unsigned N = 8>
-  class ASTOwningVector : public llvm::SmallVector<PtrTy, N> {
+  class ASTOwningVector : public SmallVector<PtrTy, N> {
     ASTOwningVector(ASTOwningVector &); // do not implement
     ASTOwningVector &operator=(ASTOwningVector &); // do not implement
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/ParsedTemplate.h b/contrib/llvm/tools/clang/include/clang/Sema/ParsedTemplate.h
index 1f572e5df4f3..735a26bd7bc4 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/ParsedTemplate.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/ParsedTemplate.h
@@ -114,7 +114,7 @@ namespace clang {
     KindType Kind;
     
     /// \brief The actual template argument representation, which may be
-    /// an \c ActionBase::TypeTy* (for a type), an ActionBase::ExprTy* (for an
+    /// an \c ActionBase::TypeTy* (for a type), an Expr* (for an
     /// expression), or an ActionBase::TemplateTy (for a template).
     void *Arg;
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/PrettyDeclStackTrace.h b/contrib/llvm/tools/clang/include/clang/Sema/PrettyDeclStackTrace.h
index b78a1c01e5e3..a31312cdc6dc 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/PrettyDeclStackTrace.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/PrettyDeclStackTrace.h
@@ -38,7 +38,7 @@ class PrettyDeclStackTraceEntry : public llvm::PrettyStackTraceEntry {
   PrettyDeclStackTraceEntry(Sema &S, Decl *D, SourceLocation Loc, const char *Msg)
     : S(S), TheDecl(D), Loc(Loc), Message(Msg) {}
 
-  virtual void print(llvm::raw_ostream &OS) const;
+  virtual void print(raw_ostream &OS) const;
 };
 
 }
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Scope.h b/contrib/llvm/tools/clang/include/clang/Sema/Scope.h
index 95d29781e704..cff8b3338d82 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Scope.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Scope.h
@@ -149,14 +149,14 @@ class Scope {
   /// maintained by the Action implementation.
   void *Entity;
 
-  typedef llvm::SmallVector<UsingDirectiveDecl *, 2> UsingDirectivesTy;
+  typedef SmallVector<UsingDirectiveDecl *, 2> UsingDirectivesTy;
   UsingDirectivesTy UsingDirectives;
 
   /// \brief Used to determine if errors occurred in this scope.
   DiagnosticErrorTrap ErrorTrap;
   
 public:
-  Scope(Scope *Parent, unsigned ScopeFlags, Diagnostic &Diag)
+  Scope(Scope *Parent, unsigned ScopeFlags, DiagnosticsEngine &Diag)
     : ErrorTrap(Diag) {
     Init(Parent, ScopeFlags);
   }
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/ScopeInfo.h b/contrib/llvm/tools/clang/include/clang/Sema/ScopeInfo.h
index 51297ae40205..9ef6d3c4b3d7 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/ScopeInfo.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/ScopeInfo.h
@@ -66,17 +66,17 @@ class FunctionScopeInfo {
 
   /// SwitchStack - This is the current set of active switch statements in the
   /// block.
-  llvm::SmallVector<SwitchStmt*, 8> SwitchStack;
+  SmallVector<SwitchStmt*, 8> SwitchStack;
 
   /// \brief The list of return statements that occur within the function or
   /// block, if there is any chance of applying the named return value
   /// optimization.
-  llvm::SmallVector<ReturnStmt*, 4> Returns;
+  SmallVector<ReturnStmt*, 4> Returns;
   
   /// \brief A list of PartialDiagnostics created but delayed within the
   /// current function scope.  These diagnostics are vetted for reachability
   /// prior to being emitted.
-  llvm::SmallVector<PossiblyUnreachableDiag, 4> PossiblyUnreachableDiags;
+  SmallVector<PossiblyUnreachableDiag, 4> PossiblyUnreachableDiags;
 
   void setHasBranchIntoScope() {
     HasBranchIntoScope = true;
@@ -95,7 +95,7 @@ class FunctionScopeInfo {
           (HasBranchProtectedScope && HasBranchIntoScope);
   }
   
-  FunctionScopeInfo(Diagnostic &Diag)
+  FunctionScopeInfo(DiagnosticsEngine &Diag)
     : IsBlockInfo(false),
       HasBranchProtectedScope(false),
       HasBranchIntoScope(false),
@@ -132,12 +132,12 @@ class BlockScopeInfo : public FunctionScopeInfo {
   llvm::DenseMap<VarDecl*, unsigned> CaptureMap;
 
   /// Captures - The captured variables.
-  llvm::SmallVector<BlockDecl::Capture, 4> Captures;
+  SmallVector<BlockDecl::Capture, 4> Captures;
 
   /// CapturesCXXThis - Whether this block captures 'this'.
   bool CapturesCXXThis;
 
-  BlockScopeInfo(Diagnostic &Diag, Scope *BlockScope, BlockDecl *Block)
+  BlockScopeInfo(DiagnosticsEngine &Diag, Scope *BlockScope, BlockDecl *Block)
     : FunctionScopeInfo(Diag), TheDecl(Block), TheScope(BlockScope),
       CapturesCXXThis(false)
   {
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Sema.h b/contrib/llvm/tools/clang/include/clang/Sema/Sema.h
index 263c1bd49ac9..22d5db294440 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Sema.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Sema.h
@@ -20,8 +20,11 @@
 #include "clang/Sema/IdentifierResolver.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ExternalSemaSource.h"
 #include "clang/Sema/LocInfoType.h"
+#include "clang/Sema/MultiInitializer.h"
 #include "clang/Sema/TypoCorrection.h"
+#include "clang/Sema/Weak.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/ExternalASTSource.h"
@@ -30,6 +33,7 @@
 #include "clang/Basic/TemplateKinds.h"
 #include "clang/Basic/TypeTraits.h"
 #include "clang/Basic/ExpressionTraits.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
@@ -54,7 +58,7 @@ namespace clang {
   class BlockDecl;
   class CXXBasePath;
   class CXXBasePaths;
-  typedef llvm::SmallVector<CXXBaseSpecifier*, 4> CXXCastPath;
+  typedef SmallVector<CXXBaseSpecifier*, 4> CXXCastPath;
   class CXXConstructorDecl;
   class CXXConversionDecl;
   class CXXDestructorDecl;
@@ -175,13 +179,6 @@ class Sema {
   typedef OpaquePtr<DeclGroupRef> DeclGroupPtrTy;
   typedef OpaquePtr<TemplateName> TemplateTy;
   typedef OpaquePtr<QualType> TypeTy;
-  typedef Attr AttrTy;
-  typedef CXXBaseSpecifier BaseTy;
-  typedef CXXCtorInitializer MemInitTy;
-  typedef Expr ExprTy;
-  typedef Stmt StmtTy;
-  typedef TemplateParameterList TemplateParamsTy;
-  typedef NestedNameSpecifier CXXScopeTy;
 
   OpenCLOptions OpenCLFeatures;
   FPOptions FPFeatures;
@@ -190,7 +187,7 @@ class Sema {
   Preprocessor &PP;
   ASTContext &Context;
   ASTConsumer &Consumer;
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
   SourceManager &SourceMgr;
 
   /// \brief Flag indicating whether or not to collect detailed statistics.
@@ -205,6 +202,10 @@ class Sema {
   /// CurContext - This is the current declaration context of parsing.
   DeclContext *CurContext;
 
+  /// \brief Generally null except when we temporarily switch decl contexts,
+  /// like in \see ActOnObjCTemporaryExitContainerContext.
+  DeclContext *OriginalLexicalContext;
+
   /// VAListTagName - The declaration name corresponding to __va_list_tag.
   /// This is used as part of a hack to omit that class from ADL results.
   DeclarationName VAListTagName;
@@ -228,16 +229,20 @@ class Sema {
   /// This array is never empty.  Clients should ignore the first
   /// element, which is used to cache a single FunctionScopeInfo
   /// that's used to parse every top-level function.
-  llvm::SmallVector<sema::FunctionScopeInfo *, 4> FunctionScopes;
+  SmallVector<sema::FunctionScopeInfo *, 4> FunctionScopes;
 
   /// ExprTemporaries - This is the stack of temporaries that are created by
   /// the current full expression.
-  llvm::SmallVector<CXXTemporary*, 8> ExprTemporaries;
+  SmallVector<CXXTemporary*, 8> ExprTemporaries;
+
+  typedef LazyVector<TypedefNameDecl *, ExternalSemaSource, 
+                     &ExternalSemaSource::ReadExtVectorDecls, 2, 2>
+    ExtVectorDeclsType;
 
   /// ExtVectorDecls - This is a list all the extended vector types. This allows
   /// us to associate a raw vector type with one of the ext_vector type names.
   /// This is only necessary for issuing pretty diagnostics.
-  llvm::SmallVector<TypedefNameDecl*, 24> ExtVectorDecls;
+  ExtVectorDeclsType ExtVectorDecls;
 
   /// FieldCollector - Collects CXXFieldDecls during parsing of C++ classes.
   llvm::OwningPtr<CXXFieldCollector> FieldCollector;
@@ -279,22 +284,38 @@ class Sema {
   ///     we find this declaration of "foo" and complain that it is
   ///     not visible.
   llvm::DenseMap<DeclarationName, NamedDecl *> LocallyScopedExternalDecls;
+  
+  /// \brief Look for a locally scoped external declaration by the given name.
+  llvm::DenseMap<DeclarationName, NamedDecl *>::iterator
+  findLocallyScopedExternalDecl(DeclarationName Name);
+  
+  typedef LazyVector<VarDecl *, ExternalSemaSource, 
+                     &ExternalSemaSource::ReadTentativeDefinitions, 2, 2>
+    TentativeDefinitionsType;
 
   /// \brief All the tentative definitions encountered in the TU.
-  llvm::SmallVector<VarDecl *, 2> TentativeDefinitions;
+  TentativeDefinitionsType TentativeDefinitions;
 
+  typedef LazyVector<const DeclaratorDecl *, ExternalSemaSource, 
+                     &ExternalSemaSource::ReadUnusedFileScopedDecls, 2, 2>
+    UnusedFileScopedDeclsType;
+  
   /// \brief The set of file scoped decls seen so far that have not been used
   /// and must warn if not used. Only contains the first declaration.
-  llvm::SmallVector<const DeclaratorDecl*, 4> UnusedFileScopedDecls;
+  UnusedFileScopedDeclsType UnusedFileScopedDecls;
+
+  typedef LazyVector<CXXConstructorDecl *, ExternalSemaSource, 
+                     &ExternalSemaSource::ReadDelegatingConstructors, 2, 2>
+    DelegatingCtorDeclsType;
 
   /// \brief All the delegating constructors seen so far in the file, used for
   /// cycle detection at the end of the TU.
-  llvm::SmallVector<CXXConstructorDecl*, 4> DelegatingCtorDecls;
+  DelegatingCtorDeclsType DelegatingCtorDecls;
 
   /// \brief All the overriding destructors seen during a class definition
   /// (there could be multiple due to nested classes) that had their exception
   /// spec checks delayed, plus the overridden destructor.
-  llvm::SmallVector<std::pair<const CXXDestructorDecl*,
+  SmallVector<std::pair<const CXXDestructorDecl*,
                               const CXXDestructorDecl*>, 2>
       DelayedDestructorExceptionSpecChecks;
 
@@ -431,32 +452,17 @@ class Sema {
   /// WeakUndeclaredIdentifiers - Identifiers contained in
   /// #pragma weak before declared. rare. may alias another
   /// identifier, declared or undeclared
-  class WeakInfo {
-    IdentifierInfo *alias;  // alias (optional)
-    SourceLocation loc;     // for diagnostics
-    bool used;              // identifier later declared?
-  public:
-    WeakInfo()
-      : alias(0), loc(SourceLocation()), used(false) {}
-    WeakInfo(IdentifierInfo *Alias, SourceLocation Loc)
-      : alias(Alias), loc(Loc), used(false) {}
-    inline IdentifierInfo * getAlias() const { return alias; }
-    inline SourceLocation getLocation() const { return loc; }
-    void setUsed(bool Used=true) { used = Used; }
-    inline bool getUsed() { return used; }
-    bool operator==(WeakInfo RHS) const {
-      return alias == RHS.getAlias() && loc == RHS.getLocation();
-    }
-    bool operator!=(WeakInfo RHS) const { return !(*this == RHS); }
-  };
   llvm::DenseMap<IdentifierInfo*,WeakInfo> WeakUndeclaredIdentifiers;
 
+  /// \brief Load weak undeclared identifiers from the external source.
+  void LoadExternalWeakUndeclaredIdentifiers();
+  
   /// WeakTopLevelDecl - Translation-unit scoped declarations generated by
   /// #pragma weak during processing of other Decls.
   /// I couldn't figure out a clean way to generate these in-line, so
   /// we store them here and handle separately -- which is a hack.
   /// It would be best to refactor this.
-  llvm::SmallVector<Decl*,2> WeakTopLevelDecl;
+  SmallVector<Decl*,2> WeakTopLevelDecl;
 
   IdentifierResolver IdResolver;
 
@@ -482,13 +488,21 @@ class Sema {
   /// have been declared.
   bool GlobalNewDeleteDeclared;
 
+
+  /// A flag that is set when parsing a -dealloc method and no [super dealloc]
+  /// call was found yet.
+  bool ObjCShouldCallSuperDealloc;
+  /// A flag that is set when parsing a -finalize method and no [super finalize]
+  /// call was found yet.
+  bool ObjCShouldCallSuperFinalize;
+
   /// \brief The set of declarations that have been referenced within
   /// a potentially evaluated expression.
-  typedef llvm::SmallVector<std::pair<SourceLocation, Decl *>, 10>
+  typedef SmallVector<std::pair<SourceLocation, Decl *>, 10>
     PotentiallyReferencedDecls;
 
   /// \brief A set of diagnostics that may be emitted.
-  typedef llvm::SmallVector<std::pair<SourceLocation, PartialDiagnostic>, 10>
+  typedef SmallVector<std::pair<SourceLocation, PartialDiagnostic>, 10>
     PotentiallyEmittedDiagnostics;
 
   /// \brief Describes how the expressions currently being parsed are
@@ -578,7 +592,7 @@ class Sema {
   };
 
   /// A stack of expression evaluation contexts.
-  llvm::SmallVector<ExpressionEvaluationContextRecord, 8> ExprEvalContexts;
+  SmallVector<ExpressionEvaluationContextRecord, 8> ExprEvalContexts;
 
   /// SpecialMemberOverloadResult - The overloading result for a special member
   /// function.
@@ -618,23 +632,21 @@ class Sema {
   /// for C++ records.
   llvm::FoldingSet<SpecialMemberOverloadResult> SpecialMemberCache;
 
-  /// \brief Whether the code handled by Sema should be considered a
-  /// complete translation unit or not.
+  /// \brief The kind of translation unit we are processing.
   ///
-  /// When true (which is generally the case), Sema will perform
+  /// When we're processing a complete translation unit, Sema will perform
   /// end-of-translation-unit semantic tasks (such as creating
   /// initializers for tentative definitions in C) once parsing has
-  /// completed. This flag will be false when building PCH files,
-  /// since a PCH file is by definition not a complete translation
-  /// unit.
-  bool CompleteTranslationUnit;
+  /// completed. Modules and precompiled headers perform different kinds of
+  /// checks.
+  TranslationUnitKind TUKind;
 
   llvm::BumpPtrAllocator BumpAlloc;
 
   /// \brief The number of SFINAE diagnostics that have been trapped.
   unsigned NumSFINAEErrors;
 
-  typedef llvm::DenseMap<ParmVarDecl *, llvm::SmallVector<ParmVarDecl *, 1> >
+  typedef llvm::DenseMap<ParmVarDecl *, SmallVector<ParmVarDecl *, 1> >
     UnparsedDefaultArgInstantiationsMap;
   
   /// \brief A mapping from parameters with unparsed default arguments to the
@@ -673,7 +685,7 @@ class Sema {
   bool isSelfExpr(Expr *RExpr);
 public:
   Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
-       bool CompleteTranslationUnit = true,
+       TranslationUnitKind TUKind = TU_Complete,
        CodeCompleteConsumer *CompletionConsumer = 0);
   ~Sema();
   
@@ -685,7 +697,7 @@ class Sema {
   OpenCLOptions &getOpenCLOptions() { return OpenCLFeatures; }
   FPOptions     &getFPOptions() { return FPFeatures; }
 
-  Diagnostic &getDiagnostics() const { return Diags; }
+  DiagnosticsEngine &getDiagnostics() const { return Diags; }
   SourceManager &getSourceManager() const { return SourceMgr; }
   const TargetAttributesSema &getTargetAttributesSema() const;
   Preprocessor &getPreprocessor() const { return PP; }
@@ -727,7 +739,7 @@ class Sema {
   /// \brief Build a partial diagnostic.
   PartialDiagnostic PDiag(unsigned DiagID = 0); // in SemaInternal.h
 
-  bool findMacroSpelling(SourceLocation &loc, llvm::StringRef name);
+  bool findMacroSpelling(SourceLocation &loc, StringRef name);
 
   ExprResult Owned(Expr* E) { return E; }
   ExprResult Owned(ExprResult R) { return R; }
@@ -754,7 +766,7 @@ class Sema {
   sema::BlockScopeInfo *getCurBlock();
 
   /// WeakTopLevelDeclDecls - access to #pragma weak-generated Decls
-  llvm::SmallVector<Decl*,2> &WeakTopLevelDecls() { return WeakTopLevelDecl; }
+  SmallVector<Decl*,2> &WeakTopLevelDecls() { return WeakTopLevelDecl; }
 
   //===--------------------------------------------------------------------===//
   // Type Analysis / Processing: SemaType.cpp.
@@ -785,6 +797,7 @@ class Sema {
   QualType BuildBlockPointerType(QualType T,
                                  SourceLocation Loc, DeclarationName Entity);
   QualType BuildParenType(QualType T);
+  QualType BuildAtomicType(QualType T, SourceLocation Loc);
 
   TypeSourceInfo *GetTypeForDeclarator(Declarator &D, Scope *S);
   TypeSourceInfo *GetTypeForDeclaratorCast(Declarator &D, QualType FromTy);
@@ -819,6 +832,10 @@ class Sema {
 
   TypeResult ActOnTypeName(Scope *S, Declarator &D);
   
+  /// \brief The parser has parsed the context-sensitive type 'instancetype'
+  /// in an Objective-C message declaration. Return the appropriate type.
+  ParsedType ActOnObjCInstanceType(SourceLocation Loc);
+  
   bool RequireCompleteType(SourceLocation Loc, QualType T,
                            const PartialDiagnostic &PD,
                            std::pair<SourceLocation, PartialDiagnostic> Note);
@@ -830,6 +847,9 @@ class Sema {
                                std::pair<SourceLocation,
                                          PartialDiagnostic> Note);
 
+  bool RequireLiteralType(SourceLocation Loc, QualType T,
+                          const PartialDiagnostic &PD,
+                          bool AllowIncompleteType = false);
 
   QualType getElaboratedType(ElaboratedTypeKeyword Keyword,
                              const CXXScopeSpec &SS, QualType T);
@@ -853,9 +873,10 @@ class Sema {
                          bool isClassName = false,
                          bool HasTrailingDot = false,
                          ParsedType ObjectType = ParsedType(),
-                         bool WantNontrivialTypeSourceInfo = false);
+                         bool WantNontrivialTypeSourceInfo = false,
+                         IdentifierInfo **CorrectedII = 0);
   TypeSpecifierType isTagName(IdentifierInfo &II, Scope *S);
-  bool isMicrosoftMissingTypename(const CXXScopeSpec *SS);
+  bool isMicrosoftMissingTypename(const CXXScopeSpec *SS, Scope *S);
   bool DiagnoseUnknownTypeName(const IdentifierInfo &II,
                                SourceLocation IILoc,
                                Scope *S,
@@ -966,8 +987,7 @@ class Sema {
   Decl *ActOnDeclarator(Scope *S, Declarator &D);
 
   Decl *HandleDeclarator(Scope *S, Declarator &D,
-                         MultiTemplateParamsArg TemplateParameterLists,
-                         bool IsFunctionDefinition);
+                         MultiTemplateParamsArg TemplateParameterLists);
   void RegisterLocallyScopedExternCDecl(NamedDecl *ND,
                                         const LookupResult &Previous,
                                         Scope *S);
@@ -978,31 +998,46 @@ class Sema {
   void CheckCastAlign(Expr *Op, QualType T, SourceRange TRange);
   void CheckTypedefForVariablyModifiedType(Scope *S, TypedefNameDecl *D);
   NamedDecl* ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                                    QualType R, TypeSourceInfo *TInfo,
-                                    LookupResult &Previous, bool &Redeclaration);
+                                    TypeSourceInfo *TInfo,
+                                    LookupResult &Previous);
   NamedDecl* ActOnTypedefNameDecl(Scope* S, DeclContext* DC, TypedefNameDecl *D,
                                   LookupResult &Previous, bool &Redeclaration);
   NamedDecl* ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                                     QualType R, TypeSourceInfo *TInfo,
+                                     TypeSourceInfo *TInfo,
                                      LookupResult &Previous,
-                                     MultiTemplateParamsArg TemplateParamLists,
-                                     bool &Redeclaration);
-  void CheckVariableDeclaration(VarDecl *NewVD, LookupResult &Previous,
-                                bool &Redeclaration);
+                                     MultiTemplateParamsArg TemplateParamLists);
+  // Returns true if the variable declaration is a redeclaration
+  bool CheckVariableDeclaration(VarDecl *NewVD, LookupResult &Previous);
   void CheckCompleteVariableDeclaration(VarDecl *var);
   NamedDecl* ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                                     QualType R, TypeSourceInfo *TInfo,
+                                     TypeSourceInfo *TInfo,
                                      LookupResult &Previous,
                                      MultiTemplateParamsArg TemplateParamLists,
-                                     bool IsFunctionDefinition,
-                                     bool &Redeclaration);
+                                     bool &AddToScope);
   bool AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD);
+
+  /// \brief The kind of constexpr declaration checking we are performing.
+  ///
+  /// The kind affects which diagnostics (if any) are emitted if the function
+  /// does not satisfy the requirements of a constexpr function declaration.
+  enum CheckConstexprKind {
+    /// \brief Check a constexpr function declaration, and produce errors if it
+    /// does not satisfy the requirements.
+    CCK_Declaration,
+    /// \brief Check a constexpr function template instantiation.
+    CCK_Instantiation,
+    /// \brief Produce notes explaining why an instantiation was not constexpr.
+    CCK_NoteNonConstexprInstantiation
+  };
+  bool CheckConstexprFunctionDecl(const FunctionDecl *FD, CheckConstexprKind CCK);
+  bool CheckConstexprFunctionBody(const FunctionDecl *FD, Stmt *Body);
+
   void DiagnoseHiddenVirtualMethods(CXXRecordDecl *DC, CXXMethodDecl *MD);
-  void CheckFunctionDeclaration(Scope *S,
+  // Returns true if the function declaration is a redeclaration
+  bool CheckFunctionDeclaration(Scope *S,
                                 FunctionDecl *NewFD, LookupResult &Previous,
-                                bool IsExplicitSpecialization,
-                                bool &Redeclaration);
-  void CheckMain(FunctionDecl *FD);
+                                bool IsExplicitSpecialization);
+  void CheckMain(FunctionDecl *FD, const DeclSpec &D);
   Decl *ActOnParamDeclarator(Scope *S, Declarator &D);
   ParmVarDecl *BuildParmVarDeclForTypedef(DeclContext *DC,
                                           SourceLocation Loc,
@@ -1021,6 +1056,7 @@ class Sema {
   bool SetParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg,
                                SourceLocation EqualLoc);
 
+  void CheckSelfReference(Decl *OrigDecl, Expr *E);
   void AddInitializerToDecl(Decl *dcl, Expr *init, bool DirectInit,
                             bool TypeMayContainAuto);
   void ActOnUninitializedDecl(Decl *dcl, bool TypeMayContainAuto);
@@ -1041,9 +1077,14 @@ class Sema {
   Decl *ActOnStartOfFunctionDef(Scope *S, Decl *D);
   void ActOnStartOfObjCMethodDef(Scope *S, Decl *D);
 
+  void computeNRVO(Stmt *Body, sema::FunctionScopeInfo *Scope);
   Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body);
   Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body, bool IsInstantiation);
 
+  /// ActOnFinishDelayedAttribute - Invoked when we have finished parsing an
+  /// attribute for which parsing is delayed.
+  void ActOnFinishDelayedAttribute(Scope *S, Decl *D, ParsedAttributes &Attrs);
+
   /// \brief Diagnose any unused parameters in the given sequence of
   /// ParmVarDecl pointers.
   void DiagnoseUnusedParameters(ParmVarDecl * const *Begin,
@@ -1062,6 +1103,26 @@ class Sema {
                               SourceLocation AsmLoc,
                               SourceLocation RParenLoc);
 
+  /// \brief The parser has processed a module import declaration.
+  ///
+  /// \param ImportLoc The location of the '__import_module__' keyword.
+  ///
+  /// \param ModuleName The name of the module.
+  ///
+  /// \param ModuleNameLoc The location of the module name.
+  DeclResult ActOnModuleImport(SourceLocation ImportLoc,
+                               IdentifierInfo &ModuleName,
+                               SourceLocation ModuleNameLoc);
+  
+  /// \brief Diagnose that \p New is a module-private redeclaration of
+  /// \p Old.
+  void diagnoseModulePrivateRedeclaration(NamedDecl *New, NamedDecl *Old,
+                                          SourceLocation ModulePrivateKeyword
+                                            = SourceLocation());
+  
+  /// \brief Retrieve a suitable printing policy.
+  PrintingPolicy getPrintingPolicy() const;
+
   /// Scope actions.
   void ActOnPopScope(SourceLocation Loc, Scope *S);
   void ActOnTranslationUnitScope(Scope *S);
@@ -1097,6 +1158,7 @@ class Sema {
                  SourceLocation KWLoc, CXXScopeSpec &SS,
                  IdentifierInfo *Name, SourceLocation NameLoc,
                  AttributeList *Attr, AccessSpecifier AS,
+                 SourceLocation ModulePrivateLoc,
                  MultiTemplateParamsArg TemplateParameterLists,
                  bool &OwnedDecl, bool &IsDependent, bool ScopedEnum,
                  bool ScopedEnumUsesClassTag, TypeResult UnderlyingType);
@@ -1118,7 +1180,7 @@ class Sema {
 
   void ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
                  IdentifierInfo *ClassName,
-                 llvm::SmallVectorImpl<Decl *> &Decls);
+                 SmallVectorImpl<Decl *> &Decls);
   Decl *ActOnField(Scope *S, Decl *TagD, SourceLocation DeclStart,
                    Declarator &D, Expr *BitfieldWidth);
 
@@ -1146,15 +1208,15 @@ class Sema {
   bool CheckNontrivialField(FieldDecl *FD);
   void DiagnoseNontrivial(const RecordType* Record, CXXSpecialMember mem);
   CXXSpecialMember getSpecialMember(const CXXMethodDecl *MD);
-  void ActOnLastBitfield(SourceLocation DeclStart, Decl *IntfDecl, 
-                         llvm::SmallVectorImpl<Decl *> &AllIvarDecls);
-  Decl *ActOnIvar(Scope *S, SourceLocation DeclStart, Decl *IntfDecl,
+  void ActOnLastBitfield(SourceLocation DeclStart, 
+                         SmallVectorImpl<Decl *> &AllIvarDecls);
+  Decl *ActOnIvar(Scope *S, SourceLocation DeclStart,
                   Declarator &D, Expr *BitfieldWidth,
                   tok::ObjCKeywordKind visibility);
 
   // This is used for both record definitions and ObjC interface declarations.
   void ActOnFields(Scope* S, SourceLocation RecLoc, Decl *TagDecl,
-                   Decl **Fields, unsigned NumFields,
+                   llvm::ArrayRef<Decl *> Fields,
                    SourceLocation LBrac, SourceLocation RBrac,
                    AttributeList *AttrList);
 
@@ -1163,6 +1225,8 @@ class Sema {
   /// struct, or union).
   void ActOnTagStartDefinition(Scope *S, Decl *TagDecl);
 
+  Decl *ActOnObjCContainerStartDefinition(Decl *IDecl);
+
   /// ActOnStartCXXMemberDeclarations - Invoked when we have parsed a
   /// C++ record definition's base-specifiers clause and are starting its
   /// member declarations.
@@ -1175,6 +1239,15 @@ class Sema {
   void ActOnTagFinishDefinition(Scope *S, Decl *TagDecl,
                                 SourceLocation RBraceLoc);
 
+  void ActOnObjCContainerFinishDefinition();
+
+  /// \brief Invoked when we must temporarily exit the objective-c container
+  /// scope for parsing/looking-up C constructs.
+  ///
+  /// Must be followed by a call to \see ActOnObjCReenterContainerContext
+  void ActOnObjCTemporaryExitContainerContext();
+  void ActOnObjCReenterContainerContext();
+
   /// ActOnTagDefinitionError - Invoked when there was an unrecoverable
   /// error parsing the definition of a tag.
   void ActOnTagDefinitionError(Scope *S, Decl *TagDecl);
@@ -1205,6 +1278,10 @@ class Sema {
   void EnterDeclaratorContext(Scope *S, DeclContext *DC);
   void ExitDeclaratorContext(Scope *S);
 
+  /// Push the parameters of D, which must be a function, into scope.
+  void ActOnReenterFunctionContext(Scope* S, Decl* D);
+  void ActOnExitFunctionContext() { PopDeclContext(); }
+
   DeclContext *getFunctionLevelDeclContext();
 
   /// getCurFunctionDecl - If inside of a function body, this returns a pointer
@@ -1314,6 +1391,7 @@ class Sema {
   bool FunctionArgTypesAreEqual(const FunctionProtoType *OldType, 
                                 const FunctionProtoType *NewType);
   
+  CastKind PrepareCastToObjCObjectPointer(ExprResult &E);
   bool CheckPointerConversion(Expr *From, QualType ToType,
                               CastKind &Kind,
                               CXXCastPath& BasePath,
@@ -1337,19 +1415,20 @@ class Sema {
                                              QualType ResultType,
                                              Expr *Value,
                                              bool AllowNRVO = true);
-  
+
   bool CanPerformCopyInitialization(const InitializedEntity &Entity,
                                     ExprResult Init);
   ExprResult PerformCopyInitialization(const InitializedEntity &Entity,
                                        SourceLocation EqualLoc,
-                                       ExprResult Init);
+                                       ExprResult Init,
+                                       bool TopLevelOfInitList = false);
   ExprResult PerformObjectArgumentInitialization(Expr *From,
                                                  NestedNameSpecifier *Qualifier,
                                                  NamedDecl *FoundDecl,
                                                  CXXMethodDecl *Method);
 
   ExprResult PerformContextuallyConvertToBool(Expr *From);
-  ExprResult PerformContextuallyConvertToObjCId(Expr *From);
+  ExprResult PerformContextuallyConvertToObjCPointer(Expr *From);
 
   ExprResult 
   ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *FromE,
@@ -1475,8 +1554,9 @@ class Sema {
                                                    bool Complain = false,
                                                    DeclAccessPair* Found = 0);
 
-  ExprResult ResolveAndFixSingleFunctionTemplateSpecialization(
-                      Expr *SrcExpr, bool DoFunctionPointerConverion = false, 
+  bool ResolveAndFixSingleFunctionTemplateSpecialization(
+                      ExprResult &SrcExpr,
+                      bool DoFunctionPointerConverion = false,
                       bool Complain = false, 
                       const SourceRange& OpRangeForComplaining = SourceRange(), 
                       QualType DestTypeForComplaining = QualType(), 
@@ -1678,6 +1758,9 @@ class Sema {
   CXXMethodDecl *LookupCopyingAssignment(CXXRecordDecl *Class, unsigned Quals,
                                          bool RValueThis, unsigned ThisQuals,
                                          bool *ConstParam = 0);
+  CXXConstructorDecl *LookupMovingConstructor(CXXRecordDecl *Class);
+  CXXMethodDecl *LookupMovingAssignment(CXXRecordDecl *Class, bool RValueThis,
+                                        unsigned ThisQuals);
   CXXDestructorDecl *LookupDestructor(CXXRecordDecl *Class);
 
   void ArgumentDependentLookup(DeclarationName Name, bool Operator,
@@ -1755,6 +1838,10 @@ class Sema {
                            bool NonInheritable = true, bool Inheritable = true);
   void ProcessDeclAttributeList(Scope *S, Decl *D, const AttributeList *AL,
                            bool NonInheritable = true, bool Inheritable = true);
+  bool ProcessAccessDeclAttributeList(AccessSpecDecl *ASDecl,
+                                      const AttributeList *AttrList);
+
+  void checkUnusedDeclAttributes(Declarator &D);
 
   bool CheckRegparmAttr(const AttributeList &attr, unsigned &value);
   bool CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC);
@@ -1762,15 +1849,26 @@ class Sema {
 
   void WarnUndefinedMethod(SourceLocation ImpLoc, ObjCMethodDecl *method,
                            bool &IncompleteImpl, unsigned DiagID);
-  void WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethod,
+  void WarnConflictingTypedMethods(ObjCMethodDecl *Method,
                                    ObjCMethodDecl *MethodDecl,
                                    bool IsProtocolMethodDecl);
+  
+  void CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
+                                   ObjCMethodDecl *Overridden,
+                                   bool IsProtocolMethodDecl);
+
+  /// WarnExactTypedMethods - This routine issues a warning if method
+  /// implementation declaration matches exactly that of its declaration.
+  void WarnExactTypedMethods(ObjCMethodDecl *Method,
+                             ObjCMethodDecl *MethodDecl,
+                             bool IsProtocolMethodDecl);
 
   bool isPropertyReadonly(ObjCPropertyDecl *PropertyDecl,
                           ObjCInterfaceDecl *IDecl);
 
   typedef llvm::DenseSet<Selector, llvm::DenseMapInfo<Selector> > SelectorSet;
-
+  typedef llvm::DenseMap<Selector, ObjCMethodDecl*> ProtocolsMethodsMap;
+                         
   /// CheckProtocolMethodDefs - This routine checks unimplemented
   /// methods declared in protocol, and those referenced by it.
   /// \param IDecl - Used for checking for methods which may have been
@@ -1788,14 +1886,6 @@ class Sema {
                                 ObjCIvarDecl **Fields, unsigned nIvars,
                                 SourceLocation Loc);
 
-  /// \brief Determine whether we can synthesize a provisional ivar for the
-  /// given name.
-  ObjCPropertyDecl *canSynthesizeProvisionalIvar(IdentifierInfo *II);
-
-  /// \brief Determine whether we can synthesize a provisional ivar for the
-  /// given property.
-  bool canSynthesizeProvisionalIvar(ObjCPropertyDecl *Property);
-
   /// ImplMethodsVsClassMethods - This is main routine to warn if any method
   /// remains unimplemented in the class or category @implementation.
   void ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
@@ -1812,6 +1902,7 @@ class Sema {
   /// properties which must be synthesized in class's @implementation.
   void DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
                                     ObjCInterfaceDecl *IDecl);
+  void DefaultSynthesizeProperties(Scope *S, Decl *D);
   
   /// CollectImmediateProperties - This routine collects all properties in
   /// the class and its conforming protocols; but not those it its super class.
@@ -1828,7 +1919,6 @@ class Sema {
   /// Called by ActOnProperty to handle @property declarations in
   ////  class extensions.
   Decl *HandlePropertyInClassExtension(Scope *S,
-                                       ObjCCategoryDecl *CDecl,
                                        SourceLocation AtLoc,
                                        FieldDeclarator &FD,
                                        Selector GetterSel,
@@ -1885,7 +1975,13 @@ class Sema {
                                   ObjCImplDecl* IMPDecl,
                                   ObjCContainerDecl* IDecl,
                                   bool &IncompleteImpl,
-                                  bool ImmediateClass);
+                                  bool ImmediateClass,
+                                  bool WarnExactMatch=false);
+
+  /// CheckCategoryVsClassMethodMatches - Checks that methods implemented in
+  /// category matches with those implemented in its primary class and
+  /// warns each time an exact match is found. 
+  void CheckCategoryVsClassMethodMatches(ObjCCategoryImplDecl *CatIMP);
 
 private:
   /// AddMethodToGlobalPool - Add an instance or factory method to the global
@@ -1912,6 +2008,10 @@ class Sema {
     AddMethodToGlobalPool(Method, impl, /*instance*/false);
   }
 
+  /// AddAnyMethodToGlobalPool - Add any method, instance or factory to global
+  /// pool.
+  void AddAnyMethodToGlobalPool(Decl *D);
+
   /// LookupInstanceMethodInGlobalPool - Returns the method and warns if
   /// there are multiple signatures.
   ObjCMethodDecl *LookupInstanceMethodInGlobalPool(Selector Sel, SourceRange R,
@@ -1937,7 +2037,7 @@ class Sema {
   /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
   /// initialization.
   void CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
-                                  llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars);
+                                  SmallVectorImpl<ObjCIvarDecl*> &Ivars);
 
   //===--------------------------------------------------------------------===//
   // Statement Parsing Callbacks: SemaStmt.cpp.
@@ -1978,7 +2078,7 @@ class Sema {
   StmtResult ActOnExprStmt(FullExprArg Expr);
 
   StmtResult ActOnNullStmt(SourceLocation SemiLoc,
-                        SourceLocation LeadingEmptyMacroLoc = SourceLocation());
+                           bool HasLeadingEmptyMacro = false);
   StmtResult ActOnCompoundStmt(SourceLocation L, SourceLocation R,
                                        MultiStmtArg Elts,
                                        bool isStmtExpr);
@@ -2022,6 +2122,8 @@ class Sema {
                           FullExprArg Third,
                           SourceLocation RParenLoc,
                           Stmt *Body);
+  ExprResult ActOnObjCForCollectionOperand(SourceLocation forLoc,
+                                           Expr *collection);
   StmtResult ActOnObjCForCollectionStmt(SourceLocation ForColLoc,
                                         SourceLocation LParenLoc,
                                         Stmt *First, Expr *Second,
@@ -2083,6 +2185,8 @@ class Sema {
   StmtResult BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw);
   StmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
                                   Scope *CurScope);
+  ExprResult ActOnObjCAtSynchronizedOperand(SourceLocation atLoc,
+                                            Expr *operand);
   StmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc,
                                          Expr *SynchExpr,
                                          Stmt *SynchBody);
@@ -2141,19 +2245,20 @@ class Sema {
     DelayedDiagnostics.popContext(state);
   }
 
-  void EmitDeprecationWarning(NamedDecl *D, llvm::StringRef Message,
+  void EmitDeprecationWarning(NamedDecl *D, StringRef Message,
                               SourceLocation Loc, 
                               const ObjCInterfaceDecl *UnknownObjCClass=0);
 
   void HandleDelayedDeprecationCheck(sema::DelayedDiagnostic &DD, Decl *Ctx);
 
   bool makeUnavailableInSystemHeader(SourceLocation loc,
-                                     llvm::StringRef message);
+                                     StringRef message);
 
   //===--------------------------------------------------------------------===//
   // Expression Parsing Callbacks: SemaExpr.cpp.
 
-  bool DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc, 
+  bool CanUseDecl(NamedDecl *D);
+  bool DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc,
                          const ObjCInterfaceDecl *UnknownObjCClass=0);
   std::string getDeletedOrUnavailableSuffix(const FunctionDecl *FD);
   bool DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *PD,
@@ -2172,12 +2277,16 @@ class Sema {
   void MarkDeclarationsReferencedInType(SourceLocation Loc, QualType T);
   void MarkDeclarationsReferencedInExpr(Expr *E);
 
+  /// \brief Try to recover by turning the given expression into a
+  /// call.  Returns true if recovery was attempted or an error was
+  /// emitted; this may also leave the ExprResult invalid.
+  bool tryToRecoverWithCall(ExprResult &E, const PartialDiagnostic &PD,
+                            bool ForceComplain = false,
+                            bool (*IsPlausibleResult)(QualType) = 0);
+
   /// \brief Figure out if an expression could be turned into a call.
   bool isExprCallable(const Expr &E, QualType &ZeroArgCallReturnTy,
                       UnresolvedSetImpl &NonTemplateOverloads);
-  /// \brief Give notes for a set of overloads.
-  void NoteOverloads(const UnresolvedSetImpl &Overloads,
-                     const SourceLocation FinalNoteLoc);
 
   /// \brief Conditionally issue a diagnostic based on the current
   /// evaluation context.
@@ -2186,17 +2295,13 @@ class Sema {
   ///  the function body is parsed, and then do a basic reachability analysis to
   ///  determine if the statement is reachable.  If it is unreachable, the
   ///  diagnostic will not be emitted.
-  bool DiagRuntimeBehavior(SourceLocation Loc, const Stmt *stmt,
+  bool DiagRuntimeBehavior(SourceLocation Loc, const Stmt *Statement,
                            const PartialDiagnostic &PD);
 
   // Primary Expressions.
   SourceRange getExprRange(Expr *E) const;
-
-  ObjCIvarDecl *SynthesizeProvisionalIvar(LookupResult &Lookup,
-                                          IdentifierInfo *II,
-                                          SourceLocation NameLoc);
   
-  ExprResult ActOnIdExpression(Scope *S, CXXScopeSpec &SS, UnqualifiedId &Name,
+  ExprResult ActOnIdExpression(Scope *S, CXXScopeSpec &SS, UnqualifiedId &Id,
                                bool HasTrailingLParen, bool IsAddressOfOperand);
 
   void DecomposeUnqualifiedId(const UnqualifiedId &Id,
@@ -2205,9 +2310,12 @@ class Sema {
                               const TemplateArgumentListInfo *&TemplateArgs);
 
   bool DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
-                           CorrectTypoContext CTC = CTC_Unknown);
+                           CorrectTypoContext CTC = CTC_Unknown,
+                           TemplateArgumentListInfo *ExplicitTemplateArgs = 0,
+                           Expr **Args = 0, unsigned NumArgs = 0);
 
-  ExprResult LookupInObjCMethod(LookupResult &R, Scope *S, IdentifierInfo *II,
+  ExprResult LookupInObjCMethod(LookupResult &LookUp, Scope *S,
+                                IdentifierInfo *II,
                                 bool AllowBuiltinCreation=false);
 
   ExprResult ActOnDependentIdExpression(const CXXScopeSpec &SS,
@@ -2248,29 +2356,30 @@ class Sema {
 
   ExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS,
                                       LookupResult &R,
-                                      bool ADL);
+                                      bool NeedsADL);
   ExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS,
                                       const DeclarationNameInfo &NameInfo,
                                       NamedDecl *D);
 
   ExprResult ActOnPredefinedExpr(SourceLocation Loc, tok::TokenKind Kind);
-  ExprResult ActOnNumericConstant(const Token &);
-  ExprResult ActOnCharacterConstant(const Token &);
-  ExprResult ActOnParenExpr(SourceLocation L, SourceLocation R, Expr *Val);
+  ExprResult ActOnNumericConstant(const Token &Tok);
+  ExprResult ActOnCharacterConstant(const Token &Tok);
+  ExprResult ActOnParenExpr(SourceLocation L, SourceLocation R, Expr *E);
   ExprResult ActOnParenOrParenListExpr(SourceLocation L,
                                        SourceLocation R,
                                        MultiExprArg Val);
 
   /// ActOnStringLiteral - The specified tokens were lexed as pasted string
   /// fragments (e.g. "foo" "bar" L"baz").
-  ExprResult ActOnStringLiteral(const Token *Toks, unsigned NumToks);
+  ExprResult ActOnStringLiteral(const Token *StringToks,
+                                unsigned NumStringToks);
 
   ExprResult ActOnGenericSelectionExpr(SourceLocation KeyLoc,
                                        SourceLocation DefaultLoc,
                                        SourceLocation RParenLoc,
                                        Expr *ControllingExpr,
-                                       MultiTypeArg Types,
-                                       MultiExprArg Exprs);
+                                       MultiTypeArg ArgTypes,
+                                       MultiExprArg ArgExprs);
   ExprResult CreateGenericSelectionExpr(SourceLocation KeyLoc,
                                         SourceLocation DefaultLoc,
                                         SourceLocation RParenLoc,
@@ -2281,13 +2390,13 @@ class Sema {
 
   // Binary/Unary Operators.  'Tok' is the token for the operator.
   ExprResult CreateBuiltinUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
-                                  Expr *InputArg);
+                                  Expr *InputExpr);
   ExprResult BuildUnaryOp(Scope *S, SourceLocation OpLoc,
-                          UnaryOperatorKind Opc, Expr *input);
+                          UnaryOperatorKind Opc, Expr *Input);
   ExprResult ActOnUnaryOp(Scope *S, SourceLocation OpLoc,
                           tok::TokenKind Op, Expr *Input);
 
-  ExprResult CreateUnaryExprOrTypeTraitExpr(TypeSourceInfo *T,
+  ExprResult CreateUnaryExprOrTypeTraitExpr(TypeSourceInfo *TInfo,
                                             SourceLocation OpLoc,
                                             UnaryExprOrTypeTrait ExprKind,
                                             SourceRange R);
@@ -2296,15 +2405,15 @@ class Sema {
   ExprResult
     ActOnUnaryExprOrTypeTraitExpr(SourceLocation OpLoc,
                                   UnaryExprOrTypeTrait ExprKind,
-                                  bool isType, void *TyOrEx,
+                                  bool IsType, void *TyOrEx,
                                   const SourceRange &ArgRange);
 
   ExprResult CheckPlaceholderExpr(Expr *E);
   bool CheckVecStepExpr(Expr *E);
 
   bool CheckUnaryExprOrTypeTraitOperand(Expr *E, UnaryExprOrTypeTrait ExprKind);
-  bool CheckUnaryExprOrTypeTraitOperand(QualType type, SourceLocation OpLoc,
-                                        SourceRange R,
+  bool CheckUnaryExprOrTypeTraitOperand(QualType ExprType, SourceLocation OpLoc,
+                                        SourceRange ExprRange,
                                         UnaryExprOrTypeTrait ExprKind);
   ExprResult ActOnSizeofParameterPackExpr(Scope *S,
                                           SourceLocation OpLoc,
@@ -2364,30 +2473,34 @@ class Sema {
                                FunctionDecl *FDecl,
                                const FunctionProtoType *Proto,
                                Expr **Args, unsigned NumArgs,
-                               SourceLocation RParenLoc);
+                               SourceLocation RParenLoc,
+                               bool ExecConfig = false);
 
   /// ActOnCallExpr - Handle a call to Fn with the specified array of arguments.
   /// This provides the location of the left/right parens and a list of comma
   /// locations.
   ExprResult ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
-                           MultiExprArg Args, SourceLocation RParenLoc,
-                           Expr *ExecConfig = 0);
+                           MultiExprArg ArgExprs, SourceLocation RParenLoc,
+                           Expr *ExecConfig = 0, bool IsExecConfig = false);
   ExprResult BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
                                    SourceLocation LParenLoc,
                                    Expr **Args, unsigned NumArgs,
                                    SourceLocation RParenLoc,
-                                   Expr *ExecConfig = 0);
+                                   Expr *Config = 0,
+                                   bool IsExecConfig = false);
 
   ExprResult ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
-                                MultiExprArg ExecConfig, SourceLocation GGGLoc);
+                                     MultiExprArg ExecConfig,
+                                     SourceLocation GGGLoc);
 
   ExprResult ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
                            Declarator &D, ParsedType &Ty,
-                           SourceLocation RParenLoc, Expr *Op);
+                           SourceLocation RParenLoc, Expr *CastExpr);
   ExprResult BuildCStyleCastExpr(SourceLocation LParenLoc,
                                  TypeSourceInfo *Ty,
                                  SourceLocation RParenLoc,
                                  Expr *Op);
+  CastKind PrepareScalarCast(ExprResult &src, QualType destType);
 
   /// \brief Build an altivec or OpenCL literal.
   ExprResult BuildVectorLiteral(SourceLocation LParenLoc,
@@ -2399,16 +2512,16 @@ class Sema {
   ExprResult ActOnCompoundLiteral(SourceLocation LParenLoc,
                                   ParsedType Ty,
                                   SourceLocation RParenLoc,
-                                  Expr *Op);
+                                  Expr *InitExpr);
 
   ExprResult BuildCompoundLiteralExpr(SourceLocation LParenLoc,
                                       TypeSourceInfo *TInfo,
                                       SourceLocation RParenLoc,
-                                      Expr *InitExpr);
+                                      Expr *LiteralExpr);
 
-  ExprResult ActOnInitList(SourceLocation LParenLoc,
-                           MultiExprArg InitList,
-                           SourceLocation RParenLoc);
+  ExprResult ActOnInitList(SourceLocation LBraceLoc,
+                           MultiExprArg InitArgList,
+                           SourceLocation RBraceLoc);
 
   ExprResult ActOnDesignatedInitializer(Designation &Desig,
                                         SourceLocation Loc,
@@ -2416,21 +2529,21 @@ class Sema {
                                         ExprResult Init);
 
   ExprResult ActOnBinOp(Scope *S, SourceLocation TokLoc,
-                        tok::TokenKind Kind, Expr *LHS, Expr *RHS);
+                        tok::TokenKind Kind, Expr *LHSExpr, Expr *RHSExpr);
   ExprResult BuildBinOp(Scope *S, SourceLocation OpLoc,
-                        BinaryOperatorKind Opc, Expr *lhs, Expr *rhs);
-  ExprResult CreateBuiltinBinOp(SourceLocation TokLoc,
-                                BinaryOperatorKind Opc, Expr *lhs, Expr *rhs);
+                        BinaryOperatorKind Opc, Expr *LHSExpr, Expr *RHSExpr);
+  ExprResult CreateBuiltinBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+                                Expr *LHSExpr, Expr *RHSExpr);
 
   /// ActOnConditionalOp - Parse a ?: operation.  Note that 'LHS' may be null
   /// in the case of a the GNU conditional expr extension.
   ExprResult ActOnConditionalOp(SourceLocation QuestionLoc,
                                 SourceLocation ColonLoc,
-                                Expr *Cond, Expr *LHS, Expr *RHS);
+                                Expr *CondExpr, Expr *LHSExpr, Expr *RHSExpr);
 
   /// ActOnAddrLabel - Parse the GNU address of label extension: "&&foo".
   ExprResult ActOnAddrLabel(SourceLocation OpLoc, SourceLocation LabLoc,
-                            LabelDecl *LD);
+                            LabelDecl *TheDecl);
   
   ExprResult ActOnStmtExpr(SourceLocation LPLoc, Stmt *SubStmt,
                            SourceLocation RPLoc); // "({..})"
@@ -2441,7 +2554,7 @@ class Sema {
     bool isBrackets;  // true if [expr], false if .ident
     union {
       IdentifierInfo *IdentInfo;
-      ExprTy *E;
+      Expr *E;
     } U;
   };
 
@@ -2454,28 +2567,26 @@ class Sema {
   ExprResult ActOnBuiltinOffsetOf(Scope *S,
                                   SourceLocation BuiltinLoc,
                                   SourceLocation TypeLoc,
-                                  ParsedType Arg1,
+                                  ParsedType ParsedArgTy,
                                   OffsetOfComponent *CompPtr,
                                   unsigned NumComponents,
                                   SourceLocation RParenLoc);
 
   // __builtin_choose_expr(constExpr, expr1, expr2)
   ExprResult ActOnChooseExpr(SourceLocation BuiltinLoc,
-                             Expr *cond, Expr *expr1,
-                             Expr *expr2, SourceLocation RPLoc);
+                             Expr *CondExpr, Expr *LHSExpr,
+                             Expr *RHSExpr, SourceLocation RPLoc);
 
   // __builtin_va_arg(expr, type)
-  ExprResult ActOnVAArg(SourceLocation BuiltinLoc,
-                        Expr *expr, ParsedType type,
+  ExprResult ActOnVAArg(SourceLocation BuiltinLoc, Expr *E, ParsedType Ty,
                         SourceLocation RPLoc);
-  ExprResult BuildVAArgExpr(SourceLocation BuiltinLoc,
-                            Expr *expr, TypeSourceInfo *TInfo,
-                            SourceLocation RPLoc);
+  ExprResult BuildVAArgExpr(SourceLocation BuiltinLoc, Expr *E,
+                            TypeSourceInfo *TInfo, SourceLocation RPLoc);
 
   // __null
   ExprResult ActOnGNUNullExpr(SourceLocation TokenLoc);
 
-  bool CheckCaseExpression(Expr *expr);
+  bool CheckCaseExpression(Expr *E);
 
   bool CheckMicrosoftIfExistsSymbol(CXXScopeSpec &SS, UnqualifiedId &Name);
 
@@ -2495,13 +2606,13 @@ class Sema {
 
   /// ActOnBlockStmtExpr - This is called when the body of a block statement
   /// literal was successfully completed.  ^(int x){...}
-  ExprResult ActOnBlockStmtExpr(SourceLocation CaretLoc,
-                                        Stmt *Body, Scope *CurScope);
+  ExprResult ActOnBlockStmtExpr(SourceLocation CaretLoc, Stmt *Body,
+                                Scope *CurScope);
 
   //===---------------------------- OpenCL Features -----------------------===//
     
   /// __builtin_astype(...)
-  ExprResult ActOnAsTypeExpr(Expr *expr, ParsedType DestTy,
+  ExprResult ActOnAsTypeExpr(Expr *E, ParsedType ParsedDestTy,
                              SourceLocation BuiltinLoc, 
                              SourceLocation RParenLoc);
   
@@ -2594,7 +2705,8 @@ class Sema {
   /// and sets it as the initializer for the the passed in VarDecl.
   bool InitializeVarWithConstructor(VarDecl *VD,
                                     CXXConstructorDecl *Constructor,
-                                    MultiExprArg Exprs);
+                                    MultiExprArg Exprs,
+                                    bool HadMultipleCandidates);
 
   /// BuildCXXConstructExpr - Creates a complete call to a constructor,
   /// including handling of its default argument expressions.
@@ -2603,16 +2715,16 @@ class Sema {
   ExprResult
   BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                         CXXConstructorDecl *Constructor, MultiExprArg Exprs,
-                        bool RequiresZeroInit, unsigned ConstructKind,
-                        SourceRange ParenRange);
+                        bool HadMultipleCandidates, bool RequiresZeroInit,
+                        unsigned ConstructKind, SourceRange ParenRange);
 
   // FIXME: Can re remove this and have the above BuildCXXConstructExpr check if
   // the constructor can be elidable?
   ExprResult
   BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                         CXXConstructorDecl *Constructor, bool Elidable,
-                        MultiExprArg Exprs, bool RequiresZeroInit,
-                        unsigned ConstructKind,
+                        MultiExprArg Exprs, bool HadMultipleCandidates,
+                        bool RequiresZeroInit, unsigned ConstructKind,
                         SourceRange ParenRange);
 
   /// BuildCXXDefaultArgExpr - Creates a CXXDefaultArgExpr, instantiating
@@ -2645,7 +2757,7 @@ class Sema {
     // any other specification (even 'none', to keep this rule simple).
     ExceptionSpecificationType ComputedEST;
     llvm::SmallPtrSet<CanQualType, 4> ExceptionsSeen;
-    llvm::SmallVector<QualType, 4> Exceptions;
+    SmallVector<QualType, 4> Exceptions;
 
     void ClearExceptions() {
       ExceptionsSeen.clear();
@@ -2710,23 +2822,33 @@ class Sema {
   std::pair<ImplicitExceptionSpecification, bool>
   ComputeDefaultedCopyAssignmentExceptionSpecAndConst(CXXRecordDecl *ClassDecl);
 
+  /// \brief Determine what sort of exception specification a defaulted move
+  /// constructor of a class will have.
+  ImplicitExceptionSpecification
+  ComputeDefaultedMoveCtorExceptionSpec(CXXRecordDecl *ClassDecl);
+
+  /// \brief Determine what sort of exception specification a defaulted move
+  /// assignment operator of a class will have.
+  ImplicitExceptionSpecification
+  ComputeDefaultedMoveAssignmentExceptionSpec(CXXRecordDecl *ClassDecl);
+
   /// \brief Determine what sort of exception specification a defaulted
   /// destructor of a class will have.
   ImplicitExceptionSpecification
   ComputeDefaultedDtorExceptionSpec(CXXRecordDecl *ClassDecl);
 
-  /// \brief Determine if a defaulted default constructor ought to be
-  /// deleted.
-  bool ShouldDeleteDefaultConstructor(CXXConstructorDecl *CD);
-
-  /// \brief Determine if a defaulted copy constructor ought to be
-  /// deleted.
-  bool ShouldDeleteCopyConstructor(CXXConstructorDecl *CD);
+  /// \brief Determine if a special member function should have a deleted
+  /// definition when it is defaulted.
+  bool ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM);
 
   /// \brief Determine if a defaulted copy assignment operator ought to be
   /// deleted.
   bool ShouldDeleteCopyAssignmentOperator(CXXMethodDecl *MD);
 
+  /// \brief Determine if a defaulted move assignment operator ought to be
+  /// deleted.
+  bool ShouldDeleteMoveAssignmentOperator(CXXMethodDecl *MD);
+
   /// \brief Determine if a defaulted destructor ought to be deleted.
   bool ShouldDeleteDestructor(CXXDestructorDecl *DD);
 
@@ -2772,9 +2894,6 @@ class Sema {
 
   /// \brief Declare the implicit copy constructor for the given class.
   ///
-  /// \param S The scope of the class, which may be NULL if this is a 
-  /// template instantiation.
-  ///
   /// \param ClassDecl The class declaration into which the implicit 
   /// copy constructor will be added.
   ///
@@ -2786,21 +2905,45 @@ class Sema {
   void DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
                                      CXXConstructorDecl *Constructor);
 
+  /// \brief Declare the implicit move constructor for the given class.
+  ///
+  /// \param ClassDecl The Class declaration into which the implicit
+  /// move constructor will be added.
+  ///
+  /// \returns The implicitly-declared move constructor, or NULL if it wasn't
+  /// declared.
+  CXXConstructorDecl *DeclareImplicitMoveConstructor(CXXRecordDecl *ClassDecl);
+                                                     
+  /// DefineImplicitMoveConstructor - Checks for feasibility of
+  /// defining this constructor as the move constructor.
+  void DefineImplicitMoveConstructor(SourceLocation CurrentLocation,
+                                     CXXConstructorDecl *Constructor);
+
   /// \brief Declare the implicit copy assignment operator for the given class.
   ///
-  /// \param S The scope of the class, which may be NULL if this is a 
-  /// template instantiation.
-  ///
   /// \param ClassDecl The class declaration into which the implicit 
-  /// copy-assignment operator will be added.
+  /// copy assignment operator will be added.
   ///
   /// \returns The implicitly-declared copy assignment operator.
   CXXMethodDecl *DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl);
   
-  /// \brief Defined an implicitly-declared copy assignment operator.
+  /// \brief Defines an implicitly-declared copy assignment operator.
   void DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
                                     CXXMethodDecl *MethodDecl);
 
+  /// \brief Declare the implicit move assignment operator for the given class.
+  ///
+  /// \param ClassDecl The Class declaration into which the implicit
+  /// move assignment operator will be added.
+  ///
+  /// \returns The implicitly-declared move assignment operator, or NULL if it
+  /// wasn't declared.
+  CXXMethodDecl *DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl);
+  
+  /// \brief Defines an implicitly-declared move assignment operator.
+  void DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
+                                    CXXMethodDecl *MethodDecl);
+
   /// \brief Force the declaration of any implicitly-declared members of this
   /// class.
   void ForceDeclarationOfImplicitMembers(CXXRecordDecl *Class);
@@ -3231,7 +3374,8 @@ class Sema {
                                   TypeSourceInfo *EncodedTypeInfo,
                                   SourceLocation RParenLoc);
   ExprResult BuildCXXMemberCallExpr(Expr *Exp, NamedDecl *FoundDecl,
-                                    CXXMethodDecl *Method);
+                                    CXXMethodDecl *Method,
+                                    bool HadMultipleCandidates);
 
   ExprResult ParseObjCEncodeExpression(SourceLocation AtLoc,
                                        SourceLocation EncodeLoc,
@@ -3259,7 +3403,7 @@ class Sema {
   Decl *ActOnStartLinkageSpecification(Scope *S,
                                        SourceLocation ExternLoc,
                                        SourceLocation LangLoc,
-                                       llvm::StringRef Lang,
+                                       StringRef Lang,
                                        SourceLocation LBraceLoc);
   Decl *ActOnFinishLinkageSpecification(Scope *S,
                                         Decl *LinkageSpec,
@@ -3272,16 +3416,16 @@ class Sema {
   bool isCurrentClassName(const IdentifierInfo &II, Scope *S,
                           const CXXScopeSpec *SS = 0);
 
-  Decl *ActOnAccessSpecifier(AccessSpecifier Access,
-                             SourceLocation ASLoc,
-                             SourceLocation ColonLoc);
+  bool ActOnAccessSpecifier(AccessSpecifier Access,
+                            SourceLocation ASLoc,
+                            SourceLocation ColonLoc,
+                            AttributeList *Attrs = 0);
 
   Decl *ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS,
                                  Declarator &D,
                                  MultiTemplateParamsArg TemplateParameterLists,
                                  Expr *BitfieldWidth, const VirtSpecifiers &VS,
-                                 Expr *Init, bool HasDeferredInit,
-                                 bool IsDefinition);
+                                 bool HasDeferredInit);
   void ActOnCXXInClassMemberInitializer(Decl *VarDecl, SourceLocation EqualLoc,
                                         Expr *Init);
 
@@ -3296,24 +3440,37 @@ class Sema {
                                     SourceLocation RParenLoc,
                                     SourceLocation EllipsisLoc);
 
-  MemInitResult BuildMemberInitializer(ValueDecl *Member, Expr **Args,
-                                       unsigned NumArgs, SourceLocation IdLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation RParenLoc);
+  MemInitResult ActOnMemInitializer(Decl *ConstructorD,
+                                    Scope *S,
+                                    CXXScopeSpec &SS,
+                                    IdentifierInfo *MemberOrBase,
+                                    ParsedType TemplateTypeTy,
+                                    SourceLocation IdLoc,
+                                    Expr *InitList,
+                                    SourceLocation EllipsisLoc);
+
+  MemInitResult BuildMemInitializer(Decl *ConstructorD,
+                                    Scope *S,
+                                    CXXScopeSpec &SS,
+                                    IdentifierInfo *MemberOrBase,
+                                    ParsedType TemplateTypeTy,
+                                    SourceLocation IdLoc,
+                                    const MultiInitializer &Init,
+                                    SourceLocation EllipsisLoc);
+
+  MemInitResult BuildMemberInitializer(ValueDecl *Member,
+                                       const MultiInitializer &Args,
+                                       SourceLocation IdLoc);
 
   MemInitResult BuildBaseInitializer(QualType BaseType,
                                      TypeSourceInfo *BaseTInfo,
-                                     Expr **Args, unsigned NumArgs,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation RParenLoc,
+                                     const MultiInitializer &Args,
                                      CXXRecordDecl *ClassDecl,
                                      SourceLocation EllipsisLoc);
 
   MemInitResult BuildDelegatingInitializer(TypeSourceInfo *TInfo,
-                                           Expr **Args, unsigned NumArgs,
+                                           const MultiInitializer &Args,
                                            SourceLocation BaseLoc,
-                                           SourceLocation RParenLoc,
-                                           SourceLocation LParenLoc,
                                            CXXRecordDecl *ClassDecl);
 
   bool SetDelegatingInitializer(CXXConstructorDecl *Constructor,
@@ -3339,7 +3496,7 @@ class Sema {
 
   /// \brief The list of vtables that are required but have not yet been
   /// materialized.
-  llvm::SmallVector<VTableUse, 16> VTableUses;
+  SmallVector<VTableUse, 16> VTableUses;
 
   /// \brief The set of classes whose vtables have been used within
   /// this translation unit, and a bit that will be true if the vtable is
@@ -3347,9 +3504,16 @@ class Sema {
   /// by code generation).
   llvm::DenseMap<CXXRecordDecl *, bool> VTablesUsed;
 
+  /// \brief Load any externally-stored vtable uses.
+  void LoadExternalVTableUses();
+  
+  typedef LazyVector<CXXRecordDecl *, ExternalSemaSource, 
+                     &ExternalSemaSource::ReadDynamicClasses, 2, 2>
+    DynamicClassesType;
+
   /// \brief A list of all of the dynamic classes in this translation
   /// unit.
-  llvm::SmallVector<CXXRecordDecl *, 16> DynamicClasses;
+  DynamicClassesType DynamicClasses;
 
   /// \brief Note that the vtable for the given class was used at the
   /// given location.
@@ -3372,7 +3536,8 @@ class Sema {
 
   void ActOnMemInitializers(Decl *ConstructorDecl,
                             SourceLocation ColonLoc,
-                            MemInitTy **MemInits, unsigned NumMemInits,
+                            CXXCtorInitializer **MemInits,
+                            unsigned NumMemInits,
                             bool AnyErrors);
 
   void CheckCompletedCXXClass(CXXRecordDecl *Record);
@@ -3401,9 +3566,9 @@ class Sema {
   FriendDecl *CheckFriendTypeDecl(SourceLocation FriendLoc,
                                   TypeSourceInfo *TSInfo);
   Decl *ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
+                            MultiTemplateParamsArg TemplateParams);
+  Decl *ActOnFriendFunctionDecl(Scope *S, Declarator &D,
                                 MultiTemplateParamsArg TemplateParams);
-  Decl *ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
-                                    MultiTemplateParamsArg TemplateParams);
 
   QualType CheckConstructorDeclarator(Declarator &D, QualType R,
                                       StorageClass& SC);
@@ -3419,6 +3584,8 @@ class Sema {
   void CheckExplicitlyDefaultedDefaultConstructor(CXXConstructorDecl *Ctor);
   void CheckExplicitlyDefaultedCopyConstructor(CXXConstructorDecl *Ctor);
   void CheckExplicitlyDefaultedCopyAssignment(CXXMethodDecl *Method);
+  void CheckExplicitlyDefaultedMoveConstructor(CXXConstructorDecl *Ctor);
+  void CheckExplicitlyDefaultedMoveAssignment(CXXMethodDecl *Method);
   void CheckExplicitlyDefaultedDestructor(CXXDestructorDecl *Dtor);
 
   //===--------------------------------------------------------------------===//
@@ -3441,7 +3608,8 @@ class Sema {
 
   bool AttachBaseSpecifiers(CXXRecordDecl *Class, CXXBaseSpecifier **Bases,
                             unsigned NumBases);
-  void ActOnBaseSpecifiers(Decl *ClassDecl, BaseTy **Bases, unsigned NumBases);
+  void ActOnBaseSpecifiers(Decl *ClassDecl, CXXBaseSpecifier **Bases,
+                           unsigned NumBases);
 
   bool IsDerivedFrom(QualType Derived, QualType Base);
   bool IsDerivedFrom(QualType Derived, QualType Base, CXXBasePaths &Paths);
@@ -3538,6 +3706,7 @@ class Sema {
                                     bool ForceCheck = false,
                                     bool ForceUnprivileged = false);
   void CheckLookupAccess(const LookupResult &R);
+  bool IsSimplyAccessible(NamedDecl *decl, CXXRecordDecl *Class);
 
   void HandleDependentAccessCheck(const DependentDiagnostic &DD,
                          const MultiLevelTemplateArgumentList &TemplateArgs);
@@ -3626,7 +3795,7 @@ class Sema {
                                       Expr *DefaultArg);
   Decl *ActOnTemplateTemplateParameter(Scope *S,
                                        SourceLocation TmpLoc,
-                                       TemplateParamsTy *Params,
+                                       TemplateParameterList *Params,
                                        SourceLocation EllipsisLoc,
                                        IdentifierInfo *ParamName,
                                        SourceLocation ParamNameLoc,
@@ -3635,7 +3804,7 @@ class Sema {
                                        SourceLocation EqualLoc,
                                        ParsedTemplateArgument DefaultArg);
 
-  TemplateParamsTy *
+  TemplateParameterList *
   ActOnTemplateParameterList(unsigned Depth,
                              SourceLocation ExportLoc,
                              SourceLocation TemplateLoc,
@@ -3672,7 +3841,8 @@ class Sema {
                                 IdentifierInfo *Name, SourceLocation NameLoc,
                                 AttributeList *Attr,
                                 TemplateParameterList *TemplateParams,
-                                AccessSpecifier AS,
+                                AccessSpecifier AS, 
+                                SourceLocation ModulePrivateLoc,
                                 unsigned NumOuterTemplateParamLists,
                             TemplateParameterList **OuterTemplateParamLists);
 
@@ -3726,6 +3896,7 @@ class Sema {
   DeclResult
   ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec, TagUseKind TUK,
                                    SourceLocation KWLoc,
+                                   SourceLocation ModulePrivateLoc,
                                    CXXScopeSpec &SS,
                                    TemplateTy Template,
                                    SourceLocation TemplateNameLoc,
@@ -3795,7 +3966,7 @@ class Sema {
                                           SourceLocation TemplateLoc,
                                           SourceLocation RAngleLoc,
                                           Decl *Param,
-                          llvm::SmallVectorImpl<TemplateArgument> &Converted);
+                          SmallVectorImpl<TemplateArgument> &Converted);
 
   /// \brief Specifies the context in which a particular template
   /// argument is being checked.
@@ -3819,7 +3990,7 @@ class Sema {
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              unsigned ArgumentPackIndex,
-                           llvm::SmallVectorImpl<TemplateArgument> &Converted,
+                           SmallVectorImpl<TemplateArgument> &Converted,
                              CheckTemplateArgumentKind CTAK = CTAK_Specified);
 
   /// \brief Check that the given template arguments can be be provided to
@@ -3847,11 +4018,11 @@ class Sema {
                                  SourceLocation TemplateLoc,
                                  TemplateArgumentListInfo &TemplateArgs,
                                  bool PartialTemplateArgs,
-                           llvm::SmallVectorImpl<TemplateArgument> &Converted);
+                           SmallVectorImpl<TemplateArgument> &Converted);
 
   bool CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
                                  const TemplateArgumentLoc &Arg,
-                           llvm::SmallVectorImpl<TemplateArgument> &Converted);
+                           SmallVectorImpl<TemplateArgument> &Converted);
 
   bool CheckTemplateArgument(TemplateTypeParmDecl *Param,
                              TypeSourceInfo *Arg);
@@ -3963,7 +4134,9 @@ class Sema {
   bool RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS);
 
   ExprResult RebuildExprInCurrentInstantiation(Expr *E);
-
+  bool RebuildTemplateParamsInCurrentInstantiation(
+                                                TemplateParameterList *Params);
+  
   std::string
   getTemplateArgumentBindingsText(const TemplateParameterList *Params,
                                   const TemplateArgumentList &Args);
@@ -4103,7 +4276,7 @@ class Sema {
   /// \param Arg The template argument that will be traversed to find
   /// unexpanded parameter packs.
   void collectUnexpandedParameterPacks(TemplateArgument Arg,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
 
   /// \brief Collect the set of unexpanded parameter packs within the given
   /// template argument.  
@@ -4111,7 +4284,7 @@ class Sema {
   /// \param Arg The template argument that will be traversed to find
   /// unexpanded parameter packs.
   void collectUnexpandedParameterPacks(TemplateArgumentLoc Arg,
-                    llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
+                    SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
 
   /// \brief Collect the set of unexpanded parameter packs within the given
   /// type.  
@@ -4119,7 +4292,7 @@ class Sema {
   /// \param T The type that will be traversed to find
   /// unexpanded parameter packs.
   void collectUnexpandedParameterPacks(QualType T,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
 
   /// \brief Collect the set of unexpanded parameter packs within the given
   /// type.  
@@ -4127,7 +4300,7 @@ class Sema {
   /// \param TL The type that will be traversed to find
   /// unexpanded parameter packs.
   void collectUnexpandedParameterPacks(TypeLoc TL,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
 
   /// \brief Invoked when parsing a template argument followed by an
   /// ellipsis, which creates a pack expansion.
@@ -4219,8 +4392,7 @@ class Sema {
   /// must be set.
   bool CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
                                        SourceRange PatternRange,
-                                     const UnexpandedParameterPack *Unexpanded,
-                                       unsigned NumUnexpanded,
+                             llvm::ArrayRef<UnexpandedParameterPack> Unexpanded,
                              const MultiLevelTemplateArgumentList &TemplateArgs,
                                        bool &ShouldExpand,
                                        bool &RetainExpansion,
@@ -4312,8 +4484,8 @@ class Sema {
   TemplateDeductionResult
   SubstituteExplicitTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
                               TemplateArgumentListInfo &ExplicitTemplateArgs,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                                 llvm::SmallVectorImpl<QualType> &ParamTypes,
+                      SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                                 SmallVectorImpl<QualType> &ParamTypes,
                                       QualType *FunctionType,
                                       sema::TemplateDeductionInfo &Info);
 
@@ -4333,11 +4505,11 @@ class Sema {
   
   TemplateDeductionResult
   FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                      SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                                   unsigned NumExplicitlySpecified,
                                   FunctionDecl *&Specialization,
                                   sema::TemplateDeductionInfo &Info,
-           llvm::SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs = 0);
+           SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs = 0);
 
   TemplateDeductionResult
   DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
@@ -4392,9 +4564,9 @@ class Sema {
   void MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs,
                                   bool OnlyDeduced,
                                   unsigned Depth,
-                                  llvm::SmallVectorImpl<bool> &Used);
+                                  SmallVectorImpl<bool> &Used);
   void MarkDeducedTemplateParameters(FunctionTemplateDecl *FunctionTemplate,
-                                     llvm::SmallVectorImpl<bool> &Deduced);
+                                     SmallVectorImpl<bool> &Deduced);
 
   //===--------------------------------------------------------------------===//
   // C++ Template Instantiation
@@ -4523,7 +4695,7 @@ class Sema {
   /// requires another template instantiation, additional
   /// instantiations are pushed onto the stack up to a
   /// user-configurable limit LangOptions::InstantiationDepth.
-  llvm::SmallVector<ActiveTemplateInstantiation, 16>
+  SmallVector<ActiveTemplateInstantiation, 16>
     ActiveTemplateInstantiations;
 
   /// \brief Whether we are in a SFINAE context that is not associated with
@@ -4580,14 +4752,14 @@ class Sema {
   ///
   /// The top of this stack is used by a fixit instantiating unresolved
   /// function calls to fix the AST to match the textual change it prints.
-  llvm::SmallVector<CallExpr *, 8> CallsUndergoingInstantiation;
+  SmallVector<CallExpr *, 8> CallsUndergoingInstantiation;
   
   /// \brief For each declaration that involved template argument deduction, the
   /// set of diagnostics that were suppressed during that template argument
   /// deduction.
   ///
   /// FIXME: Serialize this structure to the AST file.
-  llvm::DenseMap<Decl *, llvm::SmallVector<PartialDiagnosticAt, 1> >
+  llvm::DenseMap<Decl *, SmallVector<PartialDiagnosticAt, 1> >
     SuppressedDiagnostics;
   
   /// \brief A stack object to be created when performing template
@@ -4691,7 +4863,7 @@ class Sema {
   };
 
   void PrintInstantiationStack();
-
+  
   /// \brief Determines whether we are currently in a context where
   /// template argument substitution failures are not considered
   /// errors.
@@ -4805,8 +4977,8 @@ class Sema {
   bool SubstParmTypes(SourceLocation Loc, 
                       ParmVarDecl **Params, unsigned NumParams,
                       const MultiLevelTemplateArgumentList &TemplateArgs,
-                      llvm::SmallVectorImpl<QualType> &ParamTypes,
-                      llvm::SmallVectorImpl<ParmVarDecl *> *OutParams = 0);
+                      SmallVectorImpl<QualType> &ParamTypes,
+                      SmallVectorImpl<ParmVarDecl *> *OutParams = 0);
   ExprResult SubstExpr(Expr *E,
                        const MultiLevelTemplateArgumentList &TemplateArgs);
   
@@ -4827,7 +4999,7 @@ class Sema {
   /// \returns true if an error occurred, false otherwise.
   bool SubstExprs(Expr **Exprs, unsigned NumExprs, bool IsCall,
                   const MultiLevelTemplateArgumentList &TemplateArgs,
-                  llvm::SmallVectorImpl<Expr *> &Outputs);
+                  SmallVectorImpl<Expr *> &Outputs);
 
   StmtResult SubstStmt(Stmt *S,
                        const MultiLevelTemplateArgumentList &TemplateArgs);
@@ -4894,6 +5066,11 @@ class Sema {
   void InstantiateMemInitializers(CXXConstructorDecl *New,
                                   const CXXConstructorDecl *Tmpl,
                             const MultiLevelTemplateArgumentList &TemplateArgs);
+  bool InstantiateInitializer(Expr *Init,
+                            const MultiLevelTemplateArgumentList &TemplateArgs,
+                              SourceLocation &LParenLoc,
+                              ASTOwningVector<Expr*> &NewArgs,
+                              SourceLocation &RParenLoc);
 
   NamedDecl *FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
                           const MultiLevelTemplateArgumentList &TemplateArgs);
@@ -4952,7 +5129,7 @@ class Sema {
                                          IdentifierInfo *CatName,
                                          SourceLocation CatLoc);
 
-  Decl *ActOnForwardClassDeclaration(SourceLocation Loc,
+  DeclGroupPtrTy ActOnForwardClassDeclaration(SourceLocation Loc,
                                      IdentifierInfo **IdentList,
                                      SourceLocation *IdentLocs,
                                      unsigned NumElts);
@@ -4965,7 +5142,7 @@ class Sema {
   void FindProtocolDeclaration(bool WarnOnDeclarations,
                                const IdentifierLocPair *ProtocolId,
                                unsigned NumProtocols,
-                               llvm::SmallVectorImpl<Decl *> &Protocols);
+                               SmallVectorImpl<Decl *> &Protocols);
 
   /// Ensure attributes are consistent with type.
   /// \param [in, out] Attributes The attributes to check; they will
@@ -5003,7 +5180,7 @@ class Sema {
   void MatchOneProtocolPropertiesInClass(Decl *CDecl,
                                          ObjCProtocolDecl *PDecl);
 
-  void ActOnAtEnd(Scope *S, SourceRange AtEnd, Decl *classDecl,
+  void ActOnAtEnd(Scope *S, SourceRange AtEnd,
                   Decl **allMethods = 0, unsigned allNum = 0,
                   Decl **allProperties = 0, unsigned pNum = 0,
                   DeclGroupPtrTy *allTUVars = 0, unsigned tuvNum = 0);
@@ -5011,7 +5188,6 @@ class Sema {
   Decl *ActOnProperty(Scope *S, SourceLocation AtLoc,
                       FieldDeclarator &FD, ObjCDeclSpec &ODS,
                       Selector GetterSel, Selector SetterSel,
-                      Decl *ClassCategory,
                       bool *OverridingProperty,
                       tok::ObjCKeywordKind MethodImplKind,
                       DeclContext *lexicalDC = 0);
@@ -5019,7 +5195,7 @@ class Sema {
   Decl *ActOnPropertyImplDecl(Scope *S,
                               SourceLocation AtLoc,
                               SourceLocation PropertyLoc,
-                              bool ImplKind,Decl *ClassImplDecl,
+                              bool ImplKind,
                               IdentifierInfo *PropertyId,
                               IdentifierInfo *PropertyIvar,
                               SourceLocation PropertyIvarLoc);
@@ -5050,8 +5226,8 @@ class Sema {
     SourceLocation BeginLoc, // location of the + or -.
     SourceLocation EndLoc,   // location of the ; or {.
     tok::TokenKind MethodType,
-    Decl *ClassDecl, ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
-    SourceLocation SelectorStartLoc, Selector Sel,
+    ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
+    ArrayRef<SourceLocation> SelectorLocs, Selector Sel,
     // optional arguments. The number of types/arguments is obtained
     // from the Sel.getNumArgs().
     ObjCArgInfo *ArgInfo,
@@ -5112,7 +5288,7 @@ class Sema {
   ExprResult ActOnSuperMessage(Scope *S, SourceLocation SuperLoc,
                                Selector Sel,
                                SourceLocation LBracLoc,
-                               SourceLocation SelectorLoc,
+                               ArrayRef<SourceLocation> SelectorLocs,
                                SourceLocation RBracLoc,
                                MultiExprArg Args);
 
@@ -5122,7 +5298,7 @@ class Sema {
                                Selector Sel,
                                ObjCMethodDecl *Method,
                                SourceLocation LBracLoc,
-                               SourceLocation SelectorLoc,
+                               ArrayRef<SourceLocation> SelectorLocs,
                                SourceLocation RBracLoc,
                                MultiExprArg Args);
 
@@ -5130,7 +5306,7 @@ class Sema {
                                ParsedType Receiver,
                                Selector Sel,
                                SourceLocation LBracLoc,
-                               SourceLocation SelectorLoc,
+                               ArrayRef<SourceLocation> SelectorLocs,
                                SourceLocation RBracLoc,
                                MultiExprArg Args);
 
@@ -5140,7 +5316,7 @@ class Sema {
                                   Selector Sel,
                                   ObjCMethodDecl *Method,
                                   SourceLocation LBracLoc,
-                                  SourceLocation SelectorLoc,
+                                  ArrayRef<SourceLocation> SelectorLocs,
                                   SourceLocation RBracLoc,
                                   MultiExprArg Args);
 
@@ -5148,7 +5324,7 @@ class Sema {
                                   Expr *Receiver,
                                   Selector Sel,
                                   SourceLocation LBracLoc,
-                                  SourceLocation SelectorLoc,
+                                  ArrayRef<SourceLocation> SelectorLocs,
                                   SourceLocation RBracLoc,
                                   MultiExprArg Args);
 
@@ -5170,9 +5346,7 @@ class Sema {
   
   /// \brief Check whether the given new method is a valid override of the
   /// given overridden method, and set any properties that should be inherited.
-  ///
-  /// \returns True if an error occurred.
-  bool CheckObjCMethodOverride(ObjCMethodDecl *NewMethod, 
+  void CheckObjCMethodOverride(ObjCMethodDecl *NewMethod, 
                                const ObjCMethodDecl *Overridden,
                                bool IsImplementation);
 
@@ -5226,7 +5400,8 @@ class Sema {
   void ActOnPragmaVisibility(bool IsPush, const IdentifierInfo* VisType,
                              SourceLocation PragmaLoc);
 
-  NamedDecl *DeclClonePragmaWeak(NamedDecl *ND, IdentifierInfo *II);
+  NamedDecl *DeclClonePragmaWeak(NamedDecl *ND, IdentifierInfo *II,
+                                 SourceLocation Loc);
   void DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, WeakInfo &W);
 
   /// ActOnPragmaWeakID - Called on well formed #pragma weak ident.
@@ -5270,13 +5445,14 @@ class Sema {
   /// FreeVisContext - Deallocate and null out VisContext.
   void FreeVisContext();
 
-  /// AddAlignedAttr - Adds an aligned attribute to a particular declaration.
-  void AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E);
-  void AddAlignedAttr(SourceLocation AttrLoc, Decl *D, TypeSourceInfo *T);
+  /// AddCFAuditedAttribute - Check whether we're currently within
+  /// '#pragma clang arc_cf_code_audited' and, if so, consider adding
+  /// the appropriate attribute.
+  void AddCFAuditedAttribute(Decl *D);
 
-  /// CastCategory - Get the correct forwarded implicit cast result category
-  /// from the inner expression.
-  ExprValueKind CastCategory(Expr *E);
+  /// AddAlignedAttr - Adds an aligned attribute to a particular declaration.
+  void AddAlignedAttr(SourceRange AttrRange, Decl *D, Expr *E);
+  void AddAlignedAttr(SourceRange AttrRange, Decl *D, TypeSourceInfo *T);
 
   /// \brief The kind of conversion being performed.
   enum CheckedConversionKind {
@@ -5348,7 +5524,7 @@ class Sema {
                               const FunctionProtoType *Proto,
                               unsigned FirstProtoArg,
                               Expr **Args, unsigned NumArgs,
-                              llvm::SmallVector<Expr *, 8> &AllArgs,
+                              SmallVector<Expr *, 8> &AllArgs,
                               VariadicCallType CallType = VariadicDoesNotApply);
 
   // DefaultVariadicArgumentPromotion - Like DefaultArgumentPromotion, but
@@ -5361,8 +5537,8 @@ class Sema {
   // operators (C99 6.3.1.8). If both operands aren't arithmetic, this
   // routine returns the first non-arithmetic type found. The client is
   // responsible for emitting appropriate error diagnostics.
-  QualType UsualArithmeticConversions(ExprResult &lExpr, ExprResult &rExpr,
-                                      bool isCompAssign = false);
+  QualType UsualArithmeticConversions(ExprResult &LHS, ExprResult &RHS,
+                                      bool IsCompAssign = false);
 
   /// AssignConvertType - All of the 'assignment' semantic checks return this
   /// enum to indicate whether the assignment was allowed.  These checks are
@@ -5449,23 +5625,26 @@ class Sema {
   /// argument passing, variable initialization, and function return values.
   /// C99 6.5.16.
   AssignConvertType CheckAssignmentConstraints(SourceLocation Loc,
-                                               QualType lhs, QualType rhs);
+                                               QualType LHSType,
+                                               QualType RHSType);
 
   /// Check assignment constraints and prepare for a conversion of the
   /// RHS to the LHS type.
-  AssignConvertType CheckAssignmentConstraints(QualType lhs, ExprResult &rhs,
+  AssignConvertType CheckAssignmentConstraints(QualType LHSType,
+                                               ExprResult &RHS,
                                                CastKind &Kind);
 
   // CheckSingleAssignmentConstraints - Currently used by
   // CheckAssignmentOperands, and ActOnReturnStmt. Prior to type checking,
   // this routine performs the default function/array converions.
-  AssignConvertType CheckSingleAssignmentConstraints(QualType lhs,
-                                                     ExprResult &rExprRes);
+  AssignConvertType CheckSingleAssignmentConstraints(QualType LHSType,
+                                                     ExprResult &RHS,
+                                                     bool Diagnose = true);
 
   // \brief If the lhs type is a transparent union, check whether we
   // can initialize the transparent union with the given expression.
-  AssignConvertType CheckTransparentUnionArgumentConstraints(QualType lhs,
-                                                             ExprResult &rExpr);
+  AssignConvertType CheckTransparentUnionArgumentConstraints(QualType ArgType,
+                                                             ExprResult &RHS);
 
   bool IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType);
 
@@ -5473,11 +5652,13 @@ class Sema {
 
   ExprResult PerformImplicitConversion(Expr *From, QualType ToType,
                                        AssignmentAction Action,
-                                       bool AllowExplicit = false);
+                                       bool AllowExplicit = false,
+                                       bool Diagnose = true);
   ExprResult PerformImplicitConversion(Expr *From, QualType ToType,
                                        AssignmentAction Action,
                                        bool AllowExplicit,
-                                       ImplicitConversionSequence& ICS);
+                                       ImplicitConversionSequence& ICS,
+                                       bool Diagnose = true);
   ExprResult PerformImplicitConversion(Expr *From, QualType ToType,
                                        const ImplicitConversionSequence& ICS,
                                        AssignmentAction Action,
@@ -5492,41 +5673,47 @@ class Sema {
   /// or a null QualType (indicating an error diagnostic was issued).
 
   /// type checking binary operators (subroutines of CreateBuiltinBinOp).
-  QualType InvalidOperands(SourceLocation l, ExprResult &lex, ExprResult &rex);
+  QualType InvalidOperands(SourceLocation Loc, ExprResult &LHS,
+                           ExprResult &RHS);
   QualType CheckPointerToMemberOperands( // C++ 5.5
-    ExprResult &lex, ExprResult &rex, ExprValueKind &VK,
+    ExprResult &LHS, ExprResult &RHS, ExprValueKind &VK,
     SourceLocation OpLoc, bool isIndirect);
   QualType CheckMultiplyDivideOperands( // C99 6.5.5
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, bool isCompAssign,
-                                       bool isDivide);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, bool IsCompAssign,
+    bool IsDivide);
   QualType CheckRemainderOperands( // C99 6.5.5
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, bool isCompAssign = false);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
+    bool IsCompAssign = false);
   QualType CheckAdditionOperands( // C99 6.5.6
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, QualType* CompLHSTy = 0);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
+    QualType* CompLHSTy = 0);
   QualType CheckSubtractionOperands( // C99 6.5.6
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, QualType* CompLHSTy = 0);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
+    QualType* CompLHSTy = 0);
   QualType CheckShiftOperands( // C99 6.5.7
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, unsigned Opc,
-    bool isCompAssign = false);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, unsigned Opc,
+    bool IsCompAssign = false);
   QualType CheckCompareOperands( // C99 6.5.8/9
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, unsigned Opc,
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, unsigned OpaqueOpc,
                                 bool isRelational);
   QualType CheckBitwiseOperands( // C99 6.5.[10...12]
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, bool isCompAssign = false);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
+    bool IsCompAssign = false);
   QualType CheckLogicalOperands( // C99 6.5.[13,14]
-    ExprResult &lex, ExprResult &rex, SourceLocation OpLoc, unsigned Opc);
+    ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, unsigned Opc);
   // CheckAssignmentOperands is used for both simple and compound assignment.
   // For simple assignment, pass both expressions and a null converted type.
   // For compound assignment, pass both expressions and the converted type.
   QualType CheckAssignmentOperands( // C99 6.5.16.[1,2]
-    Expr *lex, ExprResult &rex, SourceLocation OpLoc, QualType convertedType);
+    Expr *LHSExpr, ExprResult &RHS, SourceLocation Loc, QualType CompoundType);
   
-  void ConvertPropertyForLValue(ExprResult &LHS, ExprResult &RHS, QualType& LHSTy);
+  void ConvertPropertyForLValue(ExprResult &LHS, ExprResult &RHS,
+                                QualType& LHSTy);
   ExprResult ConvertPropertyForRValue(Expr *E);
                                    
   QualType CheckConditionalOperands( // C99 6.5.15
-    ExprResult &cond, ExprResult &lhs, ExprResult &rhs,
-    ExprValueKind &VK, ExprObjectKind &OK, SourceLocation questionLoc);
+    ExprResult &Cond, ExprResult &LHS, ExprResult &RHS,
+    ExprValueKind &VK, ExprObjectKind &OK, SourceLocation QuestionLoc);
   QualType CXXCheckConditionalOperands( // C++ 5.16
     ExprResult &cond, ExprResult &lhs, ExprResult &rhs,
     ExprValueKind &VK, ExprObjectKind &OK, SourceLocation questionLoc);
@@ -5542,20 +5729,18 @@ class Sema {
   }
 
   QualType FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
-                                        SourceLocation questionLoc);
+                                        SourceLocation QuestionLoc);
 
-  bool DiagnoseConditionalForNull(Expr *LHS, Expr *RHS,
+  bool DiagnoseConditionalForNull(Expr *LHSExpr, Expr *RHSExpr,
                                   SourceLocation QuestionLoc);
 
   /// type checking for vector binary operators.
-  QualType CheckVectorOperands(ExprResult &lex, ExprResult &rex,
-                               SourceLocation Loc, bool isCompAssign);
-  QualType CheckVectorCompareOperands(ExprResult &lex, ExprResult &rx,
-                                      SourceLocation l, bool isRel);
+  QualType CheckVectorOperands(ExprResult &LHS, ExprResult &RHS,
+                               SourceLocation Loc, bool IsCompAssign);
+  QualType CheckVectorCompareOperands(ExprResult &LHS, ExprResult &RHS,
+                                      SourceLocation Loc, bool isRelational);
 
   /// type checking declaration initializers (C99 6.7.8)
-  bool CheckInitList(const InitializedEntity &Entity,
-                     InitListExpr *&InitList, QualType &DeclType);
   bool CheckForConstantInitializer(Expr *e, QualType t);
 
   // type checking C++ declaration initializers (C++ [dcl.init]).
@@ -5587,16 +5772,9 @@ class Sema {
                                                       bool &ObjCConversion,
                                                 bool &ObjCLifetimeConversion);
 
-  /// CheckCastTypes - Check type constraints for casting between types under
-  /// C semantics, or forward to CXXCheckCStyleCast in C++.
-  ExprResult CheckCastTypes(SourceLocation CastStartLoc, SourceRange TyRange, 
-                            QualType CastTy, Expr *CastExpr, CastKind &Kind, 
-                            ExprValueKind &VK, CXXCastPath &BasePath,
-                            bool FunctionalStyle = false);
-
-  ExprResult checkUnknownAnyCast(SourceRange TyRange, QualType castType,
-                                 Expr *castExpr, CastKind &castKind,
-                                 ExprValueKind &valueKind, CXXCastPath &BasePath);
+  ExprResult checkUnknownAnyCast(SourceRange TypeRange, QualType CastType,
+                                 Expr *CastExpr, CastKind &CastKind,
+                                 ExprValueKind &VK, CXXCastPath &Path);
 
   // CheckVectorCast - check type constraints for vectors.
   // Since vectors are an extension, there are no C standard reference for this.
@@ -5610,19 +5788,14 @@ class Sema {
   // We allow casting between vectors and integer datatypes of the same size,
   // or vectors and the element type of that vector.
   // returns the cast expr
-  ExprResult CheckExtVectorCast(SourceRange R, QualType VectorTy, Expr *CastExpr,
+  ExprResult CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *CastExpr,
                                 CastKind &Kind);
 
-  /// CXXCheckCStyleCast - Check constraints of a C-style or function-style
-  /// cast under C++ semantics.
-  ExprResult CXXCheckCStyleCast(SourceRange R, QualType CastTy, ExprValueKind &VK,
-                                Expr *CastExpr, CastKind &Kind,
-                                CXXCastPath &BasePath, bool FunctionalStyle);
-    
-  /// \brief Checks for valid expressions which can be cast to an ObjC
-  /// pointer without needing a bridge cast.
-  bool ValidObjCARCNoBridgeCastExpr(Expr *&Exp, QualType castType);
-    
+  ExprResult BuildCXXFunctionalCastExpr(TypeSourceInfo *TInfo,
+                                        SourceLocation LParenLoc,
+                                        Expr *CastExpr,
+                                        SourceLocation RParenLoc);
+
   /// \brief Checks for invalid conversions and casts between
   /// retainable pointers and other pointer kinds.
   void CheckObjCARCConversion(SourceRange castRange, QualType castType, 
@@ -5674,10 +5847,10 @@ class Sema {
   /// \param Loc - A location associated with the condition, e.g. the
   /// 'if' keyword.
   /// \return true iff there were any errors
-  ExprResult CheckBooleanCondition(Expr *CondExpr, SourceLocation Loc);
+  ExprResult CheckBooleanCondition(Expr *E, SourceLocation Loc);
 
   ExprResult ActOnBooleanCondition(Scope *S, SourceLocation Loc,
-                                           Expr *SubExpr);
+                                   Expr *SubExpr);
   
   /// DiagnoseAssignmentAsCondition - Given that an expression is
   /// being used as a boolean condition, warn if it's an assignment.
@@ -5685,7 +5858,7 @@ class Sema {
 
   /// \brief Redundant parentheses over an equality comparison can indicate
   /// that the user intended an assignment used as condition.
-  void DiagnoseEqualityWithExtraParens(ParenExpr *parenE);
+  void DiagnoseEqualityWithExtraParens(ParenExpr *ParenE);
 
   /// CheckCXXBooleanCondition - Returns true if conversion to bool is invalid.
   ExprResult CheckCXXBooleanCondition(Expr *CondExpr);
@@ -5715,6 +5888,23 @@ class Sema {
                       QualType FieldTy, const Expr *BitWidth,
                       bool *ZeroWidth = 0);
 
+  enum CUDAFunctionTarget {
+    CFT_Device,
+    CFT_Global,
+    CFT_Host,
+    CFT_HostDevice
+  };
+
+  CUDAFunctionTarget IdentifyCUDATarget(const FunctionDecl *D);
+
+  bool CheckCUDATarget(CUDAFunctionTarget CallerTarget,
+                       CUDAFunctionTarget CalleeTarget);
+
+  bool CheckCUDATarget(const FunctionDecl *Caller, const FunctionDecl *Callee) {
+    return CheckCUDATarget(IdentifyCUDATarget(Caller),
+                           IdentifyCUDATarget(Callee));
+  }
+
   /// \name Code completion
   //@{
   /// \brief Describes the context in which code completion occurs.
@@ -5782,6 +5972,7 @@ class Sema {
   void CodeCompleteCall(Scope *S, Expr *Fn, Expr **Args, unsigned NumArgs);
   void CodeCompleteInitializer(Scope *S, Decl *D);
   void CodeCompleteReturn(Scope *S);
+  void CodeCompleteAfterIf(Scope *S);
   void CodeCompleteAssignmentRHS(Scope *S, Expr *LHS);
   
   void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
@@ -5795,14 +5986,13 @@ class Sema {
                                           CXXCtorInitializer** Initializers,
                                           unsigned NumInitializers);
   
-  void CodeCompleteObjCAtDirective(Scope *S, Decl *ObjCImpDecl,
-                                   bool InInterface);
+  void CodeCompleteObjCAtDirective(Scope *S);
   void CodeCompleteObjCAtVisibility(Scope *S);
   void CodeCompleteObjCAtStatement(Scope *S);
   void CodeCompleteObjCAtExpression(Scope *S);
   void CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS);
-  void CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl);
-  void CodeCompleteObjCPropertySetter(Scope *S, Decl *ClassDecl);
+  void CodeCompleteObjCPropertyGetter(Scope *S);
+  void CodeCompleteObjCPropertySetter(Scope *S);
   void CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS, 
                                    bool IsParameter);
   void CodeCompleteObjCMessageReceiver(Scope *S);
@@ -5815,7 +6005,7 @@ class Sema {
                                     unsigned NumSelIdents,
                                     bool AtArgumentExpression,
                                     bool IsSuper = false);
-  void CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
+  void CodeCompleteObjCInstanceMessage(Scope *S, Expr *Receiver,
                                        IdentifierInfo **SelIdents,
                                        unsigned NumSelIdents,
                                        bool AtArgumentExpression,
@@ -5839,14 +6029,12 @@ class Sema {
   void CodeCompleteObjCImplementationCategory(Scope *S,
                                               IdentifierInfo *ClassName,
                                               SourceLocation ClassNameLoc);
-  void CodeCompleteObjCPropertyDefinition(Scope *S, Decl *ObjCImpDecl);
+  void CodeCompleteObjCPropertyDefinition(Scope *S);
   void CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
-                                              IdentifierInfo *PropertyName,
-                                              Decl *ObjCImpDecl);
+                                              IdentifierInfo *PropertyName);
   void CodeCompleteObjCMethodDecl(Scope *S,
                                   bool IsInstanceMethod,
-                                  ParsedType ReturnType,
-                                  Decl *IDecl);
+                                  ParsedType ReturnType);
   void CodeCompleteObjCMethodDeclSelector(Scope *S, 
                                           bool IsInstanceMethod,
                                           bool AtParameterName,
@@ -5863,7 +6051,7 @@ class Sema {
                                              unsigned Argument);
   void CodeCompleteNaturalLanguage();
   void GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,
-                  llvm::SmallVectorImpl<CodeCompletionResult> &Results);
+                  SmallVectorImpl<CodeCompletionResult> &Results);
   //@}
 
   //===--------------------------------------------------------------------===//
@@ -5874,6 +6062,8 @@ class Sema {
                                                 unsigned ByteNo) const;
 
 private:  
+  void CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr,
+                        bool isSubscript=false, bool AllowOnePastEnd=true);
   void CheckArrayAccess(const Expr *E);
   bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall);
   bool CheckBlockCall(NamedDecl *NDecl, CallExpr *TheCall);
@@ -5897,6 +6087,8 @@ class Sema {
   bool SemaBuiltinObjectSize(CallExpr *TheCall);
   bool SemaBuiltinLongjmp(CallExpr *TheCall);
   ExprResult SemaBuiltinAtomicOverloaded(ExprResult TheCallResult);
+  ExprResult SemaAtomicOpsOverloaded(ExprResult TheCallResult,
+                                     AtomicExpr::AtomicOp Op);
   bool SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum,
                               llvm::APSInt &Result);
 
@@ -5918,20 +6110,28 @@ class Sema {
                                  bool isPrintf);
 
   /// \brief Enumeration used to describe which of the memory setting or copying
-  /// functions is being checked by \c CheckMemsetcpymoveArguments().
+  /// functions is being checked by \c CheckMemaccessArguments().
   enum CheckedMemoryFunction {
     CMF_Memset,
     CMF_Memcpy,
-    CMF_Memmove
+    CMF_Memmove,
+    CMF_Memcmp,
+    CMF_Strncpy,
+    CMF_Strncmp,
+    CMF_Strncasecmp,
+    CMF_Strncat,
+    CMF_Strndup
   };
   
-  void CheckMemsetcpymoveArguments(const CallExpr *Call, 
-                                   CheckedMemoryFunction CMF,
-                                   IdentifierInfo *FnName);
+  void CheckMemaccessArguments(const CallExpr *Call, CheckedMemoryFunction CMF,
+                               IdentifierInfo *FnName);
+
+  void CheckStrlcpycatArguments(const CallExpr *Call,
+                                IdentifierInfo *FnName);
 
   void CheckReturnStackAddr(Expr *RetValExp, QualType lhsType,
                             SourceLocation ReturnLoc);
-  void CheckFloatComparison(SourceLocation loc, Expr* lex, Expr* rex);
+  void CheckFloatComparison(SourceLocation Loc, Expr* LHS, Expr* RHS);
   void CheckImplicitConversions(Expr *E, SourceLocation CC = SourceLocation());
 
   void CheckBitFieldInitialization(SourceLocation InitLoc, FieldDecl *Field,
@@ -5958,6 +6158,14 @@ class Sema {
   /// itself and in routines directly invoked from the parser and *never* from
   /// template substitution or instantiation.
   Scope *getCurScope() const { return CurScope; }
+    
+  Decl *getObjCDeclContext() const;
+
+  DeclContext *getCurLexicalContext() const {
+    return OriginalLexicalContext ? OriginalLexicalContext : CurContext;
+  }
+
+  AvailabilityResult getCurContextAvailability() const;
 };
 
 /// \brief RAII object that enters a new expression evaluation context.
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/SemaDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Sema/SemaDiagnostic.h
index 83c0999ad49c..2c4bf4b0d1b7 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/SemaDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/SemaDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define SEMASTART
 #include "clang/Basic/DiagnosticSemaKinds.inc"
 #undef DIAG
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/SemaFixItUtils.h b/contrib/llvm/tools/clang/include/clang/Sema/SemaFixItUtils.h
new file mode 100644
index 000000000000..0c1bba5078fc
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Sema/SemaFixItUtils.h
@@ -0,0 +1,91 @@
+//===--- SemaFixItUtils.h - Sema FixIts -----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines helper classes for generation of Sema FixItHints.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_SEMA_FIXITUTILS_H
+#define LLVM_CLANG_SEMA_FIXITUTILS_H
+
+#include "clang/AST/Expr.h"
+
+namespace clang {
+
+enum OverloadFixItKind {
+  OFIK_Undefined = 0,
+  OFIK_Dereference,
+  OFIK_TakeAddress,
+  OFIK_RemoveDereference,
+  OFIK_RemoveTakeAddress
+};
+
+class Sema;
+
+/// The class facilities generation and storage of conversion FixIts. Hints for
+/// new conversions are added using TryToFixConversion method. The default type
+/// conversion checker can be reset.
+struct ConversionFixItGenerator {
+  /// Performs a simple check to see if From type can be converted to To type.
+  static bool compareTypesSimple(CanQualType From,
+                                 CanQualType To,
+                                 Sema &S,
+                                 SourceLocation Loc,
+                                 ExprValueKind FromVK);
+
+  /// The list of Hints generated so far.
+  SmallVector<FixItHint, 1> Hints;
+
+  /// The number of Conversions fixed. This can be different from the size
+  /// of the Hints vector since we allow multiple FixIts per conversion.
+  unsigned NumConversionsFixed;
+
+  /// The type of fix applied. If multiple conversions are fixed, corresponds
+  /// to the kid of the very first conversion.
+  OverloadFixItKind Kind;
+
+  typedef bool (*TypeComparisonFuncTy) (const CanQualType FromTy,
+                                        const CanQualType ToTy,
+                                        Sema &S,
+                                        SourceLocation Loc,
+                                        ExprValueKind FromVK);
+  /// The type comparison function used to decide if expression FromExpr of
+  /// type FromTy can be converted to ToTy. For example, one could check if
+  /// an implicit conversion exists. Returns true if comparison exists.
+  TypeComparisonFuncTy CompareTypes;
+
+  ConversionFixItGenerator(TypeComparisonFuncTy Foo): NumConversionsFixed(0),
+                                                      Kind(OFIK_Undefined),
+                                                      CompareTypes(Foo) {}
+
+  ConversionFixItGenerator(): NumConversionsFixed(0),
+                              Kind(OFIK_Undefined),
+                              CompareTypes(compareTypesSimple) {}
+
+  /// Resets the default conversion checker method.
+  void setConversionChecker(TypeComparisonFuncTy Foo) {
+    CompareTypes = Foo;
+  }
+
+  /// If possible, generates and stores a fix for the given conversion.
+  bool tryToFixConversion(const Expr *FromExpr,
+                          const QualType FromQTy, const QualType ToQTy,
+                          Sema &S);
+
+  void clear() {
+    Hints.clear();
+    NumConversionsFixed = 0;
+  }
+
+  bool isNull() {
+    return (NumConversionsFixed == 0);
+  }
+};
+
+} // endof namespace clang
+#endif // LLVM_CLANG_SEMA_FIXITUTILS_H
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Template.h b/contrib/llvm/tools/clang/include/clang/Sema/Template.h
index a257772ee102..78f50fa9ffa4 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/Template.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Template.h
@@ -45,7 +45,7 @@ namespace clang {
   private:
     /// \brief The template argument lists, stored from the innermost template
     /// argument list (first) to the outermost template argument list (last).
-    llvm::SmallVector<ArgList, 4> TemplateArgumentLists;
+    SmallVector<ArgList, 4> TemplateArgumentLists;
     
   public:
     /// \brief Construct an empty set of template argument lists.
@@ -178,7 +178,7 @@ namespace clang {
   class LocalInstantiationScope {
   public:
     /// \brief A set of declarations.
-    typedef llvm::SmallVector<Decl *, 4> DeclArgumentPack;
+    typedef SmallVector<Decl *, 4> DeclArgumentPack;
     
   private:
     /// \brief Reference to the semantic analysis that is performing
@@ -210,7 +210,7 @@ namespace clang {
     LocalDeclsMap LocalDecls;
 
     /// \brief The set of argument packs we've allocated.
-    llvm::SmallVector<DeclArgumentPack *, 1> ArgumentPacks;
+    SmallVector<DeclArgumentPack *, 1> ArgumentPacks;
     
     /// \brief The outer scope, which contains local variable
     /// definitions from some other instantiation (that may not be
@@ -318,7 +318,7 @@ namespace clang {
     /// \brief A list of out-of-line class template partial
     /// specializations that will need to be instantiated after the
     /// enclosing class's instantiation is complete.
-    llvm::SmallVector<std::pair<ClassTemplateDecl *,
+    SmallVector<std::pair<ClassTemplateDecl *,
                                 ClassTemplatePartialSpecializationDecl *>, 4>
       OutOfLinePartialSpecs;
 
@@ -350,7 +350,8 @@ namespace clang {
                             TemplateParameterList *TemplateParams = 0);
     Decl *VisitCXXRecordDecl(CXXRecordDecl *D);
     Decl *VisitCXXMethodDecl(CXXMethodDecl *D,
-                             TemplateParameterList *TemplateParams = 0);
+                             TemplateParameterList *TemplateParams = 0,
+                             bool IsClassScopeSpecialization = false);
     Decl *VisitCXXConstructorDecl(CXXConstructorDecl *D);
     Decl *VisitCXXDestructorDecl(CXXDestructorDecl *D);
     Decl *VisitCXXConversionDecl(CXXConversionDecl *D);
@@ -367,11 +368,13 @@ namespace clang {
     Decl *VisitUsingShadowDecl(UsingShadowDecl *D);
     Decl *VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D);
     Decl *VisitUnresolvedUsingTypenameDecl(UnresolvedUsingTypenameDecl *D);
+    Decl *VisitClassScopeFunctionSpecializationDecl(
+                                      ClassScopeFunctionSpecializationDecl *D);
 
     // Base case. FIXME: Remove once we can instantiate everything.
     Decl *VisitDecl(Decl *D) {
       unsigned DiagID = SemaRef.getDiagnostics().getCustomDiagID(
-                                                            Diagnostic::Error,
+                                                   DiagnosticsEngine::Error,
                                                    "cannot instantiate %0 yet");
       SemaRef.Diag(D->getLocation(), DiagID)
         << D->getDeclKindName();
@@ -380,7 +383,7 @@ namespace clang {
     }
     
     typedef 
-      llvm::SmallVectorImpl<std::pair<ClassTemplateDecl *,
+      SmallVectorImpl<std::pair<ClassTemplateDecl *,
                                      ClassTemplatePartialSpecializationDecl *> >
         ::iterator
       delayed_partial_spec_iterator;
@@ -403,7 +406,7 @@ namespace clang {
 
     // Helper functions for instantiating methods.
     TypeSourceInfo *SubstFunctionType(FunctionDecl *D,
-                             llvm::SmallVectorImpl<ParmVarDecl *> &Params);
+                             SmallVectorImpl<ParmVarDecl *> &Params);
     bool InitFunctionInstantiation(FunctionDecl *New, FunctionDecl *Tmpl);
     bool InitMethodInstantiation(CXXMethodDecl *New, CXXMethodDecl *Tmpl);
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h b/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h
index c66697963e0d..690129ab34d1 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h
@@ -41,7 +41,7 @@ class TemplateDeductionInfo {
 
   /// \brief Warnings (and follow-on notes) that were suppressed due to 
   /// SFINAE while performing template argument deduction.
-  llvm::SmallVector<PartialDiagnosticAt, 4> SuppressedDiagnostics;
+  SmallVector<PartialDiagnosticAt, 4> SuppressedDiagnostics;
   
   // do not implement these
   TemplateDeductionInfo(const TemplateDeductionInfo&);
@@ -81,7 +81,7 @@ class TemplateDeductionInfo {
   }
   
   /// \brief Iterator over the set of suppressed diagnostics.
-  typedef llvm::SmallVectorImpl<PartialDiagnosticAt>::const_iterator 
+  typedef SmallVectorImpl<PartialDiagnosticAt>::const_iterator 
     diag_iterator;
   
   /// \brief Returns an iterator at the beginning of the sequence of suppressed
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/TypoCorrection.h b/contrib/llvm/tools/clang/include/clang/Sema/TypoCorrection.h
index 9965953538a1..9537c3031dcb 100644
--- a/contrib/llvm/tools/clang/include/clang/Sema/TypoCorrection.h
+++ b/contrib/llvm/tools/clang/include/clang/Sema/TypoCorrection.h
@@ -16,6 +16,7 @@
 #define LLVM_CLANG_SEMA_TYPOCORRECTION_H
 
 #include "clang/AST/DeclCXX.h"
+#include "llvm/ADT/SmallVector.h"
 
 namespace clang {
 
@@ -23,29 +24,31 @@ namespace clang {
 class TypoCorrection {
 public:
   TypoCorrection(const DeclarationName &Name, NamedDecl *NameDecl,
-                 NestedNameSpecifier *NNS=NULL, unsigned distance=0)
+                 NestedNameSpecifier *NNS=0, unsigned distance=0)
       : CorrectionName(Name),
         CorrectionNameSpec(NNS),
-        CorrectionDecl(NameDecl),
-        EditDistance(distance) {}
+        EditDistance(distance) {
+    if (NameDecl)
+      CorrectionDecls.push_back(NameDecl);
+  }
 
-  TypoCorrection(NamedDecl *Name, NestedNameSpecifier *NNS=NULL,
+  TypoCorrection(NamedDecl *Name, NestedNameSpecifier *NNS=0,
                  unsigned distance=0)
       : CorrectionName(Name->getDeclName()),
         CorrectionNameSpec(NNS),
-        CorrectionDecl(Name),
-        EditDistance(distance)  {}
+        EditDistance(distance) {
+    if (Name)
+      CorrectionDecls.push_back(Name);
+  }
 
-  TypoCorrection(DeclarationName Name, NestedNameSpecifier *NNS=NULL,
+  TypoCorrection(DeclarationName Name, NestedNameSpecifier *NNS=0,
                  unsigned distance=0)
       : CorrectionName(Name),
         CorrectionNameSpec(NNS),
-        CorrectionDecl(NULL),
-        EditDistance(distance)  {}
+        EditDistance(distance) {}
 
   TypoCorrection()
-      : CorrectionName(), CorrectionNameSpec(NULL), CorrectionDecl(NULL),
-        EditDistance(0) {}
+      : CorrectionNameSpec(0), EditDistance(0) {}
 
   /// \brief Gets the DeclarationName of the typo correction
   DeclarationName getCorrection() const { return CorrectionName; }
@@ -66,37 +69,69 @@ class TypoCorrection {
 
   /// \brief Gets the pointer to the declaration of the typo correction
   NamedDecl* getCorrectionDecl() const {
-    return isKeyword() ? NULL : CorrectionDecl;
+    return hasCorrectionDecl() ? *(CorrectionDecls.begin()) : 0;
   }
   template <class DeclClass>
   DeclClass *getCorrectionDeclAs() const {
     return dyn_cast_or_null<DeclClass>(getCorrectionDecl());
   }
   
+  /// \brief Clears the list of NamedDecls before adding the new one.
   void setCorrectionDecl(NamedDecl *CDecl) {
-    CorrectionDecl = CDecl;
-    if (!CorrectionName)
-      CorrectionName = CDecl->getDeclName();
+    CorrectionDecls.clear();
+    addCorrectionDecl(CDecl);
   }
 
+  /// \brief Add the given NamedDecl to the list of NamedDecls that are the
+  /// declarations associated with the DeclarationName of this TypoCorrection
+  void addCorrectionDecl(NamedDecl *CDecl);
+
   std::string getAsString(const LangOptions &LO) const;
   std::string getQuoted(const LangOptions &LO) const {
     return "'" + getAsString(LO) + "'";
   }
 
+  /// \brief Returns whether this TypoCorrection has a non-empty DeclarationName
   operator bool() const { return bool(CorrectionName); }
 
-  static inline NamedDecl *KeywordDecl() { return (NamedDecl*)-1; }
-  bool isKeyword() const { return CorrectionDecl == KeywordDecl(); }
+  /// \brief Mark this TypoCorrection as being a keyword.
+  /// Since addCorrectionDeclsand setCorrectionDecl don't allow NULL to be
+  /// added to the list of the correction's NamedDecl pointers, NULL is added
+  /// as the only element in the list to mark this TypoCorrection as a keyword.
+  void makeKeyword() {
+    CorrectionDecls.clear();
+    CorrectionDecls.push_back(0);
+  }
+
+  // Check if this TypoCorrection is a keyword by checking if the first
+  // item in CorrectionDecls is NULL.
+  bool isKeyword() const {
+    return !CorrectionDecls.empty() &&
+        CorrectionDecls.front() == 0;
+  }
 
   // Returns true if the correction either is a keyword or has a known decl.
-  bool isResolved() const { return CorrectionDecl != NULL; }
+  bool isResolved() const { return !CorrectionDecls.empty(); }
+
+  bool isOverloaded() const {
+    return CorrectionDecls.size() > 1;
+  }
+
+  typedef llvm::SmallVector<NamedDecl*, 1>::iterator decl_iterator;
+  decl_iterator begin() {
+    return isKeyword() ? CorrectionDecls.end() : CorrectionDecls.begin();
+  }
+  decl_iterator end() { return CorrectionDecls.end(); }
 
 private:
+  bool hasCorrectionDecl() const {
+    return (!isKeyword() && !CorrectionDecls.empty());
+  }
+
   // Results.
   DeclarationName CorrectionName;
   NestedNameSpecifier *CorrectionNameSpec;
-  NamedDecl *CorrectionDecl;
+  llvm::SmallVector<NamedDecl*, 1> CorrectionDecls;
   unsigned EditDistance;
 };
 
diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Weak.h b/contrib/llvm/tools/clang/include/clang/Sema/Weak.h
new file mode 100644
index 000000000000..d36b97089364
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Sema/Weak.h
@@ -0,0 +1,46 @@
+//===-- UnresolvedSet.h - Unresolved sets of declarations  ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the WeakInfo class, which is used to store
+//  information about the target of a #pragma weak directive.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_WEAK_H
+#define LLVM_CLANG_SEMA_WEAK_H
+
+#include "clang/Basic/SourceLocation.h"
+
+namespace clang {
+
+class IdentifierInfo;
+
+/// \brief Captures information about a #pragma weak directive.
+class WeakInfo {
+  IdentifierInfo *alias;  // alias (optional)
+  SourceLocation loc;     // for diagnostics
+  bool used;              // identifier later declared?
+public:
+  WeakInfo()
+    : alias(0), loc(SourceLocation()), used(false) {}
+  WeakInfo(IdentifierInfo *Alias, SourceLocation Loc)
+    : alias(Alias), loc(Loc), used(false) {}
+  inline IdentifierInfo * getAlias() const { return alias; }
+  inline SourceLocation getLocation() const { return loc; }
+  void setUsed(bool Used=true) { used = Used; }
+  inline bool getUsed() { return used; }
+  bool operator==(WeakInfo RHS) const {
+    return alias == RHS.getAlias() && loc == RHS.getLocation();
+  }
+  bool operator!=(WeakInfo RHS) const { return !(*this == RHS); }
+};
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_SEMA_WEAK_H
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h b/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h
index 11b8bed903d6..dc4d05c6f346 100644
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h
@@ -47,16 +47,28 @@ namespace clang {
     /// should be increased.
     const unsigned VERSION_MINOR = 0;
 
+    /// \brief An ID number that refers to an identifier in an AST file.
+    /// 
+    /// The ID numbers of identifiers are consecutive (in order of discovery)
+    /// and start at 1. 0 is reserved for NULL.
+    typedef uint32_t IdentifierID;
+    
     /// \brief An ID number that refers to a declaration in an AST file.
     ///
     /// The ID numbers of declarations are consecutive (in order of
-    /// discovery) and start at 2. 0 is reserved for NULL, and 1 is
-    /// reserved for the translation unit declaration.
+    /// discovery), with values below NUM_PREDEF_DECL_IDS being reserved. 
+    /// At the start of a chain of precompiled headers, declaration ID 1 is 
+    /// used for the translation unit declaration.
     typedef uint32_t DeclID;
 
     /// \brief a Decl::Kind/DeclID pair.
     typedef std::pair<uint32_t, DeclID> KindDeclIDPair;
 
+    // FIXME: Turn these into classes so we can have some type safety when
+    // we go from local ID to global and vice-versa.
+    typedef DeclID LocalDeclID;
+    typedef DeclID GlobalDeclID;
+
     /// \brief An ID number that refers to a type in an AST file.
     ///
     /// The ID of a type is partitioned into two parts: the lower
@@ -78,9 +90,15 @@ namespace clang {
 
       uint32_t getIndex() const { return Idx; }
       TypeID asTypeID(unsigned FastQuals) const {
+        if (Idx == uint32_t(-1))
+          return TypeID(-1);
+        
         return (Idx << Qualifiers::FastWidth) | FastQuals;
       }
       static TypeIdx fromTypeID(TypeID ID) {
+        if (ID == TypeID(-1))
+          return TypeIdx(-1);
+        
         return TypeIdx(ID >> Qualifiers::FastWidth);
       }
     };
@@ -103,31 +121,44 @@ namespace clang {
       }
     };
 
-    /// \brief Map that provides the ID numbers of each type within the
-    /// output stream, plus those deserialized from a chained PCH.
-    ///
-    /// The ID numbers of types are consecutive (in order of discovery)
-    /// and start at 1. 0 is reserved for NULL. When types are actually
-    /// stored in the stream, the ID number is shifted by 2 bits to
-    /// allow for the const/volatile qualifiers.
-    ///
-    /// Keys in the map never have const/volatile qualifiers.
-    typedef llvm::DenseMap<QualType, TypeIdx, UnsafeQualTypeDenseMapInfo>
-        TypeIdxMap;
-
     /// \brief An ID number that refers to an identifier in an AST file.
     typedef uint32_t IdentID;
 
-    /// \brief An ID number that refers to a macro in an AST file.
-    typedef uint32_t MacroID;
-
+    /// \brief The number of predefined identifier IDs.
+    const unsigned int NUM_PREDEF_IDENT_IDS = 1;
+    
     /// \brief An ID number that refers to an ObjC selctor in an AST file.
     typedef uint32_t SelectorID;
 
+    /// \brief The number of predefined selector IDs.
+    const unsigned int NUM_PREDEF_SELECTOR_IDS = 1;
+    
     /// \brief An ID number that refers to a set of CXXBaseSpecifiers in an 
     /// AST file.
     typedef uint32_t CXXBaseSpecifiersID;
     
+    /// \brief An ID number that refers to an entity in the detailed
+    /// preprocessing record.
+    typedef uint32_t PreprocessedEntityID;
+
+    /// \brief Source range/offset of a preprocessed entity.
+    struct PPEntityOffset {
+      /// \brief Raw source location of beginning of range.
+      unsigned Begin;
+      /// \brief Raw source location of end of range.
+      unsigned End;
+      /// \brief Offset in the AST file.
+      uint32_t BitOffset;
+
+      PPEntityOffset(SourceRange R, uint32_t BitOffset)
+        : Begin(R.getBegin().getRawEncoding()),
+          End(R.getEnd().getRawEncoding()),
+          BitOffset(BitOffset) { }
+    };
+
+    /// \brief The number of predefined preprocessed entity IDs.
+    const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1;
+
     /// \brief Describes the various kinds of blocks that occur within
     /// an AST file.
     enum BlockIDs {
@@ -290,9 +321,9 @@ namespace clang {
       /// \brief Record code for the array of unused file scoped decls.
       UNUSED_FILESCOPED_DECLS = 22,
       
-      /// \brief Record code for the table of offsets to macro definition
-      /// entries in the preprocessing record.
-      MACRO_DEFINITION_OFFSETS = 23,
+      /// \brief Record code for the table of offsets to entries in the
+      /// preprocessing record.
+      PPD_ENTITIES_OFFSETS = 23,
 
       /// \brief Record code for the array of VTable uses.
       VTABLE_USES = 24,
@@ -300,9 +331,9 @@ namespace clang {
       /// \brief Record code for the array of dynamic classes.
       DYNAMIC_CLASSES = 25,
 
-      /// \brief Record code for the chained AST metadata, including the
-      /// AST file version and the name of the PCH this depends on.
-      CHAINED_METADATA = 26,
+      /// \brief Record code for the list of other AST files imported by
+      /// this AST file.
+      IMPORTS = 26,
 
       /// \brief Record code for referenced selector pool.
       REFERENCED_SELECTOR_POOL = 27,
@@ -375,8 +406,21 @@ namespace clang {
       
       /// \brief Record code for the set of known namespaces, which are used
       /// for typo correction.
-      KNOWN_NAMESPACES = 46
+      KNOWN_NAMESPACES = 46,
 
+      /// \brief Record code for the remapping information used to relate
+      /// loaded modules to the various offsets and IDs(e.g., source location 
+      /// offests, declaration and type IDs) that are used in that module to
+      /// refer to other modules.
+      MODULE_OFFSET_MAP = 47,
+
+      /// \brief Record code for the source manager line table information,
+      /// which stores information about #line directives.
+      SOURCE_MANAGER_LINE_TABLE = 48,
+
+      /// \brief Record code for ObjC categories in a module that are chained to
+      /// an interface.
+      OBJC_CHAINED_CATEGORIES
     };
 
     /// \brief Record types used within a source manager block.
@@ -393,10 +437,7 @@ namespace clang {
       SM_SLOC_BUFFER_BLOB = 3,
       /// \brief Describes a source location entry (SLocEntry) for a
       /// macro expansion.
-      SM_SLOC_EXPANSION_ENTRY = 4,
-      /// \brief Describes the SourceManager's line table, with
-      /// information about #line directives.
-      SM_LINE_TABLE = 5
+      SM_SLOC_EXPANSION_ENTRY = 4
     };
 
     /// \brief Record types used within a preprocessor block.
@@ -507,7 +548,13 @@ namespace clang {
       /// \brief The 'unknown any' placeholder type.
       PREDEF_TYPE_UNKNOWN_ANY   = 29,
       /// \brief The placeholder type for bound member functions.
-      PREDEF_TYPE_BOUND_MEMBER  = 30
+      PREDEF_TYPE_BOUND_MEMBER  = 30,
+      /// \brief The "auto" deduction type.
+      PREDEF_TYPE_AUTO_DEDUCT   = 31,
+      /// \brief The "auto &&" deduction type.
+      PREDEF_TYPE_AUTO_RREF_DEDUCT = 32,
+      /// \brief The OpenCL 'half' / ARM NEON __fp16 type.
+      PREDEF_TYPE_HALF_ID       = 33
     };
 
     /// \brief The number of predefined type IDs that are reserved for
@@ -602,7 +649,9 @@ namespace clang {
       /// \brief A AutoType record.
       TYPE_AUTO                  = 38,
       /// \brief A UnaryTransformType record.
-      TYPE_UNARY_TRANSFORM       = 39
+      TYPE_UNARY_TRANSFORM       = 39,
+      /// \brief An AtomicType record.
+      TYPE_ATOMIC                = 40
     };
 
     /// \brief The type IDs for special types constructed by semantic
@@ -613,44 +662,65 @@ namespace clang {
     enum SpecialTypeIDs {
       /// \brief __builtin_va_list
       SPECIAL_TYPE_BUILTIN_VA_LIST             = 0,
-      /// \brief Objective-C "id" type
-      SPECIAL_TYPE_OBJC_ID                     = 1,
-      /// \brief Objective-C selector type
-      SPECIAL_TYPE_OBJC_SELECTOR               = 2,
       /// \brief Objective-C Protocol type
-      SPECIAL_TYPE_OBJC_PROTOCOL               = 3,
-      /// \brief Objective-C Class type
-      SPECIAL_TYPE_OBJC_CLASS                  = 4,
+      SPECIAL_TYPE_OBJC_PROTOCOL               = 1,
       /// \brief CFConstantString type
-      SPECIAL_TYPE_CF_CONSTANT_STRING          = 5,
-      /// \brief Objective-C fast enumeration state type
-      SPECIAL_TYPE_OBJC_FAST_ENUMERATION_STATE = 6,
+      SPECIAL_TYPE_CF_CONSTANT_STRING          = 2,
       /// \brief C FILE typedef type
-      SPECIAL_TYPE_FILE                        = 7,
+      SPECIAL_TYPE_FILE                        = 3,
       /// \brief C jmp_buf typedef type
-      SPECIAL_TYPE_jmp_buf                     = 8,
+      SPECIAL_TYPE_jmp_buf                     = 4,
       /// \brief C sigjmp_buf typedef type
-      SPECIAL_TYPE_sigjmp_buf                  = 9,
+      SPECIAL_TYPE_sigjmp_buf                  = 5,
       /// \brief Objective-C "id" redefinition type
-      SPECIAL_TYPE_OBJC_ID_REDEFINITION        = 10,
+      SPECIAL_TYPE_OBJC_ID_REDEFINITION        = 6,
       /// \brief Objective-C "Class" redefinition type
-      SPECIAL_TYPE_OBJC_CLASS_REDEFINITION     = 11,
-      /// \brief Block descriptor type for Blocks CodeGen
-      SPECIAL_TYPE_BLOCK_DESCRIPTOR            = 12,
-      /// \brief Block extedned descriptor type for Blocks CodeGen
-      SPECIAL_TYPE_BLOCK_EXTENDED_DESCRIPTOR   = 13,
+      SPECIAL_TYPE_OBJC_CLASS_REDEFINITION     = 7,
       /// \brief Objective-C "SEL" redefinition type
-      SPECIAL_TYPE_OBJC_SEL_REDEFINITION       = 14,
-      /// \brief NSConstantString type
-      SPECIAL_TYPE_NS_CONSTANT_STRING          = 15,
-      /// \brief Whether __[u]int128_t identifier is installed.
-      SPECIAL_TYPE_INT128_INSTALLED            = 16,
-      /// \brief Cached "auto" deduction type.
-      SPECIAL_TYPE_AUTO_DEDUCT                 = 17,
-      /// \brief Cached "auto &&" deduction type.
-      SPECIAL_TYPE_AUTO_RREF_DEDUCT            = 18
+      SPECIAL_TYPE_OBJC_SEL_REDEFINITION       = 8
+    };
+    
+    /// \brief The number of special type IDs.
+    const unsigned NumSpecialTypeIDs = 0;
+
+    /// \brief Predefined declaration IDs.
+    ///
+    /// These declaration IDs correspond to predefined declarations in the AST
+    /// context, such as the NULL declaration ID. Such declarations are never
+    /// actually serialized, since they will be built by the AST context when 
+    /// it is created.
+    enum PredefinedDeclIDs {
+      /// \brief The NULL declaration.
+      PREDEF_DECL_NULL_ID       = 0,
+      
+      /// \brief The translation unit.
+      PREDEF_DECL_TRANSLATION_UNIT_ID = 1,
+      
+      /// \brief The Objective-C 'id' type.
+      PREDEF_DECL_OBJC_ID_ID = 2,
+      
+      /// \brief The Objective-C 'SEL' type.
+      PREDEF_DECL_OBJC_SEL_ID = 3,
+      
+      /// \brief The Objective-C 'Class' type.
+      PREDEF_DECL_OBJC_CLASS_ID = 4,
+      
+      /// \brief The signed 128-bit integer type.
+      PREDEF_DECL_INT_128_ID = 5,
+
+      /// \brief The unsigned 128-bit integer type.
+      PREDEF_DECL_UNSIGNED_INT_128_ID = 6,
+      
+      /// \brief The internal 'instancetype' typedef.
+      PREDEF_DECL_OBJC_INSTANCETYPE_ID = 7
     };
 
+    /// \brief The number of declaration IDs that are predefined.
+    ///
+    /// For more information about predefined declarations, see the
+    /// \c PredefinedDeclIDs type and the PREDEF_DECL_*_ID constants.
+    const unsigned int NUM_PREDEF_DECL_IDS = 8;
+    
     /// \brief Record codes for each kind of declaration.
     ///
     /// These constants describe the declaration records that can occur within
@@ -658,10 +728,8 @@ namespace clang {
     /// constant describes a record for a specific declaration class
     /// in the AST.
     enum DeclCode {
-      /// \brief A TranslationUnitDecl record.
-      DECL_TRANSLATION_UNIT = 50,
       /// \brief A TypedefDecl record.
-      DECL_TYPEDEF,
+      DECL_TYPEDEF = 51,
       /// \brief A TypeAliasDecl record.
       DECL_TYPEALIAS,
       /// \brief An EnumDecl record.
@@ -786,7 +854,10 @@ namespace clang {
       DECL_INDIRECTFIELD,
       /// \brief A NonTypeTemplateParmDecl record that stores an expanded
       /// non-type template parameter pack.
-      DECL_EXPANDED_NON_TYPE_TEMPLATE_PARM_PACK
+      DECL_EXPANDED_NON_TYPE_TEMPLATE_PARM_PACK,
+      /// \brief A ClassScopeFunctionSpecializationDecl record a class scope
+      /// function specialization. (Microsoft extension).
+      DECL_CLASS_SCOPE_FUNCTION_SPECIALIZATION
     };
 
     /// \brief Record codes for each kind of statement or expression.
@@ -904,7 +975,9 @@ namespace clang {
       EXPR_BLOCK_DECL_REF,
       /// \brief A GenericSelectionExpr record.
       EXPR_GENERIC_SELECTION,
-      
+      /// \brief An AtomicExpr record.
+      EXPR_ATOMIC,
+
       // Objective-C
 
       /// \brief An ObjCStringLiteral record.
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ASTDeserializationListener.h b/contrib/llvm/tools/clang/include/clang/Serialization/ASTDeserializationListener.h
index f8cdebe5a913..588fe0e63c04 100644
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ASTDeserializationListener.h
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ASTDeserializationListener.h
@@ -45,7 +45,7 @@ class ASTDeserializationListener {
   /// \brief A selector was read from the AST file.
   virtual void SelectorRead(serialization::SelectorID iD, Selector Sel) { }
   /// \brief A macro definition was read from the AST file.
-  virtual void MacroDefinitionRead(serialization::MacroID, 
+  virtual void MacroDefinitionRead(serialization::PreprocessedEntityID, 
                                    MacroDefinition *MD) { }
 };
 
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ASTReader.h b/contrib/llvm/tools/clang/include/clang/Serialization/ASTReader.h
index 9e210c3db28c..996a13465944 100644
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ASTReader.h
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ASTReader.h
@@ -15,6 +15,9 @@
 #define LLVM_CLANG_FRONTEND_AST_READER_H
 
 #include "clang/Serialization/ASTBitCodes.h"
+#include "clang/Serialization/ContinuousRangeMap.h"
+#include "clang/Serialization/Module.h"
+#include "clang/Serialization/ModuleManager.h"
 #include "clang/Sema/ExternalSemaSource.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/DeclObjC.h"
@@ -23,6 +26,8 @@
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/APFloat.h"
@@ -31,6 +36,7 @@
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Bitcode/BitstreamReader.h"
 #include "llvm/Support/DataTypes.h"
 #include <deque>
@@ -49,6 +55,7 @@ class AddrLabelExpr;
 class ASTConsumer;
 class ASTContext;
 class ASTIdentifierIterator;
+class ASTUnit; // FIXME: Layering violation and egregious hack.
 class Attr;
 class Decl;
 class DeclContext;
@@ -64,10 +71,10 @@ class Preprocessor;
 class Sema;
 class SwitchCase;
 class ASTDeserializationListener;
+class ASTWriter;
 class ASTReader;
 class ASTDeclReader;
 class ASTStmtReader;
-class ASTIdentifierLookupTrait;
 class TypeLocReader;
 struct HeaderFileInfo;
 class VersionTuple;
@@ -77,9 +84,9 @@ struct PCHPredefinesBlock {
   FileID BufferID;
 
   /// \brief This predefines buffer in a PCH file.
-  llvm::StringRef Data;
+  StringRef Data;
 };
-typedef llvm::SmallVector<PCHPredefinesBlock, 2> PCHPredefinesBlocks;
+typedef SmallVector<PCHPredefinesBlock, 2> PCHPredefinesBlocks;
 
 /// \brief Abstract interface for callback invocations by the ASTReader.
 ///
@@ -101,7 +108,7 @@ class ASTReaderListener {
   /// \brief Receives the target triple.
   ///
   /// \returns true to indicate the target triple is invalid or false otherwise.
-  virtual bool ReadTargetTriple(llvm::StringRef Triple) {
+  virtual bool ReadTargetTriple(StringRef Triple) {
     return false;
   }
 
@@ -117,7 +124,7 @@ class ASTReaderListener {
   ///
   /// \returns true to indicate the predefines are invalid or false otherwise.
   virtual bool ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
-                                    llvm::StringRef OriginalFileName,
+                                    StringRef OriginalFileName,
                                     std::string &SuggestedPredefines,
                                     FileManager &FileMgr) {
     return false;
@@ -143,9 +150,9 @@ class PCHValidator : public ASTReaderListener {
     : PP(PP), Reader(Reader), NumHeaderInfos(0) {}
 
   virtual bool ReadLanguageOptions(const LangOptions &LangOpts);
-  virtual bool ReadTargetTriple(llvm::StringRef Triple);
+  virtual bool ReadTargetTriple(StringRef Triple);
   virtual bool ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
-                                    llvm::StringRef OriginalFileName,
+                                    StringRef OriginalFileName,
                                     std::string &SuggestedPredefines,
                                     FileManager &FileMgr);
   virtual void ReadHeaderFileInfo(const HeaderFileInfo &HFI, unsigned ID);
@@ -155,6 +162,16 @@ class PCHValidator : public ASTReaderListener {
   void Error(const char *Msg);
 };
 
+namespace serialization {    
+
+class ReadMethodPoolVisitor;
+  
+namespace reader {
+  class ASTIdentifierLookupTrait;
+}
+  
+} // end namespace serialization
+  
 /// \brief Reads an AST files chain containing the contents of a translation
 /// unit.
 ///
@@ -179,18 +196,24 @@ class ASTReader
 public:
   enum ASTReadResult { Success, Failure, IgnorePCH };
   /// \brief Types of AST files.
-  enum ASTFileType {
-    Module,   ///< File is a module proper.
-    PCH,      ///< File is a PCH file treated as such.
-    Preamble, ///< File is a PCH file treated as the preamble.
-    MainFile  ///< File is a PCH file treated as the actual main file.
-  };
   friend class PCHValidator;
   friend class ASTDeclReader;
   friend class ASTStmtReader;
   friend class ASTIdentifierIterator;
-  friend class ASTIdentifierLookupTrait;
+  friend class serialization::reader::ASTIdentifierLookupTrait;
   friend class TypeLocReader;
+  friend class ASTWriter;
+  friend class ASTUnit; // ASTUnit needs to remap source locations.
+  friend class serialization::ReadMethodPoolVisitor;
+  
+  typedef serialization::Module Module;
+  typedef serialization::ModuleKind ModuleKind;
+  typedef serialization::ModuleManager ModuleManager;
+  
+  typedef ModuleManager::ModuleIterator ModuleIterator;
+  typedef ModuleManager::ModuleConstIterator ModuleConstIterator;
+  typedef ModuleManager::ModuleReverseIterator ModuleReverseIterator;
+
 private:
   /// \brief The receiver of some callbacks invoked by ASTReader.
   llvm::OwningPtr<ASTReaderListener> Listener;
@@ -200,250 +223,50 @@ class ASTReader
 
   SourceManager &SourceMgr;
   FileManager &FileMgr;
-  Diagnostic &Diags;
-
+  DiagnosticsEngine &Diags;
+  
   /// \brief The semantic analysis object that will be processing the
   /// AST files and the translation unit that uses it.
   Sema *SemaObj;
 
   /// \brief The preprocessor that will be loading the source file.
-  Preprocessor *PP;
+  Preprocessor &PP;
 
   /// \brief The AST context into which we'll read the AST files.
-  ASTContext *Context;
+  ASTContext &Context;
       
   /// \brief The AST consumer.
   ASTConsumer *Consumer;
 
-  /// \brief AST buffers for chained PCHs created and stored in memory.
-  /// First (not depending on another) PCH in chain is in front.
-  std::vector<llvm::MemoryBuffer *> ASTBuffers;
+  /// \brief The module manager which manages modules and their dependencies
+  ModuleManager ModuleMgr;
 
-  /// \brief Information that is needed for every module.
-  struct PerFileData {
-    PerFileData(ASTFileType Ty);
-    ~PerFileData();
+  /// \brief A map of global bit offsets to the module that stores entities
+  /// at those bit offsets.
+  ContinuousRangeMap<uint64_t, Module*, 4> GlobalBitOffsetsMap;
 
-    // === General information ===
-
-    /// \brief The type of this AST file.
-    ASTFileType Type;
-
-    /// \brief The file name of the AST file.
-    std::string FileName;
-
-    /// \brief The memory buffer that stores the data associated with
-    /// this AST file.
-    llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
-
-    /// \brief The size of this file, in bits.
-    uint64_t SizeInBits;
-
-    /// \brief The bitstream reader from which we'll read the AST file.
-    llvm::BitstreamReader StreamFile;
-
-    /// \brief The main bitstream cursor for the main block.
-    llvm::BitstreamCursor Stream;
-
-    // === Source Locations ===
-
-    /// \brief Cursor used to read source location entries.
-    llvm::BitstreamCursor SLocEntryCursor;
-
-    /// \brief The number of source location entries in this AST file.
-    unsigned LocalNumSLocEntries;
-
-    /// \brief Offsets for all of the source location entries in the
-    /// AST file.
-    const uint32_t *SLocOffsets;
-
-    /// \brief The number of source location file entries in this AST file.
-    unsigned LocalNumSLocFileEntries;
-
-    /// \brief Offsets for all of the source location file entries in the
-    /// AST file.
-    const uint32_t *SLocFileOffsets;
-
-    /// \brief The entire size of this module's source location offset range.
-    unsigned LocalSLocSize;
-
-    // === Identifiers ===
-
-    /// \brief The number of identifiers in this AST file.
-    unsigned LocalNumIdentifiers;
-
-    /// \brief Offsets into the identifier table data.
-    ///
-    /// This array is indexed by the identifier ID (-1), and provides
-    /// the offset into IdentifierTableData where the string data is
-    /// stored.
-    const uint32_t *IdentifierOffsets;
-
-    /// \brief Actual data for the on-disk hash table of identifiers.
-    ///
-    /// This pointer points into a memory buffer, where the on-disk hash
-    /// table for identifiers actually lives.
-    const char *IdentifierTableData;
-
-    /// \brief A pointer to an on-disk hash table of opaque type
-    /// IdentifierHashTable.
-    void *IdentifierLookupTable;
-
-    // === Macros ===
-
-    /// \brief The cursor to the start of the preprocessor block, which stores
-    /// all of the macro definitions.
-    llvm::BitstreamCursor MacroCursor;
-
-    /// \brief The offset of the start of the set of defined macros.
-    uint64_t MacroStartOffset;
-    
-    // === Detailed PreprocessingRecord ===
-    
-    /// \brief The cursor to the start of the (optional) detailed preprocessing 
-    /// record block.
-    llvm::BitstreamCursor PreprocessorDetailCursor;
-    
-    /// \brief The offset of the start of the preprocessor detail cursor.
-    uint64_t PreprocessorDetailStartOffset;
-    
-    /// \brief The number of macro definitions in this file.
-    unsigned LocalNumMacroDefinitions;
-    
-    /// \brief Offsets of all of the macro definitions in the preprocessing
-    /// record in the AST file.
-    const uint32_t *MacroDefinitionOffsets;
-
-    // === Header search information ===
-    
-    /// \brief The number of local HeaderFileInfo structures.
-    unsigned LocalNumHeaderFileInfos;
-    
-    /// \brief Actual data for the on-disk hash table of header file 
-    /// information.
-    ///
-    /// This pointer points into a memory buffer, where the on-disk hash
-    /// table for header file information actually lives.
-    const char *HeaderFileInfoTableData;
-
-    /// \brief The on-disk hash table that contains information about each of
-    /// the header files.
-    void *HeaderFileInfoTable;
-    
-    // === Selectors ===
-
-    /// \brief The number of selectors new to this file.
-    ///
-    /// This is the number of entries in SelectorOffsets.
-    unsigned LocalNumSelectors;
-
-    /// \brief Offsets into the selector lookup table's data array
-    /// where each selector resides.
-    const uint32_t *SelectorOffsets;
-
-    /// \brief A pointer to the character data that comprises the selector table
-    ///
-    /// The SelectorOffsets table refers into this memory.
-    const unsigned char *SelectorLookupTableData;
-
-    /// \brief A pointer to an on-disk hash table of opaque type
-    /// ASTSelectorLookupTable.
-    ///
-    /// This hash table provides the IDs of all selectors, and the associated
-    /// instance and factory methods.
-    void *SelectorLookupTable;
-
-    /// \brief Method selectors used in a @selector expression. Used for
-    /// implementation of -Wselector.
-    llvm::SmallVector<uint64_t, 64> ReferencedSelectorsData;
-
-    // === Declarations ===
-      
-    /// DeclsCursor - This is a cursor to the start of the DECLS_BLOCK block. It
-    /// has read all the abbreviations at the start of the block and is ready to
-    /// jump around with these in context.
-    llvm::BitstreamCursor DeclsCursor;
-
-    /// \brief The number of declarations in this AST file.
-    unsigned LocalNumDecls;
-
-    /// \brief Offset of each declaration within the bitstream, indexed
-    /// by the declaration ID (-1).
-    const uint32_t *DeclOffsets;
-
-    /// \brief A snapshot of the pending instantiations in the chain.
-    ///
-    /// This record tracks the instantiations that Sema has to perform at the
-    /// end of the TU. It consists of a pair of values for every pending
-    /// instantiation where the first value is the ID of the decl and the second
-    /// is the instantiation location.
-    llvm::SmallVector<uint64_t, 64> PendingInstantiations;
-
-    /// \brief The number of C++ base specifier sets in this AST file.
-    unsigned LocalNumCXXBaseSpecifiers;
-    
-    /// \brief Offset of each C++ base specifier set within the bitstream,
-    /// indexed by the C++ base specifier set ID (-1).
-    const uint32_t *CXXBaseSpecifiersOffsets;
-    
-    // === Types ===
-
-    /// \brief The number of types in this AST file.
-    unsigned LocalNumTypes;
-
-    /// \brief Offset of each type within the bitstream, indexed by the
-    /// type ID, or the representation of a Type*.
-    const uint32_t *TypeOffsets;
-
-    // === Miscellaneous ===
-
-    /// \brief The AST stat cache installed for this file, if any.
-    ///
-    /// The dynamic type of this stat cache is always ASTStatCache
-    void *StatCache;
-
-    /// \brief The number of preallocated preprocessing entities in the
-    /// preprocessing record.
-    unsigned NumPreallocatedPreprocessingEntities;
-
-    /// \brief The next module in source order.
-    PerFileData *NextInSource;
-
-    /// \brief All the modules that loaded this one. Can contain NULL for
-    /// directly loaded modules.
-    llvm::SmallVector<PerFileData *, 1> Loaders;
-  };
-
-  /// \brief All loaded modules, indexed by name.
-  llvm::StringMap<PerFileData*> Modules;
-
-  /// \brief The first module in source order.
-  PerFileData *FirstInSource;
-
-  /// \brief The chain of AST files. The first entry is the one named by the
-  /// user, the last one is the one that doesn't depend on anything further.
-  /// That is, the entry I was created with -include-pch I+1.
-  llvm::SmallVector<PerFileData*, 2> Chain;
-
-  /// \brief SLocEntries that we're going to preload.
-  llvm::SmallVector<uint64_t, 64> PreloadSLocEntries;
+  /// \brief A map of negated SLocEntryIDs to the modules containing them.
+  ContinuousRangeMap<unsigned, Module*, 64> GlobalSLocEntryMap;
 
+  typedef ContinuousRangeMap<unsigned, Module*, 64> GlobalSLocOffsetMapType;
+  
+  /// \brief A map of reversed (SourceManager::MaxLoadedOffset - SLocOffset)
+  /// SourceLocation offsets to the modules containing them.
+  GlobalSLocOffsetMapType GlobalSLocOffsetMap;
+  
   /// \brief Types that have already been loaded from the chain.
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
   std::vector<QualType> TypesLoaded;
 
-  /// \brief Map that provides the ID numbers of each type within the
-  /// output stream, plus those deserialized from a chained PCH.
-  ///
-  /// The ID numbers of types are consecutive (in order of discovery)
-  /// and start at 1. 0 is reserved for NULL. When types are actually
-  /// stored in the stream, the ID number is shifted by 2 bits to
-  /// allow for the const/volatile qualifiers.
-  ///
-  /// Keys in the map never have const/volatile qualifiers.
-  serialization::TypeIdxMap TypeIdxs;
+  typedef ContinuousRangeMap<serialization::TypeID, Module *, 4>
+    GlobalTypeMapType;
+
+  /// \brief Mapping from global type IDs to the module in which the
+  /// type resides along with the offset that should be added to the
+  /// global type ID to produce a local ID.
+  GlobalTypeMapType GlobalTypeMap;
 
   /// \brief Declarations that have already been loaded from the chain.
   ///
@@ -451,50 +274,41 @@ class ASTReader
   /// = I + 1 has already been loaded.
   std::vector<Decl *> DeclsLoaded;
 
-  typedef std::pair<PerFileData *, uint64_t> FileOffset;
-  typedef llvm::SmallVector<FileOffset, 2> FileOffsetsTy;
+  typedef ContinuousRangeMap<serialization::DeclID, Module *, 4> 
+    GlobalDeclMapType;
+  
+  /// \brief Mapping from global declaration IDs to the module in which the
+  /// declaration resides.
+  GlobalDeclMapType GlobalDeclMap;
+  
+  typedef std::pair<Module *, uint64_t> FileOffset;
+  typedef SmallVector<FileOffset, 2> FileOffsetsTy;
   typedef llvm::DenseMap<serialization::DeclID, FileOffsetsTy>
       DeclUpdateOffsetsMap;
+  
   /// \brief Declarations that have modifications residing in a later file
   /// in the chain.
   DeclUpdateOffsetsMap DeclUpdateOffsets;
 
   typedef llvm::DenseMap<serialization::DeclID,
-                         std::pair<PerFileData *, uint64_t> >
+                         std::pair<Module *, uint64_t> >
       DeclReplacementMap;
   /// \brief Declarations that have been replaced in a later file in the chain.
   DeclReplacementMap ReplacedDecls;
 
-  /// \brief Information about the contents of a DeclContext.
-  struct DeclContextInfo {
-    void *NameLookupTableData; // a ASTDeclContextNameLookupTable.
-    const serialization::KindDeclIDPair *LexicalDecls;
-    unsigned NumLexicalDecls;
-  };
-  // In a full chain, there could be multiple updates to every decl context,
-  // so this is a vector. However, typically a chain is only two elements long,
-  // with only one file containing updates, so there will be only one update
-  // per decl context.
-  typedef llvm::SmallVector<DeclContextInfo, 1> DeclContextInfos;
-  typedef llvm::DenseMap<const DeclContext *, DeclContextInfos>
-      DeclContextOffsetsMap;
   // Updates for visible decls can occur for other contexts than just the
   // TU, and when we read those update records, the actual context will not
   // be available yet (unless it's the TU), so have this pending map using the
   // ID as a key. It will be realized when the context is actually loaded.
-  typedef llvm::SmallVector<void *, 1> DeclContextVisibleUpdates;
+  typedef SmallVector<std::pair<void *, Module*>, 1> DeclContextVisibleUpdates;
   typedef llvm::DenseMap<serialization::DeclID, DeclContextVisibleUpdates>
       DeclContextVisibleUpdatesPending;
 
-  /// \brief Offsets of the lexical and visible declarations for each
-  /// DeclContext.
-  DeclContextOffsetsMap DeclContextOffsets;
-
   /// \brief Updates to the visible declarations of declaration contexts that
   /// haven't been loaded yet.
   DeclContextVisibleUpdatesPending PendingVisibleUpdates;
 
-  typedef llvm::SmallVector<CXXRecordDecl *, 4> ForwardRefs;
+  typedef SmallVector<CXXRecordDecl *, 4> ForwardRefs;
   typedef llvm::DenseMap<const CXXRecordDecl *, ForwardRefs>
       PendingForwardRefsMap;
   /// \brief Forward references that have a definition but the definition decl
@@ -508,10 +322,15 @@ class ASTReader
   /// most recent declarations in another AST file.
   FirstLatestDeclIDMap FirstLatestDeclIDs;
 
+  /// \brief Set of ObjC interfaces that have categories chained to them in
+  /// other modules.
+  llvm::DenseSet<serialization::GlobalDeclID> ObjCChainedCategoriesInterfaces;
+
   /// \brief Read the records that describe the contents of declcontexts.
-  bool ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
+  bool ReadDeclContextStorage(Module &M, 
+                              llvm::BitstreamCursor &Cursor,
                               const std::pair<uint64_t, uint64_t> &Offsets,
-                              DeclContextInfo &Info);
+                              serialization::DeclContextInfo &Info);
 
   /// \brief A vector containing identifiers that have already been
   /// loaded.
@@ -521,21 +340,42 @@ class ASTReader
   /// been loaded.
   std::vector<IdentifierInfo *> IdentifiersLoaded;
 
+  typedef ContinuousRangeMap<serialization::IdentID, Module *, 4> 
+    GlobalIdentifierMapType;
+  
+  /// \brief Mapping from global identifer IDs to the module in which the
+  /// identifier resides along with the offset that should be added to the
+  /// global identifier ID to produce a local ID.
+  GlobalIdentifierMapType GlobalIdentifierMap;
+
   /// \brief A vector containing selectors that have already been loaded.
   ///
   /// This vector is indexed by the Selector ID (-1). NULL selector
   /// entries indicate that the particular selector ID has not yet
   /// been loaded.
-  llvm::SmallVector<Selector, 16> SelectorsLoaded;
+  SmallVector<Selector, 16> SelectorsLoaded;
 
-  /// \brief The macro definitions we have already loaded.
-  llvm::SmallVector<MacroDefinition *, 16> MacroDefinitionsLoaded;
+  typedef ContinuousRangeMap<serialization::SelectorID, Module *, 4> 
+    GlobalSelectorMapType;
+  
+  /// \brief Mapping from global selector IDs to the module in which the
+  /// selector resides along with the offset that should be added to the
+  /// global selector ID to produce a local ID.
+  GlobalSelectorMapType GlobalSelectorMap;
 
   /// \brief Mapping from identifiers that represent macros whose definitions
   /// have not yet been deserialized to the global offset where the macro
   /// record resides.
   llvm::DenseMap<IdentifierInfo *, uint64_t> UnreadMacroRecordOffsets;
-      
+
+  typedef ContinuousRangeMap<unsigned, Module *, 4> 
+    GlobalPreprocessedEntityMapType;
+  
+  /// \brief Mapping from global preprocessing entity IDs to the module in
+  /// which the preprocessed entity resides along with the offset that should be
+  /// added to the global preprocessing entitiy ID to produce a local ID.
+  GlobalPreprocessedEntityMapType GlobalPreprocessedEntityMap;
+  
   /// \name CodeGen-relevant special data
   /// \brief Fields containing data that is relevant to CodeGen.
   //@{
@@ -546,44 +386,56 @@ class ASTReader
   /// This contains the data loaded from all EXTERNAL_DEFINITIONS blocks in the
   /// chain. The referenced declarations are deserialized and passed to the
   /// consumer eagerly.
-  llvm::SmallVector<uint64_t, 16> ExternalDefinitions;
+  SmallVector<uint64_t, 16> ExternalDefinitions;
 
   /// \brief The IDs of all tentative definitions stored in the the chain.
   ///
   /// Sema keeps track of all tentative definitions in a TU because it has to
   /// complete them and pass them on to CodeGen. Thus, tentative definitions in
   /// the PCH chain must be eagerly deserialized.
-  llvm::SmallVector<uint64_t, 16> TentativeDefinitions;
+  SmallVector<uint64_t, 16> TentativeDefinitions;
 
   /// \brief The IDs of all CXXRecordDecls stored in the chain whose VTables are
   /// used.
   ///
   /// CodeGen has to emit VTables for these records, so they have to be eagerly
   /// deserialized.
-  llvm::SmallVector<uint64_t, 64> VTableUses;
+  SmallVector<uint64_t, 64> VTableUses;
+
+  /// \brief A snapshot of the pending instantiations in the chain.
+  ///
+  /// This record tracks the instantiations that Sema has to perform at the
+  /// end of the TU. It consists of a pair of values for every pending
+  /// instantiation where the first value is the ID of the decl and the second
+  /// is the instantiation location.
+  SmallVector<uint64_t, 64> PendingInstantiations;
 
   //@}
 
-  /// \name Diagnostic-relevant special data
+  /// \name DiagnosticsEngine-relevant special data
   /// \brief Fields containing data that is used for generating diagnostics
   //@{
 
   /// \brief A snapshot of Sema's unused file-scoped variable tracking, for
   /// generating warnings.
-  llvm::SmallVector<uint64_t, 16> UnusedFileScopedDecls;
+  SmallVector<uint64_t, 16> UnusedFileScopedDecls;
 
   /// \brief A list of all the delegating constructors we've seen, to diagnose
   /// cycles.
-  llvm::SmallVector<uint64_t, 4> DelegatingCtorDecls;
+  SmallVector<uint64_t, 4> DelegatingCtorDecls;
+  
+  /// \brief Method selectors used in a @selector expression. Used for
+  /// implementation of -Wselector.
+  SmallVector<uint64_t, 64> ReferencedSelectorsData;
 
   /// \brief A snapshot of Sema's weak undeclared identifier tracking, for
   /// generating warnings.
-  llvm::SmallVector<uint64_t, 64> WeakUndeclaredIdentifiers;
+  SmallVector<uint64_t, 64> WeakUndeclaredIdentifiers;
 
   /// \brief The IDs of type aliases for ext_vectors that exist in the chain.
   ///
   /// Used by Sema for finding sugared names for ext_vectors in diagnostics.
-  llvm::SmallVector<uint64_t, 4> ExtVectorDecls;
+  SmallVector<uint64_t, 4> ExtVectorDecls;
 
   //@}
 
@@ -595,44 +447,41 @@ class ASTReader
   ///
   /// Sema tracks these to validate that the types are consistent across all
   /// local external declarations.
-  llvm::SmallVector<uint64_t, 16> LocallyScopedExternalDecls;
+  SmallVector<uint64_t, 16> LocallyScopedExternalDecls;
 
   /// \brief The IDs of all dynamic class declarations in the chain.
   ///
   /// Sema tracks these because it checks for the key functions being defined
   /// at the end of the TU, in which case it directs CodeGen to emit the VTable.
-  llvm::SmallVector<uint64_t, 16> DynamicClasses;
+  SmallVector<uint64_t, 16> DynamicClasses;
 
   /// \brief The IDs of the declarations Sema stores directly.
   ///
   /// Sema tracks a few important decls, such as namespace std, directly.
-  llvm::SmallVector<uint64_t, 4> SemaDeclRefs;
+  SmallVector<uint64_t, 4> SemaDeclRefs;
 
   /// \brief The IDs of the types ASTContext stores directly.
   ///
   /// The AST context tracks a few important types, such as va_list, directly.
-  llvm::SmallVector<uint64_t, 16> SpecialTypes;
+  SmallVector<uint64_t, 16> SpecialTypes;
 
   /// \brief The IDs of CUDA-specific declarations ASTContext stores directly.
   ///
   /// The AST context tracks a few important decls, currently cudaConfigureCall,
   /// directly.
-  llvm::SmallVector<uint64_t, 2> CUDASpecialDeclRefs;
+  SmallVector<uint64_t, 2> CUDASpecialDeclRefs;
 
   /// \brief The floating point pragma option settings.
-  llvm::SmallVector<uint64_t, 1> FPPragmaOptions;
+  SmallVector<uint64_t, 1> FPPragmaOptions;
 
   /// \brief The OpenCL extension settings.
-  llvm::SmallVector<uint64_t, 1> OpenCLExtensions;
+  SmallVector<uint64_t, 1> OpenCLExtensions;
 
   /// \brief A list of the namespaces we've seen.
-  llvm::SmallVector<uint64_t, 4> KnownNamespaces;
+  SmallVector<uint64_t, 4> KnownNamespaces;
 
   //@}
 
-  /// \brief Diagnostic IDs and their mappings that the user changed.
-  llvm::SmallVector<uint64_t, 8> PragmaDiagMappings;
-
   /// \brief The original file name that was used to build the primary AST file,
   /// which may have been modified for relocatable-pch support.
   std::string OriginalFileName;
@@ -657,7 +506,7 @@ class ASTReader
 
   /// \brief The system include root to be used when loading the
   /// precompiled header.
-  const char *isysroot;
+  std::string isysroot;
 
   /// \brief Whether to disable the normal validation performed on precompiled
   /// headers when they are loaded.
@@ -686,9 +535,6 @@ class ASTReader
   /// \brief The number of source location entries in the chain.
   unsigned TotalNumSLocEntries;
 
-  /// \brief The next offset for a SLocEntry after everything in this reader.
-  unsigned NextSLocOffset;
-
   /// \brief The number of statements (and expressions) de-serialized
   /// from the chain.
   unsigned NumStatementsRead;
@@ -722,14 +568,20 @@ class ASTReader
   /// Number of visible decl contexts read/total.
   unsigned NumVisibleDeclContextsRead, TotalVisibleDeclContexts;
   
+  /// Total size of modules, in bits, currently loaded
+  uint64_t TotalModulesSizeInBits;
+
   /// \brief Number of Decl/types that are currently deserializing.
   unsigned NumCurrentElementsDeserializing;
 
+  /// Number of CXX base specifiers currently loaded
+  unsigned NumCXXBaseSpecifiersLoaded;
+
   /// \brief An IdentifierInfo that has been loaded but whose top-level
   /// declarations of the same name have not (yet) been loaded.
   struct PendingIdentifierInfo {
     IdentifierInfo *II;
-    llvm::SmallVector<uint32_t, 4> DeclIDs;
+    SmallVector<uint32_t, 4> DeclIDs;
   };
 
   /// \brief The set of identifiers that were read while the AST reader was
@@ -755,7 +607,7 @@ class ASTReader
   void loadAndAttachPreviousDecl(Decl *D, serialization::DeclID ID);
 
   /// \brief When reading a Stmt tree, Stmt operands are placed in this stack.
-  llvm::SmallVector<Stmt *, 16> StmtStack;
+  SmallVector<Stmt *, 16> StmtStack;
 
   /// \brief What kind of records we are reading.
   enum ReadingKind {
@@ -797,36 +649,59 @@ class ASTReader
   std::string SuggestedPredefines;
 
   /// \brief Reads a statement from the specified cursor.
-  Stmt *ReadStmtFromStream(PerFileData &F);
+  Stmt *ReadStmtFromStream(Module &F);
 
   /// \brief Get a FileEntry out of stored-in-PCH filename, making sure we take
   /// into account all the necessary relocations.
-  const FileEntry *getFileEntry(llvm::StringRef filename);
+  const FileEntry *getFileEntry(StringRef filename);
 
   void MaybeAddSystemRootToFilename(std::string &Filename);
 
-  ASTReadResult ReadASTCore(llvm::StringRef FileName, ASTFileType Type);
-  ASTReadResult ReadASTBlock(PerFileData &F);
+  ASTReadResult ReadASTCore(StringRef FileName, ModuleKind Type,
+                            Module *ImportedBy);
+  ASTReadResult ReadASTBlock(Module &F);
   bool CheckPredefinesBuffers();
-  bool ParseLineTable(PerFileData &F, llvm::SmallVectorImpl<uint64_t> &Record);
-  ASTReadResult ReadSourceManagerBlock(PerFileData &F);
-  ASTReadResult ReadSLocEntryRecord(unsigned ID);
-  PerFileData *SLocCursorForID(unsigned ID);
-  SourceLocation getImportLocation(PerFileData *F);
-  bool ParseLanguageOptions(const llvm::SmallVectorImpl<uint64_t> &Record);
+  bool ParseLineTable(Module &F, SmallVectorImpl<uint64_t> &Record);
+  ASTReadResult ReadSourceManagerBlock(Module &F);
+  ASTReadResult ReadSLocEntryRecord(int ID);
+  llvm::BitstreamCursor &SLocCursorForID(int ID);
+  SourceLocation getImportLocation(Module *F);
+  bool ParseLanguageOptions(const SmallVectorImpl<uint64_t> &Record);
 
   struct RecordLocation {
-    RecordLocation(PerFileData *M, uint64_t O)
+    RecordLocation(Module *M, uint64_t O)
       : F(M), Offset(O) {}
-    PerFileData *F;
+    Module *F;
     uint64_t Offset;
   };
 
-  QualType ReadTypeRecord(unsigned Index);
+  QualType readTypeRecord(unsigned Index);
   RecordLocation TypeCursorForIndex(unsigned Index);
   void LoadedDecl(unsigned Index, Decl *D);
-  Decl *ReadDeclRecord(unsigned Index, serialization::DeclID ID);
-  RecordLocation DeclCursorForIndex(unsigned Index, serialization::DeclID ID);
+  Decl *ReadDeclRecord(serialization::DeclID ID);
+  RecordLocation DeclCursorForID(serialization::DeclID ID);
+  void loadDeclUpdateRecords(serialization::DeclID ID, Decl *D);
+  void loadObjCChainedCategories(serialization::GlobalDeclID ID,
+                                 ObjCInterfaceDecl *D);
+  
+  RecordLocation getLocalBitOffset(uint64_t GlobalOffset);
+  uint64_t getGlobalBitOffset(Module &M, uint32_t LocalOffset);
+
+  /// \brief Returns the first preprocessed entity ID that ends after \arg BLoc.
+  serialization::PreprocessedEntityID
+    findBeginPreprocessedEntity(SourceLocation BLoc) const;
+
+  /// \brief Returns the first preprocessed entity ID that begins after \arg ELoc.
+  serialization::PreprocessedEntityID
+    findEndPreprocessedEntity(SourceLocation ELoc) const;
+
+  /// \brief \arg SLocMapI points at a chunk of a module that contains no
+  /// preprocessed entities or the entities it contains are not the ones we are
+  /// looking for. Find the next module that contains entities and return the ID
+  /// of the first entry.
+  serialization::PreprocessedEntityID
+    findNextPreprocessedEntity(
+                        GlobalSLocOffsetMapType::const_iterator SLocMapI) const;
 
   void PassInterestingDeclsToConsumer();
 
@@ -834,14 +709,14 @@ class ASTReader
   ///
   /// This routine should only be used for fatal errors that have to
   /// do with non-routine failures (e.g., corrupted AST file).
-  void Error(llvm::StringRef Msg);
-  void Error(unsigned DiagID, llvm::StringRef Arg1 = llvm::StringRef(),
-             llvm::StringRef Arg2 = llvm::StringRef());
+  void Error(StringRef Msg);
+  void Error(unsigned DiagID, StringRef Arg1 = StringRef(),
+             StringRef Arg2 = StringRef());
 
   ASTReader(const ASTReader&); // do not implement
   ASTReader &operator=(const ASTReader &); // do not implement
 public:
-  typedef llvm::SmallVector<uint64_t, 64> RecordData;
+  typedef SmallVector<uint64_t, 64> RecordData;
 
   /// \brief Load the AST file and validate its contents against the given
   /// Preprocessor.
@@ -865,46 +740,19 @@ class ASTReader
   /// help when an AST file is being used in cases where the
   /// underlying files in the file system may have changed, but
   /// parsing should still continue.
-  ASTReader(Preprocessor &PP, ASTContext *Context, const char *isysroot = 0,
+  ASTReader(Preprocessor &PP, ASTContext &Context, StringRef isysroot = "",
             bool DisableValidation = false, bool DisableStatCache = false);
 
-  /// \brief Load the AST file without using any pre-initialized Preprocessor.
-  ///
-  /// The necessary information to initialize a Preprocessor later can be
-  /// obtained by setting a ASTReaderListener.
-  ///
-  /// \param SourceMgr the source manager into which the AST file will be loaded
-  ///
-  /// \param FileMgr the file manager into which the AST file will be loaded.
-  ///
-  /// \param Diags the diagnostics system to use for reporting errors and
-  /// warnings relevant to loading the AST file.
-  ///
-  /// \param isysroot If non-NULL, the system include path specified by the
-  /// user. This is only used with relocatable PCH files. If non-NULL,
-  /// a relocatable PCH file will use the default path "/".
-  ///
-  /// \param DisableValidation If true, the AST reader will suppress most
-  /// of its regular consistency checking, allowing the use of precompiled
-  /// headers that cannot be determined to be compatible.
-  ///
-  /// \param DisableStatCache If true, the AST reader will ignore the
-  /// stat cache in the AST files. This performance pessimization can
-  /// help when an AST file is being used in cases where the
-  /// underlying files in the file system may have changed, but
-  /// parsing should still continue.
-  ASTReader(SourceManager &SourceMgr, FileManager &FileMgr,
-            Diagnostic &Diags, const char *isysroot = 0,
-            bool DisableValidation = false, bool DisableStatCache = false);
   ~ASTReader();
 
-  /// \brief Load the precompiled header designated by the given file
-  /// name.
-  ASTReadResult ReadAST(const std::string &FileName, ASTFileType Type);
+  SourceManager &getSourceManager() const { return SourceMgr; }
+  
+  /// \brief Load the AST file designated by the given file name.
+  ASTReadResult ReadAST(const std::string &FileName, ModuleKind Type);
 
   /// \brief Checks that no file that is stored in PCH is out-of-sync with
   /// the actual file in the file system.
-  ASTReadResult validateFileEntries();
+  ASTReadResult validateFileEntries(Module &M);
 
   /// \brief Set the AST callbacks listener.
   void setListener(ASTReaderListener *listener) {
@@ -914,22 +762,20 @@ class ASTReader
   /// \brief Set the AST deserialization listener.
   void setDeserializationListener(ASTDeserializationListener *Listener);
 
-  /// \brief Set the Preprocessor to use.
-  void setPreprocessor(Preprocessor &pp);
+  /// \brief Initializes the ASTContext
+  void InitializeContext();
 
-  /// \brief Sets and initializes the given Context.
-  void InitializeContext(ASTContext &Context);
-
-  /// \brief Set AST buffers for chained PCHs created and stored in memory.
-  /// First (not depending on another) PCH in chain is first in array.
-  void setASTMemoryBuffers(llvm::MemoryBuffer **bufs, unsigned numBufs) {
-    ASTBuffers.clear();
-    ASTBuffers.insert(ASTBuffers.begin(), bufs, bufs + numBufs);
+  /// \brief Add in-memory (virtual file) buffer.
+  void addInMemoryBuffer(StringRef &FileName, llvm::MemoryBuffer *Buffer) {
+    ModuleMgr.addInMemoryBuffer(FileName, Buffer);
   }
 
-  /// \brief Retrieve the name of the named (primary) AST file
-  const std::string &getFileName() const { return Chain[0]->FileName; }
+  /// \brief Retrieve the module manager.
+  ModuleManager &getModuleManager() { return ModuleMgr; }
 
+  /// \brief Retrieve the preprocessor.
+  Preprocessor &getPreprocessor() const { return PP; }
+  
   /// \brief Retrieve the name of the original source file name
   const std::string &getOriginalSourceFile() { return OriginalFileName; }
 
@@ -937,34 +783,34 @@ class ASTReader
   /// the AST file, without actually loading the AST file.
   static std::string getOriginalSourceFile(const std::string &ASTFileName,
                                            FileManager &FileMgr,
-                                           Diagnostic &Diags);
+                                           DiagnosticsEngine &Diags);
 
   /// \brief Returns the suggested contents of the predefines buffer,
   /// which contains a (typically-empty) subset of the predefines
   /// build prior to including the precompiled header.
   const std::string &getSuggestedPredefines() { return SuggestedPredefines; }
-      
-  /// \brief Read preprocessed entities into the preprocessing record.
-  virtual void ReadPreprocessedEntities();
 
-  /// \brief Read the preprocessed entity at the given offset.
-  virtual PreprocessedEntity *ReadPreprocessedEntityAtOffset(uint64_t Offset);
+  /// \brief Read a preallocated preprocessed entity from the external source.
+  ///
+  /// \returns null if an error occurred that prevented the preprocessed
+  /// entity from being loaded.
+  virtual PreprocessedEntity *ReadPreprocessedEntity(unsigned Index);
+
+  /// \brief Returns a pair of [Begin, End) indices of preallocated
+  /// preprocessed entities that \arg Range encompasses.
+  virtual std::pair<unsigned, unsigned>
+      findPreprocessedEntitiesInRange(SourceRange Range);
 
   /// \brief Read the header file information for the given file entry.
   virtual HeaderFileInfo GetHeaderFileInfo(const FileEntry *FE);
 
-  void ReadPragmaDiagnosticMappings(Diagnostic &Diag);
+  void ReadPragmaDiagnosticMappings(DiagnosticsEngine &Diag);
 
   /// \brief Returns the number of source locations found in the chain.
   unsigned getTotalNumSLocs() const {
     return TotalNumSLocEntries;
   }
 
-  /// \brief Returns the next SLocEntry offset after the chain.
-  unsigned getNextSLocOffset() const {
-    return NextSLocOffset;
-  }
-
   /// \brief Returns the number of identifiers found in the chain.
   unsigned getTotalNumIdentifiers() const {
     return static_cast<unsigned>(IdentifiersLoaded.size());
@@ -985,54 +831,110 @@ class ASTReader
     return static_cast<unsigned>(SelectorsLoaded.size());
   }
 
-  /// \brief Returns the number of macro definitions found in the chain.
-  unsigned getTotalNumMacroDefinitions() const {
-    return static_cast<unsigned>(MacroDefinitionsLoaded.size());
+  /// \brief Returns the number of preprocessed entities known to the AST
+  /// reader.
+  unsigned getTotalNumPreprocessedEntities() const {
+    unsigned Result = 0;
+    for (ModuleConstIterator I = ModuleMgr.begin(),
+        E = ModuleMgr.end(); I != E; ++I) {
+      Result += (*I)->NumPreprocessedEntities;
+    }
+    
+    return Result;
   }
       
   /// \brief Returns the number of C++ base specifiers found in the chain.
-  unsigned getTotalNumCXXBaseSpecifiers() const;
+  unsigned getTotalNumCXXBaseSpecifiers() const {
+    return NumCXXBaseSpecifiersLoaded;
+  }
       
   /// \brief Reads a TemplateArgumentLocInfo appropriate for the
   /// given TemplateArgument kind.
   TemplateArgumentLocInfo
-  GetTemplateArgumentLocInfo(PerFileData &F, TemplateArgument::ArgKind Kind,
+  GetTemplateArgumentLocInfo(Module &F, TemplateArgument::ArgKind Kind,
                              const RecordData &Record, unsigned &Idx);
 
   /// \brief Reads a TemplateArgumentLoc.
   TemplateArgumentLoc
-  ReadTemplateArgumentLoc(PerFileData &F,
+  ReadTemplateArgumentLoc(Module &F,
                           const RecordData &Record, unsigned &Idx);
 
   /// \brief Reads a declarator info from the given record.
-  TypeSourceInfo *GetTypeSourceInfo(PerFileData &F,
+  TypeSourceInfo *GetTypeSourceInfo(Module &F,
                                     const RecordData &Record, unsigned &Idx);
 
-  /// \brief Resolve and return the translation unit declaration.
-  TranslationUnitDecl *GetTranslationUnitDecl();
-
   /// \brief Resolve a type ID into a type, potentially building a new
   /// type.
   QualType GetType(serialization::TypeID ID);
 
-  /// \brief Returns the type ID associated with the given type.
-  /// If the type didn't come from the AST file the ID that is returned is
-  /// marked as "doesn't exist in AST".
-  serialization::TypeID GetTypeID(QualType T) const;
-
-  /// \brief Returns the type index associated with the given type.
-  /// If the type didn't come from the AST file the index that is returned is
-  /// marked as "doesn't exist in AST".
-  serialization::TypeIdx GetTypeIdx(QualType T) const;
+  /// \brief Resolve a local type ID within a given AST file into a type.
+  QualType getLocalType(Module &F, unsigned LocalID);
+  
+  /// \brief Map a local type ID within a given AST file into a global type ID.
+  serialization::TypeID getGlobalTypeID(Module &F, unsigned LocalID) const;
+  
+  /// \brief Read a type from the current position in the given record, which 
+  /// was read from the given AST file.
+  QualType readType(Module &F, const RecordData &Record, unsigned &Idx) {
+    if (Idx >= Record.size())
+      return QualType();
+    
+    return getLocalType(F, Record[Idx++]);
+  }
+  
+  /// \brief Map from a local declaration ID within a given module to a 
+  /// global declaration ID.
+  serialization::DeclID getGlobalDeclID(Module &F, unsigned LocalID) const;
 
+  /// \brief Returns true if global DeclID \arg ID originated from module
+  /// \arg M.
+  bool isDeclIDFromModule(serialization::GlobalDeclID ID, Module &M) const;
+  
   /// \brief Resolve a declaration ID into a declaration, potentially
   /// building a new declaration.
   Decl *GetDecl(serialization::DeclID ID);
   virtual Decl *GetExternalDecl(uint32_t ID);
 
-  /// \brief Resolve a CXXBaseSpecifiers ID into an offset into the chain
-  /// of loaded AST files.
-  uint64_t GetCXXBaseSpecifiersOffset(serialization::CXXBaseSpecifiersID ID);
+  /// \brief Reads a declaration with the given local ID in the given module.
+  Decl *GetLocalDecl(Module &F, uint32_t LocalID) {
+    return GetDecl(getGlobalDeclID(F, LocalID));
+  }
+
+  /// \brief Reads a declaration with the given local ID in the given module.
+  ///
+  /// \returns The requested declaration, casted to the given return type.
+  template<typename T>
+  T *GetLocalDeclAs(Module &F, uint32_t LocalID) {
+    return cast_or_null<T>(GetLocalDecl(F, LocalID));
+  }
+
+  /// \brief Reads a declaration ID from the given position in a record in the 
+  /// given module.
+  ///
+  /// \returns The declaration ID read from the record, adjusted to a global ID.
+  serialization::DeclID ReadDeclID(Module &F, const RecordData &Record,
+                                   unsigned &Idx);
+  
+  /// \brief Reads a declaration from the given position in a record in the
+  /// given module.
+  Decl *ReadDecl(Module &F, const RecordData &R, unsigned &I) {
+    return GetDecl(ReadDeclID(F, R, I));
+  }
+  
+  /// \brief Reads a declaration from the given position in a record in the
+  /// given module.
+  ///
+  /// \returns The declaration read from this location, casted to the given
+  /// result type.
+  template<typename T>
+  T *ReadDeclAs(Module &F, const RecordData &R, unsigned &I) {
+    return cast_or_null<T>(GetDecl(ReadDeclID(F, R, I)));
+  }
+
+  /// \brief Read a CXXBaseSpecifiers ID form the given record and
+  /// return its global bit offset.
+  uint64_t readCXXBaseSpecifiers(Module &M, const RecordData &Record, 
+                                 unsigned &Idx);
       
   virtual CXXBaseSpecifier *GetExternalCXXBaseSpecifiers(uint64_t Offset);
       
@@ -1055,8 +957,6 @@ class ASTReader
   FindExternalVisibleDeclsByName(const DeclContext *DC,
                                  DeclarationName Name);
 
-  virtual void MaterializeVisibleDecls(const DeclContext *DC);
-
   /// \brief Read all of the declarations lexically stored in a
   /// declaration context.
   ///
@@ -1072,7 +972,7 @@ class ASTReader
   /// declarations for this declaration context.
   virtual ExternalLoadResult FindExternalLexicalDecls(const DeclContext *DC,
                                         bool (*isKindWeWant)(Decl::Kind),
-                                        llvm::SmallVectorImpl<Decl*> &Decls);
+                                        SmallVectorImpl<Decl*> &Decls);
 
   /// \brief Notify ASTReader that we started deserialization of
   /// a decl or type so until FinishedDeserializing is called there may be
@@ -1093,6 +993,9 @@ class ASTReader
   /// \brief Print some statistics about AST usage.
   virtual void PrintStats();
 
+  /// \brief Dump information about the AST reader to standard error.
+  void dump();
+  
   /// Return the amount of memory used by memory buffers, breaking down
   /// by heap-backed versus mmap'ed memory.
   virtual void getMemoryBufferSizes(MemoryBufferSizes &sizes) const;
@@ -1112,7 +1015,7 @@ class ASTReader
   /// declarations will be deserialized and introduced into the declaration
   /// chain of the identifier.
   virtual IdentifierInfo *get(const char *NameStart, const char *NameEnd);
-  IdentifierInfo *get(llvm::StringRef Name) {
+  IdentifierInfo *get(StringRef Name) {
     return get(Name.begin(), Name.end());
   }
 
@@ -1131,14 +1034,42 @@ class ASTReader
   /// \brief Load the set of namespaces that are known to the external source,
   /// which will be used during typo correction.
   virtual void ReadKnownNamespaces(
-                           llvm::SmallVectorImpl<NamespaceDecl *> &Namespaces);
+                           SmallVectorImpl<NamespaceDecl *> &Namespaces);
+
+  virtual void ReadTentativeDefinitions(
+                 SmallVectorImpl<VarDecl *> &TentativeDefs);
+
+  virtual void ReadUnusedFileScopedDecls(
+                 SmallVectorImpl<const DeclaratorDecl *> &Decls);
+
+  virtual void ReadDelegatingConstructors(
+                 SmallVectorImpl<CXXConstructorDecl *> &Decls);
+
+  virtual void ReadExtVectorDecls(SmallVectorImpl<TypedefNameDecl *> &Decls);
+
+  virtual void ReadDynamicClasses(SmallVectorImpl<CXXRecordDecl *> &Decls);
+
+  virtual void ReadLocallyScopedExternalDecls(
+                 SmallVectorImpl<NamedDecl *> &Decls);
+  
+  virtual void ReadReferencedSelectors(
+                 SmallVectorImpl<std::pair<Selector, SourceLocation> > &Sels);
+
+  virtual void ReadWeakUndeclaredIdentifiers(
+                 SmallVectorImpl<std::pair<IdentifierInfo *, WeakInfo> > &WI);
+
+  virtual void ReadUsedVTables(SmallVectorImpl<ExternalVTableUse> &VTables);
+
+  virtual void ReadPendingInstantiations(
+                 SmallVectorImpl<std::pair<ValueDecl *, 
+                                           SourceLocation> > &Pending);
 
   /// \brief Load a selector from disk, registering its ID if it exists.
   void LoadSelector(Selector Sel);
 
   void SetIdentifierInfo(unsigned ID, IdentifierInfo *II);
   void SetGloballyVisibleDecls(IdentifierInfo *II,
-                               const llvm::SmallVectorImpl<uint32_t> &DeclIDs,
+                               const SmallVectorImpl<uint32_t> &DeclIDs,
                                bool Nonrecursive = false);
 
   /// \brief Report a diagnostic.
@@ -1147,92 +1078,112 @@ class ASTReader
   /// \brief Report a diagnostic.
   DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID);
 
-  IdentifierInfo *DecodeIdentifierInfo(unsigned Idx);
+  IdentifierInfo *DecodeIdentifierInfo(serialization::IdentifierID ID);
 
-  IdentifierInfo *GetIdentifierInfo(const RecordData &Record, unsigned &Idx) {
-    return DecodeIdentifierInfo(Record[Idx++]);
+  IdentifierInfo *GetIdentifierInfo(Module &M, const RecordData &Record, 
+                                    unsigned &Idx) {
+    return DecodeIdentifierInfo(getGlobalIdentifierID(M, Record[Idx++]));
   }
 
-  virtual IdentifierInfo *GetIdentifier(unsigned ID) {
+  virtual IdentifierInfo *GetIdentifier(serialization::IdentifierID ID) {
     return DecodeIdentifierInfo(ID);
   }
 
+  IdentifierInfo *getLocalIdentifier(Module &M, unsigned LocalID);
+  
+  serialization::IdentifierID getGlobalIdentifierID(Module &M, 
+                                                    unsigned LocalID);
+                                 
   /// \brief Read the source location entry with index ID.
-  virtual bool ReadSLocEntry(unsigned ID);
+  virtual bool ReadSLocEntry(int ID);
 
-  Selector DecodeSelector(unsigned Idx);
+  /// \brief Retrieve a selector from the given module with its local ID
+  /// number.
+  Selector getLocalSelector(Module &M, unsigned LocalID);
 
-  virtual Selector GetExternalSelector(uint32_t ID);
+  Selector DecodeSelector(serialization::SelectorID Idx);
+
+  virtual Selector GetExternalSelector(serialization::SelectorID ID);
   uint32_t GetNumExternalSelectors();
 
-  Selector GetSelector(const RecordData &Record, unsigned &Idx) {
-    return DecodeSelector(Record[Idx++]);
+  Selector ReadSelector(Module &M, const RecordData &Record, unsigned &Idx) {
+    return getLocalSelector(M, Record[Idx++]);
   }
+  
+  /// \brief Retrieve the global selector ID that corresponds to this
+  /// the local selector ID in a given module.
+  serialization::SelectorID getGlobalSelectorID(Module &F, 
+                                                unsigned LocalID) const;
 
   /// \brief Read a declaration name.
-  DeclarationName ReadDeclarationName(const RecordData &Record, unsigned &Idx);
-  void ReadDeclarationNameLoc(PerFileData &F,
+  DeclarationName ReadDeclarationName(Module &F, 
+                                      const RecordData &Record, unsigned &Idx);
+  void ReadDeclarationNameLoc(Module &F,
                               DeclarationNameLoc &DNLoc, DeclarationName Name,
                               const RecordData &Record, unsigned &Idx);
-  void ReadDeclarationNameInfo(PerFileData &F, DeclarationNameInfo &NameInfo,
+  void ReadDeclarationNameInfo(Module &F, DeclarationNameInfo &NameInfo,
                                const RecordData &Record, unsigned &Idx);
 
-  void ReadQualifierInfo(PerFileData &F, QualifierInfo &Info,
+  void ReadQualifierInfo(Module &F, QualifierInfo &Info,
                          const RecordData &Record, unsigned &Idx);
 
-  NestedNameSpecifier *ReadNestedNameSpecifier(const RecordData &Record,
+  NestedNameSpecifier *ReadNestedNameSpecifier(Module &F,
+                                               const RecordData &Record,
                                                unsigned &Idx);
 
-  NestedNameSpecifierLoc ReadNestedNameSpecifierLoc(PerFileData &F, 
+  NestedNameSpecifierLoc ReadNestedNameSpecifierLoc(Module &F, 
                                                     const RecordData &Record,
                                                     unsigned &Idx);
 
   /// \brief Read a template name.
-  TemplateName ReadTemplateName(PerFileData &F, const RecordData &Record, 
+  TemplateName ReadTemplateName(Module &F, const RecordData &Record, 
                                 unsigned &Idx);
 
   /// \brief Read a template argument.
-  TemplateArgument ReadTemplateArgument(PerFileData &F,
+  TemplateArgument ReadTemplateArgument(Module &F,
                                         const RecordData &Record,unsigned &Idx);
   
   /// \brief Read a template parameter list.
-  TemplateParameterList *ReadTemplateParameterList(PerFileData &F,
+  TemplateParameterList *ReadTemplateParameterList(Module &F,
                                                    const RecordData &Record,
                                                    unsigned &Idx);
   
   /// \brief Read a template argument array.
   void
-  ReadTemplateArgumentList(llvm::SmallVector<TemplateArgument, 8> &TemplArgs,
-                           PerFileData &F, const RecordData &Record,
+  ReadTemplateArgumentList(SmallVector<TemplateArgument, 8> &TemplArgs,
+                           Module &F, const RecordData &Record,
                            unsigned &Idx);
 
   /// \brief Read a UnresolvedSet structure.
-  void ReadUnresolvedSet(UnresolvedSetImpl &Set,
+  void ReadUnresolvedSet(Module &F, UnresolvedSetImpl &Set,
                          const RecordData &Record, unsigned &Idx);
 
   /// \brief Read a C++ base specifier.
-  CXXBaseSpecifier ReadCXXBaseSpecifier(PerFileData &F,
+  CXXBaseSpecifier ReadCXXBaseSpecifier(Module &F,
                                         const RecordData &Record,unsigned &Idx);
 
   /// \brief Read a CXXCtorInitializer array.
   std::pair<CXXCtorInitializer **, unsigned>
-  ReadCXXCtorInitializers(PerFileData &F, const RecordData &Record,
+  ReadCXXCtorInitializers(Module &F, const RecordData &Record,
                           unsigned &Idx);
 
   /// \brief Read a source location from raw form.
-  SourceLocation ReadSourceLocation(PerFileData &Module, unsigned Raw) {
-    (void)Module; // No remapping yet
-    return SourceLocation::getFromRawEncoding(Raw);
+  SourceLocation ReadSourceLocation(Module &Module, unsigned Raw) const {
+    SourceLocation Loc = SourceLocation::getFromRawEncoding(Raw);
+    assert(Module.SLocRemap.find(Loc.getOffset()) != Module.SLocRemap.end() &&
+           "Cannot find offset to remap.");
+    int Remap = Module.SLocRemap.find(Loc.getOffset())->second;
+    return Loc.getLocWithOffset(Remap);
   }
 
   /// \brief Read a source location.
-  SourceLocation ReadSourceLocation(PerFileData &Module,
+  SourceLocation ReadSourceLocation(Module &Module,
                                     const RecordData &Record, unsigned& Idx) {
     return ReadSourceLocation(Module, Record[Idx++]);
   }
 
   /// \brief Read a source range.
-  SourceRange ReadSourceRange(PerFileData &F,
+  SourceRange ReadSourceRange(Module &F,
                               const RecordData &Record, unsigned& Idx);
 
   /// \brief Read an integral value
@@ -1250,17 +1201,18 @@ class ASTReader
   /// \brief Read a version tuple.
   VersionTuple ReadVersionTuple(const RecordData &Record, unsigned &Idx);
 
-  CXXTemporary *ReadCXXTemporary(const RecordData &Record, unsigned &Idx);
+  CXXTemporary *ReadCXXTemporary(Module &F, const RecordData &Record, 
+                                 unsigned &Idx);
       
   /// \brief Reads attributes from the current stream position.
-  void ReadAttributes(PerFileData &F, AttrVec &Attrs,
+  void ReadAttributes(Module &F, AttrVec &Attrs,
                       const RecordData &Record, unsigned &Idx);
 
   /// \brief Reads a statement.
-  Stmt *ReadStmt(PerFileData &F);
+  Stmt *ReadStmt(Module &F);
 
   /// \brief Reads an expression.
-  Expr *ReadExpr(PerFileData &F);
+  Expr *ReadExpr(Module &F);
 
   /// \brief Reads a sub-statement operand during statement reading.
   Stmt *ReadSubStmt() {
@@ -1276,15 +1228,16 @@ class ASTReader
   Expr *ReadSubExpr();
 
   /// \brief Reads the macro record located at the given offset.
-  PreprocessedEntity *ReadMacroRecord(PerFileData &F, uint64_t Offset);
-
-  /// \brief Reads the preprocessed entity located at the current stream
-  /// position.
-  PreprocessedEntity *LoadPreprocessedEntity(PerFileData &F);
+  void ReadMacroRecord(Module &F, uint64_t Offset);
       
+  /// \brief Determine the global preprocessed entity ID that corresponds to
+  /// the given local ID within the given module.
+  serialization::PreprocessedEntityID 
+  getGlobalPreprocessedEntityID(Module &M, unsigned LocalID) const;
+  
   /// \brief Note that the identifier is a macro whose record will be loaded
   /// from the given AST file at the given (file-local) offset.
-  void SetIdentifierIsMacro(IdentifierInfo *II, PerFileData &F,
+  void SetIdentifierIsMacro(IdentifierInfo *II, Module &F,
                             uint64_t Offset);
       
   /// \brief Read the set of macros defined by this external macro source.
@@ -1297,16 +1250,13 @@ class ASTReader
   /// into the unread macro record offsets table.
   void LoadMacroDefinition(
                      llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos);
-      
-  /// \brief Retrieve the macro definition with the given ID.
-  MacroDefinition *getMacroDefinition(serialization::MacroID ID);
-
+  
   /// \brief Retrieve the AST context that this AST reader supplements.
-  ASTContext *getContext() { return Context; }
+  ASTContext &getContext() { return Context; }
 
   // \brief Contains declarations that were loaded before we have
   // access to a Sema object.
-  llvm::SmallVector<NamedDecl *, 16> PreloadedDecls;
+  SmallVector<NamedDecl *, 16> PreloadedDecls;
 
   /// \brief Retrieve the semantic analysis object used to analyze the
   /// translation unit in which the precompiled header is being
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ASTSerializationListener.h b/contrib/llvm/tools/clang/include/clang/Serialization/ASTSerializationListener.h
deleted file mode 100644
index 0c62e0b9ca5a..000000000000
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ASTSerializationListener.h
+++ /dev/null
@@ -1,44 +0,0 @@
-//===- ASTSerializationListener.h - Decl/Type PCH Write Events -*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the ASTSerializationListener class, which is notified
-//  by the ASTWriter when an entity is serialized.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_CLANG_FRONTEND_AST_SERIALIZATION_LISTENER_H
-#define LLVM_CLANG_FRONTEND_AST_SERIALIZATION_LISTENER_H
-
-#include "llvm/Support/DataTypes.h"
-
-namespace clang {
-
-class PreprocessedEntity;
-  
-/// \brief Listener object that receives callbacks when certain kinds of 
-/// entities are serialized.
-class ASTSerializationListener {
-public:
-  virtual ~ASTSerializationListener();
-  
-  /// \brief Callback invoked whenever a preprocessed entity is serialized.
-  ///
-  /// This callback will only occur when the translation unit was created with
-  /// a detailed preprocessing record.
-  ///
-  /// \param Entity The entity that has been serialized.
-  ///
-  /// \param Offset The offset (in bits) of this entity in the resulting
-  /// AST file.
-  virtual void SerializedPreprocessedEntity(PreprocessedEntity *Entity,
-                                            uint64_t Offset) = 0;
-};
-
-}
-
-#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ASTWriter.h b/contrib/llvm/tools/clang/include/clang/Serialization/ASTWriter.h
index 18e15014be70..7a49e485f2a6 100644
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ASTWriter.h
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ASTWriter.h
@@ -38,7 +38,6 @@ namespace llvm {
 namespace clang {
 
 class ASTContext;
-class ASTSerializationListener;
 class NestedNameSpecifier;
 class CXXBaseSpecifier;
 class CXXCtorInitializer;
@@ -67,20 +66,36 @@ class VersionTuple;
 class ASTWriter : public ASTDeserializationListener,
                   public ASTMutationListener {
 public:
-  typedef llvm::SmallVector<uint64_t, 64> RecordData;
-  typedef llvm::SmallVectorImpl<uint64_t> RecordDataImpl;
+  typedef SmallVector<uint64_t, 64> RecordData;
+  typedef SmallVectorImpl<uint64_t> RecordDataImpl;
 
   friend class ASTDeclWriter;
 private:
+  /// \brief Map that provides the ID numbers of each type within the
+  /// output stream, plus those deserialized from a chained PCH.
+  ///
+  /// The ID numbers of types are consecutive (in order of discovery)
+  /// and start at 1. 0 is reserved for NULL. When types are actually
+  /// stored in the stream, the ID number is shifted by 2 bits to
+  /// allow for the const/volatile qualifiers.
+  ///
+  /// Keys in the map never have const/volatile qualifiers.
+  typedef llvm::DenseMap<QualType, serialization::TypeIdx, 
+                         serialization::UnsafeQualTypeDenseMapInfo>
+    TypeIdxMap;
+
   /// \brief The bitstream writer used to emit this precompiled header.
   llvm::BitstreamWriter &Stream;
 
+  /// \brief The ASTContext we're writing.
+  ASTContext *Context;
+                    
   /// \brief The reader of existing AST files, if we're chaining.
   ASTReader *Chain;
-
-  /// \brief A listener object that receives notifications when certain 
-  /// entities are serialized.                    
-  ASTSerializationListener *SerializationListener;
+                   
+  /// \brief Indicates when the AST writing is actively performing 
+  /// serialization, rather than just queueing updates.
+  bool WritingAST;
                     
   /// \brief Stores a declaration or a type to be written to the AST file.
   class DeclOrType {
@@ -142,7 +157,7 @@ class ASTWriter : public ASTDeserializationListener,
   /// allow for the const/volatile qualifiers.
   ///
   /// Keys in the map never have const/volatile qualifiers.
-  serialization::TypeIdxMap TypeIdxs;
+  TypeIdxMap TypeIdxs;
 
   /// \brief Offset of each type in the bitstream, indexed by
   /// the type's ID.
@@ -189,23 +204,13 @@ class ASTWriter : public ASTDeserializationListener,
 
   /// \brief The set of identifiers that had macro definitions at some point.
   std::vector<const IdentifierInfo *> DeserializedMacroNames;
-                    
-  /// \brief The first ID number we can use for our own macro definitions.
-  serialization::MacroID FirstMacroID;
-  
-  /// \brief The decl ID that will be assigned to the next new macro definition.
-  serialization::MacroID NextMacroID;
   
   /// \brief Mapping from macro definitions (as they occur in the preprocessing
   /// record) to the macro IDs.
-  llvm::DenseMap<const MacroDefinition *, serialization::MacroID>
+  llvm::DenseMap<const MacroDefinition *, serialization::PreprocessedEntityID>
       MacroDefinitions;
-  
-  /// \brief Mapping from the macro definition indices in \c MacroDefinitions
-  /// to the corresponding offsets within the preprocessor block.
-  std::vector<uint32_t> MacroDefinitionOffsets;
 
-  typedef llvm::SmallVector<uint64_t, 2> UpdateRecord;
+  typedef SmallVector<uint64_t, 2> UpdateRecord;
   typedef llvm::DenseMap<const Decl *, UpdateRecord> DeclUpdateMap;
   /// \brief Mapping from declarations that came from a chained PCH to the
   /// record containing modifications to them.
@@ -228,7 +233,7 @@ class ASTWriter : public ASTDeserializationListener,
   /// headers. The declarations themselves are stored as declaration
   /// IDs, since they will be written out to an EXTERNAL_DEFINITIONS
   /// record.
-  llvm::SmallVector<uint64_t, 16> ExternalDefinitions;
+  SmallVector<uint64_t, 16> ExternalDefinitions;
 
   /// \brief DeclContexts that have received extensions since their serialized
   /// form.
@@ -242,22 +247,39 @@ class ASTWriter : public ASTDeserializationListener,
   /// \brief Decls that will be replaced in the current dependent AST file.
   DeclsToRewriteTy DeclsToRewrite;
 
+  struct ChainedObjCCategoriesData {
+    /// \brief The interface in the imported module.
+    const ObjCInterfaceDecl *Interface;
+    /// \brief The local tail category ID that got chained to the imported
+    /// interface.
+    const ObjCCategoryDecl *TailCategory;
+    
+    /// \brief ID corresponding to \c Interface.
+    serialization::DeclID InterfaceID;
+    
+    /// \brief ID corresponding to TailCategoryID.
+    serialization::DeclID TailCategoryID;
+  };
+  /// \brief ObjC categories that got chained to an interface imported from
+  /// another module.
+  SmallVector<ChainedObjCCategoriesData, 16> LocalChainedObjCCategories;
+
   /// \brief Decls that have been replaced in the current dependent AST file.
   ///
   /// When a decl changes fundamentally after being deserialized (this shouldn't
   /// happen, but the ObjC AST nodes are designed this way), it will be
   /// serialized again. In this case, it is registered here, so that the reader
   /// knows to read the updated version.
-  llvm::SmallVector<std::pair<serialization::DeclID, uint64_t>, 16>
+  SmallVector<std::pair<serialization::DeclID, uint64_t>, 16>
       ReplacedDecls;
 
   /// \brief Statements that we've encountered while serializing a
   /// declaration or type.
-  llvm::SmallVector<Stmt *, 16> StmtsToEmit;
+  SmallVector<Stmt *, 16> StmtsToEmit;
 
   /// \brief Statements collection to use for ASTWriter::AddStmt().
   /// It will point to StmtsToEmit unless it is overriden. 
-  llvm::SmallVector<Stmt *, 16> *CollectedStmts;
+  SmallVector<Stmt *, 16> *CollectedStmts;
 
   /// \brief Mapping from SwitchCase statements to IDs.
   std::map<SwitchCase *, unsigned> SwitchCaseIDs;
@@ -280,7 +302,7 @@ class ASTWriter : public ASTDeserializationListener,
   unsigned NumVisibleDeclContexts;
 
   /// \brief The offset of each CXXBaseSpecifier set within the AST.
-  llvm::SmallVector<uint32_t, 4> CXXBaseSpecifiersOffsets;
+  SmallVector<uint32_t, 4> CXXBaseSpecifiersOffsets;
                     
   /// \brief The first ID number we can use for our own base specifiers.
   serialization::CXXBaseSpecifiersID FirstCXXBaseSpecifiersID;
@@ -306,23 +328,23 @@ class ASTWriter : public ASTDeserializationListener,
                     
   /// \brief Queue of C++ base specifiers to be written to the AST file,
   /// in the order they should be written.
-  llvm::SmallVector<QueuedCXXBaseSpecifiers, 2> CXXBaseSpecifiersToWrite;
+  SmallVector<QueuedCXXBaseSpecifiers, 2> CXXBaseSpecifiersToWrite;
                     
   /// \brief Write the given subexpression to the bitstream.
   void WriteSubStmt(Stmt *S);
 
   void WriteBlockInfoBlock();
-  void WriteMetadata(ASTContext &Context, const char *isysroot,
+  void WriteMetadata(ASTContext &Context, StringRef isysroot,
                      const std::string &OutputFile);
   void WriteLanguageOptions(const LangOptions &LangOpts);
   void WriteStatCache(MemorizeStatCalls &StatCalls);
   void WriteSourceManagerBlock(SourceManager &SourceMgr,
                                const Preprocessor &PP,
-                               const char* isysroot);
-  void WritePreprocessor(const Preprocessor &PP);
-  void WriteHeaderSearch(HeaderSearch &HS, const char* isysroot);
+                               StringRef isysroot);
+  void WritePreprocessor(const Preprocessor &PP, bool IsModule);
+  void WriteHeaderSearch(HeaderSearch &HS, StringRef isysroot);
   void WritePreprocessorDetail(PreprocessingRecord &PPRec);
-  void WritePragmaDiagnosticMappings(const Diagnostic &Diag);
+  void WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag);
   void WriteCXXBaseSpecifiersOffsets();
   void WriteType(QualType T);
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context, DeclContext *DC);
@@ -330,10 +352,13 @@ class ASTWriter : public ASTDeserializationListener,
   void WriteTypeDeclOffsets();
   void WriteSelectors(Sema &SemaRef);
   void WriteReferencedSelectorsPool(Sema &SemaRef);
-  void WriteIdentifierTable(Preprocessor &PP);
+  void WriteIdentifierTable(Preprocessor &PP, bool IsModule);
   void WriteAttributes(const AttrVec &Attrs, RecordDataImpl &Record);
+  void ResolveDeclUpdatesBlocks();
   void WriteDeclUpdatesBlocks();
   void WriteDeclReplacementsBlock();
+  void ResolveChainedObjCCategories();
+  void WriteChainedObjCCategories();
   void WriteDeclContextVisibleUpdate(const DeclContext *DC);
   void WriteFPPragmaOptions(const FPOptions &Opts);
   void WriteOpenCLExtensions(Sema &SemaRef);
@@ -356,20 +381,13 @@ class ASTWriter : public ASTDeserializationListener,
   void WriteDecl(ASTContext &Context, Decl *D);
 
   void WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                    const char* isysroot, const std::string &OutputFile);
-  void WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                     const char* isysroot);
+                    StringRef isysroot, const std::string &OutputFile,
+                    bool IsModule);
   
 public:
   /// \brief Create a new precompiled header writer that outputs to
   /// the given bitstream.
   ASTWriter(llvm::BitstreamWriter &Stream);
-
-  /// \brief Set the listener that will receive notification of serialization
-  /// events.
-  void SetSerializationListener(ASTSerializationListener *Listener) {
-    SerializationListener = Listener;
-  }
                     
   /// \brief Write a precompiled header for the given semantic analysis.
   ///
@@ -379,14 +397,14 @@ class ASTWriter : public ASTDeserializationListener,
   /// \param StatCalls the object that cached all of the stat() calls made while
   /// searching for source files and headers.
   ///
-  /// \param isysroot if non-NULL, write a relocatable PCH file whose headers
-  /// are relative to the given system root.
+  /// \param IsModule Whether we're writing a module (otherwise, we're writing a
+  /// precompiled header).
   ///
-  /// \param PPRec Record of the preprocessing actions that occurred while
-  /// preprocessing this file, e.g., macro expansions
+  /// \param isysroot if non-empty, write a relocatable file whose headers
+  /// are relative to the given system root.
   void WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
                 const std::string &OutputFile,
-                const char* isysroot);
+                bool IsModule, StringRef isysroot);
 
   /// \brief Emit a source location.
   void AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record);
@@ -432,10 +450,6 @@ class ASTWriter : public ASTDeserializationListener,
            "Identifier does not name a macro");
     return MacroOffsets[II];
   }
-
-  /// \brief Retrieve the ID number corresponding to the given macro 
-  /// definition.
-  serialization::MacroID getMacroDefinitionID(MacroDefinition *MD);
   
   /// \brief Emit a reference to a type.
   void AddTypeRef(QualType T, RecordDataImpl &Record);
@@ -447,7 +461,7 @@ class ASTWriter : public ASTDeserializationListener,
   serialization::TypeID getTypeID(QualType T) const;
 
   /// \brief Force a type to be emitted and get its index.
-  serialization::TypeIdx GetOrCreateTypeIdx(QualType T);
+  serialization::TypeIdx GetOrCreateTypeIdx( QualType T);
 
   /// \brief Determine the type index of an already-emitted type.
   serialization::TypeIdx getTypeIdx(QualType T) const;
@@ -523,7 +537,7 @@ class ASTWriter : public ASTDeserializationListener,
   void AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Record);
 
   /// \brief Add a string to the given record.
-  void AddString(llvm::StringRef Str, RecordDataImpl &Record);
+  void AddString(StringRef Str, RecordDataImpl &Record);
 
   /// \brief Add a version tuple to the given record
   void AddVersionTuple(const VersionTuple &Version, RecordDataImpl &Record);
@@ -597,7 +611,8 @@ class ASTWriter : public ASTDeserializationListener,
   void TypeRead(serialization::TypeIdx Idx, QualType T);
   void DeclRead(serialization::DeclID ID, const Decl *D);
   void SelectorRead(serialization::SelectorID ID, Selector Sel);
-  void MacroDefinitionRead(serialization::MacroID ID, MacroDefinition *MD);
+  void MacroDefinitionRead(serialization::PreprocessedEntityID ID,
+                           MacroDefinition *MD);
 
   // ASTMutationListener implementation.
   virtual void CompletedTagDefinition(const TagDecl *D);
@@ -609,6 +624,8 @@ class ASTWriter : public ASTDeserializationListener,
                                               const FunctionDecl *D);
   virtual void CompletedImplicitDefinition(const FunctionDecl *D);
   virtual void StaticDataMemberInstantiated(const VarDecl *D);
+  virtual void AddedObjCCategoryToInterface(const ObjCCategoryDecl *CatD,
+                                            const ObjCInterfaceDecl *IFD);
 };
 
 /// \brief AST and semantic-analysis consumer that generates a
@@ -616,26 +633,27 @@ class ASTWriter : public ASTDeserializationListener,
 class PCHGenerator : public SemaConsumer {
   const Preprocessor &PP;
   std::string OutputFile;
-  const char *isysroot;
-  llvm::raw_ostream *Out;
+  bool IsModule;
+  std::string isysroot;
+  raw_ostream *Out;
   Sema *SemaPtr;
   MemorizeStatCalls *StatCalls; // owned by the FileManager
   std::vector<unsigned char> Buffer;
   llvm::BitstreamWriter Stream;
   ASTWriter Writer;
-  bool Chaining;
 
 protected:
   ASTWriter &getWriter() { return Writer; }
   const ASTWriter &getWriter() const { return Writer; }
 
 public:
-  PCHGenerator(const Preprocessor &PP, const std::string &OutputFile, bool Chaining,
-               const char *isysroot, llvm::raw_ostream *Out);
+  PCHGenerator(const Preprocessor &PP, StringRef OutputFile, 
+               bool IsModule,
+               StringRef isysroot, raw_ostream *Out);
+  ~PCHGenerator();
   virtual void InitializeSema(Sema &S) { SemaPtr = &S; }
   virtual void HandleTranslationUnit(ASTContext &Ctx);
   virtual ASTMutationListener *GetASTMutationListener();
-  virtual ASTSerializationListener *GetASTSerializationListener();
   virtual ASTDeserializationListener *GetASTDeserializationListener();
 };
 
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ChainedIncludesSource.h b/contrib/llvm/tools/clang/include/clang/Serialization/ChainedIncludesSource.h
index f547902ef0eb..620dbdf40c48 100644
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ChainedIncludesSource.h
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ChainedIncludesSource.h
@@ -46,10 +46,9 @@ class ChainedIncludesSource : public ExternalSemaSource {
   virtual CXXBaseSpecifier *GetExternalCXXBaseSpecifiers(uint64_t Offset);
   virtual DeclContextLookupResult
   FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name);
-  virtual void MaterializeVisibleDecls(const DeclContext *DC);
   virtual ExternalLoadResult FindExternalLexicalDecls(const DeclContext *DC,
                                         bool (*isKindWeWant)(Decl::Kind),
-                                        llvm::SmallVectorImpl<Decl*> &Result);
+                                        SmallVectorImpl<Decl*> &Result);
   virtual void CompleteType(TagDecl *Tag);
   virtual void CompleteType(ObjCInterfaceDecl *Class);
   virtual void StartedDeserializing();
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ContinuousRangeMap.h b/contrib/llvm/tools/clang/include/clang/Serialization/ContinuousRangeMap.h
new file mode 100644
index 000000000000..7f7832041072
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ContinuousRangeMap.h
@@ -0,0 +1,120 @@
+//===--- ContinuousRangeMap.h - Map with int range as key -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ContinuousRangeMap class, which is a highly
+//  specialized container used by serialization.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SERIALIZATION_CONTINUOUS_RANGE_MAP_H
+#define LLVM_CLANG_SERIALIZATION_CONTINUOUS_RANGE_MAP_H
+
+#include "llvm/ADT/SmallVector.h"
+#include <algorithm>
+#include <utility>
+
+namespace clang {
+
+/// \brief A map from continuous integer ranges to some value, with a very
+/// specialized interface.
+///
+/// CRM maps from integer ranges to values. The ranges are continuous, i.e.
+/// where one ends, the next one begins. So if the map contains the stops I0-3,
+/// the first range is from I0 to I1, the second from I1 to I2, the third from
+/// I2 to I3 and the last from I3 to infinity.
+///
+/// Ranges must be inserted in order. Inserting a new stop I4 into the map will
+/// shrink the fourth range to I3 to I4 and add the new range I4 to inf.
+template <typename Int, typename V, unsigned InitialCapacity>
+class ContinuousRangeMap {
+public:
+  typedef std::pair<Int, V> value_type;
+  typedef value_type &reference;
+  typedef const value_type &const_reference;
+  typedef value_type *pointer;
+  typedef const value_type *const_pointer;
+
+private:
+  typedef SmallVector<value_type, InitialCapacity> Representation;
+  Representation Rep;
+
+  struct Compare {
+    bool operator ()(const_reference L, Int R) const {
+      return L.first < R;
+    }
+    bool operator ()(Int L, const_reference R) const {
+      return L < R.first;
+    }
+    bool operator ()(Int L, Int R) const { 
+      return L < R;
+    }
+    bool operator ()(const_reference L, const_reference R) const {
+      return L.first < R.first;
+    }
+  };
+
+public:
+  void insert(const value_type &Val) {
+    if (!Rep.empty() && Rep.back() == Val)
+      return;
+
+    assert((Rep.empty() || Rep.back().first < Val.first) &&
+           "Must insert keys in order.");
+    Rep.push_back(Val);
+  }
+
+  typedef typename Representation::iterator iterator;
+  typedef typename Representation::const_iterator const_iterator;
+
+  iterator begin() { return Rep.begin(); }
+  iterator end() { return Rep.end(); }
+  const_iterator begin() const { return Rep.begin(); }
+  const_iterator end() const { return Rep.end(); }
+
+  iterator find(Int K) {
+    iterator I = std::upper_bound(Rep.begin(), Rep.end(), K, Compare());
+    // I points to the first entry with a key > K, which is the range that
+    // follows the one containing K.
+    if (I == Rep.begin())
+      return Rep.end();
+    --I;
+    return I;
+  }
+  const_iterator find(Int K) const {
+    return const_cast<ContinuousRangeMap*>(this)->find(K);
+  }
+
+  reference back() { return Rep.back(); }
+  const_reference back() const { return Rep.back(); }
+  
+  /// \brief An object that helps properly build a continuous range map
+  /// from a set of values.
+  class Builder {
+    ContinuousRangeMap &Self;
+    
+    Builder(const Builder&); // DO NOT IMPLEMENT
+    Builder &operator=(const Builder&); // DO NOT IMPLEMENT
+    
+  public:
+    explicit Builder(ContinuousRangeMap &Self) : Self(Self) { }
+    
+    ~Builder() {
+      std::sort(Self.Rep.begin(), Self.Rep.end(), Compare());
+    }
+    
+    void insert(const value_type &Val) {
+      Self.Rep.push_back(Val);
+    }
+  };
+  friend class Builder;
+};
+
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/Module.h b/contrib/llvm/tools/clang/include/clang/Serialization/Module.h
new file mode 100644
index 000000000000..42b5a58e085f
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/Module.h
@@ -0,0 +1,319 @@
+//===--- Module.h - Module description --------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the Module class, which describes a module that has
+//  been loaded from an AST file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SERIALIZATION_MODULE_H
+#define LLVM_CLANG_SERIALIZATION_MODULE_H
+
+#include "clang/Serialization/ASTBitCodes.h"
+#include "clang/Serialization/ContinuousRangeMap.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/Bitcode/BitstreamReader.h"
+#include <string>
+
+namespace clang { 
+
+class DeclContext;
+  
+namespace serialization {
+   
+/// \brief Specifies the kind of module that has been loaded.
+enum ModuleKind {
+  MK_Module,   ///< File is a module proper.
+  MK_PCH,      ///< File is a PCH file treated as such.
+  MK_Preamble, ///< File is a PCH file treated as the preamble.
+  MK_MainFile  ///< File is a PCH file treated as the actual main file.
+};
+
+/// \brief Information about the contents of a DeclContext.
+struct DeclContextInfo {
+  DeclContextInfo() 
+    : NameLookupTableData(), LexicalDecls(), NumLexicalDecls() {}
+  
+  void *NameLookupTableData; // an ASTDeclContextNameLookupTable.
+  const KindDeclIDPair *LexicalDecls;
+  unsigned NumLexicalDecls;
+};
+
+/// \brief Information about a module that has been loaded by the ASTReader.
+///
+/// Each instance of the Module class corresponds to a single AST file, which 
+/// may be a precompiled header, precompiled preamble, a module, or an AST file 
+/// of some sort loaded as the main file, all of which are specific formulations of
+/// the general notion of a "module". A module may depend on any number of
+/// other modules.
+class Module {
+public:  
+  Module(ModuleKind Kind);
+  ~Module();
+  
+  // === General information ===
+  
+  /// \brief The type of this module.
+  ModuleKind Kind;
+  
+  /// \brief The file name of the module file.
+  std::string FileName;
+  
+  /// \brief Whether this module has been directly imported by the
+  /// user.
+  bool DirectlyImported;
+  
+  /// \brief The memory buffer that stores the data associated with
+  /// this AST file.
+  llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
+  
+  /// \brief The size of this file, in bits.
+  uint64_t SizeInBits;
+  
+  /// \brief The global bit offset (or base) of this module
+  uint64_t GlobalBitOffset;
+  
+  /// \brief The bitstream reader from which we'll read the AST file.
+  llvm::BitstreamReader StreamFile;
+  
+  /// \brief The main bitstream cursor for the main block.
+  llvm::BitstreamCursor Stream;
+  
+  /// \brief The source location where this module was first imported.
+  SourceLocation ImportLoc;
+  
+  /// \brief The first source location in this module.
+  SourceLocation FirstLoc;
+  
+  // === Source Locations ===
+  
+  /// \brief Cursor used to read source location entries.
+  llvm::BitstreamCursor SLocEntryCursor;
+  
+  /// \brief The number of source location entries in this AST file.
+  unsigned LocalNumSLocEntries;
+  
+  /// \brief The base ID in the source manager's view of this module.
+  int SLocEntryBaseID;
+  
+  /// \brief The base offset in the source manager's view of this module.
+  unsigned SLocEntryBaseOffset;
+  
+  /// \brief Offsets for all of the source location entries in the
+  /// AST file.
+  const uint32_t *SLocEntryOffsets;
+  
+  /// \brief SLocEntries that we're going to preload.
+  SmallVector<uint64_t, 4> PreloadSLocEntries;
+
+  /// \brief The number of source location file entries in this AST file.
+  unsigned LocalNumSLocFileEntries;
+  
+  /// \brief Offsets for all of the source location file entries in the
+  /// AST file.
+  const uint32_t *SLocFileOffsets;
+  
+  /// \brief Remapping table for source locations in this module.
+  ContinuousRangeMap<uint32_t, int, 2> SLocRemap;
+  
+  // === Identifiers ===
+  
+  /// \brief The number of identifiers in this AST file.
+  unsigned LocalNumIdentifiers;
+  
+  /// \brief Offsets into the identifier table data.
+  ///
+  /// This array is indexed by the identifier ID (-1), and provides
+  /// the offset into IdentifierTableData where the string data is
+  /// stored.
+  const uint32_t *IdentifierOffsets;
+  
+  /// \brief Base identifier ID for identifiers local to this module.
+  serialization::IdentID BaseIdentifierID;
+  
+  /// \brief Remapping table for identifier IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> IdentifierRemap;
+  
+  /// \brief Actual data for the on-disk hash table of identifiers.
+  ///
+  /// This pointer points into a memory buffer, where the on-disk hash
+  /// table for identifiers actually lives.
+  const char *IdentifierTableData;
+  
+  /// \brief A pointer to an on-disk hash table of opaque type
+  /// IdentifierHashTable.
+  void *IdentifierLookupTable;
+  
+  // === Macros ===
+  
+  /// \brief The cursor to the start of the preprocessor block, which stores
+  /// all of the macro definitions.
+  llvm::BitstreamCursor MacroCursor;
+  
+  /// \brief The offset of the start of the set of defined macros.
+  uint64_t MacroStartOffset;
+  
+  // === Detailed PreprocessingRecord ===
+  
+  /// \brief The cursor to the start of the (optional) detailed preprocessing 
+  /// record block.
+  llvm::BitstreamCursor PreprocessorDetailCursor;
+  
+  /// \brief The offset of the start of the preprocessor detail cursor.
+  uint64_t PreprocessorDetailStartOffset;
+  
+  /// \brief Base preprocessed entity ID for preprocessed entities local to 
+  /// this module.
+  serialization::PreprocessedEntityID BasePreprocessedEntityID;
+  
+  /// \brief Remapping table for preprocessed entity IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> PreprocessedEntityRemap;
+  
+  const PPEntityOffset *PreprocessedEntityOffsets;
+  unsigned NumPreprocessedEntities;
+  
+  // === Header search information ===
+  
+  /// \brief The number of local HeaderFileInfo structures.
+  unsigned LocalNumHeaderFileInfos;
+  
+  /// \brief Actual data for the on-disk hash table of header file 
+  /// information.
+  ///
+  /// This pointer points into a memory buffer, where the on-disk hash
+  /// table for header file information actually lives.
+  const char *HeaderFileInfoTableData;
+  
+  /// \brief The on-disk hash table that contains information about each of
+  /// the header files.
+  void *HeaderFileInfoTable;
+  
+  /// \brief Actual data for the list of framework names used in the header
+  /// search information.
+  const char *HeaderFileFrameworkStrings;
+  
+  // === Selectors ===
+  
+  /// \brief The number of selectors new to this file.
+  ///
+  /// This is the number of entries in SelectorOffsets.
+  unsigned LocalNumSelectors;
+  
+  /// \brief Offsets into the selector lookup table's data array
+  /// where each selector resides.
+  const uint32_t *SelectorOffsets;
+  
+  /// \brief Base selector ID for selectors local to this module.
+  serialization::SelectorID BaseSelectorID;
+  
+  /// \brief Remapping table for selector IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> SelectorRemap;
+  
+  /// \brief A pointer to the character data that comprises the selector table
+  ///
+  /// The SelectorOffsets table refers into this memory.
+  const unsigned char *SelectorLookupTableData;
+  
+  /// \brief A pointer to an on-disk hash table of opaque type
+  /// ASTSelectorLookupTable.
+  ///
+  /// This hash table provides the IDs of all selectors, and the associated
+  /// instance and factory methods.
+  void *SelectorLookupTable;
+  
+  // === Declarations ===
+  
+  /// DeclsCursor - This is a cursor to the start of the DECLS_BLOCK block. It
+  /// has read all the abbreviations at the start of the block and is ready to
+  /// jump around with these in context.
+  llvm::BitstreamCursor DeclsCursor;
+  
+  /// \brief The number of declarations in this AST file.
+  unsigned LocalNumDecls;
+  
+  /// \brief Offset of each declaration within the bitstream, indexed
+  /// by the declaration ID (-1).
+  const uint32_t *DeclOffsets;
+  
+  /// \brief Base declaration ID for declarations local to this module.
+  serialization::DeclID BaseDeclID;
+  
+  /// \brief Remapping table for declaration IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> DeclRemap;
+  
+  /// \brief The number of C++ base specifier sets in this AST file.
+  unsigned LocalNumCXXBaseSpecifiers;
+  
+  /// \brief Offset of each C++ base specifier set within the bitstream,
+  /// indexed by the C++ base specifier set ID (-1).
+  const uint32_t *CXXBaseSpecifiersOffsets;
+  
+  typedef llvm::DenseMap<const DeclContext *, DeclContextInfo>
+  DeclContextInfosMap;
+  
+  /// \brief Information about the lexical and visible declarations
+  /// for each DeclContext.
+  DeclContextInfosMap DeclContextInfos;
+
+  typedef llvm::DenseMap<serialization::GlobalDeclID,
+             std::pair<serialization::LocalDeclID, serialization::LocalDeclID> >
+    ChainedObjCCategoriesMap;
+  /// \brief ObjC categories that got chained to an interface from another
+  /// module.
+  /// Key is the ID of the interface.
+  /// Value is a pair of linked category DeclIDs (head category, tail category).
+  ChainedObjCCategoriesMap ChainedObjCCategories;
+  
+  // === Types ===
+  
+  /// \brief The number of types in this AST file.
+  unsigned LocalNumTypes;
+  
+  /// \brief Offset of each type within the bitstream, indexed by the
+  /// type ID, or the representation of a Type*.
+  const uint32_t *TypeOffsets;
+  
+  /// \brief Base type ID for types local to this module as represented in 
+  /// the global type ID space.
+  serialization::TypeID BaseTypeIndex;
+  
+  /// \brief Remapping table for type IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> TypeRemap;
+  
+  // === Miscellaneous ===
+  
+  /// \brief Diagnostic IDs and their mappings that the user changed.
+  SmallVector<uint64_t, 8> PragmaDiagMappings;
+  
+  /// \brief The AST stat cache installed for this file, if any.
+  ///
+  /// The dynamic type of this stat cache is always ASTStatCache
+  void *StatCache;
+  
+  /// \brief List of modules which depend on this module
+  llvm::SetVector<Module *> ImportedBy;
+  
+  /// \brief List of modules which this module depends on
+  llvm::SetVector<Module *> Imports;
+  
+  /// \brief Determine whether this module was directly imported at
+  /// any point during translation.
+  bool isDirectlyImported() const { return DirectlyImported; }
+  
+  /// \brief Dump debugging output for this module.
+  void dump();
+};
+
+} // end namespace serialization
+
+} // end namespace clang
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ModuleManager.h b/contrib/llvm/tools/clang/include/clang/Serialization/ModuleManager.h
new file mode 100644
index 000000000000..f86915a79cd0
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ModuleManager.h
@@ -0,0 +1,156 @@
+//===--- ModuleManager.cpp - Module Manager ---------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ModuleManager class, which manages a set of loaded
+//  modules for the ASTReader.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SERIALIZATION_MODULE_MANAGER_H
+#define LLVM_CLANG_SERIALIZATION_MODULE_MANAGER_H
+
+#include "clang/Serialization/Module.h"
+#include "clang/Basic/FileManager.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace clang { 
+
+namespace serialization {
+  
+/// \brief Manages the set of modules loaded by an AST reader.
+class ModuleManager {
+  /// \brief The chain of AST files. The first entry is the one named by the
+  /// user, the last one is the one that doesn't depend on anything further.
+  llvm::SmallVector<Module*, 2> Chain;
+  
+  /// \brief All loaded modules, indexed by name.
+  llvm::DenseMap<const FileEntry *, Module *> Modules;
+  
+  /// \brief FileManager that handles translating between filenames and
+  /// FileEntry *.
+  FileManager FileMgr;
+  
+  /// \brief A lookup of in-memory (virtual file) buffers
+  llvm::DenseMap<const FileEntry *, llvm::MemoryBuffer *> InMemoryBuffers;
+  
+public:
+  typedef SmallVector<Module*, 2>::iterator ModuleIterator;
+  typedef SmallVector<Module*, 2>::const_iterator ModuleConstIterator;
+  typedef SmallVector<Module*, 2>::reverse_iterator ModuleReverseIterator;
+  typedef std::pair<uint32_t, StringRef> ModuleOffset;
+  
+  ModuleManager(const FileSystemOptions &FSO);
+  ~ModuleManager();
+  
+  /// \brief Forward iterator to traverse all loaded modules.  This is reverse
+  /// source-order.
+  ModuleIterator begin() { return Chain.begin(); }
+  /// \brief Forward iterator end-point to traverse all loaded modules
+  ModuleIterator end() { return Chain.end(); }
+  
+  /// \brief Const forward iterator to traverse all loaded modules.  This is 
+  /// in reverse source-order.
+  ModuleConstIterator begin() const { return Chain.begin(); }
+  /// \brief Const forward iterator end-point to traverse all loaded modules
+  ModuleConstIterator end() const { return Chain.end(); }
+  
+  /// \brief Reverse iterator to traverse all loaded modules.  This is in 
+  /// source order.
+  ModuleReverseIterator rbegin() { return Chain.rbegin(); }
+  /// \brief Reverse iterator end-point to traverse all loaded modules.
+  ModuleReverseIterator rend() { return Chain.rend(); }
+  
+  /// \brief Returns the primary module associated with the manager, that is,
+  /// the first module loaded
+  Module &getPrimaryModule() { return *Chain[0]; }
+  
+  /// \brief Returns the primary module associated with the manager, that is,
+  /// the first module loaded.
+  Module &getPrimaryModule() const { return *Chain[0]; }
+  
+  /// \brief Returns the module associated with the given index
+  Module &operator[](unsigned Index) const { return *Chain[Index]; }
+  
+  /// \brief Returns the module associated with the given name
+  Module *lookup(StringRef Name);
+  
+  /// \brief Returns the in-memory (virtual file) buffer with the given name
+  llvm::MemoryBuffer *lookupBuffer(StringRef Name);
+  
+  /// \brief Number of modules loaded
+  unsigned size() const { return Chain.size(); }
+  /// \brief Attempts to create a new module and add it to the list of known
+  /// modules.
+  ///
+  /// \param FileName The file name of the module to be loaded.
+  ///
+  /// \param Type The kind of module being loaded.
+  ///
+  /// \param ImportedBy The module that is importing this module, or NULL if
+  /// this module is imported directly by the user.
+  ///
+  /// \param ErrorStr Will be set to a non-empty string if any errors occurred
+  /// while trying to load the module.
+  ///
+  /// \return A pointer to the module that corresponds to this file name,
+  /// and a boolean indicating whether the module was newly added.
+  std::pair<Module *, bool> 
+  addModule(StringRef FileName, ModuleKind Type, Module *ImportedBy,
+            std::string &ErrorStr);
+  
+  /// \brief Add an in-memory buffer the list of known buffers
+  void addInMemoryBuffer(StringRef FileName, llvm::MemoryBuffer *Buffer);
+  
+  /// \brief Visit each of the modules.
+  ///
+  /// This routine visits each of the modules, starting with the
+  /// "root" modules that no other loaded modules depend on, and
+  /// proceeding to the leaf modules, visiting each module only once
+  /// during the traversal.
+  ///
+  /// This traversal is intended to support various "lookup"
+  /// operations that can find data in any of the loaded modules.
+  ///
+  /// \param Visitor A visitor function that will be invoked with each
+  /// module and the given user data pointer. The return value must be
+  /// convertible to bool; when false, the visitation continues to
+  /// modules that the current module depends on. When true, the
+  /// visitation skips any modules that the current module depends on.
+  ///
+  /// \param UserData User data associated with the visitor object, which
+  /// will be passed along to the visitor.
+  void visit(bool (*Visitor)(Module &M, void *UserData), void *UserData);
+  
+  /// \brief Visit each of the modules with a depth-first traversal.
+  ///
+  /// This routine visits each of the modules known to the module
+  /// manager using a depth-first search, starting with the first
+  /// loaded module. The traversal invokes the callback both before
+  /// traversing the children (preorder traversal) and after
+  /// traversing the children (postorder traversal).
+  ///
+  /// \param Visitor A visitor function that will be invoked with each
+  /// module and given a \c Preorder flag that indicates whether we're
+  /// visiting the module before or after visiting its children.  The
+  /// visitor may return true at any time to abort the depth-first
+  /// visitation.
+  ///
+  /// \param UserData User data ssociated with the visitor object,
+  /// which will be passed along to the user.
+  void visitDepthFirst(bool (*Visitor)(Module &M, bool Preorder, 
+                                       void *UserData), 
+                       void *UserData);
+  
+  /// \brief View the graphviz representation of the module graph.
+  void viewGraph();
+};
+
+} } // end namespace clang::serialization
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/ClangCheckers.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/ClangCheckers.h
new file mode 100644
index 000000000000..cf0a30a73d8c
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/ClangCheckers.h
@@ -0,0 +1,22 @@
+//===--- ClangCheckers.h - Provides builtin checkers ------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_STATICANALYZER_CHECKERS_CLANGCHECKERS_H
+#define LLVM_CLANG_STATICANALYZER_CHECKERS_CLANGCHECKERS_H
+
+namespace clang {
+namespace ento {
+class CheckerRegistry;
+
+void registerBuiltinCheckers(CheckerRegistry &registry);
+
+} // end namespace ento
+} // end namespace clang
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/LocalCheckers.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/LocalCheckers.h
index 2a3d43e22325..eee38e920898 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/LocalCheckers.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Checkers/LocalCheckers.h
@@ -16,33 +16,13 @@
 #define LLVM_CLANG_GR_LOCALCHECKERS_H
 
 namespace clang {
-
-class CFG;
-class Decl;
-class Diagnostic;
-class ASTContext;
-class LangOptions;
-class ParentMap;
-class LiveVariables;
-class ObjCImplementationDecl;
-class LangOptions;
-class TranslationUnitDecl;
-
 namespace ento {
 
-class PathDiagnosticClient;
-class TransferFuncs;
-class BugType;
-class BugReporter;
 class ExprEngine;
 
-TransferFuncs* MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
-                                  const LangOptions& lopts);
-
 void RegisterCallInliner(ExprEngine &Eng);
 
-} // end GR namespace
-
+} // end namespace ento
 } // end namespace clang
 
 #endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index 3acbcd685bc0..bfb7ef8964ea 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines BugReporter, a utility class for generating
-//  PathDiagnostics for analyses based on GRState.
+//  PathDiagnostics for analyses based on ProgramState.
 //
 //===----------------------------------------------------------------------===//
 
@@ -16,7 +16,9 @@
 #define LLVM_CLANG_GR_BUGREPORTER
 
 #include "clang/Basic/SourceLocation.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitor.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableList.h"
 #include "llvm/ADT/ImmutableSet.h"
@@ -26,121 +28,138 @@
 namespace clang {
 
 class ASTContext;
-class Diagnostic;
+class DiagnosticsEngine;
 class Stmt;
 class ParentMap;
 
 namespace ento {
 
 class PathDiagnostic;
-class PathDiagnosticPiece;
-class PathDiagnosticClient;
 class ExplodedNode;
 class ExplodedGraph;
+class BugReport;
 class BugReporter;
 class BugReporterContext;
 class ExprEngine;
-class GRState;
 class BugType;
 
 //===----------------------------------------------------------------------===//
 // Interface for individual bug reports.
 //===----------------------------------------------------------------------===//
 
-class BugReporterVisitor : public llvm::FoldingSetNode {
-public:
-  virtual ~BugReporterVisitor();
-  virtual PathDiagnosticPiece* VisitNode(const ExplodedNode* N,
-                                         const ExplodedNode* PrevN,
-                                         BugReporterContext& BRC) = 0;
-
-  virtual bool isOwnedByReporterContext() { return true; }
-  virtual void Profile(llvm::FoldingSetNodeID &ID) const = 0;
-};
-
-// FIXME: Combine this with RangedBugReport and remove RangedBugReport.
-class BugReport : public BugReporterVisitor {
-protected:
-  BugType& BT;
-  std::string ShortDescription;
-  std::string Description;
-  const ExplodedNode *ErrorNode;
-  mutable SourceRange R;
-
-protected:
-  friend class BugReporter;
-  friend class BugReportEquivClass;
-
-  virtual void Profile(llvm::FoldingSetNodeID& hash) const {
-    hash.AddPointer(&BT);
-    hash.AddInteger(getLocation().getRawEncoding());
-    hash.AddString(Description);
-  }
-
+/// This class provides an interface through which checkers can create
+/// individual bug reports.
+class BugReport {
 public:
   class NodeResolver {
   public:
     virtual ~NodeResolver() {}
     virtual const ExplodedNode*
-            getOriginalNode(const ExplodedNode* N) = 0;
+            getOriginalNode(const ExplodedNode *N) = 0;
   };
 
-  BugReport(BugType& bt, llvm::StringRef desc, const ExplodedNode *errornode)
-    : BT(bt), Description(desc), ErrorNode(errornode) {}
+  typedef const SourceRange *ranges_iterator;
+  typedef llvm::ImmutableList<BugReporterVisitor*>::iterator visitor_iterator;
+  typedef SmallVector<StringRef, 2> ExtraTextList;
 
-  BugReport(BugType& bt, llvm::StringRef shortDesc, llvm::StringRef desc,
+protected:
+  friend class BugReporter;
+  friend class BugReportEquivClass;
+
+  BugType& BT;
+  std::string ShortDescription;
+  std::string Description;
+  PathDiagnosticLocation Location;
+  const ExplodedNode *ErrorNode;
+  SmallVector<SourceRange, 4> Ranges;
+  ExtraTextList ExtraText;
+
+  // Not the most efficient data structure, but we use an ImmutableList for the
+  // Callbacks because it is safe to make additions to list during iteration.
+  llvm::ImmutableList<BugReporterVisitor*>::Factory F;
+  llvm::ImmutableList<BugReporterVisitor*> Callbacks;
+  llvm::FoldingSet<BugReporterVisitor> CallbacksSet;
+
+public:
+  BugReport(BugType& bt, StringRef desc, const ExplodedNode *errornode)
+    : BT(bt), Description(desc), ErrorNode(errornode),
+      Callbacks(F.getEmptyList()) {}
+
+  BugReport(BugType& bt, StringRef shortDesc, StringRef desc,
             const ExplodedNode *errornode)
-  : BT(bt), ShortDescription(shortDesc), Description(desc),
-    ErrorNode(errornode) {}
+    : BT(bt), ShortDescription(shortDesc), Description(desc),
+      ErrorNode(errornode), Callbacks(F.getEmptyList()) {}
+
+  BugReport(BugType& bt, StringRef desc, PathDiagnosticLocation l)
+    : BT(bt), Description(desc), Location(l), ErrorNode(0),
+      Callbacks(F.getEmptyList()) {}
 
   virtual ~BugReport();
 
-  virtual bool isOwnedByReporterContext() { return false; }
-
   const BugType& getBugType() const { return BT; }
   BugType& getBugType() { return BT; }
 
-  // FIXME: Perhaps this should be moved into a subclass?
-  const ExplodedNode* getErrorNode() const { return ErrorNode; }
+  const ExplodedNode *getErrorNode() const { return ErrorNode; }
 
-  // FIXME: Do we need this?  Maybe getLocation() should return a ProgramPoint
-  // object.
-  // FIXME: If we do need it, we can probably just make it private to
-  // BugReporter.
-  const Stmt* getStmt() const;
+  const StringRef getDescription() const { return Description; }
 
-  const llvm::StringRef getDescription() const { return Description; }
-
-  const llvm::StringRef getShortDescription() const {
+  const StringRef getShortDescription() const {
     return ShortDescription.empty() ? Description : ShortDescription;
   }
 
-  // FIXME: Is this needed?
-  virtual std::pair<const char**,const char**> getExtraDescriptiveText() {
-    return std::make_pair((const char**)0,(const char**)0);
+  /// \brief This allows for addition of meta data to the diagnostic.
+  ///
+  /// Currently, only the HTMLDiagnosticClient knows how to display it. 
+  void addExtraText(StringRef S) {
+    ExtraText.push_back(S);
   }
 
-  // FIXME: Perhaps move this into a subclass.
-  virtual PathDiagnosticPiece* getEndPath(BugReporterContext& BRC,
-                                          const ExplodedNode* N);
+  virtual const ExtraTextList &getExtraText() {
+    return ExtraText;
+  }
 
-  /// getLocation - Return the "definitive" location of the reported bug.
+  /// \brief Return the "definitive" location of the reported bug.
+  ///
   ///  While a bug can span an entire path, usually there is a specific
   ///  location that can be used to identify where the key issue occurred.
   ///  This location is used by clients rendering diagnostics.
-  virtual SourceLocation getLocation() const;
+  virtual PathDiagnosticLocation getLocation(const SourceManager &SM) const;
 
-  typedef const SourceRange *ranges_iterator;
+  const Stmt *getStmt() const;
 
-  /// getRanges - Returns the source ranges associated with this bug.
-  virtual std::pair<ranges_iterator, ranges_iterator> getRanges() const;
+  /// \brief Add a range to a bug report.
+  ///
+  /// Ranges are used to highlight regions of interest in the source code.
+  /// They should be at the same source code line as the BugReport location.
+  /// By default, the source range of the statement corresponding to the error
+  /// node will be used; add a single invalid range to specify absence of
+  /// ranges.
+  void addRange(SourceRange R) {
+    assert((R.isValid() || Ranges.empty()) && "Invalid range can only be used "
+                           "to specify that the report does not have a range.");
+    Ranges.push_back(R);
+  }
 
-  virtual PathDiagnosticPiece* VisitNode(const ExplodedNode* N,
-                                         const ExplodedNode* PrevN,
-                                         BugReporterContext& BR);
+  /// \brief Get the SourceRanges associated with the report.
+  virtual std::pair<ranges_iterator, ranges_iterator> getRanges();
 
-  virtual void registerInitialVisitors(BugReporterContext& BRC,
-                                       const ExplodedNode* N) {}
+  /// \brief Add custom or predefined bug report visitors to this report.
+  ///
+  /// The visitors should be used when the default trace is not sufficient.
+  /// For example, they allow constructing a more elaborate trace.
+  /// \sa registerConditionVisitor(), registerTrackNullOrUndefValue(),
+  /// registerFindLastStore(), registerNilReceiverVisitor(), and
+  /// registerVarDeclsLastStore().
+  void addVisitor(BugReporterVisitor *visitor);
+
+	/// Iterators through the custom diagnostic visitors.
+  visitor_iterator visitor_begin() { return Callbacks.begin(); }
+  visitor_iterator visitor_end() { return Callbacks.end(); }
+
+  /// Profile to identify equivalent bug reports for error report coalescing.
+  /// Reports are uniqued to ensure that we do not emit multiple diagnostics
+  /// for each bug.
+  virtual void Profile(llvm::FoldingSetNodeID& hash) const;
 };
 
 //===----------------------------------------------------------------------===//
@@ -148,7 +167,7 @@ class BugReport : public BugReporterVisitor {
 //===----------------------------------------------------------------------===//
 
 class BugReportEquivClass : public llvm::FoldingSetNode {
-  // List of *owned* BugReport objects.
+  /// List of *owned* BugReport objects.
   std::list<BugReport*> Reports;
 
   friend class BugReporter;
@@ -166,9 +185,9 @@ class BugReportEquivClass : public llvm::FoldingSetNode {
     std::list<BugReport*>::iterator impl;
   public:
     iterator(std::list<BugReport*>::iterator i) : impl(i) {}
-    iterator& operator++() { ++impl; return *this; }
-    bool operator==(const iterator& I) const { return I.impl == impl; }
-    bool operator!=(const iterator& I) const { return I.impl != impl; }
+    iterator &operator++() { ++impl; return *this; }
+    bool operator==(const iterator &I) const { return I.impl == impl; }
+    bool operator!=(const iterator &I) const { return I.impl != impl; }
     BugReport* operator*() const { return *impl; }
     BugReport* operator->() const { return *impl; }
   };
@@ -177,9 +196,9 @@ class BugReportEquivClass : public llvm::FoldingSetNode {
     std::list<BugReport*>::const_iterator impl;
   public:
     const_iterator(std::list<BugReport*>::const_iterator i) : impl(i) {}
-    const_iterator& operator++() { ++impl; return *this; }
-    bool operator==(const const_iterator& I) const { return I.impl == impl; }
-    bool operator!=(const const_iterator& I) const { return I.impl != impl; }
+    const_iterator &operator++() { ++impl; return *this; }
+    bool operator==(const const_iterator &I) const { return I.impl == impl; }
+    bool operator!=(const const_iterator &I) const { return I.impl != impl; }
     const BugReport* operator*() const { return *impl; }
     const BugReport* operator->() const { return *impl; }
   };
@@ -191,78 +210,6 @@ class BugReportEquivClass : public llvm::FoldingSetNode {
   const_iterator end() const { return const_iterator(Reports.end()); }
 };
 
-
-//===----------------------------------------------------------------------===//
-// Specialized subclasses of BugReport.
-//===----------------------------------------------------------------------===//
-
-// FIXME: Collapse this with the default BugReport class.
-class RangedBugReport : public BugReport {
-  llvm::SmallVector<SourceRange, 4> Ranges;
-public:
-  RangedBugReport(BugType& D, llvm::StringRef description,
-                  ExplodedNode *errornode)
-    : BugReport(D, description, errornode) {}
-
-  RangedBugReport(BugType& D, llvm::StringRef shortDescription,
-                  llvm::StringRef description, ExplodedNode *errornode)
-  : BugReport(D, shortDescription, description, errornode) {}
-
-  ~RangedBugReport();
-
-  // FIXME: Move this out of line.
-  void addRange(SourceRange R) {
-    assert(R.isValid());
-    Ranges.push_back(R);
-  }
-
-  virtual std::pair<ranges_iterator, ranges_iterator> getRanges() const {
-    return std::make_pair(Ranges.begin(), Ranges.end());
-  }
-  
-  virtual void Profile(llvm::FoldingSetNodeID& hash) const {
-    BugReport::Profile(hash);
-    for (llvm::SmallVectorImpl<SourceRange>::const_iterator I =
-          Ranges.begin(), E = Ranges.end(); I != E; ++I) {
-      const SourceRange range = *I;
-      if (!range.isValid())
-        continue;
-      hash.AddInteger(range.getBegin().getRawEncoding());
-      hash.AddInteger(range.getEnd().getRawEncoding());
-    }
-  }
-};
-
-class EnhancedBugReport : public RangedBugReport {
-public:
-  typedef void (*VisitorCreator)(BugReporterContext &BRcC, const void *data,
-                                 const ExplodedNode *N);
-
-private:
-  typedef std::vector<std::pair<VisitorCreator, const void*> > Creators;
-  Creators creators;
-
-public:
-  EnhancedBugReport(BugType& D, llvm::StringRef description,
-                    ExplodedNode *errornode)
-   : RangedBugReport(D, description, errornode) {}
-
-  EnhancedBugReport(BugType& D, llvm::StringRef shortDescription,
-                   llvm::StringRef description, ExplodedNode *errornode)
-    : RangedBugReport(D, shortDescription, description, errornode) {}
-
-  ~EnhancedBugReport() {}
-
-  void registerInitialVisitors(BugReporterContext& BRC, const ExplodedNode* N) {
-    for (Creators::iterator I = creators.begin(), E = creators.end(); I!=E; ++I)
-      I->first(BRC, I->second, N);
-  }
-
-  void addVisitorCreator(VisitorCreator creator, const void *data) {
-    creators.push_back(std::make_pair(creator, data));
-  }
-};
-
 //===----------------------------------------------------------------------===//
 // BugReporter and friends.
 //===----------------------------------------------------------------------===//
@@ -270,12 +217,15 @@ class EnhancedBugReport : public RangedBugReport {
 class BugReporterData {
 public:
   virtual ~BugReporterData();
-  virtual Diagnostic& getDiagnostic() = 0;
-  virtual PathDiagnosticClient* getPathDiagnosticClient() = 0;
-  virtual ASTContext& getASTContext() = 0;
+  virtual DiagnosticsEngine& getDiagnostic() = 0;
+  virtual PathDiagnosticConsumer* getPathDiagnosticConsumer() = 0;
+  virtual ASTContext &getASTContext() = 0;
   virtual SourceManager& getSourceManager() = 0;
 };
 
+/// BugReporter is a utility class for generating PathDiagnostics for analysis.
+/// It collects the BugReports and BugTypes and knows how to generate
+/// and flush the corresponding diagnostics.
 class BugReporter {
 public:
   enum Kind { BaseBRKind, GRBugReporterKind };
@@ -288,9 +238,13 @@ class BugReporter {
   const Kind kind;
   BugReporterData& D;
 
+  /// Generate and flush the diagnostics for the given bug report.
   void FlushReport(BugReportEquivClass& EQ);
 
+  /// The set of bug reports tracked by the BugReporter.
   llvm::FoldingSet<BugReportEquivClass> EQClasses;
+  /// A vector of BugReports for tracking the allocated pointers and cleanup.
+  std::vector<BugReportEquivClass *> EQClassesVector;
 
 protected:
   BugReporter(BugReporterData& d, Kind k) : BugTypes(F.getEmptySet()), kind(k),
@@ -301,63 +255,71 @@ class BugReporter {
                                     D(d) {}
   virtual ~BugReporter();
 
+  /// \brief Generate and flush diagnostics for all bug reports.
   void FlushReports();
 
   Kind getKind() const { return kind; }
 
-  Diagnostic& getDiagnostic() {
+  DiagnosticsEngine& getDiagnostic() {
     return D.getDiagnostic();
   }
 
-  PathDiagnosticClient* getPathDiagnosticClient() {
-    return D.getPathDiagnosticClient();
+  PathDiagnosticConsumer* getPathDiagnosticConsumer() {
+    return D.getPathDiagnosticConsumer();
   }
 
+  /// \brief Iterator over the set of BugTypes tracked by the BugReporter.
   typedef BugTypesTy::iterator iterator;
   iterator begin() { return BugTypes.begin(); }
   iterator end() { return BugTypes.end(); }
 
+  /// \brief Iterator over the set of BugReports tracked by the BugReporter.
   typedef llvm::FoldingSet<BugReportEquivClass>::iterator EQClasses_iterator;
   EQClasses_iterator EQClasses_begin() { return EQClasses.begin(); }
   EQClasses_iterator EQClasses_end() { return EQClasses.end(); }
 
-  ASTContext& getContext() { return D.getASTContext(); }
+  ASTContext &getContext() { return D.getASTContext(); }
 
   SourceManager& getSourceManager() { return D.getSourceManager(); }
 
   virtual void GeneratePathDiagnostic(PathDiagnostic& pathDiagnostic,
-        llvm::SmallVectorImpl<BugReport *> &bugReports) {}
+        SmallVectorImpl<BugReport *> &bugReports) {}
 
   void Register(BugType *BT);
 
+  /// \brief Add the given report to the set of reports tracked by BugReporter.
+  ///
+  /// The reports are usually generated by the checkers. Further, they are
+  /// folded based on the profile value, which is done to coalesce similar
+  /// reports.
   void EmitReport(BugReport *R);
 
-  void EmitBasicReport(llvm::StringRef BugName, llvm::StringRef BugStr,
-                       SourceLocation Loc,
+  void EmitBasicReport(StringRef BugName, StringRef BugStr,
+                       PathDiagnosticLocation Loc,
                        SourceRange* RangeBeg, unsigned NumRanges);
 
-  void EmitBasicReport(llvm::StringRef BugName, llvm::StringRef BugCategory,
-                       llvm::StringRef BugStr, SourceLocation Loc,
+  void EmitBasicReport(StringRef BugName, StringRef BugCategory,
+                       StringRef BugStr, PathDiagnosticLocation Loc,
                        SourceRange* RangeBeg, unsigned NumRanges);
 
 
-  void EmitBasicReport(llvm::StringRef BugName, llvm::StringRef BugStr,
-                       SourceLocation Loc) {
+  void EmitBasicReport(StringRef BugName, StringRef BugStr,
+                       PathDiagnosticLocation Loc) {
     EmitBasicReport(BugName, BugStr, Loc, 0, 0);
   }
 
-  void EmitBasicReport(llvm::StringRef BugName, llvm::StringRef BugCategory,
-                       llvm::StringRef BugStr, SourceLocation Loc) {
+  void EmitBasicReport(StringRef BugName, StringRef BugCategory,
+                       StringRef BugStr, PathDiagnosticLocation Loc) {
     EmitBasicReport(BugName, BugCategory, BugStr, Loc, 0, 0);
   }
 
-  void EmitBasicReport(llvm::StringRef BugName, llvm::StringRef BugStr,
-                       SourceLocation Loc, SourceRange R) {
+  void EmitBasicReport(StringRef BugName, StringRef BugStr,
+                       PathDiagnosticLocation Loc, SourceRange R) {
     EmitBasicReport(BugName, BugStr, Loc, &R, 1);
   }
 
-  void EmitBasicReport(llvm::StringRef BugName, llvm::StringRef Category,
-                       llvm::StringRef BugStr, SourceLocation Loc,
+  void EmitBasicReport(StringRef BugName, StringRef Category,
+                       StringRef BugStr, PathDiagnosticLocation Loc,
                        SourceRange R) {
     EmitBasicReport(BugName, Category, BugStr, Loc, &R, 1);
   }
@@ -369,7 +331,7 @@ class BugReporter {
 
   /// \brief Returns a BugType that is associated with the given name and
   /// category.
-  BugType *getBugTypeForName(llvm::StringRef name, llvm::StringRef category);
+  BugType *getBugTypeForName(StringRef name, StringRef category);
 };
 
 // FIXME: Get rid of GRBugReporter.  It's the wrong abstraction.
@@ -392,10 +354,10 @@ class GRBugReporter : public BugReporter {
 
   /// getStateManager - Return the state manager used by the analysis
   ///  engine.
-  GRStateManager &getStateManager();
+  ProgramStateManager &getStateManager();
 
   virtual void GeneratePathDiagnostic(PathDiagnostic &pathDiagnostic,
-                     llvm::SmallVectorImpl<BugReport*> &bugReports);
+                     SmallVectorImpl<BugReport*> &bugReports);
 
   void addNotableSymbol(SymbolRef Sym) {
     NotableSymbols.insert(Sym);
@@ -413,20 +375,10 @@ class GRBugReporter : public BugReporter {
 
 class BugReporterContext {
   GRBugReporter &BR;
-  // Not the most efficient data structure, but we use an ImmutableList for the
-  // Callbacks because it is safe to make additions to list during iteration.
-  llvm::ImmutableList<BugReporterVisitor*>::Factory F;
-  llvm::ImmutableList<BugReporterVisitor*> Callbacks;
-  llvm::FoldingSet<BugReporterVisitor> CallbacksSet;
 public:
-  BugReporterContext(GRBugReporter& br) : BR(br), Callbacks(F.getEmptyList()) {}
-  virtual ~BugReporterContext();
+  BugReporterContext(GRBugReporter& br) : BR(br) {}
 
-  void addVisitor(BugReporterVisitor* visitor);
-
-  typedef llvm::ImmutableList<BugReporterVisitor*>::iterator visitor_iterator;
-  visitor_iterator visitor_begin() { return Callbacks.begin(); }
-  visitor_iterator visitor_end() { return Callbacks.end(); }
+  virtual ~BugReporterContext() {}
 
   GRBugReporter& getBugReporter() { return BR; }
 
@@ -442,7 +394,7 @@ class BugReporterContext {
     return BR.isNotable(Sym);
   }
 
-  GRStateManager& getStateManager() {
+  ProgramStateManager& getStateManager() {
     return BR.getStateManager();
   }
 
@@ -450,7 +402,7 @@ class BugReporterContext {
     return getStateManager().getSValBuilder();
   }
 
-  ASTContext& getASTContext() {
+  ASTContext &getASTContext() {
     return BR.getContext();
   }
 
@@ -461,50 +413,6 @@ class BugReporterContext {
   virtual BugReport::NodeResolver& getNodeResolver() = 0;
 };
 
-class DiagBugReport : public RangedBugReport {
-  std::list<std::string> Strs;
-  FullSourceLoc L;
-public:
-  DiagBugReport(BugType& D, llvm::StringRef desc, FullSourceLoc l) :
-  RangedBugReport(D, desc, 0), L(l) {}
-
-  virtual ~DiagBugReport() {}
-
-  // FIXME: Move out-of-line (virtual function).
-  SourceLocation getLocation() const { return L; }
-
-  void addString(llvm::StringRef s) { Strs.push_back(s); }
-
-  typedef std::list<std::string>::const_iterator str_iterator;
-  str_iterator str_begin() const { return Strs.begin(); }
-  str_iterator str_end() const { return Strs.end(); }
-};
-
-//===----------------------------------------------------------------------===//
-//===----------------------------------------------------------------------===//
-
-namespace bugreporter {
-
-const Stmt *GetDerefExpr(const ExplodedNode *N);
-const Stmt *GetDenomExpr(const ExplodedNode *N);
-const Stmt *GetCalleeExpr(const ExplodedNode *N);
-const Stmt *GetRetValExpr(const ExplodedNode *N);
-
-void registerTrackNullOrUndefValue(BugReporterContext& BRC, const void *stmt,
-                                   const ExplodedNode* N);
-
-void registerFindLastStore(BugReporterContext& BRC, const void *memregion,
-                           const ExplodedNode *N);
-
-void registerNilReceiverVisitor(BugReporterContext &BRC);
-
-void registerVarDeclsLastStore(BugReporterContext &BRC, const void *stmt,
-                               const ExplodedNode *N);
-
-} // end namespace clang::bugreporter
-
-//===----------------------------------------------------------------------===//
-
 } // end GR namespace
 
 } // end clang namespace
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitor.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitor.h
new file mode 100644
index 000000000000..41c0a3a46b1c
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitor.h
@@ -0,0 +1,183 @@
+//===---  BugReporterVisitor.h - Generate PathDiagnostics -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file declares BugReporterVisitors, which are used to generate enhanced
+//  diagnostic traces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_GR_BUGREPORTERVISITOR
+#define LLVM_CLANG_GR_BUGREPORTERVISITOR
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
+#include "llvm/ADT/FoldingSet.h"
+
+namespace clang {
+
+namespace ento {
+
+class BugReport;
+class BugReporterContext;
+class ExplodedNode;
+class MemRegion;
+class PathDiagnosticPiece;
+
+class BugReporterVisitor : public llvm::FoldingSetNode {
+public:
+  virtual ~BugReporterVisitor();
+
+  /// \brief Return a diagnostic piece which should be associated with the
+  /// given node.
+  ///
+  /// The last parameter can be used to register a new visitor with the given
+  /// BugReport while processing a node.
+  virtual PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                         const ExplodedNode *PrevN,
+                                         BugReporterContext &BRC,
+                                         BugReport &BR) = 0;
+
+  /// \brief Provide custom definition for the final diagnostic piece on the
+  /// path - the piece, which is displayed before the path is expanded.
+  ///
+  /// If returns NULL the default implementation will be used.
+  /// Also note that at most one visitor of a BugReport should generate a
+  /// non-NULL end of path diagnostic piece.
+  virtual PathDiagnosticPiece *getEndPath(BugReporterContext &BRC,
+                                          const ExplodedNode *N,
+                                          BugReport &BR);
+
+  virtual void Profile(llvm::FoldingSetNodeID &ID) const = 0;
+
+  /// \brief Generates the default final diagnostic piece.
+  static PathDiagnosticPiece *getDefaultEndPath(BugReporterContext &BRC,
+                                                const ExplodedNode *N,
+                                                BugReport &BR);
+
+};
+
+class FindLastStoreBRVisitor : public BugReporterVisitor {
+  const MemRegion *R;
+  SVal V;
+  bool satisfied;
+  const ExplodedNode *StoreSite;
+
+public:
+  /// \brief Convenience method to create a visitor given only the MemRegion.
+  /// Returns NULL if the visitor cannot be created. For example, when the
+  /// corresponding value is unknown.
+  static BugReporterVisitor *createVisitorObject(const ExplodedNode *N,
+                                                 const MemRegion *R);
+
+  /// Creates a visitor for every VarDecl inside a Stmt and registers it with
+  /// the BugReport.
+  static void registerStatementVarDecls(BugReport &BR, const Stmt *S);
+
+  FindLastStoreBRVisitor(SVal v, const MemRegion *r)
+  : R(r), V(v), satisfied(false), StoreSite(0) {
+    assert (!V.isUnknown() && "Cannot track unknown value.");
+
+    // TODO: Does it make sense to allow undef values here?
+    // (If not, also see UndefCapturedBlockVarChecker)?
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const;
+
+  PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                 const ExplodedNode *PrevN,
+                                 BugReporterContext &BRC,
+                                 BugReport &BR);
+};
+
+class TrackConstraintBRVisitor : public BugReporterVisitor {
+  DefinedSVal Constraint;
+  const bool Assumption;
+  bool isSatisfied;
+
+public:
+  TrackConstraintBRVisitor(DefinedSVal constraint, bool assumption)
+  : Constraint(constraint), Assumption(assumption), isSatisfied(false) {}
+
+  void Profile(llvm::FoldingSetNodeID &ID) const;
+
+  PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                 const ExplodedNode *PrevN,
+                                 BugReporterContext &BRC,
+                                 BugReport &BR);
+};
+
+class NilReceiverBRVisitor : public BugReporterVisitor {
+public:
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    static int x = 0;
+    ID.AddPointer(&x);
+  }
+
+  PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                 const ExplodedNode *PrevN,
+                                 BugReporterContext &BRC,
+                                 BugReport &BR);
+};
+
+/// Visitor that tries to report interesting diagnostics from conditions.
+class ConditionBRVisitor : public BugReporterVisitor {
+public:
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    static int x = 0;
+    ID.AddPointer(&x);
+  }
+
+  virtual PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                         const ExplodedNode *Prev,
+                                         BugReporterContext &BRC,
+                                         BugReport &BR);
+
+  PathDiagnosticPiece *VisitTerminator(const Stmt *Term,
+                                       const ExplodedNode *N,
+                                       const CFGBlock *srcBlk,
+                                       const CFGBlock *dstBlk,
+                                       BugReporterContext &BRC);
+
+  PathDiagnosticPiece *VisitTrueTest(const Expr *Cond,
+                                     bool tookTrue,
+                                     BugReporterContext &BRC,
+                                     const LocationContext *LC);
+
+  PathDiagnosticPiece *VisitTrueTest(const Expr *Cond,
+                                     const DeclRefExpr *DR,
+                                     const bool tookTrue,
+                                     BugReporterContext &BRC,
+                                     const LocationContext *LC);
+
+  PathDiagnosticPiece *VisitTrueTest(const Expr *Cond,
+                                     const BinaryOperator *BExpr,
+                                     const bool tookTrue,
+                                     BugReporterContext &BRC,
+                                     const LocationContext *LC);
+
+  bool patternMatch(const Expr *Ex,
+                    llvm::raw_ostream &Out,
+                    BugReporterContext &BRC);
+};
+
+namespace bugreporter {
+
+BugReporterVisitor *getTrackNullOrUndefValueVisitor(const ExplodedNode *N,
+                                                    const Stmt *S);
+
+const Stmt *GetDerefExpr(const ExplodedNode *N);
+const Stmt *GetDenomExpr(const ExplodedNode *N);
+const Stmt *GetCalleeExpr(const ExplodedNode *N);
+const Stmt *GetRetValExpr(const ExplodedNode *N);
+
+} // end namespace clang
+} // end namespace ento
+} // end namespace bugreporter
+
+
+#endif //LLVM_CLANG_GR__BUGREPORTERVISITOR
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h
index 7b9bb03d8d05..78067cd61c1e 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_CLANG_ANALYSIS_BUGTYPE
 #define LLVM_CLANG_ANALYSIS_BUGTYPE
 
-#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "llvm/ADT/FoldingSet.h"
 #include <string>
 
@@ -22,6 +21,7 @@ namespace clang {
 
 namespace ento {
 
+class BugReporter;
 class ExplodedNode;
 class ExprEngine;
 
@@ -31,13 +31,13 @@ class BugType {
   const std::string Category;
   bool SuppressonSink;
 public:
-  BugType(llvm::StringRef name, llvm::StringRef cat)
+  BugType(StringRef name, StringRef cat)
     : Name(name), Category(cat), SuppressonSink(false) {}
   virtual ~BugType();
 
   // FIXME: Should these be made strings as well?
-  llvm::StringRef getName() const { return Name; }
-  llvm::StringRef getCategory() const { return Category; }
+  StringRef getName() const { return Name; }
+  StringRef getCategory() const { return Category; }
   
   /// isSuppressOnSink - Returns true if bug reports associated with this bug
   ///  type should be suppressed if the end node of the report is post-dominated
@@ -57,7 +57,7 @@ class BuiltinBug : public BugType {
   BuiltinBug(const char *name)
     : BugType(name, "Logic error"), desc(name) {}
   
-  llvm::StringRef getDescription() const { return desc; }
+  StringRef getDescription() const { return desc; }
 };
 
 } // end GR namespace
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h
index 6d53c097d29f..406be3cc4bd4 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h
@@ -16,6 +16,7 @@
 
 #include "clang/Basic/Diagnostic.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/PointerUnion.h"
 #include <deque>
 #include <iterator>
 #include <string>
@@ -23,42 +24,55 @@
 
 namespace clang {
 
+class AnalysisContext;
+class BinaryOperator;
+class CompoundStmt;
 class Decl;
+class LocationContext;
+class MemberExpr;
+class ParentMap;
+class ProgramPoint;
 class SourceManager;
 class Stmt;
 
 namespace ento {
 
+class ExplodedNode;
+
 //===----------------------------------------------------------------------===//
 // High-level interface for handlers of path-sensitive diagnostics.
 //===----------------------------------------------------------------------===//
 
 class PathDiagnostic;
 
-class PathDiagnosticClient : public DiagnosticClient  {
+class PathDiagnosticConsumer {
 public:
-  PathDiagnosticClient() {}
+  PathDiagnosticConsumer() {}
 
-  virtual ~PathDiagnosticClient() {}
+  virtual ~PathDiagnosticConsumer() {}
   
   virtual void
-  FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade = 0) = 0;
+  FlushDiagnostics(SmallVectorImpl<std::string> *FilesMade = 0) = 0;
   
-  void FlushDiagnostics(llvm::SmallVectorImpl<std::string> &FilesMade) {
+  void FlushDiagnostics(SmallVectorImpl<std::string> &FilesMade) {
     FlushDiagnostics(&FilesMade);
   }
   
-  virtual llvm::StringRef getName() const = 0;
+  virtual StringRef getName() const = 0;
   
-  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                const DiagnosticInfo &Info);
-  virtual void HandlePathDiagnostic(const PathDiagnostic* D) = 0;
+  void HandlePathDiagnostic(const PathDiagnostic* D);
 
   enum PathGenerationScheme { Minimal, Extensive };
   virtual PathGenerationScheme getGenerationScheme() const { return Minimal; }
   virtual bool supportsLogicalOpControlFlow() const { return false; }
   virtual bool supportsAllBlockEdges() const { return false; }
   virtual bool useVerboseDescription() const { return true; }
+
+protected:
+  /// The actual logic for handling path diagnostics, as implemented
+  /// by subclasses of PathDiagnosticConsumer.
+  virtual void HandlePathDiagnosticImpl(const PathDiagnostic* D) = 0;
+
 };
 
 //===----------------------------------------------------------------------===//
@@ -67,60 +81,143 @@ class PathDiagnosticClient : public DiagnosticClient  {
 
 class PathDiagnosticRange : public SourceRange {
 public:
-  const bool isPoint;
+  bool isPoint;
 
   PathDiagnosticRange(const SourceRange &R, bool isP = false)
     : SourceRange(R), isPoint(isP) {}
+
+  PathDiagnosticRange() : isPoint(false) {}
 };
 
+typedef llvm::PointerUnion<const LocationContext*, AnalysisContext*>
+                                                   LocationOrAnalysisContext;
+
 class PathDiagnosticLocation {
 private:
   enum Kind { RangeK, SingleLocK, StmtK, DeclK } K;
-  SourceRange R;
   const Stmt *S;
   const Decl *D;
   const SourceManager *SM;
+  FullSourceLoc Loc;
+  PathDiagnosticRange Range;
+
+  PathDiagnosticLocation(SourceLocation L, const SourceManager &sm,
+                         Kind kind)
+    : K(kind), S(0), D(0), SM(&sm),
+      Loc(genLocation(L)), Range(genRange()) {
+    assert(Loc.isValid());
+    assert(Range.isValid());
+  }
+
+  FullSourceLoc
+    genLocation(SourceLocation L = SourceLocation(),
+                LocationOrAnalysisContext LAC = (AnalysisContext*)0) const;
+
+  PathDiagnosticRange
+    genRange(LocationOrAnalysisContext LAC = (AnalysisContext*)0) const;
+
 public:
+  /// Create an invalid location.
   PathDiagnosticLocation()
     : K(SingleLocK), S(0), D(0), SM(0) {}
 
-  PathDiagnosticLocation(FullSourceLoc L)
-    : K(SingleLocK), R(L, L), S(0), D(0), SM(&L.getManager()) {}
-
-  PathDiagnosticLocation(const Stmt *s, const SourceManager &sm)
-    : K(StmtK), S(s), D(0), SM(&sm) {}
-
-  PathDiagnosticLocation(SourceRange r, const SourceManager &sm)
-    : K(RangeK), R(r), S(0), D(0), SM(&sm) {}
+  /// Create a location corresponding to the given statement.
+  PathDiagnosticLocation(const Stmt *s,
+                         const SourceManager &sm,
+                         LocationOrAnalysisContext lac)
+    : K(StmtK), S(s), D(0), SM(&sm),
+      Loc(genLocation(SourceLocation(), lac)),
+      Range(genRange(lac)) {
+    assert(Loc.isValid());
+    assert(Range.isValid());
+  }
 
+  /// Create a location corresponding to the given declaration.
   PathDiagnosticLocation(const Decl *d, const SourceManager &sm)
-    : K(DeclK), S(0), D(d), SM(&sm) {}
+    : K(DeclK), S(0), D(d), SM(&sm),
+      Loc(genLocation()), Range(genRange()) {
+    assert(Loc.isValid());
+    assert(Range.isValid());
+  }
+
+  /// Create a location corresponding to the given declaration.
+  static PathDiagnosticLocation create(const Decl *D,
+                                       const SourceManager &SM) {
+    return PathDiagnosticLocation(D, SM);
+  }
+
+  /// Create a location for the beginning of the declaration.
+  static PathDiagnosticLocation createBegin(const Decl *D,
+                                            const SourceManager &SM);
+
+  /// Create a location for the beginning of the statement.
+  static PathDiagnosticLocation createBegin(const Stmt *S,
+                                            const SourceManager &SM,
+                                            const LocationOrAnalysisContext LAC);
+
+  /// Create the location for the operator of the binary expression.
+  /// Assumes the statement has a valid location.
+  static PathDiagnosticLocation createOperatorLoc(const BinaryOperator *BO,
+                                                  const SourceManager &SM);
+
+  /// For member expressions, return the location of the '.' or '->'.
+  /// Assumes the statement has a valid location.
+  static PathDiagnosticLocation createMemberLoc(const MemberExpr *ME,
+                                                const SourceManager &SM);
+
+  /// Create a location for the beginning of the compound statement.
+  /// Assumes the statement has a valid location.
+  static PathDiagnosticLocation createBeginBrace(const CompoundStmt *CS,
+                                                 const SourceManager &SM);
+
+  /// Create a location for the end of the compound statement.
+  /// Assumes the statement has a valid location.
+  static PathDiagnosticLocation createEndBrace(const CompoundStmt *CS,
+                                               const SourceManager &SM);
+
+  /// Create a location for the beginning of the enclosing declaration body.
+  /// Defaults to the beginning of the first statement in the declaration body.
+  static PathDiagnosticLocation createDeclBegin(const LocationContext *LC,
+                                                const SourceManager &SM);
+
+  /// Constructs a location for the end of the enclosing declaration body.
+  /// Defaults to the end of brace.
+  static PathDiagnosticLocation createDeclEnd(const LocationContext *LC,
+                                                   const SourceManager &SM);
+
+  /// Create a location corresponding to the given valid ExplodedNode.
+  static PathDiagnosticLocation create(const ProgramPoint& P,
+                                       const SourceManager &SMng);
+
+  /// Create a location corresponding to the next valid ExplodedNode as end
+  /// of path location.
+  static PathDiagnosticLocation createEndOfPath(const ExplodedNode* N,
+                                                const SourceManager &SM);
+
+  /// Convert the given location into a single kind location.
+  static PathDiagnosticLocation createSingleLocation(
+                                             const PathDiagnosticLocation &PDL);
 
   bool operator==(const PathDiagnosticLocation &X) const {
-    return K == X.K && R == X.R && S == X.S && D == X.D;
+    return K == X.K && Loc == X.Loc && Range == X.Range;
   }
 
   bool operator!=(const PathDiagnosticLocation &X) const {
-    return K != X.K || R != X.R || S != X.S || D != X.D;;
-  }
-
-  PathDiagnosticLocation& operator=(const PathDiagnosticLocation &X) {
-    K = X.K;
-    R = X.R;
-    S = X.S;
-    D = X.D;
-    SM = X.SM;
-    return *this;
+    return !(*this == X);
   }
 
   bool isValid() const {
     return SM != 0;
   }
 
-  const SourceManager& getSourceManager() const { assert(isValid());return *SM;}
+  FullSourceLoc asLocation() const {
+    return Loc;
+  }
+
+  PathDiagnosticRange asRange() const {
+    return Range;
+  }
 
-  FullSourceLoc asLocation() const;
-  PathDiagnosticRange asRange() const;
   const Stmt *asStmt() const { assert(isValid()); return S; }
   const Decl *asDecl() const { assert(isValid()); return D; }
 
@@ -181,7 +278,7 @@ class PathDiagnosticPiece {
   PathDiagnosticPiece& operator=(const PathDiagnosticPiece &P);
 
 protected:
-  PathDiagnosticPiece(llvm::StringRef s, Kind k, DisplayHint hint = Below);
+  PathDiagnosticPiece(StringRef s, Kind k, DisplayHint hint = Below);
 
   PathDiagnosticPiece(Kind k, DisplayHint hint = Below);
 
@@ -191,7 +288,7 @@ class PathDiagnosticPiece {
   const std::string& getString() const { return str; }
 
   /// getDisplayHint - Return a hint indicating where the diagnostic should
-  ///  be displayed by the PathDiagnosticClient.
+  ///  be displayed by the PathDiagnosticConsumer.
   DisplayHint getDisplayHint() const { return Hint; }
 
   virtual PathDiagnosticLocation getLocation() const = 0;
@@ -199,9 +296,15 @@ class PathDiagnosticPiece {
 
   Kind getKind() const { return kind; }
 
-  void addRange(SourceRange R) { ranges.push_back(R); }
+  void addRange(SourceRange R) {
+    if (!R.isValid())
+      return;
+    ranges.push_back(R);
+  }
 
   void addRange(SourceLocation B, SourceLocation E) {
+    if (!B.isValid() || !E.isValid())
+      return;
     ranges.push_back(SourceRange(B,E));
   }
 
@@ -230,7 +333,7 @@ class PathDiagnosticPiece {
                    : &FixItHints[0] + FixItHints.size();
   }
 
-  static inline bool classof(const PathDiagnosticPiece* P) {
+  static inline bool classof(const PathDiagnosticPiece *P) {
     return true;
   }
   
@@ -242,11 +345,11 @@ class PathDiagnosticSpotPiece : public PathDiagnosticPiece {
   PathDiagnosticLocation Pos;
 public:
   PathDiagnosticSpotPiece(const PathDiagnosticLocation &pos,
-                          llvm::StringRef s,
+                          StringRef s,
                           PathDiagnosticPiece::Kind k,
                           bool addPosRange = true)
   : PathDiagnosticPiece(s, k), Pos(pos) {
-    assert(Pos.asLocation().isValid() &&
+    assert(Pos.isValid() && Pos.asLocation().isValid() &&
            "PathDiagnosticSpotPiece's must have a valid location.");
     if (addPosRange && Pos.hasRange()) addRange(Pos.asRange());
   }
@@ -261,12 +364,12 @@ class PathDiagnosticEventPiece : public PathDiagnosticSpotPiece {
 
 public:
   PathDiagnosticEventPiece(const PathDiagnosticLocation &pos,
-                           llvm::StringRef s, bool addPosRange = true)
+                           StringRef s, bool addPosRange = true)
     : PathDiagnosticSpotPiece(pos, s, Event, addPosRange) {}
 
   ~PathDiagnosticEventPiece();
 
-  static inline bool classof(const PathDiagnosticPiece* P) {
+  static inline bool classof(const PathDiagnosticPiece *P) {
     return P->getKind() == Event;
   }
 };
@@ -276,7 +379,7 @@ class PathDiagnosticControlFlowPiece : public PathDiagnosticPiece {
 public:
   PathDiagnosticControlFlowPiece(const PathDiagnosticLocation &startPos,
                                  const PathDiagnosticLocation &endPos,
-                                 llvm::StringRef s)
+                                 StringRef s)
     : PathDiagnosticPiece(s, ControlFlow) {
       LPairs.push_back(PathDiagnosticLocationPair(startPos, endPos));
     }
@@ -320,7 +423,7 @@ class PathDiagnosticControlFlowPiece : public PathDiagnosticPiece {
   const_iterator begin() const { return LPairs.begin(); }
   const_iterator end() const   { return LPairs.end(); }
 
-  static inline bool classof(const PathDiagnosticPiece* P) {
+  static inline bool classof(const PathDiagnosticPiece *P) {
     return P->getKind() == ControlFlow;
   }
   
@@ -337,7 +440,7 @@ class PathDiagnosticMacroPiece : public PathDiagnosticSpotPiece {
 
   bool containsEvent() const;
 
-  void push_back(PathDiagnosticPiece* P) { SubPieces.push_back(P); }
+  void push_back(PathDiagnosticPiece *P) { SubPieces.push_back(P); }
 
   typedef std::vector<PathDiagnosticPiece*>::iterator iterator;
   iterator begin() { return SubPieces.begin(); }
@@ -352,7 +455,7 @@ class PathDiagnosticMacroPiece : public PathDiagnosticSpotPiece {
   const_iterator begin() const { return SubPieces.begin(); }
   const_iterator end() const { return SubPieces.end(); }
 
-  static inline bool classof(const PathDiagnosticPiece* P) {
+  static inline bool classof(const PathDiagnosticPiece *P) {
     return P->getKind() == Macro;
   }
   
@@ -373,42 +476,42 @@ class PathDiagnostic : public llvm::FoldingSetNode {
 public:
   PathDiagnostic();
 
-  PathDiagnostic(llvm::StringRef bugtype, llvm::StringRef desc,
-                 llvm::StringRef category);
+  PathDiagnostic(StringRef bugtype, StringRef desc,
+                 StringRef category);
 
   ~PathDiagnostic();
 
-  llvm::StringRef getDescription() const { return Desc; }
-  llvm::StringRef getBugType() const { return BugType; }
-  llvm::StringRef getCategory() const { return Category; }
+  StringRef getDescription() const { return Desc; }
+  StringRef getBugType() const { return BugType; }
+  StringRef getCategory() const { return Category; }
 
   typedef std::deque<std::string>::const_iterator meta_iterator;
   meta_iterator meta_begin() const { return OtherDesc.begin(); }
   meta_iterator meta_end() const { return OtherDesc.end(); }
-  void addMeta(llvm::StringRef s) { OtherDesc.push_back(s); }
+  void addMeta(StringRef s) { OtherDesc.push_back(s); }
 
   PathDiagnosticLocation getLocation() const {
     assert(Size > 0 && "getLocation() requires a non-empty PathDiagnostic.");
     return rbegin()->getLocation();
   }
 
-  void push_front(PathDiagnosticPiece* piece) {
+  void push_front(PathDiagnosticPiece *piece) {
     assert(piece);
     path.push_front(piece);
     ++Size;
   }
 
-  void push_back(PathDiagnosticPiece* piece) {
+  void push_back(PathDiagnosticPiece *piece) {
     assert(piece);
     path.push_back(piece);
     ++Size;
   }
 
-  PathDiagnosticPiece* back() {
+  PathDiagnosticPiece *back() {
     return path.back();
   }
 
-  const PathDiagnosticPiece* back() const {
+  const PathDiagnosticPiece *back() const {
     return path.back();
   }
 
@@ -433,14 +536,14 @@ class PathDiagnostic : public llvm::FoldingSetNode {
   public:
     iterator(const ImplTy& i) : I(i) {}
 
-    bool operator==(const iterator& X) const { return I == X.I; }
-    bool operator!=(const iterator& X) const { return I != X.I; }
+    bool operator==(const iterator &X) const { return I == X.I; }
+    bool operator!=(const iterator &X) const { return I != X.I; }
 
     PathDiagnosticPiece& operator*() const { return **I; }
-    PathDiagnosticPiece* operator->() const { return *I; }
+    PathDiagnosticPiece *operator->() const { return *I; }
 
-    iterator& operator++() { ++I; return *this; }
-    iterator& operator--() { --I; return *this; }
+    iterator &operator++() { ++I; return *this; }
+    iterator &operator--() { --I; return *this; }
   };
 
   class const_iterator {
@@ -459,14 +562,14 @@ class PathDiagnostic : public llvm::FoldingSetNode {
   public:
     const_iterator(const ImplTy& i) : I(i) {}
 
-    bool operator==(const const_iterator& X) const { return I == X.I; }
-    bool operator!=(const const_iterator& X) const { return I != X.I; }
+    bool operator==(const const_iterator &X) const { return I == X.I; }
+    bool operator!=(const const_iterator &X) const { return I != X.I; }
 
     reference operator*() const { return **I; }
     pointer operator->() const { return *I; }
 
-    const_iterator& operator++() { ++I; return *this; }
-    const_iterator& operator--() { --I; return *this; }
+    const_iterator &operator++() { ++I; return *this; }
+    const_iterator &operator--() { --I; return *this; }
   };
 
   typedef std::reverse_iterator<iterator>       reverse_iterator;
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/Checker.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/Checker.h
index eb38bd8951e9..1e4edeb0c7ae 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/Checker.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/Checker.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_SA_CORE_CHECKER
 #define LLVM_CLANG_SA_CORE_CHECKER
 
+#include "clang/Analysis/ProgramPoint.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 #include "llvm/Support/Casting.h"
@@ -151,9 +152,10 @@ class PostObjCMessage {
 
 class Location {
   template <typename CHECKER>
-  static void _checkLocation(void *checker, const SVal &location, bool isLoad,
+  static void _checkLocation(void *checker,
+                             const SVal &location, bool isLoad, const Stmt *S,
                              CheckerContext &C) {
-    ((const CHECKER *)checker)->checkLocation(location, isLoad, C);
+    ((const CHECKER *)checker)->checkLocation(location, isLoad, S, C);
   }
 
 public:
@@ -166,9 +168,10 @@ class Location {
 
 class Bind {
   template <typename CHECKER>
-  static void _checkBind(void *checker, const SVal &location, const SVal &val,
+  static void _checkBind(void *checker,
+                         const SVal &location, const SVal &val, const Stmt *S,
                          CheckerContext &C) {
-    ((const CHECKER *)checker)->checkBind(location, val, C);
+    ((const CHECKER *)checker)->checkBind(location, val, S, C);
   }
 
 public:
@@ -227,7 +230,7 @@ class BranchCondition {
 
 class LiveSymbols {
   template <typename CHECKER>
-  static void _checkLiveSymbols(void *checker, const GRState *state,
+  static void _checkLiveSymbols(void *checker, const ProgramState *state,
                                 SymbolReaper &SR) {
     ((const CHECKER *)checker)->checkLiveSymbols(state, SR);
   }
@@ -257,15 +260,18 @@ class DeadSymbols {
 
 class RegionChanges {
   template <typename CHECKER>
-  static const GRState *_checkRegionChanges(void *checker, const GRState *state,
-                            const StoreManager::InvalidatedSymbols *invalidated,
-                                            const MemRegion * const *Begin,
-                                            const MemRegion * const *End) {
+  static const ProgramState *
+  _checkRegionChanges(void *checker,
+                      const ProgramState *state,
+                      const StoreManager::InvalidatedSymbols *invalidated,
+                      ArrayRef<const MemRegion *> Explicits,
+                      ArrayRef<const MemRegion *> Regions) {
     return ((const CHECKER *)checker)->checkRegionChanges(state, invalidated,
-                                                          Begin, End);
+                                                          Explicits, Regions);
   }
   template <typename CHECKER>
-  static bool _wantsRegionChangeUpdate(void *checker, const GRState *state) {
+  static bool _wantsRegionChangeUpdate(void *checker,
+                                       const ProgramState *state) {
     return ((const CHECKER *)checker)->wantsRegionChangeUpdate(state);
   }
 
@@ -300,8 +306,10 @@ namespace eval {
 
 class Assume {
   template <typename CHECKER>
-  static const GRState *_evalAssume(void *checker, const GRState *state,
-                                    const SVal &cond, bool assumption) {
+  static const ProgramState *_evalAssume(void *checker,
+                                         const ProgramState *state,
+                                         const SVal &cond,
+                                         bool assumption) {
     return ((const CHECKER *)checker)->evalAssume(state, cond, assumption);
   }
 
@@ -327,38 +335,73 @@ class Call {
   }
 };
 
+class InlineCall {
+  template <typename CHECKER>
+  static bool _inlineCall(void *checker, const CallExpr *CE,
+                                         ExprEngine &Eng,
+                                         ExplodedNode *Pred,
+                                         ExplodedNodeSet &Dst) {
+    return ((const CHECKER *)checker)->inlineCall(CE, Eng, Pred, Dst);
+  }
+
+public:
+  template <typename CHECKER>
+  static void _register(CHECKER *checker, CheckerManager &mgr) {
+    mgr._registerForInlineCall(
+                 CheckerManager::InlineCallFunc(checker, _inlineCall<CHECKER>));
+  }
+};
+
 } // end eval namespace
 
+class CheckerBase : public ProgramPointTag {
+public:
+  StringRef getTagDescription() const;
+
+  /// See CheckerManager::runCheckersForPrintState.
+  virtual void printState(raw_ostream &Out, const ProgramState *State,
+                          const char *NL, const char *Sep) const { }
+};
+  
 template <typename CHECK1, typename CHECK2=check::_VoidCheck,
           typename CHECK3=check::_VoidCheck, typename CHECK4=check::_VoidCheck,
           typename CHECK5=check::_VoidCheck, typename CHECK6=check::_VoidCheck,
           typename CHECK7=check::_VoidCheck, typename CHECK8=check::_VoidCheck,
           typename CHECK9=check::_VoidCheck, typename CHECK10=check::_VoidCheck,
-          typename CHECK11=check::_VoidCheck,typename CHECK12=check::_VoidCheck>
+          typename CHECK11=check::_VoidCheck,typename CHECK12=check::_VoidCheck,
+          typename CHECK13=check::_VoidCheck,typename CHECK14=check::_VoidCheck,
+          typename CHECK15=check::_VoidCheck,typename CHECK16=check::_VoidCheck>
 class Checker;
 
 template <>
 class Checker<check::_VoidCheck, check::_VoidCheck, check::_VoidCheck,
                 check::_VoidCheck, check::_VoidCheck, check::_VoidCheck,
                 check::_VoidCheck, check::_VoidCheck, check::_VoidCheck,
-                check::_VoidCheck, check::_VoidCheck, check::_VoidCheck> {
+                check::_VoidCheck, check::_VoidCheck, check::_VoidCheck,
+                check::_VoidCheck, check::_VoidCheck, check::_VoidCheck,
+                check::_VoidCheck> 
+  : public CheckerBase 
+{
 public:
   static void _register(void *checker, CheckerManager &mgr) { }
 };
 
 template <typename CHECK1, typename CHECK2, typename CHECK3, typename CHECK4,
           typename CHECK5, typename CHECK6, typename CHECK7, typename CHECK8,
-          typename CHECK9, typename CHECK10,typename CHECK11,typename CHECK12>
+          typename CHECK9, typename CHECK10,typename CHECK11,typename CHECK12,
+          typename CHECK13,typename CHECK14,typename CHECK15,typename CHECK16>
 class Checker
     : public CHECK1,
       public Checker<CHECK2, CHECK3, CHECK4, CHECK5, CHECK6, CHECK7, CHECK8,
-                       CHECK9, CHECK10, CHECK11, CHECK12> {
+                     CHECK9, CHECK10,CHECK11,CHECK12,CHECK13,CHECK14,CHECK15,
+                     CHECK16> {
 public:
   template <typename CHECKER>
   static void _register(CHECKER *checker, CheckerManager &mgr) {
     CHECK1::_register(checker, mgr);
-    Checker<CHECK2, CHECK3, CHECK4, CHECK5, CHECK6, CHECK7, CHECK8, CHECK9,
-              CHECK10, CHECK11,CHECK12>::_register(checker, mgr);
+    Checker<CHECK2, CHECK3, CHECK4, CHECK5, CHECK6, CHECK7, CHECK8,
+            CHECK9, CHECK10,CHECK11,CHECK12,CHECK13,CHECK14,CHECK15,
+            CHECK16>::_register(checker, mgr);
   }
 };
 
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
index 45d38fba0495..e3e4c49f71e1 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
@@ -27,6 +27,7 @@ namespace clang {
   class CallExpr;
 
 namespace ento {
+  class CheckerBase;
   class ExprEngine;
   class AnalysisManager;
   class BugReporter;
@@ -36,7 +37,7 @@ namespace ento {
   class ExplodedNode;
   class ExplodedNodeSet;
   class ExplodedGraph;
-  class GRState;
+  class ProgramState;
   class EndOfFunctionNodeBuilder;
   class BranchNodeBuilder;
   class MemRegion;
@@ -55,8 +56,8 @@ class CheckerFn<RET(P1, P2, P3, P4)> {
   typedef RET (*Func)(void *, P1, P2, P3, P4);
   Func Fn;
 public:
-  void *Checker;
-  CheckerFn(void *checker, Func fn) : Fn(fn), Checker(checker) { }
+  CheckerBase *Checker;
+  CheckerFn(CheckerBase *checker, Func fn) : Fn(fn), Checker(checker) { }
   RET operator()(P1 p1, P2 p2, P3 p3, P4 p4) const { 
     return Fn(Checker, p1, p2, p3, p4);
   } 
@@ -67,8 +68,8 @@ class CheckerFn<RET(P1, P2, P3)> {
   typedef RET (*Func)(void *, P1, P2, P3);
   Func Fn;
 public:
-  void *Checker;
-  CheckerFn(void *checker, Func fn) : Fn(fn), Checker(checker) { }
+  CheckerBase *Checker;
+  CheckerFn(CheckerBase *checker, Func fn) : Fn(fn), Checker(checker) { }
   RET operator()(P1 p1, P2 p2, P3 p3) const { return Fn(Checker, p1, p2, p3); } 
 };
 
@@ -77,8 +78,8 @@ class CheckerFn<RET(P1, P2)> {
   typedef RET (*Func)(void *, P1, P2);
   Func Fn;
 public:
-  void *Checker;
-  CheckerFn(void *checker, Func fn) : Fn(fn), Checker(checker) { }
+  CheckerBase *Checker;
+  CheckerFn(CheckerBase *checker, Func fn) : Fn(fn), Checker(checker) { }
   RET operator()(P1 p1, P2 p2) const { return Fn(Checker, p1, p2); } 
 };
 
@@ -87,8 +88,8 @@ class CheckerFn<RET(P1)> {
   typedef RET (*Func)(void *, P1);
   Func Fn;
 public:
-  void *Checker;
-  CheckerFn(void *checker, Func fn) : Fn(fn), Checker(checker) { }
+  CheckerBase *Checker;
+  CheckerFn(CheckerBase *checker, Func fn) : Fn(fn), Checker(checker) { }
   RET operator()(P1 p1) const { return Fn(Checker, p1); } 
 };
 
@@ -97,8 +98,8 @@ class CheckerFn<RET()> {
   typedef RET (*Func)(void *);
   Func Fn;
 public:
-  void *Checker;
-  CheckerFn(void *checker, Func fn) : Fn(fn), Checker(checker) { }
+  CheckerBase *Checker;
+  CheckerFn(CheckerBase *checker, Func fn) : Fn(fn), Checker(checker) { }
   RET operator()() const { return Fn(Checker); } 
 };
 
@@ -115,8 +116,8 @@ class CheckerManager {
 
   const LangOptions &getLangOptions() const { return LangOpts; }
 
-  typedef void *CheckerRef;
-  typedef void *CheckerTag;
+  typedef CheckerBase *CheckerRef;
+  typedef const void *CheckerTag;
   typedef CheckerFn<void ()> CheckerDtor;
 
 //===----------------------------------------------------------------------===//
@@ -157,6 +158,11 @@ class CheckerManager {
 //===----------------------------------------------------------------------===//
 
   /// \brief Run checkers for pre-visiting Stmts.
+  ///
+  /// The notification is performed for every explored CFGElement, which does
+  /// not include the control flow statements such as IfStmt.
+  ///
+  /// \sa runCheckersForBranchCondition, runCheckersForPostStmt
   void runCheckersForPreStmt(ExplodedNodeSet &Dst,
                              const ExplodedNodeSet &Src,
                              const Stmt *S,
@@ -165,6 +171,11 @@ class CheckerManager {
   }
 
   /// \brief Run checkers for post-visiting Stmts.
+  ///
+  /// The notification is performed for every explored CFGElement, which does
+  /// not include the control flow statements such as IfStmt.
+  ///
+  /// \sa runCheckersForBranchCondition, runCheckersForPreStmt
   void runCheckersForPostStmt(ExplodedNodeSet &Dst,
                               const ExplodedNodeSet &Src,
                               const Stmt *S,
@@ -224,27 +235,43 @@ class CheckerManager {
                                      BranchNodeBuilder &B, ExprEngine &Eng);
 
   /// \brief Run checkers for live symbols.
-  void runCheckersForLiveSymbols(const GRState *state,
+  ///
+  /// Allows modifying SymbolReaper object. For example, checkers can explicitly
+  /// register symbols of interest as live. These symbols will not be marked
+  /// dead and removed.
+  void runCheckersForLiveSymbols(const ProgramState *state,
                                  SymbolReaper &SymReaper);
 
   /// \brief Run checkers for dead symbols.
+  ///
+  /// Notifies checkers when symbols become dead. For example, this allows
+  /// checkers to aggressively clean up/reduce the checker state and produce
+  /// precise diagnostics.
   void runCheckersForDeadSymbols(ExplodedNodeSet &Dst,
                                  const ExplodedNodeSet &Src,
                                  SymbolReaper &SymReaper, const Stmt *S,
                                  ExprEngine &Eng);
 
   /// \brief True if at least one checker wants to check region changes.
-  bool wantsRegionChangeUpdate(const GRState *state);
+  bool wantsRegionChangeUpdate(const ProgramState *state);
 
   /// \brief Run checkers for region changes.
-  const GRState *
-  runCheckersForRegionChanges(const GRState *state,
+  ///
+  /// This corresponds to the check::RegionChanges callback.
+  /// \param state The current program state.
+  /// \param invalidated A set of all symbols potentially touched by the change.
+  /// \param ExplicitRegions The regions explicitly requested for invalidation.
+  ///   For example, in the case of a function call, these would be arguments.
+  /// \param Regions The transitive closure of accessible regions,
+  ///   i.e. all regions that may have been touched by this change.
+  const ProgramState *
+  runCheckersForRegionChanges(const ProgramState *state,
                             const StoreManager::InvalidatedSymbols *invalidated,
-                              const MemRegion * const *Begin,
-                              const MemRegion * const *End);
+                              ArrayRef<const MemRegion *> ExplicitRegions,
+                              ArrayRef<const MemRegion *> Regions);
 
   /// \brief Run checkers for handling assumptions on symbolic values.
-  const GRState *runCheckersForEvalAssume(const GRState *state,
+  const ProgramState *runCheckersForEvalAssume(const ProgramState *state,
                                           SVal Cond, bool Assumption);
 
   /// \brief Run checkers for evaluating a call.
@@ -254,10 +281,21 @@ class CheckerManager {
                               GraphExpander *defaultEval = 0);
   
   /// \brief Run checkers for the entire Translation Unit.
-  void runCheckersOnEndOfTranslationUnit(const TranslationUnitDecl* TU,
+  void runCheckersOnEndOfTranslationUnit(const TranslationUnitDecl *TU,
                                          AnalysisManager &mgr,
                                          BugReporter &BR);
 
+  /// \brief Run checkers for debug-printing a ProgramState.
+  ///
+  /// Unlike most other callbacks, any checker can simply implement the virtual
+  /// method CheckerBase::printState if it has custom data to print.
+  /// \param Out The output stream
+  /// \param State The state being printed
+  /// \param NL The preferred representation of a newline.
+  /// \param Sep The preferred separator between different kinds of data.
+  void runCheckersForPrintState(raw_ostream &Out, const ProgramState *State,
+                                const char *NL, const char *Sep);
+
 //===----------------------------------------------------------------------===//
 // Internal registration functions for AST traversing.
 //===----------------------------------------------------------------------===//
@@ -282,11 +320,13 @@ class CheckerManager {
   typedef CheckerFn<void (const ObjCMessage &, CheckerContext &)>
       CheckObjCMessageFunc;
   
-  typedef CheckerFn<void (const SVal &location, bool isLoad, CheckerContext &)>
+  typedef CheckerFn<void (const SVal &location, bool isLoad, const Stmt *S,
+                          CheckerContext &)>
       CheckLocationFunc;
   
-  typedef CheckerFn<void (const SVal &location, const SVal &val,
-                          CheckerContext &)> CheckBindFunc;
+  typedef CheckerFn<void (const SVal &location, const SVal &val, 
+                          const Stmt *S, CheckerContext &)> 
+      CheckBindFunc;
   
   typedef CheckerFn<void (ExplodedGraph &, BugReporter &, ExprEngine &)>
       CheckEndAnalysisFunc;
@@ -300,23 +340,28 @@ class CheckerManager {
   typedef CheckerFn<void (SymbolReaper &, CheckerContext &)>
       CheckDeadSymbolsFunc;
   
-  typedef CheckerFn<void (const GRState *,SymbolReaper &)> CheckLiveSymbolsFunc;
+  typedef CheckerFn<void (const ProgramState *,SymbolReaper &)> CheckLiveSymbolsFunc;
   
-  typedef CheckerFn<const GRState * (const GRState *,
+  typedef CheckerFn<const ProgramState * (const ProgramState *,
                                 const StoreManager::InvalidatedSymbols *symbols,
-                                     const MemRegion * const *begin,
-                                     const MemRegion * const *end)>
+                                    ArrayRef<const MemRegion *> ExplicitRegions,
+                                          ArrayRef<const MemRegion *> Regions)>
       CheckRegionChangesFunc;
   
-  typedef CheckerFn<bool (const GRState *)> WantsRegionChangeUpdateFunc;
+  typedef CheckerFn<bool (const ProgramState *)> WantsRegionChangeUpdateFunc;
   
-  typedef CheckerFn<const GRState * (const GRState *,
-                                     const SVal &cond, bool assumption)>
+  typedef CheckerFn<const ProgramState * (const ProgramState *,
+                                          const SVal &cond, bool assumption)>
       EvalAssumeFunc;
   
   typedef CheckerFn<bool (const CallExpr *, CheckerContext &)>
       EvalCallFunc;
 
+  typedef CheckerFn<bool (const CallExpr *, ExprEngine &Eng,
+                                            ExplodedNode *Pred,
+                                            ExplodedNodeSet &Dst)>
+      InlineCallFunc;
+
   typedef CheckerFn<void (const TranslationUnitDecl *,
                           AnalysisManager&, BugReporter &)>
       CheckEndOfTranslationUnit;
@@ -351,6 +396,8 @@ class CheckerManager {
 
   void _registerForEvalCall(EvalCallFunc checkfn);
 
+  void _registerForInlineCall(InlineCallFunc checkfn);
+
   void _registerForEndOfTranslationUnit(CheckEndOfTranslationUnit checkfn);
 
 //===----------------------------------------------------------------------===//
@@ -405,7 +452,7 @@ class CheckerManager {
 
   std::vector<CheckDeclFunc> BodyCheckers;
 
-  typedef llvm::SmallVector<CheckDeclFunc, 4> CachedDeclCheckers;
+  typedef SmallVector<CheckDeclFunc, 4> CachedDeclCheckers;
   typedef llvm::DenseMap<unsigned, CachedDeclCheckers> CachedDeclCheckersMapTy;
   CachedDeclCheckersMapTy CachedDeclCheckersMap;
 
@@ -439,7 +486,7 @@ class CheckerManager {
   };
   friend struct llvm::DenseMapInfo<CachedStmtCheckersKey>;
 
-  typedef llvm::SmallVector<CheckStmtFunc, 4> CachedStmtCheckers;
+  typedef SmallVector<CheckStmtFunc, 4> CachedStmtCheckers;
   typedef llvm::DenseMap<CachedStmtCheckersKey, CachedStmtCheckers>
       CachedStmtCheckersMapTy;
   CachedStmtCheckersMapTy CachedStmtCheckersMap;
@@ -473,10 +520,12 @@ class CheckerManager {
 
   std::vector<EvalCallFunc> EvalCallCheckers;
 
+  std::vector<InlineCallFunc> InlineCallCheckers;
+
   std::vector<CheckEndOfTranslationUnit> EndOfTranslationUnitCheckers;
 
   struct EventInfo {
-    llvm::SmallVector<CheckEventFunc, 4> Checkers;
+    SmallVector<CheckEventFunc, 4> Checkers;
     bool HasDispatcher;
     EventInfo() : HasDispatcher(false) { }
   };
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerOptInfo.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerOptInfo.h
new file mode 100644
index 000000000000..6ce5b3c5095e
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerOptInfo.h
@@ -0,0 +1,43 @@
+//===--- CheckerOptInfo.h - Specifies which checkers to use -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_STATICANALYZER_CORE_CHECKEROPTINFO_H
+#define LLVM_CLANG_STATICANALYZER_CORE_CHECKEROPTINFO_H
+
+#include "clang/Basic/LLVM.h"
+
+namespace clang {
+namespace ento {
+
+/// Represents a request to include or exclude a checker or package from a
+/// specific analysis run.
+///
+/// \sa CheckerRegistry::initializeManager
+class CheckerOptInfo {
+  StringRef Name;
+  bool Enable;
+  bool Claimed;
+
+public:
+  CheckerOptInfo(StringRef name, bool enable)
+    : Name(name), Enable(enable), Claimed(false) { }
+  
+  StringRef getName() const { return Name; }
+  bool isEnabled() const { return Enable; }
+  bool isDisabled() const { return !isEnabled(); }
+
+  bool isClaimed() const { return Claimed; }
+  bool isUnclaimed() const { return !isClaimed(); }
+  void claim() { Claimed = true; }
+};
+
+} // end namespace ento
+} // end namespace clang
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerProvider.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerProvider.h
deleted file mode 100644
index b8aaaa1a04c0..000000000000
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerProvider.h
+++ /dev/null
@@ -1,58 +0,0 @@
-//===--- CheckerProvider.h - Static Analyzer Checkers Provider --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines the Static Analyzer Checker Provider.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_SA_CORE_CHECKERPROVIDER_H
-#define LLVM_CLANG_SA_CORE_CHECKERPROVIDER_H
-
-#include "llvm/ADT/StringRef.h"
-
-namespace llvm {
-  class raw_ostream;
-}
-
-namespace clang {
-
-namespace ento {
-  class CheckerManager;
-
-class CheckerOptInfo {
-  const char *Name;
-  bool Enable;
-  bool Claimed;
-
-public:
-  CheckerOptInfo(const char *name, bool enable)
-    : Name(name), Enable(enable), Claimed(false) { }
-  
-  const char *getName() const { return Name; }
-  bool isEnabled() const { return Enable; }
-  bool isDisabled() const { return !isEnabled(); }
-
-  bool isClaimed() const { return Claimed; }
-  bool isUnclaimed() const { return !isClaimed(); }
-  void claim() { Claimed = true; }
-};
-
-class CheckerProvider {
-public:
-  virtual ~CheckerProvider();
-  virtual void registerCheckers(CheckerManager &checkerMgr,
-                          CheckerOptInfo *checkOpts, unsigned numCheckOpts) = 0;
-  virtual void printHelp(llvm::raw_ostream &OS) = 0;
-};
-
-} // end ento namespace
-
-} // end clang namespace
-
-#endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerRegistry.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerRegistry.h
new file mode 100644
index 000000000000..b59c14d32f91
--- /dev/null
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerRegistry.h
@@ -0,0 +1,132 @@
+//===--- CheckerRegistry.h - Maintains all available checkers ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRY_H
+#define LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRY_H
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/Basic/LLVM.h"
+#include <vector>
+
+// FIXME: move this information to an HTML file in docs/.
+// At the very least, a checker plugin is a dynamic library that exports
+// clang_analyzerAPIVersionString. This should be defined as follows:
+//
+//   extern "C"
+//   const char clang_analyzerAPIVersionString[] =
+//     CLANG_ANALYZER_API_VERSION_STRING;
+//
+// This is used to check whether the current version of the analyzer is known to
+// be incompatible with a plugin. Plugins with incompatible version strings,
+// or without a version string at all, will not be loaded.
+//
+// To add a custom checker to the analyzer, the plugin must also define the
+// function clang_registerCheckers. For example:
+//
+//    extern "C"
+//    void clang_registerCheckers (CheckerRegistry &registry) {
+//      registry.addChecker<MainCallChecker>("example.MainCallChecker",
+//        "Disallows calls to functions called main");
+//    }
+//
+// The first method argument is the full name of the checker, including its
+// enclosing package. By convention, the registered name of a checker is the
+// name of the associated class (the template argument).
+// The second method argument is a short human-readable description of the
+// checker.
+//
+// The clang_registerCheckers function may add any number of checkers to the
+// registry. If any checkers require additional initialization, use the three-
+// argument form of CheckerRegistry::addChecker.
+// 
+// To load a checker plugin, specify the full path to the dynamic library as
+// the argument to the -load option in the cc1 frontend. You can then enable
+// your custom checker using the -analyzer-checker:
+//
+//   clang -cc1 -load </path/to/plugin.dylib> -analyze
+//     -analyzer-checker=<example.MainCallChecker>
+//
+// For a complete working example, see examples/analyzer-plugin.
+
+
+namespace clang {
+namespace ento {
+
+#ifndef CLANG_ANALYZER_API_VERSION_STRING
+// FIXME: The Clang version string is not particularly granular;
+// the analyzer infrastructure can change a lot between releases.
+// Unfortunately, this string has to be statically embedded in each plugin,
+// so we can't just use the functions defined in Version.h.
+#include "clang/Basic/Version.h"
+#define CLANG_ANALYZER_API_VERSION_STRING CLANG_VERSION_STRING
+#endif
+
+class CheckerOptInfo;
+
+/// Manages a set of available checkers for running a static analysis.
+/// The checkers are organized into packages by full name, where including
+/// a package will recursively include all subpackages and checkers within it.
+/// For example, the checker "core.builtin.NoReturnFunctionChecker" will be
+/// included if initializeManager() is called with an option of "core",
+/// "core.builtin", or the full name "core.builtin.NoReturnFunctionChecker".
+class CheckerRegistry {
+public:
+  /// Initialization functions perform any necessary setup for a checker.
+  /// They should include a call to CheckerManager::registerChecker.
+  typedef void (*InitializationFunction)(CheckerManager &);
+  struct CheckerInfo {
+    InitializationFunction Initialize;
+    StringRef FullName;
+    StringRef Desc;
+
+    CheckerInfo(InitializationFunction fn, StringRef name, StringRef desc)
+    : Initialize(fn), FullName(name), Desc(desc) {}
+  };
+
+  typedef std::vector<CheckerInfo> CheckerInfoList;
+
+private:
+  template <typename T>
+  static void initializeManager(CheckerManager &mgr) {
+    mgr.registerChecker<T>();
+  }
+
+public:
+  /// Adds a checker to the registry. Use this non-templated overload when your
+  /// checker requires custom initialization.
+  void addChecker(InitializationFunction fn, StringRef fullName,
+                  StringRef desc);
+
+  /// Adds a checker to the registry. Use this templated overload when your
+  /// checker does not require any custom initialization.
+  template <class T>
+  void addChecker(StringRef fullName, StringRef desc) {
+    addChecker(&initializeManager<T>, fullName, desc);
+  }
+
+  /// Initializes a CheckerManager by calling the initialization functions for
+  /// all checkers specified by the given CheckerOptInfo list. The order of this
+  /// list is significant; later options can be used to reverse earlier ones.
+  /// This can be used to exclude certain checkers in an included package.
+  void initializeManager(CheckerManager &mgr,
+                         SmallVectorImpl<CheckerOptInfo> &opts) const;
+
+  /// Prints the name and description of all checkers in this registry.
+  /// This output is not intended to be machine-parseable.
+  void printHelp(raw_ostream &out, size_t maxNameChars = 30) const ;
+
+private:
+  mutable CheckerInfoList Checkers;
+  mutable llvm::StringMap<size_t> Packages;
+};
+
+} // end namespace ento
+} // end namespace clang
+
+#endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticClients.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h
similarity index 62%
rename from contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticClients.h
rename to contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h
index d02228fa30dd..d1f5a7da5599 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticClients.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h
@@ -22,18 +22,18 @@ class Preprocessor;
 
 namespace ento {
 
-class PathDiagnosticClient;
+class PathDiagnosticConsumer;
 
-PathDiagnosticClient*
-createHTMLDiagnosticClient(const std::string& prefix, const Preprocessor &PP);
+PathDiagnosticConsumer*
+createHTMLDiagnosticConsumer(const std::string& prefix, const Preprocessor &PP);
 
-PathDiagnosticClient*
-createPlistDiagnosticClient(const std::string& prefix, const Preprocessor &PP,
-                            PathDiagnosticClient *SubPD = 0);
+PathDiagnosticConsumer*
+createPlistDiagnosticConsumer(const std::string& prefix, const Preprocessor &PP,
+                              PathDiagnosticConsumer *SubPD = 0);
 
-PathDiagnosticClient*
-createTextPathDiagnosticClient(const std::string& prefix,
-                               const Preprocessor &PP);
+PathDiagnosticConsumer*
+createTextPathDiagnosticConsumer(const std::string& prefix,
+                                 const Preprocessor &PP);
 
 } // end GR namespace
 
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h
index 1ba038e6da2d..6c93f59d2094 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h
@@ -16,6 +16,7 @@
 #define LLVM_CLANG_GR_ANALYSISMANAGER_H
 
 #include "clang/Analysis/AnalysisContext.h"
+#include "clang/Frontend/AnalyzerOptions.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 
@@ -34,10 +35,10 @@ class AnalysisManager : public BugReporterData {
   LocationContextManager LocCtxMgr;
 
   ASTContext &Ctx;
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
   const LangOptions &LangInfo;
 
-  llvm::OwningPtr<PathDiagnosticClient> PD;
+  llvm::OwningPtr<PathDiagnosticConsumer> PD;
 
   // Configurable components creators.
   StoreManagerCreator CreateStoreMgr;
@@ -60,7 +61,7 @@ class AnalysisManager : public BugReporterData {
 
   bool VisualizeEGDot;
   bool VisualizeEGUbi;
-  bool PurgeDead;
+  AnalysisPurgeMode PurgeDead;
 
   /// EargerlyAssume - A flag indicating how the engine should handle
   //   expressions such as: 'x = (y != 0)'.  When this flag is true then
@@ -75,27 +76,24 @@ class AnalysisManager : public BugReporterData {
   bool EagerlyTrimEGraph;
 
 public:
-  AnalysisManager(ASTContext &ctx, Diagnostic &diags, 
-                  const LangOptions &lang, PathDiagnosticClient *pd,
+  AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags, 
+                  const LangOptions &lang, PathDiagnosticConsumer *pd,
                   StoreManagerCreator storemgr,
                   ConstraintManagerCreator constraintmgr, 
                   CheckerManager *checkerMgr,
                   idx::Indexer *idxer,
                   unsigned maxnodes, unsigned maxvisit,
-                  bool vizdot, bool vizubi, bool purge, bool eager, bool trim,
+                  bool vizdot, bool vizubi, AnalysisPurgeMode purge,
+                  bool eager, bool trim,
                   bool inlinecall, bool useUnoptimizedCFG,
                   bool addImplicitDtors, bool addInitializers,
-                  bool eagerlyTrimEGraph)
+                  bool eagerlyTrimEGraph);
+
+  /// Construct a clone of the given AnalysisManager with the given ASTContext
+  /// and DiagnosticsEngine.
+  AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
+                  AnalysisManager &ParentAM);
 
-    : AnaCtxMgr(useUnoptimizedCFG, addImplicitDtors, addInitializers),
-      Ctx(ctx), Diags(diags), LangInfo(lang), PD(pd),
-      CreateStoreMgr(storemgr), CreateConstraintMgr(constraintmgr),
-      CheckerMgr(checkerMgr), Idxer(idxer),
-      AScope(ScopeDecl), MaxNodes(maxnodes), MaxVisit(maxvisit),
-      VisualizeEGDot(vizdot), VisualizeEGUbi(vizubi), PurgeDead(purge),
-      EagerlyAssume(eager), TrimGraph(trim), InlineCall(inlinecall),
-      EagerlyTrimEGraph(eagerlyTrimEGraph) {}
-  
   ~AnalysisManager() { FlushDiagnostics(); }
   
   void ClearContexts() {
@@ -127,7 +125,7 @@ class AnalysisManager : public BugReporterData {
     return getASTContext().getSourceManager();
   }
 
-  virtual Diagnostic &getDiagnostic() {
+  virtual DiagnosticsEngine &getDiagnostic() {
     return Diags;
   }
 
@@ -135,7 +133,7 @@ class AnalysisManager : public BugReporterData {
     return LangInfo;
   }
 
-  virtual PathDiagnosticClient *getPathDiagnosticClient() {
+  virtual PathDiagnosticConsumer *getPathDiagnosticConsumer() {
     return PD.get();
   }
   
@@ -160,7 +158,7 @@ class AnalysisManager : public BugReporterData {
 
   bool shouldTrimGraph() const { return TrimGraph; }
 
-  bool shouldPurgeDead() const { return PurgeDead; }
+  AnalysisPurgeMode getPurgeMode() const { return PurgeDead; }
 
   bool shouldEagerlyAssume() const { return EagerlyAssume; }
 
@@ -174,8 +172,9 @@ class AnalysisManager : public BugReporterData {
     return AnaCtxMgr.getContext(D)->getCFG();
   }
 
-  LiveVariables *getLiveVariables(Decl const *D) {
-    return AnaCtxMgr.getContext(D)->getLiveVariables();
+  template <typename T>
+  T *getAnalysis(Decl const *D) {
+    return AnaCtxMgr.getContext(D)->getAnalysis<T>();
   }
 
   ParentMap &getParentMap(Decl const *D) {
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h
index 69495be400a3..42a15370a427 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h
@@ -27,7 +27,7 @@ namespace clang {
 
 namespace ento {
 
-  class GRState;
+class ProgramState;
 
 class CompoundValData : public llvm::FoldingSetNode {
   QualType T;
@@ -49,17 +49,17 @@ class CompoundValData : public llvm::FoldingSetNode {
 
 class LazyCompoundValData : public llvm::FoldingSetNode {
   StoreRef store;
-  const TypedRegion *region;
+  const TypedValueRegion *region;
 public:
-  LazyCompoundValData(const StoreRef &st, const TypedRegion *r)
+  LazyCompoundValData(const StoreRef &st, const TypedValueRegion *r)
     : store(st), region(r) {}
 
   const void *getStore() const { return store.getStore(); }
-  const TypedRegion *getRegion() const { return region; }
+  const TypedValueRegion *getRegion() const { return region; }
 
   static void Profile(llvm::FoldingSetNodeID& ID,
                       const StoreRef &store,
-                      const TypedRegion *region);
+                      const TypedValueRegion *region);
 
   void Profile(llvm::FoldingSetNodeID& ID) { Profile(ID, store, region); }
 };
@@ -68,25 +68,25 @@ class BasicValueFactory {
   typedef llvm::FoldingSet<llvm::FoldingSetNodeWrapper<llvm::APSInt> >
           APSIntSetTy;
 
-  ASTContext& Ctx;
+  ASTContext &Ctx;
   llvm::BumpPtrAllocator& BPAlloc;
 
   APSIntSetTy   APSIntSet;
-  void*         PersistentSVals;
-  void*         PersistentSValPairs;
+  void *        PersistentSVals;
+  void *        PersistentSValPairs;
 
   llvm::ImmutableList<SVal>::Factory SValListFactory;
   llvm::FoldingSet<CompoundValData>  CompoundValDataSet;
   llvm::FoldingSet<LazyCompoundValData> LazyCompoundValDataSet;
 
 public:
-  BasicValueFactory(ASTContext& ctx, llvm::BumpPtrAllocator& Alloc)
+  BasicValueFactory(ASTContext &ctx, llvm::BumpPtrAllocator& Alloc)
   : Ctx(ctx), BPAlloc(Alloc), PersistentSVals(0), PersistentSValPairs(0),
     SValListFactory(Alloc) {}
 
   ~BasicValueFactory();
 
-  ASTContext& getContext() const { return Ctx; }
+  ASTContext &getContext() const { return Ctx; }
 
   const llvm::APSInt& getValue(const llvm::APSInt& X);
   const llvm::APSInt& getValue(const llvm::APInt& X, bool isUnsigned);
@@ -176,7 +176,7 @@ class BasicValueFactory {
                                             llvm::ImmutableList<SVal> Vals);
 
   const LazyCompoundValData *getLazyCompoundValData(const StoreRef &store,
-                                                    const TypedRegion *region);
+                                            const TypedValueRegion *region);
 
   llvm::ImmutableList<SVal> getEmptySValList() {
     return SValListFactory.getEmptyList();
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BlockCounter.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BlockCounter.h
index 7d0fdfb5f11f..2483a79455b2 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BlockCounter.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BlockCounter.h
@@ -26,10 +26,13 @@ class StackFrameContext;
 
 namespace ento {
 
+/// \class BlockCounter
+/// \brief An abstract data type used to count the number of times a given
+/// block has been visited along a path analyzed by CoreEngine.
 class BlockCounter {
-  void* Data;
+  void *Data;
 
-  BlockCounter(void* D) : Data(D) {}
+  BlockCounter(void *D) : Data(D) {}
 
 public:
   BlockCounter() : Data(0) {}
@@ -38,7 +41,7 @@ class BlockCounter {
                          unsigned BlockID) const;
 
   class Factory {
-    void* F;
+    void *F;
   public:
     Factory(llvm::BumpPtrAllocator& Alloc);
     ~Factory();
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
index 4429c6b2a7ad..1f1478713260 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
@@ -28,26 +28,29 @@ class CheckerContext {
   ExprEngine &Eng;
   ExplodedNode *Pred;
   SaveAndRestore<bool> OldSink;
-  const void *checkerTag;
-  SaveAndRestore<ProgramPoint::Kind> OldPointKind;
   SaveOr OldHasGen;
-  const GRState *ST;
-  const Stmt *statement;
+  const ProgramPoint Location;
+  const ProgramState *ST;
   const unsigned size;
 public:
   bool *respondsToCallback;
 public:
-  CheckerContext(ExplodedNodeSet &dst, StmtNodeBuilder &builder,
-                 ExprEngine &eng, ExplodedNode *pred,
-                 const void *tag, ProgramPoint::Kind K,
+  CheckerContext(ExplodedNodeSet &dst,
+                 StmtNodeBuilder &builder,
+                 ExprEngine &eng,
+                 ExplodedNode *pred,
+                 const ProgramPoint &loc,
                  bool *respondsToCB = 0,
-                 const Stmt *stmt = 0, const GRState *st = 0)
-    : Dst(dst), B(builder), Eng(eng), Pred(pred),
+                 const ProgramState *st = 0)
+    : Dst(dst),
+      B(builder),
+      Eng(eng),
+      Pred(pred),
       OldSink(B.BuildSinks),
-      checkerTag(tag),
-      OldPointKind(B.PointKind, K),
       OldHasGen(B.hasGeneratedNode),
-      ST(st), statement(stmt), size(Dst.size()),
+      Location(loc),
+      ST(st),
+      size(Dst.size()),
       respondsToCallback(respondsToCB) {}
 
   ~CheckerContext();
@@ -69,10 +72,12 @@ class CheckerContext {
   }
 
   ExplodedNodeSet &getNodeSet() { return Dst; }
-  StmtNodeBuilder &getNodeBuilder() { return B; }
   ExplodedNode *&getPredecessor() { return Pred; }
-  const GRState *getState() { return ST ? ST : B.GetState(Pred); }
-  const Stmt *getStmt() const { return statement; }
+  const ProgramState *getState() { return ST ? ST : Pred->getState(); }
+
+  /// \brief Returns the number of times the current block has been visited
+  /// along the analyzed path.
+  unsigned getCurrentBlockCount() {return B.getCurrentBlockCount();}
 
   ASTContext &getASTContext() {
     return Eng.getContext();
@@ -90,64 +95,58 @@ class CheckerContext {
     return Eng.getSValBuilder();
   }
 
+  SymbolManager &getSymbolManager() {
+    return getSValBuilder().getSymbolManager();
+  }
+
+  bool isObjCGCEnabled() {
+    return Eng.isObjCGCEnabled();
+  }
+
+  /// \brief Generate a default checker node (containing checker tag but no
+  /// checker state changes).
   ExplodedNode *generateNode(bool autoTransition = true) {
-    assert(statement && "Only transitions with statements currently supported");
-    ExplodedNode *N = generateNodeImpl(statement, getState(), false,
-                                       checkerTag);
-    if (N && autoTransition)
-      Dst.Add(N);
-    return N;
+    return generateNode(getState(), autoTransition);
   }
   
-  ExplodedNode *generateNode(const Stmt *stmt, const GRState *state,
-                             bool autoTransition = true, const void *tag = 0) {
-    assert(state);
-    ExplodedNode *N = generateNodeImpl(stmt, state, false,
-                                       tag ? tag : checkerTag);
-    if (N && autoTransition)
-      addTransition(N);
-    return N;
-  }
-
-  ExplodedNode *generateNode(const GRState *state, ExplodedNode *pred,
+  /// \brief Generate a new checker node with the given predecessor.
+  /// Allows checkers to generate a chain of nodes.
+  ExplodedNode *generateNode(const ProgramState *state,
+                             ExplodedNode *pred,
+                             const ProgramPointTag *tag = 0,
                              bool autoTransition = true) {
-   assert(statement && "Only transitions with statements currently supported");
-    ExplodedNode *N = generateNodeImpl(statement, state, pred, false);
+    ExplodedNode *N = generateNodeImpl(state, false, pred, tag);
     if (N && autoTransition)
       addTransition(N);
     return N;
   }
 
-  ExplodedNode *generateNode(const GRState *state, bool autoTransition = true,
-                             const void *tag = 0) {
-    assert(statement && "Only transitions with statements currently supported");
-    ExplodedNode *N = generateNodeImpl(statement, state, false,
-                                       tag ? tag : checkerTag);
+  /// \brief Generate a new checker node.
+  ExplodedNode *generateNode(const ProgramState *state,
+                             bool autoTransition = true,
+                             const ProgramPointTag *tag = 0) {
+    ExplodedNode *N = generateNodeImpl(state, false, 0, tag);
     if (N && autoTransition)
       addTransition(N);
     return N;
   }
 
-  ExplodedNode *generateSink(const Stmt *stmt, const GRState *state = 0) {
-    return generateNodeImpl(stmt, state ? state : getState(), true,
-                            checkerTag);
-  }
-  
-  ExplodedNode *generateSink(const GRState *state = 0) {
-    assert(statement && "Only transitions with statements currently supported");
-    return generateNodeImpl(statement, state ? state : getState(), true,
-                            checkerTag);
+  /// \brief Generate a sink node. Generating sink stops exploration of the
+  /// given path.
+  ExplodedNode *generateSink(const ProgramState *state = 0) {
+    return generateNodeImpl(state ? state : getState(), true);
   }
 
   void addTransition(ExplodedNode *node) {
     Dst.Add(node);
   }
   
-  void addTransition(const GRState *state, const void *tag = 0) {
+  void addTransition(const ProgramState *state,
+                     const ProgramPointTag *tag = 0) {
     assert(state);
     // If the 'state' is not new, we need to check if the cached state 'ST'
     // is new.
-    if (state != getState() || (ST && ST != B.GetState(Pred)))
+    if (state != getState() || (ST && ST != Pred->getState()))
       // state is new or equals to ST.
       generateNode(state, true, tag);
     else
@@ -163,17 +162,14 @@ class CheckerContext {
   }
 
 private:
-  ExplodedNode *generateNodeImpl(const Stmt* stmt, const GRState *state,
-                             bool markAsSink, const void *tag) {
-    ExplodedNode *node = B.generateNode(stmt, state, Pred, tag);
-    if (markAsSink && node)
-      node->markAsSink();
-    return node;
-  }
+  ExplodedNode *generateNodeImpl(const ProgramState *state,
+                                 bool markAsSink,
+                                 ExplodedNode *pred = 0,
+                                 const ProgramPointTag *tag = 0) {
 
-  ExplodedNode *generateNodeImpl(const Stmt* stmt, const GRState *state,
-                                 ExplodedNode *pred, bool markAsSink) {
-   ExplodedNode *node = B.generateNode(stmt, state, pred, checkerTag);
+    ExplodedNode *node = B.generateNode(tag ? Location.withTag(tag) : Location,
+                                        state,
+                                        pred ? pred : Pred);
     if (markAsSink && node)
       node->markAsSink();
     return node;
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h
index 199b41afefae..3f6dddead8e9 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h
@@ -14,8 +14,8 @@
 #ifndef LLVM_CLANG_GR_CONSTRAINT_MANAGER_H
 #define LLVM_CLANG_GR_CONSTRAINT_MANAGER_H
 
-// FIXME: Typedef LiveSymbolsTy/DeadSymbolsTy at a more appropriate place.
-#include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 
 namespace llvm {
 class APSInt;
@@ -25,36 +25,40 @@ namespace clang {
 
 namespace ento {
 
-class GRState;
-class GRStateManager;
+class ProgramState;
+class ProgramStateManager;
 class SubEngine;
-class SVal;
 
 class ConstraintManager {
 public:
   virtual ~ConstraintManager();
-  virtual const GRState *assume(const GRState *state, DefinedSVal Cond,
-                                bool Assumption) = 0;
+  virtual const ProgramState *assume(const ProgramState *state,
+                                     DefinedSVal Cond,
+                                     bool Assumption) = 0;
 
-  std::pair<const GRState*, const GRState*> assumeDual(const GRState *state,
-                                                       DefinedSVal Cond) {
+  std::pair<const ProgramState*, const ProgramState*>
+    assumeDual(const ProgramState *state, DefinedSVal Cond)
+  {
     return std::make_pair(assume(state, Cond, true),
                           assume(state, Cond, false));
   }
 
-  virtual const llvm::APSInt* getSymVal(const GRState *state,
+  virtual const llvm::APSInt* getSymVal(const ProgramState *state,
                                         SymbolRef sym) const = 0;
 
-  virtual bool isEqual(const GRState *state, SymbolRef sym,
+  virtual bool isEqual(const ProgramState *state,
+                       SymbolRef sym,
                        const llvm::APSInt& V) const = 0;
 
-  virtual const GRState *removeDeadBindings(const GRState *state,
-                                            SymbolReaper& SymReaper) = 0;
+  virtual const ProgramState *removeDeadBindings(const ProgramState *state,
+                                                 SymbolReaper& SymReaper) = 0;
 
-  virtual void print(const GRState *state, llvm::raw_ostream& Out,
-                     const char* nl, const char *sep) = 0;
+  virtual void print(const ProgramState *state,
+                     raw_ostream &Out,
+                     const char* nl,
+                     const char *sep) = 0;
 
-  virtual void EndPath(const GRState *state) {}
+  virtual void EndPath(const ProgramState *state) {}
 
   /// canReasonAbout - Not all ConstraintManagers can accurately reason about
   ///  all SVal values.  This method returns true if the ConstraintManager can
@@ -64,9 +68,9 @@ class ConstraintManager {
   virtual bool canReasonAbout(SVal X) const = 0;
 };
 
-ConstraintManager* CreateBasicConstraintManager(GRStateManager& statemgr,
+ConstraintManager* CreateBasicConstraintManager(ProgramStateManager& statemgr,
                                                 SubEngine &subengine);
-ConstraintManager* CreateRangeConstraintManager(GRStateManager& statemgr,
+ConstraintManager* CreateRangeConstraintManager(ProgramStateManager& statemgr,
                                                 SubEngine &subengine);
 
 } // end GR namespace
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
index 2c1d07c59b68..131d39e75528 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
@@ -19,11 +19,12 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/WorkList.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/BlockCounter.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h"
 #include "llvm/ADT/OwningPtr.h"
 
 namespace clang {
 
+class ProgramPointTag;
+  
 namespace ento {
 
 //===----------------------------------------------------------------------===//
@@ -77,16 +78,17 @@ class CoreEngine {
   /// usually because it could not reason about something.
   BlocksAborted blocksAborted;
 
-  void generateNode(const ProgramPoint& Loc, const GRState* State,
-                    ExplodedNode* Pred);
+  void generateNode(const ProgramPoint &Loc,
+                    const ProgramState *State,
+                    ExplodedNode *Pred);
 
-  void HandleBlockEdge(const BlockEdge& E, ExplodedNode* Pred);
-  void HandleBlockEntrance(const BlockEntrance& E, ExplodedNode* Pred);
-  void HandleBlockExit(const CFGBlock* B, ExplodedNode* Pred);
-  void HandlePostStmt(const CFGBlock* B, unsigned StmtIdx, ExplodedNode *Pred);
+  void HandleBlockEdge(const BlockEdge &E, ExplodedNode *Pred);
+  void HandleBlockEntrance(const BlockEntrance &E, ExplodedNode *Pred);
+  void HandleBlockExit(const CFGBlock *B, ExplodedNode *Pred);
+  void HandlePostStmt(const CFGBlock *B, unsigned StmtIdx, ExplodedNode *Pred);
 
-  void HandleBranch(const Stmt* Cond, const Stmt* Term, const CFGBlock* B,
-                    ExplodedNode* Pred);
+  void HandleBranch(const Stmt *Cond, const Stmt *Term, const CFGBlock *B,
+                    ExplodedNode *Pred);
   void HandleCallEnter(const CallEnter &L, const CFGBlock *Block,
                        unsigned Index, ExplodedNode *Pred);
   void HandleCallExit(const CallExit &L, ExplodedNode *Pred);
@@ -124,9 +126,10 @@ class CoreEngine {
   /// ExecuteWorkList - Run the worklist algorithm for a maximum number of
   ///  steps.  Returns true if there is still simulation state on the worklist.
   bool ExecuteWorkList(const LocationContext *L, unsigned Steps,
-                       const GRState *InitState);
-  void ExecuteWorkListWithInitialState(const LocationContext *L, unsigned Steps,
-                                       const GRState *InitState, 
+                       const ProgramState *InitState);
+  void ExecuteWorkListWithInitialState(const LocationContext *L,
+                                       unsigned Steps,
+                                       const ProgramState *InitState, 
                                        ExplodedNodeSet &Dst);
 
   // Functions for external checking of whether we have unfinished work
@@ -160,41 +163,36 @@ class CoreEngine {
 
 class StmtNodeBuilder {
   CoreEngine& Eng;
-  const CFGBlock& B;
+  const CFGBlock &B;
   const unsigned Idx;
-  ExplodedNode* Pred;
-  GRStateManager& Mgr;
+  ExplodedNode *Pred;
+
 
 public:
   bool PurgingDeadSymbols;
   bool BuildSinks;
   bool hasGeneratedNode;
   ProgramPoint::Kind PointKind;
-  const void *Tag;
-
-  const GRState* CleanedState;
-
+  const ProgramPointTag *Tag;
 
   typedef llvm::SmallPtrSet<ExplodedNode*,5> DeferredTy;
   DeferredTy Deferred;
 
-  void GenerateAutoTransition(ExplodedNode* N);
+  void GenerateAutoTransition(ExplodedNode *N);
 
 public:
-  StmtNodeBuilder(const CFGBlock* b, unsigned idx, ExplodedNode* N,
-                    CoreEngine* e, GRStateManager &mgr);
+  StmtNodeBuilder(const CFGBlock *b,
+                  unsigned idx,
+                  ExplodedNode *N,
+                  CoreEngine* e);
 
   ~StmtNodeBuilder();
 
-  ExplodedNode* getPredecessor() const { return Pred; }
+  ExplodedNode *getPredecessor() const { return Pred; }
 
   // FIXME: This should not be exposed.
   WorkList *getWorkList() { return Eng.WList; }
 
-  void SetCleanedState(const GRState* St) {
-    CleanedState = St;
-  }
-
   BlockCounter getBlockCounter() const { return Eng.WList->getBlockCounter();}
 
   unsigned getCurrentBlockCount() const {
@@ -203,14 +201,11 @@ class StmtNodeBuilder {
                                            B.getBlockID());
   }
 
-  ExplodedNode* generateNode(PostStmt PP,const GRState* St,ExplodedNode* Pred) {
-    hasGeneratedNode = true;
-    return generateNodeInternal(PP, St, Pred);
-  }
-
-  ExplodedNode* generateNode(const Stmt *S, const GRState *St,
-                             ExplodedNode *Pred, ProgramPoint::Kind K,
-                             const void *tag = 0) {
+  ExplodedNode *generateNode(const Stmt *S,
+                             const ProgramState *St,
+                             ExplodedNode *Pred,
+                             ProgramPoint::Kind K,
+                             const ProgramPointTag *tag = 0) {
     hasGeneratedNode = true;
 
     if (PurgingDeadSymbols)
@@ -219,59 +214,65 @@ class StmtNodeBuilder {
     return generateNodeInternal(S, St, Pred, K, tag ? tag : Tag);
   }
 
-  ExplodedNode* generateNode(const Stmt *S, const GRState *St,
-                             ExplodedNode *Pred, const void *tag = 0) {
+  ExplodedNode *generateNode(const Stmt *S,
+                             const ProgramState *St,
+                             ExplodedNode *Pred,
+                             const ProgramPointTag *tag = 0) {
     return generateNode(S, St, Pred, PointKind, tag);
   }
 
-  ExplodedNode *generateNode(const ProgramPoint &PP, const GRState* State,
-                             ExplodedNode* Pred) {
+  ExplodedNode *generateNode(const ProgramPoint &PP,
+                             const ProgramState *State,
+                             ExplodedNode *Pred) {
     hasGeneratedNode = true;
     return generateNodeInternal(PP, State, Pred);
   }
 
   ExplodedNode*
-  generateNodeInternal(const ProgramPoint &PP, const GRState* State,
-                       ExplodedNode* Pred);
+  generateNodeInternal(const ProgramPoint &PP,
+                       const ProgramState *State,
+                       ExplodedNode *Pred);
 
   ExplodedNode*
-  generateNodeInternal(const Stmt* S, const GRState* State, ExplodedNode* Pred,
-                   ProgramPoint::Kind K = ProgramPoint::PostStmtKind,
-                   const void *tag = 0);
+  generateNodeInternal(const Stmt *S,
+                       const ProgramState *State,
+                       ExplodedNode *Pred,
+                       ProgramPoint::Kind K,
+                       const ProgramPointTag *tag = 0);
 
   /// getStmt - Return the current block-level expression associated with
   ///  this builder.
-  const Stmt* getStmt() const { 
+  const Stmt *getStmt() const { 
     const CFGStmt *CS = B[Idx].getAs<CFGStmt>();
     return CS ? CS->getStmt() : 0;
   }
 
   /// getBlock - Return the CFGBlock associated with the block-level expression
   ///  of this builder.
-  const CFGBlock* getBlock() const { return &B; }
+  const CFGBlock *getBlock() const { return &B; }
 
   unsigned getIndex() const { return Idx; }
 
-  const GRState* GetState(ExplodedNode* Pred) const {
-    if (Pred == getPredecessor())
-      return CleanedState;
-    else
-      return Pred->getState();
-  }
-
-  ExplodedNode* MakeNode(ExplodedNodeSet& Dst, const Stmt* S, 
-                         ExplodedNode* Pred, const GRState* St) {
+  ExplodedNode *MakeNode(ExplodedNodeSet &Dst,
+                         const Stmt *S, 
+                         ExplodedNode *Pred,
+                         const ProgramState *St) {
     return MakeNode(Dst, S, Pred, St, PointKind);
   }
 
-  ExplodedNode* MakeNode(ExplodedNodeSet& Dst, const Stmt* S,ExplodedNode* Pred,
-                         const GRState* St, ProgramPoint::Kind K);
+  ExplodedNode *MakeNode(ExplodedNodeSet &Dst,
+                         const Stmt *S,
+                         ExplodedNode *Pred,
+                         const ProgramState *St,
+                         ProgramPoint::Kind K);
 
-  ExplodedNode* MakeSinkNode(ExplodedNodeSet& Dst, const Stmt* S,
-                       ExplodedNode* Pred, const GRState* St) {
+  ExplodedNode *MakeSinkNode(ExplodedNodeSet &Dst,
+                             const Stmt *S,
+                             ExplodedNode *Pred,
+                             const ProgramState *St) {
     bool Tmp = BuildSinks;
     BuildSinks = true;
-    ExplodedNode* N = MakeNode(Dst, S, Pred, St);
+    ExplodedNode *N = MakeNode(Dst, S, Pred, St);
     BuildSinks = Tmp;
     return N;
   }
@@ -279,12 +280,12 @@ class StmtNodeBuilder {
 
 class BranchNodeBuilder {
   CoreEngine& Eng;
-  const CFGBlock* Src;
-  const CFGBlock* DstT;
-  const CFGBlock* DstF;
-  ExplodedNode* Pred;
+  const CFGBlock *Src;
+  const CFGBlock *DstT;
+  const CFGBlock *DstF;
+  ExplodedNode *Pred;
 
-  typedef llvm::SmallVector<ExplodedNode*,3> DeferredTy;
+  typedef SmallVector<ExplodedNode*,3> DeferredTy;
   DeferredTy Deferred;
 
   bool GeneratedTrue;
@@ -293,25 +294,27 @@ class BranchNodeBuilder {
   bool InFeasibleFalse;
 
 public:
-  BranchNodeBuilder(const CFGBlock* src, const CFGBlock* dstT, 
-                      const CFGBlock* dstF, ExplodedNode* pred, CoreEngine* e)
+  BranchNodeBuilder(const CFGBlock *src, const CFGBlock *dstT, 
+                      const CFGBlock *dstF, ExplodedNode *pred, CoreEngine* e)
   : Eng(*e), Src(src), DstT(dstT), DstF(dstF), Pred(pred),
     GeneratedTrue(false), GeneratedFalse(false),
     InFeasibleTrue(!DstT), InFeasibleFalse(!DstF) {}
 
   ~BranchNodeBuilder();
 
-  ExplodedNode* getPredecessor() const { return Pred; }
+  ExplodedNode *getPredecessor() const { return Pred; }
 
   const ExplodedGraph& getGraph() const { return *Eng.G; }
 
   BlockCounter getBlockCounter() const { return Eng.WList->getBlockCounter();}
 
-  ExplodedNode* generateNode(const Stmt *Condition, const GRState* State);
+  /// This function generates a new ExplodedNode but not a new
+  /// branch(block edge).
+  ExplodedNode *generateNode(const Stmt *Condition, const ProgramState *State);
 
-  ExplodedNode* generateNode(const GRState* State, bool branch);
+  ExplodedNode *generateNode(const ProgramState *State, bool branch);
 
-  const CFGBlock* getTargetBlock(bool branch) const {
+  const CFGBlock *getTargetBlock(bool branch) const {
     return branch ? DstT : DstF;
   }
 
@@ -326,21 +329,21 @@ class BranchNodeBuilder {
     return branch ? !InFeasibleTrue : !InFeasibleFalse;
   }
 
-  const GRState* getState() const {
+  const ProgramState *getState() const {
     return getPredecessor()->getState();
   }
 };
 
 class IndirectGotoNodeBuilder {
   CoreEngine& Eng;
-  const CFGBlock* Src;
-  const CFGBlock& DispatchBlock;
-  const Expr* E;
-  ExplodedNode* Pred;
+  const CFGBlock *Src;
+  const CFGBlock &DispatchBlock;
+  const Expr *E;
+  ExplodedNode *Pred;
 
 public:
-  IndirectGotoNodeBuilder(ExplodedNode* pred, const CFGBlock* src, 
-                    const Expr* e, const CFGBlock* dispatch, CoreEngine* eng)
+  IndirectGotoNodeBuilder(ExplodedNode *pred, const CFGBlock *src, 
+                    const Expr *e, const CFGBlock *dispatch, CoreEngine* eng)
     : Eng(*eng), Src(src), DispatchBlock(*dispatch), E(e), Pred(pred) {}
 
   class iterator {
@@ -350,8 +353,8 @@ class IndirectGotoNodeBuilder {
     iterator(CFGBlock::const_succ_iterator i) : I(i) {}
   public:
 
-    iterator& operator++() { ++I; return *this; }
-    bool operator!=(const iterator& X) const { return I != X.I; }
+    iterator &operator++() { ++I; return *this; }
+    bool operator!=(const iterator &X) const { return I != X.I; }
 
     const LabelDecl *getLabel() const {
       return llvm::cast<LabelStmt>((*I)->getLabel())->getDecl();
@@ -365,23 +368,24 @@ class IndirectGotoNodeBuilder {
   iterator begin() { return iterator(DispatchBlock.succ_begin()); }
   iterator end() { return iterator(DispatchBlock.succ_end()); }
 
-  ExplodedNode* generateNode(const iterator& I, const GRState* State,
+  ExplodedNode *generateNode(const iterator &I,
+                             const ProgramState *State,
                              bool isSink = false);
 
-  const Expr* getTarget() const { return E; }
+  const Expr *getTarget() const { return E; }
 
-  const GRState* getState() const { return Pred->State; }
+  const ProgramState *getState() const { return Pred->State; }
 };
 
 class SwitchNodeBuilder {
   CoreEngine& Eng;
-  const CFGBlock* Src;
-  const Expr* Condition;
-  ExplodedNode* Pred;
+  const CFGBlock *Src;
+  const Expr *Condition;
+  ExplodedNode *Pred;
 
 public:
-  SwitchNodeBuilder(ExplodedNode* pred, const CFGBlock* src,
-                      const Expr* condition, CoreEngine* eng)
+  SwitchNodeBuilder(ExplodedNode *pred, const CFGBlock *src,
+                    const Expr *condition, CoreEngine* eng)
   : Eng(*eng), Src(src), Condition(condition), Pred(pred) {}
 
   class iterator {
@@ -391,15 +395,15 @@ class SwitchNodeBuilder {
     iterator(CFGBlock::const_succ_reverse_iterator i) : I(i) {}
 
   public:
-    iterator& operator++() { ++I; return *this; }
+    iterator &operator++() { ++I; return *this; }
     bool operator!=(const iterator &X) const { return I != X.I; }
     bool operator==(const iterator &X) const { return I == X.I; }
 
-    const CaseStmt* getCase() const {
+    const CaseStmt *getCase() const {
       return llvm::cast<CaseStmt>((*I)->getLabel());
     }
 
-    const CFGBlock* getBlock() const {
+    const CFGBlock *getBlock() const {
       return *I;
     }
   };
@@ -411,14 +415,15 @@ class SwitchNodeBuilder {
     return llvm::cast<SwitchStmt>(Src->getTerminator());
   }
 
-  ExplodedNode* generateCaseStmtNode(const iterator& I, const GRState* State);
+  ExplodedNode *generateCaseStmtNode(const iterator &I,
+                                     const ProgramState *State);
 
-  ExplodedNode* generateDefaultCaseNode(const GRState* State,
+  ExplodedNode *generateDefaultCaseNode(const ProgramState *State,
                                         bool isSink = false);
 
-  const Expr* getCondition() const { return Condition; }
+  const Expr *getCondition() const { return Condition; }
 
-  const GRState* getState() const { return Pred->State; }
+  const ProgramState *getState() const { return Pred->State; }
 };
 
 class GenericNodeBuilderImpl {
@@ -426,10 +431,12 @@ class GenericNodeBuilderImpl {
   CoreEngine &engine;
   ExplodedNode *pred;
   ProgramPoint pp;
-  llvm::SmallVector<ExplodedNode*, 2> sinksGenerated;  
+  SmallVector<ExplodedNode*, 2> sinksGenerated;  
 
-  ExplodedNode *generateNodeImpl(const GRState *state, ExplodedNode *pred,
-                                 ProgramPoint programPoint, bool asSink);
+  ExplodedNode *generateNodeImpl(const ProgramState *state,
+                                 ExplodedNode *pred,
+                                 ProgramPoint programPoint,
+                                 bool asSink);
 
   GenericNodeBuilderImpl(CoreEngine &eng, ExplodedNode *pr, ProgramPoint p)
     : engine(eng), pred(pr), pp(p), hasGeneratedNode(false) {}
@@ -439,13 +446,13 @@ class GenericNodeBuilderImpl {
   
   WorkList &getWorkList() { return *engine.WList; }
   
-  ExplodedNode* getPredecessor() const { return pred; }
+  ExplodedNode *getPredecessor() const { return pred; }
   
   BlockCounter getBlockCounter() const {
     return engine.WList->getBlockCounter();
   }
   
-  const llvm::SmallVectorImpl<ExplodedNode*> &sinks() const {
+  const SmallVectorImpl<ExplodedNode*> &sinks() const {
     return sinksGenerated;
   }
 };
@@ -456,8 +463,8 @@ class GenericNodeBuilder : public GenericNodeBuilderImpl {
   GenericNodeBuilder(CoreEngine &eng, ExplodedNode *pr, const PP_T &p)
     : GenericNodeBuilderImpl(eng, pr, p) {}
 
-  ExplodedNode *generateNode(const GRState *state, ExplodedNode *pred,
-                             const void *tag, bool asSink) {
+  ExplodedNode *generateNode(const ProgramState *state, ExplodedNode *pred,
+                             const ProgramPointTag *tag, bool asSink) {
     return generateNodeImpl(state, pred, cast<PP_T>(pp).withTag(tag),
                             asSink);
   }
@@ -467,27 +474,27 @@ class GenericNodeBuilder : public GenericNodeBuilderImpl {
 
 class EndOfFunctionNodeBuilder {
   CoreEngine &Eng;
-  const CFGBlock& B;
-  ExplodedNode* Pred;
-  void *Tag;
+  const CFGBlock &B;
+  ExplodedNode *Pred;
+  const ProgramPointTag *Tag;
 
 public:
   bool hasGeneratedNode;
 
 public:
-  EndOfFunctionNodeBuilder(const CFGBlock* b, ExplodedNode* N, CoreEngine* e,
-                           void *checkerTag = 0)
-    : Eng(*e), B(*b), Pred(N), Tag(checkerTag), hasGeneratedNode(false) {}
+  EndOfFunctionNodeBuilder(const CFGBlock *b, ExplodedNode *N, CoreEngine* e,
+                           const ProgramPointTag *tag = 0)
+    : Eng(*e), B(*b), Pred(N), Tag(tag), hasGeneratedNode(false) {}
 
   ~EndOfFunctionNodeBuilder();
 
-  EndOfFunctionNodeBuilder withCheckerTag(void *tag) {
+  EndOfFunctionNodeBuilder withCheckerTag(const ProgramPointTag *tag) {
     return EndOfFunctionNodeBuilder(&B, Pred, &Eng, tag);
   }
 
   WorkList &getWorkList() { return *Eng.WList; }
 
-  ExplodedNode* getPredecessor() const { return Pred; }
+  ExplodedNode *getPredecessor() const { return Pred; }
 
   BlockCounter getBlockCounter() const {
     return Eng.WList->getBlockCounter();
@@ -499,14 +506,15 @@ class EndOfFunctionNodeBuilder {
                                            B.getBlockID());
   }
 
-  ExplodedNode* generateNode(const GRState* State, ExplodedNode *P = 0,
-                             const void *tag = 0);
+  ExplodedNode *generateNode(const ProgramState *State,
+                             ExplodedNode *P = 0,
+                             const ProgramPointTag *tag = 0);
 
-  void GenerateCallExitNode(const GRState *state);
+  void GenerateCallExitNode(const ProgramState *state);
 
-  const CFGBlock* getBlock() const { return &B; }
+  const CFGBlock *getBlock() const { return &B; }
 
-  const GRState* getState() const {
+  const ProgramState *getState() const {
     return getPredecessor()->getState();
   }
 };
@@ -535,7 +543,7 @@ class CallEnterNodeBuilder {
                          const CFGBlock *blk, unsigned idx)
     : Eng(eng), Pred(pred), CE(s), CalleeCtx(callee), Block(blk), Index(idx) {}
 
-  const GRState *getState() const { return Pred->getState(); }
+  const ProgramState *getState() const { return Pred->getState(); }
 
   const LocationContext *getLocationContext() const { 
     return Pred->getLocationContext();
@@ -549,7 +557,7 @@ class CallEnterNodeBuilder {
 
   unsigned getIndex() const { return Index; }
 
-  void generateNode(const GRState *state);
+  void generateNode(const ProgramState *state);
 };
 
 class CallExitNodeBuilder {
@@ -562,9 +570,9 @@ class CallExitNodeBuilder {
 
   const ExplodedNode *getPredecessor() const { return Pred; }
 
-  const GRState *getState() const { return Pred->getState(); }
+  const ProgramState *getState() const { return Pred->getState(); }
 
-  void generateNode(const GRState *state);
+  void generateNode(const ProgramState *state);
 }; 
 
 } // end GR namespace
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Environment.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Environment.h
index 193056e6b030..2463e23f5f79 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Environment.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Environment.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_CLANG_GR_ENVIRONMENT_H
 #define LLVM_CLANG_GR_ENVIRONMENT_H
 
-#include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 #include "llvm/ADT/ImmutableMap.h"
 
@@ -43,7 +42,7 @@ class Environment {
   Environment(BindingsTy eb)
     : ExprBindings(eb) {}
 
-  SVal lookupExpr(const Stmt* E) const;
+  SVal lookupExpr(const Stmt *E) const;
 
 public:
   typedef BindingsTy::iterator iterator;
@@ -53,7 +52,7 @@ class Environment {
 
   /// getSVal - Fetches the current binding of the expression in the
   ///  Environment.
-  SVal getSVal(const Stmt* Ex, SValBuilder& svalBuilder,
+  SVal getSVal(const Stmt *Ex, SValBuilder& svalBuilder,
 	       bool useOnlyDirectBindings = false) const;
 
   /// Profile - Profile the contents of an Environment object for use
@@ -96,8 +95,7 @@ class EnvironmentManager {
                                   SVal V);
 
   Environment removeDeadBindings(Environment Env,
-                                 SymbolReaper &SymReaper, const GRState *ST,
-                          llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
+                                 SymbolReaper &SymReaper, const ProgramState *ST);
 };
 
 } // end GR namespace
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
index e5d6876fa6b2..fdfed3da5403 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
@@ -31,7 +31,7 @@
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/Support/Casting.h"
 #include "clang/Analysis/Support/BumpVector.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 
 namespace clang {
 
@@ -67,7 +67,7 @@ class ExplodedNode : public llvm::FoldingSetNode {
       return P & 0x1;
     }
 
-    void* getPtr() const {
+    void *getPtr() const {
       assert (!getFlag());
       return reinterpret_cast<void*>(P & ~Mask);
     }
@@ -87,7 +87,7 @@ class ExplodedNode : public llvm::FoldingSetNode {
 
     bool empty() const { return (P & ~Mask) == 0; }
 
-    void addNode(ExplodedNode* N, ExplodedGraph &G);
+    void addNode(ExplodedNode *N, ExplodedGraph &G);
 
     void replaceNode(ExplodedNode *node);
 
@@ -106,7 +106,7 @@ class ExplodedNode : public llvm::FoldingSetNode {
   const ProgramPoint Location;
 
   /// State - The state associated with this node.
-  const GRState* State;
+  const ProgramState *State;
 
   /// Preds - The predecessors of this node.
   NodeGroup Preds;
@@ -116,13 +116,13 @@ class ExplodedNode : public llvm::FoldingSetNode {
 
 public:
 
-  explicit ExplodedNode(const ProgramPoint& loc, const GRState* state)
+  explicit ExplodedNode(const ProgramPoint &loc, const ProgramState *state)
     : Location(loc), State(state) {
-    const_cast<GRState*>(State)->incrementReferenceCount();
+    const_cast<ProgramState*>(State)->incrementReferenceCount();
   }
   
   ~ExplodedNode() {
-    const_cast<GRState*>(State)->decrementReferenceCount();
+    const_cast<ProgramState*>(State)->decrementReferenceCount();
   }
 
   /// getLocation - Returns the edge associated with the given node.
@@ -138,17 +138,18 @@ class ExplodedNode : public llvm::FoldingSetNode {
 
   ParentMap &getParentMap() const {return getLocationContext()->getParentMap();}
 
-  LiveVariables &getLiveVariables() const { 
-    return *getLocationContext()->getLiveVariables(); 
+  template <typename T>
+  T &getAnalysis() const {
+    return *getLocationContext()->getAnalysis<T>();
   }
 
-  const GRState* getState() const { return State; }
+  const ProgramState *getState() const { return State; }
 
   template <typename T>
   const T* getLocationAs() const { return llvm::dyn_cast<T>(&Location); }
 
   static void Profile(llvm::FoldingSetNodeID &ID,
-                      const ProgramPoint& Loc, const GRState* state) {
+                      const ProgramPoint &Loc, const ProgramState *state) {
     ID.Add(Loc);
     ID.AddPointer(state);
   }
@@ -159,7 +160,7 @@ class ExplodedNode : public llvm::FoldingSetNode {
 
   /// addPredeccessor - Adds a predecessor to the current node, and
   ///  in tandem add this node as a successor of the other node.
-  void addPredecessor(ExplodedNode* V, ExplodedGraph &G);
+  void addPredecessor(ExplodedNode *V, ExplodedGraph &G);
 
   unsigned succ_size() const { return Succs.size(); }
   unsigned pred_size() const { return Preds.size(); }
@@ -169,11 +170,11 @@ class ExplodedNode : public llvm::FoldingSetNode {
   bool isSink() const { return Succs.getFlag(); }
   void markAsSink() { Succs.setFlag(); }
 
-  ExplodedNode* getFirstPred() {
+  ExplodedNode *getFirstPred() {
     return pred_empty() ? NULL : *(pred_begin());
   }
 
-  const ExplodedNode* getFirstPred() const {
+  const ExplodedNode *getFirstPred() const {
     return const_cast<ExplodedNode*>(this)->getFirstPred();
   }
 
@@ -210,7 +211,7 @@ class ExplodedNode : public llvm::FoldingSetNode {
   class Auditor {
   public:
     virtual ~Auditor();
-    virtual void AddEdge(ExplodedNode* Src, ExplodedNode* Dst) = 0;
+    virtual void AddEdge(ExplodedNode *Src, ExplodedNode *Dst) = 0;
   };
 
   static void SetAuditor(Auditor* A);
@@ -226,7 +227,7 @@ class InterExplodedGraphMap {
   friend class ExplodedGraph;
 
 public:
-  ExplodedNode* getMappedNode(const ExplodedNode* N) const;
+  ExplodedNode *getMappedNode(const ExplodedNode *N) const;
 
   InterExplodedGraphMap() {}
   virtual ~InterExplodedGraphMap() {}
@@ -237,8 +238,8 @@ class ExplodedGraph {
   friend class CoreEngine;
 
   // Type definitions.
-  typedef llvm::SmallVector<ExplodedNode*,2>    RootsTy;
-  typedef llvm::SmallVector<ExplodedNode*,10>   EndNodesTy;
+  typedef SmallVector<ExplodedNode*,2>    RootsTy;
+  typedef SmallVector<ExplodedNode*,10>   EndNodesTy;
 
   /// Roots - The roots of the simulation graph. Usually there will be only
   /// one, but clients are free to establish multiple subgraphs within a single
@@ -275,7 +276,7 @@ class ExplodedGraph {
   ///  this pair exists, it is created.  IsNew is set to true if
   ///  the node was freshly created.
 
-  ExplodedNode* getNode(const ProgramPoint& L, const GRState *State,
+  ExplodedNode *getNode(const ProgramPoint &L, const ProgramState *State,
                         bool* IsNew = 0);
 
   ExplodedGraph* MakeEmptyGraph() const {
@@ -283,13 +284,13 @@ class ExplodedGraph {
   }
 
   /// addRoot - Add an untyped node to the set of roots.
-  ExplodedNode* addRoot(ExplodedNode* V) {
+  ExplodedNode *addRoot(ExplodedNode *V) {
     Roots.push_back(V);
     return V;
   }
 
   /// addEndOfPath - Add an untyped node to the set of EOP nodes.
-  ExplodedNode* addEndOfPath(ExplodedNode* V) {
+  ExplodedNode *addEndOfPath(ExplodedNode *V) {
     EndNodes.push_back(V);
     return V;
   }
@@ -368,18 +369,18 @@ class ExplodedNodeSet {
   ImplTy Impl;
 
 public:
-  ExplodedNodeSet(ExplodedNode* N) {
+  ExplodedNodeSet(ExplodedNode *N) {
     assert (N && !static_cast<ExplodedNode*>(N)->isSink());
     Impl.insert(N);
   }
 
   ExplodedNodeSet() {}
 
-  inline void Add(ExplodedNode* N) {
+  inline void Add(ExplodedNode *N) {
     if (N && !static_cast<ExplodedNode*>(N)->isSink()) Impl.insert(N);
   }
 
-  ExplodedNodeSet& operator=(const ExplodedNodeSet &X) {
+  ExplodedNodeSet &operator=(const ExplodedNodeSet &X) {
     Impl = X.Impl;
     return *this;
   }
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index d24036c0ec28..9bc470fdde60 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -19,9 +19,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/ExprObjC.h"
@@ -35,6 +33,8 @@ class ObjCForCollectionStmt;
 namespace ento {
 
 class AnalysisManager;
+class CallOrObjCMessage;
+class ObjCMessage;
 
 class ExprEngine : public SubEngine {
   AnalysisManager &AMgr;
@@ -49,7 +49,7 @@ class ExprEngine : public SubEngine {
   StmtNodeBuilder* Builder;
 
   /// StateMgr - Object that manages the data for all created states.
-  GRStateManager StateMgr;
+  ProgramStateManager StateMgr;
 
   /// SymMgr - Object that manages the symbol information.
   SymbolManager& SymMgr;
@@ -58,31 +58,32 @@ class ExprEngine : public SubEngine {
   SValBuilder &svalBuilder;
 
   /// EntryNode - The immediate predecessor node.
-  ExplodedNode* EntryNode;
+  ExplodedNode *EntryNode;
 
   /// CleanedState - The state for EntryNode "cleaned" of all dead
   ///  variables and symbols (as determined by a liveness analysis).
-  const GRState* CleanedState;
+  const ProgramState *CleanedState;
 
   /// currentStmt - The current block-level statement.
-  const Stmt* currentStmt;
+  const Stmt *currentStmt;
 
-  // Obj-C Class Identifiers.
+  /// Obj-C Class Identifiers.
   IdentifierInfo* NSExceptionII;
 
-  // Obj-C Selectors.
+  /// Obj-C Selectors.
   Selector* NSExceptionInstanceRaiseSelectors;
   Selector RaiseSel;
+  
+  /// Whether or not GC is enabled in this analysis.
+  bool ObjCGCEnabled;
 
   /// The BugReporter associated with this engine.  It is important that
   ///  this object be placed at the very end of member variables so that its
   ///  destructor is called before the rest of the ExprEngine is destroyed.
   GRBugReporter BR;
-  
-  llvm::OwningPtr<TransferFuncs> TF;
 
 public:
-  ExprEngine(AnalysisManager &mgr, TransferFuncs *tf);
+  ExprEngine(AnalysisManager &mgr, bool gcEnabled);
 
   ~ExprEngine();
 
@@ -94,13 +95,13 @@ class ExprEngine : public SubEngine {
   /// of the function are added into the Dst set, which represent the exit
   /// state of the function call.
   void ExecuteWorkListWithInitialState(const LocationContext *L, unsigned Steps,
-                                       const GRState *InitState, 
+                                       const ProgramState *InitState, 
                                        ExplodedNodeSet &Dst) {
     Engine.ExecuteWorkListWithInitialState(L, Steps, InitState, Dst);
   }
 
   /// getContext - Return the ASTContext associated with this analysis.
-  ASTContext& getContext() const { return AMgr.getASTContext(); }
+  ASTContext &getContext() const { return AMgr.getASTContext(); }
 
   virtual AnalysisManager &getAnalysisManager() { return AMgr; }
 
@@ -110,14 +111,11 @@ class ExprEngine : public SubEngine {
 
   SValBuilder &getSValBuilder() { return svalBuilder; }
 
-  TransferFuncs& getTF() { return *TF; }
-
   BugReporter& getBugReporter() { return BR; }
 
   StmtNodeBuilder &getBuilder() { assert(Builder); return *Builder; }
 
-  // FIXME: Remove once TransferFuncs is no longer referenced.
-  void setTransferFunction(TransferFuncs* tf);
+  bool isObjCGCEnabled() { return ObjCGCEnabled; }
 
   /// ViewGraph - Visualize the ExplodedGraph created by executing the
   ///  simulation.
@@ -127,7 +125,7 @@ class ExprEngine : public SubEngine {
 
   /// getInitialState - Return the initial state used for the root vertex
   ///  in the ExplodedGraph.
-  const GRState* getInitialState(const LocationContext *InitLoc);
+  const ProgramState *getInitialState(const LocationContext *InitLoc);
 
   ExplodedGraph& getGraph() { return G; }
   const ExplodedGraph& getGraph() const { return G; }
@@ -155,7 +153,7 @@ class ExprEngine : public SubEngine {
   
   /// ProcessBranch - Called by CoreEngine.  Used to generate successor
   ///  nodes by processing the 'effects' of a branch condition.
-  void processBranch(const Stmt* Condition, const Stmt* Term, 
+  void processBranch(const Stmt *Condition, const Stmt *Term, 
                      BranchNodeBuilder& builder);
 
   /// processIndirectGoto - Called by CoreEngine.  Used to generate successor
@@ -181,21 +179,25 @@ class ExprEngine : public SubEngine {
 
   /// evalAssume - Callback function invoked by the ConstraintManager when
   ///  making assumptions about state values.
-  const GRState *processAssume(const GRState *state, SVal cond,bool assumption);
+  const ProgramState *processAssume(const ProgramState *state, SVal cond,bool assumption);
 
-  /// wantsRegionChangeUpdate - Called by GRStateManager to determine if a
+  /// wantsRegionChangeUpdate - Called by ProgramStateManager to determine if a
   ///  region change should trigger a processRegionChanges update.
-  bool wantsRegionChangeUpdate(const GRState* state);
+  bool wantsRegionChangeUpdate(const ProgramState *state);
 
-  /// processRegionChanges - Called by GRStateManager whenever a change is made
+  /// processRegionChanges - Called by ProgramStateManager whenever a change is made
   ///  to the store. Used to update checkers that track region values.
-  const GRState *
-  processRegionChanges(const GRState *state,
+  const ProgramState *
+  processRegionChanges(const ProgramState *state,
                        const StoreManager::InvalidatedSymbols *invalidated,
-                       const MemRegion * const *Begin,
-                       const MemRegion * const *End);
+                       ArrayRef<const MemRegion *> ExplicitRegions,
+                       ArrayRef<const MemRegion *> Regions);
 
-  virtual GRStateManager& getStateManager() { return StateMgr; }
+  /// printState - Called by ProgramStateManager to print checker-specific data.
+  void printState(raw_ostream &Out, const ProgramState *State,
+                  const char *NL, const char *Sep);
+
+  virtual ProgramStateManager& getStateManager() { return StateMgr; }
 
   StoreManager& getStoreManager() { return StateMgr.getStoreManager(); }
 
@@ -222,124 +224,109 @@ class ExprEngine : public SubEngine {
 
   const CoreEngine &getCoreEngine() const { return Engine; }
 
-protected:
-  const GRState* GetState(ExplodedNode* N) {
-    return N == EntryNode ? CleanedState : N->getState();
-  }
-
 public:
-  ExplodedNode* MakeNode(ExplodedNodeSet& Dst, const Stmt* S, 
-                         ExplodedNode* Pred, const GRState* St,
+  ExplodedNode *MakeNode(ExplodedNodeSet &Dst, const Stmt *S, 
+                         ExplodedNode *Pred, const ProgramState *St,
                          ProgramPoint::Kind K = ProgramPoint::PostStmtKind,
-                         const void *tag = 0);
+                         const ProgramPointTag *tag = 0);
 
   /// Visit - Transfer function logic for all statements.  Dispatches to
   ///  other functions that handle specific kinds of statements.
-  void Visit(const Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void Visit(const Stmt *S, ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
   /// VisitArraySubscriptExpr - Transfer function for array accesses.
-  void VisitLvalArraySubscriptExpr(const ArraySubscriptExpr* Ex,
-                                   ExplodedNode* Pred,
-                                   ExplodedNodeSet& Dst);
+  void VisitLvalArraySubscriptExpr(const ArraySubscriptExpr *Ex,
+                                   ExplodedNode *Pred,
+                                   ExplodedNodeSet &Dst);
 
   /// VisitAsmStmt - Transfer function logic for inline asm.
-  void VisitAsmStmt(const AsmStmt* A, ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void VisitAsmStmt(const AsmStmt *A, ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitAsmStmtHelperOutputs(const AsmStmt* A,
+  void VisitAsmStmtHelperOutputs(const AsmStmt *A,
                                  AsmStmt::const_outputs_iterator I,
                                  AsmStmt::const_outputs_iterator E,
-                                 ExplodedNode* Pred, ExplodedNodeSet& Dst);
+                                 ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitAsmStmtHelperInputs(const AsmStmt* A,
+  void VisitAsmStmtHelperInputs(const AsmStmt *A,
                                 AsmStmt::const_inputs_iterator I,
                                 AsmStmt::const_inputs_iterator E,
-                                ExplodedNode* Pred, ExplodedNodeSet& Dst);
+                                ExplodedNode *Pred, ExplodedNodeSet &Dst);
   
   /// VisitBlockExpr - Transfer function logic for BlockExprs.
   void VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred, 
                       ExplodedNodeSet &Dst);
 
   /// VisitBinaryOperator - Transfer function logic for binary operators.
-  void VisitBinaryOperator(const BinaryOperator* B, ExplodedNode* Pred, 
-                           ExplodedNodeSet& Dst);
+  void VisitBinaryOperator(const BinaryOperator* B, ExplodedNode *Pred, 
+                           ExplodedNodeSet &Dst);
 
 
   /// VisitCall - Transfer function for function calls.
-  void VisitCallExpr(const CallExpr* CE, ExplodedNode* Pred,
-                     ExplodedNodeSet& Dst);
+  void VisitCallExpr(const CallExpr *CE, ExplodedNode *Pred,
+                     ExplodedNodeSet &Dst);
 
   /// VisitCast - Transfer function logic for all casts (implicit and explicit).
   void VisitCast(const CastExpr *CastE, const Expr *Ex, ExplodedNode *Pred,
                 ExplodedNodeSet &Dst);
 
   /// VisitCompoundLiteralExpr - Transfer function logic for compound literals.
-  void VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL, 
-                                ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void VisitCompoundLiteralExpr(const CompoundLiteralExpr *CL, 
+                                ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
   /// Transfer function logic for DeclRefExprs and BlockDeclRefExprs.
-  void VisitCommonDeclRefExpr(const Expr* DR, const NamedDecl *D,
-                              ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void VisitCommonDeclRefExpr(const Expr *DR, const NamedDecl *D,
+                              ExplodedNode *Pred, ExplodedNodeSet &Dst);
   
   /// VisitDeclStmt - Transfer function logic for DeclStmts.
-  void VisitDeclStmt(const DeclStmt* DS, ExplodedNode* Pred, 
-                     ExplodedNodeSet& Dst);
+  void VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred, 
+                     ExplodedNodeSet &Dst);
 
   /// VisitGuardedExpr - Transfer function logic for ?, __builtin_choose
-  void VisitGuardedExpr(const Expr* Ex, const Expr* L, const Expr* R, 
-                        ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void VisitGuardedExpr(const Expr *Ex, const Expr *L, const Expr *R, 
+                        ExplodedNode *Pred, ExplodedNodeSet &Dst);
   
-  void VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
-                         ExplodedNodeSet& Dst);
+  void VisitInitListExpr(const InitListExpr *E, ExplodedNode *Pred,
+                         ExplodedNodeSet &Dst);
 
   /// VisitLogicalExpr - Transfer function logic for '&&', '||'
-  void VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
-                        ExplodedNodeSet& Dst);
+  void VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
+                        ExplodedNodeSet &Dst);
 
   /// VisitMemberExpr - Transfer function for member expressions.
-  void VisitMemberExpr(const MemberExpr* M, ExplodedNode* Pred, 
-                           ExplodedNodeSet& Dst);
+  void VisitMemberExpr(const MemberExpr *M, ExplodedNode *Pred, 
+                           ExplodedNodeSet &Dst);
 
   /// Transfer function logic for ObjCAtSynchronizedStmts.
   void VisitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt *S,
                                    ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitObjCPropertyRefExpr(const ObjCPropertyRefExpr *E,
-                                ExplodedNode *Pred, ExplodedNodeSet &Dst);
-
   /// Transfer function logic for computing the lvalue of an Objective-C ivar.
-  void VisitLvalObjCIvarRefExpr(const ObjCIvarRefExpr* DR, ExplodedNode* Pred,
-                                ExplodedNodeSet& Dst);
+  void VisitLvalObjCIvarRefExpr(const ObjCIvarRefExpr *DR, ExplodedNode *Pred,
+                                ExplodedNodeSet &Dst);
 
   /// VisitObjCForCollectionStmt - Transfer function logic for
   ///  ObjCForCollectionStmt.
-  void VisitObjCForCollectionStmt(const ObjCForCollectionStmt* S, 
-                                  ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S, 
+                                  ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
-  void VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S, 
-                                     ExplodedNode* Pred,
-                                     ExplodedNodeSet& Dst, SVal ElementV);
-
-  /// VisitObjCMessageExpr - Transfer function for ObjC message expressions.
-  void VisitObjCMessageExpr(const ObjCMessageExpr* ME, ExplodedNode* Pred, 
-                            ExplodedNodeSet& Dst);
-  void VisitObjCMessage(const ObjCMessage &msg, ExplodedNodeSet &Src,
-                        ExplodedNodeSet& Dst);
+  void VisitObjCMessage(const ObjCMessage &msg, ExplodedNode *Pred,
+                        ExplodedNodeSet &Dst);
 
   /// VisitReturnStmt - Transfer function logic for return statements.
-  void VisitReturnStmt(const ReturnStmt* R, ExplodedNode* Pred, 
-                       ExplodedNodeSet& Dst);
+  void VisitReturnStmt(const ReturnStmt *R, ExplodedNode *Pred, 
+                       ExplodedNodeSet &Dst);
   
   /// VisitOffsetOfExpr - Transfer function for offsetof.
-  void VisitOffsetOfExpr(const OffsetOfExpr* Ex, ExplodedNode* Pred,
-                         ExplodedNodeSet& Dst);
+  void VisitOffsetOfExpr(const OffsetOfExpr *Ex, ExplodedNode *Pred,
+                         ExplodedNodeSet &Dst);
 
   /// VisitUnaryExprOrTypeTraitExpr - Transfer function for sizeof.
-  void VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr* Ex,
-                              ExplodedNode* Pred, ExplodedNodeSet& Dst);
+  void VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *Ex,
+                              ExplodedNode *Pred, ExplodedNodeSet &Dst);
 
   /// VisitUnaryOperator - Transfer function logic for unary operators.
-  void VisitUnaryOperator(const UnaryOperator* B, ExplodedNode* Pred, 
-                          ExplodedNodeSet& Dst);
+  void VisitUnaryOperator(const UnaryOperator* B, ExplodedNode *Pred, 
+                          ExplodedNodeSet &Dst);
 
   void VisitCXXThisExpr(const CXXThisExpr *TE, ExplodedNode *Pred, 
                         ExplodedNodeSet & Dst);
@@ -366,7 +353,8 @@ class ExprEngine : public SubEngine {
                     ExplodedNodeSet &Dst);
 
   /// Create a C++ temporary object for an rvalue.
-  void CreateCXXTemporaryObject(const Expr *Ex, ExplodedNode *Pred, 
+  void CreateCXXTemporaryObject(const MaterializeTemporaryExpr *ME,
+                                ExplodedNode *Pred, 
                                 ExplodedNodeSet &Dst);
 
   /// Synthesize CXXThisRegion.
@@ -389,8 +377,11 @@ class ExprEngine : public SubEngine {
   /// evalEagerlyAssume - Given the nodes in 'Src', eagerly assume symbolic
   ///  expressions of the form 'x != 0' and generate new nodes (stored in Dst)
   ///  with those assumptions.
-  void evalEagerlyAssume(ExplodedNodeSet& Dst, ExplodedNodeSet& Src, 
+  void evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src, 
                          const Expr *Ex);
+  
+  std::pair<const ProgramPointTag *, const ProgramPointTag*>
+    getEagerlyAssumeTags();
 
   SVal evalMinus(SVal X) {
     return X.isValid() ? svalBuilder.evalMinus(cast<NonLoc>(X)) : X;
@@ -402,36 +393,36 @@ class ExprEngine : public SubEngine {
 
 public:
 
-  SVal evalBinOp(const GRState *state, BinaryOperator::Opcode op,
+  SVal evalBinOp(const ProgramState *state, BinaryOperator::Opcode op,
                  NonLoc L, NonLoc R, QualType T) {
     return svalBuilder.evalBinOpNN(state, op, L, R, T);
   }
 
-  SVal evalBinOp(const GRState *state, BinaryOperator::Opcode op,
+  SVal evalBinOp(const ProgramState *state, BinaryOperator::Opcode op,
                  NonLoc L, SVal R, QualType T) {
     return R.isValid() ? svalBuilder.evalBinOpNN(state,op,L, cast<NonLoc>(R), T) : R;
   }
 
-  SVal evalBinOp(const GRState *ST, BinaryOperator::Opcode Op,
+  SVal evalBinOp(const ProgramState *ST, BinaryOperator::Opcode Op,
                  SVal LHS, SVal RHS, QualType T) {
     return svalBuilder.evalBinOp(ST, Op, LHS, RHS, T);
   }
   
 protected:
-  void evalObjCMessage(ExplodedNodeSet& Dst, const ObjCMessage &msg, 
-                       ExplodedNode* Pred, const GRState *state) {
-    assert (Builder && "StmtNodeBuilder must be defined.");
-    getTF().evalObjCMessage(Dst, *this, *Builder, msg, Pred, state);
-  }
+  void evalObjCMessage(ExplodedNodeSet &Dst, const ObjCMessage &msg, 
+                       ExplodedNode *Pred, const ProgramState *state);
 
-  const GRState* MarkBranch(const GRState* St, const Stmt* Terminator,
+  const ProgramState *invalidateArguments(const ProgramState *State,
+                                          const CallOrObjCMessage &Call,
+                                          const LocationContext *LC);
+
+  const ProgramState *MarkBranch(const ProgramState *St, const Stmt *Terminator,
                             bool branchTaken);
 
   /// evalBind - Handle the semantics of binding a value to a specific location.
   ///  This method is used by evalStore, VisitDeclStmt, and others.
-  void evalBind(ExplodedNodeSet& Dst, const Stmt* StoreE, ExplodedNode* Pred,
-                const GRState* St, SVal location, SVal Val,
-                bool atDeclInit = false);
+  void evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE, ExplodedNode *Pred,
+                SVal location, SVal Val, bool atDeclInit = false);
 
 public:
   // FIXME: 'tag' should be removed, and a LocationContext should be used
@@ -440,25 +431,25 @@ class ExprEngine : public SubEngine {
   // be the same as Pred->state, and when 'location' may not be the
   // same as state->getLValue(Ex).
   /// Simulate a read of the result of Ex.
-  void evalLoad(ExplodedNodeSet& Dst, const Expr* Ex, ExplodedNode* Pred,
-                const GRState* St, SVal location, const void *tag = 0,
+  void evalLoad(ExplodedNodeSet &Dst, const Expr *Ex, ExplodedNode *Pred,
+                const ProgramState *St, SVal location, const ProgramPointTag *tag = 0,
                 QualType LoadTy = QualType());
 
   // FIXME: 'tag' should be removed, and a LocationContext should be used
   // instead.
-  void evalStore(ExplodedNodeSet& Dst, const Expr* AssignE, const Expr* StoreE,
-                 ExplodedNode* Pred, const GRState* St, SVal TargetLV, SVal Val,
-                 const void *tag = 0);
+  void evalStore(ExplodedNodeSet &Dst, const Expr *AssignE, const Expr *StoreE,
+                 ExplodedNode *Pred, const ProgramState *St, SVal TargetLV, SVal Val,
+                 const ProgramPointTag *tag = 0);
 private:
-  void evalLoadCommon(ExplodedNodeSet& Dst, const Expr* Ex, ExplodedNode* Pred,
-                      const GRState* St, SVal location, const void *tag,
+  void evalLoadCommon(ExplodedNodeSet &Dst, const Expr *Ex, ExplodedNode *Pred,
+                      const ProgramState *St, SVal location, const ProgramPointTag *tag,
                       QualType LoadTy);
 
   // FIXME: 'tag' should be removed, and a LocationContext should be used
   // instead.
-  void evalLocation(ExplodedNodeSet &Dst, const Stmt *S, ExplodedNode* Pred,
-                    const GRState* St, SVal location,
-                    const void *tag, bool isLoad);
+  void evalLocation(ExplodedNodeSet &Dst, const Stmt *S, ExplodedNode *Pred,
+                    const ProgramState *St, SVal location,
+                    const ProgramPointTag *tag, bool isLoad);
 
   bool InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE, ExplodedNode *Pred);
   
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h
index 18e39d999bb4..89b47dc6ec26 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h
@@ -25,9 +25,9 @@ class StmtNodeBuilderRef {
   ExplodedNodeSet &Dst;
   StmtNodeBuilder &B;
   ExprEngine& Eng;
-  ExplodedNode* Pred;
-  const GRState* state;
-  const Stmt* stmt;
+  ExplodedNode *Pred;
+  const ProgramState *state;
+  const Stmt *stmt;
   const unsigned OldSize;
   const bool AutoCreateNode;
   SaveAndRestore<bool> OldSink;
@@ -42,9 +42,9 @@ class StmtNodeBuilderRef {
   StmtNodeBuilderRef(ExplodedNodeSet &dst,
                        StmtNodeBuilder &builder,
                        ExprEngine& eng,
-                       ExplodedNode* pred,
-                       const GRState *st,
-                       const Stmt* s, bool auto_create_node)
+                       ExplodedNode *pred,
+                       const ProgramState *st,
+                       const Stmt *s, bool auto_create_node)
   : Dst(dst), B(builder), Eng(eng), Pred(pred),
     state(st), stmt(s), OldSize(Dst.size()), AutoCreateNode(auto_create_node),
     OldSink(B.BuildSinks), OldHasGen(B.hasGeneratedNode) {}
@@ -62,13 +62,13 @@ class StmtNodeBuilderRef {
     }
   }
 
-  const GRState *getState() { return state; }
+  const ProgramState *getState() { return state; }
 
-  GRStateManager& getStateManager() {
+  ProgramStateManager& getStateManager() {
     return Eng.getStateManager();
   }
 
-  ExplodedNode* MakeNode(const GRState* state) {
+  ExplodedNode *MakeNode(const ProgramState *state) {
     return B.MakeNode(Dst, const_cast<Stmt*>(stmt), Pred, state);
   }
 };
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h
index db7a930b556e..c9941fe90c24 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h
@@ -19,15 +19,14 @@
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/ADT/FoldingSet.h"
 #include <string>
 
 namespace llvm {
 class BumpPtrAllocator;
-class raw_ostream;
 }
 
 namespace clang {
@@ -83,12 +82,13 @@ class MemRegion : public llvm::FoldingSetNode {
     // Untyped regions.
     SymbolicRegionKind,
     AllocaRegionKind,
+    BlockDataRegionKind,
     // Typed regions.
     BEG_TYPED_REGIONS,
     FunctionTextRegionKind = BEG_TYPED_REGIONS,
     BlockTextRegionKind,
-    BlockDataRegionKind,
-    CompoundLiteralRegionKind,
+    BEG_TYPED_VALUE_REGIONS,
+    CompoundLiteralRegionKind = BEG_TYPED_VALUE_REGIONS,
     CXXThisRegionKind,
     StringRegionKind,
     ElementRegionKind,
@@ -100,6 +100,7 @@ class MemRegion : public llvm::FoldingSetNode {
     END_DECL_REGIONS = ObjCIvarRegionKind,
     CXXTempObjectRegionKind,
     CXXBaseObjectRegionKind,
+    END_TYPED_VALUE_REGIONS = CXXBaseObjectRegionKind,
     END_TYPED_REGIONS = CXXBaseObjectRegionKind
   };
     
@@ -136,7 +137,7 @@ class MemRegion : public llvm::FoldingSetNode {
   /// Compute the offset within the top level memory object.
   RegionOffset getAsOffset() const;
 
-  virtual void dumpToStream(llvm::raw_ostream& os) const;
+  virtual void dumpToStream(raw_ostream &os) const;
 
   void dump() const;
 
@@ -197,7 +198,7 @@ class StaticGlobalSpaceRegion : public GlobalsSpaceRegion {
 public:
   void Profile(llvm::FoldingSetNodeID &ID) const;
   
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   const CodeTextRegion *getCodeRegion() const { return CR; }
 
@@ -214,7 +215,7 @@ class NonStaticGlobalSpaceRegion : public GlobalsSpaceRegion {
   
 public:
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static bool classof(const MemRegion *R) {
     return R->getKind() == NonStaticGlobalSpaceRegionKind;
@@ -323,14 +324,14 @@ class AllocaRegion : public SubRegion {
 protected:
   unsigned Cnt; // Block counter.  Used to distinguish different pieces of
                 // memory allocated by alloca at the same call site.
-  const Expr* Ex;
+  const Expr *Ex;
 
-  AllocaRegion(const Expr* ex, unsigned cnt, const MemRegion *superRegion)
+  AllocaRegion(const Expr *ex, unsigned cnt, const MemRegion *superRegion)
     : SubRegion(superRegion, AllocaRegionKind), Cnt(cnt), Ex(ex) {}
 
 public:
 
-  const Expr* getExpr() const { return Ex; }
+  const Expr *getExpr() const { return Ex; }
 
   bool isBoundable() const { return true; }
 
@@ -338,10 +339,10 @@ class AllocaRegion : public SubRegion {
 
   void Profile(llvm::FoldingSetNodeID& ID) const;
 
-  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const Expr* Ex,
+  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const Expr *Ex,
                             unsigned Cnt, const MemRegion *superRegion);
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static bool classof(const MemRegion* R) {
     return R->getKind() == AllocaRegionKind;
@@ -353,6 +354,26 @@ class TypedRegion : public SubRegion {
 protected:
   TypedRegion(const MemRegion* sReg, Kind k) : SubRegion(sReg, k) {}
 
+public:
+  virtual QualType getLocationType() const = 0;
+
+  QualType getDesugaredLocationType(ASTContext &Context) const {
+    return getLocationType().getDesugaredType(Context);
+  }
+
+  bool isBoundable() const { return true; }
+
+  static bool classof(const MemRegion* R) {
+    unsigned k = R->getKind();
+    return k >= BEG_TYPED_REGIONS && k <= END_TYPED_REGIONS;
+  }
+};
+
+/// TypedValueRegion - An abstract class representing regions having a typed value.
+class TypedValueRegion : public TypedRegion {
+protected:
+  TypedValueRegion(const MemRegion* sReg, Kind k) : TypedRegion(sReg, k) {}
+
 public:
   virtual QualType getValueType() const = 0;
 
@@ -370,15 +391,9 @@ class TypedRegion : public SubRegion {
     return T.getTypePtrOrNull() ? T.getDesugaredType(Context) : T;
   }
 
-  QualType getDesugaredLocationType(ASTContext &Context) const {
-    return getLocationType().getDesugaredType(Context);
-  }
-
-  bool isBoundable() const { return true; }
-
   static bool classof(const MemRegion* R) {
     unsigned k = R->getKind();
-    return k >= BEG_TYPED_REGIONS && k <= END_TYPED_REGIONS;
+    return k >= BEG_TYPED_VALUE_REGIONS && k <= END_TYPED_VALUE_REGIONS;
   }
 };
 
@@ -387,11 +402,6 @@ class CodeTextRegion : public TypedRegion {
 protected:
   CodeTextRegion(const MemRegion *sreg, Kind k) : TypedRegion(sreg, k) {}
 public:
-  QualType getValueType() const {
-    assert(0 && "Do not get the object type of a CodeTextRegion.");
-    return QualType();
-  }
-  
   bool isBoundable() const { return false; }
     
   static bool classof(const MemRegion* R) {
@@ -404,7 +414,7 @@ class CodeTextRegion : public TypedRegion {
 class FunctionTextRegion : public CodeTextRegion {
   const FunctionDecl *FD;
 public:
-  FunctionTextRegion(const FunctionDecl* fd, const MemRegion* sreg)
+  FunctionTextRegion(const FunctionDecl *fd, const MemRegion* sreg)
     : CodeTextRegion(sreg, FunctionTextRegionKind), FD(fd) {}
   
   QualType getLocationType() const {
@@ -415,7 +425,7 @@ class FunctionTextRegion : public CodeTextRegion {
     return FD;
   }
     
-  virtual void dumpToStream(llvm::raw_ostream& os) const;
+  virtual void dumpToStream(raw_ostream &os) const;
   
   void Profile(llvm::FoldingSetNodeID& ID) const;
   
@@ -456,7 +466,7 @@ class BlockTextRegion : public CodeTextRegion {
 
   AnalysisContext *getAnalysisContext() const { return AC; }
     
-  virtual void dumpToStream(llvm::raw_ostream& os) const;
+  virtual void dumpToStream(raw_ostream &os) const;
   
   void Profile(llvm::FoldingSetNodeID& ID) const;
   
@@ -509,7 +519,7 @@ class BlockDataRegion : public SubRegion {
     bool operator!=(const referenced_vars_iterator &I) const {
       return I.R != R;
     }
-    referenced_vars_iterator& operator++() {
+    referenced_vars_iterator &operator++() {
       ++R;
       return *this;
     }
@@ -518,7 +528,7 @@ class BlockDataRegion : public SubRegion {
   referenced_vars_iterator referenced_vars_begin() const;
   referenced_vars_iterator referenced_vars_end() const;  
     
-  virtual void dumpToStream(llvm::raw_ostream& os) const;
+  virtual void dumpToStream(raw_ostream &os) const;
     
   void Profile(llvm::FoldingSetNodeID& ID) const;
     
@@ -559,7 +569,7 @@ class SymbolicRegion : public SubRegion {
                             SymbolRef sym,
                             const MemRegion* superRegion);
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static bool classof(const MemRegion* R) {
     return R->getKind() == SymbolicRegionKind;
@@ -567,13 +577,13 @@ class SymbolicRegion : public SubRegion {
 };
 
 /// StringRegion - Region associated with a StringLiteral.
-class StringRegion : public TypedRegion {
+class StringRegion : public TypedValueRegion {
   friend class MemRegionManager;
   const StringLiteral* Str;
 protected:
 
   StringRegion(const StringLiteral* str, const MemRegion* sreg)
-    : TypedRegion(sreg, StringRegionKind), Str(str) {}
+    : TypedValueRegion(sreg, StringRegionKind), Str(str) {}
 
   static void ProfileRegion(llvm::FoldingSetNodeID& ID,
                             const StringLiteral* Str,
@@ -595,7 +605,7 @@ class StringRegion : public TypedRegion {
     ProfileRegion(ID, Str, superRegion);
   }
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static bool classof(const MemRegion* R) {
     return R->getKind() == StringRegionKind;
@@ -605,16 +615,16 @@ class StringRegion : public TypedRegion {
 /// CompoundLiteralRegion - A memory region representing a compound literal.
 ///   Compound literals are essentially temporaries that are stack allocated
 ///   or in the global constant pool.
-class CompoundLiteralRegion : public TypedRegion {
+class CompoundLiteralRegion : public TypedValueRegion {
 private:
   friend class MemRegionManager;
-  const CompoundLiteralExpr* CL;
+  const CompoundLiteralExpr *CL;
 
-  CompoundLiteralRegion(const CompoundLiteralExpr* cl, const MemRegion* sReg)
-    : TypedRegion(sReg, CompoundLiteralRegionKind), CL(cl) {}
+  CompoundLiteralRegion(const CompoundLiteralExpr *cl, const MemRegion* sReg)
+    : TypedValueRegion(sReg, CompoundLiteralRegionKind), CL(cl) {}
 
   static void ProfileRegion(llvm::FoldingSetNodeID& ID,
-                            const CompoundLiteralExpr* CL,
+                            const CompoundLiteralExpr *CL,
                             const MemRegion* superRegion);
 public:
   QualType getValueType() const {
@@ -625,27 +635,27 @@ class CompoundLiteralRegion : public TypedRegion {
 
   void Profile(llvm::FoldingSetNodeID& ID) const;
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
-  const CompoundLiteralExpr* getLiteralExpr() const { return CL; }
+  const CompoundLiteralExpr *getLiteralExpr() const { return CL; }
 
   static bool classof(const MemRegion* R) {
     return R->getKind() == CompoundLiteralRegionKind;
   }
 };
 
-class DeclRegion : public TypedRegion {
+class DeclRegion : public TypedValueRegion {
 protected:
-  const Decl* D;
+  const Decl *D;
 
-  DeclRegion(const Decl* d, const MemRegion* sReg, Kind k)
-    : TypedRegion(sReg, k), D(d) {}
+  DeclRegion(const Decl *d, const MemRegion* sReg, Kind k)
+    : TypedValueRegion(sReg, k), D(d) {}
 
-  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const Decl* D,
+  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const Decl *D,
                       const MemRegion* superRegion, Kind k);
 
 public:
-  const Decl* getDecl() const { return D; }
+  const Decl *getDecl() const { return D; }
   void Profile(llvm::FoldingSetNodeID& ID) const;
 
   DefinedOrUnknownSVal getExtent(SValBuilder &svalBuilder) const;
@@ -660,10 +670,10 @@ class VarRegion : public DeclRegion {
   friend class MemRegionManager;
 
   // Constructors and private methods.
-  VarRegion(const VarDecl* vd, const MemRegion* sReg)
+  VarRegion(const VarDecl *vd, const MemRegion* sReg)
     : DeclRegion(vd, sReg, VarRegionKind) {}
 
-  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const VarDecl* VD,
+  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const VarDecl *VD,
                             const MemRegion *superRegion) {
     DeclRegion::ProfileRegion(ID, VD, superRegion, VarRegionKind);
   }
@@ -680,7 +690,7 @@ class VarRegion : public DeclRegion {
     return getDecl()->getType();
   }
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static bool classof(const MemRegion* R) {
     return R->getKind() == VarRegionKind;
@@ -690,11 +700,11 @@ class VarRegion : public DeclRegion {
 /// CXXThisRegion - Represents the region for the implicit 'this' parameter
 ///  in a call to a C++ method.  This region doesn't represent the object
 ///  referred to by 'this', but rather 'this' itself.
-class CXXThisRegion : public TypedRegion {
+class CXXThisRegion : public TypedValueRegion {
   friend class MemRegionManager;
   CXXThisRegion(const PointerType *thisPointerTy,
                 const MemRegion *sReg)
-    : TypedRegion(sReg, CXXThisRegionKind), ThisPointerTy(thisPointerTy) {}
+    : TypedValueRegion(sReg, CXXThisRegionKind), ThisPointerTy(thisPointerTy) {}
 
   static void ProfileRegion(llvm::FoldingSetNodeID &ID,
                             const PointerType *PT,
@@ -707,7 +717,7 @@ class CXXThisRegion : public TypedRegion {
     return QualType(ThisPointerTy, 0);
   }
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
   
   static bool classof(const MemRegion* R) {
     return R->getKind() == CXXThisRegionKind;
@@ -720,14 +730,14 @@ class CXXThisRegion : public TypedRegion {
 class FieldRegion : public DeclRegion {
   friend class MemRegionManager;
 
-  FieldRegion(const FieldDecl* fd, const MemRegion* sReg)
+  FieldRegion(const FieldDecl *fd, const MemRegion* sReg)
     : DeclRegion(fd, sReg, FieldRegionKind) {}
 
 public:
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
-  const FieldDecl* getDecl() const { return cast<FieldDecl>(D); }
+  const FieldDecl *getDecl() const { return cast<FieldDecl>(D); }
 
   QualType getValueType() const {
     // FIXME: We can cache this if needed.
@@ -736,7 +746,7 @@ class FieldRegion : public DeclRegion {
 
   DefinedOrUnknownSVal getExtent(SValBuilder &svalBuilder) const;
 
-  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const FieldDecl* FD,
+  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const FieldDecl *FD,
                             const MemRegion* superRegion) {
     DeclRegion::ProfileRegion(ID, FD, superRegion, FieldRegionKind);
   }
@@ -750,19 +760,19 @@ class ObjCIvarRegion : public DeclRegion {
 
   friend class MemRegionManager;
 
-  ObjCIvarRegion(const ObjCIvarDecl* ivd, const MemRegion* sReg)
+  ObjCIvarRegion(const ObjCIvarDecl *ivd, const MemRegion* sReg)
     : DeclRegion(ivd, sReg, ObjCIvarRegionKind) {}
 
-  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const ObjCIvarDecl* ivd,
+  static void ProfileRegion(llvm::FoldingSetNodeID& ID, const ObjCIvarDecl *ivd,
                             const MemRegion* superRegion) {
     DeclRegion::ProfileRegion(ID, ivd, superRegion, ObjCIvarRegionKind);
   }
 
 public:
-  const ObjCIvarDecl* getDecl() const { return cast<ObjCIvarDecl>(D); }
+  const ObjCIvarDecl *getDecl() const { return cast<ObjCIvarDecl>(D); }
   QualType getValueType() const { return getDecl()->getType(); }
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static bool classof(const MemRegion* R) {
     return R->getKind() == ObjCIvarRegionKind;
@@ -789,18 +799,18 @@ class RegionRawOffset {
   CharUnits getOffset() const { return Offset; }
   const MemRegion *getRegion() const { return Region; }
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
   void dump() const;
 };
 
-class ElementRegion : public TypedRegion {
+class ElementRegion : public TypedValueRegion {
   friend class MemRegionManager;
 
   QualType ElementType;
   NonLoc Index;
 
   ElementRegion(QualType elementType, NonLoc Idx, const MemRegion* sReg)
-    : TypedRegion(sReg, ElementRegionKind),
+    : TypedValueRegion(sReg, ElementRegionKind),
       ElementType(elementType), Index(Idx) {
     assert((!isa<nonloc::ConcreteInt>(&Idx) ||
            cast<nonloc::ConcreteInt>(&Idx)->getValue().isSigned()) &&
@@ -824,7 +834,7 @@ class ElementRegion : public TypedRegion {
   /// Compute the offset within the array. The array might also be a subobject.
   RegionRawOffset getAsArrayOffset() const;
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   void Profile(llvm::FoldingSetNodeID& ID) const;
 
@@ -834,23 +844,25 @@ class ElementRegion : public TypedRegion {
 };
 
 // C++ temporary object associated with an expression.
-class CXXTempObjectRegion : public TypedRegion {
+class CXXTempObjectRegion : public TypedValueRegion {
   friend class MemRegionManager;
 
   Expr const *Ex;
 
   CXXTempObjectRegion(Expr const *E, MemRegion const *sReg) 
-    : TypedRegion(sReg, CXXTempObjectRegionKind), Ex(E) {}
+    : TypedValueRegion(sReg, CXXTempObjectRegionKind), Ex(E) {}
 
   static void ProfileRegion(llvm::FoldingSetNodeID &ID,
                             Expr const *E, const MemRegion *sReg);
   
 public:
+  const Expr *getExpr() const { return Ex; }
+
   QualType getValueType() const {
     return Ex->getType();
   }
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   void Profile(llvm::FoldingSetNodeID &ID) const;
 
@@ -861,13 +873,13 @@ class CXXTempObjectRegion : public TypedRegion {
 
 // CXXBaseObjectRegion represents a base object within a C++ object. It is 
 // identified by the base class declaration and the region of its parent object.
-class CXXBaseObjectRegion : public TypedRegion {
+class CXXBaseObjectRegion : public TypedValueRegion {
   friend class MemRegionManager;
 
   const CXXRecordDecl *decl;
 
   CXXBaseObjectRegion(const CXXRecordDecl *d, const MemRegion *sReg)
-    : TypedRegion(sReg, CXXBaseObjectRegionKind), decl(d) {}
+    : TypedValueRegion(sReg, CXXBaseObjectRegionKind), decl(d) {}
 
   static void ProfileRegion(llvm::FoldingSetNodeID &ID,
                             const CXXRecordDecl *decl, const MemRegion *sReg);
@@ -877,7 +889,7 @@ class CXXBaseObjectRegion : public TypedRegion {
 
   QualType getValueType() const;
 
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   void Profile(llvm::FoldingSetNodeID &ID) const;
 
@@ -951,13 +963,13 @@ class MemRegionManager {
   const MemSpaceRegion *getCodeRegion();
 
   /// getAllocaRegion - Retrieve a region associated with a call to alloca().
-  const AllocaRegion *getAllocaRegion(const Expr* Ex, unsigned Cnt,
+  const AllocaRegion *getAllocaRegion(const Expr *Ex, unsigned Cnt,
                                       const LocationContext *LC);
 
   /// getCompoundLiteralRegion - Retrieve the region associated with a
   ///  given CompoundLiteral.
   const CompoundLiteralRegion*
-  getCompoundLiteralRegion(const CompoundLiteralExpr* CL,
+  getCompoundLiteralRegion(const CompoundLiteralExpr *CL,
                            const LocationContext *LC);
   
   /// getCXXThisRegion - Retrieve the [artificial] region associated with the
@@ -994,7 +1006,7 @@ class MemRegionManager {
   ///  a specified FieldDecl.  'superRegion' corresponds to the containing
   ///  memory region (which typically represents the memory representing
   ///  a structure or class).
-  const FieldRegion *getFieldRegion(const FieldDecl* fd,
+  const FieldRegion *getFieldRegion(const FieldDecl *fd,
                                     const MemRegion* superRegion);
 
   const FieldRegion *getFieldRegionWithSuper(const FieldRegion *FR,
@@ -1006,7 +1018,7 @@ class MemRegionManager {
   ///   a specified Objective-c instance variable.  'superRegion' corresponds
   ///   to the containing region (which typically represents the Objective-C
   ///   object).
-  const ObjCIvarRegion *getObjCIvarRegion(const ObjCIvarDecl* ivd,
+  const ObjCIvarRegion *getObjCIvarRegion(const ObjCIvarDecl *ivd,
                                           const MemRegion* superRegion);
 
   const CXXTempObjectRegion *getCXXTempObjectRegion(Expr const *Ex,
@@ -1069,7 +1081,7 @@ class MemRegionManager {
 // Out-of-line member definitions.
 //===----------------------------------------------------------------------===//
 
-inline ASTContext& MemRegion::getContext() const {
+inline ASTContext &MemRegion::getContext() const {
   return getMemRegionManager()->getContext();
 }
   
@@ -1082,7 +1094,7 @@ inline ASTContext& MemRegion::getContext() const {
 //===----------------------------------------------------------------------===//
 
 namespace llvm {
-static inline raw_ostream& operator<<(raw_ostream& os,
+static inline raw_ostream &operator<<(raw_ostream &os,
                                       const clang::ento::MemRegion* R) {
   R->dumpToStream(os);
   return os;
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h
index 734024c2cba4..add3479804bd 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h
@@ -16,8 +16,10 @@
 #define LLVM_CLANG_STATICANALYZER_PATHSENSITIVE_OBJCMESSAGE
 
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/AST/ExprObjC.h"
+#include "clang/AST/ExprCXX.h"
+#include "llvm/ADT/PointerUnion.h"
 
 namespace clang {
 namespace ento {
@@ -86,6 +88,21 @@ class ObjCMessage {
     return 0;
   }
 
+  SVal getInstanceReceiverSVal(const ProgramState *State,
+                               const LocationContext *LC) const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    if (!isInstanceMessage())
+      return UndefinedVal();
+    if (const Expr *Ex = getInstanceReceiver())
+      return State->getSValAsScalarOrLoc(Ex);
+
+    // An instance message with no expression means we are sending to super.
+    // In this case the object reference is the same as 'self'.
+    const ImplicitParamDecl *SelfDecl = LC->getSelfDecl();
+    assert(SelfDecl && "No message receiver Expr, but not in an ObjC method");
+    return State->getSVal(State->getRegion(SelfDecl, LC));
+  }
+
   bool isInstanceMessage() const {
     assert(isValid() && "This ObjCMessage is uninitialized!");
     if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
@@ -96,7 +113,7 @@ class ObjCMessage {
   }
 
   const ObjCMethodDecl *getMethodDecl() const;
-  
+
   const ObjCInterfaceDecl *getReceiverInterface() const;
 
   SourceLocation getSuperLoc() const {
@@ -106,45 +123,63 @@ class ObjCMessage {
     return cast<ObjCPropertyRefExpr>(MsgOrPropE)->getReceiverLocation();
   }
 
-   SourceRange getSourceRange() const {
-     assert(isValid() && "This ObjCMessage is uninitialized!");
+  const Expr *getMsgOrPropExpr() const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    return MsgOrPropE;
+  }
+
+  SourceRange getSourceRange() const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
     return MsgOrPropE->getSourceRange();
   }
 
-   unsigned getNumArgs() const {
-     assert(isValid() && "This ObjCMessage is uninitialized!");
-     if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
-       return msgE->getNumArgs();
-     return isPropertySetter() ? 1 : 0;
-   }
+  unsigned getNumArgs() const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+      return msgE->getNumArgs();
+    return isPropertySetter() ? 1 : 0;
+  }
 
-   SVal getArgSVal(unsigned i, const GRState *state) const {
-     assert(isValid() && "This ObjCMessage is uninitialized!");
-     assert(i < getNumArgs() && "Invalid index for argument");
-     if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
-       return state->getSVal(msgE->getArg(i));
-     assert(isPropertySetter());
-     return SetterArgV;
-   }
+  SVal getArgSVal(unsigned i, const ProgramState *state) const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    assert(i < getNumArgs() && "Invalid index for argument");
+    if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+      return state->getSVal(msgE->getArg(i));
+    assert(isPropertySetter());
+    return SetterArgV;
+  }
 
-   QualType getArgType(unsigned i) const {
-     assert(isValid() && "This ObjCMessage is uninitialized!");
-     assert(i < getNumArgs() && "Invalid index for argument");
-     if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
-       return msgE->getArg(i)->getType();
-     assert(isPropertySetter());
-     return cast<ObjCPropertyRefExpr>(MsgOrPropE)->getType();
-   }
-   
-   const Expr *getArgExpr(unsigned i) const;
+  QualType getArgType(unsigned i) const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    assert(i < getNumArgs() && "Invalid index for argument");
+    if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+      return msgE->getArg(i)->getType();
+    assert(isPropertySetter());
+    return cast<ObjCPropertyRefExpr>(MsgOrPropE)->getType();
+  }
 
-   SourceRange getArgSourceRange(unsigned i) const {
-     assert(isValid() && "This ObjCMessage is uninitialized!");
-     assert(i < getNumArgs() && "Invalid index for argument");
-     if (const Expr *argE = getArgExpr(i))
-       return argE->getSourceRange();
-     return OriginE->getSourceRange();
-   }
+  const Expr *getArgExpr(unsigned i) const;
+
+  SourceRange getArgSourceRange(unsigned i) const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    assert(i < getNumArgs() && "Invalid index for argument");
+    if (const Expr *argE = getArgExpr(i))
+      return argE->getSourceRange();
+    return OriginE->getSourceRange();
+  }
+
+  SourceRange getReceiverSourceRange() const {
+    assert(isValid() && "This ObjCMessage is uninitialized!");
+    if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+      return msgE->getReceiverRange();
+
+    const ObjCPropertyRefExpr *propE = cast<ObjCPropertyRefExpr>(MsgOrPropE);
+    if (propE->isObjectReceiver())
+      return propE->getBase()->getSourceRange();
+
+    // FIXME: This isn't a range.
+    return propE->getReceiverLocation();
+  }
 };
 
 class ObjCPropertyGetter : public ObjCMessage {
@@ -165,58 +200,89 @@ class ObjCPropertySetter : public ObjCMessage {
   }
 };
 
-/// \brief Common wrapper for a call expression or an ObjC message, mainly to
-/// provide a common interface for handling their arguments.
+/// \brief Common wrapper for a call expression, ObjC message, or C++ 
+/// constructor, mainly to provide a common interface for their arguments.
 class CallOrObjCMessage {
-  const CallExpr *CallE;
+  llvm::PointerUnion<const CallExpr *, const CXXConstructExpr *> CallE;
   ObjCMessage Msg;
-  const GRState *State;
+  const ProgramState *State;
 public:
-  CallOrObjCMessage(const CallExpr *callE, const GRState *state)
+  CallOrObjCMessage(const CallExpr *callE, const ProgramState *state)
     : CallE(callE), State(state) {}
-  CallOrObjCMessage(const ObjCMessage &msg, const GRState *state)
-    : CallE(0), Msg(msg), State(state) {}
+  CallOrObjCMessage(const CXXConstructExpr *consE, const ProgramState *state)
+    : CallE(consE), State(state) {}
+  CallOrObjCMessage(const ObjCMessage &msg, const ProgramState *state)
+    : CallE((CallExpr *)0), Msg(msg), State(state) {}
 
   QualType getResultType(ASTContext &ctx) const;
   
   bool isFunctionCall() const {
-    return (bool) CallE;
+    return CallE && CallE.is<const CallExpr *>();
   }
-  
+
+  bool isCXXConstructExpr() const {
+    return CallE && CallE.is<const CXXConstructExpr *>();
+  }
+
+  bool isObjCMessage() const {
+    return !CallE;
+  }
+
   bool isCXXCall() const {
-    return CallE && isa<CXXMemberCallExpr>(CallE);
+    const CallExpr *ActualCallE = CallE.dyn_cast<const CallExpr *>();
+    return ActualCallE && isa<CXXMemberCallExpr>(ActualCallE);
+  }
+
+  const Expr *getOriginExpr() const {
+    if (!CallE)
+      return Msg.getOriginExpr();
+    if (const CXXConstructExpr *Ctor =
+          CallE.dyn_cast<const CXXConstructExpr *>())
+      return Ctor;
+    return CallE.get<const CallExpr *>();
   }
   
   SVal getFunctionCallee() const;
   SVal getCXXCallee() const;
+  SVal getInstanceMessageReceiver(const LocationContext *LC) const;
 
   unsigned getNumArgs() const {
-    if (CallE) return CallE->getNumArgs();
-    return Msg.getNumArgs();
+    if (!CallE)
+      return Msg.getNumArgs();
+    if (const CXXConstructExpr *Ctor =
+          CallE.dyn_cast<const CXXConstructExpr *>())
+      return Ctor->getNumArgs();
+    return CallE.get<const CallExpr *>()->getNumArgs();
   }
 
   SVal getArgSVal(unsigned i) const {
     assert(i < getNumArgs());
-    if (CallE) 
-      return State->getSVal(CallE->getArg(i));
-    return Msg.getArgSVal(i, State);
+    if (!CallE)
+      return Msg.getArgSVal(i, State);
+    return State->getSVal(getArg(i));
   }
 
-  SVal getArgSValAsScalarOrLoc(unsigned i) const;
-
   const Expr *getArg(unsigned i) const {
     assert(i < getNumArgs());
-    if (CallE)
-      return CallE->getArg(i);
-    return Msg.getArgExpr(i);
+    if (!CallE)
+      return Msg.getArgExpr(i);
+    if (const CXXConstructExpr *Ctor =
+          CallE.dyn_cast<const CXXConstructExpr *>())
+      return Ctor->getArg(i);
+    return CallE.get<const CallExpr *>()->getArg(i);
   }
 
   SourceRange getArgSourceRange(unsigned i) const {
     assert(i < getNumArgs());
     if (CallE)
-      return CallE->getArg(i)->getSourceRange();
+      return getArg(i)->getSourceRange();
     return Msg.getArgSourceRange(i);
   }
+
+  SourceRange getReceiverSourceRange() const {
+    assert(isObjCMessage());
+    return Msg.getReceiverSourceRange();
+  }
 };
 
 }
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/GRState.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
similarity index 56%
rename from contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/GRState.h
rename to contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
index 0d61d0e620eb..edae06e68c34 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/GRState.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
@@ -1,4 +1,4 @@
-//== GRState.h - Path-sensitive "State" for tracking values -----*- C++ -*--==//
+//== ProgramState.h - Path-sensitive "State" for tracking values -*- C++ -*--=//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,13 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines SymbolRef, ExprBindKey, and GRState*.
+//  This file defines SymbolRef, ExprBindKey, and ProgramState*.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_GR_VALUESTATE_H
 #define LLVM_CLANG_GR_VALUESTATE_H
 
+#include "clang/Basic/LLVM.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/Environment.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
@@ -21,12 +22,10 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableMap.h"
-#include "llvm/Support/Casting.h"
 
 namespace llvm {
 class APSInt;
 class BumpPtrAllocator;
-class raw_ostream;
 }
 
 namespace clang {
@@ -34,30 +33,31 @@ class ASTContext;
 
 namespace ento {
 
-class GRStateManager;
+class ProgramStateManager;
 
-typedef ConstraintManager* (*ConstraintManagerCreator)(GRStateManager&,
+typedef ConstraintManager* (*ConstraintManagerCreator)(ProgramStateManager&,
                                                        SubEngine&);
-typedef StoreManager* (*StoreManagerCreator)(GRStateManager&);
+typedef StoreManager* (*StoreManagerCreator)(ProgramStateManager&);
 
 //===----------------------------------------------------------------------===//
-// GRStateTrait - Traits used by the Generic Data Map of a GRState.
+// ProgramStateTrait - Traits used by the Generic Data Map of a ProgramState.
 //===----------------------------------------------------------------------===//
 
-template <typename T> struct GRStatePartialTrait;
+template <typename T> struct ProgramStatePartialTrait;
 
-template <typename T> struct GRStateTrait {
+template <typename T> struct ProgramStateTrait {
   typedef typename T::data_type data_type;
-  static inline void* GDMIndex() { return &T::TagInt; }
-  static inline void* MakeVoidPtr(data_type D) { return (void*) D; }
-  static inline data_type MakeData(void* const* P) {
+  static inline void *GDMIndex() { return &T::TagInt; }
+  static inline void *MakeVoidPtr(data_type D) { return (void*) D; }
+  static inline data_type MakeData(void *const* P) {
     return P ? (data_type) *P : (data_type) 0;
   }
 };
 
-class GRStateManager;
+class ProgramStateManager;
 
-/// GRState - This class encapsulates:
+/// \class ProgramState
+/// ProgramState - This class encapsulates:
 ///
 ///    1. A mapping from expressions to values (Environment)
 ///    2. A mapping from locations to values (Store)
@@ -65,47 +65,47 @@ class GRStateManager;
 ///
 ///  Together these represent the "abstract state" of a program.
 ///
-///  GRState is intended to be used as a functional object; that is,
+///  ProgramState is intended to be used as a functional object; that is,
 ///  once it is created and made "persistent" in a FoldingSet, its
 ///  values will never change.
-class GRState : public llvm::FoldingSetNode {
+class ProgramState : public llvm::FoldingSetNode {
 public:
   typedef llvm::ImmutableSet<llvm::APSInt*>                IntSetTy;
   typedef llvm::ImmutableMap<void*, void*>                 GenericDataMap;
 
 private:
-  void operator=(const GRState& R) const; // Do not implement.
+  void operator=(const ProgramState& R) const; // Do not implement.
 
-  friend class GRStateManager;
+  friend class ProgramStateManager;
   friend class ExplodedGraph;
   friend class ExplodedNode;
 
-  GRStateManager *stateMgr;
+  ProgramStateManager *stateMgr;
   Environment Env;           // Maps a Stmt to its current SVal.
   Store store;               // Maps a location to its current value.
   GenericDataMap   GDM;      // Custom data stored by a client of this class.
   unsigned refCount;
 
-  /// makeWithStore - Return a GRState with the same values as the current
+  /// makeWithStore - Return a ProgramState with the same values as the current
   ///  state with the exception of using the specified Store.
-  const GRState *makeWithStore(const StoreRef &store) const;
+  const ProgramState *makeWithStore(const StoreRef &store) const;
 
   void setStore(const StoreRef &storeRef);
 
 public:
 
-  /// This ctor is used when creating the first GRState object.
-  GRState(GRStateManager *mgr, const Environment& env,
+  /// This ctor is used when creating the first ProgramState object.
+  ProgramState(ProgramStateManager *mgr, const Environment& env,
           StoreRef st, GenericDataMap gdm);
     
   /// Copy ctor - We must explicitly define this or else the "Next" ptr
   ///  in FoldingSetNode will also get copied.
-  GRState(const GRState& RHS);
+  ProgramState(const ProgramState &RHS);
   
-  ~GRState();
+  ~ProgramState();
 
-  /// Return the GRStateManager associated with this state.
-  GRStateManager &getStateManager() const { return *stateMgr; }
+  /// Return the ProgramStateManager associated with this state.
+  ProgramStateManager &getStateManager() const { return *stateMgr; }
 
   /// Return true if this state is referenced by a persistent ExplodedNode.
   bool referencedByExplodedNode() const { return refCount > 0; }
@@ -124,10 +124,10 @@ class GRState : public llvm::FoldingSetNode {
 
   void setGDM(GenericDataMap gdm) { GDM = gdm; }
 
-  /// Profile - Profile the contents of a GRState object for use in a
-  ///  FoldingSet.  Two GRState objects are considered equal if they
+  /// Profile - Profile the contents of a ProgramState object for use in a
+  ///  FoldingSet.  Two ProgramState objects are considered equal if they
   ///  have the same Environment, Store, and GenericDataMap.
-  static void Profile(llvm::FoldingSetNodeID& ID, const GRState* V) {
+  static void Profile(llvm::FoldingSetNodeID& ID, const ProgramState *V) {
     V->Env.Profile(ID);
     ID.AddPointer(V->store);
     V->GDM.Profile(ID);
@@ -146,44 +146,41 @@ class GRState : public llvm::FoldingSetNode {
   // Constraints on values.
   //==---------------------------------------------------------------------==//
   //
-  // Each GRState records constraints on symbolic values.  These constraints
-  // are managed using the ConstraintManager associated with a GRStateManager.
+  // Each ProgramState records constraints on symbolic values.  These constraints
+  // are managed using the ConstraintManager associated with a ProgramStateManager.
   // As constraints gradually accrue on symbolic values, added constraints
   // may conflict and indicate that a state is infeasible (as no real values
   // could satisfy all the constraints).  This is the principal mechanism
-  // for modeling path-sensitivity in ExprEngine/GRState.
+  // for modeling path-sensitivity in ExprEngine/ProgramState.
   //
   // Various "assume" methods form the interface for adding constraints to
   // symbolic values.  A call to 'assume' indicates an assumption being placed
   // on one or symbolic values.  'assume' methods take the following inputs:
   //
-  //  (1) A GRState object representing the current state.
+  //  (1) A ProgramState object representing the current state.
   //
   //  (2) The assumed constraint (which is specific to a given "assume" method).
   //
   //  (3) A binary value "Assumption" that indicates whether the constraint is
   //      assumed to be true or false.
   //
-  // The output of "assume*" is a new GRState object with the added constraints.
+  // The output of "assume*" is a new ProgramState object with the added constraints.
   // If no new state is feasible, NULL is returned.
   //
 
-  const GRState *assume(DefinedOrUnknownSVal cond, bool assumption) const;
+  const ProgramState *assume(DefinedOrUnknownSVal cond, bool assumption) const;
 
   /// This method assumes both "true" and "false" for 'cond', and
   ///  returns both corresponding states.  It's shorthand for doing
   ///  'assume' twice.
-  std::pair<const GRState*, const GRState*>
+  std::pair<const ProgramState*, const ProgramState*>
   assume(DefinedOrUnknownSVal cond) const;
 
-  const GRState *assumeInBound(DefinedOrUnknownSVal idx,
+  const ProgramState *assumeInBound(DefinedOrUnknownSVal idx,
                                DefinedOrUnknownSVal upperBound,
                                bool assumption) const;
 
-  //==---------------------------------------------------------------------==//
-  // Utility methods for getting regions.
-  //==---------------------------------------------------------------------==//
-
+  /// Utility method for getting regions.
   const VarRegion* getRegion(const VarDecl *D, const LocationContext *LC) const;
 
   //==---------------------------------------------------------------------==//
@@ -192,52 +189,42 @@ class GRState : public llvm::FoldingSetNode {
 
   /// BindCompoundLiteral - Return the state that has the bindings currently
   ///  in this state plus the bindings for the CompoundLiteral.
-  const GRState *bindCompoundLiteral(const CompoundLiteralExpr* CL,
+  const ProgramState *bindCompoundLiteral(const CompoundLiteralExpr *CL,
                                      const LocationContext *LC,
                                      SVal V) const;
 
   /// Create a new state by binding the value 'V' to the statement 'S' in the
   /// state's environment.
-  const GRState *BindExpr(const Stmt *S, SVal V, bool Invalidate = true) const;
+  const ProgramState *BindExpr(const Stmt *S, SVal V, bool Invalidate = true) const;
 
   /// Create a new state by binding the value 'V' and location 'locaton' to the
   /// statement 'S' in the state's environment.
-  const GRState *bindExprAndLocation(const Stmt *S, SVal location, SVal V)
+  const ProgramState *bindExprAndLocation(const Stmt *S, SVal location, SVal V)
     const;
   
-  const GRState *bindDecl(const VarRegion *VR, SVal V) const;
+  const ProgramState *bindDecl(const VarRegion *VR, SVal V) const;
 
-  const GRState *bindDeclWithNoInit(const VarRegion *VR) const;
+  const ProgramState *bindDeclWithNoInit(const VarRegion *VR) const;
 
-  const GRState *bindLoc(Loc location, SVal V) const;
+  const ProgramState *bindLoc(Loc location, SVal V) const;
 
-  const GRState *bindLoc(SVal location, SVal V) const;
+  const ProgramState *bindLoc(SVal location, SVal V) const;
 
-  const GRState *bindDefault(SVal loc, SVal V) const;
+  const ProgramState *bindDefault(SVal loc, SVal V) const;
 
-  const GRState *unbindLoc(Loc LV) const;
-
-  /// invalidateRegion - Returns the state with bindings for the given region
-  ///  cleared from the store. See invalidateRegions.
-  const GRState *invalidateRegion(const MemRegion *R,
-                                  const Expr *E, unsigned BlockCount,
-                                  StoreManager::InvalidatedSymbols *IS = NULL)
-                                  const {
-    return invalidateRegions(&R, &R+1, E, BlockCount, IS, false);
-  }
+  const ProgramState *unbindLoc(Loc LV) const;
 
   /// invalidateRegions - Returns the state with bindings for the given regions
   ///  cleared from the store. The regions are provided as a continuous array
   ///  from Begin to End. Optionally invalidates global regions as well.
-  const GRState *invalidateRegions(const MemRegion * const *Begin,
-                                   const MemRegion * const *End,
+  const ProgramState *invalidateRegions(ArrayRef<const MemRegion *> Regions,
                                    const Expr *E, unsigned BlockCount,
-                                   StoreManager::InvalidatedSymbols *IS,
-                                   bool invalidateGlobals) const;
+                                   StoreManager::InvalidatedSymbols *IS = 0,
+                                   bool invalidateGlobals = false) const;
 
   /// enterStackFrame - Returns the state for entry to the given stack frame,
   ///  preserving the current state.
-  const GRState *enterStackFrame(const StackFrameContext *frame) const;
+  const ProgramState *enterStackFrame(const StackFrameContext *frame) const;
 
   /// Get the lvalue for a variable reference.
   Loc getLValue(const VarDecl *D, const LocationContext *LC) const;
@@ -260,7 +247,7 @@ class GRState : public llvm::FoldingSetNode {
   const llvm::APSInt *getSymVal(SymbolRef sym) const;
 
   /// Returns the SVal bound to the statement 'S' in the state's environment.
-  SVal getSVal(const Stmt* S, bool useOnlyDirectBindings = false) const;
+  SVal getSVal(const Stmt *S, bool useOnlyDirectBindings = false) const;
   
   SVal getSValAsScalarOrLoc(const Stmt *Ex) const;
 
@@ -273,11 +260,22 @@ class GRState : public llvm::FoldingSetNode {
 
   SVal getSValAsScalarOrLoc(const MemRegion *R) const;
   
+  /// \brief Visits the symbols reachable from the given SVal using the provided
+  /// SymbolVisitor.
+  ///
+  /// This is a convenience API. Consider using ScanReachableSymbols class
+  /// directly when making multiple scans on the same state with the same
+  /// visitor to avoid repeated initialization cost.
+  /// \sa ScanReachableSymbols
   bool scanReachableSymbols(SVal val, SymbolVisitor& visitor) const;
   
+  /// \brief Visits the symbols reachable from the SVals in the given range
+  /// using the provided SymbolVisitor.
   bool scanReachableSymbols(const SVal *I, const SVal *E,
                             SymbolVisitor &visitor) const;
   
+  /// \brief Visits the symbols reachable from the regions in the given
+  /// MemRegions range using the provided SymbolVisitor.
   bool scanReachableSymbols(const MemRegion * const *I, 
                             const MemRegion * const *E,
                             SymbolVisitor &visitor) const;
@@ -294,70 +292,62 @@ class GRState : public llvm::FoldingSetNode {
   // Accessing the Generic Data Map (GDM).
   //==---------------------------------------------------------------------==//
 
-  void* const* FindGDM(void* K) const;
+  void *const* FindGDM(void *K) const;
 
   template<typename T>
-  const GRState *add(typename GRStateTrait<T>::key_type K) const;
+  const ProgramState *add(typename ProgramStateTrait<T>::key_type K) const;
 
   template <typename T>
-  typename GRStateTrait<T>::data_type
+  typename ProgramStateTrait<T>::data_type
   get() const {
-    return GRStateTrait<T>::MakeData(FindGDM(GRStateTrait<T>::GDMIndex()));
+    return ProgramStateTrait<T>::MakeData(FindGDM(ProgramStateTrait<T>::GDMIndex()));
   }
 
   template<typename T>
-  typename GRStateTrait<T>::lookup_type
-  get(typename GRStateTrait<T>::key_type key) const {
-    void* const* d = FindGDM(GRStateTrait<T>::GDMIndex());
-    return GRStateTrait<T>::Lookup(GRStateTrait<T>::MakeData(d), key);
+  typename ProgramStateTrait<T>::lookup_type
+  get(typename ProgramStateTrait<T>::key_type key) const {
+    void *const* d = FindGDM(ProgramStateTrait<T>::GDMIndex());
+    return ProgramStateTrait<T>::Lookup(ProgramStateTrait<T>::MakeData(d), key);
   }
 
   template <typename T>
-  typename GRStateTrait<T>::context_type get_context() const;
+  typename ProgramStateTrait<T>::context_type get_context() const;
 
 
   template<typename T>
-  const GRState *remove(typename GRStateTrait<T>::key_type K) const;
+  const ProgramState *remove(typename ProgramStateTrait<T>::key_type K) const;
 
   template<typename T>
-  const GRState *remove(typename GRStateTrait<T>::key_type K,
-                        typename GRStateTrait<T>::context_type C) const;
+  const ProgramState *remove(typename ProgramStateTrait<T>::key_type K,
+                        typename ProgramStateTrait<T>::context_type C) const;
   template <typename T>
-  const GRState *remove() const;
+  const ProgramState *remove() const;
 
   template<typename T>
-  const GRState *set(typename GRStateTrait<T>::data_type D) const;
+  const ProgramState *set(typename ProgramStateTrait<T>::data_type D) const;
 
   template<typename T>
-  const GRState *set(typename GRStateTrait<T>::key_type K,
-                     typename GRStateTrait<T>::value_type E) const;
+  const ProgramState *set(typename ProgramStateTrait<T>::key_type K,
+                     typename ProgramStateTrait<T>::value_type E) const;
 
   template<typename T>
-  const GRState *set(typename GRStateTrait<T>::key_type K,
-                     typename GRStateTrait<T>::value_type E,
-                     typename GRStateTrait<T>::context_type C) const;
+  const ProgramState *set(typename ProgramStateTrait<T>::key_type K,
+                     typename ProgramStateTrait<T>::value_type E,
+                     typename ProgramStateTrait<T>::context_type C) const;
 
   template<typename T>
-  bool contains(typename GRStateTrait<T>::key_type key) const {
-    void* const* d = FindGDM(GRStateTrait<T>::GDMIndex());
-    return GRStateTrait<T>::Contains(GRStateTrait<T>::MakeData(d), key);
+  bool contains(typename ProgramStateTrait<T>::key_type key) const {
+    void *const* d = FindGDM(ProgramStateTrait<T>::GDMIndex());
+    return ProgramStateTrait<T>::Contains(ProgramStateTrait<T>::MakeData(d), key);
   }
 
-  // State pretty-printing.
-  class Printer {
-  public:
-    virtual ~Printer() {}
-    virtual void Print(llvm::raw_ostream& Out, const GRState* state,
-                       const char* nl, const char* sep) = 0;
-  };
-
   // Pretty-printing.
-  void print(llvm::raw_ostream& Out, CFG &C, const char *nl = "\n",
+  void print(raw_ostream &Out, CFG &C, const char *nl = "\n",
              const char *sep = "") const;
 
   void printStdErr(CFG &C) const;
 
-  void printDOT(llvm::raw_ostream& Out, CFG &C) const;
+  void printDOT(raw_ostream &Out, CFG &C) const;
 
 private:
   /// Increments the number of times this state is referenced by ExplodeNodes.
@@ -369,20 +359,20 @@ class GRState : public llvm::FoldingSetNode {
     --refCount;
   }
   
-  const GRState *invalidateRegionsImpl(const MemRegion * const *Begin,
-                                       const MemRegion * const *End,
-                                       const Expr *E, unsigned BlockCount,
-                                       StoreManager::InvalidatedSymbols &IS,
-                                       bool invalidateGlobals) const;
+  const ProgramState *
+  invalidateRegionsImpl(ArrayRef<const MemRegion *> Regions,
+                        const Expr *E, unsigned BlockCount,
+                        StoreManager::InvalidatedSymbols &IS,
+                        bool invalidateGlobals) const;
 };
 
-class GRStateSet {
-  typedef llvm::SmallPtrSet<const GRState*,5> ImplTy;
+class ProgramStateSet {
+  typedef llvm::SmallPtrSet<const ProgramState*,5> ImplTy;
   ImplTy Impl;
 public:
-  GRStateSet() {}
+  ProgramStateSet() {}
 
-  inline void Add(const GRState* St) {
+  inline void Add(const ProgramState *St) {
     Impl.insert(St);
   }
 
@@ -395,11 +385,11 @@ class GRStateSet {
   inline iterator end() const { return Impl.end();   }
 
   class AutoPopulate {
-    GRStateSet& S;
+    ProgramStateSet &S;
     unsigned StartSize;
-    const GRState* St;
+    const ProgramState *St;
   public:
-    AutoPopulate(GRStateSet& s, const GRState* st)
+    AutoPopulate(ProgramStateSet &s, const ProgramState *st)
       : S(s), StartSize(S.size()), St(st) {}
 
     ~AutoPopulate() {
@@ -410,12 +400,11 @@ class GRStateSet {
 };
 
 //===----------------------------------------------------------------------===//
-// GRStateManager - Factory object for GRStates.
+// ProgramStateManager - Factory object for ProgramStates.
 //===----------------------------------------------------------------------===//
 
-class GRStateManager {
-  friend class GRState;
-  friend class ExprEngine; // FIXME: Remove.
+class ProgramStateManager {
+  friend class ProgramState;
 private:
   /// Eng - The SubEngine that owns this state manager.
   SubEngine *Eng; /* Can be null. */
@@ -424,18 +413,14 @@ class GRStateManager {
   llvm::OwningPtr<StoreManager>        StoreMgr;
   llvm::OwningPtr<ConstraintManager>   ConstraintMgr;
 
-  GRState::GenericDataMap::Factory     GDMFactory;
+  ProgramState::GenericDataMap::Factory     GDMFactory;
 
   typedef llvm::DenseMap<void*,std::pair<void*,void (*)(void*)> > GDMContextsTy;
   GDMContextsTy GDMContexts;
 
-  /// Printers - A set of printer objects used for pretty-printing a GRState.
-  ///  GRStateManager owns these objects.
-  std::vector<GRState::Printer*> Printers;
-
   /// StateSet - FoldingSet containing all the states created for analyzing
   ///  a particular function.  This is used to unique states.
-  llvm::FoldingSet<GRState> StateSet;
+  llvm::FoldingSet<ProgramState> StateSet;
 
   /// Object that manages the data for all created SVals.
   llvm::OwningPtr<SValBuilder> svalBuilder;
@@ -443,15 +428,15 @@ class GRStateManager {
   /// A BumpPtrAllocator to allocate states.
   llvm::BumpPtrAllocator &Alloc;
 
-  /// A vector of recently allocated GRStates that can potentially be
+  /// A vector of recently allocated ProgramStates that can potentially be
   /// reused.
-  std::vector<GRState *> recentlyAllocatedStates;
+  std::vector<ProgramState *> recentlyAllocatedStates;
   
-  /// A vector of GRStates that we can reuse.
-  std::vector<GRState *> freeStates;
+  /// A vector of ProgramStates that we can reuse.
+  std::vector<ProgramState *> freeStates;
 
 public:
-  GRStateManager(ASTContext& Ctx,
+  ProgramStateManager(ASTContext &Ctx,
                  StoreManagerCreator CreateStoreManager,
                  ConstraintManagerCreator CreateConstraintManager,
                  llvm::BumpPtrAllocator& alloc,
@@ -465,7 +450,7 @@ class GRStateManager {
     ConstraintMgr.reset((*CreateConstraintManager)(*this, subeng));
   }
 
-  GRStateManager(ASTContext& Ctx,
+  ProgramStateManager(ASTContext &Ctx,
                  StoreManagerCreator CreateStoreManager,
                  ConstraintManager* ConstraintManagerPtr,
                  llvm::BumpPtrAllocator& alloc)
@@ -478,9 +463,9 @@ class GRStateManager {
     ConstraintMgr.reset(ConstraintManagerPtr);
   }
 
-  ~GRStateManager();
+  ~ProgramStateManager();
 
-  const GRState *getInitialState(const LocationContext *InitLoc);
+  const ProgramState *getInitialState(const LocationContext *InitLoc);
 
   ASTContext &getContext() { return svalBuilder->getContext(); }
   const ASTContext &getContext() const { return svalBuilder->getContext(); }
@@ -516,13 +501,13 @@ class GRStateManager {
   ConstraintManager& getConstraintManager() { return *ConstraintMgr; }
   SubEngine* getOwningEngine() { return Eng; }
 
-  const GRState* removeDeadBindings(const GRState* St,
+  const ProgramState *removeDeadBindings(const ProgramState *St,
                                     const StackFrameContext *LCtx,
                                     SymbolReaper& SymReaper);
 
   /// Marshal a new state for the callee in another translation unit.
   /// 'state' is owned by the caller's engine.
-  const GRState *MarshalState(const GRState *state, const StackFrameContext *L);
+  const ProgramState *MarshalState(const ProgramState *state, const StackFrameContext *L);
 
 public:
 
@@ -531,18 +516,28 @@ class GRStateManager {
   }
 
   // Methods that manipulate the GDM.
-  const GRState* addGDM(const GRState* St, void* Key, void* Data);
-  const GRState *removeGDM(const GRState *state, void *Key);
+  const ProgramState *addGDM(const ProgramState *St, void *Key, void *Data);
+  const ProgramState *removeGDM(const ProgramState *state, void *Key);
 
   // Methods that query & manipulate the Store.
 
-  void iterBindings(const GRState* state, StoreManager::BindingsHandler& F) {
+  void iterBindings(const ProgramState *state, StoreManager::BindingsHandler& F) {
     StoreMgr->iterBindings(state->getStore(), F);
   }
 
-  const GRState* getPersistentState(GRState& Impl);
-  
-  /// Periodically called by ExprEngine to recycle GRStates that were
+  const ProgramState *getPersistentState(ProgramState &Impl);
+  const ProgramState *getPersistentStateWithGDM(const ProgramState *FromState,
+                                           const ProgramState *GDMState);
+
+  bool haveEqualEnvironments(const ProgramState * S1, const ProgramState * S2) {
+    return S1->Env == S2->Env;
+  }
+
+  bool haveEqualStores(const ProgramState * S1, const ProgramState * S2) {
+    return S1->store == S2->store;
+  }
+
+  /// Periodically called by ExprEngine to recycle ProgramStates that were
   /// created but never used for creating an ExplodedNode.
   void recycleUnusedStates();
 
@@ -550,9 +545,9 @@ class GRStateManager {
   // Generic Data Map methods.
   //==---------------------------------------------------------------------==//
   //
-  // GRStateManager and GRState support a "generic data map" that allows
-  // different clients of GRState objects to embed arbitrary data within a
-  // GRState object.  The generic data map is essentially an immutable map
+  // ProgramStateManager and ProgramState support a "generic data map" that allows
+  // different clients of ProgramState objects to embed arbitrary data within a
+  // ProgramState object.  The generic data map is essentially an immutable map
   // from a "tag" (that acts as the "key" for a client) and opaque values.
   // Tags/keys and values are simply void* values.  The typical way that clients
   // generate unique tags are by taking the address of a static variable.
@@ -560,82 +555,82 @@ class GRStateManager {
   // the data pointer are immutable (and thus are essentially purely functional
   // data).
   //
-  // The templated methods below use the GRStateTrait<T> class
+  // The templated methods below use the ProgramStateTrait<T> class
   // to resolve keys into the GDM and to return data values to clients.
   //
 
   // Trait based GDM dispatch.
   template <typename T>
-  const GRState* set(const GRState* st, typename GRStateTrait<T>::data_type D) {
-    return addGDM(st, GRStateTrait<T>::GDMIndex(),
-                  GRStateTrait<T>::MakeVoidPtr(D));
+  const ProgramState *set(const ProgramState *st, typename ProgramStateTrait<T>::data_type D) {
+    return addGDM(st, ProgramStateTrait<T>::GDMIndex(),
+                  ProgramStateTrait<T>::MakeVoidPtr(D));
   }
 
   template<typename T>
-  const GRState* set(const GRState* st,
-                     typename GRStateTrait<T>::key_type K,
-                     typename GRStateTrait<T>::value_type V,
-                     typename GRStateTrait<T>::context_type C) {
+  const ProgramState *set(const ProgramState *st,
+                     typename ProgramStateTrait<T>::key_type K,
+                     typename ProgramStateTrait<T>::value_type V,
+                     typename ProgramStateTrait<T>::context_type C) {
 
-    return addGDM(st, GRStateTrait<T>::GDMIndex(),
-     GRStateTrait<T>::MakeVoidPtr(GRStateTrait<T>::Set(st->get<T>(), K, V, C)));
+    return addGDM(st, ProgramStateTrait<T>::GDMIndex(),
+     ProgramStateTrait<T>::MakeVoidPtr(ProgramStateTrait<T>::Set(st->get<T>(), K, V, C)));
   }
 
   template <typename T>
-  const GRState* add(const GRState* st,
-                     typename GRStateTrait<T>::key_type K,
-                     typename GRStateTrait<T>::context_type C) {
-    return addGDM(st, GRStateTrait<T>::GDMIndex(),
-        GRStateTrait<T>::MakeVoidPtr(GRStateTrait<T>::Add(st->get<T>(), K, C)));
+  const ProgramState *add(const ProgramState *st,
+                     typename ProgramStateTrait<T>::key_type K,
+                     typename ProgramStateTrait<T>::context_type C) {
+    return addGDM(st, ProgramStateTrait<T>::GDMIndex(),
+        ProgramStateTrait<T>::MakeVoidPtr(ProgramStateTrait<T>::Add(st->get<T>(), K, C)));
   }
 
   template <typename T>
-  const GRState* remove(const GRState* st,
-                        typename GRStateTrait<T>::key_type K,
-                        typename GRStateTrait<T>::context_type C) {
+  const ProgramState *remove(const ProgramState *st,
+                        typename ProgramStateTrait<T>::key_type K,
+                        typename ProgramStateTrait<T>::context_type C) {
 
-    return addGDM(st, GRStateTrait<T>::GDMIndex(),
-     GRStateTrait<T>::MakeVoidPtr(GRStateTrait<T>::Remove(st->get<T>(), K, C)));
+    return addGDM(st, ProgramStateTrait<T>::GDMIndex(),
+     ProgramStateTrait<T>::MakeVoidPtr(ProgramStateTrait<T>::Remove(st->get<T>(), K, C)));
   }
 
   template <typename T>
-  const GRState *remove(const GRState *st) {
-    return removeGDM(st, GRStateTrait<T>::GDMIndex());
+  const ProgramState *remove(const ProgramState *st) {
+    return removeGDM(st, ProgramStateTrait<T>::GDMIndex());
   }
 
-  void* FindGDMContext(void* index,
-                       void* (*CreateContext)(llvm::BumpPtrAllocator&),
+  void *FindGDMContext(void *index,
+                       void *(*CreateContext)(llvm::BumpPtrAllocator&),
                        void  (*DeleteContext)(void*));
 
   template <typename T>
-  typename GRStateTrait<T>::context_type get_context() {
-    void* p = FindGDMContext(GRStateTrait<T>::GDMIndex(),
-                             GRStateTrait<T>::CreateContext,
-                             GRStateTrait<T>::DeleteContext);
+  typename ProgramStateTrait<T>::context_type get_context() {
+    void *p = FindGDMContext(ProgramStateTrait<T>::GDMIndex(),
+                             ProgramStateTrait<T>::CreateContext,
+                             ProgramStateTrait<T>::DeleteContext);
 
-    return GRStateTrait<T>::MakeContext(p);
+    return ProgramStateTrait<T>::MakeContext(p);
   }
 
-  const llvm::APSInt* getSymVal(const GRState* St, SymbolRef sym) {
+  const llvm::APSInt* getSymVal(const ProgramState *St, SymbolRef sym) {
     return ConstraintMgr->getSymVal(St, sym);
   }
 
-  void EndPath(const GRState* St) {
+  void EndPath(const ProgramState *St) {
     ConstraintMgr->EndPath(St);
   }
 };
 
 
 //===----------------------------------------------------------------------===//
-// Out-of-line method definitions for GRState.
+// Out-of-line method definitions for ProgramState.
 //===----------------------------------------------------------------------===//
 
-inline const VarRegion* GRState::getRegion(const VarDecl *D,
+inline const VarRegion* ProgramState::getRegion(const VarDecl *D,
                                            const LocationContext *LC) const {
   return getStateManager().getRegionManager().getVarRegion(D, LC);
 }
 
-inline const GRState *GRState::assume(DefinedOrUnknownSVal Cond,
+inline const ProgramState *ProgramState::assume(DefinedOrUnknownSVal Cond,
                                       bool Assumption) const {
   if (Cond.isUnknown())
     return this;
@@ -644,8 +639,8 @@ inline const GRState *GRState::assume(DefinedOrUnknownSVal Cond,
                                                  Assumption);
 }
   
-inline std::pair<const GRState*, const GRState*>
-GRState::assume(DefinedOrUnknownSVal Cond) const {
+inline std::pair<const ProgramState*, const ProgramState*>
+ProgramState::assume(DefinedOrUnknownSVal Cond) const {
   if (Cond.isUnknown())
     return std::make_pair(this, this);
   
@@ -653,48 +648,48 @@ GRState::assume(DefinedOrUnknownSVal Cond) const {
                                                      cast<DefinedSVal>(Cond));
 }
 
-inline const GRState *GRState::bindLoc(SVal LV, SVal V) const {
+inline const ProgramState *ProgramState::bindLoc(SVal LV, SVal V) const {
   return !isa<Loc>(LV) ? this : bindLoc(cast<Loc>(LV), V);
 }
 
-inline Loc GRState::getLValue(const VarDecl* VD,
+inline Loc ProgramState::getLValue(const VarDecl *VD,
                                const LocationContext *LC) const {
   return getStateManager().StoreMgr->getLValueVar(VD, LC);
 }
 
-inline Loc GRState::getLValue(const StringLiteral *literal) const {
+inline Loc ProgramState::getLValue(const StringLiteral *literal) const {
   return getStateManager().StoreMgr->getLValueString(literal);
 }
 
-inline Loc GRState::getLValue(const CompoundLiteralExpr *literal,
+inline Loc ProgramState::getLValue(const CompoundLiteralExpr *literal,
                                const LocationContext *LC) const {
   return getStateManager().StoreMgr->getLValueCompoundLiteral(literal, LC);
 }
 
-inline SVal GRState::getLValue(const ObjCIvarDecl *D, SVal Base) const {
+inline SVal ProgramState::getLValue(const ObjCIvarDecl *D, SVal Base) const {
   return getStateManager().StoreMgr->getLValueIvar(D, Base);
 }
 
-inline SVal GRState::getLValue(const FieldDecl* D, SVal Base) const {
+inline SVal ProgramState::getLValue(const FieldDecl *D, SVal Base) const {
   return getStateManager().StoreMgr->getLValueField(D, Base);
 }
 
-inline SVal GRState::getLValue(QualType ElementType, SVal Idx, SVal Base) const{
+inline SVal ProgramState::getLValue(QualType ElementType, SVal Idx, SVal Base) const{
   if (NonLoc *N = dyn_cast<NonLoc>(&Idx))
     return getStateManager().StoreMgr->getLValueElement(ElementType, *N, Base);
   return UnknownVal();
 }
 
-inline const llvm::APSInt *GRState::getSymVal(SymbolRef sym) const {
+inline const llvm::APSInt *ProgramState::getSymVal(SymbolRef sym) const {
   return getStateManager().getSymVal(this, sym);
 }
 
-inline SVal GRState::getSVal(const Stmt* Ex, bool useOnlyDirectBindings) const{
+inline SVal ProgramState::getSVal(const Stmt *Ex, bool useOnlyDirectBindings) const{
   return Env.getSVal(Ex, *getStateManager().svalBuilder,
-		     useOnlyDirectBindings);
+                     useOnlyDirectBindings);
 }
 
-inline SVal GRState::getSValAsScalarOrLoc(const Stmt *S) const {
+inline SVal ProgramState::getSValAsScalarOrLoc(const Stmt *S) const {
   if (const Expr *Ex = dyn_cast<Expr>(S)) {
     QualType T = Ex->getType();
     if (Ex->isLValue() || Loc::isLocType(T) || T->isIntegerType())
@@ -704,88 +699,114 @@ inline SVal GRState::getSValAsScalarOrLoc(const Stmt *S) const {
   return UnknownVal();
 }
 
-inline SVal GRState::getRawSVal(Loc LV, QualType T) const {
+inline SVal ProgramState::getRawSVal(Loc LV, QualType T) const {
   return getStateManager().StoreMgr->Retrieve(getStore(), LV, T);
 }
 
-inline SVal GRState::getSVal(const MemRegion* R) const {
+inline SVal ProgramState::getSVal(const MemRegion* R) const {
   return getStateManager().StoreMgr->Retrieve(getStore(), loc::MemRegionVal(R));
 }
 
-inline BasicValueFactory &GRState::getBasicVals() const {
+inline BasicValueFactory &ProgramState::getBasicVals() const {
   return getStateManager().getBasicVals();
 }
 
-inline SymbolManager &GRState::getSymbolManager() const {
+inline SymbolManager &ProgramState::getSymbolManager() const {
   return getStateManager().getSymbolManager();
 }
 
 template<typename T>
-const GRState *GRState::add(typename GRStateTrait<T>::key_type K) const {
+const ProgramState *ProgramState::add(typename ProgramStateTrait<T>::key_type K) const {
   return getStateManager().add<T>(this, K, get_context<T>());
 }
 
 template <typename T>
-typename GRStateTrait<T>::context_type GRState::get_context() const {
+typename ProgramStateTrait<T>::context_type ProgramState::get_context() const {
   return getStateManager().get_context<T>();
 }
 
 template<typename T>
-const GRState *GRState::remove(typename GRStateTrait<T>::key_type K) const {
+const ProgramState *ProgramState::remove(typename ProgramStateTrait<T>::key_type K) const {
   return getStateManager().remove<T>(this, K, get_context<T>());
 }
 
 template<typename T>
-const GRState *GRState::remove(typename GRStateTrait<T>::key_type K,
-                               typename GRStateTrait<T>::context_type C) const {
+const ProgramState *ProgramState::remove(typename ProgramStateTrait<T>::key_type K,
+                               typename ProgramStateTrait<T>::context_type C) const {
   return getStateManager().remove<T>(this, K, C);
 }
 
 template <typename T>
-const GRState *GRState::remove() const {
+const ProgramState *ProgramState::remove() const {
   return getStateManager().remove<T>(this);
 }
 
 template<typename T>
-const GRState *GRState::set(typename GRStateTrait<T>::data_type D) const {
+const ProgramState *ProgramState::set(typename ProgramStateTrait<T>::data_type D) const {
   return getStateManager().set<T>(this, D);
 }
 
 template<typename T>
-const GRState *GRState::set(typename GRStateTrait<T>::key_type K,
-                            typename GRStateTrait<T>::value_type E) const {
+const ProgramState *ProgramState::set(typename ProgramStateTrait<T>::key_type K,
+                            typename ProgramStateTrait<T>::value_type E) const {
   return getStateManager().set<T>(this, K, E, get_context<T>());
 }
 
 template<typename T>
-const GRState *GRState::set(typename GRStateTrait<T>::key_type K,
-                            typename GRStateTrait<T>::value_type E,
-                            typename GRStateTrait<T>::context_type C) const {
+const ProgramState *ProgramState::set(typename ProgramStateTrait<T>::key_type K,
+                            typename ProgramStateTrait<T>::value_type E,
+                            typename ProgramStateTrait<T>::context_type C) const {
   return getStateManager().set<T>(this, K, E, C);
 }
 
 template <typename CB>
-CB GRState::scanReachableSymbols(SVal val) const {
+CB ProgramState::scanReachableSymbols(SVal val) const {
   CB cb(this);
   scanReachableSymbols(val, cb);
   return cb;
 }
   
 template <typename CB>
-CB GRState::scanReachableSymbols(const SVal *beg, const SVal *end) const {
+CB ProgramState::scanReachableSymbols(const SVal *beg, const SVal *end) const {
   CB cb(this);
   scanReachableSymbols(beg, end, cb);
   return cb;
 }
 
 template <typename CB>
-CB GRState::scanReachableSymbols(const MemRegion * const *beg,
+CB ProgramState::scanReachableSymbols(const MemRegion * const *beg,
                                  const MemRegion * const *end) const {
   CB cb(this);
   scanReachableSymbols(beg, end, cb);
   return cb;
 }
 
+/// \class ScanReachableSymbols
+/// A Utility class that allows to visit the reachable symbols using a custom
+/// SymbolVisitor.
+class ScanReachableSymbols : public SubRegionMap::Visitor  {
+  typedef llvm::DenseMap<const void*, unsigned> VisitedItems;
+
+  VisitedItems visited;
+  const ProgramState *state;
+  SymbolVisitor &visitor;
+  llvm::OwningPtr<SubRegionMap> SRM;
+public:
+
+  ScanReachableSymbols(const ProgramState *st, SymbolVisitor& v)
+    : state(st), visitor(v) {}
+
+  bool scan(nonloc::CompoundVal val);
+  bool scan(SVal val);
+  bool scan(const MemRegion *R);
+  bool scan(const SymExpr *sym);
+
+  // From SubRegionMap::Visitor.
+  bool Visit(const MemRegion* Parent, const MemRegion* SubRegion) {
+    return scan(SubRegion);
+  }
+};
+
 } // end GR namespace
 
 } // end clang namespace
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h
similarity index 70%
rename from contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h
rename to contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h
index de7b8684066f..b80d494b88cb 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h
@@ -1,4 +1,4 @@
-//==- GRStateTrait.h - Partial implementations of GRStateTrait -----*- C++ -*-//
+//ProgramStateTrait.h - Partial implementations of ProgramStateTrait -*- C++ -*-
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -8,14 +8,15 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines partial implementations of template specializations of
-//  the class GRStateTrait<>.  GRStateTrait<> is used by GRState to implement
-//  set/get methods for mapulating a GRState's generic data map.
+//  the class ProgramStateTrait<>.  ProgramStateTrait<> is used by ProgramState 
+//  to implement set/get methods for mapulating a ProgramState's
+// generic data map.
 //
 //===----------------------------------------------------------------------===//
 
 
-#ifndef LLVM_CLANG_GR_GRSTATETRAIT_H
-#define LLVM_CLANG_GR_GRSTATETRAIT_H
+#ifndef LLVM_CLANG_GR_PROGRAMSTATETRAIT_H
+#define LLVM_CLANG_GR_PROGRAMSTATETRAIT_H
 
 namespace llvm {
   class BumpPtrAllocator;
@@ -28,22 +29,22 @@ namespace llvm {
 namespace clang {
 
 namespace ento {
-  template <typename T> struct GRStatePartialTrait;
+  template <typename T> struct ProgramStatePartialTrait;
 
   // Partial-specialization for ImmutableMap.
 
   template <typename Key, typename Data, typename Info>
-  struct GRStatePartialTrait< llvm::ImmutableMap<Key,Data,Info> > {
+  struct ProgramStatePartialTrait< llvm::ImmutableMap<Key,Data,Info> > {
     typedef llvm::ImmutableMap<Key,Data,Info> data_type;
     typedef typename data_type::Factory&      context_type;
     typedef Key                               key_type;
     typedef Data                              value_type;
     typedef const value_type*                 lookup_type;
 
-    static inline data_type MakeData(void* const* p) {
+    static inline data_type MakeData(void *const* p) {
       return p ? data_type((typename data_type::TreeTy*) *p) : data_type(0);
     }
-    static inline void* MakeVoidPtr(data_type B) {
+    static inline void *MakeVoidPtr(data_type B) {
       return B.getRoot();
     }
     static lookup_type Lookup(data_type B, key_type K) {
@@ -57,15 +58,15 @@ namespace ento {
       return F.remove(B, K);
     }
 
-    static inline context_type MakeContext(void* p) {
+    static inline context_type MakeContext(void *p) {
       return *((typename data_type::Factory*) p);
     }
 
-    static void* CreateContext(llvm::BumpPtrAllocator& Alloc) {
+    static void *CreateContext(llvm::BumpPtrAllocator& Alloc) {
       return new typename data_type::Factory(Alloc);
     }
 
-    static void DeleteContext(void* Ctx) {
+    static void DeleteContext(void *Ctx) {
       delete (typename data_type::Factory*) Ctx;
     }
   };
@@ -74,16 +75,16 @@ namespace ento {
   // Partial-specialization for ImmutableSet.
 
   template <typename Key, typename Info>
-  struct GRStatePartialTrait< llvm::ImmutableSet<Key,Info> > {
+  struct ProgramStatePartialTrait< llvm::ImmutableSet<Key,Info> > {
     typedef llvm::ImmutableSet<Key,Info>      data_type;
     typedef typename data_type::Factory&      context_type;
     typedef Key                               key_type;
 
-    static inline data_type MakeData(void* const* p) {
+    static inline data_type MakeData(void *const* p) {
       return p ? data_type((typename data_type::TreeTy*) *p) : data_type(0);
     }
 
-    static inline void* MakeVoidPtr(data_type B) {
+    static inline void *MakeVoidPtr(data_type B) {
       return B.getRoot();
     }
 
@@ -99,15 +100,15 @@ namespace ento {
       return B.contains(K);
     }
 
-    static inline context_type MakeContext(void* p) {
+    static inline context_type MakeContext(void *p) {
       return *((typename data_type::Factory*) p);
     }
 
-    static void* CreateContext(llvm::BumpPtrAllocator& Alloc) {
+    static void *CreateContext(llvm::BumpPtrAllocator& Alloc) {
       return new typename data_type::Factory(Alloc);
     }
 
-    static void DeleteContext(void* Ctx) {
+    static void DeleteContext(void *Ctx) {
       delete (typename data_type::Factory*) Ctx;
     }
   };
@@ -115,7 +116,7 @@ namespace ento {
   // Partial-specialization for ImmutableList.
 
   template <typename T>
-  struct GRStatePartialTrait< llvm::ImmutableList<T> > {
+  struct ProgramStatePartialTrait< llvm::ImmutableList<T> > {
     typedef llvm::ImmutableList<T>            data_type;
     typedef T                                 key_type;
     typedef typename data_type::Factory&      context_type;
@@ -128,33 +129,33 @@ namespace ento {
       return L.contains(K);
     }
 
-    static inline data_type MakeData(void* const* p) {
+    static inline data_type MakeData(void *const* p) {
       return p ? data_type((const llvm::ImmutableListImpl<T>*) *p)
                : data_type(0);
     }
 
-    static inline void* MakeVoidPtr(data_type D) {
+    static inline void *MakeVoidPtr(data_type D) {
       return  (void*) D.getInternalPointer();
     }
 
-    static inline context_type MakeContext(void* p) {
+    static inline context_type MakeContext(void *p) {
       return *((typename data_type::Factory*) p);
     }
 
-    static void* CreateContext(llvm::BumpPtrAllocator& Alloc) {
+    static void *CreateContext(llvm::BumpPtrAllocator& Alloc) {
       return new typename data_type::Factory(Alloc);
     }
 
-    static void DeleteContext(void* Ctx) {
+    static void DeleteContext(void *Ctx) {
       delete (typename data_type::Factory*) Ctx;
     }
   };
   
   // Partial specialization for bool.
-  template <> struct GRStatePartialTrait<bool> {
+  template <> struct ProgramStatePartialTrait<bool> {
     typedef bool data_type;
 
-    static inline data_type MakeData(void* const* p) {
+    static inline data_type MakeData(void *const* p) {
       return p ? (data_type) (uintptr_t) *p
                : data_type();
     }
@@ -164,10 +165,10 @@ namespace ento {
   };
   
   // Partial specialization for unsigned.
-  template <> struct GRStatePartialTrait<unsigned> {
+  template <> struct ProgramStatePartialTrait<unsigned> {
     typedef unsigned data_type;
 
-    static inline data_type MakeData(void* const* p) {
+    static inline data_type MakeData(void *const* p) {
       return p ? (data_type) (uintptr_t) *p
                : data_type();
     }
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h
index 65eabea24077..17233e194e6d 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h
@@ -25,7 +25,7 @@ namespace clang {
 
 namespace ento {
 
-class GRState;
+class ProgramState;
 
 class SValBuilder {
 protected:
@@ -40,7 +40,7 @@ class SValBuilder {
   /// Manages the creation of memory regions.
   MemRegionManager MemMgr;
 
-  GRStateManager &StateMgr;
+  ProgramStateManager &StateMgr;
 
   /// The scalar type to use for array indices.
   const QualType ArrayIndexTy;
@@ -56,7 +56,7 @@ class SValBuilder {
 
 public:
   SValBuilder(llvm::BumpPtrAllocator &alloc, ASTContext &context,
-              GRStateManager &stateMgr)
+              ProgramStateManager &stateMgr)
     : Context(context), BasicVals(context, alloc),
       SymMgr(context, BasicVals, alloc),
       MemMgr(context, alloc),
@@ -72,29 +72,29 @@ class SValBuilder {
 
   virtual SVal evalComplement(NonLoc val) = 0;
 
-  virtual SVal evalBinOpNN(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpNN(const ProgramState *state, BinaryOperator::Opcode op,
                            NonLoc lhs, NonLoc rhs, QualType resultTy) = 0;
 
-  virtual SVal evalBinOpLL(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpLL(const ProgramState *state, BinaryOperator::Opcode op,
                            Loc lhs, Loc rhs, QualType resultTy) = 0;
 
-  virtual SVal evalBinOpLN(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpLN(const ProgramState *state, BinaryOperator::Opcode op,
                            Loc lhs, NonLoc rhs, QualType resultTy) = 0;
 
   /// getKnownValue - evaluates a given SVal. If the SVal has only one possible
   ///  (integer) value, that value is returned. Otherwise, returns NULL.
-  virtual const llvm::APSInt *getKnownValue(const GRState *state, SVal val) = 0;
+  virtual const llvm::APSInt *getKnownValue(const ProgramState *state, SVal val) = 0;
   
-  SVal evalBinOp(const GRState *state, BinaryOperator::Opcode op,
+  SVal evalBinOp(const ProgramState *state, BinaryOperator::Opcode op,
                  SVal lhs, SVal rhs, QualType type);
   
-  DefinedOrUnknownSVal evalEQ(const GRState *state, DefinedOrUnknownSVal lhs,
+  DefinedOrUnknownSVal evalEQ(const ProgramState *state, DefinedOrUnknownSVal lhs,
                               DefinedOrUnknownSVal rhs);
 
   ASTContext &getContext() { return Context; }
   const ASTContext &getContext() const { return Context; }
 
-  GRStateManager &getStateManager() { return StateMgr; }
+  ProgramStateManager &getStateManager() { return StateMgr; }
   
   QualType getConditionType() const {
     return  getContext().IntTy;
@@ -115,14 +115,14 @@ class SValBuilder {
 
   // Forwarding methods to SymbolManager.
 
-  const SymbolConjured* getConjuredSymbol(const Stmt* stmt, QualType type,
+  const SymbolConjured* getConjuredSymbol(const Stmt *stmt, QualType type,
                                           unsigned visitCount,
-                                          const void* symbolTag = 0) {
+                                          const void *symbolTag = 0) {
     return SymMgr.getConjuredSymbol(stmt, type, visitCount, symbolTag);
   }
 
-  const SymbolConjured* getConjuredSymbol(const Expr* expr, unsigned visitCount,
-                                          const void* symbolTag = 0) {
+  const SymbolConjured* getConjuredSymbol(const Expr *expr, unsigned visitCount,
+                                          const void *symbolTag = 0) {
     return SymMgr.getConjuredSymbol(expr, visitCount, symbolTag);
   }
 
@@ -130,7 +130,7 @@ class SValBuilder {
   DefinedOrUnknownSVal makeZeroVal(QualType type);
 
   /// getRegionValueSymbolVal - make a unique symbol for value of region.
-  DefinedOrUnknownSVal getRegionValueSymbolVal(const TypedRegion *region);
+  DefinedOrUnknownSVal getRegionValueSymbolVal(const TypedValueRegion *region);
 
   DefinedOrUnknownSVal getConjuredSymbolVal(const void *symbolTag,
                                             const Expr *expr, unsigned count);
@@ -139,7 +139,7 @@ class SValBuilder {
                                             unsigned count);
 
   DefinedOrUnknownSVal getDerivedRegionValueSymbolVal(
-      SymbolRef parentSymbol, const TypedRegion *region);
+      SymbolRef parentSymbol, const TypedValueRegion *region);
 
   DefinedSVal getMetadataSymbolVal(
       const void *symbolTag, const MemRegion *region,
@@ -154,7 +154,8 @@ class SValBuilder {
     return nonloc::CompoundVal(BasicVals.getCompoundValData(type, vals));
   }
 
-  NonLoc makeLazyCompoundVal(const StoreRef &store, const TypedRegion *region) {
+  NonLoc makeLazyCompoundVal(const StoreRef &store, 
+                             const TypedValueRegion *region) {
     return nonloc::LazyCompoundVal(
         BasicVals.getLazyCompoundValData(store, region));
   }
@@ -254,7 +255,7 @@ class SValBuilder {
 
 SValBuilder* createSimpleSValBuilder(llvm::BumpPtrAllocator &alloc,
                                      ASTContext &context,
-                                     GRStateManager &stateMgr);
+                                     ProgramStateManager &stateMgr);
 
 } // end GR namespace
 
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
index 0d430794e76c..5827b0042a5d 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
@@ -15,14 +15,10 @@
 #ifndef LLVM_CLANG_GR_RVALUE_H
 #define LLVM_CLANG_GR_RVALUE_H
 
+#include "clang/Basic/LLVM.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/ADT/ImmutableList.h"
 
-namespace llvm {
-  class raw_ostream;
-}
-
 //==------------------------------------------------------------------------==//
 //  Base SVal types.
 //==------------------------------------------------------------------------==//
@@ -33,12 +29,12 @@ namespace ento {
 
 class CompoundValData;
 class LazyCompoundValData;
-class GRState;
+class ProgramState;
 class BasicValueFactory;
 class MemRegion;
 class TypedRegion;
 class MemRegionManager;
-class GRStateManager;
+class ProgramStateManager;
 class SValBuilder;
 
 /// SVal - This represents a symbolic expression, which can be either
@@ -57,16 +53,16 @@ class SVal {
   enum { BaseBits = 2, BaseMask = 0x3 };
 
 protected:
-  const void* Data;
+  const void *Data;
 
   /// The lowest 2 bits are a BaseKind (0 -- 3).
   ///  The higher bits are an unsigned "kind" value.
   unsigned Kind;
 
-  explicit SVal(const void* d, bool isLoc, unsigned ValKind)
+  explicit SVal(const void *d, bool isLoc, unsigned ValKind)
   : Data(d), Kind((isLoc ? LocKind : NonLocKind) | (ValKind << BaseBits)) {}
 
-  explicit SVal(BaseKind k, const void* D = NULL)
+  explicit SVal(BaseKind k, const void *D = NULL)
     : Data(D), Kind(k) {}
 
 public:
@@ -74,7 +70,7 @@ class SVal {
   ~SVal() {}
 
   /// BufferTy - A temporary buffer to hold a set of SVals.
-  typedef llvm::SmallVector<SVal,5> BufferTy;
+  typedef SmallVector<SVal,5> BufferTy;
 
   inline unsigned getRawKind() const { return Kind; }
   inline BaseKind getBaseKind() const { return (BaseKind) (Kind & BaseMask); }
@@ -123,7 +119,7 @@ class SVal {
   /// getAsFunctionDecl - If this SVal is a MemRegionVal and wraps a
   /// CodeTextRegion wrapping a FunctionDecl, return that FunctionDecl.
   /// Otherwise return 0.
-  const FunctionDecl* getAsFunctionDecl() const;
+  const FunctionDecl *getAsFunctionDecl() const;
 
   /// getAsLocSymbol - If this SVal is a location (subclasses Loc) and
   ///  wraps a symbol, return that SymbolRef.  Otherwise return NULL.
@@ -142,22 +138,22 @@ class SVal {
 
   const MemRegion *getAsRegion() const;
 
-  void dumpToStream(llvm::raw_ostream& OS) const;
+  void dumpToStream(raw_ostream &OS) const;
   void dump() const;
 
   // Iterators.
   class symbol_iterator {
-    llvm::SmallVector<const SymExpr*, 5> itr;
+    SmallVector<const SymExpr*, 5> itr;
     void expand();
   public:
     symbol_iterator() {}
-    symbol_iterator(const SymExpr* SE);
+    symbol_iterator(const SymExpr *SE);
 
-    symbol_iterator& operator++();
+    symbol_iterator &operator++();
     SymbolRef operator*();
 
-    bool operator==(const symbol_iterator& X) const;
-    bool operator!=(const symbol_iterator& X) const;
+    bool operator==(const symbol_iterator &X) const;
+    bool operator!=(const symbol_iterator &X) const;
   };
 
   symbol_iterator symbol_begin() const {
@@ -178,13 +174,13 @@ class SVal {
 class UndefinedVal : public SVal {
 public:
   UndefinedVal() : SVal(UndefinedKind) {}
-  UndefinedVal(const void* D) : SVal(UndefinedKind, D) {}
+  UndefinedVal(const void *D) : SVal(UndefinedKind, D) {}
 
   static inline bool classof(const SVal* V) {
     return V->getBaseKind() == UndefinedKind;
   }
 
-  const void* getData() const { return Data; }
+  const void *getData() const { return Data; }
 };
 
 class DefinedOrUnknownSVal : public SVal {
@@ -195,7 +191,7 @@ class DefinedOrUnknownSVal : public SVal {
   bool isValid() const;
   
 protected:
-  explicit DefinedOrUnknownSVal(const void* d, bool isLoc, unsigned ValKind)
+  explicit DefinedOrUnknownSVal(const void *d, bool isLoc, unsigned ValKind)
     : SVal(d, isLoc, ValKind) {}
   
   explicit DefinedOrUnknownSVal(BaseKind k, void *D = NULL)
@@ -225,7 +221,7 @@ class DefinedSVal : public DefinedOrUnknownSVal {
   bool isUnknownOrUndef() const;
   bool isValid() const;  
 protected:
-  explicit DefinedSVal(const void* d, bool isLoc, unsigned ValKind)
+  explicit DefinedSVal(const void *d, bool isLoc, unsigned ValKind)
     : DefinedOrUnknownSVal(d, isLoc, ValKind) {}
 public:
   // Implement isa<T> support.
@@ -236,11 +232,11 @@ class DefinedSVal : public DefinedOrUnknownSVal {
 
 class NonLoc : public DefinedSVal {
 protected:
-  explicit NonLoc(unsigned SubKind, const void* d)
+  explicit NonLoc(unsigned SubKind, const void *d)
     : DefinedSVal(d, false, SubKind) {}
 
 public:
-  void dumpToStream(llvm::raw_ostream& Out) const;
+  void dumpToStream(raw_ostream &Out) const;
 
   // Implement isa<T> support.
   static inline bool classof(const SVal* V) {
@@ -250,11 +246,11 @@ class NonLoc : public DefinedSVal {
 
 class Loc : public DefinedSVal {
 protected:
-  explicit Loc(unsigned SubKind, const void* D)
+  explicit Loc(unsigned SubKind, const void *D)
   : DefinedSVal(const_cast<void*>(D), true, SubKind) {}
 
 public:
-  void dumpToStream(llvm::raw_ostream& Out) const;
+  void dumpToStream(raw_ostream &Out) const;
 
   Loc(const Loc& X) : DefinedSVal(X.Data, true, X.getSubKind()) {}
 
@@ -533,7 +529,7 @@ class ObjCPropRef : public Loc {
 } // end clang namespace
 
 namespace llvm {
-static inline llvm::raw_ostream& operator<<(llvm::raw_ostream& os,
+static inline raw_ostream &operator<<(raw_ostream &os,
                                             clang::ento::SVal V) {
   V.dumpToStream(os);
   return os;
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h
index cdbdf64515b4..a688d7f21979 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h
@@ -29,20 +29,20 @@ class StackFrameContext;
 
 namespace ento {
 
-class GRState;
-class GRStateManager;
+class ProgramState;
+class ProgramStateManager;
 class SubRegionMap;
 
 class StoreManager {
 protected:
   SValBuilder &svalBuilder;
-  GRStateManager &StateMgr;
+  ProgramStateManager &StateMgr;
 
   /// MRMgr - Manages region objects associated with this StoreManager.
   MemRegionManager &MRMgr;
   ASTContext &Ctx;
 
-  StoreManager(GRStateManager &stateMgr);
+  StoreManager(ProgramStateManager &stateMgr);
 
 public:
   virtual ~StoreManager() {}
@@ -60,7 +60,7 @@ class StoreManager {
   /// \param[in] state The analysis state.
   /// \param[in] loc The symbolic memory location.
   /// \param[in] val The value to bind to location \c loc.
-  /// \return A pointer to a GRState object that contains the same bindings as
+  /// \return A pointer to a ProgramState object that contains the same bindings as
   ///   \c state with the addition of having the value specified by \c val bound
   ///   to the location given for \c loc.
   virtual StoreRef Bind(Store store, Loc loc, SVal val) = 0;
@@ -73,7 +73,7 @@ class StoreManager {
   ///  for the compound literal and 'BegInit' and 'EndInit' represent an
   ///  array of initializer values.
   virtual StoreRef BindCompoundLiteral(Store store,
-                                       const CompoundLiteralExpr* cl,
+                                       const CompoundLiteralExpr *cl,
                                        const LocationContext *LC, SVal v) = 0;
 
   /// getInitialStore - Returns the initial "empty" store representing the
@@ -97,16 +97,16 @@ class StoreManager {
     return svalBuilder.makeLoc(MRMgr.getStringRegion(S));
   }
 
-  Loc getLValueCompoundLiteral(const CompoundLiteralExpr* CL,
+  Loc getLValueCompoundLiteral(const CompoundLiteralExpr *CL,
                                const LocationContext *LC) {
     return loc::MemRegionVal(MRMgr.getCompoundLiteralRegion(CL, LC));
   }
 
-  virtual SVal getLValueIvar(const ObjCIvarDecl* decl, SVal base) {
+  virtual SVal getLValueIvar(const ObjCIvarDecl *decl, SVal base) {
     return getLValueFieldOrIvar(decl, base);
   }
 
-  virtual SVal getLValueField(const FieldDecl* D, SVal Base) {
+  virtual SVal getLValueField(const FieldDecl *D, SVal Base) {
     return getLValueFieldOrIvar(D, Base);
   }
 
@@ -114,7 +114,7 @@ class StoreManager {
 
   // FIXME: This should soon be eliminated altogether; clients should deal with
   // region extents directly.
-  virtual DefinedOrUnknownSVal getSizeInElements(const GRState *state, 
+  virtual DefinedOrUnknownSVal getSizeInElements(const ProgramState *state, 
                                                  const MemRegion *region,
                                                  QualType EleTy) {
     return UnknownVal();
@@ -130,12 +130,12 @@ class StoreManager {
   }
 
   class CastResult {
-    const GRState *state;
+    const ProgramState *state;
     const MemRegion *region;
   public:
-    const GRState *getState() const { return state; }
+    const ProgramState *getState() const { return state; }
     const MemRegion* getRegion() const { return region; }
-    CastResult(const GRState *s, const MemRegion* r = 0) : state(s), region(r){}
+    CastResult(const ProgramState *s, const MemRegion* r = 0) : state(s), region(r){}
   };
 
   const ElementRegion *GetElementZeroRegion(const MemRegion *R, QualType T);
@@ -146,13 +146,15 @@ class StoreManager {
   const MemRegion *castRegion(const MemRegion *region, QualType CastToTy);
 
   virtual StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
-                                      SymbolReaper& SymReaper,
-                      llvm::SmallVectorImpl<const MemRegion*>& RegionRoots) = 0;
+                                      SymbolReaper& SymReaper) = 0;
 
   virtual StoreRef BindDecl(Store store, const VarRegion *VR, SVal initVal) = 0;
 
   virtual StoreRef BindDeclWithNoInit(Store store, const VarRegion *VR) = 0;
   
+  virtual bool includedInBindings(Store store,
+                                  const MemRegion *region) const = 0;
+  
   /// If the StoreManager supports it, increment the reference count of
   /// the specified Store object.
   virtual void incrementReferenceCount(Store store) {}
@@ -163,7 +165,7 @@ class StoreManager {
   virtual void decrementReferenceCount(Store store) {}
 
   typedef llvm::DenseSet<SymbolRef> InvalidatedSymbols;
-  typedef llvm::SmallVector<const MemRegion *, 8> InvalidatedRegions;
+  typedef SmallVector<const MemRegion *, 8> InvalidatedRegions;
 
   /// invalidateRegions - Clears out the specified regions from the store,
   ///  marking their values as unknown. Depending on the store, this may also
@@ -185,19 +187,18 @@ class StoreManager {
   ///   even if they do not currently have bindings. Pass \c NULL if this
   ///   information will not be used.
   virtual StoreRef invalidateRegions(Store store,
-                                     const MemRegion * const *Begin,
-                                     const MemRegion * const *End,
+                                     ArrayRef<const MemRegion *> Regions,
                                      const Expr *E, unsigned Count,
                                      InvalidatedSymbols &IS,
                                      bool invalidateGlobals,
-                                     InvalidatedRegions *Regions) = 0;
+                                     InvalidatedRegions *Invalidated) = 0;
 
   /// enterStackFrame - Let the StoreManager to do something when execution
   /// engine is about to execute into a callee.
-  virtual StoreRef enterStackFrame(const GRState *state,
+  virtual StoreRef enterStackFrame(const ProgramState *state,
                                    const StackFrameContext *frame);
 
-  virtual void print(Store store, llvm::raw_ostream& Out,
+  virtual void print(Store store, raw_ostream &Out,
                      const char* nl, const char *sep) = 0;
 
   class BindingsHandler {
@@ -217,11 +218,11 @@ class StoreManager {
   /// CastRetrievedVal - Used by subclasses of StoreManager to implement
   ///  implicit casts that arise from loads from regions that are reinterpreted
   ///  as another region.
-  SVal CastRetrievedVal(SVal val, const TypedRegion *region, QualType castTy,
-                        bool performTestOnly = true);
+  SVal CastRetrievedVal(SVal val, const TypedValueRegion *region, 
+                        QualType castTy, bool performTestOnly = true);
 
 private:
-  SVal getLValueFieldOrIvar(const Decl* decl, SVal base);
+  SVal getLValueFieldOrIvar(const Decl *decl, SVal base);
 };
 
 
@@ -269,11 +270,9 @@ class SubRegionMap {
   virtual bool iterSubRegions(const MemRegion *region, Visitor& V) const = 0;
 };
 
-// FIXME: Do we need to pass GRStateManager anymore?
-StoreManager *CreateBasicStoreManager(GRStateManager& StMgr);
-StoreManager *CreateRegionStoreManager(GRStateManager& StMgr);
-StoreManager *CreateFieldsOnlyRegionStoreManager(GRStateManager& StMgr);
-StoreManager *CreateFlatStoreManager(GRStateManager &StMgr);
+// FIXME: Do we need to pass ProgramStateManager anymore?
+StoreManager *CreateRegionStoreManager(ProgramStateManager& StMgr);
+StoreManager *CreateFieldsOnlyRegionStoreManager(ProgramStateManager& StMgr);
 
 } // end GR namespace
 
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/StoreRef.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/StoreRef.h
index 0662eadc93c3..d5ba003a6915 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/StoreRef.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/StoreRef.h
@@ -23,7 +23,7 @@ namespace ento {
 ///  locations to values.  At a high-level, it represents the symbolic
 ///  memory model.  Different subclasses of StoreManager may choose
 ///  different types to represent the locations and values.
-typedef const void* Store;
+typedef const void *Store;
   
 class StoreManager;
   
@@ -44,6 +44,7 @@ class StoreRef {
   ~StoreRef();
   
   Store getStore() const { return store; }
+  const StoreManager &getStoreManager() const { return mgr; }
 };
 
 }}
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h
index 1f6ea3d8c732..ae212bcf5ddf 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h
@@ -30,8 +30,8 @@ template <typename PP> class GenericNodeBuilder;
 class AnalysisManager;
 class ExplodedNodeSet;
 class ExplodedNode;
-class GRState;
-class GRStateManager;
+class ProgramState;
+class ProgramStateManager;
 class BlockCounter;
 class StmtNodeBuilder;
 class BranchNodeBuilder;
@@ -46,11 +46,11 @@ class SubEngine {
 public:
   virtual ~SubEngine() {}
 
-  virtual const GRState* getInitialState(const LocationContext *InitLoc) = 0;
+  virtual const ProgramState *getInitialState(const LocationContext *InitLoc) = 0;
 
   virtual AnalysisManager &getAnalysisManager() = 0;
 
-  virtual GRStateManager &getStateManager() = 0;
+  virtual ProgramStateManager &getStateManager() = 0;
 
   /// Called by CoreEngine. Used to generate new successor
   /// nodes by processing the 'effects' of a block-level statement.
@@ -64,7 +64,7 @@ class SubEngine {
 
   /// Called by CoreEngine.  Used to generate successor
   ///  nodes by processing the 'effects' of a branch condition.
-  virtual void processBranch(const Stmt* Condition, const Stmt* Term,
+  virtual void processBranch(const Stmt *Condition, const Stmt *Term,
                              BranchNodeBuilder& builder) = 0;
 
   /// Called by CoreEngine.  Used to generate successor
@@ -87,28 +87,32 @@ class SubEngine {
 
   /// Called by ConstraintManager. Used to call checker-specific
   /// logic for handling assumptions on symbolic values.
-  virtual const GRState* processAssume(const GRState *state,
+  virtual const ProgramState *processAssume(const ProgramState *state,
                                        SVal cond, bool assumption) = 0;
 
-  /// wantsRegionChangeUpdate - Called by GRStateManager to determine if a
+  /// wantsRegionChangeUpdate - Called by ProgramStateManager to determine if a
   ///  region change should trigger a processRegionChanges update.
-  virtual bool wantsRegionChangeUpdate(const GRState* state) = 0;
+  virtual bool wantsRegionChangeUpdate(const ProgramState *state) = 0;
 
-  /// processRegionChanges - Called by GRStateManager whenever a change is made
+  /// processRegionChanges - Called by ProgramStateManager whenever a change is made
   ///  to the store. Used to update checkers that track region values.
-  virtual const GRState *
-  processRegionChanges(const GRState *state,
+  virtual const ProgramState *
+  processRegionChanges(const ProgramState *state,
                        const StoreManager::InvalidatedSymbols *invalidated,
-                       const MemRegion* const *Begin,
-                       const MemRegion* const *End) = 0;
+                       ArrayRef<const MemRegion *> ExplicitRegions,
+                       ArrayRef<const MemRegion *> Regions) = 0;
 
 
-  inline const GRState *
-  processRegionChange(const GRState* state,
+  inline const ProgramState *
+  processRegionChange(const ProgramState *state,
                       const MemRegion* MR) {
-    return processRegionChanges(state, 0, &MR, &MR+1);
+    return processRegionChanges(state, 0, MR, MR);
   }
 
+  /// printState - Called by ProgramStateManager to print checker-specific data.
+  virtual void printState(raw_ostream &Out, const ProgramState *State,
+                          const char *NL, const char *Sep) = 0;
+
   /// Called by CoreEngine when the analysis worklist is either empty or the
   //  maximum number of analysis steps have been reached.
   virtual void processEndWorklist(bool hasWorkRemaining) = 0;
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h
index ad173bb43536..0d6e18eb3cc8 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h
@@ -18,13 +18,15 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
 #include "clang/Analysis/AnalysisContext.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/StoreRef.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DenseMap.h"
 
 namespace llvm {
 class BumpPtrAllocator;
-class raw_ostream;
 }
 
 namespace clang {
@@ -35,15 +37,15 @@ namespace ento {
   class BasicValueFactory;
   class MemRegion;
   class SubRegion;
-  class TypedRegion;
+  class TypedValueRegion;
   class VarRegion;
 
 class SymExpr : public llvm::FoldingSetNode {
 public:
-  enum Kind { BEGIN_SYMBOLS,
-              RegionValueKind, ConjuredKind, DerivedKind, ExtentKind,
+  enum Kind { RegionValueKind, ConjuredKind, DerivedKind, ExtentKind,
               MetadataKind,
-              END_SYMBOLS,
+              BEGIN_SYMBOLS = RegionValueKind,
+              END_SYMBOLS = MetadataKind,
               SymIntKind, SymSymKind };
 private:
   Kind K;
@@ -58,7 +60,7 @@ class SymExpr : public llvm::FoldingSetNode {
 
   void dump() const;
 
-  virtual void dumpToStream(llvm::raw_ostream &os) const = 0;
+  virtual void dumpToStream(raw_ostream &os) const = 0;
 
   virtual QualType getType(ASTContext&) const = 0;
   virtual void Profile(llvm::FoldingSetNodeID& profile) = 0;
@@ -82,25 +84,26 @@ class SymbolData : public SymExpr {
   SymbolID getSymbolID() const { return Sym; }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     Kind k = SE->getKind();
-    return k > BEGIN_SYMBOLS && k < END_SYMBOLS;
+    return k >= BEGIN_SYMBOLS && k <= END_SYMBOLS;
   }
 };
 
 typedef const SymbolData* SymbolRef;
+typedef llvm::SmallVector<SymbolRef, 2> SymbolRefSmallVectorTy;
 
-// A symbol representing the value of a MemRegion.
+/// A symbol representing the value of a MemRegion.
 class SymbolRegionValue : public SymbolData {
-  const TypedRegion *R;
+  const TypedValueRegion *R;
 
 public:
-  SymbolRegionValue(SymbolID sym, const TypedRegion *r)
+  SymbolRegionValue(SymbolID sym, const TypedValueRegion *r)
     : SymbolData(RegionValueKind, sym), R(r) {}
 
-  const TypedRegion* getRegion() const { return R; }
+  const TypedValueRegion* getRegion() const { return R; }
 
-  static void Profile(llvm::FoldingSetNodeID& profile, const TypedRegion* R) {
+  static void Profile(llvm::FoldingSetNodeID& profile, const TypedValueRegion* R) {
     profile.AddInteger((unsigned) RegionValueKind);
     profile.AddPointer(R);
   }
@@ -109,39 +112,39 @@ class SymbolRegionValue : public SymbolData {
     Profile(profile, R);
   }
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   QualType getType(ASTContext&) const;
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == RegionValueKind;
   }
 };
 
-// A symbol representing the result of an expression.
+/// A symbol representing the result of an expression.
 class SymbolConjured : public SymbolData {
-  const Stmt* S;
+  const Stmt *S;
   QualType T;
   unsigned Count;
-  const void* SymbolTag;
+  const void *SymbolTag;
 
 public:
-  SymbolConjured(SymbolID sym, const Stmt* s, QualType t, unsigned count,
-                 const void* symbolTag)
+  SymbolConjured(SymbolID sym, const Stmt *s, QualType t, unsigned count,
+                 const void *symbolTag)
     : SymbolData(ConjuredKind, sym), S(s), T(t), Count(count),
       SymbolTag(symbolTag) {}
 
-  const Stmt* getStmt() const { return S; }
+  const Stmt *getStmt() const { return S; }
   unsigned getCount() const { return Count; }
-  const void* getTag() const { return SymbolTag; }
+  const void *getTag() const { return SymbolTag; }
 
   QualType getType(ASTContext&) const;
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
-  static void Profile(llvm::FoldingSetNodeID& profile, const Stmt* S,
-                      QualType T, unsigned Count, const void* SymbolTag) {
+  static void Profile(llvm::FoldingSetNodeID& profile, const Stmt *S,
+                      QualType T, unsigned Count, const void *SymbolTag) {
     profile.AddInteger((unsigned) ConjuredKind);
     profile.AddPointer(S);
     profile.Add(T);
@@ -154,30 +157,30 @@ class SymbolConjured : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == ConjuredKind;
   }
 };
 
-// A symbol representing the value of a MemRegion whose parent region has 
-// symbolic value.
+/// A symbol representing the value of a MemRegion whose parent region has
+/// symbolic value.
 class SymbolDerived : public SymbolData {
   SymbolRef parentSymbol;
-  const TypedRegion *R;
+  const TypedValueRegion *R;
 
 public:
-  SymbolDerived(SymbolID sym, SymbolRef parent, const TypedRegion *r)
+  SymbolDerived(SymbolID sym, SymbolRef parent, const TypedValueRegion *r)
     : SymbolData(DerivedKind, sym), parentSymbol(parent), R(r) {}
 
   SymbolRef getParentSymbol() const { return parentSymbol; }
-  const TypedRegion *getRegion() const { return R; }
+  const TypedValueRegion *getRegion() const { return R; }
 
   QualType getType(ASTContext&) const;
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static void Profile(llvm::FoldingSetNodeID& profile, SymbolRef parent,
-                      const TypedRegion *r) {
+                      const TypedValueRegion *r) {
     profile.AddInteger((unsigned) DerivedKind);
     profile.AddPointer(r);
     profile.AddPointer(parent);
@@ -188,7 +191,7 @@ class SymbolDerived : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == DerivedKind;
   }
 };
@@ -207,7 +210,7 @@ class SymbolExtent : public SymbolData {
 
   QualType getType(ASTContext&) const;
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static void Profile(llvm::FoldingSetNodeID& profile, const SubRegion *R) {
     profile.AddInteger((unsigned) ExtentKind);
@@ -219,7 +222,7 @@ class SymbolExtent : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == ExtentKind;
   }
 };
@@ -230,23 +233,23 @@ class SymbolExtent : public SymbolData {
 ///  Intended for use by checkers.
 class SymbolMetadata : public SymbolData {
   const MemRegion* R;
-  const Stmt* S;
+  const Stmt *S;
   QualType T;
   unsigned Count;
-  const void* Tag;
+  const void *Tag;
 public:
-  SymbolMetadata(SymbolID sym, const MemRegion* r, const Stmt* s, QualType t,
-                 unsigned count, const void* tag)
+  SymbolMetadata(SymbolID sym, const MemRegion* r, const Stmt *s, QualType t,
+                 unsigned count, const void *tag)
   : SymbolData(MetadataKind, sym), R(r), S(s), T(t), Count(count), Tag(tag) {}
 
   const MemRegion *getRegion() const { return R; }
-  const Stmt* getStmt() const { return S; }
+  const Stmt *getStmt() const { return S; }
   unsigned getCount() const { return Count; }
-  const void* getTag() const { return Tag; }
+  const void *getTag() const { return Tag; }
 
   QualType getType(ASTContext&) const;
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static void Profile(llvm::FoldingSetNodeID& profile, const MemRegion *R,
                       const Stmt *S, QualType T, unsigned Count,
@@ -264,12 +267,12 @@ class SymbolMetadata : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == MetadataKind;
   }
 };
 
-// SymIntExpr - Represents symbolic expression like 'x' + 3.
+/// SymIntExpr - Represents symbolic expression like 'x' + 3.
 class SymIntExpr : public SymExpr {
   const SymExpr *LHS;
   BinaryOperator::Opcode Op;
@@ -283,11 +286,11 @@ class SymIntExpr : public SymExpr {
 
   // FIXME: We probably need to make this out-of-line to avoid redundant
   // generation of virtual functions.
-  QualType getType(ASTContext& C) const { return T; }
+  QualType getType(ASTContext &C) const { return T; }
 
   BinaryOperator::Opcode getOpcode() const { return Op; }
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   const SymExpr *getLHS() const { return LHS; }
   const llvm::APSInt &getRHS() const { return RHS; }
@@ -307,12 +310,12 @@ class SymIntExpr : public SymExpr {
   }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == SymIntKind;
   }
 };
 
-// SymSymExpr - Represents symbolic expression like 'x' + 'y'.
+/// SymSymExpr - Represents symbolic expression like 'x' + 'y'.
 class SymSymExpr : public SymExpr {
   const SymExpr *LHS;
   BinaryOperator::Opcode Op;
@@ -330,9 +333,9 @@ class SymSymExpr : public SymExpr {
 
   // FIXME: We probably need to make this out-of-line to avoid redundant
   // generation of virtual functions.
-  QualType getType(ASTContext& C) const { return T; }
+  QualType getType(ASTContext &C) const { return T; }
 
-  void dumpToStream(llvm::raw_ostream &os) const;
+  void dumpToStream(raw_ostream &os) const;
 
   static void Profile(llvm::FoldingSetNodeID& ID, const SymExpr *lhs,
                     BinaryOperator::Opcode op, const SymExpr *rhs, QualType t) {
@@ -348,48 +351,58 @@ class SymSymExpr : public SymExpr {
   }
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr* SE) {
+  static inline bool classof(const SymExpr *SE) {
     return SE->getKind() == SymSymKind;
   }
 };
 
 class SymbolManager {
   typedef llvm::FoldingSet<SymExpr> DataSetTy;
+  typedef llvm::DenseMap<SymbolRef, SymbolRefSmallVectorTy*> SymbolDependTy;
+
   DataSetTy DataSet;
+  /// Stores the extra dependencies between symbols: the data should be kept
+  /// alive as long as the key is live.
+  SymbolDependTy SymbolDependencies;
   unsigned SymbolCounter;
   llvm::BumpPtrAllocator& BPAlloc;
   BasicValueFactory &BV;
-  ASTContext& Ctx;
+  ASTContext &Ctx;
 
 public:
-  SymbolManager(ASTContext& ctx, BasicValueFactory &bv,
+  SymbolManager(ASTContext &ctx, BasicValueFactory &bv,
                 llvm::BumpPtrAllocator& bpalloc)
-    : SymbolCounter(0), BPAlloc(bpalloc), BV(bv), Ctx(ctx) {}
+    : SymbolDependencies(16), SymbolCounter(0),
+      BPAlloc(bpalloc), BV(bv), Ctx(ctx) {}
 
   ~SymbolManager();
 
   static bool canSymbolicate(QualType T);
 
-  /// Make a unique symbol for MemRegion R according to its kind.
-  const SymbolRegionValue* getRegionValueSymbol(const TypedRegion* R);
+  /// \brief Make a unique symbol for MemRegion R according to its kind.
+  const SymbolRegionValue* getRegionValueSymbol(const TypedValueRegion* R);
 
-  const SymbolConjured* getConjuredSymbol(const Stmt* E, QualType T,
+  const SymbolConjured* getConjuredSymbol(const Stmt *E, QualType T,
                                           unsigned VisitCount,
-                                          const void* SymbolTag = 0);
+                                          const void *SymbolTag = 0);
 
-  const SymbolConjured* getConjuredSymbol(const Expr* E, unsigned VisitCount,
-                                          const void* SymbolTag = 0) {
+  const SymbolConjured* getConjuredSymbol(const Expr *E, unsigned VisitCount,
+                                          const void *SymbolTag = 0) {
     return getConjuredSymbol(E, E->getType(), VisitCount, SymbolTag);
   }
 
   const SymbolDerived *getDerivedSymbol(SymbolRef parentSymbol,
-                                        const TypedRegion *R);
+                                        const TypedValueRegion *R);
 
   const SymbolExtent *getExtentSymbol(const SubRegion *R);
 
-  const SymbolMetadata* getMetadataSymbol(const MemRegion* R, const Stmt* S,
+  /// \brief Creates a metadata symbol associated with a specific region.
+  ///
+  /// VisitCount can be used to differentiate regions corresponding to
+  /// different loop iterations, thus, making the symbol path-dependent.
+  const SymbolMetadata* getMetadataSymbol(const MemRegion* R, const Stmt *S,
                                           QualType T, unsigned VisitCount,
-                                          const void* SymbolTag = 0);
+                                          const void *SymbolTag = 0);
 
   const SymIntExpr *getSymIntExpr(const SymExpr *lhs, BinaryOperator::Opcode op,
                                   const llvm::APSInt& rhs, QualType t);
@@ -406,23 +419,43 @@ class SymbolManager {
     return SE->getType(Ctx);
   }
 
+  /// \brief Add artificial symbol dependency.
+  ///
+  /// The dependent symbol should stay alive as long as the primary is alive.
+  void addSymbolDependency(const SymbolRef Primary, const SymbolRef Dependent);
+
+  const SymbolRefSmallVectorTy *getDependentSymbols(const SymbolRef Primary);
+
   ASTContext &getContext() { return Ctx; }
   BasicValueFactory &getBasicVals() { return BV; }
 };
 
 class SymbolReaper {
-  typedef llvm::DenseSet<SymbolRef> SetTy;
+  enum SymbolStatus {
+    NotProcessed,
+    HaveMarkedDependents
+  };
 
-  SetTy TheLiving;
-  SetTy MetadataInUse;
-  SetTy TheDead;
+  typedef llvm::DenseSet<SymbolRef> SymbolSetTy;
+  typedef llvm::DenseMap<SymbolRef, SymbolStatus> SymbolMapTy;
+  typedef llvm::DenseSet<const MemRegion *> RegionSetTy;
+
+  SymbolMapTy TheLiving;
+  SymbolSetTy MetadataInUse;
+  SymbolSetTy TheDead;
+
+  RegionSetTy RegionRoots;
+  
   const LocationContext *LCtx;
   const Stmt *Loc;
   SymbolManager& SymMgr;
+  StoreRef reapedStore;
+  llvm::DenseMap<const MemRegion *, unsigned> includedRegionCache;
 
 public:
-  SymbolReaper(const LocationContext *ctx, const Stmt *s, SymbolManager& symmgr)
-   : LCtx(ctx), Loc(s), SymMgr(symmgr) {}
+  SymbolReaper(const LocationContext *ctx, const Stmt *s, SymbolManager& symmgr,
+               StoreManager &storeMgr)
+   : LCtx(ctx), Loc(s), SymMgr(symmgr), reapedStore(0, storeMgr) {}
 
   ~SymbolReaper() {}
 
@@ -430,48 +463,71 @@ class SymbolReaper {
   const Stmt *getCurrentStatement() const { return Loc; }
 
   bool isLive(SymbolRef sym);
+  bool isLiveRegion(const MemRegion *region);
   bool isLive(const Stmt *ExprVal) const;
-  bool isLive(const VarRegion *VR) const;
+  bool isLive(const VarRegion *VR, bool includeStoreBindings = false) const;
 
-  // markLive - Unconditionally marks a symbol as live. This should never be
-  //  used by checkers, only by the state infrastructure such as the store and
-  //  environment. Checkers should instead use metadata symbols and markInUse.
+  /// \brief Unconditionally marks a symbol as live.
+  ///
+  /// This should never be
+  /// used by checkers, only by the state infrastructure such as the store and
+  /// environment. Checkers should instead use metadata symbols and markInUse.
   void markLive(SymbolRef sym);
 
-  // markInUse - Marks a symbol as important to a checker. For metadata symbols,
-  //  this will keep the symbol alive as long as its associated region is also
-  //  live. For other symbols, this has no effect; checkers are not permitted
-  //  to influence the life of other symbols. This should be used before any
-  //  symbol marking has occurred, i.e. in the MarkLiveSymbols callback.
+  /// \brief Marks a symbol as important to a checker.
+  ///
+  /// For metadata symbols,
+  /// this will keep the symbol alive as long as its associated region is also
+  /// live. For other symbols, this has no effect; checkers are not permitted
+  /// to influence the life of other symbols. This should be used before any
+  /// symbol marking has occurred, i.e. in the MarkLiveSymbols callback.
   void markInUse(SymbolRef sym);
 
-  // maybeDead - If a symbol is known to be live, marks the symbol as live.
-  //  Otherwise, if the symbol cannot be proven live, it is marked as dead.
-  //  Returns true if the symbol is dead, false if live.
+  /// \brief If a symbol is known to be live, marks the symbol as live.
+  ///
+  ///  Otherwise, if the symbol cannot be proven live, it is marked as dead.
+  ///  Returns true if the symbol is dead, false if live.
   bool maybeDead(SymbolRef sym);
 
-  typedef SetTy::const_iterator dead_iterator;
+  typedef SymbolSetTy::const_iterator dead_iterator;
   dead_iterator dead_begin() const { return TheDead.begin(); }
   dead_iterator dead_end() const { return TheDead.end(); }
 
   bool hasDeadSymbols() const {
     return !TheDead.empty();
   }
+  
+  typedef RegionSetTy::const_iterator region_iterator;
+  region_iterator region_begin() const { return RegionRoots.begin(); }
+  region_iterator region_end() const { return RegionRoots.end(); }
 
-  /// isDead - Returns whether or not a symbol has been confirmed dead. This
-  ///  should only be called once all marking of dead symbols has completed.
-  ///  (For checkers, this means only in the evalDeadSymbols callback.)
+  /// \brief Returns whether or not a symbol has been confirmed dead.
+  ///
+  /// This should only be called once all marking of dead symbols has completed.
+  /// (For checkers, this means only in the evalDeadSymbols callback.)
   bool isDead(SymbolRef sym) const {
     return TheDead.count(sym);
   }
+  
+  void markLive(const MemRegion *region);
+  
+  /// \brief Set to the value of the symbolic store after
+  /// StoreManager::removeDeadBindings has been called.
+  void setReapedStore(StoreRef st) { reapedStore = st; }
+
+private:
+  /// Mark the symbols dependent on the input symbol as live.
+  void markDependentsLive(SymbolRef sym);
 };
 
 class SymbolVisitor {
 public:
-  // VisitSymbol - A visitor method invoked by
-  //  GRStateManager::scanReachableSymbols.  The method returns \c true if
-  //  symbols should continue be scanned and \c false otherwise.
+  /// \brief A visitor method invoked by ProgramStateManager::scanReachableSymbols.
+  ///
+  /// The method returns \c true if symbols should continue be scanned and \c
+  /// false otherwise.
   virtual bool VisitSymbol(SymbolRef sym) = 0;
+  virtual bool VisitMemRegion(const MemRegion *region) { return true; }
   virtual ~SymbolVisitor();
 };
 
@@ -480,7 +536,7 @@ class SymbolVisitor {
 } // end clang namespace
 
 namespace llvm {
-static inline llvm::raw_ostream& operator<<(llvm::raw_ostream& os,
+static inline raw_ostream &operator<<(raw_ostream &os,
                                             const clang::ento::SymExpr *SE) {
   SE->dumpToStream(os);
   return os;
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h
deleted file mode 100644
index 23ed2be8c7a4..000000000000
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h
+++ /dev/null
@@ -1,93 +0,0 @@
-//== TransferFuncs.h - Path-Sens. Transfer Functions Interface ---*- C++ -*--=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines TransferFuncs, which provides a base-class that
-//  defines an interface for transfer functions used by ExprEngine.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_GR_TRANSFERFUNCS
-#define LLVM_CLANG_GR_TRANSFERFUNCS
-
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
-#include <vector>
-
-namespace clang {
-class ObjCMessageExpr;
-
-namespace ento {
-class ExplodedNode;
-class ExplodedNodeSet;
-class EndOfFunctionNodeBuilder;
-class ExprEngine;
-class StmtNodeBuilder;
-class StmtNodeBuilderRef;
-
-class TransferFuncs {
-public:
-  TransferFuncs() {}
-  virtual ~TransferFuncs() {}
-
-  virtual void RegisterPrinters(std::vector<GRState::Printer*>& Printers) {}
-  virtual void RegisterChecks(ExprEngine& Eng) {}
-
-
-  // Calls.
-
-  virtual void evalCall(ExplodedNodeSet& Dst,
-                        ExprEngine& Engine,
-                        StmtNodeBuilder& Builder,
-                        const CallExpr* CE, SVal L,
-                        ExplodedNode* Pred) {}
-
-  virtual void evalObjCMessage(ExplodedNodeSet& Dst,
-                               ExprEngine& Engine,
-                               StmtNodeBuilder& Builder,
-                               ObjCMessage msg,
-                               ExplodedNode* Pred,
-                               const GRState *state) {}
-
-  // Stores.
-
-  virtual void evalBind(StmtNodeBuilderRef& B, SVal location, SVal val) {}
-
-  // End-of-path and dead symbol notification.
-
-  virtual void evalEndPath(ExprEngine& Engine,
-                           EndOfFunctionNodeBuilder& Builder) {}
-
-
-  virtual void evalDeadSymbols(ExplodedNodeSet& Dst,
-                               ExprEngine& Engine,
-                               StmtNodeBuilder& Builder,
-                               ExplodedNode* Pred,
-                               const GRState* state,
-                               SymbolReaper& SymReaper) {}
-
-  // Return statements.
-  virtual void evalReturn(ExplodedNodeSet& Dst,
-                          ExprEngine& Engine,
-                          StmtNodeBuilder& Builder,
-                          const ReturnStmt* S,
-                          ExplodedNode* Pred) {}
-
-  // Assumptions.
-  virtual const GRState* evalAssume(const GRState *state,
-                                    SVal Cond, bool Assumption) {
-    return state;
-  }  
-};
-
-} // end GR namespace
-
-} // end clang namespace
-
-#endif
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/WorkList.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/WorkList.h
index 6bc9fe56f8d6..fa340753e5b7 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/WorkList.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/WorkList.h
@@ -28,20 +28,20 @@ class ExplodedNode;
 class ExplodedNodeImpl;
 
 class WorkListUnit {
-  ExplodedNode* node;
+  ExplodedNode *node;
   BlockCounter counter;
-  const CFGBlock* block;
+  const CFGBlock *block;
   unsigned blockIdx; // This is the index of the next statement.
 
 public:
-  WorkListUnit(ExplodedNode* N, BlockCounter C,
-               const CFGBlock* B, unsigned idx)
+  WorkListUnit(ExplodedNode *N, BlockCounter C,
+               const CFGBlock *B, unsigned idx)
   : node(N),
     counter(C),
     block(B),
     blockIdx(idx) {}
 
-  explicit WorkListUnit(ExplodedNode* N, BlockCounter C)
+  explicit WorkListUnit(ExplodedNode *N, BlockCounter C)
   : node(N),
     counter(C),
     block(NULL),
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h
index 9d6298f36e3b..492edd4ccb41 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h
@@ -1,4 +1,4 @@
-//===-- CheckerRegistration.h - Checker Registration Function-------*- C++ -*-===//
+//===-- CheckerRegistration.h - Checker Registration Function ---*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -10,17 +10,21 @@
 #ifndef LLVM_CLANG_SA_FRONTEND_CHECKERREGISTRATION_H
 #define LLVM_CLANG_SA_FRONTEND_CHECKERREGISTRATION_H
 
+#include "clang/Basic/LLVM.h"
+#include <string>
+
 namespace clang {
   class AnalyzerOptions;
   class LangOptions;
-  class Diagnostic;
+  class DiagnosticsEngine;
 
 namespace ento {
   class CheckerManager;
 
-CheckerManager *registerCheckers(const AnalyzerOptions &opts,
-                                 const LangOptions &langOpts,
-                                 Diagnostic &diags);
+CheckerManager *createCheckerManager(const AnalyzerOptions &opts,
+                                     const LangOptions &langOpts,
+                                     ArrayRef<std::string> plugins,
+                                     DiagnosticsEngine &diags);
 
 } // end ento namespace
 
diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
index f01418175281..838ac925533f 100644
--- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
+++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
@@ -23,10 +23,10 @@ namespace ento {
 class AnalysisAction : public ASTFrontendAction {
 protected:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile);
+                                         StringRef InFile);
 };
 
-void printCheckerHelp(llvm::raw_ostream &OS);
+void printCheckerHelp(raw_ostream &OS, ArrayRef<std::string> plugins);
 
 } // end GR namespace
 
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMT.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMT.cpp
index f1d947da677e..6e1b0e535bbb 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMT.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMT.cpp
@@ -10,6 +10,7 @@
 #include "Internals.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/Rewrite/Rewriter.h"
@@ -18,12 +19,10 @@
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/ADT/Triple.h"
-
 using namespace clang;
 using namespace arcmt;
-using llvm::StringRef;
 
-bool CapturedDiagList::clearDiagnostic(llvm::ArrayRef<unsigned> IDs,
+bool CapturedDiagList::clearDiagnostic(ArrayRef<unsigned> IDs,
                                        SourceRange range) {
   if (range.isInvalid())
     return false;
@@ -39,7 +38,7 @@ bool CapturedDiagList::clearDiagnostic(llvm::ArrayRef<unsigned> IDs,
            diagLoc.isBeforeInTranslationUnitThan(range.getEnd()))) {
       cleared = true;
       ListTy::iterator eraseS = I++;
-      while (I != List.end() && I->getLevel() == Diagnostic::Note)
+      while (I != List.end() && I->getLevel() == DiagnosticsEngine::Note)
         ++I;
       // Clear the diagnostic and any notes following it.
       List.erase(eraseS, I);
@@ -52,7 +51,7 @@ bool CapturedDiagList::clearDiagnostic(llvm::ArrayRef<unsigned> IDs,
   return cleared;
 }
 
-bool CapturedDiagList::hasDiagnostic(llvm::ArrayRef<unsigned> IDs,
+bool CapturedDiagList::hasDiagnostic(ArrayRef<unsigned> IDs,
                                      SourceRange range) const {
   if (range.isInvalid())
     return false;
@@ -74,14 +73,14 @@ bool CapturedDiagList::hasDiagnostic(llvm::ArrayRef<unsigned> IDs,
   return false;
 }
 
-void CapturedDiagList::reportDiagnostics(Diagnostic &Diags) const {
+void CapturedDiagList::reportDiagnostics(DiagnosticsEngine &Diags) const {
   for (ListTy::const_iterator I = List.begin(), E = List.end(); I != E; ++I)
     Diags.Report(*I);
 }
 
 bool CapturedDiagList::hasErrors() const {
   for (ListTy::const_iterator I = List.begin(), E = List.end(); I != E; ++I)
-    if (I->getLevel() >= Diagnostic::Error)
+    if (I->getLevel() >= DiagnosticsEngine::Error)
       return true;
 
   return false;
@@ -89,18 +88,18 @@ bool CapturedDiagList::hasErrors() const {
 
 namespace {
 
-class CaptureDiagnosticClient : public DiagnosticClient {
-  Diagnostic &Diags;
+class CaptureDiagnosticConsumer : public DiagnosticConsumer {
+  DiagnosticsEngine &Diags;
   CapturedDiagList &CapturedDiags;
 public:
-  CaptureDiagnosticClient(Diagnostic &diags,
-                          CapturedDiagList &capturedDiags)
+  CaptureDiagnosticConsumer(DiagnosticsEngine &diags,
+                           CapturedDiagList &capturedDiags)
     : Diags(diags), CapturedDiags(capturedDiags) { }
 
-  virtual void HandleDiagnostic(Diagnostic::Level level,
-                                const DiagnosticInfo &Info) {
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level level,
+                                const Diagnostic &Info) {
     if (arcmt::isARCDiagnostic(Info.getID(), Diags) ||
-        level >= Diagnostic::Error || level == Diagnostic::Note) {
+        level >= DiagnosticsEngine::Error || level == DiagnosticsEngine::Note) {
       CapturedDiags.push_back(StoredDiagnostic(level, Info));
       return;
     }
@@ -108,11 +107,17 @@ class CaptureDiagnosticClient : public DiagnosticClient {
     // Non-ARC warnings are ignored.
     Diags.setLastDiagnosticIgnored();
   }
+  
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
+    // Just drop any diagnostics that come from cloned consumers; they'll
+    // have different source managers anyway.
+    return new IgnoringDiagConsumer();
+  }
 };
 
 } // end anonymous namespace
 
-static inline llvm::StringRef SimulatorVersionDefineName() {
+static inline StringRef SimulatorVersionDefineName() {
   return "__IPHONE_OS_VERSION_MIN_REQUIRED=";
 }
 
@@ -121,11 +126,11 @@ static inline llvm::StringRef SimulatorVersionDefineName() {
 // and return the grouped values as integers, e.g:
 //   __IPHONE_OS_VERSION_MIN_REQUIRED=40201
 // will return Major=4, Minor=2, Micro=1.
-static bool GetVersionFromSimulatorDefine(llvm::StringRef define,
+static bool GetVersionFromSimulatorDefine(StringRef define,
                                           unsigned &Major, unsigned &Minor,
                                           unsigned &Micro) {
   assert(define.startswith(SimulatorVersionDefineName()));
-  llvm::StringRef name, version;
+  StringRef name, version;
   llvm::tie(name, version) = define.split('=');
   if (version.empty())
     return false;
@@ -154,7 +159,7 @@ static bool HasARCRuntime(CompilerInvocation &origCI) {
       continue;
     if (!define.startswith(SimulatorVersionDefineName()))
       continue;
-    unsigned Major, Minor, Micro;
+    unsigned Major = 0, Minor = 0, Micro = 0;
     if (GetVersionFromSimulatorDefine(define, Major, Minor, Micro) &&
         Major < 10 && Minor < 100 && Micro < 100)
       return Major >= 5;
@@ -177,7 +182,8 @@ static bool HasARCRuntime(CompilerInvocation &origCI) {
   return false;
 }
 
-CompilerInvocation *createInvocationForMigration(CompilerInvocation &origCI) {
+static CompilerInvocation *
+createInvocationForMigration(CompilerInvocation &origCI) {
   llvm::OwningPtr<CompilerInvocation> CInvok;
   CInvok.reset(new CompilerInvocation(origCI));
   CInvok->getPreprocessorOpts().ImplicitPCHInclude = std::string();
@@ -194,13 +200,29 @@ CompilerInvocation *createInvocationForMigration(CompilerInvocation &origCI) {
   return CInvok.take();
 }
 
+static void emitPremigrationErrors(const CapturedDiagList &arcDiags,
+                                   const DiagnosticOptions &diagOpts,
+                                   Preprocessor &PP) {
+  TextDiagnosticPrinter printer(llvm::errs(), diagOpts);
+  llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      new DiagnosticsEngine(DiagID, &printer, /*ShouldOwnClient=*/false));
+  Diags->setSourceManager(&PP.getSourceManager());
+  
+  printer.BeginSourceFile(PP.getLangOptions(), &PP);
+  arcDiags.reportDiagnostics(*Diags);
+  printer.EndSourceFile();
+}
+
 //===----------------------------------------------------------------------===//
 // checkForManualIssues.
 //===----------------------------------------------------------------------===//
 
 bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
-                                 llvm::StringRef Filename, InputKind Kind,
-                                 DiagnosticClient *DiagClient) {
+                                 StringRef Filename, InputKind Kind,
+                                 DiagnosticConsumer *DiagClient,
+                                 bool emitPremigrationARCErrors,
+                                 StringRef plistOut) {
   if (!origCI.getLangOpts().ObjC1)
     return false;
 
@@ -216,11 +238,11 @@ bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
 
   assert(DiagClient);
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(
-                 new Diagnostic(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
 
   // Filter of all diagnostics.
-  CaptureDiagnosticClient errRec(*Diags, capturedDiags);
+  CaptureDiagnosticConsumer errRec(*Diags, capturedDiags);
   Diags->setClient(&errRec, /*ShouldOwnClient=*/false);
 
   llvm::OwningPtr<ASTUnit> Unit(
@@ -241,9 +263,21 @@ bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
     return true;
   }
 
+  if (emitPremigrationARCErrors)
+    emitPremigrationErrors(capturedDiags, origCI.getDiagnosticOpts(),
+                           Unit->getPreprocessor());
+  if (!plistOut.empty()) {
+    SmallVector<StoredDiagnostic, 8> arcDiags;
+    for (CapturedDiagList::iterator
+           I = capturedDiags.begin(), E = capturedDiags.end(); I != E; ++I)
+      arcDiags.push_back(*I);
+    writeARCDiagsToPlist(plistOut, arcDiags,
+                         Ctx.getSourceManager(), Ctx.getLangOptions());
+  }
+
   // After parsing of source files ended, we want to reuse the
   // diagnostics objects to emit further diagnostics.
-  // We call BeginSourceFile because DiagnosticClient requires that 
+  // We call BeginSourceFile because DiagnosticConsumer requires that 
   // diagnostics with source range information are emitted only in between
   // BeginSourceFile() and EndSourceFile().
   DiagClient->BeginSourceFile(Ctx.getLangOptions(), &Unit->getPreprocessor());
@@ -266,7 +300,7 @@ bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
   // to remove it so that we don't get errors from normal compilation.
   origCI.getLangOpts().ObjCAutoRefCount = false;
 
-  return capturedDiags.hasErrors();
+  return capturedDiags.hasErrors() || testAct.hasReportedErrors();
 }
 
 //===----------------------------------------------------------------------===//
@@ -274,15 +308,18 @@ bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
 //===----------------------------------------------------------------------===//
 
 static bool applyTransforms(CompilerInvocation &origCI,
-                            llvm::StringRef Filename, InputKind Kind,
-                            DiagnosticClient *DiagClient,
-                            llvm::StringRef outputDir) {
+                            StringRef Filename, InputKind Kind,
+                            DiagnosticConsumer *DiagClient,
+                            StringRef outputDir,
+                            bool emitPremigrationARCErrors,
+                            StringRef plistOut) {
   if (!origCI.getLangOpts().ObjC1)
     return false;
 
   // Make sure checking is successful first.
   CompilerInvocation CInvokForCheck(origCI);
-  if (arcmt::checkForManualIssues(CInvokForCheck, Filename, Kind, DiagClient))
+  if (arcmt::checkForManualIssues(CInvokForCheck, Filename, Kind, DiagClient,
+                                  emitPremigrationARCErrors, plistOut))
     return true;
 
   CompilerInvocation CInvok(origCI);
@@ -300,8 +337,8 @@ static bool applyTransforms(CompilerInvocation &origCI,
   }
 
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(
-                 new Diagnostic(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
 
   if (outputDir.empty()) {
     origCI.getLangOpts().ObjCAutoRefCount = true;
@@ -315,28 +352,32 @@ static bool applyTransforms(CompilerInvocation &origCI,
 }
 
 bool arcmt::applyTransformations(CompilerInvocation &origCI,
-                                 llvm::StringRef Filename, InputKind Kind,
-                                 DiagnosticClient *DiagClient) {
-  return applyTransforms(origCI, Filename, Kind, DiagClient, llvm::StringRef());
+                                 StringRef Filename, InputKind Kind,
+                                 DiagnosticConsumer *DiagClient) {
+  return applyTransforms(origCI, Filename, Kind, DiagClient,
+                         StringRef(), false, StringRef());
 }
 
 bool arcmt::migrateWithTemporaryFiles(CompilerInvocation &origCI,
-                                      llvm::StringRef Filename, InputKind Kind,
-                                      DiagnosticClient *DiagClient,
-                                      llvm::StringRef outputDir) {
+                                      StringRef Filename, InputKind Kind,
+                                      DiagnosticConsumer *DiagClient,
+                                      StringRef outputDir,
+                                      bool emitPremigrationARCErrors,
+                                      StringRef plistOut) {
   assert(!outputDir.empty() && "Expected output directory path");
-  return applyTransforms(origCI, Filename, Kind, DiagClient, outputDir);
+  return applyTransforms(origCI, Filename, Kind, DiagClient,
+                         outputDir, emitPremigrationARCErrors, plistOut);
 }
 
 bool arcmt::getFileRemappings(std::vector<std::pair<std::string,std::string> > &
                                   remap,
-                              llvm::StringRef outputDir,
-                              DiagnosticClient *DiagClient) {
+                              StringRef outputDir,
+                              DiagnosticConsumer *DiagClient) {
   assert(!outputDir.empty());
 
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(
-                 new Diagnostic(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
 
   FileRemapper remapper;
   bool err = remapper.initFromDisk(outputDir, *Diags,
@@ -364,7 +405,8 @@ class ARCMTMacroTrackerPPCallbacks : public PPCallbacks {
   ARCMTMacroTrackerPPCallbacks(std::vector<SourceLocation> &ARCMTMacroLocs)
     : ARCMTMacroLocs(ARCMTMacroLocs) { }
 
-  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo *MI) {
+  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo *MI,
+                            SourceRange Range) {
     if (MacroNameTok.getIdentifierInfo()->getName() == getARCMTMacroName())
       ARCMTMacroLocs.push_back(MacroNameTok.getLocation());
   }
@@ -378,7 +420,7 @@ class ARCMTMacroTrackerAction : public ASTFrontendAction {
     : ARCMTMacroLocs(ARCMTMacroLocs) { }
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile) {
+                                         StringRef InFile) {
     CI.getPreprocessor().addPPCallbacks(
                               new ARCMTMacroTrackerPPCallbacks(ARCMTMacroLocs));
     return new ASTConsumer();
@@ -402,7 +444,7 @@ class RewritesApplicator : public TransformActions::RewriteReceiver {
       Listener->finish();
   }
 
-  virtual void insert(SourceLocation loc, llvm::StringRef text) {
+  virtual void insert(SourceLocation loc, StringRef text) {
     bool err = rewriter.InsertText(loc, text, /*InsertAfter=*/true,
                                    /*indentNewLines=*/true);
     if (!err && Listener)
@@ -432,13 +474,13 @@ class RewritesApplicator : public TransformActions::RewriteReceiver {
 MigrationProcess::RewriteListener::~RewriteListener() { }
 
 MigrationProcess::MigrationProcess(const CompilerInvocation &CI,
-                                   DiagnosticClient *diagClient,
-                                   llvm::StringRef outputDir)
+                                   DiagnosticConsumer *diagClient,
+                                   StringRef outputDir)
   : OrigCI(CI), DiagClient(diagClient) {
   if (!outputDir.empty()) {
     llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-    llvm::IntrusiveRefCntPtr<Diagnostic> Diags(
-                 new Diagnostic(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+    llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
     Remapper.initFromDisk(outputDir, *Diags, /*ignoreIfFilesChanges=*/true);
   }
 }
@@ -456,11 +498,11 @@ bool MigrationProcess::applyTransform(TransformFn trans,
 
   assert(DiagClient);
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(
-               new Diagnostic(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
 
   // Filter of all diagnostics.
-  CaptureDiagnosticClient errRec(*Diags, capturedDiags);
+  CaptureDiagnosticConsumer errRec(*Diags, capturedDiags);
   Diags->setClient(&errRec, /*ShouldOwnClient=*/false);
 
   llvm::OwningPtr<ARCMTMacroTrackerAction> ASTAction;
@@ -488,7 +530,7 @@ bool MigrationProcess::applyTransform(TransformFn trans,
 
   // After parsing of source files ended, we want to reuse the
   // diagnostics objects to emit further diagnostics.
-  // We call BeginSourceFile because DiagnosticClient requires that 
+  // We call BeginSourceFile because DiagnosticConsumer requires that 
   // diagnostics with source range information are emitted only in between
   // BeginSourceFile() and EndSourceFile().
   DiagClient->BeginSourceFile(Ctx.getLangOptions(), &Unit->getPreprocessor());
@@ -522,7 +564,7 @@ bool MigrationProcess::applyTransform(TransformFn trans,
     buf.write(vecOS);
     vecOS.flush();
     llvm::MemoryBuffer *memBuf = llvm::MemoryBuffer::getMemBufferCopy(
-                   llvm::StringRef(newText.data(), newText.size()), newFname);
+                   StringRef(newText.data(), newText.size()), newFname);
     llvm::SmallString<64> filePath(file->getName());
     Unit->getFileManager().FixupRelativePath(filePath);
     Remapper.remap(filePath.str(), memBuf);
@@ -535,7 +577,7 @@ bool MigrationProcess::applyTransform(TransformFn trans,
 // isARCDiagnostic.
 //===----------------------------------------------------------------------===//
 
-bool arcmt::isARCDiagnostic(unsigned diagID, Diagnostic &Diag) {
+bool arcmt::isARCDiagnostic(unsigned diagID, DiagnosticsEngine &Diag) {
   return Diag.getDiagnosticIDs()->getCategoryNumberForDiag(diagID) ==
            diag::DiagCat_Automatic_Reference_Counting_Issue;
 }
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMTActions.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMTActions.cpp
index 345c7452420e..dea867af659d 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMTActions.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/ARCMTActions.cpp
@@ -38,16 +38,26 @@ ModifyAction::ModifyAction(FrontendAction *WrappedAction)
   : WrapperFrontendAction(WrappedAction) {}
 
 bool MigrateAction::BeginInvocation(CompilerInstance &CI) {
-  return !arcmt::migrateWithTemporaryFiles(CI.getInvocation(),
+  if (arcmt::migrateWithTemporaryFiles(CI.getInvocation(),
                                            getCurrentFile(),
                                            getCurrentFileKind(),
                                            CI.getDiagnostics().getClient(),
-                                           MigrateDir);
+                                           MigrateDir,
+                                           EmitPremigrationARCErros,
+                                           PlistOut))
+    return false; // errors, stop the action.
+
+  // We only want to see diagnostics emitted by migrateWithTemporaryFiles.
+  CI.getDiagnostics().setIgnoreAllWarnings(true);
+  return true;
 }
 
 MigrateAction::MigrateAction(FrontendAction *WrappedAction,
-                             llvm::StringRef migrateDir)
-  : WrapperFrontendAction(WrappedAction), MigrateDir(migrateDir) {
+                             StringRef migrateDir,
+                             StringRef plistOut,
+                             bool emitPremigrationARCErrors)
+  : WrapperFrontendAction(WrappedAction), MigrateDir(migrateDir),
+    PlistOut(plistOut), EmitPremigrationARCErros(emitPremigrationARCErrors) {
   if (MigrateDir.empty())
     MigrateDir = "."; // user current directory if none is given.
 }
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/CMakeLists.txt b/contrib/llvm/tools/clang/lib/ARCMigrate/CMakeLists.txt
deleted file mode 100644
index 5f2711e36f26..000000000000
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/CMakeLists.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-set(LLVM_USED_LIBS clangBasic clangAST clangParse clangFrontend clangRewrite)
-
-add_clang_library(clangARCMigrate
-  ARCMT.cpp
-  ARCMTActions.cpp
-  FileRemapper.cpp
-  TransARCAssign.cpp
-  TransAutoreleasePool.cpp
-  TransBlockObjCVariable.cpp
-  TransEmptyStatementsAndDealloc.cpp
-  TransformActions.cpp
-  Transforms.cpp
-  TransProperties.cpp
-  TransRetainReleaseDealloc.cpp
-  TransUnbridgedCasts.cpp
-  TransUnusedInitDelegate.cpp
-  TransZeroOutPropsInDealloc.cpp
-  )
-
-add_dependencies(clangARCMigrate
-  ClangAttrClasses
-  ClangAttrList
-  ClangDeclNodes
-  ClangStmtNodes)
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/FileRemapper.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/FileRemapper.cpp
index db26c29a3dda..c6e6ce46d173 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/FileRemapper.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/FileRemapper.cpp
@@ -27,7 +27,7 @@ FileRemapper::~FileRemapper() {
   clear();
 }
 
-void FileRemapper::clear(llvm::StringRef outputDir) {
+void FileRemapper::clear(StringRef outputDir) {
   for (MappingsTy::iterator
          I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I)
     resetTarget(I->second);
@@ -40,7 +40,7 @@ void FileRemapper::clear(llvm::StringRef outputDir) {
   }
 }
 
-std::string FileRemapper::getRemapInfoFile(llvm::StringRef outputDir) {
+std::string FileRemapper::getRemapInfoFile(StringRef outputDir) {
   assert(!outputDir.empty());
   llvm::sys::Path dir(outputDir);
   llvm::sys::Path infoFile = dir;
@@ -48,7 +48,7 @@ std::string FileRemapper::getRemapInfoFile(llvm::StringRef outputDir) {
   return infoFile.str();
 }
 
-bool FileRemapper::initFromDisk(llvm::StringRef outputDir, Diagnostic &Diag,
+bool FileRemapper::initFromDisk(StringRef outputDir, DiagnosticsEngine &Diag,
                                 bool ignoreIfFilesChanged) {
   assert(FromToMappings.empty() &&
          "initFromDisk should be called before any remap calls");
@@ -59,38 +59,38 @@ bool FileRemapper::initFromDisk(llvm::StringRef outputDir, Diagnostic &Diag,
     return false;
 
   std::vector<std::pair<const FileEntry *, const FileEntry *> > pairs;
+  
+  llvm::OwningPtr<llvm::MemoryBuffer> fileBuf;
+  if (llvm::error_code ec = llvm::MemoryBuffer::getFile(infoFile.c_str(),
+                                                        fileBuf))
+    return report("Error opening file: " + infoFile, Diag);
+  
+  SmallVector<StringRef, 64> lines;
+  fileBuf->getBuffer().split(lines, "\n");
 
-  std::ifstream fin(infoFile.c_str());
-  if (!fin.good())
-    return report(std::string("Error opening file: ") + infoFile, Diag);
-
-  while (true) {
-    std::string fromFilename, toFilename;
-    uint64_t timeModified;
-
-    fin >> fromFilename >> timeModified >> toFilename;
-    if (fin.eof())
-      break;
-    if (!fin.good())
-      return report(std::string("Error in format of file: ") + infoFile, Diag);
-
+  for (unsigned idx = 0; idx+3 <= lines.size(); idx += 3) {
+    StringRef fromFilename = lines[idx];
+    unsigned long long timeModified;
+    lines[idx+1].getAsInteger(10, timeModified);
+    StringRef toFilename = lines[idx+2];
+    
     const FileEntry *origFE = FileMgr->getFile(fromFilename);
     if (!origFE) {
       if (ignoreIfFilesChanged)
         continue;
-      return report(std::string("File does not exist: ") + fromFilename, Diag);
+      return report("File does not exist: " + fromFilename, Diag);
     }
     const FileEntry *newFE = FileMgr->getFile(toFilename);
     if (!newFE) {
       if (ignoreIfFilesChanged)
         continue;
-      return report(std::string("File does not exist: ") + toFilename, Diag);
+      return report("File does not exist: " + toFilename, Diag);
     }
 
     if ((uint64_t)origFE->getModificationTime() != timeModified) {
       if (ignoreIfFilesChanged)
         continue;
-      return report(std::string("File was modified: ") + fromFilename, Diag);
+      return report("File was modified: " + fromFilename, Diag);
     }
 
     pairs.push_back(std::make_pair(origFE, newFE));
@@ -102,13 +102,12 @@ bool FileRemapper::initFromDisk(llvm::StringRef outputDir, Diagnostic &Diag,
   return false;
 }
 
-bool FileRemapper::flushToDisk(llvm::StringRef outputDir, Diagnostic &Diag) {
+bool FileRemapper::flushToDisk(StringRef outputDir, DiagnosticsEngine &Diag) {
   using namespace llvm::sys;
 
   bool existed;
   if (fs::create_directory(outputDir, existed) != llvm::errc::success)
-    return report(std::string("Could not create directory: ") + outputDir.str(),
-                  Diag);
+    return report("Could not create directory: " + outputDir, Diag);
 
   std::string errMsg;
   std::string infoFile = getRemapInfoFile(outputDir);
@@ -121,13 +120,13 @@ bool FileRemapper::flushToDisk(llvm::StringRef outputDir, Diagnostic &Diag) {
          I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
 
     const FileEntry *origFE = I->first;
-    llvm::SmallString<200> origPath = llvm::StringRef(origFE->getName());
+    llvm::SmallString<200> origPath = StringRef(origFE->getName());
     fs::make_absolute(origPath);
     infoOut << origPath << '\n';
     infoOut << (uint64_t)origFE->getModificationTime() << '\n';
 
     if (const FileEntry *FE = I->second.dyn_cast<const FileEntry *>()) {
-      llvm::SmallString<200> newPath = llvm::StringRef(FE->getName());
+      llvm::SmallString<200> newPath = StringRef(FE->getName());
       fs::make_absolute(newPath);
       infoOut << newPath << '\n';
     } else {
@@ -138,8 +137,7 @@ bool FileRemapper::flushToDisk(llvm::StringRef outputDir, Diagnostic &Diag) {
       tempPath += path::extension(origFE->getName());
       int fd;
       if (fs::unique_file(tempPath.str(), fd, tempPath) != llvm::errc::success)
-        return report(std::string("Could not create file: ") + tempPath.c_str(),
-                      Diag);
+        return report("Could not create file: " + tempPath.str(), Diag);
 
       llvm::raw_fd_ostream newOut(fd, /*shouldClose=*/true);
       llvm::MemoryBuffer *mem = I->second.get<llvm::MemoryBuffer *>();
@@ -156,8 +154,8 @@ bool FileRemapper::flushToDisk(llvm::StringRef outputDir, Diagnostic &Diag) {
   return false;
 }
 
-bool FileRemapper::overwriteOriginal(Diagnostic &Diag,
-                                     llvm::StringRef outputDir) {
+bool FileRemapper::overwriteOriginal(DiagnosticsEngine &Diag,
+                                     StringRef outputDir) {
   using namespace llvm::sys;
 
   for (MappingsTy::iterator
@@ -165,20 +163,15 @@ bool FileRemapper::overwriteOriginal(Diagnostic &Diag,
     const FileEntry *origFE = I->first;
     if (const FileEntry *newFE = I->second.dyn_cast<const FileEntry *>()) {
       if (fs::copy_file(newFE->getName(), origFE->getName(),
-                 fs::copy_option::overwrite_if_exists) != llvm::errc::success) {
-        std::string err = "Could not copy file '";
-        llvm::raw_string_ostream os(err);
-        os << "Could not copy file '" << newFE->getName() << "' to file '"
-           << origFE->getName() << "'";
-        os.flush();
-        return report(err, Diag);
-      }
+                 fs::copy_option::overwrite_if_exists) != llvm::errc::success)
+        return report(StringRef("Could not copy file '") + newFE->getName() +
+                      "' to file '" + origFE->getName() + "'", Diag);
     } else {
 
       bool fileExists = false;
       fs::exists(origFE->getName(), fileExists);
       if (!fileExists)
-        return report(std::string("File does not exist: ") + origFE->getName(),
+        return report(StringRef("File does not exist: ") + origFE->getName(),
                       Diag);
 
       std::string errMsg;
@@ -229,11 +222,11 @@ void FileRemapper::transferMappingsAndClear(CompilerInvocation &CI) {
   clear();
 }
 
-void FileRemapper::remap(llvm::StringRef filePath, llvm::MemoryBuffer *memBuf) {
+void FileRemapper::remap(StringRef filePath, llvm::MemoryBuffer *memBuf) {
   remap(getOriginalFile(filePath), memBuf);
 }
 
-void FileRemapper::remap(llvm::StringRef filePath, llvm::StringRef newPath) {
+void FileRemapper::remap(StringRef filePath, StringRef newPath) {
   const FileEntry *file = getOriginalFile(filePath);
   const FileEntry *newfile = FileMgr->getFile(newPath);
   remap(file, newfile);
@@ -254,7 +247,7 @@ void FileRemapper::remap(const FileEntry *file, const FileEntry *newfile) {
   ToFromMappings[newfile] = file;
 }
 
-const FileEntry *FileRemapper::getOriginalFile(llvm::StringRef filePath) {
+const FileEntry *FileRemapper::getOriginalFile(StringRef filePath) {
   const FileEntry *file = FileMgr->getFile(filePath);
   // If we are updating a file that overriden an original file,
   // actually update the original file.
@@ -283,9 +276,10 @@ void FileRemapper::resetTarget(Target &targ) {
   }
 }
 
-bool FileRemapper::report(const std::string &err, Diagnostic &Diag) {
+bool FileRemapper::report(const Twine &err, DiagnosticsEngine &Diag) {
+  llvm::SmallString<128> buf;
   unsigned ID = Diag.getDiagnosticIDs()->getCustomDiagID(DiagnosticIDs::Error,
-                                                         err);
+                                                         err.toStringRef(buf));
   Diag.Report(ID);
   return true;
 }
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/Internals.h b/contrib/llvm/tools/clang/lib/ARCMigrate/Internals.h
index 4f9b138a06ce..46f3bb626f2f 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/Internals.h
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/Internals.h
@@ -26,21 +26,30 @@ class CapturedDiagList {
 public:
   void push_back(const StoredDiagnostic &diag) { List.push_back(diag); }
 
-  bool clearDiagnostic(llvm::ArrayRef<unsigned> IDs, SourceRange range);
-  bool hasDiagnostic(llvm::ArrayRef<unsigned> IDs, SourceRange range) const;
+  bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
+  bool hasDiagnostic(ArrayRef<unsigned> IDs, SourceRange range) const;
 
-  void reportDiagnostics(Diagnostic &diags) const;
+  void reportDiagnostics(DiagnosticsEngine &diags) const;
 
   bool hasErrors() const;
+
+  typedef ListTy::const_iterator iterator;
+  iterator begin() const { return List.begin(); }
+  iterator end()   const { return List.end();   }
 };
 
+void writeARCDiagsToPlist(const std::string &outPath,
+                          ArrayRef<StoredDiagnostic> diags,
+                          SourceManager &SM, const LangOptions &LangOpts);
+
 class TransformActions {
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
   CapturedDiagList &CapturedDiags;
+  bool ReportedErrors;
   void *Impl; // TransformActionsImpl.
 
 public:
-  TransformActions(Diagnostic &diag, CapturedDiagList &capturedDiags,
+  TransformActions(DiagnosticsEngine &diag, CapturedDiagList &capturedDiags,
                    ASTContext &ctx, Preprocessor &PP);
   ~TransformActions();
 
@@ -48,21 +57,21 @@ class TransformActions {
   bool commitTransaction();
   void abortTransaction();
 
-  void insert(SourceLocation loc, llvm::StringRef text);
-  void insertAfterToken(SourceLocation loc, llvm::StringRef text);
+  void insert(SourceLocation loc, StringRef text);
+  void insertAfterToken(SourceLocation loc, StringRef text);
   void remove(SourceRange range);
   void removeStmt(Stmt *S);
-  void replace(SourceRange range, llvm::StringRef text);
+  void replace(SourceRange range, StringRef text);
   void replace(SourceRange range, SourceRange replacementRange);
-  void replaceStmt(Stmt *S, llvm::StringRef text);
-  void replaceText(SourceLocation loc, llvm::StringRef text,
-                   llvm::StringRef replacementText);
+  void replaceStmt(Stmt *S, StringRef text);
+  void replaceText(SourceLocation loc, StringRef text,
+                   StringRef replacementText);
   void increaseIndentation(SourceRange range,
                            SourceLocation parentIndent);
 
-  bool clearDiagnostic(llvm::ArrayRef<unsigned> IDs, SourceRange range);
+  bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
   bool clearAllDiagnostics(SourceRange range) {
-    return clearDiagnostic(llvm::ArrayRef<unsigned>(), range);
+    return clearDiagnostic(ArrayRef<unsigned>(), range);
   }
   bool clearDiagnostic(unsigned ID1, unsigned ID2, SourceRange range) {
     unsigned IDs[] = { ID1, ID2 };
@@ -83,16 +92,18 @@ class TransformActions {
     return CapturedDiags.hasDiagnostic(IDs, range);
   }
 
-  void reportError(llvm::StringRef error, SourceLocation loc,
+  void reportError(StringRef error, SourceLocation loc,
                    SourceRange range = SourceRange());
-  void reportNote(llvm::StringRef note, SourceLocation loc,
+  void reportNote(StringRef note, SourceLocation loc,
                   SourceRange range = SourceRange());
 
+  bool hasReportedErrors() const { return ReportedErrors; }
+
   class RewriteReceiver {
   public:
     virtual ~RewriteReceiver();
 
-    virtual void insert(SourceLocation loc, llvm::StringRef text) = 0;
+    virtual void insert(SourceLocation loc, StringRef text) = 0;
     virtual void remove(CharSourceRange range) = 0;
     virtual void increaseIndentation(CharSourceRange range,
                                      SourceLocation parentIndent) = 0;
@@ -135,9 +146,9 @@ class MigrationPass {
     : Ctx(Ctx), SemaRef(sema), TA(TA), ARCMTMacroLocs(ARCMTMacroLocs) { }
 };
 
-bool isARCDiagnostic(unsigned diagID, Diagnostic &Diag);
+bool isARCDiagnostic(unsigned diagID, DiagnosticsEngine &Diag);
 
-static inline llvm::StringRef getARCMTMacroName() {
+static inline StringRef getARCMTMacroName() {
   return "__IMPL_ARCMT_REMOVED_EXPR__";
 }
 
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/Makefile b/contrib/llvm/tools/clang/lib/ARCMigrate/Makefile
deleted file mode 100644
index 5232c5e5aff4..000000000000
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/Makefile
+++ /dev/null
@@ -1,18 +0,0 @@
-##===- clang/lib/ARCMigrate/Makefile --------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-#
-# This implements code transformation to ARC mode.
-#
-##===----------------------------------------------------------------------===##
-
-CLANG_LEVEL := ../..
-LIBRARYNAME := clangARCMigrate
-
-include $(CLANG_LEVEL)/Makefile
-
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/PlistReporter.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/PlistReporter.cpp
new file mode 100644
index 000000000000..d1bc90fdbe16
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/PlistReporter.cpp
@@ -0,0 +1,195 @@
+//===--- PlistReporter.cpp - ARC Migrate Tool Plist Reporter ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Internals.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/FileManager.h"
+using namespace clang;
+using namespace arcmt;
+
+// FIXME: This duplicates significant functionality from PlistDiagnostics.cpp,
+// it would be jolly good if there was a reusable PlistWriter or something.
+
+typedef llvm::DenseMap<FileID, unsigned> FIDMap;
+
+static void AddFID(FIDMap &FIDs, SmallVectorImpl<FileID> &V,
+                   const SourceManager &SM, SourceLocation L) {
+
+  FileID FID = SM.getFileID(SM.getExpansionLoc(L));
+  FIDMap::iterator I = FIDs.find(FID);
+  if (I != FIDs.end()) return;
+  FIDs[FID] = V.size();
+  V.push_back(FID);
+}
+
+static unsigned GetFID(const FIDMap& FIDs, const SourceManager &SM,
+                       SourceLocation L) {
+  FileID FID = SM.getFileID(SM.getExpansionLoc(L));
+  FIDMap::const_iterator I = FIDs.find(FID);
+  assert(I != FIDs.end());
+  return I->second;
+}
+
+static raw_ostream& Indent(raw_ostream& o, const unsigned indent) {
+  for (unsigned i = 0; i < indent; ++i) o << ' ';
+  return o;
+}
+
+static void EmitLocation(raw_ostream& o, const SourceManager &SM,
+                         const LangOptions &LangOpts,
+                         SourceLocation L, const FIDMap &FM,
+                         unsigned indent, bool extend = false) {
+
+  FullSourceLoc Loc(SM.getExpansionLoc(L), const_cast<SourceManager&>(SM));
+
+  // Add in the length of the token, so that we cover multi-char tokens.
+  unsigned offset =
+    extend ? Lexer::MeasureTokenLength(Loc, SM, LangOpts) - 1 : 0;
+
+  Indent(o, indent) << "<dict>\n";
+  Indent(o, indent) << " <key>line</key><integer>"
+                    << Loc.getExpansionLineNumber() << "</integer>\n";
+  Indent(o, indent) << " <key>col</key><integer>"
+                    << Loc.getExpansionColumnNumber() + offset << "</integer>\n";
+  Indent(o, indent) << " <key>file</key><integer>"
+                    << GetFID(FM, SM, Loc) << "</integer>\n";
+  Indent(o, indent) << "</dict>\n";
+}
+
+static void EmitRange(raw_ostream& o, const SourceManager &SM,
+                      const LangOptions &LangOpts,
+                      CharSourceRange R, const FIDMap &FM,
+                      unsigned indent) {
+  Indent(o, indent) << "<array>\n";
+  EmitLocation(o, SM, LangOpts, R.getBegin(), FM, indent+1);
+  EmitLocation(o, SM, LangOpts, R.getEnd(), FM, indent+1, R.isTokenRange());
+  Indent(o, indent) << "</array>\n";
+}
+
+static raw_ostream& EmitString(raw_ostream& o,
+                                     StringRef s) {
+  o << "<string>";
+  for (StringRef::const_iterator I=s.begin(), E=s.end(); I!=E; ++I) {
+    char c = *I;
+    switch (c) {
+    default:   o << c; break;
+    case '&':  o << "&amp;"; break;
+    case '<':  o << "&lt;"; break;
+    case '>':  o << "&gt;"; break;
+    case '\'': o << "&apos;"; break;
+    case '\"': o << "&quot;"; break;
+    }
+  }
+  o << "</string>";
+  return o;
+}
+
+void arcmt::writeARCDiagsToPlist(const std::string &outPath,
+                                 ArrayRef<StoredDiagnostic> diags,
+                                 SourceManager &SM,
+                                 const LangOptions &LangOpts) {
+  DiagnosticIDs DiagIDs;
+
+  // Build up a set of FIDs that we use by scanning the locations and
+  // ranges of the diagnostics.
+  FIDMap FM;
+  SmallVector<FileID, 10> Fids;
+
+  for (ArrayRef<StoredDiagnostic>::iterator
+         I = diags.begin(), E = diags.end(); I != E; ++I) {
+    const StoredDiagnostic &D = *I;
+
+    AddFID(FM, Fids, SM, D.getLocation());
+
+    for (StoredDiagnostic::range_iterator
+           RI = D.range_begin(), RE = D.range_end(); RI != RE; ++RI) {
+      AddFID(FM, Fids, SM, RI->getBegin());
+      AddFID(FM, Fids, SM, RI->getEnd());
+    }
+  }
+
+  std::string errMsg;
+  llvm::raw_fd_ostream o(outPath.c_str(), errMsg);
+  if (!errMsg.empty()) {
+    llvm::errs() << "error: could not create file: " << outPath << '\n';
+    return;
+  }
+
+  // Write the plist header.
+  o << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+  "<!DOCTYPE plist PUBLIC \"-//Apple Computer//DTD PLIST 1.0//EN\" "
+  "\"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n"
+  "<plist version=\"1.0\">\n";
+
+  // Write the root object: a <dict> containing...
+  //  - "files", an <array> mapping from FIDs to file names
+  //  - "diagnostics", an <array> containing the diagnostics
+  o << "<dict>\n"
+       " <key>files</key>\n"
+       " <array>\n";
+
+  for (SmallVectorImpl<FileID>::iterator I=Fids.begin(), E=Fids.end();
+       I!=E; ++I) {
+    o << "  ";
+    EmitString(o, SM.getFileEntryForID(*I)->getName()) << '\n';
+  }
+
+  o << " </array>\n"
+       " <key>diagnostics</key>\n"
+       " <array>\n";
+
+  for (ArrayRef<StoredDiagnostic>::iterator
+         DI = diags.begin(), DE = diags.end(); DI != DE; ++DI) {
+    
+    const StoredDiagnostic &D = *DI;
+
+    if (D.getLevel() == DiagnosticsEngine::Ignored)
+      continue;
+
+    o << "  <dict>\n";
+
+    // Output the diagnostic.
+    o << "   <key>description</key>";
+    EmitString(o, D.getMessage()) << '\n';
+    o << "   <key>category</key>";
+    EmitString(o, DiagIDs.getCategoryNameFromID(
+                          DiagIDs.getCategoryNumberForDiag(D.getID()))) << '\n';
+    o << "   <key>type</key>";
+    if (D.getLevel() >= DiagnosticsEngine::Error)
+      EmitString(o, "error") << '\n';
+    else if (D.getLevel() == DiagnosticsEngine::Warning)
+      EmitString(o, "warning") << '\n';
+    else
+      EmitString(o, "note") << '\n';
+
+    // Output the location of the bug.
+    o << "  <key>location</key>\n";
+    EmitLocation(o, SM, LangOpts, D.getLocation(), FM, 2);
+
+    // Output the ranges (if any).
+    StoredDiagnostic::range_iterator RI = D.range_begin(), RE = D.range_end();
+
+    if (RI != RE) {
+      o << "   <key>ranges</key>\n";
+      o << "   <array>\n";
+      for (; RI != RE; ++RI)
+        EmitRange(o, SM, LangOpts, *RI, FM, 4);
+      o << "   </array>\n";
+    }
+
+    // Close up the entry.
+    o << "  </dict>\n";
+  }
+
+  o << " </array>\n";
+
+  // Finish.
+  o << "</dict>\n</plist>";
+}
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransAPIUses.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransAPIUses.cpp
new file mode 100644
index 000000000000..aaa82d8dfb7a
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransAPIUses.cpp
@@ -0,0 +1,109 @@
+//===--- TransAPIUses.cpp - Tranformations to ARC mode --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// checkAPIUses:
+//
+// Emits error/fix with some API uses that are obsolete or not safe in ARC mode:
+//
+// - NSInvocation's [get/set]ReturnValue and [get/set]Argument are only safe
+//   with __unsafe_unretained objects.
+// - Calling -zone gets replaced with 'nil'.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Transforms.h"
+#include "Internals.h"
+#include "clang/Sema/SemaDiagnostic.h"
+
+using namespace clang;
+using namespace arcmt;
+using namespace trans;
+
+namespace {
+
+class APIChecker : public RecursiveASTVisitor<APIChecker> {
+  MigrationPass &Pass;
+
+  Selector getReturnValueSel, setReturnValueSel;
+  Selector getArgumentSel, setArgumentSel;
+
+  Selector zoneSel;
+public:
+  APIChecker(MigrationPass &pass) : Pass(pass) {
+    SelectorTable &sels = Pass.Ctx.Selectors;
+    IdentifierTable &ids = Pass.Ctx.Idents;
+    getReturnValueSel = sels.getUnarySelector(&ids.get("getReturnValue"));
+    setReturnValueSel = sels.getUnarySelector(&ids.get("setReturnValue"));
+
+    IdentifierInfo *selIds[2];
+    selIds[0] = &ids.get("getArgument");
+    selIds[1] = &ids.get("atIndex");
+    getArgumentSel = sels.getSelector(2, selIds);
+    selIds[0] = &ids.get("setArgument");
+    setArgumentSel = sels.getSelector(2, selIds);
+
+    zoneSel = sels.getNullarySelector(&ids.get("zone"));
+  }
+
+  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
+    // NSInvocation.
+    if (E->isInstanceMessage() &&
+        E->getReceiverInterface() &&
+        E->getReceiverInterface()->getName() == "NSInvocation") {
+      StringRef selName;
+      if (E->getSelector() == getReturnValueSel)
+        selName = "getReturnValue";
+      else if (E->getSelector() == setReturnValueSel)
+        selName = "setReturnValue";
+      else if (E->getSelector() == getArgumentSel)
+        selName = "getArgument";
+      else if (E->getSelector() == setArgumentSel)
+        selName = "setArgument";
+
+      if (selName.empty())
+        return true;
+
+      Expr *parm = E->getArg(0)->IgnoreParenCasts();
+      QualType pointee = parm->getType()->getPointeeType();
+      if (pointee.isNull())
+        return true;
+
+      if (pointee.getObjCLifetime() > Qualifiers::OCL_ExplicitNone) {
+        std::string err = "NSInvocation's ";
+        err += selName;
+        err += " is not safe to be used with an object with ownership other "
+            "than __unsafe_unretained";
+        Pass.TA.reportError(err, parm->getLocStart(), parm->getSourceRange());
+      }
+      return true;
+    }
+
+    // -zone.
+    if (E->isInstanceMessage() &&
+        E->getInstanceReceiver() &&
+        E->getSelector() == zoneSel &&
+        Pass.TA.hasDiagnostic(diag::err_unavailable,
+                              diag::err_unavailable_message,
+                              E->getInstanceReceiver()->getExprLoc())) {
+      // Calling -zone is meaningless in ARC, change it to nil.
+      Transaction Trans(Pass.TA);
+      Pass.TA.clearDiagnostic(diag::err_unavailable,
+                              diag::err_unavailable_message,
+                              E->getInstanceReceiver()->getExprLoc());
+      Pass.TA.replace(E->getSourceRange(), getNilString(Pass.Ctx));
+    }
+    return true;
+  }
+};
+
+} // anonymous namespace
+
+void trans::checkAPIUses(MigrationPass &pass) {
+  APIChecker(pass).TraverseDecl(pass.Ctx.getTranslationUnitDecl());
+}
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransARCAssign.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransARCAssign.cpp
index 8c00df5daa03..1f10196f9daa 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransARCAssign.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransARCAssign.cpp
@@ -28,7 +28,6 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransAutoreleasePool.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransAutoreleasePool.cpp
index 5b8485432c52..08561f97f962 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransAutoreleasePool.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransAutoreleasePool.cpp
@@ -36,16 +36,15 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
 class ReleaseCollector : public RecursiveASTVisitor<ReleaseCollector> {
   Decl *Dcl;
-  llvm::SmallVectorImpl<ObjCMessageExpr *> &Releases;
+  SmallVectorImpl<ObjCMessageExpr *> &Releases;
 
 public:
-  ReleaseCollector(Decl *D, llvm::SmallVectorImpl<ObjCMessageExpr *> &releases)
+  ReleaseCollector(Decl *D, SmallVectorImpl<ObjCMessageExpr *> &releases)
     : Dcl(D), Releases(releases) { }
 
   bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
@@ -82,7 +81,7 @@ class AutoreleasePoolRewriter
   }
   
   ~AutoreleasePoolRewriter() {
-    llvm::SmallVector<VarDecl *, 8> VarsToHandle;
+    SmallVector<VarDecl *, 8> VarsToHandle;
 
     for (std::map<VarDecl *, PoolVarInfo>::iterator
            I = PoolVars.begin(), E = PoolVars.end(); I != E; ++I) {
@@ -92,7 +91,7 @@ class AutoreleasePoolRewriter
       // Check that we can handle/rewrite all references of the pool.
 
       clearRefsIn(info.Dcl, info.Refs);
-      for (llvm::SmallVectorImpl<PoolScope>::iterator
+      for (SmallVectorImpl<PoolScope>::iterator
              scpI = info.Scopes.begin(),
              scpE = info.Scopes.end(); scpI != scpE; ++scpI) {
         PoolScope &scope = *scpI;
@@ -116,7 +115,7 @@ class AutoreleasePoolRewriter
       Pass.TA.removeStmt(info.Dcl);
 
       // Add "@autoreleasepool { }"
-      for (llvm::SmallVectorImpl<PoolScope>::iterator
+      for (SmallVectorImpl<PoolScope>::iterator
              scpI = info.Scopes.begin(),
              scpE = info.Scopes.end(); scpI != scpE; ++scpI) {
         PoolScope &scope = *scpI;
@@ -147,11 +146,11 @@ class AutoreleasePoolRewriter
       }
 
       // Remove rest of pool var references.
-      for (llvm::SmallVectorImpl<PoolScope>::iterator
+      for (SmallVectorImpl<PoolScope>::iterator
              scpI = info.Scopes.begin(),
              scpE = info.Scopes.end(); scpI != scpE; ++scpI) {
         PoolScope &scope = *scpI;
-        for (llvm::SmallVectorImpl<ObjCMessageExpr *>::iterator
+        for (SmallVectorImpl<ObjCMessageExpr *>::iterator
                relI = scope.Releases.begin(),
                relE = scope.Releases.end(); relI != relE; ++relI) {
           clearUnavailableDiags(*relI);
@@ -162,7 +161,7 @@ class AutoreleasePoolRewriter
   }
 
   bool VisitCompoundStmt(CompoundStmt *S) {
-    llvm::SmallVector<PoolScope, 4> Scopes;
+    SmallVector<PoolScope, 4> Scopes;
 
     for (Stmt::child_iterator
            I = S->body_begin(), E = S->body_end(); I != E; ++I) {
@@ -228,7 +227,7 @@ class AutoreleasePoolRewriter
     Stmt::child_iterator Begin;
     Stmt::child_iterator End;
     bool IsFollowedBySimpleReturnStmt;
-    llvm::SmallVector<ObjCMessageExpr *, 4> Releases;
+    SmallVector<ObjCMessageExpr *, 4> Releases;
 
     PoolScope() : PoolVar(0), CompoundParent(0), Begin(), End(),
                   IsFollowedBySimpleReturnStmt(false) { }
@@ -287,6 +286,9 @@ class AutoreleasePoolRewriter
     }
 
     bool isInScope(SourceLocation loc) {
+      if (loc.isInvalid())
+        return false;
+
       SourceManager &SM = Ctx.getSourceManager();
       if (SM.isBeforeInTranslationUnit(loc, ScopeRange.getBegin()))
         return false;
@@ -420,7 +422,7 @@ class AutoreleasePoolRewriter
   struct PoolVarInfo {
     DeclStmt *Dcl;
     ExprSet Refs;
-    llvm::SmallVector<PoolScope, 2> Scopes;
+    SmallVector<PoolScope, 2> Scopes;
 
     PoolVarInfo() : Dcl(0) { }
   };
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp
index 0e342b7a8f8c..48c0ca9cefe5 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransBlockObjCVariable.cpp
@@ -32,7 +32,6 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
@@ -75,7 +74,7 @@ class RootBlockObjCVarRewriter :
   RootBlockObjCVarRewriter(MigrationPass &pass) : Pass(pass) { }
 
   bool VisitBlockDecl(BlockDecl *block) {
-    llvm::SmallVector<VarDecl *, 4> BlockVars;
+    SmallVector<VarDecl *, 4> BlockVars;
     
     for (BlockDecl::capture_iterator
            I = block->capture_begin(), E = block->capture_end(); I != E; ++I) {
@@ -101,7 +100,7 @@ class RootBlockObjCVarRewriter :
         bool useWeak = canApplyWeak(Pass.Ctx, var->getType());
         SourceManager &SM = Pass.Ctx.getSourceManager();
         Transaction Trans(Pass.TA);
-        Pass.TA.replaceText(SM.getInstantiationLoc(attr->getLocation()),
+        Pass.TA.replaceText(SM.getExpansionLoc(attr->getLocation()),
                             "__block",
                             useWeak ? "__weak" : "__unsafe_unretained");
       }
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp
index d0bc332ff160..3ad05e683c78 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp
@@ -22,11 +22,53 @@
 #include "Transforms.h"
 #include "Internals.h"
 #include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/SourceManager.h"
 
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
+
+static bool isEmptyARCMTMacroStatement(NullStmt *S,
+                                       std::vector<SourceLocation> &MacroLocs,
+                                       ASTContext &Ctx) {
+  if (!S->hasLeadingEmptyMacro())
+    return false;
+
+  SourceLocation SemiLoc = S->getSemiLoc();
+  if (SemiLoc.isInvalid() || SemiLoc.isMacroID())
+    return false;
+
+  if (MacroLocs.empty())
+    return false;
+
+  SourceManager &SM = Ctx.getSourceManager();
+  std::vector<SourceLocation>::iterator
+    I = std::upper_bound(MacroLocs.begin(), MacroLocs.end(), SemiLoc,
+                         SourceManager::LocBeforeThanCompare(SM));
+  --I;
+  SourceLocation
+      AfterMacroLoc = I->getLocWithOffset(getARCMTMacroName().size());
+  assert(AfterMacroLoc.isFileID());
+
+  if (AfterMacroLoc == SemiLoc)
+    return true;
+
+  int RelOffs = 0;
+  if (!SM.isInSameSLocAddrSpace(AfterMacroLoc, SemiLoc, &RelOffs))
+    return false;
+  if (RelOffs < 0)
+    return false;
+
+  // We make the reasonable assumption that a semicolon after 100 characters
+  // means that it is not the next token after our macro. If this assumption
+  // fails it is not critical, we will just fail to clear out, e.g., an empty
+  // 'if'.
+  if (RelOffs - getARCMTMacroName().size() > 100)
+    return false;
+
+  SourceLocation AfterMacroSemiLoc = findSemiAfterLocation(AfterMacroLoc, Ctx);
+  return AfterMacroSemiLoc == SemiLoc;
+}
 
 namespace {
 
@@ -34,14 +76,14 @@ namespace {
 /// transformations.
 class EmptyChecker : public StmtVisitor<EmptyChecker, bool> {
   ASTContext &Ctx;
-  llvm::DenseSet<unsigned> &MacroLocs;
+  std::vector<SourceLocation> &MacroLocs;
 
 public:
-  EmptyChecker(ASTContext &ctx, llvm::DenseSet<unsigned> &macroLocs)
+  EmptyChecker(ASTContext &ctx, std::vector<SourceLocation> &macroLocs)
     : Ctx(ctx), MacroLocs(macroLocs) { }
 
   bool VisitNullStmt(NullStmt *S) {
-    return isMacroLoc(S->getLeadingEmptyMacroLoc());
+    return isEmptyARCMTMacroStatement(S, MacroLocs, Ctx);
   }
   bool VisitCompoundStmt(CompoundStmt *S) {
     if (S->body_empty())
@@ -103,23 +145,14 @@ class EmptyChecker : public StmtVisitor<EmptyChecker, bool> {
       return false;
     return Visit(S->getSubStmt());
   }
-
-private:
-  bool isMacroLoc(SourceLocation loc) {
-    if (loc.isInvalid()) return false;
-    return MacroLocs.count(loc.getRawEncoding());
-  }
 };
 
 class EmptyStatementsRemover :
                             public RecursiveASTVisitor<EmptyStatementsRemover> {
   MigrationPass &Pass;
-  llvm::DenseSet<unsigned> &MacroLocs;
 
 public:
-  EmptyStatementsRemover(MigrationPass &pass,
-                         llvm::DenseSet<unsigned> &macroLocs)
-    : Pass(pass), MacroLocs(macroLocs) { }
+  EmptyStatementsRemover(MigrationPass &pass) : Pass(pass) { }
 
   bool TraverseStmtExpr(StmtExpr *E) {
     CompoundStmt *S = E->getSubStmt();
@@ -139,17 +172,12 @@ class EmptyStatementsRemover :
     return true;
   }
 
-  bool isMacroLoc(SourceLocation loc) {
-    if (loc.isInvalid()) return false;
-    return MacroLocs.count(loc.getRawEncoding());
-  }
-
   ASTContext &getContext() { return Pass.Ctx; }
 
 private:
   void check(Stmt *S) {
     if (!S) return;
-    if (EmptyChecker(Pass.Ctx, MacroLocs).Visit(S)) {
+    if (EmptyChecker(Pass.Ctx, Pass.ARCMTMacroLocs).Visit(S)) {
       Transaction Trans(Pass.TA);
       Pass.TA.removeStmt(S);
     }
@@ -158,8 +186,8 @@ class EmptyStatementsRemover :
 
 } // anonymous namespace
 
-static bool isBodyEmpty(CompoundStmt *body,
-                        ASTContext &Ctx, llvm::DenseSet<unsigned> &MacroLocs) {
+static bool isBodyEmpty(CompoundStmt *body, ASTContext &Ctx,
+                        std::vector<SourceLocation> &MacroLocs) {
   for (CompoundStmt::body_iterator
          I = body->body_begin(), E = body->body_end(); I != E; ++I)
     if (!EmptyChecker(Ctx, MacroLocs).Visit(*I))
@@ -168,8 +196,7 @@ static bool isBodyEmpty(CompoundStmt *body,
   return true;
 }
 
-static void removeDeallocMethod(MigrationPass &pass,
-                                llvm::DenseSet<unsigned> &MacroLocs) {
+static void removeDeallocMethod(MigrationPass &pass) {
   ASTContext &Ctx = pass.Ctx;
   TransformActions &TA = pass.TA;
   DeclContext *DC = Ctx.getTranslationUnitDecl();
@@ -184,7 +211,7 @@ static void removeDeallocMethod(MigrationPass &pass,
       ObjCMethodDecl *MD = *MI;
       if (MD->getMethodFamily() == OMF_dealloc) {
         if (MD->hasBody() &&
-            isBodyEmpty(MD->getCompoundBody(), Ctx, MacroLocs)) {
+            isBodyEmpty(MD->getCompoundBody(), Ctx, pass.ARCMTMacroLocs)) {
           Transaction Trans(TA);
           TA.remove(MD->getSourceRange());
         }
@@ -195,14 +222,9 @@ static void removeDeallocMethod(MigrationPass &pass,
 }
 
 void trans::removeEmptyStatementsAndDealloc(MigrationPass &pass) {
-  llvm::DenseSet<unsigned> MacroLocs;
-  for (unsigned i = 0, e = pass.ARCMTMacroLocs.size(); i != e; ++i)
-    MacroLocs.insert(pass.ARCMTMacroLocs[i].getRawEncoding());
+  EmptyStatementsRemover(pass).TraverseDecl(pass.Ctx.getTranslationUnitDecl());
 
-  EmptyStatementsRemover(pass, MacroLocs)
-    .TraverseDecl(pass.Ctx.getTranslationUnitDecl());
-
-  removeDeallocMethod(pass, MacroLocs);
+  removeDeallocMethod(pass);
 
   for (unsigned i = 0, e = pass.ARCMTMacroLocs.size(); i != e; ++i) {
     Transaction Trans(pass.TA);
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransProperties.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransProperties.cpp
index 872c95e1a444..ca845b6b330d 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransProperties.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransProperties.cpp
@@ -40,12 +40,12 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
 class PropertiesRewriter {
   MigrationPass &Pass;
+  ObjCImplementationDecl *CurImplD;
 
   struct PropData {
     ObjCPropertyDecl *PropD;
@@ -55,7 +55,7 @@ class PropertiesRewriter {
     PropData(ObjCPropertyDecl *propD) : PropD(propD), IvarD(0), ImplD(0) { }
   };
 
-  typedef llvm::SmallVector<PropData, 2> PropsTy;
+  typedef SmallVector<PropData, 2> PropsTy;
   typedef std::map<unsigned, PropsTy> AtPropDeclsTy;
   AtPropDeclsTy AtProps;
 
@@ -63,6 +63,7 @@ class PropertiesRewriter {
   PropertiesRewriter(MigrationPass &pass) : Pass(pass) { }
 
   void doTransform(ObjCImplementationDecl *D) {
+    CurImplD = D;
     ObjCInterfaceDecl *iface = D->getClassInterface();
     if (!iface)
       return;
@@ -135,8 +136,16 @@ class PropertiesRewriter {
       return;
     }
 
-    if (propAttrs & ObjCPropertyDecl::OBJC_PR_assign)
+    if (propAttrs & ObjCPropertyDecl::OBJC_PR_assign) {
+      if (hasIvarAssignedAPlusOneObject(props)) {
+        rewriteAttribute("assign", "strong", atLoc);
+        return;
+      }
       return rewriteAssign(props, atLoc);
+    }
+
+    if (hasIvarAssignedAPlusOneObject(props))
+      return maybeAddStrongAttr(props, atLoc);
 
     return maybeAddWeakOrUnsafeUnretainedAttr(props, atLoc);
   }
@@ -163,15 +172,15 @@ class PropertiesRewriter {
   void maybeAddWeakOrUnsafeUnretainedAttr(PropsTy &props,
                                           SourceLocation atLoc) const {
     ObjCPropertyDecl::PropertyAttributeKind propAttrs = getPropertyAttrs(props);
-    if ((propAttrs & ObjCPropertyDecl::OBJC_PR_readonly) &&
-        hasNoBackingIvars(props))
-      return;
 
     bool canUseWeak = canApplyWeak(Pass.Ctx, getPropertyType(props));
-    bool addedAttr = addAttribute(canUseWeak ? "weak" : "unsafe_unretained",
-                                  atLoc);
-    if (!addedAttr)
-      canUseWeak = false;
+    if (!(propAttrs & ObjCPropertyDecl::OBJC_PR_readonly) ||
+        !hasAllIvarsBacked(props)) {
+      bool addedAttr = addAttribute(canUseWeak ? "weak" : "unsafe_unretained",
+                                    atLoc);
+      if (!addedAttr)
+        canUseWeak = false;
+    }
 
     for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
       if (isUserDeclared(I->IvarD))
@@ -187,7 +196,26 @@ class PropertiesRewriter {
     }
   }
 
-  bool rewriteAttribute(llvm::StringRef fromAttr, llvm::StringRef toAttr,
+  void maybeAddStrongAttr(PropsTy &props, SourceLocation atLoc) const {
+    ObjCPropertyDecl::PropertyAttributeKind propAttrs = getPropertyAttrs(props);
+
+    if (!(propAttrs & ObjCPropertyDecl::OBJC_PR_readonly) ||
+        !hasAllIvarsBacked(props)) {
+      addAttribute("strong", atLoc);
+    }
+
+    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
+      if (I->ImplD) {
+        Pass.TA.clearDiagnostic(diag::err_arc_assign_property_ownership,
+                                I->ImplD->getLocation());
+        Pass.TA.clearDiagnostic(
+                           diag::err_arc_objc_property_default_assign_on_object,
+                           I->ImplD->getLocation());
+      }
+    }
+  }
+
+  bool rewriteAttribute(StringRef fromAttr, StringRef toAttr,
                         SourceLocation atLoc) const {
     if (atLoc.isMacroID())
       return false;
@@ -199,7 +227,7 @@ class PropertiesRewriter {
 
     // Try to load the file buffer.
     bool invalidTemp = false;
-    llvm::StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
+    StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
     if (invalidTemp)
       return false;
 
@@ -214,7 +242,7 @@ class PropertiesRewriter {
     if (tok.isNot(tok::at)) return false;
     lexer.LexFromRawLexer(tok);
     if (tok.isNot(tok::raw_identifier)) return false;
-    if (llvm::StringRef(tok.getRawIdentifierData(), tok.getLength())
+    if (StringRef(tok.getRawIdentifierData(), tok.getLength())
           != "property")
       return false;
     lexer.LexFromRawLexer(tok);
@@ -226,7 +254,7 @@ class PropertiesRewriter {
 
     while (1) {
       if (tok.isNot(tok::raw_identifier)) return false;
-      llvm::StringRef ident(tok.getRawIdentifierData(), tok.getLength());
+      StringRef ident(tok.getRawIdentifierData(), tok.getLength());
       if (ident == fromAttr) {
         Pass.TA.replaceText(tok.getLocation(), fromAttr, toAttr);
         return true;
@@ -243,7 +271,7 @@ class PropertiesRewriter {
     return false;
   }
 
-  bool addAttribute(llvm::StringRef attr, SourceLocation atLoc) const {
+  bool addAttribute(StringRef attr, SourceLocation atLoc) const {
     if (atLoc.isMacroID())
       return false;
 
@@ -254,7 +282,7 @@ class PropertiesRewriter {
 
     // Try to load the file buffer.
     bool invalidTemp = false;
-    llvm::StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
+    StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
     if (invalidTemp)
       return false;
 
@@ -269,7 +297,7 @@ class PropertiesRewriter {
     if (tok.isNot(tok::at)) return false;
     lexer.LexFromRawLexer(tok);
     if (tok.isNot(tok::raw_identifier)) return false;
-    if (llvm::StringRef(tok.getRawIdentifierData(), tok.getLength())
+    if (StringRef(tok.getRawIdentifierData(), tok.getLength())
           != "property")
       return false;
     lexer.LexFromRawLexer(tok);
@@ -291,6 +319,45 @@ class PropertiesRewriter {
     return true;
   }
 
+  class PlusOneAssign : public RecursiveASTVisitor<PlusOneAssign> {
+    ObjCIvarDecl *Ivar;
+  public:
+    PlusOneAssign(ObjCIvarDecl *D) : Ivar(D) {}
+
+    bool VisitBinAssign(BinaryOperator *E) {
+      Expr *lhs = E->getLHS()->IgnoreParenImpCasts();
+      if (ObjCIvarRefExpr *RE = dyn_cast<ObjCIvarRefExpr>(lhs)) {
+        if (RE->getDecl() != Ivar)
+          return true;
+
+      if (ObjCMessageExpr *
+            ME = dyn_cast<ObjCMessageExpr>(E->getRHS()->IgnoreParenCasts()))
+        if (ME->getMethodFamily() == OMF_retain)
+          return false;
+
+      ImplicitCastExpr *implCE = dyn_cast<ImplicitCastExpr>(E->getRHS());
+      while (implCE && implCE->getCastKind() ==  CK_BitCast)
+        implCE = dyn_cast<ImplicitCastExpr>(implCE->getSubExpr());
+
+      if (implCE && implCE->getCastKind() == CK_ARCConsumeObject)
+        return false;
+      }
+
+      return true;
+    }
+  };
+
+  bool hasIvarAssignedAPlusOneObject(PropsTy &props) const {
+    for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
+      PlusOneAssign oneAssign(I->IvarD);
+      bool notFound = oneAssign.TraverseDecl(CurImplD);
+      if (!notFound)
+        return true;
+    }
+
+    return false;
+  }
+
   bool hasIvarWithExplicitOwnership(PropsTy &props) const {
     for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I) {
       if (isUserDeclared(I->IvarD)) {
@@ -305,9 +372,9 @@ class PropertiesRewriter {
     return false;    
   }
 
-  bool hasNoBackingIvars(PropsTy &props) const {
+  bool hasAllIvarsBacked(PropsTy &props) const {
     for (PropsTy::iterator I = props.begin(), E = props.end(); I != E; ++I)
-      if (I->IvarD)
+      if (!isUserDeclared(I->IvarD))
         return false;
 
     return true;
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp
index ed6ed0adfdf2..394f8480e112 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransRetainReleaseDealloc.cpp
@@ -25,7 +25,6 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
@@ -130,10 +129,9 @@ class RetainReleaseDeallocRemover :
       // Change the -release to "receiver = nil" in a finally to avoid a leak
       // when an exception is thrown.
       Pass.TA.replace(E->getSourceRange(), rec->getSourceRange());
-      if (Pass.Ctx.Idents.get("nil").hasMacroDefinition())
-        Pass.TA.insertAfterToken(rec->getLocEnd(), " = nil");
-      else
-        Pass.TA.insertAfterToken(rec->getLocEnd(), " = 0");
+      std::string str = " = ";
+      str += getNilString(Pass.Ctx);
+      Pass.TA.insertAfterToken(rec->getLocEnd(), str);
       return true;
     }
 
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
index 1cacd6d84e6d..69fb2e8949e4 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
@@ -36,25 +36,32 @@
 #include "Internals.h"
 #include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include "clang/Sema/SemaDiagnostic.h"
+#include "clang/AST/ParentMap.h"
 #include "clang/Basic/SourceManager.h"
 
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
 class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
   MigrationPass &Pass;
   IdentifierInfo *SelfII;
+  llvm::OwningPtr<ParentMap> StmtMap;
+
 public:
   UnbridgedCastRewriter(MigrationPass &pass) : Pass(pass) {
     SelfII = &Pass.Ctx.Idents.get("self");
   }
 
+  void transformBody(Stmt *body) {
+    StmtMap.reset(new ParentMap(body));
+    TraverseStmt(body);
+  }
+
   bool VisitCastExpr(CastExpr *E) {
-    if (E->getCastKind() != CK_AnyPointerToObjCPointerCast
+    if (E->getCastKind() != CK_CPointerToObjCPointerCast
         && E->getCastKind() != CK_BitCast)
       return true;
 
@@ -139,13 +146,21 @@ class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
   }
 
   void rewriteToBridgedCast(CastExpr *E, ObjCBridgeCastKind Kind) {
+    Transaction Trans(Pass.TA);
+    rewriteToBridgedCast(E, Kind, Trans);
+  }
+
+  void rewriteToBridgedCast(CastExpr *E, ObjCBridgeCastKind Kind,
+                            Transaction &Trans) {
     TransformActions &TA = Pass.TA;
 
     // We will remove the compiler diagnostic.
     if (!TA.hasDiagnostic(diag::err_arc_mismatched_cast,
                           diag::err_arc_cast_requires_bridge,
-                          E->getLocStart()))
+                          E->getLocStart())) {
+      Trans.abort();
       return;
+    }
 
     StringRef bridge;
     switch(Kind) {
@@ -157,7 +172,6 @@ class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
       bridge = "__bridge_retained "; break;
     }
 
-    Transaction Trans(TA);
     TA.clearDiagnostic(diag::err_arc_mismatched_cast,
                        diag::err_arc_cast_requires_bridge,
                        E->getLocStart());
@@ -168,7 +182,7 @@ class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
       llvm::SmallString<128> newCast;
       newCast += '(';
       newCast += bridge;
-      newCast += E->getType().getAsString(Pass.Ctx.PrintingPolicy);
+      newCast += E->getType().getAsString(Pass.Ctx.getPrintingPolicy());
       newCast += ')';
 
       if (isa<ParenExpr>(E->getSubExpr())) {
@@ -181,16 +195,111 @@ class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
     }
   }
 
+  void rewriteCastForCFRetain(CastExpr *castE, CallExpr *callE) {
+    Transaction Trans(Pass.TA);
+    Pass.TA.replace(callE->getSourceRange(), callE->getArg(0)->getSourceRange());
+    rewriteToBridgedCast(castE, OBC_BridgeRetained, Trans);
+  }
+
   void transformObjCToNonObjCCast(CastExpr *E) {
     if (isSelf(E->getSubExpr()))
       return rewriteToBridgedCast(E, OBC_Bridge);
+
+    CallExpr *callE;
+    if (isPassedToCFRetain(E, callE))
+      return rewriteCastForCFRetain(E, callE);
+
+    ObjCMethodFamily family = getFamilyOfMessage(E->getSubExpr());
+    if (family == OMF_retain)
+      return rewriteToBridgedCast(E, OBC_BridgeRetained);
+
+    if (family == OMF_autorelease || family == OMF_release) {
+      std::string err = "it is not safe to cast to '";
+      err += E->getType().getAsString(Pass.Ctx.getPrintingPolicy());
+      err += "' the result of '";
+      err += family == OMF_autorelease ? "autorelease" : "release";
+      err += "' message; a __bridge cast may result in a pointer to a "
+          "destroyed object and a __bridge_retained may leak the object";
+      Pass.TA.reportError(err, E->getLocStart(),
+                          E->getSubExpr()->getSourceRange());
+      Stmt *parent = E;
+      do {
+        parent = StmtMap->getParentIgnoreParenImpCasts(parent);
+      } while (parent && isa<ExprWithCleanups>(parent));
+
+      if (ReturnStmt *retS = dyn_cast_or_null<ReturnStmt>(parent)) {
+        std::string note = "remove the cast and change return type of function "
+            "to '";
+        note += E->getSubExpr()->getType().getAsString(Pass.Ctx.getPrintingPolicy());
+        note += "' to have the object automatically autoreleased";
+        Pass.TA.reportNote(note, retS->getLocStart());
+      }
+    }
+
+    if (ImplicitCastExpr *implCE = dyn_cast<ImplicitCastExpr>(E->getSubExpr())){
+      if (implCE->getCastKind() == CK_ARCConsumeObject)
+        return rewriteToBridgedCast(E, OBC_BridgeRetained);
+      if (implCE->getCastKind() == CK_ARCReclaimReturnedObject)
+        return rewriteToBridgedCast(E, OBC_Bridge);
+    }
+
+    bool isConsumed = false;
+    if (isPassedToCParamWithKnownOwnership(E, isConsumed))
+      return rewriteToBridgedCast(E, isConsumed ? OBC_BridgeRetained
+                                                : OBC_Bridge);
   }
 
-  bool isSelf(Expr *E) {
+  static ObjCMethodFamily getFamilyOfMessage(Expr *E) {
+    E = E->IgnoreParenCasts();
+    if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E))
+      return ME->getMethodFamily();
+
+    return OMF_None;
+  }
+
+  bool isPassedToCFRetain(Expr *E, CallExpr *&callE) const {
+    if ((callE = dyn_cast_or_null<CallExpr>(
+                                     StmtMap->getParentIgnoreParenImpCasts(E))))
+      if (FunctionDecl *
+            FD = dyn_cast_or_null<FunctionDecl>(callE->getCalleeDecl()))
+        if (FD->getName() == "CFRetain" && FD->getNumParams() == 1 &&
+            FD->getParent()->isTranslationUnit() &&
+            FD->getLinkage() == ExternalLinkage)
+          return true;
+
+    return false;
+  }
+
+  bool isPassedToCParamWithKnownOwnership(Expr *E, bool &isConsumed) const {
+    if (CallExpr *callE = dyn_cast_or_null<CallExpr>(
+                                     StmtMap->getParentIgnoreParenImpCasts(E)))
+      if (FunctionDecl *
+            FD = dyn_cast_or_null<FunctionDecl>(callE->getCalleeDecl())) {
+        unsigned i = 0;
+        for (unsigned e = callE->getNumArgs(); i != e; ++i) {
+          Expr *arg = callE->getArg(i);
+          if (arg == E || arg->IgnoreParenImpCasts() == E)
+            break;
+        }
+        if (i < callE->getNumArgs()) {
+          ParmVarDecl *PD = FD->getParamDecl(i);
+          if (PD->getAttr<CFConsumedAttr>()) {
+            isConsumed = true;
+            return true;
+          }
+        }
+      }
+
+    return false;
+  }
+
+  bool isSelf(Expr *E) const {
     E = E->IgnoreParenLValueCasts();
     if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-      if (DRE->getDecl()->getIdentifier() == SelfII)
-        return true;
+      if (ImplicitParamDecl *IPD = dyn_cast<ImplicitParamDecl>(DRE->getDecl()))
+        if (IPD->getIdentifier() == SelfII)
+          return true;
+
     return false;
   }
 };
@@ -198,6 +307,6 @@ class UnbridgedCastRewriter : public RecursiveASTVisitor<UnbridgedCastRewriter>{
 } // end anonymous namespace
 
 void trans::rewriteUnbridgedCasts(MigrationPass &pass) {
-  UnbridgedCastRewriter trans(pass);
+  BodyTransform<UnbridgedCastRewriter> trans(pass);
   trans.TraverseDecl(pass.Ctx.getTranslationUnitDecl());
 }
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp
index 1019ab4ff1f6..e2aa6ff93cf3 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransUnusedInitDelegate.cpp
@@ -27,7 +27,6 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp
index 07ccf70d4dfb..1dbe81114947 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransZeroOutPropsInDealloc.cpp
@@ -19,7 +19,6 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 namespace {
 
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/TransformActions.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/TransformActions.cpp
index c99940b494b9..ec676e909a3b 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/TransformActions.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/TransformActions.cpp
@@ -13,10 +13,8 @@
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/DenseSet.h"
 #include <map>
-
 using namespace clang;
 using namespace arcmt;
-using llvm::StringRef;
 
 namespace {
 
@@ -46,9 +44,9 @@ class TransformActionsImpl {
     ActionKind Kind;
     SourceLocation Loc;
     SourceRange R1, R2;
-    llvm::StringRef Text1, Text2;
+    StringRef Text1, Text2;
     Stmt *S;
-    llvm::SmallVector<unsigned, 2> DiagIDs;
+    SmallVector<unsigned, 2> DiagIDs;
   };
 
   std::vector<ActionData> CachedActions;
@@ -70,11 +68,11 @@ class TransformActionsImpl {
       SourceLocation beginLoc = range.getBegin(), endLoc = range.getEnd();
       assert(beginLoc.isValid() && endLoc.isValid());
       if (range.isTokenRange()) {
-        Begin = FullSourceLoc(srcMgr.getInstantiationLoc(beginLoc), srcMgr);
+        Begin = FullSourceLoc(srcMgr.getExpansionLoc(beginLoc), srcMgr);
         End = FullSourceLoc(getLocForEndOfToken(endLoc, srcMgr, PP), srcMgr);
       } else {
-        Begin = FullSourceLoc(srcMgr.getInstantiationLoc(beginLoc), srcMgr);
-        End = FullSourceLoc(srcMgr.getInstantiationLoc(endLoc), srcMgr);
+        Begin = FullSourceLoc(srcMgr.getExpansionLoc(beginLoc), srcMgr);
+        End = FullSourceLoc(srcMgr.getExpansionLoc(endLoc), srcMgr);
       }
       assert(Begin.isValid() && End.isValid());
     } 
@@ -104,7 +102,7 @@ class TransformActionsImpl {
     }
   };
 
-  typedef llvm::SmallVector<StringRef, 2> TextsVec;
+  typedef SmallVector<StringRef, 2> TextsVec;
   typedef std::map<FullSourceLoc, TextsVec, FullSourceLoc::BeforeThanCompare>
       InsertsMap;
   InsertsMap Inserts;
@@ -130,19 +128,19 @@ class TransformActionsImpl {
 
   bool isInTransaction() const { return IsInTransaction; }
 
-  void insert(SourceLocation loc, llvm::StringRef text);
-  void insertAfterToken(SourceLocation loc, llvm::StringRef text);
+  void insert(SourceLocation loc, StringRef text);
+  void insertAfterToken(SourceLocation loc, StringRef text);
   void remove(SourceRange range);
   void removeStmt(Stmt *S);
-  void replace(SourceRange range, llvm::StringRef text);
+  void replace(SourceRange range, StringRef text);
   void replace(SourceRange range, SourceRange replacementRange);
-  void replaceStmt(Stmt *S, llvm::StringRef text);
-  void replaceText(SourceLocation loc, llvm::StringRef text,
-                   llvm::StringRef replacementText);
+  void replaceStmt(Stmt *S, StringRef text);
+  void replaceText(SourceLocation loc, StringRef text,
+                   StringRef replacementText);
   void increaseIndentation(SourceRange range,
                            SourceLocation parentIndent);
 
-  bool clearDiagnostic(llvm::ArrayRef<unsigned> IDs, SourceRange range);
+  bool clearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
 
   void applyRewrites(TransformActions::RewriteReceiver &receiver);
 
@@ -151,17 +149,17 @@ class TransformActionsImpl {
   bool canInsertAfterToken(SourceLocation loc);
   bool canRemoveRange(SourceRange range);
   bool canReplaceRange(SourceRange range, SourceRange replacementRange);
-  bool canReplaceText(SourceLocation loc, llvm::StringRef text);
+  bool canReplaceText(SourceLocation loc, StringRef text);
 
   void commitInsert(SourceLocation loc, StringRef text);
   void commitInsertAfterToken(SourceLocation loc, StringRef text);
   void commitRemove(SourceRange range);
   void commitRemoveStmt(Stmt *S);
   void commitReplace(SourceRange range, SourceRange replacementRange);
-  void commitReplaceText(SourceLocation loc, llvm::StringRef text,
-                         llvm::StringRef replacementText);
+  void commitReplaceText(SourceLocation loc, StringRef text,
+                         StringRef replacementText);
   void commitIncreaseIndentation(SourceRange range,SourceLocation parentIndent);
-  void commitClearDiagnostic(llvm::ArrayRef<unsigned> IDs, SourceRange range);
+  void commitClearDiagnostic(ArrayRef<unsigned> IDs, SourceRange range);
 
   void addRemoval(CharSourceRange range);
   void addInsertion(SourceLocation loc, StringRef text);
@@ -364,7 +362,7 @@ void TransformActionsImpl::increaseIndentation(SourceRange range,
   CachedActions.push_back(data);
 }
 
-bool TransformActionsImpl::clearDiagnostic(llvm::ArrayRef<unsigned> IDs,
+bool TransformActionsImpl::clearDiagnostic(ArrayRef<unsigned> IDs,
                                            SourceRange range) {
   assert(IsInTransaction && "Actions only allowed during a transaction");
   if (!CapturedDiags.hasDiagnostic(IDs, range))
@@ -383,7 +381,7 @@ bool TransformActionsImpl::canInsert(SourceLocation loc) {
     return false;
 
   SourceManager &SM = Ctx.getSourceManager();
-  if (SM.isInSystemHeader(SM.getInstantiationLoc(loc)))
+  if (SM.isInSystemHeader(SM.getExpansionLoc(loc)))
     return false;
 
   if (loc.isFileID())
@@ -396,7 +394,7 @@ bool TransformActionsImpl::canInsertAfterToken(SourceLocation loc) {
     return false;
 
   SourceManager &SM = Ctx.getSourceManager();
-  if (SM.isInSystemHeader(SM.getInstantiationLoc(loc)))
+  if (SM.isInSystemHeader(SM.getExpansionLoc(loc)))
     return false;
 
   if (loc.isFileID())
@@ -418,14 +416,14 @@ bool TransformActionsImpl::canReplaceText(SourceLocation loc, StringRef text) {
     return false;
 
   SourceManager &SM = Ctx.getSourceManager();
-  loc = SM.getInstantiationLoc(loc);
+  loc = SM.getExpansionLoc(loc);
 
   // Break down the source location.
   std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
 
   // Try to load the file buffer.
   bool invalidTemp = false;
-  llvm::StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
+  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
   if (invalidTemp)
     return false;
 
@@ -479,9 +477,9 @@ void TransformActionsImpl::commitReplaceText(SourceLocation loc,
                                              StringRef text,
                                              StringRef replacementText) {
   SourceManager &SM = Ctx.getSourceManager();
-  loc = SM.getInstantiationLoc(loc);
+  loc = SM.getExpansionLoc(loc);
   // canReplaceText already checked if loc points at text.
-  SourceLocation afterText = loc.getFileLocWithOffset(text.size());
+  SourceLocation afterText = loc.getLocWithOffset(text.size());
 
   addRemoval(CharSourceRange::getCharRange(loc, afterText));
   commitInsert(loc, replacementText);  
@@ -493,17 +491,17 @@ void TransformActionsImpl::commitIncreaseIndentation(SourceRange range,
   IndentationRanges.push_back(
                  std::make_pair(CharRange(CharSourceRange::getTokenRange(range),
                                           SM, PP),
-                                SM.getInstantiationLoc(parentIndent)));
+                                SM.getExpansionLoc(parentIndent)));
 }
 
-void TransformActionsImpl::commitClearDiagnostic(llvm::ArrayRef<unsigned> IDs,
+void TransformActionsImpl::commitClearDiagnostic(ArrayRef<unsigned> IDs,
                                                  SourceRange range) {
   CapturedDiags.clearDiagnostic(IDs, range);
 }
 
 void TransformActionsImpl::addInsertion(SourceLocation loc, StringRef text) {
   SourceManager &SM = Ctx.getSourceManager();
-  loc = SM.getInstantiationLoc(loc);
+  loc = SM.getExpansionLoc(loc);
   for (std::list<CharRange>::reverse_iterator
          I = Removals.rbegin(), E = Removals.rend(); I != E; ++I) {
     if (!SM.isBeforeInTranslationUnit(loc, I->End))
@@ -591,16 +589,16 @@ SourceLocation TransformActionsImpl::getLocForEndOfToken(SourceLocation loc,
                                                          SourceManager &SM,
                                                          Preprocessor &PP) {
   if (loc.isMacroID())
-    loc = SM.getInstantiationRange(loc).second;
+    loc = SM.getExpansionRange(loc).second;
   return PP.getLocForEndOfToken(loc);
 }
 
 TransformActions::RewriteReceiver::~RewriteReceiver() { }
 
-TransformActions::TransformActions(Diagnostic &diag,
+TransformActions::TransformActions(DiagnosticsEngine &diag,
                                    CapturedDiagList &capturedDiags,
                                    ASTContext &ctx, Preprocessor &PP)
-  : Diags(diag), CapturedDiags(capturedDiags) {
+  : Diags(diag), CapturedDiags(capturedDiags), ReportedErrors(false) {
   Impl = new TransformActionsImpl(capturedDiags, ctx, PP);
 }
 
@@ -621,12 +619,12 @@ void TransformActions::abortTransaction() {
 }
 
 
-void TransformActions::insert(SourceLocation loc, llvm::StringRef text) {
+void TransformActions::insert(SourceLocation loc, StringRef text) {
   static_cast<TransformActionsImpl*>(Impl)->insert(loc, text);
 }
 
 void TransformActions::insertAfterToken(SourceLocation loc,
-                                        llvm::StringRef text) {
+                                        StringRef text) {
   static_cast<TransformActionsImpl*>(Impl)->insertAfterToken(loc, text);
 }
 
@@ -638,7 +636,7 @@ void TransformActions::removeStmt(Stmt *S) {
   static_cast<TransformActionsImpl*>(Impl)->removeStmt(S);
 }
 
-void TransformActions::replace(SourceRange range, llvm::StringRef text) {
+void TransformActions::replace(SourceRange range, StringRef text) {
   static_cast<TransformActionsImpl*>(Impl)->replace(range, text);
 }
 
@@ -647,12 +645,12 @@ void TransformActions::replace(SourceRange range,
   static_cast<TransformActionsImpl*>(Impl)->replace(range, replacementRange);
 }
 
-void TransformActions::replaceStmt(Stmt *S, llvm::StringRef text) {
+void TransformActions::replaceStmt(Stmt *S, StringRef text) {
   static_cast<TransformActionsImpl*>(Impl)->replaceStmt(S, text);
 }
 
-void TransformActions::replaceText(SourceLocation loc, llvm::StringRef text,
-                                   llvm::StringRef replacementText) {
+void TransformActions::replaceText(SourceLocation loc, StringRef text,
+                                   StringRef replacementText) {
   static_cast<TransformActionsImpl*>(Impl)->replaceText(loc, text,
                                                         replacementText);
 }
@@ -663,7 +661,7 @@ void TransformActions::increaseIndentation(SourceRange range,
                                                                 parentIndent);
 }
 
-bool TransformActions::clearDiagnostic(llvm::ArrayRef<unsigned> IDs,
+bool TransformActions::clearDiagnostic(ArrayRef<unsigned> IDs,
                                        SourceRange range) {
   return static_cast<TransformActionsImpl*>(Impl)->clearDiagnostic(IDs, range);
 }
@@ -672,7 +670,7 @@ void TransformActions::applyRewrites(RewriteReceiver &receiver) {
   static_cast<TransformActionsImpl*>(Impl)->applyRewrites(receiver);
 }
 
-void TransformActions::reportError(llvm::StringRef error, SourceLocation loc,
+void TransformActions::reportError(StringRef error, SourceLocation loc,
                                    SourceRange range) {
   assert(!static_cast<TransformActionsImpl*>(Impl)->isInTransaction() &&
          "Errors should be emitted out of a transaction");
@@ -683,9 +681,10 @@ void TransformActions::reportError(llvm::StringRef error, SourceLocation loc,
      = Diags.getDiagnosticIDs()->getCustomDiagID(DiagnosticIDs::Error,
                                                  rewriteErr);
   Diags.Report(loc, diagID) << range;
+  ReportedErrors = true;
 }
 
-void TransformActions::reportNote(llvm::StringRef note, SourceLocation loc,
+void TransformActions::reportNote(StringRef note, SourceLocation loc,
                                   SourceRange range) {
   assert(!static_cast<TransformActionsImpl*>(Impl)->isInTransaction() &&
          "Errors should be emitted out of a transaction");
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.cpp b/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.cpp
index 7bd95e54bc27..4244fafe07d4 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.cpp
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.cpp
@@ -23,7 +23,6 @@
 using namespace clang;
 using namespace arcmt;
 using namespace trans;
-using llvm::StringRef;
 
 //===----------------------------------------------------------------------===//
 // Helpers.
@@ -92,11 +91,23 @@ bool trans::canApplyWeak(ASTContext &Ctx, QualType type) {
 /// source location will be invalid.
 SourceLocation trans::findLocationAfterSemi(SourceLocation loc,
                                             ASTContext &Ctx) {
+  SourceLocation SemiLoc = findSemiAfterLocation(loc, Ctx);
+  if (SemiLoc.isInvalid())
+    return SourceLocation();
+  return SemiLoc.getLocWithOffset(1);
+}
+
+/// \brief \arg Loc is the end of a statement range. This returns the location
+/// of the semicolon following the statement.
+/// If no semicolon is found or the location is inside a macro, the returned
+/// source location will be invalid.
+SourceLocation trans::findSemiAfterLocation(SourceLocation loc,
+                                            ASTContext &Ctx) {
   SourceManager &SM = Ctx.getSourceManager();
   if (loc.isMacroID()) {
     if (!Lexer::isAtEndOfMacroExpansion(loc, SM, Ctx.getLangOptions()))
       return SourceLocation();
-    loc = SM.getInstantiationRange(loc).second;
+    loc = SM.getExpansionRange(loc).second;
   }
   loc = Lexer::getLocForEndOfToken(loc, /*Offset=*/0, SM, Ctx.getLangOptions());
 
@@ -105,7 +116,7 @@ SourceLocation trans::findLocationAfterSemi(SourceLocation loc,
 
   // Try to load the file buffer.
   bool invalidTemp = false;
-  llvm::StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
+  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
   if (invalidTemp)
     return SourceLocation();
 
@@ -120,7 +131,7 @@ SourceLocation trans::findLocationAfterSemi(SourceLocation loc,
   if (tok.isNot(tok::semi))
     return SourceLocation();
 
-  return tok.getLocation().getFileLocWithOffset(1);
+  return tok.getLocation();
 }
 
 bool trans::hasSideEffects(Expr *E, ASTContext &Ctx) {
@@ -155,7 +166,8 @@ bool trans::hasSideEffects(Expr *E, ASTContext &Ctx) {
 bool trans::isGlobalVar(Expr *E) {
   E = E->IgnoreParenCasts();
   if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-    return DRE->getDecl()->getDeclContext()->isFileContext();
+    return DRE->getDecl()->getDeclContext()->isFileContext() &&
+           DRE->getDecl()->getLinkage() == ExternalLinkage;
   if (ConditionalOperator *condOp = dyn_cast<ConditionalOperator>(E))
     return isGlobalVar(condOp->getTrueExpr()) &&
            isGlobalVar(condOp->getFalseExpr());
@@ -163,6 +175,13 @@ bool trans::isGlobalVar(Expr *E) {
   return false;  
 }
 
+StringRef trans::getNilString(ASTContext &Ctx) {
+  if (Ctx.Idents.get("nil").hasMacroDefinition())
+    return "nil";
+  else
+    return "0";
+}
+
 namespace {
 
 class ReferenceClear : public RecursiveASTVisitor<ReferenceClear> {
@@ -283,6 +302,7 @@ static void independentTransforms(MigrationPass &pass) {
   makeAssignARCSafe(pass);
   rewriteUnbridgedCasts(pass);
   rewriteBlockObjCVariable(pass);
+  checkAPIUses(pass);
 }
 
 std::vector<TransformFn> arcmt::getAllTransformations() {
diff --git a/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.h b/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.h
index b47d6d8e9b8b..5e4db56dc894 100644
--- a/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.h
+++ b/contrib/llvm/tools/clang/lib/ARCMigrate/Transforms.h
@@ -37,6 +37,7 @@ void removeZeroOutPropsInDealloc(MigrationPass &pass);
 void rewriteProperties(MigrationPass &pass);
 void rewriteBlockObjCVariable(MigrationPass &pass);
 void rewriteUnusedInitDelegate(MigrationPass &pass);
+void checkAPIUses(MigrationPass &pass);
 
 void removeEmptyStatementsAndDealloc(MigrationPass &pass);
 
@@ -53,9 +54,16 @@ bool canApplyWeak(ASTContext &Ctx, QualType type);
 /// source location will be invalid.
 SourceLocation findLocationAfterSemi(SourceLocation loc, ASTContext &Ctx);
 
+/// \brief \arg Loc is the end of a statement range. This returns the location
+/// of the semicolon following the statement.
+/// If no semicolon is found or the location is inside a macro, the returned
+/// source location will be invalid.
+SourceLocation findSemiAfterLocation(SourceLocation loc, ASTContext &Ctx);
+
 bool hasSideEffects(Expr *E, ASTContext &Ctx);
 bool isGlobalVar(Expr *E);
-
+/// \brief Returns "nil" or "0" if 'nil' macro is not actually defined.
+StringRef getNilString(ASTContext &Ctx);
 
 template <typename BODY_TRANS>
 class BodyTransform : public RecursiveASTVisitor<BodyTransform<BODY_TRANS> > {
@@ -65,7 +73,8 @@ class BodyTransform : public RecursiveASTVisitor<BodyTransform<BODY_TRANS> > {
   BodyTransform(MigrationPass &pass) : Pass(pass) { }
 
   bool TraverseStmt(Stmt *rootS) {
-    BODY_TRANS(Pass).transformBody(rootS);
+    if (rootS)
+      BODY_TRANS(Pass).transformBody(rootS);
     return true;
   }
 };
diff --git a/contrib/llvm/tools/clang/lib/AST/APValue.cpp b/contrib/llvm/tools/clang/lib/AST/APValue.cpp
index ebe99b12cdab..6f63a32dd2ed 100644
--- a/contrib/llvm/tools/clang/lib/AST/APValue.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/APValue.cpp
@@ -13,7 +13,10 @@
 
 #include "clang/AST/APValue.h"
 #include "clang/AST/CharUnits.h"
+#include "clang/Basic/Diagnostic.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 
 namespace {
@@ -89,9 +92,9 @@ static double GetApproxValue(const llvm::APFloat &F) {
   return V.convertToDouble();
 }
 
-void APValue::print(llvm::raw_ostream &OS) const {
+void APValue::print(raw_ostream &OS) const {
   switch (getKind()) {
-  default: assert(0 && "Unknown APValue kind!");
+  default: llvm_unreachable("Unknown APValue kind!");
   case Uninitialized:
     OS << "Uninitialized";
     return;
@@ -118,6 +121,49 @@ void APValue::print(llvm::raw_ostream &OS) const {
   }
 }
 
+static void WriteShortAPValueToStream(raw_ostream& Out,
+                                      const APValue& V) {
+  switch (V.getKind()) {
+  default: llvm_unreachable("Unknown APValue kind!");
+  case APValue::Uninitialized:
+    Out << "Uninitialized";
+    break;
+  case APValue::Int:
+    Out << V.getInt();
+    break;
+  case APValue::Float:
+    Out << GetApproxValue(V.getFloat());
+    break;
+  case APValue::Vector:
+    Out << '[';
+    WriteShortAPValueToStream(Out, V.getVectorElt(0));
+    for (unsigned i = 1; i != V.getVectorLength(); ++i) {
+      Out << ", ";
+      WriteShortAPValueToStream(Out, V.getVectorElt(i));
+    }
+    Out << ']';
+    break;
+  case APValue::ComplexInt:
+    Out << V.getComplexIntReal() << "+" << V.getComplexIntImag() << "i";
+    break;
+  case APValue::ComplexFloat:
+    Out << GetApproxValue(V.getComplexFloatReal()) << "+"
+        << GetApproxValue(V.getComplexFloatImag()) << "i";
+    break;
+  case APValue::LValue:
+    Out << "LValue: <todo>";
+    break;
+  }
+}
+
+const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
+                                           const APValue &V) {
+  llvm::SmallString<64> Buffer;
+  llvm::raw_svector_ostream Out(Buffer);
+  WriteShortAPValueToStream(Out, V);
+  return DB << Out.str();
+}
+
 const Expr* APValue::getLValueBase() const {
   assert(isLValue() && "Invalid accessor");
   return ((const LV*)(const void*)Data)->Base;
diff --git a/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp b/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp
index 6eada6e22f4e..462428086ddd 100644
--- a/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp
@@ -30,6 +30,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Capacity.h"
 #include "CXXABI.h"
 #include <map>
 
@@ -49,7 +50,7 @@ unsigned ASTContext::NumImplicitDestructors;
 unsigned ASTContext::NumImplicitDestructorsDeclared;
 
 enum FloatingRank {
-  FloatRank, DoubleRank, LongDoubleRank
+  HalfRank, FloatRank, DoubleRank, LongDoubleRank
 };
 
 void 
@@ -104,7 +105,7 @@ ASTContext::getCanonicalTemplateTemplateParmDecl(
   
   // Build a canonical template parameter list.
   TemplateParameterList *Params = TTP->getTemplateParameters();
-  llvm::SmallVector<NamedDecl *, 4> CanonParams;
+  SmallVector<NamedDecl *, 4> CanonParams;
   CanonParams.reserve(Params->size());
   for (TemplateParameterList::const_iterator P = Params->begin(), 
                                           PEnd = Params->end();
@@ -123,8 +124,8 @@ ASTContext::getCanonicalTemplateTemplateParmDecl(
       TypeSourceInfo *TInfo = getTrivialTypeSourceInfo(T);
       NonTypeTemplateParmDecl *Param;
       if (NTTP->isExpandedParameterPack()) {
-        llvm::SmallVector<QualType, 2> ExpandedTypes;
-        llvm::SmallVector<TypeSourceInfo *, 2> ExpandedTInfos;
+        SmallVector<QualType, 2> ExpandedTypes;
+        SmallVector<TypeSourceInfo *, 2> ExpandedTInfos;
         for (unsigned I = 0, N = NTTP->getNumExpansionTypes(); I != N; ++I) {
           ExpandedTypes.push_back(getCanonicalType(NTTP->getExpansionType(I)));
           ExpandedTInfos.push_back(
@@ -195,7 +196,7 @@ CXXABI *ASTContext::createCXXABI(const TargetInfo &T) {
   return 0;
 }
 
-static const LangAS::Map &getAddressSpaceMap(const TargetInfo &T,
+static const LangAS::Map *getAddressSpaceMap(const TargetInfo &T,
                                              const LangOptions &LOpts) {
   if (LOpts.FakeAddressSpaceMap) {
     // The fake address space map must have a distinct entry for each
@@ -205,41 +206,46 @@ static const LangAS::Map &getAddressSpaceMap(const TargetInfo &T,
       2, // opencl_local
       3  // opencl_constant
     };
-    return FakeAddrSpaceMap;
+    return &FakeAddrSpaceMap;
   } else {
-    return T.getAddressSpaceMap();
+    return &T.getAddressSpaceMap();
   }
 }
 
-ASTContext::ASTContext(const LangOptions& LOpts, SourceManager &SM,
-                       const TargetInfo &t,
+ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
+                       const TargetInfo *t,
                        IdentifierTable &idents, SelectorTable &sels,
                        Builtin::Context &builtins,
-                       unsigned size_reserve) :
-  FunctionProtoTypes(this_()),
-  TemplateSpecializationTypes(this_()),
-  DependentTemplateSpecializationTypes(this_()),
-  SubstTemplateTemplateParmPacks(this_()),
-  GlobalNestedNameSpecifier(0), IsInt128Installed(false),
-  CFConstantStringTypeDecl(0), NSConstantStringTypeDecl(0),
-  ObjCFastEnumerationStateTypeDecl(0), FILEDecl(0), 
-  jmp_bufDecl(0), sigjmp_bufDecl(0), BlockDescriptorType(0), 
-  BlockDescriptorExtendedType(0), cudaConfigureCallDecl(0),
-  NullTypeSourceInfo(QualType()),
-  SourceMgr(SM), LangOpts(LOpts), ABI(createCXXABI(t)),
-  AddrSpaceMap(getAddressSpaceMap(t, LOpts)), Target(t),
-  Idents(idents), Selectors(sels),
-  BuiltinInfo(builtins),
-  DeclarationNames(*this),
-  ExternalSource(0), Listener(0), PrintingPolicy(LOpts),
-  LastSDM(0, 0),
-  UniqueBlockByRefTypeID(0) {
-  ObjCIdRedefinitionType = QualType();
-  ObjCClassRedefinitionType = QualType();
-  ObjCSelRedefinitionType = QualType();    
+                       unsigned size_reserve,
+                       bool DelayInitialization) 
+  : FunctionProtoTypes(this_()),
+    TemplateSpecializationTypes(this_()),
+    DependentTemplateSpecializationTypes(this_()),
+    SubstTemplateTemplateParmPacks(this_()),
+    GlobalNestedNameSpecifier(0), 
+    Int128Decl(0), UInt128Decl(0),
+    ObjCIdDecl(0), ObjCSelDecl(0), ObjCClassDecl(0),
+    CFConstantStringTypeDecl(0), ObjCInstanceTypeDecl(0),
+    FILEDecl(0), 
+    jmp_bufDecl(0), sigjmp_bufDecl(0), BlockDescriptorType(0), 
+    BlockDescriptorExtendedType(0), cudaConfigureCallDecl(0),
+    NullTypeSourceInfo(QualType()),
+    SourceMgr(SM), LangOpts(LOpts), 
+    AddrSpaceMap(0), Target(t), PrintingPolicy(LOpts),
+    Idents(idents), Selectors(sels),
+    BuiltinInfo(builtins),
+    DeclarationNames(*this),
+    ExternalSource(0), Listener(0),
+    LastSDM(0, 0),
+    UniqueBlockByRefTypeID(0) 
+{
   if (size_reserve > 0) Types.reserve(size_reserve);
   TUDecl = TranslationUnitDecl::Create(*this);
-  InitBuiltinTypes();
+  
+  if (!DelayInitialization) {
+    assert(t && "No target supplied for ASTContext initialization");
+    InitBuiltinTypes(*t);
+  }
 }
 
 ASTContext::~ASTContext() {
@@ -347,6 +353,33 @@ void ASTContext::PrintStats() const {
   BumpAlloc.PrintStats();
 }
 
+TypedefDecl *ASTContext::getInt128Decl() const {
+  if (!Int128Decl) {
+    TypeSourceInfo *TInfo = getTrivialTypeSourceInfo(Int128Ty);
+    Int128Decl = TypedefDecl::Create(const_cast<ASTContext &>(*this), 
+                                     getTranslationUnitDecl(),
+                                     SourceLocation(),
+                                     SourceLocation(),
+                                     &Idents.get("__int128_t"),
+                                     TInfo);
+  }
+  
+  return Int128Decl;
+}
+
+TypedefDecl *ASTContext::getUInt128Decl() const {
+  if (!UInt128Decl) {
+    TypeSourceInfo *TInfo = getTrivialTypeSourceInfo(UnsignedInt128Ty);
+    UInt128Decl = TypedefDecl::Create(const_cast<ASTContext &>(*this), 
+                                     getTranslationUnitDecl(),
+                                     SourceLocation(),
+                                     SourceLocation(),
+                                     &Idents.get("__uint128_t"),
+                                     TInfo);
+  }
+  
+  return UInt128Decl;
+}
 
 void ASTContext::InitBuiltinType(CanQualType &R, BuiltinType::Kind K) {
   BuiltinType *Ty = new (*this, TypeAlignment) BuiltinType(K);
@@ -354,9 +387,16 @@ void ASTContext::InitBuiltinType(CanQualType &R, BuiltinType::Kind K) {
   Types.push_back(Ty);
 }
 
-void ASTContext::InitBuiltinTypes() {
+void ASTContext::InitBuiltinTypes(const TargetInfo &Target) {
+  assert((!this->Target || this->Target == &Target) &&
+         "Incorrect target reinitialization");
   assert(VoidTy.isNull() && "Context reinitialized?");
 
+  this->Target = &Target;
+  
+  ABI.reset(createCXXABI(Target));
+  AddrSpaceMap = getAddressSpaceMap(Target, LangOpts);
+  
   // C99 6.2.5p19.
   InitBuiltinType(VoidTy,              BuiltinType::Void);
 
@@ -431,11 +471,6 @@ void ASTContext::InitBuiltinTypes() {
 
   BuiltinVaListType = QualType();
 
-  // "Builtin" typedefs set by Sema::ActOnTranslationUnitScope().
-  ObjCIdTypedefType = QualType();
-  ObjCClassTypedefType = QualType();
-  ObjCSelTypedefType = QualType();
-
   // Builtin types for 'id', 'Class', and 'SEL'.
   InitBuiltinType(ObjCBuiltinIdTy, BuiltinType::ObjCId);
   InitBuiltinType(ObjCBuiltinClassTy, BuiltinType::ObjCClass);
@@ -448,9 +483,12 @@ void ASTContext::InitBuiltinTypes() {
 
   // nullptr type (C++0x 2.14.7)
   InitBuiltinType(NullPtrTy,           BuiltinType::NullPtr);
+
+  // half type (OpenCL 6.1.1.1) / ARM NEON __fp16
+  InitBuiltinType(HalfTy, BuiltinType::Half);
 }
 
-Diagnostic &ASTContext::getDiagnostics() const {
+DiagnosticsEngine &ASTContext::getDiagnostics() const {
   return SourceMgr.getDiagnostics();
 }
 
@@ -496,6 +534,24 @@ ASTContext::setInstantiatedFromStaticDataMember(VarDecl *Inst, VarDecl *Tmpl,
     = new (*this) MemberSpecializationInfo(Tmpl, TSK, PointOfInstantiation);
 }
 
+FunctionDecl *ASTContext::getClassScopeSpecializationPattern(
+                                                     const FunctionDecl *FD){
+  assert(FD && "Specialization is 0");
+  llvm::DenseMap<const FunctionDecl*, FunctionDecl *>::const_iterator Pos
+    = ClassScopeSpecializationPattern.find(FD);
+  if (Pos == ClassScopeSpecializationPattern.end())
+    return 0;
+
+  return Pos->second;
+}
+
+void ASTContext::setClassScopeSpecializationPattern(FunctionDecl *FD,
+                                        FunctionDecl *Pattern) {
+  assert(FD && "Specialization is 0");
+  assert(Pattern && "Class scope specialization pattern is 0");
+  ClassScopeSpecializationPattern[FD] = Pattern;
+}
+
 NamedDecl *
 ASTContext::getInstantiatedFromUsingDecl(UsingDecl *UUD) {
   llvm::DenseMap<UsingDecl *, NamedDecl *>::const_iterator Pos
@@ -555,35 +611,33 @@ void ASTContext::setInstantiatedFromUnnamedFieldDecl(FieldDecl *Inst,
 bool ASTContext::ZeroBitfieldFollowsNonBitfield(const FieldDecl *FD, 
                                     const FieldDecl *LastFD) const {
   return (FD->isBitField() && LastFD && !LastFD->isBitField() &&
-          FD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue() == 0);
-  
+          FD->getBitWidthValue(*this) == 0);
 }
 
 bool ASTContext::ZeroBitfieldFollowsBitfield(const FieldDecl *FD,
                                              const FieldDecl *LastFD) const {
   return (FD->isBitField() && LastFD && LastFD->isBitField() &&
-          FD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue() == 0 &&
-          LastFD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue() != 0);
-
+          FD->getBitWidthValue(*this) == 0 &&
+          LastFD->getBitWidthValue(*this) != 0);
 }
 
 bool ASTContext::BitfieldFollowsBitfield(const FieldDecl *FD,
                                          const FieldDecl *LastFD) const {
   return (FD->isBitField() && LastFD && LastFD->isBitField() &&
-          FD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue() &&
-          LastFD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue());  
+          FD->getBitWidthValue(*this) &&
+          LastFD->getBitWidthValue(*this));
 }
 
-bool ASTContext::NoneBitfieldFollowsBitfield(const FieldDecl *FD,
+bool ASTContext::NonBitfieldFollowsBitfield(const FieldDecl *FD,
                                          const FieldDecl *LastFD) const {
   return (!FD->isBitField() && LastFD && LastFD->isBitField() &&
-          LastFD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue());  
+          LastFD->getBitWidthValue(*this));
 }
 
-bool ASTContext::BitfieldFollowsNoneBitfield(const FieldDecl *FD,
+bool ASTContext::BitfieldFollowsNonBitfield(const FieldDecl *FD,
                                              const FieldDecl *LastFD) const {
   return (FD->isBitField() && LastFD && !LastFD->isBitField() &&
-          FD->getBitWidth()-> EvaluateAsInt(*this).getZExtValue());  
+          FD->getBitWidthValue(*this));
 }
 
 ASTContext::overridden_cxx_method_iterator
@@ -631,10 +685,11 @@ const llvm::fltSemantics &ASTContext::getFloatTypeSemantics(QualType T) const {
   const BuiltinType *BT = T->getAs<BuiltinType>();
   assert(BT && "Not a floating point type!");
   switch (BT->getKind()) {
-  default: assert(0 && "Not a floating point type!");
-  case BuiltinType::Float:      return Target.getFloatFormat();
-  case BuiltinType::Double:     return Target.getDoubleFormat();
-  case BuiltinType::LongDouble: return Target.getLongDoubleFormat();
+  default: llvm_unreachable("Not a floating point type!");
+  case BuiltinType::Half:       return Target->getHalfFormat();
+  case BuiltinType::Float:      return Target->getFloatFormat();
+  case BuiltinType::Double:     return Target->getDoubleFormat();
+  case BuiltinType::LongDouble: return Target->getLongDoubleFormat();
   }
 }
 
@@ -644,7 +699,7 @@ const llvm::fltSemantics &ASTContext::getFloatTypeSemantics(QualType T) const {
 /// If @p RefAsPointee, references are treated like their underlying type
 /// (for alignof), else they're treated like pointers (for CodeGen).
 CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) const {
-  unsigned Align = Target.getCharWidth();
+  unsigned Align = Target->getCharWidth();
 
   bool UseAlignAttrOnly = false;
   if (unsigned AlignFromAttr = D->getMaxAlignment()) {
@@ -654,7 +709,7 @@ CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) const {
     // *except* on a struct or struct member, where it only increases
     // alignment unless 'packed' is also specified.
     //
-    // It is an error for [[align]] to decrease alignment, so we can
+    // It is an error for alignas to decrease alignment, so we can
     // ignore that possibility;  Sema should diagnose it.
     if (isa<FieldDecl>(D)) {
       UseAlignAttrOnly = D->hasAttr<PackedAttr>() ||
@@ -684,14 +739,14 @@ CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) const {
     if (!T->isIncompleteType() && !T->isFunctionType()) {
       // Adjust alignments of declarations with array type by the
       // large-array alignment on the target.
-      unsigned MinWidth = Target.getLargeArrayMinWidth();
+      unsigned MinWidth = Target->getLargeArrayMinWidth();
       const ArrayType *arrayType;
       if (MinWidth && (arrayType = getAsArrayType(T))) {
         if (isa<VariableArrayType>(arrayType))
-          Align = std::max(Align, Target.getLargeArrayAlign());
+          Align = std::max(Align, Target->getLargeArrayAlign());
         else if (isa<ConstantArrayType>(arrayType) &&
                  MinWidth <= getTypeSize(cast<ConstantArrayType>(arrayType)))
-          Align = std::max(Align, Target.getLargeArrayAlign());
+          Align = std::max(Align, Target->getLargeArrayAlign());
 
         // Walk through any array types while we're at it.
         T = getBaseElementType(arrayType);
@@ -798,7 +853,7 @@ ASTContext::getTypeInfo(const Type *T) const {
 
   case Type::Builtin:
     switch (cast<BuiltinType>(T)->getKind()) {
-    default: assert(0 && "Unknown builtin type!");
+    default: llvm_unreachable("Unknown builtin type!");
     case BuiltinType::Void:
       // GCC extension: alignof(void) = 8 bits.
       Width = 0;
@@ -806,87 +861,91 @@ ASTContext::getTypeInfo(const Type *T) const {
       break;
 
     case BuiltinType::Bool:
-      Width = Target.getBoolWidth();
-      Align = Target.getBoolAlign();
+      Width = Target->getBoolWidth();
+      Align = Target->getBoolAlign();
       break;
     case BuiltinType::Char_S:
     case BuiltinType::Char_U:
     case BuiltinType::UChar:
     case BuiltinType::SChar:
-      Width = Target.getCharWidth();
-      Align = Target.getCharAlign();
+      Width = Target->getCharWidth();
+      Align = Target->getCharAlign();
       break;
     case BuiltinType::WChar_S:
     case BuiltinType::WChar_U:
-      Width = Target.getWCharWidth();
-      Align = Target.getWCharAlign();
+      Width = Target->getWCharWidth();
+      Align = Target->getWCharAlign();
       break;
     case BuiltinType::Char16:
-      Width = Target.getChar16Width();
-      Align = Target.getChar16Align();
+      Width = Target->getChar16Width();
+      Align = Target->getChar16Align();
       break;
     case BuiltinType::Char32:
-      Width = Target.getChar32Width();
-      Align = Target.getChar32Align();
+      Width = Target->getChar32Width();
+      Align = Target->getChar32Align();
       break;
     case BuiltinType::UShort:
     case BuiltinType::Short:
-      Width = Target.getShortWidth();
-      Align = Target.getShortAlign();
+      Width = Target->getShortWidth();
+      Align = Target->getShortAlign();
       break;
     case BuiltinType::UInt:
     case BuiltinType::Int:
-      Width = Target.getIntWidth();
-      Align = Target.getIntAlign();
+      Width = Target->getIntWidth();
+      Align = Target->getIntAlign();
       break;
     case BuiltinType::ULong:
     case BuiltinType::Long:
-      Width = Target.getLongWidth();
-      Align = Target.getLongAlign();
+      Width = Target->getLongWidth();
+      Align = Target->getLongAlign();
       break;
     case BuiltinType::ULongLong:
     case BuiltinType::LongLong:
-      Width = Target.getLongLongWidth();
-      Align = Target.getLongLongAlign();
+      Width = Target->getLongLongWidth();
+      Align = Target->getLongLongAlign();
       break;
     case BuiltinType::Int128:
     case BuiltinType::UInt128:
       Width = 128;
       Align = 128; // int128_t is 128-bit aligned on all targets.
       break;
+    case BuiltinType::Half:
+      Width = Target->getHalfWidth();
+      Align = Target->getHalfAlign();
+      break;
     case BuiltinType::Float:
-      Width = Target.getFloatWidth();
-      Align = Target.getFloatAlign();
+      Width = Target->getFloatWidth();
+      Align = Target->getFloatAlign();
       break;
     case BuiltinType::Double:
-      Width = Target.getDoubleWidth();
-      Align = Target.getDoubleAlign();
+      Width = Target->getDoubleWidth();
+      Align = Target->getDoubleAlign();
       break;
     case BuiltinType::LongDouble:
-      Width = Target.getLongDoubleWidth();
-      Align = Target.getLongDoubleAlign();
+      Width = Target->getLongDoubleWidth();
+      Align = Target->getLongDoubleAlign();
       break;
     case BuiltinType::NullPtr:
-      Width = Target.getPointerWidth(0); // C++ 3.9.1p11: sizeof(nullptr_t)
-      Align = Target.getPointerAlign(0); //   == sizeof(void*)
+      Width = Target->getPointerWidth(0); // C++ 3.9.1p11: sizeof(nullptr_t)
+      Align = Target->getPointerAlign(0); //   == sizeof(void*)
       break;
     case BuiltinType::ObjCId:
     case BuiltinType::ObjCClass:
     case BuiltinType::ObjCSel:
-      Width = Target.getPointerWidth(0); 
-      Align = Target.getPointerAlign(0);
+      Width = Target->getPointerWidth(0); 
+      Align = Target->getPointerAlign(0);
       break;
     }
     break;
   case Type::ObjCObjectPointer:
-    Width = Target.getPointerWidth(0);
-    Align = Target.getPointerAlign(0);
+    Width = Target->getPointerWidth(0);
+    Align = Target->getPointerAlign(0);
     break;
   case Type::BlockPointer: {
     unsigned AS = getTargetAddressSpace(
         cast<BlockPointerType>(T)->getPointeeType());
-    Width = Target.getPointerWidth(AS);
-    Align = Target.getPointerAlign(AS);
+    Width = Target->getPointerWidth(AS);
+    Align = Target->getPointerAlign(AS);
     break;
   }
   case Type::LValueReference:
@@ -895,14 +954,14 @@ ASTContext::getTypeInfo(const Type *T) const {
     // the pointer route.
     unsigned AS = getTargetAddressSpace(
         cast<ReferenceType>(T)->getPointeeType());
-    Width = Target.getPointerWidth(AS);
-    Align = Target.getPointerAlign(AS);
+    Width = Target->getPointerWidth(AS);
+    Align = Target->getPointerAlign(AS);
     break;
   }
   case Type::Pointer: {
     unsigned AS = getTargetAddressSpace(cast<PointerType>(T)->getPointeeType());
-    Width = Target.getPointerWidth(AS);
-    Align = Target.getPointerAlign(AS);
+    Width = Target->getPointerWidth(AS);
+    Align = Target->getPointerAlign(AS);
     break;
   }
   case Type::MemberPointer: {
@@ -1012,9 +1071,26 @@ ASTContext::getTypeInfo(const Type *T) const {
       return getTypeInfo(getCanonicalType(T));
   }
 
+  case Type::Atomic: {
+    std::pair<uint64_t, unsigned> Info
+      = getTypeInfo(cast<AtomicType>(T)->getValueType());
+    Width = Info.first;
+    Align = Info.second;
+    if (Width != 0 && Width <= Target->getMaxAtomicPromoteWidth() &&
+        llvm::isPowerOf2_64(Width)) {
+      // We can potentially perform lock-free atomic operations for this
+      // type; promote the alignment appropriately.
+      // FIXME: We could potentially promote the width here as well...
+      // is that worthwhile?  (Non-struct atomic types generally have
+      // power-of-two size anyway, but structs might not.  Requires a bit
+      // of implementation work to make sure we zero out the extra bits.)
+      Align = static_cast<unsigned>(Width);
+    }
   }
 
-  assert(Align && (Align & (Align-1)) == 0 && "Alignment must be power of 2");
+  }
+
+  assert(llvm::isPowerOf2_32(Align) && "Alignment must be power of 2");
   return std::make_pair(Width, Align);
 }
 
@@ -1063,19 +1139,6 @@ unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
   return ABIAlign;
 }
 
-/// ShallowCollectObjCIvars -
-/// Collect all ivars, including those synthesized, in the current class.
-///
-void ASTContext::ShallowCollectObjCIvars(const ObjCInterfaceDecl *OI,
-                            llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) const {
-  // FIXME. This need be removed but there are two many places which
-  // assume const-ness of ObjCInterfaceDecl
-  ObjCInterfaceDecl *IDecl = const_cast<ObjCInterfaceDecl *>(OI);
-  for (ObjCIvarDecl *Iv = IDecl->all_declared_ivar_begin(); Iv; 
-        Iv= Iv->getNextIvar())
-    Ivars.push_back(Iv);
-}
-
 /// DeepCollectObjCIvars -
 /// This routine first collects all declared, but not synthesized, ivars in
 /// super class and then collects all ivars, including those synthesized for
@@ -1084,17 +1147,16 @@ void ASTContext::ShallowCollectObjCIvars(const ObjCInterfaceDecl *OI,
 ///
 void ASTContext::DeepCollectObjCIvars(const ObjCInterfaceDecl *OI,
                                       bool leafClass,
-                            llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) const {
+                            SmallVectorImpl<const ObjCIvarDecl*> &Ivars) const {
   if (const ObjCInterfaceDecl *SuperClass = OI->getSuperClass())
     DeepCollectObjCIvars(SuperClass, false, Ivars);
   if (!leafClass) {
     for (ObjCInterfaceDecl::ivar_iterator I = OI->ivar_begin(),
          E = OI->ivar_end(); I != E; ++I)
       Ivars.push_back(*I);
-  }
-  else {
+  } else {
     ObjCInterfaceDecl *IDecl = const_cast<ObjCInterfaceDecl *>(OI);
-    for (ObjCIvarDecl *Iv = IDecl->all_declared_ivar_begin(); Iv; 
+    for (const ObjCIvarDecl *Iv = IDecl->all_declared_ivar_begin(); Iv; 
          Iv= Iv->getNextIvar())
       Ivars.push_back(Iv);
   }
@@ -1562,7 +1624,7 @@ QualType ASTContext::getConstantArrayType(QualType EltTy,
   // the target.
   llvm::APInt ArySize(ArySizeIn);
   ArySize =
-    ArySize.zextOrTrunc(Target.getPointerWidth(getTargetAddressSpace(EltTy)));
+    ArySize.zextOrTrunc(Target->getPointerWidth(getTargetAddressSpace(EltTy)));
 
   llvm::FoldingSetNodeID ID;
   ConstantArrayType::Profile(ID, EltTy, ArySize, ASM, IndexTypeQuals);
@@ -1670,6 +1732,12 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
     break;
   }
 
+  case Type::Atomic: {
+    const AtomicType *at = cast<AtomicType>(ty);
+    result = getAtomicType(getVariableArrayDecayedType(at->getValueType()));
+    break;
+  }
+
   case Type::ConstantArray: {
     const ConstantArrayType *cat = cast<ConstantArrayType>(ty);
     result = getConstantArrayType(
@@ -2026,7 +2094,7 @@ ASTContext::getFunctionType(QualType ResultTy,
   // The exception spec is not part of the canonical type.
   QualType Canonical;
   if (!isCanonical || getCanonicalCallConv(CallConv) != CallConv) {
-    llvm::SmallVector<QualType, 16> CanonicalArgs;
+    SmallVector<QualType, 16> CanonicalArgs;
     CanonicalArgs.reserve(NumArgs);
     for (unsigned i = 0; i != NumArgs; ++i)
       CanonicalArgs.push_back(getCanonicalParamType(ArgArray[i]));
@@ -2322,7 +2390,7 @@ ASTContext::getTemplateSpecializationType(TemplateName Template,
   
   unsigned NumArgs = Args.size();
 
-  llvm::SmallVector<TemplateArgument, 4> ArgVec;
+  SmallVector<TemplateArgument, 4> ArgVec;
   ArgVec.reserve(NumArgs);
   for (unsigned i = 0; i != NumArgs; ++i)
     ArgVec.push_back(Args[i].getArgument());
@@ -2389,7 +2457,7 @@ ASTContext::getCanonicalTemplateSpecializationType(TemplateName Template,
   
   // Build the canonical template specialization type.
   TemplateName CanonTemplate = getCanonicalTemplateName(Template);
-  llvm::SmallVector<TemplateArgument, 4> CanonArgs;
+  SmallVector<TemplateArgument, 4> CanonArgs;
   CanonArgs.reserve(NumArgs);
   for (unsigned I = 0; I != NumArgs; ++I)
     CanonArgs.push_back(getCanonicalTemplateArgument(Args[I]));
@@ -2509,7 +2577,7 @@ ASTContext::getDependentTemplateSpecializationType(
                                  const IdentifierInfo *Name,
                                  const TemplateArgumentListInfo &Args) const {
   // TODO: avoid this copy
-  llvm::SmallVector<TemplateArgument, 16> ArgCopy;
+  SmallVector<TemplateArgument, 16> ArgCopy;
   for (unsigned I = 0, E = Args.size(); I != E; ++I)
     ArgCopy.push_back(Args[I].getArgument());
   return getDependentTemplateSpecializationType(Keyword, NNS, Name,
@@ -2543,7 +2611,7 @@ ASTContext::getDependentTemplateSpecializationType(
   if (Keyword == ETK_None) CanonKeyword = ETK_Typename;
 
   bool AnyNonCanonArgs = false;
-  llvm::SmallVector<TemplateArgument, 16> CanonArgs(NumArgs);
+  SmallVector<TemplateArgument, 16> CanonArgs(NumArgs);
   for (unsigned I = 0; I != NumArgs; ++I) {
     CanonArgs[I] = getCanonicalTemplateArgument(Args[I]);
     if (!CanonArgs[I].structurallyEquals(Args[I]))
@@ -2647,7 +2715,7 @@ QualType ASTContext::getObjCObjectType(QualType BaseType,
   bool ProtocolsSorted = areSortedAndUniqued(Protocols, NumProtocols);
   if (!ProtocolsSorted || !BaseType.isCanonical()) {
     if (!ProtocolsSorted) {
-      llvm::SmallVector<ObjCProtocolDecl*, 8> Sorted(Protocols,
+      SmallVector<ObjCProtocolDecl*, 8> Sorted(Protocols,
                                                      Protocols + NumProtocols);
       unsigned UniqueCount = NumProtocols;
 
@@ -2737,8 +2805,7 @@ QualType ASTContext::getTypeOfExprType(Expr *tofExpr) const {
       // typeof(expr) type. Use that as our canonical type.
       toe = new (*this, TypeAlignment) TypeOfExprType(tofExpr,
                                           QualType((TypeOfExprType*)Canon, 0));
-    }
-    else {
+    } else {
       // Build a new, canonical typeof(expr) type.
       Canon
         = new (*this, TypeAlignment) DependentTypeOfExprType(*this, tofExpr);
@@ -2821,8 +2888,7 @@ QualType ASTContext::getDecltypeType(Expr *e) const {
       // decltype type. Use that as our canonical type.
       dt = new (*this, TypeAlignment) DecltypeType(e, DependentTy,
                                        QualType((DecltypeType*)Canon, 0));
-    }
-    else {
+    } else {
       // Build a new, canonical typeof(expr) type.
       Canon = new (*this, TypeAlignment) DependentDecltypeType(*this, e);
       DependentDecltypeTypes.InsertNode(Canon, InsertPos);
@@ -2869,6 +2935,34 @@ QualType ASTContext::getAutoType(QualType DeducedType) const {
   return QualType(AT, 0);
 }
 
+/// getAtomicType - Return the uniqued reference to the atomic type for
+/// the given value type.
+QualType ASTContext::getAtomicType(QualType T) const {
+  // Unique pointers, to guarantee there is only one pointer of a particular
+  // structure.
+  llvm::FoldingSetNodeID ID;
+  AtomicType::Profile(ID, T);
+
+  void *InsertPos = 0;
+  if (AtomicType *AT = AtomicTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(AT, 0);
+
+  // If the atomic value type isn't canonical, this won't be a canonical type
+  // either, so fill in the canonical type field.
+  QualType Canonical;
+  if (!T.isCanonical()) {
+    Canonical = getAtomicType(getCanonicalType(T));
+
+    // Get the new insert position for the node we care about.
+    AtomicType *NewIP = AtomicTypes.FindNodeOrInsertPos(ID, InsertPos);
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
+  }
+  AtomicType *New = new (*this, TypeAlignment) AtomicType(T, Canonical);
+  Types.push_back(New);
+  AtomicTypes.InsertNode(New, InsertPos);
+  return QualType(New, 0);
+}
+
 /// getAutoDeductType - Get type pattern for deducing against 'auto'.
 QualType ASTContext::getAutoDeductType() const {
   if (AutoDeductTy.isNull())
@@ -2898,7 +2992,7 @@ QualType ASTContext::getTagDeclType(const TagDecl *Decl) const {
 /// of the sizeof operator (C99 6.5.3.4p4). The value is target dependent and
 /// needs to agree with the definition in <stddef.h>.
 CanQualType ASTContext::getSizeType() const {
-  return getFromTargetType(Target.getSizeType());
+  return getFromTargetType(Target->getSizeType());
 }
 
 /// getSignedWCharType - Return the type of "signed wchar_t".
@@ -2918,7 +3012,7 @@ QualType ASTContext::getUnsignedWCharType() const {
 /// getPointerDiffType - Return the unique type for "ptrdiff_t" (ref?)
 /// defined in <stddef.h>. Pointer - pointer requires this (C99 6.5.6p9).
 QualType ASTContext::getPointerDiffType() const {
-  return getFromTargetType(Target.getPtrDiffType(0));
+  return getFromTargetType(Target->getPtrDiffType(0));
 }
 
 //===----------------------------------------------------------------------===//
@@ -3183,8 +3277,7 @@ ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) const {
   }
 
   // Silence GCC warning
-  assert(false && "Unhandled template argument kind");
-  return TemplateArgument();
+  llvm_unreachable("Unhandled template argument kind");
 }
 
 NestedNameSpecifier *
@@ -3397,7 +3490,8 @@ static FloatingRank getFloatingRank(QualType T) {
 
   assert(T->getAs<BuiltinType>() && "getFloatingRank(): not a floating type");
   switch (T->getAs<BuiltinType>()->getKind()) {
-  default: assert(0 && "getFloatingRank(): not a floating type");
+  default: llvm_unreachable("getFloatingRank(): not a floating type");
+  case BuiltinType::Half:       return HalfRank;
   case BuiltinType::Float:      return FloatRank;
   case BuiltinType::Double:     return DoubleRank;
   case BuiltinType::LongDouble: return LongDoubleRank;
@@ -3413,7 +3507,7 @@ QualType ASTContext::getFloatingTypeOfSizeWithinDomain(QualType Size,
   FloatingRank EltRank = getFloatingRank(Size);
   if (Domain->isComplexType()) {
     switch (EltRank) {
-    default: assert(0 && "getFloatingRank(): illegal value for rank");
+    default: llvm_unreachable("getFloatingRank(): illegal value for rank");
     case FloatRank:      return FloatComplexTy;
     case DoubleRank:     return DoubleComplexTy;
     case LongDoubleRank: return LongDoubleComplexTy;
@@ -3422,7 +3516,7 @@ QualType ASTContext::getFloatingTypeOfSizeWithinDomain(QualType Size,
 
   assert(Domain->isRealFloatingType() && "Unknown domain!");
   switch (EltRank) {
-  default: assert(0 && "getFloatingRank(): illegal value for rank");
+  default: llvm_unreachable("getFloatingRank(): illegal value for rank");
   case FloatRank:      return FloatTy;
   case DoubleRank:     return DoubleTy;
   case LongDoubleRank: return LongDoubleTy;
@@ -3454,16 +3548,16 @@ unsigned ASTContext::getIntegerRank(const Type *T) const {
 
   if (T->isSpecificBuiltinType(BuiltinType::WChar_S) ||
       T->isSpecificBuiltinType(BuiltinType::WChar_U))
-    T = getFromTargetType(Target.getWCharType()).getTypePtr();
+    T = getFromTargetType(Target->getWCharType()).getTypePtr();
 
   if (T->isSpecificBuiltinType(BuiltinType::Char16))
-    T = getFromTargetType(Target.getChar16Type()).getTypePtr();
+    T = getFromTargetType(Target->getChar16Type()).getTypePtr();
 
   if (T->isSpecificBuiltinType(BuiltinType::Char32))
-    T = getFromTargetType(Target.getChar32Type()).getTypePtr();
+    T = getFromTargetType(Target->getChar32Type()).getTypePtr();
 
   switch (cast<BuiltinType>(T)->getKind()) {
-  default: assert(0 && "getIntegerRank(): not a built-in integer");
+  default: llvm_unreachable("getIntegerRank(): not a built-in integer");
   case BuiltinType::Bool:
     return 1 + (getIntWidth(BoolTy) << 3);
   case BuiltinType::Char_S:
@@ -3504,8 +3598,7 @@ QualType ASTContext::isPromotableBitField(Expr *E) const {
 
   QualType FT = Field->getType();
 
-  llvm::APSInt BitWidthAP = Field->getBitWidth()->EvaluateAsInt(*this);
-  uint64_t BitWidth = BitWidthAP.getZExtValue();
+  uint64_t BitWidth = Field->getBitWidthValue(*this);
   uint64_t IntSize = getTypeSize(IntTy);
   // GCC extension compatibility: if the bit-field size is less than or equal
   // to the size of int, it gets promoted no matter what its type is.
@@ -3653,82 +3746,6 @@ void ASTContext::setCFConstantStringType(QualType T) {
   CFConstantStringTypeDecl = Rec->getDecl();
 }
 
-// getNSConstantStringType - Return the type used for constant NSStrings.
-QualType ASTContext::getNSConstantStringType() const {
-  if (!NSConstantStringTypeDecl) {
-    NSConstantStringTypeDecl =
-    CreateRecordDecl(*this, TTK_Struct, TUDecl,
-                     &Idents.get("__builtin_NSString"));
-    NSConstantStringTypeDecl->startDefinition();
-    
-    QualType FieldTypes[3];
-    
-    // const int *isa;
-    FieldTypes[0] = getPointerType(IntTy.withConst());
-    // const char *str;
-    FieldTypes[1] = getPointerType(CharTy.withConst());
-    // unsigned int length;
-    FieldTypes[2] = UnsignedIntTy;
-    
-    // Create fields
-    for (unsigned i = 0; i < 3; ++i) {
-      FieldDecl *Field = FieldDecl::Create(*this, NSConstantStringTypeDecl,
-                                           SourceLocation(),
-                                           SourceLocation(), 0,
-                                           FieldTypes[i], /*TInfo=*/0,
-                                           /*BitWidth=*/0,
-                                           /*Mutable=*/false,
-                                           /*HasInit=*/false);
-      Field->setAccess(AS_public);
-      NSConstantStringTypeDecl->addDecl(Field);
-    }
-    
-    NSConstantStringTypeDecl->completeDefinition();
-  }
-  
-  return getTagDeclType(NSConstantStringTypeDecl);
-}
-
-void ASTContext::setNSConstantStringType(QualType T) {
-  const RecordType *Rec = T->getAs<RecordType>();
-  assert(Rec && "Invalid NSConstantStringType");
-  NSConstantStringTypeDecl = Rec->getDecl();
-}
-
-QualType ASTContext::getObjCFastEnumerationStateType() const {
-  if (!ObjCFastEnumerationStateTypeDecl) {
-    ObjCFastEnumerationStateTypeDecl =
-      CreateRecordDecl(*this, TTK_Struct, TUDecl,
-                       &Idents.get("__objcFastEnumerationState"));
-    ObjCFastEnumerationStateTypeDecl->startDefinition();
-
-    QualType FieldTypes[] = {
-      UnsignedLongTy,
-      getPointerType(ObjCIdTypedefType),
-      getPointerType(UnsignedLongTy),
-      getConstantArrayType(UnsignedLongTy,
-                           llvm::APInt(32, 5), ArrayType::Normal, 0)
-    };
-
-    for (size_t i = 0; i < 4; ++i) {
-      FieldDecl *Field = FieldDecl::Create(*this,
-                                           ObjCFastEnumerationStateTypeDecl,
-                                           SourceLocation(),
-                                           SourceLocation(), 0,
-                                           FieldTypes[i], /*TInfo=*/0,
-                                           /*BitWidth=*/0,
-                                           /*Mutable=*/false,
-                                           /*HasInit=*/false);
-      Field->setAccess(AS_public);
-      ObjCFastEnumerationStateTypeDecl->addDecl(Field);
-    }
-
-    ObjCFastEnumerationStateTypeDecl->completeDefinition();
-  }
-
-  return getTagDeclType(ObjCFastEnumerationStateTypeDecl);
-}
-
 QualType ASTContext::getBlockDescriptorType() const {
   if (BlockDescriptorType)
     return getTagDeclType(BlockDescriptorType);
@@ -3768,12 +3785,6 @@ QualType ASTContext::getBlockDescriptorType() const {
   return getTagDeclType(BlockDescriptorType);
 }
 
-void ASTContext::setBlockDescriptorType(QualType T) {
-  const RecordType *Rec = T->getAs<RecordType>();
-  assert(Rec && "Invalid BlockDescriptorType");
-  BlockDescriptorType = Rec->getDecl();
-}
-
 QualType ASTContext::getBlockDescriptorExtendedType() const {
   if (BlockDescriptorExtendedType)
     return getTagDeclType(BlockDescriptorExtendedType);
@@ -3817,12 +3828,6 @@ QualType ASTContext::getBlockDescriptorExtendedType() const {
   return getTagDeclType(BlockDescriptorExtendedType);
 }
 
-void ASTContext::setBlockDescriptorExtendedType(QualType T) {
-  const RecordType *Rec = T->getAs<RecordType>();
-  assert(Rec && "Invalid BlockDescriptorType");
-  BlockDescriptorExtendedType = Rec->getDecl();
-}
-
 bool ASTContext::BlockRequiresCopying(QualType Ty) const {
   if (Ty->isObjCRetainableType())
     return true;
@@ -3837,7 +3842,7 @@ bool ASTContext::BlockRequiresCopying(QualType Ty) const {
 }
 
 QualType
-ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) const {
+ASTContext::BuildByRefType(StringRef DeclName, QualType Ty) const {
   //  type = struct __Block_byref_1_X {
   //    void *__isa;
   //    struct __Block_byref_1_X *__forwarding;
@@ -3869,7 +3874,7 @@ ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) const {
     Ty
   };
 
-  llvm::StringRef FieldNames[] = {
+  StringRef FieldNames[] = {
     "__isa",
     "__forwarding",
     "__flags",
@@ -3897,10 +3902,15 @@ ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) const {
   return getPointerType(getTagDeclType(T));
 }
 
-void ASTContext::setObjCFastEnumerationStateType(QualType T) {
-  const RecordType *Rec = T->getAs<RecordType>();
-  assert(Rec && "Invalid ObjCFAstEnumerationStateType");
-  ObjCFastEnumerationStateTypeDecl = Rec->getDecl();
+TypedefDecl *ASTContext::getObjCInstanceTypeDecl() {
+  if (!ObjCInstanceTypeDecl)
+    ObjCInstanceTypeDecl = TypedefDecl::Create(*this, 
+                                               getTranslationUnitDecl(),
+                                               SourceLocation(), 
+                                               SourceLocation(),
+                                               &Idents.get("instancetype"), 
+                                     getTrivialTypeSourceInfo(getObjCIdType()));
+  return ObjCInstanceTypeDecl;
 }
 
 // This returns true if a type has been typedefed to BOOL:
@@ -3962,7 +3972,6 @@ std::string ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr) const {
   S += charUnitsToString(ParmOffset);
   // Block pointer and offset.
   S += "@?0";
-  ParmOffset = PtrSize;
   
   // Argument types.
   ParmOffset = PtrSize;
@@ -4044,7 +4053,7 @@ bool ASTContext::getObjCEncodingForMethodDecl(const ObjCMethodDecl *Decl,
   // The first two arguments (self and _cmd) are pointers; account for
   // their size.
   CharUnits ParmOffset = 2 * PtrSize;
-  for (ObjCMethodDecl::param_iterator PI = Decl->param_begin(),
+  for (ObjCMethodDecl::param_const_iterator PI = Decl->param_begin(),
        E = Decl->sel_param_end(); PI != E; ++PI) {
     QualType PType = (*PI)->getType();
     CharUnits sz = getObjCEncodingTypeSize(PType);
@@ -4061,9 +4070,9 @@ bool ASTContext::getObjCEncodingForMethodDecl(const ObjCMethodDecl *Decl,
 
   // Argument types.
   ParmOffset = 2 * PtrSize;
-  for (ObjCMethodDecl::param_iterator PI = Decl->param_begin(),
+  for (ObjCMethodDecl::param_const_iterator PI = Decl->param_begin(),
        E = Decl->sel_param_end(); PI != E; ++PI) {
-    ParmVarDecl *PVDecl = *PI;
+    const ParmVarDecl *PVDecl = *PI;
     QualType PType = PVDecl->getOriginalType();
     if (const ArrayType *AT =
           dyn_cast<ArrayType>(PType->getCanonicalTypeInternal())) {
@@ -4166,6 +4175,7 @@ void ASTContext::getObjCEncodingForPropertyDecl(const ObjCPropertyDecl *PD,
     case ObjCPropertyDecl::Assign: break;
     case ObjCPropertyDecl::Copy:   S += ",C"; break;
     case ObjCPropertyDecl::Retain: S += ",&"; break;
+    case ObjCPropertyDecl::Weak:   S += ",W"; break;
     }
   }
 
@@ -4225,7 +4235,7 @@ void ASTContext::getObjCEncodingForType(QualType T, std::string& S,
 
 static char ObjCEncodingForPrimitiveKind(const ASTContext *C, QualType T) {
     switch (T->getAs<BuiltinType>()->getKind()) {
-    default: assert(0 && "Unhandled builtin type kind");
+    default: llvm_unreachable("Unhandled builtin type kind");
     case BuiltinType::Void:       return 'v';
     case BuiltinType::Bool:       return 'B';
     case BuiltinType::Char_U:
@@ -4252,10 +4262,20 @@ static char ObjCEncodingForPrimitiveKind(const ASTContext *C, QualType T) {
     }
 }
 
+static char ObjCEncodingForEnumType(const ASTContext *C, const EnumType *ET) {
+  EnumDecl *Enum = ET->getDecl();
+  
+  // The encoding of an non-fixed enum type is always 'i', regardless of size.
+  if (!Enum->isFixed())
+    return 'i';
+  
+  // The encoding of a fixed enum type matches its fixed underlying type.
+  return ObjCEncodingForPrimitiveKind(C, Enum->getIntegerType());
+}
+
 static void EncodeBitField(const ASTContext *Ctx, std::string& S,
                            QualType T, const FieldDecl *FD) {
-  const Expr *E = FD->getBitWidth();
-  assert(E && "bitfield width not there - getObjCEncodingForTypeImpl");
+  assert(FD->isBitField() && "not a bitfield - getObjCEncodingForTypeImpl");
   S += 'b';
   // The NeXT runtime encodes bit fields as b followed by the number of bits.
   // The GNU runtime requires more information; bitfields are encoded as b,
@@ -4276,13 +4296,12 @@ static void EncodeBitField(const ASTContext *Ctx, std::string& S,
     const RecordDecl *RD = FD->getParent();
     const ASTRecordLayout &RL = Ctx->getASTRecordLayout(RD);
     S += llvm::utostr(RL.getFieldOffset(FD->getFieldIndex()));
-    if (T->isEnumeralType())
-      S += 'i';
+    if (const EnumType *ET = T->getAs<EnumType>())
+      S += ObjCEncodingForEnumType(Ctx, ET);
     else
       S += ObjCEncodingForPrimitiveKind(Ctx, T);
   }
-  unsigned N = E->EvaluateAsInt(*Ctx).getZExtValue();
-  S += llvm::utostr(N);
+  S += llvm::utostr(FD->getBitWidthValue(*Ctx));
 }
 
 // FIXME: Use SmallString for accumulating string.
@@ -4342,7 +4361,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
       // Another legacy compatibility encoding. Some ObjC qualifier and type
       // combinations need to be rearranged.
       // Rewrite "in const" from "nr" to "rn"
-      if (llvm::StringRef(S).endswith("nr"))
+      if (StringRef(S).endswith("nr"))
         S.replace(S.end()-2, S.end(), "rn");
     }
 
@@ -4423,7 +4442,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
           = TemplateSpecializationType::PrintTemplateArgumentList(
                                             TemplateArgs.data(),
                                             TemplateArgs.size(),
-                                            (*this).PrintingPolicy);
+                                            (*this).getPrintingPolicy());
 
         S += TemplateArgsStr;
       }
@@ -4463,11 +4482,11 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
     return;
   }
   
-  if (T->isEnumeralType()) {
+  if (const EnumType *ET = T->getAs<EnumType>()) {
     if (FD && FD->isBitField())
       EncodeBitField(this, S, T, FD);
     else
-      S += 'i';
+      S += ObjCEncodingForEnumType(this, ET);
     return;
   }
 
@@ -4487,10 +4506,10 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
     const IdentifierInfo *II = OI->getIdentifier();
     S += II->getName();
     S += '=';
-    llvm::SmallVector<ObjCIvarDecl*, 32> Ivars;
+    SmallVector<const ObjCIvarDecl*, 32> Ivars;
     DeepCollectObjCIvars(OI, true, Ivars);
     for (unsigned i = 0, e = Ivars.size(); i != e; ++i) {
-      FieldDecl *Field = cast<FieldDecl>(Ivars[i]);
+      const FieldDecl *Field = cast<FieldDecl>(Ivars[i]);
       if (Field->isBitField())
         getObjCEncodingForTypeImpl(Field->getType(), S, false, true, Field);
       else
@@ -4573,7 +4592,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
     return;
   }
   
-  assert(0 && "@encode for type not implemented!");
+  llvm_unreachable("@encode for type not implemented!");
 }
 
 void ASTContext::getObjCEncodingForStructureImpl(RecordDecl *RDecl,
@@ -4621,8 +4640,9 @@ void ASTContext::getObjCEncodingForStructureImpl(RecordDecl *RDecl,
       if (base->isEmpty())
         continue;
       uint64_t offs = layout.getVBaseClassOffsetInBits(base);
-      FieldOrBaseOffsets.insert(FieldOrBaseOffsets.upper_bound(offs),
-                                std::make_pair(offs, base));
+      if (FieldOrBaseOffsets.find(offs) == FieldOrBaseOffsets.end())
+        FieldOrBaseOffsets.insert(FieldOrBaseOffsets.end(),
+                                  std::make_pair(offs, base));
     }
   }
 
@@ -4637,7 +4657,9 @@ void ASTContext::getObjCEncodingForStructureImpl(RecordDecl *RDecl,
   std::multimap<uint64_t, NamedDecl *>::iterator
     CurLayObj = FieldOrBaseOffsets.begin();
 
-  if (CurLayObj != FieldOrBaseOffsets.end() && CurLayObj->first != 0) {
+  if ((CurLayObj != FieldOrBaseOffsets.end() && CurLayObj->first != 0) ||
+      (CurLayObj == FieldOrBaseOffsets.end() &&
+         CXXRec && CXXRec->isDynamicClass())) {
     assert(CXXRec && CXXRec->isDynamicClass() &&
            "Offset 0 was empty but no VTable ?");
     if (FD) {
@@ -4695,7 +4717,7 @@ void ASTContext::getObjCEncodingForStructureImpl(RecordDecl *RDecl,
 
       if (field->isBitField()) {
         EncodeBitField(this, S, field->getType(), field);
-        CurOffs += field->getBitWidth()->EvaluateAsInt(*this).getZExtValue();
+        CurOffs += field->getBitWidthValue(*this);
       } else {
         QualType qt = field->getType();
         getLegacyIntegralTypeEncoding(qt);
@@ -4731,20 +4753,48 @@ void ASTContext::setBuiltinVaListType(QualType T) {
   BuiltinVaListType = T;
 }
 
-void ASTContext::setObjCIdType(QualType T) {
-  ObjCIdTypedefType = T;
+TypedefDecl *ASTContext::getObjCIdDecl() const {
+  if (!ObjCIdDecl) {
+    QualType T = getObjCObjectType(ObjCBuiltinIdTy, 0, 0);
+    T = getObjCObjectPointerType(T);
+    TypeSourceInfo *IdInfo = getTrivialTypeSourceInfo(T);
+    ObjCIdDecl = TypedefDecl::Create(const_cast<ASTContext &>(*this),
+                                     getTranslationUnitDecl(),
+                                     SourceLocation(), SourceLocation(),
+                                     &Idents.get("id"), IdInfo);
+  }
+  
+  return ObjCIdDecl;
 }
 
-void ASTContext::setObjCSelType(QualType T) {
-  ObjCSelTypedefType = T;
+TypedefDecl *ASTContext::getObjCSelDecl() const {
+  if (!ObjCSelDecl) {
+    QualType SelT = getPointerType(ObjCBuiltinSelTy);
+    TypeSourceInfo *SelInfo = getTrivialTypeSourceInfo(SelT);
+    ObjCSelDecl = TypedefDecl::Create(const_cast<ASTContext &>(*this),
+                                      getTranslationUnitDecl(),
+                                      SourceLocation(), SourceLocation(),
+                                      &Idents.get("SEL"), SelInfo);
+  }
+  return ObjCSelDecl;
 }
 
 void ASTContext::setObjCProtoType(QualType QT) {
   ObjCProtoType = QT;
 }
 
-void ASTContext::setObjCClassType(QualType T) {
-  ObjCClassTypedefType = T;
+TypedefDecl *ASTContext::getObjCClassDecl() const {
+  if (!ObjCClassDecl) {
+    QualType T = getObjCObjectType(ObjCBuiltinClassTy, 0, 0);
+    T = getObjCObjectPointerType(T);
+    TypeSourceInfo *ClassInfo = getTrivialTypeSourceInfo(T);
+    ObjCClassDecl = TypedefDecl::Create(const_cast<ASTContext &>(*this),
+                                        getTranslationUnitDecl(),
+                                        SourceLocation(), SourceLocation(),
+                                        &Idents.get("Class"), ClassInfo);
+  }
+  
+  return ObjCClassDecl;
 }
 
 void ASTContext::setObjCConstantStringInterface(ObjCInterfaceDecl *Decl) {
@@ -4925,8 +4975,7 @@ CanQualType ASTContext::getFromTargetType(unsigned Type) const {
   case TargetInfo::UnsignedLongLong: return UnsignedLongLongTy;
   }
 
-  assert(false && "Unhandled TargetInfo::IntType value");
-  return CanQualType();
+  llvm_unreachable("Unhandled TargetInfo::IntType value");
 }
 
 //===----------------------------------------------------------------------===//
@@ -4937,7 +4986,7 @@ CanQualType ASTContext::getFromTargetType(unsigned Type) const {
 /// garbage collection attribute.
 ///
 Qualifiers::GC ASTContext::getObjCGCAttrKind(QualType Ty) const {
-  if (getLangOptions().getGCMode() == LangOptions::NonGC)
+  if (getLangOptions().getGC() == LangOptions::NonGC)
     return Qualifiers::GCNone;
 
   assert(getLangOptions().ObjC1);
@@ -5261,7 +5310,7 @@ static
 void getIntersectionOfProtocols(ASTContext &Context,
                                 const ObjCObjectPointerType *LHSOPT,
                                 const ObjCObjectPointerType *RHSOPT,
-      llvm::SmallVectorImpl<ObjCProtocolDecl *> &IntersectionOfProtocols) {
+      SmallVectorImpl<ObjCProtocolDecl *> &IntersectionOfProtocols) {
   
   const ObjCObjectType* LHS = LHSOPT->getObjectType();
   const ObjCObjectType* RHS = RHSOPT->getObjectType();
@@ -5287,8 +5336,7 @@ void getIntersectionOfProtocols(ASTContext &Context,
     for (unsigned i = 0; i < RHSNumProtocols; ++i)
       if (InheritedProtocolSet.count(RHSProtocols[i]))
         IntersectionOfProtocols.push_back(RHSProtocols[i]);
-  }
-  else {
+  } else {
     llvm::SmallPtrSet<ObjCProtocolDecl *, 8> RHSInheritedProtocols;
     Context.CollectInheritedProtocols(RHS->getInterface(),
                                       RHSInheritedProtocols);
@@ -5317,7 +5365,7 @@ QualType ASTContext::areCommonBaseCompatible(
   do {
     LHS = cast<ObjCInterfaceType>(getObjCInterfaceType(LDecl));
     if (canAssignObjCInterfaces(LHS, RHS)) {
-      llvm::SmallVector<ObjCProtocolDecl *, 8> Protocols;
+      SmallVector<ObjCProtocolDecl *, 8> Protocols;
       getIntersectionOfProtocols(*this, Lptr, Rptr, Protocols);
 
       QualType Result = QualType(LHS, 0);
@@ -5552,13 +5600,13 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
   if (lbaseInfo.getProducesResult() != rbaseInfo.getProducesResult())
     return QualType();
 
-  // It's noreturn if either type is.
+  // functypes which return are preferred over those that do not.
+  if (lbaseInfo.getNoReturn() && !rbaseInfo.getNoReturn())
+    allLTypes = false;
+  else if (!lbaseInfo.getNoReturn() && rbaseInfo.getNoReturn())
+    allRTypes = false;
   // FIXME: some uses, e.g. conditional exprs, really want this to be 'both'.
   bool NoReturn = lbaseInfo.getNoReturn() || rbaseInfo.getNoReturn();
-  if (NoReturn != lbaseInfo.getNoReturn())
-    allLTypes = false;
-  if (NoReturn != rbaseInfo.getNoReturn())
-    allRTypes = false;
 
   FunctionType::ExtInfo einfo = lbaseInfo.withNoReturn(NoReturn);
 
@@ -5579,8 +5627,12 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     if (lproto->getTypeQuals() != rproto->getTypeQuals())
       return QualType();
 
+    if (LangOpts.ObjCAutoRefCount &&
+        !FunctionTypesMatchOnNSConsumedAttrs(rproto, lproto))
+      return QualType();
+      
     // Check argument compatibility
-    llvm::SmallVector<QualType, 10> types;
+    SmallVector<QualType, 10> types;
     for (unsigned i = 0; i < lproto_nargs; i++) {
       QualType largtype = lproto->getArgType(i).getUnqualifiedType();
       QualType rargtype = rproto->getArgType(i).getUnqualifiedType();
@@ -5603,6 +5655,7 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
       if (getCanonicalType(argtype) != getCanonicalType(rargtype))
         allRTypes = false;
     }
+      
     if (allLTypes) return lhs;
     if (allRTypes) return rhs;
 
@@ -5756,22 +5809,19 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #include "clang/AST/TypeNodes.def"
-    assert(false && "Non-canonical and dependent types shouldn't get here");
-    return QualType();
+    llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   case Type::LValueReference:
   case Type::RValueReference:
   case Type::MemberPointer:
-    assert(false && "C++ should never be in mergeTypes");
-    return QualType();
+    llvm_unreachable("C++ should never be in mergeTypes");
 
   case Type::ObjCInterface:
   case Type::IncompleteArray:
   case Type::VariableArray:
   case Type::FunctionProto:
   case Type::ExtVector:
-    assert(false && "Types are eliminated above");
-    return QualType();
+    llvm_unreachable("Types are eliminated above");
 
   case Type::Pointer:
   {
@@ -5809,6 +5859,24 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
       return RHS;
     return getBlockPointerType(ResultType);
   }
+  case Type::Atomic:
+  {
+    // Merge two pointer types, while trying to preserve typedef info
+    QualType LHSValue = LHS->getAs<AtomicType>()->getValueType();
+    QualType RHSValue = RHS->getAs<AtomicType>()->getValueType();
+    if (Unqualified) {
+      LHSValue = LHSValue.getUnqualifiedType();
+      RHSValue = RHSValue.getUnqualifiedType();
+    }
+    QualType ResultType = mergeTypes(LHSValue, RHSValue, false, 
+                                     Unqualified);
+    if (ResultType.isNull()) return QualType();
+    if (getCanonicalType(LHSValue) == getCanonicalType(ResultType))
+      return LHS;
+    if (getCanonicalType(RHSValue) == getCanonicalType(ResultType))
+      return RHS;
+    return getAtomicType(ResultType);
+  }
   case Type::ConstantArray:
   {
     const ConstantArrayType* LCAT = getAsConstantArrayType(LHS);
@@ -5904,6 +5972,26 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
   return QualType();
 }
 
+bool ASTContext::FunctionTypesMatchOnNSConsumedAttrs(
+                   const FunctionProtoType *FromFunctionType,
+                   const FunctionProtoType *ToFunctionType) {
+  if (FromFunctionType->hasAnyConsumedArgs() != 
+      ToFunctionType->hasAnyConsumedArgs())
+    return false;
+  FunctionProtoType::ExtProtoInfo FromEPI = 
+    FromFunctionType->getExtProtoInfo();
+  FunctionProtoType::ExtProtoInfo ToEPI = 
+    ToFunctionType->getExtProtoInfo();
+  if (FromEPI.ConsumedArguments && ToEPI.ConsumedArguments)
+    for (unsigned ArgIdx = 0, NumArgs = FromFunctionType->getNumArgs();
+         ArgIdx != NumArgs; ++ArgIdx)  {
+      if (FromEPI.ConsumedArguments[ArgIdx] != 
+          ToEPI.ConsumedArguments[ArgIdx])
+        return false;
+    }
+  return true;
+}
+
 /// mergeObjCGCQualifiers - This routine merges ObjC's GC attribute of 'LHS' and
 /// 'RHS' attributes and returns the merged version; including for function
 /// return types.
@@ -6022,8 +6110,7 @@ QualType ASTContext::getCorrespondingUnsignedType(QualType T) {
   case BuiltinType::Int128:
     return UnsignedInt128Ty;
   default:
-    assert(0 && "Unexpected signed integer type");
-    return QualType();
+    llvm_unreachable("Unexpected signed integer type");
   }
 }
 
@@ -6080,7 +6167,7 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
 
   // Read the base type.
   switch (*Str++) {
-  default: assert(0 && "Unknown builtin type letter!");
+  default: llvm_unreachable("Unknown builtin type letter!");
   case 'v':
     assert(HowLong == 0 && !Signed && !Unsigned &&
            "Bad modifiers used with 'v'!");
@@ -6183,7 +6270,11 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
     assert(!RequiresICE && "Can't require complex ICE");
     Type = Context.getComplexType(ElementType);
     break;
-  }      
+  }  
+  case 'Y' : {
+    Type = Context.getPointerDiffType();
+    break;
+  }
   case 'P':
     Type = Context.getFILEType();
     if (Type.isNull()) {
@@ -6247,7 +6338,7 @@ QualType ASTContext::GetBuiltinType(unsigned Id,
                                     unsigned *IntegerConstantArgs) const {
   const char *TypeStr = BuiltinInfo.GetTypeString(Id);
 
-  llvm::SmallVector<QualType, 8> ArgTypes;
+  SmallVector<QualType, 8> ArgTypes;
 
   bool RequiresICE = false;
   Error = GE_None;
@@ -6405,7 +6496,7 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // Forward declarations aren't required.
     if (!FD->doesThisDeclarationHaveABody())
-      return false;
+      return FD->doesDeclarationForceExternallyVisibleDefinition();
 
     // Constructors and destructors are required.
     if (FD->hasAttr<ConstructorAttr>() || FD->hasAttr<DestructorAttr>())
@@ -6431,7 +6522,7 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
       return false;
     return true;
   }
-
+  
   const VarDecl *VD = cast<VarDecl>(D);
   assert(VD->isFileVarDecl() && "Expected file scoped var");
 
@@ -6472,30 +6563,42 @@ bool ASTContext::isNearlyEmpty(const CXXRecordDecl *RD) const {
 }
 
 MangleContext *ASTContext::createMangleContext() {
-  switch (Target.getCXXABI()) {
+  switch (Target->getCXXABI()) {
   case CXXABI_ARM:
   case CXXABI_Itanium:
     return createItaniumMangleContext(*this, getDiagnostics());
   case CXXABI_Microsoft:
     return createMicrosoftMangleContext(*this, getDiagnostics());
   }
-  assert(0 && "Unsupported ABI");
-  return 0;
+  llvm_unreachable("Unsupported ABI");
 }
 
 CXXABI::~CXXABI() {}
 
 size_t ASTContext::getSideTableAllocatedMemory() const {
-  size_t bytes = 0;
-  bytes += ASTRecordLayouts.getMemorySize();
-  bytes += ObjCLayouts.getMemorySize();
-  bytes += KeyFunctions.getMemorySize();
-  bytes += ObjCImpls.getMemorySize();
-  bytes += BlockVarCopyInits.getMemorySize();
-  bytes += DeclAttrs.getMemorySize();
-  bytes += InstantiatedFromStaticDataMember.getMemorySize();
-  bytes += InstantiatedFromUsingDecl.getMemorySize();
-  bytes += InstantiatedFromUsingShadowDecl.getMemorySize();
-  bytes += InstantiatedFromUnnamedFieldDecl.getMemorySize();
-  return bytes;
+  return ASTRecordLayouts.getMemorySize()
+    + llvm::capacity_in_bytes(ObjCLayouts)
+    + llvm::capacity_in_bytes(KeyFunctions)
+    + llvm::capacity_in_bytes(ObjCImpls)
+    + llvm::capacity_in_bytes(BlockVarCopyInits)
+    + llvm::capacity_in_bytes(DeclAttrs)
+    + llvm::capacity_in_bytes(InstantiatedFromStaticDataMember)
+    + llvm::capacity_in_bytes(InstantiatedFromUsingDecl)
+    + llvm::capacity_in_bytes(InstantiatedFromUsingShadowDecl)
+    + llvm::capacity_in_bytes(InstantiatedFromUnnamedFieldDecl)
+    + llvm::capacity_in_bytes(OverriddenMethods)
+    + llvm::capacity_in_bytes(Types)
+    + llvm::capacity_in_bytes(VariableArrayTypes)
+    + llvm::capacity_in_bytes(ClassScopeSpecializationPattern);
+}
+
+void ASTContext::setParameterIndex(const ParmVarDecl *D, unsigned int index) {
+  ParamIndices[D] = index;
+}
+
+unsigned ASTContext::getParameterIndex(const ParmVarDecl *D) const {
+  ParameterIndexTable::const_iterator I = ParamIndices.find(D);
+  assert(I != ParamIndices.end() && 
+         "ParmIndices lacks entry set by ParmVarDecl");
+  return I->second;
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/ASTDiagnostic.cpp b/contrib/llvm/tools/clang/lib/AST/ASTDiagnostic.cpp
index 7c91b5cb7a00..07820dc3cc56 100644
--- a/contrib/llvm/tools/clang/lib/AST/ASTDiagnostic.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ASTDiagnostic.cpp
@@ -152,16 +152,16 @@ break; \
 /// diagnostic message
 static std::string
 ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
-                              const Diagnostic::ArgumentValue *PrevArgs,
+                              const DiagnosticsEngine::ArgumentValue *PrevArgs,
                               unsigned NumPrevArgs,
-                              llvm::SmallVectorImpl<intptr_t> &QualTypeVals) {
+                              SmallVectorImpl<intptr_t> &QualTypeVals) {
   // FIXME: Playing with std::string is really slow.
   bool ForceAKA = false;
   QualType CanTy = Ty.getCanonicalType();
-  std::string S = Ty.getAsString(Context.PrintingPolicy);
-  std::string CanS = CanTy.getAsString(Context.PrintingPolicy);
+  std::string S = Ty.getAsString(Context.getPrintingPolicy());
+  std::string CanS = CanTy.getAsString(Context.getPrintingPolicy());
 
-  for (llvm::SmallVectorImpl<intptr_t>::iterator I = QualTypeVals.begin(),
+  for (SmallVectorImpl<intptr_t>::iterator I = QualTypeVals.begin(),
        E = QualTypeVals.end(); I != E; ++I) {
     QualType CompareTy =
         QualType::getFromOpaquePtr(reinterpret_cast<void*>(*I));
@@ -170,10 +170,10 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
     QualType CompareCanTy = CompareTy.getCanonicalType();
     if (CompareCanTy == CanTy)
       continue;  // Same canonical types
-    std::string CompareS = CompareTy.getAsString(Context.PrintingPolicy);
+    std::string CompareS = CompareTy.getAsString(Context.getPrintingPolicy());
     if (CompareS != S)
       continue;  // Original strings are different
-    std::string CompareCanS = CompareCanTy.getAsString(Context.PrintingPolicy);
+    std::string CompareCanS = CompareCanTy.getAsString(Context.getPrintingPolicy());
     if (CompareCanS == CanS)
       continue;  // No new info from canonical type
 
@@ -186,7 +186,7 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
   bool Repeated = false;
   for (unsigned i = 0; i != NumPrevArgs; ++i) {
     // TODO: Handle ak_declcontext case.
-    if (PrevArgs[i].first == Diagnostic::ak_qualtype) {
+    if (PrevArgs[i].first == DiagnosticsEngine::ak_qualtype) {
       void *Ptr = (void*)PrevArgs[i].second;
       QualType PrevTy(QualType::getFromOpaquePtr(Ptr));
       if (PrevTy == Ty) {
@@ -205,7 +205,7 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
       if (DesugaredTy == Ty) {
         DesugaredTy = Ty.getCanonicalType();
       }
-      std::string akaStr = DesugaredTy.getAsString(Context.PrintingPolicy);
+      std::string akaStr = DesugaredTy.getAsString(Context.getPrintingPolicy());
       if (akaStr != S) {
         S = "'" + S + "' (aka '" + akaStr + "')";
         return S;
@@ -218,25 +218,25 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
 }
 
 void clang::FormatASTNodeDiagnosticArgument(
-    Diagnostic::ArgumentKind Kind,
+    DiagnosticsEngine::ArgumentKind Kind,
     intptr_t Val,
     const char *Modifier,
     unsigned ModLen,
     const char *Argument,
     unsigned ArgLen,
-    const Diagnostic::ArgumentValue *PrevArgs,
+    const DiagnosticsEngine::ArgumentValue *PrevArgs,
     unsigned NumPrevArgs,
-    llvm::SmallVectorImpl<char> &Output,
+    SmallVectorImpl<char> &Output,
     void *Cookie,
-    llvm::SmallVectorImpl<intptr_t> &QualTypeVals) {
+    SmallVectorImpl<intptr_t> &QualTypeVals) {
   ASTContext &Context = *static_cast<ASTContext*>(Cookie);
   
   std::string S;
   bool NeedQuotes = true;
   
   switch (Kind) {
-    default: assert(0 && "unknown ArgumentKind");
-    case Diagnostic::ak_qualtype: {
+    default: llvm_unreachable("unknown ArgumentKind");
+    case DiagnosticsEngine::ak_qualtype: {
       assert(ModLen == 0 && ArgLen == 0 &&
              "Invalid modifier for QualType argument");
       
@@ -246,7 +246,7 @@ void clang::FormatASTNodeDiagnosticArgument(
       NeedQuotes = false;
       break;
     }
-    case Diagnostic::ak_declarationname: {
+    case DiagnosticsEngine::ak_declarationname: {
       DeclarationName N = DeclarationName::getFromOpaqueInteger(Val);
       S = N.getAsString();
       
@@ -260,7 +260,7 @@ void clang::FormatASTNodeDiagnosticArgument(
                "Invalid modifier for DeclarationName argument");
       break;
     }
-    case Diagnostic::ak_nameddecl: {
+    case DiagnosticsEngine::ak_nameddecl: {
       bool Qualified;
       if (ModLen == 1 && Modifier[0] == 'q' && ArgLen == 0)
         Qualified = true;
@@ -269,18 +269,18 @@ void clang::FormatASTNodeDiagnosticArgument(
                "Invalid modifier for NamedDecl* argument");
         Qualified = false;
       }
-      reinterpret_cast<NamedDecl*>(Val)->
-      getNameForDiagnostic(S, Context.PrintingPolicy, Qualified);
+      const NamedDecl *ND = reinterpret_cast<const NamedDecl*>(Val);
+      ND->getNameForDiagnostic(S, Context.getPrintingPolicy(), Qualified);
       break;
     }
-    case Diagnostic::ak_nestednamespec: {
+    case DiagnosticsEngine::ak_nestednamespec: {
       llvm::raw_string_ostream OS(S);
       reinterpret_cast<NestedNameSpecifier*>(Val)->print(OS,
-                                                        Context.PrintingPolicy);
+                                                        Context.getPrintingPolicy());
       NeedQuotes = false;
       break;
     }
-    case Diagnostic::ak_declcontext: {
+    case DiagnosticsEngine::ak_declcontext: {
       DeclContext *DC = reinterpret_cast<DeclContext *> (Val);
       assert(DC && "Should never have a null declaration context");
       
@@ -305,7 +305,7 @@ void clang::FormatASTNodeDiagnosticArgument(
           S += "function ";
         
         S += "'";
-        ND->getNameForDiagnostic(S, Context.PrintingPolicy, true);
+        ND->getNameForDiagnostic(S, Context.getPrintingPolicy(), true);
         S += "'";
       }
       NeedQuotes = false;
diff --git a/contrib/llvm/tools/clang/lib/AST/ASTImporter.cpp b/contrib/llvm/tools/clang/lib/AST/ASTImporter.cpp
index f5e392f88d0b..af66b04c3ad7 100644
--- a/contrib/llvm/tools/clang/lib/AST/ASTImporter.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ASTImporter.cpp
@@ -59,6 +59,7 @@ namespace {
     QualType VisitFunctionNoProtoType(const FunctionNoProtoType *T);
     QualType VisitFunctionProtoType(const FunctionProtoType *T);
     // FIXME: UnresolvedUsingType
+    QualType VisitParenType(const ParenType *T);
     QualType VisitTypedefType(const TypedefType *T);
     QualType VisitTypeOfExprType(const TypeOfExprType *T);
     // FIXME: DependentTypeOfExprType
@@ -83,16 +84,20 @@ namespace {
     bool ImportDeclParts(NamedDecl *D, DeclContext *&DC, 
                          DeclContext *&LexicalDC, DeclarationName &Name, 
                          SourceLocation &Loc);
+    void ImportDefinitionIfNeeded(Decl *FromD, Decl *ToD = 0);
     void ImportDeclarationNameLoc(const DeclarationNameInfo &From,
                                   DeclarationNameInfo& To);
     void ImportDeclContext(DeclContext *FromDC, bool ForceImport = false);
-    bool ImportDefinition(RecordDecl *From, RecordDecl *To);
+    bool ImportDefinition(RecordDecl *From, RecordDecl *To, 
+                          bool ForceImport = false);
+    bool ImportDefinition(EnumDecl *From, EnumDecl *To,
+                          bool ForceImport = false);
     TemplateParameterList *ImportTemplateParameterList(
                                                  TemplateParameterList *Params);
     TemplateArgument ImportTemplateArgument(const TemplateArgument &From);
     bool ImportTemplateArguments(const TemplateArgument *FromArgs,
                                  unsigned NumFromArgs,
-                               llvm::SmallVectorImpl<TemplateArgument> &ToArgs);
+                               SmallVectorImpl<TemplateArgument> &ToArgs);
     bool IsStructuralMatch(RecordDecl *FromRecord, RecordDecl *ToRecord);
     bool IsStructuralMatch(EnumDecl *FromEnum, EnumDecl *ToRecord);
     bool IsStructuralMatch(ClassTemplateDecl *From, ClassTemplateDecl *To);
@@ -805,12 +810,74 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
     break;
   }
-      
+
+  case Type::Atomic: {
+    if (!IsStructurallyEquivalent(Context,
+                                  cast<AtomicType>(T1)->getValueType(),
+                                  cast<AtomicType>(T2)->getValueType()))
+      return false;
+    break;
+  }
+
   } // end switch
 
   return true;
 }
 
+/// \brief Determine structural equivalence of two fields.
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     FieldDecl *Field1, FieldDecl *Field2) {
+  RecordDecl *Owner2 = cast<RecordDecl>(Field2->getDeclContext());
+  
+  if (!IsStructurallyEquivalent(Context, 
+                                Field1->getType(), Field2->getType())) {
+    Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+    << Context.C2.getTypeDeclType(Owner2);
+    Context.Diag2(Field2->getLocation(), diag::note_odr_field)
+    << Field2->getDeclName() << Field2->getType();
+    Context.Diag1(Field1->getLocation(), diag::note_odr_field)
+    << Field1->getDeclName() << Field1->getType();
+    return false;
+  }
+  
+  if (Field1->isBitField() != Field2->isBitField()) {
+    Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+    << Context.C2.getTypeDeclType(Owner2);
+    if (Field1->isBitField()) {
+      Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
+      << Field1->getDeclName() << Field1->getType()
+      << Field1->getBitWidthValue(Context.C1);
+      Context.Diag2(Field2->getLocation(), diag::note_odr_not_bit_field)
+      << Field2->getDeclName();
+    } else {
+      Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
+      << Field2->getDeclName() << Field2->getType()
+      << Field2->getBitWidthValue(Context.C2);
+      Context.Diag1(Field1->getLocation(), diag::note_odr_not_bit_field)
+      << Field1->getDeclName();
+    }
+    return false;
+  }
+  
+  if (Field1->isBitField()) {
+    // Make sure that the bit-fields are the same length.
+    unsigned Bits1 = Field1->getBitWidthValue(Context.C1);
+    unsigned Bits2 = Field2->getBitWidthValue(Context.C2);
+    
+    if (Bits1 != Bits2) {
+      Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+      << Context.C2.getTypeDeclType(Owner2);
+      Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
+      << Field2->getDeclName() << Field2->getType() << Bits2;
+      Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
+      << Field1->getDeclName() << Field1->getType() << Bits1;
+      return false;
+    }
+  }
+
+  return true;
+}
+
 /// \brief Determine structural equivalence of two records.
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      RecordDecl *D1, RecordDecl *D2) {
@@ -928,61 +995,8 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
     }
     
-    if (!IsStructurallyEquivalent(Context, 
-                                  Field1->getType(), Field2->getType())) {
-      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-        << Context.C2.getTypeDeclType(D2);
-      Context.Diag2(Field2->getLocation(), diag::note_odr_field)
-        << Field2->getDeclName() << Field2->getType();
-      Context.Diag1(Field1->getLocation(), diag::note_odr_field)
-        << Field1->getDeclName() << Field1->getType();
-      return false;
-    }
-    
-    if (Field1->isBitField() != Field2->isBitField()) {
-      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-        << Context.C2.getTypeDeclType(D2);
-      if (Field1->isBitField()) {
-        llvm::APSInt Bits;
-        Field1->getBitWidth()->isIntegerConstantExpr(Bits, Context.C1);
-        Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
-          << Field1->getDeclName() << Field1->getType()
-          << Bits.toString(10, false);
-        Context.Diag2(Field2->getLocation(), diag::note_odr_not_bit_field)
-          << Field2->getDeclName();
-      } else {
-        llvm::APSInt Bits;
-        Field2->getBitWidth()->isIntegerConstantExpr(Bits, Context.C2);
-        Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
-          << Field2->getDeclName() << Field2->getType()
-          << Bits.toString(10, false);
-        Context.Diag1(Field1->getLocation(), 
-                          diag::note_odr_not_bit_field)
-        << Field1->getDeclName();
-      }
-      return false;
-    }
-    
-    if (Field1->isBitField()) {
-      // Make sure that the bit-fields are the same length.
-      llvm::APSInt Bits1, Bits2;
-      if (!Field1->getBitWidth()->isIntegerConstantExpr(Bits1, Context.C1))
-        return false;
-      if (!Field2->getBitWidth()->isIntegerConstantExpr(Bits2, Context.C2))
-        return false;
-      
-      if (!IsSameValue(Bits1, Bits2)) {
-        Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-          << Context.C2.getTypeDeclType(D2);
-        Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
-          << Field2->getDeclName() << Field2->getType()
-          << Bits2.toString(10, false);
-        Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
-          << Field1->getDeclName() << Field1->getType()
-          << Bits1.toString(10, false);
-        return false;
-      }
-    }
+    if (!IsStructurallyEquivalent(Context, *Field1, *Field2))
+      return false;    
   }
   
   if (Field2 != Field2End) {
@@ -1360,6 +1374,7 @@ QualType ASTNodeImporter::VisitBuiltinType(const BuiltinType *T) {
   case BuiltinType::Long : return Importer.getToContext().LongTy;
   case BuiltinType::LongLong : return Importer.getToContext().LongLongTy;
   case BuiltinType::Int128 : return Importer.getToContext().Int128Ty;
+  case BuiltinType::Half: return Importer.getToContext().HalfTy;
   case BuiltinType::Float: return Importer.getToContext().FloatTy;
   case BuiltinType::Double: return Importer.getToContext().DoubleTy;
   case BuiltinType::LongDouble: return Importer.getToContext().LongDoubleTy;
@@ -1518,7 +1533,7 @@ QualType ASTNodeImporter::VisitFunctionProtoType(const FunctionProtoType *T) {
     return QualType();
   
   // Import argument types
-  llvm::SmallVector<QualType, 4> ArgTypes;
+  SmallVector<QualType, 4> ArgTypes;
   for (FunctionProtoType::arg_type_iterator A = T->arg_type_begin(),
                                          AEnd = T->arg_type_end();
        A != AEnd; ++A) {
@@ -1529,7 +1544,7 @@ QualType ASTNodeImporter::VisitFunctionProtoType(const FunctionProtoType *T) {
   }
   
   // Import exception types
-  llvm::SmallVector<QualType, 4> ExceptionTypes;
+  SmallVector<QualType, 4> ExceptionTypes;
   for (FunctionProtoType::exception_iterator E = T->exception_begin(),
                                           EEnd = T->exception_end();
        E != EEnd; ++E) {
@@ -1546,6 +1561,14 @@ QualType ASTNodeImporter::VisitFunctionProtoType(const FunctionProtoType *T) {
                                                  ArgTypes.size(), EPI);
 }
 
+QualType ASTNodeImporter::VisitParenType(const ParenType *T) {
+  QualType ToInnerType = Importer.Import(T->getInnerType());
+  if (ToInnerType.isNull())
+    return QualType();
+    
+  return Importer.getToContext().getParenType(ToInnerType);
+}
+
 QualType ASTNodeImporter::VisitTypedefType(const TypedefType *T) {
   TypedefNameDecl *ToDecl
              = dyn_cast_or_null<TypedefNameDecl>(Importer.Import(T->getDecl()));
@@ -1628,7 +1651,7 @@ QualType ASTNodeImporter::VisitTemplateSpecializationType(
   if (ToTemplate.isNull())
     return QualType();
   
-  llvm::SmallVector<TemplateArgument, 2> ToTemplateArgs;
+  SmallVector<TemplateArgument, 2> ToTemplateArgs;
   if (ImportTemplateArguments(T->getArgs(), T->getNumArgs(), ToTemplateArgs))
     return QualType();
   
@@ -1677,7 +1700,7 @@ QualType ASTNodeImporter::VisitObjCObjectType(const ObjCObjectType *T) {
   if (ToBaseType.isNull())
     return QualType();
 
-  llvm::SmallVector<ObjCProtocolDecl *, 4> Protocols;
+  SmallVector<ObjCProtocolDecl *, 4> Protocols;
   for (ObjCObjectType::qual_iterator P = T->qual_begin(), 
                                      PEnd = T->qual_end();
        P != PEnd; ++P) {
@@ -1731,6 +1754,35 @@ bool ASTNodeImporter::ImportDeclParts(NamedDecl *D, DeclContext *&DC,
   return false;
 }
 
+void ASTNodeImporter::ImportDefinitionIfNeeded(Decl *FromD, Decl *ToD) {
+  if (!FromD)
+    return;
+  
+  if (!ToD) {
+    ToD = Importer.Import(FromD);
+    if (!ToD)
+      return;
+  }
+  
+  if (RecordDecl *FromRecord = dyn_cast<RecordDecl>(FromD)) {
+    if (RecordDecl *ToRecord = cast_or_null<RecordDecl>(ToD)) {
+      if (FromRecord->getDefinition() && !ToRecord->getDefinition()) {
+        ImportDefinition(FromRecord, ToRecord);
+      }
+    }
+    return;
+  }
+
+  if (EnumDecl *FromEnum = dyn_cast<EnumDecl>(FromD)) {
+    if (EnumDecl *ToEnum = cast_or_null<EnumDecl>(ToD)) {
+      if (FromEnum->getDefinition() && !ToEnum->getDefinition()) {
+        ImportDefinition(FromEnum, ToEnum);
+      }
+    }
+    return;
+  }
+}
+
 void
 ASTNodeImporter::ImportDeclarationNameLoc(const DeclarationNameInfo &From,
                                           DeclarationNameInfo& To) {
@@ -1761,16 +1813,13 @@ ASTNodeImporter::ImportDeclarationNameLoc(const DeclarationNameInfo &From,
     To.setNamedTypeInfo(Importer.Import(FromTInfo));
     return;
   }
-    assert(0 && "Unknown name kind.");
+    llvm_unreachable("Unknown name kind.");
   }
 }
 
 void ASTNodeImporter::ImportDeclContext(DeclContext *FromDC, bool ForceImport) {
   if (Importer.isMinimalImport() && !ForceImport) {
-    if (DeclContext *ToDC = Importer.ImportContext(FromDC)) {
-      ToDC->setHasExternalLexicalStorage();
-      ToDC->setHasExternalVisibleStorage();
-    }
+    Importer.ImportContext(FromDC);
     return;
   }
   
@@ -1781,8 +1830,9 @@ void ASTNodeImporter::ImportDeclContext(DeclContext *FromDC, bool ForceImport) {
     Importer.Import(*From);
 }
 
-bool ASTNodeImporter::ImportDefinition(RecordDecl *From, RecordDecl *To) {
-  if (To->getDefinition())
+bool ASTNodeImporter::ImportDefinition(RecordDecl *From, RecordDecl *To, 
+                                       bool ForceImport) {
+  if (To->getDefinition() || To->isBeingDefined())
     return false;
   
   To->startDefinition();
@@ -1791,7 +1841,7 @@ bool ASTNodeImporter::ImportDefinition(RecordDecl *From, RecordDecl *To) {
   if (CXXRecordDecl *ToCXX = dyn_cast<CXXRecordDecl>(To)) {
     CXXRecordDecl *FromCXX = cast<CXXRecordDecl>(From);
     
-    llvm::SmallVector<CXXBaseSpecifier *, 4> Bases;
+    SmallVector<CXXBaseSpecifier *, 4> Bases;
     for (CXXRecordDecl::base_class_iterator 
                   Base1 = FromCXX->bases_begin(),
             FromBaseEnd = FromCXX->bases_end();
@@ -1804,7 +1854,10 @@ bool ASTNodeImporter::ImportDefinition(RecordDecl *From, RecordDecl *To) {
       SourceLocation EllipsisLoc;
       if (Base1->isPackExpansion())
         EllipsisLoc = Importer.Import(Base1->getEllipsisLoc());
-      
+
+      // Ensure that we have a definition for the base.
+      ImportDefinitionIfNeeded(Base1->getType()->getAsCXXRecordDecl());
+        
       Bases.push_back(
                     new (Importer.getToContext()) 
                       CXXBaseSpecifier(Importer.Import(Base1->getSourceRange()),
@@ -1818,14 +1871,39 @@ bool ASTNodeImporter::ImportDefinition(RecordDecl *From, RecordDecl *To) {
       ToCXX->setBases(Bases.data(), Bases.size());
   }
   
-  ImportDeclContext(From);
+  ImportDeclContext(From, ForceImport);
   To->completeDefinition();
   return false;
 }
 
+bool ASTNodeImporter::ImportDefinition(EnumDecl *From, EnumDecl *To, 
+                                       bool ForceImport) {
+  if (To->getDefinition() || To->isBeingDefined())
+    return false;
+  
+  To->startDefinition();
+
+  QualType T = Importer.Import(Importer.getFromContext().getTypeDeclType(From));
+  if (T.isNull())
+    return true;
+  
+  QualType ToPromotionType = Importer.Import(From->getPromotionType());
+  if (ToPromotionType.isNull())
+    return true;
+  
+  ImportDeclContext(From, ForceImport);
+  
+  // FIXME: we might need to merge the number of positive or negative bits
+  // if the enumerator lists don't match.
+  To->completeDefinition(T, ToPromotionType,
+                         From->getNumPositiveBits(),
+                         From->getNumNegativeBits());
+  return false;
+}
+
 TemplateParameterList *ASTNodeImporter::ImportTemplateParameterList(
                                                 TemplateParameterList *Params) {
-  llvm::SmallVector<NamedDecl *, 4> ToParams;
+  SmallVector<NamedDecl *, 4> ToParams;
   ToParams.reserve(Params->size());
   for (TemplateParameterList::iterator P = Params->begin(), 
                                     PEnd = Params->end();
@@ -1892,7 +1970,7 @@ ASTNodeImporter::ImportTemplateArgument(const TemplateArgument &From) {
     return TemplateArgument();
       
   case TemplateArgument::Pack: {
-    llvm::SmallVector<TemplateArgument, 2> ToPack;
+    SmallVector<TemplateArgument, 2> ToPack;
     ToPack.reserve(From.pack_size());
     if (ImportTemplateArguments(From.pack_begin(), From.pack_size(), ToPack))
       return TemplateArgument();
@@ -1910,7 +1988,7 @@ ASTNodeImporter::ImportTemplateArgument(const TemplateArgument &From) {
 
 bool ASTNodeImporter::ImportTemplateArguments(const TemplateArgument *FromArgs,
                                               unsigned NumFromArgs,
-                              llvm::SmallVectorImpl<TemplateArgument> &ToArgs) {
+                              SmallVectorImpl<TemplateArgument> &ToArgs) {
   for (unsigned I = 0; I != NumFromArgs; ++I) {
     TemplateArgument To = ImportTemplateArgument(FromArgs[I]);
     if (To.isNull() && !FromArgs[I].isNull())
@@ -1969,20 +2047,20 @@ Decl *ASTNodeImporter::VisitNamespaceDecl(NamespaceDecl *D) {
     else
       MergeWithNamespace = cast<NamespaceDecl>(DC)->getAnonymousNamespace();
   } else {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(Decl::IDNS_Namespace))
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(Name, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(Decl::IDNS_Namespace))
         continue;
       
-      if (NamespaceDecl *FoundNS = dyn_cast<NamespaceDecl>(*Lookup.first)) {
+      if (NamespaceDecl *FoundNS = dyn_cast<NamespaceDecl>(FoundDecls[I])) {
         MergeWithNamespace = FoundNS;
         ConflictingDecls.clear();
         break;
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -2029,21 +2107,21 @@ Decl *ASTNodeImporter::VisitTypedefNameDecl(TypedefNameDecl *D, bool IsAlias) {
   // seen a typedef with the same name (that we can merge with) or any
   // other entity by that name (which name lookup could conflict with).
   if (!DC->isFunctionOrMethod()) {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
     unsigned IDNS = Decl::IDNS_Ordinary;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(IDNS))
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(Name, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(IDNS))
         continue;
       if (TypedefNameDecl *FoundTypedef =
-            dyn_cast<TypedefNameDecl>(*Lookup.first)) {
+            dyn_cast<TypedefNameDecl>(FoundDecls[I])) {
         if (Importer.IsStructurallyEquivalent(D->getUnderlyingType(),
                                             FoundTypedef->getUnderlyingType()))
           return Importer.Imported(D, FoundTypedef);
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -2065,15 +2143,16 @@ Decl *ASTNodeImporter::VisitTypedefNameDecl(TypedefNameDecl *D, bool IsAlias) {
   SourceLocation StartL = Importer.Import(D->getLocStart());
   TypedefNameDecl *ToTypedef;
   if (IsAlias)
+    ToTypedef = TypeAliasDecl::Create(Importer.getToContext(), DC,
+                                      StartL, Loc,
+                                      Name.getAsIdentifierInfo(),
+                                      TInfo);
+  else
     ToTypedef = TypedefDecl::Create(Importer.getToContext(), DC,
                                     StartL, Loc,
                                     Name.getAsIdentifierInfo(),
                                     TInfo);
-  else
-    ToTypedef = TypeAliasDecl::Create(Importer.getToContext(), DC,
-                                  StartL, Loc,
-                                  Name.getAsIdentifierInfo(),
-                                  TInfo);
+  
   ToTypedef->setAccess(D->getAccess());
   ToTypedef->setLexicalDeclContext(LexicalDC);
   Importer.Imported(D, ToTypedef);
@@ -2109,14 +2188,14 @@ Decl *ASTNodeImporter::VisitEnumDecl(EnumDecl *D) {
   
   // We may already have an enum of the same name; try to find and match it.
   if (!DC->isFunctionOrMethod() && SearchName) {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(IDNS))
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(SearchName, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(IDNS))
         continue;
       
-      Decl *Found = *Lookup.first;
+      Decl *Found = FoundDecls[I];
       if (TypedefNameDecl *Typedef = dyn_cast<TypedefNameDecl>(Found)) {
         if (const TagType *Tag = Typedef->getUnderlyingType()->getAs<TagType>())
           Found = Tag->getDecl();
@@ -2127,7 +2206,7 @@ Decl *ASTNodeImporter::VisitEnumDecl(EnumDecl *D) {
           return Importer.Imported(D, FoundEnum);
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -2157,25 +2236,9 @@ Decl *ASTNodeImporter::VisitEnumDecl(EnumDecl *D) {
   D2->setIntegerType(ToIntegerType);
   
   // Import the definition
-  if (D->isDefinition()) {
-    QualType T = Importer.Import(Importer.getFromContext().getTypeDeclType(D));
-    if (T.isNull())
-      return 0;
+  if (D->isCompleteDefinition() && ImportDefinition(D, D2))
+    return 0;
 
-    QualType ToPromotionType = Importer.Import(D->getPromotionType());
-    if (ToPromotionType.isNull())
-      return 0;
-    
-    D2->startDefinition();
-    ImportDeclContext(D);
-
-    // FIXME: we might need to merge the number of positive or negative bits
-    // if the enumerator lists don't match.
-    D2->completeDefinition(T, ToPromotionType,
-                           D->getNumPositiveBits(),
-                           D->getNumNegativeBits());
-  }
-  
   return D2;
 }
 
@@ -2211,14 +2274,14 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
   // We may already have a record of the same name; try to find and match it.
   RecordDecl *AdoptDecl = 0;
   if (!DC->isFunctionOrMethod() && SearchName) {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(IDNS))
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(SearchName, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(IDNS))
         continue;
       
-      Decl *Found = *Lookup.first;
+      Decl *Found = FoundDecls[I];
       if (TypedefNameDecl *Typedef = dyn_cast<TypedefNameDecl>(Found)) {
         if (const TagType *Tag = Typedef->getUnderlyingType()->getAs<TagType>())
           Found = Tag->getDecl();
@@ -2226,7 +2289,7 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
       
       if (RecordDecl *FoundRecord = dyn_cast<RecordDecl>(Found)) {
         if (RecordDecl *FoundDef = FoundRecord->getDefinition()) {
-          if (!D->isDefinition() || IsStructuralMatch(D, FoundDef)) {
+          if (!D->isCompleteDefinition() || IsStructuralMatch(D, FoundDef)) {
             // The record types structurally match, or the "from" translation
             // unit only had a forward declaration anyway; call it the same
             // function.
@@ -2241,7 +2304,7 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
         }          
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -2274,7 +2337,7 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
   
   Importer.Imported(D, D2);
 
-  if (D->isDefinition() && ImportDefinition(D, D2))
+  if (D->isCompleteDefinition() && ImportDefinition(D, D2))
     return 0;
   
   return D2;
@@ -2295,15 +2358,15 @@ Decl *ASTNodeImporter::VisitEnumConstantDecl(EnumConstantDecl *D) {
   // Determine whether there are any other declarations with the same name and 
   // in the same context.
   if (!LexicalDC->isFunctionOrMethod()) {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
     unsigned IDNS = Decl::IDNS_Ordinary;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(IDNS))
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(Name, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(IDNS))
         continue;
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -2341,15 +2404,15 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   // Try to find a function in our own ("to") context with the same name, same
   // type, and in the same context as the function we're importing.
   if (!LexicalDC->isFunctionOrMethod()) {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
     unsigned IDNS = Decl::IDNS_Ordinary;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(IDNS))
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(Name, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(IDNS))
         continue;
     
-      if (FunctionDecl *FoundFunction = dyn_cast<FunctionDecl>(*Lookup.first)) {
+      if (FunctionDecl *FoundFunction = dyn_cast<FunctionDecl>(FoundDecls[I])) {
         if (isExternalLinkage(FoundFunction->getLinkage()) &&
             isExternalLinkage(D->getLinkage())) {
           if (Importer.IsStructurallyEquivalent(D->getType(), 
@@ -2374,7 +2437,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
         }
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -2396,7 +2459,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
     return 0;
   
   // Import the function parameters.
-  llvm::SmallVector<ParmVarDecl *, 8> Parameters;
+  SmallVector<ParmVarDecl *, 8> Parameters;
   for (FunctionDecl::param_iterator P = D->param_begin(), PEnd = D->param_end();
        P != PEnd; ++P) {
     ParmVarDecl *ToP = cast_or_null<ParmVarDecl>(Importer.Import(*P));
@@ -2416,7 +2479,8 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
                                             NameInfo, T, TInfo, 
                                             FromConstructor->isExplicit(),
                                             D->isInlineSpecified(), 
-                                            D->isImplicit());
+                                            D->isImplicit(),
+                                            D->isConstexpr());
   } else if (isa<CXXDestructorDecl>(D)) {
     ToFunction = CXXDestructorDecl::Create(Importer.getToContext(),
                                            cast<CXXRecordDecl>(DC),
@@ -2432,6 +2496,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
                                            NameInfo, T, TInfo,
                                            D->isInlineSpecified(),
                                            FromConversion->isExplicit(),
+                                           D->isConstexpr(),
                                            Importer.Import(D->getLocEnd()));
   } else if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
     ToFunction = CXXMethodDecl::Create(Importer.getToContext(), 
@@ -2441,6 +2506,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
                                        Method->isStatic(),
                                        Method->getStorageClassAsWritten(),
                                        Method->isInlineSpecified(),
+                                       D->isConstexpr(),
                                        Importer.Import(D->getLocEnd()));
   } else {
     ToFunction = FunctionDecl::Create(Importer.getToContext(), DC,
@@ -2448,7 +2514,8 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
                                       NameInfo, T, TInfo, D->getStorageClass(),
                                       D->getStorageClassAsWritten(),
                                       D->isInlineSpecified(),
-                                      D->hasWrittenPrototype());
+                                      D->hasWrittenPrototype(),
+                                      D->isConstexpr());
   }
 
   // Import the qualifier, if any.
@@ -2465,7 +2532,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
     Parameters[I]->setOwningFunction(ToFunction);
     ToFunction->addDecl(Parameters[I]);
   }
-  ToFunction->setParams(Parameters.data(), Parameters.size());
+  ToFunction->setParams(Parameters);
 
   // FIXME: Other bits to merge?
 
@@ -2499,6 +2566,25 @@ Decl *ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   if (ImportDeclParts(D, DC, LexicalDC, Name, Loc))
     return 0;
   
+  // Determine whether we've already imported this field. 
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+    if (FieldDecl *FoundField = dyn_cast<FieldDecl>(FoundDecls[I])) {
+      if (Importer.IsStructurallyEquivalent(D->getType(), 
+                                            FoundField->getType())) {
+        Importer.Imported(D, FoundField);
+        return FoundField;
+      }
+      
+      Importer.ToDiag(Loc, diag::err_odr_field_type_inconsistent)
+        << Name << D->getType() << FoundField->getType();
+      Importer.ToDiag(FoundField->getLocation(), diag::note_odr_value_here)
+        << FoundField->getType();
+      return 0;
+    }
+  }
+
   // Import the type.
   QualType T = Importer.Import(D->getType());
   if (T.isNull())
@@ -2531,6 +2617,26 @@ Decl *ASTNodeImporter::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
   if (ImportDeclParts(D, DC, LexicalDC, Name, Loc))
     return 0;
 
+  // Determine whether we've already imported this field. 
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+    if (IndirectFieldDecl *FoundField 
+                                = dyn_cast<IndirectFieldDecl>(FoundDecls[I])) {
+      if (Importer.IsStructurallyEquivalent(D->getType(), 
+                                            FoundField->getType())) {
+        Importer.Imported(D, FoundField);
+        return FoundField;
+      }
+      
+      Importer.ToDiag(Loc, diag::err_odr_field_type_inconsistent)
+        << Name << D->getType() << FoundField->getType();
+      Importer.ToDiag(FoundField->getLocation(), diag::note_odr_value_here)
+        << FoundField->getType();
+      return 0;
+    }
+  }
+
   // Import the type.
   QualType T = Importer.Import(D->getType());
   if (T.isNull())
@@ -2568,10 +2674,10 @@ Decl *ASTNodeImporter::VisitObjCIvarDecl(ObjCIvarDecl *D) {
     return 0;
   
   // Determine whether we've already imported this ivar 
-  for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-       Lookup.first != Lookup.second; 
-       ++Lookup.first) {
-    if (ObjCIvarDecl *FoundIvar = dyn_cast<ObjCIvarDecl>(*Lookup.first)) {
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+    if (ObjCIvarDecl *FoundIvar = dyn_cast<ObjCIvarDecl>(FoundDecls[I])) {
       if (Importer.IsStructurallyEquivalent(D->getType(), 
                                             FoundIvar->getType())) {
         Importer.Imported(D, FoundIvar);
@@ -2621,15 +2727,15 @@ Decl *ASTNodeImporter::VisitVarDecl(VarDecl *D) {
   // in the same context as the variable we're importing.
   if (D->isFileVarDecl()) {
     VarDecl *MergeWithVar = 0;
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
     unsigned IDNS = Decl::IDNS_Ordinary;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(IDNS))
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(Name, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(IDNS))
         continue;
       
-      if (VarDecl *FoundVar = dyn_cast<VarDecl>(*Lookup.first)) {
+      if (VarDecl *FoundVar = dyn_cast<VarDecl>(FoundDecls[I])) {
         // We have found a variable that we may need to merge with. Check it.
         if (isExternalLinkage(FoundVar->getLinkage()) &&
             isExternalLinkage(D->getLinkage())) {
@@ -2668,7 +2774,7 @@ Decl *ASTNodeImporter::VisitVarDecl(VarDecl *D) {
         }
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
 
     if (MergeWithVar) {
@@ -2794,10 +2900,10 @@ Decl *ASTNodeImporter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   if (ImportDeclParts(D, DC, LexicalDC, Name, Loc))
     return 0;
   
-  for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-       Lookup.first != Lookup.second; 
-       ++Lookup.first) {
-    if (ObjCMethodDecl *FoundMethod = dyn_cast<ObjCMethodDecl>(*Lookup.first)) {
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+    if (ObjCMethodDecl *FoundMethod = dyn_cast<ObjCMethodDecl>(FoundDecls[I])) {
       if (FoundMethod->isInstanceMethod() != D->isInstanceMethod())
         continue;
 
@@ -2872,6 +2978,7 @@ Decl *ASTNodeImporter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
                              D->isInstanceMethod(),
                              D->isVariadic(),
                              D->isSynthesized(),
+                             D->isImplicit(),
                              D->isDefined(),
                              D->getImplementationControl(),
                              D->hasRelatedResultType());
@@ -2880,7 +2987,7 @@ Decl *ASTNodeImporter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   // deal with implicit parameters.
 
   // Import the parameters
-  llvm::SmallVector<ParmVarDecl *, 5> ToParams;
+  SmallVector<ParmVarDecl *, 5> ToParams;
   for (ObjCMethodDecl::param_iterator FromP = D->param_begin(),
                                    FromPEnd = D->param_end();
        FromP != FromPEnd; 
@@ -2897,9 +3004,9 @@ Decl *ASTNodeImporter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
     ToParams[I]->setOwningFunction(ToMethod);
     ToMethod->addDecl(ToParams[I]);
   }
-  ToMethod->setMethodParams(Importer.getToContext(), 
-                            ToParams.data(), ToParams.size(),
-                            ToParams.size());
+  SmallVector<SourceLocation, 12> SelLocs;
+  D->getSelectorLocs(SelLocs);
+  ToMethod->setMethodParams(Importer.getToContext(), ToParams, SelLocs); 
 
   ToMethod->setLexicalDeclContext(LexicalDC);
   Importer.Imported(D, ToMethod);
@@ -2926,21 +3033,18 @@ Decl *ASTNodeImporter::VisitObjCCategoryDecl(ObjCCategoryDecl *D) {
   ObjCCategoryDecl *ToCategory = MergeWithCategory;
   if (!ToCategory) {
     ToCategory = ObjCCategoryDecl::Create(Importer.getToContext(), DC,
-                                          Importer.Import(D->getAtLoc()),
+                                          Importer.Import(D->getAtStartLoc()),
                                           Loc, 
                                        Importer.Import(D->getCategoryNameLoc()), 
-                                          Name.getAsIdentifierInfo());
+                                          Name.getAsIdentifierInfo(),
+                                          ToInterface);
     ToCategory->setLexicalDeclContext(LexicalDC);
     LexicalDC->addDecl(ToCategory);
     Importer.Imported(D, ToCategory);
     
-    // Link this category into its class's category list.
-    ToCategory->setClassInterface(ToInterface);
-    ToCategory->insertNextClassCategory();
-    
     // Import protocols
-    llvm::SmallVector<ObjCProtocolDecl *, 4> Protocols;
-    llvm::SmallVector<SourceLocation, 4> ProtocolLocs;
+    SmallVector<ObjCProtocolDecl *, 4> Protocols;
+    SmallVector<SourceLocation, 4> ProtocolLocs;
     ObjCCategoryDecl::protocol_loc_iterator FromProtoLoc
       = D->protocol_loc_begin();
     for (ObjCCategoryDecl::protocol_iterator FromProto = D->protocol_begin(),
@@ -2989,21 +3093,22 @@ Decl *ASTNodeImporter::VisitObjCProtocolDecl(ObjCProtocolDecl *D) {
     return 0;
 
   ObjCProtocolDecl *MergeWithProtocol = 0;
-  for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-       Lookup.first != Lookup.second; 
-       ++Lookup.first) {
-    if (!(*Lookup.first)->isInIdentifierNamespace(Decl::IDNS_ObjCProtocol))
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+    if (!FoundDecls[I]->isInIdentifierNamespace(Decl::IDNS_ObjCProtocol))
       continue;
     
-    if ((MergeWithProtocol = dyn_cast<ObjCProtocolDecl>(*Lookup.first)))
+    if ((MergeWithProtocol = dyn_cast<ObjCProtocolDecl>(FoundDecls[I])))
       break;
   }
   
   ObjCProtocolDecl *ToProto = MergeWithProtocol;
   if (!ToProto || ToProto->isForwardDecl()) {
     if (!ToProto) {
-      ToProto = ObjCProtocolDecl::Create(Importer.getToContext(), DC, Loc,
-                                         Name.getAsIdentifierInfo());
+      ToProto = ObjCProtocolDecl::Create(Importer.getToContext(), DC,
+                                         Name.getAsIdentifierInfo(), Loc,
+                                         Importer.Import(D->getAtStartLoc()));
       ToProto->setForwardDecl(D->isForwardDecl());
       ToProto->setLexicalDeclContext(LexicalDC);
       LexicalDC->addDecl(ToProto);
@@ -3011,8 +3116,8 @@ Decl *ASTNodeImporter::VisitObjCProtocolDecl(ObjCProtocolDecl *D) {
     Importer.Imported(D, ToProto);
 
     // Import protocols
-    llvm::SmallVector<ObjCProtocolDecl *, 4> Protocols;
-    llvm::SmallVector<SourceLocation, 4> ProtocolLocs;
+    SmallVector<ObjCProtocolDecl *, 4> Protocols;
+    SmallVector<SourceLocation, 4> ProtocolLocs;
     ObjCProtocolDecl::protocol_loc_iterator 
       FromProtoLoc = D->protocol_loc_begin();
     for (ObjCProtocolDecl::protocol_iterator FromProto = D->protocol_begin(),
@@ -3049,23 +3154,22 @@ Decl *ASTNodeImporter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
     return 0;
 
   ObjCInterfaceDecl *MergeWithIface = 0;
-  for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-       Lookup.first != Lookup.second; 
-       ++Lookup.first) {
-    if (!(*Lookup.first)->isInIdentifierNamespace(Decl::IDNS_Ordinary))
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+    if (!FoundDecls[I]->isInIdentifierNamespace(Decl::IDNS_Ordinary))
       continue;
     
-    if ((MergeWithIface = dyn_cast<ObjCInterfaceDecl>(*Lookup.first)))
+    if ((MergeWithIface = dyn_cast<ObjCInterfaceDecl>(FoundDecls[I])))
       break;
   }
   
   ObjCInterfaceDecl *ToIface = MergeWithIface;
   if (!ToIface || ToIface->isForwardDecl()) {
     if (!ToIface) {
-      ToIface = ObjCInterfaceDecl::Create(Importer.getToContext(),
-                                          DC, Loc,
-                                          Name.getAsIdentifierInfo(),
-                                          Importer.Import(D->getClassLoc()),
+      ToIface = ObjCInterfaceDecl::Create(Importer.getToContext(), DC,
+                                          Importer.Import(D->getAtStartLoc()),
+                                          Name.getAsIdentifierInfo(), Loc,
                                           D->isForwardDecl(),
                                           D->isImplicitInterfaceDecl());
       ToIface->setForwardDecl(D->isForwardDecl());
@@ -3085,8 +3189,8 @@ Decl *ASTNodeImporter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
     }
     
     // Import protocols
-    llvm::SmallVector<ObjCProtocolDecl *, 4> Protocols;
-    llvm::SmallVector<SourceLocation, 4> ProtocolLocs;
+    SmallVector<ObjCProtocolDecl *, 4> Protocols;
+    SmallVector<SourceLocation, 4> ProtocolLocs;
     ObjCInterfaceDecl::protocol_loc_iterator 
       FromProtoLoc = D->protocol_loc_begin();
     
@@ -3175,9 +3279,10 @@ Decl *ASTNodeImporter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
       return 0;
     
     ToImpl = ObjCCategoryImplDecl::Create(Importer.getToContext(), DC,
-                                          Importer.Import(D->getLocation()),
                                           Importer.Import(D->getIdentifier()),
-                                          Category->getClassInterface());
+                                          Category->getClassInterface(),
+                                          Importer.Import(D->getLocation()),
+                                          Importer.Import(D->getAtStartLoc()));
     
     DeclContext *LexicalDC = DC;
     if (D->getDeclContext() != D->getLexicalDeclContext()) {
@@ -3219,8 +3324,9 @@ Decl *ASTNodeImporter::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
     // now.
     Impl = ObjCImplementationDecl::Create(Importer.getToContext(),
                                   Importer.ImportContext(D->getDeclContext()),
+                                          Iface, Super,
                                           Importer.Import(D->getLocation()),
-                                          Iface, Super);
+                                          Importer.Import(D->getAtStartLoc()));
     
     if (D->getDeclContext() != D->getLexicalDeclContext()) {
       DeclContext *LexicalDC
@@ -3279,11 +3385,11 @@ Decl *ASTNodeImporter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
     return 0;
 
   // Check whether we have already imported this property.
-  for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-       Lookup.first != Lookup.second; 
-       ++Lookup.first) {
+  llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+  DC->localUncachedLookup(Name, FoundDecls);
+  for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
     if (ObjCPropertyDecl *FoundProp
-                                = dyn_cast<ObjCPropertyDecl>(*Lookup.first)) {
+                                = dyn_cast<ObjCPropertyDecl>(FoundDecls[I])) {
       // Check property types.
       if (!Importer.IsStructurallyEquivalent(D->getType(), 
                                              FoundProp->getType())) {
@@ -3430,8 +3536,8 @@ ASTNodeImporter::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D) {
   // Import the location of this declaration.
   SourceLocation Loc = Importer.Import(D->getLocation());
   
-  llvm::SmallVector<ObjCProtocolDecl *, 4> Protocols;
-  llvm::SmallVector<SourceLocation, 4> Locations;
+  SmallVector<ObjCProtocolDecl *, 4> Protocols;
+  SmallVector<SourceLocation, 4> Locations;
   ObjCForwardProtocolDecl::protocol_loc_iterator FromProtoLoc
     = D->protocol_loc_begin();
   for (ObjCForwardProtocolDecl::protocol_iterator FromProto
@@ -3472,25 +3578,14 @@ Decl *ASTNodeImporter::VisitObjCClassDecl(ObjCClassDecl *D) {
   
   // Import the location of this declaration.
   SourceLocation Loc = Importer.Import(D->getLocation());
-
-  llvm::SmallVector<ObjCInterfaceDecl *, 4> Interfaces;
-  llvm::SmallVector<SourceLocation, 4> Locations;
-  for (ObjCClassDecl::iterator From = D->begin(), FromEnd = D->end();
-       From != FromEnd; ++From) {
-    ObjCInterfaceDecl *ToIface
-      = cast_or_null<ObjCInterfaceDecl>(Importer.Import(From->getInterface()));
-    if (!ToIface)
-      continue;
-    
-    Interfaces.push_back(ToIface);
-    Locations.push_back(Importer.Import(From->getLocation()));
-  }
-  
+  ObjCClassDecl::ObjCClassRef *From = D->getForwardDecl();
+  ObjCInterfaceDecl *ToIface
+    = cast_or_null<ObjCInterfaceDecl>(Importer.Import(From->getInterface()));
   ObjCClassDecl *ToClass = ObjCClassDecl::Create(Importer.getToContext(), DC,
-                                                 Loc, 
-                                                 Interfaces.data(),
-                                                 Locations.data(),
-                                                 Interfaces.size());
+                                        Loc,
+                                        ToIface,
+                                        Importer.Import(From->getLocation()));
+    
   ToClass->setLexicalDeclContext(LexicalDC);
   LexicalDC->addDecl(ToClass);
   Importer.Imported(D, ToClass);
@@ -3594,14 +3689,14 @@ Decl *ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   
   // We may already have a template of the same name; try to find and match it.
   if (!DC->isFunctionOrMethod()) {
-    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
-    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
-         Lookup.first != Lookup.second; 
-         ++Lookup.first) {
-      if (!(*Lookup.first)->isInIdentifierNamespace(Decl::IDNS_Ordinary))
+    SmallVector<NamedDecl *, 4> ConflictingDecls;
+    llvm::SmallVector<NamedDecl *, 2> FoundDecls;
+    DC->localUncachedLookup(Name, FoundDecls);
+    for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
+      if (!FoundDecls[I]->isInIdentifierNamespace(Decl::IDNS_Ordinary))
         continue;
       
-      Decl *Found = *Lookup.first;
+      Decl *Found = FoundDecls[I];
       if (ClassTemplateDecl *FoundTemplate 
                                         = dyn_cast<ClassTemplateDecl>(Found)) {
         if (IsStructuralMatch(D, FoundTemplate)) {
@@ -3614,7 +3709,7 @@ Decl *ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
         }         
       }
       
-      ConflictingDecls.push_back(*Lookup.first);
+      ConflictingDecls.push_back(FoundDecls[I]);
     }
     
     if (!ConflictingDecls.empty()) {
@@ -3660,7 +3755,8 @@ Decl *ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   Importer.Imported(D, D2);
   Importer.Imported(DTemplated, D2Templated);
 
-  if (DTemplated->isDefinition() && !D2Templated->isDefinition()) {
+  if (DTemplated->isCompleteDefinition() &&
+      !D2Templated->isCompleteDefinition()) {
     // FIXME: Import definition!
   }
   
@@ -3704,7 +3800,7 @@ Decl *ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   SourceLocation IdLoc = Importer.Import(D->getLocation());
 
   // Import template arguments.
-  llvm::SmallVector<TemplateArgument, 2> TemplateArgs;
+  SmallVector<TemplateArgument, 2> TemplateArgs;
   if (ImportTemplateArguments(D->getTemplateArgs().data(), 
                               D->getTemplateArgs().size(),
                               TemplateArgs))
@@ -3722,7 +3818,7 @@ Decl *ASTNodeImporter::VisitClassTemplateSpecializationDecl(
     // FIXME: Check for specialization vs. instantiation errors.
     
     if (RecordDecl *FoundDef = D2->getDefinition()) {
-      if (!D->isDefinition() || IsStructuralMatch(D, FoundDef)) {
+      if (!D->isCompleteDefinition() || IsStructuralMatch(D, FoundDef)) {
         // The record types structurally match, or the "from" translation
         // unit only had a forward declaration anyway; call it the same
         // function.
@@ -3752,7 +3848,7 @@ Decl *ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   }
   Importer.Imported(D, D2);
   
-  if (D->isDefinition() && ImportDefinition(D, D2))
+  if (D->isCompleteDefinition() && ImportDefinition(D, D2))
     return 0;
   
   return D2;
@@ -3792,14 +3888,17 @@ Expr *ASTNodeImporter::VisitDeclRefExpr(DeclRefExpr *E) {
   QualType T = Importer.Import(E->getType());
   if (T.isNull())
     return 0;
-  
-  return DeclRefExpr::Create(Importer.getToContext(), 
-                             Importer.Import(E->getQualifierLoc()),
-                             ToD,
-                             Importer.Import(E->getLocation()),
-                             T, E->getValueKind(),
-                             FoundD,
-                             /*FIXME:TemplateArgs=*/0);
+
+  DeclRefExpr *DRE = DeclRefExpr::Create(Importer.getToContext(), 
+                                         Importer.Import(E->getQualifierLoc()),
+                                         ToD,
+                                         Importer.Import(E->getLocation()),
+                                         T, E->getValueKind(),
+                                         FoundD,
+                                         /*FIXME:TemplateArgs=*/0);
+  if (E->hadMultipleCandidates())
+    DRE->setHadMultipleCandidates(true);
+  return DRE;
 }
 
 Expr *ASTNodeImporter::VisitIntegerLiteral(IntegerLiteral *E) {
@@ -3817,8 +3916,8 @@ Expr *ASTNodeImporter::VisitCharacterLiteral(CharacterLiteral *E) {
   if (T.isNull())
     return 0;
   
-  return new (Importer.getToContext()) CharacterLiteral(E->getValue(), 
-                                                        E->isWide(), T,
+  return new (Importer.getToContext()) CharacterLiteral(E->getValue(),
+                                                        E->getKind(), T,
                                           Importer.Import(E->getLocation()));
 }
 
@@ -4024,13 +4123,17 @@ Decl *ASTImporter::Import(Decl *FromD) {
   if (!FromD)
     return 0;
 
+  ASTNodeImporter Importer(*this);
+
   // Check whether we've already imported this declaration.  
   llvm::DenseMap<Decl *, Decl *>::iterator Pos = ImportedDecls.find(FromD);
-  if (Pos != ImportedDecls.end())
-    return Pos->second;
+  if (Pos != ImportedDecls.end()) {
+    Decl *ToD = Pos->second;
+    Importer.ImportDefinitionIfNeeded(FromD, ToD);
+    return ToD;
+  }
   
   // Import the type
-  ASTNodeImporter Importer(*this);
   Decl *ToD = Importer.Visit(FromD);
   if (!ToD)
     return 0;
@@ -4045,7 +4148,7 @@ Decl *ASTImporter::Import(Decl *FromD) {
   } else if (TypedefNameDecl *FromTypedef = dyn_cast<TypedefNameDecl>(FromD)) {
     // When we've finished transforming a typedef, see whether it was the
     // typedef for an anonymous tag.
-    for (llvm::SmallVector<TagDecl *, 4>::iterator
+    for (SmallVector<TagDecl *, 4>::iterator
                FromTag = AnonTagsWithPendingTypedefs.begin(), 
             FromTagEnd = AnonTagsWithPendingTypedefs.end();
          FromTag != FromTagEnd; ++FromTag) {
@@ -4252,7 +4355,7 @@ SourceLocation ASTImporter::Import(SourceLocation FromLoc) {
   std::pair<FileID, unsigned> Decomposed = FromSM.getDecomposedLoc(FromLoc);
   SourceManager &ToSM = ToContext.getSourceManager();
   return ToSM.getLocForStartOfFile(Import(Decomposed.first))
-             .getFileLocWithOffset(Decomposed.second);
+             .getLocWithOffset(Decomposed.second);
 }
 
 SourceRange ASTImporter::Import(SourceRange FromRange) {
@@ -4306,6 +4409,23 @@ void ASTImporter::ImportDefinition(Decl *From) {
   
   if (DeclContext *FromDC = cast<DeclContext>(From)) {
     ASTNodeImporter Importer(*this);
+      
+    if (RecordDecl *ToRecord = dyn_cast<RecordDecl>(To)) {
+      if (!ToRecord->getDefinition()) {
+        Importer.ImportDefinition(cast<RecordDecl>(FromDC), ToRecord, 
+                                  /*ForceImport=*/true);
+        return;
+      }      
+    }
+
+    if (EnumDecl *ToEnum = dyn_cast<EnumDecl>(To)) {
+      if (!ToEnum->getDefinition()) {
+        Importer.ImportDefinition(cast<EnumDecl>(FromDC), ToEnum, 
+                                  /*ForceImport=*/true);
+        return;
+      }      
+    }
+
     Importer.ImportDeclContext(FromDC, true);
   }
 }
@@ -4378,7 +4498,7 @@ Selector ASTImporter::Import(Selector FromSel) {
   if (FromSel.isNull())
     return Selector();
 
-  llvm::SmallVector<IdentifierInfo *, 4> Idents;
+  SmallVector<IdentifierInfo *, 4> Idents;
   Idents.push_back(Import(FromSel.getIdentifierInfoForSlot(0)));
   for (unsigned I = 1, N = FromSel.getNumArgs(); I < N; ++I)
     Idents.push_back(Import(FromSel.getIdentifierInfoForSlot(I)));
diff --git a/contrib/llvm/tools/clang/lib/AST/CXXInheritance.cpp b/contrib/llvm/tools/clang/lib/AST/CXXInheritance.cpp
index 9ffe1f865689..f29bfd13192d 100644
--- a/contrib/llvm/tools/clang/lib/AST/CXXInheritance.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/CXXInheritance.cpp
@@ -119,7 +119,7 @@ bool CXXRecordDecl::isProvablyNotDerivedFrom(const CXXRecordDecl *Base) const {
 bool CXXRecordDecl::forallBases(ForallBasesCallback *BaseMatches,
                                 void *OpaqueData,
                                 bool AllowShortCircuit) const {
-  llvm::SmallVector<const CXXRecordDecl*, 8> Queue;
+  SmallVector<const CXXRecordDecl*, 8> Queue;
 
   const CXXRecordDecl *Record = this;
   bool AllMatches = true;
@@ -425,7 +425,7 @@ FindNestedNameSpecifierMember(const CXXBaseSpecifier *Specifier,
 
 void OverridingMethods::add(unsigned OverriddenSubobject, 
                             UniqueVirtualMethod Overriding) {
-  llvm::SmallVector<UniqueVirtualMethod, 4> &SubobjectOverrides
+  SmallVector<UniqueVirtualMethod, 4> &SubobjectOverrides
     = Overrides[OverriddenSubobject];
   if (std::find(SubobjectOverrides.begin(), SubobjectOverrides.end(), 
                 Overriding) == SubobjectOverrides.end())
@@ -556,7 +556,7 @@ void FinalOverriderCollector::Collect(const CXXRecordDecl *RD,
     // overrides.
     typedef std::pair<CXXMethodDecl::method_iterator, 
                       CXXMethodDecl::method_iterator> OverriddenMethods;
-    llvm::SmallVector<OverriddenMethods, 4> Stack;
+    SmallVector<OverriddenMethods, 4> Stack;
     Stack.push_back(std::make_pair(CanonM->begin_overridden_methods(),
                                    CanonM->end_overridden_methods()));
     while (!Stack.empty()) {
@@ -623,11 +623,11 @@ CXXRecordDecl::getFinalOverriders(CXXFinalOverriderMap &FinalOverriders) const {
                                   SOEnd = OM->second.end();
          SO != SOEnd; 
          ++SO) {
-      llvm::SmallVector<UniqueVirtualMethod, 4> &Overriding = SO->second;
+      SmallVector<UniqueVirtualMethod, 4> &Overriding = SO->second;
       if (Overriding.size() < 2)
         continue;
 
-      for (llvm::SmallVector<UniqueVirtualMethod, 4>::iterator 
+      for (SmallVector<UniqueVirtualMethod, 4>::iterator 
              Pos = Overriding.begin(), PosEnd = Overriding.end();
            Pos != PosEnd;
            /* increment in loop */) {
@@ -642,7 +642,7 @@ CXXRecordDecl::getFinalOverriders(CXXFinalOverriderMap &FinalOverriders) const {
         // in a base class subobject that hides the virtual base class
         // subobject.
         bool Hidden = false;
-        for (llvm::SmallVector<UniqueVirtualMethod, 4>::iterator
+        for (SmallVector<UniqueVirtualMethod, 4>::iterator
                OP = Overriding.begin(), OPEnd = Overriding.end();
              OP != OPEnd && !Hidden; 
              ++OP) {
diff --git a/contrib/llvm/tools/clang/lib/AST/Decl.cpp b/contrib/llvm/tools/clang/lib/AST/Decl.cpp
index 4c323da7eee3..95d52cb0fa1b 100644
--- a/contrib/llvm/tools/clang/lib/AST/Decl.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/Decl.cpp
@@ -28,6 +28,8 @@
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 
+#include <algorithm>
+
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -51,7 +53,7 @@ static llvm::Optional<Visibility> getVisibilityOf(const Decl *D) {
 
   // If we're on Mac OS X, an 'availability' for Mac OS X attribute
   // implies visibility(default).
-  if (D->getASTContext().Target.getTriple().isOSDarwin()) {
+  if (D->getASTContext().getTargetInfo().getTriple().isOSDarwin()) {
     for (specific_attr_iterator<AvailabilityAttr> 
               A = D->specific_attr_begin<AvailabilityAttr>(),
            AEnd = D->specific_attr_end<AvailabilityAttr>();
@@ -818,7 +820,7 @@ std::string NamedDecl::getQualifiedNameAsString(const PrintingPolicy &P) const {
   if (Ctx->isFunctionOrMethod())
     return getNameAsString();
 
-  typedef llvm::SmallVector<const DeclContext *, 8> ContextsTy;
+  typedef SmallVector<const DeclContext *, 8> ContextsTy;
   ContextsTy Contexts;
 
   // Collect contexts.
@@ -845,18 +847,18 @@ std::string NamedDecl::getQualifiedNameAsString(const PrintingPolicy &P) const {
       if (ND->isAnonymousNamespace())
         OS << "<anonymous namespace>";
       else
-        OS << ND;
+        OS << *ND;
     } else if (const RecordDecl *RD = dyn_cast<RecordDecl>(*I)) {
       if (!RD->getIdentifier())
         OS << "<anonymous " << RD->getKindName() << '>';
       else
-        OS << RD;
+        OS << *RD;
     } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
       const FunctionProtoType *FT = 0;
       if (FD->hasWrittenPrototype())
         FT = dyn_cast<FunctionProtoType>(FD->getType()->getAs<FunctionType>());
 
-      OS << FD << '(';
+      OS << *FD << '(';
       if (FT) {
         unsigned NumParams = FD->getNumParams();
         for (unsigned i = 0; i < NumParams; ++i) {
@@ -875,13 +877,13 @@ std::string NamedDecl::getQualifiedNameAsString(const PrintingPolicy &P) const {
       }
       OS << ')';
     } else {
-      OS << cast<NamedDecl>(*I);
+      OS << *cast<NamedDecl>(*I);
     }
     OS << "::";
   }
 
   if (getDeclName())
-    OS << this;
+    OS << *this;
   else
     OS << "<anonymous>";
 
@@ -1003,8 +1005,7 @@ void DeclaratorDecl::setQualifierInfo(NestedNameSpecifierLoc QualifierLoc) {
     }
     // Set qualifier info.
     getExtInfo()->QualifierLoc = QualifierLoc;
-  }
-  else {
+  } else {
     // Here Qualifier == 0, i.e., we are removing the qualifier (if any).
     if (hasExtInfo()) {
       if (getExtInfo()->NumTemplParamLists == 0) {
@@ -1120,15 +1121,16 @@ QualifierInfo::setTemplateParameterListsInfo(ASTContext &Context,
 
 const char *VarDecl::getStorageClassSpecifierString(StorageClass SC) {
   switch (SC) {
-  case SC_None:          break;
-  case SC_Auto:          return "auto"; break;
-  case SC_Extern:        return "extern"; break;
-  case SC_PrivateExtern: return "__private_extern__"; break;
-  case SC_Register:      return "register"; break;
-  case SC_Static:        return "static"; break;
+  case SC_None:                 break;
+  case SC_Auto:                 return "auto";
+  case SC_Extern:               return "extern";
+  case SC_OpenCLWorkGroupLocal: return "<<work-group-local>>";
+  case SC_PrivateExtern:        return "__private_extern__";
+  case SC_Register:             return "register";
+  case SC_Static:               return "static";
   }
 
-  assert(0 && "Invalid storage class");
+  llvm_unreachable("Invalid storage class");
   return 0;
 }
 
@@ -1388,6 +1390,16 @@ ParmVarDecl *ParmVarDecl::Create(ASTContext &C, DeclContext *DC,
                              S, SCAsWritten, DefArg);
 }
 
+SourceRange ParmVarDecl::getSourceRange() const {
+  if (!hasInheritedDefaultArg()) {
+    SourceRange ArgRange = getDefaultArgRange();
+    if (ArgRange.isValid())
+      return SourceRange(getOuterLocStart(), ArgRange.getEnd());
+  }
+
+  return DeclaratorDecl::getSourceRange();
+}
+
 Expr *ParmVarDecl::getDefaultArg() {
   assert(!hasUnparsedDefaultArg() && "Default argument is not yet parsed!");
   assert(!hasUninstantiatedDefaultArg() &&
@@ -1429,6 +1441,15 @@ bool ParmVarDecl::isParameterPack() const {
   return isa<PackExpansionType>(getType());
 }
 
+void ParmVarDecl::setParameterIndexLarge(unsigned parameterIndex) {
+  getASTContext().setParameterIndex(this, parameterIndex);
+  ParmVarDeclBits.ParameterIndex = ParameterIndexSentinel;
+}
+
+unsigned ParmVarDecl::getParameterIndexLarge() const {
+  return getASTContext().getParameterIndex(this);
+}
+
 //===----------------------------------------------------------------------===//
 // FunctionDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -1678,20 +1699,14 @@ unsigned FunctionDecl::getNumParams() const {
 }
 
 void FunctionDecl::setParams(ASTContext &C,
-                             ParmVarDecl **NewParamInfo, unsigned NumParams) {
+                             llvm::ArrayRef<ParmVarDecl *> NewParamInfo) {
   assert(ParamInfo == 0 && "Already has param info!");
-  assert(NumParams == getNumParams() && "Parameter count mismatch!");
+  assert(NewParamInfo.size() == getNumParams() && "Parameter count mismatch!");
 
   // Zero params -> null pointer.
-  if (NumParams) {
-    void *Mem = C.Allocate(sizeof(ParmVarDecl*)*NumParams);
-    ParamInfo = new (Mem) ParmVarDecl*[NumParams];
-    memcpy(ParamInfo, NewParamInfo, sizeof(ParmVarDecl*)*NumParams);
-
-    // Update source range. The check below allows us to set EndRangeLoc before
-    // setting the parameters.
-    if (EndRangeLoc.isInvalid() || EndRangeLoc == getLocation())
-      EndRangeLoc = NewParamInfo[NumParams-1]->getLocEnd();
+  if (!NewParamInfo.empty()) {
+    ParamInfo = new (C) ParmVarDecl*[NewParamInfo.size()];
+    std::copy(NewParamInfo.begin(), NewParamInfo.end(), ParamInfo);
   }
 }
 
@@ -1762,6 +1777,33 @@ bool FunctionDecl::isInlined() const {
   return false;
 }
 
+/// \brief For a function declaration in C or C++, determine whether this
+/// declaration causes the definition to be externally visible.
+///
+/// Determines whether this is the first non-inline redeclaration of an inline
+/// function in a language where "inline" does not normally require an
+/// externally visible definition.
+bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const {
+  assert(!doesThisDeclarationHaveABody() &&
+         "Must have a declaration without a body.");
+
+  ASTContext &Context = getASTContext();
+
+  // In C99 mode, a function may have an inline definition (causing it to
+  // be deferred) then redeclared later.  As a special case, "extern inline"
+  // is not required to produce an external symbol.
+  if (Context.getLangOptions().GNUInline || !Context.getLangOptions().C99 ||
+      Context.getLangOptions().CPlusPlus)
+    return false;
+  if (getLinkage() != ExternalLinkage || isInlineSpecified())
+    return false;
+  const FunctionDecl *Definition = 0;
+  if (hasBody(Definition))
+    return Definition->isInlined() &&
+           Definition->isInlineDefinitionExternallyVisible();
+  return false;
+}
+
 /// \brief For an inline function definition in C or C++, determine whether the 
 /// definition will be externally visible.
 ///
@@ -1814,7 +1856,12 @@ bool FunctionDecl::isInlineDefinitionExternallyVisible() const {
     // Only consider file-scope declarations in this test.
     if (!Redecl->getLexicalDeclContext()->isTranslationUnit())
       continue;
-    
+
+    // Only consider explicit declarations; the presence of a builtin for a
+    // libcall shouldn't affect whether a definition is externally visible.
+    if (Redecl->isImplicit())
+      continue;
+
     if (!Redecl->isInlineSpecified() || Redecl->getStorageClass() == SC_Extern) 
       return true; // Not an inline definition
   }
@@ -1857,8 +1904,7 @@ FunctionDecl::TemplatedKind FunctionDecl::getTemplatedKind() const {
                                <DependentFunctionTemplateSpecializationInfo*>())
     return TK_DependentFunctionTemplateSpecialization;
 
-  assert(false && "Did we miss a TemplateOrSpecialization type?");
-  return TK_NonTemplate;
+  llvm_unreachable("Did we miss a TemplateOrSpecialization type?");
 }
 
 FunctionDecl *FunctionDecl::getInstantiatedFromMemberFunction() const {
@@ -1890,13 +1936,17 @@ bool FunctionDecl::isImplicitlyInstantiable() const {
   
   switch (getTemplateSpecializationKind()) {
   case TSK_Undeclared:
-  case TSK_ExplicitSpecialization:
   case TSK_ExplicitInstantiationDefinition:
     return false;
       
   case TSK_ImplicitInstantiation:
     return true;
 
+  // It is possible to instantiate TSK_ExplicitSpecialization kind
+  // if the FunctionDecl has a class scope specialization pattern.
+  case TSK_ExplicitSpecialization:
+    return getClassScopeSpecializationPattern() != 0;
+
   case TSK_ExplicitInstantiationDeclaration:
     // Handled below.
     break;
@@ -1919,6 +1969,10 @@ bool FunctionDecl::isImplicitlyInstantiable() const {
 }                      
    
 FunctionDecl *FunctionDecl::getTemplateInstantiationPattern() const {
+  // Handle class scope explicit specialization special case.
+  if (getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
+    return getClassScopeSpecializationPattern();
+
   if (FunctionTemplateDecl *Primary = getPrimaryTemplate()) {
     while (Primary->getInstantiatedFromMemberTemplate()) {
       // If we have hit a point where the user provided a specialization of
@@ -1944,6 +1998,10 @@ FunctionTemplateDecl *FunctionDecl::getPrimaryTemplate() const {
   return 0;
 }
 
+FunctionDecl *FunctionDecl::getClassScopeSpecializationPattern() const {
+    return getASTContext().getClassScopeSpecializationPattern(this);
+}
+
 const TemplateArgumentList *
 FunctionDecl::getTemplateSpecializationArgs() const {
   if (FunctionTemplateSpecializationInfo *Info
@@ -1954,7 +2012,7 @@ FunctionDecl::getTemplateSpecializationArgs() const {
   return 0;
 }
 
-const TemplateArgumentListInfo *
+const ASTTemplateArgumentListInfo *
 FunctionDecl::getTemplateSpecializationArgsAsWritten() const {
   if (FunctionTemplateSpecializationInfo *Info
         = TemplateOrSpecialization
@@ -2069,7 +2127,7 @@ FunctionDecl::setTemplateSpecializationKind(TemplateSpecializationKind TSK,
         MSInfo->getPointOfInstantiation().isInvalid())
       MSInfo->setPointOfInstantiation(PointOfInstantiation);
   } else
-    assert(false && "Function cannot have a template specialization kind");
+    llvm_unreachable("Function cannot have a template specialization kind");
 }
 
 SourceLocation FunctionDecl::getPointOfInstantiation() const {
@@ -2134,6 +2192,12 @@ bool FieldDecl::isAnonymousStructOrUnion() const {
   return false;
 }
 
+unsigned FieldDecl::getBitWidthValue(const ASTContext &Ctx) const {
+  assert(isBitField() && "not a bitfield");
+  Expr *BitWidth = InitializerOrBitWidth.getPointer();
+  return BitWidth->EvaluateKnownConstInt(Ctx).getZExtValue();
+}
+
 unsigned FieldDecl::getFieldIndex() const {
   if (CachedFieldIndex) return CachedFieldIndex - 1;
 
@@ -2165,8 +2229,8 @@ unsigned FieldDecl::getFieldIndex() const {
 }
 
 SourceRange FieldDecl::getSourceRange() const {
-  if (isBitField())
-    return SourceRange(getInnerLocStart(), getBitWidth()->getLocEnd());
+  if (const Expr *E = InitializerOrBitWidth.getPointer())
+    return SourceRange(getInnerLocStart(), E->getLocEnd());
   return DeclaratorDecl::getSourceRange();
 }
 
@@ -2218,22 +2282,22 @@ void TagDecl::completeDefinition() {
           cast<CXXRecordDecl>(this)->hasDefinition()) &&
          "definition completed but not started");
 
-  IsDefinition = true;
+  IsCompleteDefinition = true;
   IsBeingDefined = false;
 
   if (ASTMutationListener *L = getASTMutationListener())
     L->CompletedTagDefinition(this);
 }
 
-TagDecl* TagDecl::getDefinition() const {
-  if (isDefinition())
+TagDecl *TagDecl::getDefinition() const {
+  if (isCompleteDefinition())
     return const_cast<TagDecl *>(this);
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(this))
     return CXXRD->getDefinition();
 
   for (redecl_iterator R = redecls_begin(), REnd = redecls_end();
        R != REnd; ++R)
-    if (R->isDefinition())
+    if (R->isCompleteDefinition())
       return *R;
 
   return 0;
@@ -2246,8 +2310,7 @@ void TagDecl::setQualifierInfo(NestedNameSpecifierLoc QualifierLoc) {
       TypedefNameDeclOrQualifier = new (getASTContext()) ExtInfo;
     // Set qualifier info.
     getExtInfo()->QualifierLoc = QualifierLoc;
-  }
-  else {
+  } else {
     // Here Qualifier == 0, i.e., we are removing the qualifier (if any).
     if (hasExtInfo()) {
       if (getExtInfo()->NumTemplParamLists == 0) {
@@ -2296,7 +2359,7 @@ void EnumDecl::completeDefinition(QualType NewType,
                                   QualType NewPromotionType,
                                   unsigned NumPositiveBits,
                                   unsigned NumNegativeBits) {
-  assert(!isDefinition() && "Cannot redefine enums!");
+  assert(!isCompleteDefinition() && "Cannot redefine enums!");
   if (!IntegerType)
     IntegerType = NewType.getTypePtr();
   PromotionType = NewPromotionType;
@@ -2349,7 +2412,7 @@ RecordDecl::field_iterator RecordDecl::field_begin() const {
 /// completeDefinition - Notes that the definition of this type is now
 /// complete.
 void RecordDecl::completeDefinition() {
-  assert(!isDefinition() && "Cannot redefine record!");
+  assert(!isCompleteDefinition() && "Cannot redefine record!");
   TagDecl::completeDefinition();
 }
 
@@ -2360,7 +2423,7 @@ void RecordDecl::LoadFieldsFromExternalStorage() const {
   // Notify that we have a RecordDecl doing some initialization.
   ExternalASTSource::Deserializing TheFields(Source);
 
-  llvm::SmallVector<Decl*, 64> Decls;
+  SmallVector<Decl*, 64> Decls;
   LoadedFieldsFromExternalStorage = true;  
   switch (Source->FindExternalLexicalDeclsBy<FieldDecl>(this, Decls)) {
   case ELR_Success:
@@ -2380,23 +2443,22 @@ void RecordDecl::LoadFieldsFromExternalStorage() const {
   if (Decls.empty())
     return;
 
-  llvm::tie(FirstDecl, LastDecl) = BuildDeclChain(Decls);
+  llvm::tie(FirstDecl, LastDecl) = BuildDeclChain(Decls,
+                                                 /*FieldsAlreadyLoaded=*/false);
 }
 
 //===----------------------------------------------------------------------===//
 // BlockDecl Implementation
 //===----------------------------------------------------------------------===//
 
-void BlockDecl::setParams(ParmVarDecl **NewParamInfo,
-                          unsigned NParms) {
+void BlockDecl::setParams(llvm::ArrayRef<ParmVarDecl *> NewParamInfo) {
   assert(ParamInfo == 0 && "Already has param info!");
 
   // Zero params -> null pointer.
-  if (NParms) {
-    NumParams = NParms;
-    void *Mem = getASTContext().Allocate(sizeof(ParmVarDecl*)*NumParams);
-    ParamInfo = new (Mem) ParmVarDecl*[NumParams];
-    memcpy(ParamInfo, NewParamInfo, sizeof(ParmVarDecl*)*NumParams);
+  if (!NewParamInfo.empty()) {
+    NumParams = NewParamInfo.size();
+    ParamInfo = new (getASTContext()) ParmVarDecl*[NewParamInfo.size()];
+    std::copy(NewParamInfo.begin(), NewParamInfo.end(), ParamInfo);
   }
 }
 
@@ -2481,10 +2543,12 @@ FunctionDecl *FunctionDecl::Create(ASTContext &C, DeclContext *DC,
                                    QualType T, TypeSourceInfo *TInfo,
                                    StorageClass SC, StorageClass SCAsWritten,
                                    bool isInlineSpecified, 
-                                   bool hasWrittenPrototype) {
+                                   bool hasWrittenPrototype,
+                                   bool isConstexprSpecified) {
   FunctionDecl *New = new (C) FunctionDecl(Function, DC, StartLoc, NameInfo,
                                            T, TInfo, SC, SCAsWritten,
-                                           isInlineSpecified);
+                                           isInlineSpecified,
+                                           isConstexprSpecified);
   New->HasWrittenPrototype = hasWrittenPrototype;
   return New;
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp b/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp
index b2806f092cbd..321e40b43899 100644
--- a/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp
@@ -43,7 +43,7 @@ static bool StatSwitch = false;
 
 const char *Decl::getDeclKindName() const {
   switch (DeclKind) {
-  default: assert(0 && "Declaration not in DeclNodes.inc!");
+  default: llvm_unreachable("Declaration not in DeclNodes.inc!");
 #define DECL(DERIVED, BASE) case DERIVED: return #DERIVED;
 #define ABSTRACT_DECL(DECL)
 #include "clang/AST/DeclNodes.inc"
@@ -62,7 +62,7 @@ void Decl::setInvalidDecl(bool Invalid) {
 
 const char *DeclContext::getDeclKindName() const {
   switch (DeclKind) {
-  default: assert(0 && "Declaration context not in DeclNodes.inc!");
+  default: llvm_unreachable("Declaration context not in DeclNodes.inc!");
 #define DECL(DERIVED, BASE) case Decl::DERIVED: return #DERIVED;
 #define ABSTRACT_DECL(DECL)
 #include "clang/AST/DeclNodes.inc"
@@ -100,7 +100,7 @@ void Decl::PrintStats() {
 
 void Decl::add(Kind k) {
   switch (k) {
-  default: assert(0 && "Declaration not in DeclNodes.inc!");
+  default: llvm_unreachable("Declaration not in DeclNodes.inc!");
 #define DECL(DERIVED, BASE) case DERIVED: ++n##DERIVED##s; break;
 #define ABSTRACT_DECL(DECL)
 #include "clang/AST/DeclNodes.inc"
@@ -133,14 +133,18 @@ bool Decl::isFunctionOrFunctionTemplate() const {
   return isa<FunctionDecl>(this) || isa<FunctionTemplateDecl>(this);
 }
 
-bool Decl::isDefinedOutsideFunctionOrMethod() const {
-  for (const DeclContext *DC = getDeclContext(); 
-       DC && !DC->isTranslationUnit(); 
+bool Decl::isTemplateDecl() const {
+  return isa<TemplateDecl>(this);
+}
+
+const DeclContext *Decl::getParentFunctionOrMethod() const {
+  for (const DeclContext *DC = getDeclContext();
+       DC && !DC->isTranslationUnit() && !DC->isNamespace(); 
        DC = DC->getParent())
     if (DC->isFunctionOrMethod())
-      return false;
+      return DC;
 
-  return true;
+  return 0;
 }
 
 
@@ -148,7 +152,7 @@ bool Decl::isDefinedOutsideFunctionOrMethod() const {
 // PrettyStackTraceDecl Implementation
 //===----------------------------------------------------------------------===//
 
-void PrettyStackTraceDecl::print(llvm::raw_ostream &OS) const {
+void PrettyStackTraceDecl::print(raw_ostream &OS) const {
   SourceLocation TheLoc = Loc;
   if (TheLoc.isInvalid() && TheDecl)
     TheLoc = TheDecl->getLocation();
@@ -265,13 +269,13 @@ bool Decl::isReferenced() const {
 static AvailabilityResult CheckAvailability(ASTContext &Context,
                                             const AvailabilityAttr *A,
                                             std::string *Message) {
-  llvm::StringRef TargetPlatform = Context.Target.getPlatformName();
-  llvm::StringRef PrettyPlatformName
+  StringRef TargetPlatform = Context.getTargetInfo().getPlatformName();
+  StringRef PrettyPlatformName
     = AvailabilityAttr::getPrettyPlatformName(TargetPlatform);
   if (PrettyPlatformName.empty())
     PrettyPlatformName = TargetPlatform;
 
-  VersionTuple TargetMinVersion = Context.Target.getPlatformMinVersion();
+  VersionTuple TargetMinVersion = Context.getTargetInfo().getPlatformMinVersion();
   if (TargetMinVersion.empty())
     return AR_Available;
 
@@ -493,6 +497,7 @@ unsigned Decl::getIdentifierNamespaceForKind(Kind DeclKind) {
     case UsingDirective:
     case ClassTemplateSpecialization:
     case ClassTemplatePartialSpecialization:
+    case ClassScopeFunctionSpecialization:
     case ObjCImplementation:
     case ObjCCategory:
     case ObjCCategoryImpl:
@@ -566,8 +571,7 @@ Decl *Decl::castFromDeclContext (const DeclContext *D) {
       if (DK >= first##NAME && DK <= last##NAME) \
         return static_cast<NAME##Decl*>(const_cast<DeclContext*>(D));
 #include "clang/AST/DeclNodes.inc"
-      assert(false && "a decl that inherits DeclContext isn't handled");
-      return 0;
+      llvm_unreachable("a decl that inherits DeclContext isn't handled");
   }
 }
 
@@ -586,8 +590,7 @@ DeclContext *Decl::castToDeclContext(const Decl *D) {
       if (DK >= first##NAME && DK <= last##NAME)                  \
         return static_cast<NAME##Decl*>(const_cast<Decl*>(D));
 #include "clang/AST/DeclNodes.inc"
-      assert(false && "a decl that inherits DeclContext isn't handled");
-      return 0;
+      llvm_unreachable("a decl that inherits DeclContext isn't handled");
   }
 }
 
@@ -627,7 +630,8 @@ void Decl::CheckAccessDeclContext() const {
       isa<ParmVarDecl>(this) ||
       // FIXME: a ClassTemplateSpecialization or CXXRecordDecl can have
       // AS_none as access specifier.
-      isa<CXXRecordDecl>(this))
+      isa<CXXRecordDecl>(this) ||
+      isa<ClassScopeFunctionSpecializationDecl>(this))
     return;
 
   assert(Access != AS_none &&
@@ -812,11 +816,15 @@ DeclContext *DeclContext::getNextContext() {
 }
 
 std::pair<Decl *, Decl *>
-DeclContext::BuildDeclChain(const llvm::SmallVectorImpl<Decl*> &Decls) {
+DeclContext::BuildDeclChain(const SmallVectorImpl<Decl*> &Decls,
+                            bool FieldsAlreadyLoaded) {
   // Build up a chain of declarations via the Decl::NextDeclInContext field.
   Decl *FirstNewDecl = 0;
   Decl *PrevDecl = 0;
   for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
+    if (FieldsAlreadyLoaded && isa<FieldDecl>(Decls[I]))
+      continue;
+
     Decl *D = Decls[I];
     if (PrevDecl)
       PrevDecl->NextDeclInContext = D;
@@ -838,9 +846,9 @@ DeclContext::LoadLexicalDeclsFromExternalStorage() const {
 
   // Notify that we have a DeclContext that is initializing.
   ExternalASTSource::Deserializing ADeclContext(Source);
-
+  
   // Load the external declarations, if any.
-  llvm::SmallVector<Decl*, 64> Decls;
+  SmallVector<Decl*, 64> Decls;
   ExternalLexicalStorage = false;
   switch (Source->FindExternalLexicalDecls(this, Decls)) {
   case ELR_Success:
@@ -855,17 +863,16 @@ DeclContext::LoadLexicalDeclsFromExternalStorage() const {
     return;
 
   // We may have already loaded just the fields of this record, in which case
-  // don't add the decls, just replace the FirstDecl/LastDecl chain.
+  // we need to ignore them.
+  bool FieldsAlreadyLoaded = false;
   if (const RecordDecl *RD = dyn_cast<RecordDecl>(this))
-    if (RD->LoadedFieldsFromExternalStorage) {
-      llvm::tie(FirstDecl, LastDecl) = BuildDeclChain(Decls);
-      return;
-    }
-
+    FieldsAlreadyLoaded = RD->LoadedFieldsFromExternalStorage;
+  
   // Splice the newly-read declarations into the beginning of the list
   // of declarations.
   Decl *ExternalFirst, *ExternalLast;
-  llvm::tie(ExternalFirst, ExternalLast) = BuildDeclChain(Decls);
+  llvm::tie(ExternalFirst, ExternalLast) = BuildDeclChain(Decls,
+                                                          FieldsAlreadyLoaded);
   ExternalLast->NextDeclInContext = FirstDecl;
   FirstDecl = ExternalFirst;
   if (!LastDecl)
@@ -890,7 +897,7 @@ ExternalASTSource::SetNoExternalVisibleDeclsForName(const DeclContext *DC,
 DeclContext::lookup_result
 ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
                                                   DeclarationName Name,
-                                    llvm::SmallVectorImpl<NamedDecl*> &Decls) {
+                                                  ArrayRef<NamedDecl*> Decls) {
   ASTContext &Context = DC->getParentASTContext();;
 
   StoredDeclsMap *Map;
@@ -898,35 +905,17 @@ ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
     Map = DC->CreateStoredDeclsMap(Context);
 
   StoredDeclsList &List = (*Map)[Name];
-  for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
+  for (ArrayRef<NamedDecl*>::iterator
+         I = Decls.begin(), E = Decls.end(); I != E; ++I) {
     if (List.isNull())
-      List.setOnlyValue(Decls[I]);
+      List.setOnlyValue(*I);
     else
-      List.AddSubsequentDecl(Decls[I]);
+      List.AddSubsequentDecl(*I);
   }
 
   return List.getLookupResult();
 }
 
-void ExternalASTSource::MaterializeVisibleDeclsForName(const DeclContext *DC,
-                                                       DeclarationName Name,
-                                     llvm::SmallVectorImpl<NamedDecl*> &Decls) {
-  assert(DC->LookupPtr);
-  StoredDeclsMap &Map = *DC->LookupPtr;
-
-  // If there's an entry in the table the visible decls for this name have
-  // already been deserialized.
-  if (Map.find(Name) == Map.end()) {
-    StoredDeclsList &List = Map[Name];
-    for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
-      if (List.isNull())
-        List.setOnlyValue(Decls[I]);
-      else
-        List.AddSubsequentDecl(Decls[I]);
-    }
-  }
-}
-
 DeclContext::decl_iterator DeclContext::noload_decls_begin() const {
   return decl_iterator(FirstDecl);
 }
@@ -939,8 +928,6 @@ DeclContext::decl_iterator DeclContext::decls_begin() const {
   if (hasExternalLexicalStorage())
     LoadLexicalDeclsFromExternalStorage();
 
-  // FIXME: Check whether we need to load some declarations from
-  // external storage.
   return decl_iterator(FirstDecl);
 }
 
@@ -988,6 +975,9 @@ void DeclContext::removeDecl(Decl *D) {
   if (isa<NamedDecl>(D)) {
     NamedDecl *ND = cast<NamedDecl>(D);
 
+    // Remove only decls that have a name
+    if (!ND->getDeclName()) return;
+
     StoredDeclsMap *Map = getPrimaryContext()->LookupPtr;
     if (!Map) return;
 
@@ -1038,12 +1028,10 @@ void DeclContext::buildLookup(DeclContext *DCtx) {
         if (D->getDeclContext() == DCtx)
           makeDeclVisibleInContextImpl(ND);
 
-      // Insert any forward-declared Objective-C interfaces into the lookup
+      // Insert any forward-declared Objective-C interface into the lookup
       // data structure.
       if (ObjCClassDecl *Class = dyn_cast<ObjCClassDecl>(*D))
-        for (ObjCClassDecl::iterator I = Class->begin(), IEnd = Class->end();
-             I != IEnd; ++I)
-          makeDeclVisibleInContextImpl(I->getInterface());
+        makeDeclVisibleInContextImpl(Class->getForwardInterfaceDecl());
       
       // If this declaration is itself a transparent declaration context or
       // inline namespace, add its members (recursively).
@@ -1093,6 +1081,38 @@ DeclContext::lookup(DeclarationName Name) const {
   return const_cast<DeclContext*>(this)->lookup(Name);
 }
 
+void DeclContext::localUncachedLookup(DeclarationName Name, 
+                                  llvm::SmallVectorImpl<NamedDecl *> &Results) {
+  Results.clear();
+  
+  // If there's no external storage, just perform a normal lookup and copy
+  // the results.
+  if (!hasExternalVisibleStorage() && !hasExternalLexicalStorage()) {
+    lookup_result LookupResults = lookup(Name);
+    Results.insert(Results.end(), LookupResults.first, LookupResults.second);
+    return;
+  }
+
+  // If we have a lookup table, check there first. Maybe we'll get lucky.
+  if (LookupPtr) {
+    StoredDeclsMap::iterator Pos = LookupPtr->find(Name);
+    if (Pos != LookupPtr->end()) {
+      Results.insert(Results.end(), 
+                     Pos->second.getLookupResult().first,
+                     Pos->second.getLookupResult().second);
+      return;
+    }
+  }
+  
+  // Slow case: grovel through the declarations in our chain looking for 
+  // matches.
+  for (Decl *D = FirstDecl; D; D = D->getNextDeclInContext()) {
+    if (NamedDecl *ND = dyn_cast<NamedDecl>(D))
+      if (ND->getDeclName() == Name)
+        Results.push_back(ND);
+  }
+}
+
 DeclContext *DeclContext::getRedeclContext() {
   DeclContext *Ctx = this;
   // Skip through transparent contexts.
@@ -1205,15 +1225,6 @@ void DeclContext::makeDeclVisibleInContextImpl(NamedDecl *D) {
   DeclNameEntries.AddSubsequentDecl(D);
 }
 
-void DeclContext::MaterializeVisibleDeclsFromExternalStorage() {
-  ExternalASTSource *Source = getParentASTContext().getExternalSource();
-  assert(hasExternalVisibleStorage() && Source && "No external storage?");
-
-  if (!LookupPtr)
-    CreateStoredDeclsMap(getParentASTContext());
-  Source->MaterializeVisibleDecls(this);
-}
-
 /// Returns iterator range [First, Last) of UsingDirectiveDecls stored within
 /// this context.
 DeclContext::udir_iterator_range
diff --git a/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp b/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp
index 4b59bf37d74b..f3da67c4ff81 100644
--- a/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp
@@ -35,15 +35,16 @@ CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
     Abstract(false), IsStandardLayout(true), HasNoNonEmptyBases(true),
     HasPrivateFields(false), HasProtectedFields(false), HasPublicFields(false),
     HasMutableFields(false), HasTrivialDefaultConstructor(true),
-    HasConstExprNonCopyMoveConstructor(false), HasTrivialCopyConstructor(true),
+    HasConstexprNonCopyMoveConstructor(false), HasTrivialCopyConstructor(true),
     HasTrivialMoveConstructor(true), HasTrivialCopyAssignment(true),
     HasTrivialMoveAssignment(true), HasTrivialDestructor(true),
     HasNonLiteralTypeFieldsOrBases(false), ComputedVisibleConversions(false),
     UserProvidedDefaultConstructor(false), DeclaredDefaultConstructor(false),
     DeclaredCopyConstructor(false), DeclaredMoveConstructor(false),
     DeclaredCopyAssignment(false), DeclaredMoveAssignment(false),
-    DeclaredDestructor(false), NumBases(0), NumVBases(0), Bases(), VBases(),
-    Definition(D), FirstFriend(0) {
+    DeclaredDestructor(false), FailedImplicitMoveConstructor(false),
+    FailedImplicitMoveAssignment(false), NumBases(0), NumVBases(0), Bases(),
+    VBases(), Definition(D), FirstFriend(0) {
 }
 
 CXXRecordDecl::CXXRecordDecl(Kind K, TagKind TK, DeclContext *DC,
@@ -89,7 +90,7 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
   llvm::SmallPtrSet<CanQualType, 8> SeenVBaseTypes;
   
   // The virtual bases of this class.
-  llvm::SmallVector<const CXXBaseSpecifier *, 8> VBases;
+  SmallVector<const CXXBaseSpecifier *, 8> VBases;
 
   data().Bases = new(C) CXXBaseSpecifier [NumBases];
   data().NumBases = NumBases;
@@ -287,7 +288,7 @@ bool CXXRecordDecl::isTriviallyCopyable() const {
 /// (if there is one).
 static CXXMethodDecl *
 GetBestOverloadCandidateSimple(
-  const llvm::SmallVectorImpl<std::pair<CXXMethodDecl *, Qualifiers> > &Cands) {
+  const SmallVectorImpl<std::pair<CXXMethodDecl *, Qualifiers> > &Cands) {
   if (Cands.empty())
     return 0;
   if (Cands.size() == 1)
@@ -313,7 +314,7 @@ CXXConstructorDecl *CXXRecordDecl::getCopyConstructor(unsigned TypeQuals) const{
     = Context.DeclarationNames.getCXXConstructorName(
                                           Context.getCanonicalType(ClassType));
   unsigned FoundTQs;
-  llvm::SmallVector<std::pair<CXXMethodDecl *, Qualifiers>, 4> Found;
+  SmallVector<std::pair<CXXMethodDecl *, Qualifiers>, 4> Found;
   DeclContext::lookup_const_iterator Con, ConEnd;
   for (llvm::tie(Con, ConEnd) = this->lookup(ConstructorName);
        Con != ConEnd; ++Con) {
@@ -349,7 +350,7 @@ CXXMethodDecl *CXXRecordDecl::getCopyAssignmentOperator(bool ArgIsConst) const {
   QualType Class = Context.getTypeDeclType(const_cast<CXXRecordDecl *>(this));
   DeclarationName Name = Context.DeclarationNames.getCXXOperatorName(OO_Equal);
   
-  llvm::SmallVector<std::pair<CXXMethodDecl *, Qualifiers>, 4> Found;
+  SmallVector<std::pair<CXXMethodDecl *, Qualifiers>, 4> Found;
   DeclContext::lookup_const_iterator Op, OpEnd;
   for (llvm::tie(Op, OpEnd) = this->lookup(Name); Op != OpEnd; ++Op) {
     // C++ [class.copy]p9:
@@ -502,18 +503,18 @@ NotASpecialMember:;
     // Note that we have a user-declared constructor.
     data().UserDeclaredConstructor = true;
 
-    // FIXME: Under C++0x, /only/ special member functions may be user-provided.
-    //        This is probably a defect.
-    bool UserProvided = false;
+    // Technically, "user-provided" is only defined for special member
+    // functions, but the intent of the standard is clearly that it should apply
+    // to all functions.
+    bool UserProvided = Constructor->isUserProvided();
 
     // C++0x [class.ctor]p5:
     //   A default constructor is trivial if it is not user-provided [...]
     if (Constructor->isDefaultConstructor()) {
       data().DeclaredDefaultConstructor = true;
-      if (Constructor->isUserProvided()) {
+      if (UserProvided) {
         data().HasTrivialDefaultConstructor = false;
         data().UserProvidedDefaultConstructor = true;
-        UserProvided = true;
       }
     }
 
@@ -527,10 +528,8 @@ NotASpecialMember:;
         // C++0x [class.copy]p13:
         //   A copy/move constructor for class X is trivial if it is not
         //   user-provided [...]
-        if (Constructor->isUserProvided()) {
+        if (UserProvided)
           data().HasTrivialCopyConstructor = false;
-          UserProvided = true;
-        }
       } else if (Constructor->isMoveConstructor()) {
         data().UserDeclaredMoveConstructor = true;
         data().DeclaredMoveConstructor = true;
@@ -538,17 +537,14 @@ NotASpecialMember:;
         // C++0x [class.copy]p13:
         //   A copy/move constructor for class X is trivial if it is not
         //   user-provided [...]
-        if (Constructor->isUserProvided()) {
+        if (UserProvided)
           data().HasTrivialMoveConstructor = false;
-          UserProvided = true;
-        }
       }
     }
-    if (Constructor->isConstExpr() &&
-        !Constructor->isCopyOrMoveConstructor()) {
-      // Record if we see any constexpr constructors which are niether copy
+    if (Constructor->isConstexpr() && !Constructor->isCopyOrMoveConstructor()) {
+      // Record if we see any constexpr constructors which are neither copy
       // nor move constructors.
-      data().HasConstExprNonCopyMoveConstructor = true;
+      data().HasConstexprNonCopyMoveConstructor = true;
     }
 
     // C++ [dcl.init.aggr]p1:
@@ -657,6 +653,13 @@ NotASpecialMember:;
   
   // Handle non-static data members.
   if (FieldDecl *Field = dyn_cast<FieldDecl>(D)) {
+    // C++ [class.bit]p2:
+    //   A declaration for a bit-field that omits the identifier declares an 
+    //   unnamed bit-field. Unnamed bit-fields are not members and cannot be 
+    //   initialized.
+    if (Field->isUnnamedBitfield())
+      return;
+    
     // C++ [dcl.init.aggr]p1:
     //   An aggregate is an array or a class (clause 9) with [...] no
     //   private or protected non-static data members (clause 11).
@@ -675,7 +678,7 @@ NotASpecialMember:;
     case AS_private:    data().HasPrivateFields = true;   break;
     case AS_protected:  data().HasProtectedFields = true; break;
     case AS_public:     data().HasPublicFields = true;    break;
-    case AS_none:       assert(0 && "Invalid access specifier");
+    case AS_none:       llvm_unreachable("Invalid access specifier");
     };
     if ((data().HasPrivateFields + data().HasProtectedFields +
          data().HasPublicFields) > 1)
@@ -716,7 +719,11 @@ NotASpecialMember:;
     }
 
     // Record if this field is the first non-literal field or base.
-    if (!hasNonLiteralTypeFieldsOrBases() && !T->isLiteralType())
+    // As a slight variation on the standard, we regard mutable members as being
+    // non-literal, since mutating a constexpr variable would break C++11
+    // constant expression semantics.
+    if ((!hasNonLiteralTypeFieldsOrBases() && !T->isLiteralType()) ||
+        Field->isMutable())
       data().HasNonLiteralTypeFieldsOrBases = true;
 
     if (Field->hasInClassInitializer()) {
@@ -827,15 +834,11 @@ NotASpecialMember:;
 
     // If this is not a zero-length bit-field, then the class is not empty.
     if (data().Empty) {
-      if (!Field->getBitWidth())
+      if (!Field->isBitField() ||
+          (!Field->getBitWidth()->isTypeDependent() &&
+           !Field->getBitWidth()->isValueDependent() &&
+           Field->getBitWidthValue(Context) != 0))
         data().Empty = false;
-      else if (!Field->getBitWidth()->isTypeDependent() &&
-               !Field->getBitWidth()->isValueDependent()) {
-        llvm::APSInt Bits;
-        if (Field->getBitWidth()->isIntegerConstantExpr(Bits, Context))
-          if (!!Bits)
-            data().Empty = false;
-      } 
     }
   }
   
@@ -1057,7 +1060,7 @@ CXXRecordDecl::setTemplateSpecializationKind(TemplateSpecializationKind TSK) {
     return;
   }
   
-  assert(false && "Not a class template or member class specialization");
+  llvm_unreachable("Not a class template or member class specialization");
 }
 
 CXXDestructorDecl *CXXRecordDecl::getDestructor() const {
@@ -1160,9 +1163,10 @@ CXXMethodDecl::Create(ASTContext &C, CXXRecordDecl *RD,
                       const DeclarationNameInfo &NameInfo,
                       QualType T, TypeSourceInfo *TInfo,
                       bool isStatic, StorageClass SCAsWritten, bool isInline,
-                      SourceLocation EndLocation) {
+                      bool isConstexpr, SourceLocation EndLocation) {
   return new (C) CXXMethodDecl(CXXMethod, RD, StartLoc, NameInfo, T, TInfo,
-                               isStatic, SCAsWritten, isInline, EndLocation);
+                               isStatic, SCAsWritten, isInline, isConstexpr,
+                               EndLocation);
 }
 
 bool CXXMethodDecl::isUsualDeallocationFunction() const {
@@ -1402,7 +1406,7 @@ SourceRange CXXCtorInitializer::getSourceRange() const {
 CXXConstructorDecl *
 CXXConstructorDecl::Create(ASTContext &C, EmptyShell Empty) {
   return new (C) CXXConstructorDecl(0, SourceLocation(), DeclarationNameInfo(),
-                                    QualType(), 0, false, false, false);
+                                    QualType(), 0, false, false, false, false);
 }
 
 CXXConstructorDecl *
@@ -1410,14 +1414,14 @@ CXXConstructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
                            SourceLocation StartLoc,
                            const DeclarationNameInfo &NameInfo,
                            QualType T, TypeSourceInfo *TInfo,
-                           bool isExplicit,
-                           bool isInline,
-                           bool isImplicitlyDeclared) {
+                           bool isExplicit, bool isInline,
+                           bool isImplicitlyDeclared, bool isConstexpr) {
   assert(NameInfo.getName().getNameKind()
          == DeclarationName::CXXConstructorName &&
          "Name must refer to a constructor");
   return new (C) CXXConstructorDecl(RD, StartLoc, NameInfo, T, TInfo,
-                                    isExplicit, isInline, isImplicitlyDeclared);
+                                    isExplicit, isInline, isImplicitlyDeclared,
+                                    isConstexpr);
 }
 
 bool CXXConstructorDecl::isDefaultConstructor() const {
@@ -1545,8 +1549,7 @@ CXXDestructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
                           SourceLocation StartLoc,
                           const DeclarationNameInfo &NameInfo,
                           QualType T, TypeSourceInfo *TInfo,
-                          bool isInline,
-                          bool isImplicitlyDeclared) {
+                          bool isInline, bool isImplicitlyDeclared) {
   assert(NameInfo.getName().getNameKind()
          == DeclarationName::CXXDestructorName &&
          "Name must refer to a destructor");
@@ -1557,7 +1560,7 @@ CXXDestructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
 CXXConversionDecl *
 CXXConversionDecl::Create(ASTContext &C, EmptyShell Empty) {
   return new (C) CXXConversionDecl(0, SourceLocation(), DeclarationNameInfo(),
-                                   QualType(), 0, false, false,
+                                   QualType(), 0, false, false, false,
                                    SourceLocation());
 }
 
@@ -1567,12 +1570,13 @@ CXXConversionDecl::Create(ASTContext &C, CXXRecordDecl *RD,
                           const DeclarationNameInfo &NameInfo,
                           QualType T, TypeSourceInfo *TInfo,
                           bool isInline, bool isExplicit,
-                          SourceLocation EndLocation) {
+                          bool isConstexpr, SourceLocation EndLocation) {
   assert(NameInfo.getName().getNameKind()
          == DeclarationName::CXXConversionFunctionName &&
          "Name must refer to a conversion function");
   return new (C) CXXConversionDecl(RD, StartLoc, NameInfo, T, TInfo,
-                                   isInline, isExplicit, EndLocation);
+                                   isInline, isExplicit, isConstexpr,
+                                   EndLocation);
 }
 
 LinkageSpecDecl *LinkageSpecDecl::Create(ASTContext &C,
@@ -1696,8 +1700,7 @@ static const char *getAccessName(AccessSpecifier AS) {
   switch (AS) {
     default:
     case AS_none:
-      assert("Invalid access specifier!");
-      return 0;
+      llvm_unreachable("Invalid access specifier!");
     case AS_public:
       return "public";
     case AS_private:
diff --git a/contrib/llvm/tools/clang/lib/AST/DeclObjC.cpp b/contrib/llvm/tools/clang/lib/AST/DeclObjC.cpp
index 557b681d2fa6..a589b7f9d3e9 100644
--- a/contrib/llvm/tools/clang/lib/AST/DeclObjC.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DeclObjC.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Stmt.h"
+#include "clang/AST/ASTMutationListener.h"
 #include "llvm/ADT/STLExtras.h"
 using namespace clang;
 
@@ -185,7 +186,7 @@ void ObjCInterfaceDecl::mergeClassExtensionProtocolList(
   // Check for duplicate protocol in class's protocol list.
   // This is O(n*m). But it is extremely rare and number of protocols in
   // class or its extension are very few.
-  llvm::SmallVector<ObjCProtocolDecl*, 8> ProtocolRefs;
+  SmallVector<ObjCProtocolDecl*, 8> ProtocolRefs;
   for (unsigned i = 0; i < ExtNum; i++) {
     bool protocolExists = false;
     ObjCProtocolDecl *ProtoInExtension = ExtList[i];
@@ -337,17 +338,60 @@ ObjCMethodDecl *ObjCMethodDecl::Create(ASTContext &C,
                                        bool isInstance,
                                        bool isVariadic,
                                        bool isSynthesized,
+                                       bool isImplicitlyDeclared,
                                        bool isDefined,
                                        ImplementationControl impControl,
-                                       bool HasRelatedResultType,
-                                       unsigned numSelectorArgs) {
+                                       bool HasRelatedResultType) {
   return new (C) ObjCMethodDecl(beginLoc, endLoc,
                                 SelInfo, T, ResultTInfo, contextDecl,
                                 isInstance,
-                                isVariadic, isSynthesized, isDefined,
+                                isVariadic, isSynthesized, isImplicitlyDeclared,
+                                isDefined,
                                 impControl,
-                                HasRelatedResultType,
-                                numSelectorArgs);
+                                HasRelatedResultType);
+}
+
+void ObjCMethodDecl::setAsRedeclaration(const ObjCMethodDecl *PrevMethod) {
+  assert(PrevMethod);
+  getASTContext().setObjCMethodRedeclaration(PrevMethod, this);
+  IsRedeclaration = true;
+  PrevMethod->HasRedeclaration = true;
+}
+
+void ObjCMethodDecl::setParamsAndSelLocs(ASTContext &C,
+                                         ArrayRef<ParmVarDecl*> Params,
+                                         ArrayRef<SourceLocation> SelLocs) {
+  ParamsAndSelLocs = 0;
+  NumParams = Params.size();
+  if (Params.empty() && SelLocs.empty())
+    return;
+
+  unsigned Size = sizeof(ParmVarDecl *) * NumParams +
+                  sizeof(SourceLocation) * SelLocs.size();
+  ParamsAndSelLocs = C.Allocate(Size);
+  std::copy(Params.begin(), Params.end(), getParams());
+  std::copy(SelLocs.begin(), SelLocs.end(), getStoredSelLocs());
+}
+
+void ObjCMethodDecl::getSelectorLocs(
+                               SmallVectorImpl<SourceLocation> &SelLocs) const {
+  for (unsigned i = 0, e = getNumSelectorLocs(); i != e; ++i)
+    SelLocs.push_back(getSelectorLoc(i));
+}
+
+void ObjCMethodDecl::setMethodParams(ASTContext &C,
+                                     ArrayRef<ParmVarDecl*> Params,
+                                     ArrayRef<SourceLocation> SelLocs) {
+  assert((!SelLocs.empty() || isImplicit()) &&
+         "No selector locs for non-implicit method");
+  if (isImplicit())
+    return setParamsAndSelLocs(C, Params, ArrayRef<SourceLocation>());
+
+  SelLocsKind = hasStandardSelectorLocs(getSelector(), SelLocs, Params, EndLoc);
+  if (SelLocsKind != SelLoc_NonStandard)
+    return setParamsAndSelLocs(C, Params, ArrayRef<SourceLocation>());
+
+  setParamsAndSelLocs(C, Params, SelLocs);
 }
 
 /// \brief A definition will return its interface declaration.
@@ -356,6 +400,11 @@ ObjCMethodDecl *ObjCMethodDecl::Create(ASTContext &C,
 ObjCMethodDecl *ObjCMethodDecl::getNextRedeclaration() {
   ASTContext &Ctx = getASTContext();
   ObjCMethodDecl *Redecl = 0;
+  if (HasRedeclaration)
+    Redecl = const_cast<ObjCMethodDecl*>(Ctx.getObjCMethodRedeclaration(this));
+  if (Redecl)
+    return Redecl;
+
   Decl *CtxD = cast<Decl>(getDeclContext());
 
   if (ObjCInterfaceDecl *IFD = dyn_cast<ObjCInterfaceDecl>(CtxD)) {
@@ -377,6 +426,12 @@ ObjCMethodDecl *ObjCMethodDecl::getNextRedeclaration() {
       Redecl = CatD->getMethod(getSelector(), isInstanceMethod());
   }
 
+  if (!Redecl && isRedeclaration()) {
+    // This is the last redeclaration, go back to the first method.
+    return cast<ObjCContainerDecl>(CtxD)->getMethod(getSelector(),
+                                                    isInstanceMethod());
+  }
+
   return Redecl ? Redecl : this;
 }
 
@@ -444,6 +499,7 @@ ObjCMethodFamily ObjCMethodDecl::getMethodFamily() const {
 
   // These selectors have a conventional meaning only for instance methods.
   case OMF_dealloc:
+  case OMF_finalize:
   case OMF_retain:
   case OMF_release:
   case OMF_autorelease:
@@ -545,8 +601,7 @@ ObjCInterfaceDecl *ObjCMethodDecl::getClassInterface() {
     return IMD->getClassInterface();
 
   assert(!isa<ObjCProtocolDecl>(getDeclContext()) && "It's a protocol method");
-  assert(false && "unknown method context");
-  return 0;
+  llvm_unreachable("unknown method context");
 }
 
 //===----------------------------------------------------------------------===//
@@ -566,11 +621,10 @@ ObjCInterfaceDecl *ObjCInterfaceDecl::Create(ASTContext &C,
 ObjCInterfaceDecl::
 ObjCInterfaceDecl(DeclContext *DC, SourceLocation atLoc, IdentifierInfo *Id,
                   SourceLocation CLoc, bool FD, bool isInternal)
-  : ObjCContainerDecl(ObjCInterface, DC, atLoc, Id),
+  : ObjCContainerDecl(ObjCInterface, DC, Id, CLoc, atLoc),
     TypeForDecl(0), SuperClass(0),
     CategoryList(0), IvarList(0), 
-    ForwardDecl(FD), InternalInterface(isInternal), ExternallyCompleted(false),
-    ClassLoc(CLoc) {
+    ForwardDecl(FD), InternalInterface(isInternal), ExternallyCompleted(false) {
 }
 
 void ObjCInterfaceDecl::LoadExternalDefinition() const {
@@ -756,8 +810,7 @@ ObjCIvarDecl *ObjCIvarDecl::Create(ASTContext &C, ObjCContainerDecl *DC,
         ID = IM->getClassInterface();
         if (BW)
           IM->setHasSynthBitfield(true);
-      }
-      else {
+      } else {
         ObjCCategoryDecl *CD = cast<ObjCCategoryDecl>(DC);
         ID = CD->getClassInterface();
         if (BW)
@@ -778,8 +831,7 @@ const ObjCInterfaceDecl *ObjCIvarDecl::getContainingInterface() const {
   default:
   case ObjCCategoryImpl:
   case ObjCProtocol:
-    assert(0 && "invalid ivar container!");
-    return 0;
+    llvm_unreachable("invalid ivar container!");
 
     // Ivars can only appear in class extension categories.
   case ObjCCategory: {
@@ -812,9 +864,10 @@ ObjCAtDefsFieldDecl
 //===----------------------------------------------------------------------===//
 
 ObjCProtocolDecl *ObjCProtocolDecl::Create(ASTContext &C, DeclContext *DC,
-                                           SourceLocation L,
-                                           IdentifierInfo *Id) {
-  return new (C) ObjCProtocolDecl(DC, L, Id);
+                                           IdentifierInfo *Id,
+                                           SourceLocation nameLoc,
+                                           SourceLocation atStartLoc) {
+  return new (C) ObjCProtocolDecl(DC, Id, nameLoc, atStartLoc);
 }
 
 ObjCProtocolDecl *ObjCProtocolDecl::lookupProtocolNamed(IdentifierInfo *Name) {
@@ -850,36 +903,31 @@ ObjCMethodDecl *ObjCProtocolDecl::lookupMethod(Selector Sel,
 //===----------------------------------------------------------------------===//
 
 ObjCClassDecl::ObjCClassDecl(DeclContext *DC, SourceLocation L,
-                             ObjCInterfaceDecl *const *Elts,
-                             const SourceLocation *Locs,
-                             unsigned nElts,
+                             ObjCInterfaceDecl *const Elt,
+                             const SourceLocation Loc,
                              ASTContext &C)
   : Decl(ObjCClass, DC, L) {
-  setClassList(C, Elts, Locs, nElts);
-}
-
-void ObjCClassDecl::setClassList(ASTContext &C, ObjCInterfaceDecl*const*List,
-                                 const SourceLocation *Locs, unsigned Num) {
-  ForwardDecls = (ObjCClassRef*) C.Allocate(sizeof(ObjCClassRef)*Num,
-                                            llvm::alignOf<ObjCClassRef>());
-  for (unsigned i = 0; i < Num; ++i)
-    new (&ForwardDecls[i]) ObjCClassRef(List[i], Locs[i]);
-  
-  NumDecls = Num;
+  setClass(C, Elt, Loc);
 }
 
 ObjCClassDecl *ObjCClassDecl::Create(ASTContext &C, DeclContext *DC,
                                      SourceLocation L,
-                                     ObjCInterfaceDecl *const *Elts,
-                                     const SourceLocation *Locs,
-                                     unsigned nElts) {
-  return new (C) ObjCClassDecl(DC, L, Elts, Locs, nElts, C);
+                                     ObjCInterfaceDecl *const Elt,
+                                     const SourceLocation Loc) {
+  return new (C) ObjCClassDecl(DC, L, Elt, Loc, C);
 }
 
+void ObjCClassDecl::setClass(ASTContext &C, ObjCInterfaceDecl*const Cls,
+                             const SourceLocation Loc) {
+    
+  ForwardDecl = (ObjCClassRef*) C.Allocate(sizeof(ObjCClassRef),
+                                           llvm::alignOf<ObjCClassRef>());
+  new (ForwardDecl) ObjCClassRef(Cls, Loc);
+}
+    
 SourceRange ObjCClassDecl::getSourceRange() const {
   // FIXME: We should include the semicolon
-  assert(NumDecls);
-  return SourceRange(getLocation(), ForwardDecls[NumDecls-1].getLocation());
+  return SourceRange(getLocation(), ForwardDecl->getLocation());
 }
 
 //===----------------------------------------------------------------------===//
@@ -912,8 +960,25 @@ ObjCCategoryDecl *ObjCCategoryDecl::Create(ASTContext &C, DeclContext *DC,
                                            SourceLocation AtLoc, 
                                            SourceLocation ClassNameLoc,
                                            SourceLocation CategoryNameLoc,
-                                           IdentifierInfo *Id) {
-  return new (C) ObjCCategoryDecl(DC, AtLoc, ClassNameLoc, CategoryNameLoc, Id);
+                                           IdentifierInfo *Id,
+                                           ObjCInterfaceDecl *IDecl) {
+  ObjCCategoryDecl *CatDecl = new (C) ObjCCategoryDecl(DC, AtLoc, ClassNameLoc,
+                                                       CategoryNameLoc, Id,
+                                                       IDecl);
+  if (IDecl) {
+    // Link this category into its class's category list.
+    CatDecl->NextClassCategory = IDecl->getCategoryList();
+    IDecl->setCategoryList(CatDecl);
+    if (ASTMutationListener *L = C.getASTMutationListener())
+      L->AddedObjCCategoryToInterface(CatDecl, IDecl);
+  }
+
+  return CatDecl;
+}
+
+ObjCCategoryDecl *ObjCCategoryDecl::Create(ASTContext &C, EmptyShell Empty) {
+  return new (C) ObjCCategoryDecl(0, SourceLocation(), SourceLocation(),
+                                  SourceLocation(), 0, 0);
 }
 
 ObjCCategoryImplDecl *ObjCCategoryDecl::getImplementation() const {
@@ -932,9 +997,12 @@ void ObjCCategoryDecl::setImplementation(ObjCCategoryImplDecl *ImplD) {
 
 ObjCCategoryImplDecl *
 ObjCCategoryImplDecl::Create(ASTContext &C, DeclContext *DC,
-                             SourceLocation L,IdentifierInfo *Id,
-                             ObjCInterfaceDecl *ClassInterface) {
-  return new (C) ObjCCategoryImplDecl(DC, L, Id, ClassInterface);
+                             IdentifierInfo *Id,
+                             ObjCInterfaceDecl *ClassInterface,
+                             SourceLocation nameLoc,
+                             SourceLocation atStartLoc) {
+  return new (C) ObjCCategoryImplDecl(DC, Id, ClassInterface,
+                                      nameLoc, atStartLoc);
 }
 
 ObjCCategoryDecl *ObjCCategoryImplDecl::getCategoryDecl() const {
@@ -997,7 +1065,7 @@ FindPropertyImplDecl(IdentifierInfo *Id) const {
   return 0;
 }
 
-llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
+raw_ostream &clang::operator<<(raw_ostream &OS,
                                      const ObjCCategoryImplDecl *CID) {
   OS << CID->getName();
   return OS;
@@ -1009,13 +1077,28 @@ llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
 
 ObjCImplementationDecl *
 ObjCImplementationDecl::Create(ASTContext &C, DeclContext *DC,
-                               SourceLocation L,
                                ObjCInterfaceDecl *ClassInterface,
-                               ObjCInterfaceDecl *SuperDecl) {
-  return new (C) ObjCImplementationDecl(DC, L, ClassInterface, SuperDecl);
+                               ObjCInterfaceDecl *SuperDecl,
+                               SourceLocation nameLoc,
+                               SourceLocation atStartLoc) {
+  return new (C) ObjCImplementationDecl(DC, ClassInterface, SuperDecl,
+                                        nameLoc, atStartLoc);
 }
 
-llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
+void ObjCImplementationDecl::setIvarInitializers(ASTContext &C,
+                                             CXXCtorInitializer ** initializers,
+                                                 unsigned numInitializers) {
+  if (numInitializers > 0) {
+    NumIvarInitializers = numInitializers;
+    CXXCtorInitializer **ivarInitializers =
+    new (C) CXXCtorInitializer*[NumIvarInitializers];
+    memcpy(ivarInitializers, initializers,
+           numInitializers * sizeof(CXXCtorInitializer*));
+    IvarInitializers = ivarInitializers;
+  }
+}
+
+raw_ostream &clang::operator<<(raw_ostream &OS,
                                      const ObjCImplementationDecl *ID) {
   OS << ID->getName();
   return OS;
diff --git a/contrib/llvm/tools/clang/lib/AST/DeclPrinter.cpp b/contrib/llvm/tools/clang/lib/AST/DeclPrinter.cpp
index 19554a3baaea..08a1ab5723cd 100644
--- a/contrib/llvm/tools/clang/lib/AST/DeclPrinter.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DeclPrinter.cpp
@@ -24,22 +24,25 @@ using namespace clang;
 
 namespace {
   class DeclPrinter : public DeclVisitor<DeclPrinter> {
-    llvm::raw_ostream &Out;
+    raw_ostream &Out;
     ASTContext &Context;
     PrintingPolicy Policy;
     unsigned Indentation;
+    bool PrintInstantiation;
 
-    llvm::raw_ostream& Indent() { return Indent(Indentation); }
-    llvm::raw_ostream& Indent(unsigned Indentation);
-    void ProcessDeclGroup(llvm::SmallVectorImpl<Decl*>& Decls);
+    raw_ostream& Indent() { return Indent(Indentation); }
+    raw_ostream& Indent(unsigned Indentation);
+    void ProcessDeclGroup(SmallVectorImpl<Decl*>& Decls);
 
     void Print(AccessSpecifier AS);
 
   public:
-    DeclPrinter(llvm::raw_ostream &Out, ASTContext &Context,
+    DeclPrinter(raw_ostream &Out, ASTContext &Context,
                 const PrintingPolicy &Policy,
-                unsigned Indentation = 0)
-      : Out(Out), Context(Context), Policy(Policy), Indentation(Indentation) { }
+                unsigned Indentation = 0,
+                bool PrintInstantiation = false)
+      : Out(Out), Context(Context), Policy(Policy), Indentation(Indentation),
+        PrintInstantiation(PrintInstantiation) { }
 
     void VisitDeclContext(DeclContext *DC, bool Indent = true);
 
@@ -62,6 +65,8 @@ namespace {
     void VisitCXXRecordDecl(CXXRecordDecl *D);
     void VisitLinkageSpecDecl(LinkageSpecDecl *D);
     void VisitTemplateDecl(const TemplateDecl *D);
+    void VisitFunctionTemplateDecl(FunctionTemplateDecl *D);
+    void VisitClassTemplateDecl(ClassTemplateDecl *D);
     void VisitObjCMethodDecl(ObjCMethodDecl *D);
     void VisitObjCClassDecl(ObjCClassDecl *D);
     void VisitObjCImplementationDecl(ObjCImplementationDecl *D);
@@ -77,16 +82,20 @@ namespace {
     void VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D);
     void VisitUsingDecl(UsingDecl *D);
     void VisitUsingShadowDecl(UsingShadowDecl *D);
+
+    void PrintTemplateParameters(const TemplateParameterList *Params,
+                                 const TemplateArgumentList *Args);
   };
 }
 
-void Decl::print(llvm::raw_ostream &Out, unsigned Indentation) const {
-  print(Out, getASTContext().PrintingPolicy, Indentation);
+void Decl::print(raw_ostream &Out, unsigned Indentation,
+                 bool PrintInstantiation) const {
+  print(Out, getASTContext().getPrintingPolicy(), Indentation, PrintInstantiation);
 }
 
-void Decl::print(llvm::raw_ostream &Out, const PrintingPolicy &Policy,
-                 unsigned Indentation) const {
-  DeclPrinter Printer(Out, getASTContext(), Policy, Indentation);
+void Decl::print(raw_ostream &Out, const PrintingPolicy &Policy,
+                 unsigned Indentation, bool PrintInstantiation) const {
+  DeclPrinter Printer(Out, getASTContext(), Policy, Indentation, PrintInstantiation);
   Printer.Visit(const_cast<Decl*>(this));
 }
 
@@ -105,7 +114,7 @@ static QualType GetBaseType(QualType T) {
     else if (const VectorType *VTy = BaseType->getAs<VectorType>())
       BaseType = VTy->getElementType();
     else
-      assert(0 && "Unknown declarator!");
+      llvm_unreachable("Unknown declarator!");
   }
   return BaseType;
 }
@@ -119,7 +128,7 @@ static QualType getDeclType(Decl* D) {
 }
 
 void Decl::printGroup(Decl** Begin, unsigned NumDecls,
-                      llvm::raw_ostream &Out, const PrintingPolicy &Policy,
+                      raw_ostream &Out, const PrintingPolicy &Policy,
                       unsigned Indentation) {
   if (NumDecls == 1) {
     (*Begin)->print(Out, Policy, Indentation);
@@ -132,7 +141,7 @@ void Decl::printGroup(Decl** Begin, unsigned NumDecls,
     ++Begin;
 
   PrintingPolicy SubPolicy(Policy);
-  if (TD && TD->isDefinition()) {
+  if (TD && TD->isCompleteDefinition()) {
     TD->print(Out, Policy, Indentation);
     Out << " ";
     SubPolicy.SuppressTag = true;
@@ -159,7 +168,7 @@ void DeclContext::dumpDeclContext() const {
     DC = DC->getParent();
   
   ASTContext &Ctx = cast<TranslationUnitDecl>(DC)->getASTContext();
-  DeclPrinter Printer(llvm::errs(), Ctx, Ctx.PrintingPolicy, 0);
+  DeclPrinter Printer(llvm::errs(), Ctx, Ctx.getPrintingPolicy(), 0);
   Printer.VisitDeclContext(const_cast<DeclContext *>(this), /*Indent=*/false);
 }
 
@@ -167,13 +176,13 @@ void Decl::dump() const {
   print(llvm::errs());
 }
 
-llvm::raw_ostream& DeclPrinter::Indent(unsigned Indentation) {
+raw_ostream& DeclPrinter::Indent(unsigned Indentation) {
   for (unsigned i = 0; i != Indentation; ++i)
     Out << "  ";
   return Out;
 }
 
-void DeclPrinter::ProcessDeclGroup(llvm::SmallVectorImpl<Decl*>& Decls) {
+void DeclPrinter::ProcessDeclGroup(SmallVectorImpl<Decl*>& Decls) {
   this->Indent();
   Decl::printGroup(Decls.data(), Decls.size(), Out, Policy, Indentation);
   Out << ";\n";
@@ -183,7 +192,7 @@ void DeclPrinter::ProcessDeclGroup(llvm::SmallVectorImpl<Decl*>& Decls) {
 
 void DeclPrinter::Print(AccessSpecifier AS) {
   switch(AS) {
-  case AS_none:      assert(0 && "No access specifier!"); break;
+  case AS_none:      llvm_unreachable("No access specifier!");
   case AS_public:    Out << "public"; break;
   case AS_protected: Out << "protected"; break;
   case AS_private:   Out << "private"; break;
@@ -198,7 +207,7 @@ void DeclPrinter::VisitDeclContext(DeclContext *DC, bool Indent) {
   if (Indent)
     Indentation += Policy.Indentation;
 
-  llvm::SmallVector<Decl*, 2> Decls;
+  SmallVector<Decl*, 2> Decls;
   for (DeclContext::decl_iterator D = DC->decls_begin(), DEnd = DC->decls_end();
        D != DEnd; ++D) {
 
@@ -304,8 +313,12 @@ void DeclPrinter::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
 void DeclPrinter::VisitTypedefDecl(TypedefDecl *D) {
   std::string S = D->getNameAsString();
   D->getUnderlyingType().getAsStringInternal(S, Policy);
-  if (!Policy.SuppressSpecifiers)
+  if (!Policy.SuppressSpecifiers) {
     Out << "typedef ";
+    
+    if (D->isModulePrivate())
+      Out << "__module_private__ ";
+  }
   Out << S;
 }
 
@@ -315,6 +328,8 @@ void DeclPrinter::VisitTypeAliasDecl(TypeAliasDecl *D) {
 }
 
 void DeclPrinter::VisitEnumDecl(EnumDecl *D) {
+  if (!Policy.SuppressSpecifiers && D->isModulePrivate())
+    Out << "__module_private__ ";
   Out << "enum ";
   if (D->isScoped()) {
     if (D->isScopedUsingClassTag())
@@ -322,7 +337,7 @@ void DeclPrinter::VisitEnumDecl(EnumDecl *D) {
     else
       Out << "struct ";
   }
-  Out << D;
+  Out << *D;
 
   if (D->isFixed()) {
     std::string Underlying;
@@ -330,7 +345,7 @@ void DeclPrinter::VisitEnumDecl(EnumDecl *D) {
     Out << " : " << Underlying;
   }
 
-  if (D->isDefinition()) {
+  if (D->isCompleteDefinition()) {
     Out << " {\n";
     VisitDeclContext(D);
     Indent() << "}";
@@ -338,11 +353,13 @@ void DeclPrinter::VisitEnumDecl(EnumDecl *D) {
 }
 
 void DeclPrinter::VisitRecordDecl(RecordDecl *D) {
+  if (!Policy.SuppressSpecifiers && D->isModulePrivate())
+    Out << "__module_private__ ";
   Out << D->getKindName();
   if (D->getIdentifier())
-    Out << ' ' << D;
+    Out << ' ' << *D;
 
-  if (D->isDefinition()) {
+  if (D->isCompleteDefinition()) {
     Out << " {\n";
     VisitDeclContext(D);
     Indent() << "}";
@@ -350,7 +367,7 @@ void DeclPrinter::VisitRecordDecl(RecordDecl *D) {
 }
 
 void DeclPrinter::VisitEnumConstantDecl(EnumConstantDecl *D) {
-  Out << D;
+  Out << *D;
   if (Expr *Init = D->getInitExpr()) {
     Out << " = ";
     Init->printPretty(Out, Context, 0, Policy, Indentation);
@@ -364,11 +381,13 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     case SC_Extern: Out << "extern "; break;
     case SC_Static: Out << "static "; break;
     case SC_PrivateExtern: Out << "__private_extern__ "; break;
-    case SC_Auto: case SC_Register: llvm_unreachable("invalid for functions");
+    case SC_Auto: case SC_Register: case SC_OpenCLWorkGroupLocal:
+      llvm_unreachable("invalid for functions");
     }
 
-    if (D->isInlineSpecified())           Out << "inline ";
+    if (D->isInlineSpecified())  Out << "inline ";
     if (D->isVirtualAsWritten()) Out << "virtual ";
+    if (D->isModulePrivate())    Out << "__module_private__ ";
   }
 
   PrintingPolicy SubPolicy(Policy);
@@ -449,6 +468,10 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 
     if (D->hasAttr<NoReturnAttr>())
       Proto += " __attribute((noreturn))";
+
+    if (D->hasAttr<ReturnsTwiceAttr>())
+      Proto += " __attribute((returns_twice))";
+
     if (CXXConstructorDecl *CDecl = dyn_cast<CXXConstructorDecl>(D)) {
       bool HasInitializerList = false;
       for (CXXConstructorDecl::init_const_iterator B = CDecl->init_begin(),
@@ -468,10 +491,9 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 
         if (BMInitializer->isAnyMemberInitializer()) {
           FieldDecl *FD = BMInitializer->getAnyMember();
-          Out << FD;
+          Out << *FD;
         } else {
-          Out << QualType(BMInitializer->getBaseClass(),
-                          0).getAsString(Policy);
+          Out << QualType(BMInitializer->getBaseClass(), 0).getAsString(Policy);
         }
         
         Out << "(";
@@ -549,6 +571,8 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
   if (!Policy.SuppressSpecifiers && D->isMutable())
     Out << "mutable ";
+  if (!Policy.SuppressSpecifiers && D->isModulePrivate())
+    Out << "__module_private__ ";
 
   std::string Name = D->getNameAsString();
   D->getType().getAsStringInternal(Name, Policy);
@@ -577,6 +601,8 @@ void DeclPrinter::VisitVarDecl(VarDecl *D) {
 
   if (!Policy.SuppressSpecifiers && D->isThreadSpecified())
     Out << "__thread ";
+  if (!Policy.SuppressSpecifiers && D->isModulePrivate())
+    Out << "__module_private__ ";
 
   std::string Name = D->getNameAsString();
   QualType T = D->getType();
@@ -621,7 +647,7 @@ void DeclPrinter::VisitStaticAssertDecl(StaticAssertDecl *D) {
 // C++ declarations
 //----------------------------------------------------------------------------
 void DeclPrinter::VisitNamespaceDecl(NamespaceDecl *D) {
-  Out << "namespace " << D << " {\n";
+  Out << "namespace " << *D << " {\n";
   VisitDeclContext(D);
   Indent() << "}";
 }
@@ -630,22 +656,24 @@ void DeclPrinter::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   Out << "using namespace ";
   if (D->getQualifier())
     D->getQualifier()->print(Out, Policy);
-  Out << D->getNominatedNamespaceAsWritten();
+  Out << *D->getNominatedNamespaceAsWritten();
 }
 
 void DeclPrinter::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
-  Out << "namespace " << D << " = ";
+  Out << "namespace " << *D << " = ";
   if (D->getQualifier())
     D->getQualifier()->print(Out, Policy);
-  Out << D->getAliasedNamespace();
+  Out << *D->getAliasedNamespace();
 }
 
 void DeclPrinter::VisitCXXRecordDecl(CXXRecordDecl *D) {
+  if (!Policy.SuppressSpecifiers && D->isModulePrivate())
+    Out << "__module_private__ ";
   Out << D->getKindName();
   if (D->getIdentifier())
-    Out << ' ' << D;
+    Out << ' ' << *D;
 
-  if (D->isDefinition()) {
+  if (D->isCompleteDefinition()) {
     // Print the base classes
     if (D->getNumBases()) {
       Out << " : ";
@@ -694,10 +722,13 @@ void DeclPrinter::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
     Visit(*D->decls_begin());
 }
 
-void DeclPrinter::VisitTemplateDecl(const TemplateDecl *D) {
+void DeclPrinter::PrintTemplateParameters(
+    const TemplateParameterList *Params, const TemplateArgumentList *Args = 0) {
+  assert(Params);
+  assert(!Args || Params->size() == Args->size());
+
   Out << "template <";
 
-  TemplateParameterList *Params = D->getTemplateParameters();
   for (unsigned i = 0, e = Params->size(); i != e; ++i) {
     if (i != 0)
       Out << ", ";
@@ -716,7 +747,10 @@ void DeclPrinter::VisitTemplateDecl(const TemplateDecl *D) {
 
       Out << TTP->getNameAsString();
 
-      if (TTP->hasDefaultArgument()) {
+      if (Args) {
+        Out << " = ";
+        Args->get(i).print(Policy, Out);
+      } else if (TTP->hasDefaultArgument()) {
         Out << " = ";
         Out << TTP->getDefaultArgument().getAsString(Policy);
       };
@@ -732,7 +766,10 @@ void DeclPrinter::VisitTemplateDecl(const TemplateDecl *D) {
         Out << Name->getName();
       }
 
-      if (NTTP->hasDefaultArgument()) {
+      if (Args) {
+        Out << " = ";
+        Args->get(i).print(Policy, Out);
+      } else if (NTTP->hasDefaultArgument()) {
         Out << " = ";
         NTTP->getDefaultArgument()->printPretty(Out, Context, 0, Policy,
                                                 Indentation);
@@ -745,6 +782,10 @@ void DeclPrinter::VisitTemplateDecl(const TemplateDecl *D) {
   }
 
   Out << "> ";
+}
+
+void DeclPrinter::VisitTemplateDecl(const TemplateDecl *D) {
+  PrintTemplateParameters(D->getTemplateParameters());
 
   if (const TemplateTemplateParmDecl *TTP =
         dyn_cast<TemplateTemplateParmDecl>(D)) {
@@ -757,17 +798,39 @@ void DeclPrinter::VisitTemplateDecl(const TemplateDecl *D) {
   }
 }
 
+void DeclPrinter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
+  if (PrintInstantiation) {
+    TemplateParameterList *Params = D->getTemplateParameters();
+    for (FunctionTemplateDecl::spec_iterator I = D->spec_begin(), E = D->spec_end();
+         I != E; ++I) {
+      PrintTemplateParameters(Params, (*I)->getTemplateSpecializationArgs());
+      Visit(*I);
+    }
+  }
+
+  return VisitRedeclarableTemplateDecl(D);
+}
+
+void DeclPrinter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
+  if (PrintInstantiation) {
+    TemplateParameterList *Params = D->getTemplateParameters();
+    for (ClassTemplateDecl::spec_iterator I = D->spec_begin(), E = D->spec_end();
+         I != E; ++I) {
+      PrintTemplateParameters(Params, &(*I)->getTemplateArgs());
+      Visit(*I);
+      Out << '\n';
+    }
+  }
+
+  return VisitRedeclarableTemplateDecl(D);
+}
+
 //----------------------------------------------------------------------------
 // Objective-C declarations
 //----------------------------------------------------------------------------
 
 void DeclPrinter::VisitObjCClassDecl(ObjCClassDecl *D) {
-  Out << "@class ";
-  for (ObjCClassDecl::iterator I = D->begin(), E = D->end();
-       I != E; ++I) {
-    if (I != D->begin()) Out << ", ";
-    Out << I->getInterface();
-  }
+  Out << "@class " << *D->getForwardInterfaceDecl();
 }
 
 void DeclPrinter::VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
@@ -783,9 +846,9 @@ void DeclPrinter::VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
   for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
        E = OMD->param_end(); PI != E; ++PI) {
     // FIXME: selector is missing here!
-    pos = name.find_first_of(":", lastPos);
+    pos = name.find_first_of(':', lastPos);
     Out << " " << name.substr(lastPos, pos - lastPos);
-    Out << ":(" << (*PI)->getType().getAsString(Policy) << ')' << *PI;
+    Out << ":(" << (*PI)->getType().getAsString(Policy) << ')' << **PI;
     lastPos = pos + 1;
   }
 
@@ -807,7 +870,7 @@ void DeclPrinter::VisitObjCImplementationDecl(ObjCImplementationDecl *OID) {
   ObjCInterfaceDecl *SID = OID->getSuperClass();
 
   if (SID)
-    Out << "@implementation " << I << " : " << SID;
+    Out << "@implementation " << I << " : " << *SID;
   else
     Out << "@implementation " << I;
   Out << "\n";
@@ -820,7 +883,7 @@ void DeclPrinter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *OID) {
   ObjCInterfaceDecl *SID = OID->getSuperClass();
 
   if (SID)
-    Out << "@interface " << I << " : " << SID;
+    Out << "@interface " << I << " : " << *SID;
   else
     Out << "@interface " << I;
 
@@ -829,7 +892,7 @@ void DeclPrinter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *OID) {
   if (!Protocols.empty()) {
     for (ObjCList<ObjCProtocolDecl>::iterator I = Protocols.begin(),
          E = Protocols.end(); I != E; ++I)
-      Out << (I == Protocols.begin() ? '<' : ',') << *I;
+      Out << (I == Protocols.begin() ? '<' : ',') << **I;
   }
 
   if (!Protocols.empty())
@@ -840,7 +903,7 @@ void DeclPrinter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *OID) {
     Indentation += Policy.Indentation;
     for (ObjCInterfaceDecl::ivar_iterator I = OID->ivar_begin(),
          E = OID->ivar_end(); I != E; ++I) {
-      Indent() << (*I)->getType().getAsString(Policy) << ' ' << *I << ";\n";
+      Indent() << (*I)->getType().getAsString(Policy) << ' ' << **I << ";\n";
     }
     Indentation -= Policy.Indentation;
     Out << "}\n";
@@ -857,18 +920,18 @@ void DeclPrinter::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D) {
          E = D->protocol_end();
        I != E; ++I) {
     if (I != D->protocol_begin()) Out << ", ";
-    Out << *I;
+    Out << **I;
   }
 }
 
 void DeclPrinter::VisitObjCProtocolDecl(ObjCProtocolDecl *PID) {
-  Out << "@protocol " << PID << '\n';
+  Out << "@protocol " << *PID << '\n';
   VisitDeclContext(PID, false);
   Out << "@end";
 }
 
 void DeclPrinter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *PID) {
-  Out << "@implementation " << PID->getClassInterface() << '(' << PID << ")\n";
+  Out << "@implementation " << *PID->getClassInterface() << '(' << *PID <<")\n";
 
   VisitDeclContext(PID, false);
   Out << "@end";
@@ -876,7 +939,7 @@ void DeclPrinter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *PID) {
 }
 
 void DeclPrinter::VisitObjCCategoryDecl(ObjCCategoryDecl *PID) {
-  Out << "@interface " << PID->getClassInterface() << '(' << PID << ")\n";
+  Out << "@interface " << *PID->getClassInterface() << '(' << *PID << ")\n";
   VisitDeclContext(PID, false);
   Out << "@end";
 
@@ -884,8 +947,8 @@ void DeclPrinter::VisitObjCCategoryDecl(ObjCCategoryDecl *PID) {
 }
 
 void DeclPrinter::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *AID) {
-  Out << "@compatibility_alias " << AID
-      << ' ' << AID->getClassInterface() << ";\n";
+  Out << "@compatibility_alias " << *AID
+      << ' ' << *AID->getClassInterface() << ";\n";
 }
 
 /// PrintObjCPropertyDecl - print a property declaration.
@@ -904,58 +967,60 @@ void DeclPrinter::VisitObjCPropertyDecl(ObjCPropertyDecl *PDecl) {
         ObjCPropertyDecl::OBJC_PR_readonly) {
       Out << (first ? ' ' : ',') << "readonly";
       first = false;
-  }
+    }
 
-  if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_getter) {
-    Out << (first ? ' ' : ',') << "getter = "
-        << PDecl->getGetterName().getAsString();
-    first = false;
-  }
-  if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_setter) {
-    Out << (first ? ' ' : ',') << "setter = "
-        << PDecl->getSetterName().getAsString();
-    first = false;
-  }
+    if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_getter) {
+      Out << (first ? ' ' : ',') << "getter = "
+          << PDecl->getGetterName().getAsString();
+      first = false;
+    }
+    if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_setter) {
+      Out << (first ? ' ' : ',') << "setter = "
+          << PDecl->getSetterName().getAsString();
+      first = false;
+    }
 
-  if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_assign) {
-    Out << (first ? ' ' : ',') << "assign";
-    first = false;
-  }
+    if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_assign) {
+      Out << (first ? ' ' : ',') << "assign";
+      first = false;
+    }
 
-  if (PDecl->getPropertyAttributes() &
-      ObjCPropertyDecl::OBJC_PR_readwrite) {
-    Out << (first ? ' ' : ',') << "readwrite";
-    first = false;
-  }
+    if (PDecl->getPropertyAttributes() &
+        ObjCPropertyDecl::OBJC_PR_readwrite) {
+      Out << (first ? ' ' : ',') << "readwrite";
+      first = false;
+    }
 
-  if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_retain) {
-    Out << (first ? ' ' : ',') << "retain";
-    first = false;
-  }
+    if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_retain) {
+      Out << (first ? ' ' : ',') << "retain";
+      first = false;
+    }
 
-  if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_strong) {
-    Out << (first ? ' ' : ',') << "strong";
-    first = false;
-  }
+    if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_strong) {
+      Out << (first ? ' ' : ',') << "strong";
+      first = false;
+    }
 
-  if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_copy) {
-    Out << (first ? ' ' : ',') << "copy";
-    first = false;
-  }
+    if (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_copy) {
+      Out << (first ? ' ' : ',') << "copy";
+      first = false;
+    }
 
-  if (PDecl->getPropertyAttributes() &
-      ObjCPropertyDecl::OBJC_PR_nonatomic) {
-    Out << (first ? ' ' : ',') << "nonatomic";
-    first = false;
+    if (PDecl->getPropertyAttributes() &
+        ObjCPropertyDecl::OBJC_PR_nonatomic) {
+      Out << (first ? ' ' : ',') << "nonatomic";
+      first = false;
+    }
+    if (PDecl->getPropertyAttributes() &
+        ObjCPropertyDecl::OBJC_PR_atomic) {
+      Out << (first ? ' ' : ',') << "atomic";
+      first = false;
+    }
+    
+    (void) first; // Silence dead store warning due to idiomatic code.
+    Out << " )";
   }
-  if (PDecl->getPropertyAttributes() &
-      ObjCPropertyDecl::OBJC_PR_atomic) {
-    Out << (first ? ' ' : ',') << "atomic";
-    first = false;
-  }
-  Out << " )";
-  }
-  Out << ' ' << PDecl->getType().getAsString(Policy) << ' ' << PDecl;
+  Out << ' ' << PDecl->getType().getAsString(Policy) << ' ' << *PDecl;
 }
 
 void DeclPrinter::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *PID) {
@@ -963,15 +1028,15 @@ void DeclPrinter::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *PID) {
     Out << "@synthesize ";
   else
     Out << "@dynamic ";
-  Out << PID->getPropertyDecl();
+  Out << *PID->getPropertyDecl();
   if (PID->getPropertyIvarDecl())
-    Out << '=' << PID->getPropertyIvarDecl();
+    Out << '=' << *PID->getPropertyIvarDecl();
 }
 
 void DeclPrinter::VisitUsingDecl(UsingDecl *D) {
   Out << "using ";
   D->getQualifier()->print(Out, Policy);
-  Out << D;
+  Out << *D;
 }
 
 void
diff --git a/contrib/llvm/tools/clang/lib/AST/DeclTemplate.cpp b/contrib/llvm/tools/clang/lib/AST/DeclTemplate.cpp
index bc375d0ad2d3..558a4ccac96b 100644
--- a/contrib/llvm/tools/clang/lib/AST/DeclTemplate.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DeclTemplate.cpp
@@ -348,7 +348,7 @@ void ClassTemplateDecl::AddPartialSpecialization(
 }
 
 void ClassTemplateDecl::getPartialSpecializations(
-          llvm::SmallVectorImpl<ClassTemplatePartialSpecializationDecl *> &PS) {
+          SmallVectorImpl<ClassTemplatePartialSpecializationDecl *> &PS) {
   llvm::FoldingSet<ClassTemplatePartialSpecializationDecl> &PartialSpecs
     = getPartialSpecializations();
   PS.clear();
@@ -406,7 +406,7 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
   //  pack.
   ASTContext &Context = getASTContext();
   TemplateParameterList *Params = getTemplateParameters();
-  llvm::SmallVector<TemplateArgument, 16> TemplateArgs;
+  SmallVector<TemplateArgument, 16> TemplateArgs;
   TemplateArgs.resize(Params->size());
   GenerateInjectedTemplateArgs(getASTContext(), Params, TemplateArgs.data());
   CommonPtr->InjectedClassNameType
@@ -563,6 +563,24 @@ TemplateArgumentList::CreateCopy(ASTContext &Context,
   return new (Mem) TemplateArgumentList(StoredArgs, NumArgs, true);
 }
 
+FunctionTemplateSpecializationInfo *
+FunctionTemplateSpecializationInfo::Create(ASTContext &C, FunctionDecl *FD,
+                                           FunctionTemplateDecl *Template,
+                                           TemplateSpecializationKind TSK,
+                                       const TemplateArgumentList *TemplateArgs,
+                          const TemplateArgumentListInfo *TemplateArgsAsWritten,
+                                           SourceLocation POI) {
+  const ASTTemplateArgumentListInfo *ArgsAsWritten = 0;
+  if (TemplateArgsAsWritten)
+    ArgsAsWritten = ASTTemplateArgumentListInfo::Create(C,
+                                                        *TemplateArgsAsWritten);
+
+  return new (C) FunctionTemplateSpecializationInfo(FD, Template, TSK,
+                                                    TemplateArgs,
+                                                    ArgsAsWritten,
+                                                    POI);
+}
+
 //===----------------------------------------------------------------------===//
 // ClassTemplateSpecializationDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -638,15 +656,27 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
 
 SourceRange
 ClassTemplateSpecializationDecl::getSourceRange() const {
-  if (!ExplicitInfo)
-    return SourceRange();
-  SourceLocation Begin = getExternLoc();
-  if (Begin.isInvalid())
-    Begin = getTemplateKeywordLoc();
-  SourceLocation End = getRBraceLoc();
-  if (End.isInvalid())
-    End = getTypeAsWritten()->getTypeLoc().getEndLoc();
-  return SourceRange(Begin, End);
+  if (ExplicitInfo) {
+    SourceLocation Begin = getExternLoc();
+    if (Begin.isInvalid())
+      Begin = getTemplateKeywordLoc();
+    SourceLocation End = getRBraceLoc();
+    if (End.isInvalid())
+      End = getTypeAsWritten()->getTypeLoc().getEndLoc();
+    return SourceRange(Begin, End);
+  }
+  else {
+    // No explicit info available.
+    llvm::PointerUnion<ClassTemplateDecl *,
+                       ClassTemplatePartialSpecializationDecl *>
+      inst_from = getInstantiatedFrom();
+    if (inst_from.isNull())
+      return getSpecializedTemplate()->getSourceRange();
+    if (ClassTemplateDecl *ctd = inst_from.dyn_cast<ClassTemplateDecl*>())
+      return ctd->getSourceRange();
+    return inst_from.get<ClassTemplatePartialSpecializationDecl*>()
+      ->getSourceRange();
+  }
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/contrib/llvm/tools/clang/lib/AST/DeclarationName.cpp b/contrib/llvm/tools/clang/lib/AST/DeclarationName.cpp
index 72c0e9da7f1d..bf647ed7ac35 100644
--- a/contrib/llvm/tools/clang/lib/AST/DeclarationName.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DeclarationName.cpp
@@ -193,8 +193,7 @@ DeclarationName::NameKind DeclarationName::getNameKind() const {
   }
 
   // Can't actually get here.
-  assert(0 && "This should be unreachable!");
-  return Identifier;
+  llvm_unreachable("This should be unreachable!");
 }
 
 bool DeclarationName::isDependentName() const {
@@ -209,7 +208,7 @@ std::string DeclarationName::getAsString() const {
   return OS.str();
 }
 
-void DeclarationName::printName(llvm::raw_ostream &OS) const {
+void DeclarationName::printName(raw_ostream &OS) const {
   switch (getNameKind()) {
   case Identifier:
     if (const IdentifierInfo *II = getAsIdentifierInfo())
@@ -225,7 +224,7 @@ void DeclarationName::printName(llvm::raw_ostream &OS) const {
   case CXXConstructorName: {
     QualType ClassType = getCXXNameType();
     if (const RecordType *ClassRec = ClassType->getAs<RecordType>())
-      OS << ClassRec->getDecl();
+      OS << *ClassRec->getDecl();
     else
       OS << ClassType.getAsString();
     return;
@@ -235,7 +234,7 @@ void DeclarationName::printName(llvm::raw_ostream &OS) const {
     OS << '~';
     QualType Type = getCXXNameType();
     if (const RecordType *Rec = Type->getAs<RecordType>())
-      OS << Rec->getDecl();
+      OS << *Rec->getDecl();
     else
       OS << Type.getAsString();
     return;
@@ -266,7 +265,7 @@ void DeclarationName::printName(llvm::raw_ostream &OS) const {
     OS << "operator ";
     QualType Type = getCXXNameType();
     if (const RecordType *Rec = Type->getAs<RecordType>())
-      OS << Rec->getDecl();
+      OS << *Rec->getDecl();
     else
       OS << Type.getAsString();
     return;
@@ -276,7 +275,7 @@ void DeclarationName::printName(llvm::raw_ostream &OS) const {
     return;
   }
 
-  assert(false && "Unexpected declaration name kind");
+  llvm_unreachable("Unexpected declaration name kind");
 }
 
 QualType DeclarationName::getCXXNameType() const {
@@ -338,9 +337,8 @@ void *DeclarationName::getFETokenInfoAsVoid() const {
     return getCXXLiteralIdentifier()->getFETokenInfo<void>();
 
   default:
-    assert(false && "Declaration name has no FETokenInfo");
+    llvm_unreachable("Declaration name has no FETokenInfo");
   }
-  return 0;
 }
 
 void DeclarationName::setFETokenInfo(void *T) {
@@ -364,7 +362,7 @@ void DeclarationName::setFETokenInfo(void *T) {
     break;
 
   default:
-    assert(false && "Declaration name has no FETokenInfo");
+    llvm_unreachable("Declaration name has no FETokenInfo");
   }
 }
 
@@ -562,7 +560,7 @@ std::string DeclarationNameInfo::getAsString() const {
   return OS.str();
 }
 
-void DeclarationNameInfo::printName(llvm::raw_ostream &OS) const {
+void DeclarationNameInfo::printName(raw_ostream &OS) const {
   switch (Name.getNameKind()) {
   case DeclarationName::Identifier:
   case DeclarationName::ObjCZeroArgSelector:
@@ -588,7 +586,7 @@ void DeclarationNameInfo::printName(llvm::raw_ostream &OS) const {
       Name.printName(OS);
     return;
   }
-  assert(false && "Unexpected declaration name kind");
+  llvm_unreachable("Unexpected declaration name kind");
 }
 
 SourceLocation DeclarationNameInfo::getEndLoc() const {
@@ -621,6 +619,5 @@ SourceLocation DeclarationNameInfo::getEndLoc() const {
   case DeclarationName::CXXUsingDirective:
     return NameLoc;
   }
-  assert(false && "Unexpected declaration name kind");
-  return SourceLocation();
+  llvm_unreachable("Unexpected declaration name kind");
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/DumpXML.cpp b/contrib/llvm/tools/clang/lib/AST/DumpXML.cpp
index dfe0119f597d..2568adaa5a44 100644
--- a/contrib/llvm/tools/clang/lib/AST/DumpXML.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/DumpXML.cpp
@@ -53,9 +53,9 @@ enum NodeState {
 };
 
 struct Node {
-  llvm::StringRef Name;
+  StringRef Name;
   NodeState State;
-  Node(llvm::StringRef name) : Name(name), State(NS_Attrs) {}
+  Node(StringRef name) : Name(name), State(NS_Attrs) {}
 
   bool isDoneWithAttrs() const { return State != NS_Attrs; }
 };
@@ -159,7 +159,7 @@ template <class Impl> struct XMLTypeVisitor {
 #undef DISPATCH  
 };
 
-static llvm::StringRef getTypeKindName(Type *T) {
+static StringRef getTypeKindName(Type *T) {
   switch (T->getTypeClass()) {
 #define TYPE(DERIVED, BASE) case Type::DERIVED: return #DERIVED "Type";
 #define ABSTRACT_TYPE(DERIVED, BASE)
@@ -172,11 +172,11 @@ static llvm::StringRef getTypeKindName(Type *T) {
 
 struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
                    public XMLTypeVisitor<XMLDumper> {
-  llvm::raw_ostream &out;
+  raw_ostream &out;
   ASTContext &Context;
-  llvm::SmallVector<Node, 16> Stack;
+  SmallVector<Node, 16> Stack;
   unsigned Indent;
-  explicit XMLDumper(llvm::raw_ostream &OS, ASTContext &context)
+  explicit XMLDumper(raw_ostream &OS, ASTContext &context)
     : out(OS), Context(context), Indent(0) {}
 
   void indent() {
@@ -185,7 +185,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
   }
 
   /// Push a new node on the stack.
-  void push(llvm::StringRef name) {
+  void push(StringRef name) {
     if (!Stack.empty()) {
       assert(Stack.back().isDoneWithAttrs());
       if (Stack.back().State == NS_LazyChildren) {
@@ -200,7 +200,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
   }
 
   /// Set the given attribute to the given value.
-  void set(llvm::StringRef attr, llvm::StringRef value) {
+  void set(StringRef attr, StringRef value) {
     assert(!Stack.empty() && !Stack.back().isDoneWithAttrs());
     out << ' ' << attr << '=' << '"' << value << '"'; // TODO: quotation
   }
@@ -226,7 +226,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
 
   //---- General utilities -------------------------------------------//
 
-  void setPointer(llvm::StringRef prop, const void *p) {
+  void setPointer(StringRef prop, const void *p) {
     llvm::SmallString<10> buffer;
     llvm::raw_svector_ostream os(buffer);
     os << p;
@@ -238,11 +238,11 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
     setPointer("ptr", p);
   }
 
-  void setInteger(llvm::StringRef prop, const llvm::APSInt &v) {
+  void setInteger(StringRef prop, const llvm::APSInt &v) {
     set(prop, v.toString(10));
   }
 
-  void setInteger(llvm::StringRef prop, unsigned n) {
+  void setInteger(StringRef prop, unsigned n) {
     llvm::SmallString<10> buffer;
     llvm::raw_svector_ostream os(buffer);
     os << n;
@@ -250,7 +250,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
     set(prop, buffer);
   }
 
-  void setFlag(llvm::StringRef prop, bool flag) {
+  void setFlag(StringRef prop, bool flag) {
     if (flag) set(prop, "true");
   }
 
@@ -268,7 +268,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
   class TemporaryContainer {
     XMLDumper &Dumper;
   public:
-    TemporaryContainer(XMLDumper &dumper, llvm::StringRef name)
+    TemporaryContainer(XMLDumper &dumper, StringRef name)
       : Dumper(dumper) {
       Dumper.push(name);
       Dumper.completeAttrs();
@@ -303,7 +303,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
     completeAttrs();
     pop();
   }
-  void visitDeclRef(llvm::StringRef Name, Decl *D) {
+  void visitDeclRef(StringRef Name, Decl *D) {
     TemporaryContainer C(*this, Name);
     if (D) visitDeclRef(D);
   }
@@ -423,7 +423,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
 
   // LinkageSpecDecl
   void visitLinkageSpecDeclAttrs(LinkageSpecDecl *D) {
-    llvm::StringRef lang = "";
+    StringRef lang = "";
     switch (D->getLanguage()) {
     case LinkageSpecDecl::lang_c: lang = "C"; break;
     case LinkageSpecDecl::lang_cxx: lang = "C++"; break;
@@ -742,8 +742,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
 
   // ObjCClassDecl
   void visitObjCClassDeclChildren(ObjCClassDecl *D) {
-    for (ObjCClassDecl::iterator I = D->begin(), E = D->end(); I != E; ++I)
-      visitDeclRef(I->getInterface());
+    visitDeclRef(D->getForwardInterfaceDecl());
   }
 
   // ObjCInterfaceDecl
@@ -860,7 +859,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
   }
 
   // ObjCIvarDecl
-  void setAccessControl(llvm::StringRef prop, ObjCIvarDecl::AccessControl AC) {
+  void setAccessControl(StringRef prop, ObjCIvarDecl::AccessControl AC) {
     switch (AC) {
     case ObjCIvarDecl::None: return set(prop, "none");
     case ObjCIvarDecl::Private: return set(prop, "private");
@@ -1031,13 +1030,13 @@ void Decl::dumpXML() const {
   dumpXML(llvm::errs());
 }
 
-void Decl::dumpXML(llvm::raw_ostream &out) const {
+void Decl::dumpXML(raw_ostream &out) const {
   XMLDumper(out, getASTContext()).dispatch(const_cast<Decl*>(this));
 }
 
 #else /* ifndef NDEBUG */
 
 void Decl::dumpXML() const {}
-void Decl::dumpXML(llvm::raw_ostream &out) const {}
+void Decl::dumpXML(raw_ostream &out) const {}
 
 #endif
diff --git a/contrib/llvm/tools/clang/lib/AST/Expr.cpp b/contrib/llvm/tools/clang/lib/AST/Expr.cpp
index 4611ae369969..b0bcfe09f6dc 100644
--- a/contrib/llvm/tools/clang/lib/AST/Expr.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/Expr.cpp
@@ -130,57 +130,6 @@ SourceLocation Expr::getExprLoc() const {
 // Primary Expressions.
 //===----------------------------------------------------------------------===//
 
-void ExplicitTemplateArgumentList::initializeFrom(
-                                      const TemplateArgumentListInfo &Info) {
-  LAngleLoc = Info.getLAngleLoc();
-  RAngleLoc = Info.getRAngleLoc();
-  NumTemplateArgs = Info.size();
-
-  TemplateArgumentLoc *ArgBuffer = getTemplateArgs();
-  for (unsigned i = 0; i != NumTemplateArgs; ++i)
-    new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
-}
-
-void ExplicitTemplateArgumentList::initializeFrom(
-                                          const TemplateArgumentListInfo &Info,
-                                                  bool &Dependent, 
-                                                  bool &InstantiationDependent,
-                                       bool &ContainsUnexpandedParameterPack) {
-  LAngleLoc = Info.getLAngleLoc();
-  RAngleLoc = Info.getRAngleLoc();
-  NumTemplateArgs = Info.size();
-
-  TemplateArgumentLoc *ArgBuffer = getTemplateArgs();
-  for (unsigned i = 0; i != NumTemplateArgs; ++i) {
-    Dependent = Dependent || Info[i].getArgument().isDependent();
-    InstantiationDependent = InstantiationDependent || 
-                             Info[i].getArgument().isInstantiationDependent();
-    ContainsUnexpandedParameterPack 
-      = ContainsUnexpandedParameterPack || 
-        Info[i].getArgument().containsUnexpandedParameterPack();
-
-    new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
-  }
-}
-
-void ExplicitTemplateArgumentList::copyInto(
-                                      TemplateArgumentListInfo &Info) const {
-  Info.setLAngleLoc(LAngleLoc);
-  Info.setRAngleLoc(RAngleLoc);
-  for (unsigned I = 0; I != NumTemplateArgs; ++I)
-    Info.addArgument(getTemplateArgs()[I]);
-}
-
-std::size_t ExplicitTemplateArgumentList::sizeFor(unsigned NumTemplateArgs) {
-  return sizeof(ExplicitTemplateArgumentList) +
-         sizeof(TemplateArgumentLoc) * NumTemplateArgs;
-}
-
-std::size_t ExplicitTemplateArgumentList::sizeFor(
-                                      const TemplateArgumentListInfo &Info) {
-  return sizeFor(Info.size());
-}
-
 /// \brief Compute the type-, value-, and instantiation-dependence of a 
 /// declaration reference
 /// based on the declaration being referenced.
@@ -325,7 +274,8 @@ DeclRefExpr::DeclRefExpr(NestedNameSpecifierLoc QualifierLoc,
     if (InstantiationDependent)
       setInstantiationDependent(true);
   }
-      
+  DeclRefExprBits.HadMultipleCandidates = 0;
+
   computeDependence();
 }
 
@@ -360,7 +310,7 @@ DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
   if (FoundD)
     Size += sizeof(NamedDecl *);
   if (TemplateArgs)
-    Size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
+    Size += ASTTemplateArgumentListInfo::sizeFor(*TemplateArgs);
 
   void *Mem = Context.Allocate(Size, llvm::alignOf<DeclRefExpr>());
   return new (Mem) DeclRefExpr(QualifierLoc, D, NameInfo, FoundD, TemplateArgs,
@@ -378,7 +328,7 @@ DeclRefExpr *DeclRefExpr::CreateEmpty(ASTContext &Context,
   if (HasFoundDecl)
     Size += sizeof(NamedDecl *);
   if (HasExplicitTemplateArgs)
-    Size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
+    Size += ASTTemplateArgumentListInfo::sizeFor(NumTemplateArgs);
 
   void *Mem = Context.Allocate(Size, llvm::alignOf<DeclRefExpr>());
   return new (Mem) DeclRefExpr(EmptyShell());
@@ -463,7 +413,7 @@ std::string PredefinedExpr::ComputeName(IdentType IT, const Decl *CurrentDecl) {
     // For incorrect code, there might not be an ObjCInterfaceDecl.  Do
     // a null check to avoid a crash.
     if (const ObjCInterfaceDecl *ID = MD->getClassInterface())
-      Out << ID;
+      Out << *ID;
 
     if (const ObjCCategoryImplDecl *CID =
         dyn_cast<ObjCCategoryImplDecl>(MD->getDeclContext()))
@@ -532,9 +482,8 @@ double FloatingLiteral::getValueAsApproximateDouble() const {
   return V.convertToDouble();
 }
 
-StringLiteral *StringLiteral::Create(ASTContext &C, llvm::StringRef Str,
-                                     bool Wide,
-                                     bool Pascal, QualType Ty,
+StringLiteral *StringLiteral::Create(ASTContext &C, StringRef Str,
+                                     StringKind Kind, bool Pascal, QualType Ty,
                                      const SourceLocation *Loc,
                                      unsigned NumStrs) {
   // Allocate enough space for the StringLiteral plus an array of locations for
@@ -549,7 +498,7 @@ StringLiteral *StringLiteral::Create(ASTContext &C, llvm::StringRef Str,
   memcpy(AStrData, Str.data(), Str.size());
   SL->StrData = AStrData;
   SL->ByteLength = Str.size();
-  SL->IsWide = Wide;
+  SL->Kind = Kind;
   SL->IsPascal = Pascal;
   SL->TokLocs[0] = Loc[0];
   SL->NumConcatenated = NumStrs;
@@ -570,7 +519,7 @@ StringLiteral *StringLiteral::CreateEmpty(ASTContext &C, unsigned NumStrs) {
   return SL;
 }
 
-void StringLiteral::setString(ASTContext &C, llvm::StringRef Str) {
+void StringLiteral::setString(ASTContext &C, StringRef Str) {
   char *AStrData = new (C, 1) char[Str.size()];
   memcpy(AStrData, Str.data(), Str.size());
   StrData = AStrData;
@@ -587,8 +536,8 @@ void StringLiteral::setString(ASTContext &C, llvm::StringRef Str) {
 SourceLocation StringLiteral::
 getLocationOfByte(unsigned ByteNo, const SourceManager &SM,
                   const LangOptions &Features, const TargetInfo &Target) const {
-  assert(!isWide() && "This doesn't work for wide strings yet");
-  
+  assert(Kind == StringLiteral::Ascii && "This only works for ASCII strings");
+
   // Loop over all of the tokens in this string until we find the one that
   // contains the byte we're looking for.
   unsigned TokNo = 0;
@@ -604,7 +553,7 @@ getLocationOfByte(unsigned ByteNo, const SourceManager &SM,
     // Re-lex the token to get its length and original spelling.
     std::pair<FileID, unsigned> LocInfo =SM.getDecomposedLoc(StrTokSpellingLoc);
     bool Invalid = false;
-    llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+    StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
     if (Invalid)
       return StrTokSpellingLoc;
     
@@ -647,7 +596,7 @@ getLocationOfByte(unsigned ByteNo, const SourceManager &SM,
 /// corresponds to, e.g. "sizeof" or "[pre]++".
 const char *UnaryOperator::getOpcodeStr(Opcode Op) {
   switch (Op) {
-  default: assert(0 && "Unknown unary operator");
+  default: llvm_unreachable("Unknown unary operator");
   case UO_PostInc: return "++";
   case UO_PostDec: return "--";
   case UO_PreInc:  return "++";
@@ -667,7 +616,7 @@ const char *UnaryOperator::getOpcodeStr(Opcode Op) {
 UnaryOperatorKind
 UnaryOperator::getOverloadedOpcode(OverloadedOperatorKind OO, bool Postfix) {
   switch (OO) {
-  default: assert(false && "No unary operator for overloaded function");
+  default: llvm_unreachable("No unary operator for overloaded function");
   case OO_PlusPlus:   return Postfix ? UO_PostInc : UO_PreInc;
   case OO_MinusMinus: return Postfix ? UO_PostDec : UO_PreDec;
   case OO_Amp:        return UO_AddrOf;
@@ -771,7 +720,13 @@ CallExpr::CallExpr(ASTContext &C, StmtClass SC, unsigned NumPreArgs,
 }
 
 Decl *CallExpr::getCalleeDecl() {
-  Expr *CEE = getCallee()->IgnoreParenCasts();
+  Expr *CEE = getCallee()->IgnoreParenImpCasts();
+    
+  while (SubstNonTypeTemplateParmExpr *NTTP
+                                = dyn_cast<SubstNonTypeTemplateParmExpr>(CEE)) {
+    CEE = NTTP->getReplacement()->IgnoreParenCasts();
+  }
+  
   // If we're calling a dereference, look at the pointer instead.
   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(CEE)) {
     if (BO->isPtrMemOp())
@@ -947,7 +902,7 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
     Size += sizeof(MemberNameQualifier);
 
   if (targs)
-    Size += ExplicitTemplateArgumentList::sizeFor(*targs);
+    Size += ASTTemplateArgumentListInfo::sizeFor(*targs);
 
   void *Mem = C.Allocate(Size, llvm::alignOf<MemberExpr>());
   MemberExpr *E = new (Mem) MemberExpr(base, isarrow, memberdecl, nameinfo,
@@ -1008,6 +963,94 @@ SourceRange MemberExpr::getSourceRange() const {
   return SourceRange(StartLoc, EndLoc);
 }
 
+void CastExpr::CheckCastConsistency() const {
+  switch (getCastKind()) {
+  case CK_DerivedToBase:
+  case CK_UncheckedDerivedToBase:
+  case CK_DerivedToBaseMemberPointer:
+  case CK_BaseToDerived:
+  case CK_BaseToDerivedMemberPointer:
+    assert(!path_empty() && "Cast kind should have a base path!");
+    break;
+
+  case CK_CPointerToObjCPointerCast:
+    assert(getType()->isObjCObjectPointerType());
+    assert(getSubExpr()->getType()->isPointerType());
+    goto CheckNoBasePath;
+
+  case CK_BlockPointerToObjCPointerCast:
+    assert(getType()->isObjCObjectPointerType());
+    assert(getSubExpr()->getType()->isBlockPointerType());
+    goto CheckNoBasePath;
+
+  case CK_BitCast:
+    // Arbitrary casts to C pointer types count as bitcasts.
+    // Otherwise, we should only have block and ObjC pointer casts
+    // here if they stay within the type kind.
+    if (!getType()->isPointerType()) {
+      assert(getType()->isObjCObjectPointerType() == 
+             getSubExpr()->getType()->isObjCObjectPointerType());
+      assert(getType()->isBlockPointerType() == 
+             getSubExpr()->getType()->isBlockPointerType());
+    }
+    goto CheckNoBasePath;
+
+  case CK_AnyPointerToBlockPointerCast:
+    assert(getType()->isBlockPointerType());
+    assert(getSubExpr()->getType()->isAnyPointerType() &&
+           !getSubExpr()->getType()->isBlockPointerType());
+    goto CheckNoBasePath;
+
+  // These should not have an inheritance path.
+  case CK_Dynamic:
+  case CK_ToUnion:
+  case CK_ArrayToPointerDecay:
+  case CK_FunctionToPointerDecay:
+  case CK_NullToMemberPointer:
+  case CK_NullToPointer:
+  case CK_ConstructorConversion:
+  case CK_IntegralToPointer:
+  case CK_PointerToIntegral:
+  case CK_ToVoid:
+  case CK_VectorSplat:
+  case CK_IntegralCast:
+  case CK_IntegralToFloating:
+  case CK_FloatingToIntegral:
+  case CK_FloatingCast:
+  case CK_ObjCObjectLValueCast:
+  case CK_FloatingRealToComplex:
+  case CK_FloatingComplexToReal:
+  case CK_FloatingComplexCast:
+  case CK_FloatingComplexToIntegralComplex:
+  case CK_IntegralRealToComplex:
+  case CK_IntegralComplexToReal:
+  case CK_IntegralComplexCast:
+  case CK_IntegralComplexToFloatingComplex:
+  case CK_ARCProduceObject:
+  case CK_ARCConsumeObject:
+  case CK_ARCReclaimReturnedObject:
+  case CK_ARCExtendBlockObject:
+    assert(!getType()->isBooleanType() && "unheralded conversion to bool");
+    goto CheckNoBasePath;
+
+  case CK_Dependent:
+  case CK_LValueToRValue:
+  case CK_GetObjCProperty:
+  case CK_NoOp:
+  case CK_PointerToBoolean:
+  case CK_IntegralToBoolean:
+  case CK_FloatingToBoolean:
+  case CK_MemberPointerToBoolean:
+  case CK_FloatingComplexToBoolean:
+  case CK_IntegralComplexToBoolean:
+  case CK_LValueBitCast:            // -> bool&
+  case CK_UserDefinedConversion:    // operator bool()
+  CheckNoBasePath:
+    assert(path_empty() && "Cast kind should not have a base path!");
+    break;
+  }
+}
+
 const char *CastExpr::getCastKindName() const {
   switch (getCastKind()) {
   case CK_Dependent:
@@ -1072,8 +1115,10 @@ const char *CastExpr::getCastKindName() const {
     return "FloatingToBoolean";
   case CK_MemberPointerToBoolean:
     return "MemberPointerToBoolean";
-  case CK_AnyPointerToObjCPointerCast:
-    return "AnyPointerToObjCPointerCast";
+  case CK_CPointerToObjCPointerCast:
+    return "CPointerToObjCPointerCast";
+  case CK_BlockPointerToObjCPointerCast:
+    return "BlockPointerToObjCPointerCast";
   case CK_AnyPointerToBlockPointerCast:
     return "AnyPointerToBlockPointerCast";
   case CK_ObjCObjectLValueCast:
@@ -1098,12 +1143,14 @@ const char *CastExpr::getCastKindName() const {
     return "IntegralComplexCast";
   case CK_IntegralComplexToFloatingComplex:
     return "IntegralComplexToFloatingComplex";
-  case CK_ObjCConsumeObject:
-    return "ObjCConsumeObject";
-  case CK_ObjCProduceObject:
-    return "ObjCProduceObject";
-  case CK_ObjCReclaimReturnedObject:
-    return "ObjCReclaimReturnedObject";
+  case CK_ARCConsumeObject:
+    return "ARCConsumeObject";
+  case CK_ARCProduceObject:
+    return "ARCProduceObject";
+  case CK_ARCReclaimReturnedObject:
+    return "ARCReclaimReturnedObject";
+  case CK_ARCExtendBlockObject:
+    return "ARCCExtendBlockObject";
   }
 
   llvm_unreachable("Unhandled cast kind!");
@@ -1243,7 +1290,7 @@ const char *BinaryOperator::getOpcodeStr(Opcode Op) {
 BinaryOperatorKind
 BinaryOperator::getOverloadedOpcode(OverloadedOperatorKind OO) {
   switch (OO) {
-  default: assert(false && "Not an overloadable binary operator");
+  default: llvm_unreachable("Not an overloadable binary operator");
   case OO_Plus: return BO_Add;
   case OO_Minus: return BO_Sub;
   case OO_Star: return BO_Mul;
@@ -1491,6 +1538,7 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
   }
   case CompoundAssignOperatorClass:
   case VAArgExprClass:
+  case AtomicExprClass:
     return false;
 
   case ConditionalOperatorClass: {
@@ -1525,8 +1573,24 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
     R2 = cast<ArraySubscriptExpr>(this)->getRHS()->getSourceRange();
     return true;
 
+  case CXXOperatorCallExprClass: {
+    // We warn about operator== and operator!= even when user-defined operator
+    // overloads as there is no reasonable way to define these such that they
+    // have non-trivial, desirable side-effects. See the -Wunused-comparison
+    // warning: these operators are commonly typo'ed, and so warning on them
+    // provides additional value as well. If this list is updated,
+    // DiagnoseUnusedComparison should be as well.
+    const CXXOperatorCallExpr *Op = cast<CXXOperatorCallExpr>(this);
+    if (Op->getOperator() == OO_EqualEqual ||
+        Op->getOperator() == OO_ExclaimEqual) {
+      Loc = Op->getOperatorLoc();
+      R1 = Op->getSourceRange();
+      return true;
+    }
+
+    // Fallthrough for generic call handling.
+  }
   case CallExprClass:
-  case CXXOperatorCallExprClass:
   case CXXMemberCallExprClass: {
     // If this is a direct call, get the callee.
     const CallExpr *CE = cast<CallExpr>(this);
@@ -1666,8 +1730,15 @@ bool Expr::isOBJCGCCandidate(ASTContext &Ctx) const {
                                                       ->isOBJCGCCandidate(Ctx);
   case CStyleCastExprClass:
     return cast<CStyleCastExpr>(E)->getSubExpr()->isOBJCGCCandidate(Ctx);
+  case BlockDeclRefExprClass:
   case DeclRefExprClass: {
-    const Decl *D = cast<DeclRefExpr>(E)->getDecl();
+    
+    const Decl *D;
+    if (const BlockDeclRefExpr *BDRE = dyn_cast<BlockDeclRefExpr>(E))
+        D = BDRE->getDecl();
+    else 
+        D = cast<DeclRefExpr>(E)->getDecl();
+        
     if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
       if (VD->hasGlobalStorage())
         return true;
@@ -2024,7 +2095,11 @@ Expr *Expr::IgnoreParenCasts() {
       E = Materialize->GetTemporaryExpr();
       continue;
     }
-      
+    if (SubstNonTypeTemplateParmExpr *NTTP
+                                  = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
+      E = NTTP->getReplacement();
+      continue;
+    }      
     return E;
   }
 }
@@ -2058,6 +2133,10 @@ Expr *Expr::IgnoreParenLValueCasts() {
                                       = dyn_cast<MaterializeTemporaryExpr>(E)) {
       E = Materialize->GetTemporaryExpr();
       continue;
+    } else if (SubstNonTypeTemplateParmExpr *NTTP
+                                  = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
+      E = NTTP->getReplacement();
+      continue;
     }
     break;
   }
@@ -2092,6 +2171,11 @@ Expr *Expr::IgnoreParenImpCasts() {
       E = Materialize->GetTemporaryExpr();
       continue;
     }
+    if (SubstNonTypeTemplateParmExpr *NTTP
+                                  = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
+      E = NTTP->getReplacement();
+      continue;
+    }
     return E;
   }
 }
@@ -2149,6 +2233,12 @@ Expr *Expr::IgnoreParenNoopCasts(ASTContext &Ctx) {
       }
     }
 
+    if (SubstNonTypeTemplateParmExpr *NTTP
+                                  = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
+      E = NTTP->getReplacement();
+      continue;
+    }
+    
     return E;
   }
 }
@@ -2397,7 +2487,7 @@ bool Expr::isConstantInitializer(ASTContext &Ctx, bool IsForRef) const {
     break;
       
   case MaterializeTemporaryExprClass:
-    return llvm::cast<MaterializeTemporaryExpr>(this)->GetTemporaryExpr()
+    return cast<MaterializeTemporaryExpr>(this)->GetTemporaryExpr()
                                             ->isConstantInitializer(Ctx, false);
   }
   return isEvaluatable(Ctx);
@@ -2414,9 +2504,7 @@ Expr::isNullPointerConstant(ASTContext &Ctx,
   if (isValueDependent()) {
     switch (NPC) {
     case NPC_NeverValueDependent:
-      assert(false && "Unexpected value dependent expression!");
-      // If the unthinkable happens, fall through to the safest alternative.
-        
+      llvm_unreachable("Unexpected value dependent expression!");
     case NPC_ValueDependentIsNull:
       if (isTypeDependent() || getType()->isIntegralType(Ctx))
         return NPCK_ZeroInteger;
@@ -2576,7 +2664,7 @@ unsigned ExtVectorElementExpr::getNumElements() const {
 bool ExtVectorElementExpr::containsDuplicateElements() const {
   // FIXME: Refactor this code to an accessor on the AST node which returns the
   // "type" of component access, and share with code below and in Sema.
-  llvm::StringRef Comp = Accessor->getName();
+  StringRef Comp = Accessor->getName();
 
   // Halving swizzles do not contain duplicate elements.
   if (Comp == "hi" || Comp == "lo" || Comp == "even" || Comp == "odd")
@@ -2587,7 +2675,7 @@ bool ExtVectorElementExpr::containsDuplicateElements() const {
     Comp = Comp.substr(1);
 
   for (unsigned i = 0, e = Comp.size(); i != e; ++i)
-    if (Comp.substr(i + 1).find(Comp[i]) != llvm::StringRef::npos)
+    if (Comp.substr(i + 1).find(Comp[i]) != StringRef::npos)
         return true;
 
   return false;
@@ -2595,8 +2683,8 @@ bool ExtVectorElementExpr::containsDuplicateElements() const {
 
 /// getEncodedElementAccess - We encode the fields as a llvm ConstantArray.
 void ExtVectorElementExpr::getEncodedElementAccess(
-                                  llvm::SmallVectorImpl<unsigned> &Elts) const {
-  llvm::StringRef Comp = Accessor->getName();
+                                  SmallVectorImpl<unsigned> &Elts) const {
+  StringRef Comp = Accessor->getName();
   if (Comp[0] == 's' || Comp[0] == 'S')
     Comp = Comp.substr(1);
 
@@ -2630,23 +2718,23 @@ ObjCMessageExpr::ObjCMessageExpr(QualType T,
                                  bool IsInstanceSuper,
                                  QualType SuperType,
                                  Selector Sel, 
-                                 SourceLocation SelLoc,
+                                 ArrayRef<SourceLocation> SelLocs,
+                                 SelectorLocationsKind SelLocsK,
                                  ObjCMethodDecl *Method,
-                                 Expr **Args, unsigned NumArgs,
+                                 ArrayRef<Expr *> Args,
                                  SourceLocation RBracLoc)
   : Expr(ObjCMessageExprClass, T, VK, OK_Ordinary,
          /*TypeDependent=*/false, /*ValueDependent=*/false,
          /*InstantiationDependent=*/false,
          /*ContainsUnexpandedParameterPack=*/false),
-    NumArgs(NumArgs), Kind(IsInstanceSuper? SuperInstance : SuperClass),
-    HasMethod(Method != 0), IsDelegateInitCall(false), SuperLoc(SuperLoc),
     SelectorOrMethod(reinterpret_cast<uintptr_t>(Method? Method
                                                        : Sel.getAsOpaquePtr())),
-    SelectorLoc(SelLoc), LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
+    Kind(IsInstanceSuper? SuperInstance : SuperClass),
+    HasMethod(Method != 0), IsDelegateInitCall(false), SuperLoc(SuperLoc),
+    LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
 {
+  initArgsAndSelLocs(Args, SelLocs, SelLocsK);
   setReceiverPointer(SuperType.getAsOpaquePtr());
-  if (NumArgs)
-    memcpy(getArgs(), Args, NumArgs * sizeof(Expr *));
 }
 
 ObjCMessageExpr::ObjCMessageExpr(QualType T,
@@ -2654,33 +2742,22 @@ ObjCMessageExpr::ObjCMessageExpr(QualType T,
                                  SourceLocation LBracLoc,
                                  TypeSourceInfo *Receiver,
                                  Selector Sel,
-                                 SourceLocation SelLoc,
+                                 ArrayRef<SourceLocation> SelLocs,
+                                 SelectorLocationsKind SelLocsK,
                                  ObjCMethodDecl *Method,
-                                 Expr **Args, unsigned NumArgs,
+                                 ArrayRef<Expr *> Args,
                                  SourceLocation RBracLoc)
   : Expr(ObjCMessageExprClass, T, VK, OK_Ordinary, T->isDependentType(),
          T->isDependentType(), T->isInstantiationDependentType(),
          T->containsUnexpandedParameterPack()),
-    NumArgs(NumArgs), Kind(Class),
-    HasMethod(Method != 0), IsDelegateInitCall(false),
     SelectorOrMethod(reinterpret_cast<uintptr_t>(Method? Method
                                                        : Sel.getAsOpaquePtr())),
-    SelectorLoc(SelLoc), LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
+    Kind(Class),
+    HasMethod(Method != 0), IsDelegateInitCall(false),
+    LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
 {
+  initArgsAndSelLocs(Args, SelLocs, SelLocsK);
   setReceiverPointer(Receiver);
-  Expr **MyArgs = getArgs();
-  for (unsigned I = 0; I != NumArgs; ++I) {
-    if (Args[I]->isTypeDependent())
-      ExprBits.TypeDependent = true;
-    if (Args[I]->isValueDependent())
-      ExprBits.ValueDependent = true;
-    if (Args[I]->isInstantiationDependent())
-      ExprBits.InstantiationDependent = true;
-    if (Args[I]->containsUnexpandedParameterPack())
-      ExprBits.ContainsUnexpandedParameterPack = true;
-  
-    MyArgs[I] = Args[I];
-  }
 }
 
 ObjCMessageExpr::ObjCMessageExpr(QualType T,
@@ -2688,23 +2765,31 @@ ObjCMessageExpr::ObjCMessageExpr(QualType T,
                                  SourceLocation LBracLoc,
                                  Expr *Receiver,
                                  Selector Sel, 
-                                 SourceLocation SelLoc,
+                                 ArrayRef<SourceLocation> SelLocs,
+                                 SelectorLocationsKind SelLocsK,
                                  ObjCMethodDecl *Method,
-                                 Expr **Args, unsigned NumArgs,
+                                 ArrayRef<Expr *> Args,
                                  SourceLocation RBracLoc)
   : Expr(ObjCMessageExprClass, T, VK, OK_Ordinary, Receiver->isTypeDependent(),
          Receiver->isTypeDependent(),
          Receiver->isInstantiationDependent(),
          Receiver->containsUnexpandedParameterPack()),
-    NumArgs(NumArgs), Kind(Instance),
-    HasMethod(Method != 0), IsDelegateInitCall(false),
     SelectorOrMethod(reinterpret_cast<uintptr_t>(Method? Method
                                                        : Sel.getAsOpaquePtr())),
-    SelectorLoc(SelLoc), LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
+    Kind(Instance),
+    HasMethod(Method != 0), IsDelegateInitCall(false),
+    LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
 {
+  initArgsAndSelLocs(Args, SelLocs, SelLocsK);
   setReceiverPointer(Receiver);
+}
+
+void ObjCMessageExpr::initArgsAndSelLocs(ArrayRef<Expr *> Args,
+                                         ArrayRef<SourceLocation> SelLocs,
+                                         SelectorLocationsKind SelLocsK) {
+  setNumArgs(Args.size());
   Expr **MyArgs = getArgs();
-  for (unsigned I = 0; I != NumArgs; ++I) {
+  for (unsigned I = 0; I != Args.size(); ++I) {
     if (Args[I]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (Args[I]->isValueDependent())
@@ -2716,6 +2801,10 @@ ObjCMessageExpr::ObjCMessageExpr(QualType T,
   
     MyArgs[I] = Args[I];
   }
+
+  SelLocsKind = SelLocsK;
+  if (SelLocsK == SelLoc_NonStandard)
+    std::copy(SelLocs.begin(), SelLocs.end(), getStoredSelLocs());
 }
 
 ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
@@ -2725,16 +2814,15 @@ ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
                                          bool IsInstanceSuper,
                                          QualType SuperType,
                                          Selector Sel, 
-                                         SourceLocation SelLoc,
+                                         ArrayRef<SourceLocation> SelLocs,
                                          ObjCMethodDecl *Method,
-                                         Expr **Args, unsigned NumArgs,
+                                         ArrayRef<Expr *> Args,
                                          SourceLocation RBracLoc) {
-  unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
-    NumArgs * sizeof(Expr *);
-  void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
+  SelectorLocationsKind SelLocsK;
+  ObjCMessageExpr *Mem = alloc(Context, Args, RBracLoc, SelLocs, Sel, SelLocsK);
   return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, SuperLoc, IsInstanceSuper,
-                                   SuperType, Sel, SelLoc, Method, Args,NumArgs, 
-                                   RBracLoc);
+                                   SuperType, Sel, SelLocs, SelLocsK,
+                                   Method, Args, RBracLoc);
 }
 
 ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
@@ -2742,15 +2830,14 @@ ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
                                          SourceLocation LBracLoc,
                                          TypeSourceInfo *Receiver,
                                          Selector Sel, 
-                                         SourceLocation SelLoc,
+                                         ArrayRef<SourceLocation> SelLocs,
                                          ObjCMethodDecl *Method,
-                                         Expr **Args, unsigned NumArgs,
+                                         ArrayRef<Expr *> Args,
                                          SourceLocation RBracLoc) {
-  unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
-    NumArgs * sizeof(Expr *);
-  void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
-  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, Receiver, Sel, SelLoc,
-                                   Method, Args, NumArgs, RBracLoc);
+  SelectorLocationsKind SelLocsK;
+  ObjCMessageExpr *Mem = alloc(Context, Args, RBracLoc, SelLocs, Sel, SelLocsK);
+  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, Receiver, Sel,
+                                   SelLocs, SelLocsK, Method, Args, RBracLoc);
 }
 
 ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
@@ -2758,25 +2845,50 @@ ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
                                          SourceLocation LBracLoc,
                                          Expr *Receiver,
                                          Selector Sel,
-                                         SourceLocation SelLoc,
+                                         ArrayRef<SourceLocation> SelLocs,
                                          ObjCMethodDecl *Method,
-                                         Expr **Args, unsigned NumArgs,
+                                         ArrayRef<Expr *> Args,
                                          SourceLocation RBracLoc) {
-  unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
-    NumArgs * sizeof(Expr *);
-  void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
-  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, Receiver, Sel, SelLoc,
-                                   Method, Args, NumArgs, RBracLoc);
+  SelectorLocationsKind SelLocsK;
+  ObjCMessageExpr *Mem = alloc(Context, Args, RBracLoc, SelLocs, Sel, SelLocsK);
+  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, Receiver, Sel,
+                                   SelLocs, SelLocsK, Method, Args, RBracLoc);
 }
 
 ObjCMessageExpr *ObjCMessageExpr::CreateEmpty(ASTContext &Context, 
-                                              unsigned NumArgs) {
-  unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
-    NumArgs * sizeof(Expr *);
-  void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
+                                              unsigned NumArgs,
+                                              unsigned NumStoredSelLocs) {
+  ObjCMessageExpr *Mem = alloc(Context, NumArgs, NumStoredSelLocs);
   return new (Mem) ObjCMessageExpr(EmptyShell(), NumArgs);
 }
 
+ObjCMessageExpr *ObjCMessageExpr::alloc(ASTContext &C,
+                                        ArrayRef<Expr *> Args,
+                                        SourceLocation RBraceLoc,
+                                        ArrayRef<SourceLocation> SelLocs,
+                                        Selector Sel,
+                                        SelectorLocationsKind &SelLocsK) {
+  SelLocsK = hasStandardSelectorLocs(Sel, SelLocs, Args, RBraceLoc);
+  unsigned NumStoredSelLocs = (SelLocsK == SelLoc_NonStandard) ? SelLocs.size()
+                                                               : 0;
+  return alloc(C, Args.size(), NumStoredSelLocs);
+}
+
+ObjCMessageExpr *ObjCMessageExpr::alloc(ASTContext &C,
+                                        unsigned NumArgs,
+                                        unsigned NumStoredSelLocs) {
+  unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
+    NumArgs * sizeof(Expr *) + NumStoredSelLocs * sizeof(SourceLocation);
+  return (ObjCMessageExpr *)C.Allocate(Size,
+                                     llvm::AlignOf<ObjCMessageExpr>::Alignment);
+}
+
+void ObjCMessageExpr::getSelectorLocs(
+                               SmallVectorImpl<SourceLocation> &SelLocs) const {
+  for (unsigned i = 0, e = getNumSelectorLocs(); i != e; ++i)
+    SelLocs.push_back(getSelectorLoc(i));
+}
+
 SourceRange ObjCMessageExpr::getReceiverRange() const {
   switch (getReceiverKind()) {
   case Instance:
@@ -2830,7 +2942,7 @@ ObjCInterfaceDecl *ObjCMessageExpr::getReceiverInterface() const {
   return 0;
 }
 
-llvm::StringRef ObjCBridgedCastExpr::getBridgeKindName() const {
+StringRef ObjCBridgedCastExpr::getBridgeKindName() const {
   switch (getBridgeKind()) {
   case OBC_Bridge:
     return "__bridge";
@@ -2844,7 +2956,7 @@ llvm::StringRef ObjCBridgedCastExpr::getBridgeKindName() const {
 }
 
 bool ChooseExpr::isConditionTrue(const ASTContext &C) const {
-  return getCond()->EvaluateAsInt(C) != 0;
+  return getCond()->EvaluateKnownConstInt(C) != 0;
 }
 
 ShuffleVectorExpr::ShuffleVectorExpr(ASTContext &C, Expr **args, unsigned nexpr,
@@ -3203,3 +3315,24 @@ BlockDeclRefExpr::BlockDeclRefExpr(VarDecl *d, QualType t, ExprValueKind VK,
   ExprBits.ValueDependent = ValueDependent;
   ExprBits.InstantiationDependent = InstantiationDependent;
 }
+
+
+AtomicExpr::AtomicExpr(SourceLocation BLoc, Expr **args, unsigned nexpr,
+                       QualType t, AtomicOp op, SourceLocation RP)
+  : Expr(AtomicExprClass, t, VK_RValue, OK_Ordinary,
+         false, false, false, false),
+    NumSubExprs(nexpr), BuiltinLoc(BLoc), RParenLoc(RP), Op(op)
+{
+  for (unsigned i = 0; i < nexpr; i++) {
+    if (args[i]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (args[i]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (args[i]->isInstantiationDependent())
+      ExprBits.InstantiationDependent = true;
+    if (args[i]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
+    SubExprs[i] = args[i];
+  }
+}
diff --git a/contrib/llvm/tools/clang/lib/AST/ExprCXX.cpp b/contrib/llvm/tools/clang/lib/AST/ExprCXX.cpp
index f92afffb5851..ad5ec8b8331a 100644
--- a/contrib/llvm/tools/clang/lib/AST/ExprCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ExprCXX.cpp
@@ -49,6 +49,7 @@ CXXNewExpr::CXXNewExpr(ASTContext &C, bool globalNew, FunctionDecl *operatorNew,
                        SourceRange TypeIdParens, Expr *arraySize,
                        CXXConstructorDecl *constructor, bool initializer,
                        Expr **constructorArgs, unsigned numConsArgs,
+                       bool HadMultipleCandidates,
                        FunctionDecl *operatorDelete,
                        bool usualArrayDeleteWantsSize, QualType ty,
                        TypeSourceInfo *AllocatedTypeInfo,
@@ -61,6 +62,7 @@ CXXNewExpr::CXXNewExpr(ASTContext &C, bool globalNew, FunctionDecl *operatorNew,
          ty->containsUnexpandedParameterPack()),
     GlobalNew(globalNew), Initializer(initializer),
     UsualArrayDeleteWantsSize(usualArrayDeleteWantsSize),
+    HadMultipleCandidates(HadMultipleCandidates),
     SubExprs(0), OperatorNew(operatorNew),
     OperatorDelete(operatorDelete), Constructor(constructor),
     AllocatedTypeInfo(AllocatedTypeInfo), TypeIdParens(TypeIdParens),
@@ -202,7 +204,7 @@ UnresolvedLookupExpr::Create(ASTContext &C,
                              UnresolvedSetIterator End) 
 {
   void *Mem = C.Allocate(sizeof(UnresolvedLookupExpr) + 
-                         ExplicitTemplateArgumentList::sizeFor(Args));
+                         ASTTemplateArgumentListInfo::sizeFor(Args));
   return new (Mem) UnresolvedLookupExpr(C, NamingClass, QualifierLoc, NameInfo,
                                         ADL, /*Overload*/ true, &Args,
                                         Begin, End, /*StdIsAssociated=*/false);
@@ -213,7 +215,7 @@ UnresolvedLookupExpr::CreateEmpty(ASTContext &C, bool HasExplicitTemplateArgs,
                                   unsigned NumTemplateArgs) {
   std::size_t size = sizeof(UnresolvedLookupExpr);
   if (HasExplicitTemplateArgs)
-    size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
+    size += ASTTemplateArgumentListInfo::sizeFor(NumTemplateArgs);
 
   void *Mem = C.Allocate(size, llvm::alignOf<UnresolvedLookupExpr>());
   UnresolvedLookupExpr *E = new (Mem) UnresolvedLookupExpr(EmptyShell());
@@ -332,7 +334,7 @@ DependentScopeDeclRefExpr::DependentScopeDeclRefExpr(QualType T,
     bool ContainsUnexpandedParameterPack
       = ExprBits.ContainsUnexpandedParameterPack;
 
-    reinterpret_cast<ExplicitTemplateArgumentList*>(this+1)
+    reinterpret_cast<ASTTemplateArgumentListInfo*>(this+1)
       ->initializeFrom(*Args, Dependent, InstantiationDependent,
                        ContainsUnexpandedParameterPack);
     
@@ -347,7 +349,7 @@ DependentScopeDeclRefExpr::Create(ASTContext &C,
                                   const TemplateArgumentListInfo *Args) {
   std::size_t size = sizeof(DependentScopeDeclRefExpr);
   if (Args)
-    size += ExplicitTemplateArgumentList::sizeFor(*Args);
+    size += ASTTemplateArgumentListInfo::sizeFor(*Args);
   void *Mem = C.Allocate(size);
   return new (Mem) DependentScopeDeclRefExpr(C.DependentTy, QualifierLoc, 
                                              NameInfo, Args);
@@ -359,7 +361,7 @@ DependentScopeDeclRefExpr::CreateEmpty(ASTContext &C,
                                        unsigned NumTemplateArgs) {
   std::size_t size = sizeof(DependentScopeDeclRefExpr);
   if (HasExplicitTemplateArgs)
-    size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
+    size += ASTTemplateArgumentListInfo::sizeFor(NumTemplateArgs);
   void *Mem = C.Allocate(size);
   DependentScopeDeclRefExpr *E 
     = new (Mem) DependentScopeDeclRefExpr(QualType(), NestedNameSpecifierLoc(),
@@ -628,11 +630,13 @@ CXXTemporaryObjectExpr::CXXTemporaryObjectExpr(ASTContext &C,
                                                Expr **Args,
                                                unsigned NumArgs,
                                                SourceRange parenRange,
+                                               bool HadMultipleCandidates,
                                                bool ZeroInitialization)
   : CXXConstructExpr(C, CXXTemporaryObjectExprClass, 
                      Type->getType().getNonReferenceType(), 
                      Type->getTypeLoc().getBeginLoc(),
-                     Cons, false, Args, NumArgs, ZeroInitialization,
+                     Cons, false, Args, NumArgs,
+                     HadMultipleCandidates, ZeroInitialization,
                      CXXConstructExpr::CK_Complete, parenRange),
     Type(Type) {
 }
@@ -646,11 +650,13 @@ CXXConstructExpr *CXXConstructExpr::Create(ASTContext &C, QualType T,
                                            SourceLocation Loc,
                                            CXXConstructorDecl *D, bool Elidable,
                                            Expr **Args, unsigned NumArgs,
+                                           bool HadMultipleCandidates,
                                            bool ZeroInitialization,
                                            ConstructionKind ConstructKind,
                                            SourceRange ParenRange) {
   return new (C) CXXConstructExpr(C, CXXConstructExprClass, T, Loc, D, 
-                                  Elidable, Args, NumArgs, ZeroInitialization,
+                                  Elidable, Args, NumArgs,
+                                  HadMultipleCandidates, ZeroInitialization,
                                   ConstructKind, ParenRange);
 }
 
@@ -658,16 +664,18 @@ CXXConstructExpr::CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
                                    SourceLocation Loc,
                                    CXXConstructorDecl *D, bool elidable,
                                    Expr **args, unsigned numargs,
-                                   bool ZeroInitialization, 
+                                   bool HadMultipleCandidates,
+                                   bool ZeroInitialization,
                                    ConstructionKind ConstructKind,
                                    SourceRange ParenRange)
   : Expr(SC, T, VK_RValue, OK_Ordinary,
          T->isDependentType(), T->isDependentType(),
          T->isInstantiationDependentType(),
          T->containsUnexpandedParameterPack()),
-    Constructor(D), Loc(Loc), ParenRange(ParenRange), Elidable(elidable),
-    ZeroInitialization(ZeroInitialization), ConstructKind(ConstructKind),
-    Args(0), NumArgs(numargs) 
+    Constructor(D), Loc(Loc), ParenRange(ParenRange),  NumArgs(numargs),
+    Elidable(elidable), HadMultipleCandidates(HadMultipleCandidates),
+    ZeroInitialization(ZeroInitialization),
+    ConstructKind(ConstructKind), Args(0)
 {
   if (NumArgs) {
     Args = new (C) Stmt*[NumArgs];
@@ -838,7 +846,7 @@ CXXDependentScopeMemberExpr::Create(ASTContext &C,
 
   std::size_t size = sizeof(CXXDependentScopeMemberExpr);
   if (TemplateArgs)
-    size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
+    size += ASTTemplateArgumentListInfo::sizeFor(*TemplateArgs);
 
   void *Mem = C.Allocate(size, llvm::alignOf<CXXDependentScopeMemberExpr>());
   return new (Mem) CXXDependentScopeMemberExpr(C, Base, BaseType,
@@ -859,7 +867,7 @@ CXXDependentScopeMemberExpr::CreateEmpty(ASTContext &C,
                                                DeclarationNameInfo());
 
   std::size_t size = sizeof(CXXDependentScopeMemberExpr) +
-                     ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
+                     ASTTemplateArgumentListInfo::sizeFor(NumTemplateArgs);
   void *Mem = C.Allocate(size, llvm::alignOf<CXXDependentScopeMemberExpr>());
   CXXDependentScopeMemberExpr *E
     =  new (Mem) CXXDependentScopeMemberExpr(C, 0, QualType(),
@@ -947,7 +955,7 @@ UnresolvedMemberExpr::Create(ASTContext &C,
                              UnresolvedSetIterator End) {
   std::size_t size = sizeof(UnresolvedMemberExpr);
   if (TemplateArgs)
-    size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
+    size += ASTTemplateArgumentListInfo::sizeFor(*TemplateArgs);
 
   void *Mem = C.Allocate(size, llvm::alignOf<UnresolvedMemberExpr>());
   return new (Mem) UnresolvedMemberExpr(C, 
@@ -961,7 +969,7 @@ UnresolvedMemberExpr::CreateEmpty(ASTContext &C, bool HasExplicitTemplateArgs,
                                   unsigned NumTemplateArgs) {
   std::size_t size = sizeof(UnresolvedMemberExpr);
   if (HasExplicitTemplateArgs)
-    size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
+    size += ASTTemplateArgumentListInfo::sizeFor(NumTemplateArgs);
 
   void *Mem = C.Allocate(size, llvm::alignOf<UnresolvedMemberExpr>());
   UnresolvedMemberExpr *E = new (Mem) UnresolvedMemberExpr(EmptyShell());
diff --git a/contrib/llvm/tools/clang/lib/AST/ExprClassification.cpp b/contrib/llvm/tools/clang/lib/AST/ExprClassification.cpp
index e7888a6aa7b3..49c68213aa43 100644
--- a/contrib/llvm/tools/clang/lib/AST/ExprClassification.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ExprClassification.cpp
@@ -162,6 +162,7 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::SubstNonTypeTemplateParmPackExprClass:
   case Expr::AsTypeExprClass:
   case Expr::ObjCIndirectCopyRestoreExprClass:
+  case Expr::AtomicExprClass:
     return Cl::CL_PRValue;
 
     // Next come the complicated cases.
@@ -393,7 +394,7 @@ static Cl::Kinds ClassifyUnnamed(ASTContext &Ctx, QualType T) {
 
   // C++ [expr.call]p10: A function call is an lvalue if the result type is an
   //   lvalue reference type or an rvalue reference to function type, an xvalue
-  //   if the result type is an rvalue refernence to object type, and a prvalue
+  //   if the result type is an rvalue reference to object type, and a prvalue
   //   otherwise.
   if (T->isLValueReferenceType())
     return Cl::CL_LValue;
diff --git a/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp b/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp
index 786155af281d..df75bc8a735f 100644
--- a/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp
@@ -59,6 +59,8 @@ namespace {
 
     EvalInfo(const ASTContext &ctx, Expr::EvalResult &evalresult)
       : Ctx(ctx), EvalResult(evalresult) {}
+
+    const LangOptions &getLangOpts() { return Ctx.getLangOptions(); }
   };
 
   struct ComplexValue {
@@ -378,18 +380,22 @@ class ExprEvaluatorBase
   RetTy DerivedError(const Expr *E) {
     return static_cast<Derived*>(this)->Error(E);
   }
+  RetTy DerivedValueInitialization(const Expr *E) {
+    return static_cast<Derived*>(this)->ValueInitialization(E);
+  }
 
 protected:
   EvalInfo &Info;
   typedef ConstStmtVisitor<Derived, RetTy> StmtVisitorTy;
   typedef ExprEvaluatorBase ExprEvaluatorBaseTy;
 
+  RetTy ValueInitialization(const Expr *E) { return DerivedError(E); }
+
 public:
   ExprEvaluatorBase(EvalInfo &Info) : Info(Info) {}
 
   RetTy VisitStmt(const Stmt *) {
-    assert(0 && "Expression evaluator should not be called on stmts");
-    return DerivedError(0);
+    llvm_unreachable("Expression evaluator should not be called on stmts");
   }
   RetTy VisitExpr(const Expr *E) {
     return DerivedError(E);
@@ -436,6 +442,23 @@ class ExprEvaluatorBase
                                  : DerivedError(E));
     return DerivedSuccess(*value, E);
   }
+
+  RetTy VisitInitListExpr(const InitListExpr *E) {
+    if (Info.getLangOpts().CPlusPlus0x) {
+      if (E->getNumInits() == 0)
+        return DerivedValueInitialization(E);
+      if (E->getNumInits() == 1)
+        return StmtVisitorTy::Visit(E->getInit(0));
+    }
+    return DerivedError(E);
+  }
+  RetTy VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
+    return DerivedValueInitialization(E);
+  }
+  RetTy VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *E) {
+    return DerivedValueInitialization(E);
+  }
+
 };
 
 }
@@ -447,6 +470,7 @@ namespace {
 class LValueExprEvaluator
   : public ExprEvaluatorBase<LValueExprEvaluator, bool> {
   LValue &Result;
+  const Decl *PrevDecl;
 
   bool Success(const Expr *E) {
     Result.Base = E;
@@ -456,7 +480,7 @@ class LValueExprEvaluator
 public:
 
   LValueExprEvaluator(EvalInfo &info, LValue &Result) :
-    ExprEvaluatorBaseTy(info), Result(Result) {}
+    ExprEvaluatorBaseTy(info), Result(Result), PrevDecl(0) {}
 
   bool Success(const APValue &V, const Expr *E) {
     Result.setFrom(V);
@@ -481,9 +505,13 @@ class LValueExprEvaluator
       return false;
 
     case CK_NoOp:
+    case CK_LValueBitCast:
       return Visit(E->getSubExpr());
+
+    // FIXME: Support CK_DerivedToBase and friends.
     }
   }
+
   // FIXME: Missing: __real__, __imag__
 
 };
@@ -501,10 +529,16 @@ bool LValueExprEvaluator::VisitDeclRefExpr(const DeclRefExpr *E) {
       return Success(E);
     // Reference parameters can refer to anything even if they have an
     // "initializer" in the form of a default argument.
-    if (!isa<ParmVarDecl>(VD))
+    if (!isa<ParmVarDecl>(VD)) {
       // FIXME: Check whether VD might be overridden!
+
+      // Check for recursive initializers of references.
+      if (PrevDecl == VD)
+        return Error(E);
+      PrevDecl = VD;
       if (const Expr *Init = VD->getAnyInitializer())
         return Visit(Init);
+    }
   }
 
   return ExprEvaluatorBaseTy::VisitDeclRefExpr(E);
@@ -585,6 +619,9 @@ class PointerExprEvaluator
   bool Error(const Stmt *S) {
     return false;
   }
+  bool ValueInitialization(const Expr *E) {
+    return Success((Expr*)0);
+  }
 
   bool VisitBinaryOperator(const BinaryOperator *E);
   bool VisitCastExpr(const CastExpr* E);
@@ -599,10 +636,8 @@ class PointerExprEvaluator
       return Success(E);
     return false;
   }
-  bool VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E)
-      { return Success((Expr*)0); }
   bool VisitCXXNullPtrLiteralExpr(const CXXNullPtrLiteralExpr *E)
-      { return Success((Expr*)0); }
+      { return ValueInitialization(E); }
 
   // FIXME: Missing: @protocol, @selector
 };
@@ -667,7 +702,8 @@ bool PointerExprEvaluator::VisitCastExpr(const CastExpr* E) {
 
   case CK_NoOp:
   case CK_BitCast:
-  case CK_AnyPointerToObjCPointerCast:
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
     return Visit(SubExpr);
 
@@ -761,11 +797,11 @@ namespace {
 
     APValue Success(const APValue &V, const Expr *E) { return V; }
     APValue Error(const Expr *E) { return APValue(); }
+    APValue ValueInitialization(const Expr *E)
+      { return GetZeroVector(E->getType()); }
 
     APValue VisitUnaryReal(const UnaryOperator *E)
       { return Visit(E->getSubExpr()); }
-    APValue VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E)
-      { return GetZeroVector(E->getType()); }
     APValue VisitCastExpr(const CastExpr* E);
     APValue VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
     APValue VisitInitListExpr(const InitListExpr *E);
@@ -812,7 +848,7 @@ APValue VectorExprEvaluator::VisitCastExpr(const CastExpr* E) {
     }
 
     // Splat and create vector APValue.
-    llvm::SmallVector<APValue, 4> Elts(NElts, Result);
+    SmallVector<APValue, 4> Elts(NElts, Result);
     return APValue(&Elts[0], Elts.size());
   }
   case CK_BitCast: {
@@ -829,7 +865,7 @@ APValue VectorExprEvaluator::VisitCastExpr(const CastExpr* E) {
     assert((EltTy->isIntegerType() || EltTy->isRealFloatingType()) &&
            "Vectors must be composed of ints or floats");
 
-    llvm::SmallVector<APValue, 4> Elts;
+    SmallVector<APValue, 4> Elts;
     for (unsigned i = 0; i != NElts; ++i) {
       APSInt Tmp = Init.extOrTrunc(EltWidth);
 
@@ -862,7 +898,7 @@ VectorExprEvaluator::VisitInitListExpr(const InitListExpr *E) {
   unsigned NumElements = VT->getNumElements();
 
   QualType EltTy = VT->getElementType();
-  llvm::SmallVector<APValue, 4> Elements;
+  SmallVector<APValue, 4> Elements;
 
   // If a vector is initialized with a single element, that value
   // becomes every element of the vector, not just the first.
@@ -926,7 +962,7 @@ VectorExprEvaluator::GetZeroVector(QualType T) {
     ZeroElement =
         APValue(APFloat::getZero(Info.Ctx.getFloatTypeSemantics(EltTy)));
 
-  llvm::SmallVector<APValue, 4> Elements(VT->getNumElements(), ZeroElement);
+  SmallVector<APValue, 4> Elements(VT->getNumElements(), ZeroElement);
   return APValue(&Elements[0], Elements.size());
 }
 
@@ -999,6 +1035,8 @@ class IntExprEvaluator
     return Error(E->getLocStart(), diag::note_invalid_subexpr_in_ice, E);
   }
 
+  bool ValueInitialization(const Expr *E) { return Success(0, E); }
+
   //===--------------------------------------------------------------------===//
   //                            Visitor Methods
   //===--------------------------------------------------------------------===//
@@ -1039,16 +1077,9 @@ class IntExprEvaluator
     return Success(E->getValue(), E);
   }
 
+  // Note, GNU defines __null as an integer, not a pointer.
   bool VisitGNUNullExpr(const GNUNullExpr *E) {
-    return Success(0, E);
-  }
-
-  bool VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *E) {
-    return Success(0, E);
-  }
-
-  bool VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
-    return Success(0, E);
+    return ValueInitialization(E);
   }
 
   bool VisitUnaryTypeTraitExpr(const UnaryTypeTraitExpr *E) {
@@ -1072,7 +1103,7 @@ class IntExprEvaluator
 
   bool VisitCXXNoexceptExpr(const CXXNoexceptExpr *E);
   bool VisitSizeOfPackExpr(const SizeOfPackExpr *E);
-    
+
 private:
   CharUnits GetAlignOfExpr(const Expr *E);
   CharUnits GetAlignOfType(QualType T);
@@ -1210,7 +1241,7 @@ static int EvaluateBuiltinClassifyType(const CallExpr *E) {
   else if (ArgTy->isUnionType())
     return union_type_class;
   else  // FIXME: offset_type_class, method_type_class, & lang_type_class?
-    assert(0 && "CallExpr::isBuiltinClassifyType(): unimplemented type");
+    llvm_unreachable("CallExpr::isBuiltinClassifyType(): unimplemented type");
   return -1;
 }
 
@@ -1267,7 +1298,7 @@ bool IntExprEvaluator::VisitCallExpr(const CallExpr *E) {
     // If evaluating the argument has side-effects we can't determine
     // the size of the object and lower it to unknown now.
     if (E->getArg(0)->HasSideEffects(Info.Ctx)) {
-      if (E->getArg(1)->EvaluateAsInt(Info.Ctx).getZExtValue() <= 1)
+      if (E->getArg(1)->EvaluateKnownConstInt(Info.Ctx).getZExtValue() <= 1)
         return Success(-1ULL, E);
       return Success(0, E);
     }
@@ -1284,8 +1315,8 @@ bool IntExprEvaluator::VisitCallExpr(const CallExpr *E) {
     return Success(E->getArg(0)->isEvaluatable(Info.Ctx), E);
       
   case Builtin::BI__builtin_eh_return_data_regno: {
-    int Operand = E->getArg(0)->EvaluateAsInt(Info.Ctx).getZExtValue();
-    Operand = Info.Ctx.Target.getEHDataRegisterNumber(Operand);
+    int Operand = E->getArg(0)->EvaluateKnownConstInt(Info.Ctx).getZExtValue();
+    Operand = Info.Ctx.getTargetInfo().getEHDataRegisterNumber(Operand);
     return Success(Operand, E);
   }
 
@@ -1300,9 +1331,9 @@ bool IntExprEvaluator::VisitCallExpr(const CallExpr *E) {
                = dyn_cast<StringLiteral>(E->getArg(0)->IgnoreParenImpCasts())) {
       // The string literal may have embedded null characters. Find the first
       // one and truncate there.
-      llvm::StringRef Str = S->getString();
-      llvm::StringRef::size_type Pos = Str.find(0);
-      if (Pos != llvm::StringRef::npos)
+      StringRef Str = S->getString();
+      StringRef::size_type Pos = Str.find(0);
+      if (Pos != StringRef::npos)
         Str = Str.substr(0, Pos);
       
       return Success(Str.size(), E);
@@ -1419,7 +1450,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
 
     switch (E->getOpcode()) {
     default:
-      assert(0 && "Invalid binary operator!");
+      llvm_unreachable("Invalid binary operator!");
     case BO_LT:
       return Success(CR == APFloat::cmpLessThan, E);
     case BO_GT:
@@ -1801,7 +1832,8 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_VectorSplat:
   case CK_IntegralToFloating:
   case CK_FloatingCast:
-  case CK_AnyPointerToObjCPointerCast:
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_ObjCObjectLValueCast:
   case CK_FloatingRealToComplex:
@@ -1818,9 +1850,10 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_GetObjCProperty:
   case CK_LValueBitCast:
   case CK_UserDefinedConversion:
-  case CK_ObjCProduceObject:
-  case CK_ObjCConsumeObject:
-  case CK_ObjCReclaimReturnedObject:
+  case CK_ARCProduceObject:
+  case CK_ARCConsumeObject:
+  case CK_ARCReclaimReturnedObject:
+  case CK_ARCExtendBlockObject:
     return false;
 
   case CK_LValueToRValue:
@@ -1943,13 +1976,17 @@ class FloatExprEvaluator
     return false;
   }
 
+  bool ValueInitialization(const Expr *E) {
+    Result = APFloat::getZero(Info.Ctx.getFloatTypeSemantics(E->getType()));
+    return true;
+  }
+
   bool VisitCallExpr(const CallExpr *E);
 
   bool VisitUnaryOperator(const UnaryOperator *E);
   bool VisitBinaryOperator(const BinaryOperator *E);
   bool VisitFloatingLiteral(const FloatingLiteral *E);
   bool VisitCastExpr(const CastExpr *E);
-  bool VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *E);
 
   bool VisitUnaryReal(const UnaryOperator *E);
   bool VisitUnaryImag(const UnaryOperator *E);
@@ -2219,11 +2256,6 @@ bool FloatExprEvaluator::VisitCastExpr(const CastExpr *E) {
   return false;
 }
 
-bool FloatExprEvaluator::VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *E) {
-  Result = APFloat::getZero(Info.Ctx.getFloatTypeSemantics(E->getType()));
-  return true;
-}
-
 //===----------------------------------------------------------------------===//
 // Complex Evaluation (for float and integer)
 //===----------------------------------------------------------------------===//
@@ -2255,7 +2287,7 @@ class ComplexExprEvaluator
 
   bool VisitBinaryOperator(const BinaryOperator *E);
   bool VisitUnaryOperator(const UnaryOperator *E);
-  // FIXME Missing: ImplicitValueInitExpr
+  // FIXME Missing: ImplicitValueInitExpr, InitListExpr
 };
 } // end anonymous namespace
 
@@ -2318,16 +2350,18 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_FloatingToIntegral:
   case CK_FloatingToBoolean:
   case CK_FloatingCast:
-  case CK_AnyPointerToObjCPointerCast:
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_ObjCObjectLValueCast:
   case CK_FloatingComplexToReal:
   case CK_FloatingComplexToBoolean:
   case CK_IntegralComplexToReal:
   case CK_IntegralComplexToBoolean:
-  case CK_ObjCProduceObject:
-  case CK_ObjCConsumeObject:
-  case CK_ObjCReclaimReturnedObject:
+  case CK_ARCProduceObject:
+  case CK_ARCConsumeObject:
+  case CK_ARCReclaimReturnedObject:
+  case CK_ARCExtendBlockObject:
     llvm_unreachable("invalid cast kind for complex value");
 
   case CK_LValueToRValue:
@@ -2633,6 +2667,13 @@ bool Expr::EvaluateAsBooleanCondition(bool &Result,
   return HandleConversionToBool(this, Result, Info);
 }
 
+bool Expr::EvaluateAsInt(APSInt &Result, const ASTContext &Ctx) const {
+  EvalResult Scratch;
+  EvalInfo Info(Ctx, Scratch);
+
+  return EvaluateInteger(this, Result, Info) && !Scratch.HasSideEffects;
+}
+
 bool Expr::EvaluateAsLValue(EvalResult &Result, const ASTContext &Ctx) const {
   EvalInfo Info(Ctx, Result);
 
@@ -2671,7 +2712,7 @@ bool Expr::HasSideEffects(const ASTContext &Ctx) const {
   return HasSideEffect(Info).Visit(this);
 }
 
-APSInt Expr::EvaluateAsInt(const ASTContext &Ctx) const {
+APSInt Expr::EvaluateKnownConstInt(const ASTContext &Ctx) const {
   EvalResult EvalResult;
   bool Result = Evaluate(EvalResult, Ctx);
   (void)Result;
@@ -2755,7 +2796,6 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::CompoundAssignOperatorClass:
   case Expr::CompoundLiteralExprClass:
   case Expr::ExtVectorElementExprClass:
-  case Expr::InitListExprClass:
   case Expr::DesignatedInitExprClass:
   case Expr::ImplicitValueInitExprClass:
   case Expr::ParenListExprClass:
@@ -2800,6 +2840,18 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::AsTypeExprClass:
   case Expr::ObjCIndirectCopyRestoreExprClass:
   case Expr::MaterializeTemporaryExprClass:
+  case Expr::AtomicExprClass:
+    return ICEDiag(2, E->getLocStart());
+
+  case Expr::InitListExprClass:
+    if (Ctx.getLangOptions().CPlusPlus0x) {
+      const InitListExpr *ILE = cast<InitListExpr>(E);
+      if (ILE->getNumInits() == 0)
+        return NoDiag();
+      if (ILE->getNumInits() == 1)
+        return CheckICE(ILE->getInit(0), Ctx);
+      // Fall through for more than 1 expression.
+    }
     return ICEDiag(2, E->getLocStart());
 
   case Expr::SizeOfPackExprClass:
@@ -2963,11 +3015,11 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
         // Evaluate gives an error for undefined Div/Rem, so make sure
         // we don't evaluate one.
         if (LHSResult.Val == 0 && RHSResult.Val == 0) {
-          llvm::APSInt REval = Exp->getRHS()->EvaluateAsInt(Ctx);
+          llvm::APSInt REval = Exp->getRHS()->EvaluateKnownConstInt(Ctx);
           if (REval == 0)
             return ICEDiag(1, E->getLocStart());
           if (REval.isSigned() && REval.isAllOnesValue()) {
-            llvm::APSInt LEval = Exp->getLHS()->EvaluateAsInt(Ctx);
+            llvm::APSInt LEval = Exp->getLHS()->EvaluateKnownConstInt(Ctx);
             if (LEval.isMinSignedValue())
               return ICEDiag(1, E->getLocStart());
           }
@@ -2998,11 +3050,11 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
       //   evaluated are not considered.
       if (Ctx.getLangOptions().CPlusPlus0x && LHSResult.Val == 0) {
         if (Exp->getOpcode() == BO_LAnd && 
-            Exp->getLHS()->EvaluateAsInt(Ctx) == 0)
+            Exp->getLHS()->EvaluateKnownConstInt(Ctx) == 0)
           return LHSResult;
 
         if (Exp->getOpcode() == BO_LOr &&
-            Exp->getLHS()->EvaluateAsInt(Ctx) != 0)
+            Exp->getLHS()->EvaluateKnownConstInt(Ctx) != 0)
           return LHSResult;
       }
 
@@ -3012,7 +3064,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
         // to actually check the condition to see whether the side
         // with the comma is evaluated.
         if ((Exp->getOpcode() == BO_LAnd) !=
-            (Exp->getLHS()->EvaluateAsInt(Ctx) == 0))
+            (Exp->getLHS()->EvaluateKnownConstInt(Ctx) == 0))
           return RHSResult;
         return NoDiag();
       }
@@ -3031,11 +3083,17 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::CXXConstCastExprClass: 
   case Expr::ObjCBridgedCastExprClass: {
     const Expr *SubExpr = cast<CastExpr>(E)->getSubExpr();
-    if (SubExpr->getType()->isIntegralOrEnumerationType())
+    switch (cast<CastExpr>(E)->getCastKind()) {
+    case CK_LValueToRValue:
+    case CK_NoOp:
+    case CK_IntegralToBoolean:
+    case CK_IntegralCast:
       return CheckICE(SubExpr, Ctx);
-    if (isa<FloatingLiteral>(SubExpr->IgnoreParens()))
-      return NoDiag();
-    return ICEDiag(2, E->getLocStart());
+    default:
+      if (isa<FloatingLiteral>(SubExpr->IgnoreParens()))
+        return NoDiag();
+      return ICEDiag(2, E->getLocStart());
+    }
   }
   case Expr::BinaryConditionalOperatorClass: {
     const BinaryConditionalOperator *Exp = cast<BinaryConditionalOperator>(E);
@@ -3045,7 +3103,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
     if (FalseResult.Val == 2) return FalseResult;
     if (CommonResult.Val == 1) return CommonResult;
     if (FalseResult.Val == 1 &&
-        Exp->getCommon()->EvaluateAsInt(Ctx) == 0) return NoDiag();
+        Exp->getCommon()->EvaluateKnownConstInt(Ctx) == 0) return NoDiag();
     return FalseResult;
   }
   case Expr::ConditionalOperatorClass: {
@@ -3072,7 +3130,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
     //   subexpressions of [...] conditional (5.16) operations that
     //   are not evaluated are not considered
     bool TrueBranch = Ctx.getLangOptions().CPlusPlus0x
-      ? Exp->getCond()->EvaluateAsInt(Ctx) != 0
+      ? Exp->getCond()->EvaluateKnownConstInt(Ctx) != 0
       : false;
     ICEDiag TrueResult = NoDiag();
     if (!Ctx.getLangOptions().CPlusPlus0x || TrueBranch)
@@ -3092,7 +3150,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
     // Rare case where the diagnostics depend on which side is evaluated
     // Note that if we get here, CondResult is 0, and at least one of
     // TrueResult and FalseResult is non-zero.
-    if (Exp->getCond()->EvaluateAsInt(Ctx) == 0) {
+    if (Exp->getCond()->EvaluateKnownConstInt(Ctx) == 0) {
       return FalseResult;
     }
     return TrueResult;
diff --git a/contrib/llvm/tools/clang/lib/AST/ExternalASTSource.cpp b/contrib/llvm/tools/clang/lib/AST/ExternalASTSource.cpp
index b96d65a729ca..fd616dbc9d2f 100644
--- a/contrib/llvm/tools/clang/lib/AST/ExternalASTSource.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ExternalASTSource.cpp
@@ -49,12 +49,10 @@ ExternalASTSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
   return DeclContext::lookup_result();
 }
 
-void ExternalASTSource::MaterializeVisibleDecls(const DeclContext *DC) { }
-
 ExternalLoadResult 
 ExternalASTSource::FindExternalLexicalDecls(const DeclContext *DC,
                                             bool (*isKindWeWant)(Decl::Kind),
-                                         llvm::SmallVectorImpl<Decl*> &Result) {
+                                         SmallVectorImpl<Decl*> &Result) {
   return ELR_AlreadyLoaded;
 }
 
diff --git a/contrib/llvm/tools/clang/lib/AST/InheritViz.cpp b/contrib/llvm/tools/clang/lib/AST/InheritViz.cpp
index c47a9dadbadd..b70520f44dc5 100644
--- a/contrib/llvm/tools/clang/lib/AST/InheritViz.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/InheritViz.cpp
@@ -33,12 +33,12 @@ namespace clang {
 /// vs. non-virtual bases.
 class InheritanceHierarchyWriter {
   ASTContext& Context;
-  llvm::raw_ostream &Out;
+  raw_ostream &Out;
   std::map<QualType, int, QualTypeOrdering> DirectBaseCount;
   std::set<QualType, QualTypeOrdering> KnownVirtualBases;
 
 public:
-  InheritanceHierarchyWriter(ASTContext& Context, llvm::raw_ostream& Out)
+  InheritanceHierarchyWriter(ASTContext& Context, raw_ostream& Out)
     : Context(Context), Out(Out) { }
 
   void WriteGraph(QualType Type) {
@@ -55,7 +55,7 @@ class InheritanceHierarchyWriter {
   /// WriteNodeReference - Write out a reference to the given node,
   /// using a unique identifier for each direct base and for the
   /// (only) virtual base.
-  llvm::raw_ostream& WriteNodeReference(QualType Type, bool FromVirtual);
+  raw_ostream& WriteNodeReference(QualType Type, bool FromVirtual);
 };
 
 void InheritanceHierarchyWriter::WriteNode(QualType Type, bool FromVirtual) {
@@ -120,7 +120,7 @@ void InheritanceHierarchyWriter::WriteNode(QualType Type, bool FromVirtual) {
 /// WriteNodeReference - Write out a reference to the given node,
 /// using a unique identifier for each direct base and for the
 /// (only) virtual base.
-llvm::raw_ostream&
+raw_ostream&
 InheritanceHierarchyWriter::WriteNodeReference(QualType Type,
                                                bool FromVirtual) {
   QualType CanonType = Context.getCanonicalType(Type);
diff --git a/contrib/llvm/tools/clang/lib/AST/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/AST/ItaniumCXXABI.cpp
index 30aece3ee73b..0027dbf1915b 100644
--- a/contrib/llvm/tools/clang/lib/AST/ItaniumCXXABI.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ItaniumCXXABI.cpp
@@ -53,7 +53,7 @@ class ItaniumCXXABI : public CXXABI {
 
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
     CharUnits PointerSize = 
-      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
+      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
     return Layout.getNonVirtualSize() == PointerSize;
   }
 };
diff --git a/contrib/llvm/tools/clang/lib/AST/ItaniumMangle.cpp b/contrib/llvm/tools/clang/lib/AST/ItaniumMangle.cpp
index ec9863b298e6..acedf70f29c5 100644
--- a/contrib/llvm/tools/clang/lib/AST/ItaniumMangle.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ItaniumMangle.cpp
@@ -73,7 +73,7 @@ class ItaniumMangleContext : public MangleContext {
   
 public:
   explicit ItaniumMangleContext(ASTContext &Context,
-                                Diagnostic &Diags)
+                                DiagnosticsEngine &Diags)
     : MangleContext(Context, Diags) { }
 
   uint64_t getAnonymousStructId(const TagDecl *TD) {
@@ -92,30 +92,30 @@ class ItaniumMangleContext : public MangleContext {
   /// @{
 
   bool shouldMangleDeclName(const NamedDecl *D);
-  void mangleName(const NamedDecl *D, llvm::raw_ostream &);
+  void mangleName(const NamedDecl *D, raw_ostream &);
   void mangleThunk(const CXXMethodDecl *MD,
                    const ThunkInfo &Thunk,
-                   llvm::raw_ostream &);
+                   raw_ostream &);
   void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
                           const ThisAdjustment &ThisAdjustment,
-                          llvm::raw_ostream &);
+                          raw_ostream &);
   void mangleReferenceTemporary(const VarDecl *D,
-                                llvm::raw_ostream &);
+                                raw_ostream &);
   void mangleCXXVTable(const CXXRecordDecl *RD,
-                       llvm::raw_ostream &);
+                       raw_ostream &);
   void mangleCXXVTT(const CXXRecordDecl *RD,
-                    llvm::raw_ostream &);
+                    raw_ostream &);
   void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
                            const CXXRecordDecl *Type,
-                           llvm::raw_ostream &);
-  void mangleCXXRTTI(QualType T, llvm::raw_ostream &);
-  void mangleCXXRTTIName(QualType T, llvm::raw_ostream &);
+                           raw_ostream &);
+  void mangleCXXRTTI(QualType T, raw_ostream &);
+  void mangleCXXRTTIName(QualType T, raw_ostream &);
   void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
-                     llvm::raw_ostream &);
+                     raw_ostream &);
   void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
-                     llvm::raw_ostream &);
+                     raw_ostream &);
 
-  void mangleItaniumGuardVariable(const VarDecl *D, llvm::raw_ostream &);
+  void mangleItaniumGuardVariable(const VarDecl *D, raw_ostream &);
 
   void mangleInitDiscriminator() {
     Discriminator = 0;
@@ -136,7 +136,7 @@ class ItaniumMangleContext : public MangleContext {
 /// CXXNameMangler - Manage the mangling of a single name.
 class CXXNameMangler {
   ItaniumMangleContext &Context;
-  llvm::raw_ostream &Out;
+  raw_ostream &Out;
 
   /// The "structor" is the top-level declaration being mangled, if
   /// that's not a template specialization; otherwise it's the pattern
@@ -191,7 +191,7 @@ class CXXNameMangler {
   ASTContext &getASTContext() const { return Context.getASTContext(); }
 
 public:
-  CXXNameMangler(ItaniumMangleContext &C, llvm::raw_ostream &Out_,
+  CXXNameMangler(ItaniumMangleContext &C, raw_ostream &Out_,
                  const NamedDecl *D = 0)
     : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(0),
       SeqID(0) {
@@ -199,11 +199,11 @@ class CXXNameMangler {
     assert(!D || (!isa<CXXDestructorDecl>(D) &&
                   !isa<CXXConstructorDecl>(D)));
   }
-  CXXNameMangler(ItaniumMangleContext &C, llvm::raw_ostream &Out_,
+  CXXNameMangler(ItaniumMangleContext &C, raw_ostream &Out_,
                  const CXXConstructorDecl *D, CXXCtorType Type)
     : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type),
       SeqID(0) { }
-  CXXNameMangler(ItaniumMangleContext &C, llvm::raw_ostream &Out_,
+  CXXNameMangler(ItaniumMangleContext &C, raw_ostream &Out_,
                  const CXXDestructorDecl *D, CXXDtorType Type)
     : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type),
       SeqID(0) { }
@@ -219,9 +219,9 @@ class CXXNameMangler {
     free(result);
   }
 #endif
-  llvm::raw_ostream &getStream() { return Out; }
+  raw_ostream &getStream() { return Out; }
 
-  void mangle(const NamedDecl *D, llvm::StringRef Prefix = "_Z");
+  void mangle(const NamedDecl *D, StringRef Prefix = "_Z");
   void mangleCallOffset(int64_t NonVirtual, int64_t Virtual);
   void mangleNumber(const llvm::APSInt &I);
   void mangleNumber(int64_t Number);
@@ -310,7 +310,7 @@ class CXXNameMangler {
   void mangleCXXCtorType(CXXCtorType T);
   void mangleCXXDtorType(CXXDtorType T);
 
-  void mangleTemplateArgs(const ExplicitTemplateArgumentList &TemplateArgs);
+  void mangleTemplateArgs(const ASTTemplateArgumentListInfo &TemplateArgs);
   void mangleTemplateArgs(TemplateName Template,
                           const TemplateArgument *TemplateArgs,
                           unsigned NumTemplateArgs);  
@@ -385,7 +385,7 @@ bool ItaniumMangleContext::shouldMangleDeclName(const NamedDecl *D) {
   return true;
 }
 
-void CXXNameMangler::mangle(const NamedDecl *D, llvm::StringRef Prefix) {
+void CXXNameMangler::mangle(const NamedDecl *D, StringRef Prefix) {
   // Any decl can be declared with __asm("foo") on it, and this takes precedence
   // over all other naming in the .o file.
   if (const AsmLabelAttr *ALA = D->getAttr<AsmLabelAttr>()) {
@@ -397,8 +397,8 @@ void CXXNameMangler::mangle(const NamedDecl *D, llvm::StringRef Prefix) {
     // llvm mangler on ELF is a nop, so we can just avoid adding the \01
     // marker.  We also avoid adding the marker if this is an alias for an
     // LLVM intrinsic.
-    llvm::StringRef UserLabelPrefix =
-      getASTContext().Target.getUserLabelPrefix();
+    StringRef UserLabelPrefix =
+      getASTContext().getTargetInfo().getUserLabelPrefix();
     if (!UserLabelPrefix.empty() && !ALA->getLabel().startswith("llvm."))
       Out << '\01';  // LLVM IR Marker for __asm("foo")
 
@@ -788,6 +788,7 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
     case Type::ObjCObject:
     case Type::ObjCInterface:
     case Type::ObjCObjectPointer:
+    case Type::Atomic:
       llvm_unreachable("type is illegal as a nested name specifier");
 
     case Type::SubstTemplateTypeParmPack:
@@ -1069,8 +1070,7 @@ void CXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
   case DeclarationName::ObjCZeroArgSelector:
   case DeclarationName::ObjCOneArgSelector:
   case DeclarationName::ObjCMultiArgSelector:
-    assert(false && "Can't mangle Objective-C selector names here!");
-    break;
+    llvm_unreachable("Can't mangle Objective-C selector names here!");
 
   case DeclarationName::CXXConstructorName:
     if (ND == Structor)
@@ -1124,8 +1124,7 @@ void CXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
     break;
 
   case DeclarationName::CXXUsingDirective:
-    assert(false && "Can't mangle a using directive name!");
-    break;
+    llvm_unreachable("Can't mangle a using directive name!");
   }
 }
 
@@ -1512,8 +1511,7 @@ CXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO, unsigned Arity) {
 
   case OO_None:
   case NUM_OVERLOADED_OPERATORS:
-    assert(false && "Not an overloaded operator");
-    break;
+    llvm_unreachable("Not an overloaded operator");
   }
 }
 
@@ -1538,7 +1536,7 @@ void CXXNameMangler::mangleQualifiers(Qualifiers Quals) {
     Out << 'U' << ASString.size() << ASString;
   }
   
-  llvm::StringRef LifetimeName;
+  StringRef LifetimeName;
   switch (Quals.getObjCLifetime()) {
   // Objective-C ARC Extension:
   //
@@ -1706,7 +1704,7 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
   // UNSUPPORTED:    ::= Dd # IEEE 754r decimal floating point (64 bits)
   // UNSUPPORTED:    ::= De # IEEE 754r decimal floating point (128 bits)
   // UNSUPPORTED:    ::= Df # IEEE 754r decimal floating point (32 bits)
-  // UNSUPPORTED:    ::= Dh # IEEE 754r half-precision floating point (16 bits)
+  //                 ::= Dh # IEEE 754r half-precision floating point (16 bits)
   //                 ::= Di # char32_t
   //                 ::= Ds # char16_t
   //                 ::= Dn # std::nullptr_t (i.e., decltype(nullptr))
@@ -1731,6 +1729,7 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
   case BuiltinType::Long: Out << 'l'; break;
   case BuiltinType::LongLong: Out << 'x'; break;
   case BuiltinType::Int128: Out << 'n'; break;
+  case BuiltinType::Half: Out << "Dh"; break;
   case BuiltinType::Float: Out << 'f'; break;
   case BuiltinType::Double: Out << 'd'; break;
   case BuiltinType::LongDouble: Out << 'e'; break;
@@ -2114,6 +2113,13 @@ void CXXNameMangler::mangleType(const AutoType *T) {
     mangleType(D);
 }
 
+void CXXNameMangler::mangleType(const AtomicType *T) {
+  // <type> ::= U <source-name> <type>	# vendor extended type qualifier
+  // (Until there's a standardized mangling...)
+  Out << "U7_Atomic";
+  mangleType(T->getValueType());
+}
+
 void CXXNameMangler::mangleIntegerLiteral(QualType T,
                                           const llvm::APSInt &Value) {
   //  <expr-primary> ::= L <type> <value number> E # integer literal
@@ -2250,10 +2256,11 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   case Expr::CXXNoexceptExprClass:
   case Expr::CUDAKernelCallExprClass:
   case Expr::AsTypeExprClass:
+  case Expr::AtomicExprClass:
   {
     // As bad as this diagnostic is, it's better than crashing.
-    Diagnostic &Diags = Context.getDiags();
-    unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
+    DiagnosticsEngine &Diags = Context.getDiags();
+    unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                      "cannot yet mangle expression type %0");
     Diags.Report(E->getExprLoc(), DiagID)
       << E->getStmtClassName() << E->getSourceRange();
@@ -2262,9 +2269,9 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
 
   // Even gcc-4.5 doesn't mangle this.
   case Expr::BinaryConditionalOperatorClass: {
-    Diagnostic &Diags = Context.getDiags();
+    DiagnosticsEngine &Diags = Context.getDiags();
     unsigned DiagID =
-      Diags.getCustomDiagID(Diagnostic::Error,
+      Diags.getCustomDiagID(DiagnosticsEngine::Error,
                 "?: operator with omitted middle operand cannot be mangled");
     Diags.Report(E->getExprLoc(), DiagID)
       << E->getStmtClassName() << E->getSourceRange();
@@ -2411,7 +2418,8 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
       QualType T = (ImplicitlyConvertedToType.isNull() || 
                     !ImplicitlyConvertedToType->isIntegerType())? SAE->getType()
                                                     : ImplicitlyConvertedToType;
-      mangleIntegerLiteral(T, SAE->EvaluateAsInt(Context.getASTContext()));
+      llvm::APSInt V = SAE->EvaluateKnownConstInt(Context.getASTContext());
+      mangleIntegerLiteral(T, V);
       break;
     }
     
@@ -2423,8 +2431,8 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
       Out << 'a';
       break;
     case UETT_VecStep:
-      Diagnostic &Diags = Context.getDiags();
-      unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
+      DiagnosticsEngine &Diags = Context.getDiags();
+      unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                      "cannot yet mangle vec_step expression");
       Diags.Report(DiagID);
       return;
@@ -2526,7 +2534,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   case Expr::ObjCBridgedCastExprClass: {
     // Mangle ownership casts as a vendor extended operator __bridge, 
     // __bridge_transfer, or __bridge_retain.
-    llvm::StringRef Kind = cast<ObjCBridgedCastExpr>(E)->getBridgeKindName();
+    StringRef Kind = cast<ObjCBridgedCastExpr>(E)->getBridgeKindName();
     Out << "v1U" << Kind.size() << Kind;
   }
   // Fall through to mangle the cast itself.
@@ -2817,7 +2825,7 @@ void CXXNameMangler::mangleCXXDtorType(CXXDtorType T) {
 }
 
 void CXXNameMangler::mangleTemplateArgs(
-                          const ExplicitTemplateArgumentList &TemplateArgs) {
+                          const ASTTemplateArgumentListInfo &TemplateArgs) {
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   for (unsigned i = 0, e = TemplateArgs.NumTemplateArgs; i != e; ++i)
@@ -3021,7 +3029,7 @@ bool CXXNameMangler::mangleSubstitution(uintptr_t Ptr) {
     }
 
     Out << 'S'
-        << llvm::StringRef(BufferPtr, llvm::array_endof(Buffer)-BufferPtr)
+        << StringRef(BufferPtr, llvm::array_endof(Buffer)-BufferPtr)
         << '_';
   }
 
@@ -3196,7 +3204,7 @@ void CXXNameMangler::addSubstitution(uintptr_t Ptr) {
 /// emit the identifier of the declaration (\c D->getIdentifier()) as its
 /// name.
 void ItaniumMangleContext::mangleName(const NamedDecl *D,
-                                      llvm::raw_ostream &Out) {
+                                      raw_ostream &Out) {
   assert((isa<FunctionDecl>(D) || isa<VarDecl>(D)) &&
           "Invalid mangleName() call, argument is not a variable or function!");
   assert(!isa<CXXConstructorDecl>(D) && !isa<CXXDestructorDecl>(D) &&
@@ -3212,21 +3220,21 @@ void ItaniumMangleContext::mangleName(const NamedDecl *D,
 
 void ItaniumMangleContext::mangleCXXCtor(const CXXConstructorDecl *D,
                                          CXXCtorType Type,
-                                         llvm::raw_ostream &Out) {
+                                         raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out, D, Type);
   Mangler.mangle(D);
 }
 
 void ItaniumMangleContext::mangleCXXDtor(const CXXDestructorDecl *D,
                                          CXXDtorType Type,
-                                         llvm::raw_ostream &Out) {
+                                         raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out, D, Type);
   Mangler.mangle(D);
 }
 
 void ItaniumMangleContext::mangleThunk(const CXXMethodDecl *MD,
                                        const ThunkInfo &Thunk,
-                                       llvm::raw_ostream &Out) {
+                                       raw_ostream &Out) {
   //  <special-name> ::= T <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
   //  <special-name> ::= Tc <call-offset> <call-offset> <base encoding>
@@ -3256,7 +3264,7 @@ void
 ItaniumMangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD,
                                          CXXDtorType Type,
                                          const ThisAdjustment &ThisAdjustment,
-                                         llvm::raw_ostream &Out) {
+                                         raw_ostream &Out) {
   //  <special-name> ::= T <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
   CXXNameMangler Mangler(*this, Out, DD, Type);
@@ -3272,7 +3280,7 @@ ItaniumMangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD,
 /// mangleGuardVariable - Returns the mangled name for a guard variable
 /// for the passed in VarDecl.
 void ItaniumMangleContext::mangleItaniumGuardVariable(const VarDecl *D,
-                                                      llvm::raw_ostream &Out) {
+                                                      raw_ostream &Out) {
   //  <special-name> ::= GV <object name>       # Guard variable for one-time
   //                                            # initialization
   CXXNameMangler Mangler(*this, Out);
@@ -3281,7 +3289,7 @@ void ItaniumMangleContext::mangleItaniumGuardVariable(const VarDecl *D,
 }
 
 void ItaniumMangleContext::mangleReferenceTemporary(const VarDecl *D,
-                                                    llvm::raw_ostream &Out) {
+                                                    raw_ostream &Out) {
   // We match the GCC mangling here.
   //  <special-name> ::= GR <object name>
   CXXNameMangler Mangler(*this, Out);
@@ -3290,7 +3298,7 @@ void ItaniumMangleContext::mangleReferenceTemporary(const VarDecl *D,
 }
 
 void ItaniumMangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
-                                           llvm::raw_ostream &Out) {
+                                           raw_ostream &Out) {
   // <special-name> ::= TV <type>  # virtual table
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTV";
@@ -3298,7 +3306,7 @@ void ItaniumMangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
 }
 
 void ItaniumMangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
-                                        llvm::raw_ostream &Out) {
+                                        raw_ostream &Out) {
   // <special-name> ::= TT <type>  # VTT structure
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTT";
@@ -3308,7 +3316,7 @@ void ItaniumMangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
 void ItaniumMangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD,
                                                int64_t Offset,
                                                const CXXRecordDecl *Type,
-                                               llvm::raw_ostream &Out) {
+                                               raw_ostream &Out) {
   // <special-name> ::= TC <type> <offset number> _ <base type>
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTC";
@@ -3319,7 +3327,7 @@ void ItaniumMangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD,
 }
 
 void ItaniumMangleContext::mangleCXXRTTI(QualType Ty,
-                                         llvm::raw_ostream &Out) {
+                                         raw_ostream &Out) {
   // <special-name> ::= TI <type>  # typeinfo structure
   assert(!Ty.hasQualifiers() && "RTTI info cannot have top-level qualifiers");
   CXXNameMangler Mangler(*this, Out);
@@ -3328,7 +3336,7 @@ void ItaniumMangleContext::mangleCXXRTTI(QualType Ty,
 }
 
 void ItaniumMangleContext::mangleCXXRTTIName(QualType Ty,
-                                             llvm::raw_ostream &Out) {
+                                             raw_ostream &Out) {
   // <special-name> ::= TS <type>  # typeinfo name (null terminated byte string)
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTS";
@@ -3336,6 +3344,6 @@ void ItaniumMangleContext::mangleCXXRTTIName(QualType Ty,
 }
 
 MangleContext *clang::createItaniumMangleContext(ASTContext &Context,
-                                                 Diagnostic &Diags) {
+                                                 DiagnosticsEngine &Diags) {
   return new ItaniumMangleContext(Context, Diags);
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/Mangle.cpp b/contrib/llvm/tools/clang/lib/AST/Mangle.cpp
index c3f3b11caccf..5cb8f47519ec 100644
--- a/contrib/llvm/tools/clang/lib/AST/Mangle.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/Mangle.cpp
@@ -37,9 +37,9 @@ using namespace clang;
 namespace {
 
 static void mangleFunctionBlock(MangleContext &Context,
-                                llvm::StringRef Outer,
+                                StringRef Outer,
                                 const BlockDecl *BD,
-                                llvm::raw_ostream &Out) {
+                                raw_ostream &Out) {
   Out << "__" << Outer << "_block_invoke_" << Context.getBlockId(BD, true);
 }
 
@@ -60,13 +60,13 @@ static void checkMangleDC(const DeclContext *DC, const BlockDecl *BD) {
 }
 
 void MangleContext::mangleGlobalBlock(const BlockDecl *BD,
-                                      llvm::raw_ostream &Out) {
+                                      raw_ostream &Out) {
   Out << "__block_global_" << getBlockId(BD, false);
 }
 
 void MangleContext::mangleCtorBlock(const CXXConstructorDecl *CD,
                                     CXXCtorType CT, const BlockDecl *BD,
-                                    llvm::raw_ostream &ResStream) {
+                                    raw_ostream &ResStream) {
   checkMangleDC(CD, BD);
   llvm::SmallString<64> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
@@ -77,7 +77,7 @@ void MangleContext::mangleCtorBlock(const CXXConstructorDecl *CD,
 
 void MangleContext::mangleDtorBlock(const CXXDestructorDecl *DD,
                                     CXXDtorType DT, const BlockDecl *BD,
-                                    llvm::raw_ostream &ResStream) {
+                                    raw_ostream &ResStream) {
   checkMangleDC(DD, BD);
   llvm::SmallString<64> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
@@ -87,7 +87,7 @@ void MangleContext::mangleDtorBlock(const CXXDestructorDecl *DD,
 }
 
 void MangleContext::mangleBlock(const DeclContext *DC, const BlockDecl *BD,
-                                llvm::raw_ostream &Out) {
+                                raw_ostream &Out) {
   assert(!isa<CXXConstructorDecl>(DC) && !isa<CXXDestructorDecl>(DC));
   checkMangleDC(DC, BD);
 
@@ -113,7 +113,7 @@ void MangleContext::mangleBlock(const DeclContext *DC, const BlockDecl *BD,
 }
 
 void MangleContext::mangleObjCMethodName(const ObjCMethodDecl *MD,
-                                         llvm::raw_ostream &Out) {
+                                         raw_ostream &Out) {
   llvm::SmallString<64> Name;
   llvm::raw_svector_ostream OS(Name);
   
@@ -129,7 +129,7 @@ void MangleContext::mangleObjCMethodName(const ObjCMethodDecl *MD,
 }
 
 void MangleContext::mangleBlock(const BlockDecl *BD,
-                                llvm::raw_ostream &Out) {
+                                raw_ostream &Out) {
   const DeclContext *DC = BD->getDeclContext();
   while (isa<BlockDecl>(DC) || isa<EnumDecl>(DC))
     DC = DC->getParent();
diff --git a/contrib/llvm/tools/clang/lib/AST/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/AST/MicrosoftCXXABI.cpp
index 206f6dd0c9eb..f33d6fe1f563 100644
--- a/contrib/llvm/tools/clang/lib/AST/MicrosoftCXXABI.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/MicrosoftCXXABI.cpp
@@ -30,7 +30,7 @@ class MicrosoftCXXABI : public CXXABI {
   unsigned getMemberPointerSize(const MemberPointerType *MPT) const;
 
   CallingConv getDefaultMethodCallConv() const {
-    if (Context.Target.getTriple().getArch() == llvm::Triple::x86)
+    if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
       return CC_X86ThisCall;
     else
       return CC_C;
@@ -45,7 +45,7 @@ class MicrosoftCXXABI : public CXXABI {
     
     // In the Microsoft ABI, classes can have one or two vtable pointers.
     CharUnits PointerSize = 
-      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
+      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
     return Layout.getNonVirtualSize() == PointerSize ||
       Layout.getNonVirtualSize() == PointerSize * 2;
   }    
diff --git a/contrib/llvm/tools/clang/lib/AST/MicrosoftMangle.cpp b/contrib/llvm/tools/clang/lib/AST/MicrosoftMangle.cpp
index 4f920f9aaca7..1515db49fe3c 100644
--- a/contrib/llvm/tools/clang/lib/AST/MicrosoftMangle.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/MicrosoftMangle.cpp
@@ -29,15 +29,15 @@ namespace {
 /// Microsoft Visual C++ ABI.
 class MicrosoftCXXNameMangler {
   MangleContext &Context;
-  llvm::raw_ostream &Out;
+  raw_ostream &Out;
 
   ASTContext &getASTContext() const { return Context.getASTContext(); }
 
 public:
-  MicrosoftCXXNameMangler(MangleContext &C, llvm::raw_ostream &Out_)
+  MicrosoftCXXNameMangler(MangleContext &C, raw_ostream &Out_)
   : Context(C), Out(Out_) { }
 
-  void mangle(const NamedDecl *D, llvm::StringRef Prefix = "?");
+  void mangle(const NamedDecl *D, StringRef Prefix = "?");
   void mangleName(const NamedDecl *ND);
   void mangleFunctionEncoding(const FunctionDecl *FD);
   void mangleVariableEncoding(const VarDecl *VD);
@@ -78,30 +78,30 @@ class MicrosoftCXXNameMangler {
 class MicrosoftMangleContext : public MangleContext {
 public:
   MicrosoftMangleContext(ASTContext &Context,
-                         Diagnostic &Diags) : MangleContext(Context, Diags) { }
+                   DiagnosticsEngine &Diags) : MangleContext(Context, Diags) { }
   virtual bool shouldMangleDeclName(const NamedDecl *D);
-  virtual void mangleName(const NamedDecl *D, llvm::raw_ostream &Out);
+  virtual void mangleName(const NamedDecl *D, raw_ostream &Out);
   virtual void mangleThunk(const CXXMethodDecl *MD,
                            const ThunkInfo &Thunk,
-                           llvm::raw_ostream &);
+                           raw_ostream &);
   virtual void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
                                   const ThisAdjustment &ThisAdjustment,
-                                  llvm::raw_ostream &);
+                                  raw_ostream &);
   virtual void mangleCXXVTable(const CXXRecordDecl *RD,
-                               llvm::raw_ostream &);
+                               raw_ostream &);
   virtual void mangleCXXVTT(const CXXRecordDecl *RD,
-                            llvm::raw_ostream &);
+                            raw_ostream &);
   virtual void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
                                    const CXXRecordDecl *Type,
-                                   llvm::raw_ostream &);
-  virtual void mangleCXXRTTI(QualType T, llvm::raw_ostream &);
-  virtual void mangleCXXRTTIName(QualType T, llvm::raw_ostream &);
+                                   raw_ostream &);
+  virtual void mangleCXXRTTI(QualType T, raw_ostream &);
+  virtual void mangleCXXRTTIName(QualType T, raw_ostream &);
   virtual void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
-                             llvm::raw_ostream &);
+                             raw_ostream &);
   virtual void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
-                             llvm::raw_ostream &);
+                             raw_ostream &);
   virtual void mangleReferenceTemporary(const clang::VarDecl *,
-                                        llvm::raw_ostream &);
+                                        raw_ostream &);
 };
 
 }
@@ -154,7 +154,7 @@ bool MicrosoftMangleContext::shouldMangleDeclName(const NamedDecl *D) {
 }
 
 void MicrosoftCXXNameMangler::mangle(const NamedDecl *D,
-                                     llvm::StringRef Prefix) {
+                                     StringRef Prefix) {
   // MSVC doesn't mangle C++ names the same way it mangles extern "C" names.
   // Therefore it's really important that we don't decorate the
   // name with leading underscores or leading/trailing at signs. So, emit a
@@ -332,16 +332,13 @@ MicrosoftCXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
     case DeclarationName::ObjCZeroArgSelector:
     case DeclarationName::ObjCOneArgSelector:
     case DeclarationName::ObjCMultiArgSelector:
-      assert(false && "Can't mangle Objective-C selector names here!");
-      break;
+      llvm_unreachable("Can't mangle Objective-C selector names here!");
       
     case DeclarationName::CXXConstructorName:
-      assert(false && "Can't mangle constructors yet!");
-      break;
+      llvm_unreachable("Can't mangle constructors yet!");
       
     case DeclarationName::CXXDestructorName:
-      assert(false && "Can't mangle destructors yet!");
-      break;
+      llvm_unreachable("Can't mangle destructors yet!");
       
     case DeclarationName::CXXConversionFunctionName:
       // <operator-name> ::= ?B # (cast)
@@ -355,12 +352,10 @@ MicrosoftCXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
       
     case DeclarationName::CXXLiteralOperatorName:
       // FIXME: Was this added in VS2010? Does MS even know how to mangle this?
-      assert(false && "Don't know how to mangle literal operators yet!");
-      break;
+      llvm_unreachable("Don't know how to mangle literal operators yet!");
       
     case DeclarationName::CXXUsingDirective:
-      assert(false && "Can't mangle a using directive name!");
-      break;
+      llvm_unreachable("Can't mangle a using directive name!");
   }
 }
 
@@ -513,13 +508,11 @@ void MicrosoftCXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO) {
   case OO_Array_Delete: Out << "?_V"; break;
     
   case OO_Conditional:
-    assert(false && "Don't know how to mangle ?:");
-    break;
+    llvm_unreachable("Don't know how to mangle ?:");
     
   case OO_None:
   case NUM_OVERLOADED_OPERATORS:
-    assert(false && "Not an overloaded operator");
-    break;
+    llvm_unreachable("Not an overloaded operator");
   }
 }
 
@@ -712,18 +705,17 @@ void MicrosoftCXXNameMangler::mangleType(const BuiltinType *T) {
   case BuiltinType::Dependent:
   case BuiltinType::UnknownAny:
   case BuiltinType::BoundMember:
-    assert(false &&
+    llvm_unreachable(
            "Overloaded and dependent types shouldn't get to name mangling");
-    break;
   case BuiltinType::ObjCId: Out << "PAUobjc_object@@"; break;
   case BuiltinType::ObjCClass: Out << "PAUobjc_class@@"; break;
   case BuiltinType::ObjCSel: Out << "PAUobjc_selector@@"; break;
 
   case BuiltinType::Char16:
   case BuiltinType::Char32:
+  case BuiltinType::Half:
   case BuiltinType::NullPtr:
-    assert(false && "Don't know how to mangle this type");
-    break;
+    llvm_unreachable("Don't know how to mangle this type");
   }
 }
 
@@ -869,7 +861,7 @@ void MicrosoftCXXNameMangler::mangleCallingConvention(const FunctionType *T,
     CC = IsInstMethod ? getASTContext().getDefaultMethodCallConv() : CC_C;
   switch (CC) {
     default:
-      assert(0 && "Unsupported CC for mangling");
+      llvm_unreachable("Unsupported CC for mangling");
     case CC_Default:
     case CC_C: Out << 'A'; break;
     case CC_X86Pascal: Out << 'C'; break;
@@ -890,7 +882,7 @@ void MicrosoftCXXNameMangler::mangleThrowSpecification(
 }
 
 void MicrosoftCXXNameMangler::mangleType(const UnresolvedUsingType *T) {
-  assert(false && "Don't know how to mangle UnresolvedUsingTypes yet!");
+  llvm_unreachable("Don't know how to mangle UnresolvedUsingTypes yet!");
 }
 
 // <type>        ::= <union-type> | <struct-type> | <class-type> | <enum-type>
@@ -954,7 +946,7 @@ void MicrosoftCXXNameMangler::mangleType(const IncompleteArrayType *T) {
   mangleType(static_cast<const ArrayType *>(T), false);
 }
 void MicrosoftCXXNameMangler::mangleExtraDimensions(QualType ElementTy) {
-  llvm::SmallVector<llvm::APInt, 3> Dimensions;
+  SmallVector<llvm::APInt, 3> Dimensions;
   for (;;) {
     if (ElementTy->isConstantArrayType()) {
       const ConstantArrayType *CAT =
@@ -962,10 +954,10 @@ void MicrosoftCXXNameMangler::mangleExtraDimensions(QualType ElementTy) {
       Dimensions.push_back(CAT->getSize());
       ElementTy = CAT->getElementType();
     } else if (ElementTy->isVariableArrayType()) {
-      assert(false && "Don't know how to mangle VLAs!");
+      llvm_unreachable("Don't know how to mangle VLAs!");
     } else if (ElementTy->isDependentSizedArrayType()) {
       // The dependent expression has to be folded into a constant (TODO).
-      assert(false && "Don't know how to mangle dependent-sized arrays!");
+      llvm_unreachable("Don't know how to mangle dependent-sized arrays!");
     } else if (ElementTy->isIncompleteArrayType()) continue;
     else break;
   }
@@ -999,12 +991,12 @@ void MicrosoftCXXNameMangler::mangleType(const MemberPointerType *T) {
 }
 
 void MicrosoftCXXNameMangler::mangleType(const TemplateTypeParmType *T) {
-  assert(false && "Don't know how to mangle TemplateTypeParmTypes yet!");
+  llvm_unreachable("Don't know how to mangle TemplateTypeParmTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(
                                        const SubstTemplateTypeParmPackType *T) {
-  assert(false && 
+  llvm_unreachable(
          "Don't know how to mangle SubstTemplateTypeParmPackTypes yet!");
 }
 
@@ -1045,21 +1037,22 @@ void MicrosoftCXXNameMangler::mangleType(const LValueReferenceType *T) {
 }
 
 void MicrosoftCXXNameMangler::mangleType(const RValueReferenceType *T) {
-  assert(false && "Don't know how to mangle RValueReferenceTypes yet!");
+  llvm_unreachable("Don't know how to mangle RValueReferenceTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const ComplexType *T) {
-  assert(false && "Don't know how to mangle ComplexTypes yet!");
+  llvm_unreachable("Don't know how to mangle ComplexTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const VectorType *T) {
-  assert(false && "Don't know how to mangle VectorTypes yet!");
+  llvm_unreachable("Don't know how to mangle VectorTypes yet!");
 }
 void MicrosoftCXXNameMangler::mangleType(const ExtVectorType *T) {
-  assert(false && "Don't know how to mangle ExtVectorTypes yet!");
+  llvm_unreachable("Don't know how to mangle ExtVectorTypes yet!");
 }
 void MicrosoftCXXNameMangler::mangleType(const DependentSizedExtVectorType *T) {
-  assert(false && "Don't know how to mangle DependentSizedExtVectorTypes yet!");
+  llvm_unreachable(
+                  "Don't know how to mangle DependentSizedExtVectorTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const ObjCInterfaceType *T) {
@@ -1080,49 +1073,53 @@ void MicrosoftCXXNameMangler::mangleType(const BlockPointerType *T) {
 }
 
 void MicrosoftCXXNameMangler::mangleType(const InjectedClassNameType *T) {
-  assert(false && "Don't know how to mangle InjectedClassNameTypes yet!");
+  llvm_unreachable("Don't know how to mangle InjectedClassNameTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const TemplateSpecializationType *T) {
-  assert(false && "Don't know how to mangle TemplateSpecializationTypes yet!");
+  llvm_unreachable("Don't know how to mangle TemplateSpecializationTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const DependentNameType *T) {
-  assert(false && "Don't know how to mangle DependentNameTypes yet!");
+  llvm_unreachable("Don't know how to mangle DependentNameTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(
                                  const DependentTemplateSpecializationType *T) {
-  assert(false &&
+  llvm_unreachable(
          "Don't know how to mangle DependentTemplateSpecializationTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const PackExpansionType *T) {
-  assert(false && "Don't know how to mangle PackExpansionTypes yet!");
+  llvm_unreachable("Don't know how to mangle PackExpansionTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const TypeOfType *T) {
-  assert(false && "Don't know how to mangle TypeOfTypes yet!");
+  llvm_unreachable("Don't know how to mangle TypeOfTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const TypeOfExprType *T) {
-  assert(false && "Don't know how to mangle TypeOfExprTypes yet!");
+  llvm_unreachable("Don't know how to mangle TypeOfExprTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const DecltypeType *T) {
-  assert(false && "Don't know how to mangle DecltypeTypes yet!");
+  llvm_unreachable("Don't know how to mangle DecltypeTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const UnaryTransformType *T) {
-  assert(false && "Don't know how to mangle UnaryTransformationTypes yet!");
+  llvm_unreachable("Don't know how to mangle UnaryTransformationTypes yet!");
 }
 
 void MicrosoftCXXNameMangler::mangleType(const AutoType *T) {
-  assert(false && "Don't know how to mangle AutoTypes yet!");
+  llvm_unreachable("Don't know how to mangle AutoTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const AtomicType *T) {
+  llvm_unreachable("Don't know how to mangle AtomicTypes yet!");
 }
 
 void MicrosoftMangleContext::mangleName(const NamedDecl *D,
-                                        llvm::raw_ostream &Out) {
+                                        raw_ostream &Out) {
   assert((isa<FunctionDecl>(D) || isa<VarDecl>(D)) &&
          "Invalid mangleName() call, argument is not a variable or function!");
   assert(!isa<CXXConstructorDecl>(D) && !isa<CXXDestructorDecl>(D) &&
@@ -1137,53 +1134,53 @@ void MicrosoftMangleContext::mangleName(const NamedDecl *D,
 }
 void MicrosoftMangleContext::mangleThunk(const CXXMethodDecl *MD,
                                          const ThunkInfo &Thunk,
-                                         llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle thunks!");
+                                         raw_ostream &) {
+  llvm_unreachable("Can't yet mangle thunks!");
 }
 void MicrosoftMangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD,
                                                 CXXDtorType Type,
                                                 const ThisAdjustment &,
-                                                llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle destructor thunks!");
+                                                raw_ostream &) {
+  llvm_unreachable("Can't yet mangle destructor thunks!");
 }
 void MicrosoftMangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
-                                             llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle virtual tables!");
+                                             raw_ostream &) {
+  llvm_unreachable("Can't yet mangle virtual tables!");
 }
 void MicrosoftMangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
-                                          llvm::raw_ostream &) {
+                                          raw_ostream &) {
   llvm_unreachable("The MS C++ ABI does not have virtual table tables!");
 }
 void MicrosoftMangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD,
                                                  int64_t Offset,
                                                  const CXXRecordDecl *Type,
-                                                 llvm::raw_ostream &) {
+                                                 raw_ostream &) {
   llvm_unreachable("The MS C++ ABI does not have constructor vtables!");
 }
 void MicrosoftMangleContext::mangleCXXRTTI(QualType T,
-                                           llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle RTTI!");
+                                           raw_ostream &) {
+  llvm_unreachable("Can't yet mangle RTTI!");
 }
 void MicrosoftMangleContext::mangleCXXRTTIName(QualType T,
-                                               llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle RTTI names!");
+                                               raw_ostream &) {
+  llvm_unreachable("Can't yet mangle RTTI names!");
 }
 void MicrosoftMangleContext::mangleCXXCtor(const CXXConstructorDecl *D,
                                            CXXCtorType Type,
-                                           llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle constructors!");
+                                           raw_ostream &) {
+  llvm_unreachable("Can't yet mangle constructors!");
 }
 void MicrosoftMangleContext::mangleCXXDtor(const CXXDestructorDecl *D,
                                            CXXDtorType Type,
-                                           llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle destructors!");
+                                           raw_ostream &) {
+  llvm_unreachable("Can't yet mangle destructors!");
 }
 void MicrosoftMangleContext::mangleReferenceTemporary(const clang::VarDecl *,
-                                                      llvm::raw_ostream &) {
-  assert(false && "Can't yet mangle reference temporaries!");
+                                                      raw_ostream &) {
+  llvm_unreachable("Can't yet mangle reference temporaries!");
 }
 
 MangleContext *clang::createMicrosoftMangleContext(ASTContext &Context,
-                                                   Diagnostic &Diags) {
+                                                   DiagnosticsEngine &Diags) {
   return new MicrosoftMangleContext(Context, Diags);
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/NestedNameSpecifier.cpp b/contrib/llvm/tools/clang/lib/AST/NestedNameSpecifier.cpp
index f6d4f2513ce4..1ff2e7177b3b 100644
--- a/contrib/llvm/tools/clang/lib/AST/NestedNameSpecifier.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/NestedNameSpecifier.cpp
@@ -218,7 +218,7 @@ bool NestedNameSpecifier::containsUnexpandedParameterPack() const {
 /// \brief Print this nested name specifier to the given output
 /// stream.
 void
-NestedNameSpecifier::print(llvm::raw_ostream &OS,
+NestedNameSpecifier::print(raw_ostream &OS,
                            const PrintingPolicy &Policy) const {
   if (getPrefix())
     getPrefix()->print(OS, Policy);
@@ -569,7 +569,7 @@ void NestedNameSpecifierLocBuilder::MakeTrivial(ASTContext &Context,
   // Construct bogus (but well-formed) source information for the 
   // nested-name-specifier.
   BufferSize = 0;
-  llvm::SmallVector<NestedNameSpecifier *, 4> Stack;
+  SmallVector<NestedNameSpecifier *, 4> Stack;
   for (NestedNameSpecifier *NNS = Qualifier; NNS; NNS = NNS->getPrefix())
     Stack.push_back(NNS);
   while (!Stack.empty()) {
diff --git a/contrib/llvm/tools/clang/lib/AST/ParentMap.cpp b/contrib/llvm/tools/clang/lib/AST/ParentMap.cpp
index b7b2005e9f84..5eef83a81a8b 100644
--- a/contrib/llvm/tools/clang/lib/AST/ParentMap.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ParentMap.cpp
@@ -66,6 +66,14 @@ Stmt *ParentMap::getParentIgnoreParenCasts(Stmt *S) const {
   return S;  
 }
 
+Stmt *ParentMap::getParentIgnoreParenImpCasts(Stmt *S) const {
+  do {
+    S = getParent(S);
+  } while (S && isa<Expr>(S) && cast<Expr>(S)->IgnoreParenImpCasts() != S);
+
+  return S;
+}
+
 Stmt *ParentMap::getOuterParenParent(Stmt *S) const {
   Stmt *Paren = 0;
   while (isa<ParenExpr>(S)) {
diff --git a/contrib/llvm/tools/clang/lib/AST/RecordLayout.cpp b/contrib/llvm/tools/clang/lib/AST/RecordLayout.cpp
index 035c48fd0827..ccc591a28f32 100644
--- a/contrib/llvm/tools/clang/lib/AST/RecordLayout.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/RecordLayout.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/RecordLayout.h"
+#include "clang/Basic/TargetInfo.h"
 
 using namespace clang;
 
@@ -42,7 +43,7 @@ ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx, CharUnits size,
 // Constructor for C++ records.
 ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx,
                                  CharUnits size, CharUnits alignment,
-                                 CharUnits datasize,
+                                 CharUnits vbptroffset, CharUnits datasize,
                                  const uint64_t *fieldoffsets,
                                  unsigned fieldcount,
                                  CharUnits nonvirtualsize,
@@ -67,15 +68,20 @@ ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx,
   CXXInfo->SizeOfLargestEmptySubobject = SizeOfLargestEmptySubobject;
   CXXInfo->BaseOffsets = BaseOffsets;
   CXXInfo->VBaseOffsets = VBaseOffsets;
+  CXXInfo->VBPtrOffset = vbptroffset;
 
 #ifndef NDEBUG
     if (const CXXRecordDecl *PrimaryBase = getPrimaryBase()) {
-      if (isPrimaryBaseVirtual())
+      if (isPrimaryBaseVirtual()) {
+        // Microsoft ABI doesn't have primary virtual base
+        if (Ctx.getTargetInfo().getCXXABI() != CXXABI_Microsoft) {
         assert(getVBaseClassOffset(PrimaryBase).isZero() &&
                "Primary virtual base must be at offset 0!");
-      else
+        }
+      } else {
         assert(getBaseClassOffsetInBits(PrimaryBase) == 0 &&
                "Primary base must be at offset 0!");
+      }
     }
 #endif        
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/RecordLayoutBuilder.cpp b/contrib/llvm/tools/clang/lib/AST/RecordLayoutBuilder.cpp
index 5636a6f0f64a..bbd3fc016017 100644
--- a/contrib/llvm/tools/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -44,7 +44,7 @@ struct BaseSubobjectInfo {
   bool IsVirtual;
 
   /// Bases - Information about the base subobjects.
-  llvm::SmallVector<BaseSubobjectInfo*, 4> Bases;
+  SmallVector<BaseSubobjectInfo*, 4> Bases;
 
   /// PrimaryVirtualBaseInfo - Holds the base info for the primary virtual base
   /// of this base info (if one exists).
@@ -64,7 +64,7 @@ class EmptySubobjectMap {
   const CXXRecordDecl *Class;
 
   /// EmptyClassOffsets - A map from offsets to empty record decls.
-  typedef llvm::SmallVector<const CXXRecordDecl *, 1> ClassVectorTy;
+  typedef SmallVector<const CXXRecordDecl *, 1> ClassVectorTy;
   typedef llvm::DenseMap<CharUnits, ClassVectorTy> EmptyClassOffsetsMapTy;
   EmptyClassOffsetsMapTy EmptyClassOffsets;
   
@@ -556,7 +556,7 @@ class RecordLayoutBuilder {
   /// \brief The alignment if attribute packed is not used.
   CharUnits UnpackedAlignment;
 
-  llvm::SmallVector<uint64_t, 16> FieldOffsets;
+  SmallVector<uint64_t, 16> FieldOffsets;
 
   /// Packed - Whether the record is packed or not.
   unsigned Packed : 1;
@@ -592,6 +592,9 @@ class RecordLayoutBuilder {
   /// out is virtual.
   bool PrimaryBaseIsVirtual;
 
+  /// VBPtrOffset - Virtual base table offset. Only for MS layout.
+  CharUnits VBPtrOffset;
+
   typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> BaseOffsetsMapTy;
 
   /// Bases - base classes and their offsets in the record.
@@ -613,16 +616,17 @@ class RecordLayoutBuilder {
   llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBases;
 
   RecordLayoutBuilder(const ASTContext &Context, EmptySubobjectMap
-                      *EmptySubobjects)
+                      *EmptySubobjects, CharUnits Alignment)
     : Context(Context), EmptySubobjects(EmptySubobjects), Size(0), 
-      Alignment(CharUnits::One()), UnpackedAlignment(Alignment),
+      Alignment(Alignment), UnpackedAlignment(Alignment),
       Packed(false), IsUnion(false), 
       IsMac68kAlign(false), IsMsStruct(false),
       UnfilledBitsInLastByte(0), MaxFieldAlignment(CharUnits::Zero()), 
       DataSize(0), NonVirtualSize(CharUnits::Zero()), 
       NonVirtualAlignment(CharUnits::One()), 
       ZeroLengthBitfield(0), PrimaryBase(0), 
-      PrimaryBaseIsVirtual(false), FirstNearlyEmptyVBase(0) { }
+      PrimaryBaseIsVirtual(false), VBPtrOffset(CharUnits::fromQuantity(-1)),
+      FirstNearlyEmptyVBase(0) { }
 
   void Layout(const RecordDecl *D);
   void Layout(const CXXRecordDecl *D);
@@ -633,6 +637,8 @@ class RecordLayoutBuilder {
   void LayoutWideBitField(uint64_t FieldSize, uint64_t TypeSize,
                           bool FieldPacked, const FieldDecl *D);
   void LayoutBitField(const FieldDecl *D);
+  void MSLayoutVirtualBases(const CXXRecordDecl *RD);
+  void MSLayout(const CXXRecordDecl *RD);
 
   /// BaseSubobjectInfoAllocator - Allocator for BaseSubobjectInfo objects.
   llvm::SpecificBumpPtrAllocator<BaseSubobjectInfo> BaseSubobjectInfoAllocator;
@@ -663,7 +669,7 @@ class RecordLayoutBuilder {
 
   void SelectPrimaryVBase(const CXXRecordDecl *RD);
 
-  virtual CharUnits GetVirtualPointersSize(const CXXRecordDecl *RD) const;
+  CharUnits GetVirtualPointersSize(const CXXRecordDecl *RD) const;
 
   /// LayoutNonVirtualBases - Determines the primary base class (if any) and
   /// lays it out. Will then proceed to lay out all non-virtual base clasess.
@@ -713,6 +719,8 @@ class RecordLayoutBuilder {
   void setSize(CharUnits NewSize) { Size = Context.toBits(NewSize); }
   void setSize(uint64_t NewSize) { Size = NewSize; }
 
+  CharUnits getAligment() const { return Alignment; }
+
   CharUnits getDataSize() const { 
     assert(DataSize % Context.getCharWidth() == 0);
     return Context.toCharUnitsFromBits(DataSize); 
@@ -722,6 +730,11 @@ class RecordLayoutBuilder {
   void setDataSize(CharUnits NewSize) { DataSize = Context.toBits(NewSize); }
   void setDataSize(uint64_t NewSize) { DataSize = NewSize; }
 
+  bool HasVBPtr(const CXXRecordDecl *RD) const;
+  bool HasNewVirtualFunction(const CXXRecordDecl *RD) const;
+
+  /// Add vbptr or vfptr to layout.
+  void AddVPointer();
 
   RecordLayoutBuilder(const RecordLayoutBuilder&);   // DO NOT IMPLEMENT
   void operator=(const RecordLayoutBuilder&); // DO NOT IMPLEMENT
@@ -729,6 +742,8 @@ class RecordLayoutBuilder {
   static const CXXMethodDecl *ComputeKeyFunction(const CXXRecordDecl *RD);
 
   virtual ~RecordLayoutBuilder() { }
+
+  CharUnits GetVBPtrOffset() const { return VBPtrOffset; }
 };
 } // end anonymous namespace
 
@@ -765,7 +780,7 @@ RecordLayoutBuilder::SelectPrimaryVBase(const CXXRecordDecl *RD) {
 
 CharUnits
 RecordLayoutBuilder::GetVirtualPointersSize(const CXXRecordDecl *RD) const {
-  return Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
+  return Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
 }
 
 /// DeterminePrimaryBase - Determine the primary base of the given class.
@@ -825,7 +840,7 @@ void RecordLayoutBuilder::DeterminePrimaryBase(const CXXRecordDecl *RD) {
   setDataSize(getSize());
 
   CharUnits UnpackedBaseAlign = 
-    Context.toCharUnitsFromBits(Context.Target.getPointerAlign(0));
+    Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerAlign(0));
   CharUnits BaseAlign = (Packed) ? CharUnits::One() : UnpackedBaseAlign;
 
   // The maximum field alignment overrides base align.
@@ -1046,6 +1061,45 @@ RecordLayoutBuilder::AddPrimaryVirtualBaseOffsets(const BaseSubobjectInfo *Info,
   }
 }
 
+void RecordLayoutBuilder::AddVPointer() {
+  CharUnits PtrWidth = 
+    Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
+  setSize(getSize() + PtrWidth);
+  setDataSize(getSize());
+
+  if (Alignment > PtrWidth) {
+    setSize(getSize() + (Alignment - PtrWidth));
+    setDataSize(getSize());
+  }
+}
+
+bool 
+RecordLayoutBuilder::HasNewVirtualFunction(const CXXRecordDecl *RD) const {
+  for (CXXRecordDecl::method_iterator method = RD->method_begin();
+       method != RD->method_end();
+       ++method) {
+    if (method->isVirtual() &&
+      !method->size_overridden_methods()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool
+RecordLayoutBuilder::HasVBPtr(const CXXRecordDecl *RD) const {
+  if (!RD->getNumBases())
+    return false;
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    if (!I->isVirtual()) {
+      return false;
+    }
+  }
+  return true;
+}
+
 void
 RecordLayoutBuilder::LayoutVirtualBases(const CXXRecordDecl *RD,
                                         const CXXRecordDecl *MostDerivedClass) {
@@ -1157,6 +1211,11 @@ void RecordLayoutBuilder::InitializeLayout(const Decl *D) {
   
   IsMsStruct = D->hasAttr<MsStructAttr>();
 
+  // Honor the default struct packing maximum alignment flag.
+  if (unsigned DefaultMaxFieldAlignment = Context.getLangOptions().PackStruct) {
+    MaxFieldAlignment = CharUnits::fromQuantity(DefaultMaxFieldAlignment);
+  }
+
   // mac68k alignment supersedes maximum field alignment and attribute aligned,
   // and forces all structures to have 2-byte alignment. The IBM docs on it
   // allude to additional (more complicated) semantics, especially with regard
@@ -1184,6 +1243,11 @@ void RecordLayoutBuilder::Layout(const RecordDecl *D) {
 }
 
 void RecordLayoutBuilder::Layout(const CXXRecordDecl *RD) {
+  if (Context.getTargetInfo().getCXXABI() == CXXABI_Microsoft) {
+    MSLayout(RD);
+    return ;
+  }
+
   InitializeLayout(RD);
 
   // Lay out the vtable and the non-virtual bases.
@@ -1193,7 +1257,7 @@ void RecordLayoutBuilder::Layout(const CXXRecordDecl *RD) {
 
   NonVirtualSize = Context.toCharUnitsFromBits(
         llvm::RoundUpToAlignment(getSizeInBits(), 
-                                 Context.Target.getCharAlign()));
+                                 Context.getTargetInfo().getCharAlign()));
   NonVirtualAlignment = Alignment;
 
   // Lay out the virtual bases and add the primary virtual base offsets.
@@ -1242,10 +1306,9 @@ void RecordLayoutBuilder::Layout(const ObjCInterfaceDecl *D) {
   }
 
   InitializeLayout(D);
-  ObjCInterfaceDecl *OI = const_cast<ObjCInterfaceDecl*>(D);
   // Layout each ivar sequentially.
-  for (ObjCIvarDecl *IVD = OI->all_declared_ivar_begin(); 
-       IVD; IVD = IVD->getNextIvar())
+  for (const ObjCIvarDecl *IVD = D->all_declared_ivar_begin(); IVD;
+       IVD = IVD->getNextIvar())
     LayoutField(IVD);
 
   // Finally, round the size of the total struct up to the alignment of the
@@ -1271,8 +1334,8 @@ void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
         continue;
       // FIXME. streamline these conditions into a simple one.
       else if (Context.BitfieldFollowsBitfield(FD, LastFD) ||
-               Context.BitfieldFollowsNoneBitfield(FD, LastFD) ||
-               Context.NoneBitfieldFollowsBitfield(FD, LastFD)) {
+               Context.BitfieldFollowsNonBitfield(FD, LastFD) ||
+               Context.NonBitfieldFollowsBitfield(FD, LastFD)) {
         // 1) Adjacent bit fields are packed into the same 1-, 2-, or
         // 4-byte allocation unit if the integral types are the same
         // size and if the next bit field fits into the current
@@ -1299,14 +1362,14 @@ void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
         if (TypeSizeLastFD != TypeSize) {
           if (RemainingInAlignment &&
               LastFD && LastFD->isBitField() &&
-              LastFD->getBitWidth()->EvaluateAsInt(Context).getZExtValue()) {
+              LastFD->getBitWidthValue(Context)) {
             // If previous field was a bitfield with some remaining unfilled
             // bits, pad the field so current field starts on its type boundary.
             uint64_t FieldOffset = 
             getDataSizeInBits() - UnfilledBitsInLastByte;
             uint64_t NewSizeInBits = RemainingInAlignment + FieldOffset;
             setDataSize(llvm::RoundUpToAlignment(NewSizeInBits,
-                                                 Context.Target.getCharAlign()));
+                                                 Context.getTargetInfo().getCharAlign()));
             setSize(std::max(getSizeInBits(), getDataSizeInBits()));
             RemainingInAlignment = 0;
           }
@@ -1325,13 +1388,12 @@ void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
           uint64_t NewSizeInBits = 
             llvm::RoundUpToAlignment(UnpaddedFieldOffset, FieldAlign);
           setDataSize(llvm::RoundUpToAlignment(NewSizeInBits,
-                                               Context.Target.getCharAlign()));
+                                               Context.getTargetInfo().getCharAlign()));
           UnfilledBitsInLastByte = getDataSizeInBits() - NewSizeInBits;
           setSize(std::max(getSizeInBits(), getDataSizeInBits()));
         }
         if (FD->isBitField()) {
-          uint64_t FieldSize = 
-            FD->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
+          uint64_t FieldSize = FD->getBitWidthValue(Context);
           assert (FieldSize > 0 && "LayoutFields - ms_struct layout");
           if (RemainingInAlignment < FieldSize)
             RemainingInAlignment = TypeSize - FieldSize;
@@ -1340,8 +1402,7 @@ void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
         }
       }
       else if (FD->isBitField()) {
-        uint64_t FieldSize = 
-          FD->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
+        uint64_t FieldSize = FD->getBitWidthValue(Context);
         std::pair<uint64_t, unsigned> FieldInfo = 
           Context.getTypeInfo(FD->getType());
         uint64_t TypeSize = FieldInfo.first;
@@ -1349,18 +1410,23 @@ void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
       }
       LastFD = FD;
     }
+    else if (!Context.getTargetInfo().useBitFieldTypeAlignment() &&
+             Context.getTargetInfo().useZeroLengthBitfieldAlignment()) {             
+      FieldDecl *FD =  (*Field);
+      if (FD->isBitField() && FD->getBitWidthValue(Context) == 0)
+        ZeroLengthBitfield = FD;
+    }
     LayoutField(*Field);
   }
   if (IsMsStruct && RemainingInAlignment &&
-      LastFD && LastFD->isBitField() &&
-      LastFD->getBitWidth()->EvaluateAsInt(Context).getZExtValue()) {
+      LastFD && LastFD->isBitField() && LastFD->getBitWidthValue(Context)) {
     // If we ended a bitfield before the full length of the type then
     // pad the struct out to the full length of the last type.
     uint64_t FieldOffset = 
       getDataSizeInBits() - UnfilledBitsInLastByte;
     uint64_t NewSizeInBits = RemainingInAlignment + FieldOffset;
     setDataSize(llvm::RoundUpToAlignment(NewSizeInBits,
-                                         Context.Target.getCharAlign()));
+                                         Context.getTargetInfo().getCharAlign()));
     setSize(std::max(getSizeInBits(), getDataSizeInBits()));
   }
 }
@@ -1405,15 +1471,15 @@ void RecordLayoutBuilder::LayoutWideBitField(uint64_t FieldSize,
     setDataSize(std::max(getDataSizeInBits(), FieldSize));
     FieldOffset = 0;
   } else {
-    // The bitfield is allocated starting at the next offset aligned appropriately
-    // for T', with length n bits.
+    // The bitfield is allocated starting at the next offset aligned 
+    // appropriately for T', with length n bits.
     FieldOffset = llvm::RoundUpToAlignment(getDataSizeInBits(), 
                                            Context.toBits(TypeAlign));
 
     uint64_t NewSizeInBits = FieldOffset + FieldSize;
 
     setDataSize(llvm::RoundUpToAlignment(NewSizeInBits, 
-                                         Context.Target.getCharAlign()));
+                                         Context.getTargetInfo().getCharAlign()));
     UnfilledBitsInLastByte = getDataSizeInBits() - NewSizeInBits;
   }
 
@@ -1434,7 +1500,7 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   bool FieldPacked = Packed || D->hasAttr<PackedAttr>();
   uint64_t UnpaddedFieldOffset = getDataSizeInBits() - UnfilledBitsInLastByte;
   uint64_t FieldOffset = IsUnion ? 0 : UnpaddedFieldOffset;
-  uint64_t FieldSize = D->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
+  uint64_t FieldSize = D->getBitWidthValue(Context);
 
   std::pair<uint64_t, unsigned> FieldInfo = Context.getTypeInfo(D->getType());
   uint64_t TypeSize = FieldInfo.first;
@@ -1443,21 +1509,32 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   // This check is needed for 'long long' in -m32 mode.
   if (IsMsStruct && (TypeSize > FieldAlign))
     FieldAlign = TypeSize;
-  
+
   if (ZeroLengthBitfield) {
-    // If a zero-length bitfield is inserted after a bitfield,
-    // and the alignment of the zero-length bitfield is
-    // greater than the member that follows it, `bar', `bar' 
-    // will be aligned as the type of the zero-length bitfield.
-    if (ZeroLengthBitfield != D) {
-      std::pair<uint64_t, unsigned> FieldInfo = 
-        Context.getTypeInfo(ZeroLengthBitfield->getType());
-      unsigned ZeroLengthBitfieldAlignment = FieldInfo.second;
-      // Ignore alignment of subsequent zero-length bitfields.
-      if ((ZeroLengthBitfieldAlignment > FieldAlign) || (FieldSize == 0))
-        FieldAlign = ZeroLengthBitfieldAlignment;
-      if (FieldSize)
-        ZeroLengthBitfield = 0;
+    std::pair<uint64_t, unsigned> FieldInfo;
+    unsigned ZeroLengthBitfieldAlignment;
+    if (IsMsStruct) {
+      // If a zero-length bitfield is inserted after a bitfield,
+      // and the alignment of the zero-length bitfield is
+      // greater than the member that follows it, `bar', `bar' 
+      // will be aligned as the type of the zero-length bitfield.
+      if (ZeroLengthBitfield != D) {
+        FieldInfo = Context.getTypeInfo(ZeroLengthBitfield->getType());
+        ZeroLengthBitfieldAlignment = FieldInfo.second;
+        // Ignore alignment of subsequent zero-length bitfields.
+        if ((ZeroLengthBitfieldAlignment > FieldAlign) || (FieldSize == 0))
+          FieldAlign = ZeroLengthBitfieldAlignment;
+        if (FieldSize)
+          ZeroLengthBitfield = 0;
+      }
+    } else {
+      // The alignment of a zero-length bitfield affects the alignment
+      // of the next member.  The alignment is the max of the zero 
+      // length bitfield's alignment and a target specific fixed value.
+      unsigned ZeroLengthBitfieldBoundary =
+        Context.getTargetInfo().getZeroLengthBitfieldBoundary();
+      if (ZeroLengthBitfieldBoundary > FieldAlign)
+        FieldAlign = ZeroLengthBitfieldBoundary;
     }
   }
 
@@ -1470,10 +1547,11 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   // was unnecessary (-Wpacked).
   unsigned UnpackedFieldAlign = FieldAlign;
   uint64_t UnpackedFieldOffset = FieldOffset;
-  if (!Context.Target.useBitFieldTypeAlignment())
+  if (!Context.getTargetInfo().useBitFieldTypeAlignment() && !ZeroLengthBitfield)
     UnpackedFieldAlign = 1;
 
-  if (FieldPacked || !Context.Target.useBitFieldTypeAlignment())
+  if (FieldPacked || 
+      (!Context.getTargetInfo().useBitFieldTypeAlignment() && !ZeroLengthBitfield))
     FieldAlign = 1;
   FieldAlign = std::max(FieldAlign, D->getMaxAlignment());
   UnpackedFieldAlign = std::max(UnpackedFieldAlign, D->getMaxAlignment());
@@ -1494,10 +1572,14 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
     UnpackedFieldOffset = llvm::RoundUpToAlignment(UnpackedFieldOffset,
                                                    UnpackedFieldAlign);
 
-  // Padding members don't affect overall alignment.
-  if (!D->getIdentifier())
+  // Padding members don't affect overall alignment, unless zero length bitfield
+  // alignment is enabled.
+  if (!D->getIdentifier() && !Context.getTargetInfo().useZeroLengthBitfieldAlignment())
     FieldAlign = UnpackedFieldAlign = 1;
 
+  if (!IsMsStruct)
+    ZeroLengthBitfield = 0;
+
   // Place this field at the current location.
   FieldOffsets.push_back(FieldOffset);
 
@@ -1512,7 +1594,7 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
     uint64_t NewSizeInBits = FieldOffset + FieldSize;
 
     setDataSize(llvm::RoundUpToAlignment(NewSizeInBits, 
-                                         Context.Target.getCharAlign()));
+                                         Context.getTargetInfo().getCharAlign()));
     UnfilledBitsInLastByte = getDataSizeInBits() - NewSizeInBits;
   }
 
@@ -1552,25 +1634,36 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
   } else if (const ReferenceType *RT = D->getType()->getAs<ReferenceType>()) {
     unsigned AS = RT->getPointeeType().getAddressSpace();
     FieldSize = 
-      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(AS));
+      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(AS));
     FieldAlign = 
-      Context.toCharUnitsFromBits(Context.Target.getPointerAlign(AS));
+      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerAlign(AS));
   } else {
     std::pair<CharUnits, CharUnits> FieldInfo = 
       Context.getTypeInfoInChars(D->getType());
     FieldSize = FieldInfo.first;
     FieldAlign = FieldInfo.second;
-    
+
     if (ZeroLengthBitfield) {
-      // If a zero-length bitfield is inserted after a bitfield,
-      // and the alignment of the zero-length bitfield is
-      // greater than the member that follows it, `bar', `bar' 
-      // will be aligned as the type of the zero-length bitfield.
-      std::pair<CharUnits, CharUnits> FieldInfo = 
-        Context.getTypeInfoInChars(ZeroLengthBitfield->getType());
-      CharUnits ZeroLengthBitfieldAlignment = FieldInfo.second;
-      if (ZeroLengthBitfieldAlignment > FieldAlign)
-        FieldAlign = ZeroLengthBitfieldAlignment;
+      CharUnits ZeroLengthBitfieldBoundary = 
+        Context.toCharUnitsFromBits(
+          Context.getTargetInfo().getZeroLengthBitfieldBoundary());
+      if (ZeroLengthBitfieldBoundary == CharUnits::Zero()) {
+        // If a zero-length bitfield is inserted after a bitfield,
+        // and the alignment of the zero-length bitfield is
+        // greater than the member that follows it, `bar', `bar' 
+        // will be aligned as the type of the zero-length bitfield.
+        std::pair<CharUnits, CharUnits> FieldInfo = 
+          Context.getTypeInfoInChars(ZeroLengthBitfield->getType());
+        CharUnits ZeroLengthBitfieldAlignment = FieldInfo.second;        
+        if (ZeroLengthBitfieldAlignment > FieldAlign)
+          FieldAlign = ZeroLengthBitfieldAlignment;
+      } else if (ZeroLengthBitfieldBoundary > FieldAlign) {
+        // Align 'bar' based on a fixed alignment specified by the target.
+        assert(Context.getTargetInfo().useZeroLengthBitfieldAlignment() &&
+               "ZeroLengthBitfieldBoundary should only be used in conjunction"
+               " with useZeroLengthBitfieldAlignment.");
+        FieldAlign = ZeroLengthBitfieldBoundary;
+      }
       ZeroLengthBitfield = 0;
     }
 
@@ -1641,6 +1734,104 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
   UpdateAlignment(FieldAlign, UnpackedFieldAlign);
 }
 
+void RecordLayoutBuilder::MSLayoutVirtualBases(const CXXRecordDecl *RD) {
+
+  if (!RD->getNumVBases())
+    return;
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->vbases_begin(),
+       E = RD->vbases_end(); I != E; ++I) {
+
+    const CXXRecordDecl* BaseDecl = I->getType()->getAsCXXRecordDecl();
+    const BaseSubobjectInfo* BaseInfo = VirtualBaseInfo.lookup(BaseDecl);
+
+    assert(BaseInfo && "Did not find virtual base info!");
+    
+    LayoutVirtualBase(BaseInfo);
+  }
+}
+
+void RecordLayoutBuilder::MSLayout(const CXXRecordDecl *RD) {
+
+  bool IsVBPtrAddedToLayout = false;
+
+  InitializeLayout(RD);
+
+  if (HasVBPtr(RD)) {
+    // If all bases are virtual and the class declares a new virtual function,
+    // MSVC builds a vfptr.
+    if (HasNewVirtualFunction(RD)) {
+      AddVPointer();
+    }
+
+    VBPtrOffset = getSize();
+    AddVPointer();
+    IsVBPtrAddedToLayout = true;
+
+    ComputeBaseSubobjectInfo(RD);
+  } else {
+    LayoutNonVirtualBases(RD);
+  }
+
+  if (RD->getNumVBases() &&
+      !IsVBPtrAddedToLayout) {
+    // Add vbptr.
+    VBPtrOffset = getSize();
+    AddVPointer();
+  }
+
+  LayoutFields(RD);
+
+  NonVirtualSize = Context.toCharUnitsFromBits(
+                           llvm::RoundUpToAlignment(getSizeInBits(),
+                           Context.getTargetInfo().getCharAlign()));
+  NonVirtualAlignment = Alignment;
+
+  if (NonVirtualSize != NonVirtualSize.RoundUpToAlignment(Alignment)) {
+    CharUnits AlignMember = 
+      NonVirtualSize.RoundUpToAlignment(Alignment) - NonVirtualSize;
+
+    setSize(getSize() + AlignMember);
+    setDataSize(getSize());
+
+    NonVirtualSize = Context.toCharUnitsFromBits(
+                             llvm::RoundUpToAlignment(getSizeInBits(),
+                             Context.getTargetInfo().getCharAlign()));
+  }
+
+  MSLayoutVirtualBases(RD);
+
+  VisitedVirtualBases.clear();
+
+  // Finally, round the size of the total struct up to the alignment of the
+  // struct itself.
+  if (!RD->getNumVBases())
+    FinishLayout(RD);
+
+#ifndef NDEBUG
+  // Check that we have base offsets for all bases.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+    E = RD->bases_end(); I != E; ++I) {
+      if (I->isVirtual())
+        continue;
+
+      const CXXRecordDecl *BaseDecl =
+        cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+      assert(Bases.count(BaseDecl) && "Did not find base offset!");
+  }
+
+  // And all virtual bases.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->vbases_begin(),
+    E = RD->vbases_end(); I != E; ++I) {
+      const CXXRecordDecl *BaseDecl =
+        cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+      assert(VBases.count(BaseDecl) && "Did not find base offset!");
+  }
+#endif
+}
+
 void RecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
   // In C++, records cannot be of size 0.
   if (Context.getLangOptions().CPlusPlus && getSizeInBits() == 0) {
@@ -1663,7 +1854,7 @@ void RecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
   CharUnits UnpackedSize = Context.toCharUnitsFromBits(UnpackedSizeInBits);
   setSize(llvm::RoundUpToAlignment(getSizeInBits(), Context.toBits(Alignment)));
 
-  unsigned CharBitNum = Context.Target.getCharWidth();
+  unsigned CharBitNum = Context.getTargetInfo().getCharWidth();
   if (const RecordDecl *RD = dyn_cast<RecordDecl>(D)) {
     // Warn if padding was introduced to the struct/class/union.
     if (getSizeInBits() > UnpaddedSize) {
@@ -1718,7 +1909,12 @@ void RecordLayoutBuilder::CheckFieldPadding(uint64_t Offset,
   if (isa<ObjCIvarDecl>(D))
     return;
 
-  unsigned CharBitNum = Context.Target.getCharWidth();
+  // Don't warn about structs created without a SourceLocation.  This can
+  // be done by clients of the AST, such as codegen.
+  if (D->getLocation().isInvalid())
+    return;
+  
+  unsigned CharBitNum = Context.getTargetInfo().getCharWidth();
 
   // Warn if padding was introduced to the struct/class.
   if (!IsUnion && Offset > UnpaddedOffset) {
@@ -1802,36 +1998,18 @@ RecordLayoutBuilder::Diag(SourceLocation Loc, unsigned DiagID) {
   return Context.getDiagnostics().Report(Loc, DiagID);
 }
 
-namespace {
-  // This class implements layout specific to the Microsoft ABI.
-  class MSRecordLayoutBuilder : public RecordLayoutBuilder {
-  public:
-    MSRecordLayoutBuilder(const ASTContext& Ctx,
-                          EmptySubobjectMap *EmptySubobjects) :
-      RecordLayoutBuilder(Ctx, EmptySubobjects) {}
-
-    virtual CharUnits GetVirtualPointersSize(const CXXRecordDecl *RD) const;
-  };
-}
-
-CharUnits
-MSRecordLayoutBuilder::GetVirtualPointersSize(const CXXRecordDecl *RD) const {
-  // We should reserve space for two pointers if the class has both
-  // virtual functions and virtual bases.
-  CharUnits PointerWidth = 
-    Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
-  if (RD->isPolymorphic() && RD->getNumVBases() > 0)
-    return 2 * PointerWidth;
-  return PointerWidth;
-}
-
 /// getASTRecordLayout - Get or compute information about the layout of the
 /// specified record (struct/union/class), which indicates its size and field
 /// position information.
 const ASTRecordLayout &
 ASTContext::getASTRecordLayout(const RecordDecl *D) const {
+  // These asserts test different things.  A record has a definition
+  // as soon as we begin to parse the definition.  That definition is
+  // not a complete definition (which is what isDefinition() tests)
+  // until we *finish* parsing the definition.
   D = D->getDefinition();
   assert(D && "Cannot get layout of forward declarations!");
+  assert(D->isCompleteDefinition() && "Cannot layout type before complete!");
 
   // Look up this layout, if already laid out, return what we have.
   // Note that we can't save a reference to the entry because this function
@@ -1844,25 +2022,44 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
   if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
     EmptySubobjectMap EmptySubobjects(*this, RD);
 
-    // When compiling for Microsoft, use the special MS builder.
     llvm::OwningPtr<RecordLayoutBuilder> Builder;
-    switch (Target.getCXXABI()) {
-    default:
-      Builder.reset(new RecordLayoutBuilder(*this, &EmptySubobjects));
-      break;
-    case CXXABI_Microsoft:
-      Builder.reset(new MSRecordLayoutBuilder(*this, &EmptySubobjects));
-    }
+    CharUnits TargetAlign = CharUnits::One();
+
+    Builder.reset(new RecordLayoutBuilder(*this,
+                                          &EmptySubobjects,
+                                          TargetAlign));
+
     // Recover resources if we crash before exiting this method.
     llvm::CrashRecoveryContextCleanupRegistrar<RecordLayoutBuilder>
       RecordBuilderCleanup(Builder.get());
     
     Builder->Layout(RD);
 
+    TargetAlign = Builder->getAligment();
+    
+    if (getTargetInfo().getCXXABI() == CXXABI_Microsoft &&
+        TargetAlign.getQuantity() > 4) {
+      // MSVC rounds the vtable pointer to the struct alignment in what must
+      // be a multi-pass operation. For now, let the builder figure out the
+      // alignment and recalculate the layout once its known.
+      Builder.reset(new RecordLayoutBuilder(*this,
+                                            &EmptySubobjects,
+                                            TargetAlign));
+
+      Builder->Layout(RD);
+
+      // Recover resources if we crash before exiting this method.
+      llvm::CrashRecoveryContextCleanupRegistrar<RecordLayoutBuilder>
+        RecordBuilderCleanup(Builder.get());
+    }
+
     // FIXME: This is not always correct. See the part about bitfields at
     // http://www.codesourcery.com/public/cxx-abi/abi.html#POD for more info.
     // FIXME: IsPODForThePurposeOfLayout should be stored in the record layout.
-    bool IsPODForThePurposeOfLayout = cast<CXXRecordDecl>(D)->isPOD();
+    // This does not affect the calculations of MSVC layouts
+    bool IsPODForThePurposeOfLayout = 
+      (getTargetInfo().getCXXABI() == CXXABI_Microsoft) ||
+      cast<CXXRecordDecl>(D)->isPOD();
 
     // FIXME: This should be done in FinalizeLayout.
     CharUnits DataSize =
@@ -1873,6 +2070,7 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
     NewEntry =
       new (*this) ASTRecordLayout(*this, Builder->getSize(), 
                                   Builder->Alignment,
+                                  Builder->GetVBPtrOffset(),
                                   DataSize, 
                                   Builder->FieldOffsets.data(),
                                   Builder->FieldOffsets.size(),
@@ -1883,7 +2081,7 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
                                   Builder->PrimaryBaseIsVirtual,
                                   Builder->Bases, Builder->VBases);
   } else {
-    RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0);
+    RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0, CharUnits::One());
     Builder.Layout(D);
 
     NewEntry =
@@ -1915,8 +2113,8 @@ const CXXMethodDecl *ASTContext::getKeyFunction(const CXXRecordDecl *RD) {
   return Entry;
 }
 
-/// getInterfaceLayoutImpl - Get or compute information about the
-/// layout of the given interface.
+/// getObjCLayout - Get or compute information about the layout of the
+/// given interface.
 ///
 /// \param Impl - If given, also include the layout of the interface's
 /// implementation. This may differ by including synthesized ivars.
@@ -1942,7 +2140,7 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
       return getObjCLayout(D, 0);
   }
 
-  RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0);
+  RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0, CharUnits::One());
   Builder.Layout(D);
 
   const ASTRecordLayout *NewEntry =
@@ -1957,13 +2155,13 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
   return *NewEntry;
 }
 
-static void PrintOffset(llvm::raw_ostream &OS,
+static void PrintOffset(raw_ostream &OS,
                         CharUnits Offset, unsigned IndentLevel) {
   OS << llvm::format("%4d | ", Offset.getQuantity());
   OS.indent(IndentLevel * 2);
 }
 
-static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
+static void DumpCXXRecordLayout(raw_ostream &OS,
                                 const CXXRecordDecl *RD, const ASTContext &C,
                                 CharUnits Offset,
                                 unsigned IndentLevel,
@@ -1982,12 +2180,22 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
   IndentLevel++;
 
   const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+  bool HasVbptr = Layout.getVBPtrOffset() != CharUnits::fromQuantity(-1);
 
   // Vtable pointer.
   if (RD->isDynamicClass() && !PrimaryBase) {
     PrintOffset(OS, Offset, IndentLevel);
-    OS << '(' << RD << " vtable pointer)\n";
+    OS << '(' << *RD << " vtable pointer)\n";
   }
+  
+  if (HasVbptr && !PrimaryBase) {
+    PrintOffset(OS, Offset + Layout.getVBPtrOffset(), IndentLevel);
+    OS << '(' << *RD << " vbtable pointer)\n";
+
+    // one vbtable per class
+    HasVbptr = false;
+  }
+
   // Dump (non-virtual) bases
   for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
          E = RD->bases_end(); I != E; ++I) {
@@ -2005,6 +2213,11 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
                         Base == PrimaryBase ? "(primary base)" : "(base)",
                         /*IncludeVirtualBases=*/false);
   }
+  // vbptr
+  if (HasVbptr) {
+    PrintOffset(OS, Offset + Layout.getVBPtrOffset(), IndentLevel);
+    OS << '(' << *RD << " vbtable pointer)\n";
+  }
 
   // Dump fields.
   uint64_t FieldNo = 0;
@@ -2024,7 +2237,7 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
     }
 
     PrintOffset(OS, FieldOffset, IndentLevel);
-    OS << Field->getType().getAsString() << ' ' << Field << '\n';
+    OS << Field->getType().getAsString() << ' ' << *Field << '\n';
   }
 
   if (!IncludeVirtualBases)
@@ -2053,7 +2266,7 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
 }
 
 void ASTContext::DumpRecordLayout(const RecordDecl *RD,
-                                  llvm::raw_ostream &OS) const {
+                                  raw_ostream &OS) const {
   const ASTRecordLayout &Info = getASTRecordLayout(RD);
 
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
diff --git a/contrib/llvm/tools/clang/lib/AST/SelectorLocationsKind.cpp b/contrib/llvm/tools/clang/lib/AST/SelectorLocationsKind.cpp
new file mode 100644
index 000000000000..671207a7f2d9
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/AST/SelectorLocationsKind.cpp
@@ -0,0 +1,128 @@
+//===--- SelectorLocationsKind.cpp - Kind of selector locations -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Describes whether the identifier locations for a selector are "standard"
+// or not.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/SelectorLocationsKind.h"
+#include "clang/AST/Expr.h"
+
+using namespace clang;
+
+static SourceLocation getStandardSelLoc(unsigned Index,
+                                        Selector Sel,
+                                        bool WithArgSpace,
+                                        SourceLocation ArgLoc,
+                                        SourceLocation EndLoc) {
+  unsigned NumSelArgs = Sel.getNumArgs();
+  if (NumSelArgs == 0) {
+    assert(Index == 0);
+    if (EndLoc.isInvalid())
+      return SourceLocation();
+    IdentifierInfo *II = Sel.getIdentifierInfoForSlot(0);
+    unsigned Len = II ? II->getLength() : 0;
+    return EndLoc.getLocWithOffset(-Len);
+  }
+
+  assert(Index < NumSelArgs);
+  if (ArgLoc.isInvalid())
+    return SourceLocation();
+  IdentifierInfo *II = Sel.getIdentifierInfoForSlot(Index);
+  unsigned Len = /* selector id */ (II ? II->getLength() : 0) + /* ':' */ 1;
+  if (WithArgSpace)
+    ++Len;
+  return ArgLoc.getLocWithOffset(-Len);
+}
+
+namespace {
+
+template <typename T>
+SourceLocation getArgLoc(T* Arg);
+
+template <>
+SourceLocation getArgLoc<Expr>(Expr *Arg) {
+  return Arg->getLocStart();
+}
+
+template <>
+SourceLocation getArgLoc<ParmVarDecl>(ParmVarDecl *Arg) {
+  SourceLocation Loc = Arg->getLocStart();
+  if (Loc.isInvalid())
+    return Loc;
+  // -1 to point to left paren of the method parameter's type.
+  return Loc.getLocWithOffset(-1);
+}
+
+template <typename T>
+SourceLocation getArgLoc(unsigned Index, ArrayRef<T*> Args) {
+  return Index < Args.size() ? getArgLoc(Args[Index]) : SourceLocation();
+}
+
+template <typename T>
+SelectorLocationsKind hasStandardSelLocs(Selector Sel,
+                                         ArrayRef<SourceLocation> SelLocs,
+                                         ArrayRef<T *> Args,
+                                         SourceLocation EndLoc) {
+  // Are selector locations in standard position with no space between args ?
+  unsigned i;
+  for (i = 0; i != SelLocs.size(); ++i) {
+    if (SelLocs[i] != getStandardSelectorLoc(i, Sel, /*WithArgSpace=*/false,
+                                             Args, EndLoc))
+      break;
+  }
+  if (i == SelLocs.size())
+    return SelLoc_StandardNoSpace;
+
+  // Are selector locations in standard position with space between args ?
+  for (i = 0; i != SelLocs.size(); ++i) {
+    if (SelLocs[i] != getStandardSelectorLoc(i, Sel, /*WithArgSpace=*/true,
+                                             Args, EndLoc))
+      return SelLoc_NonStandard;
+  }
+
+  return SelLoc_StandardWithSpace;
+}
+
+} // anonymous namespace
+
+SelectorLocationsKind
+clang::hasStandardSelectorLocs(Selector Sel,
+                               ArrayRef<SourceLocation> SelLocs,
+                               ArrayRef<Expr *> Args,
+                               SourceLocation EndLoc) {
+  return hasStandardSelLocs(Sel, SelLocs, Args, EndLoc);
+}
+
+SourceLocation clang::getStandardSelectorLoc(unsigned Index,
+                                             Selector Sel,
+                                             bool WithArgSpace,
+                                             ArrayRef<Expr *> Args,
+                                             SourceLocation EndLoc) {
+  return getStandardSelLoc(Index, Sel, WithArgSpace,
+                           getArgLoc(Index, Args), EndLoc);
+}
+
+SelectorLocationsKind
+clang::hasStandardSelectorLocs(Selector Sel,
+                               ArrayRef<SourceLocation> SelLocs,
+                               ArrayRef<ParmVarDecl *> Args,
+                               SourceLocation EndLoc) {
+  return hasStandardSelLocs(Sel, SelLocs, Args, EndLoc);
+}
+
+SourceLocation clang::getStandardSelectorLoc(unsigned Index,
+                                             Selector Sel,
+                                             bool WithArgSpace,
+                                             ArrayRef<ParmVarDecl *> Args,
+                                             SourceLocation EndLoc) {
+  return getStandardSelLoc(Index, Sel, WithArgSpace,
+                           getArgLoc(Index, Args), EndLoc);
+}
diff --git a/contrib/llvm/tools/clang/lib/AST/Stmt.cpp b/contrib/llvm/tools/clang/lib/AST/Stmt.cpp
index fd6f21d43b59..e7b87e4db679 100644
--- a/contrib/llvm/tools/clang/lib/AST/Stmt.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/Stmt.cpp
@@ -97,6 +97,22 @@ Stmt *Stmt::IgnoreImplicit() {
   return s;
 }
 
+/// \brief Strip off all label-like statements.
+///
+/// This will strip off label statements, case statements, and default
+/// statements recursively.
+const Stmt *Stmt::stripLabelLikeStatements() const {
+  const Stmt *S = this;
+  while (true) {
+    if (const LabelStmt *LS = dyn_cast<LabelStmt>(S))
+      S = LS->getSubStmt();
+    else if (const SwitchCase *SC = dyn_cast<SwitchCase>(S))
+      S = SC->getSubStmt();
+    else
+      return S;
+  }
+}
+
 namespace {
   struct good {};
   struct bad {};
@@ -214,7 +230,7 @@ Expr *AsmStmt::getOutputExpr(unsigned i) {
 /// getOutputConstraint - Return the constraint string for the specified
 /// output operand.  All output constraints are known to be non-empty (either
 /// '=' or '+').
-llvm::StringRef AsmStmt::getOutputConstraint(unsigned i) const {
+StringRef AsmStmt::getOutputConstraint(unsigned i) const {
   return getOutputConstraintLiteral(i)->getString();
 }
 
@@ -238,7 +254,7 @@ void AsmStmt::setInputExpr(unsigned i, Expr *E) {
 
 /// getInputConstraint - Return the specified input constraint.  Unlike output
 /// constraints, these can be empty.
-llvm::StringRef AsmStmt::getInputConstraint(unsigned i) const {
+StringRef AsmStmt::getInputConstraint(unsigned i) const {
   return getInputConstraintLiteral(i)->getString();
 }
 
@@ -277,7 +293,7 @@ void AsmStmt::setOutputsAndInputsAndClobbers(ASTContext &C,
 /// getNamedOperand - Given a symbolic operand reference like %[foo],
 /// translate this into a numeric value needed to reference the same operand.
 /// This returns -1 if the operand name is invalid.
-int AsmStmt::getNamedOperand(llvm::StringRef SymbolicName) const {
+int AsmStmt::getNamedOperand(StringRef SymbolicName) const {
   unsigned NumPlusOperands = 0;
 
   // Check if this is an output operand.
@@ -297,9 +313,9 @@ int AsmStmt::getNamedOperand(llvm::StringRef SymbolicName) const {
 /// AnalyzeAsmString - Analyze the asm string of the current asm, decomposing
 /// it into pieces.  If the asm string is erroneous, emit errors and return
 /// true, otherwise return false.
-unsigned AsmStmt::AnalyzeAsmString(llvm::SmallVectorImpl<AsmStringPiece>&Pieces,
+unsigned AsmStmt::AnalyzeAsmString(SmallVectorImpl<AsmStringPiece>&Pieces,
                                    ASTContext &C, unsigned &DiagOffs) const {
-  llvm::StringRef Str = getAsmString()->getString();
+  StringRef Str = getAsmString()->getString();
   const char *StrStart = Str.begin();
   const char *StrEnd = Str.end();
   const char *CurPtr = StrStart;
@@ -326,7 +342,7 @@ unsigned AsmStmt::AnalyzeAsmString(llvm::SmallVectorImpl<AsmStringPiece>&Pieces,
   // asm string.
   std::string CurStringPiece;
 
-  bool HasVariants = !C.Target.hasNoAsmVariants();
+  bool HasVariants = !C.getTargetInfo().hasNoAsmVariants();
   
   while (1) {
     // Done with the string?
@@ -416,7 +432,7 @@ unsigned AsmStmt::AnalyzeAsmString(llvm::SmallVectorImpl<AsmStringPiece>&Pieces,
       if (NameEnd == CurPtr)
         return diag::err_asm_empty_symbolic_operand_name;
 
-      llvm::StringRef SymbolicName(CurPtr, NameEnd - CurPtr);
+      StringRef SymbolicName(CurPtr, NameEnd - CurPtr);
 
       int N = getNamedOperand(SymbolicName);
       if (N == -1) {
diff --git a/contrib/llvm/tools/clang/lib/AST/StmtDumper.cpp b/contrib/llvm/tools/clang/lib/AST/StmtDumper.cpp
index fb024f33ab30..2968739c227a 100644
--- a/contrib/llvm/tools/clang/lib/AST/StmtDumper.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/StmtDumper.cpp
@@ -27,7 +27,7 @@ using namespace clang;
 namespace  {
   class StmtDumper : public StmtVisitor<StmtDumper> {
     SourceManager *SM;
-    llvm::raw_ostream &OS;
+    raw_ostream &OS;
     unsigned IndentLevel;
 
     /// MaxDepth - When doing a normal dump (not dumpAll) we only want to dump
@@ -41,7 +41,7 @@ namespace  {
     unsigned LastLocLine;
 
   public:
-    StmtDumper(SourceManager *sm, llvm::raw_ostream &os, unsigned maxDepth)
+    StmtDumper(SourceManager *sm, raw_ostream &os, unsigned maxDepth)
       : SM(sm), OS(os), IndentLevel(0-1), MaxDepth(maxDepth) {
       LastLocFilename = "";
       LastLocLine = ~0U;
@@ -235,9 +235,9 @@ void StmtDumper::DumpDeclarator(Decl *D) {
   // nodes are where they need to be.
   if (TypedefDecl *localType = dyn_cast<TypedefDecl>(D)) {
     OS << "\"typedef " << localType->getUnderlyingType().getAsString()
-       << ' ' << localType << '"';
+       << ' ' << *localType << '"';
   } else if (TypeAliasDecl *localType = dyn_cast<TypeAliasDecl>(D)) {
-    OS << "\"using " << localType << " = "
+    OS << "\"using " << *localType << " = "
        << localType->getUnderlyingType().getAsString() << '"';
   } else if (ValueDecl *VD = dyn_cast<ValueDecl>(D)) {
     OS << "\"";
@@ -294,7 +294,7 @@ void StmtDumper::DumpDeclarator(Decl *D) {
     DumpSubTree(SAD->getMessage());
     OS << ");\"";
   } else {
-    assert(0 && "Unexpected decl");
+    llvm_unreachable("Unexpected decl");
   }
 }
 
@@ -333,7 +333,7 @@ void StmtDumper::VisitExpr(Expr *Node) {
   DumpExpr(Node);
 }
 
-static void DumpBasePath(llvm::raw_ostream &OS, CastExpr *Node) {
+static void DumpBasePath(raw_ostream &OS, CastExpr *Node) {
   if (Node->path_empty())
     return;
 
@@ -407,7 +407,7 @@ void StmtDumper::VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node) {
   DumpExpr(Node);
 
   OS << " " << Node->getDecl()->getDeclKindName()
-     << "Decl='" << Node->getDecl()
+     << "Decl='" << *Node->getDecl()
      << "' " << (void*)Node->getDecl();
   if (Node->isFreeIvar())
     OS << " isFreeIvar";
@@ -416,7 +416,7 @@ void StmtDumper::VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node) {
 void StmtDumper::VisitPredefinedExpr(PredefinedExpr *Node) {
   DumpExpr(Node);
   switch (Node->getIdentType()) {
-  default: assert(0 && "unknown case");
+  default: llvm_unreachable("unknown case");
   case PredefinedExpr::Func:           OS <<  " __func__"; break;
   case PredefinedExpr::Function:       OS <<  " __FUNCTION__"; break;
   case PredefinedExpr::PrettyFunction: OS <<  " __PRETTY_FUNCTION__";break;
@@ -443,8 +443,13 @@ void StmtDumper::VisitStringLiteral(StringLiteral *Str) {
   DumpExpr(Str);
   // FIXME: this doesn't print wstrings right.
   OS << " ";
-  if (Str->isWide())
-    OS << "L";
+  switch (Str->getKind()) {
+  case StringLiteral::Ascii: break; // No prefix
+  case StringLiteral::Wide:  OS << 'L'; break;
+  case StringLiteral::UTF8:  OS << "u8"; break;
+  case StringLiteral::UTF16: OS << 'u'; break;
+  case StringLiteral::UTF32: OS << 'U'; break;
+  }
   OS << '"';
   OS.write_escaped(Str->getString());
   OS << '"';
@@ -475,7 +480,7 @@ void StmtDumper::VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *Node) {
 void StmtDumper::VisitMemberExpr(MemberExpr *Node) {
   DumpExpr(Node);
   OS << " " << (Node->isArrow() ? "->" : ".")
-     << Node->getMemberDecl() << ' '
+     << *Node->getMemberDecl() << ' '
      << (void*)Node->getMemberDecl();
 }
 void StmtDumper::VisitExtVectorElementExpr(ExtVectorElementExpr *Node) {
@@ -552,7 +557,8 @@ void StmtDumper::VisitCXXThisExpr(CXXThisExpr *Node) {
 
 void StmtDumper::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *Node) {
   DumpExpr(Node);
-  OS << " functional cast to " << Node->getTypeAsWritten().getAsString();
+  OS << " functional cast to " << Node->getTypeAsWritten().getAsString()
+     << " <" << Node->getCastKindName() << ">";
 }
 
 void StmtDumper::VisitCXXConstructExpr(CXXConstructExpr *Node) {
@@ -637,7 +643,7 @@ void StmtDumper::VisitObjCSelectorExpr(ObjCSelectorExpr *Node) {
 void StmtDumper::VisitObjCProtocolExpr(ObjCProtocolExpr *Node) {
   DumpExpr(Node);
 
-  OS << ' ' << Node->getProtocol();
+  OS << ' ' <<* Node->getProtocol();
 }
 
 void StmtDumper::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
@@ -656,7 +662,7 @@ void StmtDumper::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
       OS << "(null)";
     OS << "\"";
   } else {
-    OS << " Kind=PropertyRef Property=\"" << Node->getExplicitProperty() << '"';
+    OS << " Kind=PropertyRef Property=\"" << *Node->getExplicitProperty() <<'"';
   }
 
   if (Node->isSuperReceiver())
@@ -674,7 +680,7 @@ void Stmt::dump(SourceManager &SM) const {
   dump(llvm::errs(), SM);
 }
 
-void Stmt::dump(llvm::raw_ostream &OS, SourceManager &SM) const {
+void Stmt::dump(raw_ostream &OS, SourceManager &SM) const {
   StmtDumper P(&SM, OS, 4);
   P.DumpSubTree(const_cast<Stmt*>(this));
   OS << "\n";
diff --git a/contrib/llvm/tools/clang/lib/AST/StmtPrinter.cpp b/contrib/llvm/tools/clang/lib/AST/StmtPrinter.cpp
index f705a84c7cf4..daaa354ac342 100644
--- a/contrib/llvm/tools/clang/lib/AST/StmtPrinter.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/StmtPrinter.cpp
@@ -28,14 +28,14 @@ using namespace clang;
 
 namespace  {
   class StmtPrinter : public StmtVisitor<StmtPrinter> {
-    llvm::raw_ostream &OS;
+    raw_ostream &OS;
     ASTContext &Context;
     unsigned IndentLevel;
     clang::PrinterHelper* Helper;
     PrintingPolicy Policy;
 
   public:
-    StmtPrinter(llvm::raw_ostream &os, ASTContext &C, PrinterHelper* helper,
+    StmtPrinter(raw_ostream &os, ASTContext &C, PrinterHelper* helper,
                 const PrintingPolicy &Policy,
                 unsigned Indentation = 0)
       : OS(os), Context(C), IndentLevel(Indentation), Helper(helper),
@@ -76,7 +76,7 @@ namespace  {
         OS << "<null expr>";
     }
 
-    llvm::raw_ostream &Indent(int Delta = 0) {
+    raw_ostream &Indent(int Delta = 0) {
       for (int i = 0, e = IndentLevel+Delta; i < e; ++i)
         OS << "  ";
       return OS;
@@ -124,7 +124,7 @@ void StmtPrinter::PrintRawDecl(Decl *D) {
 
 void StmtPrinter::PrintRawDeclStmt(DeclStmt *S) {
   DeclStmt::decl_iterator Begin = S->decl_begin(), End = S->decl_end();
-  llvm::SmallVector<Decl*, 2> Decls;
+  SmallVector<Decl*, 2> Decls;
   for ( ; Begin != End; ++Begin)
     Decls.push_back(*Begin);
 
@@ -564,7 +564,7 @@ void StmtPrinter::VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node) {
     PrintExpr(Node->getBase());
     OS << (Node->isArrow() ? "->" : ".");
   }
-  OS << Node->getDecl();
+  OS << *Node->getDecl();
 }
 
 void StmtPrinter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
@@ -584,7 +584,7 @@ void StmtPrinter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
 void StmtPrinter::VisitPredefinedExpr(PredefinedExpr *Node) {
   switch (Node->getIdentType()) {
     default:
-      assert(0 && "unknown case");
+      llvm_unreachable("unknown case");
     case PredefinedExpr::Func:
       OS << "__func__";
       break;
@@ -599,8 +599,14 @@ void StmtPrinter::VisitPredefinedExpr(PredefinedExpr *Node) {
 
 void StmtPrinter::VisitCharacterLiteral(CharacterLiteral *Node) {
   unsigned value = Node->getValue();
-  if (Node->isWide())
-    OS << "L";
+
+  switch (Node->getKind()) {
+  case CharacterLiteral::Ascii: break; // no prefix.
+  case CharacterLiteral::Wide:  OS << 'L'; break;
+  case CharacterLiteral::UTF16: OS << 'u'; break;
+  case CharacterLiteral::UTF32: OS << 'U'; break;
+  }
+
   switch (value) {
   case '\\':
     OS << "'\\\\'";
@@ -652,7 +658,7 @@ void StmtPrinter::VisitIntegerLiteral(IntegerLiteral *Node) {
 
   // Emit suffixes.  Integer literals are always a builtin integer type.
   switch (Node->getType()->getAs<BuiltinType>()->getKind()) {
-  default: assert(0 && "Unexpected type for integer literal!");
+  default: llvm_unreachable("Unexpected type for integer literal!");
   case BuiltinType::Int:       break; // no suffix.
   case BuiltinType::UInt:      OS << 'U'; break;
   case BuiltinType::Long:      OS << 'L'; break;
@@ -662,8 +668,9 @@ void StmtPrinter::VisitIntegerLiteral(IntegerLiteral *Node) {
   }
 }
 void StmtPrinter::VisitFloatingLiteral(FloatingLiteral *Node) {
-  // FIXME: print value more precisely.
-  OS << Node->getValueAsApproximateDouble();
+  llvm::SmallString<16> Str;
+  Node->getValue().toString(Str);
+  OS << Str;
 }
 
 void StmtPrinter::VisitImaginaryLiteral(ImaginaryLiteral *Node) {
@@ -672,12 +679,18 @@ void StmtPrinter::VisitImaginaryLiteral(ImaginaryLiteral *Node) {
 }
 
 void StmtPrinter::VisitStringLiteral(StringLiteral *Str) {
-  if (Str->isWide()) OS << 'L';
+  switch (Str->getKind()) {
+  case StringLiteral::Ascii: break; // no prefix.
+  case StringLiteral::Wide:  OS << 'L'; break;
+  case StringLiteral::UTF8:  OS << "u8"; break;
+  case StringLiteral::UTF16: OS << 'u'; break;
+  case StringLiteral::UTF32: OS << 'U'; break;
+  }
   OS << '"';
 
   // FIXME: this doesn't print wstrings right.
-  llvm::StringRef StrData = Str->getString();
-  for (llvm::StringRef::iterator I = StrData.begin(), E = StrData.end(); 
+  StringRef StrData = Str->getString();
+  for (StringRef::iterator I = StrData.begin(), E = StrData.end(); 
                                                              I != E; ++I) {
     unsigned char Char = *I;
 
@@ -998,6 +1011,59 @@ void StmtPrinter::VisitVAArgExpr(VAArgExpr *Node) {
   OS << ")";
 }
 
+void StmtPrinter::VisitAtomicExpr(AtomicExpr *Node) {
+  const char *Name = 0;
+  switch (Node->getOp()) {
+    case AtomicExpr::Load:
+      Name = "__atomic_load(";
+      break;
+    case AtomicExpr::Store:
+      Name = "__atomic_store(";
+      break;
+    case AtomicExpr::CmpXchgStrong:
+      Name = "__atomic_compare_exchange_strong(";
+      break;
+    case AtomicExpr::CmpXchgWeak:
+      Name = "__atomic_compare_exchange_weak(";
+      break;
+    case AtomicExpr::Xchg:
+      Name = "__atomic_exchange(";
+      break;
+    case AtomicExpr::Add:
+      Name = "__atomic_fetch_add(";
+      break;
+    case AtomicExpr::Sub:
+      Name = "__atomic_fetch_sub(";
+      break;
+    case AtomicExpr::And:
+      Name = "__atomic_fetch_and(";
+      break;
+    case AtomicExpr::Or:
+      Name = "__atomic_fetch_or(";
+      break;
+    case AtomicExpr::Xor:
+      Name = "__atomic_fetch_xor(";
+      break;
+  }
+  OS << Name;
+  PrintExpr(Node->getPtr());
+  OS << ", ";
+  if (Node->getOp() != AtomicExpr::Load) {
+    PrintExpr(Node->getVal1());
+    OS << ", ";
+  }
+  if (Node->isCmpXChg()) {
+    PrintExpr(Node->getVal2());
+    OS << ", ";
+  }
+  PrintExpr(Node->getOrder());
+  if (Node->isCmpXChg()) {
+    OS << ", ";
+    PrintExpr(Node->getOrderFail());
+  }
+  OS << ")";
+}
+
 // C++
 void StmtPrinter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *Node) {
   const char *OpStrings[NUM_OVERLOADED_OPERATORS] = {
@@ -1039,7 +1105,7 @@ void StmtPrinter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *Node) {
     OS << ' ' << OpStrings[Kind] << ' ';
     PrintExpr(Node->getArg(1));
   } else {
-    assert(false && "unknown overloaded operator");
+    llvm_unreachable("unknown overloaded operator");
   }
 }
 
@@ -1438,7 +1504,7 @@ void StmtPrinter::VisitObjCSelectorExpr(ObjCSelectorExpr *Node) {
 }
 
 void StmtPrinter::VisitObjCProtocolExpr(ObjCProtocolExpr *Node) {
-  OS << "@protocol(" << Node->getProtocol() << ')';
+  OS << "@protocol(" << *Node->getProtocol() << ')';
 }
 
 void StmtPrinter::VisitObjCMessageExpr(ObjCMessageExpr *Mess) {
@@ -1520,7 +1586,7 @@ void StmtPrinter::VisitBlockExpr(BlockExpr *Node) {
 }
 
 void StmtPrinter::VisitBlockDeclRefExpr(BlockDeclRefExpr *Node) {
-  OS << Node->getDecl();
+  OS << *Node->getDecl();
 }
 
 void StmtPrinter::VisitOpaqueValueExpr(OpaqueValueExpr *Node) {}
@@ -1541,7 +1607,7 @@ void Stmt::dumpPretty(ASTContext& Context) const {
               PrintingPolicy(Context.getLangOptions()));
 }
 
-void Stmt::printPretty(llvm::raw_ostream &OS, ASTContext& Context,
+void Stmt::printPretty(raw_ostream &OS, ASTContext& Context,
                        PrinterHelper* Helper,
                        const PrintingPolicy &Policy,
                        unsigned Indentation) const {
diff --git a/contrib/llvm/tools/clang/lib/AST/StmtProfile.cpp b/contrib/llvm/tools/clang/lib/AST/StmtProfile.cpp
index 120c9e50a92a..214378a974a6 100644
--- a/contrib/llvm/tools/clang/lib/AST/StmtProfile.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/StmtProfile.cpp
@@ -252,7 +252,7 @@ void StmtProfiler::VisitIntegerLiteral(const IntegerLiteral *S) {
 
 void StmtProfiler::VisitCharacterLiteral(const CharacterLiteral *S) {
   VisitExpr(S);
-  ID.AddBoolean(S->isWide());
+  ID.AddInteger(S->getKind());
   ID.AddInteger(S->getValue());
 }
 
@@ -269,7 +269,7 @@ void StmtProfiler::VisitImaginaryLiteral(const ImaginaryLiteral *S) {
 void StmtProfiler::VisitStringLiteral(const StringLiteral *S) {
   VisitExpr(S);
   ID.AddString(S->getString());
-  ID.AddBoolean(S->isWide());
+  ID.AddInteger(S->getKind());
 }
 
 void StmtProfiler::VisitParenExpr(const ParenExpr *S) {
@@ -468,6 +468,11 @@ void StmtProfiler::VisitGenericSelectionExpr(const GenericSelectionExpr *S) {
   }
 }
 
+void StmtProfiler::VisitAtomicExpr(const AtomicExpr *S) {
+  VisitExpr(S);
+  ID.AddInteger(S->getOp());
+}
+
 static Stmt::StmtClass DecodeOperatorCall(const CXXOperatorCallExpr *S,
                                           UnaryOperatorKind &UnaryOp,
                                           BinaryOperatorKind &BinaryOp) {
diff --git a/contrib/llvm/tools/clang/lib/AST/TemplateBase.cpp b/contrib/llvm/tools/clang/lib/AST/TemplateBase.cpp
index 56c6e7bc47c1..0c011a8ef090 100644
--- a/contrib/llvm/tools/clang/lib/AST/TemplateBase.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/TemplateBase.cpp
@@ -33,7 +33,7 @@ using namespace clang;
 ///
 /// \param Out the raw_ostream instance to use for printing.
 static void printIntegral(const TemplateArgument &TemplArg,
-                          llvm::raw_ostream &Out) {
+                          raw_ostream &Out) {
   const ::clang::Type *T = TemplArg.getIntegralType().getTypePtr();
   const llvm::APSInt *Val = TemplArg.getAsIntegral();
 
@@ -68,8 +68,7 @@ TemplateArgument TemplateArgument::CreatePackCopy(ASTContext &Context,
 bool TemplateArgument::isDependent() const {
   switch (getKind()) {
   case Null:
-    assert(false && "Should not have a NULL template argument");
-    return false;
+    llvm_unreachable("Should not have a NULL template argument");
 
   case Type:
     return getAsType()->isDependentType();
@@ -107,8 +106,7 @@ bool TemplateArgument::isDependent() const {
 bool TemplateArgument::isInstantiationDependent() const {
   switch (getKind()) {
   case Null:
-    assert(false && "Should not have a NULL template argument");
-    return false;
+    llvm_unreachable("Should not have a NULL template argument");
     
   case Type:
     return getAsType()->isInstantiationDependentType();
@@ -309,7 +307,7 @@ TemplateArgument TemplateArgument::getPackExpansionPattern() const {
 }
 
 void TemplateArgument::print(const PrintingPolicy &Policy, 
-                             llvm::raw_ostream &Out) const {
+                             raw_ostream &Out) const {
   switch (getKind()) {
   case Null:
     Out << "<no value>";
@@ -530,3 +528,65 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
   
   return DB;
 }
+
+const ASTTemplateArgumentListInfo *
+ASTTemplateArgumentListInfo::Create(ASTContext &C,
+                                    const TemplateArgumentListInfo &List) {
+  std::size_t size = sizeof(CXXDependentScopeMemberExpr) +
+                     ASTTemplateArgumentListInfo::sizeFor(List);
+  void *Mem = C.Allocate(size, llvm::alignOf<ASTTemplateArgumentListInfo>());
+  ASTTemplateArgumentListInfo *TAI = new (Mem) ASTTemplateArgumentListInfo();
+  TAI->initializeFrom(List);
+  return TAI;
+}
+
+void ASTTemplateArgumentListInfo::initializeFrom(
+                                      const TemplateArgumentListInfo &Info) {
+  LAngleLoc = Info.getLAngleLoc();
+  RAngleLoc = Info.getRAngleLoc();
+  NumTemplateArgs = Info.size();
+
+  TemplateArgumentLoc *ArgBuffer = getTemplateArgs();
+  for (unsigned i = 0; i != NumTemplateArgs; ++i)
+    new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
+}
+
+void ASTTemplateArgumentListInfo::initializeFrom(
+                                          const TemplateArgumentListInfo &Info,
+                                                  bool &Dependent, 
+                                                  bool &InstantiationDependent,
+                                       bool &ContainsUnexpandedParameterPack) {
+  LAngleLoc = Info.getLAngleLoc();
+  RAngleLoc = Info.getRAngleLoc();
+  NumTemplateArgs = Info.size();
+
+  TemplateArgumentLoc *ArgBuffer = getTemplateArgs();
+  for (unsigned i = 0; i != NumTemplateArgs; ++i) {
+    Dependent = Dependent || Info[i].getArgument().isDependent();
+    InstantiationDependent = InstantiationDependent || 
+                             Info[i].getArgument().isInstantiationDependent();
+    ContainsUnexpandedParameterPack 
+      = ContainsUnexpandedParameterPack || 
+        Info[i].getArgument().containsUnexpandedParameterPack();
+
+    new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
+  }
+}
+
+void ASTTemplateArgumentListInfo::copyInto(
+                                      TemplateArgumentListInfo &Info) const {
+  Info.setLAngleLoc(LAngleLoc);
+  Info.setRAngleLoc(RAngleLoc);
+  for (unsigned I = 0; I != NumTemplateArgs; ++I)
+    Info.addArgument(getTemplateArgs()[I]);
+}
+
+std::size_t ASTTemplateArgumentListInfo::sizeFor(unsigned NumTemplateArgs) {
+  return sizeof(ASTTemplateArgumentListInfo) +
+         sizeof(TemplateArgumentLoc) * NumTemplateArgs;
+}
+
+std::size_t ASTTemplateArgumentListInfo::sizeFor(
+                                      const TemplateArgumentListInfo &Info) {
+  return sizeFor(Info.size());
+}
diff --git a/contrib/llvm/tools/clang/lib/AST/TemplateName.cpp b/contrib/llvm/tools/clang/lib/AST/TemplateName.cpp
index 1f7b19aae471..a0487ba0593b 100644
--- a/contrib/llvm/tools/clang/lib/AST/TemplateName.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/TemplateName.cpp
@@ -125,16 +125,16 @@ bool TemplateName::containsUnexpandedParameterPack() const {
 }
 
 void
-TemplateName::print(llvm::raw_ostream &OS, const PrintingPolicy &Policy,
+TemplateName::print(raw_ostream &OS, const PrintingPolicy &Policy,
                     bool SuppressNNS) const {
   if (TemplateDecl *Template = Storage.dyn_cast<TemplateDecl *>())
-    OS << Template;
+    OS << *Template;
   else if (QualifiedTemplateName *QTN = getAsQualifiedTemplateName()) {
     if (!SuppressNNS)
       QTN->getQualifier()->print(OS, Policy);
     if (QTN->hasTemplateKeyword())
       OS << "template ";
-    OS << QTN->getDecl();
+    OS << *QTN->getDecl();
   } else if (DependentTemplateName *DTN = getAsDependentTemplateName()) {
     if (!SuppressNNS && DTN->getQualifier())
       DTN->getQualifier()->print(OS, Policy);
diff --git a/contrib/llvm/tools/clang/lib/AST/Type.cpp b/contrib/llvm/tools/clang/lib/AST/Type.cpp
index 08971eb03421..44eeec004f57 100644
--- a/contrib/llvm/tools/clang/lib/AST/Type.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/Type.cpp
@@ -42,6 +42,26 @@ bool Qualifiers::isStrictSupersetOf(Qualifiers Other) const {
      (hasObjCLifetime() && !Other.hasObjCLifetime()));
 }
 
+const IdentifierInfo* QualType::getBaseTypeIdentifier() const {
+  const Type* ty = getTypePtr();
+  NamedDecl *ND = NULL;
+  if (ty->isPointerType() || ty->isReferenceType())
+    return ty->getPointeeType().getBaseTypeIdentifier();
+  else if (ty->isRecordType())
+    ND = ty->getAs<RecordType>()->getDecl();
+  else if (ty->isEnumeralType())
+    ND = ty->getAs<EnumType>()->getDecl();
+  else if (ty->getTypeClass() == Type::Typedef)
+    ND = ty->getAs<TypedefType>()->getDecl();
+  else if (ty->isArrayType())
+    return ty->castAsArrayTypeUnsafe()->
+        getElementType().getBaseTypeIdentifier();
+
+  if (ND)
+    return ND->getIdentifier();
+  return NULL;
+}
+
 bool QualType::isConstant(QualType T, ASTContext &Ctx) {
   if (T.isConstQualified())
     return true;
@@ -635,6 +655,18 @@ bool Type::isWideCharType() const {
   return false;
 }
 
+bool Type::isChar16Type() const {
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
+    return BT->getKind() == BuiltinType::Char16;
+  return false;
+}
+
+bool Type::isChar32Type() const {
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
+    return BT->getKind() == BuiltinType::Char32;
+  return false;
+}
+
 /// \brief Determine whether this type is any of the built-in character
 /// types.
 bool Type::isAnyCharacterType() const {
@@ -734,9 +766,16 @@ bool Type::hasUnsignedIntegerRepresentation() const {
     return isUnsignedIntegerType();
 }
 
+bool Type::isHalfType() const {
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
+    return BT->getKind() == BuiltinType::Half;
+  // FIXME: Should we allow complex __fp16? Probably not.
+  return false;
+}
+
 bool Type::isFloatingType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() >= BuiltinType::Float &&
+    return BT->getKind() >= BuiltinType::Half &&
            BT->getKind() <= BuiltinType::LongDouble;
   if (const ComplexType *CT = dyn_cast<ComplexType>(CanonicalType))
     return CT->getElementType()->isFloatingType();
@@ -801,14 +840,16 @@ Type::ScalarTypeKind Type::getScalarTypeKind() const {
   const Type *T = CanonicalType.getTypePtr();
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(T)) {
     if (BT->getKind() == BuiltinType::Bool) return STK_Bool;
-    if (BT->getKind() == BuiltinType::NullPtr) return STK_Pointer;
+    if (BT->getKind() == BuiltinType::NullPtr) return STK_CPointer;
     if (BT->isInteger()) return STK_Integral;
     if (BT->isFloatingPoint()) return STK_Floating;
     llvm_unreachable("unknown scalar builtin type");
-  } else if (isa<PointerType>(T) ||
-             isa<BlockPointerType>(T) ||
-             isa<ObjCObjectPointerType>(T)) {
-    return STK_Pointer;
+  } else if (isa<PointerType>(T)) {
+    return STK_CPointer;
+  } else if (isa<BlockPointerType>(T)) {
+    return STK_BlockPointer;
+  } else if (isa<ObjCObjectPointerType>(T)) {
+    return STK_ObjCObjectPointer;
   } else if (isa<MemberPointerType>(T)) {
     return STK_MemberPointer;
   } else if (isa<EnumType>(T)) {
@@ -821,7 +862,6 @@ Type::ScalarTypeKind Type::getScalarTypeKind() const {
   }
 
   llvm_unreachable("unknown scalar type");
-  return STK_Pointer;
 }
 
 /// \brief Determines whether the type is a C++ aggregate type or C
@@ -872,7 +912,7 @@ bool Type::isIncompleteType() const {
   case Record:
     // A tagged type (struct/union/enum/class) is incomplete if the decl is a
     // forward declaration, but not a full definition (C99 6.2.5p22).
-    return !cast<TagType>(CanonicalType)->getDecl()->isDefinition();
+    return !cast<TagType>(CanonicalType)->getDecl()->isCompleteDefinition();
   case ConstantArray:
     // An array is incomplete if its element type is incomplete
     // (C++ [dcl.array]p1).
@@ -1073,7 +1113,7 @@ bool Type::isLiteralType() const {
   // C++0x [basic.types]p10:
   //   A type is a literal type if it is:
   //   [...]
-  //   -- an array of literal type
+  //   -- an array of literal type.
   // Extension: variable arrays cannot be literal types, since they're
   // runtime-sized.
   if (isVariableArrayType())
@@ -1093,33 +1133,31 @@ bool Type::isLiteralType() const {
   // C++0x [basic.types]p10:
   //   A type is a literal type if it is:
   //    -- a scalar type; or
-  // As an extension, Clang treats vector types as Scalar types.
-  if (BaseTy->isScalarType() || BaseTy->isVectorType()) return true;
+  // As an extension, Clang treats vector types as literal types.
+  if (BaseTy->isScalarType() || BaseTy->isVectorType())
+    return true;
   //    -- a reference type; or
-  if (BaseTy->isReferenceType()) return true;
+  if (BaseTy->isReferenceType())
+    return true;
   //    -- a class type that has all of the following properties:
   if (const RecordType *RT = BaseTy->getAs<RecordType>()) {
+    //    -- a trivial destructor,
+    //    -- every constructor call and full-expression in the
+    //       brace-or-equal-initializers for non-static data members (if any)
+    //       is a constant expression,
+    //    -- it is an aggregate type or has at least one constexpr
+    //       constructor or constructor template that is not a copy or move
+    //       constructor, and
+    //    -- all non-static data members and base classes of literal types
+    //
+    // We resolve DR1361 by ignoring the second bullet.
     if (const CXXRecordDecl *ClassDecl =
-        dyn_cast<CXXRecordDecl>(RT->getDecl())) {
-      //    -- a trivial destructor,
-      if (!ClassDecl->hasTrivialDestructor()) return false;
-      //    -- every constructor call and full-expression in the
-      //       brace-or-equal-initializers for non-static data members (if any)
-      //       is a constant expression,
-      // FIXME: C++0x: Clang doesn't yet support non-static data member
-      // declarations with initializers, or constexprs.
-      //    -- it is an aggregate type or has at least one constexpr
-      //       constructor or constructor template that is not a copy or move
-      //       constructor, and
-      if (!ClassDecl->isAggregate() &&
-          !ClassDecl->hasConstExprNonCopyMoveConstructor())
-        return false;
-      //    -- all non-static data members and base classes of literal types
-      if (ClassDecl->hasNonLiteralTypeFieldsOrBases()) return false;
-    }
+        dyn_cast<CXXRecordDecl>(RT->getDecl()))
+      return ClassDecl->isLiteral();
 
     return true;
   }
+
   return false;
 }
 
@@ -1158,7 +1196,7 @@ bool Type::isStandardLayoutType() const {
 }
 
 // This is effectively the intersection of isTrivialType and
-// isStandardLayoutType. We implement it dircetly to avoid redundant
+// isStandardLayoutType. We implement it directly to avoid redundant
 // conversions from a type to a CXXRecordDecl.
 bool QualType::isCXX11PODType(ASTContext &Context) const {
   const Type *ty = getTypePtr();
@@ -1426,10 +1464,10 @@ const char *Type::getTypeClassName() const {
   return 0;
 }
 
-const char *BuiltinType::getName(const LangOptions &LO) const {
+const char *BuiltinType::getName(const PrintingPolicy &Policy) const {
   switch (getKind()) {
   case Void:              return "void";
-  case Bool:              return LO.Bool ? "bool" : "_Bool";
+  case Bool:              return Policy.Bool ? "bool" : "_Bool";
   case Char_S:            return "char";
   case Char_U:            return "char";
   case SChar:             return "signed char";
@@ -1444,6 +1482,7 @@ const char *BuiltinType::getName(const LangOptions &LO) const {
   case ULong:             return "unsigned long";
   case ULongLong:         return "unsigned long long";
   case UInt128:           return "__uint128_t";
+  case Half:              return "half";
   case Float:             return "float";
   case Double:            return "double";
   case LongDouble:        return "long double";
@@ -1481,7 +1520,7 @@ QualType QualType::getNonLValueExprType(ASTContext &Context) const {
   return *this;
 }
 
-llvm::StringRef FunctionType::getNameForCallConv(CallingConv CC) {
+StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   switch (CC) {
   case CC_Default: 
     llvm_unreachable("no name for default cc");
@@ -1716,7 +1755,7 @@ static TagDecl *getInterestingTagDecl(TagDecl *decl) {
   for (TagDecl::redecl_iterator I = decl->redecls_begin(),
                                 E = decl->redecls_end();
        I != E; ++I) {
-    if (I->isDefinition() || I->isBeingDefined())
+    if (I->isCompleteDefinition() || I->isBeingDefined())
       return *I;
   }
   // If there's no definition (not even in progress), return what we have.
@@ -2082,6 +2121,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
     return Cache::get(cast<ObjCObjectType>(T)->getBaseType());
   case Type::ObjCObjectPointer:
     return Cache::get(cast<ObjCObjectPointerType>(T)->getPointeeType());
+  case Type::Atomic:
+    return Cache::get(cast<AtomicType>(T)->getValueType());
   }
 
   llvm_unreachable("unhandled type class");
@@ -2227,13 +2268,13 @@ QualType::DestructionKind QualType::isDestructedTypeImpl(QualType type) {
   /// with non-trivial destructors.
   const CXXRecordDecl *record =
     type->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
-  if (record && !record->hasTrivialDestructor())
+  if (record && record->hasDefinition() && !record->hasTrivialDestructor())
     return DK_cxx_destructor;
 
   return DK_none;
 }
 
-bool QualType::hasTrivialCopyAssignment(ASTContext &Context) const {
+bool QualType::hasTrivialAssignment(ASTContext &Context, bool Copying) const {
   switch (getObjCLifetime()) {
   case Qualifiers::OCL_None:
     break;
@@ -2249,7 +2290,8 @@ bool QualType::hasTrivialCopyAssignment(ASTContext &Context) const {
   
   if (const CXXRecordDecl *Record 
             = getTypePtr()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl())
-    return Record->hasTrivialCopyAssignment();
+    return Copying ? Record->hasTrivialCopyAssignment() :
+                     Record->hasTrivialMoveAssignment();
   
   return true;
 }
diff --git a/contrib/llvm/tools/clang/lib/AST/TypeLoc.cpp b/contrib/llvm/tools/clang/lib/AST/TypeLoc.cpp
index 34e7693e3075..8e8b227e1f09 100644
--- a/contrib/llvm/tools/clang/lib/AST/TypeLoc.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/TypeLoc.cpp
@@ -206,7 +206,7 @@ TypeSpecifierType BuiltinTypeLoc::getWrittenTypeSpec() const {
     case BuiltinType::Char_S:
       return TST_char;
     case BuiltinType::Char16:
-      return TST_char16;        
+      return TST_char16;
     case BuiltinType::Char32:
       return TST_char32;
     case BuiltinType::WChar_S:
@@ -225,6 +225,7 @@ TypeSpecifierType BuiltinTypeLoc::getWrittenTypeSpec() const {
     case BuiltinType::Long:
     case BuiltinType::LongLong:
     case BuiltinType::Int128:
+    case BuiltinType::Half:
     case BuiltinType::Float:
     case BuiltinType::Double:
     case BuiltinType::LongDouble:
diff --git a/contrib/llvm/tools/clang/lib/AST/TypePrinter.cpp b/contrib/llvm/tools/clang/lib/AST/TypePrinter.cpp
index b89d2aa31676..fb7b918ca2fe 100644
--- a/contrib/llvm/tools/clang/lib/AST/TypePrinter.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/TypePrinter.cpp
@@ -123,6 +123,7 @@ void TypePrinter::print(const Type *T, Qualifiers Quals, std::string &buffer) {
     case Type::DependentTemplateSpecialization:
     case Type::ObjCObject:
     case Type::ObjCInterface:
+    case Type::Atomic:
       CanPrefixQualifiers = true;
       break;
       
@@ -205,11 +206,11 @@ void TypePrinter::print(const Type *T, Qualifiers Quals, std::string &buffer) {
 
 void TypePrinter::printBuiltin(const BuiltinType *T, std::string &S) {
   if (S.empty()) {
-    S = T->getName(Policy.LangOpts);
+    S = T->getName(Policy);
   } else {
     // Prefix the basic type, e.g. 'int X'.
     S = ' ' + S;
-    S = T->getName(Policy.LangOpts) + S;
+    S = T->getName(Policy) + S;
   }
 }
 
@@ -581,6 +582,16 @@ void TypePrinter::printAuto(const AutoType *T, std::string &S) {
   }
 }
 
+void TypePrinter::printAtomic(const AtomicType *T, std::string &S) {
+  if (!S.empty())
+    S = ' ' + S;
+  std::string Str;
+  IncludeStrongLifetimeRAII Strong(Policy);
+  print(T->getValueType(), Str);
+
+  S = "_Atomic(" + Str + ")" + S;
+}
+
 /// Appends the given scope to the end of a string.
 void TypePrinter::AppendScope(DeclContext *DC, std::string &Buffer) {
   if (DC->isTranslationUnit()) return;
diff --git a/contrib/llvm/tools/clang/lib/AST/VTTBuilder.cpp b/contrib/llvm/tools/clang/lib/AST/VTTBuilder.cpp
new file mode 100644
index 000000000000..f5ff624cf069
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/AST/VTTBuilder.cpp
@@ -0,0 +1,212 @@
+//===--- VTTBuilder.cpp - C++ VTT layout builder --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with generation of the layout of virtual table
+// tables (VTT).
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/VTTBuilder.h"
+#include "clang/AST/CXXInheritance.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/Basic/TargetInfo.h"
+#include "llvm/Support/Format.h"
+#include <algorithm>
+#include <cstdio>
+
+using namespace clang;
+
+#define DUMP_OVERRIDERS 0
+
+VTTBuilder::VTTBuilder(ASTContext &Ctx,
+                       const CXXRecordDecl *MostDerivedClass,
+                       bool GenerateDefinition)
+  : Ctx(Ctx), MostDerivedClass(MostDerivedClass), 
+  MostDerivedClassLayout(Ctx.getASTRecordLayout(MostDerivedClass)),
+    GenerateDefinition(GenerateDefinition) {
+  // Lay out this VTT.
+  LayoutVTT(BaseSubobject(MostDerivedClass, CharUnits::Zero()), 
+            /*BaseIsVirtual=*/false);
+}
+
+void VTTBuilder::AddVTablePointer(BaseSubobject Base, uint64_t VTableIndex,
+                                  const CXXRecordDecl *VTableClass) {
+  // Store the vtable pointer index if we're generating the primary VTT.
+  if (VTableClass == MostDerivedClass) {
+    assert(!SecondaryVirtualPointerIndices.count(Base) &&
+           "A virtual pointer index already exists for this base subobject!");
+    SecondaryVirtualPointerIndices[Base] = VTTComponents.size();
+  }
+
+  if (!GenerateDefinition) {
+    VTTComponents.push_back(VTTComponent());
+    return;
+  }
+
+  VTTComponents.push_back(VTTComponent(VTableIndex, Base));
+}
+
+void VTTBuilder::LayoutSecondaryVTTs(BaseSubobject Base) {
+  const CXXRecordDecl *RD = Base.getBase();
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    
+    // Don't layout virtual bases.
+    if (I->isVirtual())
+        continue;
+
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+    CharUnits BaseOffset = Base.getBaseOffset() + 
+      Layout.getBaseClassOffset(BaseDecl);
+   
+    // Layout the VTT for this base.
+    LayoutVTT(BaseSubobject(BaseDecl, BaseOffset), /*BaseIsVirtual=*/false);
+  }
+}
+
+void
+VTTBuilder::LayoutSecondaryVirtualPointers(BaseSubobject Base, 
+                                           bool BaseIsMorallyVirtual,
+                                           uint64_t VTableIndex,
+                                           const CXXRecordDecl *VTableClass,
+                                           VisitedVirtualBasesSetTy &VBases) {
+  const CXXRecordDecl *RD = Base.getBase();
+  
+  // We're not interested in bases that don't have virtual bases, and not
+  // morally virtual bases.
+  if (!RD->getNumVBases() && !BaseIsMorallyVirtual)
+    return;
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Itanium C++ ABI 2.6.2:
+    //   Secondary virtual pointers are present for all bases with either
+    //   virtual bases or virtual function declarations overridden along a 
+    //   virtual path.
+    //
+    // If the base class is not dynamic, we don't want to add it, nor any
+    // of its base classes.
+    if (!BaseDecl->isDynamicClass())
+      continue;
+    
+    bool BaseDeclIsMorallyVirtual = BaseIsMorallyVirtual;
+    bool BaseDeclIsNonVirtualPrimaryBase = false;
+    CharUnits BaseOffset;
+    if (I->isVirtual()) {
+      // Ignore virtual bases that we've already visited.
+      if (!VBases.insert(BaseDecl))
+        continue;
+      
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      BaseDeclIsMorallyVirtual = true;
+    } else {
+      const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+      
+      BaseOffset = Base.getBaseOffset() + 
+        Layout.getBaseClassOffset(BaseDecl);
+      
+      if (!Layout.isPrimaryBaseVirtual() &&
+          Layout.getPrimaryBase() == BaseDecl)
+        BaseDeclIsNonVirtualPrimaryBase = true;
+    }
+
+    // Itanium C++ ABI 2.6.2:
+    //   Secondary virtual pointers: for each base class X which (a) has virtual
+    //   bases or is reachable along a virtual path from D, and (b) is not a
+    //   non-virtual primary base, the address of the virtual table for X-in-D
+    //   or an appropriate construction virtual table.
+    if (!BaseDeclIsNonVirtualPrimaryBase &&
+        (BaseDecl->getNumVBases() || BaseDeclIsMorallyVirtual)) {
+      // Add the vtable pointer.
+      AddVTablePointer(BaseSubobject(BaseDecl, BaseOffset), VTableIndex, 
+                       VTableClass);
+    }
+
+    // And lay out the secondary virtual pointers for the base class.
+    LayoutSecondaryVirtualPointers(BaseSubobject(BaseDecl, BaseOffset),
+                                   BaseDeclIsMorallyVirtual, VTableIndex, 
+                                   VTableClass, VBases);
+  }
+}
+
+void 
+VTTBuilder::LayoutSecondaryVirtualPointers(BaseSubobject Base, 
+                                           uint64_t VTableIndex) {
+  VisitedVirtualBasesSetTy VBases;
+  LayoutSecondaryVirtualPointers(Base, /*BaseIsMorallyVirtual=*/false,
+                                 VTableIndex, Base.getBase(), VBases);
+}
+
+void VTTBuilder::LayoutVirtualVTTs(const CXXRecordDecl *RD,
+                                   VisitedVirtualBasesSetTy &VBases) {
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+    
+    // Check if this is a virtual base.
+    if (I->isVirtual()) {
+      // Check if we've seen this base before.
+      if (!VBases.insert(BaseDecl))
+        continue;
+    
+      CharUnits BaseOffset = 
+        MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      
+      LayoutVTT(BaseSubobject(BaseDecl, BaseOffset), /*BaseIsVirtual=*/true);
+    }
+    
+    // We only need to layout virtual VTTs for this base if it actually has
+    // virtual bases.
+    if (BaseDecl->getNumVBases())
+      LayoutVirtualVTTs(BaseDecl, VBases);
+  }
+}
+
+void VTTBuilder::LayoutVTT(BaseSubobject Base, bool BaseIsVirtual) {
+  const CXXRecordDecl *RD = Base.getBase();
+
+  // Itanium C++ ABI 2.6.2:
+  //   An array of virtual table addresses, called the VTT, is declared for 
+  //   each class type that has indirect or direct virtual base classes.
+  if (RD->getNumVBases() == 0)
+    return;
+
+  bool IsPrimaryVTT = Base.getBase() == MostDerivedClass;
+
+  if (!IsPrimaryVTT) {
+    // Remember the sub-VTT index.
+    SubVTTIndicies[Base] = VTTComponents.size();
+  }
+
+  uint64_t VTableIndex = VTTVTables.size();
+  VTTVTables.push_back(VTTVTable(Base, BaseIsVirtual));
+
+  // Add the primary vtable pointer.
+  AddVTablePointer(Base, VTableIndex, RD);
+
+  // Add the secondary VTTs.
+  LayoutSecondaryVTTs(Base);
+  
+  // Add the secondary virtual pointers.
+  LayoutSecondaryVirtualPointers(Base, VTableIndex);
+  
+  // If this is the primary VTT, we want to lay out virtual VTTs as well.
+  if (IsPrimaryVTT) {
+    VisitedVirtualBasesSetTy VBases;
+    LayoutVirtualVTTs(Base.getBase(), VBases);
+  }
+}
diff --git a/contrib/llvm/tools/clang/lib/AST/VTableBuilder.cpp b/contrib/llvm/tools/clang/lib/AST/VTableBuilder.cpp
new file mode 100644
index 000000000000..7765817759a4
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/AST/VTableBuilder.cpp
@@ -0,0 +1,2404 @@
+//===--- VTableBuilder.cpp - C++ vtable layout builder --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with generation of the layout of virtual tables.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/VTableBuilder.h"
+#include "clang/AST/CXXInheritance.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/Basic/TargetInfo.h"
+#include "llvm/Support/Format.h"
+#include <algorithm>
+#include <cstdio>
+
+using namespace clang;
+
+#define DUMP_OVERRIDERS 0
+
+namespace {
+
+/// BaseOffset - Represents an offset from a derived class to a direct or
+/// indirect base class.
+struct BaseOffset {
+  /// DerivedClass - The derived class.
+  const CXXRecordDecl *DerivedClass;
+  
+  /// VirtualBase - If the path from the derived class to the base class
+  /// involves a virtual base class, this holds its declaration.
+  const CXXRecordDecl *VirtualBase;
+
+  /// NonVirtualOffset - The offset from the derived class to the base class.
+  /// (Or the offset from the virtual base class to the base class, if the 
+  /// path from the derived class to the base class involves a virtual base
+  /// class.
+  CharUnits NonVirtualOffset;
+  
+  BaseOffset() : DerivedClass(0), VirtualBase(0), 
+    NonVirtualOffset(CharUnits::Zero()) { }
+  BaseOffset(const CXXRecordDecl *DerivedClass,
+             const CXXRecordDecl *VirtualBase, CharUnits NonVirtualOffset)
+    : DerivedClass(DerivedClass), VirtualBase(VirtualBase), 
+    NonVirtualOffset(NonVirtualOffset) { }
+
+  bool isEmpty() const { return NonVirtualOffset.isZero() && !VirtualBase; }
+};
+
+/// FinalOverriders - Contains the final overrider member functions for all
+/// member functions in the base subobjects of a class.
+class FinalOverriders {
+public:
+  /// OverriderInfo - Information about a final overrider.
+  struct OverriderInfo {
+    /// Method - The method decl of the overrider.
+    const CXXMethodDecl *Method;
+
+    /// Offset - the base offset of the overrider in the layout class.
+    CharUnits Offset;
+    
+    OverriderInfo() : Method(0), Offset(CharUnits::Zero()) { }
+  };
+
+private:
+  /// MostDerivedClass - The most derived class for which the final overriders
+  /// are stored.
+  const CXXRecordDecl *MostDerivedClass;
+  
+  /// MostDerivedClassOffset - If we're building final overriders for a 
+  /// construction vtable, this holds the offset from the layout class to the
+  /// most derived class.
+  const CharUnits MostDerivedClassOffset;
+
+  /// LayoutClass - The class we're using for layout information. Will be 
+  /// different than the most derived class if the final overriders are for a
+  /// construction vtable.  
+  const CXXRecordDecl *LayoutClass;  
+
+  ASTContext &Context;
+  
+  /// MostDerivedClassLayout - the AST record layout of the most derived class.
+  const ASTRecordLayout &MostDerivedClassLayout;
+
+  /// MethodBaseOffsetPairTy - Uniquely identifies a member function
+  /// in a base subobject.
+  typedef std::pair<const CXXMethodDecl *, CharUnits> MethodBaseOffsetPairTy;
+
+  typedef llvm::DenseMap<MethodBaseOffsetPairTy,
+                         OverriderInfo> OverridersMapTy;
+  
+  /// OverridersMap - The final overriders for all virtual member functions of 
+  /// all the base subobjects of the most derived class.
+  OverridersMapTy OverridersMap;
+  
+  /// SubobjectsToOffsetsMapTy - A mapping from a base subobject (represented
+  /// as a record decl and a subobject number) and its offsets in the most
+  /// derived class as well as the layout class.
+  typedef llvm::DenseMap<std::pair<const CXXRecordDecl *, unsigned>, 
+                         CharUnits> SubobjectOffsetMapTy;
+
+  typedef llvm::DenseMap<const CXXRecordDecl *, unsigned> SubobjectCountMapTy;
+  
+  /// ComputeBaseOffsets - Compute the offsets for all base subobjects of the
+  /// given base.
+  void ComputeBaseOffsets(BaseSubobject Base, bool IsVirtual,
+                          CharUnits OffsetInLayoutClass,
+                          SubobjectOffsetMapTy &SubobjectOffsets,
+                          SubobjectOffsetMapTy &SubobjectLayoutClassOffsets,
+                          SubobjectCountMapTy &SubobjectCounts);
+
+  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
+  
+  /// dump - dump the final overriders for a base subobject, and all its direct
+  /// and indirect base subobjects.
+  void dump(raw_ostream &Out, BaseSubobject Base,
+            VisitedVirtualBasesSetTy& VisitedVirtualBases);
+  
+public:
+  FinalOverriders(const CXXRecordDecl *MostDerivedClass,
+                  CharUnits MostDerivedClassOffset,
+                  const CXXRecordDecl *LayoutClass);
+
+  /// getOverrider - Get the final overrider for the given method declaration in
+  /// the subobject with the given base offset. 
+  OverriderInfo getOverrider(const CXXMethodDecl *MD, 
+                             CharUnits BaseOffset) const {
+    assert(OverridersMap.count(std::make_pair(MD, BaseOffset)) && 
+           "Did not find overrider!");
+    
+    return OverridersMap.lookup(std::make_pair(MD, BaseOffset));
+  }
+  
+  /// dump - dump the final overriders.
+  void dump() {
+    VisitedVirtualBasesSetTy VisitedVirtualBases;
+    dump(llvm::errs(), BaseSubobject(MostDerivedClass, CharUnits::Zero()), 
+         VisitedVirtualBases);
+  }
+  
+};
+
+#define DUMP_OVERRIDERS 0
+
+FinalOverriders::FinalOverriders(const CXXRecordDecl *MostDerivedClass,
+                                 CharUnits MostDerivedClassOffset,
+                                 const CXXRecordDecl *LayoutClass)
+  : MostDerivedClass(MostDerivedClass), 
+  MostDerivedClassOffset(MostDerivedClassOffset), LayoutClass(LayoutClass),
+  Context(MostDerivedClass->getASTContext()),
+  MostDerivedClassLayout(Context.getASTRecordLayout(MostDerivedClass)) {
+
+  // Compute base offsets.
+  SubobjectOffsetMapTy SubobjectOffsets;
+  SubobjectOffsetMapTy SubobjectLayoutClassOffsets;
+  SubobjectCountMapTy SubobjectCounts;
+  ComputeBaseOffsets(BaseSubobject(MostDerivedClass, CharUnits::Zero()), 
+                     /*IsVirtual=*/false,
+                     MostDerivedClassOffset, 
+                     SubobjectOffsets, SubobjectLayoutClassOffsets, 
+                     SubobjectCounts);
+
+  // Get the the final overriders.
+  CXXFinalOverriderMap FinalOverriders;
+  MostDerivedClass->getFinalOverriders(FinalOverriders);
+
+  for (CXXFinalOverriderMap::const_iterator I = FinalOverriders.begin(),
+       E = FinalOverriders.end(); I != E; ++I) {
+    const CXXMethodDecl *MD = I->first;
+    const OverridingMethods& Methods = I->second;
+
+    for (OverridingMethods::const_iterator I = Methods.begin(),
+         E = Methods.end(); I != E; ++I) {
+      unsigned SubobjectNumber = I->first;
+      assert(SubobjectOffsets.count(std::make_pair(MD->getParent(), 
+                                                   SubobjectNumber)) &&
+             "Did not find subobject offset!");
+      
+      CharUnits BaseOffset = SubobjectOffsets[std::make_pair(MD->getParent(),
+                                                            SubobjectNumber)];
+
+      assert(I->second.size() == 1 && "Final overrider is not unique!");
+      const UniqueVirtualMethod &Method = I->second.front();
+
+      const CXXRecordDecl *OverriderRD = Method.Method->getParent();
+      assert(SubobjectLayoutClassOffsets.count(
+             std::make_pair(OverriderRD, Method.Subobject))
+             && "Did not find subobject offset!");
+      CharUnits OverriderOffset =
+        SubobjectLayoutClassOffsets[std::make_pair(OverriderRD, 
+                                                   Method.Subobject)];
+
+      OverriderInfo& Overrider = OverridersMap[std::make_pair(MD, BaseOffset)];
+      assert(!Overrider.Method && "Overrider should not exist yet!");
+      
+      Overrider.Offset = OverriderOffset;
+      Overrider.Method = Method.Method;
+    }
+  }
+
+#if DUMP_OVERRIDERS
+  // And dump them (for now).
+  dump();
+#endif
+}
+
+static BaseOffset ComputeBaseOffset(ASTContext &Context, 
+                                    const CXXRecordDecl *DerivedRD,
+                                    const CXXBasePath &Path) {
+  CharUnits NonVirtualOffset = CharUnits::Zero();
+
+  unsigned NonVirtualStart = 0;
+  const CXXRecordDecl *VirtualBase = 0;
+  
+  // First, look for the virtual base class.
+  for (unsigned I = 0, E = Path.size(); I != E; ++I) {
+    const CXXBasePathElement &Element = Path[I];
+    
+    if (Element.Base->isVirtual()) {
+      // FIXME: Can we break when we find the first virtual base?
+      // (If we can't, can't we just iterate over the path in reverse order?)
+      NonVirtualStart = I + 1;
+      QualType VBaseType = Element.Base->getType();
+      VirtualBase = 
+        cast<CXXRecordDecl>(VBaseType->getAs<RecordType>()->getDecl());
+    }
+  }
+  
+  // Now compute the non-virtual offset.
+  for (unsigned I = NonVirtualStart, E = Path.size(); I != E; ++I) {
+    const CXXBasePathElement &Element = Path[I];
+    
+    // Check the base class offset.
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(Element.Class);
+
+    const RecordType *BaseType = Element.Base->getType()->getAs<RecordType>();
+    const CXXRecordDecl *Base = cast<CXXRecordDecl>(BaseType->getDecl());
+
+    NonVirtualOffset += Layout.getBaseClassOffset(Base);
+  }
+  
+  // FIXME: This should probably use CharUnits or something. Maybe we should
+  // even change the base offsets in ASTRecordLayout to be specified in 
+  // CharUnits.
+  return BaseOffset(DerivedRD, VirtualBase, NonVirtualOffset);
+  
+}
+
+static BaseOffset ComputeBaseOffset(ASTContext &Context, 
+                                    const CXXRecordDecl *BaseRD,
+                                    const CXXRecordDecl *DerivedRD) {
+  CXXBasePaths Paths(/*FindAmbiguities=*/false,
+                     /*RecordPaths=*/true, /*DetectVirtual=*/false);
+  
+  if (!const_cast<CXXRecordDecl *>(DerivedRD)->
+      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseRD), Paths)) {
+    llvm_unreachable("Class must be derived from the passed in base class!");
+  }
+
+  return ComputeBaseOffset(Context, DerivedRD, Paths.front());
+}
+
+static BaseOffset
+ComputeReturnAdjustmentBaseOffset(ASTContext &Context, 
+                                  const CXXMethodDecl *DerivedMD,
+                                  const CXXMethodDecl *BaseMD) {
+  const FunctionType *BaseFT = BaseMD->getType()->getAs<FunctionType>();
+  const FunctionType *DerivedFT = DerivedMD->getType()->getAs<FunctionType>();
+  
+  // Canonicalize the return types.
+  CanQualType CanDerivedReturnType = 
+    Context.getCanonicalType(DerivedFT->getResultType());
+  CanQualType CanBaseReturnType = 
+    Context.getCanonicalType(BaseFT->getResultType());
+  
+  assert(CanDerivedReturnType->getTypeClass() == 
+         CanBaseReturnType->getTypeClass() && 
+         "Types must have same type class!");
+  
+  if (CanDerivedReturnType == CanBaseReturnType) {
+    // No adjustment needed.
+    return BaseOffset();
+  }
+  
+  if (isa<ReferenceType>(CanDerivedReturnType)) {
+    CanDerivedReturnType = 
+      CanDerivedReturnType->getAs<ReferenceType>()->getPointeeType();
+    CanBaseReturnType = 
+      CanBaseReturnType->getAs<ReferenceType>()->getPointeeType();
+  } else if (isa<PointerType>(CanDerivedReturnType)) {
+    CanDerivedReturnType = 
+      CanDerivedReturnType->getAs<PointerType>()->getPointeeType();
+    CanBaseReturnType = 
+      CanBaseReturnType->getAs<PointerType>()->getPointeeType();
+  } else {
+    llvm_unreachable("Unexpected return type!");
+  }
+  
+  // We need to compare unqualified types here; consider
+  //   const T *Base::foo();
+  //   T *Derived::foo();
+  if (CanDerivedReturnType.getUnqualifiedType() == 
+      CanBaseReturnType.getUnqualifiedType()) {
+    // No adjustment needed.
+    return BaseOffset();
+  }
+  
+  const CXXRecordDecl *DerivedRD = 
+    cast<CXXRecordDecl>(cast<RecordType>(CanDerivedReturnType)->getDecl());
+  
+  const CXXRecordDecl *BaseRD = 
+    cast<CXXRecordDecl>(cast<RecordType>(CanBaseReturnType)->getDecl());
+
+  return ComputeBaseOffset(Context, BaseRD, DerivedRD);
+}
+
+void 
+FinalOverriders::ComputeBaseOffsets(BaseSubobject Base, bool IsVirtual,
+                              CharUnits OffsetInLayoutClass,
+                              SubobjectOffsetMapTy &SubobjectOffsets,
+                              SubobjectOffsetMapTy &SubobjectLayoutClassOffsets,
+                              SubobjectCountMapTy &SubobjectCounts) {
+  const CXXRecordDecl *RD = Base.getBase();
+  
+  unsigned SubobjectNumber = 0;
+  if (!IsVirtual)
+    SubobjectNumber = ++SubobjectCounts[RD];
+
+  // Set up the subobject to offset mapping.
+  assert(!SubobjectOffsets.count(std::make_pair(RD, SubobjectNumber))
+         && "Subobject offset already exists!");
+  assert(!SubobjectLayoutClassOffsets.count(std::make_pair(RD, SubobjectNumber)) 
+         && "Subobject offset already exists!");
+
+  SubobjectOffsets[std::make_pair(RD, SubobjectNumber)] = Base.getBaseOffset();
+  SubobjectLayoutClassOffsets[std::make_pair(RD, SubobjectNumber)] =
+    OffsetInLayoutClass;
+  
+  // Traverse our bases.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    CharUnits BaseOffset;
+    CharUnits BaseOffsetInLayoutClass;
+    if (I->isVirtual()) {
+      // Check if we've visited this virtual base before.
+      if (SubobjectOffsets.count(std::make_pair(BaseDecl, 0)))
+        continue;
+
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      BaseOffsetInLayoutClass = 
+        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
+    } else {
+      const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+      CharUnits Offset = Layout.getBaseClassOffset(BaseDecl);
+    
+      BaseOffset = Base.getBaseOffset() + Offset;
+      BaseOffsetInLayoutClass = OffsetInLayoutClass + Offset;
+    }
+
+    ComputeBaseOffsets(BaseSubobject(BaseDecl, BaseOffset), 
+                       I->isVirtual(), BaseOffsetInLayoutClass, 
+                       SubobjectOffsets, SubobjectLayoutClassOffsets, 
+                       SubobjectCounts);
+  }
+}
+
+void FinalOverriders::dump(raw_ostream &Out, BaseSubobject Base,
+                           VisitedVirtualBasesSetTy &VisitedVirtualBases) {
+  const CXXRecordDecl *RD = Base.getBase();
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+    
+    // Ignore bases that don't have any virtual member functions.
+    if (!BaseDecl->isPolymorphic())
+      continue;
+
+    CharUnits BaseOffset;
+    if (I->isVirtual()) {
+      if (!VisitedVirtualBases.insert(BaseDecl)) {
+        // We've visited this base before.
+        continue;
+      }
+      
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+    } else {
+      BaseOffset = Layout.getBaseClassOffset(BaseDecl) + Base.getBaseOffset();
+    }
+
+    dump(Out, BaseSubobject(BaseDecl, BaseOffset), VisitedVirtualBases);
+  }
+
+  Out << "Final overriders for (" << RD->getQualifiedNameAsString() << ", ";
+  Out << Base.getBaseOffset().getQuantity() << ")\n";
+
+  // Now dump the overriders for this base subobject.
+  for (CXXRecordDecl::method_iterator I = RD->method_begin(), 
+       E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *MD = *I;
+
+    if (!MD->isVirtual())
+      continue;
+  
+    OverriderInfo Overrider = getOverrider(MD, Base.getBaseOffset());
+
+    Out << "  " << MD->getQualifiedNameAsString() << " - (";
+    Out << Overrider.Method->getQualifiedNameAsString();
+    Out << ", " << ", " << Overrider.Offset.getQuantity() << ')';
+
+    BaseOffset Offset;
+    if (!Overrider.Method->isPure())
+      Offset = ComputeReturnAdjustmentBaseOffset(Context, Overrider.Method, MD);
+
+    if (!Offset.isEmpty()) {
+      Out << " [ret-adj: ";
+      if (Offset.VirtualBase)
+        Out << Offset.VirtualBase->getQualifiedNameAsString() << " vbase, ";
+             
+      Out << Offset.NonVirtualOffset.getQuantity() << " nv]";
+    }
+    
+    Out << "\n";
+  }  
+}
+
+/// VCallOffsetMap - Keeps track of vcall offsets when building a vtable.
+struct VCallOffsetMap {
+  
+  typedef std::pair<const CXXMethodDecl *, CharUnits> MethodAndOffsetPairTy;
+  
+  /// Offsets - Keeps track of methods and their offsets.
+  // FIXME: This should be a real map and not a vector.
+  SmallVector<MethodAndOffsetPairTy, 16> Offsets;
+
+  /// MethodsCanShareVCallOffset - Returns whether two virtual member functions
+  /// can share the same vcall offset.
+  static bool MethodsCanShareVCallOffset(const CXXMethodDecl *LHS,
+                                         const CXXMethodDecl *RHS);
+
+public:
+  /// AddVCallOffset - Adds a vcall offset to the map. Returns true if the
+  /// add was successful, or false if there was already a member function with
+  /// the same signature in the map.
+  bool AddVCallOffset(const CXXMethodDecl *MD, CharUnits OffsetOffset);
+  
+  /// getVCallOffsetOffset - Returns the vcall offset offset (relative to the
+  /// vtable address point) for the given virtual member function.
+  CharUnits getVCallOffsetOffset(const CXXMethodDecl *MD);
+  
+  // empty - Return whether the offset map is empty or not.
+  bool empty() const { return Offsets.empty(); }
+};
+
+static bool HasSameVirtualSignature(const CXXMethodDecl *LHS,
+                                    const CXXMethodDecl *RHS) {
+  ASTContext &C = LHS->getASTContext(); // TODO: thread this down
+  CanQual<FunctionProtoType>
+    LT = C.getCanonicalType(LHS->getType()).getAs<FunctionProtoType>(),
+    RT = C.getCanonicalType(RHS->getType()).getAs<FunctionProtoType>();
+
+  // Fast-path matches in the canonical types.
+  if (LT == RT) return true;
+
+  // Force the signatures to match.  We can't rely on the overrides
+  // list here because there isn't necessarily an inheritance
+  // relationship between the two methods.
+  if (LT.getQualifiers() != RT.getQualifiers() ||
+      LT->getNumArgs() != RT->getNumArgs())
+    return false;
+  for (unsigned I = 0, E = LT->getNumArgs(); I != E; ++I)
+    if (LT->getArgType(I) != RT->getArgType(I))
+      return false;
+  return true;
+}
+
+bool VCallOffsetMap::MethodsCanShareVCallOffset(const CXXMethodDecl *LHS,
+                                                const CXXMethodDecl *RHS) {
+  assert(LHS->isVirtual() && "LHS must be virtual!");
+  assert(RHS->isVirtual() && "LHS must be virtual!");
+  
+  // A destructor can share a vcall offset with another destructor.
+  if (isa<CXXDestructorDecl>(LHS))
+    return isa<CXXDestructorDecl>(RHS);
+
+  // FIXME: We need to check more things here.
+  
+  // The methods must have the same name.
+  DeclarationName LHSName = LHS->getDeclName();
+  DeclarationName RHSName = RHS->getDeclName();
+  if (LHSName != RHSName)
+    return false;
+
+  // And the same signatures.
+  return HasSameVirtualSignature(LHS, RHS);
+}
+
+bool VCallOffsetMap::AddVCallOffset(const CXXMethodDecl *MD, 
+                                    CharUnits OffsetOffset) {
+  // Check if we can reuse an offset.
+  for (unsigned I = 0, E = Offsets.size(); I != E; ++I) {
+    if (MethodsCanShareVCallOffset(Offsets[I].first, MD))
+      return false;
+  }
+  
+  // Add the offset.
+  Offsets.push_back(MethodAndOffsetPairTy(MD, OffsetOffset));
+  return true;
+}
+
+CharUnits VCallOffsetMap::getVCallOffsetOffset(const CXXMethodDecl *MD) {
+  // Look for an offset.
+  for (unsigned I = 0, E = Offsets.size(); I != E; ++I) {
+    if (MethodsCanShareVCallOffset(Offsets[I].first, MD))
+      return Offsets[I].second;
+  }
+  
+  llvm_unreachable("Should always find a vcall offset offset!");
+}
+
+/// VCallAndVBaseOffsetBuilder - Class for building vcall and vbase offsets.
+class VCallAndVBaseOffsetBuilder {
+public:
+  typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> 
+    VBaseOffsetOffsetsMapTy;
+
+private:
+  /// MostDerivedClass - The most derived class for which we're building vcall
+  /// and vbase offsets.
+  const CXXRecordDecl *MostDerivedClass;
+  
+  /// LayoutClass - The class we're using for layout information. Will be 
+  /// different than the most derived class if we're building a construction
+  /// vtable.
+  const CXXRecordDecl *LayoutClass;
+  
+  /// Context - The ASTContext which we will use for layout information.
+  ASTContext &Context;
+
+  /// Components - vcall and vbase offset components
+  typedef SmallVector<VTableComponent, 64> VTableComponentVectorTy;
+  VTableComponentVectorTy Components;
+  
+  /// VisitedVirtualBases - Visited virtual bases.
+  llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBases;
+  
+  /// VCallOffsets - Keeps track of vcall offsets.
+  VCallOffsetMap VCallOffsets;
+
+
+  /// VBaseOffsetOffsets - Contains the offsets of the virtual base offsets,
+  /// relative to the address point.
+  VBaseOffsetOffsetsMapTy VBaseOffsetOffsets;
+  
+  /// FinalOverriders - The final overriders of the most derived class.
+  /// (Can be null when we're not building a vtable of the most derived class).
+  const FinalOverriders *Overriders;
+
+  /// AddVCallAndVBaseOffsets - Add vcall offsets and vbase offsets for the
+  /// given base subobject.
+  void AddVCallAndVBaseOffsets(BaseSubobject Base, bool BaseIsVirtual,
+                               CharUnits RealBaseOffset);
+  
+  /// AddVCallOffsets - Add vcall offsets for the given base subobject.
+  void AddVCallOffsets(BaseSubobject Base, CharUnits VBaseOffset);
+  
+  /// AddVBaseOffsets - Add vbase offsets for the given class.
+  void AddVBaseOffsets(const CXXRecordDecl *Base, 
+                       CharUnits OffsetInLayoutClass);
+  
+  /// getCurrentOffsetOffset - Get the current vcall or vbase offset offset in
+  /// chars, relative to the vtable address point.
+  CharUnits getCurrentOffsetOffset() const;
+  
+public:
+  VCallAndVBaseOffsetBuilder(const CXXRecordDecl *MostDerivedClass,
+                             const CXXRecordDecl *LayoutClass,
+                             const FinalOverriders *Overriders,
+                             BaseSubobject Base, bool BaseIsVirtual,
+                             CharUnits OffsetInLayoutClass)
+    : MostDerivedClass(MostDerivedClass), LayoutClass(LayoutClass), 
+    Context(MostDerivedClass->getASTContext()), Overriders(Overriders) {
+      
+    // Add vcall and vbase offsets.
+    AddVCallAndVBaseOffsets(Base, BaseIsVirtual, OffsetInLayoutClass);
+  }
+  
+  /// Methods for iterating over the components.
+  typedef VTableComponentVectorTy::const_reverse_iterator const_iterator;
+  const_iterator components_begin() const { return Components.rbegin(); }
+  const_iterator components_end() const { return Components.rend(); }
+  
+  const VCallOffsetMap &getVCallOffsets() const { return VCallOffsets; }
+  const VBaseOffsetOffsetsMapTy &getVBaseOffsetOffsets() const {
+    return VBaseOffsetOffsets;
+  }
+};
+  
+void 
+VCallAndVBaseOffsetBuilder::AddVCallAndVBaseOffsets(BaseSubobject Base,
+                                                    bool BaseIsVirtual,
+                                                    CharUnits RealBaseOffset) {
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(Base.getBase());
+  
+  // Itanium C++ ABI 2.5.2:
+  //   ..in classes sharing a virtual table with a primary base class, the vcall
+  //   and vbase offsets added by the derived class all come before the vcall
+  //   and vbase offsets required by the base class, so that the latter may be
+  //   laid out as required by the base class without regard to additions from
+  //   the derived class(es).
+
+  // (Since we're emitting the vcall and vbase offsets in reverse order, we'll
+  // emit them for the primary base first).
+  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
+    bool PrimaryBaseIsVirtual = Layout.isPrimaryBaseVirtual();
+
+    CharUnits PrimaryBaseOffset;
+    
+    // Get the base offset of the primary base.
+    if (PrimaryBaseIsVirtual) {
+      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 &&
+             "Primary vbase should have a zero offset!");
+      
+      const ASTRecordLayout &MostDerivedClassLayout =
+        Context.getASTRecordLayout(MostDerivedClass);
+      
+      PrimaryBaseOffset = 
+        MostDerivedClassLayout.getVBaseClassOffset(PrimaryBase);
+    } else {
+      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
+             "Primary base should have a zero offset!");
+
+      PrimaryBaseOffset = Base.getBaseOffset();
+    }
+
+    AddVCallAndVBaseOffsets(
+      BaseSubobject(PrimaryBase,PrimaryBaseOffset),
+      PrimaryBaseIsVirtual, RealBaseOffset);
+  }
+
+  AddVBaseOffsets(Base.getBase(), RealBaseOffset);
+
+  // We only want to add vcall offsets for virtual bases.
+  if (BaseIsVirtual)
+    AddVCallOffsets(Base, RealBaseOffset);
+}
+
+CharUnits VCallAndVBaseOffsetBuilder::getCurrentOffsetOffset() const {
+  // OffsetIndex is the index of this vcall or vbase offset, relative to the 
+  // vtable address point. (We subtract 3 to account for the information just
+  // above the address point, the RTTI info, the offset to top, and the
+  // vcall offset itself).
+  int64_t OffsetIndex = -(int64_t)(3 + Components.size());
+    
+  CharUnits PointerWidth = 
+    Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
+  CharUnits OffsetOffset = PointerWidth * OffsetIndex;
+  return OffsetOffset;
+}
+
+void VCallAndVBaseOffsetBuilder::AddVCallOffsets(BaseSubobject Base, 
+                                                 CharUnits VBaseOffset) {
+  const CXXRecordDecl *RD = Base.getBase();
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+
+  // Handle the primary base first.
+  // We only want to add vcall offsets if the base is non-virtual; a virtual
+  // primary base will have its vcall and vbase offsets emitted already.
+  if (PrimaryBase && !Layout.isPrimaryBaseVirtual()) {
+    // Get the base offset of the primary base.
+    assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
+           "Primary base should have a zero offset!");
+
+    AddVCallOffsets(BaseSubobject(PrimaryBase, Base.getBaseOffset()),
+                    VBaseOffset);
+  }
+  
+  // Add the vcall offsets.
+  for (CXXRecordDecl::method_iterator I = RD->method_begin(),
+       E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *MD = *I;
+    
+    if (!MD->isVirtual())
+      continue;
+
+    CharUnits OffsetOffset = getCurrentOffsetOffset();
+    
+    // Don't add a vcall offset if we already have one for this member function
+    // signature.
+    if (!VCallOffsets.AddVCallOffset(MD, OffsetOffset))
+      continue;
+
+    CharUnits Offset = CharUnits::Zero();
+
+    if (Overriders) {
+      // Get the final overrider.
+      FinalOverriders::OverriderInfo Overrider = 
+        Overriders->getOverrider(MD, Base.getBaseOffset());
+      
+      /// The vcall offset is the offset from the virtual base to the object 
+      /// where the function was overridden.
+      Offset = Overrider.Offset - VBaseOffset;
+    }
+    
+    Components.push_back(
+      VTableComponent::MakeVCallOffset(Offset));
+  }
+
+  // And iterate over all non-virtual bases (ignoring the primary base).
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+  
+    if (I->isVirtual())
+      continue;
+
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+    if (BaseDecl == PrimaryBase)
+      continue;
+
+    // Get the base offset of this base.
+    CharUnits BaseOffset = Base.getBaseOffset() + 
+      Layout.getBaseClassOffset(BaseDecl);
+    
+    AddVCallOffsets(BaseSubobject(BaseDecl, BaseOffset), 
+                    VBaseOffset);
+  }
+}
+
+void 
+VCallAndVBaseOffsetBuilder::AddVBaseOffsets(const CXXRecordDecl *RD,
+                                            CharUnits OffsetInLayoutClass) {
+  const ASTRecordLayout &LayoutClassLayout = 
+    Context.getASTRecordLayout(LayoutClass);
+
+  // Add vbase offsets.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Check if this is a virtual base that we haven't visited before.
+    if (I->isVirtual() && VisitedVirtualBases.insert(BaseDecl)) {
+      CharUnits Offset = 
+        LayoutClassLayout.getVBaseClassOffset(BaseDecl) - OffsetInLayoutClass;
+
+      // Add the vbase offset offset.
+      assert(!VBaseOffsetOffsets.count(BaseDecl) &&
+             "vbase offset offset already exists!");
+
+      CharUnits VBaseOffsetOffset = getCurrentOffsetOffset();
+      VBaseOffsetOffsets.insert(
+          std::make_pair(BaseDecl, VBaseOffsetOffset));
+
+      Components.push_back(
+          VTableComponent::MakeVBaseOffset(Offset));
+    }
+
+    // Check the base class looking for more vbase offsets.
+    AddVBaseOffsets(BaseDecl, OffsetInLayoutClass);
+  }
+}
+
+/// VTableBuilder - Class for building vtable layout information.
+class VTableBuilder {
+public:
+  /// PrimaryBasesSetVectorTy - A set vector of direct and indirect 
+  /// primary bases.
+  typedef llvm::SmallSetVector<const CXXRecordDecl *, 8> 
+    PrimaryBasesSetVectorTy;
+  
+  typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> 
+    VBaseOffsetOffsetsMapTy;
+  
+  typedef llvm::DenseMap<BaseSubobject, uint64_t> 
+    AddressPointsMapTy;
+
+private:
+  /// VTables - Global vtable information.
+  VTableContext &VTables;
+  
+  /// MostDerivedClass - The most derived class for which we're building this
+  /// vtable.
+  const CXXRecordDecl *MostDerivedClass;
+
+  /// MostDerivedClassOffset - If we're building a construction vtable, this
+  /// holds the offset from the layout class to the most derived class.
+  const CharUnits MostDerivedClassOffset;
+  
+  /// MostDerivedClassIsVirtual - Whether the most derived class is a virtual 
+  /// base. (This only makes sense when building a construction vtable).
+  bool MostDerivedClassIsVirtual;
+  
+  /// LayoutClass - The class we're using for layout information. Will be 
+  /// different than the most derived class if we're building a construction
+  /// vtable.
+  const CXXRecordDecl *LayoutClass;
+  
+  /// Context - The ASTContext which we will use for layout information.
+  ASTContext &Context;
+  
+  /// FinalOverriders - The final overriders of the most derived class.
+  const FinalOverriders Overriders;
+
+  /// VCallOffsetsForVBases - Keeps track of vcall offsets for the virtual
+  /// bases in this vtable.
+  llvm::DenseMap<const CXXRecordDecl *, VCallOffsetMap> VCallOffsetsForVBases;
+
+  /// VBaseOffsetOffsets - Contains the offsets of the virtual base offsets for
+  /// the most derived class.
+  VBaseOffsetOffsetsMapTy VBaseOffsetOffsets;
+  
+  /// Components - The components of the vtable being built.
+  SmallVector<VTableComponent, 64> Components;
+
+  /// AddressPoints - Address points for the vtable being built.
+  AddressPointsMapTy AddressPoints;
+
+  /// MethodInfo - Contains information about a method in a vtable.
+  /// (Used for computing 'this' pointer adjustment thunks.
+  struct MethodInfo {
+    /// BaseOffset - The base offset of this method.
+    const CharUnits BaseOffset;
+    
+    /// BaseOffsetInLayoutClass - The base offset in the layout class of this
+    /// method.
+    const CharUnits BaseOffsetInLayoutClass;
+    
+    /// VTableIndex - The index in the vtable that this method has.
+    /// (For destructors, this is the index of the complete destructor).
+    const uint64_t VTableIndex;
+    
+    MethodInfo(CharUnits BaseOffset, CharUnits BaseOffsetInLayoutClass, 
+               uint64_t VTableIndex)
+      : BaseOffset(BaseOffset), 
+      BaseOffsetInLayoutClass(BaseOffsetInLayoutClass),
+      VTableIndex(VTableIndex) { }
+    
+    MethodInfo() 
+      : BaseOffset(CharUnits::Zero()), 
+      BaseOffsetInLayoutClass(CharUnits::Zero()), 
+      VTableIndex(0) { }
+  };
+  
+  typedef llvm::DenseMap<const CXXMethodDecl *, MethodInfo> MethodInfoMapTy;
+  
+  /// MethodInfoMap - The information for all methods in the vtable we're
+  /// currently building.
+  MethodInfoMapTy MethodInfoMap;
+  
+  typedef llvm::DenseMap<uint64_t, ThunkInfo> VTableThunksMapTy;
+  
+  /// VTableThunks - The thunks by vtable index in the vtable currently being 
+  /// built.
+  VTableThunksMapTy VTableThunks;
+
+  typedef SmallVector<ThunkInfo, 1> ThunkInfoVectorTy;
+  typedef llvm::DenseMap<const CXXMethodDecl *, ThunkInfoVectorTy> ThunksMapTy;
+  
+  /// Thunks - A map that contains all the thunks needed for all methods in the
+  /// most derived class for which the vtable is currently being built.
+  ThunksMapTy Thunks;
+  
+  /// AddThunk - Add a thunk for the given method.
+  void AddThunk(const CXXMethodDecl *MD, const ThunkInfo &Thunk);
+  
+  /// ComputeThisAdjustments - Compute the 'this' pointer adjustments for the
+  /// part of the vtable we're currently building.
+  void ComputeThisAdjustments();
+  
+  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
+
+  /// PrimaryVirtualBases - All known virtual bases who are a primary base of
+  /// some other base.
+  VisitedVirtualBasesSetTy PrimaryVirtualBases;
+
+  /// ComputeReturnAdjustment - Compute the return adjustment given a return
+  /// adjustment base offset.
+  ReturnAdjustment ComputeReturnAdjustment(BaseOffset Offset);
+  
+  /// ComputeThisAdjustmentBaseOffset - Compute the base offset for adjusting
+  /// the 'this' pointer from the base subobject to the derived subobject.
+  BaseOffset ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
+                                             BaseSubobject Derived) const;
+
+  /// ComputeThisAdjustment - Compute the 'this' pointer adjustment for the
+  /// given virtual member function, its offset in the layout class and its
+  /// final overrider.
+  ThisAdjustment 
+  ComputeThisAdjustment(const CXXMethodDecl *MD, 
+                        CharUnits BaseOffsetInLayoutClass,
+                        FinalOverriders::OverriderInfo Overrider);
+
+  /// AddMethod - Add a single virtual member function to the vtable
+  /// components vector.
+  void AddMethod(const CXXMethodDecl *MD, ReturnAdjustment ReturnAdjustment);
+
+  /// IsOverriderUsed - Returns whether the overrider will ever be used in this
+  /// part of the vtable. 
+  ///
+  /// Itanium C++ ABI 2.5.2:
+  ///
+  ///   struct A { virtual void f(); };
+  ///   struct B : virtual public A { int i; };
+  ///   struct C : virtual public A { int j; };
+  ///   struct D : public B, public C {};
+  ///
+  ///   When B and C are declared, A is a primary base in each case, so although
+  ///   vcall offsets are allocated in the A-in-B and A-in-C vtables, no this
+  ///   adjustment is required and no thunk is generated. However, inside D
+  ///   objects, A is no longer a primary base of C, so if we allowed calls to
+  ///   C::f() to use the copy of A's vtable in the C subobject, we would need
+  ///   to adjust this from C* to B::A*, which would require a third-party 
+  ///   thunk. Since we require that a call to C::f() first convert to A*, 
+  ///   C-in-D's copy of A's vtable is never referenced, so this is not 
+  ///   necessary.
+  bool IsOverriderUsed(const CXXMethodDecl *Overrider,
+                       CharUnits BaseOffsetInLayoutClass,
+                       const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
+                       CharUnits FirstBaseOffsetInLayoutClass) const;
+
+  
+  /// AddMethods - Add the methods of this base subobject and all its
+  /// primary bases to the vtable components vector.
+  void AddMethods(BaseSubobject Base, CharUnits BaseOffsetInLayoutClass,
+                  const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
+                  CharUnits FirstBaseOffsetInLayoutClass,
+                  PrimaryBasesSetVectorTy &PrimaryBases);
+
+  // LayoutVTable - Layout the vtable for the given base class, including its
+  // secondary vtables and any vtables for virtual bases.
+  void LayoutVTable();
+
+  /// LayoutPrimaryAndSecondaryVTables - Layout the primary vtable for the
+  /// given base subobject, as well as all its secondary vtables.
+  ///
+  /// \param BaseIsMorallyVirtual whether the base subobject is a virtual base
+  /// or a direct or indirect base of a virtual base.
+  ///
+  /// \param BaseIsVirtualInLayoutClass - Whether the base subobject is virtual
+  /// in the layout class. 
+  void LayoutPrimaryAndSecondaryVTables(BaseSubobject Base,
+                                        bool BaseIsMorallyVirtual,
+                                        bool BaseIsVirtualInLayoutClass,
+                                        CharUnits OffsetInLayoutClass);
+  
+  /// LayoutSecondaryVTables - Layout the secondary vtables for the given base
+  /// subobject.
+  ///
+  /// \param BaseIsMorallyVirtual whether the base subobject is a virtual base
+  /// or a direct or indirect base of a virtual base.
+  void LayoutSecondaryVTables(BaseSubobject Base, bool BaseIsMorallyVirtual,
+                              CharUnits OffsetInLayoutClass);
+
+  /// DeterminePrimaryVirtualBases - Determine the primary virtual bases in this
+  /// class hierarchy.
+  void DeterminePrimaryVirtualBases(const CXXRecordDecl *RD, 
+                                    CharUnits OffsetInLayoutClass,
+                                    VisitedVirtualBasesSetTy &VBases);
+
+  /// LayoutVTablesForVirtualBases - Layout vtables for all virtual bases of the
+  /// given base (excluding any primary bases).
+  void LayoutVTablesForVirtualBases(const CXXRecordDecl *RD, 
+                                    VisitedVirtualBasesSetTy &VBases);
+
+  /// isBuildingConstructionVTable - Return whether this vtable builder is
+  /// building a construction vtable.
+  bool isBuildingConstructorVTable() const { 
+    return MostDerivedClass != LayoutClass;
+  }
+
+public:
+  VTableBuilder(VTableContext &VTables, const CXXRecordDecl *MostDerivedClass,
+                CharUnits MostDerivedClassOffset, 
+                bool MostDerivedClassIsVirtual, const 
+                CXXRecordDecl *LayoutClass)
+    : VTables(VTables), MostDerivedClass(MostDerivedClass),
+    MostDerivedClassOffset(MostDerivedClassOffset), 
+    MostDerivedClassIsVirtual(MostDerivedClassIsVirtual), 
+    LayoutClass(LayoutClass), Context(MostDerivedClass->getASTContext()), 
+    Overriders(MostDerivedClass, MostDerivedClassOffset, LayoutClass) {
+
+    LayoutVTable();
+
+    if (Context.getLangOptions().DumpVTableLayouts)
+      dumpLayout(llvm::errs());
+  }
+
+  uint64_t getNumThunks() const {
+    return Thunks.size();
+  }
+
+  ThunksMapTy::const_iterator thunks_begin() const {
+    return Thunks.begin();
+  }
+
+  ThunksMapTy::const_iterator thunks_end() const {
+    return Thunks.end();
+  }
+
+  const VBaseOffsetOffsetsMapTy &getVBaseOffsetOffsets() const {
+    return VBaseOffsetOffsets;
+  }
+
+  const AddressPointsMapTy &getAddressPoints() const {
+    return AddressPoints;
+  }
+
+  /// getNumVTableComponents - Return the number of components in the vtable
+  /// currently built.
+  uint64_t getNumVTableComponents() const {
+    return Components.size();
+  }
+
+  const VTableComponent *vtable_component_begin() const {
+    return Components.begin();
+  }
+  
+  const VTableComponent *vtable_component_end() const {
+    return Components.end();
+  }
+  
+  AddressPointsMapTy::const_iterator address_points_begin() const {
+    return AddressPoints.begin();
+  }
+
+  AddressPointsMapTy::const_iterator address_points_end() const {
+    return AddressPoints.end();
+  }
+
+  VTableThunksMapTy::const_iterator vtable_thunks_begin() const {
+    return VTableThunks.begin();
+  }
+
+  VTableThunksMapTy::const_iterator vtable_thunks_end() const {
+    return VTableThunks.end();
+  }
+
+  /// dumpLayout - Dump the vtable layout.
+  void dumpLayout(raw_ostream&);
+};
+
+void VTableBuilder::AddThunk(const CXXMethodDecl *MD, const ThunkInfo &Thunk) {
+  assert(!isBuildingConstructorVTable() && 
+         "Can't add thunks for construction vtable");
+
+  SmallVector<ThunkInfo, 1> &ThunksVector = Thunks[MD];
+  
+  // Check if we have this thunk already.
+  if (std::find(ThunksVector.begin(), ThunksVector.end(), Thunk) != 
+      ThunksVector.end())
+    return;
+  
+  ThunksVector.push_back(Thunk);
+}
+
+typedef llvm::SmallPtrSet<const CXXMethodDecl *, 8> OverriddenMethodsSetTy;
+
+/// ComputeAllOverriddenMethods - Given a method decl, will return a set of all
+/// the overridden methods that the function decl overrides.
+static void 
+ComputeAllOverriddenMethods(const CXXMethodDecl *MD,
+                            OverriddenMethodsSetTy& OverriddenMethods) {
+  assert(MD->isVirtual() && "Method is not virtual!");
+
+  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+       E = MD->end_overridden_methods(); I != E; ++I) {
+    const CXXMethodDecl *OverriddenMD = *I;
+    
+    OverriddenMethods.insert(OverriddenMD);
+    
+    ComputeAllOverriddenMethods(OverriddenMD, OverriddenMethods);
+  }
+}
+
+void VTableBuilder::ComputeThisAdjustments() {
+  // Now go through the method info map and see if any of the methods need
+  // 'this' pointer adjustments.
+  for (MethodInfoMapTy::const_iterator I = MethodInfoMap.begin(),
+       E = MethodInfoMap.end(); I != E; ++I) {
+    const CXXMethodDecl *MD = I->first;
+    const MethodInfo &MethodInfo = I->second;
+
+    // Ignore adjustments for unused function pointers.
+    uint64_t VTableIndex = MethodInfo.VTableIndex;
+    if (Components[VTableIndex].getKind() == 
+        VTableComponent::CK_UnusedFunctionPointer)
+      continue;
+    
+    // Get the final overrider for this method.
+    FinalOverriders::OverriderInfo Overrider =
+      Overriders.getOverrider(MD, MethodInfo.BaseOffset);
+    
+    // Check if we need an adjustment at all.
+    if (MethodInfo.BaseOffsetInLayoutClass == Overrider.Offset) {
+      // When a return thunk is needed by a derived class that overrides a
+      // virtual base, gcc uses a virtual 'this' adjustment as well. 
+      // While the thunk itself might be needed by vtables in subclasses or
+      // in construction vtables, there doesn't seem to be a reason for using
+      // the thunk in this vtable. Still, we do so to match gcc.
+      if (VTableThunks.lookup(VTableIndex).Return.isEmpty())
+        continue;
+    }
+
+    ThisAdjustment ThisAdjustment =
+      ComputeThisAdjustment(MD, MethodInfo.BaseOffsetInLayoutClass, Overrider);
+
+    if (ThisAdjustment.isEmpty())
+      continue;
+
+    // Add it.
+    VTableThunks[VTableIndex].This = ThisAdjustment;
+
+    if (isa<CXXDestructorDecl>(MD)) {
+      // Add an adjustment for the deleting destructor as well.
+      VTableThunks[VTableIndex + 1].This = ThisAdjustment;
+    }
+  }
+
+  /// Clear the method info map.
+  MethodInfoMap.clear();
+  
+  if (isBuildingConstructorVTable()) {
+    // We don't need to store thunk information for construction vtables.
+    return;
+  }
+
+  for (VTableThunksMapTy::const_iterator I = VTableThunks.begin(),
+       E = VTableThunks.end(); I != E; ++I) {
+    const VTableComponent &Component = Components[I->first];
+    const ThunkInfo &Thunk = I->second;
+    const CXXMethodDecl *MD;
+    
+    switch (Component.getKind()) {
+    default:
+      llvm_unreachable("Unexpected vtable component kind!");
+    case VTableComponent::CK_FunctionPointer:
+      MD = Component.getFunctionDecl();
+      break;
+    case VTableComponent::CK_CompleteDtorPointer:
+      MD = Component.getDestructorDecl();
+      break;
+    case VTableComponent::CK_DeletingDtorPointer:
+      // We've already added the thunk when we saw the complete dtor pointer.
+      continue;
+    }
+
+    if (MD->getParent() == MostDerivedClass)
+      AddThunk(MD, Thunk);
+  }
+}
+
+ReturnAdjustment VTableBuilder::ComputeReturnAdjustment(BaseOffset Offset) {
+  ReturnAdjustment Adjustment;
+  
+  if (!Offset.isEmpty()) {
+    if (Offset.VirtualBase) {
+      // Get the virtual base offset offset.
+      if (Offset.DerivedClass == MostDerivedClass) {
+        // We can get the offset offset directly from our map.
+        Adjustment.VBaseOffsetOffset = 
+          VBaseOffsetOffsets.lookup(Offset.VirtualBase).getQuantity();
+      } else {
+        Adjustment.VBaseOffsetOffset = 
+          VTables.getVirtualBaseOffsetOffset(Offset.DerivedClass,
+                                             Offset.VirtualBase).getQuantity();
+      }
+    }
+
+    Adjustment.NonVirtual = Offset.NonVirtualOffset.getQuantity();
+  }
+  
+  return Adjustment;
+}
+
+BaseOffset
+VTableBuilder::ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
+                                               BaseSubobject Derived) const {
+  const CXXRecordDecl *BaseRD = Base.getBase();
+  const CXXRecordDecl *DerivedRD = Derived.getBase();
+  
+  CXXBasePaths Paths(/*FindAmbiguities=*/true,
+                     /*RecordPaths=*/true, /*DetectVirtual=*/true);
+
+  if (!const_cast<CXXRecordDecl *>(DerivedRD)->
+      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseRD), Paths)) {
+    llvm_unreachable("Class must be derived from the passed in base class!");
+  }
+
+  // We have to go through all the paths, and see which one leads us to the
+  // right base subobject.
+  for (CXXBasePaths::const_paths_iterator I = Paths.begin(), E = Paths.end();
+       I != E; ++I) {
+    BaseOffset Offset = ComputeBaseOffset(Context, DerivedRD, *I);
+    
+    CharUnits OffsetToBaseSubobject = Offset.NonVirtualOffset;
+    
+    if (Offset.VirtualBase) {
+      // If we have a virtual base class, the non-virtual offset is relative
+      // to the virtual base class offset.
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+      
+      /// Get the virtual base offset, relative to the most derived class 
+      /// layout.
+      OffsetToBaseSubobject += 
+        LayoutClassLayout.getVBaseClassOffset(Offset.VirtualBase);
+    } else {
+      // Otherwise, the non-virtual offset is relative to the derived class 
+      // offset.
+      OffsetToBaseSubobject += Derived.getBaseOffset();
+    }
+    
+    // Check if this path gives us the right base subobject.
+    if (OffsetToBaseSubobject == Base.getBaseOffset()) {
+      // Since we're going from the base class _to_ the derived class, we'll
+      // invert the non-virtual offset here.
+      Offset.NonVirtualOffset = -Offset.NonVirtualOffset;
+      return Offset;
+    }      
+  }
+  
+  return BaseOffset();
+}
+  
+ThisAdjustment 
+VTableBuilder::ComputeThisAdjustment(const CXXMethodDecl *MD, 
+                                     CharUnits BaseOffsetInLayoutClass,
+                                     FinalOverriders::OverriderInfo Overrider) {
+  // Ignore adjustments for pure virtual member functions.
+  if (Overrider.Method->isPure())
+    return ThisAdjustment();
+  
+  BaseSubobject OverriddenBaseSubobject(MD->getParent(), 
+                                        BaseOffsetInLayoutClass);
+  
+  BaseSubobject OverriderBaseSubobject(Overrider.Method->getParent(),
+                                       Overrider.Offset);
+  
+  // Compute the adjustment offset.
+  BaseOffset Offset = ComputeThisAdjustmentBaseOffset(OverriddenBaseSubobject,
+                                                      OverriderBaseSubobject);
+  if (Offset.isEmpty())
+    return ThisAdjustment();
+
+  ThisAdjustment Adjustment;
+  
+  if (Offset.VirtualBase) {
+    // Get the vcall offset map for this virtual base.
+    VCallOffsetMap &VCallOffsets = VCallOffsetsForVBases[Offset.VirtualBase];
+
+    if (VCallOffsets.empty()) {
+      // We don't have vcall offsets for this virtual base, go ahead and
+      // build them.
+      VCallAndVBaseOffsetBuilder Builder(MostDerivedClass, MostDerivedClass,
+                                         /*FinalOverriders=*/0,
+                                         BaseSubobject(Offset.VirtualBase,
+                                                       CharUnits::Zero()),
+                                         /*BaseIsVirtual=*/true,
+                                         /*OffsetInLayoutClass=*/
+                                             CharUnits::Zero());
+        
+      VCallOffsets = Builder.getVCallOffsets();
+    }
+      
+    Adjustment.VCallOffsetOffset = 
+      VCallOffsets.getVCallOffsetOffset(MD).getQuantity();
+  }
+
+  // Set the non-virtual part of the adjustment.
+  Adjustment.NonVirtual = Offset.NonVirtualOffset.getQuantity();
+  
+  return Adjustment;
+}
+  
+void 
+VTableBuilder::AddMethod(const CXXMethodDecl *MD,
+                         ReturnAdjustment ReturnAdjustment) {
+  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+    assert(ReturnAdjustment.isEmpty() && 
+           "Destructor can't have return adjustment!");
+
+    // Add both the complete destructor and the deleting destructor.
+    Components.push_back(VTableComponent::MakeCompleteDtor(DD));
+    Components.push_back(VTableComponent::MakeDeletingDtor(DD));
+  } else {
+    // Add the return adjustment if necessary.
+    if (!ReturnAdjustment.isEmpty())
+      VTableThunks[Components.size()].Return = ReturnAdjustment;
+
+    // Add the function.
+    Components.push_back(VTableComponent::MakeFunction(MD));
+  }
+}
+
+/// OverridesIndirectMethodInBase - Return whether the given member function
+/// overrides any methods in the set of given bases. 
+/// Unlike OverridesMethodInBase, this checks "overriders of overriders".
+/// For example, if we have:
+///
+/// struct A { virtual void f(); }
+/// struct B : A { virtual void f(); }
+/// struct C : B { virtual void f(); }
+///
+/// OverridesIndirectMethodInBase will return true if given C::f as the method 
+/// and { A } as the set of bases.
+static bool
+OverridesIndirectMethodInBases(const CXXMethodDecl *MD,
+                               VTableBuilder::PrimaryBasesSetVectorTy &Bases) {
+  if (Bases.count(MD->getParent()))
+    return true;
+  
+  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+       E = MD->end_overridden_methods(); I != E; ++I) {
+    const CXXMethodDecl *OverriddenMD = *I;
+    
+    // Check "indirect overriders".
+    if (OverridesIndirectMethodInBases(OverriddenMD, Bases))
+      return true;
+  }
+   
+  return false;
+}
+
+bool 
+VTableBuilder::IsOverriderUsed(const CXXMethodDecl *Overrider,
+                               CharUnits BaseOffsetInLayoutClass,
+                               const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
+                               CharUnits FirstBaseOffsetInLayoutClass) const {
+  // If the base and the first base in the primary base chain have the same
+  // offsets, then this overrider will be used.
+  if (BaseOffsetInLayoutClass == FirstBaseOffsetInLayoutClass)
+   return true;
+
+  // We know now that Base (or a direct or indirect base of it) is a primary
+  // base in part of the class hierarchy, but not a primary base in the most 
+  // derived class.
+  
+  // If the overrider is the first base in the primary base chain, we know
+  // that the overrider will be used.
+  if (Overrider->getParent() == FirstBaseInPrimaryBaseChain)
+    return true;
+  
+  VTableBuilder::PrimaryBasesSetVectorTy PrimaryBases;
+
+  const CXXRecordDecl *RD = FirstBaseInPrimaryBaseChain;
+  PrimaryBases.insert(RD);
+
+  // Now traverse the base chain, starting with the first base, until we find
+  // the base that is no longer a primary base.
+  while (true) {
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+    const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+    
+    if (!PrimaryBase)
+      break;
+    
+    if (Layout.isPrimaryBaseVirtual()) {
+      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 && 
+             "Primary base should always be at offset 0!");
+
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+
+      // Now check if this is the primary base that is not a primary base in the
+      // most derived class.
+      if (LayoutClassLayout.getVBaseClassOffset(PrimaryBase) !=
+          FirstBaseOffsetInLayoutClass) {
+        // We found it, stop walking the chain.
+        break;
+      }
+    } else {
+      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 && 
+             "Primary base should always be at offset 0!");
+    }
+    
+    if (!PrimaryBases.insert(PrimaryBase))
+      llvm_unreachable("Found a duplicate primary base!");
+
+    RD = PrimaryBase;
+  }
+  
+  // If the final overrider is an override of one of the primary bases,
+  // then we know that it will be used.
+  return OverridesIndirectMethodInBases(Overrider, PrimaryBases);
+}
+
+/// FindNearestOverriddenMethod - Given a method, returns the overridden method
+/// from the nearest base. Returns null if no method was found.
+static const CXXMethodDecl * 
+FindNearestOverriddenMethod(const CXXMethodDecl *MD,
+                            VTableBuilder::PrimaryBasesSetVectorTy &Bases) {
+  OverriddenMethodsSetTy OverriddenMethods;
+  ComputeAllOverriddenMethods(MD, OverriddenMethods);
+  
+  for (int I = Bases.size(), E = 0; I != E; --I) {
+    const CXXRecordDecl *PrimaryBase = Bases[I - 1];
+
+    // Now check the overriden methods.
+    for (OverriddenMethodsSetTy::const_iterator I = OverriddenMethods.begin(),
+         E = OverriddenMethods.end(); I != E; ++I) {
+      const CXXMethodDecl *OverriddenMD = *I;
+      
+      // We found our overridden method.
+      if (OverriddenMD->getParent() == PrimaryBase)
+        return OverriddenMD;
+    }
+  }
+  
+  return 0;
+}  
+
+void
+VTableBuilder::AddMethods(BaseSubobject Base, CharUnits BaseOffsetInLayoutClass,
+                          const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
+                          CharUnits FirstBaseOffsetInLayoutClass,
+                          PrimaryBasesSetVectorTy &PrimaryBases) {
+  const CXXRecordDecl *RD = Base.getBase();
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
+    CharUnits PrimaryBaseOffset;
+    CharUnits PrimaryBaseOffsetInLayoutClass;
+    if (Layout.isPrimaryBaseVirtual()) {
+      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 &&
+             "Primary vbase should have a zero offset!");
+      
+      const ASTRecordLayout &MostDerivedClassLayout =
+        Context.getASTRecordLayout(MostDerivedClass);
+      
+      PrimaryBaseOffset = 
+        MostDerivedClassLayout.getVBaseClassOffset(PrimaryBase);
+      
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+
+      PrimaryBaseOffsetInLayoutClass =
+        LayoutClassLayout.getVBaseClassOffset(PrimaryBase);
+    } else {
+      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
+             "Primary base should have a zero offset!");
+
+      PrimaryBaseOffset = Base.getBaseOffset();
+      PrimaryBaseOffsetInLayoutClass = BaseOffsetInLayoutClass;
+    }
+
+    AddMethods(BaseSubobject(PrimaryBase, PrimaryBaseOffset),
+               PrimaryBaseOffsetInLayoutClass, FirstBaseInPrimaryBaseChain, 
+               FirstBaseOffsetInLayoutClass, PrimaryBases);
+    
+    if (!PrimaryBases.insert(PrimaryBase))
+      llvm_unreachable("Found a duplicate primary base!");
+  }
+
+  // Now go through all virtual member functions and add them.
+  for (CXXRecordDecl::method_iterator I = RD->method_begin(),
+       E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *MD = *I;
+  
+    if (!MD->isVirtual())
+      continue;
+
+    // Get the final overrider.
+    FinalOverriders::OverriderInfo Overrider = 
+      Overriders.getOverrider(MD, Base.getBaseOffset());
+
+    // Check if this virtual member function overrides a method in a primary
+    // base. If this is the case, and the return type doesn't require adjustment
+    // then we can just use the member function from the primary base.
+    if (const CXXMethodDecl *OverriddenMD = 
+          FindNearestOverriddenMethod(MD, PrimaryBases)) {
+      if (ComputeReturnAdjustmentBaseOffset(Context, MD, 
+                                            OverriddenMD).isEmpty()) {
+        // Replace the method info of the overridden method with our own
+        // method.
+        assert(MethodInfoMap.count(OverriddenMD) && 
+               "Did not find the overridden method!");
+        MethodInfo &OverriddenMethodInfo = MethodInfoMap[OverriddenMD];
+        
+        MethodInfo MethodInfo(Base.getBaseOffset(), BaseOffsetInLayoutClass,
+                              OverriddenMethodInfo.VTableIndex);
+
+        assert(!MethodInfoMap.count(MD) &&
+               "Should not have method info for this method yet!");
+        
+        MethodInfoMap.insert(std::make_pair(MD, MethodInfo));
+        MethodInfoMap.erase(OverriddenMD);
+        
+        // If the overridden method exists in a virtual base class or a direct
+        // or indirect base class of a virtual base class, we need to emit a
+        // thunk if we ever have a class hierarchy where the base class is not
+        // a primary base in the complete object.
+        if (!isBuildingConstructorVTable() && OverriddenMD != MD) {
+          // Compute the this adjustment.
+          ThisAdjustment ThisAdjustment =
+            ComputeThisAdjustment(OverriddenMD, BaseOffsetInLayoutClass,
+                                  Overrider);
+
+          if (ThisAdjustment.VCallOffsetOffset &&
+              Overrider.Method->getParent() == MostDerivedClass) {
+
+            // There's no return adjustment from OverriddenMD and MD,
+            // but that doesn't mean there isn't one between MD and
+            // the final overrider.
+            BaseOffset ReturnAdjustmentOffset =
+              ComputeReturnAdjustmentBaseOffset(Context, Overrider.Method, MD);
+            ReturnAdjustment ReturnAdjustment = 
+              ComputeReturnAdjustment(ReturnAdjustmentOffset);
+
+            // This is a virtual thunk for the most derived class, add it.
+            AddThunk(Overrider.Method, 
+                     ThunkInfo(ThisAdjustment, ReturnAdjustment));
+          }
+        }
+
+        continue;
+      }
+    }
+
+    // Insert the method info for this method.
+    MethodInfo MethodInfo(Base.getBaseOffset(), BaseOffsetInLayoutClass,
+                          Components.size());
+
+    assert(!MethodInfoMap.count(MD) &&
+           "Should not have method info for this method yet!");
+    MethodInfoMap.insert(std::make_pair(MD, MethodInfo));
+
+    // Check if this overrider is going to be used.
+    const CXXMethodDecl *OverriderMD = Overrider.Method;
+    if (!IsOverriderUsed(OverriderMD, BaseOffsetInLayoutClass,
+                         FirstBaseInPrimaryBaseChain, 
+                         FirstBaseOffsetInLayoutClass)) {
+      Components.push_back(VTableComponent::MakeUnusedFunction(OverriderMD));
+      continue;
+    }
+    
+    // Check if this overrider needs a return adjustment.
+    // We don't want to do this for pure virtual member functions.
+    BaseOffset ReturnAdjustmentOffset;
+    if (!OverriderMD->isPure()) {
+      ReturnAdjustmentOffset = 
+        ComputeReturnAdjustmentBaseOffset(Context, OverriderMD, MD);
+    }
+
+    ReturnAdjustment ReturnAdjustment = 
+      ComputeReturnAdjustment(ReturnAdjustmentOffset);
+    
+    AddMethod(Overrider.Method, ReturnAdjustment);
+  }
+}
+
+void VTableBuilder::LayoutVTable() {
+  LayoutPrimaryAndSecondaryVTables(BaseSubobject(MostDerivedClass,
+                                                 CharUnits::Zero()),
+                                   /*BaseIsMorallyVirtual=*/false,
+                                   MostDerivedClassIsVirtual,
+                                   MostDerivedClassOffset);
+  
+  VisitedVirtualBasesSetTy VBases;
+  
+  // Determine the primary virtual bases.
+  DeterminePrimaryVirtualBases(MostDerivedClass, MostDerivedClassOffset, 
+                               VBases);
+  VBases.clear();
+  
+  LayoutVTablesForVirtualBases(MostDerivedClass, VBases);
+
+  // -fapple-kext adds an extra entry at end of vtbl.
+  bool IsAppleKext = Context.getLangOptions().AppleKext;
+  if (IsAppleKext)
+    Components.push_back(VTableComponent::MakeVCallOffset(CharUnits::Zero()));
+}
+  
+void
+VTableBuilder::LayoutPrimaryAndSecondaryVTables(BaseSubobject Base,
+                                                bool BaseIsMorallyVirtual,
+                                                bool BaseIsVirtualInLayoutClass,
+                                                CharUnits OffsetInLayoutClass) {
+  assert(Base.getBase()->isDynamicClass() && "class does not have a vtable!");
+
+  // Add vcall and vbase offsets for this vtable.
+  VCallAndVBaseOffsetBuilder Builder(MostDerivedClass, LayoutClass, &Overriders,
+                                     Base, BaseIsVirtualInLayoutClass, 
+                                     OffsetInLayoutClass);
+  Components.append(Builder.components_begin(), Builder.components_end());
+  
+  // Check if we need to add these vcall offsets.
+  if (BaseIsVirtualInLayoutClass && !Builder.getVCallOffsets().empty()) {
+    VCallOffsetMap &VCallOffsets = VCallOffsetsForVBases[Base.getBase()];
+    
+    if (VCallOffsets.empty())
+      VCallOffsets = Builder.getVCallOffsets();
+  }
+
+  // If we're laying out the most derived class we want to keep track of the
+  // virtual base class offset offsets.
+  if (Base.getBase() == MostDerivedClass)
+    VBaseOffsetOffsets = Builder.getVBaseOffsetOffsets();
+
+  // Add the offset to top.
+  CharUnits OffsetToTop = MostDerivedClassOffset - OffsetInLayoutClass;
+  Components.push_back(
+    VTableComponent::MakeOffsetToTop(OffsetToTop));
+  
+  // Next, add the RTTI.
+  Components.push_back(VTableComponent::MakeRTTI(MostDerivedClass));
+  
+  uint64_t AddressPoint = Components.size();
+
+  // Now go through all virtual member functions and add them.
+  PrimaryBasesSetVectorTy PrimaryBases;
+  AddMethods(Base, OffsetInLayoutClass,
+             Base.getBase(), OffsetInLayoutClass, 
+             PrimaryBases);
+
+  // Compute 'this' pointer adjustments.
+  ComputeThisAdjustments();
+
+  // Add all address points.
+  const CXXRecordDecl *RD = Base.getBase();
+  while (true) {
+    AddressPoints.insert(std::make_pair(
+      BaseSubobject(RD, OffsetInLayoutClass),
+      AddressPoint));
+
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+    const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+    
+    if (!PrimaryBase)
+      break;
+    
+    if (Layout.isPrimaryBaseVirtual()) {
+      // Check if this virtual primary base is a primary base in the layout
+      // class. If it's not, we don't want to add it.
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+
+      if (LayoutClassLayout.getVBaseClassOffset(PrimaryBase) !=
+          OffsetInLayoutClass) {
+        // We don't want to add this class (or any of its primary bases).
+        break;
+      }
+    }
+
+    RD = PrimaryBase;
+  }
+
+  // Layout secondary vtables.
+  LayoutSecondaryVTables(Base, BaseIsMorallyVirtual, OffsetInLayoutClass);
+}
+
+void VTableBuilder::LayoutSecondaryVTables(BaseSubobject Base,
+                                           bool BaseIsMorallyVirtual,
+                                           CharUnits OffsetInLayoutClass) {
+  // Itanium C++ ABI 2.5.2:
+  //   Following the primary virtual table of a derived class are secondary 
+  //   virtual tables for each of its proper base classes, except any primary
+  //   base(s) with which it shares its primary virtual table.
+
+  const CXXRecordDecl *RD = Base.getBase();
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+  
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    // Ignore virtual bases, we'll emit them later.
+    if (I->isVirtual())
+      continue;
+    
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Ignore bases that don't have a vtable.
+    if (!BaseDecl->isDynamicClass())
+      continue;
+
+    if (isBuildingConstructorVTable()) {
+      // Itanium C++ ABI 2.6.4:
+      //   Some of the base class subobjects may not need construction virtual
+      //   tables, which will therefore not be present in the construction
+      //   virtual table group, even though the subobject virtual tables are
+      //   present in the main virtual table group for the complete object.
+      if (!BaseIsMorallyVirtual && !BaseDecl->getNumVBases())
+        continue;
+    }
+
+    // Get the base offset of this base.
+    CharUnits RelativeBaseOffset = Layout.getBaseClassOffset(BaseDecl);
+    CharUnits BaseOffset = Base.getBaseOffset() + RelativeBaseOffset;
+    
+    CharUnits BaseOffsetInLayoutClass = 
+      OffsetInLayoutClass + RelativeBaseOffset;
+    
+    // Don't emit a secondary vtable for a primary base. We might however want 
+    // to emit secondary vtables for other bases of this base.
+    if (BaseDecl == PrimaryBase) {
+      LayoutSecondaryVTables(BaseSubobject(BaseDecl, BaseOffset),
+                             BaseIsMorallyVirtual, BaseOffsetInLayoutClass);
+      continue;
+    }
+
+    // Layout the primary vtable (and any secondary vtables) for this base.
+    LayoutPrimaryAndSecondaryVTables(
+      BaseSubobject(BaseDecl, BaseOffset),
+      BaseIsMorallyVirtual,
+      /*BaseIsVirtualInLayoutClass=*/false,
+      BaseOffsetInLayoutClass);
+  }
+}
+
+void
+VTableBuilder::DeterminePrimaryVirtualBases(const CXXRecordDecl *RD,
+                                            CharUnits OffsetInLayoutClass,
+                                            VisitedVirtualBasesSetTy &VBases) {
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  
+  // Check if this base has a primary base.
+  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
+
+    // Check if it's virtual.
+    if (Layout.isPrimaryBaseVirtual()) {
+      bool IsPrimaryVirtualBase = true;
+
+      if (isBuildingConstructorVTable()) {
+        // Check if the base is actually a primary base in the class we use for
+        // layout.
+        const ASTRecordLayout &LayoutClassLayout =
+          Context.getASTRecordLayout(LayoutClass);
+
+        CharUnits PrimaryBaseOffsetInLayoutClass =
+          LayoutClassLayout.getVBaseClassOffset(PrimaryBase);
+        
+        // We know that the base is not a primary base in the layout class if 
+        // the base offsets are different.
+        if (PrimaryBaseOffsetInLayoutClass != OffsetInLayoutClass)
+          IsPrimaryVirtualBase = false;
+      }
+        
+      if (IsPrimaryVirtualBase)
+        PrimaryVirtualBases.insert(PrimaryBase);
+    }
+  }
+
+  // Traverse bases, looking for more primary virtual bases.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    CharUnits BaseOffsetInLayoutClass;
+    
+    if (I->isVirtual()) {
+      if (!VBases.insert(BaseDecl))
+        continue;
+      
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+
+      BaseOffsetInLayoutClass = 
+        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
+    } else {
+      BaseOffsetInLayoutClass = 
+        OffsetInLayoutClass + Layout.getBaseClassOffset(BaseDecl);
+    }
+
+    DeterminePrimaryVirtualBases(BaseDecl, BaseOffsetInLayoutClass, VBases);
+  }
+}
+
+void
+VTableBuilder::LayoutVTablesForVirtualBases(const CXXRecordDecl *RD, 
+                                            VisitedVirtualBasesSetTy &VBases) {
+  // Itanium C++ ABI 2.5.2:
+  //   Then come the virtual base virtual tables, also in inheritance graph
+  //   order, and again excluding primary bases (which share virtual tables with
+  //   the classes for which they are primary).
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Check if this base needs a vtable. (If it's virtual, not a primary base
+    // of some other class, and we haven't visited it before).
+    if (I->isVirtual() && BaseDecl->isDynamicClass() && 
+        !PrimaryVirtualBases.count(BaseDecl) && VBases.insert(BaseDecl)) {
+      const ASTRecordLayout &MostDerivedClassLayout =
+        Context.getASTRecordLayout(MostDerivedClass);
+      CharUnits BaseOffset = 
+        MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      
+      const ASTRecordLayout &LayoutClassLayout =
+        Context.getASTRecordLayout(LayoutClass);
+      CharUnits BaseOffsetInLayoutClass = 
+        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
+
+      LayoutPrimaryAndSecondaryVTables(
+        BaseSubobject(BaseDecl, BaseOffset),
+        /*BaseIsMorallyVirtual=*/true,
+        /*BaseIsVirtualInLayoutClass=*/true,
+        BaseOffsetInLayoutClass);
+    }
+    
+    // We only need to check the base for virtual base vtables if it actually
+    // has virtual bases.
+    if (BaseDecl->getNumVBases())
+      LayoutVTablesForVirtualBases(BaseDecl, VBases);
+  }
+}
+
+/// dumpLayout - Dump the vtable layout.
+void VTableBuilder::dumpLayout(raw_ostream& Out) {
+
+  if (isBuildingConstructorVTable()) {
+    Out << "Construction vtable for ('";
+    Out << MostDerivedClass->getQualifiedNameAsString() << "', ";
+    Out << MostDerivedClassOffset.getQuantity() << ") in '";
+    Out << LayoutClass->getQualifiedNameAsString();
+  } else {
+    Out << "Vtable for '";
+    Out << MostDerivedClass->getQualifiedNameAsString();
+  }
+  Out << "' (" << Components.size() << " entries).\n";
+
+  // Iterate through the address points and insert them into a new map where
+  // they are keyed by the index and not the base object.
+  // Since an address point can be shared by multiple subobjects, we use an
+  // STL multimap.
+  std::multimap<uint64_t, BaseSubobject> AddressPointsByIndex;
+  for (AddressPointsMapTy::const_iterator I = AddressPoints.begin(), 
+       E = AddressPoints.end(); I != E; ++I) {
+    const BaseSubobject& Base = I->first;
+    uint64_t Index = I->second;
+    
+    AddressPointsByIndex.insert(std::make_pair(Index, Base));
+  }
+  
+  for (unsigned I = 0, E = Components.size(); I != E; ++I) {
+    uint64_t Index = I;
+
+    Out << llvm::format("%4d | ", I);
+
+    const VTableComponent &Component = Components[I];
+
+    // Dump the component.
+    switch (Component.getKind()) {
+
+    case VTableComponent::CK_VCallOffset:
+      Out << "vcall_offset ("
+          << Component.getVCallOffset().getQuantity() 
+          << ")";
+      break;
+
+    case VTableComponent::CK_VBaseOffset:
+      Out << "vbase_offset ("
+          << Component.getVBaseOffset().getQuantity()
+          << ")";
+      break;
+
+    case VTableComponent::CK_OffsetToTop:
+      Out << "offset_to_top ("
+          << Component.getOffsetToTop().getQuantity()
+          << ")";
+      break;
+    
+    case VTableComponent::CK_RTTI:
+      Out << Component.getRTTIDecl()->getQualifiedNameAsString() << " RTTI";
+      break;
+    
+    case VTableComponent::CK_FunctionPointer: {
+      const CXXMethodDecl *MD = Component.getFunctionDecl();
+
+      std::string Str = 
+        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual, 
+                                    MD);
+      Out << Str;
+      if (MD->isPure())
+        Out << " [pure]";
+
+      ThunkInfo Thunk = VTableThunks.lookup(I);
+      if (!Thunk.isEmpty()) {
+        // If this function pointer has a return adjustment, dump it.
+        if (!Thunk.Return.isEmpty()) {
+          Out << "\n       [return adjustment: ";
+          Out << Thunk.Return.NonVirtual << " non-virtual";
+          
+          if (Thunk.Return.VBaseOffsetOffset) {
+            Out << ", " << Thunk.Return.VBaseOffsetOffset;
+            Out << " vbase offset offset";
+          }
+
+          Out << ']';
+        }
+
+        // If this function pointer has a 'this' pointer adjustment, dump it.
+        if (!Thunk.This.isEmpty()) {
+          Out << "\n       [this adjustment: ";
+          Out << Thunk.This.NonVirtual << " non-virtual";
+          
+          if (Thunk.This.VCallOffsetOffset) {
+            Out << ", " << Thunk.This.VCallOffsetOffset;
+            Out << " vcall offset offset";
+          }
+
+          Out << ']';
+        }          
+      }
+
+      break;
+    }
+
+    case VTableComponent::CK_CompleteDtorPointer: 
+    case VTableComponent::CK_DeletingDtorPointer: {
+      bool IsComplete = 
+        Component.getKind() == VTableComponent::CK_CompleteDtorPointer;
+      
+      const CXXDestructorDecl *DD = Component.getDestructorDecl();
+      
+      Out << DD->getQualifiedNameAsString();
+      if (IsComplete)
+        Out << "() [complete]";
+      else
+        Out << "() [deleting]";
+
+      if (DD->isPure())
+        Out << " [pure]";
+
+      ThunkInfo Thunk = VTableThunks.lookup(I);
+      if (!Thunk.isEmpty()) {
+        // If this destructor has a 'this' pointer adjustment, dump it.
+        if (!Thunk.This.isEmpty()) {
+          Out << "\n       [this adjustment: ";
+          Out << Thunk.This.NonVirtual << " non-virtual";
+          
+          if (Thunk.This.VCallOffsetOffset) {
+            Out << ", " << Thunk.This.VCallOffsetOffset;
+            Out << " vcall offset offset";
+          }
+          
+          Out << ']';
+        }          
+      }        
+
+      break;
+    }
+
+    case VTableComponent::CK_UnusedFunctionPointer: {
+      const CXXMethodDecl *MD = Component.getUnusedFunctionDecl();
+
+      std::string Str = 
+        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual, 
+                                    MD);
+      Out << "[unused] " << Str;
+      if (MD->isPure())
+        Out << " [pure]";
+    }
+
+    }
+
+    Out << '\n';
+    
+    // Dump the next address point.
+    uint64_t NextIndex = Index + 1;
+    if (AddressPointsByIndex.count(NextIndex)) {
+      if (AddressPointsByIndex.count(NextIndex) == 1) {
+        const BaseSubobject &Base = 
+          AddressPointsByIndex.find(NextIndex)->second;
+        
+        Out << "       -- (" << Base.getBase()->getQualifiedNameAsString();
+        Out << ", " << Base.getBaseOffset().getQuantity();
+        Out << ") vtable address --\n";
+      } else {
+        CharUnits BaseOffset =
+          AddressPointsByIndex.lower_bound(NextIndex)->second.getBaseOffset();
+        
+        // We store the class names in a set to get a stable order.
+        std::set<std::string> ClassNames;
+        for (std::multimap<uint64_t, BaseSubobject>::const_iterator I =
+             AddressPointsByIndex.lower_bound(NextIndex), E =
+             AddressPointsByIndex.upper_bound(NextIndex); I != E; ++I) {
+          assert(I->second.getBaseOffset() == BaseOffset &&
+                 "Invalid base offset!");
+          const CXXRecordDecl *RD = I->second.getBase();
+          ClassNames.insert(RD->getQualifiedNameAsString());
+        }
+        
+        for (std::set<std::string>::const_iterator I = ClassNames.begin(),
+             E = ClassNames.end(); I != E; ++I) {
+          Out << "       -- (" << *I;
+          Out << ", " << BaseOffset.getQuantity() << ") vtable address --\n";
+        }
+      }
+    }
+  }
+
+  Out << '\n';
+  
+  if (isBuildingConstructorVTable())
+    return;
+  
+  if (MostDerivedClass->getNumVBases()) {
+    // We store the virtual base class names and their offsets in a map to get
+    // a stable order.
+
+    std::map<std::string, CharUnits> ClassNamesAndOffsets;
+    for (VBaseOffsetOffsetsMapTy::const_iterator I = VBaseOffsetOffsets.begin(),
+         E = VBaseOffsetOffsets.end(); I != E; ++I) {
+      std::string ClassName = I->first->getQualifiedNameAsString();
+      CharUnits OffsetOffset = I->second;
+      ClassNamesAndOffsets.insert(
+          std::make_pair(ClassName, OffsetOffset));
+    }
+    
+    Out << "Virtual base offset offsets for '";
+    Out << MostDerivedClass->getQualifiedNameAsString() << "' (";
+    Out << ClassNamesAndOffsets.size();
+    Out << (ClassNamesAndOffsets.size() == 1 ? " entry" : " entries") << ").\n";
+
+    for (std::map<std::string, CharUnits>::const_iterator I =
+         ClassNamesAndOffsets.begin(), E = ClassNamesAndOffsets.end(); 
+         I != E; ++I)
+      Out << "   " << I->first << " | " << I->second.getQuantity() << '\n';
+
+    Out << "\n";
+  }
+  
+  if (!Thunks.empty()) {
+    // We store the method names in a map to get a stable order.
+    std::map<std::string, const CXXMethodDecl *> MethodNamesAndDecls;
+    
+    for (ThunksMapTy::const_iterator I = Thunks.begin(), E = Thunks.end();
+         I != E; ++I) {
+      const CXXMethodDecl *MD = I->first;
+      std::string MethodName = 
+        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual,
+                                    MD);
+      
+      MethodNamesAndDecls.insert(std::make_pair(MethodName, MD));
+    }
+
+    for (std::map<std::string, const CXXMethodDecl *>::const_iterator I =
+         MethodNamesAndDecls.begin(), E = MethodNamesAndDecls.end(); 
+         I != E; ++I) {
+      const std::string &MethodName = I->first;
+      const CXXMethodDecl *MD = I->second;
+
+      ThunkInfoVectorTy ThunksVector = Thunks[MD];
+      std::sort(ThunksVector.begin(), ThunksVector.end());
+
+      Out << "Thunks for '" << MethodName << "' (" << ThunksVector.size();
+      Out << (ThunksVector.size() == 1 ? " entry" : " entries") << ").\n";
+      
+      for (unsigned I = 0, E = ThunksVector.size(); I != E; ++I) {
+        const ThunkInfo &Thunk = ThunksVector[I];
+
+        Out << llvm::format("%4d | ", I);
+        
+        // If this function pointer has a return pointer adjustment, dump it.
+        if (!Thunk.Return.isEmpty()) {
+          Out << "return adjustment: " << Thunk.This.NonVirtual;
+          Out << " non-virtual";
+          if (Thunk.Return.VBaseOffsetOffset) {
+            Out << ", " << Thunk.Return.VBaseOffsetOffset;
+            Out << " vbase offset offset";
+          }
+
+          if (!Thunk.This.isEmpty())
+            Out << "\n       ";
+        }
+
+        // If this function pointer has a 'this' pointer adjustment, dump it.
+        if (!Thunk.This.isEmpty()) {
+          Out << "this adjustment: ";
+          Out << Thunk.This.NonVirtual << " non-virtual";
+          
+          if (Thunk.This.VCallOffsetOffset) {
+            Out << ", " << Thunk.This.VCallOffsetOffset;
+            Out << " vcall offset offset";
+          }
+        }
+        
+        Out << '\n';
+      }
+      
+      Out << '\n';
+    }
+  }
+
+  // Compute the vtable indices for all the member functions.
+  // Store them in a map keyed by the index so we'll get a sorted table.
+  std::map<uint64_t, std::string> IndicesMap;
+
+  for (CXXRecordDecl::method_iterator i = MostDerivedClass->method_begin(),
+       e = MostDerivedClass->method_end(); i != e; ++i) {
+    const CXXMethodDecl *MD = *i;
+    
+    // We only want virtual member functions.
+    if (!MD->isVirtual())
+      continue;
+
+    std::string MethodName =
+      PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual,
+                                  MD);
+
+    if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+      IndicesMap[VTables.getMethodVTableIndex(GlobalDecl(DD, Dtor_Complete))] =
+        MethodName + " [complete]";
+      IndicesMap[VTables.getMethodVTableIndex(GlobalDecl(DD, Dtor_Deleting))] =
+        MethodName + " [deleting]";
+    } else {
+      IndicesMap[VTables.getMethodVTableIndex(MD)] = MethodName;
+    }
+  }
+
+  // Print the vtable indices for all the member functions.
+  if (!IndicesMap.empty()) {
+    Out << "VTable indices for '";
+    Out << MostDerivedClass->getQualifiedNameAsString();
+    Out << "' (" << IndicesMap.size() << " entries).\n";
+
+    for (std::map<uint64_t, std::string>::const_iterator I = IndicesMap.begin(),
+         E = IndicesMap.end(); I != E; ++I) {
+      uint64_t VTableIndex = I->first;
+      const std::string &MethodName = I->second;
+
+      Out << llvm::format(" %4u | ", VTableIndex) << MethodName << '\n';
+    }
+  }
+
+  Out << '\n';
+}
+  
+}
+
+VTableLayout::VTableLayout(uint64_t NumVTableComponents,
+                           const VTableComponent *VTableComponents,
+                           uint64_t NumVTableThunks,
+                           const VTableThunkTy *VTableThunks,
+                           const AddressPointsMapTy &AddressPoints)
+  : NumVTableComponents(NumVTableComponents),
+    VTableComponents(new VTableComponent[NumVTableComponents]),
+    NumVTableThunks(NumVTableThunks),
+    VTableThunks(new VTableThunkTy[NumVTableThunks]),
+    AddressPoints(AddressPoints) {
+  std::copy(VTableComponents, VTableComponents+NumVTableComponents,
+            this->VTableComponents);
+  std::copy(VTableThunks, VTableThunks+NumVTableThunks, this->VTableThunks);
+}
+
+VTableLayout::~VTableLayout() {
+  delete[] VTableComponents;
+}
+
+VTableContext::~VTableContext() {
+  llvm::DeleteContainerSeconds(VTableLayouts);
+}
+
+static void 
+CollectPrimaryBases(const CXXRecordDecl *RD, ASTContext &Context,
+                    VTableBuilder::PrimaryBasesSetVectorTy &PrimaryBases) {
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+
+  if (!PrimaryBase)
+    return;
+
+  CollectPrimaryBases(PrimaryBase, Context, PrimaryBases);
+
+  if (!PrimaryBases.insert(PrimaryBase))
+    llvm_unreachable("Found a duplicate primary base!");
+}
+
+void VTableContext::ComputeMethodVTableIndices(const CXXRecordDecl *RD) {
+  
+  // Itanium C++ ABI 2.5.2:
+  //   The order of the virtual function pointers in a virtual table is the 
+  //   order of declaration of the corresponding member functions in the class.
+  //
+  //   There is an entry for any virtual function declared in a class, 
+  //   whether it is a new function or overrides a base class function, 
+  //   unless it overrides a function from the primary base, and conversion
+  //   between their return types does not require an adjustment. 
+
+  int64_t CurrentIndex = 0;
+  
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+  
+  if (PrimaryBase) {
+    assert(PrimaryBase->isCompleteDefinition() && 
+           "Should have the definition decl of the primary base!");
+
+    // Since the record decl shares its vtable pointer with the primary base
+    // we need to start counting at the end of the primary base's vtable.
+    CurrentIndex = getNumVirtualFunctionPointers(PrimaryBase);
+  }
+
+  // Collect all the primary bases, so we can check whether methods override
+  // a method from the base.
+  VTableBuilder::PrimaryBasesSetVectorTy PrimaryBases;
+  CollectPrimaryBases(RD, Context, PrimaryBases);
+
+  const CXXDestructorDecl *ImplicitVirtualDtor = 0;
+  
+  for (CXXRecordDecl::method_iterator i = RD->method_begin(),
+       e = RD->method_end(); i != e; ++i) {
+    const CXXMethodDecl *MD = *i;
+
+    // We only want virtual methods.
+    if (!MD->isVirtual())
+      continue;
+
+    // Check if this method overrides a method in the primary base.
+    if (const CXXMethodDecl *OverriddenMD = 
+          FindNearestOverriddenMethod(MD, PrimaryBases)) {
+      // Check if converting from the return type of the method to the 
+      // return type of the overridden method requires conversion.
+      if (ComputeReturnAdjustmentBaseOffset(Context, MD, 
+                                            OverriddenMD).isEmpty()) {
+        // This index is shared between the index in the vtable of the primary
+        // base class.
+        if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+          const CXXDestructorDecl *OverriddenDD = 
+            cast<CXXDestructorDecl>(OverriddenMD);
+          
+          // Add both the complete and deleting entries.
+          MethodVTableIndices[GlobalDecl(DD, Dtor_Complete)] = 
+            getMethodVTableIndex(GlobalDecl(OverriddenDD, Dtor_Complete));
+          MethodVTableIndices[GlobalDecl(DD, Dtor_Deleting)] = 
+            getMethodVTableIndex(GlobalDecl(OverriddenDD, Dtor_Deleting));
+        } else {
+          MethodVTableIndices[MD] = getMethodVTableIndex(OverriddenMD);
+        }
+        
+        // We don't need to add an entry for this method.
+        continue;
+      }
+    }
+    
+    if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+      if (MD->isImplicit()) {
+        assert(!ImplicitVirtualDtor && 
+               "Did already see an implicit virtual dtor!");
+        ImplicitVirtualDtor = DD;
+        continue;
+      } 
+
+      // Add the complete dtor.
+      MethodVTableIndices[GlobalDecl(DD, Dtor_Complete)] = CurrentIndex++;
+      
+      // Add the deleting dtor.
+      MethodVTableIndices[GlobalDecl(DD, Dtor_Deleting)] = CurrentIndex++;
+    } else {
+      // Add the entry.
+      MethodVTableIndices[MD] = CurrentIndex++;
+    }
+  }
+
+  if (ImplicitVirtualDtor) {
+    // Itanium C++ ABI 2.5.2:
+    //   If a class has an implicitly-defined virtual destructor, 
+    //   its entries come after the declared virtual function pointers.
+
+    // Add the complete dtor.
+    MethodVTableIndices[GlobalDecl(ImplicitVirtualDtor, Dtor_Complete)] = 
+      CurrentIndex++;
+    
+    // Add the deleting dtor.
+    MethodVTableIndices[GlobalDecl(ImplicitVirtualDtor, Dtor_Deleting)] = 
+      CurrentIndex++;
+  }
+  
+  NumVirtualFunctionPointers[RD] = CurrentIndex;
+}
+
+uint64_t VTableContext::getNumVirtualFunctionPointers(const CXXRecordDecl *RD) {
+  llvm::DenseMap<const CXXRecordDecl *, uint64_t>::iterator I = 
+    NumVirtualFunctionPointers.find(RD);
+  if (I != NumVirtualFunctionPointers.end())
+    return I->second;
+
+  ComputeMethodVTableIndices(RD);
+
+  I = NumVirtualFunctionPointers.find(RD);
+  assert(I != NumVirtualFunctionPointers.end() && "Did not find entry!");
+  return I->second;
+}
+      
+uint64_t VTableContext::getMethodVTableIndex(GlobalDecl GD) {
+  MethodVTableIndicesTy::iterator I = MethodVTableIndices.find(GD);
+  if (I != MethodVTableIndices.end())
+    return I->second;
+  
+  const CXXRecordDecl *RD = cast<CXXMethodDecl>(GD.getDecl())->getParent();
+
+  ComputeMethodVTableIndices(RD);
+
+  I = MethodVTableIndices.find(GD);
+  assert(I != MethodVTableIndices.end() && "Did not find index!");
+  return I->second;
+}
+
+CharUnits 
+VTableContext::getVirtualBaseOffsetOffset(const CXXRecordDecl *RD, 
+                                          const CXXRecordDecl *VBase) {
+  ClassPairTy ClassPair(RD, VBase);
+  
+  VirtualBaseClassOffsetOffsetsMapTy::iterator I = 
+    VirtualBaseClassOffsetOffsets.find(ClassPair);
+  if (I != VirtualBaseClassOffsetOffsets.end())
+    return I->second;
+  
+  VCallAndVBaseOffsetBuilder Builder(RD, RD, /*FinalOverriders=*/0,
+                                     BaseSubobject(RD, CharUnits::Zero()),
+                                     /*BaseIsVirtual=*/false,
+                                     /*OffsetInLayoutClass=*/CharUnits::Zero());
+
+  for (VCallAndVBaseOffsetBuilder::VBaseOffsetOffsetsMapTy::const_iterator I =
+       Builder.getVBaseOffsetOffsets().begin(), 
+       E = Builder.getVBaseOffsetOffsets().end(); I != E; ++I) {
+    // Insert all types.
+    ClassPairTy ClassPair(RD, I->first);
+    
+    VirtualBaseClassOffsetOffsets.insert(
+        std::make_pair(ClassPair, I->second));
+  }
+  
+  I = VirtualBaseClassOffsetOffsets.find(ClassPair);
+  assert(I != VirtualBaseClassOffsetOffsets.end() && "Did not find index!");
+  
+  return I->second;
+}
+
+static VTableLayout *CreateVTableLayout(const VTableBuilder &Builder) {
+  SmallVector<VTableLayout::VTableThunkTy, 1>
+    VTableThunks(Builder.vtable_thunks_begin(), Builder.vtable_thunks_end());
+  std::sort(VTableThunks.begin(), VTableThunks.end());
+
+  return new VTableLayout(Builder.getNumVTableComponents(),
+                          Builder.vtable_component_begin(),
+                          VTableThunks.size(),
+                          VTableThunks.data(),
+                          Builder.getAddressPoints());
+}
+
+void VTableContext::ComputeVTableRelatedInformation(const CXXRecordDecl *RD) {
+  const VTableLayout *&Entry = VTableLayouts[RD];
+
+  // Check if we've computed this information before.
+  if (Entry)
+    return;
+
+  VTableBuilder Builder(*this, RD, CharUnits::Zero(), 
+                        /*MostDerivedClassIsVirtual=*/0, RD);
+  Entry = CreateVTableLayout(Builder);
+
+  // Add the known thunks.
+  Thunks.insert(Builder.thunks_begin(), Builder.thunks_end());
+
+  // If we don't have the vbase information for this class, insert it.
+  // getVirtualBaseOffsetOffset will compute it separately without computing
+  // the rest of the vtable related information.
+  if (!RD->getNumVBases())
+    return;
+  
+  const RecordType *VBaseRT = 
+    RD->vbases_begin()->getType()->getAs<RecordType>();
+  const CXXRecordDecl *VBase = cast<CXXRecordDecl>(VBaseRT->getDecl());
+  
+  if (VirtualBaseClassOffsetOffsets.count(std::make_pair(RD, VBase)))
+    return;
+  
+  for (VTableBuilder::VBaseOffsetOffsetsMapTy::const_iterator I =
+       Builder.getVBaseOffsetOffsets().begin(), 
+       E = Builder.getVBaseOffsetOffsets().end(); I != E; ++I) {
+    // Insert all types.
+    ClassPairTy ClassPair(RD, I->first);
+    
+    VirtualBaseClassOffsetOffsets.insert(std::make_pair(ClassPair, I->second));
+  }
+}
+
+VTableLayout *VTableContext::createConstructionVTableLayout(
+                                          const CXXRecordDecl *MostDerivedClass,
+                                          CharUnits MostDerivedClassOffset,
+                                          bool MostDerivedClassIsVirtual,
+                                          const CXXRecordDecl *LayoutClass) {
+  VTableBuilder Builder(*this, MostDerivedClass, MostDerivedClassOffset, 
+                        MostDerivedClassIsVirtual, LayoutClass);
+  return CreateVTableLayout(Builder);
+}
diff --git a/contrib/llvm/tools/clang/lib/Analysis/AnalysisContext.cpp b/contrib/llvm/tools/clang/lib/Analysis/AnalysisContext.cpp
index 678f02fd71fb..3dd194b8e80a 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/AnalysisContext.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/AnalysisContext.cpp
@@ -24,25 +24,44 @@
 #include "clang/Analysis/CFG.h"
 #include "clang/Analysis/CFGStmtMap.h"
 #include "clang/Analysis/Support/BumpVector.h"
+#include "clang/Analysis/Support/SaveAndRestore.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace clang;
 
+typedef llvm::DenseMap<const void *, ManagedAnalysis *> ManagedAnalysisMap;
+
 AnalysisContext::AnalysisContext(const Decl *d,
                                  idx::TranslationUnit *tu,
-                                 bool useUnoptimizedCFG,
-                                 bool addehedges,
-                                 bool addImplicitDtors,
-                                 bool addInitializers)
+                                 const CFG::BuildOptions &buildOptions)
   : D(d), TU(tu),
+    cfgBuildOptions(buildOptions),
     forcedBlkExprs(0),
-    builtCFG(false), builtCompleteCFG(false),
-    useUnoptimizedCFG(useUnoptimizedCFG),
-    ReferencedBlockVars(0)
-{
+    builtCFG(false),
+    builtCompleteCFG(false),
+    ReferencedBlockVars(0),
+    ManagedAnalyses(0)
+{  
   cfgBuildOptions.forcedBlkExprs = &forcedBlkExprs;
-  cfgBuildOptions.AddEHEdges = addehedges;
+}
+
+AnalysisContext::AnalysisContext(const Decl *d,
+                                 idx::TranslationUnit *tu)
+: D(d), TU(tu),
+  forcedBlkExprs(0),
+  builtCFG(false),
+  builtCompleteCFG(false),
+  ReferencedBlockVars(0),
+  ManagedAnalyses(0)
+{  
+  cfgBuildOptions.forcedBlkExprs = &forcedBlkExprs;
+}
+
+AnalysisContextManager::AnalysisContextManager(bool useUnoptimizedCFG,
+                                               bool addImplicitDtors,
+                                               bool addInitializers) {
+  cfgBuildOptions.PruneTriviallyFalseEdges = !useUnoptimizedCFG;
   cfgBuildOptions.AddImplicitDtors = addImplicitDtors;
   cfgBuildOptions.AddInitializers = addInitializers;
 }
@@ -53,7 +72,7 @@ void AnalysisContextManager::clear() {
   Contexts.clear();
 }
 
-Stmt *AnalysisContext::getBody() {
+Stmt *AnalysisContext::getBody() const {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
     return FD->getBody();
   else if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D))
@@ -95,7 +114,7 @@ AnalysisContext::getBlockForRegisteredExpression(const Stmt *stmt) {
 }
 
 CFG *AnalysisContext::getCFG() {
-  if (useUnoptimizedCFG)
+  if (!cfgBuildOptions.PruneTriviallyFalseEdges)
     return getUnoptimizedCFG();
 
   if (!builtCFG) {
@@ -110,9 +129,10 @@ CFG *AnalysisContext::getCFG() {
 
 CFG *AnalysisContext::getUnoptimizedCFG() {
   if (!builtCompleteCFG) {
-    CFG::BuildOptions B = cfgBuildOptions;
-    B.PruneTriviallyFalseEdges = false;
-    completeCFG.reset(CFG::buildCFG(D, getBody(), &D->getASTContext(), B));
+    SaveAndRestore<bool> NotPrune(cfgBuildOptions.PruneTriviallyFalseEdges,
+                                  false);
+    completeCFG.reset(CFG::buildCFG(D, getBody(), &D->getASTContext(),
+                                    cfgBuildOptions));
     // Even when the cfg is not successfully built, we don't
     // want to try building it again.
     builtCompleteCFG = true;
@@ -160,36 +180,11 @@ PseudoConstantAnalysis *AnalysisContext::getPseudoConstantAnalysis() {
   return PCA.get();
 }
 
-LiveVariables *AnalysisContext::getLiveVariables() {
-  if (!liveness) {
-    if (CFG *c = getCFG()) {
-      liveness.reset(new LiveVariables(*this));
-      liveness->runOnCFG(*c);
-      liveness->runOnAllBlocks(*c, 0, true);
-    }
-  }
-
-  return liveness.get();
-}
-
-LiveVariables *AnalysisContext::getRelaxedLiveVariables() {
-  if (!relaxedLiveness)
-    if (CFG *c = getCFG()) {
-      relaxedLiveness.reset(new LiveVariables(*this, false));
-      relaxedLiveness->runOnCFG(*c);
-      relaxedLiveness->runOnAllBlocks(*c, 0, true);
-    }
-
-  return relaxedLiveness.get();
-}
-
 AnalysisContext *AnalysisContextManager::getContext(const Decl *D,
                                                     idx::TranslationUnit *TU) {
   AnalysisContext *&AC = Contexts[D];
   if (!AC)
-    AC = new AnalysisContext(D, TU, UseUnoptimizedCFG, false,
-        AddImplicitDtors, AddInitializers);
-
+    AC = new AnalysisContext(D, TU, cfgBuildOptions);
   return AC;
 }
 
@@ -201,7 +196,7 @@ void LocationContext::ProfileCommon(llvm::FoldingSetNodeID &ID,
                                     ContextKind ck,
                                     AnalysisContext *ctx,
                                     const LocationContext *parent,
-                                    const void* data) {
+                                    const void *data) {
   ID.AddInteger(ck);
   ID.AddPointer(ctx);
   ID.AddPointer(parent);
@@ -392,13 +387,29 @@ AnalysisContext::getReferencedBlockVars(const BlockDecl *BD) {
   return std::make_pair(V->begin(), V->end());
 }
 
+ManagedAnalysis *&AnalysisContext::getAnalysisImpl(const void *tag) {
+  if (!ManagedAnalyses)
+    ManagedAnalyses = new ManagedAnalysisMap();
+  ManagedAnalysisMap *M = (ManagedAnalysisMap*) ManagedAnalyses;
+  return (*M)[tag];
+}
+
 //===----------------------------------------------------------------------===//
 // Cleanup.
 //===----------------------------------------------------------------------===//
 
+ManagedAnalysis::~ManagedAnalysis() {}
+
 AnalysisContext::~AnalysisContext() {
   delete forcedBlkExprs;
   delete ReferencedBlockVars;
+  // Release the managed analyses.
+  if (ManagedAnalyses) {
+    ManagedAnalysisMap *M = (ManagedAnalysisMap*) ManagedAnalyses;
+    for (ManagedAnalysisMap::iterator I = M->begin(), E = M->end(); I!=E; ++I)
+      delete I->second;  
+    delete M;
+  }
 }
 
 AnalysisContextManager::~AnalysisContextManager() {
diff --git a/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp b/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp
index f231c147f11e..83c7384db0f4 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp
@@ -29,9 +29,9 @@ using namespace clang;
 
 namespace {
 
-static SourceLocation GetEndLoc(Decl* D) {
-  if (VarDecl* VD = dyn_cast<VarDecl>(D))
-    if (Expr* Ex = VD->getInit())
+static SourceLocation GetEndLoc(Decl *D) {
+  if (VarDecl *VD = dyn_cast<VarDecl>(D))
+    if (Expr *Ex = VD->getInit())
       return Ex->getSourceRange().getEnd();
   return D->getLocation();
 }
@@ -121,16 +121,16 @@ class LocalScope {
         *this = Scope->Prev;
     }
 
-    VarDecl* const* operator->() const {
+    VarDecl *const* operator->() const {
       assert (Scope && "Dereferencing invalid iterator is not allowed");
       assert (VarIter != 0 && "Iterator has invalid value of VarIter member");
       return &Scope->Vars[VarIter - 1];
     }
-    VarDecl* operator*() const {
+    VarDecl *operator*() const {
       return *this->operator->();
     }
 
-    const_iterator& operator++() {
+    const_iterator &operator++() {
       if (!Scope)
         return *this;
 
@@ -146,10 +146,10 @@ class LocalScope {
       return P;
     }
 
-    bool operator==(const const_iterator& rhs) const {
+    bool operator==(const const_iterator &rhs) const {
       return Scope == rhs.Scope && VarIter == rhs.VarIter;
     }
-    bool operator!=(const const_iterator& rhs) const {
+    bool operator!=(const const_iterator &rhs) const {
       return !(*this == rhs);
     }
 
@@ -179,7 +179,7 @@ class LocalScope {
   /// Begin of scope in direction of CFG building (backwards).
   const_iterator begin() const { return const_iterator(*this, Vars.size()); }
 
-  void addVar(VarDecl* VD) {
+  void addVar(VarDecl *VD) {
     Vars.push_back(VD, ctx);
   }
 };
@@ -205,7 +205,7 @@ int LocalScope::const_iterator::distance(LocalScope::const_iterator L) {
 /// and  LocalScope::const_iterator that specifies position in LocalScope graph.
 struct BlockScopePosPair {
   BlockScopePosPair() : block(0) {}
-  BlockScopePosPair(CFGBlock* b, LocalScope::const_iterator scopePos)
+  BlockScopePosPair(CFGBlock *b, LocalScope::const_iterator scopePos)
       : block(b), scopePosition(scopePos) {}
 
   CFGBlock *block;
@@ -252,13 +252,13 @@ class CFGBuilder {
   ASTContext *Context;
   llvm::OwningPtr<CFG> cfg;
 
-  CFGBlock* Block;
-  CFGBlock* Succ;
+  CFGBlock *Block;
+  CFGBlock *Succ;
   JumpTarget ContinueJumpTarget;
   JumpTarget BreakJumpTarget;
-  CFGBlock* SwitchTerminatedBlock;
-  CFGBlock* DefaultCaseBlock;
-  CFGBlock* TryTerminatedBlock;
+  CFGBlock *SwitchTerminatedBlock;
+  CFGBlock *DefaultCaseBlock;
+  CFGBlock *TryTerminatedBlock;
   
   // Current position in local scope.
   LocalScope::const_iterator ScopePos;
@@ -305,7 +305,7 @@ class CFGBuilder {
   // Visitors to walk an AST and construct the CFG.
   CFGBlock *VisitAddrLabelExpr(AddrLabelExpr *A, AddStmtChoice asc);
   CFGBlock *VisitBinaryOperator(BinaryOperator *B, AddStmtChoice asc);
-  CFGBlock *VisitBlockExpr(BlockExpr* E, AddStmtChoice asc);
+  CFGBlock *VisitBlockExpr(BlockExpr *E, AddStmtChoice asc);
   CFGBlock *VisitBreakStmt(BreakStmt *B);
   CFGBlock *VisitCXXCatchStmt(CXXCatchStmt *S);
   CFGBlock *VisitExprWithCleanups(ExprWithCleanups *E,
@@ -328,11 +328,11 @@ class CFGBuilder {
                                      AddStmtChoice asc);
   CFGBlock *VisitContinueStmt(ContinueStmt *C);
   CFGBlock *VisitDeclStmt(DeclStmt *DS);
-  CFGBlock *VisitDeclSubExpr(DeclStmt* DS);
+  CFGBlock *VisitDeclSubExpr(DeclStmt *DS);
   CFGBlock *VisitDefaultStmt(DefaultStmt *D);
   CFGBlock *VisitDoStmt(DoStmt *D);
   CFGBlock *VisitForStmt(ForStmt *F);
-  CFGBlock *VisitGotoStmt(GotoStmt* G);
+  CFGBlock *VisitGotoStmt(GotoStmt *G);
   CFGBlock *VisitIfStmt(IfStmt *I);
   CFGBlock *VisitImplicitCastExpr(ImplicitCastExpr *E, AddStmtChoice asc);
   CFGBlock *VisitIndirectGotoStmt(IndirectGotoStmt *I);
@@ -343,7 +343,7 @@ class CFGBuilder {
   CFGBlock *VisitObjCAtThrowStmt(ObjCAtThrowStmt *S);
   CFGBlock *VisitObjCAtTryStmt(ObjCAtTryStmt *S);
   CFGBlock *VisitObjCForCollectionStmt(ObjCForCollectionStmt *S);
-  CFGBlock *VisitReturnStmt(ReturnStmt* R);
+  CFGBlock *VisitReturnStmt(ReturnStmt *R);
   CFGBlock *VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *E,
                                           AddStmtChoice asc);
   CFGBlock *VisitStmtExpr(StmtExpr *S, AddStmtChoice asc);
@@ -353,7 +353,7 @@ class CFGBuilder {
 
   CFGBlock *Visit(Stmt *S, AddStmtChoice asc = AddStmtChoice::NotAlwaysAdd);
   CFGBlock *VisitStmt(Stmt *S, AddStmtChoice asc);
-  CFGBlock *VisitChildren(Stmt* S);
+  CFGBlock *VisitChildren(Stmt *S);
 
   // Visitors to walk an AST and generate destructors of temporaries in
   // full expression.
@@ -367,34 +367,35 @@ class CFGBuilder {
                                             bool BindToTemporary);
 
   // NYS == Not Yet Supported
-  CFGBlock* NYS() {
+  CFGBlock *NYS() {
     badCFG = true;
     return Block;
   }
 
   void autoCreateBlock() { if (!Block) Block = createBlock(); }
   CFGBlock *createBlock(bool add_successor = true);
+  CFGBlock *createNoReturnBlock();
 
   CFGBlock *addStmt(Stmt *S) {
     return Visit(S, AddStmtChoice::AlwaysAdd);
   }
   CFGBlock *addInitializer(CXXCtorInitializer *I);
   void addAutomaticObjDtors(LocalScope::const_iterator B,
-                            LocalScope::const_iterator E, Stmt* S);
+                            LocalScope::const_iterator E, Stmt *S);
   void addImplicitDtorsForDestructor(const CXXDestructorDecl *DD);
 
   // Local scopes creation.
   LocalScope* createOrReuseLocalScope(LocalScope* Scope);
 
-  void addLocalScopeForStmt(Stmt* S);
-  LocalScope* addLocalScopeForDeclStmt(DeclStmt* DS, LocalScope* Scope = NULL);
-  LocalScope* addLocalScopeForVarDecl(VarDecl* VD, LocalScope* Scope = NULL);
+  void addLocalScopeForStmt(Stmt *S);
+  LocalScope* addLocalScopeForDeclStmt(DeclStmt *DS, LocalScope* Scope = NULL);
+  LocalScope* addLocalScopeForVarDecl(VarDecl *VD, LocalScope* Scope = NULL);
 
-  void addLocalScopeAndDtors(Stmt* S);
+  void addLocalScopeAndDtors(Stmt *S);
 
   // Interface to CFGBlock - adding CFGElements.
   void appendStmt(CFGBlock *B, const Stmt *S) {
-    if (alwaysAdd(S))
+    if (alwaysAdd(S) && cachedEntry)
       cachedEntry->second = B;
 
     // All block-level expressions should have already been IgnoreParens()ed.
@@ -413,12 +414,11 @@ class CFGBuilder {
   void appendTemporaryDtor(CFGBlock *B, CXXBindTemporaryExpr *E) {
     B->appendTemporaryDtor(E, cfg->getBumpVectorContext());
   }
+  void appendAutomaticObjDtor(CFGBlock *B, VarDecl *VD, Stmt *S) {
+    B->appendAutomaticObjDtor(VD, S, cfg->getBumpVectorContext());
+  }
 
-  void insertAutomaticObjDtors(CFGBlock* Blk, CFGBlock::iterator I,
-    LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt* S);
-  void appendAutomaticObjDtors(CFGBlock* Blk, LocalScope::const_iterator B,
-      LocalScope::const_iterator E, Stmt* S);
-  void prependAutomaticObjDtorsWithTerminator(CFGBlock* Blk,
+  void prependAutomaticObjDtorsWithTerminator(CFGBlock *Blk,
       LocalScope::const_iterator B, LocalScope::const_iterator E);
 
   void addSuccessor(CFGBlock *B, CFGBlock *S) {
@@ -437,20 +437,12 @@ class CFGBuilder {
   /// tryEvaluateBool - Try and evaluate the Stmt and return 0 or 1
   /// if we can evaluate to a known value, otherwise return -1.
   TryResult tryEvaluateBool(Expr *S) {
-    Expr::EvalResult Result;
-    if (!tryEvaluate(S, Result))
+    bool Result;
+    if (!BuildOpts.PruneTriviallyFalseEdges ||
+        S->isTypeDependent() || S->isValueDependent() ||
+        !S->EvaluateAsBooleanCondition(Result, *Context))
       return TryResult();
-    
-    if (Result.Val.isInt())
-      return Result.Val.getInt().getBoolValue();
-
-    if (Result.Val.isLValue()) {
-      const Expr *e = Result.Val.getLValueBase();
-      const CharUnits &c = Result.Val.getLValueOffset();        
-      if (!e && c.isZero())
-        return false;        
-    }
-    return TryResult();
+    return Result;
   }
   
 };
@@ -461,15 +453,17 @@ inline bool AddStmtChoice::alwaysAdd(CFGBuilder &builder,
 }
 
 bool CFGBuilder::alwaysAdd(const Stmt *stmt) {
+  bool shouldAdd = BuildOpts.alwaysAdd(stmt);
+  
   if (!BuildOpts.forcedBlkExprs)
-    return false;
+    return shouldAdd;
 
   if (lastLookup == stmt) {  
     if (cachedEntry) {
       assert(cachedEntry->first == stmt);
       return true;
     }
-    return false;
+    return shouldAdd;
   }
   
   lastLookup = stmt;
@@ -480,13 +474,13 @@ bool CFGBuilder::alwaysAdd(const Stmt *stmt) {
   if (!fb) {
     // No need to update 'cachedEntry', since it will always be null.
     assert(cachedEntry == 0);
-    return false;
+    return shouldAdd;
   }
 
   CFG::BuildOptions::ForcedBlkExprs::iterator itr = fb->find(stmt);
   if (itr == fb->end()) {
     cachedEntry = 0;
-    return false;
+    return shouldAdd;
   }
 
   cachedEntry = &*itr;
@@ -512,7 +506,7 @@ static const VariableArrayType *FindVA(const Type *t) {
 ///  body (compound statement).  The ownership of the returned CFG is
 ///  transferred to the caller.  If CFG construction fails, this method returns
 ///  NULL.
-CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement) {
+CFG* CFGBuilder::buildCFG(const Decl *D, Stmt *Statement) {
   assert(cfg.get());
   if (!Statement)
     return NULL;
@@ -552,8 +546,8 @@ CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement) {
   for (BackpatchBlocksTy::iterator I = BackpatchBlocks.begin(),
                                    E = BackpatchBlocks.end(); I != E; ++I ) {
 
-    CFGBlock* B = I->block;
-    GotoStmt* G = cast<GotoStmt>(B->getTerminator());
+    CFGBlock *B = I->block;
+    GotoStmt *G = cast<GotoStmt>(B->getTerminator());
     LabelMapTy::iterator LI = LabelMap.find(G->getLabel());
 
     // If there is no target for the goto, then we are looking at an
@@ -567,7 +561,7 @@ CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement) {
   }
 
   // Add successors to the Indirect Goto Dispatch block (if we have one).
-  if (CFGBlock* B = cfg->getIndirectGotoBlock())
+  if (CFGBlock *B = cfg->getIndirectGotoBlock())
     for (LabelSetTy::iterator I = AddressTakenLabels.begin(),
                               E = AddressTakenLabels.end(); I != E; ++I ) {
       
@@ -589,13 +583,23 @@ CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement) {
 
 /// createBlock - Used to lazily create blocks that are connected
 ///  to the current (global) succcessor.
-CFGBlock* CFGBuilder::createBlock(bool add_successor) {
-  CFGBlock* B = cfg->createBlock();
+CFGBlock *CFGBuilder::createBlock(bool add_successor) {
+  CFGBlock *B = cfg->createBlock();
   if (add_successor && Succ)
     addSuccessor(B, Succ);
   return B;
 }
 
+/// createNoReturnBlock - Used to create a block is a 'noreturn' point in the
+/// CFG. It is *not* connected to the current (global) successor, and instead
+/// directly tied to the exit block in order to be reachable.
+CFGBlock *CFGBuilder::createNoReturnBlock() {
+  CFGBlock *B = createBlock(false);
+  B->setHasNoReturnElement();
+  addSuccessor(B, &cfg->getExit());
+  return B;
+}
+
 /// addInitializer - Add C++ base or member initializer element to CFG.
 CFGBlock *CFGBuilder::addInitializer(CXXCtorInitializer *I) {
   if (!BuildOpts.AddInitializers)
@@ -638,15 +642,41 @@ CFGBlock *CFGBuilder::addInitializer(CXXCtorInitializer *I) {
 /// for objects in range of local scope positions. Use S as trigger statement
 /// for destructors.
 void CFGBuilder::addAutomaticObjDtors(LocalScope::const_iterator B,
-                                      LocalScope::const_iterator E, Stmt* S) {
+                                      LocalScope::const_iterator E, Stmt *S) {
   if (!BuildOpts.AddImplicitDtors)
     return;
 
   if (B == E)
     return;
 
-  autoCreateBlock();
-  appendAutomaticObjDtors(Block, B, E, S);
+  CFGBlock::iterator InsertPos;
+
+  // We need to append the destructors in reverse order, but any one of them
+  // may be a no-return destructor which changes the CFG. As a result, buffer
+  // this sequence up and replay them in reverse order when appending onto the
+  // CFGBlock(s).
+  SmallVector<VarDecl*, 10> Decls;
+  Decls.reserve(B.distance(E));
+  for (LocalScope::const_iterator I = B; I != E; ++I)
+    Decls.push_back(*I);
+
+  for (SmallVectorImpl<VarDecl*>::reverse_iterator I = Decls.rbegin(),
+                                                   E = Decls.rend();
+       I != E; ++I) {
+    // If this destructor is marked as a no-return destructor, we need to
+    // create a new block for the destructor which does not have as a successor
+    // anything built thus far: control won't flow out of this block.
+    QualType Ty = (*I)->getType().getNonReferenceType();
+    if (const ArrayType *AT = Context->getAsArrayType(Ty))
+      Ty = AT->getElementType();
+    const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
+    if (cast<FunctionType>(Dtor->getType())->getNoReturnAttr())
+      Block = createNoReturnBlock();
+    else
+      autoCreateBlock();
+
+    appendAutomaticObjDtor(Block, *I, S);
+  }
 }
 
 /// addImplicitDtorsForDestructor - Add implicit destructors generated for
@@ -711,7 +741,7 @@ LocalScope* CFGBuilder::createOrReuseLocalScope(LocalScope* Scope) {
 
 /// addLocalScopeForStmt - Add LocalScope to local scopes tree for statement
 /// that should create implicit scope (e.g. if/else substatements). 
-void CFGBuilder::addLocalScopeForStmt(Stmt* S) {
+void CFGBuilder::addLocalScopeForStmt(Stmt *S) {
   if (!BuildOpts.AddImplicitDtors)
     return;
 
@@ -721,9 +751,7 @@ void CFGBuilder::addLocalScopeForStmt(Stmt* S) {
   if (CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {
     for (CompoundStmt::body_iterator BI = CS->body_begin(), BE = CS->body_end()
         ; BI != BE; ++BI) {
-      Stmt *SI = *BI;
-      if (LabelStmt *LS = dyn_cast<LabelStmt>(SI))
-        SI = LS->getSubStmt();
+      Stmt *SI = (*BI)->stripLabelLikeStatements();
       if (DeclStmt *DS = dyn_cast<DeclStmt>(SI))
         Scope = addLocalScopeForDeclStmt(DS, Scope);
     }
@@ -732,22 +760,20 @@ void CFGBuilder::addLocalScopeForStmt(Stmt* S) {
 
   // For any other statement scope will be implicit and as such will be
   // interesting only for DeclStmt.
-  if (LabelStmt *LS = dyn_cast<LabelStmt>(S))
-    S = LS->getSubStmt();
-  if (DeclStmt *DS = dyn_cast<DeclStmt>(S))
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(S->stripLabelLikeStatements()))
     addLocalScopeForDeclStmt(DS);
 }
 
 /// addLocalScopeForDeclStmt - Add LocalScope for declaration statement. Will
 /// reuse Scope if not NULL.
-LocalScope* CFGBuilder::addLocalScopeForDeclStmt(DeclStmt* DS,
+LocalScope* CFGBuilder::addLocalScopeForDeclStmt(DeclStmt *DS,
                                                  LocalScope* Scope) {
   if (!BuildOpts.AddImplicitDtors)
     return Scope;
 
   for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end()
       ; DI != DE; ++DI) {
-    if (VarDecl* VD = dyn_cast<VarDecl>(*DI))
+    if (VarDecl *VD = dyn_cast<VarDecl>(*DI))
       Scope = addLocalScopeForVarDecl(VD, Scope);
   }
   return Scope;
@@ -756,7 +782,7 @@ LocalScope* CFGBuilder::addLocalScopeForDeclStmt(DeclStmt* DS,
 /// addLocalScopeForVarDecl - Add LocalScope for variable declaration. It will
 /// create add scope for automatic objects and temporary objects bound to
 /// const reference. Will reuse Scope if not NULL.
-LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl* VD,
+LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl *VD,
                                                 LocalScope* Scope) {
   if (!BuildOpts.AddImplicitDtors)
     return Scope;
@@ -788,7 +814,7 @@ LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl* VD,
   }
 
   // Check if type is a C++ class with non-trivial destructor.
-  if (const CXXRecordDecl* CD = QT->getAsCXXRecordDecl())
+  if (const CXXRecordDecl *CD = QT->getAsCXXRecordDecl())
     if (!CD->hasTrivialDestructor()) {
       // Add the variable to scope
       Scope = createOrReuseLocalScope(Scope);
@@ -800,7 +826,7 @@ LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl* VD,
 
 /// addLocalScopeAndDtors - For given statement add local scope for it and
 /// add destructors that will cleanup the scope. Will reuse Scope if not NULL.
-void CFGBuilder::addLocalScopeAndDtors(Stmt* S) {
+void CFGBuilder::addLocalScopeAndDtors(Stmt *S) {
   if (!BuildOpts.AddImplicitDtors)
     return;
 
@@ -809,40 +835,27 @@ void CFGBuilder::addLocalScopeAndDtors(Stmt* S) {
   addAutomaticObjDtors(ScopePos, scopeBeginPos, S);
 }
 
-/// insertAutomaticObjDtors - Insert destructor CFGElements for variables with
-/// automatic storage duration to CFGBlock's elements vector. Insertion will be
-/// performed in place specified with iterator.
-void CFGBuilder::insertAutomaticObjDtors(CFGBlock* Blk, CFGBlock::iterator I,
-    LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt* S) {
-  BumpVectorContext& C = cfg->getBumpVectorContext();
-  I = Blk->beginAutomaticObjDtorsInsert(I, B.distance(E), C);
-  while (B != E)
-    I = Blk->insertAutomaticObjDtor(I, *B++, S);
-}
-
-/// appendAutomaticObjDtors - Append destructor CFGElements for variables with
-/// automatic storage duration to CFGBlock's elements vector. Elements will be
-/// appended to physical end of the vector which happens to be logical
-/// beginning.
-void CFGBuilder::appendAutomaticObjDtors(CFGBlock* Blk,
-    LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt* S) {
-  insertAutomaticObjDtors(Blk, Blk->begin(), B, E, S);
-}
-
 /// prependAutomaticObjDtorsWithTerminator - Prepend destructor CFGElements for
 /// variables with automatic storage duration to CFGBlock's elements vector.
 /// Elements will be prepended to physical beginning of the vector which
 /// happens to be logical end. Use blocks terminator as statement that specifies
 /// destructors call site.
-void CFGBuilder::prependAutomaticObjDtorsWithTerminator(CFGBlock* Blk,
+/// FIXME: This mechanism for adding automatic destructors doesn't handle
+/// no-return destructors properly.
+void CFGBuilder::prependAutomaticObjDtorsWithTerminator(CFGBlock *Blk,
     LocalScope::const_iterator B, LocalScope::const_iterator E) {
-  insertAutomaticObjDtors(Blk, Blk->end(), B, E, Blk->getTerminator());
+  BumpVectorContext &C = cfg->getBumpVectorContext();
+  CFGBlock::iterator InsertPos
+    = Blk->beginAutomaticObjDtorsInsert(Blk->end(), B.distance(E), C);
+  for (LocalScope::const_iterator I = B; I != E; ++I)
+    InsertPos = Blk->insertAutomaticObjDtor(InsertPos, *I,
+                                            Blk->getTerminator());
 }
 
 /// Visit - Walk the subtree of a statement and add extra
 ///   blocks for ternary operators, &&, and ||.  We also process "," and
 ///   DeclStmts (which may contain nested control-flow).
-CFGBlock* CFGBuilder::Visit(Stmt * S, AddStmtChoice asc) {
+CFGBlock *CFGBuilder::Visit(Stmt * S, AddStmtChoice asc) {
   if (!S) {
     badCFG = true;
     return 0;
@@ -996,7 +1009,7 @@ CFGBlock *CFGBuilder::VisitStmt(Stmt *S, AddStmtChoice asc) {
 }
 
 /// VisitChildren - Visit the children of a Stmt.
-CFGBlock *CFGBuilder::VisitChildren(Stmt* Terminator) {
+CFGBlock *CFGBuilder::VisitChildren(Stmt *Terminator) {
   CFGBlock *lastBlock = Block;  
   for (Stmt::child_range I = Terminator->children(); I; ++I)
     if (Stmt *child = *I)
@@ -1031,20 +1044,20 @@ CFGBlock *CFGBuilder::VisitUnaryOperator(UnaryOperator *U,
 CFGBlock *CFGBuilder::VisitBinaryOperator(BinaryOperator *B,
                                           AddStmtChoice asc) {
   if (B->isLogicalOp()) { // && or ||
-    CFGBlock* ConfluenceBlock = Block ? Block : createBlock();
+    CFGBlock *ConfluenceBlock = Block ? Block : createBlock();
     appendStmt(ConfluenceBlock, B);
 
     if (badCFG)
       return 0;
 
     // create the block evaluating the LHS
-    CFGBlock* LHSBlock = createBlock(false);
+    CFGBlock *LHSBlock = createBlock(false);
     LHSBlock->setTerminator(B);
 
     // create the block evaluating the RHS
     Succ = ConfluenceBlock;
     Block = NULL;
-    CFGBlock* RHSBlock = addStmt(B->getRHS());
+    CFGBlock *RHSBlock = addStmt(B->getRHS());
 
     if (RHSBlock) {
       if (badCFG)
@@ -1191,13 +1204,13 @@ CFGBlock *CFGBuilder::VisitCallExpr(CallExpr *C, AddStmtChoice asc) {
       return 0;
   }
 
-  Block = createBlock(!NoReturn);
+  if (NoReturn)
+    Block = createNoReturnBlock();
+  else
+    Block = createBlock();
+
   appendStmt(Block, C);
 
-  if (NoReturn) {
-    // Wire this to the exit block directly.
-    addSuccessor(Block, &cfg->getExit());
-  }
   if (AddEHEdge) {
     // Add exceptional edges.
     if (TryTerminatedBlock)
@@ -1211,7 +1224,7 @@ CFGBlock *CFGBuilder::VisitCallExpr(CallExpr *C, AddStmtChoice asc) {
 
 CFGBlock *CFGBuilder::VisitChooseExpr(ChooseExpr *C,
                                       AddStmtChoice asc) {
-  CFGBlock* ConfluenceBlock = Block ? Block : createBlock();
+  CFGBlock *ConfluenceBlock = Block ? Block : createBlock();
   appendStmt(ConfluenceBlock, C);
   if (badCFG)
     return 0;
@@ -1219,13 +1232,13 @@ CFGBlock *CFGBuilder::VisitChooseExpr(ChooseExpr *C,
   AddStmtChoice alwaysAdd = asc.withAlwaysAdd(true);
   Succ = ConfluenceBlock;
   Block = NULL;
-  CFGBlock* LHSBlock = Visit(C->getLHS(), alwaysAdd);
+  CFGBlock *LHSBlock = Visit(C->getLHS(), alwaysAdd);
   if (badCFG)
     return 0;
 
   Succ = ConfluenceBlock;
   Block = NULL;
-  CFGBlock* RHSBlock = Visit(C->getRHS(), alwaysAdd);
+  CFGBlock *RHSBlock = Visit(C->getRHS(), alwaysAdd);
   if (badCFG)
     return 0;
 
@@ -1239,9 +1252,9 @@ CFGBlock *CFGBuilder::VisitChooseExpr(ChooseExpr *C,
 }
 
 
-CFGBlock* CFGBuilder::VisitCompoundStmt(CompoundStmt* C) {
+CFGBlock *CFGBuilder::VisitCompoundStmt(CompoundStmt *C) {
   addLocalScopeAndDtors(C);
-  CFGBlock* LastBlock = Block;
+  CFGBlock *LastBlock = Block;
 
   for (CompoundStmt::reverse_body_iterator I=C->body_rbegin(), E=C->body_rend();
        I != E; ++I ) {
@@ -1264,7 +1277,7 @@ CFGBlock *CFGBuilder::VisitConditionalOperator(AbstractConditionalOperator *C,
 
   // Create the confluence block that will "merge" the results of the ternary
   // expression.
-  CFGBlock* ConfluenceBlock = Block ? Block : createBlock();
+  CFGBlock *ConfluenceBlock = Block ? Block : createBlock();
   appendStmt(ConfluenceBlock, C);
   if (badCFG)
     return 0;
@@ -1277,7 +1290,7 @@ CFGBlock *CFGBuilder::VisitConditionalOperator(AbstractConditionalOperator *C,
   //  e.g: x ?: y is shorthand for: x ? x : y;
   Succ = ConfluenceBlock;
   Block = NULL;
-  CFGBlock* LHSBlock = 0;
+  CFGBlock *LHSBlock = 0;
   const Expr *trueExpr = C->getTrueExpr();
   if (trueExpr != opaqueValue) {
     LHSBlock = Visit(C->getTrueExpr(), alwaysAdd);
@@ -1290,7 +1303,7 @@ CFGBlock *CFGBuilder::VisitConditionalOperator(AbstractConditionalOperator *C,
 
   // Create the block for the RHS expression.
   Succ = ConfluenceBlock;
-  CFGBlock* RHSBlock = Visit(C->getFalseExpr(), alwaysAdd);
+  CFGBlock *RHSBlock = Visit(C->getFalseExpr(), alwaysAdd);
   if (badCFG)
     return 0;
 
@@ -1331,7 +1344,7 @@ CFGBlock *CFGBuilder::VisitDeclStmt(DeclStmt *DS) {
   CFGBlock *B = 0;
 
   // FIXME: Add a reverse iterator for DeclStmt to avoid this extra copy.
-  typedef llvm::SmallVector<Decl*,10> BufTy;
+  typedef SmallVector<Decl*,10> BufTy;
   BufTy Buf(DS->decl_begin(), DS->decl_end());
 
   for (BufTy::reverse_iterator I = Buf.rbegin(), E = Buf.rend(); I != E; ++I) {
@@ -1355,7 +1368,7 @@ CFGBlock *CFGBuilder::VisitDeclStmt(DeclStmt *DS) {
 
 /// VisitDeclSubExpr - Utility method to add block-level expressions for
 /// DeclStmts and initializers in them.
-CFGBlock *CFGBuilder::VisitDeclSubExpr(DeclStmt* DS) {
+CFGBlock *CFGBuilder::VisitDeclSubExpr(DeclStmt *DS) {
   assert(DS->isSingleDecl() && "Can handle single declarations only.");
   Decl *D = DS->getSingleDecl();
  
@@ -1414,7 +1427,7 @@ CFGBlock *CFGBuilder::VisitDeclSubExpr(DeclStmt* DS) {
   return Block;
 }
 
-CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
+CFGBlock *CFGBuilder::VisitIfStmt(IfStmt *I) {
   // We may see an if statement in the middle of a basic block, or it may be the
   // first statement we are processing.  In either case, we create a new basic
   // block.  First, we create the blocks for the then...else statements, and
@@ -1428,7 +1441,7 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
 
   // Create local scope for possible condition variable.
   // Store scope position. Add implicit destructor.
-  if (VarDecl* VD = I->getConditionVariable()) {
+  if (VarDecl *VD = I->getConditionVariable()) {
     LocalScope::const_iterator BeginScopePos = ScopePos;
     addLocalScopeForVarDecl(VD);
     addAutomaticObjDtors(ScopePos, BeginScopePos, I);
@@ -1443,9 +1456,9 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
   }
 
   // Process the false branch.
-  CFGBlock* ElseBlock = Succ;
+  CFGBlock *ElseBlock = Succ;
 
-  if (Stmt* Else = I->getElse()) {
+  if (Stmt *Else = I->getElse()) {
     SaveAndRestore<CFGBlock*> sv(Succ);
 
     // NULL out Block so that the recursive call to Visit will
@@ -1468,9 +1481,9 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
   }
 
   // Process the true branch.
-  CFGBlock* ThenBlock;
+  CFGBlock *ThenBlock;
   {
-    Stmt* Then = I->getThen();
+    Stmt *Then = I->getThen();
     assert(Then);
     SaveAndRestore<CFGBlock*> sv(Succ);
     Block = NULL;
@@ -1526,7 +1539,7 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
 }
 
 
-CFGBlock* CFGBuilder::VisitReturnStmt(ReturnStmt* R) {
+CFGBlock *CFGBuilder::VisitReturnStmt(ReturnStmt *R) {
   // If we were in the middle of a block we stop processing that block.
   //
   // NOTE: If a "return" appears in the middle of a block, this means that the
@@ -1546,7 +1559,7 @@ CFGBlock* CFGBuilder::VisitReturnStmt(ReturnStmt* R) {
   return VisitStmt(R, AddStmtChoice::AlwaysAdd);
 }
 
-CFGBlock* CFGBuilder::VisitLabelStmt(LabelStmt *L) {
+CFGBlock *CFGBuilder::VisitLabelStmt(LabelStmt *L) {
   // Get the block of the labeled statement.  Add it to our map.
   addStmt(L->getSubStmt());
   CFGBlock *LabelBlock = Block;
@@ -1575,7 +1588,7 @@ CFGBlock* CFGBuilder::VisitLabelStmt(LabelStmt *L) {
   return LabelBlock;
 }
 
-CFGBlock* CFGBuilder::VisitGotoStmt(GotoStmt* G) {
+CFGBlock *CFGBuilder::VisitGotoStmt(GotoStmt *G) {
   // Goto is a control-flow statement.  Thus we stop processing the current
   // block and create a new one.
 
@@ -1597,8 +1610,8 @@ CFGBlock* CFGBuilder::VisitGotoStmt(GotoStmt* G) {
   return Block;
 }
 
-CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
-  CFGBlock* LoopSuccessor = NULL;
+CFGBlock *CFGBuilder::VisitForStmt(ForStmt *F) {
+  CFGBlock *LoopSuccessor = NULL;
 
   // Save local scope position because in case of condition variable ScopePos
   // won't be restored when traversing AST.
@@ -1607,11 +1620,11 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
   // Create local scope for init statement and possible condition variable.
   // Add destructor for init statement and condition variable.
   // Store scope position for continue statement.
-  if (Stmt* Init = F->getInit())
+  if (Stmt *Init = F->getInit())
     addLocalScopeForStmt(Init);
   LocalScope::const_iterator LoopBeginScopePos = ScopePos;
 
-  if (VarDecl* VD = F->getConditionVariable())
+  if (VarDecl *VD = F->getConditionVariable())
     addLocalScopeForVarDecl(VD);
   LocalScope::const_iterator ContinueScopePos = ScopePos;
 
@@ -1634,15 +1647,15 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
   // Because of short-circuit evaluation, the condition of the loop can span
   // multiple basic blocks.  Thus we need the "Entry" and "Exit" blocks that
   // evaluate the condition.
-  CFGBlock* ExitConditionBlock = createBlock(false);
-  CFGBlock* EntryConditionBlock = ExitConditionBlock;
+  CFGBlock *ExitConditionBlock = createBlock(false);
+  CFGBlock *EntryConditionBlock = ExitConditionBlock;
 
   // Set the terminator for the "exit" condition block.
   ExitConditionBlock->setTerminator(F);
 
   // Now add the actual condition to the condition block.  Because the condition
   // itself may contain control-flow, new blocks may be created.
-  if (Stmt* C = F->getCond()) {
+  if (Stmt *C = F->getCond()) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
     if (badCFG)
@@ -1691,7 +1704,7 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
     // Loop body should end with destructor of Condition variable (if any).
     addAutomaticObjDtors(ScopePos, LoopBeginScopePos, F);
 
-    if (Stmt* I = F->getInc()) {
+    if (Stmt *I = F->getInc()) {
       // Generate increment code in its own basic block.  This is the target of
       // continue statements.
       Succ = addStmt(I);
@@ -1723,7 +1736,7 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
 
     // Now populate the body block, and in the process create new blocks as we
     // walk the body of the loop.
-    CFGBlock* BodyBlock = addStmt(F->getBody());
+    CFGBlock *BodyBlock = addStmt(F->getBody());
 
     if (!BodyBlock)
       BodyBlock = ContinueJumpTarget.block;//can happen for "for (...;...;...);"
@@ -1740,7 +1753,7 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
 
   // If the loop contains initialization, create a new block for those
   // statements.  This block can also contain statements that precede the loop.
-  if (Stmt* I = F->getInit()) {
+  if (Stmt *I = F->getInit()) {
     Block = createBlock();
     return addStmt(I);
   }
@@ -1760,7 +1773,7 @@ CFGBlock *CFGBuilder::VisitMemberExpr(MemberExpr *M, AddStmtChoice asc) {
   return Visit(M->getBase());
 }
 
-CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
+CFGBlock *CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
   // Objective-C fast enumeration 'for' statements:
   //  http://developer.apple.com/documentation/Cocoa/Conceptual/ObjectiveC
   //
@@ -1793,7 +1806,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   //   a DeclStmt and the other returns a DeclRefExpr.
   //
 
-  CFGBlock* LoopSuccessor = 0;
+  CFGBlock *LoopSuccessor = 0;
 
   if (Block) {
     if (badCFG)
@@ -1804,8 +1817,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
     LoopSuccessor = Succ;
 
   // Build the condition blocks.
-  CFGBlock* ExitConditionBlock = createBlock(false);
-  CFGBlock* EntryConditionBlock = ExitConditionBlock;
+  CFGBlock *ExitConditionBlock = createBlock(false);
 
   // Set the terminator for the "exit" condition block.
   ExitConditionBlock->setTerminator(S);
@@ -1819,7 +1831,8 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   // Walk the 'element' expression to see if there are any side-effects.  We
   // generate new blocks as necessary.  We DON'T add the statement by default to
   // the CFG unless it contains control-flow.
-  EntryConditionBlock = Visit(S->getElement(), AddStmtChoice::NotAlwaysAdd);
+  CFGBlock *EntryConditionBlock = Visit(S->getElement(),
+                                        AddStmtChoice::NotAlwaysAdd);
   if (Block) {
     if (badCFG)
       return 0;
@@ -1840,7 +1853,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
     BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos);
     ContinueJumpTarget = JumpTarget(EntryConditionBlock, ScopePos);
 
-    CFGBlock* BodyBlock = addStmt(S->getBody());
+    CFGBlock *BodyBlock = addStmt(S->getBody());
 
     if (!BodyBlock)
       BodyBlock = EntryConditionBlock; // can happen for "for (X in Y) ;"
@@ -1862,7 +1875,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   return addStmt(S->getCollection());
 }
 
-CFGBlock* CFGBuilder::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt* S) {
+CFGBlock *CFGBuilder::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   // FIXME: Add locking 'primitives' to CFG for @synchronized.
 
   // Inline the body.
@@ -1886,13 +1899,13 @@ CFGBlock* CFGBuilder::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt* S) {
   return addStmt(S->getSynchExpr());
 }
 
-CFGBlock* CFGBuilder::VisitObjCAtTryStmt(ObjCAtTryStmt* S) {
+CFGBlock *CFGBuilder::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
   // FIXME
   return NYS();
 }
 
-CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
-  CFGBlock* LoopSuccessor = NULL;
+CFGBlock *CFGBuilder::VisitWhileStmt(WhileStmt *W) {
+  CFGBlock *LoopSuccessor = NULL;
 
   // Save local scope position because in case of condition variable ScopePos
   // won't be restored when traversing AST.
@@ -1901,7 +1914,7 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
   // Create local scope for possible condition variable.
   // Store scope position for continue statement.
   LocalScope::const_iterator LoopBeginScopePos = ScopePos;
-  if (VarDecl* VD = W->getConditionVariable()) {
+  if (VarDecl *VD = W->getConditionVariable()) {
     addLocalScopeForVarDecl(VD);
     addAutomaticObjDtors(ScopePos, LoopBeginScopePos, W);
   }
@@ -1919,8 +1932,8 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
   // Because of short-circuit evaluation, the condition of the loop can span
   // multiple basic blocks.  Thus we need the "Entry" and "Exit" blocks that
   // evaluate the condition.
-  CFGBlock* ExitConditionBlock = createBlock(false);
-  CFGBlock* EntryConditionBlock = ExitConditionBlock;
+  CFGBlock *ExitConditionBlock = createBlock(false);
+  CFGBlock *EntryConditionBlock = ExitConditionBlock;
 
   // Set the terminator for the "exit" condition block.
   ExitConditionBlock->setTerminator(W);
@@ -1928,7 +1941,7 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
   // Now add the actual condition to the condition block.  Because the condition
   // itself may contain control-flow, new blocks may be created.  Thus we update
   // "Succ" after adding the condition.
-  if (Stmt* C = W->getCond()) {
+  if (Stmt *C = W->getCond()) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
     // The condition might finish the current 'Block'.
@@ -1990,7 +2003,7 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
       addLocalScopeAndDtors(W->getBody());
 
     // Create the body.  The returned block is the entry to the loop body.
-    CFGBlock* BodyBlock = addStmt(W->getBody());
+    CFGBlock *BodyBlock = addStmt(W->getBody());
 
     if (!BodyBlock)
       BodyBlock = ContinueJumpTarget.block; // can happen for "while(...) ;"
@@ -2017,13 +2030,13 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
 }
 
 
-CFGBlock *CFGBuilder::VisitObjCAtCatchStmt(ObjCAtCatchStmt* S) {
+CFGBlock *CFGBuilder::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   // FIXME: For now we pretend that @catch and the code it contains does not
   //  exit.
   return Block;
 }
 
-CFGBlock* CFGBuilder::VisitObjCAtThrowStmt(ObjCAtThrowStmt* S) {
+CFGBlock *CFGBuilder::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
   // FIXME: This isn't complete.  We basically treat @throw like a return
   //  statement.
 
@@ -2042,7 +2055,7 @@ CFGBlock* CFGBuilder::VisitObjCAtThrowStmt(ObjCAtThrowStmt* S) {
   return VisitStmt(S, AddStmtChoice::AlwaysAdd);
 }
 
-CFGBlock* CFGBuilder::VisitCXXThrowExpr(CXXThrowExpr* T) {
+CFGBlock *CFGBuilder::VisitCXXThrowExpr(CXXThrowExpr *T) {
   // If we were in the middle of a block we stop processing that block.
   if (badCFG)
     return 0;
@@ -2062,8 +2075,8 @@ CFGBlock* CFGBuilder::VisitCXXThrowExpr(CXXThrowExpr* T) {
   return VisitStmt(T, AddStmtChoice::AlwaysAdd);
 }
 
-CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
-  CFGBlock* LoopSuccessor = NULL;
+CFGBlock *CFGBuilder::VisitDoStmt(DoStmt *D) {
+  CFGBlock *LoopSuccessor = NULL;
 
   // "do...while" is a control-flow statement.  Thus we stop processing the
   // current block.
@@ -2077,15 +2090,15 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
   // Because of short-circuit evaluation, the condition of the loop can span
   // multiple basic blocks.  Thus we need the "Entry" and "Exit" blocks that
   // evaluate the condition.
-  CFGBlock* ExitConditionBlock = createBlock(false);
-  CFGBlock* EntryConditionBlock = ExitConditionBlock;
+  CFGBlock *ExitConditionBlock = createBlock(false);
+  CFGBlock *EntryConditionBlock = ExitConditionBlock;
 
   // Set the terminator for the "exit" condition block.
   ExitConditionBlock->setTerminator(D);
 
   // Now add the actual condition to the condition block.  Because the condition
   // itself may contain control-flow, new blocks may be created.
-  if (Stmt* C = D->getCond()) {
+  if (Stmt *C = D->getCond()) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
     if (Block) {
@@ -2101,7 +2114,7 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
   const TryResult &KnownVal = tryEvaluateBool(D->getCond());
 
   // Process the loop body.
-  CFGBlock* BodyBlock = NULL;
+  CFGBlock *BodyBlock = NULL;
   {
     assert(D->getBody());
 
@@ -2165,7 +2178,7 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
   return BodyBlock;
 }
 
-CFGBlock* CFGBuilder::VisitContinueStmt(ContinueStmt* C) {
+CFGBlock *CFGBuilder::VisitContinueStmt(ContinueStmt *C) {
   // "continue" is a control-flow statement.  Thus we stop processing the
   // current block.
   if (badCFG)
@@ -2202,13 +2215,12 @@ CFGBlock *CFGBuilder::VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *E,
          VA != 0; VA = FindVA(VA->getElementType().getTypePtr()))
       lastBlock = addStmt(VA->getSizeExpr());
   }
-
   return lastBlock;
 }
 
 /// VisitStmtExpr - Utility method to handle (nested) statement
 ///  expressions (a GCC extension).
-CFGBlock* CFGBuilder::VisitStmtExpr(StmtExpr *SE, AddStmtChoice asc) {
+CFGBlock *CFGBuilder::VisitStmtExpr(StmtExpr *SE, AddStmtChoice asc) {
   if (asc.alwaysAdd(*this, SE)) {
     autoCreateBlock();
     appendStmt(Block, SE);
@@ -2216,10 +2228,10 @@ CFGBlock* CFGBuilder::VisitStmtExpr(StmtExpr *SE, AddStmtChoice asc) {
   return VisitCompoundStmt(SE->getSubStmt());
 }
 
-CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
+CFGBlock *CFGBuilder::VisitSwitchStmt(SwitchStmt *Terminator) {
   // "switch" is a control-flow statement.  Thus we stop processing the current
   // block.
-  CFGBlock* SwitchSuccessor = NULL;
+  CFGBlock *SwitchSuccessor = NULL;
 
   // Save local scope position because in case of condition variable ScopePos
   // won't be restored when traversing AST.
@@ -2227,7 +2239,7 @@ CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
 
   // Create local scope for possible condition variable.
   // Store scope position. Add implicit destructor.
-  if (VarDecl* VD = Terminator->getConditionVariable()) {
+  if (VarDecl *VD = Terminator->getConditionVariable()) {
     LocalScope::const_iterator SwitchBeginScopePos = ScopePos;
     addLocalScopeForVarDecl(VD);
     addAutomaticObjDtors(ScopePos, SwitchBeginScopePos, Terminator);
@@ -2323,11 +2335,8 @@ static bool shouldAddCase(bool &switchExclusivelyCovered,
   if (!switchExclusivelyCovered) {
     if (switchCond->Val.isInt()) {
       // Evaluate the LHS of the case value.
-      Expr::EvalResult V1;
-      CS->getLHS()->Evaluate(V1, Ctx);
-      assert(V1.Val.isInt());
+      const llvm::APSInt &lhsInt = CS->getLHS()->EvaluateKnownConstInt(Ctx);
       const llvm::APSInt &condInt = switchCond->Val.getInt();
-      const llvm::APSInt &lhsInt = V1.Val.getInt();
       
       if (condInt == lhsInt) {
         addCase = true;
@@ -2336,10 +2345,8 @@ static bool shouldAddCase(bool &switchExclusivelyCovered,
       else if (condInt < lhsInt) {
         if (const Expr *RHS = CS->getRHS()) {
           // Evaluate the RHS of the case value.
-          Expr::EvalResult V2;
-          RHS->Evaluate(V2, Ctx);
-          assert(V2.Val.isInt());
-          if (V2.Val.getInt() <= condInt) {
+          const llvm::APSInt &V2 = RHS->EvaluateKnownConstInt(Ctx);
+          if (V2 <= condInt) {
             addCase = true;
             switchExclusivelyCovered = true;
           }
@@ -2352,7 +2359,7 @@ static bool shouldAddCase(bool &switchExclusivelyCovered,
   return addCase;  
 }
 
-CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
+CFGBlock *CFGBuilder::VisitCaseStmt(CaseStmt *CS) {
   // CaseStmts are essentially labels, so they are the first statement in a
   // block.
   CFGBlock *TopBlock = 0, *LastBlock = 0;
@@ -2383,7 +2390,7 @@ CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
     addStmt(Sub);
   }
 
-  CFGBlock* CaseBlock = Block;
+  CFGBlock *CaseBlock = Block;
   if (!CaseBlock)
     CaseBlock = createBlock();
 
@@ -2416,7 +2423,7 @@ CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
   return Succ;
 }
 
-CFGBlock* CFGBuilder::VisitDefaultStmt(DefaultStmt* Terminator) {
+CFGBlock *CFGBuilder::VisitDefaultStmt(DefaultStmt *Terminator) {
   if (Terminator->getSubStmt())
     addStmt(Terminator->getSubStmt());
 
@@ -2450,7 +2457,7 @@ CFGBlock* CFGBuilder::VisitDefaultStmt(DefaultStmt* Terminator) {
 CFGBlock *CFGBuilder::VisitCXXTryStmt(CXXTryStmt *Terminator) {
   // "try"/"catch" is a control-flow statement.  Thus we stop processing the
   // current block.
-  CFGBlock* TrySuccessor = NULL;
+  CFGBlock *TrySuccessor = NULL;
 
   if (Block) {
     if (badCFG)
@@ -2492,8 +2499,8 @@ CFGBlock *CFGBuilder::VisitCXXTryStmt(CXXTryStmt *Terminator) {
   Succ = TrySuccessor;
 
   // Save the current "try" context.
-  SaveAndRestore<CFGBlock*> save_try(TryTerminatedBlock);
-  TryTerminatedBlock = NewTryTerminatedBlock;
+  SaveAndRestore<CFGBlock*> save_try(TryTerminatedBlock, NewTryTerminatedBlock);
+  cfg->addTryDispatchBlock(TryTerminatedBlock);
 
   assert(Terminator->getTryBlock() && "try must contain a non-NULL body");
   Block = NULL;
@@ -2501,7 +2508,7 @@ CFGBlock *CFGBuilder::VisitCXXTryStmt(CXXTryStmt *Terminator) {
   return Block;
 }
 
-CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
+CFGBlock *CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt *CS) {
   // CXXCatchStmt are treated like labels, so they are the first statement in a
   // block.
 
@@ -2511,7 +2518,7 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
 
   // Create local scope for possible exception variable.
   // Store scope position. Add implicit destructor.
-  if (VarDecl* VD = CS->getExceptionDecl()) {
+  if (VarDecl *VD = CS->getExceptionDecl()) {
     LocalScope::const_iterator BeginScopePos = ScopePos;
     addLocalScopeForVarDecl(VD);
     addAutomaticObjDtors(ScopePos, BeginScopePos, CS);
@@ -2520,7 +2527,7 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
   if (CS->getHandlerBlock())
     addStmt(CS->getHandlerBlock());
 
-  CFGBlock* CatchBlock = Block;
+  CFGBlock *CatchBlock = Block;
   if (!CatchBlock)
     CatchBlock = createBlock();
 
@@ -2535,7 +2542,7 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
   return CatchBlock;
 }
 
-CFGBlock* CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt* S) {
+CFGBlock *CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt *S) {
   // C++0x for-range statements are specified as [stmt.ranged]:
   //
   // {
@@ -2563,7 +2570,7 @@ CFGBlock* CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt* S) {
 
   // "for" is a control-flow statement.  Thus we stop processing the current
   // block.
-  CFGBlock* LoopSuccessor = NULL;
+  CFGBlock *LoopSuccessor = NULL;
   if (Block) {
     if (badCFG)
       return 0;
@@ -2577,7 +2584,7 @@ CFGBlock* CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt* S) {
   BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos);
 
   // The block for the __begin != __end expression.
-  CFGBlock* ConditionBlock = createBlock(false);
+  CFGBlock *ConditionBlock = createBlock(false);
   ConditionBlock->setTerminator(S);
 
   // Now add the actual condition to the condition block.
@@ -2713,9 +2720,9 @@ CFGBlock *CFGBuilder::VisitImplicitCastExpr(ImplicitCastExpr *E,
   return Visit(E->getSubExpr(), AddStmtChoice());
 }
 
-CFGBlock* CFGBuilder::VisitIndirectGotoStmt(IndirectGotoStmt* I) {
+CFGBlock *CFGBuilder::VisitIndirectGotoStmt(IndirectGotoStmt *I) {
   // Lazily create the indirect-goto dispatch block if there isn't one already.
-  CFGBlock* IBlock = cfg->getIndirectGotoBlock();
+  CFGBlock *IBlock = cfg->getIndirectGotoBlock();
 
   if (!IBlock) {
     IBlock = createBlock(false);
@@ -2774,7 +2781,7 @@ CFGBlock *CFGBuilder::VisitChildrenForTemporaryDtors(Stmt *E) {
   // When visiting children for destructors we want to visit them in reverse
   // order. Because there's no reverse iterator for children must to reverse
   // them in helper vector.
-  typedef llvm::SmallVector<Stmt *, 4> ChildrenVect;
+  typedef SmallVector<Stmt *, 4> ChildrenVect;
   ChildrenVect ChildrenRev;
   for (Stmt::child_range I = E->children(); I; ++I) {
     if (*I) ChildrenRev.push_back(*I);
@@ -2864,7 +2871,16 @@ CFGBlock *CFGBuilder::VisitCXXBindTemporaryExprForTemporaryDtors(
   if (!BindToTemporary) {
     // If lifetime of temporary is not prolonged (by assigning to constant
     // reference) add destructor for it.
-    autoCreateBlock();
+
+    // If the destructor is marked as a no-return destructor, we need to create
+    // a new block for the destructor which does not have as a successor
+    // anything built thus far. Control won't flow out of this block.
+    const CXXDestructorDecl *Dtor = E->getTemporary()->getDestructor();
+    if (cast<FunctionType>(Dtor->getType())->getNoReturnAttr())
+      Block = createNoReturnBlock();
+    else
+      autoCreateBlock();
+
     appendTemporaryDtor(Block, E);
     B = Block;
   }
@@ -2937,7 +2953,7 @@ CFGBlock *CFGBuilder::VisitConditionalOperatorForTemporaryDtors(
 /// createBlock - Constructs and adds a new CFGBlock to the CFG.  The block has
 ///  no successors or predecessors.  If this is the first block created in the
 ///  CFG, it is automatically set to be the Entry and Exit of the CFG.
-CFGBlock* CFG::createBlock() {
+CFGBlock *CFG::createBlock() {
   bool first_block = begin() == end();
 
   // Create the block.
@@ -2955,7 +2971,7 @@ CFGBlock* CFG::createBlock() {
 
 /// buildCFG - Constructs a CFG from an AST.  Ownership of the returned
 ///  CFG is returned to the caller.
-CFG* CFG::buildCFG(const Decl *D, Stmt* Statement, ASTContext *C,
+CFG* CFG::buildCFG(const Decl *D, Stmt *Statement, ASTContext *C,
     const BuildOptions &BO) {
   CFGBuilder Builder(C, BO);
   return Builder.buildCFG(D, Statement);
@@ -3013,17 +3029,17 @@ namespace {
   typedef llvm::DenseMap<const Stmt*,unsigned> BlkExprMapTy;
 }
 
-static void FindSubExprAssignments(Stmt *S,
-                                   llvm::SmallPtrSet<Expr*,50>& Set) {
+static void FindSubExprAssignments(const Stmt *S,
+                                   llvm::SmallPtrSet<const Expr*,50>& Set) {
   if (!S)
     return;
 
-  for (Stmt::child_range I = S->children(); I; ++I) {
-    Stmt *child = *I;
+  for (Stmt::const_child_range I = S->children(); I; ++I) {
+    const Stmt *child = *I;
     if (!child)
       continue;
 
-    if (BinaryOperator* B = dyn_cast<BinaryOperator>(child))
+    if (const BinaryOperator* B = dyn_cast<BinaryOperator>(child))
       if (B->isAssignmentOp()) Set.insert(B);
 
     FindSubExprAssignments(child, Set);
@@ -3037,7 +3053,7 @@ static BlkExprMapTy* PopulateBlkExprMap(CFG& cfg) {
   // assignments that we want to *possibly* register as a block-level
   // expression.  Basically, if an assignment occurs both in a subexpression and
   // at the block-level, it is a block-level expression.
-  llvm::SmallPtrSet<Expr*,50> SubExprAssignments;
+  llvm::SmallPtrSet<const Expr*,50> SubExprAssignments;
 
   for (CFG::iterator I=cfg.begin(), E=cfg.end(); I != E; ++I)
     for (CFGBlock::iterator BI=(*I)->begin(), EI=(*I)->end(); BI != EI; ++BI)
@@ -3053,19 +3069,19 @@ static BlkExprMapTy* PopulateBlkExprMap(CFG& cfg) {
       const CFGStmt *CS = BI->getAs<CFGStmt>();
       if (!CS)
         continue;
-      if (Expr* Exp = dyn_cast<Expr>(CS->getStmt())) {
+      if (const Expr *Exp = dyn_cast<Expr>(CS->getStmt())) {
         assert((Exp->IgnoreParens() == Exp) && "No parens on block-level exps");
 
-        if (BinaryOperator* B = dyn_cast<BinaryOperator>(Exp)) {
+        if (const BinaryOperator* B = dyn_cast<BinaryOperator>(Exp)) {
           // Assignment expressions that are not nested within another
           // expression are really "statements" whose value is never used by
           // another expression.
           if (B->isAssignmentOp() && !SubExprAssignments.count(Exp))
             continue;
-        } else if (const StmtExpr* SE = dyn_cast<StmtExpr>(Exp)) {
+        } else if (const StmtExpr *SE = dyn_cast<StmtExpr>(Exp)) {
           // Special handling for statement expressions.  The last statement in
           // the statement expression is also a block-level expr.
-          const CompoundStmt* C = SE->getSubStmt();
+          const CompoundStmt *C = SE->getSubStmt();
           if (!C->body_empty()) {
             const Stmt *Last = C->body_back();
             if (const Expr *LastEx = dyn_cast<Expr>(Last))
@@ -3082,7 +3098,7 @@ static BlkExprMapTy* PopulateBlkExprMap(CFG& cfg) {
 
     // Look at terminators.  The condition is a block-level expression.
 
-    Stmt* S = (*I)->getTerminatorCondition();
+    Stmt *S = (*I)->getTerminatorCondition();
 
     if (S && M->find(S) == M->end()) {
       unsigned x = M->size();
@@ -3093,7 +3109,7 @@ static BlkExprMapTy* PopulateBlkExprMap(CFG& cfg) {
   return M;
 }
 
-CFG::BlkExprNumTy CFG::getBlkExprNum(const Stmt* S) {
+CFG::BlkExprNumTy CFG::getBlkExprNum(const Stmt *S) {
   assert(S != NULL);
   if (!BlkExprMap) { BlkExprMap = (void*) PopulateBlkExprMap(*this); }
 
@@ -3223,7 +3239,7 @@ class StmtPrinterHelper : public PrinterHelper  {
   void setBlockID(signed i) { currentBlock = i; }
   void setStmtID(unsigned i) { currentStmt = i; }
 
-  virtual bool handledStmt(Stmt* S, llvm::raw_ostream& OS) {
+  virtual bool handledStmt(Stmt *S, raw_ostream &OS) {
     StmtMapTy::iterator I = StmtMap.find(S);
 
     if (I == StmtMap.end())
@@ -3238,7 +3254,7 @@ class StmtPrinterHelper : public PrinterHelper  {
     return true;
   }
 
-  bool handleDecl(const Decl* D, llvm::raw_ostream& OS) {
+  bool handleDecl(const Decl *D, raw_ostream &OS) {
     DeclMapTy::iterator I = DeclMap.find(D);
 
     if (I == DeclMap.end())
@@ -3260,30 +3276,30 @@ namespace {
 class CFGBlockTerminatorPrint
   : public StmtVisitor<CFGBlockTerminatorPrint,void> {
 
-  llvm::raw_ostream& OS;
+  raw_ostream &OS;
   StmtPrinterHelper* Helper;
   PrintingPolicy Policy;
 public:
-  CFGBlockTerminatorPrint(llvm::raw_ostream& os, StmtPrinterHelper* helper,
+  CFGBlockTerminatorPrint(raw_ostream &os, StmtPrinterHelper* helper,
                           const PrintingPolicy &Policy)
     : OS(os), Helper(helper), Policy(Policy) {}
 
-  void VisitIfStmt(IfStmt* I) {
+  void VisitIfStmt(IfStmt *I) {
     OS << "if ";
     I->getCond()->printPretty(OS,Helper,Policy);
   }
 
   // Default case.
-  void VisitStmt(Stmt* Terminator) {
+  void VisitStmt(Stmt *Terminator) {
     Terminator->printPretty(OS, Helper, Policy);
   }
 
-  void VisitForStmt(ForStmt* F) {
+  void VisitForStmt(ForStmt *F) {
     OS << "for (" ;
     if (F->getInit())
       OS << "...";
     OS << "; ";
-    if (Stmt* C = F->getCond())
+    if (Stmt *C = F->getCond())
       C->printPretty(OS, Helper, Policy);
     OS << "; ";
     if (F->getInc())
@@ -3291,24 +3307,24 @@ class CFGBlockTerminatorPrint
     OS << ")";
   }
 
-  void VisitWhileStmt(WhileStmt* W) {
+  void VisitWhileStmt(WhileStmt *W) {
     OS << "while " ;
-    if (Stmt* C = W->getCond())
+    if (Stmt *C = W->getCond())
       C->printPretty(OS, Helper, Policy);
   }
 
-  void VisitDoStmt(DoStmt* D) {
+  void VisitDoStmt(DoStmt *D) {
     OS << "do ... while ";
-    if (Stmt* C = D->getCond())
+    if (Stmt *C = D->getCond())
       C->printPretty(OS, Helper, Policy);
   }
 
-  void VisitSwitchStmt(SwitchStmt* Terminator) {
+  void VisitSwitchStmt(SwitchStmt *Terminator) {
     OS << "switch ";
     Terminator->getCond()->printPretty(OS, Helper, Policy);
   }
 
-  void VisitCXXTryStmt(CXXTryStmt* CS) {
+  void VisitCXXTryStmt(CXXTryStmt *CS) {
     OS << "try ...";
   }
 
@@ -3317,13 +3333,13 @@ class CFGBlockTerminatorPrint
     OS << " ? ... : ...";
   }
 
-  void VisitChooseExpr(ChooseExpr* C) {
+  void VisitChooseExpr(ChooseExpr *C) {
     OS << "__builtin_choose_expr( ";
     C->getCond()->printPretty(OS, Helper, Policy);
     OS << " )";
   }
 
-  void VisitIndirectGotoStmt(IndirectGotoStmt* I) {
+  void VisitIndirectGotoStmt(IndirectGotoStmt *I) {
     OS << "goto *";
     I->getTarget()->printPretty(OS, Helper, Policy);
   }
@@ -3344,26 +3360,26 @@ class CFGBlockTerminatorPrint
         OS << " && ...";
         return;
       default:
-        assert(false && "Invalid logical operator.");
+        llvm_unreachable("Invalid logical operator.");
     }
   }
 
-  void VisitExpr(Expr* E) {
+  void VisitExpr(Expr *E) {
     E->printPretty(OS, Helper, Policy);
   }
 };
 } // end anonymous namespace
 
-static void print_elem(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
+static void print_elem(raw_ostream &OS, StmtPrinterHelper* Helper,
                        const CFGElement &E) {
   if (const CFGStmt *CS = E.getAs<CFGStmt>()) {
-    Stmt *S = CS->getStmt();
+    const Stmt *S = CS->getStmt();
     
     if (Helper) {
 
       // special printing for statement-expressions.
-      if (StmtExpr* SE = dyn_cast<StmtExpr>(S)) {
-        CompoundStmt* Sub = SE->getSubStmt();
+      if (const StmtExpr *SE = dyn_cast<StmtExpr>(S)) {
+        const CompoundStmt *Sub = SE->getSubStmt();
 
         if (Sub->children()) {
           OS << "({ ... ; ";
@@ -3373,7 +3389,7 @@ static void print_elem(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
         }
       }
       // special printing for comma expressions.
-      if (BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
+      if (const BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
         if (B->getOpcode() == BO_Comma) {
           OS << "... , ";
           Helper->handledStmt(B->getRHS(),OS);
@@ -3401,7 +3417,7 @@ static void print_elem(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
     else OS << I->getAnyMember()->getName();
 
     OS << "(";
-    if (Expr* IE = I->getInit())
+    if (Expr *IE = I->getInit())
       IE->printPretty(OS, Helper, PrintingPolicy(Helper->getLangOpts()));
     OS << ")";
 
@@ -3410,7 +3426,7 @@ static void print_elem(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
     else OS << " (Member initializer)\n";
 
   } else if (const CFGAutomaticObjDtor *DE = E.getAs<CFGAutomaticObjDtor>()){
-    const VarDecl* VD = DE->getVarDecl();
+    const VarDecl *VD = DE->getVarDecl();
     Helper->handleDecl(VD, OS);
 
     const Type* T = VD->getType().getTypePtr();
@@ -3445,8 +3461,8 @@ static void print_elem(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
   }
 }
 
-static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
-                        const CFGBlock& B,
+static void print_block(raw_ostream &OS, const CFG* cfg,
+                        const CFGBlock &B,
                         StmtPrinterHelper* Helper, bool print_edges) {
 
   if (Helper) Helper->setBlockID(B.getBlockID());
@@ -3464,14 +3480,14 @@ static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
     OS << " ]\n";
 
   // Print the label of this block.
-  if (Stmt* Label = const_cast<Stmt*>(B.getLabel())) {
+  if (Stmt *Label = const_cast<Stmt*>(B.getLabel())) {
 
     if (print_edges)
       OS << "    ";
 
-    if (LabelStmt* L = dyn_cast<LabelStmt>(Label))
+    if (LabelStmt *L = dyn_cast<LabelStmt>(Label))
       OS << L->getName();
-    else if (CaseStmt* C = dyn_cast<CaseStmt>(Label)) {
+    else if (CaseStmt *C = dyn_cast<CaseStmt>(Label)) {
       OS << "case ";
       C->getLHS()->printPretty(OS, Helper,
                                PrintingPolicy(Helper->getLangOpts()));
@@ -3492,7 +3508,7 @@ static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
       OS << ")";
 
     } else
-      assert(false && "Invalid label statement in CFGBlock.");
+      llvm_unreachable("Invalid label statement in CFGBlock.");
 
     OS << ":\n";
   }
@@ -3571,7 +3587,7 @@ static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
 void CFG::dump(const LangOptions &LO) const { print(llvm::errs(), LO); }
 
 /// print - A simple pretty printer of a CFG that outputs to an ostream.
-void CFG::print(llvm::raw_ostream &OS, const LangOptions &LO) const {
+void CFG::print(raw_ostream &OS, const LangOptions &LO) const {
   StmtPrinterHelper Helper(this, LO);
 
   // Print the entry block.
@@ -3598,25 +3614,25 @@ void CFGBlock::dump(const CFG* cfg, const LangOptions &LO) const {
 
 /// print - A simple pretty printer of a CFGBlock that outputs to an ostream.
 ///   Generally this will only be called from CFG::print.
-void CFGBlock::print(llvm::raw_ostream& OS, const CFG* cfg,
+void CFGBlock::print(raw_ostream &OS, const CFG* cfg,
                      const LangOptions &LO) const {
   StmtPrinterHelper Helper(cfg, LO);
   print_block(OS, cfg, *this, &Helper, true);
 }
 
 /// printTerminator - A simple pretty printer of the terminator of a CFGBlock.
-void CFGBlock::printTerminator(llvm::raw_ostream &OS,
+void CFGBlock::printTerminator(raw_ostream &OS,
                                const LangOptions &LO) const {
   CFGBlockTerminatorPrint TPrinter(OS, NULL, PrintingPolicy(LO));
   TPrinter.Visit(const_cast<Stmt*>(getTerminator().getStmt()));
 }
 
-Stmt* CFGBlock::getTerminatorCondition() {
+Stmt *CFGBlock::getTerminatorCondition() {
   Stmt *Terminator = this->Terminator;
   if (!Terminator)
     return NULL;
 
-  Expr* E = NULL;
+  Expr *E = NULL;
 
   switch (Terminator->getStmtClass()) {
     default:
@@ -3693,7 +3709,7 @@ struct DOTGraphTraits<const CFG*> : public DefaultDOTGraphTraits {
 
   DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {}
 
-  static std::string getNodeLabel(const CFGBlock* Node, const CFG* Graph) {
+  static std::string getNodeLabel(const CFGBlock *Node, const CFG* Graph) {
 
 #ifndef NDEBUG
     std::string OutSStr;
diff --git a/contrib/llvm/tools/clang/lib/Analysis/CFGReachabilityAnalysis.cpp b/contrib/llvm/tools/clang/lib/Analysis/CFGReachabilityAnalysis.cpp
index 65cd0898573c..e77e72fa9fb1 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/CFGReachabilityAnalysis.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/CFGReachabilityAnalysis.cpp
@@ -40,7 +40,7 @@ bool CFGReverseBlockReachabilityAnalysis::isReachable(const CFGBlock *Src,
 // Maps reachability to a common node by walking the predecessors of the
 // destination node.
 void CFGReverseBlockReachabilityAnalysis::mapReachability(const CFGBlock *Dst) {
-  llvm::SmallVector<const CFGBlock *, 11> worklist;
+  SmallVector<const CFGBlock *, 11> worklist;
   llvm::BitVector visited(analyzed.size());
   
   ReachableSet &DstReachability = reachable[Dst->getBlockID()];
diff --git a/contrib/llvm/tools/clang/lib/Analysis/CFGStmtMap.cpp b/contrib/llvm/tools/clang/lib/Analysis/CFGStmtMap.cpp
index 1fd5eedfeb86..16df67678df5 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/CFGStmtMap.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/CFGStmtMap.cpp
@@ -19,7 +19,7 @@
 
 using namespace clang;
 
-typedef llvm::DenseMap<Stmt*,CFGBlock*> SMap;
+typedef llvm::DenseMap<const Stmt*, CFGBlock*> SMap;
 static SMap *AsMap(void *m) { return (SMap*) m; }
 
 CFGStmtMap::~CFGStmtMap() { delete AsMap(M); }
diff --git a/contrib/llvm/tools/clang/lib/Analysis/CocoaConventions.cpp b/contrib/llvm/tools/clang/lib/Analysis/CocoaConventions.cpp
index 90f7092f90ee..8acd1892f9c7 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/CocoaConventions.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/CocoaConventions.cpp
@@ -17,12 +17,9 @@
 #include "clang/AST/DeclObjC.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
-
 using namespace clang;
 using namespace ento;
 
-using llvm::StringRef;
-
 // The "fundamental rule" for naming conventions of methods:
 //  (url broken into two lines)
 //  http://developer.apple.com/documentation/Cocoa/Conceptual/
@@ -43,6 +40,7 @@ cocoa::NamingConvention cocoa::deriveNamingConvention(Selector S,
   case OMF_None:
   case OMF_autorelease:
   case OMF_dealloc:
+  case OMF_finalize:
   case OMF_release:
   case OMF_retain:
   case OMF_retainCount:
@@ -63,11 +61,11 @@ cocoa::NamingConvention cocoa::deriveNamingConvention(Selector S,
   return NoConvention;
 }
 
-bool cocoa::isRefType(QualType RetTy, llvm::StringRef Prefix,
-                      llvm::StringRef Name) {
+bool cocoa::isRefType(QualType RetTy, StringRef Prefix,
+                      StringRef Name) {
   // Recursively walk the typedef stack, allowing typedefs of reference types.
   while (const TypedefType *TD = dyn_cast<TypedefType>(RetTy.getTypePtr())) {
-    llvm::StringRef TDName = TD->getDecl()->getIdentifier()->getName();
+    StringRef TDName = TD->getDecl()->getIdentifier()->getName();
     if (TDName.startswith(Prefix) && TDName.endswith("Ref"))
       return true;
     
@@ -127,10 +125,16 @@ bool cocoa::isCocoaObjectRef(QualType Ty) {
   return false;
 }
 
-bool coreFoundation::followsCreateRule(llvm::StringRef functionName) {
-  llvm::StringRef::iterator it = functionName.begin();
-  llvm::StringRef::iterator start = it;
-  llvm::StringRef::iterator endI = functionName.end();
+bool coreFoundation::followsCreateRule(const FunctionDecl *fn) {
+  // For now, *just* base this on the function name, not on anything else.
+
+  const IdentifierInfo *ident = fn->getIdentifier();
+  if (!ident) return false;
+  StringRef functionName = ident->getName();
+  
+  StringRef::iterator it = functionName.begin();
+  StringRef::iterator start = it;
+  StringRef::iterator endI = functionName.end();
     
   while (true) {
     // Scan for the start of 'create' or 'copy'.
@@ -138,6 +142,10 @@ bool coreFoundation::followsCreateRule(llvm::StringRef functionName) {
       // Search for the first character.  It can either be 'C' or 'c'.
       char ch = *it;
       if (ch == 'C' || ch == 'c') {
+        // Make sure this isn't something like 'recreate' or 'Scopy'.
+        if (ch == 'c' && it != start && isalpha(*(it - 1)))
+          continue;
+
         ++it;
         break;
       }
@@ -149,14 +157,13 @@ bool coreFoundation::followsCreateRule(llvm::StringRef functionName) {
     
     // Scan for *lowercase* 'reate' or 'opy', followed by no lowercase
     // character.
-    llvm::StringRef suffix = functionName.substr(it - start);
+    StringRef suffix = functionName.substr(it - start);
     if (suffix.startswith("reate")) {
       it += 5;
     }
     else if (suffix.startswith("opy")) {
       it += 3;
-    }
-    else {
+    } else {
       // Keep scanning.
       continue;
     }
diff --git a/contrib/llvm/tools/clang/lib/Analysis/FormatString.cpp b/contrib/llvm/tools/clang/lib/Analysis/FormatString.cpp
index 7c1e4535292a..a26f0ad6cd50 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/FormatString.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/FormatString.cpp
@@ -209,8 +209,7 @@ clang::analyze_format_string::ParseLengthModifier(FormatSpecifier &FS,
 bool ArgTypeResult::matchesType(ASTContext &C, QualType argTy) const {
   switch (K) {
     case InvalidTy:
-      assert(false && "ArgTypeResult must be valid");
-      return true;
+      llvm_unreachable("ArgTypeResult must be valid");
 
     case UnknownTy:
       return true;
@@ -312,8 +311,7 @@ bool ArgTypeResult::matchesType(ASTContext &C, QualType argTy) const {
 QualType ArgTypeResult::getRepresentativeType(ASTContext &C) const {
   switch (K) {
     case InvalidTy:
-      assert(false && "No representative type for Invalid ArgTypeResult");
-      // Fall-through.
+      llvm_unreachable("No representative type for Invalid ArgTypeResult");
     case UnknownTy:
       return QualType();
     case SpecificTy:
@@ -379,7 +377,7 @@ analyze_format_string::LengthModifier::toString() const {
 // Methods on OptionalAmount.
 //===----------------------------------------------------------------------===//
 
-void OptionalAmount::toString(llvm::raw_ostream &os) const {
+void OptionalAmount::toString(raw_ostream &os) const {
   switch (hs) {
   case Invalid:
   case NotSpecified:
diff --git a/contrib/llvm/tools/clang/lib/Analysis/LiveVariables.cpp b/contrib/llvm/tools/clang/lib/Analysis/LiveVariables.cpp
index 7b36f85ab62c..62c5455e0f22 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/LiveVariables.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/LiveVariables.cpp
@@ -1,392 +1,674 @@
-//=- LiveVariables.cpp - Live Variable Analysis for Source CFGs -*- C++ --*-==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements Live Variables analysis for source-level CFGs.
-//
-//===----------------------------------------------------------------------===//
-
 #include "clang/Analysis/Analyses/LiveVariables.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/Stmt.h"
 #include "clang/Analysis/CFG.h"
-#include "clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h"
-#include "clang/Analysis/FlowSensitive/DataflowSolver.h"
-#include "clang/Analysis/Support/SaveAndRestore.h"
 #include "clang/Analysis/AnalysisContext.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/raw_ostream.h"
+#include "clang/AST/StmtVisitor.h"
+
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/DenseMap.h"
+
+#include <deque>
+#include <algorithm>
+#include <vector>
 
 using namespace clang;
 
-//===----------------------------------------------------------------------===//
-// Useful constants.
-//===----------------------------------------------------------------------===//
-
-static const bool Alive = true;
-static const bool Dead = false;
-
-//===----------------------------------------------------------------------===//
-// Dataflow initialization logic.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class RegisterDecls
-  : public CFGRecStmtDeclVisitor<RegisterDecls> {
-
-  LiveVariables::AnalysisDataTy& AD;
-
-  typedef llvm::SmallVector<VarDecl*, 20> AlwaysLiveTy;
-  AlwaysLiveTy AlwaysLive;
-
-
-public:
-  RegisterDecls(LiveVariables::AnalysisDataTy& ad) : AD(ad) {}
-
-  ~RegisterDecls() {
-
-    AD.AlwaysLive.resetValues(AD);
-
-    for (AlwaysLiveTy::iterator I = AlwaysLive.begin(), E = AlwaysLive.end();
-         I != E; ++ I)
-      AD.AlwaysLive(*I, AD) = Alive;
-  }
-
-  void VisitImplicitParamDecl(ImplicitParamDecl* IPD) {
-    // Register the VarDecl for tracking.
-    AD.Register(IPD);
-  }
-
-  void VisitVarDecl(VarDecl* VD) {
-    // Register the VarDecl for tracking.
-    AD.Register(VD);
-
-    // Does the variable have global storage?  If so, it is always live.
-    if (VD->hasGlobalStorage())
-      AlwaysLive.push_back(VD);
-  }
-
-  CFG& getCFG() { return AD.getCFG(); }
-};
-} // end anonymous namespace
-
-LiveVariables::LiveVariables(AnalysisContext &AC, bool killAtAssign) {
-  // Register all referenced VarDecls.
-  CFG &cfg = *AC.getCFG();
-  getAnalysisData().setCFG(cfg);
-  getAnalysisData().setContext(AC.getASTContext());
-  getAnalysisData().AC = &AC;
-  getAnalysisData().killAtAssign = killAtAssign;
-
-  RegisterDecls R(getAnalysisData());
-  cfg.VisitBlockStmts(R);
-
-  // Register all parameters even if they didn't occur in the function body.
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(AC.getDecl()))
-    for (FunctionDecl::param_const_iterator PI = FD->param_begin(), 
-           PE = FD->param_end(); PI != PE; ++PI)
-      getAnalysisData().Register(*PI);
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer functions.
-//===----------------------------------------------------------------------===//
-
 namespace {
 
-class TransferFuncs : public CFGRecStmtVisitor<TransferFuncs>{
-  LiveVariables::AnalysisDataTy& AD;
-  LiveVariables::ValTy LiveState;
-  const CFGBlock *currentBlock;
-public:
-  TransferFuncs(LiveVariables::AnalysisDataTy& ad) : AD(ad), currentBlock(0) {}
-
-  LiveVariables::ValTy& getVal() { return LiveState; }
-  CFG& getCFG() { return AD.getCFG(); }
-
-  void VisitDeclRefExpr(DeclRefExpr* DR);
-  void VisitBinaryOperator(BinaryOperator* B);
-  void VisitBlockExpr(BlockExpr *B);
-  void VisitAssign(BinaryOperator* B);
-  void VisitDeclStmt(DeclStmt* DS);
-  void BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt* S);
-  void VisitUnaryOperator(UnaryOperator* U);
-  void Visit(Stmt *S);
-  void VisitTerminator(CFGBlock* B);
+// FIXME: This is copy-pasted from ThreadSafety.c.  I wanted a patch that
+// contained working code before refactoring the implementation of both
+// files.
+class CFGBlockSet {
+  llvm::BitVector VisitedBlockIDs;
   
-  /// VisitConditionVariableInit - Handle the initialization of condition
-  ///  variables at branches.  Valid statements include IfStmt, ForStmt,
-  ///  WhileStmt, and SwitchStmt.
-  void VisitConditionVariableInit(Stmt *S);
-
-  void SetTopValue(LiveVariables::ValTy& V) {
-    V = AD.AlwaysLive;
+public:
+  // po_iterator requires this iterator, but the only interface needed is the
+  // value_type typedef.
+  struct iterator {
+    typedef const CFGBlock *value_type;
+  };
+  
+  CFGBlockSet() {}
+  CFGBlockSet(const CFG *G) : VisitedBlockIDs(G->getNumBlockIDs(), false) {}
+  
+  /// \brief Set the bit associated with a particular CFGBlock.
+  /// This is the important method for the SetType template parameter.
+  bool insert(const CFGBlock *Block) {
+    // Note that insert() is called by po_iterator, which doesn't check to make
+    // sure that Block is non-null.  Moreover, the CFGBlock iterator will
+    // occasionally hand out null pointers for pruned edges, so we catch those
+    // here.
+    if (Block == 0)
+      return false;  // if an edge is trivially false.
+    if (VisitedBlockIDs.test(Block->getBlockID()))
+      return false;
+    VisitedBlockIDs.set(Block->getBlockID());
+    return true;
   }
   
-  void setCurrentBlock(const CFGBlock *block) {
-    currentBlock = block;
+  /// \brief Check if the bit for a CFGBlock has been already set.
+  /// This method is for tracking visited blocks in the main threadsafety loop.
+  /// Block must not be null.
+  bool alreadySet(const CFGBlock *Block) {
+    return VisitedBlockIDs.test(Block->getBlockID());
   }
 };
 
-void TransferFuncs::Visit(Stmt *S) {
-
-  if (S == getCurrentBlkStmt()) {
-
-    if (AD.Observer)
-      AD.Observer->ObserveStmt(S, currentBlock, AD, LiveState);
-
-    if (getCFG().isBlkExpr(S))
-      LiveState(S, AD) = Dead;
-
-    StmtVisitor<TransferFuncs,void>::Visit(S);
+/// \brief We create a helper class which we use to iterate through CFGBlocks in
+/// the topological order.
+class TopologicallySortedCFG {
+  typedef llvm::po_iterator<const CFG*, CFGBlockSet, true>  po_iterator;
+  
+  std::vector<const CFGBlock*> Blocks;
+  
+  typedef llvm::DenseMap<const CFGBlock *, unsigned> BlockOrderTy;
+  BlockOrderTy BlockOrder;
+  
+  
+public:
+  typedef std::vector<const CFGBlock*>::reverse_iterator iterator;
+  
+  TopologicallySortedCFG(const CFG *CFGraph) {
+    Blocks.reserve(CFGraph->getNumBlockIDs());
+    CFGBlockSet BSet(CFGraph);
+    
+    for (po_iterator I = po_iterator::begin(CFGraph, BSet),
+         E = po_iterator::end(CFGraph, BSet); I != E; ++I) {
+      BlockOrder[*I] = Blocks.size() + 1;
+      Blocks.push_back(*I);      
+    }
   }
-  else if (!getCFG().isBlkExpr(S)) {
-
-    if (AD.Observer)
-      AD.Observer->ObserveStmt(S, currentBlock, AD, LiveState);
-
-    StmtVisitor<TransferFuncs,void>::Visit(S);
-
+  
+  iterator begin() {
+    return Blocks.rbegin();
   }
-  else {
-    // For block-level expressions, mark that they are live.
-    LiveState(S, AD) = Alive;
+  
+  iterator end() {
+    return Blocks.rend();
+  }
+  
+  bool empty() {
+    return begin() == end();
+  }
+  
+  struct BlockOrderCompare;
+  friend struct BlockOrderCompare;
+
+  struct BlockOrderCompare {
+    const TopologicallySortedCFG &TSC;
+  public:
+    BlockOrderCompare(const TopologicallySortedCFG &tsc) : TSC(tsc) {}
+    
+    bool operator()(const CFGBlock *b1, const CFGBlock *b2) const {
+      TopologicallySortedCFG::BlockOrderTy::const_iterator b1It = TSC.BlockOrder.find(b1);
+      TopologicallySortedCFG::BlockOrderTy::const_iterator b2It = TSC.BlockOrder.find(b2);
+      
+      unsigned b1V = (b1It == TSC.BlockOrder.end()) ? 0 : b1It->second;
+      unsigned b2V = (b2It == TSC.BlockOrder.end()) ? 0 : b2It->second;
+      return b1V > b2V;
+    }  
+  };
+  
+  BlockOrderCompare getComparator() const {
+    return BlockOrderCompare(*this);
+  }
+};
+
+class DataflowWorklist {
+  SmallVector<const CFGBlock *, 20> worklist;
+  llvm::BitVector enqueuedBlocks;
+  TopologicallySortedCFG TSC;
+public:
+  DataflowWorklist(const CFG &cfg)
+    : enqueuedBlocks(cfg.getNumBlockIDs()),
+      TSC(&cfg) {}
+  
+  void enqueueBlock(const CFGBlock *block);
+  void enqueueSuccessors(const CFGBlock *block);
+  void enqueuePredecessors(const CFGBlock *block);
+
+  const CFGBlock *dequeue();
+
+  void sortWorklist();
+};
+
+}
+
+void DataflowWorklist::enqueueBlock(const clang::CFGBlock *block) {
+  if (block && !enqueuedBlocks[block->getBlockID()]) {
+    enqueuedBlocks[block->getBlockID()] = true;
+    worklist.push_back(block);
   }
 }
   
-void TransferFuncs::VisitConditionVariableInit(Stmt *S) {
-  assert(!getCFG().isBlkExpr(S));
-  CFGRecStmtVisitor<TransferFuncs>::VisitConditionVariableInit(S);
-}
+void DataflowWorklist::enqueueSuccessors(const clang::CFGBlock *block) {
+  const unsigned OldWorklistSize = worklist.size();
+  for (CFGBlock::const_succ_iterator I = block->succ_begin(),
+       E = block->succ_end(); I != E; ++I) {
+    enqueueBlock(*I);
+  }
 
-void TransferFuncs::VisitTerminator(CFGBlock* B) {
-
-  const Stmt* E = B->getTerminatorCondition();
-
-  if (!E)
+  if (OldWorklistSize == 0 || OldWorklistSize == worklist.size())
     return;
 
-  assert (getCFG().isBlkExpr(E));
-  LiveState(E, AD) = Alive;
+  sortWorklist();
 }
 
-void TransferFuncs::VisitDeclRefExpr(DeclRefExpr* DR) {
-  if (VarDecl* V = dyn_cast<VarDecl>(DR->getDecl()))
-    LiveState(V, AD) = Alive;
-}
+void DataflowWorklist::enqueuePredecessors(const clang::CFGBlock *block) {
+  const unsigned OldWorklistSize = worklist.size();
+  for (CFGBlock::const_pred_iterator I = block->pred_begin(),
+       E = block->pred_end(); I != E; ++I) {
+    enqueueBlock(*I);
+  }
   
-void TransferFuncs::VisitBlockExpr(BlockExpr *BE) {
-  AnalysisContext::referenced_decls_iterator I, E;
-  llvm::tie(I, E) = AD.AC->getReferencedBlockVars(BE->getBlockDecl());
-  for ( ; I != E ; ++I) {
-    DeclBitVector_Types::Idx i = AD.getIdx(*I);
-    if (i.isValid())
-      LiveState.getBit(i) = Alive;
+  if (OldWorklistSize == 0 || OldWorklistSize == worklist.size())
+    return;
+
+  sortWorklist();
+}
+
+void DataflowWorklist::sortWorklist() {
+  std::sort(worklist.begin(), worklist.end(), TSC.getComparator());
+}
+
+
+const CFGBlock *DataflowWorklist::dequeue() {
+  if (worklist.empty())
+    return 0;
+  const CFGBlock *b = worklist.back();
+  worklist.pop_back();
+  enqueuedBlocks[b->getBlockID()] = false;
+  return b;
+}
+
+namespace {
+class LiveVariablesImpl {
+public:  
+  AnalysisContext &analysisContext;
+  std::vector<LiveVariables::LivenessValues> cfgBlockValues;
+  llvm::ImmutableSet<const Stmt *>::Factory SSetFact;
+  llvm::ImmutableSet<const VarDecl *>::Factory DSetFact;
+  llvm::DenseMap<const CFGBlock *, LiveVariables::LivenessValues> blocksEndToLiveness;
+  llvm::DenseMap<const CFGBlock *, LiveVariables::LivenessValues> blocksBeginToLiveness;
+  llvm::DenseMap<const Stmt *, LiveVariables::LivenessValues> stmtsToLiveness;
+  llvm::DenseMap<const DeclRefExpr *, unsigned> inAssignment;
+  const bool killAtAssign;
+  
+  LiveVariables::LivenessValues
+  merge(LiveVariables::LivenessValues valsA,
+        LiveVariables::LivenessValues valsB);
+      
+  LiveVariables::LivenessValues runOnBlock(const CFGBlock *block,
+                                           LiveVariables::LivenessValues val,
+                                           LiveVariables::Observer *obs = 0);
+
+  void dumpBlockLiveness(const SourceManager& M);
+
+  LiveVariablesImpl(AnalysisContext &ac, bool KillAtAssign)
+    : analysisContext(ac),
+      SSetFact(false), // Do not canonicalize ImmutableSets by default.
+      DSetFact(false), // This is a *major* performance win.
+      killAtAssign(KillAtAssign) {}
+};
+}
+
+static LiveVariablesImpl &getImpl(void *x) {
+  return *((LiveVariablesImpl *) x);
+}
+
+//===----------------------------------------------------------------------===//
+// Operations and queries on LivenessValues.
+//===----------------------------------------------------------------------===//
+
+bool LiveVariables::LivenessValues::isLive(const Stmt *S) const {
+  return liveStmts.contains(S);
+}
+
+bool LiveVariables::LivenessValues::isLive(const VarDecl *D) const {
+  return liveDecls.contains(D);
+}
+
+namespace {
+  template <typename SET>
+  SET mergeSets(SET A, SET B) {
+    if (A.isEmpty())
+      return B;
+    
+    for (typename SET::iterator it = B.begin(), ei = B.end(); it != ei; ++it) {
+      A = A.add(*it);
+    }
+    return A;
   }
 }
 
-void TransferFuncs::VisitBinaryOperator(BinaryOperator* B) {
-  if (B->isAssignmentOp()) VisitAssign(B);
-  else VisitStmt(B);
+LiveVariables::LivenessValues
+LiveVariablesImpl::merge(LiveVariables::LivenessValues valsA,
+                         LiveVariables::LivenessValues valsB) {  
+  
+  llvm::ImmutableSetRef<const Stmt *>
+    SSetRefA(valsA.liveStmts.getRootWithoutRetain(), SSetFact.getTreeFactory()),
+    SSetRefB(valsB.liveStmts.getRootWithoutRetain(), SSetFact.getTreeFactory());
+                                                
+  
+  llvm::ImmutableSetRef<const VarDecl *>
+    DSetRefA(valsA.liveDecls.getRootWithoutRetain(), DSetFact.getTreeFactory()),
+    DSetRefB(valsB.liveDecls.getRootWithoutRetain(), DSetFact.getTreeFactory());
+  
+
+  SSetRefA = mergeSets(SSetRefA, SSetRefB);
+  DSetRefA = mergeSets(DSetRefA, DSetRefB);
+  
+  // asImmutableSet() canonicalizes the tree, allowing us to do an easy
+  // comparison afterwards.
+  return LiveVariables::LivenessValues(SSetRefA.asImmutableSet(),
+                                       DSetRefA.asImmutableSet());  
 }
 
-void
-TransferFuncs::BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
+bool LiveVariables::LivenessValues::equals(const LivenessValues &V) const {
+  return liveStmts == V.liveStmts && liveDecls == V.liveDecls;
+}
 
-  // This is a block-level expression.  Its value is 'dead' before this point.
-  LiveState(S, AD) = Dead;
+//===----------------------------------------------------------------------===//
+// Query methods.
+//===----------------------------------------------------------------------===//
 
-  // This represents a 'use' of the collection.
-  Visit(S->getCollection());
+static bool isAlwaysAlive(const VarDecl *D) {
+  return D->hasGlobalStorage();
+}
 
-  // This represents a 'kill' for the variable.
-  Stmt* Element = S->getElement();
-  DeclRefExpr* DR = 0;
-  VarDecl* VD = 0;
+bool LiveVariables::isLive(const CFGBlock *B, const VarDecl *D) {
+  return isAlwaysAlive(D) || getImpl(impl).blocksEndToLiveness[B].isLive(D);
+}
 
-  if (DeclStmt* DS = dyn_cast<DeclStmt>(Element))
-    VD = cast<VarDecl>(DS->getSingleDecl());
-  else {
-    Expr* ElemExpr = cast<Expr>(Element)->IgnoreParens();
-    if ((DR = dyn_cast<DeclRefExpr>(ElemExpr)))
-      VD = cast<VarDecl>(DR->getDecl());
-    else {
-      Visit(ElemExpr);
+bool LiveVariables::isLive(const Stmt *S, const VarDecl *D) {
+  return isAlwaysAlive(D) || getImpl(impl).stmtsToLiveness[S].isLive(D);
+}
+
+bool LiveVariables::isLive(const Stmt *Loc, const Stmt *S) {
+  return getImpl(impl).stmtsToLiveness[Loc].isLive(S);
+}
+
+//===----------------------------------------------------------------------===//
+// Dataflow computation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+class TransferFunctions : public StmtVisitor<TransferFunctions> {
+  LiveVariablesImpl &LV;
+  LiveVariables::LivenessValues &val;
+  LiveVariables::Observer *observer;
+  const CFGBlock *currentBlock;
+public:
+  TransferFunctions(LiveVariablesImpl &im,
+                    LiveVariables::LivenessValues &Val,
+                    LiveVariables::Observer *Observer,
+                    const CFGBlock *CurrentBlock)
+  : LV(im), val(Val), observer(Observer), currentBlock(CurrentBlock) {}
+
+  void VisitBinaryOperator(BinaryOperator *BO);
+  void VisitBlockExpr(BlockExpr *BE);
+  void VisitDeclRefExpr(DeclRefExpr *DR);  
+  void VisitDeclStmt(DeclStmt *DS);
+  void VisitObjCForCollectionStmt(ObjCForCollectionStmt *OS);
+  void VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *UE);
+  void VisitUnaryOperator(UnaryOperator *UO);
+  void Visit(Stmt *S);
+};
+}
+
+static const VariableArrayType *FindVA(QualType Ty) {
+  const Type *ty = Ty.getTypePtr();
+  while (const ArrayType *VT = dyn_cast<ArrayType>(ty)) {
+    if (const VariableArrayType *VAT = dyn_cast<VariableArrayType>(VT))
+      if (VAT->getSizeExpr())
+        return VAT;
+    
+    ty = VT->getElementType().getTypePtr();
+  }
+  
+  return 0;
+}
+
+void TransferFunctions::Visit(Stmt *S) {
+  if (observer)
+    observer->observeStmt(S, currentBlock, val);
+  
+  StmtVisitor<TransferFunctions>::Visit(S);
+  
+  if (isa<Expr>(S)) {
+    val.liveStmts = LV.SSetFact.remove(val.liveStmts, S);
+  }
+
+  // Mark all children expressions live.
+  
+  switch (S->getStmtClass()) {
+    default:
+      break;
+    case Stmt::StmtExprClass: {
+      // For statement expressions, look through the compound statement.
+      S = cast<StmtExpr>(S)->getSubStmt();
+      break;
+    }
+    case Stmt::CXXMemberCallExprClass: {
+      // Include the implicit "this" pointer as being live.
+      CXXMemberCallExpr *CE = cast<CXXMemberCallExpr>(S);
+      if (Expr *ImplicitObj = CE->getImplicitObjectArgument()) {
+        ImplicitObj = ImplicitObj->IgnoreParens();        
+        val.liveStmts = LV.SSetFact.add(val.liveStmts, ImplicitObj);
+      }
+      break;
+    }
+    case Stmt::DeclStmtClass: {
+      const DeclStmt *DS = cast<DeclStmt>(S);
+      if (const VarDecl *VD = dyn_cast<VarDecl>(DS->getSingleDecl())) {
+        for (const VariableArrayType* VA = FindVA(VD->getType());
+             VA != 0; VA = FindVA(VA->getElementType())) {
+          val.liveStmts = LV.SSetFact.add(val.liveStmts,
+                                          VA->getSizeExpr()->IgnoreParens());
+        }
+      }
+      break;
+    }
+    // FIXME: These cases eventually shouldn't be needed.
+    case Stmt::ExprWithCleanupsClass: {
+      S = cast<ExprWithCleanups>(S)->getSubExpr();
+      break;
+    }
+    case Stmt::CXXBindTemporaryExprClass: {
+      S = cast<CXXBindTemporaryExpr>(S)->getSubExpr();
+      break;
+    }
+    case Stmt::UnaryExprOrTypeTraitExprClass: {
+      // No need to unconditionally visit subexpressions.
       return;
     }
   }
-
-  if (VD) {
-    LiveState(VD, AD) = Dead;
-    if (AD.Observer && DR) { AD.Observer->ObserverKill(DR); }
-  }
-}
-
-
-void TransferFuncs::VisitUnaryOperator(UnaryOperator* U) {
-  Expr *E = U->getSubExpr();
-
-  switch (U->getOpcode()) {
-  case UO_PostInc:
-  case UO_PostDec:
-  case UO_PreInc:
-  case UO_PreDec:
-    // Walk through the subexpressions, blasting through ParenExprs
-    // until we either find a DeclRefExpr or some non-DeclRefExpr
-    // expression.
-    if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(E->IgnoreParens()))
-      if (VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl())) {
-        // Treat the --/++ operator as a kill.
-        if (AD.Observer) { AD.Observer->ObserverKill(DR); }
-        LiveState(VD, AD) = Alive;
-        return VisitDeclRefExpr(DR);
-      }
-
-    // Fall-through.
-
-  default:
-    return Visit(E);
-  }
-}
-
-void TransferFuncs::VisitAssign(BinaryOperator* B) {
-  Expr* LHS = B->getLHS();
-
-  // Assigning to a variable?
-  if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(LHS->IgnoreParens())) {
-    // Assignments to references don't kill the ref's address
-    if (DR->getDecl()->getType()->isReferenceType()) {
-      VisitDeclRefExpr(DR);
-    } else {
-      if (AD.killAtAssign) {
-        // Update liveness inforamtion.
-        unsigned bit = AD.getIdx(DR->getDecl());
-        LiveState.getDeclBit(bit) = Dead | AD.AlwaysLive.getDeclBit(bit);
-
-        if (AD.Observer) { AD.Observer->ObserverKill(DR); }
-      }
-      // Handle things like +=, etc., which also generate "uses"
-      // of a variable.  Do this just by visiting the subexpression.
-      if (B->getOpcode() != BO_Assign)
-        VisitDeclRefExpr(DR);
+  
+  for (Stmt::child_iterator it = S->child_begin(), ei = S->child_end();
+       it != ei; ++it) {
+    if (Stmt *child = *it) {
+      if (Expr *Ex = dyn_cast<Expr>(child))
+        child = Ex->IgnoreParens();
+               
+      val.liveStmts = LV.SSetFact.add(val.liveStmts, child);
     }
   }
-  else // Not assigning to a variable.  Process LHS as usual.
-    Visit(LHS);
-
-  Visit(B->getRHS());
 }
 
-void TransferFuncs::VisitDeclStmt(DeclStmt* DS) {
-  // Declarations effectively "kill" a variable since they cannot
-  // possibly be live before they are declared.
+void TransferFunctions::VisitBinaryOperator(BinaryOperator *B) {
+  if (B->isAssignmentOp()) {
+    if (!LV.killAtAssign)
+      return;
+    
+    // Assigning to a variable?
+    Expr *LHS = B->getLHS()->IgnoreParens();
+    
+    if (DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LHS))
+      if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
+        // Assignments to references don't kill the ref's address
+        if (VD->getType()->isReferenceType())
+          return;
+
+        if (!isAlwaysAlive(VD)) {
+          // The variable is now dead.
+          val.liveDecls = LV.DSetFact.remove(val.liveDecls, VD);
+        }
+
+        if (observer)
+          observer->observerKill(DR);
+      }
+  }
+}
+
+void TransferFunctions::VisitBlockExpr(BlockExpr *BE) {
+  AnalysisContext::referenced_decls_iterator I, E;
+  llvm::tie(I, E) =
+    LV.analysisContext.getReferencedBlockVars(BE->getBlockDecl());
+  for ( ; I != E ; ++I) {
+    const VarDecl *VD = *I;
+    if (isAlwaysAlive(VD))
+      continue;
+    val.liveDecls = LV.DSetFact.add(val.liveDecls, VD);
+  }
+}
+
+void TransferFunctions::VisitDeclRefExpr(DeclRefExpr *DR) {
+  if (const VarDecl *D = dyn_cast<VarDecl>(DR->getDecl()))
+    if (!isAlwaysAlive(D) && LV.inAssignment.find(DR) == LV.inAssignment.end())
+      val.liveDecls = LV.DSetFact.add(val.liveDecls, D);
+}
+
+void TransferFunctions::VisitDeclStmt(DeclStmt *DS) {
   for (DeclStmt::decl_iterator DI=DS->decl_begin(), DE = DS->decl_end();
        DI != DE; ++DI)
-    if (VarDecl* VD = dyn_cast<VarDecl>(*DI)) {
-      // Update liveness information by killing the VarDecl.
-      unsigned bit = AD.getIdx(VD);
-      LiveState.getDeclBit(bit) = Dead | AD.AlwaysLive.getDeclBit(bit);
+    if (VarDecl *VD = dyn_cast<VarDecl>(*DI)) {
+      if (!isAlwaysAlive(VD))
+        val.liveDecls = LV.DSetFact.remove(val.liveDecls, VD);
+    }
+}
 
-      // The initializer is evaluated after the variable comes into scope, but
-      // before the DeclStmt (which binds the value to the variable).
-      // Since this is a reverse dataflow analysis, we must evaluate the
-      // transfer function for this expression after the DeclStmt.  If the
-      // initializer references the variable (which is bad) then we extend
-      // its liveness.
-      if (Expr* Init = VD->getInit())
-        Visit(Init);
+void TransferFunctions::VisitObjCForCollectionStmt(ObjCForCollectionStmt *OS) {
+  // Kill the iteration variable.
+  DeclRefExpr *DR = 0;
+  const VarDecl *VD = 0;
 
-      if (const VariableArrayType* VT =
-            AD.getContext().getAsVariableArrayType(VD->getType())) {
-        StmtIterator I(const_cast<VariableArrayType*>(VT));
-        StmtIterator E;
-        for (; I != E; ++I) Visit(*I);
+  Stmt *element = OS->getElement();
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(element)) {
+    VD = cast<VarDecl>(DS->getSingleDecl());
+  }
+  else if ((DR = dyn_cast<DeclRefExpr>(cast<Expr>(element)->IgnoreParens()))) {
+    VD = cast<VarDecl>(DR->getDecl());
+  }
+  
+  if (VD) {
+    val.liveDecls = LV.DSetFact.remove(val.liveDecls, VD);
+    if (observer && DR)
+      observer->observerKill(DR);
+  }
+}
+
+void TransferFunctions::
+VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *UE)
+{
+  // While sizeof(var) doesn't technically extend the liveness of 'var', it
+  // does extent the liveness of metadata if 'var' is a VariableArrayType.
+  // We handle that special case here.
+  if (UE->getKind() != UETT_SizeOf || UE->isArgumentType())
+    return;
+
+  const Expr *subEx = UE->getArgumentExpr();
+  if (subEx->getType()->isVariableArrayType()) {
+    assert(subEx->isLValue());
+    val.liveStmts = LV.SSetFact.add(val.liveStmts, subEx->IgnoreParens());
+  }
+}
+
+void TransferFunctions::VisitUnaryOperator(UnaryOperator *UO) {
+  // Treat ++/-- as a kill.
+  // Note we don't actually have to do anything if we don't have an observer,
+  // since a ++/-- acts as both a kill and a "use".
+  if (!observer)
+    return;
+  
+  switch (UO->getOpcode()) {
+  default:
+    return;
+  case UO_PostInc:
+  case UO_PostDec:    
+  case UO_PreInc:
+  case UO_PreDec:
+    break;
+  }
+  
+  if (DeclRefExpr *DR = dyn_cast<DeclRefExpr>(UO->getSubExpr()->IgnoreParens()))
+    if (isa<VarDecl>(DR->getDecl())) {
+      // Treat ++/-- as a kill.
+      observer->observerKill(DR);
+    }
+}
+
+LiveVariables::LivenessValues
+LiveVariablesImpl::runOnBlock(const CFGBlock *block,
+                              LiveVariables::LivenessValues val,
+                              LiveVariables::Observer *obs) {
+
+  TransferFunctions TF(*this, val, obs, block);
+  
+  // Visit the terminator (if any).
+  if (const Stmt *term = block->getTerminator())
+    TF.Visit(const_cast<Stmt*>(term));
+  
+  // Apply the transfer function for all Stmts in the block.
+  for (CFGBlock::const_reverse_iterator it = block->rbegin(),
+       ei = block->rend(); it != ei; ++it) {
+    const CFGElement &elem = *it;
+    if (!isa<CFGStmt>(elem))
+      continue;
+    
+    const Stmt *S = cast<CFGStmt>(elem).getStmt();
+    TF.Visit(const_cast<Stmt*>(S));
+    stmtsToLiveness[S] = val;
+  }
+  return val;
+}
+
+void LiveVariables::runOnAllBlocks(LiveVariables::Observer &obs) {
+  const CFG *cfg = getImpl(impl).analysisContext.getCFG();
+  for (CFG::const_iterator it = cfg->begin(), ei = cfg->end(); it != ei; ++it)
+    getImpl(impl).runOnBlock(*it, getImpl(impl).blocksEndToLiveness[*it], &obs);    
+}
+
+LiveVariables::LiveVariables(void *im) : impl(im) {} 
+
+LiveVariables::~LiveVariables() {
+  delete (LiveVariablesImpl*) impl;
+}
+
+LiveVariables *
+LiveVariables::computeLiveness(AnalysisContext &AC,
+                                 bool killAtAssign) {
+
+  // No CFG?  Bail out.
+  CFG *cfg = AC.getCFG();
+  if (!cfg)
+    return 0;
+
+  LiveVariablesImpl *LV = new LiveVariablesImpl(AC, killAtAssign);
+
+  // Construct the dataflow worklist.  Enqueue the exit block as the
+  // start of the analysis.
+  DataflowWorklist worklist(*cfg);
+  llvm::BitVector everAnalyzedBlock(cfg->getNumBlockIDs());
+
+  // FIXME: we should enqueue using post order.
+  for (CFG::const_iterator it = cfg->begin(), ei = cfg->end(); it != ei; ++it) {
+    const CFGBlock *block = *it;
+    worklist.enqueueBlock(block);
+    
+    // FIXME: Scan for DeclRefExprs using in the LHS of an assignment.
+    // We need to do this because we lack context in the reverse analysis
+    // to determine if a DeclRefExpr appears in such a context, and thus
+    // doesn't constitute a "use".
+    if (killAtAssign)
+      for (CFGBlock::const_iterator bi = block->begin(), be = block->end();
+           bi != be; ++bi) {
+        if (const CFGStmt *cs = bi->getAs<CFGStmt>()) {
+          if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(cs->getStmt())) {
+            if (BO->getOpcode() == BO_Assign) {
+              if (const DeclRefExpr *DR =
+                    dyn_cast<DeclRefExpr>(BO->getLHS()->IgnoreParens())) {
+                LV->inAssignment[DR] = 1;
+              }
+            }
+          }
+        }
+      }
+  }
+  
+  worklist.sortWorklist();
+  
+  while (const CFGBlock *block = worklist.dequeue()) {
+    // Determine if the block's end value has changed.  If not, we
+    // have nothing left to do for this block.
+    LivenessValues &prevVal = LV->blocksEndToLiveness[block];
+    
+    // Merge the values of all successor blocks.
+    LivenessValues val;
+    for (CFGBlock::const_succ_iterator it = block->succ_begin(),
+                                       ei = block->succ_end(); it != ei; ++it) {
+      if (const CFGBlock *succ = *it) {     
+        val = LV->merge(val, LV->blocksBeginToLiveness[succ]);
       }
     }
+    
+    if (!everAnalyzedBlock[block->getBlockID()])
+      everAnalyzedBlock[block->getBlockID()] = true;
+    else if (prevVal.equals(val))
+      continue;
+
+    prevVal = val;
+    
+    // Update the dataflow value for the start of this block.
+    LV->blocksBeginToLiveness[block] = LV->runOnBlock(block, val);
+    
+    // Enqueue the value to the predecessors.
+    worklist.enqueuePredecessors(block);
+  }
+  
+  return new LiveVariables(LV);
 }
 
-} // end anonymous namespace
-
-//===----------------------------------------------------------------------===//
-// Merge operator: if something is live on any successor block, it is live
-//  in the current block (a set union).
-//===----------------------------------------------------------------------===//
-
-namespace {
-  typedef StmtDeclBitVector_Types::Union Merge;
-  typedef DataflowSolver<LiveVariables, TransferFuncs, Merge> Solver;
-} // end anonymous namespace
-
-//===----------------------------------------------------------------------===//
-// External interface to run Liveness analysis.
-//===----------------------------------------------------------------------===//
-
-void LiveVariables::runOnCFG(CFG& cfg) {
-  Solver S(*this);
-  S.runOnCFG(cfg);
+static bool compare_entries(const CFGBlock *A, const CFGBlock *B) {
+  return A->getBlockID() < B->getBlockID();
 }
 
-void LiveVariables::runOnAllBlocks(const CFG& cfg,
-                                   LiveVariables::ObserverTy* Obs,
-                                   bool recordStmtValues) {
-  Solver S(*this);
-  SaveAndRestore<LiveVariables::ObserverTy*> SRObs(getAnalysisData().Observer,
-                                                   Obs);
-  S.runOnAllBlocks(cfg, recordStmtValues);
+static bool compare_vd_entries(const Decl *A, const Decl *B) {
+  SourceLocation ALoc = A->getLocStart();
+  SourceLocation BLoc = B->getLocStart();
+  return ALoc.getRawEncoding() < BLoc.getRawEncoding();
 }
 
-//===----------------------------------------------------------------------===//
-// liveness queries
-//
-
-bool LiveVariables::isLive(const CFGBlock* B, const VarDecl* D) const {
-  DeclBitVector_Types::Idx i = getAnalysisData().getIdx(D);
-  return i.isValid() ? getBlockData(B).getBit(i) : false;
+void LiveVariables::dumpBlockLiveness(const SourceManager &M) {
+  getImpl(impl).dumpBlockLiveness(M);
 }
 
-bool LiveVariables::isLive(const ValTy& Live, const VarDecl* D) const {
-  DeclBitVector_Types::Idx i = getAnalysisData().getIdx(D);
-  return i.isValid() ? Live.getBit(i) : false;
-}
+void LiveVariablesImpl::dumpBlockLiveness(const SourceManager &M) {
+  std::vector<const CFGBlock *> vec;
+  for (llvm::DenseMap<const CFGBlock *, LiveVariables::LivenessValues>::iterator
+       it = blocksEndToLiveness.begin(), ei = blocksEndToLiveness.end();
+       it != ei; ++it) {
+    vec.push_back(it->first);    
+  }
+  std::sort(vec.begin(), vec.end(), compare_entries);
 
-bool LiveVariables::isLive(const Stmt* Loc, const Stmt* StmtVal) const {
-  return getStmtData(Loc)(StmtVal,getAnalysisData());
-}
+  std::vector<const VarDecl*> declVec;
 
-bool LiveVariables::isLive(const Stmt* Loc, const VarDecl* D) const {
-  return getStmtData(Loc)(D,getAnalysisData());
-}
-
-//===----------------------------------------------------------------------===//
-// printing liveness state for debugging
-//
-
-void LiveVariables::dumpLiveness(const ValTy& V, const SourceManager& SM) const {
-  const AnalysisDataTy& AD = getAnalysisData();
-
-  for (AnalysisDataTy::decl_iterator I = AD.begin_decl(),
-                                     E = AD.end_decl(); I!=E; ++I)
-    if (V.getDeclBit(I->second)) {
-      llvm::errs() << "  " << I->first->getIdentifier()->getName() << " <";
-      I->first->getLocation().dump(SM);
+  for (std::vector<const CFGBlock *>::iterator
+        it = vec.begin(), ei = vec.end(); it != ei; ++it) {
+    llvm::errs() << "\n[ B" << (*it)->getBlockID()
+                 << " (live variables at block exit) ]\n";
+    
+    LiveVariables::LivenessValues vals = blocksEndToLiveness[*it];
+    declVec.clear();
+    
+    for (llvm::ImmutableSet<const VarDecl *>::iterator si =
+          vals.liveDecls.begin(),
+          se = vals.liveDecls.end(); si != se; ++si) {
+      declVec.push_back(*si);      
+    }
+    
+    std::sort(declVec.begin(), declVec.end(), compare_vd_entries);
+    
+    for (std::vector<const VarDecl*>::iterator di = declVec.begin(),
+         de = declVec.end(); di != de; ++di) {
+      llvm::errs() << " " << (*di)->getDeclName().getAsString()
+                   << " <";
+      (*di)->getLocation().dump(M);
       llvm::errs() << ">\n";
     }
-}
-
-void LiveVariables::dumpBlockLiveness(const SourceManager& M) const {
-  for (BlockDataMapTy::const_iterator I = getBlockDataMap().begin(),
-       E = getBlockDataMap().end(); I!=E; ++I) {
-    llvm::errs() << "\n[ B" << I->first->getBlockID()
-                 << " (live variables at block exit) ]\n";
-    dumpLiveness(I->second,M);
   }
-
-  llvm::errs() << "\n";
+  llvm::errs() << "\n";  
 }
+
+const void *LiveVariables::getTag() { static int x; return &x; }
+const void *RelaxedLiveVariables::getTag() { static int x; return &x; }
diff --git a/contrib/llvm/tools/clang/lib/Analysis/PrintfFormatString.cpp b/contrib/llvm/tools/clang/lib/Analysis/PrintfFormatString.cpp
index ce2690f45cae..2bb39cfa1b36 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/PrintfFormatString.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/PrintfFormatString.cpp
@@ -38,8 +38,7 @@ static bool ParsePrecision(FormatStringHandler &H, PrintfSpecifier &FS,
                            unsigned *argIndex) {
   if (argIndex) {
     FS.setPrecision(ParseNonPositionAmount(Beg, E, *argIndex));
-  }
-  else {
+  } else {
     const OptionalAmount Amt = ParsePositionAmount(H, Start, Beg, E,
                                            analyze_format_string::PrecisionPos);
     if (Amt.isInvalid())
@@ -404,6 +403,7 @@ bool PrintfSpecifier::fixType(QualType QT) {
   case BuiltinType::Char32:
   case BuiltinType::UInt128:
   case BuiltinType::Int128:
+  case BuiltinType::Half:
     // Integral types which are non-trivial to correct.
     return false;
 
@@ -477,15 +477,14 @@ bool PrintfSpecifier::fixType(QualType QT) {
       CS.setKind(ConversionSpecifier::uArg);
     HasAlternativeForm = 0;
     HasPlusPrefix = 0;
-  }
-  else {
-    assert(0 && "Unexpected type");
+  } else {
+    llvm_unreachable("Unexpected type");
   }
 
   return true;
 }
 
-void PrintfSpecifier::toString(llvm::raw_ostream &os) const {
+void PrintfSpecifier::toString(raw_ostream &os) const {
   // Whilst some features have no defined order, we are using the order
   // appearing in the C99 standard (ISO/IEC 9899:1999 (E) 7.19.6.1)
   os << "%";
diff --git a/contrib/llvm/tools/clang/lib/Analysis/ProgramPoint.cpp b/contrib/llvm/tools/clang/lib/Analysis/ProgramPoint.cpp
new file mode 100644
index 000000000000..3a0bbd5640d9
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Analysis/ProgramPoint.cpp
@@ -0,0 +1,51 @@
+//==- ProgramPoint.cpp - Program Points for Path-Sensitive Analysis -*- C++ -*-/
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the interface ProgramPoint, which identifies a
+//  distinct location in a function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/ProgramPoint.h"
+
+using namespace clang;
+
+ProgramPointTag::~ProgramPointTag() {}
+
+ProgramPoint ProgramPoint::getProgramPoint(const Stmt *S, ProgramPoint::Kind K,
+                                           const LocationContext *LC,
+                                           const ProgramPointTag *tag){
+  switch (K) {
+    default:
+      llvm_unreachable("Unhandled ProgramPoint kind");
+    case ProgramPoint::PreStmtKind:
+      return PreStmt(S, LC, tag);
+    case ProgramPoint::PostStmtKind:
+      return PostStmt(S, LC, tag);
+    case ProgramPoint::PreLoadKind:
+      return PreLoad(S, LC, tag);
+    case ProgramPoint::PostLoadKind:
+      return PostLoad(S, LC, tag);
+    case ProgramPoint::PreStoreKind:
+      return PreStore(S, LC, tag);
+    case ProgramPoint::PostStoreKind:
+      return PostStore(S, LC, tag);
+    case ProgramPoint::PostLValueKind:
+      return PostLValue(S, LC, tag);
+    case ProgramPoint::PostPurgeDeadSymbolsKind:
+      return PostPurgeDeadSymbols(S, LC, tag);
+  }
+}
+
+SimpleProgramPointTag::SimpleProgramPointTag(StringRef description)
+  : desc(description) {}
+
+StringRef SimpleProgramPointTag::getTagDescription() const {
+  return desc;
+}
diff --git a/contrib/llvm/tools/clang/lib/Analysis/PseudoConstantAnalysis.cpp b/contrib/llvm/tools/clang/lib/Analysis/PseudoConstantAnalysis.cpp
index ff96eb4a0a73..8f24c432b157 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/PseudoConstantAnalysis.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/PseudoConstantAnalysis.cpp
@@ -83,7 +83,7 @@ void PseudoConstantAnalysis::RunAnalysis() {
   WorkList.push_back(DeclBody);
 
   while (!WorkList.empty()) {
-    const Stmt* Head = WorkList.front();
+    const Stmt *Head = WorkList.front();
     WorkList.pop_front();
 
     if (const Expr *Ex = dyn_cast<Expr>(Head))
diff --git a/contrib/llvm/tools/clang/lib/Analysis/ReachableCode.cpp b/contrib/llvm/tools/clang/lib/Analysis/ReachableCode.cpp
index c5b17fc77bb2..49317718c381 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/ReachableCode.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/ReachableCode.cpp
@@ -25,47 +25,170 @@
 
 using namespace clang;
 
-static SourceLocation GetUnreachableLoc(const CFGBlock &b, SourceRange &R1,
+namespace {
+class DeadCodeScan {
+  llvm::BitVector Visited;
+  llvm::BitVector &Reachable;
+  llvm::SmallVector<const CFGBlock *, 10> WorkList;
+  
+  typedef llvm::SmallVector<std::pair<const CFGBlock *, const Stmt *>, 12>
+      DeferredLocsTy;
+  
+  DeferredLocsTy DeferredLocs;
+  
+public:
+  DeadCodeScan(llvm::BitVector &reachable)
+    : Visited(reachable.size()),
+      Reachable(reachable) {}
+  
+  void enqueue(const CFGBlock *block);  
+  unsigned scanBackwards(const CFGBlock *Start,
+                         clang::reachable_code::Callback &CB);
+  
+  bool isDeadCodeRoot(const CFGBlock *Block);
+  
+  const Stmt *findDeadCode(const CFGBlock *Block);
+  
+  void reportDeadCode(const Stmt *S,
+                      clang::reachable_code::Callback &CB);
+};
+}
+
+void DeadCodeScan::enqueue(const CFGBlock *block) {  
+  unsigned blockID = block->getBlockID();
+  if (Reachable[blockID] || Visited[blockID])
+    return;
+  Visited[blockID] = true;
+  WorkList.push_back(block);
+}
+
+bool DeadCodeScan::isDeadCodeRoot(const clang::CFGBlock *Block) {
+  bool isDeadRoot = true;
+  
+  for (CFGBlock::const_pred_iterator I = Block->pred_begin(),
+        E = Block->pred_end(); I != E; ++I) {
+    if (const CFGBlock *PredBlock = *I) {
+      unsigned blockID = PredBlock->getBlockID();
+      if (Visited[blockID]) {
+        isDeadRoot = false;
+        continue;
+      }
+      if (!Reachable[blockID]) {
+        isDeadRoot = false;
+        Visited[blockID] = true;
+        WorkList.push_back(PredBlock);
+        continue;
+      }
+    }
+  }
+  
+  return isDeadRoot;
+}
+
+static bool isValidDeadStmt(const Stmt *S) {
+  if (S->getLocStart().isInvalid())
+    return false;
+  if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(S))
+    return BO->getOpcode() != BO_Comma;
+  return true;
+}
+
+const Stmt *DeadCodeScan::findDeadCode(const clang::CFGBlock *Block) {
+  for (CFGBlock::const_iterator I = Block->begin(), E = Block->end(); I!=E; ++I)
+    if (const CFGStmt *CS = I->getAs<CFGStmt>()) {
+      const Stmt *S = CS->getStmt();
+      if (isValidDeadStmt(S))
+        return S;
+    }
+  
+  if (CFGTerminator T = Block->getTerminator()) {
+    const Stmt *S = T.getStmt();
+    if (isValidDeadStmt(S))
+      return S;    
+  }
+
+  return 0;
+}
+
+static int SrcCmp(const void *p1, const void *p2) {
+  return
+    ((std::pair<const CFGBlock *, const Stmt *>*) p2)->second->getLocStart() <
+    ((std::pair<const CFGBlock *, const Stmt *>*) p1)->second->getLocStart();
+}
+
+unsigned DeadCodeScan::scanBackwards(const clang::CFGBlock *Start,
+                                     clang::reachable_code::Callback &CB) {
+
+  unsigned count = 0;
+  enqueue(Start);
+  
+  while (!WorkList.empty()) {
+    const CFGBlock *Block = WorkList.pop_back_val();
+
+    // It is possible that this block has been marked reachable after
+    // it was enqueued.
+    if (Reachable[Block->getBlockID()])
+      continue;
+
+    // Look for any dead code within the block.
+    const Stmt *S = findDeadCode(Block);
+    
+    if (!S) {
+      // No dead code.  Possibly an empty block.  Look at dead predecessors.
+      for (CFGBlock::const_pred_iterator I = Block->pred_begin(),
+           E = Block->pred_end(); I != E; ++I) {
+        if (const CFGBlock *predBlock = *I)
+          enqueue(predBlock);
+      }
+      continue;
+    }
+    
+    // Specially handle macro-expanded code.
+    if (S->getLocStart().isMacroID()) {
+      count += clang::reachable_code::ScanReachableFromBlock(Block, Reachable);
+      continue;
+    }
+
+    if (isDeadCodeRoot(Block)) {
+      reportDeadCode(S, CB);
+      count += clang::reachable_code::ScanReachableFromBlock(Block, Reachable);
+    }
+    else {
+      // Record this statement as the possibly best location in a
+      // strongly-connected component of dead code for emitting a
+      // warning.
+      DeferredLocs.push_back(std::make_pair(Block, S));
+    }
+  }
+
+  // If we didn't find a dead root, then report the dead code with the
+  // earliest location.
+  if (!DeferredLocs.empty()) {
+    llvm::array_pod_sort(DeferredLocs.begin(), DeferredLocs.end(), SrcCmp);
+    for (DeferredLocsTy::iterator I = DeferredLocs.begin(),
+          E = DeferredLocs.end(); I != E; ++I) {
+      const CFGBlock *block = I->first;
+      if (Reachable[block->getBlockID()])
+        continue;
+      reportDeadCode(I->second, CB);
+      count += clang::reachable_code::ScanReachableFromBlock(block, Reachable);
+    }
+  }
+    
+  return count;
+}
+
+static SourceLocation GetUnreachableLoc(const Stmt *S,
+                                        SourceRange &R1,
                                         SourceRange &R2) {
-  const Stmt *S = 0;
-  unsigned sn = 0;
   R1 = R2 = SourceRange();
 
-  if (sn < b.size()) {
-    const CFGStmt *CS = b[sn].getAs<CFGStmt>();
-    if (!CS)
-      return SourceLocation();
-
-    S = CS->getStmt(); 
-  } else if (b.getTerminator())
-    S = b.getTerminator();
-  else
-    return SourceLocation();
-
   if (const Expr *Ex = dyn_cast<Expr>(S))
     S = Ex->IgnoreParenImpCasts();
 
   switch (S->getStmtClass()) {
     case Expr::BinaryOperatorClass: {
       const BinaryOperator *BO = cast<BinaryOperator>(S);
-      if (BO->getOpcode() == BO_Comma) {
-        if (sn+1 < b.size())
-          return b[sn+1].getAs<CFGStmt>()->getStmt()->getLocStart();
-        const CFGBlock *n = &b;
-        while (1) {
-          if (n->getTerminator())
-            return n->getTerminator()->getLocStart();
-          if (n->succ_size() != 1)
-            return SourceLocation();
-          n = n[0].succ_begin()[0];
-          if (n->pred_size() != 1)
-            return SourceLocation();
-          if (!n->empty())
-            return n[0][0].getAs<CFGStmt>()->getStmt()->getLocStart();
-        }
-      }
-      R1 = BO->getLHS()->getSourceRange();
-      R2 = BO->getRHS()->getSourceRange();
       return BO->getOperatorLoc();
     }
     case Expr::UnaryOperatorClass: {
@@ -120,177 +243,87 @@ static SourceLocation GetUnreachableLoc(const CFGBlock &b, SourceRange &R1,
   return S->getLocStart();
 }
 
-static SourceLocation MarkLiveTop(const CFGBlock *Start,
-                                  llvm::BitVector &reachable,
-                                  SourceManager &SM) {
-
-  // Prep work worklist.
-  llvm::SmallVector<const CFGBlock*, 32> WL;
-  WL.push_back(Start);
-
+void DeadCodeScan::reportDeadCode(const Stmt *S,
+                                  clang::reachable_code::Callback &CB) {
   SourceRange R1, R2;
-  SourceLocation top = GetUnreachableLoc(*Start, R1, R2);
-
-  bool FromMainFile = false;
-  bool FromSystemHeader = false;
-  bool TopValid = false;
-
-  if (top.isValid()) {
-    FromMainFile = SM.isFromMainFile(top);
-    FromSystemHeader = SM.isInSystemHeader(top);
-    TopValid = true;
-  }
-
-  // Solve
-  CFGBlock::FilterOptions FO;
-  FO.IgnoreDefaultsWithCoveredEnums = 1;
-
-  while (!WL.empty()) {
-    const CFGBlock *item = WL.back();
-    WL.pop_back();
-
-    SourceLocation c = GetUnreachableLoc(*item, R1, R2);
-    if (c.isValid()
-        && (!TopValid
-            || (SM.isFromMainFile(c) && !FromMainFile)
-            || (FromSystemHeader && !SM.isInSystemHeader(c))
-            || SM.isBeforeInTranslationUnit(c, top))) {
-          top = c;
-          FromMainFile = SM.isFromMainFile(top);
-          FromSystemHeader = SM.isInSystemHeader(top);
-        }
-
-    reachable.set(item->getBlockID());
-    for (CFGBlock::filtered_succ_iterator I =
-	   item->filtered_succ_start_end(FO); I.hasMore(); ++I)
-      if (const CFGBlock *B = *I) {
-        unsigned blockID = B->getBlockID();
-        if (!reachable[blockID]) {
-          reachable.set(blockID);
-          WL.push_back(B);
-        }
-      }
-  }
-
-  return top;
+  SourceLocation Loc = GetUnreachableLoc(S, R1, R2);
+  CB.HandleUnreachable(Loc, R1, R2);
 }
 
-static int LineCmp(const void *p1, const void *p2) {
-  SourceLocation *Line1 = (SourceLocation *)p1;
-  SourceLocation *Line2 = (SourceLocation *)p2;
-  return !(*Line1 < *Line2);
-}
-
-namespace {
-struct ErrLoc {
-  SourceLocation Loc;
-  SourceRange R1;
-  SourceRange R2;
-  ErrLoc(SourceLocation l, SourceRange r1, SourceRange r2)
-  : Loc(l), R1(r1), R2(r2) { }
-};
-}
 namespace clang { namespace reachable_code {
-
-/// ScanReachableFromBlock - Mark all blocks reachable from Start.
-/// Returns the total number of blocks that were marked reachable.
-unsigned ScanReachableFromBlock(const CFGBlock &Start,
+  
+unsigned ScanReachableFromBlock(const CFGBlock *Start,
                                 llvm::BitVector &Reachable) {
   unsigned count = 0;
-  llvm::SmallVector<const CFGBlock*, 32> WL;
-
+  
   // Prep work queue
-  Reachable.set(Start.getBlockID());
-  ++count;
-  WL.push_back(&Start);
-
+  SmallVector<const CFGBlock*, 32> WL;
+  
+  // The entry block may have already been marked reachable
+  // by the caller.
+  if (!Reachable[Start->getBlockID()]) {
+    ++count;
+    Reachable[Start->getBlockID()] = true;
+  }
+  
+  WL.push_back(Start);
+  
   // Find the reachable blocks from 'Start'.
-  CFGBlock::FilterOptions FO;
-  FO.IgnoreDefaultsWithCoveredEnums = 1;
-
   while (!WL.empty()) {
-    const CFGBlock *item = WL.back();
-    WL.pop_back();
-
-      // Look at the successors and mark then reachable.
-    for (CFGBlock::filtered_succ_iterator I= item->filtered_succ_start_end(FO);
-         I.hasMore(); ++I)
+    const CFGBlock *item = WL.pop_back_val();
+    
+    // Look at the successors and mark then reachable.
+    for (CFGBlock::const_succ_iterator I = item->succ_begin(), 
+         E = item->succ_end(); I != E; ++I)
       if (const CFGBlock *B = *I) {
         unsigned blockID = B->getBlockID();
         if (!Reachable[blockID]) {
           Reachable.set(blockID);
-          ++count;
           WL.push_back(B);
+          ++count;
         }
       }
   }
   return count;
 }
-
+  
 void FindUnreachableCode(AnalysisContext &AC, Callback &CB) {
   CFG *cfg = AC.getCFG();
   if (!cfg)
     return;
 
-  // Scan for reachable blocks.
+  // Scan for reachable blocks from the entrance of the CFG.  
+  // If there are no unreachable blocks, we're done.
   llvm::BitVector reachable(cfg->getNumBlockIDs());
-  unsigned numReachable = ScanReachableFromBlock(cfg->getEntry(), reachable);
-
-    // If there are no unreachable blocks, we're done.
+  unsigned numReachable = ScanReachableFromBlock(&cfg->getEntry(), reachable);
   if (numReachable == cfg->getNumBlockIDs())
     return;
+  
+  // If there aren't explicit EH edges, we should include the 'try' dispatch
+  // blocks as roots.
+  if (!AC.getCFGBuildOptions().AddEHEdges) {
+    for (CFG::try_block_iterator I = cfg->try_blocks_begin(),
+         E = cfg->try_blocks_end() ; I != E; ++I) {
+      numReachable += ScanReachableFromBlock(*I, reachable);
+    }
+    if (numReachable == cfg->getNumBlockIDs())
+      return;
+  }
 
-  SourceRange R1, R2;
-
-  llvm::SmallVector<ErrLoc, 24> lines;
-  bool AddEHEdges = AC.getAddEHEdges();
-
-  // First, give warnings for blocks with no predecessors, as they
-  // can't be part of a loop.
+  // There are some unreachable blocks.  We need to find the root blocks that
+  // contain code that should be considered unreachable.  
   for (CFG::iterator I = cfg->begin(), E = cfg->end(); I != E; ++I) {
-    CFGBlock &b = **I;
-    if (!reachable[b.getBlockID()]) {
-      if (b.pred_empty()) {
-        if (!AddEHEdges
-        && dyn_cast_or_null<CXXTryStmt>(b.getTerminator().getStmt())) {
-            // When not adding EH edges from calls, catch clauses
-            // can otherwise seem dead.  Avoid noting them as dead.
-          numReachable += ScanReachableFromBlock(b, reachable);
-          continue;
-        }
-        SourceLocation c = GetUnreachableLoc(b, R1, R2);
-        if (!c.isValid()) {
-            // Blocks without a location can't produce a warning, so don't mark
-            // reachable blocks from here as live.
-          reachable.set(b.getBlockID());
-          ++numReachable;
-          continue;
-        }
-        lines.push_back(ErrLoc(c, R1, R2));
-          // Avoid excessive errors by marking everything reachable from here
-        numReachable += ScanReachableFromBlock(b, reachable);
-      }
-    }
+    const CFGBlock *block = *I;
+    // A block may have been marked reachable during this loop.
+    if (reachable[block->getBlockID()])
+      continue;
+    
+    DeadCodeScan DS(reachable);
+    numReachable += DS.scanBackwards(block, CB);
+    
+    if (numReachable == cfg->getNumBlockIDs())
+      return;
   }
-
-  if (numReachable < cfg->getNumBlockIDs()) {
-      // And then give warnings for the tops of loops.
-    for (CFG::iterator I = cfg->begin(), E = cfg->end(); I != E; ++I) {
-      CFGBlock &b = **I;
-      if (!reachable[b.getBlockID()])
-          // Avoid excessive errors by marking everything reachable from here
-        lines.push_back(ErrLoc(MarkLiveTop(&b, reachable,
-                                         AC.getASTContext().getSourceManager()),
-                               SourceRange(), SourceRange()));
-    }
-  }
-
-  llvm::array_pod_sort(lines.begin(), lines.end(), LineCmp);
-
-  for (llvm::SmallVectorImpl<ErrLoc>::iterator I=lines.begin(), E=lines.end();
-       I != E; ++I)
-    if (I->Loc.isValid())
-      CB.HandleUnreachable(I->Loc, I->R1, I->R2);
 }
 
 }} // end namespace clang::reachable_code
diff --git a/contrib/llvm/tools/clang/lib/Analysis/ThreadSafety.cpp b/contrib/llvm/tools/clang/lib/Analysis/ThreadSafety.cpp
new file mode 100644
index 000000000000..5a12913c1c94
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Analysis/ThreadSafety.cpp
@@ -0,0 +1,799 @@
+//===- ThreadSafety.cpp ----------------------------------------*- C++ --*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// A intra-procedural analysis for thread safety (e.g. deadlocks and race
+// conditions), based off of an annotation system.
+//
+// See http://clang.llvm.org/docs/LanguageExtensions.html#threadsafety for more
+// information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/Analyses/ThreadSafety.h"
+#include "clang/Analysis/AnalysisContext.h"
+#include "clang/Analysis/CFG.h"
+#include "clang/Analysis/CFGStmtMap.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/StmtCXX.h"
+#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/ImmutableMap.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include <algorithm>
+#include <vector>
+
+using namespace clang;
+using namespace thread_safety;
+
+// Key method definition
+ThreadSafetyHandler::~ThreadSafetyHandler() {}
+
+// Helper function
+static Expr *getParent(Expr *Exp) {
+  if (MemberExpr *ME = dyn_cast<MemberExpr>(Exp))
+    return ME->getBase();
+  if (CXXMemberCallExpr *CE = dyn_cast<CXXMemberCallExpr>(Exp))
+    return CE->getImplicitObjectArgument();
+  return 0;
+}
+
+namespace {
+/// \brief Implements a set of CFGBlocks using a BitVector.
+///
+/// This class contains a minimal interface, primarily dictated by the SetType
+/// template parameter of the llvm::po_iterator template, as used with external
+/// storage. We also use this set to keep track of which CFGBlocks we visit
+/// during the analysis.
+class CFGBlockSet {
+  llvm::BitVector VisitedBlockIDs;
+
+public:
+  // po_iterator requires this iterator, but the only interface needed is the
+  // value_type typedef.
+  struct iterator {
+    typedef const CFGBlock *value_type;
+  };
+
+  CFGBlockSet() {}
+  CFGBlockSet(const CFG *G) : VisitedBlockIDs(G->getNumBlockIDs(), false) {}
+
+  /// \brief Set the bit associated with a particular CFGBlock.
+  /// This is the important method for the SetType template parameter.
+  bool insert(const CFGBlock *Block) {
+    // Note that insert() is called by po_iterator, which doesn't check to make
+    // sure that Block is non-null.  Moreover, the CFGBlock iterator will
+    // occasionally hand out null pointers for pruned edges, so we catch those
+    // here.
+    if (Block == 0)
+      return false;  // if an edge is trivially false.
+    if (VisitedBlockIDs.test(Block->getBlockID()))
+      return false;
+    VisitedBlockIDs.set(Block->getBlockID());
+    return true;
+  }
+
+  /// \brief Check if the bit for a CFGBlock has been already set.
+  /// This method is for tracking visited blocks in the main threadsafety loop.
+  /// Block must not be null.
+  bool alreadySet(const CFGBlock *Block) {
+    return VisitedBlockIDs.test(Block->getBlockID());
+  }
+};
+
+/// \brief We create a helper class which we use to iterate through CFGBlocks in
+/// the topological order.
+class TopologicallySortedCFG {
+  typedef llvm::po_iterator<const CFG*, CFGBlockSet, true>  po_iterator;
+
+  std::vector<const CFGBlock*> Blocks;
+
+public:
+  typedef std::vector<const CFGBlock*>::reverse_iterator iterator;
+
+  TopologicallySortedCFG(const CFG *CFGraph) {
+    Blocks.reserve(CFGraph->getNumBlockIDs());
+    CFGBlockSet BSet(CFGraph);
+
+    for (po_iterator I = po_iterator::begin(CFGraph, BSet),
+         E = po_iterator::end(CFGraph, BSet); I != E; ++I) {
+      Blocks.push_back(*I);
+    }
+  }
+
+  iterator begin() {
+    return Blocks.rbegin();
+  }
+
+  iterator end() {
+    return Blocks.rend();
+  }
+
+  bool empty() {
+    return begin() == end();
+  }
+};
+
+/// \brief A MutexID object uniquely identifies a particular mutex, and
+/// is built from an Expr* (i.e. calling a lock function).
+///
+/// Thread-safety analysis works by comparing lock expressions.  Within the
+/// body of a function, an expression such as "x->foo->bar.mu" will resolve to
+/// a particular mutex object at run-time.  Subsequent occurrences of the same
+/// expression (where "same" means syntactic equality) will refer to the same
+/// run-time object if three conditions hold:
+/// (1) Local variables in the expression, such as "x" have not changed.
+/// (2) Values on the heap that affect the expression have not changed.
+/// (3) The expression involves only pure function calls.
+/// The current implementation assumes, but does not verify, that multiple uses
+/// of the same lock expression satisfies these criteria.
+///
+/// Clang introduces an additional wrinkle, which is that it is difficult to
+/// derive canonical expressions, or compare expressions directly for equality.
+/// Thus, we identify a mutex not by an Expr, but by the set of named
+/// declarations that are referenced by the Expr.  In other words,
+/// x->foo->bar.mu will be a four element vector with the Decls for
+/// mu, bar, and foo, and x.  The vector will uniquely identify the expression
+/// for all practical purposes.
+///
+/// Note we will need to perform substitution on "this" and function parameter
+/// names when constructing a lock expression.
+///
+/// For example:
+/// class C { Mutex Mu;  void lock() EXCLUSIVE_LOCK_FUNCTION(this->Mu); };
+/// void myFunc(C *X) { ... X->lock() ... }
+/// The original expression for the mutex acquired by myFunc is "this->Mu", but
+/// "X" is substituted for "this" so we get X->Mu();
+///
+/// For another example:
+/// foo(MyList *L) EXCLUSIVE_LOCKS_REQUIRED(L->Mu) { ... }
+/// MyList *MyL;
+/// foo(MyL);  // requires lock MyL->Mu to be held
+class MutexID {
+  SmallVector<NamedDecl*, 2> DeclSeq;
+
+  /// Build a Decl sequence representing the lock from the given expression.
+  /// Recursive function that bottoms out when the final DeclRefExpr is reached.
+  // FIXME: Lock expressions that involve array indices or function calls.
+  // FIXME: Deal with LockReturned attribute.
+  void buildMutexID(Expr *Exp, Expr *Parent) {
+    if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Exp)) {
+      NamedDecl *ND = cast<NamedDecl>(DRE->getDecl()->getCanonicalDecl());
+      DeclSeq.push_back(ND);
+    } else if (MemberExpr *ME = dyn_cast<MemberExpr>(Exp)) {
+      NamedDecl *ND = ME->getMemberDecl();
+      DeclSeq.push_back(ND);
+      buildMutexID(ME->getBase(), Parent);
+    } else if (isa<CXXThisExpr>(Exp)) {
+      if (Parent)
+        buildMutexID(Parent, 0);
+      else
+        return; // mutexID is still valid in this case
+    } else if (CastExpr *CE = dyn_cast<CastExpr>(Exp))
+      buildMutexID(CE->getSubExpr(), Parent);
+    else
+      DeclSeq.clear(); // invalid lock expression
+  }
+
+public:
+  MutexID(Expr *LExpr, Expr *ParentExpr) {
+    buildMutexID(LExpr, ParentExpr);
+  }
+
+  /// If we encounter part of a lock expression we cannot parse
+  bool isValid() const {
+    return !DeclSeq.empty();
+  }
+
+  bool operator==(const MutexID &other) const {
+    return DeclSeq == other.DeclSeq;
+  }
+
+  bool operator!=(const MutexID &other) const {
+    return !(*this == other);
+  }
+
+  // SmallVector overloads Operator< to do lexicographic ordering. Note that
+  // we use pointer equality (and <) to compare NamedDecls. This means the order
+  // of MutexIDs in a lockset is nondeterministic. In order to output
+  // diagnostics in a deterministic ordering, we must order all diagnostics to
+  // output by SourceLocation when iterating through this lockset.
+  bool operator<(const MutexID &other) const {
+    return DeclSeq < other.DeclSeq;
+  }
+
+  /// \brief Returns the name of the first Decl in the list for a given MutexID;
+  /// e.g. the lock expression foo.bar() has name "bar".
+  /// The caret will point unambiguously to the lock expression, so using this
+  /// name in diagnostics is a way to get simple, and consistent, mutex names.
+  /// We do not want to output the entire expression text for security reasons.
+  StringRef getName() const {
+    assert(isValid());
+    return DeclSeq.front()->getName();
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    for (SmallVectorImpl<NamedDecl*>::const_iterator I = DeclSeq.begin(),
+         E = DeclSeq.end(); I != E; ++I) {
+      ID.AddPointer(*I);
+    }
+  }
+};
+
+/// \brief This is a helper class that stores info about the most recent
+/// accquire of a Lock.
+///
+/// The main body of the analysis maps MutexIDs to LockDatas.
+struct LockData {
+  SourceLocation AcquireLoc;
+
+  /// \brief LKind stores whether a lock is held shared or exclusively.
+  /// Note that this analysis does not currently support either re-entrant
+  /// locking or lock "upgrading" and "downgrading" between exclusive and
+  /// shared.
+  ///
+  /// FIXME: add support for re-entrant locking and lock up/downgrading
+  LockKind LKind;
+
+  LockData(SourceLocation AcquireLoc, LockKind LKind)
+    : AcquireLoc(AcquireLoc), LKind(LKind) {}
+
+  bool operator==(const LockData &other) const {
+    return AcquireLoc == other.AcquireLoc && LKind == other.LKind;
+  }
+
+  bool operator!=(const LockData &other) const {
+    return !(*this == other);
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+      ID.AddInteger(AcquireLoc.getRawEncoding());
+      ID.AddInteger(LKind);
+    }
+};
+
+/// A Lockset maps each MutexID (defined above) to information about how it has
+/// been locked.
+typedef llvm::ImmutableMap<MutexID, LockData> Lockset;
+
+/// \brief We use this class to visit different types of expressions in
+/// CFGBlocks, and build up the lockset.
+/// An expression may cause us to add or remove locks from the lockset, or else
+/// output error messages related to missing locks.
+/// FIXME: In future, we may be able to not inherit from a visitor.
+class BuildLockset : public StmtVisitor<BuildLockset> {
+  ThreadSafetyHandler &Handler;
+  Lockset LSet;
+  Lockset::Factory &LocksetFactory;
+
+  // Helper functions
+  void removeLock(SourceLocation UnlockLoc, Expr *LockExp, Expr *Parent);
+  void addLock(SourceLocation LockLoc, Expr *LockExp, Expr *Parent,
+               LockKind LK);
+  const ValueDecl *getValueDecl(Expr *Exp);
+  void warnIfMutexNotHeld (const NamedDecl *D, Expr *Exp, AccessKind AK,
+                           Expr *MutexExp, ProtectedOperationKind POK);
+  void checkAccess(Expr *Exp, AccessKind AK);
+  void checkDereference(Expr *Exp, AccessKind AK);
+
+  template <class AttrType>
+  void addLocksToSet(LockKind LK, Attr *Attr, CXXMemberCallExpr *Exp);
+
+  /// \brief Returns true if the lockset contains a lock, regardless of whether
+  /// the lock is held exclusively or shared.
+  bool locksetContains(MutexID Lock) const {
+    return LSet.lookup(Lock);
+  }
+
+  /// \brief Returns true if the lockset contains a lock with the passed in
+  /// locktype.
+  bool locksetContains(MutexID Lock, LockKind KindRequested) const {
+    const LockData *LockHeld = LSet.lookup(Lock);
+    return (LockHeld && KindRequested == LockHeld->LKind);
+  }
+
+  /// \brief Returns true if the lockset contains a lock with at least the
+  /// passed in locktype. So for example, if we pass in LK_Shared, this function
+  /// returns true if the lock is held LK_Shared or LK_Exclusive. If we pass in
+  /// LK_Exclusive, this function returns true if the lock is held LK_Exclusive.
+  bool locksetContainsAtLeast(MutexID Lock, LockKind KindRequested) const {
+    switch (KindRequested) {
+      case LK_Shared:
+        return locksetContains(Lock);
+      case LK_Exclusive:
+        return locksetContains(Lock, KindRequested);
+    }
+    llvm_unreachable("Unknown LockKind");
+  }
+
+public:
+  BuildLockset(ThreadSafetyHandler &Handler, Lockset LS, Lockset::Factory &F)
+    : StmtVisitor<BuildLockset>(), Handler(Handler), LSet(LS),
+      LocksetFactory(F) {}
+
+  Lockset getLockset() {
+    return LSet;
+  }
+
+  void VisitUnaryOperator(UnaryOperator *UO);
+  void VisitBinaryOperator(BinaryOperator *BO);
+  void VisitCastExpr(CastExpr *CE);
+  void VisitCXXMemberCallExpr(CXXMemberCallExpr *Exp);
+};
+
+/// \brief Add a new lock to the lockset, warning if the lock is already there.
+/// \param LockLoc The source location of the acquire
+/// \param LockExp The lock expression corresponding to the lock to be added
+void BuildLockset::addLock(SourceLocation LockLoc, Expr *LockExp, Expr *Parent,
+                           LockKind LK) {
+  // FIXME: deal with acquired before/after annotations. We can write a first
+  // pass that does the transitive lookup lazily, and refine afterwards.
+  MutexID Mutex(LockExp, Parent);
+  if (!Mutex.isValid()) {
+    Handler.handleInvalidLockExp(LockExp->getExprLoc());
+    return;
+  }
+
+  LockData NewLock(LockLoc, LK);
+
+  // FIXME: Don't always warn when we have support for reentrant locks.
+  if (locksetContains(Mutex))
+    Handler.handleDoubleLock(Mutex.getName(), LockLoc);
+  LSet = LocksetFactory.add(LSet, Mutex, NewLock);
+}
+
+/// \brief Remove a lock from the lockset, warning if the lock is not there.
+/// \param LockExp The lock expression corresponding to the lock to be removed
+/// \param UnlockLoc The source location of the unlock (only used in error msg)
+void BuildLockset::removeLock(SourceLocation UnlockLoc, Expr *LockExp,
+                              Expr *Parent) {
+  MutexID Mutex(LockExp, Parent);
+  if (!Mutex.isValid()) {
+    Handler.handleInvalidLockExp(LockExp->getExprLoc());
+    return;
+  }
+
+  Lockset NewLSet = LocksetFactory.remove(LSet, Mutex);
+  if(NewLSet == LSet)
+    Handler.handleUnmatchedUnlock(Mutex.getName(), UnlockLoc);
+
+  LSet = NewLSet;
+}
+
+/// \brief Gets the value decl pointer from DeclRefExprs or MemberExprs
+const ValueDecl *BuildLockset::getValueDecl(Expr *Exp) {
+  if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Exp))
+    return DR->getDecl();
+
+  if (const MemberExpr *ME = dyn_cast<MemberExpr>(Exp))
+    return ME->getMemberDecl();
+
+  return 0;
+}
+
+/// \brief Warn if the LSet does not contain a lock sufficient to protect access
+/// of at least the passed in AccessType.
+void BuildLockset::warnIfMutexNotHeld(const NamedDecl *D, Expr *Exp,
+                                      AccessKind AK, Expr *MutexExp,
+                                      ProtectedOperationKind POK) {
+  LockKind LK = getLockKindFromAccessKind(AK);
+  Expr *Parent = getParent(Exp);
+  MutexID Mutex(MutexExp, Parent);
+  if (!Mutex.isValid())
+    Handler.handleInvalidLockExp(MutexExp->getExprLoc());
+  else if (!locksetContainsAtLeast(Mutex, LK))
+    Handler.handleMutexNotHeld(D, POK, Mutex.getName(), LK, Exp->getExprLoc());
+}
+
+
+/// \brief This method identifies variable dereferences and checks pt_guarded_by
+/// and pt_guarded_var annotations. Note that we only check these annotations
+/// at the time a pointer is dereferenced.
+/// FIXME: We need to check for other types of pointer dereferences
+/// (e.g. [], ->) and deal with them here.
+/// \param Exp An expression that has been read or written.
+void BuildLockset::checkDereference(Expr *Exp, AccessKind AK) {
+  UnaryOperator *UO = dyn_cast<UnaryOperator>(Exp);
+  if (!UO || UO->getOpcode() != clang::UO_Deref)
+    return;
+  Exp = UO->getSubExpr()->IgnoreParenCasts();
+
+  const ValueDecl *D = getValueDecl(Exp);
+  if(!D || !D->hasAttrs())
+    return;
+
+  if (D->getAttr<PtGuardedVarAttr>() && LSet.isEmpty())
+    Handler.handleNoMutexHeld(D, POK_VarDereference, AK, Exp->getExprLoc());
+
+  const AttrVec &ArgAttrs = D->getAttrs();
+  for(unsigned i = 0, Size = ArgAttrs.size(); i < Size; ++i)
+    if (PtGuardedByAttr *PGBAttr = dyn_cast<PtGuardedByAttr>(ArgAttrs[i]))
+      warnIfMutexNotHeld(D, Exp, AK, PGBAttr->getArg(), POK_VarDereference);
+}
+
+/// \brief Checks guarded_by and guarded_var attributes.
+/// Whenever we identify an access (read or write) of a DeclRefExpr or
+/// MemberExpr, we need to check whether there are any guarded_by or
+/// guarded_var attributes, and make sure we hold the appropriate mutexes.
+void BuildLockset::checkAccess(Expr *Exp, AccessKind AK) {
+  const ValueDecl *D = getValueDecl(Exp);
+  if(!D || !D->hasAttrs())
+    return;
+
+  if (D->getAttr<GuardedVarAttr>() && LSet.isEmpty())
+    Handler.handleNoMutexHeld(D, POK_VarAccess, AK, Exp->getExprLoc());
+
+  const AttrVec &ArgAttrs = D->getAttrs();
+  for(unsigned i = 0, Size = ArgAttrs.size(); i < Size; ++i)
+    if (GuardedByAttr *GBAttr = dyn_cast<GuardedByAttr>(ArgAttrs[i]))
+      warnIfMutexNotHeld(D, Exp, AK, GBAttr->getArg(), POK_VarAccess);
+}
+
+/// \brief For unary operations which read and write a variable, we need to
+/// check whether we hold any required mutexes. Reads are checked in
+/// VisitCastExpr.
+void BuildLockset::VisitUnaryOperator(UnaryOperator *UO) {
+  switch (UO->getOpcode()) {
+    case clang::UO_PostDec:
+    case clang::UO_PostInc:
+    case clang::UO_PreDec:
+    case clang::UO_PreInc: {
+      Expr *SubExp = UO->getSubExpr()->IgnoreParenCasts();
+      checkAccess(SubExp, AK_Written);
+      checkDereference(SubExp, AK_Written);
+      break;
+    }
+    default:
+      break;
+  }
+}
+
+/// For binary operations which assign to a variable (writes), we need to check
+/// whether we hold any required mutexes.
+/// FIXME: Deal with non-primitive types.
+void BuildLockset::VisitBinaryOperator(BinaryOperator *BO) {
+  if (!BO->isAssignmentOp())
+    return;
+  Expr *LHSExp = BO->getLHS()->IgnoreParenCasts();
+  checkAccess(LHSExp, AK_Written);
+  checkDereference(LHSExp, AK_Written);
+}
+
+/// Whenever we do an LValue to Rvalue cast, we are reading a variable and
+/// need to ensure we hold any required mutexes.
+/// FIXME: Deal with non-primitive types.
+void BuildLockset::VisitCastExpr(CastExpr *CE) {
+  if (CE->getCastKind() != CK_LValueToRValue)
+    return;
+  Expr *SubExp = CE->getSubExpr()->IgnoreParenCasts();
+  checkAccess(SubExp, AK_Read);
+  checkDereference(SubExp, AK_Read);
+}
+
+/// \brief This function, parameterized by an attribute type, is used to add a
+/// set of locks specified as attribute arguments to the lockset.
+template <typename AttrType>
+void BuildLockset::addLocksToSet(LockKind LK, Attr *Attr,
+                                 CXXMemberCallExpr *Exp) {
+  typedef typename AttrType::args_iterator iterator_type;
+  SourceLocation ExpLocation = Exp->getExprLoc();
+  Expr *Parent = Exp->getImplicitObjectArgument();
+  AttrType *SpecificAttr = cast<AttrType>(Attr);
+
+  if (SpecificAttr->args_size() == 0) {
+    // The mutex held is the "this" object.
+    addLock(ExpLocation, Parent, 0, LK);
+    return;
+  }
+
+  for (iterator_type I = SpecificAttr->args_begin(),
+       E = SpecificAttr->args_end(); I != E; ++I)
+    addLock(ExpLocation, *I, Parent, LK);
+}
+
+/// \brief When visiting CXXMemberCallExprs we need to examine the attributes on
+/// the method that is being called and add, remove or check locks in the
+/// lockset accordingly.
+///
+/// FIXME: For classes annotated with one of the guarded annotations, we need
+/// to treat const method calls as reads and non-const method calls as writes,
+/// and check that the appropriate locks are held. Non-const method calls with
+/// the same signature as const method calls can be also treated as reads.
+///
+/// FIXME: We need to also visit CallExprs to catch/check global functions.
+///
+/// FIXME: Do not flag an error for member variables accessed in constructors/
+/// destructors
+void BuildLockset::VisitCXXMemberCallExpr(CXXMemberCallExpr *Exp) {
+  NamedDecl *D = dyn_cast_or_null<NamedDecl>(Exp->getCalleeDecl());
+
+  SourceLocation ExpLocation = Exp->getExprLoc();
+  Expr *Parent = Exp->getImplicitObjectArgument();
+
+  if(!D || !D->hasAttrs())
+    return;
+
+  AttrVec &ArgAttrs = D->getAttrs();
+  for(unsigned i = 0; i < ArgAttrs.size(); ++i) {
+    Attr *Attr = ArgAttrs[i];
+    switch (Attr->getKind()) {
+      // When we encounter an exclusive lock function, we need to add the lock
+      // to our lockset with kind exclusive.
+      case attr::ExclusiveLockFunction:
+        addLocksToSet<ExclusiveLockFunctionAttr>(LK_Exclusive, Attr, Exp);
+        break;
+
+      // When we encounter a shared lock function, we need to add the lock
+      // to our lockset with kind shared.
+      case attr::SharedLockFunction:
+        addLocksToSet<SharedLockFunctionAttr>(LK_Shared, Attr, Exp);
+        break;
+
+      // When we encounter an unlock function, we need to remove unlocked
+      // mutexes from the lockset, and flag a warning if they are not there.
+      case attr::UnlockFunction: {
+        UnlockFunctionAttr *UFAttr = cast<UnlockFunctionAttr>(Attr);
+
+        if (UFAttr->args_size() == 0) { // The lock held is the "this" object.
+          removeLock(ExpLocation, Parent, 0);
+          break;
+        }
+
+        for (UnlockFunctionAttr::args_iterator I = UFAttr->args_begin(),
+             E = UFAttr->args_end(); I != E; ++I)
+          removeLock(ExpLocation, *I, Parent);
+        break;
+      }
+
+      case attr::ExclusiveLocksRequired: {
+        ExclusiveLocksRequiredAttr *ELRAttr =
+            cast<ExclusiveLocksRequiredAttr>(Attr);
+
+        for (ExclusiveLocksRequiredAttr::args_iterator
+             I = ELRAttr->args_begin(), E = ELRAttr->args_end(); I != E; ++I)
+          warnIfMutexNotHeld(D, Exp, AK_Written, *I, POK_FunctionCall);
+        break;
+      }
+
+      case attr::SharedLocksRequired: {
+        SharedLocksRequiredAttr *SLRAttr = cast<SharedLocksRequiredAttr>(Attr);
+
+        for (SharedLocksRequiredAttr::args_iterator I = SLRAttr->args_begin(),
+             E = SLRAttr->args_end(); I != E; ++I)
+          warnIfMutexNotHeld(D, Exp, AK_Read, *I, POK_FunctionCall);
+        break;
+      }
+
+      case attr::LocksExcluded: {
+        LocksExcludedAttr *LEAttr = cast<LocksExcludedAttr>(Attr);
+        for (LocksExcludedAttr::args_iterator I = LEAttr->args_begin(),
+            E = LEAttr->args_end(); I != E; ++I) {
+          MutexID Mutex(*I, Parent);
+          if (!Mutex.isValid())
+            Handler.handleInvalidLockExp((*I)->getExprLoc());
+          else if (locksetContains(Mutex))
+            Handler.handleFunExcludesLock(D->getName(), Mutex.getName(),
+                                          ExpLocation);
+        }
+        break;
+      }
+
+      // Ignore other (non thread-safety) attributes
+      default:
+        break;
+    }
+  }
+}
+
+} // end anonymous namespace
+
+/// \brief Compute the intersection of two locksets and issue warnings for any
+/// locks in the symmetric difference.
+///
+/// This function is used at a merge point in the CFG when comparing the lockset
+/// of each branch being merged. For example, given the following sequence:
+/// A; if () then B; else C; D; we need to check that the lockset after B and C
+/// are the same. In the event of a difference, we use the intersection of these
+/// two locksets at the start of D.
+static Lockset intersectAndWarn(ThreadSafetyHandler &Handler,
+                                const Lockset LSet1, const Lockset LSet2,
+                                Lockset::Factory &Fact, LockErrorKind LEK) {
+  Lockset Intersection = LSet1;
+  for (Lockset::iterator I = LSet2.begin(), E = LSet2.end(); I != E; ++I) {
+    const MutexID &LSet2Mutex = I.getKey();
+    const LockData &LSet2LockData = I.getData();
+    if (const LockData *LD = LSet1.lookup(LSet2Mutex)) {
+      if (LD->LKind != LSet2LockData.LKind) {
+        Handler.handleExclusiveAndShared(LSet2Mutex.getName(),
+                                         LSet2LockData.AcquireLoc,
+                                         LD->AcquireLoc);
+        if (LD->LKind != LK_Exclusive)
+          Intersection = Fact.add(Intersection, LSet2Mutex, LSet2LockData);
+      }
+    } else {
+      Handler.handleMutexHeldEndOfScope(LSet2Mutex.getName(),
+                                        LSet2LockData.AcquireLoc, LEK);
+    }
+  }
+
+  for (Lockset::iterator I = LSet1.begin(), E = LSet1.end(); I != E; ++I) {
+    if (!LSet2.contains(I.getKey())) {
+      const MutexID &Mutex = I.getKey();
+      const LockData &MissingLock = I.getData();
+      Handler.handleMutexHeldEndOfScope(Mutex.getName(),
+                                        MissingLock.AcquireLoc, LEK);
+      Intersection = Fact.remove(Intersection, Mutex);
+    }
+  }
+  return Intersection;
+}
+
+static Lockset addLock(ThreadSafetyHandler &Handler,
+                       Lockset::Factory &LocksetFactory,
+                       Lockset &LSet, Expr *LockExp, LockKind LK,
+                       SourceLocation Loc) {
+  MutexID Mutex(LockExp, 0);
+  if (!Mutex.isValid()) {
+    Handler.handleInvalidLockExp(LockExp->getExprLoc());
+    return LSet;
+  }
+  LockData NewLock(Loc, LK);
+  return LocksetFactory.add(LSet, Mutex, NewLock);
+}
+
+namespace clang {
+namespace thread_safety {
+/// \brief Check a function's CFG for thread-safety violations.
+///
+/// We traverse the blocks in the CFG, compute the set of mutexes that are held
+/// at the end of each block, and issue warnings for thread safety violations.
+/// Each block in the CFG is traversed exactly once.
+void runThreadSafetyAnalysis(AnalysisContext &AC,
+                             ThreadSafetyHandler &Handler) {
+  CFG *CFGraph = AC.getCFG();
+  if (!CFGraph) return;
+  const Decl *D = AC.getDecl();
+  if (D && D->getAttr<NoThreadSafetyAnalysisAttr>()) return;
+
+  Lockset::Factory LocksetFactory;
+
+  // FIXME: Swith to SmallVector? Otherwise improve performance impact?
+  std::vector<Lockset> EntryLocksets(CFGraph->getNumBlockIDs(),
+                                     LocksetFactory.getEmptyMap());
+  std::vector<Lockset> ExitLocksets(CFGraph->getNumBlockIDs(),
+                                    LocksetFactory.getEmptyMap());
+
+  // We need to explore the CFG via a "topological" ordering.
+  // That way, we will be guaranteed to have information about required
+  // predecessor locksets when exploring a new block.
+  TopologicallySortedCFG SortedGraph(CFGraph);
+  CFGBlockSet VisitedBlocks(CFGraph);
+
+  if (!SortedGraph.empty() && D->hasAttrs()) {
+    const CFGBlock *FirstBlock = *SortedGraph.begin();
+    Lockset &InitialLockset = EntryLocksets[FirstBlock->getBlockID()];
+    const AttrVec &ArgAttrs = D->getAttrs();
+    for(unsigned i = 0; i < ArgAttrs.size(); ++i) {
+      Attr *Attr = ArgAttrs[i];
+      SourceLocation AttrLoc = Attr->getLocation();
+      if (SharedLocksRequiredAttr *SLRAttr
+            = dyn_cast<SharedLocksRequiredAttr>(Attr)) {
+        for (SharedLocksRequiredAttr::args_iterator
+            SLRIter = SLRAttr->args_begin(),
+            SLREnd = SLRAttr->args_end(); SLRIter != SLREnd; ++SLRIter)
+          InitialLockset = addLock(Handler, LocksetFactory, InitialLockset,
+                                   *SLRIter, LK_Shared,
+                                   AttrLoc);
+      } else if (ExclusiveLocksRequiredAttr *ELRAttr
+                   = dyn_cast<ExclusiveLocksRequiredAttr>(Attr)) {
+        for (ExclusiveLocksRequiredAttr::args_iterator
+            ELRIter = ELRAttr->args_begin(),
+            ELREnd = ELRAttr->args_end(); ELRIter != ELREnd; ++ELRIter)
+          InitialLockset = addLock(Handler, LocksetFactory, InitialLockset,
+                                   *ELRIter, LK_Exclusive,
+                                   AttrLoc);
+      }
+    }
+  }
+
+  for (TopologicallySortedCFG::iterator I = SortedGraph.begin(),
+       E = SortedGraph.end(); I!= E; ++I) {
+    const CFGBlock *CurrBlock = *I;
+    int CurrBlockID = CurrBlock->getBlockID();
+
+    VisitedBlocks.insert(CurrBlock);
+
+    // Use the default initial lockset in case there are no predecessors.
+    Lockset &Entryset = EntryLocksets[CurrBlockID];
+    Lockset &Exitset = ExitLocksets[CurrBlockID];
+
+    // Iterate through the predecessor blocks and warn if the lockset for all
+    // predecessors is not the same. We take the entry lockset of the current
+    // block to be the intersection of all previous locksets.
+    // FIXME: By keeping the intersection, we may output more errors in future
+    // for a lock which is not in the intersection, but was in the union. We
+    // may want to also keep the union in future. As an example, let's say
+    // the intersection contains Mutex L, and the union contains L and M.
+    // Later we unlock M. At this point, we would output an error because we
+    // never locked M; although the real error is probably that we forgot to
+    // lock M on all code paths. Conversely, let's say that later we lock M.
+    // In this case, we should compare against the intersection instead of the
+    // union because the real error is probably that we forgot to unlock M on
+    // all code paths.
+    bool LocksetInitialized = false;
+    for (CFGBlock::const_pred_iterator PI = CurrBlock->pred_begin(),
+         PE  = CurrBlock->pred_end(); PI != PE; ++PI) {
+
+      // if *PI -> CurrBlock is a back edge
+      if (*PI == 0 || !VisitedBlocks.alreadySet(*PI))
+        continue;
+
+      int PrevBlockID = (*PI)->getBlockID();
+      if (!LocksetInitialized) {
+        Entryset = ExitLocksets[PrevBlockID];
+        LocksetInitialized = true;
+      } else {
+        Entryset = intersectAndWarn(Handler, Entryset,
+                                    ExitLocksets[PrevBlockID], LocksetFactory,
+                                    LEK_LockedSomePredecessors);
+      }
+    }
+
+    BuildLockset LocksetBuilder(Handler, Entryset, LocksetFactory);
+    for (CFGBlock::const_iterator BI = CurrBlock->begin(),
+         BE = CurrBlock->end(); BI != BE; ++BI) {
+      if (const CFGStmt *CfgStmt = dyn_cast<CFGStmt>(&*BI))
+        LocksetBuilder.Visit(const_cast<Stmt*>(CfgStmt->getStmt()));
+    }
+    Exitset = LocksetBuilder.getLockset();
+
+    // For every back edge from CurrBlock (the end of the loop) to another block
+    // (FirstLoopBlock) we need to check that the Lockset of Block is equal to
+    // the one held at the beginning of FirstLoopBlock. We can look up the
+    // Lockset held at the beginning of FirstLoopBlock in the EntryLockSets map.
+    for (CFGBlock::const_succ_iterator SI = CurrBlock->succ_begin(),
+         SE  = CurrBlock->succ_end(); SI != SE; ++SI) {
+
+      // if CurrBlock -> *SI is *not* a back edge
+      if (*SI == 0 || !VisitedBlocks.alreadySet(*SI))
+        continue;
+
+      CFGBlock *FirstLoopBlock = *SI;
+      Lockset PreLoop = EntryLocksets[FirstLoopBlock->getBlockID()];
+      Lockset LoopEnd = ExitLocksets[CurrBlockID];
+      intersectAndWarn(Handler, LoopEnd, PreLoop, LocksetFactory,
+                       LEK_LockedSomeLoopIterations);
+    }
+  }
+
+  Lockset InitialLockset = EntryLocksets[CFGraph->getEntry().getBlockID()];
+  Lockset FinalLockset = ExitLocksets[CFGraph->getExit().getBlockID()];
+
+  // FIXME: Should we call this function for all blocks which exit the function?
+  intersectAndWarn(Handler, InitialLockset, FinalLockset, LocksetFactory,
+                   LEK_LockedAtEndOfFunction);
+}
+
+/// \brief Helper function that returns a LockKind required for the given level
+/// of access.
+LockKind getLockKindFromAccessKind(AccessKind AK) {
+  switch (AK) {
+    case AK_Read :
+      return LK_Shared;
+    case AK_Written :
+      return LK_Exclusive;
+  }
+  llvm_unreachable("Unknown AccessKind");
+}
+}} // end namespace clang::thread_safety
diff --git a/contrib/llvm/tools/clang/lib/Analysis/UninitializedValues.cpp b/contrib/llvm/tools/clang/lib/Analysis/UninitializedValues.cpp
index 1d6959d81b16..9e98560b655d 100644
--- a/contrib/llvm/tools/clang/lib/Analysis/UninitializedValues.cpp
+++ b/contrib/llvm/tools/clang/lib/Analysis/UninitializedValues.cpp
@@ -123,13 +123,7 @@ class CFGBlockValues {
   bool hasNoDeclarations() const {
     return declToIndex.size() == 0;
   }
-  
-  bool hasEntry(const VarDecl *vd) const {
-    return declToIndex.getValueIndex(vd).hasValue();
-  }
-  
-  bool hasValues(const CFGBlock *block);
-  
+
   void resetScratch();
   ValueVector &getScratch() { return scratch; }
   
@@ -170,7 +164,7 @@ ValueVector &CFGBlockValues::lazyCreate(ValueVector *&bv) {
 /// This function pattern matches for a '&&' or '||' that appears at
 /// the beginning of a CFGBlock that also (1) has a terminator and 
 /// (2) has no other elements.  If such an expression is found, it is returned.
-static BinaryOperator *getLogicalOperatorInChain(const CFGBlock *block) {
+static const BinaryOperator *getLogicalOperatorInChain(const CFGBlock *block) {
   if (block->empty())
     return 0;
 
@@ -178,7 +172,7 @@ static BinaryOperator *getLogicalOperatorInChain(const CFGBlock *block) {
   if (!cstmt)
     return 0;
 
-  BinaryOperator *b = llvm::dyn_cast_or_null<BinaryOperator>(cstmt->getStmt());
+  const BinaryOperator *b = dyn_cast_or_null<BinaryOperator>(cstmt->getStmt());
   
   if (!b || !b->isLogicalOp())
     return 0;
@@ -209,11 +203,6 @@ ValueVector &CFGBlockValues::getValueVector(const CFGBlock *block,
   return lazyCreate(vals[idx].first);
 }
 
-bool CFGBlockValues::hasValues(const CFGBlock *block) {
-  unsigned idx = block->getBlockID();
-  return vals[idx].second != 0;  
-}
-
 BVPair &CFGBlockValues::getValueVectors(const clang::CFGBlock *block,
                                         bool shouldLazyCreate) {
   unsigned idx = block->getBlockID();
@@ -223,13 +212,6 @@ BVPair &CFGBlockValues::getValueVectors(const clang::CFGBlock *block,
   return vals[idx];
 }
 
-void CFGBlockValues::mergeIntoScratch(ValueVector const &source,
-                                      bool isFirst) {
-  if (isFirst)
-    scratch = source;
-  else
-    scratch |= source;
-}
 #if 0
 static void printVector(const CFGBlock *block, ValueVector &bv,
                         unsigned num) {
@@ -240,8 +222,24 @@ static void printVector(const CFGBlock *block, ValueVector &bv,
   }
   llvm::errs() << " : " << num << '\n';
 }
+
+static void printVector(const char *name, ValueVector const &bv) {
+  llvm::errs() << name << " : ";
+  for (unsigned i = 0; i < bv.size(); ++i) {
+    llvm::errs() << ' ' << bv[i];
+  }
+  llvm::errs() << "\n";
+}
 #endif
 
+void CFGBlockValues::mergeIntoScratch(ValueVector const &source,
+                                      bool isFirst) {
+  if (isFirst)
+    scratch = source;
+  else
+    scratch |= source;
+}
+
 bool CFGBlockValues::updateValueVectorWithScratch(const CFGBlock *block) {
   ValueVector &dst = getValueVector(block, 0);
   bool changed = (dst != scratch);
@@ -283,7 +281,7 @@ ValueVector::reference CFGBlockValues::operator[](const VarDecl *vd) {
 
 namespace {
 class DataflowWorklist {
-  llvm::SmallVector<const CFGBlock *, 20> worklist;
+  SmallVector<const CFGBlock *, 20> worklist;
   llvm::BitVector enqueuedBlocks;
 public:
   DataflowWorklist(const CFG &cfg) : enqueuedBlocks(cfg.getNumBlockIDs()) {}
@@ -336,23 +334,34 @@ class FindVarResult {
   const VarDecl *getDecl() const { return vd; }
 };
   
-class TransferFunctions : public CFGRecStmtVisitor<TransferFunctions> {
+class TransferFunctions : public StmtVisitor<TransferFunctions> {
   CFGBlockValues &vals;
   const CFG &cfg;
   AnalysisContext &ac;
   UninitVariablesHandler *handler;
-  const DeclRefExpr *currentDR;
-  const Expr *currentVoidCast;
-  const bool flagBlockUses;
+  
+  /// The last DeclRefExpr seen when analyzing a block.  Used to
+  /// cheat when detecting cases when the address of a variable is taken.
+  DeclRefExpr *lastDR;
+  
+  /// The last lvalue-to-rvalue conversion of a variable whose value
+  /// was uninitialized.  Normally this results in a warning, but it is
+  /// possible to either silence the warning in some cases, or we
+  /// propagate the uninitialized value.
+  CastExpr *lastLoad;
+  
+  /// For some expressions, we want to ignore any post-processing after
+  /// visitation.
+  bool skipProcessUses;
+  
 public:
   TransferFunctions(CFGBlockValues &vals, const CFG &cfg,
                     AnalysisContext &ac,
-                    UninitVariablesHandler *handler,
-                    bool flagBlockUses)
-    : vals(vals), cfg(cfg), ac(ac), handler(handler), currentDR(0),
-      currentVoidCast(0), flagBlockUses(flagBlockUses) {}
+                    UninitVariablesHandler *handler)
+    : vals(vals), cfg(cfg), ac(ac), handler(handler),
+      lastDR(0), lastLoad(0),
+      skipProcessUses(false) {}
   
-  const CFG &getCFG() { return cfg; }
   void reportUninit(const DeclRefExpr *ex, const VarDecl *vd,
                     bool isAlwaysUninit);
 
@@ -362,53 +371,59 @@ class TransferFunctions : public CFGRecStmtVisitor<TransferFunctions> {
   void VisitUnaryOperator(UnaryOperator *uo);
   void VisitBinaryOperator(BinaryOperator *bo);
   void VisitCastExpr(CastExpr *ce);
-  void VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *se);
-  void VisitCXXTypeidExpr(CXXTypeidExpr *E);
-  void BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt *fs);
+  void VisitObjCForCollectionStmt(ObjCForCollectionStmt *fs);
+  void Visit(Stmt *s);
   
   bool isTrackedVar(const VarDecl *vd) {
     return ::isTrackedVar(vd, cast<DeclContext>(ac.getDecl()));
   }
   
   FindVarResult findBlockVarDecl(Expr *ex);
+  
+  void ProcessUses(Stmt *s = 0);
 };
 }
 
+static const Expr *stripCasts(ASTContext &C, const Expr *Ex) {
+  while (Ex) {
+    Ex = Ex->IgnoreParenNoopCasts(C);
+    if (const CastExpr *CE = dyn_cast<CastExpr>(Ex)) {
+      if (CE->getCastKind() == CK_LValueBitCast) {
+        Ex = CE->getSubExpr();
+        continue;
+      }
+    }
+    break;
+  }
+  return Ex;
+}
+
 void TransferFunctions::reportUninit(const DeclRefExpr *ex,
                                      const VarDecl *vd, bool isAlwaysUnit) {
   if (handler) handler->handleUseOfUninitVariable(ex, vd, isAlwaysUnit);
 }
 
-FindVarResult TransferFunctions::findBlockVarDecl(Expr* ex) {
-  if (DeclRefExpr* dr = dyn_cast<DeclRefExpr>(ex->IgnoreParenCasts()))
+FindVarResult TransferFunctions::findBlockVarDecl(Expr *ex) {
+  if (DeclRefExpr *dr = dyn_cast<DeclRefExpr>(ex->IgnoreParenCasts()))
     if (VarDecl *vd = dyn_cast<VarDecl>(dr->getDecl()))
       if (isTrackedVar(vd))
         return FindVarResult(vd, dr);  
   return FindVarResult(0, 0);
 }
 
-void TransferFunctions::BlockStmt_VisitObjCForCollectionStmt(
-    ObjCForCollectionStmt *fs) {
-  
-  Visit(fs->getCollection());
-  
+void TransferFunctions::VisitObjCForCollectionStmt(ObjCForCollectionStmt *fs) {
   // This represents an initialization of the 'element' value.
   Stmt *element = fs->getElement();
-  const VarDecl* vd = 0;
+  const VarDecl *vd = 0;
   
-  if (DeclStmt* ds = dyn_cast<DeclStmt>(element)) {
+  if (DeclStmt *ds = dyn_cast<DeclStmt>(element)) {
     vd = cast<VarDecl>(ds->getSingleDecl());
     if (!isTrackedVar(vd))
       vd = 0;
-  }
-  else {
+  } else {
     // Initialize the value of the reference variable.
     const FindVarResult &res = findBlockVarDecl(cast<Expr>(element));
     vd = res.getDecl();
-    if (!vd) {
-      Visit(element);
-      return;
-    }
   }
   
   if (vd)
@@ -416,14 +431,10 @@ void TransferFunctions::BlockStmt_VisitObjCForCollectionStmt(
 }
 
 void TransferFunctions::VisitBlockExpr(BlockExpr *be) {
-  if (!flagBlockUses || !handler)
-    return;
   const BlockDecl *bd = be->getBlockDecl();
   for (BlockDecl::capture_const_iterator i = bd->capture_begin(),
         e = bd->capture_end() ; i != e; ++i) {
     const VarDecl *vd = i->getVariable();
-    if (!vd->hasLocalStorage())
-      continue;
     if (!isTrackedVar(vd))
       continue;
     if (i->isByRef()) {
@@ -431,19 +442,27 @@ void TransferFunctions::VisitBlockExpr(BlockExpr *be) {
       continue;
     }
     Value v = vals[vd];
-    if (isUninitialized(v))
+    if (handler && isUninitialized(v))
       handler->handleUseOfUninitVariable(be, vd, isAlwaysUninit(v));
   }
 }
 
+void TransferFunctions::VisitDeclRefExpr(DeclRefExpr *dr) {
+  // Record the last DeclRefExpr seen.  This is an lvalue computation.
+  // We use this value to later detect if a variable "escapes" the analysis.
+  if (const VarDecl *vd = dyn_cast<VarDecl>(dr->getDecl()))
+    if (isTrackedVar(vd)) {
+      ProcessUses();
+      lastDR = dr;
+    }
+}
+
 void TransferFunctions::VisitDeclStmt(DeclStmt *ds) {
   for (DeclStmt::decl_iterator DI = ds->decl_begin(), DE = ds->decl_end();
        DI != DE; ++DI) {
     if (VarDecl *vd = dyn_cast<VarDecl>(*DI)) {
       if (isTrackedVar(vd)) {
         if (Expr *init = vd->getInit()) {
-          Visit(init);
-
           // If the initializer consists solely of a reference to itself, we
           // explicitly mark the variable as uninitialized. This allows code
           // like the following:
@@ -454,56 +473,48 @@ void TransferFunctions::VisitDeclStmt(DeclStmt *ds) {
           // clients can detect this pattern and adjust their reporting
           // appropriately, but we need to continue to analyze subsequent uses
           // of the variable.
-          DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(init->IgnoreParenImpCasts());
-          vals[vd] = (DRE && DRE->getDecl() == vd) ? Uninitialized
-                                                   : Initialized;
+          if (init == lastLoad) {
+            const DeclRefExpr *DR
+              = cast<DeclRefExpr>(stripCasts(ac.getASTContext(),
+                                             lastLoad->getSubExpr()));
+            if (DR->getDecl() == vd) {
+              // int x = x;
+              // Propagate uninitialized value, but don't immediately report
+              // a problem.
+              vals[vd] = Uninitialized;
+              lastLoad = 0;
+              lastDR = 0;
+              if (handler)
+                handler->handleSelfInit(vd);
+              return;
+            }
+          }
+
+          // All other cases: treat the new variable as initialized.
+          // This is a minor optimization to reduce the propagation
+          // of the analysis, since we will have already reported
+          // the use of the uninitialized value (which visiting the
+          // initializer).
+          vals[vd] = Initialized;
         }
-      } else if (Stmt *init = vd->getInit()) {
-        Visit(init);
       }
     }
   }
 }
 
-void TransferFunctions::VisitDeclRefExpr(DeclRefExpr *dr) {
-  // We assume that DeclRefExprs wrapped in an lvalue-to-rvalue cast
-  // cannot be block-level expressions.  Therefore, we determine if
-  // a DeclRefExpr is involved in a "load" by comparing it to the current
-  // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
-  // If a DeclRefExpr is not involved in a load, we are essentially computing
-  // its address, either for assignment to a reference or via the '&' operator.
-  // In such cases, treat the variable as being initialized, since this
-  // analysis isn't powerful enough to do alias tracking.
-  if (dr != currentDR)
-    if (const VarDecl *vd = dyn_cast<VarDecl>(dr->getDecl()))
-      if (isTrackedVar(vd))
-        vals[vd] = Initialized;
-}
-
 void TransferFunctions::VisitBinaryOperator(clang::BinaryOperator *bo) {
   if (bo->isAssignmentOp()) {
     const FindVarResult &res = findBlockVarDecl(bo->getLHS());
-    if (const VarDecl* vd = res.getDecl()) {
-      // We assume that DeclRefExprs wrapped in a BinaryOperator "assignment"
-      // cannot be block-level expressions.  Therefore, we determine if
-      // a DeclRefExpr is involved in a "load" by comparing it to the current
-      // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
-      SaveAndRestore<const DeclRefExpr*> lastDR(currentDR, 
-                                                res.getDeclRefExpr());
-      Visit(bo->getRHS());
-      Visit(bo->getLHS());
-
+    if (const VarDecl *vd = res.getDecl()) {
       ValueVector::reference val = vals[vd];
       if (isUninitialized(val)) {
         if (bo->getOpcode() != BO_Assign)
           reportUninit(res.getDeclRefExpr(), vd, isAlwaysUninit(val));
-        val = Initialized;
+        else
+          val = Initialized;
       }
-      return;
     }
   }
-  Visit(bo->getRHS());
-  Visit(bo->getLHS());
 }
 
 void TransferFunctions::VisitUnaryOperator(clang::UnaryOperator *uo) {
@@ -514,86 +525,88 @@ void TransferFunctions::VisitUnaryOperator(clang::UnaryOperator *uo) {
     case clang::UO_PreInc: {
       const FindVarResult &res = findBlockVarDecl(uo->getSubExpr());
       if (const VarDecl *vd = res.getDecl()) {
-        // We assume that DeclRefExprs wrapped in a unary operator ++/--
-        // cannot be block-level expressions.  Therefore, we determine if
-        // a DeclRefExpr is involved in a "load" by comparing it to the current
-        // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
-        SaveAndRestore<const DeclRefExpr*> lastDR(currentDR, 
-                                                  res.getDeclRefExpr());
-        Visit(uo->getSubExpr());
+        assert(res.getDeclRefExpr() == lastDR);
+        // We null out lastDR to indicate we have fully processed it
+        // and we don't want the auto-value setting in Visit().
+        lastDR = 0;
 
         ValueVector::reference val = vals[vd];
-        if (isUninitialized(val)) {
+        if (isUninitialized(val))
           reportUninit(res.getDeclRefExpr(), vd, isAlwaysUninit(val));
-          // Don't cascade warnings.
-          val = Initialized;
-        }
-        return;
       }
       break;
     }
     default:
       break;
   }
-  Visit(uo->getSubExpr());
 }
 
 void TransferFunctions::VisitCastExpr(clang::CastExpr *ce) {
   if (ce->getCastKind() == CK_LValueToRValue) {
     const FindVarResult &res = findBlockVarDecl(ce->getSubExpr());
-    if (const VarDecl *vd = res.getDecl()) {
-      // We assume that DeclRefExprs wrapped in an lvalue-to-rvalue cast
-      // cannot be block-level expressions.  Therefore, we determine if
-      // a DeclRefExpr is involved in a "load" by comparing it to the current
-      // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
-      // Here we update 'currentDR' to be the one associated with this
-      // lvalue-to-rvalue cast.  Then, when we analyze the DeclRefExpr, we
-      // will know that we are not computing its lvalue for other purposes
-      // than to perform a load.
-      SaveAndRestore<const DeclRefExpr*> lastDR(currentDR, 
-                                                res.getDeclRefExpr());
-      Visit(ce->getSubExpr());
-      if (currentVoidCast != ce) {
-        Value val = vals[vd];
-        if (isUninitialized(val)) {
-          reportUninit(res.getDeclRefExpr(), vd, isAlwaysUninit(val));
-          // Don't cascade warnings.
-          vals[vd] = Initialized;
-        }
-      }
-      return;
+    if (res.getDecl()) {
+      assert(res.getDeclRefExpr() == lastDR);
+      lastLoad = ce;
     }
   }
+  else if (ce->getCastKind() == CK_NoOp ||
+           ce->getCastKind() == CK_LValueBitCast) {
+    skipProcessUses = true;
+  }
   else if (CStyleCastExpr *cse = dyn_cast<CStyleCastExpr>(ce)) {
     if (cse->getType()->isVoidType()) {
       // e.g. (void) x;
-      SaveAndRestore<const Expr *>
-        lastVoidCast(currentVoidCast, cse->getSubExpr()->IgnoreParens());
-      Visit(cse->getSubExpr());
+      if (lastLoad == cse->getSubExpr()) {
+        // Squelch any detected load of an uninitialized value if
+        // we cast it to void.
+        lastLoad = 0;
+        lastDR = 0;
+      }
+    }
+  }
+}
+
+void TransferFunctions::Visit(clang::Stmt *s) {
+  skipProcessUses = false;
+  StmtVisitor<TransferFunctions>::Visit(s);
+  if (!skipProcessUses)
+    ProcessUses(s);
+}
+
+void TransferFunctions::ProcessUses(Stmt *s) {
+  // This method is typically called after visiting a CFGElement statement
+  // in the CFG.  We delay processing of reporting many loads of uninitialized
+  // values until here.
+  if (lastLoad) {
+    // If we just visited the lvalue-to-rvalue cast, there is nothing
+    // left to do.
+    if (lastLoad == s)
+      return;
+
+    const DeclRefExpr *DR =
+      cast<DeclRefExpr>(stripCasts(ac.getASTContext(),
+                                   lastLoad->getSubExpr()));
+    const VarDecl *VD = cast<VarDecl>(DR->getDecl());
+
+    // If we reach here, we may have seen a load of an uninitialized value
+    // and it hasn't been casted to void or otherwise handled.  In this
+    // situation, report the incident.
+    if (isUninitialized(vals[VD]))
+      reportUninit(DR, VD, isAlwaysUninit(vals[VD]));
+
+    lastLoad = 0;
+
+    if (DR == lastDR) {
+      lastDR = 0;
       return;
     }
   }
-  Visit(ce->getSubExpr());
-}
 
-void TransferFunctions::VisitUnaryExprOrTypeTraitExpr(
-                                          UnaryExprOrTypeTraitExpr *se) {
-  if (se->getKind() == UETT_SizeOf) {
-    if (se->getType()->isConstantSizeType())
-      return;
-    // Handle VLAs.
-    Visit(se->getArgumentExpr());
-  }
-}
-
-void TransferFunctions::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
-  // typeid(expression) is potentially evaluated when the argument is
-  // a glvalue of polymorphic type. (C++ 5.2.8p2-3)
-  if (!E->isTypeOperand() && E->Classify(ac.getASTContext()).isGLValue()) {
-    QualType SubExprTy = E->getExprOperand()->getType();
-    if (const RecordType *Record = SubExprTy->getAs<RecordType>())
-      if (cast<CXXRecordDecl>(Record->getDecl())->isPolymorphic())
-        Visit(E->getExprOperand());
+  // Any other uses of 'lastDR' involve taking an lvalue of variable.
+  // In this case, it "escapes" the analysis.
+  if (lastDR && lastDR != s) {
+    vals[cast<VarDecl>(lastDR->getDecl())] = Initialized;
+    lastDR = 0;
   }
 }
 
@@ -604,8 +617,7 @@ void TransferFunctions::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
 static bool runOnBlock(const CFGBlock *block, const CFG &cfg,
                        AnalysisContext &ac, CFGBlockValues &vals,
                        llvm::BitVector &wasAnalyzed,
-                       UninitVariablesHandler *handler = 0,
-                       bool flagBlockUses = false) {
+                       UninitVariablesHandler *handler = 0) {
   
   wasAnalyzed[block->getBlockID()] = true;
   
@@ -623,8 +635,7 @@ static bool runOnBlock(const CFGBlock *block, const CFG &cfg,
       vals.mergeIntoScratch(*(vB.second ? vB.second : vB.first), false);
       valsAB.first = vA.first;
       valsAB.second = &vals.getScratch();
-    }
-    else {
+    } else {
       // Merge the 'T' bits from the first and second.
       assert(b->getOpcode() == BO_LOr);
       vals.mergeIntoScratch(*vA.first, true);
@@ -640,17 +651,21 @@ static bool runOnBlock(const CFGBlock *block, const CFG &cfg,
   bool isFirst = true;
   for (CFGBlock::const_pred_iterator I = block->pred_begin(),
        E = block->pred_end(); I != E; ++I) {
-    vals.mergeIntoScratch(vals.getValueVector(*I, block), isFirst);
-    isFirst = false;
+    const CFGBlock *pred = *I;
+    if (wasAnalyzed[pred->getBlockID()]) {
+      vals.mergeIntoScratch(vals.getValueVector(pred, block), isFirst);
+      isFirst = false;
+    }
   }
   // Apply the transfer function.
-  TransferFunctions tf(vals, cfg, ac, handler, flagBlockUses);
+  TransferFunctions tf(vals, cfg, ac, handler);
   for (CFGBlock::const_iterator I = block->begin(), E = block->end(); 
        I != E; ++I) {
     if (const CFGStmt *cs = dyn_cast<CFGStmt>(&*I)) {
-      tf.BlockStmt_Visit(cs->getStmt());
+      tf.Visit(const_cast<Stmt*>(cs->getStmt()));
     }
   }
+  tf.ProcessUses();
   return vals.updateValueVectorWithScratch(block);
 }
 
@@ -685,6 +700,7 @@ void clang::runUninitializedVariablesAnalysis(
   llvm::BitVector previouslyVisited(cfg.getNumBlockIDs());
   worklist.enqueueSuccessors(&cfg.getEntry());
   llvm::BitVector wasAnalyzed(cfg.getNumBlockIDs(), false);
+  wasAnalyzed[cfg.getEntry().getBlockID()] = true;
 
   while (const CFGBlock *block = worklist.dequeue()) {
     // Did the block change?
@@ -697,9 +713,9 @@ void clang::runUninitializedVariablesAnalysis(
   
   // Run through the blocks one more time, and report uninitialized variabes.
   for (CFG::const_iterator BI = cfg.begin(), BE = cfg.end(); BI != BE; ++BI) {
-    if (wasAnalyzed[(*BI)->getBlockID()]) {
-      runOnBlock(*BI, cfg, ac, vals, wasAnalyzed, &handler,
-                 /* flagBlockUses */ true);
+    const CFGBlock *block = *BI;
+    if (wasAnalyzed[block->getBlockID()]) {
+      runOnBlock(block, cfg, ac, vals, wasAnalyzed, &handler);
       ++stats.NumBlockVisits;
     }
   }
diff --git a/contrib/llvm/tools/clang/lib/Basic/Builtins.cpp b/contrib/llvm/tools/clang/lib/Basic/Builtins.cpp
index 7df24a03b03a..7bdcdc688552 100644
--- a/contrib/llvm/tools/clang/lib/Basic/Builtins.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/Builtins.cpp
@@ -32,11 +32,15 @@ const Builtin::Info &Builtin::Context::GetRecord(unsigned ID) const {
   return TSRecords[ID - Builtin::FirstTSBuiltin];
 }
 
-Builtin::Context::Context(const TargetInfo &Target) {
+Builtin::Context::Context() {
   // Get the target specific builtins from the target.
   TSRecords = 0;
   NumTSRecords = 0;
-  Target.getTargetBuiltins(TSRecords, NumTSRecords);
+}
+
+void Builtin::Context::InitializeTarget(const TargetInfo &Target) {
+  assert(NumTSRecords == 0 && "Already initialized target?");
+  Target.getTargetBuiltins(TSRecords, NumTSRecords);  
 }
 
 /// InitializeBuiltins - Mark the identifiers for all the builtins with their
@@ -59,7 +63,7 @@ void Builtin::Context::InitializeBuiltins(IdentifierTable &Table,
 }
 
 void
-Builtin::Context::GetBuiltinNames(llvm::SmallVectorImpl<const char *> &Names,
+Builtin::Context::GetBuiltinNames(SmallVectorImpl<const char *> &Names,
                                   bool NoBuiltins) {
   // Final all target-independent names
   for (unsigned i = Builtin::NotBuiltin+1; i != Builtin::FirstTSBuiltin; ++i)
diff --git a/contrib/llvm/tools/clang/lib/Basic/Diagnostic.cpp b/contrib/llvm/tools/clang/lib/Basic/Diagnostic.cpp
index ae363a0df0d0..e5f390196d80 100644
--- a/contrib/llvm/tools/clang/lib/Basic/Diagnostic.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/Diagnostic.cpp
@@ -20,21 +20,22 @@
 
 using namespace clang;
 
-static void DummyArgToStringFn(Diagnostic::ArgumentKind AK, intptr_t QT,
+static void DummyArgToStringFn(DiagnosticsEngine::ArgumentKind AK, intptr_t QT,
                                const char *Modifier, unsigned ML,
                                const char *Argument, unsigned ArgLen,
-                               const Diagnostic::ArgumentValue *PrevArgs,
+                               const DiagnosticsEngine::ArgumentValue *PrevArgs,
                                unsigned NumPrevArgs,
-                               llvm::SmallVectorImpl<char> &Output,
+                               SmallVectorImpl<char> &Output,
                                void *Cookie,
-                               llvm::SmallVectorImpl<intptr_t> &QualTypeVals) {
+                               SmallVectorImpl<intptr_t> &QualTypeVals) {
   const char *Str = "<can't format argument>";
   Output.append(Str, Str+strlen(Str));
 }
 
 
-Diagnostic::Diagnostic(const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &diags,
-                       DiagnosticClient *client, bool ShouldOwnClient)
+DiagnosticsEngine::DiagnosticsEngine(
+                       const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &diags,
+                       DiagnosticConsumer *client, bool ShouldOwnClient)
   : Diags(diags), Client(client), OwnsDiagClient(ShouldOwnClient),
     SourceMgr(0) {
   ArgToStringFn = DummyArgToStringFn;
@@ -43,6 +44,7 @@ Diagnostic::Diagnostic(const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &diags,
   AllExtensionsSilenced = 0;
   IgnoreAllWarnings = false;
   WarningsAsErrors = false;
+  EnableAllWarnings = false;
   ErrorsAsFatal = false;
   SuppressSystemWarnings = false;
   SuppressAllDiagnostics = false;
@@ -55,12 +57,13 @@ Diagnostic::Diagnostic(const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &diags,
   Reset();
 }
 
-Diagnostic::~Diagnostic() {
+DiagnosticsEngine::~DiagnosticsEngine() {
   if (OwnsDiagClient)
     delete Client;
 }
 
-void Diagnostic::setClient(DiagnosticClient *client, bool ShouldOwnClient) {
+void DiagnosticsEngine::setClient(DiagnosticConsumer *client,
+                                  bool ShouldOwnClient) {
   if (OwnsDiagClient && Client)
     delete Client;
   
@@ -68,11 +71,11 @@ void Diagnostic::setClient(DiagnosticClient *client, bool ShouldOwnClient) {
   OwnsDiagClient = ShouldOwnClient;
 }
 
-void Diagnostic::pushMappings(SourceLocation Loc) {
+void DiagnosticsEngine::pushMappings(SourceLocation Loc) {
   DiagStateOnPushStack.push_back(GetCurDiagState());
 }
 
-bool Diagnostic::popMappings(SourceLocation Loc) {
+bool DiagnosticsEngine::popMappings(SourceLocation Loc) {
   if (DiagStateOnPushStack.empty())
     return false;
 
@@ -84,7 +87,7 @@ bool Diagnostic::popMappings(SourceLocation Loc) {
   return true;
 }
 
-void Diagnostic::Reset() {
+void DiagnosticsEngine::Reset() {
   ErrorOccurred = false;
   FatalErrorOccurred = false;
   UnrecoverableErrorOccurred = false;
@@ -92,11 +95,13 @@ void Diagnostic::Reset() {
   NumWarnings = 0;
   NumErrors = 0;
   NumErrorsSuppressed = 0;
+  TrapNumErrorsOccurred = 0;
+  TrapNumUnrecoverableErrorsOccurred = 0;
   
   CurDiagID = ~0U;
   // Set LastDiagLevel to an "unset" state. If we set it to 'Ignored', notes
-  // using a Diagnostic associated to a translation unit that follow
-  // diagnostics from a Diagnostic associated to anoter t.u. will not be
+  // using a DiagnosticsEngine associated to a translation unit that follow
+  // diagnostics from a DiagnosticsEngine associated to anoter t.u. will not be
   // displayed.
   LastDiagLevel = (DiagnosticIDs::Level)-1;
   DelayedDiagID = 0;
@@ -112,8 +117,8 @@ void Diagnostic::Reset() {
   PushDiagStatePoint(&DiagStates.back(), SourceLocation());
 }
 
-void Diagnostic::SetDelayedDiagnostic(unsigned DiagID, llvm::StringRef Arg1,
-                                      llvm::StringRef Arg2) {
+void DiagnosticsEngine::SetDelayedDiagnostic(unsigned DiagID, StringRef Arg1,
+                                      StringRef Arg2) {
   if (DelayedDiagID)
     return;
 
@@ -122,15 +127,15 @@ void Diagnostic::SetDelayedDiagnostic(unsigned DiagID, llvm::StringRef Arg1,
   DelayedDiagArg2 = Arg2.str();
 }
 
-void Diagnostic::ReportDelayed() {
+void DiagnosticsEngine::ReportDelayed() {
   Report(DelayedDiagID) << DelayedDiagArg1 << DelayedDiagArg2;
   DelayedDiagID = 0;
   DelayedDiagArg1.clear();
   DelayedDiagArg2.clear();
 }
 
-Diagnostic::DiagStatePointsTy::iterator
-Diagnostic::GetDiagStatePointForLoc(SourceLocation L) const {
+DiagnosticsEngine::DiagStatePointsTy::iterator
+DiagnosticsEngine::GetDiagStatePointForLoc(SourceLocation L) const {
   assert(!DiagStatePoints.empty());
   assert(DiagStatePoints.front().Loc.isInvalid() &&
          "Should have created a DiagStatePoint for command-line");
@@ -155,7 +160,7 @@ Diagnostic::GetDiagStatePointForLoc(SourceLocation L) const {
 ///
 /// \param The source location that this change of diagnostic state should
 /// take affect. It can be null if we are setting the latest state.
-void Diagnostic::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
+void DiagnosticsEngine::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
                                       SourceLocation L) {
   assert(Diag < diag::DIAG_UPPER_LIMIT &&
          "Can only map builtin diagnostics");
@@ -167,10 +172,19 @@ void Diagnostic::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
   bool isPragma = L.isValid();
   FullSourceLoc Loc(L, *SourceMgr);
   FullSourceLoc LastStateChangePos = DiagStatePoints.back().Loc;
+  DiagnosticMappingInfo MappingInfo = DiagnosticMappingInfo::Make(
+    Map, /*IsUser=*/true, isPragma);
+
+  // If this is a pragma mapping, then set the diagnostic mapping flags so that
+  // we override command line options.
+  if (isPragma) {
+    MappingInfo.setNoWarningAsError(true);
+    MappingInfo.setNoErrorAsFatal(true);
+  }
 
   // Common case; setting all the diagnostics of a group in one place.
   if (Loc.isInvalid() || Loc == LastStateChangePos) {
-    setDiagnosticMappingInternal(Diag, Map, GetCurDiagState(), true, isPragma);
+    GetCurDiagState()->setMappingInfo(Diag, MappingInfo);
     return;
   }
 
@@ -183,7 +197,7 @@ void Diagnostic::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
     // the new state became active.
     DiagStates.push_back(*GetCurDiagState());
     PushDiagStatePoint(&DiagStates.back(), Loc);
-    setDiagnosticMappingInternal(Diag, Map, GetCurDiagState(), true, isPragma);
+    GetCurDiagState()->setMappingInfo(Diag, MappingInfo);
     return;
   }
 
@@ -196,12 +210,12 @@ void Diagnostic::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
   // Update all diagnostic states that are active after the given location.
   for (DiagStatePointsTy::iterator
          I = Pos+1, E = DiagStatePoints.end(); I != E; ++I) {
-    setDiagnosticMappingInternal(Diag, Map, I->State, true, isPragma);
+    GetCurDiagState()->setMappingInfo(Diag, MappingInfo);
   }
 
   // If the location corresponds to an existing point, just update its state.
   if (Pos->Loc == Loc) {
-    setDiagnosticMappingInternal(Diag, Map, Pos->State, true, isPragma);
+    GetCurDiagState()->setMappingInfo(Diag, MappingInfo);
     return;
   }
 
@@ -210,12 +224,86 @@ void Diagnostic::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
   Pos->Loc.isBeforeInTranslationUnitThan(Loc);
   DiagStates.push_back(*Pos->State);
   DiagState *NewState = &DiagStates.back();
-  setDiagnosticMappingInternal(Diag, Map, NewState, true, isPragma);
+  GetCurDiagState()->setMappingInfo(Diag, MappingInfo);
   DiagStatePoints.insert(Pos+1, DiagStatePoint(NewState,
                                                FullSourceLoc(Loc, *SourceMgr)));
 }
 
-void Diagnostic::Report(const StoredDiagnostic &storedDiag) {
+bool DiagnosticsEngine::setDiagnosticGroupMapping(
+  StringRef Group, diag::Mapping Map, SourceLocation Loc)
+{
+  // Get the diagnostics in this group.
+  llvm::SmallVector<diag::kind, 8> GroupDiags;
+  if (Diags->getDiagnosticsInGroup(Group, GroupDiags))
+    return true;
+
+  // Set the mapping.
+  for (unsigned i = 0, e = GroupDiags.size(); i != e; ++i)
+    setDiagnosticMapping(GroupDiags[i], Map, Loc);
+
+  return false;
+}
+
+bool DiagnosticsEngine::setDiagnosticGroupWarningAsError(StringRef Group,
+                                                         bool Enabled) {
+  // If we are enabling this feature, just set the diagnostic mappings to map to
+  // errors.
+  if (Enabled)
+    return setDiagnosticGroupMapping(Group, diag::MAP_ERROR);
+
+  // Otherwise, we want to set the diagnostic mapping's "no Werror" bit, and
+  // potentially downgrade anything already mapped to be a warning.
+
+  // Get the diagnostics in this group.
+  llvm::SmallVector<diag::kind, 8> GroupDiags;
+  if (Diags->getDiagnosticsInGroup(Group, GroupDiags))
+    return true;
+
+  // Perform the mapping change.
+  for (unsigned i = 0, e = GroupDiags.size(); i != e; ++i) {
+    DiagnosticMappingInfo &Info = GetCurDiagState()->getOrAddMappingInfo(
+      GroupDiags[i]);
+
+    if (Info.getMapping() == diag::MAP_ERROR ||
+        Info.getMapping() == diag::MAP_FATAL)
+      Info.setMapping(diag::MAP_WARNING);
+
+    Info.setNoWarningAsError(true);
+  }
+
+  return false;
+}
+
+bool DiagnosticsEngine::setDiagnosticGroupErrorAsFatal(StringRef Group,
+                                                       bool Enabled) {
+  // If we are enabling this feature, just set the diagnostic mappings to map to
+  // fatal errors.
+  if (Enabled)
+    return setDiagnosticGroupMapping(Group, diag::MAP_FATAL);
+
+  // Otherwise, we want to set the diagnostic mapping's "no Werror" bit, and
+  // potentially downgrade anything already mapped to be an error.
+
+  // Get the diagnostics in this group.
+  llvm::SmallVector<diag::kind, 8> GroupDiags;
+  if (Diags->getDiagnosticsInGroup(Group, GroupDiags))
+    return true;
+
+  // Perform the mapping change.
+  for (unsigned i = 0, e = GroupDiags.size(); i != e; ++i) {
+    DiagnosticMappingInfo &Info = GetCurDiagState()->getOrAddMappingInfo(
+      GroupDiags[i]);
+
+    if (Info.getMapping() == diag::MAP_FATAL)
+      Info.setMapping(diag::MAP_ERROR);
+
+    Info.setNoErrorAsFatal(true);
+  }
+
+  return false;
+}
+
+void DiagnosticsEngine::Report(const StoredDiagnostic &storedDiag) {
   assert(CurDiagID == ~0U && "Multiple diagnostics in flight at once!");
 
   CurDiagLoc = storedDiag.getLocation();
@@ -232,19 +320,20 @@ void Diagnostic::Report(const StoredDiagnostic &storedDiag) {
     DiagRanges[i++] = *RI;
 
   NumFixItHints = storedDiag.fixit_size();
-  assert(NumFixItHints < Diagnostic::MaxFixItHints && "Too many fix-it hints!");
+  assert(NumFixItHints < DiagnosticsEngine::MaxFixItHints &&
+      "Too many fix-it hints!");
   i = 0;
   for (StoredDiagnostic::fixit_iterator
          FI = storedDiag.fixit_begin(),
          FE = storedDiag.fixit_end(); FI != FE; ++FI)
     FixItHints[i++] = *FI;
 
-  assert(Client && "DiagnosticClient not set!");
+  assert(Client && "DiagnosticConsumer not set!");
   Level DiagLevel = storedDiag.getLevel();
-  DiagnosticInfo Info(this, storedDiag.getMessage());
+  Diagnostic Info(this, storedDiag.getMessage());
   Client->HandleDiagnostic(DiagLevel, Info);
   if (Client->IncludeInDiagnosticCounts()) {
-    if (DiagLevel == Diagnostic::Warning)
+    if (DiagLevel == DiagnosticsEngine::Warning)
       ++NumWarnings;
   }
 
@@ -263,11 +352,11 @@ bool DiagnosticBuilder::Emit() {
   if (DiagObj == 0) return false;
 
   // When emitting diagnostics, we set the final argument count into
-  // the Diagnostic object.
+  // the DiagnosticsEngine object.
   FlushCounts();
 
   // Process the diagnostic, sending the accumulated information to the
-  // DiagnosticClient.
+  // DiagnosticConsumer.
   bool Emitted = DiagObj->ProcessDiag();
 
   // Clear out the current diagnostic object.
@@ -285,16 +374,16 @@ bool DiagnosticBuilder::Emit() {
 }
 
 
-DiagnosticClient::~DiagnosticClient() {}
+DiagnosticConsumer::~DiagnosticConsumer() {}
 
-void DiagnosticClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                        const DiagnosticInfo &Info) {
+void DiagnosticConsumer::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                        const Diagnostic &Info) {
   if (!IncludeInDiagnosticCounts())
     return;
 
-  if (DiagLevel == Diagnostic::Warning)
+  if (DiagLevel == DiagnosticsEngine::Warning)
     ++NumWarnings;
-  else if (DiagLevel >= Diagnostic::Error)
+  else if (DiagLevel >= DiagnosticsEngine::Error)
     ++NumErrors;
 }
 
@@ -337,9 +426,9 @@ static const char *ScanFormat(const char *I, const char *E, char Target) {
 /// "%2" has a value from 0-2.  If the value is 0, the diagnostic prints 'foo'.
 /// If the value is 1, it prints 'bar'.  If it has the value 2, it prints 'baz'.
 /// This is very useful for certain classes of variant diagnostics.
-static void HandleSelectModifier(const DiagnosticInfo &DInfo, unsigned ValNo,
+static void HandleSelectModifier(const Diagnostic &DInfo, unsigned ValNo,
                                  const char *Argument, unsigned ArgumentLen,
-                                 llvm::SmallVectorImpl<char> &OutStr) {
+                                 SmallVectorImpl<char> &OutStr) {
   const char *ArgumentEnd = Argument+ArgumentLen;
 
   // Skip over 'ValNo' |'s.
@@ -362,7 +451,7 @@ static void HandleSelectModifier(const DiagnosticInfo &DInfo, unsigned ValNo,
 /// letter 's' to the string if the value is not 1.  This is used in cases like
 /// this:  "you idiot, you have %4 parameter%s4!".
 static void HandleIntegerSModifier(unsigned ValNo,
-                                   llvm::SmallVectorImpl<char> &OutStr) {
+                                   SmallVectorImpl<char> &OutStr) {
   if (ValNo != 1)
     OutStr.push_back('s');
 }
@@ -372,7 +461,7 @@ static void HandleIntegerSModifier(unsigned ValNo,
 /// to the first ordinal.  Currently this is hard-coded to use the
 /// English form.
 static void HandleOrdinalModifier(unsigned ValNo,
-                                  llvm::SmallVectorImpl<char> &OutStr) {
+                                  SmallVectorImpl<char> &OutStr) {
   assert(ValNo != 0 && "ValNo must be strictly positive!");
 
   llvm::raw_svector_ostream Out(OutStr);
@@ -495,9 +584,9 @@ static bool EvalPluralExpr(unsigned ValNo, const char *Start, const char *End) {
 /// {1:form0|[2,4]:form1|:form2}
 /// Polish (requires repeated form):
 /// {1:form0|%100=[10,20]:form2|%10=[2,4]:form1|:form2}
-static void HandlePluralModifier(const DiagnosticInfo &DInfo, unsigned ValNo,
+static void HandlePluralModifier(const Diagnostic &DInfo, unsigned ValNo,
                                  const char *Argument, unsigned ArgumentLen,
-                                 llvm::SmallVectorImpl<char> &OutStr) {
+                                 SmallVectorImpl<char> &OutStr) {
   const char *ArgumentEnd = Argument + ArgumentLen;
   while (1) {
     assert(Argument < ArgumentEnd && "Plural expression didn't match.");
@@ -523,34 +612,34 @@ static void HandlePluralModifier(const DiagnosticInfo &DInfo, unsigned ValNo,
 /// FormatDiagnostic - Format this diagnostic into a string, substituting the
 /// formal arguments into the %0 slots.  The result is appended onto the Str
 /// array.
-void DiagnosticInfo::
-FormatDiagnostic(llvm::SmallVectorImpl<char> &OutStr) const {
+void Diagnostic::
+FormatDiagnostic(SmallVectorImpl<char> &OutStr) const {
   if (!StoredDiagMessage.empty()) {
     OutStr.append(StoredDiagMessage.begin(), StoredDiagMessage.end());
     return;
   }
 
-  llvm::StringRef Diag = 
+  StringRef Diag = 
     getDiags()->getDiagnosticIDs()->getDescription(getID());
 
   FormatDiagnostic(Diag.begin(), Diag.end(), OutStr);
 }
 
-void DiagnosticInfo::
+void Diagnostic::
 FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
-                 llvm::SmallVectorImpl<char> &OutStr) const {
+                 SmallVectorImpl<char> &OutStr) const {
 
   /// FormattedArgs - Keep track of all of the arguments formatted by
   /// ConvertArgToString and pass them into subsequent calls to
   /// ConvertArgToString, allowing the implementation to avoid redundancies in
   /// obvious cases.
-  llvm::SmallVector<Diagnostic::ArgumentValue, 8> FormattedArgs;
+  SmallVector<DiagnosticsEngine::ArgumentValue, 8> FormattedArgs;
 
   /// QualTypeVals - Pass a vector of arrays so that QualType names can be
   /// compared to see if more information is needed to be printed.
-  llvm::SmallVector<intptr_t, 2> QualTypeVals;
+  SmallVector<intptr_t, 2> QualTypeVals;
   for (unsigned i = 0, e = getNumArgs(); i < e; ++i)
-    if (getArgKind(i) == Diagnostic::ak_qualtype)
+    if (getArgKind(i) == DiagnosticsEngine::ak_qualtype)
       QualTypeVals.push_back(getRawArg(i));
 
   while (DiagStr != DiagEnd) {
@@ -600,17 +689,17 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
     assert(isdigit(*DiagStr) && "Invalid format for argument in diagnostic");
     unsigned ArgNo = *DiagStr++ - '0';
 
-    Diagnostic::ArgumentKind Kind = getArgKind(ArgNo);
+    DiagnosticsEngine::ArgumentKind Kind = getArgKind(ArgNo);
     
     switch (Kind) {
     // ---- STRINGS ----
-    case Diagnostic::ak_std_string: {
+    case DiagnosticsEngine::ak_std_string: {
       const std::string &S = getArgStdStr(ArgNo);
       assert(ModifierLen == 0 && "No modifiers for strings yet");
       OutStr.append(S.begin(), S.end());
       break;
     }
-    case Diagnostic::ak_c_string: {
+    case DiagnosticsEngine::ak_c_string: {
       const char *S = getArgCStr(ArgNo);
       assert(ModifierLen == 0 && "No modifiers for strings yet");
 
@@ -622,7 +711,7 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
       break;
     }
     // ---- INTEGERS ----
-    case Diagnostic::ak_sint: {
+    case DiagnosticsEngine::ak_sint: {
       int Val = getArgSInt(ArgNo);
 
       if (ModifierIs(Modifier, ModifierLen, "select")) {
@@ -641,7 +730,7 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
       }
       break;
     }
-    case Diagnostic::ak_uint: {
+    case DiagnosticsEngine::ak_uint: {
       unsigned Val = getArgUInt(ArgNo);
 
       if (ModifierIs(Modifier, ModifierLen, "select")) {
@@ -660,7 +749,7 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
       break;
     }
     // ---- NAMES and TYPES ----
-    case Diagnostic::ak_identifierinfo: {
+    case DiagnosticsEngine::ak_identifierinfo: {
       const IdentifierInfo *II = getArgIdentifier(ArgNo);
       assert(ModifierLen == 0 && "No modifiers for strings yet");
 
@@ -674,11 +763,11 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
       llvm::raw_svector_ostream(OutStr) << '\'' << II->getName() << '\'';
       break;
     }
-    case Diagnostic::ak_qualtype:
-    case Diagnostic::ak_declarationname:
-    case Diagnostic::ak_nameddecl:
-    case Diagnostic::ak_nestednamespec:
-    case Diagnostic::ak_declcontext:
+    case DiagnosticsEngine::ak_qualtype:
+    case DiagnosticsEngine::ak_declarationname:
+    case DiagnosticsEngine::ak_nameddecl:
+    case DiagnosticsEngine::ak_nestednamespec:
+    case DiagnosticsEngine::ak_declcontext:
       getDiags()->ConvertArgToString(Kind, getRawArg(ArgNo),
                                      Modifier, ModifierLen,
                                      Argument, ArgumentLen,
@@ -690,10 +779,10 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
     // Remember this argument info for subsequent formatting operations.  Turn
     // std::strings into a null terminated string to make it be the same case as
     // all the other ones.
-    if (Kind != Diagnostic::ak_std_string)
+    if (Kind != DiagnosticsEngine::ak_std_string)
       FormattedArgs.push_back(std::make_pair(Kind, getRawArg(ArgNo)));
     else
-      FormattedArgs.push_back(std::make_pair(Diagnostic::ak_c_string,
+      FormattedArgs.push_back(std::make_pair(DiagnosticsEngine::ak_c_string,
                                         (intptr_t)getArgStdStr(ArgNo).c_str()));
     
   }
@@ -701,12 +790,12 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
 
 StoredDiagnostic::StoredDiagnostic() { }
 
-StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, unsigned ID,
-                                   llvm::StringRef Message)
+StoredDiagnostic::StoredDiagnostic(DiagnosticsEngine::Level Level, unsigned ID,
+                                   StringRef Message)
   : ID(ID), Level(Level), Loc(), Message(Message) { }
 
-StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, 
-                                   const DiagnosticInfo &Info)
+StoredDiagnostic::StoredDiagnostic(DiagnosticsEngine::Level Level, 
+                                   const Diagnostic &Info)
   : ID(Info.getID()), Level(Level) 
 {
   assert((Info.getLocation().isInvalid() || Info.hasSourceManager()) &&
@@ -726,13 +815,23 @@ StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level,
     FixIts.push_back(Info.getFixItHint(I));
 }
 
+StoredDiagnostic::StoredDiagnostic(DiagnosticsEngine::Level Level, unsigned ID,
+                                   StringRef Message, FullSourceLoc Loc,
+                                   ArrayRef<CharSourceRange> Ranges,
+                                   ArrayRef<FixItHint> Fixits)
+  : ID(ID), Level(Level), Loc(Loc), Message(Message) 
+{
+  this->Ranges.assign(Ranges.begin(), Ranges.end());
+  this->FixIts.assign(FixIts.begin(), FixIts.end());
+}
+
 StoredDiagnostic::~StoredDiagnostic() { }
 
 /// IncludeInDiagnosticCounts - This method (whose default implementation
 ///  returns true) indicates whether the diagnostics handled by this
-///  DiagnosticClient should be included in the number of diagnostics
-///  reported by Diagnostic.
-bool DiagnosticClient::IncludeInDiagnosticCounts() const { return true; }
+///  DiagnosticConsumer should be included in the number of diagnostics
+///  reported by DiagnosticsEngine.
+bool DiagnosticConsumer::IncludeInDiagnosticCounts() const { return true; }
 
 PartialDiagnostic::StorageAllocator::StorageAllocator() {
   for (unsigned I = 0; I != NumCached; ++I)
diff --git a/contrib/llvm/tools/clang/lib/Basic/DiagnosticIDs.cpp b/contrib/llvm/tools/clang/lib/Basic/DiagnosticIDs.cpp
index 147ba7e99e74..9481287c08cf 100644
--- a/contrib/llvm/tools/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/DiagnosticIDs.cpp
@@ -21,6 +21,8 @@
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Sema/SemaDiagnostic.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/ErrorHandling.h"
 
 #include <map>
 using namespace clang;
@@ -45,6 +47,8 @@ struct StaticDiagInfoRec {
   unsigned Class : 3;
   unsigned SFINAE : 1;
   unsigned AccessControl : 1;
+  unsigned WarnNoWerror : 1;
+  unsigned WarnShowInSystemHeader : 1;
   unsigned Category : 5;
 
   uint8_t  NameLen;
@@ -61,21 +65,21 @@ struct StaticDiagInfoRec {
   const char *BriefExplanationStr;
   const char *FullExplanationStr;
 
-  llvm::StringRef getName() const {
-    return llvm::StringRef(NameStr, NameLen);
+  StringRef getName() const {
+    return StringRef(NameStr, NameLen);
   }
-  llvm::StringRef getOptionGroup() const {
-    return llvm::StringRef(OptionGroupStr, OptionGroupLen);
+  StringRef getOptionGroup() const {
+    return StringRef(OptionGroupStr, OptionGroupLen);
   }
 
-  llvm::StringRef getDescription() const {
-    return llvm::StringRef(DescriptionStr, DescriptionLen);
+  StringRef getDescription() const {
+    return StringRef(DescriptionStr, DescriptionLen);
   }
-  llvm::StringRef getBriefExplanation() const {
-    return llvm::StringRef(BriefExplanationStr, BriefExplanationLen);
+  StringRef getBriefExplanation() const {
+    return StringRef(BriefExplanationStr, BriefExplanationLen);
   }
-  llvm::StringRef getFullExplanation() const {
-    return llvm::StringRef(FullExplanationStr, FullExplanationLen);
+  StringRef getFullExplanation() const {
+    return StringRef(FullExplanationStr, FullExplanationLen);
   }
 
   bool operator<(const StaticDiagInfoRec &RHS) const {
@@ -88,8 +92,8 @@ struct StaticDiagNameIndexRec {
   unsigned short DiagID;
   uint8_t NameLen;
 
-  llvm::StringRef getName() const {
-    return llvm::StringRef(NameStr, NameLen);
+  StringRef getName() const {
+    return StringRef(NameStr, NameLen);
   }
 
   bool operator<(const StaticDiagNameIndexRec &RHS) const {
@@ -114,8 +118,10 @@ class StringSizerHelper {
 
 static const StaticDiagInfoRec StaticDiagInfo[] = {
 #define DIAG(ENUM,CLASS,DEFAULT_MAPPING,DESC,GROUP,               \
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL)                   \
-  { diag::ENUM, DEFAULT_MAPPING, CLASS, SFINAE, ACCESS, CATEGORY, \
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,              \
+             CATEGORY,BRIEF,FULL)                                 \
+  { diag::ENUM, DEFAULT_MAPPING, CLASS, SFINAE, ACCESS,           \
+    NOWERROR, SHOWINSYSHEADER, CATEGORY,                          \
     STR_SIZE(#ENUM, uint8_t), STR_SIZE(GROUP, uint8_t),           \
     STR_SIZE(DESC, uint16_t), STR_SIZE(BRIEF, uint16_t),          \
     STR_SIZE(FULL, uint16_t),                                     \
@@ -129,7 +135,7 @@ static const StaticDiagInfoRec StaticDiagInfo[] = {
 #include "clang/Basic/DiagnosticSemaKinds.inc"
 #include "clang/Basic/DiagnosticAnalysisKinds.inc"
 #undef DIAG
-  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
 };
 
 static const unsigned StaticDiagInfoSize =
@@ -166,7 +172,8 @@ static const StaticDiagInfoRec *GetDiagInfo(unsigned DiagID) {
 #endif
 
   // Search the diagnostic table with a binary search.
-  StaticDiagInfoRec Find = { DiagID, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0,0,0 };
+  StaticDiagInfoRec Find = { static_cast<unsigned short>(DiagID),
+                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
   const StaticDiagInfoRec *Found =
     std::lower_bound(StaticDiagInfo, StaticDiagInfo + StaticDiagInfoSize, Find);
@@ -177,19 +184,36 @@ static const StaticDiagInfoRec *GetDiagInfo(unsigned DiagID) {
   return Found;
 }
 
-static unsigned GetDefaultDiagMapping(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->Mapping;
-  return diag::MAP_FATAL;
+static DiagnosticMappingInfo GetDefaultDiagMappingInfo(unsigned DiagID) {
+  DiagnosticMappingInfo Info = DiagnosticMappingInfo::Make(
+    diag::MAP_FATAL, /*IsUser=*/false, /*IsPragma=*/false);
+
+  if (const StaticDiagInfoRec *StaticInfo = GetDiagInfo(DiagID)) {
+    Info.setMapping((diag::Mapping) StaticInfo->Mapping);
+
+    if (StaticInfo->WarnNoWerror) {
+      assert(Info.getMapping() == diag::MAP_WARNING &&
+             "Unexpected mapping with no-Werror bit!");
+      Info.setNoWarningAsError(true);
+    }
+
+    if (StaticInfo->WarnShowInSystemHeader) {
+      assert(Info.getMapping() == diag::MAP_WARNING &&
+             "Unexpected mapping with show-in-system-header bit!");
+      Info.setShowInSystemHeader(true);
+    }
+  }
+
+  return Info;
 }
 
 /// getWarningOptionForDiag - Return the lowest-level warning option that
 /// enables the specified diagnostic.  If there is no -Wfoo flag that controls
 /// the diagnostic, this returns null.
-llvm::StringRef DiagnosticIDs::getWarningOptionForDiag(unsigned DiagID) {
+StringRef DiagnosticIDs::getWarningOptionForDiag(unsigned DiagID) {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getOptionGroup();
-  return llvm::StringRef();
+  return StringRef();
 }
 
 /// getCategoryNumberForDiag - Return the category number that a specified
@@ -206,12 +230,28 @@ namespace {
     const char *NameStr;
     uint8_t NameLen;
 
-    llvm::StringRef getName() const {
-      return llvm::StringRef(NameStr, NameLen);
+    StringRef getName() const {
+      return StringRef(NameStr, NameLen);
     }
   };
 }
 
+// Unfortunately, the split between DiagnosticIDs and Diagnostic is not
+// particularly clean, but for now we just implement this method here so we can
+// access GetDefaultDiagMapping.
+DiagnosticMappingInfo &DiagnosticsEngine::DiagState::getOrAddMappingInfo(
+  diag::kind Diag)
+{
+  std::pair<iterator, bool> Result = DiagMap.insert(
+    std::make_pair(Diag, DiagnosticMappingInfo()));
+
+  // Initialize the entry if we added it.
+  if (Result.second)
+    Result.first->second = GetDefaultDiagMappingInfo(Diag);
+
+  return Result.first->second;
+}
+
 static const StaticDiagCategoryRec CategoryNameTable[] = {
 #define GET_CATEGORY_TABLE
 #define CATEGORY(X, ENUM) { X, STR_SIZE(X, uint8_t) },
@@ -228,9 +268,9 @@ unsigned DiagnosticIDs::getNumberOfCategories() {
 /// getCategoryNameFromID - Given a category ID, return the name of the
 /// category, an empty string if CategoryID is zero, or null if CategoryID is
 /// invalid.
-llvm::StringRef DiagnosticIDs::getCategoryNameFromID(unsigned CategoryID) {
+StringRef DiagnosticIDs::getCategoryNameFromID(unsigned CategoryID) {
   if (CategoryID >= getNumberOfCategories())
-   return llvm::StringRef();
+   return StringRef();
   return CategoryNameTable[CategoryID].getName();
 }
 
@@ -256,20 +296,23 @@ DiagnosticIDs::getDiagnosticSFINAEResponse(unsigned DiagID) {
 }
 
 /// getName - Given a diagnostic ID, return its name
-llvm::StringRef DiagnosticIDs::getName(unsigned DiagID) {
+StringRef DiagnosticIDs::getName(unsigned DiagID) {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getName();
-  return llvm::StringRef();
+  return StringRef();
 }
 
 /// getIdFromName - Given a diagnostic name, return its ID, or 0
-unsigned DiagnosticIDs::getIdFromName(llvm::StringRef Name) {
+unsigned DiagnosticIDs::getIdFromName(StringRef Name) {
   const StaticDiagNameIndexRec *StaticDiagNameIndexEnd =
     StaticDiagNameIndex + StaticDiagNameIndexSize;
   
   if (Name.empty()) { return diag::DIAG_UPPER_LIMIT; }
   
-  StaticDiagNameIndexRec Find = { Name.data(), 0, Name.size() };
+  assert(Name.size() == static_cast<uint8_t>(Name.size()) &&
+         "Name is too long");
+  StaticDiagNameIndexRec Find = { Name.data(), 0,
+                                  static_cast<uint8_t>(Name.size()) };
   
   const StaticDiagNameIndexRec *Found =
     std::lower_bound( StaticDiagNameIndex, StaticDiagNameIndexEnd, Find);
@@ -282,18 +325,18 @@ unsigned DiagnosticIDs::getIdFromName(llvm::StringRef Name) {
 
 /// getBriefExplanation - Given a diagnostic ID, return a brief explanation
 /// of the issue
-llvm::StringRef DiagnosticIDs::getBriefExplanation(unsigned DiagID) {
+StringRef DiagnosticIDs::getBriefExplanation(unsigned DiagID) {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getBriefExplanation();
-  return llvm::StringRef();
+  return StringRef();
 }
 
 /// getFullExplanation - Given a diagnostic ID, return a full explanation
 /// of the issue
-llvm::StringRef DiagnosticIDs::getFullExplanation(unsigned DiagID) {
+StringRef DiagnosticIDs::getFullExplanation(unsigned DiagID) {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getFullExplanation();
-  return llvm::StringRef();
+  return StringRef();
 }
 
 /// getBuiltinDiagClass - Return the class field of the diagnostic.
@@ -304,6 +347,35 @@ static unsigned getBuiltinDiagClass(unsigned DiagID) {
   return ~0U;
 }
 
+//===----------------------------------------------------------------------===//
+// diag_iterator
+//===----------------------------------------------------------------------===//
+
+llvm::StringRef DiagnosticIDs::diag_iterator::getDiagName() const {
+  return static_cast<const StaticDiagNameIndexRec*>(impl)->getName();
+}
+
+unsigned DiagnosticIDs::diag_iterator::getDiagID() const {
+  return static_cast<const StaticDiagNameIndexRec*>(impl)->DiagID;
+}
+
+DiagnosticIDs::diag_iterator &DiagnosticIDs::diag_iterator::operator++() {
+  const StaticDiagNameIndexRec* ptr =
+    static_cast<const StaticDiagNameIndexRec*>(impl);;
+  ++ptr;
+  impl = ptr;
+  return *this;
+}
+
+DiagnosticIDs::diag_iterator DiagnosticIDs::diags_begin() {
+  return DiagnosticIDs::diag_iterator(StaticDiagNameIndex);
+}
+
+DiagnosticIDs::diag_iterator DiagnosticIDs::diags_end() {
+  return DiagnosticIDs::diag_iterator(StaticDiagNameIndex +
+                                      StaticDiagNameIndexSize);
+}
+
 //===----------------------------------------------------------------------===//
 // Custom Diagnostic information
 //===----------------------------------------------------------------------===//
@@ -318,7 +390,7 @@ namespace clang {
 
       /// getDescription - Return the description of the specified custom
       /// diagnostic.
-      llvm::StringRef getDescription(unsigned DiagID) const {
+      StringRef getDescription(unsigned DiagID) const {
         assert(this && DiagID-DIAG_UPPER_LIMIT < DiagInfo.size() &&
                "Invalid diagnosic ID");
         return DiagInfo[DiagID-DIAG_UPPER_LIMIT].second;
@@ -331,7 +403,7 @@ namespace clang {
         return DiagInfo[DiagID-DIAG_UPPER_LIMIT].first;
       }
 
-      unsigned getOrCreateDiagID(DiagnosticIDs::Level L, llvm::StringRef Message,
+      unsigned getOrCreateDiagID(DiagnosticIDs::Level L, StringRef Message,
                                  DiagnosticIDs &Diags) {
         DiagDesc D(L, Message);
         // Check to see if it already exists.
@@ -366,7 +438,7 @@ DiagnosticIDs::~DiagnosticIDs() {
 /// getCustomDiagID - Return an ID for a diagnostic with the specified message
 /// and level.  If this is the first request for this diagnosic, it is
 /// registered and created, otherwise the existing ID is returned.
-unsigned DiagnosticIDs::getCustomDiagID(Level L, llvm::StringRef Message) {
+unsigned DiagnosticIDs::getCustomDiagID(Level L, StringRef Message) {
   if (CustomDiagInfo == 0)
     CustomDiagInfo = new diag::CustomDiagInfo();
   return CustomDiagInfo->getOrCreateDiagID(L, Message, *this);
@@ -400,32 +472,39 @@ bool DiagnosticIDs::isBuiltinExtensionDiag(unsigned DiagID,
       getBuiltinDiagClass(DiagID) != CLASS_EXTENSION)
     return false;
   
-  EnabledByDefault = GetDefaultDiagMapping(DiagID) != diag::MAP_IGNORE;
+  EnabledByDefault =
+    GetDefaultDiagMappingInfo(DiagID).getMapping() != diag::MAP_IGNORE;
   return true;
 }
 
+bool DiagnosticIDs::isDefaultMappingAsError(unsigned DiagID) {
+  if (DiagID >= diag::DIAG_UPPER_LIMIT)
+    return false;
+
+  return GetDefaultDiagMappingInfo(DiagID).getMapping() == diag::MAP_ERROR;
+}
+
 /// getDescription - Given a diagnostic ID, return a description of the
 /// issue.
-llvm::StringRef DiagnosticIDs::getDescription(unsigned DiagID) const {
+StringRef DiagnosticIDs::getDescription(unsigned DiagID) const {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getDescription();
   return CustomDiagInfo->getDescription(DiagID);
 }
 
-/// getDiagnosticLevel - Based on the way the client configured the Diagnostic
-/// object, classify the specified diagnostic ID into a Level, consumable by
-/// the DiagnosticClient.
+/// getDiagnosticLevel - Based on the way the client configured the
+/// DiagnosticsEngine object, classify the specified diagnostic ID into a Level,
+/// by consumable the DiagnosticClient.
 DiagnosticIDs::Level
 DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
-                                  const Diagnostic &Diag,
-                                  diag::Mapping *mapping) const {
+                                  const DiagnosticsEngine &Diag) const {
   // Handle custom diagnostics, which cannot be mapped.
   if (DiagID >= diag::DIAG_UPPER_LIMIT)
     return CustomDiagInfo->getLevel(DiagID);
 
   unsigned DiagClass = getBuiltinDiagClass(DiagID);
   assert(DiagClass != CLASS_NOTE && "Cannot get diagnostic level of a note!");
-  return getDiagnosticLevel(DiagID, DiagClass, Loc, Diag, mapping);
+  return getDiagnosticLevel(DiagID, DiagClass, Loc, Diag);
 }
 
 /// \brief Based on the way the client configured the Diagnostic
@@ -437,130 +516,119 @@ DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
 DiagnosticIDs::Level
 DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, unsigned DiagClass,
                                   SourceLocation Loc,
-                                  const Diagnostic &Diag,
-                                  diag::Mapping *mapping) const {
+                                  const DiagnosticsEngine &Diag) const {
   // Specific non-error diagnostics may be mapped to various levels from ignored
   // to error.  Errors can only be mapped to fatal.
   DiagnosticIDs::Level Result = DiagnosticIDs::Fatal;
 
-  Diagnostic::DiagStatePointsTy::iterator
+  DiagnosticsEngine::DiagStatePointsTy::iterator
     Pos = Diag.GetDiagStatePointForLoc(Loc);
-  Diagnostic::DiagState *State = Pos->State;
+  DiagnosticsEngine::DiagState *State = Pos->State;
 
-  // Get the mapping information, if unset, compute it lazily.
-  unsigned MappingInfo = Diag.getDiagnosticMappingInfo((diag::kind)DiagID,
-                                                       State);
-  if (MappingInfo == 0) {
-    MappingInfo = GetDefaultDiagMapping(DiagID);
-    Diag.setDiagnosticMappingInternal(DiagID, MappingInfo, State, false, false);
-  }
-  
-  if (mapping)
-    *mapping = (diag::Mapping) (MappingInfo & 7);
+  // Get the mapping information, or compute it lazily.
+  DiagnosticMappingInfo &MappingInfo = State->getOrAddMappingInfo(
+    (diag::kind)DiagID);
 
-  bool ShouldEmitInSystemHeader = false;
-
-  switch (MappingInfo & 7) {
-  default: assert(0 && "Unknown mapping!");
+  switch (MappingInfo.getMapping()) {
+  default: llvm_unreachable("Unknown mapping!");
   case diag::MAP_IGNORE:
-    // Ignore this, unless this is an extension diagnostic and we're mapping
-    // them onto warnings or errors.
-    if (!isBuiltinExtensionDiag(DiagID) ||  // Not an extension
-        Diag.ExtBehavior == Diagnostic::Ext_Ignore || // Ext ignored
-        (MappingInfo & 8) != 0)             // User explicitly mapped it.
-      return DiagnosticIDs::Ignored;
+    Result = DiagnosticIDs::Ignored;
+    break;
+  case diag::MAP_WARNING:
     Result = DiagnosticIDs::Warning;
-    if (Diag.ExtBehavior == Diagnostic::Ext_Error) Result = DiagnosticIDs::Error;
-    if (Result == DiagnosticIDs::Error && Diag.ErrorsAsFatal)
-      Result = DiagnosticIDs::Fatal;
     break;
   case diag::MAP_ERROR:
     Result = DiagnosticIDs::Error;
-    if (Diag.ErrorsAsFatal)
-      Result = DiagnosticIDs::Fatal;
     break;
   case diag::MAP_FATAL:
     Result = DiagnosticIDs::Fatal;
     break;
-  case diag::MAP_WARNING_SHOW_IN_SYSTEM_HEADER:
-    ShouldEmitInSystemHeader = true;
-    // continue as MAP_WARNING.
-  case diag::MAP_WARNING:
-    // If warnings are globally mapped to ignore or error, do it.
-    if (Diag.IgnoreAllWarnings)
-      return DiagnosticIDs::Ignored;
-
-    Result = DiagnosticIDs::Warning;
-
-    // If this is an extension diagnostic and we're in -pedantic-error mode, and
-    // if the user didn't explicitly map it, upgrade to an error.
-    if (Diag.ExtBehavior == Diagnostic::Ext_Error &&
-        (MappingInfo & 8) == 0 &&
-        isBuiltinExtensionDiag(DiagID))
-      Result = DiagnosticIDs::Error;
-
-    if (Diag.WarningsAsErrors)
-      Result = DiagnosticIDs::Error;
-    if (Result == DiagnosticIDs::Error && Diag.ErrorsAsFatal)
-      Result = DiagnosticIDs::Fatal;
-    break;
-
-  case diag::MAP_WARNING_NO_WERROR:
-    // Diagnostics specified with -Wno-error=foo should be set to warnings, but
-    // not be adjusted by -Werror or -pedantic-errors.
-    Result = DiagnosticIDs::Warning;
-
-    // If warnings are globally mapped to ignore or error, do it.
-    if (Diag.IgnoreAllWarnings)
-      return DiagnosticIDs::Ignored;
-
-    break;
-
-  case diag::MAP_ERROR_NO_WFATAL:
-    // Diagnostics specified as -Wno-fatal-error=foo should be errors, but
-    // unaffected by -Wfatal-errors.
-    Result = DiagnosticIDs::Error;
-    break;
   }
 
-  // Okay, we're about to return this as a "diagnostic to emit" one last check:
-  // if this is any sort of extension warning, and if we're in an __extension__
-  // block, silence it.
-  if (Diag.AllExtensionsSilenced && isBuiltinExtensionDiag(DiagID))
+  // Upgrade ignored diagnostics if -Weverything is enabled.
+  if (Diag.EnableAllWarnings && Result == DiagnosticIDs::Ignored &&
+      !MappingInfo.isUser())
+    Result = DiagnosticIDs::Warning;
+
+  // Ignore -pedantic diagnostics inside __extension__ blocks.
+  // (The diagnostics controlled by -pedantic are the extension diagnostics
+  // that are not enabled by default.)
+  bool EnabledByDefault;
+  bool IsExtensionDiag = isBuiltinExtensionDiag(DiagID, EnabledByDefault);
+  if (Diag.AllExtensionsSilenced && IsExtensionDiag && !EnabledByDefault)
     return DiagnosticIDs::Ignored;
 
-  // If we are in a system header, we ignore it.
-  // We also want to ignore extensions and warnings in -Werror and
+  // For extension diagnostics that haven't been explicitly mapped, check if we
+  // should upgrade the diagnostic.
+  if (IsExtensionDiag && !MappingInfo.isUser()) {
+    switch (Diag.ExtBehavior) {
+    case DiagnosticsEngine::Ext_Ignore:
+      break; 
+    case DiagnosticsEngine::Ext_Warn:
+      // Upgrade ignored diagnostics to warnings.
+      if (Result == DiagnosticIDs::Ignored)
+        Result = DiagnosticIDs::Warning;
+      break;
+    case DiagnosticsEngine::Ext_Error:
+      // Upgrade ignored or warning diagnostics to errors.
+      if (Result == DiagnosticIDs::Ignored || Result == DiagnosticIDs::Warning)
+        Result = DiagnosticIDs::Error;
+      break;
+    }
+  }
+
+  // At this point, ignored errors can no longer be upgraded.
+  if (Result == DiagnosticIDs::Ignored)
+    return Result;
+
+  // Honor -w, which is lower in priority than pedantic-errors, but higher than
+  // -Werror.
+  if (Result == DiagnosticIDs::Warning && Diag.IgnoreAllWarnings)
+    return DiagnosticIDs::Ignored;
+
+  // If -Werror is enabled, map warnings to errors unless explicitly disabled.
+  if (Result == DiagnosticIDs::Warning) {
+    if (Diag.WarningsAsErrors && !MappingInfo.hasNoWarningAsError())
+      Result = DiagnosticIDs::Error;
+  }
+
+  // If -Wfatal-errors is enabled, map errors to fatal unless explicity
+  // disabled.
+  if (Result == DiagnosticIDs::Error) {
+    if (Diag.ErrorsAsFatal && !MappingInfo.hasNoErrorAsFatal())
+      Result = DiagnosticIDs::Fatal;
+  }
+
+  // If we are in a system header, we ignore it. We look at the diagnostic class
+  // because we also want to ignore extensions and warnings in -Werror and
   // -pedantic-errors modes, which *map* warnings/extensions to errors.
   if (Result >= DiagnosticIDs::Warning &&
       DiagClass != CLASS_ERROR &&
       // Custom diagnostics always are emitted in system headers.
       DiagID < diag::DIAG_UPPER_LIMIT &&
-      !ShouldEmitInSystemHeader &&
+      !MappingInfo.hasShowInSystemHeader() &&
       Diag.SuppressSystemWarnings &&
       Loc.isValid() &&
       Diag.getSourceManager().isInSystemHeader(
-          Diag.getSourceManager().getInstantiationLoc(Loc)))
+          Diag.getSourceManager().getExpansionLoc(Loc)))
     return DiagnosticIDs::Ignored;
 
   return Result;
 }
 
-namespace {
-  struct WarningOption {
-    // Be safe with the size of 'NameLen' because we don't statically check if
-    // the size will fit in the field; the struct size won't decrease with a
-    // shorter type anyway.
-    size_t NameLen;
-    const char *NameStr;
-    const short *Members;
-    const short *SubGroups;
+struct clang::WarningOption {
+  // Be safe with the size of 'NameLen' because we don't statically check if
+  // the size will fit in the field; the struct size won't decrease with a
+  // shorter type anyway.
+  size_t NameLen;
+  const char *NameStr;
+  const short *Members;
+  const short *SubGroups;
 
-    llvm::StringRef getName() const {
-      return llvm::StringRef(NameStr, NameLen);
-    }
-  };
-}
+  StringRef getName() const {
+    return StringRef(NameStr, NameLen);
+  }
+};
 
 #define GET_DIAG_ARRAYS
 #include "clang/Basic/DiagnosticGroups.inc"
@@ -580,54 +648,43 @@ static bool WarningOptionCompare(const WarningOption &LHS,
   return LHS.getName() < RHS.getName();
 }
 
-static void MapGroupMembers(const WarningOption *Group, diag::Mapping Mapping,
-                            SourceLocation Loc, Diagnostic &Diag) {
-  // Option exists, poke all the members of its diagnostic set.
+void DiagnosticIDs::getDiagnosticsInGroup(
+  const WarningOption *Group,
+  llvm::SmallVectorImpl<diag::kind> &Diags) const
+{
+  // Add the members of the option diagnostic set.
   if (const short *Member = Group->Members) {
     for (; *Member != -1; ++Member)
-      Diag.setDiagnosticMapping(*Member, Mapping, Loc);
+      Diags.push_back(*Member);
   }
 
-  // Enable/disable all subgroups along with this one.
+  // Add the members of the subgroups.
   if (const short *SubGroups = Group->SubGroups) {
     for (; *SubGroups != (short)-1; ++SubGroups)
-      MapGroupMembers(&OptionTable[(short)*SubGroups], Mapping, Loc, Diag);
+      getDiagnosticsInGroup(&OptionTable[(short)*SubGroups], Diags);
   }
 }
 
-/// setDiagnosticGroupMapping - Change an entire diagnostic group (e.g.
-/// "unknown-pragmas" to have the specified mapping.  This returns true and
-/// ignores the request if "Group" was unknown, false otherwise.
-bool DiagnosticIDs::setDiagnosticGroupMapping(llvm::StringRef Group,
-                                              diag::Mapping Map,
-                                              SourceLocation Loc,
-                                              Diagnostic &Diag) const {
-  assert((Loc.isValid() ||
-          Diag.DiagStatePoints.empty() ||
-          Diag.DiagStatePoints.back().Loc.isInvalid()) &&
-         "Loc should be invalid only when the mapping comes from command-line");
-  assert((Loc.isInvalid() || Diag.DiagStatePoints.empty() ||
-          Diag.DiagStatePoints.back().Loc.isInvalid() ||
-          !Diag.SourceMgr->isBeforeInTranslationUnit(Loc,
-                                            Diag.DiagStatePoints.back().Loc)) &&
-         "Source location of new mapping is before the previous one!");
-
+bool DiagnosticIDs::getDiagnosticsInGroup(
+  StringRef Group,
+  llvm::SmallVectorImpl<diag::kind> &Diags) const
+{
   WarningOption Key = { Group.size(), Group.data(), 0, 0 };
   const WarningOption *Found =
   std::lower_bound(OptionTable, OptionTable + OptionTableSize, Key,
                    WarningOptionCompare);
   if (Found == OptionTable + OptionTableSize ||
       Found->getName() != Group)
-    return true;  // Option not found.
+    return true; // Option not found.
 
-  MapGroupMembers(Found, Map, Loc, Diag);
+  getDiagnosticsInGroup(Found, Diags);
   return false;
 }
 
 /// ProcessDiag - This is the method used to report a diagnostic that is
 /// finally fully formed.
-bool DiagnosticIDs::ProcessDiag(Diagnostic &Diag) const {
-  DiagnosticInfo Info(&Diag);
+bool DiagnosticIDs::ProcessDiag(DiagnosticsEngine &Diag) const {
+  Diagnostic Info(&Diag);
 
   if (Diag.SuppressAllDiagnostics)
     return false;
@@ -665,6 +722,13 @@ bool DiagnosticIDs::ProcessDiag(Diagnostic &Diag) const {
     Diag.LastDiagLevel = DiagLevel;
   }
 
+  // Update counts for DiagnosticErrorTrap even if a fatal error occurred.
+  if (DiagLevel >= DiagnosticIDs::Error) {
+    ++Diag.TrapNumErrorsOccurred;
+    if (isUnrecoverable(DiagID))
+      ++Diag.TrapNumUnrecoverableErrorsOccurred;
+  }
+
   // If a fatal error has already been emitted, silence all subsequent
   // diagnostics.
   if (Diag.FatalErrorOccurred) {
@@ -685,27 +749,26 @@ bool DiagnosticIDs::ProcessDiag(Diagnostic &Diag) const {
     return false;
 
   if (DiagLevel >= DiagnosticIDs::Error) {
-    Diag.TrapErrorOccurred = true;
-    if (isUnrecoverable(DiagID)) {
-      Diag.TrapUnrecoverableErrorOccurred = true;
+    if (isUnrecoverable(DiagID))
       Diag.UnrecoverableErrorOccurred = true;
-    }
     
     if (Diag.Client->IncludeInDiagnosticCounts()) {
       Diag.ErrorOccurred = true;
       ++Diag.NumErrors;
     }
 
-    // If we've emitted a lot of errors, emit a fatal error after it to stop a
-    // flood of bogus errors.
-    if (Diag.ErrorLimit && Diag.NumErrors >= Diag.ErrorLimit &&
-        DiagLevel == DiagnosticIDs::Error)
+    // If we've emitted a lot of errors, emit a fatal error instead of it to 
+    // stop a flood of bogus errors.
+    if (Diag.ErrorLimit && Diag.NumErrors > Diag.ErrorLimit &&
+        DiagLevel == DiagnosticIDs::Error) {
       Diag.SetDelayedDiagnostic(diag::fatal_too_many_errors);
+      return false;
+    }
   }
 
   // If we have any Fix-Its, make sure that all of the Fix-Its point into
-  // source locations that aren't macro instantiations. If any point into
-  // macro instantiations, remove all of the Fix-Its.
+  // source locations that aren't macro expansions. If any point into macro
+  // expansions, remove all of the Fix-Its.
   for (unsigned I = 0, N = Diag.NumFixItHints; I != N; ++I) {
     const FixItHint &FixIt = Diag.FixItHints[I];
     if (FixIt.RemoveRange.isInvalid() ||
@@ -717,7 +780,7 @@ bool DiagnosticIDs::ProcessDiag(Diagnostic &Diag) const {
   }
   
   // Finally, report it.
-  Diag.Client->HandleDiagnostic((Diagnostic::Level)DiagLevel, Info);
+  Diag.Client->HandleDiagnostic((DiagnosticsEngine::Level)DiagLevel, Info);
   if (Diag.Client->IncludeInDiagnosticCounts()) {
     if (DiagLevel == DiagnosticIDs::Warning)
       ++Diag.NumWarnings;
diff --git a/contrib/llvm/tools/clang/lib/Basic/FileManager.cpp b/contrib/llvm/tools/clang/lib/Basic/FileManager.cpp
index f747c534c0ed..c1f715ed0592 100644
--- a/contrib/llvm/tools/clang/lib/Basic/FileManager.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/FileManager.cpp
@@ -217,25 +217,26 @@ void FileManager::removeStatCache(FileSystemStatCache *statCache) {
 /// \brief Retrieve the directory that the given file name resides in.
 /// Filename can point to either a real file or a virtual file.
 static const DirectoryEntry *getDirectoryFromFile(FileManager &FileMgr,
-                                                  llvm::StringRef Filename) {
+                                                  StringRef Filename,
+                                                  bool CacheFailure) {
   if (Filename.empty())
     return NULL;
 
   if (llvm::sys::path::is_separator(Filename[Filename.size() - 1]))
     return NULL;  // If Filename is a directory.
 
-  llvm::StringRef DirName = llvm::sys::path::parent_path(Filename);
+  StringRef DirName = llvm::sys::path::parent_path(Filename);
   // Use the current directory if file has no path component.
   if (DirName.empty())
     DirName = ".";
 
-  return FileMgr.getDirectory(DirName);
+  return FileMgr.getDirectory(DirName, CacheFailure);
 }
 
 /// Add all ancestors of the given path (pointing to either a file or
 /// a directory) as virtual directories.
-void FileManager::addAncestorsAsVirtualDirs(llvm::StringRef Path) {
-  llvm::StringRef DirName = llvm::sys::path::parent_path(Path);
+void FileManager::addAncestorsAsVirtualDirs(StringRef Path) {
+  StringRef DirName = llvm::sys::path::parent_path(Path);
   if (DirName.empty())
     return;
 
@@ -263,7 +264,8 @@ void FileManager::addAncestorsAsVirtualDirs(llvm::StringRef Path) {
 /// (real or virtual).  This returns NULL if the directory doesn't
 /// exist.
 ///
-const DirectoryEntry *FileManager::getDirectory(llvm::StringRef DirName) {
+const DirectoryEntry *FileManager::getDirectory(StringRef DirName,
+                                                bool CacheFailure) {
   ++NumDirLookups;
   llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt =
     SeenDirEntries.GetOrCreateValue(DirName);
@@ -287,6 +289,8 @@ const DirectoryEntry *FileManager::getDirectory(llvm::StringRef DirName) {
   struct stat StatBuf;
   if (getStatValue(InterndDirName, StatBuf, 0/*directory lookup*/)) {
     // There's no real directory at the given path.
+    if (!CacheFailure)
+      SeenDirEntries.erase(DirName);
     return 0;
   }
 
@@ -309,7 +313,8 @@ const DirectoryEntry *FileManager::getDirectory(llvm::StringRef DirName) {
 /// getFile - Lookup, cache, and verify the specified file (real or
 /// virtual).  This returns NULL if the file doesn't exist.
 ///
-const FileEntry *FileManager::getFile(llvm::StringRef Filename, bool openFile) {
+const FileEntry *FileManager::getFile(StringRef Filename, bool openFile,
+                                      bool CacheFailure) {
   ++NumFileLookups;
 
   // See if there is already an entry in the map.
@@ -335,10 +340,15 @@ const FileEntry *FileManager::getFile(llvm::StringRef Filename, bool openFile) {
   // subdirectory.  This will let us avoid having to waste time on known-to-fail
   // searches when we go to find sys/bar.h, because all the search directories
   // without a 'sys' subdir will get a cached failure result.
-  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename);
-  if (DirInfo == 0)  // Directory doesn't exist, file can't exist.
+  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename,
+                                                       CacheFailure);
+  if (DirInfo == 0) {  // Directory doesn't exist, file can't exist.
+    if (!CacheFailure)
+      SeenFileEntries.erase(Filename);
+    
     return 0;
-
+  }
+  
   // FIXME: Use the directory info to prune this, before doing the stat syscall.
   // FIXME: This will reduce the # syscalls.
 
@@ -347,6 +357,9 @@ const FileEntry *FileManager::getFile(llvm::StringRef Filename, bool openFile) {
   struct stat StatBuf;
   if (getStatValue(InterndFileName, StatBuf, &FileDescriptor)) {
     // There's no real file at the given path.
+    if (!CacheFailure)
+      SeenFileEntries.erase(Filename);
+    
     return 0;
   }
 
@@ -381,7 +394,7 @@ const FileEntry *FileManager::getFile(llvm::StringRef Filename, bool openFile) {
 }
 
 const FileEntry *
-FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
+FileManager::getVirtualFile(StringRef Filename, off_t Size,
                             time_t ModificationTime) {
   ++NumFileLookups;
 
@@ -404,7 +417,8 @@ FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
   // Now that all ancestors of Filename are in the cache, the
   // following call is guaranteed to find the DirectoryEntry from the
   // cache.
-  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename);
+  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename,
+                                                       /*CacheFailure=*/true);
   assert(DirInfo &&
          "The directory of a virtual file should already be in the cache.");
 
@@ -451,8 +465,8 @@ FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
   return UFE;
 }
 
-void FileManager::FixupRelativePath(llvm::SmallVectorImpl<char> &path) const {
-  llvm::StringRef pathRef(path.data(), path.size());
+void FileManager::FixupRelativePath(SmallVectorImpl<char> &path) const {
+  StringRef pathRef(path.data(), path.size());
 
   if (FileSystemOpts.WorkingDir.empty() 
       || llvm::sys::path::is_absolute(pathRef))
@@ -499,7 +513,7 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) {
 }
 
 llvm::MemoryBuffer *FileManager::
-getBufferForFile(llvm::StringRef Filename, std::string *ErrorStr) {
+getBufferForFile(StringRef Filename, std::string *ErrorStr) {
   llvm::OwningPtr<llvm::MemoryBuffer> Result;
   llvm::error_code ec;
   if (FileSystemOpts.WorkingDir.empty()) {
@@ -537,7 +551,7 @@ bool FileManager::getStatValue(const char *Path, struct stat &StatBuf,
                                   StatCache.get());
 }
 
-bool FileManager::getNoncachedStatValue(llvm::StringRef Path, 
+bool FileManager::getNoncachedStatValue(StringRef Path, 
                                         struct stat &StatBuf) {
   llvm::SmallString<128> FilePath(Path);
   FixupRelativePath(FilePath);
@@ -546,7 +560,7 @@ bool FileManager::getNoncachedStatValue(llvm::StringRef Path,
 }
 
 void FileManager::GetUniqueIDMapping(
-                   llvm::SmallVectorImpl<const FileEntry *> &UIDToFiles) const {
+                   SmallVectorImpl<const FileEntry *> &UIDToFiles) const {
   UIDToFiles.clear();
   UIDToFiles.resize(NextFileUID);
   
@@ -558,7 +572,7 @@ void FileManager::GetUniqueIDMapping(
       UIDToFiles[FE->getValue()->getUID()] = FE->getValue();
   
   // Map virtual file entries
-  for (llvm::SmallVector<FileEntry*, 4>::const_iterator 
+  for (SmallVector<FileEntry*, 4>::const_iterator 
          VFE = VirtualFileEntries.begin(), VFEEnd = VirtualFileEntries.end();
        VFE != VFEEnd; ++VFE)
     if (*VFE && *VFE != NON_EXISTENT_FILE)
diff --git a/contrib/llvm/tools/clang/lib/Basic/IdentifierTable.cpp b/contrib/llvm/tools/clang/lib/Basic/IdentifierTable.cpp
index 188e2d46f587..38f09a08af3c 100644
--- a/contrib/llvm/tools/clang/lib/Basic/IdentifierTable.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/IdentifierTable.cpp
@@ -32,6 +32,7 @@ IdentifierInfo::IdentifierInfo() {
   ObjCOrBuiltinID = 0;
   HasMacro = false;
   IsExtension = false;
+  IsCXX11CompatKeyword = false;
   IsPoisoned = false;
   IsCPPOperatorKeyword = false;
   NeedsHandleIdentifier = false;
@@ -55,7 +56,7 @@ namespace {
   class EmptyLookupIterator : public IdentifierIterator
   {
   public:
-    virtual llvm::StringRef Next() { return llvm::StringRef(); }
+    virtual StringRef Next() { return StringRef(); }
   };
 }
 
@@ -102,11 +103,11 @@ namespace {
 /// identifiers because they are language keywords.  This causes the lexer to
 /// automatically map matching identifiers to specialized token codes.
 ///
-/// The C90/C99/CPP/CPP0x flags are set to 2 if the token should be
-/// enabled in the specified langauge, set to 1 if it is an extension
-/// in the specified language, and set to 0 if disabled in the
-/// specified language.
-static void AddKeyword(llvm::StringRef Keyword,
+/// The C90/C99/CPP/CPP0x flags are set to 3 if the token is a keyword in a
+/// future language standard, set to 2 if the token should be enabled in the
+/// specified langauge, set to 1 if it is an extension in the specified
+/// language, and set to 0 if disabled in the specified language.
+static void AddKeyword(StringRef Keyword,
                        tok::TokenKind TokenCode, unsigned Flags,
                        const LangOptions &LangOpts, IdentifierTable &Table) {
   unsigned AddResult = 0;
@@ -115,7 +116,7 @@ static void AddKeyword(llvm::StringRef Keyword,
   else if (LangOpts.CPlusPlus0x && (Flags & KEYCXX0X)) AddResult = 2;
   else if (LangOpts.C99 && (Flags & KEYC99)) AddResult = 2;
   else if (LangOpts.GNUKeywords && (Flags & KEYGNU)) AddResult = 1;
-  else if (LangOpts.Microsoft && (Flags & KEYMS)) AddResult = 1;
+  else if (LangOpts.MicrosoftExt && (Flags & KEYMS)) AddResult = 1;
   else if (LangOpts.Borland && (Flags & KEYBORLAND)) AddResult = 1;
   else if (LangOpts.Bool && (Flags & BOOLSUPPORT)) AddResult = 2;
   else if (LangOpts.AltiVec && (Flags & KEYALTIVEC)) AddResult = 2;
@@ -123,17 +124,20 @@ static void AddKeyword(llvm::StringRef Keyword,
   else if (!LangOpts.CPlusPlus && (Flags & KEYNOCXX)) AddResult = 2;
   else if (LangOpts.C1X && (Flags & KEYC1X)) AddResult = 2;
   else if (LangOpts.ObjCAutoRefCount && (Flags & KEYARC)) AddResult = 2;
-           
+  else if (LangOpts.CPlusPlus && (Flags & KEYCXX0X)) AddResult = 3;
+
   // Don't add this keyword if disabled in this language.
   if (AddResult == 0) return;
 
-  IdentifierInfo &Info = Table.get(Keyword, TokenCode);
+  IdentifierInfo &Info =
+      Table.get(Keyword, AddResult == 3 ? tok::identifier : TokenCode);
   Info.setIsExtensionToken(AddResult == 1);
+  Info.setIsCXX11CompatKeyword(AddResult == 3);
 }
 
 /// AddCXXOperatorKeyword - Register a C++ operator keyword alternative
 /// representations.
-static void AddCXXOperatorKeyword(llvm::StringRef Keyword,
+static void AddCXXOperatorKeyword(StringRef Keyword,
                                   tok::TokenKind TokenCode,
                                   IdentifierTable &Table) {
   IdentifierInfo &Info = Table.get(Keyword, TokenCode);
@@ -142,7 +146,7 @@ static void AddCXXOperatorKeyword(llvm::StringRef Keyword,
 
 /// AddObjCKeyword - Register an Objective-C @keyword like "class" "selector" or
 /// "property".
-static void AddObjCKeyword(llvm::StringRef Name,
+static void AddObjCKeyword(StringRef Name,
                            tok::ObjCKeywordKind ObjCID,
                            IdentifierTable &Table) {
   Table.get(Name).setObjCKeywordID(ObjCID);
@@ -153,20 +157,20 @@ static void AddObjCKeyword(llvm::StringRef Name,
 void IdentifierTable::AddKeywords(const LangOptions &LangOpts) {
   // Add keywords and tokens for the current language.
 #define KEYWORD(NAME, FLAGS) \
-  AddKeyword(llvm::StringRef(#NAME), tok::kw_ ## NAME,  \
+  AddKeyword(StringRef(#NAME), tok::kw_ ## NAME,  \
              FLAGS, LangOpts, *this);
 #define ALIAS(NAME, TOK, FLAGS) \
-  AddKeyword(llvm::StringRef(NAME), tok::kw_ ## TOK,  \
+  AddKeyword(StringRef(NAME), tok::kw_ ## TOK,  \
              FLAGS, LangOpts, *this);
 #define CXX_KEYWORD_OPERATOR(NAME, ALIAS) \
   if (LangOpts.CXXOperatorNames)          \
-    AddCXXOperatorKeyword(llvm::StringRef(#NAME), tok::ALIAS, *this);
+    AddCXXOperatorKeyword(StringRef(#NAME), tok::ALIAS, *this);
 #define OBJC1_AT_KEYWORD(NAME) \
   if (LangOpts.ObjC1)          \
-    AddObjCKeyword(llvm::StringRef(#NAME), tok::objc_##NAME, *this);
+    AddObjCKeyword(StringRef(#NAME), tok::objc_##NAME, *this);
 #define OBJC2_AT_KEYWORD(NAME) \
   if (LangOpts.ObjC2)          \
-    AddObjCKeyword(llvm::StringRef(#NAME), tok::objc_##NAME, *this);
+    AddObjCKeyword(StringRef(#NAME), tok::objc_##NAME, *this);
 #define TESTING_KEYWORD(NAME, FLAGS)
 #include "clang/Basic/TokenKinds.def"
 
@@ -217,6 +221,7 @@ tok::PPKeywordKind IdentifierInfo::getPPKeywordID() const {
   CASE(12, 'i', 'c', include_next);
 
   CASE(16, '_', 'i', __include_macros);
+  CASE(16, '_', 'e', __export_macro__);
 #undef CASE
 #undef HASH
   }
@@ -336,9 +341,9 @@ IdentifierInfo *Selector::getIdentifierInfoForSlot(unsigned argIndex) const {
   return SI->getIdentifierInfoForSlot(argIndex);
 }
 
-llvm::StringRef Selector::getNameForSlot(unsigned int argIndex) const {
+StringRef Selector::getNameForSlot(unsigned int argIndex) const {
   IdentifierInfo *II = getIdentifierInfoForSlot(argIndex);
-  return II? II->getName() : llvm::StringRef();
+  return II? II->getName() : StringRef();
 }
 
 std::string MultiKeywordSelector::getName() const {
@@ -377,7 +382,7 @@ std::string Selector::getAsString() const {
 /// Interpreting the given string using the normal CamelCase
 /// conventions, determine whether the given string starts with the
 /// given "word", which is assumed to end in a lowercase letter.
-static bool startsWithWord(llvm::StringRef name, llvm::StringRef word) {
+static bool startsWithWord(StringRef name, StringRef word) {
   if (name.size() < word.size()) return false;
   return ((name.size() == word.size() ||
            !islower(name[word.size()]))
@@ -388,10 +393,11 @@ ObjCMethodFamily Selector::getMethodFamilyImpl(Selector sel) {
   IdentifierInfo *first = sel.getIdentifierInfoForSlot(0);
   if (!first) return OMF_None;
 
-  llvm::StringRef name = first->getName();
+  StringRef name = first->getName();
   if (sel.isUnarySelector()) {
     if (name == "autorelease") return OMF_autorelease;
     if (name == "dealloc") return OMF_dealloc;
+    if (name == "finalize") return OMF_finalize;
     if (name == "release") return OMF_release;
     if (name == "retain") return OMF_retain;
     if (name == "retainCount") return OMF_retainCount;
@@ -491,4 +497,3 @@ const char *clang::getOperatorSpelling(OverloadedOperatorKind Operator) {
 
   return 0;
 }
-
diff --git a/contrib/llvm/tools/clang/lib/Basic/LangOptions.cpp b/contrib/llvm/tools/clang/lib/Basic/LangOptions.cpp
new file mode 100644
index 000000000000..5f479dbb771c
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Basic/LangOptions.cpp
@@ -0,0 +1,30 @@
+//===--- LangOptions.cpp - C Language Family Language Options ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the LangOptions class.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Basic/LangOptions.h"
+
+using namespace clang;
+
+LangOptions::LangOptions() {
+#define LANGOPT(Name, Bits, Default, Description) Name = Default;
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) set##Name(Default);
+#include "clang/Basic/LangOptions.def"
+}
+
+void LangOptions::resetNonModularOptions() {
+#define LANGOPT(Name, Bits, Default, Description)
+#define BENIGN_LANGOPT(Name, Bits, Default, Description) Name = Default;
+#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  Name = Default;
+#include "clang/Basic/LangOptions.def"
+}
+
diff --git a/contrib/llvm/tools/clang/lib/Basic/SourceLocation.cpp b/contrib/llvm/tools/clang/lib/Basic/SourceLocation.cpp
index 5062d43f58f2..6e4f3e6ad7a4 100644
--- a/contrib/llvm/tools/clang/lib/Basic/SourceLocation.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/SourceLocation.cpp
@@ -23,7 +23,7 @@ using namespace clang;
 // PrettyStackTraceLoc
 //===----------------------------------------------------------------------===//
 
-void PrettyStackTraceLoc::print(llvm::raw_ostream &OS) const {
+void PrettyStackTraceLoc::print(raw_ostream &OS) const {
   if (Loc.isValid()) {
     Loc.print(OS, SM);
     OS << ": ";
@@ -35,7 +35,7 @@ void PrettyStackTraceLoc::print(llvm::raw_ostream &OS) const {
 // SourceLocation
 //===----------------------------------------------------------------------===//
 
-void SourceLocation::print(llvm::raw_ostream &OS, const SourceManager &SM)const{
+void SourceLocation::print(raw_ostream &OS, const SourceManager &SM)const{
   if (!isValid()) {
     OS << "<invalid loc>";
     return;
@@ -48,13 +48,13 @@ void SourceLocation::print(llvm::raw_ostream &OS, const SourceManager &SM)const{
       OS << "<invalid>";
       return;
     }
-    // The instantiation and spelling pos is identical for file locs.
+    // The macro expansion and spelling pos is identical for file locs.
     OS << PLoc.getFilename() << ':' << PLoc.getLine()
        << ':' << PLoc.getColumn();
     return;
   }
 
-  SM.getInstantiationLoc(*this).print(OS, SM);
+  SM.getExpansionLoc(*this).print(OS, SM);
 
   OS << " <Spelling=";
   SM.getSpellingLoc(*this).print(OS, SM);
@@ -75,9 +75,9 @@ FileID FullSourceLoc::getFileID() const {
 }
 
 
-FullSourceLoc FullSourceLoc::getInstantiationLoc() const {
+FullSourceLoc FullSourceLoc::getExpansionLoc() const {
   assert(isValid());
-  return FullSourceLoc(SrcMgr->getInstantiationLoc(*this), *SrcMgr);
+  return FullSourceLoc(SrcMgr->getExpansionLoc(*this), *SrcMgr);
 }
 
 FullSourceLoc FullSourceLoc::getSpellingLoc() const {
@@ -85,14 +85,14 @@ FullSourceLoc FullSourceLoc::getSpellingLoc() const {
   return FullSourceLoc(SrcMgr->getSpellingLoc(*this), *SrcMgr);
 }
 
-unsigned FullSourceLoc::getInstantiationLineNumber(bool *Invalid) const {
+unsigned FullSourceLoc::getExpansionLineNumber(bool *Invalid) const {
   assert(isValid());
-  return SrcMgr->getInstantiationLineNumber(*this, Invalid);
+  return SrcMgr->getExpansionLineNumber(*this, Invalid);
 }
 
-unsigned FullSourceLoc::getInstantiationColumnNumber(bool *Invalid) const {
+unsigned FullSourceLoc::getExpansionColumnNumber(bool *Invalid) const {
   assert(isValid());
-  return SrcMgr->getInstantiationColumnNumber(*this, Invalid);
+  return SrcMgr->getExpansionColumnNumber(*this, Invalid);
 }
 
 unsigned FullSourceLoc::getSpellingLineNumber(bool *Invalid) const {
@@ -125,7 +125,7 @@ const llvm::MemoryBuffer* FullSourceLoc::getBuffer(bool *Invalid) const {
   return SrcMgr->getBuffer(SrcMgr->getFileID(*this), Invalid);
 }
 
-llvm::StringRef FullSourceLoc::getBufferData(bool *Invalid) const {
+StringRef FullSourceLoc::getBufferData(bool *Invalid) const {
   return getBuffer(Invalid)->getBuffer();
 }
 
diff --git a/contrib/llvm/tools/clang/lib/Basic/SourceManager.cpp b/contrib/llvm/tools/clang/lib/Basic/SourceManager.cpp
index 45922c15527f..364663ee3265 100644
--- a/contrib/llvm/tools/clang/lib/Basic/SourceManager.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/SourceManager.cpp
@@ -17,10 +17,12 @@
 #include "clang/Basic/FileManager.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Capacity.h"
 #include <algorithm>
 #include <string>
 #include <cstring>
@@ -39,9 +41,8 @@ ContentCache::~ContentCache() {
     delete Buffer.getPointer();
 }
 
-/// getSizeBytesMapped - Returns the number of bytes actually mapped for
-///  this ContentCache.  This can be 0 if the MemBuffer was not actually
-///  instantiated.
+/// getSizeBytesMapped - Returns the number of bytes actually mapped for this
+/// ContentCache. This can be 0 if the MemBuffer was not actually expanded.
 unsigned ContentCache::getSizeBytesMapped() const {
   return Buffer.getPointer() ? Buffer.getPointer()->getBufferSize() : 0;
 }
@@ -78,7 +79,7 @@ void ContentCache::replaceBuffer(const llvm::MemoryBuffer *B,
   Buffer.setInt(DoNotFree? DoNotFreeFlag : 0);
 }
 
-const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
+const llvm::MemoryBuffer *ContentCache::getBuffer(DiagnosticsEngine &Diag,
                                                   const SourceManager &SM,
                                                   SourceLocation Loc,
                                                   bool *Invalid) const {
@@ -105,7 +106,7 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
   // that we are in an inconsistent situation and error out as quickly as
   // possible.
   if (!Buffer.getPointer()) {
-    const llvm::StringRef FillStr("<<<MISSING SOURCE FILE>>>\n");
+    const StringRef FillStr("<<<MISSING SOURCE FILE>>>\n");
     Buffer.setPointer(MemoryBuffer::getNewMemBuffer(ContentsEntry->getSize(), 
                                                     "<invalid>"));
     char *Ptr = const_cast<char*>(Buffer.getPointer()->getBufferStart());
@@ -143,7 +144,7 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
   // If the buffer is valid, check to see if it has a UTF Byte Order Mark
   // (BOM).  We only support UTF-8 with and without a BOM right now.  See
   // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
-  llvm::StringRef BufStr = Buffer.getPointer()->getBuffer();
+  StringRef BufStr = Buffer.getPointer()->getBuffer();
   const char *InvalidBOM = llvm::StringSwitch<const char *>(BufStr)
     .StartsWith("\xFE\xFF", "UTF-16 (BE)")
     .StartsWith("\xFF\xFE", "UTF-16 (LE)")
@@ -169,7 +170,7 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
   return Buffer.getPointer();
 }
 
-unsigned LineTableInfo::getLineTableFilenameID(llvm::StringRef Name) {
+unsigned LineTableInfo::getLineTableFilenameID(StringRef Name) {
   // Look up the filename in the string table, returning the pre-existing value
   // if it exists.
   llvm::StringMapEntry<unsigned> &Entry =
@@ -186,7 +187,7 @@ unsigned LineTableInfo::getLineTableFilenameID(llvm::StringRef Name) {
 /// AddLineNote - Add a line note to the line table that indicates that there
 /// is a #line at the specified FID/Offset location which changes the presumed
 /// location to LineNo/FilenameID.
-void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset,
+void LineTableInfo::AddLineNote(int FID, unsigned Offset,
                                 unsigned LineNo, int FilenameID) {
   std::vector<LineEntry> &Entries = LineEntries[FID];
 
@@ -217,7 +218,7 @@ void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset,
 /// presumed #include stack.  If it is 1, this is a file entry, if it is 2 then
 /// this is a file exit.  FileKind specifies whether this is a system header or
 /// extern C system header.
-void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset,
+void LineTableInfo::AddLineNote(int FID, unsigned Offset,
                                 unsigned LineNo, int FilenameID,
                                 unsigned EntryExit,
                                 SrcMgr::CharacteristicKind FileKind) {
@@ -251,7 +252,7 @@ void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset,
 
 /// FindNearestLineEntry - Find the line entry nearest to FID that is before
 /// it.  If there is no line entry before Offset in FID, return null.
-const LineEntry *LineTableInfo::FindNearestLineEntry(unsigned FID,
+const LineEntry *LineTableInfo::FindNearestLineEntry(int FID,
                                                      unsigned Offset) {
   const std::vector<LineEntry> &Entries = LineEntries[FID];
   assert(!Entries.empty() && "No #line entries for this FID after all!");
@@ -270,14 +271,14 @@ const LineEntry *LineTableInfo::FindNearestLineEntry(unsigned FID,
 
 /// \brief Add a new line entry that has already been encoded into
 /// the internal representation of the line table.
-void LineTableInfo::AddEntry(unsigned FID,
+void LineTableInfo::AddEntry(int FID,
                              const std::vector<LineEntry> &Entries) {
   LineEntries[FID] = Entries;
 }
 
 /// getLineTableFilenameID - Return the uniqued ID for the specified filename.
 ///
-unsigned SourceManager::getLineTableFilenameID(llvm::StringRef Name) {
+unsigned SourceManager::getLineTableFilenameID(StringRef Name) {
   if (LineTable == 0)
     LineTable = new LineTableInfo();
   return LineTable->getLineTableFilenameID(Name);
@@ -289,7 +290,7 @@ unsigned SourceManager::getLineTableFilenameID(llvm::StringRef Name) {
 /// unspecified.
 void SourceManager::AddLineNote(SourceLocation Loc, unsigned LineNo,
                                 int FilenameID) {
-  std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
 
   bool Invalid = false;
   const SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
@@ -319,7 +320,7 @@ void SourceManager::AddLineNote(SourceLocation Loc, unsigned LineNo,
     return AddLineNote(Loc, LineNo, FilenameID);
   }
 
-  std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
 
   bool Invalid = false;
   const SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
@@ -362,7 +363,7 @@ LineTableInfo &SourceManager::getLineTable() {
 // Private 'Create' methods.
 //===----------------------------------------------------------------------===//
 
-SourceManager::SourceManager(Diagnostic &Diag, FileManager &FileMgr)
+SourceManager::SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr)
   : Diag(Diag), FileMgr(FileMgr), OverridenFilesKeepOriginalName(true),
     ExternalSLocEntries(0), LineTable(0), NumLinearScans(0),
     NumBinaryProbes(0), FakeBufferForRecovery(0) {
@@ -387,11 +388,18 @@ SourceManager::~SourceManager() {
   }
   
   delete FakeBufferForRecovery;
+
+  for (llvm::DenseMap<FileID, MacroArgsMap *>::iterator
+         I = MacroArgsCacheMap.begin(),E = MacroArgsCacheMap.end(); I!=E; ++I) {
+    delete I->second;
+  }
 }
 
 void SourceManager::clearIDTables() {
   MainFileID = FileID();
-  SLocEntryTable.clear();
+  LocalSLocEntryTable.clear();
+  LoadedSLocEntryTable.clear();
+  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = 0;
   LastFileIDLookup = FileID();
@@ -399,9 +407,10 @@ void SourceManager::clearIDTables() {
   if (LineTable)
     LineTable->clear();
 
-  // Use up FileID #0 as an invalid instantiation.
-  NextOffset = 0;
-  createInstantiationLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1);
+  // Use up FileID #0 as an invalid expansion.
+  NextLocalOffset = 0;
+  CurrentLoadedOffset = MaxLoadedOffset;
+  createExpansionLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1);
 }
 
 /// getOrCreateContentCache - Create or return a cached ContentCache for the
@@ -452,33 +461,16 @@ SourceManager::createMemBufferContentCache(const MemoryBuffer *Buffer) {
   return Entry;
 }
 
-void SourceManager::PreallocateSLocEntries(ExternalSLocEntrySource *Source,
-                                           unsigned NumSLocEntries,
-                                           unsigned NextOffset) {
-  ExternalSLocEntries = Source;
-  this->NextOffset = NextOffset;
-  unsigned CurPrealloc = SLocEntryLoaded.size();
-  // If we've ever preallocated, we must not count the dummy entry.
-  if (CurPrealloc) --CurPrealloc;
-  SLocEntryLoaded.resize(NumSLocEntries + 1);
-  SLocEntryLoaded[0] = true;
-  SLocEntryTable.resize(SLocEntryTable.size() + NumSLocEntries - CurPrealloc);
-}
-
-void SourceManager::ClearPreallocatedSLocEntries() {
-  unsigned I = 0;
-  for (unsigned N = SLocEntryLoaded.size(); I != N; ++I)
-    if (!SLocEntryLoaded[I])
-      break;
-
-  // We've already loaded all preallocated source location entries.
-  if (I == SLocEntryLoaded.size())
-    return;
-
-  // Remove everything from location I onward.
-  SLocEntryTable.resize(I);
-  SLocEntryLoaded.clear();
-  ExternalSLocEntries = 0;
+std::pair<int, unsigned>
+SourceManager::AllocateLoadedSLocEntries(unsigned NumSLocEntries,
+                                         unsigned TotalSize) {
+  assert(ExternalSLocEntries && "Don't have an external sloc source");
+  LoadedSLocEntryTable.resize(LoadedSLocEntryTable.size() + NumSLocEntries);
+  SLocEntryLoaded.resize(LoadedSLocEntryTable.size());
+  CurrentLoadedOffset -= TotalSize;
+  assert(CurrentLoadedOffset >= NextLocalOffset && "Out of source locations");
+  int ID = LoadedSLocEntryTable.size();
+  return std::make_pair(-ID - 1, CurrentLoadedOffset);
 }
 
 /// \brief As part of recovering from missing or changed content, produce a
@@ -492,7 +484,7 @@ const llvm::MemoryBuffer *SourceManager::getFakeBufferForRecovery() const {
 }
 
 //===----------------------------------------------------------------------===//
-// Methods to create new FileID's and instantiations.
+// Methods to create new FileID's and macro expansions.
 //===----------------------------------------------------------------------===//
 
 /// createFileID - Create a new FileID for the specified ContentCache and
@@ -501,77 +493,76 @@ const llvm::MemoryBuffer *SourceManager::getFakeBufferForRecovery() const {
 FileID SourceManager::createFileID(const ContentCache *File,
                                    SourceLocation IncludePos,
                                    SrcMgr::CharacteristicKind FileCharacter,
-                                   unsigned PreallocatedID,
-                                   unsigned Offset) {
-  if (PreallocatedID) {
-    // If we're filling in a preallocated ID, just load in the file
-    // entry and return.
-    assert(PreallocatedID < SLocEntryLoaded.size() &&
-           "Preallocate ID out-of-range");
-    assert(!SLocEntryLoaded[PreallocatedID] &&
-           "Source location entry already loaded");
-    assert(Offset && "Preallocate source location cannot have zero offset");
-    SLocEntryTable[PreallocatedID]
-      = SLocEntry::get(Offset, FileInfo::get(IncludePos, File, FileCharacter));
-    SLocEntryLoaded[PreallocatedID] = true;
-    FileID FID = FileID::get(PreallocatedID);
-    return FID;
+                                   int LoadedID, unsigned LoadedOffset) {
+  if (LoadedID < 0) {
+    assert(LoadedID != -1 && "Loading sentinel FileID");
+    unsigned Index = unsigned(-LoadedID) - 2;
+    assert(Index < LoadedSLocEntryTable.size() && "FileID out of range");
+    assert(!SLocEntryLoaded[Index] && "FileID already loaded");
+    LoadedSLocEntryTable[Index] = SLocEntry::get(LoadedOffset,
+        FileInfo::get(IncludePos, File, FileCharacter));
+    SLocEntryLoaded[Index] = true;
+    return FileID::get(LoadedID);
   }
-
-  SLocEntryTable.push_back(SLocEntry::get(NextOffset,
-                                          FileInfo::get(IncludePos, File,
-                                                        FileCharacter)));
+  LocalSLocEntryTable.push_back(SLocEntry::get(NextLocalOffset,
+                                               FileInfo::get(IncludePos, File,
+                                                             FileCharacter)));
   unsigned FileSize = File->getSize();
-  assert(NextOffset+FileSize+1 > NextOffset && "Ran out of source locations!");
-  NextOffset += FileSize+1;
+  assert(NextLocalOffset + FileSize + 1 > NextLocalOffset &&
+         NextLocalOffset + FileSize + 1 <= CurrentLoadedOffset &&
+         "Ran out of source locations!");
+  // We do a +1 here because we want a SourceLocation that means "the end of the
+  // file", e.g. for the "no newline at the end of the file" diagnostic.
+  NextLocalOffset += FileSize + 1;
 
   // Set LastFileIDLookup to the newly created file.  The next getFileID call is
   // almost guaranteed to be from that file.
-  FileID FID = FileID::get(SLocEntryTable.size()-1);
+  FileID FID = FileID::get(LocalSLocEntryTable.size()-1);
   return LastFileIDLookup = FID;
 }
 
 SourceLocation
-SourceManager::createMacroArgInstantiationLoc(SourceLocation SpellingLoc,
-                                              SourceLocation ILoc,
-                                              unsigned TokLength) {
-  InstantiationInfo II =
-    InstantiationInfo::createForMacroArg(SpellingLoc, ILoc);
-  return createInstantiationLocImpl(II, TokLength);
-}
-
-SourceLocation SourceManager::createInstantiationLoc(SourceLocation SpellingLoc,
-                                                     SourceLocation ILocStart,
-                                                     SourceLocation ILocEnd,
-                                                     unsigned TokLength,
-                                                     unsigned PreallocatedID,
-                                                     unsigned Offset) {
-  InstantiationInfo II =
-    InstantiationInfo::create(SpellingLoc, ILocStart, ILocEnd);
-  return createInstantiationLocImpl(II, TokLength, PreallocatedID, Offset);
+SourceManager::createMacroArgExpansionLoc(SourceLocation SpellingLoc,
+                                          SourceLocation ExpansionLoc,
+                                          unsigned TokLength) {
+  ExpansionInfo Info = ExpansionInfo::createForMacroArg(SpellingLoc,
+                                                        ExpansionLoc);
+  return createExpansionLocImpl(Info, TokLength);
 }
 
 SourceLocation
-SourceManager::createInstantiationLocImpl(const InstantiationInfo &II,
-                                          unsigned TokLength,
-                                          unsigned PreallocatedID,
-                                          unsigned Offset) {
-  if (PreallocatedID) {
-    // If we're filling in a preallocated ID, just load in the
-    // instantiation entry and return.
-    assert(PreallocatedID < SLocEntryLoaded.size() &&
-           "Preallocate ID out-of-range");
-    assert(!SLocEntryLoaded[PreallocatedID] &&
-           "Source location entry already loaded");
-    assert(Offset && "Preallocate source location cannot have zero offset");
-    SLocEntryTable[PreallocatedID] = SLocEntry::get(Offset, II);
-    SLocEntryLoaded[PreallocatedID] = true;
-    return SourceLocation::getMacroLoc(Offset);
+SourceManager::createExpansionLoc(SourceLocation SpellingLoc,
+                                  SourceLocation ExpansionLocStart,
+                                  SourceLocation ExpansionLocEnd,
+                                  unsigned TokLength,
+                                  int LoadedID,
+                                  unsigned LoadedOffset) {
+  ExpansionInfo Info = ExpansionInfo::create(SpellingLoc, ExpansionLocStart,
+                                             ExpansionLocEnd);
+  return createExpansionLocImpl(Info, TokLength, LoadedID, LoadedOffset);
+}
+
+SourceLocation
+SourceManager::createExpansionLocImpl(const ExpansionInfo &Info,
+                                      unsigned TokLength,
+                                      int LoadedID,
+                                      unsigned LoadedOffset) {
+  if (LoadedID < 0) {
+    assert(LoadedID != -1 && "Loading sentinel FileID");
+    unsigned Index = unsigned(-LoadedID) - 2;
+    assert(Index < LoadedSLocEntryTable.size() && "FileID out of range");
+    assert(!SLocEntryLoaded[Index] && "FileID already loaded");
+    LoadedSLocEntryTable[Index] = SLocEntry::get(LoadedOffset, Info);
+    SLocEntryLoaded[Index] = true;
+    return SourceLocation::getMacroLoc(LoadedOffset);
   }
-  SLocEntryTable.push_back(SLocEntry::get(NextOffset, II));
-  assert(NextOffset+TokLength+1 > NextOffset && "Ran out of source locations!");
-  NextOffset += TokLength+1;
-  return SourceLocation::getMacroLoc(NextOffset-(TokLength+1));
+  LocalSLocEntryTable.push_back(SLocEntry::get(NextLocalOffset, Info));
+  assert(NextLocalOffset + TokLength + 1 > NextLocalOffset &&
+         NextLocalOffset + TokLength + 1 <= CurrentLoadedOffset &&
+         "Ran out of source locations!");
+  // See createFileID for that +1.
+  NextLocalOffset += TokLength + 1;
+  return SourceLocation::getMacroLoc(NextLocalOffset - (TokLength + 1));
 }
 
 const llvm::MemoryBuffer *
@@ -602,9 +593,9 @@ void SourceManager::overrideFileContents(const FileEntry *SourceFile,
   OverriddenFiles[SourceFile] = NewFile;
 }
 
-llvm::StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const {
+StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const {
   bool MyInvalid = false;
-  const SLocEntry &SLoc = getSLocEntry(FID.ID, &MyInvalid);
+  const SLocEntry &SLoc = getSLocEntry(FID, &MyInvalid);
   if (!SLoc.isFile() || MyInvalid) {
     if (Invalid) 
       *Invalid = true;
@@ -627,18 +618,32 @@ llvm::StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const {
 // SourceLocation manipulation methods.
 //===----------------------------------------------------------------------===//
 
-/// getFileIDSlow - Return the FileID for a SourceLocation.  This is a very hot
-/// method that is used for all SourceManager queries that start with a
-/// SourceLocation object.  It is responsible for finding the entry in
-/// SLocEntryTable which contains the specified location.
+/// \brief Return the FileID for a SourceLocation.
 ///
+/// This is the cache-miss path of getFileID. Not as hot as that function, but
+/// still very important. It is responsible for finding the entry in the
+/// SLocEntry tables that contains the specified location.
 FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
   if (!SLocOffset)
     return FileID::get(0);
 
+  // Now it is time to search for the correct file. See where the SLocOffset
+  // sits in the global view and consult local or loaded buffers for it.
+  if (SLocOffset < NextLocalOffset)
+    return getFileIDLocal(SLocOffset);
+  return getFileIDLoaded(SLocOffset);
+}
+
+/// \brief Return the FileID for a SourceLocation with a low offset.
+///
+/// This function knows that the SourceLocation is in a local buffer, not a
+/// loaded one.
+FileID SourceManager::getFileIDLocal(unsigned SLocOffset) const {
+  assert(SLocOffset < NextLocalOffset && "Bad function choice");
+
   // After the first and second level caches, I see two common sorts of
-  // behavior: 1) a lot of searched FileID's are "near" the cached file location
-  // or are "near" the cached instantiation location.  2) others are just
+  // behavior: 1) a lot of searched FileID's are "near" the cached file
+  // location or are "near" the cached expansion location. 2) others are just
   // completely random and may be a very long way away.
   //
   // To handle this, we do a linear search for up to 8 steps to catch #1 quickly
@@ -649,12 +654,13 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
   // most newly created FileID.
   std::vector<SrcMgr::SLocEntry>::const_iterator I;
 
-  if (SLocEntryTable[LastFileIDLookup.ID].getOffset() < SLocOffset) {
+  if (LastFileIDLookup.ID < 0 ||
+      LocalSLocEntryTable[LastFileIDLookup.ID].getOffset() < SLocOffset) {
     // Neither loc prunes our search.
-    I = SLocEntryTable.end();
+    I = LocalSLocEntryTable.end();
   } else {
     // Perhaps it is near the file point.
-    I = SLocEntryTable.begin()+LastFileIDLookup.ID;
+    I = LocalSLocEntryTable.begin()+LastFileIDLookup.ID;
   }
 
   // Find the FileID that contains this.  "I" is an iterator that points to a
@@ -662,25 +668,12 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
   unsigned NumProbes = 0;
   while (1) {
     --I;
-    if (ExternalSLocEntries) {
-      bool Invalid = false;
-      getSLocEntry(FileID::get(I - SLocEntryTable.begin()), &Invalid);
-      if (Invalid)
-        return FileID::get(0);
-    }
-    
     if (I->getOffset() <= SLocOffset) {
-#if 0
-      printf("lin %d -> %d [%s] %d %d\n", SLocOffset,
-             I-SLocEntryTable.begin(),
-             I->isInstantiation() ? "inst" : "file",
-             LastFileIDLookup.ID,  int(SLocEntryTable.end()-I));
-#endif
-      FileID Res = FileID::get(I-SLocEntryTable.begin());
+      FileID Res = FileID::get(int(I - LocalSLocEntryTable.begin()));
 
-      // If this isn't an instantiation, remember it.  We have good locality
-      // across FileID lookups.
-      if (!I->isInstantiation())
+      // If this isn't an expansion, remember it.  We have good locality across
+      // FileID lookups.
+      if (!I->isExpansion())
         LastFileIDLookup = Res;
       NumLinearScans += NumProbes+1;
       return Res;
@@ -691,7 +684,7 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
 
   // Convert "I" back into an index.  We know that it is an entry whose index is
   // larger than the offset we are looking for.
-  unsigned GreaterIndex = I-SLocEntryTable.begin();
+  unsigned GreaterIndex = I - LocalSLocEntryTable.begin();
   // LessIndex - This is the lower bound of the range that we're searching.
   // We know that the offset corresponding to the FileID is is less than
   // SLocOffset.
@@ -700,8 +693,7 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
   while (1) {
     bool Invalid = false;
     unsigned MiddleIndex = (GreaterIndex-LessIndex)/2+LessIndex;
-    unsigned MidOffset = getSLocEntry(FileID::get(MiddleIndex), &Invalid)
-                                                                  .getOffset();
+    unsigned MidOffset = getLocalSLocEntry(MiddleIndex, &Invalid).getOffset();
     if (Invalid)
       return FileID::get(0);
     
@@ -715,18 +707,14 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
     }
 
     // If the middle index contains the value, succeed and return.
+    // FIXME: This could be made faster by using a function that's aware of
+    // being in the local area.
     if (isOffsetInFileID(FileID::get(MiddleIndex), SLocOffset)) {
-#if 0
-      printf("bin %d -> %d [%s] %d %d\n", SLocOffset,
-             I-SLocEntryTable.begin(),
-             I->isInstantiation() ? "inst" : "file",
-             LastFileIDLookup.ID, int(SLocEntryTable.end()-I));
-#endif
       FileID Res = FileID::get(MiddleIndex);
 
-      // If this isn't an instantiation, remember it.  We have good locality
+      // If this isn't a macro expansion, remember it.  We have good locality
       // across FileID lookups.
-      if (!I->isInstantiation())
+      if (!LocalSLocEntryTable[MiddleIndex].isExpansion())
         LastFileIDLookup = Res;
       NumBinaryProbes += NumProbes;
       return Res;
@@ -737,17 +725,82 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const {
   }
 }
 
+/// \brief Return the FileID for a SourceLocation with a high offset.
+///
+/// This function knows that the SourceLocation is in a loaded buffer, not a
+/// local one.
+FileID SourceManager::getFileIDLoaded(unsigned SLocOffset) const {
+  assert(SLocOffset >= CurrentLoadedOffset && "Bad function choice");
+
+  // Sanity checking, otherwise a bug may lead to hanging in release build.
+  if (SLocOffset < CurrentLoadedOffset)
+    return FileID();
+
+  // Essentially the same as the local case, but the loaded array is sorted
+  // in the other direction.
+
+  // First do a linear scan from the last lookup position, if possible.
+  unsigned I;
+  int LastID = LastFileIDLookup.ID;
+  if (LastID >= 0 || getLoadedSLocEntryByID(LastID).getOffset() < SLocOffset)
+    I = 0;
+  else
+    I = (-LastID - 2) + 1;
+
+  unsigned NumProbes;
+  for (NumProbes = 0; NumProbes < 8; ++NumProbes, ++I) {
+    // Make sure the entry is loaded!
+    const SrcMgr::SLocEntry &E = getLoadedSLocEntry(I);
+    if (E.getOffset() <= SLocOffset) {
+      FileID Res = FileID::get(-int(I) - 2);
+
+      if (!E.isExpansion())
+        LastFileIDLookup = Res;
+      NumLinearScans += NumProbes + 1;
+      return Res;
+    }
+  }
+
+  // Linear scan failed. Do the binary search. Note the reverse sorting of the
+  // table: GreaterIndex is the one where the offset is greater, which is
+  // actually a lower index!
+  unsigned GreaterIndex = I;
+  unsigned LessIndex = LoadedSLocEntryTable.size();
+  NumProbes = 0;
+  while (1) {
+    ++NumProbes;
+    unsigned MiddleIndex = (LessIndex - GreaterIndex) / 2 + GreaterIndex;
+    const SrcMgr::SLocEntry &E = getLoadedSLocEntry(MiddleIndex);
+
+    ++NumProbes;
+
+    if (E.getOffset() > SLocOffset) {
+      GreaterIndex = MiddleIndex;
+      continue;
+    }
+
+    if (isOffsetInFileID(FileID::get(-int(MiddleIndex) - 2), SLocOffset)) {
+      FileID Res = FileID::get(-int(MiddleIndex) - 2);
+      if (!E.isExpansion())
+        LastFileIDLookup = Res;
+      NumBinaryProbes += NumProbes;
+      return Res;
+    }
+
+    LessIndex = MiddleIndex;
+  }
+}
+
 SourceLocation SourceManager::
-getInstantiationLocSlowCase(SourceLocation Loc) const {
+getExpansionLocSlowCase(SourceLocation Loc) const {
   do {
     // Note: If Loc indicates an offset into a token that came from a macro
     // expansion (e.g. the 5th character of the token) we do not want to add
-    // this offset when going to the instantiation location.  The instatiation
+    // this offset when going to the expansion location.  The expansion
     // location is the macro invocation, which the offset has nothing to do
     // with.  This is unlike when we get the spelling loc, because the offset
     // directly correspond to the token whose spelling we're inspecting.
-    Loc = getSLocEntry(getFileID(Loc)).getInstantiation()
-                   .getInstantiationLocStart();
+    Loc = getSLocEntry(getFileID(Loc)).getExpansion().getExpansionLocStart();
   } while (!Loc.isFileID());
 
   return Loc;
@@ -756,23 +809,32 @@ getInstantiationLocSlowCase(SourceLocation Loc) const {
 SourceLocation SourceManager::getSpellingLocSlowCase(SourceLocation Loc) const {
   do {
     std::pair<FileID, unsigned> LocInfo = getDecomposedLoc(Loc);
-    Loc = getSLocEntry(LocInfo.first).getInstantiation().getSpellingLoc();
-    Loc = Loc.getFileLocWithOffset(LocInfo.second);
+    Loc = getSLocEntry(LocInfo.first).getExpansion().getSpellingLoc();
+    Loc = Loc.getLocWithOffset(LocInfo.second);
+  } while (!Loc.isFileID());
+  return Loc;
+}
+
+SourceLocation SourceManager::getFileLocSlowCase(SourceLocation Loc) const {
+  do {
+    if (isMacroArgExpansion(Loc))
+      Loc = getImmediateSpellingLoc(Loc);
+    else
+      Loc = getImmediateExpansionRange(Loc).first;
   } while (!Loc.isFileID());
   return Loc;
 }
 
 
 std::pair<FileID, unsigned>
-SourceManager::getDecomposedInstantiationLocSlowCase(
+SourceManager::getDecomposedExpansionLocSlowCase(
                                              const SrcMgr::SLocEntry *E) const {
-  // If this is an instantiation record, walk through all the instantiation
-  // points.
+  // If this is an expansion record, walk through all the expansion points.
   FileID FID;
   SourceLocation Loc;
   unsigned Offset;
   do {
-    Loc = E->getInstantiation().getInstantiationLocStart();
+    Loc = E->getExpansion().getExpansionLocStart();
 
     FID = getFileID(Loc);
     E = &getSLocEntry(FID);
@@ -785,16 +847,16 @@ SourceManager::getDecomposedInstantiationLocSlowCase(
 std::pair<FileID, unsigned>
 SourceManager::getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
                                                 unsigned Offset) const {
-  // If this is an instantiation record, walk through all the instantiation
-  // points.
+  // If this is an expansion record, walk through all the expansion points.
   FileID FID;
   SourceLocation Loc;
   do {
-    Loc = E->getInstantiation().getSpellingLoc();
+    Loc = E->getExpansion().getSpellingLoc();
+    Loc = Loc.getLocWithOffset(Offset);
 
     FID = getFileID(Loc);
     E = &getSLocEntry(FID);
-    Offset += Loc.getOffset()-E->getOffset();
+    Offset = Loc.getOffset()-E->getOffset();
   } while (!Loc.isFileID());
 
   return std::make_pair(FID, Offset);
@@ -807,45 +869,45 @@ SourceManager::getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
 SourceLocation SourceManager::getImmediateSpellingLoc(SourceLocation Loc) const{
   if (Loc.isFileID()) return Loc;
   std::pair<FileID, unsigned> LocInfo = getDecomposedLoc(Loc);
-  Loc = getSLocEntry(LocInfo.first).getInstantiation().getSpellingLoc();
-  return Loc.getFileLocWithOffset(LocInfo.second);
+  Loc = getSLocEntry(LocInfo.first).getExpansion().getSpellingLoc();
+  return Loc.getLocWithOffset(LocInfo.second);
 }
 
 
-/// getImmediateInstantiationRange - Loc is required to be an instantiation
-/// location.  Return the start/end of the instantiation information.
+/// getImmediateExpansionRange - Loc is required to be an expansion location.
+/// Return the start/end of the expansion information.
 std::pair<SourceLocation,SourceLocation>
-SourceManager::getImmediateInstantiationRange(SourceLocation Loc) const {
-  assert(Loc.isMacroID() && "Not an instantiation loc!");
-  const InstantiationInfo &II = getSLocEntry(getFileID(Loc)).getInstantiation();
-  return II.getInstantiationLocRange();
+SourceManager::getImmediateExpansionRange(SourceLocation Loc) const {
+  assert(Loc.isMacroID() && "Not a macro expansion loc!");
+  const ExpansionInfo &Expansion = getSLocEntry(getFileID(Loc)).getExpansion();
+  return Expansion.getExpansionLocRange();
 }
 
-/// getInstantiationRange - Given a SourceLocation object, return the
-/// range of tokens covered by the instantiation in the ultimate file.
+/// getExpansionRange - Given a SourceLocation object, return the range of
+/// tokens covered by the expansion in the ultimate file.
 std::pair<SourceLocation,SourceLocation>
-SourceManager::getInstantiationRange(SourceLocation Loc) const {
+SourceManager::getExpansionRange(SourceLocation Loc) const {
   if (Loc.isFileID()) return std::make_pair(Loc, Loc);
 
   std::pair<SourceLocation,SourceLocation> Res =
-    getImmediateInstantiationRange(Loc);
+    getImmediateExpansionRange(Loc);
 
-  // Fully resolve the start and end locations to their ultimate instantiation
+  // Fully resolve the start and end locations to their ultimate expansion
   // points.
   while (!Res.first.isFileID())
-    Res.first = getImmediateInstantiationRange(Res.first).first;
+    Res.first = getImmediateExpansionRange(Res.first).first;
   while (!Res.second.isFileID())
-    Res.second = getImmediateInstantiationRange(Res.second).second;
+    Res.second = getImmediateExpansionRange(Res.second).second;
   return Res;
 }
 
-bool SourceManager::isMacroArgInstantiation(SourceLocation Loc) const {
+bool SourceManager::isMacroArgExpansion(SourceLocation Loc) const {
   if (!Loc.isMacroID()) return false;
 
   FileID FID = getFileID(Loc);
   const SrcMgr::SLocEntry *E = &getSLocEntry(FID);
-  const SrcMgr::InstantiationInfo &II = E->getInstantiation();
-  return II.isMacroArgInstantiation();
+  const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
+  return Expansion.isMacroArgExpansion();
 }
 
 
@@ -913,10 +975,10 @@ unsigned SourceManager::getSpellingColumnNumber(SourceLocation Loc,
   return getColumnNumber(LocInfo.first, LocInfo.second, Invalid);
 }
 
-unsigned SourceManager::getInstantiationColumnNumber(SourceLocation Loc,
-                                                     bool *Invalid) const {
+unsigned SourceManager::getExpansionColumnNumber(SourceLocation Loc,
+                                                 bool *Invalid) const {
   if (isInvalid(Loc, Invalid)) return 0;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
   return getColumnNumber(LocInfo.first, LocInfo.second, Invalid);
 }
 
@@ -927,10 +989,10 @@ unsigned SourceManager::getPresumedColumnNumber(SourceLocation Loc,
 }
 
 static LLVM_ATTRIBUTE_NOINLINE void
-ComputeLineNumbers(Diagnostic &Diag, ContentCache *FI,
+ComputeLineNumbers(DiagnosticsEngine &Diag, ContentCache *FI,
                    llvm::BumpPtrAllocator &Alloc,
                    const SourceManager &SM, bool &Invalid);
-static void ComputeLineNumbers(Diagnostic &Diag, ContentCache *FI, 
+static void ComputeLineNumbers(DiagnosticsEngine &Diag, ContentCache *FI,
                                llvm::BumpPtrAllocator &Alloc,
                                const SourceManager &SM, bool &Invalid) {
   // Note that calling 'getBuffer()' may lazily page in the file.
@@ -941,7 +1003,7 @@ static void ComputeLineNumbers(Diagnostic &Diag, ContentCache *FI,
 
   // Find the file offsets of all of the *physical* source lines.  This does
   // not look at trigraphs, escaped newlines, or anything else tricky.
-  llvm::SmallVector<unsigned, 256> LineOffsets;
+  SmallVector<unsigned, 256> LineOffsets;
 
   // Line #1 starts at char 0.
   LineOffsets.push_back(0);
@@ -1108,10 +1170,10 @@ unsigned SourceManager::getSpellingLineNumber(SourceLocation Loc,
   std::pair<FileID, unsigned> LocInfo = getDecomposedSpellingLoc(Loc);
   return getLineNumber(LocInfo.first, LocInfo.second);
 }
-unsigned SourceManager::getInstantiationLineNumber(SourceLocation Loc, 
-                                                   bool *Invalid) const {
+unsigned SourceManager::getExpansionLineNumber(SourceLocation Loc,
+                                               bool *Invalid) const {
   if (isInvalid(Loc, Invalid)) return 0;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
   return getLineNumber(LocInfo.first, LocInfo.second);
 }
 unsigned SourceManager::getPresumedLineNumber(SourceLocation Loc,
@@ -1131,7 +1193,7 @@ unsigned SourceManager::getPresumedLineNumber(SourceLocation Loc,
 SrcMgr::CharacteristicKind
 SourceManager::getFileCharacteristic(SourceLocation Loc) const {
   assert(!Loc.isInvalid() && "Can't get file characteristic of invalid loc!");
-  std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
   bool Invalid = false;
   const SLocEntry &SEntry = getSLocEntry(LocInfo.first, &Invalid);
   if (Invalid || !SEntry.isFile())
@@ -1172,13 +1234,13 @@ const char *SourceManager::getBufferName(SourceLocation Loc,
 /// or GNU line marker directives.  This provides a view on the data that a
 /// user should see in diagnostics, for example.
 ///
-/// Note that a presumed location is always given as the instantiation point
-/// of an instantiation location, not at the spelling location.
+/// Note that a presumed location is always given as the expansion point of an
+/// expansion location, not at the spelling location.
 PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc) const {
   if (Loc.isInvalid()) return PresumedLoc();
 
-  // Presumed locations are always for instantiation points.
-  std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
+  // Presumed locations are always for expansion points.
+  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
 
   bool Invalid = false;
   const SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
@@ -1229,7 +1291,7 @@ PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc) const {
       // Handle virtual #include manipulation.
       if (Entry->IncludeOffset) {
         IncludeLoc = getLocForStartOfFile(LocInfo.first);
-        IncludeLoc = IncludeLoc.getFileLocWithOffset(Entry->IncludeOffset);
+        IncludeLoc = IncludeLoc.getLocWithOffset(Entry->IncludeOffset);
       }
     }
   }
@@ -1237,6 +1299,25 @@ PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc) const {
   return PresumedLoc(Filename, LineNo, ColNo, IncludeLoc);
 }
 
+/// \brief The size of the SLocEnty that \arg FID represents.
+unsigned SourceManager::getFileIDSize(FileID FID) const {
+  bool Invalid = false;
+  const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+  if (Invalid)
+    return 0;
+
+  int ID = FID.ID;
+  unsigned NextOffset;
+  if ((ID > 0 && unsigned(ID+1) == local_sloc_entry_size()))
+    NextOffset = getNextLocalOffset();
+  else if (ID+1 == -1)
+    NextOffset = MaxLoadedOffset;
+  else
+    NextOffset = getSLocEntry(FileID::get(ID+1)).getOffset();
+
+  return NextOffset - Entry.getOffset() - 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Other miscellaneous methods.
 //===----------------------------------------------------------------------===//
@@ -1259,24 +1340,36 @@ static llvm::Optional<ino_t> getActualFileInode(const FileEntry *File) {
 /// \brief Get the source location for the given file:line:col triplet.
 ///
 /// If the source file is included multiple times, the source location will
-/// be based upon the first inclusion.
-SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
-                                          unsigned Line, unsigned Col) {
+/// be based upon an arbitrary inclusion.
+SourceLocation SourceManager::translateFileLineCol(const FileEntry *SourceFile,
+                                                  unsigned Line,
+                                                  unsigned Col) const {
   assert(SourceFile && "Null source file!");
   assert(Line && Col && "Line and column should start from 1!");
 
+  FileID FirstFID = translateFile(SourceFile);
+  return translateLineCol(FirstFID, Line, Col);
+}
+
+/// \brief Get the FileID for the given file.
+///
+/// If the source file is included multiple times, the FileID will be the
+/// first inclusion.
+FileID SourceManager::translateFile(const FileEntry *SourceFile) const {
+  assert(SourceFile && "Null source file!");
+
   // Find the first file ID that corresponds to the given file.
   FileID FirstFID;
 
   // First, check the main file ID, since it is common to look for a
   // location in the main file.
   llvm::Optional<ino_t> SourceFileInode;
-  llvm::Optional<llvm::StringRef> SourceFileName;
+  llvm::Optional<StringRef> SourceFileName;
   if (!MainFileID.isInvalid()) {
     bool Invalid = false;
     const SLocEntry &MainSLoc = getSLocEntry(MainFileID, &Invalid);
     if (Invalid)
-      return SourceLocation();
+      return FileID();
     
     if (MainSLoc.isFile()) {
       const ContentCache *MainContentCache
@@ -1308,12 +1401,12 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
 
   if (FirstFID.isInvalid()) {
     // The location we're looking for isn't in the main file; look
-    // through all of the source locations.
-    for (unsigned I = 0, N = sloc_entry_size(); I != N; ++I) {
+    // through all of the local source locations.
+    for (unsigned I = 0, N = local_sloc_entry_size(); I != N; ++I) {
       bool Invalid = false;
-      const SLocEntry &SLoc = getSLocEntry(I, &Invalid);
+      const SLocEntry &SLoc = getLocalSLocEntry(I, &Invalid);
       if (Invalid)
-        return SourceLocation();
+        return FileID();
       
       if (SLoc.isFile() && 
           SLoc.getFile().getContentCache() &&
@@ -1322,6 +1415,18 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
         break;
       }
     }
+    // If that still didn't help, try the modules.
+    if (FirstFID.isInvalid()) {
+      for (unsigned I = 0, N = loaded_sloc_entry_size(); I != N; ++I) {
+        const SLocEntry &SLoc = getLoadedSLocEntry(I);
+        if (SLoc.isFile() && 
+            SLoc.getFile().getContentCache() &&
+            SLoc.getFile().getContentCache()->OrigEntry == SourceFile) {
+          FirstFID = FileID::get(-int(I) - 2);
+          break;
+        }
+      }
+    }
   }
 
   // If we haven't found what we want yet, try again, but this time stat()
@@ -1333,10 +1438,12 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
       (SourceFileInode ||
        (SourceFileInode = getActualFileInode(SourceFile)))) {
     bool Invalid = false;
-    for (unsigned I = 0, N = sloc_entry_size(); I != N; ++I) {
-      const SLocEntry &SLoc = getSLocEntry(I, &Invalid);
+    for (unsigned I = 0, N = local_sloc_entry_size(); I != N; ++I) {
+      FileID IFileID;
+      IFileID.ID = I;
+      const SLocEntry &SLoc = getSLocEntry(IFileID, &Invalid);
       if (Invalid)
-        return SourceLocation();
+        return FileID();
       
       if (SLoc.isFile()) { 
         const ContentCache *FileContentCache 
@@ -1355,20 +1462,38 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
       }
     }      
   }
-    
-  if (FirstFID.isInvalid())
+  
+  return FirstFID;
+}
+
+/// \brief Get the source location in \arg FID for the given line:col.
+/// Returns null location if \arg FID is not a file SLocEntry.
+SourceLocation SourceManager::translateLineCol(FileID FID,
+                                               unsigned Line,
+                                               unsigned Col) const {
+  if (FID.isInvalid())
     return SourceLocation();
 
+  bool Invalid = false;
+  const SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+  if (Invalid)
+    return SourceLocation();
+  
+  if (!Entry.isFile())
+    return SourceLocation();
+
+  SourceLocation FileLoc = SourceLocation::getFileLoc(Entry.getOffset());
+
   if (Line == 1 && Col == 1)
-    return getLocForStartOfFile(FirstFID);
+    return FileLoc;
 
   ContentCache *Content
-    = const_cast<ContentCache *>(getOrCreateContentCache(SourceFile));
+    = const_cast<ContentCache *>(Entry.getFile().getContentCache());
   if (!Content)
     return SourceLocation();
     
   // If this is the first use of line information for this buffer, compute the
-  /// SourceLineCache for it on demand.
+  // SourceLineCache for it on demand.
   if (Content->SourceLineCache == 0) {
     bool MyInvalid = false;
     ComputeLineNumbers(Diag, Content, ContentCacheAlloc, *this, MyInvalid);
@@ -1380,33 +1505,150 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
     unsigned Size = Content->getBuffer(Diag, *this)->getBufferSize();
     if (Size > 0)
       --Size;
-    return getLocForStartOfFile(FirstFID).getFileLocWithOffset(Size);
+    return FileLoc.getLocWithOffset(Size);
   }
 
   unsigned FilePos = Content->SourceLineCache[Line - 1];
   const char *Buf = Content->getBuffer(Diag, *this)->getBufferStart() + FilePos;
   unsigned BufLength = Content->getBuffer(Diag, *this)->getBufferEnd() - Buf;
+  if (BufLength == 0)
+    return FileLoc.getLocWithOffset(FilePos);
+
   unsigned i = 0;
 
   // Check that the given column is valid.
   while (i < BufLength-1 && i < Col-1 && Buf[i] != '\n' && Buf[i] != '\r')
     ++i;
   if (i < Col-1)
-    return getLocForStartOfFile(FirstFID).getFileLocWithOffset(FilePos + i);
+    return FileLoc.getLocWithOffset(FilePos + i);
 
-  return getLocForStartOfFile(FirstFID).getFileLocWithOffset(FilePos + Col - 1);
+  return FileLoc.getLocWithOffset(FilePos + Col - 1);
 }
 
-/// Given a decomposed source location, move it up the include/instantiation
-/// stack to the parent source location.  If this is possible, return the
-/// decomposed version of the parent in Loc and return false.  If Loc is the
-/// top-level entry, return true and don't modify it.
+/// \brief Compute a map of macro argument chunks to their expanded source
+/// location. Chunks that are not part of a macro argument will map to an
+/// invalid source location. e.g. if a file contains one macro argument at
+/// offset 100 with length 10, this is how the map will be formed:
+///     0   -> SourceLocation()
+///     100 -> Expanded macro arg location
+///     110 -> SourceLocation()
+void SourceManager::computeMacroArgsCache(MacroArgsMap *&CachePtr,
+                                          FileID FID) const {
+  assert(!FID.isInvalid());
+  assert(!CachePtr);
+
+  CachePtr = new MacroArgsMap();
+  MacroArgsMap &MacroArgsCache = *CachePtr;
+  // Initially no macro argument chunk is present.
+  MacroArgsCache.insert(std::make_pair(0, SourceLocation()));
+
+  int ID = FID.ID;
+  while (1) {
+    ++ID;
+    // Stop if there are no more FileIDs to check.
+    if (ID > 0) {
+      if (unsigned(ID) >= local_sloc_entry_size())
+        return;
+    } else if (ID == -1) {
+      return;
+    }
+
+    const SrcMgr::SLocEntry &Entry = getSLocEntryByID(ID);
+    if (Entry.isFile()) {
+      SourceLocation IncludeLoc = Entry.getFile().getIncludeLoc();
+      if (IncludeLoc.isInvalid())
+        continue;
+      if (!isInFileID(IncludeLoc, FID))
+        return; // No more files/macros that may be "contained" in this file.
+
+      // Skip the files/macros of the #include'd file, we only care about macros
+      // that lexed macro arguments from our file.
+      if (Entry.getFile().NumCreatedFIDs)
+        ID += Entry.getFile().NumCreatedFIDs - 1/*because of next ++ID*/;
+      continue;
+    }
+
+    if (!Entry.getExpansion().isMacroArgExpansion())
+      continue;
+ 
+    SourceLocation SpellLoc =
+        getSpellingLoc(Entry.getExpansion().getSpellingLoc());
+    unsigned BeginOffs;
+    if (!isInFileID(SpellLoc, FID, &BeginOffs))
+      return; // No more files/macros that may be "contained" in this file.
+    unsigned EndOffs = BeginOffs + getFileIDSize(FileID::get(ID));
+
+    // Add a new chunk for this macro argument. A previous macro argument chunk
+    // may have been lexed again, so e.g. if the map is
+    //     0   -> SourceLocation()
+    //     100 -> Expanded loc #1
+    //     110 -> SourceLocation()
+    // and we found a new macro FileID that lexed from offet 105 with length 3,
+    // the new map will be:
+    //     0   -> SourceLocation()
+    //     100 -> Expanded loc #1
+    //     105 -> Expanded loc #2
+    //     108 -> Expanded loc #1
+    //     110 -> SourceLocation()
+    //
+    // Since re-lexed macro chunks will always be the same size or less of
+    // previous chunks, we only need to find where the ending of the new macro
+    // chunk is mapped to and update the map with new begin/end mappings.
+
+    MacroArgsMap::iterator I = MacroArgsCache.upper_bound(EndOffs);
+    --I;
+    SourceLocation EndOffsMappedLoc = I->second;
+    MacroArgsCache[BeginOffs] = SourceLocation::getMacroLoc(Entry.getOffset());
+    MacroArgsCache[EndOffs] = EndOffsMappedLoc;
+  }
+}
+
+/// \brief If \arg Loc points inside a function macro argument, the returned
+/// location will be the macro location in which the argument was expanded.
+/// If a macro argument is used multiple times, the expanded location will
+/// be at the first expansion of the argument.
+/// e.g.
+///   MY_MACRO(foo);
+///             ^
+/// Passing a file location pointing at 'foo', will yield a macro location
+/// where 'foo' was expanded into.
+SourceLocation
+SourceManager::getMacroArgExpandedLocation(SourceLocation Loc) const {
+  if (Loc.isInvalid() || !Loc.isFileID())
+    return Loc;
+
+  FileID FID;
+  unsigned Offset;
+  llvm::tie(FID, Offset) = getDecomposedLoc(Loc);
+  if (FID.isInvalid())
+    return Loc;
+
+  MacroArgsMap *&MacroArgsCache = MacroArgsCacheMap[FID];
+  if (!MacroArgsCache)
+    computeMacroArgsCache(MacroArgsCache, FID);
+
+  assert(!MacroArgsCache->empty());
+  MacroArgsMap::iterator I = MacroArgsCache->upper_bound(Offset);
+  --I;
+  
+  unsigned MacroArgBeginOffs = I->first;
+  SourceLocation MacroArgExpandedLoc = I->second;
+  if (MacroArgExpandedLoc.isValid())
+    return MacroArgExpandedLoc.getLocWithOffset(Offset - MacroArgBeginOffs);
+
+  return Loc;
+}
+
+/// Given a decomposed source location, move it up the include/expansion stack
+/// to the parent source location.  If this is possible, return the decomposed
+/// version of the parent in Loc and return false.  If Loc is the top-level
+/// entry, return true and don't modify it.
 static bool MoveUpIncludeHierarchy(std::pair<FileID, unsigned> &Loc,
                                    const SourceManager &SM) {
   SourceLocation UpperLoc;
   const SrcMgr::SLocEntry &Entry = SM.getSLocEntry(Loc.first);
-  if (Entry.isInstantiation())
-    UpperLoc = Entry.getInstantiation().getInstantiationLocStart();
+  if (Entry.isExpansion())
+    UpperLoc = Entry.getExpansion().getExpansionLocEnd();
   else
     UpperLoc = Entry.getFile().getIncludeLoc();
   
@@ -1427,11 +1669,6 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   if (LHS == RHS)
     return false;
 
-  // If both locations are macro instantiations, the order of their offsets
-  // reflect the order that the tokens, pointed to by these locations, were
-  // instantiated (during parsing each token that is instantiated by a macro,
-  // expands the SLocEntries).
-
   std::pair<FileID, unsigned> LOffs = getDecomposedLoc(LHS);
   std::pair<FileID, unsigned> ROffs = getDecomposedLoc(RHS);
 
@@ -1445,41 +1682,28 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
     return IsBeforeInTUCache.getCachedResult(LOffs.second, ROffs.second);
 
   // Okay, we missed in the cache, start updating the cache for this query.
-  IsBeforeInTUCache.setQueryFIDs(LOffs.first, ROffs.first);
+  IsBeforeInTUCache.setQueryFIDs(LOffs.first, ROffs.first,
+                          /*isLFIDBeforeRFID=*/LOffs.first.ID < ROffs.first.ID);
 
-  // "Traverse" the include/instantiation stacks of both locations and try to
-  // find a common "ancestor".  FileIDs build a tree-like structure that
-  // reflects the #include hierarchy, and this algorithm needs to find the
-  // nearest common ancestor between the two locations.  For example, if you
-  // have a.c that includes b.h and c.h, and are comparing a location in b.h to
-  // a location in c.h, we need to find that their nearest common ancestor is
-  // a.c, and compare the locations of the two #includes to find their relative
-  // ordering.
-  //
-  // SourceManager assigns FileIDs in order of parsing.  This means that an
-  // includee always has a larger FileID than an includer.  While you might
-  // think that we could just compare the FileID's here, that doesn't work to
-  // compare a point at the end of a.c with a point within c.h.  Though c.h has
-  // a larger FileID, we have to compare the include point of c.h to the
-  // location in a.c.
-  //
-  // Despite not being able to directly compare FileID's, we can tell that a
-  // larger FileID is necessarily more deeply nested than a lower one and use
-  // this information to walk up the tree to the nearest common ancestor.
+  // We need to find the common ancestor. The only way of doing this is to
+  // build the complete include chain for one and then walking up the chain
+  // of the other looking for a match.
+  // We use a map from FileID to Offset to store the chain. Easier than writing
+  // a custom set hash info that only depends on the first part of a pair.
+  typedef llvm::DenseMap<FileID, unsigned> LocSet;
+  LocSet LChain;
   do {
-    // If LOffs is larger than ROffs, then LOffs must be more deeply nested than
-    // ROffs, walk up the #include chain.
-    if (LOffs.first.ID > ROffs.first.ID) {
-      if (MoveUpIncludeHierarchy(LOffs, *this))
-        break; // We reached the top.
-      
-    } else {
-      // Otherwise, ROffs is larger than LOffs, so ROffs must be more deeply
-      // nested than LOffs, walk up the #include chain.
-      if (MoveUpIncludeHierarchy(ROffs, *this))
-        break; // We reached the top.
-    }
-  } while (LOffs.first != ROffs.first);
+    LChain.insert(LOffs);
+    // We catch the case where LOffs is in a file included by ROffs and
+    // quit early. The other way round unfortunately remains suboptimal.
+  } while (LOffs.first != ROffs.first && !MoveUpIncludeHierarchy(LOffs, *this));
+  LocSet::iterator I;
+  while((I = LChain.find(ROffs.first)) == LChain.end()) {
+    if (MoveUpIncludeHierarchy(ROffs, *this))
+      break; // Met at topmost file.
+  }
+  if (I != LChain.end())
+    LOffs = *I;
 
   // If we exited because we found a nearest common ancestor, compare the
   // locations within the common file and cache them.
@@ -1488,26 +1712,21 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
     return IsBeforeInTUCache.getCachedResult(LOffs.second, ROffs.second);
   }
 
-  // There is no common ancestor, most probably because one location is in the
-  // predefines buffer or an AST file.
-  // FIXME: We should rearrange the external interface so this simply never
-  // happens; it can't conceptually happen. Also see PR5662.
-  IsBeforeInTUCache.setQueryFIDs(FileID(), FileID()); // Don't try caching.
-
-  // Zip both entries up to the top level record.
-  while (!MoveUpIncludeHierarchy(LOffs, *this)) /*empty*/;
-  while (!MoveUpIncludeHierarchy(ROffs, *this)) /*empty*/;
-  
-  // If exactly one location is a memory buffer, assume it precedes the other.
-  
-  // Strip off macro instantation locations, going up to the top-level File
-  // SLocEntry.
-  bool LIsMB = getFileEntryForID(LOffs.first) == 0;
-  bool RIsMB = getFileEntryForID(ROffs.first) == 0;
-  if (LIsMB != RIsMB)
-    return LIsMB;
-
-  // Otherwise, just assume FileIDs were created in order.
+  // This can happen if a location is in a built-ins buffer.
+  // But see PR5662.
+  // Clear the lookup cache, it depends on a common location.
+  IsBeforeInTUCache.clear();
+  bool LIsBuiltins = strcmp("<built-in>",
+                            getBuffer(LOffs.first)->getBufferIdentifier()) == 0;
+  bool RIsBuiltins = strcmp("<built-in>",
+                            getBuffer(ROffs.first)->getBufferIdentifier()) == 0;
+  // built-in is before non-built-in
+  if (LIsBuiltins != RIsBuiltins)
+    return LIsBuiltins;
+  assert(LIsBuiltins && RIsBuiltins &&
+         "Non-built-in locations must be rooted in the main file");
+  // Both are in built-in buffers, but from different files. We just claim that
+  // lower IDs come first.
   return LOffs.first < ROffs.first;
 }
 
@@ -1517,20 +1736,26 @@ void SourceManager::PrintStats() const {
   llvm::errs() << "\n*** Source Manager Stats:\n";
   llvm::errs() << FileInfos.size() << " files mapped, " << MemBufferInfos.size()
                << " mem buffers mapped.\n";
-  llvm::errs() << SLocEntryTable.size() << " SLocEntry's allocated ("
-               << SLocEntryTable.capacity()*sizeof(SrcMgr::SLocEntry)
+  llvm::errs() << LocalSLocEntryTable.size() << " local SLocEntry's allocated ("
+               << llvm::capacity_in_bytes(LocalSLocEntryTable)
                << " bytes of capacity), "
-               << NextOffset << "B of Sloc address space used.\n";
-
+               << NextLocalOffset << "B of Sloc address space used.\n";
+  llvm::errs() << LoadedSLocEntryTable.size()
+               << " loaded SLocEntries allocated, "
+               << MaxLoadedOffset - CurrentLoadedOffset
+               << "B of Sloc address space used.\n";
+  
   unsigned NumLineNumsComputed = 0;
   unsigned NumFileBytesMapped = 0;
   for (fileinfo_iterator I = fileinfo_begin(), E = fileinfo_end(); I != E; ++I){
     NumLineNumsComputed += I->second->SourceLineCache != 0;
     NumFileBytesMapped  += I->second->getSizeBytesMapped();
   }
+  unsigned NumMacroArgsComputed = MacroArgsCacheMap.size();
 
   llvm::errs() << NumFileBytesMapped << " bytes of files mapped, "
-               << NumLineNumsComputed << " files with line #'s computed.\n";
+               << NumLineNumsComputed << " files with line #'s computed, "
+               << NumMacroArgsComputed << " files with macro args computed.\n";
   llvm::errs() << "FileID scans: " << NumLinearScans << " linear, "
                << NumBinaryProbes << " binary.\n";
 }
@@ -1557,3 +1782,11 @@ SourceManager::MemoryBufferSizes SourceManager::getMemoryBufferSizes() const {
   return MemoryBufferSizes(malloc_bytes, mmap_bytes);
 }
 
+size_t SourceManager::getDataStructureSizes() const {
+  return llvm::capacity_in_bytes(MemBufferInfos)
+    + llvm::capacity_in_bytes(LocalSLocEntryTable)
+    + llvm::capacity_in_bytes(LoadedSLocEntryTable)
+    + llvm::capacity_in_bytes(SLocEntryLoaded)
+    + llvm::capacity_in_bytes(FileInfos)
+    + llvm::capacity_in_bytes(OverriddenFiles);
+}
diff --git a/contrib/llvm/tools/clang/lib/Basic/TargetInfo.cpp b/contrib/llvm/tools/clang/lib/Basic/TargetInfo.cpp
index 30a9bdb31774..593db2b901c7 100644
--- a/contrib/llvm/tools/clang/lib/Basic/TargetInfo.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/TargetInfo.cpp
@@ -16,6 +16,7 @@
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <cctype>
 #include <cstdlib>
 using namespace clang;
@@ -33,6 +34,8 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   IntWidth = IntAlign = 32;
   LongWidth = LongAlign = 32;
   LongLongWidth = LongLongAlign = 64;
+  HalfWidth = 16;
+  HalfAlign = 16;
   FloatWidth = 32;
   FloatAlign = 32;
   DoubleWidth = 64;
@@ -41,6 +44,7 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   LongDoubleAlign = 64;
   LargeArrayMinWidth = 0;
   LargeArrayAlign = 0;
+  MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 0;
   SizeType = UnsignedLong;
   PtrDiffType = SignedLong;
   IntMaxType = SignedLongLong;
@@ -53,6 +57,9 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   Int64Type = SignedLongLong;
   SigAtomicType = SignedInt;
   UseBitFieldTypeAlignment = true;
+  UseZeroLengthBitfieldAlignment = false;
+  ZeroLengthBitfieldBoundary = 0;
+  HalfFormat = &llvm::APFloat::IEEEhalf;
   FloatFormat = &llvm::APFloat::IEEEsingle;
   DoubleFormat = &llvm::APFloat::IEEEdouble;
   LongDoubleFormat = &llvm::APFloat::IEEEdouble;
@@ -60,6 +67,8 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
                       "i64:64:64-f32:32:32-f64:64:64-n32";
   UserLabelPrefix = "_";
   MCountName = "mcount";
+  RegParmMax = 0;
+  SSERegParmMax = 0;
   HasAlignMac68kSupport = false;
 
   // Default to no types using fpret.
@@ -83,7 +92,7 @@ TargetInfo::~TargetInfo() {}
 /// For example, SignedShort -> "short".
 const char *TargetInfo::getTypeName(IntType T) {
   switch (T) {
-  default: assert(0 && "not an integer!");
+  default: llvm_unreachable("not an integer!");
   case SignedShort:      return "short";
   case UnsignedShort:    return "unsigned short";
   case SignedInt:        return "int";
@@ -99,7 +108,7 @@ const char *TargetInfo::getTypeName(IntType T) {
 /// integer type enum. For example, SignedLong -> "L".
 const char *TargetInfo::getTypeConstantSuffix(IntType T) {
   switch (T) {
-  default: assert(0 && "not an integer!");
+  default: llvm_unreachable("not an integer!");
   case SignedShort:
   case SignedInt:        return "";
   case SignedLong:       return "L";
@@ -115,7 +124,7 @@ const char *TargetInfo::getTypeConstantSuffix(IntType T) {
 /// enum. For example, SignedInt -> getIntWidth().
 unsigned TargetInfo::getTypeWidth(IntType T) const {
   switch (T) {
-  default: assert(0 && "not an integer!");
+  default: llvm_unreachable("not an integer!");
   case SignedShort:
   case UnsignedShort:    return getShortWidth();
   case SignedInt:
@@ -131,7 +140,7 @@ unsigned TargetInfo::getTypeWidth(IntType T) const {
 /// enum. For example, SignedInt -> getIntAlign().
 unsigned TargetInfo::getTypeAlign(IntType T) const {
   switch (T) {
-  default: assert(0 && "not an integer!");
+  default: llvm_unreachable("not an integer!");
   case SignedShort:
   case UnsignedShort:    return getShortAlign();
   case SignedInt:
@@ -147,7 +156,7 @@ unsigned TargetInfo::getTypeAlign(IntType T) const {
 /// the type is signed; false otherwise.
 bool TargetInfo::isTypeSigned(IntType T) {
   switch (T) {
-  default: assert(0 && "not an integer!");
+  default: llvm_unreachable("not an integer!");
   case SignedShort:
   case SignedInt:
   case SignedLong:
@@ -174,7 +183,7 @@ void TargetInfo::setForcedLangOptions(LangOptions &Opts) {
 //===----------------------------------------------------------------------===//
 
 
-static llvm::StringRef removeGCCRegisterPrefix(llvm::StringRef Name) {
+static StringRef removeGCCRegisterPrefix(StringRef Name) {
   if (Name[0] == '%' || Name[0] == '#')
     Name = Name.substr(1);
 
@@ -184,7 +193,7 @@ static llvm::StringRef removeGCCRegisterPrefix(llvm::StringRef Name) {
 /// isValidClobber - Returns whether the passed in string is
 /// a valid clobber in an inline asm statement. This is used by
 /// Sema.
-bool TargetInfo::isValidClobber(llvm::StringRef Name) const {
+bool TargetInfo::isValidClobber(StringRef Name) const {
   return (isValidGCCRegisterName(Name) ||
 	  Name == "memory" || Name == "cc");
 }
@@ -192,7 +201,7 @@ bool TargetInfo::isValidClobber(llvm::StringRef Name) const {
 /// isValidGCCRegisterName - Returns whether the passed in string
 /// is a valid register name according to GCC. This is used by Sema for
 /// inline asm statements.
-bool TargetInfo::isValidGCCRegisterName(llvm::StringRef Name) const {
+bool TargetInfo::isValidGCCRegisterName(StringRef Name) const {
   if (Name.empty())
     return false;
 
@@ -248,8 +257,8 @@ bool TargetInfo::isValidGCCRegisterName(llvm::StringRef Name) const {
   return false;
 }
 
-llvm::StringRef
-TargetInfo::getNormalizedGCCRegisterName(llvm::StringRef Name) const {
+StringRef
+TargetInfo::getNormalizedGCCRegisterName(StringRef Name) const {
   assert(isValidGCCRegisterName(Name) && "Invalid register passed in");
 
   // Get rid of any register prefix.
diff --git a/contrib/llvm/tools/clang/lib/Basic/Targets.cpp b/contrib/llvm/tools/clang/lib/Basic/Targets.cpp
index 3518ea6f7986..6d0d7bb1dd6e 100644
--- a/contrib/llvm/tools/clang/lib/Basic/Targets.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/Targets.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Type.h"
 #include <algorithm>
 using namespace clang;
@@ -37,7 +38,7 @@ using namespace clang;
 /// DefineStd - Define a macro name and standard variants.  For example if
 /// MacroName is "unix", then this will define "__unix", "__unix__", and "unix"
 /// when in GNU mode.
-static void DefineStd(MacroBuilder &Builder, llvm::StringRef MacroName,
+static void DefineStd(MacroBuilder &Builder, StringRef MacroName,
                       const LangOptions &Opts) {
   assert(MacroName[0] != '_' && "Identifier should be in the user's namespace");
 
@@ -77,7 +78,7 @@ class OSTargetInfo : public TgtInfo {
 
 static void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
                              const llvm::Triple &Triple,
-                             llvm::StringRef &PlatformName,
+                             StringRef &PlatformName,
                              VersionTuple &PlatformMinVersion) {
   Builder.defineMacro("__APPLE_CC__", "5621");
   Builder.defineMacro("__APPLE__");
@@ -89,7 +90,7 @@ static void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
     Builder.defineMacro("__weak", "__attribute__((objc_gc(weak)))");
 
     // Darwin defines __strong even in C mode (just to nothing).
-    if (Opts.getGCMode() != LangOptions::NonGC)
+    if (Opts.getGC() != LangOptions::NonGC)
       Builder.defineMacro("__strong", "__attribute__((objc_gc(strong)))");
     else
       Builder.defineMacro("__strong", "");
@@ -146,6 +147,14 @@ static void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
     }
   }
 
+  // If -ccc-host-triple arch-pc-win32-macho option specified, we're
+  // generating code for Win32 ABI. No need to emit
+  // __ENVIRONMENT_XX_OS_VERSION_MIN_REQUIRED__.
+  if (PlatformName == "win32") {
+    PlatformMinVersion = VersionTuple(Maj, Min, Rev);
+    return;
+  }
+
   // Set the appropriate OS version define.
   if (PlatformName == "ios") {
     assert(Maj < 10 && Min < 100 && Rev < 100 && "Invalid version!");
@@ -194,9 +203,9 @@ class DarwinTargetInfo : public OSTargetInfo<Target> {
       this->MCountName = "\01mcount";
     }
 
-  virtual std::string isValidSectionSpecifier(llvm::StringRef SR) const {
+  virtual std::string isValidSectionSpecifier(StringRef SR) const {
     // Let MCSectionMachO validate this.
-    llvm::StringRef Segment, Section;
+    StringRef Segment, Section;
     unsigned TAA, StubSize;
     bool HasTAA;
     return llvm::MCSectionMachO::ParseSectionSpecifier(SR, Segment, Section,
@@ -238,11 +247,12 @@ class FreeBSDTargetInfo : public OSTargetInfo<Target> {
                             MacroBuilder &Builder) const {
     // FreeBSD defines; list based off of gcc output
 
-    // FIXME: Move version number handling to llvm::Triple.
-    llvm::StringRef Release = Triple.getOSName().substr(strlen("freebsd"), 1);
+    unsigned Release = Triple.getOSMajorVersion();
+    if (Release == 0U)
+      Release = 8U;
 
-    Builder.defineMacro("__FreeBSD__", Release);
-    Builder.defineMacro("__FreeBSD_cc_version", Release + "00001");
+    Builder.defineMacro("__FreeBSD__", Twine(Release));
+    Builder.defineMacro("__FreeBSD_cc_version", Twine(Release * 100000U + 1U));
     Builder.defineMacro("__KPRINTF_ATTRIBUTE__");
     DefineStd(Builder, "unix", Opts);
     Builder.defineMacro("__ELF__");
@@ -495,9 +505,9 @@ class WindowsTargetInfo : public OSTargetInfo<Target> {
       Builder.defineMacro("_MT");
 
     if (Opts.MSCVersion != 0)
-      Builder.defineMacro("_MSC_VER", llvm::Twine(Opts.MSCVersion));
+      Builder.defineMacro("_MSC_VER", Twine(Opts.MSCVersion));
 
-    if (Opts.Microsoft) {
+    if (Opts.MicrosoftExt) {
       Builder.defineMacro("_MSC_EXTENSIONS");
 
       if (Opts.CPlusPlus0x) {
@@ -869,13 +879,41 @@ class DarwinPPC64TargetInfo :
 } // end anonymous namespace.
 
 namespace {
+  static const unsigned PTXAddrSpaceMap[] = {
+    0,    // opencl_global
+    4,    // opencl_local
+    1     // opencl_constant
+  };
   class PTXTargetInfo : public TargetInfo {
     static const char * const GCCRegNames[];
     static const Builtin::Info BuiltinInfo[];
+    std::vector<llvm::StringRef> AvailableFeatures;
   public:
     PTXTargetInfo(const std::string& triple) : TargetInfo(triple) {
       TLSSupported = false;
       LongWidth = LongAlign = 64;
+      AddrSpaceMap = &PTXAddrSpaceMap;
+      // Define available target features
+      // These must be defined in sorted order!      
+      AvailableFeatures.push_back("compute10");
+      AvailableFeatures.push_back("compute11");
+      AvailableFeatures.push_back("compute12");
+      AvailableFeatures.push_back("compute13");
+      AvailableFeatures.push_back("compute20");
+      AvailableFeatures.push_back("double");
+      AvailableFeatures.push_back("no-fma");
+      AvailableFeatures.push_back("ptx20");
+      AvailableFeatures.push_back("ptx21");
+      AvailableFeatures.push_back("ptx22");
+      AvailableFeatures.push_back("ptx23");
+      AvailableFeatures.push_back("sm10");
+      AvailableFeatures.push_back("sm11");
+      AvailableFeatures.push_back("sm12");
+      AvailableFeatures.push_back("sm13");
+      AvailableFeatures.push_back("sm20");
+      AvailableFeatures.push_back("sm21");
+      AvailableFeatures.push_back("sm22");
+      AvailableFeatures.push_back("sm23");
     }
     virtual void getTargetDefines(const LangOptions &Opts,
                                   MacroBuilder &Builder) const {
@@ -908,6 +946,10 @@ namespace {
       // FIXME: implement
       return "typedef char* __builtin_va_list;";
     }
+
+    virtual bool setFeatureEnabled(llvm::StringMap<bool> &Features,
+                                   const std::string &Name,
+                                   bool Enabled) const;
   };
 
   const Builtin::Info PTXTargetInfo::BuiltinInfo[] = {
@@ -927,6 +969,17 @@ namespace {
     NumNames = llvm::array_lengthof(GCCRegNames);
   }
 
+  bool PTXTargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
+                                        const std::string &Name,
+                                        bool Enabled) const {
+    if(std::binary_search(AvailableFeatures.begin(), AvailableFeatures.end(),
+                          Name)) {
+      Features[Name] = Enabled;
+      return true;
+    } else {
+      return false;
+    }
+  }
 
   class PTX32TargetInfo : public PTXTargetInfo {
   public:
@@ -1119,10 +1172,134 @@ class X86TargetInfo : public TargetInfo {
   bool HasAES;
   bool HasAVX;
 
+  /// \brief Enumeration of all of the X86 CPUs supported by Clang.
+  ///
+  /// Each enumeration represents a particular CPU supported by Clang. These
+  /// loosely correspond to the options passed to '-march' or '-mtune' flags.
+  enum CPUKind {
+    CK_Generic,
+
+    /// \name i386
+    /// i386-generation processors.
+    //@{
+    CK_i386,
+    //@}
+
+    /// \name i486
+    /// i486-generation processors.
+    //@{
+    CK_i486,
+    CK_WinChipC6,
+    CK_WinChip2,
+    CK_C3,
+    //@}
+
+    /// \name i586
+    /// i586-generation processors, P5 microarchitecture based.
+    //@{
+    CK_i586,
+    CK_Pentium,
+    CK_PentiumMMX,
+    //@}
+
+    /// \name i686
+    /// i686-generation processors, P6 / Pentium M microarchitecture based.
+    //@{
+    CK_i686,
+    CK_PentiumPro,
+    CK_Pentium2,
+    CK_Pentium3,
+    CK_Pentium3M,
+    CK_PentiumM,
+    CK_C3_2,
+
+    /// This enumerator is a bit odd, as GCC no longer accepts -march=yonah.
+    /// Clang however has some logic to suport this.
+    // FIXME: Warn, deprecate, and potentially remove this.
+    CK_Yonah,
+    //@}
+
+    /// \name Netburst
+    /// Netburst microarchitecture based processors.
+    //@{
+    CK_Pentium4,
+    CK_Pentium4M,
+    CK_Prescott,
+    CK_Nocona,
+    //@}
+
+    /// \name Core
+    /// Core microarchitecture based processors.
+    //@{
+    CK_Core2,
+
+    /// This enumerator, like \see CK_Yonah, is a bit odd. It is another
+    /// codename which GCC no longer accepts as an option to -march, but Clang
+    /// has some logic for recognizing it.
+    // FIXME: Warn, deprecate, and potentially remove this.
+    CK_Penryn,
+    //@}
+
+    /// \name Atom
+    /// Atom processors
+    //@{
+    CK_Atom,
+    //@}
+
+    /// \name Nehalem
+    /// Nehalem microarchitecture based processors.
+    //@{
+    CK_Corei7,
+    CK_Corei7AVX,
+    CK_CoreAVXi,
+    //@}
+
+    /// \name K6
+    /// K6 architecture processors.
+    //@{
+    CK_K6,
+    CK_K6_2,
+    CK_K6_3,
+    //@}
+
+    /// \name K7
+    /// K7 architecture processors.
+    //@{
+    CK_Athlon,
+    CK_AthlonThunderbird,
+    CK_Athlon4,
+    CK_AthlonXP,
+    CK_AthlonMP,
+    //@}
+
+    /// \name K8
+    /// K8 architecture processors.
+    //@{
+    CK_Athlon64,
+    CK_Athlon64SSE3,
+    CK_AthlonFX,
+    CK_K8,
+    CK_K8SSE3,
+    CK_Opteron,
+    CK_OpteronSSE3,
+
+    /// This specification is deprecated and will be removed in the future.
+    /// Users should prefer \see CK_K8.
+    // FIXME: Warn on this when the CPU is set to it.
+    CK_x86_64,
+    //@}
+
+    /// \name Geode
+    /// Geode processors.
+    //@{
+    CK_Geode
+    //@}
+  } CPU;
+
 public:
   X86TargetInfo(const std::string& triple)
     : TargetInfo(triple), SSELevel(NoSSE), MMX3DNowLevel(NoMMX3DNow),
-      HasAES(false), HasAVX(false) {
+      HasAES(false), HasAVX(false), CPU(CK_Generic) {
     LongDoubleFormat = &llvm::APFloat::x87DoubleExtended;
   }
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
@@ -1156,16 +1333,122 @@ class X86TargetInfo : public TargetInfo {
   virtual bool setFeatureEnabled(llvm::StringMap<bool> &Features,
                                  const std::string &Name,
                                  bool Enabled) const;
-  virtual void getDefaultFeatures(const std::string &CPU,
-                                  llvm::StringMap<bool> &Features) const;
+  virtual void getDefaultFeatures(llvm::StringMap<bool> &Features) const;
   virtual void HandleTargetFeatures(std::vector<std::string> &Features);
   virtual const char* getABI() const {
     return MMX3DNowLevel == NoMMX3DNow ? "no-mmx" : "";
   }
+  virtual bool setCPU(const std::string &Name) {
+    CPU = llvm::StringSwitch<CPUKind>(Name)
+      .Case("i386", CK_i386)
+      .Case("i486", CK_i486)
+      .Case("winchip-c6", CK_WinChipC6)
+      .Case("winchip2", CK_WinChip2)
+      .Case("c3", CK_C3)
+      .Case("i586", CK_i586)
+      .Case("pentium", CK_Pentium)
+      .Case("pentium-mmx", CK_PentiumMMX)
+      .Case("i686", CK_i686)
+      .Case("pentiumpro", CK_PentiumPro)
+      .Case("pentium2", CK_Pentium2)
+      .Case("pentium3", CK_Pentium3)
+      .Case("pentium3m", CK_Pentium3M)
+      .Case("pentium-m", CK_PentiumM)
+      .Case("c3-2", CK_C3_2)
+      .Case("yonah", CK_Yonah)
+      .Case("pentium4", CK_Pentium4)
+      .Case("pentium4m", CK_Pentium4M)
+      .Case("prescott", CK_Prescott)
+      .Case("nocona", CK_Nocona)
+      .Case("core2", CK_Core2)
+      .Case("penryn", CK_Penryn)
+      .Case("atom", CK_Atom)
+      .Case("corei7", CK_Corei7)
+      .Case("corei7-avx", CK_Corei7AVX)
+      .Case("core-avx-i", CK_CoreAVXi)
+      .Case("k6", CK_K6)
+      .Case("k6-2", CK_K6_2)
+      .Case("k6-3", CK_K6_3)
+      .Case("athlon", CK_Athlon)
+      .Case("athlon-tbird", CK_AthlonThunderbird)
+      .Case("athlon-4", CK_Athlon4)
+      .Case("athlon-xp", CK_AthlonXP)
+      .Case("athlon-mp", CK_AthlonMP)
+      .Case("athlon64", CK_Athlon64)
+      .Case("athlon64-sse3", CK_Athlon64SSE3)
+      .Case("athlon-fx", CK_AthlonFX)
+      .Case("k8", CK_K8)
+      .Case("k8-sse3", CK_K8SSE3)
+      .Case("opteron", CK_Opteron)
+      .Case("opteron-sse3", CK_OpteronSSE3)
+      .Case("x86-64", CK_x86_64)
+      .Case("geode", CK_Geode)
+      .Default(CK_Generic);
+
+    // Perform any per-CPU checks necessary to determine if this CPU is
+    // acceptable.
+    // FIXME: This results in terrible diagnostics. Clang just says the CPU is
+    // invalid without explaining *why*.
+    switch (CPU) {
+    case CK_Generic:
+      // No processor selected!
+      return false;
+
+    case CK_i386:
+    case CK_i486:
+    case CK_WinChipC6:
+    case CK_WinChip2:
+    case CK_C3:
+    case CK_i586:
+    case CK_Pentium:
+    case CK_PentiumMMX:
+    case CK_i686:
+    case CK_PentiumPro:
+    case CK_Pentium2:
+    case CK_Pentium3:
+    case CK_Pentium3M:
+    case CK_PentiumM:
+    case CK_Yonah:
+    case CK_C3_2:
+    case CK_Pentium4:
+    case CK_Pentium4M:
+    case CK_Prescott:
+    case CK_K6:
+    case CK_K6_2:
+    case CK_K6_3:
+    case CK_Athlon:
+    case CK_AthlonThunderbird:
+    case CK_Athlon4:
+    case CK_AthlonXP:
+    case CK_AthlonMP:
+    case CK_Geode:
+      // Only accept certain architectures when compiling in 32-bit mode.
+      if (PointerWidth != 32)
+        return false;
+
+      // Fallthrough
+    case CK_Nocona:
+    case CK_Core2:
+    case CK_Penryn:
+    case CK_Atom:
+    case CK_Corei7:
+    case CK_Corei7AVX:
+    case CK_CoreAVXi:
+    case CK_Athlon64:
+    case CK_Athlon64SSE3:
+    case CK_AthlonFX:
+    case CK_K8:
+    case CK_K8SSE3:
+    case CK_Opteron:
+    case CK_OpteronSSE3:
+    case CK_x86_64:
+      return true;
+    }
+    llvm_unreachable("Unhandled CPU kind");
+  }
 };
 
-void X86TargetInfo::getDefaultFeatures(const std::string &CPU,
-                                       llvm::StringMap<bool> &Features) const {
+void X86TargetInfo::getDefaultFeatures(llvm::StringMap<bool> &Features) const {
   // FIXME: This should not be here.
   Features["3dnow"] = false;
   Features["3dnowa"] = false;
@@ -1188,57 +1471,100 @@ void X86TargetInfo::getDefaultFeatures(const std::string &CPU,
   if (PointerWidth == 64)
     Features["sse2"] = Features["sse"] = Features["mmx"] = true;
 
-  if (CPU == "generic" || CPU == "i386" || CPU == "i486" || CPU == "i586" ||
-      CPU == "pentium" || CPU == "i686" || CPU == "pentiumpro")
-    ;
-  else if (CPU == "pentium-mmx" || CPU == "pentium2")
+  switch (CPU) {
+  case CK_Generic:
+  case CK_i386:
+  case CK_i486:
+  case CK_i586:
+  case CK_Pentium:
+  case CK_i686:
+  case CK_PentiumPro:
+    break;
+  case CK_PentiumMMX:
+  case CK_Pentium2:
     setFeatureEnabled(Features, "mmx", true);
-  else if (CPU == "pentium3") {
+    break;
+  case CK_Pentium3:
+  case CK_Pentium3M:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse", true);
-  } else if (CPU == "pentium-m" || CPU == "pentium4" || CPU == "x86-64") {
+    break;
+  case CK_PentiumM:
+  case CK_Pentium4:
+  case CK_Pentium4M:
+  case CK_x86_64:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse2", true);
-  } else if (CPU == "yonah" || CPU == "prescott" || CPU == "nocona") {
+    break;
+  case CK_Yonah:
+  case CK_Prescott:
+  case CK_Nocona:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse3", true);
-  } else if (CPU == "core2") {
+    break;
+  case CK_Core2:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "ssse3", true);
-  } else if (CPU == "penryn") {
+    break;
+  case CK_Penryn:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse4", true);
     Features["sse42"] = false;
-  } else if (CPU == "atom") {
+    break;
+  case CK_Atom:
     setFeatureEnabled(Features, "mmx", true);
-    setFeatureEnabled(Features, "sse3", true);
-  } else if (CPU == "corei7") {
+    setFeatureEnabled(Features, "ssse3", true);
+    break;
+  case CK_Corei7:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse4", true);
     setFeatureEnabled(Features, "aes", true);
-  } else if (CPU == "corei7-avx") {
+    break;
+  case CK_Corei7AVX:
+  case CK_CoreAVXi:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse4", true);
     setFeatureEnabled(Features, "aes", true);
     //setFeatureEnabled(Features, "avx", true);
-  } else if (CPU == "k6" || CPU == "winchip-c6")
+    break;
+  case CK_K6:
+  case CK_WinChipC6:
     setFeatureEnabled(Features, "mmx", true);
-  else if (CPU == "k6-2" || CPU == "k6-3" || CPU == "athlon" ||
-           CPU == "athlon-tbird" || CPU == "winchip2" || CPU == "c3") {
+    break;
+  case CK_K6_2:
+  case CK_K6_3:
+  case CK_WinChip2:
+  case CK_C3:
     setFeatureEnabled(Features, "3dnow", true);
-  } else if (CPU == "athlon-4" || CPU == "athlon-xp" || CPU == "athlon-mp") {
+    break;
+  case CK_Athlon:
+  case CK_AthlonThunderbird:
+  case CK_Geode:
+    setFeatureEnabled(Features, "3dnowa", true);
+    break;
+  case CK_Athlon4:
+  case CK_AthlonXP:
+  case CK_AthlonMP:
     setFeatureEnabled(Features, "sse", true);
     setFeatureEnabled(Features, "3dnowa", true);
-  } else if (CPU == "k8" || CPU == "opteron" || CPU == "athlon64" ||
-           CPU == "athlon-fx") {
+    break;
+  case CK_K8:
+  case CK_Opteron:
+  case CK_Athlon64:
+  case CK_AthlonFX:
     setFeatureEnabled(Features, "sse2", true);
     setFeatureEnabled(Features, "3dnowa", true);
-  } else if (CPU == "k8-sse3") {
+    break;
+  case CK_K8SSE3:
+  case CK_OpteronSSE3:
+  case CK_Athlon64SSE3:
     setFeatureEnabled(Features, "sse3", true);
     setFeatureEnabled(Features, "3dnowa", true);
-  } else if (CPU == "c3-2") {
+    break;
+  case CK_C3_2:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse", true);
+    break;
   }
 }
 
@@ -1276,7 +1602,8 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
     else if (Name == "aes")
       Features["aes"] = true;
     else if (Name == "avx")
-      Features["avx"] = true;
+      Features["avx"] = Features["sse"] = Features["sse2"] = Features["sse3"] =
+        Features["ssse3"] = Features["sse41"] = Features["sse42"] = true;
   } else {
     if (Name == "mmx")
       Features["mmx"] = Features["3dnow"] = Features["3dnowa"] = false;
@@ -1358,8 +1685,8 @@ void X86TargetInfo::HandleTargetFeatures(std::vector<std::string> &Features) {
     Features.erase(it);
 }
 
-/// X86TargetInfo::getTargetDefines - Return a set of the X86-specific #defines
-/// that are not tied to a specific subtarget.
+/// X86TargetInfo::getTargetDefines - Return the set of the X86-specific macro
+/// definitions for this particular subtarget.
 void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
                                      MacroBuilder &Builder) const {
   // Target identification.
@@ -1374,19 +1701,140 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     DefineStd(Builder, "i386", Opts);
   }
 
-  if (HasAES)
-    Builder.defineMacro("__AES__");
-
-  if (HasAVX)
-    Builder.defineMacro("__AVX__");
+  // Subtarget options.
+  // FIXME: We are hard-coding the tune parameters based on the CPU, but they
+  // truly should be based on -mtune options.
+  switch (CPU) {
+  case CK_Generic:
+    break;
+  case CK_i386:
+    // The rest are coming from the i386 define above.
+    Builder.defineMacro("__tune_i386__");
+    break;
+  case CK_i486:
+  case CK_WinChipC6:
+  case CK_WinChip2:
+  case CK_C3:
+    Builder.defineMacro("__i486");
+    Builder.defineMacro("__i486__");
+    Builder.defineMacro("__tune_i486__");
+    break;
+  case CK_PentiumMMX:
+    Builder.defineMacro("__pentium_mmx__");
+    Builder.defineMacro("__tune_pentium_mmx__");
+    // Fallthrough
+  case CK_i586:
+  case CK_Pentium:
+    Builder.defineMacro("__i586");
+    Builder.defineMacro("__i586__");
+    Builder.defineMacro("__tune_i586__");
+    Builder.defineMacro("__pentium");
+    Builder.defineMacro("__pentium__");
+    Builder.defineMacro("__tune_pentium__");
+    break;
+  case CK_Pentium3:
+  case CK_Pentium3M:
+  case CK_PentiumM:
+    Builder.defineMacro("__tune_pentium3__");
+    // Fallthrough
+  case CK_Pentium2:
+  case CK_C3_2:
+    Builder.defineMacro("__tune_pentium2__");
+    // Fallthrough
+  case CK_PentiumPro:
+    Builder.defineMacro("__tune_i686__");
+    Builder.defineMacro("__tune_pentiumpro__");
+    // Fallthrough
+  case CK_i686:
+    Builder.defineMacro("__i686");
+    Builder.defineMacro("__i686__");
+    // Strangely, __tune_i686__ isn't defined by GCC when CPU == i686.
+    Builder.defineMacro("__pentiumpro");
+    Builder.defineMacro("__pentiumpro__");
+    break;
+  case CK_Pentium4:
+  case CK_Pentium4M:
+    Builder.defineMacro("__pentium4");
+    Builder.defineMacro("__pentium4__");
+    Builder.defineMacro("__tune_pentium4__");
+    break;
+  case CK_Yonah:
+  case CK_Prescott:
+  case CK_Nocona:
+    Builder.defineMacro("__nocona");
+    Builder.defineMacro("__nocona__");
+    Builder.defineMacro("__tune_nocona__");
+    break;
+  case CK_Core2:
+  case CK_Penryn:
+    Builder.defineMacro("__core2");
+    Builder.defineMacro("__core2__");
+    Builder.defineMacro("__tune_core2__");
+    break;
+  case CK_Atom:
+    Builder.defineMacro("__atom");
+    Builder.defineMacro("__atom__");
+    Builder.defineMacro("__tune_atom__");
+    break;
+  case CK_Corei7:
+  case CK_Corei7AVX:
+  case CK_CoreAVXi:
+    Builder.defineMacro("__corei7");
+    Builder.defineMacro("__corei7__");
+    Builder.defineMacro("__tune_corei7__");
+    break;
+  case CK_K6_2:
+    Builder.defineMacro("__k6_2__");
+    Builder.defineMacro("__tune_k6_2__");
+    // Fallthrough
+  case CK_K6_3:
+    if (CPU != CK_K6_2) {  // In case of fallthrough
+      // FIXME: GCC may be enabling these in cases where some other k6
+      // architecture is specified but -m3dnow is explicitly provided. The
+      // exact semantics need to be determined and emulated here.
+      Builder.defineMacro("__k6_3__");
+      Builder.defineMacro("__tune_k6_3__");
+    }
+    // Fallthrough
+  case CK_K6:
+    Builder.defineMacro("__k6");
+    Builder.defineMacro("__k6__");
+    Builder.defineMacro("__tune_k6__");
+    break;
+  case CK_Athlon:
+  case CK_AthlonThunderbird:
+  case CK_Athlon4:
+  case CK_AthlonXP:
+  case CK_AthlonMP:
+    Builder.defineMacro("__athlon");
+    Builder.defineMacro("__athlon__");
+    Builder.defineMacro("__tune_athlon__");
+    if (SSELevel != NoSSE) {
+      Builder.defineMacro("__athlon_sse__");
+      Builder.defineMacro("__tune_athlon_sse__");
+    }
+    break;
+  case CK_K8:
+  case CK_K8SSE3:
+  case CK_x86_64:
+  case CK_Opteron:
+  case CK_OpteronSSE3:
+  case CK_Athlon64:
+  case CK_Athlon64SSE3:
+  case CK_AthlonFX:
+    Builder.defineMacro("__k8");
+    Builder.defineMacro("__k8__");
+    Builder.defineMacro("__tune_k8__");
+    break;
+  case CK_Geode:
+    Builder.defineMacro("__geode");
+    Builder.defineMacro("__geode__");
+    Builder.defineMacro("__tune_geode__");
+    break;
+  }
 
   // Target properties.
   Builder.defineMacro("__LITTLE_ENDIAN__");
-
-  // Subtarget options.
-  Builder.defineMacro("__nocona");
-  Builder.defineMacro("__nocona__");
-  Builder.defineMacro("__tune_nocona__");
   Builder.defineMacro("__REGISTER_PREFIX__", "");
 
   // Define __NO_MATH_INLINES on linux/x86 so that we don't get inline
@@ -1394,6 +1842,12 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   // backend can't deal with (PR879).
   Builder.defineMacro("__NO_MATH_INLINES");
 
+  if (HasAES)
+    Builder.defineMacro("__AES__");
+
+  if (HasAVX)
+    Builder.defineMacro("__AVX__");
+
   // Each case falls through to the previous one here.
   switch (SSELevel) {
   case SSE42:
@@ -1414,20 +1868,20 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     break;
   }
 
-  if (Opts.Microsoft && PointerWidth == 32) {
+  if (Opts.MicrosoftExt && PointerWidth == 32) {
     switch (SSELevel) {
     case SSE42:
     case SSE41:
     case SSSE3:
     case SSE3:
     case SSE2:
-      Builder.defineMacro("_M_IX86_FP", llvm::Twine(2));
+      Builder.defineMacro("_M_IX86_FP", Twine(2));
       break;
     case SSE1:
-      Builder.defineMacro("_M_IX86_FP", llvm::Twine(1));
+      Builder.defineMacro("_M_IX86_FP", Twine(1));
       break;
     default:
-      Builder.defineMacro("_M_IX86_FP", llvm::Twine(0));
+      Builder.defineMacro("_M_IX86_FP", Twine(0));
     }
   }
 
@@ -1521,7 +1975,7 @@ class X86_32TargetInfo : public X86TargetInfo {
     LongDoubleAlign = 32;
     DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-"
-                        "a0:0:64-f80:32:32-n8:16:32";
+                        "a0:0:64-f80:32:32-n8:16:32-S128";
     SizeType = UnsignedInt;
     PtrDiffType = SignedInt;
     IntPtrType = SignedInt;
@@ -1531,6 +1985,11 @@ class X86_32TargetInfo : public X86TargetInfo {
     RealTypeUsesObjCFPRet = ((1 << TargetInfo::Float) |
                              (1 << TargetInfo::Double) |
                              (1 << TargetInfo::LongDouble));
+
+    // x86-32 has atomics up to 8 bytes
+    // FIXME: Check that we actually have cmpxchg8b before setting
+    // MaxAtomicInlineWidth. (cmpxchg8b is an i586 instruction.)
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
   }
   virtual const char *getVAListDeclaration() const {
     return "typedef char* __builtin_va_list;";
@@ -1567,7 +2026,7 @@ class DarwinI386TargetInfo : public DarwinTargetInfo<X86_32TargetInfo> {
     IntPtrType = SignedLong;
     DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-"
-                        "a0:0:64-f80:128:128-n8:16:32";
+                        "a0:0:64-f80:128:128-n8:16:32-S128";
     HasAlignMac68kSupport = true;
   }
 
@@ -1585,7 +2044,7 @@ class WindowsX86_32TargetInfo : public WindowsTargetInfo<X86_32TargetInfo> {
     DoubleAlign = LongLongAlign = 64;
     DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-"
-                        "v128:128:128-a0:0:64-f80:32:32-n8:16:32";
+                        "v128:128:128-a0:0:64-f80:32:32-n8:16:32-S32";
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
@@ -1634,7 +2093,7 @@ class MinGWX86_32TargetInfo : public WindowsX86_32TargetInfo {
 
     // mingw32-gcc provides __declspec(a) as alias of __attribute__((a)).
     // In contrast, clang-cc1 provides __declspec(a) with -fms-extensions.
-    if (Opts.Microsoft)
+    if (Opts.MicrosoftExt)
       // Provide "as-is" __declspec.
       Builder.defineMacro("__declspec", "__declspec");
     else
@@ -1655,7 +2114,7 @@ class CygwinX86_32TargetInfo : public X86_32TargetInfo {
     DoubleAlign = LongLongAlign = 64;
     DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-"
-                        "a0:0:64-f80:32:32-n8:16:32";
+                        "a0:0:64-f80:32:32-n8:16:32-S32";
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
@@ -1697,9 +2156,6 @@ class RTEMSTargetInfo : public OSTargetInfo<Target> {
                             MacroBuilder &Builder) const {
     // RTEMS defines; list based off of gcc output
 
-    // FIXME: Move version number handling to llvm::Triple.
-    llvm::StringRef Release = Triple.getOSName().substr(strlen("rtems"), 1);
-
     Builder.defineMacro("__rtems__");
     Builder.defineMacro("__ELF__");
   }
@@ -1765,10 +2221,16 @@ class X86_64TargetInfo : public X86TargetInfo {
 
     DescriptionString = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-"
-                        "a0:0:64-s0:64:64-f80:128:128-n8:16:32:64";
+                        "a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128";
 
     // Use fpret only for long double.
     RealTypeUsesObjCFPRet = (1 << TargetInfo::LongDouble);
+
+    // x86-64 has atomics up to 16 bytes.
+    // FIXME: Once the backend is fixed, increase MaxAtomicInlineWidth to 128
+    // on CPUs with cmpxchg16b
+    MaxAtomicPromoteWidth = 128;
+    MaxAtomicInlineWidth = 64;
   }
   virtual const char *getVAListDeclaration() const {
     return "typedef struct __va_list_tag {"
@@ -1853,7 +2315,7 @@ class MinGWX86_64TargetInfo : public WindowsX86_64TargetInfo {
 
     // mingw32-gcc provides __declspec(a) as alias of __attribute__((a)).
     // In contrast, clang-cc1 provides __declspec(a) with -fms-extensions.
-    if (Opts.Microsoft)
+    if (Opts.MicrosoftExt)
       // Provide "as-is" __declspec.
       Builder.defineMacro("__declspec", "__declspec");
     else
@@ -1932,15 +2394,19 @@ class ARMTargetInfo : public TargetInfo {
       // so set preferred for small types to 32.
       DescriptionString = ("e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-"
                            "i64:64:64-f32:32:32-f64:64:64-"
-                           "v64:64:64-v128:64:128-a0:0:32-n32");
+                           "v64:64:64-v128:64:128-a0:0:32-n32-S64");
     } else {
       DescriptionString = ("e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                            "i64:64:64-f32:32:32-f64:64:64-"
-                           "v64:64:64-v128:64:128-a0:0:64-n32");
+                           "v64:64:64-v128:64:128-a0:0:64-n32-S64");
     }
 
     // ARM targets default to using the ARM C++ ABI.
     CXXABI = CXXABI_ARM;
+
+    // ARM has atomics up to 8 bytes
+    // FIXME: Set MaxAtomicInlineWidth if we have the feature v6e
+    MaxAtomicPromoteWidth = 64;
   }
   virtual const char *getABI() const { return ABI.c_str(); }
   virtual bool setABI(const std::string &Name) {
@@ -1958,16 +2424,27 @@ class ARMTargetInfo : public TargetInfo {
       // structures. This corresponds to PCC_BITFIELD_TYPE_MATTERS in gcc.
       UseBitFieldTypeAlignment = false;
 
+      /// Do force alignment of members that follow zero length bitfields.  If
+      /// the alignment of the zero-length bitfield is greater than the member 
+      /// that follows it, `bar', `bar' will be aligned as the  type of the 
+      /// zero length bitfield.
+      UseZeroLengthBitfieldAlignment = true;
+
+      /// gcc forces the alignment to 4 bytes, regardless of the type of the
+      /// zero length bitfield.  This corresponds to EMPTY_FIELD_BOUNDARY in
+      /// gcc.
+      ZeroLengthBitfieldBoundary = 32;
+
       if (IsThumb) {
         // Thumb1 add sp, #imm requires the immediate value be multiple of 4,
         // so set preferred for small types to 32.
         DescriptionString = ("e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-"
                              "i64:32:64-f32:32:32-f64:32:64-"
-                             "v64:32:64-v128:32:128-a0:0:32-n32");
+                             "v64:32:64-v128:32:128-a0:0:32-n32-S32");
       } else {
         DescriptionString = ("e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                              "i64:32:64-f32:32:32-f64:32:64-"
-                             "v64:32:64-v128:32:128-a0:0:32-n32");
+                             "v64:32:64-v128:32:128-a0:0:32-n32-S32");
       }
 
       // FIXME: Override "preferred align" for double and long long.
@@ -1981,8 +2458,7 @@ class ARMTargetInfo : public TargetInfo {
     return true;
   }
 
-  void getDefaultFeatures(const std::string &CPU,
-                          llvm::StringMap<bool> &Features) const {
+  void getDefaultFeatures(llvm::StringMap<bool> &Features) const {
     if (CPU == "arm1136jf-s" || CPU == "arm1176jzf-s" || CPU == "mpcore")
       Features["vfp2"] = true;
     else if (CPU == "cortex-a8" || CPU == "cortex-a9")
@@ -2027,7 +2503,7 @@ class ARMTargetInfo : public TargetInfo {
       Features.erase(it);
   }
 
-  static const char *getCPUDefineSuffix(llvm::StringRef Name) {
+  static const char *getCPUDefineSuffix(StringRef Name) {
     return llvm::StringSwitch<const char*>(Name)
       .Cases("arm8", "arm810", "4")
       .Cases("strongarm", "strongarm110", "strongarm1100", "strongarm1110", "4")
@@ -2066,7 +2542,7 @@ class ARMTargetInfo : public TargetInfo {
     Builder.defineMacro("__LITTLE_ENDIAN__");
     Builder.defineMacro("__REGISTER_PREFIX__", "");
 
-    llvm::StringRef CPUArch = getCPUDefineSuffix(CPU);
+    StringRef CPUArch = getCPUDefineSuffix(CPU);
     Builder.defineMacro("__ARM_ARCH_" + CPUArch + "__");
 
     // Subtarget options.
@@ -2129,6 +2605,9 @@ class ARMTargetInfo : public TargetInfo {
     case 'P': // VFP Floating point register double precision
       Info.setAllowsRegister();
       return true;
+    case 'Q': // A memory address that is a single base register.
+      Info.setAllowsMemory();
+      return true;
     case 'U': // a memory reference...
       switch (Name[1]) {
       case 'q': // ...ARMV4 ldrsb
@@ -2245,6 +2724,9 @@ class DarwinARMTargetInfo :
   DarwinARMTargetInfo(const std::string& triple)
     : DarwinTargetInfo<ARMTargetInfo>(triple) {
     HasAlignMac68kSupport = true;
+    // iOS always has 64-bit atomic instructions.
+    // FIXME: This should be based off of the target features in ARMTargetInfo.
+    MaxAtomicInlineWidth = 64;
   }
 };
 } // end anonymous namespace.
@@ -2590,6 +3072,12 @@ namespace {
   // target processor and program binary. TCE co-design environment is
   // publicly available in http://tce.cs.tut.fi
 
+  static const unsigned TCEOpenCLAddrSpaceMap[] = {
+      3, // opencl_global
+      4, // opencl_local
+      5  // opencl_constant
+  };
+
   class TCETargetInfo : public TargetInfo{
   public:
     TCETargetInfo(const std::string& triple) : TargetInfo(triple) {
@@ -2618,6 +3106,7 @@ namespace {
                           "i16:16:32-i32:32:32-i64:32:32-"
                           "f32:32:32-f64:32:32-v64:32:32-"
                           "v128:32:32-a0:0:32-n32";
+      AddrSpaceMap = &TCEOpenCLAddrSpaceMap;
     }
 
     virtual void getTargetDefines(const LangOptions &Opts,
@@ -2646,51 +3135,30 @@ namespace {
 }
 
 namespace {
-class MipsTargetInfo : public TargetInfo {
-  std::string ABI, CPU;
-  static const TargetInfo::GCCRegAlias GCCRegAliases[];
-  static const char * const GCCRegNames[];
+class MipsTargetInfoBase : public TargetInfo {
+  std::string CPU;
+protected:
+  std::string ABI;
 public:
-  MipsTargetInfo(const std::string& triple) : TargetInfo(triple), ABI("o32") {
-    DescriptionString = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
-                        "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";
+  MipsTargetInfoBase(const std::string& triple, const std::string& ABIStr)
+    : TargetInfo(triple), ABI(ABIStr) {
     SizeType = UnsignedInt;
     PtrDiffType = SignedInt;
   }
   virtual const char *getABI() const { return ABI.c_str(); }
-  virtual bool setABI(const std::string &Name) {
-
-    if ((Name == "o32") || (Name == "eabi")) {
-      ABI = Name;
-      return true;
-    } else
-      return false;
-  }
+  virtual bool setABI(const std::string &Name) = 0;
   virtual bool setCPU(const std::string &Name) {
     CPU = Name;
     return true;
   }
-  void getDefaultFeatures(const std::string &CPU,
-                          llvm::StringMap<bool> &Features) const {
+  void getDefaultFeatures(llvm::StringMap<bool> &Features) const {
     Features[ABI] = true;
     Features[CPU] = true;
   }
   virtual void getArchDefines(const LangOptions &Opts,
-                                MacroBuilder &Builder) const {
-    if (ABI == "o32")
-      Builder.defineMacro("__mips_o32");
-    else if (ABI == "eabi")
-      Builder.defineMacro("__mips_eabi");
-  }
+                              MacroBuilder &Builder) const = 0;
   virtual void getTargetDefines(const LangOptions &Opts,
-                                MacroBuilder &Builder) const {
-    DefineStd(Builder, "mips", Opts);
-    Builder.defineMacro("_mips");
-    DefineStd(Builder, "MIPSEB", Opts);
-    Builder.defineMacro("_MIPSEB");
-    Builder.defineMacro("__REGISTER_PREFIX__", "");
-    getArchDefines(Opts, Builder);
-  }
+                                MacroBuilder &Builder) const = 0;
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
                                  unsigned &NumRecords) const {
     // FIXME: Implement!
@@ -2699,9 +3167,24 @@ class MipsTargetInfo : public TargetInfo {
     return "typedef void* __builtin_va_list;";
   }
   virtual void getGCCRegNames(const char * const *&Names,
-                              unsigned &NumNames) const;
+                              unsigned &NumNames) const {
+    static const char * const GCCRegNames[] = {
+      "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7",
+      "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
+      "$16",  "$17",  "$18",  "$19",  "$20",  "$21",  "$22",  "$23",
+      "$24",  "$25",  "$26",  "$27",  "$28",  "$sp",  "$fp",  "$31",
+      "$f0",  "$f1",  "$f2",  "$f3",  "$f4",  "$f5",  "$f6",  "$f7",
+      "$f8",  "$f9",  "$f10", "$f11", "$f12", "$f13", "$f14", "$f15",
+      "$f16", "$f17", "$f18", "$f19", "$f20", "$f21", "$f22", "$f23",
+      "$f24", "$f25", "$f26", "$f27", "$f28", "$f29", "$f30", "$f31",
+      "hi",   "lo",   "",     "$fcc0","$fcc1","$fcc2","$fcc3","$fcc4",
+      "$fcc5","$fcc6","$fcc7"
+    };
+    Names = GCCRegNames;
+    NumNames = llvm::array_lengthof(GCCRegNames);
+  }
   virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
-                                unsigned &NumAliases) const;
+                                unsigned &NumAliases) const = 0;
   virtual bool validateAsmConstraint(const char *&Name,
                                      TargetInfo::ConstraintInfo &Info) const {
     switch (*Name) {
@@ -2722,89 +3205,296 @@ class MipsTargetInfo : public TargetInfo {
   }
 };
 
-const char * const MipsTargetInfo::GCCRegNames[] = {
-  "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7",
-  "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
-  "$16",  "$17",  "$18",  "$19",  "$20",  "$21",  "$22",  "$23",
-  "$24",  "$25",  "$26",  "$27",  "$28",  "$sp",  "$fp",  "$31",
-  "$f0",  "$f1",  "$f2",  "$f3",  "$f4",  "$f5",  "$f6",  "$f7",
-  "$f8",  "$f9",  "$f10", "$f11", "$f12", "$f13", "$f14", "$f15",
-  "$f16", "$f17", "$f18", "$f19", "$f20", "$f21", "$f22", "$f23",
-  "$f24", "$f25", "$f26", "$f27", "$f28", "$f29", "$f30", "$f31",
-  "hi",   "lo",   "",     "$fcc0","$fcc1","$fcc2","$fcc3","$fcc4",
-  "$fcc5","$fcc6","$fcc7"
-};
-
-void MipsTargetInfo::getGCCRegNames(const char * const *&Names,
-                                       unsigned &NumNames) const {
-  Names = GCCRegNames;
-  NumNames = llvm::array_lengthof(GCCRegNames);
-}
-
-const TargetInfo::GCCRegAlias MipsTargetInfo::GCCRegAliases[] = {
-  { { "at" },  "$1" },
-  { { "v0" },  "$2" },
-  { { "v1" },  "$3" },
-  { { "a0" },  "$4" },
-  { { "a1" },  "$5" },
-  { { "a2" },  "$6" },
-  { { "a3" },  "$7" },
-  { { "t0" },  "$8" },
-  { { "t1" },  "$9" },
-  { { "t2" }, "$10" },
-  { { "t3" }, "$11" },
-  { { "t4" }, "$12" },
-  { { "t5" }, "$13" },
-  { { "t6" }, "$14" },
-  { { "t7" }, "$15" },
-  { { "s0" }, "$16" },
-  { { "s1" }, "$17" },
-  { { "s2" }, "$18" },
-  { { "s3" }, "$19" },
-  { { "s4" }, "$20" },
-  { { "s5" }, "$21" },
-  { { "s6" }, "$22" },
-  { { "s7" }, "$23" },
-  { { "t8" }, "$24" },
-  { { "t9" }, "$25" },
-  { { "k0" }, "$26" },
-  { { "k1" }, "$27" },
-  { { "gp" }, "$28" },
-  { { "sp" }, "$29" },
-  { { "fp" }, "$30" },
-  { { "ra" }, "$31" }
-};
-
-void MipsTargetInfo::getGCCRegAliases(const GCCRegAlias *&Aliases,
-                                         unsigned &NumAliases) const {
-  Aliases = GCCRegAliases;
-  NumAliases = llvm::array_lengthof(GCCRegAliases);
-}
-} // end anonymous namespace.
-
-namespace {
-class MipselTargetInfo : public MipsTargetInfo {
+class Mips32TargetInfoBase : public MipsTargetInfoBase {
 public:
-  MipselTargetInfo(const std::string& triple) : MipsTargetInfo(triple) {
+  Mips32TargetInfoBase(const std::string& triple) :
+    MipsTargetInfoBase(triple, "o32") {}
+  virtual bool setABI(const std::string &Name) {
+    if ((Name == "o32") || (Name == "eabi")) {
+      ABI = Name;
+      return true;
+    } else
+      return false;
+  }
+  virtual void getArchDefines(const LangOptions &Opts,
+                              MacroBuilder &Builder) const {
+    if (ABI == "o32") {
+      Builder.defineMacro("__mips_o32");
+      Builder.defineMacro("_ABIO32", "1");
+      Builder.defineMacro("_MIPS_SIM", "_ABIO32");
+    }
+    else if (ABI == "eabi")
+      Builder.defineMacro("__mips_eabi");
+    else
+      llvm_unreachable("Invalid ABI for Mips32.");
+  }
+  virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
+                                unsigned &NumAliases) const {
+    static const TargetInfo::GCCRegAlias GCCRegAliases[] = {
+      { { "at" },  "$1" },
+      { { "v0" },  "$2" },
+      { { "v1" },  "$3" },
+      { { "a0" },  "$4" },
+      { { "a1" },  "$5" },
+      { { "a2" },  "$6" },
+      { { "a3" },  "$7" },
+      { { "t0" },  "$8" },
+      { { "t1" },  "$9" },
+      { { "t2" }, "$10" },
+      { { "t3" }, "$11" },
+      { { "t4" }, "$12" },
+      { { "t5" }, "$13" },
+      { { "t6" }, "$14" },
+      { { "t7" }, "$15" },
+      { { "s0" }, "$16" },
+      { { "s1" }, "$17" },
+      { { "s2" }, "$18" },
+      { { "s3" }, "$19" },
+      { { "s4" }, "$20" },
+      { { "s5" }, "$21" },
+      { { "s6" }, "$22" },
+      { { "s7" }, "$23" },
+      { { "t8" }, "$24" },
+      { { "t9" }, "$25" },
+      { { "k0" }, "$26" },
+      { { "k1" }, "$27" },
+      { { "gp" }, "$28" },
+      { { "sp" }, "$29" },
+      { { "fp" }, "$30" },
+      { { "ra" }, "$31" }
+    };
+    Aliases = GCCRegAliases;
+    NumAliases = llvm::array_lengthof(GCCRegAliases);
+  }
+};
+
+class Mips32EBTargetInfo : public Mips32TargetInfoBase {
+public:
+  Mips32EBTargetInfo(const std::string& triple) : Mips32TargetInfoBase(triple) {
+    DescriptionString = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
+                        "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";
+  }
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    DefineStd(Builder, "mips", Opts);
+    Builder.defineMacro("_mips");
+    DefineStd(Builder, "MIPSEB", Opts);
+    Builder.defineMacro("_MIPSEB");
+    Builder.defineMacro("__REGISTER_PREFIX__", "");
+    getArchDefines(Opts, Builder);
+  }
+};
+
+class Mips32ELTargetInfo : public Mips32TargetInfoBase {
+public:
+  Mips32ELTargetInfo(const std::string& triple) : Mips32TargetInfoBase(triple) {
     DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";
   }
-
   virtual void getTargetDefines(const LangOptions &Opts,
-                                MacroBuilder &Builder) const;
+                                MacroBuilder &Builder) const {
+    DefineStd(Builder, "mips", Opts);
+    Builder.defineMacro("_mips");
+    DefineStd(Builder, "MIPSEL", Opts);
+    Builder.defineMacro("_MIPSEL");
+    Builder.defineMacro("__REGISTER_PREFIX__", "");
+    getArchDefines(Opts, Builder);
+  }
 };
 
-void MipselTargetInfo::getTargetDefines(const LangOptions &Opts,
-                                        MacroBuilder &Builder) const {
-  DefineStd(Builder, "mips", Opts);
-  Builder.defineMacro("_mips");
-  DefineStd(Builder, "MIPSEL", Opts);
-  Builder.defineMacro("_MIPSEL");
-  Builder.defineMacro("__REGISTER_PREFIX__", "");
-  getArchDefines(Opts, Builder);
+class Mips64TargetInfoBase : public MipsTargetInfoBase {
+  virtual void SetDescriptionString(const std::string &Name) = 0;
+public:
+  Mips64TargetInfoBase(const std::string& triple) :
+    MipsTargetInfoBase(triple, "n64") {}
+  virtual bool setABI(const std::string &Name) {
+    SetDescriptionString(Name);
+    if ((Name == "n32") || (Name == "n64")) {
+      ABI = Name;
+      return true;
+    } else
+      return false;
+  }
+  virtual void getArchDefines(const LangOptions &Opts,
+                              MacroBuilder &Builder) const {
+    if (ABI == "n32") {
+      Builder.defineMacro("__mips_n32");
+      Builder.defineMacro("_ABIN32", "2");
+      Builder.defineMacro("_MIPS_SIM", "_ABIN32");
+    }
+    else if (ABI == "n64") {
+      Builder.defineMacro("__mips_n64");
+      Builder.defineMacro("_ABI64", "3");
+      Builder.defineMacro("_MIPS_SIM", "_ABI64");
+    }
+    else
+      llvm_unreachable("Invalid ABI for Mips64.");
+  }
+  virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
+                                unsigned &NumAliases) const {
+    static const TargetInfo::GCCRegAlias GCCRegAliases[] = {
+      { { "at" },  "$1" },
+      { { "v0" },  "$2" },
+      { { "v1" },  "$3" },
+      { { "a0" },  "$4" },
+      { { "a1" },  "$5" },
+      { { "a2" },  "$6" },
+      { { "a3" },  "$7" },
+      { { "a4" },  "$8" },
+      { { "a5" },  "$9" },
+      { { "a6" }, "$10" },
+      { { "a7" }, "$11" },
+      { { "t0" }, "$12" },
+      { { "t1" }, "$13" },
+      { { "t2" }, "$14" },
+      { { "t3" }, "$15" },
+      { { "s0" }, "$16" },
+      { { "s1" }, "$17" },
+      { { "s2" }, "$18" },
+      { { "s3" }, "$19" },
+      { { "s4" }, "$20" },
+      { { "s5" }, "$21" },
+      { { "s6" }, "$22" },
+      { { "s7" }, "$23" },
+      { { "t8" }, "$24" },
+      { { "t9" }, "$25" },
+      { { "k0" }, "$26" },
+      { { "k1" }, "$27" },
+      { { "gp" }, "$28" },
+      { { "sp" }, "$29" },
+      { { "fp" }, "$30" },
+      { { "ra" }, "$31" }
+    };
+    Aliases = GCCRegAliases;
+    NumAliases = llvm::array_lengthof(GCCRegAliases);
+  }
+};
+
+class Mips64EBTargetInfo : public Mips64TargetInfoBase {
+  virtual void SetDescriptionString(const std::string &Name) {
+    // Change DescriptionString only if ABI is n32.  
+    if (Name == "n32")
+      DescriptionString = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
+                          "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";      
+  }
+public:
+  Mips64EBTargetInfo(const std::string& triple) : Mips64TargetInfoBase(triple) {
+    // Default ABI is n64.  
+    DescriptionString = "E-p:64:64:64-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
+                        "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";
+  }
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    DefineStd(Builder, "mips", Opts);
+    Builder.defineMacro("_mips");
+    DefineStd(Builder, "MIPSEB", Opts);
+    Builder.defineMacro("_MIPSEB");
+    Builder.defineMacro("__REGISTER_PREFIX__", "");
+    getArchDefines(Opts, Builder);
+  }
+};
+
+class Mips64ELTargetInfo : public Mips64TargetInfoBase {
+  virtual void SetDescriptionString(const std::string &Name) {
+    // Change DescriptionString only if ABI is n32.  
+    if (Name == "n32")
+      DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
+                          "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";      
+  }
+public:
+  Mips64ELTargetInfo(const std::string& triple) : Mips64TargetInfoBase(triple) {
+    // Default ABI is n64.  
+    DescriptionString = "e-p:64:64:64-i1:8:8-i8:8:32-i16:16:32-i32:32:32-"
+                        "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";
+  }
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    DefineStd(Builder, "mips", Opts);
+    Builder.defineMacro("_mips");
+    DefineStd(Builder, "MIPSEL", Opts);
+    Builder.defineMacro("_MIPSEL");
+    Builder.defineMacro("__REGISTER_PREFIX__", "");
+    getArchDefines(Opts, Builder);
+  }
+};
+} // end anonymous namespace.
+
+namespace {
+class PNaClTargetInfo : public TargetInfo {
+public:
+  PNaClTargetInfo(const std::string& triple) : TargetInfo(triple) {
+    this->UserLabelPrefix = "";
+    this->LongAlign = 32;
+    this->LongWidth = 32;
+    this->PointerAlign = 32;
+    this->PointerWidth = 32;
+    this->IntMaxType = TargetInfo::SignedLongLong;
+    this->UIntMaxType = TargetInfo::UnsignedLongLong;
+    this->Int64Type = TargetInfo::SignedLongLong;
+    this->DoubleAlign = 64;
+    this->LongDoubleWidth = 64;
+    this->LongDoubleAlign = 64;
+    this->SizeType = TargetInfo::UnsignedInt;
+    this->PtrDiffType = TargetInfo::SignedInt;
+    this->IntPtrType = TargetInfo::SignedInt;
+    this->RegParmMax = 2;
+    DescriptionString = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-"
+                        "f32:32:32-f64:64:64-p:32:32:32-v128:32:32";
+  }
+
+  void getDefaultFeatures(llvm::StringMap<bool> &Features) const {
+  }
+  virtual void getArchDefines(const LangOptions &Opts,
+                              MacroBuilder &Builder) const {
+    Builder.defineMacro("__le32__");
+    Builder.defineMacro("__pnacl__");
+  }
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    DefineStd(Builder, "unix", Opts);
+    Builder.defineMacro("__ELF__");
+    if (Opts.POSIXThreads)
+      Builder.defineMacro("_REENTRANT");
+    if (Opts.CPlusPlus)
+      Builder.defineMacro("_GNU_SOURCE");
+
+    Builder.defineMacro("__native_client__");
+    getArchDefines(Opts, Builder);
+  }
+  virtual void getTargetBuiltins(const Builtin::Info *&Records,
+                                 unsigned &NumRecords) const {
+  }
+  virtual const char *getVAListDeclaration() const {
+    return "typedef int __builtin_va_list[4];";
+  }
+  virtual void getGCCRegNames(const char * const *&Names,
+                              unsigned &NumNames) const;
+  virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
+                                unsigned &NumAliases) const;
+  virtual bool validateAsmConstraint(const char *&Name,
+                                     TargetInfo::ConstraintInfo &Info) const {
+    return false;
+  }
+
+  virtual const char *getClobbers() const {
+    return "";
+  }
+};
+
+void PNaClTargetInfo::getGCCRegNames(const char * const *&Names,
+                                     unsigned &NumNames) const {
+  Names = NULL;
+  NumNames = 0;
+}
+
+void PNaClTargetInfo::getGCCRegAliases(const GCCRegAlias *&Aliases,
+                                       unsigned &NumAliases) const {
+  Aliases = NULL;
+  NumAliases = 0;
 }
 } // end anonymous namespace.
 
+
 //===----------------------------------------------------------------------===//
 // Driver code
 //===----------------------------------------------------------------------===//
@@ -2845,40 +3535,74 @@ static TargetInfo *AllocateTarget(const std::string &T) {
 
   case llvm::Triple::mips:
     switch (os) {
-    case llvm::Triple::Psp:
-      return new PSPTargetInfo<MipsTargetInfo>(T);
     case llvm::Triple::Linux:
-      return new LinuxTargetInfo<MipsTargetInfo>(T);
+      return new LinuxTargetInfo<Mips32EBTargetInfo>(T);
     case llvm::Triple::RTEMS:
-      return new RTEMSTargetInfo<MipsTargetInfo>(T);
+      return new RTEMSTargetInfo<Mips32EBTargetInfo>(T);
     case llvm::Triple::FreeBSD:
-      return new FreeBSDTargetInfo<MipsTargetInfo>(T);
+      return new FreeBSDTargetInfo<Mips32EBTargetInfo>(T);
     case llvm::Triple::NetBSD:
-      return new NetBSDTargetInfo<MipsTargetInfo>(T);
+      return new NetBSDTargetInfo<Mips32EBTargetInfo>(T);
     default:
-      return new MipsTargetInfo(T);
+      return new Mips32EBTargetInfo(T);
     }
 
   case llvm::Triple::mipsel:
     switch (os) {
-    case llvm::Triple::Psp:
-      return new PSPTargetInfo<MipselTargetInfo>(T);
     case llvm::Triple::Linux:
-      return new LinuxTargetInfo<MipselTargetInfo>(T);
+      return new LinuxTargetInfo<Mips32ELTargetInfo>(T);
     case llvm::Triple::RTEMS:
-      return new RTEMSTargetInfo<MipselTargetInfo>(T);
+      return new RTEMSTargetInfo<Mips32ELTargetInfo>(T);
     case llvm::Triple::FreeBSD:
-      return new FreeBSDTargetInfo<MipselTargetInfo>(T);
+      return new FreeBSDTargetInfo<Mips32ELTargetInfo>(T);
     case llvm::Triple::NetBSD:
-      return new NetBSDTargetInfo<MipselTargetInfo>(T);
+      return new NetBSDTargetInfo<Mips32ELTargetInfo>(T);
     default:
-      return new MipsTargetInfo(T);
+      return new Mips32ELTargetInfo(T);
+    }
+
+  case llvm::Triple::mips64:
+    switch (os) {
+    case llvm::Triple::Linux:
+      return new LinuxTargetInfo<Mips64EBTargetInfo>(T);
+    case llvm::Triple::RTEMS:
+      return new RTEMSTargetInfo<Mips64EBTargetInfo>(T);
+    case llvm::Triple::FreeBSD:
+      return new FreeBSDTargetInfo<Mips64EBTargetInfo>(T);
+    case llvm::Triple::NetBSD:
+      return new NetBSDTargetInfo<Mips64EBTargetInfo>(T);
+    default:
+      return new Mips64EBTargetInfo(T);
+    }
+
+  case llvm::Triple::mips64el:
+    switch (os) {
+    case llvm::Triple::Linux:
+      return new LinuxTargetInfo<Mips64ELTargetInfo>(T);
+    case llvm::Triple::RTEMS:
+      return new RTEMSTargetInfo<Mips64ELTargetInfo>(T);
+    case llvm::Triple::FreeBSD:
+      return new FreeBSDTargetInfo<Mips64ELTargetInfo>(T);
+    case llvm::Triple::NetBSD:
+      return new NetBSDTargetInfo<Mips64ELTargetInfo>(T);
+    default:
+      return new Mips64ELTargetInfo(T);
+    }
+
+  case llvm::Triple::le32:
+    switch (os) {
+      case llvm::Triple::NativeClient:
+        return new PNaClTargetInfo(T);
+      default:
+        return NULL;
     }
 
   case llvm::Triple::ppc:
     if (Triple.isOSDarwin())
       return new DarwinPPC32TargetInfo(T);
     switch (os) {
+    case llvm::Triple::Linux:
+      return new LinuxTargetInfo<PPC32TargetInfo>(T);
     case llvm::Triple::FreeBSD:
       return new FreeBSDTargetInfo<PPC32TargetInfo>(T);
     case llvm::Triple::NetBSD:
@@ -2893,6 +3617,8 @@ static TargetInfo *AllocateTarget(const std::string &T) {
     if (Triple.isOSDarwin())
       return new DarwinPPC64TargetInfo(T);
     switch (os) {
+    case llvm::Triple::Linux:
+      return new LinuxTargetInfo<PPC64TargetInfo>(T);
     case llvm::Triple::Lv2:
       return new PS3PPUTargetInfo<PPC64TargetInfo>(T);
     case llvm::Triple::FreeBSD:
@@ -2913,6 +3639,8 @@ static TargetInfo *AllocateTarget(const std::string &T) {
 
   case llvm::Triple::sparc:
     switch (os) {
+    case llvm::Triple::Linux:
+      return new LinuxTargetInfo<SparcV8TargetInfo>(T);
     case llvm::Triple::AuroraUX:
       return new AuroraUXSparcV8TargetInfo(T);
     case llvm::Triple::Solaris:
@@ -3001,7 +3729,7 @@ static TargetInfo *AllocateTarget(const std::string &T) {
 
 /// CreateTargetInfo - Return the target info object for the specified target
 /// triple.
-TargetInfo *TargetInfo::CreateTargetInfo(Diagnostic &Diags,
+TargetInfo *TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
                                          TargetOptions &Opts) {
   llvm::Triple Triple(Opts.Triple);
 
@@ -3033,7 +3761,7 @@ TargetInfo *TargetInfo::CreateTargetInfo(Diagnostic &Diags,
   // Compute the default target features, we need the target to handle this
   // because features may have dependencies on one another.
   llvm::StringMap<bool> Features;
-  Target->getDefaultFeatures(Opts.CPU, Features);
+  Target->getDefaultFeatures(Features);
 
   // Apply the user specified deltas.
   for (std::vector<std::string>::const_iterator it = Opts.Features.begin(),
diff --git a/contrib/llvm/tools/clang/lib/Basic/Version.cpp b/contrib/llvm/tools/clang/lib/Basic/Version.cpp
index 89076cabc97c..390a6851f74b 100644
--- a/contrib/llvm/tools/clang/lib/Basic/Version.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/Version.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/Version.h"
+#include "clang/Basic/LLVM.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Config/config.h"
 #include <cstring>
@@ -24,14 +25,14 @@ std::string getClangRepositoryPath() {
   return CLANG_REPOSITORY_STRING;
 #else
 #ifdef SVN_REPOSITORY
-  llvm::StringRef URL(SVN_REPOSITORY);
+  StringRef URL(SVN_REPOSITORY);
 #else
-  llvm::StringRef URL("");
+  StringRef URL("");
 #endif
 
   // If the SVN_REPOSITORY is empty, try to use the SVN keyword. This helps us
   // pick up a tag in an SVN export, for example.
-  static llvm::StringRef SVNRepository("$URL: http://llvm.org/svn/llvm-project/cfe/trunk/lib/Basic/Version.cpp $");
+  static StringRef SVNRepository("$URL: http://llvm.org/svn/llvm-project/cfe/branches/release_30/lib/Basic/Version.cpp $");
   if (URL.empty()) {
     URL = SVNRepository.slice(SVNRepository.find(':'),
                               SVNRepository.find("/lib/Basic"));
@@ -42,7 +43,7 @@ std::string getClangRepositoryPath() {
 
   // Trim path prefix off, assuming path came from standard cfe path.
   size_t Start = URL.find("cfe/");
-  if (Start != llvm::StringRef::npos)
+  if (Start != StringRef::npos)
     URL = URL.substr(Start + 4);
 
   return URL;
diff --git a/contrib/llvm/tools/clang/lib/Basic/VersionTuple.cpp b/contrib/llvm/tools/clang/lib/Basic/VersionTuple.cpp
index d5cf126ff487..77aad39cbf2c 100644
--- a/contrib/llvm/tools/clang/lib/Basic/VersionTuple.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/VersionTuple.cpp
@@ -25,7 +25,7 @@ std::string VersionTuple::getAsString() const {
   return Result;
 }
 
-llvm::raw_ostream& clang::operator<<(llvm::raw_ostream &Out, 
+raw_ostream& clang::operator<<(raw_ostream &Out, 
                                      const VersionTuple &V) {
   Out << V.getMajor();
   if (llvm::Optional<unsigned> Minor = V.getMinor())
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp
index 85f42db81f59..b9e3ed9edd19 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp
@@ -15,6 +15,7 @@
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "llvm/Module.h"
 #include "llvm/PassManager.h"
+#include "llvm/Analysis/Verifier.h"
 #include "llvm/Assembly/PrintModulePass.h"
 #include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/CodeGen/RegAllocRegistry.h"
@@ -23,21 +24,24 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/PassManagerBuilder.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegistry.h"
 #include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/IPO/PassManagerBuilder.h"
+#include "llvm/Transforms/Scalar.h"
 using namespace clang;
 using namespace llvm;
 
 namespace {
 
 class EmitAssemblyHelper {
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
   const CodeGenOptions &CodeGenOpts;
   const TargetOptions &TargetOpts;
   const LangOptions &LangOpts;
@@ -82,7 +86,7 @@ class EmitAssemblyHelper {
   bool AddEmitPasses(BackendAction Action, formatted_raw_ostream &OS);
 
 public:
-  EmitAssemblyHelper(Diagnostic &_Diags,
+  EmitAssemblyHelper(DiagnosticsEngine &_Diags,
                      const CodeGenOptions &CGOpts, const TargetOptions &TOpts,
                      const LangOptions &LOpts,
                      Module *M)
@@ -237,27 +241,18 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
   TargetMachine::setDataSections    (CodeGenOpts.DataSections);
 
   // FIXME: Parse this earlier.
-  if (CodeGenOpts.RelocationModel == "static") {
-    TargetMachine::setRelocationModel(llvm::Reloc::Static);
-  } else if (CodeGenOpts.RelocationModel == "pic") {
-    TargetMachine::setRelocationModel(llvm::Reloc::PIC_);
-  } else {
-    assert(CodeGenOpts.RelocationModel == "dynamic-no-pic" &&
-           "Invalid PIC model!");
-    TargetMachine::setRelocationModel(llvm::Reloc::DynamicNoPIC);
-  }
-  // FIXME: Parse this earlier.
+  llvm::CodeModel::Model CM;
   if (CodeGenOpts.CodeModel == "small") {
-    TargetMachine::setCodeModel(llvm::CodeModel::Small);
+    CM = llvm::CodeModel::Small;
   } else if (CodeGenOpts.CodeModel == "kernel") {
-    TargetMachine::setCodeModel(llvm::CodeModel::Kernel);
+    CM = llvm::CodeModel::Kernel;
   } else if (CodeGenOpts.CodeModel == "medium") {
-    TargetMachine::setCodeModel(llvm::CodeModel::Medium);
+    CM = llvm::CodeModel::Medium;
   } else if (CodeGenOpts.CodeModel == "large") {
-    TargetMachine::setCodeModel(llvm::CodeModel::Large);
+    CM = llvm::CodeModel::Large;
   } else {
     assert(CodeGenOpts.CodeModel.empty() && "Invalid code model!");
-    TargetMachine::setCodeModel(llvm::CodeModel::Default);
+    CM = llvm::CodeModel::Default;
   }
 
   std::vector<const char *> BackendArgs;
@@ -274,6 +269,8 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
     BackendArgs.push_back("-time-passes");
   for (unsigned i = 0, e = CodeGenOpts.BackendOptions.size(); i != e; ++i)
     BackendArgs.push_back(CodeGenOpts.BackendOptions[i].c_str());
+  if (CodeGenOpts.NoGlobalMerge)
+    BackendArgs.push_back("-global-merge=false");
   BackendArgs.push_back(0);
   llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1,
                                     const_cast<char **>(&BackendArgs[0]));
@@ -287,8 +284,20 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
       Features.AddFeature(*it);
     FeaturesStr = Features.getString();
   }
+
+  llvm::Reloc::Model RM = llvm::Reloc::Default;
+  if (CodeGenOpts.RelocationModel == "static") {
+    RM = llvm::Reloc::Static;
+  } else if (CodeGenOpts.RelocationModel == "pic") {
+    RM = llvm::Reloc::PIC_;
+  } else {
+    assert(CodeGenOpts.RelocationModel == "dynamic-no-pic" &&
+           "Invalid PIC model!");
+    RM = llvm::Reloc::DynamicNoPIC;
+  }
+
   TargetMachine *TM = TheTarget->createTargetMachine(Triple, TargetOpts.CPU,
-                                                     FeaturesStr);
+                                                     FeaturesStr, RM, CM);
 
   if (CodeGenOpts.RelaxAll)
     TM->setMCRelaxAll(true);
@@ -386,7 +395,8 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) {
   }
 }
 
-void clang::EmitBackendOutput(Diagnostic &Diags, const CodeGenOptions &CGOpts,
+void clang::EmitBackendOutput(DiagnosticsEngine &Diags,
+                              const CodeGenOptions &CGOpts,
                               const TargetOptions &TOpts,
                               const LangOptions &LOpts,
                               Module *M,
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp
index 9815d1d4ef4d..969495376642 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp
@@ -59,10 +59,10 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
                                             const CGBlockInfo &blockInfo) {
   ASTContext &C = CGM.getContext();
 
-  const llvm::Type *ulong = CGM.getTypes().ConvertType(C.UnsignedLongTy);
-  const llvm::Type *i8p = CGM.getTypes().ConvertType(C.VoidPtrTy);
+  llvm::Type *ulong = CGM.getTypes().ConvertType(C.UnsignedLongTy);
+  llvm::Type *i8p = CGM.getTypes().ConvertType(C.VoidPtrTy);
 
-  llvm::SmallVector<llvm::Constant*, 6> elements;
+  SmallVector<llvm::Constant*, 6> elements;
 
   // reserved
   elements.push_back(llvm::ConstantInt::get(ulong, 0));
@@ -243,7 +243,7 @@ static CharUnits getLowBit(CharUnits v) {
 }
 
 static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
-                             llvm::SmallVectorImpl<llvm::Type*> &elementTypes) {
+                             SmallVectorImpl<llvm::Type*> &elementTypes) {
   ASTContext &C = CGM.getContext();
 
   // The header is basically a 'struct { void *; int; int; void *; void *; }'.
@@ -280,7 +280,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
   ASTContext &C = CGM.getContext();
   const BlockDecl *block = info.getBlockDecl();
 
-  llvm::SmallVector<llvm::Type*, 8> elementTypes;
+  SmallVector<llvm::Type*, 8> elementTypes;
   initializeForBlockHeader(CGM, info, elementTypes);
 
   if (!block->hasCaptures()) {
@@ -291,7 +291,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
   }
 
   // Collect the layout chunks.
-  llvm::SmallVector<BlockLayoutChunk, 16> layout;
+  SmallVector<BlockLayoutChunk, 16> layout;
   layout.reserve(block->capturesCXXThis() +
                  (block->capture_end() - block->capture_begin()));
 
@@ -422,7 +422,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
   // which has 7 bytes of padding, as opposed to the naive solution
   // which might have less (?).
   if (endAlign < maxFieldAlign) {
-    llvm::SmallVectorImpl<BlockLayoutChunk>::iterator
+    SmallVectorImpl<BlockLayoutChunk>::iterator
       li = layout.begin() + 1, le = layout.end();
 
     // Look for something that the header end is already
@@ -433,7 +433,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
     // If we found something that's naturally aligned for the end of
     // the header, keep adding things...
     if (li != le) {
-      llvm::SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
+      SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
       for (; li != le; ++li) {
         assert(endAlign >= li->Alignment);
 
@@ -468,7 +468,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
   // Slam everything else on now.  This works because they have
   // strictly decreasing alignment and we expect that size is always a
   // multiple of alignment.
-  for (llvm::SmallVectorImpl<BlockLayoutChunk>::iterator
+  for (SmallVectorImpl<BlockLayoutChunk>::iterator
          li = layout.begin(), le = layout.end(); li != le; ++li) {
     assert(endAlign >= li->Alignment);
     li->setIndex(info, elementTypes.size());
@@ -507,7 +507,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
   // Build the block descriptor.
   llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo);
 
-  const llvm::Type *intTy = ConvertType(getContext().IntTy);
+  llvm::Type *intTy = ConvertType(getContext().IntTy);
 
   llvm::AllocaInst *blockAddr =
     CreateTempAlloca(blockInfo.StructureType, "block");
@@ -617,10 +617,9 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
       ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
                            declRef, VK_RValue);
       EmitExprAsInit(&l2r, &blockFieldPseudoVar,
-                     LValue::MakeAddr(blockField, type,
-                                      getContext().getDeclAlign(variable)
-                                                  .getQuantity(),
-                                      getContext()),
+                     MakeAddrLValue(blockField, type,
+                                    getContext().getDeclAlign(variable)
+                                                .getQuantity()),
                      /*captured by init*/ false);
     }
 
@@ -681,8 +680,8 @@ llvm::Type *CodeGenModule::getBlockDescriptorType() {
   //   const char *layout;      // reserved
   // };
   BlockDescriptorType =
-    llvm::StructType::createNamed("struct.__block_descriptor",
-                                  UnsignedLongTy, UnsignedLongTy, NULL);
+    llvm::StructType::create("struct.__block_descriptor",
+                             UnsignedLongTy, UnsignedLongTy, NULL);
 
   // Now form a pointer to that.
   BlockDescriptorType = llvm::PointerType::getUnqual(BlockDescriptorType);
@@ -703,13 +702,9 @@ llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
   //   struct __block_descriptor *__descriptor;
   // };
   GenericBlockLiteralType =
-    llvm::StructType::createNamed("struct.__block_literal_generic",
-                                  VoidPtrTy,
-                                  IntTy,
-                                  IntTy,
-                                  VoidPtrTy,
-                                  BlockDescPtrTy,
-                                  NULL);
+    llvm::StructType::create("struct.__block_literal_generic",
+                             VoidPtrTy, IntTy, IntTy, VoidPtrTy,
+                             BlockDescPtrTy, NULL);
 
   return GenericBlockLiteralType;
 }
@@ -723,7 +718,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
   llvm::Value *Callee = EmitScalarExpr(E->getCallee());
 
   // Get a pointer to the generic block literal.
-  const llvm::Type *BlockLiteralTy =
+  llvm::Type *BlockLiteralTy =
     llvm::PointerType::getUnqual(CGM.getGenericBlockLiteralType());
 
   // Bitcast the callee to a block literal.
@@ -731,9 +726,9 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
     Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal");
 
   // Get the function pointer from the literal.
-  llvm::Value *FuncPtr = Builder.CreateStructGEP(BlockLiteral, 3, "tmp");
+  llvm::Value *FuncPtr = Builder.CreateStructGEP(BlockLiteral, 3);
 
-  BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy, "tmp");
+  BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy);
 
   // Add the block literal.
   CallArgList Args;
@@ -746,20 +741,16 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
                E->arg_begin(), E->arg_end());
 
   // Load the function.
-  llvm::Value *Func = Builder.CreateLoad(FuncPtr, "tmp");
+  llvm::Value *Func = Builder.CreateLoad(FuncPtr);
 
   const FunctionType *FuncTy = FnType->castAs<FunctionType>();
-  QualType ResultType = FuncTy->getResultType();
-
-  const CGFunctionInfo &FnInfo =
-    CGM.getTypes().getFunctionInfo(ResultType, Args,
-                                   FuncTy->getExtInfo());
+  const CGFunctionInfo &FnInfo = CGM.getTypes().getFunctionInfo(Args, FuncTy);
 
   // Cast the function pointer to the right type.
-  const llvm::Type *BlockFTy =
+  llvm::Type *BlockFTy =
     CGM.getTypes().GetFunctionType(FnInfo, false);
 
-  const llvm::Type *BlockFTyPtr = llvm::PointerType::getUnqual(BlockFTy);
+  llvm::Type *BlockFTyPtr = llvm::PointerType::getUnqual(BlockFTy);
   Func = Builder.CreateBitCast(Func, BlockFTyPtr);
 
   // And call the block.
@@ -783,7 +774,7 @@ llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable,
     // to byref*.
 
     addr = Builder.CreateLoad(addr);
-    const llvm::PointerType *byrefPointerType
+    llvm::PointerType *byrefPointerType
       = llvm::PointerType::get(BuildByRefType(variable), 0);
     addr = Builder.CreateBitCast(addr, byrefPointerType,
                                  "byref.addr");
@@ -863,7 +854,7 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
   literal->setAlignment(blockInfo.BlockAlign.getQuantity());
 
   // Return a constant of the appropriately-casted type.
-  const llvm::Type *requiredType =
+  llvm::Type *requiredType =
     CGM.getTypes().ConvertType(blockInfo.getBlockExpr()->getType());
   return llvm::ConstantExpr::getBitCast(literal, requiredType);
 }
@@ -918,7 +909,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
   if (CGM.ReturnTypeUsesSRet(fnInfo))
     blockInfo.UsesStret = true;
 
-  const llvm::FunctionType *fnLLVMType =
+  llvm::FunctionType *fnLLVMType =
     CGM.getTypes().GetFunctionType(fnInfo, fnType->isVariadic());
 
   MangleBuffer name;
@@ -1005,7 +996,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
     for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
            ce = blockDecl->capture_end(); ci != ce; ++ci) {
       const VarDecl *variable = ci->getVariable();
-      DI->setLocation(variable->getLocation());
+      DI->EmitLocation(Builder, variable->getLocation());
 
       const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
       if (capture.isConstant()) {
@@ -1065,7 +1056,7 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
 
   // FIXME: it would be nice if these were mergeable with things with
   // identical semantics.
-  const llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI, false);
+  llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI, false);
 
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
@@ -1088,7 +1079,7 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
                                           true);
   StartFunction(FD, C.VoidTy, Fn, FI, args, SourceLocation());
 
-  const llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
+  llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
 
   llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
   src = Builder.CreateLoad(src);
@@ -1180,7 +1171,7 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
-  const llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI, false);
+  llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI, false);
 
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
@@ -1201,7 +1192,7 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
                                           false, true);
   StartFunction(FD, C.VoidTy, Fn, FI, args, SourceLocation());
 
-  const llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
+  llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
 
   llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
   src = Builder.CreateLoad(src);
@@ -1399,7 +1390,7 @@ class CXXByrefHelpers : public CodeGenModule::ByrefHelpers {
 
 static llvm::Constant *
 generateByrefCopyHelper(CodeGenFunction &CGF,
-                        const llvm::StructType &byrefType,
+                        llvm::StructType &byrefType,
                         CodeGenModule::ByrefHelpers &byrefInfo) {
   ASTContext &Context = CGF.getContext();
 
@@ -1416,7 +1407,7 @@ generateByrefCopyHelper(CodeGenFunction &CGF,
     CGF.CGM.getTypes().getFunctionInfo(R, args, FunctionType::ExtInfo());
 
   CodeGenTypes &Types = CGF.CGM.getTypes();
-  const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
+  llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
@@ -1438,7 +1429,7 @@ generateByrefCopyHelper(CodeGenFunction &CGF,
   CGF.StartFunction(FD, R, Fn, FI, args, SourceLocation());
 
   if (byrefInfo.needsCopy()) {
-    const llvm::Type *byrefPtrType = byrefType.getPointerTo(0);
+    llvm::Type *byrefPtrType = byrefType.getPointerTo(0);
 
     // dst->x
     llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst);
@@ -1462,7 +1453,7 @@ generateByrefCopyHelper(CodeGenFunction &CGF,
 
 /// Build the copy helper for a __block variable.
 static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
-                                            const llvm::StructType &byrefType,
+                                            llvm::StructType &byrefType,
                                             CodeGenModule::ByrefHelpers &info) {
   CodeGenFunction CGF(CGM);
   return generateByrefCopyHelper(CGF, byrefType, info);
@@ -1471,7 +1462,7 @@ static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
 /// Generate code for a __block variable's dispose helper.
 static llvm::Constant *
 generateByrefDisposeHelper(CodeGenFunction &CGF,
-                           const llvm::StructType &byrefType,
+                           llvm::StructType &byrefType,
                            CodeGenModule::ByrefHelpers &byrefInfo) {
   ASTContext &Context = CGF.getContext();
   QualType R = Context.VoidTy;
@@ -1484,7 +1475,7 @@ generateByrefDisposeHelper(CodeGenFunction &CGF,
     CGF.CGM.getTypes().getFunctionInfo(R, args, FunctionType::ExtInfo());
 
   CodeGenTypes &Types = CGF.CGM.getTypes();
-  const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
+  llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
@@ -1521,7 +1512,7 @@ generateByrefDisposeHelper(CodeGenFunction &CGF,
 
 /// Build the dispose helper for a __block variable.
 static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
-                                              const llvm::StructType &byrefType,
+                                              llvm::StructType &byrefType,
                                             CodeGenModule::ByrefHelpers &info) {
   CodeGenFunction CGF(CGM);
   return generateByrefDisposeHelper(CGF, byrefType, info);
@@ -1529,7 +1520,7 @@ static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
 
 /// 
 template <class T> static T *buildByrefHelpers(CodeGenModule &CGM,
-                                               const llvm::StructType &byrefTy,
+                                               llvm::StructType &byrefTy,
                                                T &byrefInfo) {
   // Increase the field's alignment to be at least pointer alignment,
   // since the layout of the byref struct will guarantee at least that.
@@ -1553,7 +1544,7 @@ template <class T> static T *buildByrefHelpers(CodeGenModule &CGM,
 }
 
 CodeGenModule::ByrefHelpers *
-CodeGenFunction::buildByrefHelpers(const llvm::StructType &byrefType,
+CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
                                    const AutoVarEmission &emission) {
   const VarDecl &var = *emission.Variable;
   QualType type = var.getType();
@@ -1658,18 +1649,18 @@ llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr,
 ///        T x;
 ///      } x
 ///
-const llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) {
-  std::pair<const llvm::Type *, unsigned> &Info = ByRefValueInfo[D];
+llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) {
+  std::pair<llvm::Type *, unsigned> &Info = ByRefValueInfo[D];
   if (Info.first)
     return Info.first;
   
   QualType Ty = D->getType();
 
-  llvm::SmallVector<llvm::Type *, 8> types;
+  SmallVector<llvm::Type *, 8> types;
   
   llvm::StructType *ByRefType =
-    llvm::StructType::createNamed(getLLVMContext(),
-                                "struct.__block_byref_" + D->getNameAsString());
+    llvm::StructType::create(getLLVMContext(),
+                             "struct.__block_byref_" + D->getNameAsString());
   
   // void *__isa;
   types.push_back(Int8PtrTy);
@@ -1742,7 +1733,7 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   llvm::Value *addr = emission.Address;
 
   // That's an alloca of the byref structure type.
-  const llvm::StructType *byrefType = cast<llvm::StructType>(
+  llvm::StructType *byrefType = cast<llvm::StructType>(
                  cast<llvm::PointerType>(addr->getType())->getElementType());
 
   // Build the byref helpers if necessary.  This is null if we don't need any.
@@ -1812,8 +1803,63 @@ namespace {
 /// to be done externally.
 void CodeGenFunction::enterByrefCleanup(const AutoVarEmission &emission) {
   // We don't enter this cleanup if we're in pure-GC mode.
-  if (CGM.getLangOptions().getGCMode() == LangOptions::GCOnly)
+  if (CGM.getLangOptions().getGC() == LangOptions::GCOnly)
     return;
 
   EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup, emission.Address);
 }
+
+/// Adjust the declaration of something from the blocks API.
+static void configureBlocksRuntimeObject(CodeGenModule &CGM,
+                                         llvm::Constant *C) {
+  if (!CGM.getLangOptions().BlocksRuntimeOptional) return;
+
+  llvm::GlobalValue *GV = cast<llvm::GlobalValue>(C->stripPointerCasts());
+  if (GV->isDeclaration() &&
+      GV->getLinkage() == llvm::GlobalValue::ExternalLinkage)
+    GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
+}
+
+llvm::Constant *CodeGenModule::getBlockObjectDispose() {
+  if (BlockObjectDispose)
+    return BlockObjectDispose;
+
+  llvm::Type *args[] = { Int8PtrTy, Int32Ty };
+  llvm::FunctionType *fty
+    = llvm::FunctionType::get(VoidTy, args, false);
+  BlockObjectDispose = CreateRuntimeFunction(fty, "_Block_object_dispose");
+  configureBlocksRuntimeObject(*this, BlockObjectDispose);
+  return BlockObjectDispose;
+}
+
+llvm::Constant *CodeGenModule::getBlockObjectAssign() {
+  if (BlockObjectAssign)
+    return BlockObjectAssign;
+
+  llvm::Type *args[] = { Int8PtrTy, Int8PtrTy, Int32Ty };
+  llvm::FunctionType *fty
+    = llvm::FunctionType::get(VoidTy, args, false);
+  BlockObjectAssign = CreateRuntimeFunction(fty, "_Block_object_assign");
+  configureBlocksRuntimeObject(*this, BlockObjectAssign);
+  return BlockObjectAssign;
+}
+
+llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
+  if (NSConcreteGlobalBlock)
+    return NSConcreteGlobalBlock;
+
+  NSConcreteGlobalBlock = GetOrCreateLLVMGlobal("_NSConcreteGlobalBlock",
+                                                Int8PtrTy->getPointerTo(), 0);
+  configureBlocksRuntimeObject(*this, NSConcreteGlobalBlock);
+  return NSConcreteGlobalBlock;
+}
+
+llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
+  if (NSConcreteStackBlock)
+    return NSConcreteStackBlock;
+
+  NSConcreteStackBlock = GetOrCreateLLVMGlobal("_NSConcreteStackBlock",
+                                               Int8PtrTy->getPointerTo(), 0);
+  configureBlocksRuntimeObject(*this, NSConcreteStackBlock);
+  return NSConcreteStackBlock;  
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h
index 4d8dead2be8c..6e71c1fdc041 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h
@@ -176,7 +176,7 @@ class CGBlockInfo {
   /// because it gets set later in the block-creation process.
   mutable bool UsesStret : 1;
 
-  const llvm::StructType *StructureType;
+  llvm::StructType *StructureType;
   const BlockExpr *Block;
   CharUnits BlockSize;
   CharUnits BlockAlign;
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp
index 1566bd9e6697..ec0ca424220e 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp
@@ -27,24 +27,34 @@ using namespace clang;
 using namespace CodeGen;
 using namespace llvm;
 
-static void EmitMemoryBarrier(CodeGenFunction &CGF,
-                              bool LoadLoad, bool LoadStore,
-                              bool StoreLoad, bool StoreStore,
-                              bool Device) {
-  Value *True = CGF.Builder.getTrue();
-  Value *False = CGF.Builder.getFalse();
-  Value *C[5] = { LoadLoad ? True : False,
-                  LoadStore ? True : False,
-                  StoreLoad ? True : False,
-                  StoreStore ? True : False,
-                  Device ? True : False };
-  CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(Intrinsic::memory_barrier), C);
+/// getBuiltinLibFunction - Given a builtin id for a function like
+/// "__builtin_fabsf", return a Function* for "fabsf".
+llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
+                                                  unsigned BuiltinID) {
+  assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
+
+  // Get the name, skip over the __builtin_ prefix (if necessary).
+  StringRef Name;
+  GlobalDecl D(FD);
+
+  // If the builtin has been declared explicitly with an assembler label,
+  // use the mangled name. This differs from the plain label on platforms
+  // that prefix labels.
+  if (FD->hasAttr<AsmLabelAttr>())
+    Name = getMangledName(D);
+  else
+    Name = Context.BuiltinInfo.GetName(BuiltinID) + 10;
+
+  llvm::FunctionType *Ty =
+    cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
+
+  return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
 }
 
 /// Emit the conversions required to turn the given value into an
 /// integer of the given size.
 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
-                        QualType T, const llvm::IntegerType *IntType) {
+                        QualType T, llvm::IntegerType *IntType) {
   V = CGF.EmitToMemory(V, T);
 
   if (V->getType()->isPointerTy())
@@ -55,7 +65,7 @@ static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
 }
 
 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
-                          QualType T, const llvm::Type *ResultType) {
+                          QualType T, llvm::Type *ResultType) {
   V = CGF.EmitFromMemory(V, T);
 
   if (ResultType->isPointerTy())
@@ -65,25 +75,11 @@ static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
   return V;
 }
 
-// The atomic builtins are also full memory barriers. This is a utility for
-// wrapping a call to the builtins with memory barriers.
-static Value *EmitCallWithBarrier(CodeGenFunction &CGF, Value *Fn,
-                                  ArrayRef<Value *> Args) {
-  // FIXME: We need a target hook for whether this applies to device memory or
-  // not.
-  bool Device = true;
-
-  // Create barriers both before and after the call.
-  EmitMemoryBarrier(CGF, true, true, true, true, Device);
-  Value *Result = CGF.Builder.CreateCall(Fn, Args);
-  EmitMemoryBarrier(CGF, true, true, true, true, Device);
-  return Result;
-}
-
 /// Utility to insert an atomic instruction based on Instrinsic::ID
 /// and the expression node.
 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
-                               Intrinsic::ID Id, const CallExpr *E) {
+                               llvm::AtomicRMWInst::BinOp Kind,
+                               const CallExpr *E) {
   QualType T = E->getType();
   assert(E->getArg(0)->getType()->isPointerType());
   assert(CGF.getContext().hasSameUnqualifiedType(T,
@@ -99,16 +95,15 @@ static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
                            CGF.getContext().getTypeSize(T));
   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 
-  llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
-  llvm::Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes);
-
   llvm::Value *Args[2];
   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
-  const llvm::Type *ValueType = Args[1]->getType();
+  llvm::Type *ValueType = Args[1]->getType();
   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 
-  llvm::Value *Result = EmitCallWithBarrier(CGF, AtomF, Args);
+  llvm::Value *Result =
+      CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
+                                  llvm::SequentiallyConsistent);
   Result = EmitFromInt(CGF, Result, T, ValueType);
   return RValue::get(Result);
 }
@@ -117,7 +112,8 @@ static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
 /// the expression node, where the return value is the result of the
 /// operation.
 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
-                                   Intrinsic::ID Id, const CallExpr *E,
+                                   llvm::AtomicRMWInst::BinOp Kind,
+                                   const CallExpr *E,
                                    Instruction::BinaryOps Op) {
   QualType T = E->getType();
   assert(E->getArg(0)->getType()->isPointerType());
@@ -134,16 +130,15 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
                            CGF.getContext().getTypeSize(T));
   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 
-  llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
-  llvm::Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes);
-
   llvm::Value *Args[2];
   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
-  const llvm::Type *ValueType = Args[1]->getType();
+  llvm::Type *ValueType = Args[1]->getType();
   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 
-  llvm::Value *Result = EmitCallWithBarrier(CGF, AtomF, Args);
+  llvm::Value *Result =
+      CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
+                                  llvm::SequentiallyConsistent);
   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
   Result = EmitFromInt(CGF, Result, T, ValueType);
   return RValue::get(Result);
@@ -157,21 +152,26 @@ static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) {
   
   StringRef FnName;
   switch (ValTyP->getKind()) {
-  default: assert(0 && "Isn't a scalar fp type!");
+  default: llvm_unreachable("Isn't a scalar fp type!");
   case BuiltinType::Float:      FnName = "fabsf"; break;
   case BuiltinType::Double:     FnName = "fabs"; break;
   case BuiltinType::LongDouble: FnName = "fabsl"; break;
   }
   
   // The prototype is something that takes and returns whatever V's type is.
-  llvm::Type *ArgTys[] = { V->getType() };
-  llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), ArgTys,
+  llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), V->getType(),
                                                    false);
   llvm::Value *Fn = CGF.CGM.CreateRuntimeFunction(FT, FnName);
 
   return CGF.Builder.CreateCall(Fn, V, "abs");
 }
 
+static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn,
+                              const CallExpr *E, llvm::Value *calleeValue) {
+  return CGF.EmitCall(E->getCallee()->getType(), calleeValue,
+                      ReturnValueSlot(), E->arg_begin(), E->arg_end(), Fn);
+}
+
 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
                                         unsigned BuiltinID, const CallExpr *E) {
   // See if we can constant fold this builtin.  If so, don't emit it at all.
@@ -195,7 +195,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__builtin_va_start:
   case Builtin::BI__builtin_va_end: {
     Value *ArgValue = EmitVAListRef(E->getArg(0));
-    const llvm::Type *DestType = Int8PtrTy;
+    llvm::Type *DestType = Int8PtrTy;
     if (ArgValue->getType() != DestType)
       ArgValue = Builder.CreateBitCast(ArgValue, DestType,
                                        ArgValue->getName().data());
@@ -208,7 +208,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *DstPtr = EmitVAListRef(E->getArg(0));
     Value *SrcPtr = EmitVAListRef(E->getArg(1));
 
-    const llvm::Type *Type = Int8PtrTy;
+    llvm::Type *Type = Int8PtrTy;
 
     DstPtr = Builder.CreateBitCast(DstPtr, Type);
     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
@@ -236,8 +236,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::Type *ArgType = ArgValue->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
 
-    const llvm::Type *ResultType = ConvertType(E->getType());
-    Value *Result = Builder.CreateCall(F, ArgValue, "tmp");
+    llvm::Type *ResultType = ConvertType(E->getType());
+    Value *Result = Builder.CreateCall(F, ArgValue);
     if (Result->getType() != ResultType)
       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
                                      "cast");
@@ -251,8 +251,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::Type *ArgType = ArgValue->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
 
-    const llvm::Type *ResultType = ConvertType(E->getType());
-    Value *Result = Builder.CreateCall(F, ArgValue, "tmp");
+    llvm::Type *ResultType = ConvertType(E->getType());
+    Value *Result = Builder.CreateCall(F, ArgValue);
     if (Result->getType() != ResultType)
       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
                                      "cast");
@@ -267,9 +267,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::Type *ArgType = ArgValue->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
 
-    const llvm::Type *ResultType = ConvertType(E->getType());
-    Value *Tmp = Builder.CreateAdd(Builder.CreateCall(F, ArgValue, "tmp"),
-                                   llvm::ConstantInt::get(ArgType, 1), "tmp");
+    llvm::Type *ResultType = ConvertType(E->getType());
+    Value *Tmp = Builder.CreateAdd(Builder.CreateCall(F, ArgValue),
+                                   llvm::ConstantInt::get(ArgType, 1));
     Value *Zero = llvm::Constant::getNullValue(ArgType);
     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
@@ -287,10 +287,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::Type *ArgType = ArgValue->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
 
-    const llvm::Type *ResultType = ConvertType(E->getType());
-    Value *Tmp = Builder.CreateCall(F, ArgValue, "tmp");
-    Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1),
-                                      "tmp");
+    llvm::Type *ResultType = ConvertType(E->getType());
+    Value *Tmp = Builder.CreateCall(F, ArgValue);
+    Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
     if (Result->getType() != ResultType)
       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
                                      "cast");
@@ -304,8 +303,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::Type *ArgType = ArgValue->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
 
-    const llvm::Type *ResultType = ConvertType(E->getType());
-    Value *Result = Builder.CreateCall(F, ArgValue, "tmp");
+    llvm::Type *ResultType = ConvertType(E->getType());
+    Value *Result = Builder.CreateCall(F, ArgValue);
     if (Result->getType() != ResultType)
       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
                                      "cast");
@@ -327,7 +326,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *ArgValue = EmitScalarExpr(E->getArg(0));
     llvm::Type *ArgType = ArgValue->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType);
-    return RValue::get(Builder.CreateCall(F, ArgValue, "tmp"));
+    return RValue::get(Builder.CreateCall(F, ArgValue));
   }
   case Builtin::BI__builtin_object_size: {
     // We pass this builtin onto the optimizer so that it can
@@ -381,7 +380,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *Exponent = EmitScalarExpr(E->getArg(1));
     llvm::Type *ArgType = Base->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
-    return RValue::get(Builder.CreateCall2(F, Base, Exponent, "tmp"));
+    return RValue::get(Builder.CreateCall2(F, Base, Exponent));
   }
 
   case Builtin::BI__builtin_isgreater:
@@ -396,7 +395,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *RHS = EmitScalarExpr(E->getArg(1));
 
     switch (BuiltinID) {
-    default: assert(0 && "Unknown ordered comparison");
+    default: llvm_unreachable("Unknown ordered comparison");
     case Builtin::BI__builtin_isgreater:
       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
       break;
@@ -417,13 +416,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
       break;
     }
     // ZExt bool to int type.
-    return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()),
-                                          "tmp"));
+    return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
   }
   case Builtin::BI__builtin_isnan: {
     Value *V = EmitScalarExpr(E->getArg(0));
     V = Builder.CreateFCmpUNO(V, V, "cmp");
-    return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()), "tmp"));
+    return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
   }
   
   case Builtin::BI__builtin_isinf: {
@@ -432,7 +430,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     V = EmitFAbs(*this, V, E->getArg(0)->getType());
     
     V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf");
-    return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()), "tmp"));
+    return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
   }
       
   // TODO: BI__builtin_isinf_sign
@@ -457,7 +455,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   }
 
   case Builtin::BI__builtin_isfinite: {
-    // isfinite(x) --> x == x && fabs(x) != infinity; }
+    // isfinite(x) --> x == x && fabs(x) != infinity;
     Value *V = EmitScalarExpr(E->getArg(0));
     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
     
@@ -471,7 +469,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
   case Builtin::BI__builtin_fpclassify: {
     Value *V = EmitScalarExpr(E->getArg(5));
-    const llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
+    llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
 
     // Create Result
     BasicBlock *Begin = Builder.GetInsertBlock();
@@ -530,7 +528,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BIalloca:
   case Builtin::BI__builtin_alloca: {
     Value *Size = EmitScalarExpr(E->getArg(0));
-    return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size, "tmp"));
+    return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size));
   }
   case Builtin::BIbzero:
   case Builtin::BI__builtin_bzero: {
@@ -550,11 +548,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
       
   case Builtin::BI__builtin___memcpy_chk: {
     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
-    if (!E->getArg(2)->isEvaluatable(CGM.getContext()) ||
-        !E->getArg(3)->isEvaluatable(CGM.getContext()))
+    llvm::APSInt Size, DstSize;
+    if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
+        !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
       break;
-    llvm::APSInt Size = E->getArg(2)->EvaluateAsInt(CGM.getContext());
-    llvm::APSInt DstSize = E->getArg(3)->EvaluateAsInt(CGM.getContext());
     if (Size.ugt(DstSize))
       break;
     Value *Dest = EmitScalarExpr(E->getArg(0));
@@ -575,11 +572,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
   case Builtin::BI__builtin___memmove_chk: {
     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
-    if (!E->getArg(2)->isEvaluatable(CGM.getContext()) ||
-        !E->getArg(3)->isEvaluatable(CGM.getContext()))
+    llvm::APSInt Size, DstSize;
+    if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
+        !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
       break;
-    llvm::APSInt Size = E->getArg(2)->EvaluateAsInt(CGM.getContext());
-    llvm::APSInt DstSize = E->getArg(3)->EvaluateAsInt(CGM.getContext());
     if (Size.ugt(DstSize))
       break;
     Value *Dest = EmitScalarExpr(E->getArg(0));
@@ -608,11 +604,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   }
   case Builtin::BI__builtin___memset_chk: {
     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
-    if (!E->getArg(2)->isEvaluatable(CGM.getContext()) ||
-        !E->getArg(3)->isEvaluatable(CGM.getContext()))
+    llvm::APSInt Size, DstSize;
+    if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
+        !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
       break;
-    llvm::APSInt Size = E->getArg(2)->EvaluateAsInt(CGM.getContext());
-    llvm::APSInt DstSize = E->getArg(3)->EvaluateAsInt(CGM.getContext());
     if (Size.ugt(DstSize))
       break;
     Value *Address = EmitScalarExpr(E->getArg(0));
@@ -640,13 +635,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   }
   case Builtin::BI__builtin_return_address: {
     Value *Depth = EmitScalarExpr(E->getArg(0));
-    Depth = Builder.CreateIntCast(Depth, Int32Ty, false, "tmp");
+    Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
     return RValue::get(Builder.CreateCall(F, Depth));
   }
   case Builtin::BI__builtin_frame_address: {
     Value *Depth = EmitScalarExpr(E->getArg(0));
-    Depth = Builder.CreateIntCast(Depth, Int32Ty, false, "tmp");
+    Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
     Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
     return RValue::get(Builder.CreateCall(F, Depth));
   }
@@ -661,7 +656,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     return RValue::get(Result);
   }
   case Builtin::BI__builtin_dwarf_sp_column: {
-    const llvm::IntegerType *Ty
+    llvm::IntegerType *Ty
       = cast<llvm::IntegerType>(ConvertType(E->getType()));
     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
     if (Column == -1) {
@@ -680,7 +675,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *Int = EmitScalarExpr(E->getArg(0));
     Value *Ptr = EmitScalarExpr(E->getArg(1));
 
-    const llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
+    llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
     Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
@@ -775,82 +770,82 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_lock_test_and_set:
   case Builtin::BI__sync_lock_release:
   case Builtin::BI__sync_swap:
-    assert(0 && "Shouldn't make it through sema");
+    llvm_unreachable("Shouldn't make it through sema");
   case Builtin::BI__sync_fetch_and_add_1:
   case Builtin::BI__sync_fetch_and_add_2:
   case Builtin::BI__sync_fetch_and_add_4:
   case Builtin::BI__sync_fetch_and_add_8:
   case Builtin::BI__sync_fetch_and_add_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_add, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
   case Builtin::BI__sync_fetch_and_sub_1:
   case Builtin::BI__sync_fetch_and_sub_2:
   case Builtin::BI__sync_fetch_and_sub_4:
   case Builtin::BI__sync_fetch_and_sub_8:
   case Builtin::BI__sync_fetch_and_sub_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_sub, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
   case Builtin::BI__sync_fetch_and_or_1:
   case Builtin::BI__sync_fetch_and_or_2:
   case Builtin::BI__sync_fetch_and_or_4:
   case Builtin::BI__sync_fetch_and_or_8:
   case Builtin::BI__sync_fetch_and_or_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_or, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
   case Builtin::BI__sync_fetch_and_and_1:
   case Builtin::BI__sync_fetch_and_and_2:
   case Builtin::BI__sync_fetch_and_and_4:
   case Builtin::BI__sync_fetch_and_and_8:
   case Builtin::BI__sync_fetch_and_and_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_and, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
   case Builtin::BI__sync_fetch_and_xor_1:
   case Builtin::BI__sync_fetch_and_xor_2:
   case Builtin::BI__sync_fetch_and_xor_4:
   case Builtin::BI__sync_fetch_and_xor_8:
   case Builtin::BI__sync_fetch_and_xor_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_xor, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
 
   // Clang extensions: not overloaded yet.
   case Builtin::BI__sync_fetch_and_min:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_min, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
   case Builtin::BI__sync_fetch_and_max:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_max, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
   case Builtin::BI__sync_fetch_and_umin:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_umin, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
   case Builtin::BI__sync_fetch_and_umax:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_load_umax, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
 
   case Builtin::BI__sync_add_and_fetch_1:
   case Builtin::BI__sync_add_and_fetch_2:
   case Builtin::BI__sync_add_and_fetch_4:
   case Builtin::BI__sync_add_and_fetch_8:
   case Builtin::BI__sync_add_and_fetch_16:
-    return EmitBinaryAtomicPost(*this, Intrinsic::atomic_load_add, E,
+    return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
                                 llvm::Instruction::Add);
   case Builtin::BI__sync_sub_and_fetch_1:
   case Builtin::BI__sync_sub_and_fetch_2:
   case Builtin::BI__sync_sub_and_fetch_4:
   case Builtin::BI__sync_sub_and_fetch_8:
   case Builtin::BI__sync_sub_and_fetch_16:
-    return EmitBinaryAtomicPost(*this, Intrinsic::atomic_load_sub, E,
+    return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
                                 llvm::Instruction::Sub);
   case Builtin::BI__sync_and_and_fetch_1:
   case Builtin::BI__sync_and_and_fetch_2:
   case Builtin::BI__sync_and_and_fetch_4:
   case Builtin::BI__sync_and_and_fetch_8:
   case Builtin::BI__sync_and_and_fetch_16:
-    return EmitBinaryAtomicPost(*this, Intrinsic::atomic_load_and, E,
+    return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
                                 llvm::Instruction::And);
   case Builtin::BI__sync_or_and_fetch_1:
   case Builtin::BI__sync_or_and_fetch_2:
   case Builtin::BI__sync_or_and_fetch_4:
   case Builtin::BI__sync_or_and_fetch_8:
   case Builtin::BI__sync_or_and_fetch_16:
-    return EmitBinaryAtomicPost(*this, Intrinsic::atomic_load_or, E,
+    return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
                                 llvm::Instruction::Or);
   case Builtin::BI__sync_xor_and_fetch_1:
   case Builtin::BI__sync_xor_and_fetch_2:
   case Builtin::BI__sync_xor_and_fetch_4:
   case Builtin::BI__sync_xor_and_fetch_8:
   case Builtin::BI__sync_xor_and_fetch_16:
-    return EmitBinaryAtomicPost(*this, Intrinsic::atomic_load_xor, E,
+    return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
                                 llvm::Instruction::Xor);
 
   case Builtin::BI__sync_val_compare_and_swap_1:
@@ -867,18 +862,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
       llvm::IntegerType::get(getLLVMContext(),
                              getContext().getTypeSize(T));
     llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
-    llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
-    Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap,
-                                    IntrinsicTypes);
 
     Value *Args[3];
     Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
     Args[1] = EmitScalarExpr(E->getArg(1));
-    const llvm::Type *ValueType = Args[1]->getType();
+    llvm::Type *ValueType = Args[1]->getType();
     Args[1] = EmitToInt(*this, Args[1], T, IntType);
     Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
 
-    Value *Result = EmitCallWithBarrier(*this, AtomF, Args);
+    Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
+                                                llvm::SequentiallyConsistent);
     Result = EmitFromInt(*this, Result, T, ValueType);
     return RValue::get(Result);
   }
@@ -897,9 +890,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
       llvm::IntegerType::get(getLLVMContext(),
                              getContext().getTypeSize(T));
     llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
-    llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
-    Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap,
-                                    IntrinsicTypes);
 
     Value *Args[3];
     Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
@@ -907,7 +897,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
 
     Value *OldVal = Args[1];
-    Value *PrevVal = EmitCallWithBarrier(*this, AtomF, Args);
+    Value *PrevVal = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
+                                                 llvm::SequentiallyConsistent);
     Value *Result = Builder.CreateICmpEQ(PrevVal, OldVal);
     // zext bool to int.
     Result = Builder.CreateZExt(Result, ConvertType(E->getType()));
@@ -919,14 +910,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_swap_4:
   case Builtin::BI__sync_swap_8:
   case Builtin::BI__sync_swap_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_swap, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
 
   case Builtin::BI__sync_lock_test_and_set_1:
   case Builtin::BI__sync_lock_test_and_set_2:
   case Builtin::BI__sync_lock_test_and_set_4:
   case Builtin::BI__sync_lock_test_and_set_8:
   case Builtin::BI__sync_lock_test_and_set_16:
-    return EmitBinaryAtomic(*this, Intrinsic::atomic_swap, E);
+    return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
 
   case Builtin::BI__sync_lock_release_1:
   case Builtin::BI__sync_lock_release_2:
@@ -934,32 +925,95 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_lock_release_8:
   case Builtin::BI__sync_lock_release_16: {
     Value *Ptr = EmitScalarExpr(E->getArg(0));
-    const llvm::Type *ElTy =
+    llvm::Type *ElLLVMTy =
       cast<llvm::PointerType>(Ptr->getType())->getElementType();
     llvm::StoreInst *Store = 
-      Builder.CreateStore(llvm::Constant::getNullValue(ElTy), Ptr);
-    Store->setVolatile(true);
+      Builder.CreateStore(llvm::Constant::getNullValue(ElLLVMTy), Ptr);
+    QualType ElTy = E->getArg(0)->getType()->getPointeeType();
+    CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
+    Store->setAlignment(StoreSize.getQuantity());
+    Store->setAtomic(llvm::Release);
     return RValue::get(0);
   }
 
   case Builtin::BI__sync_synchronize: {
-    // We assume like gcc appears to, that this only applies to cached memory.
-    EmitMemoryBarrier(*this, true, true, true, true, false);
+    // We assume this is supposed to correspond to a C++0x-style
+    // sequentially-consistent fence (i.e. this is only usable for
+    // synchonization, not device I/O or anything like that). This intrinsic
+    // is really badly designed in the sense that in theory, there isn't 
+    // any way to safely use it... but in practice, it mostly works
+    // to use it with non-atomic loads and stores to get acquire/release
+    // semantics.
+    Builder.CreateFence(llvm::SequentiallyConsistent);
     return RValue::get(0);
   }
 
-  case Builtin::BI__builtin_llvm_memory_barrier: {
-    Value *C[5] = {
-      EmitScalarExpr(E->getArg(0)),
-      EmitScalarExpr(E->getArg(1)),
-      EmitScalarExpr(E->getArg(2)),
-      EmitScalarExpr(E->getArg(3)),
-      EmitScalarExpr(E->getArg(4))
-    };
-    Builder.CreateCall(CGM.getIntrinsic(Intrinsic::memory_barrier), C);
+  case Builtin::BI__atomic_thread_fence:
+  case Builtin::BI__atomic_signal_fence: {
+    llvm::SynchronizationScope Scope;
+    if (BuiltinID == Builtin::BI__atomic_signal_fence)
+      Scope = llvm::SingleThread;
+    else
+      Scope = llvm::CrossThread;
+    Value *Order = EmitScalarExpr(E->getArg(0));
+    if (isa<llvm::ConstantInt>(Order)) {
+      int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+      switch (ord) {
+      case 0:  // memory_order_relaxed
+      default: // invalid order
+        break;
+      case 1:  // memory_order_consume
+      case 2:  // memory_order_acquire
+        Builder.CreateFence(llvm::Acquire, Scope);
+        break;
+      case 3:  // memory_order_release
+        Builder.CreateFence(llvm::Release, Scope);
+        break;
+      case 4:  // memory_order_acq_rel
+        Builder.CreateFence(llvm::AcquireRelease, Scope);
+        break;
+      case 5:  // memory_order_seq_cst
+        Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
+        break;
+      }
+      return RValue::get(0);
+    }
+
+    llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
+    AcquireBB = createBasicBlock("acquire", CurFn);
+    ReleaseBB = createBasicBlock("release", CurFn);
+    AcqRelBB = createBasicBlock("acqrel", CurFn);
+    SeqCstBB = createBasicBlock("seqcst", CurFn);
+    llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
+
+    Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
+    llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
+
+    Builder.SetInsertPoint(AcquireBB);
+    Builder.CreateFence(llvm::Acquire, Scope);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(1), AcquireBB);
+    SI->addCase(Builder.getInt32(2), AcquireBB);
+
+    Builder.SetInsertPoint(ReleaseBB);
+    Builder.CreateFence(llvm::Release, Scope);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(3), ReleaseBB);
+
+    Builder.SetInsertPoint(AcqRelBB);
+    Builder.CreateFence(llvm::AcquireRelease, Scope);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(4), AcqRelBB);
+
+    Builder.SetInsertPoint(SeqCstBB);
+    Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(5), SeqCstBB);
+
+    Builder.SetInsertPoint(ContBB);
     return RValue::get(0);
   }
-      
+
     // Library functions with special handling.
   case Builtin::BIsqrt:
   case Builtin::BIsqrtf:
@@ -982,7 +1036,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *Exponent = EmitScalarExpr(E->getArg(1));
     llvm::Type *ArgType = Base->getType();
     Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
-    return RValue::get(Builder.CreateCall2(F, Base, Exponent, "tmp"));
+    return RValue::get(Builder.CreateCall2(F, Base, Exponent));
   }
 
   case Builtin::BIfma:
@@ -997,8 +1051,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType);
     return RValue::get(Builder.CreateCall3(F, FirstArg,
                                               EmitScalarExpr(E->getArg(1)),
-                                              EmitScalarExpr(E->getArg(2)),
-                                              "tmp"));
+                                              EmitScalarExpr(E->getArg(2))));
   }
 
   case Builtin::BI__builtin_signbit:
@@ -1007,25 +1060,40 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     LLVMContext &C = CGM.getLLVMContext();
 
     Value *Arg = EmitScalarExpr(E->getArg(0));
-    const llvm::Type *ArgTy = Arg->getType();
+    llvm::Type *ArgTy = Arg->getType();
     if (ArgTy->isPPC_FP128Ty())
       break; // FIXME: I'm not sure what the right implementation is here.
     int ArgWidth = ArgTy->getPrimitiveSizeInBits();
-    const llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
+    llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
     Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy);
     Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy);
     Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp);
     return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType())));
   }
+  case Builtin::BI__builtin_annotation: {
+    llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
+    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
+                                      AnnVal->getType());
+
+    // Get the annotation string, go through casts. Sema requires this to be a
+    // non-wide string literal, potentially casted, so the cast<> is safe.
+    const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
+    llvm::StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
+    return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
+  }
   }
 
-  // If this is an alias for a libm function (e.g. __builtin_sin) turn it into
-  // that function.
-  if (getContext().BuiltinInfo.isLibFunction(BuiltinID) ||
-      getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
-    return EmitCall(E->getCallee()->getType(),
-                    CGM.getBuiltinLibFunction(FD, BuiltinID),
-                    ReturnValueSlot(), E->arg_begin(), E->arg_end(), FD);
+  // If this is an alias for a lib function (e.g. __builtin_sin), emit
+  // the call using the normal call path, but using the unmangled
+  // version of the function name.
+  if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
+    return emitLibraryCall(*this, FD, E,
+                           CGM.getBuiltinLibFunction(FD, BuiltinID));
+  
+  // If this is a predefined lib function (e.g. malloc), emit the call
+  // using exactly the normal call path.
+  if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
+    return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee()));
 
   // See if we have a target specific intrinsic.
   const char *Name = getContext().BuiltinInfo.GetName(BuiltinID);
@@ -1045,7 +1113,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     assert(Error == ASTContext::GE_None && "Should not codegen an error");
 
     Function *F = CGM.getIntrinsic(IntrinsicID);
-    const llvm::FunctionType *FTy = F->getFunctionType();
+    llvm::FunctionType *FTy = F->getFunctionType();
 
     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
       Value *ArgValue;
@@ -1064,7 +1132,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
       // If the intrinsic arg type is different from the builtin arg type
       // we need to do a bit cast.
-      const llvm::Type *PTy = FTy->getParamType(i);
+      llvm::Type *PTy = FTy->getParamType(i);
       if (PTy != ArgValue->getType()) {
         assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
                "Must be able to losslessly bit cast to param");
@@ -1077,7 +1145,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *V = Builder.CreateCall(F, Args);
     QualType BuiltinRetType = E->getType();
 
-    const llvm::Type *RetTy = llvm::Type::getVoidTy(getLLVMContext());
+    llvm::Type *RetTy = llvm::Type::getVoidTy(getLLVMContext());
     if (!BuiltinRetType->isVoidType()) RetTy = ConvertType(BuiltinRetType);
 
     if (RetTy != V->getType()) {
@@ -1154,12 +1222,12 @@ Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
   return Builder.CreateCall(F, Ops, name);
 }
 
-Value *CodeGenFunction::EmitNeonShiftVector(Value *V, const llvm::Type *Ty, 
+Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 
                                             bool neg) {
   ConstantInt *CI = cast<ConstantInt>(V);
   int SV = CI->getSExtValue();
   
-  const llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
+  llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
   llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
   SmallVector<llvm::Constant*, 16> CV(VTy->getNumElements(), C);
   return llvm::ConstantVector::get(CV);
@@ -1193,12 +1261,12 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     const FunctionDecl *FD = E->getDirectCallee();
     // Oddly people write this call without args on occasion and gcc accepts
     // it - it's also marked as varargs in the description file.
-    llvm::SmallVector<Value*, 2> Ops;
+    SmallVector<Value*, 2> Ops;
     for (unsigned i = 0; i < E->getNumArgs(); i++)
       Ops.push_back(EmitScalarExpr(E->getArg(i)));
-    const llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
-    const llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
-    llvm::StringRef Name = FD->getName();
+    llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
+    llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
+    StringRef Name = FD->getName();
     return Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
   }
 
@@ -1223,7 +1291,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL);
 
     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
-    Value *Tmp = Builder.CreateAlloca(Int64Ty, One, "tmp");
+    Value *Tmp = Builder.CreateAlloca(Int64Ty, One);
     Value *Val = EmitScalarExpr(E->getArg(0));
     Builder.CreateStore(Val, Tmp);
 
@@ -1236,10 +1304,41 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd");
   }
 
-  llvm::SmallVector<Value*, 4> Ops;
+  SmallVector<Value*, 4> Ops;
   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++)
     Ops.push_back(EmitScalarExpr(E->getArg(i)));
 
+  // vget_lane and vset_lane are not overloaded and do not have an extra
+  // argument that specifies the vector type.
+  switch (BuiltinID) {
+  default: break;
+  case ARM::BI__builtin_neon_vget_lane_i8:
+  case ARM::BI__builtin_neon_vget_lane_i16:
+  case ARM::BI__builtin_neon_vget_lane_i32:
+  case ARM::BI__builtin_neon_vget_lane_i64:
+  case ARM::BI__builtin_neon_vget_lane_f32:
+  case ARM::BI__builtin_neon_vgetq_lane_i8:
+  case ARM::BI__builtin_neon_vgetq_lane_i16:
+  case ARM::BI__builtin_neon_vgetq_lane_i32:
+  case ARM::BI__builtin_neon_vgetq_lane_i64:
+  case ARM::BI__builtin_neon_vgetq_lane_f32:
+    return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
+                                        "vget_lane");
+  case ARM::BI__builtin_neon_vset_lane_i8:
+  case ARM::BI__builtin_neon_vset_lane_i16:
+  case ARM::BI__builtin_neon_vset_lane_i32:
+  case ARM::BI__builtin_neon_vset_lane_i64:
+  case ARM::BI__builtin_neon_vset_lane_f32:
+  case ARM::BI__builtin_neon_vsetq_lane_i8:
+  case ARM::BI__builtin_neon_vsetq_lane_i16:
+  case ARM::BI__builtin_neon_vsetq_lane_i32:
+  case ARM::BI__builtin_neon_vsetq_lane_i64:
+  case ARM::BI__builtin_neon_vsetq_lane_f32:
+    Ops.push_back(EmitScalarExpr(E->getArg(2)));
+    return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
+  }
+
+  // Get the last argument, which specifies the vector type.
   llvm::APSInt Result;
   const Expr *Arg = E->getArg(E->getNumArgs()-1);
   if (!Arg->isIntegerConstantExpr(Result, getContext()))
@@ -1382,18 +1481,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Value *SV = llvm::ConstantVector::get(Indices);
     return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext");
   }
-  case ARM::BI__builtin_neon_vget_lane_i8:
-  case ARM::BI__builtin_neon_vget_lane_i16:
-  case ARM::BI__builtin_neon_vget_lane_i32:
-  case ARM::BI__builtin_neon_vget_lane_i64:
-  case ARM::BI__builtin_neon_vget_lane_f32:
-  case ARM::BI__builtin_neon_vgetq_lane_i8:
-  case ARM::BI__builtin_neon_vgetq_lane_i16:
-  case ARM::BI__builtin_neon_vgetq_lane_i32:
-  case ARM::BI__builtin_neon_vgetq_lane_i64:
-  case ARM::BI__builtin_neon_vgetq_lane_f32:
-    return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
-                                        "vget_lane");
   case ARM::BI__builtin_neon_vhadd_v:
   case ARM::BI__builtin_neon_vhaddq_v:
     Int = usgn ? Intrinsic::arm_neon_vhaddu : Intrinsic::arm_neon_vhadds;
@@ -1457,9 +1544,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
     Ops.push_back(GetPointeeAlignment(*this, E->getArg(1)));
-    Ops[1] = Builder.CreateCall(F,
-                                ArrayRef<Value *>(Ops.begin() + 1, Ops.end()),
-                                "vld2_lane");
+    Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     return Builder.CreateStore(Ops[1], Ops[0]);
@@ -1471,9 +1556,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
     Ops.push_back(GetPointeeAlignment(*this, E->getArg(1)));
-    Ops[1] = Builder.CreateCall(F,
-                                ArrayRef<Value *>(Ops.begin() + 1, Ops.end()),
-                                "vld3_lane");
+    Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     return Builder.CreateStore(Ops[1], Ops[0]);
@@ -1486,9 +1569,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
     Ops[5] = Builder.CreateBitCast(Ops[5], Ty);
     Ops.push_back(GetPointeeAlignment(*this, E->getArg(1)));
-    Ops[1] = Builder.CreateCall(F,
-                                ArrayRef<Value *>(Ops.begin() + 1, Ops.end()),
-                                "vld3_lane");
+    Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     return Builder.CreateStore(Ops[1], Ops[0]);
@@ -1508,7 +1589,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
       case ARM::BI__builtin_neon_vld4_dup_v:
         Int = Intrinsic::arm_neon_vld2; 
         break;
-      default: assert(0 && "unknown vld_dup intrinsic?");
+      default: llvm_unreachable("unknown vld_dup intrinsic?");
       }
       Function *F = CGM.getIntrinsic(Int, Ty);
       Value *Align = GetPointeeAlignment(*this, E->getArg(1));
@@ -1527,10 +1608,10 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     case ARM::BI__builtin_neon_vld4_dup_v:
       Int = Intrinsic::arm_neon_vld2lane; 
       break;
-    default: assert(0 && "unknown vld_dup intrinsic?");
+    default: llvm_unreachable("unknown vld_dup intrinsic?");
     }
     Function *F = CGM.getIntrinsic(Int, Ty);
-    const llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
+    llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
     
     SmallVector<Value*, 6> Args;
     Args.push_back(Ops[1]);
@@ -1562,14 +1643,14 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Int = usgn ? Intrinsic::arm_neon_vminu : Intrinsic::arm_neon_vmins;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
   case ARM::BI__builtin_neon_vmovl_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
     if (usgn)
       return Builder.CreateZExt(Ops[0], Ty, "vmovl");
     return Builder.CreateSExt(Ops[0], Ty, "vmovl");
   }
   case ARM::BI__builtin_neon_vmovn_v: {
-    const llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
+    llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
   }
@@ -1587,7 +1668,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Int = usgn ? Intrinsic::arm_neon_vpadalu : Intrinsic::arm_neon_vpadals;
     // The source operand type has twice as many elements of half the size.
     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
-    const llvm::Type *EltTy =
+    llvm::Type *EltTy =
       llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
     llvm::Type *NarrowTy =
       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
@@ -1602,7 +1683,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Int = usgn ? Intrinsic::arm_neon_vpaddlu : Intrinsic::arm_neon_vpaddls;
     // The source operand type has twice as many elements of half the size.
     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
-    const llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
+    llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
     llvm::Type *NarrowTy =
       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
     llvm::Type *Tys[2] = { Ty, NarrowTy };
@@ -1729,18 +1810,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vrsubhn_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsubhn, Ty),
                         Ops, "vrsubhn");
-  case ARM::BI__builtin_neon_vset_lane_i8:
-  case ARM::BI__builtin_neon_vset_lane_i16:
-  case ARM::BI__builtin_neon_vset_lane_i32:
-  case ARM::BI__builtin_neon_vset_lane_i64:
-  case ARM::BI__builtin_neon_vset_lane_f32:
-  case ARM::BI__builtin_neon_vsetq_lane_i8:
-  case ARM::BI__builtin_neon_vsetq_lane_i16:
-  case ARM::BI__builtin_neon_vsetq_lane_i32:
-  case ARM::BI__builtin_neon_vsetq_lane_i64:
-  case ARM::BI__builtin_neon_vsetq_lane_f32:
-    Ops.push_back(EmitScalarExpr(E->getArg(2)));
-    return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
   case ARM::BI__builtin_neon_vshl_v:
   case ARM::BI__builtin_neon_vshlq_v:
     Int = usgn ? Intrinsic::arm_neon_vshiftu : Intrinsic::arm_neon_vshifts;
@@ -1921,7 +1990,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
 }
 
 llvm::Value *CodeGenFunction::
-BuildVector(const llvm::SmallVectorImpl<llvm::Value*> &Ops) {
+BuildVector(const SmallVectorImpl<llvm::Value*> &Ops) {
   assert((Ops.size() & (Ops.size() - 1)) == 0 &&
          "Not a power-of-two sized vector!");
   bool AllConstants = true;
@@ -1949,7 +2018,7 @@ BuildVector(const llvm::SmallVectorImpl<llvm::Value*> &Ops) {
 
 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E) {
-  llvm::SmallVector<Value*, 4> Ops;
+  SmallVector<Value*, 4> Ops;
 
   // Find out if any arguments are required to be integer constant expressions.
   unsigned ICEArguments = 0;
@@ -1983,7 +2052,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   case X86::BI__builtin_ia32_psrlqi128:
   case X86::BI__builtin_ia32_psrlwi128: {
     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty, "zext");
-    const llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
+    llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
     llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);
     Ops[1] = Builder.CreateInsertElement(llvm::UndefValue::get(Ty),
                                          Ops[1], Zero, "insert");
@@ -1992,7 +2061,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     Intrinsic::ID ID = Intrinsic::not_intrinsic;
 
     switch (BuiltinID) {
-    default: assert(0 && "Unsupported shift intrinsic!");
+    default: llvm_unreachable("Unsupported shift intrinsic!");
     case X86::BI__builtin_ia32_pslldi128:
       name = "pslldi";
       ID = Intrinsic::x86_sse2_psll_d;
@@ -2046,13 +2115,13 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   case X86::BI__builtin_ia32_psrlqi:
   case X86::BI__builtin_ia32_psrlwi: {
     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty, "zext");
-    const llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 1);
+    llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 1);
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty, "bitcast");
     const char *name = 0;
     Intrinsic::ID ID = Intrinsic::not_intrinsic;
 
     switch (BuiltinID) {
-    default: assert(0 && "Unsupported shift intrinsic!");
+    default: llvm_unreachable("Unsupported shift intrinsic!");
     case X86::BI__builtin_ia32_pslldi:
       name = "pslldi";
       ID = Intrinsic::x86_mmx_psll_d;
@@ -2098,19 +2167,19 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     return Builder.CreateCall(F, Ops, "cmpss");
   }
   case X86::BI__builtin_ia32_ldmxcsr: {
-    const llvm::Type *PtrTy = Int8PtrTy;
+    llvm::Type *PtrTy = Int8PtrTy;
     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
-    Value *Tmp = Builder.CreateAlloca(Int32Ty, One, "tmp");
+    Value *Tmp = Builder.CreateAlloca(Int32Ty, One);
     Builder.CreateStore(Ops[0], Tmp);
     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
                               Builder.CreateBitCast(Tmp, PtrTy));
   }
   case X86::BI__builtin_ia32_stmxcsr: {
-    const llvm::Type *PtrTy = Int8PtrTy;
+    llvm::Type *PtrTy = Int8PtrTy;
     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
-    Value *Tmp = Builder.CreateAlloca(Int32Ty, One, "tmp");
-    One = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
-                             Builder.CreateBitCast(Tmp, PtrTy));
+    Value *Tmp = Builder.CreateAlloca(Int32Ty, One);
+    Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
+                       Builder.CreateBitCast(Tmp, PtrTy));
     return Builder.CreateLoad(Tmp, "stmxcsr");
   }
   case X86::BI__builtin_ia32_cmppd: {
@@ -2144,7 +2213,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     // If palignr is shifting the pair of input vectors less than 9 bytes,
     // emit a shuffle instruction.
     if (shiftVal <= 8) {
-      llvm::SmallVector<llvm::Constant*, 8> Indices;
+      SmallVector<llvm::Constant*, 8> Indices;
       for (unsigned i = 0; i != 8; ++i)
         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
       
@@ -2156,17 +2225,17 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     // than 16 bytes, emit a logical right shift of the destination.
     if (shiftVal < 16) {
       // MMX has these as 1 x i64 vectors for some odd optimization reasons.
-      const llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
+      llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
       
       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
       Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
       
       // create i32 constant
       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q);
-      return Builder.CreateCall(F, ArrayRef<Value *>(&Ops[0], 2), "palignr");
+      return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
     }
     
-    // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
+    // If palignr is shifting the pair of vectors more than 16 bytes, emit zero.
     return llvm::Constant::getNullValue(ConvertType(E->getType()));
   }
   case X86::BI__builtin_ia32_palignr128: {
@@ -2175,7 +2244,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     // If palignr is shifting the pair of input vectors less than 17 bytes,
     // emit a shuffle instruction.
     if (shiftVal <= 16) {
-      llvm::SmallVector<llvm::Constant*, 16> Indices;
+      SmallVector<llvm::Constant*, 16> Indices;
       for (unsigned i = 0; i != 16; ++i)
         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
       
@@ -2186,14 +2255,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     // If palignr is shifting the pair of input vectors more than 16 but less
     // than 32 bytes, emit a logical right shift of the destination.
     if (shiftVal < 32) {
-      const llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
+      llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
       
       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
       Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
       
       // create i32 constant
       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq);
-      return Builder.CreateCall(F, ArrayRef<Value *>(&Ops[0], 2), "palignr");
+      return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
     }
     
     // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
@@ -2352,7 +2421,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
 
 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E) {
-  llvm::SmallVector<Value*, 4> Ops;
+  SmallVector<Value*, 4> Ops;
 
   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
     Ops.push_back(EmitScalarExpr(E->getArg(i)));
@@ -2373,11 +2442,11 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   {
     Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
 
-    Ops[0] = Builder.CreateGEP(Ops[1], Ops[0], "tmp");
+    Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
     Ops.pop_back();
 
     switch (BuiltinID) {
-    default: assert(0 && "Unsupported ld/lvsl/lvsr intrinsic!");
+    default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
     case PPC::BI__builtin_altivec_lvx:
       ID = Intrinsic::ppc_altivec_lvx;
       break;
@@ -2412,11 +2481,11 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   case PPC::BI__builtin_altivec_stvewx:
   {
     Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
-    Ops[1] = Builder.CreateGEP(Ops[2], Ops[1], "tmp");
+    Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
     Ops.pop_back();
 
     switch (BuiltinID) {
-    default: assert(0 && "Unsupported st intrinsic!");
+    default: llvm_unreachable("Unsupported st intrinsic!");
     case PPC::BI__builtin_altivec_stvx:
       ID = Intrinsic::ppc_altivec_stvx;
       break;
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp
new file mode 100644
index 000000000000..88a0bdc821d7
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp
@@ -0,0 +1,126 @@
+//===----- CGCUDANV.cpp - Interface to NVIDIA CUDA Runtime ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides a class for CUDA code generation targeting the NVIDIA CUDA
+// runtime library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGCUDARuntime.h"
+#include "CodeGenFunction.h"
+#include "CodeGenModule.h"
+#include "clang/AST/Decl.h"
+#include "llvm/BasicBlock.h"
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Support/CallSite.h"
+
+#include <vector>
+
+using namespace clang;
+using namespace CodeGen;
+
+namespace {
+
+class CGNVCUDARuntime : public CGCUDARuntime {
+
+private:
+  llvm::Type *IntTy, *SizeTy;
+  llvm::PointerType *CharPtrTy, *VoidPtrTy;
+
+  llvm::Constant *getSetupArgumentFn() const;
+  llvm::Constant *getLaunchFn() const;
+
+public:
+  CGNVCUDARuntime(CodeGenModule &CGM);
+
+  void EmitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args);
+};
+
+}
+
+CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) : CGCUDARuntime(CGM) {
+  CodeGen::CodeGenTypes &Types = CGM.getTypes();
+  ASTContext &Ctx = CGM.getContext();
+
+  IntTy = Types.ConvertType(Ctx.IntTy);
+  SizeTy = Types.ConvertType(Ctx.getSizeType());
+
+  CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy));
+  VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy));
+}
+
+llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const {
+  // cudaError_t cudaSetupArgument(void *, size_t, size_t)
+  std::vector<llvm::Type*> Params;
+  Params.push_back(VoidPtrTy);
+  Params.push_back(SizeTy);
+  Params.push_back(SizeTy);
+  return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy,
+                                                           Params, false),
+                                   "cudaSetupArgument");
+}
+
+llvm::Constant *CGNVCUDARuntime::getLaunchFn() const {
+  // cudaError_t cudaLaunch(char *)
+  std::vector<llvm::Type*> Params;
+  Params.push_back(CharPtrTy);
+  return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy,
+                                                           Params, false),
+                                   "cudaLaunch");
+}
+
+void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF,
+                                         FunctionArgList &Args) {
+  // Build the argument value list and the argument stack struct type.
+  llvm::SmallVector<llvm::Value *, 16> ArgValues;
+  std::vector<llvm::Type *> ArgTypes;
+  for (FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
+       I != E; ++I) {
+    llvm::Value *V = CGF.GetAddrOfLocalVar(*I);
+    ArgValues.push_back(V);
+    assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType");
+    ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType());
+  }
+  llvm::StructType *ArgStackTy = llvm::StructType::get(
+      CGF.getLLVMContext(), ArgTypes);
+
+  llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
+
+  // Emit the calls to cudaSetupArgument
+  llvm::Constant *cudaSetupArgFn = getSetupArgumentFn();
+  for (unsigned I = 0, E = Args.size(); I != E; ++I) {
+    llvm::Value *Args[3];
+    llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next");
+    Args[0] = CGF.Builder.CreatePointerCast(ArgValues[I], VoidPtrTy);
+    Args[1] = CGF.Builder.CreateIntCast(
+        llvm::ConstantExpr::getSizeOf(ArgTypes[I]),
+        SizeTy, false);
+    Args[2] = CGF.Builder.CreateIntCast(
+        llvm::ConstantExpr::getOffsetOf(ArgStackTy, I),
+        SizeTy, false);
+    llvm::CallSite CS = CGF.EmitCallOrInvoke(cudaSetupArgFn, Args);
+    llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0);
+    llvm::Value *CSZero = CGF.Builder.CreateICmpEQ(CS.getInstruction(), Zero);
+    CGF.Builder.CreateCondBr(CSZero, NextBlock, EndBlock);
+    CGF.EmitBlock(NextBlock);
+  }
+
+  // Emit the call to cudaLaunch
+  llvm::Constant *cudaLaunchFn = getLaunchFn();
+  llvm::Value *Arg = CGF.Builder.CreatePointerCast(CGF.CurFn, CharPtrTy);
+  CGF.EmitCallOrInvoke(cudaLaunchFn, Arg);
+  CGF.EmitBranch(EndBlock);
+
+  CGF.EmitBlock(EndBlock);
+}
+
+CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) {
+  return new CGNVCUDARuntime(CGM);
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp
new file mode 100644
index 000000000000..77dc248d69e6
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp
@@ -0,0 +1,55 @@
+//===----- CGCUDARuntime.cpp - Interface to CUDA Runtimes -----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for CUDA code generation.  Concrete
+// subclasses of this implement code generation for specific CUDA
+// runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGCUDARuntime.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/ExprCXX.h"
+#include "CGCall.h"
+#include "CodeGenFunction.h"
+
+using namespace clang;
+using namespace CodeGen;
+
+CGCUDARuntime::~CGCUDARuntime() {}
+
+RValue CGCUDARuntime::EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
+                                             const CUDAKernelCallExpr *E,
+                                             ReturnValueSlot ReturnValue) {
+  llvm::BasicBlock *ConfigOKBlock = CGF.createBasicBlock("kcall.configok");
+  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("kcall.end");
+
+  CodeGenFunction::ConditionalEvaluation eval(CGF);
+  CGF.EmitBranchOnBoolExpr(E->getConfig(), ContBlock, ConfigOKBlock);
+
+  eval.begin(CGF);
+  CGF.EmitBlock(ConfigOKBlock);
+
+  const Decl *TargetDecl = 0;
+  if (const ImplicitCastExpr *CE = dyn_cast<ImplicitCastExpr>(E->getCallee())) {
+    if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CE->getSubExpr())) {
+      TargetDecl = DRE->getDecl();
+    }
+  }
+
+  llvm::Value *Callee = CGF.EmitScalarExpr(E->getCallee());
+  CGF.EmitCall(E->getCallee()->getType(), Callee, ReturnValue,
+               E->arg_begin(), E->arg_end(), TargetDecl);
+  CGF.EmitBranch(ContBlock);
+
+  CGF.EmitBlock(ContBlock);
+  eval.end(CGF);
+
+  return RValue::get(0);
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h
new file mode 100644
index 000000000000..a99a67ae1ae7
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h
@@ -0,0 +1,54 @@
+//===----- CGCUDARuntime.h - Interface to CUDA Runtimes ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for CUDA code generation.  Concrete
+// subclasses of this implement code generation for specific CUDA
+// runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CODEGEN_CUDARUNTIME_H
+#define CLANG_CODEGEN_CUDARUNTIME_H
+
+namespace clang {
+
+class CUDAKernelCallExpr;
+
+namespace CodeGen {
+
+class CodeGenFunction;
+class CodeGenModule;
+class FunctionArgList;
+class ReturnValueSlot;
+class RValue;
+
+class CGCUDARuntime {
+protected:
+  CodeGenModule &CGM;
+
+public:
+  CGCUDARuntime(CodeGenModule &CGM) : CGM(CGM) {}
+  virtual ~CGCUDARuntime();
+
+  virtual RValue EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
+                                        const CUDAKernelCallExpr *E,
+                                        ReturnValueSlot ReturnValue);
+  
+  virtual void EmitDeviceStubBody(CodeGenFunction &CGF,
+                                  FunctionArgList &Args) = 0;
+
+};
+
+/// Creates an instance of a CUDA runtime class.
+CGCUDARuntime *CreateNVCUDARuntime(CodeGenModule &CGM);
+
+}
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp
index f6fc202eaae2..b5e6e0d7d993 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp
@@ -138,7 +138,7 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl,
     return true;
 
   // Derive the type for the alias.
-  const llvm::PointerType *AliasType
+  llvm::PointerType *AliasType
     = getTypes().GetFunctionType(AliasDecl)->getPointerTo();
 
   // Find the referrent.  Some aliases might require a bitcast, in
@@ -154,7 +154,7 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl,
     new llvm::GlobalAlias(AliasType, Linkage, "", Aliasee, &getModule());
 
   // Switch any previous uses to the alias.
-  llvm::StringRef MangledName = getMangledName(AliasDecl);
+  StringRef MangledName = getMangledName(AliasDecl);
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
     assert(Entry->isDeclaration() && "definition already exists for alias");
@@ -214,14 +214,14 @@ CodeGenModule::GetAddrOfCXXConstructor(const CXXConstructorDecl *ctor,
                                        const CGFunctionInfo *fnInfo) {
   GlobalDecl GD(ctor, ctorType);
   
-  llvm::StringRef name = getMangledName(GD);
+  StringRef name = getMangledName(GD);
   if (llvm::GlobalValue *existing = GetGlobalValue(name))
     return existing;
 
   if (!fnInfo) fnInfo = &getTypes().getFunctionInfo(ctor, ctorType);
 
   const FunctionProtoType *proto = ctor->getType()->castAs<FunctionProtoType>();
-  const llvm::FunctionType *fnType =
+  llvm::FunctionType *fnType =
     getTypes().GetFunctionType(*fnInfo, proto->isVariadic());
   return cast<llvm::Function>(GetOrCreateLLVMFunction(name, fnType, GD,
                                                       /*ForVTable=*/false));
@@ -236,11 +236,7 @@ void CodeGenModule::EmitCXXDestructors(const CXXDestructorDecl *D) {
 
   // The destructor used for destructing this as a most-derived class;
   // call the base destructor and then destructs any virtual bases.
-  if (!D->getParent()->isAbstract() || D->isVirtual()) {
-    // We don't need to emit the complete ctor if the class is abstract,
-    // unless the destructor is virtual and needs to be in the vtable.
-    EmitGlobal(GlobalDecl(D, Dtor_Complete));
-  }
+  EmitGlobal(GlobalDecl(D, Dtor_Complete));
 
   // The destructor used for destructing this as a base class; ignores
   // virtual bases.
@@ -282,13 +278,13 @@ CodeGenModule::GetAddrOfCXXDestructor(const CXXDestructorDecl *dtor,
                                       const CGFunctionInfo *fnInfo) {
   GlobalDecl GD(dtor, dtorType);
 
-  llvm::StringRef name = getMangledName(GD);
+  StringRef name = getMangledName(GD);
   if (llvm::GlobalValue *existing = GetGlobalValue(name))
     return existing;
 
   if (!fnInfo) fnInfo = &getTypes().getFunctionInfo(dtor, dtorType);
 
-  const llvm::FunctionType *fnType =
+  llvm::FunctionType *fnType =
     getTypes().GetFunctionType(*fnInfo, false);
 
   return cast<llvm::Function>(GetOrCreateLLVMFunction(name, fnType, GD,
@@ -296,7 +292,7 @@ CodeGenModule::GetAddrOfCXXDestructor(const CXXDestructorDecl *dtor,
 }
 
 static llvm::Value *BuildVirtualCall(CodeGenFunction &CGF, uint64_t VTableIndex, 
-                                     llvm::Value *This, const llvm::Type *Ty) {
+                                     llvm::Value *This, llvm::Type *Ty) {
   Ty = Ty->getPointerTo()->getPointerTo();
   
   llvm::Value *VTable = CGF.GetVTablePtr(This, Ty);
@@ -307,9 +303,9 @@ static llvm::Value *BuildVirtualCall(CodeGenFunction &CGF, uint64_t VTableIndex,
 
 llvm::Value *
 CodeGenFunction::BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
-                                  const llvm::Type *Ty) {
+                                  llvm::Type *Ty) {
   MD = MD->getCanonicalDecl();
-  uint64_t VTableIndex = CGM.getVTables().getMethodVTableIndex(MD);
+  uint64_t VTableIndex = CGM.getVTableContext().getMethodVTableIndex(MD);
   
   return ::BuildVirtualCall(*this, VTableIndex, This, Ty);
 }
@@ -320,7 +316,7 @@ CodeGenFunction::BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
 llvm::Value *
 CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD, 
                                   NestedNameSpecifier *Qual,
-                                  const llvm::Type *Ty) {
+                                  llvm::Type *Ty) {
   llvm::Value *VTable = 0;
   assert((Qual->getKind() == NestedNameSpecifier::TypeSpec) &&
          "BuildAppleKextVirtualCall - bad Qual kind");
@@ -339,9 +335,10 @@ CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
   VTable = Builder.CreateBitCast(VTable, Ty);
   assert(VTable && "BuildVirtualCall = kext vtbl pointer is null");
   MD = MD->getCanonicalDecl();
-  uint64_t VTableIndex = CGM.getVTables().getMethodVTableIndex(MD);
+  uint64_t VTableIndex = CGM.getVTableContext().getMethodVTableIndex(MD);
   uint64_t AddressPoint = 
-    CGM.getVTables().getAddressPoint(BaseSubobject(RD, CharUnits::Zero()), RD);
+    CGM.getVTableContext().getVTableLayout(RD)
+       .getAddressPoint(BaseSubobject(RD, CharUnits::Zero()));
   VTableIndex += AddressPoint;
   llvm::Value *VFuncPtr = 
     Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt");
@@ -366,7 +363,7 @@ CodeGenFunction::BuildAppleKextVirtualDestructorCall(
     &CGM.getTypes().getFunctionInfo(cast<CXXDestructorDecl>(MD),
                                     Dtor_Complete);
     const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
-    const llvm::Type *Ty
+    llvm::Type *Ty
       = CGM.getTypes().GetFunctionType(*FInfo, FPT->isVariadic());
 
     llvm::Value *VTable = CGM.getVTables().GetAddrOfVTable(RD);
@@ -374,9 +371,10 @@ CodeGenFunction::BuildAppleKextVirtualDestructorCall(
     VTable = Builder.CreateBitCast(VTable, Ty);
     DD = cast<CXXDestructorDecl>(DD->getCanonicalDecl());
     uint64_t VTableIndex = 
-      CGM.getVTables().getMethodVTableIndex(GlobalDecl(DD, Type));
+      CGM.getVTableContext().getMethodVTableIndex(GlobalDecl(DD, Type));
     uint64_t AddressPoint =
-      CGM.getVTables().getAddressPoint(BaseSubobject(RD, CharUnits::Zero()), RD);
+      CGM.getVTableContext().getVTableLayout(RD)
+         .getAddressPoint(BaseSubobject(RD, CharUnits::Zero()));
     VTableIndex += AddressPoint;
     llvm::Value *VFuncPtr =
       Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt");
@@ -387,10 +385,10 @@ CodeGenFunction::BuildAppleKextVirtualDestructorCall(
 
 llvm::Value *
 CodeGenFunction::BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type, 
-                                  llvm::Value *This, const llvm::Type *Ty) {
+                                  llvm::Value *This, llvm::Type *Ty) {
   DD = cast<CXXDestructorDecl>(DD->getCanonicalDecl());
   uint64_t VTableIndex = 
-    CGM.getVTables().getMethodVTableIndex(GlobalDecl(DD, Type));
+    CGM.getVTableContext().getMethodVTableIndex(GlobalDecl(DD, Type));
 
   return ::BuildVirtualCall(*this, VTableIndex, This, Ty);
 }
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp
index dcc28b45cfc3..248448ccdc2e 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp
@@ -20,9 +20,9 @@ using namespace CodeGen;
 CGCXXABI::~CGCXXABI() { }
 
 static void ErrorUnsupportedABI(CodeGenFunction &CGF,
-                                llvm::StringRef S) {
-  Diagnostic &Diags = CGF.CGM.getDiags();
-  unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
+                                StringRef S) {
+  DiagnosticsEngine &Diags = CGF.CGM.getDiags();
+  unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                           "cannot yet compile %1 in this ABI");
   Diags.Report(CGF.getContext().getFullLoc(CGF.CurCodeDecl->getLocation()),
                DiagID)
@@ -49,7 +49,7 @@ llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
     MPT->getPointeeType()->getAs<FunctionProtoType>();
   const CXXRecordDecl *RD = 
     cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
-  const llvm::FunctionType *FTy = 
+  llvm::FunctionType *FTy = 
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
                                    FPT->isVariadic());
   return llvm::Constant::getNullValue(FTy->getPointerTo());
@@ -60,7 +60,7 @@ llvm::Value *CGCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
                                                     llvm::Value *MemPtr,
                                               const MemberPointerType *MPT) {
   ErrorUnsupportedABI(CGF, "loads of member pointers");
-  const llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo();
+  llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo();
   return llvm::Constant::getNullValue(Ty);
 }
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h
index 29f299a43e37..c2abf358329c 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h
@@ -15,14 +15,14 @@
 #ifndef CLANG_CODEGEN_CXXABI_H
 #define CLANG_CODEGEN_CXXABI_H
 
+#include "clang/Basic/LLVM.h"
+
 #include "CodeGenFunction.h"
 
 namespace llvm {
   class Constant;
   class Type;
   class Value;
-
-  template <class T> class SmallVectorImpl;
 }
 
 namespace clang {
@@ -151,7 +151,7 @@ class CGCXXABI {
   virtual void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                          CXXCtorType T,
                                          CanQualType &ResTy,
-                               llvm::SmallVectorImpl<CanQualType> &ArgTys) = 0;
+                               SmallVectorImpl<CanQualType> &ArgTys) = 0;
 
   /// Build the signature of the given destructor variant by adding
   /// any required parameters.  For convenience, ResTy has been
@@ -160,7 +160,7 @@ class CGCXXABI {
   virtual void BuildDestructorSignature(const CXXDestructorDecl *Dtor,
                                         CXXDtorType T,
                                         CanQualType &ResTy,
-                               llvm::SmallVectorImpl<CanQualType> &ArgTys) = 0;
+                               SmallVectorImpl<CanQualType> &ArgTys) = 0;
 
   /// Build the ABI-specific portion of the parameter list for a
   /// function.  This generally involves a 'this' parameter and
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp
index f8783ad08d00..6ae2d0c96775 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp
@@ -69,14 +69,14 @@ static CanQualType GetReturnType(QualType RetTy) {
 const CGFunctionInfo &
 CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
   return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
-                         llvm::SmallVector<CanQualType, 16>(),
+                         SmallVector<CanQualType, 16>(),
                          FTNP->getExtInfo());
 }
 
 /// \param Args - contains any initial parameters besides those
 ///   in the formal type
 static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
-                                  llvm::SmallVectorImpl<CanQualType> &ArgTys,
+                                  SmallVectorImpl<CanQualType> &ArgTys,
                                              CanQual<FunctionProtoType> FTP) {
   // FIXME: Kill copy.
   for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
@@ -87,7 +87,7 @@ static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
 
 const CGFunctionInfo &
 CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
   return ::getFunctionInfo(*this, ArgTys, FTP);
 }
 
@@ -113,7 +113,7 @@ static CallingConv getCallingConventionForDecl(const Decl *D) {
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
                                                  const FunctionProtoType *FTP) {
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
 
   // Add the 'this' pointer.
   ArgTys.push_back(GetThisType(Context, RD));
@@ -123,7 +123,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
 
   assert(!isa<CXXConstructorDecl>(MD) && "wrong method for contructors!");
   assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
@@ -137,7 +137,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
                                                     CXXCtorType Type) {
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
   ArgTys.push_back(GetThisType(Context, D->getParent()));
   CanQualType ResTy = Context.VoidTy;
 
@@ -154,7 +154,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
                                                     CXXDtorType Type) {
-  llvm::SmallVector<CanQualType, 2> ArgTys;
+  SmallVector<CanQualType, 2> ArgTys;
   ArgTys.push_back(GetThisType(Context, D->getParent()));
   CanQualType ResTy = Context.VoidTy;
 
@@ -180,11 +180,11 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
   ArgTys.push_back(Context.getCanonicalParamType(MD->getSelfDecl()->getType()));
   ArgTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
   // FIXME: Kill copy?
-  for (ObjCMethodDecl::param_iterator i = MD->param_begin(),
+  for (ObjCMethodDecl::param_const_iterator i = MD->param_begin(),
          e = MD->param_end(); i != e; ++i) {
     ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
   }
@@ -216,7 +216,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
                                                     const CallArgList &Args,
                                             const FunctionType::ExtInfo &Info) {
   // FIXME: Kill copy.
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
   for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
        i != e; ++i)
     ArgTys.push_back(Context.getCanonicalParamType(i->Ty));
@@ -227,7 +227,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
                                                     const FunctionArgList &Args,
                                             const FunctionType::ExtInfo &Info) {
   // FIXME: Kill copy.
-  llvm::SmallVector<CanQualType, 16> ArgTys;
+  SmallVector<CanQualType, 16> ArgTys;
   for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
        i != e; ++i)
     ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
@@ -235,15 +235,15 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
 }
 
 const CGFunctionInfo &CodeGenTypes::getNullaryFunctionInfo() {
-  llvm::SmallVector<CanQualType, 1> args;
+  SmallVector<CanQualType, 1> args;
   return getFunctionInfo(getContext().VoidTy, args, FunctionType::ExtInfo());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
-                           const llvm::SmallVectorImpl<CanQualType> &ArgTys,
+                           const SmallVectorImpl<CanQualType> &ArgTys,
                                             const FunctionType::ExtInfo &Info) {
 #ifndef NDEBUG
-  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
+  for (SmallVectorImpl<CanQualType>::const_iterator
          I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
     assert(I->isCanonicalAsParam());
 #endif
@@ -312,50 +312,65 @@ CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
 /***/
 
 void CodeGenTypes::GetExpandedTypes(QualType type,
-                     llvm::SmallVectorImpl<llvm::Type*> &expandedTypes) {
-  const RecordType *RT = type->getAsStructureType();
-  assert(RT && "Can only expand structure types.");
-  const RecordDecl *RD = RT->getDecl();
-  assert(!RD->hasFlexibleArrayMember() &&
-         "Cannot expand structure with flexible array.");
-
-  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+                     SmallVectorImpl<llvm::Type*> &expandedTypes) {
+  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(type)) {
+    uint64_t NumElts = AT->getSize().getZExtValue();
+    for (uint64_t Elt = 0; Elt < NumElts; ++Elt)
+      GetExpandedTypes(AT->getElementType(), expandedTypes);
+  } else if (const RecordType *RT = type->getAsStructureType()) {
+    const RecordDecl *RD = RT->getDecl();
+    assert(!RD->hasFlexibleArrayMember() &&
+           "Cannot expand structure with flexible array.");
+    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
          i != e; ++i) {
-    const FieldDecl *FD = *i;
-    assert(!FD->isBitField() &&
-           "Cannot expand structure with bit-field members.");
-
-    QualType fieldType = FD->getType();
-    if (fieldType->isRecordType())
-      GetExpandedTypes(fieldType, expandedTypes);
-    else
-      expandedTypes.push_back(ConvertType(fieldType));
-  }
+      const FieldDecl *FD = *i;
+      assert(!FD->isBitField() &&
+             "Cannot expand structure with bit-field members.");
+      GetExpandedTypes(FD->getType(), expandedTypes);
+    }
+  } else if (const ComplexType *CT = type->getAs<ComplexType>()) {
+    llvm::Type *EltTy = ConvertType(CT->getElementType());
+    expandedTypes.push_back(EltTy);
+    expandedTypes.push_back(EltTy);
+  } else
+    expandedTypes.push_back(ConvertType(type));
 }
 
 llvm::Function::arg_iterator
 CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
                                     llvm::Function::arg_iterator AI) {
-  const RecordType *RT = Ty->getAsStructureType();
-  assert(RT && "Can only expand structure types.");
-
-  RecordDecl *RD = RT->getDecl();
   assert(LV.isSimple() &&
          "Unexpected non-simple lvalue during struct expansion.");
   llvm::Value *Addr = LV.getAddress();
-  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
-         i != e; ++i) {
-    FieldDecl *FD = *i;
-    QualType FT = FD->getType();
 
-    // FIXME: What are the right qualifiers here?
-    LValue LV = EmitLValueForField(Addr, FD, 0);
-    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
-      AI = ExpandTypeFromArgs(FT, LV, AI);
-    } else {
-      EmitStoreThroughLValue(RValue::get(AI), LV);
-      ++AI;
+  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
+    unsigned NumElts = AT->getSize().getZExtValue();
+    QualType EltTy = AT->getElementType();
+    for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
+      llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
+      LValue LV = MakeAddrLValue(EltAddr, EltTy);
+      AI = ExpandTypeFromArgs(EltTy, LV, AI);
     }
+  } else if (const RecordType *RT = Ty->getAsStructureType()) {
+    RecordDecl *RD = RT->getDecl();
+    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+         i != e; ++i) {
+      FieldDecl *FD = *i;
+      QualType FT = FD->getType();
+
+      // FIXME: What are the right qualifiers here?
+      LValue LV = EmitLValueForField(Addr, FD, 0);
+      AI = ExpandTypeFromArgs(FT, LV, AI);
+    }
+  } else if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
+    QualType EltTy = CT->getElementType();
+    llvm::Value *RealAddr = Builder.CreateStructGEP(Addr, 0, "real");
+    EmitStoreThroughLValue(RValue::get(AI++), MakeAddrLValue(RealAddr, EltTy));
+    llvm::Value *ImagAddr = Builder.CreateStructGEP(Addr, 0, "imag");
+    EmitStoreThroughLValue(RValue::get(AI++), MakeAddrLValue(ImagAddr, EltTy));
+  } else {
+    EmitStoreThroughLValue(RValue::get(AI), LV);
+    ++AI;
   }
 
   return AI;
@@ -367,12 +382,12 @@ CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
 /// with an in-memory size smaller than DstSize.
 static llvm::Value *
 EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
-                                   const llvm::StructType *SrcSTy,
+                                   llvm::StructType *SrcSTy,
                                    uint64_t DstSize, CodeGenFunction &CGF) {
   // We can't dive into a zero-element struct.
   if (SrcSTy->getNumElements() == 0) return SrcPtr;
 
-  const llvm::Type *FirstElt = SrcSTy->getElementType(0);
+  llvm::Type *FirstElt = SrcSTy->getElementType(0);
 
   // If the first elt is at least as large as what we're looking for, or if the
   // first element is the same size as the whole struct, we can enter it.
@@ -386,9 +401,9 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
   SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
 
   // If the first element is a struct, recurse.
-  const llvm::Type *SrcTy =
+  llvm::Type *SrcTy =
     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
-  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
+  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
     return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
 
   return SrcPtr;
@@ -398,7 +413,7 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
 /// are either integers or pointers.  This does a truncation of the value if it
 /// is too large or a zero extension if it is too small.
 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
-                                             const llvm::Type *Ty,
+                                             llvm::Type *Ty,
                                              CodeGenFunction &CGF) {
   if (Val->getType() == Ty)
     return Val;
@@ -412,7 +427,7 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
     Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
   }
 
-  const llvm::Type *DestIntTy = Ty;
+  llvm::Type *DestIntTy = Ty;
   if (isa<llvm::PointerType>(DestIntTy))
     DestIntTy = CGF.IntPtrTy;
 
@@ -433,9 +448,9 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
 /// destination type; in this situation the values of bits which not
 /// present in the src are undefined.
 static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
-                                      const llvm::Type *Ty,
+                                      llvm::Type *Ty,
                                       CodeGenFunction &CGF) {
-  const llvm::Type *SrcTy =
+  llvm::Type *SrcTy =
     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
 
   // If SrcTy and Ty are the same, just do a load.
@@ -444,7 +459,7 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
 
   uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
 
-  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
+  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
     SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
     SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
   }
@@ -495,7 +510,7 @@ static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
                           llvm::Value *DestPtr, bool DestIsVolatile,
                           bool LowAlignment) {
   // Prefer scalar stores to first-class aggregate stores.
-  if (const llvm::StructType *STy =
+  if (llvm::StructType *STy =
         dyn_cast<llvm::StructType>(Val->getType())) {
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
       llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
@@ -519,8 +534,8 @@ static void CreateCoercedStore(llvm::Value *Src,
                                llvm::Value *DstPtr,
                                bool DstIsVolatile,
                                CodeGenFunction &CGF) {
-  const llvm::Type *SrcTy = Src->getType();
-  const llvm::Type *DstTy =
+  llvm::Type *SrcTy = Src->getType();
+  llvm::Type *DstTy =
     cast<llvm::PointerType>(DstPtr->getType())->getElementType();
   if (SrcTy == DstTy) {
     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
@@ -529,7 +544,7 @@ static void CreateCoercedStore(llvm::Value *Src,
 
   uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
 
-  if (const llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
+  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
     DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
     DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
   }
@@ -584,11 +599,11 @@ bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
     default:
       return false;
     case BuiltinType::Float:
-      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Float);
+      return getContext().getTargetInfo().useObjCFPRetForRealType(TargetInfo::Float);
     case BuiltinType::Double:
-      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Double);
+      return getContext().getTargetInfo().useObjCFPRetForRealType(TargetInfo::Double);
     case BuiltinType::LongDouble:
-      return getContext().Target.useObjCFPRetForRealType(
+      return getContext().getTargetInfo().useObjCFPRetForRealType(
         TargetInfo::LongDouble);
     }
   }
@@ -614,8 +629,8 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
   bool Inserted = FunctionsBeingProcessed.insert(&FI); (void)Inserted;
   assert(Inserted && "Recursively being processed?");
   
-  llvm::SmallVector<llvm::Type*, 8> argTypes;
-  const llvm::Type *resultType = 0;
+  SmallVector<llvm::Type*, 8> argTypes;
+  llvm::Type *resultType = 0;
 
   const ABIArgInfo &retAI = FI.getReturnInfo();
   switch (retAI.getKind()) {
@@ -632,7 +647,7 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
     resultType = llvm::Type::getVoidTy(getLLVMContext());
 
     QualType ret = FI.getReturnType();
-    const llvm::Type *ty = ConvertType(ret);
+    llvm::Type *ty = ConvertType(ret);
     unsigned addressSpace = Context.getTargetAddressSpace(ret);
     argTypes.push_back(llvm::PointerType::get(ty, addressSpace));
     break;
@@ -653,7 +668,7 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
 
     case ABIArgInfo::Indirect: {
       // indirect arguments are always on the stack, which is addr space #0.
-      const llvm::Type *LTy = ConvertTypeForMem(it->type);
+      llvm::Type *LTy = ConvertTypeForMem(it->type);
       argTypes.push_back(LTy->getPointerTo());
       break;
     }
@@ -664,7 +679,7 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
       // way is semantically identical, but fast-isel and the optimizer
       // generally likes scalar values better than FCAs.
       llvm::Type *argType = argAI.getCoerceToType();
-      if (const llvm::StructType *st = dyn_cast<llvm::StructType>(argType)) {
+      if (llvm::StructType *st = dyn_cast<llvm::StructType>(argType)) {
         for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
           argTypes.push_back(st->getElementType(i));
       } else {
@@ -685,7 +700,7 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
   return llvm::FunctionType::get(resultType, argTypes, isVariadic);
 }
 
-const llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
+llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
 
@@ -714,6 +729,8 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
 
   // FIXME: handle sseregparm someday...
   if (TargetDecl) {
+    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
+      FuncAttrs |= llvm::Attribute::ReturnsTwice;
     if (TargetDecl->hasAttr<NoThrowAttr>())
       FuncAttrs |= llvm::Attribute::NoUnwind;
     else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
@@ -724,10 +741,18 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
 
     if (TargetDecl->hasAttr<NoReturnAttr>())
       FuncAttrs |= llvm::Attribute::NoReturn;
-    if (TargetDecl->hasAttr<ConstAttr>())
+
+    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
+      FuncAttrs |= llvm::Attribute::ReturnsTwice;
+
+    // 'const' and 'pure' attribute functions are also nounwind.
+    if (TargetDecl->hasAttr<ConstAttr>()) {
       FuncAttrs |= llvm::Attribute::ReadNone;
-    else if (TargetDecl->hasAttr<PureAttr>())
+      FuncAttrs |= llvm::Attribute::NoUnwind;
+    } else if (TargetDecl->hasAttr<PureAttr>()) {
       FuncAttrs |= llvm::Attribute::ReadOnly;
+      FuncAttrs |= llvm::Attribute::NoUnwind;
+    }
     if (TargetDecl->hasAttr<MallocAttr>())
       RetAttrs |= llvm::Attribute::NoAlias;
   }
@@ -763,7 +788,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
     break;
 
   case ABIArgInfo::Expand:
-    assert(0 && "Invalid ABI kind for return argument");
+    llvm_unreachable("Invalid ABI kind for return argument");
   }
 
   if (RetAttrs)
@@ -776,7 +801,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
   else
     RegParm = CodeGenOpts.NumRegisterParameters;
 
-  unsigned PointerWidth = getContext().Target.getPointerWidth(0);
+  unsigned PointerWidth = getContext().getTargetInfo().getPointerWidth(0);
   for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
          ie = FI.arg_end(); it != ie; ++it) {
     QualType ParamType = it->type;
@@ -803,7 +828,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
       }
       // FIXME: handle sseregparm someday...
 
-      if (const llvm::StructType *STy =
+      if (llvm::StructType *STy =
             dyn_cast<llvm::StructType>(AI.getCoerceToType()))
         Index += STy->getNumElements()-1;  // 1 will be added below.
       break;
@@ -824,7 +849,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
       continue;
 
     case ABIArgInfo::Expand: {
-      llvm::SmallVector<llvm::Type*, 8> types;
+      SmallVector<llvm::Type*, 8> types;
       // FIXME: This is rather inefficient. Do we ever actually need to do
       // anything here? The result should be just reconstructed on the other
       // side, so extension should be a non-issue.
@@ -847,7 +872,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
                                          const VarDecl *var,
                                          llvm::Value *value) {
-  const llvm::Type *varType = CGF.ConvertType(var->getType());
+  llvm::Type *varType = CGF.ConvertType(var->getType());
 
   // This can happen with promotions that actually don't change the
   // underlying type, like the enum promotions.
@@ -872,7 +897,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
     if (FD->hasImplicitReturnZero()) {
       QualType RetTy = FD->getResultType().getUnqualifiedType();
-      const llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
+      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
       llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
       Builder.CreateStore(Zero, ReturnValue);
     }
@@ -887,6 +912,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
   // Name the struct return argument.
   if (CGM.ReturnTypeUsesSRet(FI)) {
     AI->setName("agg.result");
+    AI->addAttr(llvm::Attribute::NoAlias);
     ++AI;
   }
 
@@ -918,7 +944,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
           //
           // FIXME: We should have a common utility for generating an aggregate
           // copy.
-          const llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
+          llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
           CharUnits Size = getContext().getTypeSizeInChars(Ty);
           llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
           llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
@@ -954,9 +980,13 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         if (Arg->getType().isRestrictQualified())
           AI->addAttr(llvm::Attribute::NoAlias);
 
+        // Ensure the argument is the correct type.
+        if (V->getType() != ArgI.getCoerceToType())
+          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
+
         if (isPromoted)
           V = emitArgumentDemotion(*this, Arg, V);
-
+        
         EmitParmDecl(*Arg, V, ArgNo);
         break;
       }
@@ -985,13 +1015,13 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // If the coerce-to type is a first class aggregate, we flatten it and
       // pass the elements. Either way is semantically identical, but fast-isel
       // and the optimizer generally likes scalar values better than FCAs.
-      if (const llvm::StructType *STy =
+      if (llvm::StructType *STy =
             dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
         Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
 
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           assert(AI != Fn->arg_end() && "Argument mismatch!");
-          AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
+          AI->setName(Arg->getName() + ".coerce" + Twine(i));
           llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
           Builder.CreateStore(AI++, EltPtr);
         }
@@ -1025,7 +1055,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // Name the arguments used in expansion and increment AI.
       unsigned Index = 0;
       for (; AI != End; ++AI, ++Index)
-        AI->setName(Arg->getName() + "." + llvm::Twine(Index));
+        AI->setName(Arg->getName() + "." + Twine(Index));
       continue;
     }
 
@@ -1054,12 +1084,12 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
   if (BB->empty()) return 0;
   if (&BB->back() != result) return 0;
 
-  const llvm::Type *resultType = result->getType();
+  llvm::Type *resultType = result->getType();
 
   // result is in a BasicBlock and is therefore an Instruction.
   llvm::Instruction *generator = cast<llvm::Instruction>(result);
 
-  llvm::SmallVector<llvm::Instruction*,4> insnsToKill;
+  SmallVector<llvm::Instruction*,4> insnsToKill;
 
   // Look for:
   //  %generator = bitcast %type1* %generator2 to %type2*
@@ -1112,7 +1142,7 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
   }
 
   // Delete all the unnecessary instructions, from latest to earliest.
-  for (llvm::SmallVectorImpl<llvm::Instruction*>::iterator
+  for (SmallVectorImpl<llvm::Instruction*>::iterator
          i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
     (*i)->eraseFromParent();
 
@@ -1218,7 +1248,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
     break;
 
   case ABIArgInfo::Expand:
-    assert(0 && "Invalid ABI kind for return argument");
+    llvm_unreachable("Invalid ABI kind for return argument");
   }
 
   llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
@@ -1324,7 +1354,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
   // The dest and src types don't necessarily match in LLVM terms
   // because of the crazy ObjC compatibility rules.
 
-  const llvm::PointerType *destType =
+  llvm::PointerType *destType =
     cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
 
   // If the address is a constant null, just pass the appropriate null.
@@ -1406,9 +1436,14 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     return emitWritebackArg(*this, args, CRE);
   }
 
-  if (type->isReferenceType())
+  assert(type->isReferenceType() == E->isGLValue() &&
+         "reference binding to unmaterialized r-value!");
+
+  if (E->isGLValue()) {
+    assert(E->getObjectKind() == OK_Ordinary);
     return args.add(EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0),
                     type);
+  }
 
   if (hasAggregateLLVMType(type) && !E->getType()->isAnyComplexType() &&
       isa<ImplicitCastExpr>(E) &&
@@ -1427,8 +1462,8 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
 /// on the current state of the EH stack.
 llvm::CallSite
 CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
-                                  llvm::ArrayRef<llvm::Value *> Args,
-                                  const llvm::Twine &Name) {
+                                  ArrayRef<llvm::Value *> Args,
+                                  const Twine &Name) {
   llvm::BasicBlock *InvokeDest = getInvokeDest();
   if (!InvokeDest)
     return Builder.CreateCall(Callee, Args, Name);
@@ -1442,8 +1477,8 @@ CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
 
 llvm::CallSite
 CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
-                                  const llvm::Twine &Name) {
-  return EmitCallOrInvoke(Callee, llvm::ArrayRef<llvm::Value *>(), Name);
+                                  const Twine &Name) {
+  return EmitCallOrInvoke(Callee, ArrayRef<llvm::Value *>(), Name);
 }
 
 static void checkArgMatches(llvm::Value *Elt, unsigned &ArgNo,
@@ -1456,28 +1491,45 @@ static void checkArgMatches(llvm::Value *Elt, unsigned &ArgNo,
 }
 
 void CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
-                                       llvm::SmallVector<llvm::Value*,16> &Args,
+                                       SmallVector<llvm::Value*,16> &Args,
                                        llvm::FunctionType *IRFuncTy) {
-  const RecordType *RT = Ty->getAsStructureType();
-  assert(RT && "Can only expand structure types.");
-  
-  RecordDecl *RD = RT->getDecl();
-  assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
-  llvm::Value *Addr = RV.getAggregateAddr();
-  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
-       i != e; ++i) {
-    FieldDecl *FD = *i;
-    QualType FT = FD->getType();
-    
-    // FIXME: What are the right qualifiers here?
-    LValue LV = EmitLValueForField(Addr, FD, 0);
-    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
-      ExpandTypeToArgs(FT, RValue::getAggregate(LV.getAddress()),
-                       Args, IRFuncTy);
-      continue;
+  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
+    unsigned NumElts = AT->getSize().getZExtValue();
+    QualType EltTy = AT->getElementType();
+    llvm::Value *Addr = RV.getAggregateAddr();
+    for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
+      llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
+      LValue LV = MakeAddrLValue(EltAddr, EltTy);
+      RValue EltRV;
+      if (CodeGenFunction::hasAggregateLLVMType(EltTy))
+        EltRV = RValue::getAggregate(LV.getAddress());
+      else
+        EltRV = EmitLoadOfLValue(LV);
+      ExpandTypeToArgs(EltTy, EltRV, Args, IRFuncTy);
     }
+  } else if (const RecordType *RT = Ty->getAsStructureType()) {
+    RecordDecl *RD = RT->getDecl();
+    assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
+    llvm::Value *Addr = RV.getAggregateAddr();
+    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+         i != e; ++i) {
+      FieldDecl *FD = *i;
+      QualType FT = FD->getType();
     
-    RValue RV = EmitLoadOfLValue(LV);
+      // FIXME: What are the right qualifiers here?
+      LValue LV = EmitLValueForField(Addr, FD, 0);
+      RValue FldRV;
+      if (CodeGenFunction::hasAggregateLLVMType(FT))
+        FldRV = RValue::getAggregate(LV.getAddress());
+      else
+        FldRV = EmitLoadOfLValue(LV);
+      ExpandTypeToArgs(FT, FldRV, Args, IRFuncTy);
+    }
+  } else if (isa<ComplexType>(Ty)) {
+    ComplexPairTy CV = RV.getComplexVal();
+    Args.push_back(CV.first);
+    Args.push_back(CV.second);
+  } else {
     assert(RV.isScalar() &&
            "Unexpected non-scalar rvalue during struct expansion.");
 
@@ -1499,7 +1551,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                  const Decl *TargetDecl,
                                  llvm::Instruction **callOrInvoke) {
   // FIXME: We no longer need the types from CallArgs; lift up and simplify.
-  llvm::SmallVector<llvm::Value*, 16> Args;
+  SmallVector<llvm::Value*, 16> Args;
 
   // Handle struct-return functions by passing a pointer to the
   // location that we would like to return into.
@@ -1630,7 +1682,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       // If the coerce-to type is a first class aggregate, we flatten it and
       // pass the elements. Either way is semantically identical, but fast-isel
       // and the optimizer generally likes scalar values better than FCAs.
-      if (const llvm::StructType *STy =
+      if (llvm::StructType *STy =
             dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
         SrcPtr = Builder.CreateBitCast(SrcPtr,
                                        llvm::PointerType::getUnqual(STy));
@@ -1668,10 +1720,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // with unprototyped functions.
   if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
     if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
-      const llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
-      const llvm::FunctionType *CurFT =
+      llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
+      llvm::FunctionType *CurFT =
         cast<llvm::FunctionType>(CurPT->getElementType());
-      const llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
+      llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
 
       if (CE->getOpcode() == llvm::Instruction::BitCast &&
           ActualFT->getReturnType() == CurFT->getReturnType() &&
@@ -1813,11 +1865,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   }
 
   case ABIArgInfo::Expand:
-    assert(0 && "Invalid ABI kind for return argument");
+    llvm_unreachable("Invalid ABI kind for return argument");
   }
 
-  assert(0 && "Unhandled ABIArgInfo::Kind");
-  return RValue::get(0);
+  llvm_unreachable("Unhandled ABIArgInfo::Kind");
 }
 
 /* VarArg handling */
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h
index 343b944bf6c9..24ed366dd860 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h
@@ -42,7 +42,7 @@ namespace clang {
   class VarDecl;
 
 namespace CodeGen {
-  typedef llvm::SmallVector<llvm::AttributeWithIndex, 8> AttributeListType;
+  typedef SmallVector<llvm::AttributeWithIndex, 8> AttributeListType;
 
   struct CallArg {
     RValue RV;
@@ -56,7 +56,7 @@ namespace CodeGen {
   /// CallArgList - Type for representing both the value and type of
   /// arguments in a call.
   class CallArgList :
-    public llvm::SmallVector<CallArg, 16> {
+    public SmallVector<CallArg, 16> {
   public:
     struct Writeback {
       /// The original argument.
@@ -90,18 +90,18 @@ namespace CodeGen {
 
     bool hasWritebacks() const { return !Writebacks.empty(); }
 
-    typedef llvm::SmallVectorImpl<Writeback>::const_iterator writeback_iterator;
+    typedef SmallVectorImpl<Writeback>::const_iterator writeback_iterator;
     writeback_iterator writeback_begin() const { return Writebacks.begin(); }
     writeback_iterator writeback_end() const { return Writebacks.end(); }
 
   private:
-    llvm::SmallVector<Writeback, 1> Writebacks;
+    SmallVector<Writeback, 1> Writebacks;
   };
 
   /// FunctionArgList - Type for representing both the decl and type
   /// of parameters to a function. The decl must be either a
   /// ParmVarDecl or ImplicitParamDecl.
-  class FunctionArgList : public llvm::SmallVector<const VarDecl*, 16> {
+  class FunctionArgList : public SmallVector<const VarDecl*, 16> {
   };
 
   /// CGFunctionInfo - Class to encapsulate the information about a
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp
index 7dbaaf85299f..c28ecc05ded6 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp
@@ -62,7 +62,7 @@ CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
   if (Offset.isZero())
     return 0;
   
-  const llvm::Type *PtrDiffTy = 
+  llvm::Type *PtrDiffTy = 
   Types.ConvertType(getContext().getPointerDiffType());
   
   return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
@@ -95,7 +95,7 @@ CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This,
   // TODO: for complete types, this should be possible with a GEP.
   llvm::Value *V = This;
   if (Offset.isPositive()) {
-    const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
+    llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
     V = Builder.CreateBitCast(V, Int8PtrTy);
     V = Builder.CreateConstInBoundsGEP1_64(V, Offset.getQuantity());
   }
@@ -107,7 +107,7 @@ CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This,
 static llvm::Value *
 ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ThisPtr,
                                 CharUnits NonVirtual, llvm::Value *Virtual) {
-  const llvm::Type *PtrDiffTy = 
+  llvm::Type *PtrDiffTy = 
     CGF.ConvertType(CGF.getContext().getPointerDiffType());
   
   llvm::Value *NonVirtualOffset = 0;
@@ -125,7 +125,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ThisPtr,
     BaseOffset = NonVirtualOffset;
   
   // Apply the base offset.
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   ThisPtr = CGF.Builder.CreateBitCast(ThisPtr, Int8PtrTy);
   ThisPtr = CGF.Builder.CreateGEP(ThisPtr, BaseOffset, "add.ptr");
 
@@ -155,7 +155,7 @@ CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
                                      Start, PathEnd);
 
   // Get the base pointer type.
-  const llvm::Type *BasePtrTy = 
+  llvm::Type *BasePtrTy = 
     ConvertType((PathEnd[-1])->getType())->getPointerTo();
   
   if (NonVirtualOffset.isZero() && !VBase) {
@@ -225,7 +225,7 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
 
   QualType DerivedTy =
     getContext().getCanonicalType(getContext().getTagDeclType(Derived));
-  const llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo();
+  llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo();
   
   llvm::Value *NonVirtualOffset =
     CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
@@ -398,8 +398,11 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
                                               BaseClassDecl,
                                               isBaseVirtual);
 
-  AggValueSlot AggSlot = AggValueSlot::forAddr(V, Qualifiers(), 
-                                               /*Lifetime*/ true);
+  AggValueSlot AggSlot =
+    AggValueSlot::forAddr(V, Qualifiers(),
+                          AggValueSlot::IsDestructed,
+                          AggValueSlot::DoesNotNeedGCBarriers,
+                          AggValueSlot::IsNotAliased);
 
   CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
   
@@ -436,8 +439,11 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF,
       CGF.EmitComplexExprIntoAddr(MemberInit->getInit(), Dest, 
                                   LHS.isVolatileQualified());
     } else {    
-      AggValueSlot Slot = AggValueSlot::forAddr(Dest, LHS.getQuals(),
-                                                /*Lifetime*/ true);
+      AggValueSlot Slot =
+        AggValueSlot::forAddr(Dest, LHS.getQuals(),
+                              AggValueSlot::IsDestructed,
+                              AggValueSlot::DoesNotNeedGCBarriers,
+                              AggValueSlot::IsNotAliased);
       
       CGF.EmitAggExpr(MemberInit->getInit(), Slot);
     }
@@ -521,6 +527,12 @@ namespace {
     }
   };
 }
+
+static bool hasTrivialCopyOrMoveConstructor(const CXXRecordDecl *Record,
+                                            bool Moving) {
+  return Moving ? Record->hasTrivialMoveConstructor() :
+                  Record->hasTrivialCopyConstructor();
+}
   
 static void EmitMemberInitializer(CodeGenFunction &CGF,
                                   const CXXRecordDecl *ClassDecl,
@@ -547,11 +559,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
     LHS = CGF.EmitLValueForFieldInitialization(ThisPtr, Field, 0);
   }
 
-  // FIXME: If there's no initializer and the CXXCtorInitializer
-  // was implicitly generated, we shouldn't be zeroing memory.
-  if (FieldType->isArrayType() && !MemberInit->getInit()) {
-    CGF.EmitNullInitialization(LHS.getAddress(), Field->getType());
-  } else if (!CGF.hasAggregateLLVMType(Field->getType())) {
+  if (!CGF.hasAggregateLLVMType(Field->getType())) {
     if (LHS.isSimple()) {
       CGF.EmitExprAsInit(MemberInit->getInit(), Field, LHS, false);
     } else {
@@ -565,15 +573,15 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
     llvm::Value *ArrayIndexVar = 0;
     const ConstantArrayType *Array
       = CGF.getContext().getAsConstantArrayType(FieldType);
-    if (Array && Constructor->isImplicit() && 
-        Constructor->isCopyConstructor()) {
-      const llvm::Type *SizeTy
+    if (Array && Constructor->isImplicitlyDefined() &&
+        Constructor->isCopyOrMoveConstructor()) {
+      llvm::Type *SizeTy
         = CGF.ConvertType(CGF.getContext().getSizeType());
       
       // The LHS is a pointer to the first object we'll be constructing, as
       // a flat array.
       QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
-      const llvm::Type *BasePtr = CGF.ConvertType(BaseElementTy);
+      llvm::Type *BasePtr = CGF.ConvertType(BaseElementTy);
       BasePtr = llvm::PointerType::getUnqual(BasePtr);
       llvm::Value *BaseAddrPtr = CGF.Builder.CreateBitCast(LHS.getAddress(), 
                                                            BasePtr);
@@ -589,7 +597,8 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
       // constructors, perform a single aggregate copy.
       const CXXRecordDecl *Record = BaseElementTy->getAsCXXRecordDecl();
       if (BaseElementTy.isPODType(CGF.getContext()) ||
-          (Record && Record->hasTrivialCopyConstructor())) {
+          (Record && hasTrivialCopyOrMoveConstructor(Record,
+                         Constructor->isMoveConstructor()))) {
         // Find the source pointer. We knows it's the last argument because
         // we know we're in a copy constructor.
         unsigned SrcArgIndex = Args.size() - 1;
@@ -684,7 +693,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
   // delegation optimization.
   if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor)) {
     if (CGDebugInfo *DI = getDebugInfo()) 
-      DI->EmitStopPoint(Builder);
+      DI->EmitLocation(Builder, Ctor->getLocEnd());
     EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args);
     return;
   }
@@ -729,7 +738,7 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
 
   const CXXRecordDecl *ClassDecl = CD->getParent();
 
-  llvm::SmallVector<CXXCtorInitializer *, 8> MemberInitializers;
+  SmallVector<CXXCtorInitializer *, 8> MemberInitializers;
   
   for (CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
        E = CD->init_end();
@@ -971,6 +980,10 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
 
   const CXXRecordDecl *ClassDecl = DD->getParent();
 
+  // Unions have no bases and do not call field destructors.
+  if (ClassDecl->isUnion())
+    return;
+
   // The complete-destructor phase just destructs all the virtual bases.
   if (DtorType == Dtor_Complete) {
 
@@ -1018,7 +1031,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
   }
 
   // Destroy direct fields.
-  llvm::SmallVector<const FieldDecl *, 16> FieldDecls;
+  SmallVector<const FieldDecl *, 16> FieldDecls;
   for (CXXRecordDecl::field_iterator I = ClassDecl->field_begin(),
        E = ClassDecl->field_end(); I != E; ++I) {
     const FieldDecl *field = *I;
@@ -1195,7 +1208,8 @@ CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
     }
 
     assert(ArgBeg + 1 == ArgEnd && "unexpected argcount for trivial ctor");
-    assert(D->isCopyConstructor() && "trivial 1-arg ctor not a copy ctor");
+    assert(D->isCopyOrMoveConstructor() &&
+           "trivial 1-arg ctor not a copy/move ctor");
 
     const Expr *E = (*ArgBeg);
     QualType Ty = E->getType();
@@ -1217,7 +1231,8 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
                                         CallExpr::const_arg_iterator ArgEnd) {
   if (D->isTrivial()) {
     assert(ArgBeg + 1 == ArgEnd && "unexpected argcount for trivial ctor");
-    assert(D->isCopyConstructor() && "trivial 1-arg ctor not a copy ctor");
+    assert(D->isCopyOrMoveConstructor() &&
+           "trivial 1-arg ctor not a copy/move ctor");
     EmitAggregateCopy(This, Src, (*ArgBeg)->getType());
     return;
   }
@@ -1236,7 +1251,7 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
   
   // Push the src ptr.
   QualType QT = *(FPT->arg_type_begin());
-  const llvm::Type *t = CGM.getTypes().ConvertType(QT);
+  llvm::Type *t = CGM.getTypes().ConvertType(QT);
   Src = Builder.CreateBitCast(Src, t);
   Args.add(RValue::get(Src), QT);
   
@@ -1258,10 +1273,8 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
     EmitCallArg(Args, *Arg, ArgType);
   }
   
-  QualType ResultType = FPT->getResultType();
-  EmitCall(CGM.getTypes().getFunctionInfo(ResultType, Args,
-                                          FPT->getExtInfo()),
-                  Callee, ReturnValueSlot(), Args, D);
+  EmitCall(CGM.getTypes().getFunctionInfo(Args, FPT), Callee,
+           ReturnValueSlot(), Args, D);
 }
 
 void
@@ -1326,7 +1339,10 @@ CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor
   llvm::Value *ThisPtr = LoadCXXThis();
 
   AggValueSlot AggSlot =
-    AggValueSlot::forAddr(ThisPtr, Qualifiers(), /*Lifetime*/ true);
+    AggValueSlot::forAddr(ThisPtr, Qualifiers(),
+                          AggValueSlot::IsDestructed,
+                          AggValueSlot::DoesNotNeedGCBarriers,
+                          AggValueSlot::IsNotAliased);
 
   EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
 
@@ -1394,12 +1410,12 @@ CodeGenFunction::GetVirtualBaseClassOffset(llvm::Value *This,
                                            const CXXRecordDecl *BaseClassDecl) {
   llvm::Value *VTablePtr = GetVTablePtr(This, Int8PtrTy);
   CharUnits VBaseOffsetOffset = 
-    CGM.getVTables().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl);
+    CGM.getVTableContext().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl);
   
   llvm::Value *VBaseOffsetPtr = 
     Builder.CreateConstGEP1_64(VTablePtr, VBaseOffsetOffset.getQuantity(), 
                                "vbase.offset.ptr");
-  const llvm::Type *PtrDiffTy = 
+  llvm::Type *PtrDiffTy = 
     ConvertType(getContext().getPointerDiffType());
   
   VBaseOffsetPtr = Builder.CreateBitCast(VBaseOffsetPtr, 
@@ -1436,7 +1452,8 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
     // And load the address point from the VTT.
     VTableAddressPoint = Builder.CreateLoad(VTT);
   } else {
-    uint64_t AddressPoint = CGM.getVTables().getAddressPoint(Base, VTableClass);
+    uint64_t AddressPoint =
+      CGM.getVTableContext().getVTableLayout(VTableClass).getAddressPoint(Base);
     VTableAddressPoint =
       Builder.CreateConstInBoundsGEP2_64(VTable, 0, AddressPoint);
   }
@@ -1465,7 +1482,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
                                                   VirtualOffset);
 
   // Finally, store the address point.
-  const llvm::Type *AddressPointPtrTy =
+  llvm::Type *AddressPointPtrTy =
     VTableAddressPoint->getType()->getPointerTo();
   VTableField = Builder.CreateBitCast(VTableField, AddressPointPtrTy);
   Builder.CreateStore(VTableAddressPoint, VTableField);
@@ -1549,7 +1566,7 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
 }
 
 llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This,
-                                           const llvm::Type *Ty) {
+                                           llvm::Type *Ty) {
   llvm::Value *VTablePtrSrc = Builder.CreateBitCast(This, Ty->getPointerTo());
   return Builder.CreateLoad(VTablePtrSrc, "vtable");
 }
@@ -1605,7 +1622,6 @@ static const Expr *skipNoOpCastsAndParens(const Expr *E) {
 
 /// canDevirtualizeMemberFunctionCall - Checks whether the given virtual member
 /// function call on the given expr can be devirtualized.
-/// expr can be devirtualized.
 static bool canDevirtualizeMemberFunctionCall(const Expr *Base, 
                                               const CXXMethodDecl *MD) {
   // If the most derived class is marked final, we know that no subclass can
@@ -1677,7 +1693,7 @@ CodeGenFunction::EmitCXXOperatorMemberCallee(const CXXOperatorCallExpr *E,
                                              const CXXMethodDecl *MD,
                                              llvm::Value *This) {
   const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
-  const llvm::Type *Ty =
+  llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
                                    FPT->isVariadic());
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp
index 9c5dd1f23721..b2d0786cb6cd 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp
@@ -48,7 +48,7 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
 
   if (rv.isComplex()) {
     CodeGenFunction::ComplexPairTy V = rv.getComplexVal();
-    const llvm::Type *ComplexTy =
+    llvm::Type *ComplexTy =
       llvm::StructType::get(V.first->getType(), V.second->getType(),
                             (void*) 0);
     llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex");
@@ -119,16 +119,30 @@ char *EHScopeStack::allocate(size_t Size) {
 }
 
 EHScopeStack::stable_iterator
-EHScopeStack::getEnclosingEHCleanup(iterator it) const {
-  assert(it != end());
-  do {
-    if (isa<EHCleanupScope>(*it)) {
-      if (cast<EHCleanupScope>(*it).isEHCleanup())
-        return stabilize(it);
-      return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
+EHScopeStack::getInnermostActiveNormalCleanup() const {
+  for (stable_iterator si = getInnermostNormalCleanup(), se = stable_end();
+         si != se; ) {
+    EHCleanupScope &cleanup = cast<EHCleanupScope>(*find(si));
+    if (cleanup.isActive()) return si;
+    si = cleanup.getEnclosingNormalCleanup();
+  }
+  return stable_end();
+}
+
+EHScopeStack::stable_iterator EHScopeStack::getInnermostActiveEHScope() const {
+  for (stable_iterator si = getInnermostEHScope(), se = stable_end();
+         si != se; ) {
+    // Skip over inactive cleanups.
+    EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*find(si));
+    if (cleanup && !cleanup->isActive()) {
+      si = cleanup->getEnclosingEHScope();
+      continue;
     }
-    ++it;
-  } while (it != end());
+
+    // All other scopes are always active.
+    return si;
+  }
+
   return stable_end();
 }
 
@@ -146,11 +160,11 @@ void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
                                 Size,
                                 BranchFixups.size(),
                                 InnermostNormalCleanup,
-                                InnermostEHCleanup);
+                                InnermostEHScope);
   if (IsNormalCleanup)
     InnermostNormalCleanup = stable_begin();
   if (IsEHCleanup)
-    InnermostEHCleanup = stable_begin();
+    InnermostEHScope = stable_begin();
 
   return Scope->getCleanupBuffer();
 }
@@ -161,11 +175,9 @@ void EHScopeStack::popCleanup() {
   assert(isa<EHCleanupScope>(*begin()));
   EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
   InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
-  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+  InnermostEHScope = Cleanup.getEnclosingEHScope();
   StartOfData += Cleanup.getAllocatedSize();
 
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
   // Destroy the cleanup.
   Cleanup.~EHCleanupScope();
 
@@ -182,37 +194,35 @@ void EHScopeStack::popCleanup() {
   }
 }
 
-EHFilterScope *EHScopeStack::pushFilter(unsigned NumFilters) {
-  char *Buffer = allocate(EHFilterScope::getSizeForNumFilters(NumFilters));
-  CatchDepth++;
-  return new (Buffer) EHFilterScope(NumFilters);
+EHFilterScope *EHScopeStack::pushFilter(unsigned numFilters) {
+  assert(getInnermostEHScope() == stable_end());
+  char *buffer = allocate(EHFilterScope::getSizeForNumFilters(numFilters));
+  EHFilterScope *filter = new (buffer) EHFilterScope(numFilters);
+  InnermostEHScope = stable_begin();
+  return filter;
 }
 
 void EHScopeStack::popFilter() {
   assert(!empty() && "popping exception stack when not empty");
 
-  EHFilterScope &Filter = cast<EHFilterScope>(*begin());
-  StartOfData += EHFilterScope::getSizeForNumFilters(Filter.getNumFilters());
+  EHFilterScope &filter = cast<EHFilterScope>(*begin());
+  StartOfData += EHFilterScope::getSizeForNumFilters(filter.getNumFilters());
 
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched filter push/pop");
-  CatchDepth--;
+  InnermostEHScope = filter.getEnclosingEHScope();
 }
 
-EHCatchScope *EHScopeStack::pushCatch(unsigned NumHandlers) {
-  char *Buffer = allocate(EHCatchScope::getSizeForNumHandlers(NumHandlers));
-  CatchDepth++;
-  EHCatchScope *Scope = new (Buffer) EHCatchScope(NumHandlers);
-  for (unsigned I = 0; I != NumHandlers; ++I)
-    Scope->getHandlers()[I].Index = getNextEHDestIndex();
-  return Scope;
+EHCatchScope *EHScopeStack::pushCatch(unsigned numHandlers) {
+  char *buffer = allocate(EHCatchScope::getSizeForNumHandlers(numHandlers));
+  EHCatchScope *scope =
+    new (buffer) EHCatchScope(numHandlers, InnermostEHScope);
+  InnermostEHScope = stable_begin();
+  return scope;
 }
 
 void EHScopeStack::pushTerminate() {
   char *Buffer = allocate(EHTerminateScope::getSize());
-  CatchDepth++;
-  new (Buffer) EHTerminateScope(getNextEHDestIndex());
+  new (Buffer) EHTerminateScope(InnermostEHScope);
+  InnermostEHScope = stable_begin();
 }
 
 /// Remove any 'null' fixups on the stack.  However, we can't pop more
@@ -384,17 +394,6 @@ static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
   return Entry;
 }
 
-static llvm::BasicBlock *CreateEHEntry(CodeGenFunction &CGF,
-                                       EHCleanupScope &Scope) {
-  assert(Scope.isEHCleanup());
-  llvm::BasicBlock *Entry = Scope.getEHBlock();
-  if (!Entry) {
-    Entry = CGF.createBasicBlock("eh.cleanup");
-    Scope.setEHBlock(Entry);
-  }
-  return Entry;
-}
-
 /// Attempts to reduce a cleanup's entry block to a fallthrough.  This
 /// is basically llvm::MergeBlockIntoPredecessor, except
 /// simplified/optimized for the tighter constraints on cleanup blocks.
@@ -483,6 +482,49 @@ static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit,
   }
 }
 
+/// We don't need a normal entry block for the given cleanup.
+/// Optimistic fixup branches can cause these blocks to come into
+/// existence anyway;  if so, destroy it.
+///
+/// The validity of this transformation is very much specific to the
+/// exact ways in which we form branches to cleanup entries.
+static void destroyOptimisticNormalEntry(CodeGenFunction &CGF,
+                                         EHCleanupScope &scope) {
+  llvm::BasicBlock *entry = scope.getNormalBlock();
+  if (!entry) return;
+
+  // Replace all the uses with unreachable.
+  llvm::BasicBlock *unreachableBB = CGF.getUnreachableBlock();
+  for (llvm::BasicBlock::use_iterator
+         i = entry->use_begin(), e = entry->use_end(); i != e; ) {
+    llvm::Use &use = i.getUse();
+    ++i;
+
+    use.set(unreachableBB);
+    
+    // The only uses should be fixup switches.
+    llvm::SwitchInst *si = cast<llvm::SwitchInst>(use.getUser());
+    if (si->getNumCases() == 2 && si->getDefaultDest() == unreachableBB) {
+      // Replace the switch with a branch.
+      llvm::BranchInst::Create(si->getSuccessor(1), si);
+
+      // The switch operand is a load from the cleanup-dest alloca.
+      llvm::LoadInst *condition = cast<llvm::LoadInst>(si->getCondition());
+
+      // Destroy the switch.
+      si->eraseFromParent();
+
+      // Destroy the load.
+      assert(condition->getOperand(0) == CGF.NormalCleanupDest);
+      assert(condition->use_empty());
+      condition->eraseFromParent();
+    }
+  }
+  
+  assert(entry->use_empty());
+  delete entry;
+}
+
 /// Pops a cleanup block.  If the block includes a normal cleanup, the
 /// current insertion point is threaded through the cleanup, as are
 /// any branch fixups on the cleanup.
@@ -501,7 +543,10 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
 
   // Check whether we need an EH cleanup.  This is only true if we've
   // generated a lazy EH cleanup block.
-  bool RequiresEHCleanup = Scope.hasEHBranches();
+  llvm::BasicBlock *EHEntry = Scope.getCachedEHDispatchBlock();
+  assert(Scope.hasEHBranches() == (EHEntry != 0));
+  bool RequiresEHCleanup = (EHEntry != 0);
+  EHScopeStack::stable_iterator EHParent = Scope.getEnclosingEHScope();
 
   // Check the three conditions which might require a normal cleanup:
 
@@ -537,43 +582,37 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     RequiresNormalCleanup = true;
   }
 
-  EHScopeStack::Cleanup::Flags cleanupFlags;
-  if (Scope.isNormalCleanup())
-    cleanupFlags.setIsNormalCleanupKind();
-  if (Scope.isEHCleanup())
-    cleanupFlags.setIsEHCleanupKind();
-
-  // Even if we don't need the normal cleanup, we might still have
-  // prebranched fallthrough to worry about.
-  if (Scope.isNormalCleanup() && !RequiresNormalCleanup &&
-      HasPrebranchedFallthrough) {
-    assert(!IsActive);
-
-    llvm::BasicBlock *NormalEntry = Scope.getNormalBlock();
-
-    // If we're branching through this cleanup, just forward the
-    // prebranched fallthrough to the next cleanup, leaving the insert
-    // point in the old block.
+  // If we have a prebranched fallthrough into an inactive normal
+  // cleanup, rewrite it so that it leads to the appropriate place.
+  if (Scope.isNormalCleanup() && HasPrebranchedFallthrough && !IsActive) {
+    llvm::BasicBlock *prebranchDest;
+    
+    // If the prebranch is semantically branching through the next
+    // cleanup, just forward it to the next block, leaving the
+    // insertion point in the prebranched block.
     if (FallthroughIsBranchThrough) {
-      EHScope &S = *EHStack.find(Scope.getEnclosingNormalCleanup());
-      llvm::BasicBlock *EnclosingEntry = 
-        CreateNormalEntry(*this, cast<EHCleanupScope>(S));
+      EHScope &enclosing = *EHStack.find(Scope.getEnclosingNormalCleanup());
+      prebranchDest = CreateNormalEntry(*this, cast<EHCleanupScope>(enclosing));
 
-      ForwardPrebranchedFallthrough(FallthroughSource,
-                                    NormalEntry, EnclosingEntry);
-      assert(NormalEntry->use_empty() &&
-             "uses of entry remain after forwarding?");
-      delete NormalEntry;
-
-    // Otherwise, we're branching out;  just emit the next block.
+    // Otherwise, we need to make a new block.  If the normal cleanup
+    // isn't being used at all, we could actually reuse the normal
+    // entry block, but this is simpler, and it avoids conflicts with
+    // dead optimistic fixup branches.
     } else {
-      EmitBlock(NormalEntry);
-      SimplifyCleanupEntry(*this, NormalEntry);
+      prebranchDest = createBasicBlock("forwarded-prebranch");
+      EmitBlock(prebranchDest);
     }
+
+    llvm::BasicBlock *normalEntry = Scope.getNormalBlock();
+    assert(normalEntry && !normalEntry->use_empty());
+
+    ForwardPrebranchedFallthrough(FallthroughSource,
+                                  normalEntry, prebranchDest);
   }
 
   // If we don't need the cleanup at all, we're done.
   if (!RequiresNormalCleanup && !RequiresEHCleanup) {
+    destroyOptimisticNormalEntry(*this, Scope);
     EHStack.popCleanup(); // safe because there are no fixups
     assert(EHStack.getNumBranchFixups() == 0 ||
            EHStack.hasNormalCleanups());
@@ -583,7 +622,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
   // Copy the cleanup emission data out.  Note that SmallVector
   // guarantees maximal alignment for its buffer regardless of its
   // type parameter.
-  llvm::SmallVector<char, 8*sizeof(void*)> CleanupBuffer;
+  SmallVector<char, 8*sizeof(void*)> CleanupBuffer;
   CleanupBuffer.reserve(Scope.getCleanupSize());
   memcpy(CleanupBuffer.data(),
          Scope.getCleanupBuffer(), Scope.getCleanupSize());
@@ -591,63 +630,14 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
   EHScopeStack::Cleanup *Fn =
     reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
 
-  // We want to emit the EH cleanup after the normal cleanup, but go
-  // ahead and do the setup for the EH cleanup while the scope is still
-  // alive.
-  llvm::BasicBlock *EHEntry = 0;
-  llvm::SmallVector<llvm::Instruction*, 2> EHInstsToAppend;
-  if (RequiresEHCleanup) {
-    EHEntry = CreateEHEntry(*this, Scope);
-
-    // Figure out the branch-through dest if necessary.
-    llvm::BasicBlock *EHBranchThroughDest = 0;
-    if (Scope.hasEHBranchThroughs()) {
-      assert(Scope.getEnclosingEHCleanup() != EHStack.stable_end());
-      EHScope &S = *EHStack.find(Scope.getEnclosingEHCleanup());
-      EHBranchThroughDest = CreateEHEntry(*this, cast<EHCleanupScope>(S));
-    }
-
-    // If we have exactly one branch-after and no branch-throughs, we
-    // can dispatch it without a switch.
-    if (!Scope.hasEHBranchThroughs() &&
-        Scope.getNumEHBranchAfters() == 1) {
-      assert(!EHBranchThroughDest);
-
-      // TODO: remove the spurious eh.cleanup.dest stores if this edge
-      // never went through any switches.
-      llvm::BasicBlock *BranchAfterDest = Scope.getEHBranchAfterBlock(0);
-      EHInstsToAppend.push_back(llvm::BranchInst::Create(BranchAfterDest));
-    
-    // Otherwise, if we have any branch-afters, we need a switch.
-    } else if (Scope.getNumEHBranchAfters()) {
-      // The default of the switch belongs to the branch-throughs if
-      // they exist.
-      llvm::BasicBlock *Default =
-        (EHBranchThroughDest ? EHBranchThroughDest : getUnreachableBlock());
-
-      const unsigned SwitchCapacity = Scope.getNumEHBranchAfters();
-
-      llvm::LoadInst *Load =
-        new llvm::LoadInst(getEHCleanupDestSlot(), "cleanup.dest");
-      llvm::SwitchInst *Switch =
-        llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
-
-      EHInstsToAppend.push_back(Load);
-      EHInstsToAppend.push_back(Switch);
-
-      for (unsigned I = 0, E = Scope.getNumEHBranchAfters(); I != E; ++I)
-        Switch->addCase(Scope.getEHBranchAfterIndex(I),
-                        Scope.getEHBranchAfterBlock(I));
-
-    // Otherwise, we have only branch-throughs; jump to the next EH
-    // cleanup.
-    } else {
-      assert(EHBranchThroughDest);
-      EHInstsToAppend.push_back(llvm::BranchInst::Create(EHBranchThroughDest));
-    }
-  }
+  EHScopeStack::Cleanup::Flags cleanupFlags;
+  if (Scope.isNormalCleanup())
+    cleanupFlags.setIsNormalCleanupKind();
+  if (Scope.isEHCleanup())
+    cleanupFlags.setIsEHCleanupKind();
 
   if (!RequiresNormalCleanup) {
+    destroyOptimisticNormalEntry(*this, Scope);
     EHStack.popCleanup();
   } else {
     // If we have a fallthrough and no other need for the cleanup,
@@ -655,15 +645,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     if (HasFallthrough && !HasPrebranchedFallthrough &&
         !HasFixups && !HasExistingBranches) {
 
-      // Fixups can cause us to optimistically create a normal block,
-      // only to later have no real uses for it.  Just delete it in
-      // this case.
-      // TODO: we can potentially simplify all the uses after this.
-      if (Scope.getNormalBlock()) {
-        Scope.getNormalBlock()->replaceAllUsesWith(getUnreachableBlock());
-        delete Scope.getNormalBlock();
-      }
-
+      destroyOptimisticNormalEntry(*this, Scope);
       EHStack.popCleanup();
 
       EmitCleanup(*this, Fn, cleanupFlags, NormalActiveFlag);
@@ -676,18 +658,19 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
 
       // I.  Set up the fallthrough edge in.
 
+      CGBuilderTy::InsertPoint savedInactiveFallthroughIP;
+
       // If there's a fallthrough, we need to store the cleanup
       // destination index.  For fall-throughs this is always zero.
       if (HasFallthrough) {
         if (!HasPrebranchedFallthrough)
           Builder.CreateStore(Builder.getInt32(0), getNormalCleanupDestSlot());
 
-      // Otherwise, clear the IP if we don't have fallthrough because
-      // the cleanup is inactive.  We don't need to save it because
-      // it's still just FallthroughSource.
+      // Otherwise, save and clear the IP if we don't have fallthrough
+      // because the cleanup is inactive.
       } else if (FallthroughSource) {
         assert(!IsActive && "source without fallthrough for active cleanup");
-        Builder.ClearInsertionPoint();
+        savedInactiveFallthroughIP = Builder.saveAndClearIP();
       }
 
       // II.  Emit the entry block.  This implicitly branches to it if
@@ -716,7 +699,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
       }
 
       llvm::BasicBlock *FallthroughDest = 0;
-      llvm::SmallVector<llvm::Instruction*, 2> InstsToAppend;
+      SmallVector<llvm::Instruction*, 2> InstsToAppend;
 
       // If there's exactly one branch-after and no other threads,
       // we can route it without a switch.
@@ -800,25 +783,14 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
 
       // V.  Set up the fallthrough edge out.
       
-      // Case 1: a fallthrough source exists but shouldn't branch to
-      // the cleanup because the cleanup is inactive.
+      // Case 1: a fallthrough source exists but doesn't branch to the
+      // cleanup because the cleanup is inactive.
       if (!HasFallthrough && FallthroughSource) {
+        // Prebranched fallthrough was forwarded earlier.
+        // Non-prebranched fallthrough doesn't need to be forwarded.
+        // Either way, all we need to do is restore the IP we cleared before.
         assert(!IsActive);
-
-        // If we have a prebranched fallthrough, that needs to be
-        // forwarded to the right block.
-        if (HasPrebranchedFallthrough) {
-          llvm::BasicBlock *Next;
-          if (FallthroughIsBranchThrough) {
-            Next = BranchThroughDest;
-            assert(!FallthroughDest);
-          } else {
-            Next = FallthroughDest;
-          }
-
-          ForwardPrebranchedFallthrough(FallthroughSource, NormalEntry, Next);
-        }
-        Builder.SetInsertPoint(FallthroughSource);
+        Builder.restoreIP(savedInactiveFallthroughIP);
 
       // Case 2: a fallthrough source exists and should branch to the
       // cleanup, but we're not supposed to branch through to the next
@@ -864,10 +836,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     cleanupFlags.setIsForEHCleanup();
     EmitCleanup(*this, Fn, cleanupFlags, EHActiveFlag);
 
-    // Append the prepared cleanup prologue from above.
-    llvm::BasicBlock *EHExit = Builder.GetInsertBlock();
-    for (unsigned I = 0, E = EHInstsToAppend.size(); I != E; ++I)
-      EHExit->getInstList().push_back(EHInstsToAppend[I]);
+    Builder.CreateBr(getEHDispatchBlock(EHParent));
 
     Builder.restoreIP(SavedIP);
 
@@ -979,64 +948,6 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
   Builder.ClearInsertionPoint();
 }
 
-void CodeGenFunction::EmitBranchThroughEHCleanup(UnwindDest Dest) {
-  // We should never get invalid scope depths for an UnwindDest; that
-  // implies that the destination wasn't set up correctly.
-  assert(Dest.getScopeDepth().isValid() && "invalid scope depth on EH dest?");
-
-  if (!HaveInsertPoint())
-    return;
-
-  // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
-
-  // Calculate the innermost active cleanup.
-  EHScopeStack::stable_iterator
-    InnermostCleanup = EHStack.getInnermostActiveEHCleanup();
-
-  // If the destination is in the same EH cleanup scope as us, we
-  // don't need to thread through anything.
-  if (InnermostCleanup.encloses(Dest.getScopeDepth())) {
-    Builder.ClearInsertionPoint();
-    return;
-  }
-  assert(InnermostCleanup != EHStack.stable_end());
-
-  // Store the index at the start.
-  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
-  new llvm::StoreInst(Index, getEHCleanupDestSlot(), BI);
-
-  // Adjust BI to point to the first cleanup block.
-  {
-    EHCleanupScope &Scope =
-      cast<EHCleanupScope>(*EHStack.find(InnermostCleanup));
-    BI->setSuccessor(0, CreateEHEntry(*this, Scope));
-  }
-  
-  // Add this destination to all the scopes involved.
-  for (EHScopeStack::stable_iterator
-         I = InnermostCleanup, E = Dest.getScopeDepth(); ; ) {
-    assert(E.strictlyEncloses(I));
-    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
-    assert(Scope.isEHCleanup());
-    I = Scope.getEnclosingEHCleanup();
-
-    // If this is the last cleanup we're propagating through, add this
-    // as a branch-after.
-    if (I == E) {
-      Scope.addEHBranchAfter(Index, Dest.getBlock());
-      break;
-    }
-
-    // Otherwise, add it as a branch-through.  If this isn't new
-    // information, all the rest of the work has been done before.
-    if (!Scope.addEHBranchThrough(Dest.getBlock()))
-      break;
-  }
-  
-  Builder.ClearInsertionPoint();
-}
-
 static bool IsUsedAsNormalCleanup(EHScopeStack &EHStack,
                                   EHScopeStack::stable_iterator C) {
   // If we needed a normal block for any reason, that counts.
@@ -1057,18 +968,21 @@ static bool IsUsedAsNormalCleanup(EHScopeStack &EHStack,
 }
 
 static bool IsUsedAsEHCleanup(EHScopeStack &EHStack,
-                              EHScopeStack::stable_iterator C) {
+                              EHScopeStack::stable_iterator cleanup) {
   // If we needed an EH block for any reason, that counts.
-  if (cast<EHCleanupScope>(*EHStack.find(C)).getEHBlock())
+  if (EHStack.find(cleanup)->hasEHBranches())
     return true;
 
   // Check whether any enclosed cleanups were needed.
   for (EHScopeStack::stable_iterator
-         I = EHStack.getInnermostEHCleanup(); I != C; ) {
-    assert(C.strictlyEncloses(I));
-    EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
-    if (S.getEHBlock()) return true;
-    I = S.getEnclosingEHCleanup();
+         i = EHStack.getInnermostEHScope(); i != cleanup; ) {
+    assert(cleanup.strictlyEncloses(i));
+
+    EHScope &scope = *EHStack.find(i);
+    if (scope.hasEHBranches())
+      return true;
+
+    i = scope.getEnclosingEHScope();
   }
 
   return false;
@@ -1163,10 +1077,3 @@ llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
       CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
   return NormalCleanupDest;
 }
-
-llvm::Value *CodeGenFunction::getEHCleanupDestSlot() {
-  if (!EHCleanupDest)
-    EHCleanupDest =
-      CreateTempAlloca(Builder.getInt32Ty(), "eh.cleanup.dest.slot");
-  return EHCleanupDest;
-}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h
index c93ec5bb76a9..7726e442c025 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h
@@ -29,25 +29,102 @@ namespace CodeGen {
 /// A protected scope for zero-cost EH handling.
 class EHScope {
   llvm::BasicBlock *CachedLandingPad;
+  llvm::BasicBlock *CachedEHDispatchBlock;
 
-  unsigned K : 2;
+  EHScopeStack::stable_iterator EnclosingEHScope;
+
+  class CommonBitFields {
+    friend class EHScope;
+    unsigned Kind : 2;
+  };
+  enum { NumCommonBits = 2 };
 
 protected:
-  enum { BitsRemaining = 30 };
+  class CatchBitFields {
+    friend class EHCatchScope;
+    unsigned : NumCommonBits;
+
+    unsigned NumHandlers : 32 - NumCommonBits;
+  };
+
+  class CleanupBitFields {
+    friend class EHCleanupScope;
+    unsigned : NumCommonBits;
+
+    /// Whether this cleanup needs to be run along normal edges.
+    unsigned IsNormalCleanup : 1;
+
+    /// Whether this cleanup needs to be run along exception edges.
+    unsigned IsEHCleanup : 1;
+
+    /// Whether this cleanup is currently active.
+    unsigned IsActive : 1;
+
+    /// Whether the normal cleanup should test the activation flag.
+    unsigned TestFlagInNormalCleanup : 1;
+
+    /// Whether the EH cleanup should test the activation flag.
+    unsigned TestFlagInEHCleanup : 1;
+
+    /// The amount of extra storage needed by the Cleanup.
+    /// Always a multiple of the scope-stack alignment.
+    unsigned CleanupSize : 12;
+
+    /// The number of fixups required by enclosing scopes (not including
+    /// this one).  If this is the top cleanup scope, all the fixups
+    /// from this index onwards belong to this scope.
+    unsigned FixupDepth : 32 - 17 - NumCommonBits; // currently 13    
+  };
+
+  class FilterBitFields {
+    friend class EHFilterScope;
+    unsigned : NumCommonBits;
+
+    unsigned NumFilters : 32 - NumCommonBits;
+  };
+
+  union {
+    CommonBitFields CommonBits;
+    CatchBitFields CatchBits;
+    CleanupBitFields CleanupBits;
+    FilterBitFields FilterBits;
+  };
 
 public:
   enum Kind { Cleanup, Catch, Terminate, Filter };
 
-  EHScope(Kind K) : CachedLandingPad(0), K(K) {}
+  EHScope(Kind kind, EHScopeStack::stable_iterator enclosingEHScope)
+    : CachedLandingPad(0), CachedEHDispatchBlock(0),
+      EnclosingEHScope(enclosingEHScope) {
+    CommonBits.Kind = kind;
+  }
 
-  Kind getKind() const { return static_cast<Kind>(K); }
+  Kind getKind() const { return static_cast<Kind>(CommonBits.Kind); }
 
   llvm::BasicBlock *getCachedLandingPad() const {
     return CachedLandingPad;
   }
 
-  void setCachedLandingPad(llvm::BasicBlock *Block) {
-    CachedLandingPad = Block;
+  void setCachedLandingPad(llvm::BasicBlock *block) {
+    CachedLandingPad = block;
+  }
+
+  llvm::BasicBlock *getCachedEHDispatchBlock() const {
+    return CachedEHDispatchBlock;
+  }
+
+  void setCachedEHDispatchBlock(llvm::BasicBlock *block) {
+    CachedEHDispatchBlock = block;
+  }
+
+  bool hasEHBranches() const {
+    if (llvm::BasicBlock *block = getCachedEHDispatchBlock())
+      return !block->use_empty();
+    return false;
+  }
+
+  EHScopeStack::stable_iterator getEnclosingEHScope() const {
+    return EnclosingEHScope;
   }
 };
 
@@ -57,8 +134,6 @@ class EHScope {
 /// Objective C @finally blocks are represented using a cleanup scope
 /// after the catch scope.
 class EHCatchScope : public EHScope {
-  unsigned NumHandlers : BitsRemaining;
-
   // In effect, we have a flexible array member
   //   Handler Handlers[0];
   // But that's only standard in C99, not C++, so we have to do
@@ -73,8 +148,7 @@ class EHCatchScope : public EHScope {
     /// The catch handler for this type.
     llvm::BasicBlock *Block;
 
-    /// The unwind destination index for this handler.
-    unsigned Index;
+    bool isCatchAll() const { return Type == 0; }
   };
 
 private:
@@ -93,12 +167,14 @@ class EHCatchScope : public EHScope {
     return sizeof(EHCatchScope) + N * sizeof(Handler);
   }
 
-  EHCatchScope(unsigned NumHandlers)
-    : EHScope(Catch), NumHandlers(NumHandlers) {
+  EHCatchScope(unsigned numHandlers,
+               EHScopeStack::stable_iterator enclosingEHScope)
+    : EHScope(Catch, enclosingEHScope) {
+    CatchBits.NumHandlers = numHandlers;
   }
 
   unsigned getNumHandlers() const {
-    return NumHandlers;
+    return CatchBits.NumHandlers;
   }
 
   void setCatchAllHandler(unsigned I, llvm::BasicBlock *Block) {
@@ -127,44 +203,16 @@ class EHCatchScope : public EHScope {
 
 /// A cleanup scope which generates the cleanup blocks lazily.
 class EHCleanupScope : public EHScope {
-  /// Whether this cleanup needs to be run along normal edges.
-  bool IsNormalCleanup : 1;
-
-  /// Whether this cleanup needs to be run along exception edges.
-  bool IsEHCleanup : 1;
-
-  /// Whether this cleanup is currently active.
-  bool IsActive : 1;
-
-  /// Whether the normal cleanup should test the activation flag.
-  bool TestFlagInNormalCleanup : 1;
-
-  /// Whether the EH cleanup should test the activation flag.
-  bool TestFlagInEHCleanup : 1;
-
-  /// The amount of extra storage needed by the Cleanup.
-  /// Always a multiple of the scope-stack alignment.
-  unsigned CleanupSize : 12;
-
-  /// The number of fixups required by enclosing scopes (not including
-  /// this one).  If this is the top cleanup scope, all the fixups
-  /// from this index onwards belong to this scope.
-  unsigned FixupDepth : BitsRemaining - 17; // currently 13
-
   /// The nearest normal cleanup scope enclosing this one.
   EHScopeStack::stable_iterator EnclosingNormal;
 
-  /// The nearest EH cleanup scope enclosing this one.
+  /// The nearest EH scope enclosing this one.
   EHScopeStack::stable_iterator EnclosingEH;
 
   /// The dual entry/exit block along the normal edge.  This is lazily
   /// created if needed before the cleanup is popped.
   llvm::BasicBlock *NormalBlock;
 
-  /// The dual entry/exit block along the EH edge.  This is lazily
-  /// created if needed before the cleanup is popped.
-  llvm::BasicBlock *EHBlock;
-
   /// An optional i1 variable indicating whether this cleanup has been
   /// activated yet.
   llvm::AllocaInst *ActiveFlag;
@@ -178,17 +226,8 @@ class EHCleanupScope : public EHScope {
     llvm::SmallPtrSet<llvm::BasicBlock*, 4> Branches;
 
     /// Normal branch-afters.
-    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
+    SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
       BranchAfters;
-
-    /// The destinations of EH branch-afters and branch-throughs.
-    /// TODO: optimize for the extremely common case of a single
-    /// branch-through.
-    llvm::SmallPtrSet<llvm::BasicBlock*, 4> EHBranches;
-
-    /// EH branch-afters.
-    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
-    EHBranchAfters;
   };
   mutable struct ExtInfo *ExtInfo;
 
@@ -210,56 +249,64 @@ class EHCleanupScope : public EHScope {
   }
 
   size_t getAllocatedSize() const {
-    return sizeof(EHCleanupScope) + CleanupSize;
+    return sizeof(EHCleanupScope) + CleanupBits.CleanupSize;
   }
 
-  EHCleanupScope(bool IsNormal, bool IsEH, bool IsActive,
-                 unsigned CleanupSize, unsigned FixupDepth,
-                 EHScopeStack::stable_iterator EnclosingNormal,
-                 EHScopeStack::stable_iterator EnclosingEH)
-    : EHScope(EHScope::Cleanup),
-      IsNormalCleanup(IsNormal), IsEHCleanup(IsEH), IsActive(IsActive),
-      TestFlagInNormalCleanup(false), TestFlagInEHCleanup(false),
-      CleanupSize(CleanupSize), FixupDepth(FixupDepth),
-      EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
-      NormalBlock(0), EHBlock(0), ActiveFlag(0), ExtInfo(0)
-  {
-    assert(this->CleanupSize == CleanupSize && "cleanup size overflow");
+  EHCleanupScope(bool isNormal, bool isEH, bool isActive,
+                 unsigned cleanupSize, unsigned fixupDepth,
+                 EHScopeStack::stable_iterator enclosingNormal,
+                 EHScopeStack::stable_iterator enclosingEH)
+    : EHScope(EHScope::Cleanup, enclosingEH), EnclosingNormal(enclosingNormal),
+      NormalBlock(0), ActiveFlag(0), ExtInfo(0) {
+    CleanupBits.IsNormalCleanup = isNormal;
+    CleanupBits.IsEHCleanup = isEH;
+    CleanupBits.IsActive = isActive;
+    CleanupBits.TestFlagInNormalCleanup = false;
+    CleanupBits.TestFlagInEHCleanup = false;
+    CleanupBits.CleanupSize = cleanupSize;
+    CleanupBits.FixupDepth = fixupDepth;
+
+    assert(CleanupBits.CleanupSize == cleanupSize && "cleanup size overflow");
   }
 
   ~EHCleanupScope() {
     delete ExtInfo;
   }
 
-  bool isNormalCleanup() const { return IsNormalCleanup; }
+  bool isNormalCleanup() const { return CleanupBits.IsNormalCleanup; }
   llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
   void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; }
 
-  bool isEHCleanup() const { return IsEHCleanup; }
-  llvm::BasicBlock *getEHBlock() const { return EHBlock; }
-  void setEHBlock(llvm::BasicBlock *BB) { EHBlock = BB; }
+  bool isEHCleanup() const { return CleanupBits.IsEHCleanup; }
+  llvm::BasicBlock *getEHBlock() const { return getCachedEHDispatchBlock(); }
+  void setEHBlock(llvm::BasicBlock *BB) { setCachedEHDispatchBlock(BB); }
 
-  bool isActive() const { return IsActive; }
-  void setActive(bool A) { IsActive = A; }
+  bool isActive() const { return CleanupBits.IsActive; }
+  void setActive(bool A) { CleanupBits.IsActive = A; }
 
   llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; }
   void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; }
 
-  void setTestFlagInNormalCleanup() { TestFlagInNormalCleanup = true; }
-  bool shouldTestFlagInNormalCleanup() const { return TestFlagInNormalCleanup; }
+  void setTestFlagInNormalCleanup() {
+    CleanupBits.TestFlagInNormalCleanup = true;
+  }
+  bool shouldTestFlagInNormalCleanup() const {
+    return CleanupBits.TestFlagInNormalCleanup;
+  }
 
-  void setTestFlagInEHCleanup() { TestFlagInEHCleanup = true; }
-  bool shouldTestFlagInEHCleanup() const { return TestFlagInEHCleanup; }
+  void setTestFlagInEHCleanup() {
+    CleanupBits.TestFlagInEHCleanup = true;
+  }
+  bool shouldTestFlagInEHCleanup() const {
+    return CleanupBits.TestFlagInEHCleanup;
+  }
 
-  unsigned getFixupDepth() const { return FixupDepth; }
+  unsigned getFixupDepth() const { return CleanupBits.FixupDepth; }
   EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
     return EnclosingNormal;
   }
-  EHScopeStack::stable_iterator getEnclosingEHCleanup() const {
-    return EnclosingEH;
-  }
 
-  size_t getCleanupSize() const { return CleanupSize; }
+  size_t getCleanupSize() const { return CleanupBits.CleanupSize; }
   void *getCleanupBuffer() { return this + 1; }
 
   EHScopeStack::Cleanup *getCleanup() {
@@ -327,41 +374,6 @@ class EHCleanupScope : public EHScope {
     return (ExtInfo->BranchAfters.size() != ExtInfo->Branches.size());
   }
 
-  // Same stuff, only for EH branches instead of normal branches.
-  // It's quite possible that we could find a better representation
-  // for this.
-
-  bool hasEHBranches() const { return ExtInfo && !ExtInfo->EHBranches.empty(); }
-  void addEHBranchAfter(llvm::ConstantInt *Index,
-                        llvm::BasicBlock *Block) {
-    struct ExtInfo &ExtInfo = getExtInfo();
-    if (ExtInfo.EHBranches.insert(Block))
-      ExtInfo.EHBranchAfters.push_back(std::make_pair(Block, Index));
-  }
-
-  unsigned getNumEHBranchAfters() const {
-    return ExtInfo ? ExtInfo->EHBranchAfters.size() : 0;
-  }
-
-  llvm::BasicBlock *getEHBranchAfterBlock(unsigned I) const {
-    assert(I < getNumEHBranchAfters());
-    return ExtInfo->EHBranchAfters[I].first;
-  }
-
-  llvm::ConstantInt *getEHBranchAfterIndex(unsigned I) const {
-    assert(I < getNumEHBranchAfters());
-    return ExtInfo->EHBranchAfters[I].second;
-  }
-
-  bool addEHBranchThrough(llvm::BasicBlock *Block) {
-    return getExtInfo().EHBranches.insert(Block);
-  }
-
-  bool hasEHBranchThroughs() const {
-    if (!ExtInfo) return false;
-    return (ExtInfo->EHBranchAfters.size() != ExtInfo->EHBranches.size());
-  }
-
   static bool classof(const EHScope *Scope) {
     return (Scope->getKind() == Cleanup);
   }
@@ -373,8 +385,6 @@ class EHCleanupScope : public EHScope {
 ///
 /// This is used to implement C++ exception specifications.
 class EHFilterScope : public EHScope {
-  unsigned NumFilters : BitsRemaining;
-
   // Essentially ends in a flexible array member:
   // llvm::Value *FilterTypes[0];
 
@@ -387,42 +397,42 @@ class EHFilterScope : public EHScope {
   }
 
 public:
-  EHFilterScope(unsigned NumFilters) :
-    EHScope(Filter), NumFilters(NumFilters) {}
-
-  static size_t getSizeForNumFilters(unsigned NumFilters) {
-    return sizeof(EHFilterScope) + NumFilters * sizeof(llvm::Value*);
+  EHFilterScope(unsigned numFilters)
+    : EHScope(Filter, EHScopeStack::stable_end()) {
+    FilterBits.NumFilters = numFilters;
   }
 
-  unsigned getNumFilters() const { return NumFilters; }
-
-  void setFilter(unsigned I, llvm::Value *FilterValue) {
-    assert(I < getNumFilters());
-    getFilters()[I] = FilterValue;
+  static size_t getSizeForNumFilters(unsigned numFilters) {
+    return sizeof(EHFilterScope) + numFilters * sizeof(llvm::Value*);
   }
 
-  llvm::Value *getFilter(unsigned I) const {
-    assert(I < getNumFilters());
-    return getFilters()[I];
+  unsigned getNumFilters() const { return FilterBits.NumFilters; }
+
+  void setFilter(unsigned i, llvm::Value *filterValue) {
+    assert(i < getNumFilters());
+    getFilters()[i] = filterValue;
   }
 
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Filter;
+  llvm::Value *getFilter(unsigned i) const {
+    assert(i < getNumFilters());
+    return getFilters()[i];
+  }
+
+  static bool classof(const EHScope *scope) {
+    return scope->getKind() == Filter;
   }
 };
 
 /// An exceptions scope which calls std::terminate if any exception
 /// reaches it.
 class EHTerminateScope : public EHScope {
-  unsigned DestIndex : BitsRemaining;
 public:
-  EHTerminateScope(unsigned Index) : EHScope(Terminate), DestIndex(Index) {}
+  EHTerminateScope(EHScopeStack::stable_iterator enclosingEHScope)
+    : EHScope(Terminate, enclosingEHScope) {}
   static size_t getSize() { return sizeof(EHTerminateScope); }
 
-  unsigned getDestIndex() const { return DestIndex; }
-
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Terminate;
+  static bool classof(const EHScope *scope) {
+    return scope->getKind() == Terminate;
   }
 };
 
@@ -498,26 +508,17 @@ inline EHScopeStack::iterator EHScopeStack::end() const {
 inline void EHScopeStack::popCatch() {
   assert(!empty() && "popping exception stack when not empty");
 
-  assert(isa<EHCatchScope>(*begin()));
-  StartOfData += EHCatchScope::getSizeForNumHandlers(
-                          cast<EHCatchScope>(*begin()).getNumHandlers());
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
-  CatchDepth--;
+  EHCatchScope &scope = cast<EHCatchScope>(*begin());
+  InnermostEHScope = scope.getEnclosingEHScope();
+  StartOfData += EHCatchScope::getSizeForNumHandlers(scope.getNumHandlers());
 }
 
 inline void EHScopeStack::popTerminate() {
   assert(!empty() && "popping exception stack when not empty");
 
-  assert(isa<EHTerminateScope>(*begin()));
+  EHTerminateScope &scope = cast<EHTerminateScope>(*begin());
+  InnermostEHScope = scope.getEnclosingEHScope();
   StartOfData += EHTerminateScope::getSize();
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
-  CatchDepth--;
 }
 
 inline EHScopeStack::iterator EHScopeStack::find(stable_iterator sp) const {
@@ -532,28 +533,6 @@ EHScopeStack::stabilize(iterator ir) const {
   return stable_iterator(EndOfBuffer - ir.Ptr);
 }
 
-inline EHScopeStack::stable_iterator
-EHScopeStack::getInnermostActiveNormalCleanup() const {
-  for (EHScopeStack::stable_iterator
-         I = getInnermostNormalCleanup(), E = stable_end(); I != E; ) {
-    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
-    if (S.isActive()) return I;
-    I = S.getEnclosingNormalCleanup();
-  }
-  return stable_end();
-}
-
-inline EHScopeStack::stable_iterator
-EHScopeStack::getInnermostActiveEHCleanup() const {
-  for (EHScopeStack::stable_iterator
-         I = getInnermostEHCleanup(), E = stable_end(); I != E; ) {
-    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
-    if (S.isActive()) return I;
-    I = S.getEnclosingEHCleanup();
-  }
-  return stable_end();
-}
-
 }
 }
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp
index 4c1244591743..c7a9b407d264 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -33,7 +33,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Dwarf.h"
-#include "llvm/Support/Path.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetMachine.h"
 using namespace clang;
@@ -46,12 +46,46 @@ CGDebugInfo::CGDebugInfo(CodeGenModule &CGM)
 }
 
 CGDebugInfo::~CGDebugInfo() {
-  assert(RegionStack.empty() && "Region stack mismatch, stack not empty!");
+  assert(LexicalBlockStack.empty() &&
+         "Region stack mismatch, stack not empty!");
 }
 
 void CGDebugInfo::setLocation(SourceLocation Loc) {
-  if (Loc.isValid())
-    CurLoc = CGM.getContext().getSourceManager().getInstantiationLoc(Loc);
+  // If the new location isn't valid return.
+  if (!Loc.isValid()) return;
+
+  CurLoc = CGM.getContext().getSourceManager().getExpansionLoc(Loc);
+
+  // If we've changed files in the middle of a lexical scope go ahead
+  // and create a new lexical scope with file node if it's different
+  // from the one in the scope.
+  if (LexicalBlockStack.empty()) return;
+
+  SourceManager &SM = CGM.getContext().getSourceManager();
+  PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc);
+  PresumedLoc PPLoc = SM.getPresumedLoc(PrevLoc);
+
+  if (PCLoc.isInvalid() || PPLoc.isInvalid() ||
+      !strcmp(PPLoc.getFilename(), PCLoc.getFilename()))
+    return;
+
+  llvm::MDNode *LB = LexicalBlockStack.back();
+  llvm::DIScope Scope = llvm::DIScope(LB);
+  if (Scope.isLexicalBlockFile()) {
+    llvm::DILexicalBlockFile LBF = llvm::DILexicalBlockFile(LB);
+    llvm::DIDescriptor D
+      = DBuilder.createLexicalBlockFile(LBF.getScope(),
+					getOrCreateFile(CurLoc));
+    llvm::MDNode *N = D;
+    LexicalBlockStack.pop_back();
+    LexicalBlockStack.push_back(N);
+  } else if (Scope.isLexicalBlock()) {
+    llvm::DIDescriptor D
+      = DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc));
+    llvm::MDNode *N = D;
+    LexicalBlockStack.pop_back();
+    LexicalBlockStack.push_back(N);
+  }
 }
 
 /// getContextDescriptor - Get context info for the decl.
@@ -81,7 +115,7 @@ llvm::DIDescriptor CGDebugInfo::getContextDescriptor(const Decl *Context) {
 /// getFunctionName - Get function name for the given FunctionDecl. If the
 /// name is constructred on demand (e.g. C++ destructor) then the name
 /// is stored on the side.
-llvm::StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) {
+StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) {
   assert (FD && "Invalid FunctionDecl!");
   IdentifierInfo *FII = FD->getIdentifier();
   if (FII)
@@ -93,10 +127,10 @@ llvm::StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) {
   // Copy this name on the side and use its reference.
   char *StrPtr = DebugInfoNames.Allocate<char>(NS.length());
   memcpy(StrPtr, NS.data(), NS.length());
-  return llvm::StringRef(StrPtr, NS.length());
+  return StringRef(StrPtr, NS.length());
 }
 
-llvm::StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) {
+StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) {
   llvm::SmallString<256> MethodName;
   llvm::raw_svector_ostream OS(MethodName);
   OS << (OMD->isInstanceMethod() ? '-' : '+') << '[';
@@ -116,22 +150,20 @@ llvm::StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) {
 
   char *StrPtr = DebugInfoNames.Allocate<char>(OS.tell());
   memcpy(StrPtr, MethodName.begin(), OS.tell());
-  return llvm::StringRef(StrPtr, OS.tell());
+  return StringRef(StrPtr, OS.tell());
 }
 
 /// getSelectorName - Return selector name. This is used for debugging
 /// info.
-llvm::StringRef CGDebugInfo::getSelectorName(Selector S) {
-  llvm::SmallString<256> SName;
-  llvm::raw_svector_ostream OS(SName);
-  OS << S.getAsString();
-  char *StrPtr = DebugInfoNames.Allocate<char>(OS.tell());
-  memcpy(StrPtr, SName.begin(), OS.tell());
-  return llvm::StringRef(StrPtr, OS.tell());
+StringRef CGDebugInfo::getSelectorName(Selector S) {
+  const std::string &SName = S.getAsString();
+  char *StrPtr = DebugInfoNames.Allocate<char>(SName.size());
+  memcpy(StrPtr, SName.data(), SName.size());
+  return StringRef(StrPtr, SName.size());
 }
 
 /// getClassName - Get class name including template argument list.
-llvm::StringRef 
+StringRef 
 CGDebugInfo::getClassName(RecordDecl *RD) {
   ClassTemplateSpecializationDecl *Spec
     = dyn_cast<ClassTemplateSpecializationDecl>(RD);
@@ -160,7 +192,7 @@ CGDebugInfo::getClassName(RecordDecl *RD) {
   // Copy this name on the side and use its reference.
   char *StrPtr = DebugInfoNames.Allocate<char>(Buffer.length());
   memcpy(StrPtr, Buffer.data(), Buffer.length());
-  return llvm::StringRef(StrPtr, Buffer.length());
+  return StringRef(StrPtr, Buffer.length());
 }
 
 /// getOrCreateFile - Get the file debug info descriptor for the input location.
@@ -172,7 +204,7 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
 
-  if (PLoc.isInvalid() || llvm::StringRef(PLoc.getFilename()).empty())
+  if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty())
     // If the location is not valid then use main input file.
     return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory());
 
@@ -202,7 +234,7 @@ llvm::DIFile CGDebugInfo::getOrCreateMainFile() {
 /// getLineNumber - Get line number for the location. If location is invalid
 /// then use current location.
 unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) {
-  assert (CurLoc.isValid() && "Invalid current location!");
+  assert((Loc.isValid() || CurLoc.isValid()) && "Invalid current location!");
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc.isValid() ? Loc : CurLoc);
   return PLoc.isValid()? PLoc.getLine() : 0;
@@ -211,20 +243,20 @@ unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) {
 /// getColumnNumber - Get column number for the location. If location is 
 /// invalid then use current location.
 unsigned CGDebugInfo::getColumnNumber(SourceLocation Loc) {
-  assert (CurLoc.isValid() && "Invalid current location!");
+  assert((Loc.isValid() || CurLoc.isValid()) && "Invalid current location!");
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc.isValid() ? Loc : CurLoc);
   return PLoc.isValid()? PLoc.getColumn() : 0;
 }
 
-llvm::StringRef CGDebugInfo::getCurrentDirname() {
+StringRef CGDebugInfo::getCurrentDirname() {
   if (!CWDName.empty())
     return CWDName;
-  char *CompDirnamePtr = NULL;
-  llvm::sys::Path CWD = llvm::sys::Path::GetCurrentDirectory();
-  CompDirnamePtr = DebugInfoNames.Allocate<char>(CWD.size());
-  memcpy(CompDirnamePtr, CWD.c_str(), CWD.size());
-  return CWDName = llvm::StringRef(CompDirnamePtr, CWD.size());
+  llvm::SmallString<256> CWD;
+  llvm::sys::fs::current_path(CWD);
+  char *CompDirnamePtr = DebugInfoNames.Allocate<char>(CWD.size());
+  memcpy(CompDirnamePtr, CWD.data(), CWD.size());
+  return CWDName = StringRef(CompDirnamePtr, CWD.size());
 }
 
 /// CreateCompileUnit - Create new compile unit.
@@ -250,7 +282,7 @@ void CGDebugInfo::CreateCompileUnit() {
   // Save filename string.
   char *FilenamePtr = DebugInfoNames.Allocate<char>(MainFileName.length());
   memcpy(FilenamePtr, MainFileName.c_str(), MainFileName.length());
-  llvm::StringRef Filename(FilenamePtr, MainFileName.length());
+  StringRef Filename(FilenamePtr, MainFileName.length());
   
   unsigned LangTag;
   const LangOptions &LO = CGM.getLangOptions();
@@ -289,7 +321,17 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
   unsigned Encoding = 0;
   const char *BTName = NULL;
   switch (BT->getKind()) {
-  default:
+  case BuiltinType::Dependent:
+    llvm_unreachable("Unexpected builtin type Dependent");
+  case BuiltinType::Overload:
+    llvm_unreachable("Unexpected builtin type Overload");
+  case BuiltinType::BoundMember:
+    llvm_unreachable("Unexpected builtin type BoundMember");
+  case BuiltinType::UnknownAny:
+    llvm_unreachable("Unexpected builtin type UnknownAny");
+  case BuiltinType::NullPtr:
+    return DBuilder.
+      createNullPtrType(BT->getName(CGM.getContext().getLangOptions()));
   case BuiltinType::Void:
     return llvm::DIType();
   case BuiltinType::ObjCClass:
@@ -312,7 +354,7 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
     
     llvm::DIType ISATy = DBuilder.createPointerType(OCTy, Size);
 
-    llvm::SmallVector<llvm::Value *, 16> EltTys;
+    SmallVector<llvm::Value *, 16> EltTys;
     llvm::DIType FieldTy = 
       DBuilder.createMemberType(getOrCreateMainFile(), "isa",
                                 getOrCreateMainFile(), 0, Size,
@@ -334,17 +376,22 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
   case BuiltinType::Char_U: Encoding = llvm::dwarf::DW_ATE_unsigned_char; break;
   case BuiltinType::Char_S:
   case BuiltinType::SChar: Encoding = llvm::dwarf::DW_ATE_signed_char; break;
+  case BuiltinType::Char16:
+  case BuiltinType::Char32: Encoding = llvm::dwarf::DW_ATE_UTF; break;
   case BuiltinType::UShort:
   case BuiltinType::UInt:
   case BuiltinType::UInt128:
   case BuiltinType::ULong:
+  case BuiltinType::WChar_U:
   case BuiltinType::ULongLong: Encoding = llvm::dwarf::DW_ATE_unsigned; break;
   case BuiltinType::Short:
   case BuiltinType::Int:
   case BuiltinType::Int128:
   case BuiltinType::Long:
+  case BuiltinType::WChar_S:
   case BuiltinType::LongLong:  Encoding = llvm::dwarf::DW_ATE_signed; break;
   case BuiltinType::Bool:      Encoding = llvm::dwarf::DW_ATE_boolean; break;
+  case BuiltinType::Half:
   case BuiltinType::Float:
   case BuiltinType::LongDouble:
   case BuiltinType::Double:    Encoding = llvm::dwarf::DW_ATE_float; break;
@@ -432,7 +479,7 @@ llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty,
                                Ty->getPointeeType(), Unit);
 }
 
-/// CreatePointeeType - Create PointTee type. If Pointee is a record
+/// CreatePointeeType - Create Pointee type. If Pointee is a record
 /// then emit record's fwd if debug info size reduction is enabled.
 llvm::DIType CGDebugInfo::CreatePointeeType(QualType PointeeTy,
                                             llvm::DIFile Unit) {
@@ -477,7 +524,7 @@ llvm::DIType CGDebugInfo::CreatePointerLikeType(unsigned Tag,
   // Size is always the size of a pointer. We can't use getTypeSize here
   // because that does not return the correct value for references.
   unsigned AS = CGM.getContext().getTargetAddressSpace(PointeeTy);
-  uint64_t Size = CGM.getContext().Target.getPointerWidth(AS);
+  uint64_t Size = CGM.getContext().getTargetInfo().getPointerWidth(AS);
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
   return 
@@ -489,7 +536,7 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
   if (BlockLiteralGenericSet)
     return BlockLiteralGeneric;
 
-  llvm::SmallVector<llvm::Value *, 8> EltTys;
+  SmallVector<llvm::Value *, 8> EltTys;
   llvm::DIType FieldTy;
   QualType FType;
   uint64_t FieldSize, FieldOffset;
@@ -567,7 +614,7 @@ llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty,
 
 llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
                                      llvm::DIFile Unit) {
-  llvm::SmallVector<llvm::Value *, 16> EltTys;
+  SmallVector<llvm::Value *, 16> EltTys;
 
   // Add the result type at least.
   EltTys.push_back(getOrCreateType(Ty->getResultType(), Unit));
@@ -587,9 +634,9 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
   return DbgTy;
 }
 
-llvm::DIType CGDebugInfo::createFieldType(llvm::StringRef name,
+llvm::DIType CGDebugInfo::createFieldType(StringRef name,
                                           QualType type,
-                                          Expr *bitWidth,
+                                          uint64_t sizeInBitsOverride,
                                           SourceLocation loc,
                                           AccessSpecifier AS,
                                           uint64_t offsetInBits,
@@ -606,8 +653,8 @@ llvm::DIType CGDebugInfo::createFieldType(llvm::StringRef name,
   if (!type->isIncompleteArrayType()) {
     llvm::tie(sizeInBits, alignInBits) = CGM.getContext().getTypeInfo(type);
 
-    if (bitWidth)
-      sizeInBits = bitWidth->EvaluateAsInt(CGM.getContext()).getZExtValue();
+    if (sizeInBitsOverride)
+      sizeInBits = sizeInBitsOverride;
   }
 
   unsigned flags = 0;
@@ -624,7 +671,7 @@ llvm::DIType CGDebugInfo::createFieldType(llvm::StringRef name,
 /// record fields. This is used while creating debug info entry for a Record.
 void CGDebugInfo::
 CollectRecordFields(const RecordDecl *record, llvm::DIFile tunit,
-                    llvm::SmallVectorImpl<llvm::Value *> &elements,
+                    SmallVectorImpl<llvm::Value *> &elements,
                     llvm::DIType RecordTy) {
   unsigned fieldNo = 0;
   const FieldDecl *LastFD = 0;
@@ -644,7 +691,7 @@ CollectRecordFields(const RecordDecl *record, llvm::DIFile tunit,
       LastFD = field;
     }
 
-    llvm::StringRef name = field->getName();
+    StringRef name = field->getName();
     QualType type = field->getType();
 
     // Ignore unnamed fields unless they're anonymous structs/unions.
@@ -653,8 +700,14 @@ CollectRecordFields(const RecordDecl *record, llvm::DIFile tunit,
       continue;
     }
 
+    uint64_t SizeInBitsOverride = 0;
+    if (field->isBitField()) {
+      SizeInBitsOverride = field->getBitWidthValue(CGM.getContext());
+      assert(SizeInBitsOverride && "found named 0-width bitfield");
+    }
+
     llvm::DIType fieldType
-      = createFieldType(name, type, field->getBitWidth(),
+      = createFieldType(name, type, SizeInBitsOverride,
                         field->getLocation(), field->getAccess(),
                         layout.getFieldOffset(fieldNo), tunit, RecordTy);
 
@@ -674,25 +727,23 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
                       Unit);
   
   // Add "this" pointer.
-
   llvm::DIArray Args = llvm::DICompositeType(FnTy).getTypeArray();
   assert (Args.getNumElements() && "Invalid number of arguments!");
 
-  llvm::SmallVector<llvm::Value *, 16> Elts;
+  SmallVector<llvm::Value *, 16> Elts;
 
   // First element is always return type. For 'void' functions it is NULL.
   Elts.push_back(Args.getElement(0));
 
-  if (!Method->isStatic())
-  {
-        // "this" pointer is always first argument.
-        QualType ThisPtr = Method->getThisType(CGM.getContext());
-        llvm::DIType ThisPtrType =
-          DBuilder.createArtificialType(getOrCreateType(ThisPtr, Unit));
-
-        TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;
-        Elts.push_back(ThisPtrType);
-    }
+  if (!Method->isStatic()) {
+    // "this" pointer is always first argument.
+    QualType ThisPtr = Method->getThisType(CGM.getContext());
+    llvm::DIType ThisPtrType =
+      DBuilder.createArtificialType(getOrCreateType(ThisPtr, Unit));
+    
+    TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;
+    Elts.push_back(ThisPtrType);
+  }
 
   // Copy rest of the arguments.
   for (unsigned i = 1, e = Args.getNumElements(); i != e; ++i)
@@ -723,12 +774,12 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
   bool IsCtorOrDtor = 
     isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method);
   
-  llvm::StringRef MethodName = getFunctionName(Method);
+  StringRef MethodName = getFunctionName(Method);
   llvm::DIType MethodTy = getOrCreateMethodType(Method, Unit);
   
   // Since a single ctor/dtor corresponds to multiple functions, it doesn't
   // make sense to give a single ctor/dtor a linkage name.
-  llvm::StringRef MethodLinkageName;
+  StringRef MethodLinkageName;
   if (!IsCtorOrDtor && !isFunctionLocalClass(Method->getParent()))
     MethodLinkageName = CGM.getMangledName(Method);
 
@@ -750,7 +801,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
     // It doesn't make sense to give a virtual destructor a vtable index,
     // since a single destructor has two entries in the vtable.
     if (!isa<CXXDestructorDecl>(Method))
-      VIndex = CGM.getVTables().getMethodVTableIndex(Method);
+      VIndex = CGM.getVTableContext().getMethodVTableIndex(Method);
     ContainingType = RecordTy;
   }
 
@@ -774,7 +825,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
     Flags |= llvm::DIDescriptor::FlagPrototyped;
     
   llvm::DISubprogram SP =
-    DBuilder.createMethod(RecordTy , MethodName, MethodLinkageName, 
+    DBuilder.createMethod(RecordTy, MethodName, MethodLinkageName, 
                           MethodDefUnit, MethodLine,
                           MethodTy, /*isLocalToUnit=*/false, 
                           /* isDefinition=*/ false,
@@ -791,7 +842,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
 /// a Record.
 void CGDebugInfo::
 CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                          llvm::SmallVectorImpl<llvm::Value *> &EltTys,
+                          SmallVectorImpl<llvm::Value *> &EltTys,
                           llvm::DIType RecordTy) {
   for(CXXRecordDecl::method_iterator I = RD->method_begin(),
         E = RD->method_end(); I != E; ++I) {
@@ -809,11 +860,12 @@ CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
 /// a Record.
 void CGDebugInfo::
 CollectCXXFriends(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                llvm::SmallVectorImpl<llvm::Value *> &EltTys,
+                SmallVectorImpl<llvm::Value *> &EltTys,
                 llvm::DIType RecordTy) {
-
   for (CXXRecordDecl::friend_iterator BI =  RD->friend_begin(),
          BE = RD->friend_end(); BI != BE; ++BI) {
+    if ((*BI)->isUnsupportedFriend())
+      continue;
     if (TypeSourceInfo *TInfo = (*BI)->getFriendType())
       EltTys.push_back(DBuilder.createFriend(RecordTy, 
                                              getOrCreateType(TInfo->getType(), 
@@ -826,7 +878,7 @@ CollectCXXFriends(const CXXRecordDecl *RD, llvm::DIFile Unit,
 /// a Record.
 void CGDebugInfo::
 CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                llvm::SmallVectorImpl<llvm::Value *> &EltTys,
+                SmallVectorImpl<llvm::Value *> &EltTys,
                 llvm::DIType RecordTy) {
 
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
@@ -842,7 +894,8 @@ CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
       // virtual base offset offset is -ve. The code generator emits dwarf
       // expression where it expects +ve number.
       BaseOffset = 
-        0 - CGM.getVTables().getVirtualBaseOffsetOffset(RD, Base).getQuantity();
+        0 - CGM.getVTableContext()
+               .getVirtualBaseOffsetOffset(RD, Base).getQuantity();
       BFlags = llvm::DIDescriptor::FlagVirtual;
     } else
       BaseOffset = RL.getBaseClassOffsetInBits(Base);
@@ -868,7 +921,7 @@ llvm::DIArray CGDebugInfo::
 CollectTemplateParams(const TemplateParameterList *TPList,
                       const TemplateArgumentList &TAList,
                       llvm::DIFile Unit) {
-  llvm::SmallVector<llvm::Value *, 16> TemplateParams;  
+  SmallVector<llvm::Value *, 16> TemplateParams;  
   for (unsigned i = 0, e = TAList.size(); i != e; ++i) {
     const TemplateArgument &TA = TAList[i];
     const NamedDecl *ND = TPList->getParam(i);
@@ -892,9 +945,11 @@ CollectTemplateParams(const TemplateParameterList *TPList,
 /// info for function template parameters.
 llvm::DIArray CGDebugInfo::
 CollectFunctionTemplateParams(const FunctionDecl *FD, llvm::DIFile Unit) {
-  if (FD->getTemplatedKind() == FunctionDecl::TK_FunctionTemplateSpecialization){
+  if (FD->getTemplatedKind() ==
+      FunctionDecl::TK_FunctionTemplateSpecialization) {
     const TemplateParameterList *TList =
-      FD->getTemplateSpecializationInfo()->getTemplate()->getTemplateParameters();
+      FD->getTemplateSpecializationInfo()->getTemplate()
+      ->getTemplateParameters();
     return 
       CollectTemplateParams(TList, *FD->getTemplateSpecializationArgs(), Unit);
   }
@@ -936,14 +991,14 @@ llvm::DIType CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile Unit) {
 }
 
 /// getVTableName - Get vtable name for the given Class.
-llvm::StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) {
+StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) {
   // Otherwise construct gdb compatible name name.
   std::string Name = "_vptr$" + RD->getNameAsString();
 
   // Copy this name on the side and use its reference.
   char *StrPtr = DebugInfoNames.Allocate<char>(Name.length());
   memcpy(StrPtr, Name.data(), Name.length());
-  return llvm::StringRef(StrPtr, Name.length());
+  return StringRef(StrPtr, Name.length());
 }
 
 
@@ -951,7 +1006,7 @@ llvm::StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) {
 /// debug info entry in EltTys vector.
 void CGDebugInfo::
 CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                  llvm::SmallVectorImpl<llvm::Value *> &EltTys) {
+                  SmallVectorImpl<llvm::Value *> &EltTys) {
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
 
   // If there is a primary base then it will hold vtable info.
@@ -1016,11 +1071,11 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
   // it.
   TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = FwdDecl;
   // Push the struct on region stack.
-  RegionStack.push_back(FwdDeclNode);
+  LexicalBlockStack.push_back(FwdDeclNode);
   RegionMap[Ty->getDecl()] = llvm::WeakVH(FwdDecl);
 
   // Convert all the elements.
-  llvm::SmallVector<llvm::Value *, 16> EltTys;
+  SmallVector<llvm::Value *, 16> EltTys;
 
   const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD);
   if (CXXDecl) {
@@ -1040,7 +1095,7 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
           
           // Create the descriptor for static variable.
           llvm::DIFile VUnit = getOrCreateFile(V->getLocation());
-          llvm::StringRef VName = V->getName();
+          StringRef VName = V->getName();
           llvm::DIType VTy = getOrCreateType(V->getType(), VUnit);
           // Do not use DIGlobalVariable for enums.
           if (VTy.getTag() != llvm::dwarf::DW_TAG_enumeration_type) {
@@ -1062,7 +1117,7 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
       TParamsArray = CollectCXXTemplateParams(TSpecial, Unit);
   }
 
-  RegionStack.pop_back();
+  LexicalBlockStack.pop_back();
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator RI = 
     RegionMap.find(Ty->getDecl());
   if (RI != RegionMap.end())
@@ -1070,7 +1125,7 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
 
   llvm::DIDescriptor RDContext =  
     getContextDescriptor(cast<Decl>(RD->getDeclContext()));
-  llvm::StringRef RDName = RD->getName();
+  StringRef RDName = RD->getName();
   uint64_t Size = CGM.getContext().getTypeSize(Ty);
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
   llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys);
@@ -1134,8 +1189,8 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   unsigned Line = getLineNumber(ID->getLocation());
   unsigned RuntimeLang = TheCU.getLanguage();
 
-  // If this is just a forward declaration, return a special forward-declaration
-  // debug type.
+  // If this is just a forward declaration return a special forward-declaration
+  // debug type since we won't be able to lay out the entire type.
   if (ID->isForwardDecl()) {
     llvm::DIType FwdDecl =
       DBuilder.createStructType(Unit, ID->getName(),
@@ -1144,12 +1199,12 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
     return FwdDecl;
   }
 
-  // To handle recursive interface, we
-  // first generate a debug descriptor for the struct as a forward declaration.
-  // Then (if it is a definition) we go through and get debug info for all of
-  // its members.  Finally, we create a descriptor for the complete type (which
-  // may refer to the forward decl if the struct is recursive) and replace all
-  // uses of the forward declaration with the final definition.
+  // To handle a recursive interface, we first generate a debug descriptor
+  // for the struct as a forward declaration. Then (if it is a definition)
+  // we go through and get debug info for all of its members.  Finally, we
+  // create a descriptor for the complete type (which may refer to the
+  // forward decl if the struct is recursive) and replace all uses of the
+  // forward declaration with the final definition.
   llvm::DIType FwdDecl = DBuilder.createTemporaryType(DefUnit);
 
   llvm::MDNode *MN = FwdDecl;
@@ -1158,11 +1213,11 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   // it.
   TypeCache[QualType(Ty, 0).getAsOpaquePtr()] = FwdDecl;
   // Push the struct on region stack.
-  RegionStack.push_back(FwdDeclNode);
+  LexicalBlockStack.push_back(FwdDeclNode);
   RegionMap[Ty->getDecl()] = llvm::WeakVH(FwdDecl);
 
   // Convert all the elements.
-  llvm::SmallVector<llvm::Value *, 16> EltTys;
+  SmallVector<llvm::Value *, 16> EltTys;
 
   ObjCInterfaceDecl *SClass = ID->getSuperClass();
   if (SClass) {
@@ -1177,7 +1232,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   }
 
   const ASTRecordLayout &RL = CGM.getContext().getASTObjCInterfaceLayout(ID);
-
+  ObjCImplementationDecl *ImpD = ID->getImplementation();
   unsigned FieldNo = 0;
   for (ObjCIvarDecl *Field = ID->all_declared_ivar_begin(); Field;
        Field = Field->getNextIvar(), ++FieldNo) {
@@ -1185,7 +1240,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
     if (!FieldTy.isValid())
       return llvm::DIType();
     
-    llvm::StringRef FieldName = Field->getName();
+    StringRef FieldName = Field->getName();
 
     // Ignore unnamed fields.
     if (FieldName.empty())
@@ -1201,15 +1256,18 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
     if (!FType->isIncompleteArrayType()) {
 
       // Bit size, align and offset of the type.
-      FieldSize = CGM.getContext().getTypeSize(FType);
-      Expr *BitWidth = Field->getBitWidth();
-      if (BitWidth)
-        FieldSize = BitWidth->EvaluateAsInt(CGM.getContext()).getZExtValue();
-
-      FieldAlign =  CGM.getContext().getTypeAlign(FType);
+      FieldSize = Field->isBitField()
+        ? Field->getBitWidthValue(CGM.getContext())
+        : CGM.getContext().getTypeSize(FType);
+      FieldAlign = CGM.getContext().getTypeAlign(FType);
     }
 
-    uint64_t FieldOffset = RL.getFieldOffset(FieldNo);
+    // We can't know the offset of our ivar in the structure if we're using
+    // the non-fragile abi and the debugger should ignore the value anyways.
+    // Call it the FieldNo+1 due to how debuggers use the information,
+    // e.g. negating the value when it needs a lookup in the dynamic table.
+    uint64_t FieldOffset = CGM.getLangOptions().ObjCNonFragileABI ? FieldNo+1
+      : RL.getFieldOffset(FieldNo);
 
     unsigned Flags = 0;
     if (Field->getAccessControl() == ObjCIvarDecl::Protected)
@@ -1217,17 +1275,21 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
     else if (Field->getAccessControl() == ObjCIvarDecl::Private)
       Flags = llvm::DIDescriptor::FlagPrivate;
 
-    llvm::StringRef PropertyName;
-    llvm::StringRef PropertyGetter;
-    llvm::StringRef PropertySetter;
+    StringRef PropertyName;
+    StringRef PropertyGetter;
+    StringRef PropertySetter;
     unsigned PropertyAttributes = 0;
-    if (ObjCPropertyDecl *PD =
-        ID->FindPropertyVisibleInPrimaryClass(Field->getIdentifier())) {
+    ObjCPropertyDecl *PD = NULL;
+    if (ImpD)
+      if (ObjCPropertyImplDecl *PImpD = 
+          ImpD->FindPropertyImplIvarDecl(Field->getIdentifier()))
+        PD = PImpD->getPropertyDecl();
+    if (PD) {
       PropertyName = PD->getName();
       PropertyGetter = getSelectorName(PD->getGetterName());
       PropertySetter = getSelectorName(PD->getSetterName());
       PropertyAttributes = PD->getPropertyAttributes();
-    }
+    } 
     FieldTy = DBuilder.createObjCIVar(FieldName, FieldDefUnit,
                                       FieldLine, FieldSize, FieldAlign,
                                       FieldOffset, Flags, FieldTy,
@@ -1238,7 +1300,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
 
   llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys);
 
-  RegionStack.pop_back();
+  LexicalBlockStack.pop_back();
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator RI = 
     RegionMap.find(Ty->getDecl());
   if (RI != RegionMap.end())
@@ -1322,7 +1384,7 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty,
   // Add the dimensions of the array.  FIXME: This loses CV qualifiers from
   // interior arrays, do we care?  Why aren't nested arrays represented the
   // obvious/recursive way?
-  llvm::SmallVector<llvm::Value *, 8> Subscripts;
+  SmallVector<llvm::Value *, 8> Subscripts;
   QualType EltTy(Ty, 0);
   if (Ty->isIncompleteArrayType())
     EltTy = Ty->getElementType();
@@ -1339,7 +1401,8 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty,
         LowerBound = 1;
 
       // FIXME: Verify this is right for VLAs.
-      Subscripts.push_back(DBuilder.getOrCreateSubrange(LowerBound, UpperBound));
+      Subscripts.push_back(DBuilder.getOrCreateSubrange(LowerBound,
+                                                        UpperBound));
       EltTy = Ty->getElementType();
     }
   }
@@ -1395,15 +1458,22 @@ llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty,
   
   llvm::DIArray Elements = DBuilder.getOrCreateArray(ElementTypes);
 
-  return DBuilder.createStructType(U, llvm::StringRef("test"), 
+  return DBuilder.createStructType(U, StringRef("test"), 
                                    U, 0, FieldOffset, 
                                    0, 0, Elements);
 }
 
+llvm::DIType CGDebugInfo::CreateType(const AtomicType *Ty, 
+                                     llvm::DIFile U) {
+  // Ignore the atomic wrapping
+  // FIXME: What is the correct representation?
+  return getOrCreateType(Ty->getValueType(), U);
+}
+
 /// CreateEnumType - get enumeration type.
 llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED) {
   llvm::DIFile Unit = getOrCreateFile(ED->getLocation());
-  llvm::SmallVector<llvm::Value *, 16> Enumerators;
+  SmallVector<llvm::Value *, 16> Enumerators;
 
   // Create DIEnumerator elements for each enumerator.
   for (EnumDecl::enumerator_iterator
@@ -1522,7 +1592,7 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
 #define NON_CANONICAL_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #include "clang/AST/TypeNodes.def"
-    assert(false && "Dependent types cannot show up in debug information");
+    llvm_unreachable("Dependent types cannot show up in debug information");
 
   case Type::ExtVector:
   case Type::Vector:
@@ -1558,6 +1628,9 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
   case Type::MemberPointer:
     return CreateType(cast<MemberPointerType>(Ty), Unit);
 
+  case Type::Atomic:
+    return CreateType(cast<AtomicType>(Ty), Unit);
+
   case Type::Attributed:
   case Type::TemplateSpecialization:
   case Type::Elaborated:
@@ -1573,7 +1646,7 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
   }
   
   assert(Diag && "Fall through without a diagnostic?");
-  unsigned DiagID = CGM.getDiags().getCustomDiagID(Diagnostic::Error,
+  unsigned DiagID = CGM.getDiags().getCustomDiagID(DiagnosticsEngine::Error,
                                "debug information for %0 is not yet supported");
   CGM.getDiags().Report(DiagID)
     << Diag;
@@ -1582,7 +1655,7 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
 
 /// CreateMemberType - Create new member and increase Offset by FType's size.
 llvm::DIType CGDebugInfo::CreateMemberType(llvm::DIFile Unit, QualType FType,
-                                           llvm::StringRef Name,
+                                           StringRef Name,
                                            uint64_t *Offset) {
   llvm::DIType FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
   uint64_t FieldSize = CGM.getContext().getTypeSize(FType);
@@ -1627,13 +1700,14 @@ llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) {
 
 // getOrCreateFunctionType - Construct DIType. If it is a c++ method, include
 // implicit parameter "this".
-llvm::DIType CGDebugInfo::getOrCreateFunctionType(const Decl * D, QualType FnType,
+llvm::DIType CGDebugInfo::getOrCreateFunctionType(const Decl * D,
+                                                  QualType FnType,
                                                   llvm::DIFile F) {
   if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D))
     return getOrCreateMethodType(Method, F);
   else if (const ObjCMethodDecl *OMethod = dyn_cast<ObjCMethodDecl>(D)) {
     // Add "self" and "_cmd"
-    llvm::SmallVector<llvm::Value *, 16> Elts;
+    SmallVector<llvm::Value *, 16> Elts;
 
     // First element is always return type. For 'void' functions it is NULL.
     Elts.push_back(getOrCreateType(OMethod->getResultType(), F));
@@ -1642,7 +1716,7 @@ llvm::DIType CGDebugInfo::getOrCreateFunctionType(const Decl * D, QualType FnTyp
     // "cmd" pointer is always second argument.
     Elts.push_back(getOrCreateType(OMethod->getCmdDecl()->getType(), F));
     // Get rest of the arguments.
-    for (ObjCMethodDecl::param_iterator PI = OMethod->param_begin(), 
+    for (ObjCMethodDecl::param_const_iterator PI = OMethod->param_begin(), 
            PE = OMethod->param_end(); PI != PE; ++PI)
       Elts.push_back(getOrCreateType((*PI)->getType(), F));
 
@@ -1658,13 +1732,13 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
                                     llvm::Function *Fn,
                                     CGBuilderTy &Builder) {
 
-  llvm::StringRef Name;
-  llvm::StringRef LinkageName;
+  StringRef Name;
+  StringRef LinkageName;
 
-  FnBeginRegionCount.push_back(RegionStack.size());
+  FnBeginRegionCount.push_back(LexicalBlockStack.size());
 
   const Decl *D = GD.getDecl();
-
+  
   unsigned Flags = 0;
   llvm::DIFile Unit = getOrCreateFile(CurLoc);
   llvm::DIDescriptor FDContext(Unit);
@@ -1677,7 +1751,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
       llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(&*FI->second));
       if (SP.isSubprogram() && llvm::DISubprogram(SP).isDefinition()) {
         llvm::MDNode *SPN = SP;
-        RegionStack.push_back(SPN);
+        LexicalBlockStack.push_back(SPN);
         RegionMap[D] = llvm::WeakVH(SP);
         return;
       }
@@ -1687,7 +1761,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
     if (!Fn->hasInternalLinkage())
       LinkageName = CGM.getMangledName(GD);
     if (LinkageName == Name)
-      LinkageName = llvm::StringRef();
+      LinkageName = StringRef();
     if (FD->hasPrototype())
       Flags |= llvm::DIDescriptor::FlagPrototyped;
     if (const NamespaceDecl *NSDecl =
@@ -1726,121 +1800,85 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
 
   // Push function on region stack.
   llvm::MDNode *SPN = SP;
-  RegionStack.push_back(SPN);
+  LexicalBlockStack.push_back(SPN);
   RegionMap[D] = llvm::WeakVH(SP);
-
-  // Clear stack used to keep track of #line directives.
-  LineDirectiveFiles.clear();
 }
 
+/// EmitLocation - Emit metadata to indicate a change in line/column
+/// information in the source file.
+void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc) {
+  
+  // Update our current location
+  setLocation(Loc);
 
-void CGDebugInfo::EmitStopPoint(CGBuilderTy &Builder) {
   if (CurLoc.isInvalid() || CurLoc.isMacroID()) return;
 
   // Don't bother if things are the same as last time.
   SourceManager &SM = CGM.getContext().getSourceManager();
-  if (CurLoc == PrevLoc
-       || (SM.getInstantiationLineNumber(CurLoc) ==
-           SM.getInstantiationLineNumber(PrevLoc)
-           && SM.isFromSameFile(CurLoc, PrevLoc)))
+  if (CurLoc == PrevLoc ||
+      SM.getExpansionLoc(CurLoc) == SM.getExpansionLoc(PrevLoc))
     // New Builder may not be in sync with CGDebugInfo.
     if (!Builder.getCurrentDebugLocation().isUnknown())
       return;
-
+  
   // Update last state.
   PrevLoc = CurLoc;
 
-  llvm::MDNode *Scope = RegionStack.back();
+  llvm::MDNode *Scope = LexicalBlockStack.back();
   Builder.SetCurrentDebugLocation(llvm::DebugLoc::get(getLineNumber(CurLoc),
                                                       getColumnNumber(CurLoc),
                                                       Scope));
 }
 
-/// UpdateLineDirectiveRegion - Update region stack only if #line directive
-/// has introduced scope change.
-void CGDebugInfo::UpdateLineDirectiveRegion(CGBuilderTy &Builder) {
-  if (CurLoc.isInvalid() || CurLoc.isMacroID() ||
-      PrevLoc.isInvalid() || PrevLoc.isMacroID())
-    return;
-  SourceManager &SM = CGM.getContext().getSourceManager();
-  PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc);
-  PresumedLoc PPLoc = SM.getPresumedLoc(PrevLoc);
-
-  if (PCLoc.isInvalid() || PPLoc.isInvalid() ||
-      !strcmp(PPLoc.getFilename(), PCLoc.getFilename()))
-    return;
-
-  // If #line directive stack is empty then we are entering a new scope.
-  if (LineDirectiveFiles.empty()) {
-    EmitRegionStart(Builder);
-    LineDirectiveFiles.push_back(PCLoc.getFilename());
-    return;
-  }
-
-  assert (RegionStack.size() >= LineDirectiveFiles.size()
-          && "error handling  #line regions!");
-
-  bool SeenThisFile = false;
-  // Chek if current file is already seen earlier.
-  for(std::vector<const char *>::iterator I = LineDirectiveFiles.begin(),
-        E = LineDirectiveFiles.end(); I != E; ++I)
-    if (!strcmp(PCLoc.getFilename(), *I)) {
-      SeenThisFile = true;
-      break;
-    }
-
-  // If #line for this file is seen earlier then pop out #line regions.
-  if (SeenThisFile) {
-    while (!LineDirectiveFiles.empty()) {
-      const char *LastFile = LineDirectiveFiles.back();
-      RegionStack.pop_back();
-      LineDirectiveFiles.pop_back();
-      if (!strcmp(PPLoc.getFilename(), LastFile))
-        break;
-    }
-    return;
-  } 
-
-  // .. otherwise insert new #line region.
-  EmitRegionStart(Builder);
-  LineDirectiveFiles.push_back(PCLoc.getFilename());
-
-  return;
-}
-/// EmitRegionStart- Constructs the debug code for entering a declarative
-/// region - "llvm.dbg.region.start.".
-void CGDebugInfo::EmitRegionStart(CGBuilderTy &Builder) {
+/// CreateLexicalBlock - Creates a new lexical block node and pushes it on
+/// the stack.
+void CGDebugInfo::CreateLexicalBlock(SourceLocation Loc) {
   llvm::DIDescriptor D =
-    DBuilder.createLexicalBlock(RegionStack.empty() ? 
-                                llvm::DIDescriptor() : 
-                                llvm::DIDescriptor(RegionStack.back()),
-                                getOrCreateFile(CurLoc),
-                                getLineNumber(CurLoc), 
-                                getColumnNumber(CurLoc));
+    DBuilder.createLexicalBlock(LexicalBlockStack.empty() ?
+				llvm::DIDescriptor() :
+				llvm::DIDescriptor(LexicalBlockStack.back()),
+				getOrCreateFile(CurLoc),
+				getLineNumber(CurLoc),
+				getColumnNumber(CurLoc));
   llvm::MDNode *DN = D;
-  RegionStack.push_back(DN);
+  LexicalBlockStack.push_back(DN);
 }
 
-/// EmitRegionEnd - Constructs the debug code for exiting a declarative
-/// region - "llvm.dbg.region.end."
-void CGDebugInfo::EmitRegionEnd(CGBuilderTy &Builder) {
-  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
+/// EmitLexicalBlockStart - Constructs the debug code for entering a declarative
+/// region - beginning of a DW_TAG_lexical_block.
+void CGDebugInfo::EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc) {
+  // Set our current location.
+  setLocation(Loc);
 
-  // Provide an region stop point.
-  EmitStopPoint(Builder);
+  // Create a new lexical block and push it on the stack.
+  CreateLexicalBlock(Loc);
 
-  RegionStack.pop_back();
+  // Emit a line table change for the current location inside the new scope.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc::get(getLineNumber(Loc),
+  					      getColumnNumber(Loc),
+  					      LexicalBlockStack.back()));
+}
+
+/// EmitLexicalBlockEnd - Constructs the debug code for exiting a declarative
+/// region - end of a DW_TAG_lexical_block.
+void CGDebugInfo::EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc) {
+  assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
+
+  // Provide an entry in the line table for the end of the block.
+  EmitLocation(Builder, Loc);
+
+  LexicalBlockStack.pop_back();
 }
 
 /// EmitFunctionEnd - Constructs the debug code for exiting a function.
 void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) {
-  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
+  assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
   unsigned RCount = FnBeginRegionCount.back();
-  assert(RCount <= RegionStack.size() && "Region stack mismatch");
+  assert(RCount <= LexicalBlockStack.size() && "Region stack mismatch");
 
   // Pop all regions for this function.
-  while (RegionStack.size() != RCount)
-    EmitRegionEnd(Builder);
+  while (LexicalBlockStack.size() != RCount)
+    EmitLexicalBlockEnd(Builder, CurLoc);
   FnBeginRegionCount.pop_back();
 }
 
@@ -1849,7 +1887,7 @@ void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) {
 llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
                                                        uint64_t *XOffset) {
 
-  llvm::SmallVector<llvm::Value *, 5> EltTys;
+  SmallVector<llvm::Value *, 5> EltTys;
   QualType FType;
   uint64_t FieldSize, FieldOffset;
   unsigned FieldAlign;
@@ -1876,7 +1914,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
   
   CharUnits Align = CGM.getContext().getDeclAlign(VD);
   if (Align > CGM.getContext().toCharUnitsFromBits(
-        CGM.getContext().Target.getPointerAlign(0))) {
+        CGM.getContext().getTargetInfo().getPointerAlign(0))) {
     CharUnits FieldOffsetInBytes 
       = CGM.getContext().toCharUnitsFromBits(FieldOffset);
     CharUnits AlignedOffsetInBytes
@@ -1916,7 +1954,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
 void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
                               llvm::Value *Storage, 
                               unsigned ArgNo, CGBuilderTy &Builder) {
-  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
+  assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
 
   llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
   llvm::DIType Ty;
@@ -1940,7 +1978,8 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
       // If an aggregate variable has non trivial destructor or non trivial copy
       // constructor than it is pass indirectly. Let debug info know about this
       // by using reference of the aggregate type as a argument type.
-      if (!Record->hasTrivialCopyConstructor() || !Record->hasTrivialDestructor())
+      if (!Record->hasTrivialCopyConstructor() ||
+          !Record->hasTrivialDestructor())
         Ty = DBuilder.createReferenceType(Ty);
     }
   }
@@ -1951,18 +1990,18 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
   unsigned Flags = 0;
   if (VD->isImplicit())
     Flags |= llvm::DIDescriptor::FlagArtificial;
-  llvm::MDNode *Scope = RegionStack.back();
+  llvm::MDNode *Scope = LexicalBlockStack.back();
     
-  llvm::StringRef Name = VD->getName();
+  StringRef Name = VD->getName();
   if (!Name.empty()) {
     if (VD->hasAttr<BlocksAttr>()) {
       CharUnits offset = CharUnits::fromQuantity(32);
-      llvm::SmallVector<llvm::Value *, 9> addr;
-      const llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
+      SmallVector<llvm::Value *, 9> addr;
+      llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
       addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
       // offset of __forwarding field
       offset = CGM.getContext().toCharUnitsFromBits(
-        CGM.getContext().Target.getPointerWidth(0));
+        CGM.getContext().getTargetInfo().getPointerWidth(0));
       addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
       addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
       addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
@@ -1973,14 +2012,14 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
       // Create the descriptor for the variable.
       llvm::DIVariable D =
         DBuilder.createComplexVariable(Tag, 
-                                       llvm::DIDescriptor(RegionStack.back()),
+                                       llvm::DIDescriptor(Scope),
                                        VD->getName(), Unit, Line, Ty,
                                        addr, ArgNo);
       
+      // Insert an llvm.dbg.declare into the current block.
       // Insert an llvm.dbg.declare into the current block.
       llvm::Instruction *Call =
         DBuilder.insertDeclare(Storage, D, Builder.GetInsertBlock());
-      
       Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
       return;
     } 
@@ -1993,7 +2032,6 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
     // Insert an llvm.dbg.declare into the current block.
     llvm::Instruction *Call =
       DBuilder.insertDeclare(Storage, D, Builder.GetInsertBlock());
-    
     Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
     return;
   }
@@ -2008,7 +2046,7 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
            I != E; ++I) {
         FieldDecl *Field = *I;
         llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
-        llvm::StringRef FieldName = Field->getName();
+        StringRef FieldName = Field->getName();
           
         // Ignore unnamed fields. Do not ignore unnamed records.
         if (FieldName.empty() && !isa<RecordType>(Field->getType()))
@@ -2024,7 +2062,6 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
         // Insert an llvm.dbg.declare into the current block.
         llvm::Instruction *Call =
           DBuilder.insertDeclare(Storage, D, Builder.GetInsertBlock());
-          
         Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
       }
     }
@@ -2040,7 +2077,7 @@ void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD,
 void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
   const VarDecl *VD, llvm::Value *Storage, CGBuilderTy &Builder,
   const CGBlockInfo &blockInfo) {
-  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
+  assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
   
   if (Builder.GetInsertBlock() == 0)
     return;
@@ -2065,15 +2102,16 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
     target.getStructLayout(blockInfo.StructureType)
           ->getElementOffset(blockInfo.getCapture(VD).getIndex()));
 
-  llvm::SmallVector<llvm::Value *, 9> addr;
-  const llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
+  SmallVector<llvm::Value *, 9> addr;
+  llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
   addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
   addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
   if (isByRef) {
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
     // offset of __forwarding field
-    offset = CGM.getContext().toCharUnitsFromBits(target.getPointerSizeInBits());
+    offset = CGM.getContext()
+                .toCharUnitsFromBits(target.getPointerSizeInBits());
     addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
@@ -2085,14 +2123,13 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
   // Create the descriptor for the variable.
   llvm::DIVariable D =
     DBuilder.createComplexVariable(llvm::dwarf::DW_TAG_auto_variable, 
-                                   llvm::DIDescriptor(RegionStack.back()),
+                                   llvm::DIDescriptor(LexicalBlockStack.back()),
                                    VD->getName(), Unit, Line, Ty, addr);
   // Insert an llvm.dbg.declare into the current block.
-  llvm::Instruction *Call = 
+  llvm::Instruction *Call =
     DBuilder.insertDeclare(Storage, D, Builder.GetInsertPoint());
-  
-  llvm::MDNode *Scope = RegionStack.back();
-  Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+  Call->setDebugLoc(llvm::DebugLoc::get(Line, Column,
+                                        LexicalBlockStack.back()));
 }
 
 /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
@@ -2131,7 +2168,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
   const llvm::StructLayout *blockLayout =
     CGM.getTargetData().getStructLayout(block.StructureType);
 
-  llvm::SmallVector<llvm::Value*, 16> fields;
+  SmallVector<llvm::Value*, 16> fields;
   fields.push_back(createFieldType("__isa", C.VoidPtrTy, 0, loc, AS_public,
                                    blockLayout->getElementOffsetInBits(0),
                                    tunit, tunit));
@@ -2154,7 +2191,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
 
   // We want to sort the captures by offset, not because DWARF
   // requires this, but because we're paranoid about debuggers.
-  llvm::SmallVector<BlockLayoutChunk, 8> chunks;
+  SmallVector<BlockLayoutChunk, 8> chunks;
 
   // 'this' capture.
   if (blockDecl->capturesCXXThis()) {
@@ -2187,7 +2224,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
   // Sort by offset.
   llvm::array_pod_sort(chunks.begin(), chunks.end());
 
-  for (llvm::SmallVectorImpl<BlockLayoutChunk>::iterator
+  for (SmallVectorImpl<BlockLayoutChunk>::iterator
          i = chunks.begin(), e = chunks.end(); i != e; ++i) {
     uint64_t offsetInBits = i->OffsetInBits;
     const BlockDecl::Capture *capture = i->Capture;
@@ -2204,7 +2241,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
     }
 
     const VarDecl *variable = capture->getVariable();
-    llvm::StringRef name = variable->getName();
+    StringRef name = variable->getName();
 
     llvm::DIType fieldType;
     if (capture->isByRef()) {
@@ -2239,8 +2276,8 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
 
   // Get overall information about the block.
   unsigned flags = llvm::DIDescriptor::FlagArtificial;
-  llvm::MDNode *scope = RegionStack.back();
-  llvm::StringRef name = ".block_descriptor";
+  llvm::MDNode *scope = LexicalBlockStack.back();
+  StringRef name = ".block_descriptor";
 
   // Create the descriptor for the parameter.
   llvm::DIVariable debugVar =
@@ -2265,6 +2302,8 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
   llvm::DIFile Unit = getOrCreateFile(D->getLocation());
   unsigned LineNo = getLineNumber(D->getLocation());
 
+  setLocation(D->getLocation());
+
   QualType T = D->getType();
   if (T->isIncompleteArrayType()) {
 
@@ -2277,13 +2316,13 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
     T = CGM.getContext().getConstantArrayType(ET, ConstVal,
                                            ArrayType::Normal, 0);
   }
-  llvm::StringRef DeclName = D->getName();
-  llvm::StringRef LinkageName;
+  StringRef DeclName = D->getName();
+  StringRef LinkageName;
   if (D->getDeclContext() && !isa<FunctionDecl>(D->getDeclContext())
       && !isa<ObjCMethodDecl>(D->getDeclContext()))
     LinkageName = Var->getName();
   if (LinkageName == DeclName)
-    LinkageName = llvm::StringRef();
+    LinkageName = StringRef();
   llvm::DIDescriptor DContext = 
     getContextDescriptor(dyn_cast<Decl>(D->getDeclContext()));
   DBuilder.createStaticVariable(DContext, DeclName, LinkageName,
@@ -2298,7 +2337,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
   llvm::DIFile Unit = getOrCreateFile(ID->getLocation());
   unsigned LineNo = getLineNumber(ID->getLocation());
 
-  llvm::StringRef Name = ID->getName();
+  StringRef Name = ID->getName();
 
   QualType T = CGM.getContext().getObjCInterfaceType(ID);
   if (T->isIncompleteArrayType()) {
@@ -2323,7 +2362,7 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD,
                                      llvm::Constant *Init) {
   // Create the descriptor for the variable.
   llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
-  llvm::StringRef Name = VD->getName();
+  StringRef Name = VD->getName();
   llvm::DIType Ty = getOrCreateType(VD->getType(), Unit);
   if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(VD)) {
     if (const EnumDecl *ED = dyn_cast<EnumDecl>(ECD->getDeclContext()))
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h
index f87d0072e323..a4533a83d579 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h
@@ -56,21 +56,18 @@ class CGDebugInfo {
   bool BlockLiteralGenericSet;
   llvm::DIType BlockLiteralGeneric;
 
-  std::vector<llvm::TrackingVH<llvm::MDNode> > RegionStack;
+  // LexicalBlockStack - Keep track of our current nested lexical block.
+  std::vector<llvm::TrackingVH<llvm::MDNode> > LexicalBlockStack;
   llvm::DenseMap<const Decl *, llvm::WeakVH> RegionMap;
-  // FnBeginRegionCount - Keep track of RegionStack counter at the beginning
-  // of a function. This is used to pop unbalanced regions at the end of a
-  // function.
+  // FnBeginRegionCount - Keep track of LexicalBlockStack counter at the
+  // beginning of a function. This is used to pop unbalanced regions at
+  // the end of a function.
   std::vector<unsigned> FnBeginRegionCount;
 
-  /// LineDirectiveFiles - This stack is used to keep track of 
-  /// scopes introduced by #line directives.
-  std::vector<const char *> LineDirectiveFiles;
-
   /// DebugInfoNames - This is a storage for names that are
   /// constructed on demand. For example, C++ destructors, C++ operators etc..
   llvm::BumpPtrAllocator DebugInfoNames;
-  llvm::StringRef CWDName;
+  StringRef CWDName;
 
   llvm::DenseMap<const char *, llvm::WeakVH> DIFileCache;
   llvm::DenseMap<const FunctionDecl *, llvm::WeakVH> SPCache;
@@ -95,6 +92,7 @@ class CGDebugInfo {
   llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const RValueReferenceType *Ty, llvm::DIFile Unit);
   llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DIFile F);
+  llvm::DIType CreateType(const AtomicType *Ty, llvm::DIFile F);
   llvm::DIType CreateEnumType(const EnumDecl *ED);
   llvm::DIType getOrCreateMethodType(const CXXMethodDecl *Method,
                                      llvm::DIFile F);
@@ -113,17 +111,17 @@ class CGDebugInfo {
   
   void CollectCXXMemberFunctions(const CXXRecordDecl *Decl,
                                  llvm::DIFile F,
-                                 llvm::SmallVectorImpl<llvm::Value *> &E,
+                                 SmallVectorImpl<llvm::Value *> &E,
                                  llvm::DIType T);
 
   void CollectCXXFriends(const CXXRecordDecl *Decl,
                        llvm::DIFile F,
-                       llvm::SmallVectorImpl<llvm::Value *> &EltTys,
+                       SmallVectorImpl<llvm::Value *> &EltTys,
                        llvm::DIType RecordTy);
 
   void CollectCXXBases(const CXXRecordDecl *Decl,
                        llvm::DIFile F,
-                       llvm::SmallVectorImpl<llvm::Value *> &EltTys,
+                       SmallVectorImpl<llvm::Value *> &EltTys,
                        llvm::DIType RecordTy);
   
   llvm::DIArray
@@ -136,30 +134,35 @@ class CGDebugInfo {
   CollectCXXTemplateParams(const ClassTemplateSpecializationDecl *TS,
                            llvm::DIFile F);
 
-  llvm::DIType createFieldType(llvm::StringRef name, QualType type,
-                               Expr *bitWidth, SourceLocation loc,
+  llvm::DIType createFieldType(StringRef name, QualType type,
+                               uint64_t sizeInBitsOverride, SourceLocation loc,
                                AccessSpecifier AS, uint64_t offsetInBits,
                                llvm::DIFile tunit,
                                llvm::DIDescriptor scope);
   void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile F,
-                           llvm::SmallVectorImpl<llvm::Value *> &E,
+                           SmallVectorImpl<llvm::Value *> &E,
                            llvm::DIType RecordTy);
 
   void CollectVTableInfo(const CXXRecordDecl *Decl,
                          llvm::DIFile F,
-                         llvm::SmallVectorImpl<llvm::Value *> &EltTys);
+                         SmallVectorImpl<llvm::Value *> &EltTys);
 
+  // CreateLexicalBlock - Create a new lexical block node and push it on
+  // the stack.
+  void CreateLexicalBlock(SourceLocation Loc);
+  
 public:
   CGDebugInfo(CodeGenModule &CGM);
   ~CGDebugInfo();
+  void finalize() { DBuilder.finalize(); }
 
   /// setLocation - Update the current source location. If \arg loc is
   /// invalid it is ignored.
   void setLocation(SourceLocation Loc);
 
-  /// EmitStopPoint - Emit a call to llvm.dbg.stoppoint to indicate a change of
-  /// source line.
-  void EmitStopPoint(CGBuilderTy &Builder);
+  /// EmitLocation - Emit metadata to indicate a change in line/column
+  /// information in the source file.
+  void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc);
 
   /// EmitFunctionStart - Emit a call to llvm.dbg.function.start to indicate
   /// start of a new function.
@@ -169,21 +172,17 @@ class CGDebugInfo {
   /// EmitFunctionEnd - Constructs the debug code for exiting a function.
   void EmitFunctionEnd(CGBuilderTy &Builder);
 
-  /// UpdateLineDirectiveRegion - Update region stack only if #line directive
-  /// has introduced scope change.
-  void UpdateLineDirectiveRegion(CGBuilderTy &Builder);
-
   /// UpdateCompletedType - Update type cache because the type is now
   /// translated.
   void UpdateCompletedType(const TagDecl *TD);
 
-  /// EmitRegionStart - Emit a call to llvm.dbg.region.start to indicate start
-  /// of a new block.
-  void EmitRegionStart(CGBuilderTy &Builder);
+  /// EmitLexicalBlockStart - Emit metadata to indicate the beginning of a
+  /// new lexical block and push the block onto the stack.
+  void EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc);
 
-  /// EmitRegionEnd - Emit call to llvm.dbg.region.end to indicate end of a
-  /// block.
-  void EmitRegionEnd(CGBuilderTy &Builder);
+  /// EmitLexicalBlockEnd - Emit metadata to indicate the end of a new lexical
+  /// block and pop the current block.
+  void EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc);
 
   /// EmitDeclareOfAutoVariable - Emit call to llvm.dbg.declare for an automatic
   /// variable declaration.
@@ -234,7 +233,7 @@ class CGDebugInfo {
   llvm::DIDescriptor getContextDescriptor(const Decl *Decl);
 
   /// getCurrentDirname - Return current directory name.
-  llvm::StringRef getCurrentDirname();
+  StringRef getCurrentDirname();
 
   /// CreateCompileUnit - Create new compile unit.
   void CreateCompileUnit();
@@ -255,7 +254,7 @@ class CGDebugInfo {
 
   /// CreateMemberType - Create new member and increase Offset by FType's size.
   llvm::DIType CreateMemberType(llvm::DIFile Unit, QualType FType,
-                                llvm::StringRef Name, uint64_t *Offset);
+                                StringRef Name, uint64_t *Offset);
 
   /// getFunctionDeclaration - Return debug info descriptor to describe method
   /// declaration for the given method definition.
@@ -264,21 +263,21 @@ class CGDebugInfo {
   /// getFunctionName - Get function name for the given FunctionDecl. If the
   /// name is constructred on demand (e.g. C++ destructor) then the name
   /// is stored on the side.
-  llvm::StringRef getFunctionName(const FunctionDecl *FD);
+  StringRef getFunctionName(const FunctionDecl *FD);
 
   /// getObjCMethodName - Returns the unmangled name of an Objective-C method.
   /// This is the display name for the debugging info.  
-  llvm::StringRef getObjCMethodName(const ObjCMethodDecl *FD);
+  StringRef getObjCMethodName(const ObjCMethodDecl *FD);
 
   /// getSelectorName - Return selector name. This is used for debugging
   /// info.
-  llvm::StringRef getSelectorName(Selector S);
+  StringRef getSelectorName(Selector S);
 
   /// getClassName - Get class name including template argument list.
-  llvm::StringRef getClassName(RecordDecl *RD);
+  StringRef getClassName(RecordDecl *RD);
 
   /// getVTableName - Get vtable name for the given Class.
-  llvm::StringRef getVTableName(const CXXRecordDecl *Decl);
+  StringRef getVTableName(const CXXRecordDecl *Decl);
 
   /// getLineNumber - Get line number for the location. If location is invalid
   /// then use current location.
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp
index 62c3a9791d0f..a6147ea7658b 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp
@@ -14,6 +14,7 @@
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CGOpenCLRuntime.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Decl.h"
@@ -46,7 +47,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
   case Decl::Field:
   case Decl::IndirectField:
   case Decl::ObjCIvar:
-  case Decl::ObjCAtDefsField:      
+  case Decl::ObjCAtDefsField:
   case Decl::ParmVar:
   case Decl::ImplicitParam:
   case Decl::ClassTemplate:
@@ -70,7 +71,8 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
   case Decl::Friend:
   case Decl::FriendTemplate:
   case Decl::Block:
-    assert(0 && "Declaration should not be in declstmts!");
+  case Decl::ClassScopeFunctionSpecialization:
+    llvm_unreachable("Declaration should not be in declstmts!");
   case Decl::Function:  // void X();
   case Decl::Record:    // struct/union/class X;
   case Decl::Enum:      // enum X;
@@ -112,7 +114,7 @@ void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
   case SC_Register:
     return EmitAutoVarDecl(D);
   case SC_Static: {
-    llvm::GlobalValue::LinkageTypes Linkage = 
+    llvm::GlobalValue::LinkageTypes Linkage =
       llvm::GlobalValue::InternalLinkage;
 
     // If the function definition has some sort of weak linkage, its
@@ -123,26 +125,28 @@ void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
     if (getContext().getLangOptions().CPlusPlus)
       if (llvm::GlobalValue::isWeakForLinker(CurFn->getLinkage()))
         Linkage = CurFn->getLinkage();
-    
+
     return EmitStaticVarDecl(D, Linkage);
   }
   case SC_Extern:
   case SC_PrivateExtern:
     // Don't emit it now, allow it to be emitted lazily on its first use.
     return;
+  case SC_OpenCLWorkGroupLocal:
+    return CGM.getOpenCLRuntime().EmitWorkGroupLocalVarDecl(*this, D);
   }
 
-  assert(0 && "Unknown storage class");
+  llvm_unreachable("Unknown storage class");
 }
 
 static std::string GetStaticDeclName(CodeGenFunction &CGF, const VarDecl &D,
                                      const char *Separator) {
   CodeGenModule &CGM = CGF.CGM;
   if (CGF.getContext().getLangOptions().CPlusPlus) {
-    llvm::StringRef Name = CGM.getMangledName(&D);
+    StringRef Name = CGM.getMangledName(&D);
     return Name.str();
   }
-  
+
   std::string ContextName;
   if (!CGF.CurFuncDecl) {
     // Better be in a block declared in global scope.
@@ -154,15 +158,15 @@ static std::string GetStaticDeclName(CodeGenFunction &CGF, const VarDecl &D,
       ContextName = Name.getString();
     }
     else
-      assert(0 && "Unknown context for block static var decl");
+      llvm_unreachable("Unknown context for block static var decl");
   } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(CGF.CurFuncDecl)) {
-    llvm::StringRef Name = CGM.getMangledName(FD);
+    StringRef Name = CGM.getMangledName(FD);
     ContextName = Name.str();
   } else if (isa<ObjCMethodDecl>(CGF.CurFuncDecl))
     ContextName = CGF.CurFn->getName();
   else
-    assert(0 && "Unknown context for static var decl");
-  
+    llvm_unreachable("Unknown context for static var decl");
+
   return ContextName + Separator + D.getNameAsString();
 }
 
@@ -175,7 +179,7 @@ CodeGenFunction::CreateStaticVarDecl(const VarDecl &D,
 
   std::string Name = GetStaticDeclName(*this, D, Separator);
 
-  const llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(Ty);
+  llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(Ty);
   llvm::GlobalVariable *GV =
     new llvm::GlobalVariable(CGM.getModule(), LTy,
                              Ty.isConstant(getContext()), Linkage,
@@ -203,7 +207,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
     if (!getContext().getLangOptions().CPlusPlus)
       CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
     else if (Builder.GetInsertBlock()) {
-      // Since we have a static initializer, this global variable can't 
+      // Since we have a static initializer, this global variable can't
       // be constant.
       GV->setConstant(false);
 
@@ -218,7 +222,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
   // in the LLVM type system.)
   if (GV->getType()->getElementType() != Init->getType()) {
     llvm::GlobalVariable *OldGV = GV;
-    
+
     GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
                                   OldGV->isConstant(),
                                   OldGV->getLinkage(), Init, "",
@@ -226,19 +230,19 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
                                   D.isThreadSpecified(),
                            CGM.getContext().getTargetAddressSpace(D.getType()));
     GV->setVisibility(OldGV->getVisibility());
-    
+
     // Steal the name of the old global
     GV->takeName(OldGV);
-    
+
     // Replace all uses of the old global with the new global
     llvm::Constant *NewPtrForOldDecl =
     llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
     OldGV->replaceAllUsesWith(NewPtrForOldDecl);
-    
+
     // Erase the old global, since it is no longer used.
     OldGV->eraseFromParent();
   }
-  
+
   GV->setInitializer(Init);
   return GV;
 }
@@ -259,7 +263,7 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
   // Make sure to evaluate VLA bounds now so that we have them for later.
   if (D.getType()->isVariablyModifiedType())
     EmitVariablyModifiedType(D.getType());
-  
+
   // Local static block variables must be treated as globals as they may be
   // referenced in their RHS initializer block-literal expresion.
   CGM.setStaticLocalDeclAddress(&D, GV);
@@ -270,14 +274,8 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
 
   GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
 
-  // FIXME: Merge attribute handling.
-  if (const AnnotateAttr *AA = D.getAttr<AnnotateAttr>()) {
-    SourceManager &SM = CGM.getContext().getSourceManager();
-    llvm::Constant *Ann =
-      CGM.EmitAnnotateAttr(GV, AA,
-                           SM.getInstantiationLineNumber(D.getLocation()));
-    CGM.AddAnnotation(Ann);
-  }
+  if (D.hasAttr<AnnotateAttr>())
+    CGM.AddGlobalAnnotations(&D, GV);
 
   if (const SectionAttr *SA = D.getAttr<SectionAttr>())
     GV->setSection(SA->getName());
@@ -290,8 +288,8 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
   //
   // FIXME: It is really dangerous to store this in the map; if anyone
   // RAUW's the GV uses of this constant will be invalid.
-  const llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(D.getType());
-  const llvm::Type *LPtrTy =
+  llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(D.getType());
+  llvm::Type *LPtrTy =
     LTy->getPointerTo(CGM.getContext().getTargetAddressSpace(D.getType()));
   DMEntry = llvm::ConstantExpr::getBitCast(GV, LPtrTy);
 
@@ -348,7 +346,7 @@ namespace {
         CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
         CGF.EmitBlock(RunDtorBB);
       }
-          
+
       CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
                                 /*ForVirtualBase=*/false, Loc);
 
@@ -360,7 +358,7 @@ namespace {
     llvm::Value *Stack;
     CallStackRestore(llvm::Value *Stack) : Stack(Stack) {}
     void Emit(CodeGenFunction &CGF, Flags flags) {
-      llvm::Value *V = CGF.Builder.CreateLoad(Stack, "tmp");
+      llvm::Value *V = CGF.Builder.CreateLoad(Stack);
       llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
       CGF.Builder.CreateCall(F, V);
     }
@@ -384,7 +382,7 @@ namespace {
     llvm::Constant *CleanupFn;
     const CGFunctionInfo &FnInfo;
     const VarDecl &Var;
-    
+
     CallCleanupFunction(llvm::Constant *CleanupFn, const CGFunctionInfo *Info,
                         const VarDecl *Var)
       : CleanupFn(CleanupFn), FnInfo(*Info), Var(*Var) {}
@@ -441,7 +439,7 @@ static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
   case Qualifiers::OCL_Autoreleasing:
     // nothing to do
     break;
- 
+
   case Qualifiers::OCL_Weak:
     // __weak objects always get EH cleanups; otherwise, exceptions
     // could cause really nasty crashes instead of mere leaks.
@@ -508,7 +506,7 @@ void CodeGenFunction::EmitScalarInit(const Expr *init,
   // actually perform the initialization with an assign.
   bool accessedByInit = false;
   if (lifetime != Qualifiers::OCL_ExplicitNone)
-    accessedByInit = isAccessedBy(D, init);
+    accessedByInit = (capturedByInit || isAccessedBy(D, init));
   if (accessedByInit) {
     LValue tempLV = lvalue;
     // Drill down to the __block object if necessary.
@@ -519,12 +517,12 @@ void CodeGenFunction::EmitScalarInit(const Expr *init,
                                    getByRefValueLLVMField(cast<VarDecl>(D))));
     }
 
-    const llvm::PointerType *ty
+    llvm::PointerType *ty
       = cast<llvm::PointerType>(tempLV.getAddress()->getType());
     ty = cast<llvm::PointerType>(ty->getElementType());
 
     llvm::Value *zero = llvm::ConstantPointerNull::get(ty);
-    
+
     // If __weak, we want to use a barrier under certain conditions.
     if (lifetime == Qualifiers::OCL_Weak)
       EmitARCInitWeak(tempLV.getAddress(), zero);
@@ -613,7 +611,7 @@ void CodeGenFunction::EmitScalarInit(llvm::Value *init, LValue lvalue) {
     break;
   }
 
-  EmitStoreOfScalar(init, lvalue);  
+  EmitStoreOfScalar(init, lvalue);
 }
 
 /// canEmitInitWithFewStoresAfterMemset - Decide whether we can emit the
@@ -640,7 +638,7 @@ static bool canEmitInitWithFewStoresAfterMemset(llvm::Constant *Init,
     }
     return true;
   }
-  
+
   // Anything else is hard and scary.
   return false;
 }
@@ -655,7 +653,7 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
       isa<llvm::ConstantPointerNull>(Init) ||
       isa<llvm::UndefValue>(Init))
     return;
-  
+
   if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
       isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
       isa<llvm::ConstantExpr>(Init)) {
@@ -663,14 +661,14 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
       Builder.CreateStore(Init, Loc, isVolatile);
     return;
   }
-  
+
   assert((isa<llvm::ConstantStruct>(Init) || isa<llvm::ConstantArray>(Init)) &&
          "Unknown value type!");
-  
+
   for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
     llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
     if (Elt->isNullValue()) continue;
-    
+
     // Otherwise, get a pointer to the element and emit it.
     emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
                                  isVolatile, Builder);
@@ -694,8 +692,8 @@ static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init,
   // plopping in more stores.
   unsigned StoreBudget = 6;
   uint64_t SizeLimit = 32;
-  
-  return GlobalSize > SizeLimit && 
+
+  return GlobalSize > SizeLimit &&
          canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
 }
 
@@ -730,7 +728,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   llvm::Value *DeclPtr;
   if (Ty->isConstantSizeType()) {
     if (!Target.useGlobalsForAutomaticVariables()) {
-      bool NRVO = getContext().getLangOptions().ElideConstructors && 
+      bool NRVO = getContext().getLangOptions().ElideConstructors &&
                   D.isNRVOVariable();
 
       // If this value is a POD array or struct with a statically
@@ -740,7 +738,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
       // arrays as long as the initialization is trivial (e.g. if they
       // have a non-trivial destructor, but not a non-trivial constructor).
       if (D.getInit() &&
-          (Ty->isArrayType() || Ty->isRecordType()) && 
+          (Ty->isArrayType() || Ty->isRecordType()) &&
           (Ty.isPODType(getContext()) ||
            getContext().getBaseElementType(Ty)->isObjCObjectPointerType()) &&
           D.getInit()->isConstantInitializer(getContext(), false)) {
@@ -759,27 +757,27 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
         // Otherwise, tell the initialization code that we're in this case.
         emission.IsConstantAggregate = true;
       }
-      
+
       // A normal fixed sized variable becomes an alloca in the entry block,
       // unless it's an NRVO variable.
-      const llvm::Type *LTy = ConvertTypeForMem(Ty);
-      
+      llvm::Type *LTy = ConvertTypeForMem(Ty);
+
       if (NRVO) {
         // The named return value optimization: allocate this variable in the
         // return slot, so that we can elide the copy when returning this
         // variable (C++0x [class.copy]p34).
         DeclPtr = ReturnValue;
-        
+
         if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
           if (!cast<CXXRecordDecl>(RecordTy->getDecl())->hasTrivialDestructor()) {
             // Create a flag that is used to indicate when the NRVO was applied
-            // to this variable. Set it to zero to indicate that NRVO was not 
+            // to this variable. Set it to zero to indicate that NRVO was not
             // applied.
             llvm::Value *Zero = Builder.getFalse();
             llvm::Value *NRVOFlag = CreateTempAlloca(Zero->getType(), "nrvo");
             EnsureInsertPoint();
             Builder.CreateStore(Zero, NRVOFlag);
-            
+
             // Record the NRVO flag for this variable.
             NRVOFlags[&D] = NRVOFlag;
             emission.NRVOFlag = NRVOFlag;
@@ -788,13 +786,13 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
       } else {
         if (isByRef)
           LTy = BuildByRefType(&D);
-        
+
         llvm::AllocaInst *Alloc = CreateTempAlloca(LTy);
         Alloc->setName(D.getNameAsString());
 
         CharUnits allocaAlignment = alignment;
         if (isByRef)
-          allocaAlignment = std::max(allocaAlignment, 
+          allocaAlignment = std::max(allocaAlignment,
               getContext().toCharUnitsFromBits(Target.getPointerAlign(0)));
         Alloc->setAlignment(allocaAlignment.getQuantity());
         DeclPtr = Alloc;
@@ -829,7 +827,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
     QualType elementType;
     llvm::tie(elementCount, elementType) = getVLASize(Ty);
 
-    const llvm::Type *llvmTy = ConvertTypeForMem(elementType);
+    llvm::Type *llvmTy = ConvertTypeForMem(elementType);
 
     // Allocate memory for the array.
     llvm::AllocaInst *vla = Builder.CreateAlloca(llvmTy, elementCount, "vla");
@@ -853,6 +851,9 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
         DI->EmitDeclareOfAutoVariable(&D, DeclPtr, Builder);
     }
 
+  if (D.hasAttr<AnnotateAttr>())
+      EmitVarAnnotations(&D, emission.Address);
+
   return emission;
 }
 
@@ -875,6 +876,32 @@ static bool isCapturedBy(const VarDecl &var, const Expr *e) {
     return false;
   }
 
+  if (const StmtExpr *SE = dyn_cast<StmtExpr>(e)) {
+    const CompoundStmt *CS = SE->getSubStmt();
+    for (CompoundStmt::const_body_iterator BI = CS->body_begin(),
+	   BE = CS->body_end(); BI != BE; ++BI)
+      if (Expr *E = dyn_cast<Expr>((*BI))) {
+        if (isCapturedBy(var, E))
+            return true;
+      }
+      else if (DeclStmt *DS = dyn_cast<DeclStmt>((*BI))) {
+          // special case declarations
+          for (DeclStmt::decl_iterator I = DS->decl_begin(), E = DS->decl_end();
+               I != E; ++I) {
+              if (VarDecl *VD = dyn_cast<VarDecl>((*I))) {
+                Expr *Init = VD->getInit();
+                if (Init && isCapturedBy(var, Init))
+                  return true;
+              }
+          }
+      }
+      else
+        // FIXME. Make safe assumption assuming arbitrary statements cause capturing.
+        // Later, provide code to poke into statements for capture analysis.
+        return true;
+    return false;
+  }
+
   for (Stmt::const_child_range children = e->children(); children; ++children)
     if (isCapturedBy(var, cast<Expr>(*children)))
       return true;
@@ -887,14 +914,14 @@ static bool isCapturedBy(const VarDecl &var, const Expr *e) {
 static bool isTrivialInitializer(const Expr *Init) {
   if (!Init)
     return true;
-  
+
   if (const CXXConstructExpr *Construct = dyn_cast<CXXConstructExpr>(Init))
     if (CXXConstructorDecl *Constructor = Construct->getConstructor())
       if (Constructor->isTrivial() &&
           Constructor->isDefaultConstructor() &&
           !Construct->requiresZeroInitialization())
         return true;
-      
+
   return false;
 }
 void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
@@ -922,7 +949,6 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
 
   if (isTrivialInitializer(Init))
     return;
-  
 
   CharUnits alignment = emission.Alignment;
 
@@ -950,16 +976,16 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
   assert(constant != 0 && "Wasn't a simple constant init?");
 
   llvm::Value *SizeVal =
-    llvm::ConstantInt::get(IntPtrTy, 
+    llvm::ConstantInt::get(IntPtrTy,
                            getContext().getTypeSizeInChars(type).getQuantity());
 
-  const llvm::Type *BP = Int8PtrTy;
+  llvm::Type *BP = Int8PtrTy;
   if (Loc->getType() != BP)
-    Loc = Builder.CreateBitCast(Loc, BP, "tmp");
+    Loc = Builder.CreateBitCast(Loc, BP);
 
   // If the initializer is all or mostly zeros, codegen with memset then do
   // a few stores afterward.
-  if (shouldUseMemSetPlusStoresToInitialize(constant, 
+  if (shouldUseMemSetPlusStoresToInitialize(constant,
                 CGM.getTargetData().getTypeAllocSize(constant->getType()))) {
     Builder.CreateMemSet(Loc, llvm::ConstantInt::get(Int8Ty, 0), SizeVal,
                          alignment.getQuantity(), isVolatile);
@@ -968,19 +994,19 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
       emitStoresForInitAfterMemset(constant, Loc, isVolatile, Builder);
     }
   } else {
-    // Otherwise, create a temporary global with the initializer then 
+    // Otherwise, create a temporary global with the initializer then
     // memcpy from the global to the alloca.
     std::string Name = GetStaticDeclName(*this, D, ".");
     llvm::GlobalVariable *GV =
       new llvm::GlobalVariable(CGM.getModule(), constant->getType(), true,
-                               llvm::GlobalValue::InternalLinkage,
+                               llvm::GlobalValue::PrivateLinkage,
                                constant, Name, 0, false, 0);
     GV->setAlignment(alignment.getQuantity());
     GV->setUnnamedAddr(true);
-        
+
     llvm::Value *SrcPtr = GV;
     if (SrcPtr->getType() != BP)
-      SrcPtr = Builder.CreateBitCast(SrcPtr, BP, "tmp");
+      SrcPtr = Builder.CreateBitCast(SrcPtr, BP);
 
     Builder.CreateMemCpy(Loc, SrcPtr, SizeVal, alignment.getQuantity(),
                          isVolatile);
@@ -1007,7 +1033,7 @@ void CodeGenFunction::EmitExprAsInit(const Expr *init,
 
   if (type->isReferenceType()) {
     RValue rvalue = EmitReferenceBindingToExpr(init, D);
-    if (capturedByInit) 
+    if (capturedByInit)
       drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
     EmitStoreThroughLValue(rvalue, lvalue);
   } else if (!hasAggregateLLVMType(type)) {
@@ -1019,7 +1045,10 @@ void CodeGenFunction::EmitExprAsInit(const Expr *init,
     StoreComplexToAddr(complex, lvalue.getAddress(), lvalue.isVolatile());
   } else {
     // TODO: how can we delay here if D is captured by its initializer?
-    EmitAggExpr(init, AggValueSlot::forLValue(lvalue, true, false));
+    EmitAggExpr(init, AggValueSlot::forLValue(lvalue,
+                                              AggValueSlot::IsDestructed,
+                                         AggValueSlot::DoesNotNeedGCBarriers,
+                                              AggValueSlot::IsNotAliased));
   }
 }
 
@@ -1058,7 +1087,7 @@ void CodeGenFunction::emitAutoVarTypeCleanup(
   case QualType::DK_objc_strong_lifetime:
     // Suppress cleanups for pseudo-strong variables.
     if (var->isARCPseudoStrong()) return;
-    
+
     // Otherwise, consider whether to use an EH cleanup or not.
     cleanupKind = getARCCleanupKind();
 
@@ -1094,7 +1123,7 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) {
     emitAutoVarTypeCleanup(emission, dtorKind);
 
   // In GC mode, honor objc_precise_lifetime.
-  if (getLangOptions().getGCMode() != LangOptions::NonGC &&
+  if (getLangOptions().getGC() != LangOptions::NonGC &&
       D.hasAttr<ObjCPreciseLifetimeAttr>()) {
     EHStack.pushCleanup<ExtendGCLifetime>(NormalCleanup, &D);
   }
@@ -1267,11 +1296,9 @@ static void emitPartialArrayDestroy(CodeGenFunction &CGF,
   if (arrayDepth) {
     llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, arrayDepth+1);
 
-    llvm::SmallVector<llvm::Value*,4> gepIndices(arrayDepth, zero);
-    begin = CGF.Builder.CreateInBoundsGEP(begin, gepIndices.begin(),
-                                          gepIndices.end(), "pad.arraybegin");
-    end = CGF.Builder.CreateInBoundsGEP(end, gepIndices.begin(),
-                                        gepIndices.end(), "pad.arrayend");
+    SmallVector<llvm::Value*,4> gepIndices(arrayDepth, zero);
+    begin = CGF.Builder.CreateInBoundsGEP(begin, gepIndices, "pad.arraybegin");
+    end = CGF.Builder.CreateInBoundsGEP(end, gepIndices, "pad.arrayend");
   }
 
   // Destroy the array.  We don't ever need an EH cleanup because we
@@ -1330,7 +1357,7 @@ namespace {
 /// pushIrregularPartialArrayCleanup - Push an EH cleanup to destroy
 /// already-constructed elements of the given array.  The cleanup
 /// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
-/// 
+///
 /// \param elementType - the immediate element type of the array;
 ///   possibly still an array type
 /// \param array - a value of type elementType*
@@ -1349,7 +1376,7 @@ void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
 /// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy
 /// already-constructed elements of the given array.  The cleanup
 /// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
-/// 
+///
 /// \param elementType - the immediate element type of the array;
 ///   possibly still an array type
 /// \param array - a value of type elementType*
@@ -1476,4 +1503,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg,
   // Emit debug info for param declaration.
   if (CGDebugInfo *DI = getDebugInfo())
     DI->EmitDeclareOfArgVariable(&D, DeclPtr, ArgNo, Builder);
+
+  if (D.hasAttr<AnnotateAttr>())
+      EmitVarAnnotations(&D, DeclPtr);
 }
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp
index 0ae6a3d2ee90..3b8f830278b2 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -46,7 +46,9 @@ static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
   } else if (type->isAnyComplexType()) {
     CGF.EmitComplexExprIntoAddr(Init, DeclPtr, lv.isVolatile());
   } else {
-    CGF.EmitAggExpr(Init, AggValueSlot::forLValue(lv, true));
+    CGF.EmitAggExpr(Init, AggValueSlot::forLValue(lv,AggValueSlot::IsDestructed,
+                                          AggValueSlot::DoesNotNeedGCBarriers,
+                                                  AggValueSlot::IsNotAliased));
   }
 }
 
@@ -126,17 +128,16 @@ CodeGenFunction::EmitCXXGlobalDtorRegistration(llvm::Constant *DtorFn,
   }
 
   // Get the destructor function type
-  llvm::Type *ArgTys[] = { Int8PtrTy };
   llvm::Type *DtorFnTy =
     llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
-                            ArgTys, false);
+                            Int8PtrTy, false);
   DtorFnTy = llvm::PointerType::getUnqual(DtorFnTy);
 
   llvm::Type *Params[] = { DtorFnTy, Int8PtrTy, Int8PtrTy };
 
   // Get the __cxa_atexit function type
   // extern "C" int __cxa_atexit ( void (*f)(void *), void *p, void *d );
-  const llvm::FunctionType *AtExitFnTy =
+  llvm::FunctionType *AtExitFnTy =
     llvm::FunctionType::get(ConvertType(getContext().IntTy), Params, false);
 
   llvm::Constant *AtExitFn = CGM.CreateRuntimeFunction(AtExitFnTy,
@@ -167,15 +168,15 @@ void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
 
 static llvm::Function *
 CreateGlobalInitOrDestructFunction(CodeGenModule &CGM,
-                                   const llvm::FunctionType *FTy,
-                                   llvm::StringRef Name) {
+                                   llvm::FunctionType *FTy,
+                                   StringRef Name) {
   llvm::Function *Fn =
     llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
                            Name, &CGM.getModule());
   if (!CGM.getContext().getLangOptions().AppleKext) {
     // Set the section if needed.
     if (const char *Section = 
-          CGM.getContext().Target.getStaticInitSectionSpecifier())
+          CGM.getContext().getTargetInfo().getStaticInitSectionSpecifier())
       Fn->setSection(Section);
   }
 
@@ -188,7 +189,7 @@ CreateGlobalInitOrDestructFunction(CodeGenModule &CGM,
 void
 CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
                                             llvm::GlobalVariable *Addr) {
-  const llvm::FunctionType *FTy
+  llvm::FunctionType *FTy
     = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
                               false);
 
@@ -225,7 +226,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() {
   if (CXXGlobalInits.empty() && PrioritizedCXXGlobalInits.empty())
     return;
 
-  const llvm::FunctionType *FTy
+  llvm::FunctionType *FTy
     = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
                               false);
 
@@ -234,7 +235,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() {
     CreateGlobalInitOrDestructFunction(*this, FTy, "_GLOBAL__I_a");
 
   if (!PrioritizedCXXGlobalInits.empty()) {
-    llvm::SmallVector<llvm::Constant*, 8> LocalCXXGlobalInits;
+    SmallVector<llvm::Constant*, 8> LocalCXXGlobalInits;
     llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(), 
                          PrioritizedCXXGlobalInits.end()); 
     for (unsigned i = 0; i < PrioritizedCXXGlobalInits.size(); i++) {
@@ -259,7 +260,7 @@ void CodeGenModule::EmitCXXGlobalDtorFunc() {
   if (CXXGlobalDtors.empty())
     return;
 
-  const llvm::FunctionType *FTy
+  llvm::FunctionType *FTy
     = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
                               false);
 
@@ -351,7 +352,7 @@ CodeGenFunction::generateDestroyHelper(llvm::Constant *addr,
   const CGFunctionInfo &FI = 
     CGM.getTypes().getFunctionInfo(getContext().VoidTy, args, 
                                    FunctionType::ExtInfo());
-  const llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI, false);
+  llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI, false);
   llvm::Function *fn = 
     CreateGlobalInitOrDestructFunction(CGM, FTy, "__cxx_global_array_dtor");
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp
index 418bea6ee402..5e4fb9881937 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp
@@ -29,9 +29,8 @@ using namespace CodeGen;
 static llvm::Constant *getAllocateExceptionFn(CodeGenFunction &CGF) {
   // void *__cxa_allocate_exception(size_t thrown_size);
 
-  llvm::Type *ArgTys[] = { CGF.SizeTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.Int8PtrTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.Int8PtrTy, CGF.SizeTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception");
 }
@@ -39,9 +38,8 @@ static llvm::Constant *getAllocateExceptionFn(CodeGenFunction &CGF) {
 static llvm::Constant *getFreeExceptionFn(CodeGenFunction &CGF) {
   // void __cxa_free_exception(void *thrown_exception);
 
-  llvm::Type *ArgTys[] = { CGF.Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.VoidTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.VoidTy, CGF.Int8PtrTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_free_exception");
 }
@@ -51,7 +49,7 @@ static llvm::Constant *getThrowFn(CodeGenFunction &CGF) {
   //                  void (*dest) (void *));
 
   llvm::Type *Args[3] = { CGF.Int8PtrTy, CGF.Int8PtrTy, CGF.Int8PtrTy };
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGF.VoidTy, Args, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_throw");
@@ -60,7 +58,7 @@ static llvm::Constant *getThrowFn(CodeGenFunction &CGF) {
 static llvm::Constant *getReThrowFn(CodeGenFunction &CGF) {
   // void __cxa_rethrow();
 
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGF.VoidTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_rethrow");
@@ -69,9 +67,8 @@ static llvm::Constant *getReThrowFn(CodeGenFunction &CGF) {
 static llvm::Constant *getGetExceptionPtrFn(CodeGenFunction &CGF) {
   // void *__cxa_get_exception_ptr(void*);
 
-  llvm::Type *ArgTys[] = { CGF.Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.Int8PtrTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.Int8PtrTy, CGF.Int8PtrTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr");
 }
@@ -79,9 +76,8 @@ static llvm::Constant *getGetExceptionPtrFn(CodeGenFunction &CGF) {
 static llvm::Constant *getBeginCatchFn(CodeGenFunction &CGF) {
   // void *__cxa_begin_catch(void*);
 
-  llvm::Type *ArgTys[] = { CGF.Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.Int8PtrTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.Int8PtrTy, CGF.Int8PtrTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_begin_catch");
 }
@@ -89,7 +85,7 @@ static llvm::Constant *getBeginCatchFn(CodeGenFunction &CGF) {
 static llvm::Constant *getEndCatchFn(CodeGenFunction &CGF) {
   // void __cxa_end_catch();
 
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGF.VoidTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_end_catch");
@@ -98,17 +94,15 @@ static llvm::Constant *getEndCatchFn(CodeGenFunction &CGF) {
 static llvm::Constant *getUnexpectedFn(CodeGenFunction &CGF) {
   // void __cxa_call_unexepcted(void *thrown_exception);
 
-  llvm::Type *ArgTys[] = { CGF.Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.VoidTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.VoidTy, CGF.Int8PtrTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_call_unexpected");
 }
 
 llvm::Constant *CodeGenFunction::getUnwindResumeFn() {
-  llvm::Type *ArgTys[] = { Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(VoidTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(VoidTy, Int8PtrTy, /*IsVarArgs=*/false);
 
   if (CGM.getLangOptions().SjLjExceptions)
     return CGM.CreateRuntimeFunction(FTy, "_Unwind_SjLj_Resume");
@@ -116,9 +110,8 @@ llvm::Constant *CodeGenFunction::getUnwindResumeFn() {
 }
 
 llvm::Constant *CodeGenFunction::getUnwindResumeOrRethrowFn() {
-  llvm::Type *ArgTys[] = { Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(VoidTy, ArgTys, /*IsVarArgs=*/false);
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(VoidTy, Int8PtrTy, /*IsVarArgs=*/false);
 
   if (CGM.getLangOptions().SjLjExceptions)
     return CGM.CreateRuntimeFunction(FTy, "_Unwind_SjLj_Resume_or_Rethrow");
@@ -128,10 +121,10 @@ llvm::Constant *CodeGenFunction::getUnwindResumeOrRethrowFn() {
 static llvm::Constant *getTerminateFn(CodeGenFunction &CGF) {
   // void __terminate();
 
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGF.VoidTy, /*IsVarArgs=*/false);
 
-  llvm::StringRef name;
+  StringRef name;
 
   // In C++, use std::terminate().
   if (CGF.getLangOptions().CPlusPlus)
@@ -145,10 +138,9 @@ static llvm::Constant *getTerminateFn(CodeGenFunction &CGF) {
 }
 
 static llvm::Constant *getCatchallRethrowFn(CodeGenFunction &CGF,
-                                            llvm::StringRef Name) {
-  llvm::Type *ArgTys[] = { CGF.Int8PtrTy };
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.VoidTy, ArgTys, /*IsVarArgs=*/false);
+                                            StringRef Name) {
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.VoidTy, CGF.Int8PtrTy, /*IsVarArgs=*/false);
 
   return CGF.CGM.CreateRuntimeFunction(FTy, Name);
 }
@@ -247,21 +239,34 @@ static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) {
       continue;
     }
 
-    // Otherwise, it has to be a selector call.
-    if (!isa<llvm::EHSelectorInst>(User)) return false;
+    // Otherwise, it has to be a landingpad instruction.
+    llvm::LandingPadInst *LPI = dyn_cast<llvm::LandingPadInst>(User);
+    if (!LPI) return false;
 
-    llvm::EHSelectorInst *Selector = cast<llvm::EHSelectorInst>(User);
-    for (unsigned I = 2, E = Selector->getNumArgOperands(); I != E; ++I) {
+    for (unsigned I = 0, E = LPI->getNumClauses(); I != E; ++I) {
       // Look for something that would've been returned by the ObjC
       // runtime's GetEHType() method.
-      llvm::GlobalVariable *GV
-        = dyn_cast<llvm::GlobalVariable>(Selector->getArgOperand(I));
-      if (!GV) continue;
-
-      // ObjC EH selector entries are always global variables with
-      // names starting like this.
-      if (GV->getName().startswith("OBJC_EHTYPE"))
-        return false;
+      llvm::Value *Val = LPI->getClause(I)->stripPointerCasts();
+      if (LPI->isCatch(I)) {
+        // Check if the catch value has the ObjC prefix.
+        if (llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(Val))
+          // ObjC EH selector entries are always global variables with
+          // names starting like this.
+          if (GV->getName().startswith("OBJC_EHTYPE"))
+            return false;
+      } else {
+        // Check if any of the filter values have the ObjC prefix.
+        llvm::Constant *CVal = cast<llvm::Constant>(Val);
+        for (llvm::User::op_iterator
+               II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II) {
+          if (llvm::GlobalVariable *GV =
+              cast<llvm::GlobalVariable>((*II)->stripPointerCasts()))
+            // ObjC EH selector entries are always global variables with
+            // names starting like this.
+            if (GV->getName().startswith("OBJC_EHTYPE"))
+              return false;
+        }
+      }
     }
   }
 
@@ -274,7 +279,7 @@ static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) {
 /// when it really needs it.
 void CodeGenModule::SimplifyPersonality() {
   // For now, this is really a Darwin-specific operation.
-  if (!Context.Target.getTriple().isOSDarwin())
+  if (!Context.getTargetInfo().getTriple().isOSDarwin())
     return;
 
   // If we're not in ObjC++ -fexceptions, there's nothing to do.
@@ -314,12 +319,6 @@ static llvm::Constant *getCatchAllValue(CodeGenFunction &CGF) {
   return llvm::ConstantPointerNull::get(CGF.Int8PtrTy);
 }
 
-/// Returns the value to inject into a selector to indicate the
-/// presence of a cleanup.
-static llvm::Constant *getCleanupValue(CodeGenFunction &CGF) {
-  return llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
-}
-
 namespace {
   /// A cleanup to free the exception object if its initialization
   /// throws.
@@ -346,7 +345,7 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e,
 
   // __cxa_allocate_exception returns a void*;  we need to cast this
   // to the appropriate type for the object.
-  const llvm::Type *ty = CGF.ConvertTypeForMem(e->getType())->getPointerTo();
+  llvm::Type *ty = CGF.ConvertTypeForMem(e->getType())->getPointerTo();
   llvm::Value *typedAddr = CGF.Builder.CreateBitCast(addr, ty);
 
   // FIXME: this isn't quite right!  If there's a final unelided call
@@ -375,6 +374,14 @@ llvm::Value *CodeGenFunction::getEHSelectorSlot() {
   return EHSelectorSlot;
 }
 
+llvm::Value *CodeGenFunction::getExceptionFromSlot() {
+  return Builder.CreateLoad(getExceptionSlot(), "exn");
+}
+
+llvm::Value *CodeGenFunction::getSelectorFromSlot() {
+  return Builder.CreateLoad(getEHSelectorSlot(), "sel");
+}
+
 void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
   if (!E->getSubExpr()) {
     if (getInvokeDest()) {
@@ -397,7 +404,7 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
   QualType ThrowType = E->getSubExpr()->getType();
 
   // Now allocate the exception object.
-  const llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
+  llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
   uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity();
 
   llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(*this);
@@ -475,6 +482,43 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
   }
 }
 
+/// Emit the dispatch block for a filter scope if necessary.
+static void emitFilterDispatchBlock(CodeGenFunction &CGF,
+                                    EHFilterScope &filterScope) {
+  llvm::BasicBlock *dispatchBlock = filterScope.getCachedEHDispatchBlock();
+  if (!dispatchBlock) return;
+  if (dispatchBlock->use_empty()) {
+    delete dispatchBlock;
+    return;
+  }
+
+  CGF.EmitBlockAfterUses(dispatchBlock);
+
+  // If this isn't a catch-all filter, we need to check whether we got
+  // here because the filter triggered.
+  if (filterScope.getNumFilters()) {
+    // Load the selector value.
+    llvm::Value *selector = CGF.getSelectorFromSlot();
+    llvm::BasicBlock *unexpectedBB = CGF.createBasicBlock("ehspec.unexpected");
+
+    llvm::Value *zero = CGF.Builder.getInt32(0);
+    llvm::Value *failsFilter =
+      CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails");
+    CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock());
+
+    CGF.EmitBlock(unexpectedBB);
+  }
+
+  // Call __cxa_call_unexpected.  This doesn't need to be an invoke
+  // because __cxa_call_unexpected magically filters exceptions
+  // according to the last landing pad the exception was thrown
+  // into.  Seriously.
+  llvm::Value *exn = CGF.getExceptionFromSlot();
+  CGF.Builder.CreateCall(getUnexpectedFn(CGF), exn)
+    ->setDoesNotReturn();
+  CGF.Builder.CreateUnreachable();
+}
+
 void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
   if (!CGM.getLangOptions().CXXExceptions)
     return;
@@ -492,6 +536,8 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
       EHStack.popTerminate();
     }
   } else if (EST == EST_Dynamic || EST == EST_DynamicNone) {
+    EHFilterScope &filterScope = cast<EHFilterScope>(*EHStack.begin());
+    emitFilterDispatchBlock(*this, filterScope);
     EHStack.popFilter();
   }
 }
@@ -533,6 +579,50 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
   }
 }
 
+llvm::BasicBlock *
+CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) {
+  // The dispatch block for the end of the scope chain is a block that
+  // just resumes unwinding.
+  if (si == EHStack.stable_end())
+    return getEHResumeBlock();
+
+  // Otherwise, we should look at the actual scope.
+  EHScope &scope = *EHStack.find(si);
+
+  llvm::BasicBlock *dispatchBlock = scope.getCachedEHDispatchBlock();
+  if (!dispatchBlock) {
+    switch (scope.getKind()) {
+    case EHScope::Catch: {
+      // Apply a special case to a single catch-all.
+      EHCatchScope &catchScope = cast<EHCatchScope>(scope);
+      if (catchScope.getNumHandlers() == 1 &&
+          catchScope.getHandler(0).isCatchAll()) {
+        dispatchBlock = catchScope.getHandler(0).Block;
+
+      // Otherwise, make a dispatch block.
+      } else {
+        dispatchBlock = createBasicBlock("catch.dispatch");
+      }
+      break;
+    }
+
+    case EHScope::Cleanup:
+      dispatchBlock = createBasicBlock("ehcleanup");
+      break;
+
+    case EHScope::Filter:
+      dispatchBlock = createBasicBlock("filter.dispatch");
+      break;
+
+    case EHScope::Terminate:
+      dispatchBlock = getTerminateHandler();
+      break;
+    }
+    scope.setCachedEHDispatchBlock(dispatchBlock);
+  }
+  return dispatchBlock;
+}
+
 /// Check whether this is a non-EH scope, i.e. a scope which doesn't
 /// affect exception handling.  Currently, the only non-EH scopes are
 /// normal-only cleanup scopes.
@@ -629,280 +719,143 @@ const CleanupHackLevel_t CleanupHackLevel = CHL_MandatoryCleanup;
 llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   assert(EHStack.requiresLandingPad());
 
-  for (EHScopeStack::iterator ir = EHStack.begin(); ; ) {
-    assert(ir != EHStack.end() &&
-           "stack requiring landing pad is nothing but non-EH scopes?");
+  EHScope &innermostEHScope = *EHStack.find(EHStack.getInnermostEHScope());
+  switch (innermostEHScope.getKind()) {
+  case EHScope::Terminate:
+    return getTerminateLandingPad();
 
-    // If this is a terminate scope, just use the singleton terminate
-    // landing pad.
-    if (isa<EHTerminateScope>(*ir))
-      return getTerminateLandingPad();
-
-    // If this isn't an EH scope, iterate; otherwise break out.
-    if (!isNonEHScope(*ir)) break;
-    ++ir;
-
-    // We haven't checked this scope for a cached landing pad yet.
-    if (llvm::BasicBlock *LP = ir->getCachedLandingPad())
-      return LP;
+  case EHScope::Catch:
+  case EHScope::Cleanup:
+  case EHScope::Filter:
+    if (llvm::BasicBlock *lpad = innermostEHScope.getCachedLandingPad())
+      return lpad;
   }
 
   // Save the current IR generation state.
-  CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+  CGBuilderTy::InsertPoint savedIP = Builder.saveAndClearIP();
 
-  const EHPersonality &Personality = EHPersonality::get(getLangOptions());
+  const EHPersonality &personality = EHPersonality::get(getLangOptions());
 
   // Create and configure the landing pad.
-  llvm::BasicBlock *LP = createBasicBlock("lpad");
-  EmitBlock(LP);
+  llvm::BasicBlock *lpad = createBasicBlock("lpad");
+  EmitBlock(lpad);
+
+  llvm::LandingPadInst *LPadInst =
+    Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, NULL),
+                             getOpaquePersonalityFn(CGM, personality), 0);
+
+  llvm::Value *LPadExn = Builder.CreateExtractValue(LPadInst, 0);
+  Builder.CreateStore(LPadExn, getExceptionSlot());
+  llvm::Value *LPadSel = Builder.CreateExtractValue(LPadInst, 1);
+  Builder.CreateStore(LPadSel, getEHSelectorSlot());
 
   // Save the exception pointer.  It's safe to use a single exception
   // pointer per function because EH cleanups can never have nested
   // try/catches.
-  llvm::CallInst *Exn =
-    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_exception), "exn");
-  Exn->setDoesNotThrow();
-  Builder.CreateStore(Exn, getExceptionSlot());
-  
-  // Build the selector arguments.
-  llvm::SmallVector<llvm::Value*, 8> EHSelector;
-  EHSelector.push_back(Exn);
-  EHSelector.push_back(getOpaquePersonalityFn(CGM, Personality));
+  // Build the landingpad instruction.
 
   // Accumulate all the handlers in scope.
-  llvm::DenseMap<llvm::Value*, UnwindDest> EHHandlers;
-  UnwindDest CatchAll;
-  bool HasEHCleanup = false;
-  bool HasEHFilter = false;
-  llvm::SmallVector<llvm::Value*, 8> EHFilters;
+  bool hasCatchAll = false;
+  bool hasCleanup = false;
+  bool hasFilter = false;
+  SmallVector<llvm::Value*, 4> filterTypes;
+  llvm::SmallPtrSet<llvm::Value*, 4> catchTypes;
   for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end();
          I != E; ++I) {
 
     switch (I->getKind()) {
     case EHScope::Cleanup:
-      if (!HasEHCleanup)
-        HasEHCleanup = cast<EHCleanupScope>(*I).isEHCleanup();
-      // We otherwise don't care about cleanups.
+      // If we have a cleanup, remember that.
+      hasCleanup = (hasCleanup || cast<EHCleanupScope>(*I).isEHCleanup());
       continue;
 
     case EHScope::Filter: {
       assert(I.next() == EHStack.end() && "EH filter is not end of EH stack");
-      assert(!CatchAll.isValid() && "EH filter reached after catch-all");
+      assert(!hasCatchAll && "EH filter reached after catch-all");
 
-      // Filter scopes get added to the selector in weird ways.
-      EHFilterScope &Filter = cast<EHFilterScope>(*I);
-      HasEHFilter = true;
+      // Filter scopes get added to the landingpad in weird ways.
+      EHFilterScope &filter = cast<EHFilterScope>(*I);
+      hasFilter = true;
 
-      // Add all the filter values which we aren't already explicitly
-      // catching.
-      for (unsigned I = 0, E = Filter.getNumFilters(); I != E; ++I) {
-        llvm::Value *FV = Filter.getFilter(I);
-        if (!EHHandlers.count(FV))
-          EHFilters.push_back(FV);
-      }
+      // Add all the filter values.
+      for (unsigned i = 0, e = filter.getNumFilters(); i != e; ++i)
+        filterTypes.push_back(filter.getFilter(i));
       goto done;
     }
 
     case EHScope::Terminate:
       // Terminate scopes are basically catch-alls.
-      assert(!CatchAll.isValid());
-      CatchAll = UnwindDest(getTerminateHandler(),
-                            EHStack.getEnclosingEHCleanup(I),
-                            cast<EHTerminateScope>(*I).getDestIndex());
+      assert(!hasCatchAll);
+      hasCatchAll = true;
       goto done;
 
     case EHScope::Catch:
       break;
     }
 
-    EHCatchScope &Catch = cast<EHCatchScope>(*I);
-    for (unsigned HI = 0, HE = Catch.getNumHandlers(); HI != HE; ++HI) {
-      EHCatchScope::Handler Handler = Catch.getHandler(HI);
+    EHCatchScope &catchScope = cast<EHCatchScope>(*I);
+    for (unsigned hi = 0, he = catchScope.getNumHandlers(); hi != he; ++hi) {
+      EHCatchScope::Handler handler = catchScope.getHandler(hi);
 
-      // Catch-all.  We should only have one of these per catch.
-      if (!Handler.Type) {
-        assert(!CatchAll.isValid());
-        CatchAll = UnwindDest(Handler.Block,
-                              EHStack.getEnclosingEHCleanup(I),
-                              Handler.Index);
-        continue;
+      // If this is a catch-all, register that and abort.
+      if (!handler.Type) {
+        assert(!hasCatchAll);
+        hasCatchAll = true;
+        goto done;
       }
 
       // Check whether we already have a handler for this type.
-      UnwindDest &Dest = EHHandlers[Handler.Type];
-      if (Dest.isValid()) continue;
-
-      EHSelector.push_back(Handler.Type);
-      Dest = UnwindDest(Handler.Block,
-                        EHStack.getEnclosingEHCleanup(I),
-                        Handler.Index);
+      if (catchTypes.insert(handler.Type))
+        // If not, add it directly to the landingpad.
+        LPadInst->addClause(handler.Type);
     }
-
-    // Stop if we found a catch-all.
-    if (CatchAll.isValid()) break;
   }
 
  done:
-  unsigned LastToEmitInLoop = EHSelector.size();
-
-  // If we have a catch-all, add null to the selector.
-  if (CatchAll.isValid()) {
-    EHSelector.push_back(getCatchAllValue(*this));
+  // If we have a catch-all, add null to the landingpad.
+  assert(!(hasCatchAll && hasFilter));
+  if (hasCatchAll) {
+    LPadInst->addClause(getCatchAllValue(*this));
 
   // If we have an EH filter, we need to add those handlers in the
-  // right place in the selector, which is to say, at the end.
-  } else if (HasEHFilter) {
-    // Create a filter expression: an integer constant saying how many
-    // filters there are (+1 to avoid ambiguity with 0 for cleanup),
-    // followed by the filter types.  The personality routine only
-    // lands here if the filter doesn't match.
-    EHSelector.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(),
-                                                EHFilters.size() + 1));
-    EHSelector.append(EHFilters.begin(), EHFilters.end());
+  // right place in the landingpad, which is to say, at the end.
+  } else if (hasFilter) {
+    // Create a filter expression: a constant array indicating which filter
+    // types there are. The personality routine only lands here if the filter
+    // doesn't match.
+    llvm::SmallVector<llvm::Constant*, 8> Filters;
+    llvm::ArrayType *AType =
+      llvm::ArrayType::get(!filterTypes.empty() ?
+                             filterTypes[0]->getType() : Int8PtrTy,
+                           filterTypes.size());
+
+    for (unsigned i = 0, e = filterTypes.size(); i != e; ++i)
+      Filters.push_back(cast<llvm::Constant>(filterTypes[i]));
+    llvm::Constant *FilterArray = llvm::ConstantArray::get(AType, Filters);
+    LPadInst->addClause(FilterArray);
 
     // Also check whether we need a cleanup.
-    if (CleanupHackLevel == CHL_MandatoryCatchall || HasEHCleanup)
-      EHSelector.push_back(CleanupHackLevel == CHL_MandatoryCatchall
-                           ? getCatchAllValue(*this)
-                           : getCleanupValue(*this));
+    if (hasCleanup)
+      LPadInst->setCleanup(true);
 
   // Otherwise, signal that we at least have cleanups.
-  } else if (CleanupHackLevel == CHL_MandatoryCatchall || HasEHCleanup) {
-    EHSelector.push_back(CleanupHackLevel == CHL_MandatoryCatchall
-                         ? getCatchAllValue(*this)
-                         : getCleanupValue(*this));
-
-  // At the MandatoryCleanup hack level, we don't need to actually
-  // spuriously tell the unwinder that we have cleanups, but we do
-  // need to always be prepared to handle cleanups.
-  } else if (CleanupHackLevel == CHL_MandatoryCleanup) {
-    // Just don't decrement LastToEmitInLoop.
-
-  } else {
-    assert(LastToEmitInLoop > 2);
-    LastToEmitInLoop--;
+  } else if (CleanupHackLevel == CHL_MandatoryCatchall || hasCleanup) {
+    if (CleanupHackLevel == CHL_MandatoryCatchall)
+      LPadInst->addClause(getCatchAllValue(*this));
+    else
+      LPadInst->setCleanup(true);
   }
 
-  assert(EHSelector.size() >= 3 && "selector call has only two arguments!");
+  assert((LPadInst->getNumClauses() > 0 || LPadInst->isCleanup()) &&
+         "landingpad instruction has no clauses!");
 
   // Tell the backend how to generate the landing pad.
-  llvm::CallInst *Selection =
-    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_selector),
-                       EHSelector, "eh.selector");
-  Selection->setDoesNotThrow();
-
-  // Save the selector value in mandatory-cleanup mode.
-  if (CleanupHackLevel == CHL_MandatoryCleanup)
-    Builder.CreateStore(Selection, getEHSelectorSlot());
-  
-  // Select the right handler.
-  llvm::Value *llvm_eh_typeid_for =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for);
-
-  // The results of llvm_eh_typeid_for aren't reliable --- at least
-  // not locally --- so we basically have to do this as an 'if' chain.
-  // We walk through the first N-1 catch clauses, testing and chaining,
-  // and then fall into the final clause (which is either a cleanup, a
-  // filter (possibly with a cleanup), a catch-all, or another catch).
-  for (unsigned I = 2; I != LastToEmitInLoop; ++I) {
-    llvm::Value *Type = EHSelector[I];
-    UnwindDest Dest = EHHandlers[Type];
-    assert(Dest.isValid() && "no handler entry for value in selector?");
-
-    // Figure out where to branch on a match.  As a debug code-size
-    // optimization, if the scope depth matches the innermost cleanup,
-    // we branch directly to the catch handler.
-    llvm::BasicBlock *Match = Dest.getBlock();
-    bool MatchNeedsCleanup =
-      Dest.getScopeDepth() != EHStack.getInnermostEHCleanup();
-    if (MatchNeedsCleanup)
-      Match = createBasicBlock("eh.match");
-
-    llvm::BasicBlock *Next = createBasicBlock("eh.next");
-
-    // Check whether the exception matches.
-    llvm::CallInst *Id
-      = Builder.CreateCall(llvm_eh_typeid_for,
-                           Builder.CreateBitCast(Type, Int8PtrTy));
-    Id->setDoesNotThrow();
-    Builder.CreateCondBr(Builder.CreateICmpEQ(Selection, Id),
-                         Match, Next);
-    
-    // Emit match code if necessary.
-    if (MatchNeedsCleanup) {
-      EmitBlock(Match);
-      EmitBranchThroughEHCleanup(Dest);
-    }
-
-    // Continue to the next match.
-    EmitBlock(Next);
-  }
-
-  // Emit the final case in the selector.
-  // This might be a catch-all....
-  if (CatchAll.isValid()) {
-    assert(isa<llvm::ConstantPointerNull>(EHSelector.back()));
-    EmitBranchThroughEHCleanup(CatchAll);
-
-  // ...or an EH filter...
-  } else if (HasEHFilter) {
-    llvm::Value *SavedSelection = Selection;
-
-    // First, unwind out to the outermost scope if necessary.
-    if (EHStack.hasEHCleanups()) {
-      // The end here might not dominate the beginning, so we might need to
-      // save the selector if we need it.
-      llvm::AllocaInst *SelectorVar = 0;
-      if (HasEHCleanup) {
-        SelectorVar = CreateTempAlloca(Builder.getInt32Ty(), "selector.var");
-        Builder.CreateStore(Selection, SelectorVar);
-      }
-
-      llvm::BasicBlock *CleanupContBB = createBasicBlock("ehspec.cleanup.cont");
-      EmitBranchThroughEHCleanup(UnwindDest(CleanupContBB, EHStack.stable_end(),
-                                            EHStack.getNextEHDestIndex()));
-      EmitBlock(CleanupContBB);
-
-      if (HasEHCleanup)
-        SavedSelection = Builder.CreateLoad(SelectorVar, "ehspec.saved-selector");
-    }
-
-    // If there was a cleanup, we'll need to actually check whether we
-    // landed here because the filter triggered.
-    if (CleanupHackLevel != CHL_Ideal || HasEHCleanup) {
-      llvm::BasicBlock *UnexpectedBB = createBasicBlock("ehspec.unexpected");
-
-      llvm::Constant *Zero = llvm::ConstantInt::get(Int32Ty, 0);
-      llvm::Value *FailsFilter =
-        Builder.CreateICmpSLT(SavedSelection, Zero, "ehspec.fails");
-      Builder.CreateCondBr(FailsFilter, UnexpectedBB, getRethrowDest().getBlock());
-
-      EmitBlock(UnexpectedBB);
-    }
-
-    // Call __cxa_call_unexpected.  This doesn't need to be an invoke
-    // because __cxa_call_unexpected magically filters exceptions
-    // according to the last landing pad the exception was thrown
-    // into.  Seriously.
-    Builder.CreateCall(getUnexpectedFn(*this),
-                       Builder.CreateLoad(getExceptionSlot()))
-      ->setDoesNotReturn();
-    Builder.CreateUnreachable();
-
-  // ...or a normal catch handler...
-  } else if (CleanupHackLevel == CHL_Ideal && !HasEHCleanup) {
-    llvm::Value *Type = EHSelector.back();
-    EmitBranchThroughEHCleanup(EHHandlers[Type]);
-
-  // ...or a cleanup.
-  } else {
-    EmitBranchThroughEHCleanup(getRethrowDest());
-  }
+  Builder.CreateBr(getEHDispatchBlock(EHStack.getInnermostEHScope()));
 
   // Restore the old IR generation state.
-  Builder.restoreIP(SavedIP);
+  Builder.restoreIP(savedIP);
 
-  return LP;
+  return lpad;
 }
 
 namespace {
@@ -954,11 +907,11 @@ static void InitCatchParam(CodeGenFunction &CGF,
                            const VarDecl &CatchParam,
                            llvm::Value *ParamAddr) {
   // Load the exception from where the landing pad saved it.
-  llvm::Value *Exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot(), "exn");
+  llvm::Value *Exn = CGF.getExceptionFromSlot();
 
   CanQualType CatchType =
     CGF.CGM.getContext().getCanonicalType(CatchParam.getType());
-  const llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType);
+  llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType);
 
   // If we're catching by reference, we can just cast the object
   // pointer to the appropriate pointer.
@@ -1001,7 +954,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
       // pad.  The best solution is to fix the personality function.
       } else {
         // Pull the pointer for the reference type off.
-        const llvm::Type *PtrTy =
+        llvm::Type *PtrTy =
           cast<llvm::PointerType>(LLVMCatchTy)->getElementType();
 
         // Create the temporary and write the adjusted pointer into it.
@@ -1037,7 +990,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
 
     // Otherwise, it returns a pointer into the exception object.
 
-    const llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
+    llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
     llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
 
     if (IsComplex) {
@@ -1055,7 +1008,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
 
   assert(isa<RecordType>(CatchType) && "unexpected catch type!");
 
-  const llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
+  llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
 
   // Check for a copy expression.  If we don't have a copy expression,
   // that means a trivial copy is okay.
@@ -1086,8 +1039,10 @@ static void InitCatchParam(CodeGenFunction &CGF,
   CGF.EHStack.pushTerminate();
 
   // Perform the copy construction.
-  CGF.EmitAggExpr(copyExpr, AggValueSlot::forAddr(ParamAddr, Qualifiers(), 
-                                                  false));
+  CGF.EmitAggExpr(copyExpr, AggValueSlot::forAddr(ParamAddr, Qualifiers(),
+                                                  AggValueSlot::IsNotDestructed,
+                                          AggValueSlot::DoesNotNeedGCBarriers,
+                                                  AggValueSlot::IsNotAliased));
 
   // Leave the terminate scope.
   CGF.EHStack.popTerminate();
@@ -1127,7 +1082,7 @@ static void BeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *S) {
 
   VarDecl *CatchParam = S->getExceptionDecl();
   if (!CatchParam) {
-    llvm::Value *Exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot(), "exn");
+    llvm::Value *Exn = CGF.getExceptionFromSlot();
     CallBeginCatch(CGF, Exn, true);
     return;
   }
@@ -1146,16 +1101,112 @@ namespace {
   };
 }
 
+/// Emit the structure of the dispatch block for the given catch scope.
+/// It is an invariant that the dispatch block already exists.
+static void emitCatchDispatchBlock(CodeGenFunction &CGF,
+                                   EHCatchScope &catchScope) {
+  llvm::BasicBlock *dispatchBlock = catchScope.getCachedEHDispatchBlock();
+  assert(dispatchBlock);
+
+  // If there's only a single catch-all, getEHDispatchBlock returned
+  // that catch-all as the dispatch block.
+  if (catchScope.getNumHandlers() == 1 &&
+      catchScope.getHandler(0).isCatchAll()) {
+    assert(dispatchBlock == catchScope.getHandler(0).Block);
+    return;
+  }
+
+  CGBuilderTy::InsertPoint savedIP = CGF.Builder.saveIP();
+  CGF.EmitBlockAfterUses(dispatchBlock);
+
+  // Select the right handler.
+  llvm::Value *llvm_eh_typeid_for =
+    CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for);
+
+  // Load the selector value.
+  llvm::Value *selector = CGF.getSelectorFromSlot();
+
+  // Test against each of the exception types we claim to catch.
+  for (unsigned i = 0, e = catchScope.getNumHandlers(); ; ++i) {
+    assert(i < e && "ran off end of handlers!");
+    const EHCatchScope::Handler &handler = catchScope.getHandler(i);
+
+    llvm::Value *typeValue = handler.Type;
+    assert(typeValue && "fell into catch-all case!");
+    typeValue = CGF.Builder.CreateBitCast(typeValue, CGF.Int8PtrTy);
+
+    // Figure out the next block.
+    bool nextIsEnd;
+    llvm::BasicBlock *nextBlock;
+
+    // If this is the last handler, we're at the end, and the next
+    // block is the block for the enclosing EH scope.
+    if (i + 1 == e) {
+      nextBlock = CGF.getEHDispatchBlock(catchScope.getEnclosingEHScope());
+      nextIsEnd = true;
+
+    // If the next handler is a catch-all, we're at the end, and the
+    // next block is that handler.
+    } else if (catchScope.getHandler(i+1).isCatchAll()) {
+      nextBlock = catchScope.getHandler(i+1).Block;
+      nextIsEnd = true;
+
+    // Otherwise, we're not at the end and we need a new block.
+    } else {
+      nextBlock = CGF.createBasicBlock("catch.fallthrough");
+      nextIsEnd = false;
+    }
+
+    // Figure out the catch type's index in the LSDA's type table.
+    llvm::CallInst *typeIndex =
+      CGF.Builder.CreateCall(llvm_eh_typeid_for, typeValue);
+    typeIndex->setDoesNotThrow();
+
+    llvm::Value *matchesTypeIndex =
+      CGF.Builder.CreateICmpEQ(selector, typeIndex, "matches");
+    CGF.Builder.CreateCondBr(matchesTypeIndex, handler.Block, nextBlock);
+
+    // If the next handler is a catch-all, we're completely done.
+    if (nextIsEnd) {
+      CGF.Builder.restoreIP(savedIP);
+      return;
+
+    // Otherwise we need to emit and continue at that block.
+    } else {
+      CGF.EmitBlock(nextBlock);
+    }
+  }
+
+  llvm_unreachable("fell out of loop!");
+}
+
+void CodeGenFunction::popCatchScope() {
+  EHCatchScope &catchScope = cast<EHCatchScope>(*EHStack.begin());
+  if (catchScope.hasEHBranches())
+    emitCatchDispatchBlock(*this, catchScope);
+  EHStack.popCatch();
+}
+
 void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
   unsigned NumHandlers = S.getNumHandlers();
   EHCatchScope &CatchScope = cast<EHCatchScope>(*EHStack.begin());
   assert(CatchScope.getNumHandlers() == NumHandlers);
 
+  // If the catch was not required, bail out now.
+  if (!CatchScope.hasEHBranches()) {
+    EHStack.popCatch();
+    return;
+  }
+
+  // Emit the structure of the EH dispatch for this catch.
+  emitCatchDispatchBlock(*this, CatchScope);
+
   // Copy the handler blocks off before we pop the EH stack.  Emitting
   // the handlers might scribble on this memory.
-  llvm::SmallVector<EHCatchScope::Handler, 8> Handlers(NumHandlers);
+  SmallVector<EHCatchScope::Handler, 8> Handlers(NumHandlers);
   memcpy(Handlers.data(), CatchScope.begin(),
          NumHandlers * sizeof(EHCatchScope::Handler));
+
   EHStack.popCatch();
 
   // The fall-through block.
@@ -1171,12 +1222,19 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
     ImplicitRethrow = isa<CXXDestructorDecl>(CurCodeDecl) ||
                       isa<CXXConstructorDecl>(CurCodeDecl);
 
-  for (unsigned I = 0; I != NumHandlers; ++I) {
-    llvm::BasicBlock *CatchBlock = Handlers[I].Block;
-    EmitBlock(CatchBlock);
+  // Perversely, we emit the handlers backwards precisely because we
+  // want them to appear in source order.  In all of these cases, the
+  // catch block will have exactly one predecessor, which will be a
+  // particular block in the catch dispatch.  However, in the case of
+  // a catch-all, one of the dispatch blocks will branch to two
+  // different handlers, and EmitBlockAfterUses will cause the second
+  // handler to be moved before the first.
+  for (unsigned I = NumHandlers; I != 0; --I) {
+    llvm::BasicBlock *CatchBlock = Handlers[I-1].Block;
+    EmitBlockAfterUses(CatchBlock);
 
     // Catch the exception if this isn't a catch-all.
-    const CXXCatchStmt *C = S.getHandler(I);
+    const CXXCatchStmt *C = S.getHandler(I-1);
 
     // Enter a cleanup scope, including the catch variable and the
     // end-catch.
@@ -1315,7 +1373,7 @@ void CodeGenFunction::FinallyInfo::enter(CodeGenFunction &CGF,
   // In the latter case we need to pass it the exception object.
   // But we can't use the exception slot because the @finally might
   // have a landing pad (which would overwrite the exception slot).
-  const llvm::FunctionType *rethrowFnTy =
+  llvm::FunctionType *rethrowFnTy =
     cast<llvm::FunctionType>(
       cast<llvm::PointerType>(rethrowFn->getType())->getElementType());
   SavedExnVar = 0;
@@ -1358,7 +1416,8 @@ void CodeGenFunction::FinallyInfo::exit(CodeGenFunction &CGF) {
   // Leave the finally catch-all.
   EHCatchScope &catchScope = cast<EHCatchScope>(*CGF.EHStack.begin());
   llvm::BasicBlock *catchBB = catchScope.getHandler(0).Block;
-  CGF.EHStack.popCatch();
+
+  CGF.popCatchScope();
 
   // If there are any references to the catch-all block, emit it.
   if (catchBB->use_empty()) {
@@ -1371,13 +1430,13 @@ void CodeGenFunction::FinallyInfo::exit(CodeGenFunction &CGF) {
 
     // If there's a begin-catch function, call it.
     if (BeginCatchFn) {
-      exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot());
+      exn = CGF.getExceptionFromSlot();
       CGF.Builder.CreateCall(BeginCatchFn, exn)->setDoesNotThrow();
     }
 
     // If we need to remember the exception pointer to rethrow later, do so.
     if (SavedExnVar) {
-      if (!exn) exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot());
+      if (!exn) exn = CGF.getExceptionFromSlot();
       CGF.Builder.CreateStore(exn, SavedExnVar);
     }
 
@@ -1405,19 +1464,11 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
   Builder.SetInsertPoint(TerminateLandingPad);
 
   // Tell the backend that this is a landing pad.
-  llvm::CallInst *Exn =
-    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_exception), "exn");
-  Exn->setDoesNotThrow();
-
   const EHPersonality &Personality = EHPersonality::get(CGM.getLangOptions());
-  
-  // Tell the backend what the exception table should be:
-  // nothing but a catch-all.
-  llvm::Value *Args[3] = { Exn, getOpaquePersonalityFn(CGM, Personality),
-                           getCatchAllValue(*this) };
-  Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_selector),
-                     Args, "eh.selector")
-    ->setDoesNotThrow();
+  llvm::LandingPadInst *LPadInst =
+    Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, NULL),
+                             getOpaquePersonalityFn(CGM, Personality), 0);
+  LPadInst->addClause(getCatchAllValue(*this));
 
   llvm::CallInst *TerminateCall = Builder.CreateCall(getTerminateFn(*this));
   TerminateCall->setDoesNotReturn();
@@ -1451,26 +1502,26 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() {
   return TerminateHandler;
 }
 
-CodeGenFunction::UnwindDest CodeGenFunction::getRethrowDest() {
-  if (RethrowBlock.isValid()) return RethrowBlock;
+llvm::BasicBlock *CodeGenFunction::getEHResumeBlock() {
+  if (EHResumeBlock) return EHResumeBlock;
 
   CGBuilderTy::InsertPoint SavedIP = Builder.saveIP();
 
   // We emit a jump to a notional label at the outermost unwind state.
-  llvm::BasicBlock *Unwind = createBasicBlock("eh.resume");
-  Builder.SetInsertPoint(Unwind);
+  EHResumeBlock = createBasicBlock("eh.resume");
+  Builder.SetInsertPoint(EHResumeBlock);
 
   const EHPersonality &Personality = EHPersonality::get(CGM.getLangOptions());
 
   // This can always be a call because we necessarily didn't find
   // anything on the EH stack which needs our help.
-  llvm::StringRef RethrowName = Personality.getCatchallRethrowFnName();
+  StringRef RethrowName = Personality.getCatchallRethrowFnName();
   if (!RethrowName.empty()) {
     Builder.CreateCall(getCatchallRethrowFn(*this, RethrowName),
-                       Builder.CreateLoad(getExceptionSlot()))
+                       getExceptionFromSlot())
       ->setDoesNotReturn();
   } else {
-    llvm::Value *Exn = Builder.CreateLoad(getExceptionSlot());
+    llvm::Value *Exn = getExceptionFromSlot();
 
     switch (CleanupHackLevel) {
     case CHL_MandatoryCatchall:
@@ -1481,12 +1532,21 @@ CodeGenFunction::UnwindDest CodeGenFunction::getRethrowDest() {
         ->setDoesNotReturn();
       break;
     case CHL_MandatoryCleanup: {
-      // In mandatory-cleanup mode, we should use llvm.eh.resume.
-      llvm::Value *Selector = Builder.CreateLoad(getEHSelectorSlot());
-      Builder.CreateCall2(CGM.getIntrinsic(llvm::Intrinsic::eh_resume),
-                          Exn, Selector)
-        ->setDoesNotReturn();
-      break;
+      // In mandatory-cleanup mode, we should use 'resume'.
+
+      // Recreate the landingpad's return value for the 'resume' instruction.
+      llvm::Value *Exn = getExceptionFromSlot();
+      llvm::Value *Sel = getSelectorFromSlot();
+
+      llvm::Type *LPadType = llvm::StructType::get(Exn->getType(),
+                                                   Sel->getType(), NULL);
+      llvm::Value *LPadVal = llvm::UndefValue::get(LPadType);
+      LPadVal = Builder.CreateInsertValue(LPadVal, Exn, 0, "lpad.val");
+      LPadVal = Builder.CreateInsertValue(LPadVal, Sel, 1, "lpad.val");
+
+      Builder.CreateResume(LPadVal);
+      Builder.restoreIP(SavedIP);
+      return EHResumeBlock;
     }
     case CHL_Ideal:
       // In an idealized mode where we don't have to worry about the
@@ -1502,7 +1562,5 @@ CodeGenFunction::UnwindDest CodeGenFunction::getRethrowDest() {
 
   Builder.restoreIP(SavedIP);
 
-  RethrowBlock = UnwindDest(Unwind, EHStack.stable_end(), 0);
-  return RethrowBlock;
+  return EHResumeBlock;
 }
-
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGException.h b/contrib/llvm/tools/clang/lib/CodeGen/CGException.h
index 5a743b51f66f..d0216160d50f 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGException.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGException.h
@@ -24,15 +24,15 @@ namespace CodeGen {
 
 /// The exceptions personality for a function.  When 
 class EHPersonality {
-  llvm::StringRef PersonalityFn;
+  StringRef PersonalityFn;
 
   // If this is non-null, this personality requires a non-standard
   // function for rethrowing an exception after a catchall cleanup.
   // This function must have prototype void(void*).
-  llvm::StringRef CatchallRethrowFn;
+  StringRef CatchallRethrowFn;
 
-  EHPersonality(llvm::StringRef PersonalityFn,
-                llvm::StringRef CatchallRethrowFn = llvm::StringRef())
+  EHPersonality(StringRef PersonalityFn,
+                StringRef CatchallRethrowFn = StringRef())
     : PersonalityFn(PersonalityFn),
       CatchallRethrowFn(CatchallRethrowFn) {}
 
@@ -46,8 +46,8 @@ class EHPersonality {
   static const EHPersonality GNU_CPlusPlus;
   static const EHPersonality GNU_CPlusPlus_SJLJ;
 
-  llvm::StringRef getPersonalityFnName() const { return PersonalityFn; }
-  llvm::StringRef getCatchallRethrowFnName() const { return CatchallRethrowFn; }
+  StringRef getPersonalityFnName() const { return PersonalityFn; }
+  StringRef getCatchallRethrowFnName() const { return CatchallRethrowFn; }
 };
 
 }
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp
index a7e8003eaab5..bd4e553991b3 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp
@@ -18,6 +18,7 @@
 #include "CGDebugInfo.h"
 #include "CGRecordLayout.h"
 #include "CGObjCRuntime.h"
+#include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/Frontend/CodeGenOptions.h"
@@ -34,7 +35,7 @@ llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
   unsigned addressSpace =
     cast<llvm::PointerType>(value->getType())->getAddressSpace();
 
-  const llvm::PointerType *destType = Int8PtrTy;
+  llvm::PointerType *destType = Int8PtrTy;
   if (addressSpace)
     destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
 
@@ -44,8 +45,8 @@ llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
 
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
 /// block.
-llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(const llvm::Type *Ty,
-                                                    const llvm::Twine &Name) {
+llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
+                                                    const Twine &Name) {
   if (!Builder.isNamePreserving())
     return new llvm::AllocaInst(Ty, 0, "", AllocaInsertPt);
   return new llvm::AllocaInst(Ty, 0, Name, AllocaInsertPt);
@@ -59,7 +60,7 @@ void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var,
 }
 
 llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
-                                                const llvm::Twine &Name) {
+                                                const Twine &Name) {
   llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name);
   // FIXME: Should we prefer the preferred type alignment here?
   CharUnits Align = getContext().getTypeAlignInChars(Ty);
@@ -68,7 +69,7 @@ llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
 }
 
 llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
-                                                 const llvm::Twine &Name) {
+                                                 const Twine &Name) {
   llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
   // FIXME: Should we prefer the preferred type alignment here?
   CharUnits Align = getContext().getTypeAlignInChars(Ty);
@@ -136,7 +137,10 @@ void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
   if (E->getType()->isAnyComplexType())
     EmitComplexExprIntoAddr(E, Location, Quals.hasVolatile());
   else if (hasAggregateLLVMType(E->getType()))
-    EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals, IsInit));
+    EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
+                                         AggValueSlot::IsDestructed_t(IsInit),
+                                         AggValueSlot::DoesNotNeedGCBarriers,
+                                         AggValueSlot::IsAliased_t(!IsInit)));
   else {
     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
     LValue LV = MakeAddrLValue(Location, E->getType());
@@ -174,7 +178,7 @@ namespace {
 }
 
 static llvm::Value *
-CreateReferenceTemporary(CodeGenFunction& CGF, QualType Type,
+CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
                          const NamedDecl *InitializedDecl) {
   if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
     if (VD->hasGlobalStorage()) {
@@ -183,7 +187,7 @@ CreateReferenceTemporary(CodeGenFunction& CGF, QualType Type,
       CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
       Out.flush();
 
-      const llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
+      llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
   
       // Create the reference temporary.
       llvm::GlobalValue *RefTemp =
@@ -310,7 +314,7 @@ EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
       return ReferenceTemporary;
     }
     
-    llvm::SmallVector<SubobjectAdjustment, 2> Adjustments;
+    SmallVector<SubobjectAdjustment, 2> Adjustments;
     while (true) {
       E = E->IgnoreParens();
 
@@ -354,8 +358,12 @@ EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
         !E->getType()->isAnyComplexType()) {
       ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(), 
                                                     InitializedDecl);
+      AggValueSlot::IsDestructed_t isDestructed
+        = AggValueSlot::IsDestructed_t(InitializedDecl != 0);
       AggSlot = AggValueSlot::forAddr(ReferenceTemporary, Qualifiers(),
-                                      InitializedDecl != 0);
+                                      isDestructed,
+                                      AggValueSlot::DoesNotNeedGCBarriers,
+                                      AggValueSlot::IsNotAliased);
     }
     
     if (InitializedDecl) {
@@ -466,7 +474,8 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
   else {
     switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
     case Qualifiers::OCL_None:
-      assert(0 && "Not a reference temporary that needs to be deallocated");
+      llvm_unreachable(
+                      "Not a reference temporary that needs to be deallocated");
     case Qualifiers::OCL_ExplicitNone:
     case Qualifiers::OCL_Autoreleasing:
       // Nothing to do.
@@ -578,7 +587,7 @@ RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
     return RValue::get(0);
   
   if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
-    const llvm::Type *EltTy = ConvertType(CTy->getElementType());
+    llvm::Type *EltTy = ConvertType(CTy->getElementType());
     llvm::Value *U = llvm::UndefValue::get(EltTy);
     return RValue::getComplex(std::make_pair(U, U));
   }
@@ -652,7 +661,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     return EmitVAArgExprLValue(cast<VAArgExpr>(E));
   case Expr::DeclRefExprClass:
     return EmitDeclRefLValue(cast<DeclRefExpr>(E));
-  case Expr::ParenExprClass:return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
+  case Expr::ParenExprClass:
+    return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
   case Expr::GenericSelectionExprClass:
     return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
   case Expr::PredefinedExprClass:
@@ -731,7 +741,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue) {
 llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
                                               unsigned Alignment, QualType Ty,
                                               llvm::MDNode *TBAAInfo) {
-  llvm::LoadInst *Load = Builder.CreateLoad(Addr, "tmp");
+  llvm::LoadInst *Load = Builder.CreateLoad(Addr);
   if (Volatile)
     Load->setVolatile(true);
   if (Alignment)
@@ -812,7 +822,7 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV) {
 
   if (LV.isVectorElt()) {
     llvm::Value *Vec = Builder.CreateLoad(LV.getVectorAddr(),
-                                          LV.isVolatileQualified(), "tmp");
+                                          LV.isVolatileQualified());
     return RValue::get(Builder.CreateExtractElement(Vec, LV.getVectorIdx(),
                                                     "vecext"));
   }
@@ -833,7 +843,7 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
   const CGBitFieldInfo &Info = LV.getBitFieldInfo();
 
   // Get the output type.
-  const llvm::Type *ResLTy = ConvertType(LV.getType());
+  llvm::Type *ResLTy = ConvertType(LV.getType());
   unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
 
   // Compute the result as an OR of all of the individual component accesses.
@@ -857,7 +867,7 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
     }
 
     // Cast to the access type.
-    const llvm::Type *PTy = llvm::Type::getIntNPtrTy(getLLVMContext(),
+    llvm::Type *PTy = llvm::Type::getIntNPtrTy(getLLVMContext(),
                                                      AI.AccessWidth,
                        CGM.getContext().getTargetAddressSpace(LV.getType()));
     Ptr = Builder.CreateBitCast(Ptr, PTy);
@@ -905,7 +915,7 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
 // appropriate shufflevector.
 RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
   llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddr(),
-                                        LV.isVolatileQualified(), "tmp");
+                                        LV.isVolatileQualified());
 
   const llvm::Constant *Elts = LV.getExtVectorElts();
 
@@ -915,13 +925,13 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
   if (!ExprVT) {
     unsigned InIdx = getAccessedFieldNo(0, Elts);
     llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
-    return RValue::get(Builder.CreateExtractElement(Vec, Elt, "tmp"));
+    return RValue::get(Builder.CreateExtractElement(Vec, Elt));
   }
 
   // Always use shuffle vector to try to retain the original program structure
   unsigned NumResultElts = ExprVT->getNumElements();
 
-  llvm::SmallVector<llvm::Constant*, 4> Mask;
+  SmallVector<llvm::Constant*, 4> Mask;
   for (unsigned i = 0; i != NumResultElts; ++i) {
     unsigned InIdx = getAccessedFieldNo(i, Elts);
     Mask.push_back(llvm::ConstantInt::get(Int32Ty, InIdx));
@@ -929,7 +939,7 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
 
   llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
   Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()),
-                                    MaskV, "tmp");
+                                    MaskV);
   return RValue::get(Vec);
 }
 
@@ -943,7 +953,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst) {
     if (Dst.isVectorElt()) {
       // Read/modify/write the vector, inserting the new element.
       llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddr(),
-                                            Dst.isVolatileQualified(), "tmp");
+                                            Dst.isVolatileQualified());
       Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
                                         Dst.getVectorIdx(), "vecins");
       Builder.CreateStore(Vec, Dst.getVectorAddr(),Dst.isVolatileQualified());
@@ -1002,7 +1012,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst) {
     llvm::Value *src = Src.getScalarVal();
     if (Dst.isObjCIvar()) {
       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
-      const llvm::Type *ResultType = ConvertType(getContext().LongTy);
+      llvm::Type *ResultType = ConvertType(getContext().LongTy);
       llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp());
       llvm::Value *dst = RHS;
       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
@@ -1029,7 +1039,7 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
   const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
 
   // Get the output type.
-  const llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
+  llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
   unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
 
   // Get the source value, truncated to the width of the bit-field.
@@ -1045,7 +1055,7 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
   // Return the new value of the bit-field, if requested.
   if (Result) {
     // Cast back to the proper type for result.
-    const llvm::Type *SrcTy = Src.getScalarVal()->getType();
+    llvm::Type *SrcTy = Src.getScalarVal()->getType();
     llvm::Value *ReloadVal = Builder.CreateIntCast(SrcVal, SrcTy, false,
                                                    "bf.reload.val");
 
@@ -1082,10 +1092,10 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
     }
 
     // Cast to the access type.
-    const llvm::Type *AccessLTy =
+    llvm::Type *AccessLTy =
       llvm::Type::getIntNTy(getLLVMContext(), AI.AccessWidth);
 
-    const llvm::Type *PTy = AccessLTy->getPointerTo(addressSpace);
+    llvm::Type *PTy = AccessLTy->getPointerTo(addressSpace);
     Ptr = Builder.CreateBitCast(Ptr, PTy);
 
     // Extract the piece of the bit-field value to write in this access, limited
@@ -1134,7 +1144,7 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
   // This access turns into a read/modify/write of the vector.  Load the input
   // value now.
   llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddr(),
-                                        Dst.isVolatileQualified(), "tmp");
+                                        Dst.isVolatileQualified());
   const llvm::Constant *Elts = Dst.getExtVectorElts();
 
   llvm::Value *SrcVal = Src.getScalarVal();
@@ -1147,7 +1157,7 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
       // Use shuffle vector is the src and destination are the same number of
       // elements and restore the vector mask since it is on the side it will be
       // stored.
-      llvm::SmallVector<llvm::Constant*, 4> Mask(NumDstElts);
+      SmallVector<llvm::Constant*, 4> Mask(NumDstElts);
       for (unsigned i = 0; i != NumSrcElts; ++i) {
         unsigned InIdx = getAccessedFieldNo(i, Elts);
         Mask[InIdx] = llvm::ConstantInt::get(Int32Ty, i);
@@ -1156,13 +1166,13 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
       llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
       Vec = Builder.CreateShuffleVector(SrcVal,
                                         llvm::UndefValue::get(Vec->getType()),
-                                        MaskV, "tmp");
+                                        MaskV);
     } else if (NumDstElts > NumSrcElts) {
       // Extended the source vector to the same length and then shuffle it
       // into the destination.
       // FIXME: since we're shuffling with undef, can we just use the indices
       //        into that?  This could be simpler.
-      llvm::SmallVector<llvm::Constant*, 4> ExtMask;
+      SmallVector<llvm::Constant*, 4> ExtMask;
       unsigned i;
       for (i = 0; i != NumSrcElts; ++i)
         ExtMask.push_back(llvm::ConstantInt::get(Int32Ty, i));
@@ -1172,9 +1182,9 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
       llvm::Value *ExtSrcVal =
         Builder.CreateShuffleVector(SrcVal,
                                     llvm::UndefValue::get(SrcVal->getType()),
-                                    ExtMaskV, "tmp");
+                                    ExtMaskV);
       // build identity
-      llvm::SmallVector<llvm::Constant*, 4> Mask;
+      SmallVector<llvm::Constant*, 4> Mask;
       for (unsigned i = 0; i != NumDstElts; ++i)
         Mask.push_back(llvm::ConstantInt::get(Int32Ty, i));
 
@@ -1184,16 +1194,16 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
         Mask[Idx] = llvm::ConstantInt::get(Int32Ty, i+NumDstElts);
       }
       llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
-      Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV, "tmp");
+      Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV);
     } else {
       // We should never shorten the vector
-      assert(0 && "unexpected shorten vector length");
+      llvm_unreachable("unexpected shorten vector length");
     }
   } else {
     // If the Src is a scalar (not a vector) it must be updating one element.
     unsigned InIdx = getAccessedFieldNo(0, Elts);
     llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
-    Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt, "tmp");
+    Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
   }
 
   Builder.CreateStore(Vec, Dst.getExtVectorAddr(), Dst.isVolatileQualified());
@@ -1203,11 +1213,23 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
 // generating write-barries API. It is currently a global, ivar,
 // or neither.
 static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
-                                 LValue &LV) {
-  if (Ctx.getLangOptions().getGCMode() == LangOptions::NonGC)
+                                 LValue &LV,
+                                 bool IsMemberAccess=false) {
+  if (Ctx.getLangOptions().getGC() == LangOptions::NonGC)
     return;
   
   if (isa<ObjCIvarRefExpr>(E)) {
+    QualType ExpTy = E->getType();
+    if (IsMemberAccess && ExpTy->isPointerType()) {
+      // If ivar is a structure pointer, assigning to field of
+      // this struct follows gcc's behavior and makes it a non-ivar 
+      // writer-barrier conservatively.
+      ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
+      if (ExpTy->isRecordType()) {
+        LV.setObjCIvar(false);
+        return;
+      }
+    }
     LV.setObjCIvar(true);
     ObjCIvarRefExpr *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr*>(E));
     LV.setBaseIvarExp(Exp->getBase());
@@ -1227,12 +1249,12 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
   }
   
   if (const UnaryOperator *Exp = dyn_cast<UnaryOperator>(E)) {
-    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV);
+    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
     return;
   }
   
   if (const ParenExpr *Exp = dyn_cast<ParenExpr>(E)) {
-    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV);
+    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
     if (LV.isObjCIvar()) {
       // If cast is to a structure pointer, follow gcc's behavior and make it
       // a non-ivar write-barrier.
@@ -1251,17 +1273,17 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
   }
 
   if (const ImplicitCastExpr *Exp = dyn_cast<ImplicitCastExpr>(E)) {
-    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV);
+    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
     return;
   }
   
   if (const CStyleCastExpr *Exp = dyn_cast<CStyleCastExpr>(E)) {
-    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV);
+    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
     return;
   }
 
   if (const ObjCBridgedCastExpr *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
-    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV);
+    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
     return;
   }
 
@@ -1277,9 +1299,9 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
       LV.setGlobalObjCRef(false);
     return;
   }
-  
+
   if (const MemberExpr *Exp = dyn_cast<MemberExpr>(E)) {
-    setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
+    setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
     // We don't know if member is an 'ivar', but this flag is looked at
     // only in the context of LV.isObjCIvar().
     LV.setObjCArray(E->getType()->isArrayType());
@@ -1290,7 +1312,7 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
 static llvm::Value *
 EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
                                 llvm::Value *V, llvm::Type *IRType,
-                                llvm::StringRef Name = llvm::StringRef()) {
+                                StringRef Name = StringRef()) {
   unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
   return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
 }
@@ -1302,7 +1324,7 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
 
   llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
   if (VD->getType()->isReferenceType())
-    V = CGF.Builder.CreateLoad(V, "tmp");
+    V = CGF.Builder.CreateLoad(V);
   
   V = EmitBitCastOfLValueToProperType(CGF, V,
                                 CGF.getTypes().ConvertTypeForMem(E->getType()));
@@ -1325,7 +1347,7 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
       QualType NoProtoType =
           CGF.getContext().getFunctionNoProtoType(Proto->getResultType());
       NoProtoType = CGF.getContext().getPointerType(NoProtoType);
-      V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType), "tmp");
+      V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType));
     }
   }
   unsigned Alignment = CGF.getContext().getDeclAlign(FD).getQuantity();
@@ -1361,7 +1383,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
       V = BuildBlockByrefAddress(V, VD);
     
     if (VD->getType()->isReferenceType())
-      V = Builder.CreateLoad(V, "tmp");
+      V = Builder.CreateLoad(V);
 
     V = EmitBitCastOfLValueToProperType(*this, V,
                                     getTypes().ConvertTypeForMem(E->getType()));
@@ -1378,7 +1400,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
   if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(ND))
     return EmitFunctionDeclLValue(*this, E, fn);
 
-  assert(false && "Unhandled DeclRefExpr");
+  llvm_unreachable("Unhandled DeclRefExpr");
   
   // an invalid LValue, but the assert will
   // ensure that this point is never reached.
@@ -1398,7 +1420,7 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
 
   QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
   switch (E->getOpcode()) {
-  default: assert(0 && "Unknown unary operator lvalue!");
+  default: llvm_unreachable("Unknown unary operator lvalue!");
   case UO_Deref: {
     QualType T = E->getSubExpr()->getType()->getPointeeType();
     assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
@@ -1411,7 +1433,7 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
     // But, we continue to generate __strong write barrier on indirect write
     // into a pointer to object.
     if (getContext().getLangOptions().ObjC1 &&
-        getContext().getLangOptions().getGCMode() != LangOptions::NonGC &&
+        getContext().getLangOptions().getGC() != LangOptions::NonGC &&
         LV.isObjCWeak())
       LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
     return LV;
@@ -1474,7 +1496,7 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
     std::string GlobalVarName;
 
     switch (Type) {
-    default: assert(0 && "Invalid type");
+    default: llvm_unreachable("Invalid type");
     case PredefinedExpr::Func:
       GlobalVarName = "__func__.";
       break;
@@ -1486,7 +1508,7 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
       break;
     }
 
-    llvm::StringRef FnName = CurFn->getName();
+    StringRef FnName = CurFn->getName();
     if (FnName.startswith("\01"))
       FnName = FnName.substr(1);
     GlobalVarName += FnName;
@@ -1646,9 +1668,9 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
     ArrayAlignment = ArrayLV.getAlignment();
 
     if (getContext().getLangOptions().isSignedOverflowDefined())
-      Address = Builder.CreateGEP(ArrayPtr, Args, Args+2, "arrayidx");
+      Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
     else
-      Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, Args+2, "arrayidx");
+      Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
   } else {
     // The base must be a pointer, which is not an aggregate.  Emit it.
     llvm::Value *Base = EmitScalarExpr(E->getBase());
@@ -1672,7 +1694,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
   LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
 
   if (getContext().getLangOptions().ObjC1 &&
-      getContext().getLangOptions().getGCMode() != LangOptions::NonGC) {
+      getContext().getLangOptions().getGC() != LangOptions::NonGC) {
     LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
     setObjCGCLValueClass(getContext(), E, LV);
   }
@@ -1681,10 +1703,10 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
 
 static
 llvm::Constant *GenerateConstantVector(llvm::LLVMContext &VMContext,
-                                       llvm::SmallVector<unsigned, 4> &Elts) {
-  llvm::SmallVector<llvm::Constant*, 4> CElts;
+                                       SmallVector<unsigned, 4> &Elts) {
+  SmallVector<llvm::Constant*, 4> CElts;
 
-  const llvm::Type *Int32Ty = llvm::Type::getInt32Ty(VMContext);
+  llvm::Type *Int32Ty = llvm::Type::getInt32Ty(VMContext);
   for (unsigned i = 0, e = Elts.size(); i != e; ++i)
     CElts.push_back(llvm::ConstantInt::get(Int32Ty, Elts[i]));
 
@@ -1725,7 +1747,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
     E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
   
   // Encode the element access list into a vector of unsigned indices.
-  llvm::SmallVector<unsigned, 4> Indices;
+  SmallVector<unsigned, 4> Indices;
   E->getEncodedElementAccess(Indices);
 
   if (Base.isSimple()) {
@@ -1735,7 +1757,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
   assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
 
   llvm::Constant *BaseElts = Base.getExtVectorElts();
-  llvm::SmallVector<llvm::Constant *, 4> CElts;
+  SmallVector<llvm::Constant *, 4> CElts;
 
   for (unsigned i = 0, e = Indices.size(); i != e; ++i) {
     if (isa<llvm::ConstantAggregateZero>(BaseElts))
@@ -1784,8 +1806,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
     return EmitFunctionDeclLValue(*this, E, FD);
 
-  assert(false && "Unhandled member declaration!");
-  return LValue();
+  llvm_unreachable("Unhandled member declaration!");
 }
 
 LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value *BaseValue,
@@ -1867,6 +1888,9 @@ LValue CodeGenFunction::EmitLValueForField(llvm::Value *baseAddr,
                                          CGM.getTypes().ConvertTypeForMem(type),
                                          field->getName());
 
+  if (field->hasAttr<AnnotateAttr>())
+    addr = EmitFieldAnnotations(field, addr);
+
   unsigned alignment = getContext().getDeclAlign(field).getQuantity();
   LValue LV = MakeAddrLValue(addr, type, alignment);
   LV.getQuals().addCVRQualifiers(cvr);
@@ -1896,7 +1920,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value *BaseValue,
   const CGRecordLayout &RL =
     CGM.getTypes().getCGRecordLayout(Field->getParent());
   unsigned idx = RL.getLLVMFieldNo(Field);
-  llvm::Value *V = Builder.CreateStructGEP(BaseValue, idx, "tmp");
+  llvm::Value *V = Builder.CreateStructGEP(BaseValue, idx);
   assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
 
   
@@ -1904,7 +1928,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value *BaseValue,
   // for both unions and structs.  A union needs a bitcast, a struct element
   // will need a bitcast if the LLVM type laid out doesn't match the desired
   // type.
-  const llvm::Type *llvmType = ConvertTypeForMem(FieldType);
+  llvm::Type *llvmType = ConvertTypeForMem(FieldType);
   unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
   V = Builder.CreateBitCast(V, llvmType->getPointerTo(AS));
   
@@ -2048,9 +2072,10 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   case CK_BaseToDerivedMemberPointer:
   case CK_MemberPointerToBoolean:
   case CK_AnyPointerToBlockPointerCast:
-  case CK_ObjCProduceObject:
-  case CK_ObjCConsumeObject:
-  case CK_ObjCReclaimReturnedObject: {
+  case CK_ARCProduceObject:
+  case CK_ARCConsumeObject:
+  case CK_ARCReclaimReturnedObject:
+  case CK_ARCExtendBlockObject: {
     // These casts only produce lvalues when we're binding a reference to a 
     // temporary realized from a (converted) pure rvalue. Emit the expression
     // as a value, copy it into a temporary, and return an lvalue referring to
@@ -2069,7 +2094,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
 
   case CK_ConstructorConversion:
   case CK_UserDefinedConversion:
-  case CK_AnyPointerToObjCPointerCast:
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
     return EmitLValue(E->getSubExpr());
   
   case CK_UncheckedDerivedToBase:
@@ -2143,8 +2169,7 @@ LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
 
 LValue CodeGenFunction::EmitMaterializeTemporaryExpr(
                                            const MaterializeTemporaryExpr *E) {
-  RValue RV = EmitReferenceBindingToExpr(E->GetTemporaryExpr(),
-                                         /*InitializedDecl=*/0);
+  RValue RV = EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
   return MakeAddrLValue(RV.getScalarVal(), E->getType());
 }
 
@@ -2155,11 +2180,8 @@ LValue CodeGenFunction::EmitMaterializeTemporaryExpr(
 
 RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, 
                                      ReturnValueSlot ReturnValue) {
-  if (CGDebugInfo *DI = getDebugInfo()) {
-    DI->setLocation(E->getLocStart());
-    DI->UpdateLineDirectiveRegion(Builder);
-    DI->EmitStopPoint(Builder);
-  }
+  if (CGDebugInfo *DI = getDebugInfo())
+    DI->EmitLocation(Builder, E->getLocStart());
 
   // Builtins never have block type.
   if (E->getCallee()->getType()->isBlockPointerType())
@@ -2168,14 +2190,13 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
   if (const CXXMemberCallExpr *CE = dyn_cast<CXXMemberCallExpr>(E))
     return EmitCXXMemberCallExpr(CE, ReturnValue);
 
-  const Decl *TargetDecl = 0;
-  if (const ImplicitCastExpr *CE = dyn_cast<ImplicitCastExpr>(E->getCallee())) {
-    if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CE->getSubExpr())) {
-      TargetDecl = DRE->getDecl();
-      if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(TargetDecl))
-        if (unsigned builtinID = FD->getBuiltinID())
-          return EmitBuiltinExpr(FD, builtinID, E);
-    }
+  if (const CUDAKernelCallExpr *CE = dyn_cast<CUDAKernelCallExpr>(E))
+    return EmitCUDAKernelCallExpr(CE, ReturnValue);
+
+  const Decl *TargetDecl = E->getCalleeDecl();
+  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
+    if (unsigned builtinID = FD->getBuiltinID())
+      return EmitBuiltinExpr(FD, builtinID, E);
   }
 
   if (const CXXOperatorCallExpr *CE = dyn_cast<CXXOperatorCallExpr>(E))
@@ -2306,7 +2327,7 @@ LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
   assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
          && "binding l-value to type which needs a temporary");
-  AggValueSlot Slot = CreateAggTemp(E->getType(), "tmp");
+  AggValueSlot Slot = CreateAggTemp(E->getType());
   EmitCXXConstructExpr(E, Slot);
   return MakeAddrLValue(Slot.getAddr(), E->getType());
 }
@@ -2319,7 +2340,7 @@ CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
 LValue
 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
   AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
-  Slot.setLifetimeExternallyManaged();
+  Slot.setExternallyDestructed();
   EmitAggExpr(E->getSubExpr(), Slot);
   EmitCXXTemporary(E->getTemporary(), Slot.getAddr());
   return MakeAddrLValue(Slot.getAddr(), E->getType());
@@ -2406,8 +2427,35 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
   CallArgList Args;
   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
 
-  return EmitCall(CGM.getTypes().getFunctionInfo(Args, FnType),
-                  Callee, ReturnValue, Args, TargetDecl);
+  const CGFunctionInfo &FnInfo = CGM.getTypes().getFunctionInfo(Args, FnType);
+
+  // C99 6.5.2.2p6:
+  //   If the expression that denotes the called function has a type
+  //   that does not include a prototype, [the default argument
+  //   promotions are performed]. If the number of arguments does not
+  //   equal the number of parameters, the behavior is undefined. If
+  //   the function is defined with a type that includes a prototype,
+  //   and either the prototype ends with an ellipsis (, ...) or the
+  //   types of the arguments after promotion are not compatible with
+  //   the types of the parameters, the behavior is undefined. If the
+  //   function is defined with a type that does not include a
+  //   prototype, and the types of the arguments after promotion are
+  //   not compatible with those of the parameters after promotion,
+  //   the behavior is undefined [except in some trivial cases].
+  // That is, in the general case, we should assume that a call
+  // through an unprototyped function type works like a *non-variadic*
+  // call.  The way we make this work is to cast to the exact type
+  // of the promoted arguments.
+  if (isa<FunctionNoProtoType>(FnType) &&
+      !getTargetHooks().isNoProtoCallVariadic(FnType->getCallConv())) {
+    assert(cast<llvm::FunctionType>(Callee->getType()->getContainedType(0))
+             ->isVarArg());
+    llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo, false);
+    CalleeTy = CalleeTy->getPointerTo();
+    Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast");
+  }
+
+  return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl);
 }
 
 LValue CodeGenFunction::
@@ -2428,3 +2476,279 @@ EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
 
   return MakeAddrLValue(AddV, MPT->getPointeeType());
 }
+
+static void
+EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
+             llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
+             uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
+  if (E->isCmpXChg()) {
+    // Note that cmpxchg only supports specifying one ordering and
+    // doesn't support weak cmpxchg, at least at the moment.
+    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
+    LoadVal1->setAlignment(Align);
+    llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
+    LoadVal2->setAlignment(Align);
+    llvm::AtomicCmpXchgInst *CXI =
+        CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
+    CXI->setVolatile(E->isVolatile());
+    llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
+    StoreVal1->setAlignment(Align);
+    llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
+    CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
+    return;
+  }
+
+  if (E->getOp() == AtomicExpr::Load) {
+    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
+    Load->setAtomic(Order);
+    Load->setAlignment(Size);
+    Load->setVolatile(E->isVolatile());
+    llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
+    StoreDest->setAlignment(Align);
+    return;
+  }
+
+  if (E->getOp() == AtomicExpr::Store) {
+    assert(!Dest && "Store does not return a value");
+    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
+    LoadVal1->setAlignment(Align);
+    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
+    Store->setAtomic(Order);
+    Store->setAlignment(Size);
+    Store->setVolatile(E->isVolatile());
+    return;
+  }
+
+  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
+  switch (E->getOp()) {
+    case AtomicExpr::CmpXchgWeak:
+    case AtomicExpr::CmpXchgStrong:
+    case AtomicExpr::Store:
+    case AtomicExpr::Load:  assert(0 && "Already handled!");
+    case AtomicExpr::Add:   Op = llvm::AtomicRMWInst::Add;  break;
+    case AtomicExpr::Sub:   Op = llvm::AtomicRMWInst::Sub;  break;
+    case AtomicExpr::And:   Op = llvm::AtomicRMWInst::And;  break;
+    case AtomicExpr::Or:    Op = llvm::AtomicRMWInst::Or;   break;
+    case AtomicExpr::Xor:   Op = llvm::AtomicRMWInst::Xor;  break;
+    case AtomicExpr::Xchg:  Op = llvm::AtomicRMWInst::Xchg; break;
+  }
+  llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
+  LoadVal1->setAlignment(Align);
+  llvm::AtomicRMWInst *RMWI =
+      CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
+  RMWI->setVolatile(E->isVolatile());
+  llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(RMWI, Dest);
+  StoreDest->setAlignment(Align);
+}
+
+// This function emits any expression (scalar, complex, or aggregate)
+// into a temporary alloca.
+static llvm::Value *
+EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
+  llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
+  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
+                       /*Init*/ true);
+  return DeclPtr;
+}
+
+static RValue ConvertTempToRValue(CodeGenFunction &CGF, QualType Ty,
+                                  llvm::Value *Dest) {
+  if (Ty->isAnyComplexType())
+    return RValue::getComplex(CGF.LoadComplexFromAddr(Dest, false));
+  if (CGF.hasAggregateLLVMType(Ty))
+    return RValue::getAggregate(Dest);
+  return RValue::get(CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(Dest, Ty)));
+}
+
+RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
+  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
+  QualType MemTy = AtomicTy->getAs<AtomicType>()->getValueType();
+  CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
+  uint64_t Size = sizeChars.getQuantity();
+  CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
+  unsigned Align = alignChars.getQuantity();
+  unsigned MaxInlineWidth =
+      getContext().getTargetInfo().getMaxAtomicInlineWidth();
+  bool UseLibcall = (Size != Align || Size > MaxInlineWidth);
+
+  llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
+  Ptr = EmitScalarExpr(E->getPtr());
+  Order = EmitScalarExpr(E->getOrder());
+  if (E->isCmpXChg()) {
+    Val1 = EmitScalarExpr(E->getVal1());
+    Val2 = EmitValToTemp(*this, E->getVal2());
+    OrderFail = EmitScalarExpr(E->getOrderFail());
+    (void)OrderFail; // OrderFail is unused at the moment
+  } else if ((E->getOp() == AtomicExpr::Add || E->getOp() == AtomicExpr::Sub) &&
+             MemTy->isPointerType()) {
+    // For pointers, we're required to do a bit of math: adding 1 to an int*
+    // is not the same as adding 1 to a uintptr_t.
+    QualType Val1Ty = E->getVal1()->getType();
+    llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
+    CharUnits PointeeIncAmt =
+        getContext().getTypeSizeInChars(MemTy->getPointeeType());
+    Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
+    Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
+    EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
+  } else if (E->getOp() != AtomicExpr::Load) {
+    Val1 = EmitValToTemp(*this, E->getVal1());
+  }
+
+  if (E->getOp() != AtomicExpr::Store && !Dest)
+    Dest = CreateMemTemp(E->getType(), ".atomicdst");
+
+  if (UseLibcall) {
+    // FIXME: Finalize what the libcalls are actually supposed to look like.
+    // See also http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
+    return EmitUnsupportedRValue(E, "atomic library call");
+  }
+#if 0
+  if (UseLibcall) {
+    const char* LibCallName;
+    switch (E->getOp()) {
+    case AtomicExpr::CmpXchgWeak:
+      LibCallName = "__atomic_compare_exchange_generic"; break;
+    case AtomicExpr::CmpXchgStrong:
+      LibCallName = "__atomic_compare_exchange_generic"; break;
+    case AtomicExpr::Add:   LibCallName = "__atomic_fetch_add_generic"; break;
+    case AtomicExpr::Sub:   LibCallName = "__atomic_fetch_sub_generic"; break;
+    case AtomicExpr::And:   LibCallName = "__atomic_fetch_and_generic"; break;
+    case AtomicExpr::Or:    LibCallName = "__atomic_fetch_or_generic"; break;
+    case AtomicExpr::Xor:   LibCallName = "__atomic_fetch_xor_generic"; break;
+    case AtomicExpr::Xchg:  LibCallName = "__atomic_exchange_generic"; break;
+    case AtomicExpr::Store: LibCallName = "__atomic_store_generic"; break;
+    case AtomicExpr::Load:  LibCallName = "__atomic_load_generic"; break;
+    }
+    llvm::SmallVector<QualType, 4> Params;
+    CallArgList Args;
+    QualType RetTy = getContext().VoidTy;
+    if (E->getOp() != AtomicExpr::Store && !E->isCmpXChg())
+      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
+               getContext().VoidPtrTy);
+    Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
+             getContext().VoidPtrTy);
+    if (E->getOp() != AtomicExpr::Load)
+      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
+               getContext().VoidPtrTy);
+    if (E->isCmpXChg()) {
+      Args.add(RValue::get(EmitCastToVoidPtr(Val2)),
+               getContext().VoidPtrTy);
+      RetTy = getContext().IntTy;
+    }
+    Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
+             getContext().getSizeType());
+    const CGFunctionInfo &FuncInfo =
+        CGM.getTypes().getFunctionInfo(RetTy, Args, FunctionType::ExtInfo());
+    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo, false);
+    llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
+    RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
+    if (E->isCmpXChg())
+      return Res;
+    if (E->getOp() == AtomicExpr::Store)
+      return RValue::get(0);
+    return ConvertTempToRValue(*this, E->getType(), Dest);
+  }
+#endif
+  llvm::Type *IPtrTy =
+      llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
+  llvm::Value *OrigDest = Dest;
+  Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
+  if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
+  if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
+  if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
+
+  if (isa<llvm::ConstantInt>(Order)) {
+    int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+    switch (ord) {
+    case 0:  // memory_order_relaxed
+      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                   llvm::Monotonic);
+      break;
+    case 1:  // memory_order_consume
+    case 2:  // memory_order_acquire
+      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                   llvm::Acquire);
+      break;
+    case 3:  // memory_order_release
+      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                   llvm::Release);
+      break;
+    case 4:  // memory_order_acq_rel
+      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                   llvm::AcquireRelease);
+      break;
+    case 5:  // memory_order_seq_cst
+      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                   llvm::SequentiallyConsistent);
+      break;
+    default: // invalid order
+      // We should not ever get here normally, but it's hard to
+      // enforce that in general.
+      break; 
+    }
+    if (E->getOp() == AtomicExpr::Store)
+      return RValue::get(0);
+    return ConvertTempToRValue(*this, E->getType(), OrigDest);
+  }
+
+  // Long case, when Order isn't obviously constant.
+
+  // Create all the relevant BB's
+  llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
+                   *AcqRelBB = 0, *SeqCstBB = 0;
+  MonotonicBB = createBasicBlock("monotonic", CurFn);
+  if (E->getOp() != AtomicExpr::Store)
+    AcquireBB = createBasicBlock("acquire", CurFn);
+  if (E->getOp() != AtomicExpr::Load)
+    ReleaseBB = createBasicBlock("release", CurFn);
+  if (E->getOp() != AtomicExpr::Load && E->getOp() != AtomicExpr::Store)
+    AcqRelBB = createBasicBlock("acqrel", CurFn);
+  SeqCstBB = createBasicBlock("seqcst", CurFn);
+  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
+
+  // Create the switch for the split
+  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
+  // doesn't matter unless someone is crazy enough to use something that
+  // doesn't fold to a constant for the ordering.
+  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
+  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
+
+  // Emit all the different atomics
+  Builder.SetInsertPoint(MonotonicBB);
+  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+               llvm::Monotonic);
+  Builder.CreateBr(ContBB);
+  if (E->getOp() != AtomicExpr::Store) {
+    Builder.SetInsertPoint(AcquireBB);
+    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                 llvm::Acquire);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(1), AcquireBB);
+    SI->addCase(Builder.getInt32(2), AcquireBB);
+  }
+  if (E->getOp() != AtomicExpr::Load) {
+    Builder.SetInsertPoint(ReleaseBB);
+    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                 llvm::Release);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(3), ReleaseBB);
+  }
+  if (E->getOp() != AtomicExpr::Load && E->getOp() != AtomicExpr::Store) {
+    Builder.SetInsertPoint(AcqRelBB);
+    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+                 llvm::AcquireRelease);
+    Builder.CreateBr(ContBB);
+    SI->addCase(Builder.getInt32(4), AcqRelBB);
+  }
+  Builder.SetInsertPoint(SeqCstBB);
+  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
+               llvm::SequentiallyConsistent);
+  Builder.CreateBr(ContBB);
+  SI->addCase(Builder.getInt32(5), SeqCstBB);
+
+  // Cleanup and return
+  Builder.SetInsertPoint(ContBB);
+  if (E->getOp() == AtomicExpr::Store)
+    return RValue::get(0);
+  return ConvertTempToRValue(*this, E->getType(), OrigDest);
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp
index 915ffd6034e2..97754d5c0ba6 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp
@@ -35,11 +35,18 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   AggValueSlot Dest;
   bool IgnoreResult;
 
+  /// We want to use 'dest' as the return slot except under two
+  /// conditions:
+  ///   - The destination slot requires garbage collection, so we
+  ///     need to use the GC API.
+  ///   - The destination slot is potentially aliased.
+  bool shouldUseDestForReturnSlot() const {
+    return !(Dest.requiresGCollection() || Dest.isPotentiallyAliased());
+  }
+
   ReturnValueSlot getReturnValueSlot() const {
-    // If the destination slot requires garbage collection, we can't
-    // use the real return value slot, because we have to use the GC
-    // API.
-    if (Dest.requiresGCollection()) return ReturnValueSlot();
+    if (!shouldUseDestForReturnSlot())
+      return ReturnValueSlot();
 
     return ReturnValueSlot(Dest.getAddr(), Dest.isVolatile());
   }
@@ -69,7 +76,13 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   void EmitFinalDestCopy(const Expr *E, LValue Src, bool Ignore = false);
   void EmitFinalDestCopy(const Expr *E, RValue Src, bool Ignore = false);
 
-  void EmitGCMove(const Expr *E, RValue Src);
+  void EmitMoveFromReturnSlot(const Expr *E, RValue Src);
+
+  AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
+    if (CGF.getLangOptions().getGC() && TypeRequiresGCollection(T))
+      return AggValueSlot::NeedsGCBarriers;
+    return AggValueSlot::DoesNotNeedGCBarriers;
+  }
 
   bool TypeRequiresGCollection(QualType T);
 
@@ -141,6 +154,9 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   void EmitNullInitializationToLValue(LValue Address);
   //  case Expr::ChooseExprClass:
   void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
+  void VisitAtomicExpr(AtomicExpr *E) {
+    CGF.EmitAtomicExpr(E, EnsureSlot(E->getType()).getAddr());
+  }
 };
 }  // end anonymous namespace.
 
@@ -173,23 +189,27 @@ bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
   return Record->hasObjectMember();
 }
 
-/// \brief Perform the final move to DestPtr if RequiresGCollection is set.
+/// \brief Perform the final move to DestPtr if for some reason
+/// getReturnValueSlot() didn't use it directly.
 ///
 /// The idea is that you do something like this:
 ///   RValue Result = EmitSomething(..., getReturnValueSlot());
-///   EmitGCMove(E, Result);
-/// If GC doesn't interfere, this will cause the result to be emitted
-/// directly into the return value slot.  If GC does interfere, a final
-/// move will be performed.
-void AggExprEmitter::EmitGCMove(const Expr *E, RValue Src) {
-  if (Dest.requiresGCollection()) {
-    CharUnits size = CGF.getContext().getTypeSizeInChars(E->getType());
-    const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
-    llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size.getQuantity());
-    CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, Dest.getAddr(),
-                                                    Src.getAggregateAddr(),
-                                                    SizeVal);
+///   EmitMoveFromReturnSlot(E, Result);
+///
+/// If nothing interferes, this will cause the result to be emitted
+/// directly into the return value slot.  Otherwise, a final move
+/// will be performed.
+void AggExprEmitter::EmitMoveFromReturnSlot(const Expr *E, RValue Src) {
+  if (shouldUseDestForReturnSlot()) {
+    // Logically, Dest.getAddr() should equal Src.getAggregateAddr().
+    // The possibility of undef rvalues complicates that a lot,
+    // though, so we can't really assert.
+    return;
   }
+
+  // Otherwise, do a final copy, 
+  assert(Dest.getAddr() != Src.getAggregateAddr());
+  EmitFinalDestCopy(E, Src, /*Ignore*/ true);
 }
 
 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
@@ -215,7 +235,7 @@ void AggExprEmitter::EmitFinalDestCopy(const Expr *E, RValue Src, bool Ignore) {
 
   if (Dest.requiresGCollection()) {
     CharUnits size = CGF.getContext().getTypeSizeInChars(E->getType());
-    const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
+    llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
     llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size.getQuantity());
     CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
                                                       Dest.getAddr(),
@@ -301,16 +321,15 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CK_DerivedToBase:
   case CK_BaseToDerived:
   case CK_UncheckedDerivedToBase: {
-    assert(0 && "cannot perform hierarchy conversion in EmitAggExpr: "
+    llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
                 "should have been unpacked before we got here");
-    break;
   }
 
   case CK_GetObjCProperty: {
     LValue LV = CGF.EmitLValue(E->getSubExpr());
     assert(LV.isPropertyRef());
     RValue RV = CGF.EmitLoadOfPropertyRefLValue(LV, getReturnValueSlot());
-    EmitGCMove(E, RV);
+    EmitMoveFromReturnSlot(E, RV);
     break;
   }
 
@@ -348,7 +367,8 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CK_FloatingToIntegral:
   case CK_FloatingToBoolean:
   case CK_FloatingCast:
-  case CK_AnyPointerToObjCPointerCast:
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_ObjCObjectLValueCast:
   case CK_FloatingRealToComplex:
@@ -361,9 +381,10 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CK_IntegralComplexToBoolean:
   case CK_IntegralComplexCast:
   case CK_IntegralComplexToFloatingComplex:
-  case CK_ObjCProduceObject:
-  case CK_ObjCConsumeObject:
-  case CK_ObjCReclaimReturnedObject:
+  case CK_ARCProduceObject:
+  case CK_ARCConsumeObject:
+  case CK_ARCReclaimReturnedObject:
+  case CK_ARCExtendBlockObject:
     llvm_unreachable("cast kind invalid for aggregate types");
   }
 }
@@ -375,12 +396,12 @@ void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
   }
 
   RValue RV = CGF.EmitCallExpr(E, getReturnValueSlot());
-  EmitGCMove(E, RV);
+  EmitMoveFromReturnSlot(E, RV);
 }
 
 void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   RValue RV = CGF.EmitObjCMessageExpr(E, getReturnValueSlot());
-  EmitGCMove(E, RV);
+  EmitMoveFromReturnSlot(E, RV);
 }
 
 void AggExprEmitter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
@@ -426,10 +447,9 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
         // as it may change the 'forwarding' field via call to Block_copy.
         LValue RHS = CGF.EmitLValue(E->getRHS());
         LValue LHS = CGF.EmitLValue(E->getLHS());
-        bool GCollection = false;
-        if (CGF.getContext().getLangOptions().getGCMode())
-          GCollection = TypeRequiresGCollection(E->getLHS()->getType());
-        Dest = AggValueSlot::forLValue(LHS, true, GCollection);
+        Dest = AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,
+                                       needsGC(E->getLHS()->getType()),
+                                       AggValueSlot::IsAliased);
         EmitFinalDestCopy(E, RHS, true);
         return;
       }
@@ -451,13 +471,11 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
     }
     CGF.EmitStoreThroughPropertyRefLValue(Src, LHS);
   } else {
-    bool GCollection = false;
-    if (CGF.getContext().getLangOptions().getGCMode())
-      GCollection = TypeRequiresGCollection(E->getLHS()->getType());
-
     // Codegen the RHS so that it stores directly into the LHS.
-    AggValueSlot LHSSlot = AggValueSlot::forLValue(LHS, true, 
-                                                   GCollection);
+    AggValueSlot LHSSlot =
+      AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed, 
+                              needsGC(E->getLHS()->getType()),
+                              AggValueSlot::IsAliased);
     CGF.EmitAggExpr(E->getRHS(), LHSSlot, false);
     EmitFinalDestCopy(E, LHS, true);
   }
@@ -476,7 +494,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
 
   // Save whether the destination's lifetime is externally managed.
-  bool DestLifetimeManaged = Dest.isLifetimeExternallyManaged();
+  bool isExternallyDestructed = Dest.isExternallyDestructed();
 
   eval.begin(CGF);
   CGF.EmitBlock(LHSBlock);
@@ -489,8 +507,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   // If the result of an agg expression is unused, then the emission
   // of the LHS might need to create a destination slot.  That's fine
   // with us, and we can safely emit the RHS into the same slot, but
-  // we shouldn't claim that its lifetime is externally managed.
-  Dest.setLifetimeExternallyManaged(DestLifetimeManaged);
+  // we shouldn't claim that it's already being destructed.
+  Dest.setExternallyDestructed(isExternallyDestructed);
 
   eval.begin(CGF);
   CGF.EmitBlock(RHSBlock);
@@ -518,16 +536,17 @@ void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
 
 void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
   // Ensure that we have a slot, but if we already do, remember
-  // whether its lifetime was externally managed.
-  bool WasManaged = Dest.isLifetimeExternallyManaged();
+  // whether it was externally destructed.
+  bool wasExternallyDestructed = Dest.isExternallyDestructed();
   Dest = EnsureSlot(E->getType());
-  Dest.setLifetimeExternallyManaged();
+
+  // We're going to push a destructor if there isn't already one.
+  Dest.setExternallyDestructed();
 
   Visit(E->getSubExpr());
 
-  // Set up the temporary's destructor if its lifetime wasn't already
-  // being managed.
-  if (!WasManaged)
+  // Push that destructor we promised.
+  if (!wasExternallyDestructed)
     CGF.EmitCXXTemporary(E->getTemporary(), Dest.getAddr());
 }
 
@@ -596,7 +615,10 @@ AggExprEmitter::EmitInitializationToLValue(Expr* E, LValue LV) {
   } else if (type->isAnyComplexType()) {
     CGF.EmitComplexExprIntoAddr(E, LV.getAddress(), false);
   } else if (CGF.hasAggregateLLVMType(type)) {
-    CGF.EmitAggExpr(E, AggValueSlot::forLValue(LV, true, false,
+    CGF.EmitAggExpr(E, AggValueSlot::forLValue(LV,
+                                               AggValueSlot::IsDestructed,
+                                      AggValueSlot::DoesNotNeedGCBarriers,
+                                               AggValueSlot::IsNotAliased,
                                                Dest.isZeroed()));
   } else if (LV.isSimple()) {
     CGF.EmitScalarInit(E, /*D=*/0, LV, /*Captured=*/false);
@@ -647,9 +669,9 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 
   // Handle initialization of an array.
   if (E->getType()->isArrayType()) {
-    const llvm::PointerType *APType =
+    llvm::PointerType *APType =
       cast<llvm::PointerType>(DestPtr->getType());
-    const llvm::ArrayType *AType =
+    llvm::ArrayType *AType =
       cast<llvm::ArrayType>(APType->getElementType());
 
     uint64_t NumInitElements = E->getNumInits();
@@ -676,7 +698,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
     llvm::Value *indices[] = { zero, zero };
     llvm::Value *begin =
-      Builder.CreateInBoundsGEP(DestPtr, indices, indices+2, "arrayinit.begin");
+      Builder.CreateInBoundsGEP(DestPtr, indices, "arrayinit.begin");
 
     // Exception safety requires us to destroy all the
     // already-constructed members if an initializer throws.
@@ -839,7 +861,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 
   // We'll need to enter cleanup scopes in case any of the member
   // initializers throw an exception.
-  llvm::SmallVector<EHScopeStack::stable_iterator, 16> cleanups;
+  SmallVector<EHScopeStack::stable_iterator, 16> cleanups;
 
   // Here we iterate over the fields; this makes it simpler to both
   // default-initialize fields and skip over unnamed fields.
@@ -948,7 +970,7 @@ static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
         // Reference values are always non-null and have the width of a pointer.
         if (Field->getType()->isReferenceType())
           NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
-              CGF.getContext().Target.getPointerWidth(0));
+              CGF.getContext().getTargetInfo().getPointerWidth(0));
         else
           NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
       }
@@ -999,7 +1021,7 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
   CharUnits Align = TypeInfo.second;
 
   llvm::Value *Loc = Slot.getAddr();
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   
   Loc = CGF.Builder.CreateBitCast(Loc, BP);
   CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, 
@@ -1036,7 +1058,9 @@ LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
   assert(hasAggregateLLVMType(E->getType()) && "Invalid argument!");
   llvm::Value *Temp = CreateMemTemp(E->getType());
   LValue LV = MakeAddrLValue(Temp, E->getType());
-  EmitAggExpr(E, AggValueSlot::forLValue(LV, false));
+  EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed,
+                                         AggValueSlot::DoesNotNeedGCBarriers,
+                                         AggValueSlot::IsNotAliased));
   return LV;
 }
 
@@ -1049,7 +1073,9 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
       CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl());
       assert((Record->hasTrivialCopyConstructor() || 
-              Record->hasTrivialCopyAssignment()) &&
+              Record->hasTrivialCopyAssignment() ||
+              Record->hasTrivialMoveConstructor() ||
+              Record->hasTrivialMoveAssignment()) &&
              "Trying to aggregate-copy a type without a trivial copy "
              "constructor or assignment operator");
       // Ignore empty classes in C++.
@@ -1088,24 +1114,24 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
   // we need to use a different call here.  We use isVolatile to indicate when
   // either the source or the destination is volatile.
 
-  const llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType());
-  const llvm::Type *DBP =
+  llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType());
+  llvm::Type *DBP =
     llvm::Type::getInt8PtrTy(getLLVMContext(), DPT->getAddressSpace());
-  DestPtr = Builder.CreateBitCast(DestPtr, DBP, "tmp");
+  DestPtr = Builder.CreateBitCast(DestPtr, DBP);
 
-  const llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType());
-  const llvm::Type *SBP =
+  llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType());
+  llvm::Type *SBP =
     llvm::Type::getInt8PtrTy(getLLVMContext(), SPT->getAddressSpace());
-  SrcPtr = Builder.CreateBitCast(SrcPtr, SBP, "tmp");
+  SrcPtr = Builder.CreateBitCast(SrcPtr, SBP);
 
   // Don't do any of the memmove_collectable tests if GC isn't set.
-  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC) {
+  if (CGM.getLangOptions().getGC() == LangOptions::NonGC) {
     // fall through
   } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
     RecordDecl *Record = RecordTy->getDecl();
     if (Record->hasObjectMember()) {
       CharUnits size = TypeInfo.first;
-      const llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
+      llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
       llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size.getQuantity());
       CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, 
                                                     SizeVal);
@@ -1116,7 +1142,7 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
     if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
       if (RecordTy->getDecl()->hasObjectMember()) {
         CharUnits size = TypeInfo.first;
-        const llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
+        llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
         llvm::Value *SizeVal = 
           llvm::ConstantInt::get(SizeTy, size.getQuantity());
         CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp
index 4396f567f2f9..78db5903de54 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Frontend/CodeGenOptions.h"
 #include "CodeGenFunction.h"
+#include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
 #include "CGDebugInfo.h"
@@ -206,16 +207,17 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
         cast<CXXConstructorDecl>(MD)->isDefaultConstructor())
       return RValue::get(0);
 
-    if (MD->isCopyAssignmentOperator()) {
-      // We don't like to generate the trivial copy assignment operator when
-      // it isn't necessary; just produce the proper effect here.
+    if (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) {
+      // We don't like to generate the trivial copy/move assignment operator
+      // when it isn't necessary; just produce the proper effect here.
       llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
       EmitAggregateCopy(This, RHS, CE->getType());
       return RValue::get(This);
     }
     
     if (isa<CXXConstructorDecl>(MD) && 
-        cast<CXXConstructorDecl>(MD)->isCopyConstructor()) {
+        cast<CXXConstructorDecl>(MD)->isCopyOrMoveConstructor()) {
+      // Trivial move and copy ctor are the same.
       llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
       EmitSynthesizedCXXCopyCtorCall(cast<CXXConstructorDecl>(MD), This, RHS,
                                      CE->arg_begin(), CE->arg_end());
@@ -236,7 +238,7 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
     FInfo = &CGM.getTypes().getFunctionInfo(MD);
 
   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
-  const llvm::Type *Ty
+  llvm::Type *Ty
     = CGM.getTypes().GetFunctionType(*FInfo, FPT->isVariadic());
 
   // C++ [class.virtual]p12:
@@ -333,16 +335,12 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
   LValue LV = EmitLValue(E->getArg(0));
   llvm::Value *This = LV.getAddress();
 
-  if (MD->isCopyAssignmentOperator()) {
-    const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(MD->getDeclContext());
-    if (ClassDecl->hasTrivialCopyAssignment()) {
-      assert(!ClassDecl->hasUserDeclaredCopyAssignment() &&
-             "EmitCXXOperatorMemberCallExpr - user declared copy assignment");
-      llvm::Value *Src = EmitLValue(E->getArg(1)).getAddress();
-      QualType Ty = E->getType();
-      EmitAggregateCopy(This, Src, Ty);
-      return RValue::get(This);
-    }
+  if ((MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) &&
+      MD->isTrivial()) {
+    llvm::Value *Src = EmitLValue(E->getArg(1)).getAddress();
+    QualType Ty = E->getType();
+    EmitAggregateCopy(This, Src, Ty);
+    return RValue::get(This);
   }
 
   llvm::Value *Callee = EmitCXXOperatorMemberCallee(E, MD, This);
@@ -350,6 +348,54 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
                            E->arg_begin() + 1, E->arg_end());
 }
 
+RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
+                                               ReturnValueSlot ReturnValue) {
+  return CGM.getCUDARuntime().EmitCUDAKernelCallExpr(*this, E, ReturnValue);
+}
+
+static void EmitNullBaseClassInitialization(CodeGenFunction &CGF,
+                                            llvm::Value *DestPtr,
+                                            const CXXRecordDecl *Base) {
+  if (Base->isEmpty())
+    return;
+
+  DestPtr = CGF.EmitCastToVoidPtr(DestPtr);
+
+  const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(Base);
+  CharUnits Size = Layout.getNonVirtualSize();
+  CharUnits Align = Layout.getNonVirtualAlign();
+
+  llvm::Value *SizeVal = CGF.CGM.getSize(Size);
+
+  // If the type contains a pointer to data member we can't memset it to zero.
+  // Instead, create a null constant and copy it to the destination.
+  // TODO: there are other patterns besides zero that we can usefully memset,
+  // like -1, which happens to be the pattern used by member-pointers.
+  // TODO: isZeroInitializable can be over-conservative in the case where a
+  // virtual base contains a member pointer.
+  if (!CGF.CGM.getTypes().isZeroInitializable(Base)) {
+    llvm::Constant *NullConstant = CGF.CGM.EmitNullConstantForBase(Base);
+
+    llvm::GlobalVariable *NullVariable = 
+      new llvm::GlobalVariable(CGF.CGM.getModule(), NullConstant->getType(),
+                               /*isConstant=*/true, 
+                               llvm::GlobalVariable::PrivateLinkage,
+                               NullConstant, Twine());
+    NullVariable->setAlignment(Align.getQuantity());
+    llvm::Value *SrcPtr = CGF.EmitCastToVoidPtr(NullVariable);
+
+    // Get and call the appropriate llvm.memcpy overload.
+    CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align.getQuantity());
+    return;
+  } 
+  
+  // Otherwise, just memset the whole thing to zero.  This is legal
+  // because in LLVM, all default initializers (other than the ones we just
+  // handled above) are guaranteed to have a bit pattern of all zeros.
+  CGF.Builder.CreateMemSet(DestPtr, CGF.Builder.getInt8(0), SizeVal,
+                           Align.getQuantity());
+}
+
 void
 CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,
                                       AggValueSlot Dest) {
@@ -360,8 +406,19 @@ CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,
   // constructor, as can be the case with a non-user-provided default
   // constructor, emit the zero initialization now, unless destination is
   // already zeroed.
-  if (E->requiresZeroInitialization() && !Dest.isZeroed())
-    EmitNullInitialization(Dest.getAddr(), E->getType());
+  if (E->requiresZeroInitialization() && !Dest.isZeroed()) {
+    switch (E->getConstructionKind()) {
+    case CXXConstructExpr::CK_Delegating:
+      assert(0 && "Delegating constructor should not need zeroing");
+    case CXXConstructExpr::CK_Complete:
+      EmitNullInitialization(Dest.getAddr(), E->getType());
+      break;
+    case CXXConstructExpr::CK_VirtualBase:
+    case CXXConstructExpr::CK_NonVirtualBase:
+      EmitNullBaseClassInitialization(*this, Dest.getAddr(), CD->getParent());
+      break;
+    }
+  }
   
   // If this is a call to a trivial default constructor, do nothing.
   if (CD->isTrivial() && CD->isDefaultConstructor())
@@ -483,7 +540,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
   // the cookie size would bring the total size >= 0.
   bool isSigned 
     = e->getArraySize()->getType()->isSignedIntegerOrEnumerationType();
-  const llvm::IntegerType *numElementsType
+  llvm::IntegerType *numElementsType
     = cast<llvm::IntegerType>(numElements->getType());
   unsigned numElementsWidth = numElementsType->getBitWidth();
 
@@ -703,63 +760,85 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const CXXNewExpr *E,
                                 AllocType.isVolatileQualified());
   else {
     AggValueSlot Slot
-      = AggValueSlot::forAddr(NewPtr, AllocType.getQualifiers(), true);
+      = AggValueSlot::forAddr(NewPtr, AllocType.getQualifiers(),
+                              AggValueSlot::IsDestructed,
+                              AggValueSlot::DoesNotNeedGCBarriers,
+                              AggValueSlot::IsNotAliased);
     CGF.EmitAggExpr(Init, Slot);
   }
 }
 
 void
 CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, 
-                                         llvm::Value *NewPtr,
-                                         llvm::Value *NumElements) {
+                                         QualType elementType,
+                                         llvm::Value *beginPtr,
+                                         llvm::Value *numElements) {
   // We have a POD type.
   if (E->getNumConstructorArgs() == 0)
     return;
-  
-  const llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
-  
-  // Create a temporary for the loop index and initialize it with 0.
-  llvm::Value *IndexPtr = CreateTempAlloca(SizeTy, "loop.index");
-  llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy);
-  Builder.CreateStore(Zero, IndexPtr);
-  
-  // Start the loop with a block that tests the condition.
-  llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
-  llvm::BasicBlock *AfterFor = createBasicBlock("for.end");
-  
-  EmitBlock(CondBlock);
-  
-  llvm::BasicBlock *ForBody = createBasicBlock("for.body");
-  
-  // Generate: if (loop-index < number-of-elements fall to the loop body,
-  // otherwise, go to the block after the for-loop.
-  llvm::Value *Counter = Builder.CreateLoad(IndexPtr);
-  llvm::Value *IsLess = Builder.CreateICmpULT(Counter, NumElements, "isless");
-  // If the condition is true, execute the body.
-  Builder.CreateCondBr(IsLess, ForBody, AfterFor);
-  
-  EmitBlock(ForBody);
-  
-  llvm::BasicBlock *ContinueBlock = createBasicBlock("for.inc");
-  // Inside the loop body, emit the constructor call on the array element.
-  Counter = Builder.CreateLoad(IndexPtr);
-  llvm::Value *Address = Builder.CreateInBoundsGEP(NewPtr, Counter, 
-                                                   "arrayidx");
-  StoreAnyExprIntoOneUnit(*this, E, Address);
-  
-  EmitBlock(ContinueBlock);
-  
-  // Emit the increment of the loop counter.
-  llvm::Value *NextVal = llvm::ConstantInt::get(SizeTy, 1);
-  Counter = Builder.CreateLoad(IndexPtr);
-  NextVal = Builder.CreateAdd(Counter, NextVal, "inc");
-  Builder.CreateStore(NextVal, IndexPtr);
-  
-  // Finally, branch back up to the condition for the next iteration.
-  EmitBranch(CondBlock);
-  
-  // Emit the fall-through block.
-  EmitBlock(AfterFor, true);
+
+  // Check if the number of elements is constant.
+  bool checkZero = true;
+  if (llvm::ConstantInt *constNum = dyn_cast<llvm::ConstantInt>(numElements)) {
+    // If it's constant zero, skip the whole loop.
+    if (constNum->isZero()) return;
+
+    checkZero = false;
+  }
+
+  // Find the end of the array, hoisted out of the loop.
+  llvm::Value *endPtr =
+    Builder.CreateInBoundsGEP(beginPtr, numElements, "array.end");
+
+  // Create the continuation block.
+  llvm::BasicBlock *contBB = createBasicBlock("new.loop.end");
+
+  // If we need to check for zero, do so now.
+  if (checkZero) {
+    llvm::BasicBlock *nonEmptyBB = createBasicBlock("new.loop.nonempty");
+    llvm::Value *isEmpty = Builder.CreateICmpEQ(beginPtr, endPtr,
+                                                "array.isempty");
+    Builder.CreateCondBr(isEmpty, contBB, nonEmptyBB);
+    EmitBlock(nonEmptyBB);
+  }
+
+  // Enter the loop.
+  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
+  llvm::BasicBlock *loopBB = createBasicBlock("new.loop");
+
+  EmitBlock(loopBB);
+
+  // Set up the current-element phi.
+  llvm::PHINode *curPtr =
+    Builder.CreatePHI(beginPtr->getType(), 2, "array.cur");
+  curPtr->addIncoming(beginPtr, entryBB);
+
+  // Enter a partial-destruction cleanup if necessary.
+  QualType::DestructionKind dtorKind = elementType.isDestructedType();
+  EHScopeStack::stable_iterator cleanup;
+  if (needsEHCleanup(dtorKind)) {
+    pushRegularPartialArrayCleanup(beginPtr, curPtr, elementType,
+                                   getDestroyer(dtorKind));
+    cleanup = EHStack.stable_begin();
+  }
+
+  // Emit the initializer into this element.
+  StoreAnyExprIntoOneUnit(*this, E, curPtr);
+
+  // Leave the cleanup if we entered one.
+  if (cleanup != EHStack.stable_end())
+    DeactivateCleanupBlock(cleanup);
+
+  // Advance to the next element.
+  llvm::Value *nextPtr = Builder.CreateConstGEP1_32(curPtr, 1, "array.next");
+
+  // Check whether we've gotten to the end of the array and, if so,
+  // exit the loop.
+  llvm::Value *isEnd = Builder.CreateICmpEQ(nextPtr, endPtr, "array.atend");
+  Builder.CreateCondBr(isEnd, contBB, loopBB);
+  curPtr->addIncoming(nextPtr, Builder.GetInsertBlock());
+
+  EmitBlock(contBB);
 }
 
 static void EmitZeroMemSet(CodeGenFunction &CGF, QualType T,
@@ -771,6 +850,7 @@ static void EmitZeroMemSet(CodeGenFunction &CGF, QualType T,
 }
                        
 static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
+                               QualType ElementType,
                                llvm::Value *NewPtr,
                                llvm::Value *NumElements,
                                llvm::Value *AllocSizeWithoutCookie) {
@@ -783,11 +863,10 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
         if (!E->hasInitializer() || Ctor->getParent()->isEmpty())
           return;
       
-        if (CGF.CGM.getTypes().isZeroInitializable(E->getAllocatedType())) {
+        if (CGF.CGM.getTypes().isZeroInitializable(ElementType)) {
           // Optimization: since zero initialization will just set the memory
           // to all zeroes, generate a single memset to do it in one shot.
-          EmitZeroMemSet(CGF, E->getAllocatedType(), NewPtr, 
-                         AllocSizeWithoutCookie);
+          EmitZeroMemSet(CGF, ElementType, NewPtr, AllocSizeWithoutCookie);
           return;
         }
 
@@ -803,11 +882,10 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
                isa<ImplicitValueInitExpr>(E->getConstructorArg(0))) {
       // Optimization: since zero initialization will just set the memory
       // to all zeroes, generate a single memset to do it in one shot.
-      EmitZeroMemSet(CGF, E->getAllocatedType(), NewPtr, 
-                     AllocSizeWithoutCookie);
-      return;      
+      EmitZeroMemSet(CGF, ElementType, NewPtr, AllocSizeWithoutCookie);
+      return;
     } else {
-      CGF.EmitNewArrayInitializer(E, NewPtr, NumElements);
+      CGF.EmitNewArrayInitializer(E, ElementType, NewPtr, NumElements);
       return;
     }
   }
@@ -819,7 +897,7 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
     if (E->hasInitializer() && 
         !Ctor->getParent()->hasUserDeclaredConstructor() &&
         !Ctor->getParent()->isEmpty())
-      CGF.EmitNullInitialization(NewPtr, E->getAllocatedType());
+      CGF.EmitNullInitialization(NewPtr, ElementType);
       
     CGF.EmitCXXConstructorCall(Ctor, Ctor_Complete, /*ForVirtualBase=*/false, 
                                NewPtr, E->constructor_arg_begin(),
@@ -1086,15 +1164,6 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     Builder.CreateCondBr(isNull, contBB, notNullBB);
     EmitBlock(notNullBB);
   }
-  
-  assert((allocSize == allocSizeWithoutCookie) ==
-         CalculateCookiePadding(*this, E).isZero());
-  if (allocSize != allocSizeWithoutCookie) {
-    assert(E->isArray());
-    allocation = CGM.getCXXABI().InitializeArrayCookie(*this, allocation,
-                                                       numElements,
-                                                       E, allocType);
-  }
 
   // If there's an operator delete, enter a cleanup to call it if an
   // exception is thrown.
@@ -1105,21 +1174,28 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     operatorDeleteCleanup = EHStack.stable_begin();
   }
 
-  const llvm::Type *elementPtrTy
+  assert((allocSize == allocSizeWithoutCookie) ==
+         CalculateCookiePadding(*this, E).isZero());
+  if (allocSize != allocSizeWithoutCookie) {
+    assert(E->isArray());
+    allocation = CGM.getCXXABI().InitializeArrayCookie(*this, allocation,
+                                                       numElements,
+                                                       E, allocType);
+  }
+
+  llvm::Type *elementPtrTy
     = ConvertTypeForMem(allocType)->getPointerTo(AS);
   llvm::Value *result = Builder.CreateBitCast(allocation, elementPtrTy);
 
+  EmitNewInitializer(*this, E, allocType, result, numElements,
+                     allocSizeWithoutCookie);
   if (E->isArray()) {
-    EmitNewInitializer(*this, E, result, numElements, allocSizeWithoutCookie);
-
     // NewPtr is a pointer to the base element type.  If we're
     // allocating an array of arrays, we'll need to cast back to the
     // array pointer type.
-    const llvm::Type *resultType = ConvertTypeForMem(E->getType());
+    llvm::Type *resultType = ConvertTypeForMem(E->getType());
     if (result->getType() != resultType)
       result = Builder.CreateBitCast(result, resultType);
-  } else {
-    EmitNewInitializer(*this, E, result, numElements, allocSizeWithoutCookie);
   }
 
   // Deactivate the 'operator delete' cleanup if we finished
@@ -1206,7 +1282,7 @@ static void EmitObjectDelete(CodeGenFunction &CGF,
   const CXXDestructorDecl *Dtor = 0;
   if (const RecordType *RT = ElementType->getAs<RecordType>()) {
     CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-    if (!RD->hasTrivialDestructor()) {
+    if (RD->hasDefinition() && !RD->hasTrivialDestructor()) {
       Dtor = RD->getDestructor();
 
       if (Dtor->isVirtual()) {
@@ -1218,7 +1294,7 @@ static void EmitObjectDelete(CodeGenFunction &CGF,
                                                     ElementType);
         }
         
-        const llvm::Type *Ty =
+        llvm::Type *Ty =
           CGF.getTypes().GetFunctionType(CGF.getTypes().getFunctionInfo(Dtor,
                                                                Dtor_Complete),
                                          /*isVariadic=*/false);
@@ -1307,7 +1383,7 @@ namespace {
       // Pass the original requested size as the second argument.
       if (DeleteFTy->getNumArgs() == 2) {
         QualType size_t = DeleteFTy->getArgType(1);
-        const llvm::IntegerType *SizeTy
+        llvm::IntegerType *SizeTy
           = cast<llvm::IntegerType>(CGF.ConvertType(size_t));
         
         CharUnits ElementTypeSize =
@@ -1406,7 +1482,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
   QualType DeleteTy = Arg->getType()->getAs<PointerType>()->getPointeeType();
   if (DeleteTy->isConstantArrayType()) {
     llvm::Value *Zero = Builder.getInt32(0);
-    llvm::SmallVector<llvm::Value*,8> GEP;
+    SmallVector<llvm::Value*,8> GEP;
 
     GEP.push_back(Zero); // point at the outermost array
 
@@ -1420,7 +1496,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
       GEP.push_back(Zero);
     }
 
-    Ptr = Builder.CreateInBoundsGEP(Ptr, GEP.begin(), GEP.end(), "del.first");
+    Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, "del.first");
   }
 
   assert(ConvertTypeForMem(DeleteTy) ==
@@ -1439,8 +1515,8 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
 static llvm::Constant *getBadTypeidFn(CodeGenFunction &CGF) {
   // void __cxa_bad_typeid();
   
-  const llvm::Type *VoidTy = llvm::Type::getVoidTy(CGF.getLLVMContext());
-  const llvm::FunctionType *FTy =
+  llvm::Type *VoidTy = llvm::Type::getVoidTy(CGF.getLLVMContext());
+  llvm::FunctionType *FTy =
   llvm::FunctionType::get(VoidTy, false);
   
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_typeid");
@@ -1454,7 +1530,7 @@ static void EmitBadTypeidCall(CodeGenFunction &CGF) {
 
 static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF,
                                          const Expr *E, 
-                                         const llvm::Type *StdTypeInfoPtrTy) {
+                                         llvm::Type *StdTypeInfoPtrTy) {
   // Get the vtable pointer.
   llvm::Value *ThisPtr = CGF.EmitLValue(E).getAddress();
 
@@ -1487,7 +1563,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF,
 }
 
 llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
-  const llvm::Type *StdTypeInfoPtrTy = 
+  llvm::Type *StdTypeInfoPtrTy = 
     ConvertType(E->getType())->getPointerTo();
   
   if (E->isTypeOperand()) {
@@ -1528,7 +1604,7 @@ static llvm::Constant *getDynamicCastFn(CodeGenFunction &CGF) {
 
   llvm::Type *Args[4] = { Int8PtrTy, Int8PtrTy, Int8PtrTy, PtrDiffTy };
   
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(Int8PtrTy, Args, false);
   
   return CGF.CGM.CreateRuntimeFunction(FTy, "__dynamic_cast");
@@ -1537,8 +1613,8 @@ static llvm::Constant *getDynamicCastFn(CodeGenFunction &CGF) {
 static llvm::Constant *getBadCastFn(CodeGenFunction &CGF) {
   // void __cxa_bad_cast();
   
-  const llvm::Type *VoidTy = llvm::Type::getVoidTy(CGF.getLLVMContext());
-  const llvm::FunctionType *FTy =
+  llvm::Type *VoidTy = llvm::Type::getVoidTy(CGF.getLLVMContext());
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(VoidTy, false);
   
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_cast");
@@ -1554,9 +1630,9 @@ static llvm::Value *
 EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
                     QualType SrcTy, QualType DestTy,
                     llvm::BasicBlock *CastEnd) {
-  const llvm::Type *PtrDiffLTy = 
+  llvm::Type *PtrDiffLTy = 
     CGF.ConvertType(CGF.getContext().getPointerDiffType());
-  const llvm::Type *DestLTy = CGF.ConvertType(DestTy);
+  llvm::Type *DestLTy = CGF.ConvertType(DestTy);
 
   if (const PointerType *PTy = DestTy->getAs<PointerType>()) {
     if (PTy->getPointeeType()->isVoidType()) {
@@ -1626,7 +1702,7 @@ EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
 
 static llvm::Value *EmitDynamicCastToNull(CodeGenFunction &CGF,
                                           QualType DestTy) {
-  const llvm::Type *DestLTy = CGF.ConvertType(DestTy);
+  llvm::Type *DestLTy = CGF.ConvertType(DestTy);
   if (DestTy->isPointerType())
     return llvm::Constant::getNullValue(DestLTy);
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp
index 35cff1d72714..4a31bcfbe9e0 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp
@@ -103,8 +103,7 @@ class ComplexExprEmitter
     
   ComplexPairTy VisitStmt(Stmt *S) {
     S->dump(CGF.getContext().getSourceManager());
-    assert(0 && "Stmt can't have complex result type!");
-    return ComplexPairTy();
+    llvm_unreachable("Stmt can't have complex result type!");
   }
   ComplexPairTy VisitExpr(Expr *S);
   ComplexPairTy VisitParenExpr(ParenExpr *PE) { return Visit(PE->getSubExpr());}
@@ -119,6 +118,7 @@ class ComplexExprEmitter
 
   // l-values.
   ComplexPairTy VisitDeclRefExpr(const Expr *E) { return EmitLoadOfLValue(E); }
+  ComplexPairTy VisitBlockDeclRefExpr(const Expr *E) { return EmitLoadOfLValue(E); }
   ComplexPairTy VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
     return EmitLoadOfLValue(E);
   }
@@ -266,6 +266,10 @@ class ComplexExprEmitter
   ComplexPairTy VisitInitListExpr(InitListExpr *E);
 
   ComplexPairTy VisitVAArgExpr(VAArgExpr *E);
+
+  ComplexPairTy VisitAtomicExpr(AtomicExpr *E) {
+    return CGF.EmitAtomicExpr(E).getComplexVal();
+  }
 };
 }  // end anonymous namespace.
 
@@ -312,8 +316,8 @@ void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, llvm::Value *Ptr,
 
 ComplexPairTy ComplexExprEmitter::VisitExpr(Expr *E) {
   CGF.ErrorUnsupported(E, "complex expression");
-  const llvm::Type *EltTy =
-    CGF.ConvertType(E->getType()->getAs<ComplexType>()->getElementType());
+  llvm::Type *EltTy =
+  CGF.ConvertType(E->getType()->getAs<ComplexType>()->getElementType());
   llvm::Value *U = llvm::UndefValue::get(EltTy);
   return ComplexPairTy(U, U);
 }
@@ -402,16 +406,18 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op,
   case CK_FloatingToIntegral:
   case CK_FloatingToBoolean:
   case CK_FloatingCast:
-  case CK_AnyPointerToObjCPointerCast:
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_ObjCObjectLValueCast:
   case CK_FloatingComplexToReal:
   case CK_FloatingComplexToBoolean:
   case CK_IntegralComplexToReal:
   case CK_IntegralComplexToBoolean:
-  case CK_ObjCProduceObject:
-  case CK_ObjCConsumeObject:
-  case CK_ObjCReclaimReturnedObject:
+  case CK_ARCProduceObject:
+  case CK_ARCConsumeObject:
+  case CK_ARCReclaimReturnedObject:
+  case CK_ARCExtendBlockObject:
     llvm_unreachable("invalid cast kind for complex value");
 
   case CK_FloatingRealToComplex:
@@ -524,40 +530,40 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
   llvm::Value *DSTr, *DSTi;
   if (Op.LHS.first->getType()->isFloatingPointTy()) {
     // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
-    llvm::Value *Tmp1 = Builder.CreateFMul(LHSr, RHSr, "tmp"); // a*c
-    llvm::Value *Tmp2 = Builder.CreateFMul(LHSi, RHSi, "tmp"); // b*d
-    llvm::Value *Tmp3 = Builder.CreateFAdd(Tmp1, Tmp2, "tmp"); // ac+bd
+    llvm::Value *Tmp1 = Builder.CreateFMul(LHSr, RHSr); // a*c
+    llvm::Value *Tmp2 = Builder.CreateFMul(LHSi, RHSi); // b*d
+    llvm::Value *Tmp3 = Builder.CreateFAdd(Tmp1, Tmp2); // ac+bd
 
-    llvm::Value *Tmp4 = Builder.CreateFMul(RHSr, RHSr, "tmp"); // c*c
-    llvm::Value *Tmp5 = Builder.CreateFMul(RHSi, RHSi, "tmp"); // d*d
-    llvm::Value *Tmp6 = Builder.CreateFAdd(Tmp4, Tmp5, "tmp"); // cc+dd
+    llvm::Value *Tmp4 = Builder.CreateFMul(RHSr, RHSr); // c*c
+    llvm::Value *Tmp5 = Builder.CreateFMul(RHSi, RHSi); // d*d
+    llvm::Value *Tmp6 = Builder.CreateFAdd(Tmp4, Tmp5); // cc+dd
 
-    llvm::Value *Tmp7 = Builder.CreateFMul(LHSi, RHSr, "tmp"); // b*c
-    llvm::Value *Tmp8 = Builder.CreateFMul(LHSr, RHSi, "tmp"); // a*d
-    llvm::Value *Tmp9 = Builder.CreateFSub(Tmp7, Tmp8, "tmp"); // bc-ad
+    llvm::Value *Tmp7 = Builder.CreateFMul(LHSi, RHSr); // b*c
+    llvm::Value *Tmp8 = Builder.CreateFMul(LHSr, RHSi); // a*d
+    llvm::Value *Tmp9 = Builder.CreateFSub(Tmp7, Tmp8); // bc-ad
 
-    DSTr = Builder.CreateFDiv(Tmp3, Tmp6, "tmp");
-    DSTi = Builder.CreateFDiv(Tmp9, Tmp6, "tmp");
+    DSTr = Builder.CreateFDiv(Tmp3, Tmp6);
+    DSTi = Builder.CreateFDiv(Tmp9, Tmp6);
   } else {
     // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
-    llvm::Value *Tmp1 = Builder.CreateMul(LHSr, RHSr, "tmp"); // a*c
-    llvm::Value *Tmp2 = Builder.CreateMul(LHSi, RHSi, "tmp"); // b*d
-    llvm::Value *Tmp3 = Builder.CreateAdd(Tmp1, Tmp2, "tmp"); // ac+bd
+    llvm::Value *Tmp1 = Builder.CreateMul(LHSr, RHSr); // a*c
+    llvm::Value *Tmp2 = Builder.CreateMul(LHSi, RHSi); // b*d
+    llvm::Value *Tmp3 = Builder.CreateAdd(Tmp1, Tmp2); // ac+bd
 
-    llvm::Value *Tmp4 = Builder.CreateMul(RHSr, RHSr, "tmp"); // c*c
-    llvm::Value *Tmp5 = Builder.CreateMul(RHSi, RHSi, "tmp"); // d*d
-    llvm::Value *Tmp6 = Builder.CreateAdd(Tmp4, Tmp5, "tmp"); // cc+dd
+    llvm::Value *Tmp4 = Builder.CreateMul(RHSr, RHSr); // c*c
+    llvm::Value *Tmp5 = Builder.CreateMul(RHSi, RHSi); // d*d
+    llvm::Value *Tmp6 = Builder.CreateAdd(Tmp4, Tmp5); // cc+dd
 
-    llvm::Value *Tmp7 = Builder.CreateMul(LHSi, RHSr, "tmp"); // b*c
-    llvm::Value *Tmp8 = Builder.CreateMul(LHSr, RHSi, "tmp"); // a*d
-    llvm::Value *Tmp9 = Builder.CreateSub(Tmp7, Tmp8, "tmp"); // bc-ad
+    llvm::Value *Tmp7 = Builder.CreateMul(LHSi, RHSr); // b*c
+    llvm::Value *Tmp8 = Builder.CreateMul(LHSr, RHSi); // a*d
+    llvm::Value *Tmp9 = Builder.CreateSub(Tmp7, Tmp8); // bc-ad
 
     if (Op.Ty->getAs<ComplexType>()->getElementType()->isUnsignedIntegerType()) {
-      DSTr = Builder.CreateUDiv(Tmp3, Tmp6, "tmp");
-      DSTi = Builder.CreateUDiv(Tmp9, Tmp6, "tmp");
+      DSTr = Builder.CreateUDiv(Tmp3, Tmp6);
+      DSTi = Builder.CreateUDiv(Tmp9, Tmp6);
     } else {
-      DSTr = Builder.CreateSDiv(Tmp3, Tmp6, "tmp");
-      DSTi = Builder.CreateSDiv(Tmp9, Tmp6, "tmp");
+      DSTr = Builder.CreateSDiv(Tmp3, Tmp6);
+      DSTi = Builder.CreateSDiv(Tmp9, Tmp6);
     }
   }
 
@@ -735,12 +741,19 @@ ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) {
     Ignore = TestAndClearIgnoreImag();
     (void)Ignore;
     assert (Ignore == false && "init list ignored");
-  if (E->getNumInits())
+
+  if (E->getNumInits() == 2) {
+    llvm::Value *Real = CGF.EmitScalarExpr(E->getInit(0));
+    llvm::Value *Imag = CGF.EmitScalarExpr(E->getInit(1));
+    return ComplexPairTy(Real, Imag);
+  } else if (E->getNumInits() == 1) {
     return Visit(E->getInit(0));
+  }
 
   // Empty init list intializes to null
+  assert(E->getNumInits() == 0 && "Unexpected number of inits");
   QualType Ty = E->getType()->getAs<ComplexType>()->getElementType();
-  const llvm::Type* LTy = CGF.ConvertType(Ty);
+  llvm::Type* LTy = CGF.ConvertType(Ty);
   llvm::Value* zeroConstant = llvm::Constant::getNullValue(LTy);
   return ComplexPairTy(zeroConstant, zeroConstant);
 }
@@ -751,7 +764,7 @@ ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) {
 
   if (!ArgPtr) {
     CGF.ErrorUnsupported(E, "complex va_arg expression");
-    const llvm::Type *EltTy =
+    llvm::Type *EltTy =
       CGF.ConvertType(E->getType()->getAs<ComplexType>()->getElementType());
     llvm::Value *U = llvm::UndefValue::get(EltTy);
     return ComplexPairTy(U, U);
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp
index 45e44dda0f58..3997866ea68e 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp
@@ -138,13 +138,12 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
     // We need to add padding.
     CharUnits PadSize = Context.toCharUnitsFromBits(
       llvm::RoundUpToAlignment(FieldOffset - NextFieldOffsetInBits, 
-                               Context.Target.getCharAlign()));
+                               Context.getTargetInfo().getCharAlign()));
 
     AppendPadding(PadSize);
   }
 
-  uint64_t FieldSize =
-    Field->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
+  uint64_t FieldSize = Field->getBitWidthValue(Context);
 
   llvm::APInt FieldValue = CI->getValue();
 
@@ -213,7 +212,7 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
         // padding and then an hole for our i8 to get plopped into.
         assert(isa<llvm::ArrayType>(LastElt->getType()) &&
                "Expected array padding of undefs");
-        const llvm::ArrayType *AT = cast<llvm::ArrayType>(LastElt->getType());
+        llvm::ArrayType *AT = cast<llvm::ArrayType>(LastElt->getType());
         assert(AT->getElementType()->isIntegerTy(CharWidth) &&
                AT->getNumElements() != 0 &&
                "Expected non-empty array padding of undefs");
@@ -281,7 +280,7 @@ void ConstStructBuilder::AppendPadding(CharUnits PadSize) {
   if (PadSize.isZero())
     return;
 
-  const llvm::Type *Ty = llvm::Type::getInt8Ty(CGM.getLLVMContext());
+  llvm::Type *Ty = llvm::Type::getInt8Ty(CGM.getLLVMContext());
   if (PadSize > CharUnits::One())
     Ty = llvm::ArrayType::get(Ty, PadSize.getQuantity());
 
@@ -317,7 +316,7 @@ void ConstStructBuilder::ConvertStructToPacked() {
       CharUnits NumChars =
         AlignedElementOffsetInChars - ElementOffsetInChars;
 
-      const llvm::Type *Ty = llvm::Type::getInt8Ty(CGM.getLLVMContext());
+      llvm::Type *Ty = llvm::Type::getInt8Ty(CGM.getLLVMContext());
       if (NumChars > CharUnits::One())
         Ty = llvm::ArrayType::get(Ty, NumChars.getQuantity());
 
@@ -364,7 +363,7 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
       continue;
 
     // Don't emit anonymous bitfields, they just affect layout.
-    if (Field->isBitField() && !Field->getIdentifier()) {
+    if (Field->isUnnamedBitfield()) {
       LastFD = (*Field);
       continue;
     }
@@ -435,11 +434,11 @@ llvm::Constant *ConstStructBuilder::
   
   // Pick the type to use.  If the type is layout identical to the ConvertType
   // type then use it, otherwise use whatever the builder produced for us.
-  const llvm::StructType *STy =
+  llvm::StructType *STy =
       llvm::ConstantStruct::getTypeForElements(CGM.getLLVMContext(),
                                                Builder.Elements,Builder.Packed);
-  const llvm::Type *ILETy = CGM.getTypes().ConvertType(ILE->getType());
-  if (const llvm::StructType *ILESTy = dyn_cast<llvm::StructType>(ILETy)) {
+  llvm::Type *ILETy = CGM.getTypes().ConvertType(ILE->getType());
+  if (llvm::StructType *ILESTy = dyn_cast<llvm::StructType>(ILETy)) {
     if (ILESTy->isLayoutIdentical(STy))
       STy = ILESTy;
   }
@@ -513,7 +512,7 @@ class ConstExprEmitter :
     llvm::Constant *RHS = CGM.EmitConstantExpr(E->getRHS(),
                                                E->getRHS()->getType(), CGF);
 
-    const llvm::Type *ResultType = ConvertType(E->getType());
+    llvm::Type *ResultType = ConvertType(E->getType());
     LHS = llvm::ConstantExpr::getPtrToInt(LHS, ResultType);
     RHS = llvm::ConstantExpr::getPtrToInt(RHS, ResultType);
         
@@ -527,7 +526,7 @@ class ConstExprEmitter :
     llvm::Constant *C = CGM.EmitConstantExpr(subExpr, subExpr->getType(), CGF);
     if (!C) return 0;
 
-    const llvm::Type *destType = ConvertType(E->getType());
+    llvm::Type *destType = ConvertType(E->getType());
 
     switch (E->getCastKind()) {
     case CK_ToUnion: {
@@ -571,7 +570,8 @@ class ConstExprEmitter :
     case CK_NoOp:
       return C;
 
-    case CK_AnyPointerToObjCPointerCast:
+    case CK_CPointerToObjCPointerCast:
+    case CK_BlockPointerToObjCPointerCast:
     case CK_AnyPointerToBlockPointerCast:
     case CK_LValueBitCast:
     case CK_BitCast:
@@ -585,9 +585,10 @@ class ConstExprEmitter :
     case CK_GetObjCProperty:
     case CK_ToVoid:
     case CK_Dynamic:
-    case CK_ObjCProduceObject:
-    case CK_ObjCConsumeObject:
-    case CK_ObjCReclaimReturnedObject:
+    case CK_ARCProduceObject:
+    case CK_ARCConsumeObject:
+    case CK_ARCReclaimReturnedObject:
+    case CK_ARCExtendBlockObject:
       return 0;
 
     // These might need to be supported for constexpr.
@@ -680,9 +681,9 @@ class ConstExprEmitter :
       return Visit(ILE->getInit(0));
 
     std::vector<llvm::Constant*> Elts;
-    const llvm::ArrayType *AType =
+    llvm::ArrayType *AType =
         cast<llvm::ArrayType>(ConvertType(ILE->getType()));
-    const llvm::Type *ElemTy = AType->getElementType();
+    llvm::Type *ElemTy = AType->getElementType();
     unsigned NumElements = AType->getNumElements();
 
     // Initialising an array requires us to automatically
@@ -719,7 +720,7 @@ class ConstExprEmitter :
       std::vector<llvm::Type*> Types;
       for (unsigned i = 0; i < Elts.size(); ++i)
         Types.push_back(Elts[i]->getType());
-      const llvm::StructType *SType = llvm::StructType::get(AType->getContext(),
+      llvm::StructType *SType = llvm::StructType::get(AType->getContext(),
                                                             Types, true);
       return llvm::ConstantStruct::get(SType, Elts);
     }
@@ -740,6 +741,22 @@ class ConstExprEmitter :
   }
 
   llvm::Constant *VisitInitListExpr(InitListExpr *ILE) {
+    if (ILE->getType()->isAnyComplexType() && ILE->getNumInits() == 2) {
+      // Complex type with element initializers
+      Expr *Real = ILE->getInit(0);
+      Expr *Imag = ILE->getInit(1);
+      llvm::Constant *Complex[2];
+      Complex[0] = CGM.EmitConstantExpr(Real, Real->getType(), CGF);
+      if (!Complex[0])
+        return 0;
+      Complex[1] = CGM.EmitConstantExpr(Imag, Imag->getType(), CGF);
+      if (!Complex[1])
+        return 0;
+      llvm::StructType *STy =
+          cast<llvm::StructType>(ConvertType(ILE->getType()));
+      return llvm::ConstantStruct::get(STy, Complex);
+    }
+
     if (ILE->getType()->isScalarType()) {
       // We have a scalar in braces. Just use the first element.
       if (ILE->getNumInits() > 0) {
@@ -762,10 +779,7 @@ class ConstExprEmitter :
     if (ILE->getType()->isVectorType())
       return 0;
 
-    assert(0 && "Unable to handle InitListExpr");
-    // Get rid of control reaches end of void function warning.
-    // Not reached.
-    return 0;
+    llvm_unreachable("Unable to handle InitListExpr");
   }
 
   llvm::Constant *VisitCXXConstructExpr(CXXConstructExpr *E) {
@@ -789,8 +803,8 @@ class ConstExprEmitter :
 
     if (E->getNumArgs()) {
       assert(E->getNumArgs() == 1 && "trivial ctor with > 1 argument");
-      assert(E->getConstructor()->isCopyConstructor() &&
-             "trivial ctor has argument but isn't a copy ctor");
+      assert(E->getConstructor()->isCopyOrMoveConstructor() &&
+             "trivial ctor has argument but isn't a copy/move ctor");
 
       Expr *Arg = E->getArg(0);
       assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) &&
@@ -831,7 +845,7 @@ class ConstExprEmitter :
   }
 
   // Utility methods
-  const llvm::Type *ConvertType(QualType T) {
+  llvm::Type *ConvertType(QualType T) {
     return CGM.getTypes().ConvertType(T);
   }
 
@@ -948,10 +962,9 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
   if (Success && !Result.HasSideEffects) {
     switch (Result.Val.getKind()) {
     case APValue::Uninitialized:
-      assert(0 && "Constant expressions should be initialized.");
-      return 0;
+      llvm_unreachable("Constant expressions should be initialized.");
     case APValue::LValue: {
-      const llvm::Type *DestTy = getTypes().ConvertTypeForMem(DestType);
+      llvm::Type *DestTy = getTypes().ConvertTypeForMem(DestType);
       llvm::Constant *Offset =
         llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
                                Result.Val.getLValueOffset().getQuantity());
@@ -962,9 +975,9 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
 
         // Apply offset if necessary.
         if (!Offset->isNullValue()) {
-          const llvm::Type *Type = llvm::Type::getInt8PtrTy(VMContext);
+          llvm::Type *Type = llvm::Type::getInt8PtrTy(VMContext);
           llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(C, Type);
-          Casted = llvm::ConstantExpr::getGetElementPtr(Casted, &Offset, 1);
+          Casted = llvm::ConstantExpr::getGetElementPtr(Casted, Offset);
           C = llvm::ConstantExpr::getBitCast(Casted, C->getType());
         }
 
@@ -994,7 +1007,7 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
                                                  Result.Val.getInt());
 
       if (C->getType()->isIntegerTy(1)) {
-        const llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
+        llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
         C = llvm::ConstantExpr::getZExt(C, BoolTy);
       }
       return C;
@@ -1013,8 +1026,13 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
                                                     NULL);
       return llvm::ConstantStruct::get(STy, Complex);
     }
-    case APValue::Float:
-      return llvm::ConstantFP::get(VMContext, Result.Val.getFloat());
+    case APValue::Float: {
+      const llvm::APFloat &Init = Result.Val.getFloat();
+      if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf)
+        return llvm::ConstantInt::get(VMContext, Init.bitcastToAPInt());
+      else
+        return llvm::ConstantFP::get(VMContext, Init);
+    }
     case APValue::ComplexFloat: {
       llvm::Constant *Complex[2];
 
@@ -1030,7 +1048,7 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
       return llvm::ConstantStruct::get(STy, Complex);
     }
     case APValue::Vector: {
-      llvm::SmallVector<llvm::Constant *, 4> Inits;
+      SmallVector<llvm::Constant *, 4> Inits;
       unsigned NumElts = Result.Val.getVectorLength();
 
       if (Context.getLangOptions().AltiVec &&
@@ -1064,7 +1082,7 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
 
   llvm::Constant* C = ConstExprEmitter(*this, CGF).Visit(const_cast<Expr*>(E));
   if (C && C->getType()->isIntegerTy(1)) {
-    const llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
+    llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
     C = llvm::ConstantExpr::getZExt(C, BoolTy);
   }
   return C;
@@ -1181,14 +1199,14 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
 }
 
 static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
-                                               const llvm::Type *baseType,
+                                               llvm::Type *baseType,
                                                const CXXRecordDecl *base);
 
 static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
                                         const CXXRecordDecl *record,
                                         bool asCompleteObject) {
   const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record);
-  const llvm::StructType *structure =
+  llvm::StructType *structure =
     (asCompleteObject ? layout.getLLVMType()
                       : layout.getBaseSubobjectLLVMType());
 
@@ -1212,7 +1230,7 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
       continue;
     
     unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(base);
-    const llvm::Type *baseType = structure->getElementType(fieldIndex);
+    llvm::Type *baseType = structure->getElementType(fieldIndex);
     elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
   }
 
@@ -1245,7 +1263,7 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
       // We might have already laid this field out.
       if (elements[fieldIndex]) continue;
 
-      const llvm::Type *baseType = structure->getElementType(fieldIndex);
+      llvm::Type *baseType = structure->getElementType(fieldIndex);
       elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
     }
   }
@@ -1261,7 +1279,7 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
 
 /// Emit the null constant for a base subobject.
 static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
-                                               const llvm::Type *baseType,
+                                               llvm::Type *baseType,
                                                const CXXRecordDecl *base) {
   const CGRecordLayout &baseLayout = CGM.getTypes().getCGRecordLayout(base);
 
@@ -1277,7 +1295,7 @@ static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
   // Otherwise, some bases are represented as arrays of i8 if the size
   // of the base is smaller than its corresponding LLVM type.  Figure
   // out how many elements this base array has.
-  const llvm::ArrayType *baseArrayType = cast<llvm::ArrayType>(baseType);
+  llvm::ArrayType *baseArrayType = cast<llvm::ArrayType>(baseType);
   unsigned numBaseElements = baseArrayType->getNumElements();
 
   // Fill in null data member pointers.
@@ -1287,7 +1305,7 @@ static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
 
   // Now go through all other elements and zero them out.
   if (numBaseElements) {
-    const llvm::Type *i8 = llvm::Type::getInt8Ty(CGM.getLLVMContext());
+    llvm::Type *i8 = llvm::Type::getInt8Ty(CGM.getLLVMContext());
     llvm::Constant *i8_zero = llvm::Constant::getNullValue(i8);
     for (unsigned i = 0; i != numBaseElements; ++i) {
       if (!baseElements[i])
@@ -1312,7 +1330,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
     for (unsigned i = 0; i != NumElements; ++i)
       Array[i] = Element;
 
-    const llvm::ArrayType *ATy =
+    llvm::ArrayType *ATy =
       cast<llvm::ArrayType>(getTypes().ConvertTypeForMem(T));
     return llvm::ConstantArray::get(ATy, Array);
   }
@@ -1330,3 +1348,8 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
   //   A NULL pointer is represented as -1.
   return getCXXABI().EmitNullMemberPointer(T->castAs<MemberPointerType>());
 }
+
+llvm::Constant *
+CodeGenModule::EmitNullConstantForBase(const CXXRecordDecl *Record) {
+  return ::EmitNullConstant(*this, Record, false);
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp
index a73e667e780e..3a9fbeed9d69 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp
@@ -78,7 +78,7 @@ class ScalarExprEmitter
     return I;
   }
 
-  const llvm::Type *ConvertType(QualType T) { return CGF.ConvertType(T); }
+  llvm::Type *ConvertType(QualType T) { return CGF.ConvertType(T); }
   LValue EmitLValue(const Expr *E) { return CGF.EmitLValue(E); }
   LValue EmitCheckedLValue(const Expr *E) { return CGF.EmitCheckedLValue(E); }
 
@@ -153,8 +153,7 @@ class ScalarExprEmitter
     
   Value *VisitStmt(Stmt *S) {
     S->dump(CGF.getContext().getSourceManager());
-    assert(0 && "Stmt can't have complex result type!");
-    return 0;
+    llvm_unreachable("Stmt can't have complex result type!");
   }
   Value *VisitExpr(Expr *S);
   
@@ -343,6 +342,10 @@ class ScalarExprEmitter
   }
     
   // C++
+  Value *VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E) {
+    return EmitLoadOfLValue(E);
+  }
+    
   Value *VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
     return Visit(DAE->getExpr());
   }
@@ -510,6 +513,7 @@ class ScalarExprEmitter
     return CGF.EmitObjCStringLiteral(E);
   }
   Value *VisitAsTypeExpr(AsTypeExpr *CE);
+  Value *VisitAtomicExpr(AtomicExpr *AE);
 };
 }  // end anonymous namespace.
 
@@ -548,14 +552,24 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
 
   if (DstType->isVoidType()) return 0;
 
+  llvm::Type *SrcTy = Src->getType();
+
+  // Floating casts might be a bit special: if we're doing casts to / from half
+  // FP, we should go via special intrinsics.
+  if (SrcType->isHalfType()) {
+    Src = Builder.CreateCall(CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16), Src);
+    SrcType = CGF.getContext().FloatTy;
+    SrcTy = llvm::Type::getFloatTy(VMContext);
+  }
+
   // Handle conversions to bool first, they are special: comparisons against 0.
   if (DstType->isBooleanType())
     return EmitConversionToBool(Src, SrcType);
 
-  const llvm::Type *DstTy = ConvertType(DstType);
+  llvm::Type *DstTy = ConvertType(DstType);
 
   // Ignore conversions like int -> uint.
-  if (Src->getType() == DstTy)
+  if (SrcTy == DstTy)
     return Src;
 
   // Handle pointer conversions next: pointers can only be converted to/from
@@ -563,13 +577,13 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
   // some native types (like Obj-C id) may map to a pointer type.
   if (isa<llvm::PointerType>(DstTy)) {
     // The source value may be an integer, or a pointer.
-    if (isa<llvm::PointerType>(Src->getType()))
+    if (isa<llvm::PointerType>(SrcTy))
       return Builder.CreateBitCast(Src, DstTy, "conv");
 
     assert(SrcType->isIntegerType() && "Not ptr->ptr or int->ptr conversion?");
     // First, convert to the correct width so that we control the kind of
     // extension.
-    const llvm::Type *MiddleTy = CGF.IntPtrTy;
+    llvm::Type *MiddleTy = CGF.IntPtrTy;
     bool InputSigned = SrcType->isSignedIntegerOrEnumerationType();
     llvm::Value* IntResult =
         Builder.CreateIntCast(Src, MiddleTy, InputSigned, "conv");
@@ -577,7 +591,7 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
     return Builder.CreateIntToPtr(IntResult, DstTy, "conv");
   }
 
-  if (isa<llvm::PointerType>(Src->getType())) {
+  if (isa<llvm::PointerType>(SrcTy)) {
     // Must be an ptr to int cast.
     assert(isa<llvm::IntegerType>(DstTy) && "not ptr->int?");
     return Builder.CreatePtrToInt(Src, DstTy, "conv");
@@ -592,10 +606,10 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
     // Insert the element in element zero of an undef vector
     llvm::Value *UnV = llvm::UndefValue::get(DstTy);
     llvm::Value *Idx = Builder.getInt32(0);
-    UnV = Builder.CreateInsertElement(UnV, Elt, Idx, "tmp");
+    UnV = Builder.CreateInsertElement(UnV, Elt, Idx);
 
     // Splat the element across to all elements
-    llvm::SmallVector<llvm::Constant*, 16> Args;
+    SmallVector<llvm::Constant*, 16> Args;
     unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
     for (unsigned i = 0; i != NumElements; ++i)
       Args.push_back(Builder.getInt32(0));
@@ -606,34 +620,47 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
   }
 
   // Allow bitcast from vector to integer/fp of the same size.
-  if (isa<llvm::VectorType>(Src->getType()) ||
+  if (isa<llvm::VectorType>(SrcTy) ||
       isa<llvm::VectorType>(DstTy))
     return Builder.CreateBitCast(Src, DstTy, "conv");
 
   // Finally, we have the arithmetic types: real int/float.
-  if (isa<llvm::IntegerType>(Src->getType())) {
+  Value *Res = NULL;
+  llvm::Type *ResTy = DstTy;
+
+  // Cast to half via float
+  if (DstType->isHalfType())
+    DstTy = llvm::Type::getFloatTy(VMContext);
+
+  if (isa<llvm::IntegerType>(SrcTy)) {
     bool InputSigned = SrcType->isSignedIntegerOrEnumerationType();
     if (isa<llvm::IntegerType>(DstTy))
-      return Builder.CreateIntCast(Src, DstTy, InputSigned, "conv");
+      Res = Builder.CreateIntCast(Src, DstTy, InputSigned, "conv");
     else if (InputSigned)
-      return Builder.CreateSIToFP(Src, DstTy, "conv");
+      Res = Builder.CreateSIToFP(Src, DstTy, "conv");
     else
-      return Builder.CreateUIToFP(Src, DstTy, "conv");
-  }
-
-  assert(Src->getType()->isFloatingPointTy() && "Unknown real conversion");
-  if (isa<llvm::IntegerType>(DstTy)) {
+      Res = Builder.CreateUIToFP(Src, DstTy, "conv");
+  } else if (isa<llvm::IntegerType>(DstTy)) {
+    assert(SrcTy->isFloatingPointTy() && "Unknown real conversion");
     if (DstType->isSignedIntegerOrEnumerationType())
-      return Builder.CreateFPToSI(Src, DstTy, "conv");
+      Res = Builder.CreateFPToSI(Src, DstTy, "conv");
     else
-      return Builder.CreateFPToUI(Src, DstTy, "conv");
+      Res = Builder.CreateFPToUI(Src, DstTy, "conv");
+  } else {
+    assert(SrcTy->isFloatingPointTy() && DstTy->isFloatingPointTy() &&
+           "Unknown real conversion");
+    if (DstTy->getTypeID() < SrcTy->getTypeID())
+      Res = Builder.CreateFPTrunc(Src, DstTy, "conv");
+    else
+      Res = Builder.CreateFPExt(Src, DstTy, "conv");
   }
 
-  assert(DstTy->isFloatingPointTy() && "Unknown real conversion");
-  if (DstTy->getTypeID() < Src->getType()->getTypeID())
-    return Builder.CreateFPTrunc(Src, DstTy, "conv");
-  else
-    return Builder.CreateFPExt(Src, DstTy, "conv");
+  if (DstTy != ResTy) {
+    assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion");
+    Res = Builder.CreateCall(CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16), Res);
+  }
+
+  return Res;
 }
 
 /// EmitComplexToScalarConversion - Emit a conversion from the specified complex
@@ -686,14 +713,14 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
     Value *RHS = CGF.EmitScalarExpr(E->getExpr(1));
     Value *Mask;
     
-    const llvm::VectorType *LTy = cast<llvm::VectorType>(LHS->getType());
+    llvm::VectorType *LTy = cast<llvm::VectorType>(LHS->getType());
     unsigned LHSElts = LTy->getNumElements();
 
     if (E->getNumSubExprs() == 3) {
       Mask = CGF.EmitScalarExpr(E->getExpr(2));
       
       // Shuffle LHS & RHS into one input vector.
-      llvm::SmallVector<llvm::Constant*, 32> concat;
+      SmallVector<llvm::Constant*, 32> concat;
       for (unsigned i = 0; i != LHSElts; ++i) {
         concat.push_back(Builder.getInt32(2*i));
         concat.push_back(Builder.getInt32(2*i+1));
@@ -706,7 +733,7 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
       Mask = RHS;
     }
     
-    const llvm::VectorType *MTy = cast<llvm::VectorType>(Mask->getType());
+    llvm::VectorType *MTy = cast<llvm::VectorType>(Mask->getType());
     llvm::Constant* EltMask;
     
     // Treat vec3 like vec4.
@@ -721,7 +748,7 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
                                        (1 << llvm::Log2_32(LHSElts))-1);
              
     // Mask off the high bits of each shuffle index.
-    llvm::SmallVector<llvm::Constant *, 32> MaskV;
+    SmallVector<llvm::Constant *, 32> MaskV;
     for (unsigned i = 0, e = MTy->getNumElements(); i != e; ++i)
       MaskV.push_back(EltMask);
     
@@ -734,7 +761,7 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
     //   n = extract mask i
     //   x = extract val n
     //   newv = insert newv, x, i
-    const llvm::VectorType *RTy = llvm::VectorType::get(LTy->getElementType(),
+    llvm::VectorType *RTy = llvm::VectorType::get(LTy->getElementType(),
                                                         MTy->getNumElements());
     Value* NewV = llvm::UndefValue::get(RTy);
     for (unsigned i = 0, e = MTy->getNumElements(); i != e; ++i) {
@@ -760,8 +787,8 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
   Value* V2 = CGF.EmitScalarExpr(E->getExpr(1));
   
   // Handle vec3 special since the index will be off by one for the RHS.
-  const llvm::VectorType *VTy = cast<llvm::VectorType>(V1->getType());
-  llvm::SmallVector<llvm::Constant*, 32> indices;
+  llvm::VectorType *VTy = cast<llvm::VectorType>(V1->getType());
+  SmallVector<llvm::Constant*, 32> indices;
   for (unsigned i = 2; i < E->getNumSubExprs(); i++) {
     unsigned Idx = E->getShuffleMaskIdx(CGF.getContext(), i-2);
     if (VTy->getNumElements() == 3 && Idx > 3)
@@ -815,7 +842,7 @@ Value *ScalarExprEmitter::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
 }
 
 static llvm::Constant *getMaskElt(llvm::ShuffleVectorInst *SVI, unsigned Idx,
-                                  unsigned Off, const llvm::Type *I32Ty) {
+                                  unsigned Off, llvm::Type *I32Ty) {
   int MV = SVI->getMaskValue(Idx);
   if (MV == -1) 
     return llvm::UndefValue::get(I32Ty);
@@ -831,12 +858,17 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
   if (E->hadArrayRangeDesignator())
     CGF.ErrorUnsupported(E, "GNU array range designator extension");
   
-  const llvm::VectorType *VType =
+  llvm::VectorType *VType =
     dyn_cast<llvm::VectorType>(ConvertType(E->getType()));
   
-  // We have a scalar in braces. Just use the first element.
-  if (!VType)
+  if (!VType) {
+    if (NumInitElements == 0) {
+      // C++11 value-initialization for the scalar.
+      return EmitNullValue(E->getType());
+    }
+    // We have a scalar in braces. Just use the first element.
     return Visit(E->getInit(0));
+  }
   
   unsigned ResElts = VType->getNumElements();
   
@@ -851,9 +883,9 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
   for (unsigned i = 0; i != NumInitElements; ++i) {
     Expr *IE = E->getInit(i);
     Value *Init = Visit(IE);
-    llvm::SmallVector<llvm::Constant*, 16> Args;
+    SmallVector<llvm::Constant*, 16> Args;
     
-    const llvm::VectorType *VVT = dyn_cast<llvm::VectorType>(Init->getType());
+    llvm::VectorType *VVT = dyn_cast<llvm::VectorType>(Init->getType());
     
     // Handle scalar elements.  If the scalar initializer is actually one
     // element of a different vector of the same width, use shuffle instead of 
@@ -911,7 +943,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
     if (isa<ExtVectorElementExpr>(IE)) {
       llvm::ShuffleVectorInst *SVI = cast<llvm::ShuffleVectorInst>(Init);
       Value *SVOp = SVI->getOperand(0);
-      const llvm::VectorType *OpTy = cast<llvm::VectorType>(SVOp->getType());
+      llvm::VectorType *OpTy = cast<llvm::VectorType>(SVOp->getType());
       
       if (OpTy->getNumElements() == ResElts) {
         for (unsigned j = 0; j != CurIdx; ++j) {
@@ -968,7 +1000,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
   
   // FIXME: evaluate codegen vs. shuffling against constant null vector.
   // Emit remaining default initializers.
-  const llvm::Type *EltTy = VType->getElementType();
+  llvm::Type *EltTy = VType->getElementType();
   
   // Emit remaining default initializers
   for (/* Do not initialize i*/; CurIdx < ResElts; ++CurIdx) {
@@ -1023,8 +1055,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
                           ConvertType(CGF.getContext().getPointerType(DestTy)));
     return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy));
   }
-      
-  case CK_AnyPointerToObjCPointerCast:
+
+  case CK_CPointerToObjCPointerCast:
+  case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_BitCast: {
     Value *Src = Visit(const_cast<Expr*>(E));
@@ -1075,7 +1108,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
       V = Builder.CreateStructGEP(V, 0, "arraydecay");
     }
 
-    return V;
+    // Make sure the array decay ends up being the right type.  This matters if
+    // the array type was of an incomplete type.
+    return CGF.Builder.CreateBitCast(V, ConvertType(CE->getType()));
   }
   case CK_FunctionToPointerDecay:
     return EmitLValue(E).getAddress();
@@ -1108,15 +1143,17 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     return CGF.CGM.getCXXABI().EmitMemberPointerConversion(CGF, CE, Src);
   }
 
-  case CK_ObjCProduceObject:
+  case CK_ARCProduceObject:
     return CGF.EmitARCRetainScalarExpr(E);
-  case CK_ObjCConsumeObject:
+  case CK_ARCConsumeObject:
     return CGF.EmitObjCConsumeObject(E->getType(), Visit(E));
-  case CK_ObjCReclaimReturnedObject: {
+  case CK_ARCReclaimReturnedObject: {
     llvm::Value *value = Visit(E);
     value = CGF.EmitARCRetainAutoreleasedReturnValue(value);
     return CGF.EmitObjCConsumeObject(E->getType(), value);
   }
+  case CK_ARCExtendBlockObject:
+    return CGF.EmitARCExtendBlockObject(E);
 
   case CK_FloatingRealToComplex:
   case CK_FloatingComplexCast:
@@ -1147,7 +1184,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
 
     // First, convert to the correct width so that we control the kind of
     // extension.
-    const llvm::Type *MiddleTy = CGF.IntPtrTy;
+    llvm::Type *MiddleTy = CGF.IntPtrTy;
     bool InputSigned = E->getType()->isSignedIntegerOrEnumerationType();
     llvm::Value* IntResult =
       Builder.CreateIntCast(Src, MiddleTy, InputSigned, "conv");
@@ -1163,16 +1200,16 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     return 0;
   }
   case CK_VectorSplat: {
-    const llvm::Type *DstTy = ConvertType(DestTy);
+    llvm::Type *DstTy = ConvertType(DestTy);
     Value *Elt = Visit(const_cast<Expr*>(E));
 
     // Insert the element in element zero of an undef vector
     llvm::Value *UnV = llvm::UndefValue::get(DstTy);
     llvm::Value *Idx = Builder.getInt32(0);
-    UnV = Builder.CreateInsertElement(UnV, Elt, Idx, "tmp");
+    UnV = Builder.CreateInsertElement(UnV, Elt, Idx);
 
     // Splat the element across to all elements
-    llvm::SmallVector<llvm::Constant*, 16> Args;
+    SmallVector<llvm::Constant*, 16> Args;
     unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
     llvm::Constant *Zero = Builder.getInt32(0);
     for (unsigned i = 0; i < NumElements; i++)
@@ -1188,7 +1225,6 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_FloatingToIntegral:
   case CK_FloatingCast:
     return EmitScalarConversion(Visit(E), E->getType(), DestTy);
-
   case CK_IntegralToBoolean:
     return EmitIntToBoolConversion(Visit(E));
   case CK_PointerToBoolean:
@@ -1253,10 +1289,8 @@ EmitAddConsiderOverflowBehavior(const UnaryOperator *E,
     BinOp.Opcode = BO_Add;
     BinOp.E = E;
     return EmitOverflowCheckedBinOp(BinOp);
-    break;
   }
-  assert(false && "Unknown SignedOverflowBehaviorTy");
-  return 0;
+  llvm_unreachable("Unknown SignedOverflowBehaviorTy");
 }
 
 llvm::Value *
@@ -1344,6 +1378,14 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
   } else if (type->isRealFloatingType()) {
     // Add the inc/dec to the real part.
     llvm::Value *amt;
+
+    if (type->isHalfType()) {
+      // Another special case: half FP increment should be done via float
+      value =
+    Builder.CreateCall(CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16),
+                       input);
+    }
+
     if (value->getType()->isFloatTy())
       amt = llvm::ConstantFP::get(VMContext,
                                   llvm::APFloat(static_cast<float>(amount)));
@@ -1359,6 +1401,11 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     }
     value = Builder.CreateFAdd(value, amt, isInc ? "inc" : "dec");
 
+    if (type->isHalfType())
+      value =
+       Builder.CreateCall(CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16),
+                          value);
+
   // Objective-C pointer types.
   } else {
     const ObjCObjectPointerType *OPT = type->castAs<ObjCObjectPointerType>();
@@ -1375,13 +1422,13 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
       value = Builder.CreateInBoundsGEP(value, sizeValue, "incdec.objptr");
     value = Builder.CreateBitCast(value, input->getType());
   }
-  
+
   // Store the updated result through the lvalue.
   if (LV.isBitField())
     CGF.EmitStoreThroughBitfieldLValue(RValue::get(value), LV, &value);
   else
     CGF.EmitStoreThroughLValue(RValue::get(value), LV);
-  
+
   // If this is a postinc, return the value read from memory, otherwise use the
   // updated value.
   return isPre ? value : input;
@@ -1432,7 +1479,7 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
 
   // Loop over the components of the offsetof to compute the value.
   unsigned n = E->getNumComponents();
-  const llvm::Type* ResultType = ConvertType(E->getType());
+  llvm::Type* ResultType = ConvertType(E->getType());
   llvm::Value* Result = llvm::Constant::getNullValue(ResultType);
   QualType CurrentType = E->getTypeSourceInfo()->getType();
   for (unsigned i = 0; i != n; ++i) {
@@ -1686,7 +1733,7 @@ void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck(
                          llvm::next(insertPt));
   llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn);
 
-  const llvm::IntegerType *Ty = cast<llvm::IntegerType>(Zero->getType());
+  llvm::IntegerType *Ty = cast<llvm::IntegerType>(Zero->getType());
 
   if (Ops.Ty->hasSignedIntegerRepresentation()) {
     llvm::Value *IntMin =
@@ -1769,7 +1816,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
     IID = llvm::Intrinsic::smul_with_overflow;
     break;
   default:
-    assert(false && "Unsupported operation for overflow detection");
+    llvm_unreachable("Unsupported operation for overflow detection");
     IID = 0;
   }
   OpID <<= 1;
@@ -2065,7 +2112,7 @@ enum IntrinsicType { VCMPEQ, VCMPGT };
 static llvm::Intrinsic::ID GetIntrinsic(IntrinsicType IT,
                                         BuiltinType::Kind ElemKind) {
   switch (ElemKind) {
-  default: assert(0 && "unexpected element type");
+  default: llvm_unreachable("unexpected element type");
   case BuiltinType::Char_U:
   case BuiltinType::UChar:
     return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequb_p :
@@ -2135,7 +2182,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
       BuiltinType::Kind ElementKind = BTy->getKind();
 
       switch(E->getOpcode()) {
-      default: assert(0 && "is not a comparison operation");
+      default: llvm_unreachable("is not a comparison operation");
       case BO_EQ:
         CR6 = CR6_LT;
         ID = GetIntrinsic(VCMPEQ, ElementKind);
@@ -2294,7 +2341,7 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
 }
 
 Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
-  const llvm::Type *ResTy = ConvertType(E->getType());
+  llvm::Type *ResTy = ConvertType(E->getType());
   
   // If we have 0 && RHS, see if we can elide RHS, if so, just return 0.
   // If we have 1 && X, just emit X without inserting the control flow.
@@ -2349,7 +2396,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
 }
 
 Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
-  const llvm::Type *ResTy = ConvertType(E->getType());
+  llvm::Type *ResTy = ConvertType(E->getType());
   
   // If we have 1 || RHS, see if we can elide RHS, if so, just return 1.
   // If we have 0 || X, just emit X without inserting the control flow.
@@ -2471,11 +2518,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
     llvm::Value *LHS = Visit(lhsExpr);
     llvm::Value *RHS = Visit(rhsExpr);
     
-    const llvm::Type *condType = ConvertType(condExpr->getType());
-    const llvm::VectorType *vecTy = cast<llvm::VectorType>(condType);
+    llvm::Type *condType = ConvertType(condExpr->getType());
+    llvm::VectorType *vecTy = cast<llvm::VectorType>(condType);
     
     unsigned numElem = vecTy->getNumElements();      
-    const llvm::Type *elemType = vecTy->getElementType();
+    llvm::Type *elemType = vecTy->getElementType();
     
     std::vector<llvm::Constant*> Zvals;
     for (unsigned i = 0; i < numElem; ++i)
@@ -2493,7 +2540,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
     llvm::Value *RHSTmp = RHS;
     llvm::Value *LHSTmp = LHS;
     bool wasCast = false;
-    const llvm::VectorType *rhsVTy = cast<llvm::VectorType>(RHS->getType());
+    llvm::VectorType *rhsVTy = cast<llvm::VectorType>(RHS->getType());
     if (rhsVTy->getElementType()->isFloatTy()) {
       RHSTmp = Builder.CreateBitCast(RHS, tmp2->getType());
       LHSTmp = Builder.CreateBitCast(LHS, tmp->getType());
@@ -2578,11 +2625,11 @@ Value *ScalarExprEmitter::VisitBlockExpr(const BlockExpr *block) {
 
 Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) {
   Value *Src  = CGF.EmitScalarExpr(E->getSrcExpr());
-  const llvm::Type *DstTy = ConvertType(E->getType());
+  llvm::Type *DstTy = ConvertType(E->getType());
   
   // Going from vec4->vec3 or vec3->vec4 is a special case and requires
   // a shuffle vector instead of a bitcast.
-  const llvm::Type *SrcTy = Src->getType();
+  llvm::Type *SrcTy = Src->getType();
   if (isa<llvm::VectorType>(DstTy) && isa<llvm::VectorType>(SrcTy)) {
     unsigned numElementsDst = cast<llvm::VectorType>(DstTy)->getNumElements();
     unsigned numElementsSrc = cast<llvm::VectorType>(SrcTy)->getNumElements();
@@ -2592,15 +2639,15 @@ Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) {
       
       // In the case of going from int4->float3, a bitcast is needed before
       // doing a shuffle.
-      const llvm::Type *srcElemTy = 
+      llvm::Type *srcElemTy = 
       cast<llvm::VectorType>(SrcTy)->getElementType();
-      const llvm::Type *dstElemTy = 
+      llvm::Type *dstElemTy = 
       cast<llvm::VectorType>(DstTy)->getElementType();
       
       if ((srcElemTy->isIntegerTy() && dstElemTy->isFloatTy())
           || (srcElemTy->isFloatTy() && dstElemTy->isIntegerTy())) {
         // Create a float type of the same size as the source or destination.
-        const llvm::VectorType *newSrcTy = llvm::VectorType::get(dstElemTy,
+        llvm::VectorType *newSrcTy = llvm::VectorType::get(dstElemTy,
                                                                  numElementsSrc);
         
         Src = Builder.CreateBitCast(Src, newSrcTy, "astypeCast");
@@ -2608,7 +2655,7 @@ Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) {
       
       llvm::Value *UnV = llvm::UndefValue::get(Src->getType());
       
-      llvm::SmallVector<llvm::Constant*, 3> Args;
+      SmallVector<llvm::Constant*, 3> Args;
       Args.push_back(Builder.getInt32(0));
       Args.push_back(Builder.getInt32(1));
       Args.push_back(Builder.getInt32(2));
@@ -2626,6 +2673,10 @@ Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) {
   return Builder.CreateBitCast(Src, DstTy, "astype");
 }
 
+Value *ScalarExprEmitter::VisitAtomicExpr(AtomicExpr *E) {
+  return CGF.EmitAtomicExpr(E).getScalarVal();
+}
+
 //===----------------------------------------------------------------------===//
 //                         Entry Point into this File
 //===----------------------------------------------------------------------===//
@@ -2678,7 +2729,7 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
   // object->isa or (*object).isa
   // Generate code as for: *(Class*)object
   // build Class* type
-  const llvm::Type *ClassPtrTy = ConvertType(E->getType());
+  llvm::Type *ClassPtrTy = ConvertType(E->getType());
 
   Expr *BaseExpr = E->getBase();
   if (BaseExpr->isRValue()) {
@@ -2744,8 +2795,7 @@ LValue CodeGenFunction::EmitCompoundAssignmentLValue(
   case BO_LOr:
   case BO_Assign:
   case BO_Comma:
-    assert(false && "Not valid compound assignment operators");
-    break;
+    llvm_unreachable("Not valid compound assignment operators");
   }
    
   llvm_unreachable("Unhandled compound assignment operator");
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp
index 426cca00140c..51f20534d116 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp
@@ -33,7 +33,7 @@ tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e);
 /// Given the address of a variable of pointer type, find the correct
 /// null to store into it.
 static llvm::Constant *getNullForVariable(llvm::Value *addr) {
-  const llvm::Type *type =
+  llvm::Type *type =
     cast<llvm::PointerType>(addr->getType())->getElementType();
   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(type));
 }
@@ -80,6 +80,53 @@ static RValue AdjustRelatedResultType(CodeGenFunction &CGF,
                                                CGF.ConvertType(E->getType())));
 }
 
+/// Decide whether to extend the lifetime of the receiver of a
+/// returns-inner-pointer message.
+static bool
+shouldExtendReceiverForInnerPointerMessage(const ObjCMessageExpr *message) {
+  switch (message->getReceiverKind()) {
+
+  // For a normal instance message, we should extend unless the
+  // receiver is loaded from a variable with precise lifetime.
+  case ObjCMessageExpr::Instance: {
+    const Expr *receiver = message->getInstanceReceiver();
+    const ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(receiver);
+    if (!ice || ice->getCastKind() != CK_LValueToRValue) return true;
+    receiver = ice->getSubExpr()->IgnoreParens();
+
+    // Only __strong variables.
+    if (receiver->getType().getObjCLifetime() != Qualifiers::OCL_Strong)
+      return true;
+
+    // All ivars and fields have precise lifetime.
+    if (isa<MemberExpr>(receiver) || isa<ObjCIvarRefExpr>(receiver))
+      return false;
+
+    // Otherwise, check for variables.
+    const DeclRefExpr *declRef = dyn_cast<DeclRefExpr>(ice->getSubExpr());
+    if (!declRef) return true;
+    const VarDecl *var = dyn_cast<VarDecl>(declRef->getDecl());
+    if (!var) return true;
+
+    // All variables have precise lifetime except local variables with
+    // automatic storage duration that aren't specially marked.
+    return (var->hasLocalStorage() &&
+            !var->hasAttr<ObjCPreciseLifetimeAttr>());
+  }
+
+  case ObjCMessageExpr::Class:
+  case ObjCMessageExpr::SuperClass:
+    // It's never necessary for class objects.
+    return false;
+
+  case ObjCMessageExpr::SuperInstance:
+    // We generally assume that 'self' lives throughout a method call.
+    return false;
+  }
+
+  llvm_unreachable("invalid receiver kind");
+}
+
 RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
                                             ReturnValueSlot Return) {
   // Only the lookup mechanism and first two arguments of the method
@@ -88,6 +135,8 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
 
   bool isDelegateInit = E->isDelegateInitCall();
 
+  const ObjCMethodDecl *method = E->getMethodDecl();
+
   // We don't retain the receiver in delegate init calls, and this is
   // safe because the receiver value is always loaded from 'self',
   // which we zero out.  We don't want to Block_copy block receivers,
@@ -95,8 +144,8 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
   bool retainSelf =
     (!isDelegateInit &&
      CGM.getLangOptions().ObjCAutoRefCount &&
-     E->getMethodDecl() &&
-     E->getMethodDecl()->hasAttr<NSConsumesSelfAttr>());
+     method &&
+     method->hasAttr<NSConsumesSelfAttr>());
 
   CGObjCRuntime &Runtime = CGM.getObjCRuntime();
   bool isSuperMessage = false;
@@ -112,8 +161,7 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
       TryEmitResult ter = tryEmitARCRetainScalarExpr(*this,
                                                    E->getInstanceReceiver());
       Receiver = ter.getPointer();
-      if (!ter.getInt())
-        Receiver = EmitARCRetainNonBlock(Receiver);
+      if (ter.getInt()) retainSelf = false;
     } else
       Receiver = EmitScalarExpr(E->getInstanceReceiver());
     break;
@@ -126,9 +174,6 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
     assert(OID && "Invalid Objective-C class message send");
     Receiver = Runtime.GetClass(Builder, OID);
     isClassMessage = true;
-
-    if (retainSelf)
-      Receiver = EmitARCRetainNonBlock(Receiver);
     break;
   }
 
@@ -136,9 +181,6 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
     ReceiverType = E->getSuperType();
     Receiver = LoadObjCSelf();
     isSuperMessage = true;
-
-    if (retainSelf)
-      Receiver = EmitARCRetainNonBlock(Receiver);
     break;
 
   case ObjCMessageExpr::SuperClass:
@@ -146,17 +188,25 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
     Receiver = LoadObjCSelf();
     isSuperMessage = true;
     isClassMessage = true;
-
-    if (retainSelf)
-      Receiver = EmitARCRetainNonBlock(Receiver);
     break;
   }
 
+  if (retainSelf)
+    Receiver = EmitARCRetainNonBlock(Receiver);
+
+  // In ARC, we sometimes want to "extend the lifetime"
+  // (i.e. retain+autorelease) of receivers of returns-inner-pointer
+  // messages.
+  if (getLangOptions().ObjCAutoRefCount && method &&
+      method->hasAttr<ObjCReturnsInnerPointerAttr>() &&
+      shouldExtendReceiverForInnerPointerMessage(E))
+    Receiver = EmitARCRetainAutorelease(ReceiverType, Receiver);
+
   QualType ResultType =
-    E->getMethodDecl() ? E->getMethodDecl()->getResultType() : E->getType();
+    method ? method->getResultType() : E->getType();
 
   CallArgList Args;
-  EmitCallArgs(Args, E->getMethodDecl(), E->arg_begin(), E->arg_end());
+  EmitCallArgs(Args, method, E->arg_begin(), E->arg_end());
 
   // For delegate init calls in ARC, do an unsafe store of null into
   // self.  This represents the call taking direct ownership of that
@@ -189,12 +239,12 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
                                               Receiver,
                                               isClassMessage,
                                               Args,
-                                              E->getMethodDecl());
+                                              method);
   } else {
     result = Runtime.GenerateMessageSend(*this, Return, ResultType,
                                          E->getSelector(),
                                          Receiver, Args, OID,
-                                         E->getMethodDecl());
+                                         method);
   }
 
   // For delegate init calls in ARC, implicitly store the result of
@@ -206,14 +256,14 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
 
     // The delegate return type isn't necessarily a matching type; in
     // fact, it's quite likely to be 'id'.
-    const llvm::Type *selfTy =
+    llvm::Type *selfTy =
       cast<llvm::PointerType>(selfAddr->getType())->getElementType();
     newSelf = Builder.CreateBitCast(newSelf, selfTy);
 
     Builder.CreateStore(newSelf, selfAddr);
   }
 
-  return AdjustRelatedResultType(*this, E, E->getMethodDecl(), result);
+  return AdjustRelatedResultType(*this, E, method, result);
 }
 
 namespace {
@@ -263,7 +313,7 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
   args.push_back(OMD->getSelfDecl());
   args.push_back(OMD->getCmdDecl());
 
-  for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
+  for (ObjCMethodDecl::param_const_iterator PI = OMD->param_begin(),
        E = OMD->param_end(); PI != E; ++PI)
     args.push_back(*PI);
 
@@ -285,39 +335,6 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
 static llvm::Value *emitARCRetainLoadOfScalar(CodeGenFunction &CGF,
                                               LValue lvalue, QualType type);
 
-void CodeGenFunction::GenerateObjCGetterBody(ObjCIvarDecl *Ivar, 
-                                             bool IsAtomic, bool IsStrong) {
-  LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
-                                Ivar, 0);
-  llvm::Value *GetCopyStructFn =
-  CGM.getObjCRuntime().GetGetStructFunction();
-  CodeGenTypes &Types = CGM.getTypes();
-  // objc_copyStruct (ReturnValue, &structIvar, 
-  //                  sizeof (Type of Ivar), isAtomic, false);
-  CallArgList Args;
-  RValue RV = RValue::get(Builder.CreateBitCast(ReturnValue, VoidPtrTy));
-  Args.add(RV, getContext().VoidPtrTy);
-  RV = RValue::get(Builder.CreateBitCast(LV.getAddress(), VoidPtrTy));
-  Args.add(RV, getContext().VoidPtrTy);
-  // sizeof (Type of Ivar)
-  CharUnits Size =  getContext().getTypeSizeInChars(Ivar->getType());
-  llvm::Value *SizeVal =
-  llvm::ConstantInt::get(Types.ConvertType(getContext().LongTy),
-                         Size.getQuantity());
-  Args.add(RValue::get(SizeVal), getContext().LongTy);
-  llvm::Value *isAtomic =
-  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 
-                         IsAtomic ? 1 : 0);
-  Args.add(RValue::get(isAtomic), getContext().BoolTy);
-  llvm::Value *hasStrong =
-  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 
-                         IsStrong ? 1 : 0);
-  Args.add(RValue::get(hasStrong), getContext().BoolTy);
-  EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
-                                 FunctionType::ExtInfo()),
-           GetCopyStructFn, ReturnValueSlot(), Args);
-}
-
 /// Generate an Objective-C method.  An Objective-C method is a C function with
 /// its pointer, name, and types registered in the class struture.
 void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) {
@@ -326,218 +343,599 @@ void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) {
   FinishFunction(OMD->getBodyRBrace());
 }
 
-// FIXME: I wasn't sure about the synthesis approach. If we end up generating an
-// AST for the whole body we can just fall back to having a GenerateFunction
-// which takes the body Stmt.
+/// emitStructGetterCall - Call the runtime function to load a property
+/// into the return value slot.
+static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, 
+                                 bool isAtomic, bool hasStrong) {
+  ASTContext &Context = CGF.getContext();
+
+  llvm::Value *src =
+    CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(),
+                          ivar, 0).getAddress();
+
+  // objc_copyStruct (ReturnValue, &structIvar, 
+  //                  sizeof (Type of Ivar), isAtomic, false);
+  CallArgList args;
+
+  llvm::Value *dest = CGF.Builder.CreateBitCast(CGF.ReturnValue, CGF.VoidPtrTy);
+  args.add(RValue::get(dest), Context.VoidPtrTy);
+
+  src = CGF.Builder.CreateBitCast(src, CGF.VoidPtrTy);
+  args.add(RValue::get(src), Context.VoidPtrTy);
+
+  CharUnits size = CGF.getContext().getTypeSizeInChars(ivar->getType());
+  args.add(RValue::get(CGF.CGM.getSize(size)), Context.getSizeType());
+  args.add(RValue::get(CGF.Builder.getInt1(isAtomic)), Context.BoolTy);
+  args.add(RValue::get(CGF.Builder.getInt1(hasStrong)), Context.BoolTy);
+
+  llvm::Value *fn = CGF.CGM.getObjCRuntime().GetGetStructFunction();
+  CGF.EmitCall(CGF.getTypes().getFunctionInfo(Context.VoidTy, args,
+                                              FunctionType::ExtInfo()),
+               fn, ReturnValueSlot(), args);
+}
+
+/// Determine whether the given architecture supports unaligned atomic
+/// accesses.  They don't have to be fast, just faster than a function
+/// call and a mutex.
+static bool hasUnalignedAtomics(llvm::Triple::ArchType arch) {
+  // FIXME: Allow unaligned atomic load/store on x86.  (It is not
+  // currently supported by the backend.)
+  return 0;
+}
+
+/// Return the maximum size that permits atomic accesses for the given
+/// architecture.
+static CharUnits getMaxAtomicAccessSize(CodeGenModule &CGM,
+                                        llvm::Triple::ArchType arch) {
+  // ARM has 8-byte atomic accesses, but it's not clear whether we
+  // want to rely on them here.
+
+  // In the default case, just assume that any size up to a pointer is
+  // fine given adequate alignment.
+  return CharUnits::fromQuantity(CGM.PointerSizeInBytes);
+}
+
+namespace {
+  class PropertyImplStrategy {
+  public:
+    enum StrategyKind {
+      /// The 'native' strategy is to use the architecture's provided
+      /// reads and writes.
+      Native,
+
+      /// Use objc_setProperty and objc_getProperty.
+      GetSetProperty,
+
+      /// Use objc_setProperty for the setter, but use expression
+      /// evaluation for the getter.
+      SetPropertyAndExpressionGet,
+
+      /// Use objc_copyStruct.
+      CopyStruct,
+
+      /// The 'expression' strategy is to emit normal assignment or
+      /// lvalue-to-rvalue expressions.
+      Expression
+    };
+
+    StrategyKind getKind() const { return StrategyKind(Kind); }
+
+    bool hasStrongMember() const { return HasStrong; }
+    bool isAtomic() const { return IsAtomic; }
+    bool isCopy() const { return IsCopy; }
+
+    CharUnits getIvarSize() const { return IvarSize; }
+    CharUnits getIvarAlignment() const { return IvarAlignment; }
+
+    PropertyImplStrategy(CodeGenModule &CGM,
+                         const ObjCPropertyImplDecl *propImpl);
+
+  private:
+    unsigned Kind : 8;
+    unsigned IsAtomic : 1;
+    unsigned IsCopy : 1;
+    unsigned HasStrong : 1;
+
+    CharUnits IvarSize;
+    CharUnits IvarAlignment;
+  };
+}
+
+/// Pick an implementation strategy for the the given property synthesis.
+PropertyImplStrategy::PropertyImplStrategy(CodeGenModule &CGM,
+                                     const ObjCPropertyImplDecl *propImpl) {
+  const ObjCPropertyDecl *prop = propImpl->getPropertyDecl();
+  ObjCPropertyDecl::SetterKind setterKind = prop->getSetterKind();
+
+  IsCopy = (setterKind == ObjCPropertyDecl::Copy);
+  IsAtomic = prop->isAtomic();
+  HasStrong = false; // doesn't matter here.
+
+  // Evaluate the ivar's size and alignment.
+  ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
+  QualType ivarType = ivar->getType();
+  llvm::tie(IvarSize, IvarAlignment)
+    = CGM.getContext().getTypeInfoInChars(ivarType);
+
+  // If we have a copy property, we always have to use getProperty/setProperty.
+  // TODO: we could actually use setProperty and an expression for non-atomics.
+  if (IsCopy) {
+    Kind = GetSetProperty;
+    return;
+  }
+
+  // Handle retain.
+  if (setterKind == ObjCPropertyDecl::Retain) {
+    // In GC-only, there's nothing special that needs to be done.
+    if (CGM.getLangOptions().getGC() == LangOptions::GCOnly) {
+      // fallthrough
+
+    // In ARC, if the property is non-atomic, use expression emission,
+    // which translates to objc_storeStrong.  This isn't required, but
+    // it's slightly nicer.
+    } else if (CGM.getLangOptions().ObjCAutoRefCount && !IsAtomic) {
+      Kind = Expression;
+      return;
+
+    // Otherwise, we need to at least use setProperty.  However, if
+    // the property isn't atomic, we can use normal expression
+    // emission for the getter.
+    } else if (!IsAtomic) {
+      Kind = SetPropertyAndExpressionGet;
+      return;
+
+    // Otherwise, we have to use both setProperty and getProperty.
+    } else {
+      Kind = GetSetProperty;
+      return;
+    }
+  }
+
+  // If we're not atomic, just use expression accesses.
+  if (!IsAtomic) {
+    Kind = Expression;
+    return;
+  }
+
+  // Properties on bitfield ivars need to be emitted using expression
+  // accesses even if they're nominally atomic.
+  if (ivar->isBitField()) {
+    Kind = Expression;
+    return;
+  }
+
+  // GC-qualified or ARC-qualified ivars need to be emitted as
+  // expressions.  This actually works out to being atomic anyway,
+  // except for ARC __strong, but that should trigger the above code.
+  if (ivarType.hasNonTrivialObjCLifetime() ||
+      (CGM.getLangOptions().getGC() &&
+       CGM.getContext().getObjCGCAttrKind(ivarType))) {
+    Kind = Expression;
+    return;
+  }
+
+  // Compute whether the ivar has strong members.
+  if (CGM.getLangOptions().getGC())
+    if (const RecordType *recordType = ivarType->getAs<RecordType>())
+      HasStrong = recordType->getDecl()->hasObjectMember();
+
+  // We can never access structs with object members with a native
+  // access, because we need to use write barriers.  This is what
+  // objc_copyStruct is for.
+  if (HasStrong) {
+    Kind = CopyStruct;
+    return;
+  }
+
+  // Otherwise, this is target-dependent and based on the size and
+  // alignment of the ivar.
+
+  // If the size of the ivar is not a power of two, give up.  We don't
+  // want to get into the business of doing compare-and-swaps.
+  if (!IvarSize.isPowerOfTwo()) {
+    Kind = CopyStruct;
+    return;
+  }
+
+  llvm::Triple::ArchType arch =
+    CGM.getContext().getTargetInfo().getTriple().getArch();
+
+  // Most architectures require memory to fit within a single cache
+  // line, so the alignment has to be at least the size of the access.
+  // Otherwise we have to grab a lock.
+  if (IvarAlignment < IvarSize && !hasUnalignedAtomics(arch)) {
+    Kind = CopyStruct;
+    return;
+  }
+
+  // If the ivar's size exceeds the architecture's maximum atomic
+  // access size, we have to use CopyStruct.
+  if (IvarSize > getMaxAtomicAccessSize(CGM, arch)) {
+    Kind = CopyStruct;
+    return;
+  }
+
+  // Otherwise, we can use native loads and stores.
+  Kind = Native;
+}
 
 /// GenerateObjCGetter - Generate an Objective-C property getter
 /// function. The given Decl must be an ObjCImplementationDecl. @synthesize
 /// is illegal within a category.
 void CodeGenFunction::GenerateObjCGetter(ObjCImplementationDecl *IMP,
                                          const ObjCPropertyImplDecl *PID) {
-  ObjCIvarDecl *Ivar = PID->getPropertyIvarDecl();
   const ObjCPropertyDecl *PD = PID->getPropertyDecl();
-  bool IsAtomic =
-    !(PD->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_nonatomic);
   ObjCMethodDecl *OMD = PD->getGetterMethodDecl();
   assert(OMD && "Invalid call to generate getter (empty method)");
   StartObjCMethod(OMD, IMP->getClassInterface(), PID->getLocStart());
-  
-  // Determine if we should use an objc_getProperty call for
-  // this. Non-atomic properties are directly evaluated.
-  // atomic 'copy' and 'retain' properties are also directly
-  // evaluated in gc-only mode.
-  if (CGM.getLangOptions().getGCMode() != LangOptions::GCOnly &&
-      IsAtomic &&
-      (PD->getSetterKind() == ObjCPropertyDecl::Copy ||
-       PD->getSetterKind() == ObjCPropertyDecl::Retain)) {
-    llvm::Value *GetPropertyFn =
-      CGM.getObjCRuntime().GetPropertyGetFunction();
 
-    if (!GetPropertyFn) {
-      CGM.ErrorUnsupported(PID, "Obj-C getter requiring atomic copy");
-      FinishFunction();
+  generateObjCGetterBody(IMP, PID);
+
+  FinishFunction();
+}
+
+static bool hasTrivialGetExpr(const ObjCPropertyImplDecl *propImpl) {
+  const Expr *getter = propImpl->getGetterCXXConstructor();
+  if (!getter) return true;
+
+  // Sema only makes only of these when the ivar has a C++ class type,
+  // so the form is pretty constrained.
+
+  // If the property has a reference type, we might just be binding a
+  // reference, in which case the result will be a gl-value.  We should
+  // treat this as a non-trivial operation.
+  if (getter->isGLValue())
+    return false;
+
+  // If we selected a trivial copy-constructor, we're okay.
+  if (const CXXConstructExpr *construct = dyn_cast<CXXConstructExpr>(getter))
+    return (construct->getConstructor()->isTrivial());
+
+  // The constructor might require cleanups (in which case it's never
+  // trivial).
+  assert(isa<ExprWithCleanups>(getter));
+  return false;
+}
+
+void
+CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
+                                        const ObjCPropertyImplDecl *propImpl) {
+  // If there's a non-trivial 'get' expression, we just have to emit that.
+  if (!hasTrivialGetExpr(propImpl)) {
+    ReturnStmt ret(SourceLocation(), propImpl->getGetterCXXConstructor(),
+                   /*nrvo*/ 0);
+    EmitReturnStmt(ret);
+    return;
+  }
+
+  const ObjCPropertyDecl *prop = propImpl->getPropertyDecl();
+  QualType propType = prop->getType();
+  ObjCMethodDecl *getterMethod = prop->getGetterMethodDecl();
+
+  ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();  
+
+  // Pick an implementation strategy.
+  PropertyImplStrategy strategy(CGM, propImpl);
+  switch (strategy.getKind()) {
+  case PropertyImplStrategy::Native: {
+    LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, 0);
+
+    // Currently, all atomic accesses have to be through integer
+    // types, so there's no point in trying to pick a prettier type.
+    llvm::Type *bitcastType =
+      llvm::Type::getIntNTy(getLLVMContext(),
+                            getContext().toBits(strategy.getIvarSize()));
+    bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay
+
+    // Perform an atomic load.  This does not impose ordering constraints.
+    llvm::Value *ivarAddr = LV.getAddress();
+    ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType);
+    llvm::LoadInst *load = Builder.CreateLoad(ivarAddr, "load");
+    load->setAlignment(strategy.getIvarAlignment().getQuantity());
+    load->setAtomic(llvm::Unordered);
+
+    // Store that value into the return address.  Doing this with a
+    // bitcast is likely to produce some pretty ugly IR, but it's not
+    // the *most* terrible thing in the world.
+    Builder.CreateStore(load, Builder.CreateBitCast(ReturnValue, bitcastType));
+
+    // Make sure we don't do an autorelease.
+    AutoreleaseResult = false;
+    return;
+  }
+
+  case PropertyImplStrategy::GetSetProperty: {
+    llvm::Value *getPropertyFn =
+      CGM.getObjCRuntime().GetPropertyGetFunction();
+    if (!getPropertyFn) {
+      CGM.ErrorUnsupported(propImpl, "Obj-C getter requiring atomic copy");
       return;
     }
 
     // Return (ivar-type) objc_getProperty((id) self, _cmd, offset, true).
     // FIXME: Can't this be simpler? This might even be worse than the
     // corresponding gcc code.
-    CodeGenTypes &Types = CGM.getTypes();
-    ValueDecl *Cmd = OMD->getCmdDecl();
-    llvm::Value *CmdVal = Builder.CreateLoad(LocalDeclMap[Cmd], "cmd");
-    QualType IdTy = getContext().getObjCIdType();
-    llvm::Value *SelfAsId =
-      Builder.CreateBitCast(LoadObjCSelf(), Types.ConvertType(IdTy));
-    llvm::Value *Offset = EmitIvarOffset(IMP->getClassInterface(), Ivar);
-    llvm::Value *True =
-      llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 1);
-    CallArgList Args;
-    Args.add(RValue::get(SelfAsId), IdTy);
-    Args.add(RValue::get(CmdVal), Cmd->getType());
-    Args.add(RValue::get(Offset), getContext().getPointerDiffType());
-    Args.add(RValue::get(True), getContext().BoolTy);
+    llvm::Value *cmd =
+      Builder.CreateLoad(LocalDeclMap[getterMethod->getCmdDecl()], "cmd");
+    llvm::Value *self = Builder.CreateBitCast(LoadObjCSelf(), VoidPtrTy);
+    llvm::Value *ivarOffset =
+      EmitIvarOffset(classImpl->getClassInterface(), ivar);
+
+    CallArgList args;
+    args.add(RValue::get(self), getContext().getObjCIdType());
+    args.add(RValue::get(cmd), getContext().getObjCSelType());
+    args.add(RValue::get(ivarOffset), getContext().getPointerDiffType());
+    args.add(RValue::get(Builder.getInt1(strategy.isAtomic())),
+             getContext().BoolTy);
+
     // FIXME: We shouldn't need to get the function info here, the
     // runtime already should have computed it to build the function.
-    RValue RV = EmitCall(Types.getFunctionInfo(PD->getType(), Args,
-                                               FunctionType::ExtInfo()),
-                         GetPropertyFn, ReturnValueSlot(), Args);
+    RValue RV = EmitCall(getTypes().getFunctionInfo(propType, args,
+                                                    FunctionType::ExtInfo()),
+                         getPropertyFn, ReturnValueSlot(), args);
+
     // We need to fix the type here. Ivars with copy & retain are
     // always objects so we don't need to worry about complex or
     // aggregates.
     RV = RValue::get(Builder.CreateBitCast(RV.getScalarVal(),
-                                           Types.ConvertType(PD->getType())));
-    EmitReturnOfRValue(RV, PD->getType());
+                                           getTypes().ConvertType(propType)));
+
+    EmitReturnOfRValue(RV, propType);
 
     // objc_getProperty does an autorelease, so we should suppress ours.
     AutoreleaseResult = false;
-  } else {
-    const llvm::Triple &Triple = getContext().Target.getTriple();
-    QualType IVART = Ivar->getType();
-    if (IsAtomic &&
-        IVART->isScalarType() &&
-        (Triple.getArch() == llvm::Triple::arm ||
-         Triple.getArch() == llvm::Triple::thumb) &&
-        (getContext().getTypeSizeInChars(IVART) 
-         > CharUnits::fromQuantity(4)) &&
-        CGM.getObjCRuntime().GetGetStructFunction()) {
-      GenerateObjCGetterBody(Ivar, true, false);
-    }
-    else if (IsAtomic &&
-             (IVART->isScalarType() && !IVART->isRealFloatingType()) &&
-             Triple.getArch() == llvm::Triple::x86 &&
-             (getContext().getTypeSizeInChars(IVART) 
-              > CharUnits::fromQuantity(4)) &&
-             CGM.getObjCRuntime().GetGetStructFunction()) {
-      GenerateObjCGetterBody(Ivar, true, false);
-    }
-    else if (IsAtomic &&
-             (IVART->isScalarType() && !IVART->isRealFloatingType()) &&
-             Triple.getArch() == llvm::Triple::x86_64 &&
-             (getContext().getTypeSizeInChars(IVART) 
-              > CharUnits::fromQuantity(8)) &&
-             CGM.getObjCRuntime().GetGetStructFunction()) {
-      GenerateObjCGetterBody(Ivar, true, false);
-    }
-    else if (IVART->isAnyComplexType()) {
-      LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
-                                    Ivar, 0);
-      ComplexPairTy Pair = LoadComplexFromAddr(LV.getAddress(),
-                                               LV.isVolatileQualified());
-      StoreComplexToAddr(Pair, ReturnValue, LV.isVolatileQualified());
-    }
-    else if (hasAggregateLLVMType(IVART)) {
-      bool IsStrong = false;
-      if ((IsStrong = IvarTypeWithAggrGCObjects(IVART))
-          && CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect
-          && CGM.getObjCRuntime().GetGetStructFunction()) {
-        GenerateObjCGetterBody(Ivar, IsAtomic, IsStrong);
-      }
-      else {
-        const CXXRecordDecl *classDecl = IVART->getAsCXXRecordDecl();
-        
-        if (PID->getGetterCXXConstructor() &&
-            classDecl && !classDecl->hasTrivialDefaultConstructor()) {
-          ReturnStmt *Stmt = 
-            new (getContext()) ReturnStmt(SourceLocation(), 
-                                          PID->getGetterCXXConstructor(),
-                                          0);
-          EmitReturnStmt(*Stmt);
-        } else if (IsAtomic &&
-                   !IVART->isAnyComplexType() &&
-                   Triple.getArch() == llvm::Triple::x86 &&
-                   (getContext().getTypeSizeInChars(IVART) 
-                    > CharUnits::fromQuantity(4)) &&
-                   CGM.getObjCRuntime().GetGetStructFunction()) {
-          GenerateObjCGetterBody(Ivar, true, false);
-        }
-        else if (IsAtomic &&
-                 !IVART->isAnyComplexType() &&
-                 Triple.getArch() == llvm::Triple::x86_64 &&
-                 (getContext().getTypeSizeInChars(IVART) 
-                  > CharUnits::fromQuantity(8)) &&
-                 CGM.getObjCRuntime().GetGetStructFunction()) {
-          GenerateObjCGetterBody(Ivar, true, false);
-        }
-        else {
-          LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
-                                        Ivar, 0);
-          EmitAggregateCopy(ReturnValue, LV.getAddress(), IVART);
-        }
-      }
-    } 
-    else {
-        LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
-                                      Ivar, 0);
-        QualType propType = PD->getType();
 
-        llvm::Value *value;
-        if (propType->isReferenceType()) {
-          value = LV.getAddress();
-        } else {
-          // In ARC, we want to emit this retained.
-          if (getLangOptions().ObjCAutoRefCount &&
-              PD->getType()->isObjCRetainableType())
-            value = emitARCRetainLoadOfScalar(*this, LV, IVART);
-          else
-            value = EmitLoadOfLValue(LV).getScalarVal();
-
-          value = Builder.CreateBitCast(value, ConvertType(propType));
-        }
-
-        EmitReturnOfRValue(RValue::get(value), propType);
-    }
+    return;
   }
 
-  FinishFunction();
+  case PropertyImplStrategy::CopyStruct:
+    emitStructGetterCall(*this, ivar, strategy.isAtomic(),
+                         strategy.hasStrongMember());
+    return;
+
+  case PropertyImplStrategy::Expression:
+  case PropertyImplStrategy::SetPropertyAndExpressionGet: {
+    LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, 0);
+
+    QualType ivarType = ivar->getType();
+    if (ivarType->isAnyComplexType()) {
+      ComplexPairTy pair = LoadComplexFromAddr(LV.getAddress(),
+                                               LV.isVolatileQualified());
+      StoreComplexToAddr(pair, ReturnValue, LV.isVolatileQualified());
+    } else if (hasAggregateLLVMType(ivarType)) {
+      // The return value slot is guaranteed to not be aliased, but
+      // that's not necessarily the same as "on the stack", so
+      // we still potentially need objc_memmove_collectable.
+      EmitAggregateCopy(ReturnValue, LV.getAddress(), ivarType);
+    } else {
+      llvm::Value *value;
+      if (propType->isReferenceType()) {
+        value = LV.getAddress();
+      } else {
+        // We want to load and autoreleaseReturnValue ARC __weak ivars.
+        if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
+          value = emitARCRetainLoadOfScalar(*this, LV, ivarType);
+
+        // Otherwise we want to do a simple load, suppressing the
+        // final autorelease.
+        } else {
+          value = EmitLoadOfLValue(LV).getScalarVal();
+          AutoreleaseResult = false;
+        }
+
+        value = Builder.CreateBitCast(value, ConvertType(propType));
+      }
+      
+      EmitReturnOfRValue(RValue::get(value), propType);
+    }
+    return;
+  }
+
+  }
+  llvm_unreachable("bad @property implementation strategy!");
 }
 
-void CodeGenFunction::GenerateObjCAtomicSetterBody(ObjCMethodDecl *OMD,
-                                                   ObjCIvarDecl *Ivar) {
+/// emitStructSetterCall - Call the runtime function to store the value
+/// from the first formal parameter into the given ivar.
+static void emitStructSetterCall(CodeGenFunction &CGF, ObjCMethodDecl *OMD,
+                                 ObjCIvarDecl *ivar) {
   // objc_copyStruct (&structIvar, &Arg, 
   //                  sizeof (struct something), true, false);
-  llvm::Value *GetCopyStructFn =
-  CGM.getObjCRuntime().GetSetStructFunction();
-  CodeGenTypes &Types = CGM.getTypes();
-  CallArgList Args;
-  LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), Ivar, 0);
-  RValue RV =
-    RValue::get(Builder.CreateBitCast(LV.getAddress(),
-                Types.ConvertType(getContext().VoidPtrTy)));
-  Args.add(RV, getContext().VoidPtrTy);
-  llvm::Value *Arg = LocalDeclMap[*OMD->param_begin()];
-  llvm::Value *ArgAsPtrTy =
-  Builder.CreateBitCast(Arg,
-                      Types.ConvertType(getContext().VoidPtrTy));
-  RV = RValue::get(ArgAsPtrTy);
-  Args.add(RV, getContext().VoidPtrTy);
-  // sizeof (Type of Ivar)
-  CharUnits Size =  getContext().getTypeSizeInChars(Ivar->getType());
-  llvm::Value *SizeVal =
-  llvm::ConstantInt::get(Types.ConvertType(getContext().LongTy), 
-                         Size.getQuantity());
-  Args.add(RValue::get(SizeVal), getContext().LongTy);
-  llvm::Value *True =
-  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 1);
-  Args.add(RValue::get(True), getContext().BoolTy);
-  llvm::Value *False =
-  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 0);
-  Args.add(RValue::get(False), getContext().BoolTy);
-  EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
-                                 FunctionType::ExtInfo()),
-           GetCopyStructFn, ReturnValueSlot(), Args);
+  CallArgList args;
+
+  // The first argument is the address of the ivar.
+  llvm::Value *ivarAddr = CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(),
+                                                CGF.LoadObjCSelf(), ivar, 0)
+    .getAddress();
+  ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy);
+  args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy);
+
+  // The second argument is the address of the parameter variable.
+  ParmVarDecl *argVar = *OMD->param_begin();
+  DeclRefExpr argRef(argVar, argVar->getType(), VK_LValue, SourceLocation());
+  llvm::Value *argAddr = CGF.EmitLValue(&argRef).getAddress();
+  argAddr = CGF.Builder.CreateBitCast(argAddr, CGF.Int8PtrTy);
+  args.add(RValue::get(argAddr), CGF.getContext().VoidPtrTy);
+
+  // The third argument is the sizeof the type.
+  llvm::Value *size =
+    CGF.CGM.getSize(CGF.getContext().getTypeSizeInChars(ivar->getType()));
+  args.add(RValue::get(size), CGF.getContext().getSizeType());
+
+  // The fourth argument is the 'isAtomic' flag.
+  args.add(RValue::get(CGF.Builder.getTrue()), CGF.getContext().BoolTy);
+
+  // The fifth argument is the 'hasStrong' flag.
+  // FIXME: should this really always be false?
+  args.add(RValue::get(CGF.Builder.getFalse()), CGF.getContext().BoolTy);
+
+  llvm::Value *copyStructFn = CGF.CGM.getObjCRuntime().GetSetStructFunction();
+  CGF.EmitCall(CGF.getTypes().getFunctionInfo(CGF.getContext().VoidTy, args,
+                                              FunctionType::ExtInfo()),
+               copyStructFn, ReturnValueSlot(), args);
 }
 
-static bool
-IvarAssignHasTrvialAssignment(const ObjCPropertyImplDecl *PID,
-                              QualType IvarT) {
-  bool HasTrvialAssignment = true;
-  if (PID->getSetterCXXAssignment()) {
-    const CXXRecordDecl *classDecl = IvarT->getAsCXXRecordDecl();
-    HasTrvialAssignment = 
-      (!classDecl || classDecl->hasTrivialCopyAssignment());
+static bool hasTrivialSetExpr(const ObjCPropertyImplDecl *PID) {
+  Expr *setter = PID->getSetterCXXAssignment();
+  if (!setter) return true;
+
+  // Sema only makes only of these when the ivar has a C++ class type,
+  // so the form is pretty constrained.
+
+  // An operator call is trivial if the function it calls is trivial.
+  // This also implies that there's nothing non-trivial going on with
+  // the arguments, because operator= can only be trivial if it's a
+  // synthesized assignment operator and therefore both parameters are
+  // references.
+  if (CallExpr *call = dyn_cast<CallExpr>(setter)) {
+    if (const FunctionDecl *callee
+          = dyn_cast_or_null<FunctionDecl>(call->getCalleeDecl()))
+      if (callee->isTrivial())
+        return true;
+    return false;
   }
-  return HasTrvialAssignment;
+
+  assert(isa<ExprWithCleanups>(setter));
+  return false;
+}
+
+void
+CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
+                                        const ObjCPropertyImplDecl *propImpl) {
+  // Just use the setter expression if Sema gave us one and it's
+  // non-trivial.  There's no way to do this atomically.
+  if (!hasTrivialSetExpr(propImpl)) {
+    EmitStmt(propImpl->getSetterCXXAssignment());
+    return;
+  }
+
+  const ObjCPropertyDecl *prop = propImpl->getPropertyDecl();
+  ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();  
+  ObjCMethodDecl *setterMethod = prop->getSetterMethodDecl();
+
+  PropertyImplStrategy strategy(CGM, propImpl);
+  switch (strategy.getKind()) {
+  case PropertyImplStrategy::Native: {
+    llvm::Value *argAddr = LocalDeclMap[*setterMethod->param_begin()];
+
+    LValue ivarLValue =
+      EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, /*quals*/ 0);
+    llvm::Value *ivarAddr = ivarLValue.getAddress();
+
+    // Currently, all atomic accesses have to be through integer
+    // types, so there's no point in trying to pick a prettier type.
+    llvm::Type *bitcastType =
+      llvm::Type::getIntNTy(getLLVMContext(),
+                            getContext().toBits(strategy.getIvarSize()));
+    bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay
+
+    // Cast both arguments to the chosen operation type.
+    argAddr = Builder.CreateBitCast(argAddr, bitcastType);
+    ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType);
+
+    // This bitcast load is likely to cause some nasty IR.
+    llvm::Value *load = Builder.CreateLoad(argAddr);
+
+    // Perform an atomic store.  There are no memory ordering requirements.
+    llvm::StoreInst *store = Builder.CreateStore(load, ivarAddr);
+    store->setAlignment(strategy.getIvarAlignment().getQuantity());
+    store->setAtomic(llvm::Unordered);
+    return;
+  }
+
+  case PropertyImplStrategy::GetSetProperty:
+  case PropertyImplStrategy::SetPropertyAndExpressionGet: {
+    llvm::Value *setPropertyFn =
+      CGM.getObjCRuntime().GetPropertySetFunction();
+    if (!setPropertyFn) {
+      CGM.ErrorUnsupported(propImpl, "Obj-C setter requiring atomic copy");
+      return;
+    }
+
+    // Emit objc_setProperty((id) self, _cmd, offset, arg,
+    //                       <is-atomic>, <is-copy>).
+    llvm::Value *cmd =
+      Builder.CreateLoad(LocalDeclMap[setterMethod->getCmdDecl()]);
+    llvm::Value *self =
+      Builder.CreateBitCast(LoadObjCSelf(), VoidPtrTy);
+    llvm::Value *ivarOffset =
+      EmitIvarOffset(classImpl->getClassInterface(), ivar);
+    llvm::Value *arg = LocalDeclMap[*setterMethod->param_begin()];
+    arg = Builder.CreateBitCast(Builder.CreateLoad(arg, "arg"), VoidPtrTy);
+
+    CallArgList args;
+    args.add(RValue::get(self), getContext().getObjCIdType());
+    args.add(RValue::get(cmd), getContext().getObjCSelType());
+    args.add(RValue::get(ivarOffset), getContext().getPointerDiffType());
+    args.add(RValue::get(arg), getContext().getObjCIdType());
+    args.add(RValue::get(Builder.getInt1(strategy.isAtomic())),
+             getContext().BoolTy);
+    args.add(RValue::get(Builder.getInt1(strategy.isCopy())),
+             getContext().BoolTy);
+    // FIXME: We shouldn't need to get the function info here, the runtime
+    // already should have computed it to build the function.
+    EmitCall(getTypes().getFunctionInfo(getContext().VoidTy, args,
+                                        FunctionType::ExtInfo()),
+             setPropertyFn, ReturnValueSlot(), args);
+    return;
+  }
+
+  case PropertyImplStrategy::CopyStruct:
+    emitStructSetterCall(*this, setterMethod, ivar);
+    return;
+
+  case PropertyImplStrategy::Expression:
+    break;
+  }
+
+  // Otherwise, fake up some ASTs and emit a normal assignment.
+  ValueDecl *selfDecl = setterMethod->getSelfDecl();
+  DeclRefExpr self(selfDecl, selfDecl->getType(), VK_LValue, SourceLocation());
+  ImplicitCastExpr selfLoad(ImplicitCastExpr::OnStack,
+                            selfDecl->getType(), CK_LValueToRValue, &self,
+                            VK_RValue);
+  ObjCIvarRefExpr ivarRef(ivar, ivar->getType().getNonReferenceType(),
+                          SourceLocation(), &selfLoad, true, true);
+
+  ParmVarDecl *argDecl = *setterMethod->param_begin();
+  QualType argType = argDecl->getType().getNonReferenceType();
+  DeclRefExpr arg(argDecl, argType, VK_LValue, SourceLocation());
+  ImplicitCastExpr argLoad(ImplicitCastExpr::OnStack,
+                           argType.getUnqualifiedType(), CK_LValueToRValue,
+                           &arg, VK_RValue);
+    
+  // The property type can differ from the ivar type in some situations with
+  // Objective-C pointer types, we can always bit cast the RHS in these cases.
+  // The following absurdity is just to ensure well-formed IR.
+  CastKind argCK = CK_NoOp;
+  if (ivarRef.getType()->isObjCObjectPointerType()) {
+    if (argLoad.getType()->isObjCObjectPointerType())
+      argCK = CK_BitCast;
+    else if (argLoad.getType()->isBlockPointerType())
+      argCK = CK_BlockPointerToObjCPointerCast;
+    else
+      argCK = CK_CPointerToObjCPointerCast;
+  } else if (ivarRef.getType()->isBlockPointerType()) {
+     if (argLoad.getType()->isBlockPointerType())
+      argCK = CK_BitCast;
+    else
+      argCK = CK_AnyPointerToBlockPointerCast;
+  } else if (ivarRef.getType()->isPointerType()) {
+    argCK = CK_BitCast;
+  }
+  ImplicitCastExpr argCast(ImplicitCastExpr::OnStack,
+                           ivarRef.getType(), argCK, &argLoad,
+                           VK_RValue);
+  Expr *finalArg = &argLoad;
+  if (!getContext().hasSameUnqualifiedType(ivarRef.getType(),
+                                           argLoad.getType()))
+    finalArg = &argCast;
+
+
+  BinaryOperator assign(&ivarRef, finalArg, BO_Assign,
+                        ivarRef.getType(), VK_RValue, OK_Ordinary,
+                        SourceLocation());
+  EmitStmt(&assign);
 }
 
 /// GenerateObjCSetter - Generate an Objective-C property setter
@@ -545,136 +943,12 @@ IvarAssignHasTrvialAssignment(const ObjCPropertyImplDecl *PID,
 /// is illegal within a category.
 void CodeGenFunction::GenerateObjCSetter(ObjCImplementationDecl *IMP,
                                          const ObjCPropertyImplDecl *PID) {
-  ObjCIvarDecl *Ivar = PID->getPropertyIvarDecl();
   const ObjCPropertyDecl *PD = PID->getPropertyDecl();
   ObjCMethodDecl *OMD = PD->getSetterMethodDecl();
   assert(OMD && "Invalid call to generate setter (empty method)");
   StartObjCMethod(OMD, IMP->getClassInterface(), PID->getLocStart());
-  const llvm::Triple &Triple = getContext().Target.getTriple();
-  QualType IVART = Ivar->getType();
-  bool IsCopy = PD->getSetterKind() == ObjCPropertyDecl::Copy;
-  bool IsAtomic =
-    !(PD->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_nonatomic);
 
-  // Determine if we should use an objc_setProperty call for
-  // this. Properties with 'copy' semantics always use it, as do
-  // non-atomic properties with 'release' semantics as long as we are
-  // not in gc-only mode.
-  if (IsCopy ||
-      (CGM.getLangOptions().getGCMode() != LangOptions::GCOnly &&
-       PD->getSetterKind() == ObjCPropertyDecl::Retain)) {
-    llvm::Value *SetPropertyFn =
-      CGM.getObjCRuntime().GetPropertySetFunction();
-
-    if (!SetPropertyFn) {
-      CGM.ErrorUnsupported(PID, "Obj-C getter requiring atomic copy");
-      FinishFunction();
-      return;
-    }
-
-    // Emit objc_setProperty((id) self, _cmd, offset, arg,
-    //                       <is-atomic>, <is-copy>).
-    // FIXME: Can't this be simpler? This might even be worse than the
-    // corresponding gcc code.
-    CodeGenTypes &Types = CGM.getTypes();
-    ValueDecl *Cmd = OMD->getCmdDecl();
-    llvm::Value *CmdVal = Builder.CreateLoad(LocalDeclMap[Cmd], "cmd");
-    QualType IdTy = getContext().getObjCIdType();
-    llvm::Value *SelfAsId =
-      Builder.CreateBitCast(LoadObjCSelf(), Types.ConvertType(IdTy));
-    llvm::Value *Offset = EmitIvarOffset(IMP->getClassInterface(), Ivar);
-    llvm::Value *Arg = LocalDeclMap[*OMD->param_begin()];
-    llvm::Value *ArgAsId =
-      Builder.CreateBitCast(Builder.CreateLoad(Arg, "arg"),
-                            Types.ConvertType(IdTy));
-    llvm::Value *True =
-      llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 1);
-    llvm::Value *False =
-      llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 0);
-    CallArgList Args;
-    Args.add(RValue::get(SelfAsId), IdTy);
-    Args.add(RValue::get(CmdVal), Cmd->getType());
-    Args.add(RValue::get(Offset), getContext().getPointerDiffType());
-    Args.add(RValue::get(ArgAsId), IdTy);
-    Args.add(RValue::get(IsAtomic ? True : False),  getContext().BoolTy);
-    Args.add(RValue::get(IsCopy ? True : False), getContext().BoolTy);
-    // FIXME: We shouldn't need to get the function info here, the runtime
-    // already should have computed it to build the function.
-    EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
-                                   FunctionType::ExtInfo()),
-             SetPropertyFn,
-             ReturnValueSlot(), Args);
-  } else if (IsAtomic && hasAggregateLLVMType(IVART) &&
-             !IVART->isAnyComplexType() &&
-             IvarAssignHasTrvialAssignment(PID, IVART) &&
-             ((Triple.getArch() == llvm::Triple::x86 &&
-              (getContext().getTypeSizeInChars(IVART)
-               > CharUnits::fromQuantity(4))) ||
-              (Triple.getArch() == llvm::Triple::x86_64 &&
-              (getContext().getTypeSizeInChars(IVART)
-               > CharUnits::fromQuantity(8))))
-             && CGM.getObjCRuntime().GetSetStructFunction()) {
-          // objc_copyStruct (&structIvar, &Arg, 
-          //                  sizeof (struct something), true, false);
-    GenerateObjCAtomicSetterBody(OMD, Ivar);
-  } else if (PID->getSetterCXXAssignment()) {
-    EmitIgnoredExpr(PID->getSetterCXXAssignment());
-  } else {
-    if (IsAtomic &&
-        IVART->isScalarType() &&
-        (Triple.getArch() == llvm::Triple::arm ||
-         Triple.getArch() == llvm::Triple::thumb) &&
-        (getContext().getTypeSizeInChars(IVART)
-          > CharUnits::fromQuantity(4)) &&
-        CGM.getObjCRuntime().GetGetStructFunction()) {
-      GenerateObjCAtomicSetterBody(OMD, Ivar);
-    }
-    else if (IsAtomic &&
-             (IVART->isScalarType() && !IVART->isRealFloatingType()) &&
-             Triple.getArch() == llvm::Triple::x86 &&
-             (getContext().getTypeSizeInChars(IVART)
-              > CharUnits::fromQuantity(4)) &&
-             CGM.getObjCRuntime().GetGetStructFunction()) {
-      GenerateObjCAtomicSetterBody(OMD, Ivar);
-    }
-    else if (IsAtomic &&
-             (IVART->isScalarType() && !IVART->isRealFloatingType()) &&
-             Triple.getArch() == llvm::Triple::x86_64 &&
-             (getContext().getTypeSizeInChars(IVART)
-              > CharUnits::fromQuantity(8)) &&
-             CGM.getObjCRuntime().GetGetStructFunction()) {
-      GenerateObjCAtomicSetterBody(OMD, Ivar);
-    }
-    else {
-      // FIXME: Find a clean way to avoid AST node creation.
-      SourceLocation Loc = PID->getLocStart();
-      ValueDecl *Self = OMD->getSelfDecl();
-      ObjCIvarDecl *Ivar = PID->getPropertyIvarDecl();
-      DeclRefExpr Base(Self, Self->getType(), VK_RValue, Loc);
-      ParmVarDecl *ArgDecl = *OMD->param_begin();
-      QualType T = ArgDecl->getType();
-      if (T->isReferenceType())
-        T = cast<ReferenceType>(T)->getPointeeType();
-      DeclRefExpr Arg(ArgDecl, T, VK_LValue, Loc);
-      ObjCIvarRefExpr IvarRef(Ivar, Ivar->getType(), Loc, &Base, true, true);
-    
-      // The property type can differ from the ivar type in some situations with
-      // Objective-C pointer types, we can always bit cast the RHS in these cases.
-      if (getContext().getCanonicalType(Ivar->getType()) !=
-          getContext().getCanonicalType(ArgDecl->getType())) {
-        ImplicitCastExpr ArgCasted(ImplicitCastExpr::OnStack,
-                                   Ivar->getType(), CK_BitCast, &Arg,
-                                   VK_RValue);
-        BinaryOperator Assign(&IvarRef, &ArgCasted, BO_Assign,
-                              Ivar->getType(), VK_RValue, OK_Ordinary, Loc);
-        EmitStmt(&Assign);
-      } else {
-        BinaryOperator Assign(&IvarRef, &Arg, BO_Assign,
-                              Ivar->getType(), VK_RValue, OK_Ordinary, Loc);
-        EmitStmt(&Assign);
-      }
-    }
-  }
+  generateObjCSetterBody(IMP, PID);
 
   FinishFunction();
 }
@@ -716,9 +990,8 @@ static void emitCXXDestructMethod(CodeGenFunction &CGF,
 
   llvm::Value *self = CGF.LoadObjCSelf();
 
-  ObjCInterfaceDecl *iface
-    = const_cast<ObjCInterfaceDecl*>(impl->getClassInterface());
-  for (ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
+  const ObjCInterfaceDecl *iface = impl->getClassInterface();
+  for (const ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
        ivar; ivar = ivar->getNextIvar()) {
     QualType type = ivar->getType();
 
@@ -758,7 +1031,7 @@ void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
     // Suppress the final autorelease in ARC.
     AutoreleaseResult = false;
 
-    llvm::SmallVector<CXXCtorInitializer *, 8> IvarInitializers;
+    SmallVector<CXXCtorInitializer *, 8> IvarInitializers;
     for (ObjCImplementationDecl::init_const_iterator B = IMP->init_begin(),
            E = IMP->init_end(); B != E; ++B) {
       CXXCtorInitializer *IvarInit = (*B);
@@ -766,7 +1039,10 @@ void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
       ObjCIvarDecl  *Ivar = cast<ObjCIvarDecl>(Field);
       LValue LV = EmitLValueForIvar(TypeOfSelfObject(), 
                                     LoadObjCSelf(), Ivar, 0);
-      EmitAggExpr(IvarInit->getInit(), AggValueSlot::forLValue(LV, true));
+      EmitAggExpr(IvarInit->getInit(),
+                  AggValueSlot::forLValue(LV, AggValueSlot::IsDestructed,
+                                          AggValueSlot::DoesNotNeedGCBarriers,
+                                          AggValueSlot::IsNotAliased));
     }
     // constructor returns 'self'.
     CodeGenTypes &Types = CGM.getTypes();
@@ -791,7 +1067,7 @@ bool CodeGenFunction::IndirectObjCSetterArg(const CGFunctionInfo &FI) {
 }
 
 bool CodeGenFunction::IvarTypeWithAggrGCObjects(QualType Ty) {
-  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC)
+  if (CGM.getLangOptions().getGC() == LangOptions::NonGC)
     return false;
   if (const RecordType *FDTTy = Ty.getTypePtr()->getAs<RecordType>())
     return FDTTy->getDecl()->hasObjectMember();
@@ -896,7 +1172,7 @@ void CodeGenFunction::EmitStoreThroughPropertyRefLValue(RValue Src,
   if (Src.isScalar()) {
     llvm::Value *SrcVal = Src.getScalarVal();
     QualType DstType = getContext().getCanonicalType(ArgType);
-    const llvm::Type *DstTy = ConvertType(DstType);
+    llvm::Type *DstTy = ConvertType(DstType);
     if (SrcVal->getType() != DstTy)
       Src = 
         RValue::get(EmitScalarConversion(SrcVal, E->getType(), DstType));
@@ -932,10 +1208,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   }
 
   CGDebugInfo *DI = getDebugInfo();
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getBegin());
-    DI->EmitRegionStart(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin());
 
   // The local variable comes into scope immediately.
   AutoVarEmission variable = AutoVarEmission::invalid();
@@ -943,10 +1217,9 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
     variable = EmitAutoVarAlloca(*cast<VarDecl>(SD->getSingleDecl()));
 
   JumpDest LoopEnd = getJumpDestInCurrentScope("forcoll.end");
-  JumpDest AfterBody = getJumpDestInCurrentScope("forcoll.next");
 
   // Fast enumeration state.
-  QualType StateTy = getContext().getObjCFastEnumerationStateType();
+  QualType StateTy = CGM.getObjCFastEnumerationStateType();
   llvm::Value *StatePtr = CreateMemTemp(StateTy, "state.ptr");
   EmitNullInitialization(StatePtr, StateTy);
 
@@ -968,8 +1241,20 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
                                       ArrayType::Normal, 0);
   llvm::Value *ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr");
 
-  // Emit the collection pointer.
-  llvm::Value *Collection = EmitScalarExpr(S.getCollection());
+  // Emit the collection pointer.  In ARC, we do a retain.
+  llvm::Value *Collection;
+  if (getLangOptions().ObjCAutoRefCount) {
+    Collection = EmitARCRetainScalarExpr(S.getCollection());
+
+    // Enter a cleanup to do the release.
+    EmitObjCConsumeObject(S.getCollection()->getType(), Collection);
+  } else {
+    Collection = EmitScalarExpr(S.getCollection());
+  }
+
+  // The 'continue' label needs to appear within the cleanup for the
+  // collection object.
+  JumpDest AfterBody = getJumpDestInCurrentScope("forcoll.next");
 
   // Send it our message:
   CallArgList Args;
@@ -985,7 +1270,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   Args.add(RValue::get(ItemsPtr), getContext().getPointerType(ItemsTy));
 
   // The third argument is the capacity of that temporary array.
-  const llvm::Type *UnsignedLongLTy = ConvertType(getContext().UnsignedLongTy);
+  llvm::Type *UnsignedLongLTy = ConvertType(getContext().UnsignedLongTy);
   llvm::Constant *Count = llvm::ConstantInt::get(UnsignedLongLTy, NumItems);
   Args.add(RValue::get(Count), getContext().UnsignedLongTy);
 
@@ -1053,8 +1338,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   EmitBlock(WasMutatedBB);
   llvm::Value *V =
     Builder.CreateBitCast(Collection,
-                          ConvertType(getContext().getObjCIdType()),
-                          "tmp");
+                          ConvertType(getContext().getObjCIdType()));
   CallArgList Args2;
   Args2.add(RValue::get(V), getContext().getObjCIdType());
   // FIXME: We shouldn't need to get the function info here, the runtime already
@@ -1089,7 +1373,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
     elementType = cast<Expr>(S.getElement())->getType();
     elementIsVariable = false;
   }
-  const llvm::Type *convertedElementType = ConvertType(elementType);
+  llvm::Type *convertedElementType = ConvertType(elementType);
 
   // Fetch the buffer out of the enumeration state.
   // TODO: this pointer should actually be invariant between
@@ -1179,10 +1463,12 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
     EmitStoreThroughLValue(RValue::get(null), elementLValue);
   }
 
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getEnd());
-    DI->EmitRegionEnd(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
+
+  // Leave the cleanup we entered in ARC.
+  if (getLangOptions().ObjCAutoRefCount)
+    PopCleanupBlock();
 
   EmitBlock(LoopEnd.getBlock());
 }
@@ -1200,7 +1486,7 @@ void CodeGenFunction::EmitObjCAtSynchronizedStmt(
   CGM.getObjCRuntime().EmitSynchronizedStmt(*this, S);
 }
 
-/// Produce the code for a CK_ObjCProduceObject.  Just does a
+/// Produce the code for a CK_ARCProduceObject.  Just does a
 /// primitive retain.
 llvm::Value *CodeGenFunction::EmitObjCProduceObject(QualType type,
                                                     llvm::Value *value) {
@@ -1209,63 +1495,22 @@ llvm::Value *CodeGenFunction::EmitObjCProduceObject(QualType type,
 
 namespace {
   struct CallObjCRelease : EHScopeStack::Cleanup {
-    CallObjCRelease(QualType type, llvm::Value *ptr, llvm::Value *condition)
-      : type(type), ptr(ptr), condition(condition) {}
-    QualType type;
-    llvm::Value *ptr;
-    llvm::Value *condition;
+    CallObjCRelease(llvm::Value *object) : object(object) {}
+    llvm::Value *object;
 
     void Emit(CodeGenFunction &CGF, Flags flags) {
-      llvm::Value *object;
-
-      // If we're in a conditional branch, we had to stash away in an
-      // alloca the pointer to be released.
-      llvm::BasicBlock *cont = 0;
-      if (condition) {
-        llvm::BasicBlock *release = CGF.createBasicBlock("release.yes");
-        cont = CGF.createBasicBlock("release.cont");
-
-        llvm::Value *cond = CGF.Builder.CreateLoad(condition);
-        CGF.Builder.CreateCondBr(cond, release, cont);
-        CGF.EmitBlock(release);
-        object = CGF.Builder.CreateLoad(ptr);
-      } else {
-        object = ptr;
-      }
-
       CGF.EmitARCRelease(object, /*precise*/ true);
-
-      if (cont) CGF.EmitBlock(cont);
     }
   };
 }
 
-/// Produce the code for a CK_ObjCConsumeObject.  Does a primitive
+/// Produce the code for a CK_ARCConsumeObject.  Does a primitive
 /// release at the end of the full-expression.
 llvm::Value *CodeGenFunction::EmitObjCConsumeObject(QualType type,
                                                     llvm::Value *object) {
   // If we're in a conditional branch, we need to make the cleanup
-  // conditional.  FIXME: this really needs to be supported by the
-  // environment.
-  llvm::AllocaInst *cond;
-  llvm::Value *ptr;
-  if (isInConditionalBranch()) {
-    cond = CreateTempAlloca(Builder.getInt1Ty(), "release.cond");
-    ptr = CreateTempAlloca(object->getType(), "release.value");
-
-    // The alloca is false until we get here.
-    // FIXME: er. doesn't this need to be set at the start of the condition?
-    InitTempAlloca(cond, Builder.getFalse());
-
-    // Then it turns true.
-    Builder.CreateStore(Builder.getTrue(), cond);
-    Builder.CreateStore(object, ptr);
-  } else {
-    cond = 0;
-    ptr = object;
-  }
-
-  EHStack.pushCleanup<CallObjCRelease>(getARCCleanupKind(), type, ptr, cond);
+  // conditional.
+  pushFullExprCleanup<CallObjCRelease>(getARCCleanupKind(), object);
   return object;
 }
 
@@ -1276,8 +1521,8 @@ llvm::Value *CodeGenFunction::EmitObjCExtendObjectLifetime(QualType type,
 
 
 static llvm::Constant *createARCRuntimeFunction(CodeGenModule &CGM,
-                                                const llvm::FunctionType *type,
-                                                llvm::StringRef fnName) {
+                                                llvm::FunctionType *type,
+                                                StringRef fnName) {
   llvm::Constant *fn = CGM.CreateRuntimeFunction(type, fnName);
 
   // In -fobjc-no-arc-runtime, emit weak references to the runtime
@@ -1295,18 +1540,18 @@ static llvm::Constant *createARCRuntimeFunction(CodeGenModule &CGM,
 static llvm::Value *emitARCValueOperation(CodeGenFunction &CGF,
                                           llvm::Value *value,
                                           llvm::Constant *&fn,
-                                          llvm::StringRef fnName) {
+                                          StringRef fnName) {
   if (isa<llvm::ConstantPointerNull>(value)) return value;
 
   if (!fn) {
     std::vector<llvm::Type*> args(1, CGF.Int8PtrTy);
-    const llvm::FunctionType *fnType =
+    llvm::FunctionType *fnType =
       llvm::FunctionType::get(CGF.Int8PtrTy, args, false);
     fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
   }
 
   // Cast the argument to 'id'.
-  const llvm::Type *origType = value->getType();
+  llvm::Type *origType = value->getType();
   value = CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy);
 
   // Call the function.
@@ -1322,16 +1567,16 @@ static llvm::Value *emitARCValueOperation(CodeGenFunction &CGF,
 static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF,
                                          llvm::Value *addr,
                                          llvm::Constant *&fn,
-                                         llvm::StringRef fnName) {
+                                         StringRef fnName) {
   if (!fn) {
     std::vector<llvm::Type*> args(1, CGF.Int8PtrPtrTy);
-    const llvm::FunctionType *fnType =
+    llvm::FunctionType *fnType =
       llvm::FunctionType::get(CGF.Int8PtrTy, args, false);
     fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
   }
 
   // Cast the argument to 'id*'.
-  const llvm::Type *origType = addr->getType();
+  llvm::Type *origType = addr->getType();
   addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy);
 
   // Call the function.
@@ -1353,7 +1598,7 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF,
                                           llvm::Value *addr,
                                           llvm::Value *value,
                                           llvm::Constant *&fn,
-                                          llvm::StringRef fnName,
+                                          StringRef fnName,
                                           bool ignored) {
   assert(cast<llvm::PointerType>(addr->getType())->getElementType()
            == value->getType());
@@ -1363,12 +1608,12 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF,
     argTypes[0] = CGF.Int8PtrPtrTy;
     argTypes[1] = CGF.Int8PtrTy;
 
-    const llvm::FunctionType *fnType
+    llvm::FunctionType *fnType
       = llvm::FunctionType::get(CGF.Int8PtrTy, argTypes, false);
     fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
   }
 
-  const llvm::Type *origType = value->getType();
+  llvm::Type *origType = value->getType();
 
   addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy);
   value = CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy);
@@ -1387,12 +1632,12 @@ static void emitARCCopyOperation(CodeGenFunction &CGF,
                                  llvm::Value *dst,
                                  llvm::Value *src,
                                  llvm::Constant *&fn,
-                                 llvm::StringRef fnName) {
+                                 StringRef fnName) {
   assert(dst->getType() == src->getType());
 
   if (!fn) {
     std::vector<llvm::Type*> argTypes(2, CGF.Int8PtrPtrTy);
-    const llvm::FunctionType *fnType
+    llvm::FunctionType *fnType
       = llvm::FunctionType::get(CGF.Builder.getVoidTy(), argTypes, false);
     fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
   }
@@ -1409,7 +1654,7 @@ static void emitARCCopyOperation(CodeGenFunction &CGF,
 ///   call i8* @objc_retainBlock(i8* %value)
 llvm::Value *CodeGenFunction::EmitARCRetain(QualType type, llvm::Value *value) {
   if (type->isBlockPointerType())
-    return EmitARCRetainBlock(value);
+    return EmitARCRetainBlock(value, /*mandatory*/ false);
   else
     return EmitARCRetainNonBlock(value);
 }
@@ -1424,10 +1669,32 @@ llvm::Value *CodeGenFunction::EmitARCRetainNonBlock(llvm::Value *value) {
 
 /// Retain the given block, with _Block_copy semantics.
 ///   call i8* @objc_retainBlock(i8* %value)
-llvm::Value *CodeGenFunction::EmitARCRetainBlock(llvm::Value *value) {
-  return emitARCValueOperation(*this, value,
-                               CGM.getARCEntrypoints().objc_retainBlock,
-                               "objc_retainBlock");
+///
+/// \param mandatory - If false, emit the call with metadata
+/// indicating that it's okay for the optimizer to eliminate this call
+/// if it can prove that the block never escapes except down the stack.
+llvm::Value *CodeGenFunction::EmitARCRetainBlock(llvm::Value *value,
+                                                 bool mandatory) {
+  llvm::Value *result
+    = emitARCValueOperation(*this, value,
+                            CGM.getARCEntrypoints().objc_retainBlock,
+                            "objc_retainBlock");
+
+  // If the copy isn't mandatory, add !clang.arc.copy_on_escape to
+  // tell the optimizer that it doesn't need to do this copy if the
+  // block doesn't escape, where being passed as an argument doesn't
+  // count as escaping.
+  if (!mandatory && isa<llvm::Instruction>(result)) {
+    llvm::CallInst *call
+      = cast<llvm::CallInst>(result->stripPointerCasts());
+    assert(call->getCalledValue() == CGM.getARCEntrypoints().objc_retainBlock);
+
+    SmallVector<llvm::Value*,1> args;
+    call->setMetadata("clang.arc.copy_on_escape",
+                      llvm::MDNode::get(Builder.getContext(), args));
+  }
+
+  return result;
 }
 
 /// Retain the given object which is the result of a function call.
@@ -1442,7 +1709,7 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
   llvm::InlineAsm *&marker
     = CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker;
   if (!marker) {
-    llvm::StringRef assembly
+    StringRef assembly
       = CGM.getTargetCodeGenInfo()
            .getARCRetainAutoreleasedReturnValueMarker();
 
@@ -1468,8 +1735,7 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
       assert(metadata->getNumOperands() <= 1);
       if (metadata->getNumOperands() == 0) {
         llvm::Value *string = llvm::MDString::get(getLLVMContext(), assembly);
-        llvm::Value *args[] = { string };
-        metadata->addOperand(llvm::MDNode::get(getLLVMContext(), args));
+        metadata->addOperand(llvm::MDNode::get(getLLVMContext(), string));
       }
     }
   }
@@ -1490,7 +1756,7 @@ void CodeGenFunction::EmitARCRelease(llvm::Value *value, bool precise) {
   llvm::Constant *&fn = CGM.getARCEntrypoints().objc_release;
   if (!fn) {
     std::vector<llvm::Type*> args(1, Int8PtrTy);
-    const llvm::FunctionType *fnType =
+    llvm::FunctionType *fnType =
       llvm::FunctionType::get(Builder.getVoidTy(), args, false);
     fn = createARCRuntimeFunction(CGM, fnType, "objc_release");
   }
@@ -1503,7 +1769,7 @@ void CodeGenFunction::EmitARCRelease(llvm::Value *value, bool precise) {
   call->setDoesNotThrow();
 
   if (!precise) {
-    llvm::SmallVector<llvm::Value*,1> args;
+    SmallVector<llvm::Value*,1> args;
     call->setMetadata("clang.imprecise_release",
                       llvm::MDNode::get(Builder.getContext(), args));
   }
@@ -1520,7 +1786,7 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(llvm::Value *addr,
   llvm::Constant *&fn = CGM.getARCEntrypoints().objc_storeStrong;
   if (!fn) {
     llvm::Type *argTypes[] = { Int8PtrPtrTy, Int8PtrTy };
-    const llvm::FunctionType *fnType
+    llvm::FunctionType *fnType
       = llvm::FunctionType::get(Builder.getVoidTy(), argTypes, false);
     fn = createARCRuntimeFunction(CGM, fnType, "objc_storeStrong");
   }
@@ -1607,9 +1873,9 @@ llvm::Value *CodeGenFunction::EmitARCRetainAutorelease(QualType type,
 
   if (isa<llvm::ConstantPointerNull>(value)) return value;
 
-  const llvm::Type *origType = value->getType();
+  llvm::Type *origType = value->getType();
   value = Builder.CreateBitCast(value, Int8PtrTy);
-  value = EmitARCRetainBlock(value);
+  value = EmitARCRetainBlock(value, /*mandatory*/ true);
   value = EmitARCAutorelease(value);
   return Builder.CreateBitCast(value, origType);
 }
@@ -1674,7 +1940,7 @@ void CodeGenFunction::EmitARCDestroyWeak(llvm::Value *addr) {
   llvm::Constant *&fn = CGM.getARCEntrypoints().objc_destroyWeak;
   if (!fn) {
     std::vector<llvm::Type*> args(1, Int8PtrPtrTy);
-    const llvm::FunctionType *fnType =
+    llvm::FunctionType *fnType =
       llvm::FunctionType::get(Builder.getVoidTy(), args, false);
     fn = createARCRuntimeFunction(CGM, fnType, "objc_destroyWeak");
   }
@@ -1709,7 +1975,7 @@ void CodeGenFunction::EmitARCCopyWeak(llvm::Value *dst, llvm::Value *src) {
 llvm::Value *CodeGenFunction::EmitObjCAutoreleasePoolPush() {
   llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPush;
   if (!fn) {
-    const llvm::FunctionType *fnType =
+    llvm::FunctionType *fnType =
       llvm::FunctionType::get(Int8PtrTy, false);
     fn = createARCRuntimeFunction(CGM, fnType, "objc_autoreleasePoolPush");
   }
@@ -1728,7 +1994,7 @@ void CodeGenFunction::EmitObjCAutoreleasePoolPop(llvm::Value *value) {
   llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPop;
   if (!fn) {
     std::vector<llvm::Type*> args(1, Int8PtrTy);
-    const llvm::FunctionType *fnType =
+    llvm::FunctionType *fnType =
       llvm::FunctionType::get(Builder.getVoidTy(), args, false);
 
     // We don't want to use a weak import here; instead we should not
@@ -1851,6 +2117,24 @@ static TryEmitResult tryEmitARCRetainLoadOfScalar(CodeGenFunction &CGF,
   e = e->IgnoreParens();
   QualType type = e->getType();
 
+  // If we're loading retained from a __strong xvalue, we can avoid 
+  // an extra retain/release pair by zeroing out the source of this
+  // "move" operation.
+  if (e->isXValue() &&
+      !type.isConstQualified() &&
+      type.getObjCLifetime() == Qualifiers::OCL_Strong) {
+    // Emit the lvalue.
+    LValue lv = CGF.EmitLValue(e);
+    
+    // Load the object pointer.
+    llvm::Value *result = CGF.EmitLoadOfLValue(lv).getScalarVal();
+    
+    // Set the source pointer to NULL.
+    CGF.EmitStoreOfScalar(getNullForVariable(lv.getAddress()), lv);
+    
+    return TryEmitResult(result, true);
+  }
+
   // As a very special optimization, in ARC++, if the l-value is the
   // result of a non-volatile assignment, do a simple retain of the
   // result of the call to objc_storeWeak instead of reloading.
@@ -1913,35 +2197,53 @@ static llvm::Value *emitARCRetainAfterCall(CodeGenFunction &CGF,
   }
 }
 
+/// Determine whether it might be important to emit a separate
+/// objc_retain_block on the result of the given expression, or
+/// whether it's okay to just emit it in a +1 context.
+static bool shouldEmitSeparateBlockRetain(const Expr *e) {
+  assert(e->getType()->isBlockPointerType());
+  e = e->IgnoreParens();
+
+  // For future goodness, emit block expressions directly in +1
+  // contexts if we can.
+  if (isa<BlockExpr>(e))
+    return false;
+
+  if (const CastExpr *cast = dyn_cast<CastExpr>(e)) {
+    switch (cast->getCastKind()) {
+    // Emitting these operations in +1 contexts is goodness.
+    case CK_LValueToRValue:
+    case CK_ARCReclaimReturnedObject:
+    case CK_ARCConsumeObject:
+    case CK_ARCProduceObject:
+      return false;
+
+    // These operations preserve a block type.
+    case CK_NoOp:
+    case CK_BitCast:
+      return shouldEmitSeparateBlockRetain(cast->getSubExpr());
+
+    // These operations are known to be bad (or haven't been considered).
+    case CK_AnyPointerToBlockPointerCast:
+    default:
+      return true;
+    }
+  }
+
+  return true;
+}
+
 static TryEmitResult
 tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e) {
+  // Look through cleanups.
+  if (const ExprWithCleanups *cleanups = dyn_cast<ExprWithCleanups>(e)) {
+    CodeGenFunction::RunCleanupsScope scope(CGF);
+    return tryEmitARCRetainScalarExpr(CGF, cleanups->getSubExpr());
+  }
+
   // The desired result type, if it differs from the type of the
   // ultimate opaque expression.
-  const llvm::Type *resultType = 0;
-
-  // If we're loading retained from a __strong xvalue, we can avoid 
-  // an extra retain/release pair by zeroing out the source of this
-  // "move" operation.
-  if (e->isXValue() && !e->getType().isConstQualified() &&
-      e->getType().getObjCLifetime() == Qualifiers::OCL_Strong) {
-    // Emit the lvalue
-    LValue lv = CGF.EmitLValue(e);
-    
-    // Load the object pointer and cast it to the appropriate type.
-    QualType exprType = e->getType();
-    llvm::Value *result = CGF.EmitLoadOfLValue(lv).getScalarVal();
-    
-    if (resultType)
-      result = CGF.Builder.CreateBitCast(result, resultType);
-    
-    // Set the source pointer to NULL.
-    llvm::Value *null 
-      = llvm::ConstantPointerNull::get(
-                            cast<llvm::PointerType>(CGF.ConvertType(exprType)));
-    CGF.EmitStoreOfScalar(null, lv);
-    
-    return TryEmitResult(result, true);
-  }
+  llvm::Type *resultType = 0;
 
   while (true) {
     e = e->IgnoreParens();
@@ -1969,7 +2271,8 @@ tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e) {
       // These casts can change the type, so remember that and
       // soldier on.  We only need to remember the outermost such
       // cast, though.
-      case CK_AnyPointerToObjCPointerCast:
+      case CK_CPointerToObjCPointerCast:
+      case CK_BlockPointerToObjCPointerCast:
       case CK_AnyPointerToBlockPointerCast:
       case CK_BitCast:
         if (!resultType)
@@ -1980,15 +2283,49 @@ tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e) {
 
       // For consumptions, just emit the subexpression and thus elide
       // the retain/release pair.
-      case CK_ObjCConsumeObject: {
+      case CK_ARCConsumeObject: {
         llvm::Value *result = CGF.EmitScalarExpr(ce->getSubExpr());
         if (resultType) result = CGF.Builder.CreateBitCast(result, resultType);
         return TryEmitResult(result, true);
       }
 
+      // Block extends are net +0.  Naively, we could just recurse on
+      // the subexpression, but actually we need to ensure that the
+      // value is copied as a block, so there's a little filter here.
+      case CK_ARCExtendBlockObject: {
+        llvm::Value *result; // will be a +0 value
+
+        // If we can't safely assume the sub-expression will produce a
+        // block-copied value, emit the sub-expression at +0.
+        if (shouldEmitSeparateBlockRetain(ce->getSubExpr())) {
+          result = CGF.EmitScalarExpr(ce->getSubExpr());
+
+        // Otherwise, try to emit the sub-expression at +1 recursively.
+        } else {
+          TryEmitResult subresult
+            = tryEmitARCRetainScalarExpr(CGF, ce->getSubExpr());
+          result = subresult.getPointer();
+
+          // If that produced a retained value, just use that,
+          // possibly casting down.
+          if (subresult.getInt()) {
+            if (resultType)
+              result = CGF.Builder.CreateBitCast(result, resultType);
+            return TryEmitResult(result, true);
+          }
+
+          // Otherwise it's +0.
+        }
+
+        // Retain the object as a block, then cast down.
+        result = CGF.EmitARCRetainBlock(result, /*mandatory*/ true);
+        if (resultType) result = CGF.Builder.CreateBitCast(result, resultType);
+        return TryEmitResult(result, true);
+      }
+
       // For reclaims, emit the subexpression as a retained call and
       // skip the consumption.
-      case CK_ObjCReclaimReturnedObject: {
+      case CK_ARCReclaimReturnedObject: {
         llvm::Value *result = emitARCRetainCall(CGF, ce->getSubExpr());
         if (resultType) result = CGF.Builder.CreateBitCast(result, resultType);
         return TryEmitResult(result, true);
@@ -2067,6 +2404,48 @@ CodeGenFunction::EmitARCRetainAutoreleaseScalarExpr(const Expr *e) {
   return value;
 }
 
+llvm::Value *CodeGenFunction::EmitARCExtendBlockObject(const Expr *e) {
+  llvm::Value *result;
+  bool doRetain;
+
+  if (shouldEmitSeparateBlockRetain(e)) {
+    result = EmitScalarExpr(e);
+    doRetain = true;
+  } else {
+    TryEmitResult subresult = tryEmitARCRetainScalarExpr(*this, e);
+    result = subresult.getPointer();
+    doRetain = !subresult.getInt();
+  }
+
+  if (doRetain)
+    result = EmitARCRetainBlock(result, /*mandatory*/ true);
+  return EmitObjCConsumeObject(e->getType(), result);
+}
+
+llvm::Value *CodeGenFunction::EmitObjCThrowOperand(const Expr *expr) {
+  // In ARC, retain and autorelease the expression.
+  if (getLangOptions().ObjCAutoRefCount) {
+    // Do so before running any cleanups for the full-expression.
+    // tryEmitARCRetainScalarExpr does make an effort to do things
+    // inside cleanups, but there are crazy cases like
+    //   @throw A().foo;
+    // where a full retain+autorelease is required and would
+    // otherwise happen after the destructor for the temporary.
+    CodeGenFunction::RunCleanupsScope cleanups(*this);
+    if (const ExprWithCleanups *ewc = dyn_cast<ExprWithCleanups>(expr))
+      expr = ewc->getSubExpr();
+
+    return EmitARCRetainAutoreleaseScalarExpr(expr);
+  }
+
+  // Otherwise, use the normal scalar-expression emission.  The
+  // exception machinery doesn't do anything special with the
+  // exception like retaining it, so there's no safety associated with
+  // only running cleanups after the throw has started, and when it
+  // matters it tends to be substantially inferior code.
+  return EmitScalarExpr(expr);
+}
+
 std::pair<LValue,llvm::Value*>
 CodeGenFunction::EmitARCStoreStrong(const BinaryOperator *e,
                                     bool ignored) {
@@ -2074,10 +2453,20 @@ CodeGenFunction::EmitARCStoreStrong(const BinaryOperator *e,
   TryEmitResult result = tryEmitARCRetainScalarExpr(*this, e->getRHS());
   llvm::Value *value = result.getPointer();
 
+  bool hasImmediateRetain = result.getInt();
+
+  // If we didn't emit a retained object, and the l-value is of block
+  // type, then we need to emit the block-retain immediately in case
+  // it invalidates the l-value.
+  if (!hasImmediateRetain && e->getType()->isBlockPointerType()) {
+    value = EmitARCRetainBlock(value, /*mandatory*/ false);
+    hasImmediateRetain = true;
+  }
+
   LValue lvalue = EmitLValue(e->getLHS());
 
   // If the RHS was emitted retained, expand this.
-  if (result.getInt()) {
+  if (hasImmediateRetain) {
     llvm::Value *oldValue =
       EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatileQualified(),
                        lvalue.getAlignment(), e->getType(),
@@ -2111,10 +2500,8 @@ void CodeGenFunction::EmitObjCAutoreleasePoolStmt(
   const CompoundStmt &S = cast<CompoundStmt>(*subStmt);
 
   CGDebugInfo *DI = getDebugInfo();
-  if (DI) {
-    DI->setLocation(S.getLBracLoc());
-    DI->EmitRegionStart(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockStart(Builder, S.getLBracLoc());
 
   // Keep track of the current cleanup stack depth.
   RunCleanupsScope Scope(*this);
@@ -2130,19 +2517,16 @@ void CodeGenFunction::EmitObjCAutoreleasePoolStmt(
        E = S.body_end(); I != E; ++I)
     EmitStmt(*I);
 
-  if (DI) {
-    DI->setLocation(S.getRBracLoc());
-    DI->EmitRegionEnd(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockEnd(Builder, S.getRBracLoc());
 }
 
 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
 /// make sure it survives garbage collection until this point.
 void CodeGenFunction::EmitExtendGCLifetime(llvm::Value *object) {
   // We just use an inline assembly.
-  llvm::Type *paramTypes[] = { VoidPtrTy };
   llvm::FunctionType *extenderType
-    = llvm::FunctionType::get(VoidTy, paramTypes, /*variadic*/ false);
+    = llvm::FunctionType::get(VoidTy, VoidPtrTy, /*variadic*/ false);
   llvm::Value *extender
     = llvm::InlineAsm::get(extenderType,
                            /* assembly */ "",
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp
index 61027feb5cb9..d3da649fbbf3 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -36,12 +36,11 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Target/TargetData.h"
 
-#include <stdarg.h>
+#include <cstdarg>
 
 
 using namespace clang;
 using namespace CodeGen;
-using llvm::dyn_cast;
 
 
 namespace {
@@ -82,7 +81,7 @@ class LazyRuntimeFunction {
      if (!Function) {
        if (0 == FunctionName) return 0;
        // We put the return type on the end of the vector, so pop it back off
-       const llvm::Type *RetTy = ArgTys.back();
+       llvm::Type *RetTy = ArgTys.back();
        ArgTys.pop_back();
        llvm::FunctionType *FTy = llvm::FunctionType::get(RetTy, ArgTys, false);
        Function =
@@ -111,17 +110,17 @@ class CGObjCGNU : public CGObjCRuntime {
   llvm::Module &TheModule;
   /// strut objc_super.  Used for sending messages to super.  This structure
   /// contains the receiver (object) and the expected class.
-  const llvm::StructType *ObjCSuperTy;
+  llvm::StructType *ObjCSuperTy;
   /// struct objc_super*.  The type of the argument to the superclass message
   /// lookup functions.  
-  const llvm::PointerType *PtrToObjCSuperTy;
+  llvm::PointerType *PtrToObjCSuperTy;
   /// LLVM type for selectors.  Opaque pointer (i8*) unless a header declaring
   /// SEL is included in a header somewhere, in which case it will be whatever
   /// type is declared in that header, most likely {i8*, i8*}.
   llvm::PointerType *SelectorTy;
   /// LLVM i8 type.  Cached here to avoid repeatedly getting it in all of the
   /// places where it's used
-  const llvm::IntegerType *Int8Ty;
+  llvm::IntegerType *Int8Ty;
   /// Pointer to i8 - LLVM type of char*, for all of the places where the
   /// runtime needs to deal with C strings.
   llvm::PointerType *PtrToInt8Ty;
@@ -138,7 +137,7 @@ class CGObjCGNU : public CGObjCRuntime {
   llvm::PointerType *IdTy;
   /// Pointer to a pointer to an Objective-C object.  Used in the new ABI
   /// message lookup function and some GC-related functions.
-  const llvm::PointerType *PtrToIdTy;
+  llvm::PointerType *PtrToIdTy;
   /// The clang type of id.  Used when using the clang CGCall infrastructure to
   /// call Objective-C methods.
   CanQualType ASTIdTy;
@@ -153,14 +152,20 @@ class CGObjCGNU : public CGObjCRuntime {
   /// compatibility with GCC...
   llvm::IntegerType *LongTy;
   /// LLVM type for C size_t.  Used in various runtime data structures.
-  const llvm::IntegerType *SizeTy;
+  llvm::IntegerType *SizeTy;
+  /// LLVM type for C intptr_t.  
+  llvm::IntegerType *IntPtrTy;
   /// LLVM type for C ptrdiff_t.  Mainly used in property accessor functions.
-  const llvm::IntegerType *PtrDiffTy;
+  llvm::IntegerType *PtrDiffTy;
   /// LLVM type for C int*.  Used for GCC-ABI-compatible non-fragile instance
   /// variables.
-  const llvm::PointerType *PtrToIntTy;
+  llvm::PointerType *PtrToIntTy;
   /// LLVM type for Objective-C BOOL type.
-  const llvm::Type *BoolTy;
+  llvm::Type *BoolTy;
+  /// 32-bit integer type, to save us needing to look it up every time it's used.
+  llvm::IntegerType *Int32Ty;
+  /// 64-bit integer type, to save us needing to look it up every time it's used.
+  llvm::IntegerType *Int64Ty;
   /// Metadata kind used to tie method lookups to message sends.  The GNUstep
   /// runtime provides some LLVM passes that can use this to do things like
   /// automatic IMP caching and speculative inlining.
@@ -171,7 +176,7 @@ class CGObjCGNU : public CGObjCRuntime {
   llvm::Constant *MakeConstantString(const std::string &Str,
                                      const std::string &Name="") {
     llvm::Constant *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
-    return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros, 2);
+    return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros);
   }
   /// Emits a linkonce_odr string, whose name is the prefix followed by the
   /// string value.  This allows the linker to combine the strings between
@@ -186,14 +191,14 @@ class CGObjCGNU : public CGObjCRuntime {
       ConstStr = new llvm::GlobalVariable(TheModule, value->getType(), true,
               llvm::GlobalValue::LinkOnceODRLinkage, value, prefix + Str);
     }
-    return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros, 2);
+    return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros);
   }
   /// Generates a global structure, initialized by the elements in the vector.
   /// The element types must match the types of the structure elements in the
   /// first argument.
-  llvm::GlobalVariable *MakeGlobal(const llvm::StructType *Ty,
-                                   std::vector<llvm::Constant*> &V,
-                                   llvm::StringRef Name="",
+  llvm::GlobalVariable *MakeGlobal(llvm::StructType *Ty,
+                                   llvm::ArrayRef<llvm::Constant*> V,
+                                   StringRef Name="",
                                    llvm::GlobalValue::LinkageTypes linkage
                                          =llvm::GlobalValue::InternalLinkage) {
     llvm::Constant *C = llvm::ConstantStruct::get(Ty, V);
@@ -203,9 +208,9 @@ class CGObjCGNU : public CGObjCRuntime {
   /// Generates a global array.  The vector must contain the same number of
   /// elements that the array type declares, of the type specified as the array
   /// element type.
-  llvm::GlobalVariable *MakeGlobal(const llvm::ArrayType *Ty,
-                                   std::vector<llvm::Constant*> &V,
-                                   llvm::StringRef Name="",
+  llvm::GlobalVariable *MakeGlobal(llvm::ArrayType *Ty,
+                                   llvm::ArrayRef<llvm::Constant*> V,
+                                   StringRef Name="",
                                    llvm::GlobalValue::LinkageTypes linkage
                                          =llvm::GlobalValue::InternalLinkage) {
     llvm::Constant *C = llvm::ConstantArray::get(Ty, V);
@@ -214,9 +219,9 @@ class CGObjCGNU : public CGObjCRuntime {
   }
   /// Generates a global array, inferring the array type from the specified
   /// element type and the size of the initialiser.  
-  llvm::GlobalVariable *MakeGlobalArray(const llvm::Type *Ty,
-                                        std::vector<llvm::Constant*> &V,
-                                        llvm::StringRef Name="",
+  llvm::GlobalVariable *MakeGlobalArray(llvm::Type *Ty,
+                                        llvm::ArrayRef<llvm::Constant*> V,
+                                        StringRef Name="",
                                         llvm::GlobalValue::LinkageTypes linkage
                                          =llvm::GlobalValue::InternalLinkage) {
     llvm::ArrayType *ArrayTy = llvm::ArrayType::get(Ty, V.size());
@@ -225,7 +230,7 @@ class CGObjCGNU : public CGObjCRuntime {
   /// Ensures that the value has the required type, by inserting a bitcast if
   /// required.  This function lets us avoid inserting bitcasts that are
   /// redundant.
-  llvm::Value* EnforceType(CGBuilderTy B, llvm::Value *V, const llvm::Type *Ty){
+  llvm::Value* EnforceType(CGBuilderTy B, llvm::Value *V, llvm::Type *Ty){
     if (V->getType() == Ty) return V;
     return B.CreateBitCast(V, Ty);
   }
@@ -268,7 +273,7 @@ class CGObjCGNU : public CGObjCRuntime {
   /// Type of the selector map.  This is roughly equivalent to the structure
   /// used in the GNUstep runtime, which maintains a list of all of the valid
   /// types for a selector in a table.
-  typedef llvm::DenseMap<Selector, llvm::SmallVector<TypedSelector, 2> >
+  typedef llvm::DenseMap<Selector, SmallVector<TypedSelector, 2> >
     SelectorMap;
   /// A map from selectors to selector types.  This allows us to emit all
   /// selectors of the same name and type together.
@@ -332,18 +337,18 @@ class CGObjCGNU : public CGObjCRuntime {
   /// metadata.  This is used purely for introspection in the fragile ABI.  In
   /// the non-fragile ABI, it's used for instance variable fixup.
   llvm::Constant *GenerateIvarList(
-      const llvm::SmallVectorImpl<llvm::Constant *>  &IvarNames,
-      const llvm::SmallVectorImpl<llvm::Constant *>  &IvarTypes,
-      const llvm::SmallVectorImpl<llvm::Constant *>  &IvarOffsets);
+      const SmallVectorImpl<llvm::Constant *>  &IvarNames,
+      const SmallVectorImpl<llvm::Constant *>  &IvarTypes,
+      const SmallVectorImpl<llvm::Constant *>  &IvarOffsets);
   /// Generates a method list structure.  This is a structure containing a size
   /// and an array of structures containing method metadata.
   ///
   /// This structure is used by both classes and categories, and contains a next
   /// pointer allowing them to be chained together in a linked list.
-  llvm::Constant *GenerateMethodList(const llvm::StringRef &ClassName,
-      const llvm::StringRef &CategoryName,
-      const llvm::SmallVectorImpl<Selector>  &MethodSels,
-      const llvm::SmallVectorImpl<llvm::Constant *>  &MethodTypes,
+  llvm::Constant *GenerateMethodList(const StringRef &ClassName,
+      const StringRef &CategoryName,
+      const SmallVectorImpl<Selector>  &MethodSels,
+      const SmallVectorImpl<llvm::Constant *>  &MethodTypes,
       bool isClassMethodList);
   /// Emits an empty protocol.  This is used for @protocol() where no protocol
   /// is found.  The runtime will (hopefully) fix up the pointer to refer to the
@@ -352,12 +357,12 @@ class CGObjCGNU : public CGObjCRuntime {
   /// Generates a list of property metadata structures.  This follows the same
   /// pattern as method and instance variable metadata lists.
   llvm::Constant *GeneratePropertyList(const ObjCImplementationDecl *OID,
-        llvm::SmallVectorImpl<Selector> &InstanceMethodSels,
-        llvm::SmallVectorImpl<llvm::Constant*> &InstanceMethodTypes);
+        SmallVectorImpl<Selector> &InstanceMethodSels,
+        SmallVectorImpl<llvm::Constant*> &InstanceMethodTypes);
   /// Generates a list of referenced protocols.  Classes, categories, and
   /// protocols all use this structure.
   llvm::Constant *GenerateProtocolList(
-      const llvm::SmallVectorImpl<std::string> &Protocols);
+      const SmallVectorImpl<std::string> &Protocols);
   /// To ensure that all protocols are seen by the runtime, we add a category on
   /// a class defined in the runtime, declaring no methods, but adopting the
   /// protocols.  This is a horribly ugly hack, but it allows us to collect all
@@ -376,12 +381,14 @@ class CGObjCGNU : public CGObjCRuntime {
       llvm::Constant *Protocols,
       llvm::Constant *IvarOffsets,
       llvm::Constant *Properties,
+      llvm::Constant *StrongIvarBitmap,
+      llvm::Constant *WeakIvarBitmap,
       bool isMeta=false);
   /// Generates a method list.  This is used by protocols to define the required
   /// and optional methods.
   llvm::Constant *GenerateProtocolMethodList(
-      const llvm::SmallVectorImpl<llvm::Constant *>  &MethodNames,
-      const llvm::SmallVectorImpl<llvm::Constant *>  &MethodTypes);
+      const SmallVectorImpl<llvm::Constant *>  &MethodNames,
+      const SmallVectorImpl<llvm::Constant *>  &MethodTypes);
   /// Returns a selector with the specified type encoding.  An empty string is
   /// used to return an untyped selector (with the types field set to NULL).
   llvm::Value *GetSelector(CGBuilderTy &Builder, Selector Sel,
@@ -403,12 +410,24 @@ class CGObjCGNU : public CGObjCRuntime {
                                  llvm::Value *&Receiver,
                                  llvm::Value *cmd,
                                  llvm::MDNode *node) = 0;
-  /// Looks up the method for sending a message to a superclass.  This mechanism
-  /// differs between the GCC and GNU runtimes, so this method must be
-  /// overridden in subclasses.
+  /// Looks up the method for sending a message to a superclass.  This
+  /// mechanism differs between the GCC and GNU runtimes, so this method must
+  /// be overridden in subclasses.
   virtual llvm::Value *LookupIMPSuper(CodeGenFunction &CGF,
                                       llvm::Value *ObjCSuper,
                                       llvm::Value *cmd) = 0;
+  /// Libobjc2 uses a bitfield representation where small(ish) bitfields are
+  /// stored in a 64-bit value with the low bit set to 1 and the remaining 63
+  /// bits set to their values, LSB first, while larger ones are stored in a
+  /// structure of this / form:
+  /// 
+  /// struct { int32_t length; int32_t values[length]; };
+  ///
+  /// The values in the array are stored in host-endian format, with the least
+  /// significant bit being assumed to come first in the bitfield.  Therefore,
+  /// a bitfield with the 64th bit set will be (int64_t)&{ 2, [0, 1<<31] },
+  /// while a bitfield / with the 63rd bit set will be 1<<64.
+  llvm::Constant *MakeBitField(llvm::SmallVectorImpl<bool> &bits);
 public:
   CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
       unsigned protocolClassVersion);
@@ -622,7 +641,7 @@ class CGObjCGNUstep : public CGObjCGNU {
         // void *__cxa_begin_catch(void *e)
         EnterCatchFn.init(&CGM, "__cxa_begin_catch", PtrTy, PtrTy, NULL);
         // void __cxa_end_catch(void)
-        EnterCatchFn.init(&CGM, "__cxa_end_catch", VoidTy, NULL);
+        ExitCatchFn.init(&CGM, "__cxa_end_catch", VoidTy, NULL);
         // void _Unwind_Resume_or_Rethrow(void*)
         ExceptionReThrowFn.init(&CGM, "_Unwind_Resume_or_Rethrow", VoidTy, PtrTy, NULL);
       }
@@ -650,13 +669,13 @@ void CGObjCGNU::EmitClassRef(const std::string &className) {
     llvm::GlobalValue::WeakAnyLinkage, ClassSymbol, symbolRef);
 }
 
-static std::string SymbolNameForMethod(const llvm::StringRef &ClassName,
-    const llvm::StringRef &CategoryName, const Selector MethodName,
+static std::string SymbolNameForMethod(const StringRef &ClassName,
+    const StringRef &CategoryName, const Selector MethodName,
     bool isClassMethod) {
   std::string MethodNameColonStripped = MethodName.getAsString();
   std::replace(MethodNameColonStripped.begin(), MethodNameColonStripped.end(),
       ':', '_');
-  return (llvm::Twine(isClassMethod ? "_c_" : "_i_") + ClassName + "_" +
+  return (Twine(isClassMethod ? "_c_" : "_i_") + ClassName + "_" +
     CategoryName + "_" + MethodNameColonStripped).str();
 }
 
@@ -697,6 +716,12 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
   PtrToIntTy = llvm::PointerType::getUnqual(IntTy);
   PtrTy = PtrToInt8Ty;
 
+  Int32Ty = llvm::Type::getInt32Ty(VMContext);
+  Int64Ty = llvm::Type::getInt64Ty(VMContext);
+
+  IntPtrTy =
+      TheModule.getPointerSize() == llvm::Module::Pointer32 ? Int32Ty : Int64Ty;
+
   // Object type
   QualType UnqualIdTy = CGM.getContext().getObjCIdType();
   ASTIdTy = CanQualType();
@@ -744,11 +769,11 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
               true));
 
   const LangOptions &Opts = CGM.getLangOptions();
-  if ((Opts.getGCMode() != LangOptions::NonGC) || Opts.ObjCAutoRefCount)
+  if ((Opts.getGC() != LangOptions::NonGC) || Opts.ObjCAutoRefCount)
     RuntimeVersion = 10;
 
   // Don't bother initialising the GC stuff unless we're compiling in GC mode
-  if (Opts.getGCMode() != LangOptions::NonGC) {
+  if (Opts.getGC() != LangOptions::NonGC) {
     // This is a bit of an hack.  We should sort this out by having a proper
     // CGObjCGNUstep subclass for GC, but we may want to really support the old
     // ABI and GC added in ObjectiveC2.framework, so we fudge it a bit for now
@@ -793,9 +818,8 @@ llvm::Value *CGObjCGNU::GetClassNamed(CGBuilderTy &Builder,
     EmitClassRef(Name);
   ClassName = Builder.CreateStructGEP(ClassName, 0);
 
-  llvm::Type *ArgTys[] = { PtrToInt8Ty };
   llvm::Constant *ClassLookupFn =
-    CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, ArgTys, true),
+    CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, PtrToInt8Ty, true),
                               "objc_lookup_class");
   return Builder.CreateCall(ClassLookupFn, ClassName);
 }
@@ -813,11 +837,11 @@ llvm::Value *CGObjCGNU::EmitNSAutoreleasePoolClassRef(CGBuilderTy &Builder) {
 llvm::Value *CGObjCGNU::GetSelector(CGBuilderTy &Builder, Selector Sel,
     const std::string &TypeEncoding, bool lval) {
 
-  llvm::SmallVector<TypedSelector, 2> &Types = SelectorTable[Sel];
+  SmallVector<TypedSelector, 2> &Types = SelectorTable[Sel];
   llvm::GlobalAlias *SelValue = 0;
 
 
-  for (llvm::SmallVectorImpl<TypedSelector>::iterator i = Types.begin(),
+  for (SmallVectorImpl<TypedSelector>::iterator i = Types.begin(),
       e = Types.end() ; i!=e ; i++) {
     if (i->first == TypeEncoding) {
       SelValue = i->second;
@@ -918,7 +942,7 @@ llvm::Constant *CGObjCGNU::GetEHType(QualType T) {
             llvm::GlobalValue::ExternalLinkage, 0, vtableName);
   }
   llvm::Constant *Two = llvm::ConstantInt::get(IntTy, 2);
-  Vtable = llvm::ConstantExpr::getGetElementPtr(Vtable, &Two, 1);
+  Vtable = llvm::ConstantExpr::getGetElementPtr(Vtable, Two);
   Vtable = llvm::ConstantExpr::getBitCast(Vtable, PtrToInt8Ty);
 
   llvm::Constant *typeName =
@@ -972,7 +996,7 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
                                     const CallArgList &CallArgs,
                                     const ObjCMethodDecl *Method) {
   CGBuilderTy &Builder = CGF.Builder;
-  if (CGM.getLangOptions().getGCMode() == LangOptions::GCOnly) {
+  if (CGM.getLangOptions().getGC() == LangOptions::GCOnly) {
     if (Sel == RetainSel || Sel == AutoreleaseSel) {
       return RValue::get(EnforceType(Builder, Receiver,
                   CGM.getTypes().ConvertType(ResultType)));
@@ -999,13 +1023,11 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
   if (isCategoryImpl) {
     llvm::Constant *classLookupFunction = 0;
     if (IsClassMessage)  {
-      llvm::Type *ArgTys[] = { PtrTy };
       classLookupFunction = CGM.CreateRuntimeFunction(llvm::FunctionType::get(
-            IdTy, ArgTys, true), "objc_get_meta_class");
+            IdTy, PtrTy, true), "objc_get_meta_class");
     } else {
-      llvm::Type *ArgTys[] = { PtrTy };
       classLookupFunction = CGM.CreateRuntimeFunction(llvm::FunctionType::get(
-            IdTy, ArgTys, true), "objc_get_class");
+            IdTy, PtrTy, true), "objc_get_class");
     }
     ReceiverClass = Builder.CreateCall(classLookupFunction,
         MakeConstantString(Class->getNameAsString()));
@@ -1048,7 +1070,7 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
   Builder.CreateStore(ReceiverClass, Builder.CreateStructGEP(ObjCSuper, 1));
 
   ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy);
-  const llvm::FunctionType *impType =
+  llvm::FunctionType *impType =
     Types.GetFunctionType(FnInfo, Method ? Method->isVariadic() : false);
 
   // Get the IMP
@@ -1082,7 +1104,7 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
   CGBuilderTy &Builder = CGF.Builder;
 
   // Strip out message sends to retain / release in GC mode
-  if (CGM.getLangOptions().getGCMode() == LangOptions::GCOnly) {
+  if (CGM.getLangOptions().getGC() == LangOptions::GCOnly) {
     if (Sel == RetainSel || Sel == AutoreleaseSel) {
       return RValue::get(EnforceType(Builder, Receiver,
                   CGM.getTypes().ConvertType(ResultType)));
@@ -1148,7 +1170,7 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
   CodeGenTypes &Types = CGM.getTypes();
   const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs,
                                                        FunctionType::ExtInfo());
-  const llvm::FunctionType *impType =
+  llvm::FunctionType *impType =
     Types.GetFunctionType(FnInfo, Method ? Method->isVariadic() : false);
   imp = EnforceType(Builder, imp, llvm::PointerType::getUnqual(impType));
 
@@ -1175,7 +1197,7 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
     } else if (msgRet.isAggregate()) {
       llvm::Value *v = msgRet.getAggregateAddr();
       llvm::PHINode *phi = Builder.CreatePHI(v->getType(), 2);
-      const llvm::PointerType *RetTy = cast<llvm::PointerType>(v->getType());
+      llvm::PointerType *RetTy = cast<llvm::PointerType>(v->getType());
       llvm::AllocaInst *NullVal = 
           CGF.CreateTempAlloca(RetTy->getElementType(), "null");
       CGF.InitTempAlloca(NullVal,
@@ -1201,10 +1223,10 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
 
 /// Generates a MethodList.  Used in construction of a objc_class and
 /// objc_category structures.
-llvm::Constant *CGObjCGNU::GenerateMethodList(const llvm::StringRef &ClassName,
-                                              const llvm::StringRef &CategoryName,
-    const llvm::SmallVectorImpl<Selector> &MethodSels,
-    const llvm::SmallVectorImpl<llvm::Constant *> &MethodTypes,
+llvm::Constant *CGObjCGNU::GenerateMethodList(const StringRef &ClassName,
+                                              const StringRef &CategoryName,
+    const SmallVectorImpl<Selector> &MethodSels,
+    const SmallVectorImpl<llvm::Constant *> &MethodTypes,
     bool isClassMethodList) {
   if (MethodSels.empty())
     return NULLPtr;
@@ -1239,8 +1261,7 @@ llvm::Constant *CGObjCGNU::GenerateMethodList(const llvm::StringRef &ClassName,
                                                          Methods);
 
   // Structure containing list pointer, array and array count
-  llvm::StructType *ObjCMethodListTy =
-    llvm::StructType::createNamed(VMContext, "");
+  llvm::StructType *ObjCMethodListTy = llvm::StructType::create(VMContext);
   llvm::Type *NextPtrTy = llvm::PointerType::getUnqual(ObjCMethodListTy);
   ObjCMethodListTy->setBody(
       NextPtrTy,
@@ -1251,8 +1272,7 @@ llvm::Constant *CGObjCGNU::GenerateMethodList(const llvm::StringRef &ClassName,
   Methods.clear();
   Methods.push_back(llvm::ConstantPointerNull::get(
         llvm::PointerType::getUnqual(ObjCMethodListTy)));
-  Methods.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
-        MethodTypes.size()));
+  Methods.push_back(llvm::ConstantInt::get(Int32Ty, MethodTypes.size()));
   Methods.push_back(MethodArray);
 
   // Create an instance of the structure
@@ -1261,9 +1281,9 @@ llvm::Constant *CGObjCGNU::GenerateMethodList(const llvm::StringRef &ClassName,
 
 /// Generates an IvarList.  Used in construction of a objc_class.
 llvm::Constant *CGObjCGNU::GenerateIvarList(
-    const llvm::SmallVectorImpl<llvm::Constant *>  &IvarNames,
-    const llvm::SmallVectorImpl<llvm::Constant *>  &IvarTypes,
-    const llvm::SmallVectorImpl<llvm::Constant *>  &IvarOffsets) {
+    const SmallVectorImpl<llvm::Constant *>  &IvarNames,
+    const SmallVectorImpl<llvm::Constant *>  &IvarTypes,
+    const SmallVectorImpl<llvm::Constant *>  &IvarOffsets) {
   if (IvarNames.size() == 0)
     return NULLPtr;
   // Get the method structure type.
@@ -1312,6 +1332,8 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure(
     llvm::Constant *Protocols,
     llvm::Constant *IvarOffsets,
     llvm::Constant *Properties,
+    llvm::Constant *StrongIvarBitmap,
+    llvm::Constant *WeakIvarBitmap,
     bool isMeta) {
   // Set up the class structure
   // Note:  Several of these are char*s when they should be ids.  This is
@@ -1339,6 +1361,8 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure(
       LongTy,                 // abi_version
       IvarOffsets->getType(), // ivar_offsets
       Properties->getType(),  // properties
+      Int64Ty,                // strong_pointers
+      Int64Ty,                // weak_pointers
       NULL);
   llvm::Constant *Zero = llvm::ConstantInt::get(LongTy, 0);
   // Fill in the structure
@@ -1363,9 +1387,11 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure(
   Elements.push_back(NULLPtr);
   Elements.push_back(llvm::ConstantExpr::getBitCast(Protocols, PtrTy));
   Elements.push_back(NULLPtr);
-  Elements.push_back(Zero);
+  Elements.push_back(llvm::ConstantInt::get(LongTy, 1));
   Elements.push_back(IvarOffsets);
   Elements.push_back(Properties);
+  Elements.push_back(StrongIvarBitmap);
+  Elements.push_back(WeakIvarBitmap);
   // Create an instance of the structure
   // This is now an externally visible symbol, so that we can speed up class
   // messages in the next ABI.
@@ -1374,8 +1400,8 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure(
 }
 
 llvm::Constant *CGObjCGNU::GenerateProtocolMethodList(
-    const llvm::SmallVectorImpl<llvm::Constant *>  &MethodNames,
-    const llvm::SmallVectorImpl<llvm::Constant *>  &MethodTypes) {
+    const SmallVectorImpl<llvm::Constant *>  &MethodNames,
+    const SmallVectorImpl<llvm::Constant *>  &MethodTypes) {
   // Get the method structure type.
   llvm::StructType *ObjCMethodDescTy = llvm::StructType::get(
     PtrToInt8Ty, // Really a selector, but the runtime does the casting for us.
@@ -1403,7 +1429,7 @@ llvm::Constant *CGObjCGNU::GenerateProtocolMethodList(
 
 // Create the protocol list structure used in classes, categories and so on
 llvm::Constant *CGObjCGNU::GenerateProtocolList(
-    const llvm::SmallVectorImpl<std::string> &Protocols) {
+    const SmallVectorImpl<std::string> &Protocols) {
   llvm::ArrayType *ProtocolArrayTy = llvm::ArrayType::get(PtrToInt8Ty,
       Protocols.size());
   llvm::StructType *ProtocolListTy = llvm::StructType::get(
@@ -1438,15 +1464,15 @@ llvm::Constant *CGObjCGNU::GenerateProtocolList(
 llvm::Value *CGObjCGNU::GenerateProtocolRef(CGBuilderTy &Builder,
                                             const ObjCProtocolDecl *PD) {
   llvm::Value *protocol = ExistingProtocols[PD->getNameAsString()];
-  const llvm::Type *T =
+  llvm::Type *T =
     CGM.getTypes().ConvertType(CGM.getContext().getObjCProtoType());
   return Builder.CreateBitCast(protocol, llvm::PointerType::getUnqual(T));
 }
 
 llvm::Constant *CGObjCGNU::GenerateEmptyProtocol(
   const std::string &ProtocolName) {
-  llvm::SmallVector<std::string, 0> EmptyStringVector;
-  llvm::SmallVector<llvm::Constant*, 0> EmptyConstantVector;
+  SmallVector<std::string, 0> EmptyStringVector;
+  SmallVector<llvm::Constant*, 0> EmptyConstantVector;
 
   llvm::Constant *ProtocolList = GenerateProtocolList(EmptyStringVector);
   llvm::Constant *MethodList =
@@ -1465,8 +1491,7 @@ llvm::Constant *CGObjCGNU::GenerateEmptyProtocol(
   // The isa pointer must be set to a magic number so the runtime knows it's
   // the correct layout.
   Elements.push_back(llvm::ConstantExpr::getIntToPtr(
-        llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
-          ProtocolVersion), IdTy));
+        llvm::ConstantInt::get(Int32Ty, ProtocolVersion), IdTy));
   Elements.push_back(MakeConstantString(ProtocolName, ".objc_protocol_name"));
   Elements.push_back(ProtocolList);
   Elements.push_back(MethodList);
@@ -1479,14 +1504,14 @@ llvm::Constant *CGObjCGNU::GenerateEmptyProtocol(
 void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
   ASTContext &Context = CGM.getContext();
   std::string ProtocolName = PD->getNameAsString();
-  llvm::SmallVector<std::string, 16> Protocols;
+  SmallVector<std::string, 16> Protocols;
   for (ObjCProtocolDecl::protocol_iterator PI = PD->protocol_begin(),
        E = PD->protocol_end(); PI != E; ++PI)
     Protocols.push_back((*PI)->getNameAsString());
-  llvm::SmallVector<llvm::Constant*, 16> InstanceMethodNames;
-  llvm::SmallVector<llvm::Constant*, 16> InstanceMethodTypes;
-  llvm::SmallVector<llvm::Constant*, 16> OptionalInstanceMethodNames;
-  llvm::SmallVector<llvm::Constant*, 16> OptionalInstanceMethodTypes;
+  SmallVector<llvm::Constant*, 16> InstanceMethodNames;
+  SmallVector<llvm::Constant*, 16> InstanceMethodTypes;
+  SmallVector<llvm::Constant*, 16> OptionalInstanceMethodNames;
+  SmallVector<llvm::Constant*, 16> OptionalInstanceMethodTypes;
   for (ObjCProtocolDecl::instmeth_iterator iter = PD->instmeth_begin(),
        E = PD->instmeth_end(); iter != E; iter++) {
     std::string TypeStr;
@@ -1502,10 +1527,10 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
     }
   }
   // Collect information about class methods:
-  llvm::SmallVector<llvm::Constant*, 16> ClassMethodNames;
-  llvm::SmallVector<llvm::Constant*, 16> ClassMethodTypes;
-  llvm::SmallVector<llvm::Constant*, 16> OptionalClassMethodNames;
-  llvm::SmallVector<llvm::Constant*, 16> OptionalClassMethodTypes;
+  SmallVector<llvm::Constant*, 16> ClassMethodNames;
+  SmallVector<llvm::Constant*, 16> ClassMethodTypes;
+  SmallVector<llvm::Constant*, 16> OptionalClassMethodNames;
+  SmallVector<llvm::Constant*, 16> OptionalClassMethodTypes;
   for (ObjCProtocolDecl::classmeth_iterator
          iter = PD->classmeth_begin(), endIter = PD->classmeth_end();
        iter != endIter ; iter++) {
@@ -1626,8 +1651,7 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
   // The isa pointer must be set to a magic number so the runtime knows it's
   // the correct layout.
   Elements.push_back(llvm::ConstantExpr::getIntToPtr(
-        llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
-          ProtocolVersion), IdTy));
+        llvm::ConstantInt::get(Int32Ty, ProtocolVersion), IdTy));
   Elements.push_back(MakeConstantString(ProtocolName, ".objc_protocol_name"));
   Elements.push_back(ProtocolList);
   Elements.push_back(InstanceMethodList);
@@ -1642,8 +1666,8 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
 }
 void CGObjCGNU::GenerateProtocolHolderCategory(void) {
   // Collect information about instance methods
-  llvm::SmallVector<Selector, 1> MethodSels;
-  llvm::SmallVector<llvm::Constant*, 1> MethodTypes;
+  SmallVector<Selector, 1> MethodSels;
+  SmallVector<llvm::Constant*, 1> MethodTypes;
 
   std::vector<llvm::Constant*> Elements;
   const std::string ClassName = "__ObjC_Protocol_Holder_Ugly_Hack";
@@ -1686,12 +1710,55 @@ void CGObjCGNU::GenerateProtocolHolderCategory(void) {
             PtrTy, PtrTy, PtrTy, NULL), Elements), PtrTy));
 }
 
+/// Libobjc2 uses a bitfield representation where small(ish) bitfields are
+/// stored in a 64-bit value with the low bit set to 1 and the remaining 63
+/// bits set to their values, LSB first, while larger ones are stored in a
+/// structure of this / form:
+/// 
+/// struct { int32_t length; int32_t values[length]; };
+///
+/// The values in the array are stored in host-endian format, with the least
+/// significant bit being assumed to come first in the bitfield.  Therefore, a
+/// bitfield with the 64th bit set will be (int64_t)&{ 2, [0, 1<<31] }, while a
+/// bitfield / with the 63rd bit set will be 1<<64.
+llvm::Constant *CGObjCGNU::MakeBitField(llvm::SmallVectorImpl<bool> &bits) {
+  int bitCount = bits.size();
+  if (bitCount < 64) {
+    uint64_t val = 1;
+    for (int i=0 ; i<bitCount ; ++i) {
+      if (bits[i]) val |= 1ULL<<(i+1);
+    }
+    return llvm::ConstantInt::get(Int64Ty, val);
+  }
+  llvm::SmallVector<llvm::Constant*, 8> values;
+  int v=0;
+  while (v < bitCount) {
+    int32_t word = 0;
+    for (int i=0 ; (i<32) && (v<bitCount)  ; ++i) {
+      if (bits[v]) word |= 1<<i;
+      v++;
+    }
+    values.push_back(llvm::ConstantInt::get(Int32Ty, word));
+  }
+  llvm::ArrayType *arrayTy = llvm::ArrayType::get(Int32Ty, values.size());
+  llvm::Constant *array = llvm::ConstantArray::get(arrayTy, values);
+  llvm::Constant *fields[2] = {
+      llvm::ConstantInt::get(Int32Ty, values.size()),
+      array };
+  llvm::Constant *GS = MakeGlobal(llvm::StructType::get(Int32Ty, arrayTy,
+        NULL), fields);
+  llvm::Constant *ptr = llvm::ConstantExpr::getPtrToInt(GS, IntPtrTy);
+  if (IntPtrTy != Int64Ty)
+    ptr = llvm::ConstantExpr::getZExt(ptr, Int64Ty);
+  return ptr;
+}
+
 void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
   std::string ClassName = OCD->getClassInterface()->getNameAsString();
   std::string CategoryName = OCD->getNameAsString();
   // Collect information about instance methods
-  llvm::SmallVector<Selector, 16> InstanceMethodSels;
-  llvm::SmallVector<llvm::Constant*, 16> InstanceMethodTypes;
+  SmallVector<Selector, 16> InstanceMethodSels;
+  SmallVector<llvm::Constant*, 16> InstanceMethodTypes;
   for (ObjCCategoryImplDecl::instmeth_iterator
          iter = OCD->instmeth_begin(), endIter = OCD->instmeth_end();
        iter != endIter ; iter++) {
@@ -1702,8 +1769,8 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
   }
 
   // Collect information about class methods
-  llvm::SmallVector<Selector, 16> ClassMethodSels;
-  llvm::SmallVector<llvm::Constant*, 16> ClassMethodTypes;
+  SmallVector<Selector, 16> ClassMethodSels;
+  SmallVector<llvm::Constant*, 16> ClassMethodTypes;
   for (ObjCCategoryImplDecl::classmeth_iterator
          iter = OCD->classmeth_begin(), endIter = OCD->classmeth_end();
        iter != endIter ; iter++) {
@@ -1714,7 +1781,7 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
   }
 
   // Collect the names of referenced protocols
-  llvm::SmallVector<std::string, 16> Protocols;
+  SmallVector<std::string, 16> Protocols;
   const ObjCCategoryDecl *CatDecl = OCD->getCategoryDecl();
   const ObjCList<ObjCProtocolDecl> &Protos = CatDecl->getReferencedProtocols();
   for (ObjCList<ObjCProtocolDecl>::iterator I = Protos.begin(),
@@ -1741,8 +1808,8 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
 }
 
 llvm::Constant *CGObjCGNU::GeneratePropertyList(const ObjCImplementationDecl *OID,
-        llvm::SmallVectorImpl<Selector> &InstanceMethodSels,
-        llvm::SmallVectorImpl<llvm::Constant*> &InstanceMethodTypes) {
+        SmallVectorImpl<Selector> &InstanceMethodSels,
+        SmallVectorImpl<llvm::Constant*> &InstanceMethodTypes) {
   ASTContext &Context = CGM.getContext();
   //
   // Property metadata: name, attributes, isSynthesized, setter name, setter
@@ -1845,11 +1912,13 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
     Context.getASTObjCImplementationLayout(OID).getSize().getQuantity();
 
   // Collect information about instance variables.
-  llvm::SmallVector<llvm::Constant*, 16> IvarNames;
-  llvm::SmallVector<llvm::Constant*, 16> IvarTypes;
-  llvm::SmallVector<llvm::Constant*, 16> IvarOffsets;
+  SmallVector<llvm::Constant*, 16> IvarNames;
+  SmallVector<llvm::Constant*, 16> IvarTypes;
+  SmallVector<llvm::Constant*, 16> IvarOffsets;
 
   std::vector<llvm::Constant*> IvarOffsetValues;
+  SmallVector<bool, 16> WeakIvars;
+  SmallVector<bool, 16> StrongIvars;
 
   int superInstanceSize = !SuperClassDecl ? 0 :
     Context.getASTObjCInterfaceLayout(SuperClassDecl).getSize().getQuantity();
@@ -1859,12 +1928,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
     instanceSize = 0 - (instanceSize - superInstanceSize);
   }
 
-  // Collect declared and synthesized ivars.
-  llvm::SmallVector<ObjCIvarDecl*, 16> OIvars;
-  CGM.getContext().ShallowCollectObjCIvars(ClassDecl, OIvars);
-
-  for (unsigned i = 0, e = OIvars.size(); i != e; ++i) {
-      ObjCIvarDecl *IVD = OIvars[i];
+  for (const ObjCIvarDecl *IVD = ClassDecl->all_declared_ivar_begin(); IVD;
+       IVD = IVD->getNextIvar()) {
       // Store the name
       IvarNames.push_back(MakeConstantString(IVD->getNameAsString()));
       // Get the type encoding for this ivar
@@ -1896,14 +1961,30 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
           IVD->getNameAsString());
       IvarOffsets.push_back(OffsetValue);
       IvarOffsetValues.push_back(OffsetVar);
+      Qualifiers::ObjCLifetime lt = IVD->getType().getQualifiers().getObjCLifetime();
+      switch (lt) {
+        case Qualifiers::OCL_Strong:
+          StrongIvars.push_back(true);
+          WeakIvars.push_back(false);
+          break;
+        case Qualifiers::OCL_Weak:
+          StrongIvars.push_back(false);
+          WeakIvars.push_back(true);
+          break;
+        default:
+          StrongIvars.push_back(false);
+          WeakIvars.push_back(false);
+      }
   }
+  llvm::Constant *StrongIvarBitmap = MakeBitField(StrongIvars);
+  llvm::Constant *WeakIvarBitmap = MakeBitField(WeakIvars);
   llvm::GlobalVariable *IvarOffsetArray =
     MakeGlobalArray(PtrToIntTy, IvarOffsetValues, ".ivar.offsets");
 
 
   // Collect information about instance methods
-  llvm::SmallVector<Selector, 16> InstanceMethodSels;
-  llvm::SmallVector<llvm::Constant*, 16> InstanceMethodTypes;
+  SmallVector<Selector, 16> InstanceMethodSels;
+  SmallVector<llvm::Constant*, 16> InstanceMethodTypes;
   for (ObjCImplementationDecl::instmeth_iterator
          iter = OID->instmeth_begin(), endIter = OID->instmeth_end();
        iter != endIter ; iter++) {
@@ -1918,8 +1999,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
 
 
   // Collect information about class methods
-  llvm::SmallVector<Selector, 16> ClassMethodSels;
-  llvm::SmallVector<llvm::Constant*, 16> ClassMethodTypes;
+  SmallVector<Selector, 16> ClassMethodSels;
+  SmallVector<llvm::Constant*, 16> ClassMethodTypes;
   for (ObjCImplementationDecl::classmeth_iterator
          iter = OID->classmeth_begin(), endIter = OID->classmeth_end();
        iter != endIter ; iter++) {
@@ -1929,7 +2010,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
     ClassMethodTypes.push_back(MakeConstantString(TypeStr));
   }
   // Collect the names of referenced protocols
-  llvm::SmallVector<std::string, 16> Protocols;
+  SmallVector<std::string, 16> Protocols;
   const ObjCList<ObjCProtocolDecl> &Protos =ClassDecl->getReferencedProtocols();
   for (ObjCList<ObjCProtocolDecl>::iterator I = Protos.begin(),
        E = Protos.end(); I != E; ++I)
@@ -1945,7 +2026,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
     SuperClass = llvm::ConstantPointerNull::get(PtrToInt8Ty);
   }
   // Empty vector used to construct empty method lists
-  llvm::SmallVector<llvm::Constant*, 1>  empty;
+  SmallVector<llvm::Constant*, 1>  empty;
   // Generate the method and instance variable lists
   llvm::Constant *MethodList = GenerateMethodList(ClassName, "",
       InstanceMethodSels, InstanceMethodTypes, false);
@@ -1963,20 +2044,20 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
   // setting up the alias.  These are: The base address for the global, the
   // ivar array (second field), the ivar in this list (set for each ivar), and
   // the offset (third field in ivar structure)
-  const llvm::Type *IndexTy = llvm::Type::getInt32Ty(VMContext);
+  llvm::Type *IndexTy = Int32Ty;
   llvm::Constant *offsetPointerIndexes[] = {Zeros[0],
       llvm::ConstantInt::get(IndexTy, 1), 0,
       llvm::ConstantInt::get(IndexTy, 2) };
 
-
-  for (unsigned i = 0, e = OIvars.size(); i != e; ++i) {
-      ObjCIvarDecl *IVD = OIvars[i];
+  unsigned ivarIndex = 0;
+  for (const ObjCIvarDecl *IVD = ClassDecl->all_declared_ivar_begin(); IVD;
+       IVD = IVD->getNextIvar()) {
       const std::string Name = "__objc_ivar_offset_" + ClassName + '.'
           + IVD->getNameAsString();
-      offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, i);
+      offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, ivarIndex);
       // Get the correct ivar field
       llvm::Constant *offsetValue = llvm::ConstantExpr::getGetElementPtr(
-              IvarList, offsetPointerIndexes, 4);
+              IvarList, offsetPointerIndexes);
       // Get the existing variable, if one exists.
       llvm::GlobalVariable *offset = TheModule.getNamedGlobal(Name);
       if (offset) {
@@ -1990,11 +2071,14 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
           offset = new llvm::GlobalVariable(TheModule, offsetValue->getType(),
                   false, llvm::GlobalValue::ExternalLinkage, offsetValue, Name);
       }
+      ++ivarIndex;
   }
+  llvm::Constant *Zero64 = llvm::ConstantInt::get(Int64Ty, 0);
   //Generate metaclass for class methods
   llvm::Constant *MetaClassStruct = GenerateClassStructure(NULLPtr,
       NULLPtr, 0x12L, ClassName.c_str(), 0, Zeros[0], GenerateIvarList(
-        empty, empty, empty), ClassMethodList, NULLPtr, NULLPtr, NULLPtr, true);
+        empty, empty, empty), ClassMethodList, NULLPtr,
+      NULLPtr, NULLPtr, Zero64, Zero64, true);
 
   // Generate the class structure
   llvm::Constant *ClassStruct =
@@ -2002,7 +2086,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
                            ClassName.c_str(), 0,
       llvm::ConstantInt::get(LongTy, instanceSize), IvarList,
       MethodList, GenerateProtocolList(Protocols), IvarOffsetArray,
-      Properties);
+      Properties, StrongIvarBitmap, WeakIvarBitmap);
 
   // Resolve the class aliases, if they exist.
   if (ClassPtrAlias) {
@@ -2033,7 +2117,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
   // Add all referenced protocols to a category.
   GenerateProtocolHolderCategory();
 
-  const llvm::StructType *SelStructTy = dyn_cast<llvm::StructType>(
+  llvm::StructType *SelStructTy = dyn_cast<llvm::StructType>(
           SelectorTy->getElementType());
   llvm::Type *SelStructPtrTy = SelectorTy;
   if (SelStructTy == 0) {
@@ -2049,7 +2133,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
         ConstantStrings.size() + 1);
     ConstantStrings.push_back(NULLPtr);
 
-    llvm::StringRef StringClass = CGM.getLangOptions().ObjCConstantStringClass;
+    StringRef StringClass = CGM.getLangOptions().ObjCConstantStringClass;
 
     if (StringClass.empty()) StringClass = "NXConstantString";
 
@@ -2088,8 +2172,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
     std::string SelNameStr = iter->first.getAsString();
     llvm::Constant *SelName = ExportUniqueString(SelNameStr, ".objc_sel_name");
 
-    llvm::SmallVectorImpl<TypedSelector> &Types = iter->second;
-    for (llvm::SmallVectorImpl<TypedSelector>::iterator i = Types.begin(),
+    SmallVectorImpl<TypedSelector> &Types = iter->second;
+    for (SmallVectorImpl<TypedSelector>::iterator i = Types.begin(),
         e = Types.end() ; i!=e ; i++) {
 
       llvm::Constant *SelectorTypeEncoding = NULLPtr;
@@ -2126,10 +2210,10 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
   for (unsigned int i=0 ; i<SelectorCount ; i++) {
 
     llvm::Constant *Idxs[] = {Zeros[0],
-      llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), i), Zeros[0]};
+      llvm::ConstantInt::get(Int32Ty, i), Zeros[0]};
     // FIXME: We're generating redundant loads and stores here!
     llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr(SelectorList,
-        Idxs, 2);
+        makeArrayRef(Idxs, 2));
     // If selectors are defined as an opaque type, cast the pointer to this
     // type.
     SelPtr = llvm::ConstantExpr::getBitCast(SelPtr, SelectorTy);
@@ -2177,7 +2261,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
   Elements.push_back(SymTab);
 
   if (RuntimeVersion >= 10)
-    switch (CGM.getLangOptions().getGCMode()) {
+    switch (CGM.getLangOptions().getGC()) {
       case LangOptions::GCOnly:
         Elements.push_back(llvm::ConstantInt::get(IntTy, 2));
         break;
@@ -2205,9 +2289,9 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
   CGBuilderTy Builder(VMContext);
   Builder.SetInsertPoint(EntryBB);
 
-  llvm::Type *ArgTys[] = { llvm::PointerType::getUnqual(ModuleTy) };
   llvm::FunctionType *FT =
-    llvm::FunctionType::get(Builder.getVoidTy(), ArgTys, true);
+    llvm::FunctionType::get(Builder.getVoidTy(),
+                            llvm::PointerType::getUnqual(ModuleTy), true);
   llvm::Value *Register = CGM.CreateRuntimeFunction(FT, "__objc_exec_class");
   Builder.CreateCall(Register, Module);
   Builder.CreateRetVoid();
@@ -2219,13 +2303,13 @@ llvm::Function *CGObjCGNU::GenerateMethod(const ObjCMethodDecl *OMD,
                                           const ObjCContainerDecl *CD) {
   const ObjCCategoryImplDecl *OCD =
     dyn_cast<ObjCCategoryImplDecl>(OMD->getDeclContext());
-  llvm::StringRef CategoryName = OCD ? OCD->getName() : "";
-  llvm::StringRef ClassName = CD->getName();
+  StringRef CategoryName = OCD ? OCD->getName() : "";
+  StringRef ClassName = CD->getName();
   Selector MethodName = OMD->getSelector();
   bool isClassMethod = !OMD->isInstanceMethod();
 
   CodeGenTypes &Types = CGM.getTypes();
-  const llvm::FunctionType *MethodTy =
+  llvm::FunctionType *MethodTy =
     Types.GetFunctionType(Types.getFunctionInfo(OMD), OMD->isVariadic());
   std::string FunctionName = SymbolNameForMethod(ClassName, CategoryName,
       MethodName, isClassMethod);
@@ -2285,15 +2369,14 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
   llvm::Value *ExceptionAsObject;
 
   if (const Expr *ThrowExpr = S.getThrowExpr()) {
-    llvm::Value *Exception = CGF.EmitScalarExpr(ThrowExpr);
+    llvm::Value *Exception = CGF.EmitObjCThrowOperand(ThrowExpr);
     ExceptionAsObject = Exception;
   } else {
     assert((!CGF.ObjCEHValueStack.empty() && CGF.ObjCEHValueStack.back()) &&
            "Unexpected rethrow outside @catch block.");
     ExceptionAsObject = CGF.ObjCEHValueStack.back();
   }
-  ExceptionAsObject =
-      CGF.Builder.CreateBitCast(ExceptionAsObject, IdTy, "tmp");
+  ExceptionAsObject = CGF.Builder.CreateBitCast(ExceptionAsObject, IdTy);
 
   // Note: This may have to be an invoke, if we want to support constructs like:
   // @try {
@@ -2341,7 +2424,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
     B.CreateCall2(GlobalAssignFn, src, dst);
   else
     // FIXME. Add threadloca assign API
-    assert(false && "EmitObjCGlobalAssign - Threal Local API NYI");
+    llvm_unreachable("EmitObjCGlobalAssign - Threal Local API NYI");
 }
 
 void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
@@ -2396,15 +2479,15 @@ llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable(
               const_cast<ObjCInterfaceDecl *>(ID)))
       Offset = ComputeIvarBaseOffset(CGM, ID, Ivar);
 
-    llvm::ConstantInt *OffsetGuess =
-      llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), Offset, "ivar");
+    llvm::ConstantInt *OffsetGuess = llvm::ConstantInt::get(Int32Ty, Offset,
+                             /*isSigned*/true);
     // Don't emit the guess in non-PIC code because the linker will not be able
     // to replace it with the real version for a library.  In non-PIC code you
     // must compile with the fragile ABI if you want to use ivars from a
     // GCC-compiled class.
     if (CGM.getLangOptions().PICLevel) {
       llvm::GlobalVariable *IvarOffsetGV = new llvm::GlobalVariable(TheModule,
-            llvm::Type::getInt32Ty(VMContext), false,
+            Int32Ty, false,
             llvm::GlobalValue::PrivateLinkage, OffsetGuess, Name+".guess");
       IvarOffsetPointer = new llvm::GlobalVariable(TheModule,
             IvarOffsetGV->getType(), false, llvm::GlobalValue::LinkOnceAnyLinkage,
@@ -2432,10 +2515,9 @@ LValue CGObjCGNU::EmitObjCValueForIvar(CodeGenFunction &CGF,
 static const ObjCInterfaceDecl *FindIvarInterface(ASTContext &Context,
                                                   const ObjCInterfaceDecl *OID,
                                                   const ObjCIvarDecl *OIVD) {
-  llvm::SmallVector<ObjCIvarDecl*, 16> Ivars;
-  Context.ShallowCollectObjCIvars(OID, Ivars);
-  for (unsigned k = 0, e = Ivars.size(); k != e; ++k) {
-    if (OIVD == Ivars[k])
+  for (const ObjCIvarDecl *next = OID->all_declared_ivar_begin(); next;
+       next = next->getNextIvar()) {
+    if (OIVD == next)
       return OID;
   }
 
@@ -2461,12 +2543,12 @@ llvm::Value *CGObjCGNU::EmitIvarOffset(CodeGenFunction &CGF,
     llvm::Value *Offset = TheModule.getGlobalVariable(name);
     if (!Offset)
       Offset = new llvm::GlobalVariable(TheModule, IntTy,
-          false, llvm::GlobalValue::CommonLinkage,
-          0, name);
+          false, llvm::GlobalValue::LinkOnceAnyLinkage,
+          llvm::Constant::getNullValue(IntTy), name);
     return CGF.Builder.CreateLoad(Offset);
   }
   uint64_t Offset = ComputeIvarBaseOffset(CGF.CGM, Interface, Ivar);
-  return llvm::ConstantInt::get(PtrDiffTy, Offset, "ivar");
+  return llvm::ConstantInt::get(PtrDiffTy, Offset, /*isSigned*/true);
 }
 
 CGObjCRuntime *
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp
index 010b9e174e46..308e0c7d3786 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp
@@ -205,14 +205,14 @@ class ObjCCommonTypesHelper {
     CodeGen::CodeGenTypes &Types = CGM.getTypes();
     ASTContext &Ctx = CGM.getContext();
     // id objc_getProperty (id, SEL, ptrdiff_t, bool)
-    llvm::SmallVector<CanQualType,4> Params;
+    SmallVector<CanQualType,4> Params;
     CanQualType IdType = Ctx.getCanonicalParamType(Ctx.getObjCIdType());
     CanQualType SelType = Ctx.getCanonicalParamType(Ctx.getObjCSelType());
     Params.push_back(IdType);
     Params.push_back(SelType);
     Params.push_back(Ctx.getPointerDiffType()->getCanonicalTypeUnqualified());
     Params.push_back(Ctx.BoolTy);
-    const llvm::FunctionType *FTy =
+    llvm::FunctionType *FTy =
       Types.GetFunctionType(Types.getFunctionInfo(IdType, Params,
                                                   FunctionType::ExtInfo()),
                             false);
@@ -223,7 +223,7 @@ class ObjCCommonTypesHelper {
     CodeGen::CodeGenTypes &Types = CGM.getTypes();
     ASTContext &Ctx = CGM.getContext();
     // void objc_setProperty (id, SEL, ptrdiff_t, id, bool, bool)
-    llvm::SmallVector<CanQualType,6> Params;
+    SmallVector<CanQualType,6> Params;
     CanQualType IdType = Ctx.getCanonicalParamType(Ctx.getObjCIdType());
     CanQualType SelType = Ctx.getCanonicalParamType(Ctx.getObjCSelType());
     Params.push_back(IdType);
@@ -232,7 +232,7 @@ class ObjCCommonTypesHelper {
     Params.push_back(IdType);
     Params.push_back(Ctx.BoolTy);
     Params.push_back(Ctx.BoolTy);
-    const llvm::FunctionType *FTy =
+    llvm::FunctionType *FTy =
       Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params,
                                                   FunctionType::ExtInfo()),
                             false);
@@ -244,13 +244,13 @@ class ObjCCommonTypesHelper {
     CodeGen::CodeGenTypes &Types = CGM.getTypes();
     ASTContext &Ctx = CGM.getContext();
     // void objc_copyStruct (void *, const void *, size_t, bool, bool)
-    llvm::SmallVector<CanQualType,5> Params;
+    SmallVector<CanQualType,5> Params;
     Params.push_back(Ctx.VoidPtrTy);
     Params.push_back(Ctx.VoidPtrTy);
     Params.push_back(Ctx.LongTy);
     Params.push_back(Ctx.BoolTy);
     Params.push_back(Ctx.BoolTy);
-    const llvm::FunctionType *FTy =
+    llvm::FunctionType *FTy =
       Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params,
                                                   FunctionType::ExtInfo()),
                             false);
@@ -261,9 +261,9 @@ class ObjCCommonTypesHelper {
     CodeGen::CodeGenTypes &Types = CGM.getTypes();
     ASTContext &Ctx = CGM.getContext();
     // void objc_enumerationMutation (id)
-    llvm::SmallVector<CanQualType,1> Params;
+    SmallVector<CanQualType,1> Params;
     Params.push_back(Ctx.getCanonicalParamType(Ctx.getObjCIdType()));
-    const llvm::FunctionType *FTy =
+    llvm::FunctionType *FTy =
       Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params,
                                                   FunctionType::ExtInfo()),
                             false);
@@ -669,8 +669,8 @@ class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
   unsigned ObjCABI;
 
   // gc ivar layout bitmap calculation helper caches.
-  llvm::SmallVector<GC_IVAR, 16> SkipIvars;
-  llvm::SmallVector<GC_IVAR, 16> IvarsInfo;
+  SmallVector<GC_IVAR, 16> SkipIvars;
+  SmallVector<GC_IVAR, 16> IvarsInfo;
 
   /// LazySymbols - Symbols to generate a lazy reference for. See
   /// DefinedSymbols and FinishModule().
@@ -733,7 +733,7 @@ class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
   /// \param[out] NameOut - The return value.
   void GetNameForMethod(const ObjCMethodDecl *OMD,
                         const ObjCContainerDecl *CD,
-                        llvm::SmallVectorImpl<char> &NameOut);
+                        SmallVectorImpl<char> &NameOut);
 
   /// GetMethodVarName - Return a unique constant for the given
   /// selector's name. The return value has type char *.
@@ -775,7 +775,7 @@ class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
   void BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
                            const llvm::StructLayout *Layout,
                            const RecordDecl *RD,
-                           const llvm::SmallVectorImpl<FieldDecl*> &RecFields,
+                           const SmallVectorImpl<const FieldDecl*> &RecFields,
                            unsigned int BytePos, bool ForStrongLayout,
                            bool &HasUnion);
 
@@ -786,7 +786,7 @@ class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
 
   /// EmitPropertyList - Emit the given property list. The return
   /// value has type PropertyListPtrTy.
-  llvm::Constant *EmitPropertyList(llvm::Twine Name,
+  llvm::Constant *EmitPropertyList(Twine Name,
                                    const Decl *Container,
                                    const ObjCContainerDecl *OCD,
                                    const ObjCCommonTypesHelper &ObjCTypes);
@@ -817,7 +817,7 @@ class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
   /// \param Align - The alignment for the variable, or 0.
   /// \param AddToUsed - Whether the variable should be added to
   /// "llvm.used".
-  llvm::GlobalVariable *CreateMetadataVar(llvm::Twine Name,
+  llvm::GlobalVariable *CreateMetadataVar(Twine Name,
                                           llvm::Constant *Init,
                                           const char *Section,
                                           unsigned Align,
@@ -923,7 +923,7 @@ class CGObjCMac : public CGObjCCommonMac {
 
   /// EmitMethodList - Emit the method list for the given
   /// implementation. The return value has type MethodListPtrTy.
-  llvm::Constant *EmitMethodList(llvm::Twine Name,
+  llvm::Constant *EmitMethodList(Twine Name,
                                  const char *Section,
                                  const ConstantVector &Methods);
 
@@ -938,7 +938,7 @@ class CGObjCMac : public CGObjCCommonMac {
   ///  - begin, end: The method list to output.
   ///
   /// The return value has type MethodDescriptionListPtrTy.
-  llvm::Constant *EmitMethodDescList(llvm::Twine Name,
+  llvm::Constant *EmitMethodDescList(Twine Name,
                                      const char *Section,
                                      const ConstantVector &Methods);
 
@@ -964,7 +964,7 @@ class CGObjCMac : public CGObjCCommonMac {
 
   /// EmitProtocolList - Generate the list of referenced
   /// protocols. The return value has type ProtocolListPtrTy.
-  llvm::Constant *EmitProtocolList(llvm::Twine Name,
+  llvm::Constant *EmitProtocolList(Twine Name,
                                    ObjCProtocolDecl::protocol_iterator begin,
                                    ObjCProtocolDecl::protocol_iterator end);
 
@@ -1060,8 +1060,7 @@ class CGObjCMac : public CGObjCCommonMac {
   /// GetClassGlobal - Return the global variable for the Objective-C
   /// class of the given name.
   virtual llvm::GlobalVariable *GetClassGlobal(const std::string &Name) {
-    assert(false && "CGObjCMac::GetClassGlobal");
-    return 0;
+    llvm_unreachable("CGObjCMac::GetClassGlobal");
   }
 };
 
@@ -1117,7 +1116,7 @@ class CGObjCNonFragileABIMac : public CGObjCCommonMac {
 
   /// EmitMethodList - Emit the method list for the given
   /// implementation. The return value has type MethodListnfABITy.
-  llvm::Constant *EmitMethodList(llvm::Twine Name,
+  llvm::Constant *EmitMethodList(Twine Name,
                                  const char *Section,
                                  const ConstantVector &Methods);
   /// EmitIvarList - Emit the ivar list for the given
@@ -1144,7 +1143,7 @@ class CGObjCNonFragileABIMac : public CGObjCCommonMac {
 
   /// EmitProtocolList - Generate the list of referenced
   /// protocols. The return value has type ProtocolListPtrTy.
-  llvm::Constant *EmitProtocolList(llvm::Twine Name,
+  llvm::Constant *EmitProtocolList(Twine Name,
                                    ObjCProtocolDecl::protocol_iterator begin,
                                    ObjCProtocolDecl::protocol_iterator end);
 
@@ -1375,7 +1374,7 @@ static llvm::Constant *getConstantGEP(llvm::LLVMContext &VMContext,
     llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx0),
     llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx1)
   };
-  return llvm::ConstantExpr::getGetElementPtr(C, Idxs, 2);
+  return llvm::ConstantExpr::getGetElementPtr(C, Idxs);
 }
 
 /// hasObjCExceptionAttribute - Return true if this class or any super
@@ -1418,12 +1417,12 @@ llvm::Constant *CGObjCMac::GetEHType(QualType T) {
   if (T->isObjCIdType() ||
       T->isObjCQualifiedIdType()) {
     return CGM.GetAddrOfRTTIDescriptor(
-              CGM.getContext().ObjCIdRedefinitionType, /*ForEH=*/true);
+              CGM.getContext().getObjCIdRedefinitionType(), /*ForEH=*/true);
   }
   if (T->isObjCClassType() ||
       T->isObjCQualifiedClassType()) {
     return CGM.GetAddrOfRTTIDescriptor(
-             CGM.getContext().ObjCClassRedefinitionType, /*ForEH=*/true);
+             CGM.getContext().getObjCClassRedefinitionType(), /*ForEH=*/true);
   }
   if (T->isObjCObjectPointerType())
     return CGM.GetAddrOfRTTIDescriptor(T,  /*ForEH=*/true);
@@ -1510,7 +1509,7 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
   }
   // FIXME: We shouldn't need to do this cast, rectify the ASTContext and
   // ObjCTypes types.
-  const llvm::Type *ClassTy =
+  llvm::Type *ClassTy =
     CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType());
   Target = CGF.Builder.CreateBitCast(Target, ClassTy);
   CGF.Builder.CreateStore(Target,
@@ -1549,7 +1548,7 @@ CGObjCCommonMac::EmitMessageSend(CodeGen::CodeGenFunction &CGF,
                                  const ObjCCommonTypesHelper &ObjCTypes) {
   CallArgList ActualArgs;
   if (!IsSuper)
-    Arg0 = CGF.Builder.CreateBitCast(Arg0, ObjCTypes.ObjectPtrTy, "tmp");
+    Arg0 = CGF.Builder.CreateBitCast(Arg0, ObjCTypes.ObjectPtrTy);
   ActualArgs.add(RValue::get(Arg0), Arg0Ty);
   ActualArgs.add(RValue::get(Sel), CGF.getContext().getObjCSelType());
   ActualArgs.addFrom(CallArgs);
@@ -1557,7 +1556,7 @@ CGObjCCommonMac::EmitMessageSend(CodeGen::CodeGenFunction &CGF,
   CodeGenTypes &Types = CGM.getTypes();
   const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs,
                                                        FunctionType::ExtInfo());
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     Types.GetFunctionType(FnInfo, Method ? Method->isVariadic() : false);
 
   if (Method)
@@ -1605,15 +1604,15 @@ llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
   llvm::Constant *nullPtr = 
     llvm::Constant::getNullValue(llvm::Type::getInt8PtrTy(VMContext));
 
-  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC &&
+  if (CGM.getLangOptions().getGC() == LangOptions::NonGC &&
       !CGM.getLangOptions().ObjCAutoRefCount)
     return nullPtr;
 
   bool hasUnion = false;
   SkipIvars.clear();
   IvarsInfo.clear();
-  unsigned WordSizeInBits = CGM.getContext().Target.getPointerWidth(0);
-  unsigned ByteSizeInBits = CGM.getContext().Target.getCharWidth();
+  unsigned WordSizeInBits = CGM.getContext().getTargetInfo().getPointerWidth(0);
+  unsigned ByteSizeInBits = CGM.getContext().getTargetInfo().getCharWidth();
   
   // __isa is the first field in block descriptor and must assume by runtime's
   // convention that it is GC'able.
@@ -1878,7 +1877,7 @@ CGObjCMac::EmitProtocolExtension(const ObjCProtocolDecl *PD,
   };
 */
 llvm::Constant *
-CGObjCMac::EmitProtocolList(llvm::Twine Name,
+CGObjCMac::EmitProtocolList(Twine Name,
                             ObjCProtocolDecl::protocol_iterator begin,
                             ObjCProtocolDecl::protocol_iterator end) {
   std::vector<llvm::Constant*> ProtocolRefs;
@@ -1942,7 +1941,7 @@ void CGObjCCommonMac::PushProtocolProperties(llvm::SmallPtrSet<const IdentifierI
   struct _objc_property[prop_count];
   };
 */
-llvm::Constant *CGObjCCommonMac::EmitPropertyList(llvm::Twine Name,
+llvm::Constant *CGObjCCommonMac::EmitPropertyList(Twine Name,
                                        const Decl *Container,
                                        const ObjCContainerDecl *OCD,
                                        const ObjCCommonTypesHelper &ObjCTypes) {
@@ -2014,7 +2013,7 @@ CGObjCMac::GetMethodDescriptionConstant(const ObjCMethodDecl *MD) {
                                    Desc);
 }
 
-llvm::Constant *CGObjCMac::EmitMethodDescList(llvm::Twine Name,
+llvm::Constant *CGObjCMac::EmitMethodDescList(Twine Name,
                                               const char *Section,
                                               const ConstantVector &Methods) {
   // Return null for empty list.
@@ -2407,10 +2406,9 @@ llvm::Constant *CGObjCMac::EmitIvarList(const ObjCImplementationDecl *ID,
   if (ForClass)
     return llvm::Constant::getNullValue(ObjCTypes.IvarListPtrTy);
 
-  ObjCInterfaceDecl *OID =
-    const_cast<ObjCInterfaceDecl*>(ID->getClassInterface());
+  const ObjCInterfaceDecl *OID = ID->getClassInterface();
 
-  for (ObjCIvarDecl *IVD = OID->all_declared_ivar_begin(); 
+  for (const ObjCIvarDecl *IVD = OID->all_declared_ivar_begin(); 
        IVD; IVD = IVD->getNextIvar()) {
     // Ignore unnamed bit-fields.
     if (!IVD->getDeclName())
@@ -2476,7 +2474,7 @@ llvm::Constant *CGObjCMac::GetMethodConstant(const ObjCMethodDecl *MD) {
   return llvm::ConstantStruct::get(ObjCTypes.MethodTy, Method);
 }
 
-llvm::Constant *CGObjCMac::EmitMethodList(llvm::Twine Name,
+llvm::Constant *CGObjCMac::EmitMethodList(Twine Name,
                                           const char *Section,
                                           const ConstantVector &Methods) {
   // Return null for empty list.
@@ -2501,7 +2499,7 @@ llvm::Function *CGObjCCommonMac::GenerateMethod(const ObjCMethodDecl *OMD,
   GetNameForMethod(OMD, CD, Name);
 
   CodeGenTypes &Types = CGM.getTypes();
-  const llvm::FunctionType *MethodTy =
+  llvm::FunctionType *MethodTy =
     Types.GetFunctionType(Types.getFunctionInfo(OMD), OMD->isVariadic());
   llvm::Function *Method =
     llvm::Function::Create(MethodTy,
@@ -2514,12 +2512,12 @@ llvm::Function *CGObjCCommonMac::GenerateMethod(const ObjCMethodDecl *OMD,
 }
 
 llvm::GlobalVariable *
-CGObjCCommonMac::CreateMetadataVar(llvm::Twine Name,
+CGObjCCommonMac::CreateMetadataVar(Twine Name,
                                    llvm::Constant *Init,
                                    const char *Section,
                                    unsigned Align,
                                    bool AddToUsed) {
-  const llvm::Type *Ty = Init->getType();
+  llvm::Type *Ty = Init->getType();
   llvm::GlobalVariable *GV =
     new llvm::GlobalVariable(CGM.getModule(), Ty, false,
                              llvm::GlobalValue::InternalLinkage, Init, Name);
@@ -2627,7 +2625,7 @@ namespace {
 
   class FragileHazards {
     CodeGenFunction &CGF;
-    llvm::SmallVector<llvm::Value*, 20> Locals;
+    SmallVector<llvm::Value*, 20> Locals;
     llvm::DenseSet<llvm::BasicBlock*> BlocksBeforeTry;
 
     llvm::InlineAsm *ReadHazard;
@@ -2754,7 +2752,6 @@ void FragileHazards::collectLocals() {
   llvm::DenseSet<llvm::Value*> AllocasToIgnore;
   addIfPresent(AllocasToIgnore, CGF.ReturnValue);
   addIfPresent(AllocasToIgnore, CGF.NormalCleanupDest);
-  addIfPresent(AllocasToIgnore, CGF.EHCleanupDest);
 
   // Collect all the allocas currently in the function.  This is
   // probably way too aggressive.
@@ -2766,7 +2763,7 @@ void FragileHazards::collectLocals() {
 }
 
 llvm::FunctionType *FragileHazards::GetAsmFnType() {
-  llvm::SmallVector<llvm::Type *, 16> tys(Locals.size());
+  SmallVector<llvm::Type *, 16> tys(Locals.size());
   for (unsigned i = 0, e = Locals.size(); i != e; ++i)
     tys[i] = Locals[i]->getType();
   return llvm::FunctionType::get(CGF.VoidTy, tys, false);
@@ -2958,7 +2955,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
   llvm::Value *GEPIndexes[] = { Zero, Zero, Zero };
   llvm::Value *SetJmpBuffer =
-    CGF.Builder.CreateGEP(ExceptionData, GEPIndexes, GEPIndexes+3, "setjmp_buffer");
+    CGF.Builder.CreateGEP(ExceptionData, GEPIndexes, "setjmp_buffer");
   llvm::CallInst *SetJmpResult =
     CGF.Builder.CreateCall(ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result");
   SetJmpResult->setDoesNotThrow();
@@ -3119,8 +3116,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // Initialize the catch variable.
       llvm::Value *Tmp =
         CGF.Builder.CreateBitCast(Caught,
-                                  CGF.ConvertType(CatchParam->getType()),
-                                  "tmp");
+                                  CGF.ConvertType(CatchParam->getType()));
       CGF.Builder.CreateStore(Tmp, CGF.GetAddrOfLocalVar(CatchParam));
 
       CGF.EmitStmt(CatchStmt->getCatchBody());
@@ -3208,9 +3204,9 @@ void CGObjCMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
   llvm::Value *ExceptionAsObject;
 
   if (const Expr *ThrowExpr = S.getThrowExpr()) {
-    llvm::Value *Exception = CGF.EmitScalarExpr(ThrowExpr);
+    llvm::Value *Exception = CGF.EmitObjCThrowOperand(ThrowExpr);
     ExceptionAsObject =
-      CGF.Builder.CreateBitCast(Exception, ObjCTypes.ObjectPtrTy, "tmp");
+      CGF.Builder.CreateBitCast(Exception, ObjCTypes.ObjectPtrTy);
   } else {
     assert((!CGF.ObjCEHValueStack.empty() && CGF.ObjCEHValueStack.back()) &&
            "Unexpected rethrow outside @catch block.");
@@ -3230,7 +3226,7 @@ void CGObjCMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
 ///
 llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
                                           llvm::Value *AddrWeakObj) {
-  const llvm::Type* DestTy =
+  llvm::Type* DestTy =
     cast<llvm::PointerType>(AddrWeakObj->getType())->getElementType();
   AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj,
                                           ObjCTypes.PtrObjectPtrTy);
@@ -3245,7 +3241,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
 ///
 void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                    llvm::Value *src, llvm::Value *dst) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -3266,7 +3262,7 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
 void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
                                      llvm::Value *src, llvm::Value *dst,
                                      bool threadlocal) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -3292,7 +3288,7 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                    llvm::Value *src, llvm::Value *dst,
                                    llvm::Value *ivarOffset) {
   assert(ivarOffset && "EmitObjCIvarAssign - ivarOffset is NULL");
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -3312,7 +3308,7 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
 ///
 void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
                                          llvm::Value *src, llvm::Value *dst) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -3386,15 +3382,15 @@ void CGObjCCommonMac::EmitImageInfo() {
   unsigned flags = 0;
 
   // FIXME: Fix and continue?
-  if (CGM.getLangOptions().getGCMode() != LangOptions::NonGC)
+  if (CGM.getLangOptions().getGC() != LangOptions::NonGC)
     flags |= eImageInfo_GarbageCollected;
-  if (CGM.getLangOptions().getGCMode() == LangOptions::GCOnly)
+  if (CGM.getLangOptions().getGC() == LangOptions::GCOnly)
     flags |= eImageInfo_GCOnly;
 
   // We never allow @synthesize of a superclass property.
   flags |= eImageInfo_CorrectedSynthesize;
 
-  const llvm::Type *Int32Ty = llvm::Type::getInt32Ty(VMContext);
+  llvm::Type *Int32Ty = llvm::Type::getInt32Ty(VMContext);
   
   // Emitted as int[2];
   llvm::Constant *values[2] = {
@@ -3498,7 +3494,7 @@ llvm::Value *CGObjCMac::EmitClassRefFromId(CGBuilderTy &Builder,
                       4, true);
   }
   
-  return Builder.CreateLoad(Entry, "tmp");
+  return Builder.CreateLoad(Entry);
 }
 
 llvm::Value *CGObjCMac::EmitClassRef(CGBuilderTy &Builder,
@@ -3527,7 +3523,7 @@ llvm::Value *CGObjCMac::EmitSelector(CGBuilderTy &Builder, Selector Sel,
 
   if (lvalue)
     return Entry;
-  return Builder.CreateLoad(Entry, "tmp");
+  return Builder.CreateLoad(Entry);
 }
 
 llvm::Constant *CGObjCCommonMac::GetClassName(IdentifierInfo *Ident) {
@@ -3551,13 +3547,6 @@ llvm::Function *CGObjCCommonMac::GetMethodDefinition(const ObjCMethodDecl *MD) {
   if (I != MethodDefinitions.end())
     return I->second;
 
-  if (MD->hasBody() && MD->getPCHLevel() > 0) {
-    // MD isn't emitted yet because it comes from PCH.
-    CGM.EmitTopLevelDecl(const_cast<ObjCMethodDecl*>(MD));
-    assert(MethodDefinitions[MD] && "EmitTopLevelDecl didn't emit the method!");
-    return MethodDefinitions[MD];
-  }
-
   return NULL;
 }
 
@@ -3574,8 +3563,8 @@ void CGObjCCommonMac::BuildAggrIvarRecordLayout(const RecordType *RT,
                                                 bool &HasUnion) {
   const RecordDecl *RD = RT->getDecl();
   // FIXME - Use iterator.
-  llvm::SmallVector<FieldDecl*, 16> Fields(RD->field_begin(), RD->field_end());
-  const llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0));
+  SmallVector<const FieldDecl*, 16> Fields(RD->field_begin(), RD->field_end());
+  llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0));
   const llvm::StructLayout *RecLayout =
     CGM.getTargetData().getStructLayout(cast<llvm::StructType>(Ty));
 
@@ -3586,15 +3575,15 @@ void CGObjCCommonMac::BuildAggrIvarRecordLayout(const RecordType *RT,
 void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
                              const llvm::StructLayout *Layout,
                              const RecordDecl *RD,
-                             const llvm::SmallVectorImpl<FieldDecl*> &RecFields,
+                             const SmallVectorImpl<const FieldDecl*> &RecFields,
                              unsigned int BytePos, bool ForStrongLayout,
                              bool &HasUnion) {
   bool IsUnion = (RD && RD->isUnion());
   uint64_t MaxUnionIvarSize = 0;
   uint64_t MaxSkippedUnionIvarSize = 0;
-  FieldDecl *MaxField = 0;
-  FieldDecl *MaxSkippedField = 0;
-  FieldDecl *LastFieldBitfieldOrUnnamed = 0;
+  const FieldDecl *MaxField = 0;
+  const FieldDecl *MaxSkippedField = 0;
+  const FieldDecl *LastFieldBitfieldOrUnnamed = 0;
   uint64_t MaxFieldOffset = 0;
   uint64_t MaxSkippedFieldOffset = 0;
   uint64_t LastBitfieldOrUnnamedOffset = 0;
@@ -3602,16 +3591,16 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
 
   if (RecFields.empty())
     return;
-  unsigned WordSizeInBits = CGM.getContext().Target.getPointerWidth(0);
-  unsigned ByteSizeInBits = CGM.getContext().Target.getCharWidth();
+  unsigned WordSizeInBits = CGM.getContext().getTargetInfo().getPointerWidth(0);
+  unsigned ByteSizeInBits = CGM.getContext().getTargetInfo().getCharWidth();
   if (!RD && CGM.getLangOptions().ObjCAutoRefCount) {
-    FieldDecl *FirstField = RecFields[0];
+    const FieldDecl *FirstField = RecFields[0];
     FirstFieldDelta = 
       ComputeIvarBaseOffset(CGM, OI, cast<ObjCIvarDecl>(FirstField));
   }
   
   for (unsigned i = 0, e = RecFields.size(); i != e; ++i) {
-    FieldDecl *Field = RecFields[i];
+    const FieldDecl *Field = RecFields[i];
     uint64_t FieldOffset;
     if (RD) {
       // Note that 'i' here is actually the field index inside RD of Field,
@@ -3721,9 +3710,8 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
   if (LastFieldBitfieldOrUnnamed) {
     if (LastFieldBitfieldOrUnnamed->isBitField()) {
       // Last field was a bitfield. Must update skip info.
-      Expr *BitWidth = LastFieldBitfieldOrUnnamed->getBitWidth();
-      uint64_t BitFieldSize =
-        BitWidth->EvaluateAsInt(CGM.getContext()).getZExtValue();
+      uint64_t BitFieldSize
+          = LastFieldBitfieldOrUnnamed->getBitWidthValue(CGM.getContext());
       GC_IVAR skivar;
       skivar.ivar_bytepos = BytePos + LastBitfieldOrUnnamedOffset;
       skivar.ivar_size = (BitFieldSize / ByteSizeInBits)
@@ -3754,10 +3742,10 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
 /// filled already by the caller.
 llvm::Constant *CGObjCCommonMac::BuildIvarLayoutBitmap(std::string& BitMap) {
   unsigned int WordsToScan, WordsToSkip;
-  const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+  llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
   
   // Build the string of skip/scan nibbles
-  llvm::SmallVector<SKIP_SCAN, 32> SkipScanIvars;
+  SmallVector<SKIP_SCAN, 32> SkipScanIvars;
   unsigned int WordSize =
   CGM.getTypes().getTargetData().getTypeAllocSize(PtrTy);
   if (IvarsInfo[0].ivar_bytepos == 0) {
@@ -3898,25 +3886,24 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
   bool ForStrongLayout) {
   bool hasUnion = false;
 
-  const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC &&
+  llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+  if (CGM.getLangOptions().getGC() == LangOptions::NonGC &&
       !CGM.getLangOptions().ObjCAutoRefCount)
     return llvm::Constant::getNullValue(PtrTy);
 
-  ObjCInterfaceDecl *OI = 
-    const_cast<ObjCInterfaceDecl*>(OMD->getClassInterface());
-  llvm::SmallVector<FieldDecl*, 32> RecFields;
+  const ObjCInterfaceDecl *OI = OMD->getClassInterface();
+  SmallVector<const FieldDecl*, 32> RecFields;
   if (CGM.getLangOptions().ObjCAutoRefCount) {
-    for (ObjCIvarDecl *IVD = OI->all_declared_ivar_begin(); 
+    for (const ObjCIvarDecl *IVD = OI->all_declared_ivar_begin(); 
          IVD; IVD = IVD->getNextIvar())
       RecFields.push_back(cast<FieldDecl>(IVD));
   }
   else {
-    llvm::SmallVector<ObjCIvarDecl*, 32> Ivars;
+    SmallVector<const ObjCIvarDecl*, 32> Ivars;
     CGM.getContext().DeepCollectObjCIvars(OI, true, Ivars);
 
-    for (unsigned k = 0, e = Ivars.size(); k != e; ++k)
-      RecFields.push_back(cast<FieldDecl>(Ivars[k]));
+    // FIXME: This is not ideal; we shouldn't have to do this copy.
+    RecFields.append(Ivars.begin(), Ivars.end());
   }
 
   if (RecFields.empty())
@@ -4036,7 +4023,7 @@ CGObjCCommonMac::GetPropertyTypeString(const ObjCPropertyDecl *PD,
 
 void CGObjCCommonMac::GetNameForMethod(const ObjCMethodDecl *D,
                                        const ObjCContainerDecl *CD,
-                                       llvm::SmallVectorImpl<char> &Name) {
+                                       SmallVectorImpl<char> &Name) {
   llvm::raw_svector_ostream OS(Name);
   assert (CD && "Missing container decl in GetNameForMethod");
   OS << '\01' << (D->isInstanceMethod() ? '-' : '+')
@@ -4125,7 +4112,7 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
 
   // FIXME: It would be nice to unify this with the opaque type, so that the IR
   // comes out a bit cleaner.
-  const llvm::Type *T = Types.ConvertType(Ctx.getObjCProtoType());
+  llvm::Type *T = Types.ConvertType(Ctx.getObjCProtoType());
   ExternalProtocolPtrTy = llvm::PointerType::getUnqual(T);
 
   // I'm not sure I like this. The implicit coordination is a bit
@@ -4160,8 +4147,8 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
   //   char *name;
   //   char *attributes;
   // }
-  PropertyTy = llvm::StructType::createNamed("struct._prop_t",
-                                             Int8PtrTy, Int8PtrTy, NULL);
+  PropertyTy = llvm::StructType::create("struct._prop_t",
+                                        Int8PtrTy, Int8PtrTy, NULL);
 
   // struct _prop_list_t {
   //   uint32_t entsize;      // sizeof(struct _prop_t)
@@ -4169,10 +4156,8 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
   //   struct _prop_t prop_list[count_of_properties];
   // }
   PropertyListTy =
-    llvm::StructType::createNamed("struct._prop_list_t",
-                                  IntTy, IntTy,
-                                  llvm::ArrayType::get(PropertyTy, 0),
-                                  NULL);
+    llvm::StructType::create("struct._prop_list_t", IntTy, IntTy,
+                             llvm::ArrayType::get(PropertyTy, 0), NULL);
   // struct _prop_list_t *
   PropertyListPtrTy = llvm::PointerType::getUnqual(PropertyListTy);
 
@@ -4181,12 +4166,12 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
   //   char *method_type;
   //   char *_imp;
   // }
-  MethodTy = llvm::StructType::createNamed("struct._objc_method",
-                                           SelectorPtrTy, Int8PtrTy, Int8PtrTy,
-                                           NULL);
+  MethodTy = llvm::StructType::create("struct._objc_method",
+                                      SelectorPtrTy, Int8PtrTy, Int8PtrTy,
+                                      NULL);
 
   // struct _objc_cache *
-  CacheTy = llvm::StructType::createNamed(VMContext, "struct._objc_cache");
+  CacheTy = llvm::StructType::create(VMContext, "struct._objc_cache");
   CachePtrTy = llvm::PointerType::getUnqual(CacheTy);
     
 }
@@ -4198,18 +4183,17 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   char *types;
   // }
   MethodDescriptionTy =
-    llvm::StructType::createNamed("struct._objc_method_description",
-                                  SelectorPtrTy, Int8PtrTy, NULL);
+    llvm::StructType::create("struct._objc_method_description",
+                             SelectorPtrTy, Int8PtrTy, NULL);
 
   // struct _objc_method_description_list {
   //   int count;
   //   struct _objc_method_description[1];
   // }
   MethodDescriptionListTy =
-    llvm::StructType::createNamed("struct._objc_method_description_list",
-                                  IntTy,
-                                  llvm::ArrayType::get(MethodDescriptionTy, 0),
-                                  NULL);
+    llvm::StructType::create("struct._objc_method_description_list",
+                             IntTy,
+                             llvm::ArrayType::get(MethodDescriptionTy, 0),NULL);
 
   // struct _objc_method_description_list *
   MethodDescriptionListPtrTy =
@@ -4224,12 +4208,10 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   struct _objc_property_list *instance_properties;
   // }
   ProtocolExtensionTy =
-    llvm::StructType::createNamed("struct._objc_protocol_extension",
-                                  IntTy,
-                                  MethodDescriptionListPtrTy,
-                                  MethodDescriptionListPtrTy,
-                                  PropertyListPtrTy,
-                                  NULL);
+    llvm::StructType::create("struct._objc_protocol_extension",
+                             IntTy, MethodDescriptionListPtrTy,
+                             MethodDescriptionListPtrTy, PropertyListPtrTy,
+                             NULL);
 
   // struct _objc_protocol_extension *
   ProtocolExtensionPtrTy = llvm::PointerType::getUnqual(ProtocolExtensionTy);
@@ -4237,10 +4219,10 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   // Handle recursive construction of Protocol and ProtocolList types
 
   ProtocolTy =
-    llvm::StructType::createNamed(VMContext, "struct._objc_protocol");
+    llvm::StructType::create(VMContext, "struct._objc_protocol");
 
   ProtocolListTy =
-    llvm::StructType::createNamed(VMContext, "struct._objc_protocol_list");
+    llvm::StructType::create(VMContext, "struct._objc_protocol_list");
   ProtocolListTy->setBody(llvm::PointerType::getUnqual(ProtocolListTy),
                           LongTy,
                           llvm::ArrayType::get(ProtocolTy, 0),
@@ -4271,26 +4253,26 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   char *ivar_type;
   //   int  ivar_offset;
   // }
-  IvarTy = llvm::StructType::createNamed("struct._objc_ivar",
-                                         Int8PtrTy, Int8PtrTy, IntTy, NULL);
+  IvarTy = llvm::StructType::create("struct._objc_ivar",
+                                    Int8PtrTy, Int8PtrTy, IntTy, NULL);
 
   // struct _objc_ivar_list *
   IvarListTy =
-    llvm::StructType::createNamed(VMContext, "struct._objc_ivar_list");
+    llvm::StructType::create(VMContext, "struct._objc_ivar_list");
   IvarListPtrTy = llvm::PointerType::getUnqual(IvarListTy);
 
   // struct _objc_method_list *
   MethodListTy =
-    llvm::StructType::createNamed(VMContext, "struct._objc_method_list");
+    llvm::StructType::create(VMContext, "struct._objc_method_list");
   MethodListPtrTy = llvm::PointerType::getUnqual(MethodListTy);
 
   // struct _objc_class_extension *
   ClassExtensionTy =
-    llvm::StructType::createNamed("struct._objc_class_extension",
-                                  IntTy, Int8PtrTy, PropertyListPtrTy, NULL);
+    llvm::StructType::create("struct._objc_class_extension",
+                             IntTy, Int8PtrTy, PropertyListPtrTy, NULL);
   ClassExtensionPtrTy = llvm::PointerType::getUnqual(ClassExtensionTy);
 
-  ClassTy = llvm::StructType::createNamed(VMContext, "struct._objc_class");
+  ClassTy = llvm::StructType::create(VMContext, "struct._objc_class");
 
   // struct _objc_class {
   //   Class isa;
@@ -4331,10 +4313,10 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   struct _objc_property_list *instance_properties;// category's @property
   // }
   CategoryTy =
-    llvm::StructType::createNamed("struct._objc_category",
-                                  Int8PtrTy, Int8PtrTy, MethodListPtrTy,
-                                  MethodListPtrTy, ProtocolListPtrTy,
-                                  IntTy, PropertyListPtrTy, NULL);
+    llvm::StructType::create("struct._objc_category",
+                             Int8PtrTy, Int8PtrTy, MethodListPtrTy,
+                             MethodListPtrTy, ProtocolListPtrTy,
+                             IntTy, PropertyListPtrTy, NULL);
 
   // Global metadata structures
 
@@ -4346,9 +4328,9 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   char *defs[cls_def_cnt + cat_def_cnt];
   // }
   SymtabTy =
-    llvm::StructType::createNamed("struct._objc_symtab",
-                                  LongTy, SelectorPtrTy, ShortTy, ShortTy,
-                                  llvm::ArrayType::get(Int8PtrTy, 0), NULL);
+    llvm::StructType::create("struct._objc_symtab",
+                             LongTy, SelectorPtrTy, ShortTy, ShortTy,
+                             llvm::ArrayType::get(Int8PtrTy, 0), NULL);
   SymtabPtrTy = llvm::PointerType::getUnqual(SymtabTy);
 
   // struct _objc_module {
@@ -4358,8 +4340,8 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
   //   struct _objc_symtab* symtab;
   //  }
   ModuleTy =
-    llvm::StructType::createNamed("struct._objc_module",
-                                  LongTy, LongTy, Int8PtrTy, SymtabPtrTy, NULL);
+    llvm::StructType::create("struct._objc_module",
+                             LongTy, LongTy, Int8PtrTy, SymtabPtrTy, NULL);
 
 
   // FIXME: This is the size of the setjmp buffer and should be target
@@ -4371,7 +4353,7 @@ ObjCTypesHelper::ObjCTypesHelper(CodeGen::CodeGenModule &cgm)
     llvm::Type::getInt8PtrTy(VMContext), 4);
 
   ExceptionDataTy =
-    llvm::StructType::createNamed("struct._objc_exception_data",
+    llvm::StructType::create("struct._objc_exception_data",
                          llvm::ArrayType::get(llvm::Type::getInt32Ty(VMContext),
                                               SetJmpBufferSize),
                          StackPtrTy, NULL);
@@ -4386,10 +4368,8 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   struct _objc_method method_list[method_count];
   // }
   MethodListnfABITy =
-    llvm::StructType::createNamed("struct.__method_list_t",
-                                  IntTy, IntTy,
-                                  llvm::ArrayType::get(MethodTy, 0),
-                                  NULL);
+    llvm::StructType::create("struct.__method_list_t", IntTy, IntTy,
+                             llvm::ArrayType::get(MethodTy, 0), NULL);
   // struct method_list_t *
   MethodListnfABIPtrTy = llvm::PointerType::getUnqual(MethodListnfABITy);
 
@@ -4408,20 +4388,14 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
 
   // Holder for struct _protocol_list_t *
   ProtocolListnfABITy =
-    llvm::StructType::createNamed(VMContext, "struct._objc_protocol_list");
+    llvm::StructType::create(VMContext, "struct._objc_protocol_list");
 
   ProtocolnfABITy =
-    llvm::StructType::createNamed("struct._protocol_t",
-                                  ObjectPtrTy, Int8PtrTy,
-                              llvm::PointerType::getUnqual(ProtocolListnfABITy),
-                                  MethodListnfABIPtrTy,
-                                  MethodListnfABIPtrTy,
-                                  MethodListnfABIPtrTy,
-                                  MethodListnfABIPtrTy,
-                                  PropertyListPtrTy,
-                                  IntTy,
-                                  IntTy,
-                                  NULL);
+    llvm::StructType::create("struct._protocol_t", ObjectPtrTy, Int8PtrTy,
+                             llvm::PointerType::getUnqual(ProtocolListnfABITy),
+                             MethodListnfABIPtrTy, MethodListnfABIPtrTy,
+                             MethodListnfABIPtrTy, MethodListnfABIPtrTy,
+                             PropertyListPtrTy, IntTy, IntTy, NULL);
 
   // struct _protocol_t*
   ProtocolnfABIPtrTy = llvm::PointerType::getUnqual(ProtocolnfABITy);
@@ -4445,13 +4419,9 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   uint32_t size;
   // }
   IvarnfABITy =
-    llvm::StructType::createNamed("struct._ivar_t",
-                                  llvm::PointerType::getUnqual(LongTy),
-                                  Int8PtrTy,
-                                  Int8PtrTy,
-                                  IntTy,
-                                  IntTy,
-                                  NULL);
+    llvm::StructType::create("struct._ivar_t",
+                             llvm::PointerType::getUnqual(LongTy),
+                             Int8PtrTy, Int8PtrTy, IntTy, IntTy, NULL);
 
   // struct _ivar_list_t {
   //   uint32 entsize;  // sizeof(struct _ivar_t)
@@ -4459,10 +4429,8 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   struct _iver_t list[count];
   // }
   IvarListnfABITy =
-    llvm::StructType::createNamed("struct._ivar_list_t",
-                                  IntTy, IntTy,
-                                  llvm::ArrayType::get(IvarnfABITy, 0),
-                                  NULL);
+    llvm::StructType::create("struct._ivar_list_t", IntTy, IntTy,
+                             llvm::ArrayType::get(IvarnfABITy, 0), NULL);
 
   IvarListnfABIPtrTy = llvm::PointerType::getUnqual(IvarListnfABITy);
 
@@ -4481,18 +4449,12 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   // }
 
   // FIXME. Add 'reserved' field in 64bit abi mode!
-  ClassRonfABITy = llvm::StructType::createNamed("struct._class_ro_t",
-                                                 IntTy,
-                                                 IntTy,
-                                                 IntTy,
-                                                 Int8PtrTy,
-                                                 Int8PtrTy,
-                                                 MethodListnfABIPtrTy,
-                                                 ProtocolListnfABIPtrTy,
-                                                 IvarListnfABIPtrTy,
-                                                 Int8PtrTy,
-                                                 PropertyListPtrTy,
-                                                 NULL);
+  ClassRonfABITy = llvm::StructType::create("struct._class_ro_t",
+                                            IntTy, IntTy, IntTy, Int8PtrTy,
+                                            Int8PtrTy, MethodListnfABIPtrTy,
+                                            ProtocolListnfABIPtrTy,
+                                            IvarListnfABIPtrTy,
+                                            Int8PtrTy, PropertyListPtrTy, NULL);
 
   // ImpnfABITy - LLVM for id (*)(id, SEL, ...)
   llvm::Type *params[] = { ObjectPtrTy, SelectorPtrTy };
@@ -4507,7 +4469,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   struct class_ro_t *ro;
   // }
 
-  ClassnfABITy = llvm::StructType::createNamed(VMContext, "struct._class_t");
+  ClassnfABITy = llvm::StructType::create(VMContext, "struct._class_t");
   ClassnfABITy->setBody(llvm::PointerType::getUnqual(ClassnfABITy),
                         llvm::PointerType::getUnqual(ClassnfABITy),
                         CachePtrTy,
@@ -4526,14 +4488,13 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   const struct _protocol_list_t * const protocols;
   //   const struct _prop_list_t * const properties;
   // }
-  CategorynfABITy = llvm::StructType::createNamed("struct._category_t",
-                                                  Int8PtrTy,
-                                                  ClassnfABIPtrTy,
-                                                  MethodListnfABIPtrTy,
-                                                  MethodListnfABIPtrTy,
-                                                  ProtocolListnfABIPtrTy,
-                                                  PropertyListPtrTy,
-                                                  NULL);
+  CategorynfABITy = llvm::StructType::create("struct._category_t",
+                                             Int8PtrTy, ClassnfABIPtrTy,
+                                             MethodListnfABIPtrTy,
+                                             MethodListnfABIPtrTy,
+                                             ProtocolListnfABIPtrTy,
+                                             PropertyListPtrTy,
+                                             NULL);
 
   // New types for nonfragile abi messaging.
   CodeGen::CodeGenTypes &Types = CGM.getTypes();
@@ -4569,8 +4530,8 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   SEL name;
   // };
   SuperMessageRefTy =
-    llvm::StructType::createNamed("struct._super_message_ref_t",
-                                  ImpnfABITy, SelectorPtrTy, NULL);
+    llvm::StructType::create("struct._super_message_ref_t",
+                             ImpnfABITy, SelectorPtrTy, NULL);
 
   // SuperMessageRefPtrTy - LLVM for struct _super_message_ref_t*
   SuperMessageRefPtrTy = llvm::PointerType::getUnqual(SuperMessageRefTy);
@@ -4582,11 +4543,9 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
   //   Class        cls;
   // };
   EHTypeTy =
-    llvm::StructType::createNamed("struct._objc_typeinfo",
-                                  llvm::PointerType::getUnqual(Int8PtrTy),
-                                  Int8PtrTy,
-                                  ClassnfABIPtrTy,
-                                  NULL);
+    llvm::StructType::create("struct._objc_typeinfo",
+                             llvm::PointerType::getUnqual(Int8PtrTy),
+                             Int8PtrTy, ClassnfABIPtrTy, NULL);
   EHTypePtrTy = llvm::PointerType::getUnqual(EHTypeTy);
 }
 
@@ -4694,7 +4653,7 @@ bool CGObjCNonFragileABIMac::isVTableDispatchedSelector(Selector Sel) {
 
     // These are vtable-based if GC is disabled.
     // Optimistically use vtable dispatch for hybrid compiles.
-    if (CGM.getLangOptions().getGCMode() != LangOptions::GCOnly) {
+    if (CGM.getLangOptions().getGC() != LangOptions::GCOnly) {
       VTableDispatchMethods.insert(GetNullarySelector("retain"));
       VTableDispatchMethods.insert(GetNullarySelector("release"));
       VTableDispatchMethods.insert(GetNullarySelector("autorelease"));
@@ -4710,7 +4669,7 @@ bool CGObjCNonFragileABIMac::isVTableDispatchedSelector(Selector Sel) {
 
     // These are vtable-based if GC is enabled.
     // Optimistically use vtable dispatch for hybrid compiles.
-    if (CGM.getLangOptions().getGCMode() != LangOptions::NonGC) {
+    if (CGM.getLangOptions().getGC() != LangOptions::NonGC) {
       VTableDispatchMethods.insert(GetNullarySelector("hash"));
       VTableDispatchMethods.insert(GetUnarySelector("addObject"));
     
@@ -5035,7 +4994,7 @@ llvm::Value *CGObjCNonFragileABIMac::GenerateProtocolRef(CGBuilderTy &Builder,
 
   llvm::GlobalVariable *PTGV = CGM.getModule().getGlobalVariable(ProtocolName);
   if (PTGV)
-    return Builder.CreateLoad(PTGV, "tmp");
+    return Builder.CreateLoad(PTGV);
   PTGV = new llvm::GlobalVariable(
     CGM.getModule(),
     Init->getType(), false,
@@ -5045,7 +5004,7 @@ llvm::Value *CGObjCNonFragileABIMac::GenerateProtocolRef(CGBuilderTy &Builder,
   PTGV->setSection("__DATA, __objc_protorefs, coalesced, no_dead_strip");
   PTGV->setVisibility(llvm::GlobalValue::HiddenVisibility);
   CGM.AddUsedGlobal(PTGV);
-  return Builder.CreateLoad(PTGV, "tmp");
+  return Builder.CreateLoad(PTGV);
 }
 
 /// GenerateCategory - Build metadata for a category implementation.
@@ -5167,7 +5126,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetMethodConstant(
 ///   struct _objc_method method_list[method_count];
 /// }
 ///
-llvm::Constant *CGObjCNonFragileABIMac::EmitMethodList(llvm::Twine Name,
+llvm::Constant *CGObjCNonFragileABIMac::EmitMethodList(Twine Name,
                                                        const char *Section,
                                                 const ConstantVector &Methods) {
   // Return null for empty list.
@@ -5258,13 +5217,12 @@ llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList(
 
   std::vector<llvm::Constant*> Ivars, Ivar(5);
 
-  ObjCInterfaceDecl *OID = 
-    const_cast<ObjCInterfaceDecl*>(ID->getClassInterface());
+  const ObjCInterfaceDecl *OID = ID->getClassInterface();
   assert(OID && "CGObjCNonFragileABIMac::EmitIvarList - null interface");
 
   // FIXME. Consolidate this with similar code in GenerateClass.
 
-  for (ObjCIvarDecl *IVD = OID->all_declared_ivar_begin(); 
+  for (const ObjCIvarDecl *IVD = OID->all_declared_ivar_begin(); 
        IVD; IVD = IVD->getNextIvar()) {
     // Ignore unnamed bit-fields.
     if (!IVD->getDeclName())
@@ -5273,7 +5231,7 @@ llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList(
                                 ComputeIvarBaseOffset(CGM, ID, IVD));
     Ivar[1] = GetMethodVarName(IVD->getIdentifier());
     Ivar[2] = GetMethodVarType(IVD);
-    const llvm::Type *FieldTy =
+    llvm::Type *FieldTy =
       CGM.getTypes().ConvertTypeForMem(IVD->getType());
     unsigned Size = CGM.getTargetData().getTypeAllocSize(FieldTy);
     unsigned Align = CGM.getContext().getPreferredTypeAlign(
@@ -5461,7 +5419,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol(
 /// @endcode
 ///
 llvm::Constant *
-CGObjCNonFragileABIMac::EmitProtocolList(llvm::Twine Name,
+CGObjCNonFragileABIMac::EmitProtocolList(Twine Name,
                                       ObjCProtocolDecl::protocol_iterator begin,
                                       ObjCProtocolDecl::protocol_iterator end) {
   std::vector<llvm::Constant*> ProtocolRefs;
@@ -5669,7 +5627,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF,
   callee = CGF.Builder.CreateLoad(callee, "msgSend_fn");
 
   bool variadic = method ? method->isVariadic() : false;
-  const llvm::FunctionType *fnType = 
+  llvm::FunctionType *fnType = 
     CGF.getTypes().GetFunctionType(fnInfo, variadic);
   callee = CGF.Builder.CreateBitCast(callee,
                                      llvm::PointerType::getUnqual(fnType));
@@ -5730,7 +5688,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitClassRefFromId(CGBuilderTy &Builder,
     CGM.AddUsedGlobal(Entry);
   }
   
-  return Builder.CreateLoad(Entry, "tmp");
+  return Builder.CreateLoad(Entry);
 }
 
 llvm::Value *CGObjCNonFragileABIMac::EmitClassRef(CGBuilderTy &Builder,
@@ -5764,7 +5722,7 @@ CGObjCNonFragileABIMac::EmitSuperClassRef(CGBuilderTy &Builder,
     CGM.AddUsedGlobal(Entry);
   }
 
-  return Builder.CreateLoad(Entry, "tmp");
+  return Builder.CreateLoad(Entry);
 }
 
 /// EmitMetaClassRef - Return a Value * of the address of _class_t
@@ -5774,7 +5732,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CGBuilderTy &Builder,
                                                       const ObjCInterfaceDecl *ID) {
   llvm::GlobalVariable * &Entry = MetaClassReferences[ID->getIdentifier()];
   if (Entry)
-    return Builder.CreateLoad(Entry, "tmp");
+    return Builder.CreateLoad(Entry);
 
   std::string MetaClassName(getMetaclassSymbolPrefix() + ID->getNameAsString());
   llvm::GlobalVariable *MetaClassGV = GetClassGlobal(MetaClassName);
@@ -5790,7 +5748,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CGBuilderTy &Builder,
   Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip");
   CGM.AddUsedGlobal(Entry);
 
-  return Builder.CreateLoad(Entry, "tmp");
+  return Builder.CreateLoad(Entry);
 }
 
 /// GetClass - Return a reference to the class for the given interface
@@ -5847,7 +5805,7 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
 
   // FIXME: We shouldn't need to do this cast, rectify the ASTContext and
   // ObjCTypes types.
-  const llvm::Type *ClassTy =
+  llvm::Type *ClassTy =
     CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType());
   Target = CGF.Builder.CreateBitCast(Target, ClassTy);
   CGF.Builder.CreateStore(Target,
@@ -5881,7 +5839,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CGBuilderTy &Builder,
 
   if (lval)
     return Entry;
-  return Builder.CreateLoad(Entry, "tmp");
+  return Builder.CreateLoad(Entry);
 }
 /// EmitObjCIvarAssign - Code gen for assigning to a __strong object.
 /// objc_assign_ivar (id src, id *dst, ptrdiff_t)
@@ -5890,7 +5848,7 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                                 llvm::Value *src,
                                                 llvm::Value *dst,
                                                 llvm::Value *ivarOffset) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -5911,7 +5869,7 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
 void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
   CodeGen::CodeGenFunction &CGF,
   llvm::Value *src, llvm::Value *dst) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -5944,7 +5902,7 @@ void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable(
 llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead(
   CodeGen::CodeGenFunction &CGF,
   llvm::Value *AddrWeakObj) {
-  const llvm::Type* DestTy =
+  llvm::Type* DestTy =
     cast<llvm::PointerType>(AddrWeakObj->getType())->getElementType();
   AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj, ObjCTypes.PtrObjectPtrTy);
   llvm::Value *read_weak = CGF.Builder.CreateCall(ObjCTypes.getGcReadWeakFn(),
@@ -5958,7 +5916,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead(
 ///
 void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                                 llvm::Value *src, llvm::Value *dst) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -5979,7 +5937,7 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
 void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
                                           llvm::Value *src, llvm::Value *dst,
                                           bool threadlocal) {
-  const llvm::Type * SrcTy = src->getType();
+  llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
@@ -6043,9 +6001,8 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
 void CGObjCNonFragileABIMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
                                            const ObjCAtThrowStmt &S) {
   if (const Expr *ThrowExpr = S.getThrowExpr()) {
-    llvm::Value *Exception = CGF.EmitScalarExpr(ThrowExpr);
-    Exception = CGF.Builder.CreateBitCast(Exception, ObjCTypes.ObjectPtrTy,
-                                          "tmp");
+    llvm::Value *Exception = CGF.EmitObjCThrowOperand(ThrowExpr);
+    Exception = CGF.Builder.CreateBitCast(Exception, ObjCTypes.ObjectPtrTy);
     CGF.EmitCallOrInvoke(ObjCTypes.getExceptionThrowFn(), Exception)
       .setDoesNotReturn();
   } else {
@@ -6096,7 +6053,7 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID,
     llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), 2);
 
   std::vector<llvm::Constant*> Values(3);
-  Values[0] = llvm::ConstantExpr::getGetElementPtr(VTableGV, &VTableIdx, 1);
+  Values[0] = llvm::ConstantExpr::getGetElementPtr(VTableGV, VTableIdx);
   Values[1] = GetClassName(ID->getIdentifier());
   Values[2] = GetClassGlobal(ClassName);
   llvm::Constant *Init =
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp
index 09c8d0b28f9c..ef426ce6ed9b 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -52,9 +52,8 @@ static uint64_t LookupFieldBitOffset(CodeGen::CodeGenModule &CGM,
   // implemented. This should be fixed to get the information from the layout
   // directly.
   unsigned Index = 0;
-  ObjCInterfaceDecl *IDecl = const_cast<ObjCInterfaceDecl*>(Container);
 
-  for (ObjCIvarDecl *IVD = IDecl->all_declared_ivar_begin(); 
+  for (const ObjCIvarDecl *IVD = Container->all_declared_ivar_begin(); 
        IVD; IVD = IVD->getNextIvar()) {
     if (Ivar == IVD)
       break;
@@ -86,9 +85,9 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
                                                unsigned CVRQualifiers,
                                                llvm::Value *Offset) {
   // Compute (type*) ( (char *) BaseValue + Offset)
-  const llvm::Type *I8Ptr = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *I8Ptr = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   QualType IvarTy = Ivar->getType();
-  const llvm::Type *LTy = CGF.CGM.getTypes().ConvertTypeForMem(IvarTy);
+  llvm::Type *LTy = CGF.CGM.getTypes().ConvertTypeForMem(IvarTy);
   llvm::Value *V = CGF.Builder.CreateBitCast(BaseValue, I8Ptr);
   V = CGF.Builder.CreateInBoundsGEP(V, Offset, "add.ptr");
   V = CGF.Builder.CreateBitCast(V, llvm::PointerType::getUnqual(LTy));
@@ -118,10 +117,9 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
   uint64_t TypeSizeInBits = CGF.CGM.getContext().toBits(RL.getSize());
   uint64_t FieldBitOffset = LookupFieldBitOffset(CGF.CGM, OID, 0, Ivar);
   uint64_t BitOffset = FieldBitOffset % CGF.CGM.getContext().getCharWidth();
-  uint64_t ContainingTypeAlign = CGF.CGM.getContext().Target.getCharAlign();
+  uint64_t ContainingTypeAlign = CGF.CGM.getContext().getTargetInfo().getCharAlign();
   uint64_t ContainingTypeSize = TypeSizeInBits - (FieldBitOffset - BitOffset);
-  uint64_t BitFieldSize =
-    Ivar->getBitWidth()->EvaluateAsInt(CGF.getContext()).getZExtValue();
+  uint64_t BitFieldSize = Ivar->getBitWidthValue(CGF.getContext());
 
   // Allocate a new CGBitFieldInfo object to describe this access.
   //
@@ -178,7 +176,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
     FinallyInfo.enter(CGF, Finally->getFinallyBody(),
                       beginCatchFn, endCatchFn, exceptionRethrowFn);
 
-  llvm::SmallVector<CatchHandler, 8> Handlers;
+  SmallVector<CatchHandler, 8> Handlers;
 
   // Enter the catch, if there is one.
   if (S.getNumCatchStmts()) {
@@ -212,7 +210,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
 
   // Leave the try.
   if (S.getNumCatchStmts())
-    CGF.EHStack.popCatch();
+    CGF.popCatchScope();
 
   // Remember where we were.
   CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
@@ -222,7 +220,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
     CatchHandler &Handler = Handlers[I];
 
     CGF.EmitBlock(Handler.Block);
-    llvm::Value *RawExn = CGF.Builder.CreateLoad(CGF.getExceptionSlot());
+    llvm::Value *RawExn = CGF.getExceptionFromSlot();
 
     // Enter the catch.
     llvm::Value *Exn = RawExn;
@@ -244,7 +242,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
 
     // Bind the catch parameter if it exists.
     if (const VarDecl *CatchParam = Handler.Variable) {
-      const llvm::Type *CatchType = CGF.ConvertType(CatchParam->getType());
+      llvm::Type *CatchType = CGF.ConvertType(CatchParam->getType());
       llvm::Value *CastExn = CGF.Builder.CreateBitCast(Exn, CatchType);
 
       CGF.EmitAutoVarDecl(*CatchParam);
@@ -289,21 +287,26 @@ void CGObjCRuntime::EmitAtSynchronizedStmt(CodeGenFunction &CGF,
                                            const ObjCAtSynchronizedStmt &S,
                                            llvm::Function *syncEnterFn,
                                            llvm::Function *syncExitFn) {
-  // Evaluate the lock operand.  This should dominate the cleanup.
-  llvm::Value *SyncArg =
-    CGF.EmitScalarExpr(S.getSynchExpr());
+  CodeGenFunction::RunCleanupsScope cleanups(CGF);
+
+  // Evaluate the lock operand.  This is guaranteed to dominate the
+  // ARC release and lock-release cleanups.
+  const Expr *lockExpr = S.getSynchExpr();
+  llvm::Value *lock;
+  if (CGF.getLangOptions().ObjCAutoRefCount) {
+    lock = CGF.EmitARCRetainScalarExpr(lockExpr);
+    lock = CGF.EmitObjCConsumeObject(lockExpr->getType(), lock);
+  } else {
+    lock = CGF.EmitScalarExpr(lockExpr);
+  }
+  lock = CGF.Builder.CreateBitCast(lock, CGF.VoidPtrTy);
 
   // Acquire the lock.
-  SyncArg = CGF.Builder.CreateBitCast(SyncArg, syncEnterFn->getFunctionType()->getParamType(0));
-  CGF.Builder.CreateCall(syncEnterFn, SyncArg);
+  CGF.Builder.CreateCall(syncEnterFn, lock)->setDoesNotThrow();
 
   // Register an all-paths cleanup to release the lock.
-  CGF.EHStack.pushCleanup<CallSyncExit>(NormalAndEHCleanup, syncExitFn,
-      SyncArg);
+  CGF.EHStack.pushCleanup<CallSyncExit>(NormalAndEHCleanup, syncExitFn, lock);
 
   // Emit the body of the statement.
   CGF.EmitStmt(S.getSynchBody());
-
-  // Pop the lock-release cleanup.
-  CGF.PopCleanupBlock();
 }
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h
index 7accc70c9623..4fa47a740aaf 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h
@@ -208,8 +208,7 @@ class CGObjCRuntime {
   
   
   virtual llvm::Value *EmitNSAutoreleasePoolClassRef(CGBuilderTy &Builder) {
-    assert(false &&"autoreleasepool unsupported in this ABI");
-    return 0;
+    llvm_unreachable("autoreleasepool unsupported in this ABI");
   }
   
   /// EnumerationMutationFunction - Return the function that's called by the
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp
new file mode 100644
index 000000000000..3a0e116e5ab1
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp
@@ -0,0 +1,28 @@
+//===----- CGOpenCLRuntime.cpp - Interface to OpenCL Runtimes -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for OpenCL code generation.  Concrete
+// subclasses of this implement code generation for specific OpenCL
+// runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGOpenCLRuntime.h"
+#include "CodeGenFunction.h"
+#include "llvm/GlobalValue.h"
+
+using namespace clang;
+using namespace CodeGen;
+
+CGOpenCLRuntime::~CGOpenCLRuntime() {}
+
+void CGOpenCLRuntime::EmitWorkGroupLocalVarDecl(CodeGenFunction &CGF,
+                                                const VarDecl &D) {
+  return CGF.EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage);
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h
new file mode 100644
index 000000000000..9a8430fb7500
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h
@@ -0,0 +1,46 @@
+//===----- CGOpenCLRuntime.h - Interface to OpenCL Runtimes -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for OpenCL code generation.  Concrete
+// subclasses of this implement code generation for specific OpenCL
+// runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CODEGEN_OPENCLRUNTIME_H
+#define CLANG_CODEGEN_OPENCLRUNTIME_H
+
+namespace clang {
+
+class VarDecl;
+
+namespace CodeGen {
+
+class CodeGenFunction;
+class CodeGenModule;
+
+class CGOpenCLRuntime {
+protected:
+  CodeGenModule &CGM;
+
+public:
+  CGOpenCLRuntime(CodeGenModule &CGM) : CGM(CGM) {}
+  virtual ~CGOpenCLRuntime();
+
+  /// Emit the IR required for a work-group-local variable declaration, and add
+  /// an entry to CGF's LocalDeclMap for D.  The base class does this using
+  /// CodeGenFunction::EmitStaticVarDecl to emit an internal global for D.
+  virtual void EmitWorkGroupLocalVarDecl(CodeGenFunction &CGF,
+                                         const VarDecl &D);
+};
+
+}
+}
+
+#endif
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGRTTI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGRTTI.cpp
index e564c7070525..fbdb2984830b 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGRTTI.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGRTTI.cpp
@@ -26,10 +26,10 @@ class RTTIBuilder {
   CodeGenModule &CGM;  // Per-module state.
   llvm::LLVMContext &VMContext;
   
-  const llvm::Type *Int8PtrTy;
+  llvm::Type *Int8PtrTy;
   
   /// Fields - The fields of the RTTI descriptor currently being built.
-  llvm::SmallVector<llvm::Constant *, 16> Fields;
+  SmallVector<llvm::Constant *, 16> Fields;
 
   /// GetAddrOfTypeName - Returns the mangled type name of the given type.
   llvm::GlobalVariable *
@@ -120,7 +120,7 @@ RTTIBuilder::GetAddrOfTypeName(QualType Ty,
   llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
   // We know that the mangled name of the type starts at index 4 of the
   // mangled name of the typename, so we can just index into it in order to
@@ -141,7 +141,7 @@ llvm::Constant *RTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
   llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
   // Look for an existing global.
   llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name);
@@ -185,6 +185,7 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
     case BuiltinType::ULong:
     case BuiltinType::LongLong:
     case BuiltinType::ULongLong:
+    case BuiltinType::Half:
     case BuiltinType::Float:
     case BuiltinType::Double:
     case BuiltinType::LongDouble:
@@ -203,7 +204,7 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
     case BuiltinType::ObjCId:
     case BuiltinType::ObjCClass:
     case BuiltinType::ObjCSel:
-      assert(false && "FIXME: Objective-C types are unsupported!");
+      llvm_unreachable("FIXME: Objective-C types are unsupported!");
   }
   
   // Silent gcc.
@@ -267,7 +268,7 @@ static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM, QualType Ty) {
 
 /// IsIncompleteClassType - Returns whether the given record type is incomplete.
 static bool IsIncompleteClassType(const RecordType *RecordTy) {
-  return !RecordTy->getDecl()->isDefinition();
+  return !RecordTy->getDecl()->isCompleteDefinition();
 }  
 
 /// ContainsIncompleteClassType - Returns whether the given type contains an
@@ -393,17 +394,18 @@ void RTTIBuilder::BuildVTablePointer(const Type *Ty) {
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #include "clang/AST/TypeNodes.def"
-    assert(false && "Non-canonical and dependent types shouldn't get here");
+    llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   case Type::LValueReference:
   case Type::RValueReference:
-    assert(false && "References shouldn't get here");
+    llvm_unreachable("References shouldn't get here");
 
   case Type::Builtin:
   // GCC treats vector and complex types as fundamental types.
   case Type::Vector:
   case Type::ExtVector:
   case Type::Complex:
+  case Type::Atomic:
   // FIXME: GCC treats block pointers as fundamental types?!
   case Type::BlockPointer:
     // abi::__fundamental_type_info.
@@ -479,12 +481,12 @@ void RTTIBuilder::BuildVTablePointer(const Type *Ty) {
   llvm::Constant *VTable = 
     CGM.getModule().getOrInsertGlobal(VTableName, Int8PtrTy);
     
-  const llvm::Type *PtrDiffTy = 
+  llvm::Type *PtrDiffTy = 
     CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
 
   // The vtable address point is 2.
   llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2);
-  VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, &Two, 1);
+  VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Two);
   VTable = llvm::ConstantExpr::getBitCast(VTable, Int8PtrTy);
 
   Fields.push_back(VTable);
@@ -533,7 +535,7 @@ maybeUpdateRTTILinkage(CodeGenModule &CGM, llvm::GlobalVariable *GV,
   llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
   llvm::GlobalVariable *TypeNameGV = CGM.getModule().getNamedGlobal(Name);
 
@@ -553,7 +555,7 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
   llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
   if (OldGV && !OldGV->isDeclaration()) {
@@ -580,7 +582,7 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   // And the name.
   llvm::GlobalVariable *TypeName = GetAddrOfTypeName(Ty, Linkage);
 
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
   Fields.push_back(llvm::ConstantExpr::getBitCast(TypeName, Int8PtrTy));
 
   switch (Ty->getTypeClass()) {
@@ -590,7 +592,7 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #include "clang/AST/TypeNodes.def"
-    assert(false && "Non-canonical and dependent types shouldn't get here");
+    llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   // GCC treats vector types as fundamental types.
   case Type::Builtin:
@@ -604,7 +606,7 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
 
   case Type::LValueReference:
   case Type::RValueReference:
-    assert(false && "References shouldn't get here");
+    llvm_unreachable("References shouldn't get here");
 
   case Type::ConstantArray:
   case Type::IncompleteArray:
@@ -656,6 +658,10 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   case Type::MemberPointer:
     BuildPointerToMemberTypeInfo(cast<MemberPointerType>(Ty));
     break;
+
+  case Type::Atomic:
+    // No fields, at least for the moment.
+    break;
   }
 
   llvm::Constant *Init = llvm::ConstantStruct::getAnon(Fields);
@@ -822,7 +828,7 @@ static unsigned ComputeVMIClassTypeInfoFlags(const CXXRecordDecl *RD) {
 /// classes with bases that do not satisfy the abi::__si_class_type_info 
 /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
 void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
-  const llvm::Type *UnsignedIntLTy = 
+  llvm::Type *UnsignedIntLTy = 
     CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy);
   
   // Itanium C++ ABI 2.9.5p6c:
@@ -840,7 +846,7 @@ void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
   if (!RD->getNumBases())
     return;
   
-  const llvm::Type *LongLTy = 
+  llvm::Type *LongLTy = 
     CGM.getTypes().ConvertType(CGM.getContext().LongTy);
 
   // Now add the base class descriptions.
@@ -879,7 +885,7 @@ void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
     CharUnits Offset;
     if (Base->isVirtual())
       Offset = 
-        CGM.getVTables().getVirtualBaseOffsetOffset(RD, BaseDecl);
+        CGM.getVTableContext().getVirtualBaseOffsetOffset(RD, BaseDecl);
     else {
       const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
       Offset = Layout.getBaseClassOffset(BaseDecl);
@@ -916,7 +922,7 @@ void RTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) {
   if (ContainsIncompleteClassType(UnqualifiedPointeeTy))
     Flags |= PTI_Incomplete;
 
-  const llvm::Type *UnsignedIntLTy = 
+  llvm::Type *UnsignedIntLTy = 
     CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy);
   Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, Flags));
   
@@ -953,7 +959,7 @@ void RTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
   if (IsIncompleteClassType(ClassType))
     Flags |= PTI_ContainingClassIncomplete;
   
-  const llvm::Type *UnsignedIntLTy = 
+  llvm::Type *UnsignedIntLTy = 
     CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy);
   Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, Flags));
   
@@ -977,12 +983,12 @@ llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
   // FIXME: should we even be calling this method if RTTI is disabled
   // and it's not for EH?
   if (!ForEH && !getContext().getLangOptions().RTTI) {
-    const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+    llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
     return llvm::Constant::getNullValue(Int8PtrTy);
   }
   
   if (ForEH && Ty->isObjCObjectPointerType() && !Features.NeXTRuntime) {
-    return Runtime->GetEHType(Ty);
+    return ObjCRuntime->GetEHType(Ty);
   }
 
   return RTTIBuilder(*this).BuildTypeInfo(Ty);
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h
index 8a450298f70a..25a0a508f188 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h
@@ -10,12 +10,13 @@
 #ifndef CLANG_CODEGEN_CGRECORDLAYOUT_H
 #define CLANG_CODEGEN_CGRECORDLAYOUT_H
 
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/DerivedTypes.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Decl.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/DerivedTypes.h"
+
 namespace llvm {
-  class raw_ostream;
   class StructType;
 }
 
@@ -144,7 +145,7 @@ class CGBitFieldInfo {
 
   /// @}
 
-  void print(llvm::raw_ostream &OS) const;
+  void print(raw_ostream &OS) const;
   void dump() const;
 
   /// \brief Given a bit-field decl, build an appropriate helper object for
@@ -270,7 +271,7 @@ class CGRecordLayout {
     return it->second;
   }
 
-  void print(llvm::raw_ostream &OS) const;
+  void print(raw_ostream &OS) const;
   void dump() const;
 };
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 2b07bafa0096..6475ccac0389 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -35,7 +35,7 @@ class CGRecordLayoutBuilder {
 public:
   /// FieldTypes - Holds the LLVM types that the struct is created from.
   /// 
-  llvm::SmallVector<llvm::Type *, 16> FieldTypes;
+  SmallVector<llvm::Type *, 16> FieldTypes;
 
   /// BaseSubobjectType - Holds the LLVM type for the non-virtual part
   /// of the struct. For example, consider:
@@ -174,7 +174,7 @@ class CGRecordLayoutBuilder {
   /// the passed size.
   void AppendTailPadding(CharUnits RecordSize);
 
-  CharUnits getTypeAlignment(const llvm::Type *Ty) const;
+  CharUnits getTypeAlignment(llvm::Type *Ty) const;
 
   /// getAlignmentAsLLVMStruct - Returns the maximum alignment of all the
   /// LLVM element types.
@@ -230,7 +230,7 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
                                uint64_t FieldSize,
                                uint64_t ContainingTypeSizeInBits,
                                unsigned ContainingTypeAlign) {
-  const llvm::Type *Ty = Types.ConvertTypeForMem(FD->getType());
+  llvm::Type *Ty = Types.ConvertTypeForMem(FD->getType());
   CharUnits TypeSizeInBytes =
     CharUnits::fromQuantity(Types.getTargetData().getTypeAllocSize(Ty));
   uint64_t TypeSizeInBits = Types.getContext().toBits(TypeSizeInBytes);
@@ -363,15 +363,14 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
 
 void CGRecordLayoutBuilder::LayoutBitField(const FieldDecl *D,
                                            uint64_t fieldOffset) {
-  uint64_t fieldSize =
-    D->getBitWidth()->EvaluateAsInt(Types.getContext()).getZExtValue();
+  uint64_t fieldSize = D->getBitWidthValue(Types.getContext());
 
   if (fieldSize == 0)
     return;
 
   uint64_t nextFieldOffsetInBits = Types.getContext().toBits(NextFieldOffset);
   CharUnits numBytesToAppend;
-  unsigned charAlign = Types.getContext().Target.getCharAlign();
+  unsigned charAlign = Types.getContext().getTargetInfo().getCharAlign();
 
   if (fieldOffset < nextFieldOffsetInBits && !BitsAvailableInLastField) {
     assert(fieldOffset % charAlign == 0 && 
@@ -492,8 +491,7 @@ llvm::Type *
 CGRecordLayoutBuilder::LayoutUnionField(const FieldDecl *Field,
                                         const ASTRecordLayout &Layout) {
   if (Field->isBitField()) {
-    uint64_t FieldSize =
-      Field->getBitWidth()->EvaluateAsInt(Types.getContext()).getZExtValue();
+    uint64_t FieldSize = Field->getBitWidthValue(Types.getContext());
 
     // Ignore zero sized bit fields.
     if (FieldSize == 0)
@@ -502,7 +500,7 @@ CGRecordLayoutBuilder::LayoutUnionField(const FieldDecl *Field,
     llvm::Type *FieldTy = llvm::Type::getInt8Ty(Types.getLLVMContext());
     CharUnits NumBytesToAppend = Types.getContext().toCharUnitsFromBits(
       llvm::RoundUpToAlignment(FieldSize, 
-                               Types.getContext().Target.getCharAlign()));
+                               Types.getContext().getTargetInfo().getCharAlign()));
 
     if (NumBytesToAppend > CharUnits::One())
       FieldTy = llvm::ArrayType::get(FieldTy, NumBytesToAppend.getQuantity());
@@ -672,10 +670,10 @@ CGRecordLayoutBuilder::LayoutNonVirtualBases(const CXXRecordDecl *RD,
   // Check if we need to add a vtable pointer.
   if (RD->isDynamicClass()) {
     if (!PrimaryBase) {
-      const llvm::Type *FunctionType =
+      llvm::Type *FunctionType =
         llvm::FunctionType::get(llvm::Type::getInt32Ty(Types.getLLVMContext()),
                                 /*isVarArg=*/true);
-      const llvm::Type *VTableTy = FunctionType->getPointerTo();
+      llvm::Type *VTableTy = FunctionType->getPointerTo();
 
       assert(NextFieldOffset.isZero() &&
              "VTable pointer must come first!");
@@ -735,8 +733,8 @@ CGRecordLayoutBuilder::ComputeNonVirtualBaseType(const CXXRecordDecl *RD) {
   }
 
   
-  BaseSubobjectType = llvm::StructType::createNamed(Types.getLLVMContext(), "",
-                                                    FieldTypes, Packed);
+  BaseSubobjectType = llvm::StructType::create(Types.getLLVMContext(),
+                                               FieldTypes, "", Packed);
   Types.addRecordTypeName(RD, BaseSubobjectType, ".base");
 
   // Pull the padding back off.
@@ -882,7 +880,7 @@ void CGRecordLayoutBuilder::AppendBytes(CharUnits numBytes) {
   AppendField(NextFieldOffset, getByteArrayType(numBytes));
 }
 
-CharUnits CGRecordLayoutBuilder::getTypeAlignment(const llvm::Type *Ty) const {
+CharUnits CGRecordLayoutBuilder::getTypeAlignment(llvm::Type *Ty) const {
   if (Packed)
     return CharUnits::One();
 
@@ -983,7 +981,7 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D,
   }
                                      
   // Verify that the LLVM and AST field offsets agree.
-  const llvm::StructType *ST =
+  llvm::StructType *ST =
     dyn_cast<llvm::StructType>(RL->getLLVMType());
   const llvm::StructLayout *SL = getTargetData().getStructLayout(ST);
 
@@ -1037,7 +1035,7 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D,
   return RL;
 }
 
-void CGRecordLayout::print(llvm::raw_ostream &OS) const {
+void CGRecordLayout::print(raw_ostream &OS) const {
   OS << "<CGRecordLayout\n";
   OS << "  LLVMType:" << *CompleteObjectType << "\n";
   if (BaseSubobjectType)
@@ -1071,7 +1069,7 @@ void CGRecordLayout::dump() const {
   print(llvm::errs());
 }
 
-void CGBitFieldInfo::print(llvm::raw_ostream &OS) const {
+void CGBitFieldInfo::print(raw_ostream &OS) const {
   OS << "<CGBitFieldInfo";
   OS << " Size:" << Size;
   OS << " IsSigned:" << IsSigned << "\n";
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp
index 07bddb797265..c56931bbc6fa 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp
@@ -31,20 +31,19 @@ using namespace CodeGen;
 
 void CodeGenFunction::EmitStopPoint(const Stmt *S) {
   if (CGDebugInfo *DI = getDebugInfo()) {
+    SourceLocation Loc;
     if (isa<DeclStmt>(S))
-      DI->setLocation(S->getLocEnd());
+      Loc = S->getLocEnd();
     else
-      DI->setLocation(S->getLocStart());
-    DI->UpdateLineDirectiveRegion(Builder);
-    DI->EmitStopPoint(Builder);
+      Loc = S->getLocStart();
+    DI->EmitLocation(Builder, Loc);
   }
 }
 
 void CodeGenFunction::EmitStmt(const Stmt *S) {
   assert(S && "Null statement?");
 
-  // Check if we can handle this without bothering to generate an
-  // insert point or debug info.
+  // These statements have their own debug info handling.
   if (EmitSimpleStmt(S))
     return;
 
@@ -137,11 +136,11 @@ void CodeGenFunction::EmitStmt(const Stmt *S) {
     EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));
     break;
   case Stmt::ObjCAtCatchStmtClass:
-    assert(0 && "@catch statements should be handled by EmitObjCAtTryStmt");
-    break;
+    llvm_unreachable(
+                    "@catch statements should be handled by EmitObjCAtTryStmt");
   case Stmt::ObjCAtFinallyStmtClass:
-    assert(0 && "@finally statements should be handled by EmitObjCAtTryStmt");
-    break;
+    llvm_unreachable(
+                  "@finally statements should be handled by EmitObjCAtTryStmt");
   case Stmt::ObjCAtThrowStmtClass:
     EmitObjCAtThrowStmt(cast<ObjCAtThrowStmt>(*S));
     break;
@@ -192,10 +191,8 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
                              "LLVM IR generation of compound statement ('{}')");
 
   CGDebugInfo *DI = getDebugInfo();
-  if (DI) {
-    DI->setLocation(S.getLBracLoc());
-    DI->EmitRegionStart(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockStart(Builder, S.getLBracLoc());
 
   // Keep track of the current cleanup stack depth.
   RunCleanupsScope Scope(*this);
@@ -204,10 +201,8 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
        E = S.body_end()-GetLast; I != E; ++I)
     EmitStmt(*I);
 
-  if (DI) {
-    DI->setLocation(S.getRBracLoc());
-    DI->EmitRegionEnd(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockEnd(Builder, S.getRBracLoc());
 
   RValue RV;
   if (!GetLast)
@@ -286,6 +281,23 @@ void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
   Builder.ClearInsertionPoint();
 }
 
+void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
+  bool inserted = false;
+  for (llvm::BasicBlock::use_iterator
+         i = block->use_begin(), e = block->use_end(); i != e; ++i) {
+    if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(*i)) {
+      CurFn->getBasicBlockList().insertAfter(insn->getParent(), block);
+      inserted = true;
+      break;
+    }
+  }
+
+  if (!inserted)
+    CurFn->getBasicBlockList().push_back(block);
+
+  Builder.SetInsertPoint(block);
+}
+
 CodeGenFunction::JumpDest
 CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
   JumpDest &Dest = LabelMap[D];
@@ -555,10 +567,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
   RunCleanupsScope ForScope(*this);
 
   CGDebugInfo *DI = getDebugInfo();
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getBegin());
-    DI->EmitRegionStart(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin());
 
   // Evaluate the first part before the loop.
   if (S.getInit())
@@ -637,10 +647,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
 
   ForScope.ForceCleanup();
 
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getEnd());
-    DI->EmitRegionEnd(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
 
   // Emit the fall-through block.
   EmitBlock(LoopExit.getBlock(), true);
@@ -652,10 +660,8 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
   RunCleanupsScope ForScope(*this);
 
   CGDebugInfo *DI = getDebugInfo();
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getBegin());
-    DI->EmitRegionStart(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin());
 
   // Evaluate the first pieces before the loop.
   EmitStmt(S.getRangeStmt());
@@ -711,10 +717,8 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
 
   ForScope.ForceCleanup();
 
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getEnd());
-    DI->EmitRegionEnd(Builder);
-  }
+  if (DI)
+    DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
 
   // Emit the fall-through block.
   EmitBlock(LoopExit.getBlock(), true);
@@ -767,7 +771,10 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
   } else if (RV->getType()->isAnyComplexType()) {
     EmitComplexExprIntoAddr(RV, ReturnValue, false);
   } else {
-    EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue, Qualifiers(), true));
+    EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue, Qualifiers(),
+                                          AggValueSlot::IsDestructed,
+                                          AggValueSlot::DoesNotNeedGCBarriers,
+                                          AggValueSlot::IsNotAliased));
   }
 
   EmitBranchThroughCleanup(ReturnBlock);
@@ -816,8 +823,8 @@ void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
 void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
   assert(S.getRHS() && "Expected RHS value in CaseStmt");
 
-  llvm::APSInt LHS = S.getLHS()->EvaluateAsInt(getContext());
-  llvm::APSInt RHS = S.getRHS()->EvaluateAsInt(getContext());
+  llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
+  llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
 
   // Emit the code for this case. We do this first to make sure it is
   // properly chained from our predecessor before generating the
@@ -856,7 +863,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
 
   // Emit range check.
   llvm::Value *Diff =
-    Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS),  "tmp");
+    Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));
   llvm::Value *Cond =
     Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");
   Builder.CreateCondBr(Cond, CaseDest, FalseDest);
@@ -876,7 +883,7 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
   }
 
   llvm::ConstantInt *CaseVal =
-    Builder.getInt(S.getLHS()->EvaluateAsInt(getContext()));
+    Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
 
   // If the body of the case is just a 'break', and if there was no fallthrough,
   // try to not emit an empty block.
@@ -917,7 +924,7 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
   while (NextCase && NextCase->getRHS() == 0) {
     CurCase = NextCase;
     llvm::ConstantInt *CaseVal = 
-      Builder.getInt(CurCase->getLHS()->EvaluateAsInt(getContext()));
+      Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
     SwitchInsn->addCase(CaseVal, CaseDest);
     NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
   }
@@ -961,7 +968,7 @@ enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
 static CSFC_Result CollectStatementsForCase(const Stmt *S,
                                             const SwitchCase *Case,
                                             bool &FoundCase,
-                              llvm::SmallVectorImpl<const Stmt*> &ResultStmts) {
+                              SmallVectorImpl<const Stmt*> &ResultStmts) {
   // If this is a null statement, just succeed.
   if (S == 0)
     return Case ? CSFC_Success : CSFC_FallThrough;
@@ -1086,7 +1093,7 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
 /// for more details.
 static bool FindCaseStatementsForValue(const SwitchStmt &S,
                                        const llvm::APInt &ConstantCondValue,
-                                llvm::SmallVectorImpl<const Stmt*> &ResultStmts,
+                                SmallVectorImpl<const Stmt*> &ResultStmts,
                                        ASTContext &C) {
   // First step, find the switch case that is being branched to.  We can do this
   // efficiently by scanning the SwitchCase list.
@@ -1107,7 +1114,7 @@ static bool FindCaseStatementsForValue(const SwitchStmt &S,
     if (CS->getRHS()) return false;
     
     // If we found our case, remember it as 'case'.
-    if (CS->getLHS()->EvaluateAsInt(C) == ConstantCondValue)
+    if (CS->getLHS()->EvaluateKnownConstInt(C) == ConstantCondValue)
       break;
   }
   
@@ -1147,7 +1154,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
   // emit the live case statement (if any) of the switch.
   llvm::APInt ConstantCondValue;
   if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {
-    llvm::SmallVector<const Stmt*, 4> CaseStmts;
+    SmallVector<const Stmt*, 4> CaseStmts;
     if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
                                    getContext())) {
       RunCleanupsScope ExecutedScope(*this);
@@ -1219,7 +1226,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
 
 static std::string
 SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
-                 llvm::SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=0) {
+                 SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=0) {
   std::string Result;
 
   while (*Constraint) {
@@ -1276,7 +1283,7 @@ AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
   AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
   if (!Attr)
     return Constraint;
-  llvm::StringRef Register = Attr->getLabel();
+  StringRef Register = Attr->getLabel();
   assert(Target.isValidGCCRegisterName(Register));
   // We're using validateOutputConstraint here because we only care if
   // this is a register constraint.
@@ -1301,7 +1308,7 @@ CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
     if (!CodeGenFunction::hasAggregateLLVMType(InputType)) {
       Arg = EmitLoadOfLValue(InputValue).getScalarVal();
     } else {
-      const llvm::Type *Ty = ConvertType(InputType);
+      llvm::Type *Ty = ConvertType(InputType);
       uint64_t Size = CGM.getTargetData().getTypeSizeInBits(Ty);
       if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
         Ty = llvm::IntegerType::get(getLLVMContext(), Size);
@@ -1341,11 +1348,11 @@ llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
 /// asm.
 static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
                                       CodeGenFunction &CGF) {
-  llvm::SmallVector<llvm::Value *, 8> Locs;
+  SmallVector<llvm::Value *, 8> Locs;
   // Add the location of the first line to the MDNode.
   Locs.push_back(llvm::ConstantInt::get(CGF.Int32Ty,
                                         Str->getLocStart().getRawEncoding()));
-  llvm::StringRef StrVal = Str->getString();
+  StringRef StrVal = Str->getString();
   if (!StrVal.empty()) {
     const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
     const LangOptions &LangOpts = CGF.CGM.getLangOptions();
@@ -1367,7 +1374,7 @@ static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
 void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   // Analyze the asm string to decompose it into its pieces.  We know that Sema
   // has already done this, so it is guaranteed to be successful.
-  llvm::SmallVector<AsmStmt::AsmStringPiece, 4> Pieces;
+  SmallVector<AsmStmt::AsmStringPiece, 4> Pieces;
   unsigned DiagOffs;
   S.AnalyzeAsmString(Pieces, getContext(), DiagOffs);
 
@@ -1384,8 +1391,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   }
 
   // Get all the output and input constraints together.
-  llvm::SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
-  llvm::SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
+  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
+  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
 
   for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
     TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i),
@@ -1530,14 +1537,18 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
         // Use ptrtoint as appropriate so that we can do our extension.
         if (isa<llvm::PointerType>(Arg->getType()))
           Arg = Builder.CreatePtrToInt(Arg, IntPtrTy);
-        const llvm::Type *OutputTy = ConvertType(OutputType);
+        llvm::Type *OutputTy = ConvertType(OutputType);
         if (isa<llvm::IntegerType>(OutputTy))
           Arg = Builder.CreateZExt(Arg, OutputTy);
-        else
+        else if (isa<llvm::PointerType>(OutputTy))
+          Arg = Builder.CreateZExt(Arg, IntPtrTy);
+        else {
+          assert(OutputTy->isFloatingPointTy() && "Unexpected output type");
           Arg = Builder.CreateFPExt(Arg, OutputTy);
+        }
       }
     }
-    if (const llvm::Type* AdjTy = 
+    if (llvm::Type* AdjTy = 
               getTargetHooks().adjustInlineAsmType(*this, InputConstraint,
                                                    Arg->getType()))
       Arg = Builder.CreateBitCast(Arg, AdjTy);
@@ -1556,7 +1567,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
   // Clobbers
   for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
-    llvm::StringRef Clobber = S.getClobber(i)->getString();
+    StringRef Clobber = S.getClobber(i)->getString();
 
     if (Clobber != "memory" && Clobber != "cc")
     Clobber = Target.getNormalizedGCCRegisterName(Clobber);
@@ -1577,7 +1588,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
     Constraints += MachineClobbers;
   }
 
-  const llvm::Type *ResultType;
+  llvm::Type *ResultType;
   if (ResultRegTypes.empty())
     ResultType = llvm::Type::getVoidTy(getLLVMContext());
   else if (ResultRegTypes.size() == 1)
@@ -1585,7 +1596,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   else
     ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);
 
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(ResultType, ArgTypes, false);
 
   llvm::InlineAsm *IA =
@@ -1615,7 +1626,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
     // If the result type of the LLVM IR asm doesn't match the result type of
     // the expression, do the conversion.
     if (ResultRegTypes[i] != ResultTruncRegTypes[i]) {
-      const llvm::Type *TruncTy = ResultTruncRegTypes[i];
+      llvm::Type *TruncTy = ResultTruncRegTypes[i];
       
       // Truncate the integer result to the right size, note that TruncTy can be
       // a pointer.
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp
index cec02cdfc235..ea7b8cb49794 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp
@@ -14,383 +14,81 @@
 #include "CodeGenModule.h"
 #include "CGCXXABI.h"
 #include "clang/AST/RecordLayout.h"
+#include "clang/AST/VTTBuilder.h"
 using namespace clang;
 using namespace CodeGen;
 
 #define D1(x)
 
-namespace {
-
-/// VTT builder - Class for building VTT layout information.
-class VTTBuilder {
-  
-  CodeGenModule &CGM;
-
-  /// MostDerivedClass - The most derived class for which we're building this
-  /// vtable.
-  const CXXRecordDecl *MostDerivedClass;
-
-  typedef llvm::SmallVector<llvm::Constant *, 64> VTTComponentsVectorTy;
-  
-  /// VTTComponents - The VTT components.
-  VTTComponentsVectorTy VTTComponents;
-  
-  /// MostDerivedClassLayout - the AST record layout of the most derived class.
-  const ASTRecordLayout &MostDerivedClassLayout;
-
-  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
-
-  typedef llvm::DenseMap<BaseSubobject, uint64_t> AddressPointsMapTy;
-
-  /// SubVTTIndicies - The sub-VTT indices for the bases of the most derived
-  /// class.
-  llvm::DenseMap<BaseSubobject, uint64_t> SubVTTIndicies;
-
-  /// SecondaryVirtualPointerIndices - The secondary virtual pointer indices of
-  /// all subobjects of the most derived class.
-  llvm::DenseMap<BaseSubobject, uint64_t> SecondaryVirtualPointerIndices;
-
-  /// GenerateDefinition - Whether the VTT builder should generate LLVM IR for
-  /// the VTT.
-  bool GenerateDefinition;
-
-  /// The linkage to use for any construction vtables required by this VTT.
-  /// Only required if we're building a definition.
-  llvm::GlobalVariable::LinkageTypes LinkageForConstructionVTables;
-  
-  /// GetAddrOfVTable - Returns the address of the vtable for the base class in
-  /// the given vtable class.
-  ///
-  /// \param AddressPoints - If the returned vtable is a construction vtable,
-  /// this will hold the address points for it.
-  llvm::Constant *GetAddrOfVTable(BaseSubobject Base, bool BaseIsVirtual,
-                                  AddressPointsMapTy& AddressPoints);
-
-  /// AddVTablePointer - Add a vtable pointer to the VTT currently being built.
-  ///
-  /// \param AddressPoints - If the vtable is a construction vtable, this has
-  /// the address points for it.
-  void AddVTablePointer(BaseSubobject Base, llvm::Constant *VTable,
-                        const CXXRecordDecl *VTableClass,
-                        const AddressPointsMapTy& AddressPoints);
-                        
-  /// LayoutSecondaryVTTs - Lay out the secondary VTTs of the given base 
-  /// subobject.
-  void LayoutSecondaryVTTs(BaseSubobject Base);
-  
-  /// LayoutSecondaryVirtualPointers - Lay out the secondary virtual pointers
-  /// for the given base subobject.
-  ///
-  /// \param BaseIsMorallyVirtual whether the base subobject is a virtual base
-  /// or a direct or indirect base of a virtual base.
-  ///
-  /// \param AddressPoints - If the vtable is a construction vtable, this has
-  /// the address points for it.
-  void LayoutSecondaryVirtualPointers(BaseSubobject Base, 
-                                      bool BaseIsMorallyVirtual,
-                                      llvm::Constant *VTable,
-                                      const CXXRecordDecl *VTableClass,
-                                      const AddressPointsMapTy& AddressPoints,
-                                      VisitedVirtualBasesSetTy &VBases);
-  
-  /// LayoutSecondaryVirtualPointers - Lay out the secondary virtual pointers
-  /// for the given base subobject.
-  ///
-  /// \param AddressPoints - If the vtable is a construction vtable, this has
-  /// the address points for it.
-  void LayoutSecondaryVirtualPointers(BaseSubobject Base, 
-                                      llvm::Constant *VTable,
-                                      const AddressPointsMapTy& AddressPoints);
-
-  /// LayoutVirtualVTTs - Lay out the VTTs for the virtual base classes of the
-  /// given record decl.
-  void LayoutVirtualVTTs(const CXXRecordDecl *RD,
-                         VisitedVirtualBasesSetTy &VBases);
-  
-  /// LayoutVTT - Will lay out the VTT for the given subobject, including any
-  /// secondary VTTs, secondary virtual pointers and virtual VTTs.
-  void LayoutVTT(BaseSubobject Base, bool BaseIsVirtual);
-  
-public:
-  VTTBuilder(CodeGenModule &CGM, const CXXRecordDecl *MostDerivedClass,
-             bool GenerateDefinition,
-             llvm::GlobalVariable::LinkageTypes LinkageForConstructionVTables
-               = (llvm::GlobalVariable::LinkageTypes) -1);
-
-  // getVTTComponents - Returns a reference to the VTT components.
-  const VTTComponentsVectorTy &getVTTComponents() const {
-    return VTTComponents;
-  }
-  
-  /// getSubVTTIndicies - Returns a reference to the sub-VTT indices.
-  const llvm::DenseMap<BaseSubobject, uint64_t> &getSubVTTIndicies() const {
-    return SubVTTIndicies;
-  }
-  
-  /// getSecondaryVirtualPointerIndices - Returns a reference to the secondary
-  /// virtual pointer indices.
-  const llvm::DenseMap<BaseSubobject, uint64_t> &
-  getSecondaryVirtualPointerIndices() const {
-    return SecondaryVirtualPointerIndices;
-  }
-
-};
-
-VTTBuilder::VTTBuilder(CodeGenModule &CGM,
-                       const CXXRecordDecl *MostDerivedClass,
-                       bool GenerateDefinition,
-          llvm::GlobalVariable::LinkageTypes LinkageForConstructionVTables)
-  : CGM(CGM), MostDerivedClass(MostDerivedClass), 
-  MostDerivedClassLayout(CGM.getContext().getASTRecordLayout(MostDerivedClass)),
-    GenerateDefinition(GenerateDefinition),
-    LinkageForConstructionVTables(LinkageForConstructionVTables) {
-  assert(!GenerateDefinition ||
-         LinkageForConstructionVTables
-           != (llvm::GlobalVariable::LinkageTypes) -1);
-    
-  // Lay out this VTT.
-  LayoutVTT(BaseSubobject(MostDerivedClass, CharUnits::Zero()), 
-            /*BaseIsVirtual=*/false);
-}
-
-llvm::Constant *
-VTTBuilder::GetAddrOfVTable(BaseSubobject Base, bool BaseIsVirtual, 
-                            AddressPointsMapTy& AddressPoints) {
-  if (!GenerateDefinition)
-    return 0;
-  
-  if (Base.getBase() == MostDerivedClass) {
-    assert(Base.getBaseOffset().isZero() &&
+llvm::Constant *GetAddrOfVTTVTable(CodeGenVTables &CGVT,
+                                   const CXXRecordDecl *MostDerivedClass,
+                                   const VTTVTable &VTable,
+                                   llvm::GlobalVariable::LinkageTypes Linkage,
+                       llvm::DenseMap<BaseSubobject, uint64_t> &AddressPoints) {
+  if (VTable.getBase() == MostDerivedClass) {
+    assert(VTable.getBaseOffset().isZero() &&
            "Most derived class vtable must have a zero offset!");
     // This is a regular vtable.
-    return CGM.getVTables().GetAddrOfVTable(MostDerivedClass);
+    return CGVT.GetAddrOfVTable(MostDerivedClass);
   }
   
-  return CGM.getVTables().GenerateConstructionVTable(MostDerivedClass, 
-                                                     Base, BaseIsVirtual,
-                                           LinkageForConstructionVTables,
-                                                     AddressPoints);
-}
-
-void VTTBuilder::AddVTablePointer(BaseSubobject Base, llvm::Constant *VTable,
-                                  const CXXRecordDecl *VTableClass,
-                                  const AddressPointsMapTy& AddressPoints) {
-  // Store the vtable pointer index if we're generating the primary VTT.
-  if (VTableClass == MostDerivedClass) {
-    assert(!SecondaryVirtualPointerIndices.count(Base) &&
-           "A virtual pointer index already exists for this base subobject!");
-    SecondaryVirtualPointerIndices[Base] = VTTComponents.size();
-  }
-
-  if (!GenerateDefinition) {
-    VTTComponents.push_back(0);
-    return;
-  }
-
-  uint64_t AddressPoint;
-  if (VTableClass != MostDerivedClass) {
-    // The vtable is a construction vtable, look in the construction vtable
-    // address points.
-    AddressPoint = AddressPoints.lookup(Base);
-    assert(AddressPoint != 0 && "Did not find ctor vtable address point!");
-  } else {
-    // Just get the address point for the regular vtable.
-    AddressPoint = CGM.getVTables().getAddressPoint(Base, VTableClass);
-    assert(AddressPoint != 0 && "Did not find vtable address point!");
-  }
-
-  if (!AddressPoint) AddressPoint = 0;
-  
-  llvm::Value *Idxs[] = {
-    llvm::ConstantInt::get(llvm::Type::getInt64Ty(CGM.getLLVMContext()), 0),
-    llvm::ConstantInt::get(llvm::Type::getInt64Ty(CGM.getLLVMContext()), 
-                           AddressPoint)
-  };
-  
-  llvm::Constant *Init = 
-    llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Idxs, 2);
-  
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
-  Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy);
-  
-  VTTComponents.push_back(Init);
-}
-
-void VTTBuilder::LayoutSecondaryVTTs(BaseSubobject Base) {
-  const CXXRecordDecl *RD = Base.getBase();
-
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    
-    // Don't layout virtual bases.
-    if (I->isVirtual())
-        continue;
-
-    const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
-    CharUnits BaseOffset = Base.getBaseOffset() + 
-      Layout.getBaseClassOffset(BaseDecl);
-   
-    // Layout the VTT for this base.
-    LayoutVTT(BaseSubobject(BaseDecl, BaseOffset), /*BaseIsVirtual=*/false);
-  }
-}
-
-void
-VTTBuilder::LayoutSecondaryVirtualPointers(BaseSubobject Base, 
-                                        bool BaseIsMorallyVirtual,
-                                        llvm::Constant *VTable,
-                                        const CXXRecordDecl *VTableClass,
-                                        const AddressPointsMapTy& AddressPoints,
-                                        VisitedVirtualBasesSetTy &VBases) {
-  const CXXRecordDecl *RD = Base.getBase();
-  
-  // We're not interested in bases that don't have virtual bases, and not
-  // morally virtual bases.
-  if (!RD->getNumVBases() && !BaseIsMorallyVirtual)
-    return;
-
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    // Itanium C++ ABI 2.6.2:
-    //   Secondary virtual pointers are present for all bases with either
-    //   virtual bases or virtual function declarations overridden along a 
-    //   virtual path.
-    //
-    // If the base class is not dynamic, we don't want to add it, nor any
-    // of its base classes.
-    if (!BaseDecl->isDynamicClass())
-      continue;
-    
-    bool BaseDeclIsMorallyVirtual = BaseIsMorallyVirtual;
-    bool BaseDeclIsNonVirtualPrimaryBase = false;
-    CharUnits BaseOffset;
-    if (I->isVirtual()) {
-      // Ignore virtual bases that we've already visited.
-      if (!VBases.insert(BaseDecl))
-        continue;
-      
-      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
-      BaseDeclIsMorallyVirtual = true;
-    } else {
-      const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
-      
-      BaseOffset = Base.getBaseOffset() + 
-        Layout.getBaseClassOffset(BaseDecl);
-      
-      if (!Layout.isPrimaryBaseVirtual() &&
-          Layout.getPrimaryBase() == BaseDecl)
-        BaseDeclIsNonVirtualPrimaryBase = true;
-    }
-
-    // Itanium C++ ABI 2.6.2:
-    //   Secondary virtual pointers: for each base class X which (a) has virtual
-    //   bases or is reachable along a virtual path from D, and (b) is not a
-    //   non-virtual primary base, the address of the virtual table for X-in-D
-    //   or an appropriate construction virtual table.
-    if (!BaseDeclIsNonVirtualPrimaryBase &&
-        (BaseDecl->getNumVBases() || BaseDeclIsMorallyVirtual)) {
-      // Add the vtable pointer.
-      AddVTablePointer(BaseSubobject(BaseDecl, BaseOffset), VTable, 
-                       VTableClass, AddressPoints);
-    }
-
-    // And lay out the secondary virtual pointers for the base class.
-    LayoutSecondaryVirtualPointers(BaseSubobject(BaseDecl, BaseOffset),
-                                   BaseDeclIsMorallyVirtual, VTable, 
-                                   VTableClass, AddressPoints, VBases);
-  }
-}
-
-void 
-VTTBuilder::LayoutSecondaryVirtualPointers(BaseSubobject Base, 
-                                      llvm::Constant *VTable,
-                                      const AddressPointsMapTy& AddressPoints) {
-  VisitedVirtualBasesSetTy VBases;
-  LayoutSecondaryVirtualPointers(Base, /*BaseIsMorallyVirtual=*/false,
-                                 VTable, Base.getBase(), AddressPoints, VBases);
-}
-
-void VTTBuilder::LayoutVirtualVTTs(const CXXRecordDecl *RD,
-                                   VisitedVirtualBasesSetTy &VBases) {
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-    
-    // Check if this is a virtual base.
-    if (I->isVirtual()) {
-      // Check if we've seen this base before.
-      if (!VBases.insert(BaseDecl))
-        continue;
-    
-      CharUnits BaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
-      
-      LayoutVTT(BaseSubobject(BaseDecl, BaseOffset), /*BaseIsVirtual=*/true);
-    }
-    
-    // We only need to layout virtual VTTs for this base if it actually has
-    // virtual bases.
-    if (BaseDecl->getNumVBases())
-      LayoutVirtualVTTs(BaseDecl, VBases);
-  }
-}
-
-void VTTBuilder::LayoutVTT(BaseSubobject Base, bool BaseIsVirtual) {
-  const CXXRecordDecl *RD = Base.getBase();
-
-  // Itanium C++ ABI 2.6.2:
-  //   An array of virtual table addresses, called the VTT, is declared for 
-  //   each class type that has indirect or direct virtual base classes.
-  if (RD->getNumVBases() == 0)
-    return;
-
-  bool IsPrimaryVTT = Base.getBase() == MostDerivedClass;
-
-  if (!IsPrimaryVTT) {
-    // Remember the sub-VTT index.
-    SubVTTIndicies[Base] = VTTComponents.size();
-  }
-
-  AddressPointsMapTy AddressPoints;
-  llvm::Constant *VTable = GetAddrOfVTable(Base, BaseIsVirtual, AddressPoints);
-
-  // Add the primary vtable pointer.
-  AddVTablePointer(Base, VTable, RD, AddressPoints);
-
-  // Add the secondary VTTs.
-  LayoutSecondaryVTTs(Base);
-  
-  // Add the secondary virtual pointers.
-  LayoutSecondaryVirtualPointers(Base, VTable, AddressPoints);
-  
-  // If this is the primary VTT, we want to lay out virtual VTTs as well.
-  if (IsPrimaryVTT) {
-    VisitedVirtualBasesSetTy VBases;
-    LayoutVirtualVTTs(Base.getBase(), VBases);
-  }
-}
-  
+  return CGVT.GenerateConstructionVTable(MostDerivedClass, 
+                                         VTable.getBaseSubobject(),
+                                         VTable.isVirtual(),
+                                         Linkage,
+                                         AddressPoints);
 }
 
 void
 CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT,
                                   llvm::GlobalVariable::LinkageTypes Linkage,
                                   const CXXRecordDecl *RD) {
-  VTTBuilder Builder(CGM, RD, /*GenerateDefinition=*/true, Linkage);
+  VTTBuilder Builder(CGM.getContext(), RD, /*GenerateDefinition=*/true);
 
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
-  const llvm::ArrayType *ArrayType = 
+  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext()),
+             *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
+  llvm::ArrayType *ArrayType = 
     llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size());
   
-  llvm::Constant *Init = 
-    llvm::ConstantArray::get(ArrayType, Builder.getVTTComponents());
+  SmallVector<llvm::Constant *, 8> VTables;
+  SmallVector<VTableAddressPointsMapTy, 8> VTableAddressPoints;
+  for (const VTTVTable *i = Builder.getVTTVTables().begin(),
+                       *e = Builder.getVTTVTables().end(); i != e; ++i) {
+    VTableAddressPoints.push_back(VTableAddressPointsMapTy());
+    VTables.push_back(GetAddrOfVTTVTable(*this, RD, *i, Linkage,
+                                         VTableAddressPoints.back()));
+  }
+
+  SmallVector<llvm::Constant *, 8> VTTComponents;
+  for (const VTTComponent *i = Builder.getVTTComponents().begin(),
+                          *e = Builder.getVTTComponents().end(); i != e; ++i) {
+    const VTTVTable &VTTVT = Builder.getVTTVTables()[i->VTableIndex];
+    llvm::Constant *VTable = VTables[i->VTableIndex];
+    uint64_t AddressPoint;
+    if (VTTVT.getBase() == RD) {
+      // Just get the address point for the regular vtable.
+      AddressPoint = VTContext.getVTableLayout(RD)
+                              .getAddressPoint(i->VTableBase);
+      assert(AddressPoint != 0 && "Did not find vtable address point!");
+    } else {
+      AddressPoint = VTableAddressPoints[i->VTableIndex].lookup(i->VTableBase);
+      assert(AddressPoint != 0 && "Did not find ctor vtable address point!");
+    }
+
+     llvm::Value *Idxs[] = {
+       llvm::ConstantInt::get(Int64Ty, 0),
+       llvm::ConstantInt::get(Int64Ty, AddressPoint)
+     };
+
+     llvm::Constant *Init = 
+       llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Idxs);
+
+     Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy);
+
+     VTTComponents.push_back(Init);
+  }
+
+  llvm::Constant *Init = llvm::ConstantArray::get(ArrayType, VTTComponents);
 
   VTT->setInitializer(Init);
 
@@ -408,15 +106,16 @@ llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTT(const CXXRecordDecl *RD) {
   llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().mangleCXXVTT(RD, Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
-  ComputeVTableRelatedInformation(RD, /*VTableRequired=*/true);
+  // This will also defer the definition of the VTT.
+  (void) GetAddrOfVTable(RD);
 
-  VTTBuilder Builder(CGM, RD, /*GenerateDefinition=*/false);
+  VTTBuilder Builder(CGM.getContext(), RD, /*GenerateDefinition=*/false);
 
-  const llvm::Type *Int8PtrTy = 
+  llvm::Type *Int8PtrTy = 
     llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
-  const llvm::ArrayType *ArrayType = 
+  llvm::ArrayType *ArrayType = 
     llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size());
 
   llvm::GlobalVariable *GV =
@@ -452,7 +151,7 @@ uint64_t CodeGenVTables::getSubVTTIndex(const CXXRecordDecl *RD,
   if (I != SubVTTIndicies.end())
     return I->second;
   
-  VTTBuilder Builder(CGM, RD, /*GenerateDefinition=*/false);
+  VTTBuilder Builder(CGM.getContext(), RD, /*GenerateDefinition=*/false);
 
   for (llvm::DenseMap<BaseSubobject, uint64_t>::const_iterator I =
        Builder.getSubVTTIndicies().begin(), 
@@ -478,7 +177,7 @@ CodeGenVTables::getSecondaryVirtualPointerIndex(const CXXRecordDecl *RD,
   if (I != SecondaryVirtualPointerIndices.end())
     return I->second;
 
-  VTTBuilder Builder(CGM, RD, /*GenerateDefinition=*/false);
+  VTTBuilder Builder(CGM.getContext(), RD, /*GenerateDefinition=*/false);
 
   // Insert all secondary vpointer indices.
   for (llvm::DenseMap<BaseSubobject, uint64_t>::const_iterator I = 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp
index c161b79fd3a0..a306c857e5aa 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp
@@ -28,2399 +28,8 @@
 using namespace clang;
 using namespace CodeGen;
 
-namespace {
-
-/// BaseOffset - Represents an offset from a derived class to a direct or
-/// indirect base class.
-struct BaseOffset {
-  /// DerivedClass - The derived class.
-  const CXXRecordDecl *DerivedClass;
-  
-  /// VirtualBase - If the path from the derived class to the base class
-  /// involves a virtual base class, this holds its declaration.
-  const CXXRecordDecl *VirtualBase;
-
-  /// NonVirtualOffset - The offset from the derived class to the base class.
-  /// (Or the offset from the virtual base class to the base class, if the 
-  /// path from the derived class to the base class involves a virtual base
-  /// class.
-  CharUnits NonVirtualOffset;
-  
-  BaseOffset() : DerivedClass(0), VirtualBase(0), 
-    NonVirtualOffset(CharUnits::Zero()) { }
-  BaseOffset(const CXXRecordDecl *DerivedClass,
-             const CXXRecordDecl *VirtualBase, CharUnits NonVirtualOffset)
-    : DerivedClass(DerivedClass), VirtualBase(VirtualBase), 
-    NonVirtualOffset(NonVirtualOffset) { }
-
-  bool isEmpty() const { return NonVirtualOffset.isZero() && !VirtualBase; }
-};
-
-/// FinalOverriders - Contains the final overrider member functions for all
-/// member functions in the base subobjects of a class.
-class FinalOverriders {
-public:
-  /// OverriderInfo - Information about a final overrider.
-  struct OverriderInfo {
-    /// Method - The method decl of the overrider.
-    const CXXMethodDecl *Method;
-
-    /// Offset - the base offset of the overrider in the layout class.
-    CharUnits Offset;
-    
-    OverriderInfo() : Method(0), Offset(CharUnits::Zero()) { }
-  };
-
-private:
-  /// MostDerivedClass - The most derived class for which the final overriders
-  /// are stored.
-  const CXXRecordDecl *MostDerivedClass;
-  
-  /// MostDerivedClassOffset - If we're building final overriders for a 
-  /// construction vtable, this holds the offset from the layout class to the
-  /// most derived class.
-  const CharUnits MostDerivedClassOffset;
-
-  /// LayoutClass - The class we're using for layout information. Will be 
-  /// different than the most derived class if the final overriders are for a
-  /// construction vtable.  
-  const CXXRecordDecl *LayoutClass;  
-
-  ASTContext &Context;
-  
-  /// MostDerivedClassLayout - the AST record layout of the most derived class.
-  const ASTRecordLayout &MostDerivedClassLayout;
-
-  /// MethodBaseOffsetPairTy - Uniquely identifies a member function
-  /// in a base subobject.
-  typedef std::pair<const CXXMethodDecl *, CharUnits> MethodBaseOffsetPairTy;
-
-  typedef llvm::DenseMap<MethodBaseOffsetPairTy,
-                         OverriderInfo> OverridersMapTy;
-  
-  /// OverridersMap - The final overriders for all virtual member functions of 
-  /// all the base subobjects of the most derived class.
-  OverridersMapTy OverridersMap;
-  
-  /// SubobjectsToOffsetsMapTy - A mapping from a base subobject (represented
-  /// as a record decl and a subobject number) and its offsets in the most
-  /// derived class as well as the layout class.
-  typedef llvm::DenseMap<std::pair<const CXXRecordDecl *, unsigned>, 
-                         CharUnits> SubobjectOffsetMapTy;
-
-  typedef llvm::DenseMap<const CXXRecordDecl *, unsigned> SubobjectCountMapTy;
-  
-  /// ComputeBaseOffsets - Compute the offsets for all base subobjects of the
-  /// given base.
-  void ComputeBaseOffsets(BaseSubobject Base, bool IsVirtual,
-                          CharUnits OffsetInLayoutClass,
-                          SubobjectOffsetMapTy &SubobjectOffsets,
-                          SubobjectOffsetMapTy &SubobjectLayoutClassOffsets,
-                          SubobjectCountMapTy &SubobjectCounts);
-
-  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
-  
-  /// dump - dump the final overriders for a base subobject, and all its direct
-  /// and indirect base subobjects.
-  void dump(llvm::raw_ostream &Out, BaseSubobject Base,
-            VisitedVirtualBasesSetTy& VisitedVirtualBases);
-  
-public:
-  FinalOverriders(const CXXRecordDecl *MostDerivedClass,
-                  CharUnits MostDerivedClassOffset,
-                  const CXXRecordDecl *LayoutClass);
-
-  /// getOverrider - Get the final overrider for the given method declaration in
-  /// the subobject with the given base offset. 
-  OverriderInfo getOverrider(const CXXMethodDecl *MD, 
-                             CharUnits BaseOffset) const {
-    assert(OverridersMap.count(std::make_pair(MD, BaseOffset)) && 
-           "Did not find overrider!");
-    
-    return OverridersMap.lookup(std::make_pair(MD, BaseOffset));
-  }
-  
-  /// dump - dump the final overriders.
-  void dump() {
-    VisitedVirtualBasesSetTy VisitedVirtualBases;
-    dump(llvm::errs(), BaseSubobject(MostDerivedClass, CharUnits::Zero()), 
-         VisitedVirtualBases);
-  }
-  
-};
-
-#define DUMP_OVERRIDERS 0
-
-FinalOverriders::FinalOverriders(const CXXRecordDecl *MostDerivedClass,
-                                 CharUnits MostDerivedClassOffset,
-                                 const CXXRecordDecl *LayoutClass)
-  : MostDerivedClass(MostDerivedClass), 
-  MostDerivedClassOffset(MostDerivedClassOffset), LayoutClass(LayoutClass),
-  Context(MostDerivedClass->getASTContext()),
-  MostDerivedClassLayout(Context.getASTRecordLayout(MostDerivedClass)) {
-
-  // Compute base offsets.
-  SubobjectOffsetMapTy SubobjectOffsets;
-  SubobjectOffsetMapTy SubobjectLayoutClassOffsets;
-  SubobjectCountMapTy SubobjectCounts;
-  ComputeBaseOffsets(BaseSubobject(MostDerivedClass, CharUnits::Zero()), 
-                     /*IsVirtual=*/false,
-                     MostDerivedClassOffset, 
-                     SubobjectOffsets, SubobjectLayoutClassOffsets, 
-                     SubobjectCounts);
-
-  // Get the the final overriders.
-  CXXFinalOverriderMap FinalOverriders;
-  MostDerivedClass->getFinalOverriders(FinalOverriders);
-
-  for (CXXFinalOverriderMap::const_iterator I = FinalOverriders.begin(),
-       E = FinalOverriders.end(); I != E; ++I) {
-    const CXXMethodDecl *MD = I->first;
-    const OverridingMethods& Methods = I->second;
-
-    for (OverridingMethods::const_iterator I = Methods.begin(),
-         E = Methods.end(); I != E; ++I) {
-      unsigned SubobjectNumber = I->first;
-      assert(SubobjectOffsets.count(std::make_pair(MD->getParent(), 
-                                                   SubobjectNumber)) &&
-             "Did not find subobject offset!");
-      
-      CharUnits BaseOffset = SubobjectOffsets[std::make_pair(MD->getParent(),
-                                                            SubobjectNumber)];
-
-      assert(I->second.size() == 1 && "Final overrider is not unique!");
-      const UniqueVirtualMethod &Method = I->second.front();
-
-      const CXXRecordDecl *OverriderRD = Method.Method->getParent();
-      assert(SubobjectLayoutClassOffsets.count(
-             std::make_pair(OverriderRD, Method.Subobject))
-             && "Did not find subobject offset!");
-      CharUnits OverriderOffset =
-        SubobjectLayoutClassOffsets[std::make_pair(OverriderRD, 
-                                                   Method.Subobject)];
-
-      OverriderInfo& Overrider = OverridersMap[std::make_pair(MD, BaseOffset)];
-      assert(!Overrider.Method && "Overrider should not exist yet!");
-      
-      Overrider.Offset = OverriderOffset;
-      Overrider.Method = Method.Method;
-    }
-  }
-
-#if DUMP_OVERRIDERS
-  // And dump them (for now).
-  dump();
-#endif
-}
-
-static BaseOffset ComputeBaseOffset(ASTContext &Context, 
-                                    const CXXRecordDecl *DerivedRD,
-                                    const CXXBasePath &Path) {
-  CharUnits NonVirtualOffset = CharUnits::Zero();
-
-  unsigned NonVirtualStart = 0;
-  const CXXRecordDecl *VirtualBase = 0;
-  
-  // First, look for the virtual base class.
-  for (unsigned I = 0, E = Path.size(); I != E; ++I) {
-    const CXXBasePathElement &Element = Path[I];
-    
-    if (Element.Base->isVirtual()) {
-      // FIXME: Can we break when we find the first virtual base?
-      // (If we can't, can't we just iterate over the path in reverse order?)
-      NonVirtualStart = I + 1;
-      QualType VBaseType = Element.Base->getType();
-      VirtualBase = 
-        cast<CXXRecordDecl>(VBaseType->getAs<RecordType>()->getDecl());
-    }
-  }
-  
-  // Now compute the non-virtual offset.
-  for (unsigned I = NonVirtualStart, E = Path.size(); I != E; ++I) {
-    const CXXBasePathElement &Element = Path[I];
-    
-    // Check the base class offset.
-    const ASTRecordLayout &Layout = Context.getASTRecordLayout(Element.Class);
-
-    const RecordType *BaseType = Element.Base->getType()->getAs<RecordType>();
-    const CXXRecordDecl *Base = cast<CXXRecordDecl>(BaseType->getDecl());
-
-    NonVirtualOffset += Layout.getBaseClassOffset(Base);
-  }
-  
-  // FIXME: This should probably use CharUnits or something. Maybe we should
-  // even change the base offsets in ASTRecordLayout to be specified in 
-  // CharUnits.
-  return BaseOffset(DerivedRD, VirtualBase, NonVirtualOffset);
-  
-}
-
-static BaseOffset ComputeBaseOffset(ASTContext &Context, 
-                                    const CXXRecordDecl *BaseRD,
-                                    const CXXRecordDecl *DerivedRD) {
-  CXXBasePaths Paths(/*FindAmbiguities=*/false,
-                     /*RecordPaths=*/true, /*DetectVirtual=*/false);
-  
-  if (!const_cast<CXXRecordDecl *>(DerivedRD)->
-      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseRD), Paths)) {
-    assert(false && "Class must be derived from the passed in base class!");
-    return BaseOffset();
-  }
-
-  return ComputeBaseOffset(Context, DerivedRD, Paths.front());
-}
-
-static BaseOffset
-ComputeReturnAdjustmentBaseOffset(ASTContext &Context, 
-                                  const CXXMethodDecl *DerivedMD,
-                                  const CXXMethodDecl *BaseMD) {
-  const FunctionType *BaseFT = BaseMD->getType()->getAs<FunctionType>();
-  const FunctionType *DerivedFT = DerivedMD->getType()->getAs<FunctionType>();
-  
-  // Canonicalize the return types.
-  CanQualType CanDerivedReturnType = 
-    Context.getCanonicalType(DerivedFT->getResultType());
-  CanQualType CanBaseReturnType = 
-    Context.getCanonicalType(BaseFT->getResultType());
-  
-  assert(CanDerivedReturnType->getTypeClass() == 
-         CanBaseReturnType->getTypeClass() && 
-         "Types must have same type class!");
-  
-  if (CanDerivedReturnType == CanBaseReturnType) {
-    // No adjustment needed.
-    return BaseOffset();
-  }
-  
-  if (isa<ReferenceType>(CanDerivedReturnType)) {
-    CanDerivedReturnType = 
-      CanDerivedReturnType->getAs<ReferenceType>()->getPointeeType();
-    CanBaseReturnType = 
-      CanBaseReturnType->getAs<ReferenceType>()->getPointeeType();
-  } else if (isa<PointerType>(CanDerivedReturnType)) {
-    CanDerivedReturnType = 
-      CanDerivedReturnType->getAs<PointerType>()->getPointeeType();
-    CanBaseReturnType = 
-      CanBaseReturnType->getAs<PointerType>()->getPointeeType();
-  } else {
-    assert(false && "Unexpected return type!");
-  }
-  
-  // We need to compare unqualified types here; consider
-  //   const T *Base::foo();
-  //   T *Derived::foo();
-  if (CanDerivedReturnType.getUnqualifiedType() == 
-      CanBaseReturnType.getUnqualifiedType()) {
-    // No adjustment needed.
-    return BaseOffset();
-  }
-  
-  const CXXRecordDecl *DerivedRD = 
-    cast<CXXRecordDecl>(cast<RecordType>(CanDerivedReturnType)->getDecl());
-  
-  const CXXRecordDecl *BaseRD = 
-    cast<CXXRecordDecl>(cast<RecordType>(CanBaseReturnType)->getDecl());
-
-  return ComputeBaseOffset(Context, BaseRD, DerivedRD);
-}
-
-void 
-FinalOverriders::ComputeBaseOffsets(BaseSubobject Base, bool IsVirtual,
-                              CharUnits OffsetInLayoutClass,
-                              SubobjectOffsetMapTy &SubobjectOffsets,
-                              SubobjectOffsetMapTy &SubobjectLayoutClassOffsets,
-                              SubobjectCountMapTy &SubobjectCounts) {
-  const CXXRecordDecl *RD = Base.getBase();
-  
-  unsigned SubobjectNumber = 0;
-  if (!IsVirtual)
-    SubobjectNumber = ++SubobjectCounts[RD];
-
-  // Set up the subobject to offset mapping.
-  assert(!SubobjectOffsets.count(std::make_pair(RD, SubobjectNumber))
-         && "Subobject offset already exists!");
-  assert(!SubobjectLayoutClassOffsets.count(std::make_pair(RD, SubobjectNumber)) 
-         && "Subobject offset already exists!");
-
-  SubobjectOffsets[std::make_pair(RD, SubobjectNumber)] = Base.getBaseOffset();
-  SubobjectLayoutClassOffsets[std::make_pair(RD, SubobjectNumber)] =
-    OffsetInLayoutClass;
-  
-  // Traverse our bases.
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    CharUnits BaseOffset;
-    CharUnits BaseOffsetInLayoutClass;
-    if (I->isVirtual()) {
-      // Check if we've visited this virtual base before.
-      if (SubobjectOffsets.count(std::make_pair(BaseDecl, 0)))
-        continue;
-
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-
-      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
-      BaseOffsetInLayoutClass = 
-        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
-    } else {
-      const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-      CharUnits Offset = Layout.getBaseClassOffset(BaseDecl);
-    
-      BaseOffset = Base.getBaseOffset() + Offset;
-      BaseOffsetInLayoutClass = OffsetInLayoutClass + Offset;
-    }
-
-    ComputeBaseOffsets(BaseSubobject(BaseDecl, BaseOffset), 
-                       I->isVirtual(), BaseOffsetInLayoutClass, 
-                       SubobjectOffsets, SubobjectLayoutClassOffsets, 
-                       SubobjectCounts);
-  }
-}
-
-void FinalOverriders::dump(llvm::raw_ostream &Out, BaseSubobject Base,
-                           VisitedVirtualBasesSetTy &VisitedVirtualBases) {
-  const CXXRecordDecl *RD = Base.getBase();
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-    
-    // Ignore bases that don't have any virtual member functions.
-    if (!BaseDecl->isPolymorphic())
-      continue;
-
-    CharUnits BaseOffset;
-    if (I->isVirtual()) {
-      if (!VisitedVirtualBases.insert(BaseDecl)) {
-        // We've visited this base before.
-        continue;
-      }
-      
-      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
-    } else {
-      BaseOffset = Layout.getBaseClassOffset(BaseDecl) + Base.getBaseOffset();
-    }
-
-    dump(Out, BaseSubobject(BaseDecl, BaseOffset), VisitedVirtualBases);
-  }
-
-  Out << "Final overriders for (" << RD->getQualifiedNameAsString() << ", ";
-  Out << Base.getBaseOffset().getQuantity() << ")\n";
-
-  // Now dump the overriders for this base subobject.
-  for (CXXRecordDecl::method_iterator I = RD->method_begin(), 
-       E = RD->method_end(); I != E; ++I) {
-    const CXXMethodDecl *MD = *I;
-
-    if (!MD->isVirtual())
-      continue;
-  
-    OverriderInfo Overrider = getOverrider(MD, Base.getBaseOffset());
-
-    Out << "  " << MD->getQualifiedNameAsString() << " - (";
-    Out << Overrider.Method->getQualifiedNameAsString();
-    Out << ", " << ", " << Overrider.Offset.getQuantity() << ')';
-
-    BaseOffset Offset;
-    if (!Overrider.Method->isPure())
-      Offset = ComputeReturnAdjustmentBaseOffset(Context, Overrider.Method, MD);
-
-    if (!Offset.isEmpty()) {
-      Out << " [ret-adj: ";
-      if (Offset.VirtualBase)
-        Out << Offset.VirtualBase->getQualifiedNameAsString() << " vbase, ";
-             
-      Out << Offset.NonVirtualOffset.getQuantity() << " nv]";
-    }
-    
-    Out << "\n";
-  }  
-}
-
-/// VTableComponent - Represents a single component in a vtable.
-class VTableComponent {
-public:
-  enum Kind {
-    CK_VCallOffset,
-    CK_VBaseOffset,
-    CK_OffsetToTop,
-    CK_RTTI,
-    CK_FunctionPointer,
-    
-    /// CK_CompleteDtorPointer - A pointer to the complete destructor.
-    CK_CompleteDtorPointer,
-    
-    /// CK_DeletingDtorPointer - A pointer to the deleting destructor.
-    CK_DeletingDtorPointer,
-    
-    /// CK_UnusedFunctionPointer - In some cases, a vtable function pointer
-    /// will end up never being called. Such vtable function pointers are
-    /// represented as a CK_UnusedFunctionPointer. 
-    CK_UnusedFunctionPointer
-  };
-
-  static VTableComponent MakeVCallOffset(CharUnits Offset) {
-    return VTableComponent(CK_VCallOffset, Offset);
-  }
-
-  static VTableComponent MakeVBaseOffset(CharUnits Offset) {
-    return VTableComponent(CK_VBaseOffset, Offset);
-  }
-
-  static VTableComponent MakeOffsetToTop(CharUnits Offset) {
-    return VTableComponent(CK_OffsetToTop, Offset);
-  }
-  
-  static VTableComponent MakeRTTI(const CXXRecordDecl *RD) {
-    return VTableComponent(CK_RTTI, reinterpret_cast<uintptr_t>(RD));
-  }
-
-  static VTableComponent MakeFunction(const CXXMethodDecl *MD) {
-    assert(!isa<CXXDestructorDecl>(MD) && 
-           "Don't use MakeFunction with destructors!");
-
-    return VTableComponent(CK_FunctionPointer, 
-                           reinterpret_cast<uintptr_t>(MD));
-  }
-  
-  static VTableComponent MakeCompleteDtor(const CXXDestructorDecl *DD) {
-    return VTableComponent(CK_CompleteDtorPointer,
-                           reinterpret_cast<uintptr_t>(DD));
-  }
-
-  static VTableComponent MakeDeletingDtor(const CXXDestructorDecl *DD) {
-    return VTableComponent(CK_DeletingDtorPointer, 
-                           reinterpret_cast<uintptr_t>(DD));
-  }
-
-  static VTableComponent MakeUnusedFunction(const CXXMethodDecl *MD) {
-    assert(!isa<CXXDestructorDecl>(MD) && 
-           "Don't use MakeUnusedFunction with destructors!");
-    return VTableComponent(CK_UnusedFunctionPointer,
-                           reinterpret_cast<uintptr_t>(MD));                           
-  }
-
-  static VTableComponent getFromOpaqueInteger(uint64_t I) {
-    return VTableComponent(I);
-  }
-
-  /// getKind - Get the kind of this vtable component.
-  Kind getKind() const {
-    return (Kind)(Value & 0x7);
-  }
-
-  CharUnits getVCallOffset() const {
-    assert(getKind() == CK_VCallOffset && "Invalid component kind!");
-    
-    return getOffset();
-  }
-
-  CharUnits getVBaseOffset() const {
-    assert(getKind() == CK_VBaseOffset && "Invalid component kind!");
-    
-    return getOffset();
-  }
-
-  CharUnits getOffsetToTop() const {
-    assert(getKind() == CK_OffsetToTop && "Invalid component kind!");
-    
-    return getOffset();
-  }
-  
-  const CXXRecordDecl *getRTTIDecl() const {
-    assert(getKind() == CK_RTTI && "Invalid component kind!");
-    
-    return reinterpret_cast<CXXRecordDecl *>(getPointer());
-  }
-  
-  const CXXMethodDecl *getFunctionDecl() const {
-    assert(getKind() == CK_FunctionPointer);
-    
-    return reinterpret_cast<CXXMethodDecl *>(getPointer());
-  }
-
-  const CXXDestructorDecl *getDestructorDecl() const {
-    assert((getKind() == CK_CompleteDtorPointer ||
-            getKind() == CK_DeletingDtorPointer) && "Invalid component kind!");
-    
-    return reinterpret_cast<CXXDestructorDecl *>(getPointer());
-  }
-
-  const CXXMethodDecl *getUnusedFunctionDecl() const {
-    assert(getKind() == CK_UnusedFunctionPointer);
-    
-    return reinterpret_cast<CXXMethodDecl *>(getPointer());
-  }
-  
-private:
-  VTableComponent(Kind ComponentKind, CharUnits Offset) {
-    assert((ComponentKind == CK_VCallOffset || 
-            ComponentKind == CK_VBaseOffset ||
-            ComponentKind == CK_OffsetToTop) && "Invalid component kind!");
-    assert(Offset.getQuantity() <= ((1LL << 56) - 1) && "Offset is too big!");
-    
-    Value = ((Offset.getQuantity() << 3) | ComponentKind);
-  }
-
-  VTableComponent(Kind ComponentKind, uintptr_t Ptr) {
-    assert((ComponentKind == CK_RTTI || 
-            ComponentKind == CK_FunctionPointer ||
-            ComponentKind == CK_CompleteDtorPointer ||
-            ComponentKind == CK_DeletingDtorPointer ||
-            ComponentKind == CK_UnusedFunctionPointer) &&
-            "Invalid component kind!");
-    
-    assert((Ptr & 7) == 0 && "Pointer not sufficiently aligned!");
-    
-    Value = Ptr | ComponentKind;
-  }
-  
-  CharUnits getOffset() const {
-    assert((getKind() == CK_VCallOffset || getKind() == CK_VBaseOffset ||
-            getKind() == CK_OffsetToTop) && "Invalid component kind!");
-    
-    return CharUnits::fromQuantity(Value >> 3);
-  }
-
-  uintptr_t getPointer() const {
-    assert((getKind() == CK_RTTI || 
-            getKind() == CK_FunctionPointer ||
-            getKind() == CK_CompleteDtorPointer ||
-            getKind() == CK_DeletingDtorPointer ||
-            getKind() == CK_UnusedFunctionPointer) &&
-           "Invalid component kind!");
-    
-    return static_cast<uintptr_t>(Value & ~7ULL);
-  }
-  
-  explicit VTableComponent(uint64_t Value)
-    : Value(Value) { }
-
-  /// The kind is stored in the lower 3 bits of the value. For offsets, we
-  /// make use of the facts that classes can't be larger than 2^55 bytes,
-  /// so we store the offset in the lower part of the 61 bytes that remain.
-  /// (The reason that we're not simply using a PointerIntPair here is that we
-  /// need the offsets to be 64-bit, even when on a 32-bit machine).
-  int64_t Value;
-};
-
-/// VCallOffsetMap - Keeps track of vcall offsets when building a vtable.
-struct VCallOffsetMap {
-  
-  typedef std::pair<const CXXMethodDecl *, CharUnits> MethodAndOffsetPairTy;
-  
-  /// Offsets - Keeps track of methods and their offsets.
-  // FIXME: This should be a real map and not a vector.
-  llvm::SmallVector<MethodAndOffsetPairTy, 16> Offsets;
-
-  /// MethodsCanShareVCallOffset - Returns whether two virtual member functions
-  /// can share the same vcall offset.
-  static bool MethodsCanShareVCallOffset(const CXXMethodDecl *LHS,
-                                         const CXXMethodDecl *RHS);
-
-public:
-  /// AddVCallOffset - Adds a vcall offset to the map. Returns true if the
-  /// add was successful, or false if there was already a member function with
-  /// the same signature in the map.
-  bool AddVCallOffset(const CXXMethodDecl *MD, CharUnits OffsetOffset);
-  
-  /// getVCallOffsetOffset - Returns the vcall offset offset (relative to the
-  /// vtable address point) for the given virtual member function.
-  CharUnits getVCallOffsetOffset(const CXXMethodDecl *MD);
-  
-  // empty - Return whether the offset map is empty or not.
-  bool empty() const { return Offsets.empty(); }
-};
-
-static bool HasSameVirtualSignature(const CXXMethodDecl *LHS,
-                                    const CXXMethodDecl *RHS) {
-  ASTContext &C = LHS->getASTContext(); // TODO: thread this down
-  CanQual<FunctionProtoType>
-    LT = C.getCanonicalType(LHS->getType()).getAs<FunctionProtoType>(),
-    RT = C.getCanonicalType(RHS->getType()).getAs<FunctionProtoType>();
-
-  // Fast-path matches in the canonical types.
-  if (LT == RT) return true;
-
-  // Force the signatures to match.  We can't rely on the overrides
-  // list here because there isn't necessarily an inheritance
-  // relationship between the two methods.
-  if (LT.getQualifiers() != RT.getQualifiers() ||
-      LT->getNumArgs() != RT->getNumArgs())
-    return false;
-  for (unsigned I = 0, E = LT->getNumArgs(); I != E; ++I)
-    if (LT->getArgType(I) != RT->getArgType(I))
-      return false;
-  return true;
-}
-
-bool VCallOffsetMap::MethodsCanShareVCallOffset(const CXXMethodDecl *LHS,
-                                                const CXXMethodDecl *RHS) {
-  assert(LHS->isVirtual() && "LHS must be virtual!");
-  assert(RHS->isVirtual() && "LHS must be virtual!");
-  
-  // A destructor can share a vcall offset with another destructor.
-  if (isa<CXXDestructorDecl>(LHS))
-    return isa<CXXDestructorDecl>(RHS);
-
-  // FIXME: We need to check more things here.
-  
-  // The methods must have the same name.
-  DeclarationName LHSName = LHS->getDeclName();
-  DeclarationName RHSName = RHS->getDeclName();
-  if (LHSName != RHSName)
-    return false;
-
-  // And the same signatures.
-  return HasSameVirtualSignature(LHS, RHS);
-}
-
-bool VCallOffsetMap::AddVCallOffset(const CXXMethodDecl *MD, 
-                                    CharUnits OffsetOffset) {
-  // Check if we can reuse an offset.
-  for (unsigned I = 0, E = Offsets.size(); I != E; ++I) {
-    if (MethodsCanShareVCallOffset(Offsets[I].first, MD))
-      return false;
-  }
-  
-  // Add the offset.
-  Offsets.push_back(MethodAndOffsetPairTy(MD, OffsetOffset));
-  return true;
-}
-
-CharUnits VCallOffsetMap::getVCallOffsetOffset(const CXXMethodDecl *MD) {
-  // Look for an offset.
-  for (unsigned I = 0, E = Offsets.size(); I != E; ++I) {
-    if (MethodsCanShareVCallOffset(Offsets[I].first, MD))
-      return Offsets[I].second;
-  }
-  
-  assert(false && "Should always find a vcall offset offset!");
-  return CharUnits::Zero();
-}
-
-/// VCallAndVBaseOffsetBuilder - Class for building vcall and vbase offsets.
-class VCallAndVBaseOffsetBuilder {
-public:
-  typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> 
-    VBaseOffsetOffsetsMapTy;
-
-private:
-  /// MostDerivedClass - The most derived class for which we're building vcall
-  /// and vbase offsets.
-  const CXXRecordDecl *MostDerivedClass;
-  
-  /// LayoutClass - The class we're using for layout information. Will be 
-  /// different than the most derived class if we're building a construction
-  /// vtable.
-  const CXXRecordDecl *LayoutClass;
-  
-  /// Context - The ASTContext which we will use for layout information.
-  ASTContext &Context;
-
-  /// Components - vcall and vbase offset components
-  typedef llvm::SmallVector<VTableComponent, 64> VTableComponentVectorTy;
-  VTableComponentVectorTy Components;
-  
-  /// VisitedVirtualBases - Visited virtual bases.
-  llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBases;
-  
-  /// VCallOffsets - Keeps track of vcall offsets.
-  VCallOffsetMap VCallOffsets;
-
-
-  /// VBaseOffsetOffsets - Contains the offsets of the virtual base offsets,
-  /// relative to the address point.
-  VBaseOffsetOffsetsMapTy VBaseOffsetOffsets;
-  
-  /// FinalOverriders - The final overriders of the most derived class.
-  /// (Can be null when we're not building a vtable of the most derived class).
-  const FinalOverriders *Overriders;
-
-  /// AddVCallAndVBaseOffsets - Add vcall offsets and vbase offsets for the
-  /// given base subobject.
-  void AddVCallAndVBaseOffsets(BaseSubobject Base, bool BaseIsVirtual,
-                               CharUnits RealBaseOffset);
-  
-  /// AddVCallOffsets - Add vcall offsets for the given base subobject.
-  void AddVCallOffsets(BaseSubobject Base, CharUnits VBaseOffset);
-  
-  /// AddVBaseOffsets - Add vbase offsets for the given class.
-  void AddVBaseOffsets(const CXXRecordDecl *Base, 
-                       CharUnits OffsetInLayoutClass);
-  
-  /// getCurrentOffsetOffset - Get the current vcall or vbase offset offset in
-  /// chars, relative to the vtable address point.
-  CharUnits getCurrentOffsetOffset() const;
-  
-public:
-  VCallAndVBaseOffsetBuilder(const CXXRecordDecl *MostDerivedClass,
-                             const CXXRecordDecl *LayoutClass,
-                             const FinalOverriders *Overriders,
-                             BaseSubobject Base, bool BaseIsVirtual,
-                             CharUnits OffsetInLayoutClass)
-    : MostDerivedClass(MostDerivedClass), LayoutClass(LayoutClass), 
-    Context(MostDerivedClass->getASTContext()), Overriders(Overriders) {
-      
-    // Add vcall and vbase offsets.
-    AddVCallAndVBaseOffsets(Base, BaseIsVirtual, OffsetInLayoutClass);
-  }
-  
-  /// Methods for iterating over the components.
-  typedef VTableComponentVectorTy::const_reverse_iterator const_iterator;
-  const_iterator components_begin() const { return Components.rbegin(); }
-  const_iterator components_end() const { return Components.rend(); }
-  
-  const VCallOffsetMap &getVCallOffsets() const { return VCallOffsets; }
-  const VBaseOffsetOffsetsMapTy &getVBaseOffsetOffsets() const {
-    return VBaseOffsetOffsets;
-  }
-};
-  
-void 
-VCallAndVBaseOffsetBuilder::AddVCallAndVBaseOffsets(BaseSubobject Base,
-                                                    bool BaseIsVirtual,
-                                                    CharUnits RealBaseOffset) {
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(Base.getBase());
-  
-  // Itanium C++ ABI 2.5.2:
-  //   ..in classes sharing a virtual table with a primary base class, the vcall
-  //   and vbase offsets added by the derived class all come before the vcall
-  //   and vbase offsets required by the base class, so that the latter may be
-  //   laid out as required by the base class without regard to additions from
-  //   the derived class(es).
-
-  // (Since we're emitting the vcall and vbase offsets in reverse order, we'll
-  // emit them for the primary base first).
-  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
-    bool PrimaryBaseIsVirtual = Layout.isPrimaryBaseVirtual();
-
-    CharUnits PrimaryBaseOffset;
-    
-    // Get the base offset of the primary base.
-    if (PrimaryBaseIsVirtual) {
-      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 &&
-             "Primary vbase should have a zero offset!");
-      
-      const ASTRecordLayout &MostDerivedClassLayout =
-        Context.getASTRecordLayout(MostDerivedClass);
-      
-      PrimaryBaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(PrimaryBase);
-    } else {
-      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
-             "Primary base should have a zero offset!");
-
-      PrimaryBaseOffset = Base.getBaseOffset();
-    }
-
-    AddVCallAndVBaseOffsets(
-      BaseSubobject(PrimaryBase,PrimaryBaseOffset),
-      PrimaryBaseIsVirtual, RealBaseOffset);
-  }
-
-  AddVBaseOffsets(Base.getBase(), RealBaseOffset);
-
-  // We only want to add vcall offsets for virtual bases.
-  if (BaseIsVirtual)
-    AddVCallOffsets(Base, RealBaseOffset);
-}
-
-CharUnits VCallAndVBaseOffsetBuilder::getCurrentOffsetOffset() const {
-  // OffsetIndex is the index of this vcall or vbase offset, relative to the 
-  // vtable address point. (We subtract 3 to account for the information just
-  // above the address point, the RTTI info, the offset to top, and the
-  // vcall offset itself).
-  int64_t OffsetIndex = -(int64_t)(3 + Components.size());
-    
-  CharUnits PointerWidth = 
-    Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
-  CharUnits OffsetOffset = PointerWidth * OffsetIndex;
-  return OffsetOffset;
-}
-
-void VCallAndVBaseOffsetBuilder::AddVCallOffsets(BaseSubobject Base, 
-                                                 CharUnits VBaseOffset) {
-  const CXXRecordDecl *RD = Base.getBase();
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-
-  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
-
-  // Handle the primary base first.
-  // We only want to add vcall offsets if the base is non-virtual; a virtual
-  // primary base will have its vcall and vbase offsets emitted already.
-  if (PrimaryBase && !Layout.isPrimaryBaseVirtual()) {
-    // Get the base offset of the primary base.
-    assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
-           "Primary base should have a zero offset!");
-
-    AddVCallOffsets(BaseSubobject(PrimaryBase, Base.getBaseOffset()),
-                    VBaseOffset);
-  }
-  
-  // Add the vcall offsets.
-  for (CXXRecordDecl::method_iterator I = RD->method_begin(),
-       E = RD->method_end(); I != E; ++I) {
-    const CXXMethodDecl *MD = *I;
-    
-    if (!MD->isVirtual())
-      continue;
-
-    CharUnits OffsetOffset = getCurrentOffsetOffset();
-    
-    // Don't add a vcall offset if we already have one for this member function
-    // signature.
-    if (!VCallOffsets.AddVCallOffset(MD, OffsetOffset))
-      continue;
-
-    CharUnits Offset = CharUnits::Zero();
-
-    if (Overriders) {
-      // Get the final overrider.
-      FinalOverriders::OverriderInfo Overrider = 
-        Overriders->getOverrider(MD, Base.getBaseOffset());
-      
-      /// The vcall offset is the offset from the virtual base to the object 
-      /// where the function was overridden.
-      Offset = Overrider.Offset - VBaseOffset;
-    }
-    
-    Components.push_back(
-      VTableComponent::MakeVCallOffset(Offset));
-  }
-
-  // And iterate over all non-virtual bases (ignoring the primary base).
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-  
-    if (I->isVirtual())
-      continue;
-
-    const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-    if (BaseDecl == PrimaryBase)
-      continue;
-
-    // Get the base offset of this base.
-    CharUnits BaseOffset = Base.getBaseOffset() + 
-      Layout.getBaseClassOffset(BaseDecl);
-    
-    AddVCallOffsets(BaseSubobject(BaseDecl, BaseOffset), 
-                    VBaseOffset);
-  }
-}
-
-void 
-VCallAndVBaseOffsetBuilder::AddVBaseOffsets(const CXXRecordDecl *RD,
-                                            CharUnits OffsetInLayoutClass) {
-  const ASTRecordLayout &LayoutClassLayout = 
-    Context.getASTRecordLayout(LayoutClass);
-
-  // Add vbase offsets.
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    // Check if this is a virtual base that we haven't visited before.
-    if (I->isVirtual() && VisitedVirtualBases.insert(BaseDecl)) {
-      CharUnits Offset = 
-        LayoutClassLayout.getVBaseClassOffset(BaseDecl) - OffsetInLayoutClass;
-
-      // Add the vbase offset offset.
-      assert(!VBaseOffsetOffsets.count(BaseDecl) &&
-             "vbase offset offset already exists!");
-
-      CharUnits VBaseOffsetOffset = getCurrentOffsetOffset();
-      VBaseOffsetOffsets.insert(
-          std::make_pair(BaseDecl, VBaseOffsetOffset));
-
-      Components.push_back(
-          VTableComponent::MakeVBaseOffset(Offset));
-    }
-
-    // Check the base class looking for more vbase offsets.
-    AddVBaseOffsets(BaseDecl, OffsetInLayoutClass);
-  }
-}
-
-/// VTableBuilder - Class for building vtable layout information.
-class VTableBuilder {
-public:
-  /// PrimaryBasesSetVectorTy - A set vector of direct and indirect 
-  /// primary bases.
-  typedef llvm::SmallSetVector<const CXXRecordDecl *, 8> 
-    PrimaryBasesSetVectorTy;
-  
-  typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> 
-    VBaseOffsetOffsetsMapTy;
-  
-  typedef llvm::DenseMap<BaseSubobject, uint64_t> 
-    AddressPointsMapTy;
-
-private:
-  /// VTables - Global vtable information.
-  CodeGenVTables &VTables;
-  
-  /// MostDerivedClass - The most derived class for which we're building this
-  /// vtable.
-  const CXXRecordDecl *MostDerivedClass;
-
-  /// MostDerivedClassOffset - If we're building a construction vtable, this
-  /// holds the offset from the layout class to the most derived class.
-  const CharUnits MostDerivedClassOffset;
-  
-  /// MostDerivedClassIsVirtual - Whether the most derived class is a virtual 
-  /// base. (This only makes sense when building a construction vtable).
-  bool MostDerivedClassIsVirtual;
-  
-  /// LayoutClass - The class we're using for layout information. Will be 
-  /// different than the most derived class if we're building a construction
-  /// vtable.
-  const CXXRecordDecl *LayoutClass;
-  
-  /// Context - The ASTContext which we will use for layout information.
-  ASTContext &Context;
-  
-  /// FinalOverriders - The final overriders of the most derived class.
-  const FinalOverriders Overriders;
-
-  /// VCallOffsetsForVBases - Keeps track of vcall offsets for the virtual
-  /// bases in this vtable.
-  llvm::DenseMap<const CXXRecordDecl *, VCallOffsetMap> VCallOffsetsForVBases;
-
-  /// VBaseOffsetOffsets - Contains the offsets of the virtual base offsets for
-  /// the most derived class.
-  VBaseOffsetOffsetsMapTy VBaseOffsetOffsets;
-  
-  /// Components - The components of the vtable being built.
-  llvm::SmallVector<VTableComponent, 64> Components;
-
-  /// AddressPoints - Address points for the vtable being built.
-  AddressPointsMapTy AddressPoints;
-
-  /// MethodInfo - Contains information about a method in a vtable.
-  /// (Used for computing 'this' pointer adjustment thunks.
-  struct MethodInfo {
-    /// BaseOffset - The base offset of this method.
-    const CharUnits BaseOffset;
-    
-    /// BaseOffsetInLayoutClass - The base offset in the layout class of this
-    /// method.
-    const CharUnits BaseOffsetInLayoutClass;
-    
-    /// VTableIndex - The index in the vtable that this method has.
-    /// (For destructors, this is the index of the complete destructor).
-    const uint64_t VTableIndex;
-    
-    MethodInfo(CharUnits BaseOffset, CharUnits BaseOffsetInLayoutClass, 
-               uint64_t VTableIndex)
-      : BaseOffset(BaseOffset), 
-      BaseOffsetInLayoutClass(BaseOffsetInLayoutClass),
-      VTableIndex(VTableIndex) { }
-    
-    MethodInfo() 
-      : BaseOffset(CharUnits::Zero()), 
-      BaseOffsetInLayoutClass(CharUnits::Zero()), 
-      VTableIndex(0) { }
-  };
-  
-  typedef llvm::DenseMap<const CXXMethodDecl *, MethodInfo> MethodInfoMapTy;
-  
-  /// MethodInfoMap - The information for all methods in the vtable we're
-  /// currently building.
-  MethodInfoMapTy MethodInfoMap;
-  
-  typedef llvm::DenseMap<uint64_t, ThunkInfo> VTableThunksMapTy;
-  
-  /// VTableThunks - The thunks by vtable index in the vtable currently being 
-  /// built.
-  VTableThunksMapTy VTableThunks;
-
-  typedef llvm::SmallVector<ThunkInfo, 1> ThunkInfoVectorTy;
-  typedef llvm::DenseMap<const CXXMethodDecl *, ThunkInfoVectorTy> ThunksMapTy;
-  
-  /// Thunks - A map that contains all the thunks needed for all methods in the
-  /// most derived class for which the vtable is currently being built.
-  ThunksMapTy Thunks;
-  
-  /// AddThunk - Add a thunk for the given method.
-  void AddThunk(const CXXMethodDecl *MD, const ThunkInfo &Thunk);
-  
-  /// ComputeThisAdjustments - Compute the 'this' pointer adjustments for the
-  /// part of the vtable we're currently building.
-  void ComputeThisAdjustments();
-  
-  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
-
-  /// PrimaryVirtualBases - All known virtual bases who are a primary base of
-  /// some other base.
-  VisitedVirtualBasesSetTy PrimaryVirtualBases;
-
-  /// ComputeReturnAdjustment - Compute the return adjustment given a return
-  /// adjustment base offset.
-  ReturnAdjustment ComputeReturnAdjustment(BaseOffset Offset);
-  
-  /// ComputeThisAdjustmentBaseOffset - Compute the base offset for adjusting
-  /// the 'this' pointer from the base subobject to the derived subobject.
-  BaseOffset ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
-                                             BaseSubobject Derived) const;
-
-  /// ComputeThisAdjustment - Compute the 'this' pointer adjustment for the
-  /// given virtual member function, its offset in the layout class and its
-  /// final overrider.
-  ThisAdjustment 
-  ComputeThisAdjustment(const CXXMethodDecl *MD, 
-                        CharUnits BaseOffsetInLayoutClass,
-                        FinalOverriders::OverriderInfo Overrider);
-
-  /// AddMethod - Add a single virtual member function to the vtable
-  /// components vector.
-  void AddMethod(const CXXMethodDecl *MD, ReturnAdjustment ReturnAdjustment);
-
-  /// IsOverriderUsed - Returns whether the overrider will ever be used in this
-  /// part of the vtable. 
-  ///
-  /// Itanium C++ ABI 2.5.2:
-  ///
-  ///   struct A { virtual void f(); };
-  ///   struct B : virtual public A { int i; };
-  ///   struct C : virtual public A { int j; };
-  ///   struct D : public B, public C {};
-  ///
-  ///   When B and C are declared, A is a primary base in each case, so although
-  ///   vcall offsets are allocated in the A-in-B and A-in-C vtables, no this
-  ///   adjustment is required and no thunk is generated. However, inside D
-  ///   objects, A is no longer a primary base of C, so if we allowed calls to
-  ///   C::f() to use the copy of A's vtable in the C subobject, we would need
-  ///   to adjust this from C* to B::A*, which would require a third-party 
-  ///   thunk. Since we require that a call to C::f() first convert to A*, 
-  ///   C-in-D's copy of A's vtable is never referenced, so this is not 
-  ///   necessary.
-  bool IsOverriderUsed(const CXXMethodDecl *Overrider,
-                       CharUnits BaseOffsetInLayoutClass,
-                       const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
-                       CharUnits FirstBaseOffsetInLayoutClass) const;
-
-  
-  /// AddMethods - Add the methods of this base subobject and all its
-  /// primary bases to the vtable components vector.
-  void AddMethods(BaseSubobject Base, CharUnits BaseOffsetInLayoutClass,
-                  const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
-                  CharUnits FirstBaseOffsetInLayoutClass,
-                  PrimaryBasesSetVectorTy &PrimaryBases);
-
-  // LayoutVTable - Layout the vtable for the given base class, including its
-  // secondary vtables and any vtables for virtual bases.
-  void LayoutVTable();
-
-  /// LayoutPrimaryAndSecondaryVTables - Layout the primary vtable for the
-  /// given base subobject, as well as all its secondary vtables.
-  ///
-  /// \param BaseIsMorallyVirtual whether the base subobject is a virtual base
-  /// or a direct or indirect base of a virtual base.
-  ///
-  /// \param BaseIsVirtualInLayoutClass - Whether the base subobject is virtual
-  /// in the layout class. 
-  void LayoutPrimaryAndSecondaryVTables(BaseSubobject Base,
-                                        bool BaseIsMorallyVirtual,
-                                        bool BaseIsVirtualInLayoutClass,
-                                        CharUnits OffsetInLayoutClass);
-  
-  /// LayoutSecondaryVTables - Layout the secondary vtables for the given base
-  /// subobject.
-  ///
-  /// \param BaseIsMorallyVirtual whether the base subobject is a virtual base
-  /// or a direct or indirect base of a virtual base.
-  void LayoutSecondaryVTables(BaseSubobject Base, bool BaseIsMorallyVirtual,
-                              CharUnits OffsetInLayoutClass);
-
-  /// DeterminePrimaryVirtualBases - Determine the primary virtual bases in this
-  /// class hierarchy.
-  void DeterminePrimaryVirtualBases(const CXXRecordDecl *RD, 
-                                    CharUnits OffsetInLayoutClass,
-                                    VisitedVirtualBasesSetTy &VBases);
-
-  /// LayoutVTablesForVirtualBases - Layout vtables for all virtual bases of the
-  /// given base (excluding any primary bases).
-  void LayoutVTablesForVirtualBases(const CXXRecordDecl *RD, 
-                                    VisitedVirtualBasesSetTy &VBases);
-
-  /// isBuildingConstructionVTable - Return whether this vtable builder is
-  /// building a construction vtable.
-  bool isBuildingConstructorVTable() const { 
-    return MostDerivedClass != LayoutClass;
-  }
-
-public:
-  VTableBuilder(CodeGenVTables &VTables, const CXXRecordDecl *MostDerivedClass,
-                CharUnits MostDerivedClassOffset, 
-                bool MostDerivedClassIsVirtual, const 
-                CXXRecordDecl *LayoutClass)
-    : VTables(VTables), MostDerivedClass(MostDerivedClass),
-    MostDerivedClassOffset(MostDerivedClassOffset), 
-    MostDerivedClassIsVirtual(MostDerivedClassIsVirtual), 
-    LayoutClass(LayoutClass), Context(MostDerivedClass->getASTContext()), 
-    Overriders(MostDerivedClass, MostDerivedClassOffset, LayoutClass) {
-
-    LayoutVTable();
-  }
-
-  ThunksMapTy::const_iterator thunks_begin() const {
-    return Thunks.begin();
-  }
-
-  ThunksMapTy::const_iterator thunks_end() const {
-    return Thunks.end();
-  }
-
-  const VBaseOffsetOffsetsMapTy &getVBaseOffsetOffsets() const {
-    return VBaseOffsetOffsets;
-  }
-
-  /// getNumVTableComponents - Return the number of components in the vtable
-  /// currently built.
-  uint64_t getNumVTableComponents() const {
-    return Components.size();
-  }
-
-  const uint64_t *vtable_components_data_begin() const {
-    return reinterpret_cast<const uint64_t *>(Components.begin());
-  }
-  
-  const uint64_t *vtable_components_data_end() const {
-    return reinterpret_cast<const uint64_t *>(Components.end());
-  }
-  
-  AddressPointsMapTy::const_iterator address_points_begin() const {
-    return AddressPoints.begin();
-  }
-
-  AddressPointsMapTy::const_iterator address_points_end() const {
-    return AddressPoints.end();
-  }
-
-  VTableThunksMapTy::const_iterator vtable_thunks_begin() const {
-    return VTableThunks.begin();
-  }
-
-  VTableThunksMapTy::const_iterator vtable_thunks_end() const {
-    return VTableThunks.end();
-  }
-
-  /// dumpLayout - Dump the vtable layout.
-  void dumpLayout(llvm::raw_ostream&);
-};
-
-void VTableBuilder::AddThunk(const CXXMethodDecl *MD, const ThunkInfo &Thunk) {
-  assert(!isBuildingConstructorVTable() && 
-         "Can't add thunks for construction vtable");
-
-  llvm::SmallVector<ThunkInfo, 1> &ThunksVector = Thunks[MD];
-  
-  // Check if we have this thunk already.
-  if (std::find(ThunksVector.begin(), ThunksVector.end(), Thunk) != 
-      ThunksVector.end())
-    return;
-  
-  ThunksVector.push_back(Thunk);
-}
-
-typedef llvm::SmallPtrSet<const CXXMethodDecl *, 8> OverriddenMethodsSetTy;
-
-/// ComputeAllOverriddenMethods - Given a method decl, will return a set of all
-/// the overridden methods that the function decl overrides.
-static void 
-ComputeAllOverriddenMethods(const CXXMethodDecl *MD,
-                            OverriddenMethodsSetTy& OverriddenMethods) {
-  assert(MD->isVirtual() && "Method is not virtual!");
-
-  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
-       E = MD->end_overridden_methods(); I != E; ++I) {
-    const CXXMethodDecl *OverriddenMD = *I;
-    
-    OverriddenMethods.insert(OverriddenMD);
-    
-    ComputeAllOverriddenMethods(OverriddenMD, OverriddenMethods);
-  }
-}
-
-void VTableBuilder::ComputeThisAdjustments() {
-  // Now go through the method info map and see if any of the methods need
-  // 'this' pointer adjustments.
-  for (MethodInfoMapTy::const_iterator I = MethodInfoMap.begin(),
-       E = MethodInfoMap.end(); I != E; ++I) {
-    const CXXMethodDecl *MD = I->first;
-    const MethodInfo &MethodInfo = I->second;
-
-    // Ignore adjustments for unused function pointers.
-    uint64_t VTableIndex = MethodInfo.VTableIndex;
-    if (Components[VTableIndex].getKind() == 
-        VTableComponent::CK_UnusedFunctionPointer)
-      continue;
-    
-    // Get the final overrider for this method.
-    FinalOverriders::OverriderInfo Overrider =
-      Overriders.getOverrider(MD, MethodInfo.BaseOffset);
-    
-    // Check if we need an adjustment at all.
-    if (MethodInfo.BaseOffsetInLayoutClass == Overrider.Offset) {
-      // When a return thunk is needed by a derived class that overrides a
-      // virtual base, gcc uses a virtual 'this' adjustment as well. 
-      // While the thunk itself might be needed by vtables in subclasses or
-      // in construction vtables, there doesn't seem to be a reason for using
-      // the thunk in this vtable. Still, we do so to match gcc.
-      if (VTableThunks.lookup(VTableIndex).Return.isEmpty())
-        continue;
-    }
-
-    ThisAdjustment ThisAdjustment =
-      ComputeThisAdjustment(MD, MethodInfo.BaseOffsetInLayoutClass, Overrider);
-
-    if (ThisAdjustment.isEmpty())
-      continue;
-
-    // Add it.
-    VTableThunks[VTableIndex].This = ThisAdjustment;
-
-    if (isa<CXXDestructorDecl>(MD)) {
-      // Add an adjustment for the deleting destructor as well.
-      VTableThunks[VTableIndex + 1].This = ThisAdjustment;
-    }
-  }
-
-  /// Clear the method info map.
-  MethodInfoMap.clear();
-  
-  if (isBuildingConstructorVTable()) {
-    // We don't need to store thunk information for construction vtables.
-    return;
-  }
-
-  for (VTableThunksMapTy::const_iterator I = VTableThunks.begin(),
-       E = VTableThunks.end(); I != E; ++I) {
-    const VTableComponent &Component = Components[I->first];
-    const ThunkInfo &Thunk = I->second;
-    const CXXMethodDecl *MD;
-    
-    switch (Component.getKind()) {
-    default:
-      llvm_unreachable("Unexpected vtable component kind!");
-    case VTableComponent::CK_FunctionPointer:
-      MD = Component.getFunctionDecl();
-      break;
-    case VTableComponent::CK_CompleteDtorPointer:
-      MD = Component.getDestructorDecl();
-      break;
-    case VTableComponent::CK_DeletingDtorPointer:
-      // We've already added the thunk when we saw the complete dtor pointer.
-      continue;
-    }
-
-    if (MD->getParent() == MostDerivedClass)
-      AddThunk(MD, Thunk);
-  }
-}
-
-ReturnAdjustment VTableBuilder::ComputeReturnAdjustment(BaseOffset Offset) {
-  ReturnAdjustment Adjustment;
-  
-  if (!Offset.isEmpty()) {
-    if (Offset.VirtualBase) {
-      // Get the virtual base offset offset.
-      if (Offset.DerivedClass == MostDerivedClass) {
-        // We can get the offset offset directly from our map.
-        Adjustment.VBaseOffsetOffset = 
-          VBaseOffsetOffsets.lookup(Offset.VirtualBase).getQuantity();
-      } else {
-        Adjustment.VBaseOffsetOffset = 
-          VTables.getVirtualBaseOffsetOffset(Offset.DerivedClass,
-                                             Offset.VirtualBase).getQuantity();
-      }
-    }
-
-    Adjustment.NonVirtual = Offset.NonVirtualOffset.getQuantity();
-  }
-  
-  return Adjustment;
-}
-
-BaseOffset
-VTableBuilder::ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
-                                               BaseSubobject Derived) const {
-  const CXXRecordDecl *BaseRD = Base.getBase();
-  const CXXRecordDecl *DerivedRD = Derived.getBase();
-  
-  CXXBasePaths Paths(/*FindAmbiguities=*/true,
-                     /*RecordPaths=*/true, /*DetectVirtual=*/true);
-
-  if (!const_cast<CXXRecordDecl *>(DerivedRD)->
-      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseRD), Paths)) {
-    assert(false && "Class must be derived from the passed in base class!");
-    return BaseOffset();
-  }
-
-  // We have to go through all the paths, and see which one leads us to the
-  // right base subobject.
-  for (CXXBasePaths::const_paths_iterator I = Paths.begin(), E = Paths.end();
-       I != E; ++I) {
-    BaseOffset Offset = ComputeBaseOffset(Context, DerivedRD, *I);
-    
-    CharUnits OffsetToBaseSubobject = Offset.NonVirtualOffset;
-    
-    if (Offset.VirtualBase) {
-      // If we have a virtual base class, the non-virtual offset is relative
-      // to the virtual base class offset.
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-      
-      /// Get the virtual base offset, relative to the most derived class 
-      /// layout.
-      OffsetToBaseSubobject += 
-        LayoutClassLayout.getVBaseClassOffset(Offset.VirtualBase);
-    } else {
-      // Otherwise, the non-virtual offset is relative to the derived class 
-      // offset.
-      OffsetToBaseSubobject += Derived.getBaseOffset();
-    }
-    
-    // Check if this path gives us the right base subobject.
-    if (OffsetToBaseSubobject == Base.getBaseOffset()) {
-      // Since we're going from the base class _to_ the derived class, we'll
-      // invert the non-virtual offset here.
-      Offset.NonVirtualOffset = -Offset.NonVirtualOffset;
-      return Offset;
-    }      
-  }
-  
-  return BaseOffset();
-}
-  
-ThisAdjustment 
-VTableBuilder::ComputeThisAdjustment(const CXXMethodDecl *MD, 
-                                     CharUnits BaseOffsetInLayoutClass,
-                                     FinalOverriders::OverriderInfo Overrider) {
-  // Ignore adjustments for pure virtual member functions.
-  if (Overrider.Method->isPure())
-    return ThisAdjustment();
-  
-  BaseSubobject OverriddenBaseSubobject(MD->getParent(), 
-                                        BaseOffsetInLayoutClass);
-  
-  BaseSubobject OverriderBaseSubobject(Overrider.Method->getParent(),
-                                       Overrider.Offset);
-  
-  // Compute the adjustment offset.
-  BaseOffset Offset = ComputeThisAdjustmentBaseOffset(OverriddenBaseSubobject,
-                                                      OverriderBaseSubobject);
-  if (Offset.isEmpty())
-    return ThisAdjustment();
-
-  ThisAdjustment Adjustment;
-  
-  if (Offset.VirtualBase) {
-    // Get the vcall offset map for this virtual base.
-    VCallOffsetMap &VCallOffsets = VCallOffsetsForVBases[Offset.VirtualBase];
-
-    if (VCallOffsets.empty()) {
-      // We don't have vcall offsets for this virtual base, go ahead and
-      // build them.
-      VCallAndVBaseOffsetBuilder Builder(MostDerivedClass, MostDerivedClass,
-                                         /*FinalOverriders=*/0,
-                                         BaseSubobject(Offset.VirtualBase,
-                                                       CharUnits::Zero()),
-                                         /*BaseIsVirtual=*/true,
-                                         /*OffsetInLayoutClass=*/
-                                             CharUnits::Zero());
-        
-      VCallOffsets = Builder.getVCallOffsets();
-    }
-      
-    Adjustment.VCallOffsetOffset = 
-      VCallOffsets.getVCallOffsetOffset(MD).getQuantity();
-  }
-
-  // Set the non-virtual part of the adjustment.
-  Adjustment.NonVirtual = Offset.NonVirtualOffset.getQuantity();
-  
-  return Adjustment;
-}
-  
-void 
-VTableBuilder::AddMethod(const CXXMethodDecl *MD,
-                         ReturnAdjustment ReturnAdjustment) {
-  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
-    assert(ReturnAdjustment.isEmpty() && 
-           "Destructor can't have return adjustment!");
-
-    // Add both the complete destructor and the deleting destructor.
-    Components.push_back(VTableComponent::MakeCompleteDtor(DD));
-    Components.push_back(VTableComponent::MakeDeletingDtor(DD));
-  } else {
-    // Add the return adjustment if necessary.
-    if (!ReturnAdjustment.isEmpty())
-      VTableThunks[Components.size()].Return = ReturnAdjustment;
-
-    // Add the function.
-    Components.push_back(VTableComponent::MakeFunction(MD));
-  }
-}
-
-/// OverridesIndirectMethodInBase - Return whether the given member function
-/// overrides any methods in the set of given bases. 
-/// Unlike OverridesMethodInBase, this checks "overriders of overriders".
-/// For example, if we have:
-///
-/// struct A { virtual void f(); }
-/// struct B : A { virtual void f(); }
-/// struct C : B { virtual void f(); }
-///
-/// OverridesIndirectMethodInBase will return true if given C::f as the method 
-/// and { A } as the set of bases.
-static bool
-OverridesIndirectMethodInBases(const CXXMethodDecl *MD,
-                               VTableBuilder::PrimaryBasesSetVectorTy &Bases) {
-  if (Bases.count(MD->getParent()))
-    return true;
-  
-  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
-       E = MD->end_overridden_methods(); I != E; ++I) {
-    const CXXMethodDecl *OverriddenMD = *I;
-    
-    // Check "indirect overriders".
-    if (OverridesIndirectMethodInBases(OverriddenMD, Bases))
-      return true;
-  }
-   
-  return false;
-}
-
-bool 
-VTableBuilder::IsOverriderUsed(const CXXMethodDecl *Overrider,
-                               CharUnits BaseOffsetInLayoutClass,
-                               const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
-                               CharUnits FirstBaseOffsetInLayoutClass) const {
-  // If the base and the first base in the primary base chain have the same
-  // offsets, then this overrider will be used.
-  if (BaseOffsetInLayoutClass == FirstBaseOffsetInLayoutClass)
-   return true;
-
-  // We know now that Base (or a direct or indirect base of it) is a primary
-  // base in part of the class hierarchy, but not a primary base in the most 
-  // derived class.
-  
-  // If the overrider is the first base in the primary base chain, we know
-  // that the overrider will be used.
-  if (Overrider->getParent() == FirstBaseInPrimaryBaseChain)
-    return true;
-  
-  VTableBuilder::PrimaryBasesSetVectorTy PrimaryBases;
-
-  const CXXRecordDecl *RD = FirstBaseInPrimaryBaseChain;
-  PrimaryBases.insert(RD);
-
-  // Now traverse the base chain, starting with the first base, until we find
-  // the base that is no longer a primary base.
-  while (true) {
-    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-    const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
-    
-    if (!PrimaryBase)
-      break;
-    
-    if (Layout.isPrimaryBaseVirtual()) {
-      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 && 
-             "Primary base should always be at offset 0!");
-
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-
-      // Now check if this is the primary base that is not a primary base in the
-      // most derived class.
-      if (LayoutClassLayout.getVBaseClassOffset(PrimaryBase) !=
-          FirstBaseOffsetInLayoutClass) {
-        // We found it, stop walking the chain.
-        break;
-      }
-    } else {
-      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 && 
-             "Primary base should always be at offset 0!");
-    }
-    
-    if (!PrimaryBases.insert(PrimaryBase))
-      assert(false && "Found a duplicate primary base!");
-
-    RD = PrimaryBase;
-  }
-  
-  // If the final overrider is an override of one of the primary bases,
-  // then we know that it will be used.
-  return OverridesIndirectMethodInBases(Overrider, PrimaryBases);
-}
-
-/// FindNearestOverriddenMethod - Given a method, returns the overridden method
-/// from the nearest base. Returns null if no method was found.
-static const CXXMethodDecl * 
-FindNearestOverriddenMethod(const CXXMethodDecl *MD,
-                            VTableBuilder::PrimaryBasesSetVectorTy &Bases) {
-  OverriddenMethodsSetTy OverriddenMethods;
-  ComputeAllOverriddenMethods(MD, OverriddenMethods);
-  
-  for (int I = Bases.size(), E = 0; I != E; --I) {
-    const CXXRecordDecl *PrimaryBase = Bases[I - 1];
-
-    // Now check the overriden methods.
-    for (OverriddenMethodsSetTy::const_iterator I = OverriddenMethods.begin(),
-         E = OverriddenMethods.end(); I != E; ++I) {
-      const CXXMethodDecl *OverriddenMD = *I;
-      
-      // We found our overridden method.
-      if (OverriddenMD->getParent() == PrimaryBase)
-        return OverriddenMD;
-    }
-  }
-  
-  return 0;
-}  
-
-void
-VTableBuilder::AddMethods(BaseSubobject Base, CharUnits BaseOffsetInLayoutClass,
-                          const CXXRecordDecl *FirstBaseInPrimaryBaseChain,
-                          CharUnits FirstBaseOffsetInLayoutClass,
-                          PrimaryBasesSetVectorTy &PrimaryBases) {
-  const CXXRecordDecl *RD = Base.getBase();
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-
-  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
-    CharUnits PrimaryBaseOffset;
-    CharUnits PrimaryBaseOffsetInLayoutClass;
-    if (Layout.isPrimaryBaseVirtual()) {
-      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 &&
-             "Primary vbase should have a zero offset!");
-      
-      const ASTRecordLayout &MostDerivedClassLayout =
-        Context.getASTRecordLayout(MostDerivedClass);
-      
-      PrimaryBaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(PrimaryBase);
-      
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-
-      PrimaryBaseOffsetInLayoutClass =
-        LayoutClassLayout.getVBaseClassOffset(PrimaryBase);
-    } else {
-      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
-             "Primary base should have a zero offset!");
-
-      PrimaryBaseOffset = Base.getBaseOffset();
-      PrimaryBaseOffsetInLayoutClass = BaseOffsetInLayoutClass;
-    }
-
-    AddMethods(BaseSubobject(PrimaryBase, PrimaryBaseOffset),
-               PrimaryBaseOffsetInLayoutClass, FirstBaseInPrimaryBaseChain, 
-               FirstBaseOffsetInLayoutClass, PrimaryBases);
-    
-    if (!PrimaryBases.insert(PrimaryBase))
-      assert(false && "Found a duplicate primary base!");
-  }
-
-  // Now go through all virtual member functions and add them.
-  for (CXXRecordDecl::method_iterator I = RD->method_begin(),
-       E = RD->method_end(); I != E; ++I) {
-    const CXXMethodDecl *MD = *I;
-  
-    if (!MD->isVirtual())
-      continue;
-
-    // Get the final overrider.
-    FinalOverriders::OverriderInfo Overrider = 
-      Overriders.getOverrider(MD, Base.getBaseOffset());
-
-    // Check if this virtual member function overrides a method in a primary
-    // base. If this is the case, and the return type doesn't require adjustment
-    // then we can just use the member function from the primary base.
-    if (const CXXMethodDecl *OverriddenMD = 
-          FindNearestOverriddenMethod(MD, PrimaryBases)) {
-      if (ComputeReturnAdjustmentBaseOffset(Context, MD, 
-                                            OverriddenMD).isEmpty()) {
-        // Replace the method info of the overridden method with our own
-        // method.
-        assert(MethodInfoMap.count(OverriddenMD) && 
-               "Did not find the overridden method!");
-        MethodInfo &OverriddenMethodInfo = MethodInfoMap[OverriddenMD];
-        
-        MethodInfo MethodInfo(Base.getBaseOffset(), BaseOffsetInLayoutClass,
-                              OverriddenMethodInfo.VTableIndex);
-
-        assert(!MethodInfoMap.count(MD) &&
-               "Should not have method info for this method yet!");
-        
-        MethodInfoMap.insert(std::make_pair(MD, MethodInfo));
-        MethodInfoMap.erase(OverriddenMD);
-        
-        // If the overridden method exists in a virtual base class or a direct
-        // or indirect base class of a virtual base class, we need to emit a
-        // thunk if we ever have a class hierarchy where the base class is not
-        // a primary base in the complete object.
-        if (!isBuildingConstructorVTable() && OverriddenMD != MD) {
-          // Compute the this adjustment.
-          ThisAdjustment ThisAdjustment =
-            ComputeThisAdjustment(OverriddenMD, BaseOffsetInLayoutClass,
-                                  Overrider);
-
-          if (ThisAdjustment.VCallOffsetOffset &&
-              Overrider.Method->getParent() == MostDerivedClass) {
-
-            // There's no return adjustment from OverriddenMD and MD,
-            // but that doesn't mean there isn't one between MD and
-            // the final overrider.
-            BaseOffset ReturnAdjustmentOffset =
-              ComputeReturnAdjustmentBaseOffset(Context, Overrider.Method, MD);
-            ReturnAdjustment ReturnAdjustment = 
-              ComputeReturnAdjustment(ReturnAdjustmentOffset);
-
-            // This is a virtual thunk for the most derived class, add it.
-            AddThunk(Overrider.Method, 
-                     ThunkInfo(ThisAdjustment, ReturnAdjustment));
-          }
-        }
-
-        continue;
-      }
-    }
-
-    // Insert the method info for this method.
-    MethodInfo MethodInfo(Base.getBaseOffset(), BaseOffsetInLayoutClass,
-                          Components.size());
-
-    assert(!MethodInfoMap.count(MD) &&
-           "Should not have method info for this method yet!");
-    MethodInfoMap.insert(std::make_pair(MD, MethodInfo));
-
-    // Check if this overrider is going to be used.
-    const CXXMethodDecl *OverriderMD = Overrider.Method;
-    if (!IsOverriderUsed(OverriderMD, BaseOffsetInLayoutClass,
-                         FirstBaseInPrimaryBaseChain, 
-                         FirstBaseOffsetInLayoutClass)) {
-      Components.push_back(VTableComponent::MakeUnusedFunction(OverriderMD));
-      continue;
-    }
-    
-    // Check if this overrider needs a return adjustment.
-    // We don't want to do this for pure virtual member functions.
-    BaseOffset ReturnAdjustmentOffset;
-    if (!OverriderMD->isPure()) {
-      ReturnAdjustmentOffset = 
-        ComputeReturnAdjustmentBaseOffset(Context, OverriderMD, MD);
-    }
-
-    ReturnAdjustment ReturnAdjustment = 
-      ComputeReturnAdjustment(ReturnAdjustmentOffset);
-    
-    AddMethod(Overrider.Method, ReturnAdjustment);
-  }
-}
-
-void VTableBuilder::LayoutVTable() {
-  LayoutPrimaryAndSecondaryVTables(BaseSubobject(MostDerivedClass,
-                                                 CharUnits::Zero()),
-                                   /*BaseIsMorallyVirtual=*/false,
-                                   MostDerivedClassIsVirtual,
-                                   MostDerivedClassOffset);
-  
-  VisitedVirtualBasesSetTy VBases;
-  
-  // Determine the primary virtual bases.
-  DeterminePrimaryVirtualBases(MostDerivedClass, MostDerivedClassOffset, 
-                               VBases);
-  VBases.clear();
-  
-  LayoutVTablesForVirtualBases(MostDerivedClass, VBases);
-}
-  
-void
-VTableBuilder::LayoutPrimaryAndSecondaryVTables(BaseSubobject Base,
-                                                bool BaseIsMorallyVirtual,
-                                                bool BaseIsVirtualInLayoutClass,
-                                                CharUnits OffsetInLayoutClass) {
-  assert(Base.getBase()->isDynamicClass() && "class does not have a vtable!");
-
-  // Add vcall and vbase offsets for this vtable.
-  VCallAndVBaseOffsetBuilder Builder(MostDerivedClass, LayoutClass, &Overriders,
-                                     Base, BaseIsVirtualInLayoutClass, 
-                                     OffsetInLayoutClass);
-  Components.append(Builder.components_begin(), Builder.components_end());
-  
-  // Check if we need to add these vcall offsets.
-  if (BaseIsVirtualInLayoutClass && !Builder.getVCallOffsets().empty()) {
-    VCallOffsetMap &VCallOffsets = VCallOffsetsForVBases[Base.getBase()];
-    
-    if (VCallOffsets.empty())
-      VCallOffsets = Builder.getVCallOffsets();
-  }
-
-  // If we're laying out the most derived class we want to keep track of the
-  // virtual base class offset offsets.
-  if (Base.getBase() == MostDerivedClass)
-    VBaseOffsetOffsets = Builder.getVBaseOffsetOffsets();
-
-  // Add the offset to top.
-  CharUnits OffsetToTop = MostDerivedClassOffset - OffsetInLayoutClass;
-  Components.push_back(
-    VTableComponent::MakeOffsetToTop(OffsetToTop));
-  
-  // Next, add the RTTI.
-  Components.push_back(VTableComponent::MakeRTTI(MostDerivedClass));
-  
-  uint64_t AddressPoint = Components.size();
-
-  // Now go through all virtual member functions and add them.
-  PrimaryBasesSetVectorTy PrimaryBases;
-  AddMethods(Base, OffsetInLayoutClass,
-             Base.getBase(), OffsetInLayoutClass, 
-             PrimaryBases);
-
-  // Compute 'this' pointer adjustments.
-  ComputeThisAdjustments();
-
-  // Add all address points.
-  const CXXRecordDecl *RD = Base.getBase();
-  while (true) {
-    AddressPoints.insert(std::make_pair(
-      BaseSubobject(RD, OffsetInLayoutClass),
-      AddressPoint));
-
-    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-    const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
-    
-    if (!PrimaryBase)
-      break;
-    
-    if (Layout.isPrimaryBaseVirtual()) {
-      // Check if this virtual primary base is a primary base in the layout
-      // class. If it's not, we don't want to add it.
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-
-      if (LayoutClassLayout.getVBaseClassOffset(PrimaryBase) !=
-          OffsetInLayoutClass) {
-        // We don't want to add this class (or any of its primary bases).
-        break;
-      }
-    }
-
-    RD = PrimaryBase;
-  }
-
-  // Layout secondary vtables.
-  LayoutSecondaryVTables(Base, BaseIsMorallyVirtual, OffsetInLayoutClass);
-}
-
-void VTableBuilder::LayoutSecondaryVTables(BaseSubobject Base,
-                                           bool BaseIsMorallyVirtual,
-                                           CharUnits OffsetInLayoutClass) {
-  // Itanium C++ ABI 2.5.2:
-  //   Following the primary virtual table of a derived class are secondary 
-  //   virtual tables for each of its proper base classes, except any primary
-  //   base(s) with which it shares its primary virtual table.
-
-  const CXXRecordDecl *RD = Base.getBase();
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
-  
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    // Ignore virtual bases, we'll emit them later.
-    if (I->isVirtual())
-      continue;
-    
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    // Ignore bases that don't have a vtable.
-    if (!BaseDecl->isDynamicClass())
-      continue;
-
-    if (isBuildingConstructorVTable()) {
-      // Itanium C++ ABI 2.6.4:
-      //   Some of the base class subobjects may not need construction virtual
-      //   tables, which will therefore not be present in the construction
-      //   virtual table group, even though the subobject virtual tables are
-      //   present in the main virtual table group for the complete object.
-      if (!BaseIsMorallyVirtual && !BaseDecl->getNumVBases())
-        continue;
-    }
-
-    // Get the base offset of this base.
-    CharUnits RelativeBaseOffset = Layout.getBaseClassOffset(BaseDecl);
-    CharUnits BaseOffset = Base.getBaseOffset() + RelativeBaseOffset;
-    
-    CharUnits BaseOffsetInLayoutClass = 
-      OffsetInLayoutClass + RelativeBaseOffset;
-    
-    // Don't emit a secondary vtable for a primary base. We might however want 
-    // to emit secondary vtables for other bases of this base.
-    if (BaseDecl == PrimaryBase) {
-      LayoutSecondaryVTables(BaseSubobject(BaseDecl, BaseOffset),
-                             BaseIsMorallyVirtual, BaseOffsetInLayoutClass);
-      continue;
-    }
-
-    // Layout the primary vtable (and any secondary vtables) for this base.
-    LayoutPrimaryAndSecondaryVTables(
-      BaseSubobject(BaseDecl, BaseOffset),
-      BaseIsMorallyVirtual,
-      /*BaseIsVirtualInLayoutClass=*/false,
-      BaseOffsetInLayoutClass);
-  }
-}
-
-void
-VTableBuilder::DeterminePrimaryVirtualBases(const CXXRecordDecl *RD,
-                                            CharUnits OffsetInLayoutClass,
-                                            VisitedVirtualBasesSetTy &VBases) {
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-  
-  // Check if this base has a primary base.
-  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
-
-    // Check if it's virtual.
-    if (Layout.isPrimaryBaseVirtual()) {
-      bool IsPrimaryVirtualBase = true;
-
-      if (isBuildingConstructorVTable()) {
-        // Check if the base is actually a primary base in the class we use for
-        // layout.
-        const ASTRecordLayout &LayoutClassLayout =
-          Context.getASTRecordLayout(LayoutClass);
-
-        CharUnits PrimaryBaseOffsetInLayoutClass =
-          LayoutClassLayout.getVBaseClassOffset(PrimaryBase);
-        
-        // We know that the base is not a primary base in the layout class if 
-        // the base offsets are different.
-        if (PrimaryBaseOffsetInLayoutClass != OffsetInLayoutClass)
-          IsPrimaryVirtualBase = false;
-      }
-        
-      if (IsPrimaryVirtualBase)
-        PrimaryVirtualBases.insert(PrimaryBase);
-    }
-  }
-
-  // Traverse bases, looking for more primary virtual bases.
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    CharUnits BaseOffsetInLayoutClass;
-    
-    if (I->isVirtual()) {
-      if (!VBases.insert(BaseDecl))
-        continue;
-      
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-
-      BaseOffsetInLayoutClass = 
-        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
-    } else {
-      BaseOffsetInLayoutClass = 
-        OffsetInLayoutClass + Layout.getBaseClassOffset(BaseDecl);
-    }
-
-    DeterminePrimaryVirtualBases(BaseDecl, BaseOffsetInLayoutClass, VBases);
-  }
-}
-
-void
-VTableBuilder::LayoutVTablesForVirtualBases(const CXXRecordDecl *RD, 
-                                            VisitedVirtualBasesSetTy &VBases) {
-  // Itanium C++ ABI 2.5.2:
-  //   Then come the virtual base virtual tables, also in inheritance graph
-  //   order, and again excluding primary bases (which share virtual tables with
-  //   the classes for which they are primary).
-  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-       E = RD->bases_end(); I != E; ++I) {
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-    // Check if this base needs a vtable. (If it's virtual, not a primary base
-    // of some other class, and we haven't visited it before).
-    if (I->isVirtual() && BaseDecl->isDynamicClass() && 
-        !PrimaryVirtualBases.count(BaseDecl) && VBases.insert(BaseDecl)) {
-      const ASTRecordLayout &MostDerivedClassLayout =
-        Context.getASTRecordLayout(MostDerivedClass);
-      CharUnits BaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
-      
-      const ASTRecordLayout &LayoutClassLayout =
-        Context.getASTRecordLayout(LayoutClass);
-      CharUnits BaseOffsetInLayoutClass = 
-        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
-
-      LayoutPrimaryAndSecondaryVTables(
-        BaseSubobject(BaseDecl, BaseOffset),
-        /*BaseIsMorallyVirtual=*/true,
-        /*BaseIsVirtualInLayoutClass=*/true,
-        BaseOffsetInLayoutClass);
-    }
-    
-    // We only need to check the base for virtual base vtables if it actually
-    // has virtual bases.
-    if (BaseDecl->getNumVBases())
-      LayoutVTablesForVirtualBases(BaseDecl, VBases);
-  }
-}
-
-/// dumpLayout - Dump the vtable layout.
-void VTableBuilder::dumpLayout(llvm::raw_ostream& Out) {
-
-  if (isBuildingConstructorVTable()) {
-    Out << "Construction vtable for ('";
-    Out << MostDerivedClass->getQualifiedNameAsString() << "', ";
-    Out << MostDerivedClassOffset.getQuantity() << ") in '";
-    Out << LayoutClass->getQualifiedNameAsString();
-  } else {
-    Out << "Vtable for '";
-    Out << MostDerivedClass->getQualifiedNameAsString();
-  }
-  Out << "' (" << Components.size() << " entries).\n";
-
-  // Iterate through the address points and insert them into a new map where
-  // they are keyed by the index and not the base object.
-  // Since an address point can be shared by multiple subobjects, we use an
-  // STL multimap.
-  std::multimap<uint64_t, BaseSubobject> AddressPointsByIndex;
-  for (AddressPointsMapTy::const_iterator I = AddressPoints.begin(), 
-       E = AddressPoints.end(); I != E; ++I) {
-    const BaseSubobject& Base = I->first;
-    uint64_t Index = I->second;
-    
-    AddressPointsByIndex.insert(std::make_pair(Index, Base));
-  }
-  
-  for (unsigned I = 0, E = Components.size(); I != E; ++I) {
-    uint64_t Index = I;
-
-    Out << llvm::format("%4d | ", I);
-
-    const VTableComponent &Component = Components[I];
-
-    // Dump the component.
-    switch (Component.getKind()) {
-
-    case VTableComponent::CK_VCallOffset:
-      Out << "vcall_offset ("
-          << Component.getVCallOffset().getQuantity() 
-          << ")";
-      break;
-
-    case VTableComponent::CK_VBaseOffset:
-      Out << "vbase_offset ("
-          << Component.getVBaseOffset().getQuantity()
-          << ")";
-      break;
-
-    case VTableComponent::CK_OffsetToTop:
-      Out << "offset_to_top ("
-          << Component.getOffsetToTop().getQuantity()
-          << ")";
-      break;
-    
-    case VTableComponent::CK_RTTI:
-      Out << Component.getRTTIDecl()->getQualifiedNameAsString() << " RTTI";
-      break;
-    
-    case VTableComponent::CK_FunctionPointer: {
-      const CXXMethodDecl *MD = Component.getFunctionDecl();
-
-      std::string Str = 
-        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual, 
-                                    MD);
-      Out << Str;
-      if (MD->isPure())
-        Out << " [pure]";
-
-      ThunkInfo Thunk = VTableThunks.lookup(I);
-      if (!Thunk.isEmpty()) {
-        // If this function pointer has a return adjustment, dump it.
-        if (!Thunk.Return.isEmpty()) {
-          Out << "\n       [return adjustment: ";
-          Out << Thunk.Return.NonVirtual << " non-virtual";
-          
-          if (Thunk.Return.VBaseOffsetOffset) {
-            Out << ", " << Thunk.Return.VBaseOffsetOffset;
-            Out << " vbase offset offset";
-          }
-
-          Out << ']';
-        }
-
-        // If this function pointer has a 'this' pointer adjustment, dump it.
-        if (!Thunk.This.isEmpty()) {
-          Out << "\n       [this adjustment: ";
-          Out << Thunk.This.NonVirtual << " non-virtual";
-          
-          if (Thunk.This.VCallOffsetOffset) {
-            Out << ", " << Thunk.This.VCallOffsetOffset;
-            Out << " vcall offset offset";
-          }
-
-          Out << ']';
-        }          
-      }
-
-      break;
-    }
-
-    case VTableComponent::CK_CompleteDtorPointer: 
-    case VTableComponent::CK_DeletingDtorPointer: {
-      bool IsComplete = 
-        Component.getKind() == VTableComponent::CK_CompleteDtorPointer;
-      
-      const CXXDestructorDecl *DD = Component.getDestructorDecl();
-      
-      Out << DD->getQualifiedNameAsString();
-      if (IsComplete)
-        Out << "() [complete]";
-      else
-        Out << "() [deleting]";
-
-      if (DD->isPure())
-        Out << " [pure]";
-
-      ThunkInfo Thunk = VTableThunks.lookup(I);
-      if (!Thunk.isEmpty()) {
-        // If this destructor has a 'this' pointer adjustment, dump it.
-        if (!Thunk.This.isEmpty()) {
-          Out << "\n       [this adjustment: ";
-          Out << Thunk.This.NonVirtual << " non-virtual";
-          
-          if (Thunk.This.VCallOffsetOffset) {
-            Out << ", " << Thunk.This.VCallOffsetOffset;
-            Out << " vcall offset offset";
-          }
-          
-          Out << ']';
-        }          
-      }        
-
-      break;
-    }
-
-    case VTableComponent::CK_UnusedFunctionPointer: {
-      const CXXMethodDecl *MD = Component.getUnusedFunctionDecl();
-
-      std::string Str = 
-        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual, 
-                                    MD);
-      Out << "[unused] " << Str;
-      if (MD->isPure())
-        Out << " [pure]";
-    }
-
-    }
-
-    Out << '\n';
-    
-    // Dump the next address point.
-    uint64_t NextIndex = Index + 1;
-    if (AddressPointsByIndex.count(NextIndex)) {
-      if (AddressPointsByIndex.count(NextIndex) == 1) {
-        const BaseSubobject &Base = 
-          AddressPointsByIndex.find(NextIndex)->second;
-        
-        Out << "       -- (" << Base.getBase()->getQualifiedNameAsString();
-        Out << ", " << Base.getBaseOffset().getQuantity();
-        Out << ") vtable address --\n";
-      } else {
-        CharUnits BaseOffset =
-          AddressPointsByIndex.lower_bound(NextIndex)->second.getBaseOffset();
-        
-        // We store the class names in a set to get a stable order.
-        std::set<std::string> ClassNames;
-        for (std::multimap<uint64_t, BaseSubobject>::const_iterator I =
-             AddressPointsByIndex.lower_bound(NextIndex), E =
-             AddressPointsByIndex.upper_bound(NextIndex); I != E; ++I) {
-          assert(I->second.getBaseOffset() == BaseOffset &&
-                 "Invalid base offset!");
-          const CXXRecordDecl *RD = I->second.getBase();
-          ClassNames.insert(RD->getQualifiedNameAsString());
-        }
-        
-        for (std::set<std::string>::const_iterator I = ClassNames.begin(),
-             E = ClassNames.end(); I != E; ++I) {
-          Out << "       -- (" << *I;
-          Out << ", " << BaseOffset.getQuantity() << ") vtable address --\n";
-        }
-      }
-    }
-  }
-
-  Out << '\n';
-  
-  if (isBuildingConstructorVTable())
-    return;
-  
-  if (MostDerivedClass->getNumVBases()) {
-    // We store the virtual base class names and their offsets in a map to get
-    // a stable order.
-
-    std::map<std::string, CharUnits> ClassNamesAndOffsets;
-    for (VBaseOffsetOffsetsMapTy::const_iterator I = VBaseOffsetOffsets.begin(),
-         E = VBaseOffsetOffsets.end(); I != E; ++I) {
-      std::string ClassName = I->first->getQualifiedNameAsString();
-      CharUnits OffsetOffset = I->second;
-      ClassNamesAndOffsets.insert(
-          std::make_pair(ClassName, OffsetOffset));
-    }
-    
-    Out << "Virtual base offset offsets for '";
-    Out << MostDerivedClass->getQualifiedNameAsString() << "' (";
-    Out << ClassNamesAndOffsets.size();
-    Out << (ClassNamesAndOffsets.size() == 1 ? " entry" : " entries") << ").\n";
-
-    for (std::map<std::string, CharUnits>::const_iterator I =
-         ClassNamesAndOffsets.begin(), E = ClassNamesAndOffsets.end(); 
-         I != E; ++I)
-      Out << "   " << I->first << " | " << I->second.getQuantity() << '\n';
-
-    Out << "\n";
-  }
-  
-  if (!Thunks.empty()) {
-    // We store the method names in a map to get a stable order.
-    std::map<std::string, const CXXMethodDecl *> MethodNamesAndDecls;
-    
-    for (ThunksMapTy::const_iterator I = Thunks.begin(), E = Thunks.end();
-         I != E; ++I) {
-      const CXXMethodDecl *MD = I->first;
-      std::string MethodName = 
-        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual,
-                                    MD);
-      
-      MethodNamesAndDecls.insert(std::make_pair(MethodName, MD));
-    }
-
-    for (std::map<std::string, const CXXMethodDecl *>::const_iterator I =
-         MethodNamesAndDecls.begin(), E = MethodNamesAndDecls.end(); 
-         I != E; ++I) {
-      const std::string &MethodName = I->first;
-      const CXXMethodDecl *MD = I->second;
-
-      ThunkInfoVectorTy ThunksVector = Thunks[MD];
-      std::sort(ThunksVector.begin(), ThunksVector.end());
-
-      Out << "Thunks for '" << MethodName << "' (" << ThunksVector.size();
-      Out << (ThunksVector.size() == 1 ? " entry" : " entries") << ").\n";
-      
-      for (unsigned I = 0, E = ThunksVector.size(); I != E; ++I) {
-        const ThunkInfo &Thunk = ThunksVector[I];
-
-        Out << llvm::format("%4d | ", I);
-        
-        // If this function pointer has a return pointer adjustment, dump it.
-        if (!Thunk.Return.isEmpty()) {
-          Out << "return adjustment: " << Thunk.This.NonVirtual;
-          Out << " non-virtual";
-          if (Thunk.Return.VBaseOffsetOffset) {
-            Out << ", " << Thunk.Return.VBaseOffsetOffset;
-            Out << " vbase offset offset";
-          }
-
-          if (!Thunk.This.isEmpty())
-            Out << "\n       ";
-        }
-
-        // If this function pointer has a 'this' pointer adjustment, dump it.
-        if (!Thunk.This.isEmpty()) {
-          Out << "this adjustment: ";
-          Out << Thunk.This.NonVirtual << " non-virtual";
-          
-          if (Thunk.This.VCallOffsetOffset) {
-            Out << ", " << Thunk.This.VCallOffsetOffset;
-            Out << " vcall offset offset";
-          }
-        }
-        
-        Out << '\n';
-      }
-      
-      Out << '\n';
-    }
-  }
-
-  // Compute the vtable indices for all the member functions.
-  // Store them in a map keyed by the index so we'll get a sorted table.
-  std::map<uint64_t, std::string> IndicesMap;
-
-  for (CXXRecordDecl::method_iterator i = MostDerivedClass->method_begin(),
-       e = MostDerivedClass->method_end(); i != e; ++i) {
-    const CXXMethodDecl *MD = *i;
-    
-    // We only want virtual member functions.
-    if (!MD->isVirtual())
-      continue;
-
-    std::string MethodName =
-      PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual,
-                                  MD);
-
-    if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
-      IndicesMap[VTables.getMethodVTableIndex(GlobalDecl(DD, Dtor_Complete))] =
-        MethodName + " [complete]";
-      IndicesMap[VTables.getMethodVTableIndex(GlobalDecl(DD, Dtor_Deleting))] =
-        MethodName + " [deleting]";
-    } else {
-      IndicesMap[VTables.getMethodVTableIndex(MD)] = MethodName;
-    }
-  }
-
-  // Print the vtable indices for all the member functions.
-  if (!IndicesMap.empty()) {
-    Out << "VTable indices for '";
-    Out << MostDerivedClass->getQualifiedNameAsString();
-    Out << "' (" << IndicesMap.size() << " entries).\n";
-
-    for (std::map<uint64_t, std::string>::const_iterator I = IndicesMap.begin(),
-         E = IndicesMap.end(); I != E; ++I) {
-      uint64_t VTableIndex = I->first;
-      const std::string &MethodName = I->second;
-
-      Out << llvm::format(" %4u | ", VTableIndex) << MethodName << '\n';
-    }
-  }
-
-  Out << '\n';
-}
-  
-}
-
-static void 
-CollectPrimaryBases(const CXXRecordDecl *RD, ASTContext &Context,
-                    VTableBuilder::PrimaryBasesSetVectorTy &PrimaryBases) {
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
-
-  if (!PrimaryBase)
-    return;
-
-  CollectPrimaryBases(PrimaryBase, Context, PrimaryBases);
-
-  if (!PrimaryBases.insert(PrimaryBase))
-    assert(false && "Found a duplicate primary base!");
-}
-
-void CodeGenVTables::ComputeMethodVTableIndices(const CXXRecordDecl *RD) {
-  
-  // Itanium C++ ABI 2.5.2:
-  //   The order of the virtual function pointers in a virtual table is the 
-  //   order of declaration of the corresponding member functions in the class.
-  //
-  //   There is an entry for any virtual function declared in a class, 
-  //   whether it is a new function or overrides a base class function, 
-  //   unless it overrides a function from the primary base, and conversion
-  //   between their return types does not require an adjustment. 
-
-  int64_t CurrentIndex = 0;
-  
-  const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
-  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
-  
-  if (PrimaryBase) {
-    assert(PrimaryBase->isDefinition() && 
-           "Should have the definition decl of the primary base!");
-
-    // Since the record decl shares its vtable pointer with the primary base
-    // we need to start counting at the end of the primary base's vtable.
-    CurrentIndex = getNumVirtualFunctionPointers(PrimaryBase);
-  }
-
-  // Collect all the primary bases, so we can check whether methods override
-  // a method from the base.
-  VTableBuilder::PrimaryBasesSetVectorTy PrimaryBases;
-  CollectPrimaryBases(RD, CGM.getContext(), PrimaryBases);
-
-  const CXXDestructorDecl *ImplicitVirtualDtor = 0;
-  
-  for (CXXRecordDecl::method_iterator i = RD->method_begin(),
-       e = RD->method_end(); i != e; ++i) {
-    const CXXMethodDecl *MD = *i;
-
-    // We only want virtual methods.
-    if (!MD->isVirtual())
-      continue;
-
-    // Check if this method overrides a method in the primary base.
-    if (const CXXMethodDecl *OverriddenMD = 
-          FindNearestOverriddenMethod(MD, PrimaryBases)) {
-      // Check if converting from the return type of the method to the 
-      // return type of the overridden method requires conversion.
-      if (ComputeReturnAdjustmentBaseOffset(CGM.getContext(), MD, 
-                                            OverriddenMD).isEmpty()) {
-        // This index is shared between the index in the vtable of the primary
-        // base class.
-        if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
-          const CXXDestructorDecl *OverriddenDD = 
-            cast<CXXDestructorDecl>(OverriddenMD);
-          
-          // Add both the complete and deleting entries.
-          MethodVTableIndices[GlobalDecl(DD, Dtor_Complete)] = 
-            getMethodVTableIndex(GlobalDecl(OverriddenDD, Dtor_Complete));
-          MethodVTableIndices[GlobalDecl(DD, Dtor_Deleting)] = 
-            getMethodVTableIndex(GlobalDecl(OverriddenDD, Dtor_Deleting));
-        } else {
-          MethodVTableIndices[MD] = getMethodVTableIndex(OverriddenMD);
-        }
-        
-        // We don't need to add an entry for this method.
-        continue;
-      }
-    }
-    
-    if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
-      if (MD->isImplicit()) {
-        assert(!ImplicitVirtualDtor && 
-               "Did already see an implicit virtual dtor!");
-        ImplicitVirtualDtor = DD;
-        continue;
-      } 
-
-      // Add the complete dtor.
-      MethodVTableIndices[GlobalDecl(DD, Dtor_Complete)] = CurrentIndex++;
-      
-      // Add the deleting dtor.
-      MethodVTableIndices[GlobalDecl(DD, Dtor_Deleting)] = CurrentIndex++;
-    } else {
-      // Add the entry.
-      MethodVTableIndices[MD] = CurrentIndex++;
-    }
-  }
-
-  if (ImplicitVirtualDtor) {
-    // Itanium C++ ABI 2.5.2:
-    //   If a class has an implicitly-defined virtual destructor, 
-    //   its entries come after the declared virtual function pointers.
-
-    // Add the complete dtor.
-    MethodVTableIndices[GlobalDecl(ImplicitVirtualDtor, Dtor_Complete)] = 
-      CurrentIndex++;
-    
-    // Add the deleting dtor.
-    MethodVTableIndices[GlobalDecl(ImplicitVirtualDtor, Dtor_Deleting)] = 
-      CurrentIndex++;
-  }
-  
-  NumVirtualFunctionPointers[RD] = CurrentIndex;
-}
+CodeGenVTables::CodeGenVTables(CodeGenModule &CGM)
+  : CGM(CGM), VTContext(CGM.getContext()) { }
 
 bool CodeGenVTables::ShouldEmitVTableInThisTU(const CXXRecordDecl *RD) {
   assert(RD->isDynamicClass() && "Non dynamic classes have no VTable.");
@@ -2449,75 +58,6 @@ bool CodeGenVTables::ShouldEmitVTableInThisTU(const CXXRecordDecl *RD) {
   return KeyFunction->hasBody();
 }
 
-uint64_t CodeGenVTables::getNumVirtualFunctionPointers(const CXXRecordDecl *RD) {
-  llvm::DenseMap<const CXXRecordDecl *, uint64_t>::iterator I = 
-    NumVirtualFunctionPointers.find(RD);
-  if (I != NumVirtualFunctionPointers.end())
-    return I->second;
-
-  ComputeMethodVTableIndices(RD);
-
-  I = NumVirtualFunctionPointers.find(RD);
-  assert(I != NumVirtualFunctionPointers.end() && "Did not find entry!");
-  return I->second;
-}
-      
-uint64_t CodeGenVTables::getMethodVTableIndex(GlobalDecl GD) {
-  MethodVTableIndicesTy::iterator I = MethodVTableIndices.find(GD);
-  if (I != MethodVTableIndices.end())
-    return I->second;
-  
-  const CXXRecordDecl *RD = cast<CXXMethodDecl>(GD.getDecl())->getParent();
-
-  ComputeMethodVTableIndices(RD);
-
-  I = MethodVTableIndices.find(GD);
-  assert(I != MethodVTableIndices.end() && "Did not find index!");
-  return I->second;
-}
-
-CharUnits 
-CodeGenVTables::getVirtualBaseOffsetOffset(const CXXRecordDecl *RD, 
-                                           const CXXRecordDecl *VBase) {
-  ClassPairTy ClassPair(RD, VBase);
-  
-  VirtualBaseClassOffsetOffsetsMapTy::iterator I = 
-    VirtualBaseClassOffsetOffsets.find(ClassPair);
-  if (I != VirtualBaseClassOffsetOffsets.end())
-    return I->second;
-  
-  VCallAndVBaseOffsetBuilder Builder(RD, RD, /*FinalOverriders=*/0,
-                                     BaseSubobject(RD, CharUnits::Zero()),
-                                     /*BaseIsVirtual=*/false,
-                                     /*OffsetInLayoutClass=*/CharUnits::Zero());
-
-  for (VCallAndVBaseOffsetBuilder::VBaseOffsetOffsetsMapTy::const_iterator I =
-       Builder.getVBaseOffsetOffsets().begin(), 
-       E = Builder.getVBaseOffsetOffsets().end(); I != E; ++I) {
-    // Insert all types.
-    ClassPairTy ClassPair(RD, I->first);
-    
-    VirtualBaseClassOffsetOffsets.insert(
-        std::make_pair(ClassPair, I->second));
-  }
-  
-  I = VirtualBaseClassOffsetOffsets.find(ClassPair);
-  assert(I != VirtualBaseClassOffsetOffsets.end() && "Did not find index!");
-  
-  return I->second;
-}
-
-uint64_t
-CodeGenVTables::getAddressPoint(BaseSubobject Base, const CXXRecordDecl *RD) {
-  assert(AddressPoints.count(std::make_pair(RD, Base)) &&
-         "Did not find address point!");
-
-  uint64_t AddressPoint = AddressPoints.lookup(std::make_pair(RD, Base));
-  assert(AddressPoint && "Address point must not be zero!");
-
-  return AddressPoint;
-}
-
 llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD, 
                                               const ThunkInfo &Thunk) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
@@ -2532,7 +72,7 @@ llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD,
     getCXXABI().getMangleContext().mangleThunk(MD, Thunk, Out);
   Out.flush();
 
-  const llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(GD);
+  llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(GD);
   return GetOrCreateLLVMFunction(Name, Ty, GD, /*ForVTable=*/true);
 }
 
@@ -2543,7 +83,7 @@ static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
   if (!NonVirtualAdjustment && !VirtualAdjustment)
     return Ptr;
 
-  const llvm::Type *Int8PtrTy = 
+  llvm::Type *Int8PtrTy = 
     llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   
   llvm::Value *V = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
@@ -2554,7 +94,7 @@ static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
   }
 
   if (VirtualAdjustment) {
-    const llvm::Type *PtrDiffTy = 
+    llvm::Type *PtrDiffTy = 
       CGF.ConvertType(CGF.getContext().getPointerDiffType());
 
     // Do the virtual adjustment.
@@ -2704,7 +244,7 @@ void CodeGenFunction::GenerateVarArgsThunk(
   QualType ResultType = FPT->getResultType();
 
   // Get the original function
-  const llvm::Type *Ty =
+  llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(FnInfo, /*IsVariadic*/true);
   llvm::Value *Callee = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true);
   llvm::Function *BaseFn = cast<llvm::Function>(Callee);
@@ -2811,7 +351,7 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn,
   }
 
   // Get our callee.
-  const llvm::Type *Ty =
+  llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(GD),
                                    FPT->isVariadic());
   llvm::Value *Callee = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true);
@@ -2881,7 +421,7 @@ void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk,
            "Shouldn't replace non-declaration");
 
     // Remove the name from the old thunk function and get a new thunk.
-    OldThunkFn->setName(llvm::StringRef());
+    OldThunkFn->setName(StringRef());
     Entry = CGM.GetAddrOfThunk(GD, Thunk);
     
     // If needed, replace the old thunk with a bitcast.
@@ -2953,122 +493,27 @@ void CodeGenVTables::EmitThunks(GlobalDecl GD)
   if (isa<CXXDestructorDecl>(MD) && GD.getDtorType() == Dtor_Base)
     return;
 
-  const CXXRecordDecl *RD = MD->getParent();
-  
-  // Compute VTable related info for this class.
-  ComputeVTableRelatedInformation(RD, false);
-  
-  ThunksMapTy::const_iterator I = Thunks.find(MD);
-  if (I == Thunks.end()) {
-    // We did not find a thunk for this method.
-    return;
-  }
-
-  const ThunkInfoVectorTy &ThunkInfoVector = I->second;
-  for (unsigned I = 0, E = ThunkInfoVector.size(); I != E; ++I)
-    EmitThunk(GD, ThunkInfoVector[I], /*UseAvailableExternallyLinkage=*/false);
-}
-
-void CodeGenVTables::ComputeVTableRelatedInformation(const CXXRecordDecl *RD,
-                                                     bool RequireVTable) {
-  VTableLayoutData &Entry = VTableLayoutMap[RD];
-
-  // We may need to generate a definition for this vtable.
-  if (RequireVTable && !Entry.getInt()) {
-    if (ShouldEmitVTableInThisTU(RD))
-      CGM.DeferredVTables.push_back(RD);
-
-    Entry.setInt(true);
-  }
-  
-  // Check if we've computed this information before.
-  if (Entry.getPointer())
+  const VTableContext::ThunkInfoVectorTy *ThunkInfoVector =
+    VTContext.getThunkInfo(MD);
+  if (!ThunkInfoVector)
     return;
 
-  VTableBuilder Builder(*this, RD, CharUnits::Zero(), 
-                        /*MostDerivedClassIsVirtual=*/0, RD);
-
-  // Add the VTable layout.
-  uint64_t NumVTableComponents = Builder.getNumVTableComponents();
-  // -fapple-kext adds an extra entry at end of vtbl.
-  bool IsAppleKext = CGM.getContext().getLangOptions().AppleKext;
-  if (IsAppleKext)
-    NumVTableComponents += 1;
-
-  uint64_t *LayoutData = new uint64_t[NumVTableComponents + 1];
-  if (IsAppleKext)
-    LayoutData[NumVTableComponents] = 0;
-  Entry.setPointer(LayoutData);
-
-  // Store the number of components.
-  LayoutData[0] = NumVTableComponents;
-
-  // Store the components.
-  std::copy(Builder.vtable_components_data_begin(),
-            Builder.vtable_components_data_end(),
-            &LayoutData[1]);
-
-  // Add the known thunks.
-  Thunks.insert(Builder.thunks_begin(), Builder.thunks_end());
-  
-  // Add the thunks needed in this vtable.
-  assert(!VTableThunksMap.count(RD) && 
-         "Thunks already exists for this vtable!");
-
-  VTableThunksTy &VTableThunks = VTableThunksMap[RD];
-  VTableThunks.append(Builder.vtable_thunks_begin(),
-                      Builder.vtable_thunks_end());
-  
-  // Sort them.
-  std::sort(VTableThunks.begin(), VTableThunks.end());
-  
-  // Add the address points.
-  for (VTableBuilder::AddressPointsMapTy::const_iterator I =
-       Builder.address_points_begin(), E = Builder.address_points_end();
-       I != E; ++I) {
-    
-    uint64_t &AddressPoint = AddressPoints[std::make_pair(RD, I->first)];
-    
-    // Check if we already have the address points for this base.
-    assert(!AddressPoint && "Address point already exists for this base!");
-    
-    AddressPoint = I->second;
-  }
-  
-  // If we don't have the vbase information for this class, insert it.
-  // getVirtualBaseOffsetOffset will compute it separately without computing
-  // the rest of the vtable related information.
-  if (!RD->getNumVBases())
-    return;
-  
-  const RecordType *VBaseRT = 
-    RD->vbases_begin()->getType()->getAs<RecordType>();
-  const CXXRecordDecl *VBase = cast<CXXRecordDecl>(VBaseRT->getDecl());
-  
-  if (VirtualBaseClassOffsetOffsets.count(std::make_pair(RD, VBase)))
-    return;
-  
-  for (VTableBuilder::VBaseOffsetOffsetsMapTy::const_iterator I =
-       Builder.getVBaseOffsetOffsets().begin(), 
-       E = Builder.getVBaseOffsetOffsets().end(); I != E; ++I) {
-    // Insert all types.
-    ClassPairTy ClassPair(RD, I->first);
-    
-    VirtualBaseClassOffsetOffsets.insert(
-        std::make_pair(ClassPair, I->second));
-  }
+  for (unsigned I = 0, E = ThunkInfoVector->size(); I != E; ++I)
+    EmitThunk(GD, (*ThunkInfoVector)[I],
+              /*UseAvailableExternallyLinkage=*/false);
 }
 
 llvm::Constant *
 CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
-                                        const uint64_t *Components, 
+                                        const VTableComponent *Components, 
                                         unsigned NumComponents,
-                                        const VTableThunksTy &VTableThunks) {
-  llvm::SmallVector<llvm::Constant *, 64> Inits;
+                                const VTableLayout::VTableThunkTy *VTableThunks,
+                                        unsigned NumVTableThunks) {
+  SmallVector<llvm::Constant *, 64> Inits;
 
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
+  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
   
-  const llvm::Type *PtrDiffTy = 
+  llvm::Type *PtrDiffTy = 
     CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
 
   QualType ClassType = CGM.getContext().getTagDeclType(RD);
@@ -3079,8 +524,7 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
   llvm::Constant* PureVirtualFn = 0;
 
   for (unsigned I = 0; I != NumComponents; ++I) {
-    VTableComponent Component = 
-      VTableComponent::getFromOpaqueInteger(Components[I]);
+    VTableComponent Component = Components[I];
 
     llvm::Constant *Init = 0;
 
@@ -3126,7 +570,7 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
       if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) {
         // We have a pure virtual member function.
         if (!PureVirtualFn) {
-          const llvm::FunctionType *Ty = 
+          llvm::FunctionType *Ty = 
             llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), 
                                     /*isVarArg=*/false);
           PureVirtualFn = 
@@ -3138,7 +582,7 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
         Init = PureVirtualFn;
       } else {
         // Check if we should use a thunk.
-        if (NextVTableThunkIndex < VTableThunks.size() &&
+        if (NextVTableThunkIndex < NumVTableThunks &&
             VTableThunks[NextVTableThunkIndex].first == I) {
           const ThunkInfo &Thunk = VTableThunks[NextVTableThunkIndex].second;
         
@@ -3147,7 +591,7 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
 
           NextVTableThunkIndex++;
         } else {
-          const llvm::Type *Ty = CGM.getTypes().GetFunctionTypeForVTable(GD);
+          llvm::Type *Ty = CGM.getTypes().GetFunctionTypeForVTable(GD);
         
           Init = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true);
         }
@@ -3170,46 +614,45 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
 }
 
 llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTable(const CXXRecordDecl *RD) {
+  llvm::GlobalVariable *&VTable = VTables[RD];
+  if (VTable)
+    return VTable;
+
+  // We may need to generate a definition for this vtable.
+  if (ShouldEmitVTableInThisTU(RD))
+    CGM.DeferredVTables.push_back(RD);
+
   llvm::SmallString<256> OutName;
   llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().mangleCXXVTable(RD, Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
-  ComputeVTableRelatedInformation(RD, /*VTableRequired=*/true);
-  
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
+  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
   llvm::ArrayType *ArrayType = 
-    llvm::ArrayType::get(Int8PtrTy, getNumVTableComponents(RD));
+    llvm::ArrayType::get(Int8PtrTy,
+                        VTContext.getVTableLayout(RD).getNumVTableComponents());
 
-  llvm::GlobalVariable *GV =
+  VTable =
     CGM.CreateOrReplaceCXXRuntimeVariable(Name, ArrayType, 
                                           llvm::GlobalValue::ExternalLinkage);
-  GV->setUnnamedAddr(true);
-  return GV;
+  VTable->setUnnamedAddr(true);
+  return VTable;
 }
 
 void
 CodeGenVTables::EmitVTableDefinition(llvm::GlobalVariable *VTable,
                                      llvm::GlobalVariable::LinkageTypes Linkage,
                                      const CXXRecordDecl *RD) {
-  // Dump the vtable layout if necessary.
-  if (CGM.getLangOptions().DumpVTableLayouts) {
-    VTableBuilder Builder(*this, RD, CharUnits::Zero(), 
-                          /*MostDerivedClassIsVirtual=*/0, RD);
+  const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
 
-    Builder.dumpLayout(llvm::errs());
-  }
-
-  assert(VTableThunksMap.count(RD) && 
-         "No thunk status for this record decl!");
-  
-  const VTableThunksTy& Thunks = VTableThunksMap[RD];
-  
   // Create and set the initializer.
   llvm::Constant *Init = 
-    CreateVTableInitializer(RD, getVTableComponentsData(RD),
-                            getNumVTableComponents(RD), Thunks);
+    CreateVTableInitializer(RD,
+                            VTLayout.vtable_component_begin(),
+                            VTLayout.getNumVTableComponents(),
+                            VTLayout.vtable_thunk_begin(),
+                            VTLayout.getNumVTableThunks());
   VTable->setInitializer(Init);
   
   // Set the correct linkage.
@@ -3225,17 +668,13 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
                                       bool BaseIsVirtual, 
                                    llvm::GlobalVariable::LinkageTypes Linkage,
                                       VTableAddressPointsMapTy& AddressPoints) {
-  VTableBuilder Builder(*this, Base.getBase(), 
-                        Base.getBaseOffset(), 
-                        /*MostDerivedClassIsVirtual=*/BaseIsVirtual, RD);
-
-  // Dump the vtable layout if necessary.
-  if (CGM.getLangOptions().DumpVTableLayouts)
-    Builder.dumpLayout(llvm::errs());
+  llvm::OwningPtr<VTableLayout> VTLayout(
+    VTContext.createConstructionVTableLayout(Base.getBase(),
+                                             Base.getBaseOffset(),
+                                             BaseIsVirtual, RD));
 
   // Add the address points.
-  AddressPoints.insert(Builder.address_points_begin(),
-                       Builder.address_points_end());
+  AddressPoints = VTLayout->getAddressPoints();
 
   // Get the mangled construction vtable name.
   llvm::SmallString<256> OutName;
@@ -3244,11 +683,11 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
     mangleCXXCtorVTable(RD, Base.getBaseOffset().getQuantity(), Base.getBase(), 
                         Out);
   Out.flush();
-  llvm::StringRef Name = OutName.str();
+  StringRef Name = OutName.str();
 
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
+  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
   llvm::ArrayType *ArrayType = 
-    llvm::ArrayType::get(Int8PtrTy, Builder.getNumVTableComponents());
+    llvm::ArrayType::get(Int8PtrTy, VTLayout->getNumVTableComponents());
 
   // Create the variable that will hold the construction vtable.
   llvm::GlobalVariable *VTable = 
@@ -3258,19 +697,13 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
   // V-tables are always unnamed_addr.
   VTable->setUnnamedAddr(true);
 
-  // Add the thunks.
-  VTableThunksTy VTableThunks;
-  VTableThunks.append(Builder.vtable_thunks_begin(),
-                      Builder.vtable_thunks_end());
-
-  // Sort them.
-  std::sort(VTableThunks.begin(), VTableThunks.end());
-
   // Create and set the initializer.
   llvm::Constant *Init = 
     CreateVTableInitializer(Base.getBase(), 
-                            Builder.vtable_components_data_begin(), 
-                            Builder.getNumVTableComponents(), VTableThunks);
+                            VTLayout->vtable_component_begin(), 
+                            VTLayout->getNumVTableComponents(),
+                            VTLayout->vtable_thunk_begin(),
+                            VTLayout->getNumVTableThunks());
   VTable->setInitializer(Init);
   
   return VTable;
@@ -3279,13 +712,10 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
 void 
 CodeGenVTables::GenerateClassData(llvm::GlobalVariable::LinkageTypes Linkage,
                                   const CXXRecordDecl *RD) {
-  llvm::GlobalVariable *&VTable = VTables[RD];
-  if (VTable) {
-    assert(VTable->getInitializer() && "VTable doesn't have a definition!");
+  llvm::GlobalVariable *VTable = GetAddrOfVTable(RD);
+  if (VTable->hasInitializer())
     return;
-  }
 
-  VTable = GetAddrOfVTable(RD);
   EmitVTableDefinition(VTable, Linkage, RD);
 
   if (RD->getNumVBases()) {
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h
index eff6e56c1f80..828330e5e3c4 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h
@@ -17,8 +17,10 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/GlobalVariable.h"
 #include "clang/Basic/ABI.h"
+#include "clang/AST/BaseSubobject.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/GlobalDecl.h"
+#include "clang/AST/VTableBuilder.h"
 
 namespace clang {
   class CXXRecordDecl;
@@ -26,145 +28,18 @@ namespace clang {
 namespace CodeGen {
   class CodeGenModule;
 
-// BaseSubobject - Uniquely identifies a direct or indirect base class. 
-// Stores both the base class decl and the offset from the most derived class to
-// the base class.
-class BaseSubobject {
-  /// Base - The base class declaration.
-  const CXXRecordDecl *Base;
-  
-  /// BaseOffset - The offset from the most derived class to the base class.
-  CharUnits BaseOffset;
-  
-public:
-  BaseSubobject(const CXXRecordDecl *Base, CharUnits BaseOffset)
-    : Base(Base), BaseOffset(BaseOffset) { }
-  
-  /// getBase - Returns the base class declaration.
-  const CXXRecordDecl *getBase() const { return Base; }
-
-  /// getBaseOffset - Returns the base class offset.
-  CharUnits getBaseOffset() const { return BaseOffset; }
-
-  friend bool operator==(const BaseSubobject &LHS, const BaseSubobject &RHS) {
-    return LHS.Base == RHS.Base && LHS.BaseOffset == RHS.BaseOffset;
- }
-};
-
-} // end namespace CodeGen
-} // end namespace clang
-
-namespace llvm {
-
-template<> struct DenseMapInfo<clang::CodeGen::BaseSubobject> {
-  static clang::CodeGen::BaseSubobject getEmptyKey() {
-    return clang::CodeGen::BaseSubobject(
-      DenseMapInfo<const clang::CXXRecordDecl *>::getEmptyKey(),
-      clang::CharUnits::fromQuantity(DenseMapInfo<int64_t>::getEmptyKey()));
-  }
-
-  static clang::CodeGen::BaseSubobject getTombstoneKey() {
-    return clang::CodeGen::BaseSubobject(
-      DenseMapInfo<const clang::CXXRecordDecl *>::getTombstoneKey(),
-      clang::CharUnits::fromQuantity(DenseMapInfo<int64_t>::getTombstoneKey()));
-  }
-
-  static unsigned getHashValue(const clang::CodeGen::BaseSubobject &Base) {
-    return 
-      DenseMapInfo<const clang::CXXRecordDecl *>::getHashValue(Base.getBase()) ^
-      DenseMapInfo<int64_t>::getHashValue(Base.getBaseOffset().getQuantity());
-  }
-
-  static bool isEqual(const clang::CodeGen::BaseSubobject &LHS, 
-                      const clang::CodeGen::BaseSubobject &RHS) {
-    return LHS == RHS;
-  }
-};
-
-// It's OK to treat BaseSubobject as a POD type.
-template <> struct isPodLike<clang::CodeGen::BaseSubobject> {
-  static const bool value = true;
-};
-
-}
-
-namespace clang {
-namespace CodeGen {
-
 class CodeGenVTables {
   CodeGenModule &CGM;
 
-  /// MethodVTableIndices - Contains the index (relative to the vtable address
-  /// point) where the function pointer for a virtual function is stored.
-  typedef llvm::DenseMap<GlobalDecl, int64_t> MethodVTableIndicesTy;
-  MethodVTableIndicesTy MethodVTableIndices;
-
-  typedef std::pair<const CXXRecordDecl *,
-                    const CXXRecordDecl *> ClassPairTy;
-
-  /// VirtualBaseClassOffsetOffsets - Contains the vtable offset (relative to 
-  /// the address point) in chars where the offsets for virtual bases of a class
-  /// are stored.
-  typedef llvm::DenseMap<ClassPairTy, CharUnits> 
-    VirtualBaseClassOffsetOffsetsMapTy;
-  VirtualBaseClassOffsetOffsetsMapTy VirtualBaseClassOffsetOffsets;
+  VTableContext VTContext;
 
   /// VTables - All the vtables which have been defined.
   llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
   
-  /// NumVirtualFunctionPointers - Contains the number of virtual function 
-  /// pointers in the vtable for a given record decl.
-  llvm::DenseMap<const CXXRecordDecl *, uint64_t> NumVirtualFunctionPointers;
-
-  typedef llvm::SmallVector<ThunkInfo, 1> ThunkInfoVectorTy;
-  typedef llvm::DenseMap<const CXXMethodDecl *, ThunkInfoVectorTy> ThunksMapTy;
-  
-  /// Thunks - Contains all thunks that a given method decl will need.
-  ThunksMapTy Thunks;
-
-  // The layout entry and a bool indicating whether we've actually emitted
-  // the vtable.
-  typedef llvm::PointerIntPair<uint64_t *, 1, bool> VTableLayoutData;
-  typedef llvm::DenseMap<const CXXRecordDecl *, VTableLayoutData>
-    VTableLayoutMapTy;
-  
-  /// VTableLayoutMap - Stores the vtable layout for all record decls.
-  /// The layout is stored as an array of 64-bit integers, where the first
-  /// integer is the number of vtable entries in the layout, and the subsequent
-  /// integers are the vtable components.
-  VTableLayoutMapTy VTableLayoutMap;
-
-  typedef std::pair<const CXXRecordDecl *, BaseSubobject> BaseSubobjectPairTy;
-  typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> AddressPointsMapTy;
-  
-  /// Address points - Address points for all vtables.
-  AddressPointsMapTy AddressPoints;
-
   /// VTableAddressPointsMapTy - Address points for a single vtable.
   typedef llvm::DenseMap<BaseSubobject, uint64_t> VTableAddressPointsMapTy;
 
-  typedef llvm::SmallVector<std::pair<uint64_t, ThunkInfo>, 1> 
-    VTableThunksTy;
-  
-  typedef llvm::DenseMap<const CXXRecordDecl *, VTableThunksTy>
-    VTableThunksMapTy;
-  
-  /// VTableThunksMap - Contains thunks needed by vtables.
-  VTableThunksMapTy VTableThunksMap;
-  
-  uint64_t getNumVTableComponents(const CXXRecordDecl *RD) const {
-    assert(VTableLayoutMap.count(RD) && "No vtable layout for this class!");
-    
-    return VTableLayoutMap.lookup(RD).getPointer()[0];
-  }
-
-  const uint64_t *getVTableComponentsData(const CXXRecordDecl *RD) const {
-    assert(VTableLayoutMap.count(RD) && "No vtable layout for this class!");
-
-    uint64_t *Components = VTableLayoutMap.lookup(RD).getPointer();
-    return &Components[1];
-  }
-
+  typedef std::pair<const CXXRecordDecl *, BaseSubobject> BaseSubobjectPairTy;
   typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SubVTTIndiciesMapTy;
   
   /// SubVTTIndicies - Contains indices into the various sub-VTTs.
@@ -177,12 +52,6 @@ class CodeGenVTables {
   /// indices.
   SecondaryVirtualPointerIndicesMapTy SecondaryVirtualPointerIndices;
 
-  /// getNumVirtualFunctionPointers - Return the number of virtual function
-  /// pointers in the vtable for a given record decl.
-  uint64_t getNumVirtualFunctionPointers(const CXXRecordDecl *RD);
-  
-  void ComputeMethodVTableIndices(const CXXRecordDecl *RD);
-
   /// EmitThunk - Emit a single thunk.
   void EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk, 
                  bool UseAvailableExternallyLinkage);
@@ -193,24 +62,20 @@ class CodeGenVTables {
   /// doesn't contain any incomplete types.
   void MaybeEmitThunkAvailableExternally(GlobalDecl GD, const ThunkInfo &Thunk);
 
-  /// ComputeVTableRelatedInformation - Compute and store all vtable related
-  /// information (vtable layout, vbase offset offsets, thunks etc) for the
-  /// given record decl.
-  void ComputeVTableRelatedInformation(const CXXRecordDecl *RD,
-                                       bool VTableRequired);
-
   /// CreateVTableInitializer - Create a vtable initializer for the given record
   /// decl.
   /// \param Components - The vtable components; this is really an array of
   /// VTableComponents.
   llvm::Constant *CreateVTableInitializer(const CXXRecordDecl *RD,
-                                          const uint64_t *Components, 
+                                          const VTableComponent *Components, 
                                           unsigned NumComponents,
-                                          const VTableThunksTy &VTableThunks);
+                                const VTableLayout::VTableThunkTy *VTableThunks,
+                                          unsigned NumVTableThunks);
 
 public:
-  CodeGenVTables(CodeGenModule &CGM)
-    : CGM(CGM) { }
+  CodeGenVTables(CodeGenModule &CGM);
+
+  VTableContext &getVTableContext() { return VTContext; }
 
   /// \brief True if the VTable of this record must be emitted in the
   /// translation unit.
@@ -230,19 +95,6 @@ class CodeGenVTables {
   uint64_t getSecondaryVirtualPointerIndex(const CXXRecordDecl *RD,
                                            BaseSubobject Base);
 
-  /// getMethodVTableIndex - Return the index (relative to the vtable address
-  /// point) where the function pointer for the given virtual function is
-  /// stored.
-  uint64_t getMethodVTableIndex(GlobalDecl GD);
-
-  /// getVirtualBaseOffsetOffset - Return the offset in chars (relative to the
-  /// vtable address point) where the offset of the virtual base that contains 
-  /// the given base is stored, otherwise, if no virtual base contains the given
-  /// class, return 0.  Base must be a virtual base class or an unambigious
-  /// base.
-  CharUnits getVirtualBaseOffsetOffset(const CXXRecordDecl *RD,
-                                       const CXXRecordDecl *VBase);
-
   /// getAddressPoint - Get the address point of the given subobject in the
   /// class decl.
   uint64_t getAddressPoint(BaseSubobject Base, const CXXRecordDecl *RD);
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h b/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h
index 4d0b8410e451..489e600b3ddf 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h
@@ -337,65 +337,90 @@ class AggValueSlot {
 
   // Qualifiers
   Qualifiers Quals;
+
+  /// DestructedFlag - This is set to true if some external code is
+  /// responsible for setting up a destructor for the slot.  Otherwise
+  /// the code which constructs it should push the appropriate cleanup.
+  bool DestructedFlag : 1;
+
+  /// ObjCGCFlag - This is set to true if writing to the memory in the
+  /// slot might require calling an appropriate Objective-C GC
+  /// barrier.  The exact interaction here is unnecessarily mysterious.
+  bool ObjCGCFlag : 1;
   
-  // Associated flags.
-  bool LifetimeFlag : 1;
-  bool RequiresGCollection : 1;
-  
-  /// IsZeroed - This is set to true if the destination is known to be zero
-  /// before the assignment into it.  This means that zero fields don't need to
-  /// be set.
-  bool IsZeroed : 1;
+  /// ZeroedFlag - This is set to true if the memory in the slot is
+  /// known to be zero before the assignment into it.  This means that
+  /// zero fields don't need to be set.
+  bool ZeroedFlag : 1;
+
+  /// AliasedFlag - This is set to true if the slot might be aliased
+  /// and it's not undefined behavior to access it through such an
+  /// alias.  Note that it's always undefined behavior to access a C++
+  /// object that's under construction through an alias derived from
+  /// outside the construction process.
+  ///
+  /// This flag controls whether calls that produce the aggregate
+  /// value may be evaluated directly into the slot, or whether they
+  /// must be evaluated into an unaliased temporary and then memcpy'ed
+  /// over.  Since it's invalid in general to memcpy a non-POD C++
+  /// object, it's important that this flag never be set when
+  /// evaluating an expression which constructs such an object.
+  bool AliasedFlag : 1;
 
 public:
+  enum IsAliased_t { IsNotAliased, IsAliased };
+  enum IsDestructed_t { IsNotDestructed, IsDestructed };
+  enum IsZeroed_t { IsNotZeroed, IsZeroed };
+  enum NeedsGCBarriers_t { DoesNotNeedGCBarriers, NeedsGCBarriers };
+
   /// ignored - Returns an aggregate value slot indicating that the
   /// aggregate value is being ignored.
   static AggValueSlot ignored() {
     AggValueSlot AV;
     AV.Addr = 0;
     AV.Quals = Qualifiers();
-    AV.LifetimeFlag = AV.RequiresGCollection = AV.IsZeroed =0;
+    AV.DestructedFlag = AV.ObjCGCFlag = AV.ZeroedFlag = AV.AliasedFlag = false;
     return AV;
   }
 
   /// forAddr - Make a slot for an aggregate value.
   ///
-  /// \param Volatile - true if the slot should be volatile-initialized
+  /// \param quals - The qualifiers that dictate how the slot should
+  /// be initialied. Only 'volatile' and the Objective-C lifetime
+  /// qualifiers matter.
   ///
-  /// \param Qualifiers - The qualifiers that dictate how the slot
-  /// should be initialied. Only 'volatile' and the Objective-C
-  /// lifetime qualifiers matter.
-  ///
-  /// \param LifetimeExternallyManaged - true if the slot's lifetime
-  ///   is being externally managed; false if a destructor should be
-  ///   registered for any temporaries evaluated into the slot
-  /// \param RequiresGCollection - true if the slot is located
+  /// \param isDestructed - true if something else is responsible
+  ///   for calling destructors on this object
+  /// \param needsGC - true if the slot is potentially located
   ///   somewhere that ObjC GC calls should be emitted for
-  static AggValueSlot forAddr(llvm::Value *Addr, Qualifiers Quals,
-                              bool LifetimeExternallyManaged,
-                              bool RequiresGCollection = false,
-                              bool IsZeroed = false) {
+  static AggValueSlot forAddr(llvm::Value *addr, Qualifiers quals,
+                              IsDestructed_t isDestructed,
+                              NeedsGCBarriers_t needsGC,
+                              IsAliased_t isAliased,
+                              IsZeroed_t isZeroed = IsNotZeroed) {
     AggValueSlot AV;
-    AV.Addr = Addr;
-    AV.Quals = Quals;
-    AV.LifetimeFlag = LifetimeExternallyManaged;
-    AV.RequiresGCollection = RequiresGCollection;
-    AV.IsZeroed = IsZeroed;
+    AV.Addr = addr;
+    AV.Quals = quals;
+    AV.DestructedFlag = isDestructed;
+    AV.ObjCGCFlag = needsGC;
+    AV.ZeroedFlag = isZeroed;
+    AV.AliasedFlag = isAliased;
     return AV;
   }
 
-  static AggValueSlot forLValue(LValue LV, bool LifetimeExternallyManaged,
-                                bool RequiresGCollection = false,
-                                bool IsZeroed = false) {
+  static AggValueSlot forLValue(LValue LV, IsDestructed_t isDestructed,
+                                NeedsGCBarriers_t needsGC,
+                                IsAliased_t isAliased,
+                                IsZeroed_t isZeroed = IsNotZeroed) {
     return forAddr(LV.getAddress(), LV.getQuals(),
-                   LifetimeExternallyManaged, RequiresGCollection, IsZeroed);
+                   isDestructed, needsGC, isAliased, isZeroed);
   }
 
-  bool isLifetimeExternallyManaged() const {
-    return LifetimeFlag;
+  IsDestructed_t isExternallyDestructed() const {
+    return IsDestructed_t(DestructedFlag);
   }
-  void setLifetimeExternallyManaged(bool Managed = true) {
-    LifetimeFlag = Managed;
+  void setExternallyDestructed(bool destructed = true) {
+    DestructedFlag = destructed;
   }
 
   Qualifiers getQualifiers() const { return Quals; }
@@ -408,8 +433,8 @@ class AggValueSlot {
     return Quals.getObjCLifetime();
   }
 
-  bool requiresGCollection() const {
-    return RequiresGCollection;
+  NeedsGCBarriers_t requiresGCollection() const {
+    return NeedsGCBarriers_t(ObjCGCFlag);
   }
   
   llvm::Value *getAddr() const {
@@ -420,13 +445,17 @@ class AggValueSlot {
     return Addr == 0;
   }
 
+  IsAliased_t isPotentiallyAliased() const {
+    return IsAliased_t(AliasedFlag);
+  }
+
   RValue asRValue() const {
     return RValue::getAggregate(getAddr(), isVolatile());
   }
   
-  void setZeroed(bool V = true) { IsZeroed = V; }
-  bool isZeroed() const {
-    return IsZeroed;
+  void setZeroed(bool V = true) { ZeroedFlag = V; }
+  IsZeroed_t isZeroed() const {
+    return IsZeroed_t(ZeroedFlag);
   }
 };
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp
index 263e01e4f18a..68dd5c94dc01 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp
@@ -30,12 +30,12 @@ using namespace llvm;
 
 namespace clang {
   class BackendConsumer : public ASTConsumer {
-    Diagnostic &Diags;
+    DiagnosticsEngine &Diags;
     BackendAction Action;
     const CodeGenOptions &CodeGenOpts;
     const TargetOptions &TargetOpts;
     const LangOptions &LangOpts;
-    llvm::raw_ostream *AsmOutStream;
+    raw_ostream *AsmOutStream;
     ASTContext *Context;
 
     Timer LLVMIRGeneration;
@@ -45,12 +45,12 @@ namespace clang {
     llvm::OwningPtr<llvm::Module> TheModule;
 
   public:
-    BackendConsumer(BackendAction action, Diagnostic &_Diags,
+    BackendConsumer(BackendAction action, DiagnosticsEngine &_Diags,
                     const CodeGenOptions &compopts,
                     const TargetOptions &targetopts,
                     const LangOptions &langopts,
                     bool TimePasses,
-                    const std::string &infile, llvm::raw_ostream *OS,
+                    const std::string &infile, raw_ostream *OS,
                     LLVMContext &C) :
       Diags(_Diags),
       Action(action),
@@ -185,7 +185,7 @@ static FullSourceLoc ConvertBackendLocation(const llvm::SMDiagnostic &D,
   // Translate the offset into the file.
   unsigned Offset = D.getLoc().getPointer()  - LBuf->getBufferStart();
   SourceLocation NewLoc =
-  CSM.getLocForStartOfFile(FID).getFileLocWithOffset(Offset);
+  CSM.getLocForStartOfFile(FID).getLocWithOffset(Offset);
   return FullSourceLoc(NewLoc, CSM);
 }
 
@@ -199,7 +199,7 @@ void BackendConsumer::InlineAsmDiagHandler2(const llvm::SMDiagnostic &D,
   // we re-format the SMDiagnostic in terms of a clang diagnostic.
 
   // Strip "error: " off the start of the message string.
-  llvm::StringRef Message = D.getMessage();
+  StringRef Message = D.getMessage();
   if (Message.startswith("error: "))
     Message = Message.substr(7);
 
@@ -259,7 +259,7 @@ llvm::LLVMContext *CodeGenAction::takeLLVMContext() {
 }
 
 static raw_ostream *GetOutputStream(CompilerInstance &CI,
-                                    llvm::StringRef InFile,
+                                    StringRef InFile,
                                     BackendAction Action) {
   switch (Action) {
   case Backend_EmitAssembly:
@@ -275,14 +275,13 @@ static raw_ostream *GetOutputStream(CompilerInstance &CI,
     return CI.createDefaultOutputFile(true, InFile, "o");
   }
 
-  assert(0 && "Invalid action!");
-  return 0;
+  llvm_unreachable("Invalid action!");
 }
 
 ASTConsumer *CodeGenAction::CreateASTConsumer(CompilerInstance &CI,
-                                              llvm::StringRef InFile) {
+                                              StringRef InFile) {
   BackendAction BA = static_cast<BackendAction>(Act);
-  llvm::OwningPtr<llvm::raw_ostream> OS(GetOutputStream(CI, InFile, BA));
+  llvm::OwningPtr<raw_ostream> OS(GetOutputStream(CI, InFile, BA));
   if (BA != Backend_EmitNothing && !OS)
     return 0;
 
@@ -320,17 +319,17 @@ void CodeGenAction::ExecuteAction() {
     TheModule.reset(ParseIR(MainFileCopy, Err, *VMContext));
     if (!TheModule) {
       // Translate from the diagnostic info to the SourceManager location.
-      SourceLocation Loc = SM.getLocation(
+      SourceLocation Loc = SM.translateFileLineCol(
         SM.getFileEntryForID(SM.getMainFileID()), Err.getLineNo(),
         Err.getColumnNo() + 1);
 
       // Get a custom diagnostic for the error. We strip off a leading
       // diagnostic code if there is one.
-      llvm::StringRef Msg = Err.getMessage();
+      StringRef Msg = Err.getMessage();
       if (Msg.startswith("error: "))
         Msg = Msg.substr(7);
-      unsigned DiagID = CI.getDiagnostics().getCustomDiagID(Diagnostic::Error,
-                                                            Msg);
+      unsigned DiagID = CI.getDiagnostics().getCustomDiagID(
+          DiagnosticsEngine::Error, Msg);
 
       CI.getDiagnostics().Report(Loc, DiagID);
       return;
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp
index 702897a7c448..8191f021da4a 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -13,6 +13,7 @@
 
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CGDebugInfo.h"
 #include "CGException.h"
@@ -30,10 +31,10 @@ using namespace CodeGen;
 
 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm)
   : CodeGenTypeCache(cgm), CGM(cgm),
-    Target(CGM.getContext().Target), Builder(cgm.getModule().getContext()),
+    Target(CGM.getContext().getTargetInfo()), Builder(cgm.getModule().getContext()),
     AutoreleaseResult(false), BlockInfo(0), BlockPointer(0),
-    NormalCleanupDest(0), EHCleanupDest(0), NextCleanupDestIndex(1),
-    ExceptionSlot(0), EHSelectorSlot(0),
+    NormalCleanupDest(0), NextCleanupDestIndex(1),
+    EHResumeBlock(0), ExceptionSlot(0), EHSelectorSlot(0),
     DebugInfo(0), DisableDebugInfo(false), DidCallStackSave(false),
     IndirectBranch(0), SwitchInsn(0), CaseRangeBlock(0), UnreachableBlock(0),
     CXXThisDecl(0), CXXThisValue(0), CXXVTTDecl(0), CXXVTTValue(0),
@@ -86,6 +87,10 @@ bool CodeGenFunction::hasAggregateLLVMType(QualType type) {
   case Type::ObjCObject:
   case Type::ObjCInterface:
     return true;
+
+  // In IRGen, atomic types are just the underlying type
+  case Type::Atomic:
+    return hasAggregateLLVMType(type->getAs<AtomicType>()->getValueType());
   }
   llvm_unreachable("unknown type kind!");
 }
@@ -116,7 +121,8 @@ void CodeGenFunction::EmitReturnBlock() {
       dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->use_begin());
     if (BI && BI->isUnconditional() &&
         BI->getSuccessor(0) == ReturnBlock.getBlock()) {
-      // Reset insertion point and delete the branch.
+      // Reset insertion point, including debug location, and delete the branch.
+      Builder.SetCurrentDebugLocation(BI->getDebugLoc());
       Builder.SetInsertPoint(BI->getParent());
       BI->eraseFromParent();
       delete ReturnBlock.getBlock();
@@ -189,7 +195,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     }
   }
 
-  EmitIfUsed(*this, RethrowBlock.getBlock());
+  EmitIfUsed(*this, EHResumeBlock);
   EmitIfUsed(*this, TerminateLandingPad);
   EmitIfUsed(*this, TerminateHandler);
   EmitIfUsed(*this, UnreachableBlock);
@@ -215,7 +221,7 @@ void CodeGenFunction::EmitFunctionInstrumentation(const char *Fn) {
   // void __cyg_profile_func_{enter,exit} (void *this_fn, void *call_site);
   llvm::PointerType *PointerTy = Int8PtrTy;
   llvm::Type *ProfileFuncArgs[] = { PointerTy, PointerTy };
-  const llvm::FunctionType *FunctionTy =
+  llvm::FunctionType *FunctionTy =
     llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
                             ProfileFuncArgs, false);
 
@@ -345,6 +351,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     if (Ty->isVariablyModifiedType())
       EmitVariablyModifiedType(Ty);
   }
+  // Emit a location at the end of the prologue.
+  if (CGDebugInfo *DI = getDebugInfo())
+    DI->EmitLocation(Builder, StartLoc);
 }
 
 void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args) {
@@ -364,9 +373,12 @@ static void TryMarkNoThrow(llvm::Function *F) {
   for (llvm::Function::iterator FI = F->begin(), FE = F->end(); FI != FE; ++FI)
     for (llvm::BasicBlock::iterator
            BI = FI->begin(), BE = FI->end(); BI != BE; ++BI)
-      if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(&*BI))
+      if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(&*BI)) {
         if (!Call->doesNotThrow())
           return;
+      } else if (isa<llvm::ResumeInst>(&*BI)) {
+        return;
+      }
   F->setDoesNotThrow(true);
 }
 
@@ -400,6 +412,10 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
     EmitDestructorBody(Args);
   else if (isa<CXXConstructorDecl>(FD))
     EmitConstructorBody(Args);
+  else if (getContext().getLangOptions().CUDA &&
+           !CGM.getCodeGenOpts().CUDAIsDevice &&
+           FD->hasAttr<CUDAGlobalAttr>())
+    CGM.getCUDARuntime().EmitDeviceStubBody(*this, Args);
   else
     EmitFunctionBody(Args);
 
@@ -645,7 +661,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
   llvm::Value *baseSizeInChars
     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSizeAndAlign.first.getQuantity());
 
-  const llvm::Type *i8p = Builder.getInt8PtrTy();
+  llvm::Type *i8p = Builder.getInt8PtrTy();
 
   llvm::Value *begin = Builder.CreateBitCast(dest, i8p, "vla.begin");
   llvm::Value *end = Builder.CreateInBoundsGEP(dest, sizeInChars, "vla.end");
@@ -690,9 +706,9 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
   // Cast the dest ptr to the appropriate i8 pointer type.
   unsigned DestAS =
     cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
-  const llvm::Type *BP = Builder.getInt8PtrTy(DestAS);
+  llvm::Type *BP = Builder.getInt8PtrTy(DestAS);
   if (DestPtr->getType() != BP)
-    DestPtr = Builder.CreateBitCast(DestPtr, BP, "tmp");
+    DestPtr = Builder.CreateBitCast(DestPtr, BP);
 
   // Get size and alignment info for this aggregate.
   std::pair<CharUnits, CharUnits> TypeInfo = 
@@ -740,7 +756,7 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
       new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
                                /*isConstant=*/true, 
                                llvm::GlobalVariable::PrivateLinkage,
-                               NullConstant, llvm::Twine());
+                               NullConstant, Twine());
     llvm::Value *SrcPtr =
       Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy());
 
@@ -818,7 +834,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
   // We have some number of constant-length arrays, so addr should
   // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
   // down to the first element of addr.
-  llvm::SmallVector<llvm::Value*, 8> gepIndices;
+  SmallVector<llvm::Value*, 8> gepIndices;
 
   // GEP down to the array type.
   llvm::ConstantInt *zero = Builder.getInt32(0);
@@ -828,7 +844,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
   // constant-length arrays than to re-evaluate the array bounds.
   uint64_t countFromCLAs = 1;
 
-  const llvm::ArrayType *llvmArrayType =
+  llvm::ArrayType *llvmArrayType =
     cast<llvm::ArrayType>(
       cast<llvm::PointerType>(addr->getType())->getElementType());
   while (true) {
@@ -850,8 +866,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
   baseType = arrayType->getElementType();
 
   // Create the actual GEP.
-  addr = Builder.CreateInBoundsGEP(addr, gepIndices.begin(),
-                                   gepIndices.end(), "array.begin");
+  addr = Builder.CreateInBoundsGEP(addr, gepIndices, "array.begin");
 
   llvm::Value *numElements
     = llvm::ConstantInt::get(SizeTy, countFromCLAs);
@@ -975,6 +990,10 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
     case Type::FunctionNoProto:
       type = cast<FunctionType>(ty)->getResultType();
       break;
+
+    case Type::Atomic:
+      type = cast<AtomicType>(ty)->getValueType();
+      break;
     }
   } while (type->isVariablyModifiedType());
 }
@@ -1018,3 +1037,50 @@ void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
   // In theory, we could try to duplicate the peepholes now, but whatever.
   protection.Inst->eraseFromParent();
 }
+
+llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Value *AnnotationFn,
+                                                 llvm::Value *AnnotatedVal,
+                                                 llvm::StringRef AnnotationStr,
+                                                 SourceLocation Location) {
+  llvm::Value *Args[4] = {
+    AnnotatedVal,
+    Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr), Int8PtrTy),
+    Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location), Int8PtrTy),
+    CGM.EmitAnnotationLineNo(Location)
+  };
+  return Builder.CreateCall(AnnotationFn, Args);
+}
+
+void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
+  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
+  // FIXME We create a new bitcast for every annotation because that's what
+  // llvm-gcc was doing.
+  for (specific_attr_iterator<AnnotateAttr>
+       ai = D->specific_attr_begin<AnnotateAttr>(),
+       ae = D->specific_attr_end<AnnotateAttr>(); ai != ae; ++ai)
+    EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
+                       Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
+                       (*ai)->getAnnotation(), D->getLocation());
+}
+
+llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
+                                                   llvm::Value *V) {
+  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
+  llvm::Type *VTy = V->getType();
+  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
+                                    CGM.Int8PtrTy);
+
+  for (specific_attr_iterator<AnnotateAttr>
+       ai = D->specific_attr_begin<AnnotateAttr>(),
+       ae = D->specific_attr_end<AnnotateAttr>(); ai != ae; ++ai) {
+    // FIXME Always emit the cast inst so we can differentiate between
+    // annotation on the first field of a struct and annotation on the struct
+    // itself.
+    if (VTy != CGM.Int8PtrTy)
+      V = Builder.Insert(new llvm::BitCastInst(V, CGM.Int8PtrTy));
+    V = EmitAnnotationCall(F, V, (*ai)->getAnnotation(), D->getLocation());
+    V = Builder.CreateBitCast(V, VTy);
+  }
+
+  return V;
+}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h
index f27ed947b8d8..157623da8fdd 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h
@@ -325,16 +325,8 @@ class EHScopeStack {
   /// The innermost normal cleanup on the stack.
   stable_iterator InnermostNormalCleanup;
 
-  /// The innermost EH cleanup on the stack.
-  stable_iterator InnermostEHCleanup;
-
-  /// The number of catches on the stack.
-  unsigned CatchDepth;
-
-  /// The current EH destination index.  Reset to FirstCatchIndex
-  /// whenever the last EH cleanup is popped.
-  unsigned NextEHDestIndex;
-  enum { FirstEHDestIndex = 1 };
+  /// The innermost EH scope on the stack.
+  stable_iterator InnermostEHScope;
 
   /// The current set of branch fixups.  A branch fixup is a jump to
   /// an as-yet unemitted label, i.e. a label for which we don't yet
@@ -353,7 +345,7 @@ class EHScopeStack {
   ///     A a;
   ///    foo:
   ///     bar();
-  llvm::SmallVector<BranchFixup, 8> BranchFixups;
+  SmallVector<BranchFixup, 8> BranchFixups;
 
   char *allocate(size_t Size);
 
@@ -362,8 +354,7 @@ class EHScopeStack {
 public:
   EHScopeStack() : StartOfBuffer(0), EndOfBuffer(0), StartOfData(0),
                    InnermostNormalCleanup(stable_end()),
-                   InnermostEHCleanup(stable_end()),
-                   CatchDepth(0), NextEHDestIndex(FirstEHDestIndex) {}
+                   InnermostEHScope(stable_end()) {}
   ~EHScopeStack() { delete[] StartOfBuffer; }
 
   // Variadic templates would make this not terrible.
@@ -435,8 +426,7 @@ class EHScopeStack {
     return new (Buffer) T(N, a0, a1, a2);
   }
 
-  /// Pops a cleanup scope off the stack.  This should only be called
-  /// by CodeGenFunction::PopCleanupBlock.
+  /// Pops a cleanup scope off the stack.  This is private to CGCleanup.cpp.
   void popCleanup();
 
   /// Push a set of catch handlers on the stack.  The catch is
@@ -444,7 +434,7 @@ class EHScopeStack {
   /// set on it.
   class EHCatchScope *pushCatch(unsigned NumHandlers);
 
-  /// Pops a catch scope off the stack.
+  /// Pops a catch scope off the stack.  This is private to CGException.cpp.
   void popCatch();
 
   /// Push an exceptions filter on the stack.
@@ -463,7 +453,7 @@ class EHScopeStack {
   bool empty() const { return StartOfData == EndOfBuffer; }
 
   bool requiresLandingPad() const {
-    return (CatchDepth || hasEHCleanups());
+    return InnermostEHScope != stable_end();
   }
 
   /// Determines whether there are any normal cleanups on the stack.
@@ -476,19 +466,13 @@ class EHScopeStack {
   stable_iterator getInnermostNormalCleanup() const {
     return InnermostNormalCleanup;
   }
-  stable_iterator getInnermostActiveNormalCleanup() const; // CGException.h
+  stable_iterator getInnermostActiveNormalCleanup() const;
 
-  /// Determines whether there are any EH cleanups on the stack.
-  bool hasEHCleanups() const {
-    return InnermostEHCleanup != stable_end();
+  stable_iterator getInnermostEHScope() const {
+    return InnermostEHScope;
   }
 
-  /// Returns the innermost EH cleanup on the stack, or stable_end()
-  /// if there are no EH cleanups.
-  stable_iterator getInnermostEHCleanup() const {
-    return InnermostEHCleanup;
-  }
-  stable_iterator getInnermostActiveEHCleanup() const; // CGException.h
+  stable_iterator getInnermostActiveEHScope() const;
 
   /// An unstable reference to a scope-stack depth.  Invalidated by
   /// pushes but not pops.
@@ -515,10 +499,6 @@ class EHScopeStack {
   /// Translates an iterator into a stable_iterator.
   stable_iterator stabilize(iterator it) const;
 
-  /// Finds the nearest cleanup enclosing the given iterator.
-  /// Returns stable_iterator::invalid() if there are no such cleanups.
-  stable_iterator getEnclosingEHCleanup(iterator it) const;
-
   /// Turn a stable reference to a scope depth into a unstable pointer
   /// to the EH stack.
   iterator find(stable_iterator save) const;
@@ -547,9 +527,6 @@ class EHScopeStack {
   /// Clears the branch-fixups list.  This should only be called by
   /// ResolveAllBranchFixups.
   void clearFixups() { BranchFixups.clear(); }
-
-  /// Gets the next EH destination index.
-  unsigned getNextEHDestIndex() { return NextEHDestIndex++; }
 };
 
 /// CodeGenFunction - This class organizes the per-function state that is used
@@ -580,26 +557,6 @@ class CodeGenFunction : public CodeGenTypeCache {
     unsigned Index;
   };
 
-  /// An unwind destination is an abstract label, branching to which
-  /// may require a jump out through EH cleanups.
-  struct UnwindDest {
-    UnwindDest() : Block(0), ScopeDepth(), Index(0) {}
-    UnwindDest(llvm::BasicBlock *Block,
-               EHScopeStack::stable_iterator Depth,
-               unsigned Index)
-      : Block(Block), ScopeDepth(Depth), Index(Index) {}
-
-    bool isValid() const { return Block != 0; }
-    llvm::BasicBlock *getBlock() const { return Block; }
-    EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
-    unsigned getDestIndex() const { return Index; }
-
-  private:
-    llvm::BasicBlock *Block;
-    EHScopeStack::stable_iterator ScopeDepth;
-    unsigned Index;
-  };
-
   CodeGenModule &CGM;  // Per-module state.
   const TargetInfo &Target;
 
@@ -629,9 +586,6 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// iff the function has no return value.
   llvm::Value *ReturnValue;
 
-  /// RethrowBlock - Unified rethrow block.
-  UnwindDest RethrowBlock;
-
   /// AllocaInsertPoint - This is an instruction in the entry block before which
   /// we prefer to insert allocas.
   llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
@@ -652,16 +606,18 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// i32s containing the indexes of the cleanup destinations.
   llvm::AllocaInst *NormalCleanupDest;
-  llvm::AllocaInst *EHCleanupDest;
 
   unsigned NextCleanupDestIndex;
 
-  /// The exception slot.  All landing pads write the current
-  /// exception pointer into this alloca.
+  /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
+  llvm::BasicBlock *EHResumeBlock;
+
+  /// The exception slot.  All landing pads write the current exception pointer
+  /// into this alloca.
   llvm::Value *ExceptionSlot;
 
-  /// The selector slot.  Under the MandatoryCleanup model, all
-  /// landing pads write the current selector value into this alloca.
+  /// The selector slot.  Under the MandatoryCleanup model, all landing pads
+  /// write the current selector value into this alloca.
   llvm::AllocaInst *EHSelectorSlot;
 
   /// Emits a landing pad for the current EH stack.
@@ -681,7 +637,7 @@ class CodeGenFunction : public CodeGenTypeCache {
 public:
   /// ObjCEHValueStack - Stack of Objective-C exception values, used for
   /// rethrows.
-  llvm::SmallVector<llvm::Value*, 8> ObjCEHValueStack;
+  SmallVector<llvm::Value*, 8> ObjCEHValueStack;
 
   /// A class controlling the emission of a finally block.
   class FinallyInfo {
@@ -872,7 +828,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// The given basic block lies in the current EH scope, but may be a
   /// target of a potentially scope-crossing jump; get a stable handle
   /// to which we can perform this jump later.
-  JumpDest getJumpDestInCurrentScope(llvm::StringRef Name = llvm::StringRef()) {
+  JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
     return getJumpDestInCurrentScope(createBasicBlock(Name));
   }
 
@@ -886,14 +842,13 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// a conservatively correct answer for this method.
   bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
 
-  /// EmitBranchThroughEHCleanup - Emit a branch from the current
-  /// insert block through the EH cleanup handling code (if any) and
-  /// then on to \arg Dest.
-  void EmitBranchThroughEHCleanup(UnwindDest Dest);
+  /// popCatchScope - Pops the catch scope at the top of the EHScope
+  /// stack, emitting any required code (other than the catch handlers
+  /// themselves).
+  void popCatchScope();
 
-  /// getRethrowDest - Returns the unified outermost-scope rethrow
-  /// destination.
-  UnwindDest getRethrowDest();
+  llvm::BasicBlock *getEHResumeBlock();
+  llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
 
   /// An object to manage conditionally-evaluated expressions.
   class ConditionalEvaluation {
@@ -1089,7 +1044,7 @@ class CodeGenFunction : public CodeGenTypeCache {
     JumpDest BreakBlock;
     JumpDest ContinueBlock;
   };
-  llvm::SmallVector<BreakContinue, 8> BreakContinueStack;
+  SmallVector<BreakContinue, 8> BreakContinueStack;
 
   /// SwitchInsn - This is nearest current switch instruction. It is null if if
   /// current context is not in a switch.
@@ -1135,7 +1090,7 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// ByrefValueInfoMap - For each __block variable, contains a pair of the LLVM
   /// type as well as the field number that contains the actual data.
-  llvm::DenseMap<const ValueDecl *, std::pair<const llvm::Type *,
+  llvm::DenseMap<const ValueDecl *, std::pair<llvm::Type *,
                                               unsigned> > ByRefValueInfo;
 
   llvm::BasicBlock *TerminateLandingPad;
@@ -1161,13 +1116,17 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   const LangOptions &getLangOptions() const { return CGM.getLangOptions(); }
 
-  /// Returns a pointer to the function's exception object slot, which
-  /// is assigned in every landing pad.
+  /// Returns a pointer to the function's exception object and selector slot,
+  /// which is assigned in every landing pad.
   llvm::Value *getExceptionSlot();
   llvm::Value *getEHSelectorSlot();
 
+  /// Returns the contents of the function's exception object and selector
+  /// slots.
+  llvm::Value *getExceptionFromSlot();
+  llvm::Value *getSelectorFromSlot();
+
   llvm::Value *getNormalCleanupDestSlot();
-  llvm::Value *getEHCleanupDestSlot();
 
   llvm::BasicBlock *getUnreachableBlock() {
     if (!UnreachableBlock) {
@@ -1248,9 +1207,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
   void GenerateObjCGetter(ObjCImplementationDecl *IMP,
                           const ObjCPropertyImplDecl *PID);
-  void GenerateObjCGetterBody(ObjCIvarDecl *Ivar, bool IsAtomic, bool IsStrong);
-  void GenerateObjCAtomicSetterBody(ObjCMethodDecl *OMD,
-                                    ObjCIvarDecl *Ivar);
+  void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
+                              const ObjCPropertyImplDecl *propImpl);
 
   void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
                                   ObjCMethodDecl *MD, bool ctor);
@@ -1259,6 +1217,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// for the given property.
   void GenerateObjCSetter(ObjCImplementationDecl *IMP,
                           const ObjCPropertyImplDecl *PID);
+  void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
+                              const ObjCPropertyImplDecl *propImpl);
   bool IndirectObjCSetterArg(const CGFunctionInfo &FI);
   bool IvarTypeWithAggrGCObjects(QualType Ty);
 
@@ -1269,7 +1229,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   llvm::Value *EmitBlockLiteral(const BlockExpr *);
   llvm::Constant *BuildDescriptorBlockDecl(const BlockExpr *,
                                            const CGBlockInfo &Info,
-                                           const llvm::StructType *,
+                                           llvm::StructType *,
                                            llvm::Constant *BlockVarLayout);
 
   llvm::Function *GenerateBlockFunction(GlobalDecl GD,
@@ -1298,7 +1258,7 @@ class CodeGenFunction : public CodeGenTypeCache {
     return GetAddrOfBlockDecl(E->getDecl(), E->isByRef());
   }
   llvm::Value *GetAddrOfBlockDecl(const VarDecl *var, bool ByRef);
-  const llvm::Type *BuildByRefType(const VarDecl *var);
+  llvm::Type *BuildByRefType(const VarDecl *var);
 
   void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
                     const CGFunctionInfo &FnInfo);
@@ -1352,7 +1312,7 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// GetVTablePtr - Return the Value of the vtable pointer member pointed
   /// to by This.
-  llvm::Value *GetVTablePtr(llvm::Value *This, const llvm::Type *Ty);
+  llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty);
 
   /// EnterDtorCleanups - Enter the cleanups necessary to complete the
   /// given phase of destruction for a destructor.  The end result
@@ -1415,7 +1375,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   static bool hasAggregateLLVMType(QualType T);
 
   /// createBasicBlock - Create an LLVM basic block.
-  llvm::BasicBlock *createBasicBlock(llvm::StringRef name = "",
+  llvm::BasicBlock *createBasicBlock(StringRef name = "",
                                      llvm::Function *parent = 0,
                                      llvm::BasicBlock *before = 0) {
 #ifdef NDEBUG
@@ -1444,6 +1404,10 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// means the block can be ignored if it is unreachable.
   void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false);
 
+  /// EmitBlockAfterUses - Emit the given block somewhere hopefully
+  /// near its uses, and leave the insertion point in it.
+  void EmitBlockAfterUses(llvm::BasicBlock *BB);
+
   /// EmitBranch - Emit a branch to the specified basic block from the current
   /// insert block, taking care to avoid creation of branches from dummy
   /// blocks. It is legal to call this function even if there is no current
@@ -1486,8 +1450,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// CreateTempAlloca - This creates a alloca and inserts it into the entry
   /// block. The caller is responsible for setting an appropriate alignment on
   /// the alloca.
-  llvm::AllocaInst *CreateTempAlloca(const llvm::Type *Ty,
-                                     const llvm::Twine &Name = "tmp");
+  llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty,
+                                     const Twine &Name = "tmp");
 
   /// InitTempAlloca - Provide an initial value for the given alloca.
   void InitTempAlloca(llvm::AllocaInst *Alloca, llvm::Value *Value);
@@ -1497,17 +1461,19 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// value needs to be stored into an alloca (for example, to avoid explicit
   /// PHI construction), but the type is the IR type, not the type appropriate
   /// for storing in memory.
-  llvm::AllocaInst *CreateIRTemp(QualType T, const llvm::Twine &Name = "tmp");
+  llvm::AllocaInst *CreateIRTemp(QualType T, const Twine &Name = "tmp");
 
   /// CreateMemTemp - Create a temporary memory object of the given type, with
   /// appropriate alignment.
-  llvm::AllocaInst *CreateMemTemp(QualType T, const llvm::Twine &Name = "tmp");
+  llvm::AllocaInst *CreateMemTemp(QualType T, const Twine &Name = "tmp");
 
   /// CreateAggTemp - Create a temporary memory object for the given
   /// aggregate type.
-  AggValueSlot CreateAggTemp(QualType T, const llvm::Twine &Name = "tmp") {
+  AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp") {
     return AggValueSlot::forAddr(CreateMemTemp(T, Name), T.getQualifiers(),
-                                 false);
+                                 AggValueSlot::IsNotDestructed,
+                                 AggValueSlot::DoesNotNeedGCBarriers,
+                                 AggValueSlot::IsNotAliased);
   }
 
   /// Emit a cast to void* in the appropriate address space.
@@ -1708,8 +1674,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
                              bool ForVirtualBase, llvm::Value *This);
 
-  void EmitNewArrayInitializer(const CXXNewExpr *E, llvm::Value *NewPtr,
-                               llvm::Value *NumElements);
+  void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
+                               llvm::Value *NewPtr, llvm::Value *NumElements);
 
   void EmitCXXTemporary(const CXXTemporary *Temporary, llvm::Value *Ptr);
 
@@ -2074,18 +2040,18 @@ class CodeGenFunction : public CodeGenTypeCache {
                       ReturnValueSlot ReturnValue = ReturnValueSlot());
 
   llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee,
-                                  llvm::ArrayRef<llvm::Value *> Args,
-                                  const llvm::Twine &Name = "");
+                                  ArrayRef<llvm::Value *> Args,
+                                  const Twine &Name = "");
   llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee,
-                                  const llvm::Twine &Name = "");
+                                  const Twine &Name = "");
 
   llvm::Value *BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
-                                const llvm::Type *Ty);
+                                llvm::Type *Ty);
   llvm::Value *BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type,
-                                llvm::Value *This, const llvm::Type *Ty);
+                                llvm::Value *This, llvm::Type *Ty);
   llvm::Value *BuildAppleKextVirtualCall(const CXXMethodDecl *MD, 
                                          NestedNameSpecifier *Qual,
-                                         const llvm::Type *Ty);
+                                         llvm::Type *Ty);
   
   llvm::Value *BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
                                                    CXXDtorType Type, 
@@ -2110,6 +2076,9 @@ class CodeGenFunction : public CodeGenTypeCache {
                                        const CXXMethodDecl *MD,
                                        ReturnValueSlot ReturnValue);
 
+  RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
+                                ReturnValueSlot ReturnValue);
+
 
   RValue EmitBuiltinExpr(const FunctionDecl *FD,
                          unsigned BuiltinID, const CallExpr *E);
@@ -2122,14 +2091,14 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitNeonCall(llvm::Function *F,
-                            llvm::SmallVectorImpl<llvm::Value*> &O,
+                            SmallVectorImpl<llvm::Value*> &O,
                             const char *name,
                             unsigned shift = 0, bool rightshift = false);
   llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
-  llvm::Value *EmitNeonShiftVector(llvm::Value *V, const llvm::Type *Ty,
+  llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
                                    bool negateForRightShift);
 
-  llvm::Value *BuildVector(const llvm::SmallVectorImpl<llvm::Value*> &Ops);
+  llvm::Value *BuildVector(const SmallVectorImpl<llvm::Value*> &Ops);
   llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
@@ -2163,7 +2132,7 @@ class CodeGenFunction : public CodeGenTypeCache {
                                       bool ignored);
   llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
   llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
-  llvm::Value *EmitARCRetainBlock(llvm::Value *value);
+  llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
   void EmitARCRelease(llvm::Value *value, bool precise);
   llvm::Value *EmitARCAutorelease(llvm::Value *value);
   llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
@@ -2175,10 +2144,13 @@ class CodeGenFunction : public CodeGenTypeCache {
   std::pair<LValue,llvm::Value*>
   EmitARCStoreStrong(const BinaryOperator *e, bool ignored);
 
+  llvm::Value *EmitObjCThrowOperand(const Expr *expr);
+
   llvm::Value *EmitObjCProduceObject(QualType T, llvm::Value *Ptr);
   llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
   llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
 
+  llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
   llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
   llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
 
@@ -2311,6 +2283,25 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   void EmitCXXThrowExpr(const CXXThrowExpr *E);
 
+  RValue EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest = 0);
+
+  //===--------------------------------------------------------------------===//
+  //                         Annotations Emission
+  //===--------------------------------------------------------------------===//
+
+  /// Emit an annotation call (intrinsic or builtin).
+  llvm::Value *EmitAnnotationCall(llvm::Value *AnnotationFn,
+                                  llvm::Value *AnnotatedVal,
+                                  llvm::StringRef AnnotationStr,
+                                  SourceLocation Location);
+
+  /// Emit local annotations for the local variable V, declared by D.
+  void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
+
+  /// Emit field annotations for the given field & value. Returns the
+  /// annotation result.
+  llvm::Value *EmitFieldAnnotations(const FieldDecl *D, llvm::Value *V);
+
   //===--------------------------------------------------------------------===//
   //                             Internal Helpers
   //===--------------------------------------------------------------------===//
@@ -2370,7 +2361,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// Ty, into individual arguments on the provided vector \arg Args. See
   /// ABIArgInfo::Expand.
   void ExpandTypeToArgs(QualType Ty, RValue Src,
-                        llvm::SmallVector<llvm::Value*, 16> &Args,
+                        SmallVector<llvm::Value*, 16> &Args,
                         llvm::FunctionType *IRFuncTy);
 
   llvm::Value* EmitAsmInput(const AsmStmt &S,
@@ -2439,7 +2430,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitDeclMetadata();
 
   CodeGenModule::ByrefHelpers *
-  buildByrefHelpers(const llvm::StructType &byrefType,
+  buildByrefHelpers(llvm::StructType &byrefType,
                     const AutoVarEmission &emission);
 };
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp
index 0668039a892c..924ec8448e86 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp
@@ -16,8 +16,10 @@
 #include "CodeGenFunction.h"
 #include "CodeGenTBAA.h"
 #include "CGCall.h"
+#include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
+#include "CGOpenCLRuntime.h"
 #include "TargetInfo.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "clang/AST/ASTContext.h"
@@ -27,7 +29,6 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Mangle.h"
 #include "clang/AST/RecordLayout.h"
-#include "clang/Basic/Builtins.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -44,8 +45,10 @@
 using namespace clang;
 using namespace CodeGen;
 
+static const char AnnotationSection[] = "llvm.metadata";
+
 static CGCXXABI &createCXXABI(CodeGenModule &CGM) {
-  switch (CGM.getContext().Target.getCXXABI()) {
+  switch (CGM.getContext().getTargetInfo().getCXXABI()) {
   case CXXABI_ARM: return *CreateARMCXXABI(CGM);
   case CXXABI_Itanium: return *CreateItaniumCXXABI(CGM);
   case CXXABI_Microsoft: return *CreateMicrosoftCXXABI(CGM);
@@ -58,22 +61,25 @@ static CGCXXABI &createCXXABI(CodeGenModule &CGM) {
 
 CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
                              llvm::Module &M, const llvm::TargetData &TD,
-                             Diagnostic &diags)
+                             DiagnosticsEngine &diags)
   : Context(C), Features(C.getLangOptions()), CodeGenOpts(CGO), TheModule(M),
     TheTargetData(TD), TheTargetCodeGenInfo(0), Diags(diags),
     ABI(createCXXABI(*this)), 
     Types(C, M, TD, getTargetCodeGenInfo().getABIInfo(), ABI, CGO),
     TBAA(0),
-    VTables(*this), Runtime(0), DebugInfo(0), ARCData(0), RRData(0),
-    CFConstantStringClassRef(0), ConstantStringClassRef(0),
+    VTables(*this), ObjCRuntime(0), OpenCLRuntime(0), CUDARuntime(0),
+    DebugInfo(0), ARCData(0), RRData(0), CFConstantStringClassRef(0),
+    ConstantStringClassRef(0), NSConstantStringType(0),
     VMContext(M.getContext()),
-    NSConcreteGlobalBlockDecl(0), NSConcreteStackBlockDecl(0),
     NSConcreteGlobalBlock(0), NSConcreteStackBlock(0),
-    BlockObjectAssignDecl(0), BlockObjectDisposeDecl(0),
     BlockObjectAssign(0), BlockObjectDispose(0),
     BlockDescriptorType(0), GenericBlockLiteralType(0) {
   if (Features.ObjC1)
-     createObjCRuntime();
+    createObjCRuntime();
+  if (Features.OpenCL)
+    createOpenCLRuntime();
+  if (Features.CUDA)
+    createCUDARuntime();
 
   // Enable TBAA unless it's suppressed.
   if (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0)
@@ -98,17 +104,20 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
   Int8Ty = llvm::Type::getInt8Ty(LLVMContext);
   Int32Ty = llvm::Type::getInt32Ty(LLVMContext);
   Int64Ty = llvm::Type::getInt64Ty(LLVMContext);
-  PointerWidthInBits = C.Target.getPointerWidth(0);
+  PointerWidthInBits = C.getTargetInfo().getPointerWidth(0);
   PointerAlignInBytes =
-    C.toCharUnitsFromBits(C.Target.getPointerAlign(0)).getQuantity();
-  IntTy = llvm::IntegerType::get(LLVMContext, C.Target.getIntWidth());
+    C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity();
+  IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth());
   IntPtrTy = llvm::IntegerType::get(LLVMContext, PointerWidthInBits);
   Int8PtrTy = Int8Ty->getPointerTo(0);
   Int8PtrPtrTy = Int8PtrTy->getPointerTo(0);
 }
 
 CodeGenModule::~CodeGenModule() {
-  delete Runtime;
+  delete ObjCRuntime;
+  delete OpenCLRuntime;
+  delete CUDARuntime;
+  delete TheTargetCodeGenInfo;
   delete &ABI;
   delete TBAA;
   delete DebugInfo;
@@ -118,21 +127,29 @@ CodeGenModule::~CodeGenModule() {
 
 void CodeGenModule::createObjCRuntime() {
   if (!Features.NeXTRuntime)
-    Runtime = CreateGNUObjCRuntime(*this);
+    ObjCRuntime = CreateGNUObjCRuntime(*this);
   else
-    Runtime = CreateMacObjCRuntime(*this);
+    ObjCRuntime = CreateMacObjCRuntime(*this);
+}
+
+void CodeGenModule::createOpenCLRuntime() {
+  OpenCLRuntime = new CGOpenCLRuntime(*this);
+}
+
+void CodeGenModule::createCUDARuntime() {
+  CUDARuntime = CreateNVCUDARuntime(*this);
 }
 
 void CodeGenModule::Release() {
   EmitDeferred();
   EmitCXXGlobalInitFunc();
   EmitCXXGlobalDtorFunc();
-  if (Runtime)
-    if (llvm::Function *ObjCInitFunction = Runtime->ModuleInitFunction())
+  if (ObjCRuntime)
+    if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction())
       AddGlobalCtor(ObjCInitFunction);
   EmitCtorList(GlobalCtors, "llvm.global_ctors");
   EmitCtorList(GlobalDtors, "llvm.global_dtors");
-  EmitAnnotations();
+  EmitGlobalAnnotations();
   EmitLLVMUsed();
 
   SimplifyPersonality();
@@ -142,6 +159,9 @@ void CodeGenModule::Release() {
 
   if (getCodeGenOpts().EmitGcovArcs || getCodeGenOpts().EmitGcovNotes)
     EmitCoverageFile();
+
+  if (DebugInfo)
+    DebugInfo->finalize();
 }
 
 void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
@@ -163,11 +183,11 @@ void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst,
 }
 
 bool CodeGenModule::isTargetDarwin() const {
-  return getContext().Target.getTriple().isOSDarwin();
+  return getContext().getTargetInfo().getTriple().isOSDarwin();
 }
 
-void CodeGenModule::Error(SourceLocation loc, llvm::StringRef error) {
-  unsigned diagID = getDiags().getCustomDiagID(Diagnostic::Error, error);
+void CodeGenModule::Error(SourceLocation loc, StringRef error) {
+  unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, error);
   getDiags().Report(Context.getFullLoc(loc), diagID);
 }
 
@@ -177,7 +197,7 @@ void CodeGenModule::ErrorUnsupported(const Stmt *S, const char *Type,
                                      bool OmitOnError) {
   if (OmitOnError && getDiags().hasErrorOccurred())
     return;
-  unsigned DiagID = getDiags().getCustomDiagID(Diagnostic::Error,
+  unsigned DiagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error,
                                                "cannot compile this %0 yet");
   std::string Msg = Type;
   getDiags().Report(Context.getFullLoc(S->getLocStart()), DiagID)
@@ -190,7 +210,7 @@ void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type,
                                      bool OmitOnError) {
   if (OmitOnError && getDiags().hasErrorOccurred())
     return;
-  unsigned DiagID = getDiags().getCustomDiagID(Diagnostic::Error,
+  unsigned DiagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error,
                                                "cannot compile this %0 yet");
   std::string Msg = Type;
   getDiags().Report(Context.getFullLoc(D->getLocation()), DiagID) << Msg;
@@ -281,10 +301,10 @@ void CodeGenModule::setTypeVisibility(llvm::GlobalValue *GV,
   GV->setUnnamedAddr(true);
 }
 
-llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
+StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
 
-  llvm::StringRef &Str = MangledDeclNames[GD.getCanonicalDecl()];
+  StringRef &Str = MangledDeclNames[GD.getCanonicalDecl()];
   if (!Str.empty())
     return Str;
 
@@ -313,7 +333,7 @@ llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   char *Name = MangledNamesAllocator.Allocate<char>(Length);
   std::copy(Buffer.begin(), Buffer.end(), Name);
   
-  Str = llvm::StringRef(Name, Length);
+  Str = StringRef(Name, Length);
   
   return Str;
 }
@@ -333,7 +353,7 @@ void CodeGenModule::getBlockMangledName(GlobalDecl GD, MangleBuffer &Buffer,
     MangleCtx.mangleBlock(cast<DeclContext>(D), BD, Out);
 }
 
-llvm::GlobalValue *CodeGenModule::GetGlobalValue(llvm::StringRef Name) {
+llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) {
   return getModule().getNamedValue(Name);
 }
 
@@ -380,22 +400,6 @@ void CodeGenModule::EmitCtorList(const CtorList &Fns, const char *GlobalName) {
   }
 }
 
-void CodeGenModule::EmitAnnotations() {
-  if (Annotations.empty())
-    return;
-
-  // Create a new global variable for the ConstantStruct in the Module.
-  llvm::Constant *Array =
-  llvm::ConstantArray::get(llvm::ArrayType::get(Annotations[0]->getType(),
-                                                Annotations.size()),
-                           Annotations);
-  llvm::GlobalValue *gv =
-  new llvm::GlobalVariable(TheModule, Array->getType(), false,
-                           llvm::GlobalValue::AppendingLinkage, Array,
-                           "llvm.global.annotations");
-  gv->setSection("llvm.metadata");
-}
-
 llvm::GlobalValue::LinkageTypes
 CodeGenModule::getFunctionLinkage(const FunctionDecl *D) {
   GVALinkage Linkage = getContext().GetGVALinkageForFunction(D);
@@ -413,7 +417,12 @@ CodeGenModule::getFunctionLinkage(const FunctionDecl *D) {
   // definition somewhere else, so we can use available_externally linkage.
   if (Linkage == GVA_C99Inline)
     return llvm::Function::AvailableExternallyLinkage;
-  
+
+  // Note that Apple's kernel linker doesn't support symbol
+  // coalescing, so we need to avoid linkonce and weak linkages there.
+  // Normally, this means we just map to internal, but for explicit
+  // instantiations we'll map to external.
+
   // In C++, the compiler has to emit a definition in every translation unit
   // that references the function.  We should use linkonce_odr because
   // a) if all references in this translation unit are optimized away, we
@@ -432,7 +441,7 @@ CodeGenModule::getFunctionLinkage(const FunctionDecl *D) {
   if (Linkage == GVA_ExplicitTemplateInstantiation)
     return !Context.getLangOptions().AppleKext
              ? llvm::Function::WeakODRLinkage
-             : llvm::Function::InternalLinkage;
+             : llvm::Function::ExternalLinkage;
   
   // Otherwise, we have strong external linkage.
   assert(Linkage == GVA_StrongExternal);
@@ -460,29 +469,54 @@ void CodeGenModule::SetLLVMFunctionAttributes(const Decl *D,
   F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
 }
 
+/// Determines whether the language options require us to model
+/// unwind exceptions.  We treat -fexceptions as mandating this
+/// except under the fragile ObjC ABI with only ObjC exceptions
+/// enabled.  This means, for example, that C with -fexceptions
+/// enables this.
+static bool hasUnwindExceptions(const LangOptions &Features) {
+  // If exceptions are completely disabled, obviously this is false.
+  if (!Features.Exceptions) return false;
+
+  // If C++ exceptions are enabled, this is true.
+  if (Features.CXXExceptions) return true;
+
+  // If ObjC exceptions are enabled, this depends on the ABI.
+  if (Features.ObjCExceptions) {
+    if (!Features.ObjCNonFragileABI) return false;
+  }
+
+  return true;
+}
+
 void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
                                                            llvm::Function *F) {
   if (CodeGenOpts.UnwindTables)
     F->setHasUWTable();
 
-  if (!Features.Exceptions && !Features.ObjCNonFragileABI)
+  if (!hasUnwindExceptions(Features))
     F->addFnAttr(llvm::Attribute::NoUnwind);
 
-  if (D->hasAttr<AlwaysInlineAttr>())
-    F->addFnAttr(llvm::Attribute::AlwaysInline);
-
-  if (D->hasAttr<NakedAttr>())
+  if (D->hasAttr<NakedAttr>()) {
+    // Naked implies noinline: we should not be inlining such functions.
     F->addFnAttr(llvm::Attribute::Naked);
+    F->addFnAttr(llvm::Attribute::NoInline);
+  }
 
   if (D->hasAttr<NoInlineAttr>())
     F->addFnAttr(llvm::Attribute::NoInline);
 
+  // (noinline wins over always_inline, and we can't specify both in IR)
+  if (D->hasAttr<AlwaysInlineAttr>() &&
+      !F->hasFnAttr(llvm::Attribute::NoInline))
+    F->addFnAttr(llvm::Attribute::AlwaysInline);
+
   if (isa<CXXConstructorDecl>(D) || isa<CXXDestructorDecl>(D))
     F->setUnnamedAddr(true);
 
-  if (Features.getStackProtectorMode() == LangOptions::SSPOn)
+  if (Features.getStackProtector() == LangOptions::SSPOn)
     F->addFnAttr(llvm::Attribute::StackProtect);
-  else if (Features.getStackProtectorMode() == LangOptions::SSPReq)
+  else if (Features.getStackProtector() == LangOptions::SSPReq)
     F->addFnAttr(llvm::Attribute::StackProtectReq);
   
   unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
@@ -570,7 +604,7 @@ void CodeGenModule::EmitLLVMUsed() {
   if (LLVMUsed.empty())
     return;
 
-  const llvm::Type *i8PTy = llvm::Type::getInt8PtrTy(VMContext);
+  llvm::Type *i8PTy = llvm::Type::getInt8PtrTy(VMContext);
 
   // Convert LLVMUsed to what ConstantArray needs.
   std::vector<llvm::Constant*> UsedArray;
@@ -597,7 +631,7 @@ void CodeGenModule::EmitLLVMUsed() {
 void CodeGenModule::EmitDeferred() {
   // Emit code for any potentially referenced deferred decls.  Since a
   // previously unused static decl may become used during the generation of code
-  // for a static function, iterate until no  changes are made.
+  // for a static function, iterate until no changes are made.
 
   while (!DeferredDeclsToEmit.empty() || !DeferredVTables.empty()) {
     if (!DeferredVTables.empty()) {
@@ -618,7 +652,7 @@ void CodeGenModule::EmitDeferred() {
     // ignore these cases.
     //
     // TODO: That said, looking this up multiple times is very wasteful.
-    llvm::StringRef Name = getMangledName(D);
+    StringRef Name = getMangledName(D);
     llvm::GlobalValue *CGRef = GetGlobalValue(Name);
     assert(CGRef && "Deferred decl wasn't referenced?");
 
@@ -635,54 +669,78 @@ void CodeGenModule::EmitDeferred() {
   }
 }
 
-/// EmitAnnotateAttr - Generate the llvm::ConstantStruct which contains the
-/// annotation information for a given GlobalValue.  The annotation struct is
-/// {i8 *, i8 *, i8 *, i32}.  The first field is a constant expression, the
-/// GlobalValue being annotated.  The second field is the constant string
-/// created from the AnnotateAttr's annotation.  The third field is a constant
-/// string containing the name of the translation unit.  The fourth field is
-/// the line number in the file of the annotated value declaration.
-///
-/// FIXME: this does not unique the annotation string constants, as llvm-gcc
-///        appears to.
-///
+void CodeGenModule::EmitGlobalAnnotations() {
+  if (Annotations.empty())
+    return;
+
+  // Create a new global variable for the ConstantStruct in the Module.
+  llvm::Constant *Array = llvm::ConstantArray::get(llvm::ArrayType::get(
+    Annotations[0]->getType(), Annotations.size()), Annotations);
+  llvm::GlobalValue *gv = new llvm::GlobalVariable(getModule(),
+    Array->getType(), false, llvm::GlobalValue::AppendingLinkage, Array,
+    "llvm.global.annotations");
+  gv->setSection(AnnotationSection);
+}
+
+llvm::Constant *CodeGenModule::EmitAnnotationString(llvm::StringRef Str) {
+  llvm::StringMap<llvm::Constant*>::iterator i = AnnotationStrings.find(Str);
+  if (i != AnnotationStrings.end())
+    return i->second;
+
+  // Not found yet, create a new global.
+  llvm::Constant *s = llvm::ConstantArray::get(getLLVMContext(), Str, true);
+  llvm::GlobalValue *gv = new llvm::GlobalVariable(getModule(), s->getType(),
+    true, llvm::GlobalValue::PrivateLinkage, s, ".str");
+  gv->setSection(AnnotationSection);
+  gv->setUnnamedAddr(true);
+  AnnotationStrings[Str] = gv;
+  return gv;
+}
+
+llvm::Constant *CodeGenModule::EmitAnnotationUnit(SourceLocation Loc) {
+  SourceManager &SM = getContext().getSourceManager();
+  PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+  if (PLoc.isValid())
+    return EmitAnnotationString(PLoc.getFilename());
+  return EmitAnnotationString(SM.getBufferName(Loc));
+}
+
+llvm::Constant *CodeGenModule::EmitAnnotationLineNo(SourceLocation L) {
+  SourceManager &SM = getContext().getSourceManager();
+  PresumedLoc PLoc = SM.getPresumedLoc(L);
+  unsigned LineNo = PLoc.isValid() ? PLoc.getLine() :
+    SM.getExpansionLineNumber(L);
+  return llvm::ConstantInt::get(Int32Ty, LineNo);
+}
+
 llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
                                                 const AnnotateAttr *AA,
-                                                unsigned LineNo) {
-  llvm::Module *M = &getModule();
-
-  // get [N x i8] constants for the annotation string, and the filename string
-  // which are the 2nd and 3rd elements of the global annotation structure.
-  const llvm::Type *SBP = llvm::Type::getInt8PtrTy(VMContext);
-  llvm::Constant *anno = llvm::ConstantArray::get(VMContext,
-                                                  AA->getAnnotation(), true);
-  llvm::Constant *unit = llvm::ConstantArray::get(VMContext,
-                                                  M->getModuleIdentifier(),
-                                                  true);
-
-  // Get the two global values corresponding to the ConstantArrays we just
-  // created to hold the bytes of the strings.
-  llvm::GlobalValue *annoGV =
-    new llvm::GlobalVariable(*M, anno->getType(), false,
-                             llvm::GlobalValue::PrivateLinkage, anno,
-                             GV->getName());
-  // translation unit name string, emitted into the llvm.metadata section.
-  llvm::GlobalValue *unitGV =
-    new llvm::GlobalVariable(*M, unit->getType(), false,
-                             llvm::GlobalValue::PrivateLinkage, unit,
-                             ".str");
-  unitGV->setUnnamedAddr(true);
+                                                SourceLocation L) {
+  // Get the globals for file name, annotation, and the line number.
+  llvm::Constant *AnnoGV = EmitAnnotationString(AA->getAnnotation()),
+                 *UnitGV = EmitAnnotationUnit(L),
+                 *LineNoCst = EmitAnnotationLineNo(L);
 
   // Create the ConstantStruct for the global annotation.
   llvm::Constant *Fields[4] = {
-    llvm::ConstantExpr::getBitCast(GV, SBP),
-    llvm::ConstantExpr::getBitCast(annoGV, SBP),
-    llvm::ConstantExpr::getBitCast(unitGV, SBP),
-    llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), LineNo)
+    llvm::ConstantExpr::getBitCast(GV, Int8PtrTy),
+    llvm::ConstantExpr::getBitCast(AnnoGV, Int8PtrTy),
+    llvm::ConstantExpr::getBitCast(UnitGV, Int8PtrTy),
+    LineNoCst
   };
   return llvm::ConstantStruct::getAnon(Fields);
 }
 
+void CodeGenModule::AddGlobalAnnotations(const ValueDecl *D,
+                                         llvm::GlobalValue *GV) {
+  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
+  // Get the struct elements for these annotations.
+  for (specific_attr_iterator<AnnotateAttr>
+       ai = D->specific_attr_begin<AnnotateAttr>(),
+       ae = D->specific_attr_end<AnnotateAttr>(); ai != ae; ++ai)
+    Annotations.push_back(EmitAnnotateAttr(GV, *ai, D->getLocation()));
+}
+
 bool CodeGenModule::MayDeferGeneration(const ValueDecl *Global) {
   // Never defer when EmitAllDecls is specified.
   if (Features.EmitAllDecls)
@@ -695,7 +753,7 @@ llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
   const AliasAttr *AA = VD->getAttr<AliasAttr>();
   assert(AA && "No alias?");
 
-  const llvm::Type *DeclTy = getTypes().ConvertTypeForMem(VD->getType());
+  llvm::Type *DeclTy = getTypes().ConvertTypeForMem(VD->getType());
 
   // See if there is already something with the target's name in the module.
   llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee());
@@ -728,34 +786,45 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
   if (Global->hasAttr<AliasAttr>())
     return EmitAliasDefinition(GD);
 
+  // If this is CUDA, be selective about which declarations we emit.
+  if (Features.CUDA) {
+    if (CodeGenOpts.CUDAIsDevice) {
+      if (!Global->hasAttr<CUDADeviceAttr>() &&
+          !Global->hasAttr<CUDAGlobalAttr>() &&
+          !Global->hasAttr<CUDAConstantAttr>() &&
+          !Global->hasAttr<CUDASharedAttr>())
+        return;
+    } else {
+      if (!Global->hasAttr<CUDAHostAttr>() && (
+            Global->hasAttr<CUDADeviceAttr>() ||
+            Global->hasAttr<CUDAConstantAttr>() ||
+            Global->hasAttr<CUDASharedAttr>()))
+        return;
+    }
+  }
+
   // Ignore declarations, they will be emitted on their first use.
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Global)) {
-    if (FD->getIdentifier()) {
-      llvm::StringRef Name = FD->getName();
-      if (Name == "_Block_object_assign") {
-        BlockObjectAssignDecl = FD;
-      } else if (Name == "_Block_object_dispose") {
-        BlockObjectDisposeDecl = FD;
-      }
-    }
-
     // Forward declarations are emitted lazily on first use.
-    if (!FD->doesThisDeclarationHaveABody())
+    if (!FD->doesThisDeclarationHaveABody()) {
+      if (!FD->doesDeclarationForceExternallyVisibleDefinition())
+        return;
+
+      const FunctionDecl *InlineDefinition = 0;
+      FD->getBody(InlineDefinition);
+
+      StringRef MangledName = getMangledName(GD);
+      llvm::StringMap<GlobalDecl>::iterator DDI =
+          DeferredDecls.find(MangledName);
+      if (DDI != DeferredDecls.end())
+        DeferredDecls.erase(DDI);
+      EmitGlobalDefinition(InlineDefinition);
       return;
+    }
   } else {
     const VarDecl *VD = cast<VarDecl>(Global);
     assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
 
-    if (VD->getIdentifier()) {
-      llvm::StringRef Name = VD->getName();
-      if (Name == "_NSConcreteGlobalBlock") {
-        NSConcreteGlobalBlockDecl = VD;
-      } else if (Name == "_NSConcreteStackBlock") {
-        NSConcreteStackBlockDecl = VD;
-      }
-    }
-
-
     if (VD->isThisDeclarationADefinition() != VarDecl::Definition)
       return;
   }
@@ -778,7 +847,7 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
   
   // If the value has already been used, add it directly to the
   // DeferredDeclsToEmit list.
-  llvm::StringRef MangledName = getMangledName(GD);
+  StringRef MangledName = getMangledName(GD);
   if (GetGlobalValue(MangledName))
     DeferredDeclsToEmit.push_back(GD);
   else {
@@ -827,7 +896,7 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD) {
   if (const VarDecl *VD = dyn_cast<VarDecl>(D))
     return EmitGlobalVarDefinition(VD);
   
-  assert(0 && "Invalid argument to EmitGlobalDefinition()");
+  llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
 }
 
 /// GetOrCreateLLVMFunction - If the specified mangled name is not in the
@@ -838,8 +907,8 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD) {
 /// If D is non-null, it specifies a decl that correspond to this.  This is used
 /// to set the attributes on the function when it is first created.
 llvm::Constant *
-CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
-                                       const llvm::Type *Ty,
+CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName,
+                                       llvm::Type *Ty,
                                        GlobalDecl D, bool ForVTable,
                                        llvm::Attributes ExtraAttrs) {
   // Lookup the entry, lazily creating it if necessary.
@@ -865,7 +934,7 @@ CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
   // sure not to try to set attributes.
   bool IsIncompleteFunction = false;
 
-  const llvm::FunctionType *FTy;
+  llvm::FunctionType *FTy;
   if (isa<llvm::FunctionType>(Ty)) {
     FTy = cast<llvm::FunctionType>(Ty);
   } else {
@@ -935,21 +1004,21 @@ CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
 /// non-null, then this function will use the specified type if it has to
 /// create it (this occurs when we see a definition of the function).
 llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD,
-                                                 const llvm::Type *Ty,
+                                                 llvm::Type *Ty,
                                                  bool ForVTable) {
   // If there was no specific requested type, just convert it now.
   if (!Ty)
     Ty = getTypes().ConvertType(cast<ValueDecl>(GD.getDecl())->getType());
   
-  llvm::StringRef MangledName = getMangledName(GD);
+  StringRef MangledName = getMangledName(GD);
   return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable);
 }
 
 /// CreateRuntimeFunction - Create a new runtime function with the specified
 /// type and name.
 llvm::Constant *
-CodeGenModule::CreateRuntimeFunction(const llvm::FunctionType *FTy,
-                                     llvm::StringRef Name,
+CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy,
+                                     StringRef Name,
                                      llvm::Attributes ExtraAttrs) {
   return GetOrCreateLLVMFunction(Name, FTy, GlobalDecl(), /*ForVTable=*/false,
                                  ExtraAttrs);
@@ -979,8 +1048,8 @@ static bool DeclIsConstantGlobal(ASTContext &Context, const VarDecl *D,
 /// If D is non-null, it specifies a decl that correspond to this.  This is used
 /// to set the attributes on the global when it is first created.
 llvm::Constant *
-CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
-                                     const llvm::PointerType *Ty,
+CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName,
+                                     llvm::PointerType *Ty,
                                      const VarDecl *D,
                                      bool UnnamedAddr) {
   // Lookup the entry, lazily creating it if necessary.
@@ -1049,8 +1118,8 @@ CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
 
 
 llvm::GlobalVariable *
-CodeGenModule::CreateOrReplaceCXXRuntimeVariable(llvm::StringRef Name, 
-                                      const llvm::Type *Ty,
+CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name, 
+                                      llvm::Type *Ty,
                                       llvm::GlobalValue::LinkageTypes Linkage) {
   llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name);
   llvm::GlobalVariable *OldGV = 0;
@@ -1092,24 +1161,24 @@ CodeGenModule::CreateOrReplaceCXXRuntimeVariable(llvm::StringRef Name,
 /// then it will be greated with the specified type instead of whatever the
 /// normal requested type would be.
 llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D,
-                                                  const llvm::Type *Ty) {
+                                                  llvm::Type *Ty) {
   assert(D->hasGlobalStorage() && "Not a global variable");
   QualType ASTTy = D->getType();
   if (Ty == 0)
     Ty = getTypes().ConvertTypeForMem(ASTTy);
 
-  const llvm::PointerType *PTy =
+  llvm::PointerType *PTy =
     llvm::PointerType::get(Ty, getContext().getTargetAddressSpace(ASTTy));
 
-  llvm::StringRef MangledName = getMangledName(D);
+  StringRef MangledName = getMangledName(D);
   return GetOrCreateLLVMGlobal(MangledName, PTy, D);
 }
 
 /// CreateRuntimeVariable - Create a new runtime global variable with the
 /// specified type and name.
 llvm::Constant *
-CodeGenModule::CreateRuntimeVariable(const llvm::Type *Ty,
-                                     llvm::StringRef Name) {
+CodeGenModule::CreateRuntimeVariable(llvm::Type *Ty,
+                                     StringRef Name) {
   return GetOrCreateLLVMGlobal(Name, llvm::PointerType::getUnqual(Ty), 0,
                                true);
 }
@@ -1121,7 +1190,7 @@ void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
     // If we have not seen a reference to this variable yet, place it
     // into the deferred declarations table to be emitted if needed
     // later.
-    llvm::StringRef MangledName = getMangledName(D);
+    StringRef MangledName = getMangledName(D);
     if (!GetGlobalValue(MangledName)) {
       DeferredDecls[MangledName] = D;
       return;
@@ -1207,7 +1276,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) {
   return llvm::GlobalVariable::LinkOnceODRLinkage;
 }
 
-CharUnits CodeGenModule::GetTargetTypeStoreSize(const llvm::Type *Ty) const {
+CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
     return Context.toCharUnitsFromBits(
       TheTargetData.getTypeStoreSizeInBits(Ty));
 }
@@ -1253,7 +1322,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     }
   }
 
-  const llvm::Type* InitType = Init->getType();
+  llvm::Type* InitType = Init->getType();
   llvm::Constant *Entry = GetAddrOfGlobalVar(D, InitType);
 
   // Strip off a bitcast if we got one back.
@@ -1282,7 +1351,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
         getContext().getTargetAddressSpace(ASTTy)) {
 
     // Move the old entry aside so that we'll create a new one.
-    Entry->setName(llvm::StringRef());
+    Entry->setName(StringRef());
 
     // Make a new global with the correct type, this is now guaranteed to work.
     GV = cast<llvm::GlobalVariable>(GetAddrOfGlobalVar(D, InitType));
@@ -1296,11 +1365,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     cast<llvm::GlobalValue>(Entry)->eraseFromParent();
   }
 
-  if (const AnnotateAttr *AA = D->getAttr<AnnotateAttr>()) {
-    SourceManager &SM = Context.getSourceManager();
-    AddAnnotation(EmitAnnotateAttr(GV, AA,
-                              SM.getInstantiationLineNumber(D->getLocation())));
-  }
+  if (D->hasAttr<AnnotateAttr>())
+    AddGlobalAnnotations(D, GV);
 
   GV->setInitializer(Init);
 
@@ -1326,10 +1392,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     EmitCXXGlobalVarDeclInitFunc(D, GV);
 
   // Emit global variable debug information.
-  if (CGDebugInfo *DI = getModuleDebugInfo()) {
-    DI->setLocation(D->getLocation());
+  if (CGDebugInfo *DI = getModuleDebugInfo())
     DI->EmitGlobalVariable(GV, D);
-  }
 }
 
 llvm::GlobalValue::LinkageTypes
@@ -1377,8 +1441,8 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
   llvm::Function *OldFn = dyn_cast<llvm::Function>(Old);
   if (OldFn == 0) return;
 
-  const llvm::Type *NewRetTy = NewFn->getReturnType();
-  llvm::SmallVector<llvm::Value*, 4> ArgList;
+  llvm::Type *NewRetTy = NewFn->getReturnType();
+  SmallVector<llvm::Value*, 4> ArgList;
 
   for (llvm::Value::use_iterator UI = OldFn->use_begin(), E = OldFn->use_end();
        UI != E; ) {
@@ -1394,6 +1458,17 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
     if (CI->getType() != NewRetTy && !CI->use_empty())
       continue;
 
+    // Get the attribute list.
+    llvm::SmallVector<llvm::AttributeWithIndex, 8> AttrVec;
+    llvm::AttrListPtr AttrList = CI->getAttributes();
+
+    // Get any return attributes.
+    llvm::Attributes RAttrs = AttrList.getRetAttributes();
+
+    // Add the return attributes.
+    if (RAttrs)
+      AttrVec.push_back(llvm::AttributeWithIndex::get(0, RAttrs));
+
     // If the function was passed too few arguments, don't transform.  If extra
     // arguments were passed, we silently drop them.  If any of the types
     // mismatch, we don't transform.
@@ -1406,10 +1481,17 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
         DontTransform = true;
         break;
       }
+
+      // Add any parameter attributes.
+      if (llvm::Attributes PAttrs = AttrList.getParamAttributes(ArgNo + 1))
+        AttrVec.push_back(llvm::AttributeWithIndex::get(ArgNo + 1, PAttrs));
     }
     if (DontTransform)
       continue;
 
+    if (llvm::Attributes FnAttrs =  AttrList.getFnAttributes())
+      AttrVec.push_back(llvm::AttributeWithIndex::get(~0, FnAttrs));
+
     // Okay, we can transform this.  Create the new call instruction and copy
     // over the required information.
     ArgList.append(CS.arg_begin(), CS.arg_begin() + ArgNo);
@@ -1417,7 +1499,8 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
     ArgList.clear();
     if (!NewCall->getType()->isVoidTy())
       NewCall->takeName(CI);
-    NewCall->setAttributes(CI->getAttributes());
+    NewCall->setAttributes(llvm::AttrListPtr::get(AttrVec.begin(),
+                                                  AttrVec.end()));
     NewCall->setCallingConv(CI->getCallingConv());
 
     // Finally, remove the old call, replacing any uses with the new one.
@@ -1440,7 +1523,7 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD) {
   bool variadic = false;
   if (const FunctionProtoType *fpt = D->getType()->getAs<FunctionProtoType>())
     variadic = fpt->isVariadic();
-  const llvm::FunctionType *Ty = getTypes().GetFunctionType(FI, variadic);
+  llvm::FunctionType *Ty = getTypes().GetFunctionType(FI, variadic);
 
   // Get or create the prototype for the function.
   llvm::Constant *Entry = GetAddrOfFunction(GD, Ty);
@@ -1467,7 +1550,7 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD) {
     // (e.g. "int f()") and then a definition of a different type
     // (e.g. "int f(int x)").  Move the old function aside so that it
     // doesn't interfere with GetAddrOfFunction.
-    OldFn->setName(llvm::StringRef());
+    OldFn->setName(StringRef());
     llvm::Function *NewFn = cast<llvm::Function>(GetAddrOfFunction(GD, Ty));
 
     // If this is an implementation of a function without a prototype, try to
@@ -1510,6 +1593,8 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD) {
     AddGlobalCtor(Fn, CA->getPriority());
   if (const DestructorAttr *DA = D->getAttr<DestructorAttr>())
     AddGlobalDtor(Fn, DA->getPriority());
+  if (D->hasAttr<AnnotateAttr>())
+    AddGlobalAnnotations(D, Fn);
 }
 
 void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
@@ -1517,7 +1602,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
   const AliasAttr *AA = D->getAttr<AliasAttr>();
   assert(AA && "Not an alias?");
 
-  llvm::StringRef MangledName = getMangledName(GD);
+  StringRef MangledName = getMangledName(GD);
 
   // If there is a definition in the module, then it wins over the alias.
   // This is dubious, but allow it to be safe.  Just ignore the alias.
@@ -1525,7 +1610,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
   if (Entry && !Entry->isDeclaration())
     return;
 
-  const llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
+  llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
 
   // Create a reference to the named value.  This ensures that it is emitted
   // if a deferred decl.
@@ -1582,37 +1667,8 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
   SetCommonAttributes(D, GA);
 }
 
-/// getBuiltinLibFunction - Given a builtin id for a function like
-/// "__builtin_fabsf", return a Function* for "fabsf".
-llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
-                                                  unsigned BuiltinID) {
-  assert((Context.BuiltinInfo.isLibFunction(BuiltinID) ||
-          Context.BuiltinInfo.isPredefinedLibFunction(BuiltinID)) &&
-         "isn't a lib fn");
-
-  // Get the name, skip over the __builtin_ prefix (if necessary).
-  llvm::StringRef Name;
-  GlobalDecl D(FD);
-
-  // If the builtin has been declared explicitly with an assembler label,
-  // use the mangled name. This differs from the plain label on platforms
-  // that prefix labels.
-  if (FD->hasAttr<AsmLabelAttr>())
-    Name = getMangledName(D);
-  else if (Context.BuiltinInfo.isLibFunction(BuiltinID))
-    Name = Context.BuiltinInfo.GetName(BuiltinID) + 10;
-  else
-    Name = Context.BuiltinInfo.GetName(BuiltinID);
-
-
-  const llvm::FunctionType *Ty =
-    cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
-
-  return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
-}
-
 llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,
-                                            llvm::ArrayRef<llvm::Type*> Tys) {
+                                            ArrayRef<llvm::Type*> Tys) {
   return llvm::Intrinsic::getDeclaration(&getModule(), (llvm::Intrinsic::ID)IID,
                                          Tys);
 }
@@ -1623,7 +1679,7 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
                          bool TargetIsLSB,
                          bool &IsUTF16,
                          unsigned &StringLength) {
-  llvm::StringRef String = Literal->getString();
+  StringRef String = Literal->getString();
   unsigned NumBytes = String.size();
 
   // Check for simple case.
@@ -1633,7 +1689,7 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
   }
 
   // Otherwise, convert the UTF8 literals into a byte string.
-  llvm::SmallVector<UTF16, 128> ToBuf(NumBytes);
+  SmallVector<UTF16, 128> ToBuf(NumBytes);
   const UTF8 *FromPtr = (UTF8 *)String.data();
   UTF16 *ToPtr = &ToBuf[0];
 
@@ -1665,7 +1721,7 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
   AsBytes.push_back(0);
 
   IsUTF16 = true;
-  return Map.GetOrCreateValue(llvm::StringRef(AsBytes.data(), AsBytes.size()));
+  return Map.GetOrCreateValue(StringRef(AsBytes.data(), AsBytes.size()));
 }
 
 static llvm::StringMapEntry<llvm::Constant*> &
@@ -1673,7 +1729,7 @@ GetConstantStringEntry(llvm::StringMap<llvm::Constant*> &Map,
 		       const StringLiteral *Literal,
 		       unsigned &StringLength)
 {
-	llvm::StringRef String = Literal->getString();
+	StringRef String = Literal->getString();
 	StringLength = String.size();
 	return Map.GetOrCreateValue(String);
 }
@@ -1696,18 +1752,18 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
 
   // If we don't already have it, get __CFConstantStringClassReference.
   if (!CFConstantStringClassRef) {
-    const llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
+    llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
     Ty = llvm::ArrayType::get(Ty, 0);
     llvm::Constant *GV = CreateRuntimeVariable(Ty,
                                            "__CFConstantStringClassReference");
     // Decay array -> ptr
     CFConstantStringClassRef =
-      llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
+      llvm::ConstantExpr::getGetElementPtr(GV, Zeros);
   }
 
   QualType CFTy = getContext().getCFConstantStringType();
 
-  const llvm::StructType *STy =
+  llvm::StructType *STy =
     cast<llvm::StructType>(getTypes().ConvertType(CFTy));
 
   std::vector<llvm::Constant*> Fields(4);
@@ -1716,7 +1772,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   Fields[0] = CFConstantStringClassRef;
 
   // Flags.
-  const llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy);
+  llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy);
   Fields[1] = isUTF16 ? llvm::ConstantInt::get(Ty, 0x07d0) :
     llvm::ConstantInt::get(Ty, 0x07C8);
 
@@ -1750,7 +1806,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
     CharUnits Align = getContext().getTypeAlignInChars(getContext().CharTy);
     GV->setAlignment(Align.getQuantity());
   }
-  Fields[2] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
+  Fields[2] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros);
 
   // String length.
   Ty = getTypes().ConvertType(getContext().LongTy);
@@ -1761,13 +1817,23 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   GV = new llvm::GlobalVariable(getModule(), C->getType(), true,
                                 llvm::GlobalVariable::PrivateLinkage, C,
                                 "_unnamed_cfstring_");
-  if (const char *Sect = getContext().Target.getCFStringSection())
+  if (const char *Sect = getContext().getTargetInfo().getCFStringSection())
     GV->setSection(Sect);
   Entry.setValue(GV);
 
   return GV;
 }
 
+static RecordDecl *
+CreateRecordDecl(const ASTContext &Ctx, RecordDecl::TagKind TK,
+                 DeclContext *DC, IdentifierInfo *Id) {
+  SourceLocation Loc;
+  if (Ctx.getLangOptions().CPlusPlus)
+    return CXXRecordDecl::Create(Ctx, TK, DC, Loc, Loc, Id);
+  else
+    return RecordDecl::Create(Ctx, TK, DC, Loc, Loc, Id);
+}
+
 llvm::Constant *
 CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) {
   unsigned StringLength = 0;
@@ -1784,7 +1850,7 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) {
   // If we don't already have it, get _NSConstantStringClassReference.
   if (!ConstantStringClassRef) {
     std::string StringClass(getLangOptions().ObjCConstantStringClass);
-    const llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
+    llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
     llvm::Constant *GV;
     if (Features.ObjCNonFragileABI) {
       std::string str = 
@@ -1792,25 +1858,54 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) {
                             : "OBJC_CLASS_$_" + StringClass;
       GV = getObjCRuntime().GetClassGlobal(str);
       // Make sure the result is of the correct type.
-      const llvm::Type *PTy = llvm::PointerType::getUnqual(Ty);
+      llvm::Type *PTy = llvm::PointerType::getUnqual(Ty);
       ConstantStringClassRef =
         llvm::ConstantExpr::getBitCast(GV, PTy);
     } else {
       std::string str =
         StringClass.empty() ? "_NSConstantStringClassReference"
                             : "_" + StringClass + "ClassReference";
-      const llvm::Type *PTy = llvm::ArrayType::get(Ty, 0);
+      llvm::Type *PTy = llvm::ArrayType::get(Ty, 0);
       GV = CreateRuntimeVariable(PTy, str);
       // Decay array -> ptr
       ConstantStringClassRef = 
-        llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
+        llvm::ConstantExpr::getGetElementPtr(GV, Zeros);
     }
   }
-  
-  QualType NSTy = getContext().getNSConstantStringType();
-  
-  const llvm::StructType *STy =
-  cast<llvm::StructType>(getTypes().ConvertType(NSTy));
+
+  if (!NSConstantStringType) {
+    // Construct the type for a constant NSString.
+    RecordDecl *D = CreateRecordDecl(Context, TTK_Struct, 
+                                     Context.getTranslationUnitDecl(),
+                                   &Context.Idents.get("__builtin_NSString"));
+    D->startDefinition();
+      
+    QualType FieldTypes[3];
+    
+    // const int *isa;
+    FieldTypes[0] = Context.getPointerType(Context.IntTy.withConst());
+    // const char *str;
+    FieldTypes[1] = Context.getPointerType(Context.CharTy.withConst());
+    // unsigned int length;
+    FieldTypes[2] = Context.UnsignedIntTy;
+    
+    // Create fields
+    for (unsigned i = 0; i < 3; ++i) {
+      FieldDecl *Field = FieldDecl::Create(Context, D,
+                                           SourceLocation(),
+                                           SourceLocation(), 0,
+                                           FieldTypes[i], /*TInfo=*/0,
+                                           /*BitWidth=*/0,
+                                           /*Mutable=*/false,
+                                           /*HasInit=*/false);
+      Field->setAccess(AS_public);
+      D->addDecl(Field);
+    }
+    
+    D->completeDefinition();
+    QualType NSTy = Context.getTagDeclType(D);
+    NSConstantStringType = cast<llvm::StructType>(getTypes().ConvertType(NSTy));
+  }
   
   std::vector<llvm::Constant*> Fields(3);
   
@@ -1831,28 +1926,63 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) {
   GV->setUnnamedAddr(true);
   CharUnits Align = getContext().getTypeAlignInChars(getContext().CharTy);
   GV->setAlignment(Align.getQuantity());
-  Fields[1] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
+  Fields[1] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros);
   
   // String length.
-  const llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy);
+  llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy);
   Fields[2] = llvm::ConstantInt::get(Ty, StringLength);
   
   // The struct.
-  C = llvm::ConstantStruct::get(STy, Fields);
+  C = llvm::ConstantStruct::get(NSConstantStringType, Fields);
   GV = new llvm::GlobalVariable(getModule(), C->getType(), true,
                                 llvm::GlobalVariable::PrivateLinkage, C,
                                 "_unnamed_nsstring_");
   // FIXME. Fix section.
   if (const char *Sect = 
         Features.ObjCNonFragileABI 
-          ? getContext().Target.getNSStringNonFragileABISection() 
-          : getContext().Target.getNSStringSection())
+          ? getContext().getTargetInfo().getNSStringNonFragileABISection() 
+          : getContext().getTargetInfo().getNSStringSection())
     GV->setSection(Sect);
   Entry.setValue(GV);
   
   return GV;
 }
 
+QualType CodeGenModule::getObjCFastEnumerationStateType() {
+  if (ObjCFastEnumerationStateType.isNull()) {
+    RecordDecl *D = CreateRecordDecl(Context, TTK_Struct, 
+                                     Context.getTranslationUnitDecl(),
+                      &Context.Idents.get("__objcFastEnumerationState"));
+    D->startDefinition();
+    
+    QualType FieldTypes[] = {
+      Context.UnsignedLongTy,
+      Context.getPointerType(Context.getObjCIdType()),
+      Context.getPointerType(Context.UnsignedLongTy),
+      Context.getConstantArrayType(Context.UnsignedLongTy,
+                           llvm::APInt(32, 5), ArrayType::Normal, 0)
+    };
+    
+    for (size_t i = 0; i < 4; ++i) {
+      FieldDecl *Field = FieldDecl::Create(Context,
+                                           D,
+                                           SourceLocation(),
+                                           SourceLocation(), 0,
+                                           FieldTypes[i], /*TInfo=*/0,
+                                           /*BitWidth=*/0,
+                                           /*Mutable=*/false,
+                                           /*HasInit=*/false);
+      Field->setAccess(AS_public);
+      D->addDecl(Field);
+    }
+    
+    D->completeDefinition();
+    ObjCFastEnumerationStateType = Context.getTagDeclType(D);
+  }
+  
+  return ObjCFastEnumerationStateType;
+}
+
 /// GetStringForStringLiteral - Return the appropriate bytes for a
 /// string literal, properly padded to match the literal type.
 std::string CodeGenModule::GetStringForStringLiteral(const StringLiteral *E) {
@@ -1864,8 +1994,20 @@ std::string CodeGenModule::GetStringForStringLiteral(const StringLiteral *E) {
   // Resize the string to the right size.
   uint64_t RealLen = CAT->getSize().getZExtValue();
 
-  if (E->isWide())
-    RealLen *= Context.Target.getWCharWidth() / Context.getCharWidth();
+  switch (E->getKind()) {
+  case StringLiteral::Ascii:
+  case StringLiteral::UTF8:
+    break;
+  case StringLiteral::Wide:
+    RealLen *= Context.getTargetInfo().getWCharWidth() / Context.getCharWidth();
+    break;
+  case StringLiteral::UTF16:
+    RealLen *= Context.getTargetInfo().getChar16Width() / Context.getCharWidth();
+    break;
+  case StringLiteral::UTF32:
+    RealLen *= Context.getTargetInfo().getChar32Width() / Context.getCharWidth();
+    break;
+  }
 
   std::string Str = E->getString().str();
   Str.resize(RealLen, '\0');
@@ -1879,8 +2021,11 @@ llvm::Constant *
 CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S) {
   // FIXME: This can be more efficient.
   // FIXME: We shouldn't need to bitcast the constant in the wide string case.
-  llvm::Constant *C = GetAddrOfConstantString(GetStringForStringLiteral(S));
-  if (S->isWide()) {
+  CharUnits Align = getContext().getTypeAlignInChars(S->getType());
+  llvm::Constant *C = GetAddrOfConstantString(GetStringForStringLiteral(S),
+                                              /* GlobalName */ 0,
+                                              Align.getQuantity());
+  if (S->isWide() || S->isUTF16() || S->isUTF32()) {
     llvm::Type *DestTy =
         llvm::PointerType::getUnqual(getTypes().ConvertType(S->getType()));
     C = llvm::ConstantExpr::getBitCast(C, DestTy);
@@ -1900,10 +2045,11 @@ CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) {
 
 
 /// GenerateWritableString -- Creates storage for a string literal.
-static llvm::Constant *GenerateStringLiteral(llvm::StringRef str,
+static llvm::GlobalVariable *GenerateStringLiteral(StringRef str,
                                              bool constant,
                                              CodeGenModule &CGM,
-                                             const char *GlobalName) {
+                                             const char *GlobalName,
+                                             unsigned Alignment) {
   // Create Constant for this string literal. Don't add a '\0'.
   llvm::Constant *C =
       llvm::ConstantArray::get(CGM.getLLVMContext(), str, false);
@@ -1913,7 +2059,7 @@ static llvm::Constant *GenerateStringLiteral(llvm::StringRef str,
     new llvm::GlobalVariable(CGM.getModule(), C->getType(), constant,
                              llvm::GlobalValue::PrivateLinkage,
                              C, GlobalName);
-  GV->setAlignment(1);
+  GV->setAlignment(Alignment);
   GV->setUnnamedAddr(true);
   return GV;
 }
@@ -1926,8 +2072,9 @@ static llvm::Constant *GenerateStringLiteral(llvm::StringRef str,
 /// Feature.WriteableStrings.
 ///
 /// The result has pointer to array type.
-llvm::Constant *CodeGenModule::GetAddrOfConstantString(llvm::StringRef Str,
-                                                       const char *GlobalName) {
+llvm::Constant *CodeGenModule::GetAddrOfConstantString(StringRef Str,
+                                                       const char *GlobalName,
+                                                       unsigned Alignment) {
   bool IsConstant = !Features.WritableStrings;
 
   // Get the default prefix if a name wasn't specified.
@@ -1936,27 +2083,32 @@ llvm::Constant *CodeGenModule::GetAddrOfConstantString(llvm::StringRef Str,
 
   // Don't share any string literals if strings aren't constant.
   if (!IsConstant)
-    return GenerateStringLiteral(Str, false, *this, GlobalName);
+    return GenerateStringLiteral(Str, false, *this, GlobalName, Alignment);
 
-  llvm::StringMapEntry<llvm::Constant *> &Entry =
+  llvm::StringMapEntry<llvm::GlobalVariable *> &Entry =
     ConstantStringMap.GetOrCreateValue(Str);
 
-  if (Entry.getValue())
-    return Entry.getValue();
+  if (llvm::GlobalVariable *GV = Entry.getValue()) {
+    if (Alignment > GV->getAlignment()) {
+      GV->setAlignment(Alignment);
+    }
+    return GV;
+  }
 
   // Create a global variable for this.
-  llvm::Constant *C = GenerateStringLiteral(Str, true, *this, GlobalName);
-  Entry.setValue(C);
-  return C;
+  llvm::GlobalVariable *GV = GenerateStringLiteral(Str, true, *this, GlobalName, Alignment);
+  Entry.setValue(GV);
+  return GV;
 }
 
 /// GetAddrOfConstantCString - Returns a pointer to a character
 /// array containing the literal and a terminating '\0'
 /// character. The result has pointer to array type.
 llvm::Constant *CodeGenModule::GetAddrOfConstantCString(const std::string &Str,
-                                                        const char *GlobalName){
-  llvm::StringRef StrWithNull(Str.c_str(), Str.size() + 1);
-  return GetAddrOfConstantString(StrWithNull, GlobalName);
+                                                        const char *GlobalName,
+                                                        unsigned Alignment) {
+  StringRef StrWithNull(Str.c_str(), Str.size() + 1);
+  return GetAddrOfConstantString(StrWithNull, GlobalName, Alignment);
 }
 
 /// EmitObjCPropertyImplementations - Emit information for synthesized
@@ -1988,9 +2140,8 @@ void CodeGenModule::EmitObjCPropertyImplementations(const
 }
 
 static bool needsDestructMethod(ObjCImplementationDecl *impl) {
-  ObjCInterfaceDecl *iface
-    = const_cast<ObjCInterfaceDecl*>(impl->getClassInterface());
-  for (ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
+  const ObjCInterfaceDecl *iface = impl->getClassInterface();
+  for (const ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
        ivar; ivar = ivar->getNextIvar())
     if (ivar->getType().isDestructedType())
       return true;
@@ -2007,8 +2158,10 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
     Selector cxxSelector = getContext().Selectors.getSelector(0, &II);
     ObjCMethodDecl *DTORMethod =
       ObjCMethodDecl::Create(getContext(), D->getLocation(), D->getLocation(),
-                             cxxSelector, getContext().VoidTy, 0, D, true,
-                             false, true, false, ObjCMethodDecl::Required);
+                             cxxSelector, getContext().VoidTy, 0, D,
+                             /*isInstance=*/true, /*isVariadic=*/false,
+                          /*isSynthesized=*/true, /*isImplicitlyDeclared=*/true,
+                             /*isDefined=*/false, ObjCMethodDecl::Required);
     D->addInstanceMethod(DTORMethod);
     CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, DTORMethod, false);
     D->setHasCXXStructors(true);
@@ -2024,9 +2177,14 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
   // The constructor returns 'self'.
   ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create(getContext(), 
                                                 D->getLocation(),
-                                                D->getLocation(), cxxSelector,
+                                                D->getLocation(),
+                                                cxxSelector,
                                                 getContext().getObjCIdType(), 0, 
-                                                D, true, false, true, false,
+                                                D, /*isInstance=*/true,
+                                                /*isVariadic=*/false,
+                                                /*isSynthesized=*/true,
+                                                /*isImplicitlyDeclared=*/true,
+                                                /*isDefined=*/false,
                                                 ObjCMethodDecl::Required);
   D->addInstanceMethod(CTORMethod);
   CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, CTORMethod, true);
@@ -2134,13 +2292,13 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
   }
 
   case Decl::ObjCProtocol:
-    Runtime->GenerateProtocol(cast<ObjCProtocolDecl>(D));
+    ObjCRuntime->GenerateProtocol(cast<ObjCProtocolDecl>(D));
     break;
 
   case Decl::ObjCCategoryImpl:
     // Categories have properties but don't support synthesize so we
     // can ignore them here.
-    Runtime->GenerateCategory(cast<ObjCCategoryImplDecl>(D));
+    ObjCRuntime->GenerateCategory(cast<ObjCCategoryImplDecl>(D));
     break;
 
   case Decl::ObjCImplementation: {
@@ -2149,7 +2307,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
       Context.ResetObjCLayout(OMD->getClassInterface());
     EmitObjCPropertyImplementations(OMD);
     EmitObjCIvarInitializations(OMD);
-    Runtime->GenerateClass(OMD);
+    ObjCRuntime->GenerateClass(OMD);
     break;
   }
   case Decl::ObjCMethod: {
@@ -2169,11 +2327,13 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
 
   case Decl::FileScopeAsm: {
     FileScopeAsmDecl *AD = cast<FileScopeAsmDecl>(D);
-    llvm::StringRef AsmString = AD->getAsmString()->getString();
+    StringRef AsmString = AD->getAsmString()->getString();
 
     const std::string &S = getModule().getModuleInlineAsm();
     if (S.empty())
       getModule().setModuleInlineAsm(AsmString);
+    else if (*--S.end() == '\n')
+      getModule().setModuleInlineAsm(S + AsmString.str());
     else
       getModule().setModuleInlineAsm(S + '\n' + AsmString.str());
     break;
@@ -2191,7 +2351,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
 static llvm::Constant *GetPointerConstant(llvm::LLVMContext &Context,
                                           const void *Ptr) {
   uintptr_t PtrInt = reinterpret_cast<uintptr_t>(Ptr);
-  const llvm::Type *i64 = llvm::Type::getInt64Ty(Context);
+  llvm::Type *i64 = llvm::Type::getInt64Ty(Context);
   return llvm::ConstantInt::get(i64, PtrInt);
 }
 
@@ -2222,7 +2382,7 @@ void CodeGenModule::EmitDeclMetadata() {
   llvm::NamedMDNode *GlobalMetadata = 0;
 
   // StaticLocalDeclMap
-  for (llvm::DenseMap<GlobalDecl,llvm::StringRef>::iterator
+  for (llvm::DenseMap<GlobalDecl,StringRef>::iterator
          I = MangledDeclNames.begin(), E = MangledDeclNames.end();
        I != E; ++I) {
     llvm::GlobalValue *Addr = getModule().getNamedValue(I->second);
@@ -2273,81 +2433,3 @@ void CodeGenModule::EmitCoverageFile() {
     }
   }
 }
-
-///@name Custom Runtime Function Interfaces
-///@{
-//
-// FIXME: These can be eliminated once we can have clients just get the required
-// AST nodes from the builtin tables.
-
-llvm::Constant *CodeGenModule::getBlockObjectDispose() {
-  if (BlockObjectDispose)
-    return BlockObjectDispose;
-
-  // If we saw an explicit decl, use that.
-  if (BlockObjectDisposeDecl) {
-    return BlockObjectDispose = GetAddrOfFunction(
-      BlockObjectDisposeDecl,
-      getTypes().GetFunctionType(BlockObjectDisposeDecl));
-  }
-
-  // Otherwise construct the function by hand.
-  llvm::Type *args[] = { Int8PtrTy, Int32Ty };
-  const llvm::FunctionType *fty
-    = llvm::FunctionType::get(VoidTy, args, false);
-  return BlockObjectDispose =
-    CreateRuntimeFunction(fty, "_Block_object_dispose");
-}
-
-llvm::Constant *CodeGenModule::getBlockObjectAssign() {
-  if (BlockObjectAssign)
-    return BlockObjectAssign;
-
-  // If we saw an explicit decl, use that.
-  if (BlockObjectAssignDecl) {
-    return BlockObjectAssign = GetAddrOfFunction(
-      BlockObjectAssignDecl,
-      getTypes().GetFunctionType(BlockObjectAssignDecl));
-  }
-
-  // Otherwise construct the function by hand.
-  llvm::Type *args[] = { Int8PtrTy, Int8PtrTy, Int32Ty };
-  const llvm::FunctionType *fty
-    = llvm::FunctionType::get(VoidTy, args, false);
-  return BlockObjectAssign =
-    CreateRuntimeFunction(fty, "_Block_object_assign");
-}
-
-llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
-  if (NSConcreteGlobalBlock)
-    return NSConcreteGlobalBlock;
-
-  // If we saw an explicit decl, use that.
-  if (NSConcreteGlobalBlockDecl) {
-    return NSConcreteGlobalBlock = GetAddrOfGlobalVar(
-      NSConcreteGlobalBlockDecl,
-      getTypes().ConvertType(NSConcreteGlobalBlockDecl->getType()));
-  }
-
-  // Otherwise construct the variable by hand.
-  return NSConcreteGlobalBlock =
-    CreateRuntimeVariable(Int8PtrTy, "_NSConcreteGlobalBlock");
-}
-
-llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
-  if (NSConcreteStackBlock)
-    return NSConcreteStackBlock;
-
-  // If we saw an explicit decl, use that.
-  if (NSConcreteStackBlockDecl) {
-    return NSConcreteStackBlock = GetAddrOfGlobalVar(
-      NSConcreteStackBlockDecl,
-      getTypes().ConvertType(NSConcreteStackBlockDecl->getType()));
-  }
-
-  // Otherwise construct the variable by hand.
-  return NSConcreteStackBlock =
-    CreateRuntimeVariable(Int8PtrTy, "_NSConcreteStackBlock");
-}
-
-///@}
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h
index 86fb6d4d030e..8e38a8999013 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h
@@ -62,7 +62,7 @@ namespace clang {
   class VarDecl;
   class LangOptions;
   class CodeGenOptions;
-  class Diagnostic;
+  class DiagnosticsEngine;
   class AnnotateAttr;
   class CXXDestructorDecl;
   class MangleBuffer;
@@ -75,6 +75,8 @@ namespace CodeGen {
   class CGCXXABI;
   class CGDebugInfo;
   class CGObjCRuntime;
+  class CGOpenCLRuntime;
+  class CGCUDARuntime;
   class BlockFieldFlags;
   class FunctionArgList;
   
@@ -129,8 +131,12 @@ namespace CodeGen {
     /// The width of a pointer into the generic address space.
     unsigned char PointerWidthInBits;
 
-    /// The alignment of a pointer into the generic address space.
-    unsigned char PointerAlignInBytes;
+    /// The size and alignment of a pointer into the generic address
+    /// space.
+    union {
+      unsigned char PointerAlignInBytes;
+      unsigned char PointerSizeInBytes;
+    };
   };
 
 struct RREntrypoints {
@@ -212,7 +218,7 @@ class CodeGenModule : public CodeGenTypeCache {
   llvm::Module &TheModule;
   const llvm::TargetData &TheTargetData;
   mutable const TargetCodeGenInfo *TheTargetCodeGenInfo;
-  Diagnostic &Diags;
+  DiagnosticsEngine &Diags;
   CGCXXABI &ABI;
   CodeGenTypes Types;
   CodeGenTBAA *TBAA;
@@ -221,7 +227,9 @@ class CodeGenModule : public CodeGenTypeCache {
   CodeGenVTables VTables;
   friend class CodeGenVTables;
 
-  CGObjCRuntime* Runtime;
+  CGObjCRuntime* ObjCRuntime;
+  CGOpenCLRuntime* OpenCLRuntime;
+  CGCUDARuntime* CUDARuntime;
   CGDebugInfo* DebugInfo;
   ARCEntrypoints *ARCData;
   RREntrypoints *RRData;
@@ -257,13 +265,17 @@ class CodeGenModule : public CodeGenTypeCache {
   CtorList GlobalDtors;
 
   /// MangledDeclNames - A map of canonical GlobalDecls to their mangled names.
-  llvm::DenseMap<GlobalDecl, llvm::StringRef> MangledDeclNames;
+  llvm::DenseMap<GlobalDecl, StringRef> MangledDeclNames;
   llvm::BumpPtrAllocator MangledNamesAllocator;
   
+  /// Global annotations.
   std::vector<llvm::Constant*> Annotations;
 
+  /// Map used to get unique annotation strings.
+  llvm::StringMap<llvm::Constant*> AnnotationStrings;
+
   llvm::StringMap<llvm::Constant*> CFConstantStringMap;
-  llvm::StringMap<llvm::Constant*> ConstantStringMap;
+  llvm::StringMap<llvm::GlobalVariable*> ConstantStringMap;
   llvm::DenseMap<const Decl*, llvm::Value*> StaticLocalDeclMap;
 
   /// CXXGlobalInits - Global variables with initializers that need to run
@@ -279,13 +291,16 @@ class CodeGenModule : public CodeGenTypeCache {
   /// - Global variables with initializers whose order of initialization
   /// is set by init_priority attribute.
   
-  llvm::SmallVector<std::pair<OrderGlobalInits, llvm::Function*>, 8> 
+  SmallVector<std::pair<OrderGlobalInits, llvm::Function*>, 8> 
     PrioritizedCXXGlobalInits;
 
   /// CXXGlobalDtors - Global destructor functions and arguments that need to
   /// run on termination.
   std::vector<std::pair<llvm::WeakVH,llvm::Constant*> > CXXGlobalDtors;
 
+  /// @name Cache for Objective-C runtime types
+  /// @{
+
   /// CFConstantStringClassRef - Cached reference to the class for constant
   /// strings. This value has type int * but is actually an Obj-C class pointer.
   llvm::Constant *CFConstantStringClassRef;
@@ -294,21 +309,29 @@ class CodeGenModule : public CodeGenTypeCache {
   /// strings. This value has type int * but is actually an Obj-C class pointer.
   llvm::Constant *ConstantStringClassRef;
 
+  /// \brief The LLVM type corresponding to NSConstantString.
+  llvm::StructType *NSConstantStringType;
+  
+  /// \brief The type used to describe the state of a fast enumeration in
+  /// Objective-C's for..in loop.
+  QualType ObjCFastEnumerationStateType;
+  
+  /// @}
+
   /// Lazily create the Objective-C runtime
   void createObjCRuntime();
 
+  void createOpenCLRuntime();
+  void createCUDARuntime();
+
   llvm::LLVMContext &VMContext;
 
   /// @name Cache for Blocks Runtime Globals
   /// @{
 
-  const VarDecl *NSConcreteGlobalBlockDecl;
-  const VarDecl *NSConcreteStackBlockDecl;
   llvm::Constant *NSConcreteGlobalBlock;
   llvm::Constant *NSConcreteStackBlock;
 
-  const FunctionDecl *BlockObjectAssignDecl;
-  const FunctionDecl *BlockObjectDisposeDecl;
   llvm::Constant *BlockObjectAssign;
   llvm::Constant *BlockObjectDispose;
 
@@ -322,7 +345,8 @@ class CodeGenModule : public CodeGenTypeCache {
   /// @}
 public:
   CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts,
-                llvm::Module &M, const llvm::TargetData &TD, Diagnostic &Diags);
+                llvm::Module &M, const llvm::TargetData &TD,
+                DiagnosticsEngine &Diags);
 
   ~CodeGenModule();
 
@@ -332,13 +356,25 @@ class CodeGenModule : public CodeGenTypeCache {
   /// getObjCRuntime() - Return a reference to the configured
   /// Objective-C runtime.
   CGObjCRuntime &getObjCRuntime() {
-    if (!Runtime) createObjCRuntime();
-    return *Runtime;
+    if (!ObjCRuntime) createObjCRuntime();
+    return *ObjCRuntime;
   }
 
   /// hasObjCRuntime() - Return true iff an Objective-C runtime has
   /// been configured.
-  bool hasObjCRuntime() { return !!Runtime; }
+  bool hasObjCRuntime() { return !!ObjCRuntime; }
+
+  /// getOpenCLRuntime() - Return a reference to the configured OpenCL runtime.
+  CGOpenCLRuntime &getOpenCLRuntime() {
+    assert(OpenCLRuntime != 0);
+    return *OpenCLRuntime;
+  }
+
+  /// getCUDARuntime() - Return a reference to the configured CUDA runtime.
+  CGCUDARuntime &getCUDARuntime() {
+    assert(CUDARuntime != 0);
+    return *CUDARuntime;
+  }
 
   /// getCXXABI() - Return a reference to the configured C++ ABI.
   CGCXXABI &getCXXABI() { return ABI; }
@@ -369,9 +405,10 @@ class CodeGenModule : public CodeGenTypeCache {
   llvm::Module &getModule() const { return TheModule; }
   CodeGenTypes &getTypes() { return Types; }
   CodeGenVTables &getVTables() { return VTables; }
-  Diagnostic &getDiags() const { return Diags; }
+  VTableContext &getVTableContext() { return VTables.getVTableContext(); }
+  DiagnosticsEngine &getDiags() const { return Diags; }
   const llvm::TargetData &getTargetData() const { return TheTargetData; }
-  const TargetInfo &getTarget() const { return Context.Target; }
+  const TargetInfo &getTarget() const { return Context.getTargetInfo(); }
   llvm::LLVMContext &getLLVMContext() { return VMContext; }
   const TargetCodeGenInfo &getTargetCodeGenInfo();
   bool isTargetDarwin() const;
@@ -433,7 +470,7 @@ class CodeGenModule : public CodeGenTypeCache {
   /// variable with the right type will be created and all uses of the old
   /// variable will be replaced with a bitcast to the new variable.
   llvm::GlobalVariable *
-  CreateOrReplaceCXXRuntimeVariable(llvm::StringRef Name, const llvm::Type *Ty,
+  CreateOrReplaceCXXRuntimeVariable(StringRef Name, llvm::Type *Ty,
                                     llvm::GlobalValue::LinkageTypes Linkage);
 
   /// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
@@ -441,14 +478,14 @@ class CodeGenModule : public CodeGenTypeCache {
   /// then it will be greated with the specified type instead of whatever the
   /// normal requested type would be.
   llvm::Constant *GetAddrOfGlobalVar(const VarDecl *D,
-                                     const llvm::Type *Ty = 0);
+                                     llvm::Type *Ty = 0);
 
 
   /// GetAddrOfFunction - Return the address of the given function.  If Ty is
   /// non-null, then this function will use the specified type if it has to
   /// create it.
   llvm::Constant *GetAddrOfFunction(GlobalDecl GD,
-                                    const llvm::Type *Ty = 0,
+                                    llvm::Type *Ty = 0,
                                     bool ForVTable = false);
 
   /// GetAddrOfRTTIDescriptor - Get the address of the RTTI descriptor 
@@ -544,8 +581,9 @@ class CodeGenModule : public CodeGenTypeCache {
   ///
   /// \param GlobalName If provided, the name to use for the global
   /// (if one is created).
-  llvm::Constant *GetAddrOfConstantString(llvm::StringRef Str,
-                                          const char *GlobalName=0);
+  llvm::Constant *GetAddrOfConstantString(StringRef Str,
+                                          const char *GlobalName=0,
+                                          unsigned Alignment=1);
 
   /// GetAddrOfConstantCString - Returns a pointer to a character array
   /// containing the literal and a terminating '\0' character. The result has
@@ -554,7 +592,12 @@ class CodeGenModule : public CodeGenTypeCache {
   /// \param GlobalName If provided, the name to use for the global (if one is
   /// created).
   llvm::Constant *GetAddrOfConstantCString(const std::string &str,
-                                           const char *GlobalName=0);
+                                           const char *GlobalName=0,
+                                           unsigned Alignment=1);
+  
+  /// \brief Retrieve the record type that describes the state of an
+  /// Objective-C fast enumeration loop (for..in).
+  QualType getObjCFastEnumerationStateType();
   
   /// GetAddrOfCXXConstructor - Return the address of the constructor of the
   /// given type.
@@ -573,8 +616,8 @@ class CodeGenModule : public CodeGenTypeCache {
   llvm::Value *getBuiltinLibFunction(const FunctionDecl *FD,
                                      unsigned BuiltinID);
 
-  llvm::Function *getIntrinsic(unsigned IID, llvm::ArrayRef<llvm::Type*> Tys =
-                                                 llvm::ArrayRef<llvm::Type*>());
+  llvm::Function *getIntrinsic(unsigned IID, ArrayRef<llvm::Type*> Tys =
+                                                 ArrayRef<llvm::Type*>());
 
   /// EmitTopLevelDecl - Emit code for a single top level declaration.
   void EmitTopLevelDecl(Decl *D);
@@ -584,8 +627,6 @@ class CodeGenModule : public CodeGenTypeCache {
   /// metadata global.
   void AddUsedGlobal(llvm::GlobalValue *GV);
 
-  void AddAnnotation(llvm::Constant *C) { Annotations.push_back(C); }
-
   /// AddCXXDtorEntry - Add a destructor and object to add to the C++ global
   /// destructor function.
   void AddCXXDtorEntry(llvm::Constant *DtorFn, llvm::Constant *Object) {
@@ -594,14 +635,14 @@ class CodeGenModule : public CodeGenTypeCache {
 
   /// CreateRuntimeFunction - Create a new runtime function with the specified
   /// type and name.
-  llvm::Constant *CreateRuntimeFunction(const llvm::FunctionType *Ty,
-                                        llvm::StringRef Name,
+  llvm::Constant *CreateRuntimeFunction(llvm::FunctionType *Ty,
+                                        StringRef Name,
                                         llvm::Attributes ExtraAttrs =
                                           llvm::Attribute::None);
   /// CreateRuntimeVariable - Create a new runtime global variable with the
   /// specified type and name.
-  llvm::Constant *CreateRuntimeVariable(const llvm::Type *Ty,
-                                        llvm::StringRef Name);
+  llvm::Constant *CreateRuntimeVariable(llvm::Type *Ty,
+                                        StringRef Name);
 
   ///@name Custom Blocks Runtime Interfaces
   ///@{
@@ -629,11 +670,13 @@ class CodeGenModule : public CodeGenTypeCache {
   /// but not always, an LLVM null constant.
   llvm::Constant *EmitNullConstant(QualType T);
 
-  llvm::Constant *EmitAnnotateAttr(llvm::GlobalValue *GV,
-                                   const AnnotateAttr *AA, unsigned LineNo);
+  /// EmitNullConstantForBase - Return a null constant appropriate for 
+  /// zero-initializing a base class with the given type.  This is usually,
+  /// but not always, an LLVM null constant.
+  llvm::Constant *EmitNullConstantForBase(const CXXRecordDecl *Record);
 
   /// Error - Emit a general error that something can't be done.
-  void Error(SourceLocation loc, llvm::StringRef error);
+  void Error(SourceLocation loc, StringRef error);
 
   /// ErrorUnsupported - Print out an error that codegen doesn't support the
   /// specified stmt yet.
@@ -688,7 +731,7 @@ class CodeGenModule : public CodeGenTypeCache {
                               AttributeListType &PAL,
                               unsigned &CallingConv);
 
-  llvm::StringRef getMangledName(GlobalDecl GD);
+  StringRef getMangledName(GlobalDecl GD);
   void getBlockMangledName(GlobalDecl GD, MangleBuffer &Buffer,
                            const BlockDecl *BD);
 
@@ -709,7 +752,7 @@ class CodeGenModule : public CodeGenTypeCache {
 
   /// GetTargetTypeStoreSize - Return the store size, in character units, of
   /// the given LLVM type.
-  CharUnits GetTargetTypeStoreSize(const llvm::Type *Ty) const;
+  CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const;
   
   /// GetLLVMLinkageVarDefinition - Returns LLVM linkage for a global 
   /// variable.
@@ -719,17 +762,44 @@ class CodeGenModule : public CodeGenTypeCache {
   
   std::vector<const CXXRecordDecl*> DeferredVTables;
 
-private:
-  llvm::GlobalValue *GetGlobalValue(llvm::StringRef Ref);
+  /// Emit all the global annotations.
+  void EmitGlobalAnnotations();
 
-  llvm::Constant *GetOrCreateLLVMFunction(llvm::StringRef MangledName,
-                                          const llvm::Type *Ty,
+  /// Emit an annotation string.
+  llvm::Constant *EmitAnnotationString(llvm::StringRef Str);
+
+  /// Emit the annotation's translation unit.
+  llvm::Constant *EmitAnnotationUnit(SourceLocation Loc);
+
+  /// Emit the annotation line number.
+  llvm::Constant *EmitAnnotationLineNo(SourceLocation L);
+
+  /// EmitAnnotateAttr - Generate the llvm::ConstantStruct which contains the
+  /// annotation information for a given GlobalValue. The annotation struct is
+  /// {i8 *, i8 *, i8 *, i32}. The first field is a constant expression, the
+  /// GlobalValue being annotated. The second field is the constant string
+  /// created from the AnnotateAttr's annotation. The third field is a constant
+  /// string containing the name of the translation unit. The fourth field is
+  /// the line number in the file of the annotated value declaration.
+  llvm::Constant *EmitAnnotateAttr(llvm::GlobalValue *GV,
+                                   const AnnotateAttr *AA,
+                                   SourceLocation L);
+
+  /// Add global annotations that are set on D, for the global GV. Those
+  /// annotations are emitted during finalization of the LLVM code.
+  void AddGlobalAnnotations(const ValueDecl *D, llvm::GlobalValue *GV);
+
+private:
+  llvm::GlobalValue *GetGlobalValue(StringRef Ref);
+
+  llvm::Constant *GetOrCreateLLVMFunction(StringRef MangledName,
+                                          llvm::Type *Ty,
                                           GlobalDecl D,
                                           bool ForVTable,
                                           llvm::Attributes ExtraAttrs =
                                             llvm::Attribute::None);
-  llvm::Constant *GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
-                                        const llvm::PointerType *PTy,
+  llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName,
+                                        llvm::PointerType *PTy,
                                         const VarDecl *D,
                                         bool UnnamedAddr = false);
 
@@ -804,8 +874,6 @@ class CodeGenModule : public CodeGenTypeCache {
   /// suitable for use as a LLVM constructor or destructor array.
   void EmitCtorList(const CtorList &Fns, const char *GlobalName);
 
-  void EmitAnnotations(void);
-
   /// EmitFundamentalRTTIDescriptor - Emit the RTTI descriptors for the
   /// given type.
   void EmitFundamentalRTTIDescriptor(QualType Type);
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp
index 53e40b2238b3..887c1eabb0c6 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -58,7 +58,7 @@ llvm::MDNode *CodeGenTBAA::getChar() {
 
 /// getTBAAInfoForNamedType - Create a TBAA tree node with the given string
 /// as its identifier, and the given Parent node as its tree parent.
-llvm::MDNode *CodeGenTBAA::getTBAAInfoForNamedType(llvm::StringRef NameStr,
+llvm::MDNode *CodeGenTBAA::getTBAAInfoForNamedType(StringRef NameStr,
                                                    llvm::MDNode *Parent,
                                                    bool Readonly) {
   // Currently there is only one flag defined - the readonly flag.
@@ -75,7 +75,7 @@ llvm::MDNode *CodeGenTBAA::getTBAAInfoForNamedType(llvm::StringRef NameStr,
 
   // Create the mdnode.
   unsigned Len = llvm::array_lengthof(Ops) - !Flags;
-  return llvm::MDNode::get(VMContext, llvm::ArrayRef<llvm::Value*>(Ops, Len));
+  return llvm::MDNode::get(VMContext, llvm::makeArrayRef(Ops, Len));
 }
 
 static bool TypeHasMayAlias(QualType QTy) {
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.h
index c4583473a0e0..9fe51fb33141 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.h
@@ -15,7 +15,7 @@
 #ifndef CLANG_CODEGEN_CODEGENTBAA_H
 #define CLANG_CODEGEN_CODEGENTBAA_H
 
-#include "llvm/LLVMContext.h"
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
 
 namespace llvm {
@@ -55,7 +55,7 @@ class CodeGenTBAA {
   /// considered to be equivalent to it.
   llvm::MDNode *getChar();
 
-  llvm::MDNode *getTBAAInfoForNamedType(llvm::StringRef NameStr,
+  llvm::MDNode *getTBAAInfoForNamedType(StringRef NameStr,
                                         llvm::MDNode *Parent,
                                         bool Readonly = false);
 
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp
index 764688fcb12e..e0d921896580 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -29,7 +29,7 @@ using namespace CodeGen;
 CodeGenTypes::CodeGenTypes(ASTContext &Ctx, llvm::Module& M,
                            const llvm::TargetData &TD, const ABIInfo &Info,
                            CGCXXABI &CXXABI, const CodeGenOptions &CGO)
-  : Context(Ctx), Target(Ctx.Target), TheModule(M), TheTargetData(TD),
+  : Context(Ctx), Target(Ctx.getTargetInfo()), TheModule(M), TheTargetData(TD),
     TheABIInfo(Info), TheCXXABI(CXXABI), CodeGenOpts(CGO) {
   SkippedLayout = false;
 }
@@ -47,7 +47,7 @@ CodeGenTypes::~CodeGenTypes() {
 
 void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
                                      llvm::StructType *Ty,
-                                     llvm::StringRef suffix) {
+                                     StringRef suffix) {
   llvm::SmallString<256> TypeName;
   llvm::raw_svector_ostream OS(TypeName);
   OS << RD->getKindName() << '.';
@@ -263,6 +263,8 @@ void CodeGenTypes::UpdateCompletedType(const TagDecl *TD) {
 
 static llvm::Type *getTypeForFormat(llvm::LLVMContext &VMContext,
                                     const llvm::fltSemantics &format) {
+  if (&format == &llvm::APFloat::IEEEhalf)
+    return llvm::Type::getInt16Ty(VMContext);
   if (&format == &llvm::APFloat::IEEEsingle)
     return llvm::Type::getFloatTy(VMContext);
   if (&format == &llvm::APFloat::IEEEdouble)
@@ -273,8 +275,7 @@ static llvm::Type *getTypeForFormat(llvm::LLVMContext &VMContext,
     return llvm::Type::getPPC_FP128Ty(VMContext);
   if (&format == &llvm::APFloat::x87DoubleExtended)
     return llvm::Type::getX86_FP80Ty(VMContext);
-  assert(0 && "Unknown float format!");
-  return 0;
+  llvm_unreachable("Unknown float format!");
 }
 
 /// ConvertType - Convert the specified type to its LLVM form.
@@ -342,6 +343,14 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
                                  static_cast<unsigned>(Context.getTypeSize(T)));
       break;
 
+    case BuiltinType::Half:
+      // Half is special: it might be lowered to i16 (and will be storage-only
+      // type),. or can be represented as a set of native operations.
+
+      // FIXME: Ask target which kind of half FP it prefers (storage only vs
+      // native).
+      ResultType = llvm::Type::getInt16Ty(getLLVMContext());
+      break;
     case BuiltinType::Float:
     case BuiltinType::Double:
     case BuiltinType::LongDouble:
@@ -418,7 +427,15 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
   }
   case Type::ConstantArray: {
     const ConstantArrayType *A = cast<ConstantArrayType>(Ty);
-    const llvm::Type *EltTy = ConvertTypeForMem(A->getElementType());
+    llvm::Type *EltTy = ConvertTypeForMem(A->getElementType());
+    
+    // Lower arrays of undefined struct type to arrays of i8 just to have a 
+    // concrete type.
+    if (!EltTy->isSized()) {
+      SkippedLayout = true;
+      EltTy = llvm::Type::getInt8Ty(getLLVMContext());
+    }
+
     ResultType = llvm::ArrayType::get(EltTy, A->getSize().getZExtValue());
     break;
   }
@@ -502,7 +519,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     // these.
     llvm::Type *&T = InterfaceTypes[cast<ObjCInterfaceType>(Ty)];
     if (!T)
-      T = llvm::StructType::createNamed(getLLVMContext(), "");
+      T = llvm::StructType::create(getLLVMContext());
     ResultType = T;
     break;
   }
@@ -511,15 +528,15 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     // Protocol qualifications do not influence the LLVM type, we just return a
     // pointer to the underlying interface type. We don't need to worry about
     // recursive conversion.
-    const llvm::Type *T =
-      ConvertType(cast<ObjCObjectPointerType>(Ty)->getPointeeType());
+    llvm::Type *T =
+      ConvertTypeForMem(cast<ObjCObjectPointerType>(Ty)->getPointeeType());
     ResultType = T->getPointerTo();
     break;
   }
 
   case Type::Enum: {
     const EnumDecl *ED = cast<EnumType>(Ty)->getDecl();
-    if (ED->isDefinition() || ED->isFixed())
+    if (ED->isCompleteDefinition() || ED->isFixed())
       return ConvertType(ED->getIntegerType());
     // Return a placeholder 'i32' type.  This can be changed later when the
     // type is defined (see UpdateCompletedType), but is likely to be the
@@ -541,6 +558,11 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
       getCXXABI().ConvertMemberPointerType(cast<MemberPointerType>(Ty));
     break;
   }
+
+  case Type::Atomic: {
+    ResultType = ConvertTypeForMem(cast<AtomicType>(Ty)->getValueType());
+    break;
+  }
   }
   
   assert(ResultType && "Didn't convert a type?");
@@ -559,7 +581,7 @@ llvm::StructType *CodeGenTypes::ConvertRecordDeclType(const RecordDecl *RD) {
 
   // If we don't have a StructType at all yet, create the forward declaration.
   if (Entry == 0) {
-    Entry = llvm::StructType::createNamed(getLLVMContext(), "");
+    Entry = llvm::StructType::create(getLLVMContext());
     addRecordTypeName(RD, Entry, "");
   }
   llvm::StructType *Ty = Entry;
@@ -567,7 +589,7 @@ llvm::StructType *CodeGenTypes::ConvertRecordDeclType(const RecordDecl *RD) {
   // If this is still a forward declaration, or the LLVM type is already
   // complete, there's nothing more to do.
   RD = RD->getDefinition();
-  if (RD == 0 || !Ty->isOpaque())
+  if (RD == 0 || !RD->isCompleteDefinition() || !Ty->isOpaque())
     return Ty;
   
   // If converting this type would cause us to infinitely loop, don't do it!
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h
index 7c0fb8164373..7f0f8ac5f0c5 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h
@@ -93,7 +93,7 @@ class CodeGenTypes {
   /// a recursive struct conversion, set this to true.
   bool SkippedLayout;
 
-  llvm::SmallVector<const RecordDecl *, 8> DeferredRecords;
+  SmallVector<const RecordDecl *, 8> DeferredRecords;
   
 private:
   /// TypeCache - This map keeps cache of llvm::Types
@@ -138,7 +138,7 @@ class CodeGenTypes {
   /// GetFunctionTypeForVTable - Get the LLVM function type for use in a vtable,
   /// given a CXXMethodDecl. If the method to has an incomplete return type,
   /// and/or incomplete argument types, this will return the opaque type.
-  const llvm::Type *GetFunctionTypeForVTable(GlobalDecl GD);
+  llvm::Type *GetFunctionTypeForVTable(GlobalDecl GD);
 
   const CGRecordLayout &getCGRecordLayout(const RecordDecl*);
 
@@ -190,7 +190,7 @@ class CodeGenTypes {
   ///
   /// \param ArgTys - must all actually be canonical as params
   const CGFunctionInfo &getFunctionInfo(CanQualType RetTy,
-                               const llvm::SmallVectorImpl<CanQualType> &ArgTys,
+                               const SmallVectorImpl<CanQualType> &ArgTys,
                                         const FunctionType::ExtInfo &Info);
 
   /// \brief Compute a new LLVM record layout object for the given record.
@@ -200,7 +200,7 @@ class CodeGenTypes {
   /// addRecordTypeName - Compute a name from the given record decl with an
   /// optional suffix and name the given LLVM type using it.
   void addRecordTypeName(const RecordDecl *RD, llvm::StructType *Ty,
-                         llvm::StringRef suffix);
+                         StringRef suffix);
   
 
 public:  // These are internal details of CGT that shouldn't be used externally.
@@ -211,7 +211,7 @@ class CodeGenTypes {
   /// argument types it would be passed as on the provided vector \arg
   /// ArgTys. See ABIArgInfo::Expand.
   void GetExpandedTypes(QualType type,
-                        llvm::SmallVectorImpl<llvm::Type*> &expanded);
+                        SmallVectorImpl<llvm::Type*> &expanded);
 
   /// IsZeroInitializable - Return whether a type can be
   /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer.
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 0c86080fa80e..c3f635aed642 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -96,12 +96,12 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
   void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                  CXXCtorType T,
                                  CanQualType &ResTy,
-                                 llvm::SmallVectorImpl<CanQualType> &ArgTys);
+                                 SmallVectorImpl<CanQualType> &ArgTys);
 
   void BuildDestructorSignature(const CXXDestructorDecl *Dtor,
                                 CXXDtorType T,
                                 CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys);
+                                SmallVectorImpl<CanQualType> &ArgTys);
 
   void BuildInstanceFunctionParams(CodeGenFunction &CGF,
                                    QualType &ResTy,
@@ -131,12 +131,12 @@ class ARMCXXABI : public ItaniumCXXABI {
   void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                  CXXCtorType T,
                                  CanQualType &ResTy,
-                                 llvm::SmallVectorImpl<CanQualType> &ArgTys);
+                                 SmallVectorImpl<CanQualType> &ArgTys);
 
   void BuildDestructorSignature(const CXXDestructorDecl *Dtor,
                                 CXXDtorType T,
                                 CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys);
+                                SmallVectorImpl<CanQualType> &ArgTys);
 
   void BuildInstanceFunctionParams(CodeGenFunction &CGF,
                                    QualType &ResTy,
@@ -215,11 +215,11 @@ ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
   const CXXRecordDecl *RD = 
     cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
 
-  const llvm::FunctionType *FTy = 
+  llvm::FunctionType *FTy = 
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
                                    FPT->isVariadic());
 
-  const llvm::IntegerType *ptrdiff = getPtrDiffTy();
+  llvm::IntegerType *ptrdiff = getPtrDiffTy();
   llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(ptrdiff, 1);
 
   llvm::BasicBlock *FnVirtual = CGF.createBasicBlock("memptr.virtual");
@@ -259,7 +259,7 @@ ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
   CGF.EmitBlock(FnVirtual);
 
   // Cast the adjusted this to a pointer to vtable pointer and load.
-  const llvm::Type *VTableTy = Builder.getInt8PtrTy();
+  llvm::Type *VTableTy = Builder.getInt8PtrTy();
   llvm::Value *VTable = Builder.CreateBitCast(This, VTableTy->getPointerTo());
   VTable = Builder.CreateLoad(VTable, "memptr.vtable");
 
@@ -307,7 +307,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
 
   // Cast the address to the appropriate pointer type, adopting the
   // address space of the base pointer.
-  const llvm::Type *PType
+  llvm::Type *PType
     = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS);
   return Builder.CreateBitCast(Addr, PType);
 }
@@ -478,7 +478,7 @@ ItaniumCXXABI::EmitMemberPointerConversion(llvm::Constant *C,
 
 llvm::Constant *
 ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
-  const llvm::Type *ptrdiff_t = getPtrDiffTy();
+  llvm::Type *ptrdiff_t = getPtrDiffTy();
 
   // Itanium C++ ABI 2.3:
   //   A NULL pointer is represented as -1.
@@ -504,16 +504,16 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
   MD = MD->getCanonicalDecl();
 
   CodeGenTypes &Types = CGM.getTypes();
-  const llvm::Type *ptrdiff_t = getPtrDiffTy();
+  llvm::Type *ptrdiff_t = getPtrDiffTy();
 
   // Get the function pointer (or index if this is a virtual function).
   llvm::Constant *MemPtr[2];
   if (MD->isVirtual()) {
-    uint64_t Index = CGM.getVTables().getMethodVTableIndex(MD);
+    uint64_t Index = CGM.getVTableContext().getMethodVTableIndex(MD);
 
     const ASTContext &Context = getContext();
     CharUnits PointerWidth =
-      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
+      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
     uint64_t VTableOffset = (Index * PointerWidth.getQuantity());
 
     if (IsARM) {
@@ -535,7 +535,7 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
     }
   } else {
     const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
-    const llvm::Type *Ty;
+    llvm::Type *Ty;
     // Check whether the function has a computable LLVM signature.
     if (Types.isFuncTypeConvertible(FPT)) {
       // The function has a computable LLVM signature; use the correct type.
@@ -678,7 +678,7 @@ bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
 void ItaniumCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                               CXXCtorType Type,
                                               CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+                                SmallVectorImpl<CanQualType> &ArgTys) {
   ASTContext &Context = getContext();
 
   // 'this' is already there.
@@ -692,7 +692,7 @@ void ItaniumCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
 void ARMCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                           CXXCtorType Type,
                                           CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+                                SmallVectorImpl<CanQualType> &ArgTys) {
   ItaniumCXXABI::BuildConstructorSignature(Ctor, Type, ResTy, ArgTys);
   ResTy = ArgTys[0];
 }
@@ -702,7 +702,7 @@ void ARMCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
 void ItaniumCXXABI::BuildDestructorSignature(const CXXDestructorDecl *Dtor,
                                              CXXDtorType Type,
                                              CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+                                SmallVectorImpl<CanQualType> &ArgTys) {
   ASTContext &Context = getContext();
 
   // 'this' is already there.
@@ -717,7 +717,7 @@ void ItaniumCXXABI::BuildDestructorSignature(const CXXDestructorDecl *Dtor,
 void ARMCXXABI::BuildDestructorSignature(const CXXDestructorDecl *Dtor,
                                          CXXDtorType Type,
                                          CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+                                SmallVectorImpl<CanQualType> &ArgTys) {
   ItaniumCXXABI::BuildDestructorSignature(Dtor, Type, ResTy, ArgTys);
 
   if (Type != Dtor_Deleting)
@@ -784,7 +784,7 @@ void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
     return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
 
   // Destructor thunks in the ARM ABI have indeterminate results.
-  const llvm::Type *T =
+  llvm::Type *T =
     cast<llvm::PointerType>(CGF.ReturnValue->getType())->getElementType();
   RValue Undef = RValue::get(llvm::UndefValue::get(T));
   return ItaniumCXXABI::EmitReturnFromThunk(CGF, Undef, ResultType);
@@ -829,27 +829,7 @@ bool ItaniumCXXABI::NeedsArrayCookie(const CXXDeleteExpr *expr,
   if (expr->doesUsualArrayDeleteWantSize())
     return true;
 
-  // Automatic Reference Counting:
-  //   We need an array cookie for pointers with strong or weak lifetime.
-  if (getContext().getLangOptions().ObjCAutoRefCount &&
-      elementType->isObjCLifetimeType()) {
-    switch (elementType.getObjCLifetime()) {
-    case Qualifiers::OCL_None:
-    case Qualifiers::OCL_ExplicitNone:
-    case Qualifiers::OCL_Autoreleasing:
-      return false;
-
-    case Qualifiers::OCL_Strong:
-    case Qualifiers::OCL_Weak:
-      return true;
-    }
-  }
-  
-  // Otherwise, if the class has a non-trivial destructor, it always
-  // needs a cookie.
-  const CXXRecordDecl *record =
-    elementType->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
-  return (record && !record->hasTrivialDestructor());
+  return elementType.isDestructedType();
 }
 
 CharUnits ItaniumCXXABI::GetArrayCookieSize(const CXXNewExpr *expr) {
@@ -907,7 +887,7 @@ void ItaniumCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
                                     CharUnits &CookieSize) {
   // Derive a char* in the same address space as the pointer.
   unsigned AS = cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
-  const llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
+  llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
 
   // If we don't need an array cookie, bail out early.
   if (!NeedsArrayCookie(expr, ElementType)) {
@@ -919,7 +899,7 @@ void ItaniumCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
 
   QualType SizeTy = getContext().getSizeType();
   CharUnits SizeSize = getContext().getTypeSizeInChars(SizeTy);
-  const llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
+  llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
   
   CookieSize
     = std::max(SizeSize, getContext().getTypeAlignInChars(ElementType));
@@ -968,7 +948,7 @@ llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
 
   ASTContext &Ctx = getContext();
   CharUnits SizeSize = Ctx.getTypeSizeInChars(Ctx.getSizeType());
-  const llvm::IntegerType *SizeTy =
+  llvm::IntegerType *SizeTy =
     cast<llvm::IntegerType>(CGF.ConvertType(Ctx.getSizeType()));
 
   // The cookie is always at the start of the buffer.
@@ -1000,7 +980,7 @@ void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
                                 CharUnits &CookieSize) {
   // Derive a char* in the same address space as the pointer.
   unsigned AS = cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
-  const llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
+  llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
 
   // If we don't need an array cookie, bail out early.
   if (!NeedsArrayCookie(expr, ElementType)) {
@@ -1012,7 +992,7 @@ void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
 
   QualType SizeTy = getContext().getSizeType();
   CharUnits SizeSize = getContext().getTypeSizeInChars(SizeTy);
-  const llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
+  llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
   
   // The cookie size is always 2 * sizeof(size_t).
   CookieSize = 2 * SizeSize;
@@ -1036,10 +1016,9 @@ void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
 static llvm::Constant *getGuardAcquireFn(CodeGenModule &CGM,
                                          llvm::PointerType *GuardPtrTy) {
   // int __cxa_guard_acquire(__guard *guard_object);
-  llvm::Type *ArgTys[] = { GuardPtrTy };
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGM.getTypes().ConvertType(CGM.getContext().IntTy),
-                            ArgTys, /*isVarArg=*/false);
+                            GuardPtrTy, /*isVarArg=*/false);
   
   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_acquire");
 }
@@ -1047,10 +1026,9 @@ static llvm::Constant *getGuardAcquireFn(CodeGenModule &CGM,
 static llvm::Constant *getGuardReleaseFn(CodeGenModule &CGM,
                                          llvm::PointerType *GuardPtrTy) {
   // void __cxa_guard_release(__guard *guard_object);
-  llvm::Type *ArgTys[] = { GuardPtrTy };
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
-                            ArgTys, /*isVarArg=*/false);
+                            GuardPtrTy, /*isVarArg=*/false);
   
   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_release");
 }
@@ -1058,10 +1036,9 @@ static llvm::Constant *getGuardReleaseFn(CodeGenModule &CGM,
 static llvm::Constant *getGuardAbortFn(CodeGenModule &CGM,
                                        llvm::PointerType *GuardPtrTy) {
   // void __cxa_guard_abort(__guard *guard_object);
-  llvm::Type *ArgTys[] = { GuardPtrTy };
-  const llvm::FunctionType *FTy =
+  llvm::FunctionType *FTy =
     llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
-                            ArgTys, /*isVarArg=*/false);
+                            GuardPtrTy, /*isVarArg=*/false);
   
   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort");
 }
@@ -1090,7 +1067,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
   bool threadsafe =
     (getContext().getLangOptions().ThreadsafeStatics && D.isLocalVarDecl());
 
-  const llvm::IntegerType *GuardTy;
+  llvm::IntegerType *GuardTy;
 
   // If we have a global variable with internal linkage and thread-safe statics
   // are disabled, we can just let the guard variable be of type i8.
@@ -1152,21 +1129,28 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
   //     }
   } else {
     // Load the first byte of the guard variable.
-    const llvm::Type *PtrTy = Builder.getInt8PtrTy();
-    llvm::Value *V = 
-      Builder.CreateLoad(Builder.CreateBitCast(GuardVariable, PtrTy), "tmp");
+    llvm::Type *PtrTy = Builder.getInt8PtrTy();
+    llvm::LoadInst *LI = 
+      Builder.CreateLoad(Builder.CreateBitCast(GuardVariable, PtrTy));
+    LI->setAlignment(1);
 
-    IsInitialized = Builder.CreateIsNull(V, "guard.uninitialized");
+    // Itanium ABI:
+    //   An implementation supporting thread-safety on multiprocessor
+    //   systems must also guarantee that references to the initialized
+    //   object do not occur before the load of the initialization flag.
+    //
+    // In LLVM, we do this by marking the load Acquire.
+    if (threadsafe)
+      LI->setAtomic(llvm::Acquire);
+
+    IsInitialized = Builder.CreateIsNull(LI, "guard.uninitialized");
   }
 
   llvm::BasicBlock *InitCheckBlock = CGF.createBasicBlock("init.check");
   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end");
 
-  llvm::BasicBlock *NoCheckBlock = EndBlock;
-  if (threadsafe) NoCheckBlock = CGF.createBasicBlock("init.barrier");
-
   // Check if the first byte of the guard variable is zero.
-  Builder.CreateCondBr(IsInitialized, InitCheckBlock, NoCheckBlock);
+  Builder.CreateCondBr(IsInitialized, InitCheckBlock, EndBlock);
 
   CGF.EmitBlock(InitCheckBlock);
 
@@ -1200,23 +1184,5 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
     Builder.CreateStore(llvm::ConstantInt::get(GuardTy, 1), GuardVariable);
   }
 
-  // Emit an acquire memory barrier if using thread-safe statics:
-  // Itanium ABI:
-  //   An implementation supporting thread-safety on multiprocessor
-  //   systems must also guarantee that references to the initialized
-  //   object do not occur before the load of the initialization flag.
-  if (threadsafe) {
-    Builder.CreateBr(EndBlock);
-    CGF.EmitBlock(NoCheckBlock);
-
-    llvm::Value *_false = Builder.getFalse();
-    llvm::Value *_true = Builder.getTrue();
-
-    Builder.CreateCall5(CGM.getIntrinsic(llvm::Intrinsic::memory_barrier),
-                        /* load-load, load-store */ _true, _true,
-                        /* store-load, store-store */ _false, _false,
-                        /* device or I/O */ _false);
-  }
-
   CGF.EmitBlock(EndBlock);
 }
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 747e5e3222c2..e200e7961701 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -31,7 +31,7 @@ class MicrosoftCXXABI : public CGCXXABI {
   void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                  CXXCtorType Type,
                                  CanQualType &ResTy,
-                                 llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+                                 SmallVectorImpl<CanQualType> &ArgTys) {
     // 'this' is already in place
     // TODO: 'for base' flag
   }  
@@ -39,7 +39,7 @@ class MicrosoftCXXABI : public CGCXXABI {
   void BuildDestructorSignature(const CXXDestructorDecl *Ctor,
                                 CXXDtorType Type,
                                 CanQualType &ResTy,
-                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+                                SmallVectorImpl<CanQualType> &ArgTys) {
     // 'this' is already in place
     // TODO: 'for base' flag
   }
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp
index 4a2c4abbeb03..793ee9192e6b 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp
@@ -27,7 +27,7 @@ using namespace clang;
 
 namespace {
   class CodeGeneratorImpl : public CodeGenerator {
-    Diagnostic &Diags;
+    DiagnosticsEngine &Diags;
     llvm::OwningPtr<const llvm::TargetData> TD;
     ASTContext *Ctx;
     const CodeGenOptions CodeGenOpts;  // Intentionally copied in.
@@ -35,7 +35,7 @@ namespace {
     llvm::OwningPtr<llvm::Module> M;
     llvm::OwningPtr<CodeGen::CodeGenModule> Builder;
   public:
-    CodeGeneratorImpl(Diagnostic &diags, const std::string& ModuleName,
+    CodeGeneratorImpl(DiagnosticsEngine &diags, const std::string& ModuleName,
                       const CodeGenOptions &CGO, llvm::LLVMContext& C)
       : Diags(diags), CodeGenOpts(CGO), M(new llvm::Module(ModuleName, C)) {}
 
@@ -52,9 +52,9 @@ namespace {
     virtual void Initialize(ASTContext &Context) {
       Ctx = &Context;
 
-      M->setTargetTriple(Ctx->Target.getTriple().getTriple());
-      M->setDataLayout(Ctx->Target.getTargetDescription());
-      TD.reset(new llvm::TargetData(Ctx->Target.getTargetDescription()));
+      M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple());
+      M->setDataLayout(Ctx->getTargetInfo().getTargetDescription());
+      TD.reset(new llvm::TargetData(Ctx->getTargetInfo().getTargetDescription()));
       Builder.reset(new CodeGen::CodeGenModule(Context, CodeGenOpts,
                                                *M, *TD, Diags));
     }
@@ -112,7 +112,7 @@ namespace {
   };
 }
 
-CodeGenerator *clang::CreateLLVMCodeGen(Diagnostic &Diags,
+CodeGenerator *clang::CreateLLVMCodeGen(DiagnosticsEngine &Diags,
                                         const std::string& ModuleName,
                                         const CodeGenOptions &CGO,
                                         llvm::LLVMContext& C) {
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp
index df2c1bd98cca..e1dc8f7ffdbd 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp
+++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp
@@ -57,12 +57,12 @@ const llvm::TargetData &ABIInfo::getTargetData() const {
 
 
 void ABIArgInfo::dump() const {
-  llvm::raw_ostream &OS = llvm::errs();
+  raw_ostream &OS = llvm::errs();
   OS << "(ABIArgInfo Kind=";
   switch (TheKind) {
   case Direct:
     OS << "Direct Type=";
-    if (const llvm::Type *Ty = getCoerceToType())
+    if (llvm::Type *Ty = getCoerceToType())
       Ty->print(OS);
     else
       OS << "null";
@@ -87,6 +87,25 @@ void ABIArgInfo::dump() const {
 
 TargetCodeGenInfo::~TargetCodeGenInfo() { delete Info; }
 
+// If someone can figure out a general rule for this, that would be great.
+// It's probably just doomed to be platform-dependent, though.
+unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
+  // Verified for:
+  //   x86-64     FreeBSD, Linux, Darwin
+  //   x86-32     FreeBSD, Linux, Darwin
+  //   PowerPC    Linux, Darwin
+  //   ARM        Darwin (*not* EABI)
+  return 32;
+}
+
+bool TargetCodeGenInfo::isNoProtoCallVariadic(CallingConv CC) const {
+  // The following conventions are known to require this to be false:
+  //   x86_stdcall
+  //   MIPS
+  // For everything else, we just prefer false unless we opt out.
+  return false;
+}
+
 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
 
 /// isEmptyField - Return true iff a the field is "empty", that is it
@@ -348,7 +367,7 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
 
 /// UseX86_MMXType - Return true if this is an MMX type that should use the special
 /// x86_mmx type.
-bool UseX86_MMXType(const llvm::Type *IRType) {
+bool UseX86_MMXType(llvm::Type *IRType) {
   // If the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>, use the
   // special x86_mmx type.
   return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
@@ -357,7 +376,7 @@ bool UseX86_MMXType(const llvm::Type *IRType) {
 }
 
 static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
-                                          llvm::StringRef Constraint,
+                                          StringRef Constraint,
                                           llvm::Type* Ty) {
   if ((Constraint == "y" || Constraint == "&y") && Ty->isVectorTy())
     return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
@@ -428,7 +447,7 @@ class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
                                llvm::Value *Address) const;
 
   llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
-                                  llvm::StringRef Constraint,
+                                  StringRef Constraint,
                                   llvm::Type* Ty) const {
     return X86AdjustInlineAsmType(CGF, Constraint, Ty);
   }
@@ -724,8 +743,8 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
 
 llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                       CodeGenFunction &CGF) const {
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
+  llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
 
   CGBuilderTy &Builder = CGF.Builder;
   llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
@@ -765,7 +784,7 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
   CodeGen::CGBuilderTy &Builder = CGF.Builder;
   llvm::LLVMContext &Context = CGF.getLLVMContext();
 
-  const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+  llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
   llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
 
   // 0-7 are the eight integer registers;  the order is different
@@ -892,7 +911,7 @@ class X86_64ABIInfo : public ABIInfo {
   /// required strict binary compatibility with older versions of GCC
   /// may need to exempt themselves.
   bool honorsRevision0_98() const {
-    return !getContext().Target.getTriple().isOSDarwin();
+    return !getContext().getTargetInfo().getTriple().isOSDarwin();
   }
 
 public:
@@ -932,7 +951,7 @@ class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
     CodeGen::CGBuilderTy &Builder = CGF.Builder;
     llvm::LLVMContext &Context = CGF.getLLVMContext();
 
-    const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+    llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
     llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
 
     // 0-15 are the 16 integer registers.
@@ -943,11 +962,20 @@ class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
   }
 
   llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
-                                  llvm::StringRef Constraint,
+                                  StringRef Constraint,
                                   llvm::Type* Ty) const {
     return X86AdjustInlineAsmType(CGF, Constraint, Ty);
   }
 
+  bool isNoProtoCallVariadic(CallingConv CC) const {
+    // The default CC on x86-64 sets %al to the number of SSA
+    // registers used, and GCC sets this when calling an unprototyped
+    // function, so we override the default behavior.
+    if (CC == CC_Default || CC == CC_C) return true;
+
+    return TargetCodeGenInfo::isNoProtoCallVariadic(CC);
+  }
+
 };
 
 class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
@@ -964,7 +992,7 @@ class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
     CodeGen::CGBuilderTy &Builder = CGF.Builder;
     llvm::LLVMContext &Context = CGF.getLLVMContext();
 
-    const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+    llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
     llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
 
     // 0-15 are the 16 integer registers.
@@ -1309,8 +1337,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
           continue;
 
         uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
-        uint64_t Size =
-          i->getBitWidth()->EvaluateAsInt(getContext()).getZExtValue();
+        uint64_t Size = i->getBitWidthValue(getContext());
 
         uint64_t EB_Lo = Offset / 64;
         uint64_t EB_Hi = (Offset + Size - 1) / 64;
@@ -1489,14 +1516,14 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
 /// float member at the specified offset.  For example, {int,{float}} has a
 /// float at offset 4.  It is conservatively correct for this routine to return
 /// false.
-static bool ContainsFloatAtOffset(const llvm::Type *IRType, unsigned IROffset,
+static bool ContainsFloatAtOffset(llvm::Type *IRType, unsigned IROffset,
                                   const llvm::TargetData &TD) {
   // Base case if we find a float.
   if (IROffset == 0 && IRType->isFloatTy())
     return true;
 
   // If this is a struct, recurse into the field at the specified offset.
-  if (const llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
+  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
     const llvm::StructLayout *SL = TD.getStructLayout(STy);
     unsigned Elt = SL->getElementContainingOffset(IROffset);
     IROffset -= SL->getElementOffset(Elt);
@@ -1504,8 +1531,8 @@ static bool ContainsFloatAtOffset(const llvm::Type *IRType, unsigned IROffset,
   }
 
   // If this is an array, recurse into the field at the specified offset.
-  if (const llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
-    const llvm::Type *EltTy = ATy->getElementType();
+  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
+    llvm::Type *EltTy = ATy->getElementType();
     unsigned EltSize = TD.getTypeAllocSize(EltTy);
     IROffset -= IROffset/EltSize*EltSize;
     return ContainsFloatAtOffset(EltTy, IROffset, TD);
@@ -1578,7 +1605,7 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
     }
   }
 
-  if (const llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
+  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
     // If this is a struct, recurse into the field at the specified offset.
     const llvm::StructLayout *SL = getTargetData().getStructLayout(STy);
     if (IROffset < SL->getSizeInBytes()) {
@@ -1590,7 +1617,7 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
     }
   }
 
-  if (const llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
+  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
     llvm::Type *EltTy = ATy->getElementType();
     unsigned EltSize = getTargetData().getTypeAllocSize(EltTy);
     unsigned EltOffset = IROffset/EltSize*EltSize;
@@ -1678,7 +1705,7 @@ classifyReturnType(QualType RetTy) const {
 
   case SSEUp:
   case X87Up:
-    assert(0 && "Invalid classification for lo word.");
+    llvm_unreachable("Invalid classification for lo word.");
 
     // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
     // hidden argument.
@@ -1732,7 +1759,7 @@ classifyReturnType(QualType RetTy) const {
     // never occur as a hi class.
   case Memory:
   case X87:
-    assert(0 && "Invalid classification for hi word.");
+    llvm_unreachable("Invalid classification for hi word.");
 
   case ComplexX87: // Previously handled.
   case NoClass:
@@ -1820,7 +1847,7 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned &neededInt,
 
   case SSEUp:
   case X87Up:
-    assert(0 && "Invalid classification for lo word.");
+    llvm_unreachable("Invalid classification for lo word.");
 
     // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
     // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
@@ -1864,8 +1891,7 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned &neededInt,
   case Memory:
   case X87:
   case ComplexX87:
-    assert(0 && "Invalid classification for hi word.");
-    break;
+    llvm_unreachable("Invalid classification for hi word.");
 
   case NoClass: break;
 
@@ -1970,7 +1996,7 @@ static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr,
   }
 
   // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
-  const llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
+  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
   llvm::Value *Res =
     CGF.Builder.CreateBitCast(overflow_arg_area,
                               llvm::PointerType::getUnqual(LTy));
@@ -2061,22 +2087,22 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   // collect arguments from different places; often what should result in a
   // simple assembling of a structure from scattered addresses has many more
   // loads than necessary. Can we clean this up?
-  const llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
+  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
   llvm::Value *RegAddr =
     CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(VAListAddr, 3),
                            "reg_save_area");
   if (neededInt && neededSSE) {
     // FIXME: Cleanup.
     assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
-    const llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
+    llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
     llvm::Value *Tmp = CGF.CreateTempAlloca(ST);
     assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
-    const llvm::Type *TyLo = ST->getElementType(0);
-    const llvm::Type *TyHi = ST->getElementType(1);
+    llvm::Type *TyLo = ST->getElementType(0);
+    llvm::Type *TyHi = ST->getElementType(1);
     assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
            "Unexpected ABI info for mixed regs");
-    const llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
-    const llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
+    llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
+    llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
     llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset);
     llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset);
     llvm::Value *RegLoAddr = TyLo->isFloatingPointTy() ? FPAddr : GPAddr;
@@ -2104,9 +2130,9 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
     llvm::Value *RegAddrLo = CGF.Builder.CreateGEP(RegAddr, fp_offset);
     llvm::Value *RegAddrHi = CGF.Builder.CreateConstGEP1_32(RegAddrLo, 16);
     llvm::Type *DoubleTy = llvm::Type::getDoubleTy(VMContext);
-    const llvm::Type *DblPtrTy =
+    llvm::Type *DblPtrTy =
       llvm::PointerType::getUnqual(DoubleTy);
-    const llvm::StructType *ST = llvm::StructType::get(DoubleTy,
+    llvm::StructType *ST = llvm::StructType::get(DoubleTy,
                                                        DoubleTy, NULL);
     llvm::Value *V, *Tmp = CGF.CreateTempAlloca(ST);
     V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo,
@@ -2166,7 +2192,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty) const {
 
     // FIXME: mingw-w64-gcc emits 128-bit struct as i128
     if (Size == 128 &&
-        getContext().Target.getTriple().getOS() == llvm::Triple::MinGW32)
+        getContext().getTargetInfo().getTriple().getOS() == llvm::Triple::MinGW32)
       return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
                                                           Size));
 
@@ -2198,8 +2224,8 @@ void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
 llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                       CodeGenFunction &CGF) const {
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
+  llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
 
   CGBuilderTy &Builder = CGF.Builder;
   llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
@@ -2246,7 +2272,7 @@ PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
   CodeGen::CGBuilderTy &Builder = CGF.Builder;
   llvm::LLVMContext &Context = CGF.getLLVMContext();
 
-  const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+  llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
   llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
   llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
   llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
@@ -2300,6 +2326,11 @@ class ARMABIInfo : public ABIInfo {
 public:
   ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind) {}
 
+  bool isEABI() const {
+    StringRef Env = getContext().getTargetInfo().getTriple().getEnvironmentName();
+    return (Env == "gnueabi" || Env == "eabi");
+  }
+
 private:
   ABIKind getABIKind() const { return Kind; }
 
@@ -2317,11 +2348,15 @@ class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
   ARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K)
     :TargetCodeGenInfo(new ARMABIInfo(CGT, K)) {}
 
+  const ARMABIInfo &getABIInfo() const {
+    return static_cast<const ARMABIInfo&>(TargetCodeGenInfo::getABIInfo());
+  }
+
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const {
     return 13;
   }
 
-  llvm::StringRef getARCRetainAutoreleasedReturnValueMarker() const {
+  StringRef getARCRetainAutoreleasedReturnValueMarker() const {
     return "mov\tr7, r7\t\t@ marker for objc_retainAutoreleaseReturnValue";
   }
 
@@ -2330,7 +2365,7 @@ class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
     CodeGen::CGBuilderTy &Builder = CGF.Builder;
     llvm::LLVMContext &Context = CGF.getLLVMContext();
 
-    const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+    llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
     llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
 
     // 0-15 are the 16 integer registers.
@@ -2338,6 +2373,11 @@ class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
 
     return false;
   }
+
+  unsigned getSizeOfUnwindException() const {
+    if (getABIInfo().isEABI()) return 88;
+    return TargetCodeGenInfo::getSizeOfUnwindException();
+  }
 };
 
 }
@@ -2354,8 +2394,7 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
   // Calling convention as default by an ABI.
   llvm::CallingConv::ID DefaultCC;
-  llvm::StringRef Env = getContext().Target.getTriple().getEnvironmentName();
-  if (Env == "gnueabi" || Env == "eabi")
+  if (isEABI())
     DefaultCC = llvm::CallingConv::ARM_AAPCS;
   else
     DefaultCC = llvm::CallingConv::ARM_APCS;
@@ -2379,6 +2418,73 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
   }
 }
 
+/// isHomogeneousAggregate - Return true if a type is an AAPCS-VFP homogeneous
+/// aggregate.  If HAMembers is non-null, the number of base elements
+/// contained in the type is returned through it; this is used for the
+/// recursive calls that check aggregate component types.
+static bool isHomogeneousAggregate(QualType Ty, const Type *&Base,
+                                   ASTContext &Context,
+                                   uint64_t *HAMembers = 0) {
+  uint64_t Members;
+  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
+    if (!isHomogeneousAggregate(AT->getElementType(), Base, Context, &Members))
+      return false;
+    Members *= AT->getSize().getZExtValue();
+  } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
+    const RecordDecl *RD = RT->getDecl();
+    if (RD->isUnion() || RD->hasFlexibleArrayMember())
+      return false;
+    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+      if (!CXXRD->isAggregate())
+        return false;
+    }
+    Members = 0;
+    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+         i != e; ++i) {
+      const FieldDecl *FD = *i;
+      uint64_t FldMembers;
+      if (!isHomogeneousAggregate(FD->getType(), Base, Context, &FldMembers))
+        return false;
+      Members += FldMembers;
+    }
+  } else {
+    Members = 1;
+    if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
+      Members = 2;
+      Ty = CT->getElementType();
+    }
+
+    // Homogeneous aggregates for AAPCS-VFP must have base types of float,
+    // double, or 64-bit or 128-bit vectors.
+    if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
+      if (BT->getKind() != BuiltinType::Float && 
+          BT->getKind() != BuiltinType::Double)
+        return false;
+    } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
+      unsigned VecSize = Context.getTypeSize(VT);
+      if (VecSize != 64 && VecSize != 128)
+        return false;
+    } else {
+      return false;
+    }
+
+    // The base type must be the same for all members.  Vector types of the
+    // same total size are treated as being equivalent here.
+    const Type *TyPtr = Ty.getTypePtr();
+    if (!Base)
+      Base = TyPtr;
+    if (Base != TyPtr &&
+        (!Base->isVectorType() || !TyPtr->isVectorType() ||
+         Context.getTypeSize(Base) != Context.getTypeSize(TyPtr)))
+      return false;
+  }
+
+  // Homogeneous Aggregates can have at most 4 members of the base type.
+  if (HAMembers)
+    *HAMembers = Members;
+  return (Members <= 4);
+}
+
 ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty) const {
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
@@ -2398,23 +2504,26 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty) const {
   if (isRecordWithNonTrivialDestructorOrCopyConstructor(Ty))
     return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
 
+  if (getABIKind() == ARMABIInfo::AAPCS_VFP) {
+    // Homogeneous Aggregates need to be expanded.
+    const Type *Base = 0;
+    if (isHomogeneousAggregate(Ty, Base, getContext()))
+      return ABIArgInfo::getExpand();
+  }
+
   // Otherwise, pass by coercing to a structure of the appropriate size.
   //
+  // FIXME: This is kind of nasty... but there isn't much choice because the ARM
+  // backend doesn't support byval.
   // FIXME: This doesn't handle alignment > 64 bits.
-  const llvm::Type* ElemTy;
+  llvm::Type* ElemTy;
   unsigned SizeRegs;
-  if (getContext().getTypeSizeInChars(Ty) <= CharUnits::fromQuantity(64)) {
-    ElemTy = llvm::Type::getInt32Ty(getVMContext());
-    SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32;
-  } else if (getABIKind() == ARMABIInfo::APCS) {
-    // Initial ARM ByVal support is APCS-only.
-    return ABIArgInfo::getIndirect(0, /*ByVal=*/true);
-  } else {
-    // FIXME: This is kind of nasty... but there isn't much choice
-    // because most of the ARM calling conventions don't yet support
-    // byval.
+  if (getContext().getTypeAlign(Ty) > 32) {
     ElemTy = llvm::Type::getInt64Ty(getVMContext());
     SizeRegs = (getContext().getTypeSize(Ty) + 63) / 64;
+  } else {
+    ElemTy = llvm::Type::getInt32Ty(getVMContext());
+    SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32;
   }
 
   llvm::Type *STy =
@@ -2579,14 +2688,23 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy) const {
 
 llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                    CodeGenFunction &CGF) const {
-  // FIXME: Need to handle alignment
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
+  llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
 
   CGBuilderTy &Builder = CGF.Builder;
   llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
                                                        "ap");
   llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
+  // Handle address alignment for type alignment > 32 bits
+  uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
+  if (TyAlign > 4) {
+    assert((TyAlign & (TyAlign - 1)) == 0 &&
+           "Alignment is not power of 2!");
+    llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
+    AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1));
+    AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
+    Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
+  }
   llvm::Type *PTy =
     llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
   llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
@@ -2623,6 +2741,9 @@ class PTXTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   PTXTargetCodeGenInfo(CodeGenTypes &CGT)
     : TargetCodeGenInfo(new PTXABIInfo(CGT)) {}
+    
+  virtual void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
+                                   CodeGen::CodeGenModule &M) const;
 };
 
 ABIArgInfo PTXABIInfo::classifyReturnType(QualType RetTy) const {
@@ -2652,13 +2773,21 @@ void PTXABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
   // Calling convention as default by an ABI.
   llvm::CallingConv::ID DefaultCC;
-  llvm::StringRef Env = getContext().Target.getTriple().getEnvironmentName();
-  if (Env == "device")
+  const LangOptions &LangOpts = getContext().getLangOptions();
+  if (LangOpts.OpenCL || LangOpts.CUDA) {
+    // If we are in OpenCL or CUDA mode, then default to device functions
     DefaultCC = llvm::CallingConv::PTX_Device;
-  else
-    DefaultCC = llvm::CallingConv::PTX_Kernel;
-
+  } else {
+    // If we are in standard C/C++ mode, use the triple to decide on the default
+    StringRef Env = 
+      getContext().getTargetInfo().getTriple().getEnvironmentName();
+    if (Env == "device")
+      DefaultCC = llvm::CallingConv::PTX_Device;
+    else
+      DefaultCC = llvm::CallingConv::PTX_Kernel;
+  }
   FI.setEffectiveCallingConvention(DefaultCC);
+   
 }
 
 llvm::Value *PTXABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -2667,6 +2796,36 @@ llvm::Value *PTXABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return 0;
 }
 
+void PTXTargetCodeGenInfo::SetTargetAttributes(const Decl *D,
+                                               llvm::GlobalValue *GV,
+                                               CodeGen::CodeGenModule &M) const{
+  const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+  if (!FD) return;
+
+  llvm::Function *F = cast<llvm::Function>(GV);
+
+  // Perform special handling in OpenCL mode
+  if (M.getLangOptions().OpenCL) {
+    // Use OpenCL function attributes to set proper calling conventions
+    // By default, all functions are device functions
+    if (FD->hasAttr<OpenCLKernelAttr>()) {
+      // OpenCL __kernel functions get a kernel calling convention
+      F->setCallingConv(llvm::CallingConv::PTX_Kernel);
+      // And kernel functions are not subject to inlining
+      F->addFnAttr(llvm::Attribute::NoInline);
+    }
+  }
+
+  // Perform special handling in CUDA mode.
+  if (M.getLangOptions().CUDA) {
+    // CUDA __global__ functions get a kernel calling convention.  Since
+    // __global__ functions cannot be called from the device, we do not
+    // need to set the noinline attribute.
+    if (FD->getAttr<CUDAGlobalAttr>())
+      F->setCallingConv(llvm::CallingConv::PTX_Kernel);
+  }
+}
+
 }
 
 //===----------------------------------------------------------------------===//
@@ -2891,7 +3050,7 @@ void MSP430TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
       // Step 3: Emit ISR vector alias.
       unsigned Num = attr->getNumber() + 0xffe0;
       new llvm::GlobalAlias(GV->getType(), llvm::Function::ExternalLinkage,
-                            "vector_" + llvm::Twine::utohexstr(Num),
+                            "vector_" + Twine::utohexstr(Num),
                             GV, &M.getModule());
     }
   }
@@ -2904,6 +3063,7 @@ void MSP430TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
 
 namespace {
 class MipsABIInfo : public ABIInfo {
+  static const unsigned MinABIStackAlignInBytes = 4;
 public:
   MipsABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
 
@@ -2914,10 +3074,13 @@ class MipsABIInfo : public ABIInfo {
                                  CodeGenFunction &CGF) const;
 };
 
+const unsigned MipsABIInfo::MinABIStackAlignInBytes;
+
 class MIPSTargetCodeGenInfo : public TargetCodeGenInfo {
+  unsigned SizeOfUnwindException;
 public:
-  MIPSTargetCodeGenInfo(CodeGenTypes &CGT)
-    : TargetCodeGenInfo(new MipsABIInfo(CGT)) {}
+  MIPSTargetCodeGenInfo(CodeGenTypes &CGT, unsigned SZ)
+    : TargetCodeGenInfo(new MipsABIInfo(CGT)), SizeOfUnwindException(SZ) {}
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const {
     return 29;
@@ -2925,6 +3088,10 @@ class MIPSTargetCodeGenInfo : public TargetCodeGenInfo {
 
   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
                                llvm::Value *Address) const;
+
+  unsigned getSizeOfUnwindException() const {
+    return SizeOfUnwindException;
+  }
 };
 }
 
@@ -2934,6 +3101,11 @@ ABIArgInfo MipsABIInfo::classifyArgumentType(QualType Ty) const {
     if (getContext().getTypeSize(Ty) == 0)
       return ABIArgInfo::getIgnore();
 
+    // Records with non trivial destructors/constructors should not be passed
+    // by value.
+    if (isRecordWithNonTrivialDestructorOrCopyConstructor(Ty))
+      return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+
     return ABIArgInfo::getIndirect(0);
   }
 
@@ -2973,7 +3145,37 @@ void MipsABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
 llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                     CodeGenFunction &CGF) const {
-  return 0;
+  llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
+ 
+  CGBuilderTy &Builder = CGF.Builder;
+  llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
+  llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
+  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
+  llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
+  llvm::Value *AddrTyped;
+
+  if (TypeAlign > MinABIStackAlignInBytes) {
+    llvm::Value *AddrAsInt32 = CGF.Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
+    llvm::Value *Inc = llvm::ConstantInt::get(CGF.Int32Ty, TypeAlign - 1);
+    llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -TypeAlign);
+    llvm::Value *Add = CGF.Builder.CreateAdd(AddrAsInt32, Inc);
+    llvm::Value *And = CGF.Builder.CreateAnd(Add, Mask);
+    AddrTyped = CGF.Builder.CreateIntToPtr(And, PTy);
+  }
+  else
+    AddrTyped = Builder.CreateBitCast(Addr, PTy);  
+
+  llvm::Value *AlignedAddr = Builder.CreateBitCast(AddrTyped, BP);
+  TypeAlign = std::max(TypeAlign, MinABIStackAlignInBytes);
+  uint64_t Offset =
+    llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, TypeAlign);
+  llvm::Value *NextAddr =
+    Builder.CreateGEP(AlignedAddr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
+                      "ap.next");
+  Builder.CreateStore(NextAddr, VAListAddrAsBPP);
+  
+  return AddrTyped;
 }
 
 bool
@@ -2987,7 +3189,7 @@ MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
 
   // Everything on MIPS is 4 bytes.  Double-precision FP registers
   // are aliased to pairs of single-precision FP registers.
-  const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+  llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
   llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
 
   // 0-31 are the general purpose registers, $0 - $31.
@@ -3009,29 +3211,98 @@ MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
   return false;
 }
 
+//===----------------------------------------------------------------------===//
+// TCE ABI Implementation (see http://tce.cs.tut.fi). Uses mostly the defaults.
+// Currently subclassed only to implement custom OpenCL C function attribute 
+// handling.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class TCETargetCodeGenInfo : public DefaultTargetCodeGenInfo {
+public:
+  TCETargetCodeGenInfo(CodeGenTypes &CGT)
+    : DefaultTargetCodeGenInfo(CGT) {}
+
+  virtual void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
+                                   CodeGen::CodeGenModule &M) const;
+};
+
+void TCETargetCodeGenInfo::SetTargetAttributes(const Decl *D,
+                                               llvm::GlobalValue *GV,
+                                               CodeGen::CodeGenModule &M) const {
+  const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+  if (!FD) return;
+
+  llvm::Function *F = cast<llvm::Function>(GV);
+  
+  if (M.getLangOptions().OpenCL) {
+    if (FD->hasAttr<OpenCLKernelAttr>()) {
+      // OpenCL C Kernel functions are not subject to inlining
+      F->addFnAttr(llvm::Attribute::NoInline);
+          
+      if (FD->hasAttr<ReqdWorkGroupSizeAttr>()) {
+
+        // Convert the reqd_work_group_size() attributes to metadata.
+        llvm::LLVMContext &Context = F->getContext();
+        llvm::NamedMDNode *OpenCLMetadata = 
+            M.getModule().getOrInsertNamedMetadata("opencl.kernel_wg_size_info");
+
+        SmallVector<llvm::Value*, 5> Operands;
+        Operands.push_back(F);
+
+        Operands.push_back(llvm::Constant::getIntegerValue(
+                             llvm::Type::getInt32Ty(Context), 
+                             llvm::APInt(
+                               32, 
+                               FD->getAttr<ReqdWorkGroupSizeAttr>()->getXDim())));
+        Operands.push_back(llvm::Constant::getIntegerValue(
+                             llvm::Type::getInt32Ty(Context), 
+                             llvm::APInt(
+                               32,
+                               FD->getAttr<ReqdWorkGroupSizeAttr>()->getYDim())));
+        Operands.push_back(llvm::Constant::getIntegerValue(
+                             llvm::Type::getInt32Ty(Context), 
+                             llvm::APInt(
+                               32, 
+                               FD->getAttr<ReqdWorkGroupSizeAttr>()->getZDim())));
+
+        // Add a boolean constant operand for "required" (true) or "hint" (false)
+        // for implementing the work_group_size_hint attr later. Currently 
+        // always true as the hint is not yet implemented.
+        Operands.push_back(llvm::ConstantInt::getTrue(llvm::Type::getInt1Ty(Context)));
+
+        OpenCLMetadata->addOperand(llvm::MDNode::get(Context, Operands));
+      }
+    }
+  }
+}
+
+}
 
 const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   if (TheTargetCodeGenInfo)
     return *TheTargetCodeGenInfo;
 
-  // For now we just cache the TargetCodeGenInfo in CodeGenModule and don't
-  // free it.
-
-  const llvm::Triple &Triple = getContext().Target.getTriple();
+  const llvm::Triple &Triple = getContext().getTargetInfo().getTriple();
   switch (Triple.getArch()) {
   default:
     return *(TheTargetCodeGenInfo = new DefaultTargetCodeGenInfo(Types));
 
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
-    return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types));
+    return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types, 24));
+
+  case llvm::Triple::mips64:
+  case llvm::Triple::mips64el:
+    return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types, 32));
 
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
     {
       ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS;
 
-      if (strcmp(getContext().Target.getABI(), "apcs-gnu") == 0)
+      if (strcmp(getContext().getTargetInfo().getABI(), "apcs-gnu") == 0)
         Kind = ARMABIInfo::APCS;
       else if (CodeGenOpts.FloatABI == "hard")
         Kind = ARMABIInfo::AAPCS_VFP;
@@ -3055,8 +3326,11 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   case llvm::Triple::msp430:
     return *(TheTargetCodeGenInfo = new MSP430TargetCodeGenInfo(Types));
 
+  case llvm::Triple::tce:
+    return *(TheTargetCodeGenInfo = new TCETargetCodeGenInfo(Types));
+
   case llvm::Triple::x86: {
-    bool DisableMMX = strcmp(getContext().Target.getABI(), "no-mmx") == 0;
+    bool DisableMMX = strcmp(getContext().getTargetInfo().getABI(), "no-mmx") == 0;
 
     if (Triple.isOSDarwin())
       return *(TheTargetCodeGenInfo =
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h
index d5e8884cdffe..8f90c7bdd92d 100644
--- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h
+++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h
@@ -15,6 +15,8 @@
 #ifndef CLANG_CODEGEN_TARGETINFO_H
 #define CLANG_CODEGEN_TARGETINFO_H
 
+#include "clang/Basic/LLVM.h"
+#include "clang/AST/Type.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace llvm {
@@ -58,7 +60,7 @@ namespace clang {
     ///     uint64 private_1;
     ///     uint64 private_2;
     ///   };
-    unsigned getSizeOfUnwindException() const { return 32; }
+    virtual unsigned getSizeOfUnwindException() const;
 
     /// Controls whether __builtin_extend_pointer should sign-extend
     /// pointers to uint64_t or zero-extend them (the default).  Has
@@ -107,7 +109,7 @@ namespace clang {
     }
 
     virtual llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
-                                            llvm::StringRef Constraint, 
+                                            StringRef Constraint, 
                                             llvm::Type* Ty) const {
       return Ty;
     }
@@ -122,9 +124,43 @@ namespace clang {
     /// a particular instruction sequence.  This functions returns
     /// that instruction sequence in inline assembly, which will be
     /// empty if none is required.
-    virtual llvm::StringRef getARCRetainAutoreleasedReturnValueMarker() const {
+    virtual StringRef getARCRetainAutoreleasedReturnValueMarker() const {
       return "";
     }
+
+    /// Determine whether a call to an unprototyped functions under
+    /// the given calling convention should use the variadic
+    /// convention or the non-variadic convention.
+    ///
+    /// There's a good reason to make a platform's variadic calling
+    /// convention be different from its non-variadic calling
+    /// convention: the non-variadic arguments can be passed in
+    /// registers (better for performance), and the variadic arguments
+    /// can be passed on the stack (also better for performance).  If
+    /// this is done, however, unprototyped functions *must* use the
+    /// non-variadic convention, because C99 states that a call
+    /// through an unprototyped function type must succeed if the
+    /// function was defined with a non-variadic prototype with
+    /// compatible parameters.  Therefore, splitting the conventions
+    /// makes it impossible to call a variadic function through an
+    /// unprototyped type.  Since function prototypes came out in the
+    /// late 1970s, this is probably an acceptable trade-off.
+    /// Nonetheless, not all platforms are willing to make it, and in
+    /// particularly x86-64 bends over backwards to make the
+    /// conventions compatible.
+    ///
+    /// The default is false.  This is correct whenever:
+    ///   - the conventions are exactly the same, because it does not
+    ///     matter and the resulting IR will be somewhat prettier in
+    ///     certain cases; or
+    ///   - the conventions are substantively different in how they pass
+    ///     arguments, because in this case using the variadic convention
+    ///     will lead to C99 violations.
+    /// It is not necessarily correct when arguments are passed in the
+    /// same way and some out-of-band information is passed for the
+    /// benefit of variadic callees, as is the case for x86-64.
+    /// In this case the ABI should be consulted.
+    virtual bool isNoProtoCallVariadic(CallingConv CC) const;
   };
 }
 
diff --git a/contrib/llvm/tools/clang/lib/Driver/Action.cpp b/contrib/llvm/tools/clang/lib/Driver/Action.cpp
index 549ce0a7f41b..52c0dbb5f173 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Action.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Action.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/Action.h"
+#include "llvm/Support/ErrorHandling.h"
 
 #include <cassert>
 using namespace clang::driver;
@@ -31,10 +32,10 @@ const char *Action::getClassName(ActionClass AC) {
   case LinkJobClass: return "linker";
   case LipoJobClass: return "lipo";
   case DsymutilJobClass: return "dsymutil";
+  case VerifyJobClass: return "verify";
   }
 
-  assert(0 && "invalid class");
-  return 0;
+  llvm_unreachable("invalid class");
 }
 
 InputAction::InputAction(const Arg &_Input, types::ID _Type)
@@ -84,3 +85,7 @@ LipoJobAction::LipoJobAction(ActionList &Inputs, types::ID Type)
 DsymutilJobAction::DsymutilJobAction(ActionList &Inputs, types::ID Type)
   : JobAction(DsymutilJobClass, Inputs, Type) {
 }
+
+VerifyJobAction::VerifyJobAction(ActionList &Inputs, types::ID Type)
+  : JobAction(VerifyJobClass, Inputs, Type) {
+}
diff --git a/contrib/llvm/tools/clang/lib/Driver/ArgList.cpp b/contrib/llvm/tools/clang/lib/Driver/ArgList.cpp
index b8af9cc47e09..ca9b944950e1 100644
--- a/contrib/llvm/tools/clang/lib/Driver/ArgList.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/ArgList.cpp
@@ -46,6 +46,17 @@ void ArgList::append(Arg *A) {
   Args.push_back(A);
 }
 
+void ArgList::eraseArg(OptSpecifier Id) {
+  for (iterator it = begin(), ie = end(); it != ie; ) {
+    if ((*it)->getOption().matches(Id)) {
+      it = Args.erase(it);
+      ie = end();
+    } else {
+      ++it;
+    }
+  }
+}
+
 Arg *ArgList::getLastArgNoClaim(OptSpecifier Id) const {
   // FIXME: Make search efficient?
   for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it)
@@ -117,19 +128,19 @@ bool ArgList::hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default) const {
   return Default;
 }
 
-llvm::StringRef ArgList::getLastArgValue(OptSpecifier Id,
-                                         llvm::StringRef Default) const {
+StringRef ArgList::getLastArgValue(OptSpecifier Id,
+                                         StringRef Default) const {
   if (Arg *A = getLastArg(Id))
     return A->getValue(*this);
   return Default;
 }
 
 int ArgList::getLastArgIntValue(OptSpecifier Id, int Default,
-                                clang::Diagnostic &Diags) const {
+                                clang::DiagnosticsEngine &Diags) const {
   int Res = Default;
 
   if (Arg *A = getLastArg(Id)) {
-    if (llvm::StringRef(A->getValue(*this)).getAsInteger(10, Res))
+    if (StringRef(A->getValue(*this)).getAsInteger(10, Res))
       Diags.Report(diag::err_drv_invalid_int_value)
         << A->getAsString(*this) << A->getValue(*this);
   }
@@ -138,7 +149,7 @@ int ArgList::getLastArgIntValue(OptSpecifier Id, int Default,
 }
 
 std::vector<std::string> ArgList::getAllArgValues(OptSpecifier Id) const {
-  llvm::SmallVector<const char *, 16> Values;
+  SmallVector<const char *, 16> Values;
   AddAllArgValues(Values, Id);
   return std::vector<std::string>(Values.begin(), Values.end());
 }
@@ -177,7 +188,7 @@ void ArgList::AddAllArgsTranslated(ArgStringList &Output, OptSpecifier Id0,
     (*it)->claim();
 
     if (Joined) {
-      Output.push_back(MakeArgString(llvm::StringRef(Translation) +
+      Output.push_back(MakeArgString(StringRef(Translation) +
                                      (*it)->getValue(*this, 0)));
     } else {
       Output.push_back(Translation);
@@ -192,16 +203,22 @@ void ArgList::ClaimAllArgs(OptSpecifier Id0) const {
     (*it)->claim();
 }
 
-const char *ArgList::MakeArgString(const llvm::Twine &T) const {
+void ArgList::ClaimAllArgs() const {
+  for (const_iterator it = begin(), ie = end(); it != ie; ++it)
+    if (!(*it)->isClaimed())
+      (*it)->claim();
+}
+
+const char *ArgList::MakeArgString(const Twine &T) const {
   llvm::SmallString<256> Str;
   T.toVector(Str);
   return MakeArgString(Str.str());
 }
 
 const char *ArgList::GetOrMakeJoinedArgString(unsigned Index,
-                                              llvm::StringRef LHS,
-                                              llvm::StringRef RHS) const {
-  llvm::StringRef Cur = getArgString(Index);
+                                              StringRef LHS,
+                                              StringRef RHS) const {
+  StringRef Cur = getArgString(Index);
   if (Cur.size() == LHS.size() + RHS.size() &&
       Cur.startswith(LHS) && Cur.endswith(RHS))
     return Cur.data();
@@ -223,7 +240,7 @@ InputArgList::~InputArgList() {
     delete *it;
 }
 
-unsigned InputArgList::MakeIndex(llvm::StringRef String0) const {
+unsigned InputArgList::MakeIndex(StringRef String0) const {
   unsigned Index = ArgStrings.size();
 
   // Tuck away so we have a reliable const char *.
@@ -233,8 +250,8 @@ unsigned InputArgList::MakeIndex(llvm::StringRef String0) const {
   return Index;
 }
 
-unsigned InputArgList::MakeIndex(llvm::StringRef String0,
-                                 llvm::StringRef String1) const {
+unsigned InputArgList::MakeIndex(StringRef String0,
+                                 StringRef String1) const {
   unsigned Index0 = MakeIndex(String0);
   unsigned Index1 = MakeIndex(String1);
   assert(Index0 + 1 == Index1 && "Unexpected non-consecutive indices!");
@@ -242,7 +259,7 @@ unsigned InputArgList::MakeIndex(llvm::StringRef String0,
   return Index0;
 }
 
-const char *InputArgList::MakeArgString(llvm::StringRef Str) const {
+const char *InputArgList::MakeArgString(StringRef Str) const {
   return getArgString(MakeIndex(Str));
 }
 
@@ -259,7 +276,7 @@ DerivedArgList::~DerivedArgList() {
     delete *it;
 }
 
-const char *DerivedArgList::MakeArgString(llvm::StringRef Str) const {
+const char *DerivedArgList::MakeArgString(StringRef Str) const {
   return BaseArgs.MakeArgString(Str);
 }
 
@@ -270,7 +287,7 @@ Arg *DerivedArgList::MakeFlagArg(const Arg *BaseArg, const Option *Opt) const {
 }
 
 Arg *DerivedArgList::MakePositionalArg(const Arg *BaseArg, const Option *Opt,
-                                       llvm::StringRef Value) const {
+                                       StringRef Value) const {
   unsigned Index = BaseArgs.MakeIndex(Value);
   Arg *A = new Arg(Opt, Index, BaseArgs.getArgString(Index), BaseArg);
   SynthesizedArgs.push_back(A);
@@ -278,7 +295,7 @@ Arg *DerivedArgList::MakePositionalArg(const Arg *BaseArg, const Option *Opt,
 }
 
 Arg *DerivedArgList::MakeSeparateArg(const Arg *BaseArg, const Option *Opt,
-                                     llvm::StringRef Value) const {
+                                     StringRef Value) const {
   unsigned Index = BaseArgs.MakeIndex(Opt->getName(), Value);
   Arg *A = new Arg(Opt, Index, BaseArgs.getArgString(Index + 1), BaseArg);
   SynthesizedArgs.push_back(A);
@@ -286,7 +303,7 @@ Arg *DerivedArgList::MakeSeparateArg(const Arg *BaseArg, const Option *Opt,
 }
 
 Arg *DerivedArgList::MakeJoinedArg(const Arg *BaseArg, const Option *Opt,
-                                   llvm::StringRef Value) const {
+                                   StringRef Value) const {
   unsigned Index = BaseArgs.MakeIndex(Opt->getName().str() + Value.str());
   Arg *A = new Arg(Opt, Index,
                    BaseArgs.getArgString(Index) + Opt->getName().size(),
diff --git a/contrib/llvm/tools/clang/lib/Driver/Compilation.cpp b/contrib/llvm/tools/clang/lib/Driver/Compilation.cpp
index 2657faa0d3a7..d02da9588a63 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Compilation.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Compilation.cpp
@@ -16,16 +16,19 @@
 #include "clang/Driver/Options.h"
 #include "clang/Driver/ToolChain.h"
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Program.h"
 #include <sys/stat.h>
 #include <errno.h>
+
 using namespace clang::driver;
+using namespace clang;
 
 Compilation::Compilation(const Driver &D, const ToolChain &_DefaultToolChain,
                          InputArgList *_Args, DerivedArgList *_TranslatedArgs)
   : TheDriver(D), DefaultToolChain(_DefaultToolChain), Args(_Args),
-    TranslatedArgs(_TranslatedArgs) {
+    TranslatedArgs(_TranslatedArgs), Redirects(0) {
 }
 
 Compilation::~Compilation() {
@@ -43,6 +46,13 @@ Compilation::~Compilation() {
   for (ActionList::iterator it = Actions.begin(), ie = Actions.end();
        it != ie; ++it)
     delete *it;
+
+  // Free redirections of stdout/stderr.
+  if (Redirects) {
+    delete Redirects[1];
+    delete Redirects[2];
+    delete [] Redirects;
+  }
 }
 
 const DerivedArgList &Compilation::getArgsForToolChain(const ToolChain *TC,
@@ -60,14 +70,14 @@ const DerivedArgList &Compilation::getArgsForToolChain(const ToolChain *TC,
   return *Entry;
 }
 
-void Compilation::PrintJob(llvm::raw_ostream &OS, const Job &J,
+void Compilation::PrintJob(raw_ostream &OS, const Job &J,
                            const char *Terminator, bool Quote) const {
   if (const Command *C = dyn_cast<Command>(&J)) {
     OS << " \"" << C->getExecutable() << '"';
     for (ArgStringList::const_iterator it = C->getArguments().begin(),
            ie = C->getArguments().end(); it != ie; ++it) {
       OS << ' ';
-      if (!Quote) {
+      if (!Quote && !std::strpbrk(*it, " \"\\$")) {
         OS << *it;
         continue;
       }
@@ -135,9 +145,9 @@ int Compilation::ExecuteCommand(const Command &C,
   std::copy(C.getArguments().begin(), C.getArguments().end(), Argv+1);
   Argv[C.getArguments().size() + 1] = 0;
 
-  if (getDriver().CCCEcho || getDriver().CCPrintOptions ||
-      getArgs().hasArg(options::OPT_v)) {
-    llvm::raw_ostream *OS = &llvm::errs();
+  if ((getDriver().CCCEcho || getDriver().CCPrintOptions ||
+       getArgs().hasArg(options::OPT_v)) && !getDriver().CCGenDiagnostics) {
+    raw_ostream *OS = &llvm::errs();
 
     // Follow gcc implementation of CC_PRINT_OPTIONS; we could also cache the
     // output stream.
@@ -167,7 +177,7 @@ int Compilation::ExecuteCommand(const Command &C,
   std::string Error;
   int Res =
     llvm::sys::Program::ExecuteAndWait(Prog, Argv,
-                                       /*env*/0, /*redirects*/0,
+                                       /*env*/0, Redirects,
                                        /*secondsToWait*/0, /*memoryLimit*/0,
                                        &Error);
   if (!Error.empty()) {
@@ -195,3 +205,24 @@ int Compilation::ExecuteJob(const Job &J,
     return 0;
   }
 }
+
+void Compilation::initCompilationForDiagnostics(void) {
+  // Free actions and jobs.
+  DeleteContainerPointers(Actions);
+  Jobs.clear();
+
+  // Clear temporary/results file lists.
+  TempFiles.clear();
+  ResultFiles.clear();
+
+  // Remove any user specified output.  Claim any unclaimed arguments, so as
+  // to avoid emitting warnings about unused args.
+  if (TranslatedArgs->hasArg(options::OPT_o))
+    TranslatedArgs->eraseArg(options::OPT_o);
+  TranslatedArgs->ClaimAllArgs();
+
+  // Redirect stdout/stderr to /dev/null.
+  Redirects = new const llvm::sys::Path*[3]();
+  Redirects[1] = new const llvm::sys::Path();
+  Redirects[2] = new const llvm::sys::Path();
+}
diff --git a/contrib/llvm/tools/clang/lib/Driver/Driver.cpp b/contrib/llvm/tools/clang/lib/Driver/Driver.cpp
index 08605722ac50..4e2d7b623e89 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Driver.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Driver.cpp
@@ -25,7 +25,6 @@
 #include "clang/Driver/Options.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
-#include "clang/Driver/Types.h"
 
 #include "clang/Basic/Version.h"
 
@@ -33,6 +32,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/FileSystem.h"
@@ -46,11 +46,11 @@
 using namespace clang::driver;
 using namespace clang;
 
-Driver::Driver(llvm::StringRef ClangExecutable,
-               llvm::StringRef DefaultHostTriple,
-               llvm::StringRef DefaultImageName,
-               bool IsProduction, bool CXXIsProduction,
-               Diagnostic &Diags)
+Driver::Driver(StringRef ClangExecutable,
+               StringRef DefaultHostTriple,
+               StringRef DefaultImageName,
+               bool IsProduction,
+               DiagnosticsEngine &Diags)
   : Opts(createDriverOptTable()), Diags(Diags),
     ClangExecutable(ClangExecutable), UseStdLib(true),
     DefaultHostTriple(DefaultHostTriple), DefaultImageName(DefaultImageName),
@@ -60,9 +60,9 @@ Driver::Driver(llvm::StringRef ClangExecutable,
     CCLogDiagnosticsFilename(0), CCCIsCXX(false),
     CCCIsCPP(false),CCCEcho(false), CCCPrintBindings(false),
     CCPrintOptions(false), CCPrintHeaders(false), CCLogDiagnostics(false),
-    CCCGenericGCCName(""), CheckInputsExist(true), CCCUseClang(true),
-    CCCUseClangCXX(true), CCCUseClangCPP(true), CCCUsePCH(true),
-    SuppressMissingInputWarning(false) {
+    CCGenDiagnostics(false), CCCGenericGCCName(""), CheckInputsExist(true),
+    CCCUseClang(true), CCCUseClangCXX(true), CCCUseClangCPP(true),
+    CCCUsePCH(true), SuppressMissingInputWarning(false) {
   if (IsProduction) {
     // In a "production" build, only use clang on architectures we expect to
     // work, and don't use clang C++.
@@ -73,16 +73,13 @@ Driver::Driver(llvm::StringRef ClangExecutable,
     CCCClangArchs.insert(llvm::Triple::x86);
     CCCClangArchs.insert(llvm::Triple::x86_64);
     CCCClangArchs.insert(llvm::Triple::arm);
-
-    if (!CXXIsProduction)
-      CCCUseClangCXX = false;
   }
 
   Name = llvm::sys::path::stem(ClangExecutable);
   Dir  = llvm::sys::path::parent_path(ClangExecutable);
 
   // Compute the path to the resource directory.
-  llvm::StringRef ClangResourceDir(CLANG_RESOURCE_DIR);
+  StringRef ClangResourceDir(CLANG_RESOURCE_DIR);
   llvm::SmallString<128> P(Dir);
   if (ClangResourceDir != "")
     llvm::sys::path::append(P, ClangResourceDir);
@@ -96,7 +93,7 @@ Driver::~Driver() {
   delete Host;
 }
 
-InputArgList *Driver::ParseArgStrings(llvm::ArrayRef<const char *> ArgList) {
+InputArgList *Driver::ParseArgStrings(ArrayRef<const char *> ArgList) {
   llvm::PrettyStackTraceString CrashInfo("Command line argument parsing");
   unsigned MissingArgIndex, MissingArgCount;
   InputArgList *Args = getOpts().ParseArgs(ArgList.begin(), ArgList.end(),
@@ -120,6 +117,43 @@ InputArgList *Driver::ParseArgStrings(llvm::ArrayRef<const char *> ArgList) {
   return Args;
 }
 
+// Determine which compilation mode we are in. We look for options which
+// affect the phase, starting with the earliest phases, and record which
+// option we used to determine the final phase.
+phases::ID Driver::getFinalPhase(const DerivedArgList &DAL, Arg **FinalPhaseArg)
+const {
+  Arg *PhaseArg = 0;
+  phases::ID FinalPhase;
+
+  // -{E,M,MM} only run the preprocessor.
+  if (CCCIsCPP ||
+      (PhaseArg = DAL.getLastArg(options::OPT_E)) ||
+      (PhaseArg = DAL.getLastArg(options::OPT_M, options::OPT_MM))) {
+    FinalPhase = phases::Preprocess;
+
+    // -{fsyntax-only,-analyze,emit-ast,S} only run up to the compiler.
+  } else if ((PhaseArg = DAL.getLastArg(options::OPT_fsyntax_only)) ||
+             (PhaseArg = DAL.getLastArg(options::OPT_rewrite_objc)) ||
+             (PhaseArg = DAL.getLastArg(options::OPT__analyze,
+                                              options::OPT__analyze_auto)) ||
+             (PhaseArg = DAL.getLastArg(options::OPT_emit_ast)) ||
+             (PhaseArg = DAL.getLastArg(options::OPT_S))) {
+    FinalPhase = phases::Compile;
+
+    // -c only runs up to the assembler.
+  } else if ((PhaseArg = DAL.getLastArg(options::OPT_c))) {
+    FinalPhase = phases::Assemble;
+
+    // Otherwise do everything.
+  } else
+    FinalPhase = phases::Link;
+
+  if (FinalPhaseArg)
+    *FinalPhaseArg = PhaseArg;
+
+  return FinalPhase;
+}
+
 DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
   DerivedArgList *DAL = new DerivedArgList(Args);
 
@@ -142,7 +176,7 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
 
       // Add the remaining values as Xlinker arguments.
       for (unsigned i = 0, e = A->getNumValues(); i != e; ++i)
-        if (llvm::StringRef(A->getValue(Args, i)) != "--no-demangle")
+        if (StringRef(A->getValue(Args, i)) != "--no-demangle")
           DAL->AddSeparateArg(A, Opts->getOption(options::OPT_Xlinker),
                               A->getValue(Args, i));
 
@@ -154,10 +188,10 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
     // care to encourage this usage model.
     if (A->getOption().matches(options::OPT_Wp_COMMA) &&
         A->getNumValues() == 2 &&
-        (A->getValue(Args, 0) == llvm::StringRef("-MD") ||
-         A->getValue(Args, 0) == llvm::StringRef("-MMD"))) {
+        (A->getValue(Args, 0) == StringRef("-MD") ||
+         A->getValue(Args, 0) == StringRef("-MMD"))) {
       // Rewrite to -MD/-MMD along with -MF.
-      if (A->getValue(Args, 0) == llvm::StringRef("-MD"))
+      if (A->getValue(Args, 0) == StringRef("-MD"))
         DAL->AddFlagArg(A, Opts->getOption(options::OPT_MD));
       else
         DAL->AddFlagArg(A, Opts->getOption(options::OPT_MMD));
@@ -168,7 +202,7 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
 
     // Rewrite reserved library names.
     if (A->getOption().matches(options::OPT_l)) {
-      llvm::StringRef Value = A->getValue(Args);
+      StringRef Value = A->getValue(Args);
 
       // Rewrite unless -nostdlib is present.
       if (!HasNostdlib && Value == "stdc++") {
@@ -201,12 +235,20 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
   return DAL;
 }
 
-Compilation *Driver::BuildCompilation(llvm::ArrayRef<const char *> ArgList) {
+Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   llvm::PrettyStackTraceString CrashInfo("Compilation construction");
 
-  // FIXME: Handle environment options which effect driver behavior, somewhere
-  // (client?). GCC_EXEC_PREFIX, COMPILER_PATH, LIBRARY_PATH, LPATH,
-  // CC_PRINT_OPTIONS.
+  // FIXME: Handle environment options which affect driver behavior, somewhere
+  // (client?). GCC_EXEC_PREFIX, LIBRARY_PATH, LPATH, CC_PRINT_OPTIONS.
+
+  if (char *env = ::getenv("COMPILER_PATH")) {
+    StringRef CompilerPath = env;
+    while (!CompilerPath.empty()) {
+      std::pair<StringRef, StringRef> Split = CompilerPath.split(':');      
+      PrefixDirs.push_back(Split.first);
+      CompilerPath = Split.second;
+    }
+  }
 
   // FIXME: What are we going to do with -V and -b?
 
@@ -242,11 +284,11 @@ Compilation *Driver::BuildCompilation(llvm::ArrayRef<const char *> ArgList) {
   CCCUseClang = !Args->hasArg(options::OPT_ccc_no_clang);
   CCCUseClangCPP = !Args->hasArg(options::OPT_ccc_no_clang_cpp);
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_clang_archs)) {
-    llvm::StringRef Cur = A->getValue(*Args);
+    StringRef Cur = A->getValue(*Args);
 
     CCCClangArchs.clear();
     while (!Cur.empty()) {
-      std::pair<llvm::StringRef, llvm::StringRef> Split = Cur.split(',');
+      std::pair<StringRef, StringRef> Split = Cur.split(',');
 
       if (!Split.first.empty()) {
         llvm::Triple::ArchType Arch =
@@ -296,12 +338,17 @@ Compilation *Driver::BuildCompilation(llvm::ArrayRef<const char *> ArgList) {
   if (!HandleImmediateArgs(*C))
     return C;
 
+  // Construct the list of inputs.
+  InputList Inputs;
+  BuildInputs(C->getDefaultToolChain(), C->getArgs(), Inputs);
+
   // Construct the list of abstract actions to perform for this compilation.
   if (Host->useDriverDriver())
     BuildUniversalActions(C->getDefaultToolChain(), C->getArgs(),
-                          C->getActions());
+                          Inputs, C->getActions());
   else
-    BuildActions(C->getDefaultToolChain(), C->getArgs(), C->getActions());
+    BuildActions(C->getDefaultToolChain(), C->getArgs(), Inputs,
+                 C->getActions());
 
   if (CCCPrintActions) {
     PrintActions(*C);
@@ -313,7 +360,112 @@ Compilation *Driver::BuildCompilation(llvm::ArrayRef<const char *> ArgList) {
   return C;
 }
 
-int Driver::ExecuteCompilation(const Compilation &C) const {
+// When clang crashes, produce diagnostic information including the fully
+// preprocessed source file(s).  Request that the developer attach the
+// diagnostic information to a bug report.
+void Driver::generateCompilationDiagnostics(Compilation &C,
+                                            const Command *FailingCommand) {
+  Diag(clang::diag::note_drv_command_failed_diag_msg)
+    << "Please submit a bug report to " BUG_REPORT_URL " and include command"
+    " line arguments and all diagnostic information.";
+
+  // Suppress driver output and emit preprocessor output to temp file.
+  CCCIsCPP = true;
+  CCGenDiagnostics = true;
+
+  // Clear stale state and suppress tool output.
+  C.initCompilationForDiagnostics();
+  Diags.Reset();
+
+  // Construct the list of inputs.
+  InputList Inputs;
+  BuildInputs(C.getDefaultToolChain(), C.getArgs(), Inputs);
+
+  for (InputList::iterator it = Inputs.begin(), ie = Inputs.end(); it != ie;) {
+    bool IgnoreInput = false;
+
+    // Ignore input from stdin or any inputs that cannot be preprocessed.
+    if (!strcmp(it->second->getValue(C.getArgs()), "-")) {
+      Diag(clang::diag::note_drv_command_failed_diag_msg)
+        << "Error generating preprocessed source(s) - ignoring input from stdin"
+        ".";
+      IgnoreInput = true;
+    } else if (types::getPreprocessedType(it->first) == types::TY_INVALID) {
+      IgnoreInput = true;
+    }
+
+    if (IgnoreInput) {
+      it = Inputs.erase(it);
+      ie = Inputs.end();
+    } else {
+      ++it;
+    }
+  }
+
+  // Don't attempt to generate preprocessed files if multiple -arch options are
+  // used.
+  int Archs = 0;
+  for (ArgList::const_iterator it = C.getArgs().begin(), ie = C.getArgs().end();
+       it != ie; ++it) {
+    Arg *A = *it;
+    if (A->getOption().matches(options::OPT_arch)) {
+      Archs++;
+      if (Archs > 1) {
+        Diag(clang::diag::note_drv_command_failed_diag_msg)
+          << "Error generating preprocessed source(s) - cannot generate "
+          "preprocessed source with multiple -arch options.";
+        return;
+      }
+    }
+  }
+
+  if (Inputs.empty()) {
+    Diag(clang::diag::note_drv_command_failed_diag_msg)
+      << "Error generating preprocessed source(s) - no preprocessable inputs.";
+    return;
+  }
+
+  // Construct the list of abstract actions to perform for this compilation.
+  if (Host->useDriverDriver())
+    BuildUniversalActions(C.getDefaultToolChain(), C.getArgs(),
+                          Inputs, C.getActions());
+  else
+    BuildActions(C.getDefaultToolChain(), C.getArgs(), Inputs,
+                 C.getActions());
+
+  BuildJobs(C);
+
+  // If there were errors building the compilation, quit now.
+  if (Diags.hasErrorOccurred()) {
+    Diag(clang::diag::note_drv_command_failed_diag_msg)
+      << "Error generating preprocessed source(s).";
+    return;
+  }
+
+  // Generate preprocessed output.
+  FailingCommand = 0;
+  int Res = C.ExecuteJob(C.getJobs(), FailingCommand);
+
+  // If the command succeeded, we are done.
+  if (Res == 0) {
+    Diag(clang::diag::note_drv_command_failed_diag_msg)
+      << "Preprocessed source(s) are located at:";
+    ArgStringList Files = C.getTempFiles();
+    for (ArgStringList::const_iterator it = Files.begin(), ie = Files.end();
+         it != ie; ++it)
+      Diag(clang::diag::note_drv_command_failed_diag_msg) << *it;
+  } else {
+    // Failure, remove preprocessed files.
+    if (!C.getArgs().hasArg(options::OPT_save_temps))
+      C.CleanupFileList(C.getTempFiles(), true);
+
+    Diag(clang::diag::note_drv_command_failed_diag_msg)
+      << "Error generating preprocessed source(s).";
+  }
+}
+
+int Driver::ExecuteCompilation(const Compilation &C,
+                               const Command *&FailingCommand) const {
   // Just print if -### was present.
   if (C.getArgs().hasArg(options::OPT__HASH_HASH_HASH)) {
     C.PrintJob(llvm::errs(), C.getJobs(), "\n", true);
@@ -321,10 +473,9 @@ int Driver::ExecuteCompilation(const Compilation &C) const {
   }
 
   // If there were errors building the compilation, quit now.
-  if (getDiags().hasErrorOccurred())
+  if (Diags.hasErrorOccurred())
     return 1;
 
-  const Command *FailingCommand = 0;
   int Res = C.ExecuteJob(C.getJobs(), FailingCommand);
 
   // Remove temp files.
@@ -382,7 +533,7 @@ void Driver::PrintHelp(bool ShowHidden) const {
                       ShowHidden);
 }
 
-void Driver::PrintVersion(const Compilation &C, llvm::raw_ostream &OS) const {
+void Driver::PrintVersion(const Compilation &C, raw_ostream &OS) const {
   // FIXME: The following handlers should use a callback mechanism, we don't
   // know what the client would like to do.
   OS << getClangFullVersion() << '\n';
@@ -397,7 +548,7 @@ void Driver::PrintVersion(const Compilation &C, llvm::raw_ostream &OS) const {
 
 /// PrintDiagnosticCategories - Implement the --print-diagnostic-categories
 /// option.
-static void PrintDiagnosticCategories(llvm::raw_ostream &OS) {
+static void PrintDiagnosticCategories(raw_ostream &OS) {
   // Skip the empty category.
   for (unsigned i = 1, max = DiagnosticIDs::getNumberOfCategories();
        i != max; ++i)
@@ -457,7 +608,7 @@ bool Driver::HandleImmediateArgs(const Compilation &C) {
       llvm::outs() << *it;
     }
     llvm::outs() << "\n";
-    llvm::outs() << "libraries: =";
+    llvm::outs() << "libraries: =" << ResourceDir;
 
     std::string sysroot;
     if (Arg *A = C.getArgs().getLastArg(options::OPT__sysroot_EQ))
@@ -465,8 +616,7 @@ bool Driver::HandleImmediateArgs(const Compilation &C) {
 
     for (ToolChain::path_list::const_iterator it = TC.getFilePaths().begin(),
            ie = TC.getFilePaths().end(); it != ie; ++it) {
-      if (it != TC.getFilePaths().begin())
-        llvm::outs() << ':';
+      llvm::outs() << ':';
       const char *path = it->c_str();
       if (path[0] == '=')
         llvm::outs() << sysroot << path + 1;
@@ -594,12 +744,13 @@ static bool ContainsCompileOrAssembleAction(const Action *A) {
 
 void Driver::BuildUniversalActions(const ToolChain &TC,
                                    const DerivedArgList &Args,
+                                   const InputList &BAInputs,
                                    ActionList &Actions) const {
   llvm::PrettyStackTraceString CrashInfo("Building universal build actions");
   // Collect the list of architectures. Duplicates are allowed, but should only
   // be handled once (in the order seen).
   llvm::StringSet<> ArchNames;
-  llvm::SmallVector<const char *, 4> Archs;
+  SmallVector<const char *, 4> Archs;
   for (ArgList::const_iterator it = Args.begin(), ie = Args.end();
        it != ie; ++it) {
     Arg *A = *it;
@@ -638,7 +789,7 @@ void Driver::BuildUniversalActions(const ToolChain &TC,
   }
 
   ActionList SingleActions;
-  BuildActions(TC, Args, SingleActions);
+  BuildActions(TC, Args, BAInputs, SingleActions);
 
   // Add in arch bindings for every top level action, as well as lipo and
   // dsymutil steps if needed.
@@ -683,23 +834,33 @@ void Driver::BuildUniversalActions(const ToolChain &TC,
         Actions.pop_back();
 
         Actions.push_back(new DsymutilJobAction(Inputs, types::TY_dSYM));
+
+	// Verify the debug output if we're in assert mode.
+	// TODO: The verifier is noisy by default so put this under an
+	// option for now.
+	#ifndef NDEBUG
+	if (Args.hasArg(options::OPT_verify)) {
+	  ActionList VerifyInputs;
+	  VerifyInputs.push_back(Actions.back());
+	  Actions.pop_back();
+	  Actions.push_back(new VerifyJobAction(VerifyInputs,
+						types::TY_Nothing));
+	}
+        #endif
       }
     }
   }
 }
 
-void Driver::BuildActions(const ToolChain &TC, const DerivedArgList &Args,
-                          ActionList &Actions) const {
-  llvm::PrettyStackTraceString CrashInfo("Building compilation actions");
-  // Start by constructing the list of inputs and their types.
-
+// Construct a the list of inputs and their types.
+void Driver::BuildInputs(const ToolChain &TC, const DerivedArgList &Args,
+                         InputList &Inputs) const {
   // Track the current user specified (-x) input. We also explicitly track the
   // argument used to set the type; we only want to claim the type when we
   // actually use it, so we warn about unused -x arguments.
   types::ID InputType = types::TY_Nothing;
   Arg *InputTypeArg = 0;
 
-  llvm::SmallVector<std::pair<types::ID, const Arg*>, 16> Inputs;
   for (ArgList::const_iterator it = Args.begin(), ie = Args.end();
        it != ie; ++it) {
     Arg *A = *it;
@@ -803,7 +964,6 @@ void Driver::BuildActions(const ToolChain &TC, const DerivedArgList &Args,
       }
     }
   }
-
   if (CCCIsCPP && Inputs.empty()) {
     // If called as standalone preprocessor, stdin is processed
     // if no other input is present.
@@ -812,40 +972,19 @@ void Driver::BuildActions(const ToolChain &TC, const DerivedArgList &Args,
     A->claim();
     Inputs.push_back(std::make_pair(types::TY_C, A));
   }
+}
+
+void Driver::BuildActions(const ToolChain &TC, const DerivedArgList &Args,
+                          const InputList &Inputs, ActionList &Actions) const {
+  llvm::PrettyStackTraceString CrashInfo("Building compilation actions");
 
   if (!SuppressMissingInputWarning && Inputs.empty()) {
     Diag(clang::diag::err_drv_no_input_files);
     return;
   }
 
-  // Determine which compilation mode we are in. We look for options which
-  // affect the phase, starting with the earliest phases, and record which
-  // option we used to determine the final phase.
-  Arg *FinalPhaseArg = 0;
-  phases::ID FinalPhase;
-
-  // -{E,M,MM} only run the preprocessor.
-  if (CCCIsCPP ||
-      (FinalPhaseArg = Args.getLastArg(options::OPT_E)) ||
-      (FinalPhaseArg = Args.getLastArg(options::OPT_M, options::OPT_MM))) {
-    FinalPhase = phases::Preprocess;
-
-    // -{fsyntax-only,-analyze,emit-ast,S} only run up to the compiler.
-  } else if ((FinalPhaseArg = Args.getLastArg(options::OPT_fsyntax_only)) ||
-             (FinalPhaseArg = Args.getLastArg(options::OPT_rewrite_objc)) ||
-             (FinalPhaseArg = Args.getLastArg(options::OPT__analyze,
-                                              options::OPT__analyze_auto)) ||
-             (FinalPhaseArg = Args.getLastArg(options::OPT_emit_ast)) ||
-             (FinalPhaseArg = Args.getLastArg(options::OPT_S))) {
-    FinalPhase = phases::Compile;
-
-    // -c only runs up to the assembler.
-  } else if ((FinalPhaseArg = Args.getLastArg(options::OPT_c))) {
-    FinalPhase = phases::Assemble;
-
-    // Otherwise do everything.
-  } else
-    FinalPhase = phases::Link;
+  Arg *FinalPhaseArg;
+  phases::ID FinalPhase = getFinalPhase(Args, &FinalPhaseArg);
 
   // Reject -Z* at the top level, these options should never have been exposed
   // by gcc.
@@ -935,7 +1074,7 @@ Action *Driver::ConstructPhaseAction(const ArgList &Args, phases::ID Phase,
   llvm::PrettyStackTraceString CrashInfo("Constructing phase actions");
   // Build the appropriate action.
   switch (Phase) {
-  case phases::Link: assert(0 && "link action invalid here.");
+  case phases::Link: llvm_unreachable("link action invalid here.");
   case phases::Preprocess: {
     types::ID OutputTy;
     // -{M, MM} alter the output type.
@@ -971,8 +1110,7 @@ Action *Driver::ConstructPhaseAction(const ArgList &Args, phases::ID Phase,
     return new AssembleJobAction(Input, types::TY_Object);
   }
 
-  assert(0 && "invalid phase in ConstructPhaseAction");
-  return 0;
+  llvm_unreachable("invalid phase in ConstructPhaseAction");
 }
 
 bool Driver::IsUsingLTO(const ArgList &Args) const {
@@ -1048,7 +1186,8 @@ void Driver::BuildJobs(Compilation &C) const {
     Arg *A = *it;
 
     // FIXME: It would be nice to be able to send the argument to the
-    // Diagnostic, so that extra values, position, and so on could be printed.
+    // DiagnosticsEngine, so that extra values, position, and so on could be
+    // printed.
     if (!A->isClaimed()) {
       if (A->getOption().hasNoArgumentUnused())
         continue;
@@ -1091,7 +1230,7 @@ static const Tool &SelectToolForJob(Compilation &C, const ToolChain *TC,
   bool HasStatic = (C.getArgs().hasArg(options::OPT_mkernel) ||
                     C.getArgs().hasArg(options::OPT_static) ||
                     C.getArgs().hasArg(options::OPT_fapple_kext));
-  bool IsDarwin = TC->getTriple().getOS() == llvm::Triple::Darwin;
+  bool IsDarwin = TC->getTriple().isOSDarwin();
   bool IsIADefault = TC->IsIntegratedAssemblerDefault() &&
     !(HasStatic && IsDarwin);
   if (C.getArgs().hasFlag(options::OPT_integrated_as,
@@ -1176,6 +1315,11 @@ void Driver::BuildJobsForAction(Compilation &C,
     if (AtTopLevel && isa<DsymutilJobAction>(A))
       SubJobAtTopLevel = true;
 
+    // Also treat verify sub-jobs as being at the top-level. They don't
+    // produce any output and so don't need temporary output names.
+    if (AtTopLevel && isa<VerifyJobAction>(A))
+      SubJobAtTopLevel = true;
+
     InputInfo II;
     BuildJobsForAction(C, *it, TC, BoundArch,
                        SubJobAtTopLevel, LinkingOutput, II);
@@ -1198,7 +1342,7 @@ void Driver::BuildJobsForAction(Compilation &C,
                        A->getType(), BaseInput);
   }
 
-  if (CCCPrintBindings) {
+  if (CCCPrintBindings && !CCGenDiagnostics) {
     llvm::errs() << "# \"" << T.getToolChain().getTripleString() << '"'
                  << " - \"" << T.getName() << "\", inputs: [";
     for (unsigned i = 0, e = InputInfos.size(); i != e; ++i) {
@@ -1219,27 +1363,31 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
                                        bool AtTopLevel) const {
   llvm::PrettyStackTraceString CrashInfo("Computing output path");
   // Output to a user requested destination?
-  if (AtTopLevel && !isa<DsymutilJobAction>(JA)) {
+  if (AtTopLevel && !isa<DsymutilJobAction>(JA) &&
+      !isa<VerifyJobAction>(JA)) {
     if (Arg *FinalOutput = C.getArgs().getLastArg(options::OPT_o))
       return C.addResultFile(FinalOutput->getValue(C.getArgs()));
   }
 
   // Default to writing to stdout?
-  if (AtTopLevel && isa<PreprocessJobAction>(JA))
+  if (AtTopLevel && isa<PreprocessJobAction>(JA) && !CCGenDiagnostics)
     return "-";
 
   // Output to a temporary file?
-  if (!AtTopLevel && !C.getArgs().hasArg(options::OPT_save_temps)) {
+  if ((!AtTopLevel && !C.getArgs().hasArg(options::OPT_save_temps)) ||
+      CCGenDiagnostics) {
+    StringRef Name = llvm::sys::path::filename(BaseInput);
+    std::pair<StringRef, StringRef> Split = Name.split('.');
     std::string TmpName =
-      GetTemporaryPath(types::getTypeTempSuffix(JA.getType()));
+      GetTemporaryPath(Split.first, types::getTypeTempSuffix(JA.getType()));
     return C.addTempFile(C.getArgs().MakeArgString(TmpName.c_str()));
   }
 
   llvm::SmallString<128> BasePath(BaseInput);
-  llvm::StringRef BaseName;
+  StringRef BaseName;
 
   // Dsymutil actions should use the full path.
-  if (isa<DsymutilJobAction>(JA))
+  if (isa<DsymutilJobAction>(JA) || isa<VerifyJobAction>(JA))
     BaseName = BasePath;
   else
     BaseName = llvm::sys::path::filename(BasePath);
@@ -1261,12 +1409,14 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
     NamedOutput = C.getArgs().MakeArgString(Suffixed.c_str());
   }
 
-  // If we're saving temps and the temp filename conflicts with the input 
+  // If we're saving temps and the temp filename conflicts with the input
   // filename, then avoid overwriting input file.
   if (!AtTopLevel && C.getArgs().hasArg(options::OPT_save_temps) &&
-    NamedOutput == BaseName) {
+      NamedOutput == BaseName) {
+    StringRef Name = llvm::sys::path::filename(BaseInput);
+    std::pair<StringRef, StringRef> Split = Name.split('.');
     std::string TmpName =
-      GetTemporaryPath(types::getTypeTempSuffix(JA.getType()));
+      GetTemporaryPath(Split.first, types::getTypeTempSuffix(JA.getType()));
     return C.addTempFile(C.getArgs().MakeArgString(TmpName.c_str()));
   }
 
@@ -1300,6 +1450,12 @@ std::string Driver::GetFilePath(const char *Name, const ToolChain &TC) const {
       return P.str();
   }
 
+  llvm::sys::Path P(ResourceDir);
+  P.appendComponent(Name);
+  bool Exists;
+  if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
+    return P.str();
+
   const ToolChain::path_list &List = TC.getFilePaths();
   for (ToolChain::path_list::const_iterator
          it = List.begin(), ie = List.end(); it != ie; ++it) {
@@ -1318,40 +1474,53 @@ std::string Driver::GetFilePath(const char *Name, const ToolChain &TC) const {
   return Name;
 }
 
+static bool isPathExecutable(llvm::sys::Path &P, bool WantFile) {
+    bool Exists;
+    return (WantFile ? !llvm::sys::fs::exists(P.str(), Exists) && Exists
+                 : P.canExecute());
+}
+
 std::string Driver::GetProgramPath(const char *Name, const ToolChain &TC,
                                    bool WantFile) const {
+  std::string TargetSpecificExecutable(DefaultHostTriple + "-" + Name);
   // Respect a limited subset of the '-Bprefix' functionality in GCC by
   // attempting to use this prefix when lokup up program paths.
   for (Driver::prefix_list::const_iterator it = PrefixDirs.begin(),
        ie = PrefixDirs.end(); it != ie; ++it) {
     llvm::sys::Path P(*it);
+    P.appendComponent(TargetSpecificExecutable);
+    if (isPathExecutable(P, WantFile)) return P.str();
+    P.eraseComponent();
     P.appendComponent(Name);
-    bool Exists;
-    if (WantFile ? !llvm::sys::fs::exists(P.str(), Exists) && Exists
-                 : P.canExecute())
-      return P.str();
+    if (isPathExecutable(P, WantFile)) return P.str();
   }
 
   const ToolChain::path_list &List = TC.getProgramPaths();
   for (ToolChain::path_list::const_iterator
          it = List.begin(), ie = List.end(); it != ie; ++it) {
     llvm::sys::Path P(*it);
+    P.appendComponent(TargetSpecificExecutable);
+    if (isPathExecutable(P, WantFile)) return P.str();
+    P.eraseComponent();
     P.appendComponent(Name);
-    bool Exists;
-    if (WantFile ? !llvm::sys::fs::exists(P.str(), Exists) && Exists
-                 : P.canExecute())
-      return P.str();
+    if (isPathExecutable(P, WantFile)) return P.str();
   }
 
   // If all else failed, search the path.
-  llvm::sys::Path P(llvm::sys::Program::FindProgramByName(Name));
+  llvm::sys::Path
+      P(llvm::sys::Program::FindProgramByName(TargetSpecificExecutable));
+  if (!P.empty())
+    return P.str();
+
+  P = llvm::sys::Path(llvm::sys::Program::FindProgramByName(Name));
   if (!P.empty())
     return P.str();
 
   return Name;
 }
 
-std::string Driver::GetTemporaryPath(const char *Suffix) const {
+std::string Driver::GetTemporaryPath(StringRef Prefix, const char *Suffix) 
+  const {
   // FIXME: This is lame; sys::Path should provide this function (in particular,
   // it should know how to find the temporary files dir).
   std::string Error;
@@ -1363,7 +1532,7 @@ std::string Driver::GetTemporaryPath(const char *Suffix) const {
   if (!TmpDir)
     TmpDir = "/tmp";
   llvm::sys::Path P(TmpDir);
-  P.appendComponent("cc");
+  P.appendComponent(Prefix);
   if (P.makeUnique(false, &Error)) {
     Diag(clang::diag::err_drv_unable_to_make_temp) << Error;
     return "";
@@ -1388,6 +1557,8 @@ const HostInfo *Driver::GetHostInfo(const char *TripleStr) const {
   case llvm::Triple::AuroraUX:
     return createAuroraUXHostInfo(*this, Triple);
   case llvm::Triple::Darwin:
+  case llvm::Triple::MacOSX:
+  case llvm::Triple::IOS:
     return createDarwinHostInfo(*this, Triple);
   case llvm::Triple::DragonFly:
     return createDragonFlyHostInfo(*this, Triple);
diff --git a/contrib/llvm/tools/clang/lib/Driver/HostInfo.cpp b/contrib/llvm/tools/clang/lib/Driver/HostInfo.cpp
index 3b1c2c73fd74..292678bf8aa8 100644
--- a/contrib/llvm/tools/clang/lib/Driver/HostInfo.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/HostInfo.cpp
@@ -188,7 +188,7 @@ bool UnknownHostInfo::useDriverDriver() const {
 ToolChain *UnknownHostInfo::CreateToolChain(const ArgList &Args,
                                             const char *ArchName) const {
   assert(!ArchName &&
-         "Unexpected arch name on platform without driver driver support.");
+         "Unexpected arch name on platform without driver support.");
 
   // Automatically handle some instances of -m32/-m64 we know about.
   std::string Arch = getArchName();
diff --git a/contrib/llvm/tools/clang/lib/Driver/Job.cpp b/contrib/llvm/tools/clang/lib/Driver/Job.cpp
index 51055e93f558..5443d70e825c 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Job.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Job.cpp
@@ -9,6 +9,8 @@
 
 #include "clang/Driver/Job.h"
 
+#include "llvm/ADT/STLExtras.h"
+
 #include <cassert>
 using namespace clang::driver;
 
@@ -28,6 +30,10 @@ JobList::~JobList() {
     delete *it;
 }
 
+void JobList::clear() {
+  DeleteContainerPointers(Jobs);
+}
+
 void Job::addCommand(Command *C) {
   cast<JobList>(this)->addJob(C);
 }
diff --git a/contrib/llvm/tools/clang/lib/Driver/OptTable.cpp b/contrib/llvm/tools/clang/lib/Driver/OptTable.cpp
index 0252b3e5cac5..4f5390b6e26b 100644
--- a/contrib/llvm/tools/clang/lib/Driver/OptTable.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/OptTable.cpp
@@ -12,11 +12,12 @@
 #include "clang/Driver/ArgList.h"
 #include "clang/Driver/Option.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <algorithm>
-#include <cassert>
 #include <map>
 using namespace clang::driver;
 using namespace clang::driver::options;
+using namespace clang;
 
 // Ordering on Info. The ordering is *almost* lexicographic, with two
 // exceptions. First, '\0' comes at the end of the alphabet instead of
@@ -116,7 +117,7 @@ OptTable::OptTable(const Info *_OptionInfos, unsigned _NumOptionInfos)
     if (!(getInfo(i) < getInfo(i + 1))) {
       getOption(i)->dump();
       getOption(i + 1)->dump();
-      assert(0 && "Options are not in order!");
+      llvm_unreachable("Options are not in order!");
     }
   }
 #endif
@@ -268,10 +269,10 @@ static std::string getOptionHelpName(const OptTable &Opts, OptSpecifier Id) {
   // Add metavar, if used.
   switch (Opts.getOptionKind(Id)) {
   case Option::GroupClass: case Option::InputClass: case Option::UnknownClass:
-    assert(0 && "Invalid option with help text.");
+    llvm_unreachable("Invalid option with help text.");
 
   case Option::MultiArgClass:
-    assert(0 && "Cannot print metavar for this kind of option.");
+    llvm_unreachable("Cannot print metavar for this kind of option.");
 
   case Option::FlagClass:
     break;
@@ -291,7 +292,7 @@ static std::string getOptionHelpName(const OptTable &Opts, OptSpecifier Id) {
   return Name;
 }
 
-static void PrintHelpOptionList(llvm::raw_ostream &OS, llvm::StringRef Title,
+static void PrintHelpOptionList(raw_ostream &OS, StringRef Title,
                                 std::vector<std::pair<std::string,
                                 const char*> > &OptionHelp) {
   OS << Title << ":\n";
@@ -342,7 +343,7 @@ static const char *getOptionHelpGroup(const OptTable &Opts, OptSpecifier Id) {
   return getOptionHelpGroup(Opts, GroupID);
 }
 
-void OptTable::PrintHelp(llvm::raw_ostream &OS, const char *Name,
+void OptTable::PrintHelp(raw_ostream &OS, const char *Name,
                          const char *Title, bool ShowHidden) const {
   OS << "OVERVIEW: " << Title << "\n";
   OS << '\n';
diff --git a/contrib/llvm/tools/clang/lib/Driver/Option.cpp b/contrib/llvm/tools/clang/lib/Driver/Option.cpp
index 90d21a3d0b17..ee1963fd331d 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Option.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Option.cpp
@@ -12,6 +12,7 @@
 #include "clang/Driver/Arg.h"
 #include "clang/Driver/ArgList.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <algorithm>
 using namespace clang::driver;
@@ -61,7 +62,7 @@ void Option::dump() const {
   llvm::errs() << "<";
   switch (Kind) {
   default:
-    assert(0 && "Invalid kind");
+    llvm_unreachable("Invalid kind");
 #define P(N) case N: llvm::errs() << #N; break
     P(GroupClass);
     P(InputClass);
@@ -114,8 +115,7 @@ OptionGroup::OptionGroup(OptSpecifier ID, const char *Name,
 }
 
 Arg *OptionGroup::accept(const ArgList &Args, unsigned &Index) const {
-  assert(0 && "accept() should never be called on an OptionGroup");
-  return 0;
+  llvm_unreachable("accept() should never be called on an OptionGroup");
 }
 
 InputOption::InputOption(OptSpecifier ID)
@@ -123,8 +123,7 @@ InputOption::InputOption(OptSpecifier ID)
 }
 
 Arg *InputOption::accept(const ArgList &Args, unsigned &Index) const {
-  assert(0 && "accept() should never be called on an InputOption");
-  return 0;
+  llvm_unreachable("accept() should never be called on an InputOption");
 }
 
 UnknownOption::UnknownOption(OptSpecifier ID)
@@ -132,8 +131,7 @@ UnknownOption::UnknownOption(OptSpecifier ID)
 }
 
 Arg *UnknownOption::accept(const ArgList &Args, unsigned &Index) const {
-  assert(0 && "accept() should never be called on an UnknownOption");
-  return 0;
+  llvm_unreachable("accept() should never be called on an UnknownOption");
 }
 
 FlagOption::FlagOption(OptSpecifier ID, const char *Name,
diff --git a/contrib/llvm/tools/clang/lib/Driver/Phases.cpp b/contrib/llvm/tools/clang/lib/Driver/Phases.cpp
index f36000214ab0..b885eeef9832 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Phases.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Phases.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/Phases.h"
+#include "llvm/Support/ErrorHandling.h"
 
 #include <cassert>
 
@@ -22,6 +23,5 @@ const char *phases::getPhaseName(ID Id) {
   case Link: return "linker";
   }
 
-  assert(0 && "Invalid phase id.");
-  return 0;
+  llvm_unreachable("Invalid phase id.");
 }
diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp
index 74b65918f34f..d09ab16814a0 100644
--- a/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp
@@ -18,11 +18,11 @@
 #include "clang/Driver/ObjCRuntime.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Support/ErrorHandling.h"
-
 using namespace clang::driver;
+using namespace clang;
 
-ToolChain::ToolChain(const HostInfo &_Host, const llvm::Triple &_Triple)
-  : Host(_Host), Triple(_Triple) {
+ToolChain::ToolChain(const HostInfo &H, const llvm::Triple &T)
+  : Host(H), Triple(T) {
 }
 
 ToolChain::~ToolChain() {
@@ -79,7 +79,7 @@ static const char *getARMTargetCPU(const ArgList &Args,
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
     return A->getValue(Args);
 
-  llvm::StringRef MArch;
+  StringRef MArch;
   if (Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
     // Otherwise, if we have -march= choose the base CPU for that arch.
     MArch = A->getValue(Args);
@@ -134,7 +134,7 @@ static const char *getARMTargetCPU(const ArgList &Args,
 //
 // FIXME: This is redundant with -mcpu, why does LLVM use this.
 // FIXME: tblgen this, or kill it!
-static const char *getLLVMArchSuffixForARM(llvm::StringRef CPU) {
+static const char *getLLVMArchSuffixForARM(StringRef CPU) {
   if (CPU == "arm7tdmi" || CPU == "arm7tdmi-s" || CPU == "arm710t" ||
       CPU == "arm720t" || CPU == "arm9" || CPU == "arm9tdmi" ||
       CPU == "arm920" || CPU == "arm920t" || CPU == "arm922t" ||
@@ -169,7 +169,8 @@ static const char *getLLVMArchSuffixForARM(llvm::StringRef CPU) {
   return "";
 }
 
-std::string ToolChain::ComputeLLVMTriple(const ArgList &Args) const {
+std::string ToolChain::ComputeLLVMTriple(const ArgList &Args, 
+                                         types::ID InputType) const {
   switch (getTriple().getArch()) {
   default:
     return getTripleString();
@@ -182,12 +183,14 @@ std::string ToolChain::ComputeLLVMTriple(const ArgList &Args) const {
     // Thumb2 is the default for V7 on Darwin.
     //
     // FIXME: Thumb should just be another -target-feaure, not in the triple.
-    llvm::StringRef Suffix =
+    StringRef Suffix =
       getLLVMArchSuffixForARM(getARMTargetCPU(Args, Triple));
-    bool ThumbDefault =
-      (Suffix == "v7" && getTriple().getOS() == llvm::Triple::Darwin);
+    bool ThumbDefault = (Suffix == "v7" && getTriple().isOSDarwin());
     std::string ArchName = "arm";
-    if (Args.hasFlag(options::OPT_mthumb, options::OPT_mno_thumb, ThumbDefault))
+
+    // Assembly files should start in ARM mode.
+    if (InputType != types::TY_PP_Asm &&
+        Args.hasFlag(options::OPT_mthumb, options::OPT_mno_thumb, ThumbDefault))
       ArchName = "thumb";
     Triple.setArchName(ArchName + Suffix.str());
 
@@ -196,25 +199,26 @@ std::string ToolChain::ComputeLLVMTriple(const ArgList &Args) const {
   }
 }
 
-std::string ToolChain::ComputeEffectiveClangTriple(const ArgList &Args) const {
+std::string ToolChain::ComputeEffectiveClangTriple(const ArgList &Args, 
+                                                   types::ID InputType) const {
   // Diagnose use of Darwin OS deployment target arguments on non-Darwin.
   if (Arg *A = Args.getLastArg(options::OPT_mmacosx_version_min_EQ,
                                options::OPT_miphoneos_version_min_EQ,
                                options::OPT_mios_simulator_version_min_EQ))
-    getDriver().Diag(clang::diag::err_drv_clang_unsupported)
+    getDriver().Diag(diag::err_drv_clang_unsupported)
       << A->getAsString(Args);
 
-  return ComputeLLVMTriple(Args);
+  return ComputeLLVMTriple(Args, InputType);
 }
 
 ToolChain::CXXStdlibType ToolChain::GetCXXStdlibType(const ArgList &Args) const{
   if (Arg *A = Args.getLastArg(options::OPT_stdlib_EQ)) {
-    llvm::StringRef Value = A->getValue(Args);
+    StringRef Value = A->getValue(Args);
     if (Value == "libc++")
       return ToolChain::CST_Libcxx;
     if (Value == "libstdc++")
       return ToolChain::CST_Libstdcxx;
-    getDriver().Diag(clang::diag::err_drv_invalid_stdlib_name)
+    getDriver().Diag(diag::err_drv_invalid_stdlib_name)
       << A->getAsString(Args);
   }
 
diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains.cpp
index 29abb6d6b122..4c625430e193 100644
--- a/contrib/llvm/tools/clang/lib/Driver/ToolChains.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains.cpp
@@ -27,6 +27,7 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
@@ -45,12 +46,14 @@
 
 using namespace clang::driver;
 using namespace clang::driver::toolchains;
+using namespace clang;
 
 /// Darwin - Darwin tool chain for i386 and x86_64.
 
 Darwin::Darwin(const HostInfo &Host, const llvm::Triple& Triple)
   : ToolChain(Host, Triple), TargetInitialized(false),
-    ARCRuntimeForSimulator(ARCSimulator_None)
+    ARCRuntimeForSimulator(ARCSimulator_None),
+    LibCXXForSimulator(LibCXXSimulator_None)
 {
   // Compute the initial Darwin version based on the host.
   bool HadExtra;
@@ -58,7 +61,7 @@ Darwin::Darwin(const HostInfo &Host, const llvm::Triple& Triple)
   if (!Driver::GetReleaseVersion(&OSName.c_str()[6],
                                  DarwinVersion[0], DarwinVersion[1],
                                  DarwinVersion[2], HadExtra))
-    getDriver().Diag(clang::diag::err_drv_invalid_darwin_version) << OSName;
+    getDriver().Diag(diag::err_drv_invalid_darwin_version) << OSName;
 
   llvm::raw_string_ostream(MacosxVersionMin)
     << "10." << std::max(0, (int)DarwinVersion[0] - 4) << '.'
@@ -112,52 +115,40 @@ void Darwin::configureObjCRuntime(ObjCRuntime &runtime) const {
     runtime.HasTerminate = false;
 }
 
-// FIXME: Can we tablegen this?
-static const char *GetArmArchForMArch(llvm::StringRef Value) {
-  if (Value == "armv6k")
-    return "armv6";
-
-  if (Value == "armv5tej")
-    return "armv5";
-
-  if (Value == "xscale")
-    return "xscale";
-
-  if (Value == "armv4t")
-    return "armv4t";
-
-  if (Value == "armv7" || Value == "armv7-a" || Value == "armv7-r" ||
-      Value == "armv7-m" || Value == "armv7a" || Value == "armv7r" ||
-      Value == "armv7m")
-    return "armv7";
-
-  return 0;
+/// Darwin provides a blocks runtime starting in MacOS X 10.6 and iOS 3.2.
+bool Darwin::hasBlocksRuntime() const {
+  if (isTargetIPhoneOS())
+    return !isIPhoneOSVersionLT(3, 2);
+  else
+    return !isMacosxVersionLT(10, 6);
 }
 
-// FIXME: Can we tablegen this?
-static const char *GetArmArchForMCpu(llvm::StringRef Value) {
-  if (Value == "arm10tdmi" || Value == "arm1020t" || Value == "arm9e" ||
-      Value == "arm946e-s" || Value == "arm966e-s" ||
-      Value == "arm968e-s" || Value == "arm10e" ||
-      Value == "arm1020e" || Value == "arm1022e" || Value == "arm926ej-s" ||
-      Value == "arm1026ej-s")
-    return "armv5";
-
-  if (Value == "xscale")
-    return "xscale";
-
-  if (Value == "arm1136j-s" || Value == "arm1136jf-s" ||
-      Value == "arm1176jz-s" || Value == "arm1176jzf-s" ||
-      Value == "cortex-m0" )
-    return "armv6";
-
-  if (Value == "cortex-a8" || Value == "cortex-r4" || Value == "cortex-m3")
-    return "armv7";
-
-  return 0;
+static const char *GetArmArchForMArch(StringRef Value) {
+  return llvm::StringSwitch<const char*>(Value)
+    .Case("armv6k", "armv6")
+    .Case("armv5tej", "armv5")
+    .Case("xscale", "xscale")
+    .Case("armv4t", "armv4t")
+    .Case("armv7", "armv7")
+    .Cases("armv7a", "armv7-a", "armv7")
+    .Cases("armv7r", "armv7-r", "armv7")
+    .Cases("armv7m", "armv7-m", "armv7")
+    .Default(0);
 }
 
-llvm::StringRef Darwin::getDarwinArchName(const ArgList &Args) const {
+static const char *GetArmArchForMCpu(StringRef Value) {
+  return llvm::StringSwitch<const char *>(Value)
+    .Cases("arm9e", "arm946e-s", "arm966e-s", "arm968e-s", "arm926ej-s","armv5")
+    .Cases("arm10e", "arm10tdmi", "armv5")
+    .Cases("arm1020t", "arm1020e", "arm1022e", "arm1026ej-s", "armv5")
+    .Case("xscale", "xscale")
+    .Cases("arm1136j-s", "arm1136jf-s", "arm1176jz-s",
+           "arm1176jzf-s", "cortex-m0", "armv6")
+    .Cases("cortex-a8", "cortex-r4", "cortex-m3", "cortex-a9", "armv7")
+    .Default(0);
+}
+
+StringRef Darwin::getDarwinArchName(const ArgList &Args) const {
   switch (getTriple().getArch()) {
   default:
     return getArchName();
@@ -184,8 +175,9 @@ Darwin::~Darwin() {
     delete it->second;
 }
 
-std::string Darwin::ComputeEffectiveClangTriple(const ArgList &Args) const {
-  llvm::Triple Triple(ComputeLLVMTriple(Args));
+std::string Darwin::ComputeEffectiveClangTriple(const ArgList &Args,
+                                                types::ID InputType) const {
+  llvm::Triple Triple(ComputeLLVMTriple(Args, InputType));
 
   // If the target isn't initialized (e.g., an unknown Darwin platform, return
   // the default triple).
@@ -212,9 +204,10 @@ Tool &Darwin::SelectTool(const Compilation &C, const JobAction &JA,
     // Fallback to llvm-gcc for i386 kext compiles, we don't support that ABI.
     if (Inputs.size() == 1 &&
         types::isCXX(Inputs[0]->getType()) &&
-        getTriple().getOS() == llvm::Triple::Darwin &&
+        getTriple().isOSDarwin() &&
         getTriple().getArch() == llvm::Triple::x86 &&
-        C.getArgs().getLastArg(options::OPT_fapple_kext))
+        (C.getArgs().getLastArg(options::OPT_fapple_kext) ||
+         C.getArgs().getLastArg(options::OPT_mkernel)))
       Key = JA.getKind();
     else
       Key = Action::AnalyzeJobClass;
@@ -236,7 +229,7 @@ Tool &Darwin::SelectTool(const Compilation &C, const JobAction &JA,
     switch (Key) {
     case Action::InputClass:
     case Action::BindArchClass:
-      assert(0 && "Invalid tool kind.");
+      llvm_unreachable("Invalid tool kind.");
     case Action::PreprocessJobClass:
       T = new tools::darwin::Preprocess(*this); break;
     case Action::AnalyzeJobClass:
@@ -257,6 +250,8 @@ Tool &Darwin::SelectTool(const Compilation &C, const JobAction &JA,
       T = new tools::darwin::Lipo(*this); break;
     case Action::DsymutilJobClass:
       T = new tools::darwin::Dsymutil(*this); break;
+    case Action::VerifyJobClass:
+      T = new tools::darwin::VerifyDebug(*this); break;
     }
   }
 
@@ -267,8 +262,6 @@ Tool &Darwin::SelectTool(const Compilation &C, const JobAction &JA,
 DarwinClang::DarwinClang(const HostInfo &Host, const llvm::Triple& Triple)
   : Darwin(Host, Triple)
 {
-  std::string UsrPrefix = "llvm-gcc-4.2/";
-
   getProgramPaths().push_back(getDriver().getInstalledDir());
   if (getDriver().getInstalledDir() != getDriver().Dir)
     getProgramPaths().push_back(getDriver().Dir);
@@ -281,16 +274,24 @@ DarwinClang::DarwinClang(const HostInfo &Host, const llvm::Triple& Triple)
   // For fallback, we need to know how to find the GCC cc1 executables, so we
   // also add the GCC libexec paths. This is legacy code that can be removed
   // once fallback is no longer useful.
+  AddGCCLibexecPath(DarwinVersion[0]);
+  AddGCCLibexecPath(DarwinVersion[0] - 2);
+  AddGCCLibexecPath(DarwinVersion[0] - 1);
+  AddGCCLibexecPath(DarwinVersion[0] + 1);
+  AddGCCLibexecPath(DarwinVersion[0] + 2);
+}
+
+void DarwinClang::AddGCCLibexecPath(unsigned darwinVersion) {
   std::string ToolChainDir = "i686-apple-darwin";
-  ToolChainDir += llvm::utostr(DarwinVersion[0]);
+  ToolChainDir += llvm::utostr(darwinVersion);
   ToolChainDir += "/4.2.1";
 
   std::string Path = getDriver().Dir;
-  Path += "/../" + UsrPrefix + "libexec/gcc/";
+  Path += "/../llvm-gcc-4.2/libexec/gcc/";
   Path += ToolChainDir;
   getProgramPaths().push_back(Path);
 
-  Path = "/usr/" + UsrPrefix + "libexec/gcc/";
+  Path = "/usr/llvm-gcc-4.2/libexec/gcc/";
   Path += ToolChainDir;
   getProgramPaths().push_back(Path);
 }
@@ -308,7 +309,7 @@ void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
   P.appendComponent("gcc");
   switch (getTriple().getArch()) {
   default:
-    assert(0 && "Invalid Darwin arch!");
+    llvm_unreachable("Invalid Darwin arch!");
   case llvm::Triple::x86:
   case llvm::Triple::x86_64:
     P.appendComponent("i686-apple-darwin10");
@@ -332,7 +333,7 @@ void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
   case llvm::Triple::arm:
   case llvm::Triple::thumb: {
     std::string Triple = ComputeLLVMTriple(Args);
-    llvm::StringRef TripleStr = Triple;
+    StringRef TripleStr = Triple;
     if (TripleStr.startswith("armv5") || TripleStr.startswith("thumbv5"))
       ArchSpecificDir = "v5";
     else if (TripleStr.startswith("armv6") || TripleStr.startswith("thumbv6"))
@@ -364,8 +365,8 @@ void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
 
 void DarwinClang::AddLinkARCArgs(const ArgList &Args,
                                  ArgStringList &CmdArgs) const {
-  
-  CmdArgs.push_back("-force_load");    
+
+  CmdArgs.push_back("-force_load");
   llvm::sys::Path P(getDriver().ClangExecutable);
   P.eraseComponent(); // 'clang'
   P.eraseComponent(); // 'bin'
@@ -387,13 +388,13 @@ void DarwinClang::AddLinkARCArgs(const ArgList &Args,
 }
 
 void DarwinClang::AddLinkRuntimeLib(const ArgList &Args,
-                                    ArgStringList &CmdArgs, 
+                                    ArgStringList &CmdArgs,
                                     const char *DarwinStaticLib) const {
   llvm::sys::Path P(getDriver().ResourceDir);
   P.appendComponent("lib");
   P.appendComponent("darwin");
   P.appendComponent(DarwinStaticLib);
-  
+
   // For now, allow missing resource libraries to support developers who may
   // not have compiler-rt checked out or integrated into their build.
   bool Exists;
@@ -412,7 +413,7 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
   // cares. This is useful in situations where someone wants to statically link
   // something like libstdc++, and needs its runtime support routines.
   if (const Arg *A = Args.getLastArg(options::OPT_static_libgcc)) {
-    getDriver().Diag(clang::diag::err_drv_unsupported_opt)
+    getDriver().Diag(diag::err_drv_unsupported_opt)
       << A->getAsString(Args);
     return;
   }
@@ -425,8 +426,9 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
   if (isTargetIPhoneOS()) {
     // If we are compiling as iOS / simulator, don't attempt to link libgcc_s.1,
     // it never went into the SDK.
-    if (!isTargetIOSSimulator())
-        CmdArgs.push_back("-lgcc_s.1");
+    // Linking against libgcc_s.1 isn't needed for iOS 5.0+
+    if (isIPhoneOSVersionLT(5, 0) && !isTargetIOSSimulator())
+      CmdArgs.push_back("-lgcc_s.1");
 
     // We currently always need a static runtime library for iOS.
     AddLinkRuntimeLib(Args, CmdArgs, "libclang_rt.ios.a");
@@ -456,7 +458,7 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
   }
 }
 
-static inline llvm::StringRef SimulatorVersionDefineName() {
+static inline StringRef SimulatorVersionDefineName() {
   return "__IPHONE_OS_VERSION_MIN_REQUIRED";
 }
 
@@ -465,11 +467,11 @@ static inline llvm::StringRef SimulatorVersionDefineName() {
 // and return the grouped values as integers, e.g:
 //   __IPHONE_OS_VERSION_MIN_REQUIRED=40201
 // will return Major=4, Minor=2, Micro=1.
-static bool GetVersionFromSimulatorDefine(llvm::StringRef define,
+static bool GetVersionFromSimulatorDefine(StringRef define,
                                           unsigned &Major, unsigned &Minor,
                                           unsigned &Micro) {
   assert(define.startswith(SimulatorVersionDefineName()));
-  llvm::StringRef name, version;
+  StringRef name, version;
   llvm::tie(name, version) = define.split('=');
   if (version.empty())
     return false;
@@ -500,13 +502,15 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   if (!iOSVersion) {
     for (arg_iterator it = Args.filtered_begin(options::OPT_D),
            ie = Args.filtered_end(); it != ie; ++it) {
-      llvm::StringRef define = (*it)->getValue(Args);
+      StringRef define = (*it)->getValue(Args);
       if (define.startswith(SimulatorVersionDefineName())) {
-        unsigned Major, Minor, Micro;
+        unsigned Major = 0, Minor = 0, Micro = 0;
         if (GetVersionFromSimulatorDefine(define, Major, Minor, Micro) &&
             Major < 10 && Minor < 100 && Micro < 100) {
           ARCRuntimeForSimulator = Major < 5 ? ARCSimulator_NoARCRuntime
                                              : ARCSimulator_HasARCRuntime;
+          LibCXXForSimulator = Major < 5 ? LibCXXSimulator_NotAvailable
+                                         : LibCXXSimulator_Available;
         }
         break;
       }
@@ -514,61 +518,78 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   }
 
   if (OSXVersion && (iOSVersion || iOSSimVersion)) {
-    getDriver().Diag(clang::diag::err_drv_argument_not_allowed_with)
+    getDriver().Diag(diag::err_drv_argument_not_allowed_with)
           << OSXVersion->getAsString(Args)
           << (iOSVersion ? iOSVersion : iOSSimVersion)->getAsString(Args);
     iOSVersion = iOSSimVersion = 0;
   } else if (iOSVersion && iOSSimVersion) {
-    getDriver().Diag(clang::diag::err_drv_argument_not_allowed_with)
+    getDriver().Diag(diag::err_drv_argument_not_allowed_with)
           << iOSVersion->getAsString(Args)
           << iOSSimVersion->getAsString(Args);
     iOSSimVersion = 0;
   } else if (!OSXVersion && !iOSVersion && !iOSSimVersion) {
-    // If not deployment target was specified on the command line, check for
+    // If no deployment target was specified on the command line, check for
     // environment defines.
-    const char *OSXTarget = ::getenv("MACOSX_DEPLOYMENT_TARGET");
-    const char *iOSTarget = ::getenv("IPHONEOS_DEPLOYMENT_TARGET");
-    const char *iOSSimTarget = ::getenv("IOS_SIMULATOR_DEPLOYMENT_TARGET");
+    StringRef OSXTarget;
+    StringRef iOSTarget;
+    StringRef iOSSimTarget;
+    if (char *env = ::getenv("MACOSX_DEPLOYMENT_TARGET"))
+      OSXTarget = env;
+    if (char *env = ::getenv("IPHONEOS_DEPLOYMENT_TARGET"))
+      iOSTarget = env;
+    if (char *env = ::getenv("IOS_SIMULATOR_DEPLOYMENT_TARGET"))
+      iOSSimTarget = env;
 
-    // Ignore empty strings.
-    if (OSXTarget && OSXTarget[0] == '\0')
-      OSXTarget = 0;
-    if (iOSTarget && iOSTarget[0] == '\0')
-      iOSTarget = 0;
-    if (iOSSimTarget && iOSSimTarget[0] == '\0')
-      iOSSimTarget = 0;
+    // If no '-miphoneos-version-min' specified on the command line and
+    // IPHONEOS_DEPLOYMENT_TARGET is not defined, see if we can set the default
+    // based on isysroot.
+    if (iOSTarget.empty()) {
+      if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
+        StringRef first, second;
+        StringRef isysroot = A->getValue(Args);
+        llvm::tie(first, second) = isysroot.split(StringRef("SDKs/iPhoneOS"));
+        if (second != "")
+          iOSTarget = second.substr(0,3);
+      }
+    }
+
+    // If no OSX or iOS target has been specified and we're compiling for armv7,
+    // go ahead as assume we're targeting iOS.
+    if (OSXTarget.empty() && iOSTarget.empty())
+      if (getDarwinArchName(Args) == "armv7")
+        iOSTarget = "0.0";
 
     // Handle conflicting deployment targets
     //
     // FIXME: Don't hardcode default here.
 
     // Do not allow conflicts with the iOS simulator target.
-    if (iOSSimTarget && (OSXTarget || iOSTarget)) {
-      getDriver().Diag(clang::diag::err_drv_conflicting_deployment_targets)
+    if (!iOSSimTarget.empty() && (!OSXTarget.empty() || !iOSTarget.empty())) {
+      getDriver().Diag(diag::err_drv_conflicting_deployment_targets)
         << "IOS_SIMULATOR_DEPLOYMENT_TARGET"
-        << (OSXTarget ? "MACOSX_DEPLOYMENT_TARGET" :
+        << (!OSXTarget.empty() ? "MACOSX_DEPLOYMENT_TARGET" :
             "IPHONEOS_DEPLOYMENT_TARGET");
     }
 
     // Allow conflicts among OSX and iOS for historical reasons, but choose the
     // default platform.
-    if (OSXTarget && iOSTarget) {
+    if (!OSXTarget.empty() && !iOSTarget.empty()) {
       if (getTriple().getArch() == llvm::Triple::arm ||
           getTriple().getArch() == llvm::Triple::thumb)
-        OSXTarget = 0;
+        OSXTarget = "";
       else
-        iOSTarget = 0;
+        iOSTarget = "";
     }
 
-    if (OSXTarget) {
+    if (!OSXTarget.empty()) {
       const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
       OSXVersion = Args.MakeJoinedArg(0, O, OSXTarget);
       Args.append(OSXVersion);
-    } else if (iOSTarget) {
+    } else if (!iOSTarget.empty()) {
       const Option *O = Opts.getOption(options::OPT_miphoneos_version_min_EQ);
       iOSVersion = Args.MakeJoinedArg(0, O, iOSTarget);
       Args.append(iOSVersion);
-    } else if (iOSSimTarget) {
+    } else if (!iOSSimTarget.empty()) {
       const Option *O = Opts.getOption(
         options::OPT_mios_simulator_version_min_EQ);
       iOSSimVersion = Args.MakeJoinedArg(0, O, iOSSimTarget);
@@ -584,7 +605,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   // Reject invalid architecture combinations.
   if (iOSSimVersion && (getTriple().getArch() != llvm::Triple::x86 &&
                         getTriple().getArch() != llvm::Triple::x86_64)) {
-    getDriver().Diag(clang::diag::err_drv_invalid_arch_for_deployment_target)
+    getDriver().Diag(diag::err_drv_invalid_arch_for_deployment_target)
       << getTriple().getArchName() << iOSSimVersion->getAsString(Args);
   }
 
@@ -596,7 +617,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     if (!Driver::GetReleaseVersion(OSXVersion->getValue(Args), Major, Minor,
                                    Micro, HadExtra) || HadExtra ||
         Major != 10 || Minor >= 100 || Micro >= 100)
-      getDriver().Diag(clang::diag::err_drv_invalid_version_number)
+      getDriver().Diag(diag::err_drv_invalid_version_number)
         << OSXVersion->getAsString(Args);
   } else {
     const Arg *Version = iOSVersion ? iOSVersion : iOSSimVersion;
@@ -604,7 +625,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     if (!Driver::GetReleaseVersion(Version->getValue(Args), Major, Minor,
                                    Micro, HadExtra) || HadExtra ||
         Major >= 10 || Minor >= 100 || Micro >= 100)
-      getDriver().Diag(clang::diag::err_drv_invalid_version_number)
+      getDriver().Diag(diag::err_drv_invalid_version_number)
         << Version->getAsString(Args);
   }
 
@@ -708,7 +729,7 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       // triple arch, or the arch being bound.
       //
       // FIXME: Canonicalize name.
-      llvm::StringRef XarchArch = A->getValue(Args, 0);
+      StringRef XarchArch = A->getValue(Args, 0);
       if (!(XarchArch == getArchName()  ||
             (BoundArch && XarchArch == BoundArch)))
         continue;
@@ -727,11 +748,11 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       // use isDriverOption() as an approximation, although things
       // like -O4 are going to slip through.
       if (!XarchArg || Index > Prev + 1) {
-        getDriver().Diag(clang::diag::err_drv_invalid_Xarch_argument_with_args)
+        getDriver().Diag(diag::err_drv_invalid_Xarch_argument_with_args)
           << A->getAsString(Args);
         continue;
       } else if (XarchArg->getOption().isDriverOption()) {
-        getDriver().Diag(clang::diag::err_drv_invalid_Xarch_argument_isdriver)
+        getDriver().Diag(diag::err_drv_invalid_Xarch_argument_isdriver)
           << A->getAsString(Args);
         continue;
       }
@@ -768,7 +789,6 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
     case options::OPT_fapple_kext:
       DAL->append(A);
       DAL->AddFlagArg(A, Opts.getOption(options::OPT_static));
-      DAL->AddFlagArg(A, Opts.getOption(options::OPT_static));
       break;
 
     case options::OPT_dependency_file:
@@ -788,12 +808,6 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
              Opts.getOption(options::OPT_feliminate_unused_debug_symbols));
       break;
 
-    case options::OPT_fterminated_vtables:
-    case options::OPT_findirect_virtual_calls:
-      DAL->AddFlagArg(A, Opts.getOption(options::OPT_fapple_kext));
-      DAL->AddFlagArg(A, Opts.getOption(options::OPT_static));
-      break;
-
     case options::OPT_shared:
       DAL->AddFlagArg(A, Opts.getOption(options::OPT_dynamiclib));
       break;
@@ -834,7 +848,7 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
   // Add the arch options based on the particular spelling of -arch, to match
   // how the driver driver works.
   if (BoundArch) {
-    llvm::StringRef Name = BoundArch;
+    StringRef Name = BoundArch;
     const Option *MCpu = Opts.getOption(options::OPT_mcpu_EQ);
     const Option *MArch = Opts.getOption(options::OPT_march_EQ);
 
@@ -904,6 +918,28 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
   // argument.
   AddDeploymentTarget(*DAL);
 
+  // Validate the C++ standard library choice.
+  CXXStdlibType Type = GetCXXStdlibType(*DAL);
+  if (Type == ToolChain::CST_Libcxx) {
+    switch (LibCXXForSimulator) {
+    case LibCXXSimulator_None:
+      // Handle non-simulator cases.
+      if (isTargetIPhoneOS()) {
+        if (isIPhoneOSVersionLT(5, 0)) {
+          getDriver().Diag(clang::diag::err_drv_invalid_libcxx_deployment)
+            << "iOS 5.0";
+        }
+      }
+      break;
+    case LibCXXSimulator_NotAvailable:
+      getDriver().Diag(clang::diag::err_drv_invalid_libcxx_deployment)
+        << "iOS 5.0";
+      break;
+    case LibCXXSimulator_Available:
+      break;
+    }
+  }
+
   return DAL;
 }
 
@@ -946,8 +982,9 @@ bool Darwin::SupportsObjCGC() const {
 }
 
 std::string
-Darwin_Generic_GCC::ComputeEffectiveClangTriple(const ArgList &Args) const {
-  return ComputeLLVMTriple(Args);
+Darwin_Generic_GCC::ComputeEffectiveClangTriple(const ArgList &Args,
+                                                types::ID InputType) const {
+  return ComputeLLVMTriple(Args, InputType);
 }
 
 /// Generic_GCC - A tool chain using the 'gcc' command to perform
@@ -982,7 +1019,7 @@ Tool &Generic_GCC::SelectTool(const Compilation &C,
     switch (Key) {
     case Action::InputClass:
     case Action::BindArchClass:
-      assert(0 && "Invalid tool kind.");
+      llvm_unreachable("Invalid tool kind.");
     case Action::PreprocessJobClass:
       T = new tools::gcc::Preprocess(*this); break;
     case Action::PrecompileJobClass:
@@ -1002,6 +1039,8 @@ Tool &Generic_GCC::SelectTool(const Compilation &C,
       T = new tools::darwin::Lipo(*this); break;
     case Action::DsymutilJobClass:
       T = new tools::darwin::Dsymutil(*this); break;
+    case Action::VerifyJobClass:
+      T = new tools::darwin::VerifyDebug(*this); break;
     }
   }
 
@@ -1071,7 +1110,7 @@ Tool &TCEToolChain::SelectTool(const Compilation &C,
     case Action::AnalyzeJobClass:
       T = new tools::Clang(*this); break;
     default:
-     assert(false && "Unsupported action for TCE target.");
+     llvm_unreachable("Unsupported action for TCE target.");
     }
   }
   return *T;
@@ -1350,32 +1389,24 @@ static bool IsUbuntu(enum LinuxDistro Distro) {
          Distro == UbuntuNatty  || Distro == UbuntuOneiric;
 }
 
-static bool IsDebianBased(enum LinuxDistro Distro) {
-  return IsDebian(Distro) || IsUbuntu(Distro);
-}
-
+// FIXME: This should be deleted. We should assume a multilib environment, and
+// fallback gracefully if any parts of it are absent.
 static bool HasMultilib(llvm::Triple::ArchType Arch, enum LinuxDistro Distro) {
   if (Arch == llvm::Triple::x86_64) {
     bool Exists;
     if (Distro == Exherbo &&
         (llvm::sys::fs::exists("/usr/lib32/libc.so", Exists) || !Exists))
       return false;
-
-    return true;
   }
-  if (Arch == llvm::Triple::ppc64)
-    return true;
-  if ((Arch == llvm::Triple::x86 || Arch == llvm::Triple::ppc) && 
-      IsDebianBased(Distro))
-    return true;
-  return false;
+
+  return true;
 }
 
 static LinuxDistro DetectLinuxDistro(llvm::Triple::ArchType Arch) {
   llvm::OwningPtr<llvm::MemoryBuffer> File;
   if (!llvm::MemoryBuffer::getFile("/etc/lsb-release", File)) {
-    llvm::StringRef Data = File.get()->getBuffer();
-    llvm::SmallVector<llvm::StringRef, 8> Lines;
+    StringRef Data = File.get()->getBuffer();
+    SmallVector<StringRef, 8> Lines;
     Data.split(Lines, "\n");
     for (unsigned int i = 0, s = Lines.size(); i < s; ++ i) {
       if (Lines[i] == "DISTRIB_CODENAME=hardy")
@@ -1399,7 +1430,7 @@ static LinuxDistro DetectLinuxDistro(llvm::Triple::ArchType Arch) {
   }
 
   if (!llvm::MemoryBuffer::getFile("/etc/redhat-release", File)) {
-    llvm::StringRef Data = File.get()->getBuffer();
+    StringRef Data = File.get()->getBuffer();
     if (Data.startswith("Fedora release 15"))
       return Fedora15;
     else if (Data.startswith("Fedora release 14"))
@@ -1407,24 +1438,24 @@ static LinuxDistro DetectLinuxDistro(llvm::Triple::ArchType Arch) {
     else if (Data.startswith("Fedora release 13"))
       return Fedora13;
     else if (Data.startswith("Fedora release") &&
-             Data.find("Rawhide") != llvm::StringRef::npos)
+             Data.find("Rawhide") != StringRef::npos)
       return FedoraRawhide;
     else if (Data.startswith("Red Hat Enterprise Linux") &&
-             Data.find("release 6") != llvm::StringRef::npos)
+             Data.find("release 6") != StringRef::npos)
       return RHEL6;
     else if ((Data.startswith("Red Hat Enterprise Linux") ||
 	      Data.startswith("CentOS")) &&
-             Data.find("release 5") != llvm::StringRef::npos)
+             Data.find("release 5") != StringRef::npos)
       return RHEL5;
     else if ((Data.startswith("Red Hat Enterprise Linux") ||
 	      Data.startswith("CentOS")) &&
-             Data.find("release 4") != llvm::StringRef::npos)
+             Data.find("release 4") != StringRef::npos)
       return RHEL4;
     return UnknownDistro;
   }
 
   if (!llvm::MemoryBuffer::getFile("/etc/debian_version", File)) {
-    llvm::StringRef Data = File.get()->getBuffer();
+    StringRef Data = File.get()->getBuffer();
     if (Data[0] == '5')
       return DebianLenny;
     else if (Data.startswith("squeeze/sid"))
@@ -1435,7 +1466,7 @@ static LinuxDistro DetectLinuxDistro(llvm::Triple::ArchType Arch) {
   }
 
   if (!llvm::MemoryBuffer::getFile("/etc/SuSE-release", File)) {
-    llvm::StringRef Data = File.get()->getBuffer();
+    StringRef Data = File.get()->getBuffer();
     if (Data.startswith("openSUSE 11.3"))
       return OpenSuse11_3;
     else if (Data.startswith("openSUSE 11.4"))
@@ -1455,164 +1486,282 @@ static LinuxDistro DetectLinuxDistro(llvm::Triple::ArchType Arch) {
   return UnknownDistro;
 }
 
-static std::string findGCCBaseLibDir(const std::string &GccTriple) {
-  // FIXME: Using CXX_INCLUDE_ROOT is here is a bit of a hack, but
-  // avoids adding yet another option to configure/cmake.
-  // It would probably be cleaner to break it in two variables
-  // CXX_GCC_ROOT with just /foo/bar
-  // CXX_GCC_VER with 4.5.2
-  // Then we would have
-  // CXX_INCLUDE_ROOT = CXX_GCC_ROOT/include/c++/CXX_GCC_VER
-  // and this function would return
-  // CXX_GCC_ROOT/lib/gcc/CXX_INCLUDE_ARCH/CXX_GCC_VER
-  llvm::SmallString<128> CxxIncludeRoot(CXX_INCLUDE_ROOT);
-  if (CxxIncludeRoot != "") {
-    // This is of the form /foo/bar/include/c++/4.5.2/
-    if (CxxIncludeRoot.back() == '/')
-      llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the /
-    llvm::StringRef Version = llvm::sys::path::filename(CxxIncludeRoot);
-    llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the version
-    llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the c++
-    llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the include
-    std::string ret(CxxIncludeRoot.c_str());
-    ret.append("/lib/gcc/");
-    ret.append(CXX_INCLUDE_ARCH);
-    ret.append("/");
-    ret.append(Version);
-    return ret;
-  }
-  static const char* GccVersions[] = {"4.6.1", "4.6.0", "4.6",
-                                      "4.5.2", "4.5.1", "4.5",
-                                      "4.4.5", "4.4.4", "4.4.3", "4.4",
-                                      "4.3.4", "4.3.3", "4.3.2", "4.3",
-                                      "4.2.4", "4.2.3", "4.2.2", "4.2.1",
-                                      "4.2", "4.1.1"};
+/// \brief Trivial helper function to simplify code checking path existence.
+static bool PathExists(StringRef Path) {
   bool Exists;
-  for (unsigned i = 0; i < sizeof(GccVersions)/sizeof(char*); ++i) {
-    std::string Suffix = GccTriple + "/" + GccVersions[i];
-    std::string t1 = "/usr/lib/gcc/" + Suffix;
-    if (!llvm::sys::fs::exists(t1 + "/crtbegin.o", Exists) && Exists)
-      return t1;
-    std::string t2 = "/usr/lib64/gcc/" + Suffix;
-    if (!llvm::sys::fs::exists(t2 + "/crtbegin.o", Exists) && Exists)
-      return t2;
-    std::string t3 = "/usr/lib/" + GccTriple + "/gcc/" + Suffix;
-    if (!llvm::sys::fs::exists(t3 + "/crtbegin.o", Exists) && Exists)
-      return t3;
+  if (!llvm::sys::fs::exists(Path, Exists))
+    return Exists;
+  return false;
+}
+
+namespace {
+/// \brief This is a class to find a viable GCC installation for Clang to use.
+///
+/// This class tries to find a GCC installation on the system, and report
+/// information about it. It starts from the host information provided to the
+/// Driver, and has logic for fuzzing that where appropriate.
+class GCCInstallationDetector {
+  /// \brief Struct to store and manipulate GCC versions.
+  ///
+  /// We rely on assumptions about the form and structure of GCC version
+  /// numbers: they consist of at most three '.'-separated components, and each
+  /// component is a non-negative integer.
+  struct GCCVersion {
+    unsigned Major, Minor, Patch;
+
+    static GCCVersion Parse(StringRef VersionText) {
+      const GCCVersion BadVersion = {0, 0, 0};
+      std::pair<StringRef, StringRef> First = VersionText.split('.');
+      std::pair<StringRef, StringRef> Second = First.second.split('.');
+
+      GCCVersion GoodVersion = {0, 0, 0};
+      if (First.first.getAsInteger(10, GoodVersion.Major))
+        return BadVersion;
+      if (Second.first.getAsInteger(10, GoodVersion.Minor))
+        return BadVersion;
+      // We accept a number, or a string for the patch version, in case there
+      // is a strang suffix, or other mangling: '4.1.x', '4.1.2-rc3'. When it
+      // isn't a number, we just use '0' as the number but accept it.
+      if (Second.first.getAsInteger(10, GoodVersion.Patch))
+        GoodVersion.Patch = 0;
+      return GoodVersion;
+    }
+
+    bool operator<(const GCCVersion &RHS) const {
+      if (Major < RHS.Major) return true;
+      if (Major > RHS.Major) return false;
+      if (Minor < RHS.Minor) return true;
+      if (Minor > RHS.Minor) return false;
+      return Patch < RHS.Patch;
+    }
+    bool operator>(const GCCVersion &RHS) const { return RHS < *this; }
+    bool operator<=(const GCCVersion &RHS) const { return !(*this > RHS); }
+    bool operator>=(const GCCVersion &RHS) const { return !(*this < RHS); }
+  };
+
+  bool IsValid;
+  std::string GccTriple;
+
+  // FIXME: These might be better as path objects.
+  std::string GccInstallPath;
+  std::string GccParentLibPath;
+
+  llvm::SmallString<128> CxxIncludeRoot;
+
+public:
+  /// \brief Construct a GCCInstallationDetector from the driver.
+  ///
+  /// This performs all of the autodetection and sets up the various paths.
+  /// Once constructed, a GCCInstallation is esentially immutable.
+  GCCInstallationDetector(const Driver &D)
+    : IsValid(false),
+      GccTriple(D.DefaultHostTriple),
+      CxxIncludeRoot(CXX_INCLUDE_ROOT) {
+    // FIXME: Using CXX_INCLUDE_ROOT is here is a bit of a hack, but
+    // avoids adding yet another option to configure/cmake.
+    // It would probably be cleaner to break it in two variables
+    // CXX_GCC_ROOT with just /foo/bar
+    // CXX_GCC_VER with 4.5.2
+    // Then we would have
+    // CXX_INCLUDE_ROOT = CXX_GCC_ROOT/include/c++/CXX_GCC_VER
+    // and this function would return
+    // CXX_GCC_ROOT/lib/gcc/CXX_INCLUDE_ARCH/CXX_GCC_VER
+    if (CxxIncludeRoot != "") {
+      // This is of the form /foo/bar/include/c++/4.5.2/
+      if (CxxIncludeRoot.back() == '/')
+        llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the /
+      StringRef Version = llvm::sys::path::filename(CxxIncludeRoot);
+      llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the version
+      llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the c++
+      llvm::sys::path::remove_filename(CxxIncludeRoot); // remove the include
+      GccInstallPath = CxxIncludeRoot.str();
+      GccInstallPath.append("/lib/gcc/");
+      GccInstallPath.append(CXX_INCLUDE_ARCH);
+      GccInstallPath.append("/");
+      GccInstallPath.append(Version);
+      GccParentLibPath = GccInstallPath + "/../../..";
+      IsValid = true;
+      return;
+    }
+
+    llvm::Triple::ArchType HostArch = llvm::Triple(GccTriple).getArch();
+    // The library directories which may contain GCC installations.
+    SmallVector<StringRef, 4> CandidateLibDirs;
+    // The compatible GCC triples for this particular architecture.
+    SmallVector<StringRef, 10> CandidateTriples;
+    CollectLibDirsAndTriples(HostArch, CandidateLibDirs, CandidateTriples);
+
+    // Always include the default host triple as the final fallback if no
+    // specific triple is detected.
+    CandidateTriples.push_back(D.DefaultHostTriple);
+
+    // Compute the set of prefixes for our search.
+    SmallVector<std::string, 8> Prefixes(D.PrefixDirs.begin(),
+                                         D.PrefixDirs.end());
+    Prefixes.push_back(D.SysRoot);
+    Prefixes.push_back(D.SysRoot + "/usr");
+    Prefixes.push_back(D.InstalledDir + "/..");
+
+    // Loop over the various components which exist and select the best GCC
+    // installation available. GCC installs are ranked by version number.
+    GCCVersion BestVersion = {0, 0, 0};
+    for (unsigned i = 0, ie = Prefixes.size(); i < ie; ++i) {
+      if (!PathExists(Prefixes[i]))
+        continue;
+      for (unsigned j = 0, je = CandidateLibDirs.size(); j < je; ++j) {
+        const std::string LibDir = Prefixes[i] + CandidateLibDirs[j].str();
+        if (!PathExists(LibDir))
+          continue;
+        for (unsigned k = 0, ke = CandidateTriples.size(); k < ke; ++k)
+          ScanLibDirForGCCTriple(LibDir, CandidateTriples[k], BestVersion);
+      }
+    }
   }
-  return "";
+
+  /// \brief Check whether we detected a valid GCC install.
+  bool isValid() const { return IsValid; }
+
+  /// \brief Get the GCC triple for the detected install.
+  const std::string &getTriple() const { return GccTriple; }
+
+  /// \brief Get the detected GCC installation path.
+  const std::string &getInstallPath() const { return GccInstallPath; }
+
+  /// \brief Get the detected GCC parent lib path.
+  const std::string &getParentLibPath() const { return GccParentLibPath; }
+
+private:
+  static void CollectLibDirsAndTriples(llvm::Triple::ArchType HostArch,
+                                       SmallVectorImpl<StringRef> &LibDirs,
+                                       SmallVectorImpl<StringRef> &Triples) {
+    if (HostArch == llvm::Triple::arm || HostArch == llvm::Triple::thumb) {
+      static const char *const ARMLibDirs[] = { "/lib" };
+      static const char *const ARMTriples[] = { "arm-linux-gnueabi" };
+      LibDirs.append(ARMLibDirs, ARMLibDirs + llvm::array_lengthof(ARMLibDirs));
+      Triples.append(ARMTriples, ARMTriples + llvm::array_lengthof(ARMTriples));
+    } else if (HostArch == llvm::Triple::x86_64) {
+      static const char *const X86_64LibDirs[] = { "/lib64", "/lib" };
+      static const char *const X86_64Triples[] = {
+        "x86_64-linux-gnu",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-pc-linux-gnu",
+        "x86_64-redhat-linux6E",
+        "x86_64-redhat-linux",
+        "x86_64-suse-linux",
+        "x86_64-manbo-linux-gnu",
+        "x86_64-linux-gnu",
+        "x86_64-slackware-linux"
+      };
+      LibDirs.append(X86_64LibDirs,
+                     X86_64LibDirs + llvm::array_lengthof(X86_64LibDirs));
+      Triples.append(X86_64Triples,
+                     X86_64Triples + llvm::array_lengthof(X86_64Triples));
+    } else if (HostArch == llvm::Triple::x86) {
+      static const char *const X86LibDirs[] = { "/lib32", "/lib" };
+      static const char *const X86Triples[] = {
+        "i686-linux-gnu",
+        "i386-linux-gnu",
+        "i686-pc-linux-gnu",
+        "i486-linux-gnu",
+        "i686-redhat-linux",
+        "i386-redhat-linux",
+        "i586-suse-linux",
+        "i486-slackware-linux"
+      };
+      LibDirs.append(X86LibDirs, X86LibDirs + llvm::array_lengthof(X86LibDirs));
+      Triples.append(X86Triples, X86Triples + llvm::array_lengthof(X86Triples));
+    } else if (HostArch == llvm::Triple::ppc) {
+      static const char *const PPCLibDirs[] = { "/lib32", "/lib" };
+      static const char *const PPCTriples[] = {
+        "powerpc-linux-gnu",
+        "powerpc-unknown-linux-gnu"
+      };
+      LibDirs.append(PPCLibDirs, PPCLibDirs + llvm::array_lengthof(PPCLibDirs));
+      Triples.append(PPCTriples, PPCTriples + llvm::array_lengthof(PPCTriples));
+    } else if (HostArch == llvm::Triple::ppc64) {
+      static const char *const PPC64LibDirs[] = { "/lib64", "/lib" };
+      static const char *const PPC64Triples[] = {
+        "powerpc64-unknown-linux-gnu"
+      };
+      LibDirs.append(PPC64LibDirs,
+                     PPC64LibDirs + llvm::array_lengthof(PPC64LibDirs));
+      Triples.append(PPC64Triples,
+                     PPC64Triples + llvm::array_lengthof(PPC64Triples));
+    }
+  }
+
+  void ScanLibDirForGCCTriple(const std::string &LibDir,
+                              StringRef CandidateTriple,
+                              GCCVersion &BestVersion) {
+    // There are various different suffixes involving the triple we
+    // check for. We also record what is necessary to walk from each back
+    // up to the lib directory.
+    const std::string Suffixes[] = {
+      "/gcc/" + CandidateTriple.str(),
+      "/" + CandidateTriple.str() + "/gcc/" + CandidateTriple.str(),
+
+      // Ubuntu has a strange mis-matched pair of triples that this happens to
+      // match.
+      // FIXME: It may be worthwhile to generalize this and look for a second
+      // triple.
+      "/" + CandidateTriple.str() + "/gcc/i686-linux-gnu"
+    };
+    const std::string InstallSuffixes[] = {
+      "/../../..",
+      "/../../../..",
+      "/../../../.."
+    };
+    // Only look at the final, weird Ubuntu suffix for i386-linux-gnu.
+    const unsigned NumSuffixes = (llvm::array_lengthof(Suffixes) -
+                                  (CandidateTriple != "i386-linux-gnu"));
+    for (unsigned i = 0; i < NumSuffixes; ++i) {
+      StringRef Suffix = Suffixes[i];
+      llvm::error_code EC;
+      for (llvm::sys::fs::directory_iterator LI(LibDir + Suffix, EC), LE;
+           !EC && LI != LE; LI = LI.increment(EC)) {
+        StringRef VersionText = llvm::sys::path::filename(LI->path());
+        GCCVersion CandidateVersion = GCCVersion::Parse(VersionText);
+        static const GCCVersion MinVersion = { 4, 1, 1 };
+        if (CandidateVersion < MinVersion)
+          continue;
+        if (CandidateVersion <= BestVersion)
+          continue;
+        if (!PathExists(LI->path() + "/crtbegin.o"))
+          continue;
+
+        BestVersion = CandidateVersion;
+        GccTriple = CandidateTriple.str();
+        // FIXME: We hack together the directory name here instead of
+        // using LI to ensure stable path separators across Windows and
+        // Linux.
+        GccInstallPath = LibDir + Suffixes[i] + "/" + VersionText.str();
+        GccParentLibPath = GccInstallPath + InstallSuffixes[i];
+        IsValid = true;
+      }
+    }
+  }
+};
+}
+
+static void addPathIfExists(const std::string &Path,
+                            ToolChain::path_list &Paths) {
+  if (PathExists(Path)) Paths.push_back(Path);
 }
 
 Linux::Linux(const HostInfo &Host, const llvm::Triple &Triple)
   : Generic_ELF(Host, Triple) {
   llvm::Triple::ArchType Arch =
     llvm::Triple(getDriver().DefaultHostTriple).getArch();
+  const std::string &SysRoot = getDriver().SysRoot;
+  GCCInstallationDetector GCCInstallation(getDriver());
 
-  std::string Suffix32  = "";
-  if (Arch == llvm::Triple::x86_64)
-    Suffix32 = "/32";
+  // OpenSuse stores the linker with the compiler, add that to the search
+  // path.
+  ToolChain::path_list &PPaths = getProgramPaths();
+  PPaths.push_back(GCCInstallation.getParentLibPath() + "/../" +
+                   GCCInstallation.getTriple() + "/bin");
 
-  std::string Suffix64  = "";
-  if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::ppc)
-    Suffix64 = "/64";
-
-  std::string Lib32 = "lib";
-
-  bool Exists;
-  if (!llvm::sys::fs::exists("/lib32", Exists) && Exists)
-    Lib32 = "lib32";
-
-  std::string Lib64 = "lib";
-  bool Symlink;
-  if (!llvm::sys::fs::exists("/lib64", Exists) && Exists &&
-      (llvm::sys::fs::is_symlink("/lib64", Symlink) || !Symlink))
-    Lib64 = "lib64";
-
-  std::string GccTriple = "";
-  if (Arch == llvm::Triple::arm || Arch == llvm::Triple::thumb) {
-    if (!llvm::sys::fs::exists("/usr/lib/gcc/arm-linux-gnueabi", Exists) &&
-        Exists)
-      GccTriple = "arm-linux-gnueabi";
-  } else if (Arch == llvm::Triple::x86_64) {
-    if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-linux-gnu", Exists) &&
-        Exists)
-      GccTriple = "x86_64-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-unknown-linux-gnu",
-             Exists) && Exists)
-      GccTriple = "x86_64-unknown-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-pc-linux-gnu",
-             Exists) && Exists)
-      GccTriple = "x86_64-pc-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-redhat-linux6E",
-             Exists) && Exists)
-      GccTriple = "x86_64-redhat-linux6E";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-redhat-linux",
-             Exists) && Exists)
-      GccTriple = "x86_64-redhat-linux";
-    else if (!llvm::sys::fs::exists("/usr/lib64/gcc/x86_64-suse-linux",
-             Exists) && Exists)
-      GccTriple = "x86_64-suse-linux";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-manbo-linux-gnu",
-             Exists) && Exists)
-      GccTriple = "x86_64-manbo-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/x86_64-linux-gnu/gcc",
-             Exists) && Exists)
-      GccTriple = "x86_64-linux-gnu";
-  } else if (Arch == llvm::Triple::x86) {
-    if (!llvm::sys::fs::exists("/usr/lib/gcc/i686-linux-gnu", Exists) && Exists)
-      GccTriple = "i686-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i686-pc-linux-gnu", Exists) &&
-             Exists)
-      GccTriple = "i686-pc-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i486-linux-gnu", Exists) &&
-             Exists)
-      GccTriple = "i486-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i686-redhat-linux", Exists) &&
-             Exists)
-      GccTriple = "i686-redhat-linux";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i586-suse-linux", Exists) &&
-             Exists)
-      GccTriple = "i586-suse-linux";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i486-slackware-linux", Exists)
-            && Exists)
-      GccTriple = "i486-slackware-linux";
-  } else if (Arch == llvm::Triple::ppc) {
-    if (!llvm::sys::fs::exists("/usr/lib/powerpc-linux-gnu", Exists) && Exists)
-      GccTriple = "powerpc-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib/gcc/powerpc-unknown-linux-gnu",
-                                    Exists) && Exists)
-      GccTriple = "powerpc-unknown-linux-gnu";
-  } else if (Arch == llvm::Triple::ppc64) {
-    if (!llvm::sys::fs::exists("/usr/lib/gcc/powerpc64-unknown-linux-gnu",
-                               Exists) && Exists)
-      GccTriple = "powerpc64-unknown-linux-gnu";
-    else if (!llvm::sys::fs::exists("/usr/lib64/gcc/"
-                                    "powerpc64-unknown-linux-gnu", Exists) && 
-             Exists)
-      GccTriple = "powerpc64-unknown-linux-gnu";
-  }
-
-  std::string Base = findGCCBaseLibDir(GccTriple);
-  path_list &Paths = getFilePaths();
-  bool Is32Bits = (getArch() == llvm::Triple::x86 || 
-                   getArch() == llvm::Triple::ppc);
-
-  std::string Suffix;
-  std::string Lib;
-
-  if (Is32Bits) {
-    Suffix = Suffix32;
-    Lib = Lib32;
-  } else {
-    Suffix = Suffix64;
-    Lib = Lib64;
-  }
-
-  llvm::sys::Path LinkerPath(Base + "/../../../../" + GccTriple + "/bin/ld");
-  if (!llvm::sys::fs::exists(LinkerPath.str(), Exists) && Exists)
-    Linker = LinkerPath.str();
-  else
-    Linker = GetProgramPath("ld");
+  Linker = GetProgramPath("ld");
 
   LinuxDistro Distro = DetectLinuxDistro(Arch);
 
@@ -1646,29 +1795,63 @@ Linux::Linux(const HostInfo &Host, const llvm::Triple &Triple)
   if (IsOpenSuse(Distro))
     ExtraOpts.push_back("--enable-new-dtags");
 
-  if (Distro == ArchLinux)
-    Lib = "lib";
+  // The selection of paths to try here is designed to match the patterns which
+  // the GCC driver itself uses, as this is part of the GCC-compatible driver.
+  // This was determined by running GCC in a fake filesystem, creating all
+  // possible permutations of these directories, and seeing which ones it added
+  // to the link paths.
+  path_list &Paths = getFilePaths();
+  const bool Is32Bits = (getArch() == llvm::Triple::x86 ||
+                         getArch() == llvm::Triple::ppc);
 
-  Paths.push_back(Base + Suffix);
+  const std::string Suffix32 = Arch == llvm::Triple::x86_64 ? "/32" : "";
+  const std::string Suffix64 = Arch == llvm::Triple::x86_64 ? "" : "/64";
+  const std::string Suffix = Is32Bits ? Suffix32 : Suffix64;
+  const std::string Multilib = Is32Bits ? "lib32" : "lib64";
+
+  // FIXME: Because we add paths only when they exist on the system, I think we
+  // should remove the concept of 'HasMultilib'. It's more likely to break the
+  // behavior than to preserve any useful invariant on the system.
   if (HasMultilib(Arch, Distro)) {
-    if (IsOpenSuse(Distro) && Is32Bits)
-      Paths.push_back(Base + "/../../../../" + GccTriple + "/lib/../lib");
-    Paths.push_back(Base + "/../../../../" + Lib);
+    // Add the multilib suffixed paths.
+    if (GCCInstallation.isValid()) {
+      const std::string &LibPath = GCCInstallation.getParentLibPath();
+      const std::string &GccTriple = GCCInstallation.getTriple();
+      // FIXME: This OpenSuse-specific path shouldn't be needed any more, but
+      // I don't want to remove it without finding someone to test.
+      if (IsOpenSuse(Distro) && Is32Bits)
+        Paths.push_back(LibPath + "/../" + GccTriple + "/lib/../lib");
+
+      addPathIfExists(GCCInstallation.getInstallPath() + Suffix, Paths);
+      addPathIfExists(LibPath + "/../" + GccTriple + "/lib/../" + Multilib,
+                      Paths);
+      addPathIfExists(LibPath + "/../" + Multilib, Paths);
+    }
+    addPathIfExists(SysRoot + "/lib/../" + Multilib, Paths);
+    addPathIfExists(SysRoot + "/usr/lib/../" + Multilib, Paths);
+
+    // Try walking via the GCC triple path in case of multiarch GCC
+    // installations with strange symlinks.
+    if (GCCInstallation.isValid())
+      addPathIfExists(SysRoot + "/usr/lib/" + GCCInstallation.getTriple() +
+                      "/../../" + Multilib, Paths);
   }
 
-  // FIXME: This is in here to find crt1.o. It is provided by libc, and
-  // libc (like gcc), can be installed in any directory. Once we are
-  // fetching this from a config file, we should have a libc prefix.
-  Paths.push_back("/lib/../" + Lib);
-  Paths.push_back("/usr/lib/../" + Lib);
+  // Add the non-multilib suffixed paths (if potentially different).
+  if (GCCInstallation.isValid()) {
+    const std::string &LibPath = GCCInstallation.getParentLibPath();
+    const std::string &GccTriple = GCCInstallation.getTriple();
+    if (!Suffix.empty() || !HasMultilib(Arch, Distro))
+      addPathIfExists(GCCInstallation.getInstallPath(), Paths);
+    addPathIfExists(LibPath + "/../" + GccTriple + "/lib", Paths);
+    addPathIfExists(LibPath, Paths);
+  }
+  addPathIfExists(SysRoot + "/lib", Paths);
+  addPathIfExists(SysRoot + "/usr/lib", Paths);
 
-  if (!Suffix.empty())
-    Paths.push_back(Base);
-  if (IsOpenSuse(Distro))
-    Paths.push_back(Base + "/../../../../" + GccTriple + "/lib");
-  Paths.push_back(Base + "/../../..");
-  if (Arch == getArch() && IsUbuntu(Distro))
-    Paths.push_back("/usr/lib/" + GccTriple);
+  // Add a multiarch lib directory whenever it exists and is plausible.
+  if (GCCInstallation.isValid() && Arch == getArch())
+    addPathIfExists(SysRoot + "/usr/lib/" + GCCInstallation.getTriple(), Paths);
 }
 
 bool Linux::HasNativeLLVMSupport() const {
@@ -1756,6 +1939,10 @@ Tool &Windows::SelectTool(const Compilation &C, const JobAction &JA,
   else
     Key = JA.getKind();
 
+  bool UseIntegratedAs = C.getArgs().hasFlag(options::OPT_integrated_as,
+                                             options::OPT_no_integrated_as,
+                                             IsIntegratedAssemblerDefault());
+
   Tool *&T = Tools[Key];
   if (!T) {
     switch (Key) {
@@ -1763,14 +1950,19 @@ Tool &Windows::SelectTool(const Compilation &C, const JobAction &JA,
     case Action::BindArchClass:
     case Action::LipoJobClass:
     case Action::DsymutilJobClass:
-      assert(0 && "Invalid tool kind.");
+    case Action::VerifyJobClass:
+      llvm_unreachable("Invalid tool kind.");
     case Action::PreprocessJobClass:
     case Action::PrecompileJobClass:
     case Action::AnalyzeJobClass:
     case Action::CompileJobClass:
       T = new tools::Clang(*this); break;
     case Action::AssembleJobClass:
-      T = new tools::ClangAs(*this); break;
+      if (!UseIntegratedAs && getTriple().getEnvironment() == llvm::Triple::MachO)
+        T = new tools::darwin::Assemble(*this);
+      else
+        T = new tools::ClangAs(*this);
+      break;
     case Action::LinkJobClass:
       T = new tools::visualstudio::Link(*this); break;
     }
diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains.h
index d68016b3a240..dfcb253acd88 100644
--- a/contrib/llvm/tools/clang/lib/Driver/ToolChains.h
+++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains.h
@@ -66,6 +66,12 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
     ARCSimulator_NoARCRuntime
   } ARCRuntimeForSimulator;
 
+  mutable enum {
+    LibCXXSimulator_None,
+    LibCXXSimulator_NotAvailable,
+    LibCXXSimulator_Available
+  } LibCXXForSimulator;
+
 private:
   /// Whether we are targeting iPhoneOS target.
   mutable bool TargetIsIPhoneOS;
@@ -89,7 +95,8 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
   Darwin(const HostInfo &Host, const llvm::Triple& Triple);
   ~Darwin();
 
-  std::string ComputeEffectiveClangTriple(const ArgList &Args) const;
+  std::string ComputeEffectiveClangTriple(const ArgList &Args,
+                                          types::ID InputType) const;
 
   /// @name Darwin Specific Toolchain API
   /// {
@@ -139,7 +146,7 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
   /// getDarwinArchName - Get the "Darwin" arch name for a particular compiler
   /// invocation. For example, Darwin treats different ARM variations as
   /// distinct architectures.
-  llvm::StringRef getDarwinArchName(const ArgList &Args) const;
+  StringRef getDarwinArchName(const ArgList &Args) const;
 
   static bool isVersionLT(unsigned (&A)[3], unsigned (&B)[3]) {
     for (unsigned i=0; i < 3; ++i) {
@@ -187,6 +194,7 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
   virtual bool HasNativeLLVMSupport() const;
 
   virtual void configureObjCRuntime(ObjCRuntime &runtime) const;
+  virtual bool hasBlocksRuntime() const;
 
   virtual DerivedArgList *TranslateArgs(const DerivedArgList &Args,
                                         const char *BoundArch) const;
@@ -237,9 +245,12 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
     return !(!isTargetIPhoneOS() && isMacosxVersionLT(10, 6));
   }
   virtual bool IsUnwindTablesDefault() const;
-  virtual unsigned GetDefaultStackProtectorLevel() const {
-    // Stack protectors default to on for 10.6 and beyond.
-    return !isTargetIPhoneOS() && !isMacosxVersionLT(10, 6);
+  virtual unsigned GetDefaultStackProtectorLevel(bool KernelOrKext) const {
+    // Stack protectors default to on for user code on 10.5,
+    // and for everything in 10.6 and beyond
+    return !isTargetIPhoneOS() &&
+      (!isMacosxVersionLT(10, 6) ||
+         (!isMacosxVersionLT(10, 5) && !KernelOrKext));
   }
   virtual const char *GetDefaultRelocationModel() const;
   virtual const char *GetForcedPicModel() const;
@@ -257,6 +268,9 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
 
 /// DarwinClang - The Darwin toolchain used by Clang.
 class LLVM_LIBRARY_VISIBILITY DarwinClang : public Darwin {
+private:
+  void AddGCCLibexecPath(unsigned darwinVersion);
+
 public:
   DarwinClang(const HostInfo &Host, const llvm::Triple& Triple);
 
@@ -288,7 +302,8 @@ class LLVM_LIBRARY_VISIBILITY Darwin_Generic_GCC : public Generic_GCC {
   Darwin_Generic_GCC(const HostInfo &Host, const llvm::Triple& Triple)
     : Generic_GCC(Host, Triple) {}
 
-  std::string ComputeEffectiveClangTriple(const ArgList &Args) const;
+  std::string ComputeEffectiveClangTriple(const ArgList &Args,
+                                          types::ID InputType) const;
 
   virtual const char *GetDefaultRelocationModel() const { return "pic"; }
 };
diff --git a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
index 64ad1465fbd3..94849a6601eb 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
@@ -45,25 +45,14 @@
 
 using namespace clang::driver;
 using namespace clang::driver::tools;
-
-/// FindTargetProgramPath - Return path of the target specific version of
-/// ProgName.  If it doesn't exist, return path of ProgName itself.
-static std::string FindTargetProgramPath(const ToolChain &TheToolChain,
-                                         const std::string TripleString,
-                                         const char *ProgName) {
-  std::string Executable(TripleString + "-" + ProgName);
-  std::string Path(TheToolChain.GetProgramPath(Executable.c_str()));
-  if (Path != Executable)
-    return Path;
-  return TheToolChain.GetProgramPath(ProgName);
-}
+using namespace clang;
 
 /// CheckPreprocessingOptions - Perform some validation of preprocessing
 /// arguments that is shared with gcc.
 static void CheckPreprocessingOptions(const Driver &D, const ArgList &Args) {
   if (Arg *A = Args.getLastArg(options::OPT_C, options::OPT_CC))
     if (!Args.hasArg(options::OPT_E) && !D.CCCIsCPP)
-      D.Diag(clang::diag::err_drv_argument_only_allowed_with)
+      D.Diag(diag::err_drv_argument_only_allowed_with)
         << A->getAsString(Args) << "-E";
 }
 
@@ -74,14 +63,14 @@ static void CheckCodeGenerationOptions(const Driver &D, const ArgList &Args) {
   if (Args.hasArg(options::OPT_static))
     if (const Arg *A = Args.getLastArg(options::OPT_dynamic,
                                        options::OPT_mdynamic_no_pic))
-      D.Diag(clang::diag::err_drv_argument_not_allowed_with)
+      D.Diag(diag::err_drv_argument_not_allowed_with)
         << A->getAsString(Args) << "-static";
 }
 
 // Quote target names for inclusion in GNU Make dependency files.
 // Only the characters '$', '#', ' ', '\t' are quoted.
-static void QuoteTarget(llvm::StringRef Target,
-                        llvm::SmallVectorImpl<char> &Res) {
+static void QuoteTarget(StringRef Target,
+                        SmallVectorImpl<char> &Res) {
   for (unsigned i = 0, e = Target.size(); i != e; ++i) {
     switch (Target[i]) {
     case ' ':
@@ -126,7 +115,7 @@ static void AddLinkerInputs(const ToolChain &TC,
           II.getType() == types::TY_LTO_IR ||
           II.getType() == types::TY_LLVM_BC ||
           II.getType() == types::TY_LTO_BC)
-        D.Diag(clang::diag::err_drv_no_linker_llvm_support)
+        D.Diag(diag::err_drv_no_linker_llvm_support)
           << TC.getTripleString();
     }
 
@@ -168,20 +157,52 @@ static void addProfileRT(const ToolChain &TC, const ArgList &Args,
   // the link line. We cannot do the same thing because unlike gcov there is a
   // libprofile_rt.so. We used to use the -l:libprofile_rt.a syntax, but that is
   // not supported by old linkers.
-  llvm::Twine ProfileRT =
-    llvm::Twine(TC.getDriver().Dir) + "/../lib/" + "libprofile_rt.a";
+  Twine ProfileRT =
+    Twine(TC.getDriver().Dir) + "/../lib/" + "libprofile_rt.a";
 
-  if (Triple.getOS() == llvm::Triple::Darwin) {
+  if (Triple.isOSDarwin()) {
     // On Darwin, if the static library doesn't exist try the dylib.
     bool Exists;
     if (llvm::sys::fs::exists(ProfileRT.str(), Exists) || !Exists)
       ProfileRT =
-        llvm::Twine(TC.getDriver().Dir) + "/../lib/" + "libprofile_rt.dylib";
+        Twine(TC.getDriver().Dir) + "/../lib/" + "libprofile_rt.dylib";
   }
 
   CmdArgs.push_back(Args.MakeArgString(ProfileRT));
 }
 
+static void AddIncludeDirectoryList(const ArgList &Args,
+                                    ArgStringList &CmdArgs,
+                                    const char *ArgName,
+                                    const char *DirList) {
+  if (!DirList)
+    return; // Nothing to do.
+
+  StringRef Dirs(DirList);
+  if (Dirs.empty()) // Empty string should not add '.'.
+    return;
+
+  StringRef::size_type Delim;
+  while ((Delim = Dirs.find(llvm::sys::PathSeparator)) != StringRef::npos) {
+    if (Delim == 0) { // Leading colon.
+      CmdArgs.push_back(ArgName);
+      CmdArgs.push_back(".");
+    } else {
+      CmdArgs.push_back(ArgName);
+      CmdArgs.push_back(Args.MakeArgString(Dirs.substr(0, Delim)));
+    }
+    Dirs = Dirs.substr(Delim + 1);
+  }
+
+  if (Dirs.empty()) { // Trailing colon.
+    CmdArgs.push_back(ArgName);
+    CmdArgs.push_back(".");
+  } else { // Add the last path.
+    CmdArgs.push_back(ArgName);
+    CmdArgs.push_back(Args.MakeArgString(Dirs));
+  }
+}
+
 void Clang::AddPreprocessingOptions(const Driver &D,
                                     const ArgList &Args,
                                     ArgStringList &CmdArgs,
@@ -245,7 +266,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
   if (Args.hasArg(options::OPT_MG)) {
     if (!A || A->getOption().matches(options::OPT_MD) ||
               A->getOption().matches(options::OPT_MMD))
-      D.Diag(clang::diag::err_drv_mg_requires_m_or_mm);
+      D.Diag(diag::err_drv_mg_requires_m_or_mm);
     CmdArgs.push_back("-MG");
   }
 
@@ -328,7 +349,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
           continue;
         } else {
           // Ignore the PCH if not first on command line and emit warning.
-          D.Diag(clang::diag::warn_drv_pch_not_first_include)
+          D.Diag(diag::warn_drv_pch_not_first_include)
               << P.str() << A->getAsString(Args);
         }
       }
@@ -340,14 +361,15 @@ void Clang::AddPreprocessingOptions(const Driver &D,
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_D, options::OPT_U);
-  Args.AddAllArgs(CmdArgs, options::OPT_I_Group, options::OPT_F);
+  Args.AddAllArgs(CmdArgs, options::OPT_I_Group, options::OPT_F,
+                  options::OPT_index_header_map);
 
   // Add C++ include arguments, if needed.
   types::ID InputType = Inputs[0].getType();
   if (types::isCXX(InputType)) {
     bool ObjCXXAutoRefCount
       = types::isObjC(InputType) && isObjCAutoRefCount(Args);
-    getToolChain().AddClangCXXStdlibIncludeArgs(Args, CmdArgs, 
+    getToolChain().AddClangCXXStdlibIncludeArgs(Args, CmdArgs,
                                                 ObjCXXAutoRefCount);
     Args.AddAllArgs(CmdArgs, options::OPT_stdlib_EQ);
   }
@@ -363,7 +385,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
 
   // -I- is a deprecated GCC feature, reject it.
   if (Arg *A = Args.getLastArg(options::OPT_I_))
-    D.Diag(clang::diag::err_drv_I_dash_not_supported) << A->getAsString(Args);
+    D.Diag(diag::err_drv_I_dash_not_supported) << A->getAsString(Args);
 
   // If we have a --sysroot, and don't have an explicit -isysroot flag, add an
   // -isysroot to the CC1 invocation.
@@ -373,6 +395,39 @@ void Clang::AddPreprocessingOptions(const Driver &D,
       CmdArgs.push_back(A->getValue(Args));
     }
   }
+  
+  // If a module path was provided, pass it along. Otherwise, use a temporary
+  // directory.
+  if (Arg *A = Args.getLastArg(options::OPT_fmodule_cache_path)) {
+    A->claim();
+    A->render(Args, CmdArgs);
+  } else {
+    llvm::SmallString<128> DefaultModuleCache;
+    llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/false, 
+                                           DefaultModuleCache);
+    llvm::sys::path::append(DefaultModuleCache, "clang-module-cache");
+    CmdArgs.push_back("-fmodule-cache-path");
+    CmdArgs.push_back(Args.MakeArgString(DefaultModuleCache));
+  }
+  
+  Args.AddAllArgs(CmdArgs, options::OPT_fauto_module_import);
+
+  // Parse additional include paths from environment variables.
+  // CPATH - included following the user specified includes (but prior to
+  // builtin and standard includes).
+  AddIncludeDirectoryList(Args, CmdArgs, "-I", ::getenv("CPATH"));
+  // C_INCLUDE_PATH - system includes enabled when compiling C.
+  AddIncludeDirectoryList(Args, CmdArgs, "-c-isystem",
+                          ::getenv("C_INCLUDE_PATH"));
+  // CPLUS_INCLUDE_PATH - system includes enabled when compiling C++.
+  AddIncludeDirectoryList(Args, CmdArgs, "-cxx-isystem",
+                          ::getenv("CPLUS_INCLUDE_PATH"));
+  // OBJC_INCLUDE_PATH - system includes enabled when compiling ObjC.
+  AddIncludeDirectoryList(Args, CmdArgs, "-objc-isystem",
+                          ::getenv("OBJC_INCLUDE_PATH"));
+  // OBJCPLUS_INCLUDE_PATH - system includes enabled when compiling ObjC++.
+  AddIncludeDirectoryList(Args, CmdArgs, "-objcxx-isystem",
+                          ::getenv("OBJCPLUS_INCLUDE_PATH"));
 }
 
 /// getARMTargetCPU - Get the (LLVM) name of the ARM cpu we are targeting.
@@ -386,7 +441,7 @@ static const char *getARMTargetCPU(const ArgList &Args,
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
     return A->getValue(Args);
 
-  llvm::StringRef MArch;
+  StringRef MArch;
   if (Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
     // Otherwise, if we have -march= choose the base CPU for that arch.
     MArch = A->getValue(Args);
@@ -395,45 +450,27 @@ static const char *getARMTargetCPU(const ArgList &Args,
     MArch = Triple.getArchName();
   }
 
-  if (MArch == "armv2" || MArch == "armv2a")
-    return "arm2";
-  if (MArch == "armv3")
-    return "arm6";
-  if (MArch == "armv3m")
-    return "arm7m";
-  if (MArch == "armv4" || MArch == "armv4t")
-    return "arm7tdmi";
-  if (MArch == "armv5" || MArch == "armv5t")
-    return "arm10tdmi";
-  if (MArch == "armv5e" || MArch == "armv5te")
-    return "arm1026ejs";
-  if (MArch == "armv5tej")
-    return "arm926ej-s";
-  if (MArch == "armv6" || MArch == "armv6k")
-    return "arm1136jf-s";
-  if (MArch == "armv6j")
-    return "arm1136j-s";
-  if (MArch == "armv6z" || MArch == "armv6zk")
-    return "arm1176jzf-s";
-  if (MArch == "armv6t2")
-    return "arm1156t2-s";
-  if (MArch == "armv7" || MArch == "armv7a" || MArch == "armv7-a")
-    return "cortex-a8";
-  if (MArch == "armv7r" || MArch == "armv7-r")
-    return "cortex-r4";
-  if (MArch == "armv7m" || MArch == "armv7-m")
-    return "cortex-m3";
-  if (MArch == "ep9312")
-    return "ep9312";
-  if (MArch == "iwmmxt")
-    return "iwmmxt";
-  if (MArch == "xscale")
-    return "xscale";
-  if (MArch == "armv6m" || MArch == "armv6-m")
-    return "cortex-m0";
-
-  // If all else failed, return the most base CPU LLVM supports.
-  return "arm7tdmi";
+  return llvm::StringSwitch<const char *>(MArch)
+    .Cases("armv2", "armv2a","arm2")
+    .Case("armv3", "arm6")
+    .Case("armv3m", "arm7m")
+    .Cases("armv4", "armv4t", "arm7tdmi")
+    .Cases("armv5", "armv5t", "arm10tdmi")
+    .Cases("armv5e", "armv5te", "arm1026ejs")
+    .Case("armv5tej", "arm926ej-s")
+    .Cases("armv6", "armv6k", "arm1136jf-s")
+    .Case("armv6j", "arm1136j-s")
+    .Cases("armv6z", "armv6zk", "arm1176jzf-s")
+    .Case("armv6t2", "arm1156t2-s")
+    .Cases("armv7", "armv7a", "armv7-a", "cortex-a8")
+    .Cases("armv7r", "armv7-r", "cortex-r4")
+    .Cases("armv7m", "armv7-m", "cortex-m3")
+    .Case("ep9312", "ep9312")
+    .Case("iwmmxt", "iwmmxt")
+    .Case("xscale", "xscale")
+    .Cases("armv6m", "armv6-m", "cortex-m0")
+    // If all else failed, return the most base CPU LLVM supports.
+    .Default("arm7tdmi");
 }
 
 /// getLLVMArchSuffixForARM - Get the LLVM arch name to use for a particular
@@ -441,33 +478,21 @@ static const char *getARMTargetCPU(const ArgList &Args,
 //
 // FIXME: This is redundant with -mcpu, why does LLVM use this.
 // FIXME: tblgen this, or kill it!
-static const char *getLLVMArchSuffixForARM(llvm::StringRef CPU) {
-  if (CPU == "arm7tdmi" || CPU == "arm7tdmi-s" || CPU == "arm710t" ||
-      CPU == "arm720t" || CPU == "arm9" || CPU == "arm9tdmi" ||
-      CPU == "arm920" || CPU == "arm920t" || CPU == "arm922t" ||
-      CPU == "arm940t" || CPU == "ep9312")
-    return "v4t";
-
-  if (CPU == "arm10tdmi" || CPU == "arm1020t")
-    return "v5";
-
-  if (CPU == "arm9e" || CPU == "arm926ej-s" || CPU == "arm946e-s" ||
-      CPU == "arm966e-s" || CPU == "arm968e-s" || CPU == "arm10e" ||
-      CPU == "arm1020e" || CPU == "arm1022e" || CPU == "xscale" ||
-      CPU == "iwmmxt")
-    return "v5e";
-
-  if (CPU == "arm1136j-s" || CPU == "arm1136jf-s" || CPU == "arm1176jz-s" ||
-      CPU == "arm1176jzf-s" || CPU == "mpcorenovfp" || CPU == "mpcore")
-    return "v6";
-
-  if (CPU == "arm1156t2-s" || CPU == "arm1156t2f-s")
-    return "v6t2";
-
-  if (CPU == "cortex-a8" || CPU == "cortex-a9")
-    return "v7";
-
-  return "";
+static const char *getLLVMArchSuffixForARM(StringRef CPU) {
+  return llvm::StringSwitch<const char *>(CPU)
+    .Cases("arm7tdmi", "arm7tdmi-s", "arm710t", "v4t")
+    .Cases("arm720t", "arm9", "arm9tdmi", "v4t")
+    .Cases("arm920", "arm920t", "arm922t", "v4t")
+    .Cases("arm940t", "ep9312","v4t")
+    .Cases("arm10tdmi",  "arm1020t", "v5")
+    .Cases("arm9e",  "arm926ej-s",  "arm946e-s", "v5e")
+    .Cases("arm966e-s",  "arm968e-s",  "arm10e", "v5e")
+    .Cases("arm1020e",  "arm1022e",  "xscale", "iwmmxt", "v5e")
+    .Cases("arm1136j-s",  "arm1136jf-s",  "arm1176jz-s", "v6")
+    .Cases("arm1176jzf-s",  "mpcorenovfp",  "mpcore", "v6")
+    .Cases("arm1156t2-s",  "arm1156t2f-s", "v6t2")
+    .Cases("cortex-a8", "cortex-a9", "v7")
+    .Default("");
 }
 
 // FIXME: Move to target hook.
@@ -479,7 +504,7 @@ static bool isSignedCharDefault(const llvm::Triple &Triple) {
   case llvm::Triple::arm:
   case llvm::Triple::ppc:
   case llvm::Triple::ppc64:
-    if (Triple.getOS() == llvm::Triple::Darwin)
+    if (Triple.isOSDarwin())
       return true;
     return false;
 
@@ -494,12 +519,6 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
   const Driver &D = getToolChain().getDriver();
   llvm::Triple Triple = getToolChain().getTriple();
 
-  // Disable movt generation, if requested.
-#ifdef DISABLE_ARM_DARWIN_USE_MOVT
-  CmdArgs.push_back("-backend-option");
-  CmdArgs.push_back("-arm-darwin-use-movt=0");
-#endif
-
   // Select the ABI to use.
   //
   // FIXME: Support -meabi.
@@ -528,7 +547,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
 
   // Select the float ABI as determined by -msoft-float, -mhard-float, and
   // -mfloat-abi=.
-  llvm::StringRef FloatABI;
+  StringRef FloatABI;
   if (Arg *A = Args.getLastArg(options::OPT_msoft_float,
                                options::OPT_mhard_float,
                                options::OPT_mfloat_abi_EQ)) {
@@ -539,7 +558,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
     else {
       FloatABI = A->getValue(Args);
       if (FloatABI != "soft" && FloatABI != "softfp" && FloatABI != "hard") {
-        D.Diag(clang::diag::err_drv_invalid_mfloat_abi)
+        D.Diag(diag::err_drv_invalid_mfloat_abi)
           << A->getAsString(Args);
         FloatABI = "soft";
       }
@@ -550,11 +569,13 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
   if (FloatABI.empty()) {
     const llvm::Triple &Triple = getToolChain().getTriple();
     switch (Triple.getOS()) {
-    case llvm::Triple::Darwin: {
+    case llvm::Triple::Darwin:
+    case llvm::Triple::MacOSX:
+    case llvm::Triple::IOS: {
       // Darwin defaults to "softfp" for v6 and v7.
       //
       // FIXME: Factor out an ARM class so we can cache the arch somewhere.
-      llvm::StringRef ArchName =
+      StringRef ArchName =
         getLLVMArchSuffixForARM(getARMTargetCPU(Args, Triple));
       if (ArchName.startswith("v6") || ArchName.startswith("v7"))
         FloatABI = "softfp";
@@ -583,7 +604,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
       default:
         // Assume "soft", but warn the user we are guessing.
         FloatABI = "soft";
-        D.Diag(clang::diag::warn_drv_assuming_mfloat_abi_is) << "soft";
+        D.Diag(diag::warn_drv_assuming_mfloat_abi_is) << "soft";
         break;
       }
     }
@@ -630,7 +651,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
   // FIXME: Centralize feature selection, defaulting shouldn't be also in the
   // frontend target.
   if (const Arg *A = Args.getLastArg(options::OPT_mfpu_EQ)) {
-    llvm::StringRef FPU = A->getValue(Args);
+    StringRef FPU = A->getValue(Args);
 
     // Set the target features based on the FPU.
     if (FPU == "fpa" || FPU == "fpe2" || FPU == "fpe3" || FPU == "maverick") {
@@ -651,7 +672,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
       CmdArgs.push_back("-target-feature");
       CmdArgs.push_back("+neon");
     } else
-      D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
+      D.Diag(diag::err_drv_clang_unsupported) << A->getAsString(Args);
   }
 
   // Setting -msoft-float effectively disables NEON because of the GCC
@@ -670,42 +691,84 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
     CmdArgs.push_back("-arm-strict-align");
 
     // The kext linker doesn't know how to deal with movw/movt.
-#ifndef DISABLE_ARM_DARWIN_USE_MOVT
     CmdArgs.push_back("-backend-option");
     CmdArgs.push_back("-arm-darwin-use-movt=0");
-#endif
   }
+
+  // Setting -mno-global-merge disables the codegen global merge pass. Setting 
+  // -mglobal-merge has no effect as the pass is enabled by default.
+  if (Arg *A = Args.getLastArg(options::OPT_mglobal_merge,
+                               options::OPT_mno_global_merge)) {
+    if (A->getOption().matches(options::OPT_mno_global_merge))
+      CmdArgs.push_back("-mno-global-merge");
+  }
+}
+
+// Get default architecture.
+static const char* getMipsArchFromCPU(StringRef CPUName) {
+  if (CPUName == "mips32r1" || CPUName == "4ke")
+    return "mips";
+
+  assert((CPUName == "mips64r1" || CPUName == "mips64r2") &&
+         "Unexpected cpu name.");
+
+  return "mips64";
+}
+
+// Get default target cpu.
+static const char* getMipsCPUFromArch(StringRef ArchName, const Driver &D) {
+  if (ArchName == "mips" || ArchName == "mipsel")
+    return "mips32r1";
+  else if (ArchName == "mips64" || ArchName == "mips64el")
+    return "mips64r1";
+  else
+    D.Diag(diag::err_drv_invalid_arch_name) << ArchName;
+
+  return 0;
+}
+
+// Get default ABI.
+static const char* getMipsABIFromArch(StringRef ArchName) {
+    if (ArchName == "mips" || ArchName == "mipsel")
+      return "o32";
+    
+    assert((ArchName == "mips64" || ArchName == "mips64el") &&
+           "Unexpected arch name.");
+    return "n64";
 }
 
 void Clang::AddMIPSTargetArgs(const ArgList &Args,
                              ArgStringList &CmdArgs) const {
   const Driver &D = getToolChain().getDriver();
 
+  StringRef ArchName;
+  const char *CPUName;
+
+  // Set target cpu and architecture.
+  if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
+    CPUName = A->getValue(Args);
+    ArchName = getMipsArchFromCPU(CPUName);
+  }
+  else {
+    ArchName = Args.MakeArgString(getToolChain().getArchName());
+    CPUName = getMipsCPUFromArch(ArchName, D);
+  }
+
+  CmdArgs.push_back("-target-cpu");
+  CmdArgs.push_back(CPUName);
+ 
   // Select the ABI to use.
   const char *ABIName = 0;
-  if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ))
     ABIName = A->getValue(Args);
-  } else {
-    ABIName = "o32";
-  }
+  else 
+    ABIName = getMipsABIFromArch(ArchName);
 
   CmdArgs.push_back("-target-abi");
   CmdArgs.push_back(ABIName);
 
-  if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
-    llvm::StringRef MArch = A->getValue(Args);
-    CmdArgs.push_back("-target-cpu");
-
-    if ((MArch == "r2000") || (MArch == "r3000"))
-      CmdArgs.push_back("mips1");
-    else if (MArch == "r6000")
-      CmdArgs.push_back("mips2");
-    else
-      CmdArgs.push_back(Args.MakeArgString(MArch));
-  }
-
   // Select the float ABI as determined by -msoft-float, -mhard-float, and
-  llvm::StringRef FloatABI;
+  StringRef FloatABI;
   if (Arg *A = Args.getLastArg(options::OPT_msoft_float,
                                options::OPT_mhard_float)) {
     if (A->getOption().matches(options::OPT_msoft_float))
@@ -718,7 +781,7 @@ void Clang::AddMIPSTargetArgs(const ArgList &Args,
   if (FloatABI.empty()) {
     // Assume "soft", but warn the user we are guessing.
     FloatABI = "soft";
-    D.Diag(clang::diag::warn_drv_assuming_mfloat_abi_is) << "soft";
+    D.Diag(diag::warn_drv_assuming_mfloat_abi_is) << "soft";
   }
 
   if (FloatABI == "soft") {
@@ -737,13 +800,13 @@ void Clang::AddSparcTargetArgs(const ArgList &Args,
   const Driver &D = getToolChain().getDriver();
 
   if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
-    llvm::StringRef MArch = A->getValue(Args);
+    StringRef MArch = A->getValue(Args);
     CmdArgs.push_back("-target-cpu");
     CmdArgs.push_back(MArch.str().c_str());
   }
 
   // Select the float ABI as determined by -msoft-float, -mhard-float, and
-  llvm::StringRef FloatABI;
+  StringRef FloatABI;
   if (Arg *A = Args.getLastArg(options::OPT_msoft_float,
                                options::OPT_mhard_float)) {
     if (A->getOption().matches(options::OPT_msoft_float))
@@ -758,7 +821,7 @@ void Clang::AddSparcTargetArgs(const ArgList &Args,
     default:
       // Assume "soft", but warn the user we are guessing.
       FloatABI = "soft";
-      D.Diag(clang::diag::warn_drv_assuming_mfloat_abi_is) << "soft";
+      D.Diag(diag::warn_drv_assuming_mfloat_abi_is) << "soft";
       break;
     }
   }
@@ -792,7 +855,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
 
   const char *CPUName = 0;
   if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
-    if (llvm::StringRef(A->getValue(Args)) == "native") {
+    if (StringRef(A->getValue(Args)) == "native") {
       // FIXME: Reject attempts to use -march=native unless the target matches
       // the host.
       //
@@ -808,7 +871,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
   // Select the default CPU if none was given (or detection failed).
   if (!CPUName) {
     // FIXME: Need target hooks.
-    if (getToolChain().getOS().startswith("darwin")) {
+    if (getToolChain().getTriple().isOSDarwin()) {
       if (getToolChain().getArch() == llvm::Triple::x86_64)
         CPUName = "core2";
       else if (getToolChain().getArch() == llvm::Triple::x86)
@@ -856,7 +919,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
   std::vector<const char*> Features;
   for (arg_iterator it = Args.filtered_begin(options::OPT_m_x86_Features_Group),
          ie = Args.filtered_end(); it != ie; ++it) {
-    llvm::StringRef Name = (*it)->getOption().getName();
+    StringRef Name = (*it)->getOption().getName();
     (*it)->claim();
 
     // Skip over "-m".
@@ -881,7 +944,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
   }
 }
 
-static bool 
+static bool
 shouldUseExceptionTablesForObjCExceptions(unsigned objcABIVersion,
                                           const llvm::Triple &Triple) {
   // We use the zero-cost exception tables for Objective-C if the non-fragile
@@ -891,12 +954,12 @@ shouldUseExceptionTablesForObjCExceptions(unsigned objcABIVersion,
   if (objcABIVersion >= 2)
     return true;
 
-  if (Triple.getOS() != llvm::Triple::Darwin)
+  if (!Triple.isOSDarwin())
     return false;
 
   return (!Triple.isMacOSXVersionLT(10,5) &&
           (Triple.getArch() == llvm::Triple::x86_64 ||
-           Triple.getArch() == llvm::Triple::arm));  
+           Triple.getArch() == llvm::Triple::arm));
 }
 
 /// addExceptionArgs - Adds exception related arguments to the driver command
@@ -922,7 +985,7 @@ static void addExceptionArgs(const ArgList &Args, types::ID InputType,
                                options::OPT_fno_exceptions)) {
     if (A->getOption().matches(options::OPT_fexceptions))
       ExceptionsEnabled = true;
-    else 
+    else
       ExceptionsEnabled = false;
 
     DidHaveExplicitExceptionFlag = true;
@@ -938,20 +1001,20 @@ static void addExceptionArgs(const ArgList &Args, types::ID InputType,
   // Obj-C exceptions are enabled by default, regardless of -fexceptions. This
   // is not necessarily sensible, but follows GCC.
   if (types::isObjC(InputType) &&
-      Args.hasFlag(options::OPT_fobjc_exceptions, 
+      Args.hasFlag(options::OPT_fobjc_exceptions,
                    options::OPT_fno_objc_exceptions,
                    true)) {
     CmdArgs.push_back("-fobjc-exceptions");
 
-    ShouldUseExceptionTables |= 
+    ShouldUseExceptionTables |=
       shouldUseExceptionTablesForObjCExceptions(objcABIVersion, Triple);
   }
 
   if (types::isCXX(InputType)) {
     bool CXXExceptionsEnabled = ExceptionsEnabled;
 
-    if (Arg *A = Args.getLastArg(options::OPT_fcxx_exceptions, 
-                                 options::OPT_fno_cxx_exceptions, 
+    if (Arg *A = Args.getLastArg(options::OPT_fcxx_exceptions,
+                                 options::OPT_fno_cxx_exceptions,
                                  options::OPT_fexceptions,
                                  options::OPT_fno_exceptions)) {
       if (A->getOption().matches(options::OPT_fcxx_exceptions))
@@ -973,7 +1036,7 @@ static void addExceptionArgs(const ArgList &Args, types::ID InputType,
 
 static bool ShouldDisableCFI(const ArgList &Args,
                              const ToolChain &TC) {
-  if (TC.getTriple().getOS() == llvm::Triple::Darwin) {
+  if (TC.getTriple().isOSDarwin()) {
     // The native darwin assembler doesn't support cfi directives, so
     // we disable them if we think the .s file will be passed to it.
 
@@ -1079,7 +1142,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       A->claim();
 
       for (unsigned i = 0, e = A->getNumValues(); i != e; ++i) {
-        llvm::StringRef Value = A->getValue(Args, i);
+        StringRef Value = A->getValue(Args, i);
 
         if (Value == "-force_cpusubtype_ALL") {
           // Do nothing, this is the default and we don't support anything else.
@@ -1091,7 +1154,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
         } else if (Value == "--noexecstack") {
           CmdArgs.push_back("-mnoexecstack");
         } else {
-          D.Diag(clang::diag::err_drv_unsupported_option_argument)
+          D.Diag(diag::err_drv_unsupported_option_argument)
             << A->getOption().getName() << Value;
         }
       }
@@ -1217,7 +1280,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     else
       Model = getToolChain().GetDefaultRelocationModel();
   }
-  if (llvm::StringRef(Model) != "pic") {
+  if (StringRef(Model) != "pic") {
     CmdArgs.push_back("-mrelocation-model");
     CmdArgs.push_back(Model);
   }
@@ -1275,13 +1338,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Enable -mconstructor-aliases except on darwin, where we have to
   // work around a linker bug;  see <rdar://problem/7651567>.
-  if (getToolChain().getTriple().getOS() != llvm::Triple::Darwin)
+  if (!getToolChain().getTriple().isOSDarwin())
     CmdArgs.push_back("-mconstructor-aliases");
 
   // Darwin's kernel doesn't support guard variables; just die if we
   // try to use them.
-  if (KernelOrKext &&
-      getToolChain().getTriple().getOS() == llvm::Triple::Darwin)
+  if (KernelOrKext && getToolChain().getTriple().isOSDarwin())
     CmdArgs.push_back("-fforbid-guard-variables");
 
   if (Args.hasArg(options::OPT_mms_bitfields)) {
@@ -1325,6 +1387,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
+  case llvm::Triple::mips64:
+  case llvm::Triple::mips64el:
     AddMIPSTargetArgs(Args, CmdArgs);
     break;
 
@@ -1347,7 +1411,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -mno-omit-leaf-frame-pointer is the default on Darwin.
   if (Args.hasFlag(options::OPT_momit_leaf_frame_pointer,
                    options::OPT_mno_omit_leaf_frame_pointer,
-                   getToolChain().getTriple().getOS() != llvm::Triple::Darwin))
+                   !getToolChain().getTriple().isOSDarwin()))
     CmdArgs.push_back("-momit-leaf-frame-pointer");
 
   // -fno-math-errno is default.
@@ -1362,21 +1426,22 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasArg(options::OPT_fallow_unsupported)) {
     Arg *Unsupported;
     if ((Unsupported = Args.getLastArg(options::OPT_iframework)))
-      D.Diag(clang::diag::err_drv_clang_unsupported)
+      D.Diag(diag::err_drv_clang_unsupported)
         << Unsupported->getOption().getName();
 
     if (types::isCXX(InputType) &&
-        getToolChain().getTriple().getOS() == llvm::Triple::Darwin &&
+        getToolChain().getTriple().isOSDarwin() &&
         getToolChain().getTriple().getArch() == llvm::Triple::x86) {
-      if ((Unsupported = Args.getLastArg(options::OPT_fapple_kext)))
-        D.Diag(clang::diag::err_drv_clang_unsupported_opt_cxx_darwin_i386)
+      if ((Unsupported = Args.getLastArg(options::OPT_fapple_kext)) ||
+          (Unsupported = Args.getLastArg(options::OPT_mkernel)))
+        D.Diag(diag::err_drv_clang_unsupported_opt_cxx_darwin_i386)
           << Unsupported->getOption().getName();
     }
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_v);
   Args.AddLastArg(CmdArgs, options::OPT_H);
-  if (D.CCPrintHeaders) {
+  if (D.CCPrintHeaders && !D.CCGenDiagnostics) {
     CmdArgs.push_back("-header-include-file");
     CmdArgs.push_back(D.CCPrintHeadersFilename ?
                       D.CCPrintHeadersFilename : "-");
@@ -1384,7 +1449,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_P);
   Args.AddLastArg(CmdArgs, options::OPT_print_ivar_layout);
 
-  if (D.CCLogDiagnostics) {
+  if (D.CCLogDiagnostics && !D.CCGenDiagnostics) {
     CmdArgs.push_back("-diagnostic-log-file");
     CmdArgs.push_back(D.CCLogDiagnosticsFilename ?
                       D.CCLogDiagnosticsFilename : "-");
@@ -1417,9 +1482,16 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  Args.AddLastArg(CmdArgs, options::OPT_nostdinc);
-  Args.AddLastArg(CmdArgs, options::OPT_nostdincxx);
-  Args.AddLastArg(CmdArgs, options::OPT_nobuiltininc);
+  // Pass options for controlling the default header search paths.
+  if (Args.hasArg(options::OPT_nostdinc)) {
+    CmdArgs.push_back("-nostdsysteminc");
+    CmdArgs.push_back("-nobuiltininc");
+  } else {
+    if (Args.hasArg(options::OPT_nostdlibinc))
+        CmdArgs.push_back("-nostdsysteminc");
+    Args.AddLastArg(CmdArgs, options::OPT_nostdincxx);
+    Args.AddLastArg(CmdArgs, options::OPT_nobuiltininc);
+  }
 
   // Pass the path to compiler resource files.
   CmdArgs.push_back("-resource-dir");
@@ -1444,11 +1516,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
         CmdArgs.push_back("-arcmt-migrate");
         CmdArgs.push_back("-arcmt-migrate-directory");
         CmdArgs.push_back(A->getValue(Args));
+
+        Args.AddLastArg(CmdArgs, options::OPT_arcmt_migrate_report_output);
+        Args.AddLastArg(CmdArgs, options::OPT_arcmt_migrate_emit_arc_errors);
         break;
       }
     }
   }
-    
+
   // Add preprocessing options like -I, -D, etc. if we are using the
   // preprocessor.
   //
@@ -1456,6 +1531,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (types::getPreprocessedType(InputType) != types::TY_INVALID)
     AddPreprocessingOptions(D, Args, CmdArgs, Output, Inputs);
 
+  // Don't warn about "clang -c -DPIC -fPIC test.i" because libtool.m4 assumes
+  // that "The compiler can only warn and ignore the option if not recognized".
+  // When building with ccache, it will pass -D options to clang even on
+  // preprocessed inputs and configure concludes that -fPIC is not supported.
+  Args.ClaimAllArgs(options::OPT_D);
+
   // Manually translate -O to -O2 and -O4 to -O3; let clang reject
   // others.
   if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
@@ -1584,7 +1665,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     // If -fmessage-length=N was not specified, determine whether this is a
     // terminal and, if so, implicitly define -fmessage-length appropriately.
     unsigned N = llvm::sys::Process::StandardErrColumns();
-    CmdArgs.push_back(Args.MakeArgString(llvm::Twine(N)));
+    CmdArgs.push_back(Args.MakeArgString(Twine(N)));
   }
 
   if (const Arg *A = Args.getLastArg(options::OPT_fvisibility_EQ)) {
@@ -1630,7 +1711,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Forward -ftrap_function= options to the backend.
   if (Arg *A = Args.getLastArg(options::OPT_ftrap_function_EQ)) {
-    llvm::StringRef FuncName = A->getValue(Args);
+    StringRef FuncName = A->getValue(Args);
     CmdArgs.push_back("-backend-option");
     CmdArgs.push_back(Args.MakeArgString("-trap-func=" + FuncName));
   }
@@ -1660,11 +1741,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       StackProtectorLevel = 1;
     else if (A->getOption().matches(options::OPT_fstack_protector_all))
       StackProtectorLevel = 2;
-  } else
-    StackProtectorLevel = getToolChain().GetDefaultStackProtectorLevel();
+  } else {
+    StackProtectorLevel =
+      getToolChain().GetDefaultStackProtectorLevel(KernelOrKext);
+  }
   if (StackProtectorLevel) {
     CmdArgs.push_back("-stack-protector");
-    CmdArgs.push_back(Args.MakeArgString(llvm::Twine(StackProtectorLevel)));
+    CmdArgs.push_back(Args.MakeArgString(Twine(StackProtectorLevel)));
   }
 
   // Translate -mstackrealign
@@ -1672,7 +1755,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-backend-option");
     CmdArgs.push_back("-force-align-stack");
   }
-  
+
   // Forward -f options with positive and negative forms; we translate
   // these by hand.
 
@@ -1697,6 +1780,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
          Args.hasArg(options::OPT_fobjc_nonfragile_abi) &&
          !Args.hasArg(options::OPT_fno_blocks))) {
     CmdArgs.push_back("-fblocks");
+
+    if (!Args.hasArg(options::OPT_fgnu_runtime) && 
+        !getToolChain().hasBlocksRuntime())
+      CmdArgs.push_back("-fblocks-runtime-optional");
   }
 
   // -faccess-control is default.
@@ -1744,11 +1831,16 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    getToolChain().getTriple().getOS() == llvm::Triple::Win32))
     CmdArgs.push_back("-fms-extensions");
 
+  // -fms-compatibility=0 is default.
+  if (Args.hasFlag(options::OPT_fms_compatibility, options::OPT_fno_ms_compatibility,
+                   getToolChain().getTriple().getOS() == llvm::Triple::Win32))
+    CmdArgs.push_back("-fms-compatibility");
+
   // -fmsc-version=1300 is default.
   if (Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions,
                    getToolChain().getTriple().getOS() == llvm::Triple::Win32) ||
       Args.hasArg(options::OPT_fmsc_version)) {
-    llvm::StringRef msc_ver = Args.getLastArgValue(options::OPT_fmsc_version);
+    StringRef msc_ver = Args.getLastArgValue(options::OPT_fmsc_version);
     if (msc_ver.empty())
       CmdArgs.push_back("-fmsc-version=1300");
     else
@@ -1761,10 +1853,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    options::OPT_fno_borland_extensions, false))
     CmdArgs.push_back("-fborland-extensions");
 
-  // -fno-delayed-template-parsing is default.
+  // -fno-delayed-template-parsing is default, except for Windows where MSVC STL
+  // needs it.
   if (Args.hasFlag(options::OPT_fdelayed_template_parsing,
                    options::OPT_fno_delayed_template_parsing,
-                   false))
+                   getToolChain().getTriple().getOS() == llvm::Triple::Win32))
     CmdArgs.push_back("-fdelayed-template-parsing");
 
   // -fgnu-keywords default varies depending on language; only pass if
@@ -1781,96 +1874,90 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -fobjc-nonfragile-abi=0 is default.
   ObjCRuntime objCRuntime;
   unsigned objcABIVersion = 0;
-  if (types::isObjC(InputType)) {
-    bool NeXTRuntimeIsDefault
-      = (IsRewriter || getToolChain().getTriple().isOSDarwin());
-    if (Args.hasFlag(options::OPT_fnext_runtime, options::OPT_fgnu_runtime,
-                     NeXTRuntimeIsDefault)) {
-      objCRuntime.setKind(ObjCRuntime::NeXT);
-    } else {
-      CmdArgs.push_back("-fgnu-runtime");
-      objCRuntime.setKind(ObjCRuntime::GNU);
-    }
-    getToolChain().configureObjCRuntime(objCRuntime);
-    if (objCRuntime.HasARC)
-      CmdArgs.push_back("-fobjc-runtime-has-arc");
-    if (objCRuntime.HasWeak)
-      CmdArgs.push_back("-fobjc-runtime-has-weak");
-    if (objCRuntime.HasTerminate)
-      CmdArgs.push_back("-fobjc-runtime-has-terminate");
+  bool NeXTRuntimeIsDefault
+    = (IsRewriter || getToolChain().getTriple().isOSDarwin());
+  if (Args.hasFlag(options::OPT_fnext_runtime, options::OPT_fgnu_runtime,
+                   NeXTRuntimeIsDefault)) {
+    objCRuntime.setKind(ObjCRuntime::NeXT);
+  } else {
+    CmdArgs.push_back("-fgnu-runtime");
+    objCRuntime.setKind(ObjCRuntime::GNU);
+  }
+  getToolChain().configureObjCRuntime(objCRuntime);
+  if (objCRuntime.HasARC)
+    CmdArgs.push_back("-fobjc-runtime-has-arc");
+  if (objCRuntime.HasWeak)
+    CmdArgs.push_back("-fobjc-runtime-has-weak");
+  if (objCRuntime.HasTerminate)
+    CmdArgs.push_back("-fobjc-runtime-has-terminate");
 
-    // Compute the Objective-C ABI "version" to use. Version numbers are
-    // slightly confusing for historical reasons:
-    //   1 - Traditional "fragile" ABI
-    //   2 - Non-fragile ABI, version 1
-    //   3 - Non-fragile ABI, version 2
-    objcABIVersion = 1;
-    // If -fobjc-abi-version= is present, use that to set the version.
-    if (Arg *A = Args.getLastArg(options::OPT_fobjc_abi_version_EQ)) {
-      if (llvm::StringRef(A->getValue(Args)) == "1")
-        objcABIVersion = 1;
-      else if (llvm::StringRef(A->getValue(Args)) == "2")
-        objcABIVersion = 2;
-      else if (llvm::StringRef(A->getValue(Args)) == "3")
-        objcABIVersion = 3;
-      else
-        D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
-    } else {
-      // Otherwise, determine if we are using the non-fragile ABI.
-      if (Args.hasFlag(options::OPT_fobjc_nonfragile_abi,
-                       options::OPT_fno_objc_nonfragile_abi,
-                       getToolChain().IsObjCNonFragileABIDefault())) {
-        // Determine the non-fragile ABI version to use.
+  // Compute the Objective-C ABI "version" to use. Version numbers are
+  // slightly confusing for historical reasons:
+  //   1 - Traditional "fragile" ABI
+  //   2 - Non-fragile ABI, version 1
+  //   3 - Non-fragile ABI, version 2
+  objcABIVersion = 1;
+  // If -fobjc-abi-version= is present, use that to set the version.
+  if (Arg *A = Args.getLastArg(options::OPT_fobjc_abi_version_EQ)) {
+    if (StringRef(A->getValue(Args)) == "1")
+      objcABIVersion = 1;
+    else if (StringRef(A->getValue(Args)) == "2")
+      objcABIVersion = 2;
+    else if (StringRef(A->getValue(Args)) == "3")
+      objcABIVersion = 3;
+    else
+      D.Diag(diag::err_drv_clang_unsupported) << A->getAsString(Args);
+  } else {
+    // Otherwise, determine if we are using the non-fragile ABI.
+    bool NonFragileABIIsDefault
+      = (!IsRewriter && getToolChain().IsObjCNonFragileABIDefault());
+    if (Args.hasFlag(options::OPT_fobjc_nonfragile_abi,
+                     options::OPT_fno_objc_nonfragile_abi,
+                     NonFragileABIIsDefault)) {
+      // Determine the non-fragile ABI version to use.
 #ifdef DISABLE_DEFAULT_NONFRAGILEABI_TWO
-        unsigned NonFragileABIVersion = 1;
+      unsigned NonFragileABIVersion = 1;
 #else
-        unsigned NonFragileABIVersion = 2;
+      unsigned NonFragileABIVersion = 2;
 #endif
 
-        if (Arg *A = Args.getLastArg(
-              options::OPT_fobjc_nonfragile_abi_version_EQ)) {
-          if (llvm::StringRef(A->getValue(Args)) == "1")
-            NonFragileABIVersion = 1;
-          else if (llvm::StringRef(A->getValue(Args)) == "2")
-            NonFragileABIVersion = 2;
-          else
-            D.Diag(clang::diag::err_drv_clang_unsupported)
-              << A->getAsString(Args);
-        }
-
-        objcABIVersion = 1 + NonFragileABIVersion;
-      } else {
-        objcABIVersion = 1;
-      }
-    }
-
-    if (objcABIVersion == 2 || objcABIVersion == 3) {
-      CmdArgs.push_back("-fobjc-nonfragile-abi");
-
-      // -fobjc-dispatch-method is only relevant with the nonfragile-abi, and
-      // legacy is the default.
-      if (!Args.hasFlag(options::OPT_fobjc_legacy_dispatch,
-                        options::OPT_fno_objc_legacy_dispatch,
-                        getToolChain().IsObjCLegacyDispatchDefault())) {
-        if (getToolChain().UseObjCMixedDispatch())
-          CmdArgs.push_back("-fobjc-dispatch-method=mixed");
+      if (Arg *A = Args.getLastArg(
+            options::OPT_fobjc_nonfragile_abi_version_EQ)) {
+        if (StringRef(A->getValue(Args)) == "1")
+          NonFragileABIVersion = 1;
+        else if (StringRef(A->getValue(Args)) == "2")
+          NonFragileABIVersion = 2;
         else
-          CmdArgs.push_back("-fobjc-dispatch-method=non-legacy");
+          D.Diag(diag::err_drv_clang_unsupported)
+            << A->getAsString(Args);
       }
-    }
 
-    // FIXME: Don't expose -fobjc-default-synthesize-properties as a top-level
-    // driver flag yet.  This feature is still under active development
-    // and shouldn't be exposed as a user visible feature (which may change).
-    // Clang still supports this as a -cc1 option for development and testing.
-#if 0
-    // -fobjc-default-synthesize-properties=0 is default.
-    if (Args.hasFlag(options::OPT_fobjc_default_synthesize_properties,
-                     options::OPT_fno_objc_default_synthesize_properties,
-                     getToolChain().IsObjCDefaultSynthPropertiesDefault())) {
-      CmdArgs.push_back("-fobjc-default-synthesize-properties");
+      objcABIVersion = 1 + NonFragileABIVersion;
+    } else {
+      objcABIVersion = 1;
     }
-#endif
+  }
+
+  if (objcABIVersion == 1) {
+    CmdArgs.push_back("-fobjc-fragile-abi");
+  } else {
+    // -fobjc-dispatch-method is only relevant with the nonfragile-abi, and
+    // legacy is the default.
+    if (!Args.hasFlag(options::OPT_fobjc_legacy_dispatch,
+                      options::OPT_fno_objc_legacy_dispatch,
+                      getToolChain().IsObjCLegacyDispatchDefault())) {
+      if (getToolChain().UseObjCMixedDispatch())
+        CmdArgs.push_back("-fobjc-dispatch-method=mixed");
+      else
+        CmdArgs.push_back("-fobjc-dispatch-method=non-legacy");
+    }
+  }
+
+  // -fobjc-default-synthesize-properties=0 is default.
+  if (Args.hasFlag(options::OPT_fobjc_default_synthesize_properties,
+                   options::OPT_fno_objc_default_synthesize_properties,
+                   getToolChain().IsObjCDefaultSynthPropertiesDefault())) {
+    CmdArgs.push_back("-fobjc-default-synthesize-properties");
   }
 
   // Allow -fno-objc-arr to trump -fobjc-arr/-fobjc-arc.
@@ -1891,7 +1978,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // rewriter.
   if (IsRewriter)
     CmdArgs.push_back("-fno-objc-infer-related-result-type");
-  
+
   // Handle -fobjc-gc and -fobjc-gc-only. They are exclusive, and -fobjc-gc-only
   // takes precedence.
   const Arg *GCArg = Args.getLastArg(options::OPT_fobjc_gc_only);
@@ -1899,13 +1986,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     GCArg = Args.getLastArg(options::OPT_fobjc_gc);
   if (GCArg) {
     if (ARC) {
-      D.Diag(clang::diag::err_drv_objc_gc_arr)
+      D.Diag(diag::err_drv_objc_gc_arr)
         << GCArg->getAsString(Args);
     } else if (getToolChain().SupportsObjCGC()) {
       GCArg->render(Args, CmdArgs);
     } else {
       // FIXME: We should move this to a hard error.
-      D.Diag(clang::diag::warn_drv_objc_gc_unsupported)
+      D.Diag(diag::warn_drv_objc_gc_unsupported)
         << GCArg->getAsString(Args);
     }
   }
@@ -1948,11 +2035,23 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    false))
     CmdArgs.push_back("-fpascal-strings");
 
+  // Honor -fpack-struct= and -fpack-struct, if given. Note that
+  // -fno-pack-struct doesn't apply to -fpack-struct=.
+  if (Arg *A = Args.getLastArg(options::OPT_fpack_struct_EQ)) {
+    CmdArgs.push_back("-fpack-struct");
+    CmdArgs.push_back(A->getValue(Args));
+  } else if (Args.hasFlag(options::OPT_fpack_struct,
+                          options::OPT_fno_pack_struct, false)) {
+    CmdArgs.push_back("-fpack-struct");
+    CmdArgs.push_back("1");
+  }
+
   if (Args.hasArg(options::OPT_mkernel) ||
       Args.hasArg(options::OPT_fapple_kext)) {
     if (!Args.hasArg(options::OPT_fcommon))
       CmdArgs.push_back("-fno-common");
   }
+
   // -fcommon is default, only pass non-default.
   else if (!Args.hasFlag(options::OPT_fcommon, options::OPT_fno_common))
     CmdArgs.push_back("-fno-common");
@@ -1961,13 +2060,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -funsigned-bitfields.
   if (!Args.hasFlag(options::OPT_fsigned_bitfields,
                     options::OPT_funsigned_bitfields))
-    D.Diag(clang::diag::warn_drv_clang_unsupported)
+    D.Diag(diag::warn_drv_clang_unsupported)
       << Args.getLastArg(options::OPT_funsigned_bitfields)->getAsString(Args);
 
   // -fsigned-bitfields is default, and clang doesn't support -fno-for-scope.
   if (!Args.hasFlag(options::OPT_ffor_scope,
                     options::OPT_fno_for_scope))
-    D.Diag(clang::diag::err_drv_clang_unsupported)
+    D.Diag(diag::err_drv_clang_unsupported)
       << Args.getLastArg(options::OPT_fno_for_scope)->getAsString(Args);
 
   // -fcaret-diagnostics is default.
@@ -1979,7 +2078,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasFlag(options::OPT_fdiagnostics_fixit_info,
                     options::OPT_fno_diagnostics_fixit_info))
     CmdArgs.push_back("-fno-diagnostics-fixit-info");
-  
+
   // Enable -fdiagnostics-show-name by default.
   if (Args.hasFlag(options::OPT_fdiagnostics_show_name,
                    options::OPT_fno_diagnostics_show_name, false))
@@ -2055,14 +2154,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Arg *A = Args.getLastArg(options::OPT_funit_at_a_time,
                                options::OPT_fno_unit_at_a_time)) {
     if (A->getOption().matches(options::OPT_fno_unit_at_a_time))
-      D.Diag(clang::diag::warn_drv_clang_unsupported) << A->getAsString(Args);
+      D.Diag(diag::warn_drv_clang_unsupported) << A->getAsString(Args);
   }
 
   // Default to -fno-builtin-str{cat,cpy} on Darwin for ARM.
   //
   // FIXME: This is disabled until clang -cc1 supports -fno-builtin-foo. PR4941.
 #if 0
-  if (getToolChain().getTriple().getOS() == llvm::Triple::Darwin &&
+  if (getToolChain().getTriple().isOSDarwin() &&
       (getToolChain().getTriple().getArch() == llvm::Triple::arm ||
        getToolChain().getTriple().getArch() == llvm::Triple::thumb)) {
     if (!Args.hasArg(options::OPT_fbuiltin_strcat))
@@ -2077,8 +2176,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                                options::OPT_traditional_cpp)) {
     if (isa<PreprocessJobAction>(JA))
       CmdArgs.push_back("-traditional-cpp");
-    else 
-      D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
+    else
+      D.Diag(diag::err_drv_clang_unsupported) << A->getAsString(Args);
   }
 
   Args.AddLastArg(CmdArgs, options::OPT_dM);
@@ -2093,7 +2192,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
     // We translate this by hand to the -cc1 argument, since nightly test uses
     // it and developers have been trained to spell it with -mllvm.
-    if (llvm::StringRef((*it)->getValue(Args, 0)) == "-disable-llvm-optzns")
+    if (StringRef((*it)->getValue(Args, 0)) == "-disable-llvm-optzns")
       CmdArgs.push_back("-disable-llvm-optzns");
     else
       (*it)->render(Args, CmdArgs);
@@ -2145,7 +2244,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_pg))
     if (Args.hasArg(options::OPT_fomit_frame_pointer))
-      D.Diag(clang::diag::err_drv_argument_not_allowed_with)
+      D.Diag(diag::err_drv_argument_not_allowed_with)
         << "-fomit-frame-pointer" << A->getAsString(Args);
 
   // Claim some arguments which clang supports automatically.
@@ -2189,7 +2288,8 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Add the "effective" target triple.
   CmdArgs.push_back("-triple");
-  std::string TripleStr = getToolChain().ComputeEffectiveClangTriple(Args);
+  std::string TripleStr = 
+    getToolChain().ComputeEffectiveClangTriple(Args, Input.getType());
   CmdArgs.push_back(Args.MakeArgString(TripleStr));
 
   // Set the output mode, we currently only expect to be used as a real
@@ -2252,7 +2352,7 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
 
   // If using a driver driver, force the arch.
   const std::string &Arch = getToolChain().getArchName();
-  if (getToolChain().getTriple().getOS() == llvm::Triple::Darwin) {
+  if (getToolChain().getTriple().isOSDarwin()) {
     CmdArgs.push_back("-arch");
 
     // FIXME: Remove these special cases.
@@ -2298,10 +2398,10 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
     // Don't try to pass LLVM or AST inputs to a generic gcc.
     if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||
         II.getType() == types::TY_LLVM_BC || II.getType() == types::TY_LTO_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
+      D.Diag(diag::err_drv_no_linker_llvm_support)
         << getToolChain().getTripleString();
     else if (II.getType() == types::TY_AST)
-      D.Diag(clang::diag::err_drv_no_ast_support)
+      D.Diag(diag::err_drv_no_ast_support)
         << getToolChain().getTripleString();
 
     if (types::canTypeBeUserSpecified(II.getType())) {
@@ -2364,7 +2464,7 @@ void gcc::Compile::RenderExtraToolArgs(const JobAction &JA,
     CmdArgs.push_back("-c");
   else {
     if (JA.getType() != types::TY_PP_Asm)
-      D.Diag(clang::diag::err_drv_invalid_gcc_output_type)
+      D.Diag(diag::err_drv_invalid_gcc_output_type)
         << getTypeName(JA.getType());
 
     CmdArgs.push_back("-S");
@@ -2384,19 +2484,21 @@ void gcc::Link::RenderExtraToolArgs(const JobAction &JA,
 const char *darwin::CC1::getCC1Name(types::ID Type) const {
   switch (Type) {
   default:
-    assert(0 && "Unexpected type for Darwin CC1 tool.");
+    llvm_unreachable("Unexpected type for Darwin CC1 tool.");
   case types::TY_Asm:
   case types::TY_C: case types::TY_CHeader:
   case types::TY_PP_C: case types::TY_PP_CHeader:
     return "cc1";
   case types::TY_ObjC: case types::TY_ObjCHeader:
-  case types::TY_PP_ObjC: case types::TY_PP_ObjCHeader:
+  case types::TY_PP_ObjC: case types::TY_PP_ObjC_Alias:
+  case types::TY_PP_ObjCHeader:
     return "cc1obj";
   case types::TY_CXX: case types::TY_CXXHeader:
   case types::TY_PP_CXX: case types::TY_PP_CXXHeader:
     return "cc1plus";
   case types::TY_ObjCXX: case types::TY_ObjCXXHeader:
-  case types::TY_PP_ObjCXX: case types::TY_PP_ObjCXXHeader:
+  case types::TY_PP_ObjCXX: case types::TY_PP_ObjCXX_Alias:
+  case types::TY_PP_ObjCXXHeader:
     return "cc1objplus";
   }
 }
@@ -2425,14 +2527,145 @@ darwin::CC1::getDependencyFileName(const ArgList &Args,
 
   if (Arg *OutputOpt = Args.getLastArg(options::OPT_o)) {
     std::string Str(OutputOpt->getValue(Args));
-
     Res = Str.substr(0, Str.rfind('.'));
-  } else
+  } else {
     Res = darwin::CC1::getBaseInputStem(Args, Inputs);
-
+  }
   return Args.MakeArgString(Res + ".d");
 }
 
+void darwin::CC1::RemoveCC1UnsupportedArgs(ArgStringList &CmdArgs) const {
+  for (ArgStringList::iterator it = CmdArgs.begin(), ie = CmdArgs.end();
+       it != ie;) {
+
+    StringRef Option = *it;
+    bool RemoveOption = false;
+
+    // Remove -faltivec
+    if (Option.equals("-faltivec")) {
+      it = CmdArgs.erase(it);
+      ie = CmdArgs.end();
+      continue;
+    }
+
+    // Handle machine specific options.
+    if (Option.startswith("-m")) {
+      RemoveOption = llvm::StringSwitch<bool>(Option)
+        .Case("-mthumb", true)
+        .Case("-mno-thumb", true)
+        .Case("-mno-fused-madd", true)
+        .Case("-mlong-branch", true)
+        .Case("-mlongcall", true)
+        .Case("-mcpu=G4", true)
+        .Case("-mcpu=G5", true)
+        .Default(false);
+    }
+    
+    // Handle warning options.
+    if (Option.startswith("-W")) {
+      // Remove -W/-Wno- to reduce the number of cases.
+      if (Option.startswith("-Wno-"))
+        Option = Option.substr(5);
+      else
+        Option = Option.substr(2);
+      
+      RemoveOption = llvm::StringSwitch<bool>(Option)
+        .Case("address-of-temporary", true)
+        .Case("ambiguous-member-template", true)
+        .Case("analyzer-incompatible-plugin", true)
+        .Case("array-bounds", true)
+        .Case("array-bounds-pointer-arithmetic", true)
+        .Case("bind-to-temporary-copy", true)
+        .Case("bitwise-op-parentheses", true)
+        .Case("bool-conversions", true)
+        .Case("builtin-macro-redefined", true)
+        .Case("c++-hex-floats", true)
+        .Case("c++0x-compat", true)
+        .Case("c++0x-extensions", true)
+        .Case("c++0x-narrowing", true)
+        .Case("c++11-compat", true)
+        .Case("c++11-extensions", true)
+        .Case("c++11-narrowing", true)
+        .Case("conditional-uninitialized", true)
+        .Case("constant-conversion", true)
+        .Case("CFString-literal", true)
+        .Case("constant-logical-operand", true)
+        .Case("custom-atomic-properties", true)
+        .Case("default-arg-special-member", true)
+        .Case("delegating-ctor-cycles", true)
+        .Case("delete-non-virtual-dtor", true)
+        .Case("deprecated-implementations", true)
+        .Case("deprecated-writable-strings", true)
+        .Case("distributed-object-modifiers", true)
+        .Case("duplicate-method-arg", true)
+        .Case("dynamic-class-memaccess", true)
+        .Case("enum-compare", true)
+        .Case("exit-time-destructors", true)
+        .Case("gnu", true)
+        .Case("gnu-designator", true)
+        .Case("header-hygiene", true)
+        .Case("idiomatic-parentheses", true)
+        .Case("ignored-qualifiers", true)
+        .Case("implicit-atomic-properties", true)
+        .Case("incompatible-pointer-types", true)
+        .Case("incomplete-implementation", true)
+        .Case("initializer-overrides", true)
+        .Case("invalid-noreturn", true)
+        .Case("invalid-token-paste", true)
+        .Case("language-extension-token", true)
+        .Case("literal-conversion", true)
+        .Case("literal-range", true)
+        .Case("local-type-template-args", true)
+        .Case("logical-op-parentheses", true)
+        .Case("method-signatures", true)
+        .Case("microsoft", true)
+        .Case("mismatched-tags", true)
+        .Case("missing-method-return-type", true)
+        .Case("non-pod-varargs", true)
+        .Case("nonfragile-abi2", true)
+        .Case("null-arithmetic", true)
+        .Case("null-dereference", true)
+        .Case("out-of-line-declaration", true)
+        .Case("overriding-method-mismatch", true)
+        .Case("readonly-setter-attrs", true)
+        .Case("return-stack-address", true)
+        .Case("self-assign", true)
+        .Case("semicolon-before-method-body", true)
+        .Case("sentinel", true)
+        .Case("shift-overflow", true)
+        .Case("shift-sign-overflow", true)
+        .Case("sign-conversion", true)
+        .Case("sizeof-array-argument", true)
+        .Case("sizeof-pointer-memaccess", true)
+        .Case("string-compare", true)
+        .Case("super-class-method-mismatch", true)
+        .Case("tautological-compare", true)
+        .Case("typedef-redefinition", true)
+        .Case("typename-missing", true)
+        .Case("undefined-reinterpret-cast", true)
+        .Case("unknown-warning-option", true)
+        .Case("unnamed-type-template-args", true)
+        .Case("unneeded-internal-declaration", true)
+        .Case("unneeded-member-function", true)
+        .Case("unused-comparison", true)
+        .Case("unused-exception-parameter", true)
+        .Case("unused-member-function", true)
+        .Case("unused-result", true)
+        .Case("vector-conversions", true)
+        .Case("vla", true)
+        .Case("used-but-marked-unused", true)
+        .Case("weak-vtables", true)
+        .Default(false);
+    } // if (Option.startswith("-W"))
+    if (RemoveOption) {
+      it = CmdArgs.erase(it);
+      ie = CmdArgs.end();
+    } else {
+      ++it;
+    }
+  }
+}
+
 void darwin::CC1::AddCC1Args(const ArgList &Args,
                              ArgStringList &CmdArgs) const {
   const Driver &D = getToolChain().getDriver();
@@ -2470,7 +2703,7 @@ void darwin::CC1::AddCC1OptionsArgs(const ArgList &Args, ArgStringList &CmdArgs,
 
   if (Arg *A = Args.getLastArg(options::OPT_pg))
     if (Args.hasArg(options::OPT_fomit_frame_pointer))
-      D.Diag(clang::diag::err_drv_argument_not_allowed_with)
+      D.Diag(diag::err_drv_argument_not_allowed_with)
         << A->getAsString(Args) << "-fomit-frame-pointer";
 
   AddCC1Args(Args, CmdArgs);
@@ -2742,6 +2975,8 @@ void darwin::Preprocess::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_d_Group);
 
+  RemoveCC1UnsupportedArgs(CmdArgs);
+
   const char *CC1Name = getCC1Name(Inputs[0].getType());
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath(CC1Name));
@@ -2761,7 +2996,7 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
   types::ID InputType = Inputs[0].getType();
   const Arg *A;
   if ((A = Args.getLastArg(options::OPT_traditional)))
-    D.Diag(clang::diag::err_drv_argument_only_allowed_with)
+    D.Diag(diag::err_drv_argument_only_allowed_with)
       << A->getAsString(Args) << "-E";
 
   if (JA.getType() == types::TY_LLVM_IR ||
@@ -2771,11 +3006,11 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
            JA.getType() == types::TY_LTO_BC)
     CmdArgs.push_back("-emit-llvm-bc");
   else if (Output.getType() == types::TY_AST)
-    D.Diag(clang::diag::err_drv_no_ast_support)
+    D.Diag(diag::err_drv_no_ast_support)
       << getToolChain().getTripleString();
   else if (JA.getType() != types::TY_PP_Asm &&
            JA.getType() != types::TY_PCH)
-    D.Diag(clang::diag::err_drv_invalid_gcc_output_type)
+    D.Diag(diag::err_drv_invalid_gcc_output_type)
       << getTypeName(JA.getType());
 
   ArgStringList OutputArgs;
@@ -2809,7 +3044,7 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
 
       // Reject AST inputs.
       if (II.getType() == types::TY_AST) {
-        D.Diag(clang::diag::err_drv_no_ast_support)
+        D.Diag(diag::err_drv_no_ast_support)
           << getToolChain().getTripleString();
         return;
       }
@@ -2831,12 +3066,23 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-o");
     // NOTE: gcc uses a temp .s file for this, but there doesn't seem
     // to be a good reason.
-    CmdArgs.push_back("/dev/null");
+    const char *TmpPath = C.getArgs().MakeArgString(
+      D.GetTemporaryPath("cc", "s"));
+    C.addTempFile(TmpPath);
+    CmdArgs.push_back(TmpPath);
 
-    CmdArgs.push_back("--output-pch=");
-    CmdArgs.push_back(Output.getFilename());
+    // If we're emitting a pch file with the last 4 characters of ".pth"
+    // and falling back to llvm-gcc we want to use ".gch" instead.
+    std::string OutputFile(Output.getFilename());
+    size_t loc = OutputFile.rfind(".pth");
+    if (loc != std::string::npos)
+      OutputFile.replace(loc, 4, ".gch");
+    const char *Tmp = C.getArgs().MakeArgString("--output-pch="+OutputFile);
+    CmdArgs.push_back(Tmp);
   }
 
+  RemoveCC1UnsupportedArgs(CmdArgs);
+
   const char *CC1Name = getCC1Name(Inputs[0].getType());
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath(CC1Name));
@@ -2861,7 +3107,7 @@ void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   // Forward -g, assuming we are dealing with an actual assembly file.
-  if (SourceAction->getType() == types::TY_Asm || 
+  if (SourceAction->getType() == types::TY_Asm ||
       SourceAction->getType() == types::TY_PP_Asm) {
     if (Args.hasArg(options::OPT_gstabs))
       CmdArgs.push_back("--gstabs");
@@ -2903,7 +3149,7 @@ void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
 
 void darwin::DarwinTool::AddDarwinArch(const ArgList &Args,
                                        ArgStringList &CmdArgs) const {
-  llvm::StringRef ArchName = getDarwinToolChain().getDarwinArchName(Args);
+  StringRef ArchName = getDarwinToolChain().getDarwinArchName(Args);
 
   // Derived from darwin_arch spec.
   CmdArgs.push_back("-arch");
@@ -2926,7 +3172,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
     if (!Driver::GetReleaseVersion(A->getValue(Args), Version[0],
                                    Version[1], Version[2], HadExtra) ||
         HadExtra)
-      D.Diag(clang::diag::err_drv_invalid_version_number)
+      D.Diag(diag::err_drv_invalid_version_number)
         << A->getAsString(Args);
   }
 
@@ -2950,7 +3196,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
              ie = Args.filtered_end(); it != ie; ++it) {
         const Arg *A = *it;
         for (unsigned i = 0, e = A->getNumValues(); i != e; ++i)
-          if (llvm::StringRef(A->getValue(Args, i)) == "-kext")
+          if (StringRef(A->getValue(Args, i)) == "-kext")
             UsesLdClassic = true;
       }
     }
@@ -2963,7 +3209,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
   // dsymutil step.
   if (Version[0] >= 116 && D.IsUsingLTO(Args)) {
     const char *TmpPath = C.getArgs().MakeArgString(
-      D.GetTemporaryPath(types::getTypeTempSuffix(types::TY_Object)));
+      D.GetTemporaryPath("cc", types::getTypeTempSuffix(types::TY_Object)));
     C.addTempFile(TmpPath);
     CmdArgs.push_back("-object_path_lto");
     CmdArgs.push_back(TmpPath);
@@ -2991,7 +3237,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
     if ((A = Args.getLastArg(options::OPT_compatibility__version)) ||
         (A = Args.getLastArg(options::OPT_current__version)) ||
         (A = Args.getLastArg(options::OPT_install__name)))
-      D.Diag(clang::diag::err_drv_argument_only_allowed_with)
+      D.Diag(diag::err_drv_argument_only_allowed_with)
         << A->getAsString(Args) << "-dynamiclib";
 
     Args.AddLastArg(CmdArgs, options::OPT_force__flat__namespace);
@@ -3007,7 +3253,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
         (A = Args.getLastArg(options::OPT_force__flat__namespace)) ||
         (A = Args.getLastArg(options::OPT_keep__private__externs)) ||
         (A = Args.getLastArg(options::OPT_private__bundle)))
-      D.Diag(clang::diag::err_drv_argument_not_allowed_with)
+      D.Diag(diag::err_drv_argument_not_allowed_with)
         << A->getAsString(Args) << "-dynamiclib";
 
     Args.AddAllArgsTranslated(CmdArgs, options::OPT_compatibility__version,
@@ -3054,9 +3300,9 @@ void darwin::Link::AddLinkArgs(Compilation &C,
     CmdArgs.push_back("-iphoneos_version_min");
   else
     CmdArgs.push_back("-macosx_version_min");
-  CmdArgs.push_back(Args.MakeArgString(llvm::Twine(TargetVersion[0]) + "." +
-                                       llvm::Twine(TargetVersion[1]) + "." +
-                                       llvm::Twine(TargetVersion[2])));
+  CmdArgs.push_back(Args.MakeArgString(Twine(TargetVersion[0]) + "." +
+                                       Twine(TargetVersion[1]) + "." +
+                                       Twine(TargetVersion[2])));
 
   Args.AddLastArg(CmdArgs, options::OPT_nomultidefs);
   Args.AddLastArg(CmdArgs, options::OPT_multi__module);
@@ -3139,6 +3385,18 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
   // more information.
   ArgStringList CmdArgs;
 
+  /// Hack(tm) to ignore linking errors when we are doing ARC migration.
+  if (Args.hasArg(options::OPT_ccc_arcmt_check,
+                  options::OPT_ccc_arcmt_migrate)) {
+    for (ArgList::const_iterator I = Args.begin(), E = Args.end(); I != E; ++I)
+      (*I)->claim();
+    const char *Exec =
+      Args.MakeArgString(getToolChain().GetProgramPath("touch"));
+    CmdArgs.push_back(Output.getFilename());
+    C.addCommand(new Command(JA, *this, Exec, CmdArgs));
+    return;
+  }
+
   // I'm not sure why this particular decomposition exists in gcc, but
   // we follow suite for ease of comparison.
   AddLinkArgs(C, Args, CmdArgs);
@@ -3346,6 +3604,26 @@ void darwin::Dsymutil::ConstructJob(Compilation &C, const JobAction &JA,
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
+void darwin::VerifyDebug::ConstructJob(Compilation &C, const JobAction &JA,
+				       const InputInfo &Output,
+				       const InputInfoList &Inputs,
+				       const ArgList &Args,
+				       const char *LinkingOutput) const {
+  ArgStringList CmdArgs;
+  CmdArgs.push_back("--verify");
+
+  assert(Inputs.size() == 1 && "Unable to handle multiple inputs.");
+  const InputInfo &Input = Inputs[0];
+  assert(Input.isFilename() && "Unexpected verify input");
+
+  // Grabbing the output of the earlier dsymutil run.
+  CmdArgs.push_back(Input.getFilename());
+
+  const char *Exec =
+    Args.MakeArgString(getToolChain().GetProgramPath("dwarfdump"));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
+}
+
 void auroraux::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
                                       const InputInfo &Output,
                                       const InputInfoList &Inputs,
@@ -3664,9 +3942,11 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       if (Args.hasArg(options::OPT_pg))
         CmdArgs.push_back(Args.MakeArgString(
                                 getToolChain().GetFilePath("gcrt1.o")));
-      else
+      else {
+        const char *crt = Args.hasArg(options::OPT_pie) ? "Scrt1.o" : "crt1.o";
         CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath("crt1.o")));
+                                getToolChain().GetFilePath(crt)));
+      }
       CmdArgs.push_back(Args.MakeArgString(
                               getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(Args.MakeArgString(
@@ -3683,7 +3963,7 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   const ToolChain::path_list Paths = getToolChain().getFilePaths();
   for (ToolChain::path_list::const_iterator i = Paths.begin(), e = Paths.end();
        i != e; ++i)
-    CmdArgs.push_back(Args.MakeArgString(llvm::StringRef("-L") + *i));
+    CmdArgs.push_back(Args.MakeArgString(StringRef("-L") + *i));
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
   Args.AddAllArgs(CmdArgs, options::OPT_s);
@@ -3798,9 +4078,7 @@ void netbsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
   }
 
-  const char *Exec = Args.MakeArgString(FindTargetProgramPath(getToolChain(),
-                                                      ToolTriple.getTriple(),
-                                                      "as"));
+  const char *Exec = Args.MakeArgString((getToolChain().GetProgramPath("as")));
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -3916,9 +4194,7 @@ void netbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   addProfileRT(getToolChain(), Args, CmdArgs, getToolChain().getTriple());
 
-  const char *Exec = Args.MakeArgString(FindTargetProgramPath(getToolChain(),
-                                                      ToolTriple.getTriple(),
-                                                      "ld"));
+  const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("ld"));
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -3936,7 +4212,7 @@ void linuxtools::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   } else if (getToolChain().getArch() == llvm::Triple::x86_64) {
     CmdArgs.push_back("--64");
   } else if (getToolChain().getArch() == llvm::Triple::arm) {
-    llvm::StringRef MArch = getToolChain().getArchName();
+    StringRef MArch = getToolChain().getArchName();
     if (MArch == "armv7" || MArch == "armv7a" || MArch == "armv7-a")
       CmdArgs.push_back("-mfpu=neon");
   }
@@ -4000,7 +4276,7 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-m");
   if (ToolChain.getArch() == llvm::Triple::x86)
     CmdArgs.push_back("elf_i386");
-  else if (ToolChain.getArch() == llvm::Triple::arm 
+  else if (ToolChain.getArch() == llvm::Triple::arm
            ||  ToolChain.getArch() == llvm::Triple::thumb)
     CmdArgs.push_back("armelf_linux_eabi");
   else if (ToolChain.getArch() == llvm::Triple::ppc)
@@ -4071,7 +4347,7 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   for (ToolChain::path_list::const_iterator i = Paths.begin(), e = Paths.end();
        i != e; ++i)
-    CmdArgs.push_back(Args.MakeArgString(llvm::StringRef("-L") + *i));
+    CmdArgs.push_back(Args.MakeArgString(StringRef("-L") + *i));
 
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs);
 
diff --git a/contrib/llvm/tools/clang/lib/Driver/Tools.h b/contrib/llvm/tools/clang/lib/Driver/Tools.h
index 1741d051ce9f..a4f732e1c213 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Tools.h
+++ b/contrib/llvm/tools/clang/lib/Driver/Tools.h
@@ -178,6 +178,7 @@ namespace darwin {
     const char *getCC1Name(types::ID Type) const;
 
     void AddCC1Args(const ArgList &Args, ArgStringList &CmdArgs) const;
+    void RemoveCC1UnsupportedArgs(ArgStringList &CmdArgs) const;
     void AddCC1OptionsArgs(const ArgList &Args, ArgStringList &CmdArgs,
                            const InputInfoList &Inputs,
                            const ArgStringList &OutputArgs) const;
@@ -276,6 +277,21 @@ namespace darwin {
                               const ArgList &TCArgs,
                               const char *LinkingOutput) const;
   };
+
+  class LLVM_LIBRARY_VISIBILITY VerifyDebug : public DarwinTool  {
+  public:
+    VerifyDebug(const ToolChain &TC) : DarwinTool("darwin::VerifyDebug",
+						  "dwarfdump", TC) {}
+
+    virtual bool hasIntegratedCPP() const { return false; }
+
+    virtual void ConstructJob(Compilation &C, const JobAction &JA,
+			      const InputInfo &Output,
+			      const InputInfoList &Inputs,
+			      const ArgList &TCArgs,
+			      const char *LinkingOutput) const;
+  };
+
 }
 
   /// openbsd -- Directly call GNU Binutils assembler and linker
diff --git a/contrib/llvm/tools/clang/lib/Driver/Types.cpp b/contrib/llvm/tools/clang/lib/Driver/Types.cpp
index 4a4312b77fc9..d61ab68c90ad 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Types.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Types.cpp
@@ -79,10 +79,11 @@ bool types::isAcceptedByClang(ID Id) {
   case TY_C: case TY_PP_C:
   case TY_CL:
   case TY_CUDA:
-  case TY_ObjC: case TY_PP_ObjC:
+  case TY_ObjC: case TY_PP_ObjC: case TY_PP_ObjC_Alias:
   case TY_CXX: case TY_PP_CXX:
-  case TY_ObjCXX: case TY_PP_ObjCXX:
+  case TY_ObjCXX: case TY_PP_ObjCXX: case TY_PP_ObjCXX_Alias:
   case TY_CHeader: case TY_PP_CHeader:
+  case TY_CLHeader:
   case TY_ObjCHeader: case TY_PP_ObjCHeader:
   case TY_CXXHeader: case TY_PP_CXXHeader:
   case TY_ObjCXXHeader: case TY_PP_ObjCXXHeader:
@@ -110,10 +111,10 @@ bool types::isObjC(ID Id) {
   default:
     return false;
 
-  case TY_ObjC: case TY_PP_ObjC:
+  case TY_ObjC: case TY_PP_ObjC: case TY_PP_ObjC_Alias:
   case TY_ObjCXX: case TY_PP_ObjCXX:
   case TY_ObjCHeader: case TY_PP_ObjCHeader:
-  case TY_ObjCXXHeader: case TY_PP_ObjCXXHeader:
+  case TY_ObjCXXHeader: case TY_PP_ObjCXXHeader: case TY_PP_ObjCXX_Alias:
     return true;
   }
 }
@@ -144,6 +145,7 @@ types::ID types::lookupTypeForExtension(const char *Ext) {
            .Case("F", TY_Fortran)
            .Case("s", TY_PP_Asm)
            .Case("S", TY_Asm)
+           .Case("o", TY_Object)
            .Case("ii", TY_PP_CXX)
            .Case("mi", TY_PP_ObjC)
            .Case("mm", TY_ObjCXX)
diff --git a/contrib/llvm/tools/clang/lib/Frontend/ASTConsumers.cpp b/contrib/llvm/tools/clang/lib/Frontend/ASTConsumers.cpp
index 28d312a2219b..54bb28272868 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/ASTConsumers.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/ASTConsumers.cpp
@@ -31,22 +31,23 @@ using namespace clang;
 
 namespace {
   class ASTPrinter : public ASTConsumer {
-    llvm::raw_ostream &Out;
+    raw_ostream &Out;
     bool Dump;
 
   public:
-    ASTPrinter(llvm::raw_ostream* o = NULL, bool Dump = false)
+    ASTPrinter(raw_ostream* o = NULL, bool Dump = false)
       : Out(o? *o : llvm::outs()), Dump(Dump) { }
 
     virtual void HandleTranslationUnit(ASTContext &Context) {
-      PrintingPolicy Policy = Context.PrintingPolicy;
+      PrintingPolicy Policy = Context.getPrintingPolicy();
       Policy.Dump = Dump;
-      Context.getTranslationUnitDecl()->print(Out, Policy);
+      Context.getTranslationUnitDecl()->print(Out, Policy, /*Indentation=*/0,
+                                              /*PrintInstantiation=*/true);
     }
   };
 } // end anonymous namespace
 
-ASTConsumer *clang::CreateASTPrinter(llvm::raw_ostream* out) {
+ASTConsumer *clang::CreateASTPrinter(raw_ostream* out) {
   return new ASTPrinter(out);
 }
 
@@ -95,7 +96,7 @@ ASTConsumer *clang::CreateASTViewer() { return new ASTViewer(); }
 namespace {
 
 class DeclContextPrinter : public ASTConsumer {
-  llvm::raw_ostream& Out;
+  raw_ostream& Out;
 public:
   DeclContextPrinter() : Out(llvm::errs()) {}
 
@@ -117,34 +118,34 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
   case Decl::Namespace: {
     Out << "[namespace] ";
     const NamespaceDecl* ND = cast<NamespaceDecl>(DC);
-    Out << ND;
+    Out << *ND;
     break;
   }
   case Decl::Enum: {
     const EnumDecl* ED = cast<EnumDecl>(DC);
-    if (ED->isDefinition())
+    if (ED->isCompleteDefinition())
       Out << "[enum] ";
     else
       Out << "<enum> ";
-    Out << ED;
+    Out << *ED;
     break;
   }
   case Decl::Record: {
     const RecordDecl* RD = cast<RecordDecl>(DC);
-    if (RD->isDefinition())
+    if (RD->isCompleteDefinition())
       Out << "[struct] ";
     else
       Out << "<struct> ";
-    Out << RD;
+    Out << *RD;
     break;
   }
   case Decl::CXXRecord: {
     const CXXRecordDecl* RD = cast<CXXRecordDecl>(DC);
-    if (RD->isDefinition())
+    if (RD->isCompleteDefinition())
       Out << "[class] ";
     else
       Out << "<class> ";
-    Out << RD << ' ' << DC;
+    Out << *RD << ' ' << DC;
     break;
   }
   case Decl::ObjCMethod:
@@ -177,7 +178,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
       Out << "[function] ";
     else
       Out << "<function> ";
-    Out << FD;
+    Out << *FD;
     // Print the parameters.
     Out << "(";
     bool PrintComma = false;
@@ -187,7 +188,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
         Out << ", ";
       else
         PrintComma = true;
-      Out << *I;
+      Out << **I;
     }
     Out << ")";
     break;
@@ -200,7 +201,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
       Out << "(c++ method) ";
     else
       Out << "<c++ method> ";
-    Out << D;
+    Out << *D;
     // Print the parameters.
     Out << "(";
     bool PrintComma = false;
@@ -210,7 +211,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
         Out << ", ";
       else
         PrintComma = true;
-      Out << *I;
+      Out << **I;
     }
     Out << ")";
 
@@ -230,7 +231,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
       Out << "(c++ ctor) ";
     else
       Out << "<c++ ctor> ";
-    Out << D;
+    Out << *D;
     // Print the parameters.
     Out << "(";
     bool PrintComma = false;
@@ -240,7 +241,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
         Out << ", ";
       else
         PrintComma = true;
-      Out << *I;
+      Out << **I;
     }
     Out << ")";
 
@@ -259,7 +260,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
       Out << "(c++ dtor) ";
     else
       Out << "<c++ dtor> ";
-    Out << D;
+    Out << *D;
     // Check the semantic DC.
     const DeclContext* SemaDC = D->getDeclContext();
     const DeclContext* LexicalDC = D->getLexicalDeclContext();
@@ -275,7 +276,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
       Out << "(c++ conversion) ";
     else
       Out << "<c++ conversion> ";
-    Out << D;
+    Out << *D;
     // Check the semantic DC.
     const DeclContext* SemaDC = D->getDeclContext();
     const DeclContext* LexicalDC = D->getLexicalDeclContext();
@@ -285,7 +286,7 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
   }
 
   default:
-    assert(0 && "a decl that inherits DeclContext isn't handled");
+    llvm_unreachable("a decl that inherits DeclContext isn't handled");
   }
 
   Out << "\n";
@@ -322,53 +323,53 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
     }
     case Decl::IndirectField: {
       IndirectFieldDecl* IFD = cast<IndirectFieldDecl>(*I);
-      Out << "<IndirectField> " << IFD << '\n';
+      Out << "<IndirectField> " << *IFD << '\n';
       break;
     }
     case Decl::Label: {
       LabelDecl *LD = cast<LabelDecl>(*I);
-      Out << "<Label> " << LD << '\n';
+      Out << "<Label> " << *LD << '\n';
       break;
     }
     case Decl::Field: {
       FieldDecl *FD = cast<FieldDecl>(*I);
-      Out << "<field> " << FD << '\n';
+      Out << "<field> " << *FD << '\n';
       break;
     }
     case Decl::Typedef:
     case Decl::TypeAlias: {
       TypedefNameDecl* TD = cast<TypedefNameDecl>(*I);
-      Out << "<typedef> " << TD << '\n';
+      Out << "<typedef> " << *TD << '\n';
       break;
     }
     case Decl::EnumConstant: {
       EnumConstantDecl* ECD = cast<EnumConstantDecl>(*I);
-      Out << "<enum constant> " << ECD << '\n';
+      Out << "<enum constant> " << *ECD << '\n';
       break;
     }
     case Decl::Var: {
       VarDecl* VD = cast<VarDecl>(*I);
-      Out << "<var> " << VD << '\n';
+      Out << "<var> " << *VD << '\n';
       break;
     }
     case Decl::ImplicitParam: {
       ImplicitParamDecl* IPD = cast<ImplicitParamDecl>(*I);
-      Out << "<implicit parameter> " << IPD << '\n';
+      Out << "<implicit parameter> " << *IPD << '\n';
       break;
     }
     case Decl::ParmVar: {
       ParmVarDecl* PVD = cast<ParmVarDecl>(*I);
-      Out << "<parameter> " << PVD << '\n';
+      Out << "<parameter> " << *PVD << '\n';
       break;
     }
     case Decl::ObjCProperty: {
       ObjCPropertyDecl* OPD = cast<ObjCPropertyDecl>(*I);
-      Out << "<objc property> " << OPD << '\n';
+      Out << "<objc property> " << *OPD << '\n';
       break;
     }
     case Decl::FunctionTemplate: {
       FunctionTemplateDecl* FTD = cast<FunctionTemplateDecl>(*I);
-      Out << "<function template> " << FTD << '\n';
+      Out << "<function template> " << *FTD << '\n';
       break;
     }
     case Decl::FileScopeAsm: {
@@ -381,17 +382,17 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
     }
     case Decl::NamespaceAlias: {
       NamespaceAliasDecl* NAD = cast<NamespaceAliasDecl>(*I);
-      Out << "<namespace alias> " << NAD << '\n';
+      Out << "<namespace alias> " << *NAD << '\n';
       break;
     }
     case Decl::ClassTemplate: {
       ClassTemplateDecl *CTD = cast<ClassTemplateDecl>(*I);
-      Out << "<class template> " << CTD << '\n';
+      Out << "<class template> " << *CTD << '\n';
       break;
     }
     default:
       Out << "DeclKind: " << DK << '"' << *I << "\"\n";
-      assert(0 && "decl unhandled");
+      llvm_unreachable("decl unhandled");
     }
   }
 }
@@ -404,10 +405,10 @@ ASTConsumer *clang::CreateDeclContextPrinter() {
 
 namespace {
 class ASTDumpXML : public ASTConsumer {
-  llvm::raw_ostream &OS;
+  raw_ostream &OS;
 
 public:
-  ASTDumpXML(llvm::raw_ostream &OS) : OS(OS) {}
+  ASTDumpXML(raw_ostream &OS) : OS(OS) {}
 
   void HandleTranslationUnit(ASTContext &C) {
     C.getTranslationUnitDecl()->dumpXML(OS);
@@ -415,6 +416,6 @@ class ASTDumpXML : public ASTConsumer {
 };
 }
 
-ASTConsumer *clang::CreateASTDumperXML(llvm::raw_ostream &OS) {
+ASTConsumer *clang::CreateASTDumperXML(raw_ostream &OS) {
   return new ASTDumpXML(OS);
 }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/ASTMerge.cpp b/contrib/llvm/tools/clang/lib/Frontend/ASTMerge.cpp
index 3905b99b02a4..cb195d11fbca 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/ASTMerge.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/ASTMerge.cpp
@@ -17,12 +17,12 @@
 using namespace clang;
 
 ASTConsumer *ASTMergeAction::CreateASTConsumer(CompilerInstance &CI,
-                                               llvm::StringRef InFile) {
+                                               StringRef InFile) {
   return AdaptedAction->CreateASTConsumer(CI, InFile);
 }
 
 bool ASTMergeAction::BeginSourceFileAction(CompilerInstance &CI,
-                                           llvm::StringRef Filename) {
+                                           StringRef Filename) {
   // FIXME: This is a hack. We need a better way to communicate the
   // AST file, compiler instance, and file name than member variables
   // of FrontendAction.
@@ -41,8 +41,8 @@ void ASTMergeAction::ExecuteAction() {
   llvm::IntrusiveRefCntPtr<DiagnosticIDs>
       DiagIDs(CI.getDiagnostics().getDiagnosticIDs());
   for (unsigned I = 0, N = ASTFiles.size(); I != N; ++I) {
-    llvm::IntrusiveRefCntPtr<Diagnostic>
-        Diags(new Diagnostic(DiagIDs, CI.getDiagnostics().getClient(),
+    llvm::IntrusiveRefCntPtr<DiagnosticsEngine>
+        Diags(new DiagnosticsEngine(DiagIDs, CI.getDiagnostics().getClient(),
                              /*ShouldOwnClient=*/false));
     ASTUnit *Unit = ASTUnit::LoadFromASTFile(ASTFiles[I], Diags,
                                              CI.getFileSystemOpts(), false);
@@ -93,8 +93,8 @@ bool ASTMergeAction::usesPreprocessorOnly() const {
   return AdaptedAction->usesPreprocessorOnly();
 }
 
-bool ASTMergeAction::usesCompleteTranslationUnit() {
-  return AdaptedAction->usesCompleteTranslationUnit();
+TranslationUnitKind ASTMergeAction::getTranslationUnitKind() {
+  return AdaptedAction->getTranslationUnitKind();
 }
 
 bool ASTMergeAction::hasPCHSupport() const {
diff --git a/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp b/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp
index 5b0a52c65b84..032adf3876a4 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp
@@ -29,7 +29,6 @@
 #include "clang/Frontend/FrontendOptions.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Serialization/ASTReader.h"
-#include "clang/Serialization/ASTSerializationListener.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
@@ -45,6 +44,8 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Timer.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Mutex.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include <cstdlib>
 #include <cstdio>
@@ -65,7 +66,7 @@ namespace {
         Start = TimeRecord::getCurrentTime();
     }
 
-    void setOutput(const llvm::Twine &Output) {
+    void setOutput(const Twine &Output) {
       if (WantTiming)
         this->Output = Output.str();
     }
@@ -96,10 +97,9 @@ static llvm::sys::cas_flag ActiveASTUnitObjects;
 ASTUnit::ASTUnit(bool _MainFileIsAST)
   : OnlyLocalDecls(false), CaptureDiagnostics(false),
     MainFileIsAST(_MainFileIsAST), 
-    CompleteTranslationUnit(true), WantTiming(getenv("LIBCLANG_TIMING")),
+    TUKind(TU_Complete), WantTiming(getenv("LIBCLANG_TIMING")),
     OwnsRemappedFileBuffers(true),
     NumStoredDiagnosticsFromDriver(0),
-    ConcurrencyCheckValue(CheckUnlocked), 
     PreambleRebuildCounter(0), SavedMainFileBuffer(0), PreambleBuffer(0),
     ShouldCacheCodeCompletionResults(false),
     NestedMacroExpansions(true),
@@ -114,7 +114,6 @@ ASTUnit::ASTUnit(bool _MainFileIsAST)
 }
 
 ASTUnit::~ASTUnit() {
-  ConcurrencyCheckValue = CheckLocked;
   CleanTemporaryFiles();
   if (!PreambleFile.empty())
     llvm::sys::Path(PreambleFile).eraseFromDisk();
@@ -185,7 +184,7 @@ static unsigned getDeclShowContexts(NamedDecl *ND,
     
     // In Objective-C, you can only be a subclass of another Objective-C class
     if (isa<ObjCInterfaceDecl>(ND))
-      Contexts |= (1 << (CodeCompletionContext::CCC_ObjCSuperclass - 1));
+      Contexts |= (1 << (CodeCompletionContext::CCC_ObjCInterfaceName - 1));
 
     // Deal with tag names.
     if (isa<EnumDecl>(ND)) {
@@ -236,7 +235,7 @@ void ASTUnit::CacheCodeCompletionResults() {
   
   // Gather the set of global code completions.
   typedef CodeCompletionResult Result;
-  llvm::SmallVector<Result, 8> Results;
+  SmallVector<Result, 8> Results;
   CachedCompletionAllocator = new GlobalCodeCompletionAllocator;
   TheSema->GatherGlobalCodeCompletions(*CachedCompletionAllocator, Results);
   
@@ -375,33 +374,61 @@ namespace {
 /// \brief Gathers information from ASTReader that will be used to initialize
 /// a Preprocessor.
 class ASTInfoCollector : public ASTReaderListener {
+  Preprocessor &PP;
+  ASTContext &Context;
   LangOptions &LangOpt;
   HeaderSearch &HSI;
-  std::string &TargetTriple;
+  llvm::IntrusiveRefCntPtr<TargetInfo> &Target;
   std::string &Predefines;
   unsigned &Counter;
 
   unsigned NumHeaderInfos;
 
+  bool InitializedLanguage;
 public:
-  ASTInfoCollector(LangOptions &LangOpt, HeaderSearch &HSI,
-                   std::string &TargetTriple, std::string &Predefines,
+  ASTInfoCollector(Preprocessor &PP, ASTContext &Context, LangOptions &LangOpt, 
+                   HeaderSearch &HSI,
+                   llvm::IntrusiveRefCntPtr<TargetInfo> &Target,
+                   std::string &Predefines,
                    unsigned &Counter)
-    : LangOpt(LangOpt), HSI(HSI), TargetTriple(TargetTriple),
-      Predefines(Predefines), Counter(Counter), NumHeaderInfos(0) {}
+    : PP(PP), Context(Context), LangOpt(LangOpt), HSI(HSI), Target(Target),
+      Predefines(Predefines), Counter(Counter), NumHeaderInfos(0),
+      InitializedLanguage(false) {}
 
   virtual bool ReadLanguageOptions(const LangOptions &LangOpts) {
+    if (InitializedLanguage)
+      return false;
+    
     LangOpt = LangOpts;
+    
+    // Initialize the preprocessor.
+    PP.Initialize(*Target);
+    
+    // Initialize the ASTContext
+    Context.InitBuiltinTypes(*Target);
+    
+    InitializedLanguage = true;
     return false;
   }
 
-  virtual bool ReadTargetTriple(llvm::StringRef Triple) {
-    TargetTriple = Triple;
+  virtual bool ReadTargetTriple(StringRef Triple) {
+    // If we've already initialized the target, don't do it again.
+    if (Target)
+      return false;
+    
+    // FIXME: This is broken, we should store the TargetOptions in the AST file.
+    TargetOptions TargetOpts;
+    TargetOpts.ABI = "";
+    TargetOpts.CXXABI = "";
+    TargetOpts.CPU = "";
+    TargetOpts.Features.clear();
+    TargetOpts.Triple = Triple;
+    Target = TargetInfo::CreateTargetInfo(PP.getDiagnostics(), TargetOpts);
     return false;
   }
 
   virtual bool ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
-                                    llvm::StringRef OriginalFileName,
+                                    StringRef OriginalFileName,
                                     std::string &SuggestedPredefines,
                                     FileManager &FileMgr) {
     Predefines = Buffers[0].Data;
@@ -420,28 +447,34 @@ class ASTInfoCollector : public ASTReaderListener {
   }
 };
 
-class StoredDiagnosticClient : public DiagnosticClient {
-  llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiags;
+class StoredDiagnosticConsumer : public DiagnosticConsumer {
+  SmallVectorImpl<StoredDiagnostic> &StoredDiags;
   
 public:
-  explicit StoredDiagnosticClient(
-                          llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiags)
+  explicit StoredDiagnosticConsumer(
+                          SmallVectorImpl<StoredDiagnostic> &StoredDiags)
     : StoredDiags(StoredDiags) { }
   
-  virtual void HandleDiagnostic(Diagnostic::Level Level,
-                                const DiagnosticInfo &Info);
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level Level,
+                                const Diagnostic &Info);
+  
+  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
+    // Just drop any diagnostics that come from cloned consumers; they'll
+    // have different source managers anyway.
+    return new IgnoringDiagConsumer();
+  }
 };
 
 /// \brief RAII object that optionally captures diagnostics, if
 /// there is no diagnostic client to capture them already.
 class CaptureDroppedDiagnostics {
-  Diagnostic &Diags;
-  StoredDiagnosticClient Client;
-  DiagnosticClient *PreviousClient;
+  DiagnosticsEngine &Diags;
+  StoredDiagnosticConsumer Client;
+  DiagnosticConsumer *PreviousClient;
 
 public:
-  CaptureDroppedDiagnostics(bool RequestCapture, Diagnostic &Diags, 
-                          llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiags)
+  CaptureDroppedDiagnostics(bool RequestCapture, DiagnosticsEngine &Diags,
+                          SmallVectorImpl<StoredDiagnostic> &StoredDiags)
     : Diags(Diags), Client(StoredDiags), PreviousClient(0)
   {
     if (RequestCapture || Diags.getClient() == 0) {
@@ -460,10 +493,10 @@ class CaptureDroppedDiagnostics {
 
 } // anonymous namespace
 
-void StoredDiagnosticClient::HandleDiagnostic(Diagnostic::Level Level,
-                                              const DiagnosticInfo &Info) {
+void StoredDiagnosticConsumer::HandleDiagnostic(DiagnosticsEngine::Level Level,
+                                              const Diagnostic &Info) {
   // Default implementation (Warnings/errors count).
-  DiagnosticClient::HandleDiagnostic(Level, Info);
+  DiagnosticConsumer::HandleDiagnostic(Level, Info);
 
   StoredDiags.push_back(StoredDiagnostic(Level, Info));
 }
@@ -472,37 +505,32 @@ const std::string &ASTUnit::getOriginalSourceFileName() {
   return OriginalSourceFile;
 }
 
-const std::string &ASTUnit::getASTFileName() {
-  assert(isMainFileAST() && "Not an ASTUnit from an AST file!");
-  return static_cast<ASTReader *>(Ctx->getExternalSource())->getFileName();
-}
-
-llvm::MemoryBuffer *ASTUnit::getBufferForFile(llvm::StringRef Filename,
+llvm::MemoryBuffer *ASTUnit::getBufferForFile(StringRef Filename,
                                               std::string *ErrorStr) {
   assert(FileMgr);
   return FileMgr->getBufferForFile(Filename, ErrorStr);
 }
 
 /// \brief Configure the diagnostics object for use with ASTUnit.
-void ASTUnit::ConfigureDiags(llvm::IntrusiveRefCntPtr<Diagnostic> &Diags,
+void ASTUnit::ConfigureDiags(llvm::IntrusiveRefCntPtr<DiagnosticsEngine> &Diags,
                              const char **ArgBegin, const char **ArgEnd,
                              ASTUnit &AST, bool CaptureDiagnostics) {
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
     DiagnosticOptions DiagOpts;
-    DiagnosticClient *Client = 0;
+    DiagnosticConsumer *Client = 0;
     if (CaptureDiagnostics)
-      Client = new StoredDiagnosticClient(AST.StoredDiagnostics);
+      Client = new StoredDiagnosticConsumer(AST.StoredDiagnostics);
     Diags = CompilerInstance::createDiagnostics(DiagOpts, ArgEnd- ArgBegin, 
                                                 ArgBegin, Client);
   } else if (CaptureDiagnostics) {
-    Diags->setClient(new StoredDiagnosticClient(AST.StoredDiagnostics));
+    Diags->setClient(new StoredDiagnosticConsumer(AST.StoredDiagnostics));
   }
 }
 
 ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
-                                  llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
                                   const FileSystemOptions &FileSystemOpts,
                                   bool OnlyLocalDecls,
                                   RemappedFile *RemappedFiles,
@@ -513,8 +541,8 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
     ASTUnitCleanup(AST.get());
-  llvm::CrashRecoveryContextCleanupRegistrar<Diagnostic,
-    llvm::CrashRecoveryContextReleaseRefCleanup<Diagnostic> >
+  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
+    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
     DiagCleanup(Diags.getPtr());
 
   ConfigureDiags(Diags, 0, 0, *AST, CaptureDiagnostics);
@@ -576,25 +604,41 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   
   // Gather Info for preprocessor construction later on.
 
-  LangOptions LangInfo;
   HeaderSearch &HeaderInfo = *AST->HeaderInfo.get();
-  std::string TargetTriple;
   std::string Predefines;
   unsigned Counter;
 
   llvm::OwningPtr<ASTReader> Reader;
 
-  Reader.reset(new ASTReader(AST->getSourceManager(), AST->getFileManager(),
-                             AST->getDiagnostics()));
+  AST->PP = new Preprocessor(AST->getDiagnostics(), AST->ASTFileLangOpts, 
+                             /*Target=*/0, AST->getSourceManager(), HeaderInfo, 
+                             *AST, 
+                             /*IILookup=*/0,
+                             /*OwnsHeaderSearch=*/false,
+                             /*DelayInitialization=*/true);
+  Preprocessor &PP = *AST->PP;
+
+  AST->Ctx = new ASTContext(AST->ASTFileLangOpts,
+                            AST->getSourceManager(),
+                            /*Target=*/0,
+                            PP.getIdentifierTable(),
+                            PP.getSelectorTable(),
+                            PP.getBuiltinInfo(),
+                            /* size_reserve = */0,
+                            /*DelayInitialization=*/true);
+  ASTContext &Context = *AST->Ctx;
+
+  Reader.reset(new ASTReader(PP, Context));
   
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<ASTReader>
     ReaderCleanup(Reader.get());
 
-  Reader->setListener(new ASTInfoCollector(LangInfo, HeaderInfo, TargetTriple,
-                                           Predefines, Counter));
+  Reader->setListener(new ASTInfoCollector(*AST->PP, Context,
+                                           AST->ASTFileLangOpts, HeaderInfo, 
+                                           AST->Target, Predefines, Counter));
 
-  switch (Reader->ReadAST(Filename, ASTReader::MainFile)) {
+  switch (Reader->ReadAST(Filename, serialization::MK_MainFile)) {
   case ASTReader::Success:
     break;
 
@@ -606,39 +650,8 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
 
   AST->OriginalSourceFile = Reader->getOriginalSourceFile();
 
-  // AST file loaded successfully. Now create the preprocessor.
-
-  // Get information about the target being compiled for.
-  //
-  // FIXME: This is broken, we should store the TargetOptions in the AST file.
-  TargetOptions TargetOpts;
-  TargetOpts.ABI = "";
-  TargetOpts.CXXABI = "";
-  TargetOpts.CPU = "";
-  TargetOpts.Features.clear();
-  TargetOpts.Triple = TargetTriple;
-  AST->Target = TargetInfo::CreateTargetInfo(AST->getDiagnostics(),
-                                             TargetOpts);
-  AST->PP = new Preprocessor(AST->getDiagnostics(), LangInfo, *AST->Target,
-                             AST->getSourceManager(), HeaderInfo);
-  Preprocessor &PP = *AST->PP;
-
   PP.setPredefines(Reader->getSuggestedPredefines());
   PP.setCounterValue(Counter);
-  Reader->setPreprocessor(PP);
-
-  // Create and initialize the ASTContext.
-
-  AST->Ctx = new ASTContext(LangInfo,
-                            AST->getSourceManager(),
-                            *AST->Target,
-                            PP.getIdentifierTable(),
-                            PP.getSelectorTable(),
-                            PP.getBuiltinInfo(),
-                            /* size_reserve = */0);
-  ASTContext &Context = *AST->Ctx;
-
-  Reader->InitializeContext(Context);
 
   // Attach the AST reader to the AST context as an external AST
   // source, so that declarations will be deserialized from the
@@ -710,10 +723,8 @@ void AddTopLevelDeclarationToHash(Decl *D, unsigned &Hash) {
     return;
   }
   
-  if (ObjCClassDecl *Class = llvm::dyn_cast<ObjCClassDecl>(D)) {
-    for (ObjCClassDecl::iterator I = Class->begin(), IEnd = Class->end();
-         I != IEnd; ++I)
-      AddTopLevelDeclarationToHash(I->getInterface(), Hash);
+  if (ObjCClassDecl *Class = dyn_cast<ObjCClassDecl>(D)) {
+    AddTopLevelDeclarationToHash(Class->getForwardInterfaceDecl(), Hash);
     return;
   }
 }
@@ -752,7 +763,7 @@ class TopLevelDeclTrackerAction : public ASTFrontendAction {
   ASTUnit &Unit;
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile) {
+                                         StringRef InFile) {
     CI.getPreprocessor().addPPCallbacks(
      new MacroDefinitionTrackerPPCallbacks(Unit.getCurrentTopLevelHashValue()));
     return new TopLevelDeclTrackerConsumer(Unit, 
@@ -763,22 +774,20 @@ class TopLevelDeclTrackerAction : public ASTFrontendAction {
   TopLevelDeclTrackerAction(ASTUnit &_Unit) : Unit(_Unit) {}
 
   virtual bool hasCodeCompletionSupport() const { return false; }
-  virtual bool usesCompleteTranslationUnit()  { 
-    return Unit.isCompleteTranslationUnit(); 
+  virtual TranslationUnitKind getTranslationUnitKind()  { 
+    return Unit.getTranslationUnitKind(); 
   }
 };
 
-class PrecompilePreambleConsumer : public PCHGenerator, 
-                                   public ASTSerializationListener {
+class PrecompilePreambleConsumer : public PCHGenerator {
   ASTUnit &Unit;
   unsigned &Hash;                                   
   std::vector<Decl *> TopLevelDecls;
                                      
 public:
-  PrecompilePreambleConsumer(ASTUnit &Unit,
-                             const Preprocessor &PP, bool Chaining,
-                             const char *isysroot, llvm::raw_ostream *Out)
-    : PCHGenerator(PP, "", Chaining, isysroot, Out), Unit(Unit),
+  PrecompilePreambleConsumer(ASTUnit &Unit, const Preprocessor &PP, 
+                             StringRef isysroot, raw_ostream *Out)
+    : PCHGenerator(PP, "", /*IsModule=*/false, isysroot, Out), Unit(Unit),
       Hash(Unit.getCurrentTopLevelHashValue()) {
     Hash = 0;
   }
@@ -809,15 +818,6 @@ class PrecompilePreambleConsumer : public PCHGenerator,
                                       getWriter().getDeclID(TopLevelDecls[I]));
     }
   }
-                                     
-  virtual void SerializedPreprocessedEntity(PreprocessedEntity *Entity,
-                                            uint64_t Offset) {
-    Unit.addPreprocessedEntityFromPreamble(Offset);
-  }
-                                     
-  virtual ASTSerializationListener *GetASTSerializationListener() {
-    return this;
-  }
 };
 
 class PrecompilePreambleAction : public ASTFrontendAction {
@@ -827,27 +827,27 @@ class PrecompilePreambleAction : public ASTFrontendAction {
   explicit PrecompilePreambleAction(ASTUnit &Unit) : Unit(Unit) {}
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
-                                         llvm::StringRef InFile) {
+                                         StringRef InFile) {
     std::string Sysroot;
     std::string OutputFile;
-    llvm::raw_ostream *OS = 0;
-    bool Chaining;
+    raw_ostream *OS = 0;
     if (GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot,
                                                        OutputFile,
-                                                       OS, Chaining))
+                                                       OS))
       return 0;
     
-    const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
-                             Sysroot.c_str() : 0;  
+    if (!CI.getFrontendOpts().RelocatablePCH)
+      Sysroot.clear();
+
     CI.getPreprocessor().addPPCallbacks(
      new MacroDefinitionTrackerPPCallbacks(Unit.getCurrentTopLevelHashValue()));
-    return new PrecompilePreambleConsumer(Unit, CI.getPreprocessor(), Chaining,
-                                          isysroot, OS);
+    return new PrecompilePreambleConsumer(Unit, CI.getPreprocessor(), Sysroot, 
+                                          OS);
   }
 
   virtual bool hasCodeCompletionSupport() const { return false; }
   virtual bool hasASTFileSupport() const { return false; }
-  virtual bool usesCompleteTranslationUnit() { return false; }
+  virtual TranslationUnitKind getTranslationUnitKind() { return TU_Prefix; }
 };
 
 }
@@ -873,7 +873,10 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
     CICleanup(Clang.get());
 
-  Clang->setInvocation(&*Invocation);
+  llvm::IntrusiveRefCntPtr<CompilerInvocation>
+    CCInvocation(new CompilerInvocation(*Invocation));
+
+  Clang->setInvocation(CCInvocation.getPtr());
   OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].second;
     
   // Set up diagnostics, capturing any diagnostics that would
@@ -913,16 +916,13 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   
   // Clear out old caches and data.
   TopLevelDecls.clear();
-  PreprocessedEntities.clear();
   CleanTemporaryFiles();
-  PreprocessedEntitiesByFile.clear();
 
   if (!OverrideMainBuffer) {
     StoredDiagnostics.erase(
                     StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver,
                             StoredDiagnostics.end());
     TopLevelDeclsInPreamble.clear();
-    PreprocessedEntitiesInPreamble.clear();
   }
 
   // Create a file manager object to provide access to and cache the filesystem.
@@ -936,13 +936,11 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   PreprocessorOptions &PreprocessorOpts = Clang->getPreprocessorOpts();
   PreprocessorOpts.DetailedRecordIncludesNestedMacroExpansions
     = NestedMacroExpansions;
-  std::string PriorImplicitPCHInclude;
   if (OverrideMainBuffer) {
     PreprocessorOpts.addRemappedFile(OriginalSourceFile, OverrideMainBuffer);
     PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size();
     PreprocessorOpts.PrecompiledPreambleBytes.second
                                                     = PreambleEndsAtStartOfLine;
-    PriorImplicitPCHInclude = PreprocessorOpts.ImplicitPCHInclude;
     PreprocessorOpts.ImplicitPCHInclude = PreambleFile;
     PreprocessorOpts.DisablePCHValidation = true;
     
@@ -961,9 +959,6 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
 
     // Keep track of the override buffer;
     SavedMainFileBuffer = OverrideMainBuffer;
-  } else {
-    PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
-    PreprocessorOpts.PrecompiledPreambleBytes.second = false;
   }
   
   llvm::OwningPtr<TopLevelDeclTrackerAction> Act(
@@ -976,7 +971,14 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0].second,
                             Clang->getFrontendOpts().Inputs[0].first))
     goto error;
-  
+
+  if (OverrideMainBuffer) {
+    std::string ModName = PreambleFile;
+    TranslateStoredDiagnostics(Clang->getModuleManager(), ModName,
+                               getSourceManager(), PreambleDiagnostics,
+                               StoredDiagnostics);
+  }
+
   Act->Execute();
   
   // Steal the created target, context, and preprocessor.
@@ -990,21 +992,11 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   
   Act->EndSourceFile();
 
-  // Remove the overridden buffer we used for the preamble.
-  if (OverrideMainBuffer) {
-    PreprocessorOpts.eraseRemappedFile(
-                               PreprocessorOpts.remapped_file_buffer_end() - 1);
-    PreprocessorOpts.ImplicitPCHInclude = PriorImplicitPCHInclude;
-  }
-
   return false;
 
 error:
   // Remove the overridden buffer we used for the preamble.
   if (OverrideMainBuffer) {
-    PreprocessorOpts.eraseRemappedFile(
-                               PreprocessorOpts.remapped_file_buffer_end() - 1);
-    PreprocessorOpts.ImplicitPCHInclude = PriorImplicitPCHInclude;
     delete OverrideMainBuffer;
     SavedMainFileBuffer = 0;
   }
@@ -1041,7 +1033,7 @@ static std::string GetPreamblePCHPath() {
   P.createDirectoryOnDisk(true);
   P.appendComponent("preamble");
   P.appendSuffix("pch");
-  if (P.createTemporaryFileOnDisk())
+  if (P.makeUnique(/*reuse_current=*/false, /*ErrMsg*/0))
     return std::string();
   
   return P.str();
@@ -1118,12 +1110,14 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
     CreatedBuffer = true;
   }
   
-  return std::make_pair(Buffer, Lexer::ComputePreamble(Buffer, MaxLines));
+  return std::make_pair(Buffer, Lexer::ComputePreamble(Buffer,
+                                                       Invocation.getLangOpts(),
+                                                       MaxLines));
 }
 
 static llvm::MemoryBuffer *CreatePaddedMainFileBuffer(llvm::MemoryBuffer *Old,
                                                       unsigned NewSize,
-                                                      llvm::StringRef NewName) {
+                                                      StringRef NewName) {
   llvm::MemoryBuffer *Result
     = llvm::MemoryBuffer::getNewUninitMemBuffer(NewSize, NewName);
   memcpy(const_cast<char*>(Result->getBufferStart()), 
@@ -1170,7 +1164,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> > NewPreamble 
     = ComputePreamble(*PreambleInvocation, MaxLines, CreatedPreambleBuffer);
 
-  // If ComputePreamble() Take ownership of the
+  // If ComputePreamble() Take ownership of the preamble buffer.
   llvm::OwningPtr<llvm::MemoryBuffer> OwnedPreambleBuffer;
   if (CreatedPreambleBuffer)
     OwnedPreambleBuffer.reset(NewPreamble.first);
@@ -1197,7 +1191,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     if (Preamble.size() == NewPreamble.second.first &&
         PreambleEndsAtStartOfLine == NewPreamble.second.second &&
         NewPreamble.first->getBufferSize() < PreambleReservedSize-2 &&
-        memcmp(&Preamble[0], NewPreamble.first->getBufferStart(),
+        memcmp(Preamble.getBufferStart(), NewPreamble.first->getBufferStart(),
                NewPreamble.second.first) == 0) {
       // The preamble has not changed. We may be able to re-use the precompiled
       // preamble.
@@ -1271,10 +1265,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
         ProcessWarningOptions(getDiagnostics(), 
                               PreambleInvocation->getDiagnosticOpts());
         getDiagnostics().setNumWarnings(NumWarningsInPreamble);
-        if (StoredDiagnostics.size() > NumStoredDiagnosticsInPreamble)
-          StoredDiagnostics.erase(
-            StoredDiagnostics.begin() + NumStoredDiagnosticsInPreamble,
-                                  StoredDiagnostics.end());
 
         // Create a version of the main file buffer that is padded to
         // buffer size we reserved when creating the preamble.
@@ -1291,6 +1281,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
 
     // We can't reuse the previously-computed preamble. Build a new one.
     Preamble.clear();
+    PreambleDiagnostics.clear();
     llvm::sys::Path(PreambleFile).eraseFromDisk();
     PreambleRebuildCounter = 1;
   } else if (!AllowRebuild) {
@@ -1332,7 +1323,9 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
 
   // Save the preamble text for later; we'll need to compare against it for
   // subsequent reparses.
-  Preamble.assign(NewPreamble.first->getBufferStart(), 
+  StringRef MainFilename = PreambleInvocation->getFrontendOpts().Inputs[0].second;
+  Preamble.assign(FileMgr->getFile(MainFilename),
+                  NewPreamble.first->getBufferStart(), 
                   NewPreamble.first->getBufferStart() 
                                                   + NewPreamble.second.first);
   PreambleEndsAtStartOfLine = NewPreamble.second.second;
@@ -1353,7 +1346,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   
   // Tell the compiler invocation to generate a temporary precompiled header.
   FrontendOpts.ProgramAction = frontend::GeneratePCH;
-  FrontendOpts.ChainedPCH = true;
   // FIXME: Generate the precompiled header into memory?
   FrontendOpts.OutputFile = PreamblePCHPath;
   PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
@@ -1406,8 +1398,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
                           StoredDiagnostics.end());
   TopLevelDecls.clear();
   TopLevelDeclsInPreamble.clear();
-  PreprocessedEntities.clear();
-  PreprocessedEntitiesInPreamble.clear();
   
   // Create a file manager object to provide access to and cache the filesystem.
   Clang->setFileManager(new FileManager(Clang->getFileSystemOpts()));
@@ -1438,17 +1428,24 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
     Preamble.clear();
     TopLevelDeclsInPreamble.clear();
-    PreprocessedEntities.clear();
-    PreprocessedEntitiesInPreamble.clear();
     PreambleRebuildCounter = DefaultPreambleRebuildInterval;
     PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
     return 0;
   }
   
+  // Transfer any diagnostics generated when parsing the preamble into the set
+  // of preamble diagnostics.
+  PreambleDiagnostics.clear();
+  PreambleDiagnostics.insert(PreambleDiagnostics.end(), 
+                   StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver,
+                             StoredDiagnostics.end());
+  StoredDiagnostics.erase(
+                    StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver,
+                          StoredDiagnostics.end());
+  
   // Keep track of the preamble we precompiled.
   PreambleFile = FrontendOpts.OutputFile;
-  NumStoredDiagnosticsInPreamble = StoredDiagnostics.size();
   NumWarningsInPreamble = getDiagnostics().getNumWarnings();
   
   // Keep track of all of the files that the source manager knows about,
@@ -1501,69 +1498,12 @@ void ASTUnit::RealizeTopLevelDeclsFromPreamble() {
   TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end());
 }
 
-void ASTUnit::RealizePreprocessedEntitiesFromPreamble() {
-  if (!PP)
-    return;
-  
-  PreprocessingRecord *PPRec = PP->getPreprocessingRecord();
-  if (!PPRec)
-    return;
-  
-  ExternalPreprocessingRecordSource *External = PPRec->getExternalSource();
-  if (!External)
-    return;
-
-  for (unsigned I = 0, N = PreprocessedEntitiesInPreamble.size(); I != N; ++I) {
-    if (PreprocessedEntity *PE
-          = External->ReadPreprocessedEntityAtOffset(
-                                            PreprocessedEntitiesInPreamble[I]))
-      PreprocessedEntities.push_back(PE);
-  }
-  
-  if (PreprocessedEntities.empty())
-    return;
-  
-  PreprocessedEntities.insert(PreprocessedEntities.end(), 
-                              PPRec->begin(true), PPRec->end(true));
-}
-
-ASTUnit::pp_entity_iterator ASTUnit::pp_entity_begin() {
-  if (!PreprocessedEntitiesInPreamble.empty() &&
-      PreprocessedEntities.empty())
-    RealizePreprocessedEntitiesFromPreamble();
-  
-  if (PreprocessedEntities.empty())
-    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
-      return PPRec->begin(true);
-  
-  return PreprocessedEntities.begin();
-}
-
-ASTUnit::pp_entity_iterator ASTUnit::pp_entity_end() {
-  if (!PreprocessedEntitiesInPreamble.empty() &&
-      PreprocessedEntities.empty())
-    RealizePreprocessedEntitiesFromPreamble();
-  
-  if (PreprocessedEntities.empty())
-    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
-      return PPRec->end(true);
-  
-  return PreprocessedEntities.end();
-}
-
-unsigned ASTUnit::getMaxPCHLevel() const {
-  if (!getOnlyLocalDecls())
-    return Decl::MaxPCHLevel;
-
-  return 0;
-}
-
-llvm::StringRef ASTUnit::getMainFileName() const {
+StringRef ASTUnit::getMainFileName() const {
   return Invocation->getFrontendOpts().Inputs[0].second;
 }
 
 ASTUnit *ASTUnit::create(CompilerInvocation *CI,
-                         llvm::IntrusiveRefCntPtr<Diagnostic> Diags) {
+                         llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags) {
   llvm::OwningPtr<ASTUnit> AST;
   AST.reset(new ASTUnit(false));
   ConfigureDiags(Diags, 0, 0, *AST, /*CaptureDiagnostics=*/false);
@@ -1571,33 +1511,35 @@ ASTUnit *ASTUnit::create(CompilerInvocation *CI,
   AST->Invocation = CI;
   AST->FileSystemOpts = CI->getFileSystemOpts();
   AST->FileMgr = new FileManager(AST->FileSystemOpts);
-  AST->SourceMgr = new SourceManager(*Diags, *AST->FileMgr);
+  AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr);
 
   return AST.take();
 }
 
 ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
-                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
-                                             ASTFrontendAction *Action) {
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
+                                             ASTFrontendAction *Action,
+                                             ASTUnit *Unit) {
   assert(CI && "A CompilerInvocation is required");
 
-  // Create the AST unit.
-  llvm::OwningPtr<ASTUnit> AST;
-  AST.reset(new ASTUnit(false));
-  ConfigureDiags(Diags, 0, 0, *AST, /*CaptureDiagnostics*/false);
-  AST->Diagnostics = Diags;
+  llvm::OwningPtr<ASTUnit> OwnAST;
+  ASTUnit *AST = Unit;
+  if (!AST) {
+    // Create the AST unit.
+    OwnAST.reset(create(CI, Diags));
+    AST = OwnAST.get();
+  }
+  
   AST->OnlyLocalDecls = false;
   AST->CaptureDiagnostics = false;
-  AST->CompleteTranslationUnit = Action ? Action->usesCompleteTranslationUnit()
-                                        : true;
+  AST->TUKind = Action ? Action->getTranslationUnitKind() : TU_Complete;
   AST->ShouldCacheCodeCompletionResults = false;
-  AST->Invocation = CI;
 
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
-    ASTUnitCleanup(AST.get());
-  llvm::CrashRecoveryContextCleanupRegistrar<Diagnostic,
-    llvm::CrashRecoveryContextReleaseRefCleanup<Diagnostic> >
+    ASTUnitCleanup(OwnAST.get());
+  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
+    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
     DiagCleanup(Diags.getPtr());
 
   // We'll manage file buffers ourselves.
@@ -1643,9 +1585,6 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
          "IR inputs not supported here!");
 
   // Configure the various subsystems.
-  AST->FileSystemOpts = Clang->getFileSystemOpts();
-  AST->FileMgr = new FileManager(AST->FileSystemOpts);
-  AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr);
   AST->TheSema.reset();
   AST->Ctx = 0;
   AST->PP = 0;
@@ -1686,7 +1625,10 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
   
   Act->EndSourceFile();
 
-  return AST.take();
+  if (OwnAST)
+    return OwnAST.take();
+  else
+    return AST;
 }
 
 bool ASTUnit::LoadFromCompilerInvocation(bool PrecompilePreamble) {
@@ -1719,11 +1661,11 @@ bool ASTUnit::LoadFromCompilerInvocation(bool PrecompilePreamble) {
 }
 
 ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
-                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                              llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
                                              bool OnlyLocalDecls,
                                              bool CaptureDiagnostics,
                                              bool PrecompilePreamble,
-                                             bool CompleteTranslationUnit,
+                                             TranslationUnitKind TUKind,
                                              bool CacheCodeCompletionResults,
                                              bool NestedMacroExpansions) {  
   // Create the AST unit.
@@ -1733,7 +1675,7 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   AST->Diagnostics = Diags;
   AST->OnlyLocalDecls = OnlyLocalDecls;
   AST->CaptureDiagnostics = CaptureDiagnostics;
-  AST->CompleteTranslationUnit = CompleteTranslationUnit;
+  AST->TUKind = TUKind;
   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
   AST->Invocation = CI;
   AST->NestedMacroExpansions = NestedMacroExpansions;
@@ -1741,8 +1683,8 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
     ASTUnitCleanup(AST.get());
-  llvm::CrashRecoveryContextCleanupRegistrar<Diagnostic,
-    llvm::CrashRecoveryContextReleaseRefCleanup<Diagnostic> >
+  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
+    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
     DiagCleanup(Diags.getPtr());
 
   return AST->LoadFromCompilerInvocation(PrecompilePreamble)? 0 : AST.take();
@@ -1750,18 +1692,16 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
 
 ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                       const char **ArgEnd,
-                                    llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
-                                      llvm::StringRef ResourceFilesPath,
+                                    llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
+                                      StringRef ResourceFilesPath,
                                       bool OnlyLocalDecls,
                                       bool CaptureDiagnostics,
                                       RemappedFile *RemappedFiles,
                                       unsigned NumRemappedFiles,
                                       bool RemappedFilesKeepOriginalName,
                                       bool PrecompilePreamble,
-                                      bool CompleteTranslationUnit,
+                                      TranslationUnitKind TUKind,
                                       bool CacheCodeCompletionResults,
-                                      bool CXXPrecompilePreamble,
-                                      bool CXXChainedPCH,
                                       bool NestedMacroExpansions) {
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
@@ -1771,17 +1711,18 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                                 ArgBegin);
   }
 
-  llvm::SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
+  SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
   
   llvm::IntrusiveRefCntPtr<CompilerInvocation> CI;
 
   {
+
     CaptureDroppedDiagnostics Capture(CaptureDiagnostics, *Diags, 
                                       StoredDiagnostics);
 
     CI = clang::createInvocationFromCommandLine(
-                        llvm::ArrayRef<const char *>(ArgBegin, ArgEnd-ArgBegin),
-                        Diags);
+                                           llvm::makeArrayRef(ArgBegin, ArgEnd),
+                                           Diags);
     if (!CI)
       return 0;
   }
@@ -1803,16 +1744,6 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   // Override the resources path.
   CI->getHeaderSearchOpts().ResourceDir = ResourceFilesPath;
 
-  // Check whether we should precompile the preamble and/or use chained PCH.
-  // FIXME: This is a temporary hack while we debug C++ chained PCH.
-  if (CI->getLangOpts().CPlusPlus) {
-    PrecompilePreamble = PrecompilePreamble && CXXPrecompilePreamble;
-    
-    if (PrecompilePreamble && !CXXChainedPCH &&
-        !CI->getPreprocessorOpts().ImplicitPCHInclude.empty())
-      PrecompilePreamble = false;
-  }
-  
   // Create the AST unit.
   llvm::OwningPtr<ASTUnit> AST;
   AST.reset(new ASTUnit(false));
@@ -1823,10 +1754,9 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   AST->FileMgr = new FileManager(AST->FileSystemOpts);
   AST->OnlyLocalDecls = OnlyLocalDecls;
   AST->CaptureDiagnostics = CaptureDiagnostics;
-  AST->CompleteTranslationUnit = CompleteTranslationUnit;
+  AST->TUKind = TUKind;
   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
   AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size();
-  AST->NumStoredDiagnosticsInPreamble = StoredDiagnostics.size();
   AST->StoredDiagnostics.swap(StoredDiagnostics);
   AST->Invocation = CI;
   AST->NestedMacroExpansions = NestedMacroExpansions;
@@ -1837,8 +1767,8 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInvocation,
     llvm::CrashRecoveryContextReleaseRefCleanup<CompilerInvocation> >
     CICleanup(CI.getPtr());
-  llvm::CrashRecoveryContextCleanupRegistrar<Diagnostic,
-    llvm::CrashRecoveryContextReleaseRefCleanup<Diagnostic> >
+  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
+    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
     DiagCleanup(Diags.getPtr());
 
   return AST->LoadFromCompilerInvocation(PrecompilePreamble) ? 0 : AST.take();
@@ -1896,6 +1826,10 @@ bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) {
       CurrentTopLevelHashValue != CompletionCacheTopLevelHashValue)
     CacheCodeCompletionResults();
 
+  // We now need to clear out the completion allocator for
+  // clang_getCursorCompletionString; it'll be recreated if necessary.
+  CursorCompletionAllocator = 0;
+  
   return Result;
 }
 
@@ -1985,7 +1919,7 @@ static void CalculateHiddenNames(const CodeCompletionContext &Context,
   case CodeCompletionContext::CCC_Name:
   case CodeCompletionContext::CCC_PotentiallyQualifiedName:
   case CodeCompletionContext::CCC_ParenthesizedExpression:
-  case CodeCompletionContext::CCC_ObjCSuperclass:
+  case CodeCompletionContext::CCC_ObjCInterfaceName:
     break;
     
   case CodeCompletionContext::CCC_EnumTag:
@@ -2052,12 +1986,11 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
   bool AddedResult = false;
   unsigned InContexts  
     = (Context.getKind() == CodeCompletionContext::CCC_Recovery? NormalContexts
-                                            : (1 << (Context.getKind() - 1)));
-
+                                        : (1ULL << (Context.getKind() - 1)));
   // Contains the set of names that are hidden by "local" completion results.
   llvm::StringSet<llvm::BumpPtrAllocator> HiddenNames;
   typedef CodeCompletionResult Result;
-  llvm::SmallVector<Result, 8> AllResults;
+  SmallVector<Result, 8> AllResults;
   for (ASTUnit::cached_completion_iterator 
             C = AST.cached_completion_begin(),
          CEnd = AST.cached_completion_end();
@@ -2139,22 +2072,22 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
 
 
 
-void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
+void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
                            RemappedFile *RemappedFiles, 
                            unsigned NumRemappedFiles,
                            bool IncludeMacros, 
                            bool IncludeCodePatterns,
                            CodeCompleteConsumer &Consumer,
-                           Diagnostic &Diag, LangOptions &LangOpts,
+                           DiagnosticsEngine &Diag, LangOptions &LangOpts,
                            SourceManager &SourceMgr, FileManager &FileMgr,
-                   llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
-             llvm::SmallVectorImpl<const llvm::MemoryBuffer *> &OwnedBuffers) {
+                   SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
+             SmallVectorImpl<const llvm::MemoryBuffer *> &OwnedBuffers) {
   if (!Invocation)
     return;
 
   SimpleTimer CompletionTimer(WantTiming);
   CompletionTimer.setOutput("Code completion @ " + File + ":" +
-                            llvm::Twine(Line) + ":" + llvm::Twine(Column));
+                            Twine(Line) + ":" + Twine(Column));
 
   llvm::IntrusiveRefCntPtr<CompilerInvocation>
     CCInvocation(new CompilerInvocation(*Invocation));
@@ -2252,7 +2185,8 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
     llvm::sys::PathWithStatus MainPath(OriginalSourceFile);
     if (const FileStatus *CompleteFileStatus = CompleteFilePath.getFileStatus())
       if (const FileStatus *MainStatus = MainPath.getFileStatus())
-        if (CompleteFileStatus->getUniqueID() == MainStatus->getUniqueID())
+        if (CompleteFileStatus->getUniqueID() == MainStatus->getUniqueID() &&
+            Line > 1)
           OverrideMainBuffer
             = getMainBufferWithPrecompiledPreamble(*CCInvocation, false, 
                                                    Line - 1);
@@ -2272,17 +2206,6 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
     PreprocessorOpts.ImplicitPCHInclude = PreambleFile;
     PreprocessorOpts.DisablePCHValidation = true;
     
-    // The stored diagnostics have the old source manager. Copy them
-    // to our output set of stored diagnostics, updating the source
-    // manager to the one we were given.
-    for (unsigned I = NumStoredDiagnosticsFromDriver, 
-                  N = this->StoredDiagnostics.size(); 
-         I < N; ++I) {
-      StoredDiagnostics.push_back(this->StoredDiagnostics[I]);
-      FullSourceLoc Loc(StoredDiagnostics[I].getLocation(), SourceMgr);
-      StoredDiagnostics[I].setLocation(Loc);
-    }
-
     OwnedBuffers.push_back(OverrideMainBuffer);
   } else {
     PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
@@ -2296,36 +2219,58 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
   Act.reset(new SyntaxOnlyAction);
   if (Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0].second,
                            Clang->getFrontendOpts().Inputs[0].first)) {
+    if (OverrideMainBuffer) {
+      std::string ModName = PreambleFile;
+      TranslateStoredDiagnostics(Clang->getModuleManager(), ModName,
+                                 getSourceManager(), PreambleDiagnostics,
+                                 StoredDiagnostics);
+    }
     Act->Execute();
     Act->EndSourceFile();
   }
 }
 
-CXSaveError ASTUnit::Save(llvm::StringRef File) {
+CXSaveError ASTUnit::Save(StringRef File) {
   if (getDiagnostics().hasUnrecoverableErrorOccurred())
     return CXSaveError_TranslationErrors;
-  
+
+  // Write to a temporary file and later rename it to the actual file, to avoid
+  // possible race conditions.
+  llvm::SmallString<128> TempPath;
+  TempPath = File;
+  TempPath += "-%%%%%%%%";
+  int fd;
+  if (llvm::sys::fs::unique_file(TempPath.str(), fd, TempPath,
+                                 /*makeAbsolute=*/false))
+    return CXSaveError_Unknown;
+
   // FIXME: Can we somehow regenerate the stat cache here, or do we need to 
   // unconditionally create a stat cache when we parse the file?
-  std::string ErrorInfo;
-  llvm::raw_fd_ostream Out(File.str().c_str(), ErrorInfo,
-                           llvm::raw_fd_ostream::F_Binary);
-  if (!ErrorInfo.empty() || Out.has_error())
-    return CXSaveError_Unknown;
+  llvm::raw_fd_ostream Out(fd, /*shouldClose=*/true);
 
   serialize(Out);
   Out.close();
-  return Out.has_error()? CXSaveError_Unknown : CXSaveError_None;
+  if (Out.has_error())
+    return CXSaveError_Unknown;
+
+  if (llvm::error_code ec = llvm::sys::fs::rename(TempPath.str(), File)) {
+    bool exists;
+    llvm::sys::fs::remove(TempPath.str(), exists);
+    return CXSaveError_Unknown;
+  }
+
+  return CXSaveError_None;
 }
 
-bool ASTUnit::serialize(llvm::raw_ostream &OS) {
+bool ASTUnit::serialize(raw_ostream &OS) {
   if (getDiagnostics().hasErrorOccurred())
     return true;
 
   std::vector<unsigned char> Buffer;
   llvm::BitstreamWriter Stream(Buffer);
   ASTWriter Writer(Stream);
-  Writer.WriteAST(getSema(), 0, std::string(), 0);
+  // FIXME: Handle modules
+  Writer.WriteAST(getSema(), 0, std::string(), /*IsModule=*/false, "");
   
   // Write the generated bitstream to "Out".
   if (!Buffer.empty())
@@ -2333,3 +2278,168 @@ bool ASTUnit::serialize(llvm::raw_ostream &OS) {
 
   return false;
 }
+
+typedef ContinuousRangeMap<unsigned, int, 2> SLocRemap;
+
+static void TranslateSLoc(SourceLocation &L, SLocRemap &Remap) {
+  unsigned Raw = L.getRawEncoding();
+  const unsigned MacroBit = 1U << 31;
+  L = SourceLocation::getFromRawEncoding((Raw & MacroBit) |
+      ((Raw & ~MacroBit) + Remap.find(Raw & ~MacroBit)->second));
+}
+
+void ASTUnit::TranslateStoredDiagnostics(
+                          ASTReader *MMan,
+                          StringRef ModName,
+                          SourceManager &SrcMgr,
+                          const SmallVectorImpl<StoredDiagnostic> &Diags,
+                          SmallVectorImpl<StoredDiagnostic> &Out) {
+  // The stored diagnostic has the old source manager in it; update
+  // the locations to refer into the new source manager. We also need to remap
+  // all the locations to the new view. This includes the diag location, any
+  // associated source ranges, and the source ranges of associated fix-its.
+  // FIXME: There should be a cleaner way to do this.
+
+  SmallVector<StoredDiagnostic, 4> Result;
+  Result.reserve(Diags.size());
+  assert(MMan && "Don't have a module manager");
+  serialization::Module *Mod = MMan->ModuleMgr.lookup(ModName);
+  assert(Mod && "Don't have preamble module");
+  SLocRemap &Remap = Mod->SLocRemap;
+  for (unsigned I = 0, N = Diags.size(); I != N; ++I) {
+    // Rebuild the StoredDiagnostic.
+    const StoredDiagnostic &SD = Diags[I];
+    SourceLocation L = SD.getLocation();
+    TranslateSLoc(L, Remap);
+    FullSourceLoc Loc(L, SrcMgr);
+
+    SmallVector<CharSourceRange, 4> Ranges;
+    Ranges.reserve(SD.range_size());
+    for (StoredDiagnostic::range_iterator I = SD.range_begin(),
+                                          E = SD.range_end();
+         I != E; ++I) {
+      SourceLocation BL = I->getBegin();
+      TranslateSLoc(BL, Remap);
+      SourceLocation EL = I->getEnd();
+      TranslateSLoc(EL, Remap);
+      Ranges.push_back(CharSourceRange(SourceRange(BL, EL), I->isTokenRange()));
+    }
+
+    SmallVector<FixItHint, 2> FixIts;
+    FixIts.reserve(SD.fixit_size());
+    for (StoredDiagnostic::fixit_iterator I = SD.fixit_begin(),
+                                          E = SD.fixit_end();
+         I != E; ++I) {
+      FixIts.push_back(FixItHint());
+      FixItHint &FH = FixIts.back();
+      FH.CodeToInsert = I->CodeToInsert;
+      SourceLocation BL = I->RemoveRange.getBegin();
+      TranslateSLoc(BL, Remap);
+      SourceLocation EL = I->RemoveRange.getEnd();
+      TranslateSLoc(EL, Remap);
+      FH.RemoveRange = CharSourceRange(SourceRange(BL, EL),
+                                       I->RemoveRange.isTokenRange());
+    }
+
+    Result.push_back(StoredDiagnostic(SD.getLevel(), SD.getID(), 
+                                      SD.getMessage(), Loc, Ranges, FixIts));
+  }
+  Result.swap(Out);
+}
+
+SourceLocation ASTUnit::getLocation(const FileEntry *File,
+                                    unsigned Line, unsigned Col) const {
+  const SourceManager &SM = getSourceManager();
+  SourceLocation Loc = SM.translateFileLineCol(File, Line, Col);
+  return SM.getMacroArgExpandedLocation(Loc);
+}
+
+SourceLocation ASTUnit::getLocation(const FileEntry *File,
+                                    unsigned Offset) const {
+  const SourceManager &SM = getSourceManager();
+  SourceLocation FileLoc = SM.translateFileLineCol(File, 1, 1);
+  return SM.getMacroArgExpandedLocation(FileLoc.getLocWithOffset(Offset));
+}
+
+/// \brief If \arg Loc is a loaded location from the preamble, returns
+/// the corresponding local location of the main file, otherwise it returns
+/// \arg Loc.
+SourceLocation ASTUnit::mapLocationFromPreamble(SourceLocation Loc) {
+  FileID PreambleID;
+  if (SourceMgr)
+    PreambleID = SourceMgr->getPreambleFileID();
+
+  if (Loc.isInvalid() || Preamble.empty() || PreambleID.isInvalid())
+    return Loc;
+
+  unsigned Offs;
+  if (SourceMgr->isInFileID(Loc, PreambleID, &Offs) && Offs < Preamble.size()) {
+    SourceLocation FileLoc
+        = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
+    return FileLoc.getLocWithOffset(Offs);
+  }
+
+  return Loc;
+}
+
+/// \brief If \arg Loc is a local location of the main file but inside the
+/// preamble chunk, returns the corresponding loaded location from the
+/// preamble, otherwise it returns \arg Loc.
+SourceLocation ASTUnit::mapLocationToPreamble(SourceLocation Loc) {
+  FileID PreambleID;
+  if (SourceMgr)
+    PreambleID = SourceMgr->getPreambleFileID();
+
+  if (Loc.isInvalid() || Preamble.empty() || PreambleID.isInvalid())
+    return Loc;
+
+  unsigned Offs;
+  if (SourceMgr->isInFileID(Loc, SourceMgr->getMainFileID(), &Offs) &&
+      Offs < Preamble.size()) {
+    SourceLocation FileLoc = SourceMgr->getLocForStartOfFile(PreambleID);
+    return FileLoc.getLocWithOffset(Offs);
+  }
+
+  return Loc;
+}
+
+void ASTUnit::PreambleData::countLines() const {
+  NumLines = 0;
+  if (empty())
+    return;
+
+  for (std::vector<char>::const_iterator
+         I = Buffer.begin(), E = Buffer.end(); I != E; ++I) {
+    if (*I == '\n')
+      ++NumLines;
+  }
+  if (Buffer.back() != '\n')
+    ++NumLines;
+}
+
+#ifndef NDEBUG
+ASTUnit::ConcurrencyState::ConcurrencyState() {
+  Mutex = new llvm::sys::MutexImpl(/*recursive=*/true);
+}
+
+ASTUnit::ConcurrencyState::~ConcurrencyState() {
+  delete static_cast<llvm::sys::MutexImpl *>(Mutex);
+}
+
+void ASTUnit::ConcurrencyState::start() {
+  bool acquired = static_cast<llvm::sys::MutexImpl *>(Mutex)->tryacquire();
+  assert(acquired && "Concurrent access to ASTUnit!");
+}
+
+void ASTUnit::ConcurrencyState::finish() {
+  static_cast<llvm::sys::MutexImpl *>(Mutex)->release();
+}
+
+#else // NDEBUG
+
+ASTUnit::ConcurrencyState::ConcurrencyState() {}
+ASTUnit::ConcurrencyState::~ConcurrencyState() {}
+void ASTUnit::ConcurrencyState::start() {}
+void ASTUnit::ConcurrencyState::finish() {}
+
+#endif
diff --git a/contrib/llvm/tools/clang/lib/Frontend/CacheTokens.cpp b/contrib/llvm/tools/clang/lib/Frontend/CacheTokens.cpp
index 20b51893fcc2..8195445fe0e4 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/CacheTokens.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/CacheTokens.cpp
@@ -71,13 +71,13 @@ class PTHEntryKeyVariant {
 
   bool isFile() const { return Kind == IsFE; }
 
-  llvm::StringRef getString() const {
+  StringRef getString() const {
     return Kind == IsFE ? FE->getName() : Path;
   }
 
   unsigned getKind() const { return (unsigned) Kind; }
 
-  void EmitData(llvm::raw_ostream& Out) {
+  void EmitData(raw_ostream& Out) {
     switch (Kind) {
     case IsFE:
       // Emit stat information.
@@ -119,7 +119,7 @@ class FileEntryPTHEntryInfo {
   }
 
   static std::pair<unsigned,unsigned>
-  EmitKeyDataLength(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
+  EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
                     const PTHEntry& E) {
 
     unsigned n = V.getString().size() + 1 + 1;
@@ -131,14 +131,14 @@ class FileEntryPTHEntryInfo {
     return std::make_pair(n, m);
   }
 
-  static void EmitKey(llvm::raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
+  static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
     // Emit the entry kind.
     ::Emit8(Out, (unsigned) V.getKind());
     // Emit the string.
     Out.write(V.getString().data(), n - 1);
   }
 
-  static void EmitData(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
+  static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
                        const PTHEntry& E, unsigned) {
 
 
@@ -197,7 +197,7 @@ class PTHWriter {
     Out.write(Ptr, NumBytes);
   }
 
-  void EmitString(llvm::StringRef V) {
+  void EmitString(StringRef V) {
     ::Emit16(Out, V.size());
     EmitBuf(V.data(), V.size());
   }
@@ -247,7 +247,7 @@ void PTHWriter::EmitToken(const Token& T) {
   } else {
     // We cache *un-cleaned* spellings. This gives us 100% fidelity with the
     // source code.
-    llvm::StringRef s(T.getLiteralData(), T.getLength());
+    StringRef s(T.getLiteralData(), T.getLength());
 
     // Get the string entry.
     llvm::StringMapEntry<OffsetOpt> *E = &CachedStrs.GetOrCreateValue(s);
@@ -584,20 +584,20 @@ class PTHIdentifierTableTrait {
   }
 
   static std::pair<unsigned,unsigned>
-  EmitKeyDataLength(llvm::raw_ostream& Out, const PTHIdKey* key, uint32_t) {
+  EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) {
     unsigned n = key->II->getLength() + 1;
     ::Emit16(Out, n);
     return std::make_pair(n, sizeof(uint32_t));
   }
 
-  static void EmitKey(llvm::raw_ostream& Out, PTHIdKey* key, unsigned n) {
+  static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) {
     // Record the location of the key data.  This is used when generating
     // the mapping from persistent IDs to strings.
     key->FileOffset = Out.tell();
     Out.write(key->II->getNameStart(), n);
   }
 
-  static void EmitData(llvm::raw_ostream& Out, PTHIdKey*, uint32_t pID,
+  static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
                        unsigned) {
     ::Emit32(Out, pID);
   }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/CompilerInstance.cpp b/contrib/llvm/tools/clang/lib/Frontend/CompilerInstance.cpp
index c58e3af5089b..55264870b8b5 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/CompilerInstance.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/CompilerInstance.cpp
@@ -19,12 +19,13 @@
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PTHManager.h"
-#include "clang/Frontend/ChainedDiagnosticClient.h"
+#include "clang/Frontend/ChainedDiagnosticConsumer.h"
 #include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/LogDiagnosticPrinter.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
-#include "clang/Frontend/VerifyDiagnosticsClient.h"
+#include "clang/Frontend/VerifyDiagnosticConsumer.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
@@ -38,11 +39,25 @@
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/system_error.h"
+#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Config/config.h"
+
+// Support for FileLockManager
+#include <fstream>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#if LLVM_ON_WIN32
+#include <windows.h>
+#endif
+#if LLVM_ON_UNIX
+#include <unistd.h>
+#endif
+
 using namespace clang;
 
 CompilerInstance::CompilerInstance()
-  : Invocation(new CompilerInvocation()) {
+  : Invocation(new CompilerInvocation()), ModuleManager(0) {
 }
 
 CompilerInstance::~CompilerInstance() {
@@ -52,7 +67,7 @@ void CompilerInstance::setInvocation(CompilerInvocation *Value) {
   Invocation = Value;
 }
 
-void CompilerInstance::setDiagnostics(Diagnostic *Value) {
+void CompilerInstance::setDiagnostics(DiagnosticsEngine *Value) {
   Diagnostics = Value;
 }
 
@@ -64,7 +79,7 @@ void CompilerInstance::setFileManager(FileManager *Value) {
   FileMgr = Value;
 }
 
-void CompilerInstance::setSourceManager(SourceManager *Value) { 
+void CompilerInstance::setSourceManager(SourceManager *Value) {
   SourceMgr = Value;
 }
 
@@ -87,9 +102,9 @@ void CompilerInstance::setCodeCompletionConsumer(CodeCompleteConsumer *Value) {
 // Diagnostics
 static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
                               unsigned argc, const char* const *argv,
-                              Diagnostic &Diags) {
+                              DiagnosticsEngine &Diags) {
   std::string ErrorInfo;
-  llvm::OwningPtr<llvm::raw_ostream> OS(
+  llvm::OwningPtr<raw_ostream> OS(
     new llvm::raw_fd_ostream(DiagOpts.DumpBuildInformation.c_str(), ErrorInfo));
   if (!ErrorInfo.empty()) {
     Diags.Report(diag::err_fe_unable_to_open_logfile)
@@ -103,17 +118,17 @@ static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
   (*OS) << '\n';
 
   // Chain in a diagnostic client which will log the diagnostics.
-  DiagnosticClient *Logger =
+  DiagnosticConsumer *Logger =
     new TextDiagnosticPrinter(*OS.take(), DiagOpts, /*OwnsOutputStream=*/true);
-  Diags.setClient(new ChainedDiagnosticClient(Diags.takeClient(), Logger));
+  Diags.setClient(new ChainedDiagnosticConsumer(Diags.takeClient(), Logger));
 }
 
 static void SetUpDiagnosticLog(const DiagnosticOptions &DiagOpts,
                                const CodeGenOptions *CodeGenOpts,
-                               Diagnostic &Diags) {
+                               DiagnosticsEngine &Diags) {
   std::string ErrorInfo;
   bool OwnsStream = false;
-  llvm::raw_ostream *OS = &llvm::errs();
+  raw_ostream *OS = &llvm::errs();
   if (DiagOpts.DiagnosticLogFile != "-") {
     // Create the output stream.
     llvm::raw_fd_ostream *FileOS(
@@ -135,38 +150,47 @@ static void SetUpDiagnosticLog(const DiagnosticOptions &DiagOpts,
                                                           OwnsStream);
   if (CodeGenOpts)
     Logger->setDwarfDebugFlags(CodeGenOpts->DwarfDebugFlags);
-  Diags.setClient(new ChainedDiagnosticClient(Diags.takeClient(), Logger));
+  Diags.setClient(new ChainedDiagnosticConsumer(Diags.takeClient(), Logger));
 }
 
 void CompilerInstance::createDiagnostics(int Argc, const char* const *Argv,
-                                         DiagnosticClient *Client) {
+                                         DiagnosticConsumer *Client,
+                                         bool ShouldOwnClient,
+                                         bool ShouldCloneClient) {
   Diagnostics = createDiagnostics(getDiagnosticOpts(), Argc, Argv, Client,
+                                  ShouldOwnClient, ShouldCloneClient,
                                   &getCodeGenOpts());
 }
 
-llvm::IntrusiveRefCntPtr<Diagnostic> 
+llvm::IntrusiveRefCntPtr<DiagnosticsEngine>
 CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
                                     int Argc, const char* const *Argv,
-                                    DiagnosticClient *Client,
+                                    DiagnosticConsumer *Client,
+                                    bool ShouldOwnClient,
+                                    bool ShouldCloneClient,
                                     const CodeGenOptions *CodeGenOpts) {
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(new Diagnostic(DiagID));
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine>
+      Diags(new DiagnosticsEngine(DiagID));
 
   // Create the diagnostic client for reporting errors or for
   // implementing -verify.
-  if (Client)
-    Diags->setClient(Client);
-  else
+  if (Client) {
+    if (ShouldCloneClient)
+      Diags->setClient(Client->clone(*Diags), ShouldOwnClient);
+    else
+      Diags->setClient(Client, ShouldOwnClient);
+  } else
     Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
 
   // Chain in -verify checker, if requested.
   if (Opts.VerifyDiagnostics)
-    Diags->setClient(new VerifyDiagnosticsClient(*Diags, Diags->takeClient()));
+    Diags->setClient(new VerifyDiagnosticConsumer(*Diags));
 
   // Chain in -diagnostic-log-file dumper, if requested.
   if (!Opts.DiagnosticLogFile.empty())
     SetUpDiagnosticLog(Opts, CodeGenOpts, *Diags);
-  
+
   if (!Opts.DumpBuildInformation.empty())
     SetUpBuildDumpLog(Opts, Argc, Argv, *Diags);
 
@@ -191,49 +215,47 @@ void CompilerInstance::createSourceManager(FileManager &FileMgr) {
 // Preprocessor
 
 void CompilerInstance::createPreprocessor() {
-  PP = createPreprocessor(getDiagnostics(), getLangOpts(),
-                          getPreprocessorOpts(), getHeaderSearchOpts(),
-                          getDependencyOutputOpts(), getTarget(),
-                          getFrontendOpts(), getSourceManager(),
-                          getFileManager());
-}
+  const PreprocessorOptions &PPOpts = getPreprocessorOpts();
 
-Preprocessor *
-CompilerInstance::createPreprocessor(Diagnostic &Diags,
-                                     const LangOptions &LangInfo,
-                                     const PreprocessorOptions &PPOpts,
-                                     const HeaderSearchOptions &HSOpts,
-                                     const DependencyOutputOptions &DepOpts,
-                                     const TargetInfo &Target,
-                                     const FrontendOptions &FEOpts,
-                                     SourceManager &SourceMgr,
-                                     FileManager &FileMgr) {
   // Create a PTH manager if we are using some form of a token cache.
   PTHManager *PTHMgr = 0;
   if (!PPOpts.TokenCache.empty())
-    PTHMgr = PTHManager::Create(PPOpts.TokenCache, Diags);
+    PTHMgr = PTHManager::Create(PPOpts.TokenCache, getDiagnostics());
 
   // Create the Preprocessor.
-  HeaderSearch *HeaderInfo = new HeaderSearch(FileMgr);
-  Preprocessor *PP = new Preprocessor(Diags, LangInfo, Target,
-                                      SourceMgr, *HeaderInfo, PTHMgr,
-                                      /*OwnsHeaderSearch=*/true);
+  HeaderSearch *HeaderInfo = new HeaderSearch(getFileManager());
+  PP = new Preprocessor(getDiagnostics(), getLangOpts(), &getTarget(),
+                        getSourceManager(), *HeaderInfo, *this, PTHMgr,
+                        /*OwnsHeaderSearch=*/true);
 
   // Note that this is different then passing PTHMgr to Preprocessor's ctor.
   // That argument is used as the IdentifierInfoLookup argument to
   // IdentifierTable's ctor.
   if (PTHMgr) {
-    PTHMgr->setPreprocessor(PP);
+    PTHMgr->setPreprocessor(&*PP);
     PP->setPTHManager(PTHMgr);
   }
 
   if (PPOpts.DetailedRecord)
     PP->createPreprocessingRecord(
-                       PPOpts.DetailedRecordIncludesNestedMacroExpansions);
-  
-  InitializePreprocessor(*PP, PPOpts, HSOpts, FEOpts);
+                                  PPOpts.DetailedRecordIncludesNestedMacroExpansions);
+
+  InitializePreprocessor(*PP, PPOpts, getHeaderSearchOpts(), getFrontendOpts());
+
+  // Set up the module path, including the hash for the
+  // module-creation options.
+  llvm::SmallString<256> SpecificModuleCache(
+                           getHeaderSearchOpts().ModuleCachePath);
+  if (!getHeaderSearchOpts().DisableModuleHash)
+    llvm::sys::path::append(SpecificModuleCache,
+                            getInvocation().getModuleHash());
+  PP->getHeaderSearchInfo().configureModules(SpecificModuleCache,
+    getPreprocessorOpts().ModuleBuildPath.empty()
+      ? std::string()
+      : getPreprocessorOpts().ModuleBuildPath.back());
 
   // Handle generating dependencies, if requested.
+  const DependencyOutputOptions &DepOpts = getDependencyOutputOpts();
   if (!DepOpts.OutputFile.empty())
     AttachDependencyFileGen(*PP, DepOpts);
 
@@ -241,14 +263,12 @@ CompilerInstance::createPreprocessor(Diagnostic &Diags,
   if (DepOpts.ShowHeaderIncludes)
     AttachHeaderIncludeGen(*PP);
   if (!DepOpts.HeaderIncludeOutputFile.empty()) {
-    llvm::StringRef OutputPath = DepOpts.HeaderIncludeOutputFile;
+    StringRef OutputPath = DepOpts.HeaderIncludeOutputFile;
     if (OutputPath == "-")
       OutputPath = "";
     AttachHeaderIncludeGen(*PP, /*ShowAllHeaders=*/true, OutputPath,
                            /*ShowDepth=*/false);
   }
-
-  return PP;
 }
 
 // ASTContext
@@ -256,30 +276,31 @@ CompilerInstance::createPreprocessor(Diagnostic &Diags,
 void CompilerInstance::createASTContext() {
   Preprocessor &PP = getPreprocessor();
   Context = new ASTContext(getLangOpts(), PP.getSourceManager(),
-                           getTarget(), PP.getIdentifierTable(),
+                           &getTarget(), PP.getIdentifierTable(),
                            PP.getSelectorTable(), PP.getBuiltinInfo(),
                            /*size_reserve=*/ 0);
 }
 
 // ExternalASTSource
 
-void CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
+void CompilerInstance::createPCHExternalASTSource(StringRef Path,
                                                   bool DisablePCHValidation,
                                                   bool DisableStatCache,
                                                  void *DeserializationListener){
   llvm::OwningPtr<ExternalASTSource> Source;
   bool Preamble = getPreprocessorOpts().PrecompiledPreambleBytes.first != 0;
   Source.reset(createPCHExternalASTSource(Path, getHeaderSearchOpts().Sysroot,
-                                          DisablePCHValidation, 
+                                          DisablePCHValidation,
                                           DisableStatCache,
                                           getPreprocessor(), getASTContext(),
                                           DeserializationListener,
                                           Preamble));
+  ModuleManager = static_cast<ASTReader*>(Source.get());
   getASTContext().setExternalSource(Source);
 }
 
 ExternalASTSource *
-CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
+CompilerInstance::createPCHExternalASTSource(StringRef Path,
                                              const std::string &Sysroot,
                                              bool DisablePCHValidation,
                                              bool DisableStatCache,
@@ -288,14 +309,15 @@ CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
                                              void *DeserializationListener,
                                              bool Preamble) {
   llvm::OwningPtr<ASTReader> Reader;
-  Reader.reset(new ASTReader(PP, &Context,
-                             Sysroot.empty() ? 0 : Sysroot.c_str(),
+  Reader.reset(new ASTReader(PP, Context,
+                             Sysroot.empty() ? "" : Sysroot.c_str(),
                              DisablePCHValidation, DisableStatCache));
 
   Reader->setDeserializationListener(
             static_cast<ASTDeserializationListener *>(DeserializationListener));
   switch (Reader->ReadAST(Path,
-                          Preamble ? ASTReader::Preamble : ASTReader::PCH)) {
+                          Preamble ? serialization::MK_Preamble
+                                   : serialization::MK_PCH)) {
   case ASTReader::Success:
     // Set the predefines buffer as suggested by the PCH reader. Typically, the
     // predefines buffer will be empty.
@@ -316,7 +338,7 @@ CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
 
 // Code Completion
 
-static bool EnableCodeCompletion(Preprocessor &PP, 
+static bool EnableCodeCompletion(Preprocessor &PP,
                                  const std::string &Filename,
                                  unsigned Line,
                                  unsigned Column) {
@@ -371,19 +393,19 @@ CompilerInstance::createCodeCompletionConsumer(Preprocessor &PP,
                                                bool ShowMacros,
                                                bool ShowCodePatterns,
                                                bool ShowGlobals,
-                                               llvm::raw_ostream &OS) {
+                                               raw_ostream &OS) {
   if (EnableCodeCompletion(PP, Filename, Line, Column))
     return 0;
 
   // Set up the creation routine for code-completion.
-  return new PrintingCodeCompleteConsumer(ShowMacros, ShowCodePatterns, 
+  return new PrintingCodeCompleteConsumer(ShowMacros, ShowCodePatterns,
                                           ShowGlobals, OS);
 }
 
-void CompilerInstance::createSema(bool CompleteTranslationUnit,
+void CompilerInstance::createSema(TranslationUnitKind TUKind,
                                   CodeCompleteConsumer *CompletionConsumer) {
   TheSema.reset(new Sema(getPreprocessor(), getASTContext(), getASTConsumer(),
-                         CompleteTranslationUnit, CompletionConsumer));
+                         TUKind, CompletionConsumer));
 }
 
 // Output Files
@@ -418,28 +440,30 @@ void CompilerInstance::clearOutputFiles(bool EraseFiles) {
       }
     } else if (!it->Filename.empty() && EraseFiles)
       llvm::sys::Path(it->Filename).eraseFromDisk();
-      
+
   }
   OutputFiles.clear();
 }
 
 llvm::raw_fd_ostream *
 CompilerInstance::createDefaultOutputFile(bool Binary,
-                                          llvm::StringRef InFile,
-                                          llvm::StringRef Extension) {
+                                          StringRef InFile,
+                                          StringRef Extension) {
   return createOutputFile(getFrontendOpts().OutputFile, Binary,
                           /*RemoveFileOnSignal=*/true, InFile, Extension);
 }
 
 llvm::raw_fd_ostream *
-CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
+CompilerInstance::createOutputFile(StringRef OutputPath,
                                    bool Binary, bool RemoveFileOnSignal,
-                                   llvm::StringRef InFile,
-                                   llvm::StringRef Extension) {
+                                   StringRef InFile,
+                                   StringRef Extension,
+                                   bool UseTemporary) {
   std::string Error, OutputPathName, TempPathName;
   llvm::raw_fd_ostream *OS = createOutputFile(OutputPath, Error, Binary,
                                               RemoveFileOnSignal,
                                               InFile, Extension,
+                                              UseTemporary,
                                               &OutputPathName,
                                               &TempPathName);
   if (!OS) {
@@ -457,12 +481,13 @@ CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
 }
 
 llvm::raw_fd_ostream *
-CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
+CompilerInstance::createOutputFile(StringRef OutputPath,
                                    std::string &Error,
                                    bool Binary,
                                    bool RemoveFileOnSignal,
-                                   llvm::StringRef InFile,
-                                   llvm::StringRef Extension,
+                                   StringRef InFile,
+                                   StringRef Extension,
+                                   bool UseTemporary,
                                    std::string *ResultPathName,
                                    std::string *TempPathName) {
   std::string OutFile, TempFile;
@@ -478,8 +503,11 @@ CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
   } else {
     OutFile = "-";
   }
-  
-  if (OutFile != "-") {
+
+  llvm::OwningPtr<llvm::raw_fd_ostream> OS;
+  std::string OSFile;
+
+  if (UseTemporary && OutFile != "-") {
     llvm::sys::Path OutPath(OutFile);
     // Only create the temporary if we can actually write to OutPath, otherwise
     // we want to fail early.
@@ -487,21 +515,26 @@ CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
     if ((llvm::sys::fs::exists(OutPath.str(), Exists) || !Exists) ||
         (OutPath.isRegularFile() && OutPath.canWrite())) {
       // Create a temporary file.
-      llvm::sys::Path TempPath(OutFile);
-      if (!TempPath.createTemporaryFileOnDisk())
-        TempFile = TempPath.str();
+      llvm::SmallString<128> TempPath;
+      TempPath = OutFile;
+      TempPath += "-%%%%%%%%";
+      int fd;
+      if (llvm::sys::fs::unique_file(TempPath.str(), fd, TempPath,
+                               /*makeAbsolute=*/false) == llvm::errc::success) {
+        OS.reset(new llvm::raw_fd_ostream(fd, /*shouldClose=*/true));
+        OSFile = TempFile = TempPath.str();
+      }
     }
   }
 
-  std::string OSFile = OutFile;
-  if (!TempFile.empty())
-    OSFile = TempFile;
-
-  llvm::OwningPtr<llvm::raw_fd_ostream> OS(
-    new llvm::raw_fd_ostream(OSFile.c_str(), Error,
-                             (Binary ? llvm::raw_fd_ostream::F_Binary : 0)));
-  if (!Error.empty())
-    return 0;
+  if (!OS) {
+    OSFile = OutFile;
+    OS.reset(
+      new llvm::raw_fd_ostream(OSFile.c_str(), Error,
+                               (Binary ? llvm::raw_fd_ostream::F_Binary : 0)));
+    if (!Error.empty())
+      return 0;
+  }
 
   // Make sure the out stream file gets removed if we crash.
   if (RemoveFileOnSignal)
@@ -517,21 +550,18 @@ CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
 
 // Initialization Utilities
 
-bool CompilerInstance::InitializeSourceManager(llvm::StringRef InputFile) {
+bool CompilerInstance::InitializeSourceManager(StringRef InputFile) {
   return InitializeSourceManager(InputFile, getDiagnostics(), getFileManager(),
                                  getSourceManager(), getFrontendOpts());
 }
 
-bool CompilerInstance::InitializeSourceManager(llvm::StringRef InputFile,
-                                               Diagnostic &Diags,
+bool CompilerInstance::InitializeSourceManager(StringRef InputFile,
+                                               DiagnosticsEngine &Diags,
                                                FileManager &FileMgr,
                                                SourceManager &SourceMgr,
                                                const FrontendOptions &Opts) {
-  // Figure out where to get and map in the main file, unless it's already
-  // been created (e.g., by a precompiled preamble).
-  if (!SourceMgr.getMainFileID().isInvalid()) {
-    // Do nothing: the main file has already been set.
-  } else if (InputFile != "-") {
+  // Figure out where to get and map in the main file.
+  if (InputFile != "-") {
     const FileEntry *File = FileMgr.getFile(InputFile);
     if (!File) {
       Diags.Report(diag::err_fe_error_reading) << InputFile;
@@ -565,7 +595,7 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
 
   // FIXME: Take this as an argument, once all the APIs we used have moved to
   // taking it as an input instead of hard-coding llvm::errs.
-  llvm::raw_ostream &OS = llvm::errs();
+  raw_ostream &OS = llvm::errs();
 
   // Create the target instance.
   setTarget(TargetInfo::CreateTargetInfo(getDiagnostics(), getTargetOpts()));
@@ -589,7 +619,7 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
 
   if (getFrontendOpts().ShowStats)
     llvm::EnableStatistics();
-    
+
   for (unsigned i = 0, e = getFrontendOpts().Inputs.size(); i != e; ++i) {
     const std::string &InFile = getFrontendOpts().Inputs[i].second;
 
@@ -608,7 +638,7 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
     // Get the total number of warnings/errors from the client.
     unsigned NumWarnings = getDiagnostics().getClient()->getNumWarnings();
     unsigned NumErrors = getDiagnostics().getClient()->getNumErrors();
-    
+
     if (NumWarnings)
       OS << NumWarnings << " warning" << (NumWarnings == 1 ? "" : "s");
     if (NumWarnings && NumErrors)
@@ -627,4 +657,456 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
   return !getDiagnostics().getClient()->getNumErrors();
 }
 
+/// \brief Determine the appropriate source input kind based on language
+/// options.
+static InputKind getSourceInputKindFromOptions(const LangOptions &LangOpts) {
+  if (LangOpts.OpenCL)
+    return IK_OpenCL;
+  if (LangOpts.CUDA)
+    return IK_CUDA;
+  if (LangOpts.ObjC1)
+    return LangOpts.CPlusPlus? IK_ObjCXX : IK_ObjC;
+  return LangOpts.CPlusPlus? IK_CXX : IK_C;
+}
+
+namespace {
+  struct CompileModuleData {
+    CompilerInstance &Instance;
+    GeneratePCHAction &CreateModuleAction;
+  };
+}
+
+/// \brief Helper function that executes the module-generating action under
+/// a crash recovery context.
+static void doCompileModule(void *UserData) {
+  CompileModuleData &Data = *reinterpret_cast<CompileModuleData *>(UserData);
+  Data.Instance.ExecuteAction(Data.CreateModuleAction);
+}
+
+namespace {
+  /// \brief Class that manages the creation of a lock file to aid
+  /// implicit coordination between different processes.
+  ///
+  /// The implicit coordination works by creating a ".lock" file alongside
+  /// the file that we're coordinating for, using the atomicity of the file
+  /// system to ensure that only a single process can create that ".lock" file.
+  /// When the lock file is removed, the owning process has finished the
+  /// operation.
+  class LockFileManager {
+  public:
+    /// \brief Describes the state of a lock file.
+    enum LockFileState {
+      /// \brief The lock file has been created and is owned by this instance
+      /// of the object.
+      LFS_Owned,
+      /// \brief The lock file already exists and is owned by some other
+      /// instance.
+      LFS_Shared,
+      /// \brief An error occurred while trying to create or find the lock
+      /// file.
+      LFS_Error
+    };
+
+  private:
+    llvm::SmallString<128> LockFileName;
+    llvm::SmallString<128> UniqueLockFileName;
+
+    llvm::Optional<std::pair<std::string, int> > Owner;
+    llvm::Optional<llvm::error_code> Error;
+
+    LockFileManager(const LockFileManager &);
+    LockFileManager &operator=(const LockFileManager &);
+
+    static llvm::Optional<std::pair<std::string, int> >
+    readLockFile(StringRef LockFileName);
+
+    static bool processStillExecuting(StringRef Hostname, int PID);
+
+  public:
+
+    LockFileManager(StringRef FileName);
+    ~LockFileManager();
+
+    /// \brief Determine the state of the lock file.
+    LockFileState getState() const;
+
+    operator LockFileState() const { return getState(); }
+
+    /// \brief For a shared lock, wait until the owner releases the lock.
+    void waitForUnlock();
+  };
+}
+
+/// \brief Attempt to read the lock file with the given name, if it exists.
+///
+/// \param LockFileName The name of the lock file to read.
+///
+/// \returns The process ID of the process that owns this lock file
+llvm::Optional<std::pair<std::string, int> >
+LockFileManager::readLockFile(StringRef LockFileName) {
+  // Check whether the lock file exists. If not, clearly there's nothing
+  // to read, so we just return.
+  bool Exists = false;
+  if (llvm::sys::fs::exists(LockFileName, Exists) || !Exists)
+    return llvm::Optional<std::pair<std::string, int> >();
+
+  // Read the owning host and PID out of the lock file. If it appears that the
+  // owning process is dead, the lock file is invalid.
+  int PID = 0;
+  std::string Hostname;
+  std::ifstream Input(LockFileName.str().c_str());
+  if (Input >> Hostname >> PID && PID > 0 &&
+      processStillExecuting(Hostname, PID))
+    return std::make_pair(Hostname, PID);
+
+  // Delete the lock file. It's invalid anyway.
+  bool Existed;
+  llvm::sys::fs::remove(LockFileName, Existed);
+  return llvm::Optional<std::pair<std::string, int> >();
+}
+
+bool LockFileManager::processStillExecuting(StringRef Hostname, int PID) {
+#if LLVM_ON_UNIX
+  char MyHostname[256];
+  MyHostname[255] = 0;
+  MyHostname[0] = 0;
+  gethostname(MyHostname, 255);
+  // Check whether the process is dead. If so, we're done.
+  if (MyHostname == Hostname && getsid(PID) == -1 && errno == ESRCH)
+    return false;
+#endif
+
+  return true;
+}
+
+LockFileManager::LockFileManager(StringRef FileName)
+{
+  LockFileName = FileName;
+  LockFileName += ".lock";
+
+  // If the lock file already exists, don't bother to try to create our own
+  // lock file; it won't work anyway. Just figure out who owns this lock file.
+  if ((Owner = readLockFile(LockFileName)))
+    return;
+
+  // Create a lock file that is unique to this instance.
+  UniqueLockFileName = LockFileName;
+  UniqueLockFileName += "-%%%%%%%%";
+  int UniqueLockFileID;
+  if (llvm::error_code EC
+        = llvm::sys::fs::unique_file(UniqueLockFileName.str(),
+                                     UniqueLockFileID,
+                                     UniqueLockFileName,
+                                     /*makeAbsolute=*/false)) {
+    Error = EC;
+    return;
+  }
+
+  // Write our process ID to our unique lock file.
+  {
+    llvm::raw_fd_ostream Out(UniqueLockFileID, /*shouldClose=*/true);
+
+#if LLVM_ON_UNIX
+    // FIXME: move getpid() call into LLVM
+    char hostname[256];
+    hostname[255] = 0;
+    hostname[0] = 0;
+    gethostname(hostname, 255);
+    Out << hostname << ' ' << getpid();
+#else
+    Out << "localhost 1";
+#endif
+    Out.close();
+
+    if (Out.has_error()) {
+      // We failed to write out PID, so make up an excuse, remove the
+      // unique lock file, and fail.
+      Error = llvm::make_error_code(llvm::errc::no_space_on_device);
+      bool Existed;
+      llvm::sys::fs::remove(UniqueLockFileName.c_str(), Existed);
+      return;
+    }
+  }
+
+  // Create a hard link from the lock file name. If this succeeds, we're done.
+  llvm::error_code EC
+    = llvm::sys::fs::create_hard_link(UniqueLockFileName.str(),
+                                      LockFileName.str());
+  if (EC == llvm::errc::success)
+    return;
+
+  // Creating the hard link failed.
+
+#ifdef LLVM_ON_UNIX
+  // The creation of the hard link may appear to fail, but if stat'ing the
+  // unique file returns a link count of 2, then we can still declare success.
+  struct stat StatBuf;
+  if (stat(UniqueLockFileName.c_str(), &StatBuf) == 0 &&
+      StatBuf.st_nlink == 2)
+    return;
+#endif
+
+  // Someone else managed to create the lock file first. Wipe out our unique
+  // lock file (it's useless now) and read the process ID from the lock file.
+  bool Existed;
+  llvm::sys::fs::remove(UniqueLockFileName.str(), Existed);
+  if ((Owner = readLockFile(LockFileName)))
+    return;
+
+  // There is a lock file that nobody owns; try to clean it up and report
+  // an error.
+  llvm::sys::fs::remove(LockFileName.str(), Existed);
+  Error = EC;
+}
+
+LockFileManager::LockFileState LockFileManager::getState() const {
+  if (Owner)
+    return LFS_Shared;
+
+  if (Error)
+    return LFS_Error;
+
+  return LFS_Owned;
+}
+
+LockFileManager::~LockFileManager() {
+  if (getState() != LFS_Owned)
+    return;
+
+  // Since we own the lock, remove the lock file and our own unique lock file.
+  bool Existed;
+  llvm::sys::fs::remove(LockFileName.str(), Existed);
+  llvm::sys::fs::remove(UniqueLockFileName.str(), Existed);
+}
+
+void LockFileManager::waitForUnlock() {
+  if (getState() != LFS_Shared)
+    return;
+
+#if LLVM_ON_WIN32
+  unsigned long Interval = 1;
+#else
+  struct timespec Interval;
+  Interval.tv_sec = 0;
+  Interval.tv_nsec = 1000000;
+#endif
+  // Don't wait more than an hour for the file to appear.
+  const unsigned MaxSeconds = 3600;
+  do {
+    // Sleep for the designated interval, to allow the owning process time to
+    // finish up and
+    // FIXME: Should we hook in to system APIs to get a notification when the
+    // lock file is deleted?
+#if LLVM_ON_WIN32
+    Sleep(Interval);
+#else
+    nanosleep(&Interval, NULL);
+#endif
+    // If the file no longer exists, we're done.
+    bool Exists = false;
+    if (!llvm::sys::fs::exists(LockFileName.str(), Exists) && !Exists)
+      return;
+
+    if (!processStillExecuting((*Owner).first, (*Owner).second))
+      return;
+
+    // Exponentially increase the time we wait for the lock to be removed.
+#if LLVM_ON_WIN32
+    Interval *= 2;
+#else
+    Interval.tv_sec *= 2;
+    Interval.tv_nsec *= 2;
+    if (Interval.tv_nsec >= 1000000000) {
+      ++Interval.tv_sec;
+      Interval.tv_nsec -= 1000000000;
+    }
+#endif
+  } while (
+#if LLVM_ON_WIN32
+           Interval < MaxSeconds * 1000
+#else
+           Interval.tv_sec < (time_t)MaxSeconds
+#endif
+           );
+
+  // Give up.
+}
+
+/// \brief Compile a module file for the given module name with the given
+/// umbrella header, using the options provided by the importing compiler
+/// instance.
+static void compileModule(CompilerInstance &ImportingInstance,
+                          StringRef ModuleName,
+                          StringRef ModuleFileName,
+                          StringRef UmbrellaHeader) {
+  LockFileManager Locked(ModuleFileName);
+  switch (Locked) {
+  case LockFileManager::LFS_Error:
+    return;
+
+  case LockFileManager::LFS_Owned:
+    // We're responsible for building the module ourselves. Do so below.
+    break;
+
+  case LockFileManager::LFS_Shared:
+    // Someone else is responsible for building the module. Wait for them to
+    // finish.
+    Locked.waitForUnlock();
+    break;
+  }
+
+  // Construct a compiler invocation for creating this module.
+  llvm::IntrusiveRefCntPtr<CompilerInvocation> Invocation
+    (new CompilerInvocation(ImportingInstance.getInvocation()));
+
+  // For any options that aren't intended to affect how a module is built,
+  // reset them to their default values.
+  Invocation->getLangOpts().resetNonModularOptions();
+  Invocation->getPreprocessorOpts().resetNonModularOptions();
+
+  // Note that this module is part of the module build path, so that we
+  // can detect cycles in the module graph.
+  Invocation->getPreprocessorOpts().ModuleBuildPath.push_back(ModuleName);
+
+  // Set up the inputs/outputs so that we build the module from its umbrella
+  // header.
+  FrontendOptions &FrontendOpts = Invocation->getFrontendOpts();
+  FrontendOpts.OutputFile = ModuleFileName.str();
+  FrontendOpts.DisableFree = false;
+  FrontendOpts.Inputs.clear();
+  FrontendOpts.Inputs.push_back(
+    std::make_pair(getSourceInputKindFromOptions(Invocation->getLangOpts()),
+                                                 UmbrellaHeader));
+
+  Invocation->getDiagnosticOpts().VerifyDiagnostics = 0;
+
+
+  assert(ImportingInstance.getInvocation().getModuleHash() ==
+           Invocation->getModuleHash() && "Module hash mismatch!");
+
+  // Construct a compiler instance that will be used to actually create the
+  // module.
+  CompilerInstance Instance;
+  Instance.setInvocation(&*Invocation);
+  Instance.createDiagnostics(/*argc=*/0, /*argv=*/0,
+                             &ImportingInstance.getDiagnosticClient(),
+                             /*ShouldOwnClient=*/true,
+                             /*ShouldCloneClient=*/true);
+
+  // Construct a module-generating action.
+  GeneratePCHAction CreateModuleAction(true);
+
+  // Execute the action to actually build the module in-place. Use a separate
+  // thread so that we get a stack large enough.
+  const unsigned ThreadStackSize = 8 << 20;
+  llvm::CrashRecoveryContext CRC;
+  CompileModuleData Data = { Instance, CreateModuleAction };
+  CRC.RunSafelyOnThread(&doCompileModule, &Data, ThreadStackSize);
+}
+
+ModuleKey CompilerInstance::loadModule(SourceLocation ImportLoc,
+                                       IdentifierInfo &ModuleName,
+                                       SourceLocation ModuleNameLoc) {
+  // Determine what file we're searching from.
+  SourceManager &SourceMgr = getSourceManager();
+  SourceLocation ExpandedImportLoc = SourceMgr.getExpansionLoc(ImportLoc);
+  const FileEntry *CurFile
+    = SourceMgr.getFileEntryForID(SourceMgr.getFileID(ExpandedImportLoc));
+  if (!CurFile)
+    CurFile = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID());
+
+  // Search for a module with the given name.
+  std::string UmbrellaHeader;
+  std::string ModuleFileName;
+  const FileEntry *ModuleFile
+    = PP->getHeaderSearchInfo().lookupModule(ModuleName.getName(),
+                                             &ModuleFileName,
+                                             &UmbrellaHeader);
+
+  bool BuildingModule = false;
+  if (!ModuleFile && !UmbrellaHeader.empty()) {
+    // We didn't find the module, but there is an umbrella header that
+    // can be used to create the module file. Create a separate compilation
+    // module to do so.
+
+    // Check whether there is a cycle in the module graph.
+    SmallVectorImpl<std::string> &ModuleBuildPath
+      = getPreprocessorOpts().ModuleBuildPath;
+    SmallVectorImpl<std::string>::iterator Pos
+      = std::find(ModuleBuildPath.begin(), ModuleBuildPath.end(),
+                  ModuleName.getName());
+    if (Pos != ModuleBuildPath.end()) {
+      llvm::SmallString<256> CyclePath;
+      for (; Pos != ModuleBuildPath.end(); ++Pos) {
+        CyclePath += *Pos;
+        CyclePath += " -> ";
+      }
+      CyclePath += ModuleName.getName();
+
+      getDiagnostics().Report(ModuleNameLoc, diag::err_module_cycle)
+        << ModuleName.getName() << CyclePath;
+      return 0;
+    }
+
+    getDiagnostics().Report(ModuleNameLoc, diag::warn_module_build)
+      << ModuleName.getName();
+    BuildingModule = true;
+    compileModule(*this, ModuleName.getName(), ModuleFileName, UmbrellaHeader);
+    ModuleFile = PP->getHeaderSearchInfo().lookupModule(ModuleName.getName());
+  }
+
+  if (!ModuleFile) {
+    getDiagnostics().Report(ModuleNameLoc,
+                            BuildingModule? diag::err_module_not_built
+                                          : diag::err_module_not_found)
+      << ModuleName.getName()
+      << SourceRange(ImportLoc, ModuleNameLoc);
+    return 0;
+  }
+
+  // If we don't already have an ASTReader, create one now.
+  if (!ModuleManager) {
+    if (!hasASTContext())
+      createASTContext();
+
+    std::string Sysroot = getHeaderSearchOpts().Sysroot;
+    const PreprocessorOptions &PPOpts = getPreprocessorOpts();
+    ModuleManager = new ASTReader(getPreprocessor(), *Context,
+                                  Sysroot.empty() ? "" : Sysroot.c_str(),
+                                  PPOpts.DisablePCHValidation,
+                                  PPOpts.DisableStatCache);
+    if (hasASTConsumer()) {
+      ModuleManager->setDeserializationListener(
+        getASTConsumer().GetASTDeserializationListener());
+      getASTContext().setASTMutationListener(
+        getASTConsumer().GetASTMutationListener());
+    }
+    llvm::OwningPtr<ExternalASTSource> Source;
+    Source.reset(ModuleManager);
+    getASTContext().setExternalSource(Source);
+    if (hasSema())
+      ModuleManager->InitializeSema(getSema());
+    if (hasASTConsumer())
+      ModuleManager->StartTranslationUnit(&getASTConsumer());
+  }
+
+  // Try to load the module we found.
+  switch (ModuleManager->ReadAST(ModuleFile->getName(),
+                                 serialization::MK_Module)) {
+  case ASTReader::Success:
+    break;
+
+  case ASTReader::IgnorePCH:
+    // FIXME: The ASTReader will already have complained, but can we showhorn
+    // that diagnostic information into a more useful form?
+    return 0;
+
+  case ASTReader::Failure:
+    // Already complained.
+    return 0;
+  }
+
+  // FIXME: The module file's FileEntry makes a poor key indeed!
+  return (ModuleKey)ModuleFile;
+}
 
diff --git a/contrib/llvm/tools/clang/lib/Frontend/CompilerInvocation.cpp b/contrib/llvm/tools/clang/lib/Frontend/CompilerInvocation.cpp
index 0371dae304af..432407a846df 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/CompilerInvocation.cpp
@@ -60,6 +60,16 @@ static const char *getAnalysisDiagClientName(AnalysisDiagClients Kind) {
   }
 }
 
+static const char *getAnalysisPurgeModeName(AnalysisPurgeMode Kind) {
+  switch (Kind) {
+  default:
+    llvm_unreachable("Unknown analysis client!");
+#define ANALYSIS_PURGE(NAME, CMDFLAG, DESC) \
+  case NAME: return CMDFLAG;
+#include "clang/Frontend/Analyses.def"
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Serialization (to args)
 //===----------------------------------------------------------------------===//
@@ -68,7 +78,7 @@ static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts,
                                std::vector<std::string> &Res) {
   if (Opts.ShowCheckerHelp)
     Res.push_back("-analyzer-checker-help");
-  if (Opts.AnalysisStoreOpt != BasicStoreModel) {
+  if (Opts.AnalysisStoreOpt != RegionStoreModel) {
     Res.push_back("-analyzer-store");
     Res.push_back(getAnalysisStoreName(Opts.AnalysisStoreOpt));
   }
@@ -80,6 +90,10 @@ static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts,
     Res.push_back("-analyzer-output");
     Res.push_back(getAnalysisDiagClientName(Opts.AnalysisDiagOpt));
   }
+  if (Opts.AnalysisPurgeOpt != PurgeStmt) {
+    Res.push_back("-analyzer-purge");
+    Res.push_back(getAnalysisPurgeModeName(Opts.AnalysisPurgeOpt));
+  }
   if (!Opts.AnalyzeSpecificFunction.empty()) {
     Res.push_back("-analyze-function");
     Res.push_back(Opts.AnalyzeSpecificFunction);
@@ -92,8 +106,6 @@ static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts,
     Res.push_back("-analyzer-opt-analyze-nested-blocks");
   if (Opts.EagerlyAssume)
     Res.push_back("-analyzer-eagerly-assume");
-  if (!Opts.PurgeDead)
-    Res.push_back("-analyzer-no-purge-dead");
   if (Opts.TrimGraph)
     Res.push_back("-trim-egraph");
   if (Opts.VisualizeEGDot)
@@ -171,6 +183,8 @@ static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
     Res.push_back("-mcode-model");
     Res.push_back(Opts.CodeModel);
   }
+  if (Opts.CUDAIsDevice)
+    Res.push_back("-fcuda-is-device");
   if (!Opts.CXAAtExit)
     Res.push_back("-fno-use-cxa-atexit");
   if (Opts.CXXCtorDtorAliases)
@@ -207,6 +221,8 @@ static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
     Res.push_back("-mregparm");
     Res.push_back(llvm::utostr(Opts.NumRegisterParameters));
   }
+  if (Opts.NoGlobalMerge)
+    Res.push_back("-mno-global-merge");
   if (Opts.NoExecStack)
     Res.push_back("-mnoexecstack");
   if (Opts.RelaxAll)
@@ -361,7 +377,6 @@ static const char *getActionName(frontend::ActionKind Kind) {
   case frontend::ASTDumpXML:             return "-ast-dump-xml";
   case frontend::ASTPrint:               return "-ast-print";
   case frontend::ASTView:                return "-ast-view";
-  case frontend::CreateModule:           return "-create-module";
   case frontend::DumpRawTokens:          return "-dump-raw-tokens";
   case frontend::DumpTokens:             return "-dump-tokens";
   case frontend::EmitAssembly:           return "-S";
@@ -372,6 +387,7 @@ static const char *getActionName(frontend::ActionKind Kind) {
   case frontend::EmitCodeGenOnly:        return "-emit-codegen-only";
   case frontend::EmitObj:                return "-emit-obj";
   case frontend::FixIt:                  return "-fixit";
+  case frontend::GenerateModule:         return "-emit-module";
   case frontend::GeneratePCH:            return "-emit-pch";
   case frontend::GeneratePTH:            return "-emit-pth";
   case frontend::InitOnly:               return "-init-only";
@@ -404,8 +420,6 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-disable-free");
   if (Opts.RelocatablePCH)
     Res.push_back("-relocatable-pch");
-  if (Opts.ChainedPCH)
-    Res.push_back("-chained-pch");
   if (Opts.ShowHelp)
     Res.push_back("-help");
   if (Opts.ShowMacrosInCodeCompletion)
@@ -439,6 +453,12 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-arcmt-migrate-directory");
     Res.push_back(Opts.ARCMTMigrateDir);
   }
+  if (!Opts.ARCMTMigrateReportOut.empty()) {
+    Res.push_back("-arcmt-migrate-report-output");
+    Res.push_back(Opts.ARCMTMigrateReportOut);
+  }
+  if (Opts.ARCMTMigrateEmitARCErrors)
+    Res.push_back("-arcmt-migrate-emit-errors");
 
   bool NeedLang = false;
   for (unsigned i = 0, e = Opts.Inputs.size(); i != e; ++i)
@@ -491,10 +511,6 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-ast-merge");
     Res.push_back(Opts.ASTMergeFiles[i]);
   }
-  for (unsigned i = 0, e = Opts.Modules.size(); i != e; ++i) {
-    Res.push_back("-import-module");
-    Res.push_back(Opts.Modules[i]);
-  }
   for (unsigned i = 0, e = Opts.LLVMArgs.size(); i != e; ++i) {
     Res.push_back("-mllvm");
     Res.push_back(Opts.LLVMArgs[i]);
@@ -514,17 +530,43 @@ static void HeaderSearchOptsToArgs(const HeaderSearchOptions &Opts,
     if (E.IsFramework && (E.Group != frontend::Angled || !E.IsUserSupplied))
       llvm::report_fatal_error("Invalid option set!");
     if (E.IsUserSupplied) {
-      if (E.Group == frontend::After) {
+      switch (E.Group) {
+      case frontend::After:
         Res.push_back("-idirafter");
-      } else if (E.Group == frontend::Quoted) {
+        break;
+        
+      case frontend::Quoted:
         Res.push_back("-iquote");
-      } else if (E.Group == frontend::System) {
+        break;
+        
+      case frontend::System:
         Res.push_back("-isystem");
-      } else if (E.Group == frontend::CXXSystem) {
+        break;
+        
+      case frontend::IndexHeaderMap:
+        Res.push_back("-index-header-map");
+        Res.push_back(E.IsFramework? "-F" : "-I");
+        break;
+        
+      case frontend::CSystem:
+        Res.push_back("-c-isystem");
+        break;
+
+      case frontend::CXXSystem:
         Res.push_back("-cxx-isystem");
-      } else {
-        assert(E.Group == frontend::Angled && "Invalid group!");
+        break;
+
+      case frontend::ObjCSystem:
+        Res.push_back("-objc-isystem");
+        break;
+
+      case frontend::ObjCXXSystem:
+        Res.push_back("-objcxx-isystem");
+        break;
+        
+      case frontend::Angled:
         Res.push_back(E.IsFramework ? "-F" : "-I");
+        break;
       }
     } else {
       if (E.Group != frontend::Angled && E.Group != frontend::System)
@@ -535,32 +577,16 @@ static void HeaderSearchOptsToArgs(const HeaderSearchOptions &Opts,
     Res.push_back(E.Path);
   }
 
-  if (!Opts.EnvIncPath.empty()) {
-    // FIXME: Provide an option for this, and move env detection to driver.
-    llvm::report_fatal_error("Not yet implemented!");
-  }
-  if (!Opts.CEnvIncPath.empty()) {
-    // FIXME: Provide an option for this, and move env detection to driver.
-    llvm::report_fatal_error("Not yet implemented!");
-  }
-  if (!Opts.ObjCEnvIncPath.empty()) {
-    // FIXME: Provide an option for this, and move env detection to driver.
-    llvm::report_fatal_error("Not yet implemented!");
-  }
-  if (!Opts.CXXEnvIncPath.empty()) {
-    // FIXME: Provide an option for this, and move env detection to driver.
-    llvm::report_fatal_error("Not yet implemented!");
-  }
-  if (!Opts.ObjCXXEnvIncPath.empty()) {
-    // FIXME: Provide an option for this, and move env detection to driver.
-    llvm::report_fatal_error("Not yet implemented!");
-  }
   if (!Opts.ResourceDir.empty()) {
     Res.push_back("-resource-dir");
     Res.push_back(Opts.ResourceDir);
   }
-  if (!Opts.UseStandardIncludes)
-    Res.push_back("-nostdinc");
+  if (!Opts.ModuleCachePath.empty()) {
+    Res.push_back("-fmodule-cache-path");
+    Res.push_back(Opts.ModuleCachePath);
+  }
+  if (!Opts.UseStandardSystemIncludes)
+    Res.push_back("-nostdsysteminc");
   if (!Opts.UseStandardCXXIncludes)
     Res.push_back("-nostdinc++");
   if (Opts.UseLibcxx)
@@ -593,16 +619,14 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-fno-gnu-keywords");
   if (!Opts.GNUMode && Opts.GNUKeywords)
     Res.push_back("-fgnu-keywords");
-  if (Opts.Microsoft)
+  if (Opts.MicrosoftExt)
     Res.push_back("-fms-extensions");
   if (Opts.MSCVersion != 0)
     Res.push_back("-fmsc-version=" + llvm::utostr(Opts.MSCVersion));
   if (Opts.Borland)
     Res.push_back("-fborland-extensions");
-  if (Opts.ObjCNonFragileABI)
-    Res.push_back("-fobjc-nonfragile-abi");
-  if (Opts.ObjCNonFragileABI2)
-    Res.push_back("-fobjc-nonfragile-abi");
+  if (!Opts.ObjCNonFragileABI)
+    Res.push_back("-fobjc-fragile-abi");
   if (Opts.ObjCDefaultSynthProperties)
     Res.push_back("-fobjc-default-synthesize-properties");
   // NoInline is implicit.
@@ -650,6 +674,8 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-pthread");
   if (Opts.Blocks)
     Res.push_back("-fblocks");
+  if (Opts.BlocksRuntimeOptional)
+    Res.push_back("-fblocks-runtime-optional");
   if (Opts.EmitAllDecls)
     Res.push_back("-femit-all-decls");
   if (Opts.MathErrno)
@@ -692,11 +718,11 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-fshort-wchar");
   if (!Opts.ElideConstructors)
     Res.push_back("-fno-elide-constructors");
-  if (Opts.getGCMode() != LangOptions::NonGC) {
-    if (Opts.getGCMode() == LangOptions::HybridGC) {
+  if (Opts.getGC() != LangOptions::NonGC) {
+    if (Opts.getGC() == LangOptions::HybridGC) {
       Res.push_back("-fobjc-gc");
     } else {
-      assert(Opts.getGCMode() == LangOptions::GCOnly && "Invalid GC mode!");
+      assert(Opts.getGC() == LangOptions::GCOnly && "Invalid GC mode!");
       Res.push_back("-fobjc-gc-only");
     }
   }
@@ -723,9 +749,9 @@ static void LangOptsToArgs(const LangOptions &Opts,
   if (Opts.InlineVisibilityHidden)
     Res.push_back("-fvisibility-inlines-hidden");
 
-  if (Opts.getStackProtectorMode() != 0) {
+  if (Opts.getStackProtector() != 0) {
     Res.push_back("-stack-protector");
-    Res.push_back(llvm::utostr(Opts.getStackProtectorMode()));
+    Res.push_back(llvm::utostr(Opts.getStackProtector()));
   }
   if (Opts.InstantiationDepth != DefaultLangOpts.InstantiationDepth) {
     Res.push_back("-ftemplate-depth");
@@ -860,7 +886,7 @@ using namespace clang::driver::cc1options;
 //
 
 static unsigned getOptimizationLevel(ArgList &Args, InputKind IK,
-                                     Diagnostic &Diags) {
+                                     DiagnosticsEngine &Diags) {
   unsigned DefaultOpt = 0;
   if (IK == IK_OpenCL && !Args.hasArg(OPT_cl_opt_disable))
     DefaultOpt = 2;
@@ -870,11 +896,11 @@ static unsigned getOptimizationLevel(ArgList &Args, InputKind IK,
 }
 
 static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
-                              Diagnostic &Diags) {
+                              DiagnosticsEngine &Diags) {
   using namespace cc1options;
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_store)) {
-    llvm::StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue(Args);
     AnalysisStores Value = llvm::StringSwitch<AnalysisStores>(Name)
 #define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATFN) \
       .Case(CMDFLAG, NAME##Model)
@@ -889,7 +915,7 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_constraints)) {
-    llvm::StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue(Args);
     AnalysisConstraints Value = llvm::StringSwitch<AnalysisConstraints>(Name)
 #define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATFN) \
       .Case(CMDFLAG, NAME##Model)
@@ -904,7 +930,7 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_output)) {
-    llvm::StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue(Args);
     AnalysisDiagClients Value = llvm::StringSwitch<AnalysisDiagClients>(Name)
 #define ANALYSIS_DIAGNOSTICS(NAME, CMDFLAG, DESC, CREATFN, AUTOCREAT) \
       .Case(CMDFLAG, PD_##NAME)
@@ -918,6 +944,21 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
       Opts.AnalysisDiagOpt = Value;
   }
 
+  if (Arg *A = Args.getLastArg(OPT_analyzer_purge)) {
+    StringRef Name = A->getValue(Args);
+    AnalysisPurgeMode Value = llvm::StringSwitch<AnalysisPurgeMode>(Name)
+#define ANALYSIS_PURGE(NAME, CMDFLAG, DESC) \
+      .Case(CMDFLAG, NAME)
+#include "clang/Frontend/Analyses.def"
+      .Default(NumPurgeModes);
+    // FIXME: Error handling.
+    if (Value == NumPurgeModes)
+      Diags.Report(diag::err_drv_invalid_value)
+        << A->getAsString(Args) << Name;
+    else
+      Opts.AnalysisPurgeOpt = Value;
+  }
+
   Opts.ShowCheckerHelp = Args.hasArg(OPT_analyzer_checker_help);
   Opts.VisualizeEGDot = Args.hasArg(OPT_analyzer_viz_egraph_graphviz);
   Opts.VisualizeEGUbi = Args.hasArg(OPT_analyzer_viz_egraph_ubigraph);
@@ -925,7 +966,6 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   Opts.AnalyzerDisplayProgress = Args.hasArg(OPT_analyzer_display_progress);
   Opts.AnalyzeNestedBlocks =
     Args.hasArg(OPT_analyzer_opt_analyze_nested_blocks);
-  Opts.PurgeDead = !Args.hasArg(OPT_analyzer_no_purge_dead);
   Opts.EagerlyAssume = Args.hasArg(OPT_analyzer_eagerly_assume);
   Opts.AnalyzeSpecificFunction = Args.getLastArgValue(OPT_analyze_function);
   Opts.UnoptimizedCFG = Args.hasArg(OPT_analysis_UnoptimizedCFG);
@@ -946,8 +986,8 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
     bool enable = (A->getOption().getID() == OPT_analyzer_checker);
     // We can have a list of comma separated checker names, e.g:
     // '-analyzer-checker=cocoa,unix'
-    llvm::StringRef checkerList = A->getValue(Args);
-    llvm::SmallVector<llvm::StringRef, 4> checkers;
+    StringRef checkerList = A->getValue(Args);
+    SmallVector<StringRef, 4> checkers;
     checkerList.split(checkers, ",");
     for (unsigned i = 0, e = checkers.size(); i != e; ++i)
       Opts.CheckersControlList.push_back(std::make_pair(checkers[i], enable));
@@ -955,7 +995,7 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
 }
 
 static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
-                             Diagnostic &Diags) {
+                             DiagnosticsEngine &Diags) {
   using namespace cc1options;
 
   Opts.OptimizationLevel = getOptimizationLevel(Args, IK, Diags);
@@ -992,6 +1032,7 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.ObjCAutoRefCountExceptions = Args.hasArg(OPT_fobjc_arc_exceptions);
   Opts.ObjCRuntimeHasARC = Args.hasArg(OPT_fobjc_runtime_has_arc);
   Opts.ObjCRuntimeHasTerminate = Args.hasArg(OPT_fobjc_runtime_has_terminate);
+  Opts.CUDAIsDevice = Args.hasArg(OPT_fcuda_is_device);
   Opts.CXAAtExit = !Args.hasArg(OPT_fno_use_cxa_atexit);
   Opts.CXXCtorDtorAliases = Args.hasArg(OPT_mconstructor_aliases);
   Opts.CodeModel = Args.getLastArgValue(OPT_mcode_model);
@@ -1006,6 +1047,7 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.NoZeroInitializedInBSS = Args.hasArg(OPT_mno_zero_initialized_in_bss);
   Opts.BackendOptions = Args.getAllArgValues(OPT_backend_option);
   Opts.NumRegisterParameters = Args.getLastArgIntValue(OPT_mregparm, 0, Diags);
+  Opts.NoGlobalMerge = Args.hasArg(OPT_mno_global_merge);
   Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
   Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
   Opts.OmitLeafFramePointer = Args.hasArg(OPT_momit_leaf_frame_pointer);
@@ -1030,7 +1072,7 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.CoverageFile = Args.getLastArgValue(OPT_coverage_file);
 
   if (Arg *A = Args.getLastArg(OPT_fobjc_dispatch_method_EQ)) {
-    llvm::StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue(Args);
     unsigned Method = llvm::StringSwitch<unsigned>(Name)
       .Case("legacy", CodeGenOptions::Legacy)
       .Case("non-legacy", CodeGenOptions::NonLegacy)
@@ -1056,7 +1098,7 @@ static void ParseDependencyOutputArgs(DependencyOutputOptions &Opts,
 }
 
 static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
-                                Diagnostic &Diags) {
+                                DiagnosticsEngine &Diags) {
   using namespace cc1options;
   Opts.DiagnosticLogFile = Args.getLastArgValue(OPT_diagnostic_log_file);
   Opts.IgnoreWarnings = Args.hasArg(OPT_w);
@@ -1080,18 +1122,18 @@ static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
     if (A->getOption().matches(OPT_fdiagnostics_show_note_include_stack))
       Opts.ShowNoteIncludeStack = true;
 
-  llvm::StringRef ShowOverloads =
+  StringRef ShowOverloads =
     Args.getLastArgValue(OPT_fshow_overloads_EQ, "all");
   if (ShowOverloads == "best")
-    Opts.ShowOverloads = Diagnostic::Ovl_Best;
+    Opts.ShowOverloads = DiagnosticsEngine::Ovl_Best;
   else if (ShowOverloads == "all")
-    Opts.ShowOverloads = Diagnostic::Ovl_All;
+    Opts.ShowOverloads = DiagnosticsEngine::Ovl_All;
   else
     Diags.Report(diag::err_drv_invalid_value)
       << Args.getLastArg(OPT_fshow_overloads_EQ)->getAsString(Args)
       << ShowOverloads;
 
-  llvm::StringRef ShowCategory =
+  StringRef ShowCategory =
     Args.getLastArgValue(OPT_fdiagnostics_show_category, "none");
   if (ShowCategory == "none")
     Opts.ShowCategories = 0;
@@ -1104,7 +1146,7 @@ static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
       << Args.getLastArg(OPT_fdiagnostics_show_category)->getAsString(Args)
       << ShowCategory;
 
-  llvm::StringRef Format =
+  StringRef Format =
     Args.getLastArgValue(OPT_fdiagnostics_format, "clang");
   if (Format == "clang")
     Opts.Format = DiagnosticOptions::Clang;
@@ -1145,13 +1187,13 @@ static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) {
 }
 
 static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
-                                   Diagnostic &Diags) {
+                                   DiagnosticsEngine &Diags) {
   using namespace cc1options;
   Opts.ProgramAction = frontend::ParseSyntaxOnly;
   if (const Arg *A = Args.getLastArg(OPT_Action_Group)) {
     switch (A->getOption().getID()) {
     default:
-      assert(0 && "Invalid option in group!");
+      llvm_unreachable("Invalid option in group!");
     case OPT_ast_dump:
       Opts.ProgramAction = frontend::ASTDump; break;
     case OPT_ast_dump_xml:
@@ -1183,6 +1225,8 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       // fall-through!
     case OPT_fixit:
       Opts.ProgramAction = frontend::FixIt; break;
+    case OPT_emit_module:
+      Opts.ProgramAction = frontend::GenerateModule; break;
     case OPT_emit_pch:
       Opts.ProgramAction = frontend::GeneratePCH; break;
     case OPT_emit_pth:
@@ -1207,8 +1251,6 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Opts.ProgramAction = frontend::RunAnalysis; break;
     case OPT_Eonly:
       Opts.ProgramAction = frontend::RunPreprocessorOnly; break;
-    case OPT_create_module:
-      Opts.ProgramAction = frontend::CreateModule; break;
     }
   }
 
@@ -1246,7 +1288,6 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.OutputFile = Args.getLastArgValue(OPT_o);
   Opts.Plugins = Args.getAllArgValues(OPT_load);
   Opts.RelocatablePCH = Args.hasArg(OPT_relocatable_pch);
-  Opts.ChainedPCH = Args.hasArg(OPT_chained_pch);
   Opts.ShowHelp = Args.hasArg(OPT_help);
   Opts.ShowMacrosInCodeCompletion = Args.hasArg(OPT_code_completion_macros);
   Opts.ShowCodePatternsInCodeCompletion
@@ -1259,7 +1300,6 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.ASTMergeFiles = Args.getAllArgValues(OPT_ast_merge);
   Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
   Opts.FixWhatYouCan = Args.hasArg(OPT_fix_what_you_can);
-  Opts.Modules = Args.getAllArgValues(OPT_import_module);
 
   Opts.ARCMTAction = FrontendOptions::ARCMT_None;
   if (const Arg *A = Args.getLastArg(OPT_arcmt_check,
@@ -1280,6 +1320,10 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     }
   }
   Opts.ARCMTMigrateDir = Args.getLastArgValue(OPT_arcmt_migrate_directory);
+  Opts.ARCMTMigrateReportOut
+    = Args.getLastArgValue(OPT_arcmt_migrate_report_output);
+  Opts.ARCMTMigrateEmitARCErrors
+    = Args.hasArg(OPT_arcmt_migrate_emit_arc_errors);
 
   InputKind DashX = IK_None;
   if (const Arg *A = Args.getLastArg(OPT_x)) {
@@ -1296,7 +1340,9 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       .Case("objective-c-cpp-output", IK_PreprocessedObjC)
       .Case("objc-cpp-output", IK_PreprocessedObjC)
       .Case("objective-c++-cpp-output", IK_PreprocessedObjCXX)
+      .Case("objc++-cpp-output", IK_PreprocessedObjCXX)
       .Case("c-header", IK_C)
+      .Case("cl-header", IK_OpenCL)
       .Case("objective-c-header", IK_ObjC)
       .Case("c++-header", IK_CXX)
       .Case("objective-c++-header", IK_ObjCXX)
@@ -1317,7 +1363,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     InputKind IK = DashX;
     if (IK == IK_None) {
       IK = FrontendOptions::getInputKindForExtension(
-        llvm::StringRef(Inputs[i]).rsplit('.').second);
+        StringRef(Inputs[i]).rsplit('.').second);
       // FIXME: Remove this hack.
       if (i == 0)
         DashX = IK;
@@ -1350,20 +1396,35 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   Opts.Sysroot = Args.getLastArgValue(OPT_isysroot, "/");
   Opts.Verbose = Args.hasArg(OPT_v);
   Opts.UseBuiltinIncludes = !Args.hasArg(OPT_nobuiltininc);
-  Opts.UseStandardIncludes = !Args.hasArg(OPT_nostdinc);
+  Opts.UseStandardSystemIncludes = !Args.hasArg(OPT_nostdsysteminc);
   Opts.UseStandardCXXIncludes = !Args.hasArg(OPT_nostdincxx);
   if (const Arg *A = Args.getLastArg(OPT_stdlib_EQ))
     Opts.UseLibcxx = (strcmp(A->getValue(Args), "libc++") == 0);
   Opts.ResourceDir = Args.getLastArgValue(OPT_resource_dir);
-
-  // Add -I... and -F... options in order.
-  for (arg_iterator it = Args.filtered_begin(OPT_I, OPT_F),
-         ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::Angled, true,
+  Opts.ModuleCachePath = Args.getLastArgValue(OPT_fmodule_cache_path);
+  Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
+  
+  // Add -I..., -F..., and -index-header-map options in order.
+  bool IsIndexHeaderMap = false;
+  for (arg_iterator it = Args.filtered_begin(OPT_I, OPT_F, 
+                                             OPT_index_header_map),
+       ie = Args.filtered_end(); it != ie; ++it) {
+    if ((*it)->getOption().matches(OPT_index_header_map)) {
+      // -index-header-map applies to the next -I or -F.
+      IsIndexHeaderMap = true;
+      continue;
+    }
+        
+    frontend::IncludeDirGroup Group 
+      = IsIndexHeaderMap? frontend::IndexHeaderMap : frontend::Angled;
+    
+    Opts.AddPath((*it)->getValue(Args), Group, true,
                  /*IsFramework=*/ (*it)->getOption().matches(OPT_F), false);
+    IsIndexHeaderMap = false;
+  }
 
   // Add -iprefix/-iwith-prefix/-iwithprefixbefore options.
-  llvm::StringRef Prefix = ""; // FIXME: This isn't the correct default prefix.
+  StringRef Prefix = ""; // FIXME: This isn't the correct default prefix.
   for (arg_iterator it = Args.filtered_begin(OPT_iprefix, OPT_iwithprefix,
                                              OPT_iwithprefixbefore),
          ie = Args.filtered_end(); it != ie; ++it) {
@@ -1384,14 +1445,25 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   for (arg_iterator it = Args.filtered_begin(OPT_iquote),
          ie = Args.filtered_end(); it != ie; ++it)
     Opts.AddPath((*it)->getValue(Args), frontend::Quoted, true, false, false);
-  for (arg_iterator it = Args.filtered_begin(OPT_cxx_isystem, OPT_isystem,
+  for (arg_iterator it = Args.filtered_begin(OPT_isystem,
          OPT_iwithsysroot), ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args),
-                 ((*it)->getOption().matches(OPT_cxx_isystem) ?
-                   frontend::CXXSystem : frontend::System),
-                 true, false, !(*it)->getOption().matches(OPT_iwithsysroot));
+    Opts.AddPath((*it)->getValue(Args), frontend::System, true, false,
+                 !(*it)->getOption().matches(OPT_iwithsysroot));
 
-  // FIXME: Need options for the various environment variables!
+  // Add the paths for the various language specific isystem flags.
+  for (arg_iterator it = Args.filtered_begin(OPT_c_isystem),
+       ie = Args.filtered_end(); it != ie; ++it)
+    Opts.AddPath((*it)->getValue(Args), frontend::CSystem, true, false, true);
+  for (arg_iterator it = Args.filtered_begin(OPT_cxx_isystem),
+       ie = Args.filtered_end(); it != ie; ++it)
+    Opts.AddPath((*it)->getValue(Args), frontend::CXXSystem, true, false, true);
+  for (arg_iterator it = Args.filtered_begin(OPT_objc_isystem),
+       ie = Args.filtered_end(); it != ie; ++it)
+    Opts.AddPath((*it)->getValue(Args), frontend::ObjCSystem, true, false,true);
+  for (arg_iterator it = Args.filtered_begin(OPT_objcxx_isystem),
+       ie = Args.filtered_end(); it != ie; ++it)
+    Opts.AddPath((*it)->getValue(Args), frontend::ObjCXXSystem, true, false,
+                 true);
 }
 
 void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
@@ -1414,7 +1486,7 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
     case IK_None:
     case IK_AST:
     case IK_LLVM_IR:
-      assert(0 && "Invalid input kind!");
+      llvm_unreachable("Invalid input kind!");
     case IK_OpenCL:
       LangStd = LangStandard::lang_opencl;
       break;
@@ -1452,9 +1524,9 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
   // OpenCL has some additional defaults.
   if (LangStd == LangStandard::lang_opencl) {
     Opts.OpenCL = 1;
-    Opts.AltiVec = 1;
+    Opts.AltiVec = 0;
     Opts.CXXOperatorNames = 1;
-    Opts.LaxVectorConversions = 1;
+    Opts.LaxVectorConversions = 0;
     Opts.DefaultFPContract = 1;
   }
 
@@ -1475,7 +1547,7 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
 }
 
 static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
-                          Diagnostic &Diags) {
+                          DiagnosticsEngine &Diags) {
   // FIXME: Cleanup per-file based stuff.
   LangStandard::Kind LangStd = LangStandard::lang_unspecified;
   if (const Arg *A = Args.getLastArg(OPT_std_EQ)) {
@@ -1545,12 +1617,12 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
 
   if (Opts.ObjC1) {
     if (Args.hasArg(OPT_fobjc_gc_only))
-      Opts.setGCMode(LangOptions::GCOnly);
+      Opts.setGC(LangOptions::GCOnly);
     else if (Args.hasArg(OPT_fobjc_gc))
-      Opts.setGCMode(LangOptions::HybridGC);
+      Opts.setGC(LangOptions::HybridGC);
     else if (Args.hasArg(OPT_fobjc_arc)) {
       Opts.ObjCAutoRefCount = 1;
-      if (!Args.hasArg(OPT_fobjc_nonfragile_abi))
+      if (Args.hasArg(OPT_fobjc_fragile_abi))
         Diags.Report(diag::err_arc_nonfragile_abi);
     }
 
@@ -1585,7 +1657,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   if (Args.hasArg(OPT_fdelayed_template_parsing))
     Opts.DelayedTemplateParsing = 1;
 
-  llvm::StringRef Vis = Args.getLastArgValue(OPT_fvisibility, "default");
+  StringRef Vis = Args.getLastArgValue(OPT_fvisibility, "default");
   if (Vis == "default")
     Opts.setVisibilityMode(DefaultVisibility);
   else if (Vis == "hidden")
@@ -1615,7 +1687,8 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
                                    OPT_fno_dollars_in_identifiers,
                                    Opts.DollarIdents);
   Opts.PascalStrings = Args.hasArg(OPT_fpascal_strings);
-  Opts.Microsoft = Args.hasArg(OPT_fms_extensions);
+  Opts.MicrosoftExt = Args.hasArg(OPT_fms_extensions);
+  Opts.MicrosoftMode = Args.hasArg(OPT_fms_compatibility);
   Opts.MSCVersion = Args.getLastArgIntValue(OPT_fmsc_version, 0, Diags);
   Opts.Borland = Args.hasArg(OPT_fborland_extensions);
   Opts.WritableStrings = Args.hasArg(OPT_fwritable_strings);
@@ -1633,6 +1706,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
 
   Opts.RTTI = !Args.hasArg(OPT_fno_rtti);
   Opts.Blocks = Args.hasArg(OPT_fblocks);
+  Opts.BlocksRuntimeOptional = Args.hasArg(OPT_fblocks_runtime_optional);
   Opts.CharIsSigned = !Args.hasArg(OPT_fno_signed_char);
   Opts.ShortWChar = Args.hasArg(OPT_fshort_wchar);
   Opts.ShortEnums = Args.hasArg(OPT_fshort_enums);
@@ -1653,13 +1727,14 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.NeXTRuntime = !Args.hasArg(OPT_fgnu_runtime);
   Opts.ObjCConstantStringClass =
     Args.getLastArgValue(OPT_fconstant_string_class);
-  Opts.ObjCNonFragileABI = Args.hasArg(OPT_fobjc_nonfragile_abi);
+  Opts.ObjCNonFragileABI = !Args.hasArg(OPT_fobjc_fragile_abi);
   if (Opts.ObjCNonFragileABI)
     Opts.ObjCNonFragileABI2 = true;
   Opts.ObjCDefaultSynthProperties =
     Args.hasArg(OPT_fobjc_default_synthesize_properties);
   Opts.CatchUndefined = Args.hasArg(OPT_fcatch_undefined_behavior);
   Opts.EmitAllDecls = Args.hasArg(OPT_femit_all_decls);
+  Opts.PackStruct = Args.getLastArgIntValue(OPT_fpack_struct, 0, Diags);
   Opts.PICLevel = Args.getLastArgIntValue(OPT_pic_level, 0, Diags);
   Opts.Static = Args.hasArg(OPT_static_define);
   Opts.DumpRecordLayouts = Args.hasArg(OPT_fdump_record_layouts);
@@ -1696,15 +1771,15 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     Diags.Report(diag::err_drv_invalid_value)
       << Args.getLastArg(OPT_stack_protector)->getAsString(Args) << SSP;
     break;
-  case 0: Opts.setStackProtectorMode(LangOptions::SSPOff); break;
-  case 1: Opts.setStackProtectorMode(LangOptions::SSPOn);  break;
-  case 2: Opts.setStackProtectorMode(LangOptions::SSPReq); break;
+  case 0: Opts.setStackProtector(LangOptions::SSPOff); break;
+  case 1: Opts.setStackProtector(LangOptions::SSPOn);  break;
+  case 2: Opts.setStackProtector(LangOptions::SSPReq); break;
   }
 }
 
 static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
                                   FileManager &FileMgr,
-                                  Diagnostic &Diags) {
+                                  DiagnosticsEngine &Diags) {
   using namespace cc1options;
   Opts.ImplicitPCHInclude = Args.getLastArgValue(OPT_include_pch);
   Opts.ImplicitPTHInclude = Args.getLastArgValue(OPT_include_pth);
@@ -1714,6 +1789,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
     Opts.TokenCache = Opts.ImplicitPTHInclude;
   Opts.UsePredefines = !Args.hasArg(OPT_undef);
   Opts.DetailedRecord = Args.hasArg(OPT_detailed_preprocessing_record);
+  Opts.AutoModuleImport = Args.hasArg(OPT_fauto_module_import);
   Opts.DisablePCHValidation = Args.hasArg(OPT_fno_validate_pch);
 
   Opts.DumpDeserializedPCHDecls = Args.hasArg(OPT_dump_deserialized_pch_decls);
@@ -1724,12 +1800,12 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   }
 
   if (const Arg *A = Args.getLastArg(OPT_preamble_bytes_EQ)) {
-    llvm::StringRef Value(A->getValue(Args));
+    StringRef Value(A->getValue(Args));
     size_t Comma = Value.find(',');
     unsigned Bytes = 0;
     unsigned EndOfLine = 0;
 
-    if (Comma == llvm::StringRef::npos ||
+    if (Comma == StringRef::npos ||
         Value.substr(0, Comma).getAsInteger(10, Bytes) ||
         Value.substr(Comma + 1).getAsInteger(10, EndOfLine))
       Diags.Report(diag::err_drv_preamble_format);
@@ -1780,8 +1856,8 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   for (arg_iterator it = Args.filtered_begin(OPT_remap_file),
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
-    std::pair<llvm::StringRef,llvm::StringRef> Split =
-      llvm::StringRef(A->getValue(Args)).split(';');
+    std::pair<StringRef,StringRef> Split =
+      StringRef(A->getValue(Args)).split(';');
 
     if (Split.second.empty()) {
       Diags.Report(diag::err_drv_invalid_remap_file) << A->getAsString(Args);
@@ -1792,7 +1868,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   }
   
   if (Arg *A = Args.getLastArg(OPT_fobjc_arc_cxxlib_EQ)) {
-    llvm::StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue(Args);
     unsigned Library = llvm::StringSwitch<unsigned>(Name)
       .Case("libc++", ARCXX_libcxx)
       .Case("libstdc++", ARCXX_libstdcxx)
@@ -1834,7 +1910,7 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args) {
 void CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
                                         const char *const *ArgBegin,
                                         const char *const *ArgEnd,
-                                        Diagnostic &Diags) {
+                                        DiagnosticsEngine &Diags) {
   // Parse the arguments.
   llvm::OwningPtr<OptTable> Opts(createCC1OptTable());
   unsigned MissingArgIndex, MissingArgCount;
@@ -1873,3 +1949,105 @@ void CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
   ParsePreprocessorOutputArgs(Res.getPreprocessorOutputOpts(), *Args);
   ParseTargetArgs(Res.getTargetOpts(), *Args);
 }
+
+namespace {
+
+  class ModuleSignature {
+    llvm::SmallVector<uint64_t, 16> Data;
+    unsigned CurBit;
+    uint64_t CurValue;
+    
+  public:
+    ModuleSignature() : CurBit(0), CurValue(0) { }
+    
+    void add(uint64_t Value, unsigned Bits);
+    void add(StringRef Value);
+    void flush();
+    
+    llvm::APInt getAsInteger() const;
+  };
+}
+
+void ModuleSignature::add(uint64_t Value, unsigned int NumBits) {
+  CurValue |= Value << CurBit;
+  if (CurBit + NumBits < 64) {
+    CurBit += NumBits;
+    return;
+  }
+  
+  // Add the current word.
+  Data.push_back(CurValue);
+  
+  if (CurBit)
+    CurValue = Value >> (64-CurBit);
+  else
+    CurValue = 0;
+  CurBit = (CurBit+NumBits) & 63;
+}
+
+void ModuleSignature::flush() {
+  if (CurBit == 0)
+    return;
+  
+  Data.push_back(CurValue);
+  CurBit = 0;
+  CurValue = 0;
+}
+
+void ModuleSignature::add(StringRef Value) {
+  for (StringRef::iterator I = Value.begin(), IEnd = Value.end(); I != IEnd;++I)
+    add(*I, 8);
+}
+
+llvm::APInt ModuleSignature::getAsInteger() const {
+  return llvm::APInt(Data.size() * 64, Data);
+}
+
+std::string CompilerInvocation::getModuleHash() const {
+  ModuleSignature Signature;
+  
+  // Start the signature with the compiler version.
+  Signature.add(getClangFullRepositoryVersion());
+  
+  // Extend the signature with the language options
+#define LANGOPT(Name, Bits, Default, Description) \
+  Signature.add(LangOpts.Name, Bits);
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  Signature.add(static_cast<unsigned>(LangOpts.get##Name()), Bits);
+#define BENIGN_LANGOPT(Name, Bits, Default, Description)
+#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+#include "clang/Basic/LangOptions.def"
+  
+  // Extend the signature with the target triple
+  llvm::Triple T(TargetOpts.Triple);
+  Signature.add((unsigned)T.getArch(), 5);
+  Signature.add((unsigned)T.getVendor(), 4);
+  Signature.add((unsigned)T.getOS(), 5);
+  Signature.add((unsigned)T.getEnvironment(), 4);
+
+  // Extend the signature with preprocessor options.
+  Signature.add(getPreprocessorOpts().UsePredefines, 1);
+  Signature.add(getPreprocessorOpts().DetailedRecord, 1);
+  
+  // Hash the preprocessor defines.
+  // FIXME: This is terrible. Use an MD5 sum of the preprocessor defines.
+  std::vector<StringRef> MacroDefs;
+  for (std::vector<std::pair<std::string, bool/*isUndef*/> >::const_iterator 
+            I = getPreprocessorOpts().Macros.begin(),
+         IEnd = getPreprocessorOpts().Macros.end();
+       I != IEnd; ++I) {
+    if (!I->second)
+      MacroDefs.push_back(I->first);
+  }
+  llvm::array_pod_sort(MacroDefs.begin(), MacroDefs.end());
+       
+  unsigned PPHashResult = 0;
+  for (unsigned I = 0, N = MacroDefs.size(); I != N; ++I)
+    PPHashResult = llvm::HashString(MacroDefs[I], PPHashResult);
+  Signature.add(PPHashResult, 32);
+  
+  // We've generated the signature. Treat it as one large APInt that we'll
+  // encode in base-36 and return.
+  Signature.flush();
+  return Signature.getAsInteger().toString(36, /*Signed=*/false);
+}
diff --git a/contrib/llvm/tools/clang/lib/Frontend/CreateInvocationFromCommandLine.cpp b/contrib/llvm/tools/clang/lib/Frontend/CreateInvocationFromCommandLine.cpp
index 42b648aa6411..fc1508103c2e 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/CreateInvocationFromCommandLine.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/CreateInvocationFromCommandLine.cpp
@@ -29,8 +29,8 @@ using namespace clang;
 /// \return A CompilerInvocation, or 0 if none was built for the given
 /// argument vector.
 CompilerInvocation *
-clang::createInvocationFromCommandLine(llvm::ArrayRef<const char *> ArgList,
-                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags) {
+clang::createInvocationFromCommandLine(ArrayRef<const char *> ArgList,
+                            llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags) {
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
@@ -39,7 +39,7 @@ clang::createInvocationFromCommandLine(llvm::ArrayRef<const char *> ArgList,
                                                 ArgList.begin());
   }
 
-  llvm::SmallVector<const char *, 16> Args;
+  SmallVector<const char *, 16> Args;
   Args.push_back("<clang>"); // FIXME: Remove dummy argument.
   Args.insert(Args.end(), ArgList.begin(), ArgList.end());
 
@@ -49,7 +49,7 @@ clang::createInvocationFromCommandLine(llvm::ArrayRef<const char *> ArgList,
 
   // FIXME: We shouldn't have to pass in the path info.
   driver::Driver TheDriver("clang", llvm::sys::getHostTriple(),
-                           "a.out", false, false, *Diags);
+                           "a.out", false, *Diags);
 
   // Don't check that inputs exist, they may have been remapped.
   TheDriver.setCheckInputsExist(false);
@@ -74,7 +74,7 @@ clang::createInvocationFromCommandLine(llvm::ArrayRef<const char *> ArgList,
   }
 
   const driver::Command *Cmd = cast<driver::Command>(*Jobs.begin());
-  if (llvm::StringRef(Cmd->getCreator().getName()) != "clang") {
+  if (StringRef(Cmd->getCreator().getName()) != "clang") {
     Diags->Report(diag::err_fe_expected_clang_command);
     return 0;
   }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/DependencyFile.cpp b/contrib/llvm/tools/clang/lib/Frontend/DependencyFile.cpp
index 1edd09b06c4c..ff3a12392c9f 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/DependencyFile.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/DependencyFile.cpp
@@ -32,19 +32,19 @@ class DependencyFileCallback : public PPCallbacks {
   llvm::StringSet<> FilesSet;
   const Preprocessor *PP;
   std::vector<std::string> Targets;
-  llvm::raw_ostream *OS;
+  raw_ostream *OS;
   bool IncludeSystemHeaders;
   bool PhonyTarget;
   bool AddMissingHeaderDeps;
 private:
   bool FileMatchesDepCriteria(const char *Filename,
                               SrcMgr::CharacteristicKind FileType);
-  void AddFilename(llvm::StringRef Filename);
+  void AddFilename(StringRef Filename);
   void OutputDependencyFile();
 
 public:
   DependencyFileCallback(const Preprocessor *_PP,
-                         llvm::raw_ostream *_OS,
+                         raw_ostream *_OS,
                          const DependencyOutputOptions &Opts)
     : PP(_PP), Targets(Opts.Targets), OS(_OS),
       IncludeSystemHeaders(Opts.IncludeSystemHeaders),
@@ -52,15 +52,16 @@ class DependencyFileCallback : public PPCallbacks {
       AddMissingHeaderDeps(Opts.AddMissingHeaderDeps) {}
 
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType);
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID);
   virtual void InclusionDirective(SourceLocation HashLoc,
                                   const Token &IncludeTok,
-                                  llvm::StringRef FileName,
+                                  StringRef FileName,
                                   bool IsAngled,
                                   const FileEntry *File,
                                   SourceLocation EndLoc,
-                                  llvm::StringRef SearchPath,
-                                  llvm::StringRef RelativePath);
+                                  StringRef SearchPath,
+                                  StringRef RelativePath);
 
   virtual void EndOfMainFile() {
     OutputDependencyFile();
@@ -78,7 +79,7 @@ void clang::AttachDependencyFileGen(Preprocessor &PP,
   }
 
   std::string Err;
-  llvm::raw_ostream *OS(new llvm::raw_fd_ostream(Opts.OutputFile.c_str(), Err));
+  raw_ostream *OS(new llvm::raw_fd_ostream(Opts.OutputFile.c_str(), Err));
   if (!Err.empty()) {
     PP.getDiagnostics().Report(diag::err_fe_error_opening)
       << Opts.OutputFile << Err;
@@ -86,14 +87,8 @@ void clang::AttachDependencyFileGen(Preprocessor &PP,
   }
 
   // Disable the "file not found" diagnostic if the -MG option was given.
-  // FIXME: Ideally this would live in the driver, but we don't have the ability
-  // to remap individual diagnostics there without creating a DiagGroup, in
-  // which case we would need to prevent the group name from showing up in
-  // diagnostics.
-  if (Opts.AddMissingHeaderDeps) {
-    PP.getDiagnostics().setDiagnosticMapping(diag::warn_pp_file_not_found,
-                                            diag::MAP_IGNORE, SourceLocation());
-  }
+  if (Opts.AddMissingHeaderDeps)
+    PP.SetSuppressIncludeNotFoundError(true);
 
   PP.addPPCallbacks(new DependencyFileCallback(&PP, OS, Opts));
 }
@@ -113,7 +108,8 @@ bool DependencyFileCallback::FileMatchesDepCriteria(const char *Filename,
 
 void DependencyFileCallback::FileChanged(SourceLocation Loc,
                                          FileChangeReason Reason,
-                                         SrcMgr::CharacteristicKind FileType) {
+                                         SrcMgr::CharacteristicKind FileType,
+                                         FileID PrevFID) {
   if (Reason != PPCallbacks::EnterFile)
     return;
 
@@ -123,10 +119,10 @@ void DependencyFileCallback::FileChanged(SourceLocation Loc,
   SourceManager &SM = PP->getSourceManager();
 
   const FileEntry *FE =
-    SM.getFileEntryForID(SM.getFileID(SM.getInstantiationLoc(Loc)));
+    SM.getFileEntryForID(SM.getFileID(SM.getExpansionLoc(Loc)));
   if (FE == 0) return;
 
-  llvm::StringRef Filename = FE->getName();
+  StringRef Filename = FE->getName();
   if (!FileMatchesDepCriteria(Filename.data(), FileType))
     return;
 
@@ -143,24 +139,24 @@ void DependencyFileCallback::FileChanged(SourceLocation Loc,
 
 void DependencyFileCallback::InclusionDirective(SourceLocation HashLoc,
                                                 const Token &IncludeTok,
-                                                llvm::StringRef FileName,
+                                                StringRef FileName,
                                                 bool IsAngled,
                                                 const FileEntry *File,
                                                 SourceLocation EndLoc,
-                                                llvm::StringRef SearchPath,
-                                                llvm::StringRef RelativePath) {
+                                                StringRef SearchPath,
+                                                StringRef RelativePath) {
   if (AddMissingHeaderDeps && !File)
     AddFilename(FileName);
 }
 
-void DependencyFileCallback::AddFilename(llvm::StringRef Filename) {
+void DependencyFileCallback::AddFilename(StringRef Filename) {
   if (FilesSet.insert(Filename))
     Files.push_back(Filename);
 }
 
 /// PrintFilename - GCC escapes spaces, but apparently not ' or " or other
 /// scary characters.
-static void PrintFilename(llvm::raw_ostream &OS, llvm::StringRef Filename) {
+static void PrintFilename(raw_ostream &OS, StringRef Filename) {
   for (unsigned i = 0, e = Filename.size(); i != e; ++i) {
     if (Filename[i] == ' ')
       OS << '\\';
diff --git a/contrib/llvm/tools/clang/lib/Frontend/FrontendAction.cpp b/contrib/llvm/tools/clang/lib/Frontend/FrontendAction.cpp
index 0128d6ee0531..ba2d63b026d5 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/FrontendAction.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/FrontendAction.cpp
@@ -20,6 +20,7 @@
 #include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Parse/ParseAST.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
+#include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ChainedIncludesSource.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Timer.h"
@@ -65,7 +66,7 @@ class DeserializedDeclsDumper : public ASTDeserializationListener {
       if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
         if (NamesToCheck.find(ND->getNameAsString()) != NamesToCheck.end()) {
           unsigned DiagID
-            = Ctx.getDiagnostics().getCustomDiagID(Diagnostic::Error,
+            = Ctx.getDiagnostics().getCustomDiagID(DiagnosticsEngine::Error,
                                                    "%0 was deserialized");
           Ctx.getDiagnostics().Report(Ctx.getFullLoc(D->getLocation()), DiagID)
               << ND->getNameAsString();
@@ -82,7 +83,7 @@ FrontendAction::FrontendAction() : Instance(0) {}
 
 FrontendAction::~FrontendAction() {}
 
-void FrontendAction::setCurrentFile(llvm::StringRef Value, InputKind Kind,
+void FrontendAction::setCurrentFile(StringRef Value, InputKind Kind,
                                     ASTUnit *AST) {
   CurrentFile = Value;
   CurrentFileKind = Kind;
@@ -90,7 +91,7 @@ void FrontendAction::setCurrentFile(llvm::StringRef Value, InputKind Kind,
 }
 
 ASTConsumer* FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
-                                                      llvm::StringRef InFile) {
+                                                      StringRef InFile) {
   ASTConsumer* Consumer = CreateASTConsumer(CI, InFile);
   if (!Consumer)
     return 0;
@@ -123,7 +124,7 @@ ASTConsumer* FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
 }
 
 bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
-                                     llvm::StringRef Filename,
+                                     StringRef Filename,
                                      InputKind InputKind) {
   assert(!Instance && "Already processing a source file!");
   assert(!Filename.empty() && "Unexpected empty filename!");
@@ -141,7 +142,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     assert(hasASTFileSupport() &&
            "This action does not have AST file support!");
 
-    llvm::IntrusiveRefCntPtr<Diagnostic> Diags(&CI.getDiagnostics());
+    llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(&CI.getDiagnostics());
     std::string Error;
     ASTUnit *AST = ASTUnit::LoadFromASTFile(Filename, Diags,
                                             CI.getFileSystemOpts());
@@ -224,9 +225,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     } else if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
       // Use PCH.
       assert(hasPCHSupport() && "This action does not have PCH support!");
-      ASTDeserializationListener *DeserialListener
-          = CI.getInvocation().getFrontendOpts().ChainedPCH ?
-                  Consumer->GetASTDeserializationListener() : 0;
+      ASTDeserializationListener *DeserialListener =
+          Consumer->GetASTDeserializationListener();
       if (CI.getPreprocessorOpts().DumpDeserializedPCHDecls)
         DeserialListener = new DeserializedDeclsDumper(DeserialListener);
       if (!CI.getPreprocessorOpts().DeserializedPCHDeclsToErrorOn.empty())
@@ -247,7 +247,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
       goto failure;
   }
 
-  // Initialize builtin info as long as we aren't using an external AST
+  // Initialize built-in info as long as we aren't using an external AST
   // source.
   if (!CI.hasASTContext() || !CI.getASTContext().getExternalSource()) {
     Preprocessor &PP = CI.getPreprocessor();
@@ -374,26 +374,26 @@ void ASTFrontendAction::ExecuteAction() {
     CompletionConsumer = &CI.getCodeCompletionConsumer();
 
   if (!CI.hasSema())
-    CI.createSema(usesCompleteTranslationUnit(), CompletionConsumer);
+    CI.createSema(getTranslationUnitKind(), CompletionConsumer);
 
   ParseAST(CI.getSema(), CI.getFrontendOpts().ShowStats);
 }
 
 ASTConsumer *
 PreprocessorFrontendAction::CreateASTConsumer(CompilerInstance &CI,
-                                              llvm::StringRef InFile) {
+                                              StringRef InFile) {
   llvm_unreachable("Invalid CreateASTConsumer on preprocessor action!");
 }
 
 ASTConsumer *WrapperFrontendAction::CreateASTConsumer(CompilerInstance &CI,
-                                                      llvm::StringRef InFile) {
+                                                      StringRef InFile) {
   return WrappedAction->CreateASTConsumer(CI, InFile);
 }
 bool WrapperFrontendAction::BeginInvocation(CompilerInstance &CI) {
   return WrappedAction->BeginInvocation(CI);
 }
 bool WrapperFrontendAction::BeginSourceFileAction(CompilerInstance &CI,
-                                                  llvm::StringRef Filename) {
+                                                  StringRef Filename) {
   WrappedAction->setCurrentFile(getCurrentFile(), getCurrentFileKind());
   WrappedAction->setCompilerInstance(&CI);
   return WrappedAction->BeginSourceFileAction(CI, Filename);
@@ -408,8 +408,8 @@ void WrapperFrontendAction::EndSourceFileAction() {
 bool WrapperFrontendAction::usesPreprocessorOnly() const {
   return WrappedAction->usesPreprocessorOnly();
 }
-bool WrapperFrontendAction::usesCompleteTranslationUnit() {
-  return WrappedAction->usesCompleteTranslationUnit();
+TranslationUnitKind WrapperFrontendAction::getTranslationUnitKind() {
+  return WrappedAction->getTranslationUnitKind();
 }
 bool WrapperFrontendAction::hasPCHSupport() const {
   return WrappedAction->hasPCHSupport();
diff --git a/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp b/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp
index 7b06c7e49acf..6f84da96300c 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp
@@ -22,6 +22,8 @@
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/system_error.h"
+
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -29,7 +31,7 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 ASTConsumer *InitOnlyAction::CreateASTConsumer(CompilerInstance &CI,
-                                               llvm::StringRef InFile) {
+                                               StringRef InFile) {
   return new ASTConsumer();
 }
 
@@ -41,20 +43,20 @@ void InitOnlyAction::ExecuteAction() {
 //===----------------------------------------------------------------------===//
 
 ASTConsumer *ASTPrintAction::CreateASTConsumer(CompilerInstance &CI,
-                                               llvm::StringRef InFile) {
-  if (llvm::raw_ostream *OS = CI.createDefaultOutputFile(false, InFile))
+                                               StringRef InFile) {
+  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile))
     return CreateASTPrinter(OS);
   return 0;
 }
 
 ASTConsumer *ASTDumpAction::CreateASTConsumer(CompilerInstance &CI,
-                                              llvm::StringRef InFile) {
+                                              StringRef InFile) {
   return CreateASTDumper();
 }
 
 ASTConsumer *ASTDumpXMLAction::CreateASTConsumer(CompilerInstance &CI,
-                                                 llvm::StringRef InFile) {
-  llvm::raw_ostream *OS;
+                                                 StringRef InFile) {
+  raw_ostream *OS;
   if (CI.getFrontendOpts().OutputFile.empty())
     OS = &llvm::outs();
   else
@@ -64,35 +66,34 @@ ASTConsumer *ASTDumpXMLAction::CreateASTConsumer(CompilerInstance &CI,
 }
 
 ASTConsumer *ASTViewAction::CreateASTConsumer(CompilerInstance &CI,
-                                              llvm::StringRef InFile) {
+                                              StringRef InFile) {
   return CreateASTViewer();
 }
 
 ASTConsumer *DeclContextPrintAction::CreateASTConsumer(CompilerInstance &CI,
-                                                       llvm::StringRef InFile) {
+                                                       StringRef InFile) {
   return CreateDeclContextPrinter();
 }
 
 ASTConsumer *GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI,
-                                                  llvm::StringRef InFile) {
+                                                  StringRef InFile) {
   std::string Sysroot;
   std::string OutputFile;
-  llvm::raw_ostream *OS = 0;
-  bool Chaining;
-  if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OutputFile, OS, Chaining))
+  raw_ostream *OS = 0;
+  if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OutputFile, OS))
     return 0;
 
-  const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
-                             Sysroot.c_str() : 0;  
-  return new PCHGenerator(CI.getPreprocessor(), OutputFile, Chaining, isysroot, OS);
+  if (!CI.getFrontendOpts().RelocatablePCH)
+    Sysroot.clear();
+  return new PCHGenerator(CI.getPreprocessor(), OutputFile, MakeModule, 
+                          Sysroot, OS);
 }
 
 bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
-                                                    llvm::StringRef InFile,
+                                                    StringRef InFile,
                                                     std::string &Sysroot,
                                                     std::string &OutputFile,
-                                                    llvm::raw_ostream *&OS,
-                                                    bool &Chaining) {
+                                                    raw_ostream *&OS) {
   Sysroot = CI.getHeaderSearchOpts().Sysroot;
   if (CI.getFrontendOpts().RelocatablePCH && Sysroot.empty()) {
     CI.getDiagnostics().Report(diag::err_relocatable_without_isysroot);
@@ -101,19 +102,19 @@ bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
 
   // We use createOutputFile here because this is exposed via libclang, and we
   // must disable the RemoveFileOnSignal behavior.
+  // We use a temporary to avoid race conditions.
   OS = CI.createOutputFile(CI.getFrontendOpts().OutputFile, /*Binary=*/true,
-                           /*RemoveFileOnSignal=*/false, InFile);
+                           /*RemoveFileOnSignal=*/false, InFile,
+                           /*Extension=*/"", /*useTemporary=*/true);
   if (!OS)
     return true;
 
   OutputFile = CI.getFrontendOpts().OutputFile;
-  Chaining = CI.getInvocation().getFrontendOpts().ChainedPCH &&
-             !CI.getPreprocessorOpts().ImplicitPCHInclude.empty();
   return false;
 }
 
 ASTConsumer *SyntaxOnlyAction::CreateASTConsumer(CompilerInstance &CI,
-                                                 llvm::StringRef InFile) {
+                                                 StringRef InFile) {
   return new ASTConsumer();
 }
 
@@ -182,9 +183,48 @@ void PreprocessOnlyAction::ExecuteAction() {
 
 void PrintPreprocessedAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
-  // Output file needs to be set to 'Binary', to avoid converting Unix style
+  // Output file may need to be set to 'Binary', to avoid converting Unix style
   // line feeds (<LF>) to Microsoft style line feeds (<CR><LF>).
-  llvm::raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
+  //
+  // Look to see what type of line endings the file uses. If there's a
+  // CRLF, then we won't open the file up in binary mode. If there is
+  // just an LF or CR, then we will open the file up in binary mode.
+  // In this fashion, the output format should match the input format, unless
+  // the input format has inconsistent line endings.
+  //
+  // This should be a relatively fast operation since most files won't have
+  // all of their source code on a single line. However, that is still a 
+  // concern, so if we scan for too long, we'll just assume the file should
+  // be opened in binary mode.
+  bool BinaryMode = true;
+  bool InvalidFile = false;
+  const SourceManager& SM = CI.getSourceManager();
+  const llvm::MemoryBuffer *Buffer = SM.getBuffer(SM.getMainFileID(), 
+                                                     &InvalidFile);
+  if (!InvalidFile) {
+    const char *cur = Buffer->getBufferStart();
+    const char *end = Buffer->getBufferEnd();
+    const char *next = (cur != end) ? cur + 1 : end;
+
+    // Limit ourselves to only scanning 256 characters into the source
+    // file.  This is mostly a sanity check in case the file has no 
+    // newlines whatsoever.
+    if (end - cur > 256) end = cur + 256;
+	  
+    while (next < end) {
+      if (*cur == 0x0D) {  // CR
+        if (*next == 0x0A)  // CRLF
+          BinaryMode = false;
+
+        break;
+      } else if (*cur == 0x0A)  // LF
+        break;
+
+      ++cur, ++next;
+    }
+  }
+
+  raw_ostream *OS = CI.createDefaultOutputFile(BinaryMode, getCurrentFile());
   if (!OS) return;
 
   DoPrintPreprocessedInput(CI.getPreprocessor(), OS,
@@ -217,7 +257,7 @@ void PrintPreambleAction::ExecuteAction() {
   llvm::MemoryBuffer *Buffer
       = CI.getFileManager().getBufferForFile(getCurrentFile());
   if (Buffer) {
-    unsigned Preamble = Lexer::ComputePreamble(Buffer).first;
+    unsigned Preamble = Lexer::ComputePreamble(Buffer, CI.getLangOpts()).first;
     llvm::outs().write(Buffer->getBufferStart(), Preamble);
     delete Buffer;
   }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/FrontendOptions.cpp b/contrib/llvm/tools/clang/lib/Frontend/FrontendOptions.cpp
index 0a2005192a19..ea4005f7c960 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/FrontendOptions.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/FrontendOptions.cpp
@@ -11,7 +11,7 @@
 #include "llvm/ADT/StringSwitch.h"
 using namespace clang;
 
-InputKind FrontendOptions::getInputKindForExtension(llvm::StringRef Extension) {
+InputKind FrontendOptions::getInputKindForExtension(StringRef Extension) {
   return llvm::StringSwitch<InputKind>(Extension)
     .Case("ast", IK_AST)
     .Case("c", IK_C)
diff --git a/contrib/llvm/tools/clang/lib/Frontend/HeaderIncludeGen.cpp b/contrib/llvm/tools/clang/lib/Frontend/HeaderIncludeGen.cpp
index 51dec967cd6b..3b41d894ec50 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/HeaderIncludeGen.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/HeaderIncludeGen.cpp
@@ -17,7 +17,7 @@ using namespace clang;
 namespace {
 class HeaderIncludesCallback : public PPCallbacks {
   SourceManager &SM;
-  llvm::raw_ostream *OutputFile;
+  raw_ostream *OutputFile;
   unsigned CurrentIncludeDepth;
   bool HasProcessedPredefines;
   bool OwnsOutputFile;
@@ -26,7 +26,7 @@ class HeaderIncludesCallback : public PPCallbacks {
 
 public:
   HeaderIncludesCallback(const Preprocessor *PP, bool ShowAllHeaders_,
-                         llvm::raw_ostream *OutputFile_, bool OwnsOutputFile_,
+                         raw_ostream *OutputFile_, bool OwnsOutputFile_,
                          bool ShowDepth_)
     : SM(PP->getSourceManager()), OutputFile(OutputFile_),
       CurrentIncludeDepth(0), HasProcessedPredefines(false),
@@ -39,13 +39,14 @@ class HeaderIncludesCallback : public PPCallbacks {
   }
 
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType);
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID);
 };
 }
 
 void clang::AttachHeaderIncludeGen(Preprocessor &PP, bool ShowAllHeaders,
-                                   llvm::StringRef OutputPath, bool ShowDepth) {
-  llvm::raw_ostream *OutputFile = &llvm::errs();
+                                   StringRef OutputPath, bool ShowDepth) {
+  raw_ostream *OutputFile = &llvm::errs();
   bool OwnsOutputFile = false;
 
   // Open the output file, if used.
@@ -72,7 +73,8 @@ void clang::AttachHeaderIncludeGen(Preprocessor &PP, bool ShowAllHeaders,
 
 void HeaderIncludesCallback::FileChanged(SourceLocation Loc,
                                          FileChangeReason Reason,
-                                       SrcMgr::CharacteristicKind NewFileType) {
+                                       SrcMgr::CharacteristicKind NewFileType,
+                                       FileID PrevFID) {
   // Unless we are exiting a #include, make sure to skip ahead to the line the
   // #include directive was at.
   PresumedLoc UserLoc = SM.getPresumedLoc(Loc);
diff --git a/contrib/llvm/tools/clang/lib/Frontend/InitHeaderSearch.cpp b/contrib/llvm/tools/clang/lib/Frontend/InitHeaderSearch.cpp
index ae38a19be0b9..43b6d4c7e5ab 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/InitHeaderSearch.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/InitHeaderSearch.cpp
@@ -56,38 +56,34 @@ class InitHeaderSearch {
 
 public:
 
-  InitHeaderSearch(HeaderSearch &HS, bool verbose, llvm::StringRef sysroot)
+  InitHeaderSearch(HeaderSearch &HS, bool verbose, StringRef sysroot)
     : Headers(HS), Verbose(verbose), IncludeSysroot(sysroot),
       IsNotEmptyOrRoot(!(sysroot.empty() || sysroot == "/")) {
   }
 
   /// AddPath - Add the specified path to the specified group list.
-  void AddPath(const llvm::Twine &Path, IncludeDirGroup Group,
+  void AddPath(const Twine &Path, IncludeDirGroup Group,
                bool isCXXAware, bool isUserSupplied,
                bool isFramework, bool IgnoreSysRoot = false);
 
   /// AddGnuCPlusPlusIncludePaths - Add the necessary paths to support a gnu
   ///  libstdc++.
-  void AddGnuCPlusPlusIncludePaths(llvm::StringRef Base,
-                                   llvm::StringRef ArchDir,
-                                   llvm::StringRef Dir32,
-                                   llvm::StringRef Dir64,
+  void AddGnuCPlusPlusIncludePaths(StringRef Base,
+                                   StringRef ArchDir,
+                                   StringRef Dir32,
+                                   StringRef Dir64,
                                    const llvm::Triple &triple);
 
   /// AddMinGWCPlusPlusIncludePaths - Add the necessary paths to support a MinGW
   ///  libstdc++.
-  void AddMinGWCPlusPlusIncludePaths(llvm::StringRef Base,
-                                     llvm::StringRef Arch,
-                                     llvm::StringRef Version);
+  void AddMinGWCPlusPlusIncludePaths(StringRef Base,
+                                     StringRef Arch,
+                                     StringRef Version);
 
   /// AddMinGW64CXXPaths - Add the necessary paths to support
   /// libstdc++ of x86_64-w64-mingw32 aka mingw-w64.
-  void AddMinGW64CXXPaths(llvm::StringRef Base,
-                          llvm::StringRef Version);
-
-  /// AddDelimitedPaths - Add a list of paths delimited by the system PATH
-  /// separator. The processing follows that of the CPATH variable for gcc.
-  void AddDelimitedPaths(llvm::StringRef String);
+  void AddMinGW64CXXPaths(StringRef Base,
+                          StringRef Version);
 
   // AddDefaultCIncludePaths - Add paths that should always be searched.
   void AddDefaultCIncludePaths(const llvm::Triple &triple,
@@ -100,9 +96,9 @@ class InitHeaderSearch {
 
   /// AddDefaultSystemIncludePaths - Adds the default system include paths so
   ///  that e.g. stdio.h is found.
-  void AddDefaultSystemIncludePaths(const LangOptions &Lang,
-                                    const llvm::Triple &triple,
-                                    const HeaderSearchOptions &HSOpts);
+  void AddDefaultIncludePaths(const LangOptions &Lang,
+                              const llvm::Triple &triple,
+                              const HeaderSearchOptions &HSOpts);
 
   /// Realize - Merges all search path lists into one list and send it to
   /// HeaderSearch.
@@ -111,7 +107,7 @@ class InitHeaderSearch {
 
 }  // end anonymous namespace.
 
-void InitHeaderSearch::AddPath(const llvm::Twine &Path,
+void InitHeaderSearch::AddPath(const Twine &Path,
                                IncludeDirGroup Group, bool isCXXAware,
                                bool isUserSupplied, bool isFramework,
                                bool IgnoreSysRoot) {
@@ -120,7 +116,7 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
 
   // Compute the actual path, taking into consideration -isysroot.
   llvm::SmallString<256> MappedPathStorage;
-  llvm::StringRef MappedPathStr = Path.toStringRef(MappedPathStorage);
+  StringRef MappedPathStr = Path.toStringRef(MappedPathStorage);
 
   // Handle isysroot.
   if ((Group == System || Group == CXXSystem) && !IgnoreSysRoot &&
@@ -138,7 +134,7 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
 
   // Compute the DirectoryLookup type.
   SrcMgr::CharacteristicKind Type;
-  if (Group == Quoted || Group == Angled)
+  if (Group == Quoted || Group == Angled || Group == IndexHeaderMap)
     Type = SrcMgr::C_User;
   else if (isCXXAware)
     Type = SrcMgr::C_System;
@@ -160,7 +156,7 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
       if (const HeaderMap *HM = Headers.CreateHeaderMap(FE)) {
         // It is a headermap, add it to the search path.
         IncludePath.push_back(std::make_pair(Group, DirectoryLookup(HM, Type,
-                              isUserSupplied)));
+                              isUserSupplied, Group == IndexHeaderMap)));
         return;
       }
     }
@@ -171,30 +167,10 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
                  << MappedPathStr << "\"\n";
 }
 
-
-void InitHeaderSearch::AddDelimitedPaths(llvm::StringRef at) {
-  if (at.empty()) // Empty string should not add '.' path.
-    return;
-
-  llvm::StringRef::size_type delim;
-  while ((delim = at.find(llvm::sys::PathSeparator)) != llvm::StringRef::npos) {
-    if (delim == 0)
-      AddPath(".", Angled, false, true, false);
-    else
-      AddPath(at.substr(0, delim), Angled, false, true, false);
-    at = at.substr(delim + 1);
-  }
-
-  if (at.empty())
-    AddPath(".", Angled, false, true, false);
-  else
-    AddPath(at, Angled, false, true, false);
-}
-
-void InitHeaderSearch::AddGnuCPlusPlusIncludePaths(llvm::StringRef Base,
-                                                   llvm::StringRef ArchDir,
-                                                   llvm::StringRef Dir32,
-                                                   llvm::StringRef Dir64,
+void InitHeaderSearch::AddGnuCPlusPlusIncludePaths(StringRef Base,
+                                                   StringRef ArchDir,
+                                                   StringRef Dir32,
+                                                   StringRef Dir64,
                                                    const llvm::Triple &triple) {
   // Add the base dir
   AddPath(Base, CXXSystem, true, false, false);
@@ -211,9 +187,9 @@ void InitHeaderSearch::AddGnuCPlusPlusIncludePaths(llvm::StringRef Base,
   AddPath(Base + "/backward", CXXSystem, true, false, false);
 }
 
-void InitHeaderSearch::AddMinGWCPlusPlusIncludePaths(llvm::StringRef Base,
-                                                     llvm::StringRef Arch,
-                                                     llvm::StringRef Version) {
+void InitHeaderSearch::AddMinGWCPlusPlusIncludePaths(StringRef Base,
+                                                     StringRef Arch,
+                                                     StringRef Version) {
   AddPath(Base + "/" + Arch + "/" + Version + "/include/c++",
           CXXSystem, true, false, false);
   AddPath(Base + "/" + Arch + "/" + Version + "/include/c++/" + Arch,
@@ -222,8 +198,8 @@ void InitHeaderSearch::AddMinGWCPlusPlusIncludePaths(llvm::StringRef Base,
           CXXSystem, true, false, false);
 }
 
-void InitHeaderSearch::AddMinGW64CXXPaths(llvm::StringRef Base,
-                                          llvm::StringRef Version) {
+void InitHeaderSearch::AddMinGW64CXXPaths(StringRef Base,
+                                          StringRef Version) {
   // Assumes Base is HeaderSearchOpts' ResourceDir
   AddPath(Base + "/../../../include/c++/" + Version,
           CXXSystem, true, false, false);
@@ -452,14 +428,16 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
                                             const HeaderSearchOptions &HSOpts) {
   llvm::Triple::OSType os = triple.getOS();
 
-  switch (os) {
-  case llvm::Triple::FreeBSD:
-  case llvm::Triple::NetBSD:
-    break;
-  default:
-    // FIXME: temporary hack: hard-coded paths.
-    AddPath("/usr/local/include", System, true, false, false);
-    break;
+  if (HSOpts.UseStandardSystemIncludes) {
+    switch (os) {
+    case llvm::Triple::FreeBSD:
+    case llvm::Triple::NetBSD:
+      break;
+    default:
+      // FIXME: temporary hack: hard-coded paths.
+      AddPath("/usr/local/include", System, true, false, false);
+      break;
+    }
   }
 
   // Builtin includes use #include_next directives and should be positioned
@@ -472,12 +450,17 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     AddPath(P.str(), System, false, false, false, /*IgnoreSysRoot=*/ true);
   }
 
+  // All remaining additions are for system include directories, early exit if
+  // we aren't using them.
+  if (!HSOpts.UseStandardSystemIncludes)
+    return;
+
   // Add dirs specified via 'configure --with-c-include-dirs'.
-  llvm::StringRef CIncludeDirs(C_INCLUDE_DIRS);
+  StringRef CIncludeDirs(C_INCLUDE_DIRS);
   if (CIncludeDirs != "") {
-    llvm::SmallVector<llvm::StringRef, 5> dirs;
+    SmallVector<StringRef, 5> dirs;
     CIncludeDirs.split(dirs, ":");
-    for (llvm::SmallVectorImpl<llvm::StringRef>::iterator i = dirs.begin();
+    for (SmallVectorImpl<StringRef>::iterator i = dirs.begin();
          i != dirs.end();
          ++i)
       AddPath(*i, System, false, false, false);
@@ -605,9 +588,9 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
 void InitHeaderSearch::
 AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple, const HeaderSearchOptions &HSOpts) {
   llvm::Triple::OSType os = triple.getOS();
-  llvm::StringRef CxxIncludeRoot(CXX_INCLUDE_ROOT);
+  StringRef CxxIncludeRoot(CXX_INCLUDE_ROOT);
   if (CxxIncludeRoot != "") {
-    llvm::StringRef CxxIncludeArch(CXX_INCLUDE_ARCH);
+    StringRef CxxIncludeArch(CXX_INCLUDE_ARCH);
     if (CxxIncludeArch == "")
       AddGnuCPlusPlusIncludePaths(CxxIncludeRoot, triple.str().c_str(),
                                   CXX_INCLUDE_32BIT_DIR, CXX_INCLUDE_64BIT_DIR,
@@ -745,7 +728,12 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple, const HeaderSearchOp
     //===------------------------------------------------------------------===//
     // Redhat based distros.
     //===------------------------------------------------------------------===//
-    // Fedora 15
+    // Fedora 15 (GCC 4.6.1)
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.6.1",
+                                "x86_64-redhat-linux", "32", "", triple);
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.6.1",
+                                "i686-redhat-linux", "", "", triple);
+    // Fedora 15 (GCC 4.6.0)
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.6.0",
                                 "x86_64-redhat-linux", "32", "", triple);
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.6.0",
@@ -848,6 +836,17 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple, const HeaderSearchOp
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.6.0",
                                 "x86_64-unknown-linux-gnu", "", "", triple);
 
+    // Slackware gcc 4.5.2 (13.37)
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5.2",
+                                "i486-slackware-linux", "", "", triple);
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5.2",
+                                "x86_64-slackware-linux", "", "", triple);
+    // Slackware gcc 4.5.3 (-current)
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5.3",
+                                "i486-slackware-linux", "", "", triple);
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5.3",
+                                "x86_64-slackware-linux", "", "", triple);
+
     // Gentoo x86 gcc 4.5.2
     AddGnuCPlusPlusIncludePaths(
       "/usr/lib/gcc/i686-pc-linux-gnu/4.5.2/include/g++-v4",
@@ -897,6 +896,10 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple, const HeaderSearchOp
     AddGnuCPlusPlusIncludePaths(
         "/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.3/include/g++-v4",
         "x86_64-pc-linux-gnu", "32", "", triple);
+    // Gentoo amd64 gcc 4.3.4
+    AddGnuCPlusPlusIncludePaths(
+        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.3.4/include/g++-v4",
+        "x86_64-pc-linux-gnu", "", "", triple);
     // Gentoo amd64 gcc 4.3.2
     AddGnuCPlusPlusIncludePaths(
         "/usr/lib/gcc/x86_64-pc-linux-gnu/4.3.2/include/g++-v4",
@@ -947,32 +950,53 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple, const HeaderSearchOp
   }
 }
 
-void InitHeaderSearch::AddDefaultSystemIncludePaths(const LangOptions &Lang,
-                                                    const llvm::Triple &triple,
+void InitHeaderSearch::AddDefaultIncludePaths(const LangOptions &Lang,
+                                              const llvm::Triple &triple,
                                             const HeaderSearchOptions &HSOpts) {
-  if (Lang.CPlusPlus && HSOpts.UseStandardCXXIncludes) {
-    if (HSOpts.UseLibcxx)
+  if (Lang.CPlusPlus && HSOpts.UseStandardCXXIncludes &&
+      HSOpts.UseStandardSystemIncludes) {
+    if (HSOpts.UseLibcxx) {
+      if (triple.isOSDarwin()) {
+        // On Darwin, libc++ may be installed alongside the compiler in
+        // lib/c++/v1.
+        llvm::sys::Path P(HSOpts.ResourceDir);
+        if (!P.isEmpty()) {
+          P.eraseComponent();  // Remove version from foo/lib/clang/version
+          P.eraseComponent();  // Remove clang from foo/lib/clang
+          
+          // Get foo/lib/c++/v1
+          P.appendComponent("c++");
+          P.appendComponent("v1");
+          AddPath(P.str(), CXXSystem, true, false, false, true);
+        }
+      }
+      
       AddPath("/usr/include/c++/v1", CXXSystem, true, false, false);
-    else
+    } else {
       AddDefaultCPlusPlusIncludePaths(triple, HSOpts);
+    }
   }
 
   AddDefaultCIncludePaths(triple, HSOpts);
 
   // Add the default framework include paths on Darwin.
-  if (triple.isOSDarwin()) {
-    AddPath("/System/Library/Frameworks", System, true, false, true);
-    AddPath("/Library/Frameworks", System, true, false, true);
+  if (HSOpts.UseStandardSystemIncludes) {
+    if (triple.isOSDarwin()) {
+      AddPath("/System/Library/Frameworks", System, true, false, true);
+      AddPath("/Library/Frameworks", System, true, false, true);
+    }
   }
 }
 
 /// RemoveDuplicates - If there are duplicate directory entries in the specified
-/// search list, remove the later (dead) ones.
-static void RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
-                             unsigned First, bool Verbose) {
+/// search list, remove the later (dead) ones.  Returns the number of non-system
+/// headers removed, which is used to update NumAngled.
+static unsigned RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
+                                 unsigned First, bool Verbose) {
   llvm::SmallPtrSet<const DirectoryEntry *, 8> SeenDirs;
   llvm::SmallPtrSet<const DirectoryEntry *, 8> SeenFrameworkDirs;
   llvm::SmallPtrSet<const HeaderMap *, 8> SeenHeaderMaps;
+  unsigned NonSystemRemoved = 0;
   for (unsigned i = First; i != SearchList.size(); ++i) {
     unsigned DirToRemove = i;
 
@@ -1039,12 +1063,15 @@ static void RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
         llvm::errs() << "  as it is a non-system directory that duplicates "
                      << "a system directory\n";
     }
+    if (DirToRemove != i)
+      ++NonSystemRemoved;
 
     // This is reached if the current entry is a duplicate.  Remove the
     // DirToRemove (usually the current dir).
     SearchList.erase(SearchList.begin()+DirToRemove);
     --i;
   }
+  return NonSystemRemoved;
 }
 
 
@@ -1065,7 +1092,7 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) {
 
   for (path_iterator it = IncludePath.begin(), ie = IncludePath.end();
        it != ie; ++it) {
-    if (it->first == Angled)
+    if (it->first == Angled || it->first == IndexHeaderMap)
       SearchList.push_back(it->second);
   }
 
@@ -1074,7 +1101,11 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) {
 
   for (path_iterator it = IncludePath.begin(), ie = IncludePath.end();
        it != ie; ++it) {
-    if (it->first == System || (Lang.CPlusPlus && it->first == CXXSystem))
+    if (it->first == System ||
+        (!Lang.ObjC1 && !Lang.CPlusPlus && it->first == CSystem)    ||
+        (/*FIXME !Lang.ObjC1 && */Lang.CPlusPlus  && it->first == CXXSystem)  ||
+        (Lang.ObjC1  && !Lang.CPlusPlus && it->first == ObjCSystem) ||
+        (Lang.ObjC1  && Lang.CPlusPlus  && it->first == ObjCXXSystem))
       SearchList.push_back(it->second);
   }
 
@@ -1087,7 +1118,8 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) {
   // Remove duplicates across both the Angled and System directories.  GCC does
   // this and failing to remove duplicates across these two groups breaks
   // #include_next.
-  RemoveDuplicates(SearchList, NumQuoted, Verbose);
+  unsigned NonSystemRemoved = RemoveDuplicates(SearchList, NumQuoted, Verbose);
+  NumAngled -= NonSystemRemoved;
 
   bool DontSearchCurDir = false;  // TODO: set to true if -I- is set?
   Headers.SetSearchPaths(SearchList, NumQuoted, NumAngled, DontSearchCurDir);
@@ -1127,19 +1159,7 @@ void clang::ApplyHeaderSearchOptions(HeaderSearch &HS,
                  E.IgnoreSysRoot);
   }
 
-  // Add entries from CPATH and friends.
-  Init.AddDelimitedPaths(HSOpts.EnvIncPath);
-  if (Lang.CPlusPlus && Lang.ObjC1)
-    Init.AddDelimitedPaths(HSOpts.ObjCXXEnvIncPath);
-  else if (Lang.CPlusPlus)
-    Init.AddDelimitedPaths(HSOpts.CXXEnvIncPath);
-  else if (Lang.ObjC1)
-    Init.AddDelimitedPaths(HSOpts.ObjCEnvIncPath);
-  else
-    Init.AddDelimitedPaths(HSOpts.CEnvIncPath);
-
-  if (HSOpts.UseStandardIncludes)
-    Init.AddDefaultSystemIncludePaths(Lang, Triple, HSOpts);
+  Init.AddDefaultIncludePaths(Lang, Triple, HSOpts);
 
   Init.Realize(Lang);
 }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/InitPreprocessor.cpp b/contrib/llvm/tools/clang/lib/Frontend/InitPreprocessor.cpp
index 9428cd5de0f3..6f49ec474473 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/InitPreprocessor.cpp
@@ -30,15 +30,15 @@ using namespace clang;
 // Append a #define line to Buf for Macro.  Macro should be of the form XXX,
 // in which case we emit "#define XXX 1" or "XXX=Y z W" in which case we emit
 // "#define XXX Y z W".  To get a #define with no value, use "XXX=".
-static void DefineBuiltinMacro(MacroBuilder &Builder, llvm::StringRef Macro,
-                               Diagnostic &Diags) {
-  std::pair<llvm::StringRef, llvm::StringRef> MacroPair = Macro.split('=');
-  llvm::StringRef MacroName = MacroPair.first;
-  llvm::StringRef MacroBody = MacroPair.second;
+static void DefineBuiltinMacro(MacroBuilder &Builder, StringRef Macro,
+                               DiagnosticsEngine &Diags) {
+  std::pair<StringRef, StringRef> MacroPair = Macro.split('=');
+  StringRef MacroName = MacroPair.first;
+  StringRef MacroBody = MacroPair.second;
   if (MacroName.size() != Macro.size()) {
     // Per GCC -D semantics, the macro ends at \n if it exists.
-    llvm::StringRef::size_type End = MacroBody.find_first_of("\n\r");
-    if (End != llvm::StringRef::npos)
+    StringRef::size_type End = MacroBody.find_first_of("\n\r");
+    if (End != StringRef::npos)
       Diags.Report(diag::warn_fe_macro_contains_embedded_newline)
         << MacroName;
     Builder.defineMacro(MacroName, MacroBody.substr(0, End));
@@ -48,7 +48,7 @@ static void DefineBuiltinMacro(MacroBuilder &Builder, llvm::StringRef Macro,
   }
 }
 
-std::string clang::NormalizeDashIncludePath(llvm::StringRef File,
+std::string clang::NormalizeDashIncludePath(StringRef File,
                                             FileManager &FileMgr) {
   // Implicit include paths should be resolved relative to the current
   // working directory first, and then use the regular header search
@@ -70,17 +70,17 @@ std::string clang::NormalizeDashIncludePath(llvm::StringRef File,
 
 /// AddImplicitInclude - Add an implicit #include of the specified file to the
 /// predefines buffer.
-static void AddImplicitInclude(MacroBuilder &Builder, llvm::StringRef File,
+static void AddImplicitInclude(MacroBuilder &Builder, StringRef File,
                                FileManager &FileMgr) {
   Builder.append("#include \"" +
-                 llvm::Twine(NormalizeDashIncludePath(File, FileMgr)) + "\"");
+                 Twine(NormalizeDashIncludePath(File, FileMgr)) + "\"");
 }
 
 static void AddImplicitIncludeMacros(MacroBuilder &Builder,
-                                     llvm::StringRef File,
+                                     StringRef File,
                                      FileManager &FileMgr) {
   Builder.append("#__include_macros \"" +
-                 llvm::Twine(NormalizeDashIncludePath(File, FileMgr)) + "\"");
+                 Twine(NormalizeDashIncludePath(File, FileMgr)) + "\"");
   // Marker token to stop the __include_macros fetch loop.
   Builder.append("##"); // ##?
 }
@@ -88,7 +88,7 @@ static void AddImplicitIncludeMacros(MacroBuilder &Builder,
 /// AddImplicitIncludePTH - Add an implicit #include using the original file
 ///  used to generate a PTH cache.
 static void AddImplicitIncludePTH(MacroBuilder &Builder, Preprocessor &PP,
-                                  llvm::StringRef ImplicitIncludePTH) {
+                                  StringRef ImplicitIncludePTH) {
   PTHManager *P = PP.getPTHManager();
   // Null check 'P' in the corner case where it couldn't be created.
   const char *OriginalFile = P ? P->getOriginalSourceFile() : 0;
@@ -120,7 +120,7 @@ static T PickFP(const llvm::fltSemantics *Sem, T IEEESingleVal,
   return IEEEQuadVal;
 }
 
-static void DefineFloatMacros(MacroBuilder &Builder, llvm::StringRef Prefix,
+static void DefineFloatMacros(MacroBuilder &Builder, StringRef Prefix,
                               const llvm::fltSemantics *Sem) {
   const char *DenormMin, *Epsilon, *Max, *Min;
   DenormMin = PickFP(Sem, "1.40129846e-45F", "4.9406564584124654e-324",
@@ -153,27 +153,27 @@ static void DefineFloatMacros(MacroBuilder &Builder, llvm::StringRef Prefix,
 
   Builder.defineMacro(DefPrefix + "DENORM_MIN__", DenormMin);
   Builder.defineMacro(DefPrefix + "HAS_DENORM__");
-  Builder.defineMacro(DefPrefix + "DIG__", llvm::Twine(Digits));
-  Builder.defineMacro(DefPrefix + "EPSILON__", llvm::Twine(Epsilon));
+  Builder.defineMacro(DefPrefix + "DIG__", Twine(Digits));
+  Builder.defineMacro(DefPrefix + "EPSILON__", Twine(Epsilon));
   Builder.defineMacro(DefPrefix + "HAS_INFINITY__");
   Builder.defineMacro(DefPrefix + "HAS_QUIET_NAN__");
-  Builder.defineMacro(DefPrefix + "MANT_DIG__", llvm::Twine(MantissaDigits));
+  Builder.defineMacro(DefPrefix + "MANT_DIG__", Twine(MantissaDigits));
 
-  Builder.defineMacro(DefPrefix + "MAX_10_EXP__", llvm::Twine(Max10Exp));
-  Builder.defineMacro(DefPrefix + "MAX_EXP__", llvm::Twine(MaxExp));
-  Builder.defineMacro(DefPrefix + "MAX__", llvm::Twine(Max));
+  Builder.defineMacro(DefPrefix + "MAX_10_EXP__", Twine(Max10Exp));
+  Builder.defineMacro(DefPrefix + "MAX_EXP__", Twine(MaxExp));
+  Builder.defineMacro(DefPrefix + "MAX__", Twine(Max));
 
-  Builder.defineMacro(DefPrefix + "MIN_10_EXP__","("+llvm::Twine(Min10Exp)+")");
-  Builder.defineMacro(DefPrefix + "MIN_EXP__", "("+llvm::Twine(MinExp)+")");
-  Builder.defineMacro(DefPrefix + "MIN__", llvm::Twine(Min));
+  Builder.defineMacro(DefPrefix + "MIN_10_EXP__","("+Twine(Min10Exp)+")");
+  Builder.defineMacro(DefPrefix + "MIN_EXP__", "("+Twine(MinExp)+")");
+  Builder.defineMacro(DefPrefix + "MIN__", Twine(Min));
 }
 
 
 /// DefineTypeSize - Emit a macro to the predefines buffer that declares a macro
 /// named MacroName with the max value for a type with width 'TypeWidth' a
 /// signedness of 'isSigned' and with a value suffix of 'ValSuffix' (e.g. LL).
-static void DefineTypeSize(llvm::StringRef MacroName, unsigned TypeWidth,
-                           llvm::StringRef ValSuffix, bool isSigned,
+static void DefineTypeSize(StringRef MacroName, unsigned TypeWidth,
+                           StringRef ValSuffix, bool isSigned,
                            MacroBuilder &Builder) {
   llvm::APInt MaxVal = isSigned ? llvm::APInt::getSignedMaxValue(TypeWidth)
                                 : llvm::APInt::getMaxValue(TypeWidth);
@@ -182,26 +182,26 @@ static void DefineTypeSize(llvm::StringRef MacroName, unsigned TypeWidth,
 
 /// DefineTypeSize - An overloaded helper that uses TargetInfo to determine
 /// the width, suffix, and signedness of the given type
-static void DefineTypeSize(llvm::StringRef MacroName, TargetInfo::IntType Ty,
+static void DefineTypeSize(StringRef MacroName, TargetInfo::IntType Ty,
                            const TargetInfo &TI, MacroBuilder &Builder) {
   DefineTypeSize(MacroName, TI.getTypeWidth(Ty), TI.getTypeConstantSuffix(Ty), 
                  TI.isTypeSigned(Ty), Builder);
 }
 
-static void DefineType(const llvm::Twine &MacroName, TargetInfo::IntType Ty,
+static void DefineType(const Twine &MacroName, TargetInfo::IntType Ty,
                        MacroBuilder &Builder) {
   Builder.defineMacro(MacroName, TargetInfo::getTypeName(Ty));
 }
 
-static void DefineTypeWidth(llvm::StringRef MacroName, TargetInfo::IntType Ty,
+static void DefineTypeWidth(StringRef MacroName, TargetInfo::IntType Ty,
                             const TargetInfo &TI, MacroBuilder &Builder) {
-  Builder.defineMacro(MacroName, llvm::Twine(TI.getTypeWidth(Ty)));
+  Builder.defineMacro(MacroName, Twine(TI.getTypeWidth(Ty)));
 }
 
-static void DefineTypeSizeof(llvm::StringRef MacroName, unsigned BitWidth,
+static void DefineTypeSizeof(StringRef MacroName, unsigned BitWidth,
                              const TargetInfo &TI, MacroBuilder &Builder) {
   Builder.defineMacro(MacroName,
-                      llvm::Twine(BitWidth / TI.getCharWidth()));
+                      Twine(BitWidth / TI.getCharWidth()));
 }
 
 static void DefineExactWidthIntType(TargetInfo::IntType Ty, 
@@ -213,80 +213,14 @@ static void DefineExactWidthIntType(TargetInfo::IntType Ty,
   if (TypeWidth == 64)
     Ty = TI.getInt64Type();
 
-  DefineType("__INT" + llvm::Twine(TypeWidth) + "_TYPE__", Ty, Builder);
+  DefineType("__INT" + Twine(TypeWidth) + "_TYPE__", Ty, Builder);
 
-  llvm::StringRef ConstSuffix(TargetInfo::getTypeConstantSuffix(Ty));
+  StringRef ConstSuffix(TargetInfo::getTypeConstantSuffix(Ty));
   if (!ConstSuffix.empty())
-    Builder.defineMacro("__INT" + llvm::Twine(TypeWidth) + "_C_SUFFIX__",
+    Builder.defineMacro("__INT" + Twine(TypeWidth) + "_C_SUFFIX__",
                         ConstSuffix);
 }
 
-/// \brief Add definitions required for a smooth interaction between
-/// Objective-C++ automatic reference counting and libc++.
-static void AddObjCXXARCLibcxxDefines(const LangOptions &LangOpts, 
-                                      MacroBuilder &Builder) {
-  Builder.defineMacro("_LIBCPP_PREDEFINED_OBJC_ARC_ADDRESSOF");
-  
-  std::string Result;
-  {
-    // Provide overloads of the function std::__1::addressof() that accept
-    // references to lifetime-qualified objects. libc++'s (more general)
-    // std::__1::addressof() template fails to instantiate with such types,
-    // because it attempts to convert the object to a char& before 
-    // dereferencing.
-    llvm::raw_string_ostream Out(Result);
-    
-    Out << "#pragma clang diagnostic push\n"
-        << "#pragma clang diagnostic ignored \"-Wc++0x-extensions\"\n"
-        << "namespace std { inline namespace __1 {\n"
-        << "\n";
-    
-    Out << "template <class _Tp>\n"
-        << "inline __attribute__ ((__visibility__(\"hidden\"), "
-        << "__always_inline__))\n"
-        << "__attribute__((objc_ownership(strong))) _Tp*\n"
-        << "addressof(__attribute__((objc_ownership(strong))) _Tp& __x) {\n"
-        << "  return &__x;\n"
-        << "}\n"
-        << "\n";
-      
-    if (LangOpts.ObjCRuntimeHasWeak) {
-      Out << "template <class _Tp>\n"
-          << "inline __attribute__ ((__visibility__(\"hidden\"),"
-          << "__always_inline__))\n"
-          << "__attribute__((objc_ownership(weak))) _Tp*\n"
-          << "addressof(__attribute__((objc_ownership(weak))) _Tp& __x) {\n"
-          << "  return &__x;\n"
-          << "};\n"
-          << "\n";
-    }
-      
-    Out << "template <class _Tp>\n"
-        << "inline __attribute__ ((__visibility__(\"hidden\"),"
-        << "__always_inline__))\n"
-        << "__attribute__((objc_ownership(autoreleasing))) _Tp*\n"
-        << "addressof(__attribute__((objc_ownership(autoreleasing))) _Tp& __x) "
-        << "{\n"
-        << " return &__x;\n"
-        << "}\n"
-        << "\n";
-    
-    Out << "template <class _Tp>\n"
-        << "inline __attribute__ ((__visibility__(\"hidden\"), "
-        << "__always_inline__))\n"
-        << "__unsafe_unretained _Tp* addressof(__unsafe_unretained _Tp& __x)"
-        << " {\n"
-        << "  return &__x;\n"
-        << "}\n";
-      
-    Out << "\n"
-        << "} }\n"
-        << "#pragma clang diagnostic pop\n"
-        << "\n";    
-  }
-  Builder.append(Result);
-}
-
 /// \brief Add definitions required for a smooth interaction between
 /// Objective-C++ automated reference counting and libstdc++ (4.2).
 static void AddObjCXXARCLibstdcxxDefines(const LangOptions &LangOpts, 
@@ -344,7 +278,7 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI,
                                                const LangOptions &LangOpts,
                                                const FrontendOptions &FEOpts,
                                                MacroBuilder &Builder) {
-  if (!LangOpts.Microsoft && !LangOpts.TraditionalCPP)
+  if (!LangOpts.MicrosoftExt && !LangOpts.TraditionalCPP)
     Builder.defineMacro("__STDC__");
   if (LangOpts.Freestanding)
     Builder.defineMacro("__STDC_HOSTED__", "0");
@@ -412,7 +346,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   // checks that it is necessary to report 4.2.1 (the base GCC version we claim
   // compatibility with) first.
   Builder.defineMacro("__VERSION__", "\"4.2.1 Compatible " + 
-                      llvm::Twine(getClangFullCPPVersion()) + "\"");
+                      Twine(getClangFullCPPVersion()) + "\"");
 
   // Initialize language-specific preprocessor defines.
 
@@ -426,10 +360,12 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   if (LangOpts.ObjC1) {
     if (LangOpts.ObjCNonFragileABI) {
       Builder.defineMacro("__OBJC2__");
-      Builder.defineMacro("OBJC_ZEROCOST_EXCEPTIONS");
+      
+      if (LangOpts.ObjCExceptions)
+        Builder.defineMacro("OBJC_ZEROCOST_EXCEPTIONS");
     }
 
-    if (LangOpts.getGCMode() != LangOptions::NonGC)
+    if (LangOpts.getGC() != LangOptions::NonGC)
       Builder.defineMacro("__OBJC_GC__");
 
     if (LangOpts.NeXTRuntime)
@@ -452,7 +388,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     Builder.defineMacro("__BLOCKS__");
   }
 
-  if (LangOpts.Exceptions)
+  if (LangOpts.CXXExceptions)
     Builder.defineMacro("__EXCEPTIONS");
   if (LangOpts.RTTI)
     Builder.defineMacro("__GXX_RTTI");
@@ -468,7 +404,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     Builder.defineMacro("__private_extern__", "extern");
   }
 
-  if (LangOpts.Microsoft) {
+  if (LangOpts.MicrosoftExt) {
     // Both __PRETTY_FUNCTION__ and __FUNCTION__ are GCC extensions, however
     // VC++ appears to only like __FUNCTION__.
     Builder.defineMacro("__PRETTY_FUNCTION__", "__FUNCTION__");
@@ -546,7 +482,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
 
   // Define a __POINTER_WIDTH__ macro for stdint.h.
   Builder.defineMacro("__POINTER_WIDTH__",
-                      llvm::Twine((int)TI.getPointerWidth(0)));
+                      Twine((int)TI.getPointerWidth(0)));
 
   if (!LangOpts.CharIsSigned)
     Builder.defineMacro("__CHAR_UNSIGNED__");
@@ -555,7 +491,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     Builder.defineMacro("__WINT_UNSIGNED__");
 
   // Define exact-width integer types for stdint.h
-  Builder.defineMacro("__INT" + llvm::Twine(TI.getCharWidth()) + "_TYPE__",
+  Builder.defineMacro("__INT" + Twine(TI.getCharWidth()) + "_TYPE__",
                       "char");
 
   if (TI.getShortWidth() > TI.getCharWidth())
@@ -589,19 +525,19 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     Builder.defineMacro("__NO_INLINE__");
 
   if (unsigned PICLevel = LangOpts.PICLevel) {
-    Builder.defineMacro("__PIC__", llvm::Twine(PICLevel));
-    Builder.defineMacro("__pic__", llvm::Twine(PICLevel));
+    Builder.defineMacro("__PIC__", Twine(PICLevel));
+    Builder.defineMacro("__pic__", Twine(PICLevel));
   }
 
   // Macros to control C99 numerics and <float.h>
   Builder.defineMacro("__FLT_EVAL_METHOD__", "0");
   Builder.defineMacro("__FLT_RADIX__", "2");
   int Dig = PickFP(&TI.getLongDoubleFormat(), -1/*FIXME*/, 17, 21, 33, 36);
-  Builder.defineMacro("__DECIMAL_DIG__", llvm::Twine(Dig));
+  Builder.defineMacro("__DECIMAL_DIG__", Twine(Dig));
 
-  if (LangOpts.getStackProtectorMode() == LangOptions::SSPOn)
+  if (LangOpts.getStackProtector() == LangOptions::SSPOn)
     Builder.defineMacro("__SSP__");
-  else if (LangOpts.getStackProtectorMode() == LangOptions::SSPReq)
+  else if (LangOpts.getStackProtector() == LangOptions::SSPReq)
     Builder.defineMacro("__SSP_ALL__", "2");
 
   if (FEOpts.ProgramAction == frontend::RewriteObjC)
@@ -629,7 +565,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
 
 // Initialize the remapping of files to alternative contents, e.g.,
 // those specified through other files.
-static void InitializeFileRemapping(Diagnostic &Diags,
+static void InitializeFileRemapping(DiagnosticsEngine &Diags,
                                     SourceManager &SourceMgr,
                                     FileManager &FileMgr,
                                     const PreprocessorOptions &InitOpts) {
@@ -705,6 +641,10 @@ void clang::InitializePreprocessor(Preprocessor &PP,
   InitializeFileRemapping(PP.getDiagnostics(), PP.getSourceManager(),
                           PP.getFileManager(), InitOpts);
 
+  // Specify whether the preprocessor should replace #include/#import with
+  // module imports when plausible.
+  PP.setAutoModuleImport(InitOpts.AutoModuleImport);
+
   // Emit line markers for various builtin sections of the file.  We don't do
   // this in asm preprocessor mode, because "# 4" is not a line marker directive
   // in this mode.
@@ -720,10 +660,7 @@ void clang::InitializePreprocessor(Preprocessor &PP,
     if (LangOpts.ObjC1 && LangOpts.CPlusPlus && LangOpts.ObjCAutoRefCount) {
       switch (InitOpts.ObjCXXARCStandardLibrary) {
       case ARCXX_nolib:
-        break;
-
-      case ARCXX_libcxx:
-        AddObjCXXARCLibcxxDefines(LangOpts, Builder);
+        case ARCXX_libcxx:
         break;
 
       case ARCXX_libstdcxx:
@@ -778,7 +715,7 @@ void clang::InitializePreprocessor(Preprocessor &PP,
                           
   // Copy PredefinedBuffer into the Preprocessor.
   PP.setPredefines(Predefines.str());
-
+  
   // Initialize the header search object.
   ApplyHeaderSearchOptions(PP.getHeaderSearchInfo(), HSOpts,
                            PP.getLangOptions(),
diff --git a/contrib/llvm/tools/clang/lib/Frontend/LangStandards.cpp b/contrib/llvm/tools/clang/lib/Frontend/LangStandards.cpp
index af1721d0f9ec..abb521ba27f2 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/LangStandards.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/LangStandards.cpp
@@ -29,7 +29,7 @@ const LangStandard &LangStandard::getLangStandardForKind(Kind K) {
   }
 }
 
-const LangStandard *LangStandard::getLangStandardForName(llvm::StringRef Name) {
+const LangStandard *LangStandard::getLangStandardForName(StringRef Name) {
   Kind K = llvm::StringSwitch<Kind>(Name)
 #define LANGSTANDARD(id, name, desc, features) \
     .Case(name, lang_##id)
diff --git a/contrib/llvm/tools/clang/lib/Frontend/LogDiagnosticPrinter.cpp b/contrib/llvm/tools/clang/lib/Frontend/LogDiagnosticPrinter.cpp
index 78eb1b212836..8b585be90ec4 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/LogDiagnosticPrinter.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/LogDiagnosticPrinter.cpp
@@ -14,7 +14,7 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
-LogDiagnosticPrinter::LogDiagnosticPrinter(llvm::raw_ostream &os,
+LogDiagnosticPrinter::LogDiagnosticPrinter(raw_ostream &os,
                                            const DiagnosticOptions &diags,
                                            bool _OwnsOutputStream)
   : OS(os), LangOpts(0), DiagOpts(&diags),
@@ -26,15 +26,30 @@ LogDiagnosticPrinter::~LogDiagnosticPrinter() {
     delete &OS;
 }
 
-static llvm::StringRef getLevelName(Diagnostic::Level Level) {
+static StringRef getLevelName(DiagnosticsEngine::Level Level) {
   switch (Level) {
   default:
     return "<unknown>";
-  case Diagnostic::Ignored: return "ignored";
-  case Diagnostic::Note:    return "note";
-  case Diagnostic::Warning: return "warning";
-  case Diagnostic::Error:   return "error";
-  case Diagnostic::Fatal:   return "fatal error";
+  case DiagnosticsEngine::Ignored: return "ignored";
+  case DiagnosticsEngine::Note:    return "note";
+  case DiagnosticsEngine::Warning: return "warning";
+  case DiagnosticsEngine::Error:   return "error";
+  case DiagnosticsEngine::Fatal:   return "fatal error";
+  }
+}
+
+// Escape XML characters inside the raw string.
+static void emitString(llvm::raw_svector_ostream &OS, const StringRef Raw) {
+  for (StringRef::iterator I = Raw.begin(), E = Raw.end(); I != E; ++I) {
+    char c = *I;
+    switch (c) {
+    default:   OS << c; break;
+    case '&':  OS << "&amp;"; break;
+    case '<':  OS << "&lt;"; break;
+    case '>':  OS << "&gt;"; break;
+    case '\'': OS << "&apos;"; break;
+    case '\"': OS << "&quot;"; break;
+    }
   }
 }
 
@@ -42,9 +57,9 @@ void LogDiagnosticPrinter::EndSourceFile() {
   // We emit all the diagnostics in EndSourceFile. However, we don't emit any
   // entry if no diagnostics were present.
   //
-  // Note that DiagnosticClient has no "end-of-compilation" callback, so we will
-  // miss any diagnostics which are emitted after and outside the translation
-  // unit processing.
+  // Note that DiagnosticConsumer has no "end-of-compilation" callback, so we
+  // will miss any diagnostics which are emitted after and outside the
+  // translation unit processing.
   if (Entries.empty())
     return;
 
@@ -55,11 +70,15 @@ void LogDiagnosticPrinter::EndSourceFile() {
   OS << "<dict>\n";
   if (!MainFilename.empty()) {
     OS << "  <key>main-file</key>\n"
-       << "  <string>" << MainFilename << "</string>\n";
+       << "  <string>";
+    emitString(OS, MainFilename);
+    OS << "</string>\n";
   }
   if (!DwarfDebugFlags.empty()) {
     OS << "  <key>dwarf-debug-flags</key>\n"
-       << "  <string>" << DwarfDebugFlags << "</string>\n";
+       << "  <string>";
+    emitString(OS, DwarfDebugFlags);
+    OS << "</string>\n";
   }
   OS << "  <key>diagnostics</key>\n";
   OS << "  <array>\n";
@@ -68,10 +87,14 @@ void LogDiagnosticPrinter::EndSourceFile() {
 
     OS << "    <dict>\n";
     OS << "      <key>level</key>\n"
-       << "      <string>" << getLevelName(DE.DiagnosticLevel) << "</string>\n";
+       << "      <string>";
+    emitString(OS, getLevelName(DE.DiagnosticLevel));
+    OS << "</string>\n";
     if (!DE.Filename.empty()) {
       OS << "      <key>filename</key>\n"
-         << "      <string>" << DE.Filename << "</string>\n";
+         << "      <string>";
+      emitString(OS, DE.Filename);
+      OS << "</string>\n";
     }
     if (DE.Line != 0) {
       OS << "      <key>line</key>\n"
@@ -83,7 +106,9 @@ void LogDiagnosticPrinter::EndSourceFile() {
     }
     if (!DE.Message.empty()) {
       OS << "      <key>message</key>\n"
-         << "      <string>" << DE.Message << "</string>\n";
+         << "      <string>";
+      emitString(OS, DE.Message);
+      OS << "</string>\n";
     }
     OS << "    </dict>\n";
   }
@@ -93,10 +118,10 @@ void LogDiagnosticPrinter::EndSourceFile() {
   this->OS << OS.str();
 }
 
-void LogDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
-                                            const DiagnosticInfo &Info) {
+void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
+                                            const Diagnostic &Info) {
   // Default implementation (Warnings/errors count).
-  DiagnosticClient::HandleDiagnostic(Level, Info);
+  DiagnosticConsumer::HandleDiagnostic(Level, Info);
 
   // Initialize the main file name, if we haven't already fetched it.
   if (MainFilename.empty() && Info.hasSourceManager()) {
@@ -144,3 +169,9 @@ void LogDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
   // Record the diagnostic entry.
   Entries.push_back(DE);
 }
+
+DiagnosticConsumer *
+LogDiagnosticPrinter::clone(DiagnosticsEngine &Diags) const {
+  return new LogDiagnosticPrinter(OS, *DiagOpts, /*OwnsOutputStream=*/false);
+}
+
diff --git a/contrib/llvm/tools/clang/lib/Frontend/MultiplexConsumer.cpp b/contrib/llvm/tools/clang/lib/Frontend/MultiplexConsumer.cpp
index 5aa65d7a60aa..8e746f65a906 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/MultiplexConsumer.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/MultiplexConsumer.cpp
@@ -37,7 +37,7 @@ class MultiplexASTDeserializationListener
   virtual void TypeRead(serialization::TypeIdx Idx, QualType T);
   virtual void DeclRead(serialization::DeclID ID, const Decl *D);
   virtual void SelectorRead(serialization::SelectorID iD, Selector Sel);
-  virtual void MacroDefinitionRead(serialization::MacroID, 
+  virtual void MacroDefinitionRead(serialization::PreprocessedEntityID, 
                                    MacroDefinition *MD);
 private:
   std::vector<ASTDeserializationListener*> Listeners;
@@ -79,7 +79,7 @@ void MultiplexASTDeserializationListener::SelectorRead(
 }
 
 void MultiplexASTDeserializationListener::MacroDefinitionRead(
-    serialization::MacroID ID, MacroDefinition *MD) {
+    serialization::PreprocessedEntityID ID, MacroDefinition *MD) {
   for (size_t i = 0, e = Listeners.size(); i != e; ++i)
     Listeners[i]->MacroDefinitionRead(ID, MD);
 }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/contrib/llvm/tools/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index c892960a18b9..8a61f968e67f 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -33,7 +33,7 @@ using namespace clang;
 /// PrintMacroDefinition - Print a macro definition in a form that will be
 /// properly accepted back as a definition.
 static void PrintMacroDefinition(const IdentifierInfo &II, const MacroInfo &MI,
-                                 Preprocessor &PP, llvm::raw_ostream &OS) {
+                                 Preprocessor &PP, raw_ostream &OS) {
   OS << "#define " << II.getName();
 
   if (MI.isFunctionLike()) {
@@ -83,7 +83,7 @@ class PrintPPOutputPPCallbacks : public PPCallbacks {
   SourceManager &SM;
   TokenConcatenation ConcatInfo;
 public:
-  llvm::raw_ostream &OS;
+  raw_ostream &OS;
 private:
   unsigned CurLine;
 
@@ -96,7 +96,7 @@ class PrintPPOutputPPCallbacks : public PPCallbacks {
   bool DumpDefines;
   bool UseLineDirective;
 public:
-  PrintPPOutputPPCallbacks(Preprocessor &pp, llvm::raw_ostream &os,
+  PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream &os,
                            bool lineMarkers, bool defines)
      : PP(pp), SM(PP.getSourceManager()),
        ConcatInfo(PP), OS(os), DisableLineMarkers(lineMarkers),
@@ -109,7 +109,7 @@ class PrintPPOutputPPCallbacks : public PPCallbacks {
     Initialized = false;
 
     // If we're in microsoft mode, use normal #line instead of line markers.
-    UseLineDirective = PP.getLangOptions().Microsoft;
+    UseLineDirective = PP.getLangOptions().MicrosoftExt;
   }
 
   void SetEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
@@ -118,17 +118,18 @@ class PrintPPOutputPPCallbacks : public PPCallbacks {
   bool StartNewLineIfNeeded();
   
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType);
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID);
   virtual void Ident(SourceLocation Loc, const std::string &str);
   virtual void PragmaComment(SourceLocation Loc, const IdentifierInfo *Kind,
                              const std::string &Str);
-  virtual void PragmaMessage(SourceLocation Loc, llvm::StringRef Str);
+  virtual void PragmaMessage(SourceLocation Loc, StringRef Str);
   virtual void PragmaDiagnosticPush(SourceLocation Loc,
-                                    llvm::StringRef Namespace);
+                                    StringRef Namespace);
   virtual void PragmaDiagnosticPop(SourceLocation Loc,
-                                   llvm::StringRef Namespace);
-  virtual void PragmaDiagnostic(SourceLocation Loc, llvm::StringRef Namespace,
-                                diag::Mapping Map, llvm::StringRef Str);
+                                   StringRef Namespace);
+  virtual void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
+                                diag::Mapping Map, StringRef Str);
 
   bool HandleFirstTokOnLine(Token &Tok);
   bool MoveToLine(SourceLocation Loc) {
@@ -235,7 +236,8 @@ bool PrintPPOutputPPCallbacks::StartNewLineIfNeeded() {
 /// position.
 void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
                                            FileChangeReason Reason,
-                                       SrcMgr::CharacteristicKind NewFileType) {
+                                       SrcMgr::CharacteristicKind NewFileType,
+                                       FileID PrevFID) {
   // Unless we are exiting a #include, make sure to skip ahead to the line the
   // #include directive was at.
   SourceManager &SourceMgr = SM;
@@ -346,7 +348,7 @@ void PrintPPOutputPPCallbacks::PragmaComment(SourceLocation Loc,
 }
 
 void PrintPPOutputPPCallbacks::PragmaMessage(SourceLocation Loc,
-                                             llvm::StringRef Str) {
+                                             StringRef Str) {
   MoveToLine(Loc);
   OS << "#pragma message(";
 
@@ -369,22 +371,22 @@ void PrintPPOutputPPCallbacks::PragmaMessage(SourceLocation Loc,
 }
 
 void PrintPPOutputPPCallbacks::
-PragmaDiagnosticPush(SourceLocation Loc, llvm::StringRef Namespace) {
+PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) {
   MoveToLine(Loc);
   OS << "#pragma " << Namespace << " diagnostic push";
   EmittedTokensOnThisLine = true;
 }
 
 void PrintPPOutputPPCallbacks::
-PragmaDiagnosticPop(SourceLocation Loc, llvm::StringRef Namespace) {
+PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) {
   MoveToLine(Loc);
   OS << "#pragma " << Namespace << " diagnostic pop";
   EmittedTokensOnThisLine = true;
 }
 
 void PrintPPOutputPPCallbacks::
-PragmaDiagnostic(SourceLocation Loc, llvm::StringRef Namespace,
-                 diag::Mapping Map, llvm::StringRef Str) {
+PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
+                 diag::Mapping Map, StringRef Str) {
   MoveToLine(Loc);
   OS << "#pragma " << Namespace << " diagnostic ";
   switch (Map) {
@@ -419,7 +421,7 @@ bool PrintPPOutputPPCallbacks::HandleFirstTokOnLine(Token &Tok) {
 
   // Print out space characters so that the first token on a line is
   // indented for easy reading.
-  unsigned ColNo = SM.getInstantiationColumnNumber(Tok.getLocation());
+  unsigned ColNo = SM.getExpansionColumnNumber(Tok.getLocation());
 
   // This hack prevents stuff like:
   // #define HASH #
@@ -491,7 +493,7 @@ struct UnknownPragmaHandler : public PragmaHandler {
 
 static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
                                     PrintPPOutputPPCallbacks *Callbacks,
-                                    llvm::raw_ostream &OS) {
+                                    raw_ostream &OS) {
   char Buffer[256];
   Token PrevPrevTok, PrevTok;
   PrevPrevTok.startToken();
@@ -550,7 +552,7 @@ static int MacroIDCompare(const void* a, const void* b) {
   return LHS->first->getName().compare(RHS->first->getName());
 }
 
-static void DoPrintMacros(Preprocessor &PP, llvm::raw_ostream *OS) {
+static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {
   // Ignore unknown pragmas.
   PP.AddPragmaHandler(new EmptyPragmaHandler());
 
@@ -562,7 +564,7 @@ static void DoPrintMacros(Preprocessor &PP, llvm::raw_ostream *OS) {
   do PP.Lex(Tok);
   while (Tok.isNot(tok::eof));
 
-  llvm::SmallVector<id_macro_pair, 128>
+  SmallVector<id_macro_pair, 128>
     MacrosByID(PP.macro_begin(), PP.macro_end());
   llvm::array_pod_sort(MacrosByID.begin(), MacrosByID.end(), MacroIDCompare);
 
@@ -578,7 +580,7 @@ static void DoPrintMacros(Preprocessor &PP, llvm::raw_ostream *OS) {
 
 /// DoPrintPreprocessedInput - This implements -E mode.
 ///
-void clang::DoPrintPreprocessedInput(Preprocessor &PP, llvm::raw_ostream *OS,
+void clang::DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream *OS,
                                      const PreprocessorOutputOptions &Opts) {
   // Show macros with no output is handled specially.
   if (!Opts.ShowCPP) {
diff --git a/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticBuffer.cpp b/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticBuffer.cpp
index 069c86de137f..f8ea9f1361ab 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticBuffer.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticBuffer.cpp
@@ -13,39 +13,48 @@
 
 #include "clang/Frontend/TextDiagnosticBuffer.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 
 /// HandleDiagnostic - Store the errors, warnings, and notes that are
 /// reported.
 ///
-void TextDiagnosticBuffer::HandleDiagnostic(Diagnostic::Level Level,
-                                            const DiagnosticInfo &Info) {
+void TextDiagnosticBuffer::HandleDiagnostic(DiagnosticsEngine::Level Level,
+                                            const Diagnostic &Info) {
   // Default implementation (Warnings/errors count).
-  DiagnosticClient::HandleDiagnostic(Level, Info);
+  DiagnosticConsumer::HandleDiagnostic(Level, Info);
 
   llvm::SmallString<100> Buf;
   Info.FormatDiagnostic(Buf);
   switch (Level) {
-  default: assert(0 && "Diagnostic not handled during diagnostic buffering!");
-  case Diagnostic::Note:
+  default: llvm_unreachable(
+                         "Diagnostic not handled during diagnostic buffering!");
+  case DiagnosticsEngine::Note:
     Notes.push_back(std::make_pair(Info.getLocation(), Buf.str()));
     break;
-  case Diagnostic::Warning:
+  case DiagnosticsEngine::Warning:
     Warnings.push_back(std::make_pair(Info.getLocation(), Buf.str()));
     break;
-  case Diagnostic::Error:
-  case Diagnostic::Fatal:
+  case DiagnosticsEngine::Error:
+  case DiagnosticsEngine::Fatal:
     Errors.push_back(std::make_pair(Info.getLocation(), Buf.str()));
     break;
   }
 }
 
-void TextDiagnosticBuffer::FlushDiagnostics(Diagnostic &Diags) const {
+void TextDiagnosticBuffer::FlushDiagnostics(DiagnosticsEngine &Diags) const {
   // FIXME: Flush the diagnostics in order.
   for (const_iterator it = err_begin(), ie = err_end(); it != ie; ++it)
-    Diags.Report(Diags.getCustomDiagID(Diagnostic::Error, it->second.c_str()));
+    Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error,
+                 it->second.c_str()));
   for (const_iterator it = warn_begin(), ie = warn_end(); it != ie; ++it)
-    Diags.Report(Diags.getCustomDiagID(Diagnostic::Warning,it->second.c_str()));
+    Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Warning,
+                 it->second.c_str()));
   for (const_iterator it = note_begin(), ie = note_end(); it != ie; ++it)
-    Diags.Report(Diags.getCustomDiagID(Diagnostic::Note, it->second.c_str()));
+    Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Note,
+                 it->second.c_str()));
+}
+
+DiagnosticConsumer *TextDiagnosticBuffer::clone(DiagnosticsEngine &) const {
+  return new TextDiagnosticBuffer();
 }
diff --git a/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticPrinter.cpp b/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticPrinter.cpp
index e49e19a17c8d..10e7238218cb 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -18,29 +18,29 @@
 #include "clang/Lex/Lexer.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringExtras.h"
 #include <algorithm>
 using namespace clang;
 
-static const enum llvm::raw_ostream::Colors noteColor =
-  llvm::raw_ostream::BLACK;
-static const enum llvm::raw_ostream::Colors fixitColor =
-  llvm::raw_ostream::GREEN;
-static const enum llvm::raw_ostream::Colors caretColor =
-  llvm::raw_ostream::GREEN;
-static const enum llvm::raw_ostream::Colors warningColor =
-  llvm::raw_ostream::MAGENTA;
-static const enum llvm::raw_ostream::Colors errorColor = llvm::raw_ostream::RED;
-static const enum llvm::raw_ostream::Colors fatalColor = llvm::raw_ostream::RED;
+static const enum raw_ostream::Colors noteColor =
+  raw_ostream::BLACK;
+static const enum raw_ostream::Colors fixitColor =
+  raw_ostream::GREEN;
+static const enum raw_ostream::Colors caretColor =
+  raw_ostream::GREEN;
+static const enum raw_ostream::Colors warningColor =
+  raw_ostream::MAGENTA;
+static const enum raw_ostream::Colors errorColor = raw_ostream::RED;
+static const enum raw_ostream::Colors fatalColor = raw_ostream::RED;
 // Used for changing only the bold attribute.
-static const enum llvm::raw_ostream::Colors savedColor =
-  llvm::raw_ostream::SAVEDCOLOR;
+static const enum raw_ostream::Colors savedColor =
+  raw_ostream::SAVEDCOLOR;
 
 /// \brief Number of spaces to indent when word-wrapping.
 const unsigned WordWrapIndentation = 6;
 
-TextDiagnosticPrinter::TextDiagnosticPrinter(llvm::raw_ostream &os,
+TextDiagnosticPrinter::TextDiagnosticPrinter(raw_ostream &os,
                                              const DiagnosticOptions &diags,
                                              bool _OwnsOutputStream)
   : OS(os), LangOpts(0), DiagOpts(&diags),
@@ -53,105 +53,52 @@ TextDiagnosticPrinter::~TextDiagnosticPrinter() {
     delete &OS;
 }
 
-void TextDiagnosticPrinter::PrintIncludeStack(Diagnostic::Level Level,
-                                              SourceLocation Loc,
-                                              const SourceManager &SM) {
-  if (!DiagOpts->ShowNoteIncludeStack && Level == Diagnostic::Note) return;
-
-  if (Loc.isInvalid()) return;
+/// \brief Helper to recursivly walk up the include stack and print each layer
+/// on the way back down.
+static void PrintIncludeStackRecursively(raw_ostream &OS,
+                                         const SourceManager &SM,
+                                         SourceLocation Loc,
+                                         bool ShowLocation) {
+  if (Loc.isInvalid())
+    return;
 
   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
   if (PLoc.isInvalid())
     return;
-  
-  // Print out the other include frames first.
-  PrintIncludeStack(Level, PLoc.getIncludeLoc(), SM);
 
-  if (DiagOpts->ShowLocation)
+  // Print out the other include frames first.
+  PrintIncludeStackRecursively(OS, SM, PLoc.getIncludeLoc(), ShowLocation);
+
+  if (ShowLocation)
     OS << "In file included from " << PLoc.getFilename()
        << ':' << PLoc.getLine() << ":\n";
   else
     OS << "In included file:\n";
 }
 
-/// HighlightRange - Given a SourceRange and a line number, highlight (with ~'s)
-/// any characters in LineNo that intersect the SourceRange.
-void TextDiagnosticPrinter::HighlightRange(const CharSourceRange &R,
-                                           const SourceManager &SM,
-                                           unsigned LineNo, FileID FID,
-                                           std::string &CaretLine,
-                                           const std::string &SourceLine) {
-  assert(CaretLine.size() == SourceLine.size() &&
-         "Expect a correspondence between source and caret line!");
-  if (!R.isValid()) return;
+/// \brief Prints an include stack when appropriate for a particular diagnostic
+/// level and location.
+///
+/// This routine handles all the logic of suppressing particular include stacks
+/// (such as those for notes) and duplicate include stacks when repeated
+/// warnings occur within the same file. It also handles the logic of
+/// customizing the formatting and display of the include stack.
+///
+/// \param Level The diagnostic level of the message this stack pertains to.
+/// \param Loc   The include location of the current file (not the diagnostic
+///              location).
+void TextDiagnosticPrinter::PrintIncludeStack(DiagnosticsEngine::Level Level,
+                                              SourceLocation Loc,
+                                              const SourceManager &SM) {
+  // Skip redundant include stacks altogether.
+  if (LastWarningLoc == Loc)
+    return;
+  LastWarningLoc = Loc;
 
-  SourceLocation Begin = SM.getInstantiationLoc(R.getBegin());
-  SourceLocation End = SM.getInstantiationLoc(R.getEnd());
+  if (!DiagOpts->ShowNoteIncludeStack && Level == DiagnosticsEngine::Note)
+    return;
 
-  // If the End location and the start location are the same and are a macro
-  // location, then the range was something that came from a macro expansion
-  // or _Pragma.  If this is an object-like macro, the best we can do is to
-  // highlight the range.  If this is a function-like macro, we'd also like to
-  // highlight the arguments.
-  if (Begin == End && R.getEnd().isMacroID())
-    End = SM.getInstantiationRange(R.getEnd()).second;
-
-  unsigned StartLineNo = SM.getInstantiationLineNumber(Begin);
-  if (StartLineNo > LineNo || SM.getFileID(Begin) != FID)
-    return;  // No intersection.
-
-  unsigned EndLineNo = SM.getInstantiationLineNumber(End);
-  if (EndLineNo < LineNo || SM.getFileID(End) != FID)
-    return;  // No intersection.
-
-  // Compute the column number of the start.
-  unsigned StartColNo = 0;
-  if (StartLineNo == LineNo) {
-    StartColNo = SM.getInstantiationColumnNumber(Begin);
-    if (StartColNo) --StartColNo;  // Zero base the col #.
-  }
-
-  // Compute the column number of the end.
-  unsigned EndColNo = CaretLine.size();
-  if (EndLineNo == LineNo) {
-    EndColNo = SM.getInstantiationColumnNumber(End);
-    if (EndColNo) {
-      --EndColNo;  // Zero base the col #.
-
-      // Add in the length of the token, so that we cover multi-char tokens if
-      // this is a token range.
-      if (R.isTokenRange())
-        EndColNo += Lexer::MeasureTokenLength(End, SM, *LangOpts);
-    } else {
-      EndColNo = CaretLine.size();
-    }
-  }
-
-  assert(StartColNo <= EndColNo && "Invalid range!");
-
-  // Check that a token range does not highlight only whitespace.
-  if (R.isTokenRange()) {
-    // Pick the first non-whitespace column.
-    while (StartColNo < SourceLine.size() &&
-           (SourceLine[StartColNo] == ' ' || SourceLine[StartColNo] == '\t'))
-      ++StartColNo;
-
-    // Pick the last non-whitespace column.
-    if (EndColNo > SourceLine.size())
-      EndColNo = SourceLine.size();
-    while (EndColNo-1 &&
-           (SourceLine[EndColNo-1] == ' ' || SourceLine[EndColNo-1] == '\t'))
-      --EndColNo;
-
-    // If the start/end passed each other, then we are trying to highlight a
-    // range that just exists in whitespace, which must be some sort of other
-    // bug.
-    assert(StartColNo <= EndColNo && "Trying to highlight whitespace??");
-  }
-
-  // Fill the range with ~'s.
-  for (unsigned i = StartColNo; i < EndColNo; ++i)
-    CaretLine[i] = '~';
+  PrintIncludeStackRecursively(OS, SM, Loc, DiagOpts->ShowLocation);
 }
 
 /// \brief When the source code line we want to print is too long for
@@ -300,7 +247,7 @@ static SourceLocation skipToMacroArgExpansion(const SourceManager &SM,
                                                   SourceLocation StartLoc) {
   for (SourceLocation L = StartLoc; L.isMacroID();
        L = SM.getImmediateSpellingLoc(L)) {
-    if (SM.isMacroArgInstantiation(L))
+    if (SM.isMacroArgExpansion(L))
       return L;
   }
 
@@ -317,12 +264,12 @@ static SourceLocation getImmediateMacroCallerLoc(const SourceManager &SM,
   // When we have the location of (part of) an expanded parameter, its spelling
   // location points to the argument as typed into the macro call, and
   // therefore is used to locate the macro caller.
-  if (SM.isMacroArgInstantiation(Loc))
+  if (SM.isMacroArgExpansion(Loc))
     return SM.getImmediateSpellingLoc(Loc);
 
   // Otherwise, the caller of the macro is located where this macro is
   // expanded (while the spelling is part of the macro definition).
-  return SM.getImmediateInstantiationRange(Loc).first;
+  return SM.getImmediateExpansionRange(Loc).first;
 }
 
 /// Gets the location of the immediate macro callee, one level down the stack
@@ -334,34 +281,82 @@ static SourceLocation getImmediateMacroCalleeLoc(const SourceManager &SM,
   // When we have the location of (part of) an expanded parameter, its
   // expansion location points to the unexpanded paramater reference within
   // the macro definition (or callee).
-  if (SM.isMacroArgInstantiation(Loc))
-    return SM.getImmediateInstantiationRange(Loc).first;
+  if (SM.isMacroArgExpansion(Loc))
+    return SM.getImmediateExpansionRange(Loc).first;
 
   // Otherwise, the callee of the macro is located where this location was
   // spelled inside the macro definition.
   return SM.getImmediateSpellingLoc(Loc);
 }
 
-void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
-                                                CharSourceRange *Ranges,
-                                                unsigned NumRanges,
-                                                const SourceManager &SM,
-                                                const FixItHint *Hints,
-                                                unsigned NumHints,
-                                                unsigned Columns,  
-                                                unsigned OnMacroInst,
-                                                unsigned MacroSkipStart,
-                                                unsigned MacroSkipEnd) {
-  assert(LangOpts && "Unexpected diagnostic outside source file processing");
-  assert(!Loc.isInvalid() && "must have a valid source location here");
+namespace {
 
-  // If this is a macro ID, first emit information about where this was
-  // expanded (recursively) then emit information about where the token was
-  // spelled from.
-  if (!Loc.isFileID()) {
-    // Whether to suppress printing this macro expansion.
-    bool Suppressed 
-      = OnMacroInst >= MacroSkipStart && OnMacroInst < MacroSkipEnd;
+/// \brief Class to encapsulate the logic for formatting and printing a textual
+/// diagnostic message.
+///
+/// This class provides an interface for building and emitting a textual
+/// diagnostic, including all of the macro backtraces, caret diagnostics, FixIt
+/// Hints, and code snippets. In the presence of macros this involves
+/// a recursive process, synthesizing notes for each macro expansion.
+///
+/// The purpose of this class is to isolate the implementation of printing
+/// beautiful text diagnostics from any particular interfaces. The Clang
+/// DiagnosticClient is implemented through this class as is diagnostic
+/// printing coming out of libclang.
+///
+/// A brief worklist:
+/// FIXME: Sink the printing of the diagnostic message itself into this class.
+/// FIXME: Sink the printing of the include stack into this class.
+/// FIXME: Remove the TextDiagnosticPrinter as an input.
+/// FIXME: Sink the recursive printing of template instantiations into this
+/// class.
+class TextDiagnostic {
+  TextDiagnosticPrinter &Printer;
+  raw_ostream &OS;
+  const SourceManager &SM;
+  const LangOptions &LangOpts;
+  const DiagnosticOptions &DiagOpts;
+
+public:
+  TextDiagnostic(TextDiagnosticPrinter &Printer,
+                  raw_ostream &OS,
+                  const SourceManager &SM,
+                  const LangOptions &LangOpts,
+                  const DiagnosticOptions &DiagOpts)
+    : Printer(Printer), OS(OS), SM(SM), LangOpts(LangOpts), DiagOpts(DiagOpts) {
+  }
+
+  /// \brief Emit the caret and underlining text.
+  ///
+  /// Walks up the macro expansion stack printing the code snippet, caret,
+  /// underlines and FixItHint display as appropriate at each level. Walk is
+  /// accomplished by calling itself recursively.
+  ///
+  /// FIXME: Remove macro expansion from this routine, it shouldn't be tied to
+  /// caret diagnostics.
+  /// FIXME: Break up massive function into logical units.
+  ///
+  /// \param Loc The location for this caret.
+  /// \param Ranges The underlined ranges for this code snippet.
+  /// \param Hints The FixIt hints active for this diagnostic.
+  /// \param MacroSkipEnd The depth to stop skipping macro expansions.
+  /// \param OnMacroInst The current depth of the macro expansion stack.
+  void EmitCaret(SourceLocation Loc,
+            SmallVectorImpl<CharSourceRange>& Ranges,
+            ArrayRef<FixItHint> Hints,
+            unsigned &MacroDepth,
+            unsigned OnMacroInst = 0) {
+    assert(!Loc.isInvalid() && "must have a valid source location here");
+
+    // If this is a file source location, directly emit the source snippet and
+    // caret line. Also record the macro depth reached.
+    if (Loc.isFileID()) {
+      assert(MacroDepth == 0 && "We shouldn't hit a leaf node twice!");
+      MacroDepth = OnMacroInst;
+      EmitSnippetAndCaret(Loc, Ranges, Hints);
+      return;
+    }
+    // Otherwise recurse through each macro expansion layer.
 
     // When processing macros, skip over the expansions leading up to
     // a macro argument, and trace the argument's expansion stack instead.
@@ -370,21 +365,31 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
     SourceLocation OneLevelUp = getImmediateMacroCallerLoc(SM, Loc);
 
     // FIXME: Map ranges?
-    EmitCaretDiagnostic(OneLevelUp, Ranges, NumRanges, SM,
-                        Hints, NumHints, Columns,
-                        OnMacroInst + 1, MacroSkipStart, MacroSkipEnd);
+    EmitCaret(OneLevelUp, Ranges, Hints, MacroDepth, OnMacroInst + 1);
 
     // Map the location.
     Loc = getImmediateMacroCalleeLoc(SM, Loc);
 
+    unsigned MacroSkipStart = 0, MacroSkipEnd = 0;
+    if (MacroDepth > DiagOpts.MacroBacktraceLimit) {
+      MacroSkipStart = DiagOpts.MacroBacktraceLimit / 2 +
+        DiagOpts.MacroBacktraceLimit % 2;
+      MacroSkipEnd = MacroDepth - DiagOpts.MacroBacktraceLimit / 2;
+    }
+
+    // Whether to suppress printing this macro expansion.
+    bool Suppressed = (OnMacroInst >= MacroSkipStart &&
+                       OnMacroInst < MacroSkipEnd);
+
     // Map the ranges.
-    for (unsigned i = 0; i != NumRanges; ++i) {
-      CharSourceRange &R = Ranges[i];
-      SourceLocation S = R.getBegin(), E = R.getEnd();
-      if (S.isMacroID())
-        R.setBegin(getImmediateMacroCalleeLoc(SM, S));
-      if (E.isMacroID())
-        R.setEnd(getImmediateMacroCalleeLoc(SM, E));
+    for (SmallVectorImpl<CharSourceRange>::iterator I = Ranges.begin(),
+                                                    E = Ranges.end();
+         I != E; ++I) {
+      SourceLocation Start = I->getBegin(), End = I->getEnd();
+      if (Start.isMacroID())
+        I->setBegin(getImmediateMacroCalleeLoc(SM, Start));
+      if (End.isMacroID())
+        I->setEnd(getImmediateMacroCalleeLoc(SM, End));
     }
 
     if (!Suppressed) {
@@ -398,139 +403,248 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
 
       // If this diagnostic is not in the main file, print out the
       // "included from" lines.
-      if (LastWarningLoc != PLoc.getIncludeLoc()) {
-        LastWarningLoc = PLoc.getIncludeLoc();
-        PrintIncludeStack(Diagnostic::Note, LastWarningLoc, SM);
-      }
+      Printer.PrintIncludeStack(DiagnosticsEngine::Note, PLoc.getIncludeLoc(), 
+                                SM);
 
-      if (DiagOpts->ShowLocation) {
+      if (DiagOpts.ShowLocation) {
         // Emit the file/line/column that this expansion came from.
         OS << PLoc.getFilename() << ':' << PLoc.getLine() << ':';
-        if (DiagOpts->ShowColumn)
+        if (DiagOpts.ShowColumn)
           OS << PLoc.getColumn() << ':';
         OS << ' ';
       }
       OS << "note: expanded from:\n";
 
-      EmitCaretDiagnostic(Loc, Ranges, NumRanges, SM, 0, 0,
-                          Columns, OnMacroInst + 1, MacroSkipStart,
-                          MacroSkipEnd);
+      EmitSnippetAndCaret(Loc, Ranges, ArrayRef<FixItHint>());
       return;
     }
-    
+
     if (OnMacroInst == MacroSkipStart) {
       // Tell the user that we've skipped contexts.
       OS << "note: (skipping " << (MacroSkipEnd - MacroSkipStart) 
       << " expansions in backtrace; use -fmacro-backtrace-limit=0 to see "
       "all)\n";
     }
-    
-    return;
   }
-  
-  // Decompose the location into a FID/Offset pair.
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
-  FileID FID = LocInfo.first;
-  unsigned FileOffset = LocInfo.second;
 
-  // Get information about the buffer it points into.
-  bool Invalid = false;
-  const char *BufStart = SM.getBufferData(FID, &Invalid).data();
-  if (Invalid)
-    return;
+  /// \brief Emit a code snippet and caret line.
+  ///
+  /// This routine emits a single line's code snippet and caret line..
+  ///
+  /// \param Loc The location for the caret.
+  /// \param Ranges The underlined ranges for this code snippet.
+  /// \param Hints The FixIt hints active for this diagnostic.
+  void EmitSnippetAndCaret(SourceLocation Loc,
+                           SmallVectorImpl<CharSourceRange>& Ranges,
+                           ArrayRef<FixItHint> Hints) {
+    assert(!Loc.isInvalid() && "must have a valid source location here");
+    assert(Loc.isFileID() && "must have a file location here");
 
-  unsigned ColNo = SM.getColumnNumber(FID, FileOffset);
-  unsigned CaretEndColNo
-    = ColNo + Lexer::MeasureTokenLength(Loc, SM, *LangOpts);
+    // Decompose the location into a FID/Offset pair.
+    std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+    FileID FID = LocInfo.first;
+    unsigned FileOffset = LocInfo.second;
 
-  // Rewind from the current position to the start of the line.
-  const char *TokPtr = BufStart+FileOffset;
-  const char *LineStart = TokPtr-ColNo+1; // Column # is 1-based.
+    // Get information about the buffer it points into.
+    bool Invalid = false;
+    const char *BufStart = SM.getBufferData(FID, &Invalid).data();
+    if (Invalid)
+      return;
 
-
-  // Compute the line end.  Scan forward from the error position to the end of
-  // the line.
-  const char *LineEnd = TokPtr;
-  while (*LineEnd != '\n' && *LineEnd != '\r' && *LineEnd != '\0')
-    ++LineEnd;
-
-  // FIXME: This shouldn't be necessary, but the CaretEndColNo can extend past
-  // the source line length as currently being computed. See
-  // test/Misc/message-length.c.
-  CaretEndColNo = std::min(CaretEndColNo, unsigned(LineEnd - LineStart));
-
-  // Copy the line of code into an std::string for ease of manipulation.
-  std::string SourceLine(LineStart, LineEnd);
-
-  // Create a line for the caret that is filled with spaces that is the same
-  // length as the line of source code.
-  std::string CaretLine(LineEnd-LineStart, ' ');
-
-  // Highlight all of the characters covered by Ranges with ~ characters.
-  if (NumRanges) {
     unsigned LineNo = SM.getLineNumber(FID, FileOffset);
+    unsigned ColNo = SM.getColumnNumber(FID, FileOffset);
+    unsigned CaretEndColNo
+      = ColNo + Lexer::MeasureTokenLength(Loc, SM, LangOpts);
 
-    for (unsigned i = 0, e = NumRanges; i != e; ++i)
-      HighlightRange(Ranges[i], SM, LineNo, FID, CaretLine, SourceLine);
+    // Rewind from the current position to the start of the line.
+    const char *TokPtr = BufStart+FileOffset;
+    const char *LineStart = TokPtr-ColNo+1; // Column # is 1-based.
+
+
+    // Compute the line end.  Scan forward from the error position to the end of
+    // the line.
+    const char *LineEnd = TokPtr;
+    while (*LineEnd != '\n' && *LineEnd != '\r' && *LineEnd != '\0')
+      ++LineEnd;
+
+    // FIXME: This shouldn't be necessary, but the CaretEndColNo can extend past
+    // the source line length as currently being computed. See
+    // test/Misc/message-length.c.
+    CaretEndColNo = std::min(CaretEndColNo, unsigned(LineEnd - LineStart));
+
+    // Copy the line of code into an std::string for ease of manipulation.
+    std::string SourceLine(LineStart, LineEnd);
+
+    // Create a line for the caret that is filled with spaces that is the same
+    // length as the line of source code.
+    std::string CaretLine(LineEnd-LineStart, ' ');
+
+    // Highlight all of the characters covered by Ranges with ~ characters.
+    for (SmallVectorImpl<CharSourceRange>::iterator I = Ranges.begin(),
+                                                    E = Ranges.end();
+         I != E; ++I)
+      HighlightRange(*I, LineNo, FID, SourceLine, CaretLine);
+
+    // Next, insert the caret itself.
+    if (ColNo-1 < CaretLine.size())
+      CaretLine[ColNo-1] = '^';
+    else
+      CaretLine.push_back('^');
+
+    ExpandTabs(SourceLine, CaretLine);
+
+    // If we are in -fdiagnostics-print-source-range-info mode, we are trying
+    // to produce easily machine parsable output.  Add a space before the
+    // source line and the caret to make it trivial to tell the main diagnostic
+    // line from what the user is intended to see.
+    if (DiagOpts.ShowSourceRanges) {
+      SourceLine = ' ' + SourceLine;
+      CaretLine = ' ' + CaretLine;
+    }
+
+    std::string FixItInsertionLine = BuildFixItInsertionLine(LineNo,
+                                                             LineStart, LineEnd,
+                                                             Hints);
+
+    // If the source line is too long for our terminal, select only the
+    // "interesting" source region within that line.
+    unsigned Columns = DiagOpts.MessageLength;
+    if (Columns && SourceLine.size() > Columns)
+      SelectInterestingSourceRegion(SourceLine, CaretLine, FixItInsertionLine,
+                                    CaretEndColNo, Columns);
+
+    // Finally, remove any blank spaces from the end of CaretLine.
+    while (CaretLine[CaretLine.size()-1] == ' ')
+      CaretLine.erase(CaretLine.end()-1);
+
+    // Emit what we have computed.
+    OS << SourceLine << '\n';
+
+    if (DiagOpts.ShowColors)
+      OS.changeColor(caretColor, true);
+    OS << CaretLine << '\n';
+    if (DiagOpts.ShowColors)
+      OS.resetColor();
+
+    if (!FixItInsertionLine.empty()) {
+      if (DiagOpts.ShowColors)
+        // Print fixit line in color
+        OS.changeColor(fixitColor, false);
+      if (DiagOpts.ShowSourceRanges)
+        OS << ' ';
+      OS << FixItInsertionLine << '\n';
+      if (DiagOpts.ShowColors)
+        OS.resetColor();
+    }
+
+    // Print out any parseable fixit information requested by the options.
+    EmitParseableFixits(Hints);
   }
 
-  // Next, insert the caret itself.
-  if (ColNo-1 < CaretLine.size())
-    CaretLine[ColNo-1] = '^';
-  else
-    CaretLine.push_back('^');
+private:
+  /// \brief Highlight a SourceRange (with ~'s) for any characters on LineNo.
+  void HighlightRange(const CharSourceRange &R,
+                      unsigned LineNo, FileID FID,
+                      const std::string &SourceLine,
+                      std::string &CaretLine) {
+    assert(CaretLine.size() == SourceLine.size() &&
+           "Expect a correspondence between source and caret line!");
+    if (!R.isValid()) return;
 
-  // Scan the source line, looking for tabs.  If we find any, manually expand
-  // them to spaces and update the CaretLine to match.
-  for (unsigned i = 0; i != SourceLine.size(); ++i) {
-    if (SourceLine[i] != '\t') continue;
+    SourceLocation Begin = SM.getExpansionLoc(R.getBegin());
+    SourceLocation End = SM.getExpansionLoc(R.getEnd());
 
-    // Replace this tab with at least one space.
-    SourceLine[i] = ' ';
+    // If the End location and the start location are the same and are a macro
+    // location, then the range was something that came from a macro expansion
+    // or _Pragma.  If this is an object-like macro, the best we can do is to
+    // highlight the range.  If this is a function-like macro, we'd also like to
+    // highlight the arguments.
+    if (Begin == End && R.getEnd().isMacroID())
+      End = SM.getExpansionRange(R.getEnd()).second;
 
-    // Compute the number of spaces we need to insert.
-    unsigned TabStop = DiagOpts->TabStop;
-    assert(0 < TabStop && TabStop <= DiagnosticOptions::MaxTabStop &&
-           "Invalid -ftabstop value");
-    unsigned NumSpaces = ((i+TabStop)/TabStop * TabStop) - (i+1);
-    assert(NumSpaces < TabStop && "Invalid computation of space amt");
+    unsigned StartLineNo = SM.getExpansionLineNumber(Begin);
+    if (StartLineNo > LineNo || SM.getFileID(Begin) != FID)
+      return;  // No intersection.
 
-    // Insert spaces into the SourceLine.
-    SourceLine.insert(i+1, NumSpaces, ' ');
+    unsigned EndLineNo = SM.getExpansionLineNumber(End);
+    if (EndLineNo < LineNo || SM.getFileID(End) != FID)
+      return;  // No intersection.
 
-    // Insert spaces or ~'s into CaretLine.
-    CaretLine.insert(i+1, NumSpaces, CaretLine[i] == '~' ? '~' : ' ');
+    // Compute the column number of the start.
+    unsigned StartColNo = 0;
+    if (StartLineNo == LineNo) {
+      StartColNo = SM.getExpansionColumnNumber(Begin);
+      if (StartColNo) --StartColNo;  // Zero base the col #.
+    }
+
+    // Compute the column number of the end.
+    unsigned EndColNo = CaretLine.size();
+    if (EndLineNo == LineNo) {
+      EndColNo = SM.getExpansionColumnNumber(End);
+      if (EndColNo) {
+        --EndColNo;  // Zero base the col #.
+
+        // Add in the length of the token, so that we cover multi-char tokens if
+        // this is a token range.
+        if (R.isTokenRange())
+          EndColNo += Lexer::MeasureTokenLength(End, SM, LangOpts);
+      } else {
+        EndColNo = CaretLine.size();
+      }
+    }
+
+    assert(StartColNo <= EndColNo && "Invalid range!");
+
+    // Check that a token range does not highlight only whitespace.
+    if (R.isTokenRange()) {
+      // Pick the first non-whitespace column.
+      while (StartColNo < SourceLine.size() &&
+             (SourceLine[StartColNo] == ' ' || SourceLine[StartColNo] == '\t'))
+        ++StartColNo;
+
+      // Pick the last non-whitespace column.
+      if (EndColNo > SourceLine.size())
+        EndColNo = SourceLine.size();
+      while (EndColNo-1 &&
+             (SourceLine[EndColNo-1] == ' ' || SourceLine[EndColNo-1] == '\t'))
+        --EndColNo;
+
+      // If the start/end passed each other, then we are trying to highlight a
+      // range that just exists in whitespace, which must be some sort of other
+      // bug.
+      assert(StartColNo <= EndColNo && "Trying to highlight whitespace??");
+    }
+
+    // Fill the range with ~'s.
+    for (unsigned i = StartColNo; i < EndColNo; ++i)
+      CaretLine[i] = '~';
   }
 
-  // If we are in -fdiagnostics-print-source-range-info mode, we are trying to
-  // produce easily machine parsable output.  Add a space before the source line
-  // and the caret to make it trivial to tell the main diagnostic line from what
-  // the user is intended to see.
-  if (DiagOpts->ShowSourceRanges) {
-    SourceLine = ' ' + SourceLine;
-    CaretLine = ' ' + CaretLine;
-  }
+  std::string BuildFixItInsertionLine(unsigned LineNo,
+                                      const char *LineStart,
+                                      const char *LineEnd,
+                                      ArrayRef<FixItHint> Hints) {
+    std::string FixItInsertionLine;
+    if (Hints.empty() || !DiagOpts.ShowFixits)
+      return FixItInsertionLine;
 
-  std::string FixItInsertionLine;
-  if (NumHints && DiagOpts->ShowFixits) {
-    for (const FixItHint *Hint = Hints, *LastHint = Hints + NumHints;
-         Hint != LastHint; ++Hint) {
-      if (!Hint->CodeToInsert.empty()) {
+    for (ArrayRef<FixItHint>::iterator I = Hints.begin(), E = Hints.end();
+         I != E; ++I) {
+      if (!I->CodeToInsert.empty()) {
         // We have an insertion hint. Determine whether the inserted
         // code is on the same line as the caret.
         std::pair<FileID, unsigned> HintLocInfo
-          = SM.getDecomposedInstantiationLoc(Hint->RemoveRange.getBegin());
-        if (SM.getLineNumber(HintLocInfo.first, HintLocInfo.second) ==
-              SM.getLineNumber(FID, FileOffset)) {
+          = SM.getDecomposedExpansionLoc(I->RemoveRange.getBegin());
+        if (LineNo == SM.getLineNumber(HintLocInfo.first, HintLocInfo.second)) {
           // Insert the new code into the line just below the code
           // that the user wrote.
           unsigned HintColNo
             = SM.getColumnNumber(HintLocInfo.first, HintLocInfo.second);
           unsigned LastColumnModified
-            = HintColNo - 1 + Hint->CodeToInsert.size();
+            = HintColNo - 1 + I->CodeToInsert.size();
           if (LastColumnModified > FixItInsertionLine.size())
             FixItInsertionLine.resize(LastColumnModified, ' ');
-          std::copy(Hint->CodeToInsert.begin(), Hint->CodeToInsert.end(),
+          std::copy(I->CodeToInsert.begin(), I->CodeToInsert.end(),
                     FixItInsertionLine.begin() + HintColNo - 1);
         } else {
           FixItInsertionLine.clear();
@@ -538,119 +652,348 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
         }
       }
     }
+
+    if (FixItInsertionLine.empty())
+      return FixItInsertionLine;
+
     // Now that we have the entire fixit line, expand the tabs in it.
     // Since we don't want to insert spaces in the middle of a word,
     // find each word and the column it should line up with and insert
     // spaces until they match.
-    if (!FixItInsertionLine.empty()) {
-      unsigned FixItPos = 0;
-      unsigned LinePos = 0;
-      unsigned TabExpandedCol = 0;
-      unsigned LineLength = LineEnd - LineStart;
+    unsigned FixItPos = 0;
+    unsigned LinePos = 0;
+    unsigned TabExpandedCol = 0;
+    unsigned LineLength = LineEnd - LineStart;
 
-      while (FixItPos < FixItInsertionLine.size() && LinePos < LineLength) {
-        // Find the next word in the FixIt line.
-        while (FixItPos < FixItInsertionLine.size() &&
-               FixItInsertionLine[FixItPos] == ' ')
-          ++FixItPos;
-        unsigned CharDistance = FixItPos - TabExpandedCol;
+    while (FixItPos < FixItInsertionLine.size() && LinePos < LineLength) {
+      // Find the next word in the FixIt line.
+      while (FixItPos < FixItInsertionLine.size() &&
+             FixItInsertionLine[FixItPos] == ' ')
+        ++FixItPos;
+      unsigned CharDistance = FixItPos - TabExpandedCol;
 
-        // Walk forward in the source line, keeping track of
-        // the tab-expanded column.
-        for (unsigned I = 0; I < CharDistance; ++I, ++LinePos)
-          if (LinePos >= LineLength || LineStart[LinePos] != '\t')
-            ++TabExpandedCol;
-          else
-            TabExpandedCol =
-              (TabExpandedCol/DiagOpts->TabStop + 1) * DiagOpts->TabStop;
+      // Walk forward in the source line, keeping track of
+      // the tab-expanded column.
+      for (unsigned I = 0; I < CharDistance; ++I, ++LinePos)
+        if (LinePos >= LineLength || LineStart[LinePos] != '\t')
+          ++TabExpandedCol;
+        else
+          TabExpandedCol =
+            (TabExpandedCol/DiagOpts.TabStop + 1) * DiagOpts.TabStop;
 
-        // Adjust the fixit line to match this column.
-        FixItInsertionLine.insert(FixItPos, TabExpandedCol-FixItPos, ' ');
-        FixItPos = TabExpandedCol;
+      // Adjust the fixit line to match this column.
+      FixItInsertionLine.insert(FixItPos, TabExpandedCol-FixItPos, ' ');
+      FixItPos = TabExpandedCol;
 
-        // Walk to the end of the word.
-        while (FixItPos < FixItInsertionLine.size() &&
-               FixItInsertionLine[FixItPos] != ' ')
-          ++FixItPos;
-      }
+      // Walk to the end of the word.
+      while (FixItPos < FixItInsertionLine.size() &&
+             FixItInsertionLine[FixItPos] != ' ')
+        ++FixItPos;
+    }
+
+    return FixItInsertionLine;
+  }
+
+  void ExpandTabs(std::string &SourceLine, std::string &CaretLine) {
+    // Scan the source line, looking for tabs.  If we find any, manually expand
+    // them to spaces and update the CaretLine to match.
+    for (unsigned i = 0; i != SourceLine.size(); ++i) {
+      if (SourceLine[i] != '\t') continue;
+
+      // Replace this tab with at least one space.
+      SourceLine[i] = ' ';
+
+      // Compute the number of spaces we need to insert.
+      unsigned TabStop = DiagOpts.TabStop;
+      assert(0 < TabStop && TabStop <= DiagnosticOptions::MaxTabStop &&
+             "Invalid -ftabstop value");
+      unsigned NumSpaces = ((i+TabStop)/TabStop * TabStop) - (i+1);
+      assert(NumSpaces < TabStop && "Invalid computation of space amt");
+
+      // Insert spaces into the SourceLine.
+      SourceLine.insert(i+1, NumSpaces, ' ');
+
+      // Insert spaces or ~'s into CaretLine.
+      CaretLine.insert(i+1, NumSpaces, CaretLine[i] == '~' ? '~' : ' ');
     }
   }
 
-  // If the source line is too long for our terminal, select only the
-  // "interesting" source region within that line.
-  if (Columns && SourceLine.size() > Columns)
-    SelectInterestingSourceRegion(SourceLine, CaretLine, FixItInsertionLine,
-                                  CaretEndColNo, Columns);
-
-  // Finally, remove any blank spaces from the end of CaretLine.
-  while (CaretLine[CaretLine.size()-1] == ' ')
-    CaretLine.erase(CaretLine.end()-1);
-
-  // Emit what we have computed.
-  OS << SourceLine << '\n';
-
-  if (DiagOpts->ShowColors)
-    OS.changeColor(caretColor, true);
-  OS << CaretLine << '\n';
-  if (DiagOpts->ShowColors)
-    OS.resetColor();
-
-  if (!FixItInsertionLine.empty()) {
-    if (DiagOpts->ShowColors)
-      // Print fixit line in color
-      OS.changeColor(fixitColor, false);
-    if (DiagOpts->ShowSourceRanges)
-      OS << ' ';
-    OS << FixItInsertionLine << '\n';
-    if (DiagOpts->ShowColors)
-      OS.resetColor();
-  }
-
-  if (DiagOpts->ShowParseableFixits) {
+  void EmitParseableFixits(ArrayRef<FixItHint> Hints) {
+    if (!DiagOpts.ShowParseableFixits)
+      return;
 
     // We follow FixItRewriter's example in not (yet) handling
     // fix-its in macros.
-    bool BadApples = false;
-    for (const FixItHint *Hint = Hints; Hint != Hints + NumHints; ++Hint) {
-      if (Hint->RemoveRange.isInvalid() ||
-          Hint->RemoveRange.getBegin().isMacroID() ||
-          Hint->RemoveRange.getEnd().isMacroID()) {
-        BadApples = true;
-        break;
-      }
+    for (ArrayRef<FixItHint>::iterator I = Hints.begin(), E = Hints.end();
+         I != E; ++I) {
+      if (I->RemoveRange.isInvalid() ||
+          I->RemoveRange.getBegin().isMacroID() ||
+          I->RemoveRange.getEnd().isMacroID())
+        return;
     }
 
-    if (!BadApples) {
-      for (const FixItHint *Hint = Hints; Hint != Hints + NumHints; ++Hint) {
+    for (ArrayRef<FixItHint>::iterator I = Hints.begin(), E = Hints.end();
+         I != E; ++I) {
+      SourceLocation BLoc = I->RemoveRange.getBegin();
+      SourceLocation ELoc = I->RemoveRange.getEnd();
 
-        SourceLocation B = Hint->RemoveRange.getBegin();
-        SourceLocation E = Hint->RemoveRange.getEnd();
+      std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(BLoc);
+      std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(ELoc);
 
-        std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
-        std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
+      // Adjust for token ranges.
+      if (I->RemoveRange.isTokenRange())
+        EInfo.second += Lexer::MeasureTokenLength(ELoc, SM, LangOpts);
 
-        // Adjust for token ranges.
-        if (Hint->RemoveRange.isTokenRange())
-          EInfo.second += Lexer::MeasureTokenLength(E, SM, *LangOpts);
+      // We specifically do not do word-wrapping or tab-expansion here,
+      // because this is supposed to be easy to parse.
+      PresumedLoc PLoc = SM.getPresumedLoc(BLoc);
+      if (PLoc.isInvalid())
+        break;
 
-        // We specifically do not do word-wrapping or tab-expansion here,
-        // because this is supposed to be easy to parse.
-        PresumedLoc PLoc = SM.getPresumedLoc(B);
-        if (PLoc.isInvalid())
-          break;
-        
-        OS << "fix-it:\"";
-        OS.write_escaped(SM.getPresumedLoc(B).getFilename());
-        OS << "\":{" << SM.getLineNumber(BInfo.first, BInfo.second)
-          << ':' << SM.getColumnNumber(BInfo.first, BInfo.second)
-          << '-' << SM.getLineNumber(EInfo.first, EInfo.second)
-          << ':' << SM.getColumnNumber(EInfo.first, EInfo.second)
-          << "}:\"";
-        OS.write_escaped(Hint->CodeToInsert);
-        OS << "\"\n";
+      OS << "fix-it:\"";
+      OS.write_escaped(PLoc.getFilename());
+      OS << "\":{" << SM.getLineNumber(BInfo.first, BInfo.second)
+        << ':' << SM.getColumnNumber(BInfo.first, BInfo.second)
+        << '-' << SM.getLineNumber(EInfo.first, EInfo.second)
+        << ':' << SM.getColumnNumber(EInfo.first, EInfo.second)
+        << "}:\"";
+      OS.write_escaped(I->CodeToInsert);
+      OS << "\"\n";
+    }
+  }
+};
+
+} // end namespace
+
+/// Get the presumed location of a diagnostic message. This computes the
+/// presumed location for the top of any macro backtrace when present.
+static PresumedLoc getDiagnosticPresumedLoc(const SourceManager &SM,
+                                            SourceLocation Loc) {
+  // This is a condensed form of the algorithm used by EmitCaretDiagnostic to
+  // walk to the top of the macro call stack.
+  while (Loc.isMacroID()) {
+    Loc = skipToMacroArgExpansion(SM, Loc);
+    Loc = getImmediateMacroCallerLoc(SM, Loc);
+  }
+
+  return SM.getPresumedLoc(Loc);
+}
+
+/// \brief Print out the file/line/column information and include trace.
+///
+/// This method handlen the emission of the diagnostic location information.
+/// This includes extracting as much location information as is present for the
+/// diagnostic and printing it, as well as any include stack or source ranges
+/// necessary.
+void TextDiagnosticPrinter::EmitDiagnosticLoc(DiagnosticsEngine::Level Level,
+                                              const Diagnostic &Info,
+                                              const SourceManager &SM,
+                                              PresumedLoc PLoc) {
+  if (PLoc.isInvalid()) {
+    // At least print the file name if available:
+    FileID FID = SM.getFileID(Info.getLocation());
+    if (!FID.isInvalid()) {
+      const FileEntry* FE = SM.getFileEntryForID(FID);
+      if (FE && FE->getName()) {
+        OS << FE->getName();
+        if (FE->getDevice() == 0 && FE->getInode() == 0
+            && FE->getFileMode() == 0) {
+          // in PCH is a guess, but a good one:
+          OS << " (in PCH)";
+        }
+        OS << ": ";
+      }
+    }
+    return;
+  }
+  unsigned LineNo = PLoc.getLine();
+
+  if (!DiagOpts->ShowLocation)
+    return;
+
+  if (DiagOpts->ShowColors)
+    OS.changeColor(savedColor, true);
+
+  OS << PLoc.getFilename();
+  switch (DiagOpts->Format) {
+  case DiagnosticOptions::Clang: OS << ':'  << LineNo; break;
+  case DiagnosticOptions::Msvc:  OS << '('  << LineNo; break;
+  case DiagnosticOptions::Vi:    OS << " +" << LineNo; break;
+  }
+
+  if (DiagOpts->ShowColumn)
+    // Compute the column number.
+    if (unsigned ColNo = PLoc.getColumn()) {
+      if (DiagOpts->Format == DiagnosticOptions::Msvc) {
+        OS << ',';
+        ColNo--;
+      } else 
+        OS << ':';
+      OS << ColNo;
+    }
+  switch (DiagOpts->Format) {
+  case DiagnosticOptions::Clang: 
+  case DiagnosticOptions::Vi:    OS << ':';    break;
+  case DiagnosticOptions::Msvc:  OS << ") : "; break;
+  }
+
+  if (DiagOpts->ShowSourceRanges && Info.getNumRanges()) {
+    FileID CaretFileID =
+      SM.getFileID(SM.getExpansionLoc(Info.getLocation()));
+    bool PrintedRange = false;
+
+    for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i) {
+      // Ignore invalid ranges.
+      if (!Info.getRange(i).isValid()) continue;
+
+      SourceLocation B = Info.getRange(i).getBegin();
+      SourceLocation E = Info.getRange(i).getEnd();
+      B = SM.getExpansionLoc(B);
+      E = SM.getExpansionLoc(E);
+
+      // If the End location and the start location are the same and are a
+      // macro location, then the range was something that came from a
+      // macro expansion or _Pragma.  If this is an object-like macro, the
+      // best we can do is to highlight the range.  If this is a
+      // function-like macro, we'd also like to highlight the arguments.
+      if (B == E && Info.getRange(i).getEnd().isMacroID())
+        E = SM.getExpansionRange(Info.getRange(i).getEnd()).second;
+
+      std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
+      std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
+
+      // If the start or end of the range is in another file, just discard
+      // it.
+      if (BInfo.first != CaretFileID || EInfo.first != CaretFileID)
+        continue;
+
+      // Add in the length of the token, so that we cover multi-char
+      // tokens.
+      unsigned TokSize = 0;
+      if (Info.getRange(i).isTokenRange())
+        TokSize = Lexer::MeasureTokenLength(E, SM, *LangOpts);
+
+      OS << '{' << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
+        << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
+        << SM.getLineNumber(EInfo.first, EInfo.second) << ':'
+        << (SM.getColumnNumber(EInfo.first, EInfo.second)+TokSize)
+        << '}';
+      PrintedRange = true;
+    }
+
+    if (PrintedRange)
+      OS << ':';
+  }
+  OS << ' ';
+}
+
+/// \brief Print the diagonstic level to a raw_ostream.
+///
+/// Handles colorizing the level and formatting.
+static void printDiagnosticLevel(raw_ostream &OS,
+                                 DiagnosticsEngine::Level Level,
+                                 bool ShowColors) {
+  if (ShowColors) {
+    // Print diagnostic category in bold and color
+    switch (Level) {
+    case DiagnosticsEngine::Ignored:
+      llvm_unreachable("Invalid diagnostic type");
+    case DiagnosticsEngine::Note:    OS.changeColor(noteColor, true); break;
+    case DiagnosticsEngine::Warning: OS.changeColor(warningColor, true); break;
+    case DiagnosticsEngine::Error:   OS.changeColor(errorColor, true); break;
+    case DiagnosticsEngine::Fatal:   OS.changeColor(fatalColor, true); break;
+    }
+  }
+
+  switch (Level) {
+  case DiagnosticsEngine::Ignored: llvm_unreachable("Invalid diagnostic type");
+  case DiagnosticsEngine::Note:    OS << "note: "; break;
+  case DiagnosticsEngine::Warning: OS << "warning: "; break;
+  case DiagnosticsEngine::Error:   OS << "error: "; break;
+  case DiagnosticsEngine::Fatal:   OS << "fatal error: "; break;
+  }
+
+  if (ShowColors)
+    OS.resetColor();
+}
+
+/// \brief Print the diagnostic name to a raw_ostream.
+///
+/// This prints the diagnostic name to a raw_ostream if it has one. It formats
+/// the name according to the expected diagnostic message formatting:
+///   " [diagnostic_name_here]"
+static void printDiagnosticName(raw_ostream &OS, const Diagnostic &Info) {
+  if (!DiagnosticIDs::isBuiltinNote(Info.getID()))
+    OS << " [" << DiagnosticIDs::getName(Info.getID()) << "]";
+}
+
+/// \brief Print any diagnostic option information to a raw_ostream.
+///
+/// This implements all of the logic for adding diagnostic options to a message
+/// (via OS). Each relevant option is comma separated and all are enclosed in
+/// the standard bracketing: " [...]".
+static void printDiagnosticOptions(raw_ostream &OS,
+                                   DiagnosticsEngine::Level Level,
+                                   const Diagnostic &Info,
+                                   const DiagnosticOptions &DiagOpts) {
+  bool Started = false;
+  if (DiagOpts.ShowOptionNames) {
+    // Handle special cases for non-warnings early.
+    if (Info.getID() == diag::fatal_too_many_errors) {
+      OS << " [-ferror-limit=]";
+      return;
+    }
+
+    // The code below is somewhat fragile because we are essentially trying to
+    // report to the user what happened by inferring what the diagnostic engine
+    // did. Eventually it might make more sense to have the diagnostic engine
+    // include some "why" information in the diagnostic.
+
+    // If this is a warning which has been mapped to an error by the user (as
+    // inferred by checking whether the default mapping is to an error) then
+    // flag it as such. Note that diagnostics could also have been mapped by a
+    // pragma, but we don't currently have a way to distinguish this.
+    if (Level == DiagnosticsEngine::Error &&
+        DiagnosticIDs::isBuiltinWarningOrExtension(Info.getID()) &&
+        !DiagnosticIDs::isDefaultMappingAsError(Info.getID())) {
+      OS << " [-Werror";
+      Started = true;
+    }
+
+    // If the diagnostic is an extension diagnostic and not enabled by default
+    // then it must have been turned on with -pedantic.
+    bool EnabledByDefault;
+    if (DiagnosticIDs::isBuiltinExtensionDiag(Info.getID(),
+                                              EnabledByDefault) &&
+        !EnabledByDefault) {
+      OS << (Started ? "," : " [") << "-pedantic";
+      Started = true;
+    }
+
+    StringRef Opt = DiagnosticIDs::getWarningOptionForDiag(Info.getID());
+    if (!Opt.empty()) {
+      OS << (Started ? "," : " [") << "-W" << Opt;
+      Started = true;
+    }
+  }
+
+  // If the user wants to see category information, include it too.
+  if (DiagOpts.ShowCategories) {
+    unsigned DiagCategory =
+      DiagnosticIDs::getCategoryNumberForDiag(Info.getID());
+    if (DiagCategory) {
+      OS << (Started ? "," : " [");
+      Started = true;
+      if (DiagOpts.ShowCategories == 1)
+        OS << DiagCategory;
+      else {
+        assert(DiagOpts.ShowCategories == 2 && "Invalid ShowCategories value");
+        OS << DiagnosticIDs::getCategoryNameFromID(DiagCategory);
       }
     }
   }
+  if (Started)
+    OS << ']';
 }
 
 /// \brief Skip over whitespace in the string, starting at the given
@@ -659,9 +1002,7 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
 /// \returns The index of the first non-whitespace character that is
 /// greater than or equal to Idx or, if no such character exists,
 /// returns the end of the string.
-static unsigned skipWhitespace(unsigned Idx,
-                               const llvm::SmallVectorImpl<char> &Str,
-                               unsigned Length) {
+static unsigned skipWhitespace(unsigned Idx, StringRef Str, unsigned Length) {
   while (Idx < Length && isspace(Str[Idx]))
     ++Idx;
   return Idx;
@@ -692,8 +1033,7 @@ static inline char findMatchingPunctuation(char c) {
 ///
 /// \returns the index pointing one character past the end of the
 /// word.
-static unsigned findEndOfWord(unsigned Start,
-                              const llvm::SmallVectorImpl<char> &Str,
+static unsigned findEndOfWord(unsigned Start, StringRef Str,
                               unsigned Length, unsigned Column,
                               unsigned Columns) {
   assert(Start < Str.size() && "Invalid start position!");
@@ -748,38 +1088,22 @@ static unsigned findEndOfWord(unsigned Start,
 /// \brief Print the given string to a stream, word-wrapping it to
 /// some number of columns in the process.
 ///
-/// \brief OS the stream to which the word-wrapping string will be
+/// \param OS the stream to which the word-wrapping string will be
 /// emitted.
-///
-/// \brief Str the string to word-wrap and output.
-///
-/// \brief Columns the number of columns to word-wrap to.
-///
-/// \brief Column the column number at which the first character of \p
+/// \param Str the string to word-wrap and output.
+/// \param Columns the number of columns to word-wrap to.
+/// \param Column the column number at which the first character of \p
 /// Str will be printed. This will be non-zero when part of the first
 /// line has already been printed.
-///
-/// \brief Indentation the number of spaces to indent any lines beyond
+/// \param Indentation the number of spaces to indent any lines beyond
 /// the first line.
-///
 /// \returns true if word-wrapping was required, or false if the
 /// string fit on the first line.
-static bool PrintWordWrapped(llvm::raw_ostream &OS,
-                             const llvm::SmallVectorImpl<char> &Str,
+static bool printWordWrapped(raw_ostream &OS, StringRef Str,
                              unsigned Columns,
                              unsigned Column = 0,
                              unsigned Indentation = WordWrapIndentation) {
-  unsigned Length = Str.size();
-
-  // If there is a newline in this message somewhere, find that
-  // newline and split the message into the part before the newline
-  // (which will be word-wrapped) and the part from the newline one
-  // (which will be emitted unchanged).
-  for (unsigned I = 0; I != Length; ++I)
-    if (Str[I] == '\n') {
-      Length = I;
-      break;
-    }
+  const unsigned Length = std::min(Str.find('\n'), Str.size());
 
   // The string used to indent each line.
   llvm::SmallString<16> IndentStr;
@@ -803,7 +1127,7 @@ static bool PrintWordWrapped(llvm::raw_ostream &OS,
         OS << ' ';
         Column += 1;
       }
-      OS.write(&Str[WordStart], WordLength);
+      OS << Str.substr(WordStart, WordLength);
       Column += WordLength;
       continue;
     }
@@ -812,38 +1136,57 @@ static bool PrintWordWrapped(llvm::raw_ostream &OS,
     // line.
     OS << '\n';
     OS.write(&IndentStr[0], Indentation);
-    OS.write(&Str[WordStart], WordLength);
+    OS << Str.substr(WordStart, WordLength);
     Column = Indentation + WordLength;
     Wrapped = true;
   }
 
-  if (Length == Str.size())
-    return Wrapped; // We're done.
+  // Append any remaning text from the message with its existing formatting.
+  OS << Str.substr(Length);
 
-  // There is a newline in the message, followed by something that
-  // will not be word-wrapped. Print that.
-  OS.write(&Str[Length], Str.size() - Length);
-  return true;
+  return Wrapped;
 }
 
-/// Get the presumed location of a diagnostic message. This computes the
-/// presumed location for the top of any macro backtrace when present.
-static PresumedLoc getDiagnosticPresumedLoc(const SourceManager &SM,
-                                            SourceLocation Loc) {
-  // This is a condensed form of the algorithm used by EmitCaretDiagnostic to
-  // walk to the top of the macro call stack.
-  while (Loc.isMacroID()) {
-    Loc = skipToMacroArgExpansion(SM, Loc);
-    Loc = getImmediateMacroCallerLoc(SM, Loc);
+static void printDiagnosticMessage(raw_ostream &OS,
+                                   DiagnosticsEngine::Level Level,
+                                   StringRef Message,
+                                   unsigned CurrentColumn, unsigned Columns,
+                                   bool ShowColors) {
+  if (ShowColors) {
+    // Print warnings, errors and fatal errors in bold, no color
+    switch (Level) {
+    case DiagnosticsEngine::Warning: OS.changeColor(savedColor, true); break;
+    case DiagnosticsEngine::Error:   OS.changeColor(savedColor, true); break;
+    case DiagnosticsEngine::Fatal:   OS.changeColor(savedColor, true); break;
+    default: break; //don't bold notes
+    }
   }
 
-  return SM.getPresumedLoc(Loc);
+  if (Columns)
+    printWordWrapped(OS, Message, Columns, CurrentColumn);
+  else
+    OS << Message;
+
+  if (ShowColors)
+    OS.resetColor();
+  OS << '\n';
 }
 
-void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
-                                             const DiagnosticInfo &Info) {
+void TextDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
+                                             const Diagnostic &Info) {
   // Default implementation (Warnings/errors count).
-  DiagnosticClient::HandleDiagnostic(Level, Info);
+  DiagnosticConsumer::HandleDiagnostic(Level, Info);
+
+  // Render the diagnostic message into a temporary buffer eagerly. We'll use
+  // this later as we print out the diagnostic to the terminal.
+  llvm::SmallString<100> OutStr;
+  Info.FormatDiagnostic(OutStr);
+
+  llvm::raw_svector_ostream DiagMessageStream(OutStr);
+  if (DiagOpts->ShowNames)
+    printDiagnosticName(DiagMessageStream, Info);
+  printDiagnosticOptions(DiagMessageStream, Level, Info, *DiagOpts);
+
 
   // Keeps track of the the starting position of the location
   // information (e.g., "foo.c:10:4:") that precedes the error
@@ -854,232 +1197,44 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
   if (!Prefix.empty())
     OS << Prefix << ": ";
 
-  // If the location is specified, print out a file/line/col and include trace
-  // if enabled.
-  if (Info.getLocation().isValid()) {
-    const SourceManager &SM = Info.getSourceManager();
-    PresumedLoc PLoc = getDiagnosticPresumedLoc(SM, Info.getLocation());
-    if (PLoc.isInvalid()) {
-      // At least print the file name if available:
-      FileID FID = SM.getFileID(Info.getLocation());
-      if (!FID.isInvalid()) {
-        const FileEntry* FE = SM.getFileEntryForID(FID);
-        if (FE && FE->getName()) {
-          OS << FE->getName();
-          if (FE->getDevice() == 0 && FE->getInode() == 0
-              && FE->getFileMode() == 0) {
-            // in PCH is a guess, but a good one:
-            OS << " (in PCH)";
-          }
-          OS << ": ";
-        }
-      }
-    } else {
-      unsigned LineNo = PLoc.getLine();
-
-      // First, if this diagnostic is not in the main file, print out the
-      // "included from" lines.
-      if (LastWarningLoc != PLoc.getIncludeLoc()) {
-        LastWarningLoc = PLoc.getIncludeLoc();
-        PrintIncludeStack(Level, LastWarningLoc, SM);
-        StartOfLocationInfo = OS.tell();
-      }
-
-      // Compute the column number.
-      if (DiagOpts->ShowLocation) {
-        if (DiagOpts->ShowColors)
-          OS.changeColor(savedColor, true);
-
-        OS << PLoc.getFilename();
-        switch (DiagOpts->Format) {
-        case DiagnosticOptions::Clang: OS << ':'  << LineNo; break;
-        case DiagnosticOptions::Msvc:  OS << '('  << LineNo; break;
-        case DiagnosticOptions::Vi:    OS << " +" << LineNo; break;
-        }
-        if (DiagOpts->ShowColumn)
-          if (unsigned ColNo = PLoc.getColumn()) {
-            if (DiagOpts->Format == DiagnosticOptions::Msvc) {
-              OS << ',';
-              ColNo--;
-            } else 
-              OS << ':';
-            OS << ColNo;
-          }
-        switch (DiagOpts->Format) {
-        case DiagnosticOptions::Clang: 
-        case DiagnosticOptions::Vi:    OS << ':';    break;
-        case DiagnosticOptions::Msvc:  OS << ") : "; break;
-        }
-
-                
-        if (DiagOpts->ShowSourceRanges && Info.getNumRanges()) {
-          FileID CaretFileID =
-            SM.getFileID(SM.getInstantiationLoc(Info.getLocation()));
-          bool PrintedRange = false;
-
-          for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i) {
-            // Ignore invalid ranges.
-            if (!Info.getRange(i).isValid()) continue;
-
-            SourceLocation B = Info.getRange(i).getBegin();
-            SourceLocation E = Info.getRange(i).getEnd();
-            B = SM.getInstantiationLoc(B);
-            E = SM.getInstantiationLoc(E);
-
-            // If the End location and the start location are the same and are a
-            // macro location, then the range was something that came from a
-            // macro expansion or _Pragma.  If this is an object-like macro, the
-            // best we can do is to highlight the range.  If this is a
-            // function-like macro, we'd also like to highlight the arguments.
-            if (B == E && Info.getRange(i).getEnd().isMacroID())
-              E = SM.getInstantiationRange(Info.getRange(i).getEnd()).second;
-
-            std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
-            std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
-
-            // If the start or end of the range is in another file, just discard
-            // it.
-            if (BInfo.first != CaretFileID || EInfo.first != CaretFileID)
-              continue;
-
-            // Add in the length of the token, so that we cover multi-char
-            // tokens.
-            unsigned TokSize = 0;
-            if (Info.getRange(i).isTokenRange())
-              TokSize = Lexer::MeasureTokenLength(E, SM, *LangOpts);
-
-            OS << '{' << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
-               << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
-               << SM.getLineNumber(EInfo.first, EInfo.second) << ':'
-               << (SM.getColumnNumber(EInfo.first, EInfo.second)+TokSize)
-               << '}';
-            PrintedRange = true;
-          }
-
-          if (PrintedRange)
-            OS << ':';
-        }
-      }
-      OS << ' ';
-      if (DiagOpts->ShowColors)
-        OS.resetColor();
-    }
+  // Use a dedicated, simpler path for diagnostics without a valid location.
+  // This is important as if the location is missing, we may be emitting
+  // diagnostics in a context that lacks language options, a source manager, or
+  // other infrastructure necessary when emitting more rich diagnostics.
+  if (!Info.getLocation().isValid()) {
+    printDiagnosticLevel(OS, Level, DiagOpts->ShowColors);
+    printDiagnosticMessage(OS, Level, DiagMessageStream.str(),
+                           OS.tell() - StartOfLocationInfo,
+                           DiagOpts->MessageLength, DiagOpts->ShowColors);
+    OS.flush();
+    return;
   }
 
-  if (DiagOpts->ShowColors) {
-    // Print diagnostic category in bold and color
-    switch (Level) {
-    case Diagnostic::Ignored: assert(0 && "Invalid diagnostic type");
-    case Diagnostic::Note:    OS.changeColor(noteColor, true); break;
-    case Diagnostic::Warning: OS.changeColor(warningColor, true); break;
-    case Diagnostic::Error:   OS.changeColor(errorColor, true); break;
-    case Diagnostic::Fatal:   OS.changeColor(fatalColor, true); break;
-    }
-  }
+  // Assert that the rest of our infrastructure is setup properly.
+  assert(LangOpts && "Unexpected diagnostic outside source file processing");
+  assert(DiagOpts && "Unexpected diagnostic without options set");
+  assert(Info.hasSourceManager() &&
+         "Unexpected diagnostic with no source manager");
+  const SourceManager &SM = Info.getSourceManager();
+  TextDiagnostic TextDiag(*this, OS, SM, *LangOpts, *DiagOpts);
 
-  switch (Level) {
-  case Diagnostic::Ignored: assert(0 && "Invalid diagnostic type");
-  case Diagnostic::Note:    OS << "note: "; break;
-  case Diagnostic::Warning: OS << "warning: "; break;
-  case Diagnostic::Error:   OS << "error: "; break;
-  case Diagnostic::Fatal:   OS << "fatal error: "; break;
-  }
+  PresumedLoc PLoc = getDiagnosticPresumedLoc(SM, Info.getLocation());
+
+  // First, if this diagnostic is not in the main file, print out the
+  // "included from" lines.
+  PrintIncludeStack(Level, PLoc.getIncludeLoc(), SM);
+  StartOfLocationInfo = OS.tell();
+
+  // Next emit the location of this particular diagnostic.
+  EmitDiagnosticLoc(Level, Info, SM, PLoc);
 
   if (DiagOpts->ShowColors)
     OS.resetColor();
 
-  llvm::SmallString<100> OutStr;
-  Info.FormatDiagnostic(OutStr);
-
-  if (DiagOpts->ShowNames &&
-      !DiagnosticIDs::isBuiltinNote(Info.getID())) {
-    OutStr += " [";
-    OutStr += DiagnosticIDs::getName(Info.getID());
-    OutStr += "]";
-  }
-  
-  std::string OptionName;
-  if (DiagOpts->ShowOptionNames) {
-    // Was this a warning mapped to an error using -Werror or pragma?
-    if (Level == Diagnostic::Error &&
-        DiagnosticIDs::isBuiltinWarningOrExtension(Info.getID())) {
-      diag::Mapping mapping = diag::MAP_IGNORE;
-      Info.getDiags()->getDiagnosticLevel(Info.getID(), Info.getLocation(), 
-                                          &mapping);
-      if (mapping == diag::MAP_WARNING)
-        OptionName += "-Werror";
-    }
-
-    llvm::StringRef Opt = DiagnosticIDs::getWarningOptionForDiag(Info.getID());
-    if (!Opt.empty()) {
-      if (!OptionName.empty())
-        OptionName += ',';
-      OptionName += "-W";
-      OptionName += Opt;
-    } else if (Info.getID() == diag::fatal_too_many_errors) {
-      OptionName = "-ferror-limit=";
-    } else {
-      // If the diagnostic is an extension diagnostic and not enabled by default
-      // then it must have been turned on with -pedantic.
-      bool EnabledByDefault;
-      if (DiagnosticIDs::isBuiltinExtensionDiag(Info.getID(),
-                                                EnabledByDefault) &&
-          !EnabledByDefault)
-        OptionName = "-pedantic";
-    }
-  }
-  
-  // If the user wants to see category information, include it too.
-  unsigned DiagCategory = 0;
-  if (DiagOpts->ShowCategories)
-    DiagCategory = DiagnosticIDs::getCategoryNumberForDiag(Info.getID());
-
-  // If there is any categorization information, include it.
-  if (!OptionName.empty() || DiagCategory != 0) {
-    bool NeedsComma = false;
-    OutStr += " [";
-    
-    if (!OptionName.empty()) {
-      OutStr += OptionName;
-      NeedsComma = true;
-    }
-    
-    if (DiagCategory) {
-      if (NeedsComma) OutStr += ',';
-      if (DiagOpts->ShowCategories == 1)
-        OutStr += llvm::utostr(DiagCategory);
-      else {
-        assert(DiagOpts->ShowCategories == 2 && "Invalid ShowCategories value");
-        OutStr += DiagnosticIDs::getCategoryNameFromID(DiagCategory);
-      }
-    }
-    
-    OutStr += "]";
-  }
-
-  
-  if (DiagOpts->ShowColors) {
-    // Print warnings, errors and fatal errors in bold, no color
-    switch (Level) {
-    case Diagnostic::Warning: OS.changeColor(savedColor, true); break;
-    case Diagnostic::Error:   OS.changeColor(savedColor, true); break;
-    case Diagnostic::Fatal:   OS.changeColor(savedColor, true); break;
-    default: break; //don't bold notes
-    }
-  }
-
-  if (DiagOpts->MessageLength) {
-    // We will be word-wrapping the error message, so compute the
-    // column number where we currently are (after printing the
-    // location information).
-    unsigned Column = OS.tell() - StartOfLocationInfo;
-    PrintWordWrapped(OS, OutStr, DiagOpts->MessageLength, Column);
-  } else {
-    OS.write(OutStr.begin(), OutStr.size());
-  }
-  OS << '\n';
-  if (DiagOpts->ShowColors)
-    OS.resetColor();
+  printDiagnosticLevel(OS, Level, DiagOpts->ShowColors);
+  printDiagnosticMessage(OS, Level, DiagMessageStream.str(),
+                         OS.tell() - StartOfLocationInfo,
+                         DiagOpts->MessageLength, DiagOpts->ShowColors);
 
   // If caret diagnostics are enabled and we have location, we want to
   // emit the caret.  However, we only do this if the location moved
@@ -1087,56 +1242,37 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
   // was part of a different warning or error diagnostic, or if the
   // diagnostic has ranges.  We don't want to emit the same caret
   // multiple times if one loc has multiple diagnostics.
-  if (DiagOpts->ShowCarets && Info.getLocation().isValid() &&
+  if (DiagOpts->ShowCarets &&
       ((LastLoc != Info.getLocation()) || Info.getNumRanges() ||
-       (LastCaretDiagnosticWasNote && Level != Diagnostic::Note) ||
+       (LastCaretDiagnosticWasNote && Level != DiagnosticsEngine::Note) ||
        Info.getNumFixItHints())) {
     // Cache the LastLoc, it allows us to omit duplicate source/caret spewage.
     LastLoc = FullSourceLoc(Info.getLocation(), Info.getSourceManager());
-    LastCaretDiagnosticWasNote = (Level == Diagnostic::Note);
+    LastCaretDiagnosticWasNote = (Level == DiagnosticsEngine::Note);
 
     // Get the ranges into a local array we can hack on.
-    CharSourceRange Ranges[20];
-    unsigned NumRanges = Info.getNumRanges();
-    assert(NumRanges < 20 && "Out of space");
-    for (unsigned i = 0; i != NumRanges; ++i)
-      Ranges[i] = Info.getRange(i);
+    SmallVector<CharSourceRange, 20> Ranges;
+    Ranges.reserve(Info.getNumRanges());
+    for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i)
+      Ranges.push_back(Info.getRange(i));
 
-    unsigned NumHints = Info.getNumFixItHints();
-    for (unsigned i = 0; i != NumHints; ++i) {
+    for (unsigned i = 0, e = Info.getNumFixItHints(); i != e; ++i) {
       const FixItHint &Hint = Info.getFixItHint(i);
-      if (Hint.RemoveRange.isValid()) {
-        assert(NumRanges < 20 && "Out of space");
-        Ranges[NumRanges++] = Hint.RemoveRange;
-      }
+      if (Hint.RemoveRange.isValid())
+        Ranges.push_back(Hint.RemoveRange);
     }
 
-    const SourceManager &SM = LastLoc.getManager();
-    unsigned MacroInstSkipStart = 0, MacroInstSkipEnd = 0;
-    if (DiagOpts && DiagOpts->MacroBacktraceLimit && !LastLoc.isFileID()) {
-      // Compute the length of the macro-expansion backtrace, so that we
-      // can establish which steps in the macro backtrace we'll skip.
-      SourceLocation Loc = LastLoc;
-      unsigned Depth = 0;
-      do {
-        ++Depth;
-        Loc = skipToMacroArgExpansion(SM, Loc);
-        Loc = getImmediateMacroCallerLoc(SM, Loc);
-      } while (!Loc.isFileID());
-      
-      if (Depth > DiagOpts->MacroBacktraceLimit) {
-        MacroInstSkipStart = DiagOpts->MacroBacktraceLimit / 2 + 
-                             DiagOpts->MacroBacktraceLimit % 2;
-        MacroInstSkipEnd = Depth - DiagOpts->MacroBacktraceLimit / 2;
-      }
-    }        
-    
-    EmitCaretDiagnostic(LastLoc, Ranges, NumRanges, LastLoc.getManager(),
-                        Info.getFixItHints(),
-                        Info.getNumFixItHints(),
-                        DiagOpts->MessageLength, 
-                        0, MacroInstSkipStart, MacroInstSkipEnd);
+    unsigned MacroDepth = 0;
+    TextDiag.EmitCaret(LastLoc, Ranges,
+                       llvm::makeArrayRef(Info.getFixItHints(),
+                                          Info.getNumFixItHints()),
+                       MacroDepth);
   }
 
   OS.flush();
 }
+
+DiagnosticConsumer *
+TextDiagnosticPrinter::clone(DiagnosticsEngine &Diags) const {
+  return new TextDiagnosticPrinter(OS, *DiagOpts, /*OwnsOutputStream=*/false);
+}
diff --git a/contrib/llvm/tools/clang/lib/Frontend/VerifyDiagnosticsClient.cpp b/contrib/llvm/tools/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
similarity index 81%
rename from contrib/llvm/tools/clang/lib/Frontend/VerifyDiagnosticsClient.cpp
rename to contrib/llvm/tools/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index fff417e20dcb..cf35c8edc310 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/VerifyDiagnosticsClient.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -1,4 +1,4 @@
-//===--- VerifyDiagnosticsClient.cpp - Verifying Diagnostic Client --------===//
+//===---- VerifyDiagnosticConsumer.cpp - Verifying Diagnostic Client ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/VerifyDiagnosticsClient.h"
+#include "clang/Frontend/VerifyDiagnosticConsumer.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/TextDiagnosticBuffer.h"
 #include "clang/Lex/Preprocessor.h"
@@ -20,19 +20,24 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
-VerifyDiagnosticsClient::VerifyDiagnosticsClient(Diagnostic &_Diags,
-                                                 DiagnosticClient *_Primary)
-  : Diags(_Diags), PrimaryClient(_Primary),
-    Buffer(new TextDiagnosticBuffer()), CurrentPreprocessor(0) {
+VerifyDiagnosticConsumer::VerifyDiagnosticConsumer(DiagnosticsEngine &_Diags)
+  : Diags(_Diags), PrimaryClient(Diags.getClient()),
+    OwnsPrimaryClient(Diags.ownsClient()),
+    Buffer(new TextDiagnosticBuffer()), CurrentPreprocessor(0) 
+{
+  Diags.takeClient();
 }
 
-VerifyDiagnosticsClient::~VerifyDiagnosticsClient() {
-  CheckDiagnostics();
+VerifyDiagnosticConsumer::~VerifyDiagnosticConsumer() {
+  CheckDiagnostics();  
+  Diags.takeClient();
+  if (OwnsPrimaryClient)
+    delete PrimaryClient;
 }
 
-// DiagnosticClient interface.
+// DiagnosticConsumer interface.
 
-void VerifyDiagnosticsClient::BeginSourceFile(const LangOptions &LangOpts,
+void VerifyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
                                              const Preprocessor *PP) {
   // FIXME: Const hack, we screw up the preprocessor but in practice its ok
   // because it doesn't get reused. It would be better if we could make a copy
@@ -42,7 +47,7 @@ void VerifyDiagnosticsClient::BeginSourceFile(const LangOptions &LangOpts,
   PrimaryClient->BeginSourceFile(LangOpts, PP);
 }
 
-void VerifyDiagnosticsClient::EndSourceFile() {
+void VerifyDiagnosticConsumer::EndSourceFile() {
   CheckDiagnostics();
 
   PrimaryClient->EndSourceFile();
@@ -50,8 +55,12 @@ void VerifyDiagnosticsClient::EndSourceFile() {
   CurrentPreprocessor = 0;
 }
 
-void VerifyDiagnosticsClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                              const DiagnosticInfo &Info) {
+void VerifyDiagnosticConsumer::HandleDiagnostic(
+      DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
+  if (FirstErrorFID.isInvalid() && Info.hasSourceManager()) {
+    const SourceManager &SM = Info.getSourceManager();
+    FirstErrorFID = SM.getFileID(Info.getLocation());
+  }
   // Send the diagnostic to the buffer, we will check it once we reach the end
   // of the source file (or are destructed).
   Buffer->HandleDiagnostic(DiagLevel, Info);
@@ -163,7 +172,7 @@ class ParseHelper
     : Begin(Begin), End(End), C(Begin), P(Begin), PEnd(NULL) { }
 
   // Return true if string literal is next.
-  bool Next(llvm::StringRef S) {
+  bool Next(StringRef S) {
     P = C;
     PEnd = C + S.size();
     if (PEnd > End)
@@ -189,7 +198,7 @@ class ParseHelper
 
   // Return true if string literal is found.
   // When true, P marks begin-position of S in content.
-  bool Search(llvm::StringRef S) {
+  bool Search(StringRef S) {
     P = std::search(C, End, S.begin(), S.end());
     PEnd = P + S.size();
     return P != End;
@@ -278,7 +287,7 @@ static void ParseDirective(const char *CommentStart, unsigned CommentLen,
 
     // next token: {{
     if (!PH.Next("{{")) {
-      PP.Diag(Pos.getFileLocWithOffset(PH.C-PH.Begin),
+      PP.Diag(Pos.getLocWithOffset(PH.C-PH.Begin),
               diag::err_verify_missing_start) << KindStr;
       continue;
     }
@@ -287,7 +296,7 @@ static void ParseDirective(const char *CommentStart, unsigned CommentLen,
 
     // search for token: }}
     if (!PH.Search("}}")) {
-      PP.Diag(Pos.getFileLocWithOffset(PH.C-PH.Begin),
+      PP.Diag(Pos.getLocWithOffset(PH.C-PH.Begin),
               diag::err_verify_missing_end) << KindStr;
       continue;
     }
@@ -296,11 +305,11 @@ static void ParseDirective(const char *CommentStart, unsigned CommentLen,
 
     // build directive text; convert \n to newlines
     std::string Text;
-    llvm::StringRef NewlineStr = "\\n";
-    llvm::StringRef Content(ContentBegin, ContentEnd-ContentBegin);
+    StringRef NewlineStr = "\\n";
+    StringRef Content(ContentBegin, ContentEnd-ContentBegin);
     size_t CPos = 0;
     size_t FPos;
-    while ((FPos = Content.find(NewlineStr, CPos)) != llvm::StringRef::npos) {
+    while ((FPos = Content.find(NewlineStr, CPos)) != StringRef::npos) {
       Text += Content.substr(CPos, FPos-CPos);
       Text += '\n';
       CPos = FPos + NewlineStr.size();
@@ -314,7 +323,7 @@ static void ParseDirective(const char *CommentStart, unsigned CommentLen,
     if (D->isValid(Error))
       DL->push_back(D);
     else {
-      PP.Diag(Pos.getFileLocWithOffset(ContentBegin-PH.Begin),
+      PP.Diag(Pos.getLocWithOffset(ContentBegin-PH.Begin),
               diag::err_verify_invalid_content)
         << KindStr << Error;
     }
@@ -323,14 +332,12 @@ static void ParseDirective(const char *CommentStart, unsigned CommentLen,
 
 /// FindExpectedDiags - Lex the main source file to find all of the
 //   expected errors and warnings.
-static void FindExpectedDiags(Preprocessor &PP, ExpectedData &ED) {
-  // Create a raw lexer to pull all the comments out of the main file.  We don't
-  // want to look in #include'd headers for expected-error strings.
-  SourceManager &SM = PP.getSourceManager();
-  FileID FID = SM.getMainFileID();
-  if (SM.getMainFileID().isInvalid())
+static void FindExpectedDiags(Preprocessor &PP, ExpectedData &ED, FileID FID) {
+  // Create a raw lexer to pull all the comments out of FID.
+  if (FID.isInvalid())
     return;
 
+  SourceManager& SM = PP.getSourceManager();
   // Create a lexer to lex all the tokens of the main file in raw mode.
   const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
   Lexer RawLex(FID, FromFile, SM, PP.getLangOptions());
@@ -357,7 +364,7 @@ static void FindExpectedDiags(Preprocessor &PP, ExpectedData &ED) {
 /// happened. Print the map out in a nice format and return "true". If the map
 /// is empty and we're not going to print things, then return "false".
 ///
-static unsigned PrintProblem(Diagnostic &Diags, SourceManager *SourceMgr,
+static unsigned PrintProblem(DiagnosticsEngine &Diags, SourceManager *SourceMgr,
                              const_diag_iterator diag_begin,
                              const_diag_iterator diag_end,
                              const char *Kind, bool Expected) {
@@ -378,7 +385,7 @@ static unsigned PrintProblem(Diagnostic &Diags, SourceManager *SourceMgr,
   return std::distance(diag_begin, diag_end);
 }
 
-static unsigned PrintProblem(Diagnostic &Diags, SourceManager *SourceMgr,
+static unsigned PrintProblem(DiagnosticsEngine &Diags, SourceManager *SourceMgr,
                              DirectiveList &DL, const char *Kind,
                              bool Expected) {
   if (DL.empty())
@@ -403,7 +410,7 @@ static unsigned PrintProblem(Diagnostic &Diags, SourceManager *SourceMgr,
 /// CheckLists - Compare expected to seen diagnostic lists and return the
 /// the difference between them.
 ///
-static unsigned CheckLists(Diagnostic &Diags, SourceManager &SourceMgr,
+static unsigned CheckLists(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
                            const char *Label,
                            DirectiveList &Left,
                            const_diag_iterator d2_begin,
@@ -446,7 +453,7 @@ static unsigned CheckLists(Diagnostic &Diags, SourceManager &SourceMgr,
 /// were actually reported. It emits any discrepencies. Return "true" if there
 /// were problems. Return "false" otherwise.
 ///
-static unsigned CheckResults(Diagnostic &Diags, SourceManager &SourceMgr,
+static unsigned CheckResults(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
                              const TextDiagnosticBuffer &Buffer,
                              ExpectedData &ED) {
   // We want to capture the delta between what was expected and what was
@@ -471,21 +478,34 @@ static unsigned CheckResults(Diagnostic &Diags, SourceManager &SourceMgr,
   return NumProblems;
 }
 
-void VerifyDiagnosticsClient::CheckDiagnostics() {
+void VerifyDiagnosticConsumer::CheckDiagnostics() {
   ExpectedData ED;
 
   // Ensure any diagnostics go to the primary client.
-  DiagnosticClient *CurClient = Diags.takeClient();
-  Diags.setClient(PrimaryClient.get());
+  bool OwnsCurClient = Diags.ownsClient();
+  DiagnosticConsumer *CurClient = Diags.takeClient();
+  Diags.setClient(PrimaryClient, false);
 
   // If we have a preprocessor, scan the source for expected diagnostic
   // markers. If not then any diagnostics are unexpected.
   if (CurrentPreprocessor) {
-    FindExpectedDiags(*CurrentPreprocessor, ED);
+    SourceManager &SM = CurrentPreprocessor->getSourceManager();
+    // Extract expected-error strings from main file.
+    FindExpectedDiags(*CurrentPreprocessor, ED, SM.getMainFileID());
+    // Only check for expectations in other diagnostic locations
+    // if they are not the main file (via ID or FileEntry) - the main
+    // file has already been looked at, and its expectations must not
+    // be added twice.
+    if (!FirstErrorFID.isInvalid() && FirstErrorFID != SM.getMainFileID()
+        && (!SM.getFileEntryForID(FirstErrorFID)
+            || (SM.getFileEntryForID(FirstErrorFID) !=
+                SM.getFileEntryForID(SM.getMainFileID())))) {
+      FindExpectedDiags(*CurrentPreprocessor, ED, FirstErrorFID);
+      FirstErrorFID = FileID();
+    }
 
     // Check that the expected diagnostics occurred.
-    NumErrors += CheckResults(Diags, CurrentPreprocessor->getSourceManager(),
-                              *Buffer, ED);
+    NumErrors += CheckResults(Diags, SM, *Buffer, ED);
   } else {
     NumErrors += (PrintProblem(Diags, 0,
                                Buffer->err_begin(), Buffer->err_end(),
@@ -499,12 +519,20 @@ void VerifyDiagnosticsClient::CheckDiagnostics() {
   }
 
   Diags.takeClient();
-  Diags.setClient(CurClient);
+  Diags.setClient(CurClient, OwnsCurClient);
 
   // Reset the buffer, we have processed all the diagnostics in it.
   Buffer.reset(new TextDiagnosticBuffer());
 }
 
+DiagnosticConsumer *
+VerifyDiagnosticConsumer::clone(DiagnosticsEngine &Diags) const {
+  if (!Diags.getClient())
+    Diags.setClient(PrimaryClient->clone(Diags));
+  
+  return new VerifyDiagnosticConsumer(Diags);
+}
+
 Directive* Directive::Create(bool RegexKind, const SourceLocation &Location,
                              const std::string &Text, unsigned Count) {
   if (RegexKind)
diff --git a/contrib/llvm/tools/clang/lib/Frontend/Warnings.cpp b/contrib/llvm/tools/clang/lib/Frontend/Warnings.cpp
index f12b484c05e6..8fbcd4b44d79 100644
--- a/contrib/llvm/tools/clang/lib/Frontend/Warnings.cpp
+++ b/contrib/llvm/tools/clang/lib/Frontend/Warnings.cpp
@@ -31,12 +31,12 @@
 #include <algorithm>
 using namespace clang;
 
-void clang::ProcessWarningOptions(Diagnostic &Diags,
+void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
                                   const DiagnosticOptions &Opts) {
   Diags.setSuppressSystemWarnings(true);  // Default to -Wno-system-headers
   Diags.setIgnoreAllWarnings(Opts.IgnoreWarnings);
   Diags.setShowOverloads(
-    static_cast<Diagnostic::OverloadsShown>(Opts.ShowOverloads));
+    static_cast<DiagnosticsEngine::OverloadsShown>(Opts.ShowOverloads));
   
   // Handle -ferror-limit
   if (Opts.ErrorLimit)
@@ -48,14 +48,14 @@ void clang::ProcessWarningOptions(Diagnostic &Diags,
   // extension diagnostics onto WARNING or ERROR unless the user has futz'd
   // around with them explicitly.
   if (Opts.PedanticErrors)
-    Diags.setExtensionHandlingBehavior(Diagnostic::Ext_Error);
+    Diags.setExtensionHandlingBehavior(DiagnosticsEngine::Ext_Error);
   else if (Opts.Pedantic)
-    Diags.setExtensionHandlingBehavior(Diagnostic::Ext_Warn);
+    Diags.setExtensionHandlingBehavior(DiagnosticsEngine::Ext_Warn);
   else
-    Diags.setExtensionHandlingBehavior(Diagnostic::Ext_Ignore);
+    Diags.setExtensionHandlingBehavior(DiagnosticsEngine::Ext_Ignore);
 
   for (unsigned i = 0, e = Opts.Warnings.size(); i != e; ++i) {
-    llvm::StringRef Opt = Opts.Warnings[i];
+    StringRef Opt = Opts.Warnings[i];
 
     // Check to see if this warning starts with "no-", if so, this is a negative
     // form of the option.
@@ -75,11 +75,18 @@ void clang::ProcessWarningOptions(Diagnostic &Diags,
       Diags.setSuppressSystemWarnings(!isPositive);
       continue;
     }
+    
+    // -Weverything is a special case as well.  It implicitly enables all
+    // warnings, including ones not explicitly in a warning group.
+    if (Opt == "everything") {
+      Diags.setEnableAllWarnings(true);
+      continue;
+    }
 
     // -Werror/-Wno-error is a special case, not controlled by the option table.
     // It also has the "specifier" form of -Werror=foo and -Werror-foo.
     if (Opt.startswith("error")) {
-      llvm::StringRef Specifier;
+      StringRef Specifier;
       if (Opt.size() > 5) {  // Specifier must be present.
         if ((Opt[5] != '=' && Opt[5] != '-') || Opt.size() == 6) {
           Diags.Report(diag::warn_unknown_warning_specifier)
@@ -94,14 +101,18 @@ void clang::ProcessWarningOptions(Diagnostic &Diags,
         continue;
       }
 
-      // -Werror=foo maps foo to Error, -Wno-error=foo maps it to Warning.
-      Mapping = isPositive ? diag::MAP_ERROR : diag::MAP_WARNING_NO_WERROR;
-      Opt = Specifier;
+      // Set the warning as error flag for this specifier.
+      if (Diags.setDiagnosticGroupWarningAsError(Specifier, isPositive)) {
+        Diags.Report(isPositive ? diag::warn_unknown_warning_option :
+                     diag::warn_unknown_negative_warning_option)
+          << ("-W" + Opt.str());
+      }
+      continue;
     }
 
     // -Wfatal-errors is yet another special case.
     if (Opt.startswith("fatal-errors")) {
-      llvm::StringRef Specifier;
+      StringRef Specifier;
       if (Opt.size() != 12) {
         if ((Opt[12] != '=' && Opt[12] != '-') || Opt.size() == 13) {
           Diags.Report(diag::warn_unknown_warning_specifier)
@@ -116,15 +127,19 @@ void clang::ProcessWarningOptions(Diagnostic &Diags,
         continue;
       }
 
-      // -Wfatal-errors=foo maps foo to Fatal, -Wno-fatal-errors=foo
-      // maps it to Error.
-      Mapping = isPositive ? diag::MAP_FATAL : diag::MAP_ERROR_NO_WFATAL;
-      Opt = Specifier;
+      // Set the error as fatal flag for this specifier.
+      if (Diags.setDiagnosticGroupErrorAsFatal(Specifier, isPositive)) {
+        Diags.Report(isPositive ? diag::warn_unknown_warning_option :
+                     diag::warn_unknown_negative_warning_option)
+          << ("-W" + Opt.str());
+      }
+      continue;
     }
 
-    if (Diags.setDiagnosticGroupMapping(Opt, Mapping))
+    if (Diags.setDiagnosticGroupMapping(Opt, Mapping)) {
       Diags.Report(isPositive ? diag::warn_unknown_warning_option :
                    diag::warn_unknown_negative_warning_option)
           << ("-W" + Opt.str());
+    }
   }
 }
diff --git a/contrib/llvm/tools/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/contrib/llvm/tools/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index f2db3ae74191..c9af3cc3ff1e 100644
--- a/contrib/llvm/tools/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/contrib/llvm/tools/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -39,7 +39,6 @@ static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
   case ASTDumpXML:             return new ASTDumpXMLAction();
   case ASTPrint:               return new ASTPrintAction();
   case ASTView:                return new ASTViewAction();
-  case CreateModule:           return 0;
   case DumpRawTokens:          return new DumpRawTokensAction();
   case DumpTokens:             return new DumpTokensAction();
   case EmitAssembly:           return new EmitAssemblyAction();
@@ -50,7 +49,8 @@ static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
   case EmitCodeGenOnly:        return new EmitCodeGenOnlyAction();
   case EmitObj:                return new EmitObjAction();
   case FixIt:                  return new FixItAction();
-  case GeneratePCH:            return new GeneratePCHAction();
+  case GenerateModule:         return new GeneratePCHAction(true);
+  case GeneratePCH:            return new GeneratePCHAction(false);
   case GeneratePTH:            return new GeneratePTHAction();
   case InitOnly:               return new InitOnlyAction();
   case ParseSyntaxOnly:        return new SyntaxOnlyAction();
@@ -100,7 +100,10 @@ static FrontendAction *CreateFrontendAction(CompilerInstance &CI) {
     Act = new arcmt::ModifyAction(Act);
     break;
   case FrontendOptions::ARCMT_Migrate:
-    Act = new arcmt::MigrateAction(Act, CI.getFrontendOpts().ARCMTMigrateDir);
+    Act = new arcmt::MigrateAction(Act,
+                                   CI.getFrontendOpts().ARCMTMigrateDir,
+                                   CI.getFrontendOpts().ARCMTMigrateReportOut,
+                                CI.getFrontendOpts().ARCMTMigrateEmitARCErrors);
     break;
   }
 
@@ -122,12 +125,6 @@ bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
     return 0;
   }
 
-  // Honor -analyzer-checker-help.
-  if (Clang->getAnalyzerOpts().ShowCheckerHelp) {
-    ento::printCheckerHelp(llvm::outs());
-    return 0;
-  }
-
   // Honor -version.
   //
   // FIXME: Use a better -version message?
@@ -136,19 +133,6 @@ bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
     return 0;
   }
 
-  // Honor -mllvm.
-  //
-  // FIXME: Remove this, one day.
-  if (!Clang->getFrontendOpts().LLVMArgs.empty()) {
-    unsigned NumArgs = Clang->getFrontendOpts().LLVMArgs.size();
-    const char **Args = new const char*[NumArgs + 2];
-    Args[0] = "clang (LLVM option parsing)";
-    for (unsigned i = 0; i != NumArgs; ++i)
-      Args[i + 1] = Clang->getFrontendOpts().LLVMArgs[i].c_str();
-    Args[NumArgs + 1] = 0;
-    llvm::cl::ParseCommandLineOptions(NumArgs + 1, const_cast<char **>(Args));
-  }
-
   // Load any requested plugins.
   for (unsigned i = 0,
          e = Clang->getFrontendOpts().Plugins.size(); i != e; ++i) {
@@ -159,6 +143,27 @@ bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
         << Path << Error;
   }
 
+  // Honor -mllvm.
+  //
+  // FIXME: Remove this, one day.
+  // This should happen AFTER plugins have been loaded!
+  if (!Clang->getFrontendOpts().LLVMArgs.empty()) {
+    unsigned NumArgs = Clang->getFrontendOpts().LLVMArgs.size();
+    const char **Args = new const char*[NumArgs + 2];
+    Args[0] = "clang (LLVM option parsing)";
+    for (unsigned i = 0; i != NumArgs; ++i)
+      Args[i + 1] = Clang->getFrontendOpts().LLVMArgs[i].c_str();
+    Args[NumArgs + 1] = 0;
+    llvm::cl::ParseCommandLineOptions(NumArgs + 1, const_cast<char **>(Args));
+  }
+
+  // Honor -analyzer-checker-help.
+  // This should happen AFTER plugins have been loaded!
+  if (Clang->getAnalyzerOpts().ShowCheckerHelp) {
+    ento::printCheckerHelp(llvm::outs(), Clang->getFrontendOpts().Plugins);
+    return 0;
+  }
+
   // If there were errors in processing arguments, don't do anything else.
   bool Success = false;
   if (!Clang->getDiagnostics().hasErrorOccurred()) {
diff --git a/contrib/llvm/tools/clang/lib/Headers/avxintrin.h b/contrib/llvm/tools/clang/lib/Headers/avxintrin.h
index 2eb2f8562256..0a0d2e45d965 100644
--- a/contrib/llvm/tools/clang/lib/Headers/avxintrin.h
+++ b/contrib/llvm/tools/clang/lib/Headers/avxintrin.h
@@ -341,7 +341,7 @@ _mm256_dp_ps(__m256 a, __m256 b, const int c)
         (__builtin_shufflevector((__v8sf)(a), (__v8sf)(b), \
         (mask) & 0x3,                ((mask) & 0xc) >> 2, \
         (((mask) & 0x30) >> 4) + 8,  (((mask) & 0xc0) >> 6) + 8, \
-        (mask) & 0x3 + 4,            (((mask) & 0xc) >> 2) + 4, \
+        ((mask) & 0x3) + 4,          (((mask) & 0xc) >> 2) + 4, \
         (((mask) & 0x30) >> 4) + 12, (((mask) & 0xc0) >> 6) + 12))
 
 #define _mm256_shuffle_pd(a, b, mask) \
diff --git a/contrib/llvm/tools/clang/lib/Headers/emmintrin.h b/contrib/llvm/tools/clang/lib/Headers/emmintrin.h
index ee12d3caef9c..903cfde87224 100644
--- a/contrib/llvm/tools/clang/lib/Headers/emmintrin.h
+++ b/contrib/llvm/tools/clang/lib/Headers/emmintrin.h
@@ -320,6 +320,12 @@ _mm_comigt_sd(__m128d a, __m128d b)
   return __builtin_ia32_comisdgt(a, b);
 }
 
+static __inline__ int __attribute__((__always_inline__, __nodebug__))
+_mm_comige_sd(__m128d a, __m128d b)
+{
+  return __builtin_ia32_comisdge(a, b);
+}
+
 static __inline__ int __attribute__((__always_inline__, __nodebug__))
 _mm_comineq_sd(__m128d a, __m128d b)
 {
@@ -350,6 +356,12 @@ _mm_ucomigt_sd(__m128d a, __m128d b)
   return __builtin_ia32_ucomisdgt(a, b);
 }
 
+static __inline__ int __attribute__((__always_inline__, __nodebug__))
+_mm_ucomige_sd(__m128d a, __m128d b)
+{
+  return __builtin_ia32_ucomisdge(a, b);
+}
+
 static __inline__ int __attribute__((__always_inline__, __nodebug__))
 _mm_ucomineq_sd(__m128d a, __m128d b)
 {
@@ -452,7 +464,11 @@ _mm_load_pd(double const *dp)
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
 _mm_load1_pd(double const *dp)
 {
-  return (__m128d){ dp[0], dp[0] };
+  struct __mm_load1_pd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  double u = ((struct __mm_load1_pd_struct*)dp)->u;
+  return (__m128d){ u, u };
 }
 
 #define        _mm_load_pd1(dp)        _mm_load1_pd(dp)
@@ -460,7 +476,8 @@ _mm_load1_pd(double const *dp)
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
 _mm_loadr_pd(double const *dp)
 {
-  return (__m128d){ dp[1], dp[0] };
+  __m128d u = *(__m128d*)dp;
+  return __builtin_shufflevector(u, u, 1, 0);
 }
 
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
@@ -475,19 +492,31 @@ _mm_loadu_pd(double const *dp)
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
 _mm_load_sd(double const *dp)
 {
-  return (__m128d){ *dp, 0.0 };
+  struct __mm_load_sd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  double u = ((struct __mm_load_sd_struct*)dp)->u;
+  return (__m128d){ u, 0 };
 }
 
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
 _mm_loadh_pd(__m128d a, double const *dp)
 {
-  return (__m128d){ a[0], *dp };
+  struct __mm_loadh_pd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  double u = ((struct __mm_loadh_pd_struct*)dp)->u;
+  return (__m128d){ a[0], u };
 }
 
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
 _mm_loadl_pd(__m128d a, double const *dp)
 {
-  return (__m128d){ *dp, a[1] };
+  struct __mm_loadl_pd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  double u = ((struct __mm_loadl_pd_struct*)dp)->u;
+  return (__m128d){ u, a[1] }; 
 }
 
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
@@ -529,14 +558,20 @@ _mm_move_sd(__m128d a, __m128d b)
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_store_sd(double *dp, __m128d a)
 {
-  dp[0] = a[0];
+  struct __mm_store_sd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  ((struct __mm_store_sd_struct*)dp)->u = a[0];
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_store1_pd(double *dp, __m128d a)
 {
-  dp[0] = a[0];
-  dp[1] = a[0];
+  struct __mm_store1_pd_struct {
+    double u[2];
+  } __attribute__((__packed__, __may_alias__));
+  ((struct __mm_store1_pd_struct*)dp)->u[0] = a[0];
+  ((struct __mm_store1_pd_struct*)dp)->u[1] = a[0];
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
@@ -554,20 +589,26 @@ _mm_storeu_pd(double *dp, __m128d a)
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_storer_pd(double *dp, __m128d a)
 {
-  dp[0] = a[1];
-  dp[1] = a[0];
+  a = __builtin_shufflevector(a, a, 1, 0);
+  *(__m128d *)dp = a;
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_storeh_pd(double *dp, __m128d a)
 {
-  dp[0] = a[1];
+  struct __mm_storeh_pd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  ((struct __mm_storeh_pd_struct*)dp)->u = a[1];
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_storel_pd(double *dp, __m128d a)
 {
-  dp[0] = a[0];
+  struct __mm_storeh_pd_struct {
+    double u;
+  } __attribute__((__packed__, __may_alias__));
+  ((struct __mm_storeh_pd_struct*)dp)->u = a[0];
 }
 
 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
@@ -1023,7 +1064,10 @@ _mm_loadu_si128(__m128i const *p)
 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
 _mm_loadl_epi64(__m128i const *p)
 {
-  return (__m128i) { *(long long*)p, 0};
+  struct __mm_loadl_epi64_struct {
+    long long u;
+  } __attribute__((__packed__, __may_alias__));
+  return (__m128i) { ((struct __mm_loadl_epi64_struct*)p)->u, 0};
 }
 
 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
diff --git a/contrib/llvm/tools/clang/lib/Headers/float.h b/contrib/llvm/tools/clang/lib/Headers/float.h
index 6eede0b6ba47..b7cb73ae59fe 100644
--- a/contrib/llvm/tools/clang/lib/Headers/float.h
+++ b/contrib/llvm/tools/clang/lib/Headers/float.h
@@ -24,7 +24,7 @@
 #ifndef __FLOAT_H
 #define __FLOAT_H
 
-/* If we're on MinGW, fall baack to the system's float.h, which might have
+/* If we're on MinGW, fall back to the system's float.h, which might have
  * additional definitions provided for Windows.
  * For more details see http://msdn.microsoft.com/en-us/library/y0ybw9fy.aspx
  */
diff --git a/contrib/llvm/tools/clang/lib/Headers/mm_malloc.h b/contrib/llvm/tools/clang/lib/Headers/mm_malloc.h
index ec9236204bb9..5fa176187227 100644
--- a/contrib/llvm/tools/clang/lib/Headers/mm_malloc.h
+++ b/contrib/llvm/tools/clang/lib/Headers/mm_malloc.h
@@ -53,7 +53,9 @@ _mm_malloc(size_t size, size_t align)
     align = sizeof(void *);
 
   void *mallocedMemory;
-#ifdef _WIN32
+#if defined(__MINGW32__)
+  mallocedMemory = __mingw_aligned_malloc(size, align);
+#elif defined(_WIN32)
   mallocedMemory = _aligned_malloc(size, align);
 #else
   if (posix_memalign(&mallocedMemory, align, size))
diff --git a/contrib/llvm/tools/clang/lib/Headers/pmmintrin.h b/contrib/llvm/tools/clang/lib/Headers/pmmintrin.h
index 7ca386cee953..5f9b097ba65f 100644
--- a/contrib/llvm/tools/clang/lib/Headers/pmmintrin.h
+++ b/contrib/llvm/tools/clang/lib/Headers/pmmintrin.h
@@ -84,11 +84,7 @@ _mm_hsub_pd(__m128d a, __m128d b)
   return __builtin_ia32_hsubpd(a, b);
 }
 
-static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
-_mm_loaddup_pd(double const *dp)
-{
-  return (__m128d){ *dp, *dp };
-}
+#define        _mm_loaddup_pd(dp)        _mm_load1_pd(dp)
 
 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
 _mm_movedup_pd(__m128d a)
diff --git a/contrib/llvm/tools/clang/lib/Headers/stdalign.h b/contrib/llvm/tools/clang/lib/Headers/stdalign.h
new file mode 100644
index 000000000000..e7fbfa0499fd
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Headers/stdalign.h
@@ -0,0 +1,30 @@
+/*===---- stdalign.h - Standard header for alignment ------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __STDALIGN_H
+#define __STDALIGN_H
+
+#define alignas _Alignas
+#define __alignas_is_defined 1
+
+#endif /* __STDALIGN_H */
diff --git a/contrib/llvm/tools/clang/lib/Headers/tgmath.h b/contrib/llvm/tools/clang/lib/Headers/tgmath.h
index e1a00236781a..1b0b9d24c1d5 100644
--- a/contrib/llvm/tools/clang/lib/Headers/tgmath.h
+++ b/contrib/llvm/tools/clang/lib/Headers/tgmath.h
@@ -1049,19 +1049,18 @@ static long double
 
 static float
     _TG_ATTRS
-    __tg_nexttoward(float __x, float __y) {return nexttowardf(__x, __y);}
+    __tg_nexttoward(float __x, long double __y) {return nexttowardf(__x, __y);}
 
 static double
     _TG_ATTRS
-    __tg_nexttoward(double __x, double __y) {return nexttoward(__x, __y);}
+    __tg_nexttoward(double __x, long double __y) {return nexttoward(__x, __y);}
 
 static long double
     _TG_ATTRS
     __tg_nexttoward(long double __x, long double __y) {return nexttowardl(__x, __y);}
 
 #undef nexttoward
-#define nexttoward(__x, __y) __tg_nexttoward(__tg_promote2((__x), (__y))(__x), \
-                                             __tg_promote2((__x), (__y))(__y))
+#define nexttoward(__x, __y) __tg_nexttoward(__tg_promote1((__x))(__x), (__y))
 
 // remainder
 
diff --git a/contrib/llvm/tools/clang/lib/Headers/xmmintrin.h b/contrib/llvm/tools/clang/lib/Headers/xmmintrin.h
index 50f275dce054..a0bc0bb09274 100644
--- a/contrib/llvm/tools/clang/lib/Headers/xmmintrin.h
+++ b/contrib/llvm/tools/clang/lib/Headers/xmmintrin.h
@@ -501,31 +501,45 @@ _mm_cvtss_f32(__m128 a)
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_loadh_pi(__m128 a, const __m64 *p)
 {
-  __m128 b;
-  b[0] = *(float*)p;
-  b[1] = *((float*)p+1);
-  return __builtin_shufflevector(a, b, 0, 1, 4, 5);
+  typedef float __mm_loadh_pi_v2f32 __attribute__((__vector_size__(8)));
+  struct __mm_loadh_pi_struct {
+    __mm_loadh_pi_v2f32 u;
+  } __attribute__((__packed__, __may_alias__));
+  __mm_loadh_pi_v2f32 b = ((struct __mm_loadh_pi_struct*)p)->u;
+  __m128 bb = __builtin_shufflevector(b, b, 0, 1, 0, 1);
+  return __builtin_shufflevector(a, bb, 0, 1, 4, 5);
 }
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_loadl_pi(__m128 a, const __m64 *p)
 {
-  __m128 b;
-  b[0] = *(float*)p;
-  b[1] = *((float*)p+1);
-  return __builtin_shufflevector(a, b, 4, 5, 2, 3);
+  typedef float __mm_loadl_pi_v2f32 __attribute__((__vector_size__(8)));
+  struct __mm_loadl_pi_struct {
+    __mm_loadl_pi_v2f32 u;
+  } __attribute__((__packed__, __may_alias__));
+  __mm_loadl_pi_v2f32 b = ((struct __mm_loadl_pi_struct*)p)->u;
+  __m128 bb = __builtin_shufflevector(b, b, 0, 1, 0, 1);
+  return __builtin_shufflevector(a, bb, 4, 5, 2, 3);
 }
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_load_ss(const float *p)
 {
-  return (__m128){ *p, 0, 0, 0 };
+  struct __mm_load_ss_struct {
+    float u;
+  } __attribute__((__packed__, __may_alias__));
+  float u = ((struct __mm_load_ss_struct*)p)->u;
+  return (__m128){ u, 0, 0, 0 };
 }
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_load1_ps(const float *p)
 {
-  return (__m128){ *p, *p, *p, *p };
+  struct __mm_load1_ps_struct {
+    float u;
+  } __attribute__((__packed__, __may_alias__));
+  float u = ((struct __mm_load1_ps_struct*)p)->u;
+  return (__m128){ u, u, u, u };
 }
 
 #define        _mm_load_ps1(p) _mm_load1_ps(p)
@@ -541,7 +555,7 @@ _mm_loadu_ps(const float *p)
 {
   struct __loadu_ps {
     __m128 v;
-  } __attribute__((packed, may_alias));
+  } __attribute__((__packed__, __may_alias__));
   return ((struct __loadu_ps*)p)->v;
 }
 
@@ -604,7 +618,10 @@ _mm_storel_pi(__m64 *p, __m128 a)
 static __inline__ void __attribute__((__always_inline__))
 _mm_store_ss(float *p, __m128 a)
 {
-  *p = a[0];
+  struct __mm_store_ss_struct {
+    float u;
+  } __attribute__((__packed__, __may_alias__));
+  ((struct __mm_store_ss_struct*)p)->u = a[0];
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
diff --git a/contrib/llvm/tools/clang/lib/Index/ASTLocation.cpp b/contrib/llvm/tools/clang/lib/Index/ASTLocation.cpp
index bd3b5ee8bf98..66b393eb6571 100644
--- a/contrib/llvm/tools/clang/lib/Index/ASTLocation.cpp
+++ b/contrib/llvm/tools/clang/lib/Index/ASTLocation.cpp
@@ -41,7 +41,7 @@ Decl *ASTLocation::getReferencedDecl() {
     return 0;
 
   switch (getKind()) {
-  default: assert(0 && "Invalid Kind");
+  default: llvm_unreachable("Invalid Kind");
   case N_Type:
     return 0;
   case N_Decl:
@@ -60,8 +60,7 @@ SourceRange ASTLocation::getSourceRange() const {
     return SourceRange();
 
   switch (getKind()) {
-  default: assert(0 && "Invalid Kind");
-    return SourceRange();
+  default: llvm_unreachable("Invalid Kind");
   case N_Decl:
     return D->getSourceRange();
   case N_Stmt:
@@ -75,7 +74,7 @@ SourceRange ASTLocation::getSourceRange() const {
   return SourceRange();
 }
 
-void ASTLocation::print(llvm::raw_ostream &OS) const {
+void ASTLocation::print(raw_ostream &OS) const {
   if (isInvalid()) {
     OS << "<< Invalid ASTLocation >>\n";
     return;
@@ -87,7 +86,7 @@ void ASTLocation::print(llvm::raw_ostream &OS) const {
   case N_Decl:
     OS << "[Decl: " << AsDecl()->getDeclKindName() << " ";
     if (const NamedDecl *ND = dyn_cast<NamedDecl>(AsDecl()))
-      OS << ND;
+      OS << *ND;
     break;
 
   case N_Stmt:
@@ -97,7 +96,7 @@ void ASTLocation::print(llvm::raw_ostream &OS) const {
     
   case N_NamedRef:
     OS << "[NamedRef: " << AsNamedRef().ND->getDeclKindName() << " ";
-    OS << AsNamedRef().ND;
+    OS << *AsNamedRef().ND;
     break;
     
   case N_Type: {
diff --git a/contrib/llvm/tools/clang/lib/Index/CallGraph.cpp b/contrib/llvm/tools/clang/lib/Index/CallGraph.cpp
index 94790b8fbc17..741e78107f84 100644
--- a/contrib/llvm/tools/clang/lib/Index/CallGraph.cpp
+++ b/contrib/llvm/tools/clang/lib/Index/CallGraph.cpp
@@ -110,7 +110,7 @@ Decl *CallGraph::getDecl(CallGraphNode *Node) {
   return Node->getDecl(*Ctx);
 }
 
-void CallGraph::print(llvm::raw_ostream &os) {
+void CallGraph::print(raw_ostream &os) {
   for (iterator I = begin(), E = end(); I != E; ++I) {
     if (I->second->hasCallee()) {
       os << "function: " << I->first.getPrintableName()
diff --git a/contrib/llvm/tools/clang/lib/Index/Entity.cpp b/contrib/llvm/tools/clang/lib/Index/Entity.cpp
index afac05c41377..fbab6d8684fd 100644
--- a/contrib/llvm/tools/clang/lib/Index/Entity.cpp
+++ b/contrib/llvm/tools/clang/lib/Index/Entity.cpp
@@ -47,7 +47,7 @@ class EntityGetter : public DeclVisitor<EntityGetter, Entity> {
                    unsigned IdNS, bool isObjCInstanceMethod);
 
   // Get an Entity associated with the name in the global namespace.
-  Entity getGlobalEntity(llvm::StringRef Name);
+  Entity getGlobalEntity(StringRef Name);
 
   Entity VisitNamedDecl(NamedDecl *D);
   Entity VisitVarDecl(VarDecl *D);
@@ -77,7 +77,7 @@ Entity EntityGetter::getEntity(Entity Parent, DeclarationName Name,
   return Entity(New);
 }
 
-Entity EntityGetter::getGlobalEntity(llvm::StringRef Name) {
+Entity EntityGetter::getGlobalEntity(StringRef Name) {
   IdentifierInfo *II = &ProgImpl.getIdents().get(Name);
   DeclarationName GlobName(II);
   unsigned IdNS = Decl::IDNS_Ordinary;
@@ -209,7 +209,7 @@ Entity EntityImpl::get(Decl *D, Program &Prog, ProgramImpl &ProgImpl) {
 }
 
 /// \brief Get an Entity associated with a global name.
-Entity EntityImpl::get(llvm::StringRef Name, Program &Prog, 
+Entity EntityImpl::get(StringRef Name, Program &Prog, 
                        ProgramImpl &ProgImpl) {
   return EntityGetter(Prog, ProgImpl).getGlobalEntity(Name);
 }
@@ -259,7 +259,7 @@ Entity Entity::get(Decl *D, Program &Prog) {
   return EntityImpl::get(D, Prog, ProgImpl);
 }
 
-Entity Entity::get(llvm::StringRef Name, Program &Prog) {
+Entity Entity::get(StringRef Name, Program &Prog) {
   ProgramImpl &ProgImpl = *static_cast<ProgramImpl*>(Prog.Impl);
   return EntityImpl::get(Name, Prog, ProgImpl);
 }
diff --git a/contrib/llvm/tools/clang/lib/Index/EntityImpl.h b/contrib/llvm/tools/clang/lib/Index/EntityImpl.h
index da52ccfc099e..6d6a0c6d2b69 100644
--- a/contrib/llvm/tools/clang/lib/Index/EntityImpl.h
+++ b/contrib/llvm/tools/clang/lib/Index/EntityImpl.h
@@ -47,7 +47,7 @@ class EntityImpl : public llvm::FoldingSetNode {
   /// \brief Get an Entity associated with the given Decl.
   /// \returns Null if an Entity cannot refer to this Decl.
   static Entity get(Decl *D, Program &Prog, ProgramImpl &ProgImpl);
-  static Entity get(llvm::StringRef Name, Program &Prog, ProgramImpl &ProgImpl);
+  static Entity get(StringRef Name, Program &Prog, ProgramImpl &ProgImpl);
 
   std::string getPrintableName();
 
diff --git a/contrib/llvm/tools/clang/lib/Index/GlobalSelector.cpp b/contrib/llvm/tools/clang/lib/Index/GlobalSelector.cpp
index 34679185b166..2fe6f95ec9be 100644
--- a/contrib/llvm/tools/clang/lib/Index/GlobalSelector.cpp
+++ b/contrib/llvm/tools/clang/lib/Index/GlobalSelector.cpp
@@ -25,7 +25,7 @@ Selector GlobalSelector::getSelector(ASTContext &AST) const {
 
   Selector GlobSel = Selector(reinterpret_cast<uintptr_t>(Val));
 
-  llvm::SmallVector<IdentifierInfo *, 8> Ids;
+  SmallVector<IdentifierInfo *, 8> Ids;
   for (unsigned i = 0, e = GlobSel.isUnarySelector() ? 1 : GlobSel.getNumArgs();
          i != e; ++i) {
     IdentifierInfo *GlobII = GlobSel.getIdentifierInfoForSlot(i);
@@ -52,7 +52,7 @@ GlobalSelector GlobalSelector::get(Selector Sel, Program &Prog) {
 
   ProgramImpl &ProgImpl = *static_cast<ProgramImpl*>(Prog.Impl);
 
-  llvm::SmallVector<IdentifierInfo *, 8> Ids;
+  SmallVector<IdentifierInfo *, 8> Ids;
   for (unsigned i = 0, e = Sel.isUnarySelector() ? 1 : Sel.getNumArgs();
          i != e; ++i) {
     IdentifierInfo *II = Sel.getIdentifierInfoForSlot(i);
diff --git a/contrib/llvm/tools/clang/lib/Lex/HeaderMap.cpp b/contrib/llvm/tools/clang/lib/Lex/HeaderMap.cpp
index e102a6da608c..0cb564c222d5 100644
--- a/contrib/llvm/tools/clang/lib/Lex/HeaderMap.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/HeaderMap.cpp
@@ -57,7 +57,7 @@ struct HMapHeader {
 /// HashHMapKey - This is the 'well known' hash function required by the file
 /// format, used to look up keys in the hash table.  The hash table uses simple
 /// linear probing based on this function.
-static inline unsigned HashHMapKey(llvm::StringRef Str) {
+static inline unsigned HashHMapKey(StringRef Str) {
   unsigned Result = 0;
   const char *S = Str.begin(), *End = Str.end();
 
@@ -200,7 +200,7 @@ void HeaderMap::dump() const {
 /// LookupFile - Check to see if the specified relative filename is located in
 /// this HeaderMap.  If so, open it and return its FileEntry.
 const FileEntry *HeaderMap::LookupFile(
-    llvm::StringRef Filename, FileManager &FM) const {
+    StringRef Filename, FileManager &FM) const {
   const HMapHeader &Hdr = getHeader();
   unsigned NumBuckets = getEndianAdjustedWord(Hdr.NumBuckets);
 
diff --git a/contrib/llvm/tools/clang/lib/Lex/HeaderSearch.cpp b/contrib/llvm/tools/clang/lib/Lex/HeaderSearch.cpp
index 86ab9564a235..931145a8d655 100644
--- a/contrib/llvm/tools/clang/lib/Lex/HeaderSearch.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/HeaderSearch.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Capacity.h"
 #include <cstdio>
 using namespace clang;
 
@@ -97,6 +98,60 @@ const HeaderMap *HeaderSearch::CreateHeaderMap(const FileEntry *FE) {
   return 0;
 }
 
+const FileEntry *HeaderSearch::lookupModule(StringRef ModuleName,
+                                            std::string *ModuleFileName,
+                                            std::string *UmbrellaHeader) {
+  // If we don't have a module cache path, we can't do anything.
+  if (ModuleCachePath.empty()) {
+    if (ModuleFileName)
+      ModuleFileName->clear();
+    return 0;
+  }
+  
+  // Try to find the module path.
+  llvm::SmallString<256> FileName(ModuleCachePath);
+  llvm::sys::path::append(FileName, ModuleName + ".pcm");
+  if (ModuleFileName)
+    *ModuleFileName = FileName.str();
+    
+  if (const FileEntry *ModuleFile
+        = getFileMgr().getFile(FileName, /*OpenFile=*/false,
+                               /*CacheFailure=*/false))
+    return ModuleFile;
+  
+  // We didn't find the module. If we're not supposed to look for an
+  // umbrella header, this is the end of the road.
+  if (!UmbrellaHeader)
+    return 0;
+  
+  // Look in each of the framework directories for an umbrella header with
+  // the same name as the module.
+  // FIXME: We need a way for non-frameworks to provide umbrella headers.
+  llvm::SmallString<128> UmbrellaHeaderName;
+  UmbrellaHeaderName = ModuleName;
+  UmbrellaHeaderName += '/';
+  UmbrellaHeaderName += ModuleName;
+  UmbrellaHeaderName += ".h";
+  for (unsigned Idx = 0, N = SearchDirs.size(); Idx != N; ++Idx) {
+    // Skip non-framework include paths
+    if (!SearchDirs[Idx].isFramework())
+      continue;
+    
+    // Look for the umbrella header in this directory.
+    if (const FileEntry *HeaderFile
+          = SearchDirs[Idx].LookupFile(UmbrellaHeaderName, *this, 0, 0, 
+                                       StringRef(), 0)) {
+      *UmbrellaHeader = HeaderFile->getName();
+      return 0;
+    }
+  }
+  
+  // We did not find an umbrella header. Clear out the UmbrellaHeader pointee
+  // so our caller knows that we failed.
+  UmbrellaHeader->clear();
+  return 0;
+}
+
 //===----------------------------------------------------------------------===//
 // File lookup within a DirectoryLookup scope
 //===----------------------------------------------------------------------===//
@@ -116,17 +171,19 @@ const char *DirectoryLookup::getName() const {
 /// LookupFile - Lookup the specified file in this search path, returning it
 /// if it exists or returning null if not.
 const FileEntry *DirectoryLookup::LookupFile(
-    llvm::StringRef Filename,
+    StringRef Filename,
     HeaderSearch &HS,
-    llvm::SmallVectorImpl<char> *SearchPath,
-    llvm::SmallVectorImpl<char> *RelativePath) const {
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    StringRef BuildingModule,
+    StringRef *SuggestedModule) const {
   llvm::SmallString<1024> TmpDir;
   if (isNormalDir()) {
     // Concatenate the requested file onto the directory.
     TmpDir = getDir()->getName();
     llvm::sys::path::append(TmpDir, Filename);
     if (SearchPath != NULL) {
-      llvm::StringRef SearchPathRef(getDir()->getName());
+      StringRef SearchPathRef(getDir()->getName());
       SearchPath->clear();
       SearchPath->append(SearchPathRef.begin(), SearchPathRef.end());
     }
@@ -138,14 +195,15 @@ const FileEntry *DirectoryLookup::LookupFile(
   }
 
   if (isFramework())
-    return DoFrameworkLookup(Filename, HS, SearchPath, RelativePath);
+    return DoFrameworkLookup(Filename, HS, SearchPath, RelativePath,
+                             BuildingModule, SuggestedModule);
 
   assert(isHeaderMap() && "Unknown directory lookup");
   const FileEntry * const Result = getHeaderMap()->LookupFile(
       Filename, HS.getFileMgr());
   if (Result) {
     if (SearchPath != NULL) {
-      llvm::StringRef SearchPathRef(getName());
+      StringRef SearchPathRef(getName());
       SearchPath->clear();
       SearchPath->append(SearchPathRef.begin(), SearchPathRef.end());
     }
@@ -161,15 +219,18 @@ const FileEntry *DirectoryLookup::LookupFile(
 /// DoFrameworkLookup - Do a lookup of the specified file in the current
 /// DirectoryLookup, which is a framework directory.
 const FileEntry *DirectoryLookup::DoFrameworkLookup(
-    llvm::StringRef Filename,
+    StringRef Filename,
     HeaderSearch &HS,
-    llvm::SmallVectorImpl<char> *SearchPath,
-    llvm::SmallVectorImpl<char> *RelativePath) const {
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    StringRef BuildingModule,
+    StringRef *SuggestedModule) const 
+{
   FileManager &FileMgr = HS.getFileMgr();
 
   // Framework names must have a '/' in the filename.
   size_t SlashPos = Filename.find('/');
-  if (SlashPos == llvm::StringRef::npos) return 0;
+  if (SlashPos == StringRef::npos) return 0;
 
   // Find out if this is the home for the specified framework, by checking
   // HeaderSearch.  Possible answer are yes/no and unknown.
@@ -226,9 +287,16 @@ const FileEntry *DirectoryLookup::DoFrameworkLookup(
     SearchPath->append(FrameworkName.begin(), FrameworkName.end()-1);
   }
 
+  /// Determine whether this is the module we're building or not.
+  bool AutomaticImport = SuggestedModule &&
+    (BuildingModule != StringRef(Filename.begin(), SlashPos)) &&
+    !Filename.substr(SlashPos + 1).startswith("..");
+  
   FrameworkName.append(Filename.begin()+SlashPos+1, Filename.end());
   if (const FileEntry *FE = FileMgr.getFile(FrameworkName.str(),
-                                            /*openFile=*/true)) {
+                                            /*openFile=*/!AutomaticImport)) {
+    if (AutomaticImport)
+      *SuggestedModule = StringRef(Filename.begin(), SlashPos);
     return FE;
   }
 
@@ -240,7 +308,11 @@ const FileEntry *DirectoryLookup::DoFrameworkLookup(
     SearchPath->insert(SearchPath->begin()+OrigSize, Private,
                        Private+strlen(Private));
 
-  return FileMgr.getFile(FrameworkName.str(), /*openFile=*/true);
+  const FileEntry *FE = FileMgr.getFile(FrameworkName.str(), 
+                                        /*openFile=*/!AutomaticImport);
+  if (FE && AutomaticImport)
+    *SuggestedModule = StringRef(Filename.begin(), SlashPos);
+  return FE;
 }
 
 
@@ -255,13 +327,18 @@ const FileEntry *DirectoryLookup::DoFrameworkLookup(
 /// non-null, indicates where the #including file is, in case a relative search
 /// is needed.
 const FileEntry *HeaderSearch::LookupFile(
-    llvm::StringRef Filename,
+    StringRef Filename,
     bool isAngled,
     const DirectoryLookup *FromDir,
     const DirectoryLookup *&CurDir,
     const FileEntry *CurFileEnt,
-    llvm::SmallVectorImpl<char> *SearchPath,
-    llvm::SmallVectorImpl<char> *RelativePath) {
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    StringRef *SuggestedModule) 
+{
+  if (SuggestedModule)
+    *SuggestedModule = StringRef();
+    
   // If 'Filename' is absolute, check to see if it exists and no searching.
   if (llvm::sys::path::is_absolute(Filename)) {
     CurDir = 0;
@@ -279,7 +356,7 @@ const FileEntry *HeaderSearch::LookupFile(
     return FileMgr.getFile(Filename, /*openFile=*/true);
   }
 
-  // Step #0, unless disabled, check to see if the file is in the #includer's
+  // Unless disabled, check to see if the file is in the #includer's
   // directory.  This has to be based on CurFileEnt, not CurDir, because
   // CurFileEnt could be a #include of a subdirectory (#include "foo/bar.h") and
   // a subsequent include of "baz.h" should resolve to "whatever/foo/baz.h".
@@ -301,7 +378,7 @@ const FileEntry *HeaderSearch::LookupFile(
       unsigned DirInfo = getFileInfo(CurFileEnt).DirInfo;
       getFileInfo(FE).DirInfo = DirInfo;
       if (SearchPath != NULL) {
-        llvm::StringRef SearchPathRef(CurFileEnt->getDir()->getName());
+        StringRef SearchPathRef(CurFileEnt->getDir()->getName());
         SearchPath->clear();
         SearchPath->append(SearchPathRef.begin(), SearchPathRef.end());
       }
@@ -346,19 +423,56 @@ const FileEntry *HeaderSearch::LookupFile(
   // Check each directory in sequence to see if it contains this file.
   for (; i != SearchDirs.size(); ++i) {
     const FileEntry *FE =
-      SearchDirs[i].LookupFile(Filename, *this, SearchPath, RelativePath);
+      SearchDirs[i].LookupFile(Filename, *this, SearchPath, RelativePath,
+                               BuildingModule, SuggestedModule);
     if (!FE) continue;
 
     CurDir = &SearchDirs[i];
 
     // This file is a system header or C++ unfriendly if the dir is.
-    getFileInfo(FE).DirInfo = CurDir->getDirCharacteristic();
+    HeaderFileInfo &HFI = getFileInfo(FE);
+    HFI.DirInfo = CurDir->getDirCharacteristic();
 
+    // If this file is found in a header map and uses the framework style of
+    // includes, then this header is part of a framework we're building.
+    if (CurDir->isIndexHeaderMap()) {
+      size_t SlashPos = Filename.find('/');
+      if (SlashPos != StringRef::npos) {
+        HFI.IndexHeaderMapHeader = 1;
+        HFI.Framework = getUniqueFrameworkName(StringRef(Filename.begin(), 
+                                                         SlashPos));
+      }
+    }
+    
     // Remember this location for the next lookup we do.
     CacheLookup.second = i;
     return FE;
   }
 
+  // If we are including a file with a quoted include "foo.h" from inside
+  // a header in a framework that is currently being built, and we couldn't
+  // resolve "foo.h" any other way, change the include to <Foo/foo.h>, where
+  // "Foo" is the name of the framework in which the including header was found.
+  if (CurFileEnt && !isAngled && Filename.find('/') == StringRef::npos) {
+    HeaderFileInfo &IncludingHFI = getFileInfo(CurFileEnt);
+    if (IncludingHFI.IndexHeaderMapHeader) {
+      llvm::SmallString<128> ScratchFilename;
+      ScratchFilename += IncludingHFI.Framework;
+      ScratchFilename += '/';
+      ScratchFilename += Filename;
+      
+      const FileEntry *Result = LookupFile(ScratchFilename, /*isAngled=*/true,
+                                           FromDir, CurDir, CurFileEnt, 
+                                           SearchPath, RelativePath,
+                                           SuggestedModule);
+      std::pair<unsigned, unsigned> &CacheLookup 
+        = LookupFileCache.GetOrCreateValue(Filename).getValue();
+      CacheLookup.second
+        = LookupFileCache.GetOrCreateValue(ScratchFilename).getValue().second;
+      return Result;
+    }
+  }
+
   // Otherwise, didn't find it. Remember we didn't find this.
   CacheLookup.second = SearchDirs.size();
   return 0;
@@ -370,15 +484,15 @@ const FileEntry *HeaderSearch::LookupFile(
 /// is a subframework within Carbon.framework.  If so, return the FileEntry
 /// for the designated file, otherwise return null.
 const FileEntry *HeaderSearch::
-LookupSubframeworkHeader(llvm::StringRef Filename,
+LookupSubframeworkHeader(StringRef Filename,
                          const FileEntry *ContextFileEnt,
-                         llvm::SmallVectorImpl<char> *SearchPath,
-                         llvm::SmallVectorImpl<char> *RelativePath) {
+                         SmallVectorImpl<char> *SearchPath,
+                         SmallVectorImpl<char> *RelativePath) {
   assert(ContextFileEnt && "No context file?");
 
   // Framework names must have a '/' in the filename.  Find it.
   size_t SlashPos = Filename.find('/');
-  if (SlashPos == llvm::StringRef::npos) return 0;
+  if (SlashPos == StringRef::npos) return 0;
 
   // Look up the base framework name of the ContextFileEnt.
   const char *ContextName = ContextFileEnt->getName();
@@ -466,7 +580,31 @@ LookupSubframeworkHeader(llvm::StringRef Filename,
 // File Info Management.
 //===----------------------------------------------------------------------===//
 
+/// \brief Merge the header file info provided by \p OtherHFI into the current
+/// header file info (\p HFI)
+static void mergeHeaderFileInfo(HeaderFileInfo &HFI, 
+                                const HeaderFileInfo &OtherHFI) {
+  HFI.isImport |= OtherHFI.isImport;
+  HFI.isPragmaOnce |= OtherHFI.isPragmaOnce;
+  HFI.NumIncludes += OtherHFI.NumIncludes;
+  
+  if (!HFI.ControllingMacro && !HFI.ControllingMacroID) {
+    HFI.ControllingMacro = OtherHFI.ControllingMacro;
+    HFI.ControllingMacroID = OtherHFI.ControllingMacroID;
+  }
+  
+  if (OtherHFI.External) {
+    HFI.DirInfo = OtherHFI.DirInfo;
+    HFI.External = OtherHFI.External;
+    HFI.IndexHeaderMapHeader = OtherHFI.IndexHeaderMapHeader;
+  }
 
+  if (HFI.Framework.empty())
+    HFI.Framework = OtherHFI.Framework;
+  
+  HFI.Resolved = true;
+}
+                                
 /// getFileInfo - Return the HeaderFileInfo structure for the specified
 /// FileEntry.
 HeaderFileInfo &HeaderSearch::getFileInfo(const FileEntry *FE) {
@@ -474,10 +612,8 @@ HeaderFileInfo &HeaderSearch::getFileInfo(const FileEntry *FE) {
     FileInfo.resize(FE->getUID()+1);
   
   HeaderFileInfo &HFI = FileInfo[FE->getUID()];
-  if (ExternalSource && !HFI.Resolved) {
-    HFI = ExternalSource->GetHeaderFileInfo(FE);
-    HFI.Resolved = true;
-  }
+  if (ExternalSource && !HFI.Resolved)
+    mergeHeaderFileInfo(HFI, ExternalSource->GetHeaderFileInfo(FE));
   return HFI;
 }
 
@@ -488,10 +624,8 @@ bool HeaderSearch::isFileMultipleIncludeGuarded(const FileEntry *File) {
 
   // Resolve header file info from the external source, if needed.
   HeaderFileInfo &HFI = FileInfo[File->getUID()];
-  if (ExternalSource && !HFI.Resolved) {
-    HFI = ExternalSource->GetHeaderFileInfo(File);
-    HFI.Resolved = true;
-  }
+  if (ExternalSource && !HFI.Resolved)
+    mergeHeaderFileInfo(HFI, ExternalSource->GetHeaderFileInfo(File));
 
   return HFI.isPragmaOnce || HFI.ControllingMacro || HFI.ControllingMacroID;
 }
@@ -542,4 +676,14 @@ bool HeaderSearch::ShouldEnterIncludeFile(const FileEntry *File, bool isImport){
   return true;
 }
 
+size_t HeaderSearch::getTotalMemory() const {
+  return SearchDirs.capacity()
+    + llvm::capacity_in_bytes(FileInfo)
+    + llvm::capacity_in_bytes(HeaderMaps)
+    + LookupFileCache.getAllocator().getTotalMemory()
+    + FrameworkMap.getAllocator().getTotalMemory();
+}
 
+StringRef HeaderSearch::getUniqueFrameworkName(StringRef Framework) {
+  return FrameworkNames.GetOrCreateValue(Framework).getKey();
+}
diff --git a/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp b/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp
index a28b8f6e7b9f..a98d889dbc98 100644
--- a/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp
@@ -32,7 +32,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include <cctype>
+#include <cstring>
 using namespace clang;
 
 static void InitCharacterInfo();
@@ -76,7 +76,7 @@ void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
   // skip the UTF-8 BOM if it's present.
   if (BufferStart == BufferPtr) {
     // Determine the size of the BOM.
-    llvm::StringRef Buf(BufferStart, BufferEnd - BufferStart);
+    StringRef Buf(BufferStart, BufferEnd - BufferStart);
     size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
       .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM
       .Default(0);
@@ -86,7 +86,7 @@ void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
   }
 
   Is_PragmaLexer = false;
-  IsInConflictMarker = false;
+  CurrentConflictMarkerState = CMK_None;
 
   // Start of the file is a start of line.
   IsAtStartOfLine = true;
@@ -187,9 +187,9 @@ Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
 
   // Set the SourceLocation with the remapping information.  This ensures that
   // GetMappedTokenLoc will remap the tokens as they are lexed.
-  L->FileLoc = SM.createInstantiationLoc(SM.getLocForStartOfFile(SpellingFID),
-                                         ExpansionLocStart,
-                                         ExpansionLocEnd, TokLen);
+  L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID),
+                                     ExpansionLocStart,
+                                     ExpansionLocEnd, TokLen);
 
   // Ensure that the lexer thinks it is inside a directive, so that end \n will
   // return an EOD token.
@@ -217,7 +217,7 @@ std::string Lexer::Stringify(const std::string &Str, bool Charify) {
 
 /// Stringify - Convert the specified string into a C string by escaping '\'
 /// and " characters.  This does not add surrounding ""'s to the string.
-void Lexer::Stringify(llvm::SmallVectorImpl<char> &Str) {
+void Lexer::Stringify(SmallVectorImpl<char> &Str) {
   for (unsigned i = 0, e = Str.size(); i != e; ++i) {
     if (Str[i] == '\\' || Str[i] == '"') {
       Str.insert(Str.begin()+i, '\\');
@@ -235,8 +235,8 @@ void Lexer::Stringify(llvm::SmallVectorImpl<char> &Str) {
 /// after trigraph expansion and escaped-newline folding.  In particular, this
 /// wants to get the true, uncanonicalized, spelling of things like digraphs
 /// UCNs, etc.
-llvm::StringRef Lexer::getSpelling(SourceLocation loc,
-                                   llvm::SmallVectorImpl<char> &buffer,
+StringRef Lexer::getSpelling(SourceLocation loc,
+                                   SmallVectorImpl<char> &buffer,
                                    const SourceManager &SM,
                                    const LangOptions &options,
                                    bool *invalid) {
@@ -245,10 +245,10 @@ llvm::StringRef Lexer::getSpelling(SourceLocation loc,
 
   // Try to the load the file buffer.
   bool invalidTemp = false;
-  llvm::StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
+  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
   if (invalidTemp) {
     if (invalid) *invalid = true;
-    return llvm::StringRef();
+    return StringRef();
   }
 
   const char *tokenBegin = file.data() + locInfo.second;
@@ -263,7 +263,7 @@ llvm::StringRef Lexer::getSpelling(SourceLocation loc,
 
   // Common case:  no need for cleaning.
   if (!token.needsCleaning())
-    return llvm::StringRef(tokenBegin, length);
+    return StringRef(tokenBegin, length);
   
   // Hard case, we need to relex the characters into the string.
   buffer.clear();
@@ -275,7 +275,7 @@ llvm::StringRef Lexer::getSpelling(SourceLocation loc,
     ti += charSize;
   }
 
-  return llvm::StringRef(buffer.data(), buffer.size());
+  return StringRef(buffer.data(), buffer.size());
 }
 
 /// getSpelling() - Return the 'spelling' of this token.  The spelling of a
@@ -394,10 +394,10 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
 
   // If this comes from a macro expansion, we really do want the macro name, not
   // the token this macro expanded to.
-  Loc = SM.getInstantiationLoc(Loc);
+  Loc = SM.getExpansionLoc(Loc);
   std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
   bool Invalid = false;
-  llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
   if (Invalid)
     return 0;
 
@@ -415,15 +415,16 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
   return TheTok.getLength();
 }
 
-SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
-                                          const SourceManager &SM,
-                                          const LangOptions &LangOpts) {
+static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
+                                              const SourceManager &SM,
+                                              const LangOptions &LangOpts) {
+  assert(Loc.isFileID());
   std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
   if (LocInfo.first.isInvalid())
     return Loc;
   
   bool Invalid = false;
-  llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
   if (Invalid)
     return Loc;
 
@@ -448,7 +449,7 @@ SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
   }
   
   // Create a lexer starting at the beginning of this token.
-  SourceLocation LexerStartLoc = Loc.getFileLocWithOffset(-LocInfo.second);
+  SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second);
   Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
   TheLexer.SetCommentRetentionState(true);
   
@@ -474,6 +475,25 @@ SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
   return Loc;
 }
 
+SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
+                                          const SourceManager &SM,
+                                          const LangOptions &LangOpts) {
+ if (Loc.isFileID())
+   return getBeginningOfFileToken(Loc, SM, LangOpts);
+ 
+ if (!SM.isMacroArgExpansion(Loc))
+   return Loc;
+
+ SourceLocation FileLoc = SM.getSpellingLoc(Loc);
+ SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts);
+ std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc);
+ std::pair<FileID, unsigned> BeginFileLocInfo= SM.getDecomposedLoc(BeginFileLoc);
+ assert(FileLocInfo.first == BeginFileLocInfo.first &&
+        FileLocInfo.second >= BeginFileLocInfo.second);
+ return Loc.getLocWithOffset(SM.getDecomposedLoc(BeginFileLoc).second -
+                                 SM.getDecomposedLoc(FileLoc).second);
+}
+
 namespace {
   enum PreambleDirectiveKind {
     PDK_Skipped,
@@ -484,21 +504,36 @@ namespace {
 }
 
 std::pair<unsigned, bool>
-Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
+Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer,
+                       const LangOptions &Features, unsigned MaxLines) {
   // Create a lexer starting at the beginning of the file. Note that we use a
   // "fake" file source location at offset 1 so that the lexer will track our
   // position within the file.
   const unsigned StartOffset = 1;
   SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset);
-  LangOptions LangOpts;
-  Lexer TheLexer(StartLoc, LangOpts, Buffer->getBufferStart(), 
+  Lexer TheLexer(StartLoc, Features, Buffer->getBufferStart(),
                  Buffer->getBufferStart(), Buffer->getBufferEnd());
   
   bool InPreprocessorDirective = false;
   Token TheTok;
   Token IfStartTok;
   unsigned IfCount = 0;
-  unsigned Line = 0;
+
+  unsigned MaxLineOffset = 0;
+  if (MaxLines) {
+    const char *CurPtr = Buffer->getBufferStart();
+    unsigned CurLine = 0;
+    while (CurPtr != Buffer->getBufferEnd()) {
+      char ch = *CurPtr++;
+      if (ch == '\n') {
+        ++CurLine;
+        if (CurLine == MaxLines)
+          break;
+      }
+    }
+    if (CurPtr != Buffer->getBufferEnd())
+      MaxLineOffset = CurPtr - Buffer->getBufferStart();
+  }
 
   do {
     TheLexer.LexFromRawLexer(TheTok);
@@ -522,11 +557,11 @@ Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
     
     // Keep track of the # of lines in the preamble.
     if (TheTok.isAtStartOfLine()) {
-      ++Line;
+      unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
 
       // If we were asked to limit the number of lines in the preamble,
       // and we're about to exceed that limit, we're done.
-      if (MaxLines && Line >= MaxLines)
+      if (MaxLineOffset && TokOffset >= MaxLineOffset)
         break;
     }
 
@@ -539,12 +574,12 @@ Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
       Token HashTok = TheTok;
       InPreprocessorDirective = true;
       
-      // Figure out which direective this is. Since we're lexing raw tokens,
+      // Figure out which directive this is. Since we're lexing raw tokens,
       // we don't have an identifier table available. Instead, just look at
       // the raw identifier to recognize and categorize preprocessor directives.
       TheLexer.LexFromRawLexer(TheTok);
       if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
-        llvm::StringRef Keyword(TheTok.getRawIdentifierData(),
+        StringRef Keyword(TheTok.getRawIdentifierData(),
                                 TheTok.getLength());
         PreambleDirectiveKind PDK
           = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
@@ -638,7 +673,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
   // chars, this method is extremely fast.
   while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
     if (CharNo == 0)
-      return TokStart.getFileLocWithOffset(PhysOffset);
+      return TokStart.getLocWithOffset(PhysOffset);
     ++TokPtr, --CharNo, ++PhysOffset;
   }
   
@@ -658,7 +693,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
   if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
     PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
   
-  return TokStart.getFileLocWithOffset(PhysOffset);
+  return TokStart.getLocWithOffset(PhysOffset);
 }
 
 /// \brief Computes the source location just past the end of the
@@ -687,7 +722,7 @@ SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
       return SourceLocation(); // Points inside the macro expansion.
 
     // Continue and find the location just after the macro expansion.
-    Loc = SM.getInstantiationRange(Loc).second;
+    Loc = SM.getExpansionRange(Loc).second;
   }
 
   unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features);
@@ -696,14 +731,14 @@ SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
   else
     return Loc;
   
-  return Loc.getFileLocWithOffset(Len);
+  return Loc.getLocWithOffset(Len);
 }
 
 /// \brief Returns true if the given MacroID location points at the first
 /// token of the macro expansion.
 bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
-                                          const SourceManager &SM,
-                                          const LangOptions &LangOpts) {
+                                      const SourceManager &SM,
+                                      const LangOptions &LangOpts) {
   assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
 
   std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc);
@@ -713,8 +748,7 @@ bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
     return false; // Does not point at the start of token.
 
   SourceLocation expansionLoc =
-    SM.getSLocEntry(infoLoc.first)
-      .getInstantiation().getInstantiationLocStart();
+    SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart();
   if (expansionLoc.isFileID())
     return true; // No other macro expansions, this is the first.
 
@@ -734,17 +768,15 @@ bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
     return false;
 
   FileID FID = SM.getFileID(loc);
-  SourceLocation afterLoc = loc.getFileLocWithOffset(tokLen+1);
-  if (!SM.isBeforeInSourceLocationOffset(afterLoc, SM.getNextOffset()))
-    return true; // We got past the last FileID, this points to the last token.
+  SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1);
+  if (SM.isInFileID(afterLoc, FID))
+    return false; // Still in the same FileID, does not point to the last token.
 
   // FIXME: If the token comes from the macro token paste operator ('##')
   // or the stringify operator ('#') this function will always return false;
-  if (FID == SM.getFileID(afterLoc))
-    return false; // Still in the same FileID, does not point to the last token.
-  
+
   SourceLocation expansionLoc =
-    SM.getSLocEntry(FID).getInstantiation().getInstantiationLocEnd();
+    SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd();
   if (expansionLoc.isFileID())
     return true; // No other macro expansions.
 
@@ -761,7 +793,8 @@ enum {
   CHAR_LETTER   = 0x04,  // a-z,A-Z
   CHAR_NUMBER   = 0x08,  // 0-9
   CHAR_UNDER    = 0x10,  // _
-  CHAR_PERIOD   = 0x20   // .
+  CHAR_PERIOD   = 0x20,  // .
+  CHAR_RAWDEL   = 0x40   // {}[]#<>%:;?*+-/^&|~!=,"'
 };
 
 // Statically initialize CharInfo table based on ASCII character set
@@ -786,20 +819,20 @@ static const unsigned char CharInfo[256] =
    0           , 0           , 0           , 0           ,
 //32 SP         33  !         34  "         35  #
 //36  $         37  %         38  &         39  '
-   CHAR_HORZ_WS, 0           , 0           , 0           ,
-   0           , 0           , 0           , 0           ,
+   CHAR_HORZ_WS, CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
+   0           , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
 //40  (         41  )         42  *         43  +
 //44  ,         45  -         46  .         47  /
-   0           , 0           , 0           , 0           ,
-   0           , 0           , CHAR_PERIOD , 0           ,
+   0           , 0           , CHAR_RAWDEL , CHAR_RAWDEL ,
+   CHAR_RAWDEL , CHAR_RAWDEL , CHAR_PERIOD , CHAR_RAWDEL ,
 //48  0         49  1         50  2         51  3
 //52  4         53  5         54  6         55  7
    CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
    CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
 //56  8         57  9         58  :         59  ;
 //60  <         61  =         62  >         63  ?
-   CHAR_NUMBER , CHAR_NUMBER , 0           , 0           ,
-   0           , 0           , 0           , 0           ,
+   CHAR_NUMBER , CHAR_NUMBER , CHAR_RAWDEL , CHAR_RAWDEL ,
+   CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
 //64  @         65  A         66  B         67  C
 //68  D         69  E         70  F         71  G
    0           , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
@@ -814,8 +847,8 @@ static const unsigned char CharInfo[256] =
    CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
 //88  X         89  Y         90  Z         91  [
 //92  \         93  ]         94  ^         95  _
-   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 0           ,
-   0           , 0           , 0           , CHAR_UNDER  ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL ,
+   0           , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_UNDER  ,
 //96  `         97  a         98  b         99  c
 //100  d       101  e        102  f        103  g
    0           , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
@@ -829,9 +862,9 @@ static const unsigned char CharInfo[256] =
    CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
    CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
 //120  x       121  y        122  z        123  {
-//124  |        125  }        126  ~        127 DEL
-   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 0           ,
-   0           , 0           , 0           , 0
+//124  |       125  }        126  ~        127 DEL
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL ,
+   CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 0
 };
 
 static void InitCharacterInfo() {
@@ -869,6 +902,12 @@ static inline bool isHorizontalWhitespace(unsigned char c) {
   return (CharInfo[c] & CHAR_HORZ_WS) ? true : false;
 }
 
+/// isVerticalWhitespace - Return true if this character is vertical
+/// whitespace: '\n', '\r'.  Note that this returns false for '\0'.
+static inline bool isVerticalWhitespace(unsigned char c) {
+  return (CharInfo[c] & CHAR_VERT_WS) ? true : false;
+}
+
 /// isWhitespace - Return true if this character is horizontal or vertical
 /// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'.  Note that this returns false
 /// for '\0'.
@@ -883,6 +922,14 @@ static inline bool isNumberBody(unsigned char c) {
     true : false;
 }
 
+/// isRawStringDelimBody - Return true if this is the body character of a
+/// raw string delimiter.
+static inline bool isRawStringDelimBody(unsigned char c) {
+  return (CharInfo[c] &
+          (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD|CHAR_RAWDEL)) ?
+    true : false;
+}
+
 
 //===----------------------------------------------------------------------===//
 // Diagnostics forwarding code.
@@ -907,14 +954,14 @@ static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
   // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
   // characters come from spelling(FileLoc)+Offset.
   SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
-  SpellingLoc = SpellingLoc.getFileLocWithOffset(CharNo);
+  SpellingLoc = SpellingLoc.getLocWithOffset(CharNo);
 
   // Figure out the expansion loc range, which is the range covered by the
   // original _Pragma(...) sequence.
   std::pair<SourceLocation,SourceLocation> II =
-    SM.getImmediateInstantiationRange(FileLoc);
+    SM.getImmediateExpansionRange(FileLoc);
 
-  return SM.createInstantiationLoc(SpellingLoc, II.first, II.second, TokLen);
+  return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen);
 }
 
 /// getSourceLocation - Return a source location identifier for the specified
@@ -928,7 +975,7 @@ SourceLocation Lexer::getSourceLocation(const char *Loc,
   // the file id from FileLoc with the offset specified.
   unsigned CharNo = Loc-BufferStart;
   if (FileLoc.isFileID())
-    return FileLoc.getFileLocWithOffset(CharNo);
+    return FileLoc.getLocWithOffset(CharNo);
 
   // Otherwise, this is the _Pragma lexer case, which pretends that all of the
   // tokens are lexed from where the _Pragma was defined.
@@ -978,7 +1025,7 @@ static char DecodeTrigraphChar(const char *CP, Lexer *L) {
   }
 
   if (!L->isLexingRawMode())
-    L->Diag(CP-2, diag::trigraph_converted) << llvm::StringRef(&Res, 1);
+    L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1);
   return Res;
 }
 
@@ -1028,6 +1075,59 @@ const char *Lexer::SkipEscapedNewLines(const char *P) {
   }
 }
 
+/// \brief Checks that the given token is the first token that occurs after the
+/// given location (this excludes comments and whitespace). Returns the location
+/// immediately after the specified token. If the token is not found or the
+/// location is inside a macro, the returned source location will be invalid.
+SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc,
+                                        tok::TokenKind TKind,
+                                        const SourceManager &SM,
+                                        const LangOptions &LangOpts,
+                                        bool SkipTrailingWhitespaceAndNewLine) {
+  if (Loc.isMacroID()) {
+    if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts))
+      return SourceLocation();
+    Loc = SM.getExpansionRange(Loc).second;
+  }
+  Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
+
+  // Break down the source location.
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+
+  // Try to load the file buffer.
+  bool InvalidTemp = false;
+  llvm::StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp);
+  if (InvalidTemp)
+    return SourceLocation();
+
+  const char *TokenBegin = File.data() + LocInfo.second;
+
+  // Lex from the start of the given location.
+  Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
+                                      TokenBegin, File.end());
+  // Find the token.
+  Token Tok;
+  lexer.LexFromRawLexer(Tok);
+  if (Tok.isNot(TKind))
+    return SourceLocation();
+  SourceLocation TokenLoc = Tok.getLocation();
+
+  // Calculate how much whitespace needs to be skipped if any.
+  unsigned NumWhitespaceChars = 0;
+  if (SkipTrailingWhitespaceAndNewLine) {
+    const char *TokenEnd = SM.getCharacterData(TokenLoc) +
+                           Tok.getLength();
+    unsigned char C = *TokenEnd;
+    while (isHorizontalWhitespace(C)) {
+      C = *(++TokenEnd);
+      NumWhitespaceChars++;
+    }
+    if (isVerticalWhitespace(C))
+      NumWhitespaceChars++;
+  }
+
+  return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars);
+}
 
 /// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
 /// get its size, and return it.  This is tricky in several cases:
@@ -1191,6 +1291,7 @@ void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
     // preprocessor, which may macro expand it or something.
     if (II->isHandleIdentifierCase())
       PP->HandleIdentifier(Result);
+    
     return;
   }
 
@@ -1252,13 +1353,12 @@ void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
   if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
     // If we are in Microsoft mode, don't continue if the constant is hex.
     // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
-    if (!Features.Microsoft || !isHexaLiteral(BufferPtr, Features))
+    if (!Features.MicrosoftExt || !isHexaLiteral(BufferPtr, Features))
       return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
   }
 
   // If we have a hex FP constant, continue.
-  if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p') &&
-      !Features.CPlusPlus0x)
+  if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p'))
     return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
 
   // Update the location of token as well as BufferPtr.
@@ -1268,10 +1368,17 @@ void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
 }
 
 /// LexStringLiteral - Lex the remainder of a string literal, after having lexed
-/// either " or L".
-void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
+/// either " or L" or u8" or u" or U".
+void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
+                             tok::TokenKind Kind) {
   const char *NulCharacter = 0; // Does this string contain the \0 character?
 
+  if (!isLexingRawMode() &&
+      (Kind == tok::utf8_string_literal ||
+       Kind == tok::utf16_string_literal ||
+       Kind == tok::utf32_string_literal))
+    Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal);
+
   char C = getAndAdvanceChar(CurPtr, Result);
   while (C != '"') {
     // Skip escaped characters.  Escaped newlines will already be processed by
@@ -1281,16 +1388,21 @@ void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
     
     if (C == '\n' || C == '\r' ||             // Newline.
         (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
-        PP->CodeCompleteNaturalLanguage();
-      else if (!isLexingRawMode() && !Features.AsmPreprocessor)
+      if (!isLexingRawMode() && !Features.AsmPreprocessor)
         Diag(BufferPtr, diag::warn_unterminated_string);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
     }
     
-    if (C == 0)
+    if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr-1)) {
+        PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        return cutOffLexing();
+      }
+
       NulCharacter = CurPtr-1;
+    }
     C = getAndAdvanceChar(CurPtr, Result);
   }
 
@@ -1300,8 +1412,82 @@ void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
 
   // Update the location of the token as well as the BufferPtr instance var.
   const char *TokStart = BufferPtr;
-  FormTokenWithChars(Result, CurPtr,
-                     Wide ? tok::wide_string_literal : tok::string_literal);
+  FormTokenWithChars(Result, CurPtr, Kind);
+  Result.setLiteralData(TokStart);
+}
+
+/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
+/// having lexed R", LR", u8R", uR", or UR".
+void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
+                                tok::TokenKind Kind) {
+  // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
+  //  Between the initial and final double quote characters of the raw string,
+  //  any transformations performed in phases 1 and 2 (trigraphs,
+  //  universal-character-names, and line splicing) are reverted.
+
+  if (!isLexingRawMode())
+    Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal);
+
+  unsigned PrefixLen = 0;
+
+  while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen]))
+    ++PrefixLen;
+
+  // If the last character was not a '(', then we didn't lex a valid delimiter.
+  if (CurPtr[PrefixLen] != '(') {
+    if (!isLexingRawMode()) {
+      const char *PrefixEnd = &CurPtr[PrefixLen];
+      if (PrefixLen == 16) {
+        Diag(PrefixEnd, diag::err_raw_delim_too_long);
+      } else {
+        Diag(PrefixEnd, diag::err_invalid_char_raw_delim)
+          << StringRef(PrefixEnd, 1);
+      }
+    }
+
+    // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
+    // it's possible the '"' was intended to be part of the raw string, but
+    // there's not much we can do about that.
+    while (1) {
+      char C = *CurPtr++;
+
+      if (C == '"')
+        break;
+      if (C == 0 && CurPtr-1 == BufferEnd) {
+        --CurPtr;
+        break;
+      }
+    }
+
+    FormTokenWithChars(Result, CurPtr, tok::unknown);
+    return;
+  }
+
+  // Save prefix and move CurPtr past it
+  const char *Prefix = CurPtr;
+  CurPtr += PrefixLen + 1; // skip over prefix and '('
+
+  while (1) {
+    char C = *CurPtr++;
+
+    if (C == ')') {
+      // Check for prefix match and closing quote.
+      if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
+        CurPtr += PrefixLen + 1; // skip over prefix and '"'
+        break;
+      }
+    } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
+      if (!isLexingRawMode())
+        Diag(BufferPtr, diag::err_unterminated_raw_string)
+          << StringRef(Prefix, PrefixLen);
+      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+      return;
+    }
+  }
+
+  // Update the location of token as well as BufferPtr.
+  const char *TokStart = BufferPtr;
+  FormTokenWithChars(Result, CurPtr, Kind);
   Result.setLiteralData(TokStart);
 }
 
@@ -1317,7 +1503,8 @@ void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
       // Skip the escaped character.
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
-               (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
+               (C == 0 && (CurPtr-1 == BufferEnd ||  // End of file.
+                           isCodeCompletionPoint(CurPtr-1)))) {
       // If the filename is unterminated, then it must just be a lone <
       // character.  Return this as such.
       FormTokenWithChars(Result, AfterLessPos, tok::less);
@@ -1340,10 +1527,15 @@ void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
 
 
 /// LexCharConstant - Lex the remainder of a character constant, after having
-/// lexed either ' or L'.
-void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
+/// lexed either ' or L' or u' or U'.
+void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
+                            tok::TokenKind Kind) {
   const char *NulCharacter = 0; // Does this character contain the \0 character?
 
+  if (!isLexingRawMode() &&
+      (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant))
+    Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal);
+
   char C = getAndAdvanceChar(CurPtr, Result);
   if (C == '\'') {
     if (!isLexingRawMode() && !Features.AsmPreprocessor)
@@ -1360,13 +1552,17 @@ void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
-        PP->CodeCompleteNaturalLanguage();
-      else if (!isLexingRawMode() && !Features.AsmPreprocessor)
+      if (!isLexingRawMode() && !Features.AsmPreprocessor)
         Diag(BufferPtr, diag::warn_unterminated_char);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
     } else if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr-1)) {
+        PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        return cutOffLexing();
+      }
+
       NulCharacter = CurPtr-1;
     }
     C = getAndAdvanceChar(CurPtr, Result);
@@ -1378,7 +1574,7 @@ void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferPtr;
-  FormTokenWithChars(Result, CurPtr, tok::char_constant);
+  FormTokenWithChars(Result, CurPtr, Kind);
   Result.setLiteralData(TokStart);
 }
 
@@ -1451,20 +1647,28 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
   char C;
   do {
     C = *CurPtr;
-    // FIXME: Speedup BCPL comment lexing.  Just scan for a \n or \r character.
-    // If we find a \n character, scan backwards, checking to see if it's an
-    // escaped newline, like we do for block comments.
-
     // Skip over characters in the fast loop.
     while (C != 0 &&                // Potentially EOF.
-           C != '\\' &&             // Potentially escaped newline.
-           C != '?' &&              // Potentially trigraph.
            C != '\n' && C != '\r')  // Newline or DOS-style newline.
       C = *++CurPtr;
 
-    // If this is a newline, we're done.
-    if (C == '\n' || C == '\r')
-      break;  // Found the newline? Break out!
+    const char *NextLine = CurPtr;
+    if (C != 0) {
+      // We found a newline, see if it's escaped.
+      const char *EscapePtr = CurPtr-1;
+      while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace.
+        --EscapePtr;
+
+      if (*EscapePtr == '\\') // Escaped newline.
+        CurPtr = EscapePtr;
+      else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
+               EscapePtr[-2] == '?') // Trigraph-escaped newline.
+        CurPtr = EscapePtr-2;
+      else
+        break; // This is a newline, we're done.
+
+      C = *CurPtr;
+    }
 
     // Otherwise, this is a hard case.  Fall back on getAndAdvanceChar to
     // properly decode the character.  Read it in raw mode to avoid emitting
@@ -1476,6 +1680,13 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
     C = getAndAdvanceChar(CurPtr, Result);
     LexingRawMode = OldRawMode;
 
+    // If we only read only one character, then no special handling is needed.
+    // We're done and can skip forward to the newline.
+    if (C != 0 && CurPtr == OldPtr+1) {
+      CurPtr = NextLine;
+      break;
+    }
+
     // If the char that we finally got was a \n, then we must have had something
     // like \<newline><newline>.  We don't want to have consumed the second
     // newline, we want CurPtr, to end up pointing to it down below.
@@ -1492,9 +1703,9 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
         if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
           // Okay, we found a // comment that ends in a newline, if the next
           // line is also a // comment, but has spaces, don't emit a diagnostic.
-          if (isspace(C)) {
+          if (isWhitespace(C)) {
             const char *ForwardPtr = CurPtr;
-            while (isspace(*ForwardPtr))  // Skip whitespace.
+            while (isWhitespace(*ForwardPtr))  // Skip whitespace.
               ++ForwardPtr;
             if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
               break;
@@ -1507,12 +1718,16 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
     }
 
     if (CurPtr == BufferEnd+1) { 
-      if (PP && PP->isCodeCompletionFile(FileLoc))
-        PP->CodeCompleteNaturalLanguage();
-
       --CurPtr; 
       break; 
     }
+
+    if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
+      PP->CodeCompleteNaturalLanguage();
+      cutOffLexing();
+      return false;
+    }
+
   } while (C != '\n' && C != '\r');
 
   // Found but did not consume the newline.  Notify comment handlers about the
@@ -1573,7 +1788,7 @@ bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
 
   Result.setKind(tok::comment);
   PP->CreateString(&Spelling[0], Spelling.size(), Result,
-                   Result.getLocation());
+                   Result.getLocation(), Result.getLocation());
   return true;
 }
 
@@ -1667,8 +1882,7 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
   unsigned char C = getCharAndSize(CurPtr, CharSize);
   CurPtr += CharSize;
   if (C == 0 && CurPtr == BufferEnd+1) {
-    if (!isLexingRawMode() &&
-        !PP->isCodeCompletionFile(FileLoc))
+    if (!isLexingRawMode())
       Diag(BufferPtr, diag::err_unterminated_block_comment);
     --CurPtr;
 
@@ -1691,7 +1905,10 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
   while (1) {
     // Skip over all non-interesting characters until we find end of buffer or a
     // (probably ending) '/' character.
-    if (CurPtr + 24 < BufferEnd) {
+    if (CurPtr + 24 < BufferEnd &&
+        // If there is a code-completion point avoid the fast scan because it
+        // doesn't check for '\0'.
+        !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
       // While not aligned to a 16-byte boundary.
       while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
         C = *CurPtr++;
@@ -1751,9 +1968,7 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
           Diag(CurPtr-1, diag::warn_nested_block_comment);
       }
     } else if (C == 0 && CurPtr == BufferEnd+1) {
-      if (PP && PP->isCodeCompletionFile(FileLoc))
-        PP->CodeCompleteNaturalLanguage();
-      else if (!isLexingRawMode())
+      if (!isLexingRawMode())
         Diag(BufferPtr, diag::err_unterminated_block_comment);
       // Note: the user probably forgot a */.  We could continue immediately
       // after the /*, but this would involve lexing a lot of what really is the
@@ -1769,7 +1984,12 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
 
       BufferPtr = CurPtr;
       return false;
+    } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
+      PP->CodeCompleteNaturalLanguage();
+      cutOffLexing();
+      return false;
     }
+
     C = *CurPtr++;
   }
 
@@ -1826,6 +2046,12 @@ std::string Lexer::ReadToEndOfLine() {
     case 0:  // Null.
       // Found end of file?
       if (CurPtr-1 != BufferEnd) {
+        if (isCodeCompletionPoint(CurPtr-1)) {
+          PP->CodeCompleteNaturalLanguage();
+          cutOffLexing();
+          return Result;
+        }
+
         // Nope, normal character, continue.
         Result += Char;
         break;
@@ -1840,8 +2066,8 @@ std::string Lexer::ReadToEndOfLine() {
       // Next, lex the character, which should handle the EOD transition.
       Lex(Tmp);
       if (Tmp.is(tok::code_completion)) {
-        if (PP && PP->getCodeCompletionHandler())
-          PP->getCodeCompletionHandler()->CodeCompleteNaturalLanguage();
+        if (PP)
+          PP->CodeCompleteNaturalLanguage();
         Lex(Tmp);
       }
       assert(Tmp.is(tok::eod) && "Unexpected token!");
@@ -1857,22 +2083,6 @@ std::string Lexer::ReadToEndOfLine() {
 /// This returns true if Result contains a token, false if PP.Lex should be
 /// called again.
 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
-  // Check if we are performing code completion.
-  if (PP && PP->isCodeCompletionFile(FileLoc)) {
-    // We're at the end of the file, but we've been asked to consider the
-    // end of the file to be a code-completion token. Return the
-    // code-completion token.
-    Result.startToken();
-    FormTokenWithChars(Result, CurPtr, tok::code_completion);
-    
-    // Only do the eof -> code_completion translation once.
-    PP->SetCodeCompletionPoint(0, 0, 0);
-    
-    // Silence any diagnostics that occur once we hit the code-completion point.
-    PP->getDiagnostics().setSuppressAllDiagnostics(true);
-    return true;
-  }
-
   // If we hit the end of the file while parsing a preprocessor directive,
   // end the preprocessor directive first.  The next token returned will
   // then be the end of file.
@@ -1900,7 +2110,7 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
 
   // If we are in a #if directive, emit an error.
   while (!ConditionalStack.empty()) {
-    if (!PP->isCodeCompletionFile(FileLoc))
+    if (PP->getCodeCompletionFileLoc() != FileLoc)
       PP->Diag(ConditionalStack.back().IfLoc,
                diag::err_pp_unterminated_conditional);
     ConditionalStack.pop_back();
@@ -1951,15 +2161,18 @@ unsigned Lexer::isNextPPTokenLParen() {
 }
 
 /// FindConflictEnd - Find the end of a version control conflict marker.
-static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd) {
-  llvm::StringRef RestOfBuffer(CurPtr+7, BufferEnd-CurPtr-7);
-  size_t Pos = RestOfBuffer.find(">>>>>>>");
-  while (Pos != llvm::StringRef::npos) {
+static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
+                                   ConflictMarkerKind CMK) {
+  const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
+  size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
+  StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen);
+  size_t Pos = RestOfBuffer.find(Terminator);
+  while (Pos != StringRef::npos) {
     // Must occur at start of line.
     if (RestOfBuffer[Pos-1] != '\r' &&
         RestOfBuffer[Pos-1] != '\n') {
-      RestOfBuffer = RestOfBuffer.substr(Pos+7);
-      Pos = RestOfBuffer.find(">>>>>>>");
+      RestOfBuffer = RestOfBuffer.substr(Pos+TermLen);
+      Pos = RestOfBuffer.find(Terminator);
       continue;
     }
     return RestOfBuffer.data()+Pos;
@@ -1977,23 +2190,25 @@ bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
       CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
     return false;
   
-  // Check to see if we have <<<<<<<.
-  if (BufferEnd-CurPtr < 8 ||
-      llvm::StringRef(CurPtr, 7) != "<<<<<<<")
+  // Check to see if we have <<<<<<< or >>>>.
+  if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") &&
+      (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> "))
     return false;
 
   // If we have a situation where we don't care about conflict markers, ignore
   // it.
-  if (IsInConflictMarker || isLexingRawMode())
+  if (CurrentConflictMarkerState || isLexingRawMode())
     return false;
   
-  // Check to see if there is a >>>>>>> somewhere in the buffer at the start of
-  // a line to terminate this conflict marker.
-  if (FindConflictEnd(CurPtr, BufferEnd)) {
+  ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
+
+  // Check to see if there is an ending marker somewhere in the buffer at the
+  // start of a line to terminate this conflict marker.
+  if (FindConflictEnd(CurPtr, BufferEnd, Kind)) {
     // We found a match.  We are really in a conflict marker.
     // Diagnose this, and ignore to the end of line.
     Diag(CurPtr, diag::err_conflict_marker);
-    IsInConflictMarker = true;
+    CurrentConflictMarkerState = Kind;
     
     // Skip ahead to the end of line.  We know this exists because the
     // end-of-conflict marker starts with \r or \n.
@@ -2010,10 +2225,10 @@ bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
 }
 
 
-/// HandleEndOfConflictMarker - If this is a '=======' or '|||||||' or '>>>>>>>'
-/// marker, then it is the end of a conflict marker.  Handle it by ignoring up
-/// until the end of the line.  This returns true if it is a conflict marker and
-/// false if not.
+/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
+/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
+/// is the end of a conflict marker.  Handle it by ignoring up until the end of
+/// the line.  This returns true if it is a conflict marker and false if not.
 bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
   // Only a conflict marker if it starts at the beginning of a line.
   if (CurPtr != BufferStart &&
@@ -2022,18 +2237,19 @@ bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
   
   // If we have a situation where we don't care about conflict markers, ignore
   // it.
-  if (!IsInConflictMarker || isLexingRawMode())
+  if (!CurrentConflictMarkerState || isLexingRawMode())
     return false;
   
-  // Check to see if we have the marker (7 characters in a row).
-  for (unsigned i = 1; i != 7; ++i)
+  // Check to see if we have the marker (4 characters in a row).
+  for (unsigned i = 1; i != 4; ++i)
     if (CurPtr[i] != CurPtr[0])
       return false;
   
   // If we do have it, search for the end of the conflict marker.  This could
   // fail if it got skipped with a '#if 0' or something.  Note that CurPtr might
   // be the end of conflict marker.
-  if (const char *End = FindConflictEnd(CurPtr, BufferEnd)) {
+  if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
+                                        CurrentConflictMarkerState)) {
     CurPtr = End;
     
     // Skip ahead to the end of line.
@@ -2043,13 +2259,22 @@ bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
     BufferPtr = CurPtr;
     
     // No longer in the conflict marker.
-    IsInConflictMarker = false;
+    CurrentConflictMarkerState = CMK_None;
     return true;
   }
   
   return false;
 }
 
+bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
+  if (PP && PP->isCodeCompletionEnabled()) {
+    SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart);
+    return Loc == PP->getCodeCompletionLoc();
+  }
+
+  return false;
+}
+
 
 /// LexTokenInternal - This implements a simple C family lexer.  It is an
 /// extremely performance critical piece of code.  This assumes that the buffer
@@ -2102,6 +2327,14 @@ void Lexer::LexTokenInternal(Token &Result) {
       return PPCache->Lex(Result);
     }
 
+    // Check if we are performing code completion.
+    if (isCodeCompletionPoint(CurPtr-1)) {
+      // Return the code-completion token.
+      Result.startToken();
+      FormTokenWithChars(Result, CurPtr, tok::code_completion);
+      return;
+    }
+
     if (!isLexingRawMode())
       Diag(CurPtr-1, diag::null_in_file);
     Result.setFlag(Token::LeadingSpace);
@@ -2112,7 +2345,7 @@ void Lexer::LexTokenInternal(Token &Result) {
       
   case 26:  // DOS & CP/M EOF: "^Z".
     // If we're in Microsoft extensions mode, treat this as end of file.
-    if (Features.Microsoft) {
+    if (Features.MicrosoftExt) {
       // Read the PP instance variable into an automatic variable, because
       // LexEndOfFile will often delete 'this'.
       Preprocessor *PPCache = PP;
@@ -2186,6 +2419,102 @@ void Lexer::LexTokenInternal(Token &Result) {
     MIOpt.ReadToken();
     return LexNumericConstant(Result, CurPtr);
 
+  case 'u':   // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+
+    if (Features.CPlusPlus0x) {
+      Char = getCharAndSize(CurPtr, SizeTmp);
+
+      // UTF-16 string literal
+      if (Char == '"')
+        return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                                tok::utf16_string_literal);
+
+      // UTF-16 character constant
+      if (Char == '\'')
+        return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                               tok::utf16_char_constant);
+
+      // UTF-16 raw string literal
+      if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+        return LexRawStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::utf16_string_literal);
+
+      if (Char == '8') {
+        char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);
+
+        // UTF-8 string literal
+        if (Char2 == '"')
+          return LexStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::utf8_string_literal);
+
+        if (Char2 == 'R') {
+          unsigned SizeTmp3;
+          char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
+          // UTF-8 raw string literal
+          if (Char3 == '"') {
+            return LexRawStringLiteral(Result,
+                   ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               SizeTmp3, Result),
+                   tok::utf8_string_literal);
+          }
+        }
+      }
+    }
+
+    // treat u like the start of an identifier.
+    return LexIdentifier(Result, CurPtr);
+
+  case 'U':   // Identifier (Uber) or C++0x UTF-32 string literal
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+
+    if (Features.CPlusPlus0x) {
+      Char = getCharAndSize(CurPtr, SizeTmp);
+
+      // UTF-32 string literal
+      if (Char == '"')
+        return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                                tok::utf32_string_literal);
+
+      // UTF-32 character constant
+      if (Char == '\'')
+        return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                               tok::utf32_char_constant);
+
+      // UTF-32 raw string literal
+      if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+        return LexRawStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::utf32_string_literal);
+    }
+
+    // treat U like the start of an identifier.
+    return LexIdentifier(Result, CurPtr);
+
+  case 'R': // Identifier or C++0x raw string literal
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+
+    if (Features.CPlusPlus0x) {
+      Char = getCharAndSize(CurPtr, SizeTmp);
+
+      if (Char == '"')
+        return LexRawStringLiteral(Result,
+                                   ConsumeChar(CurPtr, SizeTmp, Result),
+                                   tok::string_literal);
+    }
+
+    // treat R like the start of an identifier.
+    return LexIdentifier(Result, CurPtr);
+
   case 'L':   // Identifier (Loony) or wide literal (L'x' or L"xyz").
     // Notify MIOpt that we read a non-whitespace/non-comment token.
     MIOpt.ReadToken();
@@ -2194,21 +2523,30 @@ void Lexer::LexTokenInternal(Token &Result) {
     // Wide string literal.
     if (Char == '"')
       return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
-                              true);
+                              tok::wide_string_literal);
+
+    // Wide raw string literal.
+    if (Features.CPlusPlus0x && Char == 'R' &&
+        getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+      return LexRawStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::wide_string_literal);
 
     // Wide character constant.
     if (Char == '\'')
-      return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
+      return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                             tok::wide_char_constant);
     // FALL THROUGH, treating L like the start of an identifier.
 
   // C99 6.4.2: Identifiers.
   case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
   case 'H': case 'I': case 'J': case 'K':    /*'L'*/case 'M': case 'N':
-  case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U':
+  case 'O': case 'P': case 'Q':    /*'R'*/case 'S': case 'T':    /*'U'*/
   case 'V': case 'W': case 'X': case 'Y': case 'Z':
   case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
   case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
-  case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u':
+  case 'o': case 'p': case 'q': case 'r': case 's': case 't':    /*'u'*/
   case 'v': case 'w': case 'x': case 'y': case 'z':
   case '_':
     // Notify MIOpt that we read a non-whitespace/non-comment token.
@@ -2231,13 +2569,13 @@ void Lexer::LexTokenInternal(Token &Result) {
   case '\'':
     // Notify MIOpt that we read a non-whitespace/non-comment token.
     MIOpt.ReadToken();
-    return LexCharConstant(Result, CurPtr);
+    return LexCharConstant(Result, CurPtr, tok::char_constant);
 
   // C99 6.4.5: String Literals.
   case '"':
     // Notify MIOpt that we read a non-whitespace/non-comment token.
     MIOpt.ReadToken();
-    return LexStringLiteral(Result, CurPtr, false);
+    return LexStringLiteral(Result, CurPtr, tok::string_literal);
 
   // C99 6.4.6: Punctuators.
   case '?':
@@ -2396,7 +2734,7 @@ void Lexer::LexTokenInternal(Token &Result) {
         Kind = tok::hashhash;                          // '%:%:' -> '##'
         CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                              SizeTmp2, Result);
-      } else if (Char == '@' && Features.Microsoft) {  // %:@ -> #@ -> Charize
+      } else if (Char == '@' && Features.MicrosoftExt) {// %:@ -> #@ -> Charize
         CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
         if (!isLexingRawMode())
           Diag(BufferPtr, diag::charize_microsoft_ext);
@@ -2447,6 +2785,10 @@ void Lexer::LexTokenInternal(Token &Result) {
         // If this is actually a '<<<<<<<' version control conflict marker,
         // recognize it as such and recover nicely.
         goto LexNextToken;
+      } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) {
+        // If this is '<<<<' and we're in a Perforce-style conflict marker,
+        // ignore it.
+        goto LexNextToken;
       } else if (Features.CUDA && After == '<') {
         Kind = tok::lesslessless;
         CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
@@ -2470,6 +2812,8 @@ void Lexer::LexTokenInternal(Token &Result) {
         char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
         if (After != ':' && After != '>') {
           Kind = tok::less;
+          if (!isLexingRawMode())
+            Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon);
           break;
         }
       }
@@ -2494,6 +2838,10 @@ void Lexer::LexTokenInternal(Token &Result) {
         CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                              SizeTmp2, Result);
         Kind = tok::greatergreaterequal;
+      } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) {
+        // If this is actually a '>>>>' conflict marker, recognize it as such
+        // and recover nicely.
+        goto LexNextToken;
       } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
         // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
         goto LexNextToken;
@@ -2552,7 +2900,7 @@ void Lexer::LexTokenInternal(Token &Result) {
   case '=':
     Char = getCharAndSize(CurPtr, SizeTmp);
     if (Char == '=') {
-      // If this is '=======' and we're in a conflict marker, ignore it.
+      // If this is '====' and we're in a conflict marker, ignore it.
       if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
         goto LexNextToken;
       
@@ -2570,7 +2918,7 @@ void Lexer::LexTokenInternal(Token &Result) {
     if (Char == '#') {
       Kind = tok::hashhash;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
-    } else if (Char == '@' && Features.Microsoft) {  // #@ -> Charize
+    } else if (Char == '@' && Features.MicrosoftExt) {  // #@ -> Charize
       Kind = tok::hashat;
       if (!isLexingRawMode())
         Diag(BufferPtr, diag::charize_microsoft_ext);
diff --git a/contrib/llvm/tools/clang/lib/Lex/LiteralSupport.cpp b/contrib/llvm/tools/clang/lib/Lex/LiteralSupport.cpp
index 2c96c4d4ee24..70183fd1a0ea 100644
--- a/contrib/llvm/tools/clang/lib/Lex/LiteralSupport.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/LiteralSupport.cpp
@@ -16,8 +16,8 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 
 /// HexDigitValue - Return the value of the specified hex digit, or -1 if it's
@@ -29,12 +29,31 @@ static int HexDigitValue(char C) {
   return -1;
 }
 
+static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
+  switch (kind) {
+  default: llvm_unreachable("Unknown token type!");
+  case tok::char_constant:
+  case tok::string_literal:
+  case tok::utf8_string_literal:
+    return Target.getCharWidth();
+  case tok::wide_char_constant:
+  case tok::wide_string_literal:
+    return Target.getWCharWidth();
+  case tok::utf16_char_constant:
+  case tok::utf16_string_literal:
+    return Target.getChar16Width();
+  case tok::utf32_char_constant:
+  case tok::utf32_string_literal:
+    return Target.getChar32Width();
+  }
+}
+
 /// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
 /// either a character or a string literal.
 static unsigned ProcessCharEscape(const char *&ThisTokBuf,
                                   const char *ThisTokEnd, bool &HadError,
-                                  FullSourceLoc Loc, bool IsWide,
-                                  Diagnostic *Diags, const TargetInfo &Target) {
+                                  FullSourceLoc Loc, unsigned CharWidth,
+                                  DiagnosticsEngine *Diags) {
   // Skip the '\' char.
   ++ThisTokBuf;
 
@@ -99,9 +118,6 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     }
 
     // See if any bits will be truncated when evaluated as a character.
-    unsigned CharWidth =
-      IsWide ? Target.getWCharWidth() : Target.getCharWidth();
-
     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
       Overflow = true;
       ResultChar &= ~0U >> (32-CharWidth);
@@ -129,9 +145,6 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
              ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
 
     // Check for overflow.  Reject '\777', but not L'\777'.
-    unsigned CharWidth =
-      IsWide ? Target.getWCharWidth() : Target.getCharWidth();
-
     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
       if (Diags)
         Diags->Report(Loc, diag::warn_octal_escape_too_large);
@@ -167,7 +180,7 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
 /// return the UTF32.
 static bool ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
                              uint32_t &UcnVal, unsigned short &UcnLen,
-                             FullSourceLoc Loc, Diagnostic *Diags, 
+                             FullSourceLoc Loc, DiagnosticsEngine *Diags, 
                              const LangOptions &Features) {
   if (!Features.CPlusPlus && !Features.C99 && Diags)
     Diags->Report(Loc, diag::warn_ucn_not_valid_in_c89);
@@ -220,7 +233,8 @@ static bool ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
 /// we will likely rework our support for UCN's.
 static void EncodeUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
                             char *&ResultBuf, bool &HadError,
-                            FullSourceLoc Loc, bool wide, Diagnostic *Diags, 
+                            FullSourceLoc Loc, unsigned CharByteWidth,
+                            DiagnosticsEngine *Diags,
                             const LangOptions &Features) {
   typedef uint32_t UTF32;
   UTF32 UcnVal = 0;
@@ -231,19 +245,22 @@ static void EncodeUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
     return;
   }
 
-  if (wide) {
-    (void)UcnLen;
-    assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
+  assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth) &&
+         "only character widths of 1, 2, or 4 bytes supported");
 
-    if (!Features.ShortWChar) {
-      // Note: our internal rep of wide char tokens is always little-endian.
-      *ResultBuf++ = (UcnVal & 0x000000FF);
-      *ResultBuf++ = (UcnVal & 0x0000FF00) >> 8;
-      *ResultBuf++ = (UcnVal & 0x00FF0000) >> 16;
-      *ResultBuf++ = (UcnVal & 0xFF000000) >> 24;
-      return;
-    }
+  (void)UcnLen;
+  assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
 
+  if (CharByteWidth == 4) {
+    // Note: our internal rep of wide char tokens is always little-endian.
+    *ResultBuf++ = (UcnVal & 0x000000FF);
+    *ResultBuf++ = (UcnVal & 0x0000FF00) >> 8;
+    *ResultBuf++ = (UcnVal & 0x00FF0000) >> 16;
+    *ResultBuf++ = (UcnVal & 0xFF000000) >> 24;
+    return;
+  }
+
+  if (CharByteWidth == 2) {
     // Convert to UTF16.
     if (UcnVal < (UTF32)0xFFFF) {
       *ResultBuf++ = (UcnVal & 0x000000FF);
@@ -262,6 +279,9 @@ static void EncodeUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
     *ResultBuf++ = (surrogate2 & 0x0000FF00) >> 8;
     return;
   }
+
+  assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
+
   // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
   // The conversion below was inspired by:
   //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
@@ -371,7 +391,7 @@ NumericLiteralParser(const char *begin, const char *end,
       // Done.
     } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) {
       PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
-              diag::err_invalid_decimal_digit) << llvm::StringRef(s, 1);
+              diag::err_invalid_decimal_digit) << StringRef(s, 1);
       hadError = true;
       return;
     } else if (*s == '.') {
@@ -434,7 +454,7 @@ NumericLiteralParser(const char *begin, const char *end,
       continue;  // Success.
     case 'i':
     case 'I':
-      if (PP.getLangOptions().Microsoft) {
+      if (PP.getLangOptions().MicrosoftExt) {
         if (isFPConstant || isLong || isLongLong) break;
 
         // Allow i8, i16, i32, i64, and i128.
@@ -498,7 +518,7 @@ NumericLiteralParser(const char *begin, const char *end,
     PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
             isFPConstant ? diag::err_invalid_suffix_float_constant :
                            diag::err_invalid_suffix_integer_constant)
-      << llvm::StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
+      << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
     hadError = true;
     return;
   }
@@ -528,7 +548,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
     }
     // A binary exponent can appear with or with a '.'. If dotted, the
     // binary exponent is required.
-    if ((*s == 'p' || *s == 'P') && !PP.getLangOptions().CPlusPlus0x) {
+    if (*s == 'p' || *s == 'P') {
       const char *Exponent = s;
       s++;
       saw_exponent = true;
@@ -542,12 +562,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
       }
       s = first_non_digit;
 
-      // In C++0x, we cannot support hexadecmial floating literals because
-      // they conflict with user-defined literals, so we warn in previous
-      // versions of C++ by default.
-      if (PP.getLangOptions().CPlusPlus)
-        PP.Diag(TokLoc, diag::ext_hexconstant_cplusplus);
-      else if (!PP.getLangOptions().HexFloats)
+      if (!PP.getLangOptions().HexFloats)
         PP.Diag(TokLoc, diag::ext_hexconstant_invalid);
     } else if (saw_period) {
       PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
@@ -569,7 +584,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
       // Done.
     } else if (isxdigit(*s)) {
       PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
-              diag::err_invalid_binary_digit) << llvm::StringRef(s, 1);
+              diag::err_invalid_binary_digit) << StringRef(s, 1);
       hadError = true;
     }
     // Other suffixes will be diagnosed by the caller.
@@ -599,7 +614,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
   // the code is using an incorrect base.
   if (isxdigit(*s) && *s != 'e' && *s != 'E') {
     PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
-            diag::err_invalid_octal_digit) << llvm::StringRef(s, 1);
+            diag::err_invalid_octal_digit) << StringRef(s, 1);
     hadError = true;
     return;
   }
@@ -688,7 +703,6 @@ bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
 llvm::APFloat::opStatus
 NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
   using llvm::APFloat;
-  using llvm::StringRef;
 
   unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
   return Result.convertFromString(StringRef(ThisTokBegin, n),
@@ -696,14 +710,51 @@ NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
 }
 
 
+///       character-literal: [C++0x lex.ccon]
+///         ' c-char-sequence '
+///         u' c-char-sequence '
+///         U' c-char-sequence '
+///         L' c-char-sequence '
+///       c-char-sequence:
+///         c-char
+///         c-char-sequence c-char
+///       c-char:
+///         any member of the source character set except the single-quote ',
+///           backslash \, or new-line character
+///         escape-sequence
+///         universal-character-name
+///       escape-sequence: [C++0x lex.ccon]
+///         simple-escape-sequence
+///         octal-escape-sequence
+///         hexadecimal-escape-sequence
+///       simple-escape-sequence:
+///         one of \' \" \? \\ \a \b \f \n \r \t \v
+///       octal-escape-sequence:
+///         \ octal-digit
+///         \ octal-digit octal-digit
+///         \ octal-digit octal-digit octal-digit
+///       hexadecimal-escape-sequence:
+///         \x hexadecimal-digit
+///         hexadecimal-escape-sequence hexadecimal-digit
+///       universal-character-name:
+///         \u hex-quad
+///         \U hex-quad hex-quad
+///       hex-quad:
+///         hex-digit hex-digit hex-digit hex-digit
+///
 CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
-                                     SourceLocation Loc, Preprocessor &PP) {
+                                     SourceLocation Loc, Preprocessor &PP,
+                                     tok::TokenKind kind) {
   // At this point we know that the character matches the regex "L?'.*'".
   HadError = false;
 
-  // Determine if this is a wide character.
-  IsWide = begin[0] == 'L';
-  if (IsWide) ++begin;
+  Kind = kind;
+
+  // Determine if this is a wide or UTF character.
+  if (Kind == tok::wide_char_constant || Kind == tok::utf16_char_constant ||
+      Kind == tok::utf32_char_constant) {
+    ++begin;
+  }
 
   // Skip over the entry quote.
   assert(begin[0] == '\'' && "Invalid token lexed");
@@ -730,8 +781,9 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
 
       // Is this a Universal Character Name escape?
     if (begin[0] != '\\')     // If this is a normal character, consume it.
-      ResultChar = *begin++;
+      ResultChar = (unsigned char)*begin++;
     else {                    // Otherwise, this is an escape character.
+      unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo());
       // Check for UCN.
       if (begin[1] == 'u' || begin[1] == 'U') {
         uint32_t utf32 = 0;
@@ -742,19 +794,22 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
           HadError = 1;
         }
         ResultChar = utf32;
+        if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
+          PP.Diag(Loc, diag::warn_ucn_escape_too_large);
+          ResultChar &= ~0U >> (32-CharWidth);
+        }
       } else {
         // Otherwise, this is a non-UCN escape character.  Process it.
         ResultChar = ProcessCharEscape(begin, end, HadError,
                                        FullSourceLoc(Loc,PP.getSourceManager()),
-                                       IsWide,
-                                       &PP.getDiagnostics(), PP.getTargetInfo());
+                                       CharWidth, &PP.getDiagnostics());
       }
     }
 
     // If this is a multi-character constant (e.g. 'abc'), handle it.  These are
     // implementation defined (C99 6.4.4.4p10).
     if (NumCharsSoFar) {
-      if (IsWide) {
+      if (!isAscii()) {
         // Emulate GCC's (unintentional?) behavior: L'ab' -> L'b'.
         LitVal = 0;
       } else {
@@ -776,8 +831,8 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
   if (NumCharsSoFar > 1) {
     // Warn about discarding the top bits for multi-char wide-character
     // constants (L'abcd').
-    if (IsWide)
-      PP.Diag(Loc, diag::warn_extraneous_wide_char_constant);
+    if (!isAscii())
+      PP.Diag(Loc, diag::warn_extraneous_char_constant);
     else if (NumCharsSoFar != 4)
       PP.Diag(Loc, diag::ext_multichar_character_literal);
     else
@@ -789,47 +844,62 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
   // Transfer the value from APInt to uint64_t
   Value = LitVal.getZExtValue();
 
-  if (IsWide && PP.getLangOptions().ShortWChar && Value > 0xFFFF)
-    PP.Diag(Loc, diag::warn_ucn_escape_too_large);
-
   // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
   // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
   // character constants are not sign extended in the this implementation:
   // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
-  if (!IsWide && NumCharsSoFar == 1 && (Value & 128) &&
+  if (isAscii() && NumCharsSoFar == 1 && (Value & 128) &&
       PP.getLangOptions().CharIsSigned)
     Value = (signed char)Value;
 }
 
 
-///       string-literal: [C99 6.4.5]
-///          " [s-char-sequence] "
-///         L" [s-char-sequence] "
+///       string-literal: [C++0x lex.string]
+///         encoding-prefix " [s-char-sequence] "
+///         encoding-prefix R raw-string
+///       encoding-prefix:
+///         u8
+///         u
+///         U
+///         L
 ///       s-char-sequence:
 ///         s-char
 ///         s-char-sequence s-char
 ///       s-char:
-///         any source character except the double quote ",
-///           backslash \, or newline character
-///         escape-character
+///         any member of the source character set except the double-quote ",
+///           backslash \, or new-line character
+///         escape-sequence
 ///         universal-character-name
-///       escape-character: [C99 6.4.4.4]
-///         \ escape-code
-///         universal-character-name
-///       escape-code:
-///         character-escape-code
-///         octal-escape-code
-///         hex-escape-code
-///       character-escape-code: one of
-///         n t b r f v a
-///         \ ' " ?
-///       octal-escape-code:
-///         octal-digit
-///         octal-digit octal-digit
-///         octal-digit octal-digit octal-digit
-///       hex-escape-code:
-///         x hex-digit
-///         hex-escape-code hex-digit
+///       raw-string:
+///         " d-char-sequence ( r-char-sequence ) d-char-sequence "
+///       r-char-sequence:
+///         r-char
+///         r-char-sequence r-char
+///       r-char:
+///         any member of the source character set, except a right parenthesis )
+///           followed by the initial d-char-sequence (which may be empty)
+///           followed by a double quote ".
+///       d-char-sequence:
+///         d-char
+///         d-char-sequence d-char
+///       d-char:
+///         any member of the basic source character set except:
+///           space, the left parenthesis (, the right parenthesis ),
+///           the backslash \, and the control characters representing horizontal
+///           tab, vertical tab, form feed, and newline.
+///       escape-sequence: [C++0x lex.ccon]
+///         simple-escape-sequence
+///         octal-escape-sequence
+///         hexadecimal-escape-sequence
+///       simple-escape-sequence:
+///         one of \' \" \? \\ \a \b \f \n \r \t \v
+///       octal-escape-sequence:
+///         \ octal-digit
+///         \ octal-digit octal-digit
+///         \ octal-digit octal-digit octal-digit
+///       hexadecimal-escape-sequence:
+///         \x hexadecimal-digit
+///         hexadecimal-escape-sequence hexadecimal-digit
 ///       universal-character-name:
 ///         \u hex-quad
 ///         \U hex-quad hex-quad
@@ -841,8 +911,8 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
                     Preprocessor &PP, bool Complain)
   : SM(PP.getSourceManager()), Features(PP.getLangOptions()),
     Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() : 0),
-    MaxTokenLength(0), SizeBound(0), wchar_tByteWidth(0),
-    ResultPtr(ResultBuf.data()), hadError(false), AnyWide(false), Pascal(false) {
+    MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
+    ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
   init(StringToks, NumStringToks);
 }
 
@@ -862,7 +932,7 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
   MaxTokenLength = StringToks[0].getLength();
   assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
   SizeBound = StringToks[0].getLength()-2;  // -2 for "".
-  AnyWide = StringToks[0].is(tok::wide_string_literal);
+  Kind = StringToks[0].getKind();
 
   hadError = false;
 
@@ -883,8 +953,18 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
     if (StringToks[i].getLength() > MaxTokenLength)
       MaxTokenLength = StringToks[i].getLength();
 
-    // Remember if we see any wide strings.
-    AnyWide |= StringToks[i].is(tok::wide_string_literal);
+    // Remember if we see any wide or utf-8/16/32 strings.
+    // Also check for illegal concatenations.
+    if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) {
+      if (isAscii()) {
+        Kind = StringToks[i].getKind();
+      } else {
+        if (Diags)
+          Diags->Report(FullSourceLoc(StringToks[i].getLocation(), SM),
+                        diag::err_unsupported_string_concat);
+        hadError = true;
+      }
+    }
   }
 
   // Include space for the null terminator.
@@ -892,19 +972,14 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
 
   // TODO: K&R warning: "traditional C rejects string constant concatenation"
 
-  // Get the width in bytes of wchar_t.  If no wchar_t strings are used, do not
-  // query the target.  As such, wchar_tByteWidth is only valid if AnyWide=true.
-  wchar_tByteWidth = ~0U;
-  if (AnyWide) {
-    wchar_tByteWidth = Target.getWCharWidth();
-    assert((wchar_tByteWidth & 7) == 0 && "Assumes wchar_t is byte multiple!");
-    wchar_tByteWidth /= 8;
-  }
+  // Get the width in bytes of char/wchar_t/char16_t/char32_t
+  CharByteWidth = getCharWidth(Kind, Target);
+  assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
+  CharByteWidth /= 8;
 
   // The output buffer size needs to be large enough to hold wide characters.
   // This is a worst-case assumption which basically corresponds to L"" "long".
-  if (AnyWide)
-    SizeBound *= wchar_tByteWidth;
+  SizeBound *= CharByteWidth;
 
   // Size the temporary buffer to hold the result string data.
   ResultBuf.resize(SizeBound);
@@ -929,78 +1004,82 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
       Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
                          &StringInvalid);
     if (StringInvalid) {
-      hadError = 1;
+      hadError = true;
       continue;
     }
 
     const char *ThisTokEnd = ThisTokBuf+ThisTokLen-1;  // Skip end quote.
-    bool wide = false;
     // TODO: Input character set mapping support.
 
-    // Skip L marker for wide strings.
-    if (ThisTokBuf[0] == 'L') {
-      wide = true;
+    // Skip marker for wide or unicode strings.
+    if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
       ++ThisTokBuf;
-    }
-
-    assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?");
-    ++ThisTokBuf;
-
-    // Check if this is a pascal string
-    if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
-        ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
-
-      // If the \p sequence is found in the first token, we have a pascal string
-      // Otherwise, if we already have a pascal string, ignore the first \p
-      if (i == 0) {
+      // Skip 8 of u8 marker for utf8 strings.
+      if (ThisTokBuf[0] == '8')
         ++ThisTokBuf;
-        Pascal = true;
-      } else if (Pascal)
-        ThisTokBuf += 2;
     }
 
-    while (ThisTokBuf != ThisTokEnd) {
-      // Is this a span of non-escape characters?
-      if (ThisTokBuf[0] != '\\') {
-        const char *InStart = ThisTokBuf;
-        do {
+    // Check for raw string
+    if (ThisTokBuf[0] == 'R') {
+      ThisTokBuf += 2; // skip R"
+
+      const char *Prefix = ThisTokBuf;
+      while (ThisTokBuf[0] != '(')
+        ++ThisTokBuf;
+      ++ThisTokBuf; // skip '('
+
+      // remove same number of characters from the end
+      if (ThisTokEnd >= ThisTokBuf + (ThisTokBuf - Prefix))
+        ThisTokEnd -= (ThisTokBuf - Prefix);
+
+      // Copy the string over
+      CopyStringFragment(StringRef(ThisTokBuf, ThisTokEnd - ThisTokBuf));
+    } else {
+      assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?");
+      ++ThisTokBuf; // skip "
+
+      // Check if this is a pascal string
+      if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
+          ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
+
+        // If the \p sequence is found in the first token, we have a pascal string
+        // Otherwise, if we already have a pascal string, ignore the first \p
+        if (i == 0) {
           ++ThisTokBuf;
-        } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
+          Pascal = true;
+        } else if (Pascal)
+          ThisTokBuf += 2;
+      }
 
-        // Copy the character span over.
-        unsigned Len = ThisTokBuf-InStart;
-        if (!AnyWide) {
-          memcpy(ResultPtr, InStart, Len);
-          ResultPtr += Len;
-        } else {
-          // Note: our internal rep of wide char tokens is always little-endian.
-          for (; Len; --Len, ++InStart) {
-            *ResultPtr++ = InStart[0];
-            // Add zeros at the end.
-            for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i)
-              *ResultPtr++ = 0;
-          }
+      while (ThisTokBuf != ThisTokEnd) {
+        // Is this a span of non-escape characters?
+        if (ThisTokBuf[0] != '\\') {
+          const char *InStart = ThisTokBuf;
+          do {
+            ++ThisTokBuf;
+          } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
+
+          // Copy the character span over.
+          CopyStringFragment(StringRef(InStart, ThisTokBuf - InStart));
+          continue;
         }
-        continue;
-      }
-      // Is this a Universal Character Name escape?
-      if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
-        EncodeUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
-                        hadError, FullSourceLoc(StringToks[i].getLocation(),SM),
-                        wide, Diags, Features);
-        continue;
-      }
-      // Otherwise, this is a non-UCN escape character.  Process it.
-      unsigned ResultChar =
-        ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
-                          FullSourceLoc(StringToks[i].getLocation(), SM),
-                          AnyWide, Diags, Target);
+        // Is this a Universal Character Name escape?
+        if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
+          EncodeUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
+                          hadError, FullSourceLoc(StringToks[i].getLocation(),SM),
+                          CharByteWidth, Diags, Features);
+          continue;
+        }
+        // Otherwise, this is a non-UCN escape character.  Process it.
+        unsigned ResultChar =
+          ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
+                            FullSourceLoc(StringToks[i].getLocation(), SM),
+                            CharByteWidth*8, Diags);
 
-      // Note: our internal rep of wide char tokens is always little-endian.
-      *ResultPtr++ = ResultChar & 0xFF;
+        // Note: our internal rep of wide char tokens is always little-endian.
+        *ResultPtr++ = ResultChar & 0xFF;
 
-      if (AnyWide) {
-        for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i)
+        for (unsigned i = 1, e = CharByteWidth; i != e; ++i)
           *ResultPtr++ = ResultChar >> i*8;
       }
     }
@@ -1008,8 +1087,7 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
 
   if (Pascal) {
     ResultBuf[0] = ResultPtr-&ResultBuf[0]-1;
-    if (AnyWide)
-      ResultBuf[0] /= wchar_tByteWidth;
+    ResultBuf[0] /= CharByteWidth;
 
     // Verify that pascal strings aren't too large.
     if (GetStringLength() > 256) {
@@ -1018,7 +1096,7 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
                       diag::err_pascal_string_too_long)
           << SourceRange(StringToks[0].getLocation(),
                          StringToks[NumStringToks-1].getLocation());
-      hadError = 1;
+      hadError = true;
       return;
     }
   } else if (Diags) {
@@ -1036,6 +1114,25 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
 }
 
 
+/// copyStringFragment - This function copies from Start to End into ResultPtr.
+/// Performs widening for multi-byte characters.
+void StringLiteralParser::CopyStringFragment(StringRef Fragment) {
+  // Copy the character span over.
+  if (CharByteWidth == 1) {
+    memcpy(ResultPtr, Fragment.data(), Fragment.size());
+    ResultPtr += Fragment.size();
+  } else {
+    // Note: our internal rep of wide char tokens is always little-endian.
+    for (StringRef::iterator I=Fragment.begin(), E=Fragment.end(); I!=E; ++I) {
+      *ResultPtr++ = *I;
+      // Add zeros at the end.
+      for (unsigned i = 1, e = CharByteWidth; i != e; ++i)
+        *ResultPtr++ = 0;
+    }
+  }
+}
+
+
 /// getOffsetOfStringByte - This function returns the offset of the
 /// specified byte of the string data represented by Token.  This handles
 /// advancing over escape sequences in the string.
@@ -1052,7 +1149,8 @@ unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
   if (StringInvalid)
     return 0;
 
-  assert(SpellingPtr[0] != 'L' && "Doesn't handle wide strings yet");
+  assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
+         SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
 
 
   const char *SpellingStart = SpellingPtr;
@@ -1077,7 +1175,7 @@ unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
     bool HadError = false;
     ProcessCharEscape(SpellingPtr, SpellingEnd, HadError,
                       FullSourceLoc(Tok.getLocation(), SM),
-                      false, Diags, Target);
+                      CharByteWidth*8, Diags);
     assert(!HadError && "This method isn't valid on erroneous strings");
     --ByteNo;
   }
diff --git a/contrib/llvm/tools/clang/lib/Lex/MacroArgs.cpp b/contrib/llvm/tools/clang/lib/Lex/MacroArgs.cpp
index 968c15e3c27b..1846d1c05e30 100644
--- a/contrib/llvm/tools/clang/lib/Lex/MacroArgs.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/MacroArgs.cpp
@@ -15,13 +15,15 @@
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/LexDiagnostic.h"
+
+#include <algorithm>
+
 using namespace clang;
 
 /// MacroArgs ctor function - This destroys the vector passed in.
 MacroArgs *MacroArgs::create(const MacroInfo *MI,
-                             const Token *UnexpArgTokens,
-                             unsigned NumToks, bool VarargsElided,
-                             Preprocessor &PP) {
+                             llvm::ArrayRef<Token> UnexpArgTokens,
+                             bool VarargsElided, Preprocessor &PP) {
   assert(MI->isFunctionLike() &&
          "Can't have args for an object-like macro!");
   MacroArgs **ResultEnt = 0;
@@ -31,12 +33,12 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI,
   // free list.  If so, reuse it.
   for (MacroArgs **Entry = &PP.MacroArgCache; *Entry;
        Entry = &(*Entry)->ArgCache)
-    if ((*Entry)->NumUnexpArgTokens >= NumToks &&
+    if ((*Entry)->NumUnexpArgTokens >= UnexpArgTokens.size() &&
         (*Entry)->NumUnexpArgTokens < ClosestMatch) {
       ResultEnt = Entry;
       
       // If we have an exact match, use it.
-      if ((*Entry)->NumUnexpArgTokens == NumToks)
+      if ((*Entry)->NumUnexpArgTokens == UnexpArgTokens.size())
         break;
       // Otherwise, use the best fit.
       ClosestMatch = (*Entry)->NumUnexpArgTokens;
@@ -45,21 +47,22 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI,
   MacroArgs *Result;
   if (ResultEnt == 0) {
     // Allocate memory for a MacroArgs object with the lexer tokens at the end.
-    Result = (MacroArgs*)malloc(sizeof(MacroArgs) + NumToks*sizeof(Token));
+    Result = (MacroArgs*)malloc(sizeof(MacroArgs) + 
+                                UnexpArgTokens.size() * sizeof(Token));
     // Construct the MacroArgs object.
-    new (Result) MacroArgs(NumToks, VarargsElided);
+    new (Result) MacroArgs(UnexpArgTokens.size(), VarargsElided);
   } else {
     Result = *ResultEnt;
     // Unlink this node from the preprocessors singly linked list.
     *ResultEnt = Result->ArgCache;
-    Result->NumUnexpArgTokens = NumToks;
+    Result->NumUnexpArgTokens = UnexpArgTokens.size();
     Result->VarargsElided = VarargsElided;
   }
 
   // Copy the actual unexpanded tokens to immediately after the result ptr.
-  if (NumToks)
-    memcpy(const_cast<Token*>(Result->getUnexpArgument(0)),
-           UnexpArgTokens, NumToks*sizeof(Token));
+  if (!UnexpArgTokens.empty())
+    std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), 
+              const_cast<Token*>(Result->getUnexpArgument(0)));
 
   return Result;
 }
@@ -186,7 +189,8 @@ MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI,
 ///
 Token MacroArgs::StringifyArgument(const Token *ArgToks,
                                    Preprocessor &PP, bool Charify,
-                                   SourceLocation hashInstLoc) {
+                                   SourceLocation ExpansionLocStart,
+                                   SourceLocation ExpansionLocEnd) {
   Token Tok;
   Tok.startToken();
   Tok.setKind(Charify ? tok::char_constant : tok::string_literal);
@@ -208,13 +212,21 @@ Token MacroArgs::StringifyArgument(const Token *ArgToks,
     // by 6.10.3.2p2.
     if (Tok.is(tok::string_literal) ||       // "foo"
         Tok.is(tok::wide_string_literal) ||  // L"foo"
-        Tok.is(tok::char_constant)) {        // 'x' and L'x'.
+        Tok.is(tok::utf8_string_literal) ||  // u8"foo"
+        Tok.is(tok::utf16_string_literal) || // u"foo"
+        Tok.is(tok::utf32_string_literal) || // U"foo"
+        Tok.is(tok::char_constant) ||        // 'x'
+        Tok.is(tok::wide_char_constant) ||   // L'x'.
+        Tok.is(tok::utf16_char_constant) ||  // u'x'.
+        Tok.is(tok::utf32_char_constant)) {  // U'x'.
       bool Invalid = false;
       std::string TokStr = PP.getSpelling(Tok, &Invalid);
       if (!Invalid) {
         std::string Str = Lexer::Stringify(TokStr);
         Result.append(Str.begin(), Str.end());
       }
+    } else if (Tok.is(tok::code_completion)) {
+      PP.CodeCompleteNaturalLanguage();
     } else {
       // Otherwise, just append the token.  Do some gymnastics to get the token
       // in place and avoid copies where possible.
@@ -274,7 +286,8 @@ Token MacroArgs::StringifyArgument(const Token *ArgToks,
     }
   }
 
-  PP.CreateString(&Result[0], Result.size(), Tok, hashInstLoc);
+  PP.CreateString(&Result[0], Result.size(), Tok,
+                  ExpansionLocStart, ExpansionLocEnd);
   return Tok;
 }
 
@@ -282,7 +295,8 @@ Token MacroArgs::StringifyArgument(const Token *ArgToks,
 /// that has been 'stringified' as required by the # operator.
 const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
                                                Preprocessor &PP,
-                                               SourceLocation hashInstLoc) {
+                                               SourceLocation ExpansionLocStart,
+                                               SourceLocation ExpansionLocEnd) {
   assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!");
   if (StringifiedArgs.empty()) {
     StringifiedArgs.resize(getNumArguments());
@@ -291,6 +305,8 @@ const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
   }
   if (StringifiedArgs[ArgNo].isNot(tok::string_literal))
     StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP,
-                                               /*Charify=*/false, hashInstLoc);
+                                               /*Charify=*/false,
+                                               ExpansionLocStart,
+                                               ExpansionLocEnd);
   return StringifiedArgs[ArgNo];
 }
diff --git a/contrib/llvm/tools/clang/lib/Lex/MacroArgs.h b/contrib/llvm/tools/clang/lib/Lex/MacroArgs.h
index a962dacf7c93..cf86d710adb7 100644
--- a/contrib/llvm/tools/clang/lib/Lex/MacroArgs.h
+++ b/contrib/llvm/tools/clang/lib/Lex/MacroArgs.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_CLANG_MACROARGS_H
 #define LLVM_CLANG_MACROARGS_H
 
+#include "llvm/ADT/ArrayRef.h"
+
 #include <vector>
 
 namespace clang {
@@ -58,9 +60,8 @@ class MacroArgs {
   /// MacroArgs ctor function - Create a new MacroArgs object with the specified
   /// macro and argument info.
   static MacroArgs *create(const MacroInfo *MI,
-                           const Token *UnexpArgTokens,
-                           unsigned NumArgTokens, bool VarargsElided,
-                           Preprocessor &PP);
+                           llvm::ArrayRef<Token> UnexpArgTokens,
+                           bool VarargsElided, Preprocessor &PP);
 
   /// destroy - Destroy and deallocate the memory for this object.
   ///
@@ -88,7 +89,8 @@ class MacroArgs {
   /// getStringifiedArgument - Compute, cache, and return the specified argument
   /// that has been 'stringified' as required by the # operator.
   const Token &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP,
-                                      SourceLocation hashInstLoc);
+                                      SourceLocation ExpansionLocStart,
+                                      SourceLocation ExpansionLocEnd);
 
   /// getNumArguments - Return the number of arguments passed into this macro
   /// invocation.
@@ -109,7 +111,8 @@ class MacroArgs {
   ///
   static Token StringifyArgument(const Token *ArgToks,
                                  Preprocessor &PP, bool Charify,
-                                 SourceLocation hashInstLoc);
+                                 SourceLocation ExpansionLocStart,
+                                 SourceLocation ExpansionLocEnd);
   
   
   /// deallocate - This should only be called by the Preprocessor when managing
diff --git a/contrib/llvm/tools/clang/lib/Lex/MacroInfo.cpp b/contrib/llvm/tools/clang/lib/Lex/MacroInfo.cpp
index 0a16a2567219..5a7af5639830 100644
--- a/contrib/llvm/tools/clang/lib/Lex/MacroInfo.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/MacroInfo.cpp
@@ -21,6 +21,7 @@ MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc) {
   IsGNUVarargs = false;
   IsBuiltinMacro = false;
   IsFromAST = false;
+  ChangedAfterLoad = false;
   IsDisabled = false;
   IsUsed = false;
   IsAllowRedefinitionsWithoutWarning = false;
@@ -40,6 +41,7 @@ MacroInfo::MacroInfo(const MacroInfo &MI, llvm::BumpPtrAllocator &PPAllocator) {
   IsGNUVarargs = MI.IsGNUVarargs;
   IsBuiltinMacro = MI.IsBuiltinMacro;
   IsFromAST = MI.IsFromAST;
+  ChangedAfterLoad = MI.ChangedAfterLoad;
   IsDisabled = MI.IsDisabled;
   IsUsed = MI.IsUsed;
   IsAllowRedefinitionsWithoutWarning = MI.IsAllowRedefinitionsWithoutWarning;
@@ -68,9 +70,9 @@ unsigned MacroInfo::getDefinitionLengthSlow(SourceManager &SM) const {
   assert((macroEnd.isFileID() || lastToken.is(tok::comment)) &&
          "Macro defined in macro?");
   std::pair<FileID, unsigned>
-      startInfo = SM.getDecomposedInstantiationLoc(macroStart);
+      startInfo = SM.getDecomposedExpansionLoc(macroStart);
   std::pair<FileID, unsigned>
-      endInfo = SM.getDecomposedInstantiationLoc(macroEnd);
+      endInfo = SM.getDecomposedExpansionLoc(macroEnd);
   assert(startInfo.first == endInfo.first &&
          "Macro definition spanning multiple FileIDs ?");
   assert(startInfo.second <= endInfo.second);
diff --git a/contrib/llvm/tools/clang/lib/Lex/PPCaching.cpp b/contrib/llvm/tools/clang/lib/Lex/PPCaching.cpp
index 33106591c3ba..986341b98668 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PPCaching.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PPCaching.cpp
@@ -74,6 +74,8 @@ void Preprocessor::EnterCachingLexMode() {
     return;
 
   PushIncludeMacroStack();
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_CachingLexer;
 }
 
 
diff --git a/contrib/llvm/tools/clang/lib/Lex/PPDirectives.cpp b/contrib/llvm/tools/clang/lib/Lex/PPDirectives.cpp
index 4af5fabe5c80..de50c750e4d6 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PPDirectives.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PPDirectives.cpp
@@ -17,6 +17,7 @@
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/Pragma.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
@@ -102,8 +103,8 @@ void Preprocessor::ReadMacroName(Token &MacroNameTok, char isDefineUndef) {
   if (MacroNameTok.is(tok::code_completion)) {
     if (CodeComplete)
       CodeComplete->CodeCompleteMacroName(isDefineUndef == 1);
+    setCodeCompletionReached();
     LexUnexpandedToken(MacroNameTok);
-    return;
   }
   
   // Missing macro name?
@@ -192,7 +193,8 @@ void Preprocessor::CheckEndOfDirective(const char *DirType, bool EnableMacros) {
 /// the first valid token.
 void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
                                                 bool FoundNonSkipPortion,
-                                                bool FoundElse) {
+                                                bool FoundElse,
+                                                SourceLocation ElseLoc) {
   ++NumSkipped;
   assert(CurTokenLexer == 0 && CurPPLexer && "Lexing a macro, not a file?");
 
@@ -214,6 +216,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
     if (Tok.is(tok::code_completion)) {
       if (CodeComplete)
         CodeComplete->CodeCompleteInConditionalExclusion();
+      setCodeCompletionReached();
       continue;
     }
     
@@ -222,7 +225,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
       // Emit errors for each unterminated conditional on the stack, including
       // the current one.
       while (!CurPPLexer->ConditionalStack.empty()) {
-        if (!isCodeCompletionFile(Tok.getLocation()))
+        if (CurLexer->getFileLoc() != CodeCompletionFileLoc)
           Diag(CurPPLexer->ConditionalStack.back().IfLoc,
                diag::err_pp_unterminated_conditional);
         CurPPLexer->ConditionalStack.pop_back();
@@ -275,9 +278,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
     // that we can't use Tok.getIdentifierInfo() because its lookup is disabled
     // when skipping.
     char DirectiveBuf[20];
-    llvm::StringRef Directive;
+    StringRef Directive;
     if (!Tok.needsCleaning() && Tok.getLength() < 20) {
-      Directive = llvm::StringRef(RawCharData, Tok.getLength());
+      Directive = StringRef(RawCharData, Tok.getLength());
     } else {
       std::string DirectiveStr = getSpelling(Tok);
       unsigned IdLen = DirectiveStr.size();
@@ -288,11 +291,11 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
         continue;
       }
       memcpy(DirectiveBuf, &DirectiveStr[0], IdLen);
-      Directive = llvm::StringRef(DirectiveBuf, IdLen);
+      Directive = StringRef(DirectiveBuf, IdLen);
     }
 
     if (Directive.startswith("if")) {
-      llvm::StringRef Sub = Directive.substr(2);
+      StringRef Sub = Directive.substr(2);
       if (Sub.empty() ||   // "if"
           Sub == "def" ||   // "ifdef"
           Sub == "ndef") {  // "ifndef"
@@ -307,7 +310,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
           Callbacks->Endif();
       }
     } else if (Directive[0] == 'e') {
-      llvm::StringRef Sub = Directive.substr(1);
+      StringRef Sub = Directive.substr(1);
       if (Sub == "ndif") {  // "endif"
         CheckEndOfDirective("endif");
         PPConditionalInfo CondInfo;
@@ -387,6 +390,11 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
   // of the file, just stop skipping and return to lexing whatever came after
   // the #if block.
   CurPPLexer->LexingRawMode = false;
+
+  if (Callbacks) {
+    SourceLocation BeginLoc = ElseLoc.isValid() ? ElseLoc : IfTokenLoc;
+    Callbacks->SourceRangeSkipped(SourceRange(BeginLoc, Tok.getLocation()));
+  }
 }
 
 void Preprocessor::PTHSkipExcludedConditionalBlock() {
@@ -472,12 +480,13 @@ void Preprocessor::PTHSkipExcludedConditionalBlock() {
 /// return null on failure.  isAngled indicates whether the file reference is
 /// for system #include's or not (i.e. using <> instead of "").
 const FileEntry *Preprocessor::LookupFile(
-    llvm::StringRef Filename,
+    StringRef Filename,
     bool isAngled,
     const DirectoryLookup *FromDir,
     const DirectoryLookup *&CurDir,
-    llvm::SmallVectorImpl<char> *SearchPath,
-    llvm::SmallVectorImpl<char> *RelativePath) {
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    StringRef *SuggestedModule) {
   // If the header lookup mechanism may be relative to the current file, pass in
   // info about where the current file is.
   const FileEntry *CurFileEnt = 0;
@@ -501,12 +510,13 @@ const FileEntry *Preprocessor::LookupFile(
   CurDir = CurDirLookup;
   const FileEntry *FE = HeaderInfo.LookupFile(
       Filename, isAngled, FromDir, CurDir, CurFileEnt,
-      SearchPath, RelativePath);
+      SearchPath, RelativePath, SuggestedModule);
   if (FE) return FE;
 
   // Otherwise, see if this is a subframework header.  If so, this is relative
   // to one of the headers on the #include stack.  Walk the list of the current
   // headers on the #include stack and pass them to HeaderInfo.
+  // FIXME: SuggestedModule!
   if (IsFileLexer()) {
     if ((CurFileEnt = SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
       if ((FE = HeaderInfo.LookupSubframeworkHeader(Filename, CurFileEnt,
@@ -581,6 +591,7 @@ void Preprocessor::HandleDirective(Token &Result) {
     if (CodeComplete)
       CodeComplete->CodeCompleteDirective(
                                     CurPPLexer->getConditionalStackDepth() > 0);
+    setCodeCompletionReached();
     return;
   case tok::numeric_constant:  // # 7  GNU line marker directive.
     if (getLangOptions().AsmPreprocessor)
@@ -652,6 +663,9 @@ void Preprocessor::HandleDirective(Token &Result) {
     case tok::pp_unassert:
       //isExtension = true;  // FIXME: implement #unassert
       break;
+        
+    case tok::pp___export_macro__:
+      return HandleMacroExportDirective(Result);
     }
     break;
   }
@@ -758,9 +772,13 @@ void Preprocessor::HandleLineDirective(Token &Tok) {
 
   // Enforce C99 6.10.4p3: "The digit sequence shall not specify ... a
   // number greater than 2147483647".  C90 requires that the line # be <= 32767.
-  unsigned LineLimit = Features.C99 ? 2147483648U : 32768U;
+  unsigned LineLimit = 32768U;
+  if (Features.C99 || Features.CPlusPlus0x)
+    LineLimit = 2147483648U;
   if (LineNo >= LineLimit)
     Diag(DigitTok, diag::ext_pp_line_too_big) << LineLimit;
+  else if (Features.CPlusPlus0x && LineNo >= 32768U)
+    Diag(DigitTok, diag::warn_cxx98_compat_pp_line_too_big);
 
   int FilenameID = -1;
   Token StrTok;
@@ -777,7 +795,7 @@ void Preprocessor::HandleLineDirective(Token &Tok) {
   } else {
     // Parse and validate the string, converting it into a unique ID.
     StringLiteralParser Literal(&StrTok, 1, *this);
-    assert(!Literal.AnyWide && "Didn't allow wide strings in");
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
     if (Literal.hadError)
       return DiscardUntilEndOfDirective();
     if (Literal.Pascal) {
@@ -825,7 +843,7 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit,
     // If we are leaving the current presumed file, check to make sure the
     // presumed include stack isn't empty!
     FileID CurFileID =
-      SM.getDecomposedInstantiationLoc(FlagTok.getLocation()).first;
+      SM.getDecomposedExpansionLoc(FlagTok.getLocation()).first;
     PresumedLoc PLoc = SM.getPresumedLoc(FlagTok.getLocation());
     if (PLoc.isInvalid())
       return true;
@@ -834,7 +852,7 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit,
     // different physical file, then we aren't in a "1" line marker flag region.
     SourceLocation IncLoc = PLoc.getIncludeLoc();
     if (IncLoc.isInvalid() ||
-        SM.getDecomposedInstantiationLoc(IncLoc).first != CurFileID) {
+        SM.getDecomposedExpansionLoc(IncLoc).first != CurFileID) {
       PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_pop);
       PP.DiscardUntilEndOfDirective();
       return true;
@@ -910,7 +928,7 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) {
   } else {
     // Parse and validate the string, converting it into a unique ID.
     StringLiteralParser Literal(&StrTok, 1, *this);
-    assert(!Literal.AnyWide && "Didn't allow wide strings in");
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
     if (Literal.hadError)
       return DiscardUntilEndOfDirective();
     if (Literal.Pascal) {
@@ -1000,6 +1018,37 @@ void Preprocessor::HandleIdentSCCSDirective(Token &Tok) {
   }
 }
 
+/// \brief Handle a #__export_macro__ directive.
+void Preprocessor::HandleMacroExportDirective(Token &Tok) {
+  Token MacroNameTok;
+  ReadMacroName(MacroNameTok, 2);
+  
+  // Error reading macro name?  If so, diagnostic already issued.
+  if (MacroNameTok.is(tok::eod))
+    return;
+
+  // Check to see if this is the last token on the #__export_macro__ line.
+  CheckEndOfDirective("__export_macro__");
+
+  // Okay, we finally have a valid identifier to undef.
+  MacroInfo *MI = getMacroInfo(MacroNameTok.getIdentifierInfo());
+  
+  // If the macro is not defined, this is an error.
+  if (MI == 0) {
+    Diag(MacroNameTok, diag::err_pp_export_non_macro)
+      << MacroNameTok.getIdentifierInfo();
+    return;
+  }
+  
+  // Note that this macro has now been exported.
+  MI->setExportLocation(MacroNameTok.getLocation());
+  
+  // If this macro definition came from a PCH file, mark it
+  // as having changed since serialization.
+  if (MI->isFromAST())
+    MI->setChangedAfterLoad();
+}
+
 //===----------------------------------------------------------------------===//
 // Preprocessor Include Directive Handling.
 //===----------------------------------------------------------------------===//
@@ -1011,7 +1060,7 @@ void Preprocessor::HandleIdentSCCSDirective(Token &Tok) {
 /// spelling of the filename, but is also expected to handle the case when
 /// this method decides to use a different buffer.
 bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
-                                              llvm::StringRef &Buffer) {
+                                              StringRef &Buffer) {
   // Get the text form of the filename.
   assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
 
@@ -1020,27 +1069,27 @@ bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
   if (Buffer[0] == '<') {
     if (Buffer.back() != '>') {
       Diag(Loc, diag::err_pp_expects_filename);
-      Buffer = llvm::StringRef();
+      Buffer = StringRef();
       return true;
     }
     isAngled = true;
   } else if (Buffer[0] == '"') {
     if (Buffer.back() != '"') {
       Diag(Loc, diag::err_pp_expects_filename);
-      Buffer = llvm::StringRef();
+      Buffer = StringRef();
       return true;
     }
     isAngled = false;
   } else {
     Diag(Loc, diag::err_pp_expects_filename);
-    Buffer = llvm::StringRef();
+    Buffer = StringRef();
     return true;
   }
 
   // Diagnose #include "" as invalid.
   if (Buffer.size() <= 2) {
     Diag(Loc, diag::err_pp_empty_filename);
-    Buffer = llvm::StringRef();
+    Buffer = StringRef();
     return true;
   }
 
@@ -1070,6 +1119,7 @@ bool Preprocessor::ConcatenateIncludeName(
     
     // FIXME: Provide code completion for #includes.
     if (CurTok.is(tok::code_completion)) {
+      setCodeCompletionReached();
       Lex(CurTok);
       continue;
     }
@@ -1122,7 +1172,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
 
   // Reserve a buffer to get the spelling.
   llvm::SmallString<128> FilenameBuffer;
-  llvm::StringRef Filename;
+  StringRef Filename;
   SourceLocation End;
   
   switch (FilenameTok.getKind()) {
@@ -1171,23 +1221,44 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
     return;
   }
 
+  // Complain about attempts to #include files in an audit pragma.
+  if (PragmaARCCFCodeAuditedLoc.isValid()) {
+    Diag(HashLoc, diag::err_pp_include_in_arc_cf_code_audited);
+    Diag(PragmaARCCFCodeAuditedLoc, diag::note_pragma_entered_here);
+
+    // Immediately leave the pragma.
+    PragmaARCCFCodeAuditedLoc = SourceLocation();
+  }
+
   // Search include directories.
   const DirectoryLookup *CurDir;
   llvm::SmallString<1024> SearchPath;
   llvm::SmallString<1024> RelativePath;
   // We get the raw path only if we have 'Callbacks' to which we later pass
   // the path.
+  StringRef SuggestedModule;
   const FileEntry *File = LookupFile(
       Filename, isAngled, LookupFrom, CurDir,
-      Callbacks ? &SearchPath : NULL, Callbacks ? &RelativePath : NULL);
+      Callbacks ? &SearchPath : NULL, Callbacks ? &RelativePath : NULL,
+      AutoModuleImport? &SuggestedModule : 0);
 
+  // If we are supposed to import a module rather than including the header,
+  // do so now.
+  if (!SuggestedModule.empty()) {
+    TheModuleLoader.loadModule(IncludeTok.getLocation(),
+                               Identifiers.get(SuggestedModule),
+                               FilenameTok.getLocation());
+    return;
+  }
+  
   // Notify the callback object that we've seen an inclusion directive.
   if (Callbacks)
     Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled, File,
                                   End, SearchPath, RelativePath);
 
   if (File == 0) {
-    Diag(FilenameTok, diag::warn_pp_file_not_found) << Filename;
+    if (!SuppressIncludeNotFoundError)
+      Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
     return;
   }
 
@@ -1284,7 +1355,7 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc,
 /// closing ), updating MI with what we learn.  Return true if an error occurs
 /// parsing the arg list.
 bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI) {
-  llvm::SmallVector<IdentifierInfo*, 32> Arguments;
+  SmallVector<IdentifierInfo*, 32> Arguments;
 
   Token Tok;
   while (1) {
@@ -1298,8 +1369,10 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI) {
       Diag(Tok, diag::err_pp_expected_ident_in_arg_list);
       return true;
     case tok::ellipsis:  // #define X(... -> C99 varargs
-      // Warn if use of C99 feature in non-C99 mode.
-      if (!Features.C99) Diag(Tok, diag::ext_variadic_macro);
+      if (!Features.C99)
+        Diag(Tok, Features.CPlusPlus0x ? 
+             diag::warn_cxx98_compat_variadic_macro :
+             diag::ext_variadic_macro);
 
       // Lex the token after the identifier.
       LexUnexpandedToken(Tok);
@@ -1423,7 +1496,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
 
     // Read the first token after the arg list for down below.
     LexUnexpandedToken(Tok);
-  } else if (Features.C99) {
+  } else if (Features.C99 || Features.CPlusPlus0x) {
     // C99 requires whitespace between the macro definition and the body.  Emit
     // a diagnostic for something like "#define X+".
     Diag(Tok, diag::ext_c99_whitespace_required_after_macro_name);
@@ -1564,7 +1637,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
   // warn-because-unused-macro set. If it gets used it will be removed from set.
   if (isInPrimaryFile() && // don't warn for include'd macros.
       Diags->getDiagnosticLevel(diag::pp_macro_not_used,
-                               MI->getDefinitionLoc()) != Diagnostic::Ignored) {
+          MI->getDefinitionLoc()) != DiagnosticsEngine::Ignored) {
     MI->setIsWarnIfUnused(true);
     WarnUnusedMacroLocs.insert(MI->getDefinitionLoc());
   }
@@ -1765,7 +1838,7 @@ void Preprocessor::HandleElseDirective(Token &Result) {
 
   // Finally, skip the rest of the contents of this block.
   SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
-                               /*FoundElse*/true);
+                               /*FoundElse*/true, Result.getLocation());
 
   if (Callbacks)
     Callbacks->Else();
@@ -1798,7 +1871,8 @@ void Preprocessor::HandleElifDirective(Token &ElifToken) {
 
   // Finally, skip the rest of the contents of this block.
   SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
-                               /*FoundElse*/CI.FoundElse);
+                               /*FoundElse*/CI.FoundElse,
+                               ElifToken.getLocation());
 
   if (Callbacks)
     Callbacks->Elif(SourceRange(ConditionalBegin, ConditionalEnd));
diff --git a/contrib/llvm/tools/clang/lib/Lex/PPExpressions.cpp b/contrib/llvm/tools/clang/lib/Lex/PPExpressions.cpp
index 8fcfc70a7c67..20f624a0bb12 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PPExpressions.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PPExpressions.cpp
@@ -23,6 +23,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "llvm/ADT/APSInt.h"
+#include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 
 namespace {
@@ -83,20 +84,21 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   Result.setBegin(PeekTok.getLocation());
 
   // Get the next token, don't expand it.
-  PP.LexUnexpandedToken(PeekTok);
+  PP.LexUnexpandedNonComment(PeekTok);
 
   // Two options, it can either be a pp-identifier or a (.
   SourceLocation LParenLoc;
   if (PeekTok.is(tok::l_paren)) {
     // Found a paren, remember we saw it and skip it.
     LParenLoc = PeekTok.getLocation();
-    PP.LexUnexpandedToken(PeekTok);
+    PP.LexUnexpandedNonComment(PeekTok);
   }
 
   if (PeekTok.is(tok::code_completion)) {
     if (PP.getCodeCompletionHandler())
       PP.getCodeCompletionHandler()->CodeCompleteMacroName(false);
-    PP.LexUnexpandedToken(PeekTok);
+    PP.setCodeCompletionReached();
+    PP.LexUnexpandedNonComment(PeekTok);
   }
   
   // If we don't have a pp-identifier now, this is an error.
@@ -115,12 +117,16 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     PP.markMacroAsUsed(Macro);
   }
 
-  // Consume identifier.
-  Result.setEnd(PeekTok.getLocation());
-  PP.LexUnexpandedToken(PeekTok);
+  // Invoke the 'defined' callback.
+  if (PPCallbacks *Callbacks = PP.getPPCallbacks())
+    Callbacks->Defined(PeekTok);
 
   // If we are in parens, ensure we have a trailing ).
   if (LParenLoc.isValid()) {
+    // Consume identifier.
+    Result.setEnd(PeekTok.getLocation());
+    PP.LexUnexpandedNonComment(PeekTok);
+
     if (PeekTok.isNot(tok::r_paren)) {
       PP.Diag(PeekTok.getLocation(), diag::err_pp_missing_rparen) << "defined";
       PP.Diag(LParenLoc, diag::note_matching) << "(";
@@ -129,6 +135,10 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     // Consume the ).
     Result.setEnd(PeekTok.getLocation());
     PP.LexNonComment(PeekTok);
+  } else {
+    // Consume identifier.
+    Result.setEnd(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
   }
 
   // Success, remember that we saw defined(X).
@@ -152,7 +162,8 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   if (PeekTok.is(tok::code_completion)) {
     if (PP.getCodeCompletionHandler())
       PP.getCodeCompletionHandler()->CodeCompletePreprocessorExpression();
-    PP.LexUnexpandedToken(PeekTok);
+    PP.setCodeCompletionReached();
+    PP.LexNonComment(PeekTok);
   }
       
   // If this token's spelling is a pp-identifier, check to see if it is
@@ -188,7 +199,7 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   case tok::numeric_constant: {
     llvm::SmallString<64> IntegerBuffer;
     bool NumberInvalid = false;
-    llvm::StringRef Spelling = PP.getSpelling(PeekTok, IntegerBuffer, 
+    StringRef Spelling = PP.getSpelling(PeekTok, IntegerBuffer, 
                                               &NumberInvalid);
     if (NumberInvalid)
       return true; // a diagnostic was already reported
@@ -205,9 +216,9 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     assert(Literal.isIntegerLiteral() && "Unknown ppnumber");
 
     // long long is a C99 feature.
-    if (!PP.getLangOptions().C99 && !PP.getLangOptions().CPlusPlus0x
-        && Literal.isLongLong)
-      PP.Diag(PeekTok, diag::ext_longlong);
+    if (!PP.getLangOptions().C99 && Literal.isLongLong)
+      PP.Diag(PeekTok, PP.getLangOptions().CPlusPlus0x ?
+              diag::warn_cxx98_compat_longlong : diag::ext_longlong);
 
     // Parse the integer literal into Result.
     if (Literal.GetIntegerValue(Result.Val)) {
@@ -236,15 +247,18 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     PP.LexNonComment(PeekTok);
     return false;
   }
-  case tok::char_constant: {   // 'x'
+  case tok::char_constant:          // 'x'
+  case tok::wide_char_constant: {   // L'x'
+  case tok::utf16_char_constant:    // u'x'
+  case tok::utf32_char_constant:    // U'x'
     llvm::SmallString<32> CharBuffer;
     bool CharInvalid = false;
-    llvm::StringRef ThisTok = PP.getSpelling(PeekTok, CharBuffer, &CharInvalid);
+    StringRef ThisTok = PP.getSpelling(PeekTok, CharBuffer, &CharInvalid);
     if (CharInvalid)
       return true;
 
     CharLiteralParser Literal(ThisTok.begin(), ThisTok.end(),
-                              PeekTok.getLocation(), PP);
+                              PeekTok.getLocation(), PP, PeekTok.getKind());
     if (Literal.hadError())
       return true;  // A diagnostic was already emitted.
 
@@ -255,6 +269,10 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
       NumBits = TI.getIntWidth();
     else if (Literal.isWide())
       NumBits = TI.getWCharWidth();
+    else if (Literal.isUTF16())
+      NumBits = TI.getChar16Width();
+    else if (Literal.isUTF32())
+      NumBits = TI.getChar32Width();
     else
       NumBits = TI.getCharWidth();
 
@@ -262,8 +280,9 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     llvm::APSInt Val(NumBits);
     // Set the value.
     Val = Literal.getValue();
-    // Set the signedness.
-    Val.setIsUnsigned(!PP.getLangOptions().CharIsSigned);
+    // Set the signedness. UTF-16 and UTF-32 are always unsigned
+    if (!Literal.isUTF16() && !Literal.isUTF32())
+      Val.setIsUnsigned(!PP.getLangOptions().CharIsSigned);
 
     if (Result.Val.getBitWidth() > Val.getBitWidth()) {
       Result.Val = Val.extend(Result.Val.getBitWidth());
@@ -521,7 +540,7 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
 
     bool Overflow = false;
     switch (Operator) {
-    default: assert(0 && "Unknown operator token!");
+    default: llvm_unreachable("Unknown operator token!");
     case tok::percent:
       if (RHS.Val != 0)
         Res = LHS.Val % RHS.Val;
@@ -704,7 +723,7 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
   
   // Peek ahead one token.
   Token Tok;
-  Lex(Tok);
+  LexNonComment(Tok);
   
   // C99 6.10.1p3 - All expressions are evaluated as intmax_t or uintmax_t.
   unsigned BitWidth = getTargetInfo().getIntMaxTWidth();
@@ -759,4 +778,3 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
   DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
   return ResVal.Val != 0;
 }
-
diff --git a/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp b/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp
index bf28199b888a..25a98ae47b6e 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp
@@ -89,7 +89,14 @@ void Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir,
       << std::string(SourceMgr.getBufferName(FileStart)) << "";
     return;
   }
-  
+
+  if (isCodeCompletionEnabled() &&
+      SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
+    CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
+    CodeCompletionLoc =
+        CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
+  }
+
   EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
   return;
 }
@@ -106,7 +113,9 @@ void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
   CurLexer.reset(TheLexer);
   CurPPLexer = TheLexer;
   CurDirLookup = CurDir;
-
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_Lexer;
+  
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks && !CurLexer->Is_PragmaLexer) {
     SrcMgr::CharacteristicKind FileType =
@@ -128,7 +137,9 @@ void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
   CurDirLookup = CurDir;
   CurPTHLexer.reset(PL);
   CurPPLexer = CurPTHLexer.get();
-
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_PTHLexer;
+  
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks) {
     FileID FID = CurPPLexer->getFileID();
@@ -152,6 +163,8 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
     CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]);
     CurTokenLexer->Init(Tok, ILEnd, Args);
   }
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_TokenLexer;
 }
 
 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
@@ -181,6 +194,8 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
     CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]);
     CurTokenLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
   }
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_TokenLexer;
 }
 
 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
@@ -201,9 +216,50 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
     }
   }
 
+  // Complain about reaching an EOF within arc_cf_code_audited.
+  if (PragmaARCCFCodeAuditedLoc.isValid()) {
+    Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
+
+    // Recover by leaving immediately.
+    PragmaARCCFCodeAuditedLoc = SourceLocation();
+  }
+
   // If this is a #include'd file, pop it off the include stack and continue
   // lexing the #includer file.
   if (!IncludeMacroStack.empty()) {
+
+    // If we lexed the code-completion file, act as if we reached EOF.
+    if (isCodeCompletionEnabled() && CurPPLexer &&
+        SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
+            CodeCompletionFileLoc) {
+      if (CurLexer) {
+        Result.startToken();
+        CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
+        CurLexer.reset();
+      } else {
+        assert(CurPTHLexer && "Got EOF but no current lexer set!");
+        CurPTHLexer->getEOF(Result);
+        CurPTHLexer.reset();
+      }
+
+      CurPPLexer = 0;
+      return true;
+    }
+
+    if (!isEndOfMacro && CurPPLexer &&
+        SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
+      // Notify SourceManager to record the number of FileIDs that were created
+      // during lexing of the #include'd file.
+      unsigned NumFIDs =
+          SourceMgr.local_sloc_entry_size() -
+          CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
+      SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
+    }
+
+    FileID ExitedFID;
+    if (Callbacks && !isEndOfMacro && CurPPLexer)
+      ExitedFID = CurPPLexer->getFileID();
+    
     // We're done with the #included file.
     RemoveTopOfLexerStack();
 
@@ -212,7 +268,7 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
       SrcMgr::CharacteristicKind FileType =
         SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
       Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
-                             PPCallbacks::ExitFile, FileType);
+                             PPCallbacks::ExitFile, FileType, ExitedFID);
     }
 
     // Client should lex another token.
diff --git a/contrib/llvm/tools/clang/lib/Lex/PPMacroExpansion.cpp b/contrib/llvm/tools/clang/lib/Lex/PPMacroExpansion.cpp
index ecd4d4cfc68b..e10c95c75f25 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PPMacroExpansion.cpp
@@ -21,10 +21,12 @@
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Lex/ExternalPreprocessorSource.h"
+#include "clang/Lex/LiteralSupport.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <cstdio>
 #include <ctime>
 using namespace clang;
@@ -91,9 +93,10 @@ void Preprocessor::RegisterBuiltinMacros() {
   Ident__has_attribute    = RegisterBuiltinMacro(*this, "__has_attribute");
   Ident__has_include      = RegisterBuiltinMacro(*this, "__has_include");
   Ident__has_include_next = RegisterBuiltinMacro(*this, "__has_include_next");
+  Ident__has_warning      = RegisterBuiltinMacro(*this, "__has_warning");
 
   // Microsoft Extensions.
-  if (Features.Microsoft) 
+  if (Features.MicrosoftExt) 
     Ident__pragma = RegisterBuiltinMacro(*this, "__pragma");
   else
     Ident__pragma = 0;
@@ -185,7 +188,8 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
 
   // If this is a builtin macro, like __LINE__ or _Pragma, handle it specially.
   if (MI->isBuiltinMacro()) {
-    if (Callbacks) Callbacks->MacroExpands(Identifier, MI);
+    if (Callbacks) Callbacks->MacroExpands(Identifier, MI,
+                                           Identifier.getLocation());
     ExpandBuiltinMacro(Identifier);
     return false;
   }
@@ -226,13 +230,14 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   // Notice that this macro has been used.
   markMacroAsUsed(MI);
 
-  if (Callbacks) Callbacks->MacroExpands(Identifier, MI);
-  
-  // If we started lexing a macro, enter the macro expansion body.
-
   // Remember where the token is expanded.
   SourceLocation ExpandLoc = Identifier.getLocation();
 
+  if (Callbacks) Callbacks->MacroExpands(Identifier, MI,
+                                         SourceRange(ExpandLoc, ExpansionEnd));
+  
+  // If we started lexing a macro, enter the macro expansion body.
+
   // If this macro expands to no tokens, don't bother to push it onto the
   // expansion stack, only to take it right back off.
   if (MI->getNumTokens() == 0) {
@@ -255,7 +260,6 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
       if (HadLeadingSpace) Identifier.setFlag(Token::LeadingSpace);
     }
     Identifier.setFlag(Token::LeadingEmptyMacro);
-    LastEmptyMacroExpansionLoc = ExpandLoc;
     ++NumFastMacroExpanded;
     return false;
 
@@ -284,8 +288,8 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
     // Update the tokens location to include both its expansion and physical
     // locations.
     SourceLocation Loc =
-      SourceMgr.createInstantiationLoc(Identifier.getLocation(), ExpandLoc,
-                                       ExpansionEnd,Identifier.getLength());
+      SourceMgr.createExpansionLoc(Identifier.getLocation(), ExpandLoc,
+                                   ExpansionEnd,Identifier.getLength());
     Identifier.setLocation(Loc);
 
     // If this is a disabled macro or #define X X, we must mark the result as
@@ -333,7 +337,7 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
   // ArgTokens - Build up a list of tokens that make up each argument.  Each
   // argument is separated by an EOF token.  Use a SmallVector so we can avoid
   // heap allocations in the common case.
-  llvm::SmallVector<Token, 64> ArgTokens;
+  SmallVector<Token, 64> ArgTokens;
 
   unsigned NumActuals = 0;
   while (Tok.isNot(tok::r_paren)) {
@@ -352,13 +356,6 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
       // an argument value in a macro could expand to ',' or '(' or ')'.
       LexUnexpandedToken(Tok);
 
-      if (Tok.is(tok::code_completion)) {
-        if (CodeComplete)
-          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),
-                                                  MI, NumActuals);
-        LexUnexpandedToken(Tok);
-      }
-      
       if (Tok.is(tok::eof) || Tok.is(tok::eod)) { // "#if f(<eof>" & "#if f(\n"
         Diag(MacroName, diag::err_unterm_macro_invoc);
         // Do not lose the EOF/EOD.  Return it to the client.
@@ -393,7 +390,15 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
         if (MacroInfo *MI = getMacroInfo(Tok.getIdentifierInfo()))
           if (!MI->isEnabled())
             Tok.setFlag(Token::DisableExpand);
+      } else if (Tok.is(tok::code_completion)) {
+        if (CodeComplete)
+          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),
+                                                  MI, NumActuals);
+        // Don't mark that we reached the code-completion point because the
+        // parser is going to handle the token and there will be another
+        // code-completion callback.
       }
+
       ArgTokens.push_back(Tok);
     }
 
@@ -416,8 +421,10 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
 
     // Empty arguments are standard in C99 and C++0x, and are supported as an extension in
     // other modes.
-    if (ArgTokens.size() == ArgTokenStart && !Features.C99 && !Features.CPlusPlus0x)
-      Diag(Tok, diag::ext_empty_fnmacro_arg);
+    if (ArgTokens.size() == ArgTokenStart && !Features.C99)
+      Diag(Tok, Features.CPlusPlus0x ?
+           diag::warn_cxx98_compat_empty_fnmacro_arg :
+           diag::ext_empty_fnmacro_arg);
 
     // Add a marker EOF token to the end of the token list for this argument.
     Token EOFTok;
@@ -487,8 +494,7 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
     return 0;
   }
 
-  return MacroArgs::create(MI, ArgTokens.data(), ArgTokens.size(),
-                           isVarargsElided, *this);
+  return MacroArgs::create(MI, ArgTokens, isVarargsElided, *this);
 }
 
 /// \brief Keeps macro expanded tokens for TokenLexers.
@@ -497,7 +503,7 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
 /// going to lex in the cache and when it finishes the tokens are removed
 /// from the end of the cache.
 Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer,
-                                              llvm::ArrayRef<Token> tokens) {
+                                              ArrayRef<Token> tokens) {
   assert(tokLexer);
   if (tokens.empty())
     return 0;
@@ -597,34 +603,48 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
            .Case("objc_arc", LangOpts.ObjCAutoRefCount)
            .Case("objc_arc_weak", LangOpts.ObjCAutoRefCount && 
                  LangOpts.ObjCRuntimeHasWeak)
+           .Case("objc_fixed_enum", LangOpts.ObjC2)
+           .Case("objc_instancetype", LangOpts.ObjC2)
            .Case("objc_nonfragile_abi", LangOpts.ObjCNonFragileABI)
            .Case("objc_weak_class", LangOpts.ObjCNonFragileABI)
            .Case("ownership_holds", true)
            .Case("ownership_returns", true)
            .Case("ownership_takes", true)
            // C1X features
+           .Case("c_alignas", LangOpts.C1X)
            .Case("c_generic_selections", LangOpts.C1X)
            .Case("c_static_assert", LangOpts.C1X)
            // C++0x features
            .Case("cxx_access_control_sfinae", LangOpts.CPlusPlus0x)
            .Case("cxx_alias_templates", LangOpts.CPlusPlus0x)
+           .Case("cxx_alignas", LangOpts.CPlusPlus0x)
            .Case("cxx_attributes", LangOpts.CPlusPlus0x)
            .Case("cxx_auto_type", LangOpts.CPlusPlus0x)
+         //.Case("cxx_constexpr", false);
            .Case("cxx_decltype", LangOpts.CPlusPlus0x)
            .Case("cxx_default_function_template_args", LangOpts.CPlusPlus0x)
            .Case("cxx_delegating_constructors", LangOpts.CPlusPlus0x)
            .Case("cxx_deleted_functions", LangOpts.CPlusPlus0x)
+           .Case("cxx_explicit_conversions", LangOpts.CPlusPlus0x)
+         //.Case("cxx_generalized_initializers", LangOpts.CPlusPlus0x)
+           .Case("cxx_implicit_moves", LangOpts.CPlusPlus0x)
+         //.Case("cxx_inheriting_constructors", false)
            .Case("cxx_inline_namespaces", LangOpts.CPlusPlus0x)
          //.Case("cxx_lambdas", false)
+           .Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus0x)
            .Case("cxx_noexcept", LangOpts.CPlusPlus0x)
            .Case("cxx_nullptr", LangOpts.CPlusPlus0x)
            .Case("cxx_override_control", LangOpts.CPlusPlus0x)
            .Case("cxx_range_for", LangOpts.CPlusPlus0x)
+         //.Case("cxx_raw_string_literals", false)
            .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus0x)
            .Case("cxx_rvalue_references", LangOpts.CPlusPlus0x)
            .Case("cxx_strong_enums", LangOpts.CPlusPlus0x)
            .Case("cxx_static_assert", LangOpts.CPlusPlus0x)
            .Case("cxx_trailing_return", LangOpts.CPlusPlus0x)
+         //.Case("cxx_unicode_literals", false)
+         //.Case("cxx_unrestricted_unions", false)
+         //.Case("cxx_user_literals", false)
            .Case("cxx_variadic_templates", LangOpts.CPlusPlus0x)
            // Type traits
            .Case("has_nothrow_assign", LangOpts.CPlusPlus)
@@ -639,16 +659,31 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
            .Case("is_base_of", LangOpts.CPlusPlus)
            .Case("is_class", LangOpts.CPlusPlus)
            .Case("is_convertible_to", LangOpts.CPlusPlus)
-           .Case("is_empty", LangOpts.CPlusPlus)
+            // __is_empty is available only if the horrible
+            // "struct __is_empty" parsing hack hasn't been needed in this
+            // translation unit. If it has, __is_empty reverts to a normal
+            // identifier and __has_feature(is_empty) evaluates false.
+           .Case("is_empty", 
+                 LangOpts.CPlusPlus && 
+                 PP.getIdentifierInfo("__is_empty")->getTokenID()
+                                                            != tok::identifier)
            .Case("is_enum", LangOpts.CPlusPlus)
            .Case("is_literal", LangOpts.CPlusPlus)
            .Case("is_standard_layout", LangOpts.CPlusPlus)
-           .Case("is_pod", LangOpts.CPlusPlus)
+           // __is_pod is available only if the horrible
+           // "struct __is_pod" parsing hack hasn't been needed in this
+           // translation unit. If it has, __is_pod reverts to a normal
+           // identifier and __has_feature(is_pod) evaluates false.
+           .Case("is_pod", 
+                 LangOpts.CPlusPlus && 
+                 PP.getIdentifierInfo("__is_pod")->getTokenID()
+                                                            != tok::identifier)
            .Case("is_polymorphic", LangOpts.CPlusPlus)
            .Case("is_trivial", LangOpts.CPlusPlus)
            .Case("is_trivially_copyable", LangOpts.CPlusPlus)
            .Case("is_union", LangOpts.CPlusPlus)
            .Case("tls", PP.getTargetInfo().isTLSSupported())
+           .Case("underlying_type", LangOpts.CPlusPlus)
            .Default(false);
 }
 
@@ -661,7 +696,8 @@ static bool HasExtension(const Preprocessor &PP, const IdentifierInfo *II) {
 
   // If the use of an extension results in an error diagnostic, extensions are
   // effectively unavailable, so just return false here.
-  if (PP.getDiagnostics().getExtensionHandlingBehavior()==Diagnostic::Ext_Error)
+  if (PP.getDiagnostics().getExtensionHandlingBehavior() ==
+      DiagnosticsEngine::Ext_Error)
     return false;
 
   const LangOptions &LangOpts = PP.getLangOptions();
@@ -670,12 +706,16 @@ static bool HasExtension(const Preprocessor &PP, const IdentifierInfo *II) {
   // must be less restrictive than HasFeature's.
   return llvm::StringSwitch<bool>(II->getName())
            // C1X features supported by other languages as extensions.
+           .Case("c_alignas", true)
            .Case("c_generic_selections", true)
            .Case("c_static_assert", true)
            // C++0x features supported by other languages as extensions.
            .Case("cxx_deleted_functions", LangOpts.CPlusPlus)
+           .Case("cxx_explicit_conversions", LangOpts.CPlusPlus)
            .Case("cxx_inline_namespaces", LangOpts.CPlusPlus)
+           .Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus)
            .Case("cxx_override_control", LangOpts.CPlusPlus)
+           .Case("cxx_range_for", LangOpts.CPlusPlus)
            .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus)
            .Case("cxx_rvalue_references", LangOpts.CPlusPlus)
            .Default(false);
@@ -714,7 +754,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
 
   // Reserve a buffer to get the spelling.
   llvm::SmallString<128> FilenameBuffer;
-  llvm::StringRef Filename;
+  StringRef Filename;
   SourceLocation EndLoc;
   
   switch (Tok.getKind()) {
@@ -753,7 +793,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
   // Search include directories.
   const DirectoryLookup *CurDir;
   const FileEntry *File =
-      PP.LookupFile(Filename, isAngled, LookupFrom, CurDir, NULL, NULL);
+      PP.LookupFile(Filename, isAngled, LookupFrom, CurDir, NULL, NULL, NULL);
 
   // Get the result value.  Result = true means the file exists.
   bool Result = File != 0;
@@ -837,7 +877,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     // can matter for a function-like macro that expands to contain __LINE__.
     // Skip down through expansion points until we find a file loc for the
     // end of the expansion history.
-    Loc = SourceMgr.getInstantiationRange(Loc).second;
+    Loc = SourceMgr.getExpansionRange(Loc).second;
     PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
 
     // __LINE__ expands to a simple numeric value.
@@ -874,18 +914,18 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
       ComputeDATE_TIME(DATELoc, TIMELoc, *this);
     Tok.setKind(tok::string_literal);
     Tok.setLength(strlen("\"Mmm dd yyyy\""));
-    Tok.setLocation(SourceMgr.createInstantiationLoc(DATELoc, Tok.getLocation(),
-                                                     Tok.getLocation(),
-                                                     Tok.getLength()));
+    Tok.setLocation(SourceMgr.createExpansionLoc(DATELoc, Tok.getLocation(),
+                                                 Tok.getLocation(),
+                                                 Tok.getLength()));
     return;
   } else if (II == Ident__TIME__) {
     if (!TIMELoc.isValid())
       ComputeDATE_TIME(DATELoc, TIMELoc, *this);
     Tok.setKind(tok::string_literal);
     Tok.setLength(strlen("\"hh:mm:ss\""));
-    Tok.setLocation(SourceMgr.createInstantiationLoc(TIMELoc, Tok.getLocation(),
-                                                     Tok.getLocation(),
-                                                     Tok.getLength()));
+    Tok.setLocation(SourceMgr.createExpansionLoc(TIMELoc, Tok.getLocation(),
+                                                 Tok.getLocation(),
+                                                 Tok.getLength()));
     return;
   } else if (II == Ident__INCLUDE_LEVEL__) {
     // Compute the presumed include depth of this token.  This can be affected
@@ -923,7 +963,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
       Result = "??? ??? ?? ??:??:?? ????\n";
     }
     // Surround the string with " and strip the trailing newline.
-    OS << '"' << llvm::StringRef(Result, strlen(Result)-1) << '"';
+    OS << '"' << StringRef(Result, strlen(Result)-1) << '"';
     Tok.setKind(tok::string_literal);
   } else if (II == Ident__COUNTER__) {
     // __COUNTER__ expands to a simple numeric value.
@@ -983,10 +1023,78 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
       Value = EvaluateHasIncludeNext(Tok, II, *this);
     OS << (int)Value;
     Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__has_warning) {
+    // The argument should be a parenthesized string literal.
+    // The argument to these builtins should be a parenthesized identifier.
+    SourceLocation StartLoc = Tok.getLocation();    
+    bool IsValid = false;
+    bool Value = false;
+    // Read the '('.
+    Lex(Tok);
+    do {
+      if (Tok.is(tok::l_paren)) {      
+        // Read the string.
+        Lex(Tok);
+      
+        // We need at least one string literal.
+        if (!Tok.is(tok::string_literal)) {
+          StartLoc = Tok.getLocation();
+          IsValid = false;
+          // Eat tokens until ')'.
+          do Lex(Tok); while (!(Tok.is(tok::r_paren) || Tok.is(tok::eod)));
+          break;
+        }
+        
+        // String concatenation allows multiple strings, which can even come
+        // from macro expansion.
+        SmallVector<Token, 4> StrToks;
+        while (Tok.is(tok::string_literal)) {
+          StrToks.push_back(Tok);
+          LexUnexpandedToken(Tok);
+        }
+        
+        // Is the end a ')'?
+        if (!(IsValid = Tok.is(tok::r_paren)))
+          break;
+        
+        // Concatenate and parse the strings.
+        StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
+        assert(Literal.isAscii() && "Didn't allow wide strings in");
+        if (Literal.hadError)
+          break;
+        if (Literal.Pascal) {
+          Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+          break;
+        }
+        
+        StringRef WarningName(Literal.GetString());
+        
+        if (WarningName.size() < 3 || WarningName[0] != '-' ||
+            WarningName[1] != 'W') {
+          Diag(StrToks[0].getLocation(), diag::warn_has_warning_invalid_option);
+          break;
+        }
+        
+        // Finally, check if the warning flags maps to a diagnostic group.
+        // We construct a SmallVector here to talk to getDiagnosticIDs().
+        // Although we don't use the result, this isn't a hot path, and not
+        // worth special casing.
+        llvm::SmallVector<diag::kind, 10> Diags;
+        Value = !getDiagnostics().getDiagnosticIDs()->
+          getDiagnosticsInGroup(WarningName.substr(2), Diags);
+      }
+    } while (false);
+    
+    if (!IsValid)
+      Diag(StartLoc, diag::err_warning_check_malformed);
+
+    OS << (int)Value;
+    Tok.setKind(tok::numeric_constant);
   } else {
-    assert(0 && "Unknown identifier!");
+    llvm_unreachable("Unknown identifier!");
   }
-  CreateString(OS.str().data(), OS.str().size(), Tok, Tok.getLocation());
+  CreateString(OS.str().data(), OS.str().size(), Tok,
+               Tok.getLocation(), Tok.getLocation());
 }
 
 void Preprocessor::markMacroAsUsed(MacroInfo *MI) {
diff --git a/contrib/llvm/tools/clang/lib/Lex/PTHLexer.cpp b/contrib/llvm/tools/clang/lib/Lex/PTHLexer.cpp
index e5ef0fdf20eb..e0c4cf0c16c8 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PTHLexer.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PTHLexer.cpp
@@ -73,7 +73,7 @@ void PTHLexer::Lex(Token& Tok) {
   Tok.setKind(TKind);
   Tok.setFlag(TFlags);
   assert(!LexingRawMode);
-  Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset));
+  Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
   Tok.setLength(Len);
 
   // Handle identifiers.
@@ -147,7 +147,7 @@ bool PTHLexer::LexEndOfFile(Token &Result) {
 
   // If we are in a #if directive, emit an error.
   while (!ConditionalStack.empty()) {
-    if (!PP->isCodeCompletionFile(FileStartLoc))
+    if (PP->getCodeCompletionFileLoc() != FileStartLoc)
       PP->Diag(ConditionalStack.back().IfLoc,
                diag::err_pp_unterminated_conditional);
     ConditionalStack.pop_back();
@@ -297,7 +297,7 @@ SourceLocation PTHLexer::getSourceLocation() {
   // NOTE: This is a virtual function; hence it is defined out-of-line.
   const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
   uint32_t Offset = ReadLE32(OffsetPtr);
-  return FileStartLoc.getFileLocWithOffset(Offset);
+  return FileStartLoc.getLocWithOffset(Offset);
 }
 
 //===----------------------------------------------------------------------===//
@@ -380,7 +380,7 @@ class PTHStringLookupTrait {
   }
 
   static unsigned ComputeHash(const internal_key_type& a) {
-    return llvm::HashString(llvm::StringRef(a.first, a.second));
+    return llvm::HashString(StringRef(a.first, a.second));
   }
 
   // This hopefully will just get inlined and removed by the optimizer.
@@ -431,11 +431,12 @@ PTHManager::~PTHManager() {
   free(PerIDCache);
 }
 
-static void InvalidPTH(Diagnostic &Diags, const char *Msg) {
-  Diags.Report(Diags.getCustomDiagID(Diagnostic::Error, Msg));
+static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
+  Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, Msg));
 }
 
-PTHManager *PTHManager::Create(const std::string &file, Diagnostic &Diags) {
+PTHManager *PTHManager::Create(const std::string &file,
+                               DiagnosticsEngine &Diags) {
   // Memory map the PTH file.
   llvm::OwningPtr<llvm::MemoryBuffer> File;
 
@@ -572,10 +573,10 @@ IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
   return II;
 }
 
-IdentifierInfo* PTHManager::get(llvm::StringRef Name) {
+IdentifierInfo* PTHManager::get(StringRef Name) {
   PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
   // Double check our assumption that the last character isn't '\0'.
-  assert(Name.empty() || Name.data()[Name.size()-1] != '\0');
+  assert(Name.empty() || Name.back() != '\0');
   PTHStringIdLookup::iterator I = SL.find(std::make_pair(Name.data(),
                                                          Name.size()));
   if (I == SL.end()) // No identifier found?
diff --git a/contrib/llvm/tools/clang/lib/Lex/Pragma.cpp b/contrib/llvm/tools/clang/lib/Lex/Pragma.cpp
index e6b28c13317b..f6532c2175a1 100644
--- a/contrib/llvm/tools/clang/lib/Lex/Pragma.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/Pragma.cpp
@@ -54,11 +54,11 @@ PragmaNamespace::~PragmaNamespace() {
 /// specified name.  If not, return the handler for the null identifier if it
 /// exists, otherwise return null.  If IgnoreNull is true (the default) then
 /// the null handler isn't returned on failure to match.
-PragmaHandler *PragmaNamespace::FindHandler(llvm::StringRef Name,
+PragmaHandler *PragmaNamespace::FindHandler(StringRef Name,
                                             bool IgnoreNull) const {
   if (PragmaHandler *Handler = Handlers.lookup(Name))
     return Handler;
-  return IgnoreNull ? 0 : Handlers.lookup(llvm::StringRef());
+  return IgnoreNull ? 0 : Handlers.lookup(StringRef());
 }
 
 void PragmaNamespace::AddPragma(PragmaHandler *Handler) {
@@ -85,7 +85,7 @@ void PragmaNamespace::HandlePragma(Preprocessor &PP,
   // Get the handler for this token.  If there is no handler, ignore the pragma.
   PragmaHandler *Handler
     = FindHandler(Tok.getIdentifierInfo() ? Tok.getIdentifierInfo()->getName()
-                                          : llvm::StringRef(),
+                                          : StringRef(),
                   /*IgnoreNull=*/false);
   if (Handler == 0) {
     PP.Diag(Tok, diag::warn_pragma_ignored);
@@ -210,7 +210,7 @@ void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
   }
 
   // Get the tokens enclosed within the __pragma(), as well as the final ')'.
-  llvm::SmallVector<Token, 32> PragmaToks;
+  SmallVector<Token, 32> PragmaToks;
   int NumParens = 0;
   Lex(Tok);
   while (Tok.isNot(tok::eof)) {
@@ -353,7 +353,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
   // Reserve a buffer to get the spelling.
   llvm::SmallString<128> FilenameBuffer;
   bool Invalid = false;
-  llvm::StringRef Filename = getSpelling(FilenameTok, FilenameBuffer, &Invalid);
+  StringRef Filename = getSpelling(FilenameTok, FilenameBuffer, &Invalid);
   if (Invalid)
     return;
 
@@ -366,9 +366,11 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
 
   // Search include directories for this file.
   const DirectoryLookup *CurDir;
-  const FileEntry *File = LookupFile(Filename, isAngled, 0, CurDir, NULL, NULL);
+  const FileEntry *File = LookupFile(Filename, isAngled, 0, CurDir, NULL, NULL,
+                                     NULL);
   if (File == 0) {
-    Diag(FilenameTok, diag::warn_pp_file_not_found) << Filename;
+    if (!SuppressIncludeNotFoundError)
+      Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
     return;
   }
 
@@ -436,7 +438,7 @@ void Preprocessor::HandlePragmaComment(Token &Tok) {
     // String concatenation allows multiple strings, which can even come from
     // macro expansion.
     // "foo " "bar" "Baz"
-    llvm::SmallVector<Token, 4> StrToks;
+    SmallVector<Token, 4> StrToks;
     while (Tok.is(tok::string_literal)) {
       StrToks.push_back(Tok);
       Lex(Tok);
@@ -444,7 +446,7 @@ void Preprocessor::HandlePragmaComment(Token &Tok) {
 
     // Concatenate and parse the strings.
     StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
-    assert(!Literal.AnyWide && "Didn't allow wide strings in");
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
     if (Literal.hadError)
       return;
     if (Literal.Pascal) {
@@ -512,7 +514,7 @@ void Preprocessor::HandlePragmaMessage(Token &Tok) {
   // String concatenation allows multiple strings, which can even come from
   // macro expansion.
   // "foo " "bar" "Baz"
-  llvm::SmallVector<Token, 4> StrToks;
+  SmallVector<Token, 4> StrToks;
   while (Tok.is(tok::string_literal)) {
     StrToks.push_back(Tok);
     Lex(Tok);
@@ -520,7 +522,7 @@ void Preprocessor::HandlePragmaMessage(Token &Tok) {
 
   // Concatenate and parse the strings.
   StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
-  assert(!Literal.AnyWide && "Didn't allow wide strings in");
+  assert(Literal.isAscii() && "Didn't allow wide strings in");
   if (Literal.hadError)
     return;
   if (Literal.Pascal) {
@@ -528,7 +530,7 @@ void Preprocessor::HandlePragmaMessage(Token &Tok) {
     return;
   }
 
-  llvm::StringRef MessageString(Literal.GetString());
+  StringRef MessageString(Literal.GetString());
 
   if (ExpectClosingParen) {
     if (Tok.isNot(tok::r_paren)) {
@@ -662,7 +664,7 @@ void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
 /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
 /// If 'Namespace' is non-null, then it is a token required to exist on the
 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
-void Preprocessor::AddPragmaHandler(llvm::StringRef Namespace,
+void Preprocessor::AddPragmaHandler(StringRef Namespace,
                                     PragmaHandler *Handler) {
   PragmaNamespace *InsertNS = PragmaHandlers;
 
@@ -693,7 +695,7 @@ void Preprocessor::AddPragmaHandler(llvm::StringRef Namespace,
 /// preprocessor. If \arg Namespace is non-null, then it should be the
 /// namespace that \arg Handler was added to. It is an error to remove
 /// a handler that has not been registered.
-void Preprocessor::RemovePragmaHandler(llvm::StringRef Namespace,
+void Preprocessor::RemovePragmaHandler(StringRef Namespace,
                                        PragmaHandler *Handler) {
   PragmaNamespace *NS = PragmaHandlers;
 
@@ -802,7 +804,7 @@ struct PragmaDebugHandler : public PragmaHandler {
     IdentifierInfo *II = Tok.getIdentifierInfo();
 
     if (II->isStr("assert")) {
-      assert(0 && "This is an assertion!");
+      llvm_unreachable("This is an assertion!");
     } else if (II->isStr("crash")) {
       *(volatile int*) 0x11 = 0;
     } else if (II->isStr("llvm_fatal_error")) {
@@ -889,7 +891,7 @@ struct PragmaDiagnosticHandler : public PragmaHandler {
     // String concatenation allows multiple strings, which can even come from
     // macro expansion.
     // "foo " "bar" "Baz"
-    llvm::SmallVector<Token, 4> StrToks;
+    SmallVector<Token, 4> StrToks;
     while (Tok.is(tok::string_literal)) {
       StrToks.push_back(Tok);
       PP.LexUnexpandedToken(Tok);
@@ -902,7 +904,7 @@ struct PragmaDiagnosticHandler : public PragmaHandler {
 
     // Concatenate and parse the strings.
     StringLiteralParser Literal(&StrToks[0], StrToks.size(), PP);
-    assert(!Literal.AnyWide && "Didn't allow wide strings in");
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
     if (Literal.hadError)
       return;
     if (Literal.Pascal) {
@@ -910,7 +912,7 @@ struct PragmaDiagnosticHandler : public PragmaHandler {
       return;
     }
 
-    llvm::StringRef WarningName(Literal.GetString());
+    StringRef WarningName(Literal.GetString());
 
     if (WarningName.size() < 3 || WarningName[0] != '-' ||
         WarningName[1] != 'W') {
@@ -1003,6 +1005,60 @@ struct PragmaSTDC_UnknownHandler : public PragmaHandler {
   }
 };
 
+/// PragmaARCCFCodeAuditedHandler - 
+///   #pragma clang arc_cf_code_audited begin/end
+struct PragmaARCCFCodeAuditedHandler : public PragmaHandler {
+  PragmaARCCFCodeAuditedHandler() : PragmaHandler("arc_cf_code_audited") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &NameTok) {
+    SourceLocation Loc = NameTok.getLocation();
+    bool IsBegin;
+
+    Token Tok;
+
+    // Lex the 'begin' or 'end'.
+    PP.LexUnexpandedToken(Tok);
+    const IdentifierInfo *BeginEnd = Tok.getIdentifierInfo();
+    if (BeginEnd && BeginEnd->isStr("begin")) {
+      IsBegin = true;
+    } else if (BeginEnd && BeginEnd->isStr("end")) {
+      IsBegin = false;
+    } else {
+      PP.Diag(Tok.getLocation(), diag::err_pp_arc_cf_code_audited_syntax);
+      return;
+    }
+
+    // Verify that this is followed by EOD.
+    PP.LexUnexpandedToken(Tok);
+    if (Tok.isNot(tok::eod))
+      PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma";
+
+    // The start location of the active audit.
+    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedLoc();
+
+    // The start location we want after processing this.
+    SourceLocation NewLoc;
+
+    if (IsBegin) {
+      // Complain about attempts to re-enter an audit.
+      if (BeginLoc.isValid()) {
+        PP.Diag(Loc, diag::err_pp_double_begin_of_arc_cf_code_audited);
+        PP.Diag(BeginLoc, diag::note_pragma_entered_here);
+      }
+      NewLoc = Loc;
+    } else {
+      // Complain about attempts to leave an audit that doesn't exist.
+      if (!BeginLoc.isValid()) {
+        PP.Diag(Loc, diag::err_pp_unmatched_end_of_arc_cf_code_audited);
+        return;
+      }
+      NewLoc = SourceLocation();
+    }
+
+    PP.setPragmaARCCFCodeAuditedLoc(NewLoc);
+  }
+};
+
 }  // end anonymous namespace
 
 
@@ -1026,13 +1082,14 @@ void Preprocessor::RegisterBuiltinPragmas() {
   AddPragmaHandler("clang", new PragmaDebugHandler());
   AddPragmaHandler("clang", new PragmaDependencyHandler());
   AddPragmaHandler("clang", new PragmaDiagnosticHandler("clang"));
+  AddPragmaHandler("clang", new PragmaARCCFCodeAuditedHandler());
 
   AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_UnknownHandler());
 
   // MS extensions.
-  if (Features.Microsoft) {
+  if (Features.MicrosoftExt) {
     AddPragmaHandler(new PragmaCommentHandler());
   }
 }
diff --git a/contrib/llvm/tools/clang/lib/Lex/PreprocessingRecord.cpp b/contrib/llvm/tools/clang/lib/Lex/PreprocessingRecord.cpp
index 9f93ab04502a..2816609d5f8f 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PreprocessingRecord.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PreprocessingRecord.cpp
@@ -14,8 +14,8 @@
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Token.h"
-#include "clang/Basic/IdentifierTable.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Capacity.h"
 
 using namespace clang;
 
@@ -24,7 +24,7 @@ ExternalPreprocessingRecordSource::~ExternalPreprocessingRecordSource() { }
 
 InclusionDirective::InclusionDirective(PreprocessingRecord &PPRec,
                                        InclusionKind Kind, 
-                                       llvm::StringRef FileName, 
+                                       StringRef FileName, 
                                        bool InQuotes, const FileEntry *File, 
                                        SourceRange Range)
   : PreprocessingDirective(InclusionDirectiveKind, Range), 
@@ -34,116 +34,254 @@ InclusionDirective::InclusionDirective(PreprocessingRecord &PPRec,
     = (char*)PPRec.Allocate(FileName.size() + 1, llvm::alignOf<char>());
   memcpy(Memory, FileName.data(), FileName.size());
   Memory[FileName.size()] = 0;
-  this->FileName = llvm::StringRef(Memory, FileName.size());
+  this->FileName = StringRef(Memory, FileName.size());
 }
 
-void PreprocessingRecord::MaybeLoadPreallocatedEntities() const {
-  if (!ExternalSource || LoadedPreallocatedEntities)
-    return;
-  
-  LoadedPreallocatedEntities = true;
-  ExternalSource->ReadPreprocessedEntities();
-}
-
-PreprocessingRecord::PreprocessingRecord(bool IncludeNestedMacroExpansions)
-  : IncludeNestedMacroExpansions(IncludeNestedMacroExpansions),
-    ExternalSource(0), NumPreallocatedEntities(0), 
-    LoadedPreallocatedEntities(false)
+PreprocessingRecord::PreprocessingRecord(SourceManager &SM,
+                                         bool IncludeNestedMacroExpansions)
+  : SourceMgr(SM), IncludeNestedMacroExpansions(IncludeNestedMacroExpansions),
+    ExternalSource(0)
 {
 }
 
-PreprocessingRecord::iterator 
-PreprocessingRecord::begin(bool OnlyLocalEntities) {
-  if (OnlyLocalEntities)
-    return PreprocessedEntities.begin() + NumPreallocatedEntities;
-  
-  MaybeLoadPreallocatedEntities();
-  return PreprocessedEntities.begin();
+/// \brief Returns a pair of [Begin, End) iterators of preprocessed entities
+/// that source range \arg R encompasses.
+std::pair<PreprocessingRecord::iterator, PreprocessingRecord::iterator>
+PreprocessingRecord::getPreprocessedEntitiesInRange(SourceRange Range) {
+  if (Range.isInvalid())
+    return std::make_pair(iterator(this, 0), iterator(this, 0));
+  assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
+
+  std::pair<unsigned, unsigned>
+    Local = findLocalPreprocessedEntitiesInRange(Range);
+
+  // Check if range spans local entities.
+  if (!ExternalSource || SourceMgr.isLocalSourceLocation(Range.getBegin()))
+    return std::make_pair(iterator(this, Local.first),
+                          iterator(this, Local.second));
+
+  std::pair<unsigned, unsigned>
+    Loaded = ExternalSource->findPreprocessedEntitiesInRange(Range);
+
+  // Check if range spans local entities.
+  if (Loaded.first == Loaded.second)
+    return std::make_pair(iterator(this, Local.first),
+                          iterator(this, Local.second));
+
+  unsigned TotalLoaded = LoadedPreprocessedEntities.size();
+
+  // Check if range spans loaded entities.
+  if (Local.first == Local.second)
+    return std::make_pair(iterator(this, int(Loaded.first)-TotalLoaded),
+                          iterator(this, int(Loaded.second)-TotalLoaded));
+
+  // Range spands loaded and local entities.
+  return std::make_pair(iterator(this, int(Loaded.first)-TotalLoaded),
+                        iterator(this, Local.second));
 }
 
-PreprocessingRecord::iterator PreprocessingRecord::end(bool OnlyLocalEntities) {
-  if (!OnlyLocalEntities)
-    MaybeLoadPreallocatedEntities();
-  
-  return PreprocessedEntities.end();
+std::pair<unsigned, unsigned>
+PreprocessingRecord::findLocalPreprocessedEntitiesInRange(
+                                                      SourceRange Range) const {
+  if (Range.isInvalid())
+    return std::make_pair(0,0);
+  assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
+
+  unsigned Begin = findBeginLocalPreprocessedEntity(Range.getBegin());
+  unsigned End = findEndLocalPreprocessedEntity(Range.getEnd());
+  return std::make_pair(Begin, End);
 }
 
-PreprocessingRecord::const_iterator 
-PreprocessingRecord::begin(bool OnlyLocalEntities) const {
-  if (OnlyLocalEntities)
-    return PreprocessedEntities.begin() + NumPreallocatedEntities;
-  
-  MaybeLoadPreallocatedEntities();
-  return PreprocessedEntities.begin();
+namespace {
+
+template <SourceLocation (SourceRange::*getRangeLoc)() const>
+struct PPEntityComp {
+  const SourceManager &SM;
+
+  explicit PPEntityComp(const SourceManager &SM) : SM(SM) { }
+
+  bool operator()(PreprocessedEntity *L, PreprocessedEntity *R) const {
+    SourceLocation LHS = getLoc(L);
+    SourceLocation RHS = getLoc(R);
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  bool operator()(PreprocessedEntity *L, SourceLocation RHS) const {
+    SourceLocation LHS = getLoc(L);
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  bool operator()(SourceLocation LHS, PreprocessedEntity *R) const {
+    SourceLocation RHS = getLoc(R);
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  SourceLocation getLoc(PreprocessedEntity *PPE) const {
+    SourceRange Range = PPE->getSourceRange();
+    return (Range.*getRangeLoc)();
+  }
+};
+
 }
 
-PreprocessingRecord::const_iterator 
-PreprocessingRecord::end(bool OnlyLocalEntities) const {
-  if (!OnlyLocalEntities)
-    MaybeLoadPreallocatedEntities();
-  
-  return PreprocessedEntities.end();
+unsigned PreprocessingRecord::findBeginLocalPreprocessedEntity(
+                                                     SourceLocation Loc) const {
+  if (SourceMgr.isLoadedSourceLocation(Loc))
+    return 0;
+
+  size_t Count = PreprocessedEntities.size();
+  size_t Half;
+  std::vector<PreprocessedEntity *>::const_iterator
+    First = PreprocessedEntities.begin();
+  std::vector<PreprocessedEntity *>::const_iterator I;
+
+  // Do a binary search manually instead of using std::lower_bound because
+  // The end locations of entities may be unordered (when a macro expansion
+  // is inside another macro argument), but for this case it is not important
+  // whether we get the first macro expansion or its containing macro.
+  while (Count > 0) {
+    Half = Count/2;
+    I = First;
+    std::advance(I, Half);
+    if (SourceMgr.isBeforeInTranslationUnit((*I)->getSourceRange().getEnd(),
+                                            Loc)){
+      First = I;
+      ++First;
+      Count = Count - Half - 1;
+    } else
+      Count = Half;
+  }
+
+  return First - PreprocessedEntities.begin();
+}
+
+unsigned PreprocessingRecord::findEndLocalPreprocessedEntity(
+                                                     SourceLocation Loc) const {
+  if (SourceMgr.isLoadedSourceLocation(Loc))
+    return 0;
+
+  std::vector<PreprocessedEntity *>::const_iterator
+  I = std::upper_bound(PreprocessedEntities.begin(),
+                       PreprocessedEntities.end(),
+                       Loc,
+                       PPEntityComp<&SourceRange::getBegin>(SourceMgr));
+  return I - PreprocessedEntities.begin();
 }
 
 void PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
-  PreprocessedEntities.push_back(Entity);
+  assert(Entity);
+  SourceLocation BeginLoc = Entity->getSourceRange().getBegin();
+  
+  // Check normal case, this entity begin location is after the previous one.
+  if (PreprocessedEntities.empty() ||
+      !SourceMgr.isBeforeInTranslationUnit(BeginLoc,
+                   PreprocessedEntities.back()->getSourceRange().getBegin())) {
+    PreprocessedEntities.push_back(Entity);
+    return;
+  }
+
+  // The entity's location is not after the previous one; this can happen rarely
+  // e.g. with "#include MACRO".
+  // Iterate the entities vector in reverse until we find the right place to
+  // insert the new entity.
+  for (std::vector<PreprocessedEntity *>::iterator
+         RI = PreprocessedEntities.end(), Begin = PreprocessedEntities.begin();
+       RI != Begin; --RI) {
+    std::vector<PreprocessedEntity *>::iterator I = RI;
+    --I;
+    if (!SourceMgr.isBeforeInTranslationUnit(BeginLoc,
+                                           (*I)->getSourceRange().getBegin())) {
+      PreprocessedEntities.insert(RI, Entity);
+      return;
+    }
+  }
 }
 
 void PreprocessingRecord::SetExternalSource(
-                                    ExternalPreprocessingRecordSource &Source,
-                                            unsigned NumPreallocatedEntities) {
+                                    ExternalPreprocessingRecordSource &Source) {
   assert(!ExternalSource &&
          "Preprocessing record already has an external source");
   ExternalSource = &Source;
-  this->NumPreallocatedEntities = NumPreallocatedEntities;
-  PreprocessedEntities.insert(PreprocessedEntities.begin(), 
-                              NumPreallocatedEntities, 0);
 }
 
-void PreprocessingRecord::SetPreallocatedEntity(unsigned Index, 
-                                                PreprocessedEntity *Entity) {
-  assert(Index < NumPreallocatedEntities &&"Out-of-bounds preallocated entity");
-  PreprocessedEntities[Index] = Entity;
+unsigned PreprocessingRecord::allocateLoadedEntities(unsigned NumEntities) {
+  unsigned Result = LoadedPreprocessedEntities.size();
+  LoadedPreprocessedEntities.resize(LoadedPreprocessedEntities.size() 
+                                    + NumEntities);
+  return Result;
 }
 
-void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro, 
-                                                  MacroDefinition *MD) {
-  MacroDefinitions[Macro] = MD;
+void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro,
+                                                  PPEntityID PPID) {
+  MacroDefinitions[Macro] = PPID;
+}
+
+/// \brief Retrieve the preprocessed entity at the given ID.
+PreprocessedEntity *PreprocessingRecord::getPreprocessedEntity(PPEntityID PPID){
+  if (PPID < 0) {
+    assert(unsigned(-PPID-1) < LoadedPreprocessedEntities.size() &&
+           "Out-of bounds loaded preprocessed entity");
+    return getLoadedPreprocessedEntity(LoadedPreprocessedEntities.size()+PPID);
+  }
+  assert(unsigned(PPID) < PreprocessedEntities.size() &&
+         "Out-of bounds local preprocessed entity");
+  return PreprocessedEntities[PPID];
+}
+
+/// \brief Retrieve the loaded preprocessed entity at the given index.
+PreprocessedEntity *
+PreprocessingRecord::getLoadedPreprocessedEntity(unsigned Index) {
+  assert(Index < LoadedPreprocessedEntities.size() && 
+         "Out-of bounds loaded preprocessed entity");
+  assert(ExternalSource && "No external source to load from");
+  PreprocessedEntity *&Entity = LoadedPreprocessedEntities[Index];
+  if (!Entity) {
+    Entity = ExternalSource->ReadPreprocessedEntity(Index);
+    if (!Entity) // Failed to load.
+      Entity = new (*this)
+         PreprocessedEntity(PreprocessedEntity::InvalidKind, SourceRange());
+  }
+  return Entity;
 }
 
 MacroDefinition *PreprocessingRecord::findMacroDefinition(const MacroInfo *MI) {
-  llvm::DenseMap<const MacroInfo *, MacroDefinition *>::iterator Pos
+  llvm::DenseMap<const MacroInfo *, PPEntityID>::iterator Pos
     = MacroDefinitions.find(MI);
   if (Pos == MacroDefinitions.end())
     return 0;
   
-  return Pos->second;
+  PreprocessedEntity *Entity = getPreprocessedEntity(Pos->second);
+  if (Entity->isInvalid())
+    return 0;
+  return cast<MacroDefinition>(Entity);
 }
 
-void PreprocessingRecord::MacroExpands(const Token &Id, const MacroInfo* MI) {
+void PreprocessingRecord::MacroExpands(const Token &Id, const MacroInfo* MI,
+                                       SourceRange Range) {
   if (!IncludeNestedMacroExpansions && Id.getLocation().isMacroID())
     return;
 
-  if (MacroDefinition *Def = findMacroDefinition(MI))
-    PreprocessedEntities.push_back(
-                       new (*this) MacroExpansion(Id.getIdentifierInfo(),
-                                                  Id.getLocation(), Def));
+  if (MI->isBuiltinMacro())
+    addPreprocessedEntity(
+                      new (*this) MacroExpansion(Id.getIdentifierInfo(),Range));
+  else if (MacroDefinition *Def = findMacroDefinition(MI))
+    addPreprocessedEntity(
+                       new (*this) MacroExpansion(Def, Range));
 }
 
 void PreprocessingRecord::MacroDefined(const Token &Id,
                                        const MacroInfo *MI) {
   SourceRange R(MI->getDefinitionLoc(), MI->getDefinitionEndLoc());
   MacroDefinition *Def
-      = new (*this) MacroDefinition(Id.getIdentifierInfo(),
-                                    MI->getDefinitionLoc(),
-                                    R);
-  MacroDefinitions[MI] = Def;
-  PreprocessedEntities.push_back(Def);
+      = new (*this) MacroDefinition(Id.getIdentifierInfo(), R);
+  addPreprocessedEntity(Def);
+  MacroDefinitions[MI] = getPPEntityID(PreprocessedEntities.size()-1,
+                                       /*isLoaded=*/false);
 }
 
 void PreprocessingRecord::MacroUndefined(const Token &Id,
                                          const MacroInfo *MI) {
-  llvm::DenseMap<const MacroInfo *, MacroDefinition *>::iterator Pos
+  llvm::DenseMap<const MacroInfo *, PPEntityID>::iterator Pos
     = MacroDefinitions.find(MI);
   if (Pos != MacroDefinitions.end())
     MacroDefinitions.erase(Pos);
@@ -152,12 +290,12 @@ void PreprocessingRecord::MacroUndefined(const Token &Id,
 void PreprocessingRecord::InclusionDirective(
     SourceLocation HashLoc,
     const clang::Token &IncludeTok,
-    llvm::StringRef FileName,
+    StringRef FileName,
     bool IsAngled,
     const FileEntry *File,
     clang::SourceLocation EndLoc,
-    llvm::StringRef SearchPath,
-    llvm::StringRef RelativePath) {
+    StringRef SearchPath,
+    StringRef RelativePath) {
   InclusionDirective::InclusionKind Kind = InclusionDirective::Include;
   
   switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
@@ -185,5 +323,12 @@ void PreprocessingRecord::InclusionDirective(
   clang::InclusionDirective *ID
     = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled, 
                                             File, SourceRange(HashLoc, EndLoc));
-  PreprocessedEntities.push_back(ID);
+  addPreprocessedEntity(ID);
+}
+
+size_t PreprocessingRecord::getTotalMemory() const {
+  return BumpAlloc.getTotalMemory()
+    + llvm::capacity_in_bytes(MacroDefinitions)
+    + llvm::capacity_in_bytes(PreprocessedEntities)
+    + llvm::capacity_in_bytes(LoadedPreprocessedEntities);
 }
diff --git a/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp b/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp
index e7aa286a16bf..31662ad0c116 100644
--- a/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp
@@ -35,6 +35,7 @@
 #include "clang/Lex/ScratchBuffer.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -42,83 +43,39 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Capacity.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
 
-Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
-                           const TargetInfo &target, SourceManager &SM,
-                           HeaderSearch &Headers,
+Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
+                           const TargetInfo *target, SourceManager &SM,
+                           HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
                            IdentifierInfoLookup* IILookup,
-                           bool OwnsHeaders)
+                           bool OwnsHeaders,
+                           bool DelayInitialization)
   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
-    SourceMgr(SM),
-    HeaderInfo(Headers), ExternalSource(0),
-    Identifiers(opts, IILookup), BuiltinInfo(Target), CodeComplete(0),
-    CodeCompletionFile(0), SkipMainFilePreamble(0, true), CurPPLexer(0), 
-    CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0), MIChainHead(0),
-    MICache(0) {
-  ScratchBuf = new ScratchBuffer(SourceMgr);
-  CounterValue = 0; // __COUNTER__ starts at 0.
+    SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
+    ExternalSource(0), 
+    Identifiers(opts, IILookup), CodeComplete(0),
+    CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
+    SkipMainFilePreamble(0, true), CurPPLexer(0), 
+    CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 
+    Record(0), MIChainHead(0), MICache(0) 
+{
   OwnsHeaderSearch = OwnsHeaders;
-
-  // Clear stats.
-  NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
-  NumIf = NumElse = NumEndif = 0;
-  NumEnteredSourceFiles = 0;
-  NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
-  NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
-  MaxIncludeStackDepth = 0;
-  NumSkipped = 0;
-
-  // Default to discarding comments.
-  KeepComments = false;
-  KeepMacroComments = false;
-
-  // Macro expansion is enabled.
-  DisableMacroExpansion = false;
-  InMacroArgs = false;
-  NumCachedTokenLexers = 0;
-
-  CachedLexPos = 0;
-
-  // We haven't read anything from the external source.
-  ReadMacrosFromExternalSource = false;
-
-  // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
-  // This gets unpoisoned where it is allowed.
-  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
-  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
-
-  // Initialize the pragma handlers.
-  PragmaHandlers = new PragmaNamespace(llvm::StringRef());
-  RegisterBuiltinPragmas();
-
-  // Initialize builtin macros like __LINE__ and friends.
-  RegisterBuiltinMacros();
-
-  if(Features.Borland) {
-    Ident__exception_info        = getIdentifierInfo("_exception_info");
-    Ident___exception_info       = getIdentifierInfo("__exception_info");
-    Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
-    Ident__exception_code        = getIdentifierInfo("_exception_code");
-    Ident___exception_code       = getIdentifierInfo("__exception_code");
-    Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
-    Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
-    Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
-    Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
-  } else {
-    Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
-    Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
-    Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
+  
+  if (!DelayInitialization) {
+    assert(Target && "Must provide target information for PP initialization");
+    Initialize(*Target);
   }
-
 }
 
 Preprocessor::~Preprocessor() {
   assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
-  assert(MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty() &&
+  assert(((MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty()) ||
+          isCodeCompletionReached()) &&
          "Preprocessor::HandleEndOfTokenLexer should have cleared those");
 
   while (!IncludeMacroStack.empty()) {
@@ -152,6 +109,71 @@ Preprocessor::~Preprocessor() {
   delete Callbacks;
 }
 
+void Preprocessor::Initialize(const TargetInfo &Target) {
+  assert((!this->Target || this->Target == &Target) &&
+         "Invalid override of target information");
+  this->Target = &Target;
+  
+  // Initialize information about built-ins.
+  BuiltinInfo.InitializeTarget(Target);
+  
+  ScratchBuf = new ScratchBuffer(SourceMgr);
+  CounterValue = 0; // __COUNTER__ starts at 0.
+  
+  // Clear stats.
+  NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
+  NumIf = NumElse = NumEndif = 0;
+  NumEnteredSourceFiles = 0;
+  NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
+  NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
+  MaxIncludeStackDepth = 0;
+  NumSkipped = 0;
+  
+  // Default to discarding comments.
+  KeepComments = false;
+  KeepMacroComments = false;
+  SuppressIncludeNotFoundError = false;
+  AutoModuleImport = false;
+  
+  // Macro expansion is enabled.
+  DisableMacroExpansion = false;
+  InMacroArgs = false;
+  NumCachedTokenLexers = 0;
+  
+  CachedLexPos = 0;
+  
+  // We haven't read anything from the external source.
+  ReadMacrosFromExternalSource = false;
+  
+  // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
+  // This gets unpoisoned where it is allowed.
+  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
+  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
+  
+  // Initialize the pragma handlers.
+  PragmaHandlers = new PragmaNamespace(StringRef());
+  RegisterBuiltinPragmas();
+  
+  // Initialize builtin macros like __LINE__ and friends.
+  RegisterBuiltinMacros();
+  
+  if(Features.Borland) {
+    Ident__exception_info        = getIdentifierInfo("_exception_info");
+    Ident___exception_info       = getIdentifierInfo("__exception_info");
+    Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
+    Ident__exception_code        = getIdentifierInfo("_exception_code");
+    Ident___exception_code       = getIdentifierInfo("__exception_code");
+    Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
+    Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
+    Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
+    Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
+  } else {
+    Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
+    Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
+    Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
+  } 
+}
+
 void Preprocessor::setPTHManager(PTHManager* pm) {
   PTH.reset(pm);
   FileMgr.addStatCache(PTH->createStatCache());
@@ -172,7 +194,7 @@ void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
     llvm::errs() << " [ExpandDisabled]";
   if (Tok.needsCleaning()) {
     const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
-    llvm::errs() << " [UnClean='" << llvm::StringRef(Start, Tok.getLength())
+    llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
                  << "']";
   }
 
@@ -228,7 +250,13 @@ Preprocessor::macro_begin(bool IncludeExternalMacros) const {
 }
 
 size_t Preprocessor::getTotalMemory() const {
-  return BP.getTotalMemory() + MacroExpandedTokens.capacity()*sizeof(Token);
+  return BP.getTotalMemory()
+    + llvm::capacity_in_bytes(MacroExpandedTokens)
+    + Predefines.capacity() /* Predefines buffer. */
+    + llvm::capacity_in_bytes(Macros)
+    + llvm::capacity_in_bytes(PragmaPushMacroInfo)
+    + llvm::capacity_in_bytes(PoisonReasons)
+    + llvm::capacity_in_bytes(CommentHandlers);
 }
 
 Preprocessor::macro_iterator
@@ -243,16 +271,14 @@ Preprocessor::macro_end(bool IncludeExternalMacros) const {
 }
 
 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
-                                          unsigned TruncateAtLine,
-                                          unsigned TruncateAtColumn) {
+                                          unsigned CompleteLine,
+                                          unsigned CompleteColumn) {
+  assert(File);
+  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
+  assert(!CodeCompletionFile && "Already set");
+
   using llvm::MemoryBuffer;
 
-  CodeCompletionFile = File;
-
-  // Okay to clear out the code-completion point by passing NULL.
-  if (!CodeCompletionFile)
-    return false;
-
   // Load the actual file's contents.
   bool Invalid = false;
   const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
@@ -261,7 +287,7 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
 
   // Find the byte position of the truncation point.
   const char *Position = Buffer->getBufferStart();
-  for (unsigned Line = 1; Line < TruncateAtLine; ++Line) {
+  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
     for (; *Position; ++Position) {
       if (*Position != '\r' && *Position != '\n')
         continue;
@@ -275,38 +301,37 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
     }
   }
 
-  Position += TruncateAtColumn - 1;
+  Position += CompleteColumn - 1;
 
-  // Truncate the buffer.
+  // Insert '\0' at the code-completion point.
   if (Position < Buffer->getBufferEnd()) {
-    llvm::StringRef Data(Buffer->getBufferStart(),
-                         Position-Buffer->getBufferStart());
-    MemoryBuffer *TruncatedBuffer
-      = MemoryBuffer::getMemBufferCopy(Data, Buffer->getBufferIdentifier());
-    SourceMgr.overrideFileContents(File, TruncatedBuffer);
+    CodeCompletionFile = File;
+    CodeCompletionOffset = Position - Buffer->getBufferStart();
+
+    MemoryBuffer *NewBuffer =
+        MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
+                                            Buffer->getBufferIdentifier());
+    char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
+    char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
+    *NewPos = '\0';
+    std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
+    SourceMgr.overrideFileContents(File, NewBuffer);
   }
 
   return false;
 }
 
-bool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const {
-  return CodeCompletionFile && FileLoc.isFileID() &&
-    SourceMgr.getFileEntryForID(SourceMgr.getFileID(FileLoc))
-      == CodeCompletionFile;
-}
-
 void Preprocessor::CodeCompleteNaturalLanguage() {
-  SetCodeCompletionPoint(0, 0, 0);
-  getDiagnostics().setSuppressAllDiagnostics(true);
   if (CodeComplete)
     CodeComplete->CodeCompleteNaturalLanguage();
+  setCodeCompletionReached();
 }
 
 /// getSpelling - This method is used to get the spelling of a token into a
 /// SmallVector. Note that the returned StringRef may not point to the
 /// supplied buffer if a copy can be avoided.
-llvm::StringRef Preprocessor::getSpelling(const Token &Tok,
-                                          llvm::SmallVectorImpl<char> &Buffer,
+StringRef Preprocessor::getSpelling(const Token &Tok,
+                                          SmallVectorImpl<char> &Buffer,
                                           bool *Invalid) const {
   // NOTE: this has to be checked *before* testing for an IdentifierInfo.
   if (Tok.isNot(tok::raw_identifier)) {
@@ -321,22 +346,23 @@ llvm::StringRef Preprocessor::getSpelling(const Token &Tok,
 
   const char *Ptr = Buffer.data();
   unsigned Len = getSpelling(Tok, Ptr, Invalid);
-  return llvm::StringRef(Ptr, Len);
+  return StringRef(Ptr, Len);
 }
 
 /// CreateString - Plop the specified string into a scratch buffer and return a
 /// location for it.  If specified, the source location provides a source
 /// location for the token.
 void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
-                                SourceLocation ExpansionLoc) {
+                                SourceLocation ExpansionLocStart,
+                                SourceLocation ExpansionLocEnd) {
   Tok.setLength(Len);
 
   const char *DestPtr;
   SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
 
-  if (ExpansionLoc.isValid())
-    Loc = SourceMgr.createInstantiationLoc(Loc, ExpansionLoc,
-                                           ExpansionLoc, Len);
+  if (ExpansionLocStart.isValid())
+    Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
+                                       ExpansionLocEnd, Len);
   Tok.setLocation(Loc);
 
   // If this is a raw identifier or a literal token, set the pointer data.
@@ -407,12 +433,12 @@ IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
   IdentifierInfo *II;
   if (!Identifier.needsCleaning()) {
     // No cleaning needed, just use the characters from the lexed buffer.
-    II = getIdentifierInfo(llvm::StringRef(Identifier.getRawIdentifierData(),
+    II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(),
                                            Identifier.getLength()));
   } else {
     // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
     llvm::SmallString<64> IdentifierBuffer;
-    llvm::StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
+    StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
     II = getIdentifierInfo(CleanedStr);
   }
 
@@ -487,6 +513,17 @@ void Preprocessor::HandleIdentifier(Token &Identifier) {
     }
   }
 
+  // If this identifier is a keyword in C++11, produce a warning. Don't warn if
+  // we're not considering macro expansion, since this identifier might be the
+  // name of a macro.
+  // FIXME: This warning is disabled in cases where it shouldn't be, like
+  //   "#define constexpr constexpr", "int constexpr;"
+  if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) {
+    Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
+    // Don't diagnose this keyword again in this translation unit.
+    II.setIsCXX11CompatKeyword(false);
+  }
+
   // C++ 2.11p2: If this is an alternative representation of a C++ operator,
   // then we act as if it is the actual operator and not the textual
   // representation of it.
@@ -499,6 +536,44 @@ void Preprocessor::HandleIdentifier(Token &Identifier) {
   // like "#define TY typeof", "TY(1) x".
   if (II.isExtensionToken() && !DisableMacroExpansion)
     Diag(Identifier, diag::ext_token_used);
+  
+  // If this is the '__import_module__' keyword, note that the next token
+  // indicates a module name.
+  if (II.getTokenID() == tok::kw___import_module__ &&
+      !InMacroArgs && !DisableMacroExpansion) {
+    ModuleImportLoc = Identifier.getLocation();
+    CurLexerKind = CLK_LexAfterModuleImport;
+  }
+}
+
+/// \brief Lex a token following the __import_module__ keyword.
+void Preprocessor::LexAfterModuleImport(Token &Result) {
+  // Figure out what kind of lexer we actually have.
+  if (CurLexer)
+    CurLexerKind = CLK_Lexer;
+  else if (CurPTHLexer)
+    CurLexerKind = CLK_PTHLexer;
+  else if (CurTokenLexer)
+    CurLexerKind = CLK_TokenLexer;
+  else 
+    CurLexerKind = CLK_CachingLexer;
+  
+  // Lex the next token.
+  Lex(Result);
+
+  // The token sequence 
+  //
+  //   __import_module__ identifier
+  //
+  // indicates a module import directive. We already saw the __import_module__
+  // keyword, so now we're looking for the identifier.
+  if (Result.getKind() != tok::identifier)
+    return;
+  
+  // Load the module.
+  (void)TheModuleLoader.loadModule(ModuleImportLoc,
+                                   *Result.getIdentifierInfo(), 
+                                   Result.getLocation());
 }
 
 void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
@@ -529,6 +604,8 @@ bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
   return true;
 }
 
+ModuleLoader::~ModuleLoader() { }
+
 CommentHandler::~CommentHandler() { }
 
 CodeCompletionHandler::~CodeCompletionHandler() { }
@@ -538,6 +615,7 @@ void Preprocessor::createPreprocessingRecord(
   if (Record)
     return;
   
-  Record = new PreprocessingRecord(IncludeNestedMacroExpansions);
+  Record = new PreprocessingRecord(getSourceManager(),
+                                   IncludeNestedMacroExpansions);
   addPPCallbacks(Record);
 }
diff --git a/contrib/llvm/tools/clang/lib/Lex/PreprocessorLexer.cpp b/contrib/llvm/tools/clang/lib/Lex/PreprocessorLexer.cpp
index 808a81bd5e87..0da9ef5531e7 100644
--- a/contrib/llvm/tools/clang/lib/Lex/PreprocessorLexer.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/PreprocessorLexer.cpp
@@ -17,6 +17,14 @@
 #include "clang/Basic/SourceManager.h"
 using namespace clang;
 
+PreprocessorLexer::PreprocessorLexer(Preprocessor *pp, FileID fid)
+  : PP(pp), FID(fid), InitialNumSLocEntries(0),
+    ParsingPreprocessorDirective(false),
+    ParsingFilename(false), LexingRawMode(false) {
+  if (pp)
+    InitialNumSLocEntries = pp->getSourceManager().local_sloc_entry_size();
+}
+
 /// LexIncludeFilename - After the preprocessor has parsed a #include, lex and
 /// (potentially) macro expand the filename.
 void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) {
diff --git a/contrib/llvm/tools/clang/lib/Lex/ScratchBuffer.cpp b/contrib/llvm/tools/clang/lib/Lex/ScratchBuffer.cpp
index 0e98c1751985..3d363fa4b472 100644
--- a/contrib/llvm/tools/clang/lib/Lex/ScratchBuffer.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/ScratchBuffer.cpp
@@ -53,7 +53,7 @@ SourceLocation ScratchBuffer::getToken(const char *Buf, unsigned Len,
   // diagnostic points to one.
   CurBuffer[BytesUsed-1] = '\0';
 
-  return BufferStartLoc.getFileLocWithOffset(BytesUsed-Len-1);
+  return BufferStartLoc.getLocWithOffset(BytesUsed-Len-1);
 }
 
 void ScratchBuffer::AllocScratchBuffer(unsigned RequestLen) {
diff --git a/contrib/llvm/tools/clang/lib/Lex/TokenConcatenation.cpp b/contrib/llvm/tools/clang/lib/Lex/TokenConcatenation.cpp
index 3e9e8550313c..dc6d686d6cc1 100644
--- a/contrib/llvm/tools/clang/lib/Lex/TokenConcatenation.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/TokenConcatenation.cpp
@@ -17,42 +17,53 @@
 using namespace clang;
 
 
-/// StartsWithL - Return true if the spelling of this token starts with 'L'.
-bool TokenConcatenation::StartsWithL(const Token &Tok) const {
-  if (!Tok.needsCleaning()) {
-    SourceManager &SM = PP.getSourceManager();
-    return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())) == 'L';
+/// IsStringPrefix - Return true if Str is a string prefix.
+/// 'L', 'u', 'U', or 'u8'. Including raw versions.
+static bool IsStringPrefix(StringRef Str, bool CPlusPlus0x) {
+
+  if (Str[0] == 'L' ||
+      (CPlusPlus0x && (Str[0] == 'u' || Str[0] == 'U' || Str[0] == 'R'))) {
+
+    if (Str.size() == 1)
+      return true; // "L", "u", "U", and "R"
+
+    // Check for raw flavors. Need to make sure the first character wasn't
+    // already R. Need CPlusPlus0x check for "LR".
+    if (Str[1] == 'R' && Str[0] != 'R' && Str.size() == 2 && CPlusPlus0x)
+      return true; // "LR", "uR", "UR"
+
+    // Check for "u8" and "u8R"
+    if (Str[0] == 'u' && Str[1] == '8') {
+      if (Str.size() == 2) return true; // "u8"
+      if (Str.size() == 3 && Str[2] == 'R') return true; // "u8R"
+    }
   }
 
-  if (Tok.getLength() < 256) {
-    char Buffer[256];
-    const char *TokPtr = Buffer;
-    PP.getSpelling(Tok, TokPtr);
-    return TokPtr[0] == 'L';
-  }
-
-  return PP.getSpelling(Tok)[0] == 'L';
+  return false;
 }
 
-/// IsIdentifierL - Return true if the spelling of this token is literally
-/// 'L'.
-bool TokenConcatenation::IsIdentifierL(const Token &Tok) const {
+/// IsIdentifierStringPrefix - Return true if the spelling of the token
+/// is literally 'L', 'u', 'U', or 'u8'. Including raw versions.
+bool TokenConcatenation::IsIdentifierStringPrefix(const Token &Tok) const {
+  const LangOptions &LangOpts = PP.getLangOptions();
+
   if (!Tok.needsCleaning()) {
-    if (Tok.getLength() != 1)
+    if (Tok.getLength() < 1 || Tok.getLength() > 3)
       return false;
     SourceManager &SM = PP.getSourceManager();
-    return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())) == 'L';
+    const char *Ptr = SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation()));
+    return IsStringPrefix(StringRef(Ptr, Tok.getLength()),
+                          LangOpts.CPlusPlus0x);
   }
 
   if (Tok.getLength() < 256) {
     char Buffer[256];
     const char *TokPtr = Buffer;
-    if (PP.getSpelling(Tok, TokPtr) != 1)
-      return false;
-    return TokPtr[0] == 'L';
+    unsigned length = PP.getSpelling(Tok, TokPtr);
+    return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus0x);
   }
 
-  return PP.getSpelling(Tok) == "L";
+  return IsStringPrefix(StringRef(PP.getSpelling(Tok)), LangOpts.CPlusPlus0x);
 }
 
 TokenConcatenation::TokenConcatenation(Preprocessor &pp) : PP(pp) {
@@ -132,7 +143,7 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
   // source.  If they were, it must be okay to stick them together: if there
   // were an issue, the tokens would have been lexed differently.
   if (PrevTok.getLocation().isFileID() && Tok.getLocation().isFileID() &&
-      PrevTok.getLocation().getFileLocWithOffset(PrevTok.getLength()) ==
+      PrevTok.getLocation().getLocWithOffset(PrevTok.getLength()) ==
         Tok.getLocation())
     return false;
 
@@ -179,24 +190,19 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
     if (Tok.is(tok::numeric_constant))
       return GetFirstChar(PP, Tok) != '.';
 
-    if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) /* ||
-     Tok.is(tok::wide_char_literal)*/)
+    if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) ||
+        Tok.is(tok::utf8_string_literal) || Tok.is(tok::utf16_string_literal) ||
+        Tok.is(tok::utf32_string_literal) || Tok.is(tok::wide_char_constant) ||
+        Tok.is(tok::utf16_char_constant) || Tok.is(tok::utf32_char_constant))
       return true;
 
     // If this isn't identifier + string, we're done.
     if (Tok.isNot(tok::char_constant) && Tok.isNot(tok::string_literal))
       return false;
 
-    // FIXME: need a wide_char_constant!
-
-    // If the string was a wide string L"foo" or wide char L'f', it would
-    // concat with the previous identifier into fooL"bar".  Avoid this.
-    if (StartsWithL(Tok))
-      return true;
-
     // Otherwise, this is a narrow character or string.  If the *identifier*
-    // is a literal 'L', avoid pasting L "foo" -> L"foo".
-    return IsIdentifierL(PrevTok);
+    // is a literal 'L', 'u8', 'u' or 'U', avoid pasting L "foo" -> L"foo".
+    return IsIdentifierStringPrefix(PrevTok);
   case tok::numeric_constant:
     return isalnum(FirstChar) || Tok.is(tok::numeric_constant) ||
     FirstChar == '+' || FirstChar == '-' || FirstChar == '.';
diff --git a/contrib/llvm/tools/clang/lib/Lex/TokenLexer.cpp b/contrib/llvm/tools/clang/lib/Lex/TokenLexer.cpp
index 8ff82f160033..a58054490fcd 100644
--- a/contrib/llvm/tools/clang/lib/Lex/TokenLexer.cpp
+++ b/contrib/llvm/tools/clang/lib/Lex/TokenLexer.cpp
@@ -43,7 +43,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) {
   MacroExpansionStart = SourceLocation();
 
   SourceManager &SM = PP.getSourceManager();
-  MacroStartSLocOffset = SM.getNextOffset();
+  MacroStartSLocOffset = SM.getNextLocalOffset();
 
   if (NumTokens > 0) {
     assert(Tokens[0].getLocation().isValid());
@@ -55,12 +55,12 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) {
     // definition. Tokens that get lexed directly from the definition will
     // have their locations pointing inside this chunk. This is to avoid
     // creating separate source location entries for each token.
-    SourceLocation macroStart = SM.getInstantiationLoc(Tokens[0].getLocation());
-    MacroDefStartInfo = SM.getDecomposedLoc(macroStart);
-    MacroExpansionStart = SM.createInstantiationLoc(macroStart,
-                                              ExpandLocStart,
-                                              ExpandLocEnd,
-                                              Macro->getDefinitionLength(SM));
+    MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
+    MacroDefLength = Macro->getDefinitionLength(SM);
+    MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
+                                                ExpandLocStart,
+                                                ExpandLocEnd,
+                                                MacroDefLength);
   }
 
   // If this is a function-like macro, expand the arguments and change
@@ -121,9 +121,8 @@ void TokenLexer::destroy() {
 /// Expand the arguments of a function-like macro so that we can quickly
 /// return preexpanded tokens from Tokens.
 void TokenLexer::ExpandFunctionArguments() {
-  SourceManager &SM = PP.getSourceManager();
 
-  llvm::SmallVector<Token, 128> ResultToks;
+  SmallVector<Token, 128> ResultToks;
 
   // Loop through 'Tokens', expanding them into ResultToks.  Keep
   // track of whether we change anything.  If not, no need to keep them.  If so,
@@ -144,19 +143,22 @@ void TokenLexer::ExpandFunctionArguments() {
       int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo());
       assert(ArgNo != -1 && "Token following # is not an argument?");
 
-      SourceLocation hashInstLoc;
-      if(ExpandLocStart.isValid()) {
-        hashInstLoc = getMacroExpansionLocation(CurTok.getLocation());
-        assert(hashInstLoc.isValid() && "Expected '#' to come from definition");
-      }
+      SourceLocation ExpansionLocStart =
+          getExpansionLocForMacroDefLoc(CurTok.getLocation());
+      SourceLocation ExpansionLocEnd =
+          getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
 
       Token Res;
       if (CurTok.is(tok::hash))  // Stringify
-        Res = ActualArgs->getStringifiedArgument(ArgNo, PP, hashInstLoc);
+        Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
+                                                 ExpansionLocStart,
+                                                 ExpansionLocEnd);
       else {
         // 'charify': don't bother caching these.
         Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
-                                           PP, true, hashInstLoc);
+                                           PP, true,
+                                           ExpansionLocStart,
+                                           ExpansionLocEnd);
       }
 
       // The stringified/charified string leading space flag gets set to match
@@ -225,16 +227,9 @@ void TokenLexer::ExpandFunctionArguments() {
         }
 
         if(ExpandLocStart.isValid()) {
-          SourceLocation curInst =
-              getMacroExpansionLocation(CurTok.getLocation());
-          assert(curInst.isValid() &&
-                 "Expected arg identifier to come from definition");
-          for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
-            Token &Tok = ResultToks[i];
-            Tok.setLocation(SM.createMacroArgInstantiationLoc(Tok.getLocation(),
-                                                              curInst,
-                                                              Tok.getLength()));
-          }
+          updateLocForMacroArgTokens(CurTok.getLocation(),
+                                     ResultToks.begin()+FirstResult,
+                                     ResultToks.end());
         }
 
         // If any tokens were substituted from the argument, the whitespace
@@ -282,17 +277,8 @@ void TokenLexer::ExpandFunctionArguments() {
       }
 
       if (ExpandLocStart.isValid()) {
-        SourceLocation curInst =
-            getMacroExpansionLocation(CurTok.getLocation());
-        assert(curInst.isValid() &&
-               "Expected arg identifier to come from definition");
-        for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
-               i != e; ++i) {
-          Token &Tok = ResultToks[i];
-          Tok.setLocation(SM.createMacroArgInstantiationLoc(Tok.getLocation(),
-                                                            curInst,
-                                                            Tok.getLength()));
-        }
+        updateLocForMacroArgTokens(CurTok.getLocation(),
+                                   ResultToks.end()-NumToks, ResultToks.end());
       }
 
       // If this token (the macro argument) was supposed to get leading
@@ -417,18 +403,15 @@ void TokenLexer::Lex(Token &Tok) {
   // that captures all of this.
   if (ExpandLocStart.isValid() &&   // Don't do this for token streams.
       // Check that the token's location was not already set properly.
-      SM.isBeforeInSourceLocationOffset(Tok.getLocation(),
-                                        MacroStartSLocOffset)) {
+      SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
     SourceLocation instLoc;
     if (Tok.is(tok::comment)) {
-      instLoc = SM.createInstantiationLoc(Tok.getLocation(),
-                                          ExpandLocStart,
-                                          ExpandLocEnd,
-                                          Tok.getLength());
+      instLoc = SM.createExpansionLoc(Tok.getLocation(),
+                                      ExpandLocStart,
+                                      ExpandLocEnd,
+                                      Tok.getLength());
     } else {
-      instLoc = getMacroExpansionLocation(Tok.getLocation());
-      assert(instLoc.isValid() &&
-             "Location for token not coming from definition was not set!");
+      instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
     }
 
     Tok.setLocation(instLoc);
@@ -469,6 +452,7 @@ void TokenLexer::Lex(Token &Tok) {
 bool TokenLexer::PasteTokens(Token &Tok) {
   llvm::SmallString<128> Buffer;
   const char *ResultTokStrPtr = 0;
+  SourceLocation StartLoc = Tok.getLocation();
   SourceLocation PasteOpLoc;
   do {
     // Consume the ## operator.
@@ -562,7 +546,7 @@ bool TokenLexer::PasteTokens(Token &Tok) {
       if (isInvalid) {
         // Test for the Microsoft extension of /##/ turning into // here on the
         // error path.
-        if (PP.getLangOptions().Microsoft && Tok.is(tok::slash) &&
+        if (PP.getLangOptions().MicrosoftExt && Tok.is(tok::slash) &&
             RHS.is(tok::slash)) {
           HandleMicrosoftCommentPaste(Tok);
           return true;
@@ -574,14 +558,13 @@ bool TokenLexer::PasteTokens(Token &Tok) {
           // information so that the user knows where it came from.
           SourceManager &SM = PP.getSourceManager();
           SourceLocation Loc =
-            SM.createInstantiationLoc(PasteOpLoc, ExpandLocStart,
-                                      ExpandLocEnd, 2);
+            SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
           // If we're in microsoft extensions mode, downgrade this from a hard
           // error to a warning that defaults to an error.  This allows
           // disabling it.
           PP.Diag(Loc,
-                  PP.getLangOptions().Microsoft ? diag::err_pp_bad_paste_ms 
-                                                : diag::err_pp_bad_paste)
+                  PP.getLangOptions().MicrosoftExt ? diag::err_pp_bad_paste_ms 
+                                                   : diag::err_pp_bad_paste)
             << Buffer.str();
         }
 
@@ -604,23 +587,20 @@ bool TokenLexer::PasteTokens(Token &Tok) {
     Tok = Result;
   } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash));
 
+  SourceLocation EndLoc = Tokens[CurToken - 1].getLocation();
+
   // The token's current location indicate where the token was lexed from.  We
   // need this information to compute the spelling of the token, but any
   // diagnostics for the expanded token should appear as if the token was
-  // expanded from the (##) operator. Pull this information together into
+  // expanded from the full ## expression. Pull this information together into
   // a new SourceLocation that captures all of this.
-  if (ExpandLocStart.isValid()) {
-    SourceManager &SM = PP.getSourceManager();
-    SourceLocation pasteLocInst =
-        getMacroExpansionLocation(PasteOpLoc);
-    assert(pasteLocInst.isValid() &&
-           "Expected '##' to come from definition");
-
-    Tok.setLocation(SM.createInstantiationLoc(Tok.getLocation(),
-                                              pasteLocInst,
-                                              pasteLocInst,
-                                              Tok.getLength()));
-  }
+  SourceManager &SM = PP.getSourceManager();
+  if (StartLoc.isFileID())
+    StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
+  if (EndLoc.isFileID())
+    EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
+  Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc,
+                                        Tok.getLength()));
 
   // Now that we got the result token, it will be subject to expansion.  Since
   // token pasting re-lexes the result token in raw mode, identifier information
@@ -666,22 +646,111 @@ void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) {
   PP.HandleMicrosoftCommentPaste(Tok);
 }
 
-/// \brief If \arg loc is a FileID and points inside the current macro
+/// \brief If \arg loc is a file ID and points inside the current macro
 /// definition, returns the appropriate source location pointing at the
-/// macro expansion source location entry.
-SourceLocation TokenLexer::getMacroExpansionLocation(SourceLocation loc) const {
+/// macro expansion source location entry, otherwise it returns an invalid
+/// SourceLocation.
+SourceLocation
+TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
   assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
          "Not appropriate for token streams");
-  assert(loc.isValid());
+  assert(loc.isValid() && loc.isFileID());
   
   SourceManager &SM = PP.getSourceManager();
-  unsigned relativeOffset;
-  if (loc.isFileID() &&
-      SM.isInFileID(loc,
-                    MacroDefStartInfo.first, MacroDefStartInfo.second,
-                    Macro->getDefinitionLength(SM), &relativeOffset)) {
-    return MacroExpansionStart.getFileLocWithOffset(relativeOffset);
+  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
+         "Expected loc to come from the macro definition");
+
+  unsigned relativeOffset = 0;
+  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
+  return MacroExpansionStart.getLocWithOffset(relativeOffset);
+}
+
+/// \brief Finds the tokens that are consecutive (from the same FileID)
+/// creates a single SLocEntry, and assigns SourceLocations to each token that
+/// point to that SLocEntry. e.g for
+///   assert(foo == bar);
+/// There will be a single SLocEntry for the "foo == bar" chunk and locations
+/// for the 'foo', '==', 'bar' tokens will point inside that chunk.
+///
+/// \arg begin_tokens will be updated to a position past all the found
+/// consecutive tokens.
+static void updateConsecutiveMacroArgTokens(SourceManager &SM,
+                                            SourceLocation InstLoc,
+                                            Token *&begin_tokens,
+                                            Token * end_tokens) {
+  assert(begin_tokens < end_tokens);
+
+  SourceLocation FirstLoc = begin_tokens->getLocation();
+  SourceLocation CurLoc = FirstLoc;
+
+  // Compare the source location offset of tokens and group together tokens that
+  // are close, even if their locations point to different FileIDs. e.g.
+  //
+  //  |bar    |  foo | cake   |  (3 tokens from 3 consecutive FileIDs)
+  //  ^                    ^
+  //  |bar       foo   cake|     (one SLocEntry chunk for all tokens)
+  //
+  // we can perform this "merge" since the token's spelling location depends
+  // on the relative offset.
+
+  Token *NextTok = begin_tokens + 1;
+  for (; NextTok < end_tokens; ++NextTok) {
+    int RelOffs;
+    if (!SM.isInSameSLocAddrSpace(CurLoc, NextTok->getLocation(), &RelOffs))
+      break; // Token from different local/loaded location.
+    // Check that token is not before the previous token or more than 50
+    // "characters" away.
+    if (RelOffs < 0 || RelOffs > 50)
+      break;
+    CurLoc = NextTok->getLocation();
   }
 
-  return SourceLocation();
+  // For the consecutive tokens, find the length of the SLocEntry to contain
+  // all of them.
+  Token &LastConsecutiveTok = *(NextTok-1);
+  int LastRelOffs = 0;
+  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
+                           &LastRelOffs);
+  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
+
+  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
+  SourceLocation Expansion =
+      SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
+
+  // Change the location of the tokens from the spelling location to the new
+  // expanded location.
+  for (; begin_tokens < NextTok; ++begin_tokens) {
+    Token &Tok = *begin_tokens;
+    int RelOffs = 0;
+    SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
+    Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
+  }
+}
+
+/// \brief Creates SLocEntries and updates the locations of macro argument
+/// tokens to their new expanded locations.
+///
+/// \param ArgIdDefLoc the location of the macro argument id inside the macro
+/// definition.
+/// \param Tokens the macro argument tokens to update.
+void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
+                                            Token *begin_tokens,
+                                            Token *end_tokens) {
+  SourceManager &SM = PP.getSourceManager();
+
+  SourceLocation InstLoc =
+      getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
+  
+  while (begin_tokens < end_tokens) {
+    // If there's only one token just create a SLocEntry for it.
+    if (end_tokens - begin_tokens == 1) {
+      Token &Tok = *begin_tokens;
+      Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
+                                                    InstLoc,
+                                                    Tok.getLength()));
+      return;
+    }
+
+    updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
+  }
 }
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseAST.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseAST.cpp
index 56584c96180b..fdd7d0f151a6 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseAST.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseAST.cpp
@@ -37,11 +37,11 @@ using namespace clang;
 ///
 void clang::ParseAST(Preprocessor &PP, ASTConsumer *Consumer,
                      ASTContext &Ctx, bool PrintStats,
-                     bool CompleteTranslationUnit,
+                     TranslationUnitKind TUKind,
                      CodeCompleteConsumer *CompletionConsumer) {
 
   llvm::OwningPtr<Sema> S(new Sema(PP, Ctx, *Consumer,
-                                   CompleteTranslationUnit,
+                                   TUKind,
                                    CompletionConsumer));
 
   // Recover resources if we crash before exiting this method.
@@ -93,7 +93,7 @@ void clang::ParseAST(Sema &S, bool PrintStats) {
     Consumer->HandleTopLevelDecl(ADecl.get());
   
   // Process any TopLevelDecls generated by #pragma weak.
-  for (llvm::SmallVector<Decl*,2>::iterator
+  for (SmallVector<Decl*,2>::iterator
        I = S.WeakTopLevelDecls().begin(),
        E = S.WeakTopLevelDecls().end(); I != E; ++I)
     Consumer->HandleTopLevelDecl(DeclGroupRef(*I));
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp
index f5c69981ca35..b387e9e551d4 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -21,7 +21,9 @@ using namespace clang;
 /// ParseCXXInlineMethodDef - We parsed and verified that the specified
 /// Declarator is a well formed C++ inline method definition. Now lex its body
 /// and store its tokens for parsing after the C++ class is complete.
-Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, ParsingDeclarator &D,
+Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS,
+                                      AttributeList *AccessAttrs,
+                                      ParsingDeclarator &D,
                                 const ParsedTemplateInfo &TemplateInfo,
                                 const VirtSpecifiers& VS, ExprResult& Init) {
   assert(D.isFunctionDeclarator() && "This isn't a function declarator!");
@@ -34,16 +36,25 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, ParsingDeclarator &D,
           TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->size() : 0);
 
   Decl *FnD;
+  D.setFunctionDefinition(true);
   if (D.getDeclSpec().isFriendSpecified())
-    // FIXME: Friend templates
-    FnD = Actions.ActOnFriendFunctionDecl(getCurScope(), D, true,
+    FnD = Actions.ActOnFriendFunctionDecl(getCurScope(), D,
                                           move(TemplateParams));
-  else { // FIXME: pass template information through
+  else {
     FnD = Actions.ActOnCXXMemberDeclarator(getCurScope(), AS, D,
                                            move(TemplateParams), 0, 
-                                           VS, Init.release(),
-                                           /*HasInit=*/false,
-                                           /*IsDefinition*/true);
+                                           VS, /*HasInit=*/false);
+    if (FnD) {
+      Actions.ProcessDeclAttributeList(getCurScope(), FnD, AccessAttrs,
+                                       false, true);
+      bool TypeSpecContainsAuto
+        = D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto;
+      if (Init.get())
+        Actions.AddInitializerToDecl(FnD, Init.get(), false, 
+                                     TypeSpecContainsAuto);
+      else
+        Actions.ActOnUninitializedDecl(FnD, TypeSpecContainsAuto);
+    }
   }
 
   HandleMemberFunctionDefaultArgs(D, FnD);
@@ -123,30 +134,24 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, ParsingDeclarator &D,
   CachedTokens &Toks = LM->Toks;
 
   tok::TokenKind kind = Tok.getKind();
-  // We may have a constructor initializer or function-try-block here.
-  if (kind == tok::colon || kind == tok::kw_try) {
-    // Consume everything up to (and including) the left brace.
-    if (!ConsumeAndStoreUntil(tok::l_brace, Toks)) {
-      // We didn't find the left-brace we expected after the
-      // constructor initializer.
-      if (Tok.is(tok::semi)) {
-        // We found a semicolon; complain, consume the semicolon, and
-        // don't try to parse this method later.
-        Diag(Tok.getLocation(), diag::err_expected_lbrace);
-        ConsumeAnyToken();
-        delete getCurrentClass().LateParsedDeclarations.back();
-        getCurrentClass().LateParsedDeclarations.pop_back();
-        return FnD;
-      }
+  // Consume everything up to (and including) the left brace of the
+  // function body.
+  if (ConsumeAndStoreFunctionPrologue(Toks)) {
+    // We didn't find the left-brace we expected after the
+    // constructor initializer.
+    if (Tok.is(tok::semi)) {
+      // We found a semicolon; complain, consume the semicolon, and
+      // don't try to parse this method later.
+      Diag(Tok.getLocation(), diag::err_expected_lbrace);
+      ConsumeAnyToken();
+      delete getCurrentClass().LateParsedDeclarations.back();
+      getCurrentClass().LateParsedDeclarations.pop_back();
+      return FnD;
     }
-
   } else {
-    // Begin by storing the '{' token.
-    Toks.push_back(Tok);
-    ConsumeBrace();
+    // Consume everything up to (and including) the matching right brace.
+    ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
   }
-  // Consume everything up to (and including) the matching right brace.
-  ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
 
   // If we're in a function-try-block, we need to store all the catch blocks.
   if (kind == tok::kw_try) {
@@ -398,6 +403,8 @@ void Parser::ParseLexedMethodDef(LexedMethod &LM) {
     if (!Tok.is(tok::l_brace)) {
       FnScope.Exit();
       Actions.ActOnFinishFunctionBody(LM.D, 0);
+      while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
+        ConsumeAnyToken();
       return;
     }
   } else
@@ -450,7 +457,7 @@ void Parser::ParseLexedMemberInitializers(ParsingClass &Class) {
 }
 
 void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
-  if (MI.Field->isInvalidDecl())
+  if (!MI.Field || MI.Field->isInvalidDecl())
     return;
 
   // Append the current token at the end of the new token stream so that it
@@ -551,8 +558,16 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2,
       ConsumeBrace();
       break;
 
+    case tok::code_completion:
+      Toks.push_back(Tok);
+      ConsumeCodeCompletionToken();
+      break;
+
     case tok::string_literal:
     case tok::wide_string_literal:
+    case tok::utf8_string_literal:
+    case tok::utf16_string_literal:
+    case tok::utf32_string_literal:
       Toks.push_back(Tok);
       ConsumeStringToken();
       break;
@@ -569,3 +584,70 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2,
     isFirstTokenConsumed = false;
   }
 }
+
+/// \brief Consume tokens and store them in the passed token container until
+/// we've passed the try keyword and constructor initializers and have consumed
+/// the opening brace of the function body. The opening brace will be consumed
+/// if and only if there was no error.
+///
+/// \return True on error. 
+bool Parser::ConsumeAndStoreFunctionPrologue(CachedTokens &Toks) {
+  if (Tok.is(tok::kw_try)) {
+    Toks.push_back(Tok);
+    ConsumeToken();
+  }
+  if (Tok.is(tok::colon)) {
+    // Initializers can contain braces too.
+    Toks.push_back(Tok);
+    ConsumeToken();
+
+    while (Tok.is(tok::identifier) || Tok.is(tok::coloncolon)) {
+      if (Tok.is(tok::eof) || Tok.is(tok::semi))
+        return true;
+
+      // Grab the identifier.
+      if (!ConsumeAndStoreUntil(tok::l_paren, tok::l_brace, Toks,
+                                /*StopAtSemi=*/true,
+                                /*ConsumeFinalToken=*/false))
+        return true;
+
+      tok::TokenKind kind = Tok.getKind();
+      Toks.push_back(Tok);
+      if (kind == tok::l_paren)
+        ConsumeParen();
+      else {
+        assert(kind == tok::l_brace && "Must be left paren or brace here.");
+        ConsumeBrace();
+        // In C++03, this has to be the start of the function body, which
+        // means the initializer is malformed.
+        if (!getLang().CPlusPlus0x)
+          return false;
+      }
+
+      // Grab the initializer
+      if (!ConsumeAndStoreUntil(kind == tok::l_paren ? tok::r_paren :
+                                                       tok::r_brace,
+                                Toks, /*StopAtSemi=*/true))
+        return true;
+
+      // Grab the separating comma, if any.
+      if (Tok.is(tok::comma)) {
+        Toks.push_back(Tok);
+        ConsumeToken();
+      }
+    }
+  }
+
+  // Grab any remaining garbage to be diagnosed later. We stop when we reach a
+  // brace: an opening one is the function body, while a closing one probably
+  // means we've reached the end of the class.
+  if (!ConsumeAndStoreUntil(tok::l_brace, tok::r_brace, Toks,
+                            /*StopAtSemi=*/true, /*ConsumeFinalToken=*/false))
+    return true;
+  if(Tok.isNot(tok::l_brace))
+    return true;
+
+  Toks.push_back(Tok);
+  ConsumeBrace();
+  return false;
+}
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp
index 0e17295858b9..2aa178f5ebee 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp
@@ -19,6 +19,7 @@
 #include "clang/Sema/PrettyDeclStackTrace.h"
 #include "RAIIObjectsForParser.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringSwitch.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -32,12 +33,10 @@ using namespace clang;
 /// Called type-id in C++.
 TypeResult Parser::ParseTypeName(SourceRange *Range,
                                  Declarator::TheContext Context,
-                                 ObjCDeclSpec *objcQuals,
                                  AccessSpecifier AS,
                                  Decl **OwnedType) {
   // Parse the common declaration-specifiers piece.
   DeclSpec DS(AttrFactory);
-  DS.setObjCQualifiers(objcQuals);
   ParseSpecifierQualifierList(DS, AS);
   if (OwnedType)
     *OwnedType = DS.isTypeSpecOwned() ? DS.getRepAsDecl() : 0;
@@ -54,6 +53,16 @@ TypeResult Parser::ParseTypeName(SourceRange *Range,
   return Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
 }
 
+
+/// isAttributeLateParsed - Return true if the attribute has arguments that
+/// require late parsing.
+static bool isAttributeLateParsed(const IdentifierInfo &II) {
+    return llvm::StringSwitch<bool>(II.getName())
+#include "clang/Parse/AttrLateParsed.inc"
+        .Default(false);
+}
+
+
 /// ParseGNUAttributes - Parse a non-empty attributes list.
 ///
 /// [GNU] attributes:
@@ -91,7 +100,8 @@ TypeResult Parser::ParseTypeName(SourceRange *Range,
 /// a pressing need to implement the 2 token lookahead.
 
 void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
-                                SourceLocation *endLoc) {
+                                SourceLocation *endLoc,
+                                LateParsedAttrList *LateAttrs) {
   assert(Tok.is(tok::kw___attribute) && "Not a GNU attribute list!");
 
   while (Tok.is(tok::kw___attribute)) {
@@ -108,7 +118,6 @@ void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
     // Parse the attribute-list. e.g. __attribute__(( weak, alias("__f") ))
     while (Tok.is(tok::identifier) || isDeclarationSpecifier() ||
            Tok.is(tok::comma)) {
-
       if (Tok.is(tok::comma)) {
         // allows for empty/non-empty attributes. ((__vector_size__(16),,,,))
         ConsumeToken();
@@ -118,112 +127,26 @@ void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
       IdentifierInfo *AttrName = Tok.getIdentifierInfo();
       SourceLocation AttrNameLoc = ConsumeToken();
 
-      // Availability attributes have their own grammar.
-      if (AttrName->isStr("availability"))
-        ParseAvailabilityAttribute(*AttrName, AttrNameLoc, attrs, endLoc);
-      // check if we have a "parameterized" attribute
-      else if (Tok.is(tok::l_paren)) {
-        ConsumeParen(); // ignore the left paren loc for now
+      if (Tok.is(tok::l_paren)) {
+        // handle "parameterized" attributes
+        if (LateAttrs && !ClassStack.empty() &&
+            isAttributeLateParsed(*AttrName)) {
+          // Delayed parsing is only available for attributes that occur
+          // in certain locations within a class scope.
+          LateParsedAttribute *LA =
+            new LateParsedAttribute(this, *AttrName, AttrNameLoc);
+          LateAttrs->push_back(LA);
+          getCurrentClass().LateParsedDeclarations.push_back(LA);
 
-        if (Tok.is(tok::identifier)) {
-          IdentifierInfo *ParmName = Tok.getIdentifierInfo();
-          SourceLocation ParmLoc = ConsumeToken();
+          // consume everything up to and including the matching right parens
+          ConsumeAndStoreUntil(tok::r_paren, LA->Toks, true, false);
 
-          if (Tok.is(tok::r_paren)) {
-            // __attribute__(( mode(byte) ))
-            ConsumeParen(); // ignore the right paren loc for now
-            attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                         ParmName, ParmLoc, 0, 0);
-          } else if (Tok.is(tok::comma)) {
-            ConsumeToken();
-            // __attribute__(( format(printf, 1, 2) ))
-            ExprVector ArgExprs(Actions);
-            bool ArgExprsOk = true;
-
-            // now parse the non-empty comma separated list of expressions
-            while (1) {
-              ExprResult ArgExpr(ParseAssignmentExpression());
-              if (ArgExpr.isInvalid()) {
-                ArgExprsOk = false;
-                SkipUntil(tok::r_paren);
-                break;
-              } else {
-                ArgExprs.push_back(ArgExpr.release());
-              }
-              if (Tok.isNot(tok::comma))
-                break;
-              ConsumeToken(); // Eat the comma, move to the next argument
-            }
-            if (ArgExprsOk && Tok.is(tok::r_paren)) {
-              ConsumeParen(); // ignore the right paren loc for now
-              attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                           ParmName, ParmLoc, ArgExprs.take(), ArgExprs.size());
-            }
-          }
-        } else { // not an identifier
-          switch (Tok.getKind()) {
-          case tok::r_paren:
-          // parse a possibly empty comma separated list of expressions
-            // __attribute__(( nonnull() ))
-            ConsumeParen(); // ignore the right paren loc for now
-            attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                         0, SourceLocation(), 0, 0);
-            break;
-          case tok::kw_char:
-          case tok::kw_wchar_t:
-          case tok::kw_char16_t:
-          case tok::kw_char32_t:
-          case tok::kw_bool:
-          case tok::kw_short:
-          case tok::kw_int:
-          case tok::kw_long:
-          case tok::kw___int64:
-          case tok::kw_signed:
-          case tok::kw_unsigned:
-          case tok::kw_float:
-          case tok::kw_double:
-          case tok::kw_void:
-          case tok::kw_typeof: {
-            AttributeList *attr
-              = attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                             0, SourceLocation(), 0, 0);
-            if (attr->getKind() == AttributeList::AT_IBOutletCollection)
-              Diag(Tok, diag::err_iboutletcollection_builtintype);
-            // If it's a builtin type name, eat it and expect a rparen
-            // __attribute__(( vec_type_hint(char) ))
-            ConsumeToken();
-            if (Tok.is(tok::r_paren))
-              ConsumeParen();
-            break;
-          }
-          default:
-            // __attribute__(( aligned(16) ))
-            ExprVector ArgExprs(Actions);
-            bool ArgExprsOk = true;
-
-            // now parse the list of expressions
-            while (1) {
-              ExprResult ArgExpr(ParseAssignmentExpression());
-              if (ArgExpr.isInvalid()) {
-                ArgExprsOk = false;
-                SkipUntil(tok::r_paren);
-                break;
-              } else {
-                ArgExprs.push_back(ArgExpr.release());
-              }
-              if (Tok.isNot(tok::comma))
-                break;
-              ConsumeToken(); // Eat the comma, move to the next argument
-            }
-            // Match the ')'.
-            if (ArgExprsOk && Tok.is(tok::r_paren)) {
-              ConsumeParen(); // ignore the right paren loc for now
-              attrs.addNew(AttrName, AttrNameLoc, 0,
-                           AttrNameLoc, 0, SourceLocation(),
-                           ArgExprs.take(), ArgExprs.size());
-            }
-            break;
-          }
+          Token Eof;
+          Eof.startToken();
+          Eof.setLocation(Tok.getLocation());
+          LA->Toks.push_back(Eof);
+        } else {
+          ParseGNUAttributeArgs(AttrName, AttrNameLoc, attrs, endLoc);
         }
       } else {
         attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc,
@@ -241,6 +164,133 @@ void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
   }
 }
 
+
+/// Parse the arguments to a parameterized GNU attribute
+void Parser::ParseGNUAttributeArgs(IdentifierInfo *AttrName,
+                                   SourceLocation AttrNameLoc,
+                                   ParsedAttributes &Attrs,
+                                   SourceLocation *EndLoc) {
+
+  assert(Tok.is(tok::l_paren) && "Attribute arg list not starting with '('");
+
+  // Availability attributes have their own grammar.
+  if (AttrName->isStr("availability")) {
+    ParseAvailabilityAttribute(*AttrName, AttrNameLoc, Attrs, EndLoc);
+    return;
+  }
+  // Thread safety attributes fit into the FIXME case above, so we
+  // just parse the arguments as a list of expressions
+  if (IsThreadSafetyAttribute(AttrName->getName())) {
+    ParseThreadSafetyAttribute(*AttrName, AttrNameLoc, Attrs, EndLoc);
+    return;
+  }
+
+  ConsumeParen(); // ignore the left paren loc for now
+
+  if (Tok.is(tok::identifier)) {
+    IdentifierInfo *ParmName = Tok.getIdentifierInfo();
+    SourceLocation ParmLoc = ConsumeToken();
+
+    if (Tok.is(tok::r_paren)) {
+      // __attribute__(( mode(byte) ))
+      SourceLocation RParen = ConsumeParen();
+      Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen), 0, AttrNameLoc,
+                   ParmName, ParmLoc, 0, 0);
+    } else if (Tok.is(tok::comma)) {
+      ConsumeToken();
+      // __attribute__(( format(printf, 1, 2) ))
+      ExprVector ArgExprs(Actions);
+      bool ArgExprsOk = true;
+
+      // now parse the non-empty comma separated list of expressions
+      while (1) {
+        ExprResult ArgExpr(ParseAssignmentExpression());
+        if (ArgExpr.isInvalid()) {
+          ArgExprsOk = false;
+          SkipUntil(tok::r_paren);
+          break;
+        } else {
+          ArgExprs.push_back(ArgExpr.release());
+        }
+        if (Tok.isNot(tok::comma))
+          break;
+        ConsumeToken(); // Eat the comma, move to the next argument
+      }
+      if (ArgExprsOk && Tok.is(tok::r_paren)) {
+        SourceLocation RParen = ConsumeParen();
+        Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen), 0, AttrNameLoc,
+                     ParmName, ParmLoc, ArgExprs.take(), ArgExprs.size());
+      }
+    }
+  } else { // not an identifier
+    switch (Tok.getKind()) {
+    case tok::r_paren: {
+    // parse a possibly empty comma separated list of expressions
+      // __attribute__(( nonnull() ))
+      SourceLocation RParen = ConsumeParen();
+      Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen), 0, AttrNameLoc,
+                   0, SourceLocation(), 0, 0);
+      break;
+    }
+    case tok::kw_char:
+    case tok::kw_wchar_t:
+    case tok::kw_char16_t:
+    case tok::kw_char32_t:
+    case tok::kw_bool:
+    case tok::kw_short:
+    case tok::kw_int:
+    case tok::kw_long:
+    case tok::kw___int64:
+    case tok::kw_signed:
+    case tok::kw_unsigned:
+    case tok::kw_float:
+    case tok::kw_double:
+    case tok::kw_void:
+    case tok::kw_typeof: {
+      // If it's a builtin type name, eat it and expect a rparen
+      // __attribute__(( vec_type_hint(char) ))
+      SourceLocation EndLoc = ConsumeToken();
+      if (Tok.is(tok::r_paren))
+        EndLoc = ConsumeParen();
+      AttributeList *attr
+        = Attrs.addNew(AttrName, SourceRange(AttrNameLoc, EndLoc), 0,
+                       AttrNameLoc, 0, SourceLocation(), 0, 0);
+      if (attr->getKind() == AttributeList::AT_IBOutletCollection)
+        Diag(Tok, diag::err_iboutletcollection_builtintype);
+      break;
+    }
+    default:
+      // __attribute__(( aligned(16) ))
+      ExprVector ArgExprs(Actions);
+      bool ArgExprsOk = true;
+
+      // now parse the list of expressions
+      while (1) {
+        ExprResult ArgExpr(ParseAssignmentExpression());
+        if (ArgExpr.isInvalid()) {
+          ArgExprsOk = false;
+          SkipUntil(tok::r_paren);
+          break;
+        } else {
+          ArgExprs.push_back(ArgExpr.release());
+        }
+        if (Tok.isNot(tok::comma))
+          break;
+        ConsumeToken(); // Eat the comma, move to the next argument
+      }
+      // Match the ')'.
+      if (ArgExprsOk && Tok.is(tok::r_paren)) {
+        SourceLocation RParen = ConsumeParen();
+        Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen), 0,
+                     AttrNameLoc, 0, SourceLocation(),
+                     ArgExprs.take(), ArgExprs.size());
+      }
+      break;
+    }
+  }
+}
+
+
 /// ParseMicrosoftDeclSpec - Parse an __declspec construct
 ///
 /// [MS] decl-specifier:
@@ -297,10 +347,13 @@ void Parser::ParseMicrosoftTypeAttributes(ParsedAttributes &attrs) {
   // FIXME: Allow Sema to distinguish between these and real attributes!
   while (Tok.is(tok::kw___fastcall) || Tok.is(tok::kw___stdcall) ||
          Tok.is(tok::kw___thiscall) || Tok.is(tok::kw___cdecl)   ||
-         Tok.is(tok::kw___ptr64) || Tok.is(tok::kw___w64)) {
+         Tok.is(tok::kw___ptr64) || Tok.is(tok::kw___w64) ||
+         Tok.is(tok::kw___ptr32) ||
+         Tok.is(tok::kw___unaligned)) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     SourceLocation AttrNameLoc = ConsumeToken();
-    if (Tok.is(tok::kw___ptr64) || Tok.is(tok::kw___w64))
+    if (Tok.is(tok::kw___ptr64) || Tok.is(tok::kw___w64) ||
+        Tok.is(tok::kw___ptr32))
       // FIXME: Support these properly!
       continue;
     attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
@@ -511,12 +564,11 @@ void Parser::ParseAvailabilityAttribute(IdentifierInfo &Availability,
   AvailabilityChange Changes[Unknown];
 
   // Opening '('.
-  SourceLocation LParenLoc;
-  if (!Tok.is(tok::l_paren)) {
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.consumeOpen()) {
     Diag(Tok, diag::err_expected_lparen);
     return;
   }
-  LParenLoc = ConsumeParen();
 
   // Parse the platform name,
   if (Tok.isNot(tok::identifier)) {
@@ -614,12 +666,11 @@ void Parser::ParseAvailabilityAttribute(IdentifierInfo &Availability,
   } while (true);
 
   // Closing ')'.
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
-  if (RParenLoc.isInvalid())
+  if (T.consumeClose())
     return;
 
   if (endLoc)
-    *endLoc = RParenLoc;
+    *endLoc = T.getCloseLocation();
 
   // The 'unavailable' availability cannot be combined with any other
   // availability changes. Make sure that hasn't happened.
@@ -641,7 +692,8 @@ void Parser::ParseAvailabilityAttribute(IdentifierInfo &Availability,
   }
 
   // Record this attribute
-  attrs.addNew(&Availability, AvailabilityLoc, 
+  attrs.addNew(&Availability, 
+               SourceRange(AvailabilityLoc, T.getCloseLocation()), 
                0, SourceLocation(),
                Platform, PlatformLoc,
                Changes[Introduced],
@@ -650,6 +702,172 @@ void Parser::ParseAvailabilityAttribute(IdentifierInfo &Availability,
                UnavailableLoc, false, false);
 }
 
+
+// Late Parsed Attributes:
+// See other examples of late parsing in lib/Parse/ParseCXXInlineMethods
+
+void Parser::LateParsedDeclaration::ParseLexedAttributes() {}
+
+void Parser::LateParsedClass::ParseLexedAttributes() {
+  Self->ParseLexedAttributes(*Class);
+}
+
+void Parser::LateParsedAttribute::ParseLexedAttributes() {
+  Self->ParseLexedAttribute(*this);
+}
+
+/// Wrapper class which calls ParseLexedAttribute, after setting up the
+/// scope appropriately.
+void Parser::ParseLexedAttributes(ParsingClass &Class) {
+  // Deal with templates
+  // FIXME: Test cases to make sure this does the right thing for templates.
+  bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
+  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
+                                HasTemplateScope);
+  if (HasTemplateScope)
+    Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
+
+  // Set or update the scope flags to include Scope::ThisScope.
+  bool AlreadyHasClassScope = Class.TopLevelClass;
+  unsigned ScopeFlags = Scope::ClassScope|Scope::DeclScope|Scope::ThisScope;
+  ParseScope ClassScope(this, ScopeFlags, !AlreadyHasClassScope);
+  ParseScopeFlags ClassScopeFlags(this, ScopeFlags, AlreadyHasClassScope);
+
+  for (unsigned i = 0, ni = Class.LateParsedDeclarations.size(); i < ni; ++i) {
+    Class.LateParsedDeclarations[i]->ParseLexedAttributes();
+  }
+}
+
+/// \brief Finish parsing an attribute for which parsing was delayed.
+/// This will be called at the end of parsing a class declaration
+/// for each LateParsedAttribute. We consume the saved tokens and
+/// create an attribute with the arguments filled in. We add this 
+/// to the Attribute list for the decl.
+void Parser::ParseLexedAttribute(LateParsedAttribute &LA) {
+  // Save the current token position.
+  SourceLocation OrigLoc = Tok.getLocation();
+
+  // Append the current token at the end of the new token stream so that it
+  // doesn't get lost.
+  LA.Toks.push_back(Tok);
+  PP.EnterTokenStream(LA.Toks.data(), LA.Toks.size(), true, false);
+  // Consume the previously pushed token.
+  ConsumeAnyToken();
+
+  ParsedAttributes Attrs(AttrFactory);
+  SourceLocation endLoc;
+
+  // If the Decl is templatized, add template parameters to scope.
+  bool HasTemplateScope = LA.D && LA.D->isTemplateDecl();
+  ParseScope TempScope(this, Scope::TemplateParamScope, HasTemplateScope);
+  if (HasTemplateScope)
+    Actions.ActOnReenterTemplateScope(Actions.CurScope, LA.D);
+
+  // If the Decl is on a function, add function parameters to the scope.
+  bool HasFunctionScope = LA.D && LA.D->isFunctionOrFunctionTemplate();
+  ParseScope FnScope(this, Scope::FnScope|Scope::DeclScope, HasFunctionScope);
+  if (HasFunctionScope)
+    Actions.ActOnReenterFunctionContext(Actions.CurScope, LA.D);
+
+  ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc);
+
+  if (HasFunctionScope) {
+    Actions.ActOnExitFunctionContext();
+    FnScope.Exit();  // Pop scope, and remove Decls from IdResolver
+  }
+  if (HasTemplateScope) {
+    TempScope.Exit();
+  }
+
+  // Late parsed attributes must be attached to Decls by hand.  If the
+  // LA.D is not set, then this was not done properly.
+  assert(LA.D && "No decl attached to late parsed attribute");
+  Actions.ActOnFinishDelayedAttribute(getCurScope(), LA.D, Attrs);
+
+  if (Tok.getLocation() != OrigLoc) {
+    // Due to a parsing error, we either went over the cached tokens or
+    // there are still cached tokens left, so we skip the leftover tokens.
+    // Since this is an uncommon situation that should be avoided, use the
+    // expensive isBeforeInTranslationUnit call.
+    if (PP.getSourceManager().isBeforeInTranslationUnit(Tok.getLocation(),
+                                                        OrigLoc))
+    while (Tok.getLocation() != OrigLoc && Tok.isNot(tok::eof))
+        ConsumeAnyToken();
+  }
+}
+
+/// \brief Wrapper around a case statement checking if AttrName is
+/// one of the thread safety attributes
+bool Parser::IsThreadSafetyAttribute(llvm::StringRef AttrName){
+  return llvm::StringSwitch<bool>(AttrName)
+      .Case("guarded_by", true)
+      .Case("guarded_var", true)
+      .Case("pt_guarded_by", true)
+      .Case("pt_guarded_var", true)
+      .Case("lockable", true)
+      .Case("scoped_lockable", true)
+      .Case("no_thread_safety_analysis", true)
+      .Case("acquired_after", true)
+      .Case("acquired_before", true)
+      .Case("exclusive_lock_function", true)
+      .Case("shared_lock_function", true)
+      .Case("exclusive_trylock_function", true)
+      .Case("shared_trylock_function", true)
+      .Case("unlock_function", true)
+      .Case("lock_returned", true)
+      .Case("locks_excluded", true)
+      .Case("exclusive_locks_required", true)
+      .Case("shared_locks_required", true)
+      .Default(false);
+}
+
+/// \brief Parse the contents of thread safety attributes. These
+/// should always be parsed as an expression list.
+///
+/// We need to special case the parsing due to the fact that if the first token
+/// of the first argument is an identifier, the main parse loop will store
+/// that token as a "parameter" and the rest of
+/// the arguments will be added to a list of "arguments". However,
+/// subsequent tokens in the first argument are lost. We instead parse each
+/// argument as an expression and add all arguments to the list of "arguments".
+/// In future, we will take advantage of this special case to also
+/// deal with some argument scoping issues here (for example, referring to a
+/// function parameter in the attribute on that function).
+void Parser::ParseThreadSafetyAttribute(IdentifierInfo &AttrName,
+                                        SourceLocation AttrNameLoc,
+                                        ParsedAttributes &Attrs,
+                                        SourceLocation *EndLoc) {
+  assert(Tok.is(tok::l_paren) && "Attribute arg list not starting with '('");
+
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+  
+  ExprVector ArgExprs(Actions);
+  bool ArgExprsOk = true;
+  
+  // now parse the list of expressions
+  while (1) {
+    ExprResult ArgExpr(ParseAssignmentExpression());
+    if (ArgExpr.isInvalid()) {
+      ArgExprsOk = false;
+      T.consumeClose();
+      break;
+    } else {
+      ArgExprs.push_back(ArgExpr.release());
+    }
+    if (Tok.isNot(tok::comma))
+      break;
+    ConsumeToken(); // Eat the comma, move to the next argument
+  }
+  // Match the ')'.
+  if (ArgExprsOk && !T.consumeClose()) {
+    Attrs.addNew(&AttrName, AttrNameLoc, 0, AttrNameLoc, 0, SourceLocation(),
+                 ArgExprs.take(), ArgExprs.size());
+  }
+  if (EndLoc)
+    *EndLoc = T.getCloseLocation();
+}
+
 void Parser::DiagnoseProhibitedAttributes(ParsedAttributesWithRange &attrs) {
   Diag(attrs.Range.getBegin(), diag::err_attributes_not_allowed)
     << attrs.Range;
@@ -676,6 +894,9 @@ Parser::DeclGroupPtrTy Parser::ParseDeclaration(StmtVector &Stmts,
                                                 SourceLocation &DeclEnd,
                                           ParsedAttributesWithRange &attrs) {
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
+  // Must temporarily exit the objective-c container scope for
+  // parsing c none objective-c decls.
+  ObjCDeclContextSwitch ObjCDC(*this);
   
   Decl *SingleDecl = 0;
   Decl *OwnedType = 0;
@@ -832,7 +1053,7 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
     return Actions.FinalizeDeclaratorGroup(getCurScope(), DS, &ThisDecl, 1);
   }
 
-  llvm::SmallVector<Decl *, 8> DeclsInGroup;
+  SmallVector<Decl *, 8> DeclsInGroup;
   Decl *FirstDecl = ParseDeclarationAfterDeclaratorAndAttributes(D);
   D.complete(FirstDecl);
   if (FirstDecl)
@@ -998,9 +1219,8 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
 
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteInitializer(getCurScope(), ThisDecl);
-        ConsumeCodeCompletionToken();
-        SkipUntil(tok::comma, true, true);
-        return ThisDecl;
+        cutOffParsing();
+        return 0;
       }
       
       ExprResult Init(ParseInitializer());
@@ -1019,7 +1239,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
     }
   } else if (Tok.is(tok::l_paren)) {
     // Parse C++ direct initializer: '(' expression-list ')'
-    SourceLocation LParenLoc = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
+
     ExprVector Exprs(Actions);
     CommaLocsTy CommaLocs;
 
@@ -1037,7 +1259,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
       }
     } else {
       // Match the ')'.
-      SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+      T.consumeClose();
 
       assert(!Exprs.empty() && Exprs.size()-1 == CommaLocs.size() &&
              "Unexpected number of commas!");
@@ -1047,9 +1269,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
         ExitScope();
       }
 
-      Actions.AddCXXDirectInitializerToDecl(ThisDecl, LParenLoc,
+      Actions.AddCXXDirectInitializerToDecl(ThisDecl, T.getOpenLocation(),
                                             move_arg(Exprs),
-                                            RParenLoc,
+                                            T.getCloseLocation(),
                                             TypeContainsAuto);
     }
   } else if (getLang().CPlusPlus0x && Tok.is(tok::l_brace)) {
@@ -1090,6 +1312,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
 void Parser::ParseSpecifierQualifierList(DeclSpec &DS, AccessSpecifier AS) {
   /// specifier-qualifier-list is a subset of declaration-specifiers.  Just
   /// parse declaration-specifiers and complain about extra stuff.
+  /// TODO: diagnose attribute-specifiers and alignment-specifiers.
   ParseDeclarationSpecifiers(DS, ParsedTemplateInfo(), AS);
 
   // Validate declspec for type-name.
@@ -1273,12 +1496,70 @@ Parser::getDeclSpecContextFromDeclaratorContext(unsigned Context) {
   return DSC_normal;
 }
 
+/// ParseAlignArgument - Parse the argument to an alignment-specifier.
+///
+/// FIXME: Simply returns an alignof() expression if the argument is a
+/// type. Ideally, the type should be propagated directly into Sema.
+///
+/// [C1X/C++0x] type-id
+/// [C1X]       constant-expression
+/// [C++0x]     assignment-expression
+ExprResult Parser::ParseAlignArgument(SourceLocation Start) {
+  if (isTypeIdInParens()) {
+    EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
+    SourceLocation TypeLoc = Tok.getLocation();
+    ParsedType Ty = ParseTypeName().get();
+    SourceRange TypeRange(Start, Tok.getLocation());
+    return Actions.ActOnUnaryExprOrTypeTraitExpr(TypeLoc, UETT_AlignOf, true,
+                                                Ty.getAsOpaquePtr(), TypeRange);
+  } else
+    return ParseConstantExpression();
+}
+
+/// ParseAlignmentSpecifier - Parse an alignment-specifier, and add the
+/// attribute to Attrs.
+///
+/// alignment-specifier:
+/// [C1X]   '_Alignas' '(' type-id ')'
+/// [C1X]   '_Alignas' '(' constant-expression ')'
+/// [C++0x] 'alignas' '(' type-id ')'
+/// [C++0x] 'alignas' '(' assignment-expression ')'
+void Parser::ParseAlignmentSpecifier(ParsedAttributes &Attrs,
+                                     SourceLocation *endLoc) {
+  assert((Tok.is(tok::kw_alignas) || Tok.is(tok::kw__Alignas)) &&
+         "Not an alignment-specifier!");
+
+  SourceLocation KWLoc = Tok.getLocation();
+  ConsumeToken();
+
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
+    return;
+
+  ExprResult ArgExpr = ParseAlignArgument(T.getOpenLocation());
+  if (ArgExpr.isInvalid()) {
+    SkipUntil(tok::r_paren);
+    return;
+  }
+
+  T.consumeClose();
+  if (endLoc)
+    *endLoc = T.getCloseLocation();
+
+  ExprVector ArgExprs(Actions);
+  ArgExprs.push_back(ArgExpr.release());
+  Attrs.addNew(PP.getIdentifierInfo("aligned"), KWLoc, 0, KWLoc,
+               0, T.getOpenLocation(), ArgExprs.take(), 1, false, true);
+}
+
 /// ParseDeclarationSpecifiers
 ///       declaration-specifiers: [C99 6.7]
 ///         storage-class-specifier declaration-specifiers[opt]
 ///         type-specifier declaration-specifiers[opt]
 /// [C99]   function-specifier declaration-specifiers[opt]
+/// [C1X]   alignment-specifier declaration-specifiers[opt]
 /// [GNU]   attributes declaration-specifiers[opt]
+/// [Clang] '__module_private__' declaration-specifiers[opt]
 ///
 ///       storage-class-specifier: [C99 6.7.1]
 ///         'typedef'
@@ -1316,6 +1597,9 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     switch (Tok.getKind()) {
     default:
     DoneWithDeclSpec:
+      // [C++0x] decl-specifier-seq: decl-specifier attribute-specifier-seq[opt]
+      MaybeParseCXX0XAttributes(DS.getAttributes());
+
       // If this is not a declaration specifier token, we're done reading decl
       // specifiers.  First verify that DeclSpec's are consistent.
       DS.Finish(Diags, PP);
@@ -1337,8 +1621,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         Actions.CodeCompleteDeclSpec(getCurScope(), DS,
                                      AllowNonIdentifiers, 
                                      AllowNestedNameSpecifiers);
-        ConsumeCodeCompletionToken();
-        return;
+        return cutOffParsing();
       } 
       
       if (getCurScope()->getFnParent() || getCurScope()->getBlockParent())
@@ -1352,8 +1635,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         CCC = Sema::PCC_ObjCImplementation;
       
       Actions.CodeCompleteOrdinaryName(getCurScope(), CCC);
-      ConsumeCodeCompletionToken();
-      return;
+      return cutOffParsing();
     }
 
     case tok::coloncolon: // ::foo::bar
@@ -1641,11 +1923,13 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       break;
 
     case tok::kw___ptr64:
+    case tok::kw___ptr32:
     case tok::kw___w64:
     case tok::kw___cdecl:
     case tok::kw___stdcall:
     case tok::kw___fastcall:
     case tok::kw___thiscall:
+    case tok::kw___unaligned:
       ParseMicrosoftTypeAttributes(DS.getAttributes());
       continue;
 
@@ -1661,49 +1945,47 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
     // storage-class-specifier
     case tok::kw_typedef:
-      isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_typedef, Loc, PrevSpec,
-                                         DiagID, getLang());
+      isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_typedef, Loc,
+                                         PrevSpec, DiagID);
       break;
     case tok::kw_extern:
       if (DS.isThreadSpecified())
         Diag(Tok, diag::ext_thread_before) << "extern";
-      isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_extern, Loc, PrevSpec,
-                                         DiagID, getLang());
+      isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_extern, Loc,
+                                         PrevSpec, DiagID);
       break;
     case tok::kw___private_extern__:
-      isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_private_extern, Loc,
-                                         PrevSpec, DiagID, getLang());
+      isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_private_extern,
+                                         Loc, PrevSpec, DiagID);
       break;
     case tok::kw_static:
       if (DS.isThreadSpecified())
         Diag(Tok, diag::ext_thread_before) << "static";
-      isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_static, Loc, PrevSpec,
-                                         DiagID, getLang());
+      isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_static, Loc,
+                                         PrevSpec, DiagID);
       break;
     case tok::kw_auto:
       if (getLang().CPlusPlus0x) {
         if (isKnownToBeTypeSpecifier(GetLookAheadToken(1))) {
-          isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_auto, Loc, PrevSpec,
-                                           DiagID, getLang());
+          isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_auto, Loc,
+                                             PrevSpec, DiagID);
           if (!isInvalid)
-            Diag(Tok, diag::auto_storage_class)
+            Diag(Tok, diag::ext_auto_storage_class)
               << FixItHint::CreateRemoval(DS.getStorageClassSpecLoc());
-        }
-        else
+        } else
           isInvalid = DS.SetTypeSpecType(DeclSpec::TST_auto, Loc, PrevSpec,
                                          DiagID);
-      }
-      else
-        isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_auto, Loc, PrevSpec,
-                                           DiagID, getLang());
+      } else
+        isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_auto, Loc,
+                                           PrevSpec, DiagID);
       break;
     case tok::kw_register:
-      isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_register, Loc, PrevSpec,
-                                         DiagID, getLang());
+      isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_register, Loc,
+                                         PrevSpec, DiagID);
       break;
     case tok::kw_mutable:
-      isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_mutable, Loc, PrevSpec,
-                                         DiagID, getLang());
+      isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_mutable, Loc,
+                                         PrevSpec, DiagID);
       break;
     case tok::kw___thread:
       isInvalid = DS.SetStorageClassSpecThread(Loc, PrevSpec, DiagID);
@@ -1720,6 +2002,13 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       isInvalid = DS.SetFunctionSpecExplicit(Loc, PrevSpec, DiagID);
       break;
 
+    // alignment-specifier
+    case tok::kw__Alignas:
+      if (!getLang().C1X)
+        Diag(Tok, diag::ext_c1x_alignas);
+      ParseAlignmentSpecifier(DS.getAttributes());
+      continue;
+
     // friend
     case tok::kw_friend:
       if (DSContext == DSC_class)
@@ -1731,6 +2020,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       }
       break;
 
+    // Modules
+    case tok::kw___module_private__:
+      isInvalid = DS.setModulePrivateSpec(Loc, PrevSpec, DiagID);
+      break;
+        
     // constexpr
     case tok::kw_constexpr:
       isInvalid = DS.SetConstexprSpec(Loc, PrevSpec, DiagID);
@@ -1781,6 +2075,10 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       isInvalid = DS.SetTypeSpecType(DeclSpec::TST_int, Loc, PrevSpec,
                                      DiagID);
       break;
+     case tok::kw_half:
+       isInvalid = DS.SetTypeSpecType(DeclSpec::TST_half, Loc, PrevSpec,
+                                      DiagID);
+       break;
     case tok::kw_float:
       isInvalid = DS.SetTypeSpecType(DeclSpec::TST_float, Loc, PrevSpec,
                                      DiagID);
@@ -1890,6 +2188,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
     case tok::kw___underlying_type:
       ParseUnderlyingTypeSpecifier(DS);
+      continue;
+
+    case tok::kw__Atomic:
+      ParseAtomicSpecifier(DS);
+      continue;
 
     // OpenCL qualifiers:
     case tok::kw_private: 
@@ -2004,7 +2307,8 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
   case tok::kw_typename:  // typename foo::bar
     // Annotate typenames and C++ scope specifiers.  If we get one, just
     // recurse to handle whatever we get.
-    if (TryAnnotateTypeOrScopeToken())
+    if (TryAnnotateTypeOrScopeToken(/*EnteringContext=*/false,
+                                    /*NeedType=*/true))
       return true;
     if (Tok.is(tok::identifier))
       return false;
@@ -2017,7 +2321,8 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
 
     // Annotate typenames and C++ scope specifiers.  If we get one, just
     // recurse to handle whatever we get.
-    if (TryAnnotateTypeOrScopeToken())
+    if (TryAnnotateTypeOrScopeToken(/*EnteringContext=*/false,
+                                    /*NeedType=*/true))
       return true;
     return ParseOptionalTypeSpecifier(DS, isInvalid, PrevSpec, DiagID,
                                       TemplateInfo, SuppressDeclarations);
@@ -2082,6 +2387,9 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
   case tok::kw_int:
     isInvalid = DS.SetTypeSpecType(DeclSpec::TST_int, Loc, PrevSpec, DiagID);
     break;
+  case tok::kw_half:
+    isInvalid = DS.SetTypeSpecType(DeclSpec::TST_half, Loc, PrevSpec, DiagID);
+    break;
   case tok::kw_float:
     isInvalid = DS.SetTypeSpecType(DeclSpec::TST_float, Loc, PrevSpec, DiagID);
     break;
@@ -2166,6 +2474,10 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
     ParseUnderlyingTypeSpecifier(DS);
     return true;
 
+  case tok::kw__Atomic:
+    ParseAtomicSpecifier(DS);
+    return true;
+
   // OpenCL qualifiers:
   case tok::kw_private: 
     if (!getLang().OpenCL)
@@ -2182,18 +2494,24 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
 
   // C++0x auto support.
   case tok::kw_auto:
-    if (!getLang().CPlusPlus0x)
+    // This is only called in situations where a storage-class specifier is
+    // illegal, so we can assume an auto type specifier was intended even in
+    // C++98. In C++98 mode, DeclSpec::Finish will produce an appropriate
+    // extension diagnostic.
+    if (!getLang().CPlusPlus)
       return false;
 
     isInvalid = DS.SetTypeSpecType(DeclSpec::TST_auto, Loc, PrevSpec, DiagID);
     break;
 
   case tok::kw___ptr64:
+  case tok::kw___ptr32:
   case tok::kw___w64:
   case tok::kw___cdecl:
   case tok::kw___stdcall:
   case tok::kw___fastcall:
   case tok::kw___thiscall:
+  case tok::kw___unaligned:
     ParseMicrosoftTypeAttributes(DS.getAttributes());
     return true;
 
@@ -2236,6 +2554,7 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
 ///
 void Parser::
 ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
+    
   if (Tok.is(tok::kw___extension__)) {
     // __extension__ silences extension warnings in the subexpression.
     ExtensionRAIIObject O(Diags);  // Use RAII to do this.
@@ -2314,7 +2633,9 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
   PrettyDeclStackTraceEntry CrashInfo(Actions, TagDecl, RecordLoc,
                                       "parsing struct/union body");
 
-  SourceLocation LBraceLoc = ConsumeBrace();
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  if (T.consumeOpen())
+    return;
 
   ParseScope StructScope(this, Scope::ClassScope|Scope::DeclScope);
   Actions.ActOnTagStartDefinition(getCurScope(), TagDecl);
@@ -2325,7 +2646,7 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
     Diag(Tok, diag::ext_empty_struct_union)
       << (TagType == TST_union);
 
-  llvm::SmallVector<Decl *, 32> FieldDecls;
+  SmallVector<Decl *, 32> FieldDecls;
 
   // While we still have something to read, read the declarations in the struct.
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
@@ -2347,10 +2668,10 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
       struct CFieldCallback : FieldCallback {
         Parser &P;
         Decl *TagDecl;
-        llvm::SmallVectorImpl<Decl *> &FieldDecls;
+        SmallVectorImpl<Decl *> &FieldDecls;
 
         CFieldCallback(Parser &P, Decl *TagDecl,
-                       llvm::SmallVectorImpl<Decl *> &FieldDecls) :
+                       SmallVectorImpl<Decl *> &FieldDecls) :
           P(P), TagDecl(TagDecl), FieldDecls(FieldDecls) {}
 
         virtual Decl *invoke(FieldDeclarator &FD) {
@@ -2378,7 +2699,7 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
         SkipUntil(tok::semi, true);
         continue;
       }
-      llvm::SmallVector<Decl *, 16> Fields;
+      SmallVector<Decl *, 16> Fields;
       Actions.ActOnDefs(getCurScope(), TagDecl, Tok.getLocation(),
                         Tok.getIdentifierInfo(), Fields);
       FieldDecls.insert(FieldDecls.end(), Fields.begin(), Fields.end());
@@ -2400,18 +2721,19 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
     }
   }
 
-  SourceLocation RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
+  T.consumeClose();
 
   ParsedAttributes attrs(AttrFactory);
   // If attributes exist after struct contents, parse them.
   MaybeParseGNUAttributes(attrs);
 
   Actions.ActOnFields(getCurScope(),
-                      RecordLoc, TagDecl, FieldDecls.data(), FieldDecls.size(),
-                      LBraceLoc, RBraceLoc,
+                      RecordLoc, TagDecl, FieldDecls,
+                      T.getOpenLocation(), T.getCloseLocation(),
                       attrs.getList());
   StructScope.Exit();
-  Actions.ActOnTagFinishDefinition(getCurScope(), TagDecl, RBraceLoc);
+  Actions.ActOnTagFinishDefinition(getCurScope(), TagDecl,
+                                   T.getCloseLocation());
 }
 
 /// ParseEnumSpecifier
@@ -2448,7 +2770,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   if (Tok.is(tok::code_completion)) {
     // Code completion for an enum name.
     Actions.CodeCompleteTag(getCurScope(), DeclSpec::TST_enum);
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   }
 
   bool IsScopedEnum = false;
@@ -2465,7 +2787,8 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   ParsedAttributes attrs(AttrFactory);
   MaybeParseGNUAttributes(attrs);
 
-  bool AllowFixedUnderlyingType = getLang().CPlusPlus0x || getLang().Microsoft;
+  bool AllowFixedUnderlyingType 
+    = getLang().CPlusPlus0x || getLang().MicrosoftExt || getLang().ObjC2;
 
   CXXScopeSpec &SS = DS.getTypeSpecScope();
   if (getLang().CPlusPlus) {
@@ -2568,7 +2891,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
       SourceRange Range;
       BaseType = ParseTypeName(&Range);
       
-      if (!getLang().CPlusPlus0x)
+      if (!getLang().CPlusPlus0x && !getLang().ObjC2)
         Diag(StartLoc, diag::ext_ms_enum_fixed_underlying_type)
           << Range;
     }
@@ -2615,7 +2938,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   unsigned DiagID;
   Decl *TagDecl = Actions.ActOnTag(getCurScope(), DeclSpec::TST_enum, TUK,
                                    StartLoc, SS, Name, NameLoc, attrs.getList(),
-                                   AS,
+                                   AS, DS.getModulePrivateSpecLoc(),
                                    MultiTemplateParamsArg(Actions),
                                    Owned, IsDependent, IsScopedEnum,
                                    IsScopedUsingClassTag, BaseType);
@@ -2681,13 +3004,14 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
   ParseScope EnumScope(this, Scope::DeclScope);
   Actions.ActOnTagStartDefinition(getCurScope(), EnumDecl);
 
-  SourceLocation LBraceLoc = ConsumeBrace();
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  T.consumeOpen();
 
   // C does not allow an empty enumerator-list, C++ does [dcl.enum].
   if (Tok.is(tok::r_brace) && !getLang().CPlusPlus)
     Diag(Tok, diag::error_empty_enum);
 
-  llvm::SmallVector<Decl *, 32> EnumConstantDecls;
+  SmallVector<Decl *, 32> EnumConstantDecls;
 
   Decl *LastEnumConstDecl = 0;
 
@@ -2738,18 +3062,20 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
   }
 
   // Eat the }.
-  SourceLocation RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
+  T.consumeClose();
 
   // If attributes exist after the identifier list, parse them.
   ParsedAttributes attrs(AttrFactory);
   MaybeParseGNUAttributes(attrs);
 
-  Actions.ActOnEnumBody(StartLoc, LBraceLoc, RBraceLoc, EnumDecl,
-                        EnumConstantDecls.data(), EnumConstantDecls.size(),
-                        getCurScope(), attrs.getList());
+  Actions.ActOnEnumBody(StartLoc, T.getOpenLocation(), T.getCloseLocation(),
+                        EnumDecl, EnumConstantDecls.data(),
+                        EnumConstantDecls.size(), getCurScope(),
+                        attrs.getList());
 
   EnumScope.Exit();
-  Actions.ActOnTagFinishDefinition(getCurScope(), EnumDecl, RBraceLoc);
+  Actions.ActOnTagFinishDefinition(getCurScope(), EnumDecl,
+                                   T.getCloseLocation());
 }
 
 /// isTypeSpecifierQualifier - Return true if the current token could be the
@@ -2797,6 +3123,7 @@ bool Parser::isKnownToBeTypeSpecifier(const Token &Tok) const {
   case tok::kw_char16_t:
   case tok::kw_char32_t:
   case tok::kw_int:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_bool:
@@ -2866,6 +3193,7 @@ bool Parser::isTypeSpecifierQualifier() {
   case tok::kw_char16_t:
   case tok::kw_char32_t:
   case tok::kw_int:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_bool:
@@ -2901,7 +3229,9 @@ bool Parser::isTypeSpecifierQualifier() {
   case tok::kw___thiscall:
   case tok::kw___w64:
   case tok::kw___ptr64:
+  case tok::kw___ptr32:
   case tok::kw___pascal:
+  case tok::kw___unaligned:
 
   case tok::kw___private:
   case tok::kw___local:
@@ -2915,6 +3245,10 @@ bool Parser::isTypeSpecifierQualifier() {
 
   case tok::kw_private:
     return getLang().OpenCL;
+
+  // C1x _Atomic()
+  case tok::kw__Atomic:
+    return true;
   }
 }
 
@@ -2976,6 +3310,9 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   case tok::kw_register:
   case tok::kw___thread:
 
+    // Modules
+  case tok::kw___module_private__:
+      
     // type-specifiers
   case tok::kw_short:
   case tok::kw_long:
@@ -2991,6 +3328,7 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   case tok::kw_char32_t:
 
   case tok::kw_int:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_bool:
@@ -3031,6 +3369,10 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   case tok::kw_decltype:
     return true;
 
+    // C1x _Atomic()
+  case tok::kw__Atomic:
+    return true;
+
     // GNU ObjC bizarre protocol extension: <proto1,proto2> with implicit 'id'.
   case tok::less:
     return getLang().ObjC1;
@@ -3047,8 +3389,10 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   case tok::kw___thiscall:
   case tok::kw___w64:
   case tok::kw___ptr64:
+  case tok::kw___ptr32:
   case tok::kw___forceinline:
   case tok::kw___pascal:
+  case tok::kw___unaligned:
 
   case tok::kw___private:
   case tok::kw___local:
@@ -3148,8 +3492,7 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
     switch (Tok.getKind()) {
     case tok::code_completion:
       Actions.CodeCompleteTypeQualifiers(DS);
-      ConsumeCodeCompletionToken();
-      break;
+      return cutOffParsing();
         
     case tok::kw_const:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_const   , Loc, PrevSpec, DiagID,
@@ -3180,10 +3523,12 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
 
     case tok::kw___w64:
     case tok::kw___ptr64:
+    case tok::kw___ptr32:
     case tok::kw___cdecl:
     case tok::kw___stdcall:
     case tok::kw___fastcall:
     case tok::kw___thiscall:
+    case tok::kw___unaligned:
       if (VendorAttributesAllowed) {
         ParseMicrosoftTypeAttributes(DS.getAttributes());
         continue;
@@ -3548,7 +3893,9 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
           break;
       }
       ParsedAttributes attrs(AttrFactory);
-      ParseFunctionDeclarator(ConsumeParen(), D, attrs);
+      BalancedDelimiterTracker T(*this, tok::l_paren);
+      T.consumeOpen();
+      ParseFunctionDeclarator(D, attrs, T);
     } else if (Tok.is(tok::l_square)) {
       ParseBracketDeclarator(D);
     } else {
@@ -3571,7 +3918,9 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
 ///                    parameter-type-list[opt] ')'
 ///
 void Parser::ParseParenDeclarator(Declarator &D) {
-  SourceLocation StartLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+
   assert(!D.isPastIdentifier() && "Should be called before passing identifier");
 
   // Eat any attributes before we look at whether this is a grouping or function
@@ -3596,7 +3945,8 @@ void Parser::ParseParenDeclarator(Declarator &D) {
   // Eat any Microsoft extensions.
   if  (Tok.is(tok::kw___cdecl) || Tok.is(tok::kw___stdcall) ||
        Tok.is(tok::kw___thiscall) || Tok.is(tok::kw___fastcall) ||
-       Tok.is(tok::kw___w64) || Tok.is(tok::kw___ptr64)) {
+       Tok.is(tok::kw___w64) || Tok.is(tok::kw___ptr64) ||
+       Tok.is(tok::kw___ptr32) || Tok.is(tok::kw___unaligned)) {
     ParseMicrosoftTypeAttributes(attrs);
   }
   // Eat any Borland extensions.
@@ -3633,9 +3983,10 @@ void Parser::ParseParenDeclarator(Declarator &D) {
 
     ParseDeclaratorInternal(D, &Parser::ParseDirectDeclarator);
     // Match the ')'.
-    SourceLocation EndLoc = MatchRHSPunctuation(tok::r_paren, StartLoc);
-    D.AddTypeInfo(DeclaratorChunk::getParen(StartLoc, EndLoc),
-                  attrs, EndLoc);
+    T.consumeClose();
+    D.AddTypeInfo(DeclaratorChunk::getParen(T.getOpenLocation(), 
+                                            T.getCloseLocation()),
+                  attrs, T.getCloseLocation());
 
     D.setGroupingParens(hadGroupingParens);
     return;
@@ -3647,7 +3998,7 @@ void Parser::ParseParenDeclarator(Declarator &D) {
   // ParseFunctionDeclarator to handle of argument list.
   D.SetIdentifier(0, Tok.getLocation());
 
-  ParseFunctionDeclarator(StartLoc, D, attrs, RequiresArg);
+  ParseFunctionDeclarator(D, attrs, T, RequiresArg);
 }
 
 /// ParseFunctionDeclarator - We are after the identifier and have parsed the
@@ -3668,8 +4019,9 @@ void Parser::ParseParenDeclarator(Declarator &D) {
 ///           dynamic-exception-specification
 ///           noexcept-specification
 ///
-void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
+void Parser::ParseFunctionDeclarator(Declarator &D,
                                      ParsedAttributes &attrs,
+                                     BalancedDelimiterTracker &Tracker,
                                      bool RequiresArg) {
   // lparen is already consumed!
   assert(D.isPastIdentifier() && "Should not call before identifier!");
@@ -3678,7 +4030,7 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
   // Otherwise, it is treated as a K&R-style function.
   bool HasProto = false;
   // Build up an array of information about the parsed arguments.
-  llvm::SmallVector<DeclaratorChunk::ParamInfo, 16> ParamInfo;
+  SmallVector<DeclaratorChunk::ParamInfo, 16> ParamInfo;
   // Remember where we see an ellipsis, if any.
   SourceLocation EllipsisLoc;
 
@@ -3687,20 +4039,20 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
   SourceLocation RefQualifierLoc;
   ExceptionSpecificationType ESpecType = EST_None;
   SourceRange ESpecRange;
-  llvm::SmallVector<ParsedType, 2> DynamicExceptions;
-  llvm::SmallVector<SourceRange, 2> DynamicExceptionRanges;
+  SmallVector<ParsedType, 2> DynamicExceptions;
+  SmallVector<SourceRange, 2> DynamicExceptionRanges;
   ExprResult NoexceptExpr;
   ParsedType TrailingReturnType;
   
   SourceLocation EndLoc;
-
   if (isFunctionDeclaratorIdentifierList()) {
     if (RequiresArg)
       Diag(Tok, diag::err_argument_required_after_attribute);
 
     ParseFunctionDeclaratorIdentifierList(D, ParamInfo);
 
-    EndLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    Tracker.consumeClose();
+    EndLoc = Tracker.getCloseLocation();
   } else {
     // Enter function-declaration scope, limiting any declarators to the
     // function prototype scope, including parameter declarators.
@@ -3715,7 +4067,8 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
     HasProto = ParamInfo.size() || getLang().CPlusPlus;
 
     // If we have the closing ')', eat it.
-    EndLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    Tracker.consumeClose();
+    EndLoc = Tracker.getCloseLocation();
 
     if (getLang().CPlusPlus) {
       MaybeParseCXX0XAttributes(attrs);
@@ -3745,7 +4098,10 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
 
       // Parse trailing-return-type[opt].
       if (getLang().CPlusPlus0x && Tok.is(tok::arrow)) {
-        TrailingReturnType = ParseTrailingReturnType().get();
+        SourceRange Range;
+        TrailingReturnType = ParseTrailingReturnType(Range).get();
+        if (Range.getEnd().isValid())
+          EndLoc = Range.getEnd();
       }
     }
 
@@ -3768,7 +4124,8 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
                                              DynamicExceptions.size(),
                                              NoexceptExpr.isUsable() ?
                                                NoexceptExpr.get() : 0,
-                                             LParenLoc, EndLoc, D,
+                                             Tracker.getOpenLocation(), 
+                                             EndLoc, D,
                                              TrailingReturnType),
                 attrs, EndLoc);
 }
@@ -3811,7 +4168,7 @@ bool Parser::isFunctionDeclaratorIdentifierList() {
 ///
 void Parser::ParseFunctionDeclaratorIdentifierList(
        Declarator &D,
-       llvm::SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo) {
+       SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo) {
   // If there was no identifier specified for the declarator, either we are in
   // an abstract-declarator, or we are in a parameter declarator which was found
   // to be abstract.  In abstract-declarators, identifier lists are not valid:
@@ -3890,7 +4247,7 @@ void Parser::ParseFunctionDeclaratorIdentifierList(
 void Parser::ParseParameterDeclarationClause(
        Declarator &D,
        ParsedAttributes &attrs,
-       llvm::SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo,
+       SmallVector<DeclaratorChunk::ParamInfo, 16> &ParamInfo,
        SourceLocation &EllipsisLoc) {
 
   while (1) {
@@ -3902,9 +4259,9 @@ void Parser::ParseParameterDeclarationClause(
     // Parse the declaration-specifiers.
     // Just use the ParsingDeclaration "scope" of the declarator.
     DeclSpec DS(AttrFactory);
-	
+
     // Skip any Microsoft attributes before a param.
-    if (getLang().Microsoft && Tok.is(tok::l_square))
+    if (getLang().MicrosoftExt && Tok.is(tok::l_square))
       ParseMicrosoftAttributes(DS.getAttributes());
 
     SourceLocation DSStart = Tok.getLocation();
@@ -4036,20 +4393,22 @@ void Parser::ParseParameterDeclarationClause(
 /// [C99]   direct-declarator '[' type-qual-list 'static' assignment-expr ']'
 /// [C99]   direct-declarator '[' type-qual-list[opt] '*' ']'
 void Parser::ParseBracketDeclarator(Declarator &D) {
-  SourceLocation StartLoc = ConsumeBracket();
+  BalancedDelimiterTracker T(*this, tok::l_square);
+  T.consumeOpen();
 
   // C array syntax has many features, but by-far the most common is [] and [4].
   // This code does a fast path to handle some of the most obvious cases.
   if (Tok.getKind() == tok::r_square) {
-    SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
+    T.consumeClose();
     ParsedAttributes attrs(AttrFactory);
     MaybeParseCXX0XAttributes(attrs);
     
     // Remember that we parsed the empty array type.
     ExprResult NumElements;
     D.AddTypeInfo(DeclaratorChunk::getArray(0, false, false, 0,
-                                            StartLoc, EndLoc),
-                  attrs, EndLoc);
+                                            T.getOpenLocation(),
+                                            T.getCloseLocation()),
+                  attrs, T.getCloseLocation());
     return;
   } else if (Tok.getKind() == tok::numeric_constant &&
              GetLookAheadToken(1).is(tok::r_square)) {
@@ -4057,15 +4416,16 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
     ExprResult ExprRes(Actions.ActOnNumericConstant(Tok));
     ConsumeToken();
 
-    SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
+    T.consumeClose();
     ParsedAttributes attrs(AttrFactory);
     MaybeParseCXX0XAttributes(attrs);
 
     // Remember that we parsed a array type, and remember its features.
     D.AddTypeInfo(DeclaratorChunk::getArray(0, false, 0,
                                             ExprRes.release(),
-                                            StartLoc, EndLoc),
-                  attrs, EndLoc);
+                                            T.getOpenLocation(),
+                                            T.getCloseLocation()),
+                  attrs, T.getCloseLocation());
     return;
   }
 
@@ -4122,7 +4482,7 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
     return;
   }
 
-  SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
+  T.consumeClose();
 
   ParsedAttributes attrs(AttrFactory);
   MaybeParseCXX0XAttributes(attrs);
@@ -4131,8 +4491,9 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
   D.AddTypeInfo(DeclaratorChunk::getArray(DS.getTypeQualifiers(),
                                           StaticLoc.isValid(), isStar,
                                           NumElements.release(),
-                                          StartLoc, EndLoc),
-                attrs, EndLoc);
+                                          T.getOpenLocation(),
+                                          T.getCloseLocation()),
+                attrs, T.getCloseLocation());
 }
 
 /// [GNU]   typeof-specifier:
@@ -4190,6 +4551,41 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) {
     Diag(StartLoc, DiagID) << PrevSpec;
 }
 
+/// [C1X]   atomic-specifier:
+///           _Atomic ( type-name )
+///
+void Parser::ParseAtomicSpecifier(DeclSpec &DS) {
+  assert(Tok.is(tok::kw__Atomic) && "Not an atomic specifier");
+
+  SourceLocation StartLoc = ConsumeToken();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen_after, "_Atomic")) {
+    SkipUntil(tok::r_paren);
+    return;
+  }
+
+  TypeResult Result = ParseTypeName();
+  if (Result.isInvalid()) {
+    SkipUntil(tok::r_paren);
+    return;
+  }
+
+  // Match the ')'
+  T.consumeClose();
+
+  if (T.getCloseLocation().isInvalid())
+    return;
+
+  DS.setTypeofParensRange(T.getRange());
+  DS.SetRangeEnd(T.getCloseLocation());
+
+  const char *PrevSpec = 0;
+  unsigned DiagID;
+  if (DS.SetTypeSpecType(DeclSpec::TST_atomic, StartLoc, PrevSpec,
+                         DiagID, Result.release()))
+    Diag(StartLoc, DiagID) << PrevSpec;
+}
+
 
 /// TryAltiVecVectorTokenOutOfLine - Out of line body that should only be called
 /// from TryAltiVecVectorToken.
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseDeclCXX.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseDeclCXX.cpp
index 172049c34053..4339047f1f16 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseDeclCXX.cpp
@@ -52,10 +52,12 @@ Decl *Parser::ParseNamespace(unsigned Context,
                              SourceLocation InlineLoc) {
   assert(Tok.is(tok::kw_namespace) && "Not a namespace!");
   SourceLocation NamespaceLoc = ConsumeToken();  // eat the 'namespace'.
-
+  ObjCDeclContextSwitch ObjCDC(*this);
+    
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteNamespaceDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   SourceLocation IdentLoc;
@@ -89,12 +91,12 @@ Decl *Parser::ParseNamespace(unsigned Context,
     if (InlineLoc.isValid())
       Diag(InlineLoc, diag::err_inline_namespace_alias)
           << FixItHint::CreateRemoval(InlineLoc);
-
     return ParseNamespaceAlias(NamespaceLoc, IdentLoc, Ident, DeclEnd);
   }
 
 
-  if (Tok.isNot(tok::l_brace)) {
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  if (T.consumeOpen()) {
     if (!ExtraIdent.empty()) {
       Diag(ExtraNamespaceLoc[0], diag::err_nested_namespaces_with_double_colon)
           << SourceRange(ExtraNamespaceLoc.front(), ExtraIdentLoc.back());
@@ -104,8 +106,6 @@ Decl *Parser::ParseNamespace(unsigned Context,
     return 0;
   }
 
-  SourceLocation LBrace = ConsumeBrace();
-
   if (getCurScope()->isClassScope() || getCurScope()->isTemplateParamScope() || 
       getCurScope()->isInObjcMethodScope() || getCurScope()->getBlockParent() || 
       getCurScope()->getFnParent()) {
@@ -113,7 +113,7 @@ Decl *Parser::ParseNamespace(unsigned Context,
       Diag(ExtraNamespaceLoc[0], diag::err_nested_namespaces_with_double_colon)
           << SourceRange(ExtraNamespaceLoc.front(), ExtraIdentLoc.back());
     }
-    Diag(LBrace, diag::err_namespace_nonnamespace_scope);
+    Diag(T.getOpenLocation(), diag::err_namespace_nonnamespace_scope);
     SkipUntil(tok::r_brace, false);
     return 0;
   }
@@ -156,23 +156,23 @@ Decl *Parser::ParseNamespace(unsigned Context,
 
   Decl *NamespcDecl =
     Actions.ActOnStartNamespaceDef(getCurScope(), InlineLoc, NamespaceLoc,
-                                   IdentLoc, Ident, LBrace, attrs.getList());
+                                   IdentLoc, Ident, T.getOpenLocation(), 
+                                   attrs.getList());
 
   PrettyDeclStackTraceEntry CrashInfo(Actions, NamespcDecl, NamespaceLoc,
                                       "parsing namespace");
 
-  SourceLocation RBraceLoc;
   // Parse the contents of the namespace.  This includes parsing recovery on 
   // any improperly nested namespaces.
   ParseInnerNamespace(ExtraIdentLoc, ExtraIdent, ExtraNamespaceLoc, 0,
-                      InlineLoc, LBrace, attrs, RBraceLoc);
+                      InlineLoc, attrs, T);
 
   // Leave the namespace scope.
   NamespaceScope.Exit();
 
-  Actions.ActOnFinishNamespaceDef(NamespcDecl, RBraceLoc);
+  DeclEnd = T.getCloseLocation();
+  Actions.ActOnFinishNamespaceDef(NamespcDecl, DeclEnd);
 
-  DeclEnd = RBraceLoc;
   return NamespcDecl;
 }
 
@@ -181,9 +181,8 @@ void Parser::ParseInnerNamespace(std::vector<SourceLocation>& IdentLoc,
                                  std::vector<IdentifierInfo*>& Ident,
                                  std::vector<SourceLocation>& NamespaceLoc,
                                  unsigned int index, SourceLocation& InlineLoc,
-                                 SourceLocation& LBrace,
                                  ParsedAttributes& attrs,
-                                 SourceLocation& RBraceLoc) {
+                                 BalancedDelimiterTracker &Tracker) {
   if (index == Ident.size()) {
     while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
       ParsedAttributesWithRange attrs(AttrFactory);
@@ -191,7 +190,10 @@ void Parser::ParseInnerNamespace(std::vector<SourceLocation>& IdentLoc,
       MaybeParseMicrosoftAttributes(attrs);
       ParseExternalDeclaration(attrs);
     }
-    RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBrace);
+
+    // The caller is what called check -- we are simply calling
+    // the close for it.
+    Tracker.consumeClose();
 
     return;
   }
@@ -201,14 +203,15 @@ void Parser::ParseInnerNamespace(std::vector<SourceLocation>& IdentLoc,
   Decl *NamespcDecl =
     Actions.ActOnStartNamespaceDef(getCurScope(), SourceLocation(),
                                    NamespaceLoc[index], IdentLoc[index],
-                                   Ident[index], LBrace, attrs.getList());
+                                   Ident[index], Tracker.getOpenLocation(), 
+                                   attrs.getList());
 
   ParseInnerNamespace(IdentLoc, Ident, NamespaceLoc, ++index, InlineLoc,
-                      LBrace, attrs, RBraceLoc);
+                      attrs, Tracker);
 
   NamespaceScope.Exit();
 
-  Actions.ActOnFinishNamespaceDef(NamespcDecl, RBraceLoc);
+  Actions.ActOnFinishNamespaceDef(NamespcDecl, Tracker.getCloseLocation());
 }
 
 /// ParseNamespaceAlias - Parse the part after the '=' in a namespace
@@ -224,7 +227,8 @@ Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteNamespaceAliasDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   CXXScopeSpec SS;
@@ -262,7 +266,7 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, unsigned Context) {
   assert(Tok.is(tok::string_literal) && "Not a string literal!");
   llvm::SmallString<8> LangBuffer;
   bool Invalid = false;
-  llvm::StringRef Lang = PP.getSpelling(Tok, LangBuffer, &Invalid);
+  StringRef Lang = PP.getSpelling(Tok, LangBuffer, &Invalid);
   if (Invalid)
     return 0;
 
@@ -296,7 +300,8 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, unsigned Context) {
 
   ProhibitAttributes(attrs);
 
-  SourceLocation LBrace = ConsumeBrace();
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  T.consumeOpen();
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
     ParsedAttributesWithRange attrs(AttrFactory);
     MaybeParseCXX0XAttributes(attrs);
@@ -304,9 +309,9 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, unsigned Context) {
     ParseExternalDeclaration(attrs);
   }
 
-  SourceLocation RBrace = MatchRHSPunctuation(tok::r_brace, LBrace);
+  T.consumeClose();
   return Actions.ActOnFinishLinkageSpecification(getCurScope(), LinkageSpec,
-                                                 RBrace);
+                                                 T.getCloseLocation());
 }
 
 /// ParseUsingDirectiveOrDeclaration - Parse C++ using using-declaration or
@@ -317,13 +322,15 @@ Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
                                              ParsedAttributesWithRange &attrs,
                                                Decl **OwnedType) {
   assert(Tok.is(tok::kw_using) && "Not using token");
-
+  ObjCDeclContextSwitch ObjCDC(*this);
+  
   // Eat 'using'.
   SourceLocation UsingLoc = ConsumeToken();
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteUsing(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   // 'using namespace' means this is a using-directive.
@@ -344,7 +351,7 @@ Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
   ProhibitAttributes(attrs);
 
   return ParseUsingDeclaration(Context, TemplateInfo, UsingLoc, DeclEnd,
-                               AS_none, OwnedType);
+                                    AS_none, OwnedType);
 }
 
 /// ParseUsingDirective - Parse C++ using-directive, assumes
@@ -368,7 +375,8 @@ Decl *Parser::ParseUsingDirective(unsigned Context,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteUsingDirective(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   CXXScopeSpec SS;
@@ -514,7 +522,7 @@ Decl *Parser::ParseUsingDeclaration(unsigned Context,
 
     TypeAlias = ParseTypeName(0, TemplateInfo.Kind ?
                               Declarator::AliasTemplateContext :
-                              Declarator::AliasDeclContext, 0, AS, OwnedType);
+                              Declarator::AliasDeclContext, AS, OwnedType);
   } else
     // Parse (optional) attributes (most likely GNU strong-using extension).
     MaybeParseGNUAttributes(attrs);
@@ -540,6 +548,15 @@ Decl *Parser::ParseUsingDeclaration(unsigned Context,
     return 0;
   }
 
+  // "typename" keyword is allowed for identifiers only,
+  // because it may be a type definition.
+  if (IsTypeName && Name.getKind() != UnqualifiedId::IK_Identifier) {
+    Diag(Name.getSourceRange().getBegin(), diag::err_typename_identifiers_only)
+      << FixItHint::CreateRemoval(SourceRange(TypenameLoc));
+    // Proceed parsing, but reset the IsTypeName flag.
+    IsTypeName = false;
+  }
+
   if (IsAliasDecl) {
     TemplateParameterLists *TemplateParams = TemplateInfo.TemplateParams;
     MultiTemplateParamsArg TemplateParamsArg(Actions,
@@ -571,13 +588,12 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
 
   SourceLocation StaticAssertLoc = ConsumeToken();
 
-  if (Tok.isNot(tok::l_paren)) {
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.consumeOpen()) {
     Diag(Tok, diag::err_expected_lparen);
     return 0;
   }
 
-  SourceLocation LParenLoc = ConsumeParen();
-
   ExprResult AssertExpr(ParseConstantExpression());
   if (AssertExpr.isInvalid()) {
     SkipUntil(tok::semi);
@@ -597,7 +613,7 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
   if (AssertMessage.isInvalid())
     return 0;
 
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
 
   DeclEnd = Tok.getLocation();
   ExpectAndConsumeSemi(diag::err_expected_semi_after_static_assert);
@@ -605,7 +621,7 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
   return Actions.ActOnStaticAssertDeclaration(StaticAssertLoc,
                                               AssertExpr.take(),
                                               AssertMessage.take(),
-                                              RParenLoc);
+                                              T.getCloseLocation());
 }
 
 /// ParseDecltypeSpecifier - Parse a C++0x decltype specifier.
@@ -616,11 +632,9 @@ void Parser::ParseDecltypeSpecifier(DeclSpec &DS) {
   assert(Tok.is(tok::kw_decltype) && "Not a decltype specifier");
 
   SourceLocation StartLoc = ConsumeToken();
-  SourceLocation LParenLoc = Tok.getLocation();
-
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
-                       "decltype")) {
-    SkipUntil(tok::r_paren);
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen_after,
+                       "decltype", tok::r_paren)) {
     return;
   }
 
@@ -637,13 +651,8 @@ void Parser::ParseDecltypeSpecifier(DeclSpec &DS) {
   }
 
   // Match the ')'
-  SourceLocation RParenLoc;
-  if (Tok.is(tok::r_paren))
-    RParenLoc = ConsumeParen();
-  else
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
-
-  if (RParenLoc.isInvalid())
+  T.consumeClose();
+  if (T.getCloseLocation().isInvalid())
     return;
 
   const char *PrevSpec = 0;
@@ -659,11 +668,9 @@ void Parser::ParseUnderlyingTypeSpecifier(DeclSpec &DS) {
          "Not an underlying type specifier");
 
   SourceLocation StartLoc = ConsumeToken();
-  SourceLocation LParenLoc = Tok.getLocation();
-
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
-                       "__underlying_type")) {
-    SkipUntil(tok::r_paren);
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen_after,
+                       "__underlying_type", tok::r_paren)) {
     return;
   }
 
@@ -674,13 +681,8 @@ void Parser::ParseUnderlyingTypeSpecifier(DeclSpec &DS) {
   }
 
   // Match the ')'
-  SourceLocation RParenLoc;
-  if (Tok.is(tok::r_paren))
-    RParenLoc = ConsumeParen();
-  else
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
-
-  if (RParenLoc.isInvalid())
+  T.consumeClose();
+  if (T.getCloseLocation().isInvalid())
     return;
 
   const char *PrevSpec = 0;
@@ -851,7 +853,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   if (Tok.is(tok::code_completion)) {
     // Code completion for a struct, class, or union name.
     Actions.CodeCompleteTag(getCurScope(), TagType);
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   }
 
   // C++03 [temp.explicit] 14.7.2/8:
@@ -901,7 +903,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
        Tok.is(tok::kw___is_signed) ||
        Tok.is(tok::kw___is_unsigned) ||
        Tok.is(tok::kw___is_void))) {
-    // GNU libstdc++ 4.2 and libc++ uaw certain intrinsic names as the
+    // GNU libstdc++ 4.2 and libc++ use certain intrinsic names as the
     // name of struct templates, but some are keywords in GCC >= 4.3
     // and Clang. Therefore, when we see the token sequence "struct
     // X", make X into a normal identifier rather than a keyword, to
@@ -1136,7 +1138,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       // Build the class template specialization.
       TagOrTempResult
         = Actions.ActOnClassTemplateSpecialization(getCurScope(), TagType, TUK,
-                       StartLoc, SS,
+                       StartLoc, DS.getModulePrivateSpecLoc(), SS,
                        TemplateId->Template,
                        TemplateId->TemplateNameLoc,
                        TemplateId->LAngleLoc,
@@ -1188,6 +1190,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // Declaration or definition of a class type
     TagOrTempResult = Actions.ActOnTag(getCurScope(), TagType, TUK, StartLoc,
                                        SS, Name, NameLoc, attrs.getList(), AS,
+                                       DS.getModulePrivateSpecLoc(),
                                        TParams, Owned, IsDependent, false,
                                        false, clang::TypeResult());
 
@@ -1264,7 +1267,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     case tok::kw_typedef:         // struct foo {...} typedef   x;
     case tok::kw_register:        // struct foo {...} register  x;
     case tok::kw_auto:            // struct foo {...} auto      x;
-    case tok::kw_mutable:         // struct foo {...} mutable      x;
+    case tok::kw_mutable:         // struct foo {...} mutable   x;
+    case tok::kw_constexpr:       // struct foo {...} constexpr x;
       // As shown above, type qualifiers and storage class specifiers absolutely
       // can occur after class specifiers according to the grammar.  However,
       // almost no one actually writes code like this.  If we see one of these,
@@ -1321,7 +1325,7 @@ void Parser::ParseBaseClause(Decl *ClassDecl) {
   ConsumeToken();
 
   // Build up an array of parsed base specifiers.
-  llvm::SmallVector<CXXBaseSpecifier *, 8> BaseInfo;
+  SmallVector<CXXBaseSpecifier *, 8> BaseInfo;
 
   while (true) {
     // Parse a base-specifier.
@@ -1565,7 +1569,6 @@ bool Parser::isCXX0XFinalKeyword() const {
 ///       virt-specifier:
 ///         override
 ///         final
-///         new
 /// 
 ///       pure-specifier:
 ///         '= 0'
@@ -1574,6 +1577,7 @@ bool Parser::isCXX0XFinalKeyword() const {
 ///         '=' constant-expression
 ///
 void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
+                                            AttributeList *AccessAttrs,
                                        const ParsedTemplateInfo &TemplateInfo,
                                        ParsingDeclRAIIObject *TemplateDiags) {
   if (Tok.is(tok::at)) {
@@ -1640,7 +1644,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
            "Nested template improperly parsed?");
     SourceLocation DeclEnd;
     ParseDeclarationStartingWithTemplate(Declarator::MemberContext, DeclEnd,
-                                         AS);
+                                         AS, AccessAttrs);
     return;
   }
 
@@ -1649,7 +1653,8 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     // __extension__ silences extension warnings in the subexpression.
     ExtensionRAIIObject O(Diags);  // Use RAII to do this.
     ConsumeToken();
-    return ParseCXXClassMemberDeclaration(AS, TemplateInfo, TemplateDiags);
+    return ParseCXXClassMemberDeclaration(AS, AccessAttrs,
+                                          TemplateInfo, TemplateDiags);
   }
 
   // Don't parse FOO:BAR as if it were a typo for FOO::BAR, in this context it
@@ -1699,7 +1704,9 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
 
   ParsingDeclarator DeclaratorInfo(*this, DS, Declarator::MemberContext);
   VirtSpecifiers VS;
-  ExprResult Init;
+
+  // Hold late-parsed attributes so we can attach a Decl to them later.
+  LateParsedAttrList LateParsedAttrs;
 
   if (Tok.isNot(tok::colon)) {
     // Don't parse FOO:BAR as if it were a typo for FOO::BAR.
@@ -1719,11 +1726,12 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     ParseOptionalCXX0XVirtSpecifierSeq(VS);
 
     // If attributes exist after the declarator, but before an '{', parse them.
-    MaybeParseGNUAttributes(DeclaratorInfo);
+    MaybeParseGNUAttributes(DeclaratorInfo, &LateParsedAttrs);
 
     // MSVC permits pure specifier on inline functions declared at class scope.
     // Hence check for =0 before checking for function definition.
-    if (getLang().Microsoft && Tok.is(tok::equal) &&
+    ExprResult Init;
+    if (getLang().MicrosoftExt && Tok.is(tok::equal) &&
         DeclaratorInfo.isFunctionDeclarator() && 
         NextToken().is(tok::numeric_constant)) {
       ConsumeToken();
@@ -1776,7 +1784,14 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
         return;
       }
 
-      ParseCXXInlineMethodDef(AS, DeclaratorInfo, TemplateInfo, VS, Init);
+      Decl *FunDecl =
+        ParseCXXInlineMethodDef(AS, AccessAttrs, DeclaratorInfo, TemplateInfo,
+                                VS, Init);
+
+      for (unsigned i = 0, ni = LateParsedAttrs.size(); i < ni; ++i) {
+        LateParsedAttrs[i]->setDecl(FunDecl);
+      }
+      LateParsedAttrs.clear();
 
       // Consume the ';' - it's optional unless we have a delete or default
       if (Tok.is(tok::semi)) {
@@ -1791,7 +1806,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
   //   member-declarator
   //   member-declarator-list ',' member-declarator
 
-  llvm::SmallVector<Decl *, 8> DeclsInGroup;
+  SmallVector<Decl *, 8> DeclsInGroup;
   ExprResult BitfieldSize;
 
   while (1) {
@@ -1818,31 +1833,25 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     }
 
     // If attributes exist after the declarator, parse them.
-    MaybeParseGNUAttributes(DeclaratorInfo);
+    MaybeParseGNUAttributes(DeclaratorInfo, &LateParsedAttrs);
 
     // FIXME: When g++ adds support for this, we'll need to check whether it
     // goes before or after the GNU attributes and __asm__.
     ParseOptionalCXX0XVirtSpecifierSeq(VS);
 
+    bool HasInitializer = false;
     bool HasDeferredInitializer = false;
     if (Tok.is(tok::equal) || Tok.is(tok::l_brace)) {
       if (BitfieldSize.get()) {
         Diag(Tok, diag::err_bitfield_member_init);
         SkipUntil(tok::comma, true, true);
       } else {
+        HasInitializer = true;
         HasDeferredInitializer = !DeclaratorInfo.isDeclarationOfFunction() &&
           DeclaratorInfo.getDeclSpec().getStorageClassSpec()
             != DeclSpec::SCS_static &&
           DeclaratorInfo.getDeclSpec().getStorageClassSpec()
             != DeclSpec::SCS_typedef;
-
-        if (!HasDeferredInitializer) {
-          SourceLocation EqualLoc;
-          Init = ParseCXXMemberInitializer(
-            DeclaratorInfo.isDeclarationOfFunction(), EqualLoc);
-          if (Init.isInvalid())
-            SkipUntil(tok::comma, true, true);
-        }
       }
     }
 
@@ -1854,32 +1863,30 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     if (DS.isFriendSpecified()) {
       // TODO: handle initializers, bitfields, 'delete'
       ThisDecl = Actions.ActOnFriendFunctionDecl(getCurScope(), DeclaratorInfo,
-                                                 /*IsDefinition*/ false,
                                                  move(TemplateParams));
     } else {
       ThisDecl = Actions.ActOnCXXMemberDeclarator(getCurScope(), AS,
                                                   DeclaratorInfo,
                                                   move(TemplateParams),
                                                   BitfieldSize.release(),
-                                                  VS, Init.release(),
-                                                  HasDeferredInitializer,
-                                                  /*IsDefinition*/ false);
+                                                  VS, HasDeferredInitializer);
+      if (AccessAttrs)
+        Actions.ProcessDeclAttributeList(getCurScope(), ThisDecl, AccessAttrs,
+                                         false, true);
     }
-    if (ThisDecl)
-      DeclsInGroup.push_back(ThisDecl);
-
-    if (DeclaratorInfo.isFunctionDeclarator() &&
-        DeclaratorInfo.getDeclSpec().getStorageClassSpec()
-          != DeclSpec::SCS_typedef) {
-      HandleMemberFunctionDefaultArgs(DeclaratorInfo, ThisDecl);
+    
+    // Set the Decl for any late parsed attributes
+    for (unsigned i = 0, ni = LateParsedAttrs.size(); i < ni; ++i) {
+      LateParsedAttrs[i]->setDecl(ThisDecl);
     }
+    LateParsedAttrs.clear();
 
-    DeclaratorInfo.complete(ThisDecl);
-
+    // Handle the initializer.
     if (HasDeferredInitializer) {
+      // The initializer was deferred; parse it and cache the tokens.
       if (!getLang().CPlusPlus0x)
         Diag(Tok, diag::warn_nonstatic_member_init_accepted_as_extension);
-
+      
       if (DeclaratorInfo.isArrayOfUnknownBound()) {
         // C++0x [dcl.array]p3: An array bound may also be omitted when the
         // declarator is followed by an initializer. 
@@ -1892,7 +1899,35 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
         ThisDecl->setInvalidDecl();
       } else
         ParseCXXNonStaticMemberInitializer(ThisDecl);
+    } else if (HasInitializer) {
+      // Normal initializer.
+      SourceLocation EqualLoc;
+      ExprResult Init
+        = ParseCXXMemberInitializer(DeclaratorInfo.isDeclarationOfFunction(), 
+                                    EqualLoc);
+      if (Init.isInvalid())
+        SkipUntil(tok::comma, true, true);
+      else if (ThisDecl)
+        Actions.AddInitializerToDecl(ThisDecl, Init.get(), false,
+                                   DS.getTypeSpecType() == DeclSpec::TST_auto);
+    } else if (ThisDecl && DS.getStorageClassSpec() == DeclSpec::SCS_static) {
+      // No initializer.
+      Actions.ActOnUninitializedDecl(ThisDecl, 
+                                   DS.getTypeSpecType() == DeclSpec::TST_auto);
     }
+    
+    if (ThisDecl) {
+      Actions.FinalizeDeclaration(ThisDecl);
+      DeclsInGroup.push_back(ThisDecl);
+    }
+    
+    if (DeclaratorInfo.isFunctionDeclarator() &&
+        DeclaratorInfo.getDeclSpec().getStorageClassSpec()
+          != DeclSpec::SCS_typedef) {
+      HandleMemberFunctionDefaultArgs(DeclaratorInfo, ThisDecl);
+    }
+
+    DeclaratorInfo.complete(ThisDecl);
 
     // If we don't have a comma, it is either the end of the list (a ';')
     // or an error, bail out.
@@ -1905,8 +1940,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     // Parse the next declarator.
     DeclaratorInfo.clear();
     VS.clear();
-    BitfieldSize = 0;
-    Init = 0;
+    BitfieldSize = true;
 
     // Attributes are only allowed on the second declarator.
     MaybeParseGNUAttributes(DeclaratorInfo);
@@ -1973,7 +2007,6 @@ ExprResult Parser::ParseCXXMemberInitializer(bool IsFunction,
         return ExprResult();
       }
     } else if (Tok.is(tok::kw_default)) {
-      Diag(ConsumeToken(), diag::err_default_special_members);
       if (IsFunction)
         Diag(Tok, diag::err_default_delete_in_multiple_declaration)
           << 0 /* default */;
@@ -2067,12 +2100,12 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   }
 
   assert(Tok.is(tok::l_brace));
-
-  SourceLocation LBraceLoc = ConsumeBrace();
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  T.consumeOpen();
 
   if (TagDecl)
     Actions.ActOnStartCXXMemberDeclarations(getCurScope(), TagDecl, FinalLoc,
-                                            LBraceLoc);
+                                            T.getOpenLocation());
 
   // C++ 11p3: Members of a class defined with the keyword class are private
   // by default. Members of a class defined with the keywords struct or union
@@ -2082,14 +2115,14 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
     CurAS = AS_private;
   else
     CurAS = AS_public;
+  ParsedAttributes AccessAttrs(AttrFactory);
 
-  SourceLocation RBraceLoc;
   if (TagDecl) {
     // While we still have something to read, read the member-declarations.
     while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
       // Each iteration of this loop reads one member-declaration.
 
-      if (getLang().Microsoft && (Tok.is(tok::kw___if_exists) ||
+      if (getLang().MicrosoftExt && (Tok.is(tok::kw___if_exists) ||
           Tok.is(tok::kw___if_not_exists))) {
         ParseMicrosoftIfExistsClassDeclaration((DeclSpec::TST)TagType, CurAS);
         continue;
@@ -2109,22 +2142,41 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
         // Current token is a C++ access specifier.
         CurAS = AS;
         SourceLocation ASLoc = Tok.getLocation();
+        unsigned TokLength = Tok.getLength();
         ConsumeToken();
-        if (Tok.is(tok::colon))
-          Actions.ActOnAccessSpecifier(AS, ASLoc, Tok.getLocation());
-        else
-          Diag(Tok, diag::err_expected_colon);
-        ConsumeToken();
+        AccessAttrs.clear();
+        MaybeParseGNUAttributes(AccessAttrs);
+
+        SourceLocation EndLoc;
+        if (Tok.is(tok::colon)) {
+          EndLoc = Tok.getLocation();
+          if (Actions.ActOnAccessSpecifier(AS, ASLoc, EndLoc,
+                                           AccessAttrs.getList())) {
+            // found another attribute than only annotations
+            AccessAttrs.clear();
+          }
+          ConsumeToken();
+        } else if (Tok.is(tok::semi)) {
+          EndLoc = Tok.getLocation();
+          ConsumeToken();
+          Diag(EndLoc, diag::err_expected_colon) 
+            << FixItHint::CreateReplacement(EndLoc, ":");
+        } else {
+          EndLoc = ASLoc.getLocWithOffset(TokLength);
+          Diag(EndLoc, diag::err_expected_colon) 
+            << FixItHint::CreateInsertion(EndLoc, ":");
+        }
+        Actions.ActOnAccessSpecifier(AS, ASLoc, EndLoc);
         continue;
       }
 
       // FIXME: Make sure we don't have a template here.
 
       // Parse all the comma separated declarators.
-      ParseCXXClassMemberDeclaration(CurAS);
+      ParseCXXClassMemberDeclaration(CurAS, AccessAttrs.getList());
     }
 
-    RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
+    T.consumeClose();
   } else {
     SkipUntil(tok::r_brace, false, false);
   }
@@ -2135,7 +2187,8 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
 
   if (TagDecl)
     Actions.ActOnFinishCXXMemberSpecification(getCurScope(), RecordLoc, TagDecl,
-                                              LBraceLoc, RBraceLoc,
+                                              T.getOpenLocation(), 
+                                              T.getCloseLocation(),
                                               attrs.getList());
 
   // C++0x [class.mem]p2: Within the class member-specification, the class is
@@ -2148,8 +2201,10 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   if (TagDecl && NonNestedClass) {
     // We are not inside a nested class. This class and its nested classes
     // are complete and we can parse the delayed portions of method
-    // declarations and the lexed inline method definitions.
+    // declarations and the lexed inline method definitions, along with any
+    // delayed attributes.
     SourceLocation SavedPrevTokLocation = PrevTokLocation;
+    ParseLexedAttributes(getCurrentClass());
     ParseLexedMethodDeclarations(getCurrentClass());
     ParseLexedMemberInitializers(getCurrentClass());
     ParseLexedMethodDefs(getCurrentClass());
@@ -2157,7 +2212,8 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   }
 
   if (TagDecl)
-    Actions.ActOnTagFinishDefinition(getCurScope(), TagDecl, RBraceLoc);
+    Actions.ActOnTagFinishDefinition(getCurScope(), TagDecl, 
+                                     T.getCloseLocation());
 
   // Leave the class scope.
   ParsingDef.Pop();
@@ -2192,7 +2248,7 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
   PoisonSEHIdentifiersRAIIObject PoisonSEHIdentifiers(*this, true);
   SourceLocation ColonLoc = ConsumeToken();
 
-  llvm::SmallVector<CXXCtorInitializer*, 4> MemInitializers;
+  SmallVector<CXXCtorInitializer*, 4> MemInitializers;
   bool AnyErrors = false;
 
   do {
@@ -2200,7 +2256,7 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
       Actions.CodeCompleteConstructorInitializer(ConstructorDecl, 
                                                  MemInitializers.data(), 
                                                  MemInitializers.size());
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     } else {
       MemInitResult MemInit = ParseMemInitializer(ConstructorDecl);
       if (!MemInit.isInvalid())
@@ -2270,11 +2326,20 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
 
   // Parse the '('.
   if (getLang().CPlusPlus0x && Tok.is(tok::l_brace)) {
-    // FIXME: Do something with the braced-init-list.
-    ParseBraceInitializer();
-    return true;
+    ExprResult InitList = ParseBraceInitializer();
+    if (InitList.isInvalid())
+      return true;
+
+    SourceLocation EllipsisLoc;
+    if (Tok.is(tok::ellipsis))
+      EllipsisLoc = ConsumeToken();
+
+    return Actions.ActOnMemInitializer(ConstructorDecl, getCurScope(), SS, II,
+                                       TemplateTypeTy, IdLoc, InitList.take(),
+                                       EllipsisLoc);
   } else if(Tok.is(tok::l_paren)) {
-    SourceLocation LParenLoc = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
 
     // Parse the optional expression-list.
     ExprVector ArgExprs(Actions);
@@ -2284,7 +2349,7 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
       return true;
     }
 
-    SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    T.consumeClose();
 
     SourceLocation EllipsisLoc;
     if (Tok.is(tok::ellipsis))
@@ -2292,8 +2357,8 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
 
     return Actions.ActOnMemInitializer(ConstructorDecl, getCurScope(), SS, II,
                                        TemplateTypeTy, IdLoc,
-                                       LParenLoc, ArgExprs.take(),
-                                       ArgExprs.size(), RParenLoc,
+                                       T.getOpenLocation(), ArgExprs.take(),
+                                       ArgExprs.size(), T.getCloseLocation(),
                                        EllipsisLoc);
   }
 
@@ -2313,8 +2378,8 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
 ///         'noexcept' '(' constant-expression ')'
 ExceptionSpecificationType
 Parser::MaybeParseExceptionSpecification(SourceRange &SpecificationRange,
-                    llvm::SmallVectorImpl<ParsedType> &DynamicExceptions,
-                    llvm::SmallVectorImpl<SourceRange> &DynamicExceptionRanges,
+                    SmallVectorImpl<ParsedType> &DynamicExceptions,
+                    SmallVectorImpl<SourceRange> &DynamicExceptionRanges,
                     ExprResult &NoexceptExpr) {
   ExceptionSpecificationType Result = EST_None;
 
@@ -2339,7 +2404,8 @@ Parser::MaybeParseExceptionSpecification(SourceRange &SpecificationRange,
   SourceLocation KeywordLoc = ConsumeToken();
   if (Tok.is(tok::l_paren)) {
     // There is an argument.
-    SourceLocation LParenLoc = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
     NoexceptType = EST_ComputedNoexcept;
     NoexceptExpr = ParseConstantExpression();
     // The argument must be contextually convertible to bool. We use
@@ -2347,8 +2413,8 @@ Parser::MaybeParseExceptionSpecification(SourceRange &SpecificationRange,
     if (!NoexceptExpr.isInvalid())
       NoexceptExpr = Actions.ActOnBooleanCondition(getCurScope(), KeywordLoc,
                                                    NoexceptExpr.get());
-    SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
-    NoexceptRange = SourceRange(KeywordLoc, RParenLoc);
+    T.consumeClose();
+    NoexceptRange = SourceRange(KeywordLoc, T.getCloseLocation());
   } else {
     // There is no argument.
     NoexceptType = EST_BasicNoexcept;
@@ -2386,27 +2452,26 @@ Parser::MaybeParseExceptionSpecification(SourceRange &SpecificationRange,
 ///
 ExceptionSpecificationType Parser::ParseDynamicExceptionSpecification(
                                   SourceRange &SpecificationRange,
-                                  llvm::SmallVectorImpl<ParsedType> &Exceptions,
-                                  llvm::SmallVectorImpl<SourceRange> &Ranges) {
+                                  SmallVectorImpl<ParsedType> &Exceptions,
+                                  SmallVectorImpl<SourceRange> &Ranges) {
   assert(Tok.is(tok::kw_throw) && "expected throw");
 
   SpecificationRange.setBegin(ConsumeToken());
-
-  if (!Tok.is(tok::l_paren)) {
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.consumeOpen()) {
     Diag(Tok, diag::err_expected_lparen_after) << "throw";
     SpecificationRange.setEnd(SpecificationRange.getBegin());
     return EST_DynamicNone;
   }
-  SourceLocation LParenLoc = ConsumeParen();
 
   // Parse throw(...), a Microsoft extension that means "this function
   // can throw anything".
   if (Tok.is(tok::ellipsis)) {
     SourceLocation EllipsisLoc = ConsumeToken();
-    if (!getLang().Microsoft)
+    if (!getLang().MicrosoftExt)
       Diag(EllipsisLoc, diag::ext_ellipsis_exception_spec);
-    SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
-    SpecificationRange.setEnd(RParenLoc);
+    T.consumeClose();
+    SpecificationRange.setEnd(T.getCloseLocation());
     return EST_MSAny;
   }
 
@@ -2436,13 +2501,14 @@ ExceptionSpecificationType Parser::ParseDynamicExceptionSpecification(
       break;
   }
 
-  SpecificationRange.setEnd(MatchRHSPunctuation(tok::r_paren, LParenLoc));
+  T.consumeClose();
+  SpecificationRange.setEnd(T.getCloseLocation());
   return Exceptions.empty() ? EST_DynamicNone : EST_Dynamic;
 }
 
 /// ParseTrailingReturnType - Parse a trailing return type on a new-style
 /// function declaration.
-TypeResult Parser::ParseTrailingReturnType() {
+TypeResult Parser::ParseTrailingReturnType(SourceRange &Range) {
   assert(Tok.is(tok::arrow) && "expected arrow");
 
   ConsumeToken();
@@ -2454,8 +2520,6 @@ TypeResult Parser::ParseTrailingReturnType() {
   //
   // struct X is parsed as class definition because of the trailing
   // brace.
-
-  SourceRange Range;
   return ParseTypeName(&Range);
 }
 
@@ -2519,11 +2583,12 @@ void Parser::PopParsingClass(Sema::ParsingClassState state) {
   Victim->TemplateScope = getCurScope()->getParent()->isTemplateParamScope();
 }
 
-/// ParseCXX0XAttributes - Parse a C++0x attribute-specifier. Currently only
-/// parses standard attributes.
+/// ParseCXX0XAttributeSpecifier - Parse a C++0x attribute-specifier. Currently
+/// only parses standard attributes.
 ///
 /// [C++0x] attribute-specifier:
 ///         '[' '[' attribute-list ']' ']'
+///         alignment-specifier
 ///
 /// [C++0x] attribute-list:
 ///         attribute[opt]
@@ -2554,12 +2619,18 @@ void Parser::PopParsingClass(Sema::ParsingClassState state) {
 ///         '[' balanced-token-seq ']'
 ///         '{' balanced-token-seq '}'
 ///         any token but '(', ')', '[', ']', '{', or '}'
-void Parser::ParseCXX0XAttributes(ParsedAttributesWithRange &attrs,
-                                  SourceLocation *endLoc) {
+void Parser::ParseCXX0XAttributeSpecifier(ParsedAttributes &attrs,
+                                          SourceLocation *endLoc) {
+  if (Tok.is(tok::kw_alignas)) {
+    Diag(Tok.getLocation(), diag::warn_cxx98_compat_alignas);
+    ParseAlignmentSpecifier(attrs, endLoc);
+    return;
+  }
+
   assert(Tok.is(tok::l_square) && NextToken().is(tok::l_square)
       && "Not a C++0x attribute list");
 
-  SourceLocation StartLoc = Tok.getLocation(), Loc;
+  Diag(Tok.getLocation(), diag::warn_cxx98_compat_attribute);
 
   ConsumeBracket();
   ConsumeBracket();
@@ -2617,29 +2688,6 @@ void Parser::ParseCXX0XAttributes(ParsedAttributesWithRange &attrs,
         break;
       }
 
-      // One argument; must be a type-id or assignment-expression
-      case AttributeList::AT_aligned: {
-        if (Tok.isNot(tok::l_paren)) {
-          Diag(Tok.getLocation(), diag::err_cxx0x_attribute_requires_arguments)
-            << AttrName->getName();
-          break;
-        }
-        SourceLocation ParamLoc = ConsumeParen();
-
-        ExprResult ArgExpr = ParseCXX0XAlignArgument(ParamLoc);
-
-        MatchRHSPunctuation(tok::r_paren, ParamLoc);
-
-        ExprVector ArgExprs(Actions);
-        ArgExprs.push_back(ArgExpr.release());
-        attrs.addNew(AttrName, AttrLoc, 0, AttrLoc,
-                     0, ParamLoc, ArgExprs.take(), 1,
-                     false, true);
-
-        AttrParsed = true;
-        break;
-      }
-
       // Silence warnings
       default: break;
       }
@@ -2655,31 +2703,27 @@ void Parser::ParseCXX0XAttributes(ParsedAttributesWithRange &attrs,
 
   if (ExpectAndConsume(tok::r_square, diag::err_expected_rsquare))
     SkipUntil(tok::r_square, false);
-  Loc = Tok.getLocation();
+  if (endLoc)
+    *endLoc = Tok.getLocation();
   if (ExpectAndConsume(tok::r_square, diag::err_expected_rsquare))
     SkipUntil(tok::r_square, false);
-
-  attrs.Range = SourceRange(StartLoc, Loc);
 }
 
-/// ParseCXX0XAlignArgument - Parse the argument to C++0x's [[align]]
-/// attribute.
+/// ParseCXX0XAttributes - Parse a C++0x attribute-specifier-seq.
 ///
-/// FIXME: Simply returns an alignof() expression if the argument is a
-/// type. Ideally, the type should be propagated directly into Sema.
-///
-/// [C++0x] 'align' '(' type-id ')'
-/// [C++0x] 'align' '(' assignment-expression ')'
-ExprResult Parser::ParseCXX0XAlignArgument(SourceLocation Start) {
-  if (isTypeIdInParens()) {
-    EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
-    SourceLocation TypeLoc = Tok.getLocation();
-    ParsedType Ty = ParseTypeName().get();
-    SourceRange TypeRange(Start, Tok.getLocation());
-    return Actions.ActOnUnaryExprOrTypeTraitExpr(TypeLoc, UETT_AlignOf, true,
-                                                Ty.getAsOpaquePtr(), TypeRange);
-  } else
-    return ParseConstantExpression();
+/// attribute-specifier-seq:
+///       attribute-specifier-seq[opt] attribute-specifier
+void Parser::ParseCXX0XAttributes(ParsedAttributesWithRange &attrs,
+                                  SourceLocation *endLoc) {
+  SourceLocation StartLoc = Tok.getLocation(), Loc;
+  if (!endLoc)
+    endLoc = &Loc;
+
+  do {
+    ParseCXX0XAttributeSpecifier(attrs, endLoc);
+  } while (isCXX0XAttributeSpecifier());
+
+  attrs.Range = SourceRange(StartLoc, *endLoc);
 }
 
 /// ParseMicrosoftAttributes - Parse a Microsoft attribute [Attr]
@@ -2753,7 +2797,7 @@ void Parser::ParseMicrosoftIfExistsClassDeclaration(DeclSpec::TST TagType,
     }
 
     // Parse all the comma separated declarators.
-    ParseCXXClassMemberDeclaration(CurAS);
+    ParseCXXClassMemberDeclaration(CurAS, 0);
   }
 
   if (Tok.isNot(tok::r_brace)) {
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp
index fc64ae022654..bc8bbf564005 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp
@@ -214,7 +214,8 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
 ExprResult Parser::ParseAssignmentExpression() {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return ExprError();
   }
 
   if (Tok.is(tok::kw_throw))
@@ -303,23 +304,23 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
         // Eat the colon.
         ColonLoc = ConsumeToken();
       } else {
-        // Otherwise, we're missing a ':'.  Assume that this was a typo that the
-        // user forgot.  If we're not in a macro instantiation, we can suggest a
-        // fixit hint.  If there were two spaces before the current token,
+        // Otherwise, we're missing a ':'.  Assume that this was a typo that
+        // the user forgot. If we're not in a macro expansion, we can suggest
+        // a fixit hint. If there were two spaces before the current token,
         // suggest inserting the colon in between them, otherwise insert ": ".
         SourceLocation FILoc = Tok.getLocation();
         const char *FIText = ": ";
         const SourceManager &SM = PP.getSourceManager();
         if (FILoc.isFileID() || PP.isAtStartOfMacroExpansion(FILoc)) {
-          FILoc = SM.getInstantiationLoc(FILoc);
+          FILoc = SM.getExpansionLoc(FILoc);
           bool IsInvalid = false;
           const char *SourcePtr =
-            SM.getCharacterData(FILoc.getFileLocWithOffset(-1), &IsInvalid);
+            SM.getCharacterData(FILoc.getLocWithOffset(-1), &IsInvalid);
           if (!IsInvalid && *SourcePtr == ' ') {
             SourcePtr =
-              SM.getCharacterData(FILoc.getFileLocWithOffset(-2), &IsInvalid);
+              SM.getCharacterData(FILoc.getLocWithOffset(-2), &IsInvalid);
             if (!IsInvalid && *SourcePtr == ' ') {
-              FILoc = FILoc.getFileLocWithOffset(-1);
+              FILoc = FILoc.getLocWithOffset(-1);
               FIText = ":";
             }
           }
@@ -336,7 +337,7 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
     // goes through a special hook that takes the left-hand side into account.
     if (Tok.is(tok::code_completion) && NextTokPrec == prec::Assignment) {
       Actions.CodeCompleteAssignmentRHS(getCurScope(), LHS.get());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
       return ExprError();
     }
     
@@ -769,6 +770,9 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     break;
   }
   case tok::char_constant:     // constant: character-constant
+  case tok::wide_char_constant:
+  case tok::utf16_char_constant:
+  case tok::utf32_char_constant:
     Res = Actions.ActOnCharacterConstant(Tok);
     ConsumeToken();
     break;
@@ -780,6 +784,9 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     break;
   case tok::string_literal:    // primary-expression: string-literal
   case tok::wide_string_literal:
+  case tok::utf8_string_literal:
+  case tok::utf16_string_literal:
+  case tok::utf32_string_literal:
     Res = ParseStringLiteralExpression();
     break;
   case tok::kw__Generic:   // primary-expression: generic-selection [C1X 6.5.1]
@@ -913,6 +920,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw___int64:
   case tok::kw_signed:
   case tok::kw_unsigned:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_void:
@@ -1021,18 +1029,21 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 
   case tok::kw_noexcept: { // [C++0x] 'noexcept' '(' expression ')'
     SourceLocation KeyLoc = ConsumeToken();
-    SourceLocation LParen = Tok.getLocation();
-    if (ExpectAndConsume(tok::l_paren,
-                         diag::err_expected_lparen_after, "noexcept"))
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+
+    if (T.expectAndConsume(diag::err_expected_lparen_after, "noexcept"))
       return ExprError();
     // C++ [expr.unary.noexcept]p1:
     //   The noexcept operator determines whether the evaluation of its operand,
     //   which is an unevaluated operand, can throw an exception.
     EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
     ExprResult Result = ParseExpression();
-    SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+
+    T.consumeClose();
+
     if (!Result.isInvalid())
-      Result = Actions.ActOnNoexceptExpr(KeyLoc, LParen, Result.take(), RParen);
+      Result = Actions.ActOnNoexceptExpr(KeyLoc, T.getOpenLocation(), 
+                                         Result.take(), T.getCloseLocation());
     return move(Result);
   }
 
@@ -1104,11 +1115,20 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     break;
   case tok::code_completion: {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
-    return ParseCastExpression(isUnaryExpression, isAddressOfOperand, 
-                               NotCastExpr, isTypeCast);
+    cutOffParsing();
+    return ExprError();
   }
   case tok::l_square:
+    if (getLang().CPlusPlus0x) {
+      if (getLang().ObjC1) {
+        Res = TryParseLambdaExpression();
+        if (Res.isInvalid())
+          Res = ParseObjCMessageExpression();
+        break;
+      }
+      Res = ParseLambdaExpression();
+      break;
+    }
     if (getLang().ObjC1) {
       Res = ParseObjCMessageExpression();
       break;
@@ -1154,9 +1174,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         return move(LHS);
         
       Actions.CodeCompletePostfixExpression(getCurScope(), LHS);
-      ConsumeCodeCompletionToken();
-      LHS = ExprError();
-      break;
+      cutOffParsing();
+      return ExprError();
         
     case tok::identifier:
       // If we see identifier: after an expression, and we're not already in a
@@ -1183,8 +1202,10 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       if (getLang().ObjC1 && Tok.isAtStartOfLine() &&
           isSimpleObjCMessageExpression())
         return move(LHS);
-          
-      Loc = ConsumeBracket();
+       
+      BalancedDelimiterTracker T(*this, tok::l_square);
+      T.consumeOpen();
+      Loc = T.getOpenLocation();
       ExprResult Idx;
       if (getLang().CPlusPlus0x && Tok.is(tok::l_brace))
         Idx = ParseBraceInitializer();
@@ -1200,7 +1221,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         LHS = ExprError();
 
       // Match the ']'.
-      MatchRHSPunctuation(tok::r_square, Loc);
+      T.consumeClose();
       break;
     }
 
@@ -1212,12 +1233,13 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       
       Expr *ExecConfig = 0;
 
+      BalancedDelimiterTracker LLLT(*this, tok::lesslessless);
+      BalancedDelimiterTracker PT(*this, tok::l_paren);
+
       if (OpKind == tok::lesslessless) {
         ExprVector ExecConfigExprs(Actions);
         CommaLocsTy ExecConfigCommaLocs;
-        SourceLocation LLLLoc, GGGLoc;
-
-        LLLLoc = ConsumeToken();
+        LLLT.consumeOpen();
 
         if (ParseExpressionList(ExecConfigExprs, ExecConfigCommaLocs)) {
           LHS = ExprError();
@@ -1225,11 +1247,9 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
 
         if (LHS.isInvalid()) {
           SkipUntil(tok::greatergreatergreater);
-        } else if (Tok.isNot(tok::greatergreatergreater)) {
-          MatchRHSPunctuation(tok::greatergreatergreater, LLLLoc);
+        } else if (LLLT.consumeClose()) {
+          // There was an error closing the brackets
           LHS = ExprError();
-        } else {
-          GGGLoc = ConsumeToken();
         }
 
         if (!LHS.isInvalid()) {
@@ -1241,14 +1261,17 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
 
         if (!LHS.isInvalid()) {
           ExprResult ECResult = Actions.ActOnCUDAExecConfigExpr(getCurScope(),
-                                     LLLLoc, move_arg(ExecConfigExprs), GGGLoc);
+                                    LLLT.getOpenLocation(), 
+                                    move_arg(ExecConfigExprs), 
+                                    LLLT.getCloseLocation());
           if (ECResult.isInvalid())
             LHS = ExprError();
           else
             ExecConfig = ECResult.get();
         }
       } else {
-        Loc = ConsumeParen();
+        PT.consumeOpen();
+        Loc = PT.getOpenLocation();
       }
 
       ExprVector ArgExprs(Actions);
@@ -1256,7 +1279,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteCall(getCurScope(), LHS.get(), 0, 0);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return ExprError();
       }
 
       if (OpKind == tok::l_paren || !LHS.isInvalid()) {
@@ -1272,7 +1296,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       if (LHS.isInvalid()) {
         SkipUntil(tok::r_paren);
       } else if (Tok.isNot(tok::r_paren)) {
-        MatchRHSPunctuation(tok::r_paren, Loc);
+        PT.consumeClose();
         LHS = ExprError();
       } else {
         assert((ArgExprs.size() == 0 || 
@@ -1281,7 +1305,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         LHS = Actions.ActOnCallExpr(getCurScope(), LHS.take(), Loc,
                                     move_arg(ArgExprs), Tok.getLocation(),
                                     ExecConfig);
-        ConsumeParen();
+        PT.consumeClose();
       }
 
       break;
@@ -1314,7 +1338,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         Actions.CodeCompleteMemberReferenceExpr(getCurScope(), LHS.get(),
                                                 OpLoc, OpKind == tok::arrow);
         
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return ExprError();
       }
       
       if (MayBePseudoDestructor && !LHS.isInvalid()) {
@@ -1334,7 +1359,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       if (ParseUnqualifiedId(SS, 
                              /*EnteringContext=*/false, 
                              /*AllowDestructorName=*/true,
-                             /*AllowConstructorName=*/ getLang().Microsoft, 
+                             /*AllowConstructorName=*/ getLang().MicrosoftExt, 
                              ObjectType,
                              Name))
         LHS = ExprError();
@@ -1473,11 +1498,14 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() {
     IdentifierInfo *Name = 0;
     SourceLocation NameLoc;
     if (Tok.is(tok::l_paren)) {
-      LParenLoc = ConsumeParen();
+      BalancedDelimiterTracker T(*this, tok::l_paren);
+      T.consumeOpen();
+      LParenLoc = T.getOpenLocation();
       if (Tok.is(tok::identifier)) {
         Name = Tok.getIdentifierInfo();
         NameLoc = ConsumeToken();
-        RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+        T.consumeClose();
+        RParenLoc = T.getCloseLocation();
         if (RParenLoc.isInvalid())
           RParenLoc = PP.getLocForEndOfToken(NameLoc);
       } else {
@@ -1564,11 +1592,13 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
     return ExprError(Diag(Tok, diag::err_expected_lparen_after_id)
                        << BuiltinII);
 
-  SourceLocation LParenLoc = ConsumeParen();
+  BalancedDelimiterTracker PT(*this, tok::l_paren);
+  PT.consumeOpen();
+
   // TODO: Build AST.
 
   switch (T) {
-  default: assert(0 && "Not a builtin primary expression!");
+  default: llvm_unreachable("Not a builtin primary expression!");
   case tok::kw___builtin_va_arg: {
     ExprResult Expr(ParseAssignmentExpression());
 
@@ -1607,7 +1637,7 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
     }
 
     // Keep track of the various subcomponents we see.
-    llvm::SmallVector<Sema::OffsetOfComponent, 4> Comps;
+    SmallVector<Sema::OffsetOfComponent, 4> Comps;
 
     Comps.push_back(Sema::OffsetOfComponent());
     Comps.back().isBrackets = false;
@@ -1634,7 +1664,9 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
         // offsetof-member-designator: offsetof-member-design '[' expression ']'
         Comps.push_back(Sema::OffsetOfComponent());
         Comps.back().isBrackets = true;
-        Comps.back().LocStart = ConsumeBracket();
+        BalancedDelimiterTracker ST(*this, tok::l_square);
+        ST.consumeOpen();
+        Comps.back().LocStart = ST.getOpenLocation();
         Res = ParseExpression();
         if (Res.isInvalid()) {
           SkipUntil(tok::r_paren);
@@ -1642,18 +1674,19 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
         }
         Comps.back().U.E = Res.release();
 
-        Comps.back().LocEnd =
-          MatchRHSPunctuation(tok::r_square, Comps.back().LocStart);
+        ST.consumeClose();
+        Comps.back().LocEnd = ST.getCloseLocation();
       } else {
         if (Tok.isNot(tok::r_paren)) {
-          MatchRHSPunctuation(tok::r_paren, LParenLoc);
+          PT.consumeClose();
           Res = ExprError();
         } else if (Ty.isInvalid()) {
           Res = ExprError();
         } else {
+          PT.consumeClose();
           Res = Actions.ActOnBuiltinOffsetOf(getCurScope(), StartLoc, TypeLoc,
-                                             Ty.get(), &Comps[0],
-                                             Comps.size(), ConsumeParen());
+                                             Ty.get(), &Comps[0], Comps.size(),
+                                             PT.getCloseLocation());
         }
         break;
       }
@@ -1753,7 +1786,11 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
                              SourceLocation &RParenLoc) {
   assert(Tok.is(tok::l_paren) && "Not a paren expr!");
   GreaterThanIsOperatorScope G(GreaterThanIsOperator, true);
-  SourceLocation OpenLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.consumeOpen())
+    return ExprError();
+  SourceLocation OpenLoc = T.getOpenLocation();
+
   ExprResult Result(true);
   bool isAmbiguousTypeId;
   CastTy = ParsedType();
@@ -1762,7 +1799,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     Actions.CodeCompleteOrdinaryName(getCurScope(), 
                  ExprType >= CompoundLiteral? Sema::PCC_ParenthesizedExpression
                                             : Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
     return ExprError();
   }
 
@@ -1806,7 +1843,8 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     }
              
     TypeResult Ty = ParseTypeName();
-    SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, OpenLoc);
+    T.consumeClose();
+    RParenLoc = T.getCloseLocation();
     ExprResult SubExpr = ParseCastExpression(/*isUnaryExpression=*/false);
     
     if (Ty.isInvalid() || SubExpr.isInvalid())
@@ -1825,9 +1863,11 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     // in which case we should treat it as type-id.
     // if stopIfCastExpr is false, we need to determine the context past the
     // parens, so we defer to ParseCXXAmbiguousParenExpression for that.
-    if (isAmbiguousTypeId && !stopIfCastExpr)
-      return ParseCXXAmbiguousParenExpression(ExprType, CastTy,
-                                              OpenLoc, RParenLoc);
+    if (isAmbiguousTypeId && !stopIfCastExpr) {
+      ExprResult res = ParseCXXAmbiguousParenExpression(ExprType, CastTy, T);
+      RParenLoc = T.getCloseLocation();
+      return res;
+    }
 
     // Parse the type declarator.
     DeclSpec DS(AttrFactory);
@@ -1851,11 +1891,8 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
                                               Ty.get(), 0);
     } else {          
       // Match the ')'.
-      if (Tok.is(tok::r_paren))
-        RParenLoc = ConsumeParen();
-      else
-        MatchRHSPunctuation(tok::r_paren, OpenLoc);
-
+      T.consumeClose();
+      RParenLoc = T.getCloseLocation();
       if (Tok.is(tok::l_brace)) {
         ExprType = CompoundLiteral;
         TypeResult Ty;
@@ -1939,11 +1976,8 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     return ExprError();
   }
 
-  if (Tok.is(tok::r_paren))
-    RParenLoc = ConsumeParen();
-  else
-    MatchRHSPunctuation(tok::r_paren, OpenLoc);
-
+  T.consumeClose();
+  RParenLoc = T.getCloseLocation();
   return move(Result);
 }
 
@@ -1978,7 +2012,7 @@ ExprResult Parser::ParseStringLiteralExpression() {
 
   // String concat.  Note that keywords like __func__ and __FUNCTION__ are not
   // considered to be strings for concatenation purposes.
-  llvm::SmallVector<Token, 4> StringToks;
+  SmallVector<Token, 4> StringToks;
 
   do {
     StringToks.push_back(Tok);
@@ -2007,8 +2041,8 @@ ExprResult Parser::ParseGenericSelectionExpression() {
   if (!getLang().C1X)
     Diag(KeyLoc, diag::ext_c1x_generic_selection);
 
-  SourceLocation LParenLoc = Tok.getLocation();
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen, ""))
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
     return ExprError();
 
   ExprResult ControllingExpr;
@@ -2074,11 +2108,12 @@ ExprResult Parser::ParseGenericSelectionExpression() {
     ConsumeToken();
   }
 
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
-  if (RParenLoc.isInvalid())
+  T.consumeClose();
+  if (T.getCloseLocation().isInvalid())
     return ExprError();
 
-  return Actions.ActOnGenericSelectionExpr(KeyLoc, DefaultLoc, RParenLoc,
+  return Actions.ActOnGenericSelectionExpr(KeyLoc, DefaultLoc, 
+                                           T.getCloseLocation(),
                                            ControllingExpr.release(),
                                            move_arg(Types), move_arg(Exprs));
 }
@@ -2104,8 +2139,8 @@ ExprResult Parser::ParseGenericSelectionExpression() {
 /// [C++0x]   assignment-expression
 /// [C++0x]   braced-init-list
 ///
-bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
-                            llvm::SmallVectorImpl<SourceLocation> &CommaLocs,
+bool Parser::ParseExpressionList(SmallVectorImpl<Expr*> &Exprs,
+                            SmallVectorImpl<SourceLocation> &CommaLocs,
                                  void (Sema::*Completer)(Scope *S, 
                                                            Expr *Data,
                                                            Expr **Args,
@@ -2117,7 +2152,8 @@ bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
         (Actions.*Completer)(getCurScope(), Data, Exprs.data(), Exprs.size());
       else
         Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return true;
     }
 
     ExprResult Expr;
@@ -2148,7 +2184,7 @@ bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
 void Parser::ParseBlockId() {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   }
   
   // Parse the specifier-qualifier-list piece.
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseExprCXX.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseExprCXX.cpp
index b32eeda24279..60166e880c04 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseExprCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseExprCXX.cpp
@@ -15,6 +15,7 @@
 #include "clang/Parse/Parser.h"
 #include "RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Scope.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "llvm/Support/ErrorHandling.h"
 
@@ -28,8 +29,7 @@ static int SelectDigraphErrorMessage(tok::TokenKind Kind) {
     case tok::kw_reinterpret_cast: return 3;
     case tok::kw_static_cast:      return 4;
     default:
-      assert(0 && "Unknown type for digraph error message.");
-      return -1;
+      llvm_unreachable("Unknown type for digraph error message.");
   }
 }
 
@@ -37,7 +37,7 @@ static int SelectDigraphErrorMessage(tok::TokenKind Kind) {
 static bool AreTokensAdjacent(Preprocessor &PP, Token &First, Token &Second) {
   SourceManager &SM = PP.getSourceManager();
   SourceLocation FirstLoc = SM.getSpellingLoc(First.getLocation());
-  SourceLocation FirstEnd = FirstLoc.getFileLocWithOffset(First.getLength());
+  SourceLocation FirstEnd = FirstLoc.getLocWithOffset(First.getLength());
   return FirstEnd == SM.getSpellingLoc(Second.getLocation());
 }
 
@@ -58,7 +58,7 @@ static void FixDigraph(Parser &P, Preprocessor &PP, Token &DigraphToken,
 
   // Update token information to reflect their change in token type.
   ColonToken.setKind(tok::coloncolon);
-  ColonToken.setLocation(ColonToken.getLocation().getFileLocWithOffset(-1));
+  ColonToken.setLocation(ColonToken.getLocation().getLocWithOffset(-1));
   ColonToken.setLength(2);
   DigraphToken.setKind(tok::less);
   DigraphToken.setLength(1);
@@ -69,6 +69,31 @@ static void FixDigraph(Parser &P, Preprocessor &PP, Token &DigraphToken,
     PP.EnterToken(DigraphToken);
 }
 
+// Check for '<::' which should be '< ::' instead of '[:' when following
+// a template name.
+void Parser::CheckForTemplateAndDigraph(Token &Next, ParsedType ObjectType,
+                                        bool EnteringContext,
+                                        IdentifierInfo &II, CXXScopeSpec &SS) {
+  if (!Next.is(tok::l_square) || Next.getLength() != 2)
+    return;
+
+  Token SecondToken = GetLookAheadToken(2);
+  if (!SecondToken.is(tok::colon) || !AreTokensAdjacent(PP, Next, SecondToken))
+    return;
+
+  TemplateTy Template;
+  UnqualifiedId TemplateName;
+  TemplateName.setIdentifier(&II, Tok.getLocation());
+  bool MemberOfUnknownSpecialization;
+  if (!Actions.isTemplateName(getCurScope(), SS, /*hasTemplateKeyword=*/false,
+                              TemplateName, ObjectType, EnteringContext,
+                              Template, MemberOfUnknownSpecialization))
+    return;
+
+  FixDigraph(*this, PP, Next, SecondToken, tok::kw_template,
+             /*AtDigraph*/false);
+}
+
 /// \brief Parse global scope or nested-name-specifier if present.
 ///
 /// Parses a C++ global scope specifier ('::') or nested-name-specifier (which
@@ -161,12 +186,13 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
         // Code completion for a nested-name-specifier, where the code
         // code completion token follows the '::'.
         Actions.CodeCompleteQualifiedId(getCurScope(), SS, EnteringContext);
-        SourceLocation ccLoc = ConsumeCodeCompletionToken();
         // Include code completion token into the range of the scope otherwise
         // when we try to annotate the scope tokens the dangling code completion
         // token will cause assertion in
         // Preprocessor::AnnotatePreviousCachedTokens.
-        SS.setEndLoc(ccLoc);
+        SS.setEndLoc(Tok.getLocation());
+        cutOffParsing();
+        return true;
       }
     }
 
@@ -339,28 +365,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
       continue;
     }
 
-    // Check for '<::' which should be '< ::' instead of '[:' when following
-    // a template name.
-    if (Next.is(tok::l_square) && Next.getLength() == 2) {
-      Token SecondToken = GetLookAheadToken(2);
-      if (SecondToken.is(tok::colon) &&
-          AreTokensAdjacent(PP, Next, SecondToken)) {
-        TemplateTy Template;
-        UnqualifiedId TemplateName;
-        TemplateName.setIdentifier(&II, Tok.getLocation());
-        bool MemberOfUnknownSpecialization;
-        if (Actions.isTemplateName(getCurScope(), SS,
-                                   /*hasTemplateKeyword=*/false,
-                                   TemplateName,
-                                   ObjectType,
-                                   EnteringContext,
-                                   Template,
-                                   MemberOfUnknownSpecialization)) {
-          FixDigraph(*this, PP, Next, SecondToken, tok::kw_template,
-                     /*AtDigraph*/false);
-        }
-      }
-    }
+    CheckForTemplateAndDigraph(Next, ObjectType, EnteringContext, II, SS);
 
     // nested-name-specifier:
     //   type-name '<'
@@ -396,7 +401,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
         // parse correctly as a template, so suggest the keyword 'template'
         // before 'getAs' and treat this as a dependent template name.
         unsigned DiagID = diag::err_missing_dependent_template_keyword;
-        if (getLang().Microsoft)
+        if (getLang().MicrosoftExt)
           DiagID = diag::warn_missing_dependent_template_keyword;
         
         Diag(Tok.getLocation(), DiagID)
@@ -504,6 +509,273 @@ ExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
   
 }
 
+/// ParseLambdaExpression - Parse a C++0x lambda expression.
+///
+///       lambda-expression:
+///         lambda-introducer lambda-declarator[opt] compound-statement
+///
+///       lambda-introducer:
+///         '[' lambda-capture[opt] ']'
+///
+///       lambda-capture:
+///         capture-default
+///         capture-list
+///         capture-default ',' capture-list
+///
+///       capture-default:
+///         '&'
+///         '='
+///
+///       capture-list:
+///         capture
+///         capture-list ',' capture
+///
+///       capture:
+///         identifier
+///         '&' identifier
+///         'this'
+///
+///       lambda-declarator:
+///         '(' parameter-declaration-clause ')' attribute-specifier[opt]
+///           'mutable'[opt] exception-specification[opt]
+///           trailing-return-type[opt]
+///
+ExprResult Parser::ParseLambdaExpression() {
+  // Parse lambda-introducer.
+  LambdaIntroducer Intro;
+
+  llvm::Optional<unsigned> DiagID(ParseLambdaIntroducer(Intro));
+  if (DiagID) {
+    Diag(Tok, DiagID.getValue());
+    SkipUntil(tok::r_square);
+  }
+
+  return ParseLambdaExpressionAfterIntroducer(Intro);
+}
+
+/// TryParseLambdaExpression - Use lookahead and potentially tentative
+/// parsing to determine if we are looking at a C++0x lambda expression, and parse
+/// it if we are.
+///
+/// If we are not looking at a lambda expression, returns ExprError().
+ExprResult Parser::TryParseLambdaExpression() {
+  assert(getLang().CPlusPlus0x
+         && Tok.is(tok::l_square)
+         && "Not at the start of a possible lambda expression.");
+
+  const Token Next = NextToken(), After = GetLookAheadToken(2);
+
+  // If lookahead indicates this is a lambda...
+  if (Next.is(tok::r_square) ||     // []
+      Next.is(tok::equal) ||        // [=
+      (Next.is(tok::amp) &&         // [&] or [&,
+       (After.is(tok::r_square) ||
+        After.is(tok::comma))) ||
+      (Next.is(tok::identifier) &&  // [identifier]
+       After.is(tok::r_square))) {
+    return ParseLambdaExpression();
+  }
+
+  // If lookahead indicates this is an Objective-C message...
+  if (Next.is(tok::identifier) && After.is(tok::identifier)) {
+    return ExprError();
+  }
+
+  LambdaIntroducer Intro;
+  if (TryParseLambdaIntroducer(Intro))
+    return ExprError();
+  return ParseLambdaExpressionAfterIntroducer(Intro);
+}
+
+/// ParseLambdaExpression - Parse a lambda introducer.
+///
+/// Returns a DiagnosticID if it hit something unexpected.
+llvm::Optional<unsigned> Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro) {
+  typedef llvm::Optional<unsigned> DiagResult;
+
+  assert(Tok.is(tok::l_square) && "Lambda expressions begin with '['.");
+  BalancedDelimiterTracker T(*this, tok::l_square);
+  T.consumeOpen();
+
+  Intro.Range.setBegin(T.getOpenLocation());
+
+  bool first = true;
+
+  // Parse capture-default.
+  if (Tok.is(tok::amp) &&
+      (NextToken().is(tok::comma) || NextToken().is(tok::r_square))) {
+    Intro.Default = LCD_ByRef;
+    ConsumeToken();
+    first = false;
+  } else if (Tok.is(tok::equal)) {
+    Intro.Default = LCD_ByCopy;
+    ConsumeToken();
+    first = false;
+  }
+
+  while (Tok.isNot(tok::r_square)) {
+    if (!first) {
+      if (Tok.isNot(tok::comma))
+        return DiagResult(diag::err_expected_comma_or_rsquare);
+      ConsumeToken();
+    }
+
+    first = false;
+
+    // Parse capture.
+    LambdaCaptureKind Kind = LCK_ByCopy;
+    SourceLocation Loc;
+    IdentifierInfo* Id = 0;
+
+    if (Tok.is(tok::kw_this)) {
+      Kind = LCK_This;
+      Loc = ConsumeToken();
+    } else {
+      if (Tok.is(tok::amp)) {
+        Kind = LCK_ByRef;
+        ConsumeToken();
+      }
+
+      if (Tok.is(tok::identifier)) {
+        Id = Tok.getIdentifierInfo();
+        Loc = ConsumeToken();
+      } else if (Tok.is(tok::kw_this)) {
+        // FIXME: If we want to suggest a fixit here, will need to return more
+        // than just DiagnosticID. Perhaps full DiagnosticBuilder that can be
+        // Clear()ed to prevent emission in case of tentative parsing?
+        return DiagResult(diag::err_this_captured_by_reference);
+      } else {
+        return DiagResult(diag::err_expected_capture);
+      }
+    }
+
+    Intro.addCapture(Kind, Loc, Id);
+  }
+
+  T.consumeClose();
+  Intro.Range.setEnd(T.getCloseLocation());
+
+  return DiagResult();
+}
+
+/// TryParseLambdaExpression - Tentatively parse a lambda introducer.
+///
+/// Returns true if it hit something unexpected.
+bool Parser::TryParseLambdaIntroducer(LambdaIntroducer &Intro) {
+  TentativeParsingAction PA(*this);
+
+  llvm::Optional<unsigned> DiagID(ParseLambdaIntroducer(Intro));
+
+  if (DiagID) {
+    PA.Revert();
+    return true;
+  }
+
+  PA.Commit();
+  return false;
+}
+
+/// ParseLambdaExpressionAfterIntroducer - Parse the rest of a lambda
+/// expression.
+ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
+                     LambdaIntroducer &Intro) {
+  // Parse lambda-declarator[opt].
+  DeclSpec DS(AttrFactory);
+  Declarator D(DS, Declarator::PrototypeContext);
+
+  if (Tok.is(tok::l_paren)) {
+    ParseScope PrototypeScope(this,
+                              Scope::FunctionPrototypeScope |
+                              Scope::DeclScope);
+
+    SourceLocation DeclLoc, DeclEndLoc;
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
+    DeclLoc = T.getOpenLocation();
+
+    // Parse parameter-declaration-clause.
+    ParsedAttributes Attr(AttrFactory);
+    llvm::SmallVector<DeclaratorChunk::ParamInfo, 16> ParamInfo;
+    SourceLocation EllipsisLoc;
+
+    if (Tok.isNot(tok::r_paren))
+      ParseParameterDeclarationClause(D, Attr, ParamInfo, EllipsisLoc);
+
+    T.consumeClose();
+    DeclEndLoc = T.getCloseLocation();
+
+    // Parse 'mutable'[opt].
+    SourceLocation MutableLoc;
+    if (Tok.is(tok::kw_mutable)) {
+      MutableLoc = ConsumeToken();
+      DeclEndLoc = MutableLoc;
+    }
+
+    // Parse exception-specification[opt].
+    ExceptionSpecificationType ESpecType = EST_None;
+    SourceRange ESpecRange;
+    llvm::SmallVector<ParsedType, 2> DynamicExceptions;
+    llvm::SmallVector<SourceRange, 2> DynamicExceptionRanges;
+    ExprResult NoexceptExpr;
+    ESpecType = MaybeParseExceptionSpecification(ESpecRange,
+                                                 DynamicExceptions,
+                                                 DynamicExceptionRanges,
+                                                 NoexceptExpr);
+
+    if (ESpecType != EST_None)
+      DeclEndLoc = ESpecRange.getEnd();
+
+    // Parse attribute-specifier[opt].
+    MaybeParseCXX0XAttributes(Attr, &DeclEndLoc);
+
+    // Parse trailing-return-type[opt].
+    ParsedType TrailingReturnType;
+    if (Tok.is(tok::arrow)) {
+      SourceRange Range;
+      TrailingReturnType = ParseTrailingReturnType(Range).get();
+      if (Range.getEnd().isValid())
+        DeclEndLoc = Range.getEnd();
+    }
+
+    PrototypeScope.Exit();
+
+    D.AddTypeInfo(DeclaratorChunk::getFunction(/*hasProto=*/true,
+                                           /*isVariadic=*/EllipsisLoc.isValid(),
+                                           EllipsisLoc,
+                                           ParamInfo.data(), ParamInfo.size(),
+                                           DS.getTypeQualifiers(),
+                                           /*RefQualifierIsLValueRef=*/true,
+                                           /*RefQualifierLoc=*/SourceLocation(),
+                                           MutableLoc,
+                                           ESpecType, ESpecRange.getBegin(),
+                                           DynamicExceptions.data(),
+                                           DynamicExceptionRanges.data(),
+                                           DynamicExceptions.size(),
+                                           NoexceptExpr.isUsable() ?
+                                             NoexceptExpr.get() : 0,
+                                           DeclLoc, DeclEndLoc, D,
+                                           TrailingReturnType),
+                  Attr, DeclEndLoc);
+  }
+
+  // Parse compound-statement.
+  if (Tok.is(tok::l_brace)) {
+    // FIXME: Rename BlockScope -> ClosureScope if we decide to continue using
+    // it.
+    ParseScope BodyScope(this, Scope::BlockScope | Scope::FnScope |
+                               Scope::BreakScope | Scope::ContinueScope |
+                               Scope::DeclScope);
+
+    StmtResult Stmt(ParseCompoundStatementBody());
+
+    BodyScope.Exit();
+  } else {
+    Diag(Tok, diag::err_expected_lambda_body);
+  }
+
+  return ExprEmpty();
+}
+
 /// ParseCXXCasts - This handles the various ways to cast expressions to another
 /// type.
 ///
@@ -518,7 +790,7 @@ ExprResult Parser::ParseCXXCasts() {
   const char *CastName = 0;     // For error messages
 
   switch (Kind) {
-  default: assert(0 && "Unknown C++ cast!"); abort();
+  default: llvm_unreachable("Unknown C++ cast!");
   case tok::kw_const_cast:       CastName = "const_cast";       break;
   case tok::kw_dynamic_cast:     CastName = "dynamic_cast";     break;
   case tok::kw_reinterpret_cast: CastName = "reinterpret_cast"; break;
@@ -551,21 +823,23 @@ ExprResult Parser::ParseCXXCasts() {
   if (ExpectAndConsume(tok::greater, diag::err_expected_greater))
     return ExprError(Diag(LAngleBracketLoc, diag::note_matching) << "<");
 
-  SourceLocation LParenLoc = Tok.getLocation(), RParenLoc;
+  SourceLocation LParenLoc, RParenLoc;
+  BalancedDelimiterTracker T(*this, tok::l_paren);
 
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after, CastName))
+  if (T.expectAndConsume(diag::err_expected_lparen_after, CastName))
     return ExprError();
 
   ExprResult Result = ParseExpression();
 
   // Match the ')'.
-  RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
 
   if (!Result.isInvalid() && !DeclaratorInfo.isInvalidType())
     Result = Actions.ActOnCXXNamedCast(OpLoc, Kind,
                                        LAngleBracketLoc, DeclaratorInfo,
                                        RAngleBracketLoc,
-                                       LParenLoc, Result.take(), RParenLoc);
+                                       T.getOpenLocation(), Result.take(), 
+                                       T.getCloseLocation());
 
   return move(Result);
 }
@@ -580,13 +854,13 @@ ExprResult Parser::ParseCXXTypeid() {
   assert(Tok.is(tok::kw_typeid) && "Not 'typeid'!");
 
   SourceLocation OpLoc = ConsumeToken();
-  SourceLocation LParenLoc = Tok.getLocation();
-  SourceLocation RParenLoc;
+  SourceLocation LParenLoc, RParenLoc;
+  BalancedDelimiterTracker T(*this, tok::l_paren);
 
   // typeid expressions are always parenthesized.
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
-      "typeid"))
+  if (T.expectAndConsume(diag::err_expected_lparen_after, "typeid"))
     return ExprError();
+  LParenLoc = T.getOpenLocation();
 
   ExprResult Result;
 
@@ -594,8 +868,8 @@ ExprResult Parser::ParseCXXTypeid() {
     TypeResult Ty = ParseTypeName();
 
     // Match the ')'.
-    RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
-
+    T.consumeClose();
+    RParenLoc = T.getCloseLocation();
     if (Ty.isInvalid() || RParenLoc.isInvalid())
       return ExprError();
 
@@ -618,18 +892,11 @@ ExprResult Parser::ParseCXXTypeid() {
     if (Result.isInvalid())
       SkipUntil(tok::r_paren);
     else {
-      RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+      T.consumeClose();
+      RParenLoc = T.getCloseLocation();
       if (RParenLoc.isInvalid())
         return ExprError();
 
-      // If we are a foo<int> that identifies a single function, resolve it now...  
-      Expr* e = Result.get();
-      if (e->getType() == Actions.Context.OverloadTy) {
-        ExprResult er = 
-              Actions.ResolveAndFixSingleFunctionTemplateSpecialization(e);
-        if (er.isUsable())
-          Result = er.release();
-      }
       Result = Actions.ActOnCXXTypeid(OpLoc, LParenLoc, /*isType=*/false,
                                       Result.release(), RParenLoc);
     }
@@ -647,12 +914,10 @@ ExprResult Parser::ParseCXXUuidof() {
   assert(Tok.is(tok::kw___uuidof) && "Not '__uuidof'!");
 
   SourceLocation OpLoc = ConsumeToken();
-  SourceLocation LParenLoc = Tok.getLocation();
-  SourceLocation RParenLoc;
+  BalancedDelimiterTracker T(*this, tok::l_paren);
 
   // __uuidof expressions are always parenthesized.
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
-      "__uuidof"))
+  if (T.expectAndConsume(diag::err_expected_lparen_after, "__uuidof"))
     return ExprError();
 
   ExprResult Result;
@@ -661,13 +926,14 @@ ExprResult Parser::ParseCXXUuidof() {
     TypeResult Ty = ParseTypeName();
 
     // Match the ')'.
-    RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    T.consumeClose();
 
     if (Ty.isInvalid())
       return ExprError();
 
-    Result = Actions.ActOnCXXUuidof(OpLoc, LParenLoc, /*isType=*/true,
-                                    Ty.get().getAsOpaquePtr(), RParenLoc);
+    Result = Actions.ActOnCXXUuidof(OpLoc, T.getOpenLocation(), /*isType=*/true,
+                                    Ty.get().getAsOpaquePtr(), 
+                                    T.getCloseLocation());
   } else {
     EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
     Result = ParseExpression();
@@ -676,10 +942,11 @@ ExprResult Parser::ParseCXXUuidof() {
     if (Result.isInvalid())
       SkipUntil(tok::r_paren);
     else {
-      RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+      T.consumeClose();
 
-      Result = Actions.ActOnCXXUuidof(OpLoc, LParenLoc, /*isType=*/false,
-                                      Result.release(), RParenLoc);
+      Result = Actions.ActOnCXXUuidof(OpLoc, T.getOpenLocation(),
+                                      /*isType=*/false,
+                                      Result.release(), T.getCloseLocation());
     }
   }
 
@@ -836,7 +1103,8 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
   } else {
     GreaterThanIsOperatorScope G(GreaterThanIsOperator, true);
 
-    SourceLocation LParenLoc = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
 
     ExprVector Exprs(Actions);
     CommaLocsTy CommaLocs;
@@ -849,7 +1117,7 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
     }
 
     // Match the ')'.
-    SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    T.consumeClose();
 
     // TypeRep could be null, if it references an invalid typedef.
     if (!TypeRep)
@@ -857,8 +1125,9 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
 
     assert((Exprs.size() == 0 || Exprs.size()-1 == CommaLocs.size())&&
            "Unexpected number of commas!");
-    return Actions.ActOnCXXTypeConstructExpr(TypeRep, LParenLoc, move_arg(Exprs),
-                                             RParenLoc);
+    return Actions.ActOnCXXTypeConstructExpr(TypeRep, T.getOpenLocation(), 
+                                             move_arg(Exprs),
+                                             T.getCloseLocation());
   }
 }
 
@@ -889,7 +1158,8 @@ bool Parser::ParseCXXCondition(ExprResult &ExprOut,
                                bool ConvertToBoolean) {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return true;
   }
 
   if (!isCXXConditionDeclaration()) {
@@ -969,6 +1239,7 @@ bool Parser::isCXXSimpleTypeSpecifier() const {
   case tok::kw_void:
   case tok::kw_char:
   case tok::kw_int:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_wchar_t:
@@ -1022,10 +1293,9 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) {
   switch (Tok.getKind()) {
   case tok::identifier:   // foo::bar
   case tok::coloncolon:   // ::foo::bar
-    assert(0 && "Annotation token should already be formed!");
+    llvm_unreachable("Annotation token should already be formed!");
   default:
-    assert(0 && "Not a simple-type-specifier token!");
-    abort();
+    llvm_unreachable("Not a simple-type-specifier token!");
 
   // type-name
   case tok::annot_typename: {
@@ -1074,6 +1344,9 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) {
   case tok::kw_int:
     DS.SetTypeSpecType(DeclSpec::TST_int, Loc, PrevSpec, DiagID);
     break;
+  case tok::kw_half:
+    DS.SetTypeSpecType(DeclSpec::TST_half, Loc, PrevSpec, DiagID);
+    break;
   case tok::kw_float:
     DS.SetTypeSpecType(DeclSpec::TST_float, Loc, PrevSpec, DiagID);
     break;
@@ -1394,16 +1667,15 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
       // Consume the 'new' or 'delete'.
       SymbolLocations[SymbolIdx++] = ConsumeToken();
       if (Tok.is(tok::l_square)) {
-        // Consume the '['.
-        SourceLocation LBracketLoc = ConsumeBracket();
-        // Consume the ']'.
-        SourceLocation RBracketLoc = MatchRHSPunctuation(tok::r_square,
-                                                         LBracketLoc);
-        if (RBracketLoc.isInvalid())
+        // Consume the '[' and ']'.
+        BalancedDelimiterTracker T(*this, tok::l_square);
+        T.consumeOpen();
+        T.consumeClose();
+        if (T.getCloseLocation().isInvalid())
           return true;
         
-        SymbolLocations[SymbolIdx++] = LBracketLoc;
-        SymbolLocations[SymbolIdx++] = RBracketLoc;
+        SymbolLocations[SymbolIdx++] = T.getOpenLocation();
+        SymbolLocations[SymbolIdx++] = T.getCloseLocation();
         Op = isNew? OO_Array_New : OO_Array_Delete;
       } else {
         Op = isNew? OO_New : OO_Delete;
@@ -1420,31 +1692,29 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
 #include "clang/Basic/OperatorKinds.def"
       
     case tok::l_paren: {
-      // Consume the '('.
-      SourceLocation LParenLoc = ConsumeParen();
-      // Consume the ')'.
-      SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren,
-                                                     LParenLoc);
-      if (RParenLoc.isInvalid())
+      // Consume the '(' and ')'.
+      BalancedDelimiterTracker T(*this, tok::l_paren);
+      T.consumeOpen();
+      T.consumeClose();
+      if (T.getCloseLocation().isInvalid())
         return true;
       
-      SymbolLocations[SymbolIdx++] = LParenLoc;
-      SymbolLocations[SymbolIdx++] = RParenLoc;
+      SymbolLocations[SymbolIdx++] = T.getOpenLocation();
+      SymbolLocations[SymbolIdx++] = T.getCloseLocation();
       Op = OO_Call;
       break;
     }
       
     case tok::l_square: {
-      // Consume the '['.
-      SourceLocation LBracketLoc = ConsumeBracket();
-      // Consume the ']'.
-      SourceLocation RBracketLoc = MatchRHSPunctuation(tok::r_square,
-                                                       LBracketLoc);
-      if (RBracketLoc.isInvalid())
+      // Consume the '[' and ']'.
+      BalancedDelimiterTracker T(*this, tok::l_square);
+      T.consumeOpen();
+      T.consumeClose();
+      if (T.getCloseLocation().isInvalid())
         return true;
       
-      SymbolLocations[SymbolIdx++] = LBracketLoc;
-      SymbolLocations[SymbolIdx++] = RBracketLoc;
+      SymbolLocations[SymbolIdx++] = T.getOpenLocation();
+      SymbolLocations[SymbolIdx++] = T.getCloseLocation();
       Op = OO_Subscript;
       break;
     }
@@ -1452,10 +1722,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
     case tok::code_completion: {
       // Code completion for the operator name.
       Actions.CodeCompleteOperatorName(getCurScope());
-      
-      // Consume the operator token.
-      ConsumeCodeCompletionToken();
-      
+      cutOffParsing();      
       // Don't try to parse any further.
       return true;
     }
@@ -1758,13 +2025,16 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
   Declarator DeclaratorInfo(DS, Declarator::CXXNewContext);
   if (Tok.is(tok::l_paren)) {
     // If it turns out to be a placement, we change the type location.
-    PlacementLParen = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
+    PlacementLParen = T.getOpenLocation();
     if (ParseExpressionListOrTypeId(PlacementArgs, DeclaratorInfo)) {
       SkipUntil(tok::semi, /*StopAtSemi=*/true, /*DontConsume=*/true);
       return ExprError();
     }
 
-    PlacementRParen = MatchRHSPunctuation(tok::r_paren, PlacementLParen);
+    T.consumeClose();
+    PlacementRParen = T.getCloseLocation();
     if (PlacementRParen.isInvalid()) {
       SkipUntil(tok::semi, /*StopAtSemi=*/true, /*DontConsume=*/true);
       return ExprError();
@@ -1772,18 +2042,19 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
 
     if (PlacementArgs.empty()) {
       // Reset the placement locations. There was no placement.
-      TypeIdParens = SourceRange(PlacementLParen, PlacementRParen);
+      TypeIdParens = T.getRange();
       PlacementLParen = PlacementRParen = SourceLocation();
     } else {
       // We still need the type.
       if (Tok.is(tok::l_paren)) {
-        TypeIdParens.setBegin(ConsumeParen());
+        BalancedDelimiterTracker T(*this, tok::l_paren);
+        T.consumeOpen();
         MaybeParseGNUAttributes(DeclaratorInfo);
         ParseSpecifierQualifierList(DS);
         DeclaratorInfo.SetSourceRange(DS.getSourceRange());
         ParseDeclarator(DeclaratorInfo);
-        TypeIdParens.setEnd(MatchRHSPunctuation(tok::r_paren, 
-                                                TypeIdParens.getBegin()));
+        T.consumeClose();
+        TypeIdParens = T.getRange();
       } else {
         MaybeParseGNUAttributes(DeclaratorInfo);
         if (ParseCXXTypeSpecifierSeq(DS))
@@ -1816,7 +2087,9 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
   SourceLocation ConstructorLParen, ConstructorRParen;
 
   if (Tok.is(tok::l_paren)) {
-    ConstructorLParen = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
+    ConstructorLParen = T.getOpenLocation();
     if (Tok.isNot(tok::r_paren)) {
       CommaLocsTy CommaLocs;
       if (ParseExpressionList(ConstructorArgs, CommaLocs)) {
@@ -1824,7 +2097,8 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
         return ExprError();
       }
     }
-    ConstructorRParen = MatchRHSPunctuation(tok::r_paren, ConstructorLParen);
+    T.consumeClose();
+    ConstructorRParen = T.getCloseLocation();
     if (ConstructorRParen.isInvalid()) {
       SkipUntil(tok::semi, /*StopAtSemi=*/true, /*DontConsume=*/true);
       return ExprError();
@@ -1851,7 +2125,9 @@ void Parser::ParseDirectNewDeclarator(Declarator &D) {
   // Parse the array dimensions.
   bool first = true;
   while (Tok.is(tok::l_square)) {
-    SourceLocation LLoc = ConsumeBracket();
+    BalancedDelimiterTracker T(*this, tok::l_square);
+    T.consumeOpen();
+
     ExprResult Size(first ? ParseExpression()
                                 : ParseConstantExpression());
     if (Size.isInvalid()) {
@@ -1861,15 +2137,17 @@ void Parser::ParseDirectNewDeclarator(Declarator &D) {
     }
     first = false;
 
-    SourceLocation RLoc = MatchRHSPunctuation(tok::r_square, LLoc);
+    T.consumeClose();
 
     ParsedAttributes attrs(AttrFactory);
     D.AddTypeInfo(DeclaratorChunk::getArray(0,
                                             /*static=*/false, /*star=*/false,
-                                            Size.release(), LLoc, RLoc),
-                  attrs, RLoc);
+                                            Size.release(),
+                                            T.getOpenLocation(),
+                                            T.getCloseLocation()),
+                  attrs, T.getCloseLocation());
 
-    if (RLoc.isInvalid())
+    if (T.getCloseLocation().isInvalid())
       return;
   }
 }
@@ -1885,7 +2163,7 @@ void Parser::ParseDirectNewDeclarator(Declarator &D) {
 ///                   '(' expression-list ')'
 ///
 bool Parser::ParseExpressionListOrTypeId(
-                                   llvm::SmallVectorImpl<Expr*> &PlacementArgs,
+                                   SmallVectorImpl<Expr*> &PlacementArgs,
                                          Declarator &D) {
   // The '(' was already consumed.
   if (isTypeIdInParens()) {
@@ -1921,9 +2199,11 @@ Parser::ParseCXXDeleteExpression(bool UseGlobal, SourceLocation Start) {
   bool ArrayDelete = false;
   if (Tok.is(tok::l_square)) {
     ArrayDelete = true;
-    SourceLocation LHS = ConsumeBracket();
-    SourceLocation RHS = MatchRHSPunctuation(tok::r_square, LHS);
-    if (RHS.isInvalid())
+    BalancedDelimiterTracker T(*this, tok::l_square);
+
+    T.consumeOpen();
+    T.consumeClose();
+    if (T.getCloseLocation().isInvalid())
       return ExprError();
   }
 
@@ -1936,7 +2216,7 @@ Parser::ParseCXXDeleteExpression(bool UseGlobal, SourceLocation Start) {
 
 static UnaryTypeTrait UnaryTypeTraitFromTokKind(tok::TokenKind kind) {
   switch(kind) {
-  default: assert(false && "Not a known unary type trait.");
+  default: llvm_unreachable("Not a known unary type trait.");
   case tok::kw___has_nothrow_assign:      return UTT_HasNothrowAssign;
   case tok::kw___has_nothrow_constructor: return UTT_HasNothrowConstructor;
   case tok::kw___has_nothrow_copy:           return UTT_HasNothrowCopy;
@@ -2004,7 +2284,7 @@ static ArrayTypeTrait ArrayTypeTraitFromTokKind(tok::TokenKind kind) {
 
 static ExpressionTrait ExpressionTraitFromTokKind(tok::TokenKind kind) {
   switch(kind) {
-  default: assert(false && "Not a known unary expression trait.");
+  default: llvm_unreachable("Not a known unary expression trait.");
   case tok::kw___is_lvalue_expr:             return ET_IsLValueExpr;
   case tok::kw___is_rvalue_expr:             return ET_IsRValueExpr;
   }
@@ -2021,8 +2301,8 @@ ExprResult Parser::ParseUnaryTypeTrait() {
   UnaryTypeTrait UTT = UnaryTypeTraitFromTokKind(Tok.getKind());
   SourceLocation Loc = ConsumeToken();
 
-  SourceLocation LParen = Tok.getLocation();
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen))
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
     return ExprError();
 
   // FIXME: Error reporting absolutely sucks! If the this fails to parse a type
@@ -2030,12 +2310,12 @@ ExprResult Parser::ParseUnaryTypeTrait() {
   // specifiers.
   TypeResult Ty = ParseTypeName();
 
-  SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+  T.consumeClose();
 
   if (Ty.isInvalid())
     return ExprError();
 
-  return Actions.ActOnUnaryTypeTrait(UTT, Loc, Ty.get(), RParen);
+  return Actions.ActOnUnaryTypeTrait(UTT, Loc, Ty.get(), T.getCloseLocation());
 }
 
 /// ParseBinaryTypeTrait - Parse the built-in binary type-trait
@@ -2049,8 +2329,8 @@ ExprResult Parser::ParseBinaryTypeTrait() {
   BinaryTypeTrait BTT = BinaryTypeTraitFromTokKind(Tok.getKind());
   SourceLocation Loc = ConsumeToken();
 
-  SourceLocation LParen = Tok.getLocation();
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen))
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
     return ExprError();
 
   TypeResult LhsTy = ParseTypeName();
@@ -2070,9 +2350,10 @@ ExprResult Parser::ParseBinaryTypeTrait() {
     return ExprError();
   }
 
-  SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+  T.consumeClose();
 
-  return Actions.ActOnBinaryTypeTrait(BTT, Loc, LhsTy.get(), RhsTy.get(), RParen);
+  return Actions.ActOnBinaryTypeTrait(BTT, Loc, LhsTy.get(), RhsTy.get(),
+                                      T.getCloseLocation());
 }
 
 /// ParseArrayTypeTrait - Parse the built-in array type-trait
@@ -2086,8 +2367,8 @@ ExprResult Parser::ParseArrayTypeTrait() {
   ArrayTypeTrait ATT = ArrayTypeTraitFromTokKind(Tok.getKind());
   SourceLocation Loc = ConsumeToken();
 
-  SourceLocation LParen = Tok.getLocation();
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen))
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
     return ExprError();
 
   TypeResult Ty = ParseTypeName();
@@ -2099,8 +2380,9 @@ ExprResult Parser::ParseArrayTypeTrait() {
 
   switch (ATT) {
   case ATT_ArrayRank: {
-    SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
-    return Actions.ActOnArrayTypeTrait(ATT, Loc, Ty.get(), NULL, RParen);
+    T.consumeClose();
+    return Actions.ActOnArrayTypeTrait(ATT, Loc, Ty.get(), NULL,
+                                       T.getCloseLocation());
   }
   case ATT_ArrayExtent: {
     if (ExpectAndConsume(tok::comma, diag::err_expected_comma)) {
@@ -2109,9 +2391,10 @@ ExprResult Parser::ParseArrayTypeTrait() {
     }
 
     ExprResult DimExpr = ParseExpression();
-    SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+    T.consumeClose();
 
-    return Actions.ActOnArrayTypeTrait(ATT, Loc, Ty.get(), DimExpr.get(), RParen);
+    return Actions.ActOnArrayTypeTrait(ATT, Loc, Ty.get(), DimExpr.get(),
+                                       T.getCloseLocation());
   }
   default:
     break;
@@ -2129,15 +2412,16 @@ ExprResult Parser::ParseExpressionTrait() {
   ExpressionTrait ET = ExpressionTraitFromTokKind(Tok.getKind());
   SourceLocation Loc = ConsumeToken();
 
-  SourceLocation LParen = Tok.getLocation();
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen))
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
     return ExprError();
 
   ExprResult Expr = ParseExpression();
 
-  SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+  T.consumeClose();
 
-  return Actions.ActOnExpressionTrait(ET, Loc, Expr.get(), RParen);
+  return Actions.ActOnExpressionTrait(ET, Loc, Expr.get(),
+                                      T.getCloseLocation());
 }
 
 
@@ -2147,8 +2431,7 @@ ExprResult Parser::ParseExpressionTrait() {
 ExprResult
 Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
                                          ParsedType &CastTy,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation &RParenLoc) {
+                                         BalancedDelimiterTracker &Tracker) {
   assert(getLang().CPlusPlus && "Should only be called for C++!");
   assert(ExprType == CastExpr && "Compound literals are not ambiguous!");
   assert(isTypeIdInParens() && "Not a type-id!");
@@ -2182,7 +2465,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
   // the context that follows them.
   if (!ConsumeAndStoreUntil(tok::r_paren, Toks)) {
     // We didn't find the ')' we expected.
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    Tracker.consumeClose();
     return ExprError();
   }
 
@@ -2227,15 +2510,14 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
     ParseDeclarator(DeclaratorInfo);
 
     // Match the ')'.
-    if (Tok.is(tok::r_paren))
-      RParenLoc = ConsumeParen();
-    else
-      MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    Tracker.consumeClose();
 
     if (ParseAs == CompoundLiteral) {
       ExprType = CompoundLiteral;
       TypeResult Ty = ParseTypeName();
-      return ParseCompoundLiteralExpression(Ty.get(), LParenLoc, RParenLoc);
+       return ParseCompoundLiteralExpression(Ty.get(),
+                                            Tracker.getOpenLocation(),
+                                            Tracker.getCloseLocation());
     }
 
     // We parsed '(' type-id ')' and the thing after it wasn't a '{'.
@@ -2246,9 +2528,9 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
 
     // Result is what ParseCastExpression returned earlier.
     if (!Result.isInvalid())
-      Result = Actions.ActOnCastExpr(getCurScope(), LParenLoc,
-                                     DeclaratorInfo, CastTy,
-                                     RParenLoc, Result.take());
+      Result = Actions.ActOnCastExpr(getCurScope(), Tracker.getOpenLocation(),
+                                    DeclaratorInfo, CastTy,
+                                    Tracker.getCloseLocation(), Result.take());
     return move(Result);
   }
 
@@ -2258,7 +2540,8 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
   ExprType = SimpleExpr;
   Result = ParseExpression();
   if (!Result.isInvalid() && Tok.is(tok::r_paren))
-    Result = Actions.ActOnParenExpr(LParenLoc, Tok.getLocation(), Result.take());
+    Result = Actions.ActOnParenExpr(Tracker.getOpenLocation(), 
+                                    Tok.getLocation(), Result.take());
 
   // Match the ')'.
   if (Result.isInvalid()) {
@@ -2266,10 +2549,6 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
     return ExprError();
   }
 
-  if (Tok.is(tok::r_paren))
-    RParenLoc = ConsumeParen();
-  else
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
-
+  Tracker.consumeClose();
   return move(Result);
 }
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseInit.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseInit.cpp
index 2c9278a90033..33abc932541d 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseInit.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseInit.cpp
@@ -90,7 +90,7 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
     assert(Tok.is(tok::colon) && "MayBeDesignationStart not working properly!");
     SourceLocation ColonLoc = ConsumeToken();
 
-    Diag(Tok, diag::ext_gnu_old_style_field_designator)
+    Diag(NameLoc, diag::ext_gnu_old_style_field_designator)
       << FixItHint::CreateReplacement(SourceRange(NameLoc, ColonLoc),
                                       NewSyntax.str());
 
@@ -139,7 +139,10 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
     //
     InMessageExpressionRAIIObject InMessage(*this, true);
     
-    SourceLocation StartLoc = ConsumeBracket();
+    BalancedDelimiterTracker T(*this, tok::l_square);
+    T.consumeOpen();
+    SourceLocation StartLoc = T.getOpenLocation();
+
     ExprResult Idx;
 
     // If Objective-C is enabled and this is a typename (class message
@@ -266,8 +269,9 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
                                                     StartLoc, EllipsisLoc));
     }
 
-    SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
-    Desig.getDesignator(Desig.getNumDesignators() - 1).setRBracketLoc(EndLoc);
+    T.consumeClose();
+    Desig.getDesignator(Desig.getNumDesignators() - 1).setRBracketLoc(
+                                                        T.getCloseLocation());
   }
 
   // Okay, we're done with the designator sequence.  We know that there must be
@@ -316,7 +320,9 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
 ExprResult Parser::ParseBraceInitializer() {
   InMessageExpressionRAIIObject InMessage(*this, false);
   
-  SourceLocation LBraceLoc = ConsumeBrace();
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  T.consumeOpen();
+  SourceLocation LBraceLoc = T.getOpenLocation();
 
   /// InitExprs - This is the actual list of expressions contained in the
   /// initializer.
@@ -376,12 +382,13 @@ ExprResult Parser::ParseBraceInitializer() {
     // Handle trailing comma.
     if (Tok.is(tok::r_brace)) break;
   }
-  if (InitExprsOk && Tok.is(tok::r_brace))
-    return Actions.ActOnInitList(LBraceLoc, move_arg(InitExprs),
-                                 ConsumeBrace());
 
-  // Match the '}'.
-  MatchRHSPunctuation(tok::r_brace, LBraceLoc);
+  bool closed = !T.consumeClose();
+
+  if (InitExprsOk && closed)
+    return Actions.ActOnInitList(LBraceLoc, move_arg(InitExprs),
+                                 T.getCloseLocation());
+
   return ExprError(); // an error occurred.
 }
 
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp
index 7641565d1cc3..88044d1bc9d8 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp
@@ -29,57 +29,70 @@ using namespace clang;
 /// [OBJC]  objc-protocol-definition
 /// [OBJC]  objc-method-definition
 /// [OBJC]  '@' 'end'
-Decl *Parser::ParseObjCAtDirectives() {
+Parser::DeclGroupPtrTy Parser::ParseObjCAtDirectives() {
   SourceLocation AtLoc = ConsumeToken(); // the "@"
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCAtDirective(getCurScope(), ObjCImpDecl, false);
-    ConsumeCodeCompletionToken();
+    Actions.CodeCompleteObjCAtDirective(getCurScope());
+    cutOffParsing();
+    return DeclGroupPtrTy();
   }
-
+    
+  Decl *SingleDecl = 0;
   switch (Tok.getObjCKeywordID()) {
   case tok::objc_class:
     return ParseObjCAtClassDeclaration(AtLoc);
+    break;
   case tok::objc_interface: {
     ParsedAttributes attrs(AttrFactory);
-    return ParseObjCAtInterfaceDeclaration(AtLoc, attrs);
+    SingleDecl = ParseObjCAtInterfaceDeclaration(AtLoc, attrs);
+    break;
   }
   case tok::objc_protocol: {
     ParsedAttributes attrs(AttrFactory);
-    return ParseObjCAtProtocolDeclaration(AtLoc, attrs);
+    SingleDecl = ParseObjCAtProtocolDeclaration(AtLoc, attrs);
+    break;
   }
   case tok::objc_implementation:
-    return ParseObjCAtImplementationDeclaration(AtLoc);
+    SingleDecl = ParseObjCAtImplementationDeclaration(AtLoc);
+    break;
   case tok::objc_end:
     return ParseObjCAtEndDeclaration(AtLoc);
+    break;
   case tok::objc_compatibility_alias:
-    return ParseObjCAtAliasDeclaration(AtLoc);
+    SingleDecl = ParseObjCAtAliasDeclaration(AtLoc);
+    break;
   case tok::objc_synthesize:
-    return ParseObjCPropertySynthesize(AtLoc);
+    SingleDecl = ParseObjCPropertySynthesize(AtLoc);
+    break;
   case tok::objc_dynamic:
-    return ParseObjCPropertyDynamic(AtLoc);
+    SingleDecl = ParseObjCPropertyDynamic(AtLoc);
+    break;
   default:
     Diag(AtLoc, diag::err_unexpected_at);
     SkipUntil(tok::semi);
-    return 0;
+    SingleDecl = 0;
+    break;
   }
+  return Actions.ConvertDeclToDeclGroup(SingleDecl);
 }
 
 ///
 /// objc-class-declaration:
 ///    '@' 'class' identifier-list ';'
 ///
-Decl *Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
+Parser::DeclGroupPtrTy
+Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
   ConsumeToken(); // the identifier "class"
-  llvm::SmallVector<IdentifierInfo *, 8> ClassNames;
-  llvm::SmallVector<SourceLocation, 8> ClassLocs;
+  SmallVector<IdentifierInfo *, 8> ClassNames;
+  SmallVector<SourceLocation, 8> ClassLocs;
 
 
   while (1) {
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_expected_ident);
       SkipUntil(tok::semi);
-      return 0;
+      return Actions.ConvertDeclToDeclGroup(0);
     }
     ClassNames.push_back(Tok.getIdentifierInfo());
     ClassLocs.push_back(Tok.getLocation());
@@ -93,7 +106,7 @@ Decl *Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
 
   // Consume the ';'.
   if (ExpectAndConsume(tok::semi, diag::err_expected_semi_after, "@class"))
-    return 0;
+    return Actions.ConvertDeclToDeclGroup(0);
 
   return Actions.ActOnForwardClassDeclaration(atLoc, ClassNames.data(),
                                               ClassLocs.data(),
@@ -137,7 +150,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
   // Code completion after '@interface'.
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCInterfaceDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   if (Tok.isNot(tok::identifier)) {
@@ -150,14 +164,16 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
   SourceLocation nameLoc = ConsumeToken();
   if (Tok.is(tok::l_paren) && 
       !isKnownToBeTypeSpecifier(GetLookAheadToken(1))) { // we have a category.
-    // TODO(dgregor): Use the return value from the next line to provide better
-    // recovery.
-    ConsumeParen();
-    SourceLocation categoryLoc, rparenLoc;
+    
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
+
+    SourceLocation categoryLoc;
     IdentifierInfo *categoryId = 0;
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
     
     // For ObjC2, the category name is optional (not an error).
@@ -169,24 +185,25 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
       Diag(Tok, diag::err_expected_ident); // missing category name.
       return 0;
     }
-    if (Tok.isNot(tok::r_paren)) {
-      Diag(Tok, diag::err_expected_rparen);
-      SkipUntil(tok::r_paren, false); // don't stop at ';'
+   
+    T.consumeClose();
+    if (T.getCloseLocation().isInvalid())
       return 0;
+    
+    if (!attrs.empty()) { // categories don't support attributes.
+      Diag(nameLoc, diag::err_objc_no_attributes_on_category);
+      attrs.clear();
     }
-    rparenLoc = ConsumeParen();
+    
     // Next, we need to check for any protocol references.
     SourceLocation LAngleLoc, EndProtoLoc;
-    llvm::SmallVector<Decl *, 8> ProtocolRefs;
-    llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
+    SmallVector<Decl *, 8> ProtocolRefs;
+    SmallVector<SourceLocation, 8> ProtocolLocs;
     if (Tok.is(tok::less) &&
         ParseObjCProtocolReferences(ProtocolRefs, ProtocolLocs, true,
                                     LAngleLoc, EndProtoLoc))
       return 0;
 
-    if (!attrs.empty()) // categories don't support attributes.
-      Diag(Tok, diag::err_objc_no_attributes_on_category);
-
     Decl *CategoryType =
     Actions.ActOnStartCategoryInterface(atLoc,
                                         nameId, nameLoc,
@@ -195,11 +212,11 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
                                         ProtocolRefs.size(),
                                         ProtocolLocs.data(),
                                         EndProtoLoc);
-    if (Tok.is(tok::l_brace))
-      ParseObjCClassInstanceVariables(CategoryType, tok::objc_private,
-                                      atLoc);
     
-    ParseObjCInterfaceDeclList(CategoryType, tok::objc_not_keyword);
+    if (Tok.is(tok::l_brace))
+      ParseObjCClassInstanceVariables(CategoryType, tok::objc_private, atLoc);
+      
+    ParseObjCInterfaceDeclList(tok::objc_not_keyword, CategoryType);
     return CategoryType;
   }
   // Parse a class interface.
@@ -212,7 +229,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
     // Code completion of superclass names.
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
 
     if (Tok.isNot(tok::identifier)) {
@@ -223,8 +241,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
     superClassLoc = ConsumeToken();
   }
   // Next, we need to check for any protocol references.
-  llvm::SmallVector<Decl *, 8> ProtocolRefs;
-  llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
+  SmallVector<Decl *, 8> ProtocolRefs;
+  SmallVector<SourceLocation, 8> ProtocolLocs;
   SourceLocation LAngleLoc, EndProtoLoc;
   if (Tok.is(tok::less) &&
       ParseObjCProtocolReferences(ProtocolRefs, ProtocolLocs, true,
@@ -241,7 +259,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
   if (Tok.is(tok::l_brace))
     ParseObjCClassInstanceVariables(ClsType, tok::objc_protected, atLoc);
 
-  ParseObjCInterfaceDeclList(ClsType, tok::objc_interface);
+  ParseObjCInterfaceDeclList(tok::objc_interface, ClsType);
   return ClsType;
 }
 
@@ -249,17 +267,16 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
 /// it's used, but instead it's been lifted to here to support VS2005.
 struct Parser::ObjCPropertyCallback : FieldCallback {
   Parser &P;
-  Decl *IDecl;
-  llvm::SmallVectorImpl<Decl *> &Props;
+  SmallVectorImpl<Decl *> &Props;
   ObjCDeclSpec &OCDS;
   SourceLocation AtLoc;
   tok::ObjCKeywordKind MethodImplKind;
         
-  ObjCPropertyCallback(Parser &P, Decl *IDecl,
-                       llvm::SmallVectorImpl<Decl *> &Props,
+  ObjCPropertyCallback(Parser &P, 
+                       SmallVectorImpl<Decl *> &Props,
                        ObjCDeclSpec &OCDS, SourceLocation AtLoc,
                        tok::ObjCKeywordKind MethodImplKind) :
-    P(P), IDecl(IDecl), Props(Props), OCDS(OCDS), AtLoc(AtLoc),
+    P(P), Props(Props), OCDS(OCDS), AtLoc(AtLoc),
     MethodImplKind(MethodImplKind) {
   }
 
@@ -292,7 +309,7 @@ struct Parser::ObjCPropertyCallback : FieldCallback {
     bool isOverridingProperty = false;
     Decl *Property =
       P.Actions.ActOnProperty(P.getCurScope(), AtLoc, FD, OCDS,
-                              GetterSel, SetterSel, IDecl,
+                              GetterSel, SetterSel, 
                               &isOverridingProperty,
                               MethodImplKind);
     if (!isOverridingProperty)
@@ -314,20 +331,20 @@ struct Parser::ObjCPropertyCallback : FieldCallback {
 ///     @required
 ///     @optional
 ///
-void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
-                                        tok::ObjCKeywordKind contextKey) {
-  llvm::SmallVector<Decl *, 32> allMethods;
-  llvm::SmallVector<Decl *, 16> allProperties;
-  llvm::SmallVector<DeclGroupPtrTy, 8> allTUVariables;
+void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey, 
+                                        Decl *CDecl) {
+  SmallVector<Decl *, 32> allMethods;
+  SmallVector<Decl *, 16> allProperties;
+  SmallVector<DeclGroupPtrTy, 8> allTUVariables;
   tok::ObjCKeywordKind MethodImplKind = tok::objc_not_keyword;
 
   SourceRange AtEnd;
-
+    
   while (1) {
     // If this is a method prototype, parse it.
     if (Tok.is(tok::minus) || Tok.is(tok::plus)) {
       Decl *methodPrototype =
-        ParseObjCMethodPrototype(interfaceDecl, MethodImplKind, false);
+        ParseObjCMethodPrototype(MethodImplKind, false);
       allMethods.push_back(methodPrototype);
       // Consume the ';' here, since ParseObjCMethodPrototype() is re-used for
       // method definitions.
@@ -339,7 +356,6 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
       Diag(Tok, diag::err_expected_minus_or_plus);
       ParseObjCMethodDecl(Tok.getLocation(), 
                           tok::minus, 
-                          interfaceDecl,
                           MethodImplKind, false);
       continue;
     }
@@ -358,7 +374,7 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                   ObjCImpDecl? Sema::PCC_ObjCImplementation
                                              : Sema::PCC_ObjCInterface);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     
     // If we don't have an @ directive, parse it as a function definition.
@@ -368,9 +384,6 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
       // erroneous r_brace would cause an infinite loop if not handled here.
       if (Tok.is(tok::r_brace))
         break;
-
-      // FIXME: as the name implies, this rule allows function definitions.
-      // We could pass a flag or check for functions during semantic analysis.
       ParsedAttributes attrs(AttrFactory);
       allTUVariables.push_back(ParseDeclarationOrFunctionDefinition(attrs));
       continue;
@@ -379,8 +392,8 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
     // Otherwise, we have an @ directive, eat the @.
     SourceLocation AtLoc = ConsumeToken(); // the "@"
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCAtDirective(getCurScope(), ObjCImpDecl, true);
-      ConsumeCodeCompletionToken();
+      Actions.CodeCompleteObjCAtDirective(getCurScope());
+      return cutOffParsing();
       break;
     }
 
@@ -433,9 +446,9 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
       ObjCDeclSpec OCDS;
       // Parse property attribute list, if any.
       if (Tok.is(tok::l_paren))
-        ParseObjCPropertyAttribute(OCDS, interfaceDecl);
+        ParseObjCPropertyAttribute(OCDS);
 
-      ObjCPropertyCallback Callback(*this, interfaceDecl, allProperties,
+      ObjCPropertyCallback Callback(*this, allProperties,
                                     OCDS, AtLoc, MethodImplKind);
 
       // Parse all the comma separated declarators.
@@ -450,8 +463,8 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
   // We break out of the big loop in two cases: when we see @end or when we see
   // EOF.  In the former case, eat the @end.  In the later case, emit an error.
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCAtDirective(getCurScope(), ObjCImpDecl, true);
-    ConsumeCodeCompletionToken();
+    Actions.CodeCompleteObjCAtDirective(getCurScope());
+    return cutOffParsing();
   } else if (Tok.isObjCAtKeyword(tok::objc_end))
     ConsumeToken(); // the "end" identifier
   else
@@ -459,7 +472,7 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
 
   // Insert collected methods declarations into the @interface object.
   // This passes in an invalid SourceLocation for AtEndLoc when EOF is hit.
-  Actions.ActOnAtEnd(getCurScope(), AtEnd, interfaceDecl,
+  Actions.ActOnAtEnd(getCurScope(), AtEnd,
                      allMethods.data(), allMethods.size(),
                      allProperties.data(), allProperties.size(),
                      allTUVariables.data(), allTUVariables.size());
@@ -485,20 +498,21 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
 ///     weak
 ///     unsafe_unretained
 ///
-void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl) {
+void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
   assert(Tok.getKind() == tok::l_paren);
-  SourceLocation LHSLoc = ConsumeParen(); // consume '('
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
 
   while (1) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPropertyFlags(getCurScope(), DS);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     const IdentifierInfo *II = Tok.getIdentifierInfo();
 
     // If this is not an identifier at all, bail out early.
     if (II == 0) {
-      MatchRHSPunctuation(tok::r_paren, LHSLoc);
+      T.consumeClose();
       return;
     }
 
@@ -536,10 +550,10 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl) {
 
       if (Tok.is(tok::code_completion)) {
         if (IsSetter)
-          Actions.CodeCompleteObjCPropertySetter(getCurScope(), ClassDecl);
+          Actions.CodeCompleteObjCPropertySetter(getCurScope());
         else
-          Actions.CodeCompleteObjCPropertyGetter(getCurScope(), ClassDecl);
-        ConsumeCodeCompletionToken();
+          Actions.CodeCompleteObjCPropertyGetter(getCurScope());
+        return cutOffParsing();
       }
 
       
@@ -577,7 +591,7 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl) {
     ConsumeToken();
   }
 
-  MatchRHSPunctuation(tok::r_paren, LHSLoc);
+  T.consumeClose();
 }
 
 ///   objc-method-proto:
@@ -590,14 +604,13 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl) {
 ///   objc-method-attributes:         [OBJC2]
 ///     __attribute__((deprecated))
 ///
-Decl *Parser::ParseObjCMethodPrototype(Decl *IDecl,
-                                       tok::ObjCKeywordKind MethodImplKind,
+Decl *Parser::ParseObjCMethodPrototype(tok::ObjCKeywordKind MethodImplKind,
                                        bool MethodDefinition) {
   assert((Tok.is(tok::minus) || Tok.is(tok::plus)) && "expected +/-");
 
   tok::TokenKind methodType = Tok.getKind();
   SourceLocation mLoc = ConsumeToken();
-  Decl *MDecl = ParseObjCMethodDecl(mLoc, methodType, IDecl,MethodImplKind,
+  Decl *MDecl = ParseObjCMethodDecl(mLoc, methodType, MethodImplKind,
                                     MethodDefinition);
   // Since this rule is used for both method declarations and definitions,
   // the caller is (optionally) responsible for consuming the ';'.
@@ -732,12 +745,15 @@ bool Parser::isTokIdentifier_in() const {
 ///     objc-type-qualifiers objc-type-qualifier
 ///
 void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
-                                        ObjCTypeNameContext Context) {
+                                        Declarator::TheContext Context) {
+  assert(Context == Declarator::ObjCParameterContext ||
+         Context == Declarator::ObjCResultContext);
+
   while (1) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPassingType(getCurScope(), DS, 
-                                          Context == OTN_ParameterType);
-      ConsumeCodeCompletionToken();
+                          Context == Declarator::ObjCParameterContext);
+      return cutOffParsing();
     }
     
     if (Tok.isNot(tok::identifier))
@@ -750,7 +766,7 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
 
       ObjCDeclSpec::ObjCDeclQualifier Qual;
       switch (i) {
-      default: assert(0 && "Unknown decl qualifier");
+      default: llvm_unreachable("Unknown decl qualifier");
       case objc_in:     Qual = ObjCDeclSpec::DQ_In; break;
       case objc_out:    Qual = ObjCDeclSpec::DQ_Out; break;
       case objc_inout:  Qual = ObjCDeclSpec::DQ_Inout; break;
@@ -769,30 +785,95 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
   }
 }
 
+/// Take all the decl attributes out of the given list and add
+/// them to the given attribute set.
+static void takeDeclAttributes(ParsedAttributes &attrs,
+                               AttributeList *list) {
+  while (list) {
+    AttributeList *cur = list;
+    list = cur->getNext();
+
+    if (!cur->isUsedAsTypeAttr()) {
+      // Clear out the next pointer.  We're really completely
+      // destroying the internal invariants of the declarator here,
+      // but it doesn't matter because we're done with it.
+      cur->setNext(0);
+      attrs.add(cur);
+    }
+  }
+}
+
+/// takeDeclAttributes - Take all the decl attributes from the given
+/// declarator and add them to the given list.
+static void takeDeclAttributes(ParsedAttributes &attrs,
+                               Declarator &D) {
+  // First, take ownership of all attributes.
+  attrs.getPool().takeAllFrom(D.getAttributePool());
+  attrs.getPool().takeAllFrom(D.getDeclSpec().getAttributePool());
+
+  // Now actually move the attributes over.
+  takeDeclAttributes(attrs, D.getDeclSpec().getAttributes().getList());
+  takeDeclAttributes(attrs, D.getAttributes());
+  for (unsigned i = 0, e = D.getNumTypeObjects(); i != e; ++i)
+    takeDeclAttributes(attrs,
+                  const_cast<AttributeList*>(D.getTypeObject(i).getAttrs()));
+}
+
 ///   objc-type-name:
 ///     '(' objc-type-qualifiers[opt] type-name ')'
 ///     '(' objc-type-qualifiers[opt] ')'
 ///
 ParsedType Parser::ParseObjCTypeName(ObjCDeclSpec &DS, 
-                                     ObjCTypeNameContext Context) {
+                                     Declarator::TheContext context,
+                                     ParsedAttributes *paramAttrs) {
+  assert(context == Declarator::ObjCParameterContext ||
+         context == Declarator::ObjCResultContext);
+  assert((paramAttrs != 0) == (context == Declarator::ObjCParameterContext));
+
   assert(Tok.is(tok::l_paren) && "expected (");
 
-  SourceLocation LParenLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+
   SourceLocation TypeStartLoc = Tok.getLocation();
+  ObjCDeclContextSwitch ObjCDC(*this);
 
   // Parse type qualifiers, in, inout, etc.
-  ParseObjCTypeQualifierList(DS, Context);
+  ParseObjCTypeQualifierList(DS, context);
 
   ParsedType Ty;
   if (isTypeSpecifierQualifier()) {
-    TypeResult TypeSpec =
-      ParseTypeName(0, Declarator::ObjCPrototypeContext, &DS);
-    if (!TypeSpec.isInvalid())
-      Ty = TypeSpec.get();
+    // Parse an abstract declarator.
+    DeclSpec declSpec(AttrFactory);
+    declSpec.setObjCQualifiers(&DS);
+    ParseSpecifierQualifierList(declSpec);
+    Declarator declarator(declSpec, context);
+    ParseDeclarator(declarator);
+
+    // If that's not invalid, extract a type.
+    if (!declarator.isInvalidType()) {
+      TypeResult type = Actions.ActOnTypeName(getCurScope(), declarator);
+      if (!type.isInvalid())
+        Ty = type.get();
+
+      // If we're parsing a parameter, steal all the decl attributes
+      // and add them to the decl spec.
+      if (context == Declarator::ObjCParameterContext)
+        takeDeclAttributes(*paramAttrs, declarator);
+    }
+  } else if (context == Declarator::ObjCResultContext &&
+             Tok.is(tok::identifier)) {
+    if (!Ident_instancetype)
+      Ident_instancetype = PP.getIdentifierInfo("instancetype");
+    
+    if (Tok.getIdentifierInfo() == Ident_instancetype) {
+      Ty = Actions.ActOnObjCInstanceType(Tok.getLocation());
+      ConsumeToken();
+    }
   }
-  
+
   if (Tok.is(tok::r_paren))
-    ConsumeParen();
+    T.consumeClose();
   else if (Tok.getLocation() == TypeStartLoc) {
     // If we didn't eat any tokens, then this isn't a type.
     Diag(Tok, diag::err_expected_type);
@@ -800,7 +881,7 @@ ParsedType Parser::ParseObjCTypeName(ObjCDeclSpec &DS,
   } else {
     // Otherwise, we found *something*, but didn't get a ')' in the right
     // place.  Emit an error then return what we have as the type.
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    T.consumeClose();
   }
   return Ty;
 }
@@ -835,22 +916,22 @@ ParsedType Parser::ParseObjCTypeName(ObjCDeclSpec &DS,
 ///
 Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                                   tok::TokenKind mType,
-                                  Decl *IDecl,
                                   tok::ObjCKeywordKind MethodImplKind,
                                   bool MethodDefinition) {
   ParsingDeclRAIIObject PD(*this);
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, 
-                                       /*ReturnType=*/ ParsedType(), IDecl);
-    ConsumeCodeCompletionToken();
+                                       /*ReturnType=*/ ParsedType());
+    cutOffParsing();
+    return 0;
   }
 
   // Parse the return type if present.
   ParsedType ReturnType;
   ObjCDeclSpec DSRet;
   if (Tok.is(tok::l_paren))
-    ReturnType = ParseObjCTypeName(DSRet, OTN_ResultType);
+    ReturnType = ParseObjCTypeName(DSRet, Declarator::ObjCResultContext, 0);
 
   // If attributes exist before the method, parse them.
   ParsedAttributes methodAttrs(AttrFactory);
@@ -859,8 +940,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, 
-                                       ReturnType, IDecl);
-    ConsumeCodeCompletionToken();
+                                       ReturnType);
+    cutOffParsing();
+    return 0;
   }
 
   // Now parse the selector.
@@ -876,7 +958,7 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     return 0;
   }
 
-  llvm::SmallVector<DeclaratorChunk::ParamInfo, 8> CParamInfo;
+  SmallVector<DeclaratorChunk::ParamInfo, 8> CParamInfo;
   if (Tok.isNot(tok::colon)) {
     // If attributes exist after the method, parse them.
     if (getLang().ObjC2)
@@ -885,7 +967,7 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     Selector Sel = PP.getSelectorTable().getNullarySelector(SelIdent);
     Decl *Result
          = Actions.ActOnMethodDeclaration(getCurScope(), mLoc, Tok.getLocation(),
-                                          mType, IDecl, DSRet, ReturnType, 
+                                          mType, DSRet, ReturnType, 
                                           selLoc, Sel, 0, 
                                           CParamInfo.data(), CParamInfo.size(),
                                           methodAttrs.getList(), MethodImplKind,
@@ -894,8 +976,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     return Result;
   }
 
-  llvm::SmallVector<IdentifierInfo *, 12> KeyIdents;
-  llvm::SmallVector<Sema::ObjCArgInfo, 12> ArgInfos;
+  SmallVector<IdentifierInfo *, 12> KeyIdents;
+  SmallVector<SourceLocation, 12> KeyLocs;
+  SmallVector<Sema::ObjCArgInfo, 12> ArgInfos;
   ParseScope PrototypeScope(this,
                             Scope::FunctionPrototypeScope|Scope::DeclScope);
 
@@ -914,9 +997,12 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     ArgInfo.Type = ParsedType();
     if (Tok.is(tok::l_paren)) // Parse the argument type if present.
-      ArgInfo.Type = ParseObjCTypeName(ArgInfo.DeclSpec, OTN_ParameterType);
+      ArgInfo.Type = ParseObjCTypeName(ArgInfo.DeclSpec,
+                                       Declarator::ObjCParameterContext,
+                                       &paramAttrs);
 
     // If attributes exist before the argument name, parse them.
+    // Regardless, collect all the attributes we've parsed so far.
     ArgInfo.ArgAttrs = 0;
     if (getLang().ObjC2) {
       MaybeParseGNUAttributes(paramAttrs);
@@ -925,7 +1011,6 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
-      ConsumeCodeCompletionToken();
       KeyIdents.push_back(SelIdent);
       Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(), 
                                                  mType == tok::minus,
@@ -933,8 +1018,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                                                  ReturnType,
                                                  KeyIdents.data(), 
                                                  KeyIdents.size());
-      KeyIdents.pop_back();
-      break;
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.isNot(tok::identifier)) {
@@ -948,25 +1033,25 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     ArgInfos.push_back(ArgInfo);
     KeyIdents.push_back(SelIdent);
+    KeyLocs.push_back(selLoc);
 
     // Make sure the attributes persist.
     allParamAttrs.takeAllFrom(paramAttrs.getPool());
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
-      ConsumeCodeCompletionToken();
       Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(), 
                                                  mType == tok::minus,
                                                  /*AtParameterName=*/false,
                                                  ReturnType,
                                                  KeyIdents.data(), 
                                                  KeyIdents.size());
-      break;
+      cutOffParsing();
+      return 0;
     }
     
     // Check for another keyword selector.
-    SourceLocation Loc;
-    SelIdent = ParseObjCSelectorPiece(Loc);
+    SelIdent = ParseObjCSelectorPiece(selLoc);
     if (!SelIdent && Tok.isNot(tok::colon))
       break;
     // We have a selector or a colon, continue parsing.
@@ -1001,23 +1086,18 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   if (getLang().ObjC2)
     MaybeParseGNUAttributes(methodAttrs);
   
-  if (KeyIdents.size() == 0) {
-    // Leave prototype scope.
-    PrototypeScope.Exit();
+  if (KeyIdents.size() == 0)
     return 0;
-  }
   
   Selector Sel = PP.getSelectorTable().getSelector(KeyIdents.size(),
                                                    &KeyIdents[0]);
   Decl *Result
        = Actions.ActOnMethodDeclaration(getCurScope(), mLoc, Tok.getLocation(),
-                                        mType, IDecl, DSRet, ReturnType, 
-                                        selLoc, Sel, &ArgInfos[0], 
+                                        mType, DSRet, ReturnType, 
+                                        KeyLocs, Sel, &ArgInfos[0], 
                                         CParamInfo.data(), CParamInfo.size(),
                                         methodAttrs.getList(),
                                         MethodImplKind, isVariadic, MethodDefinition);
-  // Leave prototype scope.
-  PrototypeScope.Exit();
   
   PD.complete(Result);
   return Result;
@@ -1027,21 +1107,22 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 ///     '<' identifier-list '>'
 ///
 bool Parser::
-ParseObjCProtocolReferences(llvm::SmallVectorImpl<Decl *> &Protocols,
-                            llvm::SmallVectorImpl<SourceLocation> &ProtocolLocs,
+ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
+                            SmallVectorImpl<SourceLocation> &ProtocolLocs,
                             bool WarnOnDeclarations,
                             SourceLocation &LAngleLoc, SourceLocation &EndLoc) {
   assert(Tok.is(tok::less) && "expected <");
 
   LAngleLoc = ConsumeToken(); // the "<"
 
-  llvm::SmallVector<IdentifierLocPair, 8> ProtocolIdents;
+  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
 
   while (1) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents.data(), 
                                                  ProtocolIdents.size());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return true;
     }
 
     if (Tok.isNot(tok::identifier)) {
@@ -1080,8 +1161,8 @@ bool Parser::ParseObjCProtocolQualifiers(DeclSpec &DS) {
   assert(Tok.is(tok::less) && "Protocol qualifiers start with '<'");
   assert(getLang().ObjC1 && "Protocol qualifiers only exist in Objective-C");
   SourceLocation LAngleLoc, EndProtoLoc;
-  llvm::SmallVector<Decl *, 8> ProtocolDecl;
-  llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
+  SmallVector<Decl *, 8> ProtocolDecl;
+  SmallVector<SourceLocation, 8> ProtocolLocs;
   bool Result = ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
                                             LAngleLoc, EndProtoLoc);
   DS.setProtocolQualifiers(ProtocolDecl.data(), ProtocolDecl.size(),
@@ -1116,11 +1197,13 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
                                              tok::ObjCKeywordKind visibility,
                                              SourceLocation atLoc) {
   assert(Tok.is(tok::l_brace) && "expected {");
-  llvm::SmallVector<Decl *, 32> AllIvarDecls;
-
+  SmallVector<Decl *, 32> AllIvarDecls;
+    
   ParseScope ClassScope(this, Scope::DeclScope|Scope::ClassScope);
+  ObjCDeclContextSwitch ObjCDC(*this);
 
-  SourceLocation LBraceLoc = ConsumeBrace(); // the "{"
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  T.consumeOpen();
 
   // While we still have something to read, read the instance variables.
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
@@ -1140,7 +1223,7 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCAtVisibility(getCurScope());
-        ConsumeCodeCompletionToken();
+        return cutOffParsing();
       }
       
       switch (Tok.getObjCKeywordID()) {
@@ -1160,26 +1243,28 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                        Sema::PCC_ObjCInstanceVariableList);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     
     struct ObjCIvarCallback : FieldCallback {
       Parser &P;
       Decl *IDecl;
       tok::ObjCKeywordKind visibility;
-      llvm::SmallVectorImpl<Decl *> &AllIvarDecls;
+      SmallVectorImpl<Decl *> &AllIvarDecls;
 
       ObjCIvarCallback(Parser &P, Decl *IDecl, tok::ObjCKeywordKind V,
-                       llvm::SmallVectorImpl<Decl *> &AllIvarDecls) :
+                       SmallVectorImpl<Decl *> &AllIvarDecls) :
         P(P), IDecl(IDecl), visibility(V), AllIvarDecls(AllIvarDecls) {
       }
 
       Decl *invoke(FieldDeclarator &FD) {
+        P.Actions.ActOnObjCContainerStartDefinition(IDecl);
         // Install the declarator into the interface decl.
         Decl *Field
           = P.Actions.ActOnIvar(P.getCurScope(),
                                 FD.D.getDeclSpec().getSourceRange().getBegin(),
-                                IDecl, FD.D, FD.BitfieldSize, visibility);
+                                FD.D, FD.BitfieldSize, visibility);
+        P.Actions.ActOnObjCContainerFinishDefinition();
         if (Field)
           AllIvarDecls.push_back(Field);
         return Field;
@@ -1198,13 +1283,16 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
       SkipUntil(tok::r_brace, true, true);
     }
   }
-  SourceLocation RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
-  Actions.ActOnLastBitfield(RBraceLoc, interfaceDecl, AllIvarDecls);
+  T.consumeClose();
+
+  Actions.ActOnObjCContainerStartDefinition(interfaceDecl);
+  Actions.ActOnLastBitfield(T.getCloseLocation(), AllIvarDecls);
+  Actions.ActOnObjCContainerFinishDefinition();
   // Call ActOnFields() even if we don't have any decls. This is useful
   // for code rewriting tools that need to be aware of the empty list.
   Actions.ActOnFields(getCurScope(), atLoc, interfaceDecl,
-                      AllIvarDecls.data(), AllIvarDecls.size(),
-                      LBraceLoc, RBraceLoc, 0);
+                      AllIvarDecls,
+                      T.getOpenLocation(), T.getCloseLocation(), 0);
   return;
 }
 
@@ -1232,7 +1320,8 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCProtocolDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   if (Tok.isNot(tok::identifier)) {
@@ -1251,7 +1340,7 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   }
 
   if (Tok.is(tok::comma)) { // list of forward declarations.
-    llvm::SmallVector<IdentifierLocPair, 8> ProtocolRefs;
+    SmallVector<IdentifierLocPair, 8> ProtocolRefs;
     ProtocolRefs.push_back(std::make_pair(protocolName, nameLoc));
 
     // Parse the list of forward declarations.
@@ -1282,8 +1371,8 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   // Last, and definitely not least, parse a protocol declaration.
   SourceLocation LAngleLoc, EndProtoLoc;
 
-  llvm::SmallVector<Decl *, 8> ProtocolRefs;
-  llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
+  SmallVector<Decl *, 8> ProtocolRefs;
+  SmallVector<SourceLocation, 8> ProtocolLocs;
   if (Tok.is(tok::less) &&
       ParseObjCProtocolReferences(ProtocolRefs, ProtocolLocs, false,
                                   LAngleLoc, EndProtoLoc))
@@ -1295,7 +1384,8 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
                                         ProtocolRefs.size(),
                                         ProtocolLocs.data(),
                                         EndProtoLoc, attrs.getList());
-  ParseObjCInterfaceDeclList(ProtoType, tok::objc_protocol);
+
+  ParseObjCInterfaceDeclList(tok::objc_protocol, ProtoType);
   return ProtoType;
 }
 
@@ -1318,7 +1408,8 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
   // Code completion after '@implementation'.
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCImplementationDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   if (Tok.isNot(tok::identifier)) {
@@ -1337,7 +1428,8 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
 
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.is(tok::identifier)) {
@@ -1356,6 +1448,7 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
     Decl *ImplCatType = Actions.ActOnStartCategoryImplementation(
                                     atLoc, nameId, nameLoc, categoryId,
                                     categoryLoc);
+
     ObjCImpDecl = ImplCatType;
     PendingObjCImpDecl.push_back(ObjCImpDecl);
     return 0;
@@ -1378,29 +1471,38 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
                                   superClassId, superClassLoc);
 
   if (Tok.is(tok::l_brace)) // we have ivars
-    ParseObjCClassInstanceVariables(ImplClsType/*FIXME*/, 
-                                    tok::objc_private, atLoc);
+    ParseObjCClassInstanceVariables(ImplClsType, tok::objc_private, atLoc);
+
   ObjCImpDecl = ImplClsType;
   PendingObjCImpDecl.push_back(ObjCImpDecl);
-  
   return 0;
 }
 
-Decl *Parser::ParseObjCAtEndDeclaration(SourceRange atEnd) {
+Parser::DeclGroupPtrTy
+Parser::ParseObjCAtEndDeclaration(SourceRange atEnd) {
   assert(Tok.isObjCAtKeyword(tok::objc_end) &&
          "ParseObjCAtEndDeclaration(): Expected @end");
-  Decl *Result = ObjCImpDecl;
   ConsumeToken(); // the "end" identifier
+  SmallVector<Decl *, 8> DeclsInGroup;
+  Actions.DefaultSynthesizeProperties(getCurScope(), ObjCImpDecl);
+  for (size_t i = 0; i < LateParsedObjCMethods.size(); ++i) {
+    Decl *D = ParseLexedObjCMethodDefs(*LateParsedObjCMethods[i]);
+    DeclsInGroup.push_back(D);
+  }
+  DeclsInGroup.push_back(ObjCImpDecl);
+  
   if (ObjCImpDecl) {
-    Actions.ActOnAtEnd(getCurScope(), atEnd, ObjCImpDecl);
-    ObjCImpDecl = 0;
+    Actions.ActOnAtEnd(getCurScope(), atEnd);
     PendingObjCImpDecl.pop_back();
   }
-  else {
+  else
     // missing @implementation
     Diag(atEnd.getBegin(), diag::err_expected_implementation);
-  }
-  return Result;
+    
+  LateParsedObjCMethods.clear();
+  ObjCImpDecl = 0;
+  return Actions.BuildDeclaratorGroup(
+           DeclsInGroup.data(), DeclsInGroup.size(), false);
 }
 
 Parser::DeclGroupPtrTy Parser::FinishPendingObjCActions() {
@@ -1408,7 +1510,7 @@ Parser::DeclGroupPtrTy Parser::FinishPendingObjCActions() {
   if (PendingObjCImpDecl.empty())
     return Actions.ConvertDeclToDeclGroup(0);
   Decl *ImpDecl = PendingObjCImpDecl.pop_back_val();
-  Actions.ActOnAtEnd(getCurScope(), SourceRange(), ImpDecl);
+  Actions.ActOnAtEnd(getCurScope(), SourceRange());
   return Actions.ConvertDeclToDeclGroup(ImpDecl);
 }
 
@@ -1455,8 +1557,9 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCPropertyDefinition(getCurScope(), ObjCImpDecl);
-      ConsumeCodeCompletionToken();
+      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.isNot(tok::identifier)) {
@@ -1474,9 +1577,9 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
       ConsumeToken(); // consume '='
       
       if (Tok.is(tok::code_completion)) {
-        Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId,
-                                                       ObjCImpDecl);
-        ConsumeCodeCompletionToken();
+        Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId);
+        cutOffParsing();
+        return 0;
       }
       
       if (Tok.isNot(tok::identifier)) {
@@ -1486,7 +1589,7 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
       propertyIvar = Tok.getIdentifierInfo();
       propertyIvarLoc = ConsumeToken(); // consume ivar-name
     }
-    Actions.ActOnPropertyImplDecl(getCurScope(), atLoc, propertyLoc, true, ObjCImpDecl,
+    Actions.ActOnPropertyImplDecl(getCurScope(), atLoc, propertyLoc, true,
                                   propertyId, propertyIvar, propertyIvarLoc);
     if (Tok.isNot(tok::comma))
       break;
@@ -1509,8 +1612,9 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
   ConsumeToken(); // consume dynamic
   while (true) {
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCPropertyDefinition(getCurScope(), ObjCImpDecl);
-      ConsumeCodeCompletionToken();
+      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.isNot(tok::identifier)) {
@@ -1521,7 +1625,7 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
     
     IdentifierInfo *propertyId = Tok.getIdentifierInfo();
     SourceLocation propertyLoc = ConsumeToken(); // consume property name
-    Actions.ActOnPropertyImplDecl(getCurScope(), atLoc, propertyLoc, false, ObjCImpDecl,
+    Actions.ActOnPropertyImplDecl(getCurScope(), atLoc, propertyLoc, false,
                                   propertyId, 0, SourceLocation());
 
     if (Tok.isNot(tok::comma))
@@ -1560,31 +1664,46 @@ Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
     Diag(Tok, diag::err_expected_lparen_after) << "@synchronized";
     return StmtError();
   }
+
+  // The operand is surrounded with parentheses.
   ConsumeParen();  // '('
-  ExprResult Res(ParseExpression());
-  if (Res.isInvalid()) {
-    SkipUntil(tok::semi);
-    return StmtError();
+  ExprResult operand(ParseExpression());
+
+  if (Tok.is(tok::r_paren)) {
+    ConsumeParen();  // ')'
+  } else {
+    if (!operand.isInvalid())
+      Diag(Tok, diag::err_expected_rparen);
+
+    // Skip forward until we see a left brace, but don't consume it.
+    SkipUntil(tok::l_brace, true, true);
   }
-  if (Tok.isNot(tok::r_paren)) {
-    Diag(Tok, diag::err_expected_lbrace);
-    return StmtError();
-  }
-  ConsumeParen();  // ')'
+
+  // Require a compound statement.
   if (Tok.isNot(tok::l_brace)) {
-    Diag(Tok, diag::err_expected_lbrace);
+    if (!operand.isInvalid())
+      Diag(Tok, diag::err_expected_lbrace);
     return StmtError();
   }
-  // Enter a scope to hold everything within the compound stmt.  Compound
-  // statements can always hold declarations.
-  ParseScope BodyScope(this, Scope::DeclScope);
 
-  StmtResult SynchBody(ParseCompoundStatementBody());
+  // Check the @synchronized operand now.
+  if (!operand.isInvalid())
+    operand = Actions.ActOnObjCAtSynchronizedOperand(atLoc, operand.take());
 
-  BodyScope.Exit();
-  if (SynchBody.isInvalid())
-    SynchBody = Actions.ActOnNullStmt(Tok.getLocation());
-  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, Res.take(), SynchBody.take());
+  // Parse the compound statement within a new scope.
+  ParseScope bodyScope(this, Scope::DeclScope);
+  StmtResult body(ParseCompoundStatementBody());
+  bodyScope.Exit();
+
+  // If there was a semantic or parse error earlier with the
+  // operand, fail now.
+  if (operand.isInvalid())
+    return StmtError();
+
+  if (body.isInvalid())
+    body = Actions.ActOnNullStmt(Tok.getLocation());
+
+  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, operand.get(), body.get());
 }
 
 ///  objc-try-catch-statement:
@@ -1724,7 +1843,7 @@ Parser::ParseObjCAutoreleasePoolStmt(SourceLocation atLoc) {
 ///   objc-method-def: objc-method-proto ';'[opt] '{' body '}'
 ///
 Decl *Parser::ParseObjCMethodDefinition() {
-  Decl *MDecl = ParseObjCMethodPrototype(ObjCImpDecl);
+  Decl *MDecl = ParseObjCMethodPrototype();
 
   PrettyDeclStackTraceEntry CrashInfo(Actions, MDecl, Tok.getLocation(),
                                       "parsing Objective-C method");
@@ -1749,42 +1868,26 @@ Decl *Parser::ParseObjCMethodDefinition() {
     if (Tok.isNot(tok::l_brace))
       return 0;
   }
-  SourceLocation BraceLoc = Tok.getLocation();
-
-  // Enter a scope for the method body.
-  ParseScope BodyScope(this,
-                       Scope::ObjCMethodScope|Scope::FnScope|Scope::DeclScope);
-
-  // Tell the actions module that we have entered a method definition with the
-  // specified Declarator for the method.
-  Actions.ActOnStartOfObjCMethodDef(getCurScope(), MDecl);
-
-  if (PP.isCodeCompletionEnabled()) {
-    if (trySkippingFunctionBodyForCodeCompletion()) {
-      BodyScope.Exit();
-      return Actions.ActOnFinishFunctionBody(MDecl, 0);
-    }
-  }
-
-  StmtResult FnBody(ParseCompoundStatementBody());
-
-  // If the function body could not be parsed, make a bogus compoundstmt.
-  if (FnBody.isInvalid())
-    FnBody = Actions.ActOnCompoundStmt(BraceLoc, BraceLoc,
-                                       MultiStmtArg(Actions), false);
-
-  // Leave the function body scope.
-  BodyScope.Exit();
-  
-  // TODO: Pass argument information.
-  Actions.ActOnFinishFunctionBody(MDecl, FnBody.take());
+  // Allow the rest of sema to find private method decl implementations.
+  if (MDecl)
+    Actions.AddAnyMethodToGlobalPool(MDecl);
+    
+  // Consume the tokens and store them for later parsing.
+  LexedMethod* LM = new LexedMethod(this, MDecl);
+  LateParsedObjCMethods.push_back(LM);
+  CachedTokens &Toks = LM->Toks;
+  // Begin by storing the '{' token.
+  Toks.push_back(Tok);
+  ConsumeBrace();
+  // Consume everything up to (and including) the matching right brace.
+  ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
   return MDecl;
 }
 
 StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCAtStatement(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
     return StmtError();
   }
   
@@ -1818,7 +1921,7 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
   switch (Tok.getKind()) {
   case tok::code_completion:
     Actions.CodeCompleteObjCAtExpression(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
     return ExprError();
 
   case tok::string_literal:    // primary-expression: string-literal
@@ -1984,8 +2087,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMessageReceiver(getCurScope());
-    ConsumeCodeCompletionToken();
-    SkipUntil(tok::r_square);
+    cutOffParsing();
     return ExprError();
   }
   
@@ -2116,22 +2218,23 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
     else
       Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                               0, 0, false);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return ExprError();
   }
   
   // Parse objc-selector
   SourceLocation Loc;
   IdentifierInfo *selIdent = ParseObjCSelectorPiece(Loc);
 
-  SourceLocation SelectorLoc = Loc;
-
-  llvm::SmallVector<IdentifierInfo *, 12> KeyIdents;
+  SmallVector<IdentifierInfo *, 12> KeyIdents;
+  SmallVector<SourceLocation, 12> KeyLocs;
   ExprVector KeyExprs(Actions);
 
   if (Tok.is(tok::colon)) {
     while (1) {
       // Each iteration parses a single keyword argument.
       KeyIdents.push_back(selIdent);
+      KeyLocs.push_back(Loc);
 
       if (Tok.isNot(tok::colon)) {
         Diag(Tok, diag::err_expected_colon);
@@ -2162,8 +2265,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                                   KeyIdents.size(),
                                                   /*AtArgumentEpression=*/true);
 
-        ConsumeCodeCompletionToken();
-        SkipUntil(tok::r_square);
+        cutOffParsing();
         return ExprError();
       }
       
@@ -2196,8 +2298,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                                   KeyIdents.data(), 
                                                   KeyIdents.size(),
                                                 /*AtArgumentEpression=*/false);
-        ConsumeCodeCompletionToken();
-        SkipUntil(tok::r_square);
+        cutOffParsing();
         return ExprError();
       }
             
@@ -2248,24 +2349,26 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
   SourceLocation RBracLoc = ConsumeBracket(); // consume ']'
 
   unsigned nKeys = KeyIdents.size();
-  if (nKeys == 0)
+  if (nKeys == 0) {
     KeyIdents.push_back(selIdent);
+    KeyLocs.push_back(Loc);
+  }
   Selector Sel = PP.getSelectorTable().getSelector(nKeys, &KeyIdents[0]);
 
   if (SuperLoc.isValid())
     return Actions.ActOnSuperMessage(getCurScope(), SuperLoc, Sel,
-                                     LBracLoc, SelectorLoc, RBracLoc,
+                                     LBracLoc, KeyLocs, RBracLoc,
                                      MultiExprArg(Actions, 
                                                   KeyExprs.take(),
                                                   KeyExprs.size()));
   else if (ReceiverType)
     return Actions.ActOnClassMessage(getCurScope(), ReceiverType, Sel,
-                                     LBracLoc, SelectorLoc, RBracLoc,
+                                     LBracLoc, KeyLocs, RBracLoc,
                                      MultiExprArg(Actions, 
                                                   KeyExprs.take(), 
                                                   KeyExprs.size()));
   return Actions.ActOnInstanceMessage(getCurScope(), ReceiverExpr, Sel,
-                                      LBracLoc, SelectorLoc, RBracLoc,
+                                      LBracLoc, KeyLocs, RBracLoc,
                                       MultiExprArg(Actions, 
                                                    KeyExprs.take(), 
                                                    KeyExprs.size()));
@@ -2278,7 +2381,7 @@ ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
   // @"foo" @"bar" is a valid concatenated string.  Eat any subsequent string
   // expressions.  At this point, we know that the only valid thing that starts
   // with '@' is an @"".
-  llvm::SmallVector<SourceLocation, 4> AtLocs;
+  SmallVector<SourceLocation, 4> AtLocs;
   ExprVector AtStrings(Actions);
   AtLocs.push_back(AtLoc);
   AtStrings.push_back(Res.release());
@@ -2312,17 +2415,19 @@ Parser::ParseObjCEncodeExpression(SourceLocation AtLoc) {
   if (Tok.isNot(tok::l_paren))
     return ExprError(Diag(Tok, diag::err_expected_lparen_after) << "@encode");
 
-  SourceLocation LParenLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
 
   TypeResult Ty = ParseTypeName();
 
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
 
   if (Ty.isInvalid())
     return ExprError();
 
-  return Owned(Actions.ParseObjCEncodeExpression(AtLoc, EncLoc, LParenLoc,
-                                                 Ty.get(), RParenLoc));
+  return Owned(Actions.ParseObjCEncodeExpression(AtLoc, EncLoc,
+                                                 T.getOpenLocation(), Ty.get(),
+                                                 T.getCloseLocation()));
 }
 
 ///     objc-protocol-expression
@@ -2334,7 +2439,8 @@ Parser::ParseObjCProtocolExpression(SourceLocation AtLoc) {
   if (Tok.isNot(tok::l_paren))
     return ExprError(Diag(Tok, diag::err_expected_lparen_after) << "@protocol");
 
-  SourceLocation LParenLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
 
   if (Tok.isNot(tok::identifier))
     return ExprError(Diag(Tok, diag::err_expected_ident));
@@ -2342,10 +2448,11 @@ Parser::ParseObjCProtocolExpression(SourceLocation AtLoc) {
   IdentifierInfo *protocolId = Tok.getIdentifierInfo();
   ConsumeToken();
 
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
 
   return Owned(Actions.ParseObjCProtocolExpression(protocolId, AtLoc, ProtoLoc,
-                                                   LParenLoc, RParenLoc));
+                                                   T.getOpenLocation(),
+                                                   T.getCloseLocation()));
 }
 
 ///     objc-selector-expression
@@ -2356,15 +2463,16 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
   if (Tok.isNot(tok::l_paren))
     return ExprError(Diag(Tok, diag::err_expected_lparen_after) << "@selector");
 
-  llvm::SmallVector<IdentifierInfo *, 12> KeyIdents;
-  SourceLocation LParenLoc = ConsumeParen();
+  SmallVector<IdentifierInfo *, 12> KeyIdents;
   SourceLocation sLoc;
   
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents.data(),
                                      KeyIdents.size());
-    ConsumeCodeCompletionToken();
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    cutOffParsing();
     return ExprError();
   }
   
@@ -2391,8 +2499,7 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents.data(),
                                          KeyIdents.size());
-        ConsumeCodeCompletionToken();
-        MatchRHSPunctuation(tok::r_paren, LParenLoc);
+        cutOffParsing();
         return ExprError();
       }
 
@@ -2404,8 +2511,52 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
         break;
     }
   }
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
   Selector Sel = PP.getSelectorTable().getSelector(nColons, &KeyIdents[0]);
   return Owned(Actions.ParseObjCSelectorExpression(Sel, AtLoc, SelectorLoc,
-                                                   LParenLoc, RParenLoc));
+                                                   T.getOpenLocation(),
+                                                   T.getCloseLocation()));
  }
+
+Decl *Parser::ParseLexedObjCMethodDefs(LexedMethod &LM) {
+    
+  assert(!LM.Toks.empty() && "ParseLexedObjCMethodDef - Empty body!");
+  // Append the current token at the end of the new token stream so that it
+  // doesn't get lost.
+  LM.Toks.push_back(Tok);
+  PP.EnterTokenStream(LM.Toks.data(), LM.Toks.size(), true, false);
+  
+  // MDecl might be null due to error in method prototype, etc.
+  Decl *MDecl = LM.D;
+  // Consume the previously pushed token.
+  ConsumeAnyToken();
+    
+  assert(Tok.is(tok::l_brace) && "Inline objective-c method not starting with '{'");
+  SourceLocation BraceLoc = Tok.getLocation();
+  // Enter a scope for the method body.
+  ParseScope BodyScope(this,
+                       Scope::ObjCMethodScope|Scope::FnScope|Scope::DeclScope);
+    
+  // Tell the actions module that we have entered a method definition with the
+  // specified Declarator for the method.
+  Actions.ActOnStartOfObjCMethodDef(getCurScope(), MDecl);
+    
+  if (PP.isCodeCompletionEnabled()) {
+      if (trySkippingFunctionBodyForCodeCompletion()) {
+          BodyScope.Exit();
+          return Actions.ActOnFinishFunctionBody(MDecl, 0);
+      }
+  }
+    
+  StmtResult FnBody(ParseCompoundStatementBody());
+    
+  // If the function body could not be parsed, make a bogus compoundstmt.
+  if (FnBody.isInvalid())
+    FnBody = Actions.ActOnCompoundStmt(BraceLoc, BraceLoc,
+                                       MultiStmtArg(Actions), false);
+    
+  // Leave the function body scope.
+  BodyScope.Exit();
+    
+  return Actions.ActOnFinishFunctionBody(MDecl, FnBody.take());
+}
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParsePragma.cpp b/contrib/llvm/tools/clang/lib/Parse/ParsePragma.cpp
index c30ab75b76e8..2ccb6ea88d0f 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParsePragma.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParsePragma.cpp
@@ -38,7 +38,7 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP,
   SourceLocation VisLoc = VisTok.getLocation();
 
   Token Tok;
-  PP.Lex(Tok);
+  PP.LexUnexpandedToken(Tok);
 
   const IdentifierInfo *PushPop = Tok.getIdentifierInfo();
 
@@ -49,20 +49,20 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP,
     VisType = 0;
   } else if (PushPop && PushPop->isStr("push")) {
     IsPush = true;
-    PP.Lex(Tok);
+    PP.LexUnexpandedToken(Tok);
     if (Tok.isNot(tok::l_paren)) {
       PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen)
         << "visibility";
       return;
     }
-    PP.Lex(Tok);
+    PP.LexUnexpandedToken(Tok);
     VisType = Tok.getIdentifierInfo();
     if (!VisType) {
       PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier)
         << "visibility";
       return;
     }
-    PP.Lex(Tok);
+    PP.LexUnexpandedToken(Tok);
     if (Tok.isNot(tok::r_paren)) {
       PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen)
         << "visibility";
@@ -73,7 +73,7 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP,
       << "visibility";
     return;
   }
-  PP.Lex(Tok);
+  PP.LexUnexpandedToken(Tok);
   if (Tok.isNot(tok::eod)) {
     PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
       << "visibility";
@@ -297,7 +297,7 @@ void PragmaUnusedHandler::HandlePragma(Preprocessor &PP,
   }
 
   // Lex the declaration reference(s).
-  llvm::SmallVector<Token, 5> Identifiers;
+  SmallVector<Token, 5> Identifiers;
   SourceLocation RParenLoc;
   bool LexID = true;
 
@@ -451,8 +451,11 @@ PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP,
   }
 
   OpenCLOptions &f = Actions.getOpenCLOptions();
-  if (ename->isStr("all")) {
-#define OPENCLEXT(nm)   f.nm = state;
+  // OpenCL 1.1 9.1: "The all variant sets the behavior for all extensions,
+  // overriding all previously issued extension directives, but only if the
+  // behavior is set to disable."
+  if (state == 0 && ename->isStr("all")) {
+#define OPENCLEXT(nm)   f.nm = 0;
 #include "clang/Basic/OpenCLExtensions.def"
   }
 #define OPENCLEXT(nm) else if (ename->isStr(#nm)) { f.nm = state; }
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseStmt.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseStmt.cpp
index b91bca55a9dc..a2b7cdd81b87 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseStmt.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseStmt.cpp
@@ -79,7 +79,7 @@ StmtResult
 Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
   const char *SemiError = 0;
   StmtResult Res;
-  
+
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
 
   ParsedAttributesWithRange attrs(AttrFactory);
@@ -100,20 +100,25 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
 
   case tok::code_completion:
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
-    ConsumeCodeCompletionToken();
-    return ParseStatementOrDeclaration(Stmts, OnlyStatement);
-      
+    cutOffParsing();
+    return StmtError();
+
   case tok::identifier: {
     Token Next = NextToken();
     if (Next.is(tok::colon)) { // C99 6.8.1: labeled-statement
       // identifier ':' statement
       return ParseLabeledStatement(attrs);
     }
-    
+
     if (Next.isNot(tok::coloncolon)) {
       CXXScopeSpec SS;
       IdentifierInfo *Name = Tok.getIdentifierInfo();
       SourceLocation NameLoc = Tok.getLocation();
+
+      if (getLang().CPlusPlus)
+        CheckForTemplateAndDigraph(Next, ParsedType(),
+                                   /*EnteringContext=*/false, *Name, SS);
+
       Sema::NameClassification Classification
         = Actions.ClassifyName(getCurScope(), SS, Name, NameLoc, Next);
       switch (Classification.getKind()) {
@@ -125,11 +130,11 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
           Tok.setKind(Name->getTokenID());
           goto Retry;
         }
-          
+
         // Fall through via the normal error path.
         // FIXME: This seems like it could only happen for context-sensitive
         // keywords.
-          
+
       case Sema::NC_Error:
         // Handle errors here by skipping up to the next semicolon or '}', and
         // eat the semicolon if that's what stopped us.
@@ -137,33 +142,33 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
         if (Tok.is(tok::semi))
           ConsumeToken();
         return StmtError();
-                     
+
       case Sema::NC_Unknown:
         // Either we don't know anything about this identifier, or we know that
-        // we're in a syntactic context we haven't handled yet. 
-        break;     
-          
+        // we're in a syntactic context we haven't handled yet.
+        break;
+
       case Sema::NC_Type:
         Tok.setKind(tok::annot_typename);
         setTypeAnnotation(Tok, Classification.getType());
         Tok.setAnnotationEndLoc(NameLoc);
         PP.AnnotateCachedTokens(Tok);
         break;
-          
+
       case Sema::NC_Expression:
         Tok.setKind(tok::annot_primary_expr);
         setExprAnnotation(Tok, Classification.getExpression());
         Tok.setAnnotationEndLoc(NameLoc);
         PP.AnnotateCachedTokens(Tok);
         break;
-          
+
       case Sema::NC_TypeTemplate:
       case Sema::NC_FunctionTemplate: {
         ConsumeToken(); // the identifier
         UnqualifiedId Id;
         Id.setIdentifier(Name, NameLoc);
         if (AnnotateTemplateIdToken(
-                            TemplateTy::make(Classification.getTemplateName()), 
+                            TemplateTy::make(Classification.getTemplateName()),
                                     Classification.getTemplateNameKind(),
                                     SS, Id, SourceLocation(),
                                     /*AllowTypeAnnotation=*/false)) {
@@ -172,10 +177,10 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
           SkipUntil(tok::r_brace, /*StopAtSemi=*/true, /*DontConsume=*/true);
           if (Tok.is(tok::semi))
             ConsumeToken();
-          return StmtError();        
+          return StmtError();
         }
-        
-        // If the next token is '::', jump right into parsing a 
+
+        // If the next token is '::', jump right into parsing a
         // nested-name-specifier. We don't want to leave the template-id
         // hanging.
         if (NextToken().is(tok::coloncolon) && TryAnnotateCXXScopeToken(false)){
@@ -184,22 +189,22 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
           SkipUntil(tok::r_brace, /*StopAtSemi=*/true, /*DontConsume=*/true);
           if (Tok.is(tok::semi))
             ConsumeToken();
-          return StmtError();        
+          return StmtError();
         }
-        
+
         // We've annotated a template-id, so try again now.
         goto Retry;
       }
-       
+
       case Sema::NC_NestedNameSpecifier:
         // FIXME: Implement this!
         break;
       }
     }
-    
+
     // Fall through
   }
-      
+
   default: {
     if ((getLang().CPlusPlus || !OnlyStatement) && isDeclarationStatement()) {
       SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
@@ -224,10 +229,8 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
   case tok::l_brace:                // C99 6.8.2: compound-statement
     return ParseCompoundStatement(attrs);
   case tok::semi: {                 // C99 6.8.3p3: expression[opt] ';'
-    SourceLocation LeadingEmptyMacroLoc;
-    if (Tok.hasLeadingEmptyMacro())
-      LeadingEmptyMacroLoc = PP.getLastEmptyMacroExpansionLoc();
-    return Actions.ActOnNullStmt(ConsumeToken(), LeadingEmptyMacroLoc);
+    bool HasLeadingEmptyMacro = Tok.hasLeadingEmptyMacro();
+    return Actions.ActOnNullStmt(ConsumeToken(), HasLeadingEmptyMacro);
   }
 
   case tok::kw_if:                  // C99 6.8.4.1: if-statement
@@ -297,7 +300,7 @@ Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
 StmtResult Parser::ParseExprStatement(ParsedAttributes &Attrs) {
   // If a case keyword is missing, this is where it should be inserted.
   Token OldToken = Tok;
-  
+
   // FIXME: Use the attributes
   // expression[opt] ';'
   ExprResult Expr(ParseExpression());
@@ -310,18 +313,18 @@ StmtResult Parser::ParseExprStatement(ParsedAttributes &Attrs) {
       ConsumeToken();
     return StmtError();
   }
-  
+
   if (Tok.is(tok::colon) && getCurScope()->isSwitchScope() &&
       Actions.CheckCaseExpression(Expr.get())) {
     // If a constant expression is followed by a colon inside a switch block,
     // suggest a missing case keyword.
     Diag(OldToken, diag::err_expected_case_before_expression)
       << FixItHint::CreateInsertion(OldToken.getLocation(), "case ");
-    
+
     // Recover parsing as a case statement.
     return ParseCaseStatement(Attrs, /*MissingCase=*/true, Expr);
   }
-  
+
   // Otherwise, eat the semicolon.
   ExpectAndConsumeSemi(diag::err_expected_semi_after_expr);
   return Actions.ActOnExprStmt(Actions.MakeFullExpr(Expr.get()));
@@ -458,12 +461,12 @@ StmtResult Parser::ParseLabeledStatement(ParsedAttributes &attrs) {
   // Broken substmt shouldn't prevent the label from being added to the AST.
   if (SubStmt.isInvalid())
     SubStmt = Actions.ActOnNullStmt(ColonLoc);
-  
+
   LabelDecl *LD = Actions.LookupOrCreateLabel(IdentTok.getIdentifierInfo(),
                                               IdentTok.getLocation());
   if (AttributeList *Attrs = attrs.getList())
     Actions.ProcessDeclAttributeList(Actions.CurScope, LD, Attrs);
-  
+
   return Actions.ActOnLabelStmt(IdentTok.getLocation(), LD, ColonLoc,
                                 SubStmt.get());
 }
@@ -499,7 +502,7 @@ StmtResult Parser::ParseCaseStatement(ParsedAttributes &attrs, bool MissingCase,
   // DeepestParsedCaseStmt - This is the deepest statement we have parsed, which
   // gets updated each time a new case is parsed, and whose body is unset so
   // far.  When parsing 'case 4', this is the 'case 3' node.
-  StmtTy *DeepestParsedCaseStmt = 0;
+  Stmt *DeepestParsedCaseStmt = 0;
 
   // While we have case statements, eat and stack them.
   SourceLocation ColonLoc;
@@ -509,14 +512,15 @@ StmtResult Parser::ParseCaseStatement(ParsedAttributes &attrs, bool MissingCase,
 
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteCase(getCurScope());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return StmtError();
     }
-    
+
     /// We don't want to treat 'case x : y' as a potential typo for 'case x::y'.
     /// Disable this form of error recovery while we're parsing the case
     /// expression.
     ColonProtectionRAIIObject ColonProtection(*this);
-    
+
     ExprResult LHS(MissingCase ? Expr : ParseConstantExpression());
     MissingCase = false;
     if (LHS.isInvalid()) {
@@ -537,7 +541,7 @@ StmtResult Parser::ParseCaseStatement(ParsedAttributes &attrs, bool MissingCase,
         return StmtError();
       }
     }
-    
+
     ColonProtection.restore();
 
     if (Tok.is(tok::colon)) {
@@ -554,7 +558,7 @@ StmtResult Parser::ParseCaseStatement(ParsedAttributes &attrs, bool MissingCase,
         << FixItHint::CreateInsertion(ExpectedLoc, ":");
       ColonLoc = ExpectedLoc;
     }
-    
+
     StmtResult Case =
       Actions.ActOnCaseStmt(CaseLoc, LHS.get(), DotDotDotLoc,
                             RHS.get(), ColonLoc);
@@ -631,7 +635,7 @@ StmtResult Parser::ParseDefaultStatement(ParsedAttributes &attrs) {
       << FixItHint::CreateInsertion(ExpectedLoc, ":");
     ColonLoc = ExpectedLoc;
   }
-  
+
   // Diagnose the common error "switch (X) {... default: }", which is not valid.
   if (Tok.is(tok::r_brace)) {
     SourceLocation AfterColonLoc = PP.getLocForEndOfToken(ColonLoc);
@@ -704,8 +708,9 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
                                 Tok.getLocation(),
                                 "in compound statement ('{}')");
   InMessageExpressionRAIIObject InMessage(*this, false);
-  
-  SourceLocation LBraceLoc = ConsumeBrace();  // eat the '{'.
+  BalancedDelimiterTracker T(*this, tok::l_brace);
+  if (T.consumeOpen())
+    return StmtError();
 
   StmtVector Stmts(Actions);
 
@@ -714,40 +719,40 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   while (Tok.is(tok::kw___label__)) {
     SourceLocation LabelLoc = ConsumeToken();
     Diag(LabelLoc, diag::ext_gnu_local_label);
-    
-    llvm::SmallVector<Decl *, 8> DeclsInGroup;
+
+    SmallVector<Decl *, 8> DeclsInGroup;
     while (1) {
       if (Tok.isNot(tok::identifier)) {
         Diag(Tok, diag::err_expected_ident);
         break;
       }
-      
+
       IdentifierInfo *II = Tok.getIdentifierInfo();
       SourceLocation IdLoc = ConsumeToken();
       DeclsInGroup.push_back(Actions.LookupOrCreateLabel(II, IdLoc, LabelLoc));
-      
+
       if (!Tok.is(tok::comma))
         break;
       ConsumeToken();
     }
-    
+
     DeclSpec DS(AttrFactory);
     DeclGroupPtrTy Res = Actions.FinalizeDeclaratorGroup(getCurScope(), DS,
                                       DeclsInGroup.data(), DeclsInGroup.size());
     StmtResult R = Actions.ActOnDeclStmt(Res, LabelLoc, Tok.getLocation());
-    
+
     ExpectAndConsume(tok::semi, diag::err_expected_semi_declaration);
     if (R.isUsable())
       Stmts.push_back(R.release());
   }
-  
+
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
     if (Tok.is(tok::annot_pragma_unused)) {
       HandlePragmaUnused();
       continue;
     }
 
-    if (getLang().Microsoft && (Tok.is(tok::kw___if_exists) ||
+    if (getLang().MicrosoftExt && (Tok.is(tok::kw___if_exists) ||
         Tok.is(tok::kw___if_not_exists))) {
       ParseMicrosoftIfExistsStatement(Stmts);
       continue;
@@ -803,13 +808,15 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   // We broke out of the while loop because we found a '}' or EOF.
   if (Tok.isNot(tok::r_brace)) {
     Diag(Tok, diag::err_expected_rbrace);
-    Diag(LBraceLoc, diag::note_matching) << "{";
+    Diag(T.getOpenLocation(), diag::note_matching) << "{";
     return StmtError();
   }
 
-  SourceLocation RBraceLoc = ConsumeBrace();
-  return Actions.ActOnCompoundStmt(LBraceLoc, RBraceLoc, move_arg(Stmts),
-                                   isStmtExpr);
+  if (T.consumeClose())
+    return StmtError();
+
+  return Actions.ActOnCompoundStmt(T.getOpenLocation(), T.getCloseLocation(),
+                                   move_arg(Stmts), isStmtExpr);
 }
 
 /// ParseParenExprOrCondition:
@@ -827,13 +834,15 @@ bool Parser::ParseParenExprOrCondition(ExprResult &ExprResult,
                                        Decl *&DeclResult,
                                        SourceLocation Loc,
                                        bool ConvertToBoolean) {
-  SourceLocation LParenLoc = ConsumeParen();
-  if (getLang().CPlusPlus) 
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+
+  if (getLang().CPlusPlus)
     ParseCXXCondition(ExprResult, DeclResult, Loc, ConvertToBoolean);
   else {
     ExprResult = ParseExpression();
     DeclResult = 0;
-    
+
     // If required, convert to a boolean value.
     if (!ExprResult.isInvalid() && ConvertToBoolean)
       ExprResult
@@ -852,7 +861,7 @@ bool Parser::ParseParenExprOrCondition(ExprResult &ExprResult,
   }
 
   // Otherwise the condition is valid or the rparen is present.
-  MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
   return false;
 }
 
@@ -950,9 +959,13 @@ StmtResult Parser::ParseIfStatement(ParsedAttributes &attrs) {
                           C99orCXX && Tok.isNot(tok::l_brace));
 
     ElseStmt = ParseStatement();
-    
+
     // Pop the 'else' scope if needed.
     InnerScope.Exit();
+  } else if (Tok.is(tok::code_completion)) {
+    Actions.CodeCompleteAfterIf(getCurScope());
+    cutOffParsing();
+    return StmtError();
   }
 
   IfScope.Exit();
@@ -1027,7 +1040,7 @@ StmtResult Parser::ParseSwitchStatement(ParsedAttributes &attrs) {
     = Actions.ActOnStartOfSwitchStmt(SwitchLoc, Cond.get(), CondVar);
 
   if (Switch.isInvalid()) {
-    // Skip the switch body. 
+    // Skip the switch body.
     // FIXME: This is not optimal recovery, but parsing the body is more
     // dangerous due to the presence of case and default statements, which
     // will have no place to connect back with the switch.
@@ -1038,7 +1051,7 @@ StmtResult Parser::ParseSwitchStatement(ParsedAttributes &attrs) {
       SkipUntil(tok::semi);
     return move(Switch);
   }
-  
+
   // C99 6.8.4p3 - In C99, the body of the switch statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
   // if the body isn't a compound statement to avoid push/pop in common cases.
@@ -1063,7 +1076,7 @@ StmtResult Parser::ParseSwitchStatement(ParsedAttributes &attrs) {
   if (Body.isInvalid())
     // FIXME: Remove the case statement list from the Switch statement.
     Body = Actions.ActOnNullStmt(Tok.getLocation());
-  
+
   return Actions.ActOnFinishSwitchStmt(SwitchLoc, Switch.get(), Body.get());
 }
 
@@ -1196,16 +1209,17 @@ StmtResult Parser::ParseDoStatement(ParsedAttributes &attrs) {
   }
 
   // Parse the parenthesized condition.
-  SourceLocation LPLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
   ExprResult Cond = ParseExpression();
-  SourceLocation RPLoc = MatchRHSPunctuation(tok::r_paren, LPLoc);
+  T.consumeClose();
   DoScope.Exit();
 
   if (Cond.isInvalid() || Body.isInvalid())
     return StmtError();
 
-  return Actions.ActOnDoStmt(DoLoc, Body.get(), WhileLoc, LPLoc,
-                             Cond.get(), RPLoc);
+  return Actions.ActOnDoStmt(DoLoc, Body.get(), WhileLoc, T.getOpenLocation(),
+                             Cond.get(), T.getCloseLocation());
 }
 
 /// ParseForStatement
@@ -1265,7 +1279,9 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
 
   ParseScope ForScope(this, ScopeFlags);
 
-  SourceLocation LParenLoc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+
   ExprResult Value;
 
   bool ForEach = false, ForRange = false;
@@ -1276,14 +1292,15 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
   ForRangeInit ForRangeInit;
   FullExprArg ThirdPart(Actions);
   Decl *SecondVar = 0;
-  
+
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteOrdinaryName(getCurScope(), 
+    Actions.CodeCompleteOrdinaryName(getCurScope(),
                                      C99orCXXorObjC? Sema::PCC_ForInit
                                                    : Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return StmtError();
   }
-  
+
   // Parse the first part of the for specifier.
   if (Tok.is(tok::semi)) {  // for (;
     // no first part, eat the ';'.
@@ -1302,13 +1319,16 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
 
     SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
     StmtVector Stmts(Actions);
-    DeclGroupPtrTy DG = ParseSimpleDeclaration(Stmts, Declarator::ForContext, 
+    DeclGroupPtrTy DG = ParseSimpleDeclaration(Stmts, Declarator::ForContext,
                                                DeclEnd, attrs, false,
                                                MightBeForRangeStmt ?
                                                  &ForRangeInit : 0);
     FirstPart = Actions.ActOnDeclStmt(DG, DeclStart, Tok.getLocation());
 
     if (ForRangeInit.ParsedForRangeDecl()) {
+      if (!getLang().CPlusPlus0x)
+        Diag(ForRangeInit.ColonLoc, diag::ext_for_range);
+
       ForRange = true;
     } else if (Tok.is(tok::semi)) {  // for (int x = 4;
       ConsumeToken();
@@ -1316,10 +1336,11 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
       Actions.ActOnForEachDeclStmt(DG);
       // ObjC: for (id x in expr)
       ConsumeToken(); // consume 'in'
-      
+
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCForCollection(getCurScope(), DG);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return StmtError();
       }
       Collection = ParseExpression();
     } else {
@@ -1342,10 +1363,11 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
       ConsumeToken();
     } else if (ForEach) {
       ConsumeToken(); // consume 'in'
-      
+
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCForCollection(getCurScope(), DeclGroupPtrTy());
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return StmtError();
       }
       Collection = ParseExpression();
     } else {
@@ -1373,7 +1395,7 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
       else {
         Second = ParseExpression();
         if (!Second.isInvalid())
-          Second = Actions.ActOnBooleanCondition(getCurScope(), ForLoc, 
+          Second = Actions.ActOnBooleanCondition(getCurScope(), ForLoc,
                                                  Second.get());
       }
       SecondPartIsInvalid = Second.isInvalid();
@@ -1399,18 +1421,27 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
     }
   }
   // Match the ')'.
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+  T.consumeClose();
 
   // We need to perform most of the semantic analysis for a C++0x for-range
   // statememt before parsing the body, in order to be able to deduce the type
   // of an auto-typed loop variable.
   StmtResult ForRangeStmt;
-  if (ForRange)
-    ForRangeStmt = Actions.ActOnCXXForRangeStmt(ForLoc, LParenLoc,
+  if (ForRange) {
+    ForRangeStmt = Actions.ActOnCXXForRangeStmt(ForLoc, T.getOpenLocation(),
                                                 FirstPart.take(),
                                                 ForRangeInit.ColonLoc,
                                                 ForRangeInit.RangeExpr.get(),
-                                                RParenLoc);
+                                                T.getCloseLocation());
+
+
+  // Similarly, we need to do the semantic analysis for a for-range
+  // statement immediately in order to close over temporaries correctly.
+  } else if (ForEach) {
+    if (!Collection.isInvalid())
+      Collection =
+        Actions.ActOnObjCForCollectionOperand(ForLoc, Collection.take());
+  }
 
   // C99 6.8.5p5 - In C99, the body of the if statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
@@ -1439,18 +1470,18 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
     return StmtError();
 
   if (ForEach)
-    // FIXME: It isn't clear how to communicate the late destruction of 
-    // C++ temporaries used to create the collection.
-    return Actions.ActOnObjCForCollectionStmt(ForLoc, LParenLoc,
-                                              FirstPart.take(),
-                                              Collection.take(), RParenLoc, 
-                                              Body.take());
+   return Actions.ActOnObjCForCollectionStmt(ForLoc, T.getOpenLocation(),
+                                             FirstPart.take(),
+                                             Collection.take(), 
+                                             T.getCloseLocation(),
+                                             Body.take());
 
   if (ForRange)
     return Actions.FinishCXXForRangeStmt(ForRangeStmt.take(), Body.take());
 
-  return Actions.ActOnForStmt(ForLoc, LParenLoc, FirstPart.take(), SecondPart,
-                              SecondVar, ThirdPart, RParenLoc, Body.take());
+  return Actions.ActOnForStmt(ForLoc, T.getOpenLocation(), FirstPart.take(),
+                              SecondPart, SecondVar, ThirdPart,
+                              T.getCloseLocation(), Body.take());
 }
 
 /// ParseGotoStatement
@@ -1529,11 +1560,10 @@ StmtResult Parser::ParseReturnStatement(ParsedAttributes &attrs) {
   if (Tok.isNot(tok::semi)) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteReturn(getCurScope());
-      ConsumeCodeCompletionToken();
-      SkipUntil(tok::semi, false, true);
+      cutOffParsing();
       return StmtError();
     }
-        
+
     // FIXME: This is a hack to allow something like C++0x's generalized
     // initializer lists, but only enough of this feature to allow Clang to
     // parse libstdc++ 4.5's headers.
@@ -1552,30 +1582,105 @@ StmtResult Parser::ParseReturnStatement(ParsedAttributes &attrs) {
   return Actions.ActOnReturnStmt(ReturnLoc, R.take());
 }
 
-/// FuzzyParseMicrosoftAsmStatement. When -fms-extensions is enabled, this
-/// routine is called to skip/ignore tokens that comprise the MS asm statement.
-StmtResult Parser::FuzzyParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
-  SourceLocation EndLoc;
-  if (Tok.is(tok::l_brace)) {
-    unsigned short savedBraceCount = BraceCount;
-    do {
-      EndLoc = Tok.getLocation();
-      ConsumeAnyToken();
-    } while (BraceCount > savedBraceCount && Tok.isNot(tok::eof));
-  } else {
-    // From the MS website: If used without braces, the __asm keyword means
-    // that the rest of the line is an assembly-language statement.
-    SourceManager &SrcMgr = PP.getSourceManager();
+/// ParseMicrosoftAsmStatement. When -fms-extensions/-fasm-blocks is enabled,
+/// this routine is called to collect the tokens for an MS asm statement.
+StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
+  SourceManager &SrcMgr = PP.getSourceManager();
+  SourceLocation EndLoc = AsmLoc;
+  do {
+    bool InBraces = false;
+    unsigned short savedBraceCount = 0;
+    bool InAsmComment = false;
+    FileID FID;
+    unsigned LineNo = 0;
+    unsigned NumTokensRead = 0;
+    SourceLocation LBraceLoc;
+
+    if (Tok.is(tok::l_brace)) {
+      // Braced inline asm: consume the opening brace.
+      InBraces = true;
+      savedBraceCount = BraceCount;
+      EndLoc = LBraceLoc = ConsumeBrace();
+      ++NumTokensRead;
+    } else {
+      // Single-line inline asm; compute which line it is on.
+      std::pair<FileID, unsigned> ExpAsmLoc =
+          SrcMgr.getDecomposedExpansionLoc(EndLoc);
+      FID = ExpAsmLoc.first;
+      LineNo = SrcMgr.getLineNumber(FID, ExpAsmLoc.second);
+    }
+
     SourceLocation TokLoc = Tok.getLocation();
-    unsigned LineNo = SrcMgr.getInstantiationLineNumber(TokLoc);
     do {
+      // If we hit EOF, we're done, period.
+      if (Tok.is(tok::eof))
+        break;
+      // When we consume the closing brace, we're done.
+      if (InBraces && BraceCount == savedBraceCount)
+        break;
+
+      if (!InAsmComment && Tok.is(tok::semi)) {
+        // A semicolon in an asm is the start of a comment.
+        InAsmComment = true;
+        if (InBraces) {
+          // Compute which line the comment is on.
+          std::pair<FileID, unsigned> ExpSemiLoc =
+              SrcMgr.getDecomposedExpansionLoc(TokLoc);
+          FID = ExpSemiLoc.first;
+          LineNo = SrcMgr.getLineNumber(FID, ExpSemiLoc.second);
+        }
+      } else if (!InBraces || InAsmComment) {
+        // If end-of-line is significant, check whether this token is on a
+        // new line.
+        std::pair<FileID, unsigned> ExpLoc =
+            SrcMgr.getDecomposedExpansionLoc(TokLoc);
+        if (ExpLoc.first != FID ||
+            SrcMgr.getLineNumber(ExpLoc.first, ExpLoc.second) != LineNo) {
+          // If this is a single-line __asm, we're done.
+          if (!InBraces)
+            break;
+          // We're no longer in a comment.
+          InAsmComment = false;
+        } else if (!InAsmComment && Tok.is(tok::r_brace)) {
+          // Single-line asm always ends when a closing brace is seen.
+          // FIXME: This is compatible with Apple gcc's -fasm-blocks; what
+          // does MSVC do here?
+          break;
+        }
+      }
+
+      // Consume the next token; make sure we don't modify the brace count etc.
+      // if we are in a comment.
       EndLoc = TokLoc;
-      ConsumeAnyToken();
+      if (InAsmComment)
+        PP.Lex(Tok);
+      else
+        ConsumeAnyToken();
       TokLoc = Tok.getLocation();
-    } while ((SrcMgr.getInstantiationLineNumber(TokLoc) == LineNo) &&
-             Tok.isNot(tok::r_brace) && Tok.isNot(tok::semi) &&
-             Tok.isNot(tok::eof));
-  }
+      ++NumTokensRead;
+    } while (1);
+
+    if (InBraces && BraceCount != savedBraceCount) {
+      // __asm without closing brace (this can happen at EOF).
+      Diag(Tok, diag::err_expected_rbrace);
+      Diag(LBraceLoc, diag::note_matching) << "{";
+      return StmtError();
+    } else if (NumTokensRead == 0) {
+      // Empty __asm.
+      Diag(Tok, diag::err_expected_lbrace);
+      return StmtError();
+    }
+    // Multiple adjacent asm's form together into a single asm statement
+    // in the AST.
+    if (!Tok.is(tok::kw_asm))
+      break;
+    EndLoc = ConsumeToken();
+  } while (1);
+  // FIXME: Need to actually grab the data and pass it on to Sema.  Ideally,
+  // what Sema wants is a string of the entire inline asm, with one instruction
+  // per line and all the __asm keywords stripped out, and a way of mapping
+  // from any character of that string to its location in the original source
+  // code. I'm not entirely sure how to go about that, though.
   Token t;
   t.setKind(tok::string_literal);
   t.setLiteralData("\"/*FIXME: not done*/\"");
@@ -1611,20 +1716,24 @@ StmtResult Parser::FuzzyParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
 ///         asm-clobbers ',' asm-string-literal
 ///
 /// [MS]  ms-asm-statement:
-///         '__asm' assembly-instruction ';'[opt]
-///         '__asm' '{' assembly-instruction-list '}' ';'[opt]
+///         ms-asm-block
+///         ms-asm-block ms-asm-statement
 ///
-/// [MS]  assembly-instruction-list:
-///         assembly-instruction ';'[opt]
-///         assembly-instruction-list ';' assembly-instruction ';'[opt]
+/// [MS]  ms-asm-block:
+///         '__asm' ms-asm-line '\n'
+///         '__asm' '{' ms-asm-instruction-block[opt] '}' ';'[opt]
+///
+/// [MS]  ms-asm-instruction-block
+///         ms-asm-line
+///         ms-asm-line '\n' ms-asm-instruction-block
 ///
 StmtResult Parser::ParseAsmStatement(bool &msAsm) {
   assert(Tok.is(tok::kw_asm) && "Not an asm stmt");
   SourceLocation AsmLoc = ConsumeToken();
 
-  if (getLang().Microsoft && Tok.isNot(tok::l_paren) && !isTypeQualifier()) {
+  if (getLang().MicrosoftExt && Tok.isNot(tok::l_paren) && !isTypeQualifier()) {
     msAsm = true;
-    return FuzzyParseMicrosoftAsmStatement(AsmLoc);
+    return ParseMicrosoftAsmStatement(AsmLoc);
   }
   DeclSpec DS(AttrFactory);
   SourceLocation Loc = Tok.getLocation();
@@ -1643,25 +1752,26 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
     SkipUntil(tok::r_paren);
     return StmtError();
   }
-  Loc = ConsumeParen();
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
 
   ExprResult AsmString(ParseAsmStringLiteral());
   if (AsmString.isInvalid())
     return StmtError();
 
-  llvm::SmallVector<IdentifierInfo *, 4> Names;
+  SmallVector<IdentifierInfo *, 4> Names;
   ExprVector Constraints(Actions);
   ExprVector Exprs(Actions);
   ExprVector Clobbers(Actions);
 
   if (Tok.is(tok::r_paren)) {
     // We have a simple asm expression like 'asm("foo")'.
-    SourceLocation RParenLoc = ConsumeParen();
+    T.consumeClose();
     return Actions.ActOnAsmStmt(AsmLoc, /*isSimple*/ true, isVolatile,
-                                /*NumOutputs*/ 0, /*NumInputs*/ 0, 0, 
+                                /*NumOutputs*/ 0, /*NumInputs*/ 0, 0,
                                 move_arg(Constraints), move_arg(Exprs),
                                 AsmString.take(), move_arg(Clobbers),
-                                RParenLoc);
+                                T.getCloseLocation());
   }
 
   // Parse Outputs, if present.
@@ -1670,12 +1780,12 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
     // In C++ mode, parse "::" like ": :".
     AteExtraColon = Tok.is(tok::coloncolon);
     ConsumeToken();
-  
+
     if (!AteExtraColon &&
         ParseAsmOperandsOpt(Names, Constraints, Exprs))
       return StmtError();
   }
-  
+
   unsigned NumOutputs = Names.size();
 
   // Parse Inputs, if present.
@@ -1688,7 +1798,7 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
       AteExtraColon = Tok.is(tok::coloncolon);
       ConsumeToken();
     }
-    
+
     if (!AteExtraColon &&
         ParseAsmOperandsOpt(Names, Constraints, Exprs))
       return StmtError();
@@ -1721,12 +1831,12 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
     }
   }
 
-  SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, Loc);
+  T.consumeClose();
   return Actions.ActOnAsmStmt(AsmLoc, false, isVolatile,
                               NumOutputs, NumInputs, Names.data(),
                               move_arg(Constraints), move_arg(Exprs),
                               AsmString.take(), move_arg(Clobbers),
-                              RParenLoc);
+                              T.getCloseLocation());
 }
 
 /// ParseAsmOperands - Parse the asm-operands production as used by
@@ -1742,9 +1852,9 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
 ///
 //
 // FIXME: Avoid unnecessary std::string trashing.
-bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
-                                 llvm::SmallVectorImpl<ExprTy *> &Constraints,
-                                 llvm::SmallVectorImpl<ExprTy *> &Exprs) {
+bool Parser::ParseAsmOperandsOpt(SmallVectorImpl<IdentifierInfo *> &Names,
+                                 SmallVectorImpl<Expr *> &Constraints,
+                                 SmallVectorImpl<Expr *> &Exprs) {
   // 'asm-operands' isn't present?
   if (!isTokenStringLiteral() && Tok.isNot(tok::l_square))
     return false;
@@ -1752,7 +1862,8 @@ bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
   while (1) {
     // Read the [id] if present.
     if (Tok.is(tok::l_square)) {
-      SourceLocation Loc = ConsumeBracket();
+      BalancedDelimiterTracker T(*this, tok::l_square);
+      T.consumeOpen();
 
       if (Tok.isNot(tok::identifier)) {
         Diag(Tok, diag::err_expected_ident);
@@ -1764,7 +1875,7 @@ bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
       ConsumeToken();
 
       Names.push_back(II);
-      MatchRHSPunctuation(tok::r_square, Loc);
+      T.consumeClose();
     } else
       Names.push_back(0);
 
@@ -1782,9 +1893,10 @@ bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
     }
 
     // Read the parenthesized expression.
-    SourceLocation OpenLoc = ConsumeParen();
+    BalancedDelimiterTracker T(*this, tok::l_paren);
+    T.consumeOpen();
     ExprResult Res(ParseExpression());
-    MatchRHSPunctuation(tok::r_paren, OpenLoc);
+    T.consumeClose();
     if (Res.isInvalid()) {
       SkipUntil(tok::r_paren);
       return true;
@@ -1808,7 +1920,7 @@ Decl *Parser::ParseFunctionStatementBody(Decl *Decl, ParseScope &BodyScope) {
       return Actions.ActOnFinishFunctionBody(Decl, 0);
     }
   }
-  
+
   PrettyDeclStackTraceEntry CrashInfo(Actions, Decl, LBraceLoc,
                                       "parsing function body");
 
@@ -1841,6 +1953,8 @@ Decl *Parser::ParseFunctionTryBlock(Decl *Decl, ParseScope &BodyScope) {
   // Constructor initializer list?
   if (Tok.is(tok::colon))
     ParseConstructorInitializer(Decl);
+  else
+    Actions.ActOnDefaultCtorInitializers(Decl);
 
   if (PP.isCodeCompletionEnabled()) {
     if (trySkippingFunctionBodyForCodeCompletion()) {
@@ -1977,8 +2091,8 @@ StmtResult Parser::ParseCXXCatchBlock() {
 
   SourceLocation CatchLoc = ConsumeToken();
 
-  SourceLocation LParenLoc = Tok.getLocation();
-  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen))
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume(diag::err_expected_lparen))
     return StmtError();
 
   // C++ 3.3.2p3:
@@ -1999,7 +2113,8 @@ StmtResult Parser::ParseCXXCatchBlock() {
   } else
     ConsumeToken();
 
-  if (MatchRHSPunctuation(tok::r_paren, LParenLoc).isInvalid())
+  T.consumeClose();
+  if (T.getCloseLocation().isInvalid())
     return StmtError();
 
   if (Tok.isNot(tok::l_brace))
@@ -2018,7 +2133,7 @@ void Parser::ParseMicrosoftIfExistsStatement(StmtVector &Stmts) {
   bool Result;
   if (ParseMicrosoftIfExistsCondition(Result))
     return;
-  
+
   if (Tok.isNot(tok::l_brace)) {
     Diag(Tok, diag::err_expected_lbrace);
     return;
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseTemplate.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseTemplate.cpp
index 9eab40a3ecdf..3d68a4ab9db8 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseTemplate.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseTemplate.cpp
@@ -26,12 +26,16 @@ using namespace clang;
 Decl *
 Parser::ParseDeclarationStartingWithTemplate(unsigned Context,
                                              SourceLocation &DeclEnd,
-                                             AccessSpecifier AS) {
-  if (Tok.is(tok::kw_template) && NextToken().isNot(tok::less))
+                                             AccessSpecifier AS,
+                                             AttributeList *AccessAttrs) {
+  ObjCDeclContextSwitch ObjCDC(*this);
+  
+  if (Tok.is(tok::kw_template) && NextToken().isNot(tok::less)) {
     return ParseExplicitInstantiation(SourceLocation(), ConsumeToken(),
-                                      DeclEnd);
-
-  return ParseTemplateDeclarationOrSpecialization(Context, DeclEnd, AS);
+                                           DeclEnd);
+  }
+  return ParseTemplateDeclarationOrSpecialization(Context, DeclEnd, AS,
+                                                  AccessAttrs);
 }
 
 /// \brief RAII class that manages the template parameter depth.
@@ -75,7 +79,8 @@ namespace {
 Decl *
 Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
                                                  SourceLocation &DeclEnd,
-                                                 AccessSpecifier AS) {
+                                                 AccessSpecifier AS,
+                                                 AttributeList *AccessAttrs) {
   assert((Tok.is(tok::kw_export) || Tok.is(tok::kw_template)) &&
          "Token does not start a template declaration.");
 
@@ -129,7 +134,7 @@ Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
 
     // Parse the '<' template-parameter-list '>'
     SourceLocation LAngleLoc, RAngleLoc;
-    llvm::SmallVector<Decl*, 4> TemplateParams;
+    SmallVector<Decl*, 4> TemplateParams;
     if (ParseTemplateParameters(Depth, TemplateParams, LAngleLoc,
                                 RAngleLoc)) {
       // Skip until the semi-colon or a }.
@@ -159,7 +164,7 @@ Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
                                                              isSpecialization,
                                                          LastParamListWasEmpty),
                                              ParsingTemplateParams,
-                                             DeclEnd, AS);
+                                             DeclEnd, AS, AccessAttrs);
 }
 
 /// \brief Parse a single declaration that declares a template,
@@ -188,13 +193,15 @@ Parser::ParseSingleDeclarationAfterTemplate(
                                        const ParsedTemplateInfo &TemplateInfo,
                                        ParsingDeclRAIIObject &DiagsFromTParams,
                                        SourceLocation &DeclEnd,
-                                       AccessSpecifier AS) {
+                                       AccessSpecifier AS,
+                                       AttributeList *AccessAttrs) {
   assert(TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate &&
          "Template information required");
 
   if (Context == Declarator::MemberContext) {
     // We are parsing a member template.
-    ParseCXXClassMemberDeclaration(AS, TemplateInfo, &DiagsFromTParams);
+    ParseCXXClassMemberDeclaration(AS, AccessAttrs, TemplateInfo,
+                                   &DiagsFromTParams);
     return 0;
   }
 
@@ -289,7 +296,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
 ///
 /// \returns true if an error occurred, false otherwise.
 bool Parser::ParseTemplateParameters(unsigned Depth,
-                               llvm::SmallVectorImpl<Decl*> &TemplateParams,
+                               SmallVectorImpl<Decl*> &TemplateParams,
                                      SourceLocation &LAngleLoc,
                                      SourceLocation &RAngleLoc) {
   // Get the template parameter list.
@@ -322,7 +329,7 @@ bool Parser::ParseTemplateParameters(unsigned Depth,
 ///         template-parameter-list ',' template-parameter
 bool
 Parser::ParseTemplateParameterList(unsigned Depth,
-                             llvm::SmallVectorImpl<Decl*> &TemplateParams) {
+                             SmallVectorImpl<Decl*> &TemplateParams) {
   while (1) {
     if (Decl *TmpParam
           = ParseTemplateParameter(Depth, TemplateParams.size())) {
@@ -468,8 +475,10 @@ Decl *Parser::ParseTypeParameter(unsigned Depth, unsigned Position) {
     Ellipsis = true;
     EllipsisLoc = ConsumeToken();
 
-    if (!getLang().CPlusPlus0x)
-      Diag(EllipsisLoc, diag::ext_variadic_templates);
+    Diag(EllipsisLoc,
+         getLang().CPlusPlus0x
+           ? diag::warn_cxx98_compat_variadic_templates
+           : diag::ext_variadic_templates);
   }
 
   // Grab the template parameter name (if given)
@@ -516,7 +525,7 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
 
   // Handle the template <...> part.
   SourceLocation TemplateLoc = ConsumeToken();
-  llvm::SmallVector<Decl*,8> TemplateParams;
+  SmallVector<Decl*,8> TemplateParams;
   SourceLocation LAngleLoc, RAngleLoc;
   {
     ParseScope TemplateParmScope(this, Scope::TemplateParamScope);
@@ -540,8 +549,10 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   if (Tok.is(tok::ellipsis)) {
     EllipsisLoc = ConsumeToken();
     
-    if (!getLang().CPlusPlus0x)
-      Diag(EllipsisLoc, diag::ext_variadic_templates);
+    Diag(EllipsisLoc,
+         getLang().CPlusPlus0x
+           ? diag::warn_cxx98_compat_variadic_templates
+           : diag::ext_variadic_templates);
   }
       
   // Get the identifier, if given.
@@ -558,7 +569,7 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
     return 0;
   }
 
-  TemplateParamsTy *ParamList =
+  TemplateParameterList *ParamList =
     Actions.ActOnTemplateParameterList(Depth, SourceLocation(),
                                        TemplateLoc, LAngleLoc,
                                        TemplateParams.data(),
@@ -1157,6 +1168,7 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
     FD = cast<FunctionDecl>(LMT.D);
   
   // Reinject the template parameters.
+  SmallVector<ParseScope*, 4> TemplateParamScopeStack;
   DeclaratorDecl* Declarator = dyn_cast<DeclaratorDecl>(FD);
   if (Declarator && Declarator->getNumTemplateParameterLists() != 0) {
     Actions.ActOnReenterDeclaratorTemplateScope(getCurScope(), Declarator);
@@ -1164,17 +1176,31 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
   } else {
     Actions.ActOnReenterTemplateScope(getCurScope(), LMT.D);
 
+    // Get the list of DeclContext to reenter.
+    SmallVector<DeclContext*, 4> DeclContextToReenter;
     DeclContext *DD = FD->getLexicalParent();
     while (DD && DD->isRecord()) {
-      if (ClassTemplatePartialSpecializationDecl* MD =
-                  dyn_cast_or_null<ClassTemplatePartialSpecializationDecl>(DD))
-          Actions.ActOnReenterTemplateScope(getCurScope(), MD);
-      else if (CXXRecordDecl* MD = dyn_cast_or_null<CXXRecordDecl>(DD))
-          Actions.ActOnReenterTemplateScope(getCurScope(),
-                                            MD->getDescribedClassTemplate());
-
+      DeclContextToReenter.push_back(DD);
       DD = DD->getLexicalParent();
     }
+
+    // Reenter template scopes from outmost to innermost.
+    SmallVector<DeclContext*, 4>::reverse_iterator II =
+    DeclContextToReenter.rbegin();
+    for (; II != DeclContextToReenter.rend(); ++II) {
+      if (ClassTemplatePartialSpecializationDecl* MD =
+                dyn_cast_or_null<ClassTemplatePartialSpecializationDecl>(*II)) {
+       TemplateParamScopeStack.push_back(new ParseScope(this,
+                                                   Scope::TemplateParamScope));
+        Actions.ActOnReenterTemplateScope(getCurScope(), MD);
+      } else if (CXXRecordDecl* MD = dyn_cast_or_null<CXXRecordDecl>(*II)) {
+        TemplateParamScopeStack.push_back(new ParseScope(this,
+                                                    Scope::TemplateParamScope,
+                                       MD->getDescribedClassTemplate() != 0 ));
+        Actions.ActOnReenterTemplateScope(getCurScope(),
+                                          MD->getDescribedClassTemplate());
+      }
+    }
   }
   assert(!LMT.Toks.empty() && "Empty body!");
 
@@ -1205,21 +1231,25 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
 
   if (Tok.is(tok::kw_try)) {
     ParseFunctionTryBlock(LMT.D, FnScope);
-    return;
-  }
-  if (Tok.is(tok::colon)) {
-    ParseConstructorInitializer(LMT.D);
+  } else {
+    if (Tok.is(tok::colon))
+      ParseConstructorInitializer(LMT.D);
+    else
+      Actions.ActOnDefaultCtorInitializers(LMT.D);
 
-    // Error recovery.
-    if (!Tok.is(tok::l_brace)) {
+    if (Tok.is(tok::l_brace)) {
+      ParseFunctionStatementBody(LMT.D, FnScope);
+      Actions.MarkAsLateParsedTemplate(FD, false);
+    } else
       Actions.ActOnFinishFunctionBody(LMT.D, 0);
-      return;
-    }
-  } else
-    Actions.ActOnDefaultCtorInitializers(LMT.D);
+  }
 
-  ParseFunctionStatementBody(LMT.D, FnScope);
-  Actions.MarkAsLateParsedTemplate(FD, false);
+  // Exit scopes.
+  FnScope.Exit();
+  SmallVector<ParseScope*, 4>::reverse_iterator I =
+   TemplateParamScopeStack.rbegin();
+  for (; I != TemplateParamScopeStack.rend(); ++I)
+    delete *I;
 
   DeclGroupPtrTy grp = Actions.ConvertDeclToDeclGroup(LMT.D);
   if (grp)
@@ -1229,15 +1259,10 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
 /// \brief Lex a delayed template function for late parsing.
 void Parser::LexTemplateFunctionForLateParsing(CachedTokens &Toks) {
   tok::TokenKind kind = Tok.getKind();
-  // We may have a constructor initializer or function-try-block here.
-  if (kind == tok::colon || kind == tok::kw_try)
-    ConsumeAndStoreUntil(tok::l_brace, Toks);
-  else {
-    Toks.push_back(Tok);
-    ConsumeBrace();
+  if (!ConsumeAndStoreFunctionPrologue(Toks)) {
+    // Consume everything up to (and including) the matching right brace.
+    ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
   }
-  // Consume everything up to (and including) the matching right brace.
-  ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
 
   // If we're in a function-try-block, we need to store all the catch blocks.
   if (kind == tok::kw_try) {
diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseTentative.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseTentative.cpp
index 2ba0fc673f67..d53839f3cb01 100644
--- a/contrib/llvm/tools/clang/lib/Parse/ParseTentative.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/ParseTentative.cpp
@@ -377,6 +377,7 @@ bool Parser::isCXXTypeId(TentativeCXXTypeIdContext Context, bool &isAmbiguous) {
 ///
 /// [C++0x] attribute-specifier:
 ///         '[' '[' attribute-list ']' ']'
+///         alignment-specifier
 ///
 /// [C++0x] attribute-list:
 ///         attribute[opt]
@@ -409,6 +410,9 @@ bool Parser::isCXXTypeId(TentativeCXXTypeIdContext Context, bool &isAmbiguous) {
 ///         any token but '(', ')', '[', ']', '{', or '}'
 bool Parser::isCXX0XAttributeSpecifier (bool CheckClosing,
                                         tok::TokenKind *After) {
+  if (Tok.is(tok::kw_alignas))
+    return true;
+
   if (Tok.isNot(tok::l_square) || NextToken().isNot(tok::l_square))
     return false;
   
@@ -552,7 +556,8 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
           Tok.is(tok::kw___cdecl) ||
           Tok.is(tok::kw___stdcall) ||
           Tok.is(tok::kw___fastcall) ||
-          Tok.is(tok::kw___thiscall))
+          Tok.is(tok::kw___thiscall) ||
+          Tok.is(tok::kw___unaligned))
         return TPResult::True(); // attributes indicate declaration
       TPResult TPR = TryParseDeclarator(mayBeAbstract, mayHaveIdentifier);
       if (TPR != TPResult::Ambiguous())
@@ -605,8 +610,14 @@ Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
   // Obviously starts an expression.
   case tok::numeric_constant:
   case tok::char_constant:
+  case tok::wide_char_constant:
+  case tok::utf16_char_constant:
+  case tok::utf32_char_constant:
   case tok::string_literal:
   case tok::wide_string_literal:
+  case tok::utf8_string_literal:
+  case tok::utf16_string_literal:
+  case tok::utf32_string_literal:
   case tok::l_square:
   case tok::l_paren:
   case tok::amp:
@@ -674,6 +685,7 @@ Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
   case tok::kw_const:
   case tok::kw_double:
   case tok::kw_enum:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_int:
   case tok::kw_long:
@@ -705,8 +717,10 @@ Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
   case tok::kw___stdcall:
   case tok::kw___fastcall:
   case tok::kw___thiscall:
+  case tok::kw___unaligned:
   case tok::kw___vector:
   case tok::kw___pixel:
+  case tok::kw__Atomic:
     return TPResult::False();
 
   default:
@@ -863,6 +877,9 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
   case tok::kw_virtual:
   case tok::kw_explicit:
 
+    // Modules
+  case tok::kw___module_private__:
+      
     // type-specifier:
     //   simple-type-specifier
     //   class-specifier
@@ -896,7 +913,9 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
   case tok::kw___thiscall:
   case tok::kw___w64:
   case tok::kw___ptr64:
+  case tok::kw___ptr32:
   case tok::kw___forceinline:
+  case tok::kw___unaligned:
     return TPResult::True();
 
     // Borland
@@ -976,6 +995,7 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
   case tok::kw___int64:
   case tok::kw_signed:
   case tok::kw_unsigned:
+  case tok::kw_half:
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_void:
@@ -1016,6 +1036,10 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
   case tok::kw___underlying_type:
     return TPResult::True();
 
+  // C1x _Atomic
+  case tok::kw__Atomic:
+    return TPResult::True();
+
   default:
     return TPResult::False();
   }
diff --git a/contrib/llvm/tools/clang/lib/Parse/Parser.cpp b/contrib/llvm/tools/clang/lib/Parse/Parser.cpp
index 5c502907bc15..c90964381f8d 100644
--- a/contrib/llvm/tools/clang/lib/Parse/Parser.cpp
+++ b/contrib/llvm/tools/clang/lib/Parse/Parser.cpp
@@ -72,7 +72,7 @@ Parser::Parser(Preprocessor &pp, Sema &actions)
 
 /// If a crash happens while the parser is active, print out a line indicating
 /// what the current token is.
-void PrettyStackTraceParserEntry::print(llvm::raw_ostream &OS) const {
+void PrettyStackTraceParserEntry::print(raw_ostream &OS) const {
   const Token &Tok = P.getCurToken();
   if (Tok.is(tok::eof)) {
     OS << "<eof> parser at end of file\n";
@@ -122,34 +122,6 @@ void Parser::SuggestParentheses(SourceLocation Loc, unsigned DK,
     << FixItHint::CreateInsertion(EndLoc, ")");
 }
 
-/// MatchRHSPunctuation - For punctuation with a LHS and RHS (e.g. '['/']'),
-/// this helper function matches and consumes the specified RHS token if
-/// present.  If not present, it emits a corresponding diagnostic indicating
-/// that the parser failed to match the RHS of the token at LHSLoc.
-SourceLocation Parser::MatchRHSPunctuation(tok::TokenKind RHSTok,
-                                           SourceLocation LHSLoc) {
-
-  if (Tok.is(RHSTok))
-    return ConsumeAnyToken();
-
-  SourceLocation R = Tok.getLocation();
-  const char *LHSName = "unknown";
-  diag::kind DID = diag::err_parse_error;
-  switch (RHSTok) {
-  default: break;
-  case tok::r_paren : LHSName = "("; DID = diag::err_expected_rparen; break;
-  case tok::r_brace : LHSName = "{"; DID = diag::err_expected_rbrace; break;
-  case tok::r_square: LHSName = "["; DID = diag::err_expected_rsquare; break;
-  case tok::greater:  LHSName = "<"; DID = diag::err_expected_greater; break;
-  case tok::greatergreatergreater:
-                      LHSName = "<<<"; DID = diag::err_expected_ggg; break;
-  }
-  Diag(Tok, DID);
-  Diag(LHSLoc, diag::note_matching) << LHSName;
-  SkipUntil(RHSTok);
-  return R;
-}
-
 static bool IsCommonTypo(tok::TokenKind ExpectedTok, const Token &Tok) {
   switch (ExpectedTok) {
   case tok::semi: return Tok.is(tok::colon); // : for ;
@@ -298,6 +270,9 @@ bool Parser::SkipUntil(const tok::TokenKind *Toks, unsigned NumToks,
 
     case tok::string_literal:
     case tok::wide_string_literal:
+    case tok::utf8_string_literal:
+    case tok::utf16_string_literal:
+    case tok::utf32_string_literal:
       ConsumeStringToken();
       break;
         
@@ -440,6 +415,7 @@ void Parser::Initialize() {
     ObjCTypeQuals[objc_byref] = &PP.getIdentifierTable().get("byref");
   }
 
+  Ident_instancetype = 0;
   Ident_final = 0;
   Ident_override = 0;
 
@@ -550,7 +526,12 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
                                  ParsingDeclSpec *DS) {
   DelayedCleanupPoint CleanupRAII(TopLevelDeclCleanupPool);
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
-  
+
+  if (PP.isCodeCompletionReached()) {
+    cutOffParsing();
+    return DeclGroupPtrTy();
+  }
+
   Decl *SingleDecl = 0;
   switch (Tok.getKind()) {
   case tok::semi:
@@ -590,10 +571,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
     break;
   }
   case tok::at:
-    // @ is not a legal token unless objc is enabled, no need to check for ObjC.
-    /// FIXME: ParseObjCAtDirectives should return a DeclGroup for things like
-    /// @class foo, bar;
-    SingleDecl = ParseObjCAtDirectives();
+    return ParseObjCAtDirectives();
     break;
   case tok::minus:
   case tok::plus:
@@ -608,8 +586,8 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                    ObjCImpDecl? Sema::PCC_ObjCImplementation
                                               : Sema::PCC_Namespace);
-    ConsumeCodeCompletionToken();
-    return ParseExternalDeclaration(attrs);
+    cutOffParsing();
+    return DeclGroupPtrTy();
   case tok::kw_using:
   case tok::kw_namespace:
   case tok::kw_typedef:
@@ -676,6 +654,9 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
     ParseMicrosoftIfExistsExternalDeclaration();
     return DeclGroupPtrTy();
 
+  case tok::kw___import_module__:
+    return ParseModuleImport();
+      
   default:
   dont_know:
     // We can't tell whether this is a function-definition or declaration yet.
@@ -808,6 +789,11 @@ Parser::ParseDeclarationOrFunctionDefinition(ParsedAttributes &attrs,
                                              AccessSpecifier AS) {
   ParsingDeclSpec DS(*this);
   DS.takeAttributesFrom(attrs);
+  // Must temporarily exit the objective-c container scope for
+  // parsing c constructs and re-enter objc container scope
+  // afterwards.
+  ObjCDeclContextSwitch ObjCDC(*this);
+    
   return ParseDeclarationOrFunctionDefinition(DS, AS);
 }
 
@@ -877,9 +863,9 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
     ParseScope BodyScope(this, Scope::FnScope|Scope::DeclScope);
     Scope *ParentScope = getCurScope()->getParent();
 
+    D.setFunctionDefinition(true);
     Decl *DP = Actions.HandleDeclarator(ParentScope, D,
-                                move(TemplateParameterLists),
-                                /*IsFunctionDefinition=*/true);
+                                        move(TemplateParameterLists));
     D.complete(DP);
     D.getMutableDeclSpec().abort();
 
@@ -1133,13 +1119,12 @@ Parser::ExprResult Parser::ParseSimpleAsm(SourceLocation *EndLoc) {
     ConsumeToken();
   }
 
-  if (Tok.isNot(tok::l_paren)) {
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.consumeOpen()) {
     Diag(Tok, diag::err_expected_lparen_after) << "asm";
     return ExprError();
   }
 
-  Loc = ConsumeParen();
-
   ExprResult Result(ParseAsmStringLiteral());
 
   if (Result.isInvalid()) {
@@ -1148,9 +1133,10 @@ Parser::ExprResult Parser::ParseSimpleAsm(SourceLocation *EndLoc) {
       *EndLoc = Tok.getLocation();
     ConsumeAnyToken();
   } else {
-    Loc = MatchRHSPunctuation(tok::r_paren, Loc);
+    // Close the paren and get the location of the end bracket
+    T.consumeClose();
     if (EndLoc)
-      *EndLoc = Loc;
+      *EndLoc = T.getCloseLocation();
   }
 
   return move(Result);
@@ -1190,7 +1176,7 @@ TemplateIdAnnotation *Parser::takeTemplateIdAnnotation(const Token &tok) {
 ///
 /// Note that this routine emits an error if you call it with ::new or ::delete
 /// as the current tokens, so only call it in contexts where these are invalid.
-bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
+bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext, bool NeedType) {
   assert((Tok.is(tok::identifier) || Tok.is(tok::coloncolon)
           || Tok.is(tok::kw_typename) || Tok.is(tok::annot_cxxscope)) &&
          "Cannot be a type or scope token!");
@@ -1208,7 +1194,7 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
                                        0, /*IsTypename*/true))
       return true;
     if (!SS.isSet()) {
-      if (getLang().Microsoft)
+      if (getLang().MicrosoftExt)
         Diag(Tok.getLocation(), diag::warn_expected_qualified_after_typename);
       else
         Diag(Tok.getLocation(), diag::err_expected_qualified_after_typename);
@@ -1264,13 +1250,18 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
       return true;
 
   if (Tok.is(tok::identifier)) {
+    IdentifierInfo *CorrectedII = 0;
     // Determine whether the identifier is a type name.
     if (ParsedType Ty = Actions.getTypeName(*Tok.getIdentifierInfo(),
                                             Tok.getLocation(), getCurScope(),
                                             &SS, false, 
                                             NextToken().is(tok::period),
                                             ParsedType(),
-                                            /*NonTrivialTypeSourceInfo*/true)) {
+                                            /*NonTrivialTypeSourceInfo*/true,
+                                            NeedType ? &CorrectedII : NULL)) {
+      // A FixIt was applied as a result of typo correction
+      if (CorrectedII)
+        Tok.setIdentifierInfo(CorrectedII);
       // This is a typename. Replace the current token in-place with an
       // annotation type token.
       Tok.setKind(tok::annot_typename);
@@ -1405,20 +1396,27 @@ bool Parser::isTokenEqualOrMistypedEqualEqual(unsigned DiagID) {
   return Tok.is(tok::equal);
 }
 
-void Parser::CodeCompletionRecovery() {
+SourceLocation Parser::handleUnexpectedCodeCompletionToken() {
+  assert(Tok.is(tok::code_completion));
+  PrevTokLocation = Tok.getLocation();
+
   for (Scope *S = getCurScope(); S; S = S->getParent()) {
     if (S->getFlags() & Scope::FnScope) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_RecoveryInFunction);
-      return;
+      cutOffParsing();
+      return PrevTokLocation;
     }
     
     if (S->getFlags() & Scope::ClassScope) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class);
-      return;
+      cutOffParsing();
+      return PrevTokLocation;
     }
   }
   
   Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace);
+  cutOffParsing();
+  return PrevTokLocation;
 }
 
 // Anchor the Parser::FieldCallback vtable to this translation unit.
@@ -1463,13 +1461,12 @@ bool Parser::ParseMicrosoftIfExistsCondition(bool& Result) {
   Token Condition = Tok;
   SourceLocation IfExistsLoc = ConsumeToken();
 
-  SourceLocation LParenLoc = Tok.getLocation();
-  if (Tok.isNot(tok::l_paren)) {
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.consumeOpen()) {
     Diag(Tok, diag::err_expected_lparen_after) << IfExistsLoc;
     SkipUntil(tok::semi);
     return true;
   }
-  ConsumeParen(); // eat the '('.
   
   // Parse nested-name-specifier.
   CXXScopeSpec SS;
@@ -1488,7 +1485,8 @@ bool Parser::ParseMicrosoftIfExistsCondition(bool& Result) {
     return true;
   }
 
-  if (MatchRHSPunctuation(tok::r_paren, LParenLoc).isInvalid())
+  T.consumeClose();
+  if (T.getCloseLocation().isInvalid())
     return true;
 
   // Check if the symbol exists.
@@ -1533,3 +1531,86 @@ void Parser::ParseMicrosoftIfExistsExternalDeclaration() {
   }
   ConsumeBrace();
 }
+
+Parser::DeclGroupPtrTy Parser::ParseModuleImport() {
+  assert(Tok.is(tok::kw___import_module__) && 
+         "Improper start to module import");
+  SourceLocation ImportLoc = ConsumeToken();
+  
+  // Parse the module name.
+  if (!Tok.is(tok::identifier)) {
+    Diag(Tok, diag::err_module_expected_ident);
+    SkipUntil(tok::semi);
+    return DeclGroupPtrTy();
+  }
+  
+  IdentifierInfo &ModuleName = *Tok.getIdentifierInfo();
+  SourceLocation ModuleNameLoc = ConsumeToken();
+  DeclResult Import = Actions.ActOnModuleImport(ImportLoc, ModuleName, ModuleNameLoc);
+  ExpectAndConsumeSemi(diag::err_module_expected_semi);
+  if (Import.isInvalid())
+    return DeclGroupPtrTy();
+  
+  return Actions.ConvertDeclToDeclGroup(Import.get());
+}
+
+bool Parser::BalancedDelimiterTracker::consumeOpen() {
+  // Try to consume the token we are holding
+  if (P.Tok.is(Kind)) {
+    P.QuantityTracker.push(Kind);
+    Cleanup = true;
+    if (P.QuantityTracker.getDepth(Kind) < MaxDepth) {
+      LOpen = P.ConsumeAnyToken();
+      return false;
+    } else {
+      P.Diag(P.Tok, diag::err_parser_impl_limit_overflow);
+      P.SkipUntil(tok::eof);
+    }
+  }
+  return true;
+}
+
+bool Parser::BalancedDelimiterTracker::expectAndConsume(unsigned DiagID, 
+                                            const char *Msg,
+                                            tok::TokenKind SkipToToc ) {
+  LOpen = P.Tok.getLocation();
+  if (!P.ExpectAndConsume(Kind, DiagID, Msg, SkipToToc)) {
+    P.QuantityTracker.push(Kind);
+    Cleanup = true;
+    if (P.QuantityTracker.getDepth(Kind) < MaxDepth) {
+      return false;
+    } else {
+      P.Diag(P.Tok, diag::err_parser_impl_limit_overflow);
+      P.SkipUntil(tok::eof);
+    }
+  }
+  return true;
+}
+
+bool Parser::BalancedDelimiterTracker::consumeClose() {
+  if (P.Tok.is(Close)) {
+    LClose = P.ConsumeAnyToken();
+    if (Cleanup)
+      P.QuantityTracker.pop(Kind);
+
+    Cleanup = false;
+    return false;
+  } else {
+    const char *LHSName = "unknown";
+    diag::kind DID = diag::err_parse_error;
+    switch (Close) {
+    default: break;
+    case tok::r_paren : LHSName = "("; DID = diag::err_expected_rparen; break;
+    case tok::r_brace : LHSName = "{"; DID = diag::err_expected_rbrace; break;
+    case tok::r_square: LHSName = "["; DID = diag::err_expected_rsquare; break;
+    case tok::greater:  LHSName = "<"; DID = diag::err_expected_greater; break;
+    case tok::greatergreatergreater:
+                        LHSName = "<<<"; DID = diag::err_expected_ggg; break;
+    }
+    P.Diag(P.Tok, DID);
+    P.Diag(LOpen, diag::note_matching) << LHSName;
+    if (P.SkipUntil(Close))
+      LClose = P.Tok.getLocation();
+  }
+  return true;
+}
diff --git a/contrib/llvm/tools/clang/lib/Parse/RAIIObjectsForParser.h b/contrib/llvm/tools/clang/lib/Parse/RAIIObjectsForParser.h
index 3765f92348ea..ef17aee3f512 100644
--- a/contrib/llvm/tools/clang/lib/Parse/RAIIObjectsForParser.h
+++ b/contrib/llvm/tools/clang/lib/Parse/RAIIObjectsForParser.h
@@ -30,9 +30,9 @@ namespace clang {
   class ExtensionRAIIObject {
     void operator=(const ExtensionRAIIObject &);     // DO NOT IMPLEMENT
     ExtensionRAIIObject(const ExtensionRAIIObject&); // DO NOT IMPLEMENT
-    Diagnostic &Diags;
+    DiagnosticsEngine &Diags;
   public:
-    ExtensionRAIIObject(Diagnostic &diags) : Diags(diags) {
+    ExtensionRAIIObject(DiagnosticsEngine &diags) : Diags(diags) {
       Diags.IncrementAllExtensionsSilenced();
     }
 
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/DeltaTree.cpp b/contrib/llvm/tools/clang/lib/Rewrite/DeltaTree.cpp
index 085dfd89ef15..4297dc8de62f 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/DeltaTree.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/DeltaTree.cpp
@@ -12,12 +12,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Rewrite/DeltaTree.h"
-#include "llvm/Support/Casting.h"
+#include "clang/Basic/LLVM.h"
 #include <cstring>
 #include <cstdio>
 using namespace clang;
-using llvm::cast;
-using llvm::dyn_cast;
 
 /// The DeltaTree class is a multiway search tree (BTree) structure with some
 /// fancy features.  B-Trees are generally more memory and cache efficient
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/FixItRewriter.cpp b/contrib/llvm/tools/clang/lib/Rewrite/FixItRewriter.cpp
index e50793e2775f..632c0de0742a 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/FixItRewriter.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/FixItRewriter.cpp
@@ -25,7 +25,7 @@
 
 using namespace clang;
 
-FixItRewriter::FixItRewriter(Diagnostic &Diags, SourceManager &SourceMgr,
+FixItRewriter::FixItRewriter(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
                              const LangOptions &LangOpts,
                              FixItOptions *FixItOpts)
   : Diags(Diags),
@@ -41,7 +41,7 @@ FixItRewriter::~FixItRewriter() {
   Diags.setClient(Client);
 }
 
-bool FixItRewriter::WriteFixedFile(FileID ID, llvm::raw_ostream &OS) {
+bool FixItRewriter::WriteFixedFile(FileID ID, raw_ostream &OS) {
   const RewriteBuffer *RewriteBuf = Rewrite.getRewriteBufferFor(ID);
   if (!RewriteBuf) return true;
   RewriteBuf->write(OS);
@@ -78,15 +78,15 @@ bool FixItRewriter::IncludeInDiagnosticCounts() const {
   return Client ? Client->IncludeInDiagnosticCounts() : true;
 }
 
-void FixItRewriter::HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                     const DiagnosticInfo &Info) {
+void FixItRewriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                     const Diagnostic &Info) {
   // Default implementation (Warnings/errors count).
-  DiagnosticClient::HandleDiagnostic(DiagLevel, Info);
+  DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
 
   Client->HandleDiagnostic(DiagLevel, Info);
 
   // Skip over any diagnostics that are ignored or notes.
-  if (DiagLevel <= Diagnostic::Note)
+  if (DiagLevel <= DiagnosticsEngine::Note)
     return;
 
   // Make sure that we can perform all of the modifications we
@@ -107,7 +107,8 @@ void FixItRewriter::HandleDiagnostic(Diagnostic::Level DiagLevel,
       Diag(Info.getLocation(), diag::note_fixit_in_macro);
 
     // If this was an error, refuse to perform any rewriting.
-    if (DiagLevel == Diagnostic::Error || DiagLevel == Diagnostic::Fatal) {
+    if (DiagLevel == DiagnosticsEngine::Error ||
+          DiagLevel == DiagnosticsEngine::Fatal) {
       if (++NumFailures == 1)
         Diag(Info.getLocation(), diag::note_fixit_unfixed_error);
     }
@@ -155,4 +156,9 @@ void FixItRewriter::Diag(SourceLocation Loc, unsigned DiagID) {
   Diags.setClient(this);
 }
 
+DiagnosticConsumer *FixItRewriter::clone(DiagnosticsEngine &Diags) const {
+  return new FixItRewriter(Diags, Diags.getSourceManager(), 
+                           Rewrite.getLangOpts(), FixItOpts);
+}
+
 FixItOptions::~FixItOptions() {}
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/FrontendActions.cpp b/contrib/llvm/tools/clang/lib/Rewrite/FrontendActions.cpp
index 33e79edaf93b..f00e7fd9c913 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/FrontendActions.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/FrontendActions.cpp
@@ -28,8 +28,8 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 ASTConsumer *HTMLPrintAction::CreateASTConsumer(CompilerInstance &CI,
-                                                llvm::StringRef InFile) {
-  if (llvm::raw_ostream *OS = CI.createDefaultOutputFile(false, InFile))
+                                                StringRef InFile) {
+  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile))
     return CreateHTMLPrinter(OS, CI.getPreprocessor());
   return 0;
 }
@@ -38,7 +38,7 @@ FixItAction::FixItAction() {}
 FixItAction::~FixItAction() {}
 
 ASTConsumer *FixItAction::CreateASTConsumer(CompilerInstance &CI,
-                                            llvm::StringRef InFile) {
+                                            StringRef InFile) {
   return new ASTConsumer();
 }
 
@@ -67,7 +67,7 @@ class FixItActionSuffixInserter : public FixItOptions {
 } // end anonymous namespace
 
 bool FixItAction::BeginSourceFileAction(CompilerInstance &CI,
-                                        llvm::StringRef Filename) {
+                                        StringRef Filename) {
   const FrontendOptions &FEOpts = getCompilerInstance().getFrontendOpts();
   if (!FEOpts.FixItSuffix.empty()) {
     FixItOpts.reset(new FixItActionSuffixInserter(FEOpts.FixItSuffix,
@@ -91,8 +91,8 @@ void FixItAction::EndSourceFileAction() {
 //===----------------------------------------------------------------------===//
 
 ASTConsumer *RewriteObjCAction::CreateASTConsumer(CompilerInstance &CI,
-                                                  llvm::StringRef InFile) {
-  if (llvm::raw_ostream *OS = CI.createDefaultOutputFile(false, InFile, "cpp"))
+                                                  StringRef InFile) {
+  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile, "cpp"))
     return CreateObjCRewriter(InFile, OS,
                               CI.getDiagnostics(), CI.getLangOpts(),
                               CI.getDiagnosticOpts().NoRewriteMacros);
@@ -101,7 +101,7 @@ ASTConsumer *RewriteObjCAction::CreateASTConsumer(CompilerInstance &CI,
 
 void RewriteMacrosAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
-  llvm::raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
+  raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
   if (!OS) return;
 
   RewriteMacrosInInput(CI.getPreprocessor(), OS);
@@ -109,7 +109,7 @@ void RewriteMacrosAction::ExecuteAction() {
 
 void RewriteTestAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
-  llvm::raw_ostream *OS = CI.createDefaultOutputFile(false, getCurrentFile());
+  raw_ostream *OS = CI.createDefaultOutputFile(false, getCurrentFile());
   if (!OS) return;
 
   DoRewriteTest(CI.getPreprocessor(), OS);
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/HTMLPrint.cpp b/contrib/llvm/tools/clang/lib/Rewrite/HTMLPrint.cpp
index f66bfcb2dfc4..6a89265ef2c2 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/HTMLPrint.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/HTMLPrint.cpp
@@ -32,12 +32,12 @@ using namespace clang;
 namespace {
   class HTMLPrinter : public ASTConsumer {
     Rewriter R;
-    llvm::raw_ostream *Out;
+    raw_ostream *Out;
     Preprocessor &PP;
     bool SyntaxHighlight, HighlightMacros;
 
   public:
-    HTMLPrinter(llvm::raw_ostream *OS, Preprocessor &pp,
+    HTMLPrinter(raw_ostream *OS, Preprocessor &pp,
                 bool _SyntaxHighlight, bool _HighlightMacros)
       : Out(OS), PP(pp), SyntaxHighlight(_SyntaxHighlight),
         HighlightMacros(_HighlightMacros) {}
@@ -47,7 +47,7 @@ namespace {
   };
 }
 
-ASTConsumer* clang::CreateHTMLPrinter(llvm::raw_ostream *OS,
+ASTConsumer* clang::CreateHTMLPrinter(raw_ostream *OS,
                                       Preprocessor &PP,
                                       bool SyntaxHighlight,
                                       bool HighlightMacros) {
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/HTMLRewrite.cpp b/contrib/llvm/tools/clang/lib/Rewrite/HTMLRewrite.cpp
index 0b54755dc58c..ba39602d162b 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/HTMLRewrite.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/HTMLRewrite.cpp
@@ -33,8 +33,8 @@ using namespace clang;
 void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
                           const char *StartTag, const char *EndTag) {
   SourceManager &SM = R.getSourceMgr();
-  B = SM.getInstantiationLoc(B);
-  E = SM.getInstantiationLoc(E);
+  B = SM.getExpansionLoc(B);
+  E = SM.getExpansionLoc(E);
   FileID FID = SM.getFileID(B);
   assert(SM.getFileID(E) == FID && "B/E not in the same file!");
 
@@ -140,10 +140,10 @@ void html::EscapeText(Rewriter &R, FileID FID,
       unsigned NumSpaces = 8-(ColNo&7);
       if (EscapeSpaces)
         RB.ReplaceText(FilePos, 1,
-                       llvm::StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
+                       StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
                                        "&nbsp;&nbsp;&nbsp;", 6*NumSpaces));
       else
-        RB.ReplaceText(FilePos, 1, llvm::StringRef("        ", NumSpaces));
+        RB.ReplaceText(FilePos, 1, StringRef("        ", NumSpaces));
       ColNo += NumSpaces;
       break;
     }
@@ -277,7 +277,7 @@ void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID,
   const char* FileEnd = Buf->getBufferEnd();
 
   SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID);
-  SourceLocation EndLoc = StartLoc.getFileLocWithOffset(FileEnd-FileStart);
+  SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart);
 
   std::string s;
   llvm::raw_string_ostream os(s);
@@ -397,8 +397,15 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
       HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
                      "<span class='comment'>", "</span>");
       break;
+    case tok::utf8_string_literal:
+      // Chop off the u part of u8 prefix
+      ++TokOffs;
+      --TokLen;
+      // FALL THROUGH to chop the 8
     case tok::wide_string_literal:
-      // Chop off the L prefix
+    case tok::utf16_string_literal:
+    case tok::utf32_string_literal:
+      // Chop off the L, u, U or 8 prefix
       ++TokOffs;
       --TokLen;
       // FALL THROUGH.
@@ -433,17 +440,6 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
   }
 }
 
-namespace {
-/// IgnoringDiagClient - This is a diagnostic client that just ignores all
-/// diags.
-class IgnoringDiagClient : public DiagnosticClient {
-  void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                        const DiagnosticInfo &Info) {
-    // Just ignore it.
-  }
-};
-}
-
 /// HighlightMacros - This uses the macro table state from the end of the
 /// file, to re-expand macros and insert (into the HTML) information about the
 /// macro expansions.  This won't be perfectly perfect, but it will be
@@ -486,14 +482,14 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
 
   // Temporarily change the diagnostics object so that we ignore any generated
   // diagnostics from this pass.
-  Diagnostic TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
-                      new IgnoringDiagClient);
+  DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
+                      new IgnoringDiagConsumer);
 
   // FIXME: This is a huge hack; we reuse the input preprocessor because we want
   // its state, but we aren't actually changing it (we hope). This should really
   // construct a copy of the preprocessor.
   Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
-  Diagnostic *OldDiags = &TmpPP.getDiagnostics();
+  DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
   TmpPP.setDiagnostics(TmpDiags);
 
   // Inform the preprocessor that we don't want comments.
@@ -519,7 +515,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // expansion by inserting a start tag before the macro expansion and
     // end tag after it.
     std::pair<SourceLocation, SourceLocation> LLoc =
-      SM.getInstantiationRange(Tok.getLocation());
+      SM.getExpansionRange(Tok.getLocation());
 
     // Ignore tokens whose instantiation location was not the main file.
     if (SM.getFileID(LLoc.first) != FID) {
@@ -542,7 +538,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // instantiation.  It would be really nice to pop up a window with all the
     // spelling of the tokens or something.
     while (!Tok.is(tok::eof) &&
-           SM.getInstantiationLoc(Tok.getLocation()) == LLoc.first) {
+           SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
       // Insert a newline if the macro expansion is getting large.
       if (LineLen > 60) {
         Expansion += "<br>";
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/RewriteMacros.cpp b/contrib/llvm/tools/clang/lib/Rewrite/RewriteMacros.cpp
index 0453098a56d2..d56910037c6c 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/RewriteMacros.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/RewriteMacros.cpp
@@ -87,7 +87,7 @@ static void LexRawTokensFromMainFile(Preprocessor &PP,
 
 
 /// RewriteMacrosInInput - Implement -rewrite-macros mode.
-void clang::RewriteMacrosInInput(Preprocessor &PP, llvm::raw_ostream *OS) {
+void clang::RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS) {
   SourceManager &SM = PP.getSourceManager();
 
   Rewriter Rewrite;
@@ -112,7 +112,7 @@ void clang::RewriteMacrosInInput(Preprocessor &PP, llvm::raw_ostream *OS) {
   // that aren't in the preprocessed view, we have macros that expand to no
   // tokens, or macro arguments etc.
   while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) {
-    SourceLocation PPLoc = SM.getInstantiationLoc(PPTok.getLocation());
+    SourceLocation PPLoc = SM.getExpansionLoc(PPTok.getLocation());
 
     // If PPTok is from a different source file, ignore it.
     if (!SM.isFromMainFile(PPLoc)) {
@@ -197,7 +197,7 @@ void clang::RewriteMacrosInInput(Preprocessor &PP, llvm::raw_ostream *OS) {
     while (PPOffs < RawOffs) {
       Expansion += ' ' + PP.getSpelling(PPTok);
       PP.Lex(PPTok);
-      PPLoc = SM.getInstantiationLoc(PPTok.getLocation());
+      PPLoc = SM.getExpansionLoc(PPTok.getLocation());
       PPOffs = SM.getFileOffset(PPLoc);
     }
     Expansion += ' ';
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/RewriteObjC.cpp b/contrib/llvm/tools/clang/lib/Rewrite/RewriteObjC.cpp
index 8202164ebce5..6a392ea357f1 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/RewriteObjC.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/RewriteObjC.cpp
@@ -55,7 +55,7 @@ namespace {
     };
     
     Rewriter Rewrite;
-    Diagnostic &Diags;
+    DiagnosticsEngine &Diags;
     const LangOptions &LangOpts;
     unsigned RewriteFailedDiag;
     unsigned TryFinallyContainsReturnDiag;
@@ -67,14 +67,14 @@ namespace {
     const char *MainFileStart, *MainFileEnd;
     SourceLocation LastIncLoc;
 
-    llvm::SmallVector<ObjCImplementationDecl *, 8> ClassImplementation;
-    llvm::SmallVector<ObjCCategoryImplDecl *, 8> CategoryImplementation;
+    SmallVector<ObjCImplementationDecl *, 8> ClassImplementation;
+    SmallVector<ObjCCategoryImplDecl *, 8> CategoryImplementation;
     llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCSynthesizedStructs;
     llvm::SmallPtrSet<ObjCProtocolDecl*, 8> ObjCSynthesizedProtocols;
     llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCForwardDecls;
     llvm::DenseMap<ObjCMethodDecl*, std::string> MethodInternalNames;
-    llvm::SmallVector<Stmt *, 32> Stmts;
-    llvm::SmallVector<int, 8> ObjCBcLabelNo;
+    SmallVector<Stmt *, 32> Stmts;
+    SmallVector<int, 8> ObjCBcLabelNo;
     // Remember all the @protocol(<expr>) expressions.
     llvm::SmallPtrSet<ObjCProtocolDecl *, 32> ProtocolExprDecls;
     
@@ -113,7 +113,7 @@ namespace {
     bool IsHeader;
 
     std::string InFileName;
-    llvm::raw_ostream* OutFile;
+    raw_ostream* OutFile;
 
     bool SilenceRewriteMacroWarning;
     bool objc_impl_method;
@@ -121,16 +121,16 @@ namespace {
     std::string Preamble;
 
     // Block expressions.
-    llvm::SmallVector<BlockExpr *, 32> Blocks;
-    llvm::SmallVector<int, 32> InnerDeclRefsCount;
-    llvm::SmallVector<BlockDeclRefExpr *, 32> InnerDeclRefs;
+    SmallVector<BlockExpr *, 32> Blocks;
+    SmallVector<int, 32> InnerDeclRefsCount;
+    SmallVector<BlockDeclRefExpr *, 32> InnerDeclRefs;
     
-    llvm::SmallVector<BlockDeclRefExpr *, 32> BlockDeclRefs;
+    SmallVector<BlockDeclRefExpr *, 32> BlockDeclRefs;
 
     // Block related declarations.
-    llvm::SmallVector<ValueDecl *, 8> BlockByCopyDecls;
+    SmallVector<ValueDecl *, 8> BlockByCopyDecls;
     llvm::SmallPtrSet<ValueDecl *, 8> BlockByCopyDeclsPtrSet;
-    llvm::SmallVector<ValueDecl *, 8> BlockByRefDecls;
+    SmallVector<ValueDecl *, 8> BlockByRefDecls;
     llvm::SmallPtrSet<ValueDecl *, 8> BlockByRefDeclsPtrSet;
     llvm::DenseMap<ValueDecl *, unsigned> BlockByRefDeclNo;
     llvm::SmallPtrSet<ValueDecl *, 8> ImportedBlockDecls;
@@ -161,13 +161,18 @@ namespace {
 
     // Top Level Driver code.
     virtual void HandleTopLevelDecl(DeclGroupRef D) {
-      for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I)
+      for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
+        if (isa<ObjCClassDecl>((*I))) {
+          RewriteForwardClassDecl(D);
+          break;
+        }
         HandleTopLevelSingleDecl(*I);
+      }
     }
     void HandleTopLevelSingleDecl(Decl *D);
     void HandleDeclInMainFile(Decl *D);
-    RewriteObjC(std::string inFile, llvm::raw_ostream *OS,
-                Diagnostic &D, const LangOptions &LOpts,
+    RewriteObjC(std::string inFile, raw_ostream *OS,
+                DiagnosticsEngine &D, const LangOptions &LOpts,
                 bool silenceMacroWarn);
 
     ~RewriteObjC() {}
@@ -219,7 +224,7 @@ namespace {
                    << Old->getSourceRange();
     }
 
-    void InsertText(SourceLocation Loc, llvm::StringRef Str,
+    void InsertText(SourceLocation Loc, StringRef Str,
                     bool InsertAfter = true) {
       // If insertion succeeded or warning disabled return with no warning.
       if (!Rewrite.InsertText(Loc, Str, InsertAfter) ||
@@ -230,7 +235,7 @@ namespace {
     }
 
     void ReplaceText(SourceLocation Start, unsigned OrigLength,
-                     llvm::StringRef Str) {
+                     StringRef Str) {
       // If removal succeeded or warning disabled return with no warning.
       if (!Rewrite.ReplaceText(Start, OrigLength, Str) ||
           SilenceRewriteMacroWarning)
@@ -241,7 +246,11 @@ namespace {
 
     // Syntactic Rewriting.
     void RewriteInclude();
-    void RewriteForwardClassDecl(ObjCClassDecl *Dcl);
+    void RewriteForwardClassDecl(DeclGroupRef D);
+    void RewriteForwardClassDecl(const llvm::SmallVector<Decl*, 8> &DG);
+    void RewriteForwardClassEpilogue(ObjCClassDecl *ClassDecl, 
+                                     const std::string &typedefString);
+      
     void RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
                                  ObjCImplementationDecl *IMD,
                                  ObjCCategoryImplDecl *CID);
@@ -332,17 +341,17 @@ namespace {
     void RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
                                     MethodIterator MethodEnd,
                                     bool IsInstanceMethod,
-                                    llvm::StringRef prefix,
-                                    llvm::StringRef ClassName,
+                                    StringRef prefix,
+                                    StringRef ClassName,
                                     std::string &Result);
 
     void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
-                                     llvm::StringRef prefix,
-                                     llvm::StringRef ClassName,
+                                     StringRef prefix,
+                                     StringRef ClassName,
                                      std::string &Result);
     void RewriteObjCProtocolListMetaData(const ObjCList<ObjCProtocolDecl> &Prots,
-                                         llvm::StringRef prefix,
-                                         llvm::StringRef ClassName,
+                                         StringRef prefix,
+                                         StringRef ClassName,
                                          std::string &Result);
     void SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
                                       std::string &Result);
@@ -367,24 +376,24 @@ namespace {
     void RewriteBlockPointerFunctionArgs(FunctionDecl *FD);
 
     std::string SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                      llvm::StringRef funcName, std::string Tag);
+                                      StringRef funcName, std::string Tag);
     std::string SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                      llvm::StringRef funcName, std::string Tag);
+                                      StringRef funcName, std::string Tag);
     std::string SynthesizeBlockImpl(BlockExpr *CE, 
                                     std::string Tag, std::string Desc);
     std::string SynthesizeBlockDescriptor(std::string DescTag, 
                                           std::string ImplTag,
-                                          int i, llvm::StringRef funcName,
+                                          int i, StringRef funcName,
                                           unsigned hasCopy);
     Stmt *SynthesizeBlockCall(CallExpr *Exp, const Expr* BlockExp);
     void SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                 llvm::StringRef FunName);
+                                 StringRef FunName);
     void RewriteRecordBody(RecordDecl *RD);
 
     void CollectBlockDeclRefInfo(BlockExpr *Exp);
     void GetBlockDeclRefExprs(Stmt *S);
     void GetInnerBlockDeclRefExprs(Stmt *S, 
-                llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs,
+                SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs,
                 llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts);
 
     // We avoid calling Type::isBlockPointerType(), since it operates on the
@@ -411,8 +420,14 @@ namespace {
       else if (T->isObjCQualifiedClassType())
         T = Context->getObjCClassType();
       else if (T->isObjCObjectPointerType() &&
-               T->getPointeeType()->isObjCQualifiedInterfaceType())
-        T = Context->getObjCIdType();
+               T->getPointeeType()->isObjCQualifiedInterfaceType()) {
+        if (const ObjCObjectPointerType * OBJPT =
+              T->getAsObjCInterfacePointerType()) {
+          const ObjCInterfaceType *IFaceT = OBJPT->getInterfaceType();
+          T = QualType(IFaceT, 0);
+          T = Context->getPointerType(T);
+        }
+     }
     }
     
     // FIXME: This predicate seems like it would be useful to add to ASTContext.
@@ -439,9 +454,9 @@ namespace {
                             const char *&RParen);
     void RewriteCastExpr(CStyleCastExpr *CE);
 
-    FunctionDecl *SynthBlockInitFunctionDecl(llvm::StringRef name);
+    FunctionDecl *SynthBlockInitFunctionDecl(StringRef name);
     Stmt *SynthBlockInitExpr(BlockExpr *Exp,
-            const llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs);
+            const SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs);
 
     void QuoteDoublequotes(std::string &From, std::string &To) {
       for (unsigned i = 0; i < From.length(); i++) {
@@ -456,6 +471,8 @@ namespace {
                                    const QualType *args,
                                    unsigned numArgs,
                                    bool variadic = false) {
+      if (result == Context->getObjCInstanceType())
+        result =  Context->getObjCIdType();
       FunctionProtoType::ExtProtoInfo fpi;
       fpi.Variadic = variadic;
       return Context->getFunctionType(result, args, numArgs, fpi);
@@ -505,22 +522,23 @@ static bool IsHeaderFile(const std::string &Filename) {
   return Ext == "h" || Ext == "hh" || Ext == "H";
 }
 
-RewriteObjC::RewriteObjC(std::string inFile, llvm::raw_ostream* OS,
-                         Diagnostic &D, const LangOptions &LOpts,
+RewriteObjC::RewriteObjC(std::string inFile, raw_ostream* OS,
+                         DiagnosticsEngine &D, const LangOptions &LOpts,
                          bool silenceMacroWarn)
       : Diags(D), LangOpts(LOpts), InFileName(inFile), OutFile(OS),
         SilenceRewriteMacroWarning(silenceMacroWarn) {
   IsHeader = IsHeaderFile(inFile);
-  RewriteFailedDiag = Diags.getCustomDiagID(Diagnostic::Warning,
+  RewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
                "rewriting sub-expression within a macro (may not be correct)");
-  TryFinallyContainsReturnDiag = Diags.getCustomDiagID(Diagnostic::Warning,
+  TryFinallyContainsReturnDiag = Diags.getCustomDiagID(
+               DiagnosticsEngine::Warning,
                "rewriter doesn't support user-specified control flow semantics "
                "for @try/@finally (code may not execute properly)");
 }
 
 ASTConsumer *clang::CreateObjCRewriter(const std::string& InFile,
-                                       llvm::raw_ostream* OS,
-                                       Diagnostic &Diags,
+                                       raw_ostream* OS,
+                                       DiagnosticsEngine &Diags,
                                        const LangOptions &LOpts,
                                        bool SilenceRewriteMacroWarning) {
   return new RewriteObjC(InFile, OS, Diags, LOpts, SilenceRewriteMacroWarning);
@@ -572,7 +590,7 @@ void RewriteObjC::Initialize(ASTContext &context) {
   Preamble += "struct objc_selector; struct objc_class;\n";
   Preamble += "struct __rw_objc_super { struct objc_object *object; ";
   Preamble += "struct objc_object *superClass; ";
-  if (LangOpts.Microsoft) {
+  if (LangOpts.MicrosoftExt) {
     // Add a constructor for creating temporary objects.
     Preamble += "__rw_objc_super(struct objc_object *o, struct objc_object *s) "
                 ": ";
@@ -583,7 +601,7 @@ void RewriteObjC::Initialize(ASTContext &context) {
   Preamble += "typedef struct objc_object Protocol;\n";
   Preamble += "#define _REWRITER_typedef_Protocol\n";
   Preamble += "#endif\n";
-  if (LangOpts.Microsoft) {
+  if (LangOpts.MicrosoftExt) {
     Preamble += "#define __OBJC_RW_DLLIMPORT extern \"C\" __declspec(dllimport)\n";
     Preamble += "#define __OBJC_RW_STATICIMPORT extern \"C\"\n";
   } else
@@ -592,9 +610,9 @@ void RewriteObjC::Initialize(ASTContext &context) {
   Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
   Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSendSuper";
   Preamble += "(struct objc_super *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSend_stret";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object* objc_msgSend_stret";
   Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSendSuper_stret";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object* objc_msgSendSuper_stret";
   Preamble += "(struct objc_super *, struct objc_selector *, ...);\n";
   Preamble += "__OBJC_RW_DLLIMPORT double objc_msgSend_fpret";
   Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
@@ -660,7 +678,7 @@ void RewriteObjC::Initialize(ASTContext &context) {
   Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteStackBlock[32];\n";
   Preamble += "#endif\n";
   Preamble += "#endif\n";
-  if (LangOpts.Microsoft) {
+  if (LangOpts.MicrosoftExt) {
     Preamble += "#undef __OBJC_RW_DLLIMPORT\n";
     Preamble += "#undef __OBJC_RW_STATICIMPORT\n";
     Preamble += "#ifndef KEEP_ATTRIBUTES\n";  // We use this for clang tests.
@@ -690,7 +708,7 @@ void RewriteObjC::HandleTopLevelSingleDecl(Decl *D) {
   // #included file.  If the former, rewrite it now.  If the later, check to see
   // if we rewrote the #include/#import.
   SourceLocation Loc = D->getLocation();
-  Loc = SM->getInstantiationLoc(Loc);
+  Loc = SM->getExpansionLoc(Loc);
 
   // If this is for a builtin, ignore it.
   if (Loc.isInvalid()) return;
@@ -717,8 +735,23 @@ void RewriteObjC::HandleTopLevelSingleDecl(Decl *D) {
     // Recurse into linkage specifications
     for (DeclContext::decl_iterator DI = LSD->decls_begin(),
                                  DIEnd = LSD->decls_end();
-         DI != DIEnd; ++DI)
+         DI != DIEnd; ) {
+      if (isa<ObjCClassDecl>((*DI))) {
+        SmallVector<Decl *, 8> DG;
+        Decl *D = (*DI);
+        SourceLocation Loc = D->getLocation();
+        while (DI != DIEnd &&
+               isa<ObjCClassDecl>(D) && D->getLocation() == Loc) {
+          DG.push_back(D);
+          ++DI;
+          D = (*DI);
+        }
+        RewriteForwardClassDecl(DG);
+        continue;
+      }
       HandleTopLevelSingleDecl(*DI);
+      ++DI;
+    }
   }
   // If we have a decl in the main file, see if we should rewrite it.
   if (SM->isFromMainFile(Loc))
@@ -731,7 +764,7 @@ void RewriteObjC::HandleTopLevelSingleDecl(Decl *D) {
 
 void RewriteObjC::RewriteInclude() {
   SourceLocation LocStart = SM->getLocForStartOfFile(MainFileID);
-  llvm::StringRef MainBuf = SM->getBufferData(MainFileID);
+  StringRef MainBuf = SM->getBufferData(MainFileID);
   const char *MainBufStart = MainBuf.begin();
   const char *MainBufEnd = MainBuf.end();
   size_t ImportLen = strlen("import");
@@ -747,7 +780,7 @@ void RewriteObjC::RewriteInclude() {
       if (!strncmp(BufPtr, "import", ImportLen)) {
         // replace import with include
         SourceLocation ImportLoc =
-          LocStart.getFileLocWithOffset(BufPtr-MainBufStart);
+          LocStart.getLocWithOffset(BufPtr-MainBufStart);
         ReplaceText(ImportLoc, ImportLen, "include");
         BufPtr += ImportLen;
       }
@@ -777,7 +810,7 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
   const char *semiBuf = strchr(startBuf, ';');
   assert((*semiBuf == ';') && "@synthesize: can't find ';'");
   SourceLocation onePastSemiLoc =
-    startLoc.getFileLocWithOffset(semiBuf-startBuf+1);
+    startLoc.getLocWithOffset(semiBuf-startBuf+1);
 
   if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
     return; // FIXME: is this correct?
@@ -821,7 +854,7 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
           for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
             if (i) Getr += ", ";
             std::string ParamStr = FT->getArgType(i).getAsString(
-                                                          Context->PrintingPolicy);
+                                                          Context->getPrintingPolicy());
             Getr += ParamStr;
           }
           if (FT->isVariadic()) {
@@ -886,55 +919,73 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
   InsertText(onePastSemiLoc, Setr);
 }
 
-void RewriteObjC::RewriteForwardClassDecl(ObjCClassDecl *ClassDecl) {
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = ClassDecl->getLocation();
-  const char *startBuf = SM->getCharacterData(startLoc);
-  const char *semiPtr = strchr(startBuf, ';');
+static void RewriteOneForwardClassDecl(ObjCInterfaceDecl *ForwardDecl,
+                                       std::string &typedefString) {
+  typedefString += "#ifndef _REWRITER_typedef_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += "\n";
+  typedefString += "#define _REWRITER_typedef_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += "\n";
+  typedefString += "typedef struct objc_object ";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += ";\n#endif\n";
+}
 
-  // Translate to typedef's that forward reference structs with the same name
-  // as the class. As a convenience, we include the original declaration
-  // as a comment.
+void RewriteObjC::RewriteForwardClassEpilogue(ObjCClassDecl *ClassDecl,
+                                              const std::string &typedefString) {
+    SourceLocation startLoc = ClassDecl->getLocation();
+    const char *startBuf = SM->getCharacterData(startLoc);
+    const char *semiPtr = strchr(startBuf, ';'); 
+    // Replace the @class with typedefs corresponding to the classes.
+    ReplaceText(startLoc, semiPtr-startBuf+1, typedefString);  
+}
+
+void RewriteObjC::RewriteForwardClassDecl(DeclGroupRef D) {
   std::string typedefString;
-  typedefString += "// @class ";
-  for (ObjCClassDecl::iterator I = ClassDecl->begin(), E = ClassDecl->end();
-       I != E; ++I) {
-    ObjCInterfaceDecl *ForwardDecl = I->getInterface();
-    typedefString += ForwardDecl->getNameAsString();
-    if (I+1 != E)
-      typedefString += ", ";
-    else
+  for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
+    ObjCClassDecl *ClassDecl = cast<ObjCClassDecl>(*I);
+    ObjCInterfaceDecl *ForwardDecl = ClassDecl->getForwardInterfaceDecl();
+    if (I == D.begin()) {
+      // Translate to typedef's that forward reference structs with the same name
+      // as the class. As a convenience, we include the original declaration
+      // as a comment.
+      typedefString += "// @class ";
+      typedefString += ForwardDecl->getNameAsString();
       typedefString += ";\n";
+    }
+    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
   }
-  
-  for (ObjCClassDecl::iterator I = ClassDecl->begin(), E = ClassDecl->end();
-       I != E; ++I) {
-    ObjCInterfaceDecl *ForwardDecl = I->getInterface();
-    typedefString += "#ifndef _REWRITER_typedef_";
-    typedefString += ForwardDecl->getNameAsString();
-    typedefString += "\n";
-    typedefString += "#define _REWRITER_typedef_";
-    typedefString += ForwardDecl->getNameAsString();
-    typedefString += "\n";
-    typedefString += "typedef struct objc_object ";
-    typedefString += ForwardDecl->getNameAsString();
-    typedefString += ";\n#endif\n";
-  }
+  DeclGroupRef::iterator I = D.begin();
+  RewriteForwardClassEpilogue(cast<ObjCClassDecl>(*I), typedefString);
+}
 
-  // Replace the @class with typedefs corresponding to the classes.
-  ReplaceText(startLoc, semiPtr-startBuf+1, typedefString);
+void RewriteObjC::RewriteForwardClassDecl(
+                                const llvm::SmallVector<Decl*, 8> &D) {
+  std::string typedefString;
+  for (unsigned i = 0; i < D.size(); i++) {
+    ObjCClassDecl *ClassDecl = cast<ObjCClassDecl>(D[i]);
+    ObjCInterfaceDecl *ForwardDecl = ClassDecl->getForwardInterfaceDecl();
+    if (i == 0) {
+      typedefString += "// @class ";
+      typedefString += ForwardDecl->getNameAsString();
+      typedefString += ";\n";
+    }
+    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
+  }
+  RewriteForwardClassEpilogue(cast<ObjCClassDecl>(D[0]), typedefString);
 }
 
 void RewriteObjC::RewriteMethodDeclaration(ObjCMethodDecl *Method) {
   // When method is a synthesized one, such as a getter/setter there is
   // nothing to rewrite.
-  if (Method->isSynthesized())
+  if (Method->isImplicit())
     return;
   SourceLocation LocStart = Method->getLocStart();
   SourceLocation LocEnd = Method->getLocEnd();
 
-  if (SM->getInstantiationLineNumber(LocEnd) >
-      SM->getInstantiationLineNumber(LocStart)) {
+  if (SM->getExpansionLineNumber(LocEnd) >
+      SM->getExpansionLineNumber(LocStart)) {
     InsertText(LocStart, "#if 0\n");
     ReplaceText(LocEnd, 1, ";\n#endif\n");
   } else {
@@ -1001,12 +1052,12 @@ void RewriteObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
   const char *endBuf = SM->getCharacterData(LocEnd);
   for (const char *p = startBuf; p < endBuf; p++) {
     if (*p == '@' && !strncmp(p+1, "optional", strlen("optional"))) {
-      SourceLocation OptionalLoc = LocStart.getFileLocWithOffset(p-startBuf);
+      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
       ReplaceText(OptionalLoc, strlen("@optional"), "/* @optional */");
 
     }
     else if (*p == '@' && !strncmp(p+1, "required", strlen("required"))) {
-      SourceLocation OptionalLoc = LocStart.getFileLocWithOffset(p-startBuf);
+      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
       ReplaceText(OptionalLoc, strlen("@required"), "/* @required */");
 
     }
@@ -1016,7 +1067,7 @@ void RewriteObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
 void RewriteObjC::RewriteForwardProtocolDecl(ObjCForwardProtocolDecl *PDecl) {
   SourceLocation LocStart = PDecl->getLocation();
   if (LocStart.isInvalid())
-    assert(false && "Invalid SourceLocation");
+    llvm_unreachable("Invalid SourceLocation");
   // FIXME: handle forward protocol that are declared across multiple lines.
   ReplaceText(LocStart, 0, "// ");
 }
@@ -1037,11 +1088,11 @@ void RewriteObjC::RewriteTypeIntoString(QualType T, std::string &ResultStr,
       PointeeTy = BPT->getPointeeType();
     if ((FPRetType = PointeeTy->getAs<FunctionType>())) {
       ResultStr += FPRetType->getResultType().getAsString(
-        Context->PrintingPolicy);
+        Context->getPrintingPolicy());
       ResultStr += "(*";
     }
   } else
-    ResultStr += T.getAsString(Context->PrintingPolicy);
+    ResultStr += T.getAsString(Context->getPrintingPolicy());
 }
 
 void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
@@ -1089,7 +1140,7 @@ void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
   if (OMD->isInstanceMethod()) {
     QualType selfTy = Context->getObjCInterfaceType(IDecl);
     selfTy = Context->getPointerType(selfTy);
-    if (!LangOpts.Microsoft) {
+    if (!LangOpts.MicrosoftExt) {
       if (ObjCSynthesizedStructs.count(const_cast<ObjCInterfaceDecl*>(IDecl)))
         ResultStr += "struct ";
     }
@@ -1099,10 +1150,10 @@ void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
   }
   else
     ResultStr += Context->getObjCClassType().getAsString(
-      Context->PrintingPolicy);
+      Context->getPrintingPolicy());
 
   ResultStr += " self, ";
-  ResultStr += Context->getObjCSelType().getAsString(Context->PrintingPolicy);
+  ResultStr += Context->getObjCSelType().getAsString(Context->getPrintingPolicy());
   ResultStr += " _cmd";
 
   // Method arguments.
@@ -1118,9 +1169,9 @@ void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
       QualType QT = PDecl->getType();
       // Make sure we convert "t (^)(...)" to "t (*)(...)".
       if (convertBlockPointerToFunctionPointer(QT))
-        QT.getAsStringInternal(Name, Context->PrintingPolicy);
+        QT.getAsStringInternal(Name, Context->getPrintingPolicy());
       else
-        PDecl->getType().getAsStringInternal(Name, Context->PrintingPolicy);
+        PDecl->getType().getAsStringInternal(Name, Context->getPrintingPolicy());
       ResultStr += Name;
     }
   }
@@ -1137,7 +1188,7 @@ void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
       for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
         if (i) ResultStr += ", ";
         std::string ParamStr = FT->getArgType(i).getAsString(
-          Context->PrintingPolicy);
+          Context->getPrintingPolicy());
         ResultStr += ParamStr;
       }
       if (FT->isVariadic()) {
@@ -1265,8 +1316,6 @@ Stmt *RewriteObjC::RewritePropertyOrImplicitSetter(BinaryOperator *BinOp, Expr *
   }
   
   assert(OMD && "RewritePropertyOrImplicitSetter - null OMD");
-  llvm::SmallVector<Expr *, 1> ExprVec;
-  ExprVec.push_back(newStmt);
 
   ObjCMessageExpr *MsgExpr;
   if (Super)
@@ -1278,7 +1327,7 @@ Stmt *RewriteObjC::RewritePropertyOrImplicitSetter(BinaryOperator *BinOp, Expr *
                                       /*IsInstanceSuper=*/true,
                                       SuperTy,
                                       Sel, SelectorLoc, OMD,
-                                      &ExprVec[0], 1,
+                                      newStmt,
                                       /*FIXME:*/SourceLocation());
   else {
     // FIXME. Refactor this into common code with that in 
@@ -1295,7 +1344,7 @@ Stmt *RewriteObjC::RewritePropertyOrImplicitSetter(BinaryOperator *BinOp, Expr *
                                       /*FIXME: */SourceLocation(),
                                       cast<Expr>(Receiver),
                                       Sel, SelectorLoc, OMD,
-                                      &ExprVec[0], 1,
+                                      newStmt,
                                       /*FIXME:*/SourceLocation());
   }
   Stmt *ReplacingStmt = SynthMessageExpr(MsgExpr);
@@ -1354,7 +1403,7 @@ Stmt *RewriteObjC::RewritePropertyOrImplicitGetter(Expr *PropOrGetterRefExpr) {
                                       /*IsInstanceSuper=*/true,
                                       SuperTy,
                                       Sel, SelectorLoc, OMD,
-                                      0, 0, 
+                                      ArrayRef<Expr*>(), 
                                       PropOrGetterRefExpr->getLocEnd());
   else {
     assert (Receiver && "RewritePropertyOrImplicitGetter - Receiver is null");
@@ -1368,7 +1417,7 @@ Stmt *RewriteObjC::RewritePropertyOrImplicitGetter(Expr *PropOrGetterRefExpr) {
                                       PropOrGetterRefExpr->getLocStart(),
                                       cast<Expr>(Receiver),
                                       Sel, SelectorLoc, OMD,
-                                      0, 0, 
+                                      ArrayRef<Expr*>(), 
                                       PropOrGetterRefExpr->getLocEnd());
   }
 
@@ -1611,7 +1660,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
 
   SourceLocation startLoc = S->getLocStart();
   const char *startBuf = SM->getCharacterData(startLoc);
-  llvm::StringRef elementName;
+  StringRef elementName;
   std::string elementTypeAsString;
   std::string buf;
   buf = "\n{\n\t";
@@ -1624,7 +1673,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
       // Simply use 'id' for all qualified types.
       elementTypeAsString = "id";
     else
-      elementTypeAsString = ElementType.getAsString(Context->PrintingPolicy);
+      elementTypeAsString = ElementType.getAsString(Context->getPrintingPolicy());
     buf += elementTypeAsString;
     buf += " ";
     elementName = D->getName();
@@ -1640,7 +1689,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
       // Simply use 'id' for all qualified types.
       elementTypeAsString = "id";
     else
-      elementTypeAsString = VD->getType().getAsString(Context->PrintingPolicy);
+      elementTypeAsString = VD->getType().getAsString(Context->getPrintingPolicy());
   }
 
   // struct __objcFastEnumerationState enumState = { 0 };
@@ -1667,7 +1716,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
   // Replace ')' in for '(' type elem in collection ')' with ';'
   SourceLocation rightParenLoc = S->getRParenLoc();
   const char *rparenBuf = SM->getCharacterData(rightParenLoc);
-  SourceLocation lparenLoc = startLoc.getFileLocWithOffset(rparenBuf-startBuf);
+  SourceLocation lparenLoc = startLoc.getLocWithOffset(rparenBuf-startBuf);
   buf = ";\n\t";
 
   // unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
@@ -1744,7 +1793,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
   // Insert all these *after* the statement body.
   // FIXME: If this should support Obj-C++, support CXXTryStmt
   if (isa<CompoundStmt>(S->getBody())) {
-    SourceLocation endBodyLoc = OrigEnd.getFileLocWithOffset(1);
+    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(1);
     InsertText(endBodyLoc, buf);
   } else {
     /* Need to treat single statements specially. For example:
@@ -1757,7 +1806,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
     const char *stmtBuf = SM->getCharacterData(OrigEnd);
     const char *semiBuf = strchr(stmtBuf, ';');
     assert(semiBuf && "Can't find ';'");
-    SourceLocation endBodyLoc = OrigEnd.getFileLocWithOffset(semiBuf-stmtBuf+1);
+    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(semiBuf-stmtBuf+1);
     InsertText(endBodyLoc, buf);
   }
   Stmts.pop_back();
@@ -1789,7 +1838,7 @@ Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   SourceLocation endLoc = S->getSynchBody()->getLocStart();
   const char *endBuf = SM->getCharacterData(endLoc);
   while (*endBuf != ')') endBuf--;
-  SourceLocation rparenLoc = startLoc.getFileLocWithOffset(endBuf-startBuf);
+  SourceLocation rparenLoc = startLoc.getLocWithOffset(endBuf-startBuf);
   buf = ");\n";
   // declare a new scope with two variables, _stack and _rethrow.
   buf += "/* @try scope begin */ \n{ struct _objc_exception_data {\n";
@@ -1812,9 +1861,15 @@ Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   
   std::string syncBuf;
   syncBuf += " objc_sync_exit(";
-  Expr *syncExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CK_BitCast,
-                                            S->getSynchExpr());
+
+  Expr *syncExpr = S->getSynchExpr();
+  CastKind CK = syncExpr->getType()->isObjCObjectPointerType()
+                  ? CK_BitCast :
+                syncExpr->getType()->isBlockPointerType()
+                  ? CK_BlockPointerToObjCPointerCast
+                  : CK_CPointerToObjCPointerCast;
+  syncExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                                      CK, syncExpr);
   std::string syncExprBufS;
   llvm::raw_string_ostream syncExprBuf(syncExprBufS);
   syncExpr->printPretty(syncExprBuf, *Context, 0,
@@ -1875,7 +1930,7 @@ void RewriteObjC::RewriteTryReturnStmts(Stmt *S) {
 
    const char *semiBuf = strchr(startBuf, ';');
    assert((*semiBuf == ';') && "RewriteTryReturnStmts: can't find ';'");
-   SourceLocation onePastSemiLoc = startLoc.getFileLocWithOffset(semiBuf-startBuf+1);
+   SourceLocation onePastSemiLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
 
    std::string buf;
    buf = "{ objc_exception_try_exit(&_stack); return";
@@ -1898,7 +1953,7 @@ void RewriteObjC::RewriteSyncReturnStmts(Stmt *S, std::string syncExitBuf) {
 
     const char *semiBuf = strchr(startBuf, ';');
     assert((*semiBuf == ';') && "RewriteSyncReturnStmts: can't find ';'");
-    SourceLocation onePastSemiLoc = startLoc.getFileLocWithOffset(semiBuf-startBuf+1);
+    SourceLocation onePastSemiLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
 
     std::string buf;
     buf = "{ objc_exception_try_exit(&_stack);";
@@ -1936,7 +1991,7 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
 
   SourceLocation lastCurlyLoc = startLoc;
   if (S->getNumCatchStmts()) {
-    startLoc = startLoc.getFileLocWithOffset(1);
+    startLoc = startLoc.getLocWithOffset(1);
     buf = " /* @catch begin */ else {\n";
     buf += " id _caught = objc_exception_extract(&_stack);\n";
     buf += " objc_exception_try_enter (&_stack);\n";
@@ -2007,7 +2062,7 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
       // declares the @catch parameter).
       ReplaceText(rParenLoc, bodyBuf-rParenBuf+1, " = _caught;");
     } else {
-      assert(false && "@catch rewrite bug");
+      llvm_unreachable("@catch rewrite bug");
     }
   }
   // Complete the catch list...
@@ -2017,7 +2072,7 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
            "bogus @catch body location");
 
     // Insert the last (implicit) else clause *before* the right curly brace.
-    bodyLoc = bodyLoc.getFileLocWithOffset(-1);
+    bodyLoc = bodyLoc.getLocWithOffset(-1);
     buf = "} /* last catch end */\n";
     buf += "else {\n";
     buf += " _rethrow = _caught;\n";
@@ -2045,9 +2100,9 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
     assert(*SM->getCharacterData(endLoc) == '}' &&
            "bogus @finally body location");
 
-    startLoc = startLoc.getFileLocWithOffset(1);
+    startLoc = startLoc.getLocWithOffset(1);
     InsertText(startLoc, " if (!_rethrow) objc_exception_try_exit(&_stack);\n");
-    endLoc = endLoc.getFileLocWithOffset(-1);
+    endLoc = endLoc.getLocWithOffset(-1);
     InsertText(endLoc, " if (_rethrow) objc_exception_throw(_rethrow);\n");
 
     // Set lastCurlyLoc
@@ -2071,7 +2126,7 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
       RewriteTryReturnStmts(S->getTryBody());
   }
   // Now emit the final closing curly brace...
-  lastCurlyLoc = lastCurlyLoc.getFileLocWithOffset(1);
+  lastCurlyLoc = lastCurlyLoc.getLocWithOffset(1);
   InsertText(lastCurlyLoc, " } /* @try scope end */\n");
   return 0;
 }
@@ -2100,7 +2155,7 @@ Stmt *RewriteObjC::RewriteObjCThrowStmt(ObjCAtThrowStmt *S) {
 
   const char *semiBuf = strchr(startBuf, ';');
   assert((*semiBuf == ';') && "@throw: can't find ';'");
-  SourceLocation semiLoc = startLoc.getFileLocWithOffset(semiBuf-startBuf);
+  SourceLocation semiLoc = startLoc.getLocWithOffset(semiBuf-startBuf);
   ReplaceText(semiLoc, 1, ");");
   return 0;
 }
@@ -2111,8 +2166,8 @@ Stmt *RewriteObjC::RewriteAtEncode(ObjCEncodeExpr *Exp) {
   std::string StrEncoding;
   Context->getObjCEncodingForType(Exp->getEncodedType(), StrEncoding);
   Expr *Replacement = StringLiteral::Create(*Context, StrEncoding,
-                                            false, false, StrType,
-                                            SourceLocation());
+                                            StringLiteral::Ascii, false,
+                                            StrType, SourceLocation());
   ReplaceStmt(Exp, Replacement);
 
   // Replace this subexpr in the parent.
@@ -2125,12 +2180,12 @@ Stmt *RewriteObjC::RewriteAtSelector(ObjCSelectorExpr *Exp) {
     SynthSelGetUidFunctionDecl();
   assert(SelGetUidFunctionDecl && "Can't find sel_registerName() decl");
   // Create a call to sel_registerName("selName").
-  llvm::SmallVector<Expr*, 8> SelExprs;
+  SmallVector<Expr*, 8> SelExprs;
   QualType argType = Context->getPointerType(Context->CharTy);
   SelExprs.push_back(StringLiteral::Create(*Context,
                                            Exp->getSelector().getAsString(),
-                                           false, false, argType, 
-                                           SourceLocation()));
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
   CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
                                                  &SelExprs[0], SelExprs.size());
   ReplaceStmt(Exp, SelExp);
@@ -2231,8 +2286,8 @@ void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Expr *E) {
     const char *startRef = 0, *endRef = 0;
     if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
       // Get the locations of the startRef, endRef.
-      SourceLocation LessLoc = Loc.getFileLocWithOffset(startRef-startBuf);
-      SourceLocation GreaterLoc = Loc.getFileLocWithOffset(endRef-startBuf+1);
+      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-startBuf);
+      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-startBuf+1);
       // Comment out the protocol references.
       InsertText(LessLoc, "/*");
       InsertText(GreaterLoc, "*/");
@@ -2276,8 +2331,8 @@ void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Decl *Dcl) {
     const char *startRef = 0, *endRef = 0;
     if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
       // Get the locations of the startRef, endRef.
-      SourceLocation LessLoc = Loc.getFileLocWithOffset(startRef-endBuf);
-      SourceLocation GreaterLoc = Loc.getFileLocWithOffset(endRef-endBuf+1);
+      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-endBuf);
+      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-endBuf+1);
       // Comment out the protocol references.
       InsertText(LessLoc, "/*");
       InsertText(GreaterLoc, "*/");
@@ -2299,9 +2354,9 @@ void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Decl *Dcl) {
       if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
         // Get the locations of the startRef, endRef.
         SourceLocation LessLoc =
-          Loc.getFileLocWithOffset(startRef-startFuncBuf);
+          Loc.getLocWithOffset(startRef-startFuncBuf);
         SourceLocation GreaterLoc =
-          Loc.getFileLocWithOffset(endRef-startFuncBuf+1);
+          Loc.getLocWithOffset(endRef-startFuncBuf+1);
         // Comment out the protocol references.
         InsertText(LessLoc, "/*");
         InsertText(GreaterLoc, "*/");
@@ -2330,7 +2385,7 @@ void RewriteObjC::RewriteTypeOfDecl(VarDecl *ND) {
   }
   // FIXME. This will not work for multiple declarators; as in:
   // __typeof__(a) b,c,d;
-  std::string TypeAsString(QT.getAsString(Context->PrintingPolicy));
+  std::string TypeAsString(QT.getAsString(Context->getPrintingPolicy()));
   SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
   const char *startBuf = SM->getCharacterData(DeclLoc);
   if (ND->getInit()) {
@@ -2342,13 +2397,13 @@ void RewriteObjC::RewriteTypeOfDecl(VarDecl *ND) {
       startLoc = ECE->getLParenLoc();
     else
       startLoc = E->getLocStart();
-    startLoc = SM->getInstantiationLoc(startLoc);
+    startLoc = SM->getExpansionLoc(startLoc);
     const char *endBuf = SM->getCharacterData(startLoc);
     ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
   }
   else {
     SourceLocation X = ND->getLocEnd();
-    X = SM->getInstantiationLoc(X);
+    X = SM->getExpansionLoc(X);
     const char *endBuf = SM->getCharacterData(X);
     ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
   }
@@ -2357,7 +2412,7 @@ void RewriteObjC::RewriteTypeOfDecl(VarDecl *ND) {
 // SynthSelGetUidFunctionDecl - SEL sel_registerName(const char *str);
 void RewriteObjC::SynthSelGetUidFunctionDecl() {
   IdentifierInfo *SelGetUidIdent = &Context->Idents.get("sel_registerName");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
   QualType getFuncType =
     getSimpleFunctionType(Context->getObjCSelType(), &ArgTys[0], ArgTys.size());
@@ -2380,7 +2435,7 @@ void RewriteObjC::RewriteFunctionDecl(FunctionDecl *FD) {
 }
 
 void RewriteObjC::RewriteBlockPointerType(std::string& Str, QualType Type) {
-  std::string TypeString(Type.getAsString(Context->PrintingPolicy));
+  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
   const char *argPtr = TypeString.c_str();
   if (!strchr(argPtr, '^')) {
     Str += TypeString;
@@ -2396,7 +2451,7 @@ void RewriteObjC::RewriteBlockPointerType(std::string& Str, QualType Type) {
 void RewriteObjC::RewriteBlockPointerTypeVariable(std::string& Str,
                                                   ValueDecl *VD) {
   QualType Type = VD->getType();
-  std::string TypeString(Type.getAsString(Context->PrintingPolicy));
+  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
   const char *argPtr = TypeString.c_str();
   int paren = 0;
   while (*argPtr) {
@@ -2430,7 +2485,7 @@ void RewriteObjC::RewriteBlockLiteralFunctionDecl(FunctionDecl *FD) {
   if (!proto)
     return;
   QualType Type = proto->getResultType();
-  std::string FdStr = Type.getAsString(Context->PrintingPolicy);
+  std::string FdStr = Type.getAsString(Context->getPrintingPolicy());
   FdStr += " ";
   FdStr += FD->getName();
   FdStr +=  "(";
@@ -2451,7 +2506,7 @@ void RewriteObjC::SynthSuperContructorFunctionDecl() {
   if (SuperContructorFunctionDecl)
     return;
   IdentifierInfo *msgSendIdent = &Context->Idents.get("__rw_objc_super");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   QualType argT = Context->getObjCIdType();
   assert(!argT.isNull() && "Can't find 'id' type");
   ArgTys.push_back(argT);
@@ -2469,7 +2524,7 @@ void RewriteObjC::SynthSuperContructorFunctionDecl() {
 // SynthMsgSendFunctionDecl - id objc_msgSend(id self, SEL op, ...);
 void RewriteObjC::SynthMsgSendFunctionDecl() {
   IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   QualType argT = Context->getObjCIdType();
   assert(!argT.isNull() && "Can't find 'id' type");
   ArgTys.push_back(argT);
@@ -2490,7 +2545,7 @@ void RewriteObjC::SynthMsgSendFunctionDecl() {
 // SynthMsgSendSuperFunctionDecl - id objc_msgSendSuper(struct objc_super *, SEL op, ...);
 void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
   IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSendSuper");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
                                       SourceLocation(), SourceLocation(),
                                       &Context->Idents.get("objc_super"));
@@ -2514,7 +2569,7 @@ void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
 // SynthMsgSendStretFunctionDecl - id objc_msgSend_stret(id self, SEL op, ...);
 void RewriteObjC::SynthMsgSendStretFunctionDecl() {
   IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_stret");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   QualType argT = Context->getObjCIdType();
   assert(!argT.isNull() && "Can't find 'id' type");
   ArgTys.push_back(argT);
@@ -2537,7 +2592,7 @@ void RewriteObjC::SynthMsgSendStretFunctionDecl() {
 void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
   IdentifierInfo *msgSendIdent =
     &Context->Idents.get("objc_msgSendSuper_stret");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
                                       SourceLocation(), SourceLocation(),
                                       &Context->Idents.get("objc_super"));
@@ -2561,7 +2616,7 @@ void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
 // SynthMsgSendFpretFunctionDecl - double objc_msgSend_fpret(id self, SEL op, ...);
 void RewriteObjC::SynthMsgSendFpretFunctionDecl() {
   IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_fpret");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   QualType argT = Context->getObjCIdType();
   assert(!argT.isNull() && "Can't find 'id' type");
   ArgTys.push_back(argT);
@@ -2582,7 +2637,7 @@ void RewriteObjC::SynthMsgSendFpretFunctionDecl() {
 // SynthGetClassFunctionDecl - id objc_getClass(const char *name);
 void RewriteObjC::SynthGetClassFunctionDecl() {
   IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getClass");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
   QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
                                                 &ArgTys[0], ArgTys.size());
@@ -2598,7 +2653,7 @@ void RewriteObjC::SynthGetClassFunctionDecl() {
 void RewriteObjC::SynthGetSuperClassFunctionDecl() {
   IdentifierInfo *getSuperClassIdent = 
     &Context->Idents.get("class_getSuperclass");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getObjCClassType());
   QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
                                                 &ArgTys[0], ArgTys.size());
@@ -2615,7 +2670,7 @@ void RewriteObjC::SynthGetSuperClassFunctionDecl() {
 // SynthGetMetaClassFunctionDecl - id objc_getClass(const char *name);
 void RewriteObjC::SynthGetMetaClassFunctionDecl() {
   IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getMetaClass");
-  llvm::SmallVector<QualType, 16> ArgTys;
+  SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
   QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
                                                 &ArgTys[0], ArgTys.size());
@@ -2667,7 +2722,7 @@ Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
                                            SourceLocation());
   // cast to NSConstantString *
   CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
-                                            CK_BitCast, Unop);
+                                            CK_CPointerToObjCPointerCast, Unop);
   ReplaceStmt(Exp, cast);
   // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return cast;
@@ -2770,7 +2825,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
   }
 
   // Synthesize a call to objc_msgSend().
-  llvm::SmallVector<Expr*, 8> MsgExprs;
+  SmallVector<Expr*, 8> MsgExprs;
   switch (Exp->getReceiverKind()) {
   case ObjCMessageExpr::SuperClass: {
     MsgSendFlavor = MsgSendSuperFunctionDecl;
@@ -2780,7 +2835,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
     ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
 
-    llvm::SmallVector<Expr*, 4> InitExprs;
+    SmallVector<Expr*, 4> InitExprs;
 
     // set the receiver to self, the first argument to all methods.
     InitExprs.push_back(
@@ -2793,11 +2848,12 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                         ); // set the 'receiver'.
 
     // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    llvm::SmallVector<Expr*, 8> ClsExprs;
+    SmallVector<Expr*, 8> ClsExprs;
     QualType argType = Context->getPointerType(Context->CharTy);
     ClsExprs.push_back(StringLiteral::Create(*Context,
                                    ClassDecl->getIdentifier()->getName(),
-                                   false, false, argType, SourceLocation()));
+                                   StringLiteral::Ascii, false,
+                                   argType, SourceLocation()));
     CallExpr *Cls = SynthesizeCallToFunctionDecl(GetMetaClassFunctionDecl,
                                                  &ClsExprs[0],
                                                  ClsExprs.size(),
@@ -2806,7 +2862,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // (Class)objc_getClass("CurrentClass")
     CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
                                              Context->getObjCClassType(),
-                                             CK_BitCast, Cls);
+                                             CK_CPointerToObjCPointerCast, Cls);
     ClsExprs.clear();
     ClsExprs.push_back(ArgExpr);
     Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
@@ -2823,7 +2879,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     QualType superType = getSuperStructType();
     Expr *SuperRep;
 
-    if (LangOpts.Microsoft) {
+    if (LangOpts.MicrosoftExt) {
       SynthSuperContructorFunctionDecl();
       // Simulate a contructor call...
       DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
@@ -2868,14 +2924,14 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
   }
 
   case ObjCMessageExpr::Class: {
-    llvm::SmallVector<Expr*, 8> ClsExprs;
+    SmallVector<Expr*, 8> ClsExprs;
     QualType argType = Context->getPointerType(Context->CharTy);
     ObjCInterfaceDecl *Class
       = Exp->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
     IdentifierInfo *clsName = Class->getIdentifier();
     ClsExprs.push_back(StringLiteral::Create(*Context,
                                              clsName->getName(),
-                                             false, false, 
+                                             StringLiteral::Ascii, false,
                                              argType, SourceLocation()));
     CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
                                                  &ClsExprs[0],
@@ -2891,7 +2947,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
     assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
     ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
-    llvm::SmallVector<Expr*, 4> InitExprs;
+    SmallVector<Expr*, 4> InitExprs;
 
     InitExprs.push_back(
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
@@ -2902,11 +2958,12 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                         ); // set the 'receiver'.
     
     // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    llvm::SmallVector<Expr*, 8> ClsExprs;
+    SmallVector<Expr*, 8> ClsExprs;
     QualType argType = Context->getPointerType(Context->CharTy);
     ClsExprs.push_back(StringLiteral::Create(*Context,
                                    ClassDecl->getIdentifier()->getName(),
-                                   false, false, argType, SourceLocation()));
+                                   StringLiteral::Ascii, false, argType,
+                                   SourceLocation()));
     CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
                                                  &ClsExprs[0],
                                                  ClsExprs.size(), 
@@ -2931,7 +2988,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     QualType superType = getSuperStructType();
     Expr *SuperRep;
 
-    if (LangOpts.Microsoft) {
+    if (LangOpts.MicrosoftExt) {
       SynthSuperContructorFunctionDecl();
       // Simulate a contructor call...
       DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
@@ -2975,19 +3032,25 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     Expr *recExpr = Exp->getInstanceReceiver();
     while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
       recExpr = CE->getSubExpr();
+    CastKind CK = recExpr->getType()->isObjCObjectPointerType()
+                    ? CK_BitCast : recExpr->getType()->isBlockPointerType()
+                                     ? CK_BlockPointerToObjCPointerCast
+                                     : CK_CPointerToObjCPointerCast;
+
     recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                       CK_BitCast, recExpr);
+                                       CK, recExpr);
     MsgExprs.push_back(recExpr);
     break;
   }
   }
 
   // Create a call to sel_registerName("selName"), it will be the 2nd argument.
-  llvm::SmallVector<Expr*, 8> SelExprs;
+  SmallVector<Expr*, 8> SelExprs;
   QualType argType = Context->getPointerType(Context->CharTy);
   SelExprs.push_back(StringLiteral::Create(*Context,
                                        Exp->getSelector().getAsString(),
-                                       false, false, argType, SourceLocation()));
+                                       StringLiteral::Ascii, false,
+                                       argType, SourceLocation()));
   CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
                                                  &SelExprs[0], SelExprs.size(),
                                                   StartLoc,
@@ -3005,16 +3068,42 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
         type = Context->getObjCIdType();
       // Make sure we convert "type (^)(...)" to "type (*)(...)".
       (void)convertBlockPointerToFunctionPointer(type);
-      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK_BitCast,
-                                          userExpr);
+      const Expr *SubExpr = ICE->IgnoreParenImpCasts();
+      CastKind CK;
+      if (SubExpr->getType()->isIntegralType(*Context) && 
+          type->isBooleanType()) {
+        CK = CK_IntegralToBoolean;
+      } else if (type->isObjCObjectPointerType()) {
+        if (SubExpr->getType()->isBlockPointerType()) {
+          CK = CK_BlockPointerToObjCPointerCast;
+        } else if (SubExpr->getType()->isPointerType()) {
+          CK = CK_CPointerToObjCPointerCast;
+        } else {
+          CK = CK_BitCast;
+        }
+      } else {
+        CK = CK_BitCast;
+      }
+
+      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, userExpr);
     }
     // Make id<P...> cast into an 'id' cast.
     else if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(userExpr)) {
       if (CE->getType()->isObjCQualifiedIdType()) {
         while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
           userExpr = CE->getSubExpr();
+        CastKind CK;
+        if (userExpr->getType()->isIntegralType(*Context)) {
+          CK = CK_IntegralToPointer;
+        } else if (userExpr->getType()->isBlockPointerType()) {
+          CK = CK_BlockPointerToObjCPointerCast;
+        } else if (userExpr->getType()->isPointerType()) {
+          CK = CK_CPointerToObjCPointerCast;
+        } else {
+          CK = CK_BitCast;
+        }
         userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CK_BitCast, userExpr);
+                                            CK, userExpr);
       }
     }
     MsgExprs.push_back(userExpr);
@@ -3025,7 +3114,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
   }
   // Generate the funky cast.
   CastExpr *cast;
-  llvm::SmallVector<QualType, 8> ArgTypes;
+  SmallVector<QualType, 8> ArgTypes;
   QualType returnType;
 
   // Push 'id' and 'SEL', the 2 implicit arguments.
@@ -3045,8 +3134,8 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       (void)convertBlockPointerToFunctionPointer(t);
       ArgTypes.push_back(t);
     }
-    returnType = OMD->getResultType()->isObjCQualifiedIdType()
-                   ? Context->getObjCIdType() : OMD->getResultType();
+    returnType = Exp->getType();
+    convertToUnqualifiedObjCType(returnType);
     (void)convertBlockPointerToFunctionPointer(returnType);
   } else {
     returnType = Context->getObjCIdType();
@@ -3251,7 +3340,7 @@ void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
   // SynthesizeObjCInternalStruct is ever called recursively.
   Result += "\nstruct ";
   Result += CDecl->getNameAsString();
-  if (LangOpts.Microsoft)
+  if (LangOpts.MicrosoftExt)
     Result += "_IMPL";
 
   if (NumIvars > 0) {
@@ -3275,7 +3364,7 @@ void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
     // This clause is segregated to avoid breaking the common case.
     if (BufferContainsPPDirectives(startBuf, cursor)) {
       SourceLocation L = RCDecl ? CDecl->getSuperClassLoc() :
-                                  CDecl->getClassLoc();
+                                  CDecl->getAtStartLoc();
       const char *endHeader = SM->getCharacterData(L);
       endHeader += Lexer::MeasureTokenLength(L, *SM, LangOpts);
 
@@ -3299,7 +3388,7 @@ void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
 
       // insert the super class structure definition.
       SourceLocation OnePastCurly =
-        LocStart.getFileLocWithOffset(cursor-startBuf+1);
+        LocStart.getLocWithOffset(cursor-startBuf+1);
       InsertText(OnePastCurly, Result);
     }
     cursor++; // past '{'
@@ -3307,7 +3396,7 @@ void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
     // Now comment out any visibility specifiers.
     while (cursor < endBuf) {
       if (*cursor == '@') {
-        SourceLocation atLoc = LocStart.getFileLocWithOffset(cursor-startBuf);
+        SourceLocation atLoc = LocStart.getLocWithOffset(cursor-startBuf);
         // Skip whitespace.
         for (++cursor; cursor[0] == ' ' || cursor[0] == '\t'; ++cursor)
           /*scan*/;
@@ -3324,20 +3413,20 @@ void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
       // of user code, then scan the ivar list and use needToScanForQualifiers
       // for type checking.
       else if (*cursor == '<') {
-        SourceLocation atLoc = LocStart.getFileLocWithOffset(cursor-startBuf);
+        SourceLocation atLoc = LocStart.getLocWithOffset(cursor-startBuf);
         InsertText(atLoc, "/* ");
         cursor = strchr(cursor, '>');
         cursor++;
-        atLoc = LocStart.getFileLocWithOffset(cursor-startBuf);
+        atLoc = LocStart.getLocWithOffset(cursor-startBuf);
         InsertText(atLoc, " */");
       } else if (*cursor == '^') { // rewrite block specifier.
-        SourceLocation caretLoc = LocStart.getFileLocWithOffset(cursor-startBuf);
+        SourceLocation caretLoc = LocStart.getLocWithOffset(cursor-startBuf);
         ReplaceText(caretLoc, 1, "*");
       }
       cursor++;
     }
     // Don't forget to add a ';'!!
-    InsertText(LocEnd.getFileLocWithOffset(1), ";");
+    InsertText(LocEnd.getLocWithOffset(1), ";");
   } else { // we don't have any instance variables - insert super struct.
     endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
     Result += " {\n    struct ";
@@ -3349,7 +3438,7 @@ void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
   }
   // Mark this struct as having been generated.
   if (!ObjCSynthesizedStructs.insert(CDecl))
-    assert(false && "struct already synthesize- SynthesizeObjCInternalStruct");
+    llvm_unreachable("struct already synthesize- SynthesizeObjCInternalStruct");
 }
 
 // RewriteObjCMethodsMetaData - Rewrite methods metadata for instance or
@@ -3358,8 +3447,8 @@ template<typename MethodIterator>
 void RewriteObjC::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
                                              MethodIterator MethodEnd,
                                              bool IsInstanceMethod,
-                                             llvm::StringRef prefix,
-                                             llvm::StringRef ClassName,
+                                             StringRef prefix,
+                                             StringRef ClassName,
                                              std::string &Result) {
   if (MethodBegin == MethodEnd) return;
 
@@ -3428,8 +3517,8 @@ void RewriteObjC::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
 
 /// RewriteObjCProtocolMetaData - Rewrite protocols meta-data.
 void RewriteObjC::
-RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, llvm::StringRef prefix,
-                            llvm::StringRef ClassName, std::string &Result) {
+RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, StringRef prefix,
+                            StringRef ClassName, std::string &Result) {
   static bool objc_protocol_methods = false;
 
   // Output struct protocol_methods holder of method selector and type.
@@ -3570,13 +3659,13 @@ RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, llvm::StringRef prefix,
 
   // Mark this protocol as having been generated.
   if (!ObjCSynthesizedProtocols.insert(PDecl))
-    assert(false && "protocol already synthesized");
+    llvm_unreachable("protocol already synthesized");
 
 }
 
 void RewriteObjC::
 RewriteObjCProtocolListMetaData(const ObjCList<ObjCProtocolDecl> &Protocols,
-                                llvm::StringRef prefix, llvm::StringRef ClassName,
+                                StringRef prefix, StringRef ClassName,
                                 std::string &Result) {
   if (Protocols.empty()) return;
 
@@ -3634,7 +3723,7 @@ void RewriteObjC::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
   FullCategoryName += IDecl->getNameAsString();
 
   // Build _objc_method_list for class's instance methods if needed
-  llvm::SmallVector<ObjCMethodDecl *, 32>
+  SmallVector<ObjCMethodDecl *, 32>
     InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
 
   // If any of our property implementations have associated getters or
@@ -3742,7 +3831,7 @@ void RewriteObjC::SynthesizeIvarOffsetComputation(ObjCIvarDecl *ivar,
   } else {
     Result += "__OFFSETOFIVAR__(struct ";
     Result += ivar->getContainingInterface()->getNameAsString();
-    if (LangOpts.Microsoft)
+    if (LangOpts.MicrosoftExt)
       Result += "_IMPL";
     Result += ", ";
     Result += ivar->getNameAsString();
@@ -3804,7 +3893,7 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
     Result += "\n";
 
     ObjCInterfaceDecl::ivar_iterator IVI, IVE;
-    llvm::SmallVector<ObjCIvarDecl *, 8> IVars;
+    SmallVector<ObjCIvarDecl *, 8> IVars;
     if (!IDecl->ivar_empty()) {
       for (ObjCInterfaceDecl::ivar_iterator
              IV = IDecl->ivar_begin(), IVEnd = IDecl->ivar_end();
@@ -3843,7 +3932,7 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
   }
 
   // Build _objc_method_list for class's instance methods if needed
-  llvm::SmallVector<ObjCMethodDecl *, 32>
+  SmallVector<ObjCMethodDecl *, 32>
     InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
 
   // If any of our property implementations have associated getters or
@@ -3986,7 +4075,7 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
     // class has size. Must synthesize its size.
     Result += ",sizeof(struct ";
     Result += CDecl->getNameAsString();
-    if (LangOpts.Microsoft)
+    if (LangOpts.MicrosoftExt)
       Result += "_IMPL";
     Result += ")";
   }
@@ -4104,7 +4193,7 @@ void RewriteObjC::SynthesizeMetaDataIntoBuffer(std::string &Result) {
   ", sizeof(struct _objc_module), \"\", &_OBJC_SYMBOLS\n";
   Result += "};\n\n";
 
-  if (LangOpts.Microsoft) {
+  if (LangOpts.MicrosoftExt) {
     if (ProtocolExprDecls.size()) {
       Result += "#pragma section(\".objc_protocol$B\",long,read,write)\n";
       Result += "#pragma data_seg(push, \".objc_protocol$B\")\n";
@@ -4144,12 +4233,12 @@ static bool HasLocalVariableExternalStorage(ValueDecl *VD) {
 }
 
 std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                                   llvm::StringRef funcName,
+                                                   StringRef funcName,
                                                    std::string Tag) {
   const FunctionType *AFT = CE->getFunctionType();
   QualType RT = AFT->getResultType();
   std::string StructRef = "struct " + Tag;
-  std::string S = "static " + RT.getAsString(Context->PrintingPolicy) + " __" +
+  std::string S = "static " + RT.getAsString(Context->getPrintingPolicy()) + " __" +
                   funcName.str() + "_" + "block_func_" + utostr(i);
 
   BlockDecl *BD = CE->getBlockDecl();
@@ -4173,9 +4262,9 @@ std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
       ParamStr = (*AI)->getNameAsString();
       QualType QT = (*AI)->getType();
       if (convertBlockPointerToFunctionPointer(QT))
-        QT.getAsStringInternal(ParamStr, Context->PrintingPolicy);
+        QT.getAsStringInternal(ParamStr, Context->getPrintingPolicy());
       else
-        QT.getAsStringInternal(ParamStr, Context->PrintingPolicy);      
+        QT.getAsStringInternal(ParamStr, Context->getPrintingPolicy());      
       S += ParamStr;
     }
     if (FT->isVariadic()) {
@@ -4188,7 +4277,7 @@ std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
 
   // Create local declarations to avoid rewriting all closure decl ref exprs.
   // First, emit a declaration for all "by ref" decls.
-  for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+  for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
        E = BlockByRefDecls.end(); I != E; ++I) {
     S += "  ";
     std::string Name = (*I)->getNameAsString();
@@ -4199,7 +4288,7 @@ std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
     S += Name + " = __cself->" + (*I)->getNameAsString() + "; // bound by ref\n";
   }
   // Next, emit a declaration for all "by copy" declarations.
-  for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+  for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
        E = BlockByCopyDecls.end(); I != E; ++I) {
     S += "  ";
     // Handle nested closure invocation. For example:
@@ -4224,7 +4313,7 @@ std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
       QualType QT = (*I)->getType();
       if (HasLocalVariableExternalStorage(*I))
         QT = Context->getPointerType(QT);
-      QT.getAsStringInternal(Name, Context->PrintingPolicy);
+      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
       S += Name + " = __cself->" + 
                               (*I)->getNameAsString() + "; // bound by copy\n";
     }
@@ -4238,7 +4327,7 @@ std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
 }
 
 std::string RewriteObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                                   llvm::StringRef funcName,
+                                                   StringRef funcName,
                                                    std::string Tag) {
   std::string StructRef = "struct " + Tag;
   std::string S = "static void __";
@@ -4250,12 +4339,15 @@ std::string RewriteObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
   S += "*src) {";
   for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
       E = ImportedBlockDecls.end(); I != E; ++I) {
+    ValueDecl *VD = (*I);
     S += "_Block_object_assign((void*)&dst->";
     S += (*I)->getNameAsString();
     S += ", (void*)src->";
     S += (*I)->getNameAsString();
     if (BlockByRefDeclsPtrSet.count((*I)))
       S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
+    else if (VD->getType()->isBlockPointerType())
+      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
     else
       S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
   }
@@ -4268,10 +4360,13 @@ std::string RewriteObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
   S += "*src) {";
   for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
       E = ImportedBlockDecls.end(); I != E; ++I) {
+    ValueDecl *VD = (*I);
     S += "_Block_object_dispose((void*)src->";
     S += (*I)->getNameAsString();
     if (BlockByRefDeclsPtrSet.count((*I)))
       S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
+    else if (VD->getType()->isBlockPointerType())
+      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
     else
       S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
   }
@@ -4294,7 +4389,7 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
 
   if (BlockDeclRefs.size()) {
     // Output all "by copy" declarations.
-    for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
          E = BlockByCopyDecls.end(); I != E; ++I) {
       S += "  ";
       std::string FieldName = (*I)->getNameAsString();
@@ -4316,14 +4411,14 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
         QualType QT = (*I)->getType();
         if (HasLocalVariableExternalStorage(*I))
           QT = Context->getPointerType(QT);
-        QT.getAsStringInternal(FieldName, Context->PrintingPolicy);
-        QT.getAsStringInternal(ArgName, Context->PrintingPolicy);
+        QT.getAsStringInternal(FieldName, Context->getPrintingPolicy());
+        QT.getAsStringInternal(ArgName, Context->getPrintingPolicy());
         Constructor += ", " + ArgName;
       }
       S += FieldName + ";\n";
     }
     // Output all "by ref" declarations.
-    for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
          E = BlockByRefDecls.end(); I != E; ++I) {
       S += "  ";
       std::string FieldName = (*I)->getNameAsString();
@@ -4342,7 +4437,7 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
     Constructor += ", int flags=0)";
     // Initialize all "by copy" arguments.
     bool firsTime = true;
-    for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
          E = BlockByCopyDecls.end(); I != E; ++I) {
       std::string Name = (*I)->getNameAsString();
         if (firsTime) {
@@ -4357,7 +4452,7 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
           Constructor += Name + "(_" + Name + ")";
     }
     // Initialize all "by ref" arguments.
-    for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
          E = BlockByRefDecls.end(); I != E; ++I) {
       std::string Name = (*I)->getNameAsString();
       if (firsTime) {
@@ -4396,7 +4491,7 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
 
 std::string RewriteObjC::SynthesizeBlockDescriptor(std::string DescTag, 
                                                    std::string ImplTag, int i,
-                                                   llvm::StringRef FunName,
+                                                   StringRef FunName,
                                                    unsigned hasCopy) {
   std::string S = "\nstatic struct " + DescTag;
   
@@ -4423,7 +4518,7 @@ std::string RewriteObjC::SynthesizeBlockDescriptor(std::string DescTag,
 }
 
 void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                          llvm::StringRef FunName) {
+                                          StringRef FunName) {
   // Insert declaration for the function in which block literal is used.
   if (CurFunctionDeclToDeclareForBlock && !Blocks.empty())
     RewriteBlockLiteralFunctionDecl(CurFunctionDeclToDeclareForBlock);
@@ -4512,7 +4607,7 @@ void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
 
 void RewriteObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
   SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
-  llvm::StringRef FuncName = FD->getName();
+  StringRef FuncName = FD->getName();
 
   SynthesizeBlockLiterals(FunLocStart, FuncName);
 }
@@ -4564,7 +4659,7 @@ void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
 }
 
 void RewriteObjC::GetInnerBlockDeclRefExprs(Stmt *S, 
-                llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs,
+                SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs,
                 llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts) {
   for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI) {
@@ -4603,7 +4698,7 @@ QualType RewriteObjC::convertFunctionTypeOfBlocks(const FunctionType *FT) {
   const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
   // FTP will be null for closures that don't take arguments.
   // Generate a funky cast.
-  llvm::SmallVector<QualType, 8> ArgTypes;
+  SmallVector<QualType, 8> ArgTypes;
   QualType Res = FT->getResultType();
   bool HasBlockType = convertBlockPointerToFunctionPointer(Res);
   
@@ -4673,7 +4768,7 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
   QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
 
   // Generate a funky cast.
-  llvm::SmallVector<QualType, 8> ArgTypes;
+  SmallVector<QualType, 8> ArgTypes;
 
   // Push the block argument type.
   ArgTypes.push_back(PtrBlock);
@@ -4716,7 +4811,7 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
                                                 CK_BitCast, ME);
   PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
 
-  llvm::SmallVector<Expr*, 8> BlkExprs;
+  SmallVector<Expr*, 8> BlkExprs;
   // Add the implicit argument.
   BlkExprs.push_back(BlkCast);
   // Add the user arguments.
@@ -4767,7 +4862,7 @@ Stmt *RewriteObjC::RewriteBlockDeclRefExpr(Expr *DeclRefExp) {
                                             FD->getType(), VK_LValue,
                                             OK_Ordinary);
 
-  llvm::StringRef Name = VD->getName();
+  StringRef Name = VD->getName();
   FD = FieldDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
                          &Context->Idents.get(Name), 
                          Context->VoidPtrTy, 0,
@@ -4835,7 +4930,7 @@ void RewriteObjC::RewriteCastExpr(CStyleCastExpr *CE) {
     switch (*argPtr) {
     case '^':
       // Replace the '^' with '*'.
-      LocStart = LocStart.getFileLocWithOffset(argPtr-startBuf);
+      LocStart = LocStart.getLocWithOffset(argPtr-startBuf);
       ReplaceText(LocStart, 1, "*");
       break;
     }
@@ -4855,7 +4950,7 @@ void RewriteObjC::RewriteBlockPointerFunctionArgs(FunctionDecl *FD) {
 
   parenCount++;
   // advance the location to startArgList.
-  DeclLoc = DeclLoc.getFileLocWithOffset(startArgList-startBuf);
+  DeclLoc = DeclLoc.getLocWithOffset(startArgList-startBuf);
   assert((DeclLoc.isValid()) && "Invalid DeclLoc");
 
   const char *argPtr = startArgList;
@@ -4864,7 +4959,7 @@ void RewriteObjC::RewriteBlockPointerFunctionArgs(FunctionDecl *FD) {
     switch (*argPtr) {
     case '^':
       // Replace the '^' with '*'.
-      DeclLoc = DeclLoc.getFileLocWithOffset(argPtr-startArgList);
+      DeclLoc = DeclLoc.getLocWithOffset(argPtr-startArgList);
       ReplaceText(DeclLoc, 1, "*");
       break;
     case '(':
@@ -4957,14 +5052,14 @@ void RewriteObjC::RewriteBlockPointerDecl(NamedDecl *ND) {
   else if (FieldDecl *FD = dyn_cast<FieldDecl>(ND))
     DeclT = FD->getType();
   else
-    assert(0 && "RewriteBlockPointerDecl(): Decl type not yet handled");
+    llvm_unreachable("RewriteBlockPointerDecl(): Decl type not yet handled");
 
   const char *startBuf = SM->getCharacterData(DeclLoc);
   const char *endBuf = startBuf;
   // scan backward (from the decl location) for the end of the previous decl.
   while (*startBuf != '^' && *startBuf != ';' && startBuf != MainFileStart)
     startBuf--;
-  SourceLocation Start = DeclLoc.getFileLocWithOffset(startBuf-endBuf);
+  SourceLocation Start = DeclLoc.getLocWithOffset(startBuf-endBuf);
   std::string buf;
   unsigned OrigLength=0;
   // *startBuf != '^' if we are dealing with a pointer to function that
@@ -5108,7 +5203,7 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
     DeclLoc = ND->getLocation();
   const char *startBuf = SM->getCharacterData(DeclLoc);
   SourceLocation X = ND->getLocEnd();
-  X = SM->getInstantiationLoc(X);
+  X = SM->getExpansionLoc(X);
   const char *endBuf = SM->getCharacterData(X);
   std::string Name(ND->getNameAsString());
   std::string ByrefType;
@@ -5130,7 +5225,7 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
 
   QualType T = Ty;
   (void)convertBlockPointerToFunctionPointer(T);
-  T.getAsStringInternal(Name, Context->PrintingPolicy);
+  T.getAsStringInternal(Name, Context->getPrintingPolicy());
     
   ByrefType += " " + Name + ";\n";
   ByrefType += "};\n";
@@ -5203,7 +5298,7 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
       startLoc = ECE->getLParenLoc();
     else
       startLoc = E->getLocStart();
-    startLoc = SM->getInstantiationLoc(startLoc);
+    startLoc = SM->getExpansionLoc(startLoc);
     endBuf = SM->getCharacterData(startLoc);
     ByrefType += " " + Name;
     ByrefType += " = {(void*)";
@@ -5235,7 +5330,7 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
     const char *semiBuf = strchr(startInitializerBuf, ';');
     assert((*semiBuf == ';') && "RewriteByRefVar: can't find ';'");
     SourceLocation semiLoc =
-      startLoc.getFileLocWithOffset(semiBuf-startInitializerBuf);
+      startLoc.getLocWithOffset(semiBuf-startInitializerBuf);
 
     InsertText(semiLoc, "}");
   }
@@ -5271,7 +5366,7 @@ void RewriteObjC::CollectBlockDeclRefInfo(BlockExpr *Exp) {
   }
 }
 
-FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(llvm::StringRef name) {
+FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(StringRef name) {
   IdentifierInfo *ID = &Context->Idents.get(name);
   QualType FType = Context->getFunctionNoProtoType(Context->VoidPtrTy);
   return FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
@@ -5280,7 +5375,7 @@ FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(llvm::StringRef name) {
 }
 
 Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
-          const llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs) {
+          const SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs) {
   const BlockDecl *block = Exp->getBlockDecl();
   Blocks.push_back(Exp);
 
@@ -5343,7 +5438,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
   DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, FType, VK_RValue,
                                                SourceLocation());
 
-  llvm::SmallVector<Expr*, 4> InitExprs;
+  SmallVector<Expr*, 4> InitExprs;
 
   // Initialize the block function.
   FD = SynthBlockInitFunctionDecl(Func);
@@ -5376,7 +5471,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
   if (BlockDeclRefs.size()) {
     Expr *Exp;
     // Output all "by copy" declarations.
-    for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
          E = BlockByCopyDecls.end(); I != E; ++I) {
       if (isObjCType((*I)->getType())) {
         // FIXME: Conform to ABI ([[obj retain] autorelease]).
@@ -5410,7 +5505,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
       InitExprs.push_back(Exp);
     }
     // Output all "by ref" declarations.
-    for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
          E = BlockByRefDecls.end(); I != E; ++I) {
       ValueDecl *ND = (*I);
       std::string Name(ND->getNameAsString());
@@ -5557,7 +5652,7 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
     }
 
   if (BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
-    llvm::SmallVector<BlockDeclRefExpr *, 8> InnerBlockDeclRefs;
+    SmallVector<BlockDeclRefExpr *, 8> InnerBlockDeclRefs;
     llvm::SmallPtrSet<const DeclContext *, 8> InnerContexts;
     InnerContexts.insert(BE->getBlockDecl());
     ImportedLocalExternalDecls.clear();
@@ -5568,12 +5663,20 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
     CurrentBody = BE->getBody();
     CollectPropertySetters(CurrentBody);
     PropParentMap = 0;
+    // block literal on rhs of a property-dot-sytax assignment
+    // must be replaced by its synthesize ast so getRewrittenText
+    // works as expected. In this case, what actually ends up on RHS
+    // is the blockTranscribed which is the helper function for the
+    // block literal; as in: self.c = ^() {[ace ARR];};
+    bool saveDisableReplaceStmt = DisableReplaceStmt;
+    DisableReplaceStmt = false;
     RewriteFunctionBodyOrGlobalInitializer(BE->getBody());
+    DisableReplaceStmt = saveDisableReplaceStmt;
     CurrentBody = SaveCurrentBody;
     PropParentMap = 0;
     ImportedLocalExternalDecls.clear();
     // Now we snarf the rewritten text and stash it away for later use.
-    std::string Str = Rewrite.ConvertToString(BE->getBody());
+    std::string Str = Rewrite.getRewrittenText(BE->getSourceRange());
     RewrittenBlockExprs[BE] = Str;
 
     Stmt *blockTranscribed = SynthBlockInitExpr(BE, InnerBlockDeclRefs);
@@ -5866,8 +5969,8 @@ void RewriteObjC::HandleDeclInMainFile(Decl *D) {
     ClassImplementation.push_back(CI);
   else if (ObjCCategoryImplDecl *CI = dyn_cast<ObjCCategoryImplDecl>(D))
     CategoryImplementation.push_back(CI);
-  else if (ObjCClassDecl *CD = dyn_cast<ObjCClassDecl>(D))
-    RewriteForwardClassDecl(CD);
+  else if (isa<ObjCClassDecl>(D))
+    llvm_unreachable("RewriteObjC::HandleDeclInMainFile - ObjCClassDecl");
   else if (VarDecl *VD = dyn_cast<VarDecl>(D)) {
     RewriteObjCQualifiedInterfaceTypes(VD);
     if (isTopLevelBlockPointerType(VD->getType()))
@@ -5881,7 +5984,7 @@ void RewriteObjC::HandleDeclInMainFile(Decl *D) {
       }
     } else if (VD->getType()->isRecordType()) {
       RecordDecl *RD = VD->getType()->getAs<RecordType>()->getDecl();
-      if (RD->isDefinition())
+      if (RD->isCompleteDefinition())
         RewriteRecordBody(RD);
     }
     if (VD->getInit()) {
@@ -5913,7 +6016,7 @@ void RewriteObjC::HandleDeclInMainFile(Decl *D) {
     return;
   }
   if (RecordDecl *RD = dyn_cast<RecordDecl>(D)) {
-    if (RD->isDefinition()) 
+    if (RD->isCompleteDefinition()) 
       RewriteRecordBody(RD);
     return;
   }
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/RewriteRope.cpp b/contrib/llvm/tools/clang/lib/Rewrite/RewriteRope.cpp
index cfedd4b41faf..6c211b28fd99 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/RewriteRope.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/RewriteRope.cpp
@@ -12,11 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Rewrite/RewriteRope.h"
-#include "llvm/Support/Casting.h"
+#include "clang/Basic/LLVM.h"
 #include <algorithm>
 using namespace clang;
-using llvm::dyn_cast;
-using llvm::cast;
 
 /// RewriteRope is a "strong" string class, designed to make insertions and
 /// deletions in the middle of the string nearly constant time (really, they are
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/RewriteTest.cpp b/contrib/llvm/tools/clang/lib/Rewrite/RewriteTest.cpp
index 36207000c905..c446324d3eb2 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/RewriteTest.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/RewriteTest.cpp
@@ -16,7 +16,7 @@
 #include "clang/Rewrite/TokenRewriter.h"
 #include "llvm/Support/raw_ostream.h"
 
-void clang::DoRewriteTest(Preprocessor &PP, llvm::raw_ostream* OS) {
+void clang::DoRewriteTest(Preprocessor &PP, raw_ostream* OS) {
   SourceManager &SM = PP.getSourceManager();
   const LangOptions &LangOpts = PP.getLangOptions();
 
diff --git a/contrib/llvm/tools/clang/lib/Rewrite/Rewriter.cpp b/contrib/llvm/tools/clang/lib/Rewrite/Rewriter.cpp
index 92f516074ec9..464b299ccdb0 100644
--- a/contrib/llvm/tools/clang/lib/Rewrite/Rewriter.cpp
+++ b/contrib/llvm/tools/clang/lib/Rewrite/Rewriter.cpp
@@ -17,10 +17,9 @@
 #include "clang/AST/Decl.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Basic/SourceManager.h"
-#include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
-llvm::raw_ostream &RewriteBuffer::write(llvm::raw_ostream &os) const {
+raw_ostream &RewriteBuffer::write(raw_ostream &os) const {
   // FIXME: eliminate the copy by writing out each chunk at a time
   os << std::string(begin(), end());
   return os;
@@ -84,7 +83,7 @@ void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
   }
 }
 
-void RewriteBuffer::InsertText(unsigned OrigOffset, llvm::StringRef Str,
+void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
                                bool InsertAfter) {
 
   // Nothing to insert, exit early.
@@ -101,7 +100,7 @@ void RewriteBuffer::InsertText(unsigned OrigOffset, llvm::StringRef Str,
 /// buffer with a new string.  This is effectively a combined "remove+insert"
 /// operation.
 void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                                llvm::StringRef NewStr) {
+                                StringRef NewStr) {
   unsigned RealOffset = getMappedOffset(OrigOffset, true);
   Buffer.erase(RealOffset, OrigLength);
   Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
@@ -222,7 +221,7 @@ RewriteBuffer &Rewriter::getEditBuffer(FileID FID) {
     return I->second;
   I = RewriteBuffers.insert(I, std::make_pair(FID, RewriteBuffer()));
 
-  llvm::StringRef MB = SourceMgr->getBufferData(FID);
+  StringRef MB = SourceMgr->getBufferData(FID);
   I->second.Initialize(MB.begin(), MB.end());
 
   return I->second;
@@ -230,10 +229,8 @@ RewriteBuffer &Rewriter::getEditBuffer(FileID FID) {
 
 /// InsertText - Insert the specified string at the specified location in the
 /// original buffer.
-bool Rewriter::InsertText(SourceLocation Loc, llvm::StringRef Str,
+bool Rewriter::InsertText(SourceLocation Loc, StringRef Str,
                           bool InsertAfter, bool indentNewLines) {
-  using llvm::StringRef;
-
   if (!isRewritable(Loc)) return true;
   FileID FID;
   unsigned StartOffs = getLocationOffsetAndFileID(Loc, FID);
@@ -256,7 +253,7 @@ bool Rewriter::InsertText(SourceLocation Loc, llvm::StringRef Str,
       indentSpace = MB.substr(lineOffs, i-lineOffs);
     }
 
-    llvm::SmallVector<StringRef, 4> lines;
+    SmallVector<StringRef, 4> lines;
     Str.split(lines, "\n");
 
     for (unsigned i = 0, e = lines.size(); i != e; ++i) {
@@ -273,7 +270,7 @@ bool Rewriter::InsertText(SourceLocation Loc, llvm::StringRef Str,
   return false;
 }
 
-bool Rewriter::InsertTextAfterToken(SourceLocation Loc, llvm::StringRef Str) {
+bool Rewriter::InsertTextAfterToken(SourceLocation Loc, StringRef Str) {
   if (!isRewritable(Loc)) return true;
   FileID FID;
   unsigned StartOffs = getLocationOffsetAndFileID(Loc, FID);
@@ -298,7 +295,7 @@ bool Rewriter::RemoveText(SourceLocation Start, unsigned Length,
 /// buffer with a new string.  This is effectively a combined "remove/insert"
 /// operation.
 bool Rewriter::ReplaceText(SourceLocation Start, unsigned OrigLength,
-                           llvm::StringRef NewStr) {
+                           StringRef NewStr) {
   if (!isRewritable(Start)) return true;
   FileID StartFileID;
   unsigned StartOffs = getLocationOffsetAndFileID(Start, StartFileID);
@@ -317,7 +314,7 @@ bool Rewriter::ReplaceText(SourceRange range, SourceRange replacementRange) {
   FileID FID;
   unsigned newOffs = getLocationOffsetAndFileID(replacementRange.getBegin(),
                                                 FID);
-  llvm::StringRef MB = SourceMgr->getBufferData(FID);
+  StringRef MB = SourceMgr->getBufferData(FID);
   return ReplaceText(start, origLength, MB.substr(newOffs, newLength));
 }
 
@@ -349,8 +346,6 @@ std::string Rewriter::ConvertToString(Stmt *From) {
 
 bool Rewriter::IncreaseIndentation(CharSourceRange range,
                                    SourceLocation parentIndent) {
-  using llvm::StringRef;
-
   if (range.isInvalid()) return true;
   if (!isRewritable(range.getBegin())) return true;
   if (!isRewritable(range.getEnd())) return true;
@@ -400,7 +395,7 @@ bool Rewriter::IncreaseIndentation(CharSourceRange range,
   if (!startSpace.startswith(parentSpace))
     return true;
 
-  llvm::StringRef indent = startSpace.substr(parentSpace.size());
+  StringRef indent = startSpace.substr(parentSpace.size());
 
   // Indent the lines between start/end offsets.
   RewriteBuffer &RB = getEditBuffer(FID);
diff --git a/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp b/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 7a14855e69cd..babb8af18b37 100644
--- a/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -17,6 +17,7 @@
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclCXX.h"
@@ -25,14 +26,23 @@
 #include "clang/AST/StmtObjC.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/EvaluatedExprVisitor.h"
+#include "clang/AST/StmtVisitor.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/CFG.h"
 #include "clang/Analysis/Analyses/ReachableCode.h"
 #include "clang/Analysis/Analyses/CFGReachabilityAnalysis.h"
+#include "clang/Analysis/Analyses/ThreadSafety.h"
 #include "clang/Analysis/CFGStmtMap.h"
 #include "clang/Analysis/Analyses/UninitializedValues.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/ImmutableMap.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
+#include <algorithm>
+#include <vector>
 
 using namespace clang;
 
@@ -86,7 +96,7 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
   // The CFG leaves in dead things, and we don't want the dead code paths to
   // confuse us, so we mark all live things first.
   llvm::BitVector live(cfg->getNumBlockIDs());
-  unsigned count = reachable_code::ScanReachableFromBlock(cfg->getEntry(),
+  unsigned count = reachable_code::ScanReachableFromBlock(&cfg->getEntry(),
                                                           live);
 
   bool AddEHEdges = AC.getAddEHEdges();
@@ -101,7 +111,7 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
           if (b.getTerminator() && isa<CXXTryStmt>(b.getTerminator()))
             // When not adding EH edges from calls, catch clauses
             // can otherwise seem dead.  Avoid noting them as dead.
-            count += reachable_code::ScanReachableFromBlock(b, live);
+            count += reachable_code::ScanReachableFromBlock(&b, live);
           continue;
         }
       }
@@ -126,31 +136,23 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
     if (!live[B.getBlockID()])
       continue;
 
+    // Skip blocks which contain an element marked as no-return. They don't
+    // represent actually viable edges into the exit block, so mark them as
+    // abnormal.
+    if (B.hasNoReturnElement()) {
+      HasAbnormalEdge = true;
+      continue;
+    }
+
     // Destructors can appear after the 'return' in the CFG.  This is
     // normal.  We need to look pass the destructors for the return
     // statement (if it exists).
     CFGBlock::const_reverse_iterator ri = B.rbegin(), re = B.rend();
-    bool hasNoReturnDtor = false;
-    
-    for ( ; ri != re ; ++ri) {
-      CFGElement CE = *ri;
 
-      // FIXME: The right solution is to just sever the edges in the
-      // CFG itself.
-      if (const CFGImplicitDtor *iDtor = ri->getAs<CFGImplicitDtor>())
-        if (iDtor->isNoReturn(AC.getASTContext())) {
-          hasNoReturnDtor = true;
-          HasFakeEdge = true;
-          break;
-        }
-      
-      if (isa<CFGStmt>(CE))
+    for ( ; ri != re ; ++ri)
+      if (isa<CFGStmt>(*ri))
         break;
-    }
-    
-    if (hasNoReturnDtor)
-      continue;
-    
+
     // No more CFGElements in the block?
     if (ri == re) {
       if (B.getTerminator() && isa<CXXTryStmt>(B.getTerminator())) {
@@ -163,7 +165,7 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
     }
 
     CFGStmt CS = cast<CFGStmt>(*ri);
-    Stmt *S = CS.getStmt();
+    const Stmt *S = CS.getStmt();
     if (isa<ReturnStmt>(S)) {
       HasLiveReturn = true;
       continue;
@@ -187,34 +189,13 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
       HasAbnormalEdge = true;
       continue;
     }
-
-    bool NoReturnEdge = false;
-    if (CallExpr *C = dyn_cast<CallExpr>(S)) {
-      if (std::find(B.succ_begin(), B.succ_end(), &cfg->getExit())
-            == B.succ_end()) {
-        HasAbnormalEdge = true;
-        continue;
-      }
-      Expr *CEE = C->getCallee()->IgnoreParenCasts();
-      QualType calleeType = CEE->getType();
-      if (calleeType == AC.getASTContext().BoundMemberTy) {
-        calleeType = Expr::findBoundMemberType(CEE);
-        assert(!calleeType.isNull() && "analyzing unresolved call?");
-      }
-      if (getFunctionExtInfo(calleeType).getNoReturn()) {
-        NoReturnEdge = true;
-        HasFakeEdge = true;
-      } else if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CEE)) {
-        ValueDecl *VD = DRE->getDecl();
-        if (VD->hasAttr<NoReturnAttr>()) {
-          NoReturnEdge = true;
-          HasFakeEdge = true;
-        }
-      }
+    if (std::find(B.succ_begin(), B.succ_end(), &cfg->getExit())
+        == B.succ_end()) {
+      HasAbnormalEdge = true;
+      continue;
     }
-    // FIXME: Add noreturn message sends.
-    if (NoReturnEdge == false)
-      HasPlainEdge = true;
+
+    HasPlainEdge = true;
   }
   if (!HasPlainEdge) {
     if (HasLiveReturn)
@@ -258,7 +239,23 @@ struct CheckFallThroughDiagnostics {
     if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Func))
       isVirtualMethod = Method->isVirtual();
     
-    if (!isVirtualMethod)
+    // Don't suggest that template instantiations be marked "noreturn"
+    bool isTemplateInstantiation = false;
+    if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(Func)) {
+      switch (Function->getTemplateSpecializationKind()) {
+      case TSK_Undeclared:
+      case TSK_ExplicitSpecialization:
+        break;
+        
+      case TSK_ImplicitInstantiation:
+      case TSK_ExplicitInstantiationDeclaration:
+      case TSK_ExplicitInstantiationDefinition:
+        isTemplateInstantiation = true;
+        break;
+      }
+    }
+        
+    if (!isVirtualMethod && !isTemplateInstantiation)
       D.diag_NeverFallThroughOrReturn =
         diag::warn_suggest_noreturn_function;
     else
@@ -284,25 +281,25 @@ struct CheckFallThroughDiagnostics {
     return D;
   }
 
-  bool checkDiagnostics(Diagnostic &D, bool ReturnsVoid,
+  bool checkDiagnostics(DiagnosticsEngine &D, bool ReturnsVoid,
                         bool HasNoReturn) const {
     if (funMode) {
       return (ReturnsVoid ||
               D.getDiagnosticLevel(diag::warn_maybe_falloff_nonvoid_function,
-                                   FuncLoc) == Diagnostic::Ignored)
+                                   FuncLoc) == DiagnosticsEngine::Ignored)
         && (!HasNoReturn ||
             D.getDiagnosticLevel(diag::warn_noreturn_function_has_return_expr,
-                                 FuncLoc) == Diagnostic::Ignored)
+                                 FuncLoc) == DiagnosticsEngine::Ignored)
         && (!ReturnsVoid ||
             D.getDiagnosticLevel(diag::warn_suggest_noreturn_block, FuncLoc)
-              == Diagnostic::Ignored);
+              == DiagnosticsEngine::Ignored);
     }
 
     // For blocks.
     return  ReturnsVoid && !HasNoReturn
             && (!ReturnsVoid ||
                 D.getDiagnosticLevel(diag::warn_suggest_noreturn_block, FuncLoc)
-                  == Diagnostic::Ignored);
+                  == DiagnosticsEngine::Ignored);
   }
 };
 
@@ -340,7 +337,7 @@ static void CheckFallThroughForBody(Sema &S, const Decl *D, const Stmt *Body,
     }
   }
 
-  Diagnostic &Diags = S.getDiagnostics();
+  DiagnosticsEngine &Diags = S.getDiagnostics();
 
   // Short circuit for compilation speed.
   if (CD.checkDiagnostics(Diags, ReturnsVoid, HasNoReturn))
@@ -369,9 +366,17 @@ static void CheckFallThroughForBody(Sema &S, const Decl *D, const Stmt *Body,
                  CD.diag_AlwaysFallThrough_ReturnsNonVoid);
         break;
       case NeverFallThroughOrReturn:
-        if (ReturnsVoid && !HasNoReturn && CD.diag_NeverFallThroughOrReturn)
-          S.Diag(Compound->getLBracLoc(),
-                 CD.diag_NeverFallThroughOrReturn);
+        if (ReturnsVoid && !HasNoReturn && CD.diag_NeverFallThroughOrReturn) {
+          if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+            S.Diag(Compound->getLBracLoc(), CD.diag_NeverFallThroughOrReturn)
+              << 0 << FD;
+          } else if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D)) {
+            S.Diag(Compound->getLBracLoc(), CD.diag_NeverFallThroughOrReturn)
+              << 1 << MD;
+          } else {
+            S.Diag(Compound->getLBracLoc(), CD.diag_NeverFallThroughOrReturn);
+          }
+        }
         break;
       case NeverFallThrough:
         break;
@@ -415,13 +420,58 @@ class ContainsReference : public EvaluatedExprVisitor<ContainsReference> {
 };
 }
 
+static bool SuggestInitializationFixit(Sema &S, const VarDecl *VD) {
+  // Don't issue a fixit if there is already an initializer.
+  if (VD->getInit())
+    return false;
+
+  // Suggest possible initialization (if any).
+  const char *initialization = 0;
+  QualType VariableTy = VD->getType().getCanonicalType();
+
+  if (VariableTy->isObjCObjectPointerType() ||
+      VariableTy->isBlockPointerType()) {
+    // Check if 'nil' is defined.
+    if (S.PP.getMacroInfo(&S.getASTContext().Idents.get("nil")))
+      initialization = " = nil";
+    else
+      initialization = " = 0";
+  }
+  else if (VariableTy->isRealFloatingType())
+    initialization = " = 0.0";
+  else if (VariableTy->isBooleanType() && S.Context.getLangOptions().CPlusPlus)
+    initialization = " = false";
+  else if (VariableTy->isEnumeralType())
+    return false;
+  else if (VariableTy->isPointerType() || VariableTy->isMemberPointerType()) {
+    if (S.Context.getLangOptions().CPlusPlus0x)
+      initialization = " = nullptr";
+    // Check if 'NULL' is defined.
+    else if (S.PP.getMacroInfo(&S.getASTContext().Idents.get("NULL")))
+      initialization = " = NULL";
+    else
+      initialization = " = 0";
+  }
+  else if (VariableTy->isScalarType())
+    initialization = " = 0";
+
+  if (initialization) {
+    SourceLocation loc = S.PP.getLocForEndOfToken(VD->getLocEnd());
+    S.Diag(loc, diag::note_var_fixit_add_initialization) << VD->getDeclName()
+      << FixItHint::CreateInsertion(loc, initialization);
+    return true;
+  }
+  return false;
+}
+
 /// DiagnoseUninitializedUse -- Helper function for diagnosing uses of an
 /// uninitialized variable. This manages the different forms of diagnostic
 /// emitted for particular types of uses. Returns true if the use was diagnosed
 /// as a warning. If a pariticular use is one we omit warnings for, returns
 /// false.
 static bool DiagnoseUninitializedUse(Sema &S, const VarDecl *VD,
-                                     const Expr *E, bool isAlwaysUninit) {
+                                     const Expr *E, bool isAlwaysUninit,
+                                     bool alwaysReportSelfInit = false) {
   bool isSelfInit = false;
 
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
@@ -441,7 +491,7 @@ static bool DiagnoseUninitializedUse(Sema &S, const VarDecl *VD,
       // TODO: Should we suppress maybe-uninitialized warnings for
       // variables initialized in this way?
       if (const Expr *Initializer = VD->getInit()) {
-        if (DRE == Initializer->IgnoreParenImpCasts())
+        if (!alwaysReportSelfInit && DRE == Initializer->IgnoreParenImpCasts())
           return false;
 
         ContainsReference CR(S.Context, DRE);
@@ -469,54 +519,15 @@ static bool DiagnoseUninitializedUse(Sema &S, const VarDecl *VD,
   }
 
   // Report where the variable was declared when the use wasn't within
-  // the initializer of that declaration.
-  if (!isSelfInit)
+  // the initializer of that declaration & we didn't already suggest
+  // an initialization fixit.
+  if (!isSelfInit && !SuggestInitializationFixit(S, VD))
     S.Diag(VD->getLocStart(), diag::note_uninit_var_def)
       << VD->getDeclName();
 
   return true;
 }
 
-static void SuggestInitializationFixit(Sema &S, const VarDecl *VD) {
-  // Don't issue a fixit if there is already an initializer.
-  if (VD->getInit())
-    return;
-
-  // Suggest possible initialization (if any).
-  const char *initialization = 0;
-  QualType VariableTy = VD->getType().getCanonicalType();
-
-  if (VariableTy->isObjCObjectPointerType() ||
-      VariableTy->isBlockPointerType()) {
-    // Check if 'nil' is defined.
-    if (S.PP.getMacroInfo(&S.getASTContext().Idents.get("nil")))
-      initialization = " = nil";
-    else
-      initialization = " = 0";
-  }
-  else if (VariableTy->isRealFloatingType())
-    initialization = " = 0.0";
-  else if (VariableTy->isBooleanType() && S.Context.getLangOptions().CPlusPlus)
-    initialization = " = false";
-  else if (VariableTy->isEnumeralType())
-    return;
-  else if (VariableTy->isPointerType() || VariableTy->isMemberPointerType()) {
-    // Check if 'NULL' is defined.
-    if (S.PP.getMacroInfo(&S.getASTContext().Idents.get("NULL")))
-      initialization = " = NULL";
-    else
-      initialization = " = 0";
-  }
-  else if (VariableTy->isScalarType())
-    initialization = " = 0";
-
-  if (initialization) {
-    SourceLocation loc = S.PP.getLocForEndOfToken(VD->getLocEnd());
-    S.Diag(loc, diag::note_var_fixit_add_initialization)
-      << FixItHint::CreateInsertion(loc, initialization);
-  }
-}
-
 typedef std::pair<const Expr*, bool> UninitUse;
 
 namespace {
@@ -530,8 +541,8 @@ struct SLocSort {
 
 class UninitValsDiagReporter : public UninitVariablesHandler {
   Sema &S;
-  typedef llvm::SmallVector<UninitUse, 2> UsesVec;
-  typedef llvm::DenseMap<const VarDecl *, UsesVec*> UsesMap;
+  typedef SmallVector<UninitUse, 2> UsesVec;
+  typedef llvm::DenseMap<const VarDecl *, std::pair<UsesVec*, bool> > UsesMap;
   UsesMap *uses;
   
 public:
@@ -539,17 +550,26 @@ class UninitValsDiagReporter : public UninitVariablesHandler {
   ~UninitValsDiagReporter() { 
     flushDiagnostics();
   }
-  
-  void handleUseOfUninitVariable(const Expr *ex, const VarDecl *vd,
-                                 bool isAlwaysUninit) {
+
+  std::pair<UsesVec*, bool> &getUses(const VarDecl *vd) {
     if (!uses)
       uses = new UsesMap();
-    
-    UsesVec *&vec = (*uses)[vd];
+
+    UsesMap::mapped_type &V = (*uses)[vd];
+    UsesVec *&vec = V.first;
     if (!vec)
       vec = new UsesVec();
     
-    vec->push_back(std::make_pair(ex, isAlwaysUninit));
+    return V;
+  }
+  
+  void handleUseOfUninitVariable(const Expr *ex, const VarDecl *vd,
+                                 bool isAlwaysUninit) {
+    getUses(vd).first->push_back(std::make_pair(ex, isAlwaysUninit));
+  }
+  
+  void handleSelfInit(const VarDecl *vd) {
+    getUses(vd).second = true;    
   }
   
   void flushDiagnostics() {
@@ -558,28 +578,34 @@ class UninitValsDiagReporter : public UninitVariablesHandler {
     
     for (UsesMap::iterator i = uses->begin(), e = uses->end(); i != e; ++i) {
       const VarDecl *vd = i->first;
-      UsesVec *vec = i->second;
+      const UsesMap::mapped_type &V = i->second;
 
-      bool fixitIssued = false;
-            
-      // Sort the uses by their SourceLocations.  While not strictly
-      // guaranteed to produce them in line/column order, this will provide
-      // a stable ordering.
-      std::sort(vec->begin(), vec->end(), SLocSort());
-      
-      for (UsesVec::iterator vi = vec->begin(), ve = vec->end(); vi != ve;
-           ++vi) {
-        if (!DiagnoseUninitializedUse(S, vd, vi->first,
-                                      /*isAlwaysUninit=*/vi->second))
-          continue;
+      UsesVec *vec = V.first;
+      bool hasSelfInit = V.second;
 
-        // Suggest a fixit hint the first time we diagnose a use of a variable.
-        if (!fixitIssued) {
-          SuggestInitializationFixit(S, vd);
-          fixitIssued = true;
+      // Specially handle the case where we have uses of an uninitialized 
+      // variable, but the root cause is an idiomatic self-init.  We want
+      // to report the diagnostic at the self-init since that is the root cause.
+      if (!vec->empty() && hasSelfInit)
+        DiagnoseUninitializedUse(S, vd, vd->getInit()->IgnoreParenCasts(),
+                                 true, /* alwaysReportSelfInit */ true);
+      else {
+        // Sort the uses by their SourceLocations.  While not strictly
+        // guaranteed to produce them in line/column order, this will provide
+        // a stable ordering.
+        std::sort(vec->begin(), vec->end(), SLocSort());
+        
+        for (UsesVec::iterator vi = vec->begin(), ve = vec->end(); vi != ve;
+             ++vi) {
+          if (DiagnoseUninitializedUse(S, vd, vi->first,
+                                        /*isAlwaysUninit=*/vi->second))
+            // Skip further diagnostics for this variable. We try to warn only
+            // on the first point at which a variable is used uninitialized.
+            break;
         }
       }
-
+      
+      // Release the uses vector.
       delete vec;
     }
     delete uses;
@@ -587,6 +613,132 @@ class UninitValsDiagReporter : public UninitVariablesHandler {
 };
 }
 
+
+//===----------------------------------------------------------------------===//
+// -Wthread-safety
+//===----------------------------------------------------------------------===//
+namespace clang {
+namespace thread_safety {
+typedef std::pair<SourceLocation, PartialDiagnostic> DelayedDiag;
+typedef llvm::SmallVector<DelayedDiag, 4> DiagList;
+
+struct SortDiagBySourceLocation {
+  Sema &S;
+  SortDiagBySourceLocation(Sema &S) : S(S) {}
+
+  bool operator()(const DelayedDiag &left, const DelayedDiag &right) {
+    // Although this call will be slow, this is only called when outputting
+    // multiple warnings.
+    return S.getSourceManager().isBeforeInTranslationUnit(left.first,
+                                                          right.first);
+  }
+};
+
+class ThreadSafetyReporter : public clang::thread_safety::ThreadSafetyHandler {
+  Sema &S;
+  DiagList Warnings;
+
+  // Helper functions
+  void warnLockMismatch(unsigned DiagID, Name LockName, SourceLocation Loc) {
+    PartialDiagnostic Warning = S.PDiag(DiagID) << LockName;
+    Warnings.push_back(DelayedDiag(Loc, Warning));
+  }
+
+ public:
+  ThreadSafetyReporter(Sema &S) : S(S) {}
+
+  /// \brief Emit all buffered diagnostics in order of sourcelocation.
+  /// We need to output diagnostics produced while iterating through
+  /// the lockset in deterministic order, so this function orders diagnostics
+  /// and outputs them.
+  void emitDiagnostics() {
+    SortDiagBySourceLocation SortDiagBySL(S);
+    sort(Warnings.begin(), Warnings.end(), SortDiagBySL);
+    for (DiagList::iterator I = Warnings.begin(), E = Warnings.end();
+         I != E; ++I)
+      S.Diag(I->first, I->second);
+  }
+
+  void handleInvalidLockExp(SourceLocation Loc) {
+    PartialDiagnostic Warning = S.PDiag(diag::warn_cannot_resolve_lock) << Loc;
+    Warnings.push_back(DelayedDiag(Loc, Warning));
+  }
+  void handleUnmatchedUnlock(Name LockName, SourceLocation Loc) {
+    warnLockMismatch(diag::warn_unlock_but_no_lock, LockName, Loc);
+  }
+
+  void handleDoubleLock(Name LockName, SourceLocation Loc) {
+    warnLockMismatch(diag::warn_double_lock, LockName, Loc);
+  }
+
+  void handleMutexHeldEndOfScope(Name LockName, SourceLocation Loc,
+                                 LockErrorKind LEK){
+    unsigned DiagID = 0;
+    switch (LEK) {
+      case LEK_LockedSomePredecessors:
+        DiagID = diag::warn_lock_at_end_of_scope;
+        break;
+      case LEK_LockedSomeLoopIterations:
+        DiagID = diag::warn_expecting_lock_held_on_loop;
+        break;
+      case LEK_LockedAtEndOfFunction:
+        DiagID = diag::warn_no_unlock;
+        break;
+    }
+    warnLockMismatch(DiagID, LockName, Loc);
+  }
+
+
+  void handleExclusiveAndShared(Name LockName, SourceLocation Loc1,
+                                SourceLocation Loc2) {
+    PartialDiagnostic Warning =
+      S.PDiag(diag::warn_lock_exclusive_and_shared) << LockName;
+    PartialDiagnostic Note =
+      S.PDiag(diag::note_lock_exclusive_and_shared) << LockName;
+    Warnings.push_back(DelayedDiag(Loc1, Warning));
+    Warnings.push_back(DelayedDiag(Loc2, Note));
+  }
+
+  void handleNoMutexHeld(const NamedDecl *D, ProtectedOperationKind POK,
+                         AccessKind AK, SourceLocation Loc) {
+    assert((POK == POK_VarAccess || POK == POK_VarDereference)
+             && "Only works for variables");
+    unsigned DiagID = POK == POK_VarAccess?
+                        diag::warn_variable_requires_any_lock:
+                        diag::warn_var_deref_requires_any_lock;
+    PartialDiagnostic Warning = S.PDiag(DiagID)
+      << D->getName() << getLockKindFromAccessKind(AK);
+    Warnings.push_back(DelayedDiag(Loc, Warning));
+  }
+
+  void handleMutexNotHeld(const NamedDecl *D, ProtectedOperationKind POK,
+                          Name LockName, LockKind LK, SourceLocation Loc) {
+    unsigned DiagID = 0;
+    switch (POK) {
+      case POK_VarAccess:
+        DiagID = diag::warn_variable_requires_lock;
+        break;
+      case POK_VarDereference:
+        DiagID = diag::warn_var_deref_requires_lock;
+        break;
+      case POK_FunctionCall:
+        DiagID = diag::warn_fun_requires_lock;
+        break;
+    }
+    PartialDiagnostic Warning = S.PDiag(DiagID)
+      << D->getName() << LockName << LK;
+    Warnings.push_back(DelayedDiag(Loc, Warning));
+  }
+
+  void handleFunExcludesLock(Name FunName, Name LockName, SourceLocation Loc) {
+    PartialDiagnostic Warning =
+      S.PDiag(diag::warn_fun_excludes_mutex) << FunName << LockName;
+    Warnings.push_back(DelayedDiag(Loc, Warning));
+  }
+};
+}
+}
+
 //===----------------------------------------------------------------------===//
 // AnalysisBasedWarnings - Worker object used by Sema to execute analysis-based
 //  warnings on a function, method, or block.
@@ -595,6 +747,7 @@ class UninitValsDiagReporter : public UninitVariablesHandler {
 clang::sema::AnalysisBasedWarnings::Policy::Policy() {
   enableCheckFallThrough = 1;
   enableCheckUnreachable = 0;
+  enableThreadSafetyAnalysis = 0;
 }
 
 clang::sema::AnalysisBasedWarnings::AnalysisBasedWarnings(Sema &s)
@@ -608,14 +761,18 @@ clang::sema::AnalysisBasedWarnings::AnalysisBasedWarnings(Sema &s)
     MaxUninitAnalysisVariablesPerFunction(0),
     NumUninitAnalysisBlockVisits(0),
     MaxUninitAnalysisBlockVisitsPerFunction(0) {
-  Diagnostic &D = S.getDiagnostics();
+  DiagnosticsEngine &D = S.getDiagnostics();
   DefaultPolicy.enableCheckUnreachable = (unsigned)
     (D.getDiagnosticLevel(diag::warn_unreachable, SourceLocation()) !=
-        Diagnostic::Ignored);
+        DiagnosticsEngine::Ignored);
+  DefaultPolicy.enableThreadSafetyAnalysis = (unsigned)
+    (D.getDiagnosticLevel(diag::warn_double_lock, SourceLocation()) !=
+     DiagnosticsEngine::Ignored);
+
 }
 
 static void flushDiagnostics(Sema &S, sema::FunctionScopeInfo *fscope) {
-  for (llvm::SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
+  for (SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
        i = fscope->PossiblyUnreachableDiags.begin(),
        e = fscope->PossiblyUnreachableDiags.end();
        i != e; ++i) {
@@ -635,7 +792,7 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   //     don't bother trying.
   // (2) The code already has problems; running the analysis just takes more
   //     time.
-  Diagnostic &Diags = S.getDiagnostics();
+  DiagnosticsEngine &Diags = S.getDiagnostics();
 
   // Do not do any analysis for declarations in system headers if we are
   // going to just ignore them.
@@ -656,17 +813,43 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   const Stmt *Body = D->getBody();
   assert(Body);
 
+  AnalysisContext AC(D, 0);
+
   // Don't generate EH edges for CallExprs as we'd like to avoid the n^2
   // explosion for destrutors that can result and the compile time hit.
-  AnalysisContext AC(D, 0, /*useUnoptimizedCFG=*/false, /*addehedges=*/false,
-                     /*addImplicitDtors=*/true, /*addInitializers=*/true);
+  AC.getCFGBuildOptions().PruneTriviallyFalseEdges = true;
+  AC.getCFGBuildOptions().AddEHEdges = false;
+  AC.getCFGBuildOptions().AddInitializers = true;
+  AC.getCFGBuildOptions().AddImplicitDtors = true;
+  
+  // Force that certain expressions appear as CFGElements in the CFG.  This
+  // is used to speed up various analyses.
+  // FIXME: This isn't the right factoring.  This is here for initial
+  // prototyping, but we need a way for analyses to say what expressions they
+  // expect to always be CFGElements and then fill in the BuildOptions
+  // appropriately.  This is essentially a layering violation.
+  if (P.enableCheckUnreachable) {
+    // Unreachable code analysis requires a linearized CFG.
+    AC.getCFGBuildOptions().setAllAlwaysAdd();
+  }
+  else {
+    AC.getCFGBuildOptions()
+      .setAlwaysAdd(Stmt::BinaryOperatorClass)
+      .setAlwaysAdd(Stmt::BlockExprClass)
+      .setAlwaysAdd(Stmt::CStyleCastExprClass)
+      .setAlwaysAdd(Stmt::DeclRefExprClass)
+      .setAlwaysAdd(Stmt::ImplicitCastExprClass)
+      .setAlwaysAdd(Stmt::UnaryOperatorClass);
+  }
 
+  // Construct the analysis context with the specified CFG build options.
+  
   // Emit delayed diagnostics.
   if (!fscope->PossiblyUnreachableDiags.empty()) {
     bool analyzed = false;
 
     // Register the expressions with the CFGBuilder.
-    for (llvm::SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
+    for (SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
          i = fscope->PossiblyUnreachableDiags.begin(),
          e = fscope->PossiblyUnreachableDiags.end();
          i != e; ++i) {
@@ -676,7 +859,7 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
 
     if (AC.getCFG()) {
       analyzed = true;
-      for (llvm::SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
+      for (SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
             i = fscope->PossiblyUnreachableDiags.begin(),
             e = fscope->PossiblyUnreachableDiags.end();
             i != e; ++i)
@@ -716,11 +899,18 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   // Warning: check for unreachable code
   if (P.enableCheckUnreachable)
     CheckUnreachable(S, AC);
-  
+
+  // Check for thread safety violations
+  if (P.enableThreadSafetyAnalysis) {
+    thread_safety::ThreadSafetyReporter Reporter(S);
+    thread_safety::runThreadSafetyAnalysis(AC, Reporter);
+    Reporter.emitDiagnostics();
+  }
+
   if (Diags.getDiagnosticLevel(diag::warn_uninit_var, D->getLocStart())
-      != Diagnostic::Ignored ||
+      != DiagnosticsEngine::Ignored ||
       Diags.getDiagnosticLevel(diag::warn_maybe_uninit_var, D->getLocStart())
-      != Diagnostic::Ignored) {
+      != DiagnosticsEngine::Ignored) {
     if (CFG *cfg = AC.getCFG()) {
       UninitValsDiagReporter reporter(S);
       UninitVariablesAnalysisStats stats;
diff --git a/contrib/llvm/tools/clang/lib/Sema/AttributeList.cpp b/contrib/llvm/tools/clang/lib/Sema/AttributeList.cpp
index 5a8330bbfd77..13a0edec28d6 100644
--- a/contrib/llvm/tools/clang/lib/Sema/AttributeList.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/AttributeList.cpp
@@ -98,7 +98,7 @@ AttributePool::createIntegerAttribute(ASTContext &C, IdentifierInfo *Name,
 }
 
 AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
-  llvm::StringRef AttrName = Name->getName();
+  StringRef AttrName = Name->getName();
 
   // Normalize the attribute name, __foo__ becomes foo.
   if (AttrName.startswith("__") && AttrName.endswith("__"))
@@ -159,10 +159,11 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("address_space", AT_address_space)
     .Case("opencl_image_access", AT_opencl_image_access)
     .Case("always_inline", AT_always_inline)
-    .Case("returns_twice", IgnoredAttribute)
+    .Case("returns_twice", AT_returns_twice)
     .Case("vec_type_hint", IgnoredAttribute)
     .Case("objc_exception", AT_objc_exception)
     .Case("objc_method_family", AT_objc_method_family)
+    .Case("objc_returns_inner_pointer", AT_objc_returns_inner_pointer)
     .Case("ext_vector_type", AT_ext_vector_type)
     .Case("neon_vector_type", AT_neon_vector_type)
     .Case("neon_polyvector_type", AT_neon_polyvector_type)
@@ -170,15 +171,18 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("analyzer_noreturn", AT_analyzer_noreturn)
     .Case("warn_unused_result", AT_warn_unused_result)
     .Case("carries_dependency", AT_carries_dependency)
+    .Case("ns_bridged", AT_ns_bridged)
     .Case("ns_consumed", AT_ns_consumed)
     .Case("ns_consumes_self", AT_ns_consumes_self)
     .Case("ns_returns_autoreleased", AT_ns_returns_autoreleased)
     .Case("ns_returns_not_retained", AT_ns_returns_not_retained)
     .Case("ns_returns_retained", AT_ns_returns_retained)
+    .Case("cf_audited_transfer", AT_cf_audited_transfer)
     .Case("cf_consumed", AT_cf_consumed)
     .Case("cf_returns_not_retained", AT_cf_returns_not_retained)
     .Case("cf_returns_retained", AT_cf_returns_retained)
     .Case("cf_returns_autoreleased", AT_cf_returns_autoreleased)
+    .Case("cf_unknown_transfer", AT_cf_unknown_transfer)
     .Case("ns_consumes_self", AT_ns_consumes_self)
     .Case("ns_consumed", AT_ns_consumed)
     .Case("objc_ownership", AT_objc_ownership)
@@ -209,5 +213,23 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("uuid", AT_uuid)
     .Case("pcs", AT_pcs)
     .Case("ms_struct", AT_MsStruct)
+    .Case("guarded_var", AT_guarded_var)
+    .Case("pt_guarded_var", AT_pt_guarded_var)
+    .Case("scoped_lockable", AT_scoped_lockable)
+    .Case("lockable", AT_lockable)
+    .Case("no_thread_safety_analysis", AT_no_thread_safety_analysis)
+    .Case("guarded_by", AT_guarded_by)
+    .Case("pt_guarded_by", AT_pt_guarded_by)
+    .Case("acquired_after", AT_acquired_after)
+    .Case("acquired_before", AT_acquired_before)
+    .Case("exclusive_lock_function", AT_exclusive_lock_function)
+    .Case("exclusive_locks_required", AT_exclusive_locks_required)
+    .Case("exclusive_trylock_function", AT_exclusive_trylock_function)
+    .Case("lock_returned", AT_lock_returned)
+    .Case("locks_excluded", AT_locks_excluded)
+    .Case("shared_lock_function", AT_shared_lock_function)
+    .Case("shared_locks_required", AT_shared_locks_required)
+    .Case("shared_trylock_function", AT_shared_trylock_function)
+    .Case("unlock_function", AT_unlock_function)
     .Default(UnknownAttribute);
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/CodeCompleteConsumer.cpp b/contrib/llvm/tools/clang/lib/Sema/CodeCompleteConsumer.cpp
index d7dc5b2538c6..da9860361bc3 100644
--- a/contrib/llvm/tools/clang/lib/Sema/CodeCompleteConsumer.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/CodeCompleteConsumer.cpp
@@ -26,7 +26,6 @@
 #include <functional>
 
 using namespace clang;
-using llvm::StringRef;
 
 //===----------------------------------------------------------------------===//
 // Code completion context implementation
@@ -68,7 +67,7 @@ bool CodeCompletionContext::wantConstructorResults() const {
   case CCC_OtherWithMacros:
   case CCC_ObjCInstanceMessage:
   case CCC_ObjCClassMessage:
-  case CCC_ObjCSuperclass:
+  case CCC_ObjCInterfaceName:
   case CCC_ObjCCategoryName:
     return false;
   }
@@ -191,14 +190,36 @@ CodeCompletionString::Chunk::CreateCurrentParameter(
 CodeCompletionString::CodeCompletionString(const Chunk *Chunks, 
                                            unsigned NumChunks,
                                            unsigned Priority, 
-                                           CXAvailabilityKind Availability) 
-  : NumChunks(NumChunks), Priority(Priority), Availability(Availability) 
+                                           CXAvailabilityKind Availability,
+                                           const char **Annotations,
+                                           unsigned NumAnnotations)
+  : NumChunks(NumChunks), NumAnnotations(NumAnnotations)
+  , Priority(Priority), Availability(Availability)
 { 
+  assert(NumChunks <= 0xffff);
+  assert(NumAnnotations <= 0xffff);
+
   Chunk *StoredChunks = reinterpret_cast<Chunk *>(this + 1);
   for (unsigned I = 0; I != NumChunks; ++I)
     StoredChunks[I] = Chunks[I];
+
+  const char **StoredAnnotations = reinterpret_cast<const char **>(StoredChunks + NumChunks);
+  for (unsigned I = 0; I != NumAnnotations; ++I)
+    StoredAnnotations[I] = Annotations[I];
 }
 
+unsigned CodeCompletionString::getAnnotationCount() const {
+  return NumAnnotations;
+}
+
+const char *CodeCompletionString::getAnnotation(unsigned AnnotationNr) const {
+  if (AnnotationNr < NumAnnotations)
+    return reinterpret_cast<const char * const*>(end())[AnnotationNr];
+  else
+    return 0;
+}
+
+
 std::string CodeCompletionString::getAsString() const {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
@@ -228,14 +249,14 @@ const char *CodeCompletionString::getTypedText() const {
   return 0;
 }
 
-const char *CodeCompletionAllocator::CopyString(llvm::StringRef String) {
+const char *CodeCompletionAllocator::CopyString(StringRef String) {
   char *Mem = (char *)Allocate(String.size() + 1, 1);
   std::copy(String.begin(), String.end(), Mem);
   Mem[String.size()] = 0;
   return Mem;
 }
 
-const char *CodeCompletionAllocator::CopyString(llvm::Twine String) {
+const char *CodeCompletionAllocator::CopyString(Twine String) {
   // FIXME: It would be more efficient to teach Twine to tell us its size and
   // then add a routine there to fill in an allocated char* with the contents
   // of the string.
@@ -245,11 +266,13 @@ const char *CodeCompletionAllocator::CopyString(llvm::Twine String) {
 
 CodeCompletionString *CodeCompletionBuilder::TakeString() {
   void *Mem = Allocator.Allocate(
-                  sizeof(CodeCompletionString) + sizeof(Chunk) * Chunks.size(), 
+                  sizeof(CodeCompletionString) + sizeof(Chunk) * Chunks.size()
+                                    + sizeof(const char *) * Annotations.size(),
                                  llvm::alignOf<CodeCompletionString>());
   CodeCompletionString *Result 
     = new (Mem) CodeCompletionString(Chunks.data(), Chunks.size(),
-                               Priority, Availability);
+                                     Priority, Availability,
+                                     Annotations.data(), Annotations.size());
   Chunks.clear();
   return Result;
 }
@@ -329,7 +352,7 @@ PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
     OS << "COMPLETION: ";
     switch (Results[I].Kind) {
     case CodeCompletionResult::RK_Declaration:
-      OS << Results[I].Declaration;
+      OS << *Results[I].Declaration;
       if (Results[I].Hidden)
         OS << " (Hidden)";
       if (CodeCompletionString *CCS 
@@ -377,7 +400,7 @@ PrintingCodeCompleteConsumer::ProcessOverloadCandidates(Sema &SemaRef,
   }
 }
 
-void CodeCompletionResult::computeCursorKindAndAvailability() {
+void CodeCompletionResult::computeCursorKindAndAvailability(bool Accessible) {
   switch (Kind) {
   case RK_Declaration:
     // Set the availability based on attributes.
@@ -419,13 +442,16 @@ void CodeCompletionResult::computeCursorKindAndAvailability() {
     // Do nothing: Patterns can come with cursor kinds!
     break;
   }
+
+  if (!Accessible)
+    Availability = CXAvailability_NotAccessible;
 }
 
 /// \brief Retrieve the name that should be used to order a result.
 ///
 /// If the name needs to be constructed as a string, that string will be
 /// saved into Saved and the returned StringRef will refer to it.
-static llvm::StringRef getOrderedName(const CodeCompletionResult &R,
+static StringRef getOrderedName(const CodeCompletionResult &R,
                                     std::string &Saved) {
   switch (R.Kind) {
     case CodeCompletionResult::RK_Keyword:
@@ -460,8 +486,8 @@ static llvm::StringRef getOrderedName(const CodeCompletionResult &R,
 bool clang::operator<(const CodeCompletionResult &X, 
                       const CodeCompletionResult &Y) {
   std::string XSaved, YSaved;
-  llvm::StringRef XStr = getOrderedName(X, XSaved);
-  llvm::StringRef YStr = getOrderedName(Y, YSaved);
+  StringRef XStr = getOrderedName(X, XSaved);
+  StringRef YStr = getOrderedName(Y, YSaved);
   int cmp = XStr.compare_lower(YStr);
   if (cmp)
     return cmp < 0;
diff --git a/contrib/llvm/tools/clang/lib/Sema/DeclSpec.cpp b/contrib/llvm/tools/clang/lib/Sema/DeclSpec.cpp
index c87f2cff5489..f0a763e4ecfd 100644
--- a/contrib/llvm/tools/clang/lib/Sema/DeclSpec.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/DeclSpec.cpp
@@ -15,6 +15,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/LocInfoType.h"
 #include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/Sema.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/NestedNameSpecifier.h"
@@ -27,7 +28,7 @@
 using namespace clang;
 
 
-static DiagnosticBuilder Diag(Diagnostic &D, SourceLocation Loc,
+static DiagnosticBuilder Diag(DiagnosticsEngine &D, SourceLocation Loc,
                               unsigned DiagID) {
   return D.Report(Loc, DiagID);
 }
@@ -242,6 +243,7 @@ bool Declarator::isDeclarationOfFunction() const {
   }
   
   switch (DS.getTypeSpecType()) {
+    case TST_atomic:
     case TST_auto:
     case TST_bool:
     case TST_char:
@@ -255,6 +257,7 @@ bool Declarator::isDeclarationOfFunction() const {
     case TST_enum:
     case TST_error:
     case TST_float:
+    case TST_half:
     case TST_int:
     case TST_struct:
     case TST_union:
@@ -371,6 +374,7 @@ const char *DeclSpec::getSpecifierName(DeclSpec::TST T) {
   case DeclSpec::TST_char16:      return "char16_t";
   case DeclSpec::TST_char32:      return "char32_t";
   case DeclSpec::TST_int:         return "int";
+  case DeclSpec::TST_half:        return "half";
   case DeclSpec::TST_float:       return "float";
   case DeclSpec::TST_double:      return "double";
   case DeclSpec::TST_bool:        return "_Bool";
@@ -388,6 +392,7 @@ const char *DeclSpec::getSpecifierName(DeclSpec::TST T) {
   case DeclSpec::TST_decltype:    return "(decltype)";
   case DeclSpec::TST_underlyingType: return "__underlying_type";
   case DeclSpec::TST_unknown_anytype: return "__unknown_anytype";
+  case DeclSpec::TST_atomic: return "_Atomic";
   case DeclSpec::TST_error:       return "(error)";
   }
   llvm_unreachable("Unknown typespec!");
@@ -403,21 +408,24 @@ const char *DeclSpec::getSpecifierName(TQ T) {
   llvm_unreachable("Unknown typespec!");
 }
 
-bool DeclSpec::SetStorageClassSpec(SCS S, SourceLocation Loc,
+bool DeclSpec::SetStorageClassSpec(Sema &S, SCS SC, SourceLocation Loc,
                                    const char *&PrevSpec,
-                                   unsigned &DiagID,
-                                   const LangOptions &Lang) {
-  // OpenCL prohibits extern, auto, register, and static
+                                   unsigned &DiagID) {
+  // OpenCL 1.1 6.8g: "The extern, static, auto and register storage-class
+  // specifiers are not supported."
   // It seems sensible to prohibit private_extern too
-  if (Lang.OpenCL) {
-    switch (S) {
+  // The cl_clang_storage_class_specifiers extension enables support for
+  // these storage-class specifiers.
+  if (S.getLangOptions().OpenCL &&
+      !S.getOpenCLOptions().cl_clang_storage_class_specifiers) {
+    switch (SC) {
     case SCS_extern:
     case SCS_private_extern:
     case SCS_auto:
     case SCS_register:
     case SCS_static:
       DiagID   = diag::err_not_opencl_storage_class_specifier;
-      PrevSpec = getSpecifierName(S);
+      PrevSpec = getSpecifierName(SC);
       return true;
     default:
       break;
@@ -425,17 +433,30 @@ bool DeclSpec::SetStorageClassSpec(SCS S, SourceLocation Loc,
   }
 
   if (StorageClassSpec != SCS_unspecified) {
+    // Maybe this is an attempt to use C++0x 'auto' outside of C++0x mode.
+    bool isInvalid = true;
+    if (TypeSpecType == TST_unspecified && S.getLangOptions().CPlusPlus) {
+      if (SC == SCS_auto)
+        return SetTypeSpecType(TST_auto, Loc, PrevSpec, DiagID);
+      if (StorageClassSpec == SCS_auto) {
+        isInvalid = SetTypeSpecType(TST_auto, StorageClassSpecLoc,
+                                    PrevSpec, DiagID);
+        assert(!isInvalid && "auto SCS -> TST recovery failed");
+      }
+    }
+
     // Changing storage class is allowed only if the previous one
     // was the 'extern' that is part of a linkage specification and
     // the new storage class is 'typedef'.
-    if (!(SCS_extern_in_linkage_spec &&
+    if (isInvalid &&
+        !(SCS_extern_in_linkage_spec &&
           StorageClassSpec == SCS_extern &&
-          S == SCS_typedef))
-      return BadSpecifier(S, (SCS)StorageClassSpec, PrevSpec, DiagID);
+          SC == SCS_typedef))
+      return BadSpecifier(SC, (SCS)StorageClassSpec, PrevSpec, DiagID);
   }
-  StorageClassSpec = S;
+  StorageClassSpec = SC;
   StorageClassSpecLoc = Loc;
-  assert((unsigned)S == StorageClassSpec && "SCS constants overflow bitfield");
+  assert((unsigned)SC == StorageClassSpec && "SCS constants overflow bitfield");
   return false;
 }
 
@@ -637,7 +658,7 @@ bool DeclSpec::SetTypeQual(TQ T, SourceLocation Loc, const char *&PrevSpec,
   TypeQualifiers |= T;
 
   switch (T) {
-  default: assert(0 && "Unknown type qualifier!");
+  default: llvm_unreachable("Unknown type qualifier!");
   case TQ_const:    TQ_constLoc = Loc; break;
   case TQ_restrict: TQ_restrictLoc = Loc; break;
   case TQ_volatile: TQ_volatileLoc = Loc; break;
@@ -682,6 +703,18 @@ bool DeclSpec::SetFriendSpec(SourceLocation Loc, const char *&PrevSpec,
   return false;
 }
 
+bool DeclSpec::setModulePrivateSpec(SourceLocation Loc, const char *&PrevSpec,
+                                    unsigned &DiagID) {
+  if (isModulePrivateSpecified()) {
+    PrevSpec = "__module_private__";
+    DiagID = diag::ext_duplicate_declspec;
+    return true;
+  }
+  
+  ModulePrivateLoc = Loc;
+  return false;
+}
+
 bool DeclSpec::SetConstexprSpec(SourceLocation Loc, const char *&PrevSpec,
                                 unsigned &DiagID) {
   // 'constexpr constexpr' is ok.
@@ -732,7 +765,7 @@ void DeclSpec::SaveStorageSpecifierAsWritten() {
 /// "_Imaginary" (lacking an FP type).  This returns a diagnostic to issue or
 /// diag::NUM_DIAGNOSTICS if there is no error.  After calling this method,
 /// DeclSpec is guaranteed self-consistent, even if an error occurred.
-void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
+void DeclSpec::Finish(DiagnosticsEngine &D, Preprocessor &PP) {
   // Before possibly changing their values, save specs as written.
   SaveWrittenBuiltinSpecs();
   SaveStorageSpecifierAsWritten();
@@ -836,6 +869,27 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
     }
   }
 
+  // If no type specifier was provided and we're parsing a language where
+  // the type specifier is not optional, but we got 'auto' as a storage
+  // class specifier, then assume this is an attempt to use C++0x's 'auto'
+  // type specifier.
+  // FIXME: Does Microsoft really support implicit int in C++?
+  if (PP.getLangOptions().CPlusPlus && !PP.getLangOptions().MicrosoftExt &&
+      TypeSpecType == TST_unspecified && StorageClassSpec == SCS_auto) {
+    TypeSpecType = TST_auto;
+    StorageClassSpec = StorageClassSpecAsWritten = SCS_unspecified;
+    TSTLoc = TSTNameLoc = StorageClassSpecLoc;
+    StorageClassSpecLoc = SourceLocation();
+  }
+  // Diagnose if we've recovered from an ill-formed 'auto' storage class
+  // specifier in a pre-C++0x dialect of C++.
+  if (!PP.getLangOptions().CPlusPlus0x && TypeSpecType == TST_auto)
+    Diag(D, TSTLoc, diag::ext_auto_type_specifier);
+  if (PP.getLangOptions().CPlusPlus && !PP.getLangOptions().CPlusPlus0x &&
+      StorageClassSpec == SCS_auto)
+    Diag(D, StorageClassSpecLoc, diag::warn_auto_storage_class)
+      << FixItHint::CreateRemoval(StorageClassSpecLoc);
+
   // C++ [class.friend]p6:
   //   No storage-class-specifier shall appear in the decl-specifier-seq
   //   of a friend declaration.
@@ -844,7 +898,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
     const char *SpecName = getSpecifierName(SC);
 
     SourceLocation SCLoc = getStorageClassSpecLoc();
-    SourceLocation SCEndLoc = SCLoc.getFileLocWithOffset(strlen(SpecName));
+    SourceLocation SCEndLoc = SCLoc.getLocWithOffset(strlen(SpecName));
 
     Diag(D, SCLoc, diag::err_friend_storage_spec)
       << SpecName
@@ -854,7 +908,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
   }
 
   assert(!TypeSpecOwned || isDeclRep((TST) TypeSpecType));
-
+ 
   // Okay, now we can infer the real type.
 
   // TODO: return "auto function" and other bad things based on the real type.
@@ -902,7 +956,7 @@ bool VirtSpecifiers::SetSpecifier(Specifier VS, SourceLocation Loc,
   Specifiers |= VS;
 
   switch (VS) {
-  default: assert(0 && "Unknown specifier!");
+  default: llvm_unreachable("Unknown specifier!");
   case VS_Override: VS_overrideLoc = Loc; break;
   case VS_Final:    VS_finalLoc = Loc; break;
   }
@@ -912,7 +966,7 @@ bool VirtSpecifiers::SetSpecifier(Specifier VS, SourceLocation Loc,
 
 const char *VirtSpecifiers::getSpecifierName(Specifier VS) {
   switch (VS) {
-  default: assert(0 && "Unknown specifier");
+  default: llvm_unreachable("Unknown specifier");
   case VS_Override: return "override";
   case VS_Final: return "final";
   }
diff --git a/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp b/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp
index c6744ed80d4b..d6c1ad13a854 100644
--- a/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp
@@ -21,7 +21,7 @@ using namespace sema;
 
 DelayedDiagnostic DelayedDiagnostic::makeDeprecation(SourceLocation Loc,
                                                      const NamedDecl *D,
-                                                     llvm::StringRef Msg) {
+                                                     StringRef Msg) {
   DelayedDiagnostic DD;
   DD.Kind = Deprecation;
   DD.Triggered = false;
diff --git a/contrib/llvm/tools/clang/lib/Sema/IdentifierResolver.cpp b/contrib/llvm/tools/clang/lib/Sema/IdentifierResolver.cpp
index 95420a316ad0..a6432670856a 100644
--- a/contrib/llvm/tools/clang/lib/Sema/IdentifierResolver.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/IdentifierResolver.cpp
@@ -73,7 +73,7 @@ void IdentifierResolver::IdDeclInfo::RemoveDecl(NamedDecl *D) {
     }
   }
 
-  assert(0 && "Didn't find this decl on its identifier's chain!");
+  llvm_unreachable("Didn't find this decl on its identifier's chain!");
 }
 
 bool
diff --git a/contrib/llvm/tools/clang/lib/Sema/JumpDiagnostics.cpp b/contrib/llvm/tools/clang/lib/Sema/JumpDiagnostics.cpp
index 59bc2637c8f3..3e74dfccbdb1 100644
--- a/contrib/llvm/tools/clang/lib/Sema/JumpDiagnostics.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/JumpDiagnostics.cpp
@@ -1,4 +1,4 @@
-//===--- JumpDiagnostics.cpp - Analyze Jump Targets for VLA issues --------===//
+//===--- JumpDiagnostics.cpp - Protected scope jump analysis ------*- C++ -*-=//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements the JumpScopeChecker class, which is used to diagnose
-// jumps that enter a VLA scope in an invalid way.
+// jumps that enter a protected scope in an invalid way.
 //
 //===----------------------------------------------------------------------===//
 
@@ -58,12 +58,12 @@ class JumpScopeChecker {
       : ParentScope(parentScope), InDiag(InDiag), OutDiag(OutDiag), Loc(L) {}
   };
 
-  llvm::SmallVector<GotoScope, 48> Scopes;
+  SmallVector<GotoScope, 48> Scopes;
   llvm::DenseMap<Stmt*, unsigned> LabelAndGotoScopes;
-  llvm::SmallVector<Stmt*, 16> Jumps;
+  SmallVector<Stmt*, 16> Jumps;
 
-  llvm::SmallVector<IndirectGotoStmt*, 4> IndirectJumps;
-  llvm::SmallVector<LabelDecl*, 4> IndirectJumpTargets;
+  SmallVector<IndirectGotoStmt*, 4> IndirectJumps;
+  SmallVector<LabelDecl*, 4> IndirectJumpTargets;
 public:
   JumpScopeChecker(Stmt *Body, Sema &S);
 private:
@@ -76,8 +76,8 @@ class JumpScopeChecker {
   void VerifyIndirectJumps();
   void DiagnoseIndirectJump(IndirectGotoStmt *IG, unsigned IGScope,
                             LabelDecl *Target, unsigned TargetScope);
-  void CheckJump(Stmt *From, Stmt *To,
-                 SourceLocation DiagLoc, unsigned JumpDiag);
+  void CheckJump(Stmt *From, Stmt *To, SourceLocation DiagLoc,
+                 unsigned JumpDiag, unsigned JumpDiagWarning);
 
   unsigned GetDeepestCommonScope(unsigned A, unsigned B);
 };
@@ -476,7 +476,8 @@ void JumpScopeChecker::VerifyJumps() {
     // With a goto,
     if (GotoStmt *GS = dyn_cast<GotoStmt>(Jump)) {
       CheckJump(GS, GS->getLabel()->getStmt(), GS->getGotoLoc(),
-                diag::err_goto_into_protected_scope);
+                diag::err_goto_into_protected_scope,
+                diag::warn_goto_into_protected_scope);
       continue;
     }
 
@@ -484,7 +485,8 @@ void JumpScopeChecker::VerifyJumps() {
     if (IndirectGotoStmt *IGS = dyn_cast<IndirectGotoStmt>(Jump)) {
       LabelDecl *Target = IGS->getConstantTarget();
       CheckJump(IGS, Target->getStmt(), IGS->getGotoLoc(),
-                diag::err_goto_into_protected_scope);
+                diag::err_goto_into_protected_scope,
+                diag::warn_goto_into_protected_scope);
       continue;
     }
 
@@ -493,7 +495,7 @@ void JumpScopeChecker::VerifyJumps() {
          SC = SC->getNextSwitchCase()) {
       assert(LabelAndGotoScopes.count(SC) && "Case not visited?");
       CheckJump(SS, SC, SC->getLocStart(),
-                diag::err_switch_into_protected_scope);
+                diag::err_switch_into_protected_scope, 0);
     }
   }
 }
@@ -532,10 +534,10 @@ void JumpScopeChecker::VerifyIndirectJumps() {
   // indirect goto.  For most code bases, this substantially cuts
   // down on the number of jump sites we'll have to consider later.
   typedef std::pair<unsigned, IndirectGotoStmt*> JumpScope;
-  llvm::SmallVector<JumpScope, 32> JumpScopes;
+  SmallVector<JumpScope, 32> JumpScopes;
   {
     llvm::DenseMap<unsigned, IndirectGotoStmt*> JumpScopesMap;
-    for (llvm::SmallVectorImpl<IndirectGotoStmt*>::iterator
+    for (SmallVectorImpl<IndirectGotoStmt*>::iterator
            I = IndirectJumps.begin(), E = IndirectJumps.end(); I != E; ++I) {
       IndirectGotoStmt *IG = *I;
       assert(LabelAndGotoScopes.count(IG) &&
@@ -554,7 +556,7 @@ void JumpScopeChecker::VerifyIndirectJumps() {
   // label whose address was taken somewhere in the function.
   // For most code bases, there will be only one such scope.
   llvm::DenseMap<unsigned, LabelDecl*> TargetScopes;
-  for (llvm::SmallVectorImpl<LabelDecl*>::iterator
+  for (SmallVectorImpl<LabelDecl*>::iterator
          I = IndirectJumpTargets.begin(), E = IndirectJumpTargets.end();
        I != E; ++I) {
     LabelDecl *TheLabel = *I;
@@ -599,7 +601,7 @@ void JumpScopeChecker::VerifyIndirectJumps() {
 
     // Walk through all the jump sites, checking that they can trivially
     // reach this label scope.
-    for (llvm::SmallVectorImpl<JumpScope>::iterator
+    for (SmallVectorImpl<JumpScope>::iterator
            I = JumpScopes.begin(), E = JumpScopes.end(); I != E; ++I) {
       unsigned Scope = I->first;
 
@@ -660,10 +662,20 @@ void JumpScopeChecker::DiagnoseIndirectJump(IndirectGotoStmt *Jump,
       S.Diag(Scopes[I].Loc, Scopes[I].InDiag);
 }
 
+/// Return true if a particular error+note combination must be downgraded
+/// to a warning in Microsoft mode.
+static bool IsMicrosoftJumpWarning(unsigned JumpDiag, unsigned InDiagNote)
+{
+    return (JumpDiag == diag::err_goto_into_protected_scope && 
+           (InDiagNote == diag::note_protected_by_variable_init || 
+            InDiagNote == diag::note_protected_by_variable_nontriv_destructor));
+}
+
+
 /// CheckJump - Validate that the specified jump statement is valid: that it is
 /// jumping within or out of its current scope, not into a deeper one.
-void JumpScopeChecker::CheckJump(Stmt *From, Stmt *To,
-                                 SourceLocation DiagLoc, unsigned JumpDiag) {
+void JumpScopeChecker::CheckJump(Stmt *From, Stmt *To, SourceLocation DiagLoc,
+                             unsigned JumpDiagError, unsigned JumpDiagWarning) {
   assert(LabelAndGotoScopes.count(From) && "Jump didn't get added to scopes?");
   unsigned FromScope = LabelAndGotoScopes[From];
 
@@ -679,19 +691,30 @@ void JumpScopeChecker::CheckJump(Stmt *From, Stmt *To,
   if (CommonScope == ToScope) return;
 
   // Pull out (and reverse) any scopes we might need to diagnose skipping.
-  llvm::SmallVector<unsigned, 10> ToScopes;
-  for (unsigned I = ToScope; I != CommonScope; I = Scopes[I].ParentScope)
-    if (Scopes[I].InDiag)
-      ToScopes.push_back(I);
+  SmallVector<unsigned, 10> ToScopesError;
+  SmallVector<unsigned, 10> ToScopesWarning;
+  for (unsigned I = ToScope; I != CommonScope; I = Scopes[I].ParentScope) {
+    if (S.getLangOptions().MicrosoftMode && JumpDiagWarning != 0 &&
+        IsMicrosoftJumpWarning(JumpDiagError, Scopes[I].InDiag))
+      ToScopesWarning.push_back(I);
+    else if (Scopes[I].InDiag)
+      ToScopesError.push_back(I);
+  }
 
-  // If the only scopes present are cleanup scopes, we're okay.
-  if (ToScopes.empty()) return;
+  // Handle warnings.
+  if (!ToScopesWarning.empty()) {
+    S.Diag(DiagLoc, JumpDiagWarning);
+    for (unsigned i = 0, e = ToScopesWarning.size(); i != e; ++i)
+      S.Diag(Scopes[ToScopesWarning[i]].Loc, Scopes[ToScopesWarning[i]].InDiag);
+  }
 
-  S.Diag(DiagLoc, JumpDiag);
-
-  // Emit diagnostics for whatever is left in ToScopes.
-  for (unsigned i = 0, e = ToScopes.size(); i != e; ++i)
-    S.Diag(Scopes[ToScopes[i]].Loc, Scopes[ToScopes[i]].InDiag);
+  // Handle errors.
+  if (!ToScopesError.empty()) {
+    S.Diag(DiagLoc, JumpDiagError);
+    // Emit diagnostics note for whatever is left in ToScopesError.
+    for (unsigned i = 0, e = ToScopesError.size(); i != e; ++i)
+      S.Diag(Scopes[ToScopesError[i]].Loc, Scopes[ToScopesError[i]].InDiag);
+  }
 }
 
 void Sema::DiagnoseInvalidJumps(Stmt *Body) {
diff --git a/contrib/llvm/tools/clang/lib/Sema/MultiInitializer.cpp b/contrib/llvm/tools/clang/lib/Sema/MultiInitializer.cpp
new file mode 100644
index 000000000000..8bd2213b5705
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Sema/MultiInitializer.cpp
@@ -0,0 +1,92 @@
+//===--- MultiInitializer.cpp - Initializer expression group ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MultiInitializer class, which can represent a list
+// initializer or a parentheses-wrapped group of expressions in a C++ member
+// initializer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Sema/MultiInitializer.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Sema.h"
+#include "clang/AST/Expr.h"
+
+using namespace clang;
+
+InitListExpr *MultiInitializer::getInitList() const {
+  return cast<InitListExpr>(InitListOrExpressions.get<Expr*>());
+}
+
+SourceLocation MultiInitializer::getStartLoc() const {
+  return isInitializerList() ? getInitList()->getLBraceLoc() : LParenLoc;
+}
+
+SourceLocation MultiInitializer::getEndLoc() const {
+  return isInitializerList() ? getInitList()->getRBraceLoc() : RParenLoc;
+}
+
+MultiInitializer::iterator MultiInitializer::begin() const {
+  return isInitializerList() ? getInitList()->getInits() : getExpressions();
+}
+
+MultiInitializer::iterator MultiInitializer::end() const {
+  if (isInitializerList()) {
+    InitListExpr *ILE = getInitList();
+    return ILE->getInits() + ILE->getNumInits();
+  }
+  return getExpressions() + NumInitializers;
+}
+
+bool MultiInitializer::isTypeDependent() const {
+  if (isInitializerList())
+    return getInitList()->isTypeDependent();
+  for (iterator I = begin(), E = end(); I != E; ++I) {
+    if ((*I)->isTypeDependent())
+      return true;
+  }
+  return false;
+}
+
+bool MultiInitializer::DiagnoseUnexpandedParameterPack(Sema &SemaRef) const {
+  if (isInitializerList())
+    return SemaRef.DiagnoseUnexpandedParameterPack(getInitList());
+  for (iterator I = begin(), E = end(); I != E; ++I) {
+    if (SemaRef.DiagnoseUnexpandedParameterPack(*I))
+      return true;
+  }
+  return false;
+}
+
+Expr *MultiInitializer::CreateInitExpr(ASTContext &Ctx, QualType T) const {
+  if (isInitializerList())
+    return InitListOrExpressions.get<Expr*>();
+
+  return new (Ctx) ParenListExpr(Ctx, LParenLoc, getExpressions(),
+                                 NumInitializers, RParenLoc, T);
+}
+
+ExprResult MultiInitializer::PerformInit(Sema &SemaRef,
+                                         InitializedEntity Entity,
+                                         InitializationKind Kind) const {
+  Expr *Single;
+  Expr **Args;
+  unsigned NumArgs;
+  if (isInitializerList()) {
+    Single = InitListOrExpressions.get<Expr*>();
+    Args = &Single;
+    NumArgs = 1;
+  } else {
+    Args = getExpressions();
+    NumArgs = NumInitializers;
+  }
+  InitializationSequence InitSeq(SemaRef, Entity, Kind, Args, NumArgs);
+  return InitSeq.Perform(SemaRef, Entity, Kind,
+                         MultiExprArg(SemaRef, Args, NumArgs), 0);
+}
diff --git a/contrib/llvm/tools/clang/lib/Sema/Sema.cpp b/contrib/llvm/tools/clang/lib/Sema/Sema.cpp
index fdf3bb3cb012..533b21cb43e9 100644
--- a/contrib/llvm/tools/clang/lib/Sema/Sema.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/Sema.cpp
@@ -34,6 +34,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/FileManager.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
 using namespace clang;
@@ -54,99 +55,54 @@ void FunctionScopeInfo::Clear() {
 
 BlockScopeInfo::~BlockScopeInfo() { }
 
+PrintingPolicy Sema::getPrintingPolicy() const {
+  PrintingPolicy Policy = Context.getPrintingPolicy();
+  Policy.Bool = getLangOptions().Bool;
+  if (!Policy.Bool) {
+    if (MacroInfo *BoolMacro = PP.getMacroInfo(&Context.Idents.get("bool"))) {
+      Policy.Bool = BoolMacro->isObjectLike() && 
+        BoolMacro->getNumTokens() == 1 &&
+        BoolMacro->getReplacementToken(0).is(tok::kw__Bool);
+    }
+  }
+  
+  return Policy;
+}
+
 void Sema::ActOnTranslationUnitScope(Scope *S) {
   TUScope = S;
   PushDeclContext(S, Context.getTranslationUnitDecl());
 
   VAListTagName = PP.getIdentifierInfo("__va_list_tag");
 
-  if (!Context.isInt128Installed() && // May be set by ASTReader.
-      PP.getTargetInfo().getPointerWidth(0) >= 64) {
-    TypeSourceInfo *TInfo;
-
-    // Install [u]int128_t for 64-bit targets.
-    TInfo = Context.getTrivialTypeSourceInfo(Context.Int128Ty);
-    PushOnScopeChains(TypedefDecl::Create(Context, CurContext,
-                                          SourceLocation(),
-                                          SourceLocation(),
-                                          &Context.Idents.get("__int128_t"),
-                                          TInfo), TUScope);
-
-    TInfo = Context.getTrivialTypeSourceInfo(Context.UnsignedInt128Ty);
-    PushOnScopeChains(TypedefDecl::Create(Context, CurContext,
-                                          SourceLocation(),
-                                          SourceLocation(),
-                                          &Context.Idents.get("__uint128_t"),
-                                          TInfo), TUScope);
-    Context.setInt128Installed();
+  if (PP.getLangOptions().ObjC1) {
+    // Synthesize "@class Protocol;
+    if (Context.getObjCProtoType().isNull()) {
+      ObjCInterfaceDecl *ProtocolDecl =
+        ObjCInterfaceDecl::Create(Context, CurContext, SourceLocation(),
+                                  &Context.Idents.get("Protocol"),
+                                  SourceLocation(), true);
+      Context.setObjCProtoType(Context.getObjCInterfaceType(ProtocolDecl));
+      PushOnScopeChains(ProtocolDecl, TUScope, false);
+    }  
   }
-
-
-  if (!PP.getLangOptions().ObjC1) return;
-
-  // Built-in ObjC types may already be set by ASTReader (hence isNull checks).
-  if (Context.getObjCSelType().isNull()) {
-    // Create the built-in typedef for 'SEL'.
-    QualType SelT = Context.getPointerType(Context.ObjCBuiltinSelTy);
-    TypeSourceInfo *SelInfo = Context.getTrivialTypeSourceInfo(SelT);
-    TypedefDecl *SelTypedef
-      = TypedefDecl::Create(Context, CurContext,
-                            SourceLocation(), SourceLocation(),
-                            &Context.Idents.get("SEL"), SelInfo);
-    PushOnScopeChains(SelTypedef, TUScope);
-    Context.setObjCSelType(Context.getTypeDeclType(SelTypedef));
-    Context.ObjCSelRedefinitionType = Context.getObjCSelType();
-  }
-
-  // Synthesize "@class Protocol;
-  if (Context.getObjCProtoType().isNull()) {
-    ObjCInterfaceDecl *ProtocolDecl =
-      ObjCInterfaceDecl::Create(Context, CurContext, SourceLocation(),
-                                &Context.Idents.get("Protocol"),
-                                SourceLocation(), true);
-    Context.setObjCProtoType(Context.getObjCInterfaceType(ProtocolDecl));
-    PushOnScopeChains(ProtocolDecl, TUScope, false);
-  }
-  // Create the built-in typedef for 'id'.
-  if (Context.getObjCIdType().isNull()) {
-    QualType T = Context.getObjCObjectType(Context.ObjCBuiltinIdTy, 0, 0);
-    T = Context.getObjCObjectPointerType(T);
-    TypeSourceInfo *IdInfo = Context.getTrivialTypeSourceInfo(T);
-    TypedefDecl *IdTypedef
-      = TypedefDecl::Create(Context, CurContext,
-                            SourceLocation(), SourceLocation(),
-                            &Context.Idents.get("id"), IdInfo);
-    PushOnScopeChains(IdTypedef, TUScope);
-    Context.setObjCIdType(Context.getTypeDeclType(IdTypedef));
-    Context.ObjCIdRedefinitionType = Context.getObjCIdType();
-  }
-  // Create the built-in typedef for 'Class'.
-  if (Context.getObjCClassType().isNull()) {
-    QualType T = Context.getObjCObjectType(Context.ObjCBuiltinClassTy, 0, 0);
-    T = Context.getObjCObjectPointerType(T);
-    TypeSourceInfo *ClassInfo = Context.getTrivialTypeSourceInfo(T);
-    TypedefDecl *ClassTypedef
-      = TypedefDecl::Create(Context, CurContext,
-                            SourceLocation(), SourceLocation(),
-                            &Context.Idents.get("Class"), ClassInfo);
-    PushOnScopeChains(ClassTypedef, TUScope);
-    Context.setObjCClassType(Context.getTypeDeclType(ClassTypedef));
-    Context.ObjCClassRedefinitionType = Context.getObjCClassType();
-  }  
 }
 
 Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
-           bool CompleteTranslationUnit,
+           TranslationUnitKind TUKind,
            CodeCompleteConsumer *CodeCompleter)
   : TheTargetAttributesSema(0), FPFeatures(pp.getLangOptions()),
     LangOpts(pp.getLangOptions()), PP(pp), Context(ctxt), Consumer(consumer),
     Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()),
     CollectStats(false), ExternalSource(0), CodeCompleter(CodeCompleter),
-    CurContext(0), PackContext(0), MSStructPragmaOn(false), VisContext(0),
+    CurContext(0), OriginalLexicalContext(0),
+    PackContext(0), MSStructPragmaOn(false), VisContext(0),
     ExprNeedsCleanups(0), LateTemplateParser(0), OpaqueParser(0),
     IdResolver(pp.getLangOptions()), CXXTypeInfoDecl(0), MSVCGuidDecl(0),
     GlobalNewDeleteDeclared(false), 
-    CompleteTranslationUnit(CompleteTranslationUnit),
+    ObjCShouldCallSuperDealloc(false),
+    ObjCShouldCallSuperFinalize(false),
+    TUKind(TUKind),
     NumSFINAEErrors(0), SuppressAccessChecking(false), 
     AccessCheckingSFINAE(false), InNonInstantiationSFINAEContext(false),
     NonInstantiationEntries(0), ArgumentPackSubstitutionIndex(-1),
@@ -181,6 +137,40 @@ void Sema::Initialize() {
   if (ExternalSemaSource *ExternalSema
       = dyn_cast_or_null<ExternalSemaSource>(Context.getExternalSource()))
     ExternalSema->InitializeSema(*this);
+
+  // Initialize predefined 128-bit integer types, if needed.
+  if (PP.getTargetInfo().getPointerWidth(0) >= 64) {
+    // If either of the 128-bit integer types are unavailable to name lookup,
+    // define them now.
+    DeclarationName Int128 = &Context.Idents.get("__int128_t");
+    if (IdentifierResolver::begin(Int128) == IdentifierResolver::end())
+      PushOnScopeChains(Context.getInt128Decl(), TUScope);
+
+    DeclarationName UInt128 = &Context.Idents.get("__uint128_t");
+    if (IdentifierResolver::begin(UInt128) == IdentifierResolver::end())
+      PushOnScopeChains(Context.getUInt128Decl(), TUScope);
+  }
+  
+
+  // Initialize predefined Objective-C types:
+  if (PP.getLangOptions().ObjC1) {
+    // If 'SEL' does not yet refer to any declarations, make it refer to the
+    // predefined 'SEL'.
+    DeclarationName SEL = &Context.Idents.get("SEL");
+    if (IdentifierResolver::begin(SEL) == IdentifierResolver::end())
+      PushOnScopeChains(Context.getObjCSelDecl(), TUScope);
+
+    // If 'id' does not yet refer to any declarations, make it refer to the
+    // predefined 'id'.
+    DeclarationName Id = &Context.Idents.get("id");
+    if (IdentifierResolver::begin(Id) == IdentifierResolver::end())
+      PushOnScopeChains(Context.getObjCIdDecl(), TUScope);
+    
+    // Create the built-in typedef for 'Class'.
+    DeclarationName Class = &Context.Idents.get("Class");
+    if (IdentifierResolver::begin(Class) == IdentifierResolver::end())
+      PushOnScopeChains(Context.getObjCClassDecl(), TUScope);
+  }
 }
 
 Sema::~Sema() {
@@ -210,7 +200,7 @@ Sema::~Sema() {
 /// make the relevant declaration unavailable instead of erroring, do
 /// so and return true.
 bool Sema::makeUnavailableInSystemHeader(SourceLocation loc,
-                                         llvm::StringRef msg) {
+                                         StringRef msg) {
   // If we're not in a function, it's an error.
   FunctionDecl *fn = dyn_cast<FunctionDecl>(CurContext);
   if (!fn) return false;
@@ -287,7 +277,9 @@ ExprResult Sema::ImpCastExprToType(Expr *E, QualType Ty,
 CastKind Sema::ScalarTypeToBooleanCastKind(QualType ScalarTy) {
   switch (ScalarTy->getScalarTypeKind()) {
   case Type::STK_Bool: return CK_NoOp;
-  case Type::STK_Pointer: return CK_PointerToBoolean;
+  case Type::STK_CPointer: return CK_PointerToBoolean;
+  case Type::STK_BlockPointer: return CK_PointerToBoolean;
+  case Type::STK_ObjCObjectPointer: return CK_PointerToBoolean;
   case Type::STK_MemberPointer: return CK_MemberPointerToBoolean;
   case Type::STK_Integral: return CK_IntegralToBoolean;
   case Type::STK_Floating: return CK_FloatingToBoolean;
@@ -297,12 +289,6 @@ CastKind Sema::ScalarTypeToBooleanCastKind(QualType ScalarTy) {
   return CK_Invalid;
 }
 
-ExprValueKind Sema::CastCategory(Expr *E) {
-  Expr::Classification Classification = E->Classify(Context);
-  return Classification.isRValue() ? VK_RValue :
-      (Classification.isLValue() ? VK_LValue : VK_XValue);
-}
-
 /// \brief Used to prune the decls of Sema's UnusedFileScopedDecls vector.
 static bool ShouldRemoveFromUnused(Sema *SemaRef, const DeclaratorDecl *D) {
   if (D->isUsed())
@@ -358,7 +344,7 @@ static void checkUndefinedInternals(Sema &S) {
   if (S.UndefinedInternals.empty()) return;
 
   // Collect all the still-undefined entities with internal linkage.
-  llvm::SmallVector<UndefinedInternal, 16> undefined;
+  SmallVector<UndefinedInternal, 16> undefined;
   for (llvm::DenseMap<NamedDecl*,SourceLocation>::iterator
          i = S.UndefinedInternals.begin(), e = S.UndefinedInternals.end();
        i != e; ++i) {
@@ -389,7 +375,7 @@ static void checkUndefinedInternals(Sema &S) {
   // the iteration order through an llvm::DenseMap.
   llvm::array_pod_sort(undefined.begin(), undefined.end());
 
-  for (llvm::SmallVectorImpl<UndefinedInternal>::iterator
+  for (SmallVectorImpl<UndefinedInternal>::iterator
          i = undefined.begin(), e = undefined.end(); i != e; ++i) {
     NamedDecl *decl = i->decl;
     S.Diag(decl->getLocation(), diag::warn_undefined_internal)
@@ -398,24 +384,41 @@ static void checkUndefinedInternals(Sema &S) {
   }
 }
 
+void Sema::LoadExternalWeakUndeclaredIdentifiers() {
+  if (!ExternalSource)
+    return;
+  
+  SmallVector<std::pair<IdentifierInfo *, WeakInfo>, 4> WeakIDs;
+  ExternalSource->ReadWeakUndeclaredIdentifiers(WeakIDs);
+  for (unsigned I = 0, N = WeakIDs.size(); I != N; ++I) {
+    llvm::DenseMap<IdentifierInfo*,WeakInfo>::iterator Pos
+      = WeakUndeclaredIdentifiers.find(WeakIDs[I].first);
+    if (Pos != WeakUndeclaredIdentifiers.end())
+      continue;
+    
+    WeakUndeclaredIdentifiers.insert(WeakIDs[I]);
+  }
+}
+
 /// ActOnEndOfTranslationUnit - This is called at the very end of the
 /// translation unit when EOF is reached and all but the top-level scope is
 /// popped.
 void Sema::ActOnEndOfTranslationUnit() {
-  // At PCH writing, implicit instantiations and VTable handling info are
-  // stored and performed when the PCH is included.
-  if (CompleteTranslationUnit) {
+  // Only complete translation units define vtables and perform implicit
+  // instantiations.
+  if (TUKind == TU_Complete) {
     // If any dynamic classes have their key function defined within
     // this translation unit, then those vtables are considered "used" and must
     // be emitted.
-    for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I) {
-      assert(!DynamicClasses[I]->isDependentType() &&
+    for (DynamicClassesType::iterator I = DynamicClasses.begin(ExternalSource),
+                                      E = DynamicClasses.end();
+         I != E; ++I) {
+      assert(!(*I)->isDependentType() &&
              "Should not see dependent types here!");
-      if (const CXXMethodDecl *KeyFunction
-          = Context.getKeyFunction(DynamicClasses[I])) {
+      if (const CXXMethodDecl *KeyFunction = Context.getKeyFunction(*I)) {
         const FunctionDecl *Definition = 0;
         if (KeyFunction->hasBody(Definition))
-          MarkVTableUsed(Definition->getLocation(), DynamicClasses[I], true);
+          MarkVTableUsed(Definition->getLocation(), *I, true);
       }
     }
 
@@ -438,13 +441,15 @@ void Sema::ActOnEndOfTranslationUnit() {
   }
   
   // Remove file scoped decls that turned out to be used.
-  UnusedFileScopedDecls.erase(std::remove_if(UnusedFileScopedDecls.begin(),
+  UnusedFileScopedDecls.erase(std::remove_if(UnusedFileScopedDecls.begin(0, 
+                                                                         true),
                                              UnusedFileScopedDecls.end(),
                               std::bind1st(std::ptr_fun(ShouldRemoveFromUnused),
                                            this)),
                               UnusedFileScopedDecls.end());
 
-  if (!CompleteTranslationUnit) {
+  if (TUKind == TU_Prefix) {
+    // Translation unit prefixes don't need any of the checking below.
     TUScope = 0;
     return;
   }
@@ -452,6 +457,7 @@ void Sema::ActOnEndOfTranslationUnit() {
   // Check for #pragma weak identifiers that were never declared
   // FIXME: This will cause diagnostics to be emitted in a non-determinstic
   // order!  Iterating over a densemap like this is bad.
+  LoadExternalWeakUndeclaredIdentifiers();
   for (llvm::DenseMap<IdentifierInfo*,WeakInfo>::iterator
        I = WeakUndeclaredIdentifiers.begin(),
        E = WeakUndeclaredIdentifiers.end(); I != E; ++I) {
@@ -461,6 +467,40 @@ void Sema::ActOnEndOfTranslationUnit() {
       << I->first;
   }
 
+  if (TUKind == TU_Module) {
+    // Mark any macros from system headers (in /usr/include) as exported, along
+    // with our own Clang headers.
+    // FIXME: This is a gross hack to deal with the fact that system headers
+    // are #include'd in many places within module headers, but are not 
+    // themselves modularized. This doesn't actually work, but it lets us
+    // focus on other issues for the moment.
+    for (Preprocessor::macro_iterator M = PP.macro_begin(false),
+                                   MEnd = PP.macro_end(false);
+         M != MEnd; ++M) {
+      if (M->second && 
+          !M->second->isExported() &&
+          !M->second->isBuiltinMacro()) {
+        SourceLocation Loc = M->second->getDefinitionLoc();
+        if (SourceMgr.isInSystemHeader(Loc)) {
+          const FileEntry *File
+            = SourceMgr.getFileEntryForID(SourceMgr.getFileID(Loc));
+          if (File && 
+              ((StringRef(File->getName()).find("lib/clang") 
+                  != StringRef::npos) ||
+               (StringRef(File->getName()).find("usr/include") 
+                  != StringRef::npos) ||
+               (StringRef(File->getName()).find("usr/local/include") 
+                  != StringRef::npos)))
+            M->second->setExportLocation(Loc);
+        }
+      }
+    }
+          
+    // Modules don't need any of the checking below.
+    TUScope = 0;
+    return;
+  }
+  
   // C99 6.9.2p2:
   //   A declaration of an identifier for an object that has file
   //   scope without an initializer, and without a storage-class
@@ -473,8 +513,12 @@ void Sema::ActOnEndOfTranslationUnit() {
   //   identifier, with the composite type as of the end of the
   //   translation unit, with an initializer equal to 0.
   llvm::SmallSet<VarDecl *, 32> Seen;
-  for (unsigned i = 0, e = TentativeDefinitions.size(); i != e; ++i) {
-    VarDecl *VD = TentativeDefinitions[i]->getActingDefinition();
+  for (TentativeDefinitionsType::iterator 
+            T = TentativeDefinitions.begin(ExternalSource),
+         TEnd = TentativeDefinitions.end();
+       T != TEnd; ++T) 
+  {
+    VarDecl *VD = (*T)->getActingDefinition();
 
     // If the tentative definition was completed, getActingDefinition() returns
     // null. If we've already seen this variable before, insert()'s second
@@ -510,16 +554,19 @@ void Sema::ActOnEndOfTranslationUnit() {
   if (LangOpts.CPlusPlus0x &&
       Diags.getDiagnosticLevel(diag::warn_delegating_ctor_cycle,
                                SourceLocation())
-        != Diagnostic::Ignored)
+        != DiagnosticsEngine::Ignored)
     CheckDelegatingCtorCycles();
 
   // If there were errors, disable 'unused' warnings since they will mostly be
   // noise.
   if (!Diags.hasErrorOccurred()) {
     // Output warning for unused file scoped decls.
-    for (llvm::SmallVectorImpl<const DeclaratorDecl*>::iterator
-           I = UnusedFileScopedDecls.begin(),
+    for (UnusedFileScopedDeclsType::iterator
+           I = UnusedFileScopedDecls.begin(ExternalSource),
            E = UnusedFileScopedDecls.end(); I != E; ++I) {
+      if (ShouldRemoveFromUnused(this, *I))
+        continue;
+      
       if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
         const FunctionDecl *DiagD;
         if (!FD->hasBody(DiagD))
@@ -631,7 +678,7 @@ Sema::SemaDiagnosticBuilder::~SemaDiagnosticBuilder() {
       // Make a copy of this suppressed diagnostic and store it with the
       // template-deduction information;
       FlushCounts();
-      DiagnosticInfo DiagInfo(&SemaRef.Diags);
+      Diagnostic DiagInfo(&SemaRef.Diags);
         
       if (*Info)
         (*Info)->addSuppressedDiagnostic(DiagInfo.getLocation(),
@@ -646,6 +693,9 @@ Sema::SemaDiagnosticBuilder::~SemaDiagnosticBuilder() {
     }
   }
   
+  // Set up the context's printing policy based on our current state.
+  SemaRef.Context.setPrintingPolicy(SemaRef.getPrintingPolicy());
+  
   // Emit the diagnostic.
   if (!this->Emit())
     return;
@@ -677,20 +727,20 @@ Sema::Diag(SourceLocation Loc, const PartialDiagnostic& PD) {
   return Builder;
 }
 
-/// \brief Looks through the macro-instantiation chain for the given
-/// location, looking for a macro instantiation with the given name.
+/// \brief Looks through the macro-expansion chain for the given
+/// location, looking for a macro expansion with the given name.
 /// If one is found, returns true and sets the location to that
-/// instantiation loc.
-bool Sema::findMacroSpelling(SourceLocation &locref, llvm::StringRef name) {
+/// expansion loc.
+bool Sema::findMacroSpelling(SourceLocation &locref, StringRef name) {
   SourceLocation loc = locref;
   if (!loc.isMacroID()) return false;
 
   // There's no good way right now to look at the intermediate
-  // instantiations, so just jump to the instantiation location.
-  loc = getSourceManager().getInstantiationLoc(loc);
+  // expansions, so just jump to the expansion location.
+  loc = getSourceManager().getExpansionLoc(loc);
 
   // If that's written with the name, stop here.
-  llvm::SmallVector<char, 16> buffer;
+  SmallVector<char, 16> buffer;
   if (getPreprocessor().getSpelling(loc, buffer) == name) {
     locref = loc;
     return true;
@@ -754,7 +804,7 @@ void Sema::PopFunctionOrBlockScope(const AnalysisBasedWarnings::Policy *WP,
   if (WP && D)
     AnalysisWarnings.IssueWarnings(*WP, Scope, D, blkExpr);
   else {
-    for (llvm::SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
+    for (SmallVectorImpl<sema::PossiblyUnreachableDiag>::iterator
          i = Scope->PossiblyUnreachableDiags.begin(),
          e = Scope->PossiblyUnreachableDiags.end();
          i != e; ++i) {
@@ -790,10 +840,10 @@ ExternalSemaSource::ReadMethodPool(Selector Sel) {
 }
 
 void ExternalSemaSource::ReadKnownNamespaces(
-                           llvm::SmallVectorImpl<NamespaceDecl *> &Namespaces) {  
+                           SmallVectorImpl<NamespaceDecl *> &Namespaces) {  
 }
 
-void PrettyDeclStackTraceEntry::print(llvm::raw_ostream &OS) const {
+void PrettyDeclStackTraceEntry::print(raw_ostream &OS) const {
   SourceLocation Loc = this->Loc;
   if (!Loc.isValid() && TheDecl) Loc = TheDecl->getLocation();
   if (Loc.isValid()) {
@@ -820,27 +870,38 @@ void PrettyDeclStackTraceEntry::print(llvm::raw_ostream &OS) const {
 /// \param ZeroArgCallReturnTy - If the expression can be turned into a call
 ///  with no arguments, this parameter is set to the type returned by such a
 ///  call; otherwise, it is set to an empty QualType.
-/// \param NonTemplateOverloads - If the expression is an overloaded function
+/// \param OverloadSet - If the expression is an overloaded function
 ///  name, this parameter is populated with the decls of the various overloads.
 bool Sema::isExprCallable(const Expr &E, QualType &ZeroArgCallReturnTy,
-                          UnresolvedSetImpl &NonTemplateOverloads) {
+                          UnresolvedSetImpl &OverloadSet) {
   ZeroArgCallReturnTy = QualType();
-  NonTemplateOverloads.clear();
-  if (const OverloadExpr *Overloads = dyn_cast<OverloadExpr>(&E)) {
+  OverloadSet.clear();
+
+  if (E.getType() == Context.OverloadTy) {
+    OverloadExpr::FindResult FR = OverloadExpr::find(const_cast<Expr*>(&E));
+    const OverloadExpr *Overloads = FR.Expression;
+
     for (OverloadExpr::decls_iterator it = Overloads->decls_begin(),
          DeclsEnd = Overloads->decls_end(); it != DeclsEnd; ++it) {
-      // Our overload set may include TemplateDecls, which we'll ignore for our
-      // present purpose.
-      if (const FunctionDecl *OverloadDecl = dyn_cast<FunctionDecl>(*it)) {
-        NonTemplateOverloads.addDecl(*it);
+      OverloadSet.addDecl(*it);
+
+      // Check whether the function is a non-template which takes no
+      // arguments.
+      if (const FunctionDecl *OverloadDecl
+            = dyn_cast<FunctionDecl>((*it)->getUnderlyingDecl())) {
         if (OverloadDecl->getMinRequiredArguments() == 0)
           ZeroArgCallReturnTy = OverloadDecl->getResultType();
       }
     }
+
+    // Ignore overloads that are pointer-to-member constants.
+    if (FR.HasFormOfMemberPointer)
+      return false;
+
     return true;
   }
 
-  if (const DeclRefExpr *DeclRef = dyn_cast<DeclRefExpr>(&E)) {
+  if (const DeclRefExpr *DeclRef = dyn_cast<DeclRefExpr>(E.IgnoreParens())) {
     if (const FunctionDecl *Fun = dyn_cast<FunctionDecl>(DeclRef->getDecl())) {
       if (Fun->getMinRequiredArguments() == 0)
         ZeroArgCallReturnTy = Fun->getResultType();
@@ -887,8 +948,8 @@ bool Sema::isExprCallable(const Expr &E, QualType &ZeroArgCallReturnTy,
 ///  -fshow-overloads=best, this is the location to attach to the note about too
 ///  many candidates. Typically this will be the location of the original
 ///  ill-formed expression.
-void Sema::NoteOverloads(const UnresolvedSetImpl &Overloads,
-                         const SourceLocation FinalNoteLoc) {
+static void noteOverloads(Sema &S, const UnresolvedSetImpl &Overloads,
+                          const SourceLocation FinalNoteLoc) {
   int ShownOverloads = 0;
   int SuppressedOverloads = 0;
   for (UnresolvedSetImpl::iterator It = Overloads.begin(),
@@ -896,15 +957,86 @@ void Sema::NoteOverloads(const UnresolvedSetImpl &Overloads,
     // FIXME: Magic number for max shown overloads stolen from
     // OverloadCandidateSet::NoteCandidates.
     if (ShownOverloads >= 4 &&
-        Diags.getShowOverloads() == Diagnostic::Ovl_Best) {
+        S.Diags.getShowOverloads() == DiagnosticsEngine::Ovl_Best) {
       ++SuppressedOverloads;
       continue;
     }
-    Diag(cast<FunctionDecl>(*It)->getSourceRange().getBegin(),
-         diag::note_member_ref_possible_intended_overload);
+
+    NamedDecl *Fn = (*It)->getUnderlyingDecl();
+    S.Diag(Fn->getLocStart(), diag::note_possible_target_of_call);
     ++ShownOverloads;
   }
+
   if (SuppressedOverloads)
-    Diag(FinalNoteLoc, diag::note_ovl_too_many_candidates)
-        << SuppressedOverloads;
+    S.Diag(FinalNoteLoc, diag::note_ovl_too_many_candidates)
+      << SuppressedOverloads;
+}
+
+static void notePlausibleOverloads(Sema &S, SourceLocation Loc,
+                                   const UnresolvedSetImpl &Overloads,
+                                   bool (*IsPlausibleResult)(QualType)) {
+  if (!IsPlausibleResult)
+    return noteOverloads(S, Overloads, Loc);
+
+  UnresolvedSet<2> PlausibleOverloads;
+  for (OverloadExpr::decls_iterator It = Overloads.begin(),
+         DeclsEnd = Overloads.end(); It != DeclsEnd; ++It) {
+    const FunctionDecl *OverloadDecl = cast<FunctionDecl>(*It);
+    QualType OverloadResultTy = OverloadDecl->getResultType();
+    if (IsPlausibleResult(OverloadResultTy))
+      PlausibleOverloads.addDecl(It.getDecl());
+  }
+  noteOverloads(S, PlausibleOverloads, Loc);
+}
+
+/// Determine whether the given expression can be called by just
+/// putting parentheses after it.  Notably, expressions with unary
+/// operators can't be because the unary operator will start parsing
+/// outside the call.
+static bool IsCallableWithAppend(Expr *E) {
+  E = E->IgnoreImplicit();
+  return (!isa<CStyleCastExpr>(E) &&
+          !isa<UnaryOperator>(E) &&
+          !isa<BinaryOperator>(E) &&
+          !isa<CXXOperatorCallExpr>(E));
+}
+
+bool Sema::tryToRecoverWithCall(ExprResult &E, const PartialDiagnostic &PD,
+                                bool ForceComplain,
+                                bool (*IsPlausibleResult)(QualType)) {
+  SourceLocation Loc = E.get()->getExprLoc();
+  SourceRange Range = E.get()->getSourceRange();
+
+  QualType ZeroArgCallTy;
+  UnresolvedSet<4> Overloads;
+  if (isExprCallable(*E.get(), ZeroArgCallTy, Overloads) &&
+      !ZeroArgCallTy.isNull() &&
+      (!IsPlausibleResult || IsPlausibleResult(ZeroArgCallTy))) {
+    // At this point, we know E is potentially callable with 0
+    // arguments and that it returns something of a reasonable type,
+    // so we can emit a fixit and carry on pretending that E was
+    // actually a CallExpr.
+    SourceLocation ParenInsertionLoc =
+      PP.getLocForEndOfToken(Range.getEnd());
+    Diag(Loc, PD) 
+      << /*zero-arg*/ 1 << Range
+      << (IsCallableWithAppend(E.get())
+          ? FixItHint::CreateInsertion(ParenInsertionLoc, "()")
+          : FixItHint());
+    notePlausibleOverloads(*this, Loc, Overloads, IsPlausibleResult);
+
+    // FIXME: Try this before emitting the fixit, and suppress diagnostics
+    // while doing so.
+    E = ActOnCallExpr(0, E.take(), ParenInsertionLoc,
+                      MultiExprArg(*this, 0, 0),
+                      ParenInsertionLoc.getLocWithOffset(1));
+    return true;
+  }
+
+  if (!ForceComplain) return false;
+
+  Diag(Loc, PD) << /*not zero-arg*/ 0 << Range;
+  notePlausibleOverloads(*this, Loc, Overloads, IsPlausibleResult);
+  E = ExprError();
+  return true;
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaAccess.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaAccess.cpp
index a26737e00e4d..6cd923032685 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaAccess.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaAccess.cpp
@@ -103,7 +103,11 @@ struct EffectiveContext {
       } else if (isa<FunctionDecl>(DC)) {
         FunctionDecl *Function = cast<FunctionDecl>(DC)->getCanonicalDecl();
         Functions.push_back(Function);
-        DC = Function->getDeclContext();
+        
+        if (Function->getFriendObjectKind())
+          DC = Function->getLexicalDeclContext();
+        else
+          DC = Function->getDeclContext();
       } else if (DC->isFileContext()) {
         break;
       } else {
@@ -126,11 +130,11 @@ struct EffectiveContext {
     return Inner;
   }
 
-  typedef llvm::SmallVectorImpl<CXXRecordDecl*>::const_iterator record_iterator;
+  typedef SmallVectorImpl<CXXRecordDecl*>::const_iterator record_iterator;
 
   DeclContext *Inner;
-  llvm::SmallVector<FunctionDecl*, 4> Functions;
-  llvm::SmallVector<CXXRecordDecl*, 4> Records;
+  SmallVector<FunctionDecl*, 4> Functions;
+  SmallVector<CXXRecordDecl*, 4> Records;
   bool Dependent;
 };
 
@@ -146,10 +150,8 @@ struct AccessTarget : public AccessedEntity {
                MemberNonce _,
                CXXRecordDecl *NamingClass,
                DeclAccessPair FoundDecl,
-               QualType BaseObjectType,
-               bool IsUsingDecl = false)
-    : AccessedEntity(Context, Member, NamingClass, FoundDecl, BaseObjectType),
-      IsUsingDeclaration(IsUsingDecl) {
+               QualType BaseObjectType)
+    : AccessedEntity(Context, Member, NamingClass, FoundDecl, BaseObjectType) {
     initialize();
   }
 
@@ -218,7 +220,6 @@ struct AccessTarget : public AccessedEntity {
     DeclaringClass = DeclaringClass->getCanonicalDecl();
   }
 
-  bool IsUsingDeclaration : 1;
   bool HasInstanceContext : 1;
   mutable bool CalculatedInstanceContext : 1;
   mutable const CXXRecordDecl *InstanceContext;
@@ -260,7 +261,7 @@ static AccessResult IsDerivedFromInclusive(const CXXRecordDecl *Derived,
     return AR_dependent;
 
   AccessResult OnFailure = AR_inaccessible;
-  llvm::SmallVector<const CXXRecordDecl*, 8> Queue; // actually a stack
+  SmallVector<const CXXRecordDecl*, 8> Queue; // actually a stack
 
   while (true) {
     for (CXXRecordDecl::base_class_const_iterator
@@ -421,7 +422,7 @@ static AccessResult MatchesFriend(Sema &S,
 
   // Check whether the friend is the template of a class in the
   // context chain.
-  for (llvm::SmallVectorImpl<CXXRecordDecl*>::const_iterator
+  for (SmallVectorImpl<CXXRecordDecl*>::const_iterator
          I = EC.Records.begin(), E = EC.Records.end(); I != E; ++I) {
     CXXRecordDecl *Record = *I;
 
@@ -472,7 +473,7 @@ static AccessResult MatchesFriend(Sema &S,
                                   FunctionDecl *Friend) {
   AccessResult OnFailure = AR_inaccessible;
 
-  for (llvm::SmallVectorImpl<FunctionDecl*>::const_iterator
+  for (SmallVectorImpl<FunctionDecl*>::const_iterator
          I = EC.Functions.begin(), E = EC.Functions.end(); I != E; ++I) {
     if (Friend == *I)
       return AR_accessible;
@@ -493,7 +494,7 @@ static AccessResult MatchesFriend(Sema &S,
 
   AccessResult OnFailure = AR_inaccessible;
 
-  for (llvm::SmallVectorImpl<FunctionDecl*>::const_iterator
+  for (SmallVectorImpl<FunctionDecl*>::const_iterator
          I = EC.Functions.begin(), E = EC.Functions.end(); I != E; ++I) {
 
     FunctionTemplateDecl *FTD = (*I)->getPrimaryTemplate();
@@ -584,7 +585,7 @@ struct ProtectedFriendContext {
   bool EverDependent;
 
   /// The path down to the current base class.
-  llvm::SmallVector<const CXXRecordDecl*, 20> CurPath;
+  SmallVector<const CXXRecordDecl*, 20> CurPath;
 
   ProtectedFriendContext(Sema &S, const EffectiveContext &EC,
                          const CXXRecordDecl *InstanceContext,
@@ -1273,7 +1274,7 @@ static AccessResult CheckEffectiveAccess(Sema &S,
                                          AccessTarget &Entity) {
   assert(Entity.getAccess() != AS_public && "called for public access!");
 
-  if (S.getLangOptions().Microsoft &&
+  if (S.getLangOptions().MicrosoftMode &&
       IsMicrosoftUsingDeclarationAccessBug(S, Loc, Entity))
     return AR_accessible;
 
@@ -1554,8 +1555,7 @@ Sema::AccessResult Sema::CheckMemberOperatorAccess(SourceLocation OpLoc,
       Found.getAccess() == AS_public)
     return AR_accessible;
 
-  const RecordType *RT = ObjectExpr->getType()->getAs<RecordType>();
-  assert(RT && "found member operator but object expr not of record type");
+  const RecordType *RT = ObjectExpr->getType()->castAs<RecordType>();
   CXXRecordDecl *NamingClass = cast<CXXRecordDecl>(RT->getDecl());
 
   AccessTarget Entity(Context, AccessTarget::Member, NamingClass, Found,
@@ -1638,13 +1638,34 @@ void Sema::CheckLookupAccess(const LookupResult &R) {
     if (I.getAccess() != AS_public) {
       AccessTarget Entity(Context, AccessedEntity::Member,
                           R.getNamingClass(), I.getPair(),
-                          R.getBaseObjectType(), R.isUsingDeclaration());
+                          R.getBaseObjectType());
       Entity.setDiag(diag::err_access);
       CheckAccess(*this, R.getNameLoc(), Entity);
     }
   }
 }
 
+/// Checks access to Decl from the given class. The check will take access
+/// specifiers into account, but no member access expressions and such.
+///
+/// \param Decl the declaration to check if it can be accessed
+/// \param Class the class/context from which to start the search
+/// \return true if the Decl is accessible from the Class, false otherwise.
+bool Sema::IsSimplyAccessible(NamedDecl *Decl, CXXRecordDecl *Class) {
+  if (!Class || !Decl->isCXXClassMember())
+    return true;
+
+  QualType qType = Class->getTypeForDecl()->getCanonicalTypeInternal();
+  AccessTarget Entity(Context, AccessedEntity::Member, Class,
+                      DeclAccessPair::make(Decl, Decl->getAccess()),
+                      qType);
+  if (Entity.getAccess() == AS_public)
+    return true;
+
+  EffectiveContext EC(CurContext);
+  return ::IsAccessible(*this, EC, Entity) != ::AR_inaccessible;
+}
+
 void Sema::ActOnStartSuppressingAccessChecks() {
   assert(!SuppressAccessChecking &&
          "Tried to start access check suppression when already started.");
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaAttr.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaAttr.cpp
index 53dd297aebb3..77410db01f58 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaAttr.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaAttr.cpp
@@ -189,7 +189,7 @@ void Sema::ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind,
 }
 
 void Sema::ActOnPragmaPack(PragmaPackKind Kind, IdentifierInfo *Name,
-                           ExprTy *alignment, SourceLocation PragmaLoc,
+                           Expr *alignment, SourceLocation PragmaLoc,
                            SourceLocation LParenLoc, SourceLocation RParenLoc) {
   Expr *Alignment = static_cast<Expr *>(alignment);
 
@@ -265,7 +265,7 @@ void Sema::ActOnPragmaPack(PragmaPackKind Kind, IdentifierInfo *Name,
     break;
 
   default:
-    assert(0 && "Invalid #pragma pack kind.");
+    llvm_unreachable("Invalid #pragma pack kind.");
   }
 }
 
@@ -300,6 +300,18 @@ void Sema::ActOnPragmaUnused(const Token &IdTok, Scope *curScope,
   VD->addAttr(::new (Context) UnusedAttr(IdTok.getLocation(), Context));
 }
 
+void Sema::AddCFAuditedAttribute(Decl *D) {
+  SourceLocation Loc = PP.getPragmaARCCFCodeAuditedLoc();
+  if (!Loc.isValid()) return;
+
+  // Don't add a redundant or conflicting attribute.
+  if (D->hasAttr<CFAuditedTransferAttr>() ||
+      D->hasAttr<CFUnknownTransferAttr>())
+    return;
+
+  D->addAttr(::new (Context) CFAuditedTransferAttr(Loc, Context));
+}
+
 typedef std::vector<std::pair<unsigned, SourceLocation> > VisStack;
 enum { NoVisibility = (unsigned) -1 };
 
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaCXXScopeSpec.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaCXXScopeSpec.cpp
index 5f8c9c62a409..360a0404b8f4 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaCXXScopeSpec.cpp
@@ -137,8 +137,7 @@ DeclContext *Sema::computeDeclContext(const CXXScopeSpec &SS,
 
   switch (NNS->getKind()) {
   case NestedNameSpecifier::Identifier:
-    assert(false && "Dependent nested-name-specifier has no DeclContext");
-    break;
+    llvm_unreachable("Dependent nested-name-specifier has no DeclContext");
 
   case NestedNameSpecifier::Namespace:
     return NNS->getAsNamespace();
@@ -238,7 +237,7 @@ bool Sema::RequireCompleteDeclContext(CXXScopeSpec &SS,
     // until we see a definition, so awkwardly pull out this special
     // case.
     if (const EnumType *enumType = dyn_cast_or_null<EnumType>(tagType)) {
-      if (!enumType->getDecl()->isDefinition()) {
+      if (!enumType->getDecl()->isCompleteDefinition()) {
         Diag(loc, diag::err_incomplete_nested_name_spec)
           << type << SS.getRange();
         SS.SetInvalid(SS.getRange());
@@ -615,6 +614,31 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S,
     LookupName(Found, S);
   }
 
+  // In Microsoft mode, if we are within a templated function and we can't
+  // resolve Identifier, then extend the SS with Identifier. This will have 
+  // the effect of resolving Identifier during template instantiation. 
+  // The goal is to be able to resolve a function call whose
+  // nested-name-specifier is located inside a dependent base class.
+  // Example: 
+  //
+  // class C {
+  // public:
+  //    static void foo2() {  }
+  // };
+  // template <class T> class A { public: typedef C D; };
+  //
+  // template <class T> class B : public A<T> {
+  // public:
+  //   void foo() { D::foo2(); }
+  // };
+  if (getLangOptions().MicrosoftExt) {
+    DeclContext *DC = LookupCtx ? LookupCtx : CurContext;
+    if (DC->isDependentContext() && DC->isFunctionOrMethod()) {
+      SS.Extend(Context, &Identifier, IdentifierLoc, CCLoc);
+      return false;
+    }
+  }
+
   unsigned DiagID;
   if (!Found.empty())
     DiagID = diag::err_expected_class_or_namespace;
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaCXXCast.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp
similarity index 78%
rename from contrib/llvm/tools/clang/lib/Sema/SemaCXXCast.cpp
rename to contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp
index d053d5a9e9fe..8bd9351e0bd4 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaCXXCast.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp
@@ -1,4 +1,4 @@
-//===--- SemaNamedCast.cpp - Semantic Analysis for Named Casts ------------===//
+//===--- SemaCast.cpp - Semantic Analysis for Casts -----------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,13 +7,17 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file implements semantic analysis for C++ named casts.
+//  This file implements semantic analysis for cast expressions, including
+//  1) C-style casts like '(int) x'
+//  2) C++ functional casts like 'int(x)'
+//  3) C++ named casts like 'static_cast<int>(x)'
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprObjC.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/Basic/PartialDiagnostic.h"
@@ -39,29 +43,83 @@ enum CastType {
   CT_Functional   ///< Type(expr)
 };
 
+namespace {
+  struct CastOperation {
+    CastOperation(Sema &S, QualType destType, ExprResult src)
+      : Self(S), SrcExpr(src), DestType(destType),
+        ResultType(destType.getNonLValueExprType(S.Context)),
+        ValueKind(Expr::getValueKindForType(destType)),
+        Kind(CK_Dependent) {
 
+      if (const BuiltinType *placeholder =
+            src.get()->getType()->getAsPlaceholderType()) {
+        PlaceholderKind = placeholder->getKind();
+      } else {
+        PlaceholderKind = (BuiltinType::Kind) 0;
+      }
+    }
 
+    Sema &Self;
+    ExprResult SrcExpr;
+    QualType DestType;
+    QualType ResultType;
+    ExprValueKind ValueKind;
+    CastKind Kind;
+    BuiltinType::Kind PlaceholderKind;
+    CXXCastPath BasePath;
 
-static void CheckConstCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                           ExprValueKind &VK,
-                           const SourceRange &OpRange,
-                           const SourceRange &DestRange);
-static void CheckReinterpretCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                                 ExprValueKind &VK,
-                                 const SourceRange &OpRange,
-                                 const SourceRange &DestRange,
-                                 CastKind &Kind);
-static void CheckStaticCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                            ExprValueKind &VK,
-                            const SourceRange &OpRange,
-                            CastKind &Kind,
-                            CXXCastPath &BasePath);
-static void CheckDynamicCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                             ExprValueKind &VK,
-                             const SourceRange &OpRange,
-                             const SourceRange &DestRange,
-                             CastKind &Kind,
-                             CXXCastPath &BasePath);
+    SourceRange OpRange;
+    SourceRange DestRange;
+
+    // Top-level semantics-checking routines.
+    void CheckConstCast();
+    void CheckReinterpretCast();
+    void CheckStaticCast();
+    void CheckDynamicCast();
+    void CheckCXXCStyleCast(bool FunctionalCast);
+    void CheckCStyleCast();
+
+    // Internal convenience methods.
+
+    /// Try to handle the given placeholder expression kind.  Return
+    /// true if the source expression has the appropriate placeholder
+    /// kind.  A placeholder can only be claimed once.
+    bool claimPlaceholder(BuiltinType::Kind K) {
+      if (PlaceholderKind != K) return false;
+
+      PlaceholderKind = (BuiltinType::Kind) 0;
+      return true;
+    }
+
+    bool isPlaceholder() const {
+      return PlaceholderKind != 0;
+    }
+    bool isPlaceholder(BuiltinType::Kind K) const {
+      return PlaceholderKind == K;
+    }
+
+    void checkCastAlign() {
+      Self.CheckCastAlign(SrcExpr.get(), DestType, OpRange);
+    }
+
+    void checkObjCARCConversion(Sema::CheckedConversionKind CCK) {
+      Expr *src = SrcExpr.get();
+      Self.CheckObjCARCConversion(OpRange, DestType, src, CCK);
+      SrcExpr = src;
+    }
+
+    /// Check for and handle non-overload placeholder expressions.
+    void checkNonOverloadPlaceholders() {
+      if (!isPlaceholder() || isPlaceholder(BuiltinType::Overload))
+        return;
+
+      SrcExpr = Self.CheckPlaceholderExpr(SrcExpr.take());
+      if (SrcExpr.isInvalid())
+        return;
+      PlaceholderKind = (BuiltinType::Kind) 0;
+    }
+  };
+}
 
 static bool CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType,
                                bool CheckCVR, bool CheckObjCLifetime);
@@ -162,70 +220,61 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
   ExprResult Ex = Owned(E);
   QualType DestType = DestTInfo->getType();
 
-  SourceRange OpRange(OpLoc, Parens.getEnd());
-  SourceRange DestRange = AngleBrackets;
-
   // If the type is dependent, we won't do the semantic analysis now.
   // FIXME: should we check this in a more fine-grained manner?
   bool TypeDependent = DestType->isDependentType() || Ex.get()->isTypeDependent();
 
-  ExprValueKind VK = VK_RValue;
-  if (TypeDependent)
-    VK = Expr::getValueKindForType(DestType);
+  CastOperation Op(*this, DestType, E);
+  Op.OpRange = SourceRange(OpLoc, Parens.getEnd());
+  Op.DestRange = AngleBrackets;
 
   switch (Kind) {
   default: llvm_unreachable("Unknown C++ cast!");
 
   case tok::kw_const_cast:
     if (!TypeDependent) {
-      CheckConstCast(*this, Ex, DestType, VK, OpRange, DestRange);
-      if (Ex.isInvalid())
+      Op.CheckConstCast();
+      if (Op.SrcExpr.isInvalid())
         return ExprError();
     }
-    return Owned(CXXConstCastExpr::Create(Context,
-                                        DestType.getNonLValueExprType(Context),
-                                          VK, Ex.take(), DestTInfo, OpLoc,
+    return Owned(CXXConstCastExpr::Create(Context, Op.ResultType, Op.ValueKind,
+                                          Op.SrcExpr.take(), DestTInfo, OpLoc,
                                           Parens.getEnd()));
 
   case tok::kw_dynamic_cast: {
-    CastKind Kind = CK_Dependent;
-    CXXCastPath BasePath;
     if (!TypeDependent) {
-      CheckDynamicCast(*this, Ex, DestType, VK, OpRange, DestRange,
-                       Kind, BasePath);
-      if (Ex.isInvalid())
+      Op.CheckDynamicCast();
+      if (Op.SrcExpr.isInvalid())
         return ExprError();
     }
-    return Owned(CXXDynamicCastExpr::Create(Context,
-                                          DestType.getNonLValueExprType(Context),
-                                            VK, Kind, Ex.take(), &BasePath, DestTInfo,
-                                            OpLoc, Parens.getEnd()));
+    return Owned(CXXDynamicCastExpr::Create(Context, Op.ResultType,
+                                            Op.ValueKind, Op.Kind,
+                                            Op.SrcExpr.take(), &Op.BasePath,
+                                            DestTInfo, OpLoc, Parens.getEnd()));
   }
   case tok::kw_reinterpret_cast: {
-    CastKind Kind = CK_Dependent;
     if (!TypeDependent) {
-      CheckReinterpretCast(*this, Ex, DestType, VK, OpRange, DestRange, Kind);
-      if (Ex.isInvalid())
+      Op.CheckReinterpretCast();
+      if (Op.SrcExpr.isInvalid())
         return ExprError();
     }
-    return Owned(CXXReinterpretCastExpr::Create(Context,
-                                  DestType.getNonLValueExprType(Context),
-                                  VK, Kind, Ex.take(), 0,
-                                  DestTInfo, OpLoc, Parens.getEnd()));
+    return Owned(CXXReinterpretCastExpr::Create(Context, Op.ResultType,
+                                                Op.ValueKind, Op.Kind,
+                                                Op.SrcExpr.take(), 0,
+                                                DestTInfo, OpLoc,
+                                                Parens.getEnd()));
   }
   case tok::kw_static_cast: {
-    CastKind Kind = CK_Dependent;
-    CXXCastPath BasePath;
     if (!TypeDependent) {
-      CheckStaticCast(*this, Ex, DestType, VK, OpRange, Kind, BasePath);
-      if (Ex.isInvalid())
+      Op.CheckStaticCast();
+      if (Op.SrcExpr.isInvalid())
         return ExprError();
     }
     
-    return Owned(CXXStaticCastExpr::Create(Context,
-                                         DestType.getNonLValueExprType(Context),
-                                           VK, Kind, Ex.take(), &BasePath,
-                                           DestTInfo, OpLoc, Parens.getEnd()));
+    return Owned(CXXStaticCastExpr::Create(Context, Op.ResultType, Op.ValueKind,
+                                           Op.Kind, Op.SrcExpr.take(),
+                                           &Op.BasePath, DestTInfo, OpLoc,
+                                           Parens.getEnd()));
   }
   }
 
@@ -410,7 +459,7 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType,
 
   QualType UnwrappedSrcType = Self.Context.getCanonicalType(SrcType), 
            UnwrappedDestType = Self.Context.getCanonicalType(DestType);
-  llvm::SmallVector<Qualifiers, 8> cv1, cv2;
+  SmallVector<Qualifiers, 8> cv1, cv2;
 
   // Find the qualifiers. We only care about cvr-qualifiers for the 
   // purpose of this check, because other qualifiers (address spaces, 
@@ -442,7 +491,7 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType,
   QualType SrcConstruct = Self.Context.VoidTy;
   QualType DestConstruct = Self.Context.VoidTy;
   ASTContext &Context = Self.Context;
-  for (llvm::SmallVector<Qualifiers, 8>::reverse_iterator i1 = cv1.rbegin(),
+  for (SmallVector<Qualifiers, 8>::reverse_iterator i1 = cv1.rbegin(),
                                                           i2 = cv2.rbegin();
        i1 != cv1.rend(); ++i1, ++i2) {
     SrcConstruct
@@ -461,13 +510,9 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType,
 /// CheckDynamicCast - Check that a dynamic_cast\<DestType\>(SrcExpr) is valid.
 /// Refer to C++ 5.2.7 for details. Dynamic casts are used mostly for runtime-
 /// checked downcasts in class hierarchies.
-static void
-CheckDynamicCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                 ExprValueKind &VK, const SourceRange &OpRange,
-                 const SourceRange &DestRange, CastKind &Kind,
-                 CXXCastPath &BasePath) {
-  QualType OrigDestType = DestType, OrigSrcType = SrcExpr.get()->getType();
-  DestType = Self.Context.getCanonicalType(DestType);
+void CastOperation::CheckDynamicCast() {
+  QualType OrigSrcType = SrcExpr.get()->getType();
+  QualType DestType = Self.Context.getCanonicalType(this->DestType);
 
   // C++ 5.2.7p1: T shall be a pointer or reference to a complete class type,
   //   or "pointer to cv void".
@@ -479,12 +524,9 @@ CheckDynamicCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
     DestPointee = DestPointer->getPointeeType();
   } else if ((DestReference = DestType->getAs<ReferenceType>())) {
     DestPointee = DestReference->getPointeeType();
-    VK = isa<LValueReferenceType>(DestReference) ? VK_LValue 
-       : isa<RValueReferenceType>(DestReference) ? VK_XValue
-       : VK_RValue;
   } else {
     Self.Diag(OpRange.getBegin(), diag::err_bad_dynamic_cast_not_ref_or_ptr)
-      << OrigDestType << DestRange;
+      << this->DestType << DestRange;
     return;
   }
 
@@ -519,7 +561,7 @@ CheckDynamicCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
   } else if (DestReference->isLValueReferenceType()) {
     if (!SrcExpr.get()->isLValue()) {
       Self.Diag(OpRange.getBegin(), diag::err_bad_cxx_cast_rvalue)
-        << CT_Dynamic << OrigSrcType << OrigDestType << OpRange;
+        << CT_Dynamic << OrigSrcType << this->DestType << OpRange;
     }
     SrcPointee = SrcType;
   } else {
@@ -547,7 +589,7 @@ CheckDynamicCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
   // C++ 5.2.7p1: The dynamic_cast operator shall not cast away constness.
   if (!DestPointee.isAtLeastAsQualifiedAs(SrcPointee)) {
     Self.Diag(OpRange.getBegin(), diag::err_bad_cxx_cast_qualifiers_away)
-      << CT_Dynamic << OrigSrcType << OrigDestType << OpRange;
+      << CT_Dynamic << OrigSrcType << this->DestType << OpRange;
     return;
   }
 
@@ -595,11 +637,8 @@ CheckDynamicCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
 /// like this:
 /// const char *str = "literal";
 /// legacy_function(const_cast\<char*\>(str));
-void
-CheckConstCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, ExprValueKind &VK,
-               const SourceRange &OpRange, const SourceRange &DestRange) {
-  VK = Expr::getValueKindForType(DestType);
-  if (VK == VK_RValue) {
+void CastOperation::CheckConstCast() {
+  if (ValueKind == VK_RValue && !isPlaceholder(BuiltinType::Overload)) {
     SrcExpr = Self.DefaultFunctionArrayLvalueConversion(SrcExpr.take());
     if (SrcExpr.isInvalid()) // if conversion failed, don't report another error
       return;
@@ -617,12 +656,8 @@ CheckConstCast(Sema &Self, ExprResult &SrcExpr, QualType DestType, ExprValueKind
 /// Refer to C++ 5.2.10 for details. reinterpret_cast is typically used in code
 /// like this:
 /// char *bytes = reinterpret_cast\<char*\>(int_ptr);
-void
-CheckReinterpretCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                     ExprValueKind &VK, const SourceRange &OpRange,
-                     const SourceRange &DestRange, CastKind &Kind) {
-  VK = Expr::getValueKindForType(DestType);
-  if (VK == VK_RValue) {
+void CastOperation::CheckReinterpretCast() {
+  if (ValueKind == VK_RValue && !isPlaceholder(BuiltinType::Overload)) {
     SrcExpr = Self.DefaultFunctionArrayLvalueConversion(SrcExpr.take());
     if (SrcExpr.isInvalid()) // if conversion failed, don't report another error
       return;
@@ -647,10 +682,7 @@ CheckReinterpretCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
       diagnoseBadCast(Self, msg, CT_Reinterpret, OpRange, SrcExpr.get(), DestType);
     }
   } else if (tcr == TC_Success && Self.getLangOptions().ObjCAutoRefCount) {
-    Expr *Exp = SrcExpr.get();
-    // Note that Exp does not change with CCK_OtherCast cast type
-    Self.CheckObjCARCConversion(OpRange, DestType,
-                                Exp, Sema::CCK_OtherCast);
+    checkObjCARCConversion(Sema::CCK_OtherCast);
   }
 }
 
@@ -658,36 +690,34 @@ CheckReinterpretCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
 /// CheckStaticCast - Check that a static_cast\<DestType\>(SrcExpr) is valid.
 /// Refer to C++ 5.2.9 for details. Static casts are mostly used for making
 /// implicit conversions explicit and getting rid of data loss warnings.
-void
-CheckStaticCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                ExprValueKind &VK, const SourceRange &OpRange,
-                CastKind &Kind, CXXCastPath &BasePath) {
+void CastOperation::CheckStaticCast() {
+  if (isPlaceholder()) {
+    checkNonOverloadPlaceholders();
+    if (SrcExpr.isInvalid())
+      return;
+  }
+
   // This test is outside everything else because it's the only case where
   // a non-lvalue-reference target type does not lead to decay.
   // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
   if (DestType->isVoidType()) {
-    SrcExpr = Self.IgnoredValueConversions(SrcExpr.take());
-    if (SrcExpr.isInvalid()) // if conversion failed, don't report another error
-      return;
-    if (SrcExpr.get()->getType() == Self.Context.OverloadTy) {
-      ExprResult SingleFunctionExpression = 
-         Self.ResolveAndFixSingleFunctionTemplateSpecialization(SrcExpr.get(), 
+    Kind = CK_ToVoid;
+
+    if (claimPlaceholder(BuiltinType::Overload)) {
+      Self.ResolveAndFixSingleFunctionTemplateSpecialization(SrcExpr, 
                 false, // Decay Function to ptr 
                 true, // Complain
                 OpRange, DestType, diag::err_bad_static_cast_overload);
-        if (SingleFunctionExpression.isUsable())
-        {
-          SrcExpr = SingleFunctionExpression;
-          Kind = CK_ToVoid;
-        }
+      if (SrcExpr.isInvalid())
+        return;
     }
-    else
-      Kind = CK_ToVoid;
+
+    SrcExpr = Self.IgnoredValueConversions(SrcExpr.take());
     return;
   }
 
-  VK = Expr::getValueKindForType(DestType);
-  if (VK == VK_RValue && !DestType->isRecordType()) {
+  if (ValueKind == VK_RValue && !DestType->isRecordType() &&
+      !isPlaceholder(BuiltinType::Overload)) {
     SrcExpr = Self.DefaultFunctionArrayLvalueConversion(SrcExpr.take());
     if (SrcExpr.isInvalid()) // if conversion failed, don't report another error
       return;
@@ -711,16 +741,12 @@ CheckStaticCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
     }
   } else if (tcr == TC_Success) {
     if (Kind == CK_BitCast)
-      Self.CheckCastAlign(SrcExpr.get(), DestType, OpRange);
-    if (Self.getLangOptions().ObjCAutoRefCount) {
-      Expr *Exp = SrcExpr.get();
-      // Note that Exp does not change with CCK_OtherCast cast type
-      Self.CheckObjCARCConversion(OpRange, DestType,
-                                  Exp, Sema::CCK_OtherCast);
-    }
+      checkCastAlign();
+    if (Self.getLangOptions().ObjCAutoRefCount)
+      checkObjCARCConversion(Sema::CCK_OtherCast);
+  } else if (Kind == CK_BitCast) {
+    checkCastAlign();
   }
-  else if (Kind == CK_BitCast)
-    Self.CheckCastAlign(SrcExpr.get(), DestType, OpRange);
 }
 
 /// TryStaticCast - Check if a static cast can be performed, and do so if
@@ -815,11 +841,12 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
   // The same goes for reverse floating point promotion/conversion and
   // floating-integral conversions. Again, only floating->enum is relevant.
   if (DestType->isEnumeralType()) {
-    if (SrcType->isComplexType() || SrcType->isVectorType()) {
-      // Fall through - these cannot be converted.
-    } else if (SrcType->isArithmeticType() || SrcType->isEnumeralType()) {
+    if (SrcType->isIntegralOrEnumerationType()) {
       Kind = CK_IntegralCast;
       return TC_Success;
+    } else if (SrcType->isRealFloatingType())   {
+      Kind = CK_FloatingToIntegral;
+      return TC_Success;
     }
   }
 
@@ -870,7 +897,7 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
       else if (DestType->isObjCObjectPointerType()) {
         // allow both c-style cast and static_cast of objective-c pointers as 
         // they are pervasive.
-        Kind = CK_AnyPointerToObjCPointerCast;
+        Kind = CK_CPointerToObjCPointerCast;
         return TC_Success;
       }
       else if (CStyle && DestType->isBlockPointerType()) {
@@ -1373,7 +1400,7 @@ void Sema::CheckCompatibleReinterpretCast(QualType SrcType, QualType DestType,
                         diag::warn_undefined_reinterpret_cast;
 
   if (Diags.getDiagnosticLevel(DiagID, Range.getBegin()) ==
-          Diagnostic::Ignored) {
+          DiagnosticsEngine::Ignored) {
     return;
   }
 
@@ -1430,17 +1457,19 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
   // Is the source an overloaded name? (i.e. &foo)
   // If so, reinterpret_cast can not help us here (13.4, p1, bullet 5) ...
   if (SrcType == Self.Context.OverloadTy) {
-    // ... unless foo<int> resolves to an lvalue unambiguously
-    ExprResult SingleFunctionExpr = 
-        Self.ResolveAndFixSingleFunctionTemplateSpecialization(SrcExpr.get(), 
+    // ... unless foo<int> resolves to an lvalue unambiguously.
+    // TODO: what if this fails because of DiagnoseUseOfDecl or something
+    // like it?
+    ExprResult SingleFunctionExpr = SrcExpr;
+    if (Self.ResolveAndFixSingleFunctionTemplateSpecialization(
+          SingleFunctionExpr,
           Expr::getValueKindForType(DestType) == VK_RValue // Convert Fun to Ptr 
-        );
-    if (SingleFunctionExpr.isUsable()) {
+        ) && SingleFunctionExpr.isUsable()) {
       SrcExpr = move(SingleFunctionExpr);
       SrcType = SrcExpr.get()->getType();
-    }      
-    else  
+    } else {
       return TC_NotApplicable;
+    }
   }
 
   if (const ReferenceType *DestTypeTmp = DestType->getAs<ReferenceType>()) {
@@ -1592,7 +1621,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
     //   integral type size doesn't matter.
     if ((Self.Context.getTypeSize(SrcType) >
          Self.Context.getTypeSize(DestType)) &&
-         !Self.getLangOptions().Microsoft) {
+         !Self.getLangOptions().MicrosoftExt) {
       msg = diag::err_bad_reinterpret_cast_small_int;
       return TC_Failed;
     }
@@ -1629,16 +1658,28 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
       (DestType->isBlockPointerType() && SrcType->isObjCObjectPointerType()))
     return TC_NotApplicable;
 
+  if (IsLValueCast) {
+    Kind = CK_LValueBitCast;
+  } else if (DestType->isObjCObjectPointerType()) {
+    Kind = Self.PrepareCastToObjCObjectPointer(SrcExpr);
+  } else if (DestType->isBlockPointerType()) {
+    if (!SrcType->isBlockPointerType()) {
+      Kind = CK_AnyPointerToBlockPointerCast;
+    } else {
+      Kind = CK_BitCast;
+    }
+  } else {
+    Kind = CK_BitCast;
+  }
+
   // Any pointer can be cast to an Objective-C pointer type with a C-style
   // cast.
   if (CStyle && DestType->isObjCObjectPointerType()) {
-    Kind = CK_AnyPointerToObjCPointerCast;
     return TC_Success;
   }
     
   // Not casting away constness, so the only remaining check is for compatible
   // pointer categories.
-  Kind = IsLValueCast? CK_LValueBitCast : CK_BitCast;
 
   if (SrcType->isFunctionPointerType()) {
     if (DestType->isFunctionPointerType()) {
@@ -1673,65 +1714,64 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
   return TC_Success;
 }                                     
 
-ExprResult
-Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, ExprValueKind &VK,
-                         Expr *CastExpr, CastKind &Kind, 
-                         CXXCastPath &BasePath,
-                         bool FunctionalStyle) {
-  // This test is outside everything else because it's the only case where
-  // a non-lvalue-reference target type does not lead to decay.
-  // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
-  if (CastTy->isVoidType()) {
-    Kind = CK_ToVoid;
-
-    ExprResult CastExprRes = IgnoredValueConversions(CastExpr);    
-    if (CastExprRes.isInvalid())
-      return ExprError();
-    CastExpr = CastExprRes.take();
-
-    if (CastExpr->getType() == Context.BoundMemberTy)
-      return CheckPlaceholderExpr(CastExpr); // will always fail
-
-    if (CastExpr->getType() == Context.OverloadTy) {
-      ExprResult SingleFunctionExpr = 
-        ResolveAndFixSingleFunctionTemplateSpecialization(
-                  CastExpr, /* Decay Function to ptr */ false, 
-                  /* Complain */ true, R, CastTy, 
-                  diag::err_bad_cstyle_cast_overload);
-      if (SingleFunctionExpr.isInvalid())
-        return ExprError();
-      CastExpr = SingleFunctionExpr.take();
+void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle) {
+  // Handle placeholders.
+  if (isPlaceholder()) {
+    // C-style casts can resolve __unknown_any types.
+    if (claimPlaceholder(BuiltinType::UnknownAny)) {
+      SrcExpr = Self.checkUnknownAnyCast(DestRange, DestType,
+                                         SrcExpr.get(), Kind,
+                                         ValueKind, BasePath);
+      return;
     }
 
-    assert(!CastExpr->getType()->isPlaceholderType());
+    checkNonOverloadPlaceholders();
+    if (SrcExpr.isInvalid())
+      return;
+  }  
 
-    return Owned(CastExpr);
+  // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
+  // This test is outside everything else because it's the only case where
+  // a non-lvalue-reference target type does not lead to decay.
+  if (DestType->isVoidType()) {
+    Kind = CK_ToVoid;
+
+    if (claimPlaceholder(BuiltinType::Overload)) {
+      Self.ResolveAndFixSingleFunctionTemplateSpecialization(
+                  SrcExpr, /* Decay Function to ptr */ false, 
+                  /* Complain */ true, DestRange, DestType,
+                  diag::err_bad_cstyle_cast_overload);
+      if (SrcExpr.isInvalid())
+        return;
+    }
+
+    SrcExpr = Self.IgnoredValueConversions(SrcExpr.take());
+    if (SrcExpr.isInvalid())
+      return;
+
+    return;
   }
 
-  // Make sure we determine the value kind before we bail out for
-  // dependent types.
-  VK = Expr::getValueKindForType(CastTy);
-
   // If the type is dependent, we won't do any other semantic analysis now.
-  if (CastTy->isDependentType() || CastExpr->isTypeDependent()) {
-    Kind = CK_Dependent;
-    return Owned(CastExpr);
+  if (DestType->isDependentType() || SrcExpr.get()->isTypeDependent()) {
+    assert(Kind == CK_Dependent);
+    return;
   }
 
-  if (VK == VK_RValue && !CastTy->isRecordType()) {
-    ExprResult CastExprRes = DefaultFunctionArrayLvalueConversion(CastExpr);
-    if (CastExprRes.isInvalid())
-      return ExprError();
-    CastExpr = CastExprRes.take();
+  if (ValueKind == VK_RValue && !DestType->isRecordType() &&
+      !isPlaceholder(BuiltinType::Overload)) {
+    SrcExpr = Self.DefaultFunctionArrayLvalueConversion(SrcExpr.take());
+    if (SrcExpr.isInvalid())
+      return;
   }
 
   // AltiVec vector initialization with a single literal.
-  if (const VectorType *vecTy = CastTy->getAs<VectorType>())
+  if (const VectorType *vecTy = DestType->getAs<VectorType>())
     if (vecTy->getVectorKind() == VectorType::AltiVecVector
-        && (CastExpr->getType()->isIntegerType()
-            || CastExpr->getType()->isFloatingType())) {
+        && (SrcExpr.get()->getType()->isIntegerType()
+            || SrcExpr.get()->getType()->isFloatingType())) {
       Kind = CK_VectorSplat;
-      return Owned(CastExpr);
+      return;
     }
 
   // C++ [expr.cast]p5: The conversions performed by
@@ -1746,8 +1786,8 @@ Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, ExprValueKind &VK,
   //   even if a cast resulting from that interpretation is ill-formed.
   // In plain language, this means trying a const_cast ...
   unsigned msg = diag::err_bad_cxx_cast_generic;
-  TryCastResult tcr = TryConstCast(*this, CastExpr, CastTy, /*CStyle*/true,
-                                   msg);
+  TryCastResult tcr = TryConstCast(Self, SrcExpr.get(), DestType,
+                                   /*CStyle*/true, msg);
   if (tcr == TC_Success)
     Kind = CK_NoOp;
 
@@ -1756,47 +1796,274 @@ Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, ExprValueKind &VK,
                      : Sema::CCK_CStyleCast;
   if (tcr == TC_NotApplicable) {
     // ... or if that is not possible, a static_cast, ignoring const, ...
-    ExprResult CastExprRes = Owned(CastExpr);
-    tcr = TryStaticCast(*this, CastExprRes, CastTy, CCK, R, msg, Kind, 
-                        BasePath);
-    if (CastExprRes.isInvalid())
-      return ExprError();
-    CastExpr = CastExprRes.take();
+    tcr = TryStaticCast(Self, SrcExpr, DestType, CCK, OpRange,
+                        msg, Kind, BasePath);
+    if (SrcExpr.isInvalid())
+      return;
+
     if (tcr == TC_NotApplicable) {
       // ... and finally a reinterpret_cast, ignoring const.
-      CastExprRes = Owned(CastExpr);
-      tcr = TryReinterpretCast(*this, CastExprRes, CastTy, /*CStyle*/true, R,
-                               msg, Kind);
-      if (CastExprRes.isInvalid())
-        return ExprError();
-      CastExpr = CastExprRes.take();
+      tcr = TryReinterpretCast(Self, SrcExpr, DestType, /*CStyle*/true,
+                               OpRange, msg, Kind);
+      if (SrcExpr.isInvalid())
+        return;
     }
   }
 
-  if (getLangOptions().ObjCAutoRefCount && tcr == TC_Success)
-    CheckObjCARCConversion(R, CastTy, CastExpr, CCK);
+  if (Self.getLangOptions().ObjCAutoRefCount && tcr == TC_Success)
+    checkObjCARCConversion(CCK);
 
   if (tcr != TC_Success && msg != 0) {
-    if (CastExpr->getType() == Context.OverloadTy) {
+    if (SrcExpr.get()->getType() == Self.Context.OverloadTy) {
       DeclAccessPair Found;
-      FunctionDecl *Fn = ResolveAddressOfOverloadedFunction(CastExpr,
-                                CastTy,
-                                /* Complain */ true,
+      FunctionDecl *Fn = Self.ResolveAddressOfOverloadedFunction(SrcExpr.get(),
+                                DestType,
+                                /*Complain*/ true,
                                 Found);
       
       assert(!Fn && "cast failed but able to resolve overload expression!!");
       (void)Fn;
 
     } else {
-      diagnoseBadCast(*this, msg, (FunctionalStyle ? CT_Functional : CT_CStyle),
-                      R, CastExpr, CastTy);
+      diagnoseBadCast(Self, msg, (FunctionalStyle ? CT_Functional : CT_CStyle),
+                      OpRange, SrcExpr.get(), DestType);
+    }
+  } else if (Kind == CK_BitCast) {
+    checkCastAlign();
+  }
+
+  // Clear out SrcExpr if there was a fatal error.
+  if (tcr != TC_Success)
+    SrcExpr = ExprError();
+}
+
+/// Check the semantics of a C-style cast operation, in C.
+void CastOperation::CheckCStyleCast() {
+  assert(!Self.getLangOptions().CPlusPlus);
+
+  // Handle placeholders.
+  if (isPlaceholder()) {
+    // C-style casts can resolve __unknown_any types.
+    if (claimPlaceholder(BuiltinType::UnknownAny)) {
+      SrcExpr = Self.checkUnknownAnyCast(DestRange, DestType,
+                                         SrcExpr.get(), Kind,
+                                         ValueKind, BasePath);
+      return;
+    }
+
+    // We allow overloads in C, but we don't allow them to be resolved
+    // by anything except calls.
+    SrcExpr = Self.CheckPlaceholderExpr(SrcExpr.take());
+    if (SrcExpr.isInvalid())
+      return;
+  }  
+
+  assert(!isPlaceholder());
+
+  // C99 6.5.4p2: the cast type needs to be void or scalar and the expression
+  // type needs to be scalar.
+  if (DestType->isVoidType()) {
+    // We don't necessarily do lvalue-to-rvalue conversions on this.
+    SrcExpr = Self.IgnoredValueConversions(SrcExpr.take());
+    if (SrcExpr.isInvalid())
+      return;
+
+    // Cast to void allows any expr type.
+    Kind = CK_ToVoid;
+    return;
+  }
+
+  SrcExpr = Self.DefaultFunctionArrayLvalueConversion(SrcExpr.take());
+  if (SrcExpr.isInvalid())
+    return;
+  QualType SrcType = SrcExpr.get()->getType();
+
+  if (Self.RequireCompleteType(OpRange.getBegin(), DestType,
+                               diag::err_typecheck_cast_to_incomplete)) {
+    SrcExpr = ExprError();
+    return;
+  }
+
+  if (!DestType->isScalarType() && !DestType->isVectorType()) {
+    const RecordType *DestRecordTy = DestType->getAs<RecordType>();
+
+    if (DestRecordTy && Self.Context.hasSameUnqualifiedType(DestType, SrcType)){
+      // GCC struct/union extension: allow cast to self.
+      Self.Diag(OpRange.getBegin(), diag::ext_typecheck_cast_nonscalar)
+        << DestType << SrcExpr.get()->getSourceRange();
+      Kind = CK_NoOp;
+      return;
+    }
+
+    // GCC's cast to union extension.
+    if (DestRecordTy && DestRecordTy->getDecl()->isUnion()) {
+      RecordDecl *RD = DestRecordTy->getDecl();
+      RecordDecl::field_iterator Field, FieldEnd;
+      for (Field = RD->field_begin(), FieldEnd = RD->field_end();
+           Field != FieldEnd; ++Field) {
+        if (Self.Context.hasSameUnqualifiedType(Field->getType(), SrcType) &&
+            !Field->isUnnamedBitfield()) {
+          Self.Diag(OpRange.getBegin(), diag::ext_typecheck_cast_to_union)
+            << SrcExpr.get()->getSourceRange();
+          break;
+        }
+      }
+      if (Field == FieldEnd) {
+        Self.Diag(OpRange.getBegin(), diag::err_typecheck_cast_to_union_no_type)
+          << SrcType << SrcExpr.get()->getSourceRange();
+        SrcExpr = ExprError();
+        return;
+      }
+      Kind = CK_ToUnion;
+      return;
+    }
+
+    // Reject any other conversions to non-scalar types.
+    Self.Diag(OpRange.getBegin(), diag::err_typecheck_cond_expect_scalar)
+      << DestType << SrcExpr.get()->getSourceRange();
+    SrcExpr = ExprError();
+    return;
+  }
+
+  // The type we're casting to is known to be a scalar or vector.
+
+  // Require the operand to be a scalar or vector.
+  if (!SrcType->isScalarType() && !SrcType->isVectorType()) {
+    Self.Diag(SrcExpr.get()->getExprLoc(),
+              diag::err_typecheck_expect_scalar_operand)
+      << SrcType << SrcExpr.get()->getSourceRange();
+    SrcExpr = ExprError();
+    return;
+  }
+
+  if (DestType->isExtVectorType()) {
+    SrcExpr = Self.CheckExtVectorCast(OpRange, DestType, SrcExpr.take(), Kind);
+    return;
+  }
+
+  if (const VectorType *DestVecTy = DestType->getAs<VectorType>()) {
+    if (DestVecTy->getVectorKind() == VectorType::AltiVecVector &&
+          (SrcType->isIntegerType() || SrcType->isFloatingType())) {
+      Kind = CK_VectorSplat;
+    } else if (Self.CheckVectorCast(OpRange, DestType, SrcType, Kind)) {
+      SrcExpr = ExprError();
+    }
+    return;
+  }
+
+  if (SrcType->isVectorType()) {
+    if (Self.CheckVectorCast(OpRange, SrcType, DestType, Kind))
+      SrcExpr = ExprError();
+    return;
+  }
+
+  // The source and target types are both scalars, i.e.
+  //   - arithmetic types (fundamental, enum, and complex)
+  //   - all kinds of pointers
+  // Note that member pointers were filtered out with C++, above.
+
+  if (isa<ObjCSelectorExpr>(SrcExpr.get())) {
+    Self.Diag(SrcExpr.get()->getExprLoc(), diag::err_cast_selector_expr);
+    SrcExpr = ExprError();
+    return;
+  }
+
+  // If either type is a pointer, the other type has to be either an
+  // integer or a pointer.
+  if (!DestType->isArithmeticType()) {
+    if (!SrcType->isIntegralType(Self.Context) && SrcType->isArithmeticType()) {
+      Self.Diag(SrcExpr.get()->getExprLoc(),
+                diag::err_cast_pointer_from_non_pointer_int)
+        << SrcType << SrcExpr.get()->getSourceRange();
+      SrcExpr = ExprError();
+      return;
+    }
+  } else if (!SrcType->isArithmeticType()) {
+    if (!DestType->isIntegralType(Self.Context) &&
+        DestType->isArithmeticType()) {
+      Self.Diag(SrcExpr.get()->getLocStart(),
+           diag::err_cast_pointer_to_non_pointer_int)
+        << SrcType << SrcExpr.get()->getSourceRange();
+      SrcExpr = ExprError();
+      return;
     }
   }
-  else if (Kind == CK_BitCast)
-    CheckCastAlign(CastExpr, CastTy, R);
 
-  if (tcr != TC_Success)
+  // ARC imposes extra restrictions on casts.
+  if (Self.getLangOptions().ObjCAutoRefCount) {
+    checkObjCARCConversion(Sema::CCK_CStyleCast);
+    if (SrcExpr.isInvalid())
+      return;
+    
+    if (const PointerType *CastPtr = DestType->getAs<PointerType>()) {
+      if (const PointerType *ExprPtr = SrcType->getAs<PointerType>()) {
+        Qualifiers CastQuals = CastPtr->getPointeeType().getQualifiers();
+        Qualifiers ExprQuals = ExprPtr->getPointeeType().getQualifiers();
+        if (CastPtr->getPointeeType()->isObjCLifetimeType() && 
+            ExprPtr->getPointeeType()->isObjCLifetimeType() &&
+            !CastQuals.compatiblyIncludesObjCLifetime(ExprQuals)) {
+          Self.Diag(SrcExpr.get()->getLocStart(), 
+                    diag::err_typecheck_incompatible_ownership)
+            << SrcType << DestType << Sema::AA_Casting
+            << SrcExpr.get()->getSourceRange();
+          return;
+        }
+      }
+    } 
+    else if (!Self.CheckObjCARCUnavailableWeakConversion(DestType, SrcType)) {
+      Self.Diag(SrcExpr.get()->getLocStart(), 
+                diag::err_arc_convesion_of_weak_unavailable)
+        << 1 << SrcType << DestType << SrcExpr.get()->getSourceRange();
+      SrcExpr = ExprError();
+      return;
+    }
+  }
+  
+  Kind = Self.PrepareScalarCast(SrcExpr, DestType);
+  if (SrcExpr.isInvalid())
+    return;
+
+  if (Kind == CK_BitCast)
+    checkCastAlign();
+}
+
+ExprResult Sema::BuildCStyleCastExpr(SourceLocation LPLoc,
+                                     TypeSourceInfo *CastTypeInfo,
+                                     SourceLocation RPLoc,
+                                     Expr *CastExpr) {
+  CastOperation Op(*this, CastTypeInfo->getType(), CastExpr);  
+  Op.DestRange = CastTypeInfo->getTypeLoc().getSourceRange();
+  Op.OpRange = SourceRange(LPLoc, CastExpr->getLocEnd());
+
+  if (getLangOptions().CPlusPlus) {
+    Op.CheckCXXCStyleCast(/*FunctionalStyle=*/ false);
+  } else {
+    Op.CheckCStyleCast();
+  }
+
+  if (Op.SrcExpr.isInvalid())
     return ExprError();
 
-  return Owned(CastExpr);
+  return Owned(CStyleCastExpr::Create(Context, Op.ResultType, Op.ValueKind,
+                                      Op.Kind, Op.SrcExpr.take(), &Op.BasePath,
+                                      CastTypeInfo, LPLoc, RPLoc));
+}
+
+ExprResult Sema::BuildCXXFunctionalCastExpr(TypeSourceInfo *CastTypeInfo,
+                                            SourceLocation LPLoc,
+                                            Expr *CastExpr,
+                                            SourceLocation RPLoc) {
+  CastOperation Op(*this, CastTypeInfo->getType(), CastExpr);
+  Op.DestRange = CastTypeInfo->getTypeLoc().getSourceRange();
+  Op.OpRange = SourceRange(Op.DestRange.getBegin(), CastExpr->getLocEnd());
+
+  Op.CheckCXXCStyleCast(/*FunctionalStyle=*/ true);
+  if (Op.SrcExpr.isInvalid())
+    return ExprError();
+
+  return Owned(CXXFunctionalCastExpr::Create(Context, Op.ResultType,
+                                             Op.ValueKind, CastTypeInfo,
+                                             Op.DestRange.getBegin(),
+                                             Op.Kind, Op.SrcExpr.take(),
+                                             &Op.BasePath, RPLoc));
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp
index b032b8a0d4c0..eaf7bfa14754 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp
@@ -12,8 +12,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Sema/Initialization.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Initialization.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Analysis/Analyses/FormatString.h"
 #include "clang/AST/ASTContext.h"
@@ -87,6 +89,19 @@ static bool checkArgCount(Sema &S, CallExpr *call, unsigned desiredArgCount) {
     << call->getArg(1)->getSourceRange();
 }
 
+/// CheckBuiltinAnnotationString - Checks that string argument to the builtin
+/// annotation is a non wide string literal.
+static bool CheckBuiltinAnnotationString(Sema &S, Expr *Arg) {
+  Arg = Arg->IgnoreParenCasts();
+  StringLiteral *Literal = dyn_cast<StringLiteral>(Arg);
+  if (!Literal || !Literal->isAscii()) {
+    S.Diag(Arg->getLocStart(), diag::err_builtin_annotation_not_string_constant)
+      << Arg->getSourceRange();
+    return true;
+  }
+  return false;
+}
+
 ExprResult
 Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   ExprResult TheCallResult(Owned(TheCall));
@@ -183,12 +198,38 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   case Builtin::BI__sync_lock_release:
   case Builtin::BI__sync_swap:
     return SemaBuiltinAtomicOverloaded(move(TheCallResult));
+  case Builtin::BI__atomic_load:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Load);
+  case Builtin::BI__atomic_store:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Store);
+  case Builtin::BI__atomic_exchange:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Xchg);
+  case Builtin::BI__atomic_compare_exchange_strong:
+    return SemaAtomicOpsOverloaded(move(TheCallResult),
+                                   AtomicExpr::CmpXchgStrong);
+  case Builtin::BI__atomic_compare_exchange_weak:
+    return SemaAtomicOpsOverloaded(move(TheCallResult),
+                                   AtomicExpr::CmpXchgWeak);
+  case Builtin::BI__atomic_fetch_add:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Add);
+  case Builtin::BI__atomic_fetch_sub:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Sub);
+  case Builtin::BI__atomic_fetch_and:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::And);
+  case Builtin::BI__atomic_fetch_or:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Or);
+  case Builtin::BI__atomic_fetch_xor:
+    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::Xor);
+  case Builtin::BI__builtin_annotation:
+    if (CheckBuiltinAnnotationString(*this, TheCall->getArg(1)))
+      return ExprError();
+    break;
   }
   
   // Since the target specific builtins for each arch overlap, only check those
   // of the arch we are compiling for.
   if (BuiltinID >= Builtin::FirstTSBuiltin) {
-    switch (Context.Target.getTriple().getArch()) {
+    switch (Context.getTargetInfo().getTriple().getArch()) {
       case llvm::Triple::arm:
       case llvm::Triple::thumb:
         if (CheckARMBuiltinFunctionCall(BuiltinID, TheCall))
@@ -319,7 +360,7 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall) {
                           TheCall->getCallee()->getLocStart());
   }
 
-  // Memset/memcpy/memmove handling
+  // Builtin handling
   int CMF = -1;
   switch (FDecl->getBuiltinID()) {
   case Builtin::BI__builtin_memset:
@@ -339,7 +380,40 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall) {
   case Builtin::BImemmove:
     CMF = CMF_Memmove;
     break;
+
+  case Builtin::BIstrlcpy:
+  case Builtin::BIstrlcat:
+    CheckStrlcpycatArguments(TheCall, FnInfo);
+    break;
     
+  case Builtin::BI__builtin_memcmp:
+    CMF = CMF_Memcmp;
+    break;
+    
+  case Builtin::BI__builtin_strncpy:
+  case Builtin::BI__builtin___strncpy_chk:
+  case Builtin::BIstrncpy:
+    CMF = CMF_Strncpy;
+    break;
+
+  case Builtin::BI__builtin_strncmp:
+    CMF = CMF_Strncmp;
+    break;
+
+  case Builtin::BI__builtin_strncasecmp:
+    CMF = CMF_Strncasecmp;
+    break;
+
+  case Builtin::BI__builtin_strncat:
+  case Builtin::BIstrncat:
+    CMF = CMF_Strncat;
+    break;
+
+  case Builtin::BI__builtin_strndup:
+  case Builtin::BIstrndup:
+    CMF = CMF_Strndup;
+    break;
+
   default:
     if (FDecl->getLinkage() == ExternalLinkage &&
         (!getLangOptions().CPlusPlus || FDecl->isExternC())) {
@@ -349,12 +423,25 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall) {
         CMF = CMF_Memcpy;
       else if (FnInfo->isStr("memmove"))
         CMF = CMF_Memmove;
+      else if (FnInfo->isStr("memcmp"))
+        CMF = CMF_Memcmp;
+      else if (FnInfo->isStr("strncpy"))
+        CMF = CMF_Strncpy;
+      else if (FnInfo->isStr("strncmp"))
+        CMF = CMF_Strncmp;
+      else if (FnInfo->isStr("strncasecmp"))
+        CMF = CMF_Strncasecmp;
+      else if (FnInfo->isStr("strncat"))
+        CMF = CMF_Strncat;
+      else if (FnInfo->isStr("strndup"))
+        CMF = CMF_Strndup;
     }
     break;
   }
    
+  // Memset/memcpy/memmove handling
   if (CMF != -1)
-    CheckMemsetcpymoveArguments(TheCall, CheckedMemoryFunction(CMF), FnInfo);
+    CheckMemaccessArguments(TheCall, CheckedMemoryFunction(CMF), FnInfo);
 
   return false;
 }
@@ -384,6 +471,170 @@ bool Sema::CheckBlockCall(NamedDecl *NDecl, CallExpr *TheCall) {
   return false;
 }
 
+ExprResult
+Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult, AtomicExpr::AtomicOp Op) {
+  CallExpr *TheCall = cast<CallExpr>(TheCallResult.get());
+  DeclRefExpr *DRE =cast<DeclRefExpr>(TheCall->getCallee()->IgnoreParenCasts());
+
+  // All these operations take one of the following four forms:
+  // T   __atomic_load(_Atomic(T)*, int)                              (loads)
+  // T*  __atomic_add(_Atomic(T*)*, ptrdiff_t, int)         (pointer add/sub)
+  // int __atomic_compare_exchange_strong(_Atomic(T)*, T*, T, int, int)
+  //                                                                (cmpxchg)
+  // T   __atomic_exchange(_Atomic(T)*, T, int)             (everything else)
+  // where T is an appropriate type, and the int paremeterss are for orderings.
+  unsigned NumVals = 1;
+  unsigned NumOrders = 1;
+  if (Op == AtomicExpr::Load) {
+    NumVals = 0;
+  } else if (Op == AtomicExpr::CmpXchgWeak || Op == AtomicExpr::CmpXchgStrong) {
+    NumVals = 2;
+    NumOrders = 2;
+  }
+
+  if (TheCall->getNumArgs() < NumVals+NumOrders+1) {
+    Diag(TheCall->getLocEnd(), diag::err_typecheck_call_too_few_args)
+      << 0 << NumVals+NumOrders+1 << TheCall->getNumArgs()
+      << TheCall->getCallee()->getSourceRange();
+    return ExprError();
+  } else if (TheCall->getNumArgs() > NumVals+NumOrders+1) {
+    Diag(TheCall->getArg(NumVals+NumOrders+1)->getLocStart(),
+         diag::err_typecheck_call_too_many_args)
+      << 0 << NumVals+NumOrders+1 << TheCall->getNumArgs()
+      << TheCall->getCallee()->getSourceRange();
+    return ExprError();
+  }
+
+  // Inspect the first argument of the atomic operation.  This should always be
+  // a pointer to an _Atomic type.
+  Expr *Ptr = TheCall->getArg(0);
+  Ptr = DefaultFunctionArrayLvalueConversion(Ptr).get();
+  const PointerType *pointerType = Ptr->getType()->getAs<PointerType>();
+  if (!pointerType) {
+    Diag(DRE->getLocStart(), diag::err_atomic_op_needs_atomic)
+      << Ptr->getType() << Ptr->getSourceRange();
+    return ExprError();
+  }
+
+  QualType AtomTy = pointerType->getPointeeType();
+  if (!AtomTy->isAtomicType()) {
+    Diag(DRE->getLocStart(), diag::err_atomic_op_needs_atomic)
+      << Ptr->getType() << Ptr->getSourceRange();
+    return ExprError();
+  }
+  QualType ValType = AtomTy->getAs<AtomicType>()->getValueType();
+
+  if ((Op == AtomicExpr::Add || Op == AtomicExpr::Sub) &&
+      !ValType->isIntegerType() && !ValType->isPointerType()) {
+    Diag(DRE->getLocStart(), diag::err_atomic_op_needs_atomic_int_or_ptr)
+      << Ptr->getType() << Ptr->getSourceRange();
+    return ExprError();
+  }
+
+  if (!ValType->isIntegerType() &&
+      (Op == AtomicExpr::And || Op == AtomicExpr::Or || Op == AtomicExpr::Xor)){
+    Diag(DRE->getLocStart(), diag::err_atomic_op_logical_needs_atomic_int)
+      << Ptr->getType() << Ptr->getSourceRange();
+    return ExprError();
+  }
+
+  switch (ValType.getObjCLifetime()) {
+  case Qualifiers::OCL_None:
+  case Qualifiers::OCL_ExplicitNone:
+    // okay
+    break;
+
+  case Qualifiers::OCL_Weak:
+  case Qualifiers::OCL_Strong:
+  case Qualifiers::OCL_Autoreleasing:
+    Diag(DRE->getLocStart(), diag::err_arc_atomic_ownership)
+      << ValType << Ptr->getSourceRange();
+    return ExprError();
+  }
+
+  QualType ResultType = ValType;
+  if (Op == AtomicExpr::Store)
+    ResultType = Context.VoidTy;
+  else if (Op == AtomicExpr::CmpXchgWeak || Op == AtomicExpr::CmpXchgStrong)
+    ResultType = Context.BoolTy;
+
+  // The first argument --- the pointer --- has a fixed type; we
+  // deduce the types of the rest of the arguments accordingly.  Walk
+  // the remaining arguments, converting them to the deduced value type.
+  for (unsigned i = 1; i != NumVals+NumOrders+1; ++i) {
+    ExprResult Arg = TheCall->getArg(i);
+    QualType Ty;
+    if (i < NumVals+1) {
+      // The second argument to a cmpxchg is a pointer to the data which will
+      // be exchanged. The second argument to a pointer add/subtract is the
+      // amount to add/subtract, which must be a ptrdiff_t.  The third
+      // argument to a cmpxchg and the second argument in all other cases
+      // is the type of the value.
+      if (i == 1 && (Op == AtomicExpr::CmpXchgWeak ||
+                     Op == AtomicExpr::CmpXchgStrong))
+         Ty = Context.getPointerType(ValType.getUnqualifiedType());
+      else if (!ValType->isIntegerType() &&
+               (Op == AtomicExpr::Add || Op == AtomicExpr::Sub))
+        Ty = Context.getPointerDiffType();
+      else
+        Ty = ValType;
+    } else {
+      // The order(s) are always converted to int.
+      Ty = Context.IntTy;
+    }
+    InitializedEntity Entity =
+        InitializedEntity::InitializeParameter(Context, Ty, false);
+    Arg = PerformCopyInitialization(Entity, SourceLocation(), Arg);
+    if (Arg.isInvalid())
+      return true;
+    TheCall->setArg(i, Arg.get());
+  }
+
+  SmallVector<Expr*, 5> SubExprs;
+  SubExprs.push_back(Ptr);
+  if (Op == AtomicExpr::Load) {
+    SubExprs.push_back(TheCall->getArg(1)); // Order
+  } else if (Op != AtomicExpr::CmpXchgWeak && Op != AtomicExpr::CmpXchgStrong) {
+    SubExprs.push_back(TheCall->getArg(2)); // Order
+    SubExprs.push_back(TheCall->getArg(1)); // Val1
+  } else {
+    SubExprs.push_back(TheCall->getArg(3)); // Order
+    SubExprs.push_back(TheCall->getArg(1)); // Val1
+    SubExprs.push_back(TheCall->getArg(2)); // Val2
+    SubExprs.push_back(TheCall->getArg(4)); // OrderFail
+  }
+
+  return Owned(new (Context) AtomicExpr(TheCall->getCallee()->getLocStart(),
+                                        SubExprs.data(), SubExprs.size(),
+                                        ResultType, Op,
+                                        TheCall->getRParenLoc()));
+}
+
+
+/// checkBuiltinArgument - Given a call to a builtin function, perform
+/// normal type-checking on the given argument, updating the call in
+/// place.  This is useful when a builtin function requires custom
+/// type-checking for some of its arguments but not necessarily all of
+/// them.
+///
+/// Returns true on error.
+static bool checkBuiltinArgument(Sema &S, CallExpr *E, unsigned ArgIndex) {
+  FunctionDecl *Fn = E->getDirectCallee();
+  assert(Fn && "builtin call without direct callee!");
+
+  ParmVarDecl *Param = Fn->getParamDecl(ArgIndex);
+  InitializedEntity Entity =
+    InitializedEntity::InitializeParameter(S.Context, Param);
+
+  ExprResult Arg = E->getArg(0);
+  Arg = S.PerformCopyInitialization(Entity, SourceLocation(), Arg);
+  if (Arg.isInvalid())
+    return true;
+
+  E->setArg(ArgIndex, Arg.take());
+  return false;
+}
+
 /// SemaBuiltinAtomicOverloaded - We have a call to a function like
 /// __sync_fetch_and_add, which is an overloaded function based on the pointer
 /// type of its first argument.  The main ActOnCallExpr routines have already
@@ -441,6 +692,9 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
     return ExprError();
   }
 
+  // Strip any qualifiers off ValType.
+  ValType = ValType.getUnqualifiedType();
+
   // The majority of builtins return a value, but a few have special return
   // types, so allow them to override appropriately below.
   QualType ResultType = ValType;
@@ -494,7 +748,7 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   unsigned BuiltinID = FDecl->getBuiltinID();
   unsigned BuiltinIndex, NumFixed = 1;
   switch (BuiltinID) {
-  default: assert(0 && "Unknown overloaded atomic builtin!");
+  default: llvm_unreachable("Unknown overloaded atomic builtin!");
   case Builtin::BI__sync_fetch_and_add: BuiltinIndex = 0; break;
   case Builtin::BI__sync_fetch_and_sub: BuiltinIndex = 1; break;
   case Builtin::BI__sync_fetch_and_or:  BuiltinIndex = 2; break;
@@ -559,11 +813,10 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
 
     // GCC does an implicit conversion to the pointer or integer ValType.  This
     // can fail in some cases (1i -> int**), check for this error case now.
-    CastKind Kind = CK_Invalid;
-    ExprValueKind VK = VK_RValue;
-    CXXCastPath BasePath;
-    Arg = CheckCastTypes(Arg.get()->getLocStart(), Arg.get()->getSourceRange(), 
-                         ValType, Arg.take(), Kind, VK, BasePath);
+    // Initialize the argument.
+    InitializedEntity Entity = InitializedEntity::InitializeParameter(Context,
+                                                   ValType, /*consume*/ false);
+    Arg = PerformCopyInitialization(Entity, SourceLocation(), Arg);
     if (Arg.isInvalid())
       return ExprError();
 
@@ -573,17 +826,23 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
     // pass in 42.  The 42 gets converted to char.  This is even more strange
     // for things like 45.123 -> char, etc.
     // FIXME: Do this check.
-    Arg = ImpCastExprToType(Arg.take(), ValType, Kind, VK, &BasePath);
-    TheCall->setArg(i+1, Arg.get());
+    TheCall->setArg(i+1, Arg.take());
   }
 
-  // Switch the DeclRefExpr to refer to the new decl.
-  DRE->setDecl(NewBuiltinDecl);
-  DRE->setType(NewBuiltinDecl->getType());
+  ASTContext& Context = this->getASTContext();
+
+  // Create a new DeclRefExpr to refer to the new decl.
+  DeclRefExpr* NewDRE = DeclRefExpr::Create(
+      Context,
+      DRE->getQualifierLoc(),
+      NewBuiltinDecl,
+      DRE->getLocation(),
+      NewBuiltinDecl->getType(),
+      DRE->getValueKind());
 
   // Set the callee in the CallExpr.
   // FIXME: This leaks the original parens and implicit casts.
-  ExprResult PromotedCall = UsualUnaryConversions(DRE);
+  ExprResult PromotedCall = UsualUnaryConversions(NewDRE);
   if (PromotedCall.isInvalid())
     return ExprError();
   TheCall->setCallee(PromotedCall.take());
@@ -596,7 +855,6 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   return move(TheCallResult);
 }
 
-
 /// CheckObjCString - Checks that the argument to the builtin
 /// CFString constructor is correct
 /// Note: It might also make sense to do the UTF-16 conversion here (would
@@ -605,16 +863,16 @@ bool Sema::CheckObjCString(Expr *Arg) {
   Arg = Arg->IgnoreParenCasts();
   StringLiteral *Literal = dyn_cast<StringLiteral>(Arg);
 
-  if (!Literal || Literal->isWide()) {
+  if (!Literal || !Literal->isAscii()) {
     Diag(Arg->getLocStart(), diag::err_cfstring_literal_not_string_constant)
       << Arg->getSourceRange();
     return true;
   }
 
   if (Literal->containsNonAsciiOrNull()) {
-    llvm::StringRef String = Literal->getString();
+    StringRef String = Literal->getString();
     unsigned NumBytes = String.size();
-    llvm::SmallVector<UTF16, 128> ToBuf(NumBytes);
+    SmallVector<UTF16, 128> ToBuf(NumBytes);
     const UTF8 *FromPtr = (UTF8 *)String.data();
     UTF16 *ToPtr = &ToBuf[0];
     
@@ -649,6 +907,10 @@ bool Sema::SemaBuiltinVAStart(CallExpr *TheCall) {
       << 0 /*function call*/ << 2 << TheCall->getNumArgs();
   }
 
+  // Type-check the first argument normally.
+  if (checkBuiltinArgument(*this, TheCall, 0))
+    return true;
+
   // Determine whether the current function is variadic or not.
   BlockScopeInfo *CurBlock = getCurBlock();
   bool isVariadic;
@@ -844,7 +1106,7 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
                << TheCall->getArg(i)->getSourceRange());
   }
 
-  llvm::SmallVector<Expr*, 32> exprs;
+  SmallVector<Expr*, 32> exprs;
 
   for (unsigned i = 0, e = TheCall->getNumArgs(); i != e; i++) {
     exprs.push_back(TheCall->getArg(i));
@@ -1238,7 +1500,7 @@ getSpecifierRange(const char *startSpecifier, unsigned specifierLen) {
   SourceLocation End   = getLocationOfByte(startSpecifier + specifierLen - 1);
 
   // Advance the end SourceLocation by one due to half-open ranges.
-  End = End.getFileLocWithOffset(1);
+  End = End.getLocWithOffset(1);
 
   return CharSourceRange::getCharRange(Start, End);
 }
@@ -1322,7 +1584,7 @@ CheckFormatHandler::HandleInvalidConversionSpecifier(unsigned argIndex,
   }
   
   S.Diag(Loc, diag::warn_format_invalid_conversion)
-    << llvm::StringRef(csStart, csLen)
+    << StringRef(csStart, csLen)
     << getSpecifierRange(startSpec, specifierLen);
   
   return keepGoing;
@@ -1838,7 +2100,7 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
                              bool isPrintf) {
   
   // CHECK: is the format string a wide literal?
-  if (FExpr->isWide()) {
+  if (!FExpr->isAscii()) {
     Diag(FExpr->getLocStart(),
          diag::warn_format_string_is_wide_literal)
     << OrigFormatExpr->getSourceRange();
@@ -1846,12 +2108,13 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
   }
   
   // Str - The format string.  NOTE: this is NOT null-terminated!
-  llvm::StringRef StrRef = FExpr->getString();
+  StringRef StrRef = FExpr->getString();
   const char *Str = StrRef.data();
   unsigned StrLen = StrRef.size();
+  const unsigned numDataArgs = TheCall->getNumArgs() - firstDataArg;
   
   // CHECK: empty format string?
-  if (StrLen == 0) {
+  if (StrLen == 0 && numDataArgs > 0) {
     Diag(FExpr->getLocStart(), diag::warn_empty_format_string)
     << OrigFormatExpr->getSourceRange();
     return;
@@ -1859,9 +2122,8 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
   
   if (isPrintf) {
     CheckPrintfHandler H(*this, FExpr, OrigFormatExpr, firstDataArg,
-                         TheCall->getNumArgs() - firstDataArg,
-                         isa<ObjCStringLiteral>(OrigFormatExpr), Str,
-                         HasVAListArg, TheCall, format_idx);
+                         numDataArgs, isa<ObjCStringLiteral>(OrigFormatExpr),
+                         Str, HasVAListArg, TheCall, format_idx);
   
     bool FormatExtensions = getLangOptions().FormatExtensions;
     if (!analyze_format_string::ParsePrintfString(H, Str, Str + StrLen,
@@ -1870,9 +2132,8 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
   }
   else {
     CheckScanfHandler H(*this, FExpr, OrigFormatExpr, firstDataArg,
-                        TheCall->getNumArgs() - firstDataArg,
-                        isa<ObjCStringLiteral>(OrigFormatExpr), Str,
-                        HasVAListArg, TheCall, format_idx);
+                        numDataArgs, isa<ObjCStringLiteral>(OrigFormatExpr),
+                        Str, HasVAListArg, TheCall, format_idx);
     
     if (!analyze_format_string::ParseScanfString(H, Str, Str + StrLen))
       H.DoneProcessing();
@@ -1916,19 +2177,22 @@ static QualType getSizeOfArgType(const Expr* E) {
 /// \brief Check for dangerous or invalid arguments to memset().
 ///
 /// This issues warnings on known problematic, dangerous or unspecified
-/// arguments to the standard 'memset', 'memcpy', and 'memmove' function calls.
+/// arguments to the standard 'memset', 'memcpy', 'memmove', and 'memcmp'
+/// function calls.
 ///
 /// \param Call The call expression to diagnose.
-void Sema::CheckMemsetcpymoveArguments(const CallExpr *Call,
-                                       CheckedMemoryFunction CMF,
-                                       IdentifierInfo *FnName) {
+void Sema::CheckMemaccessArguments(const CallExpr *Call,
+                                   CheckedMemoryFunction CMF,
+                                   IdentifierInfo *FnName) {
   // It is possible to have a non-standard definition of memset.  Validate
   // we have enough arguments, and if not, abort further checking.
-  if (Call->getNumArgs() < 3)
+  unsigned ExpectedNumArgs = (CMF == CMF_Strndup ? 2 : 3);
+  if (Call->getNumArgs() < ExpectedNumArgs)
     return;
 
-  unsigned LastArg = (CMF == CMF_Memset? 1 : 2);
-  const Expr *LenExpr = Call->getArg(2)->IgnoreParenImpCasts();
+  unsigned LastArg = (CMF == CMF_Memset || CMF == CMF_Strndup ? 1 : 2);
+  unsigned LenArg = (CMF == CMF_Strndup ? 1 : 2);
+  const Expr *LenExpr = Call->getArg(LenArg)->IgnoreParenImpCasts();
 
   // We have special checking when the length is a sizeof expression.
   QualType SizeOfArgTy = getSizeOfArgType(LenExpr);
@@ -1962,6 +2226,8 @@ void Sema::CheckMemsetcpymoveArguments(const CallExpr *Call,
         llvm::FoldingSetNodeID DestID;
         Dest->Profile(DestID, Context, true);
         if (DestID == SizeOfArgID) {
+          // TODO: For strncpy() and friends, this could suggest sizeof(dst)
+          //       over sizeof(src) as well.
           unsigned ActionIdx = 0; // Default is to suggest dereferencing.
           if (const UnaryOperator *UnaryOp = dyn_cast<UnaryOperator>(Dest))
             if (UnaryOp->getOpcode() == UO_AddrOf)
@@ -1969,9 +2235,10 @@ void Sema::CheckMemsetcpymoveArguments(const CallExpr *Call,
           if (Context.getTypeSize(PointeeTy) == Context.getCharWidth())
             ActionIdx = 2; // If the pointee's size is sizeof(char),
                            // suggest an explicit length.
+          unsigned DestSrcSelect = (CMF == CMF_Strndup ? 1 : ArgIdx);
           DiagRuntimeBehavior(SizeOfArg->getExprLoc(), Dest,
                               PDiag(diag::warn_sizeof_pointer_expr_memaccess)
-                                << FnName << ArgIdx << ActionIdx
+                                << FnName << DestSrcSelect << ActionIdx
                                 << Dest->getSourceRange()
                                 << SizeOfArg->getSourceRange());
           break;
@@ -1993,22 +2260,25 @@ void Sema::CheckMemsetcpymoveArguments(const CallExpr *Call,
         }
       }
 
-      unsigned DiagID;
-
       // Always complain about dynamic classes.
       if (isDynamicClassType(PointeeTy))
-        DiagID = diag::warn_dyn_class_memaccess;
+        DiagRuntimeBehavior(
+          Dest->getExprLoc(), Dest,
+          PDiag(diag::warn_dyn_class_memaccess)
+            << (CMF == CMF_Memcmp ? ArgIdx + 2 : ArgIdx) << FnName << PointeeTy
+            // "overwritten" if we're warning about the destination for any call
+            // but memcmp; otherwise a verb appropriate to the call.
+            << (ArgIdx == 0 && CMF != CMF_Memcmp ? 0 : (unsigned)CMF)
+            << Call->getCallee()->getSourceRange());
       else if (PointeeTy.hasNonTrivialObjCLifetime() && CMF != CMF_Memset)
-        DiagID = diag::warn_arc_object_memaccess;
+        DiagRuntimeBehavior(
+          Dest->getExprLoc(), Dest,
+          PDiag(diag::warn_arc_object_memaccess)
+            << ArgIdx << FnName << PointeeTy
+            << Call->getCallee()->getSourceRange());
       else
         continue;
 
-      DiagRuntimeBehavior(
-        Dest->getExprLoc(), Dest,
-        PDiag(DiagID)
-          << ArgIdx << FnName << PointeeTy 
-          << Call->getCallee()->getSourceRange());
-
       DiagRuntimeBehavior(
         Dest->getExprLoc(), Dest,
         PDiag(diag::note_bad_memaccess_silence)
@@ -2018,10 +2288,107 @@ void Sema::CheckMemsetcpymoveArguments(const CallExpr *Call,
   }
 }
 
+// A little helper routine: ignore addition and subtraction of integer literals.
+// This intentionally does not ignore all integer constant expressions because
+// we don't want to remove sizeof().
+static const Expr *ignoreLiteralAdditions(const Expr *Ex, ASTContext &Ctx) {
+  Ex = Ex->IgnoreParenCasts();
+
+  for (;;) {
+    const BinaryOperator * BO = dyn_cast<BinaryOperator>(Ex);
+    if (!BO || !BO->isAdditiveOp())
+      break;
+
+    const Expr *RHS = BO->getRHS()->IgnoreParenCasts();
+    const Expr *LHS = BO->getLHS()->IgnoreParenCasts();
+    
+    if (isa<IntegerLiteral>(RHS))
+      Ex = LHS;
+    else if (isa<IntegerLiteral>(LHS))
+      Ex = RHS;
+    else
+      break;
+  }
+
+  return Ex;
+}
+
+// Warn if the user has made the 'size' argument to strlcpy or strlcat
+// be the size of the source, instead of the destination.
+void Sema::CheckStrlcpycatArguments(const CallExpr *Call,
+                                    IdentifierInfo *FnName) {
+
+  // Don't crash if the user has the wrong number of arguments
+  if (Call->getNumArgs() != 3)
+    return;
+
+  const Expr *SrcArg = ignoreLiteralAdditions(Call->getArg(1), Context);
+  const Expr *SizeArg = ignoreLiteralAdditions(Call->getArg(2), Context);
+  const Expr *CompareWithSrc = NULL;
+  
+  // Look for 'strlcpy(dst, x, sizeof(x))'
+  if (const Expr *Ex = getSizeOfExprArg(SizeArg))
+    CompareWithSrc = Ex;
+  else {
+    // Look for 'strlcpy(dst, x, strlen(x))'
+    if (const CallExpr *SizeCall = dyn_cast<CallExpr>(SizeArg)) {
+      if (SizeCall->isBuiltinCall(Context) == Builtin::BIstrlen
+          && SizeCall->getNumArgs() == 1)
+        CompareWithSrc = ignoreLiteralAdditions(SizeCall->getArg(0), Context);
+    }
+  }
+
+  if (!CompareWithSrc)
+    return;
+
+  // Determine if the argument to sizeof/strlen is equal to the source
+  // argument.  In principle there's all kinds of things you could do
+  // here, for instance creating an == expression and evaluating it with
+  // EvaluateAsBooleanCondition, but this uses a more direct technique:
+  const DeclRefExpr *SrcArgDRE = dyn_cast<DeclRefExpr>(SrcArg);
+  if (!SrcArgDRE)
+    return;
+  
+  const DeclRefExpr *CompareWithSrcDRE = dyn_cast<DeclRefExpr>(CompareWithSrc);
+  if (!CompareWithSrcDRE || 
+      SrcArgDRE->getDecl() != CompareWithSrcDRE->getDecl())
+    return;
+  
+  const Expr *OriginalSizeArg = Call->getArg(2);
+  Diag(CompareWithSrcDRE->getLocStart(), diag::warn_strlcpycat_wrong_size)
+    << OriginalSizeArg->getSourceRange() << FnName;
+  
+  // Output a FIXIT hint if the destination is an array (rather than a
+  // pointer to an array).  This could be enhanced to handle some
+  // pointers if we know the actual size, like if DstArg is 'array+2'
+  // we could say 'sizeof(array)-2'.
+  const Expr *DstArg = Call->getArg(0)->IgnoreParenImpCasts();
+  QualType DstArgTy = DstArg->getType();
+  
+  // Only handle constant-sized or VLAs, but not flexible members.
+  if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(DstArgTy)) {
+    // Only issue the FIXIT for arrays of size > 1.
+    if (CAT->getSize().getSExtValue() <= 1)
+      return;
+  } else if (!DstArgTy->isVariableArrayType()) {
+    return;
+  }
+
+  llvm::SmallString<128> sizeString;
+  llvm::raw_svector_ostream OS(sizeString);
+  OS << "sizeof(";
+  DstArg->printPretty(OS, Context, 0, getPrintingPolicy());
+  OS << ")";
+  
+  Diag(OriginalSizeArg->getLocStart(), diag::note_strlcpycat_wrong_size)
+    << FixItHint::CreateReplacement(OriginalSizeArg->getSourceRange(),
+                                    OS.str());
+}
+
 //===--- CHECK: Return Address of Stack Variable --------------------------===//
 
-static Expr *EvalVal(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars);
-static Expr *EvalAddr(Expr* E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars);
+static Expr *EvalVal(Expr *E, SmallVectorImpl<DeclRefExpr *> &refVars);
+static Expr *EvalAddr(Expr* E, SmallVectorImpl<DeclRefExpr *> &refVars);
 
 /// CheckReturnStackAddr - Check if a return statement returns the address
 ///   of a stack variable.
@@ -2030,7 +2397,7 @@ Sema::CheckReturnStackAddr(Expr *RetValExp, QualType lhsType,
                            SourceLocation ReturnLoc) {
 
   Expr *stackE = 0;
-  llvm::SmallVector<DeclRefExpr *, 8> refVars;
+  SmallVector<DeclRefExpr *, 8> refVars;
 
   // Perform checking for returned stack addresses, local blocks,
   // label addresses or references to temporaries.
@@ -2112,7 +2479,7 @@ Sema::CheckReturnStackAddr(Expr *RetValExp, QualType lhsType,
 ///   * arbitrary interplay between "&" and "*" operators
 ///   * pointer arithmetic from an address of a stack variable
 ///   * taking the address of an array element where the array is on the stack
-static Expr *EvalAddr(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars) {
+static Expr *EvalAddr(Expr *E, SmallVectorImpl<DeclRefExpr *> &refVars) {
   if (E->isTypeDependent())
       return NULL;
 
@@ -2257,7 +2624,7 @@ static Expr *EvalAddr(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars) {
 
 ///  EvalVal - This function is complements EvalAddr in the mutual recursion.
 ///   See the comments for EvalAddr for more details.
-static Expr *EvalVal(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars) {
+static Expr *EvalVal(Expr *E, SmallVectorImpl<DeclRefExpr *> &refVars) {
 do {
   // We should only be called for evaluating non-pointer expressions, or
   // expressions with a pointer type that are not used as references but instead
@@ -2374,11 +2741,11 @@ do {
 /// Check for comparisons of floating point operands using != and ==.
 /// Issue a warning if these are no self-comparisons, as they are not likely
 /// to do what the programmer intended.
-void Sema::CheckFloatComparison(SourceLocation loc, Expr* lex, Expr *rex) {
+void Sema::CheckFloatComparison(SourceLocation Loc, Expr* LHS, Expr *RHS) {
   bool EmitWarning = true;
 
-  Expr* LeftExprSansParen = lex->IgnoreParenImpCasts();
-  Expr* RightExprSansParen = rex->IgnoreParenImpCasts();
+  Expr* LeftExprSansParen = LHS->IgnoreParenImpCasts();
+  Expr* RightExprSansParen = RHS->IgnoreParenImpCasts();
 
   // Special case: check for x == x (which is OK).
   // Do not emit warnings for such cases.
@@ -2417,8 +2784,8 @@ void Sema::CheckFloatComparison(SourceLocation loc, Expr* lex, Expr *rex) {
 
   // Emit the diagnostic.
   if (EmitWarning)
-    Diag(loc, diag::warn_floatingpoint_eq)
-      << lex->getSourceRange() << rex->getSourceRange();
+    Diag(Loc, diag::warn_floatingpoint_eq)
+      << LHS->getSourceRange() << RHS->getSourceRange();
 }
 
 //===--- CHECK: Integer mixed-sign comparisons (-Wsign-compare) --------===//
@@ -2462,7 +2829,7 @@ struct IntRange {
     // For enum types, use the known bit width of the enumerators.
     if (const EnumType *ET = dyn_cast<EnumType>(T)) {
       EnumDecl *Enum = ET->getDecl();
-      if (!Enum->isDefinition())
+      if (!Enum->isCompleteDefinition())
         return IntRange(C.getIntWidth(QualType(T, 0)), false);
 
       unsigned NumPositive = Enum->getNumPositiveBits();
@@ -2490,7 +2857,7 @@ struct IntRange {
     if (const ComplexType *CT = dyn_cast<ComplexType>(T))
       T = CT->getElementType().getTypePtr();
     if (const EnumType *ET = dyn_cast<EnumType>(T))
-      T = ET->getDecl()->getIntegerType().getTypePtr();
+      T = C.getCanonicalType(ET->getDecl()->getIntegerType()).getTypePtr();
 
     const BuiltinType *BT = cast<BuiltinType>(T);
     assert(BT->isInteger());
@@ -2767,14 +3134,9 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     IntRange::forValueOfType(C, E->getType());
   }
 
-  FieldDecl *BitField = E->getBitField();
-  if (BitField) {
-    llvm::APSInt BitWidthAP = BitField->getBitWidth()->EvaluateAsInt(C);
-    unsigned BitWidth = BitWidthAP.getZExtValue();
-
-    return IntRange(BitWidth, 
+  if (FieldDecl *BitField = E->getBitField())
+    return IntRange(BitField->getBitWidthValue(C),
                     BitField->getType()->isUnsignedIntegerOrEnumerationType());
-  }
 
   return IntRange::forValueOfType(C, E->getType());
 }
@@ -2883,10 +3245,7 @@ void AnalyzeImpConvsInComparison(Sema &S, BinaryOperator *E) {
 
 /// \brief Implements -Wsign-compare.
 ///
-/// \param lex the left-hand expression
-/// \param rex the right-hand expression
-/// \param OpLoc the location of the joining operator
-/// \param BinOpc binary opcode or 0
+/// \param E the binary operator to check for warnings
 void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   // The type the comparison is being performed in.
   QualType T = E->getLHS()->getType();
@@ -2903,20 +3262,20 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
       || E->isValueDependent() || E->isIntegerConstantExpr(S.Context))
     return AnalyzeImpConvsInComparison(S, E);
 
-  Expr *lex = E->getLHS()->IgnoreParenImpCasts();
-  Expr *rex = E->getRHS()->IgnoreParenImpCasts();
+  Expr *LHS = E->getLHS()->IgnoreParenImpCasts();
+  Expr *RHS = E->getRHS()->IgnoreParenImpCasts();
 
   // Check to see if one of the (unmodified) operands is of different
   // signedness.
   Expr *signedOperand, *unsignedOperand;
-  if (lex->getType()->hasSignedIntegerRepresentation()) {
-    assert(!rex->getType()->hasSignedIntegerRepresentation() &&
+  if (LHS->getType()->hasSignedIntegerRepresentation()) {
+    assert(!RHS->getType()->hasSignedIntegerRepresentation() &&
            "unsigned comparison between two signed integer expressions?");
-    signedOperand = lex;
-    unsignedOperand = rex;
-  } else if (rex->getType()->hasSignedIntegerRepresentation()) {
-    signedOperand = rex;
-    unsignedOperand = lex;
+    signedOperand = LHS;
+    unsignedOperand = RHS;
+  } else if (RHS->getType()->hasSignedIntegerRepresentation()) {
+    signedOperand = RHS;
+    unsignedOperand = LHS;
   } else {
     CheckTrivialUnsignedComparison(S, E);
     return AnalyzeImpConvsInComparison(S, E);
@@ -2927,8 +3286,8 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
 
   // Go ahead and analyze implicit conversions in the operands.  Note
   // that we skip the implicit conversions on both sides.
-  AnalyzeImplicitConversions(S, lex, E->getOperatorLoc());
-  AnalyzeImplicitConversions(S, rex, E->getOperatorLoc());
+  AnalyzeImplicitConversions(S, LHS, E->getOperatorLoc());
+  AnalyzeImplicitConversions(S, RHS, E->getOperatorLoc());
 
   // If the signed range is non-negative, -Wsign-compare won't fire,
   // but we should still check for comparisons which are always true
@@ -2953,8 +3312,8 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   }
 
   S.Diag(E->getOperatorLoc(), diag::warn_mixed_sign_comparison)
-    << lex->getType() << rex->getType()
-    << lex->getSourceRange() << rex->getSourceRange();
+    << LHS->getType() << RHS->getType()
+    << LHS->getSourceRange() << RHS->getSourceRange();
 }
 
 /// Analyzes an attempt to assign the given value to a bitfield.
@@ -2979,16 +3338,14 @@ bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
 
   Expr *OriginalInit = Init->IgnoreParenImpCasts();
 
-  llvm::APSInt Width(32);
   Expr::EvalResult InitValue;
-  if (!Bitfield->getBitWidth()->isIntegerConstantExpr(Width, S.Context) ||
-      !OriginalInit->Evaluate(InitValue, S.Context) ||
+  if (!OriginalInit->Evaluate(InitValue, S.Context) ||
       !InitValue.Val.isInt())
     return false;
 
   const llvm::APSInt &Value = InitValue.Val.getInt();
   unsigned OriginalWidth = Value.getBitWidth();
-  unsigned FieldWidth = Width.getZExtValue();
+  unsigned FieldWidth = Bitfield->getBitWidthValue(S.Context);
 
   if (OriginalWidth <= FieldWidth)
     return false;
@@ -3049,34 +3406,22 @@ void DiagnoseImpCast(Sema &S, Expr *E, QualType T, SourceLocation CContext,
   DiagnoseImpCast(S, E, E->getType(), T, CContext, diag);
 }
 
-/// Diagnose an implicit cast from a literal expression. Also attemps to supply
-/// fixit hints when the cast wouldn't lose information to simply write the
-/// expression with the expected type.
+/// Diagnose an implicit cast from a literal expression. Does not warn when the
+/// cast wouldn't lose information.
 void DiagnoseFloatingLiteralImpCast(Sema &S, FloatingLiteral *FL, QualType T,
                                     SourceLocation CContext) {
-  // Emit the primary warning first, then try to emit a fixit hint note if
-  // reasonable.
-  S.Diag(FL->getExprLoc(), diag::warn_impcast_literal_float_to_integer)
-    << FL->getType() << T << FL->getSourceRange() << SourceRange(CContext);
-
-  const llvm::APFloat &Value = FL->getValue();
-
-  // Don't attempt to fix PPC double double literals.
-  if (&Value.getSemantics() == &llvm::APFloat::PPCDoubleDouble)
-    return;
-
-  // Try to convert this exactly to an integer.
+  // Try to convert the literal exactly to an integer. If we can, don't warn.
   bool isExact = false;
+  const llvm::APFloat &Value = FL->getValue();
   llvm::APSInt IntegerValue(S.Context.getIntWidth(T),
                             T->hasUnsignedIntegerRepresentation());
   if (Value.convertToInteger(IntegerValue,
                              llvm::APFloat::rmTowardZero, &isExact)
-      != llvm::APFloat::opOK || !isExact)
+      == llvm::APFloat::opOK && isExact)
     return;
 
-  std::string LiteralValue = IntegerValue.toString(10);
-  S.Diag(FL->getExprLoc(), diag::note_fix_integral_float_as_integer)
-    << FixItHint::CreateReplacement(FL->getSourceRange(), LiteralValue);
+  S.Diag(FL->getExprLoc(), diag::warn_impcast_literal_float_to_integer)
+    << FL->getType() << T << FL->getSourceRange() << SourceRange(CContext);
 }
 
 std::string PrettyPrintInRange(const llvm::APSInt &Value, IntRange Range) {
@@ -3102,17 +3447,24 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
   if (Source == Target) return;
   if (Target->isDependentType()) return;
 
-  // If the conversion context location is invalid don't complain.
-  // We also don't want to emit a warning if the issue occurs from the
-  // instantiation of a system macro.  The problem is that 'getSpellingLoc()'
-  // is slow, so we delay this check as long as possible.  Once we detect
-  // we are in that scenario, we just return.
+  // If the conversion context location is invalid don't complain. We also
+  // don't want to emit a warning if the issue occurs from the expansion of
+  // a system macro. The problem is that 'getSpellingLoc()' is slow, so we
+  // delay this check as long as possible. Once we detect we are in that
+  // scenario, we just return.
   if (CC.isInvalid())
     return;
 
-  // Never diagnose implicit casts to bool.
-  if (Target->isSpecificBuiltinType(BuiltinType::Bool))
-    return;
+  // Diagnose implicit casts to bool.
+  if (Target->isSpecificBuiltinType(BuiltinType::Bool)) {
+    if (isa<StringLiteral>(E))
+      // Warn on string literal to bool.  Checks for string literals in logical
+      // expressions, for instances, assert(0 && "error here"), is prevented
+      // by a check in AnalyzeImplicitConversions().
+      return DiagnoseImpCast(S, E, T, CC,
+                             diag::warn_impcast_string_literal_to_bool);
+    return; // Other casts to bool are not checked.
+  }
 
   // Strip vector types.
   if (isa<VectorType>(Source)) {
@@ -3180,6 +3532,11 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
         return;
       
       Expr *InnerE = E->IgnoreParenImpCasts();
+      // We also want to warn on, e.g., "int i = -1.234"
+      if (UnaryOperator *UOp = dyn_cast<UnaryOperator>(InnerE))
+        if (UOp->getOpcode() == UO_Minus || UOp->getOpcode() == UO_Plus)
+          InnerE = UOp->getSubExpr()->IgnoreParenImpCasts();
+
       if (FloatingLiteral *FL = dyn_cast<FloatingLiteral>(InnerE)) {
         DiagnoseFloatingLiteralImpCast(S, FL, T, CC);
       } else {
@@ -3314,29 +3671,16 @@ void CheckConditionalOperator(Sema &S, ConditionalOperator *E, QualType T) {
                                  CC))
     return;
 
-  // ...and -Wsign-compare isn't...
-  if (!S.Diags.getDiagnosticLevel(diag::warn_mixed_sign_conditional, CC))
-    return;
-
   // ...then check whether it would have warned about either of the
   // candidates for a signedness conversion to the condition type.
-  if (E->getType() != T) {
-    Suspicious = false;
-    CheckImplicitConversion(S, E->getTrueExpr()->IgnoreParenImpCasts(),
+  if (E->getType() == T) return;
+ 
+  Suspicious = false;
+  CheckImplicitConversion(S, E->getTrueExpr()->IgnoreParenImpCasts(),
+                          E->getType(), CC, &Suspicious);
+  if (!Suspicious)
+    CheckImplicitConversion(S, E->getFalseExpr()->IgnoreParenImpCasts(),
                             E->getType(), CC, &Suspicious);
-    if (!Suspicious)
-      CheckImplicitConversion(S, E->getFalseExpr()->IgnoreParenImpCasts(),
-                              E->getType(), CC, &Suspicious);
-    if (!Suspicious)
-      return;
-  }
-
-  // If so, emit a diagnostic under -Wsign-compare.
-  Expr *lex = E->getTrueExpr()->IgnoreParenImpCasts();
-  Expr *rex = E->getFalseExpr()->IgnoreParenImpCasts();
-  S.Diag(E->getQuestionLoc(), diag::warn_mixed_sign_conditional)
-    << lex->getType() << rex->getType()
-    << lex->getSourceRange() << rex->getSourceRange();
 }
 
 /// AnalyzeImplicitConversions - Find and report any interesting
@@ -3346,6 +3690,9 @@ void AnalyzeImplicitConversions(Sema &S, Expr *OrigE, SourceLocation CC) {
   QualType T = OrigE->getType();
   Expr *E = OrigE->IgnoreParenImpCasts();
 
+  if (E->isTypeDependent() || E->isValueDependent())
+    return;
+
   // For conditional operators, we analyze the arguments as if they
   // were being fed directly into the output.
   if (isa<ConditionalOperator>(E)) {
@@ -3389,8 +3736,16 @@ void AnalyzeImplicitConversions(Sema &S, Expr *OrigE, SourceLocation CC) {
 
   // Now just recurse over the expression's children.
   CC = E->getExprLoc();
-  for (Stmt::child_range I = E->children(); I; ++I)
-    AnalyzeImplicitConversions(S, cast<Expr>(*I), CC);
+  BinaryOperator *BO = dyn_cast<BinaryOperator>(E);
+  bool IsLogicalOperator = BO && BO->isLogicalOp();
+  for (Stmt::child_range I = E->children(); I; ++I) {
+    Expr *ChildExpr = cast<Expr>(*I);
+    if (IsLogicalOperator &&
+        isa<StringLiteral>(ChildExpr->IgnoreParenImpCasts()))
+      // Ignore checking string literals that are in logical operators.
+      continue;
+    AnalyzeImplicitConversions(S, ChildExpr, CC);
+  }
 }
 
 } // end anonymous namespace
@@ -3411,6 +3766,11 @@ void Sema::CheckImplicitConversions(Expr *E, SourceLocation CC) {
   if (E->isTypeDependent() || E->isValueDependent())
     return;
 
+  // Check for array bounds violations in cases where the check isn't triggered
+  // elsewhere for other Expr types (like BinaryOperators), e.g. when an
+  // ArraySubscriptExpr is on the RHS of a variable initialization.
+  CheckArrayAccess(E);
+
   // This is not the right CC for (e.g.) a variable initialization.
   AnalyzeImplicitConversions(*this, E, CC);
 }
@@ -3477,7 +3837,7 @@ void Sema::CheckCastAlign(Expr *Op, QualType T, SourceRange TRange) {
   // cast; don't do it if we're ignoring -Wcast_align (as is the default).
   if (getDiagnostics().getDiagnosticLevel(diag::warn_cast_align,
                                           TRange.getBegin())
-        == Diagnostic::Ignored)
+        == DiagnosticsEngine::Ignored)
     return;
 
   // Ignore dependent types.
@@ -3515,63 +3875,164 @@ void Sema::CheckCastAlign(Expr *Op, QualType T, SourceRange TRange) {
     << TRange << Op->getSourceRange();
 }
 
-static void CheckArrayAccess_Check(Sema &S,
-                                   const clang::ArraySubscriptExpr *E) {
-  const Expr *BaseExpr = E->getBase()->IgnoreParenImpCasts();
+static const Type* getElementType(const Expr *BaseExpr) {
+  const Type* EltType = BaseExpr->getType().getTypePtr();
+  if (EltType->isAnyPointerType())
+    return EltType->getPointeeType().getTypePtr();
+  else if (EltType->isArrayType())
+    return EltType->getBaseElementTypeUnsafe();
+  return EltType;
+}
+
+/// \brief Check whether this array fits the idiom of a size-one tail padded
+/// array member of a struct.
+///
+/// We avoid emitting out-of-bounds access warnings for such arrays as they are
+/// commonly used to emulate flexible arrays in C89 code.
+static bool IsTailPaddedMemberArray(Sema &S, llvm::APInt Size,
+                                    const NamedDecl *ND) {
+  if (Size != 1 || !ND) return false;
+
+  const FieldDecl *FD = dyn_cast<FieldDecl>(ND);
+  if (!FD) return false;
+
+  // Don't consider sizes resulting from macro expansions or template argument
+  // substitution to form C89 tail-padded arrays.
+  ConstantArrayTypeLoc TL =
+    cast<ConstantArrayTypeLoc>(FD->getTypeSourceInfo()->getTypeLoc());
+  const Expr *SizeExpr = dyn_cast<IntegerLiteral>(TL.getSizeExpr());
+  if (!SizeExpr || SizeExpr->getExprLoc().isMacroID())
+    return false;
+
+  const RecordDecl *RD = dyn_cast<RecordDecl>(FD->getDeclContext());
+  if (!RD || !RD->isStruct())
+    return false;
+
+  // See if this is the last field decl in the record.
+  const Decl *D = FD;
+  while ((D = D->getNextDeclInContext()))
+    if (isa<FieldDecl>(D))
+      return false;
+  return true;
+}
+
+void Sema::CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr,
+                            bool isSubscript, bool AllowOnePastEnd) {
+  const Type* EffectiveType = getElementType(BaseExpr);
+  BaseExpr = BaseExpr->IgnoreParenCasts();
+  IndexExpr = IndexExpr->IgnoreParenCasts();
+
   const ConstantArrayType *ArrayTy =
-    S.Context.getAsConstantArrayType(BaseExpr->getType());
+    Context.getAsConstantArrayType(BaseExpr->getType());
   if (!ArrayTy)
     return;
 
-  const Expr *IndexExpr = E->getIdx();
   if (IndexExpr->isValueDependent())
     return;
   llvm::APSInt index;
-  if (!IndexExpr->isIntegerConstantExpr(index, S.Context))
+  if (!IndexExpr->isIntegerConstantExpr(index, Context))
     return;
 
-  if (index.isUnsigned() || !index.isNegative()) {
-    llvm::APInt size = ArrayTy->getSize();
-    if (!size.isStrictlyPositive())
-      return;
-    if (size.getBitWidth() > index.getBitWidth())
-      index = index.sext(size.getBitWidth());
-    else if (size.getBitWidth() < index.getBitWidth())
-      size = size.sext(index.getBitWidth());
-
-    if (index.slt(size))
-      return;
-
-    S.DiagRuntimeBehavior(E->getBase()->getLocStart(), BaseExpr,
-                          S.PDiag(diag::warn_array_index_exceeds_bounds)
-                            << index.toString(10, true)
-                            << size.toString(10, true)
-                            << IndexExpr->getSourceRange());
-  } else {
-    S.DiagRuntimeBehavior(E->getBase()->getLocStart(), BaseExpr,
-                          S.PDiag(diag::warn_array_index_precedes_bounds)
-                            << index.toString(10, true)
-                            << IndexExpr->getSourceRange());
-  }
-
   const NamedDecl *ND = NULL;
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(BaseExpr))
     ND = dyn_cast<NamedDecl>(DRE->getDecl());
   if (const MemberExpr *ME = dyn_cast<MemberExpr>(BaseExpr))
     ND = dyn_cast<NamedDecl>(ME->getMemberDecl());
+
+  if (index.isUnsigned() || !index.isNegative()) {
+    llvm::APInt size = ArrayTy->getSize();
+    if (!size.isStrictlyPositive())
+      return;
+
+    const Type* BaseType = getElementType(BaseExpr);
+    if (BaseType != EffectiveType) {
+      // Make sure we're comparing apples to apples when comparing index to size
+      uint64_t ptrarith_typesize = Context.getTypeSize(EffectiveType);
+      uint64_t array_typesize = Context.getTypeSize(BaseType);
+      // Handle ptrarith_typesize being zero, such as when casting to void*
+      if (!ptrarith_typesize) ptrarith_typesize = 1;
+      if (ptrarith_typesize != array_typesize) {
+        // There's a cast to a different size type involved
+        uint64_t ratio = array_typesize / ptrarith_typesize;
+        // TODO: Be smarter about handling cases where array_typesize is not a
+        // multiple of ptrarith_typesize
+        if (ptrarith_typesize * ratio == array_typesize)
+          size *= llvm::APInt(size.getBitWidth(), ratio);
+      }
+    }
+
+    if (size.getBitWidth() > index.getBitWidth())
+      index = index.sext(size.getBitWidth());
+    else if (size.getBitWidth() < index.getBitWidth())
+      size = size.sext(index.getBitWidth());
+
+    // For array subscripting the index must be less than size, but for pointer
+    // arithmetic also allow the index (offset) to be equal to size since
+    // computing the next address after the end of the array is legal and
+    // commonly done e.g. in C++ iterators and range-based for loops.
+    if (AllowOnePastEnd ? index.sle(size) : index.slt(size))
+      return;
+
+    // Also don't warn for arrays of size 1 which are members of some
+    // structure. These are often used to approximate flexible arrays in C89
+    // code.
+    if (IsTailPaddedMemberArray(*this, size, ND))
+      return;
+
+    unsigned DiagID = diag::warn_ptr_arith_exceeds_bounds;
+    if (isSubscript)
+      DiagID = diag::warn_array_index_exceeds_bounds;
+
+    DiagRuntimeBehavior(BaseExpr->getLocStart(), BaseExpr,
+                        PDiag(DiagID) << index.toString(10, true)
+                          << size.toString(10, true)
+                          << (unsigned)size.getLimitedValue(~0U)
+                          << IndexExpr->getSourceRange());
+  } else {
+    unsigned DiagID = diag::warn_array_index_precedes_bounds;
+    if (!isSubscript) {
+      DiagID = diag::warn_ptr_arith_precedes_bounds;
+      if (index.isNegative()) index = -index;
+    }
+
+    DiagRuntimeBehavior(BaseExpr->getLocStart(), BaseExpr,
+                        PDiag(DiagID) << index.toString(10, true)
+                          << IndexExpr->getSourceRange());
+  }
+
   if (ND)
-    S.DiagRuntimeBehavior(ND->getLocStart(), BaseExpr,
-                          S.PDiag(diag::note_array_index_out_of_bounds)
-                            << ND->getDeclName());
+    DiagRuntimeBehavior(ND->getLocStart(), BaseExpr,
+                        PDiag(diag::note_array_index_out_of_bounds)
+                          << ND->getDeclName());
 }
 
 void Sema::CheckArrayAccess(const Expr *expr) {
-  while (true) {
-    expr = expr->IgnoreParens();
+  int AllowOnePastEnd = 0;
+  while (expr) {
+    expr = expr->IgnoreParenImpCasts();
     switch (expr->getStmtClass()) {
-      case Stmt::ArraySubscriptExprClass:
-        CheckArrayAccess_Check(*this, cast<ArraySubscriptExpr>(expr));
+      case Stmt::ArraySubscriptExprClass: {
+        const ArraySubscriptExpr *ASE = cast<ArraySubscriptExpr>(expr);
+        CheckArrayAccess(ASE->getBase(), ASE->getIdx(), true,
+                         AllowOnePastEnd > 0);
         return;
+      }
+      case Stmt::UnaryOperatorClass: {
+        // Only unwrap the * and & unary operators
+        const UnaryOperator *UO = cast<UnaryOperator>(expr);
+        expr = UO->getSubExpr();
+        switch (UO->getOpcode()) {
+          case UO_AddrOf:
+            AllowOnePastEnd++;
+            break;
+          case UO_Deref:
+            AllowOnePastEnd--;
+            break;
+          default:
+            return;
+        }
+        break;
+      }
       case Stmt::ConditionalOperatorClass: {
         const ConditionalOperator *cond = cast<ConditionalOperator>(expr);
         if (const Expr *lhs = cond->getLHS())
@@ -3625,7 +4086,7 @@ static bool findRetainCycleOwner(Expr *e, RetainCycleOwner &owner) {
       case CK_BitCast:
       case CK_LValueBitCast:
       case CK_LValueToRValue:
-      case CK_ObjCReclaimReturnedObject:
+      case CK_ARCReclaimReturnedObject:
         e = cast->getSubExpr();
         continue;
 
@@ -3634,10 +4095,7 @@ static bool findRetainCycleOwner(Expr *e, RetainCycleOwner &owner) {
         const ObjCPropertyRefExpr *pre = cast->getSubExpr()->getObjCProperty();
         if (pre->isImplicitProperty()) return false;
         ObjCPropertyDecl *property = pre->getExplicitProperty();
-        if (!(property->getPropertyAttributes() &
-              (ObjCPropertyDecl::OBJC_PR_retain |
-               ObjCPropertyDecl::OBJC_PR_copy |
-               ObjCPropertyDecl::OBJC_PR_strong)) &&
+        if (!property->isRetaining() &&
             !(property->getPropertyIvarDecl() &&
               property->getPropertyIvarDecl()->getType()
                 .getObjCLifetime() == Qualifiers::OCL_Strong))
@@ -3758,7 +4216,7 @@ static void diagnoseRetainCycle(Sema &S, Expr *capturer,
 static bool isSetterLikeSelector(Selector sel) {
   if (sel.isUnarySelector()) return false;
 
-  llvm::StringRef str = sel.getNameForSlot(0);
+  StringRef str = sel.getNameForSlot(0);
   while (!str.empty() && str.front() == '_') str = str.substr(1);
   if (str.startswith("set") || str.startswith("add"))
     str = str.substr(3);
@@ -3810,7 +4268,7 @@ bool Sema::checkUnsafeAssigns(SourceLocation Loc,
     return false;
   // strip off any implicit cast added to get to the one arc-specific
   while (ImplicitCastExpr *cast = dyn_cast<ImplicitCastExpr>(RHS)) {
-    if (cast->getCastKind() == CK_ObjCConsumeObject) {
+    if (cast->getCastKind() == CK_ARCConsumeObject) {
       Diag(Loc, diag::warn_arc_retained_assign)
         << (LT == Qualifiers::OCL_ExplicitNone) 
         << RHS->getSourceRange();
@@ -3841,7 +4299,7 @@ void Sema::checkUnsafeExprAssigns(SourceLocation Loc,
     unsigned Attributes = PD->getPropertyAttributes();
     if (Attributes & ObjCPropertyDecl::OBJC_PR_assign)
       while (ImplicitCastExpr *cast = dyn_cast<ImplicitCastExpr>(RHS)) {
-        if (cast->getCastKind() == CK_ObjCConsumeObject) {
+        if (cast->getCastKind() == CK_ARCConsumeObject) {
           Diag(Loc, diag::warn_arc_retained_property_assign)
           << RHS->getSourceRange();
           return;
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaCodeComplete.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaCodeComplete.cpp
index b555c8a9aa1f..405d626ae9ba 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaCodeComplete.cpp
@@ -61,7 +61,7 @@ namespace {
     /// a single (declaration, index) mapping (the common case) but
     /// can also store a list of (declaration, index) mappings.
     class ShadowMapEntry {
-      typedef llvm::SmallVector<DeclIndexPair, 4> DeclIndexPairVector;
+      typedef SmallVector<DeclIndexPair, 4> DeclIndexPairVector;
 
       /// \brief Contains either the solitary NamedDecl * or a vector
       /// of (declaration, index) pairs.
@@ -434,7 +434,7 @@ static NestedNameSpecifier *
 getRequiredQualification(ASTContext &Context,
                          DeclContext *CurContext,
                          DeclContext *TargetContext) {
-  llvm::SmallVector<DeclContext *, 4> TargetParents;
+  SmallVector<DeclContext *, 4> TargetParents;
   
   for (DeclContext *CommonAncestor = TargetContext;
        CommonAncestor && !CommonAncestor->Encloses(CurContext);
@@ -990,9 +990,6 @@ bool ResultBuilder::IsOrdinaryName(NamedDecl *ND) const {
   else if (SemaRef.getLangOptions().ObjC1) {
     if (isa<ObjCIvarDecl>(ND))
       return true;
-    if (isa<ObjCPropertyDecl>(ND) &&
-        SemaRef.canSynthesizeProvisionalIvar(cast<ObjCPropertyDecl>(ND)))
-      return true;
   }
   
   return ND->getIdentifierNamespace() & IDNS;
@@ -1011,9 +1008,6 @@ bool ResultBuilder::IsOrdinaryNonTypeName(NamedDecl *ND) const {
   else if (SemaRef.getLangOptions().ObjC1) {
     if (isa<ObjCIvarDecl>(ND))
       return true;
-    if (isa<ObjCPropertyDecl>(ND) &&
-        SemaRef.canSynthesizeProvisionalIvar(cast<ObjCPropertyDecl>(ND)))
-      return true;
   }
  
   return ND->getIdentifierNamespace() & IDNS;
@@ -1192,8 +1186,16 @@ namespace {
     CodeCompletionDeclConsumer(ResultBuilder &Results, DeclContext *CurContext)
       : Results(Results), CurContext(CurContext) { }
     
-    virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, bool InBaseClass) {
-      Results.AddResult(ND, CurContext, Hiding, InBaseClass);
+    virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, DeclContext *Ctx,
+                           bool InBaseClass) {
+      bool Accessible = true;
+      if (Ctx) {
+        if (CXXRecordDecl *Class = dyn_cast<CXXRecordDecl>(Ctx))
+          Accessible = Results.getSema().IsSimplyAccessible(ND, Class);
+        // FIXME: ObjC access checks are missing.
+      }
+      ResultBuilder::Result Result(ND, 0, false, Accessible);
+      Results.AddResult(Result, CurContext, Hiding, InBaseClass);
     }
   };
 }
@@ -1846,6 +1848,14 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
     Results.AddResult(Result("operator"));
 }
 
+/// \brief Retrieve a printing policy suitable for code completion.
+static PrintingPolicy getCompletionPrintingPolicy(Sema &S) {
+  PrintingPolicy Policy = S.getPrintingPolicy();
+  Policy.AnonymousTagLocations = false;
+  Policy.SuppressStrongLifetime = true;
+  return Policy;
+}
+
 /// \brief Retrieve the string representation of the given type as a string
 /// that has the appropriate lifetime for code completion.
 ///
@@ -1853,15 +1863,12 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
 /// common type names.
 static const char *GetCompletionTypeString(QualType T,
                                            ASTContext &Context,
+                                           const PrintingPolicy &Policy,
                                            CodeCompletionAllocator &Allocator) {
-  PrintingPolicy Policy(Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  Policy.SuppressStrongLifetime = true;
-  
   if (!T.getLocalQualifiers()) {
     // Built-in type names are constant strings.
     if (const BuiltinType *BT = dyn_cast<BuiltinType>(T))
-      return BT->getName(Context.getLangOptions());
+      return BT->getName(Policy);
     
     // Anonymous tag types are constant strings.
     if (const TagType *TagT = dyn_cast<TagType>(T))
@@ -1885,6 +1892,7 @@ static const char *GetCompletionTypeString(QualType T,
 /// \brief If the given declaration has an associated type, add it as a result 
 /// type chunk.
 static void AddResultTypeChunk(ASTContext &Context,
+                               const PrintingPolicy &Policy,
                                NamedDecl *ND,
                                CodeCompletionBuilder &Result) {
   if (!ND)
@@ -1915,7 +1923,7 @@ static void AddResultTypeChunk(ASTContext &Context,
   if (T.isNull() || Context.hasSameType(T, Context.DependentTy))
     return;
   
-  Result.AddResultTypeChunk(GetCompletionTypeString(T, Context, 
+  Result.AddResultTypeChunk(GetCompletionTypeString(T, Context, Policy,
                                                     Result.getAllocator()));
 }
 
@@ -1933,13 +1941,32 @@ static void MaybeAddSentinel(ASTContext &Context, NamedDecl *FunctionOrMethod,
     }
 }
 
+static void appendWithSpace(std::string &Result, StringRef Text) {
+  if (!Result.empty())
+    Result += ' ';
+  Result += Text.str();
+}
+static std::string formatObjCParamQualifiers(unsigned ObjCQuals) {
+  std::string Result;
+  if (ObjCQuals & Decl::OBJC_TQ_In)
+    appendWithSpace(Result, "in");
+  else if (ObjCQuals & Decl::OBJC_TQ_Inout)
+    appendWithSpace(Result, "inout");
+  else if (ObjCQuals & Decl::OBJC_TQ_Out)
+    appendWithSpace(Result, "out");
+  if (ObjCQuals & Decl::OBJC_TQ_Bycopy)
+    appendWithSpace(Result, "bycopy");
+  else if (ObjCQuals & Decl::OBJC_TQ_Byref)
+    appendWithSpace(Result, "byref");
+  if (ObjCQuals & Decl::OBJC_TQ_Oneway)
+    appendWithSpace(Result, "oneway");
+  return Result;
+}
+
 static std::string FormatFunctionParameter(ASTContext &Context,
+                                           const PrintingPolicy &Policy,
                                            ParmVarDecl *Param,
                                            bool SuppressName = false) {
-  PrintingPolicy Policy(Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  Policy.SuppressStrongLifetime = true;
-
   bool ObjCMethodParam = isa<ObjCMethodDecl>(Param->getDeclContext());
   if (Param->getType()->isDependentType() ||
       !Param->getType()->isBlockPointerType()) {
@@ -1953,8 +1980,8 @@ static std::string FormatFunctionParameter(ASTContext &Context,
     Param->getType().getAsStringInternal(Result, Policy);
     
     if (ObjCMethodParam) {
-      Result = "(" + Result;
-      Result += ")";
+      Result = "(" + formatObjCParamQualifiers(Param->getObjCDeclQualifier())
+             + Result + ")";
       if (Param->getIdentifier() && !SuppressName)
         Result += Param->getIdentifier()->getName();
     }
@@ -2003,8 +2030,8 @@ static std::string FormatFunctionParameter(ASTContext &Context,
     Param->getType().getUnqualifiedType().getAsStringInternal(Result, Policy);
     
     if (ObjCMethodParam) {
-      Result = "(" + Result;
-      Result += ")";
+      Result = "(" + formatObjCParamQualifiers(Param->getObjCDeclQualifier())
+             + Result + ")";
       if (Param->getIdentifier())
         Result += Param->getIdentifier()->getName();
     }
@@ -2030,7 +2057,7 @@ static std::string FormatFunctionParameter(ASTContext &Context,
     for (unsigned I = 0, N = Block->getNumArgs(); I != N; ++I) {
       if (I)
         Result += ", ";
-      Result += FormatFunctionParameter(Context, Block->getArg(I));
+      Result += FormatFunctionParameter(Context, Policy, Block->getArg(I));
       
       if (I == N - 1 && BlockProto->getTypePtr()->isVariadic())
         Result += ", ...";
@@ -2046,6 +2073,7 @@ static std::string FormatFunctionParameter(ASTContext &Context,
 
 /// \brief Add function parameter chunks to the given code completion string.
 static void AddFunctionParameterChunks(ASTContext &Context,
+                                       const PrintingPolicy &Policy,
                                        FunctionDecl *Function,
                                        CodeCompletionBuilder &Result,
                                        unsigned Start = 0,
@@ -2062,7 +2090,7 @@ static void AddFunctionParameterChunks(ASTContext &Context,
       CodeCompletionBuilder Opt(Result.getAllocator());
       if (!FirstParameter)
         Opt.AddChunk(Chunk(CodeCompletionString::CK_Comma));        
-      AddFunctionParameterChunks(Context, Function, Opt, P, true);
+      AddFunctionParameterChunks(Context, Policy, Function, Opt, P, true);
       Result.AddOptionalChunk(Opt.TakeString());
       break;
     }
@@ -2075,7 +2103,8 @@ static void AddFunctionParameterChunks(ASTContext &Context,
     InOptional = false;
     
     // Format the placeholder string.
-    std::string PlaceholderStr = FormatFunctionParameter(Context, Param);
+    std::string PlaceholderStr = FormatFunctionParameter(Context, Policy, 
+                                                         Param);
         
     if (Function->isVariadic() && P == N - 1)
       PlaceholderStr += ", ...";
@@ -2097,14 +2126,12 @@ static void AddFunctionParameterChunks(ASTContext &Context,
 
 /// \brief Add template parameter chunks to the given code completion string.
 static void AddTemplateParameterChunks(ASTContext &Context,
+                                       const PrintingPolicy &Policy,
                                        TemplateDecl *Template,
                                        CodeCompletionBuilder &Result,
                                        unsigned MaxParameters = 0,
                                        unsigned Start = 0,
                                        bool InDefaultArg = false) {
-  PrintingPolicy Policy(Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-
   typedef CodeCompletionString::Chunk Chunk;
   bool FirstParameter = true;
   
@@ -2155,7 +2182,7 @@ static void AddTemplateParameterChunks(ASTContext &Context,
       CodeCompletionBuilder Opt(Result.getAllocator());
       if (!FirstParameter)
         Opt.AddChunk(Chunk(CodeCompletionString::CK_Comma));
-      AddTemplateParameterChunks(Context, Template, Opt, MaxParameters,
+      AddTemplateParameterChunks(Context, Policy, Template, Opt, MaxParameters,
                                  P - Params->begin(), true);
       Result.AddOptionalChunk(Opt.TakeString());
       break;
@@ -2180,14 +2207,15 @@ static void
 AddQualifierToCompletionString(CodeCompletionBuilder &Result, 
                                NestedNameSpecifier *Qualifier, 
                                bool QualifierIsInformative,
-                               ASTContext &Context) {
+                               ASTContext &Context,
+                               const PrintingPolicy &Policy) {
   if (!Qualifier)
     return;
   
   std::string PrintedNNS;
   {
     llvm::raw_string_ostream OS(PrintedNNS);
-    Qualifier->print(OS, Context.PrintingPolicy);
+    Qualifier->print(OS, Policy);
   }
   if (QualifierIsInformative)
     Result.AddInformativeChunk(Result.getAllocator().CopyString(PrintedNNS));
@@ -2233,8 +2261,8 @@ AddFunctionTypeQualsToCompletionString(CodeCompletionBuilder &Result,
 }
 
 /// \brief Add the name of the given declaration 
-static void AddTypedNameChunk(ASTContext &Context, NamedDecl *ND,
-                              CodeCompletionBuilder &Result) {
+static void AddTypedNameChunk(ASTContext &Context, const PrintingPolicy &Policy,
+                              NamedDecl *ND, CodeCompletionBuilder &Result) {
   typedef CodeCompletionString::Chunk Chunk;
   
   DeclarationName Name = ND->getDeclName();
@@ -2299,7 +2327,7 @@ static void AddTypedNameChunk(ASTContext &Context, NamedDecl *ND,
                   Result.getAllocator().CopyString(Record->getNameAsString()));
     if (ClassTemplateDecl *Template = Record->getDescribedClassTemplate()) {
       Result.AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
-      AddTemplateParameterChunks(Context, Template, Result);
+      AddTemplateParameterChunks(Context, Policy, Template, Result);
       Result.AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
     }
     break;
@@ -2319,10 +2347,7 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
   typedef CodeCompletionString::Chunk Chunk;
   CodeCompletionBuilder Result(Allocator, Priority, Availability);
   
-  PrintingPolicy Policy(S.Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  Policy.SuppressStrongLifetime = true;
-
+  PrintingPolicy Policy = getCompletionPrintingPolicy(S);
   if (Kind == RK_Pattern) {
     Pattern->Priority = Priority;
     Pattern->Availability = Availability;
@@ -2346,19 +2371,24 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
     
     // Format a function-like macro with placeholders for the arguments.
     Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
-    for (MacroInfo::arg_iterator A = MI->arg_begin(), AEnd = MI->arg_end();
-         A != AEnd; ++A) {
+    bool CombineVariadicArgument = false;
+    MacroInfo::arg_iterator A = MI->arg_begin(), AEnd = MI->arg_end();
+    if (MI->isVariadic() && AEnd - A > 1) {
+      AEnd -= 2;
+      CombineVariadicArgument = true;
+    }
+    for (MacroInfo::arg_iterator A = MI->arg_begin(); A != AEnd; ++A) {
       if (A != MI->arg_begin())
         Result.AddChunk(Chunk(CodeCompletionString::CK_Comma));
       
-      if (!MI->isVariadic() || A != AEnd - 1) {
+      if (!MI->isVariadic() || A + 1 != AEnd) {
         // Non-variadic argument.
         Result.AddPlaceholderChunk(
                             Result.getAllocator().CopyString((*A)->getName()));
         continue;
       }
       
-      // Variadic argument; cope with the different between GNU and C99
+      // Variadic argument; cope with the difference between GNU and C99
       // variadic macros, providing a single placeholder for the rest of the
       // arguments.
       if ((*A)->isStr("__VA_ARGS__"))
@@ -2369,6 +2399,18 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
         Result.AddPlaceholderChunk(Result.getAllocator().CopyString(Arg));
       }
     }
+     
+    if (CombineVariadicArgument) {
+      // Handle the next-to-last argument, combining it with the variadic
+      // argument.
+      std::string LastArg = (*A)->getName();
+      ++A;
+      if ((*A)->isStr("__VA_ARGS__"))
+        LastArg += ", ...";
+      else
+        LastArg += ", " + (*A)->getName().str() + "...";
+      Result.AddPlaceholderChunk(Result.getAllocator().CopyString(LastArg));
+    }
     Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
     return Result.TakeString();
   }
@@ -2382,15 +2424,21 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
     Result.AddTextChunk("::");
     return Result.TakeString();
   }
+
+  for (Decl::attr_iterator i = ND->attr_begin(); i != ND->attr_end(); ++i) {
+    if (AnnotateAttr *Attr = dyn_cast_or_null<AnnotateAttr>(*i)) {
+      Result.AddAnnotation(Result.getAllocator().CopyString(Attr->getAnnotation()));
+    }
+  }
   
-  AddResultTypeChunk(S.Context, ND, Result);
+  AddResultTypeChunk(S.Context, Policy, ND, Result);
   
   if (FunctionDecl *Function = dyn_cast<FunctionDecl>(ND)) {
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
-                                   S.Context);
-    AddTypedNameChunk(S.Context, ND, Result);
+                                   S.Context, Policy);
+    AddTypedNameChunk(S.Context, Policy, ND, Result);
     Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
-    AddFunctionParameterChunks(S.Context, Function, Result);
+    AddFunctionParameterChunks(S.Context, Policy, Function, Result);
     Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
     AddFunctionTypeQualsToCompletionString(Result, Function);
     return Result.TakeString();
@@ -2398,13 +2446,13 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
   
   if (FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(ND)) {
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
-                                   S.Context);
+                                   S.Context, Policy);
     FunctionDecl *Function = FunTmpl->getTemplatedDecl();
-    AddTypedNameChunk(S.Context, Function, Result);
+    AddTypedNameChunk(S.Context, Policy, Function, Result);
 
     // Figure out which template parameters are deduced (or have default
     // arguments).
-    llvm::SmallVector<bool, 16> Deduced;
+    SmallVector<bool, 16> Deduced;
     S.MarkDeducedTemplateParameters(FunTmpl, Deduced);
     unsigned LastDeducibleArgument;
     for (LastDeducibleArgument = Deduced.size(); LastDeducibleArgument > 0;
@@ -2437,14 +2485,14 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
       // function call, so we introduce an explicit template argument list
       // containing all of the arguments up to the first deducible argument.
       Result.AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
-      AddTemplateParameterChunks(S.Context, FunTmpl, Result, 
+      AddTemplateParameterChunks(S.Context, Policy, FunTmpl, Result, 
                                  LastDeducibleArgument);
       Result.AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
     }
     
     // Add the function parameters
     Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
-    AddFunctionParameterChunks(S.Context, Function, Result);
+    AddFunctionParameterChunks(S.Context, Policy, Function, Result);
     Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
     AddFunctionTypeQualsToCompletionString(Result, Function);
     return Result.TakeString();
@@ -2452,11 +2500,11 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
   
   if (TemplateDecl *Template = dyn_cast<TemplateDecl>(ND)) {
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
-                                   S.Context);
+                                   S.Context, Policy);
     Result.AddTypedTextChunk(
                 Result.getAllocator().CopyString(Template->getNameAsString()));
     Result.AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
-    AddTemplateParameterChunks(S.Context, Template, Result);
+    AddTemplateParameterChunks(S.Context, Policy, Template, Result);
     Result.AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
     return Result.TakeString();
   }
@@ -2490,7 +2538,7 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
         if (Idx > StartParameter)
           Result.AddChunk(CodeCompletionString::CK_HorizontalSpace);
         if (IdentifierInfo *II = Sel.getIdentifierInfoForSlot(Idx))
-          Keyword += II->getName().str();
+          Keyword += II->getName();
         Keyword += ":";
         if (Idx < StartParameter || AllParametersAreInformative)
           Result.AddInformativeChunk(Result.getAllocator().CopyString(Keyword));
@@ -2505,13 +2553,14 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
       std::string Arg;
       
       if ((*P)->getType()->isBlockPointerType() && !DeclaringEntity)
-        Arg = FormatFunctionParameter(S.Context, *P, true);
+        Arg = FormatFunctionParameter(S.Context, Policy, *P, true);
       else {
         (*P)->getType().getAsStringInternal(Arg, Policy);
-        Arg = "(" + Arg + ")";
+        Arg = "(" + formatObjCParamQualifiers((*P)->getObjCDeclQualifier()) 
+            + Arg + ")";
         if (IdentifierInfo *II = (*P)->getIdentifier())
           if (DeclaringEntity || AllParametersAreInformative)
-            Arg += II->getName().str();
+            Arg += II->getName();
       }
       
       if (Method->isVariadic() && (P + 1) == PEnd)
@@ -2543,7 +2592,7 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
 
   if (Qualifier)
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
-                                   S.Context);
+                                   S.Context, Policy);
 
   Result.AddTypedTextChunk(
                        Result.getAllocator().CopyString(ND->getNameAsString()));
@@ -2556,14 +2605,12 @@ CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
                                                                Sema &S,
                                      CodeCompletionAllocator &Allocator) const {
   typedef CodeCompletionString::Chunk Chunk;
-  PrintingPolicy Policy(S.Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  Policy.SuppressStrongLifetime = true;
+  PrintingPolicy Policy = getCompletionPrintingPolicy(S);
 
   // FIXME: Set priority, availability appropriately.
   CodeCompletionBuilder Result(Allocator, 1, CXAvailability_Available);
   FunctionDecl *FDecl = getFunction();
-  AddResultTypeChunk(S.Context, FDecl, Result);
+  AddResultTypeChunk(S.Context, Policy, FDecl, Result);
   const FunctionProtoType *Proto 
     = dyn_cast<FunctionProtoType>(getFunctionType());
   if (!FDecl && !Proto) {
@@ -2571,7 +2618,7 @@ CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
     // highlighted ellipsis.
     const FunctionType *FT = getFunctionType();
     Result.AddTextChunk(GetCompletionTypeString(FT->getResultType(),
-                                                S.Context, 
+                                                S.Context, Policy, 
                                                 Result.getAllocator()));
     Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
     Result.AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, "..."));
@@ -2624,7 +2671,7 @@ CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
   return Result.TakeString();
 }
 
-unsigned clang::getMacroUsagePriority(llvm::StringRef MacroName, 
+unsigned clang::getMacroUsagePriority(StringRef MacroName, 
                                       const LangOptions &LangOpts,
                                       bool PreferredTypeIsPointer) {
   unsigned Priority = CCP_Macro;
@@ -2689,6 +2736,7 @@ CXCursorKind clang::getCursorKindForDecl(Decl *D) {
     case Decl::TemplateTemplateParm:return CXCursor_TemplateTemplateParameter;
     case Decl::FunctionTemplate:   return CXCursor_FunctionTemplate;
     case Decl::ClassTemplate:      return CXCursor_ClassTemplate;
+    case Decl::AccessSpec:         return CXCursor_CXXAccessSpecifier;
     case Decl::ClassTemplatePartialSpecialization:
       return CXCursor_ClassTemplatePartialSpecialization;
     case Decl::UsingDirective:     return CXCursor_UsingDirective;
@@ -2850,6 +2898,7 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
       return;
   }
 
+  PrintingPolicy Policy = getCompletionPrintingPolicy(S);
   for (CXXMethodDecl::method_iterator M = Method->begin_overridden_methods(),
                                    MEnd = Method->end_overridden_methods();
        M != MEnd; ++M) {
@@ -2866,7 +2915,7 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
       if (NNS) {
         std::string Str;
         llvm::raw_string_ostream OS(Str);
-        NNS->print(OS, S.Context.PrintingPolicy);
+        NNS->print(OS, Policy);
         Builder.AddTextChunk(Results.getAllocator().CopyString(OS.str()));
       }
     } else if (!InContext->Equals(Overridden->getDeclContext()))
@@ -3059,7 +3108,7 @@ struct Sema::CodeCompleteExpressionData {
   QualType PreferredType;
   bool IntegralConstantExpression;
   bool ObjCCollection;
-  llvm::SmallVector<Decl *, 4> IgnoreDecls;
+  SmallVector<Decl *, 4> IgnoreDecls;
 };
 
 /// \brief Perform code-completion in an expression context when we know what
@@ -3146,6 +3195,7 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
   // Add nullary methods
   if (AllowNullaryMethods) {
     ASTContext &Context = Container->getASTContext();
+    PrintingPolicy Policy = getCompletionPrintingPolicy(Results.getSema());
     for (ObjCContainerDecl::method_iterator M = Container->meth_begin(),
                                          MEnd = Container->meth_end();
          M != MEnd; ++M) {
@@ -3153,7 +3203,7 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
         if (IdentifierInfo *Name = M->getSelector().getIdentifierInfoForSlot(0))
           if (AddedProperties.insert(Name)) {
             CodeCompletionBuilder Builder(Results.getAllocator());
-            AddResultTypeChunk(Context, *M, Builder);
+            AddResultTypeChunk(Context, Policy, *M, Builder);
             Builder.AddTypedTextChunk(
                             Results.getAllocator().CopyString(Name->getName()));
             
@@ -3224,7 +3274,7 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
   }
 }
 
-void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
+void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *BaseE,
                                            SourceLocation OpLoc,
                                            bool IsArrow) {
   if (!BaseE || !CodeCompleter)
@@ -3366,8 +3416,7 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
     break;
     
   default:
-    assert(false && "Unknown type specifier kind in CodeCompleteTag");
-    return;
+    llvm_unreachable("Unknown type specifier kind in CodeCompleteTag");
   }
   
   ResultBuilder Results(*this, CodeCompleter->getAllocator(), ContextKind);
@@ -3408,10 +3457,11 @@ void Sema::CodeCompleteTypeQualifiers(DeclSpec &DS) {
 void Sema::CodeCompleteCase(Scope *S) {
   if (getCurFunction()->SwitchStack.empty() || !CodeCompleter)
     return;
-  
+
   SwitchStmt *Switch = getCurFunction()->SwitchStack.back();
-  if (!Switch->getCond()->getType()->isEnumeralType()) {
-    CodeCompleteExpressionData Data(Switch->getCond()->getType());
+  QualType type = Switch->getCond()->IgnoreImplicit()->getType();
+  if (!type->isEnumeralType()) {
+    CodeCompleteExpressionData Data(type);
     Data.IntegralConstantExpression = true;
     CodeCompleteExpression(S, Data);
     return;
@@ -3419,7 +3469,7 @@ void Sema::CodeCompleteCase(Scope *S) {
   
   // Code-complete the cases of a switch statement over an enumeration type
   // by providing the list of 
-  EnumDecl *Enum = Switch->getCond()->getType()->getAs<EnumType>()->getDecl();
+  EnumDecl *Enum = type->castAs<EnumType>()->getDecl();
   
   // Determine which enumerators we have already seen in the switch statement.
   // FIXME: Ideally, we would also be able to look *past* the code-completion
@@ -3527,8 +3577,8 @@ static bool anyNullArguments(Expr **Args, unsigned NumArgs) {
   return false;
 }
 
-void Sema::CodeCompleteCall(Scope *S, ExprTy *FnIn,
-                            ExprTy **ArgsIn, unsigned NumArgs) {
+void Sema::CodeCompleteCall(Scope *S, Expr *FnIn,
+                            Expr **ArgsIn, unsigned NumArgs) {
   if (!CodeCompleter)
     return;
 
@@ -3556,7 +3606,7 @@ void Sema::CodeCompleteCall(Scope *S, ExprTy *FnIn,
   // FIXME: What if we're calling a member function?
   
   typedef CodeCompleteConsumer::OverloadCandidate ResultCandidate;
-  llvm::SmallVector<ResultCandidate, 8> Results;
+  SmallVector<ResultCandidate, 8> Results;
 
   Expr *NakedFn = Fn->IgnoreParenCasts();
   if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn))
@@ -3662,7 +3712,62 @@ void Sema::CodeCompleteReturn(Scope *S) {
     CodeCompleteExpression(S, ResultType);
 }
 
-void Sema::CodeCompleteAssignmentRHS(Scope *S, ExprTy *LHS) {
+void Sema::CodeCompleteAfterIf(Scope *S) {
+  typedef CodeCompletionResult Result;
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        mapCodeCompletionContext(*this, PCC_Statement));
+  Results.setFilter(&ResultBuilder::IsOrdinaryName);
+  Results.EnterNewScope();
+  
+  CodeCompletionDeclConsumer Consumer(Results, CurContext);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
+  
+  AddOrdinaryNameResults(PCC_Statement, S, *this, Results);
+  
+  // "else" block
+  CodeCompletionBuilder Builder(Results.getAllocator());
+  Builder.AddTypedTextChunk("else");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+  Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+  Builder.AddPlaceholderChunk("statements");
+  Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+  Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+  Results.AddResult(Builder.TakeString());
+
+  // "else if" block
+  Builder.AddTypedTextChunk("else");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddTextChunk("if");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  if (getLangOptions().CPlusPlus)
+    Builder.AddPlaceholderChunk("condition");
+  else
+    Builder.AddPlaceholderChunk("expression");
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+  Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+  Builder.AddPlaceholderChunk("statements");
+  Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+  Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+  Results.AddResult(Builder.TakeString());
+
+  Results.ExitScope();
+  
+  if (S->getFnParent())
+    AddPrettyFunctionResults(PP.getLangOptions(), Results);        
+  
+  if (CodeCompleter->includeMacros())
+    AddMacroResults(PP, Results);
+  
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompleteAssignmentRHS(Scope *S, Expr *LHS) {
   if (LHS)
     CodeCompleteExpression(S, static_cast<Expr *>(LHS)->getType());
   else
@@ -3706,7 +3811,7 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
   LookupVisibleDecls(Ctx, LookupOrdinaryName, Consumer);
 
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Name,
+                            Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
@@ -3848,10 +3953,7 @@ void Sema::CodeCompleteOperatorName(Scope *S) {
 void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
                                               CXXCtorInitializer** Initializers,
                                               unsigned NumInitializers) {
-  PrintingPolicy Policy(Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  Policy.SuppressStrongLifetime = true;
-
+  PrintingPolicy Policy = getCompletionPrintingPolicy(*this);
   CXXConstructorDecl *Constructor
     = static_cast<CXXConstructorDecl *>(ConstructorD);
   if (!Constructor)
@@ -4045,15 +4147,14 @@ static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt) {
   Results.AddResult(Result(Builder.TakeString()));
 }
 
-void Sema::CodeCompleteObjCAtDirective(Scope *S, Decl *ObjCImpDecl,
-                                       bool InInterface) {
+void Sema::CodeCompleteObjCAtDirective(Scope *S) {
   typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
-  if (ObjCImpDecl)
+  if (isa<ObjCImplDecl>(CurContext))
     AddObjCImplementationResults(getLangOptions(), Results, false);
-  else if (InInterface)
+  else if (CurContext->isObjCContainer())
     AddObjCInterfaceResults(getLangOptions(), Results, false);
   else
     AddObjCTopLevelResults(Results, false);
@@ -4425,14 +4526,14 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
 }
 
 
-void Sema::CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl) {
+void Sema::CodeCompleteObjCPropertyGetter(Scope *S) {
   typedef CodeCompletionResult Result;
 
   // Try to find the interface where getters might live.
-  ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(ClassDecl);
+  ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(CurContext);
   if (!Class) {
     if (ObjCCategoryDecl *Category
-          = dyn_cast_or_null<ObjCCategoryDecl>(ClassDecl))
+          = dyn_cast_or_null<ObjCCategoryDecl>(CurContext))
       Class = Category->getClassInterface();
 
     if (!Class)
@@ -4453,15 +4554,15 @@ void Sema::CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl) {
                             Results.data(),Results.size());
 }
 
-void Sema::CodeCompleteObjCPropertySetter(Scope *S, Decl *ObjCImplDecl) {
+void Sema::CodeCompleteObjCPropertySetter(Scope *S) {
   typedef CodeCompletionResult Result;
 
   // Try to find the interface where setters might live.
   ObjCInterfaceDecl *Class
-    = dyn_cast_or_null<ObjCInterfaceDecl>(ObjCImplDecl);
+    = dyn_cast_or_null<ObjCInterfaceDecl>(CurContext);
   if (!Class) {
     if (ObjCCategoryDecl *Category
-          = dyn_cast_or_null<ObjCCategoryDecl>(ObjCImplDecl))
+          = dyn_cast_or_null<ObjCCategoryDecl>(CurContext))
       Class = Category->getClassInterface();
 
     if (!Class)
@@ -4694,7 +4795,8 @@ static ObjCMethodDecl *AddSuperSendCompletion(Sema &S, bool NeedSuperKeyword,
   CodeCompletionBuilder Builder(Results.getAllocator());
   
   // Give this completion a return type.
-  AddResultTypeChunk(S.Context, SuperMethod, Builder);
+  AddResultTypeChunk(S.Context, getCompletionPrintingPolicy(S), SuperMethod, 
+                     Builder);
 
   // If we need the "super" keyword, add it (plus some spacing).
   if (NeedSuperKeyword) {
@@ -4955,8 +5057,13 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
                                         unsigned NumSelIdents,
                                         bool AtArgumentExpression,
                                         bool IsSuper) {
+  
+  QualType T = this->GetTypeFromParser(Receiver);
+  
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
-                        CodeCompletionContext::CCC_ObjCClassMessage);
+              CodeCompletionContext(CodeCompletionContext::CCC_ObjCClassMessage,
+                                    T, SelIdents, NumSelIdents));
+    
   AddClassMessageCompletions(*this, S, Receiver, SelIdents, NumSelIdents, 
                              AtArgumentExpression, IsSuper, Results);
   
@@ -4967,7 +5074,7 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
   // our preferred type, improving completion results.
   if (AtArgumentExpression) {
     QualType PreferredType = getPreferredArgumentTypeForMessageSend(Results, 
-                                                                    NumSelIdents);
+                                                                  NumSelIdents);
     if (PreferredType.isNull())
       CodeCompleteOrdinaryName(S, PCC_Expression);
     else
@@ -4976,11 +5083,11 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
   }
 
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_ObjCClassMessage,
+                            Results.getCompletionContext(),
                             Results.data(), Results.size());
 }
 
-void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
+void Sema::CodeCompleteObjCInstanceMessage(Scope *S, Expr *Receiver,
                                            IdentifierInfo **SelIdents,
                                            unsigned NumSelIdents,
                                            bool AtArgumentExpression,
@@ -5019,7 +5126,9 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
 
   // Build the set of methods we can see.
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
-                        CodeCompletionContext::CCC_ObjCInstanceMessage);
+           CodeCompletionContext(CodeCompletionContext::CCC_ObjCInstanceMessage,
+                                 ReceiverType, SelIdents, NumSelIdents));
+  
   Results.EnterNewScope();
 
   // If this is a send-to-super, try to add the special "super" send 
@@ -5132,7 +5241,7 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
   }
   
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_ObjCInstanceMessage,
+                            Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
@@ -5196,7 +5305,7 @@ void Sema::CodeCompleteObjCSelector(Scope *S, IdentifierInfo **SelIdents,
         }
       }
       
-      Accumulator += Sel.getNameForSlot(I).str();
+      Accumulator += Sel.getNameForSlot(I);
       Accumulator += ':';
     }
     Builder.AddTypedTextChunk(Builder.getAllocator().CopyString( Accumulator));
@@ -5303,12 +5412,11 @@ static void AddInterfaceResults(DeclContext *Ctx, DeclContext *CurContext,
 
     // Record any forward-declared interfaces we find.
     if (ObjCClassDecl *Forward = dyn_cast<ObjCClassDecl>(*D)) {
-      for (ObjCClassDecl::iterator C = Forward->begin(), CEnd = Forward->end();
-           C != CEnd; ++C)
-        if ((!OnlyForwardDeclarations || C->getInterface()->isForwardDecl()) &&
-            (!OnlyUnimplemented || !C->getInterface()->getImplementation()))
-          Results.AddResult(Result(C->getInterface(), 0), CurContext,
-                            0, false);
+      ObjCInterfaceDecl *IDecl = Forward->getForwardInterfaceDecl();
+      if ((!OnlyForwardDeclarations || IDecl->isForwardDecl()) &&
+          (!OnlyUnimplemented || !IDecl->getImplementation()))
+        Results.AddResult(Result(IDecl, 0), CurContext,
+                          0, false);
     }
   }
 }
@@ -5318,21 +5426,23 @@ void Sema::CodeCompleteObjCInterfaceDecl(Scope *S) {
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   
-  // Add all classes.
-  AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, true,
-                      false, Results);
-
+  if (CodeCompleter->includeGlobals()) {
+    // Add all classes.
+    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
+                        false, Results);
+  }
+  
   Results.ExitScope();
-  // FIXME: Use cached global completion results.
+
   HandleCodeCompleteResults(this, CodeCompleter,
-                            CodeCompletionContext::CCC_Other,
+                            CodeCompletionContext::CCC_ObjCInterfaceName,
                             Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
                                       SourceLocation ClassNameLoc) { 
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
-                        CodeCompletionContext::CCC_ObjCSuperclass);
+                        CodeCompletionContext::CCC_ObjCInterfaceName);
   Results.EnterNewScope();
   
   // Make sure that we ignore the class we're currently defining.
@@ -5341,14 +5451,16 @@ void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
   if (CurClass && isa<ObjCInterfaceDecl>(CurClass))
     Results.Ignore(CurClass);
 
-  // Add all classes.
-  AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
-                      false, Results);
-
+  if (CodeCompleter->includeGlobals()) {
+    // Add all classes.
+    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
+                        false, Results);
+  }
+  
   Results.ExitScope();
-  // FIXME: Use cached global completion results.
+
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_ObjCSuperclass,
+                            CodeCompletionContext::CCC_ObjCInterfaceName,
                             Results.data(),Results.size());
 }
 
@@ -5357,14 +5469,16 @@ void Sema::CodeCompleteObjCImplementationDecl(Scope *S) {
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
-  // Add all unimplemented classes.
-  AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
-                      true, Results);
-
+  if (CodeCompleter->includeGlobals()) {
+    // Add all unimplemented classes.
+    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
+                        true, Results);
+  }
+  
   Results.ExitScope();
-  // FIXME: Use cached global completion results.
+
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Other,
+                            CodeCompletionContext::CCC_ObjCInterfaceName,
                             Results.data(),Results.size());
 }
 
@@ -5442,14 +5556,14 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
                             Results.data(),Results.size());  
 }
 
-void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, Decl *ObjCImpDecl) {
+void Sema::CodeCompleteObjCPropertyDefinition(Scope *S) {
   typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompletionContext::CCC_Other);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container
-    = dyn_cast_or_null<ObjCContainerDecl>(ObjCImpDecl);
+    = dyn_cast_or_null<ObjCContainerDecl>(CurContext);
   if (!Container || 
       (!isa<ObjCImplementationDecl>(Container) && 
        !isa<ObjCCategoryImplDecl>(Container)))
@@ -5482,15 +5596,14 @@ void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, Decl *ObjCImpDecl) {
 }
 
 void Sema::CodeCompleteObjCPropertySynthesizeIvar(Scope *S, 
-                                                  IdentifierInfo *PropertyName,
-                                                  Decl *ObjCImpDecl) {
+                                                  IdentifierInfo *PropertyName) {
   typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompletionContext::CCC_Other);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container
-    = dyn_cast_or_null<ObjCContainerDecl>(ObjCImpDecl);
+    = dyn_cast_or_null<ObjCContainerDecl>(CurContext);
   if (!Container || 
       (!isa<ObjCImplementationDecl>(Container) && 
        !isa<ObjCCategoryImplDecl>(Container)))
@@ -5523,7 +5636,7 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
   bool SawSimilarlyNamedIvar = false;
   std::string NameWithPrefix;
   NameWithPrefix += '_';
-  NameWithPrefix += PropertyName->getName().str();
+  NameWithPrefix += PropertyName->getName();
   std::string NameWithSuffix = PropertyName->getName().str();
   NameWithSuffix += '_';
   for(; Class; Class = Class->getSuperClass()) {
@@ -5557,8 +5670,9 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
     CodeCompletionAllocator &Allocator = Results.getAllocator();
     CodeCompletionBuilder Builder(Allocator, Priority,CXAvailability_Available);
 
+    PrintingPolicy Policy = getCompletionPrintingPolicy(*this);
     Builder.AddResultTypeChunk(GetCompletionTypeString(PropertyType, Context,
-                                                       Allocator));
+                                                       Policy, Allocator));
     Builder.AddTypedTextChunk(Allocator.CopyString(NameWithPrefix));
     Results.AddResult(Result(Builder.TakeString(), Priority, 
                              CXCursor_ObjCIvarDecl));
@@ -5658,9 +5772,10 @@ static void FindImplementableMethods(ASTContext &Context,
 /// completion string.
 static void AddObjCPassingTypeChunk(QualType Type,
                                     ASTContext &Context,
+                                    const PrintingPolicy &Policy,
                                     CodeCompletionBuilder &Builder) {
   Builder.AddChunk(CodeCompletionString::CK_LeftParen);
-  Builder.AddTextChunk(GetCompletionTypeString(Type, Context, 
+  Builder.AddTextChunk(GetCompletionTypeString(Type, Context, Policy,
                                                Builder.getAllocator()));
   Builder.AddChunk(CodeCompletionString::CK_RightParen);
 }
@@ -5668,7 +5783,7 @@ static void AddObjCPassingTypeChunk(QualType Type,
 /// \brief Determine whether the given class is or inherits from a class by
 /// the given name.
 static bool InheritsFromClassNamed(ObjCInterfaceDecl *Class, 
-                                   llvm::StringRef Name) {
+                                   StringRef Name) {
   if (!Class)
     return false;
   
@@ -5690,7 +5805,8 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   if (!PropName || PropName->getLength() == 0)
     return;
   
-  
+  PrintingPolicy Policy = getCompletionPrintingPolicy(Results.getSema());
+
   // Builder that will create each code completion.
   typedef CodeCompletionResult Result;
   CodeCompletionAllocator &Allocator = Results.getAllocator();
@@ -5703,10 +5819,10 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // on demand.
   struct KeyHolder {
     CodeCompletionAllocator &Allocator;
-    llvm::StringRef Key;
+    StringRef Key;
     const char *CopiedKey;
     
-    KeyHolder(CodeCompletionAllocator &Allocator, llvm::StringRef Key)
+    KeyHolder(CodeCompletionAllocator &Allocator, StringRef Key)
     : Allocator(Allocator), Key(Key), CopiedKey(0) { }
     
     operator const char *() {
@@ -5733,7 +5849,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
       KnownSelectors.insert(Selectors.getNullarySelector(PropName)) &&
       ReturnTypeMatchesProperty && !Property->getGetterMethodDecl()) {
     if (ReturnType.isNull())
-      AddObjCPassingTypeChunk(Property->getType(), Context, Builder);
+      AddObjCPassingTypeChunk(Property->getType(), Context, Policy, Builder);
     
     Builder.AddTypedTextChunk(Key);
     Results.AddResult(Result(Builder.TakeString(), CCP_CodePattern, 
@@ -5748,7 +5864,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
        (ReturnType.isNull() && 
         (Property->getType()->isIntegerType() || 
          Property->getType()->isBooleanType())))) {
-    std::string SelectorName = (llvm::Twine("is") + UpperKey).str();
+    std::string SelectorName = (Twine("is") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getNullarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -5767,7 +5883,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // Add the normal mutator.
   if (IsInstanceMethod && ReturnTypeMatchesVoid && 
       !Property->getSetterMethodDecl()) {
-    std::string SelectorName = (llvm::Twine("set") + UpperKey).str();
+    std::string SelectorName = (Twine("set") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -5779,7 +5895,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
       Builder.AddTypedTextChunk(
                                 Allocator.CopyString(SelectorId->getName()));
       Builder.AddTypedTextChunk(":");
-      AddObjCPassingTypeChunk(Property->getType(), Context, Builder);
+      AddObjCPassingTypeChunk(Property->getType(), Context, Policy, Builder);
       Builder.AddTextChunk(Key);
       Results.AddResult(Result(Builder.TakeString(), CCP_CodePattern, 
                                CXCursor_ObjCInstanceMethodDecl));
@@ -5818,7 +5934,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // Add -(NSUInteger)countOf<key>
   if (IsInstanceMethod &&  
       (ReturnType.isNull() || ReturnType->isIntegerType())) {
-    std::string SelectorName = (llvm::Twine("countOf") + UpperKey).str();
+    std::string SelectorName = (Twine("countOf") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getNullarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -5841,7 +5957,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   if (IsInstanceMethod &&
       (ReturnType.isNull() || ReturnType->isObjCObjectPointerType())) {
     std::string SelectorName
-      = (llvm::Twine("objectIn") + UpperKey + "AtIndex").str();
+      = (Twine("objectIn") + UpperKey + "AtIndex").str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -5868,7 +5984,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
         ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl()
                                                 ->getName() == "NSArray"))) {
     std::string SelectorName
-      = (llvm::Twine(Property->getName()) + "AtIndexes").str();
+      = (Twine(Property->getName()) + "AtIndexes").str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -5889,7 +6005,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   
   // Add -(void)getKey:(type **)buffer range:(NSRange)inRange
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
-    std::string SelectorName = (llvm::Twine("get") + UpperKey).str();
+    std::string SelectorName = (Twine("get") + UpperKey).str();
     IdentifierInfo *SelectorIds[2] = {
       &Context.Idents.get(SelectorName),
       &Context.Idents.get("range")
@@ -5923,7 +6039,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   
   // - (void)insertObject:(type *)object inKeyAtIndex:(NSUInteger)index
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
-    std::string SelectorName = (llvm::Twine("in") + UpperKey + "AtIndex").str();
+    std::string SelectorName = (Twine("in") + UpperKey + "AtIndex").str();
     IdentifierInfo *SelectorIds[2] = {
       &Context.Idents.get("insertObject"),
       &Context.Idents.get(SelectorName)
@@ -5955,7 +6071,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   
   // - (void)insertKey:(NSArray *)array atIndexes:(NSIndexSet *)indexes
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
-    std::string SelectorName = (llvm::Twine("insert") + UpperKey).str();
+    std::string SelectorName = (Twine("insert") + UpperKey).str();
     IdentifierInfo *SelectorIds[2] = {
       &Context.Idents.get(SelectorName),
       &Context.Idents.get("atIndexes")
@@ -5987,7 +6103,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // -(void)removeObjectFromKeyAtIndex:(NSUInteger)index
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
     std::string SelectorName
-      = (llvm::Twine("removeObjectFrom") + UpperKey + "AtIndex").str();
+      = (Twine("removeObjectFrom") + UpperKey + "AtIndex").str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);        
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6009,7 +6125,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // -(void)removeKeyAtIndexes:(NSIndexSet *)indexes
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
     std::string SelectorName
-      = (llvm::Twine("remove") + UpperKey + "AtIndexes").str();
+      = (Twine("remove") + UpperKey + "AtIndexes").str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);        
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6031,7 +6147,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // - (void)replaceObjectInKeyAtIndex:(NSUInteger)index withObject:(id)object
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
     std::string SelectorName
-      = (llvm::Twine("replaceObjectIn") + UpperKey + "AtIndex").str();
+      = (Twine("replaceObjectIn") + UpperKey + "AtIndex").str();
     IdentifierInfo *SelectorIds[2] = {
       &Context.Idents.get(SelectorName),
       &Context.Idents.get("withObject")
@@ -6063,8 +6179,8 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // - (void)replaceKeyAtIndexes:(NSIndexSet *)indexes withKey:(NSArray *)array
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
     std::string SelectorName1 
-      = (llvm::Twine("replace") + UpperKey + "AtIndexes").str();
-    std::string SelectorName2 = (llvm::Twine("with") + UpperKey).str();
+      = (Twine("replace") + UpperKey + "AtIndexes").str();
+    std::string SelectorName2 = (Twine("with") + UpperKey).str();
     IdentifierInfo *SelectorIds[2] = {
       &Context.Idents.get(SelectorName1),
       &Context.Idents.get(SelectorName2)
@@ -6101,7 +6217,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
         ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl() &&
         ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl()
           ->getName() == "NSEnumerator"))) {
-    std::string SelectorName = (llvm::Twine("enumeratorOf") + UpperKey).str();
+    std::string SelectorName = (Twine("enumeratorOf") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getNullarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6119,7 +6235,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // - (type *)memberOfKey:(type *)object
   if (IsInstanceMethod && 
       (ReturnType.isNull() || ReturnType->isObjCObjectPointerType())) {
-    std::string SelectorName = (llvm::Twine("memberOf") + UpperKey).str();
+    std::string SelectorName = (Twine("memberOf") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6136,6 +6252,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
         Builder.AddTextChunk(" *");
       } else {
         Builder.AddTextChunk(GetCompletionTypeString(ReturnType, Context, 
+                                                     Policy,
                                                      Builder.getAllocator()));
       }
       Builder.AddChunk(CodeCompletionString::CK_RightParen);
@@ -6149,7 +6266,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // - (void)addKeyObject:(type *)object
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
     std::string SelectorName
-      = (llvm::Twine("add") + UpperKey + llvm::Twine("Object")).str();
+      = (Twine("add") + UpperKey + Twine("Object")).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6171,7 +6288,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
 
   // - (void)addKey:(NSSet *)objects
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
-    std::string SelectorName = (llvm::Twine("add") + UpperKey).str();
+    std::string SelectorName = (Twine("add") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6193,7 +6310,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   // - (void)removeKeyObject:(type *)object
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
     std::string SelectorName
-      = (llvm::Twine("remove") + UpperKey + llvm::Twine("Object")).str();
+      = (Twine("remove") + UpperKey + Twine("Object")).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6215,7 +6332,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   
   // - (void)removeKey:(NSSet *)objects
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
-    std::string SelectorName = (llvm::Twine("remove") + UpperKey).str();
+    std::string SelectorName = (Twine("remove") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6236,7 +6353,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
 
   // - (void)intersectKey:(NSSet *)objects
   if (IsInstanceMethod && ReturnTypeMatchesVoid) {
-    std::string SelectorName = (llvm::Twine("intersect") + UpperKey).str();
+    std::string SelectorName = (Twine("intersect") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getUnarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6264,7 +6381,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
         ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl()
                                                     ->getName() == "NSSet"))) {
     std::string SelectorName 
-      = (llvm::Twine("keyPathsForValuesAffecting") + UpperKey).str();
+      = (Twine("keyPathsForValuesAffecting") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getNullarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6285,7 +6402,7 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
        ReturnType->isIntegerType() || 
        ReturnType->isBooleanType())) {
     std::string SelectorName 
-      = (llvm::Twine("automaticallyNotifiesObserversOf") + UpperKey).str();
+      = (Twine("automaticallyNotifiesObserversOf") + UpperKey).str();
     IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
     if (KnownSelectors.insert(Selectors.getNullarySelector(SelectorId))) {
       if (ReturnType.isNull()) {
@@ -6303,12 +6420,15 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
 
 void Sema::CodeCompleteObjCMethodDecl(Scope *S, 
                                       bool IsInstanceMethod,
-                                      ParsedType ReturnTy,
-                                      Decl *IDecl) {
+                                      ParsedType ReturnTy) {
   // Determine the return type of the method we're declaring, if
   // provided.
   QualType ReturnType = GetTypeFromParser(ReturnTy);
-
+  Decl *IDecl = 0;
+  if (CurContext->isObjCContainer()) {
+      ObjCContainerDecl *OCD = dyn_cast<ObjCContainerDecl>(CurContext);
+      IDecl = cast<Decl>(OCD);
+  }
   // Determine where we should start searching for methods.
   ObjCContainerDecl *SearchDecl = 0;
   bool IsInImplementation = false;
@@ -6346,9 +6466,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
-  PrintingPolicy Policy(Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  Policy.SuppressStrongLifetime = true;
+  PrintingPolicy Policy = getCompletionPrintingPolicy(*this);
   for (KnownMethodsMap::iterator M = KnownMethods.begin(), 
                               MEnd = KnownMethods.end();
        M != MEnd; ++M) {
@@ -6358,7 +6476,8 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
     // If the result type was not already provided, add it to the
     // pattern as (type).
     if (ReturnType.isNull())
-      AddObjCPassingTypeChunk(Method->getResultType(), Context, Builder);
+      AddObjCPassingTypeChunk(Method->getResultType(), Context, Policy, 
+                              Builder);
 
     Selector Sel = Method->getSelector();
 
@@ -6382,7 +6501,8 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
         break;
 
       // Add the parameter type.
-      AddObjCPassingTypeChunk((*P)->getOriginalType(), Context, Builder);
+      AddObjCPassingTypeChunk((*P)->getOriginalType(), Context, Policy, 
+                              Builder);
       
       if (IdentifierInfo *Id = (*P)->getIdentifier())
         Builder.AddTextChunk(Builder.getAllocator().CopyString( Id->getName())); 
@@ -6425,7 +6545,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   // Add Key-Value-Coding and Key-Value-Observing accessor methods for all of 
   // the properties in this class and its categories.
   if (Context.getLangOptions().ObjC2) {
-    llvm::SmallVector<ObjCContainerDecl *, 4> Containers;
+    SmallVector<ObjCContainerDecl *, 4> Containers;
     Containers.push_back(SearchDecl);
     
     VisitedSelectorSet KnownSelectors;
@@ -6748,9 +6868,8 @@ void Sema::CodeCompletePreprocessorMacroArgument(Scope *S,
   // FIXME: In the future, we could provide "overload" results, much like we
   // do for function calls.
   
-  CodeCompleteOrdinaryName(S,
-                           S->getFnParent()? Sema::PCC_RecoveryInFunction 
-                                           : Sema::PCC_Namespace);
+  // Now just ignore this. There will be another code-completion callback
+  // for the expanded tokens.
 }
 
 void Sema::CodeCompleteNaturalLanguage() {
@@ -6760,7 +6879,7 @@ void Sema::CodeCompleteNaturalLanguage() {
 }
 
 void Sema::GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,
-                 llvm::SmallVectorImpl<CodeCompletionResult> &Results) {
+                 SmallVectorImpl<CodeCompletionResult> &Results) {
   ResultBuilder Builder(*this, Allocator, CodeCompletionContext::CCC_Recovery);
   if (!CodeCompleter || CodeCompleter->includeGlobals()) {
     CodeCompletionDeclConsumer Consumer(Builder, 
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp
index 9d91a48bdcd7..8d993ef61032 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp
@@ -33,11 +33,13 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Basic/PartialDiagnostic.h"
+#include "clang/Sema/DelayedDiagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 // FIXME: layering (ideally, Sema shouldn't be dependent on Lex API's)
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "llvm/ADT/Triple.h"
 #include <algorithm>
 #include <cstring>
@@ -69,7 +71,8 @@ ParsedType Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
                              Scope *S, CXXScopeSpec *SS,
                              bool isClassName, bool HasTrailingDot,
                              ParsedType ObjectTypePtr,
-                             bool WantNontrivialTypeSourceInfo) {
+                             bool WantNontrivialTypeSourceInfo,
+                             IdentifierInfo **CorrectedII) {
   // Determine where we will perform name lookup.
   DeclContext *LookupCtx = 0;
   if (ObjectTypePtr) {
@@ -144,6 +147,51 @@ ParsedType Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
   switch (Result.getResultKind()) {
   case LookupResult::NotFound:
   case LookupResult::NotFoundInCurrentInstantiation:
+    if (CorrectedII) {
+      TypoCorrection Correction = CorrectTypo(Result.getLookupNameInfo(),
+                                              Kind, S, SS, 0, false,
+                                              Sema::CTC_Type);
+      IdentifierInfo *NewII = Correction.getCorrectionAsIdentifierInfo();
+      TemplateTy Template;
+      bool MemberOfUnknownSpecialization;
+      UnqualifiedId TemplateName;
+      TemplateName.setIdentifier(NewII, NameLoc);
+      NestedNameSpecifier *NNS = Correction.getCorrectionSpecifier();
+      CXXScopeSpec NewSS, *NewSSPtr = SS;
+      if (SS && NNS) {
+        NewSS.MakeTrivial(Context, NNS, SourceRange(NameLoc));
+        NewSSPtr = &NewSS;
+      }
+      if (Correction && (NNS || NewII != &II) &&
+          // Ignore a correction to a template type as the to-be-corrected
+          // identifier is not a template (typo correction for template names
+          // is handled elsewhere).
+          !(getLangOptions().CPlusPlus && NewSSPtr &&
+            isTemplateName(S, *NewSSPtr, false, TemplateName, ParsedType(),
+                           false, Template, MemberOfUnknownSpecialization))) {
+        ParsedType Ty = getTypeName(*NewII, NameLoc, S, NewSSPtr,
+                                    isClassName, HasTrailingDot, ObjectTypePtr,
+                                    WantNontrivialTypeSourceInfo);
+        if (Ty) {
+          std::string CorrectedStr(Correction.getAsString(getLangOptions()));
+          std::string CorrectedQuotedStr(
+              Correction.getQuoted(getLangOptions()));
+          Diag(NameLoc, diag::err_unknown_typename_suggest)
+              << Result.getLookupName() << CorrectedQuotedStr
+              << FixItHint::CreateReplacement(SourceRange(NameLoc),
+                                              CorrectedStr);
+          if (NamedDecl *FirstDecl = Correction.getCorrectionDecl())
+            Diag(FirstDecl->getLocation(), diag::note_previous_decl)
+              << CorrectedQuotedStr;
+
+          if (SS && NNS)
+            SS->MakeTrivial(Context, NNS, SourceRange(NameLoc));
+          *CorrectedII = NewII;
+          return Ty;
+        }
+      }
+    }
+    // If typo correction failed or was not performed, fall through
   case LookupResult::FoundOverloaded:
   case LookupResult::FoundUnresolvedValue:
     Result.suppressDiagnostics();
@@ -269,7 +317,7 @@ DeclSpec::TST Sema::isTagName(IdentifierInfo &II, Scope *S) {
 ///   A<T>::TYPE a; // no typename required because A<T> is a base class.
 /// };
 /// @endcode
-bool Sema::isMicrosoftMissingTypename(const CXXScopeSpec *SS) {
+bool Sema::isMicrosoftMissingTypename(const CXXScopeSpec *SS, Scope *S) {
   if (CurContext->isRecord()) {
     const Type *Ty = SS->getScopeRep()->getAsType();
 
@@ -278,8 +326,9 @@ bool Sema::isMicrosoftMissingTypename(const CXXScopeSpec *SS) {
           BaseEnd = RD->bases_end(); Base != BaseEnd; ++Base)
       if (Context.hasSameUnqualifiedType(QualType(Ty, 1), Base->getType()))
         return true;
+    return S->isFunctionPrototypeScope();
   } 
-  return false;
+  return CurContext->isFunctionOrMethod() || S->isFunctionPrototypeScope();
 }
 
 bool Sema::DiagnoseUnknownTypeName(const IdentifierInfo &II, 
@@ -361,7 +410,7 @@ bool Sema::DiagnoseUnknownTypeName(const IdentifierInfo &II,
       << &II << DC << SS->getRange();
   else if (isDependentScopeSpecifier(*SS)) {
     unsigned DiagID = diag::err_typename_missing;
-    if (getLangOptions().Microsoft && isMicrosoftMissingTypename(SS))
+    if (getLangOptions().MicrosoftMode && isMicrosoftMissingTypename(SS, S))
       DiagID = diag::warn_typename_missing;
 
     Diag(SS->getRange().getBegin(), DiagID)
@@ -419,24 +468,6 @@ Sema::NameClassification Sema::ClassifyName(Scope *S,
     ExprResult E = LookupInObjCMethod(Result, S, Name, true);
     if (E.get() || E.isInvalid())
       return E;
-    
-    // Synthesize ivars lazily.
-    if (getLangOptions().ObjCDefaultSynthProperties &&
-        getLangOptions().ObjCNonFragileABI2) {
-      if (SynthesizeProvisionalIvar(Result, Name, NameLoc)) {
-        if (const ObjCPropertyDecl *Property = 
-                                          canSynthesizeProvisionalIvar(Name)) {
-          Diag(NameLoc, diag::warn_synthesized_ivar_access) << Name;
-          Diag(Property->getLocation(), diag::note_property_declare);
-        }
-
-        // FIXME: This is strange. Shouldn't we just take the ivar returned
-        // from SynthesizeProvisionalIvar and continue with that?
-        E = LookupInObjCMethod(Result, S, Name, true);   
-        if (E.get() || E.isInvalid())
-          return E;
-      }
-    }
   }
   
   bool SecondTry = false;
@@ -737,11 +768,6 @@ DeclContext *Sema::getContainingDC(DeclContext *DC) {
     return DC;
   }
 
-  // ObjCMethodDecls are parsed (for some reason) outside the context
-  // of the class.
-  if (isa<ObjCMethodDecl>(DC))
-    return DC->getLexicalParent()->getLexicalParent();
-
   return DC->getLexicalParent();
 }
 
@@ -804,6 +830,29 @@ void Sema::ExitDeclaratorContext(Scope *S) {
   // disappear.
 }
 
+
+void Sema::ActOnReenterFunctionContext(Scope* S, Decl *D) {
+  FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+  if (FunctionTemplateDecl *TFD = dyn_cast_or_null<FunctionTemplateDecl>(D)) {
+    // We assume that the caller has already called
+    // ActOnReenterTemplateScope
+    FD = TFD->getTemplatedDecl();
+  }
+  if (!FD)
+    return;
+
+  PushDeclContext(S, FD);
+  for (unsigned P = 0, NumParams = FD->getNumParams(); P < NumParams; ++P) {
+    ParmVarDecl *Param = FD->getParamDecl(P);
+    // If the parameter has an identifier, then add it to the scope
+    if (Param->getIdentifier()) {
+      S->AddDecl(Param);
+      IdResolver.AddDecl(Param);
+    }
+  }
+}
+
+
 /// \brief Determine whether we allow overloading of the function
 /// PrevDecl with another declaration.
 ///
@@ -843,7 +892,9 @@ void Sema::PushOnScopeChains(NamedDecl *D, Scope *S, bool AddToContext) {
   // Out-of-line definitions shouldn't be pushed into scope in C++.
   // Out-of-line variable and function definitions shouldn't even in C.
   if ((getLangOptions().CPlusPlus || isa<VarDecl>(D) || isa<FunctionDecl>(D)) &&
-      D->isOutOfLine())
+      D->isOutOfLine() &&
+      !D->getDeclContext()->getRedeclContext()->Equals(
+        D->getLexicalDeclContext()->getRedeclContext()))
     return;
 
   // Template instantiations should also not be pushed into scope.
@@ -1086,12 +1137,28 @@ static bool ShouldDiagnoseUnusedDecl(const NamedDecl *D) {
   return true;
 }
 
+static void GenerateFixForUnusedDecl(const NamedDecl *D, ASTContext &Ctx,
+                                     FixItHint &Hint) {
+  if (isa<LabelDecl>(D)) {
+    SourceLocation AfterColon = Lexer::findLocationAfterToken(D->getLocEnd(),
+                tok::colon, Ctx.getSourceManager(), Ctx.getLangOptions(), true);
+    if (AfterColon.isInvalid())
+      return;
+    Hint = FixItHint::CreateRemoval(CharSourceRange::
+                                    getCharRange(D->getLocStart(), AfterColon));
+  }
+  return;
+}
+
 /// DiagnoseUnusedDecl - Emit warnings about declarations that are not used
 /// unless they are marked attr(unused).
 void Sema::DiagnoseUnusedDecl(const NamedDecl *D) {
+  FixItHint Hint;
   if (!ShouldDiagnoseUnusedDecl(D))
     return;
   
+  GenerateFixForUnusedDecl(D, Context, Hint);
+
   unsigned DiagID;
   if (isa<VarDecl>(D) && cast<VarDecl>(D)->isExceptionVariable())
     DiagID = diag::warn_unused_exception_param;
@@ -1100,7 +1167,7 @@ void Sema::DiagnoseUnusedDecl(const NamedDecl *D) {
   else
     DiagID = diag::warn_unused_variable;
 
-  Diag(D->getLocation(), DiagID) << D->getDeclName();
+  Diag(D->getLocation(), DiagID) << D->getDeclName() << Hint;
 }
 
 static void CheckPoppedLabel(LabelDecl *L, Sema &S) {
@@ -1246,7 +1313,7 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
       << R;
     if (Context.BuiltinInfo.getHeaderName(BID) &&
         Diags.getDiagnosticLevel(diag::ext_implicit_lib_function_decl, Loc)
-          != Diagnostic::Ignored)
+          != DiagnosticsEngine::Ignored)
       Diag(Loc, diag::note_please_include_header)
         << Context.BuiltinInfo.getHeaderName(BID)
         << Context.BuiltinInfo.GetName(BID);
@@ -1263,7 +1330,7 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
   // Create Decl objects for each parameter, adding them to the
   // FunctionDecl.
   if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(R)) {
-    llvm::SmallVector<ParmVarDecl*, 16> Params;
+    SmallVector<ParmVarDecl*, 16> Params;
     for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
       ParmVarDecl *parm =
         ParmVarDecl::Create(Context, New, SourceLocation(),
@@ -1273,7 +1340,7 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
       parm->setScopeInfo(0, i);
       Params.push_back(parm);
     }
-    New->setParams(Params.data(), Params.size());
+    New->setParams(Params);
   }
 
   AddKnownFunctionAttributes(New);
@@ -1308,29 +1375,24 @@ void Sema::MergeTypedefNameDecl(TypedefNameDecl *New, LookupResult &OldDecls) {
     case 2:
       if (!TypeID->isStr("id"))
         break;
-      Context.ObjCIdRedefinitionType = New->getUnderlyingType();
+      Context.setObjCIdRedefinitionType(New->getUnderlyingType());
       // Install the built-in type for 'id', ignoring the current definition.
       New->setTypeForDecl(Context.getObjCIdType().getTypePtr());
       return;
     case 5:
       if (!TypeID->isStr("Class"))
         break;
-      Context.ObjCClassRedefinitionType = New->getUnderlyingType();
+      Context.setObjCClassRedefinitionType(New->getUnderlyingType());
       // Install the built-in type for 'Class', ignoring the current definition.
       New->setTypeForDecl(Context.getObjCClassType().getTypePtr());
       return;
     case 3:
       if (!TypeID->isStr("SEL"))
         break;
-      Context.ObjCSelRedefinitionType = New->getUnderlyingType();
+      Context.setObjCSelRedefinitionType(New->getUnderlyingType());
       // Install the built-in type for 'SEL', ignoring the current definition.
       New->setTypeForDecl(Context.getObjCSelType().getTypePtr());
       return;
-    case 8:
-      if (!TypeID->isStr("Protocol"))
-        break;
-      Context.setObjCProtoType(New->getUnderlyingType());
-      return;
     }
     // Fall through - the typedef name was not a builtin type.
   }
@@ -1382,7 +1444,13 @@ void Sema::MergeTypedefNameDecl(TypedefNameDecl *New, LookupResult &OldDecls) {
   if (TypedefNameDecl *Typedef = dyn_cast<TypedefNameDecl>(Old))
     New->setPreviousDeclaration(Typedef);
 
-  if (getLangOptions().Microsoft)
+  // __module_private__ is propagated to later declarations.
+  if (Old->isModulePrivate())
+    New->setModulePrivate();
+  else if (New->isModulePrivate())
+    diagnoseModulePrivateRedeclaration(New, Old);
+           
+  if (getLangOptions().MicrosoftExt)
     return;
 
   if (getLangOptions().CPlusPlus) {
@@ -1443,8 +1511,14 @@ void Sema::MergeTypedefNameDecl(TypedefNameDecl *New, LookupResult &OldDecls) {
 static bool
 DeclHasAttr(const Decl *D, const Attr *A) {
   const OwnershipAttr *OA = dyn_cast<OwnershipAttr>(A);
+  const AnnotateAttr *Ann = dyn_cast<AnnotateAttr>(A);
   for (Decl::attr_iterator i = D->attr_begin(), e = D->attr_end(); i != e; ++i)
     if ((*i)->getKind() == A->getKind()) {
+      if (Ann) {
+        if (Ann->getAnnotation() == cast<AnnotateAttr>(*i)->getAnnotation())
+          return true;
+        continue;
+      }
       // FIXME: Don't hardcode this check
       if (OA && isa<OwnershipAttr>(*i))
         return OA->getOwnKind() == cast<OwnershipAttr>(*i)->getOwnKind();
@@ -1456,7 +1530,7 @@ DeclHasAttr(const Decl *D, const Attr *A) {
 
 /// mergeDeclAttributes - Copy attributes from the Old decl to the New one.
 static void mergeDeclAttributes(Decl *newDecl, const Decl *oldDecl,
-                                ASTContext &C) {
+                                ASTContext &C, bool mergeDeprecation = true) {
   if (!oldDecl->hasAttrs())
     return;
 
@@ -1469,6 +1543,13 @@ static void mergeDeclAttributes(Decl *newDecl, const Decl *oldDecl,
   for (specific_attr_iterator<InheritableAttr>
        i = oldDecl->specific_attr_begin<InheritableAttr>(),
        e = oldDecl->specific_attr_end<InheritableAttr>(); i != e; ++i) {
+    // Ignore deprecated/unavailable/availability attributes if requested.
+    if (!mergeDeprecation &&
+        (isa<DeprecatedAttr>(*i) || 
+         isa<UnavailableAttr>(*i) ||
+         isa<AvailabilityAttr>(*i)))
+      continue;
+
     if (!DeclHasAttr(newDecl, *i)) {
       InheritableAttr *newAttr = cast<InheritableAttr>((*i)->clone(C));
       newAttr->setInherited(true);
@@ -1535,6 +1616,8 @@ Sema::CXXSpecialMember Sema::getSpecialMember(const CXXMethodDecl *MD) {
     return Sema::CXXDestructor;
   } else if (MD->isCopyAssignmentOperator()) {
     return Sema::CXXCopyAssignment;
+  } else if (MD->isMoveAssignmentOperator()) {
+    return Sema::CXXMoveAssignment;
   }
 
   return Sema::CXXInvalid;
@@ -1605,7 +1688,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
       New->getStorageClass() == SC_Static &&
       Old->getStorageClass() != SC_Static &&
       !canRedefineFunction(Old, getLangOptions())) {
-    if (getLangOptions().Microsoft) {
+    if (getLangOptions().MicrosoftExt) {
       Diag(New->getLocation(), diag::warn_static_non_static) << New;
       Diag(Old->getLocation(), PrevDiag);
     } else {
@@ -1669,6 +1752,18 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
     RequiresAdjustment = true;
   }
 
+  // Merge ns_returns_retained attribute.
+  if (OldTypeInfo.getProducesResult() != NewTypeInfo.getProducesResult()) {
+    if (NewTypeInfo.getProducesResult()) {
+      Diag(New->getLocation(), diag::err_returns_retained_mismatch);
+      Diag(Old->getLocation(), diag::note_previous_declaration);      
+      return true;
+    }
+    
+    NewTypeInfo = NewTypeInfo.withProducesResult(true);
+    RequiresAdjustment = true;
+  }
+  
   if (RequiresAdjustment) {
     NewType = Context.adjustFunctionType(NewType, NewTypeInfo);
     New->setType(QualType(NewType, 0));
@@ -1706,9 +1801,16 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
       // Preserve triviality.
       NewMethod->setTrivial(OldMethod->isTrivial());
 
+      // MSVC allows explicit template specialization at class scope:
+      // 2 CXMethodDecls referring to the same function will be injected.
+      // We don't want a redeclartion error.
+      bool IsClassScopeExplicitSpecialization =
+                              OldMethod->isFunctionTemplateSpecialization() &&
+                              NewMethod->isFunctionTemplateSpecialization();
       bool isFriend = NewMethod->getFriendObjectKind();
 
-      if (!isFriend && NewMethod->getLexicalDeclContext()->isRecord()) {
+      if (!isFriend && NewMethod->getLexicalDeclContext()->isRecord() &&
+          !IsClassScopeExplicitSpecialization) {
         //    -- Member function declarations with the same name and the
         //       same parameter types cannot be overloaded if any of them
         //       is a static member function declaration.
@@ -1790,7 +1892,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
       // The old declaration provided a function prototype, but the
       // new declaration does not. Merge in the prototype.
       assert(!OldProto->hasExceptionSpec() && "Exception spec in C");
-      llvm::SmallVector<QualType, 16> ParamTypes(OldProto->arg_type_begin(),
+      SmallVector<QualType, 16> ParamTypes(OldProto->arg_type_begin(),
                                                  OldProto->arg_type_end());
       NewQType = Context.getFunctionType(NewFuncType->getResultType(),
                                          ParamTypes.data(), ParamTypes.size(),
@@ -1799,7 +1901,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
       New->setHasInheritedPrototype();
 
       // Synthesize a parameter for each argument type.
-      llvm::SmallVector<ParmVarDecl*, 16> Params;
+      SmallVector<ParmVarDecl*, 16> Params;
       for (FunctionProtoType::arg_type_iterator
              ParamType = OldProto->arg_type_begin(),
              ParamEnd = OldProto->arg_type_end();
@@ -1815,7 +1917,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
         Params.push_back(Param);
       }
 
-      New->setParams(Params.data(), Params.size());
+      New->setParams(Params);
     }
 
     return MergeCompatibleFunctionDecls(New, Old);
@@ -1836,8 +1938,8 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
       Old->hasPrototype() && !New->hasPrototype() &&
       New->getType()->getAs<FunctionProtoType>() &&
       Old->getNumParams() == New->getNumParams()) {
-    llvm::SmallVector<QualType, 16> ArgTypes;
-    llvm::SmallVector<GNUCompatibleParamWarning, 16> Warnings;
+    SmallVector<QualType, 16> ArgTypes;
+    SmallVector<GNUCompatibleParamWarning, 16> Warnings;
     const FunctionProtoType *OldProto
       = Old->getType()->getAs<FunctionProtoType>();
     const FunctionProtoType *NewProto
@@ -1933,6 +2035,12 @@ bool Sema::MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old) {
   if (Old->isPure())
     New->setPure();
 
+  // __module_private__ is propagated to later declarations.
+  if (Old->isModulePrivate())
+    New->setModulePrivate();
+  else if (New->isModulePrivate())
+    diagnoseModulePrivateRedeclaration(New, Old);
+
   // Merge attributes from the parameters.  These can mismatch with K&R
   // declarations.
   if (New->getNumParams() == Old->getNumParams())
@@ -1949,15 +2057,20 @@ bool Sema::MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old) {
 
 void Sema::mergeObjCMethodDecls(ObjCMethodDecl *newMethod,
                                 const ObjCMethodDecl *oldMethod) {
+  // We don't want to merge unavailable and deprecated attributes
+  // except from interface to implementation.
+  bool mergeDeprecation = isa<ObjCImplDecl>(newMethod->getDeclContext());
+
   // Merge the attributes.
-  mergeDeclAttributes(newMethod, oldMethod, Context);
+  mergeDeclAttributes(newMethod, oldMethod, Context, mergeDeprecation);
 
   // Merge attributes from the parameters.
-  for (ObjCMethodDecl::param_iterator oi = oldMethod->param_begin(),
+  ObjCMethodDecl::param_const_iterator oi = oldMethod->param_begin();
+  for (ObjCMethodDecl::param_iterator
          ni = newMethod->param_begin(), ne = newMethod->param_end();
        ni != ne; ++ni, ++oi)
     mergeParamDeclAttributes(*ni, *oi, Context);
-  
+
   CheckObjCMethodOverride(newMethod, oldMethod, true);
 }
 
@@ -2111,6 +2224,12 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
     return New->setInvalidDecl();
   }
 
+  // __module_private__ is propagated to later declarations.
+  if (Old->isModulePrivate())
+    New->setModulePrivate();
+  else if (New->isModulePrivate())
+    diagnoseModulePrivateRedeclaration(New, Old);
+
   // Variables with external linkage are analyzed in FinalizeDeclaratorGroup.
 
   // FIXME: The test for external storage here seems wrong? We still
@@ -2193,6 +2312,9 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     Tag = dyn_cast<TagDecl>(TagD);
   }
 
+  if (Tag)
+    Tag->setFreeStanding();
+
   if (unsigned TypeQuals = DS.getTypeQualifiers()) {
     // Enforce C99 6.7.3p2: "Types other than pointer types derived from object
     // or incomplete types shall not be restrict-qualified."
@@ -2202,6 +2324,20 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
            << DS.getSourceRange();
   }
 
+  if (DS.isConstexprSpecified()) {
+    // C++0x [dcl.constexpr]p1: constexpr can only be applied to declarations
+    // and definitions of functions and variables.
+    if (Tag)
+      Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_tag)
+        << (DS.getTypeSpecType() == DeclSpec::TST_class ? 0 :
+            DS.getTypeSpecType() == DeclSpec::TST_struct ? 1 :
+            DS.getTypeSpecType() == DeclSpec::TST_union ? 2 : 3);
+    else
+      Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_no_declarators);
+    // Don't emit warnings after this error.
+    return TagD;
+  }
+
   if (DS.isFriendSpecified()) {
     // If we're dealing with a decl but not a TagDecl, assume that
     // whatever routines created it handled the friendship aspect.
@@ -2217,7 +2353,7 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   if (RecordDecl *Record = dyn_cast_or_null<RecordDecl>(Tag)) {
     ProcessDeclAttributeList(S, Record, DS.getAttributes().getList());
     
-    if (!Record->getDeclName() && Record->isDefinition() &&
+    if (!Record->getDeclName() && Record->isCompleteDefinition() &&
         DS.getStorageClassSpec() != DeclSpec::SCS_typedef) {
       if (getLangOptions().CPlusPlus ||
           Record->getDeclContext()->isRecord())
@@ -2230,7 +2366,7 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   }
 
   // Check for Microsoft C extension: anonymous struct.
-  if (getLangOptions().Microsoft && !getLangOptions().CPlusPlus &&
+  if (getLangOptions().MicrosoftExt && !getLangOptions().CPlusPlus &&
       CurContext->isRecord() &&
       DS.getStorageClassSpec() == DeclSpec::SCS_unspecified) {
     // Handle 2 kinds of anonymous struct:
@@ -2238,7 +2374,7 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     // and
     //   STRUCT_TYPE;  <- where STRUCT_TYPE is a typedef struct.
     RecordDecl *Record = dyn_cast_or_null<RecordDecl>(Tag);
-    if ((Record && Record->getDeclName() && !Record->isDefinition()) ||
+    if ((Record && Record->getDeclName() && !Record->isCompleteDefinition()) ||
         (DS.getTypeSpecType() == DeclSpec::TST_typename &&
          DS.getRepAsType().get()->isStructureType())) {
       Diag(DS.getSourceRange().getBegin(), diag::ext_ms_anonymous_struct)
@@ -2305,6 +2441,12 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   if (DS.isExplicitSpecified())
     Diag(DS.getExplicitSpecLoc(), diag::warn_standalone_specifier) <<"explicit";
 
+  if (DS.isModulePrivateSpecified() && 
+      Tag && Tag->getDeclContext()->isFunctionOrMethod())
+    Diag(DS.getModulePrivateSpecLoc(), diag::err_module_private_local_class)
+      << Tag->getTagKind()
+      << FixItHint::CreateRemoval(DS.getModulePrivateSpecLoc());
+
   // FIXME: Warn on useless attributes
 
   return TagD;
@@ -2379,7 +2521,7 @@ static bool InjectAnonymousStructOrUnionMembers(Sema &SemaRef, Scope *S,
                                                 DeclContext *Owner,
                                                 RecordDecl *AnonRecord,
                                                 AccessSpecifier AS,
-                              llvm::SmallVector<NamedDecl*, 2> &Chaining,
+                              SmallVector<NamedDecl*, 2> &Chaining,
                                                       bool MSAnonStruct) {
   unsigned diagKind
     = AnonRecord->isUnion() ? diag::err_anonymous_union_member_redecl
@@ -2511,8 +2653,8 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
       Invalid = true;
 
       // Recover by adding 'static'.
-      DS.SetStorageClassSpec(DeclSpec::SCS_static, SourceLocation(),
-                             PrevSpec, DiagID, getLangOptions());
+      DS.SetStorageClassSpec(*this, DeclSpec::SCS_static, SourceLocation(),
+                             PrevSpec, DiagID);
     }
     // C++ [class.union]p3:
     //   A storage class is not allowed in a declaration of an
@@ -2524,8 +2666,8 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
       Invalid = true;
 
       // Recover by removing the storage specifier.
-      DS.SetStorageClassSpec(DeclSpec::SCS_unspecified, SourceLocation(),
-                             PrevSpec, DiagID, getLangOptions());
+      DS.SetStorageClassSpec(*this, DeclSpec::SCS_unspecified, SourceLocation(),
+                             PrevSpec, DiagID);
     }
 
     // Ignore const/volatile/restrict qualifiers.
@@ -2582,7 +2724,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
         if (!MemRecord->isAnonymousStructOrUnion() &&
             MemRecord->getDeclName()) {
           // Visual C++ allows type definition in anonymous struct or union.
-          if (getLangOptions().Microsoft)
+          if (getLangOptions().MicrosoftExt)
             Diag(MemRecord->getLocation(), diag::ext_anonymous_record_with_type)
               << (int)Record->isUnion();
           else {
@@ -2606,7 +2748,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
           DK = diag::err_anonymous_record_with_static;
         
         // Visual C++ allows type definition in anonymous struct or union.
-        if (getLangOptions().Microsoft &&
+        if (getLangOptions().MicrosoftExt &&
             DK == diag::err_anonymous_record_with_type)
           Diag((*Mem)->getLocation(), diag::ext_anonymous_record_with_type)
             << (int)Record->isUnion();
@@ -2665,6 +2807,12 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
                            Record->getLocation(), /*IdentifierInfo=*/0,
                            Context.getTypeDeclType(Record),
                            TInfo, SC, SCAsWritten);
+
+    // Default-initialize the implicit variable. This initialization will be
+    // trivial in almost all cases, except if a union member has an in-class
+    // initializer:
+    //   union { int n = 0; };
+    ActOnUninitializedDecl(Anon, /*TypeMayContainAuto=*/false);
   }
   Anon->setImplicit();
 
@@ -2676,7 +2824,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
   // Inject the members of the anonymous struct/union into the owning
   // context and into the identifier resolver chain for name lookup
   // purposes.
-  llvm::SmallVector<NamedDecl*, 2> Chain;
+  SmallVector<NamedDecl*, 2> Chain;
   Chain.push_back(Anon);
 
   if (InjectAnonymousStructOrUnionMembers(*this, S, Owner, Record, AS,
@@ -2740,7 +2888,7 @@ Decl *Sema::BuildMicrosoftCAnonymousStruct(Scope *S, DeclSpec &DS,
   // Inject the members of the anonymous struct into the current
   // context and into the identifier resolver chain for name lookup
   // purposes.
-  llvm::SmallVector<NamedDecl*, 2> Chain;
+  SmallVector<NamedDecl*, 2> Chain;
   Chain.push_back(Anon);
 
   if (InjectAnonymousStructOrUnionMembers(*this, S, CurContext,
@@ -2855,26 +3003,51 @@ Sema::GetNameFromUnqualifiedId(const UnqualifiedId &Name) {
 
   } // switch (Name.getKind())
 
-  assert(false && "Unknown name kind");
-  return DeclarationNameInfo();
+  llvm_unreachable("Unknown name kind");
 }
 
-/// isNearlyMatchingFunction - Determine whether the C++ functions
-/// Declaration and Definition are "nearly" matching. This heuristic
-/// is used to improve diagnostics in the case where an out-of-line
-/// function definition doesn't match any declaration within
-/// the class or namespace.
-static bool isNearlyMatchingFunction(ASTContext &Context,
+static QualType getCoreType(QualType Ty) {
+  do {
+    if (Ty->isPointerType() || Ty->isReferenceType())
+      Ty = Ty->getPointeeType();
+    else if (Ty->isArrayType())
+      Ty = Ty->castAsArrayTypeUnsafe()->getElementType();
+    else
+      return Ty.withoutLocalFastQualifiers();
+  } while (true);
+}
+
+/// hasSimilarParameters - Determine whether the C++ functions Declaration
+/// and Definition have "nearly" matching parameters. This heuristic is
+/// used to improve diagnostics in the case where an out-of-line function
+/// definition doesn't match any declaration within the class or namespace.
+/// Also sets Params to the list of indices to the parameters that differ
+/// between the declaration and the definition. If hasSimilarParameters
+/// returns true and Params is empty, then all of the parameters match.
+static bool hasSimilarParameters(ASTContext &Context,
                                      FunctionDecl *Declaration,
-                                     FunctionDecl *Definition) {
+                                     FunctionDecl *Definition,
+                                     llvm::SmallVectorImpl<unsigned> &Params) {
+  Params.clear();
   if (Declaration->param_size() != Definition->param_size())
     return false;
   for (unsigned Idx = 0; Idx < Declaration->param_size(); ++Idx) {
     QualType DeclParamTy = Declaration->getParamDecl(Idx)->getType();
     QualType DefParamTy = Definition->getParamDecl(Idx)->getType();
 
-    if (!Context.hasSameUnqualifiedType(DeclParamTy.getNonReferenceType(),
-                                        DefParamTy.getNonReferenceType()))
+    // The parameter types are identical
+    if (Context.hasSameType(DefParamTy, DeclParamTy))
+      continue;
+
+    QualType DeclParamBaseTy = getCoreType(DeclParamTy);
+    QualType DefParamBaseTy = getCoreType(DefParamTy);
+    const IdentifierInfo *DeclTyName = DeclParamBaseTy.getBaseTypeIdentifier();
+    const IdentifierInfo *DefTyName = DefParamBaseTy.getBaseTypeIdentifier();
+
+    if (Context.hasSameUnqualifiedType(DeclParamBaseTy, DefParamBaseTy) ||
+        (DeclTyName && DeclTyName == DefTyName))
+      Params.push_back(Idx);
+    else  // The two parameters aren't even close
       return false;
   }
 
@@ -2900,7 +3073,8 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
   case DeclSpec::TST_typename:
   case DeclSpec::TST_typeofType:
   case DeclSpec::TST_decltype:
-  case DeclSpec::TST_underlyingType: {
+  case DeclSpec::TST_underlyingType:
+  case DeclSpec::TST_atomic: {
     // Grab the type from the parser.
     TypeSourceInfo *TSI = 0;
     QualType T = S.GetTypeFromParser(DS.getRepAsType(), &TSI);
@@ -2955,8 +3129,8 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
 }
 
 Decl *Sema::ActOnDeclarator(Scope *S, Declarator &D) {
-  return HandleDeclarator(S, D, MultiTemplateParamsArg(*this),
-                          /*IsFunctionDefinition=*/false);
+  D.setFunctionDefinition(false);
+  return HandleDeclarator(S, D, MultiTemplateParamsArg(*this));
 }
 
 /// DiagnoseClassNameShadow - Implement C++ [class.mem]p13:
@@ -2980,8 +3154,7 @@ bool Sema::DiagnoseClassNameShadow(DeclContext *DC,
 }
   
 Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
-                             MultiTemplateParamsArg TemplateParamLists,
-                             bool IsFunctionDefinition) {
+                             MultiTemplateParamsArg TemplateParamLists) {
   // TODO: consider using NameInfo for diagnostic.
   DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
   DeclarationName Name = NameInfo.getName();
@@ -3180,22 +3353,21 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
       D.getDeclSpec().getStorageClassSpec() != DeclSpec::SCS_typedef)
     Previous.clear();
 
-  bool Redeclaration = false;
+  bool AddToScope = true;
   if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_typedef) {
     if (TemplateParamLists.size()) {
       Diag(D.getIdentifierLoc(), diag::err_template_typedef);
       return 0;
     }
 
-    New = ActOnTypedefDeclarator(S, D, DC, R, TInfo, Previous, Redeclaration);
+    New = ActOnTypedefDeclarator(S, D, DC, TInfo, Previous);
   } else if (R->isFunctionType()) {
-    New = ActOnFunctionDeclarator(S, D, DC, R, TInfo, Previous,
+    New = ActOnFunctionDeclarator(S, D, DC, TInfo, Previous,
                                   move(TemplateParamLists),
-                                  IsFunctionDefinition, Redeclaration);
+                                  AddToScope);
   } else {
-    New = ActOnVariableDeclarator(S, D, DC, R, TInfo, Previous,
-                                  move(TemplateParamLists),
-                                  Redeclaration);
+    New = ActOnVariableDeclarator(S, D, DC, TInfo, Previous,
+                                  move(TemplateParamLists));
   }
 
   if (New == 0)
@@ -3203,7 +3375,8 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
 
   // If this has an identifier and is not an invalid redeclaration or 
   // function template specialization, add it to the scope stack.
-  if (New->getDeclName() && !(Redeclaration && New->isInvalidDecl()))
+  if (New->getDeclName() && AddToScope &&
+       !(D.isRedeclaration() && New->isInvalidDecl()))
     PushOnScopeChains(New, S);
 
   return New;
@@ -3321,6 +3494,23 @@ Sema::RegisterLocallyScopedExternCDecl(NamedDecl *ND,
   }
 }
 
+llvm::DenseMap<DeclarationName, NamedDecl *>::iterator
+Sema::findLocallyScopedExternalDecl(DeclarationName Name) {
+  if (ExternalSource) {
+    // Load locally-scoped external decls from the external source.
+    SmallVector<NamedDecl *, 4> Decls;
+    ExternalSource->ReadLocallyScopedExternalDecls(Decls);
+    for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
+      llvm::DenseMap<DeclarationName, NamedDecl *>::iterator Pos
+        = LocallyScopedExternalDecls.find(Decls[I]->getDeclName());
+      if (Pos == LocallyScopedExternalDecls.end())
+        LocallyScopedExternalDecls[Decls[I]->getDeclName()] = Decls[I];
+    }
+  }
+  
+  return LocallyScopedExternalDecls.find(Name);
+}
+
 /// \brief Diagnose function specifiers on a declaration of an identifier that
 /// does not identify a function.
 void Sema::DiagnoseFunctionSpecifiers(Declarator& D) {
@@ -3341,8 +3531,7 @@ void Sema::DiagnoseFunctionSpecifiers(Declarator& D) {
 
 NamedDecl*
 Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                             QualType R,  TypeSourceInfo *TInfo,
-                             LookupResult &Previous, bool &Redeclaration) {
+                             TypeSourceInfo *TInfo, LookupResult &Previous) {
   // Typedef declarators cannot be qualified (C++ [dcl.meaning]p1).
   if (D.getCXXScopeSpec().isSet()) {
     Diag(D.getIdentifierLoc(), diag::err_qualified_typedef_declarator)
@@ -3362,6 +3551,9 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
   if (D.getDeclSpec().isThreadSpecified())
     Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_invalid_thread);
+  if (D.getDeclSpec().isConstexprSpecified())
+    Diag(D.getDeclSpec().getConstexprSpecLoc(), diag::err_invalid_constexpr)
+      << 1;
 
   if (D.getName().Kind != UnqualifiedId::IK_Identifier) {
     Diag(D.getName().StartLocation, diag::err_typedef_not_identifier)
@@ -3369,7 +3561,7 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     return 0;
   }
 
-  TypedefDecl *NewTD = ParseTypedefDecl(S, D, R, TInfo);
+  TypedefDecl *NewTD = ParseTypedefDecl(S, D, TInfo->getType(), TInfo);
   if (!NewTD) return 0;
 
   // Handle attributes prior to checking for duplicates in MergeVarDecl
@@ -3377,7 +3569,10 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
   CheckTypedefForVariablyModifiedType(S, NewTD);
 
-  return ActOnTypedefNameDecl(S, DC, NewTD, Previous, Redeclaration);
+  bool Redeclaration = D.isRedeclaration();
+  NamedDecl *ND = ActOnTypedefNameDecl(S, DC, NewTD, Previous, Redeclaration);
+  D.setRedeclaration(Redeclaration);
+  return ND;
 }
 
 void
@@ -3557,10 +3752,9 @@ bool Sema::inferObjCARCLifetime(ValueDecl *decl) {
 
 NamedDecl*
 Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
-                              QualType R, TypeSourceInfo *TInfo,
-                              LookupResult &Previous,
-                              MultiTemplateParamsArg TemplateParamLists,
-                              bool &Redeclaration) {
+                              TypeSourceInfo *TInfo, LookupResult &Previous,
+                              MultiTemplateParamsArg TemplateParamLists) {
+  QualType R = TInfo->getType();
   DeclarationName Name = GetNameForDeclarator(D).getName();
 
   // Check that there are no default arguments (C++ only).
@@ -3585,7 +3779,7 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   IdentifierInfo *II = Name.getAsIdentifierInfo();
   if (!II) {
     Diag(D.getIdentifierLoc(), diag::err_bad_variable_name)
-      << Name.getAsString();
+      << Name;
     return 0;
   }
 
@@ -3606,6 +3800,13 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     }
   }
   
+  if (getLangOptions().OpenCL) {
+    // Set up the special work-group-local storage class for variables in the
+    // OpenCL __local address space.
+    if (R.getAddressSpace() == LangAS::opencl_local)
+      SC = SC_OpenCLWorkGroupLocal;
+  }
+
   bool isExplicitSpecialization = false;
   VarDecl *NewVD;
   if (!getLangOptions().CPlusPlus) {
@@ -3695,21 +3896,68 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
                                            TemplateParamLists.size(),
                                            TemplateParamLists.release());
     }
-  }
 
-  if (D.getDeclSpec().isThreadSpecified()) {
-    if (NewVD->hasLocalStorage())
-      Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_thread_non_global);
-    else if (!Context.Target.isTLSSupported())
-      Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_thread_unsupported);
-    else
-      NewVD->setThreadSpecified(true);
+    if (D.getDeclSpec().isConstexprSpecified()) {
+      // FIXME: once we know whether there's an initializer, apply this to
+      // static data members too.
+      if (!NewVD->isStaticDataMember() &&
+          !NewVD->isThisDeclarationADefinition()) {
+        // 'constexpr' is redundant and ill-formed on a non-defining declaration
+        // of a variable. Suggest replacing it with 'const' if appropriate.
+        SourceLocation ConstexprLoc = D.getDeclSpec().getConstexprSpecLoc();
+        SourceRange ConstexprRange(ConstexprLoc, ConstexprLoc);
+        // If the declarator is complex, we need to move the keyword to the
+        // innermost chunk as we switch it from 'constexpr' to 'const'.
+        int Kind = DeclaratorChunk::Paren;
+        for (unsigned I = 0, E = D.getNumTypeObjects(); I != E; ++I) {
+          Kind = D.getTypeObject(I).Kind;
+          if (Kind != DeclaratorChunk::Paren)
+            break;
+        }
+        if ((D.getDeclSpec().getTypeQualifiers() & DeclSpec::TQ_const) ||
+            Kind == DeclaratorChunk::Reference)
+          Diag(ConstexprLoc, diag::err_invalid_constexpr_var_decl)
+            << FixItHint::CreateRemoval(ConstexprRange);
+        else if (Kind == DeclaratorChunk::Paren)
+          Diag(ConstexprLoc, diag::err_invalid_constexpr_var_decl)
+            << FixItHint::CreateReplacement(ConstexprRange, "const");
+        else
+          Diag(ConstexprLoc, diag::err_invalid_constexpr_var_decl)
+            << FixItHint::CreateRemoval(ConstexprRange)
+            << FixItHint::CreateInsertion(D.getIdentifierLoc(), "const ");
+      } else {
+        NewVD->setConstexpr(true);
+      }
+    }
   }
 
   // Set the lexical context. If the declarator has a C++ scope specifier, the
   // lexical context will be different from the semantic context.
   NewVD->setLexicalDeclContext(CurContext);
 
+  if (D.getDeclSpec().isThreadSpecified()) {
+    if (NewVD->hasLocalStorage())
+      Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_thread_non_global);
+    else if (!Context.getTargetInfo().isTLSSupported())
+      Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_thread_unsupported);
+    else
+      NewVD->setThreadSpecified(true);
+  }
+
+  if (D.getDeclSpec().isModulePrivateSpecified()) {
+    if (isExplicitSpecialization)
+      Diag(NewVD->getLocation(), diag::err_module_private_specialization)
+        << 2
+        << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+    else if (NewVD->hasLocalStorage())
+      Diag(NewVD->getLocation(), diag::err_module_private_local)
+        << 0 << NewVD->getDeclName()
+        << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
+        << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+    else
+      NewVD->setModulePrivate();
+  }
+
   // Handle attributes prior to checking for duplicates in MergeVarDecl
   ProcessDeclAttributes(S, NewVD, D);
 
@@ -3722,7 +3970,7 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   if (Expr *E = (Expr*)D.getAsmLabel()) {
     // The parser guarantees this is a string.
     StringLiteral *SE = cast<StringLiteral>(E);
-    llvm::StringRef Label = SE->getString();
+    StringRef Label = SE->getString();
     if (S->getFnParent() != 0) {
       switch (SC) {
       case SC_None:
@@ -3730,12 +3978,13 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         Diag(E->getExprLoc(), diag::warn_asm_label_on_auto_decl) << Label;
         break;
       case SC_Register:
-        if (!Context.Target.isValidGCCRegisterName(Label))
+        if (!Context.getTargetInfo().isValidGCCRegisterName(Label))
           Diag(E->getExprLoc(), diag::err_asm_unknown_register_name) << Label;
         break;
       case SC_Static:
       case SC_Extern:
       case SC_PrivateExtern:
+      case SC_OpenCLWorkGroupLocal:
         break;
       }
     }
@@ -3754,9 +4003,9 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   FilterLookupForScope(Previous, DC, S, NewVD->hasLinkage(),
                        isExplicitSpecialization);
   
-  if (!getLangOptions().CPlusPlus)
-    CheckVariableDeclaration(NewVD, Previous, Redeclaration);
-  else {
+  if (!getLangOptions().CPlusPlus) {
+    D.setRedeclaration(CheckVariableDeclaration(NewVD, Previous));
+  } else {
     // Merge the decl with the existing one if appropriate.
     if (!Previous.empty()) {
       if (Previous.isSingleResult() &&
@@ -3777,7 +4026,7 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       NewVD->setInvalidDecl();
     }
 
-    CheckVariableDeclaration(NewVD, Previous, Redeclaration);
+    D.setRedeclaration(CheckVariableDeclaration(NewVD, Previous));
 
     // This is an explicit specialization of a static data member. Check it.
     if (isExplicitSpecialization && !NewVD->isInvalidDecl() &&
@@ -3825,7 +4074,7 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 void Sema::CheckShadow(Scope *S, VarDecl *D, const LookupResult& R) {
   // Return if warning is ignored.
   if (Diags.getDiagnosticLevel(diag::warn_decl_shadow, R.getNameLoc()) ==
-        Diagnostic::Ignored)
+        DiagnosticsEngine::Ignored)
     return;
 
   // Don't diagnose declarations at file scope.
@@ -3899,7 +4148,7 @@ void Sema::CheckShadow(Scope *S, VarDecl *D, const LookupResult& R) {
 /// \brief Check -Wshadow without the advantage of a previous lookup.
 void Sema::CheckShadow(Scope *S, VarDecl *D) {
   if (Diags.getDiagnosticLevel(diag::warn_decl_shadow, D->getLocation()) ==
-        Diagnostic::Ignored)
+        DiagnosticsEngine::Ignored)
     return;
 
   LookupResult R(*this, D->getDeclName(), D->getLocation(),
@@ -3918,18 +4167,21 @@ void Sema::CheckShadow(Scope *S, VarDecl *D) {
 /// that have been instantiated from a template.
 ///
 /// Sets NewVD->isInvalidDecl() if an error was encountered.
-void Sema::CheckVariableDeclaration(VarDecl *NewVD,
-                                    LookupResult &Previous,
-                                    bool &Redeclaration) {
+///
+/// Returns true if the variable declaration is a redeclaration.
+bool Sema::CheckVariableDeclaration(VarDecl *NewVD,
+                                    LookupResult &Previous) {
   // If the decl is already known invalid, don't check it.
   if (NewVD->isInvalidDecl())
-    return;
+    return false;
 
   QualType T = NewVD->getType();
 
   if (T->isObjCObjectType()) {
-    Diag(NewVD->getLocation(), diag::err_statically_allocated_object);
-    return NewVD->setInvalidDecl();
+    Diag(NewVD->getLocation(), diag::err_statically_allocated_object)
+      << FixItHint::CreateInsertion(NewVD->getLocation(), "*");
+    T = Context.getObjCObjectPointerType(T);
+    NewVD->setType(T);
   }
 
   // Emit an error if an address space was applied to decl with local storage.
@@ -3938,12 +4190,13 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
   // ISO/IEC TR 18037 S5.1.2
   if (NewVD->hasLocalStorage() && T.getAddressSpace() != 0) {
     Diag(NewVD->getLocation(), diag::err_as_qualified_auto_decl);
-    return NewVD->setInvalidDecl();
+    NewVD->setInvalidDecl();
+    return false;
   }
 
   if (NewVD->hasLocalStorage() && T.isObjCGCWeak()
       && !NewVD->hasAttr<BlocksAttr>()) {
-    if (getLangOptions().getGCMode() != LangOptions::NonGC)
+    if (getLangOptions().getGC() != LangOptions::NonGC)
       Diag(NewVD->getLocation(), diag::warn_gc_attribute_weak_on_local);
     else
       Diag(NewVD->getLocation(), diag::warn_attribute_weak_on_local);
@@ -3977,7 +4230,8 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
       else
         Diag(NewVD->getLocation(), diag::err_vla_decl_has_extern_linkage)
         << SizeRange;
-      return NewVD->setInvalidDecl();
+      NewVD->setInvalidDecl();
+      return false;
     }
 
     if (FixedTy.isNull()) {
@@ -3985,7 +4239,8 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
         Diag(NewVD->getLocation(), diag::err_vm_decl_in_file_scope);
       else
         Diag(NewVD->getLocation(), diag::err_vm_decl_has_extern_linkage);
-      return NewVD->setInvalidDecl();
+      NewVD->setInvalidDecl();
+      return false;
     }
 
     Diag(NewVD->getLocation(), diag::warn_illegal_constant_array_size);
@@ -3997,7 +4252,7 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
     // an extern "C" variable, look for a non-visible extern "C"
     // declaration with the same name.
     llvm::DenseMap<DeclarationName, NamedDecl *>::iterator Pos
-      = LocallyScopedExternalDecls.find(NewVD->getDeclName());
+      = findLocallyScopedExternalDecl(NewVD->getDeclName());
     if (Pos != LocallyScopedExternalDecls.end())
       Previous.addDecl(Pos->second);
   }
@@ -4005,17 +4260,20 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
   if (T->isVoidType() && !NewVD->hasExternalStorage()) {
     Diag(NewVD->getLocation(), diag::err_typecheck_decl_incomplete_type)
       << T;
-    return NewVD->setInvalidDecl();
+    NewVD->setInvalidDecl();
+    return false;
   }
 
   if (!NewVD->hasLocalStorage() && NewVD->hasAttr<BlocksAttr>()) {
     Diag(NewVD->getLocation(), diag::err_block_on_nonlocal);
-    return NewVD->setInvalidDecl();
+    NewVD->setInvalidDecl();
+    return false;
   }
 
   if (isVM && NewVD->hasAttr<BlocksAttr>()) {
     Diag(NewVD->getLocation(), diag::err_block_on_vm);
-    return NewVD->setInvalidDecl();
+    NewVD->setInvalidDecl();
+    return false;
   }
 
   // Function pointers and references cannot have qualified function type, only
@@ -4032,13 +4290,15 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
       Pointee->getAs<FunctionProtoType>()->getTypeQuals() != 0) {
     Diag(NewVD->getLocation(), diag::err_invalid_qualified_function_pointer)
         << PtrOrRef;
-    return NewVD->setInvalidDecl();
+    NewVD->setInvalidDecl();
+    return false;
   }
 
   if (!Previous.empty()) {
-    Redeclaration = true;
     MergeVarDecl(NewVD, Previous);
+    return true;
   }
+  return false;
 }
 
 /// \brief Data used with FindOverriddenMethod
@@ -4108,60 +4368,358 @@ bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
   return AddedAny;
 }
 
-static void DiagnoseInvalidRedeclaration(Sema &S, FunctionDecl *NewFD) {
-  LookupResult Prev(S, NewFD->getDeclName(), NewFD->getLocation(),
-                    Sema::LookupOrdinaryName, Sema::ForRedeclaration);
-  S.LookupQualifiedName(Prev, NewFD->getDeclContext());
-  assert(!Prev.isAmbiguous() &&
-         "Cannot have an ambiguity in previous-declaration lookup");
-  for (LookupResult::iterator Func = Prev.begin(), FuncEnd = Prev.end();
-       Func != FuncEnd; ++Func) {
-    if (isa<FunctionDecl>(*Func) &&
-        isNearlyMatchingFunction(S.Context, cast<FunctionDecl>(*Func), NewFD))
-      S.Diag((*Func)->getLocation(), diag::note_member_def_close_match);
-  }
+namespace {
+  // Struct for holding all of the extra arguments needed by
+  // DiagnoseInvalidRedeclaration to call Sema::ActOnFunctionDeclarator.
+  struct ActOnFDArgs {
+    Scope *S;
+    Declarator &D;
+    MultiTemplateParamsArg TemplateParamLists;
+    bool AddToScope;
+  };
 }
 
-NamedDecl*
-Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
-                              QualType R, TypeSourceInfo *TInfo,
-                              LookupResult &Previous,
-                              MultiTemplateParamsArg TemplateParamLists,
-                              bool IsFunctionDefinition, bool &Redeclaration) {
-  assert(R.getTypePtr()->isFunctionType());
+/// \brief Generate diagnostics for an invalid function redeclaration.
+///
+/// This routine handles generating the diagnostic messages for an invalid
+/// function redeclaration, including finding possible similar declarations
+/// or performing typo correction if there are no previous declarations with
+/// the same name.
+///
+/// Returns a NamedDecl iff typo correction was performed and substituting in
+/// the new declaration name does not cause new errors.
+static NamedDecl* DiagnoseInvalidRedeclaration(
+    Sema &SemaRef, LookupResult &Previous, FunctionDecl *NewFD,
+    ActOnFDArgs &ExtraArgs) {
+  NamedDecl *Result = NULL;
+  DeclarationName Name = NewFD->getDeclName();
+  DeclContext *NewDC = NewFD->getDeclContext();
+  LookupResult Prev(SemaRef, Name, NewFD->getLocation(),
+                    Sema::LookupOrdinaryName, Sema::ForRedeclaration);
+  llvm::SmallVector<unsigned, 1> MismatchedParams;
+  llvm::SmallVector<std::pair<FunctionDecl*, unsigned>, 1> NearMatches;
+  TypoCorrection Correction;
+  bool isFriendDecl = (SemaRef.getLangOptions().CPlusPlus &&
+                       ExtraArgs.D.getDeclSpec().isFriendSpecified());
+  unsigned DiagMsg = isFriendDecl ? diag::err_no_matching_local_friend
+                                  : diag::err_member_def_does_not_match;
 
-  // TODO: consider using NameInfo for diagnostic.
-  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
-  DeclarationName Name = NameInfo.getName();
-  FunctionDecl::StorageClass SC = SC_None;
+  NewFD->setInvalidDecl();
+  SemaRef.LookupQualifiedName(Prev, NewDC);
+  assert(!Prev.isAmbiguous() &&
+         "Cannot have an ambiguity in previous-declaration lookup");
+  if (!Prev.empty()) {
+    for (LookupResult::iterator Func = Prev.begin(), FuncEnd = Prev.end();
+         Func != FuncEnd; ++Func) {
+      FunctionDecl *FD = dyn_cast<FunctionDecl>(*Func);
+      if (FD &&
+          hasSimilarParameters(SemaRef.Context, FD, NewFD, MismatchedParams)) {
+        // Add 1 to the index so that 0 can mean the mismatch didn't
+        // involve a parameter
+        unsigned ParamNum =
+            MismatchedParams.empty() ? 0 : MismatchedParams.front() + 1;
+        NearMatches.push_back(std::make_pair(FD, ParamNum));
+      }
+    }
+  // If the qualified name lookup yielded nothing, try typo correction
+  } else if ((Correction = SemaRef.CorrectTypo(Prev.getLookupNameInfo(),
+                                         Prev.getLookupKind(), 0, 0, NewDC)) &&
+             Correction.getCorrection() != Name) {
+    // Trap errors.
+    Sema::SFINAETrap Trap(SemaRef);
+
+    // Set up everything for the call to ActOnFunctionDeclarator
+    ExtraArgs.D.SetIdentifier(Correction.getCorrectionAsIdentifierInfo(),
+                              ExtraArgs.D.getIdentifierLoc());
+    Previous.clear();
+    Previous.setLookupName(Correction.getCorrection());
+    for (TypoCorrection::decl_iterator CDecl = Correction.begin(),
+                                    CDeclEnd = Correction.end();
+         CDecl != CDeclEnd; ++CDecl) {
+      FunctionDecl *FD = dyn_cast<FunctionDecl>(*CDecl);
+      if (FD && hasSimilarParameters(SemaRef.Context, FD, NewFD,
+                                     MismatchedParams)) {
+        Previous.addDecl(FD);
+      }
+    }
+    bool wasRedeclaration = ExtraArgs.D.isRedeclaration();
+    // TODO: Refactor ActOnFunctionDeclarator so that we can call only the
+    // pieces need to verify the typo-corrected C++ declaraction and hopefully
+    // eliminate the need for the parameter pack ExtraArgs.
+    Result = SemaRef.ActOnFunctionDeclarator(ExtraArgs.S, ExtraArgs.D,
+                                             NewFD->getDeclContext(),
+                                             NewFD->getTypeSourceInfo(),
+                                             Previous,
+                                             ExtraArgs.TemplateParamLists,
+                                             ExtraArgs.AddToScope);
+    if (Trap.hasErrorOccurred()) {
+      // Pretend the typo correction never occurred
+      ExtraArgs.D.SetIdentifier(Name.getAsIdentifierInfo(),
+                                ExtraArgs.D.getIdentifierLoc());
+      ExtraArgs.D.setRedeclaration(wasRedeclaration);
+      Previous.clear();
+      Previous.setLookupName(Name);
+      Result = NULL;
+    } else {
+      for (LookupResult::iterator Func = Previous.begin(),
+                               FuncEnd = Previous.end();
+           Func != FuncEnd; ++Func) {
+        if (FunctionDecl *FD = dyn_cast<FunctionDecl>(*Func))
+          NearMatches.push_back(std::make_pair(FD, 0));
+      }
+    }
+    if (NearMatches.empty()) {
+      // Ignore the correction if it didn't yield any close FunctionDecl matches
+      Correction = TypoCorrection();
+    } else {
+      DiagMsg = isFriendDecl ? diag::err_no_matching_local_friend_suggest
+                             : diag::err_member_def_does_not_match_suggest;
+    }
+  }
+
+  if (Correction)
+    SemaRef.Diag(NewFD->getLocation(), DiagMsg)
+        << Name << NewDC << Correction.getQuoted(SemaRef.getLangOptions())
+        << FixItHint::CreateReplacement(
+            NewFD->getLocation(),
+            Correction.getAsString(SemaRef.getLangOptions()));
+  else
+    SemaRef.Diag(NewFD->getLocation(), DiagMsg)
+        << Name << NewDC << NewFD->getLocation();
+
+  bool NewFDisConst = false;
+  if (CXXMethodDecl *NewMD = dyn_cast<CXXMethodDecl>(NewFD))
+    NewFDisConst = NewMD->getTypeQualifiers() & Qualifiers::Const;
+
+  for (llvm::SmallVector<std::pair<FunctionDecl*, unsigned>, 1>::iterator
+       NearMatch = NearMatches.begin(), NearMatchEnd = NearMatches.end();
+       NearMatch != NearMatchEnd; ++NearMatch) {
+    FunctionDecl *FD = NearMatch->first;
+    bool FDisConst = false;
+    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
+      FDisConst = MD->getTypeQualifiers() & Qualifiers::Const;
+
+    if (unsigned Idx = NearMatch->second) {
+      ParmVarDecl *FDParam = FD->getParamDecl(Idx-1);
+      SemaRef.Diag(FDParam->getTypeSpecStartLoc(),
+             diag::note_member_def_close_param_match)
+          << Idx << FDParam->getType() << NewFD->getParamDecl(Idx-1)->getType();
+    } else if (Correction) {
+      SemaRef.Diag(FD->getLocation(), diag::note_previous_decl)
+          << Correction.getQuoted(SemaRef.getLangOptions());
+    } else if (FDisConst != NewFDisConst) {
+      SemaRef.Diag(FD->getLocation(), diag::note_member_def_close_const_match)
+          << NewFDisConst << FD->getSourceRange().getEnd();
+    } else
+      SemaRef.Diag(FD->getLocation(), diag::note_member_def_close_match);
+  }
+  return Result;
+}
+
+static FunctionDecl::StorageClass getFunctionStorageClass(Sema &SemaRef, Declarator &D) {
   switch (D.getDeclSpec().getStorageClassSpec()) {
-  default: assert(0 && "Unknown storage class!");
+  default: llvm_unreachable("Unknown storage class!");
   case DeclSpec::SCS_auto:
   case DeclSpec::SCS_register:
   case DeclSpec::SCS_mutable:
-    Diag(D.getDeclSpec().getStorageClassSpecLoc(),
-         diag::err_typecheck_sclass_func);
+    SemaRef.Diag(D.getDeclSpec().getStorageClassSpecLoc(),
+                 diag::err_typecheck_sclass_func);
     D.setInvalidType();
     break;
-  case DeclSpec::SCS_unspecified: SC = SC_None; break;
-  case DeclSpec::SCS_extern:      SC = SC_Extern; break;
+  case DeclSpec::SCS_unspecified: break;
+  case DeclSpec::SCS_extern: return SC_Extern;
   case DeclSpec::SCS_static: {
-    if (CurContext->getRedeclContext()->isFunctionOrMethod()) {
+    if (SemaRef.CurContext->getRedeclContext()->isFunctionOrMethod()) {
       // C99 6.7.1p5:
       //   The declaration of an identifier for a function that has
       //   block scope shall have no explicit storage-class specifier
       //   other than extern
       // See also (C++ [dcl.stc]p4).
-      Diag(D.getDeclSpec().getStorageClassSpecLoc(),
-           diag::err_static_block_func);
-      SC = SC_None;
+      SemaRef.Diag(D.getDeclSpec().getStorageClassSpecLoc(),
+                   diag::err_static_block_func);
+      break;
     } else
-      SC = SC_Static;
-    break;
+      return SC_Static;
   }
-  case DeclSpec::SCS_private_extern: SC = SC_PrivateExtern; break;
+  case DeclSpec::SCS_private_extern: return SC_PrivateExtern;
   }
 
+  // No explicit storage class has already been returned
+  return SC_None;
+}
+
+static FunctionDecl* CreateNewFunctionDecl(Sema &SemaRef, Declarator &D,
+                                           DeclContext *DC, QualType &R,
+                                           TypeSourceInfo *TInfo,
+                                           FunctionDecl::StorageClass SC,
+                                           bool &IsVirtualOkay) {
+  DeclarationNameInfo NameInfo = SemaRef.GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
+
+  FunctionDecl *NewFD = 0;
+  bool isInline = D.getDeclSpec().isInlineSpecified();
+  DeclSpec::SCS SCSpec = D.getDeclSpec().getStorageClassSpecAsWritten();
+  FunctionDecl::StorageClass SCAsWritten
+    = StorageClassSpecToFunctionDeclStorageClass(SCSpec);
+
+  if (!SemaRef.getLangOptions().CPlusPlus) {
+    // Determine whether the function was written with a
+    // prototype. This true when:
+    //   - there is a prototype in the declarator, or
+    //   - the type R of the function is some kind of typedef or other reference
+    //     to a type name (which eventually refers to a function type).
+    bool HasPrototype =
+      (D.isFunctionDeclarator() && D.getFunctionTypeInfo().hasPrototype) ||
+      (!isa<FunctionType>(R.getTypePtr()) && R->isFunctionProtoType());
+
+    NewFD = FunctionDecl::Create(SemaRef.Context, DC, D.getSourceRange().getBegin(),
+                                 NameInfo, R, TInfo, SC, SCAsWritten, isInline,
+                                 HasPrototype);
+    if (D.isInvalidType())
+      NewFD->setInvalidDecl();
+
+    // Set the lexical context.
+    NewFD->setLexicalDeclContext(SemaRef.CurContext);
+
+    return NewFD;
+  }
+
+  bool isExplicit = D.getDeclSpec().isExplicitSpecified();
+  bool isConstexpr = D.getDeclSpec().isConstexprSpecified();
+
+  // Check that the return type is not an abstract class type.
+  // For record types, this is done by the AbstractClassUsageDiagnoser once
+  // the class has been completely parsed.
+  if (!DC->isRecord() &&
+      SemaRef.RequireNonAbstractType(D.getIdentifierLoc(),
+                                     R->getAs<FunctionType>()->getResultType(),
+                                     diag::err_abstract_type_in_decl,
+                                     SemaRef.AbstractReturnType))
+    D.setInvalidType();
+
+  if (Name.getNameKind() == DeclarationName::CXXConstructorName) {
+    // This is a C++ constructor declaration.
+    assert(DC->isRecord() &&
+           "Constructors can only be declared in a member context");
+
+    R = SemaRef.CheckConstructorDeclarator(D, R, SC);
+    return CXXConstructorDecl::Create(SemaRef.Context, cast<CXXRecordDecl>(DC),
+                                      D.getSourceRange().getBegin(), NameInfo,
+                                      R, TInfo, isExplicit, isInline,
+                                      /*isImplicitlyDeclared=*/false,
+                                      isConstexpr);
+
+  } else if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
+    // This is a C++ destructor declaration.
+    if (DC->isRecord()) {
+      R = SemaRef.CheckDestructorDeclarator(D, R, SC);
+      CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
+      CXXDestructorDecl *NewDD = CXXDestructorDecl::Create(
+                                        SemaRef.Context, Record,
+                                        D.getSourceRange().getBegin(),
+                                        NameInfo, R, TInfo, isInline,
+                                        /*isImplicitlyDeclared=*/false);
+
+      // If the class is complete, then we now create the implicit exception
+      // specification. If the class is incomplete or dependent, we can't do
+      // it yet.
+      if (SemaRef.getLangOptions().CPlusPlus0x && !Record->isDependentType() &&
+          Record->getDefinition() && !Record->isBeingDefined() &&
+          R->getAs<FunctionProtoType>()->getExceptionSpecType() == EST_None) {
+        SemaRef.AdjustDestructorExceptionSpec(Record, NewDD);
+      }
+
+      IsVirtualOkay = true;
+      return NewDD;
+
+    } else {
+      SemaRef.Diag(D.getIdentifierLoc(), diag::err_destructor_not_member);
+      D.setInvalidType();
+
+      // Create a FunctionDecl to satisfy the function definition parsing
+      // code path.
+      return FunctionDecl::Create(SemaRef.Context, DC,
+                                  D.getSourceRange().getBegin(),
+                                  D.getIdentifierLoc(), Name, R, TInfo,
+                                  SC, SCAsWritten, isInline,
+                                  /*hasPrototype=*/true, isConstexpr);
+    }
+
+  } else if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName) {
+    if (!DC->isRecord()) {
+      SemaRef.Diag(D.getIdentifierLoc(),
+           diag::err_conv_function_not_member);
+      return 0;
+    }
+
+    SemaRef.CheckConversionDeclarator(D, R, SC);
+    IsVirtualOkay = true;
+    return CXXConversionDecl::Create(SemaRef.Context, cast<CXXRecordDecl>(DC),
+                                     D.getSourceRange().getBegin(), NameInfo,
+                                     R, TInfo, isInline, isExplicit,
+                                     isConstexpr, SourceLocation());
+
+  } else if (DC->isRecord()) {
+    // If the name of the function is the same as the name of the record,
+    // then this must be an invalid constructor that has a return type.
+    // (The parser checks for a return type and makes the declarator a
+    // constructor if it has no return type).
+    if (Name.getAsIdentifierInfo() &&
+        Name.getAsIdentifierInfo() == cast<CXXRecordDecl>(DC)->getIdentifier()){
+      SemaRef.Diag(D.getIdentifierLoc(), diag::err_constructor_return_type)
+        << SourceRange(D.getDeclSpec().getTypeSpecTypeLoc())
+        << SourceRange(D.getIdentifierLoc());
+      return 0;
+    }
+
+    bool isStatic = SC == SC_Static;
+
+    // [class.free]p1:
+    // Any allocation function for a class T is a static member
+    // (even if not explicitly declared static).
+    if (Name.getCXXOverloadedOperator() == OO_New ||
+        Name.getCXXOverloadedOperator() == OO_Array_New)
+      isStatic = true;
+
+    // [class.free]p6 Any deallocation function for a class X is a static member
+    // (even if not explicitly declared static).
+    if (Name.getCXXOverloadedOperator() == OO_Delete ||
+        Name.getCXXOverloadedOperator() == OO_Array_Delete)
+      isStatic = true;
+
+    IsVirtualOkay = !isStatic;
+
+    // This is a C++ method declaration.
+    return CXXMethodDecl::Create(SemaRef.Context, cast<CXXRecordDecl>(DC),
+                                 D.getSourceRange().getBegin(), NameInfo, R,
+                                 TInfo, isStatic, SCAsWritten, isInline,
+                                 isConstexpr, SourceLocation());
+
+  } else {
+    // Determine whether the function was written with a
+    // prototype. This true when:
+    //   - we're in C++ (where every function has a prototype),
+    return FunctionDecl::Create(SemaRef.Context, DC,
+                                D.getSourceRange().getBegin(),
+                                NameInfo, R, TInfo, SC, SCAsWritten, isInline,
+                                true/*HasPrototype*/, isConstexpr);
+  }
+}
+
+NamedDecl*
+Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
+                              TypeSourceInfo *TInfo, LookupResult &Previous,
+                              MultiTemplateParamsArg TemplateParamLists,
+                              bool &AddToScope) {
+  QualType R = TInfo->getType();
+
+  assert(R.getTypePtr()->isFunctionType());
+
+  // TODO: consider using NameInfo for diagnostic.
+  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
+  FunctionDecl::StorageClass SC = getFunctionStorageClass(*this, D);
+
   if (D.getDeclSpec().isThreadSpecified())
     Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_invalid_thread);
 
@@ -4169,172 +4727,38 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   if (R->getAs<FunctionType>()->getResultType()->isObjCObjectType()) {
     Diag(D.getIdentifierLoc(),
          diag::err_object_cannot_be_passed_returned_by_value) << 0
-    << R->getAs<FunctionType>()->getResultType();
-    D.setInvalidType();
+    << R->getAs<FunctionType>()->getResultType()
+    << FixItHint::CreateInsertion(D.getIdentifierLoc(), "*");
+
+    QualType T = R->getAs<FunctionType>()->getResultType();
+    T = Context.getObjCObjectPointerType(T);
+    if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(R)) {
+      FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+      R = Context.getFunctionType(T, FPT->arg_type_begin(),
+                                  FPT->getNumArgs(), EPI);
+    }
+    else if (isa<FunctionNoProtoType>(R))
+      R = Context.getFunctionNoProtoType(T);
   }
-  
-  FunctionDecl *NewFD;
-  bool isInline = D.getDeclSpec().isInlineSpecified();
+
   bool isFriend = false;
-  DeclSpec::SCS SCSpec = D.getDeclSpec().getStorageClassSpecAsWritten();
-  FunctionDecl::StorageClass SCAsWritten
-    = StorageClassSpecToFunctionDeclStorageClass(SCSpec);
   FunctionTemplateDecl *FunctionTemplate = 0;
   bool isExplicitSpecialization = false;
   bool isFunctionTemplateSpecialization = false;
+  bool isDependentClassScopeExplicitSpecialization = false;
+  bool isVirtualOkay = false;
 
-  if (!getLangOptions().CPlusPlus) {
-    // Determine whether the function was written with a
-    // prototype. This true when:
-    //   - there is a prototype in the declarator, or
-    //   - the type R of the function is some kind of typedef or other reference
-    //     to a type name (which eventually refers to a function type).
-    bool HasPrototype =
-    (D.isFunctionDeclarator() && D.getFunctionTypeInfo().hasPrototype) ||
-    (!isa<FunctionType>(R.getTypePtr()) && R->isFunctionProtoType());
-  
-    NewFD = FunctionDecl::Create(Context, DC, D.getSourceRange().getBegin(),
-                                 NameInfo, R, TInfo, SC, SCAsWritten, isInline,
-                                 HasPrototype);
-    if (D.isInvalidType())
-      NewFD->setInvalidDecl();
-    
-    // Set the lexical context.
-    NewFD->setLexicalDeclContext(CurContext);
-    // Filter out previous declarations that don't match the scope.
-    FilterLookupForScope(Previous, DC, S, NewFD->hasLinkage(),
-                         /*ExplicitInstantiationOrSpecialization=*/false);
-  } else {
-    isFriend = D.getDeclSpec().isFriendSpecified();
+  FunctionDecl *NewFD = CreateNewFunctionDecl(*this, D, DC, R, TInfo, SC,
+                                              isVirtualOkay);
+  if (!NewFD) return 0;
+
+  if (getLangOptions().CPlusPlus) {
+    bool isInline = D.getDeclSpec().isInlineSpecified();
     bool isVirtual = D.getDeclSpec().isVirtualSpecified();
     bool isExplicit = D.getDeclSpec().isExplicitSpecified();
-    bool isVirtualOkay = false;
-
-    // Check that the return type is not an abstract class type.
-    // For record types, this is done by the AbstractClassUsageDiagnoser once
-    // the class has been completely parsed.
-    if (!DC->isRecord() &&
-      RequireNonAbstractType(D.getIdentifierLoc(),
-                             R->getAs<FunctionType>()->getResultType(),
-                             diag::err_abstract_type_in_decl,
-                             AbstractReturnType))
-      D.setInvalidType();
-
-    if (Name.getNameKind() == DeclarationName::CXXConstructorName) {
-      // This is a C++ constructor declaration.
-      assert(DC->isRecord() &&
-             "Constructors can only be declared in a member context");
-
-      R = CheckConstructorDeclarator(D, R, SC);
-
-      // Create the new declaration
-      CXXConstructorDecl *NewCD = CXXConstructorDecl::Create(
-                                         Context,
-                                         cast<CXXRecordDecl>(DC),
-                                         D.getSourceRange().getBegin(),
-                                         NameInfo, R, TInfo,
-                                         isExplicit, isInline,
-                                         /*isImplicitlyDeclared=*/false);
-
-      NewFD = NewCD;
-    } else if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
-      // This is a C++ destructor declaration.
-      if (DC->isRecord()) {
-        R = CheckDestructorDeclarator(D, R, SC);
-        CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
-
-        CXXDestructorDecl *NewDD = CXXDestructorDecl::Create(Context, Record,
-                                          D.getSourceRange().getBegin(),
-                                          NameInfo, R, TInfo,
-                                          isInline,
-                                          /*isImplicitlyDeclared=*/false);
-        NewFD = NewDD;
-        isVirtualOkay = true;
-
-        // If the class is complete, then we now create the implicit exception
-        // specification. If the class is incomplete or dependent, we can't do
-        // it yet.
-        if (getLangOptions().CPlusPlus0x && !Record->isDependentType() &&
-            Record->getDefinition() && !Record->isBeingDefined() &&
-            R->getAs<FunctionProtoType>()->getExceptionSpecType() == EST_None) {
-          AdjustDestructorExceptionSpec(Record, NewDD);
-        }
-
-      } else {
-        Diag(D.getIdentifierLoc(), diag::err_destructor_not_member);
-
-        // Create a FunctionDecl to satisfy the function definition parsing
-        // code path.
-        NewFD = FunctionDecl::Create(Context, DC, D.getSourceRange().getBegin(),
-                                     D.getIdentifierLoc(), Name, R, TInfo,
-                                     SC, SCAsWritten, isInline,
-                                     /*hasPrototype=*/true);
-        D.setInvalidType();
-      }
-    } else if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName) {
-      if (!DC->isRecord()) {
-        Diag(D.getIdentifierLoc(),
-             diag::err_conv_function_not_member);
-        return 0;
-      }
-
-      CheckConversionDeclarator(D, R, SC);
-      NewFD = CXXConversionDecl::Create(Context, cast<CXXRecordDecl>(DC),
-                                        D.getSourceRange().getBegin(),
-                                        NameInfo, R, TInfo,
-                                        isInline, isExplicit,
-                                        SourceLocation());
-
-      isVirtualOkay = true;
-    } else if (DC->isRecord()) {
-      // If the of the function is the same as the name of the record, then this
-      // must be an invalid constructor that has a return type.
-      // (The parser checks for a return type and makes the declarator a
-      // constructor if it has no return type).
-      // must have an invalid constructor that has a return type
-      if (Name.getAsIdentifierInfo() &&
-          Name.getAsIdentifierInfo() == cast<CXXRecordDecl>(DC)->getIdentifier()){
-        Diag(D.getIdentifierLoc(), diag::err_constructor_return_type)
-          << SourceRange(D.getDeclSpec().getTypeSpecTypeLoc())
-          << SourceRange(D.getIdentifierLoc());
-        return 0;
-      }
-
-      bool isStatic = SC == SC_Static;
-
-      // [class.free]p1:
-      // Any allocation function for a class T is a static member
-      // (even if not explicitly declared static).
-      if (Name.getCXXOverloadedOperator() == OO_New ||
-          Name.getCXXOverloadedOperator() == OO_Array_New)
-        isStatic = true;
-
-      // [class.free]p6 Any deallocation function for a class X is a static member
-      // (even if not explicitly declared static).
-      if (Name.getCXXOverloadedOperator() == OO_Delete ||
-          Name.getCXXOverloadedOperator() == OO_Array_Delete)
-        isStatic = true;
-
-      // This is a C++ method declaration.
-      CXXMethodDecl *NewMD = CXXMethodDecl::Create(
-                                               Context, cast<CXXRecordDecl>(DC),
-                                               D.getSourceRange().getBegin(),
-                                               NameInfo, R, TInfo,
-                                               isStatic, SCAsWritten, isInline,
-                                               SourceLocation());
-      NewFD = NewMD;
-
-      isVirtualOkay = !isStatic;
-    } else {
-      // Determine whether the function was written with a
-      // prototype. This true when:
-      //   - we're in C++ (where every function has a prototype),
-      NewFD = FunctionDecl::Create(Context, DC, D.getSourceRange().getBegin(),
-                                   NameInfo, R, TInfo, SC, SCAsWritten, isInline,
-                                   true/*HasPrototype*/);
-    }
-
-    if (isFriend && !isInline && IsFunctionDefinition) {
+    bool isConstexpr = D.getDeclSpec().isConstexprSpecified();
+    isFriend = D.getDeclSpec().isFriendSpecified();
+    if (isFriend && !isInline && D.isFunctionDefinition()) {
       // C++ [class.friend]p5
       //   A function can be defined in a friend declaration of a
       //   class . . . . Such a function is implicitly inline.
@@ -4377,6 +4801,16 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
           Diag(NewFD->getLocation(), diag::err_destructor_template);
           return 0;
         }
+        
+        // If we're adding a template to a dependent context, we may need to 
+        // rebuilding some of the types used within the template parameter list, 
+        // now that we know what the current instantiation is.
+        if (DC->isDependentContext()) {
+          ContextRAII SavedContext(*this, DC);
+          if (RebuildTemplateParamsInCurrentInstantiation(TemplateParams))
+            Invalid = true;
+        }
+        
 
         FunctionTemplate = FunctionTemplateDecl::Create(Context, DC,
                                                         NewFD->getLocation(),
@@ -4437,7 +4871,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       if (FunctionTemplate)
         FunctionTemplate->setInvalidDecl();
     }
-  
+
     // C++ [dcl.fct.spec]p5:
     //   The virtual specifier shall only be used in declarations of
     //   nonstatic class member functions that appear within a
@@ -4474,7 +4908,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
           << FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
       }
     }
- 
+
     // C++ [dcl.fct.spec]p6:
     //  The explicit specifier shall be used only in the declaration of a
     //  constructor or conversion function within its class definition; see 12.3.1
@@ -4495,10 +4929,32 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       }      
     }
 
-    // Filter out previous declarations that don't match the scope.
-    FilterLookupForScope(Previous, DC, S, NewFD->hasLinkage(),
-                         isExplicitSpecialization || 
-                         isFunctionTemplateSpecialization);
+    if (isConstexpr) {
+      // C++0x [dcl.constexpr]p2: constexpr functions and constexpr constructors
+      // are implicitly inline.
+      NewFD->setImplicitlyInline();
+
+      // C++0x [dcl.constexpr]p3: functions declared constexpr are required to
+      // be either constructors or to return a literal type. Therefore,
+      // destructors cannot be declared constexpr.
+      if (isa<CXXDestructorDecl>(NewFD))
+        Diag(D.getDeclSpec().getConstexprSpecLoc(), diag::err_constexpr_dtor);
+    }
+
+    // If __module_private__ was specified, mark the function accordingly.
+    if (D.getDeclSpec().isModulePrivateSpecified()) {
+      if (isFunctionTemplateSpecialization) {
+        SourceLocation ModulePrivateLoc
+          = D.getDeclSpec().getModulePrivateSpecLoc();
+        Diag(ModulePrivateLoc, diag::err_module_private_specialization)
+          << 0
+          << FixItHint::CreateRemoval(ModulePrivateLoc);
+      } else {
+        NewFD->setModulePrivate();
+        if (FunctionTemplate)
+          FunctionTemplate->setModulePrivate();
+      }
+    }
 
     if (isFriend) {
       // For now, claim that the objects have no previous declaration.
@@ -4510,7 +4966,8 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       NewFD->setAccess(AS_public);
     }
 
-    if (isa<CXXMethodDecl>(NewFD) && DC == CurContext && IsFunctionDefinition) {
+    if (isa<CXXMethodDecl>(NewFD) && DC == CurContext &&
+        D.isFunctionDefinition()) {
       // A method is implicitly inline if it's defined in its class
       // definition.
       NewFD->setImplicitlyInline();
@@ -4530,6 +4987,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
     }
   }
+
+  // Filter out previous declarations that don't match the scope.
+  FilterLookupForScope(Previous, DC, S, NewFD->hasLinkage(),
+                       isExplicitSpecialization ||
+                       isFunctionTemplateSpecialization);
   
   // Handle GNU asm-label extension (encoded as an attribute).
   if (Expr *E = (Expr*) D.getAsmLabel()) {
@@ -4541,7 +5003,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
   // Copy the parameter declarations from the declarator D to the function
   // declaration NewFD, if they are available.  First scavenge them into Params.
-  llvm::SmallVector<ParmVarDecl*, 16> Params;
+  SmallVector<ParmVarDecl*, 16> Params;
   if (D.isFunctionDeclarator()) {
     DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
 
@@ -4603,8 +5065,9 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     assert(R->isFunctionNoProtoType() && NewFD->getNumParams() == 0 &&
            "Should not need args for typedef of non-prototype fn");
   }
+
   // Finally, we know we have the right number of parameters, install them.
-  NewFD->setParams(Params.data(), Params.size());
+  NewFD->setParams(Params);
 
   // Process the non-inheritable attributes on this declaration.
   ProcessDeclAttributes(S, NewFD, D,
@@ -4613,9 +5076,20 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   if (!getLangOptions().CPlusPlus) {
     // Perform semantic checking on the function declaration.
     bool isExplicitSpecialization=false;
-    CheckFunctionDeclaration(S, NewFD, Previous, isExplicitSpecialization,
-                             Redeclaration);
-    assert((NewFD->isInvalidDecl() || !Redeclaration ||
+    if (!NewFD->isInvalidDecl()) {
+      if (NewFD->getResultType()->isVariablyModifiedType()) {
+        // Functions returning a variably modified type violate C99 6.7.5.2p2
+        // because all functions have linkage.
+        Diag(NewFD->getLocation(), diag::err_vm_func_decl);
+        NewFD->setInvalidDecl();
+      } else {
+        if (NewFD->isMain()) 
+          CheckMain(NewFD, D.getDeclSpec());
+        D.setRedeclaration(CheckFunctionDeclaration(S, NewFD, Previous,
+                                                    isExplicitSpecialization));
+      }
+    }
+    assert((NewFD->isInvalidDecl() || !D.isRedeclaration() ||
             Previous.getResultKind() != LookupResult::FoundOverloaded) &&
            "previous declaration set still overloaded");
   } else {
@@ -4676,8 +5150,12 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     // that either the specialized function type or the specialized
     // template is dependent, and therefore matching will fail.  In
     // this case, don't check the specialization yet.
+    bool InstantiationDependent = false;
     if (isFunctionTemplateSpecialization && isFriend &&
-        (NewFD->getType()->isDependentType() || DC->isDependentContext())) {
+        (NewFD->getType()->isDependentType() || DC->isDependentContext() ||
+         TemplateSpecializationType::anyDependentTemplateArguments(
+            TemplateArgs.getArgumentArray(), TemplateArgs.size(),
+            InstantiationDependent))) {
       assert(HasExplicitTemplateArgs &&
              "friend function specialization without template args");
       if (CheckDependentFunctionTemplateSpecialization(NewFD, TemplateArgs,
@@ -4686,10 +5164,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     } else if (isFunctionTemplateSpecialization) {
       if (CurContext->isDependentContext() && CurContext->isRecord() 
           && !isFriend) {
-        Diag(NewFD->getLocation(), diag::err_function_specialization_in_class)
+        isDependentClassScopeExplicitSpecialization = true;
+        Diag(NewFD->getLocation(), getLangOptions().MicrosoftExt ? 
+          diag::ext_function_specialization_in_class :
+          diag::err_function_specialization_in_class)
           << NewFD->getDeclName();
-        NewFD->setInvalidDecl();
-        return 0;
       } else if (CheckFunctionTemplateSpecialization(NewFD,
                                   (HasExplicitTemplateArgs ? &TemplateArgs : 0),
                                                      Previous))
@@ -4719,18 +5198,33 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     }
 
     // Perform semantic checking on the function declaration.
-    CheckFunctionDeclaration(S, NewFD, Previous, isExplicitSpecialization,
-                             Redeclaration);
+    if (!isDependentClassScopeExplicitSpecialization) {
+      if (NewFD->isInvalidDecl()) {
+        // If this is a class member, mark the class invalid immediately.
+        // This avoids some consistency errors later.
+        if (CXXMethodDecl* methodDecl = dyn_cast<CXXMethodDecl>(NewFD))
+          methodDecl->getParent()->setInvalidDecl();
+      } else {
+        if (NewFD->isMain()) 
+          CheckMain(NewFD, D.getDeclSpec());
+        D.setRedeclaration(CheckFunctionDeclaration(S, NewFD, Previous,
+                                                    isExplicitSpecialization));
+      }
+    }
 
-    assert((NewFD->isInvalidDecl() || !Redeclaration ||
+    assert((NewFD->isInvalidDecl() || !D.isRedeclaration() ||
             Previous.getResultKind() != LookupResult::FoundOverloaded) &&
            "previous declaration set still overloaded");
 
+    if (NewFD->isConstexpr() && !NewFD->isInvalidDecl() &&
+        !CheckConstexprFunctionDecl(NewFD, CCK_Declaration))
+      NewFD->setInvalidDecl();
+
     NamedDecl *PrincipalDecl = (FunctionTemplate
                                 ? cast<NamedDecl>(FunctionTemplate)
                                 : NewFD);
 
-    if (isFriend && Redeclaration) {
+    if (isFriend && D.isRedeclaration()) {
       AccessSpecifier Access = AS_public;
       if (!NewFD->isInvalidDecl())
         Access = NewFD->getPreviousDeclaration()->getAccess();
@@ -4752,7 +5246,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       CheckTemplateParameterList(FunctionTemplate->getTemplateParameters(),
                                  PrevTemplate? PrevTemplate->getTemplateParameters() : 0,
                             D.getDeclSpec().isFriendSpecified()
-                              ? (IsFunctionDefinition 
+                              ? (D.isFunctionDefinition()
                                    ? TPC_FriendFunctionTemplateDefinition
                                    : TPC_FriendFunctionTemplate)
                               : (D.getCXXScopeSpec().isSet() && 
@@ -4764,7 +5258,9 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
     if (NewFD->isInvalidDecl()) {
       // Ignore all the rest of this.
-    } else if (!Redeclaration) {
+    } else if (!D.isRedeclaration()) {
+      struct ActOnFDArgs ExtraArgs = { S, D, TemplateParamLists,
+                                       AddToScope };
       // Fake up an access specifier if it's supposed to be a class member.
       if (isa<CXXRecordDecl>(NewFD->getDeclContext()))
         NewFD->setAccess(AS_public);
@@ -4784,38 +5280,43 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
             (TemplateParamLists.size() ||
              D.getCXXScopeSpec().getScopeRep()->isDependent() ||
              CurContext->isDependentContext())) {
-              // ignore these
-            } else {
-              // The user tried to provide an out-of-line definition for a
-              // function that is a member of a class or namespace, but there
-              // was no such member function declared (C++ [class.mfct]p2,
-              // C++ [namespace.memdef]p2). For example:
-              //
-              // class X {
-              //   void f() const;
-              // };
-              //
-              // void X::f() { } // ill-formed
-              //
-              // Complain about this problem, and attempt to suggest close
-              // matches (e.g., those that differ only in cv-qualifiers and
-              // whether the parameter types are references).
-              Diag(D.getIdentifierLoc(), diag::err_member_def_does_not_match)
-              << Name << DC << D.getCXXScopeSpec().getRange();
-              NewFD->setInvalidDecl();
+          // ignore these
+        } else {
+          // The user tried to provide an out-of-line definition for a
+          // function that is a member of a class or namespace, but there
+          // was no such member function declared (C++ [class.mfct]p2,
+          // C++ [namespace.memdef]p2). For example:
+          //
+          // class X {
+          //   void f() const;
+          // };
+          //
+          // void X::f() { } // ill-formed
+          //
+          // Complain about this problem, and attempt to suggest close
+          // matches (e.g., those that differ only in cv-qualifiers and
+          // whether the parameter types are references).
 
-              DiagnoseInvalidRedeclaration(*this, NewFD);
-            }
+          if (NamedDecl *Result = DiagnoseInvalidRedeclaration(*this, Previous,
+                                                               NewFD,
+                                                               ExtraArgs)) {
+            AddToScope = ExtraArgs.AddToScope;
+            return Result;
+          }
+        }
 
         // Unqualified local friend declarations are required to resolve
         // to something.
-        } else if (isFriend && cast<CXXRecordDecl>(CurContext)->isLocalClass()) {
-          Diag(D.getIdentifierLoc(), diag::err_no_matching_local_friend);
-          NewFD->setInvalidDecl();
-          DiagnoseInvalidRedeclaration(*this, NewFD);
+      } else if (isFriend && cast<CXXRecordDecl>(CurContext)->isLocalClass()) {
+        if (NamedDecl *Result = DiagnoseInvalidRedeclaration(*this, Previous,
+                                                             NewFD,
+                                                             ExtraArgs)) {
+          AddToScope = ExtraArgs.AddToScope;
+          return Result;
         }
+      }
 
-    } else if (!IsFunctionDefinition && D.getCXXScopeSpec().isSet() &&
+    } else if (!D.isFunctionDefinition() && D.getCXXScopeSpec().isSet() &&
                !isFriend && !isFunctionTemplateSpecialization &&
                !isExplicitSpecialization) {
       // An out-of-line member function declaration must also be a
@@ -4839,7 +5340,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
   // attributes declared post-definition are currently ignored
   // FIXME: This should happen during attribute merging
-  if (Redeclaration && Previous.isSingleResult()) {
+  if (D.isRedeclaration() && Previous.isSingleResult()) {
     const FunctionDecl *Def;
     FunctionDecl *PrevFD = dyn_cast<FunctionDecl>(Previous.getFoundDecl());
     if (PrevFD && PrevFD->isDefined(Def) && D.hasAttributes()) {
@@ -4871,6 +5372,10 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   if (NewFD->getLinkage() == ExternalLinkage && !DC->isRecord())
     AddPushedVisibilityAttribute(NewFD);
 
+  // If there's a #pragma clang arc_cf_code_audited in scope, consider
+  // marking the function.
+  AddCFAuditedAttribute(NewFD);
+
   // If this is a locally-scoped extern C function, update the
   // map of such names.
   if (CurContext->isFunctionOrMethod() && NewFD->isExternC()
@@ -4901,6 +5406,18 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
           Context.setcudaConfigureCallDecl(NewFD);
         }
       }
+  
+  // Here we have an function template explicit specialization at class scope.
+  // The actually specialization will be postponed to template instatiation
+  // time via the ClassScopeFunctionSpecializationDecl node.
+  if (isDependentClassScopeExplicitSpecialization) {
+    ClassScopeFunctionSpecializationDecl *NewSpec =
+                         ClassScopeFunctionSpecializationDecl::Create(
+                                Context, CurContext,  SourceLocation(), 
+                                cast<CXXMethodDecl>(NewFD));
+    CurContext->addDecl(NewSpec);
+    AddToScope = false;
+  }
 
   return NewFD;
 }
@@ -4919,29 +5436,13 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 /// an explicit specialization of the previous declaration.
 ///
 /// This sets NewFD->isInvalidDecl() to true if there was an error.
-void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
+///
+/// Returns true if the function declaration is a redeclaration.
+bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
                                     LookupResult &Previous,
-                                    bool IsExplicitSpecialization,
-                                    bool &Redeclaration) {
-  // If NewFD is already known erroneous, don't do any of this checking.
-  if (NewFD->isInvalidDecl()) {
-    // If this is a class member, mark the class invalid immediately.
-    // This avoids some consistency errors later.
-    if (isa<CXXMethodDecl>(NewFD))
-      cast<CXXMethodDecl>(NewFD)->getParent()->setInvalidDecl();
-
-    return;
-  }
-
-  if (NewFD->getResultType()->isVariablyModifiedType()) {
-    // Functions returning a variably modified type violate C99 6.7.5.2p2
-    // because all functions have linkage.
-    Diag(NewFD->getLocation(), diag::err_vm_func_decl);
-    return NewFD->setInvalidDecl();
-  }
-
-  if (NewFD->isMain()) 
-    CheckMain(NewFD);
+                                    bool IsExplicitSpecialization) {
+  assert(!NewFD->getResultType()->isVariablyModifiedType() 
+         && "Variably modified return types are not handled here");
 
   // Check for a previous declaration of this name.
   if (Previous.empty() && NewFD->isExternC()) {
@@ -4949,11 +5450,13 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     // an extern "C" function, look for a non-visible extern "C"
     // declaration with the same name.
     llvm::DenseMap<DeclarationName, NamedDecl *>::iterator Pos
-      = LocallyScopedExternalDecls.find(NewFD->getDeclName());
+      = findLocallyScopedExternalDecl(NewFD->getDeclName());
     if (Pos != LocallyScopedExternalDecls.end())
       Previous.addDecl(Pos->second);
   }
 
+  bool Redeclaration = false;
+
   // Merge or overload the declaration with an existing declaration of
   // the same name, if appropriate.
   if (!Previous.empty()) {
@@ -5003,8 +5506,10 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     if (Redeclaration) {
       // NewFD and OldDecl represent declarations that need to be
       // merged.
-      if (MergeFunctionDecl(NewFD, OldDecl))
-        return NewFD->setInvalidDecl();
+      if (MergeFunctionDecl(NewFD, OldDecl)) {
+        NewFD->setInvalidDecl();
+        return Redeclaration;
+      }
 
       Previous.clear();
       Previous.addDecl(OldDecl);
@@ -5028,6 +5533,10 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
           NewTemplateDecl->setMemberSpecialization();
           assert(OldTemplateDecl->isMemberSpecialization());
         }
+        
+        if (OldTemplateDecl->isModulePrivate())
+          NewTemplateDecl->setModulePrivate();
+        
       } else {
         if (isa<CXXMethodDecl>(NewFD)) // Set access for out-of-line definitions
           NewFD->setAccess(OldDecl->getAccess());
@@ -5054,7 +5563,8 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
                                         Context.getCanonicalType(ClassType));
         if (NewFD->getDeclName() != Name) {
           Diag(NewFD->getLocation(), diag::err_destructor_name);
-          return NewFD->setInvalidDecl();
+          NewFD->setInvalidDecl();
+          return Redeclaration;
         }
       }
     } else if (CXXConversionDecl *Conversion
@@ -5086,13 +5596,17 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
 
     // Extra checking for C++ overloaded operators (C++ [over.oper]).
     if (NewFD->isOverloadedOperator() &&
-        CheckOverloadedOperatorDeclaration(NewFD))
-      return NewFD->setInvalidDecl();
+        CheckOverloadedOperatorDeclaration(NewFD)) {
+      NewFD->setInvalidDecl();
+      return Redeclaration;
+    }
 
     // Extra checking for C++0x literal operators (C++0x [over.literal]).
     if (NewFD->getLiteralIdentifier() &&
-        CheckLiteralOperatorDeclaration(NewFD))
-      return NewFD->setInvalidDecl();
+        CheckLiteralOperatorDeclaration(NewFD)) {
+      NewFD->setInvalidDecl();
+      return Redeclaration;
+    }
 
     // In C++, check default arguments now that we have merged decls. Unless
     // the lexical context is the class, because in this case this is done
@@ -5112,24 +5626,22 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
       }
     }
   }
+  return Redeclaration;
 }
 
-void Sema::CheckMain(FunctionDecl* FD) {
+void Sema::CheckMain(FunctionDecl* FD, const DeclSpec& DS) {
   // C++ [basic.start.main]p3:  A program that declares main to be inline
   //   or static is ill-formed.
   // C99 6.7.4p4:  In a hosted environment, the inline function specifier
   //   shall not appear in a declaration of main.
   // static main is not an error under C99, but we should warn about it.
-  bool isInline = FD->isInlineSpecified();
-  bool isStatic = FD->getStorageClass() == SC_Static;
-  if (isInline || isStatic) {
-    unsigned diagID = diag::warn_unusual_main_decl;
-    if (isInline || getLangOptions().CPlusPlus)
-      diagID = diag::err_unusual_main_decl;
-
-    int which = isStatic + (isInline << 1) - 1;
-    Diag(FD->getLocation(), diagID) << which;
-  }
+  if (FD->getStorageClass() == SC_Static)
+    Diag(DS.getStorageClassSpecLoc(), getLangOptions().CPlusPlus 
+         ? diag::err_static_main : diag::warn_static_main) 
+      << FixItHint::CreateRemoval(DS.getStorageClassSpecLoc());
+  if (FD->isInlineSpecified())
+    Diag(DS.getInlineSpecLoc(), diag::err_inline_main) 
+      << FixItHint::CreateRemoval(DS.getInlineSpecLoc());
 
   QualType T = FD->getType();
   assert(T->isFunctionType() && "function decl is not of function type");
@@ -5152,7 +5664,7 @@ void Sema::CheckMain(FunctionDecl* FD) {
   // Darwin passes an undocumented fourth argument of type char**.  If
   // other platforms start sprouting these, the logic below will start
   // getting shifty.
-  if (nparams == 4 && Context.Target.getTriple().isOSDarwin())
+  if (nparams == 4 && Context.getTargetInfo().getTriple().isOSDarwin())
     HasExtraParameters = false;
 
   if (HasExtraParameters) {
@@ -5231,41 +5743,89 @@ namespace {
       : public EvaluatedExprVisitor<SelfReferenceChecker> {
     Sema &S;
     Decl *OrigDecl;
+    bool isRecordType;
+    bool isPODType;
 
   public:
     typedef EvaluatedExprVisitor<SelfReferenceChecker> Inherited;
 
     SelfReferenceChecker(Sema &S, Decl *OrigDecl) : Inherited(S.Context),
-                                                    S(S), OrigDecl(OrigDecl) { }
+                                                    S(S), OrigDecl(OrigDecl) {
+      isPODType = false;
+      isRecordType = false;
+      if (ValueDecl *VD = dyn_cast<ValueDecl>(OrigDecl)) {
+        isPODType = VD->getType().isPODType(S.Context);
+        isRecordType = VD->getType()->isRecordType();
+      }
+    }
 
     void VisitExpr(Expr *E) {
       if (isa<ObjCMessageExpr>(*E)) return;
+      if (isRecordType) {
+        Expr *expr = E;
+        if (MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
+          ValueDecl *VD = ME->getMemberDecl();
+          if (isa<EnumConstantDecl>(VD) || isa<VarDecl>(VD)) return;
+          expr = ME->getBase();
+        }
+        if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(expr)) {
+          HandleDeclRefExpr(DRE);
+          return;
+        }
+      }
       Inherited::VisitExpr(E);
     }
 
+    void VisitMemberExpr(MemberExpr *E) {
+      if (E->getType()->canDecayToPointerType()) return;
+      if (isa<FieldDecl>(E->getMemberDecl()))
+        if (DeclRefExpr *DRE
+              = dyn_cast<DeclRefExpr>(E->getBase()->IgnoreParenImpCasts())) {
+          HandleDeclRefExpr(DRE);
+          return;
+        }
+      Inherited::VisitMemberExpr(E);
+    }
+
     void VisitImplicitCastExpr(ImplicitCastExpr *E) {
-      CheckForSelfReference(E);
+      if ((!isRecordType &&E->getCastKind() == CK_LValueToRValue) ||
+          (isRecordType && E->getCastKind() == CK_NoOp)) {
+        Expr* SubExpr = E->getSubExpr()->IgnoreParenImpCasts();
+        if (MemberExpr *ME = dyn_cast<MemberExpr>(SubExpr))
+          SubExpr = ME->getBase()->IgnoreParenImpCasts();
+        if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(SubExpr)) {
+          HandleDeclRefExpr(DRE);
+          return;
+        }
+      }
       Inherited::VisitImplicitCastExpr(E);
     }
 
-    void CheckForSelfReference(ImplicitCastExpr *E) {
-      if (E->getCastKind() != CK_LValueToRValue) return;
-      Expr* SubExpr = E->getSubExpr()->IgnoreParenImpCasts();
-      DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(SubExpr);
-      if (!DRE) return;
-      Decl* ReferenceDecl = DRE->getDecl();
+    void VisitUnaryOperator(UnaryOperator *E) {
+      // For POD record types, addresses of its own members are well-defined.
+      if (isRecordType && isPODType) return;
+      Inherited::VisitUnaryOperator(E);
+    } 
+    
+    void HandleDeclRefExpr(DeclRefExpr *DRE) {
+      Decl* ReferenceDecl = DRE->getDecl(); 
       if (OrigDecl != ReferenceDecl) return;
       LookupResult Result(S, DRE->getNameInfo(), Sema::LookupOrdinaryName,
                           Sema::NotForRedeclaration);
-      S.DiagRuntimeBehavior(SubExpr->getLocStart(), SubExpr,
+      S.DiagRuntimeBehavior(DRE->getLocStart(), DRE,
                             S.PDiag(diag::warn_uninit_self_reference_in_init)
-                              << Result.getLookupName() 
+                              << Result.getLookupName()
                               << OrigDecl->getLocation()
-                              << SubExpr->getSourceRange());
+                              << DRE->getSourceRange());
     }
   };
 }
 
+/// CheckSelfReference - Warns if OrigDecl is used in expression E.
+void Sema::CheckSelfReference(Decl* OrigDecl, Expr *E) {
+  SelfReferenceChecker(*this, OrigDecl).VisitExpr(E);
+}
+
 /// AddInitializerToDecl - Adds the initializer Init to the
 /// declaration dcl. If DirectInit is true, this is C++ direct
 /// initialization rather than copy initialization.
@@ -5281,10 +5841,10 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     // Variables declared within a function/method body are handled
     // by a dataflow analysis.
     if (!vd->hasLocalStorage() && !vd->isStaticLocal())
-      SelfReferenceChecker(*this, RealDecl).VisitExpr(Init);    
+      CheckSelfReference(RealDecl, Init);    
   }
   else {
-    SelfReferenceChecker(*this, RealDecl).VisitExpr(Init);
+    CheckSelfReference(RealDecl, Init);
   }
 
   if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(RealDecl)) {
@@ -5392,6 +5952,14 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     }
   }
 
+  // OpenCL 1.1 6.5.2: "Variables allocated in the __local address space inside
+  // a kernel function cannot be initialized."
+  if (VDecl->getStorageClass() == SC_OpenCLWorkGroupLocal) {
+    Diag(VDecl->getLocation(), diag::err_local_cant_init);
+    VDecl->setInvalidDecl();
+    return;
+  }
+
   // Capture the variable that is being initialized and the style of
   // initialization.
   InitializedEntity Entity = InitializedEntity::InitializeVariable(VDecl);
@@ -5456,11 +6024,26 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     //   A member-declarator can contain a constant-initializer only
     //   if it declares a static member (9.4) of const integral or
     //   const enumeration type, see 9.4.2.
+    //
+    // C++0x [class.static.data]p3:
+    //   If a non-volatile const static data member is of integral or
+    //   enumeration type, its declaration in the class definition can
+    //   specify a brace-or-equal-initializer in which every initalizer-clause
+    //   that is an assignment-expression is a constant expression. A static
+    //   data member of literal type can be declared in the class definition
+    //   with the constexpr specifier; if so, its declaration shall specify a
+    //   brace-or-equal-initializer in which every initializer-clause that is
+    //   an assignment-expression is a constant expression.
     QualType T = VDecl->getType();
 
     // Do nothing on dependent types.
     if (T->isDependentType()) {
 
+    // Allow any 'static constexpr' members, whether or not they are of literal
+    // type. We separately check that the initializer is a constant expression,
+    // which implicitly requires the member to be of literal type.
+    } else if (VDecl->isConstexpr()) {
+
     // Require constness.
     } else if (!T.isConstQualified()) {
       Diag(VDecl->getLocation(), diag::err_in_class_initializer_non_const)
@@ -5471,7 +6054,11 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     } else if (T->isIntegralOrEnumerationType()) {
       // Check whether the expression is a constant expression.
       SourceLocation Loc;
-      if (Init->isValueDependent())
+      if (getLangOptions().CPlusPlus0x && T.isVolatileQualified())
+        // In C++0x, a non-constexpr const static data member with an
+        // in-class initializer cannot be volatile.
+        Diag(VDecl->getLocation(), diag::err_in_class_initializer_volatile);
+      else if (Init->isValueDependent())
         ; // Nothing to check.
       else if (Init->isIntegerConstantExpr(Context, &Loc))
         ; // Ok, it's an ICE!
@@ -5488,31 +6075,33 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
         VDecl->setInvalidDecl();
       }
 
-    // We allow floating-point constants as an extension in C++03, and
-    // C++0x has far more complicated rules that we don't really
-    // implement fully.
-    } else {
-      bool Allowed = false;
-      if (getLangOptions().CPlusPlus0x) {
-        Allowed = T->isLiteralType();
-      } else if (T->isFloatingType()) { // also permits complex, which is ok
-        Diag(VDecl->getLocation(), diag::ext_in_class_initializer_float_type)
-          << T << Init->getSourceRange();
-        Allowed = true;
-      }
+    // We allow floating-point constants as an extension.
+    } else if (T->isFloatingType()) { // also permits complex, which is ok
+      Diag(VDecl->getLocation(), diag::ext_in_class_initializer_float_type)
+        << T << Init->getSourceRange();
+      if (getLangOptions().CPlusPlus0x)
+        Diag(VDecl->getLocation(),
+             diag::note_in_class_initializer_float_type_constexpr)
+          << FixItHint::CreateInsertion(VDecl->getLocStart(), "constexpr ");
 
-      if (!Allowed) {
-        Diag(VDecl->getLocation(), diag::err_in_class_initializer_bad_type)
-          << T << Init->getSourceRange();
-        VDecl->setInvalidDecl();
-
-      // TODO: there are probably expressions that pass here that shouldn't.
-      } else if (!Init->isValueDependent() &&
-                 !Init->isConstantInitializer(Context, false)) {
+      if (!Init->isValueDependent() &&
+          !Init->isConstantInitializer(Context, false)) {
         Diag(Init->getExprLoc(), diag::err_in_class_initializer_non_constant)
           << Init->getSourceRange();
         VDecl->setInvalidDecl();
       }
+
+    // Suggest adding 'constexpr' in C++0x for literal types.
+    } else if (getLangOptions().CPlusPlus0x && T->isLiteralType()) {
+      Diag(VDecl->getLocation(), diag::err_in_class_initializer_literal_type)
+        << T << Init->getSourceRange()
+        << FixItHint::CreateInsertion(VDecl->getLocStart(), "constexpr ");
+      VDecl->setConstexpr(true);
+
+    } else {
+      Diag(VDecl->getLocation(), diag::err_in_class_initializer_bad_type)
+        << T << Init->getSourceRange();
+      VDecl->setInvalidDecl();
     }
   } else if (VDecl->isFileVarDecl()) {
     if (VDecl->getStorageClassAsWritten() == SC_Extern && 
@@ -5547,33 +6136,23 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     VDecl->setType(DclT);
     Init->setType(DclT);
   }
-
-  
-  // If this variable is a local declaration with record type, make sure it
-  // doesn't have a flexible member initialization.  We only support this as a
-  // global/static definition.
-  if (VDecl->hasLocalStorage())
-    if (const RecordType *RT = VDecl->getType()->getAs<RecordType>())
-      if (RT->getDecl()->hasFlexibleArrayMember()) {
-        // Check whether the initializer tries to initialize the flexible
-        // array member itself to anything other than an empty initializer list.
-        if (InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) {
-          unsigned Index = std::distance(RT->getDecl()->field_begin(),
-                                         RT->getDecl()->field_end()) - 1;
-          if (Index < ILE->getNumInits() &&
-              !(isa<InitListExpr>(ILE->getInit(Index)) &&
-                cast<InitListExpr>(ILE->getInit(Index))->getNumInits() == 0)) {
-            Diag(VDecl->getLocation(), diag::err_nonstatic_flexible_variable);
-            VDecl->setInvalidDecl();
-          }
-        }
-      }
   
   // Check any implicit conversions within the expression.
   CheckImplicitConversions(Init, VDecl->getLocation());
   
   if (!VDecl->isInvalidDecl())
     checkUnsafeAssigns(VDecl->getLocation(), VDecl->getType(), Init);
+
+  if (VDecl->isConstexpr() && !VDecl->isInvalidDecl() &&
+      !VDecl->getType()->isDependentType() &&
+      !Init->isTypeDependent() && !Init->isValueDependent() &&
+      !Init->isConstantInitializer(Context,
+                                   VDecl->getType()->isReferenceType())) {
+    // FIXME: Improve this diagnostic to explain why the initializer is not
+    // a constant expression.
+    Diag(VDecl->getLocation(), diag::err_constexpr_var_requires_const_init)
+      << VDecl << Init->getSourceRange();
+  }
   
   Init = MaybeCreateExprWithCleanups(Init);
   // Attach the initializer to the decl.
@@ -5639,6 +6218,21 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
       return;
     }
 
+    // C++0x [dcl.constexpr]p9: An object or reference declared constexpr must
+    // have an initializer.
+    // C++0x [class.static.data]p3: A static data member can be declared with
+    // the constexpr specifier; if so, its declaration shall specify
+    // a brace-or-equal-initializer.
+    //
+    // A static data member's definition may inherit an initializer from an
+    // in-class declaration.
+    if (Var->isConstexpr() && !Var->getAnyInitializer()) {
+      Diag(Var->getLocation(), diag::err_constexpr_var_requires_init)
+        << Var->getDeclName();
+      Var->setInvalidDecl();
+      return;
+    }
+
     switch (Var->isThisDeclarationADefinition()) {
     case VarDecl::Definition:
       if (!Var->isStaticDataMember() || !Var->getAnyInitializer())
@@ -5829,10 +6423,11 @@ void Sema::ActOnCXXForRangeDecl(Decl *D) {
   case SC_Register:
     Error = 4;
     break;
+  case SC_OpenCLWorkGroupLocal:
+    llvm_unreachable("Unexpected storage class");
   }
-  // FIXME: constexpr isn't allowed here.
-  //if (DS.isConstexprSpecified())
-  //  Error = 5;
+  if (VD->isConstexpr())
+    Error = 5;
   if (Error != -1) {
     Diag(VD->getOuterLocStart(), diag::err_for_range_storage_class)
       << VD->getDeclName() << Error;
@@ -5916,7 +6511,7 @@ Sema::FinalizeDeclaration(Decl *ThisDecl) {
 Sema::DeclGroupPtrTy
 Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
                               Decl **Group, unsigned NumDecls) {
-  llvm::SmallVector<Decl*, 8> Decls;
+  SmallVector<Decl*, 8> Decls;
 
   if (DS.isTypeSpecOwned())
     Decls.push_back(DS.getRepAsDecl());
@@ -5996,6 +6591,9 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
 
   if (D.getDeclSpec().isThreadSpecified())
     Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_invalid_thread);
+  if (D.getDeclSpec().isConstexprSpecified())
+    Diag(D.getDeclSpec().getConstexprSpecLoc(), diag::err_invalid_constexpr)
+      << 0;
 
   DiagnoseFunctionSpecifiers(D);
 
@@ -6074,6 +6672,12 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
 
   ProcessDeclAttributes(S, New, D);
 
+  if (D.getDeclSpec().isModulePrivateSpecified())
+    Diag(New->getLocation(), diag::err_module_private_local)
+      << 1 << New->getDeclName()
+      << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
+      << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+
   if (New->hasAttr<BlocksAttr>()) {
     Diag(New->getLocation(), diag::err_block_on_nonlocal);
   }
@@ -6157,8 +6761,9 @@ ParmVarDecl *Sema::CheckParameter(DeclContext *DC, SourceLocation StartLoc,
     //   - otherwise, it's an error
     if (T->isArrayType()) {
       if (!T.isConstQualified()) {
-        Diag(NameLoc, diag::err_arc_array_param_no_ownership)
-          << TSInfo->getTypeLoc().getSourceRange();
+        DelayedDiagnostics.add(
+            sema::DelayedDiagnostic::makeForbiddenType(
+            NameLoc, diag::err_arc_array_param_no_ownership, T, false));
       }
       lifetime = Qualifiers::OCL_ExplicitNone;
     } else {
@@ -6185,8 +6790,10 @@ ParmVarDecl *Sema::CheckParameter(DeclContext *DC, SourceLocation StartLoc,
   // passed by reference.
   if (T->isObjCObjectType()) {
     Diag(NameLoc,
-         diag::err_object_cannot_be_passed_returned_by_value) << 1 << T;
-    New->setInvalidDecl();
+         diag::err_object_cannot_be_passed_returned_by_value) << 1 << T
+      << FixItHint::CreateInsertion(NameLoc, "*");
+    T = Context.getObjCObjectPointerType(T);
+    New->setType(T);
   }
 
   // ISO/IEC TR 18037 S6.7.3: "The type of an object with automatic storage 
@@ -6241,9 +6848,9 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope,
   assert(D.isFunctionDeclarator() && "Not a function declarator!");
   Scope *ParentScope = FnBodyScope->getParent();
 
+  D.setFunctionDefinition(true);
   Decl *DP = HandleDeclarator(ParentScope, D,
-                              MultiTemplateParamsArg(*this),
-                              /*IsFunctionDefinition=*/true);
+                              MultiTemplateParamsArg(*this));
   return ActOnStartOfFunctionDef(FnBodyScope, DP);
 }
 
@@ -6379,7 +6986,7 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
     // dllimport attribute cannot be directly applied to definition.
     // Microsoft accepts dllimport for functions defined within class scope. 
     if (!DA->isInherited() &&
-        !(LangOpts.Microsoft && FD->getLexicalDeclContext()->isRecord())) {
+        !(LangOpts.MicrosoftExt && FD->getLexicalDeclContext()->isRecord())) {
       Diag(FD->getLocation(),
            diag::err_attribute_can_be_applied_only_to_symbol_declaration)
         << "dllimport";
@@ -6389,7 +6996,7 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
 
     // Visual C++ appears to not think this is an issue, so only issue
     // a warning when Microsoft extensions are disabled.
-    if (!LangOpts.Microsoft) {
+    if (!LangOpts.MicrosoftExt) {
       // If a symbol previously declared dllimport is later defined, the
       // attribute is ignored in subsequent references, and a warning is
       // emitted.
@@ -6417,7 +7024,7 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
 /// FIXME: Employ a smarter algorithm that accounts for multiple return 
 /// statements and the lifetimes of the NRVO candidates. We should be able to
 /// find a maximal set of NRVO variables.
-static void ComputeNRVO(Stmt *Body, FunctionScopeInfo *Scope) {
+void Sema::computeNRVO(Stmt *Body, FunctionScopeInfo *Scope) {
   ReturnStmt **Returns = Scope->Returns.data();
 
   const VarDecl *NRVOCandidate = 0;
@@ -6466,7 +7073,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
 
     // MSVC permits the use of pure specifier (=0) on function definition,
     // defined at class scope, warn about this non standard construct.
-    if (getLangOptions().Microsoft && FD->isPure())
+    if (getLangOptions().MicrosoftExt && FD->isPure())
       Diag(FD->getLocation(), diag::warn_pure_function_definition);
 
     if (!FD->isInvalidDecl()) {
@@ -6478,7 +7085,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
       if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(FD))
         MarkVTableUsed(FD->getLocation(), Constructor->getParent());
       
-      ComputeNRVO(Body, getCurFunction());
+      computeNRVO(Body, getCurFunction());
     }
     
     assert(FD == getCurFunctionDecl() && "Function parsing confused");
@@ -6491,11 +7098,27 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
       DiagnoseUnusedParameters(MD->param_begin(), MD->param_end());
       DiagnoseSizeOfParametersAndReturnValue(MD->param_begin(), MD->param_end(),
                                              MD->getResultType(), MD);
+      
+      if (Body)
+        computeNRVO(Body, getCurFunction());
+    }
+    if (ObjCShouldCallSuperDealloc) {
+      Diag(MD->getLocEnd(), diag::warn_objc_missing_super_dealloc);
+      ObjCShouldCallSuperDealloc = false;
+    }
+    if (ObjCShouldCallSuperFinalize) {
+      Diag(MD->getLocEnd(), diag::warn_objc_missing_super_finalize);
+      ObjCShouldCallSuperFinalize = false;
     }
   } else {
     return 0;
   }
 
+  assert(!ObjCShouldCallSuperDealloc && "This should only be set for "
+         "ObjC methods, which should have been handled in the block above.");
+  assert(!ObjCShouldCallSuperFinalize && "This should only be set for "
+         "ObjC methods, which should have been handled in the block above.");
+
   // Verify and clean out per-function state.
   if (Body) {
     // C++ constructors that have function-try-blocks can't have return
@@ -6504,8 +7127,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
     if (FD && isa<CXXConstructorDecl>(FD) && isa<CXXTryStmt>(Body))
       DiagnoseReturnInConstructorExceptionHandler(cast<CXXTryStmt>(Body));
     
-    // Verify that that gotos and switch cases don't jump into scopes illegally.
-    // Verify that that gotos and switch cases don't jump into scopes illegally.
+    // Verify that gotos and switch cases don't jump into scopes illegally.
     if (getCurFunction()->NeedsScopeChecking() &&
         !dcl->isInvalidDecl() &&
         !hasAnyUnrecoverableErrorsInThisFunction())
@@ -6532,6 +7154,10 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
       ActivePolicy = &WP;
     }
 
+    if (FD && FD->isConstexpr() && !FD->isInvalidDecl() &&
+        !CheckConstexprFunctionBody(FD, Body))
+      FD->setInvalidDecl();
+
     assert(ExprTemporaries.empty() && "Leftover temporaries in function");
     assert(!ExprNeedsCleanups && "Unaccounted cleanups in function");
   }
@@ -6552,6 +7178,15 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
   return dcl;
 }
 
+
+/// When we finish delayed parsing of an attribute, we must attach it to the
+/// relevant Decl.
+void Sema::ActOnFinishDelayedAttribute(Scope *S, Decl *D,
+                                       ParsedAttributes &Attrs) {
+  ProcessDeclAttributeList(S, D, Attrs.getList());
+}
+
+
 /// ImplicitlyDefineFunction - An undeclared identifier was used in a function
 /// call, forming a call to an implicitly defined function (per C99 6.5.1p2).
 NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
@@ -6561,7 +7196,7 @@ NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
   // this name as a function or variable. If so, use that
   // (non-visible) declaration, and complain about it.
   llvm::DenseMap<DeclarationName, NamedDecl *>::iterator Pos
-    = LocallyScopedExternalDecls.find(&II);
+    = findLocallyScopedExternalDecl(&II);
   if (Pos != LocallyScopedExternalDecls.end()) {
     Diag(Loc, diag::warn_use_out_of_scope_declaration) << Pos->second;
     Diag(Pos->second->getLocation(), diag::note_previous_declaration);
@@ -6651,6 +7286,9 @@ void Sema::AddKnownFunctionAttributes(FunctionDecl *FD) {
         FD->addAttr(::new (Context) ConstAttr(FD->getLocation(), Context));
     }
 
+    if (Context.BuiltinInfo.isReturnsTwice(BuiltinID) &&
+        !FD->getAttr<ReturnsTwiceAttr>())
+      FD->addAttr(::new (Context) ReturnsTwiceAttr(FD->getLocation(), Context));
     if (Context.BuiltinInfo.isNoThrow(BuiltinID) && !FD->getAttr<NoThrowAttr>())
       FD->addAttr(::new (Context) NoThrowAttr(FD->getLocation(), Context));
     if (Context.BuiltinInfo.isConst(BuiltinID) && !FD->getAttr<ConstAttr>())
@@ -6712,6 +7350,16 @@ TypedefDecl *Sema::ParseTypedefDecl(Scope *S, Declarator &D, QualType T,
     return NewTD;
   }
 
+  if (D.getDeclSpec().isModulePrivateSpecified()) {
+    if (CurContext->isFunctionOrMethod())
+      Diag(NewTD->getLocation(), diag::err_module_private_local)
+        << 2 << NewTD->getDeclName()
+        << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
+        << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+    else
+      NewTD->setModulePrivate();
+  }
+  
   // C++ [dcl.typedef]p8:
   //   If the typedef declaration defines an unnamed class (or
   //   enum), the first typedef-name declared by the declaration
@@ -6853,6 +7501,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
                      SourceLocation KWLoc, CXXScopeSpec &SS,
                      IdentifierInfo *Name, SourceLocation NameLoc,
                      AttributeList *Attr, AccessSpecifier AS,
+                     SourceLocation ModulePrivateLoc,
                      MultiTemplateParamsArg TemplateParameterLists,
                      bool &OwnedDecl, bool &IsDependent,
                      bool ScopedEnum, bool ScopedEnumUsesClassTag,
@@ -6892,6 +7541,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
         DeclResult Result = CheckClassTemplate(S, TagSpec, TUK, KWLoc,
                                                SS, Name, NameLoc, Attr,
                                                TemplateParams, AS,
+                                               ModulePrivateLoc,
                                            TemplateParameterLists.size() - 1,
                  (TemplateParameterList**) TemplateParameterLists.release());
         return Result.get();
@@ -6934,7 +7584,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
                                           UPPC_FixedUnderlyingType))
         EnumUnderlying = Context.IntTy.getTypePtr();
 
-    } else if (getLangOptions().Microsoft)
+    } else if (getLangOptions().MicrosoftExt)
       // Microsoft enums are always of int type.
       EnumUnderlying = Context.IntTy.getTypePtr();
   }
@@ -7233,7 +7883,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
           // For our current ASTs this shouldn't be a problem, but will
           // need to be changed with DeclGroups.
           if ((TUK == TUK_Reference && (!PrevTagDecl->getFriendObjectKind() ||
-               getLangOptions().Microsoft)) || TUK == TUK_Friend)
+               getLangOptions().MicrosoftExt)) || TUK == TUK_Friend)
             return PrevTagDecl;
 
           // Diagnose attempts to redefine a tag.
@@ -7385,7 +8035,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
         Diag(Def->getLocation(), diag::note_previous_definition);
       } else {
         unsigned DiagID = diag::ext_forward_ref_enum;
-        if (getLangOptions().Microsoft)
+        if (getLangOptions().MicrosoftExt)
           DiagID = diag::ext_ms_forward_ref_enum;
         else if (getLangOptions().CPlusPlus)
           DiagID = diag::err_forward_ref_enum;
@@ -7454,6 +8104,23 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
     AddMsStructLayoutForRecord(RD);
   }
 
+  if (PrevDecl && PrevDecl->isModulePrivate())
+    New->setModulePrivate();
+  else if (ModulePrivateLoc.isValid()) {
+    if (isExplicitSpecialization)
+      Diag(New->getLocation(), diag::err_module_private_specialization)
+        << 2
+        << FixItHint::CreateRemoval(ModulePrivateLoc);
+    else if (PrevDecl && !PrevDecl->isModulePrivate())
+      diagnoseModulePrivateRedeclaration(New, PrevDecl, ModulePrivateLoc);
+    // __module_private__ does not apply to local classes. However, we only
+    // diagnose this as an error when the declaration specifiers are
+    // freestanding. Here, we just ignore the __module_private__.
+    // foobar
+    else if (!SearchDC->isFunctionOrMethod())
+      New->setModulePrivate();
+  }
+  
   // If this is a specialization of a member class (of a class template),
   // check the specialization.
   if (isExplicitSpecialization && CheckMemberSpecialization(New, Previous))
@@ -7480,7 +8147,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   // the tag name visible.
   if (TUK == TUK_Friend)
     New->setObjectOfFriendDecl(/* PreviouslyDeclared = */ !Previous.empty() ||
-                               getLangOptions().Microsoft);
+                               getLangOptions().MicrosoftExt);
 
   // Set the access specifier.
   if (!Invalid && SearchDC->isRecord())
@@ -7530,6 +8197,16 @@ void Sema::ActOnTagStartDefinition(Scope *S, Decl *TagD) {
   PushDeclContext(S, Tag);
 }
 
+Decl *Sema::ActOnObjCContainerStartDefinition(Decl *IDecl) {
+  assert(isa<ObjCContainerDecl>(IDecl) && 
+         "ActOnObjCContainerStartDefinition - Not ObjCContainerDecl");
+  DeclContext *OCD = cast<DeclContext>(IDecl);
+  assert(getContainingDC(OCD) == CurContext &&
+      "The next DeclContext should be lexically contained in the current one.");
+  CurContext = OCD;
+  return IDecl;
+}
+
 void Sema::ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagD,
                                            SourceLocation FinalLoc,
                                            SourceLocation LBraceLoc) {
@@ -7581,6 +8258,21 @@ void Sema::ActOnTagFinishDefinition(Scope *S, Decl *TagD,
   Consumer.HandleTagDeclDefinition(Tag);
 }
 
+void Sema::ActOnObjCContainerFinishDefinition() {
+  // Exit this scope of this interface definition.
+  PopDeclContext();
+}
+
+void Sema::ActOnObjCTemporaryExitContainerContext() {
+  OriginalLexicalContext = CurContext;
+  ActOnObjCContainerFinishDefinition();
+}
+
+void Sema::ActOnObjCReenterContainerContext() {
+  ActOnObjCContainerStartDefinition(cast<Decl>(OriginalLexicalContext));
+  OriginalLexicalContext = 0;
+}
+
 void Sema::ActOnTagDefinitionError(Scope *S, Decl *TagD) {
   AdjustDeclIfTemplate(TagD);
   TagDecl *Tag = cast<TagDecl>(TagD);
@@ -7669,7 +8361,7 @@ bool Sema::VerifyBitField(SourceLocation FieldLoc, IdentifierInfo *FieldName,
 /// ActOnField - Each field of a C struct/union is passed into this in order
 /// to create a FieldDecl object for it.
 Decl *Sema::ActOnField(Scope *S, Decl *TagD, SourceLocation DeclStart,
-                       Declarator &D, ExprTy *BitfieldWidth) {
+                       Declarator &D, Expr *BitfieldWidth) {
   FieldDecl *Res = HandleField(S, cast_or_null<RecordDecl>(TagD),
                                DeclStart, D, static_cast<Expr*>(BitfieldWidth),
                                /*HasInit=*/false, AS_public);
@@ -7703,6 +8395,9 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record,
 
   if (D.getDeclSpec().isThreadSpecified())
     Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_invalid_thread);
+  if (D.getDeclSpec().isConstexprSpecified())
+    Diag(D.getDeclSpec().getConstexprSpecLoc(), diag::err_invalid_constexpr)
+      << 2;
   
   // Check to see if this name was declared as a member previously
   LookupResult Previous(*this, II, Loc, LookupMemberName, ForRedeclaration);
@@ -7735,6 +8430,9 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record,
   if (NewFD->isInvalidDecl())
     Record->setInvalidDecl();
 
+  if (D.getDeclSpec().isModulePrivateSpecified())
+    NewFD->setModulePrivate();
+  
   if (NewFD->isInvalidDecl() && PrevDecl) {
     // Don't introduce NewFD into scope; there's already something
     // with the same name in the same scope.
@@ -7968,8 +8666,7 @@ void Sema::DiagnoseNontrivial(const RecordType* T, CXXSpecialMember member) {
         }
       }
 
-      assert(0 && "found no user-declared constructors");
-      return;
+      llvm_unreachable("found no user-declared constructors");
     }
     break;
 
@@ -8054,7 +8751,7 @@ void Sema::DiagnoseNontrivial(const RecordType* T, CXXSpecialMember member) {
   case CXXDestructor:
     hasTrivial = &CXXRecordDecl::hasTrivialDestructor; break;
   default:
-    assert(0 && "unexpected special member"); return;
+    llvm_unreachable("unexpected special member");
   }
 
   // Check for nontrivial bases (and recurse).
@@ -8102,7 +8799,7 @@ void Sema::DiagnoseNontrivial(const RecordType* T, CXXSpecialMember member) {
     }
   }
 
-  assert(0 && "found no explanation for non-trivial member");
+  llvm_unreachable("found no explanation for non-trivial member");
 }
 
 /// TranslateIvarVisibility - Translate visibility from a token ID to an
@@ -8110,7 +8807,7 @@ void Sema::DiagnoseNontrivial(const RecordType* T, CXXSpecialMember member) {
 static ObjCIvarDecl::AccessControl
 TranslateIvarVisibility(tok::ObjCKeywordKind ivarVisibility) {
   switch (ivarVisibility) {
-  default: assert(0 && "Unknown visitibility kind");
+  default: llvm_unreachable("Unknown visitibility kind");
   case tok::objc_private: return ObjCIvarDecl::Private;
   case tok::objc_public: return ObjCIvarDecl::Public;
   case tok::objc_protected: return ObjCIvarDecl::Protected;
@@ -8122,8 +8819,7 @@ TranslateIvarVisibility(tok::ObjCKeywordKind ivarVisibility) {
 /// in order to create an IvarDecl object for it.
 Decl *Sema::ActOnIvar(Scope *S,
                                 SourceLocation DeclStart,
-                                Decl *IntfDecl,
-                                Declarator &D, ExprTy *BitfieldWidth,
+                                Declarator &D, Expr *BitfieldWidth,
                                 tok::ObjCKeywordKind Visibility) {
 
   IdentifierInfo *II = D.getIdentifier();
@@ -8165,7 +8861,7 @@ Decl *Sema::ActOnIvar(Scope *S,
     Visibility != tok::objc_not_keyword ? TranslateIvarVisibility(Visibility)
                                         : ObjCIvarDecl::None;
   // Must set ivar's DeclContext to its enclosing interface.
-  ObjCContainerDecl *EnclosingDecl = cast<ObjCContainerDecl>(IntfDecl);
+  ObjCContainerDecl *EnclosingDecl = cast<ObjCContainerDecl>(CurContext);
   ObjCContainerDecl *EnclosingContext;
   if (ObjCImplementationDecl *IMPDecl =
       dyn_cast<ObjCImplementationDecl>(EnclosingDecl)) {
@@ -8213,6 +8909,9 @@ Decl *Sema::ActOnIvar(Scope *S,
   if (getLangOptions().ObjCAutoRefCount && inferObjCARCLifetime(NewID))
     NewID->setInvalidDecl();
 
+  if (D.getDeclSpec().isModulePrivateSpecified())
+    NewID->setModulePrivate();
+  
   if (II) {
     // FIXME: When interfaces are DeclContexts, we'll need to add
     // these to the interface.
@@ -8227,23 +8926,19 @@ Decl *Sema::ActOnIvar(Scope *S,
 /// class and class extensions. For every class @interface and class 
 /// extension @interface, if the last ivar is a bitfield of any type, 
 /// then add an implicit `char :0` ivar to the end of that interface.
-void Sema::ActOnLastBitfield(SourceLocation DeclLoc, Decl *EnclosingDecl,
-                             llvm::SmallVectorImpl<Decl *> &AllIvarDecls) {
+void Sema::ActOnLastBitfield(SourceLocation DeclLoc,
+                             SmallVectorImpl<Decl *> &AllIvarDecls) {
   if (!LangOpts.ObjCNonFragileABI2 || AllIvarDecls.empty())
     return;
   
   Decl *ivarDecl = AllIvarDecls[AllIvarDecls.size()-1];
   ObjCIvarDecl *Ivar = cast<ObjCIvarDecl>(ivarDecl);
   
-  if (!Ivar->isBitField())
+  if (!Ivar->isBitField() || Ivar->getBitWidthValue(Context) == 0)
     return;
-  uint64_t BitFieldSize =
-    Ivar->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
-  if (BitFieldSize == 0)
-    return;
-  ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(EnclosingDecl);
+  ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(CurContext);
   if (!ID) {
-    if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(EnclosingDecl)) {
+    if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(CurContext)) {
       if (!CD->IsClassExtension())
         return;
     }
@@ -8252,13 +8947,14 @@ void Sema::ActOnLastBitfield(SourceLocation DeclLoc, Decl *EnclosingDecl,
       return;
   }
   // All conditions are met. Add a new bitfield to the tail end of ivars.
-  llvm::APInt Zero(Context.getTypeSize(Context.CharTy), 0);
-  Expr * BW = IntegerLiteral::Create(Context, Zero, Context.CharTy, DeclLoc);
+  llvm::APInt Zero(Context.getTypeSize(Context.IntTy), 0);
+  Expr * BW = IntegerLiteral::Create(Context, Zero, Context.IntTy, DeclLoc);
 
-  Ivar = ObjCIvarDecl::Create(Context, cast<ObjCContainerDecl>(EnclosingDecl),
+  Ivar = ObjCIvarDecl::Create(Context, cast<ObjCContainerDecl>(CurContext),
                               DeclLoc, DeclLoc, 0,
                               Context.CharTy, 
-                              Context.CreateTypeSourceInfo(Context.CharTy),
+                              Context.getTrivialTypeSourceInfo(Context.CharTy,
+                                                               DeclLoc),
                               ObjCIvarDecl::Private, BW,
                               true);
   AllIvarDecls.push_back(Ivar);
@@ -8266,27 +8962,25 @@ void Sema::ActOnLastBitfield(SourceLocation DeclLoc, Decl *EnclosingDecl,
 
 void Sema::ActOnFields(Scope* S,
                        SourceLocation RecLoc, Decl *EnclosingDecl,
-                       Decl **Fields, unsigned NumFields,
+                       llvm::ArrayRef<Decl *> Fields,
                        SourceLocation LBrac, SourceLocation RBrac,
                        AttributeList *Attr) {
   assert(EnclosingDecl && "missing record or interface decl");
 
   // If the decl this is being inserted into is invalid, then it may be a
   // redeclaration or some other bogus case.  Don't try to add fields to it.
-  if (EnclosingDecl->isInvalidDecl()) {
-    // FIXME: Deallocate fields?
+  if (EnclosingDecl->isInvalidDecl())
     return;
-  }
-
 
   // Verify that all the fields are okay.
   unsigned NumNamedMembers = 0;
-  llvm::SmallVector<FieldDecl*, 32> RecFields;
+  SmallVector<FieldDecl*, 32> RecFields;
 
   RecordDecl *Record = dyn_cast<RecordDecl>(EnclosingDecl);
   bool ARCErrReported = false;
-  for (unsigned i = 0; i != NumFields; ++i) {
-    FieldDecl *FD = cast<FieldDecl>(Fields[i]);
+  for (llvm::ArrayRef<Decl *>::iterator i = Fields.begin(), end = Fields.end();
+       i != end; ++i) {
+    FieldDecl *FD = cast<FieldDecl>(*i);
 
     // Get the type for the field.
     const Type *FDTy = FD->getType().getTypePtr();
@@ -8321,25 +9015,26 @@ void Sema::ActOnFields(Scope* S,
       EnclosingDecl->setInvalidDecl();
       continue;
     } else if (FDTy->isIncompleteArrayType() && Record && 
-               ((i == NumFields - 1 && !Record->isUnion()) ||
-                ((getLangOptions().Microsoft || getLangOptions().CPlusPlus) &&
-                 (i == NumFields - 1 || Record->isUnion())))) {
+               ((i + 1 == Fields.end() && !Record->isUnion()) ||
+                ((getLangOptions().MicrosoftExt ||
+                  getLangOptions().CPlusPlus) &&
+                 (i + 1 == Fields.end() || Record->isUnion())))) {
       // Flexible array member.
       // Microsoft and g++ is more permissive regarding flexible array.
       // It will accept flexible array in union and also
       // as the sole element of a struct/class.
-      if (getLangOptions().Microsoft) {
+      if (getLangOptions().MicrosoftExt) {
         if (Record->isUnion()) 
           Diag(FD->getLocation(), diag::ext_flexible_array_union_ms)
             << FD->getDeclName();
-        else if (NumFields == 1) 
+        else if (Fields.size() == 1) 
           Diag(FD->getLocation(), diag::ext_flexible_array_empty_aggregate_ms)
             << FD->getDeclName() << Record->getTagKind();
       } else if (getLangOptions().CPlusPlus) {
         if (Record->isUnion()) 
           Diag(FD->getLocation(), diag::ext_flexible_array_union_gnu)
             << FD->getDeclName();
-        else if (NumFields == 1) 
+        else if (Fields.size() == 1) 
           Diag(FD->getLocation(), diag::ext_flexible_array_empty_aggregate_gnu)
             << FD->getDeclName() << Record->getTagKind();
       } else if (NumNamedMembers < 1) {
@@ -8376,7 +9071,7 @@ void Sema::ActOnFields(Scope* S,
           // If this is a struct/class and this is not the last element, reject
           // it.  Note that GCC supports variable sized arrays in the middle of
           // structures.
-          if (i != NumFields-1)
+          if (i + 1 != Fields.end())
             Diag(FD->getLocation(), diag::ext_variable_sized_type_in_struct)
               << FD->getDeclName() << FD->getType();
           else {
@@ -8393,10 +9088,10 @@ void Sema::ActOnFields(Scope* S,
         Record->setHasObjectMember(true);
     } else if (FDTy->isObjCObjectType()) {
       /// A field cannot be an Objective-c object
-      Diag(FD->getLocation(), diag::err_statically_allocated_object);
-      FD->setInvalidDecl();
-      EnclosingDecl->setInvalidDecl();
-      continue;
+      Diag(FD->getLocation(), diag::err_statically_allocated_object)
+        << FixItHint::CreateInsertion(FD->getLocation(), "*");
+      QualType T = Context.getObjCObjectPointerType(FD->getType());
+      FD->setType(T);
     } 
     else if (!getLangOptions().CPlusPlus) {
       if (getLangOptions().ObjCAutoRefCount && Record && !ARCErrReported) {
@@ -8421,7 +9116,7 @@ void Sema::ActOnFields(Scope* S,
         }
       }
       else if (getLangOptions().ObjC1 &&
-               getLangOptions().getGCMode() != LangOptions::NonGC &&
+               getLangOptions().getGC() != LangOptions::NonGC &&
                Record && !Record->hasObjectMember()) {
         if (FD->getType()->isObjCObjectPointerType() ||
             FD->getType().isObjCGCStrong())
@@ -8650,7 +9345,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
                                           SourceLocation IdLoc,
                                           IdentifierInfo *Id,
                                           Expr *Val) {
-  unsigned IntWidth = Context.Target.getIntWidth();
+  unsigned IntWidth = Context.getTargetInfo().getIntWidth();
   llvm::APSInt EnumVal(IntWidth);
   QualType EltTy;
 
@@ -8691,7 +9386,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
           //   ... if the initializing value of an enumerator cannot be
           //   represented by the underlying type, the program is ill-formed.
           if (!isRepresentableIntegerValue(Context, EnumVal, EltTy)) {
-            if (getLangOptions().Microsoft) {
+            if (getLangOptions().MicrosoftExt) {
               Diag(IdLoc, diag::ext_enumerator_too_large) << EltTy;
               Val = ImpCastExprToType(Val, EltTy, CK_IntegralCast).take();
             } else 
@@ -8806,7 +9501,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
 Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl, Decl *lastEnumConst,
                               SourceLocation IdLoc, IdentifierInfo *Id,
                               AttributeList *Attr,
-                              SourceLocation EqualLoc, ExprTy *val) {
+                              SourceLocation EqualLoc, Expr *val) {
   EnumDecl *TheEnumDecl = cast<EnumDecl>(theEnumDecl);
   EnumConstantDecl *LastEnumConst =
     cast_or_null<EnumConstantDecl>(lastEnumConst);
@@ -8894,9 +9589,9 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
   // TODO: If the result value doesn't fit in an int, it must be a long or long
   // long value.  ISO C does not support this, but GCC does as an extension,
   // emit a warning.
-  unsigned IntWidth = Context.Target.getIntWidth();
-  unsigned CharWidth = Context.Target.getCharWidth();
-  unsigned ShortWidth = Context.Target.getShortWidth();
+  unsigned IntWidth = Context.getTargetInfo().getIntWidth();
+  unsigned CharWidth = Context.getTargetInfo().getCharWidth();
+  unsigned ShortWidth = Context.getTargetInfo().getShortWidth();
 
   // Verify that all the values are okay, compute the size of the values, and
   // reverse the list.
@@ -8969,12 +9664,12 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
       BestType = Context.IntTy;
       BestWidth = IntWidth;
     } else {
-      BestWidth = Context.Target.getLongWidth();
+      BestWidth = Context.getTargetInfo().getLongWidth();
 
       if (NumNegativeBits <= BestWidth && NumPositiveBits < BestWidth) {
         BestType = Context.LongTy;
       } else {
-        BestWidth = Context.Target.getLongLongWidth();
+        BestWidth = Context.getTargetInfo().getLongLongWidth();
 
         if (NumNegativeBits > BestWidth || NumPositiveBits >= BestWidth)
           Diag(Enum->getLocation(), diag::warn_enum_too_large);
@@ -9001,13 +9696,13 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
         = (NumPositiveBits == BestWidth || !getLangOptions().CPlusPlus)
                            ? Context.UnsignedIntTy : Context.IntTy;
     } else if (NumPositiveBits <=
-               (BestWidth = Context.Target.getLongWidth())) {
+               (BestWidth = Context.getTargetInfo().getLongWidth())) {
       BestType = Context.UnsignedLongTy;
       BestPromotionType
         = (NumPositiveBits == BestWidth || !getLangOptions().CPlusPlus)
                            ? Context.UnsignedLongTy : Context.LongTy;
     } else {
-      BestWidth = Context.Target.getLongLongWidth();
+      BestWidth = Context.getTargetInfo().getLongLongWidth();
       assert(NumPositiveBits <= BestWidth &&
              "How could an initializer get larger than ULL?");
       BestType = Context.UnsignedLongLongTy;
@@ -9094,6 +9789,33 @@ Decl *Sema::ActOnFileScopeAsmDecl(Expr *expr,
   return New;
 }
 
+DeclResult Sema::ActOnModuleImport(SourceLocation ImportLoc,
+                                   IdentifierInfo &ModuleName,
+                                   SourceLocation ModuleNameLoc) {
+  ModuleKey Module = PP.getModuleLoader().loadModule(ImportLoc, 
+                                                     ModuleName, ModuleNameLoc);
+  if (!Module)
+    return true;
+  
+  // FIXME: Actually create a declaration to describe the module import.
+  (void)Module;
+  return DeclResult((Decl *)0);
+}
+
+void 
+Sema::diagnoseModulePrivateRedeclaration(NamedDecl *New, NamedDecl *Old,
+                                         SourceLocation ModulePrivateKeyword) {
+  assert(!Old->isModulePrivate() && "Old is module-private!");
+  
+  Diag(New->getLocation(), diag::err_module_private_follows_public)
+    << New->getDeclName() << SourceRange(ModulePrivateKeyword);
+  Diag(Old->getLocation(), diag::note_previous_declaration)
+    << Old->getDeclName();
+  
+  // Drop the __module_private__ from the new declaration, since it's invalid.
+  New->setModulePrivate(false);
+}
+
 void Sema::ActOnPragmaWeakID(IdentifierInfo* Name,
                              SourceLocation PragmaLoc,
                              SourceLocation NameLoc) {
@@ -9126,3 +9848,16 @@ void Sema::ActOnPragmaWeakAlias(IdentifierInfo* Name,
       std::pair<IdentifierInfo*,WeakInfo>(AliasName, W));
   }
 }
+
+Decl *Sema::getObjCDeclContext() const {
+  return (dyn_cast_or_null<ObjCContainerDecl>(CurContext));
+}
+
+AvailabilityResult Sema::getCurContextAvailability() const {
+  const Decl *D = cast<Decl>(getCurLexicalContext());
+  // A category implicitly has the availability of the interface.
+  if (const ObjCCategoryDecl *CatD = dyn_cast<ObjCCategoryDecl>(D))
+    D = CatD->getClassInterface();
+  
+  return D->getAvailability();
+}
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp
index 61b7b3ee05e3..69baf79ad0fe 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp
@@ -15,19 +15,21 @@
 #include "TargetAttributesSema.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/DelayedDiagnostic.h"
+#include "clang/Sema/Lookup.h"
 #include "llvm/ADT/StringExtras.h"
 using namespace clang;
 using namespace sema;
 
 /// These constants match the enumerated choices of
 /// warn_attribute_wrong_decl_type and err_attribute_wrong_decl_type.
-enum {
+enum AttributeDeclKind {
   ExpectedFunction,
   ExpectedUnion,
   ExpectedVariableOrFunction,
@@ -42,7 +44,8 @@ enum {
   ExpectedClassMember,
   ExpectedVariable,
   ExpectedMethod,
-  ExpectedVariableFunctionOrLabel
+  ExpectedVariableFunctionOrLabel,
+  ExpectedFieldOrGlobalVar
 };
 
 //===----------------------------------------------------------------------===//
@@ -194,6 +197,8 @@ static inline bool isCFStringType(QualType T, ASTContext &Ctx) {
   return RD->getIdentifier() == &Ctx.Idents.get("__CFString");
 }
 
+/// \brief Check if the attribute has exactly as many args as Num. May
+/// output an error.
 static bool checkAttributeNumArgs(Sema &S, const AttributeList &Attr,
                                   unsigned int Num) {
   if (Attr.getNumArgs() != Num) {
@@ -204,6 +209,140 @@ static bool checkAttributeNumArgs(Sema &S, const AttributeList &Attr,
   return true;
 }
 
+
+/// \brief Check if the attribute has at least as many args as Num. May
+/// output an error.
+static bool checkAttributeAtLeastNumArgs(Sema &S, const AttributeList &Attr,
+                                  unsigned int Num) {
+  if (Attr.getNumArgs() < Num) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_too_few_arguments) << Num;
+    return false;
+  }
+
+  return true;
+}
+
+///
+/// \brief Check if passed in Decl is a field or potentially shared global var
+/// \return true if the Decl is a field or potentially shared global variable
+///
+static bool mayBeSharedVariable(const Decl *D) {
+  if (isa<FieldDecl>(D))
+    return true;
+  if (const VarDecl *vd = dyn_cast<VarDecl>(D))
+    return (vd->hasGlobalStorage() && !(vd->isThreadSpecified()));
+
+  return false;
+}
+
+/// \brief Check if the passed-in expression is of type int or bool.
+static bool isIntOrBool(Expr *Exp) {
+  QualType QT = Exp->getType();
+  return QT->isBooleanType() || QT->isIntegerType();
+}
+
+///
+/// \brief Check if passed in Decl is a pointer type.
+/// Note that this function may produce an error message.
+/// \return true if the Decl is a pointer type; false otherwise
+///
+static bool checkIsPointer(Sema &S, const Decl *D, const AttributeList &Attr) {
+  if (const ValueDecl *vd = dyn_cast<ValueDecl>(D)) {
+    QualType QT = vd->getType();
+    if (QT->isAnyPointerType())
+      return true;
+    S.Diag(Attr.getLoc(), diag::warn_pointer_attribute_wrong_type)
+      << Attr.getName()->getName() << QT;
+  } else {
+    S.Diag(Attr.getLoc(), diag::err_attribute_can_be_applied_only_to_value_decl)
+      << Attr.getName();
+  }
+  return false;
+}
+
+/// \brief Checks that the passed in QualType either is of RecordType or points
+/// to RecordType. Returns the relevant RecordType, null if it does not exit.
+static const RecordType *getRecordType(QualType QT) {
+  if (const RecordType *RT = QT->getAs<RecordType>())
+    return RT;
+
+  // Now check if we point to record type.
+  if (const PointerType *PT = QT->getAs<PointerType>())
+    return PT->getPointeeType()->getAs<RecordType>();
+
+  return 0;
+}
+
+/// \brief Thread Safety Analysis: Checks that the passed in RecordType
+/// resolves to a lockable object. May flag an error.
+static bool checkForLockableRecord(Sema &S, Decl *D, const AttributeList &Attr,
+                                   const RecordType *RT) {
+  // Flag error if could not get record type for this argument.
+  if (!RT) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_argument_not_class)
+      << Attr.getName();
+    return false;
+  }
+  // Flag error if the type is not lockable.
+  if (!RT->getDecl()->getAttr<LockableAttr>()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_argument_not_lockable)
+      << Attr.getName();
+    return false;
+  }
+  return true;
+}
+
+/// \brief Thread Safety Analysis: Checks that all attribute arguments, starting
+/// from Sidx, resolve to a lockable object. May flag an error.
+/// \param Sidx The attribute argument index to start checking with.
+/// \param ParamIdxOk Whether an argument can be indexing into a function
+/// parameter list.
+static bool checkAttrArgsAreLockableObjs(Sema &S, Decl *D,
+                                         const AttributeList &Attr,
+                                         SmallVectorImpl<Expr*> &Args,
+                                         int Sidx = 0,
+                                         bool ParamIdxOk = false) {
+  for(unsigned Idx = Sidx; Idx < Attr.getNumArgs(); ++Idx) {
+    Expr *ArgExp = Attr.getArg(Idx);
+
+    if (ArgExp->isTypeDependent()) {
+      // FIXME -- need to processs this again on template instantiation
+      Args.push_back(ArgExp);
+      continue;
+    }
+
+    QualType ArgTy = ArgExp->getType();
+
+    // First see if we can just cast to record type, or point to record type.
+    const RecordType *RT = getRecordType(ArgTy);
+
+    // Now check if we index into a record type function param.
+    if(!RT && ParamIdxOk) {
+      FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+      IntegerLiteral *IL = dyn_cast<IntegerLiteral>(ArgExp);
+      if(FD && IL) {
+        unsigned int NumParams = FD->getNumParams();
+        llvm::APInt ArgValue = IL->getValue();
+        uint64_t ParamIdxFromOne = ArgValue.getZExtValue();
+        uint64_t ParamIdxFromZero = ParamIdxFromOne - 1;
+        if(!ArgValue.isStrictlyPositive() || ParamIdxFromOne > NumParams) {
+          S.Diag(Attr.getLoc(), diag::err_attribute_argument_out_of_range)
+            << Attr.getName() << Idx + 1 << NumParams;
+          return false;
+        }
+        ArgTy = FD->getParamDecl(ParamIdxFromZero)->getType();
+        RT = getRecordType(ArgTy);
+      }
+    }
+
+    if (!checkForLockableRecord(S, D, Attr, RT))
+      return false;
+
+    Args.push_back(ArgExp);
+  }
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Attribute Implementations
 //===----------------------------------------------------------------------===//
@@ -212,6 +351,324 @@ static bool checkAttributeNumArgs(Sema &S, const AttributeList &Attr,
 // least add some helper functions to check most argument patterns (#
 // and types of args).
 
+static void handleGuardedVarAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                                 bool pointer = false) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeNumArgs(S, Attr, 0))
+    return;
+
+  // D must be either a member field or global (potentially shared) variable.
+  if (!mayBeSharedVariable(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFieldOrGlobalVar;
+    return;
+  }
+
+  if (pointer && !checkIsPointer(S, D, Attr))
+    return;
+
+  if (pointer)
+    D->addAttr(::new (S.Context) PtGuardedVarAttr(Attr.getRange(), S.Context));
+  else
+    D->addAttr(::new (S.Context) GuardedVarAttr(Attr.getRange(), S.Context));
+}
+
+static void handleGuardedByAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                                bool pointer = false) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeNumArgs(S, Attr, 1))
+    return;
+
+  Expr *Arg = Attr.getArg(0);
+
+  // D must be either a member field or global (potentially shared) variable.
+  if (!mayBeSharedVariable(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFieldOrGlobalVar;
+    return;
+  }
+
+  if (pointer && !checkIsPointer(S, D, Attr))
+    return;
+
+  if (Arg->isTypeDependent())
+    // FIXME: handle attributes with dependent types
+    return;
+
+  // check that the argument is lockable object
+  if (!checkForLockableRecord(S, D, Attr, getRecordType(Arg->getType())))
+    return;
+
+  if (pointer)
+    D->addAttr(::new (S.Context) PtGuardedByAttr(Attr.getRange(),
+                                                 S.Context, Arg));
+  else
+    D->addAttr(::new (S.Context) GuardedByAttr(Attr.getRange(), S.Context, Arg));
+}
+
+
+static void handleLockableAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                               bool scoped = false) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeNumArgs(S, Attr, 0))
+    return;
+
+  // FIXME: Lockable structs for C code.
+  if (!isa<CXXRecordDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedClass;
+    return;
+  }
+
+  if (scoped)
+    D->addAttr(::new (S.Context) ScopedLockableAttr(Attr.getRange(), S.Context));
+  else
+    D->addAttr(::new (S.Context) LockableAttr(Attr.getRange(), S.Context));
+}
+
+static void handleNoThreadSafetyAttr(Sema &S, Decl *D,
+                                     const AttributeList &Attr) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeNumArgs(S, Attr, 0))
+    return;
+
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  D->addAttr(::new (S.Context) NoThreadSafetyAnalysisAttr(Attr.getRange(),
+                                                          S.Context));
+}
+
+static void handleAcquireOrderAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                                   bool before) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeAtLeastNumArgs(S, Attr, 1))
+    return;
+
+  // D must be either a member field or global (potentially shared) variable.
+  ValueDecl *VD = dyn_cast<ValueDecl>(D);
+  if (!VD || !mayBeSharedVariable(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFieldOrGlobalVar;
+    return;
+  }
+
+  // Check that this attribute only applies to lockable types
+  QualType QT = VD->getType();
+  if (!QT->isDependentType()) {
+    const RecordType *RT = getRecordType(QT);
+    if (!RT || !RT->getDecl()->getAttr<LockableAttr>()) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_decl_not_lockable)
+              << Attr.getName();
+      return;
+    }
+  }
+
+  SmallVector<Expr*, 1> Args;
+  // check that all arguments are lockable objects
+  if (!checkAttrArgsAreLockableObjs(S, D, Attr, Args))
+    return;
+
+  unsigned Size = Args.size();
+  assert(Size == Attr.getNumArgs());
+  Expr **StartArg = Size == 0 ? 0 : &Args[0];
+
+  if (before)
+    D->addAttr(::new (S.Context) AcquiredBeforeAttr(Attr.getRange(), S.Context,
+                                                    StartArg, Size));
+  else
+    D->addAttr(::new (S.Context) AcquiredAfterAttr(Attr.getRange(), S.Context,
+                                                   StartArg, Size));
+}
+
+static void handleLockFunAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                              bool exclusive = false) {
+  assert(!Attr.isInvalid());
+
+  // zero or more arguments ok
+
+  // check that the attribute is applied to a function
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  // check that all arguments are lockable objects
+  SmallVector<Expr*, 1> Args;
+  if (!checkAttrArgsAreLockableObjs(S, D, Attr, Args, 0, /*ParamIdxOk=*/true))
+    return;
+
+  unsigned Size = Args.size();
+  assert(Size == Attr.getNumArgs());
+  Expr **StartArg = Size == 0 ? 0 : &Args[0];
+
+  if (exclusive)
+    D->addAttr(::new (S.Context) ExclusiveLockFunctionAttr(Attr.getRange(),
+                                                           S.Context, StartArg,
+                                                           Size));
+  else
+    D->addAttr(::new (S.Context) SharedLockFunctionAttr(Attr.getRange(),
+                                                        S.Context, StartArg,
+                                                        Size));
+}
+
+static void handleTrylockFunAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                                 bool exclusive = false) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeAtLeastNumArgs(S, Attr, 1))
+    return;
+
+
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  if (!isIntOrBool(Attr.getArg(0))) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_first_argument_not_int_or_bool)
+        << Attr.getName();
+    return;
+  }
+
+  SmallVector<Expr*, 2> Args;
+  // check that all arguments are lockable objects
+  if (!checkAttrArgsAreLockableObjs(S, D, Attr, Args, 1))
+    return;
+
+  unsigned Size = Args.size();
+  Expr **StartArg = Size == 0 ? 0 : &Args[0];
+
+  if (exclusive)
+    D->addAttr(::new (S.Context) ExclusiveTrylockFunctionAttr(Attr.getRange(),
+                                                              S.Context,
+                                                              Attr.getArg(0),
+                                                              StartArg, Size));
+  else
+    D->addAttr(::new (S.Context) SharedTrylockFunctionAttr(Attr.getRange(),
+                                                           S.Context,
+                                                           Attr.getArg(0),
+                                                           StartArg, Size));
+}
+
+static void handleLocksRequiredAttr(Sema &S, Decl *D, const AttributeList &Attr,
+                                    bool exclusive = false) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeAtLeastNumArgs(S, Attr, 1))
+    return;
+
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  // check that all arguments are lockable objects
+  SmallVector<Expr*, 1> Args;
+  if (!checkAttrArgsAreLockableObjs(S, D, Attr, Args))
+    return;
+
+  unsigned Size = Args.size();
+  assert(Size == Attr.getNumArgs());
+  Expr **StartArg = Size == 0 ? 0 : &Args[0];
+
+  if (exclusive)
+    D->addAttr(::new (S.Context) ExclusiveLocksRequiredAttr(Attr.getRange(),
+                                                            S.Context, StartArg,
+                                                            Size));
+  else
+    D->addAttr(::new (S.Context) SharedLocksRequiredAttr(Attr.getRange(),
+                                                         S.Context, StartArg,
+                                                         Size));
+}
+
+static void handleUnlockFunAttr(Sema &S, Decl *D,
+                                const AttributeList &Attr) {
+  assert(!Attr.isInvalid());
+
+  // zero or more arguments ok
+
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  // check that all arguments are lockable objects
+  SmallVector<Expr*, 1> Args;
+  if (!checkAttrArgsAreLockableObjs(S, D, Attr, Args, 0, /*ParamIdxOk=*/true))
+    return;
+
+  unsigned Size = Args.size();
+  assert(Size == Attr.getNumArgs());
+  Expr **StartArg = Size == 0 ? 0 : &Args[0];
+
+  D->addAttr(::new (S.Context) UnlockFunctionAttr(Attr.getRange(), S.Context,
+                                                  StartArg, Size));
+}
+
+static void handleLockReturnedAttr(Sema &S, Decl *D,
+                                   const AttributeList &Attr) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeNumArgs(S, Attr, 1))
+    return;
+  Expr *Arg = Attr.getArg(0);
+
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  if (Arg->isTypeDependent())
+    return;
+
+  // check that the argument is lockable object
+  if (!checkForLockableRecord(S, D, Attr, getRecordType(Arg->getType())))
+    return;
+
+  D->addAttr(::new (S.Context) LockReturnedAttr(Attr.getRange(), S.Context, Arg));
+}
+
+static void handleLocksExcludedAttr(Sema &S, Decl *D,
+                                    const AttributeList &Attr) {
+  assert(!Attr.isInvalid());
+
+  if (!checkAttributeAtLeastNumArgs(S, Attr, 1))
+    return;
+
+  if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  // check that all arguments are lockable objects
+  SmallVector<Expr*, 1> Args;
+  if (!checkAttrArgsAreLockableObjs(S, D, Attr, Args))
+    return;
+
+  unsigned Size = Args.size();
+  assert(Size == Attr.getNumArgs());
+  Expr **StartArg = Size == 0 ? 0 : &Args[0];
+
+  D->addAttr(::new (S.Context) LocksExcludedAttr(Attr.getRange(), S.Context,
+                                                 StartArg, Size));
+}
+
+
 static void handleExtVectorTypeAttr(Sema &S, Scope *scope, Decl *D,
                                     const AttributeList &Attr) {
   TypedefNameDecl *tDecl = dyn_cast<TypedefNameDecl>(D);
@@ -261,7 +718,7 @@ static void handlePackedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
 
   if (TagDecl *TD = dyn_cast<TagDecl>(D))
-    TD->addAttr(::new (S.Context) PackedAttr(Attr.getLoc(), S.Context));
+    TD->addAttr(::new (S.Context) PackedAttr(Attr.getRange(), S.Context));
   else if (FieldDecl *FD = dyn_cast<FieldDecl>(D)) {
     // If the alignment is less than or equal to 8 bits, the packed attribute
     // has no effect.
@@ -270,14 +727,14 @@ static void handlePackedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       S.Diag(Attr.getLoc(), diag::warn_attribute_ignored_for_field_of_type)
         << Attr.getName() << FD->getType();
     else
-      FD->addAttr(::new (S.Context) PackedAttr(Attr.getLoc(), S.Context));
+      FD->addAttr(::new (S.Context) PackedAttr(Attr.getRange(), S.Context));
   } else
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << Attr.getName();
 }
 
 static void handleMsStructAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   if (TagDecl *TD = dyn_cast<TagDecl>(D))
-    TD->addAttr(::new (S.Context) MsStructAttr(Attr.getLoc(), S.Context));
+    TD->addAttr(::new (S.Context) MsStructAttr(Attr.getRange(), S.Context));
   else
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << Attr.getName();
 }
@@ -290,26 +747,48 @@ static void handleIBAction(Sema &S, Decl *D, const AttributeList &Attr) {
   // The IBAction attributes only apply to instance methods.
   if (ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D))
     if (MD->isInstanceMethod()) {
-      D->addAttr(::new (S.Context) IBActionAttr(Attr.getLoc(), S.Context));
+      D->addAttr(::new (S.Context) IBActionAttr(Attr.getRange(), S.Context));
       return;
     }
 
   S.Diag(Attr.getLoc(), diag::warn_attribute_ibaction) << Attr.getName();
 }
 
+static bool checkIBOutletCommon(Sema &S, Decl *D, const AttributeList &Attr) {
+  // The IBOutlet/IBOutletCollection attributes only apply to instance
+  // variables or properties of Objective-C classes.  The outlet must also
+  // have an object reference type.
+  if (const ObjCIvarDecl *VD = dyn_cast<ObjCIvarDecl>(D)) {
+    if (!VD->getType()->getAs<ObjCObjectPointerType>()) {
+      S.Diag(Attr.getLoc(), diag::err_iboutlet_object_type)
+        << Attr.getName() << VD->getType() << 0;
+      return false;
+    }
+  }
+  else if (const ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(D)) {
+    if (!PD->getType()->getAs<ObjCObjectPointerType>()) {
+      S.Diag(Attr.getLoc(), diag::err_iboutlet_object_type) 
+        << Attr.getName() << PD->getType() << 1;
+      return false;
+    }
+  }
+  else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_iboutlet) << Attr.getName();
+    return false;
+  }
+  
+  return true;
+}
+
 static void handleIBOutlet(Sema &S, Decl *D, const AttributeList &Attr) {
   // check the attribute arguments.
   if (!checkAttributeNumArgs(S, Attr, 0))
     return;
-
-  // The IBOutlet attributes only apply to instance variables of
-  // Objective-C classes.
-  if (isa<ObjCIvarDecl>(D) || isa<ObjCPropertyDecl>(D)) {
-    D->addAttr(::new (S.Context) IBOutletAttr(Attr.getLoc(), S.Context));
+  
+  if (!checkIBOutletCommon(S, D, Attr))
     return;
-  }
 
-  S.Diag(Attr.getLoc(), diag::warn_attribute_iboutlet) << Attr.getName();
+  D->addAttr(::new (S.Context) IBOutletAttr(Attr.getRange(), S.Context));
 }
 
 static void handleIBOutletCollection(Sema &S, Decl *D,
@@ -321,25 +800,9 @@ static void handleIBOutletCollection(Sema &S, Decl *D,
     return;
   }
 
-  // The IBOutletCollection attributes only apply to instance variables of
-  // Objective-C classes.
-  if (!(isa<ObjCIvarDecl>(D) || isa<ObjCPropertyDecl>(D))) {
-    S.Diag(Attr.getLoc(), diag::warn_attribute_iboutlet) << Attr.getName();
+  if (!checkIBOutletCommon(S, D, Attr))
     return;
-  }
-  if (const ValueDecl *VD = dyn_cast<ValueDecl>(D))
-    if (!VD->getType()->getAs<ObjCObjectPointerType>()) {
-      S.Diag(Attr.getLoc(), diag::err_iboutletcollection_object_type) 
-        << VD->getType() << 0;
-      return;
-    }
-  if (const ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(D))
-    if (!PD->getType()->getAs<ObjCObjectPointerType>()) {
-      S.Diag(Attr.getLoc(), diag::err_iboutletcollection_object_type) 
-        << PD->getType() << 1;
-      return;
-    }
-  
+
   IdentifierInfo *II = Attr.getParameterName();
   if (!II)
     II = &S.Context.Idents.get("id");
@@ -360,8 +823,8 @@ static void handleIBOutletCollection(Sema &S, Decl *D,
     S.Diag(Attr.getLoc(), diag::err_iboutletcollection_type) << II;
     return;
   }
-  D->addAttr(::new (S.Context) IBOutletCollectionAttr(Attr.getLoc(), S.Context,
-                                                      QT));
+  D->addAttr(::new (S.Context) IBOutletCollectionAttr(Attr.getRange(),S.Context,
+                                                   QT, Attr.getParameterLoc()));
 }
 
 static void possibleTransparentUnionPointerType(QualType &T) {
@@ -394,7 +857,7 @@ static void handleNonNullAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   unsigned NumArgs  = getFunctionOrMethodNumArgs(D) + HasImplicitThisParam;
 
   // The nonnull attribute only applies to pointers.
-  llvm::SmallVector<unsigned, 10> NonNullArgs;
+  SmallVector<unsigned, 10> NonNullArgs;
 
   for (AttributeList::arg_iterator I=Attr.arg_begin(),
                                    E=Attr.arg_end(); I!=E; ++I) {
@@ -466,7 +929,7 @@ static void handleNonNullAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   unsigned* start = &NonNullArgs[0];
   unsigned size = NonNullArgs.size();
   llvm::array_pod_sort(start, start + size);
-  D->addAttr(::new (S.Context) NonNullAttr(Attr.getLoc(), S.Context, start,
+  D->addAttr(::new (S.Context) NonNullAttr(Attr.getRange(), S.Context, start,
                                            size));
 }
 
@@ -526,13 +989,13 @@ static void handleOwnershipAttr(Sema &S, Decl *D, const AttributeList &AL) {
   bool HasImplicitThisParam = isInstanceMethod(D);
   unsigned NumArgs  = getFunctionOrMethodNumArgs(D) + HasImplicitThisParam;
 
-  llvm::StringRef Module = AL.getParameterName()->getName();
+  StringRef Module = AL.getParameterName()->getName();
 
   // Normalize the argument, __foo__ becomes foo.
   if (Module.startswith("__") && Module.endswith("__"))
     Module = Module.substr(2, Module.size() - 4);
 
-  llvm::SmallVector<unsigned, 10> OwnershipArgs;
+  SmallVector<unsigned, 10> OwnershipArgs;
 
   for (AttributeList::arg_iterator I = AL.arg_begin(), E = AL.arg_end(); I != E;
        ++I) {
@@ -712,18 +1175,18 @@ static void handleWeakRefAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     Arg = Arg->IgnoreParenCasts();
     StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
 
-    if (Str == 0 || Str->isWide()) {
+    if (!Str || !Str->isAscii()) {
       S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
           << "weakref" << 1;
       return;
     }
     // GCC will accept anything as the argument of weakref. Should we
     // check for an existing decl?
-    D->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context,
+    D->addAttr(::new (S.Context) AliasAttr(Attr.getRange(), S.Context,
                                            Str->getString()));
   }
 
-  D->addAttr(::new (S.Context) WeakRefAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) WeakRefAttr(Attr.getRange(), S.Context));
 }
 
 static void handleAliasAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -737,20 +1200,20 @@ static void handleAliasAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   Arg = Arg->IgnoreParenCasts();
   StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
 
-  if (Str == 0 || Str->isWide()) {
+  if (!Str || !Str->isAscii()) {
     S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
       << "alias" << 1;
     return;
   }
 
-  if (S.Context.Target.getTriple().isOSDarwin()) {
+  if (S.Context.getTargetInfo().getTriple().isOSDarwin()) {
     S.Diag(Attr.getLoc(), diag::err_alias_not_supported_on_darwin);
     return;
   }
 
   // FIXME: check if target symbol exists in current file
 
-  D->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) AliasAttr(Attr.getRange(), S.Context,
                                          Str->getString()));
 }
 
@@ -765,7 +1228,7 @@ static void handleNakedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) NakedAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) NakedAttr(Attr.getRange(), S.Context));
 }
 
 static void handleAlwaysInlineAttr(Sema &S, Decl *D,
@@ -782,7 +1245,7 @@ static void handleAlwaysInlineAttr(Sema &S, Decl *D,
     return;
   }
 
-  D->addAttr(::new (S.Context) AlwaysInlineAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) AlwaysInlineAttr(Attr.getRange(), S.Context));
 }
 
 static void handleMallocAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -795,7 +1258,7 @@ static void handleMallocAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     QualType RetTy = FD->getResultType();
     if (RetTy->isAnyPointerType() || RetTy->isBlockPointerType()) {
-      D->addAttr(::new (S.Context) MallocAttr(Attr.getLoc(), S.Context));
+      D->addAttr(::new (S.Context) MallocAttr(Attr.getRange(), S.Context));
       return;
     }
   }
@@ -808,13 +1271,13 @@ static void handleMayAliasAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   if (!checkAttributeNumArgs(S, Attr, 0))
     return;
 
-  D->addAttr(::new (S.Context) MayAliasAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) MayAliasAttr(Attr.getRange(), S.Context));
 }
 
 static void handleNoCommonAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   assert(!Attr.isInvalid());
   if (isa<VarDecl>(D))
-    D->addAttr(::new (S.Context) NoCommonAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) NoCommonAttr(Attr.getRange(), S.Context));
   else
     S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
       << Attr.getName() << ExpectedVariable;
@@ -823,7 +1286,7 @@ static void handleNoCommonAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 static void handleCommonAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   assert(!Attr.isInvalid());
   if (isa<VarDecl>(D))
-    D->addAttr(::new (S.Context) CommonAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CommonAttr(Attr.getRange(), S.Context));
   else
     S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
       << Attr.getName() << ExpectedVariable;
@@ -840,7 +1303,7 @@ static void handleNoReturnAttr(Sema &S, Decl *D, const AttributeList &attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) NoReturnAttr(attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) NoReturnAttr(attr.getRange(), S.Context));
 }
 
 bool Sema::CheckNoReturnAttr(const AttributeList &attr) {
@@ -874,7 +1337,7 @@ static void handleAnalyzerNoReturnAttr(Sema &S, Decl *D,
     }
   }
   
-  D->addAttr(::new (S.Context) AnalyzerNoReturnAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) AnalyzerNoReturnAttr(Attr.getRange(), S.Context));
 }
 
 // PS3 PPU-specific.
@@ -935,7 +1398,7 @@ static void handleVecReturnAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     count++;
   }
 
-  D->addAttr(::new (S.Context) VecReturnAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) VecReturnAttr(Attr.getRange(), S.Context));
 }
 
 static void handleDependencyAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -961,7 +1424,24 @@ static void handleUnusedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) UnusedAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) UnusedAttr(Attr.getRange(), S.Context));
+}
+
+static void handleReturnsTwiceAttr(Sema &S, Decl *D,
+                                   const AttributeList &Attr) {
+  // check the attribute arguments.
+  if (Attr.hasParameterOrArguments()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+    return;
+  }
+
+  if (!isa<FunctionDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunction;
+    return;
+  }
+
+  D->addAttr(::new (S.Context) ReturnsTwiceAttr(Attr.getRange(), S.Context));
 }
 
 static void handleUsedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -982,7 +1462,7 @@ static void handleUsedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) UsedAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) UsedAttr(Attr.getRange(), S.Context));
 }
 
 static void handleConstructorAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1011,7 +1491,7 @@ static void handleConstructorAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) ConstructorAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) ConstructorAttr(Attr.getRange(), S.Context,
                                                priority));
 }
 
@@ -1041,7 +1521,7 @@ static void handleDestructorAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) DestructorAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) DestructorAttr(Attr.getRange(), S.Context,
                                               priority));
 }
 
@@ -1053,7 +1533,7 @@ static void handleDeprecatedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   }
   
   // Handle the case where deprecated attribute has a text message.
-  llvm::StringRef Str;
+  StringRef Str;
   if (NumArgs == 1) {
     StringLiteral *SE = dyn_cast<StringLiteral>(Attr.getArg(0));
     if (!SE) {
@@ -1064,7 +1544,7 @@ static void handleDeprecatedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     Str = SE->getString();
   }
 
-  D->addAttr(::new (S.Context) DeprecatedAttr(Attr.getLoc(), S.Context, Str));
+  D->addAttr(::new (S.Context) DeprecatedAttr(Attr.getRange(), S.Context, Str));
 }
 
 static void handleUnavailableAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1075,7 +1555,7 @@ static void handleUnavailableAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   }
   
   // Handle the case where unavailable attribute has a text message.
-  llvm::StringRef Str;
+  StringRef Str;
   if (NumArgs == 1) {
     StringLiteral *SE = dyn_cast<StringLiteral>(Attr.getArg(0));
     if (!SE) {
@@ -1085,7 +1565,7 @@ static void handleUnavailableAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     }
     Str = SE->getString();
   }
-  D->addAttr(::new (S.Context) UnavailableAttr(Attr.getLoc(), S.Context, Str));
+  D->addAttr(::new (S.Context) UnavailableAttr(Attr.getRange(), S.Context, Str));
 }
 
 static void handleArcWeakrefUnavailableAttr(Sema &S, Decl *D, 
@@ -1097,7 +1577,7 @@ static void handleArcWeakrefUnavailableAttr(Sema &S, Decl *D,
   }
   
   D->addAttr(::new (S.Context) ArcWeakrefUnavailableAttr(
-                                          Attr.getLoc(), S.Context));
+                                          Attr.getRange(), S.Context));
 }
 
 static void handleAvailabilityAttr(Sema &S, Decl *D,
@@ -1105,7 +1585,7 @@ static void handleAvailabilityAttr(Sema &S, Decl *D,
   IdentifierInfo *Platform = Attr.getParameterName();
   SourceLocation PlatformLoc = Attr.getParameterLoc();
 
-  llvm::StringRef PlatformName
+  StringRef PlatformName
     = AvailabilityAttr::getPrettyPlatformName(Platform->getName());
   if (PlatformName.empty()) {
     S.Diag(PlatformLoc, diag::warn_availability_unknown_platform)
@@ -1119,10 +1599,10 @@ static void handleAvailabilityAttr(Sema &S, Decl *D,
   AvailabilityChange Obsoleted = Attr.getAvailabilityObsoleted();
   bool IsUnavailable = Attr.getUnavailableLoc().isValid();
 
-  // Ensure that Introduced < Deprecated < Obsoleted (although not all
+  // Ensure that Introduced <= Deprecated <= Obsoleted (although not all
   // of these steps are needed).
   if (Introduced.isValid() && Deprecated.isValid() &&
-      !(Introduced.Version < Deprecated.Version)) {
+      !(Introduced.Version <= Deprecated.Version)) {
     S.Diag(Introduced.KeywordLoc, diag::warn_availability_version_ordering)
       << 1 << PlatformName << Deprecated.Version.getAsString()
       << 0 << Introduced.Version.getAsString();
@@ -1130,7 +1610,7 @@ static void handleAvailabilityAttr(Sema &S, Decl *D,
   }
 
   if (Introduced.isValid() && Obsoleted.isValid() &&
-      !(Introduced.Version < Obsoleted.Version)) {
+      !(Introduced.Version <= Obsoleted.Version)) {
     S.Diag(Introduced.KeywordLoc, diag::warn_availability_version_ordering)
       << 2 << PlatformName << Obsoleted.Version.getAsString()
       << 0 << Introduced.Version.getAsString();
@@ -1138,14 +1618,14 @@ static void handleAvailabilityAttr(Sema &S, Decl *D,
   }
 
   if (Deprecated.isValid() && Obsoleted.isValid() &&
-      !(Deprecated.Version < Obsoleted.Version)) {
+      !(Deprecated.Version <= Obsoleted.Version)) {
     S.Diag(Deprecated.KeywordLoc, diag::warn_availability_version_ordering)
       << 2 << PlatformName << Obsoleted.Version.getAsString()
       << 1 << Deprecated.Version.getAsString();
     return;
   }
 
-  D->addAttr(::new (S.Context) AvailabilityAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) AvailabilityAttr(Attr.getRange(), S.Context,
                                                 Platform,
                                                 Introduced.Version,
                                                 Deprecated.Version,
@@ -1162,13 +1642,13 @@ static void handleVisibilityAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   Arg = Arg->IgnoreParenCasts();
   StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
 
-  if (Str == 0 || Str->isWide()) {
+  if (!Str || !Str->isAscii()) {
     S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
       << "visibility" << 1;
     return;
   }
 
-  llvm::StringRef TypeStr = Str->getString();
+  StringRef TypeStr = Str->getString();
   VisibilityAttr::VisibilityType type;
 
   if (TypeStr == "default")
@@ -1184,7 +1664,7 @@ static void handleVisibilityAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) VisibilityAttr(Attr.getLoc(), S.Context, type));
+  D->addAttr(::new (S.Context) VisibilityAttr(Attr.getRange(), S.Context, type));
 }
 
 static void handleObjCMethodFamilyAttr(Sema &S, Decl *decl,
@@ -1207,7 +1687,7 @@ static void handleObjCMethodFamilyAttr(Sema &S, Decl *decl,
     return;
   }
 
-  llvm::StringRef param = Attr.getParameterName()->getName();
+  StringRef param = Attr.getParameterName()->getName();
   ObjCMethodFamilyAttr::FamilyKind family;
   if (param == "none")
     family = ObjCMethodFamilyAttr::OMF_None;
@@ -1236,7 +1716,7 @@ static void handleObjCMethodFamilyAttr(Sema &S, Decl *decl,
     return;
   }
 
-  method->addAttr(new (S.Context) ObjCMethodFamilyAttr(Attr.getLoc(),
+  method->addAttr(new (S.Context) ObjCMethodFamilyAttr(Attr.getRange(),
                                                        S.Context, family));
 }
 
@@ -1251,7 +1731,7 @@ static void handleObjCExceptionAttr(Sema &S, Decl *D,
     return;
   }
 
-  D->addAttr(::new (S.Context) ObjCExceptionAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) ObjCExceptionAttr(Attr.getRange(), S.Context));
 }
 
 static void handleObjCNSObject(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1267,7 +1747,7 @@ static void handleObjCNSObject(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
   }
-  D->addAttr(::new (S.Context) ObjCNSObjectAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) ObjCNSObjectAttr(Attr.getRange(), S.Context));
 }
 
 static void
@@ -1282,7 +1762,7 @@ handleOverloadableAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) OverloadableAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) OverloadableAttr(Attr.getRange(), S.Context));
 }
 
 static void handleBlocksAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1306,7 +1786,7 @@ static void handleBlocksAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) BlocksAttr(Attr.getLoc(), S.Context, type));
+  D->addAttr(::new (S.Context) BlocksAttr(Attr.getRange(), S.Context, type));
 }
 
 static void handleSentinelAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1316,7 +1796,7 @@ static void handleSentinelAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  int sentinel = 0;
+  unsigned sentinel = 0;
   if (Attr.getNumArgs() > 0) {
     Expr *E = Attr.getArg(0);
     llvm::APSInt Idx(32);
@@ -1326,16 +1806,17 @@ static void handleSentinelAttr(Sema &S, Decl *D, const AttributeList &Attr) {
        << "sentinel" << 1 << E->getSourceRange();
       return;
     }
-    sentinel = Idx.getZExtValue();
 
-    if (sentinel < 0) {
+    if (Idx.isSigned() && Idx.isNegative()) {
       S.Diag(Attr.getLoc(), diag::err_attribute_sentinel_less_than_zero)
         << E->getSourceRange();
       return;
     }
+
+    sentinel = Idx.getZExtValue();
   }
 
-  int nullPos = 0;
+  unsigned nullPos = 0;
   if (Attr.getNumArgs() > 1) {
     Expr *E = Attr.getArg(1);
     llvm::APSInt Idx(32);
@@ -1347,7 +1828,7 @@ static void handleSentinelAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     }
     nullPos = Idx.getZExtValue();
 
-    if (nullPos > 1 || nullPos < 0) {
+    if ((Idx.isSigned() && Idx.isNegative()) || nullPos > 1) {
       // FIXME: This error message could be improved, it would be nice
       // to say what the bounds actually are.
       S.Diag(Attr.getLoc(), diag::err_attribute_sentinel_not_zero_or_one)
@@ -1357,9 +1838,7 @@ static void handleSentinelAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   }
 
   if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
-    const FunctionType *FT = FD->getType()->getAs<FunctionType>();
-    assert(FT && "FunctionDecl has non-function type?");
-
+    const FunctionType *FT = FD->getType()->castAs<FunctionType>();
     if (isa<FunctionNoProtoType>(FT)) {
       S.Diag(Attr.getLoc(), diag::warn_attribute_sentinel_named_arguments);
       return;
@@ -1398,7 +1877,7 @@ static void handleSentinelAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       << Attr.getName() << ExpectedFunctionMethodOrBlock;
     return;
   }
-  D->addAttr(::new (S.Context) SentinelAttr(Attr.getLoc(), S.Context, sentinel,
+  D->addAttr(::new (S.Context) SentinelAttr(Attr.getRange(), S.Context, sentinel,
                                             nullPos));
 }
 
@@ -1425,7 +1904,7 @@ static void handleWarnUnusedResult(Sema &S, Decl *D, const AttributeList &Attr)
       return;
     }
   
-  D->addAttr(::new (S.Context) WarnUnusedResultAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) WarnUnusedResultAttr(Attr.getRange(), S.Context));
 }
 
 static void handleWeakAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1449,7 +1928,7 @@ static void handleWeakAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  nd->addAttr(::new (S.Context) WeakAttr(Attr.getLoc(), S.Context));
+  nd->addAttr(::new (S.Context) WeakAttr(Attr.getRange(), S.Context));
 }
 
 static void handleWeakImportAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1466,7 +1945,7 @@ static void handleWeakImportAttr(Sema &S, Decl *D, const AttributeList &Attr) {
              diag::warn_attribute_weak_import_invalid_on_definition)
         << "weak_import" << 2 /*variable and function*/;
     else if (isa<ObjCPropertyDecl>(D) || isa<ObjCMethodDecl>(D) ||
-             (S.Context.Target.getTriple().isOSDarwin() &&
+             (S.Context.getTargetInfo().getTriple().isOSDarwin() &&
               isa<ObjCInterfaceDecl>(D))) {
       // Nothing to warn about here.
     } else
@@ -1476,7 +1955,7 @@ static void handleWeakImportAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) WeakImportAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) WeakImportAttr(Attr.getRange(), S.Context));
 }
 
 static void handleReqdWorkGroupSize(Sema &S, Decl *D,
@@ -1497,7 +1976,7 @@ static void handleReqdWorkGroupSize(Sema &S, Decl *D,
     }
     WGSize[i] = (unsigned) ArgNum.getZExtValue();
   }
-  D->addAttr(::new (S.Context) ReqdWorkGroupSizeAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) ReqdWorkGroupSizeAttr(Attr.getRange(), S.Context,
                                                      WGSize[0], WGSize[1],
                                                      WGSize[2]));
 }
@@ -1517,7 +1996,7 @@ static void handleSectionAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   }
 
   // If the target wants to validate the section specifier, make it happen.
-  std::string Error = S.Context.Target.isValidSectionSpecifier(SE->getString());
+  std::string Error = S.Context.getTargetInfo().isValidSectionSpecifier(SE->getString());
   if (!Error.empty()) {
     S.Diag(SE->getLocStart(), diag::err_attribute_section_invalid_for_target)
     << Error;
@@ -1530,7 +2009,7 @@ static void handleSectionAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
   
-  D->addAttr(::new (S.Context) SectionAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) SectionAttr(Attr.getRange(), S.Context,
                                            SE->getString()));
 }
 
@@ -1544,9 +2023,9 @@ static void handleNothrowAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   
   if (NoThrowAttr *Existing = D->getAttr<NoThrowAttr>()) {
     if (Existing->getLocation().isInvalid())
-      Existing->setLocation(Attr.getLoc());
+      Existing->setRange(Attr.getRange());
   } else {
-    D->addAttr(::new (S.Context) NoThrowAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) NoThrowAttr(Attr.getRange(), S.Context));
   }
 }
 
@@ -1559,9 +2038,9 @@ static void handleConstAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 
   if (ConstAttr *Existing = D->getAttr<ConstAttr>()) {
    if (Existing->getLocation().isInvalid())
-     Existing->setLocation(Attr.getLoc());
+     Existing->setRange(Attr.getRange());
   } else {
-    D->addAttr(::new (S.Context) ConstAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) ConstAttr(Attr.getRange(), S.Context));
   }
 }
 
@@ -1570,7 +2049,7 @@ static void handlePureAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   if (!checkAttributeNumArgs(S, Attr, 0))
     return;
 
-  D->addAttr(::new (S.Context) PureAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) PureAttr(Attr.getRange(), S.Context));
 }
 
 static void handleCleanupAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -1629,7 +2108,7 @@ static void handleCleanupAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) CleanupAttr(Attr.getLoc(), S.Context, FD));
+  D->addAttr(::new (S.Context) CleanupAttr(Attr.getRange(), S.Context, FD));
   S.MarkDeclarationReferenced(Attr.getParameterLoc(), FD);
 }
 
@@ -1704,7 +2183,7 @@ static void handleFormatArgAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) FormatArgAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) FormatArgAttr(Attr.getRange(), S.Context,
                                              Idx.getZExtValue()));
 }
 
@@ -1719,7 +2198,7 @@ enum FormatAttrKind {
 
 /// getFormatAttrKind - Map from format attribute names to supported format
 /// types.
-static FormatAttrKind getFormatAttrKind(llvm::StringRef Format) {
+static FormatAttrKind getFormatAttrKind(StringRef Format) {
   // Check for formats that get handled specially.
   if (Format == "NSString")
     return NSStringFormat;
@@ -1788,7 +2267,7 @@ static void handleInitPriorityAttr(Sema &S, Decl *D,
     Attr.setInvalid();
     return;
   }
-  D->addAttr(::new (S.Context) InitPriorityAttr(Attr.getLoc(), S.Context,
+  D->addAttr(::new (S.Context) InitPriorityAttr(Attr.getRange(), S.Context,
                                                 prioritynum));
 }
 
@@ -1819,7 +2298,7 @@ static void handleFormatAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   unsigned NumArgs  = getFunctionOrMethodNumArgs(D) + HasImplicitThisParam;
   unsigned FirstIdx = 1;
 
-  llvm::StringRef Format = Attr.getParameterName()->getName();
+  StringRef Format = Attr.getParameterName()->getName();
 
   // Normalize the argument, __foo__ becomes foo.
   if (Format.startswith("__") && Format.endswith("__"))
@@ -1939,12 +2418,12 @@ static void handleFormatAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       // If we don't have a valid location for this attribute, adopt the
       // location.
       if (f->getLocation().isInvalid())
-        f->setLocation(Attr.getLoc());
+        f->setRange(Attr.getRange());
       return;
     }
   }
   
-  D->addAttr(::new (S.Context) FormatAttr(Attr.getLoc(), S.Context, Format,
+  D->addAttr(::new (S.Context) FormatAttr(Attr.getRange(), S.Context, Format,
                                           Idx.getZExtValue(),
                                           FirstArg.getZExtValue()));
 }
@@ -1970,7 +2449,7 @@ static void handleTransparentUnionAttr(Sema &S, Decl *D,
     return;
   }
 
-  if (!RD->isDefinition()) {
+  if (!RD->isCompleteDefinition()) {
     S.Diag(Attr.getLoc(),
         diag::warn_transparent_union_attribute_not_definition);
     return;
@@ -2013,7 +2492,7 @@ static void handleTransparentUnionAttr(Sema &S, Decl *D,
     }
   }
 
-  RD->addAttr(::new (S.Context) TransparentUnionAttr(Attr.getLoc(), S.Context));
+  RD->addAttr(::new (S.Context) TransparentUnionAttr(Attr.getRange(), S.Context));
 }
 
 static void handleAnnotateAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -2030,7 +2509,15 @@ static void handleAnnotateAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     S.Diag(ArgExpr->getLocStart(), diag::err_attribute_not_string) <<"annotate";
     return;
   }
-  D->addAttr(::new (S.Context) AnnotateAttr(Attr.getLoc(), S.Context,
+
+  // Don't duplicate annotations that are already set.
+  for (specific_attr_iterator<AnnotateAttr>
+       i = D->specific_attr_begin<AnnotateAttr>(),
+       e = D->specific_attr_end<AnnotateAttr>(); i != e; ++i) {
+      if ((*i)->getAnnotation() == SE->getString())
+          return;
+  }
+  D->addAttr(::new (S.Context) AnnotateAttr(Attr.getRange(), S.Context,
                                             SE->getString()));
 }
 
@@ -2046,20 +2533,21 @@ static void handleAlignedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   //       weaker alignment, rather than being silently ignored.
 
   if (Attr.getNumArgs() == 0) {
-    D->addAttr(::new (S.Context) AlignedAttr(Attr.getLoc(), S.Context, true, 0));
+    D->addAttr(::new (S.Context) AlignedAttr(Attr.getRange(), S.Context, true, 0));
     return;
   }
 
-  S.AddAlignedAttr(Attr.getLoc(), D, Attr.getArg(0));
+  S.AddAlignedAttr(Attr.getRange(), D, Attr.getArg(0));
 }
 
-void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E) {
+void Sema::AddAlignedAttr(SourceRange AttrRange, Decl *D, Expr *E) {
   if (E->isTypeDependent() || E->isValueDependent()) {
     // Save dependent expressions in the AST to be instantiated.
-    D->addAttr(::new (Context) AlignedAttr(AttrLoc, Context, true, E));
+    D->addAttr(::new (Context) AlignedAttr(AttrRange, Context, true, E));
     return;
   }
 
+  SourceLocation AttrLoc = AttrRange.getBegin();
   // FIXME: Cache the number on the Attr object?
   llvm::APSInt Alignment(32);
   if (!E->isIntegerConstantExpr(Alignment, Context)) {
@@ -2073,13 +2561,13 @@ void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E) {
     return;
   }
 
-  D->addAttr(::new (Context) AlignedAttr(AttrLoc, Context, true, E));
+  D->addAttr(::new (Context) AlignedAttr(AttrRange, Context, true, E));
 }
 
-void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, TypeSourceInfo *TS) {
+void Sema::AddAlignedAttr(SourceRange AttrRange, Decl *D, TypeSourceInfo *TS) {
   // FIXME: Cache the number on the Attr object if non-dependent?
   // FIXME: Perform checking of type validity
-  D->addAttr(::new (Context) AlignedAttr(AttrLoc, Context, false, TS));
+  D->addAttr(::new (Context) AlignedAttr(AttrRange, Context, false, TS));
   return;
 }
 
@@ -2104,7 +2592,7 @@ static void handleModeAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  llvm::StringRef Str = Attr.getParameterName()->getName();
+  StringRef Str = Attr.getParameterName()->getName();
 
   // Normalize the attribute name, __foo__ becomes foo.
   if (Str.startswith("__") && Str.endswith("__"))
@@ -2136,13 +2624,13 @@ static void handleModeAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     // FIXME: glibc uses 'word' to define register_t; this is narrower than a
     // pointer on PIC16 and other embedded platforms.
     if (Str == "word")
-      DestWidth = S.Context.Target.getPointerWidth(0);
+      DestWidth = S.Context.getTargetInfo().getPointerWidth(0);
     else if (Str == "byte")
-      DestWidth = S.Context.Target.getCharWidth();
+      DestWidth = S.Context.getTargetInfo().getCharWidth();
     break;
   case 7:
     if (Str == "pointer")
-      DestWidth = S.Context.Target.getPointerWidth(0);
+      DestWidth = S.Context.getTargetInfo().getPointerWidth(0);
     break;
   }
 
@@ -2153,7 +2641,7 @@ static void handleModeAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     OldTy = VD->getType();
   else {
     S.Diag(D->getLocation(), diag::err_attr_wrong_decl)
-      << "mode" << SourceRange(Attr.getLoc(), Attr.getLoc());
+      << "mode" << Attr.getRange();
     return;
   }
 
@@ -2216,12 +2704,12 @@ static void handleModeAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     if (!IntegerMode)
       NewTy = S.Context.DoubleTy;
     else if (OldTy->isSignedIntegerType())
-      if (S.Context.Target.getLongWidth() == 64)
+      if (S.Context.getTargetInfo().getLongWidth() == 64)
         NewTy = S.Context.LongTy;
       else
         NewTy = S.Context.LongLongTy;
     else
-      if (S.Context.Target.getLongWidth() == 64)
+      if (S.Context.getTargetInfo().getLongWidth() == 64)
         NewTy = S.Context.UnsignedLongTy;
       else
         NewTy = S.Context.UnsignedLongLongTy;
@@ -2264,7 +2752,7 @@ static void handleNoDebugAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) NoDebugAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) NoDebugAttr(Attr.getRange(), S.Context));
 }
 
 static void handleNoInlineAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -2279,7 +2767,7 @@ static void handleNoInlineAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) NoInlineAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) NoInlineAttr(Attr.getRange(), S.Context));
 }
 
 static void handleNoInstrumentFunctionAttr(Sema &S, Decl *D,
@@ -2295,7 +2783,7 @@ static void handleNoInstrumentFunctionAttr(Sema &S, Decl *D,
     return;
   }
 
-  D->addAttr(::new (S.Context) NoInstrumentFunctionAttr(Attr.getLoc(),
+  D->addAttr(::new (S.Context) NoInstrumentFunctionAttr(Attr.getRange(),
                                                         S.Context));
 }
 
@@ -2313,7 +2801,7 @@ static void handleConstantAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
 
-    D->addAttr(::new (S.Context) CUDAConstantAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CUDAConstantAttr(Attr.getRange(), S.Context));
   } else {
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "constant";
   }
@@ -2333,7 +2821,7 @@ static void handleDeviceAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
 
-    D->addAttr(::new (S.Context) CUDADeviceAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CUDADeviceAttr(Attr.getRange(), S.Context));
   } else {
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "device";
   }
@@ -2366,7 +2854,7 @@ static void handleGlobalAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
 
-    D->addAttr(::new (S.Context) CUDAGlobalAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CUDAGlobalAttr(Attr.getRange(), S.Context));
   } else {
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "global";
   }
@@ -2385,7 +2873,7 @@ static void handleHostAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
 
-    D->addAttr(::new (S.Context) CUDAHostAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CUDAHostAttr(Attr.getRange(), S.Context));
   } else {
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "host";
   }
@@ -2404,7 +2892,7 @@ static void handleSharedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
 
-    D->addAttr(::new (S.Context) CUDASharedAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CUDASharedAttr(Attr.getRange(), S.Context));
   } else {
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "shared";
   }
@@ -2427,7 +2915,7 @@ static void handleGNUInlineAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) GNUInlineAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) GNUInlineAttr(Attr.getRange(), S.Context));
 }
 
 static void handleCallConvAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -2447,31 +2935,31 @@ static void handleCallConvAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 
   switch (Attr.getKind()) {
   case AttributeList::AT_fastcall:
-    D->addAttr(::new (S.Context) FastCallAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) FastCallAttr(Attr.getRange(), S.Context));
     return;
   case AttributeList::AT_stdcall:
-    D->addAttr(::new (S.Context) StdCallAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) StdCallAttr(Attr.getRange(), S.Context));
     return;
   case AttributeList::AT_thiscall:
-    D->addAttr(::new (S.Context) ThisCallAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) ThisCallAttr(Attr.getRange(), S.Context));
     return;
   case AttributeList::AT_cdecl:
-    D->addAttr(::new (S.Context) CDeclAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) CDeclAttr(Attr.getRange(), S.Context));
     return;
   case AttributeList::AT_pascal:
-    D->addAttr(::new (S.Context) PascalAttr(Attr.getLoc(), S.Context));
+    D->addAttr(::new (S.Context) PascalAttr(Attr.getRange(), S.Context));
     return;
   case AttributeList::AT_pcs: {
     Expr *Arg = Attr.getArg(0);
     StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
-    if (Str == 0 || Str->isWide()) {
+    if (!Str || !Str->isAscii()) {
       S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
         << "pcs" << 1;
       Attr.setInvalid();
       return;
     }
 
-    llvm::StringRef StrRef = Str->getString();
+    StringRef StrRef = Str->getString();
     PcsAttr::PCSType PCS;
     if (StrRef == "aapcs")
       PCS = PcsAttr::AAPCS;
@@ -2483,7 +2971,7 @@ static void handleCallConvAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       return;
     }
 
-    D->addAttr(::new (S.Context) PcsAttr(Attr.getLoc(), S.Context, PCS));
+    D->addAttr(::new (S.Context) PcsAttr(Attr.getRange(), S.Context, PCS));
   }
   default:
     llvm_unreachable("unexpected attribute kind");
@@ -2493,7 +2981,7 @@ static void handleCallConvAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 
 static void handleOpenCLKernelAttr(Sema &S, Decl *D, const AttributeList &Attr){
   assert(!Attr.isInvalid());
-  D->addAttr(::new (S.Context) OpenCLKernelAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) OpenCLKernelAttr(Attr.getRange(), S.Context));
 }
 
 bool Sema::CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC) {
@@ -2519,14 +3007,14 @@ bool Sema::CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC) {
   case AttributeList::AT_pcs: {
     Expr *Arg = attr.getArg(0);
     StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
-    if (Str == 0 || Str->isWide()) {
+    if (!Str || !Str->isAscii()) {
       Diag(attr.getLoc(), diag::err_attribute_argument_n_not_string)
         << "pcs" << 1;
       attr.setInvalid();
       return true;
     }
 
-    llvm::StringRef StrRef = Str->getString();
+    StringRef StrRef = Str->getString();
     if (StrRef == "aapcs") {
       CC = CC_AAPCS;
       break;
@@ -2555,7 +3043,7 @@ static void handleRegparmAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     return;
   }
 
-  D->addAttr(::new (S.Context) RegparmAttr(Attr.getLoc(), S.Context, numParams));
+  D->addAttr(::new (S.Context) RegparmAttr(Attr.getRange(), S.Context, numParams));
 }
 
 /// Checks a regparm attribute, returning true if it is ill-formed and
@@ -2580,7 +3068,7 @@ bool Sema::CheckRegparmAttr(const AttributeList &Attr, unsigned &numParams) {
     return true;
   }
 
-  if (Context.Target.getRegParmMax() == 0) {
+  if (Context.getTargetInfo().getRegParmMax() == 0) {
     Diag(Attr.getLoc(), diag::err_attribute_regparm_wrong_platform)
       << NumParamsExpr->getSourceRange();
     Attr.setInvalid();
@@ -2588,9 +3076,9 @@ bool Sema::CheckRegparmAttr(const AttributeList &Attr, unsigned &numParams) {
   }
 
   numParams = NumParams.getZExtValue();
-  if (numParams > Context.Target.getRegParmMax()) {
+  if (numParams > Context.getTargetInfo().getRegParmMax()) {
     Diag(Attr.getLoc(), diag::err_attribute_regparm_invalid_number)
-      << Context.Target.getRegParmMax() << NumParamsExpr->getSourceRange();
+      << Context.getTargetInfo().getRegParmMax() << NumParamsExpr->getSourceRange();
     Attr.setInvalid();
     return true;
   }
@@ -2635,7 +3123,7 @@ static void handleLaunchBoundsAttr(Sema &S, Decl *D, const AttributeList &Attr){
       }
     }
 
-    D->addAttr(::new (S.Context) CUDALaunchBoundsAttr(Attr.getLoc(), S.Context,
+    D->addAttr(::new (S.Context) CUDALaunchBoundsAttr(Attr.getRange(), S.Context,
                                                       MaxThreads.getZExtValue(),
                                                      MinBlocks.getZExtValue()));
   } else {
@@ -2648,17 +3136,21 @@ static void handleLaunchBoundsAttr(Sema &S, Decl *D, const AttributeList &Attr){
 //===----------------------------------------------------------------------===//
 
 static bool isValidSubjectOfNSAttribute(Sema &S, QualType type) {
-  return type->isObjCObjectPointerType() || S.Context.isObjCNSObjectType(type);
+  return type->isDependentType() || 
+         type->isObjCObjectPointerType() || 
+         S.Context.isObjCNSObjectType(type);
 }
 static bool isValidSubjectOfCFAttribute(Sema &S, QualType type) {
-  return type->isPointerType() || isValidSubjectOfNSAttribute(S, type);
+  return type->isDependentType() || 
+         type->isPointerType() || 
+         isValidSubjectOfNSAttribute(S, type);
 }
 
 static void handleNSConsumedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   ParmVarDecl *param = dyn_cast<ParmVarDecl>(D);
   if (!param) {
     S.Diag(D->getLocStart(), diag::warn_attribute_wrong_decl_type)
-      << SourceRange(Attr.getLoc()) << Attr.getName() << ExpectedParameter;
+      << Attr.getRange() << Attr.getName() << ExpectedParameter;
     return;
   }
 
@@ -2673,25 +3165,25 @@ static void handleNSConsumedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 
   if (!typeOK) {
     S.Diag(D->getLocStart(), diag::warn_ns_attribute_wrong_parameter_type)
-      << SourceRange(Attr.getLoc()) << Attr.getName() << cf;
+      << Attr.getRange() << Attr.getName() << cf;
     return;
   }
 
   if (cf)
-    param->addAttr(::new (S.Context) CFConsumedAttr(Attr.getLoc(), S.Context));
+    param->addAttr(::new (S.Context) CFConsumedAttr(Attr.getRange(), S.Context));
   else
-    param->addAttr(::new (S.Context) NSConsumedAttr(Attr.getLoc(), S.Context));
+    param->addAttr(::new (S.Context) NSConsumedAttr(Attr.getRange(), S.Context));
 }
 
 static void handleNSConsumesSelfAttr(Sema &S, Decl *D,
                                      const AttributeList &Attr) {
   if (!isa<ObjCMethodDecl>(D)) {
     S.Diag(D->getLocStart(), diag::warn_attribute_wrong_decl_type)
-      << SourceRange(Attr.getLoc()) << Attr.getName() << ExpectedMethod;
+      << Attr.getRange() << Attr.getName() << ExpectedMethod;
     return;
   }
 
-  D->addAttr(::new (S.Context) NSConsumesSelfAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) NSConsumesSelfAttr(Attr.getRange(), S.Context));
 }
 
 static void handleNSReturnsRetainedAttr(Sema &S, Decl *D,
@@ -2710,7 +3202,7 @@ static void handleNSReturnsRetainedAttr(Sema &S, Decl *D,
     returnType = FD->getResultType();
   else {
     S.Diag(D->getLocStart(), diag::warn_attribute_wrong_decl_type)
-        << SourceRange(Attr.getLoc()) << Attr.getName()
+        << Attr.getRange() << Attr.getName()
         << ExpectedFunctionOrMethod;
     return;
   }
@@ -2735,53 +3227,143 @@ static void handleNSReturnsRetainedAttr(Sema &S, Decl *D,
 
   if (!typeOK) {
     S.Diag(D->getLocStart(), diag::warn_ns_attribute_wrong_return_type)
-      << SourceRange(Attr.getLoc())
-      << Attr.getName() << isa<ObjCMethodDecl>(D) << cf;
+      << Attr.getRange() << Attr.getName() << isa<ObjCMethodDecl>(D) << cf;
     return;
   }
 
   switch (Attr.getKind()) {
     default:
-      assert(0 && "invalid ownership attribute");
-      return;
+      llvm_unreachable("invalid ownership attribute");
     case AttributeList::AT_ns_returns_autoreleased:
-      D->addAttr(::new (S.Context) NSReturnsAutoreleasedAttr(Attr.getLoc(),
+      D->addAttr(::new (S.Context) NSReturnsAutoreleasedAttr(Attr.getRange(),
                                                              S.Context));
       return;
     case AttributeList::AT_cf_returns_not_retained:
-      D->addAttr(::new (S.Context) CFReturnsNotRetainedAttr(Attr.getLoc(),
+      D->addAttr(::new (S.Context) CFReturnsNotRetainedAttr(Attr.getRange(),
                                                             S.Context));
       return;
     case AttributeList::AT_ns_returns_not_retained:
-      D->addAttr(::new (S.Context) NSReturnsNotRetainedAttr(Attr.getLoc(),
+      D->addAttr(::new (S.Context) NSReturnsNotRetainedAttr(Attr.getRange(),
                                                             S.Context));
       return;
     case AttributeList::AT_cf_returns_retained:
-      D->addAttr(::new (S.Context) CFReturnsRetainedAttr(Attr.getLoc(),
+      D->addAttr(::new (S.Context) CFReturnsRetainedAttr(Attr.getRange(),
                                                          S.Context));
       return;
     case AttributeList::AT_ns_returns_retained:
-      D->addAttr(::new (S.Context) NSReturnsRetainedAttr(Attr.getLoc(),
+      D->addAttr(::new (S.Context) NSReturnsRetainedAttr(Attr.getRange(),
                                                          S.Context));
       return;
   };
 }
 
+static void handleObjCReturnsInnerPointerAttr(Sema &S, Decl *D,
+                                              const AttributeList &attr) {
+  SourceLocation loc = attr.getLoc();
+
+  ObjCMethodDecl *method = dyn_cast<ObjCMethodDecl>(D);
+
+  if (!isa<ObjCMethodDecl>(method)) {
+    S.Diag(method->getLocStart(), diag::err_attribute_wrong_decl_type)
+      << SourceRange(loc, loc) << attr.getName() << 13 /* methods */;
+    return;
+  }
+
+  // Check that the method returns a normal pointer.
+  QualType resultType = method->getResultType();
+    
+  if (!resultType->isReferenceType() &&
+      (!resultType->isPointerType() || resultType->isObjCRetainableType())) {
+    S.Diag(method->getLocStart(), diag::warn_ns_attribute_wrong_return_type)
+      << SourceRange(loc)
+      << attr.getName() << /*method*/ 1 << /*non-retainable pointer*/ 2;
+
+    // Drop the attribute.
+    return;
+  }
+
+  method->addAttr(
+    ::new (S.Context) ObjCReturnsInnerPointerAttr(attr.getRange(), S.Context));
+}
+
+/// Handle cf_audited_transfer and cf_unknown_transfer.
+static void handleCFTransferAttr(Sema &S, Decl *D, const AttributeList &A) {
+  if (!isa<FunctionDecl>(D)) {
+    S.Diag(D->getLocStart(), diag::err_attribute_wrong_decl_type)
+      << A.getRange() << A.getName() << 0 /*function*/;
+    return;
+  }
+
+  bool IsAudited = (A.getKind() == AttributeList::AT_cf_audited_transfer);
+
+  // Check whether there's a conflicting attribute already present.
+  Attr *Existing;
+  if (IsAudited) {
+    Existing = D->getAttr<CFUnknownTransferAttr>();
+  } else {
+    Existing = D->getAttr<CFAuditedTransferAttr>();
+  }
+  if (Existing) {
+    S.Diag(D->getLocStart(), diag::err_attributes_are_not_compatible)
+      << A.getName()
+      << (IsAudited ? "cf_unknown_transfer" : "cf_audited_transfer")
+      << A.getRange() << Existing->getRange();
+    return;
+  }
+
+  // All clear;  add the attribute.
+  if (IsAudited) {
+    D->addAttr(
+      ::new (S.Context) CFAuditedTransferAttr(A.getRange(), S.Context));
+  } else {
+    D->addAttr(
+      ::new (S.Context) CFUnknownTransferAttr(A.getRange(), S.Context));
+  }
+}
+
+static void handleNSBridgedAttr(Sema &S, Scope *Sc, Decl *D,
+                                const AttributeList &Attr) {
+  RecordDecl *RD = dyn_cast<RecordDecl>(D);
+  if (!RD || RD->isUnion()) {
+    S.Diag(D->getLocStart(), diag::err_attribute_wrong_decl_type)
+      << Attr.getRange() << Attr.getName() << 14 /*struct */;
+  }
+
+  IdentifierInfo *ParmName = Attr.getParameterName();
+
+  // In Objective-C, verify that the type names an Objective-C type.
+  // We don't want to check this outside of ObjC because people sometimes
+  // do crazy C declarations of Objective-C types.
+  if (ParmName && S.getLangOptions().ObjC1) {
+    // Check for an existing type with this name.
+    LookupResult R(S, DeclarationName(ParmName), Attr.getParameterLoc(),
+                   Sema::LookupOrdinaryName);
+    if (S.LookupName(R, Sc)) {
+      NamedDecl *Target = R.getFoundDecl();
+      if (Target && !isa<ObjCInterfaceDecl>(Target)) {
+        S.Diag(D->getLocStart(), diag::err_ns_bridged_not_interface);
+        S.Diag(Target->getLocStart(), diag::note_declared_at);
+      }
+    }
+  }
+
+  D->addAttr(::new (S.Context) NSBridgedAttr(Attr.getRange(), S.Context,
+                                             ParmName));
+}
+
 static void handleObjCOwnershipAttr(Sema &S, Decl *D,
                                     const AttributeList &Attr) {
   if (hasDeclarator(D)) return;
 
-  SourceLocation L = Attr.getLoc();
   S.Diag(D->getLocStart(), diag::err_attribute_wrong_decl_type)
-    << SourceRange(L, L) << Attr.getName() << 12 /* variable */;
+    << Attr.getRange() << Attr.getName() << 12 /* variable */;
 }
 
 static void handleObjCPreciseLifetimeAttr(Sema &S, Decl *D,
                                           const AttributeList &Attr) {
   if (!isa<VarDecl>(D) && !isa<FieldDecl>(D)) {
-    SourceLocation L = Attr.getLoc();
     S.Diag(D->getLocStart(), diag::err_attribute_wrong_decl_type)
-      << SourceRange(L, L) << Attr.getName() << 12 /* variable */;
+      << Attr.getRange() << Attr.getName() << 12 /* variable */;
     return;
   }
 
@@ -2820,7 +3402,7 @@ static void handleObjCPreciseLifetimeAttr(Sema &S, Decl *D,
   }
 
   D->addAttr(::new (S.Context)
-                 ObjCPreciseLifetimeAttr(Attr.getLoc(), S.Context));
+                 ObjCPreciseLifetimeAttr(Attr.getRange(), S.Context));
 }
 
 static bool isKnownDeclSpecAttr(const AttributeList &Attr) {
@@ -2834,20 +3416,20 @@ static bool isKnownDeclSpecAttr(const AttributeList &Attr) {
 //===----------------------------------------------------------------------===//
 
 static void handleUuidAttr(Sema &S, Decl *D, const AttributeList &Attr) {
-  if (S.LangOpts.Microsoft || S.LangOpts.Borland) {
+  if (S.LangOpts.MicrosoftExt || S.LangOpts.Borland) {
     // check the attribute arguments.
     if (!checkAttributeNumArgs(S, Attr, 1))
       return;
 
     Expr *Arg = Attr.getArg(0);
     StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
-    if (Str == 0 || Str->isWide()) {
+    if (!Str || !Str->isAscii()) {
       S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
         << "uuid" << 1;
       return;
     }
 
-    llvm::StringRef StrRef = Str->getString();
+    StringRef StrRef = Str->getString();
 
     bool IsCurly = StrRef.size() > 1 && StrRef.front() == '{' &&
                    StrRef.back() == '}';
@@ -2864,7 +3446,7 @@ static void handleUuidAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 
     // GUID format is "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" or 
     // "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}"
-    llvm::StringRef::iterator I = StrRef.begin();
+    StringRef::iterator I = StrRef.begin();
     if (IsCurly) // Skip the optional '{'
        ++I;
 
@@ -2881,7 +3463,7 @@ static void handleUuidAttr(Sema &S, Decl *D, const AttributeList &Attr) {
       I++;
     }
 
-    D->addAttr(::new (S.Context) UuidAttr(Attr.getLoc(), S.Context,
+    D->addAttr(::new (S.Context) UuidAttr(Attr.getRange(), S.Context,
                                           Str->getString()));
   } else
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "uuid";
@@ -2969,6 +3551,16 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
   case AttributeList::AT_objc_precise_lifetime:
     handleObjCPreciseLifetimeAttr(S, D, Attr); break;
 
+  case AttributeList::AT_objc_returns_inner_pointer:
+    handleObjCReturnsInnerPointerAttr(S, D, Attr); break;
+
+  case AttributeList::AT_ns_bridged:
+    handleNSBridgedAttr(S, scope, D, Attr); break;
+
+  case AttributeList::AT_cf_audited_transfer:
+  case AttributeList::AT_cf_unknown_transfer:
+    handleCFTransferAttr(S, D, Attr); break;
+
   // Checker-specific.
   case AttributeList::AT_cf_consumed:
   case AttributeList::AT_ns_consumed: handleNSConsumedAttr  (S, D, Attr); break;
@@ -2996,6 +3588,9 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
     handleArcWeakrefUnavailableAttr (S, D, Attr); 
     break;
   case AttributeList::AT_unused:      handleUnusedAttr      (S, D, Attr); break;
+  case AttributeList::AT_returns_twice:
+    handleReturnsTwiceAttr(S, D, Attr);
+    break;
   case AttributeList::AT_used:        handleUsedAttr        (S, D, Attr); break;
   case AttributeList::AT_visibility:  handleVisibilityAttr  (S, D, Attr); break;
   case AttributeList::AT_warn_unused_result: handleWarnUnusedResult(S, D, Attr);
@@ -3041,6 +3636,63 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
   case AttributeList::AT_uuid:
     handleUuidAttr(S, D, Attr);
     break;
+
+  // Thread safety attributes:
+  case AttributeList::AT_guarded_var:
+    handleGuardedVarAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_pt_guarded_var:
+    handleGuardedVarAttr(S, D, Attr, /*pointer = */true);
+    break;
+  case AttributeList::AT_scoped_lockable:
+    handleLockableAttr(S, D, Attr, /*scoped = */true);
+    break;
+  case AttributeList::AT_no_thread_safety_analysis:
+    handleNoThreadSafetyAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_lockable:
+    handleLockableAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_guarded_by:
+    handleGuardedByAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_pt_guarded_by:
+    handleGuardedByAttr(S, D, Attr, /*pointer = */true);
+    break;
+  case AttributeList::AT_exclusive_lock_function:
+    handleLockFunAttr(S, D, Attr, /*exclusive = */true);
+    break;
+  case AttributeList::AT_exclusive_locks_required:
+    handleLocksRequiredAttr(S, D, Attr, /*exclusive = */true);
+    break;
+  case AttributeList::AT_exclusive_trylock_function:
+    handleTrylockFunAttr(S, D, Attr, /*exclusive = */true);
+    break;
+  case AttributeList::AT_lock_returned:
+    handleLockReturnedAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_locks_excluded:
+    handleLocksExcludedAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_shared_lock_function:
+    handleLockFunAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_shared_locks_required:
+    handleLocksRequiredAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_shared_trylock_function:
+    handleTrylockFunAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_unlock_function:
+    handleUnlockFunAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_acquired_before:
+    handleAcquireOrderAttr(S, D, Attr, /*before = */true);
+    break;
+  case AttributeList::AT_acquired_after:
+    handleAcquireOrderAttr(S, D, Attr, /*before = */false);
+    break;
+
   default:
     // Ask target about the attribute.
     const TargetAttributesSema &TargetAttrs = S.getTargetAttributesSema();
@@ -3091,19 +3743,86 @@ void Sema::ProcessDeclAttributeList(Scope *S, Decl *D,
   }
 }
 
+// Annotation attributes are the only attributes allowed after an access
+// specifier.
+bool Sema::ProcessAccessDeclAttributeList(AccessSpecDecl *ASDecl,
+                                          const AttributeList *AttrList) {
+  for (const AttributeList* l = AttrList; l; l = l->getNext()) {
+    if (l->getKind() == AttributeList::AT_annotate) {
+      handleAnnotateAttr(*this, ASDecl, *l);
+    } else {
+      Diag(l->getLoc(), diag::err_only_annotate_after_access_spec);
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/// checkUnusedDeclAttributes - Check a list of attributes to see if it
+/// contains any decl attributes that we should warn about.
+static void checkUnusedDeclAttributes(Sema &S, const AttributeList *A) {
+  for ( ; A; A = A->getNext()) {
+    // Only warn if the attribute is an unignored, non-type attribute.
+    if (A->isUsedAsTypeAttr()) continue;
+    if (A->getKind() == AttributeList::IgnoredAttribute) continue;
+
+    if (A->getKind() == AttributeList::UnknownAttribute) {
+      S.Diag(A->getLoc(), diag::warn_unknown_attribute_ignored)
+        << A->getName() << A->getRange();
+    } else {
+      S.Diag(A->getLoc(), diag::warn_attribute_not_on_decl)
+        << A->getName() << A->getRange();
+    }
+  }
+}
+
+/// checkUnusedDeclAttributes - Given a declarator which is not being
+/// used to build a declaration, complain about any decl attributes
+/// which might be lying around on it.
+void Sema::checkUnusedDeclAttributes(Declarator &D) {
+  ::checkUnusedDeclAttributes(*this, D.getDeclSpec().getAttributes().getList());
+  ::checkUnusedDeclAttributes(*this, D.getAttributes());
+  for (unsigned i = 0, e = D.getNumTypeObjects(); i != e; ++i)
+    ::checkUnusedDeclAttributes(*this, D.getTypeObject(i).getAttrs());
+}
+
 /// DeclClonePragmaWeak - clone existing decl (maybe definition),
 /// #pragma weak needs a non-definition decl and source may not have one
-NamedDecl * Sema::DeclClonePragmaWeak(NamedDecl *ND, IdentifierInfo *II) {
+NamedDecl * Sema::DeclClonePragmaWeak(NamedDecl *ND, IdentifierInfo *II,
+                                      SourceLocation Loc) {
   assert(isa<FunctionDecl>(ND) || isa<VarDecl>(ND));
   NamedDecl *NewD = 0;
   if (FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
-    NewD = FunctionDecl::Create(FD->getASTContext(), FD->getDeclContext(),
-                                FD->getInnerLocStart(),
-                                FD->getLocation(), DeclarationName(II),
-                                FD->getType(), FD->getTypeSourceInfo());
-    if (FD->getQualifier()) {
-      FunctionDecl *NewFD = cast<FunctionDecl>(NewD);
+    FunctionDecl *NewFD;
+    // FIXME: Missing call to CheckFunctionDeclaration().
+    // FIXME: Mangling?
+    // FIXME: Is the qualifier info correct?
+    // FIXME: Is the DeclContext correct?
+    NewFD = FunctionDecl::Create(FD->getASTContext(), FD->getDeclContext(),
+                                 Loc, Loc, DeclarationName(II),
+                                 FD->getType(), FD->getTypeSourceInfo(),
+                                 SC_None, SC_None,
+                                 false/*isInlineSpecified*/,
+                                 FD->hasPrototype(),
+                                 false/*isConstexprSpecified*/);
+    NewD = NewFD;
+
+    if (FD->getQualifier())
       NewFD->setQualifierInfo(FD->getQualifierLoc());
+
+    // Fake up parameter variables; they are declared as if this were
+    // a typedef.
+    QualType FDTy = FD->getType();
+    if (const FunctionProtoType *FT = FDTy->getAs<FunctionProtoType>()) {
+      SmallVector<ParmVarDecl*, 16> Params;
+      for (FunctionProtoType::arg_type_iterator AI = FT->arg_type_begin(),
+           AE = FT->arg_type_end(); AI != AE; ++AI) {
+        ParmVarDecl *Param = BuildParmVarDeclForTypedef(NewFD, Loc, *AI);
+        Param->setScopeInfo(0, Params.size());
+        Params.push_back(Param);
+      }
+      NewFD->setParams(Params);
     }
   } else if (VarDecl *VD = dyn_cast<VarDecl>(ND)) {
     NewD = VarDecl::Create(VD->getASTContext(), VD->getDeclContext(),
@@ -3126,7 +3845,7 @@ void Sema::DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, WeakInfo &W) {
   W.setUsed(true);
   if (W.getAlias()) { // clone decl, impersonate __attribute(weak,alias(...))
     IdentifierInfo *NDId = ND->getIdentifier();
-    NamedDecl *NewD = DeclClonePragmaWeak(ND, W.getAlias());
+    NamedDecl *NewD = DeclClonePragmaWeak(ND, W.getAlias(), W.getLocation());
     NewD->addAttr(::new (Context) AliasAttr(W.getLocation(), Context,
                                             NDId->getName()));
     NewD->addAttr(::new (Context) WeakAttr(W.getLocation(), Context));
@@ -3149,15 +3868,18 @@ void Sema::ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD,
                                  bool NonInheritable, bool Inheritable) {
   // It's valid to "forward-declare" #pragma weak, in which case we
   // have to do this.
-  if (Inheritable && !WeakUndeclaredIdentifiers.empty()) {
-    if (NamedDecl *ND = dyn_cast<NamedDecl>(D)) {
-      if (IdentifierInfo *Id = ND->getIdentifier()) {
-        llvm::DenseMap<IdentifierInfo*,WeakInfo>::iterator I
-          = WeakUndeclaredIdentifiers.find(Id);
-        if (I != WeakUndeclaredIdentifiers.end() && ND->hasLinkage()) {
-          WeakInfo W = I->second;
-          DeclApplyPragmaWeak(S, ND, W);
-          WeakUndeclaredIdentifiers[Id] = W;
+  if (Inheritable) {
+    LoadExternalWeakUndeclaredIdentifiers();
+    if (!WeakUndeclaredIdentifiers.empty()) {
+      if (NamedDecl *ND = dyn_cast<NamedDecl>(D)) {
+        if (IdentifierInfo *Id = ND->getIdentifier()) {
+          llvm::DenseMap<IdentifierInfo*,WeakInfo>::iterator I
+            = WeakUndeclaredIdentifiers.find(Id);
+          if (I != WeakUndeclaredIdentifiers.end() && ND->hasLinkage()) {
+            WeakInfo W = I->second;
+            DeclApplyPragmaWeak(S, ND, W);
+            WeakUndeclaredIdentifiers[Id] = W;
+          }
         }
       }
     }
@@ -3185,7 +3907,9 @@ static bool isForbiddenTypeAllowed(Sema &S, Decl *decl) {
   // Private ivars are always okay.  Unfortunately, people don't
   // always properly make their ivars private, even in system headers.
   // Plus we need to make fields okay, too.
-  if (!isa<FieldDecl>(decl) && !isa<ObjCPropertyDecl>(decl))
+  // Function declarations in sys headers will be marked unavailable.
+  if (!isa<FieldDecl>(decl) && !isa<ObjCPropertyDecl>(decl) &&
+      !isa<FunctionDecl>(decl))
     return false;
 
   // Require it to be declared in a system header.
@@ -3200,6 +3924,17 @@ static void handleDelayedForbiddenType(Sema &S, DelayedDiagnostic &diag,
                         "this system declaration uses an unsupported type"));
     return;
   }
+  if (S.getLangOptions().ObjCAutoRefCount)
+    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(decl)) {
+      // FIXME. we may want to supress diagnostics for all
+      // kind of forbidden type messages on unavailable functions. 
+      if (FD->hasAttr<UnavailableAttr>() &&
+          diag.getForbiddenTypeDiagnostic() == 
+          diag::err_arc_array_param_no_ownership) {
+        diag.Triggered = true;
+        return;
+      }
+    }
 
   S.Diag(diag.Loc, diag.getForbiddenTypeDiagnostic())
     << diag.getForbiddenTypeOperand() << diag.getForbiddenTypeArgument();
@@ -3287,6 +4022,9 @@ static bool isDeclDeprecated(Decl *D) {
   do {
     if (D->isDeprecated())
       return true;
+    // A category implicitly has the availability of the interface.
+    if (const ObjCCategoryDecl *CatD = dyn_cast<ObjCCategoryDecl>(D))
+      return CatD->getClassInterface()->isDeprecated();
   } while ((D = cast_or_null<Decl>(D->getDeclContext())));
   return false;
 }
@@ -3306,7 +4044,7 @@ void Sema::HandleDelayedDeprecationCheck(DelayedDiagnostic &DD,
       << DD.getDeprecationDecl()->getDeclName();
 }
 
-void Sema::EmitDeprecationWarning(NamedDecl *D, llvm::StringRef Message,
+void Sema::EmitDeprecationWarning(NamedDecl *D, StringRef Message,
                                   SourceLocation Loc,
                                   const ObjCInterfaceDecl *UnknownObjCClass) {
   // Delay if we're currently parsing a declaration.
@@ -3316,7 +4054,7 @@ void Sema::EmitDeprecationWarning(NamedDecl *D, llvm::StringRef Message,
   }
 
   // Otherwise, don't warn if our current context is deprecated.
-  if (isDeclDeprecated(cast<Decl>(CurContext)))
+  if (isDeclDeprecated(cast<Decl>(getCurLexicalContext())))
     return;
   if (!Message.empty())
     Diag(Loc, diag::warn_deprecated_message) << D->getDeclName() 
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDeclCXX.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDeclCXX.cpp
index d793daf9d826..a39584a107a8 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaDeclCXX.cpp
@@ -392,7 +392,7 @@ bool Sema::MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old) {
       // MSVC accepts that default parameters be redefined for member functions
       // of template class. The new default parameter's value is ignored.
       Invalid = true;
-      if (getLangOptions().Microsoft) {
+      if (getLangOptions().MicrosoftExt) {
         CXXMethodDecl* MD = dyn_cast<CXXMethodDecl>(New);
         if (MD && MD->getParent()->getDescribedClassTemplate()) {
           // Merge the old default argument into the new parameter.
@@ -502,6 +502,20 @@ bool Sema::MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old) {
     }
   }
 
+  // C++0x [dcl.constexpr]p1: If any declaration of a function or function
+  // template has a constexpr specifier then all its declarations shall
+  // contain the constexpr specifier. [Note: An explicit specialization can
+  // differ from the template declaration with respect to the constexpr
+  // specifier. -- end note]
+  //
+  // FIXME: Don't reject changes in constexpr in explicit specializations.
+  if (New->isConstexpr() != Old->isConstexpr()) {
+    Diag(New->getLocation(), diag::err_constexpr_redecl_mismatch)
+      << New << New->isConstexpr();
+    Diag(Old->getLocation(), diag::note_previous_declaration);
+    Invalid = true;
+  }
+
   if (CheckEquivalentExceptionSpec(Old, New))
     Invalid = true;
 
@@ -602,6 +616,363 @@ void Sema::CheckCXXDefaultArguments(FunctionDecl *FD) {
   }
 }
 
+// CheckConstexprParameterTypes - Check whether a function's parameter types
+// are all literal types. If so, return true. If not, produce a suitable
+// diagnostic depending on @p CCK and return false.
+static bool CheckConstexprParameterTypes(Sema &SemaRef, const FunctionDecl *FD,
+                                         Sema::CheckConstexprKind CCK) {
+  unsigned ArgIndex = 0;
+  const FunctionProtoType *FT = FD->getType()->getAs<FunctionProtoType>();
+  for (FunctionProtoType::arg_type_iterator i = FT->arg_type_begin(),
+       e = FT->arg_type_end(); i != e; ++i, ++ArgIndex) {
+    const ParmVarDecl *PD = FD->getParamDecl(ArgIndex);
+    SourceLocation ParamLoc = PD->getLocation();
+    if (!(*i)->isDependentType() &&
+        SemaRef.RequireLiteralType(ParamLoc, *i, CCK == Sema::CCK_Declaration ?
+                            SemaRef.PDiag(diag::err_constexpr_non_literal_param)
+                                     << ArgIndex+1 << PD->getSourceRange()
+                                     << isa<CXXConstructorDecl>(FD) :
+                                   SemaRef.PDiag(),
+                                   /*AllowIncompleteType*/ true)) {
+      if (CCK == Sema::CCK_NoteNonConstexprInstantiation)
+        SemaRef.Diag(ParamLoc, diag::note_constexpr_tmpl_non_literal_param)
+          << ArgIndex+1 << PD->getSourceRange()
+          << isa<CXXConstructorDecl>(FD) << *i;
+      return false;
+    }
+  }
+  return true;
+}
+
+// CheckConstexprFunctionDecl - Check whether a function declaration satisfies
+// the requirements of a constexpr function declaration or a constexpr
+// constructor declaration. Return true if it does, false if not.
+//
+// This implements C++0x [dcl.constexpr]p3,4, as amended by N3308.
+//
+// \param CCK Specifies whether to produce diagnostics if the function does not
+// satisfy the requirements.
+bool Sema::CheckConstexprFunctionDecl(const FunctionDecl *NewFD,
+                                      CheckConstexprKind CCK) {
+  assert((CCK != CCK_NoteNonConstexprInstantiation ||
+          (NewFD->getTemplateInstantiationPattern() &&
+           NewFD->getTemplateInstantiationPattern()->isConstexpr())) &&
+         "only constexpr templates can be instantiated non-constexpr");
+
+  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(NewFD)) {
+    // C++0x [dcl.constexpr]p4:
+    //  In the definition of a constexpr constructor, each of the parameter
+    //  types shall be a literal type.
+    if (!CheckConstexprParameterTypes(*this, NewFD, CCK))
+      return false;
+
+    //  In addition, either its function-body shall be = delete or = default or
+    //  it shall satisfy the following constraints:
+    //  - the class shall not have any virtual base classes;
+    const CXXRecordDecl *RD = CD->getParent();
+    if (RD->getNumVBases()) {
+      // Note, this is still illegal if the body is = default, since the
+      // implicit body does not satisfy the requirements of a constexpr
+      // constructor. We also reject cases where the body is = delete, as
+      // required by N3308.
+      if (CCK != CCK_Instantiation) {
+        Diag(NewFD->getLocation(),
+             CCK == CCK_Declaration ? diag::err_constexpr_virtual_base
+                                    : diag::note_constexpr_tmpl_virtual_base)
+          << RD->isStruct() << RD->getNumVBases();
+        for (CXXRecordDecl::base_class_const_iterator I = RD->vbases_begin(),
+               E = RD->vbases_end(); I != E; ++I)
+          Diag(I->getSourceRange().getBegin(),
+               diag::note_constexpr_virtual_base_here) << I->getSourceRange();
+      }
+      return false;
+    }
+  } else {
+    // C++0x [dcl.constexpr]p3:
+    //  The definition of a constexpr function shall satisfy the following
+    //  constraints:
+    // - it shall not be virtual;
+    const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(NewFD);
+    if (Method && Method->isVirtual()) {
+      if (CCK != CCK_Instantiation) {
+        Diag(NewFD->getLocation(),
+             CCK == CCK_Declaration ? diag::err_constexpr_virtual
+                                    : diag::note_constexpr_tmpl_virtual);
+
+        // If it's not obvious why this function is virtual, find an overridden
+        // function which uses the 'virtual' keyword.
+        const CXXMethodDecl *WrittenVirtual = Method;
+        while (!WrittenVirtual->isVirtualAsWritten())
+          WrittenVirtual = *WrittenVirtual->begin_overridden_methods();
+        if (WrittenVirtual != Method)
+          Diag(WrittenVirtual->getLocation(), 
+               diag::note_overridden_virtual_function);
+      }
+      return false;
+    }
+
+    // - its return type shall be a literal type;
+    QualType RT = NewFD->getResultType();
+    if (!RT->isDependentType() &&
+        RequireLiteralType(NewFD->getLocation(), RT, CCK == CCK_Declaration ?
+                           PDiag(diag::err_constexpr_non_literal_return) :
+                           PDiag(),
+                           /*AllowIncompleteType*/ true)) {
+      if (CCK == CCK_NoteNonConstexprInstantiation)
+        Diag(NewFD->getLocation(),
+             diag::note_constexpr_tmpl_non_literal_return) << RT;
+      return false;
+    }
+
+    // - each of its parameter types shall be a literal type;
+    if (!CheckConstexprParameterTypes(*this, NewFD, CCK))
+      return false;
+  }
+
+  return true;
+}
+
+/// Check the given declaration statement is legal within a constexpr function
+/// body. C++0x [dcl.constexpr]p3,p4.
+///
+/// \return true if the body is OK, false if we have diagnosed a problem.
+static bool CheckConstexprDeclStmt(Sema &SemaRef, const FunctionDecl *Dcl,
+                                   DeclStmt *DS) {
+  // C++0x [dcl.constexpr]p3 and p4:
+  //  The definition of a constexpr function(p3) or constructor(p4) [...] shall
+  //  contain only
+  for (DeclStmt::decl_iterator DclIt = DS->decl_begin(),
+         DclEnd = DS->decl_end(); DclIt != DclEnd; ++DclIt) {
+    switch ((*DclIt)->getKind()) {
+    case Decl::StaticAssert:
+    case Decl::Using:
+    case Decl::UsingShadow:
+    case Decl::UsingDirective:
+    case Decl::UnresolvedUsingTypename:
+      //   - static_assert-declarations
+      //   - using-declarations,
+      //   - using-directives,
+      continue;
+
+    case Decl::Typedef:
+    case Decl::TypeAlias: {
+      //   - typedef declarations and alias-declarations that do not define
+      //     classes or enumerations,
+      TypedefNameDecl *TN = cast<TypedefNameDecl>(*DclIt);
+      if (TN->getUnderlyingType()->isVariablyModifiedType()) {
+        // Don't allow variably-modified types in constexpr functions.
+        TypeLoc TL = TN->getTypeSourceInfo()->getTypeLoc();
+        SemaRef.Diag(TL.getBeginLoc(), diag::err_constexpr_vla)
+          << TL.getSourceRange() << TL.getType()
+          << isa<CXXConstructorDecl>(Dcl);
+        return false;
+      }
+      continue;
+    }
+
+    case Decl::Enum:
+    case Decl::CXXRecord:
+      // As an extension, we allow the declaration (but not the definition) of
+      // classes and enumerations in all declarations, not just in typedef and
+      // alias declarations.
+      if (cast<TagDecl>(*DclIt)->isThisDeclarationADefinition()) {
+        SemaRef.Diag(DS->getLocStart(), diag::err_constexpr_type_definition)
+          << isa<CXXConstructorDecl>(Dcl);
+        return false;
+      }
+      continue;
+
+    case Decl::Var:
+      SemaRef.Diag(DS->getLocStart(), diag::err_constexpr_var_declaration)
+        << isa<CXXConstructorDecl>(Dcl);
+      return false;
+
+    default:
+      SemaRef.Diag(DS->getLocStart(), diag::err_constexpr_body_invalid_stmt)
+        << isa<CXXConstructorDecl>(Dcl);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/// Check that the given field is initialized within a constexpr constructor.
+///
+/// \param Dcl The constexpr constructor being checked.
+/// \param Field The field being checked. This may be a member of an anonymous
+///        struct or union nested within the class being checked.
+/// \param Inits All declarations, including anonymous struct/union members and
+///        indirect members, for which any initialization was provided.
+/// \param Diagnosed Set to true if an error is produced.
+static void CheckConstexprCtorInitializer(Sema &SemaRef,
+                                          const FunctionDecl *Dcl,
+                                          FieldDecl *Field,
+                                          llvm::SmallSet<Decl*, 16> &Inits,
+                                          bool &Diagnosed) {
+  if (Field->isUnnamedBitfield())
+    return;
+  
+  if (!Inits.count(Field)) {
+    if (!Diagnosed) {
+      SemaRef.Diag(Dcl->getLocation(), diag::err_constexpr_ctor_missing_init);
+      Diagnosed = true;
+    }
+    SemaRef.Diag(Field->getLocation(), diag::note_constexpr_ctor_missing_init);
+  } else if (Field->isAnonymousStructOrUnion()) {
+    const RecordDecl *RD = Field->getType()->castAs<RecordType>()->getDecl();
+    for (RecordDecl::field_iterator I = RD->field_begin(), E = RD->field_end();
+         I != E; ++I)
+      // If an anonymous union contains an anonymous struct of which any member
+      // is initialized, all members must be initialized.
+      if (!RD->isUnion() || Inits.count(*I))
+        CheckConstexprCtorInitializer(SemaRef, Dcl, *I, Inits, Diagnosed);
+  }
+}
+
+/// Check the body for the given constexpr function declaration only contains
+/// the permitted types of statement. C++11 [dcl.constexpr]p3,p4.
+///
+/// \return true if the body is OK, false if we have diagnosed a problem.
+bool Sema::CheckConstexprFunctionBody(const FunctionDecl *Dcl, Stmt *Body) {
+  if (isa<CXXTryStmt>(Body)) {
+    // C++0x [dcl.constexpr]p3:
+    //  The definition of a constexpr function shall satisfy the following
+    //  constraints: [...]
+    // - its function-body shall be = delete, = default, or a
+    //   compound-statement
+    //
+    // C++0x [dcl.constexpr]p4:
+    //  In the definition of a constexpr constructor, [...]
+    // - its function-body shall not be a function-try-block;
+    Diag(Body->getLocStart(), diag::err_constexpr_function_try_block)
+      << isa<CXXConstructorDecl>(Dcl);
+    return false;
+  }
+
+  // - its function-body shall be [...] a compound-statement that contains only
+  CompoundStmt *CompBody = cast<CompoundStmt>(Body);
+
+  llvm::SmallVector<SourceLocation, 4> ReturnStmts;
+  for (CompoundStmt::body_iterator BodyIt = CompBody->body_begin(),
+         BodyEnd = CompBody->body_end(); BodyIt != BodyEnd; ++BodyIt) {
+    switch ((*BodyIt)->getStmtClass()) {
+    case Stmt::NullStmtClass:
+      //   - null statements,
+      continue;
+
+    case Stmt::DeclStmtClass:
+      //   - static_assert-declarations
+      //   - using-declarations,
+      //   - using-directives,
+      //   - typedef declarations and alias-declarations that do not define
+      //     classes or enumerations,
+      if (!CheckConstexprDeclStmt(*this, Dcl, cast<DeclStmt>(*BodyIt)))
+        return false;
+      continue;
+
+    case Stmt::ReturnStmtClass:
+      //   - and exactly one return statement;
+      if (isa<CXXConstructorDecl>(Dcl))
+        break;
+
+      ReturnStmts.push_back((*BodyIt)->getLocStart());
+      // FIXME
+      // - every constructor call and implicit conversion used in initializing
+      //   the return value shall be one of those allowed in a constant
+      //   expression.
+      // Deal with this as part of a general check that the function can produce
+      // a constant expression (for [dcl.constexpr]p5).
+      continue;
+
+    default:
+      break;
+    }
+
+    Diag((*BodyIt)->getLocStart(), diag::err_constexpr_body_invalid_stmt)
+      << isa<CXXConstructorDecl>(Dcl);
+    return false;
+  }
+
+  if (const CXXConstructorDecl *Constructor
+        = dyn_cast<CXXConstructorDecl>(Dcl)) {
+    const CXXRecordDecl *RD = Constructor->getParent();
+    // - every non-static data member and base class sub-object shall be
+    //   initialized;
+    if (RD->isUnion()) {
+      // DR1359: Exactly one member of a union shall be initialized.
+      if (Constructor->getNumCtorInitializers() == 0) {
+        Diag(Dcl->getLocation(), diag::err_constexpr_union_ctor_no_init);
+        return false;
+      }
+    } else if (!Constructor->isDependentContext() &&
+               !Constructor->isDelegatingConstructor()) {
+      assert(RD->getNumVBases() == 0 && "constexpr ctor with virtual bases");
+
+      // Skip detailed checking if we have enough initializers, and we would
+      // allow at most one initializer per member.
+      bool AnyAnonStructUnionMembers = false;
+      unsigned Fields = 0;
+      for (CXXRecordDecl::field_iterator I = RD->field_begin(),
+           E = RD->field_end(); I != E; ++I, ++Fields) {
+        if ((*I)->isAnonymousStructOrUnion()) {
+          AnyAnonStructUnionMembers = true;
+          break;
+        }
+      }
+      if (AnyAnonStructUnionMembers ||
+          Constructor->getNumCtorInitializers() != RD->getNumBases() + Fields) {
+        // Check initialization of non-static data members. Base classes are
+        // always initialized so do not need to be checked. Dependent bases
+        // might not have initializers in the member initializer list.
+        llvm::SmallSet<Decl*, 16> Inits;
+        for (CXXConstructorDecl::init_const_iterator
+               I = Constructor->init_begin(), E = Constructor->init_end();
+             I != E; ++I) {
+          if (FieldDecl *FD = (*I)->getMember())
+            Inits.insert(FD);
+          else if (IndirectFieldDecl *ID = (*I)->getIndirectMember())
+            Inits.insert(ID->chain_begin(), ID->chain_end());
+        }
+
+        bool Diagnosed = false;
+        for (CXXRecordDecl::field_iterator I = RD->field_begin(),
+             E = RD->field_end(); I != E; ++I)
+          CheckConstexprCtorInitializer(*this, Dcl, *I, Inits, Diagnosed);
+        if (Diagnosed)
+          return false;
+      }
+    }
+
+    // FIXME
+    // - every constructor involved in initializing non-static data members
+    //   and base class sub-objects shall be a constexpr constructor;
+    // - every assignment-expression that is an initializer-clause appearing
+    //   directly or indirectly within a brace-or-equal-initializer for
+    //   a non-static data member that is not named by a mem-initializer-id
+    //   shall be a constant expression; and
+    // - every implicit conversion used in converting a constructor argument
+    //   to the corresponding parameter type and converting
+    //   a full-expression to the corresponding member type shall be one of
+    //   those allowed in a constant expression.
+    // Deal with these as part of a general check that the function can produce
+    // a constant expression (for [dcl.constexpr]p5).
+  } else {
+    if (ReturnStmts.empty()) {
+      Diag(Dcl->getLocation(), diag::err_constexpr_body_no_return);
+      return false;
+    }
+    if (ReturnStmts.size() > 1) {
+      Diag(ReturnStmts.back(), diag::err_constexpr_body_multiple_return);
+      for (unsigned I = 0; I < ReturnStmts.size() - 1; ++I)
+        Diag(ReturnStmts[I], diag::note_constexpr_body_previous_return);
+      return false;
+    }
+  }
+
+  return true;
+}
+
 /// isCurrentClassName - Determine whether the identifier II is the
 /// name of the class type currently being defined. In the case of
 /// nested classes, this will only return true if II is the name of
@@ -797,7 +1168,7 @@ bool Sema::AttachBaseSpecifiers(CXXRecordDecl *Class, CXXBaseSpecifier **Bases,
 /// ActOnBaseSpecifiers - Attach the given base specifiers to the
 /// class, after checking whether there are any duplicate base
 /// classes.
-void Sema::ActOnBaseSpecifiers(Decl *ClassDecl, BaseTy **Bases,
+void Sema::ActOnBaseSpecifiers(Decl *ClassDecl, CXXBaseSpecifier **Bases,
                                unsigned NumBases) {
   if (!ClassDecl || !Bases || !NumBases)
     return;
@@ -1005,19 +1376,20 @@ std::string Sema::getAmbiguousPathsDisplayString(CXXBasePaths &Paths) {
 //===----------------------------------------------------------------------===//
 
 /// ActOnAccessSpecifier - Parsed an access specifier followed by a colon.
-Decl *Sema::ActOnAccessSpecifier(AccessSpecifier Access,
-                                 SourceLocation ASLoc,
-                                 SourceLocation ColonLoc) {
+bool Sema::ActOnAccessSpecifier(AccessSpecifier Access,
+                                SourceLocation ASLoc,
+                                SourceLocation ColonLoc,
+                                AttributeList *Attrs) {
   assert(Access != AS_none && "Invalid kind for syntactic access specifier!");
   AccessSpecDecl *ASDecl = AccessSpecDecl::Create(Context, Access, CurContext,
                                                   ASLoc, ColonLoc);
   CurContext->addHiddenDecl(ASDecl);
-  return ASDecl;
+  return ProcessAccessDeclAttributeList(ASDecl, Attrs);
 }
 
 /// CheckOverrideControl - Check C++0x override control semantics.
 void Sema::CheckOverrideControl(const Decl *D) {
-  const CXXMethodDecl *MD = llvm::dyn_cast<CXXMethodDecl>(D);
+  const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D);
   if (!MD || !MD->isVirtual())
     return;
 
@@ -1060,9 +1432,8 @@ bool Sema::CheckIfOverriddenFunctionIsMarkedFinal(const CXXMethodDecl *New,
 Decl *
 Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                                MultiTemplateParamsArg TemplateParameterLists,
-                               ExprTy *BW, const VirtSpecifiers &VS,
-                               ExprTy *InitExpr, bool HasDeferredInit,
-                               bool IsDefinition) {
+                               Expr *BW, const VirtSpecifiers &VS,
+                               bool HasDeferredInit) {
   const DeclSpec &DS = D.getDeclSpec();
   DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
   DeclarationName Name = NameInfo.getName();
@@ -1073,11 +1444,9 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
     Loc = D.getSourceRange().getBegin();
 
   Expr *BitWidth = static_cast<Expr*>(BW);
-  Expr *Init = static_cast<Expr*>(InitExpr);
 
   assert(isa<CXXRecordDecl>(CurContext));
   assert(!DS.isFriendSpecified());
-  assert(!Init || !HasDeferredInit);
 
   bool isFunc = D.isDeclarationOfFunction();
 
@@ -1120,7 +1489,37 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
   Decl *Member;
   if (isInstField) {
     CXXScopeSpec &SS = D.getCXXScopeSpec();
+
+    // Data members must have identifiers for names.
+    if (Name.getNameKind() != DeclarationName::Identifier) {
+      Diag(Loc, diag::err_bad_variable_name)
+        << Name;
+      return 0;
+    }
     
+    IdentifierInfo *II = Name.getAsIdentifierInfo();
+
+    // Member field could not be with "template" keyword.
+    // So TemplateParameterLists should be empty in this case.
+    if (TemplateParameterLists.size()) {
+      TemplateParameterList* TemplateParams = TemplateParameterLists.get()[0];
+      if (TemplateParams->size()) {
+        // There is no such thing as a member field template.
+        Diag(D.getIdentifierLoc(), diag::err_template_member)
+            << II
+            << SourceRange(TemplateParams->getTemplateLoc(),
+                TemplateParams->getRAngleLoc());
+      } else {
+        // There is an extraneous 'template<>' for this member.
+        Diag(TemplateParams->getTemplateLoc(),
+            diag::err_template_member_noparams)
+            << II
+            << SourceRange(TemplateParams->getTemplateLoc(),
+                TemplateParams->getRAngleLoc());
+      }
+      return 0;
+    }
+
     if (SS.isSet() && !SS.isInvalid()) {
       // The user provided a superfluous scope specifier inside a class
       // definition:
@@ -1138,16 +1537,14 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
        
       SS.clear();
     }
-    
-    // FIXME: Check for template parameters!
-    // FIXME: Check that the name is an identifier!
+
     Member = HandleField(S, cast<CXXRecordDecl>(CurContext), Loc, D, BitWidth,
                          HasDeferredInit, AS);
     assert(Member && "HandleField never returns null");
   } else {
     assert(!HasDeferredInit);
 
-    Member = HandleDeclarator(S, D, move(TemplateParameterLists), IsDefinition);
+    Member = HandleDeclarator(S, D, move(TemplateParameterLists));
     if (!Member) {
       return 0;
     }
@@ -1214,28 +1611,15 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
 
   assert((Name || isInstField) && "No identifier for non-field ?");
 
-  if (Init)
-    AddInitializerToDecl(Member, Init, false,
-                         DS.getTypeSpecType() == DeclSpec::TST_auto);
-  else if (DS.getTypeSpecType() == DeclSpec::TST_auto &&
-           DS.getStorageClassSpec() == DeclSpec::SCS_static) {
-    // C++0x [dcl.spec.auto]p4: 'auto' can only be used in the type of a static
-    // data member if a brace-or-equal-initializer is provided.
-    Diag(Loc, diag::err_auto_var_requires_init)
-      << Name << cast<ValueDecl>(Member)->getType();
-    Member->setInvalidDecl();
-  }
-
-  FinalizeDeclaration(Member);
-
   if (isInstField)
     FieldCollector->Add(cast<FieldDecl>(Member));
   return Member;
 }
 
 /// ActOnCXXInClassMemberInitializer - This is invoked after parsing an
-/// in-class initializer for a non-static C++ class member. Such parsing
-/// is deferred until the class is complete.
+/// in-class initializer for a non-static C++ class member, and after
+/// instantiating an in-class initializer in a class template. Such actions
+/// are deferred until the class is complete.
 void
 Sema::ActOnCXXInClassMemberInitializer(Decl *D, SourceLocation EqualLoc,
                                        Expr *InitExpr) {
@@ -1319,7 +1703,21 @@ static bool FindBaseInitializer(Sema &SemaRef,
   return DirectBaseSpec || VirtualBaseSpec;
 }
 
-/// ActOnMemInitializer - Handle a C++ member initializer.
+/// \brief Handle a C++ member initializer using braced-init-list syntax.
+MemInitResult
+Sema::ActOnMemInitializer(Decl *ConstructorD,
+                          Scope *S,
+                          CXXScopeSpec &SS,
+                          IdentifierInfo *MemberOrBase,
+                          ParsedType TemplateTypeTy,
+                          SourceLocation IdLoc,
+                          Expr *InitList,
+                          SourceLocation EllipsisLoc) {
+  return BuildMemInitializer(ConstructorD, S, SS, MemberOrBase, TemplateTypeTy,
+                             IdLoc, MultiInitializer(InitList), EllipsisLoc);
+}
+
+/// \brief Handle a C++ member initializer using parentheses syntax.
 MemInitResult
 Sema::ActOnMemInitializer(Decl *ConstructorD,
                           Scope *S,
@@ -1328,9 +1726,25 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
                           ParsedType TemplateTypeTy,
                           SourceLocation IdLoc,
                           SourceLocation LParenLoc,
-                          ExprTy **Args, unsigned NumArgs,
+                          Expr **Args, unsigned NumArgs,
                           SourceLocation RParenLoc,
                           SourceLocation EllipsisLoc) {
+  return BuildMemInitializer(ConstructorD, S, SS, MemberOrBase, TemplateTypeTy,
+                             IdLoc, MultiInitializer(LParenLoc, Args, NumArgs,
+                                                     RParenLoc),
+                             EllipsisLoc);
+}
+
+/// \brief Handle a C++ member initializer.
+MemInitResult
+Sema::BuildMemInitializer(Decl *ConstructorD,
+                          Scope *S,
+                          CXXScopeSpec &SS,
+                          IdentifierInfo *MemberOrBase,
+                          ParsedType TemplateTypeTy,
+                          SourceLocation IdLoc,
+                          const MultiInitializer &Args,
+                          SourceLocation EllipsisLoc) {
   if (!ConstructorD)
     return true;
 
@@ -1365,26 +1779,23 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
       = ClassDecl->lookup(MemberOrBase);
     if (Result.first != Result.second) {
       Member = dyn_cast<FieldDecl>(*Result.first);
-    
+
       if (Member) {
         if (EllipsisLoc.isValid())
           Diag(EllipsisLoc, diag::err_pack_expansion_member_init)
-            << MemberOrBase << SourceRange(IdLoc, RParenLoc);
-        
-        return BuildMemberInitializer(Member, (Expr**)Args, NumArgs, IdLoc,
-                                    LParenLoc, RParenLoc);
+            << MemberOrBase << SourceRange(IdLoc, Args.getEndLoc());
+
+        return BuildMemberInitializer(Member, Args, IdLoc);
       }
-      
+
       // Handle anonymous union case.
       if (IndirectFieldDecl* IndirectField
             = dyn_cast<IndirectFieldDecl>(*Result.first)) {
         if (EllipsisLoc.isValid())
           Diag(EllipsisLoc, diag::err_pack_expansion_member_init)
-            << MemberOrBase << SourceRange(IdLoc, RParenLoc);
+            << MemberOrBase << SourceRange(IdLoc, Args.getEndLoc());
 
-         return BuildMemberInitializer(IndirectField, (Expr**)Args,
-                                       NumArgs, IdLoc,
-                                       LParenLoc, RParenLoc);
+         return BuildMemberInitializer(IndirectField, Args, IdLoc);
       }
     }
   }
@@ -1443,8 +1854,7 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
             Diag(Member->getLocation(), diag::note_previous_decl)
               << CorrectedQuotedStr;
 
-            return BuildMemberInitializer(Member, (Expr**)Args, NumArgs, IdLoc,
-                                          LParenLoc, RParenLoc);
+            return BuildMemberInitializer(Member, Args, IdLoc);
           }
         } else if (TypeDecl *Type = Corr.getCorrectionDeclAs<TypeDecl>()) {
           const CXXBaseSpecifier *DirectBaseSpec;
@@ -1473,7 +1883,7 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
 
       if (!TyD && BaseType.isNull()) {
         Diag(IdLoc, diag::err_mem_init_not_member_or_class)
-          << MemberOrBase << SourceRange(IdLoc, RParenLoc);
+          << MemberOrBase << SourceRange(IdLoc, Args.getEndLoc());
         return true;
       }
     }
@@ -1493,8 +1903,62 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
   if (!TInfo)
     TInfo = Context.getTrivialTypeSourceInfo(BaseType, IdLoc);
 
-  return BuildBaseInitializer(BaseType, TInfo, (Expr **)Args, NumArgs, 
-                              LParenLoc, RParenLoc, ClassDecl, EllipsisLoc);
+  return BuildBaseInitializer(BaseType, TInfo, Args, ClassDecl, EllipsisLoc);
+}
+
+/// Checks a member initializer expression for cases where reference (or
+/// pointer) members are bound to by-value parameters (or their addresses).
+static void CheckForDanglingReferenceOrPointer(Sema &S, ValueDecl *Member,
+                                               Expr *Init,
+                                               SourceLocation IdLoc) {
+  QualType MemberTy = Member->getType();
+
+  // We only handle pointers and references currently.
+  // FIXME: Would this be relevant for ObjC object pointers? Or block pointers?
+  if (!MemberTy->isReferenceType() && !MemberTy->isPointerType())
+    return;
+
+  const bool IsPointer = MemberTy->isPointerType();
+  if (IsPointer) {
+    if (const UnaryOperator *Op
+          = dyn_cast<UnaryOperator>(Init->IgnoreParenImpCasts())) {
+      // The only case we're worried about with pointers requires taking the
+      // address.
+      if (Op->getOpcode() != UO_AddrOf)
+        return;
+
+      Init = Op->getSubExpr();
+    } else {
+      // We only handle address-of expression initializers for pointers.
+      return;
+    }
+  }
+
+  if (isa<MaterializeTemporaryExpr>(Init->IgnoreParens())) {
+    // Taking the address of a temporary will be diagnosed as a hard error.
+    if (IsPointer)
+      return;
+
+    S.Diag(Init->getExprLoc(), diag::warn_bind_ref_member_to_temporary)
+      << Member << Init->getSourceRange();
+  } else if (const DeclRefExpr *DRE
+               = dyn_cast<DeclRefExpr>(Init->IgnoreParens())) {
+    // We only warn when referring to a non-reference parameter declaration.
+    const ParmVarDecl *Parameter = dyn_cast<ParmVarDecl>(DRE->getDecl());
+    if (!Parameter || Parameter->getType()->isReferenceType())
+      return;
+
+    S.Diag(Init->getExprLoc(),
+           IsPointer ? diag::warn_init_ptr_member_to_parameter_addr
+                     : diag::warn_bind_ref_member_to_parameter)
+      << Member << Parameter << Init->getSourceRange();
+  } else {
+    // Other initializers are fine.
+    return;
+  }
+
+  S.Diag(Member->getLocation(), diag::note_ref_or_ptr_member_declared_here)
+    << (unsigned)IsPointer;
 }
 
 /// Checks an initializer expression for use of uninitialized fields, such as
@@ -1566,10 +2030,9 @@ static bool InitExprContainsUninitializedFields(const Stmt *S,
 }
 
 MemInitResult
-Sema::BuildMemberInitializer(ValueDecl *Member, Expr **Args,
-                             unsigned NumArgs, SourceLocation IdLoc,
-                             SourceLocation LParenLoc,
-                             SourceLocation RParenLoc) {
+Sema::BuildMemberInitializer(ValueDecl *Member,
+                             const MultiInitializer &Args,
+                             SourceLocation IdLoc) {
   FieldDecl *DirectMember = dyn_cast<FieldDecl>(Member);
   IndirectFieldDecl *IndirectMember = dyn_cast<IndirectFieldDecl>(Member);
   assert((DirectMember || IndirectMember) &&
@@ -1582,9 +2045,13 @@ Sema::BuildMemberInitializer(ValueDecl *Member, Expr **Args,
   //   foo(foo)
   // where foo is not also a parameter to the constructor.
   // TODO: implement -Wuninitialized and fold this into that framework.
-  for (unsigned i = 0; i < NumArgs; ++i) {
+  for (MultiInitializer::iterator I = Args.begin(), E = Args.end();
+       I != E; ++I) {
     SourceLocation L;
-    if (InitExprContainsUninitializedFields(Args[i], Member, &L)) {
+    Expr *Arg = *I;
+    if (DesignatedInitExpr *DIE = dyn_cast<DesignatedInitExpr>(Arg))
+      Arg = DIE->getInit();
+    if (InitExprContainsUninitializedFields(Arg, Member, &L)) {
       // FIXME: Return true in the case when other fields are used before being
       // uninitialized. For example, let this field be the i'th field. When
       // initializing the i'th field, throw a warning if any of the >= i'th
@@ -1595,17 +2062,13 @@ Sema::BuildMemberInitializer(ValueDecl *Member, Expr **Args,
     }
   }
 
-  bool HasDependentArg = false;
-  for (unsigned i = 0; i < NumArgs; i++)
-    HasDependentArg |= Args[i]->isTypeDependent();
+  bool HasDependentArg = Args.isTypeDependent();
 
   Expr *Init;
   if (Member->getType()->isDependentType() || HasDependentArg) {
     // Can't check initialization for a member of dependent type or when
     // any of the arguments are type-dependent expressions.
-    Init = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                       RParenLoc,
-                                       Member->getType().getNonReferenceType());
+    Init = Args.CreateInitExpr(Context,Member->getType().getNonReferenceType());
 
     DiscardCleanupsInEvaluationContext();
   } else {
@@ -1614,17 +2077,14 @@ Sema::BuildMemberInitializer(ValueDecl *Member, Expr **Args,
       DirectMember ? InitializedEntity::InitializeMember(DirectMember, 0)
                    : InitializedEntity::InitializeMember(IndirectMember, 0);
     InitializationKind Kind =
-      InitializationKind::CreateDirect(IdLoc, LParenLoc, RParenLoc);
+      InitializationKind::CreateDirect(IdLoc, Args.getStartLoc(),
+                                       Args.getEndLoc());
 
-    InitializationSequence InitSeq(*this, MemberEntity, Kind, Args, NumArgs);
-
-    ExprResult MemberInit =
-      InitSeq.Perform(*this, MemberEntity, Kind,
-                      MultiExprArg(*this, Args, NumArgs), 0);
+    ExprResult MemberInit = Args.PerformInit(*this, MemberEntity, Kind);
     if (MemberInit.isInvalid())
       return true;
 
-    CheckImplicitConversions(MemberInit.get(), LParenLoc);
+    CheckImplicitConversions(MemberInit.get(), Args.getStartLoc());
 
     // C++0x [class.base.init]p7:
     //   The initialization of each base and member constitutes a
@@ -1640,31 +2100,30 @@ Sema::BuildMemberInitializer(ValueDecl *Member, Expr **Args,
     // of the information that we have about the member
     // initializer. However, deconstructing the ASTs is a dicey process,
     // and this approach is far more likely to get the corner cases right.
-    if (CurContext->isDependentContext())
-      Init = new (Context) ParenListExpr(
-          Context, LParenLoc, Args, NumArgs, RParenLoc,
-          Member->getType().getNonReferenceType());
-    else
+    if (CurContext->isDependentContext()) {
+      Init = Args.CreateInitExpr(Context,
+                                 Member->getType().getNonReferenceType());
+    } else {
       Init = MemberInit.get();
+      CheckForDanglingReferenceOrPointer(*this, Member, Init, IdLoc);
+    }
   }
 
   if (DirectMember) {
     return new (Context) CXXCtorInitializer(Context, DirectMember,
-                                                    IdLoc, LParenLoc, Init,
-                                                    RParenLoc);
+                                                    IdLoc, Args.getStartLoc(),
+                                                    Init, Args.getEndLoc());
   } else {
     return new (Context) CXXCtorInitializer(Context, IndirectMember,
-                                                    IdLoc, LParenLoc, Init,
-                                                    RParenLoc);
+                                                    IdLoc, Args.getStartLoc(),
+                                                    Init, Args.getEndLoc());
   }
 }
 
 MemInitResult
 Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo,
-                                 Expr **Args, unsigned NumArgs,
+                                 const MultiInitializer &Args,
                                  SourceLocation NameLoc,
-                                 SourceLocation LParenLoc,
-                                 SourceLocation RParenLoc,
                                  CXXRecordDecl *ClassDecl) {
   SourceLocation Loc = TInfo->getTypeLoc().getLocalSourceRange().getBegin();
   if (!LangOpts.CPlusPlus0x)
@@ -1675,13 +2134,10 @@ Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo,
   InitializedEntity DelegationEntity = InitializedEntity::InitializeDelegation(
                                      QualType(ClassDecl->getTypeForDecl(), 0));
   InitializationKind Kind =
-    InitializationKind::CreateDirect(NameLoc, LParenLoc, RParenLoc);
+    InitializationKind::CreateDirect(NameLoc, Args.getStartLoc(),
+                                     Args.getEndLoc());
 
-  InitializationSequence InitSeq(*this, DelegationEntity, Kind, Args, NumArgs);
-
-  ExprResult DelegationInit =
-    InitSeq.Perform(*this, DelegationEntity, Kind,
-                    MultiExprArg(*this, Args, NumArgs), 0);
+  ExprResult DelegationInit = Args.PerformInit(*this, DelegationEntity, Kind);
   if (DelegationInit.isInvalid())
     return true;
 
@@ -1690,7 +2146,7 @@ Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo,
     = ConExpr->getConstructor();
   assert(Constructor && "Delegating constructor with no target?");
 
-  CheckImplicitConversions(DelegationInit.get(), LParenLoc);
+  CheckImplicitConversions(DelegationInit.get(), Args.getStartLoc());
 
   // C++0x [class.base.init]p7:
   //   The initialization of each base and member constitutes a
@@ -1700,24 +2156,22 @@ Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo,
     return true;
 
   assert(!CurContext->isDependentContext());
-  return new (Context) CXXCtorInitializer(Context, Loc, LParenLoc, Constructor,
+  return new (Context) CXXCtorInitializer(Context, Loc, Args.getStartLoc(),
+                                          Constructor,
                                           DelegationInit.takeAs<Expr>(),
-                                          RParenLoc);
+                                          Args.getEndLoc());
 }
 
 MemInitResult
 Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
-                           Expr **Args, unsigned NumArgs, 
-                           SourceLocation LParenLoc, SourceLocation RParenLoc, 
+                           const MultiInitializer &Args,
                            CXXRecordDecl *ClassDecl,
                            SourceLocation EllipsisLoc) {
-  bool HasDependentArg = false;
-  for (unsigned i = 0; i < NumArgs; i++)
-    HasDependentArg |= Args[i]->isTypeDependent();
+  bool HasDependentArg = Args.isTypeDependent();
 
   SourceLocation BaseLoc
     = BaseTInfo->getTypeLoc().getLocalSourceRange().getBegin();
-  
+
   if (!BaseType->isDependentType() && !BaseType->isRecordType())
     return Diag(BaseLoc, diag::err_base_init_does_not_name_class)
              << BaseType << BaseTInfo->getTypeLoc().getLocalSourceRange();
@@ -1734,28 +2188,26 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
     // This is a pack expansion.
     if (!BaseType->containsUnexpandedParameterPack())  {
       Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
-        << SourceRange(BaseLoc, RParenLoc);
-      
+        << SourceRange(BaseLoc, Args.getEndLoc());
+
       EllipsisLoc = SourceLocation();
     }
   } else {
     // Check for any unexpanded parameter packs.
     if (DiagnoseUnexpandedParameterPack(BaseLoc, BaseTInfo, UPPC_Initializer))
       return true;
-    
-    for (unsigned I = 0; I != NumArgs; ++I)
-      if (DiagnoseUnexpandedParameterPack(Args[I]))
-        return true;
+
+    if (Args.DiagnoseUnexpandedParameterPack(*this))
+      return true;
   }
-  
+
   // Check for direct and virtual base classes.
   const CXXBaseSpecifier *DirectBaseSpec = 0;
   const CXXBaseSpecifier *VirtualBaseSpec = 0;
   if (!Dependent) { 
     if (Context.hasSameUnqualifiedType(QualType(ClassDecl->getTypeForDecl(),0),
                                        BaseType))
-      return BuildDelegatingInitializer(BaseTInfo, Args, NumArgs, BaseLoc,
-                                        LParenLoc, RParenLoc, ClassDecl);
+      return BuildDelegatingInitializer(BaseTInfo, Args, BaseLoc, ClassDecl);
 
     FindBaseInitializer(*this, ClassDecl, BaseType, DirectBaseSpec, 
                         VirtualBaseSpec);
@@ -1782,18 +2234,14 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   if (Dependent) {
     // Can't check initialization for a base of dependent type or when
     // any of the arguments are type-dependent expressions.
-    ExprResult BaseInit
-      = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                          RParenLoc, BaseType));
+    Expr *BaseInit = Args.CreateInitExpr(Context, BaseType);
 
     DiscardCleanupsInEvaluationContext();
 
-    return new (Context) CXXCtorInitializer(Context, BaseTInfo, 
-                                                    /*IsVirtual=*/false,
-                                                    LParenLoc, 
-                                                    BaseInit.takeAs<Expr>(),
-                                                    RParenLoc,
-                                                    EllipsisLoc);
+    return new (Context) CXXCtorInitializer(Context, BaseTInfo,
+                                            /*IsVirtual=*/false,
+                                            Args.getStartLoc(), BaseInit,
+                                            Args.getEndLoc(), EllipsisLoc);
   }
 
   // C++ [base.class.init]p2:
@@ -1813,18 +2261,15 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   InitializedEntity BaseEntity =
     InitializedEntity::InitializeBase(Context, BaseSpec, VirtualBaseSpec);
   InitializationKind Kind = 
-    InitializationKind::CreateDirect(BaseLoc, LParenLoc, RParenLoc);
-  
-  InitializationSequence InitSeq(*this, BaseEntity, Kind, Args, NumArgs);
-  
-  ExprResult BaseInit =
-    InitSeq.Perform(*this, BaseEntity, Kind, 
-                    MultiExprArg(*this, Args, NumArgs), 0);
+    InitializationKind::CreateDirect(BaseLoc, Args.getStartLoc(),
+                                     Args.getEndLoc());
+
+  ExprResult BaseInit = Args.PerformInit(*this, BaseEntity, Kind);
   if (BaseInit.isInvalid())
     return true;
 
-  CheckImplicitConversions(BaseInit.get(), LParenLoc);
-  
+  CheckImplicitConversions(BaseInit.get(), Args.getStartLoc());
+
   // C++0x [class.base.init]p7:
   //   The initialization of each base and member constitutes a 
   //   full-expression.
@@ -1839,24 +2284,27 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   // of the information that we have about the base
   // initializer. However, deconstructing the ASTs is a dicey process,
   // and this approach is far more likely to get the corner cases right.
-  if (CurContext->isDependentContext()) {
-    ExprResult Init
-      = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                          RParenLoc, BaseType));
-    return new (Context) CXXCtorInitializer(Context, BaseTInfo,
-                                                    BaseSpec->isVirtual(),
-                                                    LParenLoc, 
-                                                    Init.takeAs<Expr>(),
-                                                    RParenLoc,
-                                                    EllipsisLoc);
-  }
+  if (CurContext->isDependentContext())
+    BaseInit = Owned(Args.CreateInitExpr(Context, BaseType));
 
   return new (Context) CXXCtorInitializer(Context, BaseTInfo,
-                                                  BaseSpec->isVirtual(),
-                                                  LParenLoc, 
-                                                  BaseInit.takeAs<Expr>(),
-                                                  RParenLoc,
-                                                  EllipsisLoc);
+                                          BaseSpec->isVirtual(),
+                                          Args.getStartLoc(),
+                                          BaseInit.takeAs<Expr>(),
+                                          Args.getEndLoc(), EllipsisLoc);
+}
+
+// Create a static_cast\<T&&>(expr).
+static Expr *CastForMoving(Sema &SemaRef, Expr *E) {
+  QualType ExprType = E->getType();
+  QualType TargetType = SemaRef.Context.getRValueReferenceType(ExprType);
+  SourceLocation ExprLoc = E->getLocStart();
+  TypeSourceInfo *TargetLoc = SemaRef.Context.getTrivialTypeSourceInfo(
+      TargetType, ExprLoc);
+
+  return SemaRef.BuildCXXNamedCast(ExprLoc, tok::kw_static_cast, TargetLoc, E,
+                                   SourceRange(ExprLoc, ExprLoc),
+                                   E->getSourceRange()).take();
 }
 
 /// ImplicitInitializerKind - How an implicit base or member initializer should
@@ -1889,7 +2337,9 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     break;
   }
 
+  case IIK_Move:
   case IIK_Copy: {
+    bool Moving = ImplicitInitKind == IIK_Move;
     ParmVarDecl *Param = Constructor->getParamDecl(0);
     QualType ParamType = Param->getType().getNonReferenceType();
     
@@ -1897,17 +2347,22 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
       DeclRefExpr::Create(SemaRef.Context, NestedNameSpecifierLoc(), Param, 
                           Constructor->getLocation(), ParamType,
                           VK_LValue, 0);
-    
+
     // Cast to the base class to avoid ambiguities.
     QualType ArgTy = 
       SemaRef.Context.getQualifiedType(BaseSpec->getType().getUnqualifiedType(), 
                                        ParamType.getQualifiers());
 
+    if (Moving) {
+      CopyCtorArg = CastForMoving(SemaRef, CopyCtorArg);
+    }
+
     CXXCastPath BasePath;
     BasePath.push_back(BaseSpec);
     CopyCtorArg = SemaRef.ImpCastExprToType(CopyCtorArg, ArgTy,
                                             CK_UncheckedDerivedToBase,
-                                            VK_LValue, &BasePath).take();
+                                            Moving ? VK_XValue : VK_LValue,
+                                            &BasePath).take();
 
     InitializationKind InitKind
       = InitializationKind::CreateDirect(Constructor->getLocation(),
@@ -1918,9 +2373,6 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
                                MultiExprArg(&CopyCtorArg, 1));
     break;
   }
-
-  case IIK_Move:
-    assert(false && "Unhandled initializer kind!");
   }
 
   BaseInit = SemaRef.MaybeCreateExprWithCleanups(BaseInit);
@@ -1940,36 +2392,46 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
   return false;
 }
 
+static bool RefersToRValueRef(Expr *MemRef) {
+  ValueDecl *Referenced = cast<MemberExpr>(MemRef)->getMemberDecl();
+  return Referenced->getType()->isRValueReferenceType();
+}
+
 static bool
 BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
                                ImplicitInitializerKind ImplicitInitKind,
-                               FieldDecl *Field,
+                               FieldDecl *Field, IndirectFieldDecl *Indirect,
                                CXXCtorInitializer *&CXXMemberInit) {
   if (Field->isInvalidDecl())
     return true;
 
   SourceLocation Loc = Constructor->getLocation();
 
-  if (ImplicitInitKind == IIK_Copy) {
+  if (ImplicitInitKind == IIK_Copy || ImplicitInitKind == IIK_Move) {
+    bool Moving = ImplicitInitKind == IIK_Move;
     ParmVarDecl *Param = Constructor->getParamDecl(0);
     QualType ParamType = Param->getType().getNonReferenceType();
 
     // Suppress copying zero-width bitfields.
-    if (const Expr *Width = Field->getBitWidth())
-      if (Width->EvaluateAsInt(SemaRef.Context) == 0)
-        return false;
+    if (Field->isBitField() && Field->getBitWidthValue(SemaRef.Context) == 0)
+      return false;
     
     Expr *MemberExprBase = 
       DeclRefExpr::Create(SemaRef.Context, NestedNameSpecifierLoc(), Param, 
                           Loc, ParamType, VK_LValue, 0);
 
+    if (Moving) {
+      MemberExprBase = CastForMoving(SemaRef, MemberExprBase);
+    }
+
     // Build a reference to this field within the parameter.
     CXXScopeSpec SS;
     LookupResult MemberLookup(SemaRef, Field->getDeclName(), Loc,
                               Sema::LookupMemberName);
-    MemberLookup.addDecl(Field, AS_public);
+    MemberLookup.addDecl(Indirect ? cast<ValueDecl>(Indirect)
+                                  : cast<ValueDecl>(Field), AS_public);
     MemberLookup.resolveKind();
-    ExprResult CopyCtorArg 
+    ExprResult CtorArg 
       = SemaRef.BuildMemberReferenceExpr(MemberExprBase,
                                          ParamType, Loc,
                                          /*IsArrow=*/false,
@@ -1977,18 +2439,27 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
                                          /*FirstQualifierInScope=*/0,
                                          MemberLookup,
                                          /*TemplateArgs=*/0);    
-    if (CopyCtorArg.isInvalid())
+    if (CtorArg.isInvalid())
       return true;
-    
+
+    // C++11 [class.copy]p15:
+    //   - if a member m has rvalue reference type T&&, it is direct-initialized
+    //     with static_cast<T&&>(x.m);
+    if (RefersToRValueRef(CtorArg.get())) {
+      CtorArg = CastForMoving(SemaRef, CtorArg.take());
+    }
+
     // When the field we are copying is an array, create index variables for 
     // each dimension of the array. We use these index variables to subscript
     // the source array, and other clients (e.g., CodeGen) will perform the
     // necessary iteration with these index variables.
-    llvm::SmallVector<VarDecl *, 4> IndexVariables;
+    SmallVector<VarDecl *, 4> IndexVariables;
     QualType BaseType = Field->getType();
     QualType SizeType = SemaRef.Context.getSizeType();
+    bool InitializingArray = false;
     while (const ConstantArrayType *Array
                           = SemaRef.Context.getAsConstantArrayType(BaseType)) {
+      InitializingArray = true;
       // Create the iteration variable for this array index.
       IdentifierInfo *IterationVarName = 0;
       {
@@ -2009,24 +2480,30 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
         = SemaRef.BuildDeclRefExpr(IterationVar, SizeType, VK_RValue, Loc);
       assert(!IterationVarRef.isInvalid() &&
              "Reference to invented variable cannot fail!");
-      
+
       // Subscript the array with this iteration variable.
-      CopyCtorArg = SemaRef.CreateBuiltinArraySubscriptExpr(CopyCtorArg.take(),
-                                                            Loc,
+      CtorArg = SemaRef.CreateBuiltinArraySubscriptExpr(CtorArg.take(), Loc,
                                                         IterationVarRef.take(),
-                                                            Loc);
-      if (CopyCtorArg.isInvalid())
+                                                        Loc);
+      if (CtorArg.isInvalid())
         return true;
-      
+
       BaseType = Array->getElementType();
     }
-    
+
+    // The array subscript expression is an lvalue, which is wrong for moving.
+    if (Moving && InitializingArray)
+      CtorArg = CastForMoving(SemaRef, CtorArg.take());
+
     // Construct the entity that we will be initializing. For an array, this
     // will be first element in the array, which may require several levels
     // of array-subscript entities. 
-    llvm::SmallVector<InitializedEntity, 4> Entities;
+    SmallVector<InitializedEntity, 4> Entities;
     Entities.reserve(1 + IndexVariables.size());
-    Entities.push_back(InitializedEntity::InitializeMember(Field));
+    if (Indirect)
+      Entities.push_back(InitializedEntity::InitializeMember(Indirect));
+    else
+      Entities.push_back(InitializedEntity::InitializeMember(Field));
     for (unsigned I = 0, N = IndexVariables.size(); I != N; ++I)
       Entities.push_back(InitializedEntity::InitializeElement(SemaRef.Context,
                                                               0,
@@ -2036,22 +2513,31 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     InitializationKind InitKind =
       InitializationKind::CreateDirect(Loc, SourceLocation(), SourceLocation());
     
-    Expr *CopyCtorArgE = CopyCtorArg.takeAs<Expr>();
+    Expr *CtorArgE = CtorArg.takeAs<Expr>();
     InitializationSequence InitSeq(SemaRef, Entities.back(), InitKind,
-                                   &CopyCtorArgE, 1);
+                                   &CtorArgE, 1);
     
     ExprResult MemberInit
       = InitSeq.Perform(SemaRef, Entities.back(), InitKind, 
-                        MultiExprArg(&CopyCtorArgE, 1));
+                        MultiExprArg(&CtorArgE, 1));
     MemberInit = SemaRef.MaybeCreateExprWithCleanups(MemberInit);
     if (MemberInit.isInvalid())
       return true;
 
-    CXXMemberInit
-      = CXXCtorInitializer::Create(SemaRef.Context, Field, Loc, Loc,
-                                           MemberInit.takeAs<Expr>(), Loc,
-                                           IndexVariables.data(),
-                                           IndexVariables.size());
+    if (Indirect) {
+      assert(IndexVariables.size() == 0 && 
+             "Indirect field improperly initialized");
+      CXXMemberInit
+        = new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context, Indirect, 
+                                                   Loc, Loc, 
+                                                   MemberInit.takeAs<Expr>(), 
+                                                   Loc);
+    } else
+      CXXMemberInit = CXXCtorInitializer::Create(SemaRef.Context, Field, Loc, 
+                                                 Loc, MemberInit.takeAs<Expr>(), 
+                                                 Loc,
+                                                 IndexVariables.data(),
+                                                 IndexVariables.size());
     return false;
   }
 
@@ -2061,7 +2547,9 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     SemaRef.Context.getBaseElementType(Field->getType());
   
   if (FieldBaseElementType->isRecordType()) {
-    InitializedEntity InitEntity = InitializedEntity::InitializeMember(Field);
+    InitializedEntity InitEntity 
+      = Indirect? InitializedEntity::InitializeMember(Indirect)
+                : InitializedEntity::InitializeMember(Field);
     InitializationKind InitKind = 
       InitializationKind::CreateDefault(Loc);
     
@@ -2073,11 +2561,17 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     if (MemberInit.isInvalid())
       return true;
     
-    CXXMemberInit =
-      new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context,
-                                                       Field, Loc, Loc,
-                                                       MemberInit.get(),
-                                                       Loc);
+    if (Indirect)
+      CXXMemberInit = new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context,
+                                                               Indirect, Loc, 
+                                                               Loc,
+                                                               MemberInit.get(),
+                                                               Loc);
+    else
+      CXXMemberInit = new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context,
+                                                               Field, Loc, Loc,
+                                                               MemberInit.get(),
+                                                               Loc);
     return false;
   }
 
@@ -2129,21 +2623,37 @@ struct BaseAndFieldInfo {
   bool AnyErrorsInInits;
   ImplicitInitializerKind IIK;
   llvm::DenseMap<const void *, CXXCtorInitializer*> AllBaseFields;
-  llvm::SmallVector<CXXCtorInitializer*, 8> AllToInit;
+  SmallVector<CXXCtorInitializer*, 8> AllToInit;
 
   BaseAndFieldInfo(Sema &S, CXXConstructorDecl *Ctor, bool ErrorsInInits)
     : S(S), Ctor(Ctor), AnyErrorsInInits(ErrorsInInits) {
-    // FIXME: Handle implicit move constructors.
-    if (Ctor->isImplicit() && Ctor->isCopyConstructor())
+    bool Generated = Ctor->isImplicit() || Ctor->isDefaulted();
+    if (Generated && Ctor->isCopyConstructor())
       IIK = IIK_Copy;
+    else if (Generated && Ctor->isMoveConstructor())
+      IIK = IIK_Move;
     else
       IIK = IIK_Default;
   }
 };
 }
 
+/// \brief Determine whether the given indirect field declaration is somewhere
+/// within an anonymous union.
+static bool isWithinAnonymousUnion(IndirectFieldDecl *F) {
+  for (IndirectFieldDecl::chain_iterator C = F->chain_begin(), 
+                                      CEnd = F->chain_end();
+       C != CEnd; ++C)
+    if (CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>((*C)->getDeclContext()))
+      if (Record->isUnion())
+        return true;
+        
+  return false;
+}
+
 static bool CollectFieldInitializer(Sema &SemaRef, BaseAndFieldInfo &Info,
-                                    FieldDecl *Top, FieldDecl *Field) {
+                                    FieldDecl *Field, 
+                                    IndirectFieldDecl *Indirect = 0) {
 
   // Overwhelmingly common case: we have a direct initializer for this field.
   if (CXXCtorInitializer *Init = Info.AllBaseFields.lookup(Field)) {
@@ -2155,53 +2665,26 @@ static bool CollectFieldInitializer(Sema &SemaRef, BaseAndFieldInfo &Info,
   // has a brace-or-equal-initializer, the entity is initialized as specified
   // in [dcl.init].
   if (Field->hasInClassInitializer()) {
-    Info.AllToInit.push_back(
-      new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context, Field,
-                                               SourceLocation(),
-                                               SourceLocation(), 0,
-                                               SourceLocation()));
+    CXXCtorInitializer *Init;
+    if (Indirect)
+      Init = new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context, Indirect,
+                                                      SourceLocation(),
+                                                      SourceLocation(), 0,
+                                                      SourceLocation());
+    else
+      Init = new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context, Field,
+                                                      SourceLocation(),
+                                                      SourceLocation(), 0,
+                                                      SourceLocation());
+    Info.AllToInit.push_back(Init);
     return false;
   }
 
-  if (Info.IIK == IIK_Default && Field->isAnonymousStructOrUnion()) {
-    const RecordType *FieldClassType = Field->getType()->getAs<RecordType>();
-    assert(FieldClassType && "anonymous struct/union without record type");
-    CXXRecordDecl *FieldClassDecl
-      = cast<CXXRecordDecl>(FieldClassType->getDecl());
-
-    // Even though union members never have non-trivial default
-    // constructions in C++03, we still build member initializers for aggregate
-    // record types which can be union members, and C++0x allows non-trivial
-    // default constructors for union members, so we ensure that only one
-    // member is initialized for these.
-    if (FieldClassDecl->isUnion()) {
-      // First check for an explicit initializer for one field.
-      for (RecordDecl::field_iterator FA = FieldClassDecl->field_begin(),
-           EA = FieldClassDecl->field_end(); FA != EA; FA++) {
-        if (CXXCtorInitializer *Init = Info.AllBaseFields.lookup(*FA)) {
-          Info.AllToInit.push_back(Init);
-
-          // Once we've initialized a field of an anonymous union, the union
-          // field in the class is also initialized, so exit immediately.
-          return false;
-        } else if ((*FA)->isAnonymousStructOrUnion()) {
-          if (CollectFieldInitializer(SemaRef, Info, Top, *FA))
-            return true;
-        }
-      }
-
-      // FIXME: C++0x unrestricted unions might call a default constructor here.
-      return false;
-    } else {
-      // For structs, we simply descend through to initialize all members where
-      // necessary.
-      for (RecordDecl::field_iterator FA = FieldClassDecl->field_begin(),
-           EA = FieldClassDecl->field_end(); FA != EA; FA++) {
-        if (CollectFieldInitializer(SemaRef, Info, Top, *FA))
-          return true;
-      }
-    }
-  }
+  // Don't build an implicit initializer for union members if none was
+  // explicitly specified.
+  if (Field->getParent()->isUnion() ||
+      (Indirect && isWithinAnonymousUnion(Indirect)))
+    return false;
 
   // Don't try to build an implicit initializer if there were semantic
   // errors in any of the initializers (and therefore we might be
@@ -2210,7 +2693,8 @@ static bool CollectFieldInitializer(Sema &SemaRef, BaseAndFieldInfo &Info,
     return false;
 
   CXXCtorInitializer *Init = 0;
-  if (BuildImplicitMemberInitializer(Info.S, Info.Ctor, Info.IIK, Field, Init))
+  if (BuildImplicitMemberInitializer(Info.S, Info.Ctor, Info.IIK, Field,
+                                     Indirect, Init))
     return true;
 
   if (Init)
@@ -2238,12 +2722,12 @@ Sema::SetDelegatingInitializer(CXXConstructorDecl *Constructor,
 
   return false;
 }
-                               
+
 bool Sema::SetCtorInitializers(CXXConstructorDecl *Constructor,
                                CXXCtorInitializer **Initializers,
                                unsigned NumInitializers,
                                bool AnyErrors) {
-  if (Constructor->getDeclContext()->isDependentContext()) {
+  if (Constructor->isDependentContext()) {
     // Just store the initializers as written, they will be checked during
     // instantiation.
     if (NumInitializers > 0) {
@@ -2330,15 +2814,51 @@ bool Sema::SetCtorInitializers(CXXConstructorDecl *Constructor,
   }
 
   // Fields.
-  for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
-       E = ClassDecl->field_end(); Field != E; ++Field) {
-    if ((*Field)->getType()->isIncompleteArrayType()) {
-      assert(ClassDecl->hasFlexibleArrayMember() &&
-             "Incomplete array type is not valid");
+  for (DeclContext::decl_iterator Mem = ClassDecl->decls_begin(),
+                               MemEnd = ClassDecl->decls_end();
+       Mem != MemEnd; ++Mem) {
+    if (FieldDecl *F = dyn_cast<FieldDecl>(*Mem)) {
+      // C++ [class.bit]p2:
+      //   A declaration for a bit-field that omits the identifier declares an
+      //   unnamed bit-field. Unnamed bit-fields are not members and cannot be
+      //   initialized.
+      if (F->isUnnamedBitfield())
+        continue;
+      
+      if (F->getType()->isIncompleteArrayType()) {
+        assert(ClassDecl->hasFlexibleArrayMember() &&
+               "Incomplete array type is not valid");
+        continue;
+      }
+      
+      // If we're not generating the implicit copy/move constructor, then we'll
+      // handle anonymous struct/union fields based on their individual
+      // indirect fields.
+      if (F->isAnonymousStructOrUnion() && Info.IIK == IIK_Default)
+        continue;
+          
+      if (CollectFieldInitializer(*this, Info, F))
+        HadError = true;
       continue;
     }
-    if (CollectFieldInitializer(*this, Info, *Field, *Field))
-      HadError = true;
+    
+    // Beyond this point, we only consider default initialization.
+    if (Info.IIK != IIK_Default)
+      continue;
+    
+    if (IndirectFieldDecl *F = dyn_cast<IndirectFieldDecl>(*Mem)) {
+      if (F->getType()->isIncompleteArrayType()) {
+        assert(ClassDecl->hasFlexibleArrayMember() &&
+               "Incomplete array type is not valid");
+        continue;
+      }
+      
+      // Initialize each field of an anonymous struct individually.
+      if (CollectFieldInitializer(*this, Info, F->getAnonField(), F))
+        HadError = true;
+      
+      continue;        
+    }
   }
 
   NumInitializers = Info.AllToInit.size();
@@ -2414,7 +2934,7 @@ DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
     CXXCtorInitializer *Init = Inits[InitIndex];
     if (SemaRef.Diags.getDiagnosticLevel(diag::warn_initializer_out_of_order,
                                          Init->getSourceLocation())
-          != Diagnostic::Ignored) {
+          != DiagnosticsEngine::Ignored) {
       ShouldCheckOrder = true;
       break;
     }
@@ -2425,7 +2945,7 @@ DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
   // Build the list of bases and members in the order that they'll
   // actually be initialized.  The explicit initializers should be in
   // this same order but may be missing things.
-  llvm::SmallVector<const void*, 32> IdealInitKeys;
+  SmallVector<const void*, 32> IdealInitKeys;
 
   const CXXRecordDecl *ClassDecl = Constructor->getParent();
 
@@ -2445,9 +2965,13 @@ DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
 
   // 3. Direct fields.
   for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
-       E = ClassDecl->field_end(); Field != E; ++Field)
+       E = ClassDecl->field_end(); Field != E; ++Field) {
+    if (Field->isUnnamedBitfield())
+      continue;
+    
     IdealInitKeys.push_back(GetKeyForTopLevelField(*Field));
-
+  }
+  
   unsigned NumIdealInits = IdealInitKeys.size();
   unsigned IdealIndex = 0;
 
@@ -2561,7 +3085,8 @@ bool CheckRedundantUnionInit(Sema &S,
 /// ActOnMemInitializers - Handle the member initializers for a constructor.
 void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
                                 SourceLocation ColonLoc,
-                                MemInitTy **meminits, unsigned NumMemInits,
+                                CXXCtorInitializer **meminits,
+                                unsigned NumMemInits,
                                 bool AnyErrors) {
   if (!ConstructorDecl)
     return;
@@ -2630,8 +3155,9 @@ void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
 void
 Sema::MarkBaseAndMemberDestructorsReferenced(SourceLocation Location,
                                              CXXRecordDecl *ClassDecl) {
-  // Ignore dependent contexts.
-  if (ClassDecl->isDependentContext())
+  // Ignore dependent contexts. Also ignore unions, since their members never
+  // have destructors implicitly called.
+  if (ClassDecl->isDependentContext() || ClassDecl->isUnion())
     return;
 
   // FIXME: all the access-control diagnostics are positioned on the
@@ -2903,6 +3429,7 @@ struct CheckAbstractUsage {
   CheckPolymorphic(ReferenceTypeLoc)
   CheckPolymorphic(MemberPointerTypeLoc)
   CheckPolymorphic(BlockPointerTypeLoc)
+  CheckPolymorphic(AtomicTypeLoc)
 
   /// Handle all the types we haven't given a more specific
   /// implementation for above.
@@ -3016,7 +3543,7 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
     for (RecordDecl::field_iterator F = Record->field_begin(), 
                                  FEnd = Record->field_end();
          F != FEnd; ++F) {
-      if (F->hasInClassInitializer())
+      if (F->hasInClassInitializer() || F->isUnnamedBitfield())
         continue;
 
       if (F->getType()->isReferenceType() ||
@@ -3077,6 +3604,47 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
     }
   }
 
+  // C++0x [dcl.constexpr]p8: A constexpr specifier for a non-static member
+  // function that is not a constructor declares that member function to be
+  // const. [...] The class of which that function is a member shall be
+  // a literal type.
+  //
+  // It's fine to diagnose constructors here too: such constructors cannot
+  // produce a constant expression, so are ill-formed (no diagnostic required).
+  //
+  // If the class has virtual bases, any constexpr members will already have
+  // been diagnosed by the checks performed on the member declaration, so
+  // suppress this (less useful) diagnostic.
+  if (LangOpts.CPlusPlus0x && !Record->isDependentType() &&
+      !Record->isLiteral() && !Record->getNumVBases()) {
+    for (CXXRecordDecl::method_iterator M = Record->method_begin(),
+                                     MEnd = Record->method_end();
+         M != MEnd; ++M) {
+      if ((*M)->isConstexpr()) {
+        switch (Record->getTemplateSpecializationKind()) {
+        case TSK_ImplicitInstantiation:
+        case TSK_ExplicitInstantiationDeclaration:
+        case TSK_ExplicitInstantiationDefinition:
+          // If a template instantiates to a non-literal type, but its members
+          // instantiate to constexpr functions, the template is technically
+          // ill-formed, but we allow it for sanity. Such members are treated as
+          // non-constexpr.
+          (*M)->setConstexpr(false);
+          continue;
+
+        case TSK_Undeclared:
+        case TSK_ExplicitSpecialization:
+          RequireLiteralType((*M)->getLocation(), Context.getRecordType(Record),
+                             PDiag(diag::err_constexpr_method_non_literal));
+          break;
+        }
+
+        // Only produce one error per class.
+        break;
+      }
+    }
+  }
+
   // Declare inherited constructors. We do this eagerly here because:
   // - The standard requires an eager diagnostic for conflicting inherited
   //   constructors from different classes.
@@ -3114,12 +3682,14 @@ void Sema::CheckExplicitlyDefaultedMethods(CXXRecordDecl *Record) {
         break;
 
       case CXXMoveConstructor:
-      case CXXMoveAssignment:
-        Diag(MI->getLocation(), diag::err_defaulted_move_unsupported);
+        CheckExplicitlyDefaultedMoveConstructor(cast<CXXConstructorDecl>(*MI));
         break;
 
-      default:
-        // FIXME: Do moves once they exist
+      case CXXMoveAssignment:
+        CheckExplicitlyDefaultedMoveAssignment(*MI);
+        break;
+
+      case CXXInvalid:
         llvm_unreachable("non-special member explicitly defaulted!");
       }
     }
@@ -3176,7 +3746,7 @@ void Sema::CheckExplicitlyDefaultedDefaultConstructor(CXXConstructorDecl *CD) {
     return;
   }
 
-  if (ShouldDeleteDefaultConstructor(CD)) {
+  if (ShouldDeleteSpecialMember(CD, CXXDefaultConstructor)) {
     if (First) {
       CD->setDeletedAsWritten();
     } else {
@@ -3243,7 +3813,7 @@ void Sema::CheckExplicitlyDefaultedCopyConstructor(CXXConstructorDecl *CD) {
     return;
   }
 
-  if (ShouldDeleteCopyConstructor(CD)) {
+  if (ShouldDeleteSpecialMember(CD, CXXCopyConstructor)) {
     if (First) {
       CD->setDeletedAsWritten();
     } else {
@@ -3288,7 +3858,7 @@ void Sema::CheckExplicitlyDefaultedCopyAssignment(CXXMethodDecl *MD) {
                          Context.VoidTy, 0, 0, EPI)->getAs<FunctionProtoType>();
 
   QualType ArgType = OperType->getArgType(0);
-  if (!ArgType->isReferenceType()) {
+  if (!ArgType->isLValueReferenceType()) {
     Diag(MD->getLocation(), diag::err_defaulted_copy_assign_not_ref);
     HadError = true;
   } else {
@@ -3340,6 +3910,155 @@ void Sema::CheckExplicitlyDefaultedCopyAssignment(CXXMethodDecl *MD) {
   }
 }
 
+void Sema::CheckExplicitlyDefaultedMoveConstructor(CXXConstructorDecl *CD) {
+  assert(CD->isExplicitlyDefaulted() && CD->isMoveConstructor());
+
+  // Whether this was the first-declared instance of the constructor.
+  bool First = CD == CD->getCanonicalDecl();
+
+  bool HadError = false;
+  if (CD->getNumParams() != 1) {
+    Diag(CD->getLocation(), diag::err_defaulted_move_ctor_params)
+      << CD->getSourceRange();
+    HadError = true;
+  }
+
+  ImplicitExceptionSpecification Spec(
+      ComputeDefaultedMoveCtorExceptionSpec(CD->getParent()));
+
+  FunctionProtoType::ExtProtoInfo EPI = Spec.getEPI();
+  const FunctionProtoType *CtorType = CD->getType()->getAs<FunctionProtoType>(),
+                          *ExceptionType = Context.getFunctionType(
+                         Context.VoidTy, 0, 0, EPI)->getAs<FunctionProtoType>();
+
+  // Check for parameter type matching.
+  // This is a move ctor so we know it's a cv-qualified rvalue reference to T.
+  QualType ArgType = CtorType->getArgType(0);
+  if (ArgType->getPointeeType().isVolatileQualified()) {
+    Diag(CD->getLocation(), diag::err_defaulted_move_ctor_volatile_param);
+    HadError = true;
+  }
+  if (ArgType->getPointeeType().isConstQualified()) {
+    Diag(CD->getLocation(), diag::err_defaulted_move_ctor_const_param);
+    HadError = true;
+  }
+
+  if (CtorType->hasExceptionSpec()) {
+    if (CheckEquivalentExceptionSpec(
+          PDiag(diag::err_incorrect_defaulted_exception_spec)
+            << CXXMoveConstructor,
+          PDiag(),
+          ExceptionType, SourceLocation(),
+          CtorType, CD->getLocation())) {
+      HadError = true;
+    }
+  } else if (First) {
+    // We set the declaration to have the computed exception spec here.
+    // We duplicate the one parameter type.
+    EPI.ExtInfo = CtorType->getExtInfo();
+    CD->setType(Context.getFunctionType(Context.VoidTy, &ArgType, 1, EPI));
+  }
+
+  if (HadError) {
+    CD->setInvalidDecl();
+    return;
+  }
+
+  if (ShouldDeleteSpecialMember(CD, CXXMoveConstructor)) {
+    if (First) {
+      CD->setDeletedAsWritten();
+    } else {
+      Diag(CD->getLocation(), diag::err_out_of_line_default_deletes)
+        << CXXMoveConstructor;
+      CD->setInvalidDecl();
+    }
+  }
+}
+
+void Sema::CheckExplicitlyDefaultedMoveAssignment(CXXMethodDecl *MD) {
+  assert(MD->isExplicitlyDefaulted());
+
+  // Whether this was the first-declared instance of the operator
+  bool First = MD == MD->getCanonicalDecl();
+
+  bool HadError = false;
+  if (MD->getNumParams() != 1) {
+    Diag(MD->getLocation(), diag::err_defaulted_move_assign_params)
+      << MD->getSourceRange();
+    HadError = true;
+  }
+
+  QualType ReturnType =
+    MD->getType()->getAs<FunctionType>()->getResultType();
+  if (!ReturnType->isLValueReferenceType() ||
+      !Context.hasSameType(
+        Context.getCanonicalType(ReturnType->getPointeeType()),
+        Context.getCanonicalType(Context.getTypeDeclType(MD->getParent())))) {
+    Diag(MD->getLocation(), diag::err_defaulted_move_assign_return_type);
+    HadError = true;
+  }
+
+  ImplicitExceptionSpecification Spec(
+      ComputeDefaultedMoveCtorExceptionSpec(MD->getParent()));
+  
+  FunctionProtoType::ExtProtoInfo EPI = Spec.getEPI();
+  const FunctionProtoType *OperType = MD->getType()->getAs<FunctionProtoType>(),
+                          *ExceptionType = Context.getFunctionType(
+                         Context.VoidTy, 0, 0, EPI)->getAs<FunctionProtoType>();
+
+  QualType ArgType = OperType->getArgType(0);
+  if (!ArgType->isRValueReferenceType()) {
+    Diag(MD->getLocation(), diag::err_defaulted_move_assign_not_ref);
+    HadError = true;
+  } else {
+    if (ArgType->getPointeeType().isVolatileQualified()) {
+      Diag(MD->getLocation(), diag::err_defaulted_move_assign_volatile_param);
+      HadError = true;
+    }
+    if (ArgType->getPointeeType().isConstQualified()) {
+      Diag(MD->getLocation(), diag::err_defaulted_move_assign_const_param);
+      HadError = true;
+    }
+  }
+
+  if (OperType->getTypeQuals()) {
+    Diag(MD->getLocation(), diag::err_defaulted_move_assign_quals);
+    HadError = true;
+  }
+
+  if (OperType->hasExceptionSpec()) {
+    if (CheckEquivalentExceptionSpec(
+          PDiag(diag::err_incorrect_defaulted_exception_spec)
+            << CXXMoveAssignment,
+          PDiag(),
+          ExceptionType, SourceLocation(),
+          OperType, MD->getLocation())) {
+      HadError = true;
+    }
+  } else if (First) {
+    // We set the declaration to have the computed exception spec here.
+    // We duplicate the one parameter type.
+    EPI.RefQualifier = OperType->getRefQualifier();
+    EPI.ExtInfo = OperType->getExtInfo();
+    MD->setType(Context.getFunctionType(ReturnType, &ArgType, 1, EPI));
+  }
+
+  if (HadError) {
+    MD->setInvalidDecl();
+    return;
+  }
+
+  if (ShouldDeleteMoveAssignmentOperator(MD)) {
+    if (First) {
+      MD->setDeletedAsWritten();
+    } else {
+      Diag(MD->getLocation(), diag::err_out_of_line_default_deletes)
+        << CXXMoveAssignment;
+      MD->setInvalidDecl();
+    }
+  }
+}
+
 void Sema::CheckExplicitlyDefaultedDestructor(CXXDestructorDecl *DD) {
   assert(DD->isExplicitlyDefaulted());
 
@@ -3381,30 +4100,53 @@ void Sema::CheckExplicitlyDefaultedDestructor(CXXDestructorDecl *DD) {
   }
 }
 
-bool Sema::ShouldDeleteDefaultConstructor(CXXConstructorDecl *CD) {
-  CXXRecordDecl *RD = CD->getParent();
+/// This function implements the following C++0x paragraphs:
+///  - [class.ctor]/5
+///  - [class.copy]/11
+bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM) {
+  assert(!MD->isInvalidDecl());
+  CXXRecordDecl *RD = MD->getParent();
   assert(!RD->isDependentType() && "do deletion after instantiation");
   if (!LangOpts.CPlusPlus0x || RD->isInvalidDecl())
     return false;
 
-  SourceLocation Loc = CD->getLocation();
+  bool IsUnion = RD->isUnion();
+  bool IsConstructor = false;
+  bool IsAssignment = false;
+  bool IsMove = false;
+  
+  bool ConstArg = false;
 
-  // Do access control from the constructor
-  ContextRAII CtorContext(*this, CD);
+  switch (CSM) {
+  case CXXDefaultConstructor:
+    IsConstructor = true;
+    break;
+  case CXXCopyConstructor:
+    IsConstructor = true;
+    ConstArg = MD->getParamDecl(0)->getType().isConstQualified();
+    break;
+  case CXXMoveConstructor:
+    IsConstructor = true;
+    IsMove = true;
+    break;
+  default:
+    llvm_unreachable("function only currently implemented for default ctors");
+  }
+
+  SourceLocation Loc = MD->getLocation();
+
+  // Do access control from the special member function
+  ContextRAII MethodContext(*this, MD);
 
-  bool Union = RD->isUnion();
   bool AllConst = true;
 
   // We do this because we should never actually use an anonymous
   // union's constructor.
-  if (Union && RD->isAnonymousStructOrUnion())
+  if (IsUnion && RD->isAnonymousStructOrUnion())
     return false;
 
   // FIXME: We should put some diagnostic logic right into this function.
 
-  // C++0x [class.ctor]/5
-  //    A defaulted default constructor for class X is defined as deleted if:
-
   for (CXXRecordDecl::base_class_iterator BI = RD->bases_begin(),
                                           BE = RD->bases_end();
        BI != BE; ++BI) {
@@ -3415,26 +4157,41 @@ bool Sema::ShouldDeleteDefaultConstructor(CXXConstructorDecl *CD) {
     CXXRecordDecl *BaseDecl = BI->getType()->getAsCXXRecordDecl();
     assert(BaseDecl && "base isn't a CXXRecordDecl");
 
-    // -- any [direct base class] has a type with a destructor that is
-    //    deleted or inaccessible from the defaulted default constructor
-    CXXDestructorDecl *BaseDtor = LookupDestructor(BaseDecl);
-    if (BaseDtor->isDeleted())
-      return true;
-    if (CheckDestructorAccess(Loc, BaseDtor, PDiag()) !=
-        AR_accessible)
-      return true;
+    // Unless we have an assignment operator, the base's destructor must
+    // be accessible and not deleted.
+    if (!IsAssignment) {
+      CXXDestructorDecl *BaseDtor = LookupDestructor(BaseDecl);
+      if (BaseDtor->isDeleted())
+        return true;
+      if (CheckDestructorAccess(Loc, BaseDtor, PDiag()) !=
+          AR_accessible)
+        return true;
+    }
 
-    // -- any [direct base class either] has no default constructor or
-    //    overload resolution as applied to [its] default constructor
-    //    results in an ambiguity or in a function that is deleted or
-    //    inaccessible from the defaulted default constructor
-    CXXConstructorDecl *BaseDefault = LookupDefaultConstructor(BaseDecl);
-    if (!BaseDefault || BaseDefault->isDeleted())
-      return true;
+    // Finding the corresponding member in the base should lead to a
+    // unique, accessible, non-deleted function. If we are doing
+    // a destructor, we have already checked this case.
+    if (CSM != CXXDestructor) {
+      SpecialMemberOverloadResult *SMOR =
+        LookupSpecialMember(BaseDecl, CSM, ConstArg, false, false, false,
+                            false);
+      if (!SMOR->hasSuccess())
+        return true;
+      CXXMethodDecl *BaseMember = SMOR->getMethod();
+      if (IsConstructor) {
+        CXXConstructorDecl *BaseCtor = cast<CXXConstructorDecl>(BaseMember);
+        if (CheckConstructorAccess(Loc, BaseCtor, BaseCtor->getAccess(),
+                                   PDiag()) != AR_accessible)
+          return true;
 
-    if (CheckConstructorAccess(Loc, BaseDefault, BaseDefault->getAccess(),
-                               PDiag()) != AR_accessible)
-      return true;
+        // For a move operation, the corresponding operation must actually
+        // be a move operation (and not a copy selected by overload
+        // resolution) unless we are working on a trivially copyable class.
+        if (IsMove && !BaseCtor->isMoveConstructor() &&
+            !BaseDecl->isTriviallyCopyable())
+          return true;
+      }
+    }
   }
 
   for (CXXRecordDecl::base_class_iterator BI = RD->vbases_begin(),
@@ -3443,69 +4200,76 @@ bool Sema::ShouldDeleteDefaultConstructor(CXXConstructorDecl *CD) {
     CXXRecordDecl *BaseDecl = BI->getType()->getAsCXXRecordDecl();
     assert(BaseDecl && "base isn't a CXXRecordDecl");
 
-    // -- any [virtual base class] has a type with a destructor that is
-    //    delete or inaccessible from the defaulted default constructor
-    CXXDestructorDecl *BaseDtor = LookupDestructor(BaseDecl);
-    if (BaseDtor->isDeleted())
-      return true;
-    if (CheckDestructorAccess(Loc, BaseDtor, PDiag()) !=
-        AR_accessible)
-      return true;
+    // Unless we have an assignment operator, the base's destructor must
+    // be accessible and not deleted.
+    if (!IsAssignment) {
+      CXXDestructorDecl *BaseDtor = LookupDestructor(BaseDecl);
+      if (BaseDtor->isDeleted())
+        return true;
+      if (CheckDestructorAccess(Loc, BaseDtor, PDiag()) !=
+          AR_accessible)
+        return true;
+    }
 
-    // -- any [virtual base class either] has no default constructor or
-    //    overload resolution as applied to [its] default constructor
-    //    results in an ambiguity or in a function that is deleted or
-    //    inaccessible from the defaulted default constructor
-    CXXConstructorDecl *BaseDefault = LookupDefaultConstructor(BaseDecl);
-    if (!BaseDefault || BaseDefault->isDeleted())
-      return true;
+    // Finding the corresponding member in the base should lead to a
+    // unique, accessible, non-deleted function.
+    if (CSM != CXXDestructor) {
+      SpecialMemberOverloadResult *SMOR =
+        LookupSpecialMember(BaseDecl, CSM, ConstArg, false, false, false,
+                            false);
+      if (!SMOR->hasSuccess())
+        return true;
+      CXXMethodDecl *BaseMember = SMOR->getMethod();
+      if (IsConstructor) {
+        CXXConstructorDecl *BaseCtor = cast<CXXConstructorDecl>(BaseMember);
+        if (CheckConstructorAccess(Loc, BaseCtor, BaseCtor->getAccess(),
+                                   PDiag()) != AR_accessible)
+          return true;
 
-    if (CheckConstructorAccess(Loc, BaseDefault, BaseDefault->getAccess(),
-                               PDiag()) != AR_accessible)
-      return true;
+        // For a move operation, the corresponding operation must actually
+        // be a move operation (and not a copy selected by overload
+        // resolution) unless we are working on a trivially copyable class.
+        if (IsMove && !BaseCtor->isMoveConstructor() &&
+            !BaseDecl->isTriviallyCopyable())
+          return true;
+      }
+    }
   }
 
   for (CXXRecordDecl::field_iterator FI = RD->field_begin(),
                                      FE = RD->field_end();
        FI != FE; ++FI) {
-    if (FI->isInvalidDecl())
+    if (FI->isInvalidDecl() || FI->isUnnamedBitfield())
       continue;
     
     QualType FieldType = Context.getBaseElementType(FI->getType());
     CXXRecordDecl *FieldRecord = FieldType->getAsCXXRecordDecl();
 
-    // -- any non-static data member with no brace-or-equal-initializer is of
-    //    reference type
-    if (FieldType->isReferenceType() && !FI->hasInClassInitializer())
-      return true;
+    // For a default constructor, all references must be initialized in-class
+    // and, if a union, it must have a non-const member.
+    if (CSM == CXXDefaultConstructor) {
+      if (FieldType->isReferenceType() && !FI->hasInClassInitializer())
+        return true;
 
-    // -- X is a union and all its variant members are of const-qualified type
-    //    (or array thereof)
-    if (Union && !FieldType.isConstQualified())
-      AllConst = false;
+      if (IsUnion && !FieldType.isConstQualified())
+        AllConst = false;
+    // For a copy constructor, data members must not be of rvalue reference
+    // type.
+    } else if (CSM == CXXCopyConstructor) {
+      if (FieldType->isRValueReferenceType())
+        return true;
+    }
 
     if (FieldRecord) {
-      // -- X is a union-like class that has a variant member with a non-trivial
-      //    default constructor
-      if (Union && !FieldRecord->hasTrivialDefaultConstructor())
-        return true;
-
-      CXXDestructorDecl *FieldDtor = LookupDestructor(FieldRecord);
-      if (FieldDtor->isDeleted())
-        return true;
-      if (CheckDestructorAccess(Loc, FieldDtor, PDiag()) !=
-          AR_accessible)
-        return true;
-
-      // -- any non-variant non-static data member of const-qualified type (or
-      //    array thereof) with no brace-or-equal-initializer does not have a
-      //    user-provided default constructor
-      if (FieldType.isConstQualified() &&
+      // For a default constructor, a const member must have a user-provided
+      // default constructor or else be explicitly initialized.
+      if (CSM == CXXDefaultConstructor && FieldType.isConstQualified() &&
           !FI->hasInClassInitializer() &&
           !FieldRecord->hasUserProvidedDefaultConstructor())
         return true;
  
-      if (!Union && FieldRecord->isUnion() &&
+      // Some additional restrictions exist on the variant members.
+      if (!IsUnion && FieldRecord->isUnion() &&
           FieldRecord->isAnonymousStructOrUnion()) {
         // We're okay to reuse AllConst here since we only care about the
         // value otherwise if we're in a union.
@@ -3521,12 +4285,49 @@ bool Sema::ShouldDeleteDefaultConstructor(CXXConstructorDecl *CD) {
           if (!UnionFieldType.isConstQualified())
             AllConst = false;
 
-          if (UnionFieldRecord &&
-              !UnionFieldRecord->hasTrivialDefaultConstructor())
-            return true;
+          if (UnionFieldRecord) {
+            // FIXME: Checking for accessibility and validity of this
+            //        destructor is technically going beyond the
+            //        standard, but this is believed to be a defect.
+            if (!IsAssignment) {
+              CXXDestructorDecl *FieldDtor = LookupDestructor(UnionFieldRecord);
+              if (FieldDtor->isDeleted())
+                return true;
+              if (CheckDestructorAccess(Loc, FieldDtor, PDiag()) !=
+                  AR_accessible)
+                return true;
+              if (!FieldDtor->isTrivial())
+                return true;
+            }
+
+            if (CSM != CXXDestructor) {
+              SpecialMemberOverloadResult *SMOR =
+                LookupSpecialMember(UnionFieldRecord, CSM, ConstArg, false,
+                                    false, false, false);
+              // FIXME: Checking for accessibility and validity of this
+              //        corresponding member is technically going beyond the
+              //        standard, but this is believed to be a defect.
+              if (!SMOR->hasSuccess())
+                return true;
+
+              CXXMethodDecl *FieldMember = SMOR->getMethod();
+              // A member of a union must have a trivial corresponding
+              // constructor.
+              if (!FieldMember->isTrivial())
+                return true;
+
+              if (IsConstructor) {
+                CXXConstructorDecl *FieldCtor = cast<CXXConstructorDecl>(FieldMember);
+                if (CheckConstructorAccess(Loc, FieldCtor, FieldCtor->getAccess(),
+                                           PDiag()) != AR_accessible)
+                return true;
+              }
+            }
+          }
         }
 
-        if (AllConst)
+        // At least one member in each anonymous union must be non-const
+        if (CSM == CXXDefaultConstructor && AllConst)
           return true;
 
         // Don't try to initialize the anonymous union
@@ -3534,165 +4335,9 @@ bool Sema::ShouldDeleteDefaultConstructor(CXXConstructorDecl *CD) {
         continue;
       }
 
-      // -- any non-static data member with no brace-or-equal-initializer has
-      //    class type M (or array thereof) and either M has no default
-      //    constructor or overload resolution as applied to M's default
-      //    constructor results in an ambiguity or in a function that is deleted
-      //    or inaccessible from the defaulted default constructor.
-      if (!FI->hasInClassInitializer()) {
-        CXXConstructorDecl *FieldDefault = LookupDefaultConstructor(FieldRecord);
-        if (!FieldDefault || FieldDefault->isDeleted())
-          return true;
-        if (CheckConstructorAccess(Loc, FieldDefault, FieldDefault->getAccess(),
-                                   PDiag()) != AR_accessible)
-          return true;
-      }
-    } else if (!Union && FieldType.isConstQualified() &&
-               !FI->hasInClassInitializer()) {
-      // -- any non-variant non-static data member of const-qualified type (or
-      //    array thereof) with no brace-or-equal-initializer does not have a
-      //    user-provided default constructor
-      return true;
-    }
-  }
-
-  if (Union && AllConst)
-    return true;
-
-  return false;
-}
-
-bool Sema::ShouldDeleteCopyConstructor(CXXConstructorDecl *CD) {
-  CXXRecordDecl *RD = CD->getParent();
-  assert(!RD->isDependentType() && "do deletion after instantiation");
-  if (!LangOpts.CPlusPlus0x || RD->isInvalidDecl())
-    return false;
-
-  SourceLocation Loc = CD->getLocation();
-
-  // Do access control from the constructor
-  ContextRAII CtorContext(*this, CD);
-
-  bool Union = RD->isUnion();
-
-  assert(!CD->getParamDecl(0)->getType()->getPointeeType().isNull() &&
-         "copy assignment arg has no pointee type");
-  unsigned ArgQuals =
-    CD->getParamDecl(0)->getType()->getPointeeType().isConstQualified() ?
-      Qualifiers::Const : 0;
-
-  // We do this because we should never actually use an anonymous
-  // union's constructor.
-  if (Union && RD->isAnonymousStructOrUnion())
-    return false;
-
-  // FIXME: We should put some diagnostic logic right into this function.
-
-  // C++0x [class.copy]/11
-  //    A defaulted [copy] constructor for class X is defined as delete if X has:
-
-  for (CXXRecordDecl::base_class_iterator BI = RD->bases_begin(),
-                                          BE = RD->bases_end();
-       BI != BE; ++BI) {
-    // We'll handle this one later
-    if (BI->isVirtual())
-      continue;
-
-    QualType BaseType = BI->getType();
-    CXXRecordDecl *BaseDecl = BaseType->getAsCXXRecordDecl();
-    assert(BaseDecl && "base isn't a CXXRecordDecl");
-
-    // -- any [direct base class] of a type with a destructor that is deleted or
-    //    inaccessible from the defaulted constructor
-    CXXDestructorDecl *BaseDtor = LookupDestructor(BaseDecl);
-    if (BaseDtor->isDeleted())
-      return true;
-    if (CheckDestructorAccess(Loc, BaseDtor, PDiag()) !=
-        AR_accessible)
-      return true;
-
-    // -- a [direct base class] B that cannot be [copied] because overload
-    //    resolution, as applied to B's [copy] constructor, results in an
-    //    ambiguity or a function that is deleted or inaccessible from the
-    //    defaulted constructor
-    CXXConstructorDecl *BaseCtor = LookupCopyingConstructor(BaseDecl, ArgQuals);
-    if (!BaseCtor || BaseCtor->isDeleted())
-      return true;
-    if (CheckConstructorAccess(Loc, BaseCtor, BaseCtor->getAccess(), PDiag()) !=
-        AR_accessible)
-      return true;
-  }
-
-  for (CXXRecordDecl::base_class_iterator BI = RD->vbases_begin(),
-                                          BE = RD->vbases_end();
-       BI != BE; ++BI) {
-    QualType BaseType = BI->getType();
-    CXXRecordDecl *BaseDecl = BaseType->getAsCXXRecordDecl();
-    assert(BaseDecl && "base isn't a CXXRecordDecl");
-
-    // -- any [virtual base class] of a type with a destructor that is deleted or
-    //    inaccessible from the defaulted constructor
-    CXXDestructorDecl *BaseDtor = LookupDestructor(BaseDecl);
-    if (BaseDtor->isDeleted())
-      return true;
-    if (CheckDestructorAccess(Loc, BaseDtor, PDiag()) !=
-        AR_accessible)
-      return true;
-
-    // -- a [virtual base class] B that cannot be [copied] because overload
-    //    resolution, as applied to B's [copy] constructor, results in an
-    //    ambiguity or a function that is deleted or inaccessible from the
-    //    defaulted constructor
-    CXXConstructorDecl *BaseCtor = LookupCopyingConstructor(BaseDecl, ArgQuals);
-    if (!BaseCtor || BaseCtor->isDeleted())
-      return true;
-    if (CheckConstructorAccess(Loc, BaseCtor, BaseCtor->getAccess(), PDiag()) !=
-        AR_accessible)
-      return true;
-  }
-
-  for (CXXRecordDecl::field_iterator FI = RD->field_begin(),
-                                     FE = RD->field_end();
-       FI != FE; ++FI) {
-    QualType FieldType = Context.getBaseElementType(FI->getType());
-    
-    // -- for a copy constructor, a non-static data member of rvalue reference
-    //    type
-    if (FieldType->isRValueReferenceType())
-      return true;
- 
-    CXXRecordDecl *FieldRecord = FieldType->getAsCXXRecordDecl();
-
-    if (FieldRecord) {
-      // This is an anonymous union
-      if (FieldRecord->isUnion() && FieldRecord->isAnonymousStructOrUnion()) {
-        // Anonymous unions inside unions do not variant members create
-        if (!Union) {
-          for (CXXRecordDecl::field_iterator UI = FieldRecord->field_begin(),
-                                             UE = FieldRecord->field_end();
-               UI != UE; ++UI) {
-            QualType UnionFieldType = Context.getBaseElementType(UI->getType());
-            CXXRecordDecl *UnionFieldRecord =
-              UnionFieldType->getAsCXXRecordDecl();
-
-            // -- a variant member with a non-trivial [copy] constructor and X
-            //    is a union-like class
-            if (UnionFieldRecord &&
-                !UnionFieldRecord->hasTrivialCopyConstructor())
-              return true;
-          }
-        }
-
-        // Don't try to initalize an anonymous union
-        continue;
-      } else {
-         // -- a variant member with a non-trivial [copy] constructor and X is a
-         //    union-like class
-        if (Union && !FieldRecord->hasTrivialCopyConstructor())
-          return true;
-
-        // -- any [non-static data member] of a type with a destructor that is
-        //    deleted or inaccessible from the defaulted constructor
+      // Unless we're doing assignment, the field's destructor must be
+      // accessible and not deleted.
+      if (!IsAssignment) {
         CXXDestructorDecl *FieldDtor = LookupDestructor(FieldRecord);
         if (FieldDtor->isDeleted())
           return true;
@@ -3701,20 +4346,52 @@ bool Sema::ShouldDeleteCopyConstructor(CXXConstructorDecl *CD) {
           return true;
       }
 
-    // -- a [non-static data member of class type (or array thereof)] B that
-    //    cannot be [copied] because overload resolution, as applied to B's
-    //    [copy] constructor, results in an ambiguity or a function that is
-    //    deleted or inaccessible from the defaulted constructor
-      CXXConstructorDecl *FieldCtor = LookupCopyingConstructor(FieldRecord,
-                                                               ArgQuals);
-      if (!FieldCtor || FieldCtor->isDeleted())
-        return true;
-      if (CheckConstructorAccess(Loc, FieldCtor, FieldCtor->getAccess(),
-                                 PDiag()) != AR_accessible)
-        return true;
+      // Check that the corresponding member of the field is accessible,
+      // unique, and non-deleted. We don't do this if it has an explicit
+      // initialization when default-constructing.
+      if (CSM != CXXDestructor &&
+          (CSM != CXXDefaultConstructor || !FI->hasInClassInitializer())) {
+        SpecialMemberOverloadResult *SMOR =
+          LookupSpecialMember(FieldRecord, CSM, ConstArg, false, false, false,
+                              false);
+        if (!SMOR->hasSuccess())
+          return true;
+
+        CXXMethodDecl *FieldMember = SMOR->getMethod();
+        if (IsConstructor) {
+          CXXConstructorDecl *FieldCtor = cast<CXXConstructorDecl>(FieldMember);
+          if (CheckConstructorAccess(Loc, FieldCtor, FieldCtor->getAccess(),
+                                     PDiag()) != AR_accessible)
+          return true;
+
+          // For a move operation, the corresponding operation must actually
+          // be a move operation (and not a copy selected by overload
+          // resolution) unless we are working on a trivially copyable class.
+          if (IsMove && !FieldCtor->isMoveConstructor() &&
+              !FieldRecord->isTriviallyCopyable())
+            return true;
+        }
+
+        // We need the corresponding member of a union to be trivial so that
+        // we can safely copy them all simultaneously.
+        // FIXME: Note that performing the check here (where we rely on the lack
+        // of an in-class initializer) is technically ill-formed. However, this
+        // seems most obviously to be a bug in the standard.
+        if (IsUnion && !FieldMember->isTrivial())
+          return true;
+      }
+    } else if (CSM == CXXDefaultConstructor && !IsUnion &&
+               FieldType.isConstQualified() && !FI->hasInClassInitializer()) {
+      // We can't initialize a const member of non-class type to any value.
+      return true;
     }
   }
 
+  // We can't have all const members in a union when default-constructing,
+  // or else they're all nonsensical garbage values that can't be changed.
+  if (CSM == CXXDefaultConstructor && IsUnion && AllConst)
+    return true;
+
   return false;
 }
 
@@ -3742,7 +4419,7 @@ bool Sema::ShouldDeleteCopyAssignmentOperator(CXXMethodDecl *MD) {
 
   // FIXME: We should put some diagnostic logic right into this function.
 
-  // C++0x [class.copy]/11
+  // C++0x [class.copy]/20
   //    A defaulted [copy] assignment operator for class X is defined as deleted
   //    if X has:
 
@@ -3791,6 +4468,9 @@ bool Sema::ShouldDeleteCopyAssignmentOperator(CXXMethodDecl *MD) {
   for (CXXRecordDecl::field_iterator FI = RD->field_begin(),
                                      FE = RD->field_end();
        FI != FE; ++FI) {
+    if (FI->isUnnamedBitfield())
+      continue;
+    
     QualType FieldType = Context.getBaseElementType(FI->getType());
     
     // -- a non-static data member of reference type
@@ -3834,9 +4514,126 @@ bool Sema::ShouldDeleteCopyAssignmentOperator(CXXMethodDecl *MD) {
       CXXMethodDecl *CopyOper = LookupCopyingAssignment(FieldRecord, ArgQuals,
                                                         false, 0);
       if (!CopyOper || CopyOper->isDeleted())
-        return false;
+        return true;
       if (CheckDirectMemberAccess(Loc, CopyOper, PDiag()) != AR_accessible)
-        return false;
+        return true;
+    }
+  }
+
+  return false;
+}
+
+bool Sema::ShouldDeleteMoveAssignmentOperator(CXXMethodDecl *MD) {
+  CXXRecordDecl *RD = MD->getParent();
+  assert(!RD->isDependentType() && "do deletion after instantiation");
+  if (!LangOpts.CPlusPlus0x || RD->isInvalidDecl())
+    return false;
+
+  SourceLocation Loc = MD->getLocation();
+
+  // Do access control from the constructor
+  ContextRAII MethodContext(*this, MD);
+
+  bool Union = RD->isUnion();
+
+  // We do this because we should never actually use an anonymous
+  // union's constructor.
+  if (Union && RD->isAnonymousStructOrUnion())
+    return false;
+
+  // C++0x [class.copy]/20
+  //    A defaulted [move] assignment operator for class X is defined as deleted
+  //    if X has:
+
+  //    -- for the move constructor, [...] any direct or indirect virtual base
+  //       class.
+  if (RD->getNumVBases() != 0)
+    return true;
+
+  for (CXXRecordDecl::base_class_iterator BI = RD->bases_begin(),
+                                          BE = RD->bases_end();
+       BI != BE; ++BI) {
+
+    QualType BaseType = BI->getType();
+    CXXRecordDecl *BaseDecl = BaseType->getAsCXXRecordDecl();
+    assert(BaseDecl && "base isn't a CXXRecordDecl");
+
+    // -- a [direct base class] B that cannot be [moved] because overload
+    //    resolution, as applied to B's [move] assignment operator, results in
+    //    an ambiguity or a function that is deleted or inaccessible from the
+    //    assignment operator
+    CXXMethodDecl *MoveOper = LookupMovingAssignment(BaseDecl, false, 0);
+    if (!MoveOper || MoveOper->isDeleted())
+      return true;
+    if (CheckDirectMemberAccess(Loc, MoveOper, PDiag()) != AR_accessible)
+      return true;
+
+    // -- for the move assignment operator, a [direct base class] with a type
+    //    that does not have a move assignment operator and is not trivially
+    //    copyable.
+    if (!MoveOper->isMoveAssignmentOperator() &&
+        !BaseDecl->isTriviallyCopyable())
+      return true;
+  }
+
+  for (CXXRecordDecl::field_iterator FI = RD->field_begin(),
+                                     FE = RD->field_end();
+       FI != FE; ++FI) {
+    if (FI->isUnnamedBitfield())
+      continue;
+        
+    QualType FieldType = Context.getBaseElementType(FI->getType());
+    
+    // -- a non-static data member of reference type
+    if (FieldType->isReferenceType())
+      return true;
+
+    // -- a non-static data member of const non-class type (or array thereof)
+    if (FieldType.isConstQualified() && !FieldType->isRecordType())
+      return true;
+
+    CXXRecordDecl *FieldRecord = FieldType->getAsCXXRecordDecl();
+
+    if (FieldRecord) {
+      // This is an anonymous union
+      if (FieldRecord->isUnion() && FieldRecord->isAnonymousStructOrUnion()) {
+        // Anonymous unions inside unions do not variant members create
+        if (!Union) {
+          for (CXXRecordDecl::field_iterator UI = FieldRecord->field_begin(),
+                                             UE = FieldRecord->field_end();
+               UI != UE; ++UI) {
+            QualType UnionFieldType = Context.getBaseElementType(UI->getType());
+            CXXRecordDecl *UnionFieldRecord =
+              UnionFieldType->getAsCXXRecordDecl();
+
+            // -- a variant member with a non-trivial [move] assignment operator
+            //    and X is a union-like class
+            if (UnionFieldRecord &&
+                !UnionFieldRecord->hasTrivialMoveAssignment())
+              return true;
+          }
+        }
+
+        // Don't try to initalize an anonymous union
+        continue;
+      // -- a variant member with a non-trivial [move] assignment operator
+      //    and X is a union-like class
+      } else if (Union && !FieldRecord->hasTrivialMoveAssignment()) {
+          return true;
+      }
+
+      CXXMethodDecl *MoveOper = LookupMovingAssignment(FieldRecord, false, 0);
+      if (!MoveOper || MoveOper->isDeleted())
+        return true;
+      if (CheckDirectMemberAccess(Loc, MoveOper, PDiag()) != AR_accessible)
+        return true;
+
+      // -- for the move assignment operator, a [non-static data member] with a
+      //    type that does not have a move assignment operator and is not
+      //    trivially copyable.
+      if (!MoveOper->isMoveAssignmentOperator() &&
+          !FieldRecord->isTriviallyCopyable())
+        return true;
     }
   }
 
@@ -3959,7 +4756,7 @@ namespace {
     Sema *S;
     CXXMethodDecl *Method;
     llvm::SmallPtrSet<const CXXMethodDecl *, 8> OverridenAndUsingBaseMethods;
-    llvm::SmallVector<CXXMethodDecl *, 8> OverloadedMethods;
+    SmallVector<CXXMethodDecl *, 8> OverloadedMethods;
   };
 }
 
@@ -3978,7 +4775,7 @@ static bool FindHiddenVirtualMethod(const CXXBaseSpecifier *Specifier,
   assert(Name.getNameKind() == DeclarationName::Identifier);
 
   bool foundSameNameMethod = false;
-  llvm::SmallVector<CXXMethodDecl *, 8> overloadedMethods;
+  SmallVector<CXXMethodDecl *, 8> overloadedMethods;
   for (Path.Decls = BaseRecord->lookup(Name);
        Path.Decls.first != Path.Decls.second;
        ++Path.Decls.first) {
@@ -4009,7 +4806,7 @@ static bool FindHiddenVirtualMethod(const CXXBaseSpecifier *Specifier,
 /// overriding any.
 void Sema::DiagnoseHiddenVirtualMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
   if (Diags.getDiagnosticLevel(diag::warn_overloaded_virtual,
-                               MD->getLocation()) == Diagnostic::Ignored)
+                               MD->getLocation()) == DiagnosticsEngine::Ignored)
     return;
   if (MD->getDeclName().getNameKind() != DeclarationName::Identifier)
     return;
@@ -4058,10 +4855,10 @@ void Sema::ActOnFinishCXXMemberSpecification(Scope* S, SourceLocation RLoc,
 
   AdjustDeclIfTemplate(TagDecl);
 
-  ActOnFields(S, RLoc, TagDecl,
+  ActOnFields(S, RLoc, TagDecl, llvm::makeArrayRef(
               // strict aliasing violation!
               reinterpret_cast<Decl**>(FieldCollector->getCurFields()),
-              FieldCollector->getCurNumFields(), LBrac, RBrac, AttrList);
+              FieldCollector->getCurNumFields()), LBrac, RBrac, AttrList);
 
   CheckCompletedCXXClass(
                         dyn_cast_or_null<CXXRecordDecl>(TagDecl));
@@ -4946,7 +5743,7 @@ Decl *Sema::ActOnUsingDirective(Scope *S,
                                       IdentLoc, Named, CommonAncestor);
 
     if (IsUsingDirectiveInToplevelContext(CurContext) &&
-        !SourceMgr.isFromMainFile(SourceMgr.getInstantiationLoc(IdentLoc))) {
+        !SourceMgr.isFromMainFile(SourceMgr.getExpansionLoc(IdentLoc))) {
       Diag(IdentLoc, diag::warn_using_directive_in_header);
     }
 
@@ -5361,7 +6158,6 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
   // Otherwise, look up the target name.
 
   LookupResult R(*this, NameInfo, LookupOrdinaryName);
-  R.setUsingDeclaration(true);
 
   // Unlike most lookups, we don't always want to hide tag
   // declarations: tag names are visible through the using declaration
@@ -5968,7 +6764,9 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
                                  /*TInfo=*/0,
                                  /*isExplicit=*/false,
                                  /*isInline=*/true,
-                                 /*isImplicitlyDeclared=*/true);
+                                 /*isImplicitlyDeclared=*/true,
+                                 // FIXME: apply the rules for definitions here
+                                 /*isConstexpr=*/false);
   DefaultCon->setAccess(AS_public);
   DefaultCon->setDefaulted();
   DefaultCon->setImplicit();
@@ -5981,7 +6779,7 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
     PushOnScopeChains(DefaultCon, S, false);
   ClassDecl->addDecl(DefaultCon);
 
-  if (ShouldDeleteDefaultConstructor(DefaultCon))
+  if (ShouldDeleteSpecialMember(DefaultCon, CXXDefaultConstructor))
     DefaultCon->setDeletedAsWritten();
   
   return DefaultCon;
@@ -6075,7 +6873,7 @@ void Sema::DeclareInheritedConstructors(CXXRecordDecl *ClassDecl) {
   // We start with an initial pass over the base classes to collect those that
   // inherit constructors from. If there are none, we can forgo all further
   // processing.
-  typedef llvm::SmallVector<const RecordType *, 4> BasesVector;
+  typedef SmallVector<const RecordType *, 4> BasesVector;
   BasesVector BasesToInheritFrom;
   for (CXXRecordDecl::base_class_iterator BaseIt = ClassDecl->bases_begin(),
                                           BaseE = ClassDecl->bases_end();
@@ -6171,7 +6969,7 @@ void Sema::DeclareInheritedConstructors(CXXRecordDecl *ClassDecl) {
         if (params == maxParams)
           NewCtorType = BaseCtorType;
         else {
-          llvm::SmallVector<QualType, 16> Args;
+          SmallVector<QualType, 16> Args;
           for (unsigned i = 0; i < params; ++i) {
             Args.push_back(BaseCtorType->getArgType(i));
           }
@@ -6219,16 +7017,19 @@ void Sema::DeclareInheritedConstructors(CXXRecordDecl *ClassDecl) {
 
         // OK, we're there, now add the constructor.
         // C++0x [class.inhctor]p8: [...] that would be performed by a
-        //   user-writtern inline constructor [...]
+        //   user-written inline constructor [...]
         DeclarationNameInfo DNI(CreatedCtorName, UsingLoc);
         CXXConstructorDecl *NewCtor = CXXConstructorDecl::Create(
             Context, ClassDecl, UsingLoc, DNI, QualType(NewCtorType, 0),
             /*TInfo=*/0, BaseCtor->isExplicit(), /*Inline=*/true,
-            /*ImplicitlyDeclared=*/true);
+            /*ImplicitlyDeclared=*/true,
+            // FIXME: Due to a defect in the standard, we treat inherited
+            // constructors as constexpr even if that makes them ill-formed.
+            /*Constexpr=*/BaseCtor->isConstexpr());
         NewCtor->setAccess(BaseCtor->getAccess());
 
         // Build up the parameter decls and add them.
-        llvm::SmallVector<ParmVarDecl *, 16> ParamDecls;
+        SmallVector<ParmVarDecl *, 16> ParamDecls;
         for (unsigned i = 0; i < params; ++i) {
           ParamDecls.push_back(ParmVarDecl::Create(Context, NewCtor,
                                                    UsingLoc, UsingLoc,
@@ -6237,7 +7038,7 @@ void Sema::DeclareInheritedConstructors(CXXRecordDecl *ClassDecl) {
                                                    /*TInfo=*/0, SC_None,
                                                    SC_None, /*DefaultArg=*/0));
         }
-        NewCtor->setParams(ParamDecls.data(), ParamDecls.size());
+        NewCtor->setParams(ParamDecls);
         NewCtor->setInheritedConstructor(BaseCtor);
 
         PushOnScopeChains(NewCtor, S, false);
@@ -6365,7 +7166,7 @@ void Sema::DefineImplicitDestructor(SourceLocation CurrentLocation,
 
   SourceLocation Loc = Destructor->getLocation();
   Destructor->setBody(new (Context) CompoundStmt(Context, 0, 0, Loc, Loc));
-
+  Destructor->setImplicitlyDefined(true);
   Destructor->setUsed();
   MarkVTableUsed(CurrentLocation, ClassDecl);
 
@@ -6388,8 +7189,10 @@ void Sema::AdjustDestructorExceptionSpec(CXXRecordDecl *classDecl,
   ImplicitExceptionSpecification exceptSpec =
       ComputeDefaultedDtorExceptionSpec(classDecl);
 
-  // Replace the destructor's type.
-  FunctionProtoType::ExtProtoInfo epi;
+  // Replace the destructor's type, building off the existing one. Fortunately,
+  // the only thing of interest in the destructor type is its extended info.
+  // The return and arguments are fixed.
+  FunctionProtoType::ExtProtoInfo epi = dtorType->getExtProtoInfo();
   epi.ExceptionSpecType = exceptSpec.getExceptionSpecType();
   epi.NumExceptions = exceptSpec.size();
   epi.Exceptions = exceptSpec.data();
@@ -6403,41 +7206,45 @@ void Sema::AdjustDestructorExceptionSpec(CXXRecordDecl *classDecl,
   // However, we don't have a body yet, so it needs to be done somewhere else.
 }
 
-/// \brief Builds a statement that copies the given entity from \p From to
+/// \brief Builds a statement that copies/moves the given entity from \p From to
 /// \c To.
 ///
-/// This routine is used to copy the members of a class with an
-/// implicitly-declared copy assignment operator. When the entities being
+/// This routine is used to copy/move the members of a class with an
+/// implicitly-declared copy/move assignment operator. When the entities being
 /// copied are arrays, this routine builds for loops to copy them.
 ///
 /// \param S The Sema object used for type-checking.
 ///
-/// \param Loc The location where the implicit copy is being generated.
+/// \param Loc The location where the implicit copy/move is being generated.
 ///
-/// \param T The type of the expressions being copied. Both expressions must
-/// have this type.
+/// \param T The type of the expressions being copied/moved. Both expressions
+/// must have this type.
 ///
-/// \param To The expression we are copying to.
+/// \param To The expression we are copying/moving to.
 ///
-/// \param From The expression we are copying from.
+/// \param From The expression we are copying/moving from.
 ///
-/// \param CopyingBaseSubobject Whether we're copying a base subobject.
+/// \param CopyingBaseSubobject Whether we're copying/moving a base subobject.
 /// Otherwise, it's a non-static member subobject.
 ///
+/// \param Copying Whether we're copying or moving.
+///
 /// \param Depth Internal parameter recording the depth of the recursion.
 ///
 /// \returns A statement or a loop that copies the expressions.
 static StmtResult
 BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T, 
                       Expr *To, Expr *From,
-                      bool CopyingBaseSubobject, unsigned Depth = 0) {
-  // C++0x [class.copy]p30:
+                      bool CopyingBaseSubobject, bool Copying,
+                      unsigned Depth = 0) {
+  // C++0x [class.copy]p28:
   //   Each subobject is assigned in the manner appropriate to its type:
   //
-  //     - if the subobject is of class type, the copy assignment operator
-  //       for the class is used (as if by explicit qualification; that is, 
-  //       ignoring any possible virtual overriding functions in more derived 
-  //       classes);
+  //     - if the subobject is of class type, as if by a call to operator= with
+  //       the subobject as the object expression and the corresponding
+  //       subobject of x as a single function argument (as if by explicit
+  //       qualification; that is, ignoring any possible virtual overriding
+  //       functions in more derived classes);
   if (const RecordType *RecordTy = T->getAs<RecordType>()) {
     CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RecordTy->getDecl());
     
@@ -6447,14 +7254,15 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
     LookupResult OpLookup(S, Name, Loc, Sema::LookupOrdinaryName);
     S.LookupQualifiedName(OpLookup, ClassDecl, false);
     
-    // Filter out any result that isn't a copy-assignment operator.
+    // Filter out any result that isn't a copy/move-assignment operator.
     LookupResult::Filter F = OpLookup.makeFilter();
     while (F.hasNext()) {
       NamedDecl *D = F.next();
       if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D))
-        if (Method->isCopyAssignmentOperator())
+        if (Copying ? Method->isCopyAssignmentOperator() :
+                      Method->isMoveAssignmentOperator())
           continue;
-      
+
       F.erase();
     }
     F.done();
@@ -6573,11 +7381,13 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
                                                          IterationVarRef, Loc));
   To = AssertSuccess(S.CreateBuiltinArraySubscriptExpr(To, Loc,
                                                        IterationVarRef, Loc));
-  
-  // Build the copy for an individual element of the array.
+  if (!Copying) // Cast to rvalue
+    From = CastForMoving(S, From);
+
+  // Build the copy/move for an individual element of the array.
   StmtResult Copy = BuildSingleCopyAssign(S, Loc, ArrayTy->getElementType(),
                                           To, From, CopyingBaseSubobject,
-                                          Depth + 1);
+                                          Copying, Depth + 1);
   if (Copy.isInvalid())
     return StmtError();
   
@@ -6733,7 +7543,7 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
                             Context.getFunctionType(RetType, &ArgType, 1, EPI),
                             /*TInfo=*/0, /*isStatic=*/false,
                             /*StorageClassAsWritten=*/SC_None,
-                            /*isInline=*/true,
+                            /*isInline=*/true, /*isConstexpr=*/false,
                             SourceLocation());
   CopyAssignment->setAccess(AS_public);
   CopyAssignment->setDefaulted();
@@ -6746,7 +7556,7 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
                                                ArgType, /*TInfo=*/0,
                                                SC_None,
                                                SC_None, 0);
-  CopyAssignment->setParams(&FromParam, 1);
+  CopyAssignment->setParams(FromParam);
   
   // Note that we have added this copy-assignment operator.
   ++ASTContext::NumImplicitCopyAssignmentOperatorsDeclared;
@@ -6857,7 +7667,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     // Build the copy.
     StmtResult Copy = BuildSingleCopyAssign(*this, Loc, BaseType,
                                             To.get(), From,
-                                            /*CopyingBaseSubobject=*/true);
+                                            /*CopyingBaseSubobject=*/true,
+                                            /*Copying=*/true);
     if (Copy.isInvalid()) {
       Diag(CurrentLocation, diag::note_member_synthesized_at) 
         << CXXCopyAssignment << Context.getTagDeclType(ClassDecl);
@@ -6878,6 +7689,9 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
                                   FieldEnd = ClassDecl->field_end(); 
        Field != FieldEnd; ++Field) {
+    if (Field->isUnnamedBitfield())
+      continue;
+    
     // Check for members of reference type; we can't copy those.
     if (Field->getType()->isReferenceType()) {
       Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
@@ -6902,9 +7716,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     }
 
     // Suppress assigning zero-width bitfields.
-    if (const Expr *Width = Field->getBitWidth())
-      if (Width->EvaluateAsInt(Context) == 0)
-        continue;
+    if (Field->isBitField() && Field->getBitWidthValue(Context) == 0)
+      continue;
     
     QualType FieldType = Field->getType().getNonReferenceType();
     if (FieldType->isIncompleteArrayType()) {
@@ -6932,8 +7745,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     // explicit assignments, do so. This optimization only applies for arrays 
     // of scalars and arrays of class type with trivial copy-assignment 
     // operators.
-    if (FieldType->isArrayType() && 
-        BaseType.hasTrivialCopyAssignment(Context)) {
+    if (FieldType->isArrayType() && !FieldType.isVolatileQualified()
+        && BaseType.hasTrivialAssignment(Context, /*Copying=*/true)) {
       // Compute the size of the memory buffer to be copied.
       QualType SizeType = Context.getSizeType();
       llvm::APInt Size(Context.getTypeSize(SizeType), 
@@ -7020,8 +7833,9 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     
     // Build the copy of this field.
     StmtResult Copy = BuildSingleCopyAssign(*this, Loc, FieldType, 
-                                                  To.get(), From.get(),
-                                              /*CopyingBaseSubobject=*/false);
+                                            To.get(), From.get(),
+                                            /*CopyingBaseSubobject=*/false,
+                                            /*Copying=*/true);
     if (Copy.isInvalid()) {
       Diag(CurrentLocation, diag::note_member_synthesized_at) 
         << CXXCopyAssignment << Context.getTagDeclType(ClassDecl);
@@ -7066,6 +7880,436 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   }
 }
 
+Sema::ImplicitExceptionSpecification
+Sema::ComputeDefaultedMoveAssignmentExceptionSpec(CXXRecordDecl *ClassDecl) {
+  ImplicitExceptionSpecification ExceptSpec(Context);
+
+  if (ClassDecl->isInvalidDecl())
+    return ExceptSpec;
+
+  // C++0x [except.spec]p14:
+  //   An implicitly declared special member function (Clause 12) shall have an 
+  //   exception-specification. [...]
+
+  // It is unspecified whether or not an implicit move assignment operator
+  // attempts to deduplicate calls to assignment operators of virtual bases are
+  // made. As such, this exception specification is effectively unspecified.
+  // Based on a similar decision made for constness in C++0x, we're erring on
+  // the side of assuming such calls to be made regardless of whether they
+  // actually happen.
+  // Note that a move constructor is not implicitly declared when there are
+  // virtual bases, but it can still be user-declared and explicitly defaulted.
+  for (CXXRecordDecl::base_class_iterator Base = ClassDecl->bases_begin(),
+                                       BaseEnd = ClassDecl->bases_end();
+       Base != BaseEnd; ++Base) {
+    if (Base->isVirtual())
+      continue;
+
+    CXXRecordDecl *BaseClassDecl
+      = cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+    if (CXXMethodDecl *MoveAssign = LookupMovingAssignment(BaseClassDecl,
+                                                           false, 0))
+      ExceptSpec.CalledDecl(MoveAssign);
+  }
+
+  for (CXXRecordDecl::base_class_iterator Base = ClassDecl->vbases_begin(),
+                                       BaseEnd = ClassDecl->vbases_end();
+       Base != BaseEnd; ++Base) {
+    CXXRecordDecl *BaseClassDecl
+      = cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+    if (CXXMethodDecl *MoveAssign = LookupMovingAssignment(BaseClassDecl,
+                                                           false, 0))
+      ExceptSpec.CalledDecl(MoveAssign);
+  }
+
+  for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
+                                  FieldEnd = ClassDecl->field_end();
+       Field != FieldEnd;
+       ++Field) {
+    QualType FieldType = Context.getBaseElementType((*Field)->getType());
+    if (CXXRecordDecl *FieldClassDecl = FieldType->getAsCXXRecordDecl()) {
+      if (CXXMethodDecl *MoveAssign = LookupMovingAssignment(FieldClassDecl,
+                                                             false, 0))
+        ExceptSpec.CalledDecl(MoveAssign);
+    }
+  }
+
+  return ExceptSpec;
+}
+
+CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
+  // Note: The following rules are largely analoguous to the move
+  // constructor rules.
+
+  ImplicitExceptionSpecification Spec(
+      ComputeDefaultedMoveAssignmentExceptionSpec(ClassDecl));
+
+  QualType ArgType = Context.getTypeDeclType(ClassDecl);
+  QualType RetType = Context.getLValueReferenceType(ArgType);
+  ArgType = Context.getRValueReferenceType(ArgType);
+
+  //   An implicitly-declared move assignment operator is an inline public
+  //   member of its class.
+  FunctionProtoType::ExtProtoInfo EPI = Spec.getEPI();
+  DeclarationName Name = Context.DeclarationNames.getCXXOperatorName(OO_Equal);
+  SourceLocation ClassLoc = ClassDecl->getLocation();
+  DeclarationNameInfo NameInfo(Name, ClassLoc);
+  CXXMethodDecl *MoveAssignment
+    = CXXMethodDecl::Create(Context, ClassDecl, ClassLoc, NameInfo,
+                            Context.getFunctionType(RetType, &ArgType, 1, EPI),
+                            /*TInfo=*/0, /*isStatic=*/false,
+                            /*StorageClassAsWritten=*/SC_None,
+                            /*isInline=*/true,
+                            /*isConstexpr=*/false,
+                            SourceLocation());
+  MoveAssignment->setAccess(AS_public);
+  MoveAssignment->setDefaulted();
+  MoveAssignment->setImplicit();
+  MoveAssignment->setTrivial(ClassDecl->hasTrivialMoveAssignment());
+
+  // Add the parameter to the operator.
+  ParmVarDecl *FromParam = ParmVarDecl::Create(Context, MoveAssignment,
+                                               ClassLoc, ClassLoc, /*Id=*/0,
+                                               ArgType, /*TInfo=*/0,
+                                               SC_None,
+                                               SC_None, 0);
+  MoveAssignment->setParams(FromParam);
+
+  // Note that we have added this copy-assignment operator.
+  ++ASTContext::NumImplicitMoveAssignmentOperatorsDeclared;
+
+  // C++0x [class.copy]p9:
+  //   If the definition of a class X does not explicitly declare a move
+  //   assignment operator, one will be implicitly declared as defaulted if and
+  //   only if:
+  //   [...]
+  //   - the move assignment operator would not be implicitly defined as
+  //     deleted.
+  if (ShouldDeleteMoveAssignmentOperator(MoveAssignment)) {
+    // Cache this result so that we don't try to generate this over and over
+    // on every lookup, leaking memory and wasting time.
+    ClassDecl->setFailedImplicitMoveAssignment();
+    return 0;
+  }
+
+  if (Scope *S = getScopeForContext(ClassDecl))
+    PushOnScopeChains(MoveAssignment, S, false);
+  ClassDecl->addDecl(MoveAssignment);
+
+  AddOverriddenMethods(ClassDecl, MoveAssignment);
+  return MoveAssignment;
+}
+
+void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
+                                        CXXMethodDecl *MoveAssignOperator) {
+  assert((MoveAssignOperator->isDefaulted() && 
+          MoveAssignOperator->isOverloadedOperator() &&
+          MoveAssignOperator->getOverloadedOperator() == OO_Equal &&
+          !MoveAssignOperator->doesThisDeclarationHaveABody()) &&
+         "DefineImplicitMoveAssignment called for wrong function");
+
+  CXXRecordDecl *ClassDecl = MoveAssignOperator->getParent();
+
+  if (ClassDecl->isInvalidDecl() || MoveAssignOperator->isInvalidDecl()) {
+    MoveAssignOperator->setInvalidDecl();
+    return;
+  }
+  
+  MoveAssignOperator->setUsed();
+
+  ImplicitlyDefinedFunctionScope Scope(*this, MoveAssignOperator);
+  DiagnosticErrorTrap Trap(Diags);
+
+  // C++0x [class.copy]p28:
+  //   The implicitly-defined or move assignment operator for a non-union class
+  //   X performs memberwise move assignment of its subobjects. The direct base
+  //   classes of X are assigned first, in the order of their declaration in the
+  //   base-specifier-list, and then the immediate non-static data members of X
+  //   are assigned, in the order in which they were declared in the class
+  //   definition.
+
+  // The statements that form the synthesized function body.
+  ASTOwningVector<Stmt*> Statements(*this);
+
+  // The parameter for the "other" object, which we are move from.
+  ParmVarDecl *Other = MoveAssignOperator->getParamDecl(0);
+  QualType OtherRefType = Other->getType()->
+      getAs<RValueReferenceType>()->getPointeeType();
+  assert(OtherRefType.getQualifiers() == 0 &&
+         "Bad argument type of defaulted move assignment");
+
+  // Our location for everything implicitly-generated.
+  SourceLocation Loc = MoveAssignOperator->getLocation();
+
+  // Construct a reference to the "other" object. We'll be using this 
+  // throughout the generated ASTs.
+  Expr *OtherRef = BuildDeclRefExpr(Other, OtherRefType, VK_LValue, Loc).take();
+  assert(OtherRef && "Reference to parameter cannot fail!");
+  // Cast to rvalue.
+  OtherRef = CastForMoving(*this, OtherRef);
+
+  // Construct the "this" pointer. We'll be using this throughout the generated
+  // ASTs.
+  Expr *This = ActOnCXXThis(Loc).takeAs<Expr>();
+  assert(This && "Reference to this cannot fail!");
+  
+  // Assign base classes.
+  bool Invalid = false;
+  for (CXXRecordDecl::base_class_iterator Base = ClassDecl->bases_begin(),
+       E = ClassDecl->bases_end(); Base != E; ++Base) {
+    // Form the assignment:
+    //   static_cast<Base*>(this)->Base::operator=(static_cast<Base&&>(other));
+    QualType BaseType = Base->getType().getUnqualifiedType();
+    if (!BaseType->isRecordType()) {
+      Invalid = true;
+      continue;
+    }
+
+    CXXCastPath BasePath;
+    BasePath.push_back(Base);
+
+    // Construct the "from" expression, which is an implicit cast to the
+    // appropriately-qualified base type.
+    Expr *From = OtherRef;
+    From = ImpCastExprToType(From, BaseType, CK_UncheckedDerivedToBase,
+                             VK_XValue, &BasePath).take();
+
+    // Dereference "this".
+    ExprResult To = CreateBuiltinUnaryOp(Loc, UO_Deref, This);
+
+    // Implicitly cast "this" to the appropriately-qualified base type.
+    To = ImpCastExprToType(To.take(), 
+                           Context.getCVRQualifiedType(BaseType,
+                                     MoveAssignOperator->getTypeQualifiers()),
+                           CK_UncheckedDerivedToBase, 
+                           VK_LValue, &BasePath);
+
+    // Build the move.
+    StmtResult Move = BuildSingleCopyAssign(*this, Loc, BaseType,
+                                            To.get(), From,
+                                            /*CopyingBaseSubobject=*/true,
+                                            /*Copying=*/false);
+    if (Move.isInvalid()) {
+      Diag(CurrentLocation, diag::note_member_synthesized_at) 
+        << CXXMoveAssignment << Context.getTagDeclType(ClassDecl);
+      MoveAssignOperator->setInvalidDecl();
+      return;
+    }
+
+    // Success! Record the move.
+    Statements.push_back(Move.takeAs<Expr>());
+  }
+
+  // \brief Reference to the __builtin_memcpy function.
+  Expr *BuiltinMemCpyRef = 0;
+  // \brief Reference to the __builtin_objc_memmove_collectable function.
+  Expr *CollectableMemCpyRef = 0;
+
+  // Assign non-static members.
+  for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
+                                  FieldEnd = ClassDecl->field_end(); 
+       Field != FieldEnd; ++Field) {
+    if (Field->isUnnamedBitfield())
+      continue;
+
+    // Check for members of reference type; we can't move those.
+    if (Field->getType()->isReferenceType()) {
+      Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
+        << Context.getTagDeclType(ClassDecl) << 0 << Field->getDeclName();
+      Diag(Field->getLocation(), diag::note_declared_at);
+      Diag(CurrentLocation, diag::note_member_synthesized_at) 
+        << CXXMoveAssignment << Context.getTagDeclType(ClassDecl);
+      Invalid = true;
+      continue;
+    }
+
+    // Check for members of const-qualified, non-class type.
+    QualType BaseType = Context.getBaseElementType(Field->getType());
+    if (!BaseType->getAs<RecordType>() && BaseType.isConstQualified()) {
+      Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
+        << Context.getTagDeclType(ClassDecl) << 1 << Field->getDeclName();
+      Diag(Field->getLocation(), diag::note_declared_at);
+      Diag(CurrentLocation, diag::note_member_synthesized_at) 
+        << CXXMoveAssignment << Context.getTagDeclType(ClassDecl);
+      Invalid = true;      
+      continue;
+    }
+
+    // Suppress assigning zero-width bitfields.
+    if (Field->isBitField() && Field->getBitWidthValue(Context) == 0)
+      continue;
+    
+    QualType FieldType = Field->getType().getNonReferenceType();
+    if (FieldType->isIncompleteArrayType()) {
+      assert(ClassDecl->hasFlexibleArrayMember() && 
+             "Incomplete array type is not valid");
+      continue;
+    }
+    
+    // Build references to the field in the object we're copying from and to.
+    CXXScopeSpec SS; // Intentionally empty
+    LookupResult MemberLookup(*this, Field->getDeclName(), Loc,
+                              LookupMemberName);
+    MemberLookup.addDecl(*Field);
+    MemberLookup.resolveKind();
+    ExprResult From = BuildMemberReferenceExpr(OtherRef, OtherRefType,
+                                               Loc, /*IsArrow=*/false,
+                                               SS, 0, MemberLookup, 0);
+    ExprResult To = BuildMemberReferenceExpr(This, This->getType(),
+                                             Loc, /*IsArrow=*/true,
+                                             SS, 0, MemberLookup, 0);
+    assert(!From.isInvalid() && "Implicit field reference cannot fail");
+    assert(!To.isInvalid() && "Implicit field reference cannot fail");
+
+    assert(!From.get()->isLValue() && // could be xvalue or prvalue
+        "Member reference with rvalue base must be rvalue except for reference "
+        "members, which aren't allowed for move assignment.");
+
+    // If the field should be copied with __builtin_memcpy rather than via
+    // explicit assignments, do so. This optimization only applies for arrays 
+    // of scalars and arrays of class type with trivial move-assignment 
+    // operators.
+    if (FieldType->isArrayType() && !FieldType.isVolatileQualified()
+        && BaseType.hasTrivialAssignment(Context, /*Copying=*/false)) {
+      // Compute the size of the memory buffer to be copied.
+      QualType SizeType = Context.getSizeType();
+      llvm::APInt Size(Context.getTypeSize(SizeType), 
+                       Context.getTypeSizeInChars(BaseType).getQuantity());
+      for (const ConstantArrayType *Array
+              = Context.getAsConstantArrayType(FieldType);
+           Array; 
+           Array = Context.getAsConstantArrayType(Array->getElementType())) {
+        llvm::APInt ArraySize
+          = Array->getSize().zextOrTrunc(Size.getBitWidth());
+        Size *= ArraySize;
+      }
+
+      // Take the address of the field references for "from" and "to". We
+      // directly construct UnaryOperators here because semantic analysis
+      // does not permit us to take the address of an xvalue.
+      From = new (Context) UnaryOperator(From.get(), UO_AddrOf,
+                             Context.getPointerType(From.get()->getType()),
+                             VK_RValue, OK_Ordinary, Loc);
+      To = new (Context) UnaryOperator(To.get(), UO_AddrOf,
+                           Context.getPointerType(To.get()->getType()),
+                           VK_RValue, OK_Ordinary, Loc);
+          
+      bool NeedsCollectableMemCpy = 
+          (BaseType->isRecordType() && 
+           BaseType->getAs<RecordType>()->getDecl()->hasObjectMember());
+          
+      if (NeedsCollectableMemCpy) {
+        if (!CollectableMemCpyRef) {
+          // Create a reference to the __builtin_objc_memmove_collectable function.
+          LookupResult R(*this, 
+                         &Context.Idents.get("__builtin_objc_memmove_collectable"), 
+                         Loc, LookupOrdinaryName);
+          LookupName(R, TUScope, true);
+        
+          FunctionDecl *CollectableMemCpy = R.getAsSingle<FunctionDecl>();
+          if (!CollectableMemCpy) {
+            // Something went horribly wrong earlier, and we will have 
+            // complained about it.
+            Invalid = true;
+            continue;
+          }
+        
+          CollectableMemCpyRef = BuildDeclRefExpr(CollectableMemCpy, 
+                                                  CollectableMemCpy->getType(),
+                                                  VK_LValue, Loc, 0).take();
+          assert(CollectableMemCpyRef && "Builtin reference cannot fail");
+        }
+      }
+      // Create a reference to the __builtin_memcpy builtin function.
+      else if (!BuiltinMemCpyRef) {
+        LookupResult R(*this, &Context.Idents.get("__builtin_memcpy"), Loc,
+                       LookupOrdinaryName);
+        LookupName(R, TUScope, true);
+        
+        FunctionDecl *BuiltinMemCpy = R.getAsSingle<FunctionDecl>();
+        if (!BuiltinMemCpy) {
+          // Something went horribly wrong earlier, and we will have complained
+          // about it.
+          Invalid = true;
+          continue;
+        }
+
+        BuiltinMemCpyRef = BuildDeclRefExpr(BuiltinMemCpy, 
+                                            BuiltinMemCpy->getType(),
+                                            VK_LValue, Loc, 0).take();
+        assert(BuiltinMemCpyRef && "Builtin reference cannot fail");
+      }
+          
+      ASTOwningVector<Expr*> CallArgs(*this);
+      CallArgs.push_back(To.takeAs<Expr>());
+      CallArgs.push_back(From.takeAs<Expr>());
+      CallArgs.push_back(IntegerLiteral::Create(Context, Size, SizeType, Loc));
+      ExprResult Call = ExprError();
+      if (NeedsCollectableMemCpy)
+        Call = ActOnCallExpr(/*Scope=*/0,
+                             CollectableMemCpyRef,
+                             Loc, move_arg(CallArgs), 
+                             Loc);
+      else
+        Call = ActOnCallExpr(/*Scope=*/0,
+                             BuiltinMemCpyRef,
+                             Loc, move_arg(CallArgs), 
+                             Loc);
+          
+      assert(!Call.isInvalid() && "Call to __builtin_memcpy cannot fail!");
+      Statements.push_back(Call.takeAs<Expr>());
+      continue;
+    }
+    
+    // Build the move of this field.
+    StmtResult Move = BuildSingleCopyAssign(*this, Loc, FieldType, 
+                                            To.get(), From.get(),
+                                            /*CopyingBaseSubobject=*/false,
+                                            /*Copying=*/false);
+    if (Move.isInvalid()) {
+      Diag(CurrentLocation, diag::note_member_synthesized_at) 
+        << CXXMoveAssignment << Context.getTagDeclType(ClassDecl);
+      MoveAssignOperator->setInvalidDecl();
+      return;
+    }
+    
+    // Success! Record the copy.
+    Statements.push_back(Move.takeAs<Stmt>());
+  }
+
+  if (!Invalid) {
+    // Add a "return *this;"
+    ExprResult ThisObj = CreateBuiltinUnaryOp(Loc, UO_Deref, This);
+    
+    StmtResult Return = ActOnReturnStmt(Loc, ThisObj.get());
+    if (Return.isInvalid())
+      Invalid = true;
+    else {
+      Statements.push_back(Return.takeAs<Stmt>());
+
+      if (Trap.hasErrorOccurred()) {
+        Diag(CurrentLocation, diag::note_member_synthesized_at) 
+          << CXXMoveAssignment << Context.getTagDeclType(ClassDecl);
+        Invalid = true;
+      }
+    }
+  }
+
+  if (Invalid) {
+    MoveAssignOperator->setInvalidDecl();
+    return;
+  }
+  
+  StmtResult Body = ActOnCompoundStmt(Loc, Loc, move_arg(Statements),
+                                            /*isStmtExpr=*/false);
+  assert(!Body.isInvalid() && "Compound statement creation cannot fail");
+  MoveAssignOperator->setBody(Body.takeAs<Stmt>());
+
+  if (ASTMutationListener *L = getASTMutationListener()) {
+    L->CompletedImplicitDefinition(MoveAssignOperator);
+  }
+}
+
 std::pair<Sema::ImplicitExceptionSpecification, bool>
 Sema::ComputeDefaultedCopyCtorExceptionSpecAndConst(CXXRecordDecl *ClassDecl) {
   if (ClassDecl->isInvalidDecl())
@@ -7205,7 +8449,9 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
                                  /*TInfo=*/0,
                                  /*isExplicit=*/false,
                                  /*isInline=*/true,
-                                 /*isImplicitlyDeclared=*/true);
+                                 /*isImplicitlyDeclared=*/true,
+                                 // FIXME: apply the rules for definitions here
+                                 /*isConstexpr=*/false);
   CopyConstructor->setAccess(AS_public);
   CopyConstructor->setDefaulted();
   CopyConstructor->setTrivial(ClassDecl->hasTrivialCopyConstructor());
@@ -7220,7 +8466,7 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
                                                ArgType, /*TInfo=*/0,
                                                SC_None,
                                                SC_None, 0);
-  CopyConstructor->setParams(&FromParam, 1);
+  CopyConstructor->setParams(FromParam);
 
   if (Scope *S = getScopeForContext(ClassDecl))
     PushOnScopeChains(CopyConstructor, S, false);
@@ -7232,7 +8478,7 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   //   deleted; ...
   if (ClassDecl->hasUserDeclaredMoveConstructor() ||
       ClassDecl->hasUserDeclaredMoveAssignment() ||
-      ShouldDeleteCopyConstructor(CopyConstructor))
+      ShouldDeleteSpecialMember(CopyConstructor, CXXCopyConstructor))
     CopyConstructor->setDeletedAsWritten();
   
   return CopyConstructor;
@@ -7262,19 +8508,184 @@ void Sema::DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
                                                MultiStmtArg(*this, 0, 0), 
                                                /*isStmtExpr=*/false)
                                                               .takeAs<Stmt>());
+    CopyConstructor->setImplicitlyDefined(true);
   }
   
   CopyConstructor->setUsed();
-
   if (ASTMutationListener *L = getASTMutationListener()) {
     L->CompletedImplicitDefinition(CopyConstructor);
   }
 }
 
+Sema::ImplicitExceptionSpecification
+Sema::ComputeDefaultedMoveCtorExceptionSpec(CXXRecordDecl *ClassDecl) {
+  // C++ [except.spec]p14:
+  //   An implicitly declared special member function (Clause 12) shall have an 
+  //   exception-specification. [...]
+  ImplicitExceptionSpecification ExceptSpec(Context);
+  if (ClassDecl->isInvalidDecl())
+    return ExceptSpec;
+
+  // Direct base-class constructors.
+  for (CXXRecordDecl::base_class_iterator B = ClassDecl->bases_begin(),
+                                       BEnd = ClassDecl->bases_end();
+       B != BEnd; ++B) {
+    if (B->isVirtual()) // Handled below.
+      continue;
+    
+    if (const RecordType *BaseType = B->getType()->getAs<RecordType>()) {
+      CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getDecl());
+      CXXConstructorDecl *Constructor = LookupMovingConstructor(BaseClassDecl);
+      // If this is a deleted function, add it anyway. This might be conformant
+      // with the standard. This might not. I'm not sure. It might not matter.
+      if (Constructor)
+        ExceptSpec.CalledDecl(Constructor);
+    }
+  }
+
+  // Virtual base-class constructors.
+  for (CXXRecordDecl::base_class_iterator B = ClassDecl->vbases_begin(),
+                                       BEnd = ClassDecl->vbases_end();
+       B != BEnd; ++B) {
+    if (const RecordType *BaseType = B->getType()->getAs<RecordType>()) {
+      CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getDecl());
+      CXXConstructorDecl *Constructor = LookupMovingConstructor(BaseClassDecl);
+      // If this is a deleted function, add it anyway. This might be conformant
+      // with the standard. This might not. I'm not sure. It might not matter.
+      if (Constructor)
+        ExceptSpec.CalledDecl(Constructor);
+    }
+  }
+
+  // Field constructors.
+  for (RecordDecl::field_iterator F = ClassDecl->field_begin(),
+                               FEnd = ClassDecl->field_end();
+       F != FEnd; ++F) {
+    if (F->hasInClassInitializer()) {
+      if (Expr *E = F->getInClassInitializer())
+        ExceptSpec.CalledExpr(E);
+      else if (!F->isInvalidDecl())
+        ExceptSpec.SetDelayed();
+    } else if (const RecordType *RecordTy
+              = Context.getBaseElementType(F->getType())->getAs<RecordType>()) {
+      CXXRecordDecl *FieldRecDecl = cast<CXXRecordDecl>(RecordTy->getDecl());
+      CXXConstructorDecl *Constructor = LookupMovingConstructor(FieldRecDecl);
+      // If this is a deleted function, add it anyway. This might be conformant
+      // with the standard. This might not. I'm not sure. It might not matter.
+      // In particular, the problem is that this function never gets called. It
+      // might just be ill-formed because this function attempts to refer to
+      // a deleted function here.
+      if (Constructor)
+        ExceptSpec.CalledDecl(Constructor);
+    }
+  }
+
+  return ExceptSpec;
+}
+
+CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
+                                                    CXXRecordDecl *ClassDecl) {
+  ImplicitExceptionSpecification Spec(
+      ComputeDefaultedMoveCtorExceptionSpec(ClassDecl));
+
+  QualType ClassType = Context.getTypeDeclType(ClassDecl);
+  QualType ArgType = Context.getRValueReferenceType(ClassType);
+ 
+  FunctionProtoType::ExtProtoInfo EPI = Spec.getEPI();
+
+  DeclarationName Name
+    = Context.DeclarationNames.getCXXConstructorName(
+                                           Context.getCanonicalType(ClassType));
+  SourceLocation ClassLoc = ClassDecl->getLocation();
+  DeclarationNameInfo NameInfo(Name, ClassLoc);
+
+  // C++0x [class.copy]p11:
+  //   An implicitly-declared copy/move constructor is an inline public
+  //   member of its class. 
+  CXXConstructorDecl *MoveConstructor
+    = CXXConstructorDecl::Create(Context, ClassDecl, ClassLoc, NameInfo,
+                                 Context.getFunctionType(Context.VoidTy,
+                                                         &ArgType, 1, EPI),
+                                 /*TInfo=*/0,
+                                 /*isExplicit=*/false,
+                                 /*isInline=*/true,
+                                 /*isImplicitlyDeclared=*/true,
+                                 // FIXME: apply the rules for definitions here
+                                 /*isConstexpr=*/false);
+  MoveConstructor->setAccess(AS_public);
+  MoveConstructor->setDefaulted();
+  MoveConstructor->setTrivial(ClassDecl->hasTrivialMoveConstructor());
+  
+  // Add the parameter to the constructor.
+  ParmVarDecl *FromParam = ParmVarDecl::Create(Context, MoveConstructor,
+                                               ClassLoc, ClassLoc,
+                                               /*IdentifierInfo=*/0,
+                                               ArgType, /*TInfo=*/0,
+                                               SC_None,
+                                               SC_None, 0);
+  MoveConstructor->setParams(FromParam);
+
+  // C++0x [class.copy]p9:
+  //   If the definition of a class X does not explicitly declare a move
+  //   constructor, one will be implicitly declared as defaulted if and only if:
+  //   [...]
+  //   - the move constructor would not be implicitly defined as deleted.
+  if (ShouldDeleteSpecialMember(MoveConstructor, CXXMoveConstructor)) {
+    // Cache this result so that we don't try to generate this over and over
+    // on every lookup, leaking memory and wasting time.
+    ClassDecl->setFailedImplicitMoveConstructor();
+    return 0;
+  }
+
+  // Note that we have declared this constructor.
+  ++ASTContext::NumImplicitMoveConstructorsDeclared;
+
+  if (Scope *S = getScopeForContext(ClassDecl))
+    PushOnScopeChains(MoveConstructor, S, false);
+  ClassDecl->addDecl(MoveConstructor);
+
+  return MoveConstructor;
+}
+
+void Sema::DefineImplicitMoveConstructor(SourceLocation CurrentLocation,
+                                   CXXConstructorDecl *MoveConstructor) {
+  assert((MoveConstructor->isDefaulted() &&
+          MoveConstructor->isMoveConstructor() &&
+          !MoveConstructor->doesThisDeclarationHaveABody()) &&
+         "DefineImplicitMoveConstructor - call it for implicit move ctor");
+
+  CXXRecordDecl *ClassDecl = MoveConstructor->getParent();
+  assert(ClassDecl && "DefineImplicitMoveConstructor - invalid constructor");
+
+  ImplicitlyDefinedFunctionScope Scope(*this, MoveConstructor);
+  DiagnosticErrorTrap Trap(Diags);
+
+  if (SetCtorInitializers(MoveConstructor, 0, 0, /*AnyErrors=*/false) ||
+      Trap.hasErrorOccurred()) {
+    Diag(CurrentLocation, diag::note_member_synthesized_at) 
+      << CXXMoveConstructor << Context.getTagDeclType(ClassDecl);
+    MoveConstructor->setInvalidDecl();
+  }  else {
+    MoveConstructor->setBody(ActOnCompoundStmt(MoveConstructor->getLocation(),
+                                               MoveConstructor->getLocation(),
+                                               MultiStmtArg(*this, 0, 0), 
+                                               /*isStmtExpr=*/false)
+                                                              .takeAs<Stmt>());
+    MoveConstructor->setImplicitlyDefined(true);
+  }
+
+  MoveConstructor->setUsed();
+
+  if (ASTMutationListener *L = getASTMutationListener()) {
+    L->CompletedImplicitDefinition(MoveConstructor);
+  }
+}
+
 ExprResult
 Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             CXXConstructorDecl *Constructor,
                             MultiExprArg ExprArgs,
+                            bool HadMultipleCandidates,
                             bool RequiresZeroInit,
                             unsigned ConstructKind,
                             SourceRange ParenRange) {
@@ -7297,8 +8708,8 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
   }
 
   return BuildCXXConstructExpr(ConstructLoc, DeclInitType, Constructor,
-                               Elidable, move(ExprArgs), RequiresZeroInit,
-                               ConstructKind, ParenRange);
+                               Elidable, move(ExprArgs), HadMultipleCandidates,
+                               RequiresZeroInit, ConstructKind, ParenRange);
 }
 
 /// BuildCXXConstructExpr - Creates a complete call to a constructor,
@@ -7307,6 +8718,7 @@ ExprResult
 Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             CXXConstructorDecl *Constructor, bool Elidable,
                             MultiExprArg ExprArgs,
+                            bool HadMultipleCandidates,
                             bool RequiresZeroInit,
                             unsigned ConstructKind,
                             SourceRange ParenRange) {
@@ -7322,20 +8734,21 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
 
   MarkDeclarationReferenced(ConstructLoc, Constructor);
   return Owned(CXXConstructExpr::Create(Context, DeclInitType, ConstructLoc,
-                                        Constructor, Elidable, Exprs, NumExprs, 
-                                        RequiresZeroInit,
+                                        Constructor, Elidable, Exprs, NumExprs,
+                                        HadMultipleCandidates, RequiresZeroInit,
               static_cast<CXXConstructExpr::ConstructionKind>(ConstructKind),
                                         ParenRange));
 }
 
 bool Sema::InitializeVarWithConstructor(VarDecl *VD,
                                         CXXConstructorDecl *Constructor,
-                                        MultiExprArg Exprs) {
+                                        MultiExprArg Exprs,
+                                        bool HadMultipleCandidates) {
   // FIXME: Provide the correct paren SourceRange when available.
   ExprResult TempResult =
     BuildCXXConstructExpr(VD->getLocation(), VD->getType(), Constructor,
-                          move(Exprs), false, CXXConstructExpr::CK_Complete,
-                          SourceRange());
+                          move(Exprs), HadMultipleCandidates, false,
+                          CXXConstructExpr::CK_Complete, SourceRange());
   if (TempResult.isInvalid())
     return true;
 
@@ -7447,6 +8860,7 @@ void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
   // class type.
 
   if (!VDecl->getType()->isDependentType() &&
+      !VDecl->getType()->isIncompleteArrayType() && 
       RequireCompleteType(VDecl->getLocation(), VDecl->getType(),
                           diag::err_typecheck_decl_incomplete_type)) {
     VDecl->setInvalidDecl();
@@ -7522,18 +8936,34 @@ void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
     = InitializationKind::CreateDirect(VDecl->getLocation(),
                                        LParenLoc, RParenLoc);
   
+  QualType T = VDecl->getType();
   InitializationSequence InitSeq(*this, Entity, Kind, 
                                  Exprs.get(), Exprs.size());
-  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, move(Exprs));
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, move(Exprs), &T);
   if (Result.isInvalid()) {
     VDecl->setInvalidDecl();
     return;
+  } else if (T != VDecl->getType()) {
+    VDecl->setType(T);
+    Result.get()->setType(T);
   }
 
-  CheckImplicitConversions(Result.get(), LParenLoc);
   
-  Result = MaybeCreateExprWithCleanups(Result);
-  VDecl->setInit(Result.takeAs<Expr>());
+  Expr *Init = Result.get();
+  CheckImplicitConversions(Init, LParenLoc);
+  
+  if (VDecl->isConstexpr() && !VDecl->isInvalidDecl() &&
+      !Init->isValueDependent() &&
+      !Init->isConstantInitializer(Context,
+                                   VDecl->getType()->isReferenceType())) {
+    // FIXME: Improve this diagnostic to explain why the initializer is not
+    // a constant expression.
+    Diag(VDecl->getLocation(), diag::err_constexpr_var_requires_const_init)
+      << VDecl << Init->getSourceRange();
+  }
+
+  Init = MaybeCreateExprWithCleanups(Init);
+  VDecl->setInit(Init);
   VDecl->setCXXDirectInitializer(true);
 
   CheckCompleteVariableDeclaration(VDecl);
@@ -7566,7 +8996,7 @@ Sema::CompleteConstructorCall(CXXConstructorDecl *Constructor,
 
   VariadicCallType CallType = 
     Proto->isVariadic() ? VariadicConstructor : VariadicDoesNotApply;
-  llvm::SmallVector<Expr *, 8> AllArgs;
+  SmallVector<Expr *, 8> AllArgs;
   bool Invalid = GatherArgumentsForCall(Loc, Constructor,
                                         Proto, 0, Args, NumArgs, AllArgs, 
                                         CallType);
@@ -7928,6 +9358,30 @@ bool Sema::CheckLiteralOperatorDeclaration(FunctionDecl *FnDecl) {
     return true;
   }
 
+  StringRef LiteralName 
+    = FnDecl->getDeclName().getCXXLiteralIdentifier()->getName();
+  if (LiteralName[0] != '_') {
+    // C++0x [usrlit.suffix]p1:
+    //   Literal suffix identifiers that do not start with an underscore are 
+    //   reserved for future standardization.
+    bool IsHexFloat = true;
+    if (LiteralName.size() > 1 && 
+        (LiteralName[0] == 'P' || LiteralName[0] == 'p')) {
+      for (unsigned I = 1, N = LiteralName.size(); I < N; ++I) {
+        if (!isdigit(LiteralName[I])) {
+          IsHexFloat = false;
+          break;
+        }
+      }
+    }
+    
+    if (IsHexFloat)
+      Diag(FnDecl->getLocation(), diag::warn_user_literal_hexfloat)
+        << LiteralName;
+    else
+      Diag(FnDecl->getLocation(), diag::warn_user_literal_reserved);
+  }
+  
   return false;
 }
 
@@ -7940,7 +9394,7 @@ bool Sema::CheckLiteralOperatorDeclaration(FunctionDecl *FnDecl) {
 /// have any braces.
 Decl *Sema::ActOnStartLinkageSpecification(Scope *S, SourceLocation ExternLoc,
                                            SourceLocation LangLoc,
-                                           llvm::StringRef Lang,
+                                           StringRef Lang,
                                            SourceLocation LBraceLoc) {
   LinkageSpecDecl::LanguageIDs Language;
   if (Lang == "\"C\"")
@@ -8265,6 +9719,7 @@ Decl *Sema::ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc,
       return CheckClassTemplate(S, TagSpec, TUK_Friend, TagLoc,
                                 SS, Name, NameLoc, Attr,
                                 TemplateParams, AS_public,
+                                /*ModulePrivateLoc=*/SourceLocation(),
                                 TempParamLists.size() - 1,
                    (TemplateParameterList**) TempParamLists.release()).take();
     } else {
@@ -8427,7 +9882,7 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   return D;
 }
 
-Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
+Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D,
                                     MultiTemplateParamsArg TemplateParams) {
   const DeclSpec &DS = D.getDeclSpec();
 
@@ -8436,7 +9891,6 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
 
   SourceLocation Loc = D.getIdentifierLoc();
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
-  QualType T = TInfo->getType();
 
   // C++ [class.friend]p1
   //   A friend of a class is a function or class....
@@ -8448,7 +9902,7 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
   //   a declaration that does not use the syntactic form of a
   //   function declarator to have a function type, the program
   //   is ill-formed.
-  if (!T->isFunctionType()) {
+  if (!TInfo->getType()->isFunctionType()) {
     Diag(Loc, diag::err_unexpected_friend);
 
     // It might be worthwhile to try to recover by creating an
@@ -8551,6 +10005,14 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
 
     DCScope = getScopeForDeclContext(S, DC);
 
+    // C++ [class.friend]p6:
+    //   A function can be defined in a friend declaration of a class if and 
+    //   only if the class is a non-local class (9.8), the function name is
+    //   unqualified, and the function has namespace scope.
+    if (isLocal && D.isFunctionDefinition()) {
+      Diag(NameInfo.getBeginLoc(), diag::err_friend_def_in_local_class);
+    }
+    
   //   - There's a non-dependent scope specifier, in which case we
   //     compute it and do a previous lookup there for a function
   //     or function template.
@@ -8576,7 +10038,8 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
 
     if (Previous.empty()) {
       D.setInvalidType();
-      Diag(Loc, diag::err_qualified_friend_not_found) << Name << T;
+      Diag(Loc, diag::err_qualified_friend_not_found)
+          << Name << TInfo->getType();
       return 0;
     }
 
@@ -8584,6 +10047,20 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
     //   class that is not a member of the class . . .
     if (DC->Equals(CurContext))
       Diag(DS.getFriendSpecLoc(), diag::err_friend_is_member);
+    
+    if (D.isFunctionDefinition()) {
+      // C++ [class.friend]p6:
+      //   A function can be defined in a friend declaration of a class if and 
+      //   only if the class is a non-local class (9.8), the function name is
+      //   unqualified, and the function has namespace scope.
+      SemaDiagnosticBuilder DB
+        = Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def);
+      
+      DB << SS.getScopeRep();
+      if (DC->isFileContext())
+        DB << FixItHint::CreateRemoval(SS.getRange());
+      SS.clear();
+    }
 
   //   - There's a scope specifier that does not match any template
   //     parameter lists, in which case we use some arbitrary context,
@@ -8591,10 +10068,19 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
   //   - There's a scope specifier that does match some template
   //     parameter lists, which we don't handle right now.
   } else {
+    if (D.isFunctionDefinition()) {
+      // C++ [class.friend]p6:
+      //   A function can be defined in a friend declaration of a class if and 
+      //   only if the class is a non-local class (9.8), the function name is
+      //   unqualified, and the function has namespace scope.
+      Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def)
+        << SS.getScopeRep();
+    }
+    
     DC = CurContext;
     assert(isa<CXXRecordDecl>(DC) && "friend declaration not in class?");
   }
-
+  
   if (!DC->isRecord()) {
     // This implies that it has to be an operator or function.
     if (D.getName().getKind() == UnqualifiedId::IK_ConstructorName ||
@@ -8607,11 +10093,9 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
     }
   }
 
-  bool Redeclaration = false;
-  NamedDecl *ND = ActOnFunctionDeclarator(DCScope, D, DC, T, TInfo, Previous,
-                                          move(TemplateParams),
-                                          IsDefinition,
-                                          Redeclaration);
+  bool AddToScope = true;
+  NamedDecl *ND = ActOnFunctionDeclarator(DCScope, D, DC, TInfo, Previous,
+                                          move(TemplateParams), AddToScope);
   if (!ND) return 0;
 
   assert(ND->getDeclContext() == DC);
@@ -8732,15 +10216,24 @@ void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) {
       break;
     }
 
-    case CXXMoveConstructor:
-    case CXXMoveAssignment:
-      Diag(Dcl->getLocation(), diag::err_defaulted_move_unsupported);
+    case CXXMoveConstructor: {
+      CXXConstructorDecl *CD = cast<CXXConstructorDecl>(MD);
+      CheckExplicitlyDefaultedMoveConstructor(CD);
+      if (!CD->isInvalidDecl())
+        DefineImplicitMoveConstructor(DefaultLoc, CD);
       break;
+    }
 
-    default:
-      // FIXME: Do the rest once we have move functions
+    case CXXMoveAssignment: {
+      CheckExplicitlyDefaultedMoveAssignment(MD);
+      if (!MD->isInvalidDecl())
+        DefineImplicitMoveAssignment(DefaultLoc, MD);
       break;
     }
+
+    case CXXInvalid:
+      llvm_unreachable("Invalid special member.");
+    }
   } else {
     Diag(DefaultLoc, diag::err_default_special_members);
   }
@@ -8934,6 +10427,30 @@ DeclResult Sema::ActOnCXXConditionDeclaration(Scope *S, Declarator &D) {
   return Dcl;
 }
 
+void Sema::LoadExternalVTableUses() {
+  if (!ExternalSource)
+    return;
+  
+  SmallVector<ExternalVTableUse, 4> VTables;
+  ExternalSource->ReadUsedVTables(VTables);
+  SmallVector<VTableUse, 4> NewUses;
+  for (unsigned I = 0, N = VTables.size(); I != N; ++I) {
+    llvm::DenseMap<CXXRecordDecl *, bool>::iterator Pos
+      = VTablesUsed.find(VTables[I].Record);
+    // Even if a definition wasn't required before, it may be required now.
+    if (Pos != VTablesUsed.end()) {
+      if (!Pos->second && VTables[I].DefinitionRequired)
+        Pos->second = true;
+      continue;
+    }
+    
+    VTablesUsed[VTables[I].Record] = VTables[I].DefinitionRequired;
+    NewUses.push_back(VTableUse(VTables[I].Record, VTables[I].Location));
+  }
+  
+  VTableUses.insert(VTableUses.begin(), NewUses.begin(), NewUses.end());
+}
+
 void Sema::MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class,
                           bool DefinitionRequired) {
   // Ignore any vtable uses in unevaluated operands or for classes that do
@@ -8944,6 +10461,7 @@ void Sema::MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class,
     return;
 
   // Try to insert this class into the map.
+  LoadExternalVTableUses();
   Class = cast<CXXRecordDecl>(Class->getCanonicalDecl());
   std::pair<llvm::DenseMap<CXXRecordDecl *, bool>::iterator, bool>
     Pos = VTablesUsed.insert(std::make_pair(Class, DefinitionRequired));
@@ -8969,6 +10487,7 @@ void Sema::MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class,
 }
 
 bool Sema::DefineUsedVTables() {
+  LoadExternalVTableUses();
   if (VTableUses.empty())
     return false;
 
@@ -9037,7 +10556,10 @@ bool Sema::DefineUsedVTables() {
     // Optionally warn if we're emitting a weak vtable.
     if (Class->getLinkage() == ExternalLinkage &&
         Class->getTemplateSpecializationKind() != TSK_ImplicitInstantiation) {
-      if (!KeyFunction || (KeyFunction->hasBody() && KeyFunction->isInlined()))
+      const FunctionDecl *KeyFunctionDef = 0;
+      if (!KeyFunction || 
+          (KeyFunction->hasBody(KeyFunctionDef) && 
+           KeyFunctionDef->isInlined()))
         Diag(Class->getLocation(), diag::warn_weak_vtable) << Class;
     }
   }
@@ -9078,11 +10600,11 @@ void Sema::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
   if (!getLangOptions().CPlusPlus)
     return;
   if (ObjCInterfaceDecl *OID = ObjCImplementation->getClassInterface()) {
-    llvm::SmallVector<ObjCIvarDecl*, 8> ivars;
+    SmallVector<ObjCIvarDecl*, 8> ivars;
     CollectIvarsToConstructOrDestruct(OID, ivars);
     if (ivars.empty())
       return;
-    llvm::SmallVector<CXXCtorInitializer*, 32> AllToInit;
+    SmallVector<CXXCtorInitializer*, 32> AllToInit;
     for (unsigned i = 0; i < ivars.size(); i++) {
       FieldDecl *Field = ivars[i];
       if (Field->isInvalidDecl())
@@ -9199,8 +10721,8 @@ void Sema::CheckDelegatingCtorCycles() {
   llvm::SmallSet<CXXConstructorDecl*, 4>::iterator CI = Current.begin(),
                                                    CE = Current.end();
 
-  for (llvm::SmallVector<CXXConstructorDecl*, 4>::iterator
-         I = DelegatingCtorDecls.begin(),
+  for (DelegatingCtorDeclsType::iterator
+         I = DelegatingCtorDecls.begin(ExternalSource),
          E = DelegatingCtorDecls.end();
        I != E; ++I) {
    DelegatingCycleHelper(*I, Valid, Invalid, Current, *this);
@@ -9209,3 +10731,44 @@ void Sema::CheckDelegatingCtorCycles() {
   for (CI = Invalid.begin(), CE = Invalid.end(); CI != CE; ++CI)
     (*CI)->setInvalidDecl();
 }
+
+/// IdentifyCUDATarget - Determine the CUDA compilation target for this function
+Sema::CUDAFunctionTarget Sema::IdentifyCUDATarget(const FunctionDecl *D) {
+  // Implicitly declared functions (e.g. copy constructors) are
+  // __host__ __device__
+  if (D->isImplicit())
+    return CFT_HostDevice;
+
+  if (D->hasAttr<CUDAGlobalAttr>())
+    return CFT_Global;
+
+  if (D->hasAttr<CUDADeviceAttr>()) {
+    if (D->hasAttr<CUDAHostAttr>())
+      return CFT_HostDevice;
+    else
+      return CFT_Device;
+  }
+
+  return CFT_Host;
+}
+
+bool Sema::CheckCUDATarget(CUDAFunctionTarget CallerTarget,
+                           CUDAFunctionTarget CalleeTarget) {
+  // CUDA B.1.1 "The __device__ qualifier declares a function that is...
+  // Callable from the device only."
+  if (CallerTarget == CFT_Host && CalleeTarget == CFT_Device)
+    return true;
+
+  // CUDA B.1.2 "The __global__ qualifier declares a function that is...
+  // Callable from the host only."
+  // CUDA B.1.3 "The __host__ qualifier declares a function that is...
+  // Callable from the host only."
+  if ((CallerTarget == CFT_Device || CallerTarget == CFT_Global) &&
+      (CalleeTarget == CFT_Host || CalleeTarget == CFT_Global))
+    return true;
+
+  if (CallerTarget == CFT_HostDevice && CalleeTarget != CFT_HostDevice)
+    return true;
+
+  return false;
+}
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp
index aa8152b03b0a..62b4a7c0cc77 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp
@@ -106,7 +106,7 @@ bool Sema::checkInitMethod(ObjCMethodDecl *method,
   return true;
 }
 
-bool Sema::CheckObjCMethodOverride(ObjCMethodDecl *NewMethod, 
+void Sema::CheckObjCMethodOverride(ObjCMethodDecl *NewMethod, 
                                    const ObjCMethodDecl *Overridden,
                                    bool IsImplementation) {
   if (Overridden->hasRelatedResultType() && 
@@ -148,105 +148,44 @@ bool Sema::CheckObjCMethodOverride(ObjCMethodDecl *NewMethod,
         << ResultTypeRange;
     }
     
-    Diag(Overridden->getLocation(), diag::note_related_result_type_overridden)
-      << Overridden->getMethodFamily();
+    if (ObjCMethodFamily Family = Overridden->getMethodFamily())
+      Diag(Overridden->getLocation(), 
+           diag::note_related_result_type_overridden_family)
+        << Family;
+    else
+      Diag(Overridden->getLocation(), 
+           diag::note_related_result_type_overridden);
   }
-  
-  return false;
-}
-
-/// \brief Check for consistency between a given method declaration and the
-/// methods it overrides within the class hierarchy.
-///
-/// This method walks the inheritance hierarchy starting at the given 
-/// declaration context (\p DC), invoking Sema::CheckObjCMethodOverride() with
-/// the given new method (\p NewMethod) and any method it directly overrides
-/// in the hierarchy. Sema::CheckObjCMethodOverride() is responsible for
-/// checking consistency, e.g., among return types for methods that return a 
-/// related result type.
-static bool CheckObjCMethodOverrides(Sema &S, ObjCMethodDecl *NewMethod,
-                                     DeclContext *DC, 
-                                     bool SkipCurrent = true) {
-  if (!DC)
-    return false;
-  
-  if (!SkipCurrent) {
-    // Look for this method. If we find it, we're done.
-    Selector Sel = NewMethod->getSelector();
-    bool IsInstance = NewMethod->isInstanceMethod();
-    DeclContext::lookup_const_iterator Meth, MethEnd;
-    for (llvm::tie(Meth, MethEnd) = DC->lookup(Sel); Meth != MethEnd; ++Meth) {
-      ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(*Meth);
-      if (MD && MD->isInstanceMethod() == IsInstance)
-        return S.CheckObjCMethodOverride(NewMethod, MD, false);
+  if (getLangOptions().ObjCAutoRefCount) {
+    if ((NewMethod->hasAttr<NSReturnsRetainedAttr>() !=
+         Overridden->hasAttr<NSReturnsRetainedAttr>())) {
+        Diag(NewMethod->getLocation(),
+             diag::err_nsreturns_retained_attribute_mismatch) << 1;
+        Diag(Overridden->getLocation(), diag::note_previous_decl) 
+        << "method";
+    }
+    if ((NewMethod->hasAttr<NSReturnsNotRetainedAttr>() !=
+              Overridden->hasAttr<NSReturnsNotRetainedAttr>())) {
+        Diag(NewMethod->getLocation(),
+             diag::err_nsreturns_retained_attribute_mismatch) << 0;
+        Diag(Overridden->getLocation(), diag::note_previous_decl) 
+        << "method";
+    }
+    ObjCMethodDecl::param_const_iterator oi = Overridden->param_begin();
+    for (ObjCMethodDecl::param_iterator
+           ni = NewMethod->param_begin(), ne = NewMethod->param_end();
+         ni != ne; ++ni, ++oi) {
+      const ParmVarDecl *oldDecl = (*oi);
+      ParmVarDecl *newDecl = (*ni);
+      if (newDecl->hasAttr<NSConsumedAttr>() != 
+          oldDecl->hasAttr<NSConsumedAttr>()) {
+        Diag(newDecl->getLocation(),
+             diag::err_nsconsumed_attribute_mismatch);
+        Diag(oldDecl->getLocation(), diag::note_previous_decl) 
+          << "parameter";
+      }
     }
   }
-  
-  if (ObjCInterfaceDecl *Class = llvm::dyn_cast<ObjCInterfaceDecl>(DC)) {
-    // Look through categories.
-    for (ObjCCategoryDecl *Category = Class->getCategoryList();
-         Category; Category = Category->getNextClassCategory()) {
-      if (CheckObjCMethodOverrides(S, NewMethod, Category, false))
-        return true;
-    }
-
-    // Look through protocols.
-    for (ObjCList<ObjCProtocolDecl>::iterator I = Class->protocol_begin(),
-                                           IEnd = Class->protocol_end();
-         I != IEnd; ++I)
-      if (CheckObjCMethodOverrides(S, NewMethod, *I, false))
-        return true;
-    
-    // Look in our superclass.
-    return CheckObjCMethodOverrides(S, NewMethod, Class->getSuperClass(), 
-                                    false);
-  }
-  
-  if (ObjCCategoryDecl *Category = dyn_cast<ObjCCategoryDecl>(DC)) {
-    // Look through protocols.
-    for (ObjCList<ObjCProtocolDecl>::iterator I = Category->protocol_begin(),
-                                           IEnd = Category->protocol_end();
-         I != IEnd; ++I)
-      if (CheckObjCMethodOverrides(S, NewMethod, *I, false))
-        return true;
-    
-    return false;
-  }
-  
-  if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(DC)) {
-    // Look through protocols.
-    for (ObjCList<ObjCProtocolDecl>::iterator I = Protocol->protocol_begin(),
-                                           IEnd = Protocol->protocol_end();
-         I != IEnd; ++I)
-      if (CheckObjCMethodOverrides(S, NewMethod, *I, false))
-        return true;
-    
-    return false;
-  }
-  
-  return false;
-}
-
-bool Sema::CheckObjCMethodOverrides(ObjCMethodDecl *NewMethod, 
-                                    DeclContext *DC) {
-  if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(DC))
-    return ::CheckObjCMethodOverrides(*this, NewMethod, Class);
-
-  if (ObjCCategoryDecl *Category = dyn_cast<ObjCCategoryDecl>(DC))
-    return ::CheckObjCMethodOverrides(*this, NewMethod, Category);
-
-  if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(DC))
-    return ::CheckObjCMethodOverrides(*this, NewMethod, Protocol);
-
-  if (ObjCImplementationDecl *Impl = dyn_cast<ObjCImplementationDecl>(DC))
-    return ::CheckObjCMethodOverrides(*this, NewMethod, 
-                                      Impl->getClassInterface());
-  
-  if (ObjCCategoryImplDecl *CatImpl = dyn_cast<ObjCCategoryImplDecl>(DC))
-    return ::CheckObjCMethodOverrides(*this, NewMethod, 
-                                      CatImpl->getClassInterface());
-  
-  return ::CheckObjCMethodOverrides(*this, NewMethod, CurContext);
 }
 
 /// \brief Check a method declaration for compatibility with the Objective-C
@@ -256,11 +195,13 @@ static bool CheckARCMethodDecl(Sema &S, ObjCMethodDecl *method) {
   switch (family) {
   case OMF_None:
   case OMF_dealloc:
+  case OMF_finalize:
   case OMF_retain:
   case OMF_release:
   case OMF_autorelease:
   case OMF_retainCount:
   case OMF_self:
+  case OMF_performSelector:
     return false;
 
   case OMF_init:
@@ -286,11 +227,6 @@ static bool CheckARCMethodDecl(Sema &S, ObjCMethodDecl *method) {
         method->hasAttr<NSReturnsAutoreleasedAttr>())
       return false;
     break;
-    
-  case OMF_performSelector:
-    // we don't annotate performSelector's
-    return true;
-      
   }
 
   method->addAttr(new (S.Context) NSReturnsRetainedAttr(SourceLocation(),
@@ -311,6 +247,20 @@ static void DiagnoseObjCImplementedDeprecations(Sema &S,
   }
 }
 
+/// AddAnyMethodToGlobalPool - Add any method, instance or factory to global
+/// pool.
+void Sema::AddAnyMethodToGlobalPool(Decl *D) {
+  ObjCMethodDecl *MDecl = dyn_cast_or_null<ObjCMethodDecl>(D);
+    
+  // If we don't have a valid method decl, simply return.
+  if (!MDecl)
+    return;
+  if (MDecl->isInstanceMethod())
+    AddInstanceMethodToGlobalPool(MDecl, true);
+  else
+    AddFactoryMethodToGlobalPool(MDecl, true);
+}
+
 /// ActOnStartOfObjCMethodDef - This routine sets up parameters; invisible
 /// and user declared, in the method definition's AST.
 void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
@@ -321,12 +271,6 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
   if (!MDecl)
     return;
 
-  // Allow the rest of sema to find private method decl implementations.
-  if (MDecl->isInstanceMethod())
-    AddInstanceMethodToGlobalPool(MDecl, true);
-  else
-    AddFactoryMethodToGlobalPool(MDecl, true);
-  
   // Allow all of Sema to see that we are entering a method definition.
   PushDeclContext(FnBodyScope, MDecl);
   PushFunctionScope();
@@ -365,6 +309,7 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
 
     case OMF_None:
     case OMF_dealloc:
+    case OMF_finalize:
     case OMF_alloc:
     case OMF_init:
     case OMF_mutableCopy:
@@ -376,14 +321,29 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
     }
   }
 
-  // Warn on implementating deprecated methods under 
-  // -Wdeprecated-implementations flag.
-  if (ObjCInterfaceDecl *IC = MDecl->getClassInterface())
+  // Warn on deprecated methods under -Wdeprecated-implementations,
+  // and prepare for warning on missing super calls.
+  if (ObjCInterfaceDecl *IC = MDecl->getClassInterface()) {
     if (ObjCMethodDecl *IMD = 
           IC->lookupMethod(MDecl->getSelector(), MDecl->isInstanceMethod()))
       DiagnoseObjCImplementedDeprecations(*this, 
                                           dyn_cast<NamedDecl>(IMD), 
                                           MDecl->getLocation(), 0);
+
+    // If this is "dealloc" or "finalize", set some bit here.
+    // Then in ActOnSuperMessage() (SemaExprObjC), set it back to false.
+    // Finally, in ActOnFinishFunctionBody() (SemaDecl), warn if flag is set.
+    // Only do this if the current class actually has a superclass.
+    if (IC->getSuperClass()) {
+      ObjCShouldCallSuperDealloc = 
+        !(Context.getLangOptions().ObjCAutoRefCount ||
+          Context.getLangOptions().getGC() == LangOptions::GCOnly) &&
+        MDecl->getMethodFamily() == OMF_dealloc;
+      ObjCShouldCallSuperFinalize =
+        Context.getLangOptions().getGC() != LangOptions::NonGC &&
+        MDecl->getMethodFamily() == OMF_finalize;
+    }
+  }
 }
 
 Decl *Sema::
@@ -414,11 +374,11 @@ ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
 
       // Return the previous class interface.
       // FIXME: don't leak the objects passed in!
-      return IDecl;
+      return ActOnObjCContainerStartDefinition(IDecl);
     } else {
-      IDecl->setLocation(AtInterfaceLoc);
+      IDecl->setLocation(ClassLoc);
       IDecl->setForwardDecl(false);
-      IDecl->setClassLoc(ClassLoc);
+      IDecl->setAtStartLoc(AtInterfaceLoc);
       // If the forward decl was in a PCH, we need to write it again in a
       // dependent AST file.
       IDecl->setChangedSinceDeserialization(true);
@@ -522,7 +482,7 @@ ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
   }
 
   CheckObjCDeclScope(IDecl);
-  return IDecl;
+  return ActOnObjCContainerStartDefinition(IDecl);
 }
 
 /// ActOnCompatiblityAlias - this action is called after complete parsing of
@@ -618,7 +578,7 @@ Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
       Diag(PDecl->getLocation(), diag::note_previous_definition);
       // Just return the protocol we already had.
       // FIXME: don't leak the objects passed in!
-      return PDecl;
+      return ActOnObjCContainerStartDefinition(PDecl);
     }
     ObjCList<ObjCProtocolDecl> PList;
     PList.set((ObjCProtocolDecl *const*)ProtoRefs, NumProtoRefs, Context);
@@ -626,14 +586,18 @@ Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
             ProtocolName, ProtocolLoc, PDecl->getLocation(), PList);
 
     // Make sure the cached decl gets a valid start location.
-    PDecl->setLocation(AtProtoInterfaceLoc);
+    PDecl->setAtStartLoc(AtProtoInterfaceLoc);
+    PDecl->setLocation(ProtocolLoc);
     PDecl->setForwardDecl(false);
+    // Since this ObjCProtocolDecl was created by a forward declaration,
+    // we now add it to the DeclContext since it wasn't added before
+    PDecl->setLexicalDeclContext(CurContext);
     CurContext->addDecl(PDecl);
     // Repeat in dependent AST files.
     PDecl->setChangedSinceDeserialization(true);
   } else {
-    PDecl = ObjCProtocolDecl::Create(Context, CurContext,
-                                     AtProtoInterfaceLoc,ProtocolName);
+    PDecl = ObjCProtocolDecl::Create(Context, CurContext, ProtocolName,
+                                     ProtocolLoc, AtProtoInterfaceLoc);
     PushOnScopeChains(PDecl, TUScope);
     PDecl->setForwardDecl(false);
   }
@@ -647,7 +611,7 @@ Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
   }
 
   CheckObjCDeclScope(PDecl);
-  return PDecl;
+  return ActOnObjCContainerStartDefinition(PDecl);
 }
 
 /// FindProtocolDeclaration - This routine looks up protocols and
@@ -657,7 +621,7 @@ void
 Sema::FindProtocolDeclaration(bool WarnOnDeclarations,
                               const IdentifierLocPair *ProtocolId,
                               unsigned NumProtocols,
-                              llvm::SmallVectorImpl<Decl *> &Protocols) {
+                              SmallVectorImpl<Decl *> &Protocols) {
   for (unsigned i = 0; i != NumProtocols; ++i) {
     ObjCProtocolDecl *PDecl = LookupProtocol(ProtocolId[i].first,
                                              ProtocolId[i].second);
@@ -725,16 +689,16 @@ Sema::ActOnForwardProtocolDeclaration(SourceLocation AtProtocolLoc,
                                       const IdentifierLocPair *IdentList,
                                       unsigned NumElts,
                                       AttributeList *attrList) {
-  llvm::SmallVector<ObjCProtocolDecl*, 32> Protocols;
-  llvm::SmallVector<SourceLocation, 8> ProtoLocs;
+  SmallVector<ObjCProtocolDecl*, 32> Protocols;
+  SmallVector<SourceLocation, 8> ProtoLocs;
 
   for (unsigned i = 0; i != NumElts; ++i) {
     IdentifierInfo *Ident = IdentList[i].first;
     ObjCProtocolDecl *PDecl = LookupProtocol(Ident, IdentList[i].second);
     bool isNew = false;
     if (PDecl == 0) { // Not already seen?
-      PDecl = ObjCProtocolDecl::Create(Context, CurContext,
-                                       IdentList[i].second, Ident);
+      PDecl = ObjCProtocolDecl::Create(Context, CurContext, Ident,
+                                       IdentList[i].second, AtProtocolLoc);
       PushOnScopeChains(PDecl, TUScope, false);
       isNew = true;
     }
@@ -774,10 +738,10 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
     // the enclosing method declarations.  We mark the decl invalid
     // to make it clear that this isn't a valid AST.
     CDecl = ObjCCategoryDecl::Create(Context, CurContext, AtInterfaceLoc,
-                                     ClassLoc, CategoryLoc, CategoryName);
+                                     ClassLoc, CategoryLoc, CategoryName,IDecl);
     CDecl->setInvalidDecl();
     Diag(ClassLoc, diag::err_undef_interface) << ClassName;
-    return CDecl;
+    return ActOnObjCContainerStartDefinition(CDecl);
   }
 
   if (!CategoryName && IDecl->getImplementation()) {
@@ -786,19 +750,6 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
           diag::note_implementation_declared);
   }
 
-  CDecl = ObjCCategoryDecl::Create(Context, CurContext, AtInterfaceLoc,
-                                   ClassLoc, CategoryLoc, CategoryName);
-  // FIXME: PushOnScopeChains?
-  CurContext->addDecl(CDecl);
-
-  CDecl->setClassInterface(IDecl);
-  // Insert class extension to the list of class's categories.
-  if (!CategoryName)
-    CDecl->insertNextClassCategory();
-
-  // If the interface is deprecated, warn about it.
-  (void)DiagnoseUseOfDecl(IDecl, ClassLoc);
-
   if (CategoryName) {
     /// Check for duplicate interface declaration for this category
     ObjCCategoryDecl *CDeclChain;
@@ -812,10 +763,13 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
         break;
       }
     }
-    if (!CDeclChain)
-      CDecl->insertNextClassCategory();
   }
 
+  CDecl = ObjCCategoryDecl::Create(Context, CurContext, AtInterfaceLoc,
+                                   ClassLoc, CategoryLoc, CategoryName, IDecl);
+  // FIXME: PushOnScopeChains?
+  CurContext->addDecl(CDecl);
+
   if (NumProtoRefs) {
     CDecl->setProtocolList((ObjCProtocolDecl**)ProtoRefs, NumProtoRefs, 
                            ProtoLocs, Context);
@@ -826,7 +780,7 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
   }
 
   CheckObjCDeclScope(CDecl);
-  return CDecl;
+  return ActOnObjCContainerStartDefinition(CDecl);
 }
 
 /// ActOnStartCategoryImplementation - Perform semantic checks on the
@@ -845,22 +799,26 @@ Decl *Sema::ActOnStartCategoryImplementation(
       // Create and install one.
       CatIDecl = ObjCCategoryDecl::Create(Context, CurContext, SourceLocation(),
                                           SourceLocation(), SourceLocation(),
-                                          CatName);
-      CatIDecl->setClassInterface(IDecl);
-      CatIDecl->insertNextClassCategory();
+                                          CatName, IDecl);
     }
   }
 
   ObjCCategoryImplDecl *CDecl =
-    ObjCCategoryImplDecl::Create(Context, CurContext, AtCatImplLoc, CatName,
-                                 IDecl);
+    ObjCCategoryImplDecl::Create(Context, CurContext, CatName, IDecl,
+                                 ClassLoc, AtCatImplLoc);
   /// Check that class of this category is already completely declared.
-  if (!IDecl || IDecl->isForwardDecl())
+  if (!IDecl || IDecl->isForwardDecl()) {
     Diag(ClassLoc, diag::err_undef_interface) << ClassName;
+    CDecl->setInvalidDecl();
+  }
 
   // FIXME: PushOnScopeChains?
   CurContext->addDecl(CDecl);
 
+  // If the interface is deprecated/unavailable, warn/error about it.
+  if (IDecl)
+    DiagnoseUseOfDecl(IDecl, ClassLoc);
+
   /// Check that CatName, category name, is not used in another implementation.
   if (CatIDecl) {
     if (CatIDecl->getImplementation()) {
@@ -879,7 +837,7 @@ Decl *Sema::ActOnStartCategoryImplementation(
   }
 
   CheckObjCDeclScope(CDecl);
-  return CDecl;
+  return ActOnObjCContainerStartDefinition(CDecl);
 }
 
 Decl *Sema::ActOnStartClassImplementation(
@@ -969,11 +927,11 @@ Decl *Sema::ActOnStartClassImplementation(
   }
 
   ObjCImplementationDecl* IMPDecl =
-    ObjCImplementationDecl::Create(Context, CurContext, AtClassImplLoc,
-                                   IDecl, SDecl);
+    ObjCImplementationDecl::Create(Context, CurContext, IDecl, SDecl,
+                                   ClassLoc, AtClassImplLoc);
 
   if (CheckObjCDeclScope(IMPDecl))
-    return IMPDecl;
+    return ActOnObjCContainerStartDefinition(IMPDecl);
 
   // Check that there is no duplicate implementation of this class.
   if (IDecl->getImplementation()) {
@@ -990,7 +948,7 @@ Decl *Sema::ActOnStartClassImplementation(
                                         dyn_cast<NamedDecl>(IDecl), 
                                         IMPDecl->getLocation(), 1);
   }
-  return IMPDecl;
+  return ActOnObjCContainerStartDefinition(IMPDecl);
 }
 
 void Sema::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
@@ -1050,21 +1008,18 @@ void Sema::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
     assert (ClsIvar && "missing class ivar");
 
     // First, make sure the types match.
-    if (Context.getCanonicalType(ImplIvar->getType()) !=
-        Context.getCanonicalType(ClsIvar->getType())) {
+    if (!Context.hasSameType(ImplIvar->getType(), ClsIvar->getType())) {
       Diag(ImplIvar->getLocation(), diag::err_conflicting_ivar_type)
         << ImplIvar->getIdentifier()
         << ImplIvar->getType() << ClsIvar->getType();
       Diag(ClsIvar->getLocation(), diag::note_previous_definition);
-    } else if (ImplIvar->isBitField() && ClsIvar->isBitField()) {
-      Expr *ImplBitWidth = ImplIvar->getBitWidth();
-      Expr *ClsBitWidth = ClsIvar->getBitWidth();
-      if (ImplBitWidth->EvaluateAsInt(Context).getZExtValue() !=
-          ClsBitWidth->EvaluateAsInt(Context).getZExtValue()) {
-        Diag(ImplBitWidth->getLocStart(), diag::err_conflicting_ivar_bitwidth)
-          << ImplIvar->getIdentifier();
-        Diag(ClsBitWidth->getLocStart(), diag::note_previous_definition);
-      }
+    } else if (ImplIvar->isBitField() && ClsIvar->isBitField() &&
+               ImplIvar->getBitWidthValue(Context) !=
+               ClsIvar->getBitWidthValue(Context)) {
+      Diag(ImplIvar->getBitWidth()->getLocStart(),
+           diag::err_conflicting_ivar_bitwidth) << ImplIvar->getIdentifier();
+      Diag(ClsIvar->getBitWidth()->getLocStart(),
+           diag::note_previous_definition);
     }
     // Make sure the names are identical.
     if (ImplIvar->getIdentifier() != ClsIvar->getIdentifier()) {
@@ -1167,26 +1122,38 @@ static SourceRange getTypeRange(TypeSourceInfo *TSI) {
   return (TSI ? TSI->getTypeLoc().getSourceRange() : SourceRange());
 }
 
-static void CheckMethodOverrideReturn(Sema &S,
+static bool CheckMethodOverrideReturn(Sema &S,
                                       ObjCMethodDecl *MethodImpl,
                                       ObjCMethodDecl *MethodDecl,
-                                      bool IsProtocolMethodDecl) {
+                                      bool IsProtocolMethodDecl,
+                                      bool IsOverridingMode,
+                                      bool Warn) {
   if (IsProtocolMethodDecl &&
       (MethodDecl->getObjCDeclQualifier() !=
        MethodImpl->getObjCDeclQualifier())) {
-    S.Diag(MethodImpl->getLocation(), 
-           diag::warn_conflicting_ret_type_modifiers)
-        << MethodImpl->getDeclName()
-        << getTypeRange(MethodImpl->getResultTypeSourceInfo());
-    S.Diag(MethodDecl->getLocation(), diag::note_previous_declaration)
-        << getTypeRange(MethodDecl->getResultTypeSourceInfo());
+    if (Warn) {
+        S.Diag(MethodImpl->getLocation(), 
+               (IsOverridingMode ? 
+                 diag::warn_conflicting_overriding_ret_type_modifiers 
+                 : diag::warn_conflicting_ret_type_modifiers))
+          << MethodImpl->getDeclName()
+          << getTypeRange(MethodImpl->getResultTypeSourceInfo());
+        S.Diag(MethodDecl->getLocation(), diag::note_previous_declaration)
+          << getTypeRange(MethodDecl->getResultTypeSourceInfo());
+    }
+    else
+      return false;
   }
   
   if (S.Context.hasSameUnqualifiedType(MethodImpl->getResultType(),
                                        MethodDecl->getResultType()))
-    return;
+    return true;
+  if (!Warn)
+    return false;
 
-  unsigned DiagID = diag::warn_conflicting_ret_types;
+  unsigned DiagID = 
+    IsOverridingMode ? diag::warn_conflicting_overriding_ret_types 
+                     : diag::warn_conflicting_ret_types;
 
   // Mismatches between ObjC pointers go into a different warning
   // category, and sometimes they're even completely whitelisted.
@@ -1199,9 +1166,11 @@ static void CheckMethodOverrideReturn(Sema &S,
       // return types that are subclasses of the declared return type,
       // or that are more-qualified versions of the declared type.
       if (isObjCTypeSubstitutable(S.Context, IfacePtrTy, ImplPtrTy, false))
-        return;
+        return false;
 
-      DiagID = diag::warn_non_covariant_ret_types;
+      DiagID = 
+        IsOverridingMode ? diag::warn_non_covariant_overriding_ret_types 
+                          : diag::warn_non_covariant_ret_types;
     }
   }
 
@@ -1210,34 +1179,52 @@ static void CheckMethodOverrideReturn(Sema &S,
     << MethodDecl->getResultType()
     << MethodImpl->getResultType()
     << getTypeRange(MethodImpl->getResultTypeSourceInfo());
-  S.Diag(MethodDecl->getLocation(), diag::note_previous_definition)
+  S.Diag(MethodDecl->getLocation(), 
+         IsOverridingMode ? diag::note_previous_declaration 
+                          : diag::note_previous_definition)
     << getTypeRange(MethodDecl->getResultTypeSourceInfo());
+  return false;
 }
 
-static void CheckMethodOverrideParam(Sema &S,
+static bool CheckMethodOverrideParam(Sema &S,
                                      ObjCMethodDecl *MethodImpl,
                                      ObjCMethodDecl *MethodDecl,
                                      ParmVarDecl *ImplVar,
                                      ParmVarDecl *IfaceVar,
-                                     bool IsProtocolMethodDecl) {
+                                     bool IsProtocolMethodDecl,
+                                     bool IsOverridingMode,
+                                     bool Warn) {
   if (IsProtocolMethodDecl &&
       (ImplVar->getObjCDeclQualifier() !=
        IfaceVar->getObjCDeclQualifier())) {
-    S.Diag(ImplVar->getLocation(), 
-           diag::warn_conflicting_param_modifiers)
-        << getTypeRange(ImplVar->getTypeSourceInfo())
-        << MethodImpl->getDeclName();
-    S.Diag(IfaceVar->getLocation(), diag::note_previous_declaration)
-        << getTypeRange(IfaceVar->getTypeSourceInfo());   
+    if (Warn) {
+      if (IsOverridingMode)
+        S.Diag(ImplVar->getLocation(), 
+               diag::warn_conflicting_overriding_param_modifiers)
+            << getTypeRange(ImplVar->getTypeSourceInfo())
+            << MethodImpl->getDeclName();
+      else S.Diag(ImplVar->getLocation(), 
+             diag::warn_conflicting_param_modifiers)
+          << getTypeRange(ImplVar->getTypeSourceInfo())
+          << MethodImpl->getDeclName();
+      S.Diag(IfaceVar->getLocation(), diag::note_previous_declaration)
+          << getTypeRange(IfaceVar->getTypeSourceInfo());   
+    }
+    else
+      return false;
   }
       
   QualType ImplTy = ImplVar->getType();
   QualType IfaceTy = IfaceVar->getType();
   
   if (S.Context.hasSameUnqualifiedType(ImplTy, IfaceTy))
-    return;
-
-  unsigned DiagID = diag::warn_conflicting_param_types;
+    return true;
+  
+  if (!Warn)
+    return false;
+  unsigned DiagID = 
+    IsOverridingMode ? diag::warn_conflicting_overriding_param_types 
+                     : diag::warn_conflicting_param_types;
 
   // Mismatches between ObjC pointers go into a different warning
   // category, and sometimes they're even completely whitelisted.
@@ -1250,17 +1237,22 @@ static void CheckMethodOverrideParam(Sema &S,
       // implementation must accept any objects that the superclass
       // accepts, however it may also accept others.
       if (isObjCTypeSubstitutable(S.Context, ImplPtrTy, IfacePtrTy, true))
-        return;
+        return false;
 
-      DiagID = diag::warn_non_contravariant_param_types;
+      DiagID = 
+      IsOverridingMode ? diag::warn_non_contravariant_overriding_param_types 
+                       :  diag::warn_non_contravariant_param_types;
     }
   }
 
   S.Diag(ImplVar->getLocation(), DiagID)
     << getTypeRange(ImplVar->getTypeSourceInfo())
     << MethodImpl->getDeclName() << IfaceTy << ImplTy;
-  S.Diag(IfaceVar->getLocation(), diag::note_previous_definition)
+  S.Diag(IfaceVar->getLocation(), 
+         (IsOverridingMode ? diag::note_previous_declaration 
+                        : diag::note_previous_definition))
     << getTypeRange(IfaceVar->getTypeSourceInfo());
+  return false;
 }
 
 /// In ARC, check whether the conventional meanings of the two methods
@@ -1302,6 +1294,7 @@ static bool checkMethodFamilyMismatch(Sema &S, ObjCMethodDecl *impl,
   case OMF_release:
   case OMF_autorelease:
   case OMF_dealloc:
+  case OMF_finalize:
   case OMF_retainCount:
   case OMF_self:
   case OMF_performSelector:
@@ -1341,20 +1334,86 @@ void Sema::WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethodDecl,
     return;
 
   CheckMethodOverrideReturn(*this, ImpMethodDecl, MethodDecl, 
-                            IsProtocolMethodDecl);
+                            IsProtocolMethodDecl, false, 
+                            true);
 
   for (ObjCMethodDecl::param_iterator IM = ImpMethodDecl->param_begin(),
        IF = MethodDecl->param_begin(), EM = ImpMethodDecl->param_end();
-       IM != EM; ++IM, ++IF)
+       IM != EM; ++IM, ++IF) {
     CheckMethodOverrideParam(*this, ImpMethodDecl, MethodDecl, *IM, *IF,
-                             IsProtocolMethodDecl);
+                             IsProtocolMethodDecl, false, true);
+  }
 
   if (ImpMethodDecl->isVariadic() != MethodDecl->isVariadic()) {
-    Diag(ImpMethodDecl->getLocation(), diag::warn_conflicting_variadic);
+    Diag(ImpMethodDecl->getLocation(), 
+         diag::warn_conflicting_variadic);
     Diag(MethodDecl->getLocation(), diag::note_previous_declaration);
   }
 }
 
+void Sema::CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
+                                       ObjCMethodDecl *Overridden,
+                                       bool IsProtocolMethodDecl) {
+  
+  CheckMethodOverrideReturn(*this, Method, Overridden, 
+                            IsProtocolMethodDecl, true, 
+                            true);
+  
+  for (ObjCMethodDecl::param_iterator IM = Method->param_begin(),
+       IF = Overridden->param_begin(), EM = Method->param_end();
+       IM != EM; ++IM, ++IF) {
+    CheckMethodOverrideParam(*this, Method, Overridden, *IM, *IF,
+                             IsProtocolMethodDecl, true, true);
+  }
+  
+  if (Method->isVariadic() != Overridden->isVariadic()) {
+    Diag(Method->getLocation(), 
+         diag::warn_conflicting_overriding_variadic);
+    Diag(Overridden->getLocation(), diag::note_previous_declaration);
+  }
+}
+
+/// WarnExactTypedMethods - This routine issues a warning if method
+/// implementation declaration matches exactly that of its declaration.
+void Sema::WarnExactTypedMethods(ObjCMethodDecl *ImpMethodDecl,
+                                 ObjCMethodDecl *MethodDecl,
+                                 bool IsProtocolMethodDecl) {
+  // don't issue warning when protocol method is optional because primary
+  // class is not required to implement it and it is safe for protocol
+  // to implement it.
+  if (MethodDecl->getImplementationControl() == ObjCMethodDecl::Optional)
+    return;
+  // don't issue warning when primary class's method is 
+  // depecated/unavailable.
+  if (MethodDecl->hasAttr<UnavailableAttr>() ||
+      MethodDecl->hasAttr<DeprecatedAttr>())
+    return;
+  
+  bool match = CheckMethodOverrideReturn(*this, ImpMethodDecl, MethodDecl, 
+                                      IsProtocolMethodDecl, false, false);
+  if (match)
+    for (ObjCMethodDecl::param_iterator IM = ImpMethodDecl->param_begin(),
+         IF = MethodDecl->param_begin(), EM = ImpMethodDecl->param_end();
+         IM != EM; ++IM, ++IF) {
+      match = CheckMethodOverrideParam(*this, ImpMethodDecl, MethodDecl, 
+                                       *IM, *IF,
+                                       IsProtocolMethodDecl, false, false);
+      if (!match)
+        break;
+    }
+  if (match)
+    match = (ImpMethodDecl->isVariadic() == MethodDecl->isVariadic());
+  if (match)
+    match = !(MethodDecl->isClassMethod() &&
+              MethodDecl->getSelector() == GetNullarySelector("load", Context));
+  
+  if (match) {
+    Diag(ImpMethodDecl->getLocation(), 
+         diag::warn_category_method_impl_match);
+    Diag(MethodDecl->getLocation(), diag::note_method_declared_at);
+  }
+}
+
 /// FIXME: Type hierarchies in Objective-C can be deep. We could most likely
 /// improve the efficiency of selector lookups and type checking by associating
 /// with each protocol / interface / category the flattened instance tables. If
@@ -1416,7 +1475,7 @@ void Sema::CheckProtocolMethodDefs(SourceLocation ImpLoc,
             if (!MethodInClass || !MethodInClass->isSynthesized()) {
               unsigned DIAG = diag::warn_unimplemented_protocol_method;
               if (Diags.getDiagnosticLevel(DIAG, ImpLoc)
-                      != Diagnostic::Ignored) {
+                      != DiagnosticsEngine::Ignored) {
                 WarnUndefinedMethod(ImpLoc, method, IncompleteImpl, DIAG);
                 Diag(method->getLocation(), diag::note_method_declared_at);
                 Diag(CDecl->getLocation(), diag::note_required_for_protocol_at)
@@ -1434,7 +1493,8 @@ void Sema::CheckProtocolMethodDefs(SourceLocation ImpLoc,
         !ClsMap.count(method->getSelector()) &&
         (!Super || !Super->lookupClassMethod(method->getSelector()))) {
       unsigned DIAG = diag::warn_unimplemented_protocol_method;
-      if (Diags.getDiagnosticLevel(DIAG, ImpLoc) != Diagnostic::Ignored) {
+      if (Diags.getDiagnosticLevel(DIAG, ImpLoc) !=
+            DiagnosticsEngine::Ignored) {
         WarnUndefinedMethod(ImpLoc, method, IncompleteImpl, DIAG);
         Diag(method->getLocation(), diag::note_method_declared_at);
         Diag(IDecl->getLocation(), diag::note_required_for_protocol_at) <<
@@ -1458,7 +1518,8 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
                                       ObjCImplDecl* IMPDecl,
                                       ObjCContainerDecl* CDecl,
                                       bool &IncompleteImpl,
-                                      bool ImmediateClass) {
+                                      bool ImmediateClass,
+                                      bool WarnExactMatch) {
   // Check and see if instance methods in class interface have been
   // implemented in the implementation class. If so, their types match.
   for (ObjCInterfaceDecl::instmeth_iterator I = CDecl->instmeth_begin(),
@@ -1474,15 +1535,19 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
       continue;
     } else {
       ObjCMethodDecl *ImpMethodDecl =
-      IMPDecl->getInstanceMethod((*I)->getSelector());
-      ObjCMethodDecl *MethodDecl =
-      CDecl->getInstanceMethod((*I)->getSelector());
-      assert(MethodDecl &&
-             "MethodDecl is null in ImplMethodsVsClassMethods");
+        IMPDecl->getInstanceMethod((*I)->getSelector());
+      assert(CDecl->getInstanceMethod((*I)->getSelector()) &&
+             "Expected to find the method through lookup as well");
+      ObjCMethodDecl *MethodDecl = *I;
       // ImpMethodDecl may be null as in a @dynamic property.
-      if (ImpMethodDecl)
-        WarnConflictingTypedMethods(ImpMethodDecl, MethodDecl,
-                                    isa<ObjCProtocolDecl>(CDecl));
+      if (ImpMethodDecl) {
+        if (!WarnExactMatch)
+          WarnConflictingTypedMethods(ImpMethodDecl, MethodDecl,
+                                      isa<ObjCProtocolDecl>(CDecl));
+        else if (!MethodDecl->isSynthesized())
+          WarnExactTypedMethods(ImpMethodDecl, MethodDecl,
+                               isa<ObjCProtocolDecl>(CDecl));
+      }
     }
   }
 
@@ -1500,10 +1565,15 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
     } else {
       ObjCMethodDecl *ImpMethodDecl =
         IMPDecl->getClassMethod((*I)->getSelector());
-      ObjCMethodDecl *MethodDecl =
-        CDecl->getClassMethod((*I)->getSelector());
-      WarnConflictingTypedMethods(ImpMethodDecl, MethodDecl, 
-                                  isa<ObjCProtocolDecl>(CDecl));
+      assert(CDecl->getClassMethod((*I)->getSelector()) &&
+             "Expected to find the method through lookup as well");
+      ObjCMethodDecl *MethodDecl = *I;
+      if (!WarnExactMatch)
+        WarnConflictingTypedMethods(ImpMethodDecl, MethodDecl, 
+                                    isa<ObjCProtocolDecl>(CDecl));
+      else
+        WarnExactTypedMethods(ImpMethodDecl, MethodDecl,
+                             isa<ObjCProtocolDecl>(CDecl));
     }
   }
   
@@ -1514,7 +1584,7 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
       MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
                                  IMPDecl,
                                  const_cast<ObjCCategoryDecl *>(ClsExtDecl), 
-                                 IncompleteImpl, false);
+                                 IncompleteImpl, false, WarnExactMatch);
     
     // Check for any implementation of a methods declared in protocol.
     for (ObjCInterfaceDecl::all_protocol_iterator
@@ -1522,14 +1592,50 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
           E = I->all_referenced_protocol_end(); PI != E; ++PI)
       MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
                                  IMPDecl,
-                                 (*PI), IncompleteImpl, false);
-    if (I->getSuperClass())
+                                 (*PI), IncompleteImpl, false, WarnExactMatch);
+    
+    // FIXME. For now, we are not checking for extact match of methods 
+    // in category implementation and its primary class's super class. 
+    if (!WarnExactMatch && I->getSuperClass())
       MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
                                  IMPDecl,
                                  I->getSuperClass(), IncompleteImpl, false);
   }
 }
 
+/// CheckCategoryVsClassMethodMatches - Checks that methods implemented in
+/// category matches with those implemented in its primary class and
+/// warns each time an exact match is found. 
+void Sema::CheckCategoryVsClassMethodMatches(
+                                  ObjCCategoryImplDecl *CatIMPDecl) {
+  llvm::DenseSet<Selector> InsMap, ClsMap;
+  
+  for (ObjCImplementationDecl::instmeth_iterator
+       I = CatIMPDecl->instmeth_begin(), 
+       E = CatIMPDecl->instmeth_end(); I!=E; ++I)
+    InsMap.insert((*I)->getSelector());
+  
+  for (ObjCImplementationDecl::classmeth_iterator
+       I = CatIMPDecl->classmeth_begin(),
+       E = CatIMPDecl->classmeth_end(); I != E; ++I)
+    ClsMap.insert((*I)->getSelector());
+  if (InsMap.empty() && ClsMap.empty())
+    return;
+  
+  // Get category's primary class.
+  ObjCCategoryDecl *CatDecl = CatIMPDecl->getCategoryDecl();
+  if (!CatDecl)
+    return;
+  ObjCInterfaceDecl *IDecl = CatDecl->getClassInterface();
+  if (!IDecl)
+    return;
+  llvm::DenseSet<Selector> InsMapSeen, ClsMapSeen;
+  bool IncompleteImpl = false;
+  MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
+                             CatIMPDecl, IDecl,
+                             IncompleteImpl, false, true /*WarnExactMatch*/);
+}
+
 void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
                                      ObjCContainerDecl* CDecl,
                                      bool IncompleteImpl) {
@@ -1559,6 +1665,12 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
   MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
                              IMPDecl, CDecl,
                              IncompleteImpl, true);
+  
+  // check all methods implemented in category against those declared
+  // in its primary class.
+  if (ObjCCategoryImplDecl *CatDecl = 
+        dyn_cast<ObjCCategoryImplDecl>(IMPDecl))
+    CheckCategoryVsClassMethodMatches(CatDecl);
 
   // Check the protocol list for unimplemented methods in the @implementation
   // class.
@@ -1598,17 +1710,16 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
       DiagnoseUnimplementedProperties(S, IMPDecl, CDecl, InsMap);      
     } 
   } else
-    assert(false && "invalid ObjCContainerDecl type.");
+    llvm_unreachable("invalid ObjCContainerDecl type.");
 }
 
 /// ActOnForwardClassDeclaration -
-Decl *
+Sema::DeclGroupPtrTy
 Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
                                    IdentifierInfo **IdentList,
                                    SourceLocation *IdentLocs,
                                    unsigned NumElts) {
-  llvm::SmallVector<ObjCInterfaceDecl*, 32> Interfaces;
-
+  SmallVector<Decl *, 8> DeclsInGroup;
   for (unsigned i = 0; i != NumElts; ++i) {
     // Check for another declaration kind with the same name.
     NamedDecl *PrevDecl
@@ -1653,17 +1764,14 @@ Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
       PushOnScopeChains(IDecl, TUScope, false);
       CurContext->makeDeclVisibleInContext(IDecl, true);
     }
-
-    Interfaces.push_back(IDecl);
+    ObjCClassDecl *CDecl = ObjCClassDecl::Create(Context, CurContext, AtClassLoc,
+                                                 IDecl, IdentLocs[i]);
+    CurContext->addDecl(CDecl);
+    CheckObjCDeclScope(CDecl);
+    DeclsInGroup.push_back(CDecl);
   }
-
-  assert(Interfaces.size() == NumElts);
-  ObjCClassDecl *CDecl = ObjCClassDecl::Create(Context, CurContext, AtClassLoc,
-                                               Interfaces.data(), IdentLocs,
-                                               Interfaces.size());
-  CurContext->addDecl(CDecl);
-  CheckObjCDeclScope(CDecl);
-  return CDecl;
+  
+  return BuildDeclaratorGroup(DeclsInGroup.data(), DeclsInGroup.size(), false);
 }
 
 static bool tryMatchRecordTypes(ASTContext &Context,
@@ -1705,11 +1813,16 @@ static bool matchTypes(ASTContext &Context, Sema::MethodMatchStrategy strategy,
   if (!left->isScalarType() || !right->isScalarType())
     return tryMatchRecordTypes(Context, strategy, left, right);
 
-  // Make scalars agree in kind, except count bools as chars.
+  // Make scalars agree in kind, except count bools as chars, and group
+  // all non-member pointers together.
   Type::ScalarTypeKind leftSK = left->getScalarTypeKind();
   Type::ScalarTypeKind rightSK = right->getScalarTypeKind();
   if (leftSK == Type::STK_Bool) leftSK = Type::STK_Integral;
   if (rightSK == Type::STK_Bool) rightSK = Type::STK_Integral;
+  if (leftSK == Type::STK_CPointer || leftSK == Type::STK_BlockPointer)
+    leftSK = Type::STK_ObjCObjectPointer;
+  if (rightSK == Type::STK_CPointer || rightSK == Type::STK_BlockPointer)
+    rightSK = Type::STK_ObjCObjectPointer;
 
   // Note that data member pointers and function member pointers don't
   // intermix because of the size differences.
@@ -1764,12 +1877,12 @@ bool Sema::MatchTwoMethodDeclarations(const ObjCMethodDecl *left,
          != right->hasAttr<NSConsumesSelfAttr>()))
     return false;
 
-  ObjCMethodDecl::param_iterator
+  ObjCMethodDecl::param_const_iterator
     li = left->param_begin(), le = left->param_end(), ri = right->param_begin();
 
   for (; li != le; ++li, ++ri) {
     assert(ri != right->param_end() && "Param mismatch");
-    ParmVarDecl *lparm = *li, *rparm = *ri;
+    const ParmVarDecl *lparm = *li, *rparm = *ri;
 
     if (!matchTypes(Context, strategy, lparm->getType(), rparm->getType()))
       return false;
@@ -1887,7 +2000,7 @@ ObjCMethodDecl *Sema::LookupMethodInGlobalPool(Selector Sel, SourceRange R,
       (receiverIdOrClass && warn &&
        (Diags.getDiagnosticLevel(diag::warn_strict_multiple_method_decl,
                                  R.getBegin()) != 
-      Diagnostic::Ignored));
+      DiagnosticsEngine::Ignored));
     if (strictSelectorMatch)
       for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next) {
         if (!MatchTwoMethodDeclarations(MethList.Method, Next->Method,
@@ -2010,15 +2123,14 @@ void Sema::DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID,
 // Note: For class/category implemenations, allMethods/allProperties is
 // always null.
 void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
-                      Decl *ClassDecl,
                       Decl **allMethods, unsigned allNum,
                       Decl **allProperties, unsigned pNum,
                       DeclGroupPtrTy *allTUVars, unsigned tuvNum) {
-  // FIXME: If we don't have a ClassDecl, we have an error. We should consider
-  // always passing in a decl. If the decl has an error, isInvalidDecl()
-  // should be true.
-  if (!ClassDecl)
+
+  if (!CurContext->isObjCContainer())
     return;
+  ObjCContainerDecl *OCD = dyn_cast<ObjCContainerDecl>(CurContext);
+  Decl *ClassDecl = cast<Decl>(OCD);
   
   bool isInterfaceDeclKind =
         isa<ObjCInterfaceDecl>(ClassDecl) || isa<ObjCCategoryDecl>(ClassDecl)
@@ -2055,6 +2167,8 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
           Diag(PrevMethod->getLocation(), diag::note_previous_declaration);
         Method->setInvalidDecl();
       } else {
+        if (PrevMethod)
+          Method->setAsRedeclaration(PrevMethod);
         InsMap[Method->getSelector()] = Method;
         /// The following allows us to typecheck messages to "id".
         AddInstanceMethodToGlobalPool(Method);
@@ -2074,6 +2188,8 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
         Diag(PrevMethod->getLocation(), diag::note_previous_declaration);
         Method->setInvalidDecl();
       } else {
+        if (PrevMethod)
+          Method->setAsRedeclaration(PrevMethod);
         ClsMap[Method->getSelector()] = Method;
         /// The following allows us to typecheck messages to "Class".
         AddFactoryMethodToGlobalPool(Method);
@@ -2145,10 +2261,6 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
           }        
         }
       }
-      
-      if (LangOpts.ObjCDefaultSynthProperties &&
-          LangOpts.ObjCNonFragileABI2)
-        DefaultSynthesizeProperties(S, IC, IDecl);
       ImplMethodsVsClassMethods(S, IC, IDecl);
       AtomicPropertySetterGetterRules(IC, IDecl);
       DiagnoseOwningPropertyGetterSynthesis(IC);
@@ -2187,6 +2299,7 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
         }
     }
   }
+  ActOnObjCContainerFinishDefinition();
 }
 
 
@@ -2208,16 +2321,22 @@ bool containsInvalidMethodImplAttribute(const AttrVec &A) {
   return false;
 }
 
+namespace  {
+  /// \brief Describes the compatibility of a result type with its method.
+  enum ResultTypeCompatibilityKind {
+    RTC_Compatible,
+    RTC_Incompatible,
+    RTC_Unknown
+  };
+}
+
 /// \brief Check whether the declared result type of the given Objective-C
 /// method declaration is compatible with the method's class.
 ///
-static bool 
+static ResultTypeCompatibilityKind 
 CheckRelatedResultTypeCompatibility(Sema &S, ObjCMethodDecl *Method,
                                     ObjCInterfaceDecl *CurrentClass) {
   QualType ResultType = Method->getResultType();
-  SourceRange ResultTypeRange;
-  if (const TypeSourceInfo *ResultTypeInfo = Method->getResultTypeSourceInfo())
-    ResultTypeRange = ResultTypeInfo->getTypeLoc().getSourceRange();
   
   // If an Objective-C method inherits its related result type, then its 
   // declared result type must be compatible with its own class type. The
@@ -2227,52 +2346,171 @@ CheckRelatedResultTypeCompatibility(Sema &S, ObjCMethodDecl *Method,
     //   - it is id or qualified id, or
     if (ResultObjectType->isObjCIdType() ||
         ResultObjectType->isObjCQualifiedIdType())
-      return false;
+      return RTC_Compatible;
   
     if (CurrentClass) {
       if (ObjCInterfaceDecl *ResultClass 
                                       = ResultObjectType->getInterfaceDecl()) {
         //   - it is the same as the method's class type, or
         if (CurrentClass == ResultClass)
-          return false;
+          return RTC_Compatible;
         
         //   - it is a superclass of the method's class type
         if (ResultClass->isSuperClassOf(CurrentClass))
-          return false;
+          return RTC_Compatible;
       }      
+    } else {
+      // Any Objective-C pointer type might be acceptable for a protocol
+      // method; we just don't know.
+      return RTC_Unknown;
     }
   }
   
-  return true;
+  return RTC_Incompatible;
 }
 
-/// \brief Determine if any method in the global method pool has an inferred 
-/// result type.
-static bool 
-anyMethodInfersRelatedResultType(Sema &S, Selector Sel, bool IsInstance) {
-  Sema::GlobalMethodPool::iterator Pos = S.MethodPool.find(Sel);
-  if (Pos == S.MethodPool.end()) {
-    if (S.ExternalSource)
-      Pos = S.ReadMethodPool(Sel);
-    else
-      return 0;
+namespace {
+/// A helper class for searching for methods which a particular method
+/// overrides.
+class OverrideSearch {
+  Sema &S;
+  ObjCMethodDecl *Method;
+  llvm::SmallPtrSet<ObjCContainerDecl*, 8> Searched;
+  llvm::SmallPtrSet<ObjCMethodDecl*, 8> Overridden;
+  bool Recursive;
+
+public:
+  OverrideSearch(Sema &S, ObjCMethodDecl *method) : S(S), Method(method) {
+    Selector selector = method->getSelector();
+
+    // Bypass this search if we've never seen an instance/class method
+    // with this selector before.
+    Sema::GlobalMethodPool::iterator it = S.MethodPool.find(selector);
+    if (it == S.MethodPool.end()) {
+      if (!S.ExternalSource) return;
+      it = S.ReadMethodPool(selector);
+    }
+    ObjCMethodList &list =
+      method->isInstanceMethod() ? it->second.first : it->second.second;
+    if (!list.Method) return;
+
+    ObjCContainerDecl *container
+      = cast<ObjCContainerDecl>(method->getDeclContext());
+
+    // Prevent the search from reaching this container again.  This is
+    // important with categories, which override methods from the
+    // interface and each other.
+    Searched.insert(container);
+    searchFromContainer(container);
   }
-  
-  ObjCMethodList &List = IsInstance ? Pos->second.first : Pos->second.second;
-  for (ObjCMethodList *M = &List; M; M = M->Next) {
-    if (M->Method && M->Method->hasRelatedResultType())
-      return true;
-  }  
-  
-  return false;
+
+  typedef llvm::SmallPtrSet<ObjCMethodDecl*,8>::iterator iterator;
+  iterator begin() const { return Overridden.begin(); }
+  iterator end() const { return Overridden.end(); }
+
+private:
+  void searchFromContainer(ObjCContainerDecl *container) {
+    if (container->isInvalidDecl()) return;
+
+    switch (container->getDeclKind()) {
+#define OBJCCONTAINER(type, base) \
+    case Decl::type: \
+      searchFrom(cast<type##Decl>(container)); \
+      break;
+#define ABSTRACT_DECL(expansion)
+#define DECL(type, base) \
+    case Decl::type:
+#include "clang/AST/DeclNodes.inc"
+      llvm_unreachable("not an ObjC container!");
+    }
+  }
+
+  void searchFrom(ObjCProtocolDecl *protocol) {
+    // A method in a protocol declaration overrides declarations from
+    // referenced ("parent") protocols.
+    search(protocol->getReferencedProtocols());
+  }
+
+  void searchFrom(ObjCCategoryDecl *category) {
+    // A method in a category declaration overrides declarations from
+    // the main class and from protocols the category references.
+    search(category->getClassInterface());
+    search(category->getReferencedProtocols());
+  }
+
+  void searchFrom(ObjCCategoryImplDecl *impl) {
+    // A method in a category definition that has a category
+    // declaration overrides declarations from the category
+    // declaration.
+    if (ObjCCategoryDecl *category = impl->getCategoryDecl()) {
+      search(category);
+
+    // Otherwise it overrides declarations from the class.
+    } else {
+      search(impl->getClassInterface());
+    }
+  }
+
+  void searchFrom(ObjCInterfaceDecl *iface) {
+    // A method in a class declaration overrides declarations from
+
+    //   - categories,
+    for (ObjCCategoryDecl *category = iface->getCategoryList();
+           category; category = category->getNextClassCategory())
+      search(category);
+
+    //   - the super class, and
+    if (ObjCInterfaceDecl *super = iface->getSuperClass())
+      search(super);
+
+    //   - any referenced protocols.
+    search(iface->getReferencedProtocols());
+  }
+
+  void searchFrom(ObjCImplementationDecl *impl) {
+    // A method in a class implementation overrides declarations from
+    // the class interface.
+    search(impl->getClassInterface());
+  }
+
+
+  void search(const ObjCProtocolList &protocols) {
+    for (ObjCProtocolList::iterator i = protocols.begin(), e = protocols.end();
+         i != e; ++i)
+      search(*i);
+  }
+
+  void search(ObjCContainerDecl *container) {
+    // Abort if we've already searched this container.
+    if (!Searched.insert(container)) return;
+
+    // Check for a method in this container which matches this selector.
+    ObjCMethodDecl *meth = container->getMethod(Method->getSelector(),
+                                                Method->isInstanceMethod());
+
+    // If we find one, record it and bail out.
+    if (meth) {
+      Overridden.insert(meth);
+      return;
+    }
+
+    // Otherwise, search for methods that a hypothetical method here
+    // would have overridden.
+
+    // Note that we're now in a recursive case.
+    Recursive = true;
+
+    searchFromContainer(container);
+  }
+};
 }
 
 Decl *Sema::ActOnMethodDeclaration(
     Scope *S,
     SourceLocation MethodLoc, SourceLocation EndLoc,
-    tok::TokenKind MethodType, Decl *ClassDecl,
+    tok::TokenKind MethodType, 
     ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
-    SourceLocation SelectorStartLoc,
+    ArrayRef<SourceLocation> SelectorLocs,
     Selector Sel,
     // optional arguments. The number of types/arguments is obtained
     // from the Sel.getNumArgs().
@@ -2281,12 +2519,15 @@ Decl *Sema::ActOnMethodDeclaration(
     AttributeList *AttrList, tok::ObjCKeywordKind MethodDeclKind,
     bool isVariadic, bool MethodDefinition) {
   // Make sure we can establish a context for the method.
-  if (!ClassDecl) {
+  if (!CurContext->isObjCContainer()) {
     Diag(MethodLoc, diag::error_missing_method_context);
     return 0;
   }
+  ObjCContainerDecl *OCD = dyn_cast<ObjCContainerDecl>(CurContext);
+  Decl *ClassDecl = cast<Decl>(OCD); 
   QualType resultDeclType;
 
+  bool HasRelatedResultType = false;
   TypeSourceInfo *ResultTInfo = 0;
   if (ReturnType) {
     resultDeclType = GetTypeFromParser(ReturnType, &ResultTInfo);
@@ -2298,21 +2539,28 @@ Decl *Sema::ActOnMethodDeclaration(
         << 0 << resultDeclType;
       return 0;
     }    
-  } else // get the type for "id".
+    
+    HasRelatedResultType = (resultDeclType == Context.getObjCInstanceType());
+  } else { // get the type for "id".
     resultDeclType = Context.getObjCIdType();
+    Diag(MethodLoc, diag::warn_missing_method_return_type)
+      << FixItHint::CreateInsertion(SelectorLocs.front(), "(id)");
+  }
 
   ObjCMethodDecl* ObjCMethod =
-    ObjCMethodDecl::Create(Context, MethodLoc, EndLoc, Sel, resultDeclType,
+    ObjCMethodDecl::Create(Context, MethodLoc, EndLoc, Sel,
+                           resultDeclType,
                            ResultTInfo,
-                           cast<DeclContext>(ClassDecl),
+                           CurContext,
                            MethodType == tok::minus, isVariadic,
-                           false, false,
+                           /*isSynthesized=*/false,
+                           /*isImplicitlyDeclared=*/false, /*isDefined=*/false,
                            MethodDeclKind == tok::objc_optional 
                              ? ObjCMethodDecl::Optional
                              : ObjCMethodDecl::Required,
-                           false);
+                           HasRelatedResultType);
 
-  llvm::SmallVector<ParmVarDecl*, 16> Params;
+  SmallVector<ParmVarDecl*, 16> Params;
 
   for (unsigned i = 0, e = Sel.getNumArgs(); i != e; ++i) {
     QualType ArgType;
@@ -2383,20 +2631,16 @@ Decl *Sema::ActOnMethodDeclaration(
     Params.push_back(Param);
   }
   
-  ObjCMethod->setMethodParams(Context, Params.data(), Params.size(),
-                              Sel.getNumArgs());
+  ObjCMethod->setMethodParams(Context, Params, SelectorLocs);
   ObjCMethod->setObjCDeclQualifier(
     CvtQTToAstBitMask(ReturnQT.getObjCDeclQualifier()));
-  const ObjCMethodDecl *PrevMethod = 0;
 
   if (AttrList)
     ProcessDeclAttributeList(TUScope, ObjCMethod, AttrList);
 
-  const ObjCMethodDecl *InterfaceMD = 0;
-
   // Add the method now.
-  if (ObjCImplementationDecl *ImpDecl =
-        dyn_cast<ObjCImplementationDecl>(ClassDecl)) {
+  const ObjCMethodDecl *PrevMethod = 0;
+  if (ObjCImplDecl *ImpDecl = dyn_cast<ObjCImplDecl>(ClassDecl)) {
     if (MethodType == tok::minus) {
       PrevMethod = ImpDecl->getInstanceMethod(Sel);
       ImpDecl->addInstanceMethod(ObjCMethod);
@@ -2404,24 +2648,6 @@ Decl *Sema::ActOnMethodDeclaration(
       PrevMethod = ImpDecl->getClassMethod(Sel);
       ImpDecl->addClassMethod(ObjCMethod);
     }
-    InterfaceMD = ImpDecl->getClassInterface()->getMethod(Sel,
-                                                   MethodType == tok::minus);
-    
-    if (ObjCMethod->hasAttrs() &&
-        containsInvalidMethodImplAttribute(ObjCMethod->getAttrs()))
-      Diag(EndLoc, diag::warn_attribute_method_def);
-  } else if (ObjCCategoryImplDecl *CatImpDecl =
-             dyn_cast<ObjCCategoryImplDecl>(ClassDecl)) {
-    if (MethodType == tok::minus) {
-      PrevMethod = CatImpDecl->getInstanceMethod(Sel);
-      CatImpDecl->addInstanceMethod(ObjCMethod);
-    } else {
-      PrevMethod = CatImpDecl->getClassMethod(Sel);
-      CatImpDecl->addClassMethod(ObjCMethod);
-    }
-
-    if (ObjCCategoryDecl *Cat = CatImpDecl->getCategoryDecl())
-      InterfaceMD = Cat->getMethod(Sel, MethodType == tok::minus);
 
     if (ObjCMethod->hasAttrs() &&
         containsInvalidMethodImplAttribute(ObjCMethod->getAttrs()))
@@ -2429,6 +2655,7 @@ Decl *Sema::ActOnMethodDeclaration(
   } else {
     cast<DeclContext>(ClassDecl)->addDecl(ObjCMethod);
   }
+
   if (PrevMethod) {
     // You can never have two method definitions with the same name.
     Diag(ObjCMethod->getLocation(), diag::err_duplicate_method_decl)
@@ -2449,28 +2676,44 @@ Decl *Sema::ActOnMethodDeclaration(
                                    = dyn_cast<ObjCCategoryImplDecl>(ClassDecl))
       CurrentClass = CatImpl->getClassInterface();
   }
-  
-  // Merge information down from the interface declaration if we have one.
-  if (InterfaceMD) {
-    // Inherit the related result type, if we can.
-    if (InterfaceMD->hasRelatedResultType() &&
-        !CheckRelatedResultTypeCompatibility(*this, ObjCMethod, CurrentClass))
+
+  ResultTypeCompatibilityKind RTC
+    = CheckRelatedResultTypeCompatibility(*this, ObjCMethod, CurrentClass);
+
+  // Search for overridden methods and merge information down from them.
+  OverrideSearch overrides(*this, ObjCMethod);
+  for (OverrideSearch::iterator
+         i = overrides.begin(), e = overrides.end(); i != e; ++i) {
+    ObjCMethodDecl *overridden = *i;
+
+    // Propagate down the 'related result type' bit from overridden methods.
+    if (RTC != RTC_Incompatible && overridden->hasRelatedResultType())
       ObjCMethod->SetRelatedResultType();
-      
-    mergeObjCMethodDecls(ObjCMethod, InterfaceMD);
+
+    // Then merge the declarations.
+    mergeObjCMethodDecls(ObjCMethod, overridden);
+    
+    // Check for overriding methods
+    if (isa<ObjCInterfaceDecl>(ObjCMethod->getDeclContext()) || 
+        isa<ObjCImplementationDecl>(ObjCMethod->getDeclContext()))
+      CheckConflictingOverridingMethod(ObjCMethod, overridden,
+              isa<ObjCProtocolDecl>(overridden->getDeclContext()));
   }
   
   bool ARCError = false;
   if (getLangOptions().ObjCAutoRefCount)
     ARCError = CheckARCMethodDecl(*this, ObjCMethod);
 
-  if (!ObjCMethod->hasRelatedResultType() && !ARCError &&
-      getLangOptions().ObjCInferRelatedResultType) {
+  // Infer the related result type when possible.
+  if (!ARCError && RTC == RTC_Compatible &&
+      !ObjCMethod->hasRelatedResultType() &&
+      LangOpts.ObjCInferRelatedResultType) {
     bool InferRelatedResultType = false;
     switch (ObjCMethod->getMethodFamily()) {
     case OMF_None:
     case OMF_copy:
     case OMF_dealloc:
+    case OMF_finalize:
     case OMF_mutableCopy:
     case OMF_release:
     case OMF_retainCount:
@@ -2490,14 +2733,8 @@ Decl *Sema::ActOnMethodDeclaration(
       break;
     }
     
-    if (InferRelatedResultType &&
-        !CheckRelatedResultTypeCompatibility(*this, ObjCMethod, CurrentClass))
+    if (InferRelatedResultType)
       ObjCMethod->SetRelatedResultType();
-    
-    if (!InterfaceMD && 
-        anyMethodInfersRelatedResultType(*this, ObjCMethod->getSelector(),
-                                         ObjCMethod->isInstanceMethod()))
-      CheckObjCMethodOverrides(ObjCMethod, cast<DeclContext>(ClassDecl));
   }
     
   return ObjCMethod;
@@ -2506,7 +2743,12 @@ Decl *Sema::ActOnMethodDeclaration(
 bool Sema::CheckObjCDeclScope(Decl *D) {
   if (isa<TranslationUnitDecl>(CurContext->getRedeclContext()))
     return false;
-
+  // Following is also an error. But it is caused by a missing @end
+  // and diagnostic is issued elsewhere.
+  if (isa<ObjCContainerDecl>(CurContext->getRedeclContext())) {
+    return false;
+  }
+  
   Diag(D->getLocation(), diag::err_objc_decls_may_only_appear_in_global_scope);
   D->setInvalidDecl();
 
@@ -2517,7 +2759,7 @@ bool Sema::CheckObjCDeclScope(Decl *D) {
 /// instance variables of ClassName into Decls.
 void Sema::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
                      IdentifierInfo *ClassName,
-                     llvm::SmallVectorImpl<Decl*> &Decls) {
+                     SmallVectorImpl<Decl*> &Decls) {
   // Check that ClassName is a valid class
   ObjCInterfaceDecl *Class = getObjCInterfaceDecl(ClassName, DeclStart);
   if (!Class) {
@@ -2530,11 +2772,11 @@ void Sema::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
   }
 
   // Collect the instance variables
-  llvm::SmallVector<ObjCIvarDecl*, 32> Ivars;
+  SmallVector<const ObjCIvarDecl*, 32> Ivars;
   Context.DeepCollectObjCIvars(Class, true, Ivars);
   // For each ivar, create a fresh ObjCAtDefsFieldDecl.
   for (unsigned i = 0; i < Ivars.size(); i++) {
-    FieldDecl* ID = cast<FieldDecl>(Ivars[i]);
+    const FieldDecl* ID = cast<FieldDecl>(Ivars[i]);
     RecordDecl *Record = dyn_cast<RecordDecl>(TagD);
     Decl *FD = ObjCAtDefsFieldDecl::Create(Context, Record,
                                            /*FIXME: StartL=*/ID->getLocation(),
@@ -2545,7 +2787,7 @@ void Sema::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
   }
 
   // Introduce all of these fields into the appropriate scope.
-  for (llvm::SmallVectorImpl<Decl*>::iterator D = Decls.begin();
+  for (SmallVectorImpl<Decl*>::iterator D = Decls.begin();
        D != Decls.end(); ++D) {
     FieldDecl *FD = cast<FieldDecl>(*D);
     if (getLangOptions().CPlusPlus)
@@ -2647,7 +2889,7 @@ Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
 /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
 /// initialization.
 void Sema::CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
-                                llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
+                                SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
   for (ObjCIvarDecl *Iv = OI->all_declared_ivar_begin(); Iv; 
        Iv= Iv->getNextIvar()) {
     QualType QT = Context.getBaseElementType(Iv->getType());
@@ -2656,20 +2898,15 @@ void Sema::CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
   }
 }
 
-void ObjCImplementationDecl::setIvarInitializers(ASTContext &C,
-                                             CXXCtorInitializer ** initializers,
-                                                 unsigned numInitializers) {
-  if (numInitializers > 0) {
-    NumIvarInitializers = numInitializers;
-    CXXCtorInitializer **ivarInitializers =
-    new (C) CXXCtorInitializer*[NumIvarInitializers];
-    memcpy(ivarInitializers, initializers,
-           numInitializers * sizeof(CXXCtorInitializer*));
-    IvarInitializers = ivarInitializers;
-  }
-}
-
 void Sema::DiagnoseUseOfUnimplementedSelectors() {
+  // Load referenced selectors from the external source.
+  if (ExternalSource) {
+    SmallVector<std::pair<Selector, SourceLocation>, 4> Sels;
+    ExternalSource->ReadReferencedSelectors(Sels);
+    for (unsigned I = 0, N = Sels.size(); I != N; ++I)
+      ReferencedSelectors[Sels[I].first] = Sels[I].second;
+  }
+  
   // Warning will be issued only when selector table is
   // generated (which means there is at lease one implementation
   // in the TU). This is to match gcc's behavior.
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExceptionSpec.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExceptionSpec.cpp
index 7bcec315ddab..92af2d914ba0 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -101,7 +101,7 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
   bool MissingExceptionSpecification = false;
   bool MissingEmptyExceptionSpecification = false;
   unsigned DiagID = diag::err_mismatched_exception_spec;
-  if (getLangOptions().Microsoft)
+  if (getLangOptions().MicrosoftExt)
     DiagID = diag::warn_mismatched_exception_spec; 
   
   if (!CheckEquivalentExceptionSpec(PDiag(DiagID),
@@ -205,7 +205,7 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
         else
           OS << ", ";
         
-        OS << E->getAsString(Context.PrintingPolicy);
+        OS << E->getAsString(getPrintingPolicy());
       }
       OS << ")";
       break;
@@ -217,13 +217,13 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
 
     case EST_ComputedNoexcept:
       OS << "noexcept(";
-      OldProto->getNoexceptExpr()->printPretty(OS, Context, 0,
-                                               Context.PrintingPolicy);
+      OldProto->getNoexceptExpr()->printPretty(OS, Context, 0, 
+                                               getPrintingPolicy());
       OS << ")";
       break;
 
     default:
-      assert(false && "This spec type is compatible with none.");
+      llvm_unreachable("This spec type is compatible with none.");
     }
     OS.flush();
 
@@ -264,7 +264,7 @@ bool Sema::CheckEquivalentExceptionSpec(
     const FunctionProtoType *Old, SourceLocation OldLoc,
     const FunctionProtoType *New, SourceLocation NewLoc) {
   unsigned DiagID = diag::err_mismatched_exception_spec;
-  if (getLangOptions().Microsoft)
+  if (getLangOptions().MicrosoftExt)
     DiagID = diag::warn_mismatched_exception_spec; 
   return CheckEquivalentExceptionSpec(
                                       PDiag(DiagID),
@@ -717,7 +717,7 @@ bool Sema::CheckOverridingFunctionExceptionSpec(const CXXMethodDecl *New,
     }
   }
   unsigned DiagID = diag::err_override_exception_spec;
-  if (getLangOptions().Microsoft)
+  if (getLangOptions().MicrosoftExt)
     DiagID = diag::warn_override_exception_spec;
   return CheckExceptionSpecSubset(PDiag(DiagID),
                                   PDiag(diag::note_overridden_virtual_function),
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp
index 5efc36559d97..170097cc59a9 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp
@@ -36,10 +36,60 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/SemaFixItUtils.h"
 #include "clang/Sema/Template.h"
 using namespace clang;
 using namespace sema;
 
+/// \brief Determine whether the use of this declaration is valid, without
+/// emitting diagnostics.
+bool Sema::CanUseDecl(NamedDecl *D) {
+  // See if this is an auto-typed variable whose initializer we are parsing.
+  if (ParsingInitForAutoVars.count(D))
+    return false;
+
+  // See if this is a deleted function.
+  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    if (FD->isDeleted())
+      return false;
+  }
+  return true;
+}
+
+static AvailabilityResult DiagnoseAvailabilityOfDecl(Sema &S,
+                              NamedDecl *D, SourceLocation Loc,
+                              const ObjCInterfaceDecl *UnknownObjCClass) {
+  // See if this declaration is unavailable or deprecated.
+  std::string Message;
+  AvailabilityResult Result = D->getAvailability(&Message);
+  switch (Result) {
+    case AR_Available:
+    case AR_NotYetIntroduced:
+      break;
+            
+    case AR_Deprecated:
+      S.EmitDeprecationWarning(D, Message, Loc, UnknownObjCClass);
+      break;
+            
+    case AR_Unavailable:
+      if (S.getCurContextAvailability() != AR_Unavailable) {
+        if (Message.empty()) {
+          if (!UnknownObjCClass)
+            S.Diag(Loc, diag::err_unavailable) << D->getDeclName();
+          else
+            S.Diag(Loc, diag::warn_unavailable_fwdclass_message) 
+              << D->getDeclName();
+        }
+        else 
+          S.Diag(Loc, diag::err_unavailable_message) 
+            << D->getDeclName() << Message;
+          S.Diag(D->getLocation(), diag::note_unavailable_here) 
+          << isa<FunctionDecl>(D) << false;
+      }
+      break;
+    }
+    return Result;
+}
 
 /// \brief Determine whether the use of this declaration is valid, and
 /// emit any corresponding diagnostics.
@@ -50,9 +100,6 @@ using namespace sema;
 /// used, or produce an error (and return true) if a C++0x deleted
 /// function is being used.
 ///
-/// If IgnoreDeprecated is set to true, this should not warn about deprecated
-/// decls.
-///
 /// \returns true if there was an error (this declaration cannot be
 /// referenced), false otherwise.
 ///
@@ -61,10 +108,10 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc,
   if (getLangOptions().CPlusPlus && isa<FunctionDecl>(D)) {
     // If there were any diagnostics suppressed by template argument deduction,
     // emit them now.
-    llvm::DenseMap<Decl *, llvm::SmallVector<PartialDiagnosticAt, 1> >::iterator
+    llvm::DenseMap<Decl *, SmallVector<PartialDiagnosticAt, 1> >::iterator
       Pos = SuppressedDiagnostics.find(D->getCanonicalDecl());
     if (Pos != SuppressedDiagnostics.end()) {
-      llvm::SmallVectorImpl<PartialDiagnosticAt> &Suppressed = Pos->second;
+      SmallVectorImpl<PartialDiagnosticAt> &Suppressed = Pos->second;
       for (unsigned I = 0, N = Suppressed.size(); I != N; ++I)
         Diag(Suppressed[I].first, Suppressed[I].second);
       
@@ -91,40 +138,22 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc,
       return true;
     }
   }
-
-  // See if this declaration is unavailable or deprecated.
-  std::string Message;
-  switch (D->getAvailability(&Message)) {
-  case AR_Available:
-  case AR_NotYetIntroduced:
-    break;
-
-  case AR_Deprecated:
-    EmitDeprecationWarning(D, Message, Loc, UnknownObjCClass);
-    break;
-
-  case AR_Unavailable:
-    if (cast<Decl>(CurContext)->getAvailability() != AR_Unavailable) {
-      if (Message.empty()) {
-        if (!UnknownObjCClass)
-          Diag(Loc, diag::err_unavailable) << D->getDeclName();
-        else
-          Diag(Loc, diag::warn_unavailable_fwdclass_message) 
-               << D->getDeclName();
-      }
-      else 
-        Diag(Loc, diag::err_unavailable_message) 
-          << D->getDeclName() << Message;
-      Diag(D->getLocation(), diag::note_unavailable_here) 
-        << isa<FunctionDecl>(D) << false;
-    }
-    break;
-  }
+  AvailabilityResult Result =
+    DiagnoseAvailabilityOfDecl(*this, D, Loc, UnknownObjCClass);
 
   // Warn if this is used but marked unused.
   if (D->hasAttr<UnusedAttr>())
     Diag(Loc, diag::warn_used_but_marked_unused) << D->getDeclName();
-
+  // For available enumerator, it will become unavailable/deprecated
+  // if its enum declaration is as such.
+  if (Result == AR_Available)
+    if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(D)) {
+      const DeclContext *DC = ECD->getDeclContext();
+      if (const EnumDecl *TheEnumDecl = dyn_cast<EnumDecl>(DC))
+        DiagnoseAvailabilityOfDecl(*this,
+                          const_cast< EnumDecl *>(TheEnumDecl), 
+                          Loc, UnknownObjCClass);
+    }
   return false;
 }
 
@@ -145,94 +174,74 @@ std::string Sema::getDeletedOrUnavailableSuffix(const FunctionDecl *FD) {
   return std::string();
 }
 
-/// DiagnoseSentinelCalls - This routine checks on method dispatch calls
-/// (and other functions in future), which have been declared with sentinel
-/// attribute. It warns if call does not have the sentinel argument.
-///
+/// DiagnoseSentinelCalls - This routine checks whether a call or
+/// message-send is to a declaration with the sentinel attribute, and
+/// if so, it checks that the requirements of the sentinel are
+/// satisfied.
 void Sema::DiagnoseSentinelCalls(NamedDecl *D, SourceLocation Loc,
-                                 Expr **Args, unsigned NumArgs) {
+                                 Expr **args, unsigned numArgs) {
   const SentinelAttr *attr = D->getAttr<SentinelAttr>();
   if (!attr)
     return;
 
-  // FIXME: In C++0x, if any of the arguments are parameter pack
-  // expansions, we can't check for the sentinel now.
-  int sentinelPos = attr->getSentinel();
-  int nullPos = attr->getNullPos();
+  // The number of formal parameters of the declaration.
+  unsigned numFormalParams;
+
+  // The kind of declaration.  This is also an index into a %select in
+  // the diagnostic.
+  enum CalleeType { CT_Function, CT_Method, CT_Block } calleeType;
 
-  // FIXME. ObjCMethodDecl and FunctionDecl need be derived from the same common
-  // base class. Then we won't be needing two versions of the same code.
-  unsigned int i = 0;
-  bool warnNotEnoughArgs = false;
-  int isMethod = 0;
   if (ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D)) {
-    // skip over named parameters.
-    ObjCMethodDecl::param_iterator P, E = MD->param_end();
-    for (P = MD->param_begin(); (P != E && i < NumArgs); ++P) {
-      if (nullPos)
-        --nullPos;
-      else
-        ++i;
-    }
-    warnNotEnoughArgs = (P != E || i >= NumArgs);
-    isMethod = 1;
+    numFormalParams = MD->param_size();
+    calleeType = CT_Method;
   } else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
-    // skip over named parameters.
-    ObjCMethodDecl::param_iterator P, E = FD->param_end();
-    for (P = FD->param_begin(); (P != E && i < NumArgs); ++P) {
-      if (nullPos)
-        --nullPos;
-      else
-        ++i;
-    }
-    warnNotEnoughArgs = (P != E || i >= NumArgs);
-  } else if (VarDecl *V = dyn_cast<VarDecl>(D)) {
-    // block or function pointer call.
-    QualType Ty = V->getType();
-    if (Ty->isBlockPointerType() || Ty->isFunctionPointerType()) {
-      const FunctionType *FT = Ty->isFunctionPointerType()
-      ? Ty->getAs<PointerType>()->getPointeeType()->getAs<FunctionType>()
-      : Ty->getAs<BlockPointerType>()->getPointeeType()->getAs<FunctionType>();
-      if (const FunctionProtoType *Proto = dyn_cast<FunctionProtoType>(FT)) {
-        unsigned NumArgsInProto = Proto->getNumArgs();
-        unsigned k;
-        for (k = 0; (k != NumArgsInProto && i < NumArgs); k++) {
-          if (nullPos)
-            --nullPos;
-          else
-            ++i;
-        }
-        warnNotEnoughArgs = (k != NumArgsInProto || i >= NumArgs);
-      }
-      if (Ty->isBlockPointerType())
-        isMethod = 2;
-    } else
+    numFormalParams = FD->param_size();
+    calleeType = CT_Function;
+  } else if (isa<VarDecl>(D)) {
+    QualType type = cast<ValueDecl>(D)->getType();
+    const FunctionType *fn = 0;
+    if (const PointerType *ptr = type->getAs<PointerType>()) {
+      fn = ptr->getPointeeType()->getAs<FunctionType>();
+      if (!fn) return;
+      calleeType = CT_Function;
+    } else if (const BlockPointerType *ptr = type->getAs<BlockPointerType>()) {
+      fn = ptr->getPointeeType()->castAs<FunctionType>();
+      calleeType = CT_Block;
+    } else {
       return;
-  } else
-    return;
+    }
 
-  if (warnNotEnoughArgs) {
-    Diag(Loc, diag::warn_not_enough_argument) << D->getDeclName();
-    Diag(D->getLocation(), diag::note_sentinel_here) << isMethod;
+    if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fn)) {
+      numFormalParams = proto->getNumArgs();
+    } else {
+      numFormalParams = 0;
+    }
+  } else {
     return;
   }
-  int sentinel = i;
-  while (sentinelPos > 0 && i < NumArgs-1) {
-    --sentinelPos;
-    ++i;
-  }
-  if (sentinelPos > 0) {
+
+  // "nullPos" is the number of formal parameters at the end which
+  // effectively count as part of the variadic arguments.  This is
+  // useful if you would prefer to not have *any* formal parameters,
+  // but the language forces you to have at least one.
+  unsigned nullPos = attr->getNullPos();
+  assert((nullPos == 0 || nullPos == 1) && "invalid null position on sentinel");
+  numFormalParams = (nullPos > numFormalParams ? 0 : numFormalParams - nullPos);
+
+  // The number of arguments which should follow the sentinel.
+  unsigned numArgsAfterSentinel = attr->getSentinel();
+
+  // If there aren't enough arguments for all the formal parameters,
+  // the sentinel, and the args after the sentinel, complain.
+  if (numArgs < numFormalParams + numArgsAfterSentinel + 1) {
     Diag(Loc, diag::warn_not_enough_argument) << D->getDeclName();
-    Diag(D->getLocation(), diag::note_sentinel_here) << isMethod;
+    Diag(D->getLocation(), diag::note_sentinel_here) << calleeType;
     return;
   }
-  while (i < NumArgs-1) {
-    ++i;
-    ++sentinel;
-  }
-  Expr *sentinelExpr = Args[sentinel];
+
+  // Otherwise, find the sentinel expression.
+  Expr *sentinelExpr = args[numArgs - numArgsAfterSentinel - 1];
   if (!sentinelExpr) return;
-  if (sentinelExpr->isTypeDependent()) return;
   if (sentinelExpr->isValueDependent()) return;
 
   // nullptr_t is always treated as null.
@@ -246,13 +255,32 @@ void Sema::DiagnoseSentinelCalls(NamedDecl *D, SourceLocation Loc,
   // Unfortunately, __null has type 'int'.
   if (isa<GNUNullExpr>(sentinelExpr)) return;
 
-  Diag(Loc, diag::warn_missing_sentinel) << isMethod;
-  Diag(D->getLocation(), diag::note_sentinel_here) << isMethod;
+  // Pick a reasonable string to insert.  Optimistically use 'nil' or
+  // 'NULL' if those are actually defined in the context.  Only use
+  // 'nil' for ObjC methods, where it's much more likely that the
+  // variadic arguments form a list of object pointers.
+  SourceLocation MissingNilLoc
+    = PP.getLocForEndOfToken(sentinelExpr->getLocEnd());
+  std::string NullValue;
+  if (calleeType == CT_Method &&
+      PP.getIdentifierInfo("nil")->hasMacroDefinition())
+    NullValue = "nil";
+  else if (PP.getIdentifierInfo("NULL")->hasMacroDefinition())
+    NullValue = "NULL";
+  else
+    NullValue = "(void*) 0";
+
+  if (MissingNilLoc.isInvalid())
+    Diag(Loc, diag::warn_missing_sentinel) << calleeType;
+  else
+    Diag(MissingNilLoc, diag::warn_missing_sentinel) 
+      << calleeType
+      << FixItHint::CreateInsertion(MissingNilLoc, ", " + NullValue);
+  Diag(D->getLocation(), diag::note_sentinel_here) << calleeType;
 }
 
-SourceRange Sema::getExprRange(ExprTy *E) const {
-  Expr *Ex = (Expr *)E;
-  return Ex? Ex->getSourceRange() : SourceRange();
+SourceRange Sema::getExprRange(Expr *E) const {
+  return E ? E->getSourceRange() : SourceRange();
 }
 
 //===----------------------------------------------------------------------===//
@@ -261,6 +289,13 @@ SourceRange Sema::getExprRange(ExprTy *E) const {
 
 /// DefaultFunctionArrayConversion (C99 6.3.2.1p3, C99 6.3.2.1p4).
 ExprResult Sema::DefaultFunctionArrayConversion(Expr *E) {
+  // Handle any placeholder expressions which made it here.
+  if (E->getType()->isPlaceholderType()) {
+    ExprResult result = CheckPlaceholderExpr(E);
+    if (result.isInvalid()) return ExprError();
+    E = result.take();
+  }
+  
   QualType Ty = E->getType();
   assert(!Ty.isNull() && "DefaultFunctionArrayConversion - missing type");
 
@@ -306,6 +341,13 @@ static void CheckForNullPointerDereference(Sema &S, Expr *E) {
 }
 
 ExprResult Sema::DefaultLvalueConversion(Expr *E) {
+  // Handle any placeholder expressions which made it here.
+  if (E->getType()->isPlaceholderType()) {
+    ExprResult result = CheckPlaceholderExpr(E);
+    if (result.isInvalid()) return ExprError();
+    E = result.take();
+  }
+  
   // C++ [conv.lval]p1:
   //   A glvalue of a non-function, non-array type T can be
   //   converted to a prvalue.
@@ -314,6 +356,10 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) {
   QualType T = E->getType();
   assert(!T.isNull() && "r-value conversion on typeless expression?");
 
+  // We can't do lvalue-to-rvalue on atomics yet.
+  if (T->getAs<AtomicType>())
+    return Owned(E);
+
   // Create a load out of an ObjCProperty l-value, if necessary.
   if (E->getObjectKind() == OK_ObjCProperty) {
     ExprResult Res = ConvertPropertyForRValue(E);
@@ -350,14 +396,14 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) {
   // C99 6.3.2.1p2:
   //   If the lvalue has qualified type, the value has the unqualified
   //   version of the type of the lvalue; otherwise, the value has the
-  //   type of the lvalue.    
+  //   type of the lvalue.
   if (T.hasQualifiers())
     T = T.getUnqualifiedType();
 
-  CheckArrayAccess(E);
-  
-  return Owned(ImplicitCastExpr::Create(Context, T, CK_LValueToRValue,
-                                        E, 0, VK_RValue));
+  ExprResult Res = Owned(ImplicitCastExpr::Create(Context, T, CK_LValueToRValue,
+                                                  E, 0, VK_RValue));
+
+  return Res;
 }
 
 ExprResult Sema::DefaultFunctionArrayLvalueConversion(Expr *E) {
@@ -382,10 +428,15 @@ ExprResult Sema::UsualUnaryConversions(Expr *E) {
   if (Res.isInvalid())
     return Owned(E);
   E = Res.take();
-  
+
   QualType Ty = E->getType();
   assert(!Ty.isNull() && "UsualUnaryConversions - missing type");
-  
+
+  // Half FP is a bit different: it's a storage-only type, meaning that any
+  // "use" of it should be promoted to float.
+  if (Ty->isHalfType())
+    return ImpCastExprToType(Res.take(), Context.FloatTy, CK_FloatingCast);
+
   // Try to perform integral promotions if the object has a theoretically
   // promotable type.
   if (Ty->isIntegralOrUnscopedEnumerationType()) {
@@ -402,7 +453,7 @@ ExprResult Sema::UsualUnaryConversions(Expr *E) {
     //   value is converted to an int; otherwise, it is converted to an
     //   unsigned int. These are called the integer promotions. All
     //   other types are unchanged by the integer promotions.
-  
+
     QualType PTy = Context.isPromotableBitField(E);
     if (!PTy.isNull()) {
       E = ImpCastExprToType(E, PTy, CK_IntegralCast).take();
@@ -433,6 +484,28 @@ ExprResult Sema::DefaultArgumentPromotion(Expr *E) {
   if (Ty->isSpecificBuiltinType(BuiltinType::Float))
     E = ImpCastExprToType(E, Context.DoubleTy, CK_FloatingCast).take();
 
+  // C++ performs lvalue-to-rvalue conversion as a default argument
+  // promotion, even on class types, but note:
+  //   C++11 [conv.lval]p2:
+  //     When an lvalue-to-rvalue conversion occurs in an unevaluated
+  //     operand or a subexpression thereof the value contained in the
+  //     referenced object is not accessed. Otherwise, if the glvalue
+  //     has a class type, the conversion copy-initializes a temporary
+  //     of type T from the glvalue and the result of the conversion
+  //     is a prvalue for the temporary.
+  // FIXME: add some way to gate this entire thing for correctness in
+  // potentially potentially evaluated contexts.
+  if (getLangOptions().CPlusPlus && E->isGLValue() && 
+      ExprEvalContexts.back().Context != Unevaluated) {
+    ExprResult Temp = PerformCopyInitialization(
+                       InitializedEntity::InitializeTemporary(E->getType()),
+                                                E->getExprLoc(),
+                                                Owned(E));
+    if (Temp.isInvalid())
+      return ExprError();
+    E = Temp.get();
+  }
+
   return Owned(E);
 }
 
@@ -450,20 +523,17 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
     return ExprError();
   E = ExprRes.take();
 
-  // __builtin_va_start takes the second argument as a "varargs" argument, but
-  // it doesn't actually do anything with it.  It doesn't need to be non-pod
-  // etc.
-  if (FDecl && FDecl->getBuiltinID() == Builtin::BI__builtin_va_start)
-    return Owned(E);
-  
   // Don't allow one to pass an Objective-C interface to a vararg.
   if (E->getType()->isObjCObjectType() &&
     DiagRuntimeBehavior(E->getLocStart(), 0,
                         PDiag(diag::err_cannot_pass_objc_interface_to_vararg)
                           << E->getType() << CT))
     return ExprError();
-  
-  if (!E->getType().isPODType(Context)) {
+
+  // Complain about passing non-POD types through varargs. However, don't
+  // perform this check for incomplete types, which we can get here when we're
+  // in an unevaluated context.
+  if (!E->getType()->isIncompleteType() && !E->getType().isPODType(Context)) {
     // C++0x [expr.call]p7:
     //   Passing a potentially-evaluated argument of class type (Clause 9) 
     //   having a non-trivial copy constructor, a non-trivial move constructor,
@@ -507,8 +577,7 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
       ExprResult Comma = ActOnBinOp(TUScope, E->getLocStart(), tok::comma,
                                     Call.get(), E);
       if (Comma.isInvalid())
-        return ExprError();
-      
+        return ExprError();      
       E = Comma.get();
     }
   }
@@ -516,305 +585,361 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
   return Owned(E);
 }
 
+/// \brief Converts an integer to complex float type.  Helper function of
+/// UsualArithmeticConversions()
+///
+/// \return false if the integer expression is an integer type and is
+/// successfully converted to the complex type.
+static bool handleIntegerToComplexFloatConversion(Sema &S, ExprResult &IntExpr,
+                                                  ExprResult &ComplexExpr,
+                                                  QualType IntTy,
+                                                  QualType ComplexTy,
+                                                  bool SkipCast) {
+  if (IntTy->isComplexType() || IntTy->isRealFloatingType()) return true;
+  if (SkipCast) return false;
+  if (IntTy->isIntegerType()) {
+    QualType fpTy = cast<ComplexType>(ComplexTy)->getElementType();
+    IntExpr = S.ImpCastExprToType(IntExpr.take(), fpTy, CK_IntegralToFloating);
+    IntExpr = S.ImpCastExprToType(IntExpr.take(), ComplexTy,
+                                  CK_FloatingRealToComplex);
+  } else {
+    assert(IntTy->isComplexIntegerType());
+    IntExpr = S.ImpCastExprToType(IntExpr.take(), ComplexTy,
+                                  CK_IntegralComplexToFloatingComplex);
+  }
+  return false;
+}
+
+/// \brief Takes two complex float types and converts them to the same type.
+/// Helper function of UsualArithmeticConversions()
+static QualType
+handleComplexFloatToComplexFloatConverstion(Sema &S, ExprResult &LHS,
+                                            ExprResult &RHS, QualType LHSType,
+                                            QualType RHSType,
+                                            bool IsCompAssign) {
+  int order = S.Context.getFloatingTypeOrder(LHSType, RHSType);
+
+  if (order < 0) {
+    // _Complex float -> _Complex double
+    if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_FloatingComplexCast);
+    return RHSType;
+  }
+  if (order > 0)
+    // _Complex float -> _Complex double
+    RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_FloatingComplexCast);
+  return LHSType;
+}
+
+/// \brief Converts otherExpr to complex float and promotes complexExpr if
+/// necessary.  Helper function of UsualArithmeticConversions()
+static QualType handleOtherComplexFloatConversion(Sema &S,
+                                                  ExprResult &ComplexExpr,
+                                                  ExprResult &OtherExpr,
+                                                  QualType ComplexTy,
+                                                  QualType OtherTy,
+                                                  bool ConvertComplexExpr,
+                                                  bool ConvertOtherExpr) {
+  int order = S.Context.getFloatingTypeOrder(ComplexTy, OtherTy);
+
+  // If just the complexExpr is complex, the otherExpr needs to be converted,
+  // and the complexExpr might need to be promoted.
+  if (order > 0) { // complexExpr is wider
+    // float -> _Complex double
+    if (ConvertOtherExpr) {
+      QualType fp = cast<ComplexType>(ComplexTy)->getElementType();
+      OtherExpr = S.ImpCastExprToType(OtherExpr.take(), fp, CK_FloatingCast);
+      OtherExpr = S.ImpCastExprToType(OtherExpr.take(), ComplexTy,
+                                      CK_FloatingRealToComplex);
+    }
+    return ComplexTy;
+  }
+
+  // otherTy is at least as wide.  Find its corresponding complex type.
+  QualType result = (order == 0 ? ComplexTy :
+                                  S.Context.getComplexType(OtherTy));
+
+  // double -> _Complex double
+  if (ConvertOtherExpr)
+    OtherExpr = S.ImpCastExprToType(OtherExpr.take(), result,
+                                    CK_FloatingRealToComplex);
+
+  // _Complex float -> _Complex double
+  if (ConvertComplexExpr && order < 0)
+    ComplexExpr = S.ImpCastExprToType(ComplexExpr.take(), result,
+                                      CK_FloatingComplexCast);
+
+  return result;
+}
+
+/// \brief Handle arithmetic conversion with complex types.  Helper function of
+/// UsualArithmeticConversions()
+static QualType handleComplexFloatConversion(Sema &S, ExprResult &LHS,
+                                             ExprResult &RHS, QualType LHSType,
+                                             QualType RHSType,
+                                             bool IsCompAssign) {
+  // if we have an integer operand, the result is the complex type.
+  if (!handleIntegerToComplexFloatConversion(S, RHS, LHS, RHSType, LHSType,
+                                             /*skipCast*/false))
+    return LHSType;
+  if (!handleIntegerToComplexFloatConversion(S, LHS, RHS, LHSType, RHSType,
+                                             /*skipCast*/IsCompAssign))
+    return RHSType;
+
+  // This handles complex/complex, complex/float, or float/complex.
+  // When both operands are complex, the shorter operand is converted to the
+  // type of the longer, and that is the type of the result. This corresponds
+  // to what is done when combining two real floating-point operands.
+  // The fun begins when size promotion occur across type domains.
+  // From H&S 6.3.4: When one operand is complex and the other is a real
+  // floating-point type, the less precise type is converted, within it's
+  // real or complex domain, to the precision of the other type. For example,
+  // when combining a "long double" with a "double _Complex", the
+  // "double _Complex" is promoted to "long double _Complex".
+
+  bool LHSComplexFloat = LHSType->isComplexType();
+  bool RHSComplexFloat = RHSType->isComplexType();
+
+  // If both are complex, just cast to the more precise type.
+  if (LHSComplexFloat && RHSComplexFloat)
+    return handleComplexFloatToComplexFloatConverstion(S, LHS, RHS,
+                                                       LHSType, RHSType,
+                                                       IsCompAssign);
+
+  // If only one operand is complex, promote it if necessary and convert the
+  // other operand to complex.
+  if (LHSComplexFloat)
+    return handleOtherComplexFloatConversion(
+        S, LHS, RHS, LHSType, RHSType, /*convertComplexExpr*/!IsCompAssign,
+        /*convertOtherExpr*/ true);
+
+  assert(RHSComplexFloat);
+  return handleOtherComplexFloatConversion(
+      S, RHS, LHS, RHSType, LHSType, /*convertComplexExpr*/true,
+      /*convertOtherExpr*/ !IsCompAssign);
+}
+
+/// \brief Hande arithmetic conversion from integer to float.  Helper function
+/// of UsualArithmeticConversions()
+static QualType handleIntToFloatConversion(Sema &S, ExprResult &FloatExpr,
+                                           ExprResult &IntExpr,
+                                           QualType FloatTy, QualType IntTy,
+                                           bool ConvertFloat, bool ConvertInt) {
+  if (IntTy->isIntegerType()) {
+    if (ConvertInt)
+      // Convert intExpr to the lhs floating point type.
+      IntExpr = S.ImpCastExprToType(IntExpr.take(), FloatTy,
+                                    CK_IntegralToFloating);
+    return FloatTy;
+  }
+     
+  // Convert both sides to the appropriate complex float.
+  assert(IntTy->isComplexIntegerType());
+  QualType result = S.Context.getComplexType(FloatTy);
+
+  // _Complex int -> _Complex float
+  if (ConvertInt)
+    IntExpr = S.ImpCastExprToType(IntExpr.take(), result,
+                                  CK_IntegralComplexToFloatingComplex);
+
+  // float -> _Complex float
+  if (ConvertFloat)
+    FloatExpr = S.ImpCastExprToType(FloatExpr.take(), result,
+                                    CK_FloatingRealToComplex);
+
+  return result;
+}
+
+/// \brief Handle arithmethic conversion with floating point types.  Helper
+/// function of UsualArithmeticConversions()
+static QualType handleFloatConversion(Sema &S, ExprResult &LHS,
+                                      ExprResult &RHS, QualType LHSType,
+                                      QualType RHSType, bool IsCompAssign) {
+  bool LHSFloat = LHSType->isRealFloatingType();
+  bool RHSFloat = RHSType->isRealFloatingType();
+
+  // If we have two real floating types, convert the smaller operand
+  // to the bigger result.
+  if (LHSFloat && RHSFloat) {
+    int order = S.Context.getFloatingTypeOrder(LHSType, RHSType);
+    if (order > 0) {
+      RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_FloatingCast);
+      return LHSType;
+    }
+
+    assert(order < 0 && "illegal float comparison");
+    if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_FloatingCast);
+    return RHSType;
+  }
+
+  if (LHSFloat)
+    return handleIntToFloatConversion(S, LHS, RHS, LHSType, RHSType,
+                                      /*convertFloat=*/!IsCompAssign,
+                                      /*convertInt=*/ true);
+  assert(RHSFloat);
+  return handleIntToFloatConversion(S, RHS, LHS, RHSType, LHSType,
+                                    /*convertInt=*/ true,
+                                    /*convertFloat=*/!IsCompAssign);
+}
+
+/// \brief Handle conversions with GCC complex int extension.  Helper function
+/// of UsualArithmeticConversions()
+// FIXME: if the operands are (int, _Complex long), we currently
+// don't promote the complex.  Also, signedness?
+static QualType handleComplexIntConversion(Sema &S, ExprResult &LHS,
+                                           ExprResult &RHS, QualType LHSType,
+                                           QualType RHSType,
+                                           bool IsCompAssign) {
+  const ComplexType *LHSComplexInt = LHSType->getAsComplexIntegerType();
+  const ComplexType *RHSComplexInt = RHSType->getAsComplexIntegerType();
+
+  if (LHSComplexInt && RHSComplexInt) {
+    int order = S.Context.getIntegerTypeOrder(LHSComplexInt->getElementType(),
+                                              RHSComplexInt->getElementType());
+    assert(order && "inequal types with equal element ordering");
+    if (order > 0) {
+      // _Complex int -> _Complex long
+      RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_IntegralComplexCast);
+      return LHSType;
+    }
+
+    if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_IntegralComplexCast);
+    return RHSType;
+  }
+
+  if (LHSComplexInt) {
+    // int -> _Complex int
+    RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_IntegralRealToComplex);
+    return LHSType;
+  }
+
+  assert(RHSComplexInt);
+  // int -> _Complex int
+  if (!IsCompAssign)
+    LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_IntegralRealToComplex);
+  return RHSType;
+}
+
+/// \brief Handle integer arithmetic conversions.  Helper function of
+/// UsualArithmeticConversions()
+static QualType handleIntegerConversion(Sema &S, ExprResult &LHS,
+                                        ExprResult &RHS, QualType LHSType,
+                                        QualType RHSType, bool IsCompAssign) {
+  // The rules for this case are in C99 6.3.1.8
+  int order = S.Context.getIntegerTypeOrder(LHSType, RHSType);
+  bool LHSSigned = LHSType->hasSignedIntegerRepresentation();
+  bool RHSSigned = RHSType->hasSignedIntegerRepresentation();
+  if (LHSSigned == RHSSigned) {
+    // Same signedness; use the higher-ranked type
+    if (order >= 0) {
+      RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_IntegralCast);
+      return LHSType;
+    } else if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_IntegralCast);
+    return RHSType;
+  } else if (order != (LHSSigned ? 1 : -1)) {
+    // The unsigned type has greater than or equal rank to the
+    // signed type, so use the unsigned type
+    if (RHSSigned) {
+      RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_IntegralCast);
+      return LHSType;
+    } else if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_IntegralCast);
+    return RHSType;
+  } else if (S.Context.getIntWidth(LHSType) != S.Context.getIntWidth(RHSType)) {
+    // The two types are different widths; if we are here, that
+    // means the signed type is larger than the unsigned type, so
+    // use the signed type.
+    if (LHSSigned) {
+      RHS = S.ImpCastExprToType(RHS.take(), LHSType, CK_IntegralCast);
+      return LHSType;
+    } else if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), RHSType, CK_IntegralCast);
+    return RHSType;
+  } else {
+    // The signed type is higher-ranked than the unsigned type,
+    // but isn't actually any bigger (like unsigned int and long
+    // on most 32-bit systems).  Use the unsigned type corresponding
+    // to the signed type.
+    QualType result =
+      S.Context.getCorrespondingUnsignedType(LHSSigned ? LHSType : RHSType);
+    RHS = S.ImpCastExprToType(RHS.take(), result, CK_IntegralCast);
+    if (!IsCompAssign)
+      LHS = S.ImpCastExprToType(LHS.take(), result, CK_IntegralCast);
+    return result;
+  }
+}
+
 /// UsualArithmeticConversions - Performs various conversions that are common to
 /// binary operators (C99 6.3.1.8). If both operands aren't arithmetic, this
 /// routine returns the first non-arithmetic type found. The client is
 /// responsible for emitting appropriate error diagnostics.
 /// FIXME: verify the conversion rules for "complex int" are consistent with
 /// GCC.
-QualType Sema::UsualArithmeticConversions(ExprResult &lhsExpr, ExprResult &rhsExpr,
-                                          bool isCompAssign) {
-  if (!isCompAssign) {
-    lhsExpr = UsualUnaryConversions(lhsExpr.take());
-    if (lhsExpr.isInvalid())
+QualType Sema::UsualArithmeticConversions(ExprResult &LHS, ExprResult &RHS,
+                                          bool IsCompAssign) {
+  if (!IsCompAssign) {
+    LHS = UsualUnaryConversions(LHS.take());
+    if (LHS.isInvalid())
       return QualType();
   }
 
-  rhsExpr = UsualUnaryConversions(rhsExpr.take());
-  if (rhsExpr.isInvalid())
+  RHS = UsualUnaryConversions(RHS.take());
+  if (RHS.isInvalid())
     return QualType();
 
   // For conversion purposes, we ignore any qualifiers.
   // For example, "const float" and "float" are equivalent.
-  QualType lhs =
-    Context.getCanonicalType(lhsExpr.get()->getType()).getUnqualifiedType();
-  QualType rhs =
-    Context.getCanonicalType(rhsExpr.get()->getType()).getUnqualifiedType();
+  QualType LHSType =
+    Context.getCanonicalType(LHS.get()->getType()).getUnqualifiedType();
+  QualType RHSType =
+    Context.getCanonicalType(RHS.get()->getType()).getUnqualifiedType();
 
   // If both types are identical, no conversion is needed.
-  if (lhs == rhs)
-    return lhs;
+  if (LHSType == RHSType)
+    return LHSType;
 
   // If either side is a non-arithmetic type (e.g. a pointer), we are done.
   // The caller can deal with this (e.g. pointer + int).
-  if (!lhs->isArithmeticType() || !rhs->isArithmeticType())
-    return lhs;
+  if (!LHSType->isArithmeticType() || !RHSType->isArithmeticType())
+    return LHSType;
 
   // Apply unary and bitfield promotions to the LHS's type.
-  QualType lhs_unpromoted = lhs;
-  if (lhs->isPromotableIntegerType())
-    lhs = Context.getPromotedIntegerType(lhs);
-  QualType LHSBitfieldPromoteTy = Context.isPromotableBitField(lhsExpr.get());
+  QualType LHSUnpromotedType = LHSType;
+  if (LHSType->isPromotableIntegerType())
+    LHSType = Context.getPromotedIntegerType(LHSType);
+  QualType LHSBitfieldPromoteTy = Context.isPromotableBitField(LHS.get());
   if (!LHSBitfieldPromoteTy.isNull())
-    lhs = LHSBitfieldPromoteTy;
-  if (lhs != lhs_unpromoted && !isCompAssign)
-    lhsExpr = ImpCastExprToType(lhsExpr.take(), lhs, CK_IntegralCast);
+    LHSType = LHSBitfieldPromoteTy;
+  if (LHSType != LHSUnpromotedType && !IsCompAssign)
+    LHS = ImpCastExprToType(LHS.take(), LHSType, CK_IntegralCast);
 
   // If both types are identical, no conversion is needed.
-  if (lhs == rhs)
-    return lhs;
+  if (LHSType == RHSType)
+    return LHSType;
 
   // At this point, we have two different arithmetic types.
 
   // Handle complex types first (C99 6.3.1.8p1).
-  bool LHSComplexFloat = lhs->isComplexType();
-  bool RHSComplexFloat = rhs->isComplexType();
-  if (LHSComplexFloat || RHSComplexFloat) {
-    // if we have an integer operand, the result is the complex type.
-
-    if (!RHSComplexFloat && !rhs->isRealFloatingType()) {
-      if (rhs->isIntegerType()) {
-        QualType fp = cast<ComplexType>(lhs)->getElementType();
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), fp, CK_IntegralToFloating);
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_FloatingRealToComplex);
-      } else {
-        assert(rhs->isComplexIntegerType());
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralComplexToFloatingComplex);
-      }
-      return lhs;
-    }
-
-    if (!LHSComplexFloat && !lhs->isRealFloatingType()) {
-      if (!isCompAssign) {
-        // int -> float -> _Complex float
-        if (lhs->isIntegerType()) {
-          QualType fp = cast<ComplexType>(rhs)->getElementType();
-          lhsExpr = ImpCastExprToType(lhsExpr.take(), fp, CK_IntegralToFloating);
-          lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_FloatingRealToComplex);
-        } else {
-          assert(lhs->isComplexIntegerType());
-          lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralComplexToFloatingComplex);
-        }
-      }
-      return rhs;
-    }
-
-    // This handles complex/complex, complex/float, or float/complex.
-    // When both operands are complex, the shorter operand is converted to the
-    // type of the longer, and that is the type of the result. This corresponds
-    // to what is done when combining two real floating-point operands.
-    // The fun begins when size promotion occur across type domains.
-    // From H&S 6.3.4: When one operand is complex and the other is a real
-    // floating-point type, the less precise type is converted, within it's
-    // real or complex domain, to the precision of the other type. For example,
-    // when combining a "long double" with a "double _Complex", the
-    // "double _Complex" is promoted to "long double _Complex".
-    int order = Context.getFloatingTypeOrder(lhs, rhs);
-
-    // If both are complex, just cast to the more precise type.
-    if (LHSComplexFloat && RHSComplexFloat) {
-      if (order > 0) {
-        // _Complex float -> _Complex double
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_FloatingComplexCast);
-        return lhs;
-
-      } else if (order < 0) {
-        // _Complex float -> _Complex double
-        if (!isCompAssign)
-          lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_FloatingComplexCast);
-        return rhs;
-      }
-      return lhs;
-    }
-
-    // If just the LHS is complex, the RHS needs to be converted,
-    // and the LHS might need to be promoted.
-    if (LHSComplexFloat) {
-      if (order > 0) { // LHS is wider
-        // float -> _Complex double
-        QualType fp = cast<ComplexType>(lhs)->getElementType();
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), fp, CK_FloatingCast);
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_FloatingRealToComplex);
-        return lhs;        
-      }
-
-      // RHS is at least as wide.  Find its corresponding complex type.
-      QualType result = (order == 0 ? lhs : Context.getComplexType(rhs));
-
-      // double -> _Complex double
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), result, CK_FloatingRealToComplex);
-
-      // _Complex float -> _Complex double
-      if (!isCompAssign && order < 0)
-        lhsExpr = ImpCastExprToType(lhsExpr.take(), result, CK_FloatingComplexCast);
-
-      return result;
-    }
-
-    // Just the RHS is complex, so the LHS needs to be converted
-    // and the RHS might need to be promoted.
-    assert(RHSComplexFloat);
-
-    if (order < 0) { // RHS is wider
-      // float -> _Complex double
-      if (!isCompAssign) {
-        QualType fp = cast<ComplexType>(rhs)->getElementType();
-        lhsExpr = ImpCastExprToType(lhsExpr.take(), fp, CK_FloatingCast);
-        lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_FloatingRealToComplex);
-      }
-      return rhs;
-    }
-
-    // LHS is at least as wide.  Find its corresponding complex type.
-    QualType result = (order == 0 ? rhs : Context.getComplexType(lhs));
-
-    // double -> _Complex double
-    if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), result, CK_FloatingRealToComplex);
-
-    // _Complex float -> _Complex double
-    if (order > 0)
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), result, CK_FloatingComplexCast);
-
-    return result;
-  }
+  if (LHSType->isComplexType() || RHSType->isComplexType())
+    return handleComplexFloatConversion(*this, LHS, RHS, LHSType, RHSType,
+                                        IsCompAssign);
 
   // Now handle "real" floating types (i.e. float, double, long double).
-  bool LHSFloat = lhs->isRealFloatingType();
-  bool RHSFloat = rhs->isRealFloatingType();
-  if (LHSFloat || RHSFloat) {
-    // If we have two real floating types, convert the smaller operand
-    // to the bigger result.
-    if (LHSFloat && RHSFloat) {
-      int order = Context.getFloatingTypeOrder(lhs, rhs);
-      if (order > 0) {
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_FloatingCast);
-        return lhs;
-      }
-
-      assert(order < 0 && "illegal float comparison");
-      if (!isCompAssign)
-        lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_FloatingCast);
-      return rhs;
-    }
-
-    // If we have an integer operand, the result is the real floating type.
-    if (LHSFloat) {
-      if (rhs->isIntegerType()) {
-        // Convert rhs to the lhs floating point type.
-        rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralToFloating);
-        return lhs;
-      }
-
-      // Convert both sides to the appropriate complex float.
-      assert(rhs->isComplexIntegerType());
-      QualType result = Context.getComplexType(lhs);
-
-      // _Complex int -> _Complex float
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), result, CK_IntegralComplexToFloatingComplex);
-
-      // float -> _Complex float
-      if (!isCompAssign)
-        lhsExpr = ImpCastExprToType(lhsExpr.take(), result, CK_FloatingRealToComplex);
-
-      return result;
-    }
-
-    assert(RHSFloat);
-    if (lhs->isIntegerType()) {
-      // Convert lhs to the rhs floating point type.
-      if (!isCompAssign)
-        lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralToFloating);
-      return rhs;
-    }
-
-    // Convert both sides to the appropriate complex float.
-    assert(lhs->isComplexIntegerType());
-    QualType result = Context.getComplexType(rhs);
-
-    // _Complex int -> _Complex float
-    if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), result, CK_IntegralComplexToFloatingComplex);
-
-    // float -> _Complex float
-    rhsExpr = ImpCastExprToType(rhsExpr.take(), result, CK_FloatingRealToComplex);
-
-    return result;
-  }
+  if (LHSType->isRealFloatingType() || RHSType->isRealFloatingType())
+    return handleFloatConversion(*this, LHS, RHS, LHSType, RHSType,
+                                 IsCompAssign);
 
   // Handle GCC complex int extension.
-  // FIXME: if the operands are (int, _Complex long), we currently
-  // don't promote the complex.  Also, signedness?
-  const ComplexType *lhsComplexInt = lhs->getAsComplexIntegerType();
-  const ComplexType *rhsComplexInt = rhs->getAsComplexIntegerType();
-  if (lhsComplexInt && rhsComplexInt) {
-    int order = Context.getIntegerTypeOrder(lhsComplexInt->getElementType(),
-                                            rhsComplexInt->getElementType());
-    assert(order && "inequal types with equal element ordering");
-    if (order > 0) {
-      // _Complex int -> _Complex long
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralComplexCast);
-      return lhs;
-    }
-
-    if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralComplexCast);
-    return rhs;
-  } else if (lhsComplexInt) {
-    // int -> _Complex int
-    rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralRealToComplex);
-    return lhs;
-  } else if (rhsComplexInt) {
-    // int -> _Complex int
-    if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralRealToComplex);
-    return rhs;
-  }
+  if (LHSType->isComplexIntegerType() || RHSType->isComplexIntegerType())
+    return handleComplexIntConversion(*this, LHS, RHS, LHSType, RHSType,
+                                      IsCompAssign);
 
   // Finally, we have two differing integer types.
-  // The rules for this case are in C99 6.3.1.8
-  int compare = Context.getIntegerTypeOrder(lhs, rhs);
-  bool lhsSigned = lhs->hasSignedIntegerRepresentation(),
-       rhsSigned = rhs->hasSignedIntegerRepresentation();
-  if (lhsSigned == rhsSigned) {
-    // Same signedness; use the higher-ranked type
-    if (compare >= 0) {
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralCast);
-      return lhs;
-    } else if (!isCompAssign) 
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralCast);
-    return rhs;
-  } else if (compare != (lhsSigned ? 1 : -1)) {
-    // The unsigned type has greater than or equal rank to the
-    // signed type, so use the unsigned type
-    if (rhsSigned) {
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralCast);
-      return lhs;
-    } else if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralCast);
-    return rhs;
-  } else if (Context.getIntWidth(lhs) != Context.getIntWidth(rhs)) {
-    // The two types are different widths; if we are here, that
-    // means the signed type is larger than the unsigned type, so
-    // use the signed type.
-    if (lhsSigned) {
-      rhsExpr = ImpCastExprToType(rhsExpr.take(), lhs, CK_IntegralCast);
-      return lhs;
-    } else if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), rhs, CK_IntegralCast);
-    return rhs;
-  } else {
-    // The signed type is higher-ranked than the unsigned type,
-    // but isn't actually any bigger (like unsigned int and long
-    // on most 32-bit systems).  Use the unsigned type corresponding
-    // to the signed type.
-    QualType result =
-      Context.getCorrespondingUnsignedType(lhsSigned ? lhs : rhs);
-    rhsExpr = ImpCastExprToType(rhsExpr.take(), result, CK_IntegralCast);
-    if (!isCompAssign)
-      lhsExpr = ImpCastExprToType(lhsExpr.take(), result, CK_IntegralCast);
-    return result;
-  }
+  return handleIntegerConversion(*this, LHS, RHS, LHSType, RHSType,
+                                 IsCompAssign);
 }
 
 //===----------------------------------------------------------------------===//
@@ -827,13 +952,13 @@ Sema::ActOnGenericSelectionExpr(SourceLocation KeyLoc,
                                 SourceLocation DefaultLoc,
                                 SourceLocation RParenLoc,
                                 Expr *ControllingExpr,
-                                MultiTypeArg types,
-                                MultiExprArg exprs) {
-  unsigned NumAssocs = types.size();
-  assert(NumAssocs == exprs.size());
+                                MultiTypeArg ArgTypes,
+                                MultiExprArg ArgExprs) {
+  unsigned NumAssocs = ArgTypes.size();
+  assert(NumAssocs == ArgExprs.size());
 
-  ParsedType *ParsedTypes = types.release();
-  Expr **Exprs = exprs.release();
+  ParsedType *ParsedTypes = ArgTypes.release();
+  Expr **Exprs = ArgExprs.release();
 
   TypeSourceInfo **Types = new TypeSourceInfo*[NumAssocs];
   for (unsigned i = 0; i < NumAssocs; ++i) {
@@ -922,7 +1047,7 @@ Sema::CreateGenericSelectionExpr(SourceLocation KeyLoc,
                    Types, Exprs, NumAssocs, DefaultLoc,
                    RParenLoc, ContainsUnexpandedParameterPack));
 
-  llvm::SmallVector<unsigned, 1> CompatIndices;
+  SmallVector<unsigned, 1> CompatIndices;
   unsigned DefaultIndex = -1U;
   for (unsigned i = 0; i < NumAssocs; ++i) {
     if (!Types[i])
@@ -942,7 +1067,7 @@ Sema::CreateGenericSelectionExpr(SourceLocation KeyLoc,
     Diag(ControllingExpr->getLocStart(), diag::err_generic_sel_multi_match)
       << ControllingExpr->getSourceRange() << ControllingExpr->getType()
       << (unsigned) CompatIndices.size();
-    for (llvm::SmallVector<unsigned, 1>::iterator I = CompatIndices.begin(),
+    for (SmallVector<unsigned, 1>::iterator I = CompatIndices.begin(),
          E = CompatIndices.end(); I != E; ++I) {
       Diag(Types[*I]->getTypeLoc().getBeginLoc(),
            diag::note_compat_assoc)
@@ -993,16 +1118,30 @@ Sema::ActOnStringLiteral(const Token *StringToks, unsigned NumStringToks) {
   if (Literal.hadError)
     return ExprError();
 
-  llvm::SmallVector<SourceLocation, 4> StringTokLocs;
+  SmallVector<SourceLocation, 4> StringTokLocs;
   for (unsigned i = 0; i != NumStringToks; ++i)
     StringTokLocs.push_back(StringToks[i].getLocation());
 
   QualType StrTy = Context.CharTy;
-  if (Literal.AnyWide) 
+  if (Literal.isWide())
     StrTy = Context.getWCharType();
+  else if (Literal.isUTF16())
+    StrTy = Context.Char16Ty;
+  else if (Literal.isUTF32())
+    StrTy = Context.Char32Ty;
   else if (Literal.Pascal)
     StrTy = Context.UnsignedCharTy;
 
+  StringLiteral::StringKind Kind = StringLiteral::Ascii;
+  if (Literal.isWide())
+    Kind = StringLiteral::Wide;
+  else if (Literal.isUTF8())
+    Kind = StringLiteral::UTF8;
+  else if (Literal.isUTF16())
+    Kind = StringLiteral::UTF16;
+  else if (Literal.isUTF32())
+    Kind = StringLiteral::UTF32;
+
   // A C++ string literal has a const-qualified element type (C++ 2.13.4p1).
   if (getLangOptions().CPlusPlus || getLangOptions().ConstStrings)
     StrTy.addConst();
@@ -1016,7 +1155,7 @@ Sema::ActOnStringLiteral(const Token *StringToks, unsigned NumStringToks) {
 
   // Pass &StringTokLocs[0], StringTokLocs.size() to factory!
   return Owned(StringLiteral::Create(Context, Literal.GetString(),
-                                     Literal.AnyWide, Literal.Pascal, StrTy,
+                                     Kind, Literal.Pascal, StrTy,
                                      &StringTokLocs[0],
                                      StringTokLocs.size()));
 }
@@ -1091,21 +1230,21 @@ diagnoseUncapturableValueReference(Sema &S, SourceLocation loc,
 
 /// There is a well-formed capture at a particular scope level;
 /// propagate it through all the nested blocks.
-static CaptureResult propagateCapture(Sema &S, unsigned validScopeIndex,
-                                      const BlockDecl::Capture &capture) {
-  VarDecl *var = capture.getVariable();
+static CaptureResult propagateCapture(Sema &S, unsigned ValidScopeIndex,
+                                      const BlockDecl::Capture &Capture) {
+  VarDecl *var = Capture.getVariable();
 
   // Update all the inner blocks with the capture information.
-  for (unsigned i = validScopeIndex + 1, e = S.FunctionScopes.size();
+  for (unsigned i = ValidScopeIndex + 1, e = S.FunctionScopes.size();
          i != e; ++i) {
     BlockScopeInfo *innerBlock = cast<BlockScopeInfo>(S.FunctionScopes[i]);
     innerBlock->Captures.push_back(
-      BlockDecl::Capture(capture.getVariable(), capture.isByRef(),
-                         /*nested*/ true, capture.getCopyExpr()));
+      BlockDecl::Capture(Capture.getVariable(), Capture.isByRef(),
+                         /*nested*/ true, Capture.getCopyExpr()));
     innerBlock->CaptureMap[var] = innerBlock->Captures.size(); // +1
   }
 
-  return capture.isByRef() ? CR_CaptureByRef : CR_Capture;
+  return Capture.isByRef() ? CR_CaptureByRef : CR_Capture;
 }
 
 /// shouldCaptureValueReference - Determine if a reference to the
@@ -1114,9 +1253,9 @@ static CaptureResult propagateCapture(Sema &S, unsigned validScopeIndex,
 /// This also keeps the captures set in the BlockScopeInfo records
 /// up-to-date.
 static CaptureResult shouldCaptureValueReference(Sema &S, SourceLocation loc,
-                                                 ValueDecl *value) {
+                                                 ValueDecl *Value) {
   // Only variables ever require capture.
-  VarDecl *var = dyn_cast<VarDecl>(value);
+  VarDecl *var = dyn_cast<VarDecl>(Value);
   if (!var) return CR_NoCapture;
 
   // Fast path: variables from the current context never require capture.
@@ -1225,19 +1364,19 @@ static CaptureResult shouldCaptureValueReference(Sema &S, SourceLocation loc,
                           blockScope->Captures.back());
 }
 
-static ExprResult BuildBlockDeclRefExpr(Sema &S, ValueDecl *vd,
+static ExprResult BuildBlockDeclRefExpr(Sema &S, ValueDecl *VD,
                                         const DeclarationNameInfo &NameInfo,
-                                        bool byRef) {
-  assert(isa<VarDecl>(vd) && "capturing non-variable");
+                                        bool ByRef) {
+  assert(isa<VarDecl>(VD) && "capturing non-variable");
 
-  VarDecl *var = cast<VarDecl>(vd);
+  VarDecl *var = cast<VarDecl>(VD);
   assert(var->hasLocalStorage() && "capturing non-local");
-  assert(byRef == var->hasAttr<BlocksAttr>() && "byref set wrong");
+  assert(ByRef == var->hasAttr<BlocksAttr>() && "byref set wrong");
 
   QualType exprType = var->getType().getNonReferenceType();
 
   BlockDeclRefExpr *BDRE;
-  if (!byRef) {
+  if (!ByRef) {
     // The variable will be bound by copy; make it const within the
     // closure, but record that this was done in the expression.
     bool constAdded = !exprType.isConstQualified();
@@ -1268,6 +1407,20 @@ ExprResult
 Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
                        const DeclarationNameInfo &NameInfo,
                        const CXXScopeSpec *SS) {
+  if (getLangOptions().CUDA)
+    if (const FunctionDecl *Caller = dyn_cast<FunctionDecl>(CurContext))
+      if (const FunctionDecl *Callee = dyn_cast<FunctionDecl>(D)) {
+        CUDAFunctionTarget CallerTarget = IdentifyCUDATarget(Caller),
+                           CalleeTarget = IdentifyCUDATarget(Callee);
+        if (CheckCUDATarget(CallerTarget, CalleeTarget)) {
+          Diag(NameInfo.getLoc(), diag::err_ref_bad_target)
+            << CalleeTarget << D->getIdentifier() << CallerTarget;
+          Diag(D->getLocation(), diag::note_previous_decl)
+            << D->getIdentifier();
+          return ExprError();
+        }
+      }
+
   MarkDeclarationReferenced(NameInfo.getLoc(), D);
 
   Expr *E = DeclRefExpr::Create(Context,
@@ -1276,7 +1429,8 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
                                 D, NameInfo, Ty, VK);
 
   // Just in case we're building an illegal pointer-to-member.
-  if (isa<FieldDecl>(D) && cast<FieldDecl>(D)->getBitWidth())
+  FieldDecl *FD = dyn_cast<FieldDecl>(D);
+  if (FD && FD->isBitField())
     E->setObjectKind(OK_BitField);
 
   return Owned(E);
@@ -1291,10 +1445,11 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
 /// This actually loses a lot of source location information for
 /// non-standard name kinds; we should consider preserving that in
 /// some way.
-void Sema::DecomposeUnqualifiedId(const UnqualifiedId &Id,
-                                 TemplateArgumentListInfo &Buffer,
-                                 DeclarationNameInfo &NameInfo,
-                              const TemplateArgumentListInfo *&TemplateArgs) {
+void
+Sema::DecomposeUnqualifiedId(const UnqualifiedId &Id,
+                             TemplateArgumentListInfo &Buffer,
+                             DeclarationNameInfo &NameInfo,
+                             const TemplateArgumentListInfo *&TemplateArgs) {
   if (Id.getKind() == UnqualifiedId::IK_TemplateId) {
     Buffer.setLAngleLoc(Id.TemplateId->LAngleLoc);
     Buffer.setRAngleLoc(Id.TemplateId->RAngleLoc);
@@ -1319,7 +1474,9 @@ void Sema::DecomposeUnqualifiedId(const UnqualifiedId &Id,
 ///
 /// \return false if new lookup candidates were found
 bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
-                               CorrectTypoContext CTC) {
+                               CorrectTypoContext CTC,
+                               TemplateArgumentListInfo *ExplicitTemplateArgs,
+                               Expr **Args, unsigned NumArgs) {
   DeclarationName Name = R.getLookupName();
 
   unsigned diagnostic = diag::err_undeclared_var_use;
@@ -1358,6 +1515,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
           CXXMethodDecl *DepMethod = cast_or_null<CXXMethodDecl>(
               CurMethod->getInstantiatedFromMemberFunction());
           if (DepMethod) {
+            if (getLangOptions().MicrosoftExt)
+              diagnostic = diag::warn_found_via_dependent_bases_lookup;
             Diag(R.getNameLoc(), diagnostic) << Name
               << FixItHint::CreateInsertion(R.getNameLoc(), "this->");
             QualType DepThisType = DepMethod->getThisType(Context);
@@ -1373,7 +1532,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                 CXXDependentScopeMemberExpr::Create(
                     Context, DepThis, DepThisType, true, SourceLocation(),
                     SS.getWithLocInContext(Context), NULL,
-                    R.getLookupNameInfo(), &TList);
+                    R.getLookupNameInfo(),
+                    ULE->hasExplicitTemplateArgs() ? &TList : 0);
             CallsUndergoingInstantiation.back()->setCallee(DepExpr);
           } else {
             // FIXME: we should be able to handle this case too. It is correct
@@ -1405,6 +1565,30 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
     R.setLookupName(Corrected.getCorrection());
 
     if (NamedDecl *ND = Corrected.getCorrectionDecl()) {
+      if (Corrected.isOverloaded()) {
+        OverloadCandidateSet OCS(R.getNameLoc());
+        OverloadCandidateSet::iterator Best;
+        for (TypoCorrection::decl_iterator CD = Corrected.begin(),
+                                        CDEnd = Corrected.end();
+             CD != CDEnd; ++CD) {
+          if (FunctionTemplateDecl *FTD =
+                   dyn_cast<FunctionTemplateDecl>(*CD))
+            AddTemplateOverloadCandidate(
+                FTD, DeclAccessPair::make(FTD, AS_none), ExplicitTemplateArgs,
+                Args, NumArgs, OCS);
+          else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(*CD))
+            if (!ExplicitTemplateArgs || ExplicitTemplateArgs->size() == 0)
+              AddOverloadCandidate(FD, DeclAccessPair::make(FD, AS_none),
+                                   Args, NumArgs, OCS);
+        }
+        switch (OCS.BestViableFunction(*this, R.getNameLoc(), Best)) {
+          case OR_Success:
+            ND = Best->Function;
+            break;
+          default:
+            break;
+        }
+      }
       R.addDecl(ND);
       if (isa<ValueDecl>(ND) || isa<FunctionTemplateDecl>(ND)) {
         if (SS.isEmpty())
@@ -1430,7 +1614,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
         // correction, but don't make it a fix-it since we're not going
         // to recover well anyway.
         if (SS.isEmpty())
-          Diag(R.getNameLoc(), diagnostic_suggest) << Name << CorrectedQuotedStr;
+          Diag(R.getNameLoc(), diagnostic_suggest)
+            << Name << CorrectedQuotedStr;
         else
           Diag(R.getNameLoc(), diag::err_no_member_suggest)
             << Name << computeDeclContext(SS, false) << CorrectedQuotedStr
@@ -1467,96 +1652,12 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   return true;
 }
 
-ObjCPropertyDecl *Sema::canSynthesizeProvisionalIvar(IdentifierInfo *II) {
-  ObjCMethodDecl *CurMeth = getCurMethodDecl();
-  ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
-  if (!IDecl)
-    return 0;
-  ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
-  if (!ClassImpDecl)
-    return 0;
-  ObjCPropertyDecl *property = LookupPropertyDecl(IDecl, II);
-  if (!property)
-    return 0;
-  if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II))
-    if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic ||
-        PIDecl->getPropertyIvarDecl())
-      return 0;
-  return property;
-}
-
-bool Sema::canSynthesizeProvisionalIvar(ObjCPropertyDecl *Property) {
-  ObjCMethodDecl *CurMeth = getCurMethodDecl();
-  ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
-  if (!IDecl)
-    return false;
-  ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
-  if (!ClassImpDecl)
-    return false;
-  if (ObjCPropertyImplDecl *PIDecl
-                = ClassImpDecl->FindPropertyImplDecl(Property->getIdentifier()))
-    if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic ||
-        PIDecl->getPropertyIvarDecl())
-      return false;
-  
-  return true;
-}
-
-ObjCIvarDecl *Sema::SynthesizeProvisionalIvar(LookupResult &Lookup,
-                                              IdentifierInfo *II,
-                                              SourceLocation NameLoc) {
-  ObjCMethodDecl *CurMeth = getCurMethodDecl();
-  bool LookForIvars;
-  if (Lookup.empty())
-    LookForIvars = true;
-  else if (CurMeth->isClassMethod())
-    LookForIvars = false;
-  else
-    LookForIvars = (Lookup.isSingleResult() &&
-                    Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod() &&
-                    (Lookup.getAsSingle<VarDecl>() != 0));
-  if (!LookForIvars)
-    return 0;
-  
-  ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
-  if (!IDecl)
-    return 0;
-  ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
-  if (!ClassImpDecl)
-    return 0;
-  bool DynamicImplSeen = false;
-  ObjCPropertyDecl *property = LookupPropertyDecl(IDecl, II);
-  if (!property)
-    return 0;
-  if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II)) {
-    DynamicImplSeen = 
-      (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic);
-    // property implementation has a designated ivar. No need to assume a new
-    // one.
-    if (!DynamicImplSeen && PIDecl->getPropertyIvarDecl())
-      return 0;
-  }
-  if (!DynamicImplSeen) {
-    QualType PropType = Context.getCanonicalType(property->getType());
-    ObjCIvarDecl *Ivar = ObjCIvarDecl::Create(Context, ClassImpDecl, 
-                                              NameLoc, NameLoc,
-                                              II, PropType, /*Dinfo=*/0,
-                                              ObjCIvarDecl::Private,
-                                              (Expr *)0, true);
-    ClassImpDecl->addDecl(Ivar);
-    IDecl->makeDeclVisibleInContext(Ivar, false);
-    property->setPropertyIvarDecl(Ivar);
-    return Ivar;
-  }
-  return 0;
-}
-
 ExprResult Sema::ActOnIdExpression(Scope *S,
                                    CXXScopeSpec &SS,
                                    UnqualifiedId &Id,
                                    bool HasTrailingLParen,
-                                   bool isAddressOfOperand) {
-  assert(!(isAddressOfOperand && HasTrailingLParen) &&
+                                   bool IsAddressOfOperand) {
+  assert(!(IsAddressOfOperand && HasTrailingLParen) &&
          "cannot be direct & operand and have a trailing lparen");
 
   if (SS.isInvalid())
@@ -1598,7 +1699,7 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
   }
 
   if (DependentID)
-    return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
+    return ActOnDependentIdExpression(SS, NameInfo, IsAddressOfOperand,
                                       TemplateArgs);
 
   bool IvarLookupFollowUp = false;
@@ -1618,7 +1719,7 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
     
     if (MemberOfUnknownSpecialization ||
         (R.getResultKind() == LookupResult::NotFoundInCurrentInstantiation))
-      return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
+      return ActOnDependentIdExpression(SS, NameInfo, IsAddressOfOperand,
                                         TemplateArgs);
   } else {
     IvarLookupFollowUp = (!SS.isSet() && II && getCurMethodDecl());
@@ -1627,7 +1728,7 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
     // If the result might be in a dependent base class, this is a dependent 
     // id-expression.
     if (R.getResultKind() == LookupResult::NotFoundInCurrentInstantiation)
-      return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
+      return ActOnDependentIdExpression(SS, NameInfo, IsAddressOfOperand,
                                         TemplateArgs);
       
     // If this reference is in an Objective-C method, then we need to do
@@ -1640,19 +1741,6 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
       if (Expr *Ex = E.takeAs<Expr>())
         return Owned(Ex);
       
-      // Synthesize ivars lazily.
-      if (getLangOptions().ObjCDefaultSynthProperties &&
-          getLangOptions().ObjCNonFragileABI2) {
-        if (SynthesizeProvisionalIvar(R, II, NameLoc)) {
-          if (const ObjCPropertyDecl *Property = 
-                canSynthesizeProvisionalIvar(II)) {
-            Diag(NameLoc, diag::warn_synthesized_ivar_access) << II;
-            Diag(Property->getLocation(), diag::note_property_declare);
-          }
-          return ActOnIdExpression(S, SS, Id, HasTrailingLParen,
-                                   isAddressOfOperand);
-        }
-      }
       // for further use, this must be set to false if in class method.
       IvarLookupFollowUp = getCurMethodDecl()->isInstanceMethod();
     }
@@ -1676,6 +1764,16 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
     // If this name wasn't predeclared and if this is not a function
     // call, diagnose the problem.
     if (R.empty()) {
+
+      // In Microsoft mode, if we are inside a template class member function
+      // and we can't resolve an identifier then assume the identifier is type
+      // dependent. The goal is to postpone name lookup to instantiation time 
+      // to be able to search into type dependent base classes.
+      if (getLangOptions().MicrosoftMode && CurContext->isDependentContext() &&
+          isa<CXXMethodDecl>(CurContext))
+        return ActOnDependentIdExpression(SS, NameInfo, IsAddressOfOperand,
+                                          TemplateArgs);
+
       if (DiagnoseEmptyLookup(S, SS, R, CTC_Unknown))
         return ExprError();
 
@@ -1688,7 +1786,10 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
       if (ObjCIvarDecl *Ivar = R.getAsSingle<ObjCIvarDecl>()) {
         R.clear();
         ExprResult E(LookupInObjCMethod(R, S, Ivar->getIdentifier()));
-        assert(E.isInvalid() || E.get());
+        // In a hopelessly buggy code, Objective-C instance variable
+        // lookup fails and no expression will be built to reference it.
+        if (!E.isInvalid() && !E.get())
+          return ExprError();
         return move(E);
       }
     }
@@ -1723,7 +1824,7 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
   // instance method.
   if (!R.empty() && (*R.begin())->isCXXClassMember()) {
     bool MightBeImplicitMember;
-    if (!isAddressOfOperand)
+    if (!IsAddressOfOperand)
       MightBeImplicitMember = true;
     else if (!SS.isEmpty())
       MightBeImplicitMember = false;
@@ -1950,7 +2051,7 @@ Sema::PerformObjectMemberConversion(Expr *From,
   SourceRange FromRange = From->getSourceRange();
   SourceLocation FromLoc = FromRange.getBegin();
 
-  ExprValueKind VK = CastCategory(From);
+  ExprValueKind VK = From->getValueKind();
 
   // C++ [class.member.lookup]p8:
   //   [...] Ambiguities can often be resolved by qualifying a name with its
@@ -2341,7 +2442,8 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
       // If we're referring to a method with an __unknown_anytype
       // result type, make the entire expression __unknown_anytype.
       // This should only be possible with a type written directly.
-      if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(VD->getType()))
+      if (const FunctionProtoType *proto
+            = dyn_cast<FunctionProtoType>(VD->getType()))
         if (proto->getResultType() == Context.UnknownAnyTy) {
           type = Context.UnknownAnyTy;
           valueKind = VK_RValue;
@@ -2375,7 +2477,7 @@ ExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc, tok::TokenKind Kind) {
   PredefinedExpr::IdentType IT;
 
   switch (Kind) {
-  default: assert(0 && "Unknown simple primary expr!");
+  default: llvm_unreachable("Unknown simple primary expr!");
   case tok::kw___func__: IT = PredefinedExpr::Func; break; // [C99 6.4.2.2]
   case tok::kw___FUNCTION__: IT = PredefinedExpr::Function; break;
   case tok::kw___PRETTY_FUNCTION__: IT = PredefinedExpr::PrettyFunction; break;
@@ -2408,12 +2510,12 @@ ExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc, tok::TokenKind Kind) {
 ExprResult Sema::ActOnCharacterConstant(const Token &Tok) {
   llvm::SmallString<16> CharBuffer;
   bool Invalid = false;
-  llvm::StringRef ThisTok = PP.getSpelling(Tok, CharBuffer, &Invalid);
+  StringRef ThisTok = PP.getSpelling(Tok, CharBuffer, &Invalid);
   if (Invalid)
     return ExprError();
 
   CharLiteralParser Literal(ThisTok.begin(), ThisTok.end(), Tok.getLocation(),
-                            PP);
+                            PP, Tok.getKind());
   if (Literal.hadError())
     return ExprError();
 
@@ -2422,14 +2524,25 @@ ExprResult Sema::ActOnCharacterConstant(const Token &Tok) {
     Ty = Context.IntTy;   // 'x' and L'x' -> int in C.
   else if (Literal.isWide())
     Ty = Context.WCharTy; // L'x' -> wchar_t in C++.
+  else if (Literal.isUTF16())
+    Ty = Context.Char16Ty; // u'x' -> char16_t in C++0x.
+  else if (Literal.isUTF32())
+    Ty = Context.Char32Ty; // U'x' -> char32_t in C++0x.
   else if (Literal.isMultiChar())
     Ty = Context.IntTy;   // 'wxyz' -> int in C++.
   else
     Ty = Context.CharTy;  // 'x' -> char in C++
 
-  return Owned(new (Context) CharacterLiteral(Literal.getValue(),
-                                              Literal.isWide(),
-                                              Ty, Tok.getLocation()));
+  CharacterLiteral::CharacterKind Kind = CharacterLiteral::Ascii;
+  if (Literal.isWide())
+    Kind = CharacterLiteral::Wide;
+  else if (Literal.isUTF16())
+    Kind = CharacterLiteral::UTF16;
+  else if (Literal.isUTF32())
+    Kind = CharacterLiteral::UTF32;
+
+  return Owned(new (Context) CharacterLiteral(Literal.getValue(), Kind, Ty,
+                                              Tok.getLocation()));
 }
 
 ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
@@ -2437,7 +2550,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
   // cannot have a trigraph, escaped newline, radix prefix, or type suffix.
   if (Tok.getLength() == 1) {
     const char Val = PP.getSpellingOfSingleCharacterNumericConstant(Tok);
-    unsigned IntSize = Context.Target.getIntWidth();
+    unsigned IntSize = Context.getTargetInfo().getIntWidth();
     return Owned(IntegerLiteral::Create(Context, llvm::APInt(IntSize, Val-'0'),
                     Context.IntTy, Tok.getLocation()));
   }
@@ -2492,7 +2605,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
 
       Diag(Tok.getLocation(), diagnostic)
         << Ty
-        << llvm::StringRef(buffer.data(), buffer.size());
+        << StringRef(buffer.data(), buffer.size());
     }
 
     bool isExact = (result == APFloat::opOK);
@@ -2517,7 +2630,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
       Diag(Tok.getLocation(), diag::ext_longlong);
 
     // Get the value in the widest-possible width.
-    llvm::APInt ResultVal(Context.Target.getIntMaxTWidth(), 0);
+    llvm::APInt ResultVal(Context.getTargetInfo().getIntMaxTWidth(), 0);
 
     if (Literal.GetIntegerValue(ResultVal)) {
       // If this value didn't fit into uintmax_t, warn and force to ull.
@@ -2537,7 +2650,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
       unsigned Width = 0;
       if (!Literal.isLong && !Literal.isLongLong) {
         // Are int/unsigned possibilities?
-        unsigned IntSize = Context.Target.getIntWidth();
+        unsigned IntSize = Context.getTargetInfo().getIntWidth();
 
         // Does it fit in a unsigned int?
         if (ResultVal.isIntN(IntSize)) {
@@ -2552,7 +2665,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
 
       // Are long/unsigned long possibilities?
       if (Ty.isNull() && !Literal.isLongLong) {
-        unsigned LongSize = Context.Target.getLongWidth();
+        unsigned LongSize = Context.getTargetInfo().getLongWidth();
 
         // Does it fit in a unsigned long?
         if (ResultVal.isIntN(LongSize)) {
@@ -2567,7 +2680,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
 
       // Finally, check long long if needed.
       if (Ty.isNull()) {
-        unsigned LongLongSize = Context.Target.getLongLongWidth();
+        unsigned LongLongSize = Context.getTargetInfo().getLongLongWidth();
 
         // Does it fit in a unsigned long long?
         if (ResultVal.isIntN(LongLongSize)) {
@@ -2575,7 +2688,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
           // To be compatible with MSVC, hex integer literals ending with the
           // LL or i64 suffix are always signed in Microsoft mode.
           if (!Literal.isUnsigned && (ResultVal[LongLongSize-1] == 0 ||
-              (getLangOptions().Microsoft && Literal.isLongLong)))
+              (getLangOptions().MicrosoftExt && Literal.isLongLong)))
             Ty = Context.LongLongTy;
           else if (AllowUnsigned)
             Ty = Context.UnsignedLongLongTy;
@@ -2588,7 +2701,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
       if (Ty.isNull()) {
         Diag(Tok.getLocation(), diag::warn_integer_too_large_for_signed);
         Ty = Context.UnsignedLongLongTy;
-        Width = Context.Target.getLongLongWidth();
+        Width = Context.getTargetInfo().getLongLongWidth();
       }
 
       if (ResultVal.getBitWidth() != Width)
@@ -2605,8 +2718,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
   return Owned(Res);
 }
 
-ExprResult Sema::ActOnParenExpr(SourceLocation L,
-                                              SourceLocation R, Expr *E) {
+ExprResult Sema::ActOnParenExpr(SourceLocation L, SourceLocation R, Expr *E) {
   assert((E != 0) && "ActOnParenExpr() missing expr");
   return Owned(new (Context) ParenExpr(L, R, E));
 }
@@ -2672,9 +2784,9 @@ static bool CheckObjCTraitOperandConstraints(Sema &S, QualType T,
 /// expression. The logic mostly mirrors the type-based overload, but may modify
 /// the expression as it completes the type for that expression through template
 /// instantiation, etc.
-bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *Op,
+bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *E,
                                             UnaryExprOrTypeTrait ExprKind) {
-  QualType ExprTy = Op->getType();
+  QualType ExprTy = E->getType();
 
   // C++ [expr.sizeof]p2: "When applied to a reference or a reference type,
   //   the result is the size of the referenced type."
@@ -2684,36 +2796,36 @@ bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *Op,
     ExprTy = Ref->getPointeeType();
 
   if (ExprKind == UETT_VecStep)
-    return CheckVecStepTraitOperandType(*this, ExprTy, Op->getExprLoc(),
-                                        Op->getSourceRange());
+    return CheckVecStepTraitOperandType(*this, ExprTy, E->getExprLoc(),
+                                        E->getSourceRange());
 
   // Whitelist some types as extensions
-  if (!CheckExtensionTraitOperandType(*this, ExprTy, Op->getExprLoc(),
-                                      Op->getSourceRange(), ExprKind))
+  if (!CheckExtensionTraitOperandType(*this, ExprTy, E->getExprLoc(),
+                                      E->getSourceRange(), ExprKind))
     return false;
 
-  if (RequireCompleteExprType(Op,
+  if (RequireCompleteExprType(E,
                               PDiag(diag::err_sizeof_alignof_incomplete_type)
-                              << ExprKind << Op->getSourceRange(),
+                              << ExprKind << E->getSourceRange(),
                               std::make_pair(SourceLocation(), PDiag(0))))
     return true;
 
   // Completeing the expression's type may have changed it.
-  ExprTy = Op->getType();
+  ExprTy = E->getType();
   if (const ReferenceType *Ref = ExprTy->getAs<ReferenceType>())
     ExprTy = Ref->getPointeeType();
 
-  if (CheckObjCTraitOperandConstraints(*this, ExprTy, Op->getExprLoc(),
-                                       Op->getSourceRange(), ExprKind))
+  if (CheckObjCTraitOperandConstraints(*this, ExprTy, E->getExprLoc(),
+                                       E->getSourceRange(), ExprKind))
     return true;
 
   if (ExprKind == UETT_SizeOf) {
-    if (DeclRefExpr *DeclRef = dyn_cast<DeclRefExpr>(Op->IgnoreParens())) {
+    if (DeclRefExpr *DeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParens())) {
       if (ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(DeclRef->getFoundDecl())) {
         QualType OType = PVD->getOriginalType();
         QualType Type = PVD->getType();
         if (Type->isPointerType() && OType->isArrayType()) {
-          Diag(Op->getExprLoc(), diag::warn_sizeof_array_param)
+          Diag(E->getExprLoc(), diag::warn_sizeof_array_param)
             << Type << OType;
           Diag(PVD->getLocation(), diag::note_declared_at);
         }
@@ -2739,34 +2851,34 @@ bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *Op,
 ///   standard conversions are not applied to the operand of sizeof.
 ///
 /// This policy is followed for all of the unary trait expressions.
-bool Sema::CheckUnaryExprOrTypeTraitOperand(QualType exprType,
+bool Sema::CheckUnaryExprOrTypeTraitOperand(QualType ExprType,
                                             SourceLocation OpLoc,
                                             SourceRange ExprRange,
                                             UnaryExprOrTypeTrait ExprKind) {
-  if (exprType->isDependentType())
+  if (ExprType->isDependentType())
     return false;
 
   // C++ [expr.sizeof]p2: "When applied to a reference or a reference type,
   //   the result is the size of the referenced type."
   // C++ [expr.alignof]p3: "When alignof is applied to a reference type, the
   //   result shall be the alignment of the referenced type."
-  if (const ReferenceType *Ref = exprType->getAs<ReferenceType>())
-    exprType = Ref->getPointeeType();
+  if (const ReferenceType *Ref = ExprType->getAs<ReferenceType>())
+    ExprType = Ref->getPointeeType();
 
   if (ExprKind == UETT_VecStep)
-    return CheckVecStepTraitOperandType(*this, exprType, OpLoc, ExprRange);
+    return CheckVecStepTraitOperandType(*this, ExprType, OpLoc, ExprRange);
 
   // Whitelist some types as extensions
-  if (!CheckExtensionTraitOperandType(*this, exprType, OpLoc, ExprRange,
+  if (!CheckExtensionTraitOperandType(*this, ExprType, OpLoc, ExprRange,
                                       ExprKind))
     return false;
 
-  if (RequireCompleteType(OpLoc, exprType,
+  if (RequireCompleteType(OpLoc, ExprType,
                           PDiag(diag::err_sizeof_alignof_incomplete_type)
                           << ExprKind << ExprRange))
     return true;
 
-  if (CheckObjCTraitOperandConstraints(*this, exprType, OpLoc, ExprRange,
+  if (CheckObjCTraitOperandConstraints(*this, ExprType, OpLoc, ExprRange,
                                        ExprKind))
     return true;
 
@@ -2870,12 +2982,12 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc,
 /// Note that the ArgRange is invalid if isType is false.
 ExprResult
 Sema::ActOnUnaryExprOrTypeTraitExpr(SourceLocation OpLoc,
-                                    UnaryExprOrTypeTrait ExprKind, bool isType,
+                                    UnaryExprOrTypeTrait ExprKind, bool IsType,
                                     void *TyOrEx, const SourceRange &ArgRange) {
   // If error parsing type, ignore.
   if (TyOrEx == 0) return ExprError();
 
-  if (isType) {
+  if (IsType) {
     TypeSourceInfo *TInfo;
     (void) GetTypeFromParser(ParsedType::getFromOpaquePtr(TyOrEx), &TInfo);
     return CreateUnaryExprOrTypeTraitExpr(TInfo, OpLoc, ExprKind, ArgRange);
@@ -2887,7 +2999,7 @@ Sema::ActOnUnaryExprOrTypeTraitExpr(SourceLocation OpLoc,
 }
 
 static QualType CheckRealImagOperand(Sema &S, ExprResult &V, SourceLocation Loc,
-                                     bool isReal) {
+                                     bool IsReal) {
   if (V.get()->isTypeDependent())
     return S.Context.DependentTy;
 
@@ -2911,12 +3023,12 @@ static QualType CheckRealImagOperand(Sema &S, ExprResult &V, SourceLocation Loc,
   if (PR.isInvalid()) return QualType();
   if (PR.get() != V.get()) {
     V = move(PR);
-    return CheckRealImagOperand(S, V, Loc, isReal);
+    return CheckRealImagOperand(S, V, Loc, IsReal);
   }
 
   // Reject anything else.
   S.Diag(Loc, diag::err_realimag_invalid_type) << V.get()->getType()
-    << (isReal ? "__real" : "__imag");
+    << (IsReal ? "__real" : "__imag");
   return QualType();
 }
 
@@ -2927,7 +3039,7 @@ Sema::ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc,
                           tok::TokenKind Kind, Expr *Input) {
   UnaryOperatorKind Opc;
   switch (Kind) {
-  default: assert(0 && "Unknown unary op!");
+  default: llvm_unreachable("Unknown unary op!");
   case tok::plusplus:   Opc = UO_PostInc; break;
   case tok::minusminus: Opc = UO_PostDec; break;
   }
@@ -2967,7 +3079,7 @@ Sema::ActOnArraySubscriptExpr(Scope *S, Expr *Base, SourceLocation LLoc,
 
 ExprResult
 Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
-                                     Expr *Idx, SourceLocation RLoc) {
+                                      Expr *Idx, SourceLocation RLoc) {
   Expr *LHSExp = Base;
   Expr *RHSExp = Idx;
 
@@ -3190,7 +3302,8 @@ Sema::ConvertArgumentsForCall(CallExpr *Call, Expr *Fn,
                               FunctionDecl *FDecl,
                               const FunctionProtoType *Proto,
                               Expr **Args, unsigned NumArgs,
-                              SourceLocation RParenLoc) {
+                              SourceLocation RParenLoc,
+                              bool IsExecConfig) {
   // Bail out early if calling a builtin with custom typechecking.
   // We don't need to do this in the 
   if (FDecl)
@@ -3202,14 +3315,29 @@ Sema::ConvertArgumentsForCall(CallExpr *Call, Expr *Fn,
   // assignment, to the types of the corresponding parameter, ...
   unsigned NumArgsInProto = Proto->getNumArgs();
   bool Invalid = false;
+  unsigned MinArgs = FDecl ? FDecl->getMinRequiredArguments() : NumArgsInProto;
+  unsigned FnKind = Fn->getType()->isBlockPointerType()
+                       ? 1 /* block */
+                       : (IsExecConfig ? 3 /* kernel function (exec config) */
+                                       : 0 /* function */);
 
   // If too few arguments are available (and we don't have default
   // arguments for the remaining parameters), don't make the call.
   if (NumArgs < NumArgsInProto) {
-    if (!FDecl || NumArgs < FDecl->getMinRequiredArguments())
-      return Diag(RParenLoc, diag::err_typecheck_call_too_few_args)
-        << Fn->getType()->isBlockPointerType()
-        << NumArgsInProto << NumArgs << Fn->getSourceRange();
+    if (NumArgs < MinArgs) {
+      Diag(RParenLoc, MinArgs == NumArgsInProto
+                        ? diag::err_typecheck_call_too_few_args
+                        : diag::err_typecheck_call_too_few_args_at_least)
+        << FnKind
+        << MinArgs << NumArgs << Fn->getSourceRange();
+
+      // Emit the location of the prototype.
+      if (FDecl && !FDecl->getBuiltinID() && !IsExecConfig)
+        Diag(FDecl->getLocStart(), diag::note_callee_decl)
+          << FDecl;
+
+      return true;
+    }
     Call->setNumArgs(Context, NumArgsInProto);
   }
 
@@ -3218,24 +3346,25 @@ Sema::ConvertArgumentsForCall(CallExpr *Call, Expr *Fn,
   if (NumArgs > NumArgsInProto) {
     if (!Proto->isVariadic()) {
       Diag(Args[NumArgsInProto]->getLocStart(),
-           diag::err_typecheck_call_too_many_args)
-        << Fn->getType()->isBlockPointerType()
+           MinArgs == NumArgsInProto
+             ? diag::err_typecheck_call_too_many_args
+             : diag::err_typecheck_call_too_many_args_at_most)
+        << FnKind
         << NumArgsInProto << NumArgs << Fn->getSourceRange()
         << SourceRange(Args[NumArgsInProto]->getLocStart(),
                        Args[NumArgs-1]->getLocEnd());
 
       // Emit the location of the prototype.
-      if (FDecl && !FDecl->getBuiltinID())
-        Diag(FDecl->getLocStart(),
-             diag::note_typecheck_call_too_many_args)
-             << FDecl;
+      if (FDecl && !FDecl->getBuiltinID() && !IsExecConfig)
+        Diag(FDecl->getLocStart(), diag::note_callee_decl)
+          << FDecl;
       
       // This deletes the extra arguments.
       Call->setNumArgs(Context, NumArgsInProto);
       return true;
     }
   }
-  llvm::SmallVector<Expr *, 8> AllArgs;
+  SmallVector<Expr *, 8> AllArgs;
   VariadicCallType CallType =
     Proto->isVariadic() ? VariadicFunction : VariadicDoesNotApply;
   if (Fn->getType()->isBlockPointerType())
@@ -3258,7 +3387,7 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
                                   const FunctionProtoType *Proto,
                                   unsigned FirstProtoArg,
                                   Expr **Args, unsigned NumArgs,
-                                  llvm::SmallVector<Expr *, 8> &AllArgs,
+                                  SmallVector<Expr *, 8> &AllArgs,
                                   VariadicCallType CallType) {
   unsigned NumArgsInProto = Proto->getNumArgs();
   unsigned NumArgsToCheck = NumArgs;
@@ -3307,6 +3436,12 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
 
       Arg = ArgExpr.takeAs<Expr>();
     }
+
+    // Check for array bounds violations for each argument to the call. This
+    // check only triggers warnings when the argument isn't a more complex Expr
+    // with its own checking, such as a BinaryOperator.
+    CheckArrayAccess(Arg);
+
     AllArgs.push_back(Arg);
   }
 
@@ -3330,11 +3465,16 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
     // Otherwise do argument promotion, (C99 6.5.2.2p7).
     } else {
       for (unsigned i = ArgIx; i != NumArgs; ++i) {
-        ExprResult Arg = DefaultVariadicArgumentPromotion(Args[i], CallType, FDecl);
+        ExprResult Arg = DefaultVariadicArgumentPromotion(Args[i], CallType,
+                                                          FDecl);
         Invalid |= Arg.isInvalid();
         AllArgs.push_back(Arg.take());
       }
     }
+
+    // Check for array bounds violations.
+    for (unsigned i = ArgIx; i != NumArgs; ++i)
+      CheckArrayAccess(Args[i]);
   }
   return Invalid;
 }
@@ -3348,16 +3488,16 @@ static ExprResult rebuildUnknownAnyFunction(Sema &S, Expr *fn);
 /// locations.
 ExprResult
 Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
-                    MultiExprArg args, SourceLocation RParenLoc,
-                    Expr *ExecConfig) {
-  unsigned NumArgs = args.size();
+                    MultiExprArg ArgExprs, SourceLocation RParenLoc,
+                    Expr *ExecConfig, bool IsExecConfig) {
+  unsigned NumArgs = ArgExprs.size();
 
   // Since this might be a postfix expression, get rid of ParenListExprs.
   ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Fn);
   if (Result.isInvalid()) return ExprError();
   Fn = Result.take();
 
-  Expr **Args = args.release();
+  Expr **Args = ArgExprs.release();
 
   if (getLangOptions().CPlusPlus) {
     // If this is a pseudo-destructor expression, build the call immediately.
@@ -3419,8 +3559,8 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
   if (Fn->getType() == Context.OverloadTy) {
     OverloadExpr::FindResult find = OverloadExpr::find(Fn);
 
-    // We aren't supposed to apply this logic if there's an '&' involved.
-    if (!find.IsAddressOfOperand) {
+    // We aren't supposed to apply this logic for if there's an '&' involved.
+    if (!find.HasFormOfMemberPointer) {
       OverloadExpr *ovl = find.Expression;
       if (isa<UnresolvedLookupExpr>(ovl)) {
         UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(ovl);
@@ -3448,12 +3588,12 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
     NDecl = cast<MemberExpr>(NakedFn)->getMemberDecl();
 
   return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, Args, NumArgs, RParenLoc,
-                               ExecConfig);
+                               ExecConfig, IsExecConfig);
 }
 
 ExprResult
 Sema::ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
-                              MultiExprArg execConfig, SourceLocation GGGLoc) {
+                              MultiExprArg ExecConfig, SourceLocation GGGLoc) {
   FunctionDecl *ConfigDecl = Context.getcudaConfigureCallDecl();
   if (!ConfigDecl)
     return ExprError(Diag(LLLLoc, diag::err_undeclared_var_use)
@@ -3463,27 +3603,29 @@ Sema::ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
   DeclRefExpr *ConfigDR = new (Context) DeclRefExpr(
       ConfigDecl, ConfigQTy, VK_LValue, LLLLoc);
 
-  return ActOnCallExpr(S, ConfigDR, LLLLoc, execConfig, GGGLoc, 0);
+  return ActOnCallExpr(S, ConfigDR, LLLLoc, ExecConfig, GGGLoc, 0,
+                       /*IsExecConfig=*/true);
 }
 
 /// ActOnAsTypeExpr - create a new asType (bitcast) from the arguments.
 ///
 /// __builtin_astype( value, dst type )
 ///
-ExprResult Sema::ActOnAsTypeExpr(Expr *expr, ParsedType destty,
+ExprResult Sema::ActOnAsTypeExpr(Expr *E, ParsedType ParsedDestTy,
                                  SourceLocation BuiltinLoc,
                                  SourceLocation RParenLoc) {
   ExprValueKind VK = VK_RValue;
   ExprObjectKind OK = OK_Ordinary;
-  QualType DstTy = GetTypeFromParser(destty);
-  QualType SrcTy = expr->getType();
+  QualType DstTy = GetTypeFromParser(ParsedDestTy);
+  QualType SrcTy = E->getType();
   if (Context.getTypeSize(DstTy) != Context.getTypeSize(SrcTy))
     return ExprError(Diag(BuiltinLoc,
                           diag::err_invalid_astype_of_different_size)
                      << DstTy
                      << SrcTy
-                     << expr->getSourceRange());
-  return Owned(new (Context) AsTypeExpr(expr, DstTy, VK, OK, BuiltinLoc, RParenLoc));
+                     << E->getSourceRange());
+  return Owned(new (Context) AsTypeExpr(E, DstTy, VK, OK, BuiltinLoc,
+               RParenLoc));
 }
 
 /// BuildResolvedCallExpr - Build a call to a resolved expression,
@@ -3497,7 +3639,7 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
                             SourceLocation LParenLoc,
                             Expr **Args, unsigned NumArgs,
                             SourceLocation RParenLoc,
-                            Expr *Config) {
+                            Expr *Config, bool IsExecConfig) {
   FunctionDecl *FDecl = dyn_cast_or_null<FunctionDecl>(NDecl);
 
   // Promote the function operand.
@@ -3567,6 +3709,11 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
       if (!FuncT->getResultType()->isVoidType())
         return ExprError(Diag(LParenLoc, diag::err_kern_type_not_void_return)
             << Fn->getType() << Fn->getSourceRange());
+    } else {
+      // CUDA: Calls to global functions must be configured
+      if (FDecl && FDecl->hasAttr<CUDAGlobalAttr>())
+        return ExprError(Diag(LParenLoc, diag::err_global_call_not_config)
+            << FDecl->getName() << Fn->getSourceRange());
     }
   }
 
@@ -3582,7 +3729,7 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
 
   if (const FunctionProtoType *Proto = dyn_cast<FunctionProtoType>(FuncT)) {
     if (ConvertArgumentsForCall(TheCall, Fn, FDecl, Proto, Args, NumArgs,
-                                RParenLoc))
+                                RParenLoc, IsExecConfig))
       return ExprError();
   } else {
     assert(isa<FunctionNoProtoType>(FuncT) && "Unknown FunctionType!");
@@ -3682,23 +3829,23 @@ Sema::ActOnCompoundLiteral(SourceLocation LParenLoc, ParsedType Ty,
 
 ExprResult
 Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
-                               SourceLocation RParenLoc, Expr *literalExpr) {
+                               SourceLocation RParenLoc, Expr *LiteralExpr) {
   QualType literalType = TInfo->getType();
 
   if (literalType->isArrayType()) {
     if (RequireCompleteType(LParenLoc, Context.getBaseElementType(literalType),
              PDiag(diag::err_illegal_decl_array_incomplete_type)
                << SourceRange(LParenLoc,
-                              literalExpr->getSourceRange().getEnd())))
+                              LiteralExpr->getSourceRange().getEnd())))
       return ExprError();
     if (literalType->isVariableArrayType())
       return ExprError(Diag(LParenLoc, diag::err_variable_object_no_init)
-        << SourceRange(LParenLoc, literalExpr->getSourceRange().getEnd()));
+        << SourceRange(LParenLoc, LiteralExpr->getSourceRange().getEnd()));
   } else if (!literalType->isDependentType() &&
              RequireCompleteType(LParenLoc, literalType,
                       PDiag(diag::err_typecheck_decl_incomplete_type)
                         << SourceRange(LParenLoc,
-                                       literalExpr->getSourceRange().getEnd())))
+                                       LiteralExpr->getSourceRange().getEnd())))
     return ExprError();
 
   InitializedEntity Entity
@@ -3706,17 +3853,17 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
   InitializationKind Kind
     = InitializationKind::CreateCStyleCast(LParenLoc, 
                                            SourceRange(LParenLoc, RParenLoc));
-  InitializationSequence InitSeq(*this, Entity, Kind, &literalExpr, 1);
+  InitializationSequence InitSeq(*this, Entity, Kind, &LiteralExpr, 1);
   ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                       MultiExprArg(*this, &literalExpr, 1),
+                                       MultiExprArg(*this, &LiteralExpr, 1),
                                             &literalType);
   if (Result.isInvalid())
     return ExprError();
-  literalExpr = Result.get();
+  LiteralExpr = Result.get();
 
   bool isFileScope = getCurFunctionOrMethodDecl() == 0;
   if (isFileScope) { // 6.5.2.5p3
-    if (CheckForConstantInitializer(literalExpr, literalType))
+    if (CheckForConstantInitializer(LiteralExpr, literalType))
       return ExprError();
   }
 
@@ -3725,14 +3872,14 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
 
   return MaybeBindToTemporary(
            new (Context) CompoundLiteralExpr(LParenLoc, TInfo, literalType,
-                                             VK, literalExpr, isFileScope));
+                                             VK, LiteralExpr, isFileScope));
 }
 
 ExprResult
-Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg initlist,
+Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg InitArgList,
                     SourceLocation RBraceLoc) {
-  unsigned NumInit = initlist.size();
-  Expr **InitList = initlist.release();
+  unsigned NumInit = InitArgList.size();
+  Expr **InitList = InitArgList.release();
 
   // Semantic analysis for initializers is done by ActOnDeclarator() and
   // CheckInitializer() - it requires knowledge of the object being intialized.
@@ -3743,27 +3890,68 @@ Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg initlist,
   return Owned(E);
 }
 
+/// Do an explicit extend of the given block pointer if we're in ARC.
+static void maybeExtendBlockObject(Sema &S, ExprResult &E) {
+  assert(E.get()->getType()->isBlockPointerType());
+  assert(E.get()->isRValue());
+
+  // Only do this in an r-value context.
+  if (!S.getLangOptions().ObjCAutoRefCount) return;
+
+  E = ImplicitCastExpr::Create(S.Context, E.get()->getType(),
+                               CK_ARCExtendBlockObject, E.get(),
+                               /*base path*/ 0, VK_RValue);
+  S.ExprNeedsCleanups = true;
+}
+
+/// Prepare a conversion of the given expression to an ObjC object
+/// pointer type.
+CastKind Sema::PrepareCastToObjCObjectPointer(ExprResult &E) {
+  QualType type = E.get()->getType();
+  if (type->isObjCObjectPointerType()) {
+    return CK_BitCast;
+  } else if (type->isBlockPointerType()) {
+    maybeExtendBlockObject(*this, E);
+    return CK_BlockPointerToObjCPointerCast;
+  } else {
+    assert(type->isPointerType());
+    return CK_CPointerToObjCPointerCast;
+  }
+}
+
 /// Prepares for a scalar cast, performing all the necessary stages
 /// except the final cast and returning the kind required.
-static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
+CastKind Sema::PrepareScalarCast(ExprResult &Src, QualType DestTy) {
   // Both Src and Dest are scalar types, i.e. arithmetic or pointer.
   // Also, callers should have filtered out the invalid cases with
   // pointers.  Everything else should be possible.
 
   QualType SrcTy = Src.get()->getType();
-  if (S.Context.hasSameUnqualifiedType(SrcTy, DestTy))
+  if (Context.hasSameUnqualifiedType(SrcTy, DestTy))
     return CK_NoOp;
 
-  switch (SrcTy->getScalarTypeKind()) {
+  switch (Type::ScalarTypeKind SrcKind = SrcTy->getScalarTypeKind()) {
   case Type::STK_MemberPointer:
     llvm_unreachable("member pointer type in C");
 
-  case Type::STK_Pointer:
+  case Type::STK_CPointer:
+  case Type::STK_BlockPointer:
+  case Type::STK_ObjCObjectPointer:
     switch (DestTy->getScalarTypeKind()) {
-    case Type::STK_Pointer:
-      return DestTy->isObjCObjectPointerType() ?
-                CK_AnyPointerToObjCPointerCast :
-                CK_BitCast;
+    case Type::STK_CPointer:
+      return CK_BitCast;
+    case Type::STK_BlockPointer:
+      return (SrcKind == Type::STK_BlockPointer
+                ? CK_BitCast : CK_AnyPointerToBlockPointerCast);
+    case Type::STK_ObjCObjectPointer:
+      if (SrcKind == Type::STK_ObjCObjectPointer)
+        return CK_BitCast;
+      else if (SrcKind == Type::STK_CPointer)
+        return CK_CPointerToObjCPointerCast;
+      else {
+        maybeExtendBlockObject(*this, Src);
+        return CK_BlockPointerToObjCPointerCast;
+      }
     case Type::STK_Bool:
       return CK_PointerToBoolean;
     case Type::STK_Integral:
@@ -3779,8 +3967,11 @@ static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
   case Type::STK_Bool: // casting from bool is like casting from an integer
   case Type::STK_Integral:
     switch (DestTy->getScalarTypeKind()) {
-    case Type::STK_Pointer:
-      if (Src.get()->isNullPointerConstant(S.Context, Expr::NPC_ValueDependentIsNull))
+    case Type::STK_CPointer:
+    case Type::STK_ObjCObjectPointer:
+    case Type::STK_BlockPointer:
+      if (Src.get()->isNullPointerConstant(Context,
+                                           Expr::NPC_ValueDependentIsNull))
         return CK_NullToPointer;
       return CK_IntegralToPointer;
     case Type::STK_Bool:
@@ -3790,12 +3981,14 @@ static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
     case Type::STK_Floating:
       return CK_IntegralToFloating;
     case Type::STK_IntegralComplex:
-      Src = S.ImpCastExprToType(Src.take(), DestTy->getAs<ComplexType>()->getElementType(),
-                                CK_IntegralCast);
+      Src = ImpCastExprToType(Src.take(),
+                              DestTy->castAs<ComplexType>()->getElementType(),
+                              CK_IntegralCast);
       return CK_IntegralRealToComplex;
     case Type::STK_FloatingComplex:
-      Src = S.ImpCastExprToType(Src.take(), DestTy->getAs<ComplexType>()->getElementType(),
-                                CK_IntegralToFloating);
+      Src = ImpCastExprToType(Src.take(),
+                              DestTy->castAs<ComplexType>()->getElementType(),
+                              CK_IntegralToFloating);
       return CK_FloatingRealToComplex;
     case Type::STK_MemberPointer:
       llvm_unreachable("member pointer type in C");
@@ -3811,14 +4004,18 @@ static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
     case Type::STK_Integral:
       return CK_FloatingToIntegral;
     case Type::STK_FloatingComplex:
-      Src = S.ImpCastExprToType(Src.take(), DestTy->getAs<ComplexType>()->getElementType(),
-                                CK_FloatingCast);
+      Src = ImpCastExprToType(Src.take(),
+                              DestTy->castAs<ComplexType>()->getElementType(),
+                              CK_FloatingCast);
       return CK_FloatingRealToComplex;
     case Type::STK_IntegralComplex:
-      Src = S.ImpCastExprToType(Src.take(), DestTy->getAs<ComplexType>()->getElementType(),
-                                CK_FloatingToIntegral);
+      Src = ImpCastExprToType(Src.take(),
+                              DestTy->castAs<ComplexType>()->getElementType(),
+                              CK_FloatingToIntegral);
       return CK_IntegralRealToComplex;
-    case Type::STK_Pointer:
+    case Type::STK_CPointer:
+    case Type::STK_ObjCObjectPointer:
+    case Type::STK_BlockPointer:
       llvm_unreachable("valid float->pointer cast?");
     case Type::STK_MemberPointer:
       llvm_unreachable("member pointer type in C");
@@ -3832,19 +4029,22 @@ static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
     case Type::STK_IntegralComplex:
       return CK_FloatingComplexToIntegralComplex;
     case Type::STK_Floating: {
-      QualType ET = SrcTy->getAs<ComplexType>()->getElementType();
-      if (S.Context.hasSameType(ET, DestTy))
+      QualType ET = SrcTy->castAs<ComplexType>()->getElementType();
+      if (Context.hasSameType(ET, DestTy))
         return CK_FloatingComplexToReal;
-      Src = S.ImpCastExprToType(Src.take(), ET, CK_FloatingComplexToReal);
+      Src = ImpCastExprToType(Src.take(), ET, CK_FloatingComplexToReal);
       return CK_FloatingCast;
     }
     case Type::STK_Bool:
       return CK_FloatingComplexToBoolean;
     case Type::STK_Integral:
-      Src = S.ImpCastExprToType(Src.take(), SrcTy->getAs<ComplexType>()->getElementType(),
-                                CK_FloatingComplexToReal);
+      Src = ImpCastExprToType(Src.take(),
+                              SrcTy->castAs<ComplexType>()->getElementType(),
+                              CK_FloatingComplexToReal);
       return CK_FloatingToIntegral;
-    case Type::STK_Pointer:
+    case Type::STK_CPointer:
+    case Type::STK_ObjCObjectPointer:
+    case Type::STK_BlockPointer:
       llvm_unreachable("valid complex float->pointer cast?");
     case Type::STK_MemberPointer:
       llvm_unreachable("member pointer type in C");
@@ -3858,19 +4058,22 @@ static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
     case Type::STK_IntegralComplex:
       return CK_IntegralComplexCast;
     case Type::STK_Integral: {
-      QualType ET = SrcTy->getAs<ComplexType>()->getElementType();
-      if (S.Context.hasSameType(ET, DestTy))
+      QualType ET = SrcTy->castAs<ComplexType>()->getElementType();
+      if (Context.hasSameType(ET, DestTy))
         return CK_IntegralComplexToReal;
-      Src = S.ImpCastExprToType(Src.take(), ET, CK_IntegralComplexToReal);
+      Src = ImpCastExprToType(Src.take(), ET, CK_IntegralComplexToReal);
       return CK_IntegralCast;
     }
     case Type::STK_Bool:
       return CK_IntegralComplexToBoolean;
     case Type::STK_Floating:
-      Src = S.ImpCastExprToType(Src.take(), SrcTy->getAs<ComplexType>()->getElementType(),
-                                CK_IntegralComplexToReal);
+      Src = ImpCastExprToType(Src.take(),
+                              SrcTy->castAs<ComplexType>()->getElementType(),
+                              CK_IntegralComplexToReal);
       return CK_IntegralToFloating;
-    case Type::STK_Pointer:
+    case Type::STK_CPointer:
+    case Type::STK_ObjCObjectPointer:
+    case Type::STK_BlockPointer:
       llvm_unreachable("valid complex int->pointer cast?");
     case Type::STK_MemberPointer:
       llvm_unreachable("member pointer type in C");
@@ -3879,193 +4082,6 @@ static CastKind PrepareScalarCast(Sema &S, ExprResult &Src, QualType DestTy) {
   }
 
   llvm_unreachable("Unhandled scalar cast");
-  return CK_BitCast;
-}
-
-/// CheckCastTypes - Check type constraints for casting between types.
-ExprResult Sema::CheckCastTypes(SourceLocation CastStartLoc, SourceRange TyR, 
-                                QualType castType, Expr *castExpr, 
-                                CastKind& Kind, ExprValueKind &VK,
-                                CXXCastPath &BasePath, bool FunctionalStyle) {
-  if (castExpr->getType() == Context.UnknownAnyTy)
-    return checkUnknownAnyCast(TyR, castType, castExpr, Kind, VK, BasePath);
-
-  if (getLangOptions().CPlusPlus)
-    return CXXCheckCStyleCast(SourceRange(CastStartLoc,
-                                          castExpr->getLocEnd()), 
-                              castType, VK, castExpr, Kind, BasePath,
-                              FunctionalStyle);
-
-  assert(!castExpr->getType()->isPlaceholderType());
-
-  // We only support r-value casts in C.
-  VK = VK_RValue;
-
-  // C99 6.5.4p2: the cast type needs to be void or scalar and the expression
-  // type needs to be scalar.
-  if (castType->isVoidType()) {
-    // We don't necessarily do lvalue-to-rvalue conversions on this.
-    ExprResult castExprRes = IgnoredValueConversions(castExpr);
-    if (castExprRes.isInvalid())
-      return ExprError();
-    castExpr = castExprRes.take();
-
-    // Cast to void allows any expr type.
-    Kind = CK_ToVoid;
-    return Owned(castExpr);
-  }
-
-  ExprResult castExprRes = DefaultFunctionArrayLvalueConversion(castExpr);
-  if (castExprRes.isInvalid())
-    return ExprError();
-  castExpr = castExprRes.take();
-
-  if (RequireCompleteType(TyR.getBegin(), castType,
-                          diag::err_typecheck_cast_to_incomplete))
-    return ExprError();
-
-  if (!castType->isScalarType() && !castType->isVectorType()) {
-    if (Context.hasSameUnqualifiedType(castType, castExpr->getType()) &&
-        (castType->isStructureType() || castType->isUnionType())) {
-      // GCC struct/union extension: allow cast to self.
-      // FIXME: Check that the cast destination type is complete.
-      Diag(TyR.getBegin(), diag::ext_typecheck_cast_nonscalar)
-        << castType << castExpr->getSourceRange();
-      Kind = CK_NoOp;
-      return Owned(castExpr);
-    }
-
-    if (castType->isUnionType()) {
-      // GCC cast to union extension
-      RecordDecl *RD = castType->getAs<RecordType>()->getDecl();
-      RecordDecl::field_iterator Field, FieldEnd;
-      for (Field = RD->field_begin(), FieldEnd = RD->field_end();
-           Field != FieldEnd; ++Field) {
-        if (Context.hasSameUnqualifiedType(Field->getType(),
-                                           castExpr->getType()) &&
-            !Field->isUnnamedBitfield()) {
-          Diag(TyR.getBegin(), diag::ext_typecheck_cast_to_union)
-            << castExpr->getSourceRange();
-          break;
-        }
-      }
-      if (Field == FieldEnd) {
-        Diag(TyR.getBegin(), diag::err_typecheck_cast_to_union_no_type)
-          << castExpr->getType() << castExpr->getSourceRange();
-        return ExprError();
-      }
-      Kind = CK_ToUnion;
-      return Owned(castExpr);
-    }
-
-    // Reject any other conversions to non-scalar types.
-    Diag(TyR.getBegin(), diag::err_typecheck_cond_expect_scalar)
-      << castType << castExpr->getSourceRange();
-    return ExprError();
-  }
-
-  // The type we're casting to is known to be a scalar or vector.
-
-  // Require the operand to be a scalar or vector.
-  if (!castExpr->getType()->isScalarType() &&
-      !castExpr->getType()->isVectorType()) {
-    Diag(castExpr->getLocStart(),
-                diag::err_typecheck_expect_scalar_operand)
-      << castExpr->getType() << castExpr->getSourceRange();
-    return ExprError();
-  }
-
-  if (castType->isExtVectorType())
-    return CheckExtVectorCast(TyR, castType, castExpr, Kind);
-
-  if (castType->isVectorType()) {
-    if (castType->getAs<VectorType>()->getVectorKind() ==
-        VectorType::AltiVecVector &&
-          (castExpr->getType()->isIntegerType() ||
-           castExpr->getType()->isFloatingType())) {
-      Kind = CK_VectorSplat;
-      return Owned(castExpr);
-    } else if (CheckVectorCast(TyR, castType, castExpr->getType(), Kind)) {
-      return ExprError();
-    } else
-      return Owned(castExpr);
-  }
-  if (castExpr->getType()->isVectorType()) {
-    if (CheckVectorCast(TyR, castExpr->getType(), castType, Kind))
-      return ExprError();
-    else
-      return Owned(castExpr);
-  }
-
-  // The source and target types are both scalars, i.e.
-  //   - arithmetic types (fundamental, enum, and complex)
-  //   - all kinds of pointers
-  // Note that member pointers were filtered out with C++, above.
-
-  if (isa<ObjCSelectorExpr>(castExpr)) {
-    Diag(castExpr->getLocStart(), diag::err_cast_selector_expr);
-    return ExprError();
-  }
-
-  // If either type is a pointer, the other type has to be either an
-  // integer or a pointer.
-  QualType castExprType = castExpr->getType();
-  if (!castType->isArithmeticType()) {
-    if (!castExprType->isIntegralType(Context) && 
-        castExprType->isArithmeticType()) {
-      Diag(castExpr->getLocStart(),
-           diag::err_cast_pointer_from_non_pointer_int)
-        << castExprType << castExpr->getSourceRange();
-      return ExprError();
-    }
-  } else if (!castExpr->getType()->isArithmeticType()) {
-    if (!castType->isIntegralType(Context) && castType->isArithmeticType()) {
-      Diag(castExpr->getLocStart(), diag::err_cast_pointer_to_non_pointer_int)
-        << castType << castExpr->getSourceRange();
-      return ExprError();
-    }
-  }
-
-  if (getLangOptions().ObjCAutoRefCount) {
-    // Diagnose problems with Objective-C casts involving lifetime qualifiers.
-    CheckObjCARCConversion(SourceRange(CastStartLoc, castExpr->getLocEnd()), 
-                           castType, castExpr, CCK_CStyleCast);
-    
-    if (const PointerType *CastPtr = castType->getAs<PointerType>()) {
-      if (const PointerType *ExprPtr = castExprType->getAs<PointerType>()) {
-        Qualifiers CastQuals = CastPtr->getPointeeType().getQualifiers();
-        Qualifiers ExprQuals = ExprPtr->getPointeeType().getQualifiers();
-        if (CastPtr->getPointeeType()->isObjCLifetimeType() && 
-            ExprPtr->getPointeeType()->isObjCLifetimeType() &&
-            !CastQuals.compatiblyIncludesObjCLifetime(ExprQuals)) {
-          Diag(castExpr->getLocStart(), 
-               diag::err_typecheck_incompatible_ownership)
-            << castExprType << castType << AA_Casting
-            << castExpr->getSourceRange();
-          
-          return ExprError();
-        }
-      }
-    } 
-    else if (!CheckObjCARCUnavailableWeakConversion(castType, castExprType)) {
-           Diag(castExpr->getLocStart(), 
-                diag::err_arc_convesion_of_weak_unavailable) << 1
-                << castExprType << castType 
-                << castExpr->getSourceRange();
-          return ExprError();
-    }
-  }
-  
-  castExprRes = Owned(castExpr);
-  Kind = PrepareScalarCast(*this, castExprRes, castType);
-  if (castExprRes.isInvalid())
-    return ExprError();
-  castExpr = castExprRes.take();
-
-  if (Kind == CK_BitCast)
-    CheckCastAlign(castExpr, castType, TyR);
-
-  return Owned(castExpr);
 }
 
 bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
@@ -4096,8 +4112,12 @@ ExprResult Sema::CheckExtVectorCast(SourceRange R, QualType DestTy,
 
   // If SrcTy is a VectorType, the total size must match to explicitly cast to
   // an ExtVectorType.
+  // In OpenCL, casts between vectors of different types are not allowed.
+  // (See OpenCL 6.2).
   if (SrcTy->isVectorType()) {
-    if (Context.getTypeSize(DestTy) != Context.getTypeSize(SrcTy)) {
+    if (Context.getTypeSize(DestTy) != Context.getTypeSize(SrcTy)
+        || (getLangOptions().OpenCL &&
+            (DestTy.getCanonicalType() != SrcTy.getCanonicalType()))) {
       Diag(R.getBegin(),diag::err_invalid_conversion_between_ext_vectors)
         << DestTy << SrcTy << R;
       return ExprError();
@@ -4116,7 +4136,7 @@ ExprResult Sema::CheckExtVectorCast(SourceRange R, QualType DestTy,
 
   QualType DestElemTy = DestTy->getAs<ExtVectorType>()->getElementType();
   ExprResult CastExprRes = Owned(CastExpr);
-  CastKind CK = PrepareScalarCast(*this, CastExprRes, DestElemTy);
+  CastKind CK = PrepareScalarCast(CastExprRes, DestElemTy);
   if (CastExprRes.isInvalid())
     return ExprError();
   CastExpr = ImpCastExprToType(CastExprRes.take(), DestElemTy, CK).take();
@@ -4128,11 +4148,11 @@ ExprResult Sema::CheckExtVectorCast(SourceRange R, QualType DestTy,
 ExprResult
 Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
                     Declarator &D, ParsedType &Ty,
-                    SourceLocation RParenLoc, Expr *castExpr) {
-  assert(!D.isInvalidType() && (castExpr != 0) &&
+                    SourceLocation RParenLoc, Expr *CastExpr) {
+  assert(!D.isInvalidType() && (CastExpr != 0) &&
          "ActOnCastExpr(): missing type or expr");
 
-  TypeSourceInfo *castTInfo = GetTypeForDeclaratorCast(D, castExpr->getType());
+  TypeSourceInfo *castTInfo = GetTypeForDeclaratorCast(D, CastExpr->getType());
   if (D.isInvalidType())
     return ExprError();
 
@@ -4141,6 +4161,8 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
     CheckExtraCXXDefaultArguments(D);
   }
 
+  checkUnusedDeclAttributes(D);
+
   QualType castType = castTInfo->getType();
   Ty = CreateParsedType(castType, castTInfo);
 
@@ -4148,9 +4170,10 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
 
   // Check for an altivec or OpenCL literal,
   // i.e. all the elements are integer constants.
-  ParenExpr *PE = dyn_cast<ParenExpr>(castExpr);
-  ParenListExpr *PLE = dyn_cast<ParenListExpr>(castExpr);
-  if (getLangOptions().AltiVec && castType->isVectorType() && (PE || PLE)) {
+  ParenExpr *PE = dyn_cast<ParenExpr>(CastExpr);
+  ParenListExpr *PLE = dyn_cast<ParenListExpr>(CastExpr);
+  if ((getLangOptions().AltiVec || getLangOptions().OpenCL)
+       && castType->isVectorType() && (PE || PLE)) {
     if (PLE && PLE->getNumExprs() == 0) {
       Diag(PLE->getExprLoc(), diag::err_altivec_empty_initializer);
       return ExprError();
@@ -4167,37 +4190,18 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
   // If this is a vector initializer, '(' type ')' '(' init, ..., init ')'
   // then handle it as such.
   if (isVectorLiteral)
-    return BuildVectorLiteral(LParenLoc, RParenLoc, castExpr, castTInfo);
+    return BuildVectorLiteral(LParenLoc, RParenLoc, CastExpr, castTInfo);
 
   // If the Expr being casted is a ParenListExpr, handle it specially.
   // This is not an AltiVec-style cast, so turn the ParenListExpr into a
   // sequence of BinOp comma operators.
-  if (isa<ParenListExpr>(castExpr)) {
-    ExprResult Result = MaybeConvertParenListExprToParenExpr(S, castExpr);
+  if (isa<ParenListExpr>(CastExpr)) {
+    ExprResult Result = MaybeConvertParenListExprToParenExpr(S, CastExpr);
     if (Result.isInvalid()) return ExprError();
-    castExpr = Result.take();
+    CastExpr = Result.take();
   }
 
-  return BuildCStyleCastExpr(LParenLoc, castTInfo, RParenLoc, castExpr);
-}
-
-ExprResult
-Sema::BuildCStyleCastExpr(SourceLocation LParenLoc, TypeSourceInfo *Ty,
-                          SourceLocation RParenLoc, Expr *castExpr) {
-  CastKind Kind = CK_Invalid;
-  ExprValueKind VK = VK_RValue;
-  CXXCastPath BasePath;
-  ExprResult CastResult =
-    CheckCastTypes(LParenLoc, SourceRange(LParenLoc, RParenLoc), Ty->getType(), 
-                   castExpr, Kind, VK, BasePath);
-  if (CastResult.isInvalid())
-    return ExprError();
-  castExpr = CastResult.take();
-
-  return Owned(CStyleCastExpr::Create(Context,
-                                      Ty->getType().getNonLValueExprType(Context),
-                                      VK, Kind, castExpr, &BasePath, Ty,
-                                      LParenLoc, RParenLoc));
+  return BuildCStyleCastExpr(LParenLoc, castTInfo, RParenLoc, CastExpr);
 }
 
 ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
@@ -4221,7 +4225,7 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
   QualType Ty = TInfo->getType();
   assert(Ty->isVectorType() && "Expected vector type");
 
-  llvm::SmallVector<Expr *, 8> initExprs;
+  SmallVector<Expr *, 8> initExprs;
   const VectorType *VTy = Ty->getAs<VectorType>();
   unsigned numElems = Ty->getAs<VectorType>()->getNumElements();
   
@@ -4237,7 +4241,7 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
       QualType ElemTy = Ty->getAs<VectorType>()->getElementType();
       ExprResult Literal = Owned(exprs[0]);
       Literal = ImpCastExprToType(Literal.take(), ElemTy,
-                                  PrepareScalarCast(*this, Literal, ElemTy));
+                                  PrepareScalarCast(Literal, ElemTy));
       return BuildCStyleCastExpr(LParenLoc, TInfo, RParenLoc, Literal.take());
     }
     else if (numExprs < numElems) {
@@ -4258,7 +4262,7 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
         QualType ElemTy = Ty->getAs<VectorType>()->getElementType();
         ExprResult Literal = Owned(exprs[0]);
         Literal = ImpCastExprToType(Literal.take(), ElemTy,
-                                    PrepareScalarCast(*this, Literal, ElemTy));
+                                    PrepareScalarCast(Literal, ElemTy));
         return BuildCStyleCastExpr(LParenLoc, TInfo, RParenLoc, Literal.take());
     }
     
@@ -4277,10 +4281,10 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
 /// This is not an AltiVec-style cast, so turn the ParenListExpr into a sequence
 /// of comma binary operators.
 ExprResult
-Sema::MaybeConvertParenListExprToParenExpr(Scope *S, Expr *expr) {
-  ParenListExpr *E = dyn_cast<ParenListExpr>(expr);
+Sema::MaybeConvertParenListExprToParenExpr(Scope *S, Expr *OrigExpr) {
+  ParenListExpr *E = dyn_cast<ParenListExpr>(OrigExpr);
   if (!E)
-    return Owned(expr);
+    return Owned(OrigExpr);
 
   ExprResult Result(E->getExpr(0));
 
@@ -4294,8 +4298,8 @@ Sema::MaybeConvertParenListExprToParenExpr(Scope *S, Expr *expr) {
 }
 
 ExprResult Sema::ActOnParenOrParenListExpr(SourceLocation L,
-                                                  SourceLocation R,
-                                                  MultiExprArg Val) {
+                                           SourceLocation R,
+                                           MultiExprArg Val) {
   unsigned nexprs = Val.size();
   Expr **exprs = reinterpret_cast<Expr**>(Val.release());
   assert((exprs != 0) && "ActOnParenOrParenListExpr() missing expr list");
@@ -4309,18 +4313,19 @@ ExprResult Sema::ActOnParenOrParenListExpr(SourceLocation L,
 }
 
 /// \brief Emit a specialized diagnostic when one expression is a null pointer
-/// constant and the other is not a pointer.
-bool Sema::DiagnoseConditionalForNull(Expr *LHS, Expr *RHS,
+/// constant and the other is not a pointer.  Returns true if a diagnostic is
+/// emitted.
+bool Sema::DiagnoseConditionalForNull(Expr *LHSExpr, Expr *RHSExpr,
                                       SourceLocation QuestionLoc) {
-  Expr *NullExpr = LHS;
-  Expr *NonPointerExpr = RHS;
+  Expr *NullExpr = LHSExpr;
+  Expr *NonPointerExpr = RHSExpr;
   Expr::NullPointerConstantKind NullKind =
       NullExpr->isNullPointerConstant(Context,
                                       Expr::NPC_ValueDependentIsNotNull);
 
   if (NullKind == Expr::NPCK_NotNull) {
-    NullExpr = RHS;
-    NonPointerExpr = LHS;
+    NullExpr = RHSExpr;
+    NonPointerExpr = LHSExpr;
     NullKind =
         NullExpr->isNullPointerConstant(Context,
                                         Expr::NPC_ValueDependentIsNotNull);
@@ -4345,20 +4350,228 @@ bool Sema::DiagnoseConditionalForNull(Expr *LHS, Expr *RHS,
   return true;
 }
 
-/// Note that lhs is not null here, even if this is the gnu "x ?: y" extension.
-/// In that case, lhs = cond.
+/// \brief Return false if the condition expression is valid, true otherwise.
+static bool checkCondition(Sema &S, Expr *Cond) {
+  QualType CondTy = Cond->getType();
+
+  // C99 6.5.15p2
+  if (CondTy->isScalarType()) return false;
+
+  // OpenCL: Sec 6.3.i says the condition is allowed to be a vector or scalar.
+  if (S.getLangOptions().OpenCL && CondTy->isVectorType())
+    return false;
+
+  // Emit the proper error message.
+  S.Diag(Cond->getLocStart(), S.getLangOptions().OpenCL ?
+                              diag::err_typecheck_cond_expect_scalar :
+                              diag::err_typecheck_cond_expect_scalar_or_vector)
+    << CondTy;
+  return true;
+}
+
+/// \brief Return false if the two expressions can be converted to a vector,
+/// true otherwise
+static bool checkConditionalConvertScalarsToVectors(Sema &S, ExprResult &LHS,
+                                                    ExprResult &RHS,
+                                                    QualType CondTy) {
+  // Both operands should be of scalar type.
+  if (!LHS.get()->getType()->isScalarType()) {
+    S.Diag(LHS.get()->getLocStart(), diag::err_typecheck_cond_expect_scalar)
+      << CondTy;
+    return true;
+  }
+  if (!RHS.get()->getType()->isScalarType()) {
+    S.Diag(RHS.get()->getLocStart(), diag::err_typecheck_cond_expect_scalar)
+      << CondTy;
+    return true;
+  }
+
+  // Implicity convert these scalars to the type of the condition.
+  LHS = S.ImpCastExprToType(LHS.take(), CondTy, CK_IntegralCast);
+  RHS = S.ImpCastExprToType(RHS.take(), CondTy, CK_IntegralCast);
+  return false;
+}
+
+/// \brief Handle when one or both operands are void type.
+static QualType checkConditionalVoidType(Sema &S, ExprResult &LHS,
+                                         ExprResult &RHS) {
+    Expr *LHSExpr = LHS.get();
+    Expr *RHSExpr = RHS.get();
+
+    if (!LHSExpr->getType()->isVoidType())
+      S.Diag(RHSExpr->getLocStart(), diag::ext_typecheck_cond_one_void)
+        << RHSExpr->getSourceRange();
+    if (!RHSExpr->getType()->isVoidType())
+      S.Diag(LHSExpr->getLocStart(), diag::ext_typecheck_cond_one_void)
+        << LHSExpr->getSourceRange();
+    LHS = S.ImpCastExprToType(LHS.take(), S.Context.VoidTy, CK_ToVoid);
+    RHS = S.ImpCastExprToType(RHS.take(), S.Context.VoidTy, CK_ToVoid);
+    return S.Context.VoidTy;
+}
+
+/// \brief Return false if the NullExpr can be promoted to PointerTy,
+/// true otherwise.
+static bool checkConditionalNullPointer(Sema &S, ExprResult &NullExpr,
+                                        QualType PointerTy) {
+  if ((!PointerTy->isAnyPointerType() && !PointerTy->isBlockPointerType()) ||
+      !NullExpr.get()->isNullPointerConstant(S.Context,
+                                            Expr::NPC_ValueDependentIsNull))
+    return true;
+
+  NullExpr = S.ImpCastExprToType(NullExpr.take(), PointerTy, CK_NullToPointer);
+  return false;
+}
+
+/// \brief Checks compatibility between two pointers and return the resulting
+/// type.
+static QualType checkConditionalPointerCompatibility(Sema &S, ExprResult &LHS,
+                                                     ExprResult &RHS,
+                                                     SourceLocation Loc) {
+  QualType LHSTy = LHS.get()->getType();
+  QualType RHSTy = RHS.get()->getType();
+
+  if (S.Context.hasSameType(LHSTy, RHSTy)) {
+    // Two identical pointers types are always compatible.
+    return LHSTy;
+  }
+
+  QualType lhptee, rhptee;
+
+  // Get the pointee types.
+  if (const BlockPointerType *LHSBTy = LHSTy->getAs<BlockPointerType>()) {
+    lhptee = LHSBTy->getPointeeType();
+    rhptee = RHSTy->castAs<BlockPointerType>()->getPointeeType();
+  } else {
+    lhptee = LHSTy->castAs<PointerType>()->getPointeeType();
+    rhptee = RHSTy->castAs<PointerType>()->getPointeeType();
+  }
+
+  if (!S.Context.typesAreCompatible(lhptee.getUnqualifiedType(),
+                                    rhptee.getUnqualifiedType())) {
+    S.Diag(Loc, diag::warn_typecheck_cond_incompatible_pointers)
+      << LHSTy << RHSTy << LHS.get()->getSourceRange()
+      << RHS.get()->getSourceRange();
+    // In this situation, we assume void* type. No especially good
+    // reason, but this is what gcc does, and we do have to pick
+    // to get a consistent AST.
+    QualType incompatTy = S.Context.getPointerType(S.Context.VoidTy);
+    LHS = S.ImpCastExprToType(LHS.take(), incompatTy, CK_BitCast);
+    RHS = S.ImpCastExprToType(RHS.take(), incompatTy, CK_BitCast);
+    return incompatTy;
+  }
+
+  // The pointer types are compatible.
+  // C99 6.5.15p6: If both operands are pointers to compatible types *or* to
+  // differently qualified versions of compatible types, the result type is
+  // a pointer to an appropriately qualified version of the *composite*
+  // type.
+  // FIXME: Need to calculate the composite type.
+  // FIXME: Need to add qualifiers
+
+  LHS = S.ImpCastExprToType(LHS.take(), LHSTy, CK_BitCast);
+  RHS = S.ImpCastExprToType(RHS.take(), LHSTy, CK_BitCast);
+  return LHSTy;
+}
+
+/// \brief Return the resulting type when the operands are both block pointers.
+static QualType checkConditionalBlockPointerCompatibility(Sema &S,
+                                                          ExprResult &LHS,
+                                                          ExprResult &RHS,
+                                                          SourceLocation Loc) {
+  QualType LHSTy = LHS.get()->getType();
+  QualType RHSTy = RHS.get()->getType();
+
+  if (!LHSTy->isBlockPointerType() || !RHSTy->isBlockPointerType()) {
+    if (LHSTy->isVoidPointerType() || RHSTy->isVoidPointerType()) {
+      QualType destType = S.Context.getPointerType(S.Context.VoidTy);
+      LHS = S.ImpCastExprToType(LHS.take(), destType, CK_BitCast);
+      RHS = S.ImpCastExprToType(RHS.take(), destType, CK_BitCast);
+      return destType;
+    }
+    S.Diag(Loc, diag::err_typecheck_cond_incompatible_operands)
+      << LHSTy << RHSTy << LHS.get()->getSourceRange()
+      << RHS.get()->getSourceRange();
+    return QualType();
+  }
+
+  // We have 2 block pointer types.
+  return checkConditionalPointerCompatibility(S, LHS, RHS, Loc);
+}
+
+/// \brief Return the resulting type when the operands are both pointers.
+static QualType
+checkConditionalObjectPointersCompatibility(Sema &S, ExprResult &LHS,
+                                            ExprResult &RHS,
+                                            SourceLocation Loc) {
+  // get the pointer types
+  QualType LHSTy = LHS.get()->getType();
+  QualType RHSTy = RHS.get()->getType();
+
+  // get the "pointed to" types
+  QualType lhptee = LHSTy->getAs<PointerType>()->getPointeeType();
+  QualType rhptee = RHSTy->getAs<PointerType>()->getPointeeType();
+
+  // ignore qualifiers on void (C99 6.5.15p3, clause 6)
+  if (lhptee->isVoidType() && rhptee->isIncompleteOrObjectType()) {
+    // Figure out necessary qualifiers (C99 6.5.15p6)
+    QualType destPointee
+      = S.Context.getQualifiedType(lhptee, rhptee.getQualifiers());
+    QualType destType = S.Context.getPointerType(destPointee);
+    // Add qualifiers if necessary.
+    LHS = S.ImpCastExprToType(LHS.take(), destType, CK_NoOp);
+    // Promote to void*.
+    RHS = S.ImpCastExprToType(RHS.take(), destType, CK_BitCast);
+    return destType;
+  }
+  if (rhptee->isVoidType() && lhptee->isIncompleteOrObjectType()) {
+    QualType destPointee
+      = S.Context.getQualifiedType(rhptee, lhptee.getQualifiers());
+    QualType destType = S.Context.getPointerType(destPointee);
+    // Add qualifiers if necessary.
+    RHS = S.ImpCastExprToType(RHS.take(), destType, CK_NoOp);
+    // Promote to void*.
+    LHS = S.ImpCastExprToType(LHS.take(), destType, CK_BitCast);
+    return destType;
+  }
+
+  return checkConditionalPointerCompatibility(S, LHS, RHS, Loc);
+}
+
+/// \brief Return false if the first expression is not an integer and the second
+/// expression is not a pointer, true otherwise.
+static bool checkPointerIntegerMismatch(Sema &S, ExprResult &Int,
+                                        Expr* PointerExpr, SourceLocation Loc,
+                                        bool IsIntFirstExpr) {
+  if (!PointerExpr->getType()->isPointerType() ||
+      !Int.get()->getType()->isIntegerType())
+    return false;
+
+  Expr *Expr1 = IsIntFirstExpr ? Int.get() : PointerExpr;
+  Expr *Expr2 = IsIntFirstExpr ? PointerExpr : Int.get();
+
+  S.Diag(Loc, diag::warn_typecheck_cond_pointer_integer_mismatch)
+    << Expr1->getType() << Expr2->getType()
+    << Expr1->getSourceRange() << Expr2->getSourceRange();
+  Int = S.ImpCastExprToType(Int.take(), PointerExpr->getType(),
+                            CK_IntegralToPointer);
+  return true;
+}
+
+/// Note that LHS is not null here, even if this is the gnu "x ?: y" extension.
+/// In that case, LHS = cond.
 /// C99 6.5.15
-QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, ExprResult &RHS,
-                                        ExprValueKind &VK, ExprObjectKind &OK,
+QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
+                                        ExprResult &RHS, ExprValueKind &VK,
+                                        ExprObjectKind &OK,
                                         SourceLocation QuestionLoc) {
 
-  ExprResult lhsResult = CheckPlaceholderExpr(LHS.get());
-  if (!lhsResult.isUsable()) return QualType();
-  LHS = move(lhsResult);
+  ExprResult LHSResult = CheckPlaceholderExpr(LHS.get());
+  if (!LHSResult.isUsable()) return QualType();
+  LHS = move(LHSResult);
 
-  ExprResult rhsResult = CheckPlaceholderExpr(RHS.get());
-  if (!rhsResult.isUsable()) return QualType();
-  RHS = move(rhsResult);
+  ExprResult RHSResult = CheckPlaceholderExpr(RHS.get());
+  if (!RHSResult.isUsable()) return QualType();
+  RHS = move(RHSResult);
 
   // C++ is sufficiently different to merit its own checker.
   if (getLangOptions().CPlusPlus)
@@ -4382,23 +4595,8 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, ExprR
   QualType RHSTy = RHS.get()->getType();
 
   // first, check the condition.
-  if (!CondTy->isScalarType()) { // C99 6.5.15p2
-    // OpenCL: Sec 6.3.i says the condition is allowed to be a vector or scalar.
-    // Throw an error if its not either.
-    if (getLangOptions().OpenCL) {
-      if (!CondTy->isVectorType()) {
-        Diag(Cond.get()->getLocStart(), 
-             diag::err_typecheck_cond_expect_scalar_or_vector)
-          << CondTy;
-        return QualType();
-      }
-    }
-    else {
-      Diag(Cond.get()->getLocStart(), diag::err_typecheck_cond_expect_scalar)
-        << CondTy;
-      return QualType();
-    }
-  }
+  if (checkCondition(*this, Cond.get()))
+    return QualType();
 
   // Now check the two expressions.
   if (LHSTy->isVectorType() || RHSTy->isVectorType())
@@ -4407,22 +4605,9 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, ExprR
   // OpenCL: If the condition is a vector, and both operands are scalar,
   // attempt to implicity convert them to the vector type to act like the
   // built in select.
-  if (getLangOptions().OpenCL && CondTy->isVectorType()) {
-    // Both operands should be of scalar type.
-    if (!LHSTy->isScalarType()) {
-      Diag(LHS.get()->getLocStart(), diag::err_typecheck_cond_expect_scalar)
-        << CondTy;
+  if (getLangOptions().OpenCL && CondTy->isVectorType())
+    if (checkConditionalConvertScalarsToVectors(*this, LHS, RHS, CondTy))
       return QualType();
-    }
-    if (!RHSTy->isScalarType()) {
-      Diag(RHS.get()->getLocStart(), diag::err_typecheck_cond_expect_scalar)
-        << CondTy;
-      return QualType();
-    }
-    // Implicity convert these scalars to the type of the condition.
-    LHS = ImpCastExprToType(LHS.take(), CondTy, CK_IntegralCast);
-    RHS = ImpCastExprToType(RHS.take(), CondTy, CK_IntegralCast);
-  }
   
   // If both operands have arithmetic type, do the usual arithmetic conversions
   // to find a common type: C99 6.5.15p3,5.
@@ -4447,29 +4632,13 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, ExprR
   // C99 6.5.15p5: "If both operands have void type, the result has void type."
   // The following || allows only one side to be void (a GCC-ism).
   if (LHSTy->isVoidType() || RHSTy->isVoidType()) {
-    if (!LHSTy->isVoidType())
-      Diag(RHS.get()->getLocStart(), diag::ext_typecheck_cond_one_void)
-        << RHS.get()->getSourceRange();
-    if (!RHSTy->isVoidType())
-      Diag(LHS.get()->getLocStart(), diag::ext_typecheck_cond_one_void)
-        << LHS.get()->getSourceRange();
-    LHS = ImpCastExprToType(LHS.take(), Context.VoidTy, CK_ToVoid);
-    RHS = ImpCastExprToType(RHS.take(), Context.VoidTy, CK_ToVoid);
-    return Context.VoidTy;
+    return checkConditionalVoidType(*this, LHS, RHS);
   }
+
   // C99 6.5.15p6 - "if one operand is a null pointer constant, the result has
   // the type of the other operand."
-  if ((LHSTy->isAnyPointerType() || LHSTy->isBlockPointerType()) &&
-      RHS.get()->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
-    // promote the null to a pointer.
-    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_NullToPointer);
-    return LHSTy;
-  }
-  if ((RHSTy->isAnyPointerType() || RHSTy->isBlockPointerType()) &&
-      LHS.get()->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
-    LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_NullToPointer);
-    return RHSTy;
-  }
+  if (!checkConditionalNullPointer(*this, RHS, LHSTy)) return LHSTy;
+  if (!checkConditionalNullPointer(*this, LHS, RHSTy)) return RHSTy;
 
   // All objective-c pointer type analysis is done here.
   QualType compositeType = FindCompositeObjCPointerType(LHS, RHS,
@@ -4481,116 +4650,23 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, ExprR
 
 
   // Handle block pointer types.
-  if (LHSTy->isBlockPointerType() || RHSTy->isBlockPointerType()) {
-    if (!LHSTy->isBlockPointerType() || !RHSTy->isBlockPointerType()) {
-      if (LHSTy->isVoidPointerType() || RHSTy->isVoidPointerType()) {
-        QualType destType = Context.getPointerType(Context.VoidTy);
-        LHS = ImpCastExprToType(LHS.take(), destType, CK_BitCast);
-        RHS = ImpCastExprToType(RHS.take(), destType, CK_BitCast);
-        return destType;
-      }
-      Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
-      << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-      return QualType();
-    }
-    // We have 2 block pointer types.
-    if (Context.getCanonicalType(LHSTy) == Context.getCanonicalType(RHSTy)) {
-      // Two identical block pointer types are always compatible.
-      return LHSTy;
-    }
-    // The block pointer types aren't identical, continue checking.
-    QualType lhptee = LHSTy->getAs<BlockPointerType>()->getPointeeType();
-    QualType rhptee = RHSTy->getAs<BlockPointerType>()->getPointeeType();
-
-    if (!Context.typesAreCompatible(lhptee.getUnqualifiedType(),
-                                    rhptee.getUnqualifiedType())) {
-      Diag(QuestionLoc, diag::warn_typecheck_cond_incompatible_pointers)
-      << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-      // In this situation, we assume void* type. No especially good
-      // reason, but this is what gcc does, and we do have to pick
-      // to get a consistent AST.
-      QualType incompatTy = Context.getPointerType(Context.VoidTy);
-      LHS = ImpCastExprToType(LHS.take(), incompatTy, CK_BitCast);
-      RHS = ImpCastExprToType(RHS.take(), incompatTy, CK_BitCast);
-      return incompatTy;
-    }
-    // The block pointer types are compatible.
-    LHS = ImpCastExprToType(LHS.take(), LHSTy, CK_BitCast);
-    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_BitCast);
-    return LHSTy;
-  }
+  if (LHSTy->isBlockPointerType() || RHSTy->isBlockPointerType())
+    return checkConditionalBlockPointerCompatibility(*this, LHS, RHS,
+                                                     QuestionLoc);
 
   // Check constraints for C object pointers types (C99 6.5.15p3,6).
-  if (LHSTy->isPointerType() && RHSTy->isPointerType()) {
-    // get the "pointed to" types
-    QualType lhptee = LHSTy->getAs<PointerType>()->getPointeeType();
-    QualType rhptee = RHSTy->getAs<PointerType>()->getPointeeType();
-
-    // ignore qualifiers on void (C99 6.5.15p3, clause 6)
-    if (lhptee->isVoidType() && rhptee->isIncompleteOrObjectType()) {
-      // Figure out necessary qualifiers (C99 6.5.15p6)
-      QualType destPointee
-        = Context.getQualifiedType(lhptee, rhptee.getQualifiers());
-      QualType destType = Context.getPointerType(destPointee);
-      // Add qualifiers if necessary.
-      LHS = ImpCastExprToType(LHS.take(), destType, CK_NoOp);
-      // Promote to void*.
-      RHS = ImpCastExprToType(RHS.take(), destType, CK_BitCast);
-      return destType;
-    }
-    if (rhptee->isVoidType() && lhptee->isIncompleteOrObjectType()) {
-      QualType destPointee
-        = Context.getQualifiedType(rhptee, lhptee.getQualifiers());
-      QualType destType = Context.getPointerType(destPointee);
-      // Add qualifiers if necessary.
-      RHS = ImpCastExprToType(RHS.take(), destType, CK_NoOp);
-      // Promote to void*.
-      LHS = ImpCastExprToType(LHS.take(), destType, CK_BitCast);
-      return destType;
-    }
-
-    if (Context.getCanonicalType(LHSTy) == Context.getCanonicalType(RHSTy)) {
-      // Two identical pointer types are always compatible.
-      return LHSTy;
-    }
-    if (!Context.typesAreCompatible(lhptee.getUnqualifiedType(),
-                                    rhptee.getUnqualifiedType())) {
-      Diag(QuestionLoc, diag::warn_typecheck_cond_incompatible_pointers)
-        << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-      // In this situation, we assume void* type. No especially good
-      // reason, but this is what gcc does, and we do have to pick
-      // to get a consistent AST.
-      QualType incompatTy = Context.getPointerType(Context.VoidTy);
-      LHS = ImpCastExprToType(LHS.take(), incompatTy, CK_BitCast);
-      RHS = ImpCastExprToType(RHS.take(), incompatTy, CK_BitCast);
-      return incompatTy;
-    }
-    // The pointer types are compatible.
-    // C99 6.5.15p6: If both operands are pointers to compatible types *or* to
-    // differently qualified versions of compatible types, the result type is
-    // a pointer to an appropriately qualified version of the *composite*
-    // type.
-    // FIXME: Need to calculate the composite type.
-    // FIXME: Need to add qualifiers
-    LHS = ImpCastExprToType(LHS.take(), LHSTy, CK_BitCast);
-    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_BitCast);
-    return LHSTy;
-  }
+  if (LHSTy->isPointerType() && RHSTy->isPointerType())
+    return checkConditionalObjectPointersCompatibility(*this, LHS, RHS,
+                                                       QuestionLoc);
 
   // GCC compatibility: soften pointer/integer mismatch.  Note that
   // null pointers have been filtered out by this point.
-  if (RHSTy->isPointerType() && LHSTy->isIntegerType()) {
-    Diag(QuestionLoc, diag::warn_typecheck_cond_pointer_integer_mismatch)
-      << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-    LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_IntegralToPointer);
+  if (checkPointerIntegerMismatch(*this, LHS, RHS.get(), QuestionLoc,
+      /*isIntFirstExpr=*/true))
     return RHSTy;
-  }
-  if (LHSTy->isPointerType() && RHSTy->isIntegerType()) {
-    Diag(QuestionLoc, diag::warn_typecheck_cond_pointer_integer_mismatch)
-      << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_IntegralToPointer);
+  if (checkPointerIntegerMismatch(*this, RHS, LHS.get(), QuestionLoc,
+      /*isIntFirstExpr=*/false))
     return LHSTy;
-  }
 
   // Emit a better diagnostic if one of the expressions is a null pointer
   // constant and the other is not a pointer type. In this case, the user most
@@ -4600,14 +4676,15 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, ExprR
 
   // Otherwise, the operands are not compatible.
   Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
-    << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
+    << LHSTy << RHSTy << LHS.get()->getSourceRange()
+    << RHS.get()->getSourceRange();
   return QualType();
 }
 
 /// FindCompositeObjCPointerType - Helper method to find composite type of
 /// two objective-c pointer types of the two input expressions.
 QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
-                                        SourceLocation QuestionLoc) {
+                                            SourceLocation QuestionLoc) {
   QualType LHSTy = LHS.get()->getType();
   QualType RHSTy = RHS.get()->getType();
 
@@ -4615,34 +4692,34 @@ QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
   // to the pseudo-builtin, because that will be implicitly cast back to the
   // redefinition type if an attempt is made to access its fields.
   if (LHSTy->isObjCClassType() &&
-      (Context.hasSameType(RHSTy, Context.ObjCClassRedefinitionType))) {
-    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_BitCast);
+      (Context.hasSameType(RHSTy, Context.getObjCClassRedefinitionType()))) {
+    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_CPointerToObjCPointerCast);
     return LHSTy;
   }
   if (RHSTy->isObjCClassType() &&
-      (Context.hasSameType(LHSTy, Context.ObjCClassRedefinitionType))) {
-    LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_BitCast);
+      (Context.hasSameType(LHSTy, Context.getObjCClassRedefinitionType()))) {
+    LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_CPointerToObjCPointerCast);
     return RHSTy;
   }
   // And the same for struct objc_object* / id
   if (LHSTy->isObjCIdType() &&
-      (Context.hasSameType(RHSTy, Context.ObjCIdRedefinitionType))) {
-    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_BitCast);
+      (Context.hasSameType(RHSTy, Context.getObjCIdRedefinitionType()))) {
+    RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_CPointerToObjCPointerCast);
     return LHSTy;
   }
   if (RHSTy->isObjCIdType() &&
-      (Context.hasSameType(LHSTy, Context.ObjCIdRedefinitionType))) {
-    LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_BitCast);
+      (Context.hasSameType(LHSTy, Context.getObjCIdRedefinitionType()))) {
+    LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_CPointerToObjCPointerCast);
     return RHSTy;
   }
   // And the same for struct objc_selector* / SEL
   if (Context.isObjCSelType(LHSTy) &&
-      (Context.hasSameType(RHSTy, Context.ObjCSelRedefinitionType))) {
+      (Context.hasSameType(RHSTy, Context.getObjCSelRedefinitionType()))) {
     RHS = ImpCastExprToType(RHS.take(), LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (Context.isObjCSelType(RHSTy) &&
-      (Context.hasSameType(LHSTy, Context.ObjCSelRedefinitionType))) {
+      (Context.hasSameType(LHSTy, Context.getObjCSelRedefinitionType()))) {
     LHS = ImpCastExprToType(LHS.take(), RHSTy, CK_BitCast);
     return RHSTy;
   }
@@ -4653,8 +4730,8 @@ QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
       // Two identical object pointer types are always compatible.
       return LHSTy;
     }
-    const ObjCObjectPointerType *LHSOPT = LHSTy->getAs<ObjCObjectPointerType>();
-    const ObjCObjectPointerType *RHSOPT = RHSTy->getAs<ObjCObjectPointerType>();
+    const ObjCObjectPointerType *LHSOPT = LHSTy->castAs<ObjCObjectPointerType>();
+    const ObjCObjectPointerType *RHSOPT = RHSTy->castAs<ObjCObjectPointerType>();
     QualType compositeType = LHSTy;
 
     // If both operands are interfaces and either operand can be
@@ -4752,18 +4829,20 @@ static bool IsArithmeticOp(BinaryOperatorKind Opc) {
 
 /// IsArithmeticBinaryExpr - Returns true if E is an arithmetic binary
 /// expression, either using a built-in or overloaded operator,
-/// and sets *OpCode to the opcode and *RHS to the right-hand side expression.
+/// and sets *OpCode to the opcode and *RHSExprs to the right-hand side
+/// expression.
 static bool IsArithmeticBinaryExpr(Expr *E, BinaryOperatorKind *Opcode,
-                                   Expr **RHS) {
-  E = E->IgnoreParenImpCasts();
+                                   Expr **RHSExprs) {
+  // Don't strip parenthesis: we should not warn if E is in parenthesis.
+  E = E->IgnoreImpCasts();
   E = E->IgnoreConversionOperator();
-  E = E->IgnoreParenImpCasts();
+  E = E->IgnoreImpCasts();
 
   // Built-in binary operator.
   if (BinaryOperator *OP = dyn_cast<BinaryOperator>(E)) {
     if (IsArithmeticOp(OP->getOpcode())) {
       *Opcode = OP->getOpcode();
-      *RHS = OP->getRHS();
+      *RHSExprs = OP->getRHS();
       return true;
     }
   }
@@ -4782,7 +4861,7 @@ static bool IsArithmeticBinaryExpr(Expr *E, BinaryOperatorKind *Opcode,
     BinaryOperatorKind OpKind = BinaryOperator::getOverloadedOpcode(OO);
     if (IsArithmeticOp(OpKind)) {
       *Opcode = OpKind;
-      *RHS = Call->getArg(1);
+      *RHSExprs = Call->getArg(1);
       return true;
     }
   }
@@ -4817,8 +4896,8 @@ static bool ExprLooksBoolean(Expr *E) {
 static void DiagnoseConditionalPrecedence(Sema &Self,
                                           SourceLocation OpLoc,
                                           Expr *Condition,
-                                          Expr *LHS,
-                                          Expr *RHS) {
+                                          Expr *LHSExpr,
+                                          Expr *RHSExpr) {
   BinaryOperatorKind CondOpcode;
   Expr *CondRHS;
 
@@ -4841,7 +4920,7 @@ static void DiagnoseConditionalPrecedence(Sema &Self,
 
   SuggestParentheses(Self, OpLoc,
     Self.PDiag(diag::note_precedence_conditional_first),
-    SourceRange(CondRHS->getLocStart(), RHS->getLocEnd()));
+    SourceRange(CondRHS->getLocStart(), RHSExpr->getLocEnd()));
 }
 
 /// ActOnConditionalOp - Parse a ?: operation.  Note that 'LHS' may be null
@@ -4898,7 +4977,8 @@ ExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
 
   return Owned(new (Context)
     BinaryConditionalOperator(commonExpr, opaqueValue, Cond.take(), LHS.take(),
-                              RHS.take(), QuestionLoc, ColonLoc, result, VK, OK));
+                              RHS.take(), QuestionLoc, ColonLoc, result, VK,
+                              OK));
 }
 
 // checkPointerTypesForAssignment - This is a very tricky routine (despite
@@ -4907,15 +4987,15 @@ ExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
 // This circumvents the usual type rules specified in 6.2.7p1 & 6.7.5.[1-3].
 // FIXME: add a couple examples in this comment.
 static Sema::AssignConvertType
-checkPointerTypesForAssignment(Sema &S, QualType lhsType, QualType rhsType) {
-  assert(lhsType.isCanonical() && "LHS not canonicalized!");
-  assert(rhsType.isCanonical() && "RHS not canonicalized!");
+checkPointerTypesForAssignment(Sema &S, QualType LHSType, QualType RHSType) {
+  assert(LHSType.isCanonical() && "LHS not canonicalized!");
+  assert(RHSType.isCanonical() && "RHS not canonicalized!");
 
   // get the "pointed to" type (ignoring qualifiers at the top level)
   const Type *lhptee, *rhptee;
   Qualifiers lhq, rhq;
-  llvm::tie(lhptee, lhq) = cast<PointerType>(lhsType)->getPointeeType().split();
-  llvm::tie(rhptee, rhq) = cast<PointerType>(rhsType)->getPointeeType().split();
+  llvm::tie(lhptee, lhq) = cast<PointerType>(LHSType)->getPointeeType().split();
+  llvm::tie(rhptee, rhq) = cast<PointerType>(RHSType)->getPointeeType().split();
 
   Sema::AssignConvertType ConvTy = Sema::Compatible;
 
@@ -5019,6 +5099,9 @@ checkPointerTypesForAssignment(Sema &S, QualType lhsType, QualType rhsType) {
     // General pointer incompatibility takes priority over qualifiers.
     return Sema::IncompatiblePointer;
   }
+  if (!S.getLangOptions().CPlusPlus &&
+      S.IsNoReturnConversion(ltrans, rtrans, ltrans))
+    return Sema::IncompatiblePointer;
   return ConvTy;
 }
 
@@ -5027,16 +5110,16 @@ checkPointerTypesForAssignment(Sema &S, QualType lhsType, QualType rhsType) {
 /// are compatible. It is more restrict than comparing two function pointer
 // types.
 static Sema::AssignConvertType
-checkBlockPointerTypesForAssignment(Sema &S, QualType lhsType,
-                                    QualType rhsType) {
-  assert(lhsType.isCanonical() && "LHS not canonicalized!");
-  assert(rhsType.isCanonical() && "RHS not canonicalized!");
+checkBlockPointerTypesForAssignment(Sema &S, QualType LHSType,
+                                    QualType RHSType) {
+  assert(LHSType.isCanonical() && "LHS not canonicalized!");
+  assert(RHSType.isCanonical() && "RHS not canonicalized!");
 
   QualType lhptee, rhptee;
 
   // get the "pointed to" type (ignoring qualifiers at the top level)
-  lhptee = cast<BlockPointerType>(lhsType)->getPointeeType();
-  rhptee = cast<BlockPointerType>(rhsType)->getPointeeType();
+  lhptee = cast<BlockPointerType>(LHSType)->getPointeeType();
+  rhptee = cast<BlockPointerType>(RHSType)->getPointeeType();
 
   // In C++, the types have to match exactly.
   if (S.getLangOptions().CPlusPlus)
@@ -5048,7 +5131,7 @@ checkBlockPointerTypesForAssignment(Sema &S, QualType lhsType,
   if (lhptee.getLocalQualifiers() != rhptee.getLocalQualifiers())
     ConvTy = Sema::CompatiblePointerDiscardsQualifiers;
 
-  if (!S.Context.typesAreBlockPointerCompatible(lhsType, rhsType))
+  if (!S.Context.typesAreBlockPointerCompatible(LHSType, RHSType))
     return Sema::IncompatibleBlockPointer;
 
   return ConvTy;
@@ -5057,51 +5140,49 @@ checkBlockPointerTypesForAssignment(Sema &S, QualType lhsType,
 /// checkObjCPointerTypesForAssignment - Compares two objective-c pointer types
 /// for assignment compatibility.
 static Sema::AssignConvertType
-checkObjCPointerTypesForAssignment(Sema &S, QualType lhsType, QualType rhsType) {
-  assert(lhsType.isCanonical() && "LHS was not canonicalized!");
-  assert(rhsType.isCanonical() && "RHS was not canonicalized!");
+checkObjCPointerTypesForAssignment(Sema &S, QualType LHSType,
+                                   QualType RHSType) {
+  assert(LHSType.isCanonical() && "LHS was not canonicalized!");
+  assert(RHSType.isCanonical() && "RHS was not canonicalized!");
 
-  if (lhsType->isObjCBuiltinType()) {
+  if (LHSType->isObjCBuiltinType()) {
     // Class is not compatible with ObjC object pointers.
-    if (lhsType->isObjCClassType() && !rhsType->isObjCBuiltinType() &&
-        !rhsType->isObjCQualifiedClassType())
+    if (LHSType->isObjCClassType() && !RHSType->isObjCBuiltinType() &&
+        !RHSType->isObjCQualifiedClassType())
       return Sema::IncompatiblePointer;
     return Sema::Compatible;
   }
-  if (rhsType->isObjCBuiltinType()) {
-    // Class is not compatible with ObjC object pointers.
-    if (rhsType->isObjCClassType() && !lhsType->isObjCBuiltinType() &&
-        !lhsType->isObjCQualifiedClassType())
+  if (RHSType->isObjCBuiltinType()) {
+    if (RHSType->isObjCClassType() && !LHSType->isObjCBuiltinType() &&
+        !LHSType->isObjCQualifiedClassType())
       return Sema::IncompatiblePointer;
     return Sema::Compatible;
   }
-  QualType lhptee =
-  lhsType->getAs<ObjCObjectPointerType>()->getPointeeType();
-  QualType rhptee =
-  rhsType->getAs<ObjCObjectPointerType>()->getPointeeType();
+  QualType lhptee = LHSType->getAs<ObjCObjectPointerType>()->getPointeeType();
+  QualType rhptee = RHSType->getAs<ObjCObjectPointerType>()->getPointeeType();
 
   if (!lhptee.isAtLeastAsQualifiedAs(rhptee))
     return Sema::CompatiblePointerDiscardsQualifiers;
 
-  if (S.Context.typesAreCompatible(lhsType, rhsType))
+  if (S.Context.typesAreCompatible(LHSType, RHSType))
     return Sema::Compatible;
-  if (lhsType->isObjCQualifiedIdType() || rhsType->isObjCQualifiedIdType())
+  if (LHSType->isObjCQualifiedIdType() || RHSType->isObjCQualifiedIdType())
     return Sema::IncompatibleObjCQualifiedId;
   return Sema::IncompatiblePointer;
 }
 
 Sema::AssignConvertType
 Sema::CheckAssignmentConstraints(SourceLocation Loc,
-                                 QualType lhsType, QualType rhsType) {
+                                 QualType LHSType, QualType RHSType) {
   // Fake up an opaque expression.  We don't actually care about what
   // cast operations are required, so if CheckAssignmentConstraints
   // adds casts to this they'll be wasted, but fortunately that doesn't
   // usually happen on valid code.
-  OpaqueValueExpr rhs(Loc, rhsType, VK_RValue);
-  ExprResult rhsPtr = &rhs;
+  OpaqueValueExpr RHSExpr(Loc, RHSType, VK_RValue);
+  ExprResult RHSPtr = &RHSExpr;
   CastKind K = CK_Invalid;
 
-  return CheckAssignmentConstraints(lhsType, rhsPtr, K);
+  return CheckAssignmentConstraints(LHSType, RHSPtr, K);
 }
 
 /// CheckAssignmentConstraints (C99 6.5.16) - This routine currently
@@ -5122,18 +5203,22 @@ Sema::CheckAssignmentConstraints(SourceLocation Loc,
 ///
 /// Sets 'Kind' for any result kind except Incompatible.
 Sema::AssignConvertType
-Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
+Sema::CheckAssignmentConstraints(QualType LHSType, ExprResult &RHS,
                                  CastKind &Kind) {
-  QualType rhsType = rhs.get()->getType();
-  QualType origLhsType = lhsType;
+  QualType RHSType = RHS.get()->getType();
+  QualType OrigLHSType = LHSType;
 
   // Get canonical types.  We're not formatting these types, just comparing
   // them.
-  lhsType = Context.getCanonicalType(lhsType).getUnqualifiedType();
-  rhsType = Context.getCanonicalType(rhsType).getUnqualifiedType();
+  LHSType = Context.getCanonicalType(LHSType).getUnqualifiedType();
+  RHSType = Context.getCanonicalType(RHSType).getUnqualifiedType();
+
+  // We can't do assignment from/to atomics yet.
+  if (LHSType->isAtomicType())
+    return Incompatible;
 
   // Common case: no conversion required.
-  if (lhsType == rhsType) {
+  if (LHSType == RHSType) {
     Kind = CK_NoOp;
     return Compatible;
   }
@@ -5143,10 +5228,10 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   // e.g., as a parameter type in a built-in function. In this case,
   // just make sure that the type referenced is compatible with the
   // right-hand side type. The caller is responsible for adjusting
-  // lhsType so that the resulting expression does not have reference
+  // LHSType so that the resulting expression does not have reference
   // type.
-  if (const ReferenceType *lhsTypeRef = lhsType->getAs<ReferenceType>()) {
-    if (Context.typesAreCompatible(lhsTypeRef->getPointeeType(), rhsType)) {
+  if (const ReferenceType *LHSTypeRef = LHSType->getAs<ReferenceType>()) {
+    if (Context.typesAreCompatible(LHSTypeRef->getPointeeType(), RHSType)) {
       Kind = CK_LValueBitCast;
       return Compatible;
     }
@@ -5155,16 +5240,16 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
 
   // Allow scalar to ExtVector assignments, and assignments of an ExtVector type
   // to the same ExtVector type.
-  if (lhsType->isExtVectorType()) {
-    if (rhsType->isExtVectorType())
+  if (LHSType->isExtVectorType()) {
+    if (RHSType->isExtVectorType())
       return Incompatible;
-    if (rhsType->isArithmeticType()) {
+    if (RHSType->isArithmeticType()) {
       // CK_VectorSplat does T -> vector T, so first cast to the
       // element type.
-      QualType elType = cast<ExtVectorType>(lhsType)->getElementType();
-      if (elType != rhsType) {
-        Kind = PrepareScalarCast(*this, rhs, elType);
-        rhs = ImpCastExprToType(rhs.take(), elType, Kind);
+      QualType elType = cast<ExtVectorType>(LHSType)->getElementType();
+      if (elType != RHSType) {
+        Kind = PrepareScalarCast(RHS, elType);
+        RHS = ImpCastExprToType(RHS.take(), elType, Kind);
       }
       Kind = CK_VectorSplat;
       return Compatible;
@@ -5172,11 +5257,11 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // Conversions to or from vector type.
-  if (lhsType->isVectorType() || rhsType->isVectorType()) {
-    if (lhsType->isVectorType() && rhsType->isVectorType()) {
+  if (LHSType->isVectorType() || RHSType->isVectorType()) {
+    if (LHSType->isVectorType() && RHSType->isVectorType()) {
       // Allow assignments of an AltiVec vector type to an equivalent GCC
       // vector type and vice versa
-      if (Context.areCompatibleVectorTypes(lhsType, rhsType)) {
+      if (Context.areCompatibleVectorTypes(LHSType, RHSType)) {
         Kind = CK_BitCast;
         return Compatible;
       }
@@ -5185,7 +5270,7 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
       // vectors, the total size only needs to be the same. This is a bitcast;
       // no bits are changed but the result type is different.
       if (getLangOptions().LaxVectorConversions &&
-          (Context.getTypeSize(lhsType) == Context.getTypeSize(rhsType))) {
+          (Context.getTypeSize(LHSType) == Context.getTypeSize(RHSType))) {
         Kind = CK_BitCast;
         return IncompatibleVectors;
       }
@@ -5194,38 +5279,39 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // Arithmetic conversions.
-  if (lhsType->isArithmeticType() && rhsType->isArithmeticType() &&
-      !(getLangOptions().CPlusPlus && lhsType->isEnumeralType())) {
-    Kind = PrepareScalarCast(*this, rhs, lhsType);
+  if (LHSType->isArithmeticType() && RHSType->isArithmeticType() &&
+      !(getLangOptions().CPlusPlus && LHSType->isEnumeralType())) {
+    Kind = PrepareScalarCast(RHS, LHSType);
     return Compatible;
   }
 
   // Conversions to normal pointers.
-  if (const PointerType *lhsPointer = dyn_cast<PointerType>(lhsType)) {
+  if (const PointerType *LHSPointer = dyn_cast<PointerType>(LHSType)) {
     // U* -> T*
-    if (isa<PointerType>(rhsType)) {
+    if (isa<PointerType>(RHSType)) {
       Kind = CK_BitCast;
-      return checkPointerTypesForAssignment(*this, lhsType, rhsType);
+      return checkPointerTypesForAssignment(*this, LHSType, RHSType);
     }
 
     // int -> T*
-    if (rhsType->isIntegerType()) {
+    if (RHSType->isIntegerType()) {
       Kind = CK_IntegralToPointer; // FIXME: null?
       return IntToPointer;
     }
 
     // C pointers are not compatible with ObjC object pointers,
     // with two exceptions:
-    if (isa<ObjCObjectPointerType>(rhsType)) {
+    if (isa<ObjCObjectPointerType>(RHSType)) {
       //  - conversions to void*
-      if (lhsPointer->getPointeeType()->isVoidType()) {
-        Kind = CK_AnyPointerToObjCPointerCast;
+      if (LHSPointer->getPointeeType()->isVoidType()) {
+        Kind = CK_BitCast;
         return Compatible;
       }
 
       //  - conversions from 'Class' to the redefinition type
-      if (rhsType->isObjCClassType() &&
-          Context.hasSameType(lhsType, Context.ObjCClassRedefinitionType)) {
+      if (RHSType->isObjCClassType() &&
+          Context.hasSameType(LHSType, 
+                              Context.getObjCClassRedefinitionType())) {
         Kind = CK_BitCast;
         return Compatible;
       }
@@ -5235,8 +5321,8 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
     }
 
     // U^ -> void*
-    if (rhsType->getAs<BlockPointerType>()) {
-      if (lhsPointer->getPointeeType()->isVoidType()) {
+    if (RHSType->getAs<BlockPointerType>()) {
+      if (LHSPointer->getPointeeType()->isVoidType()) {
         Kind = CK_BitCast;
         return Compatible;
       }
@@ -5246,27 +5332,27 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // Conversions to block pointers.
-  if (isa<BlockPointerType>(lhsType)) {
+  if (isa<BlockPointerType>(LHSType)) {
     // U^ -> T^
-    if (rhsType->isBlockPointerType()) {
-      Kind = CK_AnyPointerToBlockPointerCast;
-      return checkBlockPointerTypesForAssignment(*this, lhsType, rhsType);
+    if (RHSType->isBlockPointerType()) {
+      Kind = CK_BitCast;
+      return checkBlockPointerTypesForAssignment(*this, LHSType, RHSType);
     }
 
     // int or null -> T^
-    if (rhsType->isIntegerType()) {
+    if (RHSType->isIntegerType()) {
       Kind = CK_IntegralToPointer; // FIXME: null
       return IntToBlockPointer;
     }
 
     // id -> T^
-    if (getLangOptions().ObjC1 && rhsType->isObjCIdType()) {
+    if (getLangOptions().ObjC1 && RHSType->isObjCIdType()) {
       Kind = CK_AnyPointerToBlockPointerCast;
       return Compatible;
     }
 
     // void* -> T^
-    if (const PointerType *RHSPT = rhsType->getAs<PointerType>())
+    if (const PointerType *RHSPT = RHSType->getAs<PointerType>())
       if (RHSPT->getPointeeType()->isVoidType()) {
         Kind = CK_AnyPointerToBlockPointerCast;
         return Compatible;
@@ -5276,48 +5362,49 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // Conversions to Objective-C pointers.
-  if (isa<ObjCObjectPointerType>(lhsType)) {
+  if (isa<ObjCObjectPointerType>(LHSType)) {
     // A* -> B*
-    if (rhsType->isObjCObjectPointerType()) {
+    if (RHSType->isObjCObjectPointerType()) {
       Kind = CK_BitCast;
       Sema::AssignConvertType result = 
-        checkObjCPointerTypesForAssignment(*this, lhsType, rhsType);
+        checkObjCPointerTypesForAssignment(*this, LHSType, RHSType);
       if (getLangOptions().ObjCAutoRefCount &&
           result == Compatible && 
-          !CheckObjCARCUnavailableWeakConversion(origLhsType, rhsType))
+          !CheckObjCARCUnavailableWeakConversion(OrigLHSType, RHSType))
         result = IncompatibleObjCWeakRef;
       return result;
     }
 
     // int or null -> A*
-    if (rhsType->isIntegerType()) {
+    if (RHSType->isIntegerType()) {
       Kind = CK_IntegralToPointer; // FIXME: null
       return IntToPointer;
     }
 
     // In general, C pointers are not compatible with ObjC object pointers,
     // with two exceptions:
-    if (isa<PointerType>(rhsType)) {
+    if (isa<PointerType>(RHSType)) {
+      Kind = CK_CPointerToObjCPointerCast;
+
       //  - conversions from 'void*'
-      if (rhsType->isVoidPointerType()) {
-        Kind = CK_AnyPointerToObjCPointerCast;
+      if (RHSType->isVoidPointerType()) {
         return Compatible;
       }
 
       //  - conversions to 'Class' from its redefinition type
-      if (lhsType->isObjCClassType() &&
-          Context.hasSameType(rhsType, Context.ObjCClassRedefinitionType)) {
-        Kind = CK_BitCast;
+      if (LHSType->isObjCClassType() &&
+          Context.hasSameType(RHSType, 
+                              Context.getObjCClassRedefinitionType())) {
         return Compatible;
       }
 
-      Kind = CK_AnyPointerToObjCPointerCast;
       return IncompatiblePointer;
     }
 
     // T^ -> A*
-    if (rhsType->isBlockPointerType()) {
-      Kind = CK_AnyPointerToObjCPointerCast;
+    if (RHSType->isBlockPointerType()) {
+      maybeExtendBlockObject(*this, RHS);
+      Kind = CK_BlockPointerToObjCPointerCast;
       return Compatible;
     }
 
@@ -5325,15 +5412,15 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // Conversions from pointers that are not covered by the above.
-  if (isa<PointerType>(rhsType)) {
+  if (isa<PointerType>(RHSType)) {
     // T* -> _Bool
-    if (lhsType == Context.BoolTy) {
+    if (LHSType == Context.BoolTy) {
       Kind = CK_PointerToBoolean;
       return Compatible;
     }
 
     // T* -> int
-    if (lhsType->isIntegerType()) {
+    if (LHSType->isIntegerType()) {
       Kind = CK_PointerToIntegral;
       return PointerToInt;
     }
@@ -5342,15 +5429,15 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // Conversions from Objective-C pointers that are not covered by the above.
-  if (isa<ObjCObjectPointerType>(rhsType)) {
+  if (isa<ObjCObjectPointerType>(RHSType)) {
     // T* -> _Bool
-    if (lhsType == Context.BoolTy) {
+    if (LHSType == Context.BoolTy) {
       Kind = CK_PointerToBoolean;
       return Compatible;
     }
 
     // T* -> int
-    if (lhsType->isIntegerType()) {
+    if (LHSType->isIntegerType()) {
       Kind = CK_PointerToIntegral;
       return PointerToInt;
     }
@@ -5359,8 +5446,8 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
   }
 
   // struct A -> struct B
-  if (isa<TagType>(lhsType) && isa<TagType>(rhsType)) {
-    if (Context.typesAreCompatible(lhsType, rhsType)) {
+  if (isa<TagType>(LHSType) && isa<TagType>(RHSType)) {
+    if (Context.typesAreCompatible(LHSType, RHSType)) {
       Kind = CK_NoOp;
       return Compatible;
     }
@@ -5371,8 +5458,9 @@ Sema::CheckAssignmentConstraints(QualType lhsType, ExprResult &rhs,
 
 /// \brief Constructs a transparent union from an expression that is
 /// used to initialize the transparent union.
-static void ConstructTransparentUnion(Sema &S, ASTContext &C, ExprResult &EResult,
-                                      QualType UnionType, FieldDecl *Field) {
+static void ConstructTransparentUnion(Sema &S, ASTContext &C,
+                                      ExprResult &EResult, QualType UnionType,
+                                      FieldDecl *Field) {
   // Build an initializer list that designates the appropriate member
   // of the transparent union.
   Expr *E = EResult.take();
@@ -5391,8 +5479,9 @@ static void ConstructTransparentUnion(Sema &S, ASTContext &C, ExprResult &EResul
 }
 
 Sema::AssignConvertType
-Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType, ExprResult &rExpr) {
-  QualType FromType = rExpr.get()->getType();
+Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType,
+                                               ExprResult &RHS) {
+  QualType RHSType = RHS.get()->getType();
 
   // If the ArgType is a Union type, we want to handle a potential
   // transparent_union GCC extension.
@@ -5411,25 +5500,26 @@ Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType, ExprResult &rEx
       // If the transparent union contains a pointer type, we allow:
       // 1) void pointer
       // 2) null pointer constant
-      if (FromType->isPointerType())
-        if (FromType->getAs<PointerType>()->getPointeeType()->isVoidType()) {
-          rExpr = ImpCastExprToType(rExpr.take(), it->getType(), CK_BitCast);
+      if (RHSType->isPointerType())
+        if (RHSType->castAs<PointerType>()->getPointeeType()->isVoidType()) {
+          RHS = ImpCastExprToType(RHS.take(), it->getType(), CK_BitCast);
           InitField = *it;
           break;
         }
 
-      if (rExpr.get()->isNullPointerConstant(Context,
-                                       Expr::NPC_ValueDependentIsNull)) {
-        rExpr = ImpCastExprToType(rExpr.take(), it->getType(), CK_NullToPointer);
+      if (RHS.get()->isNullPointerConstant(Context,
+                                           Expr::NPC_ValueDependentIsNull)) {
+        RHS = ImpCastExprToType(RHS.take(), it->getType(),
+                                CK_NullToPointer);
         InitField = *it;
         break;
       }
     }
 
     CastKind Kind = CK_Invalid;
-    if (CheckAssignmentConstraints(it->getType(), rExpr, Kind)
+    if (CheckAssignmentConstraints(it->getType(), RHS, Kind)
           == Compatible) {
-      rExpr = ImpCastExprToType(rExpr.take(), it->getType(), Kind);
+      RHS = ImpCastExprToType(RHS.take(), it->getType(), Kind);
       InitField = *it;
       break;
     }
@@ -5438,42 +5528,46 @@ Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType, ExprResult &rEx
   if (!InitField)
     return Incompatible;
 
-  ConstructTransparentUnion(*this, Context, rExpr, ArgType, InitField);
+  ConstructTransparentUnion(*this, Context, RHS, ArgType, InitField);
   return Compatible;
 }
 
 Sema::AssignConvertType
-Sema::CheckSingleAssignmentConstraints(QualType lhsType, ExprResult &rExpr) {
+Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &RHS,
+                                       bool Diagnose) {
   if (getLangOptions().CPlusPlus) {
-    if (!lhsType->isRecordType()) {
+    if (!LHSType->isRecordType() && !LHSType->isAtomicType()) {
       // C++ 5.17p3: If the left operand is not of class type, the
       // expression is implicitly converted (C++ 4) to the
       // cv-unqualified type of the left operand.
-      ExprResult Res = PerformImplicitConversion(rExpr.get(),
-                                                 lhsType.getUnqualifiedType(),
-                                                 AA_Assigning);
+      ExprResult Res = PerformImplicitConversion(RHS.get(),
+                                                 LHSType.getUnqualifiedType(),
+                                                 AA_Assigning, Diagnose);
       if (Res.isInvalid())
         return Incompatible;
       Sema::AssignConvertType result = Compatible;
       if (getLangOptions().ObjCAutoRefCount &&
-          !CheckObjCARCUnavailableWeakConversion(lhsType, rExpr.get()->getType()))
+          !CheckObjCARCUnavailableWeakConversion(LHSType,
+                                                 RHS.get()->getType()))
         result = IncompatibleObjCWeakRef;
-      rExpr = move(Res);
+      RHS = move(Res);
       return result;
     }
 
     // FIXME: Currently, we fall through and treat C++ classes like C
     // structures.
-  }  
+    // FIXME: We also fall through for atomics; not sure what should
+    // happen there, though.
+  }
 
   // C99 6.5.16.1p1: the left operand is a pointer and the right is
   // a null pointer constant.
-  if ((lhsType->isPointerType() ||
-       lhsType->isObjCObjectPointerType() ||
-       lhsType->isBlockPointerType())
-      && rExpr.get()->isNullPointerConstant(Context,
-                                      Expr::NPC_ValueDependentIsNull)) {
-    rExpr = ImpCastExprToType(rExpr.take(), lhsType, CK_NullToPointer);
+  if ((LHSType->isPointerType() ||
+       LHSType->isObjCObjectPointerType() ||
+       LHSType->isBlockPointerType())
+      && RHS.get()->isNullPointerConstant(Context,
+                                          Expr::NPC_ValueDependentIsNull)) {
+    RHS = ImpCastExprToType(RHS.take(), LHSType, CK_NullToPointer);
     return Compatible;
   }
 
@@ -5483,15 +5577,15 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, ExprResult &rExpr) {
   // expressions that suppress this implicit conversion (&, sizeof).
   //
   // Suppress this for references: C++ 8.5.3p5.
-  if (!lhsType->isReferenceType()) {
-    rExpr = DefaultFunctionArrayLvalueConversion(rExpr.take());
-    if (rExpr.isInvalid())
+  if (!LHSType->isReferenceType()) {
+    RHS = DefaultFunctionArrayLvalueConversion(RHS.take());
+    if (RHS.isInvalid())
       return Incompatible;
   }
 
   CastKind Kind = CK_Invalid;
   Sema::AssignConvertType result =
-    CheckAssignmentConstraints(lhsType, rExpr, Kind);
+    CheckAssignmentConstraints(LHSType, RHS, Kind);
 
   // C99 6.5.16.1p2: The value of the right operand is converted to the
   // type of the assignment expression.
@@ -5499,150 +5593,202 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, ExprResult &rExpr) {
   // so that we can use references in built-in functions even in C.
   // The getNonReferenceType() call makes sure that the resulting expression
   // does not have reference type.
-  if (result != Incompatible && rExpr.get()->getType() != lhsType)
-    rExpr = ImpCastExprToType(rExpr.take(), lhsType.getNonLValueExprType(Context), Kind);
+  if (result != Incompatible && RHS.get()->getType() != LHSType)
+    RHS = ImpCastExprToType(RHS.take(),
+                            LHSType.getNonLValueExprType(Context), Kind);
   return result;
 }
 
-QualType Sema::InvalidOperands(SourceLocation Loc, ExprResult &lex, ExprResult &rex) {
+QualType Sema::InvalidOperands(SourceLocation Loc, ExprResult &LHS,
+                               ExprResult &RHS) {
   Diag(Loc, diag::err_typecheck_invalid_operands)
-    << lex.get()->getType() << rex.get()->getType()
-    << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+    << LHS.get()->getType() << RHS.get()->getType()
+    << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
   return QualType();
 }
 
-QualType Sema::CheckVectorOperands(ExprResult &lex, ExprResult &rex,
-                                   SourceLocation Loc, bool isCompAssign) {
+QualType Sema::CheckVectorOperands(ExprResult &LHS, ExprResult &RHS,
+                                   SourceLocation Loc, bool IsCompAssign) {
   // For conversion purposes, we ignore any qualifiers.
   // For example, "const float" and "float" are equivalent.
-  QualType lhsType =
-    Context.getCanonicalType(lex.get()->getType()).getUnqualifiedType();
-  QualType rhsType =
-    Context.getCanonicalType(rex.get()->getType()).getUnqualifiedType();
+  QualType LHSType =
+    Context.getCanonicalType(LHS.get()->getType()).getUnqualifiedType();
+  QualType RHSType =
+    Context.getCanonicalType(RHS.get()->getType()).getUnqualifiedType();
 
   // If the vector types are identical, return.
-  if (lhsType == rhsType)
-    return lhsType;
+  if (LHSType == RHSType)
+    return LHSType;
 
   // Handle the case of equivalent AltiVec and GCC vector types
-  if (lhsType->isVectorType() && rhsType->isVectorType() &&
-      Context.areCompatibleVectorTypes(lhsType, rhsType)) {
-    if (lhsType->isExtVectorType()) {
-      rex = ImpCastExprToType(rex.take(), lhsType, CK_BitCast);
-      return lhsType;
+  if (LHSType->isVectorType() && RHSType->isVectorType() &&
+      Context.areCompatibleVectorTypes(LHSType, RHSType)) {
+    if (LHSType->isExtVectorType()) {
+      RHS = ImpCastExprToType(RHS.take(), LHSType, CK_BitCast);
+      return LHSType;
     }
 
-    if (!isCompAssign)
-      lex = ImpCastExprToType(lex.take(), rhsType, CK_BitCast);
-    return rhsType;
+    if (!IsCompAssign)
+      LHS = ImpCastExprToType(LHS.take(), RHSType, CK_BitCast);
+    return RHSType;
   }
 
   if (getLangOptions().LaxVectorConversions &&
-      Context.getTypeSize(lhsType) == Context.getTypeSize(rhsType)) {
+      Context.getTypeSize(LHSType) == Context.getTypeSize(RHSType)) {
     // If we are allowing lax vector conversions, and LHS and RHS are both
     // vectors, the total size only needs to be the same. This is a
     // bitcast; no bits are changed but the result type is different.
     // FIXME: Should we really be allowing this?
-    rex = ImpCastExprToType(rex.take(), lhsType, CK_BitCast);
-    return lhsType;
+    RHS = ImpCastExprToType(RHS.take(), LHSType, CK_BitCast);
+    return LHSType;
   }
 
   // Canonicalize the ExtVector to the LHS, remember if we swapped so we can
   // swap back (so that we don't reverse the inputs to a subtract, for instance.
   bool swapped = false;
-  if (rhsType->isExtVectorType() && !isCompAssign) {
+  if (RHSType->isExtVectorType() && !IsCompAssign) {
     swapped = true;
-    std::swap(rex, lex);
-    std::swap(rhsType, lhsType);
+    std::swap(RHS, LHS);
+    std::swap(RHSType, LHSType);
   }
 
   // Handle the case of an ext vector and scalar.
-  if (const ExtVectorType *LV = lhsType->getAs<ExtVectorType>()) {
+  if (const ExtVectorType *LV = LHSType->getAs<ExtVectorType>()) {
     QualType EltTy = LV->getElementType();
-    if (EltTy->isIntegralType(Context) && rhsType->isIntegralType(Context)) {
-      int order = Context.getIntegerTypeOrder(EltTy, rhsType);
+    if (EltTy->isIntegralType(Context) && RHSType->isIntegralType(Context)) {
+      int order = Context.getIntegerTypeOrder(EltTy, RHSType);
       if (order > 0)
-        rex = ImpCastExprToType(rex.take(), EltTy, CK_IntegralCast);
+        RHS = ImpCastExprToType(RHS.take(), EltTy, CK_IntegralCast);
       if (order >= 0) {
-        rex = ImpCastExprToType(rex.take(), lhsType, CK_VectorSplat);
-        if (swapped) std::swap(rex, lex);
-        return lhsType;
+        RHS = ImpCastExprToType(RHS.take(), LHSType, CK_VectorSplat);
+        if (swapped) std::swap(RHS, LHS);
+        return LHSType;
       }
     }
-    if (EltTy->isRealFloatingType() && rhsType->isScalarType() &&
-        rhsType->isRealFloatingType()) {
-      int order = Context.getFloatingTypeOrder(EltTy, rhsType);
+    if (EltTy->isRealFloatingType() && RHSType->isScalarType() &&
+        RHSType->isRealFloatingType()) {
+      int order = Context.getFloatingTypeOrder(EltTy, RHSType);
       if (order > 0)
-        rex = ImpCastExprToType(rex.take(), EltTy, CK_FloatingCast);
+        RHS = ImpCastExprToType(RHS.take(), EltTy, CK_FloatingCast);
       if (order >= 0) {
-        rex = ImpCastExprToType(rex.take(), lhsType, CK_VectorSplat);
-        if (swapped) std::swap(rex, lex);
-        return lhsType;
+        RHS = ImpCastExprToType(RHS.take(), LHSType, CK_VectorSplat);
+        if (swapped) std::swap(RHS, LHS);
+        return LHSType;
       }
     }
   }
 
   // Vectors of different size or scalar and non-ext-vector are errors.
-  if (swapped) std::swap(rex, lex);
+  if (swapped) std::swap(RHS, LHS);
   Diag(Loc, diag::err_typecheck_vector_not_convertable)
-    << lex.get()->getType() << rex.get()->getType()
-    << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+    << LHS.get()->getType() << RHS.get()->getType()
+    << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
   return QualType();
 }
 
-QualType Sema::CheckMultiplyDivideOperands(
-  ExprResult &lex, ExprResult &rex, SourceLocation Loc, bool isCompAssign, bool isDiv) {
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType())
-    return CheckVectorOperands(lex, rex, Loc, isCompAssign);
+// checkArithmeticNull - Detect when a NULL constant is used improperly in an
+// expression.  These are mainly cases where the null pointer is used as an
+// integer instead of a pointer.
+static void checkArithmeticNull(Sema &S, ExprResult &LHS, ExprResult &RHS,
+                                SourceLocation Loc, bool IsCompare) {
+  // The canonical way to check for a GNU null is with isNullPointerConstant,
+  // but we use a bit of a hack here for speed; this is a relatively
+  // hot path, and isNullPointerConstant is slow.
+  bool LHSNull = isa<GNUNullExpr>(LHS.get()->IgnoreParenImpCasts());
+  bool RHSNull = isa<GNUNullExpr>(RHS.get()->IgnoreParenImpCasts());
 
-  QualType compType = UsualArithmeticConversions(lex, rex, isCompAssign);
-  if (lex.isInvalid() || rex.isInvalid())
+  QualType NonNullType = LHSNull ? RHS.get()->getType() : LHS.get()->getType();
+
+  // Avoid analyzing cases where the result will either be invalid (and
+  // diagnosed as such) or entirely valid and not something to warn about.
+  if ((!LHSNull && !RHSNull) || NonNullType->isBlockPointerType() ||
+      NonNullType->isMemberPointerType() || NonNullType->isFunctionType())
+    return;
+
+  // Comparison operations would not make sense with a null pointer no matter
+  // what the other expression is.
+  if (!IsCompare) {
+    S.Diag(Loc, diag::warn_null_in_arithmetic_operation)
+        << (LHSNull ? LHS.get()->getSourceRange() : SourceRange())
+        << (RHSNull ? RHS.get()->getSourceRange() : SourceRange());
+    return;
+  }
+
+  // The rest of the operations only make sense with a null pointer
+  // if the other expression is a pointer.
+  if (LHSNull == RHSNull || NonNullType->isAnyPointerType() ||
+      NonNullType->canDecayToPointerType())
+    return;
+
+  S.Diag(Loc, diag::warn_null_in_comparison_operation)
+      << LHSNull /* LHS is NULL */ << NonNullType
+      << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
+}
+
+QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
+                                           SourceLocation Loc,
+                                           bool IsCompAssign, bool IsDiv) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/false);
+
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType())
+    return CheckVectorOperands(LHS, RHS, Loc, IsCompAssign);
+
+  QualType compType = UsualArithmeticConversions(LHS, RHS, IsCompAssign);
+  if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
 
-  if (!lex.get()->getType()->isArithmeticType() ||
-      !rex.get()->getType()->isArithmeticType())
-    return InvalidOperands(Loc, lex, rex);
+  if (!LHS.get()->getType()->isArithmeticType() ||
+      !RHS.get()->getType()->isArithmeticType())
+    return InvalidOperands(Loc, LHS, RHS);
 
   // Check for division by zero.
-  if (isDiv &&
-      rex.get()->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNotNull))
-    DiagRuntimeBehavior(Loc, rex.get(), PDiag(diag::warn_division_by_zero)
-                                     << rex.get()->getSourceRange());
+  if (IsDiv &&
+      RHS.get()->isNullPointerConstant(Context,
+                                       Expr::NPC_ValueDependentIsNotNull))
+    DiagRuntimeBehavior(Loc, RHS.get(), PDiag(diag::warn_division_by_zero)
+                                          << RHS.get()->getSourceRange());
 
   return compType;
 }
 
 QualType Sema::CheckRemainderOperands(
-  ExprResult &lex, ExprResult &rex, SourceLocation Loc, bool isCompAssign) {
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType()) {
-    if (lex.get()->getType()->hasIntegerRepresentation() && 
-        rex.get()->getType()->hasIntegerRepresentation())
-      return CheckVectorOperands(lex, rex, Loc, isCompAssign);
-    return InvalidOperands(Loc, lex, rex);
+  ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, bool IsCompAssign) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/false);
+
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType()) {
+    if (LHS.get()->getType()->hasIntegerRepresentation() && 
+        RHS.get()->getType()->hasIntegerRepresentation())
+      return CheckVectorOperands(LHS, RHS, Loc, IsCompAssign);
+    return InvalidOperands(Loc, LHS, RHS);
   }
 
-  QualType compType = UsualArithmeticConversions(lex, rex, isCompAssign);
-  if (lex.isInvalid() || rex.isInvalid())
+  QualType compType = UsualArithmeticConversions(LHS, RHS, IsCompAssign);
+  if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
 
-  if (!lex.get()->getType()->isIntegerType() || !rex.get()->getType()->isIntegerType())
-    return InvalidOperands(Loc, lex, rex);
+  if (!LHS.get()->getType()->isIntegerType() ||
+      !RHS.get()->getType()->isIntegerType())
+    return InvalidOperands(Loc, LHS, RHS);
 
   // Check for remainder by zero.
-  if (rex.get()->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNotNull))
-    DiagRuntimeBehavior(Loc, rex.get(), PDiag(diag::warn_remainder_by_zero)
-                                 << rex.get()->getSourceRange());
+  if (RHS.get()->isNullPointerConstant(Context,
+                                       Expr::NPC_ValueDependentIsNotNull))
+    DiagRuntimeBehavior(Loc, RHS.get(), PDiag(diag::warn_remainder_by_zero)
+                                 << RHS.get()->getSourceRange());
 
   return compType;
 }
 
 /// \brief Diagnose invalid arithmetic on two void pointers.
 static void diagnoseArithmeticOnTwoVoidPointers(Sema &S, SourceLocation Loc,
-                                                Expr *LHS, Expr *RHS) {
+                                                Expr *LHSExpr, Expr *RHSExpr) {
   S.Diag(Loc, S.getLangOptions().CPlusPlus
                 ? diag::err_typecheck_pointer_arith_void_type
                 : diag::ext_gnu_void_ptr)
-    << 1 /* two pointers */ << LHS->getSourceRange() << RHS->getSourceRange();
+    << 1 /* two pointers */ << LHSExpr->getSourceRange()
+                            << RHSExpr->getSourceRange();
 }
 
 /// \brief Diagnose invalid arithmetic on a void pointer.
@@ -5682,6 +5828,24 @@ static void diagnoseArithmeticOnFunctionPointer(Sema &S, SourceLocation Loc,
     << Pointer->getSourceRange();
 }
 
+/// \brief Emit error if Operand is incomplete pointer type
+///
+/// \returns True if pointer has incomplete type
+static bool checkArithmeticIncompletePointerType(Sema &S, SourceLocation Loc,
+                                                 Expr *Operand) {
+  if ((Operand->getType()->isPointerType() &&
+       !Operand->getType()->isDependentType()) ||
+      Operand->getType()->isObjCObjectPointerType()) {
+    QualType PointeeTy = Operand->getType()->getPointeeType();
+    if (S.RequireCompleteType(
+          Loc, PointeeTy,
+          S.PDiag(diag::err_typecheck_arithmetic_incomplete_type)
+            << PointeeTy << Operand->getSourceRange()))
+      return true;
+  }
+  return false;
+}
+
 /// \brief Check the validity of an arithmetic pointer operand.
 ///
 /// If the operand has pointer type, this code will check for pointer types
@@ -5704,16 +5868,7 @@ static bool checkArithmeticOpPointerOperand(Sema &S, SourceLocation Loc,
     return !S.getLangOptions().CPlusPlus;
   }
 
-  if ((Operand->getType()->isPointerType() &&
-       !Operand->getType()->isDependentType()) ||
-      Operand->getType()->isObjCObjectPointerType()) {
-    QualType PointeeTy = Operand->getType()->getPointeeType();
-    if (S.RequireCompleteType(
-          Loc, PointeeTy,
-          S.PDiag(diag::err_typecheck_arithmetic_incomplete_type)
-            << PointeeTy << Operand->getSourceRange()))
-      return false;
-  }
+  if (checkArithmeticIncompletePointerType(S, Loc, Operand)) return false;
 
   return true;
 }
@@ -5728,22 +5883,22 @@ static bool checkArithmeticOpPointerOperand(Sema &S, SourceLocation Loc,
 ///
 /// \returns True when the operand is valid to use (even if as an extension).
 static bool checkArithmeticBinOpPointerOperands(Sema &S, SourceLocation Loc,
-                                                Expr *LHS, Expr *RHS) {
-  bool isLHSPointer = LHS->getType()->isAnyPointerType();
-  bool isRHSPointer = RHS->getType()->isAnyPointerType();
+                                                Expr *LHSExpr, Expr *RHSExpr) {
+  bool isLHSPointer = LHSExpr->getType()->isAnyPointerType();
+  bool isRHSPointer = RHSExpr->getType()->isAnyPointerType();
   if (!isLHSPointer && !isRHSPointer) return true;
 
   QualType LHSPointeeTy, RHSPointeeTy;
-  if (isLHSPointer) LHSPointeeTy = LHS->getType()->getPointeeType();
-  if (isRHSPointer) RHSPointeeTy = RHS->getType()->getPointeeType();
+  if (isLHSPointer) LHSPointeeTy = LHSExpr->getType()->getPointeeType();
+  if (isRHSPointer) RHSPointeeTy = RHSExpr->getType()->getPointeeType();
 
   // Check for arithmetic on pointers to incomplete types.
   bool isLHSVoidPtr = isLHSPointer && LHSPointeeTy->isVoidType();
   bool isRHSVoidPtr = isRHSPointer && RHSPointeeTy->isVoidType();
   if (isLHSVoidPtr || isRHSVoidPtr) {
-    if (!isRHSVoidPtr) diagnoseArithmeticOnVoidPointer(S, Loc, LHS);
-    else if (!isLHSVoidPtr) diagnoseArithmeticOnVoidPointer(S, Loc, RHS);
-    else diagnoseArithmeticOnTwoVoidPointers(S, Loc, LHS, RHS);
+    if (!isRHSVoidPtr) diagnoseArithmeticOnVoidPointer(S, Loc, LHSExpr);
+    else if (!isLHSVoidPtr) diagnoseArithmeticOnVoidPointer(S, Loc, RHSExpr);
+    else diagnoseArithmeticOnTwoVoidPointers(S, Loc, LHSExpr, RHSExpr);
 
     return !S.getLangOptions().CPlusPlus;
   }
@@ -5751,160 +5906,179 @@ static bool checkArithmeticBinOpPointerOperands(Sema &S, SourceLocation Loc,
   bool isLHSFuncPtr = isLHSPointer && LHSPointeeTy->isFunctionType();
   bool isRHSFuncPtr = isRHSPointer && RHSPointeeTy->isFunctionType();
   if (isLHSFuncPtr || isRHSFuncPtr) {
-    if (!isRHSFuncPtr) diagnoseArithmeticOnFunctionPointer(S, Loc, LHS);
-    else if (!isLHSFuncPtr) diagnoseArithmeticOnFunctionPointer(S, Loc, RHS);
-    else diagnoseArithmeticOnTwoFunctionPointers(S, Loc, LHS, RHS);
+    if (!isRHSFuncPtr) diagnoseArithmeticOnFunctionPointer(S, Loc, LHSExpr);
+    else if (!isLHSFuncPtr) diagnoseArithmeticOnFunctionPointer(S, Loc,
+                                                                RHSExpr);
+    else diagnoseArithmeticOnTwoFunctionPointers(S, Loc, LHSExpr, RHSExpr);
 
     return !S.getLangOptions().CPlusPlus;
   }
 
-  Expr *Operands[] = { LHS, RHS };
-  for (unsigned i = 0; i < 2; ++i) {
-    Expr *Operand = Operands[i];
-    if ((Operand->getType()->isPointerType() &&
-         !Operand->getType()->isDependentType()) ||
-        Operand->getType()->isObjCObjectPointerType()) {
-      QualType PointeeTy = Operand->getType()->getPointeeType();
-      if (S.RequireCompleteType(
-            Loc, PointeeTy,
-            S.PDiag(diag::err_typecheck_arithmetic_incomplete_type)
-              << PointeeTy << Operand->getSourceRange()))
-        return false;
-    }
-  }
+  if (checkArithmeticIncompletePointerType(S, Loc, LHSExpr)) return false;
+  if (checkArithmeticIncompletePointerType(S, Loc, RHSExpr)) return false;
+
   return true;
 }
 
+/// \brief Check bad cases where we step over interface counts.
+static bool checkArithmethicPointerOnNonFragileABI(Sema &S,
+                                                   SourceLocation OpLoc,
+                                                   Expr *Op) {
+  assert(Op->getType()->isAnyPointerType());
+  QualType PointeeTy = Op->getType()->getPointeeType();
+  if (!PointeeTy->isObjCObjectType() || !S.LangOpts.ObjCNonFragileABI)
+    return true;
+
+  S.Diag(OpLoc, diag::err_arithmetic_nonfragile_interface)
+    << PointeeTy << Op->getSourceRange();
+  return false;
+}
+
+/// \brief Emit error when two pointers are incompatible.
+static void diagnosePointerIncompatibility(Sema &S, SourceLocation Loc,
+                                           Expr *LHSExpr, Expr *RHSExpr) {
+  assert(LHSExpr->getType()->isAnyPointerType());
+  assert(RHSExpr->getType()->isAnyPointerType());
+  S.Diag(Loc, diag::err_typecheck_sub_ptr_compatible)
+    << LHSExpr->getType() << RHSExpr->getType() << LHSExpr->getSourceRange()
+    << RHSExpr->getSourceRange();
+}
+
 QualType Sema::CheckAdditionOperands( // C99 6.5.6
-  ExprResult &lex, ExprResult &rex, SourceLocation Loc, QualType* CompLHSTy) {
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType()) {
-    QualType compType = CheckVectorOperands(lex, rex, Loc, CompLHSTy);
+  ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, QualType* CompLHSTy) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/false);
+
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType()) {
+    QualType compType = CheckVectorOperands(LHS, RHS, Loc, CompLHSTy);
     if (CompLHSTy) *CompLHSTy = compType;
     return compType;
   }
 
-  QualType compType = UsualArithmeticConversions(lex, rex, CompLHSTy);
-  if (lex.isInvalid() || rex.isInvalid())
+  QualType compType = UsualArithmeticConversions(LHS, RHS, CompLHSTy);
+  if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
 
   // handle the common case first (both operands are arithmetic).
-  if (lex.get()->getType()->isArithmeticType() &&
-      rex.get()->getType()->isArithmeticType()) {
+  if (LHS.get()->getType()->isArithmeticType() &&
+      RHS.get()->getType()->isArithmeticType()) {
     if (CompLHSTy) *CompLHSTy = compType;
     return compType;
   }
 
   // Put any potential pointer into PExp
-  Expr* PExp = lex.get(), *IExp = rex.get();
+  Expr* PExp = LHS.get(), *IExp = RHS.get();
   if (IExp->getType()->isAnyPointerType())
     std::swap(PExp, IExp);
 
-  if (PExp->getType()->isAnyPointerType()) {
-    if (IExp->getType()->isIntegerType()) {
-      if (!checkArithmeticOpPointerOperand(*this, Loc, PExp))
-        return QualType();
+  if (!PExp->getType()->isAnyPointerType())
+    return InvalidOperands(Loc, LHS, RHS);
 
-      QualType PointeeTy = PExp->getType()->getPointeeType();
+  if (!IExp->getType()->isIntegerType())
+    return InvalidOperands(Loc, LHS, RHS);
 
-      // Diagnose bad cases where we step over interface counts.
-      if (PointeeTy->isObjCObjectType() && LangOpts.ObjCNonFragileABI) {
-        Diag(Loc, diag::err_arithmetic_nonfragile_interface)
-          << PointeeTy << PExp->getSourceRange();
-        return QualType();
-      }
+  if (!checkArithmeticOpPointerOperand(*this, Loc, PExp))
+    return QualType();
 
-      if (CompLHSTy) {
-        QualType LHSTy = Context.isPromotableBitField(lex.get());
-        if (LHSTy.isNull()) {
-          LHSTy = lex.get()->getType();
-          if (LHSTy->isPromotableIntegerType())
-            LHSTy = Context.getPromotedIntegerType(LHSTy);
-        }
-        *CompLHSTy = LHSTy;
-      }
-      return PExp->getType();
+  // Diagnose bad cases where we step over interface counts.
+  if (!checkArithmethicPointerOnNonFragileABI(*this, Loc, PExp))
+    return QualType();
+
+  // Check array bounds for pointer arithemtic
+  CheckArrayAccess(PExp, IExp);
+
+  if (CompLHSTy) {
+    QualType LHSTy = Context.isPromotableBitField(LHS.get());
+    if (LHSTy.isNull()) {
+      LHSTy = LHS.get()->getType();
+      if (LHSTy->isPromotableIntegerType())
+        LHSTy = Context.getPromotedIntegerType(LHSTy);
     }
+    *CompLHSTy = LHSTy;
   }
 
-  return InvalidOperands(Loc, lex, rex);
+  return PExp->getType();
 }
 
 // C99 6.5.6
-QualType Sema::CheckSubtractionOperands(ExprResult &lex, ExprResult &rex,
-                                        SourceLocation Loc, QualType* CompLHSTy) {
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType()) {
-    QualType compType = CheckVectorOperands(lex, rex, Loc, CompLHSTy);
+QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
+                                        SourceLocation Loc,
+                                        QualType* CompLHSTy) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/false);
+
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType()) {
+    QualType compType = CheckVectorOperands(LHS, RHS, Loc, CompLHSTy);
     if (CompLHSTy) *CompLHSTy = compType;
     return compType;
   }
 
-  QualType compType = UsualArithmeticConversions(lex, rex, CompLHSTy);
-  if (lex.isInvalid() || rex.isInvalid())
+  QualType compType = UsualArithmeticConversions(LHS, RHS, CompLHSTy);
+  if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
 
   // Enforce type constraints: C99 6.5.6p3.
 
   // Handle the common case first (both operands are arithmetic).
-  if (lex.get()->getType()->isArithmeticType() &&
-      rex.get()->getType()->isArithmeticType()) {
+  if (LHS.get()->getType()->isArithmeticType() &&
+      RHS.get()->getType()->isArithmeticType()) {
     if (CompLHSTy) *CompLHSTy = compType;
     return compType;
   }
 
   // Either ptr - int   or   ptr - ptr.
-  if (lex.get()->getType()->isAnyPointerType()) {
-    QualType lpointee = lex.get()->getType()->getPointeeType();
+  if (LHS.get()->getType()->isAnyPointerType()) {
+    QualType lpointee = LHS.get()->getType()->getPointeeType();
 
     // Diagnose bad cases where we step over interface counts.
-    if (lpointee->isObjCObjectType() && LangOpts.ObjCNonFragileABI) {
-      Diag(Loc, diag::err_arithmetic_nonfragile_interface)
-        << lpointee << lex.get()->getSourceRange();
+    if (!checkArithmethicPointerOnNonFragileABI(*this, Loc, LHS.get()))
       return QualType();
-    }
 
     // The result type of a pointer-int computation is the pointer type.
-    if (rex.get()->getType()->isIntegerType()) {
-      if (!checkArithmeticOpPointerOperand(*this, Loc, lex.get()))
+    if (RHS.get()->getType()->isIntegerType()) {
+      if (!checkArithmeticOpPointerOperand(*this, Loc, LHS.get()))
         return QualType();
 
-      if (CompLHSTy) *CompLHSTy = lex.get()->getType();
-      return lex.get()->getType();
+      Expr *IExpr = RHS.get()->IgnoreParenCasts();
+      UnaryOperator negRex(IExpr, UO_Minus, IExpr->getType(), VK_RValue,
+                           OK_Ordinary, IExpr->getExprLoc());
+      // Check array bounds for pointer arithemtic
+      CheckArrayAccess(LHS.get()->IgnoreParenCasts(), &negRex);
+
+      if (CompLHSTy) *CompLHSTy = LHS.get()->getType();
+      return LHS.get()->getType();
     }
 
     // Handle pointer-pointer subtractions.
-    if (const PointerType *RHSPTy = rex.get()->getType()->getAs<PointerType>()) {
+    if (const PointerType *RHSPTy
+          = RHS.get()->getType()->getAs<PointerType>()) {
       QualType rpointee = RHSPTy->getPointeeType();
 
       if (getLangOptions().CPlusPlus) {
         // Pointee types must be the same: C++ [expr.add]
         if (!Context.hasSameUnqualifiedType(lpointee, rpointee)) {
-          Diag(Loc, diag::err_typecheck_sub_ptr_compatible)
-            << lex.get()->getType() << rex.get()->getType()
-            << lex.get()->getSourceRange() << rex.get()->getSourceRange();
-          return QualType();
+          diagnosePointerIncompatibility(*this, Loc, LHS.get(), RHS.get());
         }
       } else {
         // Pointee types must be compatible C99 6.5.6p3
         if (!Context.typesAreCompatible(
                 Context.getCanonicalType(lpointee).getUnqualifiedType(),
                 Context.getCanonicalType(rpointee).getUnqualifiedType())) {
-          Diag(Loc, diag::err_typecheck_sub_ptr_compatible)
-            << lex.get()->getType() << rex.get()->getType()
-            << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+          diagnosePointerIncompatibility(*this, Loc, LHS.get(), RHS.get());
           return QualType();
         }
       }
 
       if (!checkArithmeticBinOpPointerOperands(*this, Loc,
-                                               lex.get(), rex.get()))
+                                               LHS.get(), RHS.get()))
         return QualType();
 
-      if (CompLHSTy) *CompLHSTy = lex.get()->getType();
+      if (CompLHSTy) *CompLHSTy = LHS.get()->getType();
       return Context.getPointerDiffType();
     }
   }
 
-  return InvalidOperands(Loc, lex, rex);
+  return InvalidOperands(Loc, LHS, RHS);
 }
 
 static bool isScopedEnumerationType(QualType T) {
@@ -5913,26 +6087,27 @@ static bool isScopedEnumerationType(QualType T) {
   return false;
 }
 
-static void DiagnoseBadShiftValues(Sema& S, ExprResult &lex, ExprResult &rex,
+static void DiagnoseBadShiftValues(Sema& S, ExprResult &LHS, ExprResult &RHS,
                                    SourceLocation Loc, unsigned Opc,
-                                   QualType LHSTy) {
+                                   QualType LHSType) {
   llvm::APSInt Right;
   // Check right/shifter operand
-  if (rex.get()->isValueDependent() || !rex.get()->isIntegerConstantExpr(Right, S.Context))
+  if (RHS.get()->isValueDependent() ||
+      !RHS.get()->isIntegerConstantExpr(Right, S.Context))
     return;
 
   if (Right.isNegative()) {
-    S.DiagRuntimeBehavior(Loc, rex.get(),
+    S.DiagRuntimeBehavior(Loc, RHS.get(),
                           S.PDiag(diag::warn_shift_negative)
-                            << rex.get()->getSourceRange());
+                            << RHS.get()->getSourceRange());
     return;
   }
   llvm::APInt LeftBits(Right.getBitWidth(),
-                       S.Context.getTypeSize(lex.get()->getType()));
+                       S.Context.getTypeSize(LHS.get()->getType()));
   if (Right.uge(LeftBits)) {
-    S.DiagRuntimeBehavior(Loc, rex.get(),
+    S.DiagRuntimeBehavior(Loc, RHS.get(),
                           S.PDiag(diag::warn_shift_gt_typewidth)
-                            << rex.get()->getSourceRange());
+                            << RHS.get()->getSourceRange());
     return;
   }
   if (Opc != BO_Shl)
@@ -5943,8 +6118,9 @@ static void DiagnoseBadShiftValues(Sema& S, ExprResult &lex, ExprResult &rex,
   // integers have defined behavior modulo one more than the maximum value
   // representable in the result type, so never warn for those.
   llvm::APSInt Left;
-  if (lex.get()->isValueDependent() || !lex.get()->isIntegerConstantExpr(Left, S.Context) ||
-      LHSTy->hasUnsignedIntegerRepresentation())
+  if (LHS.get()->isValueDependent() ||
+      !LHS.get()->isIntegerConstantExpr(Left, S.Context) ||
+      LHSType->hasUnsignedIntegerRepresentation())
     return;
   llvm::APInt ResultBits =
       static_cast<llvm::APInt&>(Right) + Left.getMinSignedBits();
@@ -5964,57 +6140,62 @@ static void DiagnoseBadShiftValues(Sema& S, ExprResult &lex, ExprResult &rex,
   // turned off separately if needed.
   if (LeftBits == ResultBits - 1) {
     S.Diag(Loc, diag::warn_shift_result_sets_sign_bit)
-        << HexResult.str() << LHSTy
-        << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+        << HexResult.str() << LHSType
+        << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
     return;
   }
 
   S.Diag(Loc, diag::warn_shift_result_gt_typewidth)
-    << HexResult.str() << Result.getMinSignedBits() << LHSTy
-    << Left.getBitWidth() << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+    << HexResult.str() << Result.getMinSignedBits() << LHSType
+    << Left.getBitWidth() << LHS.get()->getSourceRange()
+    << RHS.get()->getSourceRange();
 }
 
 // C99 6.5.7
-QualType Sema::CheckShiftOperands(ExprResult &lex, ExprResult &rex, SourceLocation Loc,
-                                  unsigned Opc, bool isCompAssign) {
+QualType Sema::CheckShiftOperands(ExprResult &LHS, ExprResult &RHS,
+                                  SourceLocation Loc, unsigned Opc,
+                                  bool IsCompAssign) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/false);
+
   // C99 6.5.7p2: Each of the operands shall have integer type.
-  if (!lex.get()->getType()->hasIntegerRepresentation() || 
-      !rex.get()->getType()->hasIntegerRepresentation())
-    return InvalidOperands(Loc, lex, rex);
+  if (!LHS.get()->getType()->hasIntegerRepresentation() || 
+      !RHS.get()->getType()->hasIntegerRepresentation())
+    return InvalidOperands(Loc, LHS, RHS);
 
   // C++0x: Don't allow scoped enums. FIXME: Use something better than
   // hasIntegerRepresentation() above instead of this.
-  if (isScopedEnumerationType(lex.get()->getType()) ||
-      isScopedEnumerationType(rex.get()->getType())) {
-    return InvalidOperands(Loc, lex, rex);
+  if (isScopedEnumerationType(LHS.get()->getType()) ||
+      isScopedEnumerationType(RHS.get()->getType())) {
+    return InvalidOperands(Loc, LHS, RHS);
   }
 
   // Vector shifts promote their scalar inputs to vector type.
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType())
-    return CheckVectorOperands(lex, rex, Loc, isCompAssign);
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType())
+    return CheckVectorOperands(LHS, RHS, Loc, IsCompAssign);
 
   // Shifts don't perform usual arithmetic conversions, they just do integer
   // promotions on each operand. C99 6.5.7p3
 
   // For the LHS, do usual unary conversions, but then reset them away
   // if this is a compound assignment.
-  ExprResult old_lex = lex;
-  lex = UsualUnaryConversions(lex.take());
-  if (lex.isInvalid())
+  ExprResult OldLHS = LHS;
+  LHS = UsualUnaryConversions(LHS.take());
+  if (LHS.isInvalid())
     return QualType();
-  QualType LHSTy = lex.get()->getType();
-  if (isCompAssign) lex = old_lex;
+  QualType LHSType = LHS.get()->getType();
+  if (IsCompAssign) LHS = OldLHS;
 
   // The RHS is simpler.
-  rex = UsualUnaryConversions(rex.take());
-  if (rex.isInvalid())
+  RHS = UsualUnaryConversions(RHS.take());
+  if (RHS.isInvalid())
     return QualType();
 
   // Sanity-check shift operands
-  DiagnoseBadShiftValues(*this, lex, rex, Loc, Opc, LHSTy);
+  DiagnoseBadShiftValues(*this, LHS, RHS, Loc, Opc, LHSType);
 
   // "The type of the result is that of the promoted left operand."
-  return LHSTy;
+  return LHSType;
 }
 
 static bool IsWithinTemplateSpecialization(Decl *D) {
@@ -6027,42 +6208,125 @@ static bool IsWithinTemplateSpecialization(Decl *D) {
   return false;
 }
 
+/// If two different enums are compared, raise a warning.
+static void checkEnumComparison(Sema &S, SourceLocation Loc, ExprResult &LHS,
+                                ExprResult &RHS) {
+  QualType LHSStrippedType = LHS.get()->IgnoreParenImpCasts()->getType();
+  QualType RHSStrippedType = RHS.get()->IgnoreParenImpCasts()->getType();
+
+  const EnumType *LHSEnumType = LHSStrippedType->getAs<EnumType>();
+  if (!LHSEnumType)
+    return;
+  const EnumType *RHSEnumType = RHSStrippedType->getAs<EnumType>();
+  if (!RHSEnumType)
+    return;
+
+  // Ignore anonymous enums.
+  if (!LHSEnumType->getDecl()->getIdentifier())
+    return;
+  if (!RHSEnumType->getDecl()->getIdentifier())
+    return;
+
+  if (S.Context.hasSameUnqualifiedType(LHSStrippedType, RHSStrippedType))
+    return;
+
+  S.Diag(Loc, diag::warn_comparison_of_mixed_enum_types)
+      << LHSStrippedType << RHSStrippedType
+      << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
+}
+
+/// \brief Diagnose bad pointer comparisons.
+static void diagnoseDistinctPointerComparison(Sema &S, SourceLocation Loc,
+                                              ExprResult &LHS, ExprResult &RHS,
+                                              bool IsError) {
+  S.Diag(Loc, IsError ? diag::err_typecheck_comparison_of_distinct_pointers
+                      : diag::ext_typecheck_comparison_of_distinct_pointers)
+    << LHS.get()->getType() << RHS.get()->getType()
+    << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
+}
+
+/// \brief Returns false if the pointers are converted to a composite type,
+/// true otherwise.
+static bool convertPointersToCompositeType(Sema &S, SourceLocation Loc,
+                                           ExprResult &LHS, ExprResult &RHS) {
+  // C++ [expr.rel]p2:
+  //   [...] Pointer conversions (4.10) and qualification
+  //   conversions (4.4) are performed on pointer operands (or on
+  //   a pointer operand and a null pointer constant) to bring
+  //   them to their composite pointer type. [...]
+  //
+  // C++ [expr.eq]p1 uses the same notion for (in)equality
+  // comparisons of pointers.
+
+  // C++ [expr.eq]p2:
+  //   In addition, pointers to members can be compared, or a pointer to
+  //   member and a null pointer constant. Pointer to member conversions
+  //   (4.11) and qualification conversions (4.4) are performed to bring
+  //   them to a common type. If one operand is a null pointer constant,
+  //   the common type is the type of the other operand. Otherwise, the
+  //   common type is a pointer to member type similar (4.4) to the type
+  //   of one of the operands, with a cv-qualification signature (4.4)
+  //   that is the union of the cv-qualification signatures of the operand
+  //   types.
+
+  QualType LHSType = LHS.get()->getType();
+  QualType RHSType = RHS.get()->getType();
+  assert((LHSType->isPointerType() && RHSType->isPointerType()) ||
+         (LHSType->isMemberPointerType() && RHSType->isMemberPointerType()));
+
+  bool NonStandardCompositeType = false;
+  bool *BoolPtr = S.isSFINAEContext() ? 0 : &NonStandardCompositeType;
+  QualType T = S.FindCompositePointerType(Loc, LHS, RHS, BoolPtr);
+  if (T.isNull()) {
+    diagnoseDistinctPointerComparison(S, Loc, LHS, RHS, /*isError*/true);
+    return true;
+  }
+
+  if (NonStandardCompositeType)
+    S.Diag(Loc, diag::ext_typecheck_comparison_of_distinct_pointers_nonstandard)
+      << LHSType << RHSType << T << LHS.get()->getSourceRange()
+      << RHS.get()->getSourceRange();
+
+  LHS = S.ImpCastExprToType(LHS.take(), T, CK_BitCast);
+  RHS = S.ImpCastExprToType(RHS.take(), T, CK_BitCast);
+  return false;
+}
+
+static void diagnoseFunctionPointerToVoidComparison(Sema &S, SourceLocation Loc,
+                                                    ExprResult &LHS,
+                                                    ExprResult &RHS,
+                                                    bool IsError) {
+  S.Diag(Loc, IsError ? diag::err_typecheck_comparison_of_fptr_to_void
+                      : diag::ext_typecheck_comparison_of_fptr_to_void)
+    << LHS.get()->getType() << RHS.get()->getType()
+    << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
+}
+
 // C99 6.5.8, C++ [expr.rel]
-QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLocation Loc,
-                                    unsigned OpaqueOpc, bool isRelational) {
+QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
+                                    SourceLocation Loc, unsigned OpaqueOpc,
+                                    bool IsRelational) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/true);
+
   BinaryOperatorKind Opc = (BinaryOperatorKind) OpaqueOpc;
 
   // Handle vector comparisons separately.
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType())
-    return CheckVectorCompareOperands(lex, rex, Loc, isRelational);
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType())
+    return CheckVectorCompareOperands(LHS, RHS, Loc, IsRelational);
 
-  QualType lType = lex.get()->getType();
-  QualType rType = rex.get()->getType();
- 
-  Expr *LHSStripped = lex.get()->IgnoreParenImpCasts();
-  Expr *RHSStripped = rex.get()->IgnoreParenImpCasts();
-  QualType LHSStrippedType = LHSStripped->getType();
-  QualType RHSStrippedType = RHSStripped->getType();
+  QualType LHSType = LHS.get()->getType();
+  QualType RHSType = RHS.get()->getType();
 
-  
+  Expr *LHSStripped = LHS.get()->IgnoreParenImpCasts();
+  Expr *RHSStripped = RHS.get()->IgnoreParenImpCasts();
 
-  // Two different enums will raise a warning when compared.
-  if (const EnumType *LHSEnumType = LHSStrippedType->getAs<EnumType>()) {
-    if (const EnumType *RHSEnumType = RHSStrippedType->getAs<EnumType>()) {
-      if (LHSEnumType->getDecl()->getIdentifier() &&
-          RHSEnumType->getDecl()->getIdentifier() &&
-          !Context.hasSameUnqualifiedType(LHSStrippedType, RHSStrippedType)) {
-        Diag(Loc, diag::warn_comparison_of_mixed_enum_types)
-          << LHSStrippedType << RHSStrippedType
-          << lex.get()->getSourceRange() << rex.get()->getSourceRange();
-      }
-    }
-  }
+  checkEnumComparison(*this, Loc, LHS, RHS);
 
-  if (!lType->hasFloatingRepresentation() &&
-      !(lType->isBlockPointerType() && isRelational) &&
-      !lex.get()->getLocStart().isMacroID() &&
-      !rex.get()->getLocStart().isMacroID()) {
+  if (!LHSType->hasFloatingRepresentation() &&
+      !(LHSType->isBlockPointerType() && IsRelational) &&
+      !LHS.get()->getLocStart().isMacroID() &&
+      !RHS.get()->getLocStart().isMacroID()) {
     // For non-floating point types, check for self-comparisons of the form
     // x == x, x != x, x < x, etc.  These always evaluate to a constant, and
     // often indicate logic errors in the program.
@@ -6082,7 +6346,7 @@ QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLoca
                               << (Opc == BO_EQ
                                   || Opc == BO_LE
                                   || Opc == BO_GE));
-        } else if (lType->isArrayType() && rType->isArrayType() &&
+        } else if (LHSType->isArrayType() && RHSType->isArrayType() &&
                    !DRL->getDecl()->getType()->isReferenceType() &&
                    !DRR->getDecl()->getType()->isReferenceType()) {
             // what is it always going to eval to?
@@ -6118,13 +6382,13 @@ QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLoca
     if ((isa<StringLiteral>(LHSStripped) || isa<ObjCEncodeExpr>(LHSStripped)) &&
         !RHSStripped->isNullPointerConstant(Context,
                                             Expr::NPC_ValueDependentIsNull)) {
-      literalString = lex.get();
+      literalString = LHS.get();
       literalStringStripped = LHSStripped;
     } else if ((isa<StringLiteral>(RHSStripped) ||
                 isa<ObjCEncodeExpr>(RHSStripped)) &&
                !LHSStripped->isNullPointerConstant(Context,
                                             Expr::NPC_ValueDependentIsNull)) {
-      literalString = rex.get();
+      literalString = RHS.get();
       literalStringStripped = RHSStripped;
     }
 
@@ -6137,7 +6401,7 @@ QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLoca
       case BO_GE: resultComparison = ") >= 0"; break;
       case BO_EQ: resultComparison = ") == 0"; break;
       case BO_NE: resultComparison = ") != 0"; break;
-      default: assert(false && "Invalid comparison operator");
+      default: llvm_unreachable("Invalid comparison operator");
       }
 
       DiagRuntimeBehavior(Loc, 0,
@@ -6148,56 +6412,57 @@ QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLoca
   }
 
   // C99 6.5.8p3 / C99 6.5.9p4
-  if (lex.get()->getType()->isArithmeticType() && rex.get()->getType()->isArithmeticType()) {
-    UsualArithmeticConversions(lex, rex);
-    if (lex.isInvalid() || rex.isInvalid())
+  if (LHS.get()->getType()->isArithmeticType() &&
+      RHS.get()->getType()->isArithmeticType()) {
+    UsualArithmeticConversions(LHS, RHS);
+    if (LHS.isInvalid() || RHS.isInvalid())
       return QualType();
   }
   else {
-    lex = UsualUnaryConversions(lex.take());
-    if (lex.isInvalid())
+    LHS = UsualUnaryConversions(LHS.take());
+    if (LHS.isInvalid())
       return QualType();
 
-    rex = UsualUnaryConversions(rex.take());
-    if (rex.isInvalid())
+    RHS = UsualUnaryConversions(RHS.take());
+    if (RHS.isInvalid())
       return QualType();
   }
 
-  lType = lex.get()->getType();
-  rType = rex.get()->getType();
+  LHSType = LHS.get()->getType();
+  RHSType = RHS.get()->getType();
 
   // The result of comparisons is 'bool' in C++, 'int' in C.
   QualType ResultTy = Context.getLogicalOperationType();
 
-  if (isRelational) {
-    if (lType->isRealType() && rType->isRealType())
+  if (IsRelational) {
+    if (LHSType->isRealType() && RHSType->isRealType())
       return ResultTy;
   } else {
     // Check for comparisons of floating point operands using != and ==.
-    if (lType->hasFloatingRepresentation())
-      CheckFloatComparison(Loc, lex.get(), rex.get());
+    if (LHSType->hasFloatingRepresentation())
+      CheckFloatComparison(Loc, LHS.get(), RHS.get());
 
-    if (lType->isArithmeticType() && rType->isArithmeticType())
+    if (LHSType->isArithmeticType() && RHSType->isArithmeticType())
       return ResultTy;
   }
 
-  bool LHSIsNull = lex.get()->isNullPointerConstant(Context,
+  bool LHSIsNull = LHS.get()->isNullPointerConstant(Context,
                                               Expr::NPC_ValueDependentIsNull);
-  bool RHSIsNull = rex.get()->isNullPointerConstant(Context,
+  bool RHSIsNull = RHS.get()->isNullPointerConstant(Context,
                                               Expr::NPC_ValueDependentIsNull);
 
   // All of the following pointer-related warnings are GCC extensions, except
   // when handling null pointer constants. 
-  if (lType->isPointerType() && rType->isPointerType()) { // C99 6.5.8p2
+  if (LHSType->isPointerType() && RHSType->isPointerType()) { // C99 6.5.8p2
     QualType LCanPointeeTy =
-      Context.getCanonicalType(lType->getAs<PointerType>()->getPointeeType());
+      LHSType->castAs<PointerType>()->getPointeeType().getCanonicalType();
     QualType RCanPointeeTy =
-      Context.getCanonicalType(rType->getAs<PointerType>()->getPointeeType());
+      RHSType->castAs<PointerType>()->getPointeeType().getCanonicalType();
 
     if (getLangOptions().CPlusPlus) {
       if (LCanPointeeTy == RCanPointeeTy)
         return ResultTy;
-      if (!isRelational &&
+      if (!IsRelational &&
           (LCanPointeeTy->isVoidType() || RCanPointeeTy->isVoidType())) {
         // Valid unless comparison between non-null pointer and function pointer
         // This is a gcc extension compatibility comparison.
@@ -6205,214 +6470,178 @@ QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLoca
         // conformance with the C++ standard.
         if ((LCanPointeeTy->isFunctionType() || RCanPointeeTy->isFunctionType())
             && !LHSIsNull && !RHSIsNull) {
-          Diag(Loc, 
-               isSFINAEContext()? 
-                   diag::err_typecheck_comparison_of_fptr_to_void
-                 : diag::ext_typecheck_comparison_of_fptr_to_void)
-            << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+          diagnoseFunctionPointerToVoidComparison(
+              *this, Loc, LHS, RHS, /*isError*/ isSFINAEContext());
           
           if (isSFINAEContext())
             return QualType();
           
-          rex = ImpCastExprToType(rex.take(), lType, CK_BitCast);
+          RHS = ImpCastExprToType(RHS.take(), LHSType, CK_BitCast);
           return ResultTy;
         }
       }
 
-      // C++ [expr.rel]p2:
-      //   [...] Pointer conversions (4.10) and qualification
-      //   conversions (4.4) are performed on pointer operands (or on
-      //   a pointer operand and a null pointer constant) to bring
-      //   them to their composite pointer type. [...]
-      //
-      // C++ [expr.eq]p1 uses the same notion for (in)equality
-      // comparisons of pointers.
-      bool NonStandardCompositeType = false;
-      QualType T = FindCompositePointerType(Loc, lex, rex,
-                              isSFINAEContext()? 0 : &NonStandardCompositeType);
-      if (T.isNull()) {
-        Diag(Loc, diag::err_typecheck_comparison_of_distinct_pointers)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+      if (convertPointersToCompositeType(*this, Loc, LHS, RHS))
         return QualType();
-      } else if (NonStandardCompositeType) {
-        Diag(Loc,
-             diag::ext_typecheck_comparison_of_distinct_pointers_nonstandard)
-          << lType << rType << T
-          << lex.get()->getSourceRange() << rex.get()->getSourceRange();
-      }
-
-      lex = ImpCastExprToType(lex.take(), T, CK_BitCast);
-      rex = ImpCastExprToType(rex.take(), T, CK_BitCast);
-      return ResultTy;
+      else
+        return ResultTy;
     }
     // C99 6.5.9p2 and C99 6.5.8p2
     if (Context.typesAreCompatible(LCanPointeeTy.getUnqualifiedType(),
                                    RCanPointeeTy.getUnqualifiedType())) {
       // Valid unless a relational comparison of function pointers
-      if (isRelational && LCanPointeeTy->isFunctionType()) {
+      if (IsRelational && LCanPointeeTy->isFunctionType()) {
         Diag(Loc, diag::ext_typecheck_ordered_comparison_of_function_pointers)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+          << LHSType << RHSType << LHS.get()->getSourceRange()
+          << RHS.get()->getSourceRange();
       }
-    } else if (!isRelational &&
+    } else if (!IsRelational &&
                (LCanPointeeTy->isVoidType() || RCanPointeeTy->isVoidType())) {
       // Valid unless comparison between non-null pointer and function pointer
       if ((LCanPointeeTy->isFunctionType() || RCanPointeeTy->isFunctionType())
-          && !LHSIsNull && !RHSIsNull) {
-        Diag(Loc, diag::ext_typecheck_comparison_of_fptr_to_void)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
-      }
+          && !LHSIsNull && !RHSIsNull)
+        diagnoseFunctionPointerToVoidComparison(*this, Loc, LHS, RHS,
+                                                /*isError*/false);
     } else {
       // Invalid
-      Diag(Loc, diag::ext_typecheck_comparison_of_distinct_pointers)
-        << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+      diagnoseDistinctPointerComparison(*this, Loc, LHS, RHS, /*isError*/false);
     }
     if (LCanPointeeTy != RCanPointeeTy) {
       if (LHSIsNull && !RHSIsNull)
-        lex = ImpCastExprToType(lex.take(), rType, CK_BitCast);
+        LHS = ImpCastExprToType(LHS.take(), RHSType, CK_BitCast);
       else
-        rex = ImpCastExprToType(rex.take(), lType, CK_BitCast);
+        RHS = ImpCastExprToType(RHS.take(), LHSType, CK_BitCast);
     }
     return ResultTy;
   }
 
   if (getLangOptions().CPlusPlus) {
     // Comparison of nullptr_t with itself.
-    if (lType->isNullPtrType() && rType->isNullPtrType())
+    if (LHSType->isNullPtrType() && RHSType->isNullPtrType())
       return ResultTy;
     
     // Comparison of pointers with null pointer constants and equality
     // comparisons of member pointers to null pointer constants.
     if (RHSIsNull &&
-        ((lType->isAnyPointerType() || lType->isNullPtrType()) ||
-         (!isRelational && 
-          (lType->isMemberPointerType() || lType->isBlockPointerType())))) {
-      rex = ImpCastExprToType(rex.take(), lType, 
-                        lType->isMemberPointerType()
+        ((LHSType->isAnyPointerType() || LHSType->isNullPtrType()) ||
+         (!IsRelational && 
+          (LHSType->isMemberPointerType() || LHSType->isBlockPointerType())))) {
+      RHS = ImpCastExprToType(RHS.take(), LHSType, 
+                        LHSType->isMemberPointerType()
                           ? CK_NullToMemberPointer
                           : CK_NullToPointer);
       return ResultTy;
     }
     if (LHSIsNull &&
-        ((rType->isAnyPointerType() || rType->isNullPtrType()) ||
-         (!isRelational && 
-          (rType->isMemberPointerType() || rType->isBlockPointerType())))) {
-      lex = ImpCastExprToType(lex.take(), rType, 
-                        rType->isMemberPointerType()
+        ((RHSType->isAnyPointerType() || RHSType->isNullPtrType()) ||
+         (!IsRelational && 
+          (RHSType->isMemberPointerType() || RHSType->isBlockPointerType())))) {
+      LHS = ImpCastExprToType(LHS.take(), RHSType, 
+                        RHSType->isMemberPointerType()
                           ? CK_NullToMemberPointer
                           : CK_NullToPointer);
       return ResultTy;
     }
 
     // Comparison of member pointers.
-    if (!isRelational &&
-        lType->isMemberPointerType() && rType->isMemberPointerType()) {
-      // C++ [expr.eq]p2:
-      //   In addition, pointers to members can be compared, or a pointer to
-      //   member and a null pointer constant. Pointer to member conversions
-      //   (4.11) and qualification conversions (4.4) are performed to bring
-      //   them to a common type. If one operand is a null pointer constant,
-      //   the common type is the type of the other operand. Otherwise, the
-      //   common type is a pointer to member type similar (4.4) to the type
-      //   of one of the operands, with a cv-qualification signature (4.4)
-      //   that is the union of the cv-qualification signatures of the operand
-      //   types.
-      bool NonStandardCompositeType = false;
-      QualType T = FindCompositePointerType(Loc, lex, rex,
-                              isSFINAEContext()? 0 : &NonStandardCompositeType);
-      if (T.isNull()) {
-        Diag(Loc, diag::err_typecheck_comparison_of_distinct_pointers)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+    if (!IsRelational &&
+        LHSType->isMemberPointerType() && RHSType->isMemberPointerType()) {
+      if (convertPointersToCompositeType(*this, Loc, LHS, RHS))
         return QualType();
-      } else if (NonStandardCompositeType) {
-        Diag(Loc,
-             diag::ext_typecheck_comparison_of_distinct_pointers_nonstandard)
-          << lType << rType << T
-          << lex.get()->getSourceRange() << rex.get()->getSourceRange();
-      }
-
-      lex = ImpCastExprToType(lex.take(), T, CK_BitCast);
-      rex = ImpCastExprToType(rex.take(), T, CK_BitCast);
-      return ResultTy;
+      else
+        return ResultTy;
     }
 
     // Handle scoped enumeration types specifically, since they don't promote
     // to integers.
-    if (lex.get()->getType()->isEnumeralType() &&
-        Context.hasSameUnqualifiedType(lex.get()->getType(), rex.get()->getType()))
+    if (LHS.get()->getType()->isEnumeralType() &&
+        Context.hasSameUnqualifiedType(LHS.get()->getType(),
+                                       RHS.get()->getType()))
       return ResultTy;
   }
 
   // Handle block pointer types.
-  if (!isRelational && lType->isBlockPointerType() && rType->isBlockPointerType()) {
-    QualType lpointee = lType->getAs<BlockPointerType>()->getPointeeType();
-    QualType rpointee = rType->getAs<BlockPointerType>()->getPointeeType();
+  if (!IsRelational && LHSType->isBlockPointerType() &&
+      RHSType->isBlockPointerType()) {
+    QualType lpointee = LHSType->castAs<BlockPointerType>()->getPointeeType();
+    QualType rpointee = RHSType->castAs<BlockPointerType>()->getPointeeType();
 
     if (!LHSIsNull && !RHSIsNull &&
         !Context.typesAreCompatible(lpointee, rpointee)) {
       Diag(Loc, diag::err_typecheck_comparison_of_distinct_blocks)
-        << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+        << LHSType << RHSType << LHS.get()->getSourceRange()
+        << RHS.get()->getSourceRange();
     }
-    rex = ImpCastExprToType(rex.take(), lType, CK_BitCast);
+    RHS = ImpCastExprToType(RHS.take(), LHSType, CK_BitCast);
     return ResultTy;
   }
 
   // Allow block pointers to be compared with null pointer constants.
-  if (!isRelational
-      && ((lType->isBlockPointerType() && rType->isPointerType())
-          || (lType->isPointerType() && rType->isBlockPointerType()))) {
+  if (!IsRelational
+      && ((LHSType->isBlockPointerType() && RHSType->isPointerType())
+          || (LHSType->isPointerType() && RHSType->isBlockPointerType()))) {
     if (!LHSIsNull && !RHSIsNull) {
-      if (!((rType->isPointerType() && rType->castAs<PointerType>()
+      if (!((RHSType->isPointerType() && RHSType->castAs<PointerType>()
              ->getPointeeType()->isVoidType())
-            || (lType->isPointerType() && lType->castAs<PointerType>()
+            || (LHSType->isPointerType() && LHSType->castAs<PointerType>()
                 ->getPointeeType()->isVoidType())))
         Diag(Loc, diag::err_typecheck_comparison_of_distinct_blocks)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+          << LHSType << RHSType << LHS.get()->getSourceRange()
+          << RHS.get()->getSourceRange();
     }
     if (LHSIsNull && !RHSIsNull)
-      lex = ImpCastExprToType(lex.take(), rType, CK_BitCast);
+      LHS = ImpCastExprToType(LHS.take(), RHSType,
+                              RHSType->isPointerType() ? CK_BitCast
+                                : CK_AnyPointerToBlockPointerCast);
     else
-      rex = ImpCastExprToType(rex.take(), lType, CK_BitCast);
+      RHS = ImpCastExprToType(RHS.take(), LHSType,
+                              LHSType->isPointerType() ? CK_BitCast
+                                : CK_AnyPointerToBlockPointerCast);
     return ResultTy;
   }
 
-  if (lType->isObjCObjectPointerType() || rType->isObjCObjectPointerType()) {
-    const PointerType *LPT = lType->getAs<PointerType>();
-    const PointerType *RPT = rType->getAs<PointerType>();
+  if (LHSType->isObjCObjectPointerType() ||
+      RHSType->isObjCObjectPointerType()) {
+    const PointerType *LPT = LHSType->getAs<PointerType>();
+    const PointerType *RPT = RHSType->getAs<PointerType>();
     if (LPT || RPT) {
       bool LPtrToVoid = LPT ? LPT->getPointeeType()->isVoidType() : false;
       bool RPtrToVoid = RPT ? RPT->getPointeeType()->isVoidType() : false;
 
       if (!LPtrToVoid && !RPtrToVoid &&
-          !Context.typesAreCompatible(lType, rType)) {
-        Diag(Loc, diag::ext_typecheck_comparison_of_distinct_pointers)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+          !Context.typesAreCompatible(LHSType, RHSType)) {
+        diagnoseDistinctPointerComparison(*this, Loc, LHS, RHS,
+                                          /*isError*/false);
       }
       if (LHSIsNull && !RHSIsNull)
-        lex = ImpCastExprToType(lex.take(), rType, CK_BitCast);
+        LHS = ImpCastExprToType(LHS.take(), RHSType,
+                                RPT ? CK_BitCast :CK_CPointerToObjCPointerCast);
       else
-        rex = ImpCastExprToType(rex.take(), lType, CK_BitCast);
+        RHS = ImpCastExprToType(RHS.take(), LHSType,
+                                LPT ? CK_BitCast :CK_CPointerToObjCPointerCast);
       return ResultTy;
     }
-    if (lType->isObjCObjectPointerType() && rType->isObjCObjectPointerType()) {
-      if (!Context.areComparableObjCPointerTypes(lType, rType))
-        Diag(Loc, diag::ext_typecheck_comparison_of_distinct_pointers)
-          << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+    if (LHSType->isObjCObjectPointerType() &&
+        RHSType->isObjCObjectPointerType()) {
+      if (!Context.areComparableObjCPointerTypes(LHSType, RHSType))
+        diagnoseDistinctPointerComparison(*this, Loc, LHS, RHS,
+                                          /*isError*/false);
       if (LHSIsNull && !RHSIsNull)
-        lex = ImpCastExprToType(lex.take(), rType, CK_BitCast);
+        LHS = ImpCastExprToType(LHS.take(), RHSType, CK_BitCast);
       else
-        rex = ImpCastExprToType(rex.take(), lType, CK_BitCast);
+        RHS = ImpCastExprToType(RHS.take(), LHSType, CK_BitCast);
       return ResultTy;
     }
   }
-  if ((lType->isAnyPointerType() && rType->isIntegerType()) ||
-      (lType->isIntegerType() && rType->isAnyPointerType())) {
+  if ((LHSType->isAnyPointerType() && RHSType->isIntegerType()) ||
+      (LHSType->isIntegerType() && RHSType->isAnyPointerType())) {
     unsigned DiagID = 0;
     bool isError = false;
-    if ((LHSIsNull && lType->isIntegerType()) ||
-        (RHSIsNull && rType->isIntegerType())) {
-      if (isRelational && !getLangOptions().CPlusPlus)
+    if ((LHSIsNull && LHSType->isIntegerType()) ||
+        (RHSIsNull && RHSType->isIntegerType())) {
+      if (IsRelational && !getLangOptions().CPlusPlus)
         DiagID = diag::ext_typecheck_ordered_comparison_of_pointer_and_zero;
-    } else if (isRelational && !getLangOptions().CPlusPlus)
+    } else if (IsRelational && !getLangOptions().CPlusPlus)
       DiagID = diag::ext_typecheck_ordered_comparison_of_pointer_integer;
     else if (getLangOptions().CPlusPlus) {
       DiagID = diag::err_typecheck_comparison_of_pointer_integer;
@@ -6422,50 +6651,51 @@ QualType Sema::CheckCompareOperands(ExprResult &lex, ExprResult &rex, SourceLoca
 
     if (DiagID) {
       Diag(Loc, DiagID)
-        << lType << rType << lex.get()->getSourceRange() << rex.get()->getSourceRange();
+        << LHSType << RHSType << LHS.get()->getSourceRange()
+        << RHS.get()->getSourceRange();
       if (isError)
         return QualType();
     }
     
-    if (lType->isIntegerType())
-      lex = ImpCastExprToType(lex.take(), rType,
+    if (LHSType->isIntegerType())
+      LHS = ImpCastExprToType(LHS.take(), RHSType,
                         LHSIsNull ? CK_NullToPointer : CK_IntegralToPointer);
     else
-      rex = ImpCastExprToType(rex.take(), lType,
+      RHS = ImpCastExprToType(RHS.take(), LHSType,
                         RHSIsNull ? CK_NullToPointer : CK_IntegralToPointer);
     return ResultTy;
   }
   
   // Handle block pointers.
-  if (!isRelational && RHSIsNull
-      && lType->isBlockPointerType() && rType->isIntegerType()) {
-    rex = ImpCastExprToType(rex.take(), lType, CK_NullToPointer);
+  if (!IsRelational && RHSIsNull
+      && LHSType->isBlockPointerType() && RHSType->isIntegerType()) {
+    RHS = ImpCastExprToType(RHS.take(), LHSType, CK_NullToPointer);
     return ResultTy;
   }
-  if (!isRelational && LHSIsNull
-      && lType->isIntegerType() && rType->isBlockPointerType()) {
-    lex = ImpCastExprToType(lex.take(), rType, CK_NullToPointer);
+  if (!IsRelational && LHSIsNull
+      && LHSType->isIntegerType() && RHSType->isBlockPointerType()) {
+    LHS = ImpCastExprToType(LHS.take(), RHSType, CK_NullToPointer);
     return ResultTy;
   }
 
-  return InvalidOperands(Loc, lex, rex);
+  return InvalidOperands(Loc, LHS, RHS);
 }
 
 /// CheckVectorCompareOperands - vector comparisons are a clang extension that
 /// operates on extended vector types.  Instead of producing an IntTy result,
 /// like a scalar comparison, a vector comparison produces a vector of integer
 /// types.
-QualType Sema::CheckVectorCompareOperands(ExprResult &lex, ExprResult &rex,
+QualType Sema::CheckVectorCompareOperands(ExprResult &LHS, ExprResult &RHS,
                                           SourceLocation Loc,
-                                          bool isRelational) {
+                                          bool IsRelational) {
   // Check to make sure we're operating on vectors of the same type and width,
   // Allowing one side to be a scalar of element type.
-  QualType vType = CheckVectorOperands(lex, rex, Loc, /*isCompAssign*/false);
+  QualType vType = CheckVectorOperands(LHS, RHS, Loc, /*isCompAssign*/false);
   if (vType.isNull())
     return vType;
 
-  QualType lType = lex.get()->getType();
-  QualType rType = rex.get()->getType();
+  QualType LHSType = LHS.get()->getType();
+  QualType RHSType = RHS.get()->getType();
 
   // If AltiVec, the comparison results in a numeric type, i.e.
   // bool for C++, int for C
@@ -6475,9 +6705,9 @@ QualType Sema::CheckVectorCompareOperands(ExprResult &lex, ExprResult &rex,
   // For non-floating point types, check for self-comparisons of the form
   // x == x, x != x, x < x, etc.  These always evaluate to a constant, and
   // often indicate logic errors in the program.
-  if (!lType->hasFloatingRepresentation()) {
-    if (DeclRefExpr* DRL = dyn_cast<DeclRefExpr>(lex.get()->IgnoreParens()))
-      if (DeclRefExpr* DRR = dyn_cast<DeclRefExpr>(rex.get()->IgnoreParens()))
+  if (!LHSType->hasFloatingRepresentation()) {
+    if (DeclRefExpr* DRL = dyn_cast<DeclRefExpr>(LHS.get()->IgnoreParens()))
+      if (DeclRefExpr* DRR = dyn_cast<DeclRefExpr>(RHS.get()->IgnoreParens()))
         if (DRL->getDecl() == DRR->getDecl())
           DiagRuntimeBehavior(Loc, 0,
                               PDiag(diag::warn_comparison_always)
@@ -6487,18 +6717,18 @@ QualType Sema::CheckVectorCompareOperands(ExprResult &lex, ExprResult &rex,
   }
 
   // Check for comparisons of floating point operands using != and ==.
-  if (!isRelational && lType->hasFloatingRepresentation()) {
-    assert (rType->hasFloatingRepresentation());
-    CheckFloatComparison(Loc, lex.get(), rex.get());
+  if (!IsRelational && LHSType->hasFloatingRepresentation()) {
+    assert (RHSType->hasFloatingRepresentation());
+    CheckFloatComparison(Loc, LHS.get(), RHS.get());
   }
 
   // Return the type for the comparison, which is the same as vector type for
   // integer vectors, or an integer type of identical size and number of
   // elements for floating point vectors.
-  if (lType->hasIntegerRepresentation())
-    return lType;
+  if (LHSType->hasIntegerRepresentation())
+    return LHSType;
 
-  const VectorType *VTy = lType->getAs<VectorType>();
+  const VectorType *VTy = LHSType->getAs<VectorType>();
   unsigned TypeSize = Context.getTypeSize(VTy->getElementType());
   if (TypeSize == Context.getTypeSize(Context.IntTy))
     return Context.getExtVectorType(Context.IntTy, VTy->getNumElements());
@@ -6511,36 +6741,41 @@ QualType Sema::CheckVectorCompareOperands(ExprResult &lex, ExprResult &rex,
 }
 
 inline QualType Sema::CheckBitwiseOperands(
-  ExprResult &lex, ExprResult &rex, SourceLocation Loc, bool isCompAssign) {
-  if (lex.get()->getType()->isVectorType() || rex.get()->getType()->isVectorType()) {
-    if (lex.get()->getType()->hasIntegerRepresentation() &&
-        rex.get()->getType()->hasIntegerRepresentation())
-      return CheckVectorOperands(lex, rex, Loc, isCompAssign);
+  ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, bool IsCompAssign) {
+  checkArithmeticNull(*this, LHS, RHS, Loc, /*isCompare=*/false);
+
+  if (LHS.get()->getType()->isVectorType() ||
+      RHS.get()->getType()->isVectorType()) {
+    if (LHS.get()->getType()->hasIntegerRepresentation() &&
+        RHS.get()->getType()->hasIntegerRepresentation())
+      return CheckVectorOperands(LHS, RHS, Loc, IsCompAssign);
     
-    return InvalidOperands(Loc, lex, rex);
+    return InvalidOperands(Loc, LHS, RHS);
   }
 
-  ExprResult lexResult = Owned(lex), rexResult = Owned(rex);
-  QualType compType = UsualArithmeticConversions(lexResult, rexResult, isCompAssign);
-  if (lexResult.isInvalid() || rexResult.isInvalid())
+  ExprResult LHSResult = Owned(LHS), RHSResult = Owned(RHS);
+  QualType compType = UsualArithmeticConversions(LHSResult, RHSResult,
+                                                 IsCompAssign);
+  if (LHSResult.isInvalid() || RHSResult.isInvalid())
     return QualType();
-  lex = lexResult.take();
-  rex = rexResult.take();
+  LHS = LHSResult.take();
+  RHS = RHSResult.take();
 
-  if (lex.get()->getType()->isIntegralOrUnscopedEnumerationType() &&
-      rex.get()->getType()->isIntegralOrUnscopedEnumerationType())
+  if (LHS.get()->getType()->isIntegralOrUnscopedEnumerationType() &&
+      RHS.get()->getType()->isIntegralOrUnscopedEnumerationType())
     return compType;
-  return InvalidOperands(Loc, lex, rex);
+  return InvalidOperands(Loc, LHS, RHS);
 }
 
 inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
-  ExprResult &lex, ExprResult &rex, SourceLocation Loc, unsigned Opc) {
+  ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, unsigned Opc) {
   
   // Diagnose cases where the user write a logical and/or but probably meant a
   // bitwise one.  We do this when the LHS is a non-bool integer and the RHS
   // is a constant.
-  if (lex.get()->getType()->isIntegerType() && !lex.get()->getType()->isBooleanType() &&
-      rex.get()->getType()->isIntegerType() && !rex.get()->isValueDependent() &&
+  if (LHS.get()->getType()->isIntegerType() &&
+      !LHS.get()->getType()->isBooleanType() &&
+      RHS.get()->getType()->isIntegerType() && !RHS.get()->isValueDependent() &&
       // Don't warn in macros or template instantiations.
       !Loc.isMacroID() && ActiveTemplateInstantiations.empty()) {
     // If the RHS can be constant folded, and if it constant folds to something
@@ -6548,27 +6783,43 @@ inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
     // happened to fold to true/false) then warn.
     // Parens on the RHS are ignored.
     Expr::EvalResult Result;
-    if (rex.get()->Evaluate(Result, Context) && !Result.HasSideEffects)
-      if ((getLangOptions().Bool && !rex.get()->getType()->isBooleanType()) ||
+    if (RHS.get()->Evaluate(Result, Context) && !Result.HasSideEffects)
+      if ((getLangOptions().Bool && !RHS.get()->getType()->isBooleanType()) ||
           (Result.Val.getInt() != 0 && Result.Val.getInt() != 1)) {
         Diag(Loc, diag::warn_logical_instead_of_bitwise)
-          << rex.get()->getSourceRange()
-          << (Opc == BO_LAnd ? "&&" : "||")
-          << (Opc == BO_LAnd ? "&" : "|");
-    }
+          << RHS.get()->getSourceRange()
+          << (Opc == BO_LAnd ? "&&" : "||");
+        // Suggest replacing the logical operator with the bitwise version
+        Diag(Loc, diag::note_logical_instead_of_bitwise_change_operator)
+            << (Opc == BO_LAnd ? "&" : "|")
+            << FixItHint::CreateReplacement(SourceRange(
+                Loc, Lexer::getLocForEndOfToken(Loc, 0, getSourceManager(),
+                                                getLangOptions())),
+                                            Opc == BO_LAnd ? "&" : "|");
+        if (Opc == BO_LAnd)
+          // Suggest replacing "Foo() && kNonZero" with "Foo()"
+          Diag(Loc, diag::note_logical_instead_of_bitwise_remove_constant)
+              << FixItHint::CreateRemoval(
+                  SourceRange(
+                      Lexer::getLocForEndOfToken(LHS.get()->getLocEnd(),
+                                                 0, getSourceManager(),
+                                                 getLangOptions()),
+                      RHS.get()->getLocEnd()));
+      }
   }
   
   if (!Context.getLangOptions().CPlusPlus) {
-    lex = UsualUnaryConversions(lex.take());
-    if (lex.isInvalid())
+    LHS = UsualUnaryConversions(LHS.take());
+    if (LHS.isInvalid())
       return QualType();
 
-    rex = UsualUnaryConversions(rex.take());
-    if (rex.isInvalid())
+    RHS = UsualUnaryConversions(RHS.take());
+    if (RHS.isInvalid())
       return QualType();
 
-    if (!lex.get()->getType()->isScalarType() || !rex.get()->getType()->isScalarType())
-      return InvalidOperands(Loc, lex, rex);
+    if (!LHS.get()->getType()->isScalarType() ||
+        !RHS.get()->getType()->isScalarType())
+      return InvalidOperands(Loc, LHS, RHS);
 
     return Context.IntTy;
   }
@@ -6579,15 +6830,15 @@ inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
   // C++ [expr.log.and]p1
   // C++ [expr.log.or]p1
   // The operands are both contextually converted to type bool.
-  ExprResult lexRes = PerformContextuallyConvertToBool(lex.get());
-  if (lexRes.isInvalid())
-    return InvalidOperands(Loc, lex, rex);
-  lex = move(lexRes);
+  ExprResult LHSRes = PerformContextuallyConvertToBool(LHS.get());
+  if (LHSRes.isInvalid())
+    return InvalidOperands(Loc, LHS, RHS);
+  LHS = move(LHSRes);
 
-  ExprResult rexRes = PerformContextuallyConvertToBool(rex.get());
-  if (rexRes.isInvalid())
-    return InvalidOperands(Loc, lex, rex);
-  rex = move(rexRes);
+  ExprResult RHSRes = PerformContextuallyConvertToBool(RHS.get());
+  if (RHSRes.isInvalid())
+    return InvalidOperands(Loc, LHS, RHS);
+  RHS = move(RHSRes);
 
   // C++ [expr.log.and]p2
   // C++ [expr.log.or]p2
@@ -6758,25 +7009,26 @@ static bool CheckForModifiableLvalue(Expr *E, SourceLocation Loc, Sema &S) {
 
 
 // C99 6.5.16.1
-QualType Sema::CheckAssignmentOperands(Expr *LHS, ExprResult &RHS,
+QualType Sema::CheckAssignmentOperands(Expr *LHSExpr, ExprResult &RHS,
                                        SourceLocation Loc,
                                        QualType CompoundType) {
   // Verify that LHS is a modifiable lvalue, and emit error if not.
-  if (CheckForModifiableLvalue(LHS, Loc, *this))
+  if (CheckForModifiableLvalue(LHSExpr, Loc, *this))
     return QualType();
 
-  QualType LHSType = LHS->getType();
-  QualType RHSType = CompoundType.isNull() ? RHS.get()->getType() : CompoundType;
+  QualType LHSType = LHSExpr->getType();
+  QualType RHSType = CompoundType.isNull() ? RHS.get()->getType() :
+                                             CompoundType;
   AssignConvertType ConvTy;
   if (CompoundType.isNull()) {
     QualType LHSTy(LHSType);
     // Simple assignment "x = y".
-    if (LHS->getObjectKind() == OK_ObjCProperty) {
-      ExprResult LHSResult = Owned(LHS);
+    if (LHSExpr->getObjectKind() == OK_ObjCProperty) {
+      ExprResult LHSResult = Owned(LHSExpr);
       ConvertPropertyForLValue(LHSResult, RHS, LHSTy);
       if (LHSResult.isInvalid())
         return QualType();
-      LHS = LHSResult.take();
+      LHSExpr = LHSResult.take();
     }
     ConvTy = CheckSingleAssignmentConstraints(LHSTy, RHS);
     if (RHS.isInvalid())
@@ -6806,10 +7058,10 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, ExprResult &RHS,
            UO->getOpcode() == UO_Minus) &&
           Loc.isFileID() && UO->getOperatorLoc().isFileID() &&
           // Only if the two operators are exactly adjacent.
-          Loc.getFileLocWithOffset(1) == UO->getOperatorLoc() &&
+          Loc.getLocWithOffset(1) == UO->getOperatorLoc() &&
           // And there is a space or other character before the subexpr of the
           // unary +/-.  We don't want to warn on "x=-1".
-          Loc.getFileLocWithOffset(2) != UO->getSubExpr()->getLocStart() &&
+          Loc.getLocWithOffset(2) != UO->getSubExpr()->getLocStart() &&
           UO->getSubExpr()->getLocStart().isFileID()) {
         Diag(Loc, diag::warn_not_compound_assign)
           << (UO->getOpcode() == UO_Plus ? "+" : "-")
@@ -6819,9 +7071,9 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, ExprResult &RHS,
 
     if (ConvTy == Compatible) {
       if (LHSType.getObjCLifetime() == Qualifiers::OCL_Strong)
-        checkRetainCycles(LHS, RHS.get());
+        checkRetainCycles(LHSExpr, RHS.get());
       else if (getLangOptions().ObjCAutoRefCount)
-        checkUnsafeExprAssigns(Loc, LHS, RHS.get());
+        checkUnsafeExprAssigns(Loc, LHSExpr, RHS.get());
     }
   } else {
     // Compound assignment "x += y"
@@ -6832,10 +7084,8 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, ExprResult &RHS,
                                RHS.get(), AA_Assigning))
     return QualType();
 
-  CheckForNullPointerDereference(*this, LHS);
-  // Check for trivial buffer overflows.
-  CheckArrayAccess(LHS->IgnoreParenCasts());
-  
+  CheckForNullPointerDereference(*this, LHSExpr);
+
   // C99 6.5.16p3: The type of an assignment expression is the type of the
   // left operand unless the left operand has qualified type, in which case
   // it is the unqualified version of the type of the left operand.
@@ -6872,7 +7122,8 @@ static QualType CheckCommaOperands(Sema &S, ExprResult &LHS, ExprResult &RHS,
     if (RHS.isInvalid())
       return QualType();
     if (!RHS.get()->getType()->isVoidType())
-      S.RequireCompleteType(Loc, RHS.get()->getType(), diag::err_incomplete_type);
+      S.RequireCompleteType(Loc, RHS.get()->getType(),
+                            diag::err_incomplete_type);
   }
 
   return RHS.get()->getType();
@@ -6883,7 +7134,7 @@ static QualType CheckCommaOperands(Sema &S, ExprResult &LHS, ExprResult &RHS,
 static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
                                                ExprValueKind &VK,
                                                SourceLocation OpLoc,
-                                               bool isInc, bool isPrefix) {
+                                               bool IsInc, bool IsPrefix) {
   if (Op->isTypeDependent())
     return S.Context.DependentTy;
 
@@ -6892,7 +7143,7 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
 
   if (S.getLangOptions().CPlusPlus && ResType->isBooleanType()) {
     // Decrement of bool is not allowed.
-    if (!isInc) {
+    if (!IsInc) {
       S.Diag(OpLoc, diag::err_decrement_bool) << Op->getSourceRange();
       return QualType();
     }
@@ -6901,18 +7152,13 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
   } else if (ResType->isRealType()) {
     // OK!
   } else if (ResType->isAnyPointerType()) {
-    QualType PointeeTy = ResType->getPointeeType();
-
     // C99 6.5.2.4p2, 6.5.6p2
     if (!checkArithmeticOpPointerOperand(S, OpLoc, Op))
       return QualType();
 
     // Diagnose bad cases where we step over interface counts.
-    else if (PointeeTy->isObjCObjectType() && S.LangOpts.ObjCNonFragileABI) {
-      S.Diag(OpLoc, diag::err_arithmetic_nonfragile_interface)
-        << PointeeTy << Op->getSourceRange();
+    else if (!checkArithmethicPointerOnNonFragileABI(S, OpLoc, Op))
       return QualType();
-    }
   } else if (ResType->isAnyComplexType()) {
     // C99 does not support ++/-- on complex types, we allow as an extension.
     S.Diag(OpLoc, diag::ext_integer_increment_complex)
@@ -6921,12 +7167,12 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
     ExprResult PR = S.CheckPlaceholderExpr(Op);
     if (PR.isInvalid()) return QualType();
     return CheckIncrementDecrementOperand(S, PR.take(), VK, OpLoc,
-                                          isInc, isPrefix);
+                                          IsInc, IsPrefix);
   } else if (S.getLangOptions().AltiVec && ResType->isVectorType()) {
     // OK! ( C/C++ Language Extensions for CBEA(Version 2.6) 10.3 )
   } else {
     S.Diag(OpLoc, diag::err_typecheck_illegal_increment_decrement)
-      << ResType << int(isInc) << Op->getSourceRange();
+      << ResType << int(IsInc) << Op->getSourceRange();
     return QualType();
   }
   // At this point, we know we have a real, complex or pointer type.
@@ -6936,7 +7182,7 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
   // In C++, a prefix increment is the same type as the operand. Otherwise
   // (in C or with postfix), the increment is the unqualified type of the
   // operand.
-  if (isPrefix && S.getLangOptions().CPlusPlus) {
+  if (IsPrefix && S.getLangOptions().CPlusPlus) {
     VK = VK_LValue;
     return ResType;
   } else {
@@ -6973,7 +7219,13 @@ ExprResult Sema::ConvertPropertyForRValue(Expr *E) {
             << PRE->getBase()->getType();
     }
   }
-  
+  else {
+    // lvalue-ness of an explicit property is determined by
+    // getter type.
+    QualType ResT = PRE->getGetterResultType();
+    VK = Expr::getValueKindForType(ResT);
+  }
+    
   E = ImplicitCastExpr::Create(Context, T, CK_GetObjCProperty,
                                E, 0, VK);
   
@@ -6984,7 +7236,8 @@ ExprResult Sema::ConvertPropertyForRValue(Expr *E) {
   return Owned(E);
 }
 
-void Sema::ConvertPropertyForLValue(ExprResult &LHS, ExprResult &RHS, QualType &LHSTy) {
+void Sema::ConvertPropertyForLValue(ExprResult &LHS, ExprResult &RHS,
+                                    QualType &LHSTy) {
   assert(LHS.get()->getValueKind() == VK_LValue &&
          LHS.get()->getObjectKind() == OK_ObjCProperty);
   const ObjCPropertyRefExpr *PropRef = LHS.get()->getObjCProperty();
@@ -6996,7 +7249,7 @@ void Sema::ConvertPropertyForLValue(ExprResult &LHS, ExprResult &RHS, QualType &
     // setter, RHS expression is being passed to the setter argument. So,
     // type conversion (and comparison) is RHS to setter's argument type.
     if (const ObjCMethodDecl *SetterMD = PropRef->getImplicitPropertySetter()) {
-      ObjCMethodDecl::param_iterator P = SetterMD->param_begin();
+      ObjCMethodDecl::param_const_iterator P = SetterMD->param_begin();
       LHSTy = (*P)->getType();
       Consumed = (getLangOptions().ObjCAutoRefCount &&
                   (*P)->hasAttr<NSConsumedAttr>());
@@ -7015,7 +7268,7 @@ void Sema::ConvertPropertyForLValue(ExprResult &LHS, ExprResult &RHS, QualType &
     const ObjCMethodDecl *setter
       = PropRef->getExplicitProperty()->getSetterMethodDecl();
     if (setter) {
-      ObjCMethodDecl::param_iterator P = setter->param_begin();
+      ObjCMethodDecl::param_const_iterator P = setter->param_begin();
       LHSTy = (*P)->getType();
       Consumed = (*P)->hasAttr<NSConsumedAttr>();
     }
@@ -7092,6 +7345,23 @@ static ValueDecl *getPrimaryDecl(Expr *E) {
   }
 }
 
+namespace {
+  enum {
+    AO_Bit_Field = 0,
+    AO_Vector_Element = 1,
+    AO_Property_Expansion = 2,
+    AO_Register_Variable = 3,
+    AO_No_Error = 4
+  };
+}
+/// \brief Diagnose invalid operand for address of operations.
+///
+/// \param Type The type of operand which cannot have its address taken.
+static void diagnoseAddressOfInvalidType(Sema &S, SourceLocation Loc,
+                                         Expr *E, unsigned Type) {
+  S.Diag(Loc, diag::err_typecheck_address_of) << Type << E->getSourceRange();
+}
+
 /// CheckAddressOfOperand - The operand of & must be either a function
 /// designator or an lvalue designating an object. If it is an lvalue, the
 /// object cannot be declared with storage class register or be a bit field.
@@ -7103,8 +7373,15 @@ static QualType CheckAddressOfOperand(Sema &S, Expr *OrigOp,
                                       SourceLocation OpLoc) {
   if (OrigOp->isTypeDependent())
     return S.Context.DependentTy;
-  if (OrigOp->getType() == S.Context.OverloadTy)
+  if (OrigOp->getType() == S.Context.OverloadTy) {
+    if (!isa<OverloadExpr>(OrigOp->IgnoreParens())) {
+      S.Diag(OpLoc, diag::err_typecheck_invalid_lvalue_addrof)
+        << OrigOp->getSourceRange();
+      return QualType();
+    }
+                  
     return S.Context.OverloadTy;
+  }
   if (OrigOp->getType() == S.Context.UnknownAnyTy)
     return S.Context.UnknownAnyTy;
   if (OrigOp->getType() == S.Context.BoundMemberTy) {
@@ -7131,6 +7408,7 @@ static QualType CheckAddressOfOperand(Sema &S, Expr *OrigOp,
   }
   ValueDecl *dcl = getPrimaryDecl(op);
   Expr::LValueClassification lval = op->ClassifyLValue(S.Context);
+  unsigned AddressOfError = AO_No_Error;
 
   if (lval == Expr::LV_ClassTemporary) { 
     bool sfinae = S.isSFINAEContext();
@@ -7178,19 +7456,13 @@ static QualType CheckAddressOfOperand(Sema &S, Expr *OrigOp,
     }
   } else if (op->getObjectKind() == OK_BitField) { // C99 6.5.3.2p1
     // The operand cannot be a bit-field
-    S.Diag(OpLoc, diag::err_typecheck_address_of)
-      << "bit-field" << op->getSourceRange();
-        return QualType();
+    AddressOfError = AO_Bit_Field;
   } else if (op->getObjectKind() == OK_VectorComponent) {
     // The operand cannot be an element of a vector
-    S.Diag(OpLoc, diag::err_typecheck_address_of)
-      << "vector element" << op->getSourceRange();
-    return QualType();
+    AddressOfError = AO_Vector_Element;
   } else if (op->getObjectKind() == OK_ObjCProperty) {
     // cannot take address of a property expression.
-    S.Diag(OpLoc, diag::err_typecheck_address_of)
-      << "property expression" << op->getSourceRange();
-    return QualType();
+    AddressOfError = AO_Property_Expansion;
   } else if (dcl) { // C99 6.5.3.2p1
     // We have an lvalue with a decl. Make sure the decl is not declared
     // with the register storage-class specifier.
@@ -7199,9 +7471,7 @@ static QualType CheckAddressOfOperand(Sema &S, Expr *OrigOp,
       // variable (c++03 7.1.1P3)
       if (vd->getStorageClass() == SC_Register &&
           !S.getLangOptions().CPlusPlus) {
-        S.Diag(OpLoc, diag::err_typecheck_address_of)
-          << "register variable" << op->getSourceRange();
-        return QualType();
+        AddressOfError = AO_Register_Variable;
       }
     } else if (isa<FunctionTemplateDecl>(dcl)) {
       return S.Context.OverloadTy;
@@ -7225,8 +7495,13 @@ static QualType CheckAddressOfOperand(Sema &S, Expr *OrigOp,
                 S.Context.getTypeDeclType(cast<RecordDecl>(Ctx)).getTypePtr());
         }
       }
-    } else if (!isa<FunctionDecl>(dcl))
-      assert(0 && "Unknown/unexpected decl type");
+    } else if (!isa<FunctionDecl>(dcl) && !isa<NonTypeTemplateParmDecl>(dcl))
+      llvm_unreachable("Unknown/unexpected decl type");
+  }
+
+  if (AddressOfError != AO_No_Error) {
+    diagnoseAddressOfInvalidType(S, OpLoc, op, AddressOfError);
+    return QualType();
   }
 
   if (lval == Expr::LV_IncompleteVoidType) {
@@ -7297,7 +7572,7 @@ static inline BinaryOperatorKind ConvertTokenKindToBinaryOpcode(
   tok::TokenKind Kind) {
   BinaryOperatorKind Opc;
   switch (Kind) {
-  default: assert(0 && "Unknown binop!");
+  default: llvm_unreachable("Unknown binop!");
   case tok::periodstar:           Opc = BO_PtrMemD; break;
   case tok::arrowstar:            Opc = BO_PtrMemI; break;
   case tok::star:                 Opc = BO_Mul; break;
@@ -7338,7 +7613,7 @@ static inline UnaryOperatorKind ConvertTokenKindToUnaryOpcode(
   tok::TokenKind Kind) {
   UnaryOperatorKind Opc;
   switch (Kind) {
-  default: assert(0 && "Unknown unary op!");
+  default: llvm_unreachable("Unknown unary op!");
   case tok::plusplus:     Opc = UO_PreInc; break;
   case tok::minusminus:   Opc = UO_PreDec; break;
   case tok::amp:          Opc = UO_AddrOf; break;
@@ -7357,35 +7632,35 @@ static inline UnaryOperatorKind ConvertTokenKindToUnaryOpcode(
 /// DiagnoseSelfAssignment - Emits a warning if a value is assigned to itself.
 /// This warning is only emitted for builtin assignment operations. It is also
 /// suppressed in the event of macro expansions.
-static void DiagnoseSelfAssignment(Sema &S, Expr *lhs, Expr *rhs,
+static void DiagnoseSelfAssignment(Sema &S, Expr *LHSExpr, Expr *RHSExpr,
                                    SourceLocation OpLoc) {
   if (!S.ActiveTemplateInstantiations.empty())
     return;
   if (OpLoc.isInvalid() || OpLoc.isMacroID())
     return;
-  lhs = lhs->IgnoreParenImpCasts();
-  rhs = rhs->IgnoreParenImpCasts();
-  const DeclRefExpr *LeftDeclRef = dyn_cast<DeclRefExpr>(lhs);
-  const DeclRefExpr *RightDeclRef = dyn_cast<DeclRefExpr>(rhs);
-  if (!LeftDeclRef || !RightDeclRef ||
-      LeftDeclRef->getLocation().isMacroID() ||
-      RightDeclRef->getLocation().isMacroID())
+  LHSExpr = LHSExpr->IgnoreParenImpCasts();
+  RHSExpr = RHSExpr->IgnoreParenImpCasts();
+  const DeclRefExpr *LHSDeclRef = dyn_cast<DeclRefExpr>(LHSExpr);
+  const DeclRefExpr *RHSDeclRef = dyn_cast<DeclRefExpr>(RHSExpr);
+  if (!LHSDeclRef || !RHSDeclRef ||
+      LHSDeclRef->getLocation().isMacroID() ||
+      RHSDeclRef->getLocation().isMacroID())
     return;
-  const ValueDecl *LeftDecl =
-    cast<ValueDecl>(LeftDeclRef->getDecl()->getCanonicalDecl());
-  const ValueDecl *RightDecl =
-    cast<ValueDecl>(RightDeclRef->getDecl()->getCanonicalDecl());
-  if (LeftDecl != RightDecl)
+  const ValueDecl *LHSDecl =
+    cast<ValueDecl>(LHSDeclRef->getDecl()->getCanonicalDecl());
+  const ValueDecl *RHSDecl =
+    cast<ValueDecl>(RHSDeclRef->getDecl()->getCanonicalDecl());
+  if (LHSDecl != RHSDecl)
     return;
-  if (LeftDecl->getType().isVolatileQualified())
+  if (LHSDecl->getType().isVolatileQualified())
     return;
-  if (const ReferenceType *RefTy = LeftDecl->getType()->getAs<ReferenceType>())
+  if (const ReferenceType *RefTy = LHSDecl->getType()->getAs<ReferenceType>())
     if (RefTy->getPointeeType().isVolatileQualified())
       return;
 
   S.Diag(OpLoc, diag::warn_self_assignment)
-      << LeftDeclRef->getType()
-      << lhs->getSourceRange() << rhs->getSourceRange();
+      << LHSDeclRef->getType()
+      << LHSExpr->getSourceRange() << RHSExpr->getSourceRange();
 }
 
 /// CreateBuiltinBinOp - Creates a new built-in binary operation with
@@ -7393,8 +7668,8 @@ static void DiagnoseSelfAssignment(Sema &S, Expr *lhs, Expr *rhs,
 /// built-in operations; ActOnBinOp handles overloaded operators.
 ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
                                     BinaryOperatorKind Opc,
-                                    Expr *lhsExpr, Expr *rhsExpr) {
-  ExprResult lhs = Owned(lhsExpr), rhs = Owned(rhsExpr);
+                                    Expr *LHSExpr, Expr *RHSExpr) {
+  ExprResult LHS = Owned(LHSExpr), RHS = Owned(RHSExpr);
   QualType ResultTy;     // Result type of the binary operator.
   // The following two variables are used for compound assignment operators
   QualType CompLHSTy;    // Type of LHS after promotions for computation
@@ -7410,172 +7685,131 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   // f<int> == 0;  // resolve f<int> blindly
   // void (*p)(int); p = f<int>;  // resolve f<int> using target
   if (Opc != BO_Assign) { 
-    ExprResult resolvedLHS = CheckPlaceholderExpr(lhs.get());
+    ExprResult resolvedLHS = CheckPlaceholderExpr(LHS.get());
     if (!resolvedLHS.isUsable()) return ExprError();
-    lhs = move(resolvedLHS);
+    LHS = move(resolvedLHS);
 
-    ExprResult resolvedRHS = CheckPlaceholderExpr(rhs.get());
+    ExprResult resolvedRHS = CheckPlaceholderExpr(RHS.get());
     if (!resolvedRHS.isUsable()) return ExprError();
-    rhs = move(resolvedRHS);
-  }
-
-  // The canonical way to check for a GNU null is with isNullPointerConstant,
-  // but we use a bit of a hack here for speed; this is a relatively
-  // hot path, and isNullPointerConstant is slow.
-  bool LeftNull = isa<GNUNullExpr>(lhs.get()->IgnoreParenImpCasts());
-  bool RightNull = isa<GNUNullExpr>(rhs.get()->IgnoreParenImpCasts());
-
-  // Detect when a NULL constant is used improperly in an expression.  These
-  // are mainly cases where the null pointer is used as an integer instead
-  // of a pointer.
-  if (LeftNull || RightNull) {
-    // Avoid analyzing cases where the result will either be invalid (and
-    // diagnosed as such) or entirely valid and not something to warn about.
-    QualType LeftType = lhs.get()->getType();
-    QualType RightType = rhs.get()->getType();
-    if (!LeftType->isBlockPointerType() && !LeftType->isMemberPointerType() &&
-        !LeftType->isFunctionType() &&
-        !RightType->isBlockPointerType() &&
-        !RightType->isMemberPointerType() &&
-        !RightType->isFunctionType()) {
-      if (Opc == BO_Mul || Opc == BO_Div || Opc == BO_Rem || Opc == BO_Add ||
-          Opc == BO_Sub || Opc == BO_Shl || Opc == BO_Shr || Opc == BO_And ||
-          Opc == BO_Xor || Opc == BO_Or || Opc == BO_MulAssign ||
-          Opc == BO_DivAssign || Opc == BO_AddAssign || Opc == BO_SubAssign ||
-          Opc == BO_RemAssign || Opc == BO_ShlAssign || Opc == BO_ShrAssign ||
-          Opc == BO_AndAssign || Opc == BO_OrAssign || Opc == BO_XorAssign) {
-        // These are the operations that would not make sense with a null pointer
-        // no matter what the other expression is.
-        Diag(OpLoc, diag::warn_null_in_arithmetic_operation)
-          << (LeftNull ? lhs.get()->getSourceRange() : SourceRange())
-          << (RightNull ? rhs.get()->getSourceRange() : SourceRange());
-      } else if (Opc == BO_LE || Opc == BO_LT || Opc == BO_GE || Opc == BO_GT ||
-                 Opc == BO_EQ || Opc == BO_NE) {
-        // These are the operations that would not make sense with a null pointer
-        // if the other expression the other expression is not a pointer.
-        if (LeftNull != RightNull &&
-            !LeftType->isAnyPointerType() &&
-            !LeftType->canDecayToPointerType() &&
-            !RightType->isAnyPointerType() &&
-            !RightType->canDecayToPointerType()) {
-          Diag(OpLoc, diag::warn_null_in_arithmetic_operation)
-            << (LeftNull ? lhs.get()->getSourceRange()
-                         : rhs.get()->getSourceRange());
-        }
-      }
-    }
+    RHS = move(resolvedRHS);
   }
 
   switch (Opc) {
   case BO_Assign:
-    ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, QualType());
+    ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType());
     if (getLangOptions().CPlusPlus &&
-        lhs.get()->getObjectKind() != OK_ObjCProperty) {
-      VK = lhs.get()->getValueKind();
-      OK = lhs.get()->getObjectKind();
+        LHS.get()->getObjectKind() != OK_ObjCProperty) {
+      VK = LHS.get()->getValueKind();
+      OK = LHS.get()->getObjectKind();
     }
     if (!ResultTy.isNull())
-      DiagnoseSelfAssignment(*this, lhs.get(), rhs.get(), OpLoc);
+      DiagnoseSelfAssignment(*this, LHS.get(), RHS.get(), OpLoc);
     break;
   case BO_PtrMemD:
   case BO_PtrMemI:
-    ResultTy = CheckPointerToMemberOperands(lhs, rhs, VK, OpLoc,
+    ResultTy = CheckPointerToMemberOperands(LHS, RHS, VK, OpLoc,
                                             Opc == BO_PtrMemI);
     break;
   case BO_Mul:
   case BO_Div:
-    ResultTy = CheckMultiplyDivideOperands(lhs, rhs, OpLoc, false,
+    ResultTy = CheckMultiplyDivideOperands(LHS, RHS, OpLoc, false,
                                            Opc == BO_Div);
     break;
   case BO_Rem:
-    ResultTy = CheckRemainderOperands(lhs, rhs, OpLoc);
+    ResultTy = CheckRemainderOperands(LHS, RHS, OpLoc);
     break;
   case BO_Add:
-    ResultTy = CheckAdditionOperands(lhs, rhs, OpLoc);
+    ResultTy = CheckAdditionOperands(LHS, RHS, OpLoc);
     break;
   case BO_Sub:
-    ResultTy = CheckSubtractionOperands(lhs, rhs, OpLoc);
+    ResultTy = CheckSubtractionOperands(LHS, RHS, OpLoc);
     break;
   case BO_Shl:
   case BO_Shr:
-    ResultTy = CheckShiftOperands(lhs, rhs, OpLoc, Opc);
+    ResultTy = CheckShiftOperands(LHS, RHS, OpLoc, Opc);
     break;
   case BO_LE:
   case BO_LT:
   case BO_GE:
   case BO_GT:
-    ResultTy = CheckCompareOperands(lhs, rhs, OpLoc, Opc, true);
+    ResultTy = CheckCompareOperands(LHS, RHS, OpLoc, Opc, true);
     break;
   case BO_EQ:
   case BO_NE:
-    ResultTy = CheckCompareOperands(lhs, rhs, OpLoc, Opc, false);
+    ResultTy = CheckCompareOperands(LHS, RHS, OpLoc, Opc, false);
     break;
   case BO_And:
   case BO_Xor:
   case BO_Or:
-    ResultTy = CheckBitwiseOperands(lhs, rhs, OpLoc);
+    ResultTy = CheckBitwiseOperands(LHS, RHS, OpLoc);
     break;
   case BO_LAnd:
   case BO_LOr:
-    ResultTy = CheckLogicalOperands(lhs, rhs, OpLoc, Opc);
+    ResultTy = CheckLogicalOperands(LHS, RHS, OpLoc, Opc);
     break;
   case BO_MulAssign:
   case BO_DivAssign:
-    CompResultTy = CheckMultiplyDivideOperands(lhs, rhs, OpLoc, true,
+    CompResultTy = CheckMultiplyDivideOperands(LHS, RHS, OpLoc, true,
                                                Opc == BO_DivAssign);
     CompLHSTy = CompResultTy;
-    if (!CompResultTy.isNull() && !lhs.isInvalid() && !rhs.isInvalid())
-      ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, CompResultTy);
+    if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
+      ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy);
     break;
   case BO_RemAssign:
-    CompResultTy = CheckRemainderOperands(lhs, rhs, OpLoc, true);
+    CompResultTy = CheckRemainderOperands(LHS, RHS, OpLoc, true);
     CompLHSTy = CompResultTy;
-    if (!CompResultTy.isNull() && !lhs.isInvalid() && !rhs.isInvalid())
-      ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, CompResultTy);
+    if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
+      ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy);
     break;
   case BO_AddAssign:
-    CompResultTy = CheckAdditionOperands(lhs, rhs, OpLoc, &CompLHSTy);
-    if (!CompResultTy.isNull() && !lhs.isInvalid() && !rhs.isInvalid())
-      ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, CompResultTy);
+    CompResultTy = CheckAdditionOperands(LHS, RHS, OpLoc, &CompLHSTy);
+    if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
+      ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy);
     break;
   case BO_SubAssign:
-    CompResultTy = CheckSubtractionOperands(lhs, rhs, OpLoc, &CompLHSTy);
-    if (!CompResultTy.isNull() && !lhs.isInvalid() && !rhs.isInvalid())
-      ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, CompResultTy);
+    CompResultTy = CheckSubtractionOperands(LHS, RHS, OpLoc, &CompLHSTy);
+    if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
+      ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy);
     break;
   case BO_ShlAssign:
   case BO_ShrAssign:
-    CompResultTy = CheckShiftOperands(lhs, rhs, OpLoc, Opc, true);
+    CompResultTy = CheckShiftOperands(LHS, RHS, OpLoc, Opc, true);
     CompLHSTy = CompResultTy;
-    if (!CompResultTy.isNull() && !lhs.isInvalid() && !rhs.isInvalid())
-      ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, CompResultTy);
+    if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
+      ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy);
     break;
   case BO_AndAssign:
   case BO_XorAssign:
   case BO_OrAssign:
-    CompResultTy = CheckBitwiseOperands(lhs, rhs, OpLoc, true);
+    CompResultTy = CheckBitwiseOperands(LHS, RHS, OpLoc, true);
     CompLHSTy = CompResultTy;
-    if (!CompResultTy.isNull() && !lhs.isInvalid() && !rhs.isInvalid())
-      ResultTy = CheckAssignmentOperands(lhs.get(), rhs, OpLoc, CompResultTy);
+    if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
+      ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy);
     break;
   case BO_Comma:
-    ResultTy = CheckCommaOperands(*this, lhs, rhs, OpLoc);
-    if (getLangOptions().CPlusPlus && !rhs.isInvalid()) {
-      VK = rhs.get()->getValueKind();
-      OK = rhs.get()->getObjectKind();
+    ResultTy = CheckCommaOperands(*this, LHS, RHS, OpLoc);
+    if (getLangOptions().CPlusPlus && !RHS.isInvalid()) {
+      VK = RHS.get()->getValueKind();
+      OK = RHS.get()->getObjectKind();
     }
     break;
   }
-  if (ResultTy.isNull() || lhs.isInvalid() || rhs.isInvalid())
+  if (ResultTy.isNull() || LHS.isInvalid() || RHS.isInvalid())
     return ExprError();
+
+  // Check for array bounds violations for both sides of the BinaryOperator
+  CheckArrayAccess(LHS.get());
+  CheckArrayAccess(RHS.get());
+
   if (CompResultTy.isNull())
-    return Owned(new (Context) BinaryOperator(lhs.take(), rhs.take(), Opc,
+    return Owned(new (Context) BinaryOperator(LHS.take(), RHS.take(), Opc,
                                               ResultTy, VK, OK, OpLoc));
-  if (getLangOptions().CPlusPlus && lhs.get()->getObjectKind() != OK_ObjCProperty) {
+  if (getLangOptions().CPlusPlus && LHS.get()->getObjectKind() !=
+      OK_ObjCProperty) {
     VK = VK_LValue;
-    OK = lhs.get()->getObjectKind();
+    OK = LHS.get()->getObjectKind();
   }
-  return Owned(new (Context) CompoundAssignOperator(lhs.take(), rhs.take(), Opc,
+  return Owned(new (Context) CompoundAssignOperator(LHS.take(), RHS.take(), Opc,
                                                     ResultTy, VK, OK, CompLHSTy,
                                                     CompResultTy, OpLoc));
 }
@@ -7585,51 +7819,49 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
 /// comparison operators have higher precedence. The most typical example of
 /// such code is "flags & 0x0020 != 0", which is equivalent to "flags & 1".
 static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperatorKind Opc,
-                                      SourceLocation OpLoc,Expr *lhs,Expr *rhs){
+                                      SourceLocation OpLoc, Expr *LHSExpr,
+                                      Expr *RHSExpr) {
   typedef BinaryOperator BinOp;
-  BinOp::Opcode lhsopc = static_cast<BinOp::Opcode>(-1),
-                rhsopc = static_cast<BinOp::Opcode>(-1);
-  if (BinOp *BO = dyn_cast<BinOp>(lhs))
-    lhsopc = BO->getOpcode();
-  if (BinOp *BO = dyn_cast<BinOp>(rhs))
-    rhsopc = BO->getOpcode();
+  BinOp::Opcode LHSopc = static_cast<BinOp::Opcode>(-1),
+                RHSopc = static_cast<BinOp::Opcode>(-1);
+  if (BinOp *BO = dyn_cast<BinOp>(LHSExpr))
+    LHSopc = BO->getOpcode();
+  if (BinOp *BO = dyn_cast<BinOp>(RHSExpr))
+    RHSopc = BO->getOpcode();
 
   // Subs are not binary operators.
-  if (lhsopc == -1 && rhsopc == -1)
+  if (LHSopc == -1 && RHSopc == -1)
     return;
 
   // Bitwise operations are sometimes used as eager logical ops.
   // Don't diagnose this.
-  if ((BinOp::isComparisonOp(lhsopc) || BinOp::isBitwiseOp(lhsopc)) &&
-      (BinOp::isComparisonOp(rhsopc) || BinOp::isBitwiseOp(rhsopc)))
+  if ((BinOp::isComparisonOp(LHSopc) || BinOp::isBitwiseOp(LHSopc)) &&
+      (BinOp::isComparisonOp(RHSopc) || BinOp::isBitwiseOp(RHSopc)))
     return;
 
-  if (BinOp::isComparisonOp(lhsopc)) {
-    Self.Diag(OpLoc, diag::warn_precedence_bitwise_rel)
-        << SourceRange(lhs->getLocStart(), OpLoc)
-        << BinOp::getOpcodeStr(Opc) << BinOp::getOpcodeStr(lhsopc);
-    SuggestParentheses(Self, OpLoc,
-      Self.PDiag(diag::note_precedence_bitwise_silence)
-          << BinOp::getOpcodeStr(lhsopc),
-      lhs->getSourceRange());
-    SuggestParentheses(Self, OpLoc,
-      Self.PDiag(diag::note_precedence_bitwise_first)
-          << BinOp::getOpcodeStr(Opc),
-      SourceRange(cast<BinOp>(lhs)->getRHS()->getLocStart(), rhs->getLocEnd()));
-  } else if (BinOp::isComparisonOp(rhsopc)) {
-    Self.Diag(OpLoc, diag::warn_precedence_bitwise_rel)
-        << SourceRange(OpLoc, rhs->getLocEnd())
-        << BinOp::getOpcodeStr(Opc) << BinOp::getOpcodeStr(rhsopc);
-    SuggestParentheses(Self, OpLoc,
-      Self.PDiag(diag::note_precedence_bitwise_silence)
-          << BinOp::getOpcodeStr(rhsopc),
-      rhs->getSourceRange());
-    SuggestParentheses(Self, OpLoc,
-      Self.PDiag(diag::note_precedence_bitwise_first)
-        << BinOp::getOpcodeStr(Opc),
-      SourceRange(lhs->getLocStart(), 
-                  cast<BinOp>(rhs)->getLHS()->getLocStart()));
-  }
+  bool isLeftComp = BinOp::isComparisonOp(LHSopc);
+  bool isRightComp = BinOp::isComparisonOp(RHSopc);
+  if (!isLeftComp && !isRightComp) return;
+
+  SourceRange DiagRange = isLeftComp ? SourceRange(LHSExpr->getLocStart(),
+                                                   OpLoc)
+                                     : SourceRange(OpLoc, RHSExpr->getLocEnd());
+  std::string OpStr = isLeftComp ? BinOp::getOpcodeStr(LHSopc)
+                                 : BinOp::getOpcodeStr(RHSopc);
+  SourceRange ParensRange = isLeftComp ?
+      SourceRange(cast<BinOp>(LHSExpr)->getRHS()->getLocStart(),
+                  RHSExpr->getLocEnd())
+    : SourceRange(LHSExpr->getLocStart(),
+                  cast<BinOp>(RHSExpr)->getLHS()->getLocStart());
+
+  Self.Diag(OpLoc, diag::warn_precedence_bitwise_rel)
+    << DiagRange << BinOp::getOpcodeStr(Opc) << OpStr;
+  SuggestParentheses(Self, OpLoc,
+    Self.PDiag(diag::note_precedence_bitwise_silence) << OpStr,
+    RHSExpr->getSourceRange());
+  SuggestParentheses(Self, OpLoc,
+    Self.PDiag(diag::note_precedence_bitwise_first) << BinOp::getOpcodeStr(Opc),
+    ParensRange);
 }
 
 /// \brief It accepts a '&' expr that is inside a '|' one.
@@ -7676,11 +7908,11 @@ static bool EvaluatesAsFalse(Sema &S, Expr *E) {
 
 /// \brief Look for '&&' in the left hand of a '||' expr.
 static void DiagnoseLogicalAndInLogicalOrLHS(Sema &S, SourceLocation OpLoc,
-                                             Expr *OrLHS, Expr *OrRHS) {
-  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(OrLHS)) {
+                                             Expr *LHSExpr, Expr *RHSExpr) {
+  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(LHSExpr)) {
     if (Bop->getOpcode() == BO_LAnd) {
       // If it's "a && b || 0" don't warn since the precedence doesn't matter.
-      if (EvaluatesAsFalse(S, OrRHS))
+      if (EvaluatesAsFalse(S, RHSExpr))
         return;
       // If it's "1 && a || b" don't warn since the precedence doesn't matter.
       if (!EvaluatesAsTrue(S, Bop->getLHS()))
@@ -7698,11 +7930,11 @@ static void DiagnoseLogicalAndInLogicalOrLHS(Sema &S, SourceLocation OpLoc,
 
 /// \brief Look for '&&' in the right hand of a '||' expr.
 static void DiagnoseLogicalAndInLogicalOrRHS(Sema &S, SourceLocation OpLoc,
-                                             Expr *OrLHS, Expr *OrRHS) {
-  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(OrRHS)) {
+                                             Expr *LHSExpr, Expr *RHSExpr) {
+  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(RHSExpr)) {
     if (Bop->getOpcode() == BO_LAnd) {
       // If it's "0 || a && b" don't warn since the precedence doesn't matter.
-      if (EvaluatesAsFalse(S, OrLHS))
+      if (EvaluatesAsFalse(S, LHSExpr))
         return;
       // If it's "a || b && 1" don't warn since the precedence doesn't matter.
       if (!EvaluatesAsTrue(S, Bop->getRHS()))
@@ -7723,52 +7955,54 @@ static void DiagnoseBitwiseAndInBitwiseOr(Sema &S, SourceLocation OpLoc,
 /// DiagnoseBinOpPrecedence - Emit warnings for expressions with tricky
 /// precedence.
 static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
-                                    SourceLocation OpLoc, Expr *lhs, Expr *rhs){
+                                    SourceLocation OpLoc, Expr *LHSExpr,
+                                    Expr *RHSExpr){
   // Diagnose "arg1 'bitwise' arg2 'eq' arg3".
   if (BinaryOperator::isBitwiseOp(Opc))
-    DiagnoseBitwisePrecedence(Self, Opc, OpLoc, lhs, rhs);
+    DiagnoseBitwisePrecedence(Self, Opc, OpLoc, LHSExpr, RHSExpr);
 
   // Diagnose "arg1 & arg2 | arg3"
   if (Opc == BO_Or && !OpLoc.isMacroID()/* Don't warn in macros. */) {
-    DiagnoseBitwiseAndInBitwiseOr(Self, OpLoc, lhs);
-    DiagnoseBitwiseAndInBitwiseOr(Self, OpLoc, rhs);
+    DiagnoseBitwiseAndInBitwiseOr(Self, OpLoc, LHSExpr);
+    DiagnoseBitwiseAndInBitwiseOr(Self, OpLoc, RHSExpr);
   }
 
   // Warn about arg1 || arg2 && arg3, as GCC 4.3+ does.
   // We don't warn for 'assert(a || b && "bad")' since this is safe.
   if (Opc == BO_LOr && !OpLoc.isMacroID()/* Don't warn in macros. */) {
-    DiagnoseLogicalAndInLogicalOrLHS(Self, OpLoc, lhs, rhs);
-    DiagnoseLogicalAndInLogicalOrRHS(Self, OpLoc, lhs, rhs);
+    DiagnoseLogicalAndInLogicalOrLHS(Self, OpLoc, LHSExpr, RHSExpr);
+    DiagnoseLogicalAndInLogicalOrRHS(Self, OpLoc, LHSExpr, RHSExpr);
   }
 }
 
 // Binary Operators.  'Tok' is the token for the operator.
 ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
                             tok::TokenKind Kind,
-                            Expr *lhs, Expr *rhs) {
+                            Expr *LHSExpr, Expr *RHSExpr) {
   BinaryOperatorKind Opc = ConvertTokenKindToBinaryOpcode(Kind);
-  assert((lhs != 0) && "ActOnBinOp(): missing left expression");
-  assert((rhs != 0) && "ActOnBinOp(): missing right expression");
+  assert((LHSExpr != 0) && "ActOnBinOp(): missing left expression");
+  assert((RHSExpr != 0) && "ActOnBinOp(): missing right expression");
 
   // Emit warnings for tricky precedence issues, e.g. "bitfield & 0x4 == 0"
-  DiagnoseBinOpPrecedence(*this, Opc, TokLoc, lhs, rhs);
+  DiagnoseBinOpPrecedence(*this, Opc, TokLoc, LHSExpr, RHSExpr);
 
-  return BuildBinOp(S, TokLoc, Opc, lhs, rhs);
+  return BuildBinOp(S, TokLoc, Opc, LHSExpr, RHSExpr);
 }
 
 ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
                             BinaryOperatorKind Opc,
-                            Expr *lhs, Expr *rhs) {
+                            Expr *LHSExpr, Expr *RHSExpr) {
   if (getLangOptions().CPlusPlus) {
     bool UseBuiltinOperator;
 
-    if (lhs->isTypeDependent() || rhs->isTypeDependent()) {
+    if (LHSExpr->isTypeDependent() || RHSExpr->isTypeDependent()) {
       UseBuiltinOperator = false;
-    } else if (Opc == BO_Assign && lhs->getObjectKind() == OK_ObjCProperty) {
+    } else if (Opc == BO_Assign &&
+               LHSExpr->getObjectKind() == OK_ObjCProperty) {
       UseBuiltinOperator = true;
     } else {
-      UseBuiltinOperator = !lhs->getType()->isOverloadableType() &&
-                           !rhs->getType()->isOverloadableType();
+      UseBuiltinOperator = !LHSExpr->getType()->isOverloadableType() &&
+                           !RHSExpr->getType()->isOverloadableType();
     }
 
     if (!UseBuiltinOperator) {
@@ -7780,17 +8014,17 @@ ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
       OverloadedOperatorKind OverOp
         = BinaryOperator::getOverloadedOperator(Opc);
       if (S && OverOp != OO_None)
-        LookupOverloadedOperatorName(OverOp, S, lhs->getType(), rhs->getType(),
-                                     Functions);
+        LookupOverloadedOperatorName(OverOp, S, LHSExpr->getType(),
+                                     RHSExpr->getType(), Functions);
 
       // Build the (potentially-overloaded, potentially-dependent)
       // binary operation.
-      return CreateOverloadedBinOp(OpLoc, Opc, Functions, lhs, rhs);
+      return CreateOverloadedBinOp(OpLoc, Opc, Functions, LHSExpr, RHSExpr);
     }
   }
 
   // Build a built-in binary operation.
-  return CreateBuiltinBinOp(OpLoc, Opc, lhs, rhs);
+  return CreateBuiltinBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
 }
 
 ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
@@ -7876,6 +8110,13 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
     Input = DefaultFunctionArrayLvalueConversion(Input.take());
     if (Input.isInvalid()) return ExprError();
     resultType = Input.get()->getType();
+
+    // Though we still have to promote half FP to float...
+    if (resultType->isHalfType()) {
+      Input = ImpCastExprToType(Input.take(), Context.FloatTy, CK_FloatingCast).take();
+      resultType = Context.FloatTy;
+    }
+
     if (resultType->isDependentType())
       break;
     if (resultType->isScalarType()) {
@@ -7917,13 +8158,19 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
   if (resultType.isNull() || Input.isInvalid())
     return ExprError();
 
+  // Check for array bounds violations in the operand of the UnaryOperator,
+  // except for the '*' and '&' operators that have to be handled specially
+  // by CheckArrayAccess (as there are special cases like &array[arraysize]
+  // that are explicitly defined as valid by the standard).
+  if (Opc != UO_AddrOf && Opc != UO_Deref)
+    CheckArrayAccess(Input.get());
+
   return Owned(new (Context) UnaryOperator(Input.take(), Opc, resultType,
                                            VK, OK, OpLoc));
 }
 
 ExprResult Sema::BuildUnaryOp(Scope *S, SourceLocation OpLoc,
-                              UnaryOperatorKind Opc,
-                              Expr *Input) {
+                              UnaryOperatorKind Opc, Expr *Input) {
   if (getLangOptions().CPlusPlus && Input->getType()->isOverloadableType() &&
       UnaryOperator::getOverloadedOperator(Opc) != OO_None) {
     // Find all of the overloaded operators visible from this
@@ -7962,13 +8209,13 @@ ExprResult Sema::ActOnAddrLabel(SourceLocation OpLoc, SourceLocation LabLoc,
 /// ns_returns_retained function) and, if so, rebuild it to hoist the
 /// release out of the full-expression.  Otherwise, return null.
 /// Cannot fail.
-static Expr *maybeRebuildARCConsumingStmt(Stmt *s) {
+static Expr *maybeRebuildARCConsumingStmt(Stmt *Statement) {
   // Should always be wrapped with one of these.
-  ExprWithCleanups *cleanups = dyn_cast<ExprWithCleanups>(s);
+  ExprWithCleanups *cleanups = dyn_cast<ExprWithCleanups>(Statement);
   if (!cleanups) return 0;
 
   ImplicitCastExpr *cast = dyn_cast<ImplicitCastExpr>(cleanups->getSubExpr());
-  if (!cast || cast->getCastKind() != CK_ObjCConsumeObject)
+  if (!cast || cast->getCastKind() != CK_ARCConsumeObject)
     return 0;
 
   // Splice out the cast.  This shouldn't modify any interesting
@@ -8091,8 +8338,8 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
   bool DidWarnAboutNonPOD = false;
   QualType CurrentType = ArgTy;
   typedef OffsetOfExpr::OffsetOfNode OffsetOfNode;
-  llvm::SmallVector<OffsetOfNode, 4> Comps;
-  llvm::SmallVector<Expr*, 4> Exprs;
+  SmallVector<OffsetOfNode, 4> Comps;
+  SmallVector<Expr*, 4> Exprs;
   for (unsigned i = 0; i != NumComponents; ++i) {
     const OffsetOfComponent &OC = CompPtr[i];
     if (OC.isBrackets) {
@@ -8174,7 +8421,7 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
     //   (If the specified member is a bit-field, the behavior is undefined.)
     //
     // We diagnose this as an error.
-    if (MemberDecl->getBitWidth()) {
+    if (MemberDecl->isBitField()) {
       Diag(OC.LocEnd, diag::err_offsetof_bitfield)
         << MemberDecl->getDeclName()
         << SourceRange(BuiltinLoc, RParenLoc);
@@ -8219,13 +8466,13 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
 ExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
                                       SourceLocation BuiltinLoc,
                                       SourceLocation TypeLoc,
-                                      ParsedType argty,
+                                      ParsedType ParsedArgTy,
                                       OffsetOfComponent *CompPtr,
                                       unsigned NumComponents,
-                                      SourceLocation RPLoc) {
+                                      SourceLocation RParenLoc) {
   
   TypeSourceInfo *ArgTInfo;
-  QualType ArgTy = GetTypeFromParser(argty, &ArgTInfo);
+  QualType ArgTy = GetTypeFromParser(ParsedArgTy, &ArgTInfo);
   if (ArgTy.isNull())
     return ExprError();
 
@@ -8233,7 +8480,7 @@ ExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
     ArgTInfo = Context.getTrivialTypeSourceInfo(ArgTy, TypeLoc);
 
   return BuildBuiltinOffsetOf(BuiltinLoc, ArgTInfo, CompPtr, NumComponents, 
-                              RPLoc);
+                              RParenLoc);
 }
 
 
@@ -8279,12 +8526,12 @@ ExprResult Sema::ActOnChooseExpr(SourceLocation BuiltinLoc,
 //===----------------------------------------------------------------------===//
 
 /// ActOnBlockStart - This callback is invoked when a block literal is started.
-void Sema::ActOnBlockStart(SourceLocation CaretLoc, Scope *BlockScope) {
+void Sema::ActOnBlockStart(SourceLocation CaretLoc, Scope *CurScope) {
   BlockDecl *Block = BlockDecl::Create(Context, CurContext, CaretLoc);
-  PushBlockScope(BlockScope, Block);
+  PushBlockScope(CurScope, Block);
   CurContext->addDecl(Block);
-  if (BlockScope)
-    PushDeclContext(BlockScope, Block);
+  if (CurScope)
+    PushDeclContext(CurScope, Block);
   else
     CurContext = Block;
 }
@@ -8351,7 +8598,7 @@ void Sema::ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
     CurBlock->ReturnType = RetTy;
 
   // Push block parameters from the declarator if we had them.
-  llvm::SmallVector<ParmVarDecl*, 8> Params;
+  SmallVector<ParmVarDecl*, 8> Params;
   if (ExplicitSignature) {
     for (unsigned I = 0, E = ExplicitSignature.getNumArgs(); I != E; ++I) {
       ParmVarDecl *Param = ExplicitSignature.getArg(I);
@@ -8378,7 +8625,7 @@ void Sema::ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
 
   // Set the parameters on the block decl.
   if (!Params.empty()) {
-    CurBlock->TheDecl->setParams(Params.data(), Params.size());
+    CurBlock->TheDecl->setParams(Params);
     CheckParmsForFunctionDef(CurBlock->TheDecl->param_begin(),
                              CurBlock->TheDecl->param_end(),
                              /*CheckParameterNames=*/false);
@@ -8500,6 +8747,8 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
       getCurFunction()->setHasBranchProtectedScope();
   }
 
+  computeNRVO(Body, getCurBlock());
+  
   BlockExpr *Result = new (Context) BlockExpr(BSI->TheDecl, BlockTy);
   const AnalysisBasedWarnings::Policy &WP = AnalysisWarnings.getDefaultPolicy();
   PopFunctionOrBlockScope(&WP, Result->getBlockDecl(), Result);
@@ -8508,11 +8757,11 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
 }
 
 ExprResult Sema::ActOnVAArg(SourceLocation BuiltinLoc,
-                                        Expr *expr, ParsedType type,
+                                        Expr *E, ParsedType Ty,
                                         SourceLocation RPLoc) {
   TypeSourceInfo *TInfo;
-  GetTypeFromParser(type, &TInfo);
-  return BuildVAArgExpr(BuiltinLoc, expr, TInfo, RPLoc);
+  GetTypeFromParser(Ty, &TInfo);
+  return BuildVAArgExpr(BuiltinLoc, E, TInfo, RPLoc);
 }
 
 ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
@@ -8559,11 +8808,14 @@ ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
           << TInfo->getTypeLoc().getSourceRange()))
       return ExprError();
 
-    if (!TInfo->getType().isPODType(Context))
+    if (!TInfo->getType().isPODType(Context)) {
       Diag(TInfo->getTypeLoc().getBeginLoc(),
-          diag::warn_second_parameter_to_va_arg_not_pod)
+           TInfo->getType()->isObjCLifetimeType()
+             ? diag::warn_second_parameter_to_va_arg_ownership_qualified
+             : diag::warn_second_parameter_to_va_arg_not_pod)
         << TInfo->getType()
         << TInfo->getTypeLoc().getSourceRange();
+    }
 
     // Check for va_arg where arguments of the given type will be promoted
     // (i.e. this va_arg is guaranteed to have undefined behavior).
@@ -8591,16 +8843,15 @@ ExprResult Sema::ActOnGNUNullExpr(SourceLocation TokenLoc) {
   // The type of __null will be int or long, depending on the size of
   // pointers on the target.
   QualType Ty;
-  unsigned pw = Context.Target.getPointerWidth(0);
-  if (pw == Context.Target.getIntWidth())
+  unsigned pw = Context.getTargetInfo().getPointerWidth(0);
+  if (pw == Context.getTargetInfo().getIntWidth())
     Ty = Context.IntTy;
-  else if (pw == Context.Target.getLongWidth())
+  else if (pw == Context.getTargetInfo().getLongWidth())
     Ty = Context.LongTy;
-  else if (pw == Context.Target.getLongLongWidth())
+  else if (pw == Context.getTargetInfo().getLongLongWidth())
     Ty = Context.LongLongTy;
   else {
-    assert(!"I don't know size of pointer!");
-    Ty = Context.IntTy;
+    llvm_unreachable("I don't know size of pointer!");
   }
 
   return Owned(new (Context) GNUNullExpr(Ty, TokenLoc));
@@ -8625,7 +8876,7 @@ static void MakeObjCStringLiteralFixItHint(Sema& SemaRef, QualType DstType,
 
   // Strip off any parens and casts.
   StringLiteral *SL = dyn_cast<StringLiteral>(SrcExpr->IgnoreParenCasts());
-  if (!SL || SL->isWide())
+  if (!SL || !SL->isAscii())
     return;
 
   Hint = FixItHint::CreateInsertion(SL->getLocStart(), "@");
@@ -8644,21 +8895,31 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
   bool isInvalid = false;
   unsigned DiagKind;
   FixItHint Hint;
+  ConversionFixItGenerator ConvHints;
+  bool MayHaveConvFixit = false;
 
   switch (ConvTy) {
-  default: assert(0 && "Unknown conversion type");
+  default: llvm_unreachable("Unknown conversion type");
   case Compatible: return false;
   case PointerToInt:
     DiagKind = diag::ext_typecheck_convert_pointer_int;
+    ConvHints.tryToFixConversion(SrcExpr, SrcType, DstType, *this);
+    MayHaveConvFixit = true;
     break;
   case IntToPointer:
     DiagKind = diag::ext_typecheck_convert_int_pointer;
+    ConvHints.tryToFixConversion(SrcExpr, SrcType, DstType, *this);
+    MayHaveConvFixit = true;
     break;
   case IncompatiblePointer:
     MakeObjCStringLiteralFixItHint(*this, DstType, SrcExpr, Hint);
     DiagKind = diag::ext_typecheck_convert_incompatible_pointer;
     CheckInferredResultType = DstType->isObjCObjectPointerType() &&
       SrcType->isObjCObjectPointerType();
+    if (Hint.isNull() && !CheckInferredResultType) {
+      ConvHints.tryToFixConversion(SrcExpr, SrcType, DstType, *this);
+    }
+    MayHaveConvFixit = true;
     break;
   case IncompatiblePointerSign:
     DiagKind = diag::ext_typecheck_convert_incompatible_pointer_sign;
@@ -8722,6 +8983,8 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
     break;
   case Incompatible:
     DiagKind = diag::err_typecheck_convert_incompatible;
+    ConvHints.tryToFixConversion(SrcExpr, SrcType, DstType, *this);
+    MayHaveConvFixit = true;
     isInvalid = true;
     break;
   }
@@ -8746,8 +9009,23 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
     break;
   }
 
-  Diag(Loc, DiagKind) << FirstType << SecondType << Action
-    << SrcExpr->getSourceRange() << Hint;
+  PartialDiagnostic FDiag = PDiag(DiagKind);
+  FDiag << FirstType << SecondType << Action << SrcExpr->getSourceRange();
+
+  // If we can fix the conversion, suggest the FixIts.
+  assert(ConvHints.isNull() || Hint.isNull());
+  if (!ConvHints.isNull()) {
+    for (llvm::SmallVector<FixItHint, 1>::iterator
+        HI = ConvHints.Hints.begin(), HE = ConvHints.Hints.end();
+        HI != HE; ++HI)
+      FDiag << *HI;
+  } else {
+    FDiag << Hint;
+  }
+  if (MayHaveConvFixit) { FDiag << (unsigned) (ConvHints.Kind); }
+
+  Diag(Loc, FDiag);
+
   if (CheckInferredResultType)
     EmitRelatedResultTypeNote(SrcExpr);
   
@@ -8786,7 +9064,7 @@ bool Sema::VerifyIntegerConstantExpression(const Expr *E, llvm::APSInt *Result){
 
   if (EvalResult.Diag &&
       Diags.getDiagnosticLevel(diag::ext_expr_not_ice, EvalResult.DiagLoc)
-          != Diagnostic::Ignored)
+          != DiagnosticsEngine::Ignored)
     Diag(EvalResult.DiagLoc, EvalResult.Diag);
 
   if (Result)
@@ -8803,8 +9081,7 @@ Sema::PushExpressionEvaluationContext(ExpressionEvaluationContext NewContext) {
   ExprNeedsCleanups = false;
 }
 
-void
-Sema::PopExpressionEvaluationContext() {
+void Sema::PopExpressionEvaluationContext() {
   // Pop the current expression evaluation context off the stack.
   ExpressionEvaluationContextRecord Rec = ExprEvalContexts.back();
   ExprEvalContexts.pop_back();
@@ -8874,10 +9151,10 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
   if (D->isUsed(false))
     return;
 
-  // Mark a parameter or variable declaration "used", regardless of whether we're in a
-  // template or not. The reason for this is that unevaluated expressions
-  // (e.g. (void)sizeof()) constitute a use for warning purposes (-Wunused-variables and
-  // -Wunused-parameters)
+  // Mark a parameter or variable declaration "used", regardless of whether
+  // we're in a template or not. The reason for this is that unevaluated
+  // expressions (e.g. (void)sizeof()) constitute a use for warning purposes
+  // (-Wunused-variables and -Wunused-parameters)
   if (isa<ParmVarDecl>(D) ||
       (isa<VarDecl>(D) && D->getDeclContext()->isFunctionOrMethod())) {
     D->setUsed();
@@ -8917,15 +9194,19 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
 
   // Note that this declaration has been used.
   if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(D)) {
-    if (Constructor->isDefaulted() && Constructor->isDefaultConstructor()) {
-      if (Constructor->isTrivial())
-        return;
-      if (!Constructor->isUsed(false))
-        DefineImplicitDefaultConstructor(Loc, Constructor);
-    } else if (Constructor->isDefaulted() &&
-               Constructor->isCopyConstructor()) {
-      if (!Constructor->isUsed(false))
-        DefineImplicitCopyConstructor(Loc, Constructor);
+    if (Constructor->isDefaulted()) {
+      if (Constructor->isDefaultConstructor()) {
+        if (Constructor->isTrivial())
+          return;
+        if (!Constructor->isUsed(false))
+          DefineImplicitDefaultConstructor(Loc, Constructor);
+      } else if (Constructor->isCopyConstructor()) {
+        if (!Constructor->isUsed(false))
+          DefineImplicitCopyConstructor(Loc, Constructor);
+      } else if (Constructor->isMoveConstructor()) {
+        if (!Constructor->isUsed(false))
+          DefineImplicitMoveConstructor(Loc, Constructor);
+      }
     }
 
     MarkVTableUsed(Loc, Constructor->getParent());
@@ -8937,8 +9218,12 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
   } else if (CXXMethodDecl *MethodDecl = dyn_cast<CXXMethodDecl>(D)) {
     if (MethodDecl->isDefaulted() && MethodDecl->isOverloadedOperator() &&
         MethodDecl->getOverloadedOperator() == OO_Equal) {
-      if (!MethodDecl->isUsed(false))
-        DefineImplicitCopyAssignment(Loc, MethodDecl);
+      if (!MethodDecl->isUsed(false)) {
+        if (MethodDecl->isCopyAssignmentOperator())
+          DefineImplicitCopyAssignment(Loc, MethodDecl);
+        else
+          DefineImplicitMoveAssignment(Loc, MethodDecl);
+      }
     } else if (MethodDecl->isVirtual())
       MarkVTableUsed(Loc, MethodDecl->getParent());
   }
@@ -9147,7 +9432,7 @@ void Sema::MarkDeclarationsReferencedInExpr(Expr *E) {
 /// behavior of a program, such as passing a non-POD value through an ellipsis.
 /// Failure to do so will likely result in spurious diagnostics or failures
 /// during overload resolution or within sizeof/alignof/typeof/typeid.
-bool Sema::DiagRuntimeBehavior(SourceLocation Loc, const Stmt *stmt,
+bool Sema::DiagRuntimeBehavior(SourceLocation Loc, const Stmt *Statement,
                                const PartialDiagnostic &PD) {
   switch (ExprEvalContexts.back().Context) {
   case Unevaluated:
@@ -9156,9 +9441,9 @@ bool Sema::DiagRuntimeBehavior(SourceLocation Loc, const Stmt *stmt,
 
   case PotentiallyEvaluated:
   case PotentiallyEvaluatedIfUsed:
-    if (stmt && getCurFunctionOrMethodDecl()) {
+    if (Statement && getCurFunctionOrMethodDecl()) {
       FunctionScopes.back()->PossiblyUnreachableDiags.
-        push_back(sema::PossiblyUnreachableDiag(PD, Loc, stmt));
+        push_back(sema::PossiblyUnreachableDiag(PD, Loc, Statement));
     }
     else
       Diag(Loc, PD);
@@ -9203,8 +9488,7 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
   unsigned diagnostic = diag::warn_condition_is_assignment;
   bool IsOrAssign = false;
 
-  if (isa<BinaryOperator>(E)) {
-    BinaryOperator *Op = cast<BinaryOperator>(E);
+  if (BinaryOperator *Op = dyn_cast<BinaryOperator>(E)) {
     if (Op->getOpcode() != BO_Assign && Op->getOpcode() != BO_OrAssign)
       return;
 
@@ -9225,8 +9509,7 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
     }
 
     Loc = Op->getOperatorLoc();
-  } else if (isa<CXXOperatorCallExpr>(E)) {
-    CXXOperatorCallExpr *Op = cast<CXXOperatorCallExpr>(E);
+  } else if (CXXOperatorCallExpr *Op = dyn_cast<CXXOperatorCallExpr>(E)) {
     if (Op->getOperator() != OO_Equal && Op->getOperator() != OO_PipeEqual)
       return;
 
@@ -9255,16 +9538,16 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
 
 /// \brief Redundant parentheses over an equality comparison can indicate
 /// that the user intended an assignment used as condition.
-void Sema::DiagnoseEqualityWithExtraParens(ParenExpr *parenE) {
+void Sema::DiagnoseEqualityWithExtraParens(ParenExpr *ParenE) {
   // Don't warn if the parens came from a macro.
-  SourceLocation parenLoc = parenE->getLocStart();
+  SourceLocation parenLoc = ParenE->getLocStart();
   if (parenLoc.isInvalid() || parenLoc.isMacroID())
     return;
   // Don't warn for dependent expressions.
-  if (parenE->isTypeDependent())
+  if (ParenE->isTypeDependent())
     return;
 
-  Expr *E = parenE->IgnoreParens();
+  Expr *E = ParenE->IgnoreParens();
 
   if (BinaryOperator *opE = dyn_cast<BinaryOperator>(E))
     if (opE->getOpcode() == BO_EQ &&
@@ -9274,8 +9557,8 @@ void Sema::DiagnoseEqualityWithExtraParens(ParenExpr *parenE) {
       
       Diag(Loc, diag::warn_equality_with_extra_parens) << E->getSourceRange();
       Diag(Loc, diag::note_equality_comparison_silence)
-        << FixItHint::CreateRemoval(parenE->getSourceRange().getBegin())
-        << FixItHint::CreateRemoval(parenE->getSourceRange().getEnd());
+        << FixItHint::CreateRemoval(ParenE->getSourceRange().getBegin())
+        << FixItHint::CreateRemoval(ParenE->getSourceRange().getEnd());
       Diag(Loc, diag::note_equality_comparison_to_assign)
         << FixItHint::CreateReplacement(Loc, "=");
     }
@@ -9311,11 +9594,11 @@ ExprResult Sema::CheckBooleanCondition(Expr *E, SourceLocation Loc) {
 }
 
 ExprResult Sema::ActOnBooleanCondition(Scope *S, SourceLocation Loc,
-                                       Expr *Sub) {
-  if (!Sub)
+                                       Expr *SubExpr) {
+  if (!SubExpr)
     return ExprError();
 
-  return CheckBooleanCondition(Sub, Loc);
+  return CheckBooleanCondition(SubExpr, Loc);
 }
 
 namespace {
@@ -9333,76 +9616,76 @@ namespace {
       return ExprError();
     }
 
-    ExprResult VisitExpr(Expr *expr) {
-      S.Diag(expr->getExprLoc(), diag::err_unsupported_unknown_any_call)
-        << expr->getSourceRange();
+    ExprResult VisitExpr(Expr *E) {
+      S.Diag(E->getExprLoc(), diag::err_unsupported_unknown_any_call)
+        << E->getSourceRange();
       return ExprError();
     }
 
     /// Rebuild an expression which simply semantically wraps another
     /// expression which it shares the type and value kind of.
-    template <class T> ExprResult rebuildSugarExpr(T *expr) {
-      ExprResult subResult = Visit(expr->getSubExpr());
-      if (subResult.isInvalid()) return ExprError();
+    template <class T> ExprResult rebuildSugarExpr(T *E) {
+      ExprResult SubResult = Visit(E->getSubExpr());
+      if (SubResult.isInvalid()) return ExprError();
 
-      Expr *subExpr = subResult.take();
-      expr->setSubExpr(subExpr);
-      expr->setType(subExpr->getType());
-      expr->setValueKind(subExpr->getValueKind());
-      assert(expr->getObjectKind() == OK_Ordinary);
-      return expr;
+      Expr *SubExpr = SubResult.take();
+      E->setSubExpr(SubExpr);
+      E->setType(SubExpr->getType());
+      E->setValueKind(SubExpr->getValueKind());
+      assert(E->getObjectKind() == OK_Ordinary);
+      return E;
     }
 
-    ExprResult VisitParenExpr(ParenExpr *paren) {
-      return rebuildSugarExpr(paren);
+    ExprResult VisitParenExpr(ParenExpr *E) {
+      return rebuildSugarExpr(E);
     }
 
-    ExprResult VisitUnaryExtension(UnaryOperator *op) {
-      return rebuildSugarExpr(op);
+    ExprResult VisitUnaryExtension(UnaryOperator *E) {
+      return rebuildSugarExpr(E);
     }
 
-    ExprResult VisitUnaryAddrOf(UnaryOperator *op) {
-      ExprResult subResult = Visit(op->getSubExpr());
-      if (subResult.isInvalid()) return ExprError();
+    ExprResult VisitUnaryAddrOf(UnaryOperator *E) {
+      ExprResult SubResult = Visit(E->getSubExpr());
+      if (SubResult.isInvalid()) return ExprError();
 
-      Expr *subExpr = subResult.take();
-      op->setSubExpr(subExpr);
-      op->setType(S.Context.getPointerType(subExpr->getType()));
-      assert(op->getValueKind() == VK_RValue);
-      assert(op->getObjectKind() == OK_Ordinary);
-      return op;
+      Expr *SubExpr = SubResult.take();
+      E->setSubExpr(SubExpr);
+      E->setType(S.Context.getPointerType(SubExpr->getType()));
+      assert(E->getValueKind() == VK_RValue);
+      assert(E->getObjectKind() == OK_Ordinary);
+      return E;
     }
 
-    ExprResult resolveDecl(Expr *expr, ValueDecl *decl) {
-      if (!isa<FunctionDecl>(decl)) return VisitExpr(expr);
+    ExprResult resolveDecl(Expr *E, ValueDecl *VD) {
+      if (!isa<FunctionDecl>(VD)) return VisitExpr(E);
 
-      expr->setType(decl->getType());
+      E->setType(VD->getType());
 
-      assert(expr->getValueKind() == VK_RValue);
+      assert(E->getValueKind() == VK_RValue);
       if (S.getLangOptions().CPlusPlus &&
-          !(isa<CXXMethodDecl>(decl) &&
-            cast<CXXMethodDecl>(decl)->isInstance()))
-        expr->setValueKind(VK_LValue);
+          !(isa<CXXMethodDecl>(VD) &&
+            cast<CXXMethodDecl>(VD)->isInstance()))
+        E->setValueKind(VK_LValue);
 
-      return expr;
+      return E;
     }
 
-    ExprResult VisitMemberExpr(MemberExpr *mem) {
-      return resolveDecl(mem, mem->getMemberDecl());
+    ExprResult VisitMemberExpr(MemberExpr *E) {
+      return resolveDecl(E, E->getMemberDecl());
     }
 
-    ExprResult VisitDeclRefExpr(DeclRefExpr *ref) {
-      return resolveDecl(ref, ref->getDecl());
+    ExprResult VisitDeclRefExpr(DeclRefExpr *E) {
+      return resolveDecl(E, E->getDecl());
     }
   };
 }
 
 /// Given a function expression of unknown-any type, try to rebuild it
 /// to have a function type.
-static ExprResult rebuildUnknownAnyFunction(Sema &S, Expr *fn) {
-  ExprResult result = RebuildUnknownAnyFunction(S).Visit(fn);
-  if (result.isInvalid()) return ExprError();
-  return S.DefaultFunctionArrayConversion(result.take());
+static ExprResult rebuildUnknownAnyFunction(Sema &S, Expr *FunctionExpr) {
+  ExprResult Result = RebuildUnknownAnyFunction(S).Visit(FunctionExpr);
+  if (Result.isInvalid()) return ExprError();
+  return S.DefaultFunctionArrayConversion(Result.take());
 }
 
 namespace {
@@ -9418,80 +9701,80 @@ namespace {
     /// The current destination type.
     QualType DestType;
 
-    RebuildUnknownAnyExpr(Sema &S, QualType castType)
-      : S(S), DestType(castType) {}
+    RebuildUnknownAnyExpr(Sema &S, QualType CastType)
+      : S(S), DestType(CastType) {}
 
     ExprResult VisitStmt(Stmt *S) {
       llvm_unreachable("unexpected statement!");
       return ExprError();
     }
 
-    ExprResult VisitExpr(Expr *expr) {
-      S.Diag(expr->getExprLoc(), diag::err_unsupported_unknown_any_expr)
-        << expr->getSourceRange();
+    ExprResult VisitExpr(Expr *E) {
+      S.Diag(E->getExprLoc(), diag::err_unsupported_unknown_any_expr)
+        << E->getSourceRange();
       return ExprError();
     }
 
-    ExprResult VisitCallExpr(CallExpr *call);
-    ExprResult VisitObjCMessageExpr(ObjCMessageExpr *message);
+    ExprResult VisitCallExpr(CallExpr *E);
+    ExprResult VisitObjCMessageExpr(ObjCMessageExpr *E);
 
     /// Rebuild an expression which simply semantically wraps another
     /// expression which it shares the type and value kind of.
-    template <class T> ExprResult rebuildSugarExpr(T *expr) {
-      ExprResult subResult = Visit(expr->getSubExpr());
-      if (subResult.isInvalid()) return ExprError();
-      Expr *subExpr = subResult.take();
-      expr->setSubExpr(subExpr);
-      expr->setType(subExpr->getType());
-      expr->setValueKind(subExpr->getValueKind());
-      assert(expr->getObjectKind() == OK_Ordinary);
-      return expr;
+    template <class T> ExprResult rebuildSugarExpr(T *E) {
+      ExprResult SubResult = Visit(E->getSubExpr());
+      if (SubResult.isInvalid()) return ExprError();
+      Expr *SubExpr = SubResult.take();
+      E->setSubExpr(SubExpr);
+      E->setType(SubExpr->getType());
+      E->setValueKind(SubExpr->getValueKind());
+      assert(E->getObjectKind() == OK_Ordinary);
+      return E;
     }
 
-    ExprResult VisitParenExpr(ParenExpr *paren) {
-      return rebuildSugarExpr(paren);
+    ExprResult VisitParenExpr(ParenExpr *E) {
+      return rebuildSugarExpr(E);
     }
 
-    ExprResult VisitUnaryExtension(UnaryOperator *op) {
-      return rebuildSugarExpr(op);
+    ExprResult VisitUnaryExtension(UnaryOperator *E) {
+      return rebuildSugarExpr(E);
     }
 
-    ExprResult VisitUnaryAddrOf(UnaryOperator *op) {
-      const PointerType *ptr = DestType->getAs<PointerType>();
-      if (!ptr) {
-        S.Diag(op->getOperatorLoc(), diag::err_unknown_any_addrof)
-          << op->getSourceRange();
+    ExprResult VisitUnaryAddrOf(UnaryOperator *E) {
+      const PointerType *Ptr = DestType->getAs<PointerType>();
+      if (!Ptr) {
+        S.Diag(E->getOperatorLoc(), diag::err_unknown_any_addrof)
+          << E->getSourceRange();
         return ExprError();
       }
-      assert(op->getValueKind() == VK_RValue);
-      assert(op->getObjectKind() == OK_Ordinary);
-      op->setType(DestType);
+      assert(E->getValueKind() == VK_RValue);
+      assert(E->getObjectKind() == OK_Ordinary);
+      E->setType(DestType);
 
       // Build the sub-expression as if it were an object of the pointee type.
-      DestType = ptr->getPointeeType();
-      ExprResult subResult = Visit(op->getSubExpr());
-      if (subResult.isInvalid()) return ExprError();
-      op->setSubExpr(subResult.take());
-      return op;
+      DestType = Ptr->getPointeeType();
+      ExprResult SubResult = Visit(E->getSubExpr());
+      if (SubResult.isInvalid()) return ExprError();
+      E->setSubExpr(SubResult.take());
+      return E;
     }
 
-    ExprResult VisitImplicitCastExpr(ImplicitCastExpr *ice);
+    ExprResult VisitImplicitCastExpr(ImplicitCastExpr *E);
 
-    ExprResult resolveDecl(Expr *expr, ValueDecl *decl);
+    ExprResult resolveDecl(Expr *E, ValueDecl *VD);
 
-    ExprResult VisitMemberExpr(MemberExpr *mem) {
-      return resolveDecl(mem, mem->getMemberDecl());
+    ExprResult VisitMemberExpr(MemberExpr *E) {
+      return resolveDecl(E, E->getMemberDecl());
     }
 
-    ExprResult VisitDeclRefExpr(DeclRefExpr *ref) {
-      return resolveDecl(ref, ref->getDecl());
+    ExprResult VisitDeclRefExpr(DeclRefExpr *E) {
+      return resolveDecl(E, E->getDecl());
     }
   };
 }
 
 /// Rebuilds a call expression which yielded __unknown_anytype.
-ExprResult RebuildUnknownAnyExpr::VisitCallExpr(CallExpr *call) {
-  Expr *callee = call->getCallee();
+ExprResult RebuildUnknownAnyExpr::VisitCallExpr(CallExpr *E) {
+  Expr *CalleeExpr = E->getCallee();
 
   enum FnKind {
     FK_MemberFunction,
@@ -9499,49 +9782,49 @@ ExprResult RebuildUnknownAnyExpr::VisitCallExpr(CallExpr *call) {
     FK_BlockPointer
   };
 
-  FnKind kind;
-  QualType type = callee->getType();
-  if (type == S.Context.BoundMemberTy) {
-    assert(isa<CXXMemberCallExpr>(call) || isa<CXXOperatorCallExpr>(call));    
-    kind = FK_MemberFunction;
-    type = Expr::findBoundMemberType(callee);
-  } else if (const PointerType *ptr = type->getAs<PointerType>()) {
-    type = ptr->getPointeeType();
-    kind = FK_FunctionPointer;
+  FnKind Kind;
+  QualType CalleeType = CalleeExpr->getType();
+  if (CalleeType == S.Context.BoundMemberTy) {
+    assert(isa<CXXMemberCallExpr>(E) || isa<CXXOperatorCallExpr>(E));
+    Kind = FK_MemberFunction;
+    CalleeType = Expr::findBoundMemberType(CalleeExpr);
+  } else if (const PointerType *Ptr = CalleeType->getAs<PointerType>()) {
+    CalleeType = Ptr->getPointeeType();
+    Kind = FK_FunctionPointer;
   } else {
-    type = type->castAs<BlockPointerType>()->getPointeeType();
-    kind = FK_BlockPointer;
+    CalleeType = CalleeType->castAs<BlockPointerType>()->getPointeeType();
+    Kind = FK_BlockPointer;
   }
-  const FunctionType *fnType = type->castAs<FunctionType>();
+  const FunctionType *FnType = CalleeType->castAs<FunctionType>();
 
   // Verify that this is a legal result type of a function.
   if (DestType->isArrayType() || DestType->isFunctionType()) {
     unsigned diagID = diag::err_func_returning_array_function;
-    if (kind == FK_BlockPointer)
+    if (Kind == FK_BlockPointer)
       diagID = diag::err_block_returning_array_function;
 
-    S.Diag(call->getExprLoc(), diagID)
+    S.Diag(E->getExprLoc(), diagID)
       << DestType->isFunctionType() << DestType;
     return ExprError();
   }
 
   // Otherwise, go ahead and set DestType as the call's result.
-  call->setType(DestType.getNonLValueExprType(S.Context));
-  call->setValueKind(Expr::getValueKindForType(DestType));
-  assert(call->getObjectKind() == OK_Ordinary);
+  E->setType(DestType.getNonLValueExprType(S.Context));
+  E->setValueKind(Expr::getValueKindForType(DestType));
+  assert(E->getObjectKind() == OK_Ordinary);
 
   // Rebuild the function type, replacing the result type with DestType.
-  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType))
+  if (const FunctionProtoType *Proto = dyn_cast<FunctionProtoType>(FnType))
     DestType = S.Context.getFunctionType(DestType,
-                                         proto->arg_type_begin(),
-                                         proto->getNumArgs(),
-                                         proto->getExtProtoInfo());
+                                         Proto->arg_type_begin(),
+                                         Proto->getNumArgs(),
+                                         Proto->getExtProtoInfo());
   else
     DestType = S.Context.getFunctionNoProtoType(DestType,
-                                                fnType->getExtInfo());
+                                                FnType->getExtInfo());
 
   // Rebuild the appropriate pointer-to-function type.
-  switch (kind) {
+  switch (Kind) { 
   case FK_MemberFunction:
     // Nothing to do.
     break;
@@ -9556,121 +9839,131 @@ ExprResult RebuildUnknownAnyExpr::VisitCallExpr(CallExpr *call) {
   }
 
   // Finally, we can recurse.
-  ExprResult calleeResult = Visit(callee);
-  if (!calleeResult.isUsable()) return ExprError();
-  call->setCallee(calleeResult.take());
+  ExprResult CalleeResult = Visit(CalleeExpr);
+  if (!CalleeResult.isUsable()) return ExprError();
+  E->setCallee(CalleeResult.take());
 
   // Bind a temporary if necessary.
-  return S.MaybeBindToTemporary(call);
+  return S.MaybeBindToTemporary(E);
 }
 
-ExprResult RebuildUnknownAnyExpr::VisitObjCMessageExpr(ObjCMessageExpr *msg) {
+ExprResult RebuildUnknownAnyExpr::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   // Verify that this is a legal result type of a call.
   if (DestType->isArrayType() || DestType->isFunctionType()) {
-    S.Diag(msg->getExprLoc(), diag::err_func_returning_array_function)
+    S.Diag(E->getExprLoc(), diag::err_func_returning_array_function)
       << DestType->isFunctionType() << DestType;
     return ExprError();
   }
 
   // Rewrite the method result type if available.
-  if (ObjCMethodDecl *method = msg->getMethodDecl()) {
-    assert(method->getResultType() == S.Context.UnknownAnyTy);
-    method->setResultType(DestType);
+  if (ObjCMethodDecl *Method = E->getMethodDecl()) {
+    assert(Method->getResultType() == S.Context.UnknownAnyTy);
+    Method->setResultType(DestType);
   }
 
   // Change the type of the message.
-  msg->setType(DestType.getNonReferenceType());
-  msg->setValueKind(Expr::getValueKindForType(DestType));
+  E->setType(DestType.getNonReferenceType());
+  E->setValueKind(Expr::getValueKindForType(DestType));
 
-  return S.MaybeBindToTemporary(msg);
+  return S.MaybeBindToTemporary(E);
 }
 
-ExprResult RebuildUnknownAnyExpr::VisitImplicitCastExpr(ImplicitCastExpr *ice) {
+ExprResult RebuildUnknownAnyExpr::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   // The only case we should ever see here is a function-to-pointer decay.
-  assert(ice->getCastKind() == CK_FunctionToPointerDecay);
-  assert(ice->getValueKind() == VK_RValue);
-  assert(ice->getObjectKind() == OK_Ordinary);
+  assert(E->getCastKind() == CK_FunctionToPointerDecay);
+  assert(E->getValueKind() == VK_RValue);
+  assert(E->getObjectKind() == OK_Ordinary);
 
-  ice->setType(DestType);
+  E->setType(DestType);
 
   // Rebuild the sub-expression as the pointee (function) type.
   DestType = DestType->castAs<PointerType>()->getPointeeType();
 
-  ExprResult result = Visit(ice->getSubExpr());
-  if (!result.isUsable()) return ExprError();
+  ExprResult Result = Visit(E->getSubExpr());
+  if (!Result.isUsable()) return ExprError();
 
-  ice->setSubExpr(result.take());
-  return S.Owned(ice);
+  E->setSubExpr(Result.take());
+  return S.Owned(E);
 }
 
-ExprResult RebuildUnknownAnyExpr::resolveDecl(Expr *expr, ValueDecl *decl) {
-  ExprValueKind valueKind = VK_LValue;
-  QualType type = DestType;
+ExprResult RebuildUnknownAnyExpr::resolveDecl(Expr *E, ValueDecl *VD) {
+  ExprValueKind ValueKind = VK_LValue;
+  QualType Type = DestType;
 
   // We know how to make this work for certain kinds of decls:
 
   //  - functions
-  if (FunctionDecl *fn = dyn_cast<FunctionDecl>(decl)) {
-    // This is true because FunctionDecls must always have function
-    // type, so we can't be resolving the entire thing at once.
-    assert(type->isFunctionType());
+  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(VD)) {
+    if (const PointerType *Ptr = Type->getAs<PointerType>()) {
+      DestType = Ptr->getPointeeType();
+      ExprResult Result = resolveDecl(E, VD);
+      if (Result.isInvalid()) return ExprError();
+      return S.ImpCastExprToType(Result.take(), Type,
+                                 CK_FunctionToPointerDecay, VK_RValue);
+    }
 
-    if (CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(fn))
-      if (method->isInstance()) {
-        valueKind = VK_RValue;
-        type = S.Context.BoundMemberTy;
+    if (!Type->isFunctionType()) {
+      S.Diag(E->getExprLoc(), diag::err_unknown_any_function)
+        << VD << E->getSourceRange();
+      return ExprError();
+    }
+
+    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
+      if (MD->isInstance()) {
+        ValueKind = VK_RValue;
+        Type = S.Context.BoundMemberTy;
       }
 
     // Function references aren't l-values in C.
     if (!S.getLangOptions().CPlusPlus)
-      valueKind = VK_RValue;
+      ValueKind = VK_RValue;
 
   //  - variables
-  } else if (isa<VarDecl>(decl)) {
-    if (const ReferenceType *refTy = type->getAs<ReferenceType>()) {
-      type = refTy->getPointeeType();
-    } else if (type->isFunctionType()) {
-      S.Diag(expr->getExprLoc(), diag::err_unknown_any_var_function_type)
-        << decl << expr->getSourceRange();
+  } else if (isa<VarDecl>(VD)) {
+    if (const ReferenceType *RefTy = Type->getAs<ReferenceType>()) {
+      Type = RefTy->getPointeeType();
+    } else if (Type->isFunctionType()) {
+      S.Diag(E->getExprLoc(), diag::err_unknown_any_var_function_type)
+        << VD << E->getSourceRange();
       return ExprError();
     }
 
   //  - nothing else
   } else {
-    S.Diag(expr->getExprLoc(), diag::err_unsupported_unknown_any_decl)
-      << decl << expr->getSourceRange();
+    S.Diag(E->getExprLoc(), diag::err_unsupported_unknown_any_decl)
+      << VD << E->getSourceRange();
     return ExprError();
   }
 
-  decl->setType(DestType);
-  expr->setType(type);
-  expr->setValueKind(valueKind);
-  return S.Owned(expr);
+  VD->setType(DestType);
+  E->setType(Type);
+  E->setValueKind(ValueKind);
+  return S.Owned(E);
 }
 
 /// Check a cast of an unknown-any type.  We intentionally only
 /// trigger this for C-style casts.
-ExprResult Sema::checkUnknownAnyCast(SourceRange typeRange, QualType castType,
-                                     Expr *castExpr, CastKind &castKind,
-                                     ExprValueKind &VK, CXXCastPath &path) {
+ExprResult Sema::checkUnknownAnyCast(SourceRange TypeRange, QualType CastType,
+                                     Expr *CastExpr, CastKind &CastKind,
+                                     ExprValueKind &VK, CXXCastPath &Path) {
   // Rewrite the casted expression from scratch.
-  ExprResult result = RebuildUnknownAnyExpr(*this, castType).Visit(castExpr);
+  ExprResult result = RebuildUnknownAnyExpr(*this, CastType).Visit(CastExpr);
   if (!result.isUsable()) return ExprError();
 
-  castExpr = result.take();
-  VK = castExpr->getValueKind();
-  castKind = CK_NoOp;
+  CastExpr = result.take();
+  VK = CastExpr->getValueKind();
+  CastKind = CK_NoOp;
 
-  return castExpr;
+  return CastExpr;
 }
 
-static ExprResult diagnoseUnknownAnyExpr(Sema &S, Expr *e) {
-  Expr *orig = e;
+static ExprResult diagnoseUnknownAnyExpr(Sema &S, Expr *E) {
+  Expr *orig = E;
   unsigned diagID = diag::err_uncasted_use_of_unknown_any;
   while (true) {
-    e = e->IgnoreParenImpCasts();
-    if (CallExpr *call = dyn_cast<CallExpr>(e)) {
-      e = call->getCallee();
+    E = E->IgnoreParenImpCasts();
+    if (CallExpr *call = dyn_cast<CallExpr>(E)) {
+      E = call->getCallee();
       diagID = diag::err_uncasted_call_of_unknown_any;
     } else {
       break;
@@ -9679,20 +9972,25 @@ static ExprResult diagnoseUnknownAnyExpr(Sema &S, Expr *e) {
 
   SourceLocation loc;
   NamedDecl *d;
-  if (DeclRefExpr *ref = dyn_cast<DeclRefExpr>(e)) {
+  if (DeclRefExpr *ref = dyn_cast<DeclRefExpr>(E)) {
     loc = ref->getLocation();
     d = ref->getDecl();
-  } else if (MemberExpr *mem = dyn_cast<MemberExpr>(e)) {
+  } else if (MemberExpr *mem = dyn_cast<MemberExpr>(E)) {
     loc = mem->getMemberLoc();
     d = mem->getMemberDecl();
-  } else if (ObjCMessageExpr *msg = dyn_cast<ObjCMessageExpr>(e)) {
+  } else if (ObjCMessageExpr *msg = dyn_cast<ObjCMessageExpr>(E)) {
     diagID = diag::err_uncasted_call_of_unknown_any;
-    loc = msg->getSelectorLoc();
+    loc = msg->getSelectorStartLoc();
     d = msg->getMethodDecl();
-    assert(d && "unknown method returning __unknown_any?");
+    if (!d) {
+      S.Diag(loc, diag::err_uncasted_send_to_unknown_any_method)
+        << static_cast<unsigned>(msg->isClassMessage()) << msg->getSelector()
+        << orig->getSourceRange();
+      return ExprError();
+    }
   } else {
-    S.Diag(e->getExprLoc(), diag::err_unsupported_unknown_any_expr)
-      << e->getSourceRange();
+    S.Diag(E->getExprLoc(), diag::err_unsupported_unknown_any_expr)
+      << E->getSourceRange();
     return ExprError();
   }
 
@@ -9709,17 +10007,27 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
   QualType type = E->getType();
 
   // Overloaded expressions.
-  if (type == Context.OverloadTy)
-    return ResolveAndFixSingleFunctionTemplateSpecialization(E, false, true,
-                                                           E->getSourceRange(),
-                                                             QualType(),
-                                                   diag::err_ovl_unresolvable);
+  if (type == Context.OverloadTy) {
+    // Try to resolve a single function template specialization.
+    // This is obligatory.
+    ExprResult result = Owned(E);
+    if (ResolveAndFixSingleFunctionTemplateSpecialization(result, false)) {
+      return result;
+
+    // If that failed, try to recover with a call.
+    } else {
+      tryToRecoverWithCall(result, PDiag(diag::err_ovl_unresolvable),
+                           /*complain*/ true);
+      return result;
+    }
+  }
 
   // Bound member functions.
   if (type == Context.BoundMemberTy) {
-    Diag(E->getLocStart(), diag::err_invalid_use_of_bound_member_func)
-      << E->getSourceRange();
-    return ExprError();
+    ExprResult result = Owned(E);
+    tryToRecoverWithCall(result, PDiag(diag::err_bound_member_function),
+                         /*complain*/ true);
+    return result;
   }    
 
   // Expressions of unknown type.
@@ -9730,10 +10038,10 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
   return Owned(E);
 }
 
-bool Sema::CheckCaseExpression(Expr *expr) {
-  if (expr->isTypeDependent())
+bool Sema::CheckCaseExpression(Expr *E) {
+  if (E->isTypeDependent())
     return true;
-  if (expr->isValueDependent() || expr->isIntegerConstantExpr(Context))
-    return expr->getType()->isIntegralOrEnumerationType();
+  if (E->isValueDependent() || E->isIntegerConstantExpr(Context))
+    return E->getType()->isIntegralOrEnumerationType();
   return false;
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExprCXX.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExprCXX.cpp
index e804fcd60c74..3300444a1c1e 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaExprCXX.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaExprCXX.cpp
@@ -295,6 +295,12 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
                                 SourceLocation RParenLoc) {
   bool isUnevaluatedOperand = true;
   if (E && !E->isTypeDependent()) {
+    if (E->getType()->isPlaceholderType()) {
+      ExprResult result = CheckPlaceholderExpr(E);
+      if (result.isInvalid()) return ExprError();
+      E = result.take();
+    }
+
     QualType T = E->getType();
     if (const RecordType *RecordT = T->getAs<RecordType>()) {
       CXXRecordDecl *RecordD = cast<CXXRecordDecl>(RecordT->getDecl());
@@ -325,7 +331,7 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
     QualType UnqualT = Context.getUnqualifiedArrayType(T, Quals);
     if (!Context.hasSameType(T, UnqualT)) {
       T = UnqualT;
-      E = ImpCastExprToType(E, UnqualT, CK_NoOp, CastCategory(E)).take();
+      E = ImpCastExprToType(E, UnqualT, CK_NoOp, E->getValueKind()).take();
     }
   }
 
@@ -545,7 +551,7 @@ ExprResult Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *E,
   //   or "pointer to function returning T", [...]
   if (E->getType().hasQualifiers())
     E = ImpCastExprToType(E, E->getType().getUnqualifiedType(), CK_NoOp,
-                      CastCategory(E)).take();
+                          E->getValueKind()).take();
 
   ExprResult Res = DefaultFunctionArrayConversion(E);
   if (Res.isInvalid())
@@ -739,27 +745,10 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
   // If the expression list is a single expression, the type conversion
   // expression is equivalent (in definedness, and if defined in meaning) to the
   // corresponding cast expression.
-  //
   if (NumExprs == 1) {
-    CastKind Kind = CK_Invalid;
-    ExprValueKind VK = VK_RValue;
-    CXXCastPath BasePath;
-    ExprResult CastExpr =
-      CheckCastTypes(TInfo->getTypeLoc().getBeginLoc(),
-                     TInfo->getTypeLoc().getSourceRange(), Ty, Exprs[0],
-                     Kind, VK, BasePath,
-                     /*FunctionalStyle=*/true);
-    if (CastExpr.isInvalid())
-      return ExprError();
-    Exprs[0] = CastExpr.take();
-
+    Expr *Arg = Exprs[0];
     exprs.release();
-
-    return Owned(CXXFunctionalCastExpr::Create(Context,
-                                               Ty.getNonLValueExprType(Context),
-                                               VK, TInfo, TyBeginLoc, Kind,
-                                               Exprs[0], &BasePath,
-                                               RParenLoc));
+    return BuildCXXFunctionalCastExpr(TInfo, LParenLoc, Arg, RParenLoc);
   }
 
   InitializedEntity Entity = InitializedEntity::InitializeTemporary(TInfo);
@@ -1058,7 +1047,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     UsualArrayDeleteWantsSize
       = doesUsualArrayDeleteWantSize(*this, StartLoc, AllocType);
 
-  llvm::SmallVector<Expr *, 8> AllPlaceArgs;
+  SmallVector<Expr *, 8> AllPlaceArgs;
   if (OperatorNew) {
     // Add default arguments, if any.
     const FunctionProtoType *Proto =
@@ -1079,6 +1068,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
   bool Init = ConstructorLParen.isValid();
   // --- Choosing a constructor ---
   CXXConstructorDecl *Constructor = 0;
+  bool HadMultipleCandidates = false;
   Expr **ConsArgs = (Expr**)ConstructorArgs.get();
   unsigned NumConsArgs = ConstructorArgs.size();
   ASTOwningVector<Expr*> ConvertedConstructorArgs(*this);
@@ -1125,6 +1115,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
       if (CXXConstructExpr *Construct
                     = dyn_cast<CXXConstructExpr>(FullInitExpr)) {
         Constructor = Construct->getConstructor();
+        HadMultipleCandidates = Construct->hadMultipleCandidates();
         for (CXXConstructExpr::arg_iterator A = Construct->arg_begin(),
                                          AEnd = Construct->arg_end();
              A != AEnd; ++A)
@@ -1166,7 +1157,9 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
   return Owned(new (Context) CXXNewExpr(Context, UseGlobal, OperatorNew,
                                         PlaceArgs, NumPlaceArgs, TypeIdParens,
                                         ArraySize, Constructor, Init,
-                                        ConsArgs, NumConsArgs, OperatorDelete,
+                                        ConsArgs, NumConsArgs,
+                                        HadMultipleCandidates,
+                                        OperatorDelete,
                                         UsualArrayDeleteWantsSize,
                                         ResultType, AllocTypeInfo,
                                         StartLoc,
@@ -1246,12 +1239,12 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
   // 3) The first argument is always size_t. Append the arguments from the
   //   placement form.
 
-  llvm::SmallVector<Expr*, 8> AllocArgs(1 + NumPlaceArgs);
+  SmallVector<Expr*, 8> AllocArgs(1 + NumPlaceArgs);
   // We don't care about the actual value of this argument.
   // FIXME: Should the Sema create the expression and embed it in the syntax
   // tree? Or should the consumer just recalculate the value?
   IntegerLiteral Size(Context, llvm::APInt::getNullValue(
-                      Context.Target.getPointerWidth(0)),
+                      Context.getTargetInfo().getPointerWidth(0)),
                       Context.getSizeType(),
                       SourceLocation());
   AllocArgs[0] = &Size;
@@ -1325,7 +1318,7 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
 
   FoundDelete.suppressDiagnostics();
 
-  llvm::SmallVector<std::pair<DeclAccessPair,FunctionDecl*>, 2> Matches;
+  SmallVector<std::pair<DeclAccessPair,FunctionDecl*>, 2> Matches;
 
   // Whether we're looking for a placement operator delete is dictated
   // by whether we selected a placement operator new, not by whether
@@ -1352,7 +1345,7 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
       const FunctionProtoType *Proto
         = OperatorNew->getType()->getAs<FunctionProtoType>();
 
-      llvm::SmallVector<QualType, 4> ArgTypes;
+      SmallVector<QualType, 4> ArgTypes;
       ArgTypes.push_back(Context.VoidPtrTy);
       for (unsigned I = 1, N = Proto->getNumArgs(); I < N; ++I)
         ArgTypes.push_back(Proto->getArgType(I));
@@ -1525,8 +1518,7 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
     return true;
   }
   }
-  assert(false && "Unreachable, bad result from BestViableFunction");
-  return true;
+  llvm_unreachable("Unreachable, bad result from BestViableFunction");
 }
 
 
@@ -1671,7 +1663,7 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
                                            SourceLocation(), 0,
                                            Argument, /*TInfo=*/0,
                                            SC_None, SC_None, 0);
-  Alloc->setParams(&Param, 1);
+  Alloc->setParams(Param);
 
   // FIXME: Also add this declaration to the IdentifierResolver, but
   // make sure it is at the end of the chain to coincide with the
@@ -1691,7 +1683,7 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
 
   Found.suppressDiagnostics();
 
-  llvm::SmallVector<DeclAccessPair,4> Matches;
+  SmallVector<DeclAccessPair,4> Matches;
   for (LookupResult::iterator F = Found.begin(), FEnd = Found.end();
        F != FEnd; ++F) {
     NamedDecl *ND = (*F)->getUnderlyingDecl();
@@ -1727,7 +1719,7 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
       Diag(StartLoc, diag::err_ambiguous_suitable_delete_member_function_found)
         << Name << RD;
 
-      for (llvm::SmallVectorImpl<DeclAccessPair>::iterator
+      for (SmallVectorImpl<DeclAccessPair>::iterator
              F = Matches.begin(), FEnd = Matches.end(); F != FEnd; ++F)
         Diag((*F)->getUnderlyingDecl()->getLocation(),
              diag::note_member_declared_here) << Name;
@@ -1792,7 +1784,7 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
                               PDiag(diag::err_delete_incomplete_class_type)))
         return ExprError();
 
-      llvm::SmallVector<CXXConversionDecl*, 4> ObjectPtrConversions;
+      SmallVector<CXXConversionDecl*, 4> ObjectPtrConversions;
 
       CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
       const UnresolvedSetImpl *Conversions = RD->getVisibleConversionFunctions();
@@ -1840,24 +1832,32 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
         << Type << Ex.get()->getSourceRange());
 
     QualType Pointee = Type->getAs<PointerType>()->getPointeeType();
+    QualType PointeeElem = Context.getBaseElementType(Pointee);
+
+    if (unsigned AddressSpace = Pointee.getAddressSpace())
+      return Diag(Ex.get()->getLocStart(), 
+                  diag::err_address_space_qualified_delete)
+               << Pointee.getUnqualifiedType() << AddressSpace;
+
+    CXXRecordDecl *PointeeRD = 0;
     if (Pointee->isVoidType() && !isSFINAEContext()) {
       // The C++ standard bans deleting a pointer to a non-object type, which
       // effectively bans deletion of "void*". However, most compilers support
       // this, so we treat it as a warning unless we're in a SFINAE context.
       Diag(StartLoc, diag::ext_delete_void_ptr_operand)
         << Type << Ex.get()->getSourceRange();
-    } else if (Pointee->isFunctionType() || Pointee->isVoidType())
+    } else if (Pointee->isFunctionType() || Pointee->isVoidType()) {
       return ExprError(Diag(StartLoc, diag::err_delete_operand)
         << Type << Ex.get()->getSourceRange());
-    else if (!Pointee->isDependentType() &&
-             RequireCompleteType(StartLoc, Pointee,
-                                 PDiag(diag::warn_delete_incomplete)
-                                   << Ex.get()->getSourceRange()))
-      return ExprError();
-    else if (unsigned AddressSpace = Pointee.getAddressSpace())
-      return Diag(Ex.get()->getLocStart(), 
-                  diag::err_address_space_qualified_delete)
-               << Pointee.getUnqualifiedType() << AddressSpace;
+    } else if (!Pointee->isDependentType()) {
+      if (!RequireCompleteType(StartLoc, Pointee,
+                               PDiag(diag::warn_delete_incomplete)
+                                 << Ex.get()->getSourceRange())) {
+        if (const RecordType *RT = PointeeElem->getAs<RecordType>())
+          PointeeRD = cast<CXXRecordDecl>(RT->getDecl());
+      }
+    }
+
     // C++ [expr.delete]p2:
     //   [Note: a pointer to a const type can be the operand of a
     //   delete-expression; it is not necessary to cast away the constness
@@ -1877,12 +1877,10 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     DeclarationName DeleteName = Context.DeclarationNames.getCXXOperatorName(
                                       ArrayForm ? OO_Array_Delete : OO_Delete);
 
-    QualType PointeeElem = Context.getBaseElementType(Pointee);
-    if (const RecordType *RT = PointeeElem->getAs<RecordType>()) {
-      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-
+    if (PointeeRD) {
       if (!UseGlobal &&
-          FindDeallocationFunction(StartLoc, RD, DeleteName, OperatorDelete))
+          FindDeallocationFunction(StartLoc, PointeeRD, DeleteName,
+                                   OperatorDelete))
         return ExprError();
 
       // If we're allocating an array of records, check whether the
@@ -1900,8 +1898,8 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
           UsualArrayDeleteWantsSize = (OperatorDelete->getNumParams() == 2);
       }
 
-      if (!RD->hasTrivialDestructor())
-        if (CXXDestructorDecl *Dtor = LookupDestructor(RD)) {
+      if (!PointeeRD->hasTrivialDestructor())
+        if (CXXDestructorDecl *Dtor = LookupDestructor(PointeeRD)) {
           MarkDeclarationReferenced(StartLoc,
                                     const_cast<CXXDestructorDecl*>(Dtor));
           DiagnoseUseOfDecl(Dtor, StartLoc);
@@ -1915,10 +1913,20 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
       //   behavior is undefined.
       //
       // Note: a final class cannot be derived from, no issue there
-      if (!ArrayForm && RD->isPolymorphic() && !RD->hasAttr<FinalAttr>()) {
-        CXXDestructorDecl *dtor = RD->getDestructor();
-        if (!dtor || !dtor->isVirtual())
-          Diag(StartLoc, diag::warn_delete_non_virtual_dtor) << PointeeElem;
+      if (PointeeRD->isPolymorphic() && !PointeeRD->hasAttr<FinalAttr>()) {
+        CXXDestructorDecl *dtor = PointeeRD->getDestructor();
+        if (dtor && !dtor->isVirtual()) {
+          if (PointeeRD->isAbstract()) {
+            // If the class is abstract, we warn by default, because we're
+            // sure the code has undefined behavior.
+            Diag(StartLoc, diag::warn_delete_abstract_non_virtual_dtor)
+                << PointeeElem;
+          } else if (!ArrayForm) {
+            // Otherwise, if this is not an array delete, it's a bit suspect,
+            // but not necessarily wrong.
+            Diag(StartLoc, diag::warn_delete_non_virtual_dtor) << PointeeElem;
+          }
+        }
       }
 
     } else if (getLangOptions().ObjCAutoRefCount &&
@@ -1944,9 +1952,8 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     MarkDeclarationReferenced(StartLoc, OperatorDelete);
     
     // Check access and ambiguity of operator delete and destructor.
-    if (const RecordType *RT = PointeeElem->getAs<RecordType>()) {
-      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-      if (CXXDestructorDecl *Dtor = LookupDestructor(RD)) {
+    if (PointeeRD) {
+      if (CXXDestructorDecl *Dtor = LookupDestructor(PointeeRD)) {
           CheckDestructorAccess(Ex.get()->getExprLoc(), Dtor, 
                       PDiag(diag::err_access_dtor) << PointeeElem);
       }
@@ -2026,12 +2033,20 @@ Sema::IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType) {
           = ToPtrType->getPointeeType()->getAs<BuiltinType>()) {
         // This conversion is considered only when there is an
         // explicit appropriate pointer target type (C++ 4.2p2).
-        if (!ToPtrType->getPointeeType().hasQualifiers() &&
-            ((StrLit->isWide() && ToPointeeType->isWideCharType()) ||
-             (!StrLit->isWide() &&
-              (ToPointeeType->getKind() == BuiltinType::Char_U ||
-               ToPointeeType->getKind() == BuiltinType::Char_S))))
-          return true;
+        if (!ToPtrType->getPointeeType().hasQualifiers()) {
+          switch (StrLit->getKind()) {
+            case StringLiteral::UTF8:
+            case StringLiteral::UTF16:
+            case StringLiteral::UTF32:
+              // We don't allow UTF literals to be implicitly converted
+              break;
+            case StringLiteral::Ascii:
+              return (ToPointeeType->getKind() == BuiltinType::Char_U ||
+                      ToPointeeType->getKind() == BuiltinType::Char_S);
+            case StringLiteral::Wide:
+              return ToPointeeType->isWideCharType();
+          }
+        }
       }
 
   return false;
@@ -2042,23 +2057,28 @@ static ExprResult BuildCXXCastArgument(Sema &S,
                                        QualType Ty,
                                        CastKind Kind,
                                        CXXMethodDecl *Method,
-                                       NamedDecl *FoundDecl,
+                                       DeclAccessPair FoundDecl,
+                                       bool HadMultipleCandidates,
                                        Expr *From) {
   switch (Kind) {
-  default: assert(0 && "Unhandled cast kind!");
+  default: llvm_unreachable("Unhandled cast kind!");
   case CK_ConstructorConversion: {
+    CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(Method);
     ASTOwningVector<Expr*> ConstructorArgs(S);
 
-    if (S.CompleteConstructorCall(cast<CXXConstructorDecl>(Method),
+    if (S.CompleteConstructorCall(Constructor,
                                   MultiExprArg(&From, 1),
                                   CastLoc, ConstructorArgs))
       return ExprError();
 
-    ExprResult Result =
-    S.BuildCXXConstructExpr(CastLoc, Ty, cast<CXXConstructorDecl>(Method),
-                            move_arg(ConstructorArgs),
-                            /*ZeroInit*/ false, CXXConstructExpr::CK_Complete,
-                            SourceRange());
+    S.CheckConstructorAccess(CastLoc, Constructor, Constructor->getAccess(),
+                             S.PDiag(diag::err_access_ctor));
+    
+    ExprResult Result
+      = S.BuildCXXConstructExpr(CastLoc, Ty, cast<CXXConstructorDecl>(Method),
+                                move_arg(ConstructorArgs), 
+                                HadMultipleCandidates, /*ZeroInit*/ false, 
+                                CXXConstructExpr::CK_Complete, SourceRange());
     if (Result.isInvalid())
       return ExprError();
 
@@ -2069,10 +2089,13 @@ static ExprResult BuildCXXCastArgument(Sema &S,
     assert(!From->getType()->isPointerType() && "Arg can't have pointer type!");
 
     // Create an implicit call expr that calls it.
-    ExprResult Result = S.BuildCXXMemberCallExpr(From, FoundDecl, Method);
+    ExprResult Result = S.BuildCXXMemberCallExpr(From, FoundDecl, Method,
+                                                 HadMultipleCandidates);
     if (Result.isInvalid())
       return ExprError();
 
+    S.CheckMemberOperatorAccess(CastLoc, From, /*arg*/ 0, FoundDecl);
+
     return S.MaybeBindToTemporary(Result.get());
   }
   }
@@ -2138,6 +2161,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
                                ToType.getNonReferenceType(),
                                CastKind, cast<CXXMethodDecl>(FD),
                                ICS.UserDefined.FoundConversionFunction,
+                               ICS.UserDefined.HadMultipleCandidates,
                                From);
 
       if (CastArg.isInvalid())
@@ -2156,8 +2180,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
      return ExprError();
 
   case ImplicitConversionSequence::EllipsisConversion:
-    assert(false && "Cannot perform an ellipsis conversion");
-    return Owned(From);
+    llvm_unreachable("Cannot perform an ellipsis conversion");
 
   case ImplicitConversionSequence::BadConversion:
     return ExprError();
@@ -2198,6 +2221,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
       return BuildCXXConstructExpr(/*FIXME:ConstructLoc*/SourceLocation(),
                                    ToType, SCS.CopyConstructor,
                                    move_arg(ConstructorArgs),
+                                   /*HadMultipleCandidates*/ false,
                                    /*ZeroInit*/ false,
                                    CXXConstructExpr::CK_Complete,
                                    SourceRange());
@@ -2205,6 +2229,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     return BuildCXXConstructExpr(/*FIXME:ConstructLoc*/SourceLocation(),
                                  ToType, SCS.CopyConstructor,
                                  MultiExprArg(*this, &From, 1),
+                                 /*HadMultipleCandidates*/ false,
                                  /*ZeroInit*/ false,
                                  CXXConstructExpr::CK_Complete,
                                  SourceRange());
@@ -2241,9 +2266,6 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
       if (!From->isGLValue()) break;
     }
 
-    // Check for trivial buffer overflows.
-    CheckArrayAccess(From);
-
     FromType = FromType.getUnqualifiedType();
     From = ImplicitCastExpr::Create(Context, FromType, CK_LValueToRValue,
                                     From, 0, VK_RValue);
@@ -2262,8 +2284,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     break;
 
   default:
-    assert(false && "Improper first standard conversion");
-    break;
+    llvm_unreachable("Improper first standard conversion");
   }
 
   // Perform the second implicit conversion
@@ -2340,12 +2361,12 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
         Diag(From->getSourceRange().getBegin(),
              diag::ext_typecheck_convert_incompatible_pointer)
           << ToType << From->getType() << Action
-          << From->getSourceRange();
+          << From->getSourceRange() << 0;
       else
         Diag(From->getSourceRange().getBegin(),
              diag::ext_typecheck_convert_incompatible_pointer)
           << From->getType() << ToType << Action
-          << From->getSourceRange();
+          << From->getSourceRange() << 0;
 
       if (From->getType()->isObjCObjectPointerType() &&
           ToType->isObjCObjectPointerType())
@@ -2354,20 +2375,29 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     else if (getLangOptions().ObjCAutoRefCount &&
              !CheckObjCARCUnavailableWeakConversion(ToType, 
                                                     From->getType())) {
-           if (Action == AA_Initializing)
-             Diag(From->getSourceRange().getBegin(), 
-                  diag::err_arc_weak_unavailable_assign);
-           else
-             Diag(From->getSourceRange().getBegin(),
-                  diag::err_arc_convesion_of_weak_unavailable) 
-                  << (Action == AA_Casting) << From->getType() << ToType 
-                  << From->getSourceRange();
-         }
+      if (Action == AA_Initializing)
+        Diag(From->getSourceRange().getBegin(), 
+             diag::err_arc_weak_unavailable_assign);
+      else
+        Diag(From->getSourceRange().getBegin(),
+             diag::err_arc_convesion_of_weak_unavailable) 
+          << (Action == AA_Casting) << From->getType() << ToType 
+          << From->getSourceRange();
+    }
              
     CastKind Kind = CK_Invalid;
     CXXCastPath BasePath;
     if (CheckPointerConversion(From, ToType, Kind, BasePath, CStyle))
       return ExprError();
+
+    // Make sure we extend blocks if necessary.
+    // FIXME: doing this here is really ugly.
+    if (Kind == CK_BlockPointerToObjCPointerCast) {
+      ExprResult E = From;
+      (void) PrepareCastToObjCObjectPointer(E);
+      From = E.take();
+    }
+
     From = ImpCastExprToType(From, ToType, Kind, VK_RValue, &BasePath, CCK)
              .take();
     break;
@@ -2386,6 +2416,12 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   }
 
   case ICK_Boolean_Conversion:
+    // Perform half-to-boolean conversion via float.
+    if (From->getType()->isHalfType()) {
+      From = ImpCastExprToType(From, Context.FloatTy, CK_FloatingCast).take();
+      FromType = Context.FloatTy;
+    }
+
     From = ImpCastExprToType(From, Context.BoolTy,
                              ScalarTypeToBooleanCastKind(FromType), 
                              VK_RValue, /*BasePath=*/0, CCK).take();
@@ -2402,7 +2438,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
       return ExprError();
 
     From = ImpCastExprToType(From, ToType.getNonReferenceType(),
-                      CK_DerivedToBase, CastCategory(From),
+                      CK_DerivedToBase, From->getValueKind(),
                       &BasePath, CCK).take();
     break;
   }
@@ -2493,8 +2529,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   case ICK_Function_To_Pointer:
   case ICK_Qualification:
   case ICK_Num_Conversion_Kinds:
-    assert(false && "Improper second standard conversion");
-    break;
+    llvm_unreachable("Improper second standard conversion");
   }
 
   switch (SCS.Third) {
@@ -2506,7 +2541,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     // The qualification keeps the category of the inner expression, unless the
     // target type isn't a reference.
     ExprValueKind VK = ToType->isReferenceType() ?
-                                  CastCategory(From) : VK_RValue;
+                                  From->getValueKind() : VK_RValue;
     From = ImpCastExprToType(From, ToType.getNonLValueExprType(Context),
                              CK_NoOp, VK, /*BasePath=*/0, CCK).take();
 
@@ -2519,8 +2554,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     }
 
   default:
-    assert(false && "Improper third standard conversion");
-    break;
+    llvm_unreachable("Improper third standard conversion");
   }
 
   return Owned(From);
@@ -2835,8 +2869,12 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, UnaryTypeTrait UTT,
       LookupResult Res(Self, DeclarationNameInfo(Name, KeyLoc),
                        Sema::LookupOrdinaryName);
       if (Self.LookupQualifiedName(Res, RD)) {
+        Res.suppressDiagnostics();
         for (LookupResult::iterator Op = Res.begin(), OpEnd = Res.end();
              Op != OpEnd; ++Op) {
+          if (isa<FunctionTemplateDecl>(*Op))
+            continue;
+          
           CXXMethodDecl *Operator = cast<CXXMethodDecl>(*Op);
           if (Operator->isCopyAssignmentOperator()) {
             FoundAssign = true;
@@ -2849,7 +2887,7 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, UnaryTypeTrait UTT,
           }
         }
       }
-
+      
       return FoundAssign;
     }
     return false;
@@ -3240,34 +3278,34 @@ ExprResult Sema::BuildExpressionTrait(ExpressionTrait ET,
                                                  RParen, Context.BoolTy));
 }
 
-QualType Sema::CheckPointerToMemberOperands(ExprResult &lex, ExprResult &rex,
+QualType Sema::CheckPointerToMemberOperands(ExprResult &LHS, ExprResult &RHS,
                                             ExprValueKind &VK,
                                             SourceLocation Loc,
                                             bool isIndirect) {
-  assert(!lex.get()->getType()->isPlaceholderType() &&
-         !rex.get()->getType()->isPlaceholderType() &&
+  assert(!LHS.get()->getType()->isPlaceholderType() &&
+         !RHS.get()->getType()->isPlaceholderType() &&
          "placeholders should have been weeded out by now");
 
   // The LHS undergoes lvalue conversions if this is ->*.
   if (isIndirect) {
-    lex = DefaultLvalueConversion(lex.take());
-    if (lex.isInvalid()) return QualType();
+    LHS = DefaultLvalueConversion(LHS.take());
+    if (LHS.isInvalid()) return QualType();
   }
 
   // The RHS always undergoes lvalue conversions.
-  rex = DefaultLvalueConversion(rex.take());
-  if (rex.isInvalid()) return QualType();
+  RHS = DefaultLvalueConversion(RHS.take());
+  if (RHS.isInvalid()) return QualType();
 
   const char *OpSpelling = isIndirect ? "->*" : ".*";
   // C++ 5.5p2
   //   The binary operator .* [p3: ->*] binds its second operand, which shall
   //   be of type "pointer to member of T" (where T is a completely-defined
   //   class type) [...]
-  QualType RType = rex.get()->getType();
-  const MemberPointerType *MemPtr = RType->getAs<MemberPointerType>();
+  QualType RHSType = RHS.get()->getType();
+  const MemberPointerType *MemPtr = RHSType->getAs<MemberPointerType>();
   if (!MemPtr) {
     Diag(Loc, diag::err_bad_memptr_rhs)
-      << OpSpelling << RType << rex.get()->getSourceRange();
+      << OpSpelling << RHSType << RHS.get()->getSourceRange();
     return QualType();
   }
 
@@ -3283,21 +3321,21 @@ QualType Sema::CheckPointerToMemberOperands(ExprResult &lex, ExprResult &rex,
   //   [...] to its first operand, which shall be of class T or of a class of
   //   which T is an unambiguous and accessible base class. [p3: a pointer to
   //   such a class]
-  QualType LType = lex.get()->getType();
+  QualType LHSType = LHS.get()->getType();
   if (isIndirect) {
-    if (const PointerType *Ptr = LType->getAs<PointerType>())
-      LType = Ptr->getPointeeType();
+    if (const PointerType *Ptr = LHSType->getAs<PointerType>())
+      LHSType = Ptr->getPointeeType();
     else {
       Diag(Loc, diag::err_bad_memptr_lhs)
-        << OpSpelling << 1 << LType
+        << OpSpelling << 1 << LHSType
         << FixItHint::CreateReplacement(SourceRange(Loc), ".*");
       return QualType();
     }
   }
 
-  if (!Context.hasSameUnqualifiedType(Class, LType)) {
+  if (!Context.hasSameUnqualifiedType(Class, LHSType)) {
     // If we want to check the hierarchy, we need a complete type.
-    if (RequireCompleteType(Loc, LType, PDiag(diag::err_bad_memptr_lhs)
+    if (RequireCompleteType(Loc, LHSType, PDiag(diag::err_bad_memptr_lhs)
         << OpSpelling << (int)isIndirect)) {
       return QualType();
     }
@@ -3305,23 +3343,23 @@ QualType Sema::CheckPointerToMemberOperands(ExprResult &lex, ExprResult &rex,
                        /*DetectVirtual=*/false);
     // FIXME: Would it be useful to print full ambiguity paths, or is that
     // overkill?
-    if (!IsDerivedFrom(LType, Class, Paths) ||
+    if (!IsDerivedFrom(LHSType, Class, Paths) ||
         Paths.isAmbiguous(Context.getCanonicalType(Class))) {
       Diag(Loc, diag::err_bad_memptr_lhs) << OpSpelling
-        << (int)isIndirect << lex.get()->getType();
+        << (int)isIndirect << LHS.get()->getType();
       return QualType();
     }
     // Cast LHS to type of use.
     QualType UseType = isIndirect ? Context.getPointerType(Class) : Class;
-    ExprValueKind VK =
-        isIndirect ? VK_RValue : CastCategory(lex.get());
+    ExprValueKind VK = isIndirect ? VK_RValue : LHS.get()->getValueKind();
 
     CXXCastPath BasePath;
     BuildBasePathArray(Paths, BasePath);
-    lex = ImpCastExprToType(lex.take(), UseType, CK_DerivedToBase, VK, &BasePath);
+    LHS = ImpCastExprToType(LHS.take(), UseType, CK_DerivedToBase, VK,
+                            &BasePath);
   }
 
-  if (isa<CXXScalarValueInitExpr>(rex.get()->IgnoreParens())) {
+  if (isa<CXXScalarValueInitExpr>(RHS.get()->IgnoreParens())) {
     // Diagnose use of pointer-to-member type which when used as
     // the functional cast in a pointer-to-member expression.
     Diag(Loc, diag::err_pointer_to_member_type) << isIndirect;
@@ -3334,7 +3372,7 @@ QualType Sema::CheckPointerToMemberOperands(ExprResult &lex, ExprResult &rex,
   // The cv qualifiers are the union of those in the pointer and the left side,
   // in accordance with 5.5p5 and 5.2.5.
   QualType Result = MemPtr->getPointeeType();
-  Result = Context.getCVRQualifiedType(Result, LType.getCVRQualifiers());
+  Result = Context.getCVRQualifiedType(Result, LHSType.getCVRQualifiers());
 
   // C++0x [expr.mptr.oper]p6:
   //   In a .* expression whose object expression is an rvalue, the program is
@@ -3349,15 +3387,15 @@ QualType Sema::CheckPointerToMemberOperands(ExprResult &lex, ExprResult &rex,
       break;
 
     case RQ_LValue:
-      if (!isIndirect && !lex.get()->Classify(Context).isLValue())
+      if (!isIndirect && !LHS.get()->Classify(Context).isLValue())
         Diag(Loc, diag::err_pointer_to_member_oper_value_classify)
-          << RType << 1 << lex.get()->getSourceRange();
+          << RHSType << 1 << LHS.get()->getSourceRange();
       break;
 
     case RQ_RValue:
-      if (isIndirect || !lex.get()->Classify(Context).isRValue())
+      if (isIndirect || !LHS.get()->Classify(Context).isRValue())
         Diag(Loc, diag::err_pointer_to_member_oper_value_classify)
-          << RType << 0 << lex.get()->getSourceRange();
+          << RHSType << 0 << LHS.get()->getSourceRange();
       break;
     }
   }
@@ -3375,7 +3413,7 @@ QualType Sema::CheckPointerToMemberOperands(ExprResult &lex, ExprResult &rex,
   } else if (isIndirect) {
     VK = VK_LValue;
   } else {
-    VK = lex.get()->getValueKind();
+    VK = LHS.get()->getValueKind();
   }
 
   return Result;
@@ -3528,8 +3566,7 @@ static bool FindConditionalOverload(Sema &Self, ExprResult &LHS, ExprResult &RHS
       break;
 
     case OR_Deleted:
-      assert(false && "Conditional operator has only built-in overloads");
-      break;
+      llvm_unreachable("Conditional operator has only built-in overloads");
   }
   return true;
 }
@@ -3838,9 +3875,9 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
   // conversions in both directions. If only one works, or if the two composite
   // types are the same, we have succeeded.
   // FIXME: extended qualifiers?
-  typedef llvm::SmallVector<unsigned, 4> QualifierVector;
+  typedef SmallVector<unsigned, 4> QualifierVector;
   QualifierVector QualifierUnion;
-  typedef llvm::SmallVector<std::pair<const Type *, const Type *>, 4>
+  typedef SmallVector<std::pair<const Type *, const Type *>, 4>
       ContainingClassVector;
   ContainingClassVector MemberOfClass;
   QualType Composite1 = Context.getCanonicalType(T1),
@@ -4044,27 +4081,30 @@ ExprResult Sema::MaybeBindToTemporary(Expr *E) {
     // actual method.  FIXME: we should infer retention by selector in
     // cases where we don't have an actual method.
     } else {
-      Decl *D = 0;
+      ObjCMethodDecl *D = 0;
       if (ObjCMessageExpr *Send = dyn_cast<ObjCMessageExpr>(E)) {
         D = Send->getMethodDecl();
       } else {
         CastExpr *CE = cast<CastExpr>(E);
-        // FIXME. What other cast kinds to check for?
-        if (CE->getCastKind() == CK_ObjCProduceObject ||
-            CE->getCastKind() == CK_LValueToRValue)
-          return MaybeBindToTemporary(CE->getSubExpr());
         assert(CE->getCastKind() == CK_GetObjCProperty);
         const ObjCPropertyRefExpr *PRE = CE->getSubExpr()->getObjCProperty();
         D = (PRE->isImplicitProperty() ? PRE->getImplicitPropertyGetter() : 0);
       }
 
       ReturnsRetained = (D && D->hasAttr<NSReturnsRetainedAttr>());
+
+      // Don't do reclaims on performSelector calls; despite their
+      // return type, the invoked method doesn't necessarily actually
+      // return an object.
+      if (!ReturnsRetained &&
+          D && D->getMethodFamily() == OMF_performSelector)
+        return Owned(E);
     }
 
     ExprNeedsCleanups = true;
 
-    CastKind ck = (ReturnsRetained ? CK_ObjCConsumeObject
-                                   : CK_ObjCReclaimReturnedObject);
+    CastKind ck = (ReturnsRetained ? CK_ARCConsumeObject
+                                   : CK_ARCReclaimReturnedObject);
     return Owned(ImplicitCastExpr::Create(Context, E->getType(), ck, E, 0,
                                           VK_RValue));
   }
@@ -4172,7 +4212,7 @@ Sema::ActOnStartCXXMemberReference(Scope *S, Expr *Base, SourceLocation OpLoc,
   if (OpKind == tok::arrow) {
     // The set of types we've considered so far.
     llvm::SmallPtrSet<CanQualType,8> CTypes;
-    llvm::SmallVector<SourceLocation, 8> Locations;
+    SmallVector<SourceLocation, 8> Locations;
     CTypes.insert(Context.getCanonicalType(BaseType));
 
     while (BaseType->isRecordType()) {
@@ -4509,7 +4549,8 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
 }
 
 ExprResult Sema::BuildCXXMemberCallExpr(Expr *E, NamedDecl *FoundDecl,
-                                        CXXMethodDecl *Method) {
+                                        CXXMethodDecl *Method,
+                                        bool HadMultipleCandidates) {
   ExprResult Exp = PerformObjectArgumentInitialization(E, /*Qualifier=*/0,
                                           FoundDecl, Method);
   if (Exp.isInvalid())
@@ -4519,6 +4560,9 @@ ExprResult Sema::BuildCXXMemberCallExpr(Expr *E, NamedDecl *FoundDecl,
       new (Context) MemberExpr(Exp.take(), /*IsArrow=*/false, Method,
                                SourceLocation(), Method->getType(),
                                VK_RValue, OK_Ordinary);
+  if (HadMultipleCandidates)
+    ME->setHadMultipleCandidates(true);
+
   QualType ResultType = Method->getResultType();
   ExprValueKind VK = Expr::getValueKindForType(ResultType);
   ResultType = ResultType.getNonLValueExprType(Context);
@@ -4608,7 +4652,7 @@ ExprResult Sema::ActOnFinishFullExpr(Expr *FE) {
   if (FullExpr.isInvalid())
     return ExprError();
 
-  CheckImplicitConversions(FullExpr.get());
+  CheckImplicitConversions(FullExpr.get(), FullExpr.get()->getExprLoc());
   return MaybeCreateExprWithCleanups(FullExpr);
 }
 
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp
index 2488dc8849dd..26867c21a1aa 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp
@@ -298,7 +298,7 @@ CheckExtVectorComponent(Sema &S, QualType baseType, ExprValueKind &VK,
     // We didn't get to the end of the string. This means the component names
     // didn't come from the same set *or* we encountered an illegal name.
     S.Diag(OpLoc, diag::err_ext_vector_component_name_illegal)
-      << llvm::StringRef(compStr, 1) << SourceRange(CompLoc);
+      << StringRef(compStr, 1) << SourceRange(CompLoc);
     return QualType();
   }
 
@@ -337,10 +337,14 @@ CheckExtVectorComponent(Sema &S, QualType baseType, ExprValueKind &VK,
   QualType VT = S.Context.getExtVectorType(vecType->getElementType(), CompSize);
   // Now look up the TypeDefDecl from the vector type. Without this,
   // diagostics look bad. We want extended vector types to appear built-in.
-  for (unsigned i = 0, E = S.ExtVectorDecls.size(); i != E; ++i) {
-    if (S.ExtVectorDecls[i]->getUnderlyingType() == VT)
-      return S.Context.getTypedefType(S.ExtVectorDecls[i]);
+  for (Sema::ExtVectorDeclsType::iterator 
+         I = S.ExtVectorDecls.begin(S.ExternalSource),
+         E = S.ExtVectorDecls.end(); 
+       I != E; ++I) {
+    if ((*I)->getUnderlyingType() == VT)
+      return S.Context.getTypedefType(*I);
   }
+  
   return VT; // should never get here (a typedef type should always be found).
 }
 
@@ -949,9 +953,9 @@ static bool ShouldTryAgainWithRedefinitionType(Sema &S, ExprResult &base) {
 
   QualType redef;
   if (ty->isObjCId()) {
-    redef = S.Context.ObjCIdRedefinitionType;
+    redef = S.Context.getObjCIdRedefinitionType();
   } else if (ty->isObjCClass()) {
-    redef = S.Context.ObjCClassRedefinitionType;
+    redef = S.Context.getObjCClassRedefinitionType();
   } else {
     return false;
   }
@@ -966,6 +970,15 @@ static bool ShouldTryAgainWithRedefinitionType(Sema &S, ExprResult &base) {
   return true;
 }
 
+static bool isRecordType(QualType T) {
+  return T->isRecordType();
+}
+static bool isPointerToRecordType(QualType T) {
+  if (const PointerType *PT = T->getAs<PointerType>())
+    return PT->getPointeeType()->isRecordType();
+  return false;
+}
+
 /// Look up the given member of the given non-type-dependent
 /// expression.  This can return in one of two ways:
 ///  * If it returns a sentinel null-but-valid result, the caller will
@@ -985,6 +998,8 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
 
   // Perform default conversions.
   BaseExpr = DefaultFunctionArrayConversion(BaseExpr.take());
+  if (BaseExpr.isInvalid())
+    return ExprError();
 
   if (IsArrow) {
     BaseExpr = DefaultLvalueConversion(BaseExpr.take());
@@ -1041,6 +1056,13 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
 
   // Handle ivar access to Objective-C objects.
   if (const ObjCObjectType *OTy = BaseType->getAs<ObjCObjectType>()) {
+    if (!SS.isEmpty() && !SS.isInvalid()) {
+      Diag(SS.getRange().getBegin(), diag::err_qualified_objc_access)
+        << 1 << SS.getScopeRep()
+        << FixItHint::CreateRemoval(SS.getRange());
+      SS.clear();
+    }
+    
     IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
 
     // There are three cases for the base type:
@@ -1159,6 +1181,13 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
   // Objective-C property access.
   const ObjCObjectPointerType *OPT;
   if (!IsArrow && (OPT = BaseType->getAs<ObjCObjectPointerType>())) {
+    if (!SS.isEmpty() && !SS.isInvalid()) {
+      Diag(SS.getRange().getBegin(), diag::err_qualified_objc_access)
+        << 0 << SS.getScopeRep()
+        << FixItHint::CreateRemoval(SS.getRange());
+      SS.clear();
+    }
+
     // This actually uses the base as an r-value.
     BaseExpr = DefaultLvalueConversion(BaseExpr.take());
     if (BaseExpr.isInvalid())
@@ -1318,8 +1347,9 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
   // not just a pointer to builtin-sel again.
   if (IsArrow &&
       BaseType->isSpecificBuiltinType(BuiltinType::ObjCSel) &&
-      !Context.ObjCSelRedefinitionType->isObjCSelType()) {
-    BaseExpr = ImpCastExprToType(BaseExpr.take(), Context.ObjCSelRedefinitionType,
+      !Context.getObjCSelRedefinitionType()->isObjCSelType()) {
+    BaseExpr = ImpCastExprToType(BaseExpr.take(), 
+                                 Context.getObjCSelRedefinitionType(),
                                  CK_BitCast);
     return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
                             ObjCImpDecl, HasTemplateArgs);
@@ -1351,50 +1381,15 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
 
   // If the user is trying to apply -> or . to a function name, it's probably
   // because they forgot parentheses to call that function.
-  QualType ZeroArgCallTy;
-  UnresolvedSet<4> Overloads;
-  if (isExprCallable(*BaseExpr.get(), ZeroArgCallTy, Overloads)) {
-    if (ZeroArgCallTy.isNull()) {
-      Diag(BaseExpr.get()->getExprLoc(), diag::err_member_reference_needs_call)
-          << (Overloads.size() > 1) << 0 << BaseExpr.get()->getSourceRange();
-      UnresolvedSet<2> PlausibleOverloads;
-      for (OverloadExpr::decls_iterator It = Overloads.begin(),
-           DeclsEnd = Overloads.end(); It != DeclsEnd; ++It) {
-        const FunctionDecl *OverloadDecl = cast<FunctionDecl>(*It);
-        QualType OverloadResultTy = OverloadDecl->getResultType();
-        if ((!IsArrow && OverloadResultTy->isRecordType()) ||
-            (IsArrow && OverloadResultTy->isPointerType() &&
-             OverloadResultTy->getPointeeType()->isRecordType()))
-          PlausibleOverloads.addDecl(It.getDecl());
-      }
-      NoteOverloads(PlausibleOverloads, BaseExpr.get()->getExprLoc());
+  if (tryToRecoverWithCall(BaseExpr,
+                           PDiag(diag::err_member_reference_needs_call),
+                           /*complain*/ false,
+                           IsArrow ? &isRecordType : &isPointerToRecordType)) {
+    if (BaseExpr.isInvalid())
       return ExprError();
-    }
-    if ((!IsArrow && ZeroArgCallTy->isRecordType()) ||
-        (IsArrow && ZeroArgCallTy->isPointerType() &&
-         ZeroArgCallTy->getPointeeType()->isRecordType())) {
-      // At this point, we know BaseExpr looks like it's potentially callable
-      // with 0 arguments, and that it returns something of a reasonable type,
-      // so we can emit a fixit and carry on pretending that BaseExpr was
-      // actually a CallExpr.
-      SourceLocation ParenInsertionLoc =
-          PP.getLocForEndOfToken(BaseExpr.get()->getLocEnd());
-      Diag(BaseExpr.get()->getExprLoc(), diag::err_member_reference_needs_call)
-          << (Overloads.size() > 1) << 1 << BaseExpr.get()->getSourceRange()
-          << FixItHint::CreateInsertion(ParenInsertionLoc, "()");
-      // FIXME: Try this before emitting the fixit, and suppress diagnostics
-      // while doing so.
-      ExprResult NewBase =
-          ActOnCallExpr(0, BaseExpr.take(), ParenInsertionLoc,
-                        MultiExprArg(*this, 0, 0),
-                        ParenInsertionLoc.getFileLocWithOffset(1));
-      if (NewBase.isInvalid())
-        return ExprError();
-      BaseExpr = NewBase;
-      BaseExpr = DefaultFunctionArrayConversion(BaseExpr.take());
-      return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
-                              ObjCImpDecl, HasTemplateArgs);
-    }
+    BaseExpr = DefaultFunctionArrayConversion(BaseExpr.take());
+    return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                            ObjCImpDecl, HasTemplateArgs);
   }
 
   Diag(MemberLoc, diag::err_typecheck_member_reference_struct_union)
@@ -1427,7 +1422,7 @@ ExprResult Sema::ActOnMemberAccessExpr(Scope *S, Expr *Base,
     return ExprError();
 
   // Warn about the explicit constructor calls Microsoft extension.
-  if (getLangOptions().Microsoft &&
+  if (getLangOptions().MicrosoftExt &&
       Id.getKind() == UnqualifiedId::IK_ConstructorName)
     Diag(Id.getSourceRange().getBegin(),
          diag::ext_ms_explicit_constructor_call);
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExprObjC.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExprObjC.cpp
index 02a4682cc840..20098b21366d 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaExprObjC.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaExprObjC.cpp
@@ -16,6 +16,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Initialization.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprObjC.h"
@@ -26,6 +27,7 @@
 
 using namespace clang;
 using namespace sema;
+using llvm::makeArrayRef;
 
 ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
                                         Expr **strings,
@@ -42,13 +44,13 @@ ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
   if (NumStrings != 1) {
     // Concatenate objc strings.
     llvm::SmallString<128> StrBuf;
-    llvm::SmallVector<SourceLocation, 8> StrLocs;
+    SmallVector<SourceLocation, 8> StrLocs;
 
     for (unsigned i = 0; i != NumStrings; ++i) {
       S = Strings[i];
 
-      // ObjC strings can't be wide.
-      if (S->isWide()) {
+      // ObjC strings can't be wide or UTF.
+      if (!S->isAscii()) {
         Diag(S->getLocStart(), diag::err_cfstring_literal_not_string_constant)
           << S->getSourceRange();
         return true;
@@ -64,7 +66,7 @@ ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
     // Create the aggregate string with the appropriate content and location
     // information.
     S = StringLiteral::Create(Context, StrBuf,
-                              /*Wide=*/false, /*Pascal=*/false,
+                              StringLiteral::Ascii, /*Pascal=*/false,
                               Context.getPointerType(Context.CharTy),
                               &StrLocs[0], StrLocs.size());
   }
@@ -203,6 +205,7 @@ ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
     case OMF_None:
     case OMF_alloc:
     case OMF_copy:
+    case OMF_finalize:
     case OMF_init:
     case OMF_mutableCopy:
     case OMF_new:
@@ -270,6 +273,13 @@ ObjCMethodDecl *Sema::tryCaptureObjCSelf() {
   return method;
 }
 
+static QualType stripObjCInstanceType(ASTContext &Context, QualType T) {
+  if (T == Context.getObjCInstanceType())
+    return Context.getObjCIdType();
+  
+  return T;
+}
+
 QualType Sema::getMessageSendResultType(QualType ReceiverType,
                                         ObjCMethodDecl *Method,
                                     bool isClassMessage, bool isSuperMessage) {
@@ -282,7 +292,7 @@ QualType Sema::getMessageSendResultType(QualType ReceiverType,
   //     was a class message send, T is the declared return type of the method
   //     found
   if (Method->isInstanceMethod() && isClassMessage)
-    return Method->getSendResultType();
+    return stripObjCInstanceType(Context, Method->getSendResultType());
   
   //   - if the receiver is super, T is a pointer to the class of the 
   //     enclosing method definition
@@ -301,7 +311,7 @@ QualType Sema::getMessageSendResultType(QualType ReceiverType,
   //     T is the declared return type of the method.
   if (ReceiverType->isObjCClassType() ||
       ReceiverType->isObjCQualifiedClassType())
-    return  Method->getSendResultType();
+    return stripObjCInstanceType(Context, Method->getSendResultType());
   
   //   - if the receiver is id, qualified id, Class, or qualified Class, T
   //     is the receiver type, otherwise
@@ -327,6 +337,10 @@ void Sema::EmitRelatedResultTypeNote(const Expr *E) {
                                      MsgSend->getType()))
     return;
   
+  if (!Context.hasSameUnqualifiedType(Method->getResultType(), 
+                                      Context.getObjCInstanceType()))
+    return;
+  
   Diag(Method->getLocation(), diag::note_related_result_type_inferred)
     << Method->isInstanceMethod() << Method->getSelector()
     << MsgSend->getType();
@@ -356,8 +370,9 @@ bool Sema::CheckMessageArgumentTypes(QualType ReceiverType,
     else
       DiagID = isClassMessage ? diag::warn_class_method_not_found
                               : diag::warn_inst_method_not_found;
-    Diag(lbrac, DiagID)
-      << Sel << isClassMessage << SourceRange(lbrac, rbrac);
+    if (!getLangOptions().DebuggerSupport)
+      Diag(lbrac, DiagID)
+        << Sel << isClassMessage << SourceRange(lbrac, rbrac);
 
     // In debuggers, we want to use __unknown_anytype for these
     // results so that clients can cast them.
@@ -947,7 +962,7 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
                                    SourceLocation SuperLoc,
                                    Selector Sel,
                                    SourceLocation LBracLoc,
-                                   SourceLocation SelectorLoc,
+                                   ArrayRef<SourceLocation> SelectorLocs,
                                    SourceLocation RBracLoc,
                                    MultiExprArg Args) {
   // Determine whether we are inside a method or not.
@@ -975,13 +990,18 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
   // We are in a method whose class has a superclass, so 'super'
   // is acting as a keyword.
   if (Method->isInstanceMethod()) {
+    if (Sel.getMethodFamily() == OMF_dealloc)
+      ObjCShouldCallSuperDealloc = false;
+    if (Sel.getMethodFamily() == OMF_finalize)
+      ObjCShouldCallSuperFinalize = false;
+
     // Since we are in an instance method, this is an instance
     // message to the superclass instance.
     QualType SuperTy = Context.getObjCInterfaceType(Super);
     SuperTy = Context.getObjCObjectPointerType(SuperTy);
     return BuildInstanceMessage(0, SuperTy, SuperLoc,
                                 Sel, /*Method=*/0,
-                                LBracLoc, SelectorLoc, RBracLoc, move(Args));
+                                LBracLoc, SelectorLocs, RBracLoc, move(Args));
   }
   
   // Since we are in a class method, this is a class message to
@@ -989,7 +1009,7 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
   return BuildClassMessage(/*ReceiverTypeInfo=*/0,
                            Context.getObjCInterfaceType(Super),
                            SuperLoc, Sel, /*Method=*/0,
-                           LBracLoc, SelectorLoc, RBracLoc, move(Args));
+                           LBracLoc, SelectorLocs, RBracLoc, move(Args));
 }
 
 /// \brief Build an Objective-C class message expression.
@@ -1026,7 +1046,7 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
                                    Selector Sel,
                                    ObjCMethodDecl *Method,
                                    SourceLocation LBracLoc, 
-                                   SourceLocation SelectorLoc,
+                                   ArrayRef<SourceLocation> SelectorLocs,
                                    SourceLocation RBracLoc,
                                    MultiExprArg ArgsIn) {
   SourceLocation Loc = SuperLoc.isValid()? SuperLoc
@@ -1045,8 +1065,8 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
     assert(SuperLoc.isInvalid() && "Message to super with dependent type");
     return Owned(ObjCMessageExpr::Create(Context, ReceiverType,
                                          VK_RValue, LBracLoc, ReceiverTypeInfo,
-                                         Sel, SelectorLoc, /*Method=*/0,
-                                         Args, NumArgs, RBracLoc));
+                                         Sel, SelectorLocs, /*Method=*/0,
+                                         makeArrayRef(Args, NumArgs),RBracLoc));
   }
   
   // Find the class to which we are sending this message.
@@ -1107,12 +1127,14 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
   if (SuperLoc.isValid())
     Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc, 
                                      SuperLoc, /*IsInstanceSuper=*/false, 
-                                     ReceiverType, Sel, SelectorLoc,
-                                     Method, Args, NumArgs, RBracLoc);
+                                     ReceiverType, Sel, SelectorLocs,
+                                     Method, makeArrayRef(Args, NumArgs),
+                                     RBracLoc);
   else
     Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc, 
-                                     ReceiverTypeInfo, Sel, SelectorLoc,
-                                     Method, Args, NumArgs, RBracLoc);
+                                     ReceiverTypeInfo, Sel, SelectorLocs,
+                                     Method, makeArrayRef(Args, NumArgs),
+                                     RBracLoc);
   return MaybeBindToTemporary(Result);
 }
 
@@ -1123,7 +1145,7 @@ ExprResult Sema::ActOnClassMessage(Scope *S,
                                    ParsedType Receiver,
                                    Selector Sel,
                                    SourceLocation LBracLoc,
-                                   SourceLocation SelectorLoc,
+                                   ArrayRef<SourceLocation> SelectorLocs,
                                    SourceLocation RBracLoc,
                                    MultiExprArg Args) {
   TypeSourceInfo *ReceiverTypeInfo;
@@ -1137,7 +1159,7 @@ ExprResult Sema::ActOnClassMessage(Scope *S,
 
   return BuildClassMessage(ReceiverTypeInfo, ReceiverType, 
                            /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0,
-                           LBracLoc, SelectorLoc, RBracLoc, move(Args));
+                           LBracLoc, SelectorLocs, RBracLoc, move(Args));
 }
 
 /// \brief Build an Objective-C instance message expression.
@@ -1174,7 +1196,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                                       Selector Sel,
                                       ObjCMethodDecl *Method,
                                       SourceLocation LBracLoc, 
-                                      SourceLocation SelectorLoc,
+                                      ArrayRef<SourceLocation> SelectorLocs,
                                       SourceLocation RBracLoc,
                                       MultiExprArg ArgsIn) {
   // The location of the receiver.
@@ -1197,8 +1219,9 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
       assert(SuperLoc.isInvalid() && "Message to super with dependent type");
       return Owned(ObjCMessageExpr::Create(Context, Context.DependentTy,
                                            VK_RValue, LBracLoc, Receiver, Sel, 
-                                           SelectorLoc, /*Method=*/0,
-                                           Args, NumArgs, RBracLoc));
+                                           SelectorLocs, /*Method=*/0,
+                                           makeArrayRef(Args, NumArgs),
+                                           RBracLoc));
     }
 
     // If necessary, apply function/array conversion to the receiver.
@@ -1346,7 +1369,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
           << Receiver->getSourceRange();
         if (ReceiverType->isPointerType())
           Receiver = ImpCastExprToType(Receiver, Context.getObjCIdType(), 
-                            CK_BitCast).take();
+                            CK_CPointerToObjCPointerCast).take();
         else {
           // TODO: specialized warning on null receivers?
           bool IsNull = Receiver->isNullPointerConstant(Context,
@@ -1355,24 +1378,19 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                             IsNull ? CK_NullToPointer : CK_IntegralToPointer).take();
         }
         ReceiverType = Receiver->getType();
-      } 
-      else {
+      } else {
         ExprResult ReceiverRes;
         if (getLangOptions().CPlusPlus)
-          ReceiverRes = PerformContextuallyConvertToObjCId(Receiver);
+          ReceiverRes = PerformContextuallyConvertToObjCPointer(Receiver);
         if (ReceiverRes.isUsable()) {
           Receiver = ReceiverRes.take();
-          if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Receiver)) {
-            Receiver = ICE->getSubExpr();
-            ReceiverType = Receiver->getType();
-          }
           return BuildInstanceMessage(Receiver,
                                       ReceiverType,
                                       SuperLoc,
                                       Sel,
                                       Method,
                                       LBracLoc,
-                                      SelectorLoc,
+                                      SelectorLocs,
                                       RBracLoc,
                                       move(ArgsIn));
         } else {
@@ -1402,6 +1420,8 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                           diag::err_illegal_message_expr_incomplete_type))
     return ExprError();
 
+  SourceLocation SelLoc = SelectorLocs.front();
+
   // In ARC, forbid the user from sending messages to 
   // retain/release/autorelease/dealloc/retainCount explicitly.
   if (getLangOptions().ObjCAutoRefCount) {
@@ -1415,6 +1435,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     case OMF_None:
     case OMF_alloc:
     case OMF_copy:
+    case OMF_finalize:
     case OMF_mutableCopy:
     case OMF_new:
     case OMF_self:
@@ -1426,7 +1447,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     case OMF_autorelease:
     case OMF_retainCount:
       Diag(Loc, diag::err_arc_illegal_explicit_message)
-        << Sel << SelectorLoc;
+        << Sel << SelLoc;
       break;
     
     case OMF_performSelector:
@@ -1452,7 +1473,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                 // Issue error, unless ns_returns_not_retained.
                 if (!SelMethod->hasAttr<NSReturnsNotRetainedAttr>()) {
                   // selector names a +1 method 
-                  Diag(SelectorLoc, 
+                  Diag(SelLoc, 
                        diag::err_arc_perform_selector_retains);
                   Diag(SelMethod->getLocation(), diag::note_method_declared_at);
                 }
@@ -1461,7 +1482,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                 // +0 call. OK. unless ns_returns_retained.
                 if (SelMethod->hasAttr<NSReturnsRetainedAttr>()) {
                   // selector names a +1 method
-                  Diag(SelectorLoc, 
+                  Diag(SelLoc, 
                        diag::err_arc_perform_selector_retains);
                   Diag(SelMethod->getLocation(), diag::note_method_declared_at);
                 }
@@ -1470,7 +1491,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
           }
         } else {
           // error (may leak).
-          Diag(SelectorLoc, diag::warn_arc_perform_selector_leaks);
+          Diag(SelLoc, diag::warn_arc_perform_selector_leaks);
           Diag(Args[0]->getExprLoc(), diag::note_used_here);
         }
       }
@@ -1483,12 +1504,12 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
   if (SuperLoc.isValid())
     Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc,
                                      SuperLoc,  /*IsInstanceSuper=*/true,
-                                     ReceiverType, Sel, SelectorLoc, Method, 
-                                     Args, NumArgs, RBracLoc);
+                                     ReceiverType, Sel, SelectorLocs, Method, 
+                                     makeArrayRef(Args, NumArgs), RBracLoc);
   else
     Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc,
-                                     Receiver, Sel, SelectorLoc, Method,
-                                     Args, NumArgs, RBracLoc);
+                                     Receiver, Sel, SelectorLocs, Method,
+                                     makeArrayRef(Args, NumArgs), RBracLoc);
 
   if (getLangOptions().ObjCAutoRefCount) {
     // In ARC, annotate delegate init calls.
@@ -1520,7 +1541,7 @@ ExprResult Sema::ActOnInstanceMessage(Scope *S,
                                       Expr *Receiver, 
                                       Selector Sel,
                                       SourceLocation LBracLoc,
-                                      SourceLocation SelectorLoc,
+                                      ArrayRef<SourceLocation> SelectorLocs,
                                       SourceLocation RBracLoc,
                                       MultiExprArg Args) {
   if (!Receiver)
@@ -1528,206 +1549,360 @@ ExprResult Sema::ActOnInstanceMessage(Scope *S,
 
   return BuildInstanceMessage(Receiver, Receiver->getType(),
                               /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0, 
-                              LBracLoc, SelectorLoc, RBracLoc, move(Args));
+                              LBracLoc, SelectorLocs, RBracLoc, move(Args));
 }
 
 enum ARCConversionTypeClass {
+  /// int, void, struct A
   ACTC_none,
+
+  /// id, void (^)()
   ACTC_retainable,
-  ACTC_indirectRetainable
+
+  /// id*, id***, void (^*)(),
+  ACTC_indirectRetainable,
+
+  /// void* might be a normal C type, or it might a CF type.
+  ACTC_voidPtr,
+
+  /// struct A*
+  ACTC_coreFoundation
 };
+static bool isAnyRetainable(ARCConversionTypeClass ACTC) {
+  return (ACTC == ACTC_retainable ||
+          ACTC == ACTC_coreFoundation ||
+          ACTC == ACTC_voidPtr);
+}
+static bool isAnyCLike(ARCConversionTypeClass ACTC) {
+  return ACTC == ACTC_none ||
+         ACTC == ACTC_voidPtr ||
+         ACTC == ACTC_coreFoundation;
+}
+
 static ARCConversionTypeClass classifyTypeForARCConversion(QualType type) {
-  ARCConversionTypeClass ACTC = ACTC_retainable;
+  bool isIndirect = false;
   
   // Ignore an outermost reference type.
-  if (const ReferenceType *ref = type->getAs<ReferenceType>())
+  if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
     type = ref->getPointeeType();
+    isIndirect = true;
+  }
   
   // Drill through pointers and arrays recursively.
   while (true) {
     if (const PointerType *ptr = type->getAs<PointerType>()) {
       type = ptr->getPointeeType();
+
+      // The first level of pointer may be the innermost pointer on a CF type.
+      if (!isIndirect) {
+        if (type->isVoidType()) return ACTC_voidPtr;
+        if (type->isRecordType()) return ACTC_coreFoundation;
+      }
     } else if (const ArrayType *array = type->getAsArrayTypeUnsafe()) {
       type = QualType(array->getElementType()->getBaseElementTypeUnsafe(), 0);
     } else {
       break;
     }
-    ACTC = ACTC_indirectRetainable;
+    isIndirect = true;
   }
   
-  if (!type->isObjCRetainableType()) return ACTC_none;
-  return ACTC;
+  if (isIndirect) {
+    if (type->isObjCARCBridgableType())
+      return ACTC_indirectRetainable;
+    return ACTC_none;
+  }
+
+  if (type->isObjCARCBridgableType())
+    return ACTC_retainable;
+
+  return ACTC_none;
 }
 
 namespace {
-  /// Return true if the given expression can be reasonably converted
-  /// between a retainable pointer type and a C pointer type.
-  struct ARCCastChecker : StmtVisitor<ARCCastChecker, bool> {
+  /// A result from the cast checker.
+  enum ACCResult {
+    /// Cannot be casted.
+    ACC_invalid,
+
+    /// Can be safely retained or not retained.
+    ACC_bottom,
+
+    /// Can be casted at +0.
+    ACC_plusZero,
+
+    /// Can be casted at +1.
+    ACC_plusOne
+  };
+  ACCResult merge(ACCResult left, ACCResult right) {
+    if (left == right) return left;
+    if (left == ACC_bottom) return right;
+    if (right == ACC_bottom) return left;
+    return ACC_invalid;
+  }
+
+  /// A checker which white-lists certain expressions whose conversion
+  /// to or from retainable type would otherwise be forbidden in ARC.
+  class ARCCastChecker : public StmtVisitor<ARCCastChecker, ACCResult> {
+    typedef StmtVisitor<ARCCastChecker, ACCResult> super;
+
     ASTContext &Context;
-    ARCCastChecker(ASTContext &Context) : Context(Context) {}
-    bool VisitStmt(Stmt *s) {
-      return false;
+    ARCConversionTypeClass SourceClass;
+    ARCConversionTypeClass TargetClass;
+
+    static bool isCFType(QualType type) {
+      // Someday this can use ns_bridged.  For now, it has to do this.
+      return type->isCARCBridgableType();
     }
-    bool VisitExpr(Expr *e) {
-      return e->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull);
+
+  public:
+    ARCCastChecker(ASTContext &Context, ARCConversionTypeClass source,
+                   ARCConversionTypeClass target)
+      : Context(Context), SourceClass(source), TargetClass(target) {}
+
+    using super::Visit;
+    ACCResult Visit(Expr *e) {
+      return super::Visit(e->IgnoreParens());
+    }
+
+    ACCResult VisitStmt(Stmt *s) {
+      return ACC_invalid;
+    }
+
+    /// Null pointer constants can be casted however you please.
+    ACCResult VisitExpr(Expr *e) {
+      if (e->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNotNull))
+        return ACC_bottom;
+      return ACC_invalid;
+    }
+
+    /// Objective-C string literals can be safely casted.
+    ACCResult VisitObjCStringLiteral(ObjCStringLiteral *e) {
+      // If we're casting to any retainable type, go ahead.  Global
+      // strings are immune to retains, so this is bottom.
+      if (isAnyRetainable(TargetClass)) return ACC_bottom;
+
+      return ACC_invalid;
     }
     
-    bool VisitParenExpr(ParenExpr *e) {
-      return Visit(e->getSubExpr());
-    }
-    bool VisitCastExpr(CastExpr *e) {
+    /// Look through certain implicit and explicit casts.
+    ACCResult VisitCastExpr(CastExpr *e) {
       switch (e->getCastKind()) {
         case CK_NullToPointer:
-          return true;
+          return ACC_bottom;
+
         case CK_NoOp:
         case CK_LValueToRValue:
         case CK_BitCast:
-        case CK_AnyPointerToObjCPointerCast:
+        case CK_GetObjCProperty:
+        case CK_CPointerToObjCPointerCast:
+        case CK_BlockPointerToObjCPointerCast:
         case CK_AnyPointerToBlockPointerCast:
           return Visit(e->getSubExpr());
+
         default:
-          return false;
+          return ACC_invalid;
       }
     }
-    bool VisitUnaryExtension(UnaryOperator *e) {
+
+    /// Look through unary extension.
+    ACCResult VisitUnaryExtension(UnaryOperator *e) {
       return Visit(e->getSubExpr());
     }
-    bool VisitBinComma(BinaryOperator *e) {
+
+    /// Ignore the LHS of a comma operator.
+    ACCResult VisitBinComma(BinaryOperator *e) {
       return Visit(e->getRHS());
     }
-    bool VisitConditionalOperator(ConditionalOperator *e) {
-      // Conditional operators are okay if both sides are okay.
-      return Visit(e->getTrueExpr()) && Visit(e->getFalseExpr());
+
+    /// Conditional operators are okay if both sides are okay.
+    ACCResult VisitConditionalOperator(ConditionalOperator *e) {
+      ACCResult left = Visit(e->getTrueExpr());
+      if (left == ACC_invalid) return ACC_invalid;
+      return merge(left, Visit(e->getFalseExpr()));
     }
-    bool VisitObjCStringLiteral(ObjCStringLiteral *e) {
-      // Always white-list Objective-C string literals.
-      return true;
-    }
-    bool VisitStmtExpr(StmtExpr *e) {
+
+    /// Statement expressions are okay if their result expression is okay.
+    ACCResult VisitStmtExpr(StmtExpr *e) {
       return Visit(e->getSubStmt()->body_back());
     }
-    bool VisitDeclRefExpr(DeclRefExpr *e) {
-      // White-list references to global extern strings from system
-      // headers.
-      if (VarDecl *var = dyn_cast<VarDecl>(e->getDecl()))
-        if (var->getStorageClass() == SC_Extern &&
-            var->getType().isConstQualified() &&
-            Context.getSourceManager().isInSystemHeader(var->getLocation()))
-          return true;
-      return false;
-    }
-  };
-}
 
-bool
-Sema::ValidObjCARCNoBridgeCastExpr(Expr *&Exp, QualType castType) {
-  Expr *NewExp = Exp->IgnoreParenCasts();
-  
-  if (!isa<ObjCMessageExpr>(NewExp) && !isa<ObjCPropertyRefExpr>(NewExp)
-      && !isa<CallExpr>(NewExp))
-    return false;
-  ObjCMethodDecl *method = 0;
-  bool MethodReturnsPlusOne = false;
-  
-  if (ObjCPropertyRefExpr *PRE = dyn_cast<ObjCPropertyRefExpr>(NewExp)) {
-    method = PRE->getExplicitProperty()->getGetterMethodDecl();
-  }
-  else if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(NewExp))
-    method = ME->getMethodDecl();
-  else {
-    CallExpr *CE = cast<CallExpr>(NewExp);
-    Decl *CallDecl = CE->getCalleeDecl();
-    if (!CallDecl)
-      return false;
-    if (CallDecl->hasAttr<CFReturnsNotRetainedAttr>())
-      return true;
-    MethodReturnsPlusOne = CallDecl->hasAttr<CFReturnsRetainedAttr>();
-    if (!MethodReturnsPlusOne) {
-      if (NamedDecl *ND = dyn_cast<NamedDecl>(CallDecl))
-        if (const IdentifierInfo *Id = ND->getIdentifier())
-          if (Id->isStr("__builtin___CFStringMakeConstantString"))
-            return true;
+    /// Some declaration references are okay.
+    ACCResult VisitDeclRefExpr(DeclRefExpr *e) {
+      // References to global constants from system headers are okay.
+      // These are things like 'kCFStringTransformToLatin'.  They are
+      // can also be assumed to be immune to retains.
+      VarDecl *var = dyn_cast<VarDecl>(e->getDecl());
+      if (isAnyRetainable(TargetClass) &&
+          isAnyRetainable(SourceClass) &&
+          var &&
+          var->getStorageClass() == SC_Extern &&
+          var->getType().isConstQualified() &&
+          Context.getSourceManager().isInSystemHeader(var->getLocation())) {
+        return ACC_bottom;
+      }
+
+      // Nothing else.
+      return ACC_invalid;
     }
-  }
-  
-  if (!MethodReturnsPlusOne) {
-    if (!method)
-      return false;
-    if (method->hasAttr<CFReturnsNotRetainedAttr>())
-      return true;
-    MethodReturnsPlusOne = method->hasAttr<CFReturnsRetainedAttr>();
-    if (!MethodReturnsPlusOne) {
-      ObjCMethodFamily family = method->getSelector().getMethodFamily();
-      switch (family) {
-        case OMF_alloc:
-        case OMF_copy:
-        case OMF_mutableCopy:
-        case OMF_new:
-          MethodReturnsPlusOne = true;
-          break;
-        default:
-          break;
+
+    /// Some calls are okay.
+    ACCResult VisitCallExpr(CallExpr *e) {
+      if (FunctionDecl *fn = e->getDirectCallee())
+        if (ACCResult result = checkCallToFunction(fn))
+          return result;
+
+      return super::VisitCallExpr(e);
+    }
+
+    ACCResult checkCallToFunction(FunctionDecl *fn) {
+      // Require a CF*Ref return type.
+      if (!isCFType(fn->getResultType()))
+        return ACC_invalid;
+
+      if (!isAnyRetainable(TargetClass))
+        return ACC_invalid;
+
+      // Honor an explicit 'not retained' attribute.
+      if (fn->hasAttr<CFReturnsNotRetainedAttr>())
+        return ACC_plusZero;
+
+      // Honor an explicit 'retained' attribute, except that for
+      // now we're not going to permit implicit handling of +1 results,
+      // because it's a bit frightening.
+      if (fn->hasAttr<CFReturnsRetainedAttr>())
+        return ACC_invalid; // ACC_plusOne if we start accepting this
+
+      // Recognize this specific builtin function, which is used by CFSTR.
+      unsigned builtinID = fn->getBuiltinID();
+      if (builtinID == Builtin::BI__builtin___CFStringMakeConstantString)
+        return ACC_bottom;
+
+      // Otherwise, don't do anything implicit with an unaudited function.
+      if (!fn->hasAttr<CFAuditedTransferAttr>())
+        return ACC_invalid;
+
+      // Otherwise, it's +0 unless it follows the create convention.
+      if (ento::coreFoundation::followsCreateRule(fn))
+        return ACC_invalid; // ACC_plusOne if we start accepting this
+
+      return ACC_plusZero;
+    }
+
+    ACCResult VisitObjCMessageExpr(ObjCMessageExpr *e) {
+      return checkCallToMethod(e->getMethodDecl());
+    }
+
+    ACCResult VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *e) {
+      ObjCMethodDecl *method;
+      if (e->isExplicitProperty())
+        method = e->getExplicitProperty()->getGetterMethodDecl();
+      else
+        method = e->getImplicitPropertyGetter();
+      return checkCallToMethod(method);
+    }
+
+    ACCResult checkCallToMethod(ObjCMethodDecl *method) {
+      if (!method) return ACC_invalid;
+
+      // Check for message sends to functions returning CF types.  We
+      // just obey the Cocoa conventions with these, even though the
+      // return type is CF.
+      if (!isAnyRetainable(TargetClass) || !isCFType(method->getResultType()))
+        return ACC_invalid;
+      
+      // If the method is explicitly marked not-retained, it's +0.
+      if (method->hasAttr<CFReturnsNotRetainedAttr>())
+        return ACC_plusZero;
+
+      // If the method is explicitly marked as returning retained, or its
+      // selector follows a +1 Cocoa convention, treat it as +1.
+      if (method->hasAttr<CFReturnsRetainedAttr>())
+        return ACC_plusOne;
+
+      switch (method->getSelector().getMethodFamily()) {
+      case OMF_alloc:
+      case OMF_copy:
+      case OMF_mutableCopy:
+      case OMF_new:
+        return ACC_plusOne;
+
+      default:
+        // Otherwise, treat it as +0.
+        return ACC_plusZero;
       }
     }
-  }
-  
-  if (MethodReturnsPlusOne) {
-    TypeSourceInfo *TSInfo = 
-      Context.getTrivialTypeSourceInfo(castType, SourceLocation());
-    ExprResult ExpRes = BuildObjCBridgedCast(SourceLocation(), OBC_BridgeTransfer,
-                                             SourceLocation(), TSInfo, Exp);
-    Exp = ExpRes.take();
-  }
-  return true;
+  };
 }
 
 void 
 Sema::CheckObjCARCConversion(SourceRange castRange, QualType castType,
                              Expr *&castExpr, CheckedConversionKind CCK) {
   QualType castExprType = castExpr->getType();
+
+  // For the purposes of the classification, we assume reference types
+  // will bind to temporaries.
+  QualType effCastType = castType;
+  if (const ReferenceType *ref = castType->getAs<ReferenceType>())
+    effCastType = ref->getPointeeType();
   
   ARCConversionTypeClass exprACTC = classifyTypeForARCConversion(castExprType);
-  ARCConversionTypeClass castACTC = classifyTypeForARCConversion(castType);
+  ARCConversionTypeClass castACTC = classifyTypeForARCConversion(effCastType);
   if (exprACTC == castACTC) return;
-  if (exprACTC && castType->isIntegralType(Context)) return;
+  if (isAnyCLike(exprACTC) && isAnyCLike(castACTC)) return;
+
+  // Allow all of these types to be cast to integer types (but not
+  // vice-versa).
+  if (castACTC == ACTC_none && castType->isIntegralType(Context))
+    return;
   
   // Allow casts between pointers to lifetime types (e.g., __strong id*)
   // and pointers to void (e.g., cv void *). Casting from void* to lifetime*
   // must be explicit.
-  if (const PointerType *CastPtr = castType->getAs<PointerType>()) {
-    if (const PointerType *CastExprPtr = castExprType->getAs<PointerType>()) {
-      QualType CastPointee = CastPtr->getPointeeType();
-      QualType CastExprPointee = CastExprPtr->getPointeeType();
-      if ((CCK != CCK_ImplicitConversion && 
-           CastPointee->isObjCIndirectLifetimeType() && 
-           CastExprPointee->isVoidType()) ||
-          (CastPointee->isVoidType() && 
-           CastExprPointee->isObjCIndirectLifetimeType()))
-        return;
-    }
+  if (exprACTC == ACTC_indirectRetainable && castACTC == ACTC_voidPtr)
+    return;
+  if (castACTC == ACTC_indirectRetainable && exprACTC == ACTC_voidPtr &&
+      CCK != CCK_ImplicitConversion)
+    return;
+
+  switch (ARCCastChecker(Context, exprACTC, castACTC).Visit(castExpr)) {
+  // For invalid casts, fall through.
+  case ACC_invalid:
+    break;
+
+  // Do nothing for both bottom and +0.
+  case ACC_bottom:
+  case ACC_plusZero:
+    return;
+
+  // If the result is +1, consume it here.
+  case ACC_plusOne:
+    castExpr = ImplicitCastExpr::Create(Context, castExpr->getType(),
+                                        CK_ARCConsumeObject, castExpr,
+                                        0, VK_RValue);
+    ExprNeedsCleanups = true;
+    return;
   }
   
-  if (ARCCastChecker(Context).Visit(castExpr))
-    return;
-  
   SourceLocation loc =
-  (castRange.isValid() ? castRange.getBegin() : castExpr->getExprLoc());
+    (castRange.isValid() ? castRange.getBegin() : castExpr->getExprLoc());
   
   if (makeUnavailableInSystemHeader(loc,
-                                    "converts between Objective-C and C pointers in -fobjc-arc"))
+                "converts between Objective-C and C pointers in -fobjc-arc"))
     return;
   
   unsigned srcKind = 0;
   switch (exprACTC) {
-    case ACTC_none:
-      srcKind = (castExprType->isPointerType() ? 1 : 0);
-      break;
-    case ACTC_retainable:
-      srcKind = (castExprType->isBlockPointerType() ? 2 : 3);
-      break;
-    case ACTC_indirectRetainable:
-      srcKind = 4;
-      break;
+  case ACTC_none:
+  case ACTC_coreFoundation:
+  case ACTC_voidPtr:
+    srcKind = (castExprType->isPointerType() ? 1 : 0);
+    break;
+  case ACTC_retainable:
+    srcKind = (castExprType->isBlockPointerType() ? 2 : 3);
+    break;
+  case ACTC_indirectRetainable:
+    srcKind = 4;
+    break;
   }
   
   if (CCK == CCK_CStyleCast) {
@@ -1735,12 +1910,7 @@ Sema::CheckObjCARCConversion(SourceRange castRange, QualType castType,
     SourceLocation AfterLParen = PP.getLocForEndOfToken(castRange.getBegin());
     SourceLocation NoteLoc = AfterLParen.isValid()? AfterLParen : loc;
     
-    if (castType->isObjCARCBridgableType() && 
-        castExprType->isCARCBridgableType()) {
-      // explicit unbridged casts are allowed if the source of the cast is a 
-      // message sent to an objc method (or property access)
-      if (ValidObjCARCNoBridgeCastExpr(castExpr, castType))
-        return;
+    if (castACTC == ACTC_retainable && isAnyRetainable(exprACTC)) {
       Diag(loc, diag::err_arc_cast_requires_bridge)
         << 2
         << castExprType
@@ -1757,8 +1927,7 @@ Sema::CheckObjCARCConversion(SourceRange castRange, QualType castType,
       return;
     }
     
-    if (castType->isCARCBridgableType() && 
-        castExprType->isObjCARCBridgableType()){
+    if (exprACTC == ACTC_retainable && isAnyRetainable(castACTC)) {
       Diag(loc, diag::err_arc_cast_requires_bridge)
         << (castExprType->isBlockPointerType()? 1 : 0)
         << castExprType
@@ -1806,7 +1975,7 @@ static Expr *maybeUndoReclaimObject(Expr *e) {
   // value-propagating subexpressions --- we can't reliably rebuild
   // in-place because of expression sharing.
   if (ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(e))
-    if (ice->getCastKind() == CK_ObjCReclaimReturnedObject)
+    if (ice->getCastKind() == CK_ARCReclaimReturnedObject)
       return ice->getSubExpr();
 
   return e;
@@ -1817,14 +1986,23 @@ ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
                                       SourceLocation BridgeKeywordLoc,
                                       TypeSourceInfo *TSInfo,
                                       Expr *SubExpr) {
+  ExprResult SubResult = UsualUnaryConversions(SubExpr);
+  if (SubResult.isInvalid()) return ExprError();
+  SubExpr = SubResult.take();
+
   QualType T = TSInfo->getType();
   QualType FromType = SubExpr->getType();
 
+  CastKind CK;
+
   bool MustConsume = false;
   if (T->isDependentType() || SubExpr->isTypeDependent()) {
     // Okay: we'll build a dependent expression type.
+    CK = CK_Dependent;
   } else if (T->isObjCARCBridgableType() && FromType->isCARCBridgableType()) {
     // Casting CF -> id
+    CK = (T->isBlockPointerType() ? CK_AnyPointerToBlockPointerCast
+                                  : CK_CPointerToObjCPointerCast);
     switch (Kind) {
     case OBC_Bridge:
       break;
@@ -1854,6 +2032,7 @@ ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
     }
   } else if (T->isCARCBridgableType() && FromType->isObjCARCBridgableType()) {
     // Okay: id -> CF
+    CK = CK_BitCast;
     switch (Kind) {
     case OBC_Bridge:
       // Reclaiming a value that's going to be __bridge-casted to CF
@@ -1864,7 +2043,7 @@ ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
     case OBC_BridgeRetained:        
       // Produce the object before casting it.
       SubExpr = ImplicitCastExpr::Create(Context, FromType,
-                                         CK_ObjCProduceObject,
+                                         CK_ARCProduceObject,
                                          SubExpr, 0, VK_RValue);
       break;
       
@@ -1894,13 +2073,13 @@ ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
     return ExprError();
   }
 
-  Expr *Result = new (Context) ObjCBridgedCastExpr(LParenLoc, Kind, 
+  Expr *Result = new (Context) ObjCBridgedCastExpr(LParenLoc, Kind, CK,
                                                    BridgeKeywordLoc,
                                                    TSInfo, SubExpr);
   
   if (MustConsume) {
     ExprNeedsCleanups = true;
-    Result = ImplicitCastExpr::Create(Context, T, CK_ObjCConsumeObject, Result, 
+    Result = ImplicitCastExpr::Create(Context, T, CK_ARCConsumeObject, Result, 
                                       0, VK_RValue);    
   }
   
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaFixItUtils.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaFixItUtils.cpp
new file mode 100644
index 000000000000..8e8a46da73e4
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaFixItUtils.cpp
@@ -0,0 +1,160 @@
+//===--- SemaFixItUtils.cpp - Sema FixIts ---------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines helper classes for generation of Sema FixItHints.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaFixItUtils.h"
+
+using namespace clang;
+
+bool ConversionFixItGenerator::compareTypesSimple(CanQualType From,
+                                                  CanQualType To,
+                                                  Sema &S,
+                                                  SourceLocation Loc,
+                                                  ExprValueKind FromVK) {
+  if (!To.isAtLeastAsQualifiedAs(From))
+    return false;
+
+  From = From.getNonReferenceType();
+  To = To.getNonReferenceType();
+
+  // If both are pointer types, work with the pointee types.
+  if (isa<PointerType>(From) && isa<PointerType>(To)) {
+    From = S.Context.getCanonicalType(
+        (cast<PointerType>(From))->getPointeeType());
+    To = S.Context.getCanonicalType(
+        (cast<PointerType>(To))->getPointeeType());
+  }
+
+  const CanQualType FromUnq = From.getUnqualifiedType();
+  const CanQualType ToUnq = To.getUnqualifiedType();
+
+  if ((FromUnq == ToUnq || (S.IsDerivedFrom(FromUnq, ToUnq)) ) &&
+      To.isAtLeastAsQualifiedAs(From))
+    return true;
+  return false;
+}
+
+bool ConversionFixItGenerator::tryToFixConversion(const Expr *FullExpr,
+                                                  const QualType FromTy,
+                                                  const QualType ToTy,
+                                                  Sema &S) {
+  if (!FullExpr)
+    return false;
+
+  const CanQualType FromQTy = S.Context.getCanonicalType(FromTy);
+  const CanQualType ToQTy = S.Context.getCanonicalType(ToTy);
+  const SourceLocation Begin = FullExpr->getSourceRange().getBegin();
+  const SourceLocation End = S.PP.getLocForEndOfToken(FullExpr->getSourceRange()
+                                                      .getEnd());
+
+  // Strip the implicit casts - those are implied by the compiler, not the
+  // original source code.
+  const Expr* Expr = FullExpr->IgnoreImpCasts();
+
+  bool NeedParen = true;
+  if (isa<ArraySubscriptExpr>(Expr) ||
+      isa<CallExpr>(Expr) ||
+      isa<DeclRefExpr>(Expr) ||
+      isa<CastExpr>(Expr) ||
+      isa<CXXNewExpr>(Expr) ||
+      isa<CXXConstructExpr>(Expr) ||
+      isa<CXXDeleteExpr>(Expr) ||
+      isa<CXXNoexceptExpr>(Expr) ||
+      isa<CXXPseudoDestructorExpr>(Expr) ||
+      isa<CXXScalarValueInitExpr>(Expr) ||
+      isa<CXXThisExpr>(Expr) ||
+      isa<CXXTypeidExpr>(Expr) ||
+      isa<CXXUnresolvedConstructExpr>(Expr) ||
+      isa<ObjCMessageExpr>(Expr) ||
+      isa<ObjCPropertyRefExpr>(Expr) ||
+      isa<ObjCProtocolExpr>(Expr) ||
+      isa<MemberExpr>(Expr) ||
+      isa<ParenExpr>(FullExpr) ||
+      isa<ParenListExpr>(Expr) ||
+      isa<SizeOfPackExpr>(Expr) ||
+      isa<UnaryOperator>(Expr))
+    NeedParen = false;
+
+  // Check if the argument needs to be dereferenced:
+  //   (type * -> type) or (type * -> type &).
+  if (const PointerType *FromPtrTy = dyn_cast<PointerType>(FromQTy)) {
+    OverloadFixItKind FixKind = OFIK_Dereference;
+
+    bool CanConvert = CompareTypes(
+      S.Context.getCanonicalType(FromPtrTy->getPointeeType()), ToQTy,
+                                 S, Begin, VK_LValue);
+    if (CanConvert) {
+      // Do not suggest dereferencing a Null pointer.
+      if (Expr->IgnoreParenCasts()->
+          isNullPointerConstant(S.Context, Expr::NPC_ValueDependentIsNotNull))
+        return false;
+
+      if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Expr)) {
+        if (UO->getOpcode() == UO_AddrOf) {
+          FixKind = OFIK_RemoveTakeAddress;
+          Hints.push_back(FixItHint::CreateRemoval(
+                            CharSourceRange::getTokenRange(Begin, Begin)));
+        }
+      } else if (NeedParen) {
+        Hints.push_back(FixItHint::CreateInsertion(Begin, "*("));
+        Hints.push_back(FixItHint::CreateInsertion(End, ")"));
+      } else {
+        Hints.push_back(FixItHint::CreateInsertion(Begin, "*"));
+      }
+
+      NumConversionsFixed++;
+      if (NumConversionsFixed == 1)
+        Kind = FixKind;
+      return true;
+    }
+  }
+
+  // Check if the pointer to the argument needs to be passed:
+  //   (type -> type *) or (type & -> type *).
+  if (isa<PointerType>(ToQTy)) {
+    bool CanConvert = false;
+    OverloadFixItKind FixKind = OFIK_TakeAddress;
+
+    // Only suggest taking address of L-values.
+    if (!Expr->isLValue() || Expr->getObjectKind() != OK_Ordinary)
+      return false;
+
+    CanConvert = CompareTypes(S.Context.getPointerType(FromQTy), ToQTy,
+                              S, Begin, VK_RValue);
+    if (CanConvert) {
+
+      if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Expr)) {
+        if (UO->getOpcode() == UO_Deref) {
+          FixKind = OFIK_RemoveDereference;
+          Hints.push_back(FixItHint::CreateRemoval(
+                            CharSourceRange::getTokenRange(Begin, Begin)));
+        }
+      } else if (NeedParen) {
+        Hints.push_back(FixItHint::CreateInsertion(Begin, "&("));
+        Hints.push_back(FixItHint::CreateInsertion(End, ")"));
+      } else {
+        Hints.push_back(FixItHint::CreateInsertion(Begin, "&"));
+      }
+
+      NumConversionsFixed++;
+      if (NumConversionsFixed == 1)
+        Kind = FixKind;
+      return true;
+    }
+  }
+
+  return false;
+}
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaInit.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaInit.cpp
index 9fbcbab0b7d0..7ed3fa84c8d5 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaInit.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaInit.cpp
@@ -7,9 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements semantic analysis for initializers. The main entry
-// point is Sema::CheckInitList(), but all of the work is performed
-// within the InitListChecker class.
+// This file implements semantic analysis for initializers.
 //
 //===----------------------------------------------------------------------===//
 
@@ -24,6 +22,7 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeLoc.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
 #include <map>
 using namespace clang;
 
@@ -48,20 +47,30 @@ static Expr *IsStringInit(Expr *Init, const ArrayType *AT,
   if (SL == 0) return 0;
 
   QualType ElemTy = Context.getCanonicalType(AT->getElementType());
-  // char array can be initialized with a narrow string.
-  // Only allow char x[] = "foo";  not char x[] = L"foo";
-  if (!SL->isWide())
+
+  switch (SL->getKind()) {
+  case StringLiteral::Ascii:
+  case StringLiteral::UTF8:
+    // char array can be initialized with a narrow string.
+    // Only allow char x[] = "foo";  not char x[] = L"foo";
     return ElemTy->isCharType() ? Init : 0;
+  case StringLiteral::UTF16:
+    return ElemTy->isChar16Type() ? Init : 0;
+  case StringLiteral::UTF32:
+    return ElemTy->isChar32Type() ? Init : 0;
+  case StringLiteral::Wide:
+    // wchar_t array can be initialized with a wide string: C99 6.7.8p15 (with
+    // correction from DR343): "An array with element type compatible with a
+    // qualified or unqualified version of wchar_t may be initialized by a wide
+    // string literal, optionally enclosed in braces."
+    if (Context.typesAreCompatible(Context.getWCharType(),
+                                   ElemTy.getUnqualifiedType()))
+      return Init;
 
-  // wchar_t array can be initialized with a wide string: C99 6.7.8p15 (with
-  // correction from DR343): "An array with element type compatible with a
-  // qualified or unqualified version of wchar_t may be initialized by a wide
-  // string literal, optionally enclosed in braces."
-  if (Context.typesAreCompatible(Context.getWCharType(),
-                                 ElemTy.getUnqualifiedType()))
-    return Init;
+    return 0;
+  }
 
-  return 0;
+  llvm_unreachable("missed a StringLiteral kind?");
 }
 
 static Expr *IsStringInit(Expr *init, QualType declType, ASTContext &Context) {
@@ -160,14 +169,14 @@ namespace {
 class InitListChecker {
   Sema &SemaRef;
   bool hadError;
+  bool VerifyOnly; // no diagnostics, no structure building
   std::map<InitListExpr *, InitListExpr *> SyntacticToSemantic;
   InitListExpr *FullyStructuredList;
 
   void CheckImplicitInitList(const InitializedEntity &Entity,
                              InitListExpr *ParentIList, QualType T,
                              unsigned &Index, InitListExpr *StructuredList,
-                             unsigned &StructuredIndex,
-                             bool TopLevelObject = false);
+                             unsigned &StructuredIndex);
   void CheckExplicitInitList(const InitializedEntity &Entity,
                              InitListExpr *IList, QualType &T,
                              unsigned &Index, InitListExpr *StructuredList,
@@ -185,6 +194,11 @@ class InitListChecker {
                            unsigned &Index,
                            InitListExpr *StructuredList,
                            unsigned &StructuredIndex);
+  void CheckComplexType(const InitializedEntity &Entity,
+                        InitListExpr *IList, QualType DeclType,
+                        unsigned &Index,
+                        InitListExpr *StructuredList,
+                        unsigned &StructuredIndex);
   void CheckScalarType(const InitializedEntity &Entity,
                        InitListExpr *IList, QualType DeclType,
                        unsigned &Index,
@@ -239,9 +253,12 @@ class InitListChecker {
                                InitListExpr *ILE, bool &RequiresSecondPass);
   void FillInValueInitializations(const InitializedEntity &Entity,
                                   InitListExpr *ILE, bool &RequiresSecondPass);
+  bool CheckFlexibleArrayInit(const InitializedEntity &Entity,
+                              Expr *InitExpr, FieldDecl *Field,
+                              bool TopLevelObject);
 public:
   InitListChecker(Sema &S, const InitializedEntity &Entity,
-                  InitListExpr *IL, QualType &T);
+                  InitListExpr *IL, QualType &T, bool VerifyOnly);
   bool HadError() { return hadError; }
 
   // @brief Retrieves the fully-structured initializer list used for
@@ -434,8 +451,9 @@ InitListChecker::FillInValueInitializations(const InitializedEntity &Entity,
 
 
 InitListChecker::InitListChecker(Sema &S, const InitializedEntity &Entity,
-                                 InitListExpr *IL, QualType &T)
-  : SemaRef(S) {
+                                 InitListExpr *IL, QualType &T,
+                                 bool VerifyOnly)
+  : SemaRef(S), VerifyOnly(VerifyOnly) {
   hadError = false;
 
   unsigned newIndex = 0;
@@ -446,7 +464,7 @@ InitListChecker::InitListChecker(Sema &S, const InitializedEntity &Entity,
                         FullyStructuredList, newStructuredIndex,
                         /*TopLevelObject=*/true);
 
-  if (!hadError) {
+  if (!hadError && !VerifyOnly) {
     bool RequiresSecondPass = false;
     FillInValueInitializations(Entity, FullyStructuredList, RequiresSecondPass);
     if (RequiresSecondPass && !hadError)
@@ -472,7 +490,7 @@ int InitListChecker::numStructUnionElements(QualType DeclType) {
          Field = structDecl->field_begin(),
          FieldEnd = structDecl->field_end();
        Field != FieldEnd; ++Field) {
-    if ((*Field)->getIdentifier() || !(*Field)->isBitField())
+    if (!Field->isUnnamedBitfield())
       ++InitializableMembers;
   }
   if (structDecl->isUnion())
@@ -484,8 +502,7 @@ void InitListChecker::CheckImplicitInitList(const InitializedEntity &Entity,
                                             InitListExpr *ParentIList,
                                             QualType T, unsigned &Index,
                                             InitListExpr *StructuredList,
-                                            unsigned &StructuredIndex,
-                                            bool TopLevelObject) {
+                                            unsigned &StructuredIndex) {
   int maxElements = 0;
 
   if (T->isArrayType())
@@ -495,11 +512,12 @@ void InitListChecker::CheckImplicitInitList(const InitializedEntity &Entity,
   else if (T->isVectorType())
     maxElements = T->getAs<VectorType>()->getNumElements();
   else
-    assert(0 && "CheckImplicitInitList(): Illegal type");
+    llvm_unreachable("CheckImplicitInitList(): Illegal type");
 
   if (maxElements == 0) {
-    SemaRef.Diag(ParentIList->getInit(Index)->getLocStart(),
-                  diag::err_implicit_empty_initializer);
+    if (!VerifyOnly)
+      SemaRef.Diag(ParentIList->getInit(Index)->getLocStart(),
+                   diag::err_implicit_empty_initializer);
     ++Index;
     hadError = true;
     return;
@@ -518,29 +536,31 @@ void InitListChecker::CheckImplicitInitList(const InitializedEntity &Entity,
   CheckListElementTypes(Entity, ParentIList, T,
                         /*SubobjectIsDesignatorContext=*/false, Index,
                         StructuredSubobjectInitList,
-                        StructuredSubobjectInitIndex,
-                        TopLevelObject);
+                        StructuredSubobjectInitIndex);
   unsigned EndIndex = (Index == StartIndex? StartIndex : Index - 1);
-  StructuredSubobjectInitList->setType(T);
+  if (!VerifyOnly) {
+    StructuredSubobjectInitList->setType(T);
 
-  // Update the structured sub-object initializer so that it's ending
-  // range corresponds with the end of the last initializer it used.
-  if (EndIndex < ParentIList->getNumInits()) {
-    SourceLocation EndLoc
-      = ParentIList->getInit(EndIndex)->getSourceRange().getEnd();
-    StructuredSubobjectInitList->setRBraceLoc(EndLoc);
-  }
+    // Update the structured sub-object initializer so that it's ending
+    // range corresponds with the end of the last initializer it used.
+    if (EndIndex < ParentIList->getNumInits()) {
+      SourceLocation EndLoc
+        = ParentIList->getInit(EndIndex)->getSourceRange().getEnd();
+      StructuredSubobjectInitList->setRBraceLoc(EndLoc);
+    }
 
-  // Warn about missing braces.
-  if (T->isArrayType() || T->isRecordType()) {
-    SemaRef.Diag(StructuredSubobjectInitList->getLocStart(),
-                 diag::warn_missing_braces)
-    << StructuredSubobjectInitList->getSourceRange()
-    << FixItHint::CreateInsertion(StructuredSubobjectInitList->getLocStart(),
-                                  "{")
-    << FixItHint::CreateInsertion(SemaRef.PP.getLocForEndOfToken(
+    // Warn about missing braces.
+    if (T->isArrayType() || T->isRecordType()) {
+      SemaRef.Diag(StructuredSubobjectInitList->getLocStart(),
+                   diag::warn_missing_braces)
+        << StructuredSubobjectInitList->getSourceRange()
+        << FixItHint::CreateInsertion(
+              StructuredSubobjectInitList->getLocStart(), "{")
+        << FixItHint::CreateInsertion(
+              SemaRef.PP.getLocForEndOfToken(
                                       StructuredSubobjectInitList->getLocEnd()),
-                                  "}");
+              "}");
+    }
   }
 }
 
@@ -551,18 +571,31 @@ void InitListChecker::CheckExplicitInitList(const InitializedEntity &Entity,
                                             unsigned &StructuredIndex,
                                             bool TopLevelObject) {
   assert(IList->isExplicit() && "Illegal Implicit InitListExpr");
-  SyntacticToSemantic[IList] = StructuredList;
-  StructuredList->setSyntacticForm(IList);
+  if (!VerifyOnly) {
+    SyntacticToSemantic[IList] = StructuredList;
+    StructuredList->setSyntacticForm(IList);
+  }
   CheckListElementTypes(Entity, IList, T, /*SubobjectIsDesignatorContext=*/true,
                         Index, StructuredList, StructuredIndex, TopLevelObject);
-  QualType ExprTy = T.getNonLValueExprType(SemaRef.Context);
-  IList->setType(ExprTy);
-  StructuredList->setType(ExprTy);
+  if (!VerifyOnly) {
+    QualType ExprTy = T.getNonLValueExprType(SemaRef.Context);
+    IList->setType(ExprTy);
+    StructuredList->setType(ExprTy);
+  }
   if (hadError)
     return;
 
   if (Index < IList->getNumInits()) {
     // We have leftover initializers
+    if (VerifyOnly) {
+      if (SemaRef.getLangOptions().CPlusPlus ||
+          (SemaRef.getLangOptions().OpenCL &&
+           IList->getType()->isVectorType())) {
+        hadError = true;
+      }
+      return;
+    }
+
     if (StructuredIndex == 1 &&
         IsStringInit(StructuredList->getInit(0), T, SemaRef.Context)) {
       unsigned DK = diag::warn_excess_initializers_in_char_array_initializer;
@@ -599,7 +632,8 @@ void InitListChecker::CheckExplicitInitList(const InitializedEntity &Entity,
     }
   }
 
-  if (T->isScalarType() && !TopLevelObject)
+  if (!VerifyOnly && T->isScalarType() && IList->getNumInits() == 1 &&
+      !TopLevelObject)
     SemaRef.Diag(IList->getLocStart(), diag::warn_braces_around_scalar_init)
       << IList->getSourceRange()
       << FixItHint::CreateRemoval(IList->getLocStart())
@@ -614,7 +648,12 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
                                             InitListExpr *StructuredList,
                                             unsigned &StructuredIndex,
                                             bool TopLevelObject) {
-  if (DeclType->isScalarType()) {
+  if (DeclType->isAnyComplexType() && SubobjectIsDesignatorContext) {
+    // Explicitly braced initializer for complex type can be real+imaginary
+    // parts.
+    CheckComplexType(Entity, IList, DeclType, Index,
+                     StructuredList, StructuredIndex);
+  } else if (DeclType->isScalarType()) {
     CheckScalarType(Entity, IList, DeclType, Index,
                     StructuredList, StructuredIndex);
   } else if (DeclType->isVectorType()) {
@@ -635,12 +674,13 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
                      SubobjectIsDesignatorContext, Index,
                      StructuredList, StructuredIndex);
     } else
-      assert(0 && "Aggregate that isn't a structure or array?!");
+      llvm_unreachable("Aggregate that isn't a structure or array?!");
   } else if (DeclType->isVoidType() || DeclType->isFunctionType()) {
     // This type is invalid, issue a diagnostic.
     ++Index;
-    SemaRef.Diag(IList->getLocStart(), diag::err_illegal_initializer_type)
-      << DeclType;
+    if (!VerifyOnly)
+      SemaRef.Diag(IList->getLocStart(), diag::err_illegal_initializer_type)
+        << DeclType;
     hadError = true;
   } else if (DeclType->isRecordType()) {
     // C++ [dcl.init]p14:
@@ -651,19 +691,22 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
     // we have an initializer list and a destination type that is not
     // an aggregate.
     // FIXME: In C++0x, this is yet another form of initialization.
-    SemaRef.Diag(IList->getLocStart(), diag::err_init_non_aggr_init_list)
-      << DeclType << IList->getSourceRange();
+    if (!VerifyOnly)
+      SemaRef.Diag(IList->getLocStart(), diag::err_init_non_aggr_init_list)
+        << DeclType << IList->getSourceRange();
     hadError = true;
   } else if (DeclType->isReferenceType()) {
     CheckReferenceType(Entity, IList, DeclType, Index,
                        StructuredList, StructuredIndex);
   } else if (DeclType->isObjCObjectType()) {
-    SemaRef.Diag(IList->getLocStart(), diag::err_init_objc_class)
-      << DeclType;
+    if (!VerifyOnly)
+      SemaRef.Diag(IList->getLocStart(), diag::err_init_objc_class)
+        << DeclType;
     hadError = true;
   } else {
-    SemaRef.Diag(IList->getLocStart(), diag::err_illegal_initializer_type)
-      << DeclType;
+    if (!VerifyOnly)
+      SemaRef.Diag(IList->getLocStart(), diag::err_illegal_initializer_type)
+        << DeclType;
     hadError = true;
   }
 }
@@ -701,8 +744,10 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
     // type here, though.
 
     if (Expr *Str = IsStringInit(expr, arrayType, SemaRef.Context)) {
-      CheckStringInit(Str, ElemType, arrayType, SemaRef);
-      UpdateStructuredListElement(StructuredList, StructuredIndex, Str);
+      if (!VerifyOnly) {
+        CheckStringInit(Str, ElemType, arrayType, SemaRef);
+        UpdateStructuredListElement(StructuredList, StructuredIndex, Str);
+      }
       ++Index;
       return;
     }
@@ -712,7 +757,7 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
   } else if (SemaRef.getLangOptions().CPlusPlus) {
     // C++ [dcl.init.aggr]p12:
     //   All implicit type conversions (clause 4) are considered when
-    //   initializing the aggregate member with an ini- tializer from
+    //   initializing the aggregate member with an initializer from
     //   an initializer-list. If the initializer can initialize a
     //   member, the member is initialized. [...]
 
@@ -722,13 +767,15 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
     InitializationSequence Seq(SemaRef, Entity, Kind, &expr, 1);
 
     if (Seq) {
-      ExprResult Result =
-        Seq.Perform(SemaRef, Entity, Kind, MultiExprArg(&expr, 1));
-      if (Result.isInvalid())
-        hadError = true;
+      if (!VerifyOnly) {
+        ExprResult Result =
+          Seq.Perform(SemaRef, Entity, Kind, MultiExprArg(&expr, 1));
+        if (Result.isInvalid())
+          hadError = true;
 
-      UpdateStructuredListElement(StructuredList, StructuredIndex,
-                                  Result.takeAs<Expr>());
+        UpdateStructuredListElement(StructuredList, StructuredIndex,
+                                    Result.takeAs<Expr>());
+      }
       ++Index;
       return;
     }
@@ -745,7 +792,8 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
     //   that of the expression.
     ExprResult ExprRes = SemaRef.Owned(expr);
     if ((ElemType->isRecordType() || ElemType->isVectorType()) &&
-        SemaRef.CheckSingleAssignmentConstraints(ElemType, ExprRes)
+        SemaRef.CheckSingleAssignmentConstraints(ElemType, ExprRes,
+                                                 !VerifyOnly)
           == Sema::Compatible) {
       if (ExprRes.isInvalid())
         hadError = true;
@@ -775,25 +823,68 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
                           StructuredIndex);
     ++StructuredIndex;
   } else {
-    // We cannot initialize this element, so let
-    // PerformCopyInitialization produce the appropriate diagnostic.
-    SemaRef.PerformCopyInitialization(Entity, SourceLocation(),
-                                      SemaRef.Owned(expr));
+    if (!VerifyOnly) {
+      // We cannot initialize this element, so let
+      // PerformCopyInitialization produce the appropriate diagnostic.
+      SemaRef.PerformCopyInitialization(Entity, SourceLocation(),
+                                        SemaRef.Owned(expr),
+                                        /*TopLevelOfInitList=*/true);
+    }
     hadError = true;
     ++Index;
     ++StructuredIndex;
   }
 }
 
+void InitListChecker::CheckComplexType(const InitializedEntity &Entity,
+                                       InitListExpr *IList, QualType DeclType,
+                                       unsigned &Index,
+                                       InitListExpr *StructuredList,
+                                       unsigned &StructuredIndex) {
+  assert(Index == 0 && "Index in explicit init list must be zero");
+
+  // As an extension, clang supports complex initializers, which initialize
+  // a complex number component-wise.  When an explicit initializer list for
+  // a complex number contains two two initializers, this extension kicks in:
+  // it exepcts the initializer list to contain two elements convertible to
+  // the element type of the complex type. The first element initializes
+  // the real part, and the second element intitializes the imaginary part.
+
+  if (IList->getNumInits() != 2)
+    return CheckScalarType(Entity, IList, DeclType, Index, StructuredList,
+                           StructuredIndex);
+
+  // This is an extension in C.  (The builtin _Complex type does not exist
+  // in the C++ standard.)
+  if (!SemaRef.getLangOptions().CPlusPlus && !VerifyOnly)
+    SemaRef.Diag(IList->getLocStart(), diag::ext_complex_component_init)
+      << IList->getSourceRange();
+
+  // Initialize the complex number.
+  QualType elementType = DeclType->getAs<ComplexType>()->getElementType();
+  InitializedEntity ElementEntity =
+    InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
+
+  for (unsigned i = 0; i < 2; ++i) {
+    ElementEntity.setElementIndex(Index);
+    CheckSubElementType(ElementEntity, IList, elementType, Index,
+                        StructuredList, StructuredIndex);
+  }
+}
+
+
 void InitListChecker::CheckScalarType(const InitializedEntity &Entity,
                                       InitListExpr *IList, QualType DeclType,
                                       unsigned &Index,
                                       InitListExpr *StructuredList,
                                       unsigned &StructuredIndex) {
   if (Index >= IList->getNumInits()) {
-    SemaRef.Diag(IList->getLocStart(), diag::err_empty_scalar_initializer)
-      << IList->getSourceRange();
-    hadError = true;
+    if (!SemaRef.getLangOptions().CPlusPlus0x) {
+      if (!VerifyOnly)
+        SemaRef.Diag(IList->getLocStart(), diag::err_empty_scalar_initializer)
+          << IList->getSourceRange();
+      hadError = true;
+    }
     ++Index;
     ++StructuredIndex;
     return;
@@ -801,26 +892,36 @@ void InitListChecker::CheckScalarType(const InitializedEntity &Entity,
 
   Expr *expr = IList->getInit(Index);
   if (InitListExpr *SubIList = dyn_cast<InitListExpr>(expr)) {
-    SemaRef.Diag(SubIList->getLocStart(),
-                 diag::warn_many_braces_around_scalar_init)
-      << SubIList->getSourceRange();
+    if (!VerifyOnly)
+      SemaRef.Diag(SubIList->getLocStart(),
+                   diag::warn_many_braces_around_scalar_init)
+        << SubIList->getSourceRange();
 
     CheckScalarType(Entity, SubIList, DeclType, Index, StructuredList,
                     StructuredIndex);
     return;
   } else if (isa<DesignatedInitExpr>(expr)) {
-    SemaRef.Diag(expr->getSourceRange().getBegin(),
-                 diag::err_designator_for_scalar_init)
-      << DeclType << expr->getSourceRange();
+    if (!VerifyOnly)
+      SemaRef.Diag(expr->getSourceRange().getBegin(),
+                   diag::err_designator_for_scalar_init)
+        << DeclType << expr->getSourceRange();
     hadError = true;
     ++Index;
     ++StructuredIndex;
     return;
   }
 
+  if (VerifyOnly) {
+    if (!SemaRef.CanPerformCopyInitialization(Entity, SemaRef.Owned(expr)))
+      hadError = true;
+    ++Index;
+    return;
+  }
+
   ExprResult Result =
     SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
-                                      SemaRef.Owned(expr));
+                                      SemaRef.Owned(expr),
+                                      /*TopLevelOfInitList=*/true);
 
   Expr *ResultExpr = 0;
 
@@ -846,46 +947,57 @@ void InitListChecker::CheckReferenceType(const InitializedEntity &Entity,
                                          unsigned &Index,
                                          InitListExpr *StructuredList,
                                          unsigned &StructuredIndex) {
-  if (Index < IList->getNumInits()) {
-    Expr *expr = IList->getInit(Index);
-    if (isa<InitListExpr>(expr)) {
-      SemaRef.Diag(IList->getLocStart(), diag::err_init_non_aggr_init_list)
-        << DeclType << IList->getSourceRange();
-      hadError = true;
-      ++Index;
-      ++StructuredIndex;
-      return;
-    }
-
-    ExprResult Result =
-      SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
-                                        SemaRef.Owned(expr));
-
-    if (Result.isInvalid())
-      hadError = true;
-
-    expr = Result.takeAs<Expr>();
-    IList->setInit(Index, expr);
-
-    if (hadError)
-      ++StructuredIndex;
-    else
-      UpdateStructuredListElement(StructuredList, StructuredIndex, expr);
-    ++Index;
-  } else {
+  if (Index >= IList->getNumInits()) {
     // FIXME: It would be wonderful if we could point at the actual member. In
     // general, it would be useful to pass location information down the stack,
     // so that we know the location (or decl) of the "current object" being
     // initialized.
-    SemaRef.Diag(IList->getLocStart(),
-                  diag::err_init_reference_member_uninitialized)
-      << DeclType
-      << IList->getSourceRange();
+    if (!VerifyOnly)
+      SemaRef.Diag(IList->getLocStart(),
+                    diag::err_init_reference_member_uninitialized)
+        << DeclType
+        << IList->getSourceRange();
     hadError = true;
     ++Index;
     ++StructuredIndex;
     return;
   }
+
+  Expr *expr = IList->getInit(Index);
+  if (isa<InitListExpr>(expr)) {
+    // FIXME: Allowed in C++11.
+    if (!VerifyOnly)
+      SemaRef.Diag(IList->getLocStart(), diag::err_init_non_aggr_init_list)
+        << DeclType << IList->getSourceRange();
+    hadError = true;
+    ++Index;
+    ++StructuredIndex;
+    return;
+  }
+
+  if (VerifyOnly) {
+    if (!SemaRef.CanPerformCopyInitialization(Entity, SemaRef.Owned(expr)))
+      hadError = true;
+    ++Index;
+    return;
+  }
+
+  ExprResult Result =
+    SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
+                                      SemaRef.Owned(expr),
+                                      /*TopLevelOfInitList=*/true);
+
+  if (Result.isInvalid())
+    hadError = true;
+
+  expr = Result.takeAs<Expr>();
+  IList->setInit(Index, expr);
+
+  if (hadError)
+    ++StructuredIndex;
+  else
+    UpdateStructuredListElement(StructuredList, StructuredIndex, expr);
+  ++Index;
 }
 
 void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
@@ -906,9 +1018,17 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
     // instead of breaking it apart (which is doomed to failure anyway).
     Expr *Init = IList->getInit(Index);
     if (!isa<InitListExpr>(Init) && Init->getType()->isVectorType()) {
+      if (VerifyOnly) {
+        if (!SemaRef.CanPerformCopyInitialization(Entity, SemaRef.Owned(Init)))
+          hadError = true;
+        ++Index;
+        return;
+      }
+
       ExprResult Result =
         SemaRef.PerformCopyInitialization(Entity, Init->getLocStart(),
-                                          SemaRef.Owned(Init));
+                                          SemaRef.Owned(Init),
+                                          /*TopLevelOfInitList=*/true);
 
       Expr *ResultExpr = 0;
       if (Result.isInvalid())
@@ -924,7 +1044,8 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
       if (hadError)
         ++StructuredIndex;
       else
-        UpdateStructuredListElement(StructuredList, StructuredIndex, ResultExpr);
+        UpdateStructuredListElement(StructuredList, StructuredIndex,
+                                    ResultExpr);
       ++Index;
       return;
     }
@@ -977,11 +1098,11 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
   }
 
   // OpenCL requires all elements to be initialized.
-  if (numEltsInit != maxElements)
-    if (SemaRef.getLangOptions().OpenCL)
-      SemaRef.Diag(IList->getSourceRange().getBegin(),
-                   diag::err_vector_incorrect_num_initializers)
-        << (numEltsInit < maxElements) << maxElements << numEltsInit;
+  // FIXME: Shouldn't this set hadError to true then?
+  if (numEltsInit != maxElements && !VerifyOnly)
+    SemaRef.Diag(IList->getSourceRange().getBegin(),
+                 diag::err_vector_incorrect_num_initializers)
+      << (numEltsInit < maxElements) << maxElements << numEltsInit;
 }
 
 void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
@@ -997,14 +1118,16 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
   if (Index < IList->getNumInits()) {
     if (Expr *Str = IsStringInit(IList->getInit(Index), arrayType,
                                  SemaRef.Context)) {
-      CheckStringInit(Str, DeclType, arrayType, SemaRef);
       // We place the string literal directly into the resulting
       // initializer list. This is the only place where the structure
       // of the structured initializer list doesn't match exactly,
       // because doing so would involve allocating one character
       // constant for each string.
-      UpdateStructuredListElement(StructuredList, StructuredIndex, Str);
-      StructuredList->resizeInits(SemaRef.Context, StructuredIndex);
+      if (!VerifyOnly) {
+        CheckStringInit(Str, DeclType, arrayType, SemaRef);
+        UpdateStructuredListElement(StructuredList, StructuredIndex, Str);
+        StructuredList->resizeInits(SemaRef.Context, StructuredIndex);
+      }
       ++Index;
       return;
     }
@@ -1013,9 +1136,10 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
     // Check for VLAs; in standard C it would be possible to check this
     // earlier, but I don't know where clang accepts VLAs (gcc accepts
     // them in all sorts of strange places).
-    SemaRef.Diag(VAT->getSizeExpr()->getLocStart(),
-                  diag::err_variable_object_no_init)
-      << VAT->getSizeExpr()->getSourceRange();
+    if (!VerifyOnly)
+      SemaRef.Diag(VAT->getSizeExpr()->getLocStart(),
+                    diag::err_variable_object_no_init)
+        << VAT->getSizeExpr()->getSourceRange();
     hadError = true;
     ++Index;
     ++StructuredIndex;
@@ -1085,7 +1209,7 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
     if (!maxElementsKnown && elementIndex > maxElements)
       maxElements = elementIndex;
   }
-  if (!hadError && DeclType->isIncompleteArrayType()) {
+  if (!hadError && DeclType->isIncompleteArrayType() && !VerifyOnly) {
     // If this is an incomplete array type, the actual type needs to
     // be calculated here.
     llvm::APSInt Zero(maxElements.getBitWidth(), maxElements.isUnsigned());
@@ -1101,6 +1225,45 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
   }
 }
 
+bool InitListChecker::CheckFlexibleArrayInit(const InitializedEntity &Entity,
+                                             Expr *InitExpr,
+                                             FieldDecl *Field,
+                                             bool TopLevelObject) {
+  // Handle GNU flexible array initializers.
+  unsigned FlexArrayDiag;
+  if (isa<InitListExpr>(InitExpr) &&
+      cast<InitListExpr>(InitExpr)->getNumInits() == 0) {
+    // Empty flexible array init always allowed as an extension
+    FlexArrayDiag = diag::ext_flexible_array_init;
+  } else if (SemaRef.getLangOptions().CPlusPlus) {
+    // Disallow flexible array init in C++; it is not required for gcc
+    // compatibility, and it needs work to IRGen correctly in general.
+    FlexArrayDiag = diag::err_flexible_array_init;
+  } else if (!TopLevelObject) {
+    // Disallow flexible array init on non-top-level object
+    FlexArrayDiag = diag::err_flexible_array_init;
+  } else if (Entity.getKind() != InitializedEntity::EK_Variable) {
+    // Disallow flexible array init on anything which is not a variable.
+    FlexArrayDiag = diag::err_flexible_array_init;
+  } else if (cast<VarDecl>(Entity.getDecl())->hasLocalStorage()) {
+    // Disallow flexible array init on local variables.
+    FlexArrayDiag = diag::err_flexible_array_init;
+  } else {
+    // Allow other cases.
+    FlexArrayDiag = diag::ext_flexible_array_init;
+  }
+
+  if (!VerifyOnly) {
+    SemaRef.Diag(InitExpr->getSourceRange().getBegin(),
+                 FlexArrayDiag)
+      << InitExpr->getSourceRange().getBegin();
+    SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
+      << Field;
+  }
+
+  return FlexArrayDiag != diag::ext_flexible_array_init;
+}
+
 void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
                                             InitListExpr *IList,
                                             QualType DeclType,
@@ -1120,13 +1283,15 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
   }
 
   if (DeclType->isUnionType() && IList->getNumInits() == 0) {
-    // Value-initialize the first named member of the union.
-    RecordDecl *RD = DeclType->getAs<RecordType>()->getDecl();
-    for (RecordDecl::field_iterator FieldEnd = RD->field_end();
-         Field != FieldEnd; ++Field) {
-      if (Field->getDeclName()) {
-        StructuredList->setInitializedFieldInUnion(*Field);
-        break;
+    if (!VerifyOnly) {
+      // Value-initialize the first named member of the union.
+      RecordDecl *RD = DeclType->getAs<RecordType>()->getDecl();
+      for (RecordDecl::field_iterator FieldEnd = RD->field_end();
+           Field != FieldEnd; ++Field) {
+        if (Field->getDeclName()) {
+          StructuredList->setInitializedFieldInUnion(*Field);
+          break;
+        }
       }
     }
     return;
@@ -1186,13 +1351,18 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
     }
 
     // Make sure we can use this declaration.
-    if (SemaRef.DiagnoseUseOfDecl(*Field, 
-                                  IList->getInit(Index)->getLocStart())) {
+    bool InvalidUse;
+    if (VerifyOnly)
+      InvalidUse = !SemaRef.CanUseDecl(*Field);
+    else
+      InvalidUse = SemaRef.DiagnoseUseOfDecl(*Field,
+                                          IList->getInit(Index)->getLocStart());
+    if (InvalidUse) {
       ++Index;
       ++Field;
       hadError = true;
       continue;
-    }        
+    }
 
     InitializedEntity MemberEntity =
       InitializedEntity::InitializeMember(*Field, &Entity);
@@ -1200,7 +1370,7 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
                         StructuredList, StructuredIndex);
     InitializedSomething = true;
 
-    if (DeclType->isUnionType()) {
+    if (DeclType->isUnionType() && !VerifyOnly) {
       // Initialize the first field within the union.
       StructuredList->setInitializedFieldInUnion(*Field);
     }
@@ -1209,8 +1379,9 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
   }
 
   // Emit warnings for missing struct field initializers.
-  if (InitializedSomething && CheckForMissingFields && Field != FieldEnd &&
-      !Field->getType()->isIncompleteArrayType() && !DeclType->isUnionType()) {
+  if (!VerifyOnly && InitializedSomething && CheckForMissingFields &&
+      Field != FieldEnd && !Field->getType()->isIncompleteArrayType() &&
+      !DeclType->isUnionType()) {
     // It is possible we have one or more unnamed bitfields remaining.
     // Find first (if any) named field and emit warning.
     for (RecordDecl::field_iterator it = Field, end = RD->field_end();
@@ -1227,24 +1398,11 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
       Index >= IList->getNumInits())
     return;
 
-  // Handle GNU flexible array initializers.
-  if (!TopLevelObject &&
-      (!isa<InitListExpr>(IList->getInit(Index)) ||
-       cast<InitListExpr>(IList->getInit(Index))->getNumInits() > 0)) {
-    SemaRef.Diag(IList->getInit(Index)->getSourceRange().getBegin(),
-                  diag::err_flexible_array_init_nonempty)
-      << IList->getInit(Index)->getSourceRange().getBegin();
-    SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
-      << *Field;
+  if (CheckFlexibleArrayInit(Entity, IList->getInit(Index), *Field,
+                             TopLevelObject)) {
     hadError = true;
     ++Index;
     return;
-  } else {
-    SemaRef.Diag(IList->getInit(Index)->getSourceRange().getBegin(),
-                 diag::ext_flexible_array_init)
-      << IList->getInit(Index)->getSourceRange().getBegin();
-    SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
-      << *Field;
   }
 
   InitializedEntity MemberEntity =
@@ -1269,7 +1427,7 @@ static void ExpandAnonymousFieldDesignator(Sema &SemaRef,
   typedef DesignatedInitExpr::Designator Designator;
 
   // Build the replacement designators.
-  llvm::SmallVector<Designator, 4> Replacements;
+  SmallVector<Designator, 4> Replacements;
   for (IndirectFieldDecl::chain_iterator PI = IndirectField->chain_begin(),
        PE = IndirectField->chain_end(); PI != PE; ++PI) {
     if (PI + 1 == PE)
@@ -1304,6 +1462,18 @@ static IndirectFieldDecl *FindIndirectFieldDesignator(FieldDecl *AnonField,
   return 0;
 }
 
+static DesignatedInitExpr *CloneDesignatedInitExpr(Sema &SemaRef,
+                                                   DesignatedInitExpr *DIE) {
+  unsigned NumIndexExprs = DIE->getNumSubExprs() - 1;
+  SmallVector<Expr*, 4> IndexExprs(NumIndexExprs);
+  for (unsigned I = 0; I < NumIndexExprs; ++I)
+    IndexExprs[I] = DIE->getSubExpr(I + 1);
+  return DesignatedInitExpr::Create(SemaRef.Context, DIE->designators_begin(),
+                                    DIE->size(), IndexExprs.data(),
+                                    NumIndexExprs, DIE->getEqualOrColonLoc(),
+                                    DIE->usesGNUSyntax(), DIE->getInit());
+}
+
 /// @brief Check the well-formedness of a C99 designated initializer.
 ///
 /// Determines whether the designated initializer @p DIE, which
@@ -1342,14 +1512,14 @@ static IndirectFieldDecl *FindIndirectFieldDesignator(FieldDecl *AnonField,
 bool
 InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
                                             InitListExpr *IList,
-                                      DesignatedInitExpr *DIE,
-                                      unsigned DesigIdx,
-                                      QualType &CurrentObjectType,
-                                      RecordDecl::field_iterator *NextField,
-                                      llvm::APSInt *NextElementIndex,
-                                      unsigned &Index,
-                                      InitListExpr *StructuredList,
-                                      unsigned &StructuredIndex,
+                                            DesignatedInitExpr *DIE,
+                                            unsigned DesigIdx,
+                                            QualType &CurrentObjectType,
+                                          RecordDecl::field_iterator *NextField,
+                                            llvm::APSInt *NextElementIndex,
+                                            unsigned &Index,
+                                            InitListExpr *StructuredList,
+                                            unsigned &StructuredIndex,
                                             bool FinishSubobjectInit,
                                             bool TopLevelObject) {
   if (DesigIdx == DIE->size()) {
@@ -1374,19 +1544,21 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     return hadError && !prevHadError;
   }
 
-  bool IsFirstDesignator = (DesigIdx == 0);
-  assert((IsFirstDesignator || StructuredList) &&
-         "Need a non-designated initializer list to start from");
-
   DesignatedInitExpr::Designator *D = DIE->getDesignator(DesigIdx);
-  // Determine the structural initializer list that corresponds to the
-  // current subobject.
-  StructuredList = IsFirstDesignator? SyntacticToSemantic[IList]
-    : getStructuredSubobjectInit(IList, Index, CurrentObjectType,
-                                 StructuredList, StructuredIndex,
-                                 SourceRange(D->getStartLocation(),
-                                             DIE->getSourceRange().getEnd()));
-  assert(StructuredList && "Expected a structured initializer list");
+  bool IsFirstDesignator = (DesigIdx == 0);
+  if (!VerifyOnly) {
+    assert((IsFirstDesignator || StructuredList) &&
+           "Need a non-designated initializer list to start from");
+
+    // Determine the structural initializer list that corresponds to the
+    // current subobject.
+    StructuredList = IsFirstDesignator? SyntacticToSemantic[IList]
+      : getStructuredSubobjectInit(IList, Index, CurrentObjectType,
+                                   StructuredList, StructuredIndex,
+                                   SourceRange(D->getStartLocation(),
+                                               DIE->getSourceRange().getEnd()));
+    assert(StructuredList && "Expected a structured initializer list");
+  }
 
   if (D->isFieldDesignator()) {
     // C99 6.7.8p7:
@@ -1403,8 +1575,9 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       SourceLocation Loc = D->getDotLoc();
       if (Loc.isInvalid())
         Loc = D->getFieldLoc();
-      SemaRef.Diag(Loc, diag::err_field_designator_non_aggr)
-        << SemaRef.getLangOptions().CPlusPlus << CurrentObjectType;
+      if (!VerifyOnly)
+        SemaRef.Diag(Loc, diag::err_field_designator_non_aggr)
+          << SemaRef.getLangOptions().CPlusPlus << CurrentObjectType;
       ++Index;
       return true;
     }
@@ -1427,6 +1600,9 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       if (!KnownField && Field->isAnonymousStructOrUnion()) {
         if (IndirectFieldDecl *IF =
             FindIndirectFieldDesignator(*Field, FieldName)) {
+          // In verify mode, don't modify the original.
+          if (VerifyOnly)
+            DIE = CloneDesignatedInitExpr(SemaRef, DIE);
           ExpandAnonymousFieldDesignator(SemaRef, DIE, DesigIdx, IF);
           D = DIE->getDesignator(DesigIdx);
           break;
@@ -1441,6 +1617,11 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     }
 
     if (Field == FieldEnd) {
+      if (VerifyOnly) {
+        ++Index;
+        return true; // No typo correction when just trying this out.
+      }
+
       // There was no normal field in the struct with the designated
       // name. Perform another lookup for this name, which may find
       // something that we can't designate (e.g., a member function),
@@ -1470,6 +1651,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
             << FixItHint::CreateReplacement(D->getFieldLoc(), CorrectedStr);
           SemaRef.Diag(ReplacementField->getLocation(),
                        diag::note_previous_decl) << CorrectedQuotedStr;
+          hadError = true;
         } else {
           SemaRef.Diag(D->getFieldLoc(), diag::err_field_designator_unknown)
             << FieldName << CurrentObjectType;
@@ -1510,22 +1692,30 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     // the initializer list.
     if (RT->getDecl()->isUnion()) {
       FieldIndex = 0;
-      StructuredList->setInitializedFieldInUnion(*Field);
+      if (!VerifyOnly)
+        StructuredList->setInitializedFieldInUnion(*Field);
     }
 
     // Make sure we can use this declaration.
-    if (SemaRef.DiagnoseUseOfDecl(*Field, D->getFieldLoc())) {
+    bool InvalidUse;
+    if (VerifyOnly)
+      InvalidUse = !SemaRef.CanUseDecl(*Field);
+    else
+      InvalidUse = SemaRef.DiagnoseUseOfDecl(*Field, D->getFieldLoc());
+    if (InvalidUse) {
       ++Index;
       return true;
-    }        
+    }
 
-    // Update the designator with the field declaration.
-    D->setField(*Field);
+    if (!VerifyOnly) {
+      // Update the designator with the field declaration.
+      D->setField(*Field);
 
-    // Make sure that our non-designated initializer list has space
-    // for a subobject corresponding to this field.
-    if (FieldIndex >= StructuredList->getNumInits())
-      StructuredList->resizeInits(SemaRef.Context, FieldIndex + 1);
+      // Make sure that our non-designated initializer list has space
+      // for a subobject corresponding to this field.
+      if (FieldIndex >= StructuredList->getNumInits())
+        StructuredList->resizeInits(SemaRef.Context, FieldIndex + 1);
+    }
 
     // This designator names a flexible array member.
     if (Field->getType()->isIncompleteArrayType()) {
@@ -1533,38 +1723,36 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       if ((DesigIdx + 1) != DIE->size()) {
         // We can't designate an object within the flexible array
         // member (because GCC doesn't allow it).
-        DesignatedInitExpr::Designator *NextD
-          = DIE->getDesignator(DesigIdx + 1);
-        SemaRef.Diag(NextD->getStartLocation(),
-                      diag::err_designator_into_flexible_array_member)
-          << SourceRange(NextD->getStartLocation(),
-                         DIE->getSourceRange().getEnd());
-        SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
-          << *Field;
+        if (!VerifyOnly) {
+          DesignatedInitExpr::Designator *NextD
+            = DIE->getDesignator(DesigIdx + 1);
+          SemaRef.Diag(NextD->getStartLocation(),
+                        diag::err_designator_into_flexible_array_member)
+            << SourceRange(NextD->getStartLocation(),
+                           DIE->getSourceRange().getEnd());
+          SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
+            << *Field;
+        }
         Invalid = true;
       }
 
       if (!hadError && !isa<InitListExpr>(DIE->getInit()) &&
           !isa<StringLiteral>(DIE->getInit())) {
         // The initializer is not an initializer list.
-        SemaRef.Diag(DIE->getInit()->getSourceRange().getBegin(),
-                      diag::err_flexible_array_init_needs_braces)
-          << DIE->getInit()->getSourceRange();
-        SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
-          << *Field;
+        if (!VerifyOnly) {
+          SemaRef.Diag(DIE->getInit()->getSourceRange().getBegin(),
+                        diag::err_flexible_array_init_needs_braces)
+            << DIE->getInit()->getSourceRange();
+          SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
+            << *Field;
+        }
         Invalid = true;
       }
 
-      // Handle GNU flexible array initializers.
-      if (!Invalid && !TopLevelObject &&
-          cast<InitListExpr>(DIE->getInit())->getNumInits() > 0) {
-        SemaRef.Diag(DIE->getSourceRange().getBegin(),
-                      diag::err_flexible_array_init_nonempty)
-          << DIE->getSourceRange().getBegin();
-        SemaRef.Diag(Field->getLocation(), diag::note_flexible_array_member)
-          << *Field;
+      // Check GNU flexible array initializer.
+      if (!Invalid && CheckFlexibleArrayInit(Entity, DIE->getInit(), *Field,
+                                             TopLevelObject))
         Invalid = true;
-      }
 
       if (Invalid) {
         ++Index;
@@ -1651,8 +1839,9 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
   //      [ constant-expression ... constant-expression ]
   const ArrayType *AT = SemaRef.Context.getAsArrayType(CurrentObjectType);
   if (!AT) {
-    SemaRef.Diag(D->getLBracketLoc(), diag::err_array_designator_non_array)
-      << CurrentObjectType;
+    if (!VerifyOnly)
+      SemaRef.Diag(D->getLBracketLoc(), diag::err_array_designator_non_array)
+        << CurrentObjectType;
     ++Index;
     return true;
   }
@@ -1661,15 +1850,15 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
   llvm::APSInt DesignatedStartIndex, DesignatedEndIndex;
   if (D->isArrayDesignator()) {
     IndexExpr = DIE->getArrayIndex(*D);
-    DesignatedStartIndex = IndexExpr->EvaluateAsInt(SemaRef.Context);
+    DesignatedStartIndex = IndexExpr->EvaluateKnownConstInt(SemaRef.Context);
     DesignatedEndIndex = DesignatedStartIndex;
   } else {
     assert(D->isArrayRangeDesignator() && "Need array-range designator");
 
     DesignatedStartIndex =
-      DIE->getArrayRangeStart(*D)->EvaluateAsInt(SemaRef.Context);
+      DIE->getArrayRangeStart(*D)->EvaluateKnownConstInt(SemaRef.Context);
     DesignatedEndIndex =
-      DIE->getArrayRangeEnd(*D)->EvaluateAsInt(SemaRef.Context);
+      DIE->getArrayRangeEnd(*D)->EvaluateKnownConstInt(SemaRef.Context);
     IndexExpr = DIE->getArrayRangeEnd(*D);
 
     // Codegen can't handle evaluating array range designators that have side
@@ -1678,7 +1867,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     // elements with something that has a side effect, so codegen can emit an
     // "error unsupported" error instead of miscompiling the app.
     if (DesignatedStartIndex.getZExtValue()!=DesignatedEndIndex.getZExtValue()&&
-        DIE->getInit()->HasSideEffects(SemaRef.Context))
+        DIE->getInit()->HasSideEffects(SemaRef.Context) && !VerifyOnly)
       FullyStructuredList->sawArrayRangeDesignator();
   }
 
@@ -1691,10 +1880,11 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       = DesignatedEndIndex.extOrTrunc(MaxElements.getBitWidth());
     DesignatedEndIndex.setIsUnsigned(MaxElements.isUnsigned());
     if (DesignatedEndIndex >= MaxElements) {
-      SemaRef.Diag(IndexExpr->getSourceRange().getBegin(),
-                    diag::err_array_designator_too_large)
-        << DesignatedEndIndex.toString(10) << MaxElements.toString(10)
-        << IndexExpr->getSourceRange();
+      if (!VerifyOnly)
+        SemaRef.Diag(IndexExpr->getSourceRange().getBegin(),
+                      diag::err_array_designator_too_large)
+          << DesignatedEndIndex.toString(10) << MaxElements.toString(10)
+          << IndexExpr->getSourceRange();
       ++Index;
       return true;
     }
@@ -1713,7 +1903,8 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
 
   // Make sure that our non-designated initializer list has space
   // for a subobject corresponding to this array element.
-  if (DesignatedEndIndex.getZExtValue() >= StructuredList->getNumInits())
+  if (!VerifyOnly &&
+      DesignatedEndIndex.getZExtValue() >= StructuredList->getNumInits())
     StructuredList->resizeInits(SemaRef.Context,
                                 DesignatedEndIndex.getZExtValue() + 1);
 
@@ -1773,6 +1964,8 @@ InitListChecker::getStructuredSubobjectInit(InitListExpr *IList, unsigned Index,
                                             InitListExpr *StructuredList,
                                             unsigned StructuredIndex,
                                             SourceRange InitRange) {
+  if (VerifyOnly)
+    return 0; // No structured list in verification-only mode.
   Expr *ExistingInit = 0;
   if (!StructuredList)
     ExistingInit = SyntacticToSemantic[IList];
@@ -1844,9 +2037,6 @@ InitListChecker::getStructuredSubobjectInit(InitListExpr *IList, unsigned Index,
                                   RDecl->field_end());
   }
 
-  if (NumElements < NumInits)
-    NumElements = IList->getNumInits();
-
   Result->reserveInits(SemaRef.Context, NumElements);
 
   // Link this new initializer list into the structured initializer
@@ -1915,8 +2105,8 @@ ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
   typedef DesignatedInitExpr::Designator ASTDesignator;
 
   bool Invalid = false;
-  llvm::SmallVector<ASTDesignator, 32> Designators;
-  llvm::SmallVector<Expr *, 32> InitExpressions;
+  SmallVector<ASTDesignator, 32> Designators;
+  SmallVector<Expr *, 32> InitExpressions;
 
   // Build designators and check array designator expressions.
   for (unsigned Idx = 0; Idx < Desig.getNumDesignators(); ++Idx) {
@@ -2007,15 +2197,6 @@ ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
   return Owned(DIE);
 }
 
-bool Sema::CheckInitList(const InitializedEntity &Entity,
-                         InitListExpr *&InitList, QualType &DeclType) {
-  InitListChecker CheckInitList(*this, Entity, InitList, DeclType);
-  if (!CheckInitList.HadError())
-    InitList = CheckInitList.getFullyStructuredList();
-
-  return CheckInitList.HadError();
-}
-
 //===----------------------------------------------------------------------===//
 // Initialization entity
 //===----------------------------------------------------------------------===//
@@ -2027,9 +2208,14 @@ InitializedEntity::InitializedEntity(ASTContext &Context, unsigned Index,
   if (const ArrayType *AT = Context.getAsArrayType(Parent.getType())) {
     Kind = EK_ArrayElement;
     Type = AT->getElementType();
-  } else {
+  } else if (const VectorType *VT = Parent.getType()->getAs<VectorType>()) {
     Kind = EK_VectorElement;
-    Type = Parent.getType()->getAs<VectorType>()->getElementType();
+    Type = VT->getElementType();
+  } else {
+    const ComplexType *CT = Parent.getType()->getAs<ComplexType>();
+    assert(CT && "Unexpected type");
+    Kind = EK_ComplexElement;
+    Type = CT->getElementType();
   }
 }
 
@@ -2066,6 +2252,7 @@ DeclarationName InitializedEntity::getName() const {
   case EK_Delegating:
   case EK_ArrayElement:
   case EK_VectorElement:
+  case EK_ComplexElement:
   case EK_BlockElement:
     return DeclarationName();
   }
@@ -2091,6 +2278,7 @@ DeclaratorDecl *InitializedEntity::getDecl() const {
   case EK_Delegating:
   case EK_ArrayElement:
   case EK_VectorElement:
+  case EK_ComplexElement:
   case EK_BlockElement:
     return 0;
   }
@@ -2114,6 +2302,7 @@ bool InitializedEntity::allowsNRVO() const {
   case EK_Delegating:
   case EK_ArrayElement:
   case EK_VectorElement:
+  case EK_ComplexElement:
   case EK_BlockElement:
     break;
   }
@@ -2139,6 +2328,7 @@ void InitializationSequence::Step::Destroy() {
   case SK_QualificationConversionXValue:
   case SK_QualificationConversionLValue:
   case SK_ListInitialization:
+  case SK_ListConstructorCall:
   case SK_ConstructorInitialization:
   case SK_ZeroInitialization:
   case SK_CAssignment:
@@ -2182,6 +2372,7 @@ bool InitializationSequence::isAmbiguous() const {
   case FK_Incomplete:
   case FK_ArrayTypeMismatch:
   case FK_NonConstantArrayInit:
+  case FK_ListInitializationFailed:
     return false;
 
   case FK_ReferenceInitOverloadFailed:
@@ -2197,12 +2388,171 @@ bool InitializationSequence::isConstructorInitialization() const {
   return !Steps.empty() && Steps.back().Kind == SK_ConstructorInitialization;
 }
 
+bool InitializationSequence::endsWithNarrowing(ASTContext &Ctx,
+                                               const Expr *Initializer,
+                                               bool *isInitializerConstant,
+                                               APValue *ConstantValue) const {
+  if (Steps.empty() || Initializer->isValueDependent())
+    return false;
+
+  const Step &LastStep = Steps.back();
+  if (LastStep.Kind != SK_ConversionSequence)
+    return false;
+
+  const ImplicitConversionSequence &ICS = *LastStep.ICS;
+  const StandardConversionSequence *SCS = NULL;
+  switch (ICS.getKind()) {
+  case ImplicitConversionSequence::StandardConversion:
+    SCS = &ICS.Standard;
+    break;
+  case ImplicitConversionSequence::UserDefinedConversion:
+    SCS = &ICS.UserDefined.After;
+    break;
+  case ImplicitConversionSequence::AmbiguousConversion:
+  case ImplicitConversionSequence::EllipsisConversion:
+  case ImplicitConversionSequence::BadConversion:
+    return false;
+  }
+
+  // Check if SCS represents a narrowing conversion, according to C++0x
+  // [dcl.init.list]p7:
+  //
+  // A narrowing conversion is an implicit conversion ...
+  ImplicitConversionKind PossibleNarrowing = SCS->Second;
+  QualType FromType = SCS->getToType(0);
+  QualType ToType = SCS->getToType(1);
+  switch (PossibleNarrowing) {
+  // * from a floating-point type to an integer type, or
+  //
+  // * from an integer type or unscoped enumeration type to a floating-point
+  //   type, except where the source is a constant expression and the actual
+  //   value after conversion will fit into the target type and will produce
+  //   the original value when converted back to the original type, or
+  case ICK_Floating_Integral:
+    if (FromType->isRealFloatingType() && ToType->isIntegralType(Ctx)) {
+      *isInitializerConstant = false;
+      return true;
+    } else if (FromType->isIntegralType(Ctx) && ToType->isRealFloatingType()) {
+      llvm::APSInt IntConstantValue;
+      if (Initializer &&
+          Initializer->isIntegerConstantExpr(IntConstantValue, Ctx)) {
+        // Convert the integer to the floating type.
+        llvm::APFloat Result(Ctx.getFloatTypeSemantics(ToType));
+        Result.convertFromAPInt(IntConstantValue, IntConstantValue.isSigned(),
+                                llvm::APFloat::rmNearestTiesToEven);
+        // And back.
+        llvm::APSInt ConvertedValue = IntConstantValue;
+        bool ignored;
+        Result.convertToInteger(ConvertedValue,
+                                llvm::APFloat::rmTowardZero, &ignored);
+        // If the resulting value is different, this was a narrowing conversion.
+        if (IntConstantValue != ConvertedValue) {
+          *isInitializerConstant = true;
+          *ConstantValue = APValue(IntConstantValue);
+          return true;
+        }
+      } else {
+        // Variables are always narrowings.
+        *isInitializerConstant = false;
+        return true;
+      }
+    }
+    return false;
+
+  // * from long double to double or float, or from double to float, except
+  //   where the source is a constant expression and the actual value after
+  //   conversion is within the range of values that can be represented (even
+  //   if it cannot be represented exactly), or
+  case ICK_Floating_Conversion:
+    if (1 == Ctx.getFloatingTypeOrder(FromType, ToType)) {
+      // FromType is larger than ToType.
+      Expr::EvalResult InitializerValue;
+      // FIXME: Check whether Initializer is a constant expression according
+      // to C++0x [expr.const], rather than just whether it can be folded.
+      if (Initializer->Evaluate(InitializerValue, Ctx) &&
+          !InitializerValue.HasSideEffects && InitializerValue.Val.isFloat()) {
+        // Constant! (Except for FIXME above.)
+        llvm::APFloat FloatVal = InitializerValue.Val.getFloat();
+        // Convert the source value into the target type.
+        bool ignored;
+        llvm::APFloat::opStatus ConvertStatus = FloatVal.convert(
+          Ctx.getFloatTypeSemantics(ToType),
+          llvm::APFloat::rmNearestTiesToEven, &ignored);
+        // If there was no overflow, the source value is within the range of
+        // values that can be represented.
+        if (ConvertStatus & llvm::APFloat::opOverflow) {
+          *isInitializerConstant = true;
+          *ConstantValue = InitializerValue.Val;
+          return true;
+        }
+      } else {
+        *isInitializerConstant = false;
+        return true;
+      }
+    }
+    return false;
+
+  // * from an integer type or unscoped enumeration type to an integer type
+  //   that cannot represent all the values of the original type, except where
+  //   the source is a constant expression and the actual value after
+  //   conversion will fit into the target type and will produce the original
+  //   value when converted back to the original type.
+  case ICK_Boolean_Conversion:  // Bools are integers too.
+    if (!FromType->isIntegralOrUnscopedEnumerationType()) {
+      // Boolean conversions can be from pointers and pointers to members
+      // [conv.bool], and those aren't considered narrowing conversions.
+      return false;
+    }  // Otherwise, fall through to the integral case.
+  case ICK_Integral_Conversion: {
+    assert(FromType->isIntegralOrUnscopedEnumerationType());
+    assert(ToType->isIntegralOrUnscopedEnumerationType());
+    const bool FromSigned = FromType->isSignedIntegerOrEnumerationType();
+    const unsigned FromWidth = Ctx.getIntWidth(FromType);
+    const bool ToSigned = ToType->isSignedIntegerOrEnumerationType();
+    const unsigned ToWidth = Ctx.getIntWidth(ToType);
+
+    if (FromWidth > ToWidth ||
+        (FromWidth == ToWidth && FromSigned != ToSigned)) {
+      // Not all values of FromType can be represented in ToType.
+      llvm::APSInt InitializerValue;
+      if (Initializer->isIntegerConstantExpr(InitializerValue, Ctx)) {
+        *isInitializerConstant = true;
+        *ConstantValue = APValue(InitializerValue);
+
+        // Add a bit to the InitializerValue so we don't have to worry about
+        // signed vs. unsigned comparisons.
+        InitializerValue = InitializerValue.extend(
+          InitializerValue.getBitWidth() + 1);
+        // Convert the initializer to and from the target width and signed-ness.
+        llvm::APSInt ConvertedValue = InitializerValue;
+        ConvertedValue = ConvertedValue.trunc(ToWidth);
+        ConvertedValue.setIsSigned(ToSigned);
+        ConvertedValue = ConvertedValue.extend(InitializerValue.getBitWidth());
+        ConvertedValue.setIsSigned(InitializerValue.isSigned());
+        // If the result is different, this was a narrowing conversion.
+        return ConvertedValue != InitializerValue;
+      } else {
+        // Variables are always narrowings.
+        *isInitializerConstant = false;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  default:
+    // Other kinds of conversions are not narrowings.
+    return false;
+  }
+}
+
 void InitializationSequence::AddAddressOverloadResolutionStep(
                                                       FunctionDecl *Function,
                                                       DeclAccessPair Found) {
   Step S;
   S.Kind = SK_ResolveAddressOfOverloadedFunction;
   S.Type = Function->getType();
+  S.Function.HadMultipleCandidates = false;
   S.Function.Function = Function;
   S.Function.FoundDecl = Found;
   Steps.push_back(S);
@@ -2242,6 +2592,7 @@ void InitializationSequence::AddUserConversionStep(FunctionDecl *Function,
   Step S;
   S.Kind = SK_UserConversion;
   S.Type = T;
+  S.Function.HadMultipleCandidates = false;
   S.Function.Function = Function;
   S.Function.FoundDecl = FoundDecl;
   Steps.push_back(S);
@@ -2291,6 +2642,7 @@ InitializationSequence::AddConstructorInitializationStep(
   Step S;
   S.Kind = SK_ConstructorInitialization;
   S.Type = T;
+  S.Function.HadMultipleCandidates = false;
   S.Function.Function = Constructor;
   S.Function.FoundDecl = DeclAccessPair::make(Constructor, Access);
   Steps.push_back(S);
@@ -2391,44 +2743,33 @@ static void TryListInitialization(Sema &S,
                                   const InitializationKind &Kind,
                                   InitListExpr *InitList,
                                   InitializationSequence &Sequence) {
-  // FIXME: We only perform rudimentary checking of list
-  // initializations at this point, then assume that any list
-  // initialization of an array, aggregate, or scalar will be
-  // well-formed. When we actually "perform" list initialization, we'll
-  // do all of the necessary checking.  C++0x initializer lists will
-  // force us to perform more checking here.
-
   QualType DestType = Entity.getType();
 
-  // C++ [dcl.init]p13:
-  //   If T is a scalar type, then a declaration of the form
-  //
-  //     T x = { a };
-  //
-  //   is equivalent to
-  //
-  //     T x = a;
-  if (DestType->isScalarType()) {
-    if (InitList->getNumInits() > 1 && S.getLangOptions().CPlusPlus) {
-      Sequence.SetFailed(InitializationSequence::FK_TooManyInitsForScalar);
-      return;
-    }
-
-    // Assume scalar initialization from a single value works.
-  } else if (DestType->isAggregateType()) {
-    // Assume aggregate initialization works.
-  } else if (DestType->isVectorType()) {
-    // Assume vector initialization works.
-  } else if (DestType->isReferenceType()) {
-    // FIXME: C++0x defines behavior for this.
+  // C++ doesn't allow scalar initialization with more than one argument.
+  // But C99 complex numbers are scalars and it makes sense there.
+  if (S.getLangOptions().CPlusPlus && DestType->isScalarType() &&
+      !DestType->isAnyComplexType() && InitList->getNumInits() > 1) {
+    Sequence.SetFailed(InitializationSequence::FK_TooManyInitsForScalar);
+    return;
+  }
+  // FIXME: C++0x defines behavior for these two cases.
+  if (DestType->isReferenceType()) {
     Sequence.SetFailed(InitializationSequence::FK_ReferenceBindingToInitList);
     return;
-  } else if (DestType->isRecordType()) {
-    // FIXME: C++0x defines behavior for this
+  }
+  if (DestType->isRecordType() && !DestType->isAggregateType()) {
     Sequence.SetFailed(InitializationSequence::FK_InitListBadDestinationType);
+    return;
   }
 
-  // Add a general "list initialization" step.
+  InitListChecker CheckInitList(S, Entity, InitList,
+          DestType, /*VerifyOnly=*/true);
+  if (CheckInitList.HadError()) {
+    Sequence.SetFailed(InitializationSequence::FK_ListInitializationFailed);
+    return;
+  }
+
+  // Add the list initialization step with the built init list.
   Sequence.AddListInitializationStep(DestType);
 }
 
@@ -2767,7 +3108,7 @@ static void TryReferenceInitialization(Sema &S,
       //
       //   The constructor that would be used to make the copy shall
       //   be callable whether or not the copy is actually done.
-      if (!S.getLangOptions().CPlusPlus0x && !S.getLangOptions().Microsoft)
+      if (!S.getLangOptions().CPlusPlus0x && !S.getLangOptions().MicrosoftExt)
         Sequence.AddExtraneousCopyToTemporary(cv2T2);
     }
 
@@ -2887,6 +3228,14 @@ static void TryConstructorInitialization(Sema &S,
                                          Expr **Args, unsigned NumArgs,
                                          QualType DestType,
                                          InitializationSequence &Sequence) {
+  // Check constructor arguments for self reference.
+  if (DeclaratorDecl *DD = Entity.getDecl())
+    // Parameters arguments are occassionially constructed with itself,
+    // for instance, in recursive functions.  Skip them.
+    if (!isa<ParmVarDecl>(DD))
+      for (unsigned i = 0; i < NumArgs; ++i)
+        S.CheckSelfReference(DD, Args[i]);
+
   // Build the candidate set directly in the initialization sequence
   // structure, so that it will persist if we fail.
   OverloadCandidateSet &CandidateSet = Sequence.getFailedCandidateSet();
@@ -3577,7 +3926,7 @@ InitializationSequence::InitializationSequence(Sema &S,
 }
 
 InitializationSequence::~InitializationSequence() {
-  for (llvm::SmallVectorImpl<Step>::iterator Step = Steps.begin(),
+  for (SmallVectorImpl<Step>::iterator Step = Steps.begin(),
                                           StepEnd = Steps.end();
        Step != StepEnd; ++Step)
     Step->Destroy();
@@ -3613,6 +3962,7 @@ getAssignmentAction(const InitializedEntity &Entity) {
   case InitializedEntity::EK_Member:
   case InitializedEntity::EK_ArrayElement:
   case InitializedEntity::EK_VectorElement:
+  case InitializedEntity::EK_ComplexElement:
   case InitializedEntity::EK_BlockElement:
     return Sema::AA_Initializing;
   }
@@ -3632,6 +3982,7 @@ static bool shouldBindAsTemporary(const InitializedEntity &Entity) {
   case InitializedEntity::EK_Base:
   case InitializedEntity::EK_Delegating:
   case InitializedEntity::EK_VectorElement:
+  case InitializedEntity::EK_ComplexElement:
   case InitializedEntity::EK_Exception:
   case InitializedEntity::EK_BlockElement:
     return false;
@@ -3654,6 +4005,7 @@ static bool shouldDestroyTemporary(const InitializedEntity &Entity) {
     case InitializedEntity::EK_Base:
     case InitializedEntity::EK_Delegating:
     case InitializedEntity::EK_VectorElement:
+    case InitializedEntity::EK_ComplexElement:
     case InitializedEntity::EK_BlockElement:
       return false;
 
@@ -3739,6 +4091,7 @@ static ExprResult CopyObject(Sema &S,
   case InitializedEntity::EK_Base:
   case InitializedEntity::EK_Delegating:
   case InitializedEntity::EK_VectorElement:
+  case InitializedEntity::EK_ComplexElement:
   case InitializedEntity::EK_BlockElement:
     Loc = CurInitExpr->getLocStart();
     break;
@@ -3790,6 +4143,8 @@ static ExprResult CopyObject(Sema &S,
                                    &CurInitExpr, 1, CandidateSet, true);
   }
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, Loc, Best)) {
   case OR_Success:
@@ -3867,6 +4222,7 @@ static ExprResult CopyObject(Sema &S,
   // Actually perform the constructor call.
   CurInit = S.BuildCXXConstructExpr(Loc, T, Constructor, Elidable,
                                     move_arg(ConstructorArgs),
+                                    HadMultipleCandidates,
                                     /*ZeroInit*/ false,
                                     CXXConstructExpr::CK_Complete,
                                     SourceRange());
@@ -3982,6 +4338,7 @@ InitializationSequence::Perform(Sema &S,
   case SK_QualificationConversionXValue:
   case SK_QualificationConversionRValue:
   case SK_ConversionSequence:
+  case SK_ListConstructorCall:
   case SK_ListInitialization:
   case SK_CAssignment:
   case SK_StringInit:
@@ -4127,8 +4484,8 @@ InitializationSequence::Perform(Sema &S,
       bool IsCopy = false;
       FunctionDecl *Fn = Step->Function.Function;
       DeclAccessPair FoundFn = Step->Function.FoundDecl;
+      bool HadMultipleCandidates = Step->Function.HadMultipleCandidates;
       bool CreatedObject = false;
-      bool IsLvalue = false;
       if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(Fn)) {
         // Build a call to the selected constructor.
         ASTOwningVector<Expr*> ConstructorArgs(S);
@@ -4146,6 +4503,7 @@ InitializationSequence::Perform(Sema &S,
         // Build the an expression that constructs a temporary.
         CurInit = S.BuildCXXConstructExpr(Loc, Step->Type, Constructor,
                                           move_arg(ConstructorArgs),
+                                          HadMultipleCandidates,
                                           /*ZeroInit*/ false,
                                           CXXConstructExpr::CK_Complete,
                                           SourceRange());
@@ -4166,7 +4524,6 @@ InitializationSequence::Perform(Sema &S,
       } else {
         // Build a call to the conversion function.
         CXXConversionDecl *Conversion = cast<CXXConversionDecl>(Fn);
-        IsLvalue = Conversion->getResultType()->isLValueReferenceType();
         S.CheckMemberOperatorAccess(Kind.getLocation(), CurInit.get(), 0,
                                     FoundFn);
         S.DiagnoseUseOfDecl(FoundFn, Kind.getLocation());
@@ -4182,7 +4539,8 @@ InitializationSequence::Perform(Sema &S,
         CurInit = move(CurInitExprRes);
 
         // Build the actual call to the conversion function.
-        CurInit = S.BuildCXXMemberCallExpr(CurInit.get(), FoundFn, Conversion);
+        CurInit = S.BuildCXXMemberCallExpr(CurInit.get(), FoundFn, Conversion,
+                                           HadMultipleCandidates);
         if (CurInit.isInvalid() || !CurInit.get())
           return ExprError();
 
@@ -4206,11 +4564,10 @@ InitializationSequence::Perform(Sema &S,
         }
       }
 
-      // FIXME: xvalues
       CurInit = S.Owned(ImplicitCastExpr::Create(S.Context,
                                                  CurInit.get()->getType(),
                                                  CastKind, CurInit.get(), 0,
-                                           IsLvalue ? VK_LValue : VK_RValue));
+                                                CurInit.get()->getValueKind()));
 
       if (RequiresCopy)
         CurInit = CopyObject(S, Entity.getType().getNonReferenceType(), Entity,
@@ -4251,18 +4608,24 @@ InitializationSequence::Perform(Sema &S,
     case SK_ListInitialization: {
       InitListExpr *InitList = cast<InitListExpr>(CurInit.get());
       QualType Ty = Step->Type;
-      if (S.CheckInitList(Entity, InitList, ResultType? *ResultType : Ty))
+      InitListChecker PerformInitList(S, Entity, InitList,
+          ResultType ? *ResultType : Ty, /*VerifyOnly=*/false);
+      if (PerformInitList.HadError())
         return ExprError();
 
       CurInit.release();
-      CurInit = S.Owned(InitList);
+      CurInit = S.Owned(PerformInitList.getFullyStructuredList());
       break;
     }
 
+    case SK_ListConstructorCall:
+      assert(false && "List constructor calls not yet supported.");
+
     case SK_ConstructorInitialization: {
       unsigned NumArgs = Args.size();
       CXXConstructorDecl *Constructor
         = cast<CXXConstructorDecl>(Step->Function.Function);
+      bool HadMultipleCandidates = Step->Function.HadMultipleCandidates;
 
       // Build a call to the selected constructor.
       ASTOwningVector<Expr*> ConstructorArgs(S);
@@ -4309,6 +4672,7 @@ InitializationSequence::Perform(Sema &S,
                                                                  Exprs,
                                                                  NumExprs,
                                                          Kind.getParenRange(),
+                                                         HadMultipleCandidates,
                                              ConstructorInitRequiresZeroInit));
       } else {
         CXXConstructExpr::ConstructionKind ConstructKind =
@@ -4333,6 +4697,7 @@ InitializationSequence::Perform(Sema &S,
           CurInit = S.BuildCXXConstructExpr(Loc, Entity.getType(),
                                             Constructor, /*Elidable=*/true,
                                             move_arg(ConstructorArgs),
+                                            HadMultipleCandidates,
                                             ConstructorInitRequiresZeroInit,
                                             ConstructKind,
                                             parenRange);
@@ -4340,6 +4705,7 @@ InitializationSequence::Perform(Sema &S,
           CurInit = S.BuildCXXConstructExpr(Loc, Entity.getType(),
                                             Constructor,
                                             move_arg(ConstructorArgs),
+                                            HadMultipleCandidates,
                                             ConstructorInitRequiresZeroInit,
                                             ConstructKind,
                                             parenRange);
@@ -4427,7 +4793,7 @@ InitializationSequence::Perform(Sema &S,
     case SK_ObjCObjectConversion:
       CurInit = S.ImpCastExprToType(CurInit.take(), Step->Type,
                           CK_ObjCObjectLValueCast,
-                          S.CastCategory(CurInit.get()));
+                          CurInit.get()->getValueKind());
       break;
 
     case SK_ArrayInit:
@@ -4463,7 +4829,7 @@ InitializationSequence::Perform(Sema &S,
 
     case SK_ProduceObjCObject:
       CurInit = S.Owned(ImplicitCastExpr::Create(S.Context, Step->Type,
-                                                 CK_ObjCProduceObject,
+                                                 CK_ARCProduceObject,
                                                  CurInit.take(), 0, VK_RValue));
       break;
     }
@@ -4759,17 +5125,28 @@ bool InitializationSequence::Diagnose(Sema &S,
     }
     break;
 
-    case FK_Incomplete:
-      S.RequireCompleteType(Kind.getLocation(), DestType,
-                            diag::err_init_incomplete_type);
-      break;
+  case FK_Incomplete:
+    S.RequireCompleteType(Kind.getLocation(), DestType,
+                          diag::err_init_incomplete_type);
+    break;
+
+  case FK_ListInitializationFailed: {
+    // Run the init list checker again to emit diagnostics.
+    InitListExpr* InitList = cast<InitListExpr>(Args[0]);
+    QualType DestType = Entity.getType();
+    InitListChecker DiagnoseInitList(S, Entity, InitList,
+            DestType, /*VerifyOnly=*/false);
+    assert(DiagnoseInitList.HadError() &&
+           "Inconsistent init list check result.");
+    break;
+  }
   }
 
   PrintInitLocationNote(S, Entity);
   return true;
 }
 
-void InitializationSequence::dump(llvm::raw_ostream &OS) const {
+void InitializationSequence::dump(raw_ostream &OS) const {
   switch (SequenceKind) {
   case FailedSequence: {
     OS << "Failed sequence: ";
@@ -4857,6 +5234,9 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
     case FK_Incomplete:
       OS << "initialization of incomplete type";
       break;
+
+    case FK_ListInitializationFailed:
+      OS << "list initialization checker failure";
     }
     OS << '\n';
     return;
@@ -4906,7 +5286,7 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
       break;
 
     case SK_UserConversion:
-      OS << "user-defined conversion via " << S->Function.Function;
+      OS << "user-defined conversion via " << *S->Function.Function;
       break;
 
     case SK_QualificationConversionRValue:
@@ -4926,7 +5306,11 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
       break;
 
     case SK_ListInitialization:
-      OS << "list initialization";
+      OS << "list aggregate initialization";
+      break;
+
+    case SK_ListConstructorCall:
+      OS << "list initialization via constructor";
       break;
 
     case SK_ConstructorInitialization:
@@ -4972,6 +5356,51 @@ void InitializationSequence::dump() const {
   dump(llvm::errs());
 }
 
+static void DiagnoseNarrowingInInitList(
+    Sema& S, QualType EntityType, const Expr *InitE,
+    bool Constant, const APValue &ConstantValue) {
+  if (Constant) {
+    S.Diag(InitE->getLocStart(),
+           S.getLangOptions().CPlusPlus0x && !S.getLangOptions().MicrosoftExt
+           ? diag::err_init_list_constant_narrowing
+           : diag::warn_init_list_constant_narrowing)
+      << InitE->getSourceRange()
+      << ConstantValue
+      << EntityType.getLocalUnqualifiedType();
+  } else
+    S.Diag(InitE->getLocStart(),
+           S.getLangOptions().CPlusPlus0x && !S.getLangOptions().MicrosoftExt
+           ? diag::err_init_list_variable_narrowing
+           : diag::warn_init_list_variable_narrowing)
+      << InitE->getSourceRange()
+      << InitE->getType().getLocalUnqualifiedType()
+      << EntityType.getLocalUnqualifiedType();
+
+  llvm::SmallString<128> StaticCast;
+  llvm::raw_svector_ostream OS(StaticCast);
+  OS << "static_cast<";
+  if (const TypedefType *TT = EntityType->getAs<TypedefType>()) {
+    // It's important to use the typedef's name if there is one so that the
+    // fixit doesn't break code using types like int64_t.
+    //
+    // FIXME: This will break if the typedef requires qualification.  But
+    // getQualifiedNameAsString() includes non-machine-parsable components.
+    OS << *TT->getDecl();
+  } else if (const BuiltinType *BT = EntityType->getAs<BuiltinType>())
+    OS << BT->getName(S.getLangOptions());
+  else {
+    // Oops, we didn't find the actual type of the variable.  Don't emit a fixit
+    // with a broken cast.
+    return;
+  }
+  OS << ">(";
+  S.Diag(InitE->getLocStart(), diag::note_init_list_narrowing_override)
+    << InitE->getSourceRange()
+    << FixItHint::CreateInsertion(InitE->getLocStart(), OS.str())
+    << FixItHint::CreateInsertion(
+      S.getPreprocessor().getLocForEndOfToken(InitE->getLocEnd()), ")");
+}
+
 //===----------------------------------------------------------------------===//
 // Initialization helper functions
 //===----------------------------------------------------------------------===//
@@ -4993,7 +5422,8 @@ Sema::CanPerformCopyInitialization(const InitializedEntity &Entity,
 ExprResult
 Sema::PerformCopyInitialization(const InitializedEntity &Entity,
                                 SourceLocation EqualLoc,
-                                ExprResult Init) {
+                                ExprResult Init,
+                                bool TopLevelOfInitList) {
   if (Init.isInvalid())
     return ExprError();
 
@@ -5007,5 +5437,13 @@ Sema::PerformCopyInitialization(const InitializedEntity &Entity,
                                                            EqualLoc);
   InitializationSequence Seq(*this, Entity, Kind, &InitE, 1);
   Init.release();
+
+  bool Constant = false;
+  APValue Result;
+  if (TopLevelOfInitList &&
+      Seq.endsWithNarrowing(Context, InitE, &Constant, &Result)) {
+    DiagnoseNarrowingInInitList(*this, Entity.getType(), InitE,
+                                Constant, Result);
+  }
   return Seq.Perform(*this, Entity, Kind, MultiExprArg(&InitE, 1));
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp
index 0e448e31207d..d5bee1d0e368 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp
@@ -35,6 +35,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <limits>
 #include <list>
@@ -86,7 +87,7 @@ namespace {
   /// A collection of using directives, as used by C++ unqualified
   /// lookup.
   class UnqualUsingDirectiveSet {
-    typedef llvm::SmallVector<UnqualUsingEntry, 8> ListTy;
+    typedef SmallVector<UnqualUsingEntry, 8> ListTy;
 
     ListTy list;
     llvm::SmallPtrSet<DeclContext*, 8> visited;
@@ -147,7 +148,7 @@ namespace {
     // by its using directives, transitively) as if they appeared in
     // the given effective context.
     void addUsingDirectives(DeclContext *DC, DeclContext *EffectiveDC) {
-      llvm::SmallVector<DeclContext*,4> queue;
+      SmallVector<DeclContext*,4> queue;
       while (true) {
         DeclContext::udir_iterator I, End;
         for (llvm::tie(I, End) = DC->getUsingDirectives(); I != End; ++I) {
@@ -460,7 +461,7 @@ void LookupResult::setAmbiguousBaseSubobjectTypes(CXXBasePaths &P) {
   setAmbiguous(AmbiguousBaseSubobjectTypes);
 }
 
-void LookupResult::print(llvm::raw_ostream &Out) {
+void LookupResult::print(raw_ostream &Out) {
   Out << Decls.size() << " result(s)";
   if (isAmbiguous()) Out << ", ambiguous";
   if (Paths) Out << ", base paths present";
@@ -549,6 +550,16 @@ void Sema::ForceDeclarationOfImplicitMembers(CXXRecordDecl *Class) {
   if (!Class->hasDeclaredCopyAssignment())
     DeclareImplicitCopyAssignment(Class);
 
+  if (getLangOptions().CPlusPlus0x) {
+    // If the move constructor has not yet been declared, do so now.
+    if (Class->needsImplicitMoveConstructor())
+      DeclareImplicitMoveConstructor(Class); // might not actually do it
+
+    // If the move assignment operator has not yet been declared, do so now.
+    if (Class->needsImplicitMoveAssignment())
+      DeclareImplicitMoveAssignment(Class); // might not actually do it
+  }
+
   // If the destructor has not yet been declared, do so now.
   if (!Class->hasDeclaredDestructor())
     DeclareImplicitDestructor(Class);
@@ -585,11 +596,14 @@ static void DeclareImplicitMemberFunctionsWithName(Sema &S,
     if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC))
       if (Record->getDefinition() &&
           CanDeclareSpecialMemberFunction(S.Context, Record)) {
+        CXXRecordDecl *Class = const_cast<CXXRecordDecl *>(Record);
         if (Record->needsImplicitDefaultConstructor())
-          S.DeclareImplicitDefaultConstructor(
-                                           const_cast<CXXRecordDecl *>(Record));
+          S.DeclareImplicitDefaultConstructor(Class);
         if (!Record->hasDeclaredCopyConstructor())
-          S.DeclareImplicitCopyConstructor(const_cast<CXXRecordDecl *>(Record));
+          S.DeclareImplicitCopyConstructor(Class);
+        if (S.getLangOptions().CPlusPlus0x &&
+            Record->needsImplicitMoveConstructor())
+          S.DeclareImplicitMoveConstructor(Class);
       }
     break;
 
@@ -604,10 +618,17 @@ static void DeclareImplicitMemberFunctionsWithName(Sema &S,
     if (Name.getCXXOverloadedOperator() != OO_Equal)
       break;
 
-    if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC))
-      if (Record->getDefinition() && !Record->hasDeclaredCopyAssignment() &&
-          CanDeclareSpecialMemberFunction(S.Context, Record))
-        S.DeclareImplicitCopyAssignment(const_cast<CXXRecordDecl *>(Record));
+    if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC)) {
+      if (Record->getDefinition() &&
+          CanDeclareSpecialMemberFunction(S.Context, Record)) {
+        CXXRecordDecl *Class = const_cast<CXXRecordDecl *>(Record);
+        if (!Record->hasDeclaredCopyAssignment())
+          S.DeclareImplicitCopyAssignment(Class);
+        if (S.getLangOptions().CPlusPlus0x &&
+            Record->needsImplicitMoveAssignment())
+          S.DeclareImplicitMoveAssignment(Class);
+      }
+    }
     break;
 
   default:
@@ -648,7 +669,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
   //   name lookup. Instead, any conversion function templates visible in the
   //   context of the use are considered. [...]
   const CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
-  if (!Record->isDefinition())
+  if (!Record->isCompleteDefinition())
     return Found;
 
   const UnresolvedSetImpl *Unresolved = Record->getConversionFunctions();
@@ -1187,7 +1208,7 @@ static bool LookupQualifiedNameInUsingDirectives(Sema &S, LookupResult &R,
 
   // We have not yet looked into these namespaces, much less added
   // their "using-children" to the queue.
-  llvm::SmallVector<NamespaceDecl*, 8> Queue;
+  SmallVector<NamespaceDecl*, 8> Queue;
 
   // We have already looked into the initial namespace; seed the queue
   // with its using-children.
@@ -1332,7 +1353,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
   // Make sure that the declaration context is complete.
   assert((!isa<TagDecl>(LookupCtx) ||
           LookupCtx->isDependentContext() ||
-          cast<TagDecl>(LookupCtx)->isDefinition() ||
+          cast<TagDecl>(LookupCtx)->isCompleteDefinition() ||
           Context.getTypeDeclType(cast<TagDecl>(LookupCtx))->getAs<TagType>()
             ->isBeingDefined()) &&
          "Declaration context must already be complete!");
@@ -1802,7 +1823,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
 
   // Add direct and indirect base classes along with their associated
   // namespaces.
-  llvm::SmallVector<CXXRecordDecl *, 32> Bases;
+  SmallVector<CXXRecordDecl *, 32> Bases;
   Bases.push_back(Class);
   while (!Bases.empty()) {
     // Pop this class off the stack.
@@ -1852,7 +1873,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
   //   the types do not contribute to this set. The sets of namespaces
   //   and classes are determined in the following way:
 
-  llvm::SmallVector<const Type *, 16> Queue;
+  SmallVector<const Type *, 16> Queue;
   const Type *T = Ty->getCanonicalTypeInternal().getTypePtr();
 
   while (true) {
@@ -1979,6 +2000,12 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
     case Type::ObjCObjectPointer:
       Result.Namespaces.insert(Result.S.Context.getTranslationUnitDecl());
       break;
+
+    // Atomic types are just wrappers; use the associations of the
+    // contained type.
+    case Type::Atomic:
+      T = cast<AtomicType>(T)->getValueType().getTypePtr();
+      continue;
     }
 
     if (Queue.empty()) break;
@@ -2210,12 +2237,14 @@ Sema::SpecialMemberOverloadResult *Sema::LookupSpecialMember(CXXRecordDecl *RD,
       Name = Context.DeclarationNames.getCXXConstructorName(CanTy);
       if (!RD->hasDeclaredCopyConstructor())
         DeclareImplicitCopyConstructor(RD);
-      // TODO: Move constructors
+      if (getLangOptions().CPlusPlus0x && RD->needsImplicitMoveConstructor())
+        DeclareImplicitMoveConstructor(RD);
     } else {
       Name = Context.DeclarationNames.getCXXOperatorName(OO_Equal);
       if (!RD->hasDeclaredCopyAssignment())
         DeclareImplicitCopyAssignment(RD);
-      // TODO: Move assignment
+      if (getLangOptions().CPlusPlus0x && RD->needsImplicitMoveAssignment())
+        DeclareImplicitMoveAssignment(RD);
     }
 
     QualType ArgType = CanTy;
@@ -2358,6 +2387,15 @@ CXXConstructorDecl *Sema::LookupCopyingConstructor(CXXRecordDecl *Class,
   return cast_or_null<CXXConstructorDecl>(Result->getMethod());
 }
 
+/// \brief Look up the moving constructor for the given class.
+CXXConstructorDecl *Sema::LookupMovingConstructor(CXXRecordDecl *Class) {
+  SpecialMemberOverloadResult *Result =
+    LookupSpecialMember(Class, CXXMoveConstructor, false,
+                        false, false, false, false);
+
+  return cast_or_null<CXXConstructorDecl>(Result->getMethod());
+}
+
 /// \brief Look up the constructors for the given class.
 DeclContext::lookup_result Sema::LookupConstructors(CXXRecordDecl *Class) {
   // If the implicit constructors have not yet been declared, do so now.
@@ -2366,6 +2404,8 @@ DeclContext::lookup_result Sema::LookupConstructors(CXXRecordDecl *Class) {
       DeclareImplicitDefaultConstructor(Class);
     if (!Class->hasDeclaredCopyConstructor())
       DeclareImplicitCopyConstructor(Class);
+    if (getLangOptions().CPlusPlus0x && Class->needsImplicitMoveConstructor())
+      DeclareImplicitMoveConstructor(Class);
   }
 
   CanQualType T = Context.getCanonicalType(Context.getTypeDeclType(Class));
@@ -2394,6 +2434,20 @@ CXXMethodDecl *Sema::LookupCopyingAssignment(CXXRecordDecl *Class,
   return Result->getMethod();
 }
 
+/// \brief Look up the moving assignment operator for the given class.
+CXXMethodDecl *Sema::LookupMovingAssignment(CXXRecordDecl *Class,
+                                            bool RValueThis,
+                                            unsigned ThisQuals) {
+  assert(!(ThisQuals & ~(Qualifiers::Const | Qualifiers::Volatile)) &&
+         "non-const, non-volatile qualifiers for copy assignment this");
+  SpecialMemberOverloadResult *Result =
+    LookupSpecialMember(Class, CXXMoveAssignment, false, false, RValueThis,
+                        ThisQuals & Qualifiers::Const,
+                        ThisQuals & Qualifiers::Volatile);
+
+  return Result->getMethod();
+}
+
 /// \brief Look for the destructor of the given class.
 ///
 /// During semantic analysis, this routine should be used in lieu of
@@ -2530,24 +2584,7 @@ class VisibleDeclsRecord {
   /// \brief An entry in the shadow map, which is optimized to store a
   /// single declaration (the common case) but can also store a list
   /// of declarations.
-  class ShadowMapEntry {
-    typedef llvm::SmallVector<NamedDecl *, 4> DeclVector;
-
-    /// \brief Contains either the solitary NamedDecl * or a vector
-    /// of declarations.
-    llvm::PointerUnion<NamedDecl *, DeclVector*> DeclOrVector;
-
-  public:
-    ShadowMapEntry() : DeclOrVector() { }
-
-    void Add(NamedDecl *ND);
-    void Destroy();
-
-    // Iteration.
-    typedef NamedDecl * const *iterator;
-    iterator begin();
-    iterator end();
-  };
+  typedef llvm::TinyPtrVector<NamedDecl*> ShadowMapEntry;
 
 private:
   /// \brief A mapping from declaration names to the declarations that have
@@ -2581,7 +2618,9 @@ class VisibleDeclsRecord {
   NamedDecl *checkHidden(NamedDecl *ND);
 
   /// \brief Add a declaration to the current shadow map.
-  void add(NamedDecl *ND) { ShadowMaps.back()[ND->getDeclName()].Add(ND); }
+  void add(NamedDecl *ND) {
+    ShadowMaps.back()[ND->getDeclName()].push_back(ND);
+  }
 };
 
 /// \brief RAII object that records when we've entered a shadow context.
@@ -2596,66 +2635,12 @@ class ShadowContextRAII {
   }
 
   ~ShadowContextRAII() {
-    for (ShadowMap::iterator E = Visible.ShadowMaps.back().begin(),
-                          EEnd = Visible.ShadowMaps.back().end();
-         E != EEnd;
-         ++E)
-      E->second.Destroy();
-
     Visible.ShadowMaps.pop_back();
   }
 };
 
 } // end anonymous namespace
 
-void VisibleDeclsRecord::ShadowMapEntry::Add(NamedDecl *ND) {
-  if (DeclOrVector.isNull()) {
-    // 0 - > 1 elements: just set the single element information.
-    DeclOrVector = ND;
-    return;
-  }
-
-  if (NamedDecl *PrevND = DeclOrVector.dyn_cast<NamedDecl *>()) {
-    // 1 -> 2 elements: create the vector of results and push in the
-    // existing declaration.
-    DeclVector *Vec = new DeclVector;
-    Vec->push_back(PrevND);
-    DeclOrVector = Vec;
-  }
-
-  // Add the new element to the end of the vector.
-  DeclOrVector.get<DeclVector*>()->push_back(ND);
-}
-
-void VisibleDeclsRecord::ShadowMapEntry::Destroy() {
-  if (DeclVector *Vec = DeclOrVector.dyn_cast<DeclVector *>()) {
-    delete Vec;
-    DeclOrVector = ((NamedDecl *)0);
-  }
-}
-
-VisibleDeclsRecord::ShadowMapEntry::iterator
-VisibleDeclsRecord::ShadowMapEntry::begin() {
-  if (DeclOrVector.isNull())
-    return 0;
-
-  if (DeclOrVector.is<NamedDecl *>())
-    return DeclOrVector.getAddrOf<NamedDecl *>();
-
-  return DeclOrVector.get<DeclVector *>()->begin();
-}
-
-VisibleDeclsRecord::ShadowMapEntry::iterator
-VisibleDeclsRecord::ShadowMapEntry::end() {
-  if (DeclOrVector.isNull())
-    return 0;
-
-  if (DeclOrVector.is<NamedDecl *>())
-    return DeclOrVector.getAddrOf<NamedDecl *>() + 1;
-
-  return DeclOrVector.get<DeclVector *>()->end();
-}
-
 NamedDecl *VisibleDeclsRecord::checkHidden(NamedDecl *ND) {
   // Look through using declarations.
   ND = ND->getUnderlyingDecl();
@@ -2722,7 +2707,7 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
          D != DEnd; ++D) {
       if (NamedDecl *ND = dyn_cast<NamedDecl>(*D)) {
         if (Result.isAcceptableDecl(ND)) {
-          Consumer.FoundDecl(ND, Visited.checkHidden(ND), InBaseClass);
+          Consumer.FoundDecl(ND, Visited.checkHidden(ND), Ctx, InBaseClass);
           Visited.add(ND);
         }
       } else if (ObjCForwardProtocolDecl *ForwardProto
@@ -2733,19 +2718,17 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
              P != PEnd;
              ++P) {
           if (Result.isAcceptableDecl(*P)) {
-            Consumer.FoundDecl(*P, Visited.checkHidden(*P), InBaseClass);
+            Consumer.FoundDecl(*P, Visited.checkHidden(*P), Ctx, InBaseClass);
             Visited.add(*P);
           }
         }
       } else if (ObjCClassDecl *Class = dyn_cast<ObjCClassDecl>(*D)) {
-        for (ObjCClassDecl::iterator I = Class->begin(), IEnd = Class->end();
-             I != IEnd; ++I) {
-          ObjCInterfaceDecl *IFace = I->getInterface();
+          ObjCInterfaceDecl *IFace = Class->getForwardInterfaceDecl();
           if (Result.isAcceptableDecl(IFace)) {
-            Consumer.FoundDecl(IFace, Visited.checkHidden(IFace), InBaseClass);
+            Consumer.FoundDecl(IFace, Visited.checkHidden(IFace), Ctx,
+                               InBaseClass);
             Visited.add(IFace);
           }
-        }
       }
       
       // Visit transparent contexts and inline namespaces inside this context.
@@ -2885,7 +2868,7 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
          D != DEnd; ++D) {
       if (NamedDecl *ND = dyn_cast<NamedDecl>(*D))
         if (Result.isAcceptableDecl(ND)) {
-          Consumer.FoundDecl(ND, Visited.checkHidden(ND), false);
+          Consumer.FoundDecl(ND, Visited.checkHidden(ND), 0, false);
           Visited.add(ND);
         }
     }
@@ -2909,24 +2892,7 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
                                   Result.getNameLoc(), Sema::LookupMemberName);
           if (ObjCInterfaceDecl *IFace = Method->getClassInterface()) {
             LookupVisibleDecls(IFace, IvarResult, /*QualifiedNameLookup=*/false,
-                               /*InBaseClass=*/false, Consumer, Visited);
-
-            // Look for properties from which we can synthesize ivars, if
-            // permitted.
-            if (Result.getSema().getLangOptions().ObjCNonFragileABI2 &&
-                IFace->getImplementation() &&
-                Result.getLookupKind() == Sema::LookupOrdinaryName) {
-              for (ObjCInterfaceDecl::prop_iterator
-                        P = IFace->prop_begin(),
-                     PEnd = IFace->prop_end();
-                   P != PEnd; ++P) {
-                if (Result.getSema().canSynthesizeProvisionalIvar(*P) &&
-                    !IFace->lookupInstanceVariable((*P)->getIdentifier())) {
-                  Consumer.FoundDecl(*P, Visited.checkHidden(*P), false);
-                  Visited.add(*P);
-                }
-              }
-            }
+                               /*InBaseClass=*/false, Consumer, Visited);              
           }
         }
 
@@ -3056,7 +3022,7 @@ static const unsigned MaxTypoDistanceResultSets = 5;
 
 class TypoCorrectionConsumer : public VisibleDeclConsumer {
   /// \brief The name written that is a typo in the source.
-  llvm::StringRef Typo;
+  StringRef Typo;
 
   /// \brief The results found that have the smallest edit distance
   /// found (so far) with the typo name.
@@ -3084,11 +3050,12 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
       delete I->second;
   }
   
-  virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, bool InBaseClass);
-  void FoundName(llvm::StringRef Name);
-  void addKeywordResult(llvm::StringRef Keyword);
-  void addName(llvm::StringRef Name, NamedDecl *ND, unsigned Distance,
-               NestedNameSpecifier *NNS=NULL);
+  virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, DeclContext *Ctx,
+                         bool InBaseClass);
+  void FoundName(StringRef Name);
+  void addKeywordResult(StringRef Keyword);
+  void addName(StringRef Name, NamedDecl *ND, unsigned Distance,
+               NestedNameSpecifier *NNS=NULL, bool isKeyword=false);
   void addCorrection(TypoCorrection Correction);
 
   typedef TypoResultsMap::iterator result_iterator;
@@ -3099,7 +3066,7 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
   unsigned size() const { return BestResults.size(); }
   bool empty() const { return BestResults.empty(); }
 
-  TypoCorrection &operator[](llvm::StringRef Name) {
+  TypoCorrection &operator[](StringRef Name) {
     return (*BestResults.begin()->second)[Name];
   }
 
@@ -3115,7 +3082,7 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
 }
 
 void TypoCorrectionConsumer::FoundDecl(NamedDecl *ND, NamedDecl *Hiding,
-                                       bool InBaseClass) {
+                                       DeclContext *Ctx, bool InBaseClass) {
   // Don't consider hidden names for typo correction.
   if (Hiding)
     return;
@@ -3130,7 +3097,7 @@ void TypoCorrectionConsumer::FoundDecl(NamedDecl *ND, NamedDecl *Hiding,
   FoundName(Name->getName());
 }
 
-void TypoCorrectionConsumer::FoundName(llvm::StringRef Name) {
+void TypoCorrectionConsumer::FoundName(StringRef Name) {
   // Use a simple length-based heuristic to determine the minimum possible
   // edit distance. If the minimum isn't good enough, bail out early.
   unsigned MinED = abs((int)Name.size() - (int)Typo.size());
@@ -3156,7 +3123,7 @@ void TypoCorrectionConsumer::FoundName(llvm::StringRef Name) {
   addName(Name, NULL, ED);
 }
 
-void TypoCorrectionConsumer::addKeywordResult(llvm::StringRef Keyword) {
+void TypoCorrectionConsumer::addKeywordResult(StringRef Keyword) {
   // Compute the edit distance between the typo and this keyword.
   // If this edit distance is not worse than the best edit
   // distance we've seen so far, add it to the list of results.
@@ -3167,19 +3134,21 @@ void TypoCorrectionConsumer::addKeywordResult(llvm::StringRef Keyword) {
     return;
   }
 
-  addName(Keyword, TypoCorrection::KeywordDecl(), ED);
+  addName(Keyword, NULL, ED, NULL, true);
 }
 
-void TypoCorrectionConsumer::addName(llvm::StringRef Name,
+void TypoCorrectionConsumer::addName(StringRef Name,
                                      NamedDecl *ND,
                                      unsigned Distance,
-                                     NestedNameSpecifier *NNS) {
-  addCorrection(TypoCorrection(&SemaRef.Context.Idents.get(Name),
-                               ND, NNS, Distance));
+                                     NestedNameSpecifier *NNS,
+                                     bool isKeyword) {
+  TypoCorrection TC(&SemaRef.Context.Idents.get(Name), ND, NNS, Distance);
+  if (isKeyword) TC.makeKeyword();
+  addCorrection(TC);
 }
 
 void TypoCorrectionConsumer::addCorrection(TypoCorrection Correction) {
-  llvm::StringRef Name = Correction.getCorrectionAsIdentifierInfo()->getName();
+  StringRef Name = Correction.getCorrectionAsIdentifierInfo()->getName();
   TypoResultsMap *& Map = BestResults[Correction.getEditDistance()];
   if (!Map)
     Map = new TypoResultsMap;
@@ -3213,8 +3182,8 @@ class SpecifierInfo {
       : DeclCtx(Ctx), NameSpecifier(NNS), EditDistance(ED) {}
 };
 
-typedef llvm::SmallVector<DeclContext*, 4> DeclContextList;
-typedef llvm::SmallVector<SpecifierInfo, 16> SpecifierInfoList;
+typedef SmallVector<DeclContext*, 4> DeclContextList;
+typedef SmallVector<SpecifierInfo, 16> SpecifierInfoList;
 
 class NamespaceSpecifierSet {
   ASTContext &Context;
@@ -3264,14 +3233,14 @@ DeclContextList NamespaceSpecifierSet::BuildContextChain(DeclContext *Start) {
 }
 
 void NamespaceSpecifierSet::SortNamespaces() {
-  llvm::SmallVector<unsigned, 4> sortedDistances;
+  SmallVector<unsigned, 4> sortedDistances;
   sortedDistances.append(Distances.begin(), Distances.end());
 
   if (sortedDistances.size() > 1)
     std::sort(sortedDistances.begin(), sortedDistances.end());
 
   Specifiers.clear();
-  for (llvm::SmallVector<unsigned, 4>::iterator DI = sortedDistances.begin(),
+  for (SmallVector<unsigned, 4>::iterator DI = sortedDistances.begin(),
                                              DIEnd = sortedDistances.end();
        DI != DIEnd; ++DI) {
     SpecifierInfoList &SpecList = DistanceMap[*DI];
@@ -3648,7 +3617,7 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
                               = Context.Idents.getExternalIdentifierLookup()) {
         llvm::OwningPtr<IdentifierIterator> Iter(External->getIdentifiers());
         do {
-          llvm::StringRef Name = Iter->Next();
+          StringRef Name = Iter->Next();
           if (Name.empty())
             break;
 
@@ -3692,7 +3661,7 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
   if (getLangOptions().CPlusPlus) {
     // Load any externally-known namespaces.
     if (ExternalSource && !LoadedExternalKnownNamespaces) {
-      llvm::SmallVector<NamespaceDecl *, 4> ExternalKnownNamespaces;
+      SmallVector<NamespaceDecl *, 4> ExternalKnownNamespaces;
       LoadedExternalKnownNamespaces = true;
       ExternalSource->ReadKnownNamespaces(ExternalKnownNamespaces);
       for (unsigned I = 0, N = ExternalKnownNamespaces.size(); I != N; ++I)
@@ -3730,6 +3699,7 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
       switch (TmpRes.getResultKind()) {
       case LookupResult::NotFound:
       case LookupResult::NotFoundInCurrentInstantiation:
+      case LookupResult::FoundUnresolvedValue:
         QualifiedResults.insert(Name);
         // We didn't find this name in our scope, or didn't like what we found;
         // ignore it.
@@ -3745,12 +3715,18 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
         // We don't deal with ambiguities.
         return TypoCorrection();
 
+      case LookupResult::FoundOverloaded: {
+        // Store all of the Decls for overloaded symbols
+        for (LookupResult::iterator TRD = TmpRes.begin(),
+                                 TRDEnd = TmpRes.end();
+             TRD != TRDEnd; ++TRD)
+          I->second.addCorrectionDecl(*TRD);
+        ++I;
+        break;
+      }
+
       case LookupResult::Found:
-      case LookupResult::FoundOverloaded:
-      case LookupResult::FoundUnresolvedValue:
         I->second.setCorrectionDecl(TmpRes.getAsSingle<NamedDecl>());
-        // FIXME: This sets the CorrectionDecl to NULL for overloaded functions.
-        // It would be nice to find the right one with overload resolution.
         ++I;
         break;
       }
@@ -3786,14 +3762,23 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
 
           switch (TmpRes.getResultKind()) {
           case LookupResult::Found:
-          case LookupResult::FoundOverloaded:
-          case LookupResult::FoundUnresolvedValue:
             Consumer.addName((*QRI)->getName(), TmpRes.getAsSingle<NamedDecl>(),
                              QualifiedED, NI->NameSpecifier);
             break;
+          case LookupResult::FoundOverloaded: {
+            TypoCorrection corr(&Context.Idents.get((*QRI)->getName()), NULL,
+                                NI->NameSpecifier, QualifiedED);
+            for (LookupResult::iterator TRD = TmpRes.begin(),
+                                     TRDEnd = TmpRes.end();
+                 TRD != TRDEnd; ++TRD)
+              corr.addCorrectionDecl(*TRD);
+            Consumer.addCorrection(corr);
+            break;
+          }
           case LookupResult::NotFound:
           case LookupResult::NotFoundInCurrentInstantiation:
           case LookupResult::Ambiguous:
+          case LookupResult::FoundUnresolvedValue:
             break;
           }
         }
@@ -3870,6 +3855,18 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
   return TypoCorrection();
 }
 
+void TypoCorrection::addCorrectionDecl(NamedDecl *CDecl) {
+  if (!CDecl) return;
+
+  if (isKeyword())
+    CorrectionDecls.clear();
+
+  CorrectionDecls.push_back(CDecl);
+
+  if (!CorrectionName)
+    CorrectionName = CDecl->getDeclName();
+}
+
 std::string TypoCorrection::getAsString(const LangOptions &LO) const {
   if (CorrectionNameSpec) {
     std::string tmpBuffer;
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp
index d826ea8d84a4..751f553945cd 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp
@@ -16,6 +16,7 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprObjC.h"
+#include "clang/AST/ExprCXX.h"
 #include "llvm/ADT/DenseSet.h"
 
 using namespace clang;
@@ -24,6 +25,37 @@ using namespace clang;
 // Grammar actions.
 //===----------------------------------------------------------------------===//
 
+/// getImpliedARCOwnership - Given a set of property attributes and a
+/// type, infer an expected lifetime.  The type's ownership qualification
+/// is not considered.
+///
+/// Returns OCL_None if the attributes as stated do not imply an ownership.
+/// Never returns OCL_Autoreleasing.
+static Qualifiers::ObjCLifetime getImpliedARCOwnership(
+                               ObjCPropertyDecl::PropertyAttributeKind attrs,
+                                                QualType type) {
+  // retain, strong, copy, weak, and unsafe_unretained are only legal
+  // on properties of retainable pointer type.
+  if (attrs & (ObjCPropertyDecl::OBJC_PR_retain |
+               ObjCPropertyDecl::OBJC_PR_strong |
+               ObjCPropertyDecl::OBJC_PR_copy)) {
+    return type->getObjCARCImplicitLifetime();
+  } else if (attrs & ObjCPropertyDecl::OBJC_PR_weak) {
+    return Qualifiers::OCL_Weak;
+  } else if (attrs & ObjCPropertyDecl::OBJC_PR_unsafe_unretained) {
+    return Qualifiers::OCL_ExplicitNone;
+  }
+
+  // assign can appear on other types, so we have to check the
+  // property type.
+  if (attrs & ObjCPropertyDecl::OBJC_PR_assign &&
+      type->isObjCRetainableType()) {
+    return Qualifiers::OCL_ExplicitNone;
+  }
+
+  return Qualifiers::OCL_None;
+}
+
 /// Check the internal consistency of a property declaration.
 static void checkARCPropertyDecl(Sema &S, ObjCPropertyDecl *property) {
   if (property->isInvalidDecl()) return;
@@ -36,26 +68,23 @@ static void checkARCPropertyDecl(Sema &S, ObjCPropertyDecl *property) {
   // Nothing to do if we don't have a lifetime.
   if (propertyLifetime == Qualifiers::OCL_None) return;
 
-  Qualifiers::ObjCLifetime expectedLifetime;
-  unsigned selector;
-
-  // Strong properties should have either strong or no lifetime.
-  if (propertyKind & (ObjCPropertyDecl::OBJC_PR_retain |
-                      ObjCPropertyDecl::OBJC_PR_strong |
-                      ObjCPropertyDecl::OBJC_PR_copy)) {
-    expectedLifetime = Qualifiers::OCL_Strong;
-    selector = 0;
-  } else if (propertyKind & ObjCPropertyDecl::OBJC_PR_weak) {
-    expectedLifetime = Qualifiers::OCL_Weak;
-    selector = 1;
-  } else if (propertyKind & (ObjCPropertyDecl::OBJC_PR_assign |
-                             ObjCPropertyDecl::OBJC_PR_unsafe_unretained) &&
-             property->getType()->isObjCRetainableType()) {
-    expectedLifetime = Qualifiers::OCL_ExplicitNone;
-    selector = 2;
-  } else {
+  Qualifiers::ObjCLifetime expectedLifetime
+    = getImpliedARCOwnership(propertyKind, property->getType());
+  if (!expectedLifetime) {
     // We have a lifetime qualifier but no dominating property
-    // attribute.  That's okay.
+    // attribute.  That's okay, but restore reasonable invariants by
+    // setting the property attribute according to the lifetime
+    // qualifier.
+    ObjCPropertyDecl::PropertyAttributeKind attr;
+    if (propertyLifetime == Qualifiers::OCL_Strong) {
+      attr = ObjCPropertyDecl::OBJC_PR_strong;
+    } else if (propertyLifetime == Qualifiers::OCL_Weak) {
+      attr = ObjCPropertyDecl::OBJC_PR_weak;
+    } else {
+      assert(propertyLifetime == Qualifiers::OCL_ExplicitNone);
+      attr = ObjCPropertyDecl::OBJC_PR_unsafe_unretained;
+    }
+    property->setPropertyAttributes(attr);
     return;
   }
 
@@ -65,7 +94,7 @@ static void checkARCPropertyDecl(Sema &S, ObjCPropertyDecl *property) {
   S.Diag(property->getLocation(),
          diag::err_arc_inconsistent_property_ownership)
     << property->getDeclName()
-    << selector
+    << expectedLifetime
     << propertyLifetime;
 }
 
@@ -74,14 +103,13 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
                           ObjCDeclSpec &ODS,
                           Selector GetterSel,
                           Selector SetterSel,
-                          Decl *ClassCategory,
                           bool *isOverridingProperty,
                           tok::ObjCKeywordKind MethodImplKind,
                           DeclContext *lexicalDC) {
   unsigned Attributes = ODS.getPropertyAttributes();
   TypeSourceInfo *TSI = GetTypeForDeclarator(FD.D, S);
   QualType T = TSI->getType();
-  if ((getLangOptions().getGCMode() != LangOptions::NonGC && 
+  if ((getLangOptions().getGC() != LangOptions::NonGC && 
        T.isObjCGCWeak()) ||
       (getLangOptions().ObjCAutoRefCount &&
        T.getObjCLifetime() == Qualifiers::OCL_Weak))
@@ -101,12 +129,11 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
                     !(Attributes & ObjCDeclSpec::DQ_PR_weak)));
 
   // Proceed with constructing the ObjCPropertDecls.
-  ObjCContainerDecl *ClassDecl =
-    cast<ObjCContainerDecl>(ClassCategory);
+  ObjCContainerDecl *ClassDecl = cast<ObjCContainerDecl>(CurContext);
 
   if (ObjCCategoryDecl *CDecl = dyn_cast<ObjCCategoryDecl>(ClassDecl))
     if (CDecl->IsClassExtension()) {
-      Decl *Res = HandlePropertyInClassExtension(S, CDecl, AtLoc,
+      Decl *Res = HandlePropertyInClassExtension(S, AtLoc,
                                            FD, GetterSel, SetterSel,
                                            isAssign, isReadWrite,
                                            Attributes,
@@ -137,7 +164,7 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
 }
 
 Decl *
-Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
+Sema::HandlePropertyInClassExtension(Scope *S,
                                      SourceLocation AtLoc, FieldDeclarator &FD,
                                      Selector GetterSel, Selector SetterSel,
                                      const bool isAssign,
@@ -146,9 +173,9 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
                                      bool *isOverridingProperty,
                                      TypeSourceInfo *T,
                                      tok::ObjCKeywordKind MethodImplKind) {
-
+  ObjCCategoryDecl *CDecl = cast<ObjCCategoryDecl>(CurContext);
   // Diagnose if this property is already in continuation class.
-  DeclContext *DC = cast<DeclContext>(CDecl);
+  DeclContext *DC = CurContext;
   IdentifierInfo *PropertyId = FD.D.getIdentifier();
   ObjCInterfaceDecl *CCPrimary = CDecl->getClassInterface();
   
@@ -209,7 +236,13 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
                         /* lexicalDC = */ CDecl);
     return PDecl;
   }
-
+  if (PIDecl->getType().getCanonicalType() 
+      != PDecl->getType().getCanonicalType()) {
+    Diag(AtLoc, 
+         diag::warn_type_mismatch_continuation_class) << PDecl->getType();
+    Diag(PIDecl->getLocation(), diag::note_property_declare);
+  }
+    
   // The property 'PIDecl's readonly attribute will be over-ridden
   // with continuation class's readwrite property attribute!
   unsigned PIkind = PIDecl->getPropertyAttributesAsWritten();
@@ -235,12 +268,15 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
       ProtocolPropertyODS.
       setPropertyAttributes((ObjCDeclSpec::ObjCPropertyAttributeKind)
                             PIkind);
-
+      // Must re-establish the context from class extension to primary
+      // class context.
+      ContextRAII SavedContext(*this, CCPrimary);
+      
       Decl *ProtocolPtrTy =
         ActOnProperty(S, AtLoc, FD, ProtocolPropertyODS,
                       PIDecl->getGetterName(),
                       PIDecl->getSetterName(),
-                      CCPrimary, isOverridingProperty,
+                      isOverridingProperty,
                       MethodImplKind,
                       /* lexicalDC = */ CDecl);
       PIDecl = cast<ObjCPropertyDecl>(ProtocolPtrTy);
@@ -290,7 +326,7 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
 
   // Issue a warning if property is 'assign' as default and its object, which is
   // gc'able conforms to NSCopying protocol
-  if (getLangOptions().getGCMode() != LangOptions::NonGC &&
+  if (getLangOptions().getGC() != LangOptions::NonGC &&
       isAssign && !(Attributes & ObjCDeclSpec::DQ_PR_assign))
     if (const ObjCObjectPointerType *ObjPtrTy =
           T->getAs<ObjCObjectPointerType>()) {
@@ -392,9 +428,10 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
   if (isAssign)
     PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_assign);
 
+  // In the semantic attributes, one of nonatomic or atomic is always set.
   if (Attributes & ObjCDeclSpec::DQ_PR_nonatomic)
     PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_nonatomic);
-  else if (Attributes & ObjCDeclSpec::DQ_PR_atomic)
+  else
     PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_atomic);
 
   // 'unsafe_unretained' is alias for 'assign'.
@@ -416,82 +453,49 @@ static void checkARCPropertyImpl(Sema &S, SourceLocation propertyImplLoc,
                                  ObjCIvarDecl *ivar) {
   if (property->isInvalidDecl() || ivar->isInvalidDecl()) return;
 
-  QualType propertyType = property->getType();
-  Qualifiers::ObjCLifetime propertyLifetime = propertyType.getObjCLifetime();
-  ObjCPropertyDecl::PropertyAttributeKind propertyKind
-    = property->getPropertyAttributes();
-
   QualType ivarType = ivar->getType();
   Qualifiers::ObjCLifetime ivarLifetime = ivarType.getObjCLifetime();
-          
-  // Case 1: strong properties.
-  if (propertyLifetime == Qualifiers::OCL_Strong ||
-      (propertyKind & (ObjCPropertyDecl::OBJC_PR_retain |
-                       ObjCPropertyDecl::OBJC_PR_strong |
-                       ObjCPropertyDecl::OBJC_PR_copy))) {
-    switch (ivarLifetime) {
-    case Qualifiers::OCL_Strong:
-      // Okay.
-      return;
 
-    case Qualifiers::OCL_None:
-    case Qualifiers::OCL_Autoreleasing:
-      // These aren't valid lifetimes for object ivars;  don't diagnose twice.
-      return;
+  // The lifetime implied by the property's attributes.
+  Qualifiers::ObjCLifetime propertyLifetime =
+    getImpliedARCOwnership(property->getPropertyAttributes(),
+                           property->getType());
 
-    case Qualifiers::OCL_ExplicitNone:
-    case Qualifiers::OCL_Weak:
-      S.Diag(propertyImplLoc, diag::err_arc_strong_property_ownership)
-        << property->getDeclName()
-        << ivar->getDeclName()
-        << ivarLifetime;
-      break;
-    }
+  // We're fine if they match.
+  if (propertyLifetime == ivarLifetime) return;
 
-  // Case 2: weak properties.
-  } else if (propertyLifetime == Qualifiers::OCL_Weak ||
-             (propertyKind & ObjCPropertyDecl::OBJC_PR_weak)) {
-    switch (ivarLifetime) {
-    case Qualifiers::OCL_Weak:
-      // Okay.
-      return;
+  // These aren't valid lifetimes for object ivars;  don't diagnose twice.
+  if (ivarLifetime == Qualifiers::OCL_None ||
+      ivarLifetime == Qualifiers::OCL_Autoreleasing)
+    return;
 
-    case Qualifiers::OCL_None:
-    case Qualifiers::OCL_Autoreleasing:
-      // These aren't valid lifetimes for object ivars;  don't diagnose twice.
-      return;
+  switch (propertyLifetime) {
+  case Qualifiers::OCL_Strong:
+    S.Diag(propertyImplLoc, diag::err_arc_strong_property_ownership)
+      << property->getDeclName()
+      << ivar->getDeclName()
+      << ivarLifetime;
+    break;
 
-    case Qualifiers::OCL_ExplicitNone:
-    case Qualifiers::OCL_Strong:
-      S.Diag(propertyImplLoc, diag::error_weak_property)
-        << property->getDeclName()
-        << ivar->getDeclName();
-      break;
-    }
+  case Qualifiers::OCL_Weak:
+    S.Diag(propertyImplLoc, diag::error_weak_property)
+      << property->getDeclName()
+      << ivar->getDeclName();
+    break;
 
-  // Case 3: assign properties.
-  } else if ((propertyKind & ObjCPropertyDecl::OBJC_PR_assign) &&
-             propertyType->isObjCRetainableType()) {
-    switch (ivarLifetime) {
-    case Qualifiers::OCL_ExplicitNone:
-      // Okay.
-      return;
+  case Qualifiers::OCL_ExplicitNone:
+    S.Diag(propertyImplLoc, diag::err_arc_assign_property_ownership)
+      << property->getDeclName()
+      << ivar->getDeclName()
+      << ((property->getPropertyAttributesAsWritten() 
+           & ObjCPropertyDecl::OBJC_PR_assign) != 0);
+    break;
 
-    case Qualifiers::OCL_None:
-    case Qualifiers::OCL_Autoreleasing:
-      // These aren't valid lifetimes for object ivars;  don't diagnose twice.
-      return;
+  case Qualifiers::OCL_Autoreleasing:
+    llvm_unreachable("properties cannot be autoreleasing");
 
-    case Qualifiers::OCL_Weak:
-    case Qualifiers::OCL_Strong:
-      S.Diag(propertyImplLoc, diag::err_arc_assign_property_ownership)
-        << property->getDeclName()
-        << ivar->getDeclName();
-      break;
-    }
-
-  // Any other property should be ignored.
-  } else {
+  case Qualifiers::OCL_None:
+    // Any other property should be ignored.
     return;
   }
 
@@ -507,12 +511,11 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
                                   SourceLocation AtLoc,
                                   SourceLocation PropertyLoc,
                                   bool Synthesize,
-                                  Decl *ClassCatImpDecl,
                                   IdentifierInfo *PropertyId,
                                   IdentifierInfo *PropertyIvar,
                                   SourceLocation PropertyIvarLoc) {
   ObjCContainerDecl *ClassImpDecl =
-    cast_or_null<ObjCContainerDecl>(ClassCatImpDecl);
+    dyn_cast<ObjCContainerDecl>(CurContext);
   // Make sure we have a context for the property implementation declaration.
   if (!ClassImpDecl) {
     Diag(AtLoc, diag::error_missing_property_context);
@@ -586,61 +589,65 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
   ObjCIvarDecl *Ivar = 0;
   // Check that we have a valid, previously declared ivar for @synthesize
   if (Synthesize) {
-    if (getLangOptions().ObjCAutoRefCount &&
-        !property->hasWrittenStorageAttribute() &&
-        property->getType()->isObjCRetainableType()) {
-      Diag(PropertyLoc, diag::err_arc_objc_property_default_assign_on_object);
-      Diag(property->getLocation(), diag::note_property_declare);
-    }
-
     // @synthesize
     if (!PropertyIvar)
       PropertyIvar = PropertyId;
     ObjCPropertyDecl::PropertyAttributeKind kind 
       = property->getPropertyAttributes();
-    QualType PropType = Context.getCanonicalType(property->getType());
-    QualType PropertyIvarType = PropType;
-    if (PropType->isReferenceType())
-      PropertyIvarType = cast<ReferenceType>(PropType)->getPointeeType();
+    QualType PropType = property->getType();
+
+    QualType PropertyIvarType = PropType.getNonReferenceType();
+
+    // Add GC __weak to the ivar type if the property is weak.
+    if ((kind & ObjCPropertyDecl::OBJC_PR_weak) && 
+        getLangOptions().getGC() != LangOptions::NonGC) {
+      assert(!getLangOptions().ObjCAutoRefCount);
+      if (PropertyIvarType.isObjCGCStrong()) {
+        Diag(PropertyLoc, diag::err_gc_weak_property_strong_type);
+        Diag(property->getLocation(), diag::note_property_declare);
+      } else {
+        PropertyIvarType =
+          Context.getObjCGCQualType(PropertyIvarType, Qualifiers::Weak);
+      }
+    }
+
     // Check that this is a previously declared 'ivar' in 'IDecl' interface
     ObjCInterfaceDecl *ClassDeclared;
     Ivar = IDecl->lookupInstanceVariable(PropertyIvar, ClassDeclared);
     if (!Ivar) {
-      // In ARC, give the ivar a lifetime qualifier based on its
+      // In ARC, give the ivar a lifetime qualifier based on the
       // property attributes.
       if (getLangOptions().ObjCAutoRefCount &&
-          !PropertyIvarType.getObjCLifetime()) {
+          !PropertyIvarType.getObjCLifetime() &&
+          PropertyIvarType->isObjCRetainableType()) {
 
-        // retain/copy have retaining lifetime.
-        if (kind & (ObjCPropertyDecl::OBJC_PR_retain |
-                    ObjCPropertyDecl::OBJC_PR_strong |
-                    ObjCPropertyDecl::OBJC_PR_copy)) {
-          Qualifiers qs;
-          qs.addObjCLifetime(Qualifiers::OCL_Strong);
-          PropertyIvarType = Context.getQualifiedType(PropertyIvarType, qs);
-        }
-        else if (kind & ObjCPropertyDecl::OBJC_PR_weak) {
-          if (!getLangOptions().ObjCRuntimeHasWeak) {
+        // It's an error if we have to do this and the user didn't
+        // explicitly write an ownership attribute on the property.
+        if (!property->hasWrittenStorageAttribute() &&
+            !(kind & ObjCPropertyDecl::OBJC_PR_strong)) {
+          Diag(PropertyLoc,
+               diag::err_arc_objc_property_default_assign_on_object);
+          Diag(property->getLocation(), diag::note_property_declare);
+        } else {
+          Qualifiers::ObjCLifetime lifetime =
+            getImpliedARCOwnership(kind, PropertyIvarType);
+          assert(lifetime && "no lifetime for property?");
+
+          if (lifetime == Qualifiers::OCL_Weak &&
+              !getLangOptions().ObjCRuntimeHasWeak) {
             Diag(PropertyLoc, diag::err_arc_weak_no_runtime);
             Diag(property->getLocation(), diag::note_property_declare);
           }
+
           Qualifiers qs;
-          qs.addObjCLifetime(Qualifiers::OCL_Weak);
+          qs.addObjCLifetime(lifetime);
           PropertyIvarType = Context.getQualifiedType(PropertyIvarType, qs);   
         }
-        else if (kind & ObjCPropertyDecl::OBJC_PR_assign &&
-                 PropertyIvarType->isObjCRetainableType()) {
-            // assume that an 'assign' property synthesizes __unsafe_unretained
-            // ivar
-            Qualifiers qs;
-            qs.addObjCLifetime(Qualifiers::OCL_ExplicitNone);
-            PropertyIvarType = Context.getQualifiedType(PropertyIvarType, qs);  
-        }
       }
 
       if (kind & ObjCPropertyDecl::OBJC_PR_weak &&
           !getLangOptions().ObjCAutoRefCount &&
-          getLangOptions().getGCMode() == LangOptions::NonGC) {
+          getLangOptions().getGC() == LangOptions::NonGC) {
         Diag(PropertyLoc, diag::error_synthesize_weak_non_arc_or_gc);
         Diag(property->getLocation(), diag::note_property_declare);
       }
@@ -670,7 +677,7 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
     QualType IvarType = Context.getCanonicalType(Ivar->getType());
 
     // Check that type of property and its ivar are type compatible.
-    if (PropertyIvarType != IvarType) {
+    if (Context.getCanonicalType(PropertyIvarType) != IvarType) {
       bool compat = false;
       if (isa<ObjCObjectPointerType>(PropertyIvarType) 
           && isa<ObjCObjectPointerType>(IvarType))
@@ -709,15 +716,16 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       }
       // __weak is explicit. So it works on Canonical type.
       if ((PropType.isObjCGCWeak() && !IvarType.isObjCGCWeak() &&
-           getLangOptions().getGCMode() != LangOptions::NonGC)) {
+           getLangOptions().getGC() != LangOptions::NonGC)) {
         Diag(PropertyLoc, diag::error_weak_property)
         << property->getDeclName() << Ivar->getDeclName();
+        Diag(Ivar->getLocation(), diag::note_ivar_decl);
         // Fall thru - see previous comment
       }
       // Fall thru - see previous comment
       if ((property->getType()->isObjCObjectPointerType() ||
            PropType.isObjCGCStrong()) && IvarType.isObjCGCWeak() &&
-          getLangOptions().getGCMode() != LangOptions::NonGC) {
+          getLangOptions().getGC() != LangOptions::NonGC) {
         Diag(PropertyLoc, diag::error_strong_property)
         << property->getDeclName() << Ivar->getDeclName();
         // Fall thru - see previous comment
@@ -793,6 +801,17 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
                                             VK_LValue, SourceLocation());
       ExprResult Res = BuildBinOp(S, lhs->getLocEnd(), 
                                   BO_Assign, lhs, rhs);
+      if (property->getPropertyAttributes() & 
+          ObjCPropertyDecl::OBJC_PR_atomic) {
+        Expr *callExpr = Res.takeAs<Expr>();
+        if (const CXXOperatorCallExpr *CXXCE = 
+              dyn_cast_or_null<CXXOperatorCallExpr>(callExpr))
+          if (const FunctionDecl *FuncDecl = CXXCE->getDirectCallee())
+            if (!FuncDecl->isTrivial())
+              Diag(PropertyLoc, 
+                   diag::warn_atomic_property_nontrivial_assign_op) 
+                    << property->getType();
+      }
       PIDecl->setSetterCXXAssignment(Res.takeAs<Expr>());
     }
   }
@@ -880,7 +899,7 @@ Sema::DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
       != (SAttr & ObjCPropertyDecl::OBJC_PR_copy))
     Diag(Property->getLocation(), diag::warn_property_attribute)
       << Property->getDeclName() << "copy" << inheritedName;
-  else {
+  else if (!(SAttr & ObjCPropertyDecl::OBJC_PR_readonly)){
     unsigned CAttrRetain = 
       (CAttr & 
        (ObjCPropertyDecl::OBJC_PR_retain | ObjCPropertyDecl::OBJC_PR_strong));
@@ -917,9 +936,11 @@ Sema::DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
     QualType ConvertedType;
     if (!isObjCPointerConversion(RHSType, LHSType, 
                                  ConvertedType, IncompatibleObjC) ||
-        IncompatibleObjC)
+        IncompatibleObjC) {
         Diag(Property->getLocation(), diag::warn_property_types_are_incompatible)
         << Property->getType() << SuperProperty->getType() << inheritedName;
+      Diag(SuperProperty->getLocation(), diag::note_property_declare);
+    }
   }
 }
 
@@ -1146,7 +1167,8 @@ void Sema::CollectImmediateProperties(ObjCContainerDecl *CDecl,
       ObjCPropertyDecl *PropertyFromSuper = SuperPropMap[Prop->getIdentifier()];
       // Exclude property for protocols which conform to class's super-class, 
       // as super-class has to implement the property.
-      if (!PropertyFromSuper || PropertyFromSuper != Prop) {
+      if (!PropertyFromSuper || 
+          PropertyFromSuper->getIdentifier() != Prop->getIdentifier()) {
         ObjCPropertyDecl *&PropEntry = PropMap[Prop->getIdentifier()];
         if (!PropEntry)
           PropEntry = Prop;
@@ -1241,10 +1263,20 @@ ObjCPropertyDecl *Sema::LookupPropertyDecl(const ObjCContainerDecl *CDecl,
   return 0;
 }
 
+static IdentifierInfo * getDefaultSynthIvarName(ObjCPropertyDecl *Prop,
+                                                ASTContext &Ctx) {
+  llvm::SmallString<128> ivarName;
+  {
+    llvm::raw_svector_ostream os(ivarName);
+    os << '_' << Prop->getIdentifier()->getName();
+  }
+  return &Ctx.Idents.get(ivarName.str());
+}
+
 /// DefaultSynthesizeProperties - This routine default synthesizes all
 /// properties which must be synthesized in class's @implementation.
-void Sema::DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
-                                        ObjCInterfaceDecl *IDecl) {
+void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl* IMPDecl,
+                                       ObjCInterfaceDecl *IDecl) {
   
   llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*> PropMap;
   CollectClassPropertyImplementations(IDecl, PropMap);
@@ -1280,12 +1312,23 @@ void Sema::DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
     // Saying that they are located at the @implementation isn't really going
     // to help users.
     ActOnPropertyImplDecl(S, SourceLocation(), SourceLocation(),
-                          true,IMPDecl,
-                          Prop->getIdentifier(), Prop->getIdentifier(),
+                          true,
+                          /* property = */ Prop->getIdentifier(),
+                          /* ivar = */ getDefaultSynthIvarName(Prop, Context),
                           SourceLocation());
   }
 }
 
+void Sema::DefaultSynthesizeProperties(Scope *S, Decl *D) {
+  if (!LangOpts.ObjCDefaultSynthProperties || !LangOpts.ObjCNonFragileABI2)
+    return;
+  ObjCImplementationDecl *IC=dyn_cast_or_null<ObjCImplementationDecl>(D);
+  if (!IC)
+    return;
+  if (ObjCInterfaceDecl* IDecl = IC->getClassInterface())
+    DefaultSynthesizeProperties(S, IC, IDecl);
+}
+
 void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
                                       ObjCContainerDecl *CDecl,
                                       const llvm::DenseSet<Selector>& InsMap) {
@@ -1313,23 +1356,23 @@ void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
         PropImplMap.count(Prop) || Prop->hasAttr<UnavailableAttr>())
       continue;
     if (!InsMap.count(Prop->getGetterName())) {
-      Diag(Prop->getLocation(),
+      Diag(IMPDecl->getLocation(),
            isa<ObjCCategoryDecl>(CDecl) ?
             diag::warn_setter_getter_impl_required_in_category :
             diag::warn_setter_getter_impl_required)
       << Prop->getDeclName() << Prop->getGetterName();
-      Diag(IMPDecl->getLocation(),
-           diag::note_property_impl_required);
+      Diag(Prop->getLocation(),
+           diag::note_property_declare);
     }
 
     if (!Prop->isReadOnly() && !InsMap.count(Prop->getSetterName())) {
-      Diag(Prop->getLocation(),
+      Diag(IMPDecl->getLocation(),
            isa<ObjCCategoryDecl>(CDecl) ?
            diag::warn_setter_getter_impl_required_in_category :
            diag::warn_setter_getter_impl_required)
       << Prop->getDeclName() << Prop->getSetterName();
-      Diag(IMPDecl->getLocation(),
-           diag::note_property_impl_required);
+      Diag(Prop->getLocation(),
+           diag::note_property_declare);
     }
   }
 }
@@ -1338,7 +1381,7 @@ void
 Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
                                        ObjCContainerDecl* IDecl) {
   // Rules apply in non-GC mode only
-  if (getLangOptions().getGCMode() != LangOptions::NonGC)
+  if (getLangOptions().getGC() != LangOptions::NonGC)
     return;
   for (ObjCContainerDecl::prop_iterator I = IDecl->prop_begin(),
        E = IDecl->prop_end();
@@ -1349,10 +1392,10 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
     bool LookedUpGetterSetter = false;
 
     unsigned Attributes = Property->getPropertyAttributes();
-    unsigned AttributesAsWrittern = Property->getPropertyAttributesAsWritten();
+    unsigned AttributesAsWritten = Property->getPropertyAttributesAsWritten();
 
-    if (!(AttributesAsWrittern & ObjCPropertyDecl::OBJC_PR_atomic) &&
-        !(AttributesAsWrittern & ObjCPropertyDecl::OBJC_PR_nonatomic)) {
+    if (!(AttributesAsWritten & ObjCPropertyDecl::OBJC_PR_atomic) &&
+        !(AttributesAsWritten & ObjCPropertyDecl::OBJC_PR_nonatomic)) {
       GetterMethod = IMPDecl->getInstanceMethod(Property->getGetterName());
       SetterMethod = IMPDecl->getInstanceMethod(Property->getSetterName());
       LookedUpGetterSetter = true;
@@ -1388,7 +1431,9 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
           (GetterMethod ? GetterMethod->getLocation()
                         : SetterMethod->getLocation());
         Diag(MethodLoc, diag::warn_atomic_property_rule)
-          << Property->getIdentifier();
+          << Property->getIdentifier() << (GetterMethod != 0)
+          << (SetterMethod != 0);
+        Diag(MethodLoc, diag::note_atomic_property_fixup_suggest);
         Diag(Property->getLocation(), diag::note_property_declare);
       }
     }
@@ -1396,7 +1441,7 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
 }
 
 void Sema::DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D) {
-  if (getLangOptions().getGCMode() == LangOptions::GCOnly)
+  if (getLangOptions().getGC() == LangOptions::GCOnly)
     return;
 
   for (ObjCImplementationDecl::propimpl_iterator
@@ -1417,7 +1462,7 @@ void Sema::DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D
         if (getLangOptions().ObjCAutoRefCount)
           Diag(PID->getLocation(), diag::err_ownin_getter_rule);
         else
-          Diag(PID->getLocation(), diag::warn_ownin_getter_rule);
+          Diag(PID->getLocation(), diag::warn_owning_getter_rule);
         Diag(PD->getLocation(), diag::note_property_declare);
       }
     }
@@ -1464,7 +1509,8 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
           Context.VoidTy)
       Diag(SetterMethod->getLocation(), diag::err_setter_type_void);
     if (SetterMethod->param_size() != 1 ||
-        ((*SetterMethod->param_begin())->getType() != property->getType())) {
+        !Context.hasSameUnqualifiedType(
+          (*SetterMethod->param_begin())->getType(), property->getType())) {
       Diag(property->getLocation(),
            diag::warn_accessor_property_type_mismatch)
         << property->getDeclName()
@@ -1489,8 +1535,9 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
 
     GetterMethod = ObjCMethodDecl::Create(Context, Loc, Loc,
                              property->getGetterName(),
-                             property->getType(), 0, CD, true, false, true, 
-                             false,
+                             property->getType(), 0, CD, /*isInstance=*/true,
+                             /*isVariadic=*/false, /*isSynthesized=*/true,
+                             /*isImplicitlyDeclared=*/true, /*isDefined=*/false,
                              (property->getPropertyImplementation() ==
                               ObjCPropertyDecl::Optional) ?
                              ObjCMethodDecl::Optional :
@@ -1526,7 +1573,10 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
       SetterMethod =
         ObjCMethodDecl::Create(Context, Loc, Loc,
                                property->getSetterName(), Context.VoidTy, 0,
-                               CD, true, false, true, false,
+                               CD, /*isInstance=*/true, /*isVariadic=*/false,
+                               /*isSynthesized=*/true,
+                               /*isImplicitlyDeclared=*/true,
+                               /*isDefined=*/false,
                                (property->getPropertyImplementation() ==
                                 ObjCPropertyDecl::Optional) ?
                                 ObjCMethodDecl::Optional :
@@ -1542,7 +1592,8 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
                                                   SC_None,
                                                   SC_None,
                                                   0);
-      SetterMethod->setMethodParams(Context, &Argument, 1, 1);
+      SetterMethod->setMethodParams(Context, Argument,
+                                    ArrayRef<SourceLocation>());
 
       AddPropertyAttrs(*this, SetterMethod, property);
 
@@ -1686,7 +1737,7 @@ void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
            (Attributes & ObjCDeclSpec::DQ_PR_weak)) {
       Diag(Loc, diag::err_objc_property_attr_mutually_exclusive)
         << "retain" << "weak";
-      Attributes &= ~ObjCDeclSpec::DQ_PR_weak;
+      Attributes &= ~ObjCDeclSpec::DQ_PR_retain;
   }
   else if ((Attributes & ObjCDeclSpec::DQ_PR_strong) &&
            (Attributes & ObjCDeclSpec::DQ_PR_weak)) {
@@ -1695,6 +1746,13 @@ void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
       Attributes &= ~ObjCDeclSpec::DQ_PR_weak;
   }
 
+  if ((Attributes & ObjCDeclSpec::DQ_PR_atomic) &&
+      (Attributes & ObjCDeclSpec::DQ_PR_nonatomic)) {
+      Diag(Loc, diag::err_objc_property_attr_mutually_exclusive)
+        << "atomic" << "nonatomic";
+      Attributes &= ~ObjCDeclSpec::DQ_PR_atomic;
+  }
+
   // Warn if user supplied no assignment attribute, property is
   // readwrite, and this is an object type.
   if (!(Attributes & (ObjCDeclSpec::DQ_PR_assign | ObjCDeclSpec::DQ_PR_copy |
@@ -1703,13 +1761,19 @@ void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
                       ObjCDeclSpec::DQ_PR_weak)) &&
       !(Attributes & ObjCDeclSpec::DQ_PR_readonly) &&
       PropertyTy->isObjCObjectPointerType()) {
-    // Skip this warning in gc-only mode.
-    if (getLangOptions().getGCMode() != LangOptions::GCOnly)
-      Diag(Loc, diag::warn_objc_property_no_assignment_attribute);
+      if (getLangOptions().ObjCAutoRefCount)
+        // With arc,  @property definitions should default to (strong) when 
+        // not specified
+        PropertyDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_strong);
+      else {
+          // Skip this warning in gc-only mode.
+          if (getLangOptions().getGC() != LangOptions::GCOnly)
+            Diag(Loc, diag::warn_objc_property_no_assignment_attribute);
 
-    // If non-gc code warn that this is likely inappropriate.
-    if (getLangOptions().getGCMode() == LangOptions::NonGC)
-      Diag(Loc, diag::warn_objc_property_default_assign_on_object);
+          // If non-gc code warn that this is likely inappropriate.
+          if (getLangOptions().getGC() == LangOptions::NonGC)
+            Diag(Loc, diag::warn_objc_property_default_assign_on_object);
+      }
 
     // FIXME: Implement warning dependent on NSCopying being
     // implemented. See also:
@@ -1719,7 +1783,13 @@ void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
 
   if (!(Attributes & ObjCDeclSpec::DQ_PR_copy)
       &&!(Attributes & ObjCDeclSpec::DQ_PR_readonly)
-      && getLangOptions().getGCMode() == LangOptions::GCOnly
+      && getLangOptions().getGC() == LangOptions::GCOnly
       && PropertyTy->isBlockPointerType())
     Diag(Loc, diag::warn_objc_property_copy_missing_on_block);
+  else if (getLangOptions().ObjCAutoRefCount &&
+           (Attributes & ObjCDeclSpec::DQ_PR_retain) &&
+           !(Attributes & ObjCDeclSpec::DQ_PR_readonly) &&
+           !(Attributes & ObjCDeclSpec::DQ_PR_strong) &&
+           PropertyTy->isBlockPointerType())
+      Diag(Loc, diag::warn_objc_property_retain_of_block);
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp
index 437b2b5c6d43..b0dd5e280ebd 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp
@@ -37,11 +37,14 @@ using namespace sema;
 /// A convenience routine for creating a decayed reference to a
 /// function.
 static ExprResult
-CreateFunctionRefExpr(Sema &S, FunctionDecl *Fn,
+CreateFunctionRefExpr(Sema &S, FunctionDecl *Fn, bool HadMultipleCandidates,
                       SourceLocation Loc = SourceLocation(), 
                       const DeclarationNameLoc &LocInfo = DeclarationNameLoc()){
-  ExprResult E = S.Owned(new (S.Context) DeclRefExpr(Fn, Fn->getType(), 
-                                                     VK_LValue, Loc, LocInfo));
+  DeclRefExpr *DRE = new (S.Context) DeclRefExpr(Fn, Fn->getType(),
+                                                 VK_LValue, Loc, LocInfo);
+  if (HadMultipleCandidates)
+    DRE->setHadMultipleCandidates(true);
+  ExprResult E = S.Owned(DRE);
   E = S.DefaultFunctionArrayConversion(E.take());
   if (E.isInvalid())
     return ExprError();
@@ -258,7 +261,7 @@ isPointerConversionToVoidPointer(ASTContext& Context) const {
 /// DebugPrint - Print this standard conversion sequence to standard
 /// error. Useful for debugging overloading issues.
 void StandardConversionSequence::DebugPrint() const {
-  llvm::raw_ostream &OS = llvm::errs();
+  raw_ostream &OS = llvm::errs();
   bool PrintedSomething = false;
   if (First != ICK_Identity) {
     OS << GetImplicitConversionName(First);
@@ -297,12 +300,12 @@ void StandardConversionSequence::DebugPrint() const {
 /// DebugPrint - Print this user-defined conversion sequence to standard
 /// error. Useful for debugging overloading issues.
 void UserDefinedConversionSequence::DebugPrint() const {
-  llvm::raw_ostream &OS = llvm::errs();
+  raw_ostream &OS = llvm::errs();
   if (Before.First || Before.Second || Before.Third) {
     Before.DebugPrint();
     OS << " -> ";
   }
-  OS << '\'' << ConversionFunction << '\'';
+  OS << '\'' << *ConversionFunction << '\'';
   if (After.First || After.Second || After.Third) {
     OS << " -> ";
     After.DebugPrint();
@@ -312,7 +315,7 @@ void UserDefinedConversionSequence::DebugPrint() const {
 /// DebugPrint - Print this implicit conversion sequence to standard
 /// error. Useful for debugging overloading issues.
 void ImplicitConversionSequence::DebugPrint() const {
-  llvm::raw_ostream &OS = llvm::errs();
+  raw_ostream &OS = llvm::errs();
   switch (ConversionKind) {
   case StandardConversion:
     OS << "Standard conversion: ";
@@ -909,20 +912,22 @@ Sema::TryImplicitConversion(Expr *From, QualType ToType,
 /// explicit user-defined conversions are permitted.
 ExprResult
 Sema::PerformImplicitConversion(Expr *From, QualType ToType,
-                                AssignmentAction Action, bool AllowExplicit) {
+                                AssignmentAction Action, bool AllowExplicit,
+                                bool Diagnose) {
   ImplicitConversionSequence ICS;
-  return PerformImplicitConversion(From, ToType, Action, AllowExplicit, ICS);
+  return PerformImplicitConversion(From, ToType, Action, AllowExplicit, ICS,
+                                   Diagnose);
 }
 
 ExprResult
 Sema::PerformImplicitConversion(Expr *From, QualType ToType,
                                 AssignmentAction Action, bool AllowExplicit,
-                                ImplicitConversionSequence& ICS) {
+                                ImplicitConversionSequence& ICS,
+                                bool Diagnose) {
   // Objective-C ARC: Determine whether we will allow the writeback conversion.
   bool AllowObjCWritebackConversion
     = getLangOptions().ObjCAutoRefCount && 
       (Action == AA_Passing || Action == AA_Sending);
-  
 
   ICS = clang::TryImplicitConversion(*this, From, ToType,
                                      /*SuppressUserConversions=*/false,
@@ -930,6 +935,8 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
                                      /*InOverloadResolution=*/false,
                                      /*CStyle=*/false,
                                      AllowObjCWritebackConversion);
+  if (!Diagnose && ICS.isFailure())
+    return ExprError();
   return PerformImplicitConversion(From, ToType, ICS, Action);
 }
 
@@ -1113,10 +1120,10 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
       return false;
     }
   }
-  // Lvalue-to-rvalue conversion (C++ 4.1):
-  //   An lvalue (3.10) of a non-function, non-array type T can be
-  //   converted to an rvalue.
-  bool argIsLValue = From->isLValue();
+  // Lvalue-to-rvalue conversion (C++11 4.1):
+  //   A glvalue (3.10) of a non-function, non-array type T can
+  //   be converted to a prvalue.
+  bool argIsLValue = From->isGLValue();
   if (argIsLValue &&
       !FromType->isFunctionType() && !FromType->isArrayType() &&
       S.Context.getCanonicalType(FromType) != S.Context.OverloadTy) {
@@ -1392,12 +1399,9 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
       ToType->isIntegerType()) {
     // Determine whether the type we're converting from is signed or
     // unsigned.
-    bool FromIsSigned;
+    bool FromIsSigned = FromType->isSignedIntegerType();
     uint64_t FromSize = Context.getTypeSize(FromType);
 
-    // FIXME: Is wchar_t signed or unsigned? We assume it's signed for now.
-    FromIsSigned = true;
-
     // The types we'll try to promote to, in the appropriate
     // order. Try each of these types.
     QualType PromoteTypes[6] = {
@@ -1465,10 +1469,10 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
 /// FromType to ToType is a floating point promotion (C++ 4.6). If so,
 /// returns true and sets PromotedType to the promoted type.
 bool Sema::IsFloatingPointPromotion(QualType FromType, QualType ToType) {
-  /// An rvalue of type float can be converted to an rvalue of type
-  /// double. (C++ 4.6p1).
   if (const BuiltinType *FromBuiltin = FromType->getAs<BuiltinType>())
     if (const BuiltinType *ToBuiltin = ToType->getAs<BuiltinType>()) {
+      /// An rvalue of type float can be converted to an rvalue of type
+      /// double. (C++ 4.6p1).
       if (FromBuiltin->getKind() == BuiltinType::Float &&
           ToBuiltin->getKind() == BuiltinType::Double)
         return true;
@@ -1481,6 +1485,11 @@ bool Sema::IsFloatingPointPromotion(QualType FromType, QualType ToType) {
            FromBuiltin->getKind() == BuiltinType::Double) &&
           (ToBuiltin->getKind() == BuiltinType::LongDouble))
         return true;
+
+      // Half can be promoted to float.
+      if (FromBuiltin->getKind() == BuiltinType::Half &&
+          ToBuiltin->getKind() == BuiltinType::Float)
+        return true;
     }
 
   return false;
@@ -1668,7 +1677,7 @@ bool Sema::IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
   }
 
   // MSVC allows implicit function to void* type conversion.
-  if (getLangOptions().Microsoft && FromPointeeType->isFunctionType() &&
+  if (getLangOptions().MicrosoftExt && FromPointeeType->isFunctionType() &&
       ToPointeeType->isVoidType()) {
     ConvertedType = BuildSimilarlyQualifiedPointerType(FromTypePtr,
                                                        ToPointeeType,
@@ -2073,6 +2082,11 @@ bool Sema::IsBlockPointerConversion(QualType FromType, QualType ToType,
        // Argument types are too different. Abort.
        return false;
    }
+   if (LangOpts.ObjCAutoRefCount && 
+       !Context.FunctionTypesMatchOnNSConsumedAttrs(FromFunctionType, 
+                                                    ToFunctionType))
+     return false;
+   
    ConvertedType = ToType;
    return true;
 }
@@ -2136,8 +2150,8 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
                         PDiag(diag::warn_impcast_bool_to_null_pointer)
                           << ToType << From->getSourceRange());
 
-  if (const PointerType *FromPtrType = FromType->getAs<PointerType>())
-    if (const PointerType *ToPtrType = ToType->getAs<PointerType>()) {
+  if (const PointerType *ToPtrType = ToType->getAs<PointerType>()) {
+    if (const PointerType *FromPtrType = FromType->getAs<PointerType>()) {
       QualType FromPointeeType = FromPtrType->getPointeeType(),
                ToPointeeType   = ToPtrType->getPointeeType();
 
@@ -2155,16 +2169,23 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
         Kind = CK_DerivedToBase;
       }
     }
-  if (const ObjCObjectPointerType *FromPtrType =
-        FromType->getAs<ObjCObjectPointerType>()) {
-    if (const ObjCObjectPointerType *ToPtrType =
-          ToType->getAs<ObjCObjectPointerType>()) {
+  } else if (const ObjCObjectPointerType *ToPtrType =
+               ToType->getAs<ObjCObjectPointerType>()) {
+    if (const ObjCObjectPointerType *FromPtrType =
+          FromType->getAs<ObjCObjectPointerType>()) {
       // Objective-C++ conversions are always okay.
       // FIXME: We should have a different class of conversions for the
       // Objective-C++ implicit conversions.
       if (FromPtrType->isObjCBuiltinType() || ToPtrType->isObjCBuiltinType())
         return false;
+    } else if (FromType->isBlockPointerType()) {
+      Kind = CK_BlockPointerToObjCPointerCast;
+    } else {
+      Kind = CK_CPointerToObjCPointerCast;
     }
+  } else if (ToType->isBlockPointerType()) {
+    if (!FromType->isBlockPointerType())
+      Kind = CK_AnyPointerToBlockPointerCast;
   }
 
   // We shouldn't fall into this case unless it's valid for other
@@ -2483,6 +2504,8 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     }
   }
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, From->getLocStart(), Best, true)) {
   case OR_Success:
@@ -2504,8 +2527,9 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
         User.Before = Best->Conversions[0].Standard;
         User.EllipsisConversion = false;
       }
+      User.HadMultipleCandidates = HadMultipleCandidates;
       User.ConversionFunction = Constructor;
-      User.FoundConversionFunction = Best->FoundDecl.getDecl();
+      User.FoundConversionFunction = Best->FoundDecl;
       User.After.setAsIdentityConversion();
       User.After.setFromType(ThisType->getAs<PointerType>()->getPointeeType());
       User.After.setAllToTypes(ToType);
@@ -2521,8 +2545,9 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
       //   conversion sequence converts the source type to the
       //   implicit object parameter of the conversion function.
       User.Before = Best->Conversions[0].Standard;
+      User.HadMultipleCandidates = HadMultipleCandidates;
       User.ConversionFunction = Conversion;
-      User.FoundConversionFunction = Best->FoundDecl.getDecl();
+      User.FoundConversionFunction = Best->FoundDecl;
       User.EllipsisConversion = false;
 
       // C++ [over.ics.user]p2:
@@ -2862,6 +2887,25 @@ CompareStandardConversionSequences(Sema &S,
     }
   }
 
+  // In Microsoft mode, prefer an integral conversion to a
+  // floating-to-integral conversion if the integral conversion
+  // is between types of the same size.
+  // For example:
+  // void f(float);
+  // void f(int);
+  // int main {
+  //    long a;
+  //    f(a);
+  // }
+  // Here, MSVC will call f(int) instead of generating a compile error
+  // as clang will do in standard mode.
+  if (S.getLangOptions().MicrosoftMode &&
+      SCS1.Second == ICK_Integral_Conversion &&
+      SCS2.Second == ICK_Floating_Integral && 
+      S.Context.getTypeSize(SCS1.getFromType()) ==
+      S.Context.getTypeSize(SCS1.getToType(2)))
+    return ImplicitConversionSequence::Better;
+
   return ImplicitConversionSequence::Indistinguishable;
 }
 
@@ -3292,6 +3336,16 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
       bool DerivedToBase = false;
       bool ObjCConversion = false;
       bool ObjCLifetimeConversion = false;
+      
+      // If we are initializing an rvalue reference, don't permit conversion
+      // functions that return lvalues.
+      if (!ConvTemplate && DeclType->isRValueReferenceType()) {
+        const ReferenceType *RefType
+          = Conv->getConversionType()->getAs<LValueReferenceType>();
+        if (RefType && !RefType->getPointeeType()->isFunctionType())
+          continue;
+      }
+      
       if (!ConvTemplate &&
           S.CompareReferenceRelationship(
             DeclLoc,
@@ -3322,6 +3376,8 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
                                DeclType, CandidateSet);
   }
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, DeclLoc, Best, true)) {
   case OR_Success:
@@ -3343,8 +3399,9 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
     ICS.setUserDefined();
     ICS.UserDefined.Before = Best->Conversions[0].Standard;
     ICS.UserDefined.After = Best->FinalConversion;
+    ICS.UserDefined.HadMultipleCandidates = HadMultipleCandidates;
     ICS.UserDefined.ConversionFunction = Best->Function;
-    ICS.UserDefined.FoundConversionFunction = Best->FoundDecl.getDecl();
+    ICS.UserDefined.FoundConversionFunction = Best->FoundDecl;
     ICS.UserDefined.EllipsisConversion = false;
     assert(ICS.UserDefined.After.ReferenceBinding &&
            ICS.UserDefined.After.DirectBinding &&
@@ -3611,12 +3668,25 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
     ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false;
     ICS.Standard.ObjCLifetimeConversionBinding = false;
   } else if (ICS.isUserDefined()) {
+    // Don't allow rvalue references to bind to lvalues.
+    if (DeclType->isRValueReferenceType()) {
+      if (const ReferenceType *RefType
+            = ICS.UserDefined.ConversionFunction->getResultType()
+                ->getAs<LValueReferenceType>()) {
+        if (!RefType->getPointeeType()->isFunctionType()) {
+          ICS.setBad(BadConversionSequence::lvalue_ref_to_rvalue, Init, 
+                     DeclType);
+          return ICS;
+        }
+      }
+    }
+    
     ICS.UserDefined.After.ReferenceBinding = true;
-    ICS.Standard.IsLvalueReference = !isRValRef;
-    ICS.Standard.BindsToFunctionLvalue = T2->isFunctionType();
-    ICS.Standard.BindsToRvalue = true;
-    ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false;
-    ICS.Standard.ObjCLifetimeConversionBinding = false;
+    ICS.UserDefined.After.IsLvalueReference = !isRValRef;
+    ICS.UserDefined.After.BindsToFunctionLvalue = T2->isFunctionType();
+    ICS.UserDefined.After.BindsToRvalue = true;
+    ICS.UserDefined.After.BindsImplicitObjectArgumentWithoutRefQualifier = false;
+    ICS.UserDefined.After.ObjCLifetimeConversionBinding = false;
   }
 
   return ICS;
@@ -3647,6 +3717,18 @@ TryCopyInitialization(Sema &S, Expr *From, QualType ToType,
                                AllowObjCWritebackConversion);
 }
 
+static bool TryCopyInitialization(const CanQualType FromQTy,
+                                  const CanQualType ToQTy,
+                                  Sema &S,
+                                  SourceLocation Loc,
+                                  ExprValueKind FromVK) {
+  OpaqueValueExpr TmpExpr(Loc, FromQTy, FromVK);
+  ImplicitConversionSequence ICS =
+    TryCopyInitialization(S, &TmpExpr, ToQTy, true, true, false);
+
+  return !ICS.isBad();
+}
+
 /// TryObjectArgumentInitialization - Try to initialize the object
 /// parameter of the given member function (@c Method) from the
 /// expression @p From.
@@ -3852,25 +3934,57 @@ ExprResult Sema::PerformContextuallyConvertToBool(Expr *From) {
   return ExprError();
 }
 
-/// TryContextuallyConvertToObjCId - Attempt to contextually convert the
-/// expression From to 'id'.
-static ImplicitConversionSequence
-TryContextuallyConvertToObjCId(Sema &S, Expr *From) {
-  QualType Ty = S.Context.getObjCIdType();
-  return TryImplicitConversion(S, From, Ty,
-                               // FIXME: Are these flags correct?
-                               /*SuppressUserConversions=*/false,
-                               /*AllowExplicit=*/true,
-                               /*InOverloadResolution=*/false,
-                               /*CStyle=*/false,
-                               /*AllowObjCWritebackConversion=*/false);
+/// dropPointerConversions - If the given standard conversion sequence
+/// involves any pointer conversions, remove them.  This may change
+/// the result type of the conversion sequence.
+static void dropPointerConversion(StandardConversionSequence &SCS) {
+  if (SCS.Second == ICK_Pointer_Conversion) {
+    SCS.Second = ICK_Identity;
+    SCS.Third = ICK_Identity;
+    SCS.ToTypePtrs[2] = SCS.ToTypePtrs[1] = SCS.ToTypePtrs[0];
+  }
 }
 
-/// PerformContextuallyConvertToObjCId - Perform a contextual conversion
-/// of the expression From to 'id'.
-ExprResult Sema::PerformContextuallyConvertToObjCId(Expr *From) {
+/// TryContextuallyConvertToObjCPointer - Attempt to contextually
+/// convert the expression From to an Objective-C pointer type.
+static ImplicitConversionSequence
+TryContextuallyConvertToObjCPointer(Sema &S, Expr *From) {
+  // Do an implicit conversion to 'id'.
+  QualType Ty = S.Context.getObjCIdType();
+  ImplicitConversionSequence ICS
+    = TryImplicitConversion(S, From, Ty,
+                            // FIXME: Are these flags correct?
+                            /*SuppressUserConversions=*/false,
+                            /*AllowExplicit=*/true,
+                            /*InOverloadResolution=*/false,
+                            /*CStyle=*/false,
+                            /*AllowObjCWritebackConversion=*/false);
+
+  // Strip off any final conversions to 'id'.
+  switch (ICS.getKind()) {
+  case ImplicitConversionSequence::BadConversion:
+  case ImplicitConversionSequence::AmbiguousConversion:
+  case ImplicitConversionSequence::EllipsisConversion:
+    break;
+
+  case ImplicitConversionSequence::UserDefinedConversion:
+    dropPointerConversion(ICS.UserDefined.After);
+    break;
+
+  case ImplicitConversionSequence::StandardConversion:
+    dropPointerConversion(ICS.Standard);
+    break;
+  }
+
+  return ICS;
+}
+
+/// PerformContextuallyConvertToObjCPointer - Perform a contextual
+/// conversion of the expression From to an Objective-C pointer type.
+ExprResult Sema::PerformContextuallyConvertToObjCPointer(Expr *From) {
   QualType Ty = Context.getObjCIdType();
-  ImplicitConversionSequence ICS = TryContextuallyConvertToObjCId(*this, From);
+  ImplicitConversionSequence ICS =
+    TryContextuallyConvertToObjCPointer(*this, From);
   if (!ICS.isBad())
     return PerformImplicitConversion(From, Ty, ICS, AA_Converting);
   return ExprError();
@@ -3951,6 +4065,8 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
   const UnresolvedSetImpl *Conversions
     = cast<CXXRecordDecl>(RecordTy->getDecl())->getVisibleConversionFunctions();
 
+  bool HadMultipleCandidates = (Conversions->size() > 1);
+
   for (UnresolvedSetImpl::iterator I = Conversions->begin(),
                                    E = Conversions->end();
        I != E;
@@ -3978,7 +4094,7 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
       QualType ConvTy
         = Conversion->getConversionType().getNonReferenceType();
       std::string TypeStr;
-      ConvTy.getAsStringInternal(TypeStr, Context.PrintingPolicy);
+      ConvTy.getAsStringInternal(TypeStr, getPrintingPolicy());
 
       Diag(Loc, ExplicitConvDiag)
         << T << ConvTy
@@ -3995,7 +4111,8 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
         return ExprError();
 
       CheckMemberOperatorAccess(From->getExprLoc(), From, 0, Found);
-      ExprResult Result = BuildCXXMemberCallExpr(From, Found, Conversion);
+      ExprResult Result = BuildCXXMemberCallExpr(From, Found, Conversion,
+                                                 HadMultipleCandidates);
       if (Result.isInvalid())
         return ExprError();
 
@@ -4022,8 +4139,8 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
         << T << ConvTy->isEnumeralType() << ConvTy << From->getSourceRange();
     }
 
-    ExprResult Result = BuildCXXMemberCallExpr(From, Found,
-                          cast<CXXConversionDecl>(Found->getUnderlyingDecl()));
+    ExprResult Result = BuildCXXMemberCallExpr(From, Found, Conversion,
+                                               HadMultipleCandidates);
     if (Result.isInvalid())
       return ExprError();
 
@@ -4144,6 +4261,15 @@ Sema::AddOverloadCandidate(FunctionDecl *Function,
     return;
   }
 
+  // (CUDA B.1): Check for invalid calls between targets.
+  if (getLangOptions().CUDA)
+    if (const FunctionDecl *Caller = dyn_cast<FunctionDecl>(CurContext))
+      if (CheckCUDATarget(Caller, Function)) {
+        Candidate.Viable = false;
+        Candidate.FailureKind = ovl_fail_bad_target;
+        return;
+      }
+
   // Determine the implicit conversion sequences for each of the
   // arguments.
   Candidate.Conversions.resize(NumArgs);
@@ -4590,7 +4716,7 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
     break;
 
   default:
-    assert(false &&
+    llvm_unreachable(
            "Can only end up with a standard conversion sequence or failure");
   }
 }
@@ -4686,9 +4812,9 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
   Candidate.Conversions[0].setUserDefined();
   Candidate.Conversions[0].UserDefined.Before = ObjectInit.Standard;
   Candidate.Conversions[0].UserDefined.EllipsisConversion = false;
+  Candidate.Conversions[0].UserDefined.HadMultipleCandidates = false;
   Candidate.Conversions[0].UserDefined.ConversionFunction = Conversion;
-  Candidate.Conversions[0].UserDefined.FoundConversionFunction
-    = FoundDecl.getDecl();
+  Candidate.Conversions[0].UserDefined.FoundConversionFunction = FoundDecl;
   Candidate.Conversions[0].UserDefined.After
     = Candidate.Conversions[0].UserDefined.Before;
   Candidate.Conversions[0].UserDefined.After.setAsIdentityConversion();
@@ -4973,7 +5099,7 @@ BuiltinCandidateTypeSet::AddPointerWithMoreQualifiedTypeVariants(QualType Ty,
       buildObjCPtr = true;
     }
     else
-      assert(false && "type was not a pointer type!");
+      llvm_unreachable("type was not a pointer type!");
   }
   else
     PointeeTy = PointerTy->getPointeeType();
@@ -5230,7 +5356,7 @@ class BuiltinOperatorOverloadBuilder {
   unsigned NumArgs;
   Qualifiers VisibleTypeConversionsQuals;
   bool HasArithmeticOrEnumeralCandidateType;
-  llvm::SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes;
+  SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes;
   OverloadCandidateSet &CandidateSet;
 
   // Define some constants used to index and iterate over the arithemetic types
@@ -5366,7 +5492,7 @@ class BuiltinOperatorOverloadBuilder {
     Sema &S, Expr **Args, unsigned NumArgs,
     Qualifiers VisibleTypeConversionsQuals,
     bool HasArithmeticOrEnumeralCandidateType,
-    llvm::SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes,
+    SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes,
     OverloadCandidateSet &CandidateSet)
     : S(S), Args(Args), NumArgs(NumArgs),
       VisibleTypeConversionsQuals(VisibleTypeConversionsQuals),
@@ -6235,7 +6361,7 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
 
   bool HasNonRecordCandidateType = false;
   bool HasArithmeticOrEnumeralCandidateType = false;
-  llvm::SmallVector<BuiltinCandidateTypeSet, 2> CandidateTypes;
+  SmallVector<BuiltinCandidateTypeSet, 2> CandidateTypes;
   for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx) {
     CandidateTypes.push_back(BuiltinCandidateTypeSet(*this));
     CandidateTypes[ArgIdx].AddTypesConvertedFrom(Args[ArgIdx]->getType(),
@@ -6254,7 +6380,11 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
 
   // Exit early when no non-record types have been added to the candidate set
   // for any of the arguments to the operator.
-  if (!HasNonRecordCandidateType)
+  //
+  // We can't exit early for !, ||, or &&, since there we have always have
+  // 'bool' overloads.
+  if (!HasNonRecordCandidateType && 
+      !(Op == OO_Exclaim || Op == OO_AmpAmp || Op == OO_PipePipe))
     return;
 
   // Setup an object to manage the common state for building overloads.
@@ -6267,16 +6397,15 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
   switch (Op) {
   case OO_None:
   case NUM_OVERLOADED_OPERATORS:
-    assert(false && "Expected an overloaded operator");
-    break;
+    llvm_unreachable("Expected an overloaded operator");
 
   case OO_New:
   case OO_Delete:
   case OO_Array_New:
   case OO_Array_Delete:
   case OO_Call:
-    assert(false && "Special operators don't use AddBuiltinOperatorCandidates");
-    break;
+    llvm_unreachable(
+                    "Special operators don't use AddBuiltinOperatorCandidates");
 
   case OO_Comma:
   case OO_Arrow:
@@ -6846,6 +6975,17 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
     return;
   }
 
+  // Special diagnostic for failure to convert an initializer list, since
+  // telling the user that it has type void is not useful.
+  if (FromExpr && isa<InitListExpr>(FromExpr)) {
+    S.Diag(Fn->getLocation(), diag::note_ovl_candidate_bad_list_argument)
+      << (unsigned) FnKind << FnDesc
+      << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
+      << FromTy << ToTy << (unsigned) isObjectArgument << I+1;
+    MaybeEmitInheritedConstructorNote(S, Fn);
+    return;
+  }
+
   // Diagnose references or pointers to incomplete types differently,
   // since it's far from impossible that the incompleteness triggered
   // the failure.
@@ -6902,11 +7042,34 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
     return;
   }
 
-  // TODO: specialize more based on the kind of mismatch
-  S.Diag(Fn->getLocation(), diag::note_ovl_candidate_bad_conv)
-    << (unsigned) FnKind << FnDesc
+  if (isa<ObjCObjectPointerType>(CFromTy) &&
+      isa<PointerType>(CToTy)) {
+      Qualifiers FromQs = CFromTy.getQualifiers();
+      Qualifiers ToQs = CToTy.getQualifiers();
+      if (FromQs.getObjCLifetime() != ToQs.getObjCLifetime()) {
+        S.Diag(Fn->getLocation(), diag::note_ovl_candidate_bad_arc_conv)
+        << (unsigned) FnKind << FnDesc
+        << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
+        << FromTy << ToTy << (unsigned) isObjectArgument << I+1;
+        MaybeEmitInheritedConstructorNote(S, Fn);
+        return;
+      }
+  }
+  
+  // Emit the generic diagnostic and, optionally, add the hints to it.
+  PartialDiagnostic FDiag = S.PDiag(diag::note_ovl_candidate_bad_conv);
+  FDiag << (unsigned) FnKind << FnDesc
     << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
-    << FromTy << ToTy << (unsigned) isObjectArgument << I+1;
+    << FromTy << ToTy << (unsigned) isObjectArgument << I + 1
+    << (unsigned) (Cand->Fix.Kind);
+
+  // If we can fix the conversion, suggest the FixIts.
+  for (SmallVector<FixItHint, 1>::iterator
+      HI = Cand->Fix.Hints.begin(), HE = Cand->Fix.Hints.end();
+      HI != HE; ++HI)
+    FDiag << *HI;
+  S.Diag(Fn->getLocation(), FDiag);
+
   MaybeEmitInheritedConstructorNote(S, Fn);
 }
 
@@ -7081,6 +7244,21 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
   }
 }
 
+/// CUDA: diagnose an invalid call across targets.
+void DiagnoseBadTarget(Sema &S, OverloadCandidate *Cand) {
+  FunctionDecl *Caller = cast<FunctionDecl>(S.CurContext);
+  FunctionDecl *Callee = Cand->Function;
+
+  Sema::CUDAFunctionTarget CallerTarget = S.IdentifyCUDATarget(Caller),
+                           CalleeTarget = S.IdentifyCUDATarget(Callee);
+
+  std::string FnDesc;
+  OverloadCandidateKind FnKind = ClassifyOverloadCandidate(S, Callee, FnDesc);
+
+  S.Diag(Callee->getLocation(), diag::note_ovl_candidate_bad_target)
+      << (unsigned) FnKind << CalleeTarget << CallerTarget;
+}
+
 /// Generates a 'note' diagnostic for an overload candidate.  We've
 /// already generated a primary error at the call site.
 ///
@@ -7140,6 +7318,9 @@ void NoteFunctionCandidate(Sema &S, OverloadCandidate *Cand,
     // those conditions and diagnose them well.
     return S.NoteOverloadCandidate(Fn);
   }
+
+  case ovl_fail_bad_target:
+    return DiagnoseBadTarget(S, Cand);
   }
 }
 
@@ -7217,6 +7398,37 @@ SourceLocation GetLocationForCandidate(const OverloadCandidate *Cand) {
   return SourceLocation();
 }
 
+static unsigned
+RankDeductionFailure(const OverloadCandidate::DeductionFailureInfo &DFI) {
+  switch ((Sema::TemplateDeductionResult)DFI.Result) {
+  case Sema::TDK_Success:
+    llvm_unreachable("TDK_success while diagnosing bad deduction");
+
+  case Sema::TDK_Incomplete:
+    return 1;
+
+  case Sema::TDK_Underqualified:
+  case Sema::TDK_Inconsistent:
+    return 2;
+
+  case Sema::TDK_SubstitutionFailure:
+  case Sema::TDK_NonDeducedMismatch:
+    return 3;
+
+  case Sema::TDK_InstantiationDepth:
+  case Sema::TDK_FailedOverloadResolution:
+    return 4;
+
+  case Sema::TDK_InvalidExplicitArguments:
+    return 5;
+
+  case Sema::TDK_TooManyArguments:
+  case Sema::TDK_TooFewArguments:
+    return 6;
+  }
+  llvm_unreachable("Unhandled deduction result");
+}
+
 struct CompareOverloadCandidatesForDisplay {
   Sema &S;
   CompareOverloadCandidatesForDisplay(Sema &S) : S(S) {}
@@ -7256,6 +7468,19 @@ struct CompareOverloadCandidatesForDisplay {
         if (R->FailureKind != ovl_fail_bad_conversion)
           return true;
 
+        // The conversion that can be fixed with a smaller number of changes,
+        // comes first.
+        unsigned numLFixes = L->Fix.NumConversionsFixed;
+        unsigned numRFixes = R->Fix.NumConversionsFixed;
+        numLFixes = (numLFixes == 0) ? UINT_MAX : numLFixes;
+        numRFixes = (numRFixes == 0) ? UINT_MAX : numRFixes;
+        if (numLFixes != numRFixes) {
+          if (numLFixes < numRFixes)
+            return true;
+          else
+            return false;
+        }
+
         // If there's any ordering between the defined conversions...
         // FIXME: this might not be transitive.
         assert(L->Conversions.size() == R->Conversions.size());
@@ -7284,6 +7509,16 @@ struct CompareOverloadCandidatesForDisplay {
       } else if (R->FailureKind == ovl_fail_bad_conversion)
         return false;
 
+      if (L->FailureKind == ovl_fail_bad_deduction) {
+        if (R->FailureKind != ovl_fail_bad_deduction)
+          return true;
+
+        if (L->DeductionFailure.Result != R->DeductionFailure.Result)
+          return RankDeductionFailure(L->DeductionFailure)
+               < RankDeductionFailure(R->DeductionFailure);
+      } else if (R->FailureKind == ovl_fail_bad_deduction)
+        return false;
+
       // TODO: others?
     }
 
@@ -7300,7 +7535,7 @@ struct CompareOverloadCandidatesForDisplay {
 };
 
 /// CompleteNonViableCandidate - Normally, overload resolution only
-/// computes up to the first
+/// computes up to the first. Produces the FixIt set if possible.
 void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
                                 Expr **Args, unsigned NumArgs) {
   assert(!Cand->Viable);
@@ -7308,14 +7543,21 @@ void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
   // Don't do anything on failures other than bad conversion.
   if (Cand->FailureKind != ovl_fail_bad_conversion) return;
 
+  // We only want the FixIts if all the arguments can be corrected.
+  bool Unfixable = false;
+  // Use a implicit copy initialization to check conversion fixes.
+  Cand->Fix.setConversionChecker(TryCopyInitialization);
+
   // Skip forward to the first bad conversion.
   unsigned ConvIdx = (Cand->IgnoreObjectArgument ? 1 : 0);
   unsigned ConvCount = Cand->Conversions.size();
   while (true) {
     assert(ConvIdx != ConvCount && "no bad conversion in candidate");
     ConvIdx++;
-    if (Cand->Conversions[ConvIdx - 1].isBad())
+    if (Cand->Conversions[ConvIdx - 1].isBad()) {
+      Unfixable = !Cand->TryToFixBadConversion(ConvIdx - 1, S);
       break;
+    }
   }
 
   if (ConvIdx == ConvCount)
@@ -7360,13 +7602,17 @@ void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
   // Fill in the rest of the conversions.
   unsigned NumArgsInProto = Proto->getNumArgs();
   for (; ConvIdx != ConvCount; ++ConvIdx, ++ArgIdx) {
-    if (ArgIdx < NumArgsInProto)
+    if (ArgIdx < NumArgsInProto) {
       Cand->Conversions[ConvIdx]
         = TryCopyInitialization(S, Args[ArgIdx], Proto->getArgType(ArgIdx),
                                 SuppressUserConversions,
                                 /*InOverloadResolution=*/true,
                                 /*AllowObjCWritebackConversion=*/
                                   S.getLangOptions().ObjCAutoRefCount);
+      // Store the FixIt in the candidate if it exists.
+      if (!Unfixable && Cand->Conversions[ConvIdx].isBad())
+        Unfixable = !Cand->TryToFixBadConversion(ConvIdx, S);
+    }
     else
       Cand->Conversions[ConvIdx].setEllipsis();
   }
@@ -7384,7 +7630,7 @@ void OverloadCandidateSet::NoteCandidates(Sema &S,
                                           SourceLocation OpLoc) {
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
-  llvm::SmallVector<OverloadCandidate*, 32> Cands;
+  SmallVector<OverloadCandidate*, 32> Cands;
   if (OCD == OCD_AllCandidates) Cands.reserve(size());
   for (iterator Cand = begin(), LastCand = end(); Cand != LastCand; ++Cand) {
     if (Cand->Viable)
@@ -7403,8 +7649,9 @@ void OverloadCandidateSet::NoteCandidates(Sema &S,
 
   bool ReportedAmbiguousConversions = false;
 
-  llvm::SmallVectorImpl<OverloadCandidate*>::iterator I, E;
-  const Diagnostic::OverloadsShown ShowOverloads = S.Diags.getShowOverloads();
+  SmallVectorImpl<OverloadCandidate*>::iterator I, E;
+  const DiagnosticsEngine::OverloadsShown ShowOverloads = 
+      S.Diags.getShowOverloads();
   unsigned CandsShown = 0;
   for (I = Cands.begin(), E = Cands.end(); I != E; ++I) {
     OverloadCandidate *Cand = *I;
@@ -7412,7 +7659,7 @@ void OverloadCandidateSet::NoteCandidates(Sema &S,
     // Set an arbitrary limit on the number of candidate functions we'll spam
     // the user with.  FIXME: This limit should depend on details of the
     // candidate list.
-    if (CandsShown >= 4 && ShowOverloads == Diagnostic::Ovl_Best) {
+    if (CandsShown >= 4 && ShowOverloads == DiagnosticsEngine::Ovl_Best) {
       break;
     }
     ++CandsShown;
@@ -7485,7 +7732,7 @@ class AddressOfFunctionResolver
   OverloadExpr::FindResult OvlExprInfo; 
   OverloadExpr *OvlExpr;
   TemplateArgumentListInfo OvlExplicitTemplateArgs;
-  llvm::SmallVector<std::pair<DeclAccessPair, FunctionDecl*>, 4> Matches;
+  SmallVector<std::pair<DeclAccessPair, FunctionDecl*>, 4> Matches;
 
 public:
   AddressOfFunctionResolver(Sema &S, Expr* SourceExpr, 
@@ -7607,6 +7854,11 @@ class AddressOfFunctionResolver
       return false;
 
     if (FunctionDecl *FunDecl = dyn_cast<FunctionDecl>(Fn)) {
+      if (S.getLangOptions().CUDA)
+        if (FunctionDecl *Caller = dyn_cast<FunctionDecl>(S.CurContext))
+          if (S.CheckCUDATarget(Caller, FunDecl))
+            return false;
+
       QualType ResultTy;
       if (Context.hasSameUnqualifiedType(TargetFunctionType, 
                                          FunDecl->getType()) ||
@@ -7873,25 +8125,31 @@ Sema::ResolveSingleFunctionTemplateSpecialization(OverloadExpr *ovl,
 
 
 
-// Resolve and fix an overloaded expression that
-// can be resolved because it identifies a single function
-// template specialization
+// Resolve and fix an overloaded expression that can be resolved
+// because it identifies a single function template specialization.
+//
 // Last three arguments should only be supplied if Complain = true
-ExprResult Sema::ResolveAndFixSingleFunctionTemplateSpecialization(
-             Expr *SrcExpr, bool doFunctionPointerConverion, bool complain,
-                                  const SourceRange& OpRangeForComplaining, 
+//
+// Return true if it was logically possible to so resolve the
+// expression, regardless of whether or not it succeeded.  Always
+// returns true if 'complain' is set.
+bool Sema::ResolveAndFixSingleFunctionTemplateSpecialization(
+                      ExprResult &SrcExpr, bool doFunctionPointerConverion,
+                   bool complain, const SourceRange& OpRangeForComplaining, 
                                            QualType DestTypeForComplaining, 
                                             unsigned DiagIDForComplaining) {
-  assert(SrcExpr->getType() == Context.OverloadTy);
+  assert(SrcExpr.get()->getType() == Context.OverloadTy);
 
-  OverloadExpr::FindResult ovl = OverloadExpr::find(SrcExpr);
+  OverloadExpr::FindResult ovl = OverloadExpr::find(SrcExpr.get());
 
   DeclAccessPair found;
   ExprResult SingleFunctionExpression;
   if (FunctionDecl *fn = ResolveSingleFunctionTemplateSpecialization(
                            ovl.Expression, /*complain*/ false, &found)) {
-    if (DiagnoseUseOfDecl(fn, SrcExpr->getSourceRange().getBegin()))
-      return ExprError();
+    if (DiagnoseUseOfDecl(fn, SrcExpr.get()->getSourceRange().getBegin())) {
+      SrcExpr = ExprError();
+      return true;
+    }
 
     // It is only correct to resolve to an instance method if we're
     // resolving a form that's permitted to be a pointer to member.
@@ -7900,28 +8158,34 @@ ExprResult Sema::ResolveAndFixSingleFunctionTemplateSpecialization(
     if (!ovl.HasFormOfMemberPointer &&
         isa<CXXMethodDecl>(fn) &&
         cast<CXXMethodDecl>(fn)->isInstance()) {
-      if (complain) {
-        Diag(ovl.Expression->getExprLoc(),
-             diag::err_invalid_use_of_bound_member_func)
-          << ovl.Expression->getSourceRange();
-        // TODO: I believe we only end up here if there's a mix of
-        // static and non-static candidates (otherwise the expression
-        // would have 'bound member' type, not 'overload' type).
-        // Ideally we would note which candidate was chosen and why
-        // the static candidates were rejected.
-      }
-      
-      return ExprError();
+      if (!complain) return false;
+
+      Diag(ovl.Expression->getExprLoc(),
+           diag::err_bound_member_function)
+        << 0 << ovl.Expression->getSourceRange();
+
+      // TODO: I believe we only end up here if there's a mix of
+      // static and non-static candidates (otherwise the expression
+      // would have 'bound member' type, not 'overload' type).
+      // Ideally we would note which candidate was chosen and why
+      // the static candidates were rejected.
+      SrcExpr = ExprError();
+      return true;
     }
 
     // Fix the expresion to refer to 'fn'.
     SingleFunctionExpression =
-      Owned(FixOverloadedFunctionReference(SrcExpr, found, fn));
+      Owned(FixOverloadedFunctionReference(SrcExpr.take(), found, fn));
 
     // If desired, do function-to-pointer decay.
-    if (doFunctionPointerConverion)
+    if (doFunctionPointerConverion) {
       SingleFunctionExpression =
         DefaultFunctionArrayLvalueConversion(SingleFunctionExpression.take());
+      if (SingleFunctionExpression.isInvalid()) {
+        SrcExpr = ExprError();
+        return true;
+      }
+    }
   }
 
   if (!SingleFunctionExpression.isUsable()) {
@@ -7931,12 +8195,17 @@ ExprResult Sema::ResolveAndFixSingleFunctionTemplateSpecialization(
         << DestTypeForComplaining
         << OpRangeForComplaining 
         << ovl.Expression->getQualifierLoc().getSourceRange();
-      NoteAllOverloadCandidates(SrcExpr);
-    }      
-    return ExprError();
+      NoteAllOverloadCandidates(SrcExpr.get());
+
+      SrcExpr = ExprError();
+      return true;
+    }
+
+    return false;
   }
 
-  return SingleFunctionExpression;
+  SrcExpr = SingleFunctionExpression;
+  return true;
 }
 
 /// \brief Add a single candidate to the overload set.
@@ -8164,7 +8433,8 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
   if (!DiagnoseTwoPhaseLookup(SemaRef, Fn->getExprLoc(), SS, R,
                               ExplicitTemplateArgs, Args, NumArgs) &&
       (!EmptyLookup ||
-       SemaRef.DiagnoseEmptyLookup(S, SS, R, Sema::CTC_Expression)))
+       SemaRef.DiagnoseEmptyLookup(S, SS, R, Sema::CTC_Expression,
+                                   ExplicitTemplateArgs, Args, NumArgs)))
     return ExprError();
 
   assert(!R.empty() && "lookup results empty despite recovery");
@@ -8214,7 +8484,7 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
     if (ULE->decls_begin() + 1 == ULE->decls_end() &&
         (F = dyn_cast<FunctionDecl>(*ULE->decls_begin())) &&
         F->getBuiltinID() && F->isImplicit())
-      assert(0 && "performing ADL for builtin");
+      llvm_unreachable("performing ADL for builtin");
 
     // We don't perform ADL in C.
     assert(getLangOptions().CPlusPlus && "ADL enabled in C");
@@ -8232,9 +8502,22 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
   // If we found nothing, try to recover.
   // BuildRecoveryCallExpr diagnoses the error itself, so we just bail
   // out if it fails.
-  if (CandidateSet.empty())
+  if (CandidateSet.empty()) {
+    // In Microsoft mode, if we are inside a template class member function then
+    // create a type dependent CallExpr. The goal is to postpone name lookup
+    // to instantiation time to be able to search into type dependent base
+    // classes.
+    if (getLangOptions().MicrosoftExt && CurContext->isDependentContext() && 
+        isa<CXXMethodDecl>(CurContext)) {
+      CallExpr *CE = new (Context) CallExpr(Context, Fn, Args, NumArgs,
+                                          Context.DependentTy, VK_RValue,
+                                          RParenLoc);
+      CE->setTypeDependent(true);
+      return Owned(CE);
+    }
     return BuildRecoveryCallExpr(*this, S, Fn, ULE, LParenLoc, Args, NumArgs,
                                  RParenLoc, /*EmptyLookup=*/true);
+  }
 
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, Fn->getLocStart(), Best)) {
@@ -8379,6 +8662,8 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
   // Add builtin operator candidates.
   AddBuiltinOperatorCandidates(Op, OpLoc, &Args[0], NumArgs, CandidateSet);
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, OpLoc, Best)) {
@@ -8423,7 +8708,8 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
       ResultTy = ResultTy.getNonLValueExprType(Context);
 
       // Build the actual expression node.
-      ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl);
+      ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl,
+                                                HadMultipleCandidates);
       if (FnExpr.isInvalid())
         return ExprError();
 
@@ -8468,8 +8754,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
         << UnaryOperator::getOpcodeStr(Opc)
         << Input->getType()
         << Input->getSourceRange();
-    CandidateSet.NoteCandidates(*this, OCD_ViableCandidates,
-                                Args, NumArgs,
+    CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, NumArgs,
                                 UnaryOperator::getOpcodeStr(Opc), OpLoc);
     return ExprError();
 
@@ -8479,7 +8764,8 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
       << UnaryOperator::getOpcodeStr(Opc)
       << getDeletedOrUnavailableSuffix(Best->Function)
       << Input->getSourceRange();
-    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs,
+                                UnaryOperator::getOpcodeStr(Opc), OpLoc);
     return ExprError();
   }
 
@@ -8627,6 +8913,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
   // Add builtin operator candidates.
   AddBuiltinOperatorCandidates(Op, OpLoc, Args, 2, CandidateSet);
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, OpLoc, Best)) {
@@ -8688,7 +8976,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         ResultTy = ResultTy.getNonLValueExprType(Context);
 
         // Build the actual expression node.
-        ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl, OpLoc);
+        ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl,
+                                                  HadMultipleCandidates, OpLoc);
         if (FnExpr.isInvalid())
           return ExprError();
 
@@ -8774,7 +9063,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         << BinaryOperator::getOpcodeStr(Opc)
         << getDeletedOrUnavailableSuffix(Best->Function)
         << Args[0]->getSourceRange() << Args[1]->getSourceRange();
-      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2,
+                                  BinaryOperator::getOpcodeStr(Opc), OpLoc);
       return ExprError();
   }
 
@@ -8837,6 +9127,8 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
   // Add builtin operator candidates.
   AddBuiltinOperatorCandidates(OO_Subscript, LLoc, Args, 2, CandidateSet);
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, LLoc, Best)) {
@@ -8883,7 +9175,9 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
         DeclarationNameLoc LocInfo;
         LocInfo.CXXOperatorName.BeginOpNameLoc = LLoc.getRawEncoding();
         LocInfo.CXXOperatorName.EndOpNameLoc = RLoc.getRawEncoding();
-        ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl, LLoc, LocInfo);
+        ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl,
+                                                  HadMultipleCandidates,
+                                                  LLoc, LocInfo);
         if (FnExpr.isInvalid())
           return ExprError();
 
@@ -9059,7 +9353,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
 
 
       // Microsoft supports direct constructor calls.
-      if (getLangOptions().Microsoft && isa<CXXConstructorDecl>(Func)) {
+      if (getLangOptions().MicrosoftExt && isa<CXXConstructorDecl>(Func)) {
         AddOverloadCandidate(cast<CXXConstructorDecl>(Func), I.getPair(), Args, NumArgs,
                              CandidateSet);
       } else if ((Method = dyn_cast<CXXMethodDecl>(Func))) {
@@ -9231,8 +9525,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   }
 
   // C++ [over.call.object]p2:
-  //   In addition, for each conversion function declared in T of the
-  //   form
+  //   In addition, for each (non-explicit in C++0x) conversion function 
+  //   declared in T of the form
   //
   //        operator conversion-type-id () cv-qualifier;
   //
@@ -9262,18 +9556,23 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
       continue;
 
     CXXConversionDecl *Conv = cast<CXXConversionDecl>(D);
+    if (!Conv->isExplicit()) {
+      // Strip the reference type (if any) and then the pointer type (if
+      // any) to get down to what might be a function type.
+      QualType ConvType = Conv->getConversionType().getNonReferenceType();
+      if (const PointerType *ConvPtrType = ConvType->getAs<PointerType>())
+        ConvType = ConvPtrType->getPointeeType();
 
-    // Strip the reference type (if any) and then the pointer type (if
-    // any) to get down to what might be a function type.
-    QualType ConvType = Conv->getConversionType().getNonReferenceType();
-    if (const PointerType *ConvPtrType = ConvType->getAs<PointerType>())
-      ConvType = ConvPtrType->getPointeeType();
-
-    if (const FunctionProtoType *Proto = ConvType->getAs<FunctionProtoType>())
-      AddSurrogateCandidate(Conv, I.getPair(), ActingContext, Proto,
-                            Object.get(), Args, NumArgs, CandidateSet);
+      if (const FunctionProtoType *Proto = ConvType->getAs<FunctionProtoType>())
+      {
+        AddSurrogateCandidate(Conv, I.getPair(), ActingContext, Proto,
+                              Object.get(), Args, NumArgs, CandidateSet);
+      }
+    }
   }
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, Object.get()->getLocStart(),
@@ -9332,7 +9631,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
 
     // Create an implicit member expr to refer to the conversion operator.
     // and then call it.
-    ExprResult Call = BuildCXXMemberCallExpr(Object.get(), Best->FoundDecl, Conv);
+    ExprResult Call = BuildCXXMemberCallExpr(Object.get(), Best->FoundDecl,
+                                             Conv, HadMultipleCandidates);
     if (Call.isInvalid())
       return ExprError();
 
@@ -9368,7 +9668,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx)
     MethodArgs[ArgIdx + 1] = Args[ArgIdx];
 
-  ExprResult NewFn = CreateFunctionRefExpr(*this, Method);
+  ExprResult NewFn = CreateFunctionRefExpr(*this, Method,
+                                           HadMultipleCandidates);
   if (NewFn.isInvalid())
     return true;
 
@@ -9498,6 +9799,8 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
                        0, 0, CandidateSet, /*SuppressUserConversions=*/false);
   }
 
+  bool HadMultipleCandidates = (CandidateSet.size() > 1);
+
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, OpLoc, Best)) {
@@ -9545,7 +9848,8 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
   Base = BaseResult.take();
 
   // Build the operator call.
-  ExprResult FnExpr = CreateFunctionRefExpr(*this, Method);
+  ExprResult FnExpr = CreateFunctionRefExpr(*this, Method,
+                                            HadMultipleCandidates);
   if (FnExpr.isInvalid())
     return ExprError();
 
@@ -9648,14 +9952,16 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
       TemplateArgs = &TemplateArgsBuffer;
     }
 
-    return DeclRefExpr::Create(Context,
-                               ULE->getQualifierLoc(),
-                               Fn,
-                               ULE->getNameLoc(),
-                               Fn->getType(),
-                               VK_LValue,
-                               Found.getDecl(),
-                               TemplateArgs);
+    DeclRefExpr *DRE = DeclRefExpr::Create(Context,
+                                           ULE->getQualifierLoc(),
+                                           Fn,
+                                           ULE->getNameLoc(),
+                                           Fn->getType(),
+                                           VK_LValue,
+                                           Found.getDecl(),
+                                           TemplateArgs);
+    DRE->setHadMultipleCandidates(ULE->getNumDecls() > 1);
+    return DRE;
   }
 
   if (UnresolvedMemberExpr *MemExpr = dyn_cast<UnresolvedMemberExpr>(E)) {
@@ -9672,14 +9978,16 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
     // implicit member access, rewrite to a simple decl ref.
     if (MemExpr->isImplicitAccess()) {
       if (cast<CXXMethodDecl>(Fn)->isStatic()) {
-        return DeclRefExpr::Create(Context,
-                                   MemExpr->getQualifierLoc(),
-                                   Fn,
-                                   MemExpr->getMemberLoc(),
-                                   Fn->getType(),
-                                   VK_LValue,
-                                   Found.getDecl(),
-                                   TemplateArgs);
+        DeclRefExpr *DRE = DeclRefExpr::Create(Context,
+                                               MemExpr->getQualifierLoc(),
+                                               Fn,
+                                               MemExpr->getMemberLoc(),
+                                               Fn->getType(),
+                                               VK_LValue,
+                                               Found.getDecl(),
+                                               TemplateArgs);
+        DRE->setHadMultipleCandidates(MemExpr->getNumDecls() > 1);
+        return DRE;
       } else {
         SourceLocation Loc = MemExpr->getMemberLoc();
         if (MemExpr->getQualifier())
@@ -9701,14 +10009,16 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
       type = Context.BoundMemberTy;
     }
 
-    return MemberExpr::Create(Context, Base,
-                              MemExpr->isArrow(),
-                              MemExpr->getQualifierLoc(),
-                              Fn,
-                              Found,
-                              MemExpr->getMemberNameInfo(),
-                              TemplateArgs,
-                              type, valueKind, OK_Ordinary);
+    MemberExpr *ME = MemberExpr::Create(Context, Base,
+                                        MemExpr->isArrow(),
+                                        MemExpr->getQualifierLoc(),
+                                        Fn,
+                                        Found,
+                                        MemExpr->getMemberNameInfo(),
+                                        TemplateArgs,
+                                        type, valueKind, OK_Ordinary);
+    ME->setHadMultipleCandidates(true);
+    return ME;
   }
 
   llvm_unreachable("Invalid reference to overloaded function");
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp
index 65f431d46075..5351896204ca 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp
@@ -47,8 +47,8 @@ StmtResult Sema::ActOnExprStmt(FullExprArg expr) {
 
 
 StmtResult Sema::ActOnNullStmt(SourceLocation SemiLoc,
-                               SourceLocation LeadingEmptyMacroLoc) {
-  return Owned(new (Context) NullStmt(SemiLoc, LeadingEmptyMacroLoc));
+                               bool HasLeadingEmptyMacro) {
+  return Owned(new (Context) NullStmt(SemiLoc, HasLeadingEmptyMacro));
 }
 
 StmtResult Sema::ActOnDeclStmt(DeclGroupPtrTy dg, SourceLocation StartLoc,
@@ -92,6 +92,56 @@ void Sema::ActOnForEachDeclStmt(DeclGroupPtrTy dg) {
   }
 }
 
+/// \brief Diagnose unused '==' and '!=' as likely typos for '=' or '|='.
+///
+/// Adding a cast to void (or other expression wrappers) will prevent the
+/// warning from firing.
+static bool DiagnoseUnusedComparison(Sema &S, const Expr *E) {
+  SourceLocation Loc;
+  bool IsNotEqual, CanAssign;
+
+  if (const BinaryOperator *Op = dyn_cast<BinaryOperator>(E)) {
+    if (Op->getOpcode() != BO_EQ && Op->getOpcode() != BO_NE)
+      return false;
+
+    Loc = Op->getOperatorLoc();
+    IsNotEqual = Op->getOpcode() == BO_NE;
+    CanAssign = Op->getLHS()->IgnoreParenImpCasts()->isLValue();
+  } else if (const CXXOperatorCallExpr *Op = dyn_cast<CXXOperatorCallExpr>(E)) {
+    if (Op->getOperator() != OO_EqualEqual &&
+        Op->getOperator() != OO_ExclaimEqual)
+      return false;
+
+    Loc = Op->getOperatorLoc();
+    IsNotEqual = Op->getOperator() == OO_ExclaimEqual;
+    CanAssign = Op->getArg(0)->IgnoreParenImpCasts()->isLValue();
+  } else {
+    // Not a typo-prone comparison.
+    return false;
+  }
+
+  // Suppress warnings when the operator, suspicious as it may be, comes from
+  // a macro expansion.
+  if (Loc.isMacroID())
+    return false;
+
+  S.Diag(Loc, diag::warn_unused_comparison)
+    << (unsigned)IsNotEqual << E->getSourceRange();
+
+  // If the LHS is a plausible entity to assign to, provide a fixit hint to
+  // correct common typos.
+  if (CanAssign) {
+    if (IsNotEqual)
+      S.Diag(Loc, diag::note_inequality_comparison_to_or_assign)
+        << FixItHint::CreateReplacement(Loc, "|=");
+    else
+      S.Diag(Loc, diag::note_equality_comparison_to_assign)
+        << FixItHint::CreateReplacement(Loc, "=");
+  }
+
+  return true;
+}
+
 void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
   if (const LabelStmt *Label = dyn_cast_or_null<LabelStmt>(S))
     return DiagnoseUnusedExprResult(Label->getSubStmt());
@@ -114,6 +164,9 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
   if (const CXXBindTemporaryExpr *TempExpr = dyn_cast<CXXBindTemporaryExpr>(E))
     E = TempExpr->getSubExpr();
 
+  if (DiagnoseUnusedComparison(*this, E))
+    return;
+
   E = E->IgnoreParenImpCasts();
   if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
     if (E->getType()->isVoidType())
@@ -123,7 +176,7 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
     // a more specific message to make it clear what is happening.
     if (const Decl *FD = CE->getCalleeDecl()) {
       if (FD->getAttr<WarnUnusedResultAttr>()) {
-        Diag(Loc, diag::warn_unused_call) << R1 << R2 << "warn_unused_result";
+        Diag(Loc, diag::warn_unused_result) << R1 << R2;
         return;
       }
       if (FD->getAttr<PureAttr>()) {
@@ -142,7 +195,7 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
     }
     const ObjCMethodDecl *MD = ME->getMethodDecl();
     if (MD && MD->getAttr<WarnUnusedResultAttr>()) {
-      Diag(Loc, diag::warn_unused_call) << R1 << R2 << "warn_unused_result";
+      Diag(Loc, diag::warn_unused_result) << R1 << R2;
       return;
     }
   } else if (isa<ObjCPropertyRefExpr>(E)) {
@@ -238,6 +291,8 @@ Sema::ActOnCaseStmt(SourceLocation CaseLoc, Expr *LHSVal,
 
 /// ActOnCaseStmtBody - This installs a statement as the body of a case.
 void Sema::ActOnCaseStmtBody(Stmt *caseStmt, Stmt *SubStmt) {
+  DiagnoseUnusedExprResult(SubStmt);
+
   CaseStmt *CS = static_cast<CaseStmt*>(caseStmt);
   CS->setSubStmt(SubStmt);
 }
@@ -245,6 +300,8 @@ void Sema::ActOnCaseStmtBody(Stmt *caseStmt, Stmt *SubStmt) {
 StmtResult
 Sema::ActOnDefaultStmt(SourceLocation DefaultLoc, SourceLocation ColonLoc,
                        Stmt *SubStmt, Scope *CurScope) {
+  DiagnoseUnusedExprResult(SubStmt);
+
   if (getCurFunction()->SwitchStack.empty()) {
     Diag(DefaultLoc, diag::err_default_not_in_switch);
     return Owned(SubStmt);
@@ -407,13 +464,12 @@ static bool EqEnumVals(const std::pair<llvm::APSInt, EnumConstantDecl*>& lhs,
 
 /// GetTypeBeforeIntegralPromotion - Returns the pre-promotion type of
 /// potentially integral-promoted expression @p expr.
-static QualType GetTypeBeforeIntegralPromotion(const Expr* expr) {
-  if (const CastExpr *ImplicitCast = dyn_cast<ImplicitCastExpr>(expr)) {
-    const Expr *ExprBeforePromotion = ImplicitCast->getSubExpr();
-    QualType TypeBeforePromotion = ExprBeforePromotion->getType();
-    if (TypeBeforePromotion->isIntegralOrEnumerationType()) {
-      return TypeBeforePromotion;
-    }
+static QualType GetTypeBeforeIntegralPromotion(Expr *&expr) {
+  if (ExprWithCleanups *cleanups = dyn_cast<ExprWithCleanups>(expr))
+    expr = cleanups->getSubExpr();
+  while (ImplicitCastExpr *impcast = dyn_cast<ImplicitCastExpr>(expr)) {
+    if (impcast->getCastKind() != CK_IntegralCast) break;
+    expr = impcast->getSubExpr();
   }
   return expr->getType();
 }
@@ -449,6 +505,11 @@ Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, Expr *Cond,
   if (CondResult.isInvalid()) return StmtError();
   Cond = CondResult.take();
 
+  // C99 6.8.4.2p5 - Integer promotions are performed on the controlling expr.
+  CondResult = UsualUnaryConversions(Cond);
+  if (CondResult.isInvalid()) return StmtError();
+  Cond = CondResult.take();
+
   if (!CondVar) {
     CheckImplicitConversions(Cond, SwitchLoc);
     CondResult = MaybeCreateExprWithCleanups(Cond);
@@ -482,21 +543,14 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
   SS->setBody(BodyStmt, SwitchLoc);
   getCurFunction()->SwitchStack.pop_back();
 
-  if (SS->getCond() == 0)
-    return StmtError();
-
   Expr *CondExpr = SS->getCond();
+  if (!CondExpr) return StmtError();
+
+  QualType CondType = CondExpr->getType();
+
   Expr *CondExprBeforePromotion = CondExpr;
   QualType CondTypeBeforePromotion =
-      GetTypeBeforeIntegralPromotion(CondExpr);
-
-  // C99 6.8.4.2p5 - Integer promotions are performed on the controlling expr.
-  ExprResult CondResult = UsualUnaryConversions(CondExpr);
-  if (CondResult.isInvalid())
-    return StmtError();
-  CondExpr = CondResult.take();
-  QualType CondType = CondExpr->getType();
-  SS->setCond(CondExpr);
+      GetTypeBeforeIntegralPromotion(CondExprBeforePromotion);
 
   // C++ 6.4.2.p2:
   // Integral promotions are performed (on the switch condition).
@@ -533,7 +587,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
   // Accumulate all of the case values in a vector so that we can sort them
   // and detect duplicates.  This vector contains the APInt for the case after
   // it has been converted to the condition type.
-  typedef llvm::SmallVector<std::pair<llvm::APSInt, CaseStmt*>, 64> CaseValsTy;
+  typedef SmallVector<std::pair<llvm::APSInt, CaseStmt*>, 64> CaseValsTy;
   CaseValsTy CaseVals;
 
   // Keep track of any GNU case ranges we see.  The APSInt is the low value.
@@ -572,7 +626,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
         break;
       }
 
-      llvm::APSInt LoVal = Lo->EvaluateAsInt(Context);
+      llvm::APSInt LoVal = Lo->EvaluateKnownConstInt(Context);
 
       // Convert the value to the same width/sign as the condition.
       ConvertIntegerToTypeWarnOnOverflow(LoVal, CondWidth, CondIsSigned,
@@ -651,7 +705,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
         llvm::APSInt &LoVal = CaseRanges[i].first;
         CaseStmt *CR = CaseRanges[i].second;
         Expr *Hi = CR->getRHS();
-        llvm::APSInt HiVal = Hi->EvaluateAsInt(Context);
+        llvm::APSInt HiVal = Hi->EvaluateKnownConstInt(Context);
 
         // Convert the value to the same width/sign as the condition.
         ConvertIntegerToTypeWarnOnOverflow(HiVal, CondWidth, CondIsSigned,
@@ -750,7 +804,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
     // If switch has default case, then ignore it.
     if (!CaseListIsErroneous  && !HasConstantCond && ET) {
       const EnumDecl *ED = ET->getDecl();
-      typedef llvm::SmallVector<std::pair<llvm::APSInt, EnumConstantDecl*>, 64>
+      typedef SmallVector<std::pair<llvm::APSInt, EnumConstantDecl*>, 64>
         EnumValsTy;
       EnumValsTy EnumVals;
 
@@ -791,7 +845,8 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
               << ED->getDeclName();
           }
 
-          llvm::APSInt Hi = RI->second->getRHS()->EvaluateAsInt(Context);
+          llvm::APSInt Hi = 
+            RI->second->getRHS()->EvaluateKnownConstInt(Context);
           AdjustAPSInt(Hi, CondWidth, CondIsSigned);
           while (EI != EIend && EI->first < Hi)
             EI++;
@@ -806,7 +861,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
       CaseRangesTy::const_iterator RI = CaseRanges.begin();
       bool hasCasesNotInSwitch = false;
 
-      llvm::SmallVector<DeclarationName,8> UnhandledNames;
+      SmallVector<DeclarationName,8> UnhandledNames;
 
       for (EnumValsTy::const_iterator EI = EnumVals.begin(); EI != EIend; EI++){
         // Drop unneeded case values
@@ -819,7 +874,8 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
 
         // Drop unneeded case ranges
         for (; RI != CaseRanges.end(); RI++) {
-          llvm::APSInt Hi = RI->second->getRHS()->EvaluateAsInt(Context);
+          llvm::APSInt Hi =
+            RI->second->getRHS()->EvaluateKnownConstInt(Context);
           AdjustAPSInt(Hi, CondWidth, CondIsSigned);
           if (EI->first <= Hi)
             break;
@@ -965,6 +1021,76 @@ StmtResult Sema::ActOnForEachLValueExpr(Expr *E) {
   return Owned(static_cast<Stmt*>(Result.get()));
 }
 
+ExprResult
+Sema::ActOnObjCForCollectionOperand(SourceLocation forLoc, Expr *collection) {
+  assert(collection);
+
+  // Bail out early if we've got a type-dependent expression.
+  if (collection->isTypeDependent()) return Owned(collection);
+
+  // Perform normal l-value conversion.
+  ExprResult result = DefaultFunctionArrayLvalueConversion(collection);
+  if (result.isInvalid())
+    return ExprError();
+  collection = result.take();
+
+  // The operand needs to have object-pointer type.
+  // TODO: should we do a contextual conversion?
+  const ObjCObjectPointerType *pointerType =
+    collection->getType()->getAs<ObjCObjectPointerType>();
+  if (!pointerType)
+    return Diag(forLoc, diag::err_collection_expr_type)
+             << collection->getType() << collection->getSourceRange();
+
+  // Check that the operand provides
+  //   - countByEnumeratingWithState:objects:count:
+  const ObjCObjectType *objectType = pointerType->getObjectType();
+  ObjCInterfaceDecl *iface = objectType->getInterface();
+
+  // If we have a forward-declared type, we can't do this check.
+  if (iface && iface->isForwardDecl()) {
+    // This is ill-formed under ARC.
+    if (getLangOptions().ObjCAutoRefCount) {
+      Diag(forLoc, diag::err_arc_collection_forward)
+        << pointerType->getPointeeType() << collection->getSourceRange();
+    }
+
+    // Otherwise, if we have any useful type information, check that
+    // the type declares the appropriate method.
+  } else if (iface || !objectType->qual_empty()) {
+    IdentifierInfo *selectorIdents[] = {
+      &Context.Idents.get("countByEnumeratingWithState"),
+      &Context.Idents.get("objects"),
+      &Context.Idents.get("count")
+    };
+    Selector selector = Context.Selectors.getSelector(3, &selectorIdents[0]);
+
+    ObjCMethodDecl *method = 0;
+
+    // If there's an interface, look in both the public and private APIs.
+    if (iface) {
+      method = iface->lookupInstanceMethod(selector);
+      if (!method) method = LookupPrivateInstanceMethod(selector, iface);
+    }
+
+    // Also check protocol qualifiers.
+    if (!method)
+      method = LookupMethodInQualifiedType(selector, pointerType,
+                                           /*instance*/ true);
+
+    // If we didn't find it anywhere, give up.
+    if (!method) {
+      Diag(forLoc, diag::warn_collection_expr_type)
+        << collection->getType() << selector << collection->getSourceRange();
+    }
+
+    // TODO: check for an incompatible signature?
+  }
+
+  // Wrap up any cleanups in the expression.
+  return Owned(MaybeCreateExprWithCleanups(collection));
+}
+
 StmtResult
 Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
                                  SourceLocation LParenLoc,
@@ -1000,38 +1126,7 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
         Diag(ForLoc, diag::err_selector_element_type)
           << FirstType << First->getSourceRange();
   }
-  if (Second && !Second->isTypeDependent()) {
-    ExprResult Result = DefaultFunctionArrayLvalueConversion(Second);
-    if (Result.isInvalid())
-      return StmtError();
-    Second = Result.take();
-    QualType SecondType = Second->getType();
-    if (!SecondType->isObjCObjectPointerType())
-      Diag(ForLoc, diag::err_collection_expr_type)
-        << SecondType << Second->getSourceRange();
-    else if (const ObjCObjectPointerType *OPT =
-             SecondType->getAsObjCInterfacePointerType()) {
-      llvm::SmallVector<IdentifierInfo *, 4> KeyIdents;
-      IdentifierInfo* selIdent =
-        &Context.Idents.get("countByEnumeratingWithState");
-      KeyIdents.push_back(selIdent);
-      selIdent = &Context.Idents.get("objects");
-      KeyIdents.push_back(selIdent);
-      selIdent = &Context.Idents.get("count");
-      KeyIdents.push_back(selIdent);
-      Selector CSelector = Context.Selectors.getSelector(3, &KeyIdents[0]);
-      if (ObjCInterfaceDecl *IDecl = OPT->getInterfaceDecl()) {
-        if (!IDecl->isForwardDecl() &&
-            !IDecl->lookupInstanceMethod(CSelector) &&
-            !LookupMethodInQualifiedType(CSelector, OPT, true)) {
-          // Must further look into private implementation methods.
-          if (!LookupPrivateInstanceMethod(CSelector, IDecl))
-            Diag(ForLoc, diag::warn_collection_expr_type)
-              << SecondType << CSelector << Second->getSourceRange();
-        }
-      }
-    }
-  }
+
   return Owned(new (Context) ObjCForCollectionStmt(First, Second, Body,
                                                    ForLoc, RParenLoc));
 }
@@ -1244,10 +1339,16 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
   if (!BeginEndDecl.get() && !RangeVarType->isDependentType()) {
     SourceLocation RangeLoc = RangeVar->getLocation();
 
-    ExprResult RangeRef = BuildDeclRefExpr(RangeVar,
-                                           RangeVarType.getNonReferenceType(),
-                                           VK_LValue, ColonLoc);
-    if (RangeRef.isInvalid())
+    const QualType RangeVarNonRefType = RangeVarType.getNonReferenceType();
+
+    ExprResult BeginRangeRef = BuildDeclRefExpr(RangeVar, RangeVarNonRefType,
+                                                VK_LValue, ColonLoc);
+    if (BeginRangeRef.isInvalid())
+      return StmtError();
+
+    ExprResult EndRangeRef = BuildDeclRefExpr(RangeVar, RangeVarNonRefType,
+                                              VK_LValue, ColonLoc);
+    if (EndRangeRef.isInvalid())
       return StmtError();
 
     QualType AutoType = Context.getAutoDeductType();
@@ -1275,8 +1376,8 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
       //   the program is ill-formed;
 
       // begin-expr is __range.
-      BeginExpr = RangeRef;
-      if (FinishForRangeVarDecl(*this, BeginVar, RangeRef.get(), ColonLoc,
+      BeginExpr = BeginRangeRef;
+      if (FinishForRangeVarDecl(*this, BeginVar, BeginRangeRef.get(), ColonLoc,
                                 diag::err_for_range_iter_deduction_failure)) {
         NoteForRangeBeginEndFunction(*this, BeginExpr.get(), BEF_begin);
         return StmtError();
@@ -1294,12 +1395,11 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
       else {
         // Can't be a DependentSizedArrayType or an IncompleteArrayType since
         // UnqAT is not incomplete and Range is not type-dependent.
-        assert(0 && "Unexpected array type in for-range");
-        return StmtError();
+        llvm_unreachable("Unexpected array type in for-range");
       }
 
       // end-expr is __range + __bound.
-      EndExpr = ActOnBinOp(S, ColonLoc, tok::plus, RangeRef.get(),
+      EndExpr = ActOnBinOp(S, ColonLoc, tok::plus, EndRangeRef.get(),
                            BoundExpr.get());
       if (EndExpr.isInvalid())
         return StmtError();
@@ -1340,13 +1440,14 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
 
       BeginExpr = BuildForRangeBeginEndCall(*this, S, ColonLoc, BeginVar,
                                             BEF_begin, BeginNameInfo,
-                                            BeginMemberLookup, RangeRef.get());
+                                            BeginMemberLookup,
+                                            BeginRangeRef.get());
       if (BeginExpr.isInvalid())
         return StmtError();
 
       EndExpr = BuildForRangeBeginEndCall(*this, S, ColonLoc, EndVar,
                                           BEF_end, EndNameInfo,
-                                          EndMemberLookup, RangeRef.get());
+                                          EndMemberLookup, EndRangeRef.get());
       if (EndExpr.isInvalid())
         return StmtError();
     }
@@ -1366,11 +1467,16 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
       BuildDeclaratorGroup(BeginEndDecls, 2, /*TypeMayContainAuto=*/false);
     BeginEndDecl = ActOnDeclStmt(BeginEndGroup, ColonLoc, ColonLoc);
 
-    ExprResult BeginRef = BuildDeclRefExpr(BeginVar,
-                                           BeginType.getNonReferenceType(),
+    const QualType BeginRefNonRefType = BeginType.getNonReferenceType();
+    ExprResult BeginRef = BuildDeclRefExpr(BeginVar, BeginRefNonRefType,
                                            VK_LValue, ColonLoc);
+    if (BeginRef.isInvalid())
+      return StmtError();
+
     ExprResult EndRef = BuildDeclRefExpr(EndVar, EndType.getNonReferenceType(),
                                          VK_LValue, ColonLoc);
+    if (EndRef.isInvalid())
+      return StmtError();
 
     // Build and check __begin != __end expression.
     NotEqExpr = ActOnBinOp(S, ColonLoc, tok::exclaimequal,
@@ -1385,6 +1491,11 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
     }
 
     // Build and check ++__begin expression.
+    BeginRef = BuildDeclRefExpr(BeginVar, BeginRefNonRefType,
+                                VK_LValue, ColonLoc);
+    if (BeginRef.isInvalid())
+      return StmtError();
+
     IncrExpr = ActOnUnaryOp(S, ColonLoc, tok::plusplus, BeginRef.get());
     IncrExpr = ActOnFinishFullExpr(IncrExpr.get());
     if (IncrExpr.isInvalid()) {
@@ -1393,6 +1504,11 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
     }
 
     // Build and check *__begin  expression.
+    BeginRef = BuildDeclRefExpr(BeginVar, BeginRefNonRefType,
+                                VK_LValue, ColonLoc);
+    if (BeginRef.isInvalid())
+      return StmtError();
+
     ExprResult DerefExpr = ActOnUnaryOp(S, ColonLoc, tok::star, BeginRef.get());
     if (DerefExpr.isInvalid()) {
       NoteForRangeBeginEndFunction(*this, BeginExpr.get(), BEF_begin);
@@ -1651,55 +1767,50 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   // Otherwise, verify that this result type matches the previous one.  We are
   // pickier with blocks than for normal functions because we don't have GCC
   // compatibility to worry about here.
-  ReturnStmt *Result = 0;
-  if (CurBlock->ReturnType->isVoidType()) {
-    if (RetValExp && !RetValExp->isTypeDependent() &&
-        (!getLangOptions().CPlusPlus || !RetValExp->getType()->isVoidType())) {
+  const VarDecl *NRVOCandidate = 0;
+  if (FnRetType->isDependentType()) {
+    // Delay processing for now.  TODO: there are lots of dependent
+    // types we can conclusively prove aren't void.
+  } else if (FnRetType->isVoidType()) {
+    if (RetValExp &&
+        !(getLangOptions().CPlusPlus &&
+          (RetValExp->isTypeDependent() ||
+           RetValExp->getType()->isVoidType()))) {
       Diag(ReturnLoc, diag::err_return_block_has_expr);
       RetValExp = 0;
     }
-    Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, 0);
   } else if (!RetValExp) {
-    if (!CurBlock->ReturnType->isDependentType())
-      return StmtError(Diag(ReturnLoc, diag::err_block_return_missing_expr));
+    return StmtError(Diag(ReturnLoc, diag::err_block_return_missing_expr));
+  } else if (!RetValExp->isTypeDependent()) {
+    // we have a non-void block with an expression, continue checking
 
-    Result = new (Context) ReturnStmt(ReturnLoc, 0, 0);
-  } else {
-    const VarDecl *NRVOCandidate = 0;
+    // C99 6.8.6.4p3(136): The return statement is not an assignment. The
+    // overlap restriction of subclause 6.5.16.1 does not apply to the case of
+    // function return.
 
-    if (!FnRetType->isDependentType() && !RetValExp->isTypeDependent()) {
-      // we have a non-void block with an expression, continue checking
-
-      // C99 6.8.6.4p3(136): The return statement is not an assignment. The
-      // overlap restriction of subclause 6.5.16.1 does not apply to the case of
-      // function return.
-
-      // In C++ the return statement is handled via a copy initialization.
-      // the C version of which boils down to CheckSingleAssignmentConstraints.
-      NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
-      InitializedEntity Entity = InitializedEntity::InitializeResult(ReturnLoc,
-                                                                     FnRetType,
+    // In C++ the return statement is handled via a copy initialization.
+    // the C version of which boils down to CheckSingleAssignmentConstraints.
+    NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
+    InitializedEntity Entity = InitializedEntity::InitializeResult(ReturnLoc,
+                                                                   FnRetType,
                                                            NRVOCandidate != 0);
-      ExprResult Res = PerformMoveOrCopyInitialization(Entity, NRVOCandidate,
-                                                       FnRetType, RetValExp);
-      if (Res.isInvalid()) {
-        // FIXME: Cleanup temporaries here, anyway?
-        return StmtError();
-      }
-
-      if (RetValExp) {
-        CheckImplicitConversions(RetValExp, ReturnLoc);
-        RetValExp = MaybeCreateExprWithCleanups(RetValExp);
-      }
-
-      RetValExp = Res.takeAs<Expr>();
-      if (RetValExp)
-        CheckReturnStackAddr(RetValExp, FnRetType, ReturnLoc);
+    ExprResult Res = PerformMoveOrCopyInitialization(Entity, NRVOCandidate,
+                                                     FnRetType, RetValExp);
+    if (Res.isInvalid()) {
+      // FIXME: Cleanup temporaries here, anyway?
+      return StmtError();
     }
-
-    Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, NRVOCandidate);
+    RetValExp = Res.take();
+    CheckReturnStackAddr(RetValExp, FnRetType, ReturnLoc);
   }
 
+  if (RetValExp) {
+    CheckImplicitConversions(RetValExp, ReturnLoc);
+    RetValExp = MaybeCreateExprWithCleanups(RetValExp);
+  }
+  ReturnStmt *Result = new (Context) ReturnStmt(ReturnLoc, RetValExp,
+                                                NRVOCandidate);
+
   // If we need to check for the named return value optimization, save the
   // return statement in our scope for later processing.
   if (getLangOptions().CPlusPlus && FnRetType->isRecordType() && 
@@ -1883,7 +1994,7 @@ static bool CheckAsmLValue(const Expr *E, Sema &S) {
 /// isOperandMentioned - Return true if the specified operand # is mentioned
 /// anywhere in the decomposed asm string.
 static bool isOperandMentioned(unsigned OpNo, 
-                         llvm::ArrayRef<AsmStmt::AsmStringPiece> AsmStrPieces) {
+                         ArrayRef<AsmStmt::AsmStringPiece> AsmStrPieces) {
   for (unsigned p = 0, e = AsmStrPieces.size(); p != e; ++p) {
     const AsmStmt::AsmStringPiece &Piece = AsmStrPieces[p];
     if (!Piece.isOperand()) continue;
@@ -1910,25 +2021,25 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
   StringLiteral *AsmString = cast<StringLiteral>(asmString);
   StringLiteral **Clobbers = reinterpret_cast<StringLiteral**>(clobbers.get());
 
-  llvm::SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
+  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
 
   // The parser verifies that there is a string literal here.
-  if (AsmString->isWide())
+  if (!AsmString->isAscii())
     return StmtError(Diag(AsmString->getLocStart(),diag::err_asm_wide_character)
       << AsmString->getSourceRange());
 
   for (unsigned i = 0; i != NumOutputs; i++) {
     StringLiteral *Literal = Constraints[i];
-    if (Literal->isWide())
+    if (!Literal->isAscii())
       return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
         << Literal->getSourceRange());
 
-    llvm::StringRef OutputName;
+    StringRef OutputName;
     if (Names[i])
       OutputName = Names[i]->getName();
 
     TargetInfo::ConstraintInfo Info(Literal->getString(), OutputName);
-    if (!Context.Target.validateOutputConstraint(Info))
+    if (!Context.getTargetInfo().validateOutputConstraint(Info))
       return StmtError(Diag(Literal->getLocStart(),
                             diag::err_asm_invalid_output_constraint)
                        << Info.getConstraintStr());
@@ -1944,20 +2055,20 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
     OutputConstraintInfos.push_back(Info);
   }
 
-  llvm::SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
+  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
 
   for (unsigned i = NumOutputs, e = NumOutputs + NumInputs; i != e; i++) {
     StringLiteral *Literal = Constraints[i];
-    if (Literal->isWide())
+    if (!Literal->isAscii())
       return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
         << Literal->getSourceRange());
 
-    llvm::StringRef InputName;
+    StringRef InputName;
     if (Names[i])
       InputName = Names[i]->getName();
 
     TargetInfo::ConstraintInfo Info(Literal->getString(), InputName);
-    if (!Context.Target.validateInputConstraint(OutputConstraintInfos.data(),
+    if (!Context.getTargetInfo().validateInputConstraint(OutputConstraintInfos.data(),
                                                 NumOutputs, Info)) {
       return StmtError(Diag(Literal->getLocStart(),
                             diag::err_asm_invalid_input_constraint)
@@ -1995,13 +2106,13 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
   // Check that the clobbers are valid.
   for (unsigned i = 0; i != NumClobbers; i++) {
     StringLiteral *Literal = Clobbers[i];
-    if (Literal->isWide())
+    if (!Literal->isAscii())
       return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
         << Literal->getSourceRange());
 
-    llvm::StringRef Clobber = Literal->getString();
+    StringRef Clobber = Literal->getString();
 
-    if (!Context.Target.isValidClobber(Clobber))
+    if (!Context.getTargetInfo().isValidClobber(Clobber))
       return StmtError(Diag(Literal->getLocStart(),
                   diag::err_asm_unknown_register_name) << Clobber);
   }
@@ -2012,7 +2123,7 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
                           AsmString, NumClobbers, Clobbers, RParenLoc);
   // Validate the asm string, ensuring it makes sense given the operands we
   // have.
-  llvm::SmallVector<AsmStmt::AsmStringPiece, 8> Pieces;
+  SmallVector<AsmStmt::AsmStringPiece, 8> Pieces;
   unsigned DiagOffs;
   if (unsigned DiagID = NS->AnalyzeAsmString(Pieces, Context, DiagOffs)) {
     Diag(getLocationOfStringLiteralByte(AsmString, DiagOffs), DiagID)
@@ -2033,6 +2144,10 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
     unsigned InputOpNo = i+NumOutputs;
     Expr *OutputExpr = Exprs[TiedTo];
     Expr *InputExpr = Exprs[InputOpNo];
+
+    if (OutputExpr->isTypeDependent() || InputExpr->isTypeDependent())
+      continue;
+
     QualType InTy = InputExpr->getType();
     QualType OutTy = OutputExpr->getType();
     if (Context.hasSameType(InTy, OutTy))
@@ -2154,6 +2269,7 @@ Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
 StmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc,
                                                   Expr *Throw) {
   if (Throw) {
+    Throw = MaybeCreateExprWithCleanups(Throw);
     ExprResult Result = DefaultLvalueConversion(Throw);
     if (Result.isInvalid())
       return StmtError();
@@ -2188,29 +2304,36 @@ Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
     if (!AtCatchParent)
       return StmtError(Diag(AtLoc, diag::error_rethrow_used_outside_catch));
   }
-
+  
   return BuildObjCAtThrowStmt(AtLoc, Throw);
 }
 
+ExprResult
+Sema::ActOnObjCAtSynchronizedOperand(SourceLocation atLoc, Expr *operand) {
+  ExprResult result = DefaultLvalueConversion(operand);
+  if (result.isInvalid())
+    return ExprError();
+  operand = result.take();
+
+  // Make sure the expression type is an ObjC pointer or "void *".
+  QualType type = operand->getType();
+  if (!type->isDependentType() &&
+      !type->isObjCObjectPointerType()) {
+    const PointerType *pointerType = type->getAs<PointerType>();
+    if (!pointerType || !pointerType->getPointeeType()->isVoidType())
+      return Diag(atLoc, diag::error_objc_synchronized_expects_object)
+               << type << operand->getSourceRange();
+  }
+
+  // The operand to @synchronized is a full-expression.
+  return MaybeCreateExprWithCleanups(operand);
+}
+
 StmtResult
 Sema::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SyncExpr,
                                   Stmt *SyncBody) {
+  // We can't jump into or indirect-jump out of a @synchronized block.
   getCurFunction()->setHasBranchProtectedScope();
-
-  ExprResult Result = DefaultLvalueConversion(SyncExpr);
-  if (Result.isInvalid())
-    return StmtError();
-
-  SyncExpr = Result.take();
-  // Make sure the expression type is an ObjC pointer or "void *".
-  if (!SyncExpr->getType()->isDependentType() &&
-      !SyncExpr->getType()->isObjCObjectPointerType()) {
-    const PointerType *PT = SyncExpr->getType()->getAs<PointerType>();
-    if (!PT || !PT->getPointeeType()->isVoidType())
-      return StmtError(Diag(AtLoc, diag::error_objc_synchronized_expects_object)
-                       << SyncExpr->getType() << SyncExpr->getSourceRange());
-  }
-
   return Owned(new (Context) ObjCAtSynchronizedStmt(AtLoc, SyncExpr, SyncBody));
 }
 
@@ -2278,10 +2401,10 @@ Sema::ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
          "The parser shouldn't call this if there are no handlers.");
   Stmt **Handlers = RawHandlers.get();
 
-  llvm::SmallVector<TypeWithHandler, 8> TypesWithHandlers;
+  SmallVector<TypeWithHandler, 8> TypesWithHandlers;
 
   for (unsigned i = 0; i < NumHandlers; ++i) {
-    CXXCatchStmt *Handler = llvm::cast<CXXCatchStmt>(Handlers[i]);
+    CXXCatchStmt *Handler = cast<CXXCatchStmt>(Handlers[i]);
     if (!Handler->getExceptionDecl()) {
       if (i < NumHandlers - 1)
         return StmtError(Diag(Handler->getLocStart(),
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp
index 3ac190e1d4c8..8dda34c8ab5d 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp
@@ -344,10 +344,12 @@ void Sema::LookupTemplateName(LookupResult &Found,
     if (FoundOuter.empty()) {
       //   - if the name is not found, the name found in the class of the
       //     object expression is used, otherwise
-    } else if (!FoundOuter.getAsSingle<ClassTemplateDecl>()) {
+    } else if (!FoundOuter.getAsSingle<ClassTemplateDecl>() ||
+               FoundOuter.isAmbiguous()) {
       //   - if the name is found in the context of the entire
       //     postfix-expression and does not name a class template, the name
       //     found in the class of the object expression is used, otherwise
+      FoundOuter.clear();
     } else if (!Found.isSuppressingDiagnostics()) {
       //   - if the name found is a class template, it must refer to the same
       //     entity as the one found in the class of the object expression,
@@ -422,7 +424,7 @@ bool Sema::DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl) {
   assert(PrevDecl->isTemplateParameter() && "Not a template parameter");
 
   // Microsoft Visual C++ permits template parameters to be shadowed.
-  if (getLangOptions().Microsoft)
+  if (getLangOptions().MicrosoftExt)
     return false;
 
   // C++ [temp.local]p4:
@@ -712,7 +714,7 @@ Decl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
 /// has been parsed. S is the current scope.
 Decl *Sema::ActOnTemplateTemplateParameter(Scope* S,
                                            SourceLocation TmpLoc,
-                                           TemplateParamsTy *Params,
+                                           TemplateParameterList *Params,
                                            SourceLocation EllipsisLoc,
                                            IdentifierInfo *Name,
                                            SourceLocation NameLoc,
@@ -783,7 +785,7 @@ Decl *Sema::ActOnTemplateTemplateParameter(Scope* S,
 
 /// ActOnTemplateParameterList - Builds a TemplateParameterList that
 /// contains the template parameters in Params/NumParams.
-Sema::TemplateParamsTy *
+TemplateParameterList *
 Sema::ActOnTemplateParameterList(unsigned Depth,
                                  SourceLocation ExportLoc,
                                  SourceLocation TemplateLoc,
@@ -809,7 +811,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
                          IdentifierInfo *Name, SourceLocation NameLoc,
                          AttributeList *Attr,
                          TemplateParameterList *TemplateParams,
-                         AccessSpecifier AS,
+                         AccessSpecifier AS, SourceLocation ModulePrivateLoc,
                          unsigned NumOuterTemplateParamLists,
                          TemplateParameterList** OuterTemplateParamLists) {
   assert(TemplateParams && TemplateParams->size() > 0 &&
@@ -844,6 +846,15 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
     if (RequireCompleteDeclContext(SS, SemanticContext))
       return true;
 
+    // If we're adding a template to a dependent context, we may need to 
+    // rebuilding some of the types used within the template parameter list, 
+    // now that we know what the current instantiation is.
+    if (SemanticContext->isDependentContext()) {
+      ContextRAII SavedContext(*this, SemanticContext);
+      if (RebuildTemplateParamsInCurrentInstantiation(TemplateParams))
+        Invalid = true;
+    }
+        
     LookupQualifiedName(Previous, SemanticContext);
   } else {
     SemanticContext = CurContext;
@@ -943,7 +954,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
         // definition, as part of error recovery?
         return true;
       }
-    }
+    }    
   } else if (PrevDecl && PrevDecl->isTemplateParameter()) {
     // Maybe we will complain about the shadowed template parameter.
     DiagnoseTemplateParameterShadow(NameLoc, PrevDecl);
@@ -997,7 +1008,17 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
                                 DeclarationName(Name), TemplateParams,
                                 NewClass, PrevClassTemplate);
   NewClass->setDescribedClassTemplate(NewTemplate);
-
+  
+  if (PrevClassTemplate && PrevClassTemplate->isModulePrivate()) {
+    NewTemplate->setModulePrivate();
+  } else if (ModulePrivateLoc.isValid()) {
+    if (PrevClassTemplate && !PrevClassTemplate->isModulePrivate())
+      diagnoseModulePrivateRedeclaration(NewTemplate, PrevClassTemplate,
+                                         ModulePrivateLoc);
+    else
+      NewTemplate->setModulePrivate();
+  }
+  
   // Build the type for the class template declaration now.
   QualType T = NewTemplate->getInjectedClassNameSpecialization();
   T = Context.getInjectedClassNameType(NewClass, T);
@@ -1519,7 +1540,7 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
   // The sequence of nested types to which we will match up the template
   // parameter lists. We first build this list by starting with the type named
   // by the nested-name-specifier and walking out until we run out of types.
-  llvm::SmallVector<QualType, 4> NestedTypes;
+  SmallVector<QualType, 4> NestedTypes;
   QualType T;
   if (SS.getScopeRep()) {
     if (CXXRecordDecl *Record 
@@ -1888,7 +1909,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
 
   // Check that the template argument list is well-formed for this
   // template.
-  llvm::SmallVector<TemplateArgument, 4> Converted;
+  SmallVector<TemplateArgument, 4> Converted;
   if (CheckTemplateArgumentList(Template, TemplateLoc, TemplateArgs,
                                 false, Converted))
     return QualType();
@@ -1995,7 +2016,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
       Decl = ClassTemplateSpecializationDecl::Create(Context,
                             ClassTemplate->getTemplatedDecl()->getTagKind(),
                                                 ClassTemplate->getDeclContext(),
-                                                ClassTemplate->getLocation(),
+                            ClassTemplate->getTemplatedDecl()->getLocStart(),
                                                 ClassTemplate->getLocation(),
                                                      ClassTemplate,
                                                      Converted.data(),
@@ -2129,7 +2150,7 @@ TypeResult Sema::ActOnTagTemplateIdType(TagUseKind TUK,
   
   QualType Result = CheckTemplateIdType(Template, TemplateLoc, TemplateArgs);
   if (Result.isNull())
-    return TypeResult();
+    return TypeResult(true);
   
   // Check the tag kind
   if (const RecordType *RT = Result->getAs<RecordType>()) {
@@ -2311,7 +2332,8 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
     return TNK_Dependent_template_name;
 
   case UnqualifiedId::IK_LiteralOperatorId:
-    assert(false && "We don't support these; Parse shouldn't have allowed propagation");
+    llvm_unreachable(
+            "We don't support these; Parse shouldn't have allowed propagation");
 
   default:
     break;
@@ -2327,7 +2349,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
 
 bool Sema::CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
                                      const TemplateArgumentLoc &AL,
-                          llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+                          SmallVectorImpl<TemplateArgument> &Converted) {
   const TemplateArgument &Arg = AL.getArgument();
 
   // Check template type parameter.
@@ -2408,7 +2430,7 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              TemplateTypeParmDecl *Param,
-                         llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+                         SmallVectorImpl<TemplateArgument> &Converted) {
   TypeSourceInfo *ArgType = Param->getDefaultArgumentInfo();
 
   // If the argument type is dependent, instantiate it now based
@@ -2461,7 +2483,7 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              NonTypeTemplateParmDecl *Param,
-                        llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+                        SmallVectorImpl<TemplateArgument> &Converted) {
   TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
                                     Converted.data(), Converted.size());
 
@@ -2507,7 +2529,7 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              TemplateTemplateParmDecl *Param,
-                       llvm::SmallVectorImpl<TemplateArgument> &Converted,
+                       SmallVectorImpl<TemplateArgument> &Converted,
                              NestedNameSpecifierLoc &QualifierLoc) {
   TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
                                     Converted.data(), Converted.size());
@@ -2543,7 +2565,7 @@ Sema::SubstDefaultTemplateArgumentIfAvailable(TemplateDecl *Template,
                                               SourceLocation TemplateLoc,
                                               SourceLocation RAngleLoc,
                                               Decl *Param,
-                      llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+                      SmallVectorImpl<TemplateArgument> &Converted) {
    if (TemplateTypeParmDecl *TypeParm = dyn_cast<TemplateTypeParmDecl>(Param)) {
     if (!TypeParm->hasDefaultArgument())
       return TemplateArgumentLoc();
@@ -2629,7 +2651,7 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
                                  SourceLocation TemplateLoc,
                                  SourceLocation RAngleLoc,
                                  unsigned ArgumentPackIndex,
-                            llvm::SmallVectorImpl<TemplateArgument> &Converted,
+                            SmallVectorImpl<TemplateArgument> &Converted,
                                  CheckTemplateArgumentKind CTAK) {
   // Check template type parameters.
   if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
@@ -2669,8 +2691,7 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
 
     switch (Arg.getArgument().getKind()) {
     case TemplateArgument::Null:
-      assert(false && "Should never see a NULL template argument here");
-      return true;
+      llvm_unreachable("Should never see a NULL template argument here");
 
     case TemplateArgument::Expression: {
       TemplateArgument Result;
@@ -2792,8 +2813,7 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
 
   switch (Arg.getArgument().getKind()) {
   case TemplateArgument::Null:
-    assert(false && "Should never see a NULL template argument here");
-    return true;
+    llvm_unreachable("Should never see a NULL template argument here");
 
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
@@ -2834,7 +2854,7 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
                                      SourceLocation TemplateLoc,
                                      TemplateArgumentListInfo &TemplateArgs,
                                      bool PartialTemplateArgs,
-                          llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+                          SmallVectorImpl<TemplateArgument> &Converted) {
   TemplateParameterList *Params = Template->getTemplateParameters();
   unsigned NumParams = Params->size();
   unsigned NumArgs = TemplateArgs.size();
@@ -2871,7 +2891,7 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
   //   corresponding parameter declared by the template in its
   //   template-parameter-list.
   bool isTemplateTemplateParameter = isa<TemplateTemplateParmDecl>(Template);
-  llvm::SmallVector<TemplateArgument, 2> ArgumentPack;
+  SmallVector<TemplateArgument, 2> ArgumentPack;
   TemplateParameterList::iterator Param = Params->begin(),
                                ParamEnd = Params->end();
   unsigned ArgIdx = 0;
@@ -3235,6 +3255,10 @@ bool UnnamedLocalNoLinkageFinder::VisitObjCObjectPointerType(
   return false;
 }
 
+bool UnnamedLocalNoLinkageFinder::VisitAtomicType(const AtomicType* T) {
+  return Visit(T->getValueType());
+}
+
 bool UnnamedLocalNoLinkageFinder::VisitTagDecl(const TagDecl *Tag) {
   if (Tag->getDeclContext()->isFunctionOrMethod()) {
     S.Diag(SR.getBegin(), diag::ext_template_arg_local_type)
@@ -3358,7 +3382,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
     }
   }
 
-  if (S.getLangOptions().Microsoft && isa<CXXUuidofExpr>(Arg)) {
+  if (S.getLangOptions().MicrosoftExt && isa<CXXUuidofExpr>(Arg)) {
     Converted = TemplateArgument(ArgIn);
     return false;
   }
@@ -3820,8 +3844,9 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     }
 
     Converted = TemplateArgument(Value,
-                                 ParamType->isEnumeralType() ? ParamType
-                                                             : IntegerType);
+                                 ParamType->isEnumeralType() 
+                                   ? Context.getCanonicalType(ParamType)
+                                   : IntegerType);
     return Owned(Arg);
   }
 
@@ -3892,7 +3917,8 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     bool ObjCLifetimeConversion;
     if (IsQualificationConversion(ArgType, ParamType.getNonReferenceType(),
                                   false, ObjCLifetimeConversion)) {
-      Arg = ImpCastExprToType(Arg, ParamType, CK_NoOp, CastCategory(Arg)).take();
+      Arg = ImpCastExprToType(Arg, ParamType, CK_NoOp,
+                              Arg->getValueKind()).take();
     } else if (!Context.hasSameUnqualifiedType(ArgType,
                                            ParamType.getNonReferenceType())) {
       // We can't perform this conversion.
@@ -3963,7 +3989,8 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     // Types match exactly: nothing more to do here.
   } else if (IsQualificationConversion(ArgType, ParamType, false, 
                                        ObjCLifetimeConversion)) {
-    Arg = ImpCastExprToType(Arg, ParamType, CK_NoOp, CastCategory(Arg)).take();
+    Arg = ImpCastExprToType(Arg, ParamType, CK_NoOp,
+                            Arg->getValueKind()).take();
   } else {
     // We can't perform this conversion.
     Diag(Arg->getSourceRange().getBegin(),
@@ -4131,10 +4158,22 @@ Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
   assert(Arg.getKind() == TemplateArgument::Integral &&
          "Operation is only valid for integral template arguments");
   QualType T = Arg.getIntegralType();
-  if (T->isCharType() || T->isWideCharType())
+  if (T->isAnyCharacterType()) {
+    CharacterLiteral::CharacterKind Kind;
+    if (T->isWideCharType())
+      Kind = CharacterLiteral::Wide;
+    else if (T->isChar16Type())
+      Kind = CharacterLiteral::UTF16;
+    else if (T->isChar32Type())
+      Kind = CharacterLiteral::UTF32;
+    else
+      Kind = CharacterLiteral::Ascii;
+
     return Owned(new (Context) CharacterLiteral(
-                                             Arg.getAsIntegral()->getZExtValue(),
-                                             T->isWideCharType(), T, Loc));
+                                            Arg.getAsIntegral()->getZExtValue(),
+                                            Kind, T, Loc));
+  }
+
   if (T->isBooleanType())
     return Owned(new (Context) CXXBoolLiteralExpr(
                                             Arg.getAsIntegral()->getBoolValue(),
@@ -4496,9 +4535,18 @@ static bool CheckTemplateSpecializationScope(Sema &S,
   }
 
   if (S.CurContext->isRecord() && !IsPartialSpecialization) {
-    S.Diag(Loc, diag::err_template_spec_decl_class_scope)
-      << Specialized;
-    return true;
+    if (S.getLangOptions().MicrosoftExt) {
+      // Do not warn for class scope explicit specialization during
+      // instantiation, warning was already emitted during pattern
+      // semantic analysis.
+      if (!S.ActiveTemplateInstantiations.size())
+        S.Diag(Loc, diag::ext_function_specialization_in_class)
+          << Specialized;
+    } else {
+      S.Diag(Loc, diag::err_template_spec_decl_class_scope)
+        << Specialized;
+      return true;
+    }
   }
 
   // C++ [temp.class.spec]p6:
@@ -4653,7 +4701,7 @@ static bool CheckNonTypeClassTemplatePartialSpecializationArgs(Sema &S,
 /// \returns true if there was an error, false otherwise.
 static bool CheckClassTemplatePartialSpecializationArgs(Sema &S,
                                         TemplateParameterList *TemplateParams,
-                       llvm::SmallVectorImpl<TemplateArgument> &TemplateArgs) {
+                       SmallVectorImpl<TemplateArgument> &TemplateArgs) {
   const TemplateArgument *ArgList = TemplateArgs.data();
 
   for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
@@ -4691,6 +4739,7 @@ DeclResult
 Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                        TagUseKind TUK,
                                        SourceLocation KWLoc,
+                                       SourceLocation ModulePrivateLoc,
                                        CXXScopeSpec &SS,
                                        TemplateTy TemplateD,
                                        SourceLocation TemplateNameLoc,
@@ -4821,7 +4870,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
 
   // Check that the template argument list is well-formed for this
   // template.
-  llvm::SmallVector<TemplateArgument, 4> Converted;
+  SmallVector<TemplateArgument, 4> Converted;
   if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc,
                                 TemplateArgs, false, Converted))
     return true;
@@ -4907,14 +4956,14 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
       //   -- The argument list of the specialization shall not be identical
       //      to the implicit argument list of the primary template.
       Diag(TemplateNameLoc, diag::err_partial_spec_args_match_primary_template)
-      << (TUK == TUK_Definition)
-      << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
+        << (TUK == TUK_Definition)
+        << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
       return CheckClassTemplate(S, TagSpec, TUK, KWLoc, SS,
                                 ClassTemplate->getIdentifier(),
                                 TemplateNameLoc,
                                 Attr,
                                 TemplateParams,
-                                AS_none,
+                                AS_none, /*ModulePrivateLoc=*/SourceLocation(),
                                 TemplateParameterLists.size() - 1,
                   (TemplateParameterList**) TemplateParameterLists.release());
     }
@@ -4956,7 +5005,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     // partial specialization are deducible from the template
     // arguments. If not, this class template partial specialization
     // will never be used.
-    llvm::SmallVector<bool, 8> DeducibleParams;
+    SmallVector<bool, 8> DeducibleParams;
     DeducibleParams.resize(TemplateParams->size());
     MarkUsedTemplateParameters(Partial->getTemplateArgs(), true,
                                TemplateParams->getDepth(),
@@ -5056,6 +5105,11 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   if (Attr)
     ProcessDeclAttributeList(S, Specialization, Attr);
 
+  if (ModulePrivateLoc.isValid())
+    Diag(Specialization->getLocation(), diag::err_module_private_specialization)
+      << (isPartialSpecialization? 1 : 0)
+      << FixItHint::CreateRemoval(ModulePrivateLoc);
+  
   // Build the fully-sugared type for this class template
   // specialization as the user wrote in the specialization
   // itself. This means that we'll pretty-print the type retrieved
@@ -5105,7 +5159,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
 Decl *Sema::ActOnTemplateDeclarator(Scope *S,
                               MultiTemplateParamsArg TemplateParameterLists,
                                     Declarator &D) {
-  return HandleDeclarator(S, D, move(TemplateParameterLists), false);
+  return HandleDeclarator(S, D, move(TemplateParameterLists));
 }
 
 Decl *Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
@@ -5120,9 +5174,9 @@ Decl *Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
 
   Scope *ParentScope = FnBodyScope->getParent();
 
+  D.setFunctionDefinition(true);
   Decl *DP = HandleDeclarator(ParentScope, D,
-                              move(TemplateParameterLists),
-                              /*IsFunctionDefinition=*/true);
+                              move(TemplateParameterLists));
   if (FunctionTemplateDecl *FunctionTemplate
         = dyn_cast_or_null<FunctionTemplateDecl>(DP))
     return ActOnStartOfFunctionDef(FnBodyScope,
@@ -5176,8 +5230,7 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
   switch (NewTSK) {
   case TSK_Undeclared:
   case TSK_ImplicitInstantiation:
-    assert(false && "Don't check implicit instantiations here");
-    return false;
+    llvm_unreachable("Don't check implicit instantiations here");
 
   case TSK_ExplicitSpecialization:
     switch (PrevTSK) {
@@ -5309,9 +5362,7 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
     break;
   }
 
-  assert(false && "Missing specialization/instantiation case?");
-
-  return false;
+  llvm_unreachable("Missing specialization/instantiation case?");
 }
 
 /// \brief Perform semantic analysis for the given dependent function
@@ -5482,12 +5533,10 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
   // Take copies of (semantic and syntactic) template argument lists.
   const TemplateArgumentList* TemplArgs = new (Context)
     TemplateArgumentList(Specialization->getTemplateSpecializationArgs());
-  const TemplateArgumentListInfo* TemplArgsAsWritten = ExplicitTemplateArgs
-    ? new (Context) TemplateArgumentListInfo(*ExplicitTemplateArgs) : 0;
   FD->setFunctionTemplateSpecialization(Specialization->getPrimaryTemplate(),
                                         TemplArgs, /*InsertPos=*/0,
                                     SpecInfo->getTemplateSpecializationKind(),
-                                        TemplArgsAsWritten);
+                                        ExplicitTemplateArgs);
   FD->setStorageClass(Specialization->getStorageClass());
   
   // The "previous declaration" for this function template specialization is
@@ -5793,7 +5842,7 @@ Sema::ActOnExplicitInstantiation(Scope *S,
 
   // Check that the template argument list is well-formed for this
   // template.
-  llvm::SmallVector<TemplateArgument, 4> Converted;
+  SmallVector<TemplateArgument, 4> Converted;
   if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc,
                                 TemplateArgs, false, Converted))
     return true;
@@ -5949,6 +5998,7 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   bool IsDependent = false;
   Decl *TagD = ActOnTag(S, TagSpec, Sema::TUK_Reference,
                         KWLoc, SS, Name, NameLoc, Attr, AS_none,
+                        /*ModulePrivateLoc=*/SourceLocation(),
                         MultiTemplateParamsArg(*this, 0, 0),
                         Owned, IsDependent, false, false,
                         TypeResult());
@@ -6114,9 +6164,12 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   if (D.getDeclSpec().isInlineSpecified() && getLangOptions().CPlusPlus0x)
     Diag(D.getDeclSpec().getInlineSpecLoc(),
          diag::err_explicit_instantiation_inline)
-      <<FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
-
-  // FIXME: check for constexpr specifier.
+      << FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
+  if (D.getDeclSpec().isConstexprSpecified())
+    // FIXME: Add a fix-it to remove the 'constexpr' and add a 'const' if one is
+    // not already specified.
+    Diag(D.getDeclSpec().getConstexprSpecLoc(),
+         diag::err_explicit_instantiation_constexpr);
 
   // C++0x [temp.explicit]p2:
   //   There are two forms of explicit instantiation: an explicit instantiation
@@ -6658,6 +6711,45 @@ bool Sema::RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS) {
   return false;
 }
 
+/// \brief Rebuild the template parameters now that we know we're in a current
+/// instantiation.
+bool Sema::RebuildTemplateParamsInCurrentInstantiation(
+                                               TemplateParameterList *Params) {
+  for (unsigned I = 0, N = Params->size(); I != N; ++I) {
+    Decl *Param = Params->getParam(I);
+    
+    // There is nothing to rebuild in a type parameter.
+    if (isa<TemplateTypeParmDecl>(Param))
+      continue;
+    
+    // Rebuild the template parameter list of a template template parameter.
+    if (TemplateTemplateParmDecl *TTP 
+        = dyn_cast<TemplateTemplateParmDecl>(Param)) {
+      if (RebuildTemplateParamsInCurrentInstantiation(
+            TTP->getTemplateParameters()))
+        return true;
+      
+      continue;
+    }
+    
+    // Rebuild the type of a non-type template parameter.
+    NonTypeTemplateParmDecl *NTTP = cast<NonTypeTemplateParmDecl>(Param);
+    TypeSourceInfo *NewTSI 
+      = RebuildTypeInCurrentInstantiation(NTTP->getTypeSourceInfo(), 
+                                          NTTP->getLocation(), 
+                                          NTTP->getDeclName());
+    if (!NewTSI)
+      return true;
+    
+    if (NewTSI != NTTP->getTypeSourceInfo()) {
+      NTTP->setTypeSourceInfo(NewTSI);
+      NTTP->setType(NewTSI->getType());
+    }
+  }
+  
+  return false;
+}
+
 /// \brief Produces a formatted string that describes the binding of
 /// template parameters to template arguments.
 std::string
@@ -6692,7 +6784,7 @@ Sema::getTemplateArgumentBindingsText(const TemplateParameterList *Params,
     }
 
     Out << " = ";
-    Args[I].print(Context.PrintingPolicy, Out);
+    Args[I].print(getPrintingPolicy(), Out);
   }
 
   Out << ']';
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateDeduction.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateDeduction.cpp
index dcb4ff286060..93ea89d6285d 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -85,7 +85,7 @@ DeduceTemplateArguments(Sema &S,
                         const TemplateArgument &Param,
                         TemplateArgument Arg,
                         TemplateDeductionInfo &Info,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced);
+                      SmallVectorImpl<DeducedTemplateArgument> &Deduced);
 
 /// \brief Whether template argument deduction for two reference parameters
 /// resulted in the argument type, parameter type, or neither type being more
@@ -117,10 +117,10 @@ DeduceTemplateArguments(Sema &S,
                         QualType Param,
                         QualType Arg,
                         TemplateDeductionInfo &Info,
-                        llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                        SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         unsigned TDF,
                         bool PartialOrdering = false,
-                      llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *
+                      SmallVectorImpl<RefParamPartialOrderingComparison> *
                                                       RefParamComparisons = 0);
 
 static Sema::TemplateDeductionResult
@@ -129,7 +129,7 @@ DeduceTemplateArguments(Sema &S,
                         const TemplateArgument *Params, unsigned NumParams,
                         const TemplateArgument *Args, unsigned NumArgs,
                         TemplateDeductionInfo &Info,
-                        llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                        SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         bool NumberOfArgumentsMustMatch = true);
 
 /// \brief If the given expression is of a form that permits the deduction
@@ -288,7 +288,7 @@ DeduceNonTypeTemplateArgument(Sema &S,
                               llvm::APSInt Value, QualType ValueType,
                               bool DeducedFromArrayBound,
                               TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
 
@@ -316,7 +316,7 @@ DeduceNonTypeTemplateArgument(Sema &S,
                               NonTypeTemplateParmDecl *NTTP,
                               Expr *Value,
                               TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
   assert((Value->isTypeDependent() || Value->isValueDependent()) &&
@@ -347,7 +347,7 @@ DeduceNonTypeTemplateArgument(Sema &S,
                               NonTypeTemplateParmDecl *NTTP,
                               Decl *D,
                               TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
 
@@ -372,7 +372,7 @@ DeduceTemplateArguments(Sema &S,
                         TemplateName Param,
                         TemplateName Arg,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   TemplateDecl *ParamDecl = Param.getAsTemplateDecl();
   if (!ParamDecl) {
     // The parameter type is dependent and is not a template template parameter,
@@ -431,7 +431,7 @@ DeduceTemplateArguments(Sema &S,
                         const TemplateSpecializationType *Param,
                         QualType Arg,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(Arg.isCanonical() && "Argument type must be canonical");
 
   // Check whether the template argument is a dependent template-id.
@@ -546,11 +546,11 @@ static TemplateParameter makeTemplateParameter(Decl *D) {
 /// \brief Prepare to perform template argument deduction for all of the
 /// arguments in a set of argument packs.
 static void PrepareArgumentPackDeduction(Sema &S,
-                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                             const llvm::SmallVectorImpl<unsigned> &PackIndices,
-                     llvm::SmallVectorImpl<DeducedTemplateArgument> &SavedPacks,
-         llvm::SmallVectorImpl<
-           llvm::SmallVector<DeducedTemplateArgument, 4> > &NewlyDeducedPacks) {
+                       SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                             const SmallVectorImpl<unsigned> &PackIndices,
+                     SmallVectorImpl<DeducedTemplateArgument> &SavedPacks,
+         SmallVectorImpl<
+           SmallVector<DeducedTemplateArgument, 4> > &NewlyDeducedPacks) {
   // Save the deduced template arguments for each parameter pack expanded
   // by this pack expansion, then clear out the deduction.
   for (unsigned I = 0, N = PackIndices.size(); I != N; ++I) {
@@ -581,11 +581,11 @@ static Sema::TemplateDeductionResult
 FinishArgumentPackDeduction(Sema &S,
                             TemplateParameterList *TemplateParams,
                             bool HasAnyArguments,
-                        llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                            const llvm::SmallVectorImpl<unsigned> &PackIndices,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &SavedPacks,
-        llvm::SmallVectorImpl<
-          llvm::SmallVector<DeducedTemplateArgument, 4> > &NewlyDeducedPacks,
+                        SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                            const SmallVectorImpl<unsigned> &PackIndices,
+                    SmallVectorImpl<DeducedTemplateArgument> &SavedPacks,
+        SmallVectorImpl<
+          SmallVector<DeducedTemplateArgument, 4> > &NewlyDeducedPacks,
                             TemplateDeductionInfo &Info) {
   // Build argument packs for each of the parameter packs expanded by this
   // pack expansion.
@@ -668,10 +668,10 @@ DeduceTemplateArguments(Sema &S,
                         const QualType *Params, unsigned NumParams,
                         const QualType *Args, unsigned NumArgs,
                         TemplateDeductionInfo &Info,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                      SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         unsigned TDF,
                         bool PartialOrdering = false,
-                        llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *
+                        SmallVectorImpl<RefParamPartialOrderingComparison> *
                                                      RefParamComparisons = 0) {
   // Fast-path check to see if we have too many/too few arguments.
   if (NumParams != NumArgs &&
@@ -733,11 +733,11 @@ DeduceTemplateArguments(Sema &S,
 
     // Compute the set of template parameter indices that correspond to
     // parameter packs expanded by the pack expansion.
-    llvm::SmallVector<unsigned, 2> PackIndices;
+    SmallVector<unsigned, 2> PackIndices;
     QualType Pattern = Expansion->getPattern();
     {
       llvm::BitVector SawIndices(TemplateParams->size());
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       S.collectUnexpandedParameterPacks(Pattern, Unexpanded);
       for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
         unsigned Depth, Index;
@@ -753,9 +753,9 @@ DeduceTemplateArguments(Sema &S,
     // Keep track of the deduced template arguments for each parameter pack
     // expanded by this pack expansion (the outer index) and for each
     // template argument (the inner SmallVectors).
-    llvm::SmallVector<llvm::SmallVector<DeducedTemplateArgument, 4>, 2>
+    SmallVector<SmallVector<DeducedTemplateArgument, 4>, 2>
       NewlyDeducedPacks(PackIndices.size());
-    llvm::SmallVector<DeducedTemplateArgument, 2>
+    SmallVector<DeducedTemplateArgument, 2>
       SavedPacks(PackIndices.size());
     PrepareArgumentPackDeduction(S, Deduced, PackIndices, SavedPacks,
                                  NewlyDeducedPacks);
@@ -862,10 +862,10 @@ DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         QualType ParamIn, QualType ArgIn,
                         TemplateDeductionInfo &Info,
-                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                     SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         unsigned TDF,
                         bool PartialOrdering,
-    llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *RefParamComparisons) {
+    SmallVectorImpl<RefParamPartialOrderingComparison> *RefParamComparisons) {
   // We only want to look at the canonical types, since typedefs and
   // sugar are not part of template argument deduction.
   QualType Param = S.Context.getCanonicalType(ParamIn);
@@ -977,6 +977,10 @@ DeduceTemplateArguments(Sema &S,
   //     cv-list T
   if (const TemplateTypeParmType *TemplateTypeParm
         = Param->getAs<TemplateTypeParmType>()) {
+    // Just skip any attempts to deduce from a placeholder type.
+    if (Arg->isPlaceholderType())
+      return Sema::TDK_Success;
+    
     unsigned Index = TemplateTypeParm->getIndex();
     bool RecanonicalizeArg = false;
 
@@ -1017,6 +1021,17 @@ DeduceTemplateArguments(Sema &S,
     if (ParamQs.hasObjCLifetime())
       DeducedQs.removeObjCLifetime();
     
+    // Objective-C ARC:
+    //   If template deduction would produce a lifetime qualifier on a type
+    //   that is not a lifetime type, template argument deduction fails.
+    if (ParamQs.hasObjCLifetime() && !DeducedType->isObjCLifetimeType() &&
+        !DeducedType->isDependentType()) {
+      Info.Param = cast<TemplateTypeParmDecl>(TemplateParams->getParam(Index));
+      Info.FirstArg = TemplateArgument(Param);
+      Info.SecondArg = TemplateArgument(Arg);
+      return Sema::TDK_Underqualified;      
+    }
+    
     // Objective-C ARC:
     //   If template deduction would produce an argument type with lifetime type
     //   but no lifetime qualifier, the __strong lifetime qualifier is inferred.
@@ -1101,7 +1116,17 @@ DeduceTemplateArguments(Sema &S,
                                        Info, Deduced, TDF);
 
       return Sema::TDK_NonDeducedMismatch;
-      
+
+    //     _Atomic T   [extension]
+    case Type::Atomic:
+      if (const AtomicType *AtomicArg = Arg->getAs<AtomicType>())
+        return DeduceTemplateArguments(S, TemplateParams,
+                                       cast<AtomicType>(Param)->getValueType(),
+                                       AtomicArg->getValueType(),
+                                       Info, Deduced, TDF);
+
+      return Sema::TDK_NonDeducedMismatch;
+
     //     T *
     case Type::Pointer: {
       QualType PointeeType;
@@ -1306,10 +1331,10 @@ DeduceTemplateArguments(Sema &S,
           // Visited contains the set of nodes we have already visited, while
           // ToVisit is our stack of records that we still need to visit.
           llvm::SmallPtrSet<const RecordType *, 8> Visited;
-          llvm::SmallVector<const RecordType *, 8> ToVisit;
+          SmallVector<const RecordType *, 8> ToVisit;
           ToVisit.push_back(RecordT);
           bool Successful = false;
-          llvm::SmallVectorImpl<DeducedTemplateArgument> DeducedOrig(0);
+          SmallVectorImpl<DeducedTemplateArgument> DeducedOrig(0);
           DeducedOrig = Deduced;
           while (!ToVisit.empty()) {
             // Retrieve the next class in the inheritance hierarchy.
@@ -1515,7 +1540,7 @@ DeduceTemplateArguments(Sema &S,
                         const TemplateArgument &Param,
                         TemplateArgument Arg,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   // If the template argument is a pack expansion, perform template argument
   // deduction against the pattern of that expansion. This only occurs during
   // partial ordering.
@@ -1524,8 +1549,7 @@ DeduceTemplateArguments(Sema &S,
 
   switch (Param.getKind()) {
   case TemplateArgument::Null:
-    assert(false && "Null template argument in parameter list");
-    break;
+    llvm_unreachable("Null template argument in parameter list");
 
   case TemplateArgument::Type:
     if (Arg.getKind() == TemplateArgument::Type)
@@ -1667,7 +1691,7 @@ DeduceTemplateArguments(Sema &S,
                         const TemplateArgument *Params, unsigned NumParams,
                         const TemplateArgument *Args, unsigned NumArgs,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         bool NumberOfArgumentsMustMatch) {
   // C++0x [temp.deduct.type]p9:
   //   If the template argument list of P contains a pack expansion that is not
@@ -1720,10 +1744,10 @@ DeduceTemplateArguments(Sema &S,
 
     // Compute the set of template parameter indices that correspond to
     // parameter packs expanded by the pack expansion.
-    llvm::SmallVector<unsigned, 2> PackIndices;
+    SmallVector<unsigned, 2> PackIndices;
     {
       llvm::BitVector SawIndices(TemplateParams->size());
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       S.collectUnexpandedParameterPacks(Pattern, Unexpanded);
       for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
         unsigned Depth, Index;
@@ -1742,9 +1766,9 @@ DeduceTemplateArguments(Sema &S,
 
     // Save the deduced template arguments for each parameter pack expanded
     // by this pack expansion, then clear out the deduction.
-    llvm::SmallVector<DeducedTemplateArgument, 2>
+    SmallVector<DeducedTemplateArgument, 2>
       SavedPacks(PackIndices.size());
-    llvm::SmallVector<llvm::SmallVector<DeducedTemplateArgument, 4>, 2>
+    SmallVector<SmallVector<DeducedTemplateArgument, 4>, 2>
       NewlyDeducedPacks(PackIndices.size());
     PrepareArgumentPackDeduction(S, Deduced, PackIndices, SavedPacks,
                                  NewlyDeducedPacks);
@@ -1799,7 +1823,7 @@ DeduceTemplateArguments(Sema &S,
                         const TemplateArgumentList &ParamList,
                         const TemplateArgumentList &ArgList,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+                    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   return DeduceTemplateArguments(S, TemplateParams,
                                  ParamList.data(), ParamList.size(),
                                  ArgList.data(), ArgList.size(),
@@ -1815,8 +1839,7 @@ static bool isSameTemplateArg(ASTContext &Context,
 
   switch (X.getKind()) {
     case TemplateArgument::Null:
-      assert(false && "Comparing NULL template argument");
-      break;
+      llvm_unreachable("Comparing NULL template argument");
 
     case TemplateArgument::Type:
       return Context.getCanonicalType(X.getAsType()) ==
@@ -1940,11 +1963,11 @@ static bool ConvertDeducedTemplateArgument(Sema &S, NamedDecl *Param,
                                            unsigned ArgumentPackIndex,
                                            TemplateDeductionInfo &Info,
                                            bool InFunctionTemplate,
-                             llvm::SmallVectorImpl<TemplateArgument> &Output) {
+                             SmallVectorImpl<TemplateArgument> &Output) {
   if (Arg.getKind() == TemplateArgument::Pack) {
     // This is a template argument pack, so check each of its arguments against
     // the template parameter.
-    llvm::SmallVector<TemplateArgument, 2> PackedArgsBuilder;
+    SmallVector<TemplateArgument, 2> PackedArgsBuilder;
     for (TemplateArgument::pack_iterator PA = Arg.pack_begin(),
                                       PAEnd = Arg.pack_end();
          PA != PAEnd; ++PA) {
@@ -1996,7 +2019,7 @@ static Sema::TemplateDeductionResult
 FinishTemplateArgumentDeduction(Sema &S,
                                 ClassTemplatePartialSpecializationDecl *Partial,
                                 const TemplateArgumentList &TemplateArgs,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                      SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                                 TemplateDeductionInfo &Info) {
   // Trap errors.
   Sema::SFINAETrap Trap(S);
@@ -2006,7 +2029,7 @@ FinishTemplateArgumentDeduction(Sema &S,
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  llvm::SmallVector<TemplateArgument, 4> Builder;
+  SmallVector<TemplateArgument, 4> Builder;
   TemplateParameterList *PartialParams = Partial->getTemplateParameters();
   for (unsigned I = 0, N = PartialParams->size(); I != N; ++I) {
     NamedDecl *Param = PartialParams->getParam(I);
@@ -2089,7 +2112,7 @@ FinishTemplateArgumentDeduction(Sema &S,
     return Sema::TDK_SubstitutionFailure;
   }
 
-  llvm::SmallVector<TemplateArgument, 4> ConvertedInstArgs;
+  SmallVector<TemplateArgument, 4> ConvertedInstArgs;
   if (S.CheckTemplateArgumentList(ClassTemplate, Partial->getLocation(),
                                   InstArgs, false, ConvertedInstArgs))
     return Sema::TDK_SubstitutionFailure;
@@ -2125,7 +2148,7 @@ Sema::DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial,
   //   specialization can be deduced from the actual template argument
   //   list (14.8.2).
   SFINAETrap Trap(*this);
-  llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
   Deduced.resize(Partial->getTemplateParameters()->size());
   if (TemplateDeductionResult Result
         = ::DeduceTemplateArguments(*this,
@@ -2183,8 +2206,8 @@ Sema::TemplateDeductionResult
 Sema::SubstituteExplicitTemplateArguments(
                                       FunctionTemplateDecl *FunctionTemplate,
                                TemplateArgumentListInfo &ExplicitTemplateArgs,
-                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                                 llvm::SmallVectorImpl<QualType> &ParamTypes,
+                       SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                                 SmallVectorImpl<QualType> &ParamTypes,
                                           QualType *FunctionType,
                                           TemplateDeductionInfo &Info) {
   FunctionDecl *Function = FunctionTemplate->getTemplatedDecl();
@@ -2214,7 +2237,7 @@ Sema::SubstituteExplicitTemplateArguments(
   //   declaration order of their corresponding template-parameters. The
   //   template argument list shall not specify more template-arguments than
   //   there are corresponding template-parameters.
-  llvm::SmallVector<TemplateArgument, 4> Builder;
+  SmallVector<TemplateArgument, 4> Builder;
 
   // Enter a new template instantiation context where we check the
   // explicitly-specified template arguments against this function template,
@@ -2420,11 +2443,11 @@ CheckOriginalCallArgDeduction(Sema &S, Sema::OriginalCallArg OriginalArg,
 /// which the deduced argument types should be compared.
 Sema::TemplateDeductionResult
 Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
-                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                       SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                                       unsigned NumExplicitlySpecified,
                                       FunctionDecl *&Specialization,
                                       TemplateDeductionInfo &Info,
-        llvm::SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs) {
+        SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs) {
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
 
@@ -2446,7 +2469,7 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  llvm::SmallVector<TemplateArgument, 4> Builder;
+  SmallVector<TemplateArgument, 4> Builder;
   for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
     NamedDecl *Param = TemplateParams->getParam(I);
 
@@ -2566,7 +2589,7 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
   Specialization = cast_or_null<FunctionDecl>(
                       SubstDecl(FunctionTemplate->getTemplatedDecl(), Owner,
                          MultiLevelTemplateArgumentList(*DeducedArgumentList)));
-  if (!Specialization)
+  if (!Specialization || Specialization->isInvalidDecl())
     return TDK_SubstitutionFailure;
 
   assert(Specialization->getPrimaryTemplate()->getCanonicalDecl() ==
@@ -2578,6 +2601,14 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
       !Trap.hasErrorOccurred())
     Info.take();
 
+  // There may have been an error that did not prevent us from constructing a
+  // declaration. Mark the declaration invalid and return with a substitution
+  // failure.
+  if (Trap.hasErrorOccurred()) {
+    Specialization->setInvalidDecl(true);
+    return TDK_SubstitutionFailure;
+  }
+
   if (OriginalCallArgs) {
     // C++ [temp.deduct.call]p4:
     //   In general, the deduction process attempts to find template argument
@@ -2596,20 +2627,12 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
     }
   }
   
-  // There may have been an error that did not prevent us from constructing a
-  // declaration. Mark the declaration invalid and return with a substitution
-  // failure.
-  if (Trap.hasErrorOccurred()) {
-    Specialization->setInvalidDecl(true);
-    return TDK_SubstitutionFailure;
-  }
-
   // If we suppressed any diagnostics while performing template argument
   // deduction, and if we haven't already instantiated this declaration,
   // keep track of these diagnostics. They'll be emitted if this specialization
   // is actually used.
   if (Info.diag_begin() != Info.diag_end()) {
-    llvm::DenseMap<Decl *, llvm::SmallVector<PartialDiagnosticAt, 1> >::iterator
+    llvm::DenseMap<Decl *, SmallVector<PartialDiagnosticAt, 1> >::iterator
       Pos = SuppressedDiagnostics.find(Specialization->getCanonicalDecl());
     if (Pos == SuppressedDiagnostics.end())
         SuppressedDiagnostics[Specialization->getCanonicalDecl()]
@@ -2710,7 +2733,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
     //   Type deduction is done independently for each P/A pair, and
     //   the deduced template argument values are then combined.
     // So we do not reject deductions which were made elsewhere.
-    llvm::SmallVector<DeducedTemplateArgument, 8>
+    SmallVector<DeducedTemplateArgument, 8>
       Deduced(TemplateParams->size());
     TemplateDeductionInfo Info(S.Context, Ovl->getNameLoc());
     Sema::TemplateDeductionResult Result
@@ -2900,8 +2923,8 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   LocalInstantiationScope InstScope(*this);
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
-  llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
-  llvm::SmallVector<QualType, 4> ParamTypes;
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
+  SmallVector<QualType, 4> ParamTypes;
   unsigned NumExplicitlySpecified = 0;
   if (ExplicitTemplateArgs) {
     TemplateDeductionResult Result =
@@ -2924,7 +2947,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   // Deduce template arguments from the function parameters.
   Deduced.resize(TemplateParams->size());
   unsigned ArgIdx = 0;
-  llvm::SmallVector<OriginalCallArg, 4> OriginalCallArgs;
+  SmallVector<OriginalCallArg, 4> OriginalCallArgs;
   for (unsigned ParamIdx = 0, NumParams = ParamTypes.size();
        ParamIdx != NumParams; ++ParamIdx) {
     QualType OrigParamType = ParamTypes[ParamIdx];
@@ -2946,16 +2969,19 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
                                                     TDF))
         continue;
 
+      // If we have nothing to deduce, we're done.
+      if (!hasDeducibleTemplateParameters(*this, FunctionTemplate, ParamType))
+        continue;
+
       // Keep track of the argument type and corresponding parameter index,
       // so we can check for compatibility between the deduced A and A.
-      if (hasDeducibleTemplateParameters(*this, FunctionTemplate, ParamType))
-        OriginalCallArgs.push_back(OriginalCallArg(OrigParamType, ArgIdx-1, 
-                                                   ArgType));
+      OriginalCallArgs.push_back(OriginalCallArg(OrigParamType, ArgIdx-1, 
+                                                 ArgType));
 
       if (TemplateDeductionResult Result
-          = ::DeduceTemplateArguments(*this, TemplateParams,
-                                      ParamType, ArgType, Info, Deduced,
-                                      TDF))
+            = ::DeduceTemplateArguments(*this, TemplateParams,
+                                        ParamType, ArgType, Info, Deduced,
+                                        TDF))
         return Result;
 
       continue;
@@ -2974,10 +3000,10 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
       break;
 
     QualType ParamPattern = ParamExpansion->getPattern();
-    llvm::SmallVector<unsigned, 2> PackIndices;
+    SmallVector<unsigned, 2> PackIndices;
     {
       llvm::BitVector SawIndices(TemplateParams->size());
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       collectUnexpandedParameterPacks(ParamPattern, Unexpanded);
       for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
         unsigned Depth, Index;
@@ -2993,9 +3019,9 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
     // Keep track of the deduced template arguments for each parameter pack
     // expanded by this pack expansion (the outer index) and for each
     // template argument (the inner SmallVectors).
-    llvm::SmallVector<llvm::SmallVector<DeducedTemplateArgument, 4>, 2>
+    SmallVector<SmallVector<DeducedTemplateArgument, 4>, 2>
       NewlyDeducedPacks(PackIndices.size());
-    llvm::SmallVector<DeducedTemplateArgument, 2>
+    SmallVector<DeducedTemplateArgument, 2>
       SavedPacks(PackIndices.size());
     PrepareArgumentPackDeduction(*this, Deduced, PackIndices, SavedPacks,
                                  NewlyDeducedPacks);
@@ -3095,9 +3121,9 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
 
   // Substitute any explicit template arguments.
   LocalInstantiationScope InstScope(*this);
-  llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
   unsigned NumExplicitlySpecified = 0;
-  llvm::SmallVector<QualType, 4> ParamTypes;
+  SmallVector<QualType, 4> ParamTypes;
   if (ExplicitTemplateArgs) {
     if (TemplateDeductionResult Result
           = SubstituteExplicitTemplateArguments(FunctionTemplate,
@@ -3205,7 +3231,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   //   A) as described in 14.8.2.4.
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
-  llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
   Deduced.resize(TemplateParams->size());
 
   // C++0x [temp.deduct.conv]p4:
@@ -3226,7 +3252,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   // both P and A are pointers or member pointers. In this case, we
   // just ignore cv-qualifiers completely).
   if ((P->isPointerType() && A->isPointerType()) ||
-      (P->isMemberPointerType() && P->isMemberPointerType()))
+      (P->isMemberPointerType() && A->isMemberPointerType()))
     TDF |= TDF_IgnoreQualifiers;
   if (TemplateDeductionResult Result
         = ::DeduceTemplateArguments(*this, TemplateParams,
@@ -3342,7 +3368,7 @@ Sema::DeduceAutoType(TypeSourceInfo *Type, Expr *Init,
   QualType FuncParam = FuncParamInfo->getType();
 
   // Deduce type of TemplParam in Func(Init)
-  llvm::SmallVector<DeducedTemplateArgument, 1> Deduced;
+  SmallVector<DeducedTemplateArgument, 1> Deduced;
   Deduced.resize(1);
   QualType InitType = Init->getType();
   unsigned TDF = 0;
@@ -3380,12 +3406,12 @@ static void
 MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
                            bool OnlyDeduced,
                            unsigned Level,
-                           llvm::SmallVectorImpl<bool> &Deduced);
+                           SmallVectorImpl<bool> &Deduced);
 
 /// \brief If this is a non-static member function,
 static void MaybeAddImplicitObjectParameterType(ASTContext &Context,
                                                 CXXMethodDecl *Method,
-                                 llvm::SmallVectorImpl<QualType> &ArgTypes) {
+                                 SmallVectorImpl<QualType> &ArgTypes) {
   if (Method->isStatic())
     return;
 
@@ -3414,7 +3440,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
                                      FunctionTemplateDecl *FT2,
                                      TemplatePartialOrderingContext TPOC,
                                      unsigned NumCallArguments,
-    llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *RefParamComparisons) {
+    SmallVectorImpl<RefParamPartialOrderingComparison> *RefParamComparisons) {
   FunctionDecl *FD1 = FT1->getTemplatedDecl();
   FunctionDecl *FD2 = FT2->getTemplatedDecl();
   const FunctionProtoType *Proto1 = FD1->getType()->getAs<FunctionProtoType>();
@@ -3422,7 +3448,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
 
   assert(Proto1 && Proto2 && "Function templates must have prototypes");
   TemplateParameterList *TemplateParams = FT2->getTemplateParameters();
-  llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
   Deduced.resize(TemplateParams->size());
 
   // C++0x [temp.deduct.partial]p3:
@@ -3454,7 +3480,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
     // C++98/03 doesn't have this provision, so instead we drop the
     // first argument of the free function or static member, which
     // seems to match existing practice.
-    llvm::SmallVector<QualType, 4> Args1;
+    SmallVector<QualType, 4> Args1;
     unsigned Skip1 = !S.getLangOptions().CPlusPlus0x &&
       IsNonStatic2 && !IsNonStatic1;
     if (S.getLangOptions().CPlusPlus0x && IsNonStatic1 && !IsNonStatic2)
@@ -3462,7 +3488,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
     Args1.insert(Args1.end(),
                  Proto1->arg_type_begin() + Skip1, Proto1->arg_type_end());
 
-    llvm::SmallVector<QualType, 4> Args2;
+    SmallVector<QualType, 4> Args2;
     Skip2 = !S.getLangOptions().CPlusPlus0x &&
       IsNonStatic1 && !IsNonStatic2;
     if (S.getLangOptions().CPlusPlus0x && IsNonStatic2 && !IsNonStatic1)
@@ -3524,7 +3550,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
   }
 
   // Figure out which template parameters were used.
-  llvm::SmallVector<bool, 4> UsedParameters;
+  SmallVector<bool, 4> UsedParameters;
   UsedParameters.resize(TemplateParams->size());
   switch (TPOC) {
   case TPOC_Call: {
@@ -3605,7 +3631,7 @@ Sema::getMoreSpecializedTemplate(FunctionTemplateDecl *FT1,
                                  SourceLocation Loc,
                                  TemplatePartialOrderingContext TPOC,
                                  unsigned NumCallArguments) {
-  llvm::SmallVector<RefParamPartialOrderingComparison, 4> RefParamComparisons;
+  SmallVector<RefParamPartialOrderingComparison, 4> RefParamComparisons;
   bool Better1 = isAtLeastAsSpecializedAs(*this, Loc, FT1, FT2, TPOC,
                                           NumCallArguments, 0);
   bool Better2 = isAtLeastAsSpecializedAs(*this, Loc, FT2, FT1, TPOC,
@@ -3853,7 +3879,7 @@ Sema::getMoreSpecializedPartialSpecialization(
   // know that every template parameter is deducible from the class
   // template partial specialization's template arguments, for
   // example.
-  llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
   TemplateDeductionInfo Info(Context, Loc);
 
   QualType PT1 = PS1->getInjectedSpecializationType();
@@ -3899,7 +3925,7 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            const TemplateArgument &TemplateArg,
                            bool OnlyDeduced,
                            unsigned Depth,
-                           llvm::SmallVectorImpl<bool> &Used);
+                           SmallVectorImpl<bool> &Used);
 
 /// \brief Mark the template parameters that are used by the given
 /// expression.
@@ -3908,7 +3934,7 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            const Expr *E,
                            bool OnlyDeduced,
                            unsigned Depth,
-                           llvm::SmallVectorImpl<bool> &Used) {
+                           SmallVectorImpl<bool> &Used) {
   // We can deduce from a pack expansion.
   if (const PackExpansionExpr *Expansion = dyn_cast<PackExpansionExpr>(E))
     E = Expansion->getPattern();
@@ -3939,7 +3965,7 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            NestedNameSpecifier *NNS,
                            bool OnlyDeduced,
                            unsigned Depth,
-                           llvm::SmallVectorImpl<bool> &Used) {
+                           SmallVectorImpl<bool> &Used) {
   if (!NNS)
     return;
 
@@ -3956,7 +3982,7 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            TemplateName Name,
                            bool OnlyDeduced,
                            unsigned Depth,
-                           llvm::SmallVectorImpl<bool> &Used) {
+                           SmallVectorImpl<bool> &Used) {
   if (TemplateDecl *Template = Name.getAsTemplateDecl()) {
     if (TemplateTemplateParmDecl *TTP
           = dyn_cast<TemplateTemplateParmDecl>(Template)) {
@@ -3980,7 +4006,7 @@ static void
 MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
                            bool OnlyDeduced,
                            unsigned Depth,
-                           llvm::SmallVectorImpl<bool> &Used) {
+                           SmallVectorImpl<bool> &Used) {
   if (T.isNull())
     return;
 
@@ -4113,6 +4139,13 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
                                  OnlyDeduced, Depth, Used);
     break;
 
+  case Type::Atomic:
+    if (!OnlyDeduced)
+      MarkUsedTemplateParameters(SemaRef,
+                                 cast<AtomicType>(T)->getValueType(),
+                                 OnlyDeduced, Depth, Used);
+    break;
+
   case Type::DependentName:
     if (!OnlyDeduced)
       MarkUsedTemplateParameters(SemaRef,
@@ -4206,7 +4239,7 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            const TemplateArgument &TemplateArg,
                            bool OnlyDeduced,
                            unsigned Depth,
-                           llvm::SmallVectorImpl<bool> &Used) {
+                           SmallVectorImpl<bool> &Used) {
   switch (TemplateArg.getKind()) {
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
@@ -4251,7 +4284,7 @@ MarkUsedTemplateParameters(Sema &SemaRef,
 void
 Sema::MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs,
                                  bool OnlyDeduced, unsigned Depth,
-                                 llvm::SmallVectorImpl<bool> &Used) {
+                                 SmallVectorImpl<bool> &Used) {
   // C++0x [temp.deduct.type]p9:
   //   If the template argument list of P contains a pack expansion that is not
   //   the last template argument, the entire template argument list is a
@@ -4269,7 +4302,7 @@ Sema::MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs,
 /// call to the given function template.
 void
 Sema::MarkDeducedTemplateParameters(FunctionTemplateDecl *FunctionTemplate,
-                                    llvm::SmallVectorImpl<bool> &Deduced) {
+                                    SmallVectorImpl<bool> &Deduced) {
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
   Deduced.clear();
@@ -4289,7 +4322,7 @@ bool hasDeducibleTemplateParameters(Sema &S,
 
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
-  llvm::SmallVector<bool, 4> Deduced;
+  SmallVector<bool, 4> Deduced;
   Deduced.resize(TemplateParams->size());
   ::MarkUsedTemplateParameters(S, T, true, TemplateParams->getDepth(), 
                                Deduced);
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiate.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 1988f1445814..301bf6a11260 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -98,8 +98,9 @@ Sema::getTemplateInstantiationArgs(NamedDecl *D,
     // Add template arguments from a function template specialization.
     else if (FunctionDecl *Function = dyn_cast<FunctionDecl>(Ctx)) {
       if (!RelativeToPrimary &&
-          Function->getTemplateSpecializationKind() 
-                                                  == TSK_ExplicitSpecialization)
+          (Function->getTemplateSpecializationKind() == 
+                                                  TSK_ExplicitSpecialization &&
+           !Function->getClassScopeSpecializationPattern()))
         break;
           
       if (const TemplateArgumentList *TemplateArgs
@@ -428,7 +429,7 @@ void Sema::PrintInstantiationStack() {
 
   // FIXME: In all of these cases, we need to show the template arguments
   unsigned InstantiationIdx = 0;
-  for (llvm::SmallVector<ActiveTemplateInstantiation, 16>::reverse_iterator
+  for (SmallVector<ActiveTemplateInstantiation, 16>::reverse_iterator
          Active = ActiveTemplateInstantiations.rbegin(),
          ActiveEnd = ActiveTemplateInstantiations.rend();
        Active != ActiveEnd;
@@ -483,7 +484,7 @@ void Sema::PrintInstantiationStack() {
         = TemplateSpecializationType::PrintTemplateArgumentList(
                                                          Active->TemplateArgs,
                                                       Active->NumTemplateArgs,
-                                                      Context.PrintingPolicy);
+                                                      getPrintingPolicy());
       Diags.Report(Active->PointOfInstantiation,
                    diag::note_default_arg_instantiation_here)
         << (Template->getNameAsString() + TemplateArgsStr)
@@ -537,7 +538,7 @@ void Sema::PrintInstantiationStack() {
         = TemplateSpecializationType::PrintTemplateArgumentList(
                                                          Active->TemplateArgs,
                                                       Active->NumTemplateArgs,
-                                                      Context.PrintingPolicy);
+                                                      getPrintingPolicy());
       Diags.Report(Active->PointOfInstantiation,
                    diag::note_default_function_arg_instantiation_here)
         << (FD->getNameAsString() + TemplateArgsStr)
@@ -591,7 +592,6 @@ void Sema::PrintInstantiationStack() {
 }
 
 llvm::Optional<TemplateDeductionInfo *> Sema::isSFINAEContext() const {
-  using llvm::SmallVector;
   if (InNonInstantiationSFINAEContext)
     return llvm::Optional<TemplateDeductionInfo *>(0);
 
@@ -681,14 +681,12 @@ namespace {
 
     bool TryExpandParameterPacks(SourceLocation EllipsisLoc,
                                  SourceRange PatternRange,
-                                 const UnexpandedParameterPack *Unexpanded,
-                                 unsigned NumUnexpanded,
+                             llvm::ArrayRef<UnexpandedParameterPack> Unexpanded,
                                  bool &ShouldExpand,
                                  bool &RetainExpansion,
                                  llvm::Optional<unsigned> &NumExpansions) {
       return getSema().CheckParameterPacksForExpansion(EllipsisLoc, 
                                                        PatternRange, Unexpanded,
-                                                       NumUnexpanded, 
                                                        TemplateArgs, 
                                                        ShouldExpand,
                                                        RetainExpansion,
@@ -1535,8 +1533,8 @@ ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm,
 bool Sema::SubstParmTypes(SourceLocation Loc, 
                           ParmVarDecl **Params, unsigned NumParams,
                           const MultiLevelTemplateArgumentList &TemplateArgs,
-                          llvm::SmallVectorImpl<QualType> &ParamTypes,
-                          llvm::SmallVectorImpl<ParmVarDecl *> *OutParams) {
+                          SmallVectorImpl<QualType> &ParamTypes,
+                          SmallVectorImpl<ParmVarDecl *> *OutParams) {
   assert(!ActiveTemplateInstantiations.empty() &&
          "Cannot perform an instantiation without some context on the "
          "instantiation stack");
@@ -1558,7 +1556,7 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
                           CXXRecordDecl *Pattern,
                           const MultiLevelTemplateArgumentList &TemplateArgs) {
   bool Invalid = false;
-  llvm::SmallVector<CXXBaseSpecifier*, 4> InstantiatedBases;
+  SmallVector<CXXBaseSpecifier*, 4> InstantiatedBases;
   for (ClassTemplateSpecializationDecl::base_class_iterator
          Base = Pattern->bases_begin(), BaseEnd = Pattern->bases_end();
        Base != BaseEnd; ++Base) {
@@ -1572,7 +1570,7 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
     if (Base->isPackExpansion()) {
       // This is a pack expansion. See whether we should expand it now, or
       // wait until later.
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       collectUnexpandedParameterPacks(Base->getTypeSourceInfo()->getTypeLoc(),
                                       Unexpanded);
       bool ShouldExpand = false;
@@ -1580,7 +1578,7 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
       llvm::Optional<unsigned> NumExpansions;
       if (CheckParameterPacksForExpansion(Base->getEllipsisLoc(), 
                                           Base->getSourceRange(),
-                                          Unexpanded.data(), Unexpanded.size(),
+                                          Unexpanded,
                                           TemplateArgs, ShouldExpand, 
                                           RetainExpansion,
                                           NumExpansions)) {
@@ -1755,8 +1753,8 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     Invalid = true;
 
   TemplateDeclInstantiator Instantiator(*this, Instantiation, TemplateArgs);
-  llvm::SmallVector<Decl*, 4> Fields;
-  llvm::SmallVector<std::pair<FieldDecl*, FieldDecl*>, 4>
+  SmallVector<Decl*, 4> Fields;
+  SmallVector<std::pair<FieldDecl*, FieldDecl*>, 4>
     FieldsWithMemberInitializers;
   for (RecordDecl::decl_iterator Member = Pattern->decls_begin(),
          MemberEnd = Pattern->decls_end();
@@ -1796,9 +1794,8 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   }
 
   // Finish checking fields.
-  ActOnFields(0, Instantiation->getLocation(), Instantiation,
-              Fields.data(), Fields.size(), SourceLocation(), SourceLocation(),
-              0);
+  ActOnFields(0, Instantiation->getLocation(), Instantiation, Fields, 
+              SourceLocation(), SourceLocation(), 0);
   CheckCompletedCXXClass(Instantiation);
 
   // Attach any in-class member initializers now the class is complete.
@@ -1806,39 +1803,24 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
     FieldDecl *OldField = FieldsWithMemberInitializers[I].first;
     FieldDecl *NewField = FieldsWithMemberInitializers[I].second;
     Expr *OldInit = OldField->getInClassInitializer();
-    ExprResult NewInit = SubstExpr(OldInit, TemplateArgs);
 
-    // If the initialization is no longer dependent, check it now.
-    if ((OldField->getType()->isDependentType() || OldInit->isTypeDependent() ||
-         OldInit->isValueDependent()) &&
-        !NewField->getType()->isDependentType() &&
-        !NewInit.get()->isTypeDependent() &&
-        !NewInit.get()->isValueDependent()) {
-      // FIXME: handle list-initialization
-      SourceLocation EqualLoc = NewField->getLocation();
-      NewInit = PerformCopyInitialization(
-        InitializedEntity::InitializeMember(NewField), EqualLoc,
-        NewInit.release());
-
-      if (!NewInit.isInvalid()) {
-        CheckImplicitConversions(NewInit.get(), EqualLoc);
-
-        // C++0x [class.base.init]p7:
-        //   The initialization of each base and member constitutes a
-        //   full-expression.
-        NewInit = MaybeCreateExprWithCleanups(NewInit);
-      }
-    }
-
-    if (NewInit.isInvalid())
+    SourceLocation LParenLoc, RParenLoc;
+    ASTOwningVector<Expr*> NewArgs(*this);
+    if (InstantiateInitializer(OldInit, TemplateArgs, LParenLoc, NewArgs,
+                               RParenLoc))
       NewField->setInvalidDecl();
-    else
-      NewField->setInClassInitializer(NewInit.release());
+    else {
+      assert(NewArgs.size() == 1 && "wrong number of in-class initializers");
+      ActOnCXXInClassMemberInitializer(NewField, LParenLoc, NewArgs[0]);
+    }
   }
 
   if (!FieldsWithMemberInitializers.empty())
     ActOnFinishDelayedMemberInitializers(Instantiation);
 
+  if (TSK == TSK_ImplicitInstantiation)
+    Instantiation->setRBraceLoc(Pattern->getRBraceLoc());
+
   if (Instantiation->isInvalidDecl())
     Invalid = true;
   else {
@@ -1931,8 +1913,8 @@ Sema::InstantiateClassTemplateSpecialization(
   //   specialization with the template argument lists of the partial
   //   specializations.
   typedef PartialSpecMatchResult MatchResult;
-  llvm::SmallVector<MatchResult, 4> Matched;
-  llvm::SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
+  SmallVector<MatchResult, 4> Matched;
+  SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
   Template->getPartialSpecializations(PartialSpecs);
   for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I) {
     ClassTemplatePartialSpecializationDecl *Partial = PartialSpecs[I];
@@ -1954,10 +1936,10 @@ Sema::InstantiateClassTemplateSpecialization(
   // If we're dealing with a member template where the template parameters
   // have been instantiated, this provides the original template parameters
   // from which the member template's parameters were instantiated.
-  llvm::SmallVector<const NamedDecl *, 4> InstantiatedTemplateParameters;
+  SmallVector<const NamedDecl *, 4> InstantiatedTemplateParameters;
   
   if (Matched.size() >= 1) {
-    llvm::SmallVector<MatchResult, 4>::iterator Best = Matched.begin();
+    SmallVector<MatchResult, 4>::iterator Best = Matched.begin();
     if (Matched.size() == 1) {
       //   -- If exactly one matching specialization is found, the
       //      instantiation is generated from that specialization.
@@ -1970,7 +1952,7 @@ Sema::InstantiateClassTemplateSpecialization(
       //      specialized than all of the other matching
       //      specializations, then the use of the class template is
       //      ambiguous and the program is ill-formed.
-      for (llvm::SmallVector<MatchResult, 4>::iterator P = Best + 1,
+      for (SmallVector<MatchResult, 4>::iterator P = Best + 1,
                                                     PEnd = Matched.end();
            P != PEnd; ++P) {
         if (getMoreSpecializedPartialSpecialization(P->Partial, Best->Partial,
@@ -1982,7 +1964,7 @@ Sema::InstantiateClassTemplateSpecialization(
       // Determine if the best partial specialization is more specialized than
       // the others.
       bool Ambiguous = false;
-      for (llvm::SmallVector<MatchResult, 4>::iterator P = Matched.begin(),
+      for (SmallVector<MatchResult, 4>::iterator P = Matched.begin(),
                                                     PEnd = Matched.end();
            P != PEnd; ++P) {
         if (P != Best &&
@@ -2001,7 +1983,7 @@ Sema::InstantiateClassTemplateSpecialization(
           << ClassTemplateSpec;
         
         // Print the matching partial specializations.
-        for (llvm::SmallVector<MatchResult, 4>::iterator P = Matched.begin(),
+        for (SmallVector<MatchResult, 4>::iterator P = Matched.begin(),
                                                       PEnd = Matched.end();
              P != PEnd; ++P)
           Diag(P->Partial->getLocation(), diag::note_partial_spec_match)
@@ -2240,7 +2222,7 @@ Sema::SubstExpr(Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs) {
 
 bool Sema::SubstExprs(Expr **Exprs, unsigned NumExprs, bool IsCall,
                       const MultiLevelTemplateArgumentList &TemplateArgs,
-                      llvm::SmallVectorImpl<Expr *> &Outputs) {
+                      SmallVectorImpl<Expr *> &Outputs) {
   if (NumExprs == 0)
     return false;
   
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 29385e50b8c1..02a05d518251 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -29,14 +29,14 @@ bool TemplateDeclInstantiator::SubstQualifier(const DeclaratorDecl *OldDecl,
                                               DeclaratorDecl *NewDecl) {
   if (!OldDecl->getQualifierLoc())
     return false;
-  
+
   NestedNameSpecifierLoc NewQualifierLoc
-    = SemaRef.SubstNestedNameSpecifierLoc(OldDecl->getQualifierLoc(), 
+    = SemaRef.SubstNestedNameSpecifierLoc(OldDecl->getQualifierLoc(),
                                           TemplateArgs);
-  
+
   if (!NewQualifierLoc)
     return true;
-  
+
   NewDecl->setQualifierInfo(NewQualifierLoc);
   return false;
 }
@@ -45,14 +45,14 @@ bool TemplateDeclInstantiator::SubstQualifier(const TagDecl *OldDecl,
                                               TagDecl *NewDecl) {
   if (!OldDecl->getQualifierLoc())
     return false;
-  
+
   NestedNameSpecifierLoc NewQualifierLoc
-  = SemaRef.SubstNestedNameSpecifierLoc(OldDecl->getQualifierLoc(), 
+  = SemaRef.SubstNestedNameSpecifierLoc(OldDecl->getQualifierLoc(),
                                         TemplateArgs);
-  
+
   if (!NewQualifierLoc)
     return true;
-  
+
   NewDecl->setQualifierInfo(NewQualifierLoc);
   return false;
 }
@@ -79,7 +79,7 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
         else {
           TypeSourceInfo *Result = SubstType(Aligned->getAlignmentType(),
                                              TemplateArgs,
-                                             Aligned->getLocation(), 
+                                             Aligned->getLocation(),
                                              DeclarationName());
           if (Result)
             AddAlignedAttr(Aligned->getLocation(), New, Result);
@@ -96,8 +96,7 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
 
 Decl *
 TemplateDeclInstantiator::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
-  assert(false && "Translation units cannot be instantiated");
-  return D;
+  llvm_unreachable("Translation units cannot be instantiated");
 }
 
 Decl *
@@ -110,8 +109,7 @@ TemplateDeclInstantiator::VisitLabelDecl(LabelDecl *D) {
 
 Decl *
 TemplateDeclInstantiator::VisitNamespaceDecl(NamespaceDecl *D) {
-  assert(false && "Namespaces cannot be instantiated");
-  return D;
+  llvm_unreachable("Namespaces cannot be instantiated");
 }
 
 Decl *
@@ -165,13 +163,13 @@ Decl *TemplateDeclInstantiator::InstantiateTypedefNameDecl(TypedefNameDecl *D,
       newTag->setTypedefNameForAnonDecl(Typedef);
     }
   }
-  
+
   if (TypedefNameDecl *Prev = D->getPreviousDeclaration()) {
     NamedDecl *InstPrev = SemaRef.FindInstantiatedDecl(D->getLocation(), Prev,
                                                        TemplateArgs);
     if (!InstPrev)
       return 0;
-    
+
     Typedef->setPreviousDeclaration(cast<TypedefNameDecl>(InstPrev));
   }
 
@@ -230,7 +228,7 @@ TemplateDeclInstantiator::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
 
   if (!PrevAliasTemplate)
     Inst->setInstantiatedFromMemberTemplate(D);
-  
+
   Owner->addDecl(Inst);
 
   return Inst;
@@ -239,8 +237,6 @@ TemplateDeclInstantiator::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
 /// \brief Instantiate an initializer, breaking it into separate
 /// initialization arguments.
 ///
-/// \param S The semantic analysis object.
-///
 /// \param Init The initializer to instantiate.
 ///
 /// \param TemplateArgs Template arguments to be substituted into the
@@ -249,11 +245,11 @@ TemplateDeclInstantiator::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
 /// \param NewArgs Will be filled in with the instantiation arguments.
 ///
 /// \returns true if an error occurred, false otherwise
-static bool InstantiateInitializer(Sema &S, Expr *Init,
+bool Sema::InstantiateInitializer(Expr *Init,
                             const MultiLevelTemplateArgumentList &TemplateArgs,
-                                   SourceLocation &LParenLoc,
-                                   ASTOwningVector<Expr*> &NewArgs,
-                                   SourceLocation &RParenLoc) {
+                                  SourceLocation &LParenLoc,
+                                  ASTOwningVector<Expr*> &NewArgs,
+                                  SourceLocation &RParenLoc) {
   NewArgs.clear();
   LParenLoc = SourceLocation();
   RParenLoc = SourceLocation();
@@ -273,24 +269,24 @@ static bool InstantiateInitializer(Sema &S, Expr *Init,
   if (ParenListExpr *ParenList = dyn_cast<ParenListExpr>(Init)) {
     LParenLoc = ParenList->getLParenLoc();
     RParenLoc = ParenList->getRParenLoc();
-    return S.SubstExprs(ParenList->getExprs(), ParenList->getNumExprs(),
-                        true, TemplateArgs, NewArgs);
+    return SubstExprs(ParenList->getExprs(), ParenList->getNumExprs(),
+                      true, TemplateArgs, NewArgs);
   }
 
   if (CXXConstructExpr *Construct = dyn_cast<CXXConstructExpr>(Init)) {
     if (!isa<CXXTemporaryObjectExpr>(Construct)) {
-      if (S.SubstExprs(Construct->getArgs(), Construct->getNumArgs(), true,
-                       TemplateArgs, NewArgs))
+      if (SubstExprs(Construct->getArgs(), Construct->getNumArgs(), true,
+                     TemplateArgs, NewArgs))
         return true;
 
       // FIXME: Fake locations!
-      LParenLoc = S.PP.getLocForEndOfToken(Init->getLocStart());
+      LParenLoc = PP.getLocForEndOfToken(Init->getLocStart());
       RParenLoc = LParenLoc;
       return false;
     }
   }
- 
-  ExprResult Result = S.SubstExpr(Init, TemplateArgs);
+
+  ExprResult Result = SubstExpr(Init, TemplateArgs);
   if (Result.isInvalid())
     return true;
 
@@ -319,7 +315,7 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
       << D->isStaticDataMember() << DI->getType();
     return 0;
   }
-  
+
   // Build the instantiated declaration
   VarDecl *Var = VarDecl::Create(SemaRef.Context, Owner,
                                  D->getInnerLocStart(),
@@ -342,21 +338,20 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
     Var->setLexicalDeclContext(D->getLexicalDeclContext());
 
   Var->setAccess(D->getAccess());
-  
+
   if (!D->isStaticDataMember()) {
     Var->setUsed(D->isUsed(false));
     Var->setReferenced(D->isReferenced());
   }
-  
+
   // FIXME: In theory, we could have a previous declaration for variables that
   // are not static data members.
-  bool Redeclaration = false;
   // FIXME: having to fake up a LookupResult is dumb.
   LookupResult Previous(SemaRef, Var->getDeclName(), Var->getLocation(),
                         Sema::LookupOrdinaryName, Sema::ForRedeclaration);
   if (D->isStaticDataMember())
     SemaRef.LookupQualifiedName(Previous, Owner, false);
-  SemaRef.CheckVariableDeclaration(Var, Previous, Redeclaration);
+  SemaRef.CheckVariableDeclaration(Var, Previous);
 
   if (D->isOutOfLine()) {
     if (!D->isStaticDataMember())
@@ -368,13 +363,13 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
       SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Var);
   }
   SemaRef.InstantiateAttrs(TemplateArgs, D, Var);
-  
+
   // Link instantiations of static data members back to the template from
   // which they were instantiated.
   if (Var->isStaticDataMember())
-    SemaRef.Context.setInstantiatedFromStaticDataMember(Var, D, 
+    SemaRef.Context.setInstantiatedFromStaticDataMember(Var, D,
                                                      TSK_ImplicitInstantiation);
-  
+
   if (Var->getAnyInitializer()) {
     // We already have an initializer in the class.
   } else if (D->getInit()) {
@@ -386,8 +381,8 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
     // Instantiate the initializer.
     SourceLocation LParenLoc, RParenLoc;
     ASTOwningVector<Expr*> InitArgs(SemaRef);
-    if (!InstantiateInitializer(SemaRef, D->getInit(), TemplateArgs, LParenLoc,
-                                InitArgs, RParenLoc)) {
+    if (!SemaRef.InstantiateInitializer(D->getInit(), TemplateArgs, LParenLoc,
+                                        InitArgs, RParenLoc)) {
       bool TypeMayContainAuto = true;
       // Attach the initializer to the declaration, if we have one.
       if (InitArgs.size() == 0)
@@ -409,7 +404,7 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
       // because of a bogus initializer.
       Var->setInvalidDecl();
     }
-    
+
     SemaRef.PopExpressionEvaluationContext();
   } else if ((!Var->isStaticDataMember() || Var->isOutOfLine()) &&
              !Var->isCXXForRangeDecl())
@@ -489,14 +484,14 @@ Decl *TemplateDeclInstantiator::VisitFieldDecl(FieldDecl *D) {
   }
 
   SemaRef.InstantiateAttrs(TemplateArgs, D, Field);
-  
+
   if (Invalid)
     Field->setInvalidDecl();
 
   if (!Field->getDeclName()) {
     // Keep track of where this decl came from.
     SemaRef.Context.setInstantiatedFromUnnamedFieldDecl(Field, D);
-  } 
+  }
   if (CXXRecordDecl *Parent= dyn_cast<CXXRecordDecl>(Field->getDeclContext())) {
     if (Parent->isAnonymousStructOrUnion() &&
         Parent->getRedeclContext()->isFunctionOrMethod())
@@ -518,11 +513,11 @@ Decl *TemplateDeclInstantiator::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
   for (IndirectFieldDecl::chain_iterator PI =
        D->chain_begin(), PE = D->chain_end();
        PI != PE; ++PI) {
-    NamedDecl *Next = SemaRef.FindInstantiatedDecl(D->getLocation(), *PI, 
+    NamedDecl *Next = SemaRef.FindInstantiatedDecl(D->getLocation(), *PI,
                                               TemplateArgs);
     if (!Next)
       return 0;
-    
+
     NamedChain[i++] = Next;
   }
 
@@ -560,13 +555,13 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
     FriendDecl *FD = SemaRef.CheckFriendTypeDecl(D->getFriendLoc(), InstTy);
     if (!FD)
       return 0;
-    
+
     FD->setAccess(AS_public);
     FD->setUnsupportedFriend(D->isUnsupportedFriend());
     Owner->addDecl(FD);
     return FD;
-  } 
-  
+  }
+
   NamedDecl *ND = D->getFriendDecl();
   assert(ND && "friend decl must be a decl or a type!");
 
@@ -578,7 +573,7 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
   if (!NewND) return 0;
 
   FriendDecl *FD =
-    FriendDecl::Create(SemaRef.Context, Owner, D->getLocation(), 
+    FriendDecl::Create(SemaRef.Context, Owner, D->getLocation(),
                        cast<NamedDecl>(NewND), D->getFriendLoc());
   FD->setAccess(AS_public);
   FD->setUnsupportedFriend(D->isUnsupportedFriend());
@@ -620,7 +615,7 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
                                                        TemplateArgs,
                                                        UnderlyingLoc,
                                                        DeclarationName()));
-      
+
       if (!Enum->getIntegerTypeSourceInfo())
         Enum->setIntegerType(SemaRef.Context.IntTy);
     }
@@ -641,8 +636,8 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
 
   if (D->getDeclContext()->isFunctionOrMethod())
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Enum);
-    
-  llvm::SmallVector<Decl*, 4> Enumerators;
+
+  SmallVector<Decl*, 4> Enumerators;
 
   EnumConstantDecl *LastEnumConst = 0;
   for (EnumDecl::enumerator_iterator EC = D->enumerator_begin(),
@@ -683,7 +678,7 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
       Enum->addDecl(EnumConst);
       Enumerators.push_back(EnumConst);
       LastEnumConst = EnumConst;
-      
+
       if (D->getDeclContext()->isFunctionOrMethod()) {
         // If the enumeration is within a function or method, record the enum
         // constant as a local.
@@ -703,8 +698,7 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
 }
 
 Decl *TemplateDeclInstantiator::VisitEnumConstantDecl(EnumConstantDecl *D) {
-  assert(false && "EnumConstantDecls can only occur within EnumDecls.");
-  return 0;
+  llvm_unreachable("EnumConstantDecls can only occur within EnumDecls.");
 }
 
 Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
@@ -789,7 +783,7 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
       // template parameters of the original declaration. In this one
       // case, we don't complain about the ill-formed friend
       // declaration.
-      if (isFriend && Pattern->getIdentifier() && 
+      if (isFriend && Pattern->getIdentifier() &&
           Pattern->getIdentifier()->isStr("_Map_base") &&
           DC->isNamespace() &&
           cast<NamespaceDecl>(DC)->getIdentifier() &&
@@ -812,7 +806,7 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
       // Make sure the parameter lists match.
       if (!SemaRef.TemplateParameterListsAreEqual(InstParams, PrevParams,
-                                                  Complain, 
+                                                  Complain,
                                                   Sema::TPL_TemplateMatch)) {
         if (Complain)
           return 0;
@@ -859,7 +853,7 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
     if (!PrevClassTemplate)
       Inst->setInstantiatedFromMemberTemplate(D);
   }
-  
+
   // Trigger creation of the type for the instantiation.
   SemaRef.Context.getInjectedClassNameType(RecordInst,
                                     Inst->getInjectedClassNameSpecialization());
@@ -869,14 +863,14 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
     DC->makeDeclVisibleInContext(Inst, /*Recoverable*/ false);
     return Inst;
   }
-  
+
   Owner->addDecl(Inst);
 
   if (!PrevClassTemplate) {
     // Queue up any out-of-line partial specializations of this member
     // class template; the client will force their instantiation once
     // the enclosing class has been instantiated.
-    llvm::SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
+    SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
     D->getPartialSpecializations(PartialSpecs);
     for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I)
       if (PartialSpecs[I]->isOutOfLine())
@@ -890,19 +884,19 @@ Decl *
 TemplateDeclInstantiator::VisitClassTemplatePartialSpecializationDecl(
                                    ClassTemplatePartialSpecializationDecl *D) {
   ClassTemplateDecl *ClassTemplate = D->getSpecializedTemplate();
-  
+
   // Lookup the already-instantiated declaration in the instantiation
   // of the class template and return that.
   DeclContext::lookup_result Found
     = Owner->lookup(ClassTemplate->getDeclName());
   if (Found.first == Found.second)
     return 0;
-  
+
   ClassTemplateDecl *InstClassTemplate
     = dyn_cast<ClassTemplateDecl>(*Found.first);
   if (!InstClassTemplate)
     return 0;
-  
+
   if (ClassTemplatePartialSpecializationDecl *Result
         = InstClassTemplate->findPartialSpecInstantiatedFromMember(D))
     return Result;
@@ -914,7 +908,7 @@ Decl *
 TemplateDeclInstantiator::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   // Create a local instantiation scope for this function template, which
   // will contain the instantiations of the template parameters and then get
-  // merged with the local instantiation scope for the function template 
+  // merged with the local instantiation scope for the function template
   // itself.
   LocalInstantiationScope Scope(SemaRef);
 
@@ -922,16 +916,16 @@ TemplateDeclInstantiator::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   TemplateParameterList *InstParams = SubstTemplateParams(TempParams);
   if (!InstParams)
     return NULL;
-  
+
   FunctionDecl *Instantiated = 0;
   if (CXXMethodDecl *DMethod = dyn_cast<CXXMethodDecl>(D->getTemplatedDecl()))
-    Instantiated = cast_or_null<FunctionDecl>(VisitCXXMethodDecl(DMethod, 
+    Instantiated = cast_or_null<FunctionDecl>(VisitCXXMethodDecl(DMethod,
                                                                  InstParams));
   else
     Instantiated = cast_or_null<FunctionDecl>(VisitFunctionDecl(
-                                                          D->getTemplatedDecl(), 
+                                                          D->getTemplatedDecl(),
                                                                 InstParams));
-  
+
   if (!Instantiated)
     return 0;
 
@@ -939,10 +933,10 @@ TemplateDeclInstantiator::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
 
   // Link the instantiated function template declaration to the function
   // template from which it was instantiated.
-  FunctionTemplateDecl *InstTemplate 
+  FunctionTemplateDecl *InstTemplate
     = Instantiated->getDescribedFunctionTemplate();
   InstTemplate->setAccess(D->getAccess());
-  assert(InstTemplate && 
+  assert(InstTemplate &&
          "VisitFunctionDecl/CXXMethodDecl didn't create a template!");
 
   bool isFriend = (InstTemplate->getFriendObjectKind() != Decl::FOK_None);
@@ -952,7 +946,7 @@ TemplateDeclInstantiator::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   if (!InstTemplate->getInstantiatedFromMemberTemplate() &&
       !(isFriend && !D->getTemplatedDecl()->isThisDeclarationADefinition()))
     InstTemplate->setInstantiatedFromMemberTemplate(D);
-  
+
   // Make declarations visible in the appropriate context.
   if (!isFriend)
     Owner->addDecl(InstTemplate);
@@ -1018,7 +1012,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
   FunctionTemplateDecl *FunctionTemplate = D->getDescribedFunctionTemplate();
   void *InsertPos = 0;
   if (FunctionTemplate && !TemplateParams) {
-    std::pair<const TemplateArgument *, unsigned> Innermost 
+    std::pair<const TemplateArgument *, unsigned> Innermost
       = TemplateArgs.getInnermost();
 
     FunctionDecl *SpecFunc
@@ -1038,11 +1032,11 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
 
   bool MergeWithParentScope = (TemplateParams != 0) ||
     Owner->isFunctionOrMethod() ||
-    !(isa<Decl>(Owner) && 
+    !(isa<Decl>(Owner) &&
       cast<Decl>(Owner)->isDefinedOutsideFunctionOrMethod());
   LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
 
-  llvm::SmallVector<ParmVarDecl *, 4> Params;
+  SmallVector<ParmVarDecl *, 4> Params;
   TypeSourceInfo *TInfo = D->getTypeSourceInfo();
   TInfo = SubstFunctionType(D, Params);
   if (!TInfo)
@@ -1068,7 +1062,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     DC = SemaRef.computeDeclContext(SS);
     if (!DC) return 0;
   } else {
-    DC = SemaRef.FindInstantiatedContext(D->getLocation(), D->getDeclContext(), 
+    DC = SemaRef.FindInstantiatedContext(D->getLocation(), D->getDeclContext(),
                                          TemplateArgs);
   }
 
@@ -1076,7 +1070,8 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
       FunctionDecl::Create(SemaRef.Context, DC, D->getInnerLocStart(),
                            D->getLocation(), D->getDeclName(), T, TInfo,
                            D->getStorageClass(), D->getStorageClassAsWritten(),
-                           D->isInlineSpecified(), D->hasWrittenPrototype());
+                           D->isInlineSpecified(), D->hasWrittenPrototype(),
+                           /*isConstexpr*/ false);
 
   if (QualifierLoc)
     Function->setQualifierInfo(QualifierLoc);
@@ -1110,7 +1105,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
       Params.push_back(Param);
     }
   }
-  Function->setParams(Params.data(), Params.size());
+  Function->setParams(Params);
 
   SourceLocation InstantiateAtPOI;
   if (TemplateParams) {
@@ -1124,7 +1119,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     //
     //   X<int> x;
     //
-    // We are instantiating the friend function template "f" within X<int>, 
+    // We are instantiating the friend function template "f" within X<int>,
     // which means substituting int for T, but leaving "f" as a friend function
     // template.
     // Build the function template itself.
@@ -1144,25 +1139,28 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     }
   } else if (FunctionTemplate) {
     // Record this function template specialization.
-    std::pair<const TemplateArgument *, unsigned> Innermost 
+    std::pair<const TemplateArgument *, unsigned> Innermost
       = TemplateArgs.getInnermost();
     Function->setFunctionTemplateSpecialization(FunctionTemplate,
                             TemplateArgumentList::CreateCopy(SemaRef.Context,
                                                              Innermost.first,
                                                              Innermost.second),
                                                 InsertPos);
-  } else if (isFriend && D->isThisDeclarationADefinition()) {
-    // TODO: should we remember this connection regardless of whether
-    // the friend declaration provided a body?
+  } else if (isFriend) {
+    // Note, we need this connection even if the friend doesn't have a body.
+    // Its body may exist but not have been attached yet due to deferred
+    // parsing.
+    // FIXME: It might be cleaner to set this when attaching the body to the
+    // friend function declaration, however that would require finding all the
+    // instantiations and modifying them.
     Function->setInstantiationOfMemberFunction(D, TSK_ImplicitInstantiation);
   }
-    
+
   if (InitFunctionInstantiation(Function, D))
     Function->setInvalidDecl();
 
-  bool Redeclaration = false;
   bool isExplicitSpecialization = false;
-    
+
   LookupResult Previous(SemaRef, Function->getDeclName(), SourceLocation(),
                         Sema::LookupOrdinaryName, Sema::ForRedeclaration);
 
@@ -1194,15 +1192,15 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
                                                     &ExplicitArgs,
                                                     Previous))
       Function->setInvalidDecl();
-                                          
+
     isExplicitSpecialization = true;
 
   } else if (TemplateParams || !FunctionTemplate) {
-    // Look only into the namespace where the friend would be declared to 
-    // find a previous declaration. This is the innermost enclosing namespace, 
+    // Look only into the namespace where the friend would be declared to
+    // find a previous declaration. This is the innermost enclosing namespace,
     // as described in ActOnFriendFunctionDecl.
     SemaRef.LookupQualifiedName(Previous, DC);
-    
+
     // In C++, the previous declaration we find might be a tag type
     // (class or enum). In this case, the new declaration will hide the
     // tag type. Note that this does does not apply if we're declaring a
@@ -1210,9 +1208,9 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     if (Previous.isSingleTagDecl())
       Previous.clear();
   }
-  
+
   SemaRef.CheckFunctionDeclaration(/*Scope*/ 0, Function, Previous,
-                                   isExplicitSpecialization, Redeclaration);
+                                   isExplicitSpecialization);
 
   NamedDecl *PrincipalDecl = (TemplateParams
                               ? cast<NamedDecl>(FunctionTemplate)
@@ -1238,11 +1236,11 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
       const FunctionDecl *Definition = 0;
       if (Function->isDefined(Definition) &&
           Definition->getTemplateSpecializationKind() == TSK_Undeclared) {
-        SemaRef.Diag(Function->getLocation(), diag::err_redefinition) 
+        SemaRef.Diag(Function->getLocation(), diag::err_redefinition)
           << Function->getDeclName();
         SemaRef.Diag(Definition->getLocation(), diag::note_previous_definition);
-        Function->setInvalidDecl();        
-      } 
+        Function->setInvalidDecl();
+      }
       // Check for redefinitions due to other instantiations of this or
       // a similar friend function.
       else for (FunctionDecl::redecl_iterator R = Function->redecls_begin(),
@@ -1269,7 +1267,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
           if (const FunctionDecl *RPattern
               = R->getTemplateInstantiationPattern())
             if (RPattern->isDefined(RPattern)) {
-              SemaRef.Diag(Function->getLocation(), diag::err_redefinition) 
+              SemaRef.Diag(Function->getLocation(), diag::err_redefinition)
                 << Function->getDeclName();
               SemaRef.Diag(R->getLocation(), diag::note_previous_definition);
               Function->setInvalidDecl();
@@ -1290,14 +1288,15 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
 
 Decl *
 TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
-                                      TemplateParameterList *TemplateParams) {
+                                      TemplateParameterList *TemplateParams,
+                                      bool IsClassScopeSpecialization) {
   FunctionTemplateDecl *FunctionTemplate = D->getDescribedFunctionTemplate();
   void *InsertPos = 0;
   if (FunctionTemplate && !TemplateParams) {
     // We are creating a function template specialization from a function
     // template. Check whether there is already a function template
     // specialization for this particular set of template arguments.
-    std::pair<const TemplateArgument *, unsigned> Innermost 
+    std::pair<const TemplateArgument *, unsigned> Innermost
       = TemplateArgs.getInnermost();
 
     FunctionDecl *SpecFunc
@@ -1316,12 +1315,12 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     isFriend = (D->getFriendObjectKind() != Decl::FOK_None);
 
   bool MergeWithParentScope = (TemplateParams != 0) ||
-    !(isa<Decl>(Owner) && 
+    !(isa<Decl>(Owner) &&
       cast<Decl>(Owner)->isDefinedOutsideFunctionOrMethod());
   LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
 
   // Instantiate enclosing template arguments for friends.
-  llvm::SmallVector<TemplateParameterList *, 4> TempParamLists;
+  SmallVector<TemplateParameterList *, 4> TempParamLists;
   unsigned NumTempParamLists = 0;
   if (isFriend && (NumTempParamLists = D->getNumTemplateParameterLists())) {
     TempParamLists.set_size(NumTempParamLists);
@@ -1334,7 +1333,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     }
   }
 
-  llvm::SmallVector<ParmVarDecl *, 4> Params;
+  SmallVector<ParmVarDecl *, 4> Params;
   TypeSourceInfo *TInfo = D->getTypeSourceInfo();
   TInfo = SubstFunctionType(D, Params);
   if (!TInfo)
@@ -1352,18 +1351,18 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   // synthesized in the method declaration.
   if (!isa<FunctionProtoType>(T.IgnoreParens())) {
     assert(!Params.size() && "Instantiating type could not yield parameters");
-    llvm::SmallVector<QualType, 4> ParamTypes;
-    if (SemaRef.SubstParmTypes(D->getLocation(), D->param_begin(), 
-                               D->getNumParams(), TemplateArgs, ParamTypes, 
+    SmallVector<QualType, 4> ParamTypes;
+    if (SemaRef.SubstParmTypes(D->getLocation(), D->param_begin(),
+                               D->getNumParams(), TemplateArgs, ParamTypes,
                                &Params))
-      return 0;    
+      return 0;
   }
 
   NestedNameSpecifierLoc QualifierLoc = D->getQualifierLoc();
   if (QualifierLoc) {
     QualifierLoc = SemaRef.SubstNestedNameSpecifierLoc(QualifierLoc,
                                                  TemplateArgs);
-    if (!QualifierLoc) 
+    if (!QualifierLoc)
       return 0;
   }
 
@@ -1396,7 +1395,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
                                         StartLoc, NameInfo, T, TInfo,
                                         Constructor->isExplicit(),
                                         Constructor->isInlineSpecified(),
-                                        false);
+                                        false, /*isConstexpr*/ false);
   } else if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(D)) {
     Method = CXXDestructorDecl::Create(SemaRef.Context, Record,
                                        StartLoc, NameInfo, T, TInfo,
@@ -1407,6 +1406,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
                                        StartLoc, NameInfo, T, TInfo,
                                        Conversion->isInlineSpecified(),
                                        Conversion->isExplicit(),
+                                       /*isConstexpr*/ false,
                                        Conversion->getLocEnd());
   } else {
     Method = CXXMethodDecl::Create(SemaRef.Context, Record,
@@ -1414,7 +1414,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
                                    D->isStatic(),
                                    D->getStorageClassAsWritten(),
                                    D->isInlineSpecified(),
-                                   D->getLocEnd());
+                                   /*isConstexpr*/ false, D->getLocEnd());
   }
 
   if (QualifierLoc)
@@ -1446,7 +1446,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     Method->setDescribedFunctionTemplate(FunctionTemplate);
   } else if (FunctionTemplate) {
     // Record this function template specialization.
-    std::pair<const TemplateArgument *, unsigned> Innermost 
+    std::pair<const TemplateArgument *, unsigned> Innermost
       = TemplateArgs.getInnermost();
     Method->setFunctionTemplateSpecialization(FunctionTemplate,
                          TemplateArgumentList::CreateCopy(SemaRef.Context,
@@ -1457,7 +1457,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     // Record that this is an instantiation of a member function.
     Method->setInstantiationOfMemberFunction(D, TSK_ImplicitInstantiation);
   }
-  
+
   // If we are instantiating a member function defined
   // out-of-line, the instantiation will have the same lexical
   // context (which will be a namespace scope) as the template.
@@ -1475,7 +1475,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   // Attach the parameters
   for (unsigned P = 0; P < Params.size(); ++P)
     Params[P]->setOwningFunction(Method);
-  Method->setParams(Params.data(), Params.size());
+  Method->setParams(Params);
 
   if (InitMethodInstantiation(Method, D))
     Method->setInvalidDecl();
@@ -1494,8 +1494,8 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
       Previous.clear();
   }
 
-  bool Redeclaration = false;
-  SemaRef.CheckFunctionDeclaration(0, Method, Previous, false, Redeclaration);
+  if (!IsClassScopeSpecialization)
+    SemaRef.CheckFunctionDeclaration(0, Method, Previous, false);
 
   if (D->isPure())
     SemaRef.CheckPureMethod(Method, SourceRange());
@@ -1514,7 +1514,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
                             : Method);
     if (isFriend)
       Record->makeDeclVisibleInContext(DeclToAdd);
-    else
+    else if (!IsClassScopeSpecialization)
       Owner->addDecl(DeclToAdd);
   }
 
@@ -1558,14 +1558,14 @@ Decl *TemplateDeclInstantiator::VisitTemplateTypeParmDecl(
                                  D->wasDeclaredWithTypename(),
                                  D->isParameterPack());
   Inst->setAccess(AS_public);
-  
-  if (D->hasDefaultArgument())
-    Inst->setDefaultArgument(D->getDefaultArgumentInfo(), false);  
 
-  // Introduce this template parameter's instantiation into the instantiation 
+  if (D->hasDefaultArgument())
+    Inst->setDefaultArgument(D->getDefaultArgumentInfo(), false);
+
+  // Introduce this template parameter's instantiation into the instantiation
   // scope.
   SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Inst);
-  
+
   return Inst;
 }
 
@@ -1573,26 +1573,26 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
                                                  NonTypeTemplateParmDecl *D) {
   // Substitute into the type of the non-type template parameter.
   TypeLoc TL = D->getTypeSourceInfo()->getTypeLoc();
-  llvm::SmallVector<TypeSourceInfo *, 4> ExpandedParameterPackTypesAsWritten;
-  llvm::SmallVector<QualType, 4> ExpandedParameterPackTypes;
+  SmallVector<TypeSourceInfo *, 4> ExpandedParameterPackTypesAsWritten;
+  SmallVector<QualType, 4> ExpandedParameterPackTypes;
   bool IsExpandedParameterPack = false;
-  TypeSourceInfo *DI;  
+  TypeSourceInfo *DI;
   QualType T;
   bool Invalid = false;
 
   if (D->isExpandedParameterPack()) {
-    // The non-type template parameter pack is an already-expanded pack 
+    // The non-type template parameter pack is an already-expanded pack
     // expansion of types. Substitute into each of the expanded types.
     ExpandedParameterPackTypes.reserve(D->getNumExpansionTypes());
     ExpandedParameterPackTypesAsWritten.reserve(D->getNumExpansionTypes());
     for (unsigned I = 0, N = D->getNumExpansionTypes(); I != N; ++I) {
       TypeSourceInfo *NewDI =SemaRef.SubstType(D->getExpansionTypeSourceInfo(I),
                                                TemplateArgs,
-                                               D->getLocation(), 
+                                               D->getLocation(),
                                                D->getDeclName());
       if (!NewDI)
         return 0;
-      
+
       ExpandedParameterPackTypesAsWritten.push_back(NewDI);
       QualType NewT =SemaRef.CheckNonTypeTemplateParameterType(NewDI->getType(),
                                                               D->getLocation());
@@ -1600,7 +1600,7 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
         return 0;
       ExpandedParameterPackTypes.push_back(NewT);
     }
-    
+
     IsExpandedParameterPack = true;
     DI = D->getTypeSourceInfo();
     T = DI->getType();
@@ -1610,9 +1610,9 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
     // types.
     PackExpansionTypeLoc Expansion = cast<PackExpansionTypeLoc>(TL);
     TypeLoc Pattern = Expansion.getPatternLoc();
-    llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+    SmallVector<UnexpandedParameterPack, 2> Unexpanded;
     SemaRef.collectUnexpandedParameterPacks(Pattern, Unexpanded);
-    
+
     // Determine whether the set of unexpanded parameter packs can and should
     // be expanded.
     bool Expand = true;
@@ -1622,22 +1622,21 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
     llvm::Optional<unsigned> NumExpansions = OrigNumExpansions;
     if (SemaRef.CheckParameterPacksForExpansion(Expansion.getEllipsisLoc(),
                                                 Pattern.getSourceRange(),
-                                                Unexpanded.data(),
-                                                Unexpanded.size(),
+                                                Unexpanded,
                                                 TemplateArgs,
-                                                Expand, RetainExpansion, 
+                                                Expand, RetainExpansion,
                                                 NumExpansions))
       return 0;
-    
+
     if (Expand) {
       for (unsigned I = 0; I != *NumExpansions; ++I) {
         Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, I);
         TypeSourceInfo *NewDI = SemaRef.SubstType(Pattern, TemplateArgs,
-                                                  D->getLocation(), 
+                                                  D->getLocation(),
                                                   D->getDeclName());
         if (!NewDI)
           return 0;
-        
+
         ExpandedParameterPackTypesAsWritten.push_back(NewDI);
         QualType NewT = SemaRef.CheckNonTypeTemplateParameterType(
                                                               NewDI->getType(),
@@ -1646,7 +1645,7 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
           return 0;
         ExpandedParameterPackTypes.push_back(NewT);
       }
-      
+
       // Note that we have an expanded parameter pack. The "type" of this
       // expanded parameter pack is the original expansion type, but callers
       // will end up using the expanded parameter pack types for type-checking.
@@ -1658,62 +1657,62 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
       // pattern and create a new pack expansion type.
       Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, -1);
       TypeSourceInfo *NewPattern = SemaRef.SubstType(Pattern, TemplateArgs,
-                                                     D->getLocation(), 
+                                                     D->getLocation(),
                                                      D->getDeclName());
       if (!NewPattern)
         return 0;
-      
+
       DI = SemaRef.CheckPackExpansion(NewPattern, Expansion.getEllipsisLoc(),
                                       NumExpansions);
       if (!DI)
         return 0;
-      
+
       T = DI->getType();
     }
   } else {
     // Simple case: substitution into a parameter that is not a parameter pack.
-    DI = SemaRef.SubstType(D->getTypeSourceInfo(), TemplateArgs, 
+    DI = SemaRef.SubstType(D->getTypeSourceInfo(), TemplateArgs,
                            D->getLocation(), D->getDeclName());
     if (!DI)
       return 0;
-    
+
     // Check that this type is acceptable for a non-type template parameter.
-    T = SemaRef.CheckNonTypeTemplateParameterType(DI->getType(), 
+    T = SemaRef.CheckNonTypeTemplateParameterType(DI->getType(),
                                                   D->getLocation());
     if (T.isNull()) {
       T = SemaRef.Context.IntTy;
       Invalid = true;
     }
   }
-  
+
   NonTypeTemplateParmDecl *Param;
   if (IsExpandedParameterPack)
-    Param = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner, 
+    Param = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner,
                                             D->getInnerLocStart(),
                                             D->getLocation(),
-                                    D->getDepth() - TemplateArgs.getNumLevels(), 
-                                            D->getPosition(), 
+                                    D->getDepth() - TemplateArgs.getNumLevels(),
+                                            D->getPosition(),
                                             D->getIdentifier(), T,
                                             DI,
                                             ExpandedParameterPackTypes.data(),
                                             ExpandedParameterPackTypes.size(),
                                     ExpandedParameterPackTypesAsWritten.data());
   else
-    Param = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner, 
+    Param = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner,
                                             D->getInnerLocStart(),
                                             D->getLocation(),
-                                    D->getDepth() - TemplateArgs.getNumLevels(), 
-                                            D->getPosition(), 
-                                            D->getIdentifier(), T, 
+                                    D->getDepth() - TemplateArgs.getNumLevels(),
+                                            D->getPosition(),
+                                            D->getIdentifier(), T,
                                             D->isParameterPack(), DI);
-  
+
   Param->setAccess(AS_public);
   if (Invalid)
     Param->setInvalidDecl();
-  
+
   Param->setDefaultArgument(D->getDefaultArgument(), false);
-  
-  // Introduce this template parameter's instantiation into the instantiation 
+
+  // Introduce this template parameter's instantiation into the instantiation
   // scope.
   SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Param);
   return Param;
@@ -1732,34 +1731,34 @@ TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
     InstParams = SubstTemplateParams(TempParams);
     if (!InstParams)
       return NULL;
-  }  
-  
+  }
+
   // Build the template template parameter.
   TemplateTemplateParmDecl *Param
     = TemplateTemplateParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
-                                   D->getDepth() - TemplateArgs.getNumLevels(), 
-                                       D->getPosition(), D->isParameterPack(), 
+                                   D->getDepth() - TemplateArgs.getNumLevels(),
+                                       D->getPosition(), D->isParameterPack(),
                                        D->getIdentifier(), InstParams);
   Param->setDefaultArgument(D->getDefaultArgument(), false);
   Param->setAccess(AS_public);
-  
-  // Introduce this template parameter's instantiation into the instantiation 
+
+  // Introduce this template parameter's instantiation into the instantiation
   // scope.
   SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Param);
-  
+
   return Param;
 }
 
 Decl *TemplateDeclInstantiator::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   // Using directives are never dependent (and never contain any types or
   // expressions), so they require no explicit instantiation work.
-  
+
   UsingDirectiveDecl *Inst
     = UsingDirectiveDecl::Create(SemaRef.Context, Owner, D->getLocation(),
-                                 D->getNamespaceKeyLocation(), 
+                                 D->getNamespaceKeyLocation(),
                                  D->getQualifierLoc(),
-                                 D->getIdentLocation(), 
-                                 D->getNominatedNamespace(), 
+                                 D->getIdentLocation(),
+                                 D->getNominatedNamespace(),
                                  D->getCommonAncestor());
   Owner->addDecl(Inst);
   return Inst;
@@ -1863,7 +1862,7 @@ Decl *TemplateDeclInstantiator::VisitUsingShadowDecl(UsingShadowDecl *D) {
 Decl * TemplateDeclInstantiator
     ::VisitUnresolvedUsingTypenameDecl(UnresolvedUsingTypenameDecl *D) {
   NestedNameSpecifierLoc QualifierLoc
-    = SemaRef.SubstNestedNameSpecifierLoc(D->getQualifierLoc(), 
+    = SemaRef.SubstNestedNameSpecifierLoc(D->getQualifierLoc(),
                                           TemplateArgs);
   if (!QualifierLoc)
     return 0;
@@ -1891,7 +1890,7 @@ Decl * TemplateDeclInstantiator
       = SemaRef.SubstNestedNameSpecifierLoc(D->getQualifierLoc(), TemplateArgs);
   if (!QualifierLoc)
     return 0;
-  
+
   CXXScopeSpec SS;
   SS.Adopt(QualifierLoc);
 
@@ -1909,6 +1908,29 @@ Decl * TemplateDeclInstantiator
   return UD;
 }
 
+
+Decl *TemplateDeclInstantiator::VisitClassScopeFunctionSpecializationDecl(
+                                     ClassScopeFunctionSpecializationDecl *Decl) {
+  CXXMethodDecl *OldFD = Decl->getSpecialization();
+  CXXMethodDecl *NewFD = cast<CXXMethodDecl>(VisitCXXMethodDecl(OldFD, 0, true));
+
+  LookupResult Previous(SemaRef, NewFD->getNameInfo(), Sema::LookupOrdinaryName,
+                        Sema::ForRedeclaration);
+
+  SemaRef.LookupQualifiedName(Previous, SemaRef.CurContext);
+  if (SemaRef.CheckFunctionTemplateSpecialization(NewFD, 0, Previous)) {
+    NewFD->setInvalidDecl();
+    return NewFD;
+  }
+
+  // Associate the specialization with the pattern.
+  FunctionDecl *Specialization = cast<FunctionDecl>(Previous.getFoundDecl());
+  assert(Specialization && "Class scope Specialization is null");
+  SemaRef.Context.setClassScopeSpecializationPattern(Specialization, OldFD);
+
+  return NewFD;
+}
+
 Decl *Sema::SubstDecl(Decl *D, DeclContext *Owner,
                       const MultiLevelTemplateArgumentList &TemplateArgs) {
   TemplateDeclInstantiator Instantiator(*this, Owner, TemplateArgs);
@@ -1930,7 +1952,7 @@ TemplateDeclInstantiator::SubstTemplateParams(TemplateParameterList *L) {
   bool Invalid = false;
 
   unsigned N = L->size();
-  typedef llvm::SmallVector<NamedDecl *, 8> ParamVector;
+  typedef SmallVector<NamedDecl *, 8> ParamVector;
   ParamVector Params;
   Params.reserve(N);
   for (TemplateParameterList::iterator PI = L->begin(), PE = L->end();
@@ -1951,13 +1973,13 @@ TemplateDeclInstantiator::SubstTemplateParams(TemplateParameterList *L) {
   return InstL;
 }
 
-/// \brief Instantiate the declaration of a class template partial 
+/// \brief Instantiate the declaration of a class template partial
 /// specialization.
 ///
 /// \param ClassTemplate the (instantiated) class template that is partially
 // specialized by the instantiation of \p PartialSpec.
 ///
-/// \param PartialSpec the (uninstantiated) class template partial 
+/// \param PartialSpec the (uninstantiated) class template partial
 /// specialization that we are instantiating.
 ///
 /// \returns The instantiated partial specialization, if successful; otherwise,
@@ -1970,28 +1992,28 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   // specialization, which will contain the instantiations of the template
   // parameters.
   LocalInstantiationScope Scope(SemaRef);
-  
+
   // Substitute into the template parameters of the class template partial
   // specialization.
   TemplateParameterList *TempParams = PartialSpec->getTemplateParameters();
   TemplateParameterList *InstParams = SubstTemplateParams(TempParams);
   if (!InstParams)
     return 0;
-  
+
   // Substitute into the template arguments of the class template partial
   // specialization.
   TemplateArgumentListInfo InstTemplateArgs; // no angle locations
-  if (SemaRef.Subst(PartialSpec->getTemplateArgsAsWritten(), 
-                    PartialSpec->getNumTemplateArgsAsWritten(), 
+  if (SemaRef.Subst(PartialSpec->getTemplateArgsAsWritten(),
+                    PartialSpec->getNumTemplateArgsAsWritten(),
                     InstTemplateArgs, TemplateArgs))
     return 0;
-  
+
   // Check that the template argument list is well-formed for this
   // class template.
-  llvm::SmallVector<TemplateArgument, 4> Converted;
-  if (SemaRef.CheckTemplateArgumentList(ClassTemplate, 
+  SmallVector<TemplateArgument, 4> Converted;
+  if (SemaRef.CheckTemplateArgumentList(ClassTemplate,
                                         PartialSpec->getLocation(),
-                                        InstTemplateArgs, 
+                                        InstTemplateArgs,
                                         false,
                                         Converted))
     return 0;
@@ -2002,10 +2024,10 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   ClassTemplateSpecializationDecl *PrevDecl
     = ClassTemplate->findPartialSpecialization(Converted.data(),
                                                Converted.size(), InsertPos);
-  
+
   // Build the canonical type that describes the converted template
   // arguments of the class template partial specialization.
-  QualType CanonType 
+  QualType CanonType
     = SemaRef.Context.getTemplateSpecializationType(TemplateName(ClassTemplate),
                                                     Converted.data(),
                                                     Converted.size());
@@ -2023,7 +2045,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
                                                     PartialSpec->getLocation(),
                                                     InstTemplateArgs,
                                                     CanonType);
-  
+
   if (PrevDecl) {
     // We've already seen a partial specialization with the same template
     // parameters and template arguments. This can happen, for example, when
@@ -2046,17 +2068,17 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
       << SemaRef.Context.getTypeDeclType(PrevDecl);
     return 0;
   }
-  
-  
+
+
   // Create the class template partial specialization declaration.
   ClassTemplatePartialSpecializationDecl *InstPartialSpec
-    = ClassTemplatePartialSpecializationDecl::Create(SemaRef.Context, 
+    = ClassTemplatePartialSpecializationDecl::Create(SemaRef.Context,
                                                      PartialSpec->getTagKind(),
-                                                     Owner, 
+                                                     Owner,
                                                      PartialSpec->getLocStart(),
                                                      PartialSpec->getLocation(),
                                                      InstParams,
-                                                     ClassTemplate, 
+                                                     ClassTemplate,
                                                      Converted.data(),
                                                      Converted.size(),
                                                      InstTemplateArgs,
@@ -2069,7 +2091,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
 
   InstPartialSpec->setInstantiatedFromMember(PartialSpec);
   InstPartialSpec->setTypeAsWritten(WrittenTy);
-  
+
   // Add this partial specialization to the set of class template partial
   // specializations.
   ClassTemplate->AddPartialSpecialization(InstPartialSpec, InsertPos);
@@ -2078,7 +2100,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
 
 TypeSourceInfo*
 TemplateDeclInstantiator::SubstFunctionType(FunctionDecl *D,
-                              llvm::SmallVectorImpl<ParmVarDecl *> &Params) {
+                              SmallVectorImpl<ParmVarDecl *> &Params) {
   TypeSourceInfo *OldTInfo = D->getTypeSourceInfo();
   assert(OldTInfo && "substituting function without type source info");
   assert(Params.empty() && "parameter vector is non-empty at start");
@@ -2104,7 +2126,7 @@ TemplateDeclInstantiator::SubstFunctionType(FunctionDecl *D,
         if (!OldParam->isParameterPack() ||
             (NewIdx < NumNewParams &&
              NewProtoLoc->getArg(NewIdx)->isParameterPack())) {
-          // Simple case: normal parameter, or a parameter pack that's 
+          // Simple case: normal parameter, or a parameter pack that's
           // instantiated to a (still-dependent) parameter pack.
           ParmVarDecl *NewParam = NewProtoLoc->getArg(NewIdx++);
           Params.push_back(NewParam);
@@ -2112,7 +2134,7 @@ TemplateDeclInstantiator::SubstFunctionType(FunctionDecl *D,
                                                                NewParam);
           continue;
         }
-        
+
         // Parameter pack: make the instantiation an argument pack.
         SemaRef.CurrentInstantiationScope->MakeInstantiatedLocalArgPack(
                                                                       OldParam);
@@ -2184,43 +2206,42 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
   if (Proto->hasExceptionSpec() || Proto->getNoReturnAttr()) {
     // The function has an exception specification or a "noreturn"
     // attribute. Substitute into each of the exception types.
-    llvm::SmallVector<QualType, 4> Exceptions;
+    SmallVector<QualType, 4> Exceptions;
     for (unsigned I = 0, N = Proto->getNumExceptions(); I != N; ++I) {
       // FIXME: Poor location information!
       if (const PackExpansionType *PackExpansion
             = Proto->getExceptionType(I)->getAs<PackExpansionType>()) {
         // We have a pack expansion. Instantiate it.
-        llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+        SmallVector<UnexpandedParameterPack, 2> Unexpanded;
         SemaRef.collectUnexpandedParameterPacks(PackExpansion->getPattern(),
                                                 Unexpanded);
-        assert(!Unexpanded.empty() && 
+        assert(!Unexpanded.empty() &&
                "Pack expansion without parameter packs?");
 
         bool Expand = false;
         bool RetainExpansion = false;
         llvm::Optional<unsigned> NumExpansions
                                           = PackExpansion->getNumExpansions();
-        if (SemaRef.CheckParameterPacksForExpansion(New->getLocation(), 
+        if (SemaRef.CheckParameterPacksForExpansion(New->getLocation(),
                                                     SourceRange(),
-                                                    Unexpanded.data(), 
-                                                    Unexpanded.size(),
+                                                    Unexpanded,
                                                     TemplateArgs,
-                                                    Expand, 
+                                                    Expand,
                                                     RetainExpansion,
                                                     NumExpansions))
           break;
 
         if (!Expand) {
           // We can't expand this pack expansion into separate arguments yet;
-          // just substitute into the pattern and create a new pack expansion 
+          // just substitute into the pattern and create a new pack expansion
           // type.
           Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, -1);
-          QualType T = SemaRef.SubstType(PackExpansion->getPattern(), 
+          QualType T = SemaRef.SubstType(PackExpansion->getPattern(),
                                          TemplateArgs,
                                        New->getLocation(), New->getDeclName());
           if (T.isNull())
             break;
-          
+
           T = SemaRef.Context.getPackExpansionType(T, NumExpansions);
           Exceptions.push_back(T);
           continue;
@@ -2230,8 +2251,8 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
         bool Invalid = false;
         for (unsigned ArgIdx = 0; ArgIdx != *NumExpansions; ++ArgIdx) {
           Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, ArgIdx);
-          
-          QualType T = SemaRef.SubstType(PackExpansion->getPattern(), 
+
+          QualType T = SemaRef.SubstType(PackExpansion->getPattern(),
                                          TemplateArgs,
                                        New->getLocation(), New->getDeclName());
           if (T.isNull()) {
@@ -2247,11 +2268,11 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
 
         continue;
       }
-      
+
       QualType T
         = SemaRef.SubstType(Proto->getExceptionType(I), TemplateArgs,
                             New->getLocation(), New->getDeclName());
-      if (T.isNull() || 
+      if (T.isNull() ||
           SemaRef.CheckSpecifiedExceptionType(T, New->getLocation()))
         continue;
 
@@ -2262,10 +2283,24 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
       EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
       ExprResult E = SemaRef.SubstExpr(OldNoexceptExpr, TemplateArgs);
       if (E.isUsable())
+        E = SemaRef.CheckBooleanCondition(E.get(), E.get()->getLocStart());
+    
+      if (E.isUsable()) {
+        SourceLocation ErrLoc;
+        llvm::APSInt NoexceptVal;
         NoexceptExpr = E.take();
+        if (!NoexceptExpr->isTypeDependent() &&
+            !NoexceptExpr->isValueDependent() &&
+            !NoexceptExpr->isIntegerConstantExpr(NoexceptVal, SemaRef.Context,
+                                                 &ErrLoc, /*evaluated=*/false)){
+          SemaRef.Diag(ErrLoc, diag::err_noexcept_needs_constant_expression)
+            << NoexceptExpr->getSourceRange();
+          NoexceptExpr = 0;
+        }
+      }
     }
 
-    // Rebuild the function type 
+    // Rebuild the function type
 
     FunctionProtoType::ExtProtoInfo EPI = Proto->getExtProtoInfo();
     EPI.ExceptionSpecType = Proto->getExceptionSpecType();
@@ -2283,6 +2318,13 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
                                                  EPI));
   }
 
+  // C++0x [dcl.constexpr]p6: If the instantiated template specialization of
+  // a constexpr function template satisfies the requirements for a constexpr
+  // function, then it is a constexpr function.
+  if (Tmpl->isConstexpr() &&
+      SemaRef.CheckConstexprFunctionDecl(New, Sema::CCK_Instantiation))
+    New->setConstexpr(true);
+
   const FunctionDecl* Definition = Tmpl;
 
   // Get the definition. Leaves the variable unchanged if undefined.
@@ -2337,8 +2379,10 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   if (Function->isInvalidDecl() || Function->isDefined())
     return;
 
-  // Never instantiate an explicit specialization.
-  if (Function->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
+  // Never instantiate an explicit specialization except if it is a class scope
+  // explicit specialization.
+  if (Function->getTemplateSpecializationKind() == TSK_ExplicitSpecialization &&
+      !Function->getClassScopeSpecializationPattern())
     return;
 
   // Find the function body that we'll be substituting.
@@ -2362,7 +2406,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   // Call the LateTemplateParser callback if there a need to late parse
-  // a templated function definition. 
+  // a templated function definition.
   if (!Pattern && PatternDecl->isLateTemplateParsed() &&
       LateTemplateParser) {
     LateTemplateParser(OpaqueParser, PatternDecl);
@@ -2372,16 +2416,16 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   if (!Pattern && !PatternDecl->isDefaulted()) {
     if (DefinitionRequired) {
       if (Function->getPrimaryTemplate())
-        Diag(PointOfInstantiation, 
+        Diag(PointOfInstantiation,
              diag::err_explicit_instantiation_undefined_func_template)
           << Function->getPrimaryTemplate();
       else
-        Diag(PointOfInstantiation, 
+        Diag(PointOfInstantiation,
              diag::err_explicit_instantiation_undefined_member)
           << 1 << Function->getDeclName() << Function->getDeclContext();
-      
+
       if (PatternDecl)
-        Diag(PatternDecl->getLocation(), 
+        Diag(PatternDecl->getLocation(),
              diag::note_explicit_instantiation_here);
       Function->setInvalidDecl();
     } else if (Function->getTemplateSpecializationKind()
@@ -2404,19 +2448,19 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
 
   InstantiatingTemplate Inst(*this, PointOfInstantiation, Function);
   if (Inst)
-    return;  
-  
+    return;
+
   // If we're performing recursive template instantiation, create our own
   // queue of pending implicit instantiations that we will instantiate later,
   // while we're still within our own instantiation context.
-  llvm::SmallVector<VTableUse, 16> SavedVTableUses;
+  SmallVector<VTableUse, 16> SavedVTableUses;
   std::deque<PendingImplicitInstantiation> SavedPendingInstantiations;
   if (Recursive) {
     VTableUses.swap(SavedVTableUses);
     PendingInstantiations.swap(SavedPendingInstantiations);
   }
 
-  EnterExpressionEvaluationContext EvalContext(*this, 
+  EnterExpressionEvaluationContext EvalContext(*this,
                                                Sema::PotentiallyEvaluated);
   ActOnStartOfFunctionDef(0, Function);
 
@@ -2445,11 +2489,11 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
       ++FParamIdx;
       continue;
     }
-    
+
     // Expand the parameter pack.
     Scope.MakeInstantiatedLocalArgPack(PatternParam);
-    for (unsigned NumFParams = Function->getNumParams(); 
-         FParamIdx < NumFParams; 
+    for (unsigned NumFParams = Function->getNumParams();
+         FParamIdx < NumFParams;
          ++FParamIdx) {
       ParmVarDecl *FunctionParam = Function->getParamDecl(FParamIdx);
       FunctionParam->setDeclName(PatternParam->getDeclName());
@@ -2481,7 +2525,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
 
     if (Body.isInvalid())
       Function->setInvalidDecl();
-    
+
     ActOnFinishFunctionBody(Function, Body.get(),
                             /*IsInstantiation=*/true);
   }
@@ -2545,7 +2589,7 @@ void Sema::InstantiateStaticDataMemberDefinition(
   // Find the out-of-line definition of this static data member.
   VarDecl *Def = Var->getInstantiatedFromStaticDataMember();
   assert(Def && "This data member was not instantiated from a template?");
-  assert(Def->isStaticDataMember() && "Not a static data member?");  
+  assert(Def->isStaticDataMember() && "Not a static data member?");
   Def = Def->getOutOfLineDefinition();
 
   if (!Def) {
@@ -2555,7 +2599,7 @@ void Sema::InstantiateStaticDataMemberDefinition(
     // another translation unit.
     if (DefinitionRequired) {
       Def = Var->getInstantiatedFromStaticDataMember();
-      Diag(PointOfInstantiation, 
+      Diag(PointOfInstantiation,
            diag::err_explicit_instantiation_undefined_member)
         << 2 << Var->getDeclName() << Var->getDeclContext();
       Diag(Def->getLocation(), diag::note_explicit_instantiation_here);
@@ -2571,12 +2615,12 @@ void Sema::InstantiateStaticDataMemberDefinition(
   // Never instantiate an explicit specialization.
   if (Var->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
     return;
-  
+
   // C++0x [temp.explicit]p9:
   //   Except for inline functions, other explicit instantiation declarations
   //   have the effect of suppressing the implicit instantiation of the entity
   //   to which they refer.
-  if (Var->getTemplateSpecializationKind() 
+  if (Var->getTemplateSpecializationKind()
         == TSK_ExplicitInstantiationDeclaration)
     return;
 
@@ -2591,7 +2635,7 @@ void Sema::InstantiateStaticDataMemberDefinition(
   // If we're performing recursive template instantiation, create our own
   // queue of pending implicit instantiations that we will instantiate later,
   // while we're still within our own instantiation context.
-  llvm::SmallVector<VTableUse, 16> SavedVTableUses;
+  SmallVector<VTableUse, 16> SavedVTableUses;
   std::deque<PendingImplicitInstantiation> SavedPendingInstantiations;
   if (Recursive) {
     VTableUses.swap(SavedVTableUses);
@@ -2639,14 +2683,26 @@ void Sema::InstantiateStaticDataMemberDefinition(
   }
 }
 
+static MultiInitializer CreateMultiInitializer(SmallVectorImpl<Expr*> &Args,
+                                               const CXXCtorInitializer *Init) {
+  // FIXME: This is a hack that will do slightly the wrong thing for an
+  // initializer of the form foo({...}).
+  // The right thing to do would be to modify InstantiateInitializer to create
+  // the MultiInitializer.
+  if (Args.size() == 1 && isa<InitListExpr>(Args[0]))
+    return MultiInitializer(Args[0]);
+  return MultiInitializer(Init->getLParenLoc(), Args.data(),
+                          Args.size(), Init->getRParenLoc());
+}
+
 void
 Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
                                  const CXXConstructorDecl *Tmpl,
                            const MultiLevelTemplateArgumentList &TemplateArgs) {
 
-  llvm::SmallVector<MemInitTy*, 4> NewInits;
+  SmallVector<CXXCtorInitializer*, 4> NewInits;
   bool AnyErrors = false;
-  
+
   // Instantiate all the initializers.
   for (CXXConstructorDecl::init_const_iterator Inits = Tmpl->init_begin(),
                                             InitsEnd = Tmpl->init_end();
@@ -2662,20 +2718,19 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
     ASTOwningVector<Expr*> NewArgs(*this);
 
     SourceLocation EllipsisLoc;
-    
+
     if (Init->isPackExpansion()) {
       // This is a pack expansion. We should expand it now.
       TypeLoc BaseTL = Init->getBaseClassInfo()->getTypeLoc();
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       collectUnexpandedParameterPacks(BaseTL, Unexpanded);
       bool ShouldExpand = false;
       bool RetainExpansion = false;
       llvm::Optional<unsigned> NumExpansions;
-      if (CheckParameterPacksForExpansion(Init->getEllipsisLoc(), 
+      if (CheckParameterPacksForExpansion(Init->getEllipsisLoc(),
                                           BaseTL.getSourceRange(),
-                                          Unexpanded.data(), 
-                                          Unexpanded.size(),
-                                          TemplateArgs, ShouldExpand, 
+                                          Unexpanded,
+                                          TemplateArgs, ShouldExpand,
                                           RetainExpansion,
                                           NumExpansions)) {
         AnyErrors = true;
@@ -2683,22 +2738,22 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
         continue;
       }
       assert(ShouldExpand && "Partial instantiation of base initializer?");
-      
-      // Loop over all of the arguments in the argument pack(s), 
+
+      // Loop over all of the arguments in the argument pack(s),
       for (unsigned I = 0; I != *NumExpansions; ++I) {
         Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(*this, I);
 
         // Instantiate the initializer.
-        if (InstantiateInitializer(*this, Init->getInit(), TemplateArgs, 
+        if (InstantiateInitializer(Init->getInit(), TemplateArgs,
                                    LParenLoc, NewArgs, RParenLoc)) {
           AnyErrors = true;
           break;
         }
 
         // Instantiate the base type.
-        TypeSourceInfo *BaseTInfo = SubstType(Init->getBaseClassInfo(), 
-                                              TemplateArgs, 
-                                              Init->getSourceLocation(), 
+        TypeSourceInfo *BaseTInfo = SubstType(Init->getBaseClassInfo(),
+                                              TemplateArgs,
+                                              Init->getSourceLocation(),
                                               New->getDeclName());
         if (!BaseTInfo) {
           AnyErrors = true;
@@ -2706,52 +2761,45 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
         }
 
         // Build the initializer.
-        MemInitResult NewInit = BuildBaseInitializer(BaseTInfo->getType(), 
-                                                     BaseTInfo,
-                                                     (Expr **)NewArgs.data(),
-                                                     NewArgs.size(),
-                                                     Init->getLParenLoc(),
-                                                     Init->getRParenLoc(),
+        MultiInitializer MultiInit(CreateMultiInitializer(NewArgs, Init));
+        MemInitResult NewInit = BuildBaseInitializer(BaseTInfo->getType(),
+                                                     BaseTInfo, MultiInit,
                                                      New->getParent(),
                                                      SourceLocation());
         if (NewInit.isInvalid()) {
           AnyErrors = true;
           break;
         }
-        
+
         NewInits.push_back(NewInit.get());
         NewArgs.clear();
       }
-      
+
       continue;
     }
 
     // Instantiate the initializer.
-    if (InstantiateInitializer(*this, Init->getInit(), TemplateArgs, 
+    if (InstantiateInitializer(Init->getInit(), TemplateArgs,
                                LParenLoc, NewArgs, RParenLoc)) {
       AnyErrors = true;
       continue;
     }
-    
+
     MemInitResult NewInit;
     if (Init->isBaseInitializer()) {
-      TypeSourceInfo *BaseTInfo = SubstType(Init->getBaseClassInfo(), 
-                                            TemplateArgs, 
-                                            Init->getSourceLocation(), 
+      TypeSourceInfo *BaseTInfo = SubstType(Init->getBaseClassInfo(),
+                                            TemplateArgs,
+                                            Init->getSourceLocation(),
                                             New->getDeclName());
       if (!BaseTInfo) {
         AnyErrors = true;
         New->setInvalidDecl();
         continue;
       }
-      
-      NewInit = BuildBaseInitializer(BaseTInfo->getType(), BaseTInfo,
-                                     (Expr **)NewArgs.data(),
-                                     NewArgs.size(),
-                                     Init->getLParenLoc(),
-                                     Init->getRParenLoc(),
-                                     New->getParent(),
-                                     EllipsisLoc);
+
+      MultiInitializer MultiInit(CreateMultiInitializer(NewArgs, Init));
+      NewInit = BuildBaseInitializer(BaseTInfo->getType(), BaseTInfo, MultiInit,
+                                     New->getParent(), EllipsisLoc);
     } else if (Init->isMemberInitializer()) {
       FieldDecl *Member = cast_or_null<FieldDecl>(FindInstantiatedDecl(
                                                      Init->getMemberLocation(),
@@ -2763,11 +2811,9 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
         continue;
       }
 
-      NewInit = BuildMemberInitializer(Member, (Expr **)NewArgs.data(),
-                                       NewArgs.size(),
-                                       Init->getSourceLocation(),
-                                       Init->getLParenLoc(),
-                                       Init->getRParenLoc());
+      MultiInitializer MultiInit(CreateMultiInitializer(NewArgs, Init));
+      NewInit = BuildMemberInitializer(Member, MultiInit,
+                                       Init->getSourceLocation());
     } else if (Init->isIndirectMemberInitializer()) {
       IndirectFieldDecl *IndirectMember =
          cast_or_null<IndirectFieldDecl>(FindInstantiatedDecl(
@@ -2777,14 +2823,12 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
       if (!IndirectMember) {
         AnyErrors = true;
         New->setInvalidDecl();
-        continue;        
+        continue;
       }
-      
-      NewInit = BuildMemberInitializer(IndirectMember, (Expr **)NewArgs.data(),
-                                       NewArgs.size(),
-                                       Init->getSourceLocation(),
-                                       Init->getLParenLoc(),
-                                       Init->getRParenLoc());
+
+      MultiInitializer MultiInit(CreateMultiInitializer(NewArgs, Init));
+      NewInit = BuildMemberInitializer(IndirectMember, MultiInit,
+                                       Init->getSourceLocation());
     }
 
     if (NewInit.isInvalid()) {
@@ -2794,7 +2838,7 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
       // FIXME: It would be nice if ASTOwningVector had a release function.
       NewArgs.take();
 
-      NewInits.push_back((MemInitTy *)NewInit.get());
+      NewInits.push_back(NewInit.get());
     }
   }
 
@@ -2824,20 +2868,20 @@ static bool isInstantiationOf(ClassTemplateDecl *Pattern,
 static bool isInstantiationOf(FunctionTemplateDecl *Pattern,
                               FunctionTemplateDecl *Instance) {
   Pattern = Pattern->getCanonicalDecl();
-  
+
   do {
     Instance = Instance->getCanonicalDecl();
     if (Pattern == Instance) return true;
     Instance = Instance->getInstantiatedFromMemberTemplate();
   } while (Instance);
-  
+
   return false;
 }
 
-static bool 
+static bool
 isInstantiationOf(ClassTemplatePartialSpecializationDecl *Pattern,
                   ClassTemplatePartialSpecializationDecl *Instance) {
-  Pattern 
+  Pattern
     = cast<ClassTemplatePartialSpecializationDecl>(Pattern->getCanonicalDecl());
   do {
     Instance = cast<ClassTemplatePartialSpecializationDecl>(
@@ -2846,7 +2890,7 @@ isInstantiationOf(ClassTemplatePartialSpecializationDecl *Pattern,
       return true;
     Instance = Instance->getInstantiatedFromMember();
   } while (Instance);
-  
+
   return false;
 }
 
@@ -3052,11 +3096,11 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
     typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
     llvm::PointerUnion<Decl *, DeclArgumentPack *> *Found
       = CurrentInstantiationScope->findInstantiationOf(D);
-    
+
     if (Found) {
       if (Decl *FD = Found->dyn_cast<Decl *>())
         return cast<NamedDecl>(FD);
-      
+
       unsigned PackIdx = ArgumentPackSubstitutionIndex;
       return cast<NamedDecl>((*Found->get<DeclArgumentPack *>())[PackIdx]);
     }
@@ -3064,10 +3108,10 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
     // If we didn't find the decl, then we must have a label decl that hasn't
     // been found yet.  Lazily instantiate it and return it now.
     assert(isa<LabelDecl>(D));
-    
+
     Decl *Inst = SubstDecl(D, CurContext, TemplateArgs);
     assert(Inst && "Failed to instantiate label??");
-    
+
     CurrentInstantiationScope->InstantiatedLocal(D, Inst);
     return cast<LabelDecl>(Inst);
   }
@@ -3075,7 +3119,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
   if (CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(D)) {
     if (!Record->isDependentContext())
       return D;
-    
+
     // If the RecordDecl is actually the injected-class-name or a
     // "templated" declaration for a class template, class template
     // partial specialization, or a member class of a class template,
@@ -3083,7 +3127,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
     // or partial specialization to find the new DeclContext.
     QualType T;
     ClassTemplateDecl *ClassTemplate = Record->getDescribedClassTemplate();
-    
+
     if (ClassTemplate) {
       T = ClassTemplate->getInjectedClassNameSpecialization();
     } else if (ClassTemplatePartialSpecializationDecl *PartialSpec
@@ -3096,26 +3140,26 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
       assert(isa<InjectedClassNameType>(T) &&
              "type of partial specialization is not an InjectedClassNameType");
       T = cast<InjectedClassNameType>(T)->getInjectedSpecializationType();
-    }  
-    
+    }
+
     if (!T.isNull()) {
       // Substitute into the injected-class-name to get the type
       // corresponding to the instantiation we want, which may also be
       // the current instantiation (if we're in a template
       // definition). This substitution should never fail, since we
       // know we can instantiate the injected-class-name or we
-      // wouldn't have gotten to the injected-class-name!  
+      // wouldn't have gotten to the injected-class-name!
 
       // FIXME: Can we use the CurrentInstantiationScope to avoid this
       // extra instantiation in the common case?
       T = SubstType(T, TemplateArgs, Loc, DeclarationName());
       assert(!T.isNull() && "Instantiation of injected-class-name cannot fail.");
-    
+
       if (!T->isDependentType()) {
         assert(T->isRecordType() && "Instantiation must produce a record type");
         return T->getAs<RecordType>()->getDecl();
       }
-    
+
       // We are performing "partial" template instantiation to create
       // the member declarations for the members of a class template
       // specialization. Therefore, D is actually referring to something
@@ -3128,7 +3172,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
            DC = DC->getParent()) {
         if (ClassTemplateSpecializationDecl *Spec
                           = dyn_cast<ClassTemplateSpecializationDecl>(DC))
-          if (isInstantiationOf(ClassTemplate, 
+          if (isInstantiationOf(ClassTemplate,
                                 Spec->getSpecializedTemplate()))
             return Spec;
 
@@ -3139,10 +3183,10 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
       // We're performing "instantiation" of a member of the current
       // instantiation while we are type-checking the
       // definition. Compute the declaration context and return that.
-      assert(!SawNonDependentContext && 
+      assert(!SawNonDependentContext &&
              "No dependent context while instantiating record");
       DeclContext *DC = computeDeclContext(T);
-      assert(DC && 
+      assert(DC &&
              "Unable to find declaration for the current instantiation");
       return cast<CXXRecordDecl>(DC);
     }
@@ -3153,7 +3197,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
 
   if (!ParentDC->isDependentContext())
     return D;
-  
+
   ParentDC = FindInstantiatedContext(Loc, ParentDC, TemplateArgs);
   if (!ParentDC)
     return 0;
@@ -3207,7 +3251,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
         // declaration failed to instantiate. There's no point in complaining
         // further, since this is normal in invalid code.
       } else if (IsBeingInstantiated) {
-        // The class in which this member exists is currently being 
+        // The class in which this member exists is currently being
         // instantiated, and we haven't gotten around to instantiating this
         // member yet. This can happen when the code uses forward declarations
         // of member classes, and introduces ordering dependencies via
@@ -3221,7 +3265,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
         llvm_unreachable("Unable to find instantiation of declaration!");
       }
     }
-    
+
     D = Result;
   }
 
@@ -3231,6 +3275,14 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
 /// \brief Performs template instantiation for all implicit template
 /// instantiations we have seen until this point.
 void Sema::PerformPendingInstantiations(bool LocalOnly) {
+  // Load pending instantiations from the external source.
+  if (!LocalOnly && ExternalSource) {
+    SmallVector<std::pair<ValueDecl *, SourceLocation>, 4> Pending;
+    ExternalSource->ReadPendingInstantiations(Pending);
+    PendingInstantiations.insert(PendingInstantiations.begin(),
+                                 Pending.begin(), Pending.end());
+  }
+
   while (!PendingLocalImplicitInstantiations.empty() ||
          (!LocalOnly && !PendingInstantiations.empty())) {
     PendingImplicitInstantiation Inst;
@@ -3267,7 +3319,7 @@ void Sema::PerformPendingInstantiations(bool LocalOnly) {
     // and removed the need for implicit instantiation.
     switch (Var->getMostRecentDeclaration()->getTemplateSpecializationKind()) {
     case TSK_Undeclared:
-      assert(false && "Cannot instantitiate an undeclared specialization.");
+      llvm_unreachable("Cannot instantitiate an undeclared specialization.");
     case TSK_ExplicitInstantiationDeclaration:
     case TSK_ExplicitSpecialization:
       continue;  // No longer need to instantiate this type.
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateVariadic.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateVariadic.cpp
index daa1523363c0..e383db935064 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -32,11 +32,11 @@ namespace {
     typedef RecursiveASTVisitor<CollectUnexpandedParameterPacksVisitor>
       inherited;
 
-    llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded;
+    SmallVectorImpl<UnexpandedParameterPack> &Unexpanded;
 
   public:
     explicit CollectUnexpandedParameterPacksVisitor(
-                  llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded)
+                  SmallVectorImpl<UnexpandedParameterPack> &Unexpanded)
       : Unexpanded(Unexpanded) { }
 
     bool shouldWalkTypesOfTypeLocs() const { return false; }
@@ -158,9 +158,9 @@ namespace {
 static void 
 DiagnoseUnexpandedParameterPacks(Sema &S, SourceLocation Loc,
                                  Sema::UnexpandedParameterPackContext UPPC,
-             const llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
-  llvm::SmallVector<SourceLocation, 4> Locations;
-  llvm::SmallVector<IdentifierInfo *, 4> Names;
+             const SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  SmallVector<SourceLocation, 4> Locations;
+  SmallVector<IdentifierInfo *, 4> Names;
   llvm::SmallPtrSet<IdentifierInfo *, 4> NamesKnown;
 
   for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
@@ -201,7 +201,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(SourceLocation Loc,
   if (!T->getType()->containsUnexpandedParameterPack())
     return false;
 
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseTypeLoc(
                                                               T->getTypeLoc());
   assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
@@ -217,7 +217,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(Expr *E,
   if (!E->containsUnexpandedParameterPack())
     return false;
 
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseStmt(E);
   assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
   DiagnoseUnexpandedParameterPacks(*this, E->getLocStart(), UPPC, Unexpanded);
@@ -233,7 +233,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(const CXXScopeSpec &SS,
       !SS.getScopeRep()->containsUnexpandedParameterPack())
     return false;
 
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded)
     .TraverseNestedNameSpecifier(SS.getScopeRep());
   assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
@@ -270,7 +270,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(const DeclarationNameInfo &NameInfo,
     break;
   }
 
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded)
     .TraverseType(NameInfo.getName().getCXXNameType());
   assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
@@ -285,7 +285,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(SourceLocation Loc,
   if (Template.isNull() || !Template.containsUnexpandedParameterPack())
     return false;
 
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded)
     .TraverseTemplateName(Template);
   assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
@@ -299,7 +299,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(TemplateArgumentLoc Arg,
       !Arg.getArgument().containsUnexpandedParameterPack())
     return false;
   
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded)
     .TraverseTemplateArgumentLoc(Arg);
   assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
@@ -308,24 +308,24 @@ bool Sema::DiagnoseUnexpandedParameterPack(TemplateArgumentLoc Arg,
 }
 
 void Sema::collectUnexpandedParameterPacks(TemplateArgument Arg,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
   CollectUnexpandedParameterPacksVisitor(Unexpanded)
     .TraverseTemplateArgument(Arg);
 }
 
 void Sema::collectUnexpandedParameterPacks(TemplateArgumentLoc Arg,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
   CollectUnexpandedParameterPacksVisitor(Unexpanded)
     .TraverseTemplateArgumentLoc(Arg);
 }
 
 void Sema::collectUnexpandedParameterPacks(QualType T,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
   CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseType(T);  
 }  
 
 void Sema::collectUnexpandedParameterPacks(TypeLoc TL,
-                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+                   SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
   CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseTypeLoc(TL);  
 }  
 
@@ -462,8 +462,7 @@ getDepthAndIndex(NamedDecl *ND) {
 
 bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
                                            SourceRange PatternRange,
-                                     const UnexpandedParameterPack *Unexpanded,
-                                           unsigned NumUnexpanded,
+                                   ArrayRef<UnexpandedParameterPack> Unexpanded,
                              const MultiLevelTemplateArgumentList &TemplateArgs,
                                            bool &ShouldExpand,
                                            bool &RetainExpansion,
@@ -473,19 +472,21 @@ bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
   std::pair<IdentifierInfo *, SourceLocation> FirstPack;
   bool HaveFirstPack = false;
   
-  for (unsigned I = 0; I != NumUnexpanded; ++I) {
+  for (ArrayRef<UnexpandedParameterPack>::iterator i = Unexpanded.begin(),
+                                                 end = Unexpanded.end();
+                                                  i != end; ++i) {
     // Compute the depth and index for this parameter pack.
     unsigned Depth = 0, Index = 0;
     IdentifierInfo *Name;
     bool IsFunctionParameterPack = false;
     
     if (const TemplateTypeParmType *TTP
-        = Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>()) {
+        = i->first.dyn_cast<const TemplateTypeParmType *>()) {
       Depth = TTP->getDepth();
       Index = TTP->getIndex();
       Name = TTP->getIdentifier();
     } else {
-      NamedDecl *ND = Unexpanded[I].first.get<NamedDecl *>();
+      NamedDecl *ND = i->first.get<NamedDecl *>();
       if (isa<ParmVarDecl>(ND))
         IsFunctionParameterPack = true;
       else
@@ -502,7 +503,7 @@ bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
       
       llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation
         = CurrentInstantiationScope->findInstantiationOf(
-                                        Unexpanded[I].first.get<NamedDecl *>());
+                                        i->first.get<NamedDecl *>());
       if (Instantiation->is<DeclArgumentPack *>()) {
         // We could expand this function parameter pack.
         NewPackSize = Instantiation->get<DeclArgumentPack *>()->size();
@@ -545,7 +546,7 @@ bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
       // Record it.
       NumExpansions = NewPackSize;
       FirstPack.first = Name;
-      FirstPack.second = Unexpanded[I].second;
+      FirstPack.second = i->second;
       HaveFirstPack = true;
       continue;
     }
@@ -557,11 +558,11 @@ bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
       if (HaveFirstPack)
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict)
           << FirstPack.first << Name << *NumExpansions << NewPackSize
-          << SourceRange(FirstPack.second) << SourceRange(Unexpanded[I].second);
+          << SourceRange(FirstPack.second) << SourceRange(i->second);
       else
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict_multilevel)
           << Name << *NumExpansions << NewPackSize
-          << SourceRange(Unexpanded[I].second);
+          << SourceRange(i->second);
       return true;
     }
   }
@@ -572,7 +573,7 @@ bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
 unsigned Sema::getNumArgumentsInExpansion(QualType T, 
                           const MultiLevelTemplateArgumentList &TemplateArgs) {
   QualType Pattern = cast<PackExpansionType>(T)->getPattern();
-  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
   CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseType(Pattern);
 
   for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
@@ -618,7 +619,8 @@ bool Sema::containsUnexpandedParameterPacks(Declarator &D) {
   switch (DS.getTypeSpecType()) {
   case TST_typename:
   case TST_typeofType:
-  case TST_underlyingType: {
+  case TST_underlyingType:
+  case TST_atomic: {
     QualType T = DS.getRepAsType().get();
     if (!T.isNull() && T->containsUnexpandedParameterPack())
       return true;
@@ -639,6 +641,7 @@ bool Sema::containsUnexpandedParameterPacks(Declarator &D) {
   case TST_char16:
   case TST_char32:
   case TST_int:
+  case TST_half:
   case TST_float:
   case TST_double:
   case TST_bool:
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaType.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaType.cpp
index f3e73ec5a723..2b563a50a99f 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaType.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaType.cpp
@@ -52,18 +52,18 @@ static bool isOmittedBlockReturnType(const Declarator &D) {
 /// doesn't apply to the given type.
 static void diagnoseBadTypeAttribute(Sema &S, const AttributeList &attr,
                                      QualType type) {
-  bool useInstantiationLoc = false;
+  bool useExpansionLoc = false;
 
   unsigned diagID = 0;
   switch (attr.getKind()) {
   case AttributeList::AT_objc_gc:
     diagID = diag::warn_pointer_attribute_wrong_type;
-    useInstantiationLoc = true;
+    useExpansionLoc = true;
     break;
 
   case AttributeList::AT_objc_ownership:
     diagID = diag::warn_objc_object_attribute_wrong_type;
-    useInstantiationLoc = true;
+    useExpansionLoc = true;
     break;
 
   default:
@@ -73,10 +73,10 @@ static void diagnoseBadTypeAttribute(Sema &S, const AttributeList &attr,
   }
 
   SourceLocation loc = attr.getLoc();
-  llvm::StringRef name = attr.getName()->getName();
+  StringRef name = attr.getName()->getName();
 
   // The GC attributes are usually written with macros;  special-case them.
-  if (useInstantiationLoc && loc.isMacroID() && attr.getParameterName()) {
+  if (useExpansionLoc && loc.isMacroID() && attr.getParameterName()) {
     if (attr.getParameterName()->isStr("strong")) {
       if (S.findMacroSpelling(loc, "__strong")) name = "__strong";
     } else if (attr.getParameterName()->isStr("weak")) {
@@ -125,11 +125,11 @@ namespace {
     bool hasSavedAttrs;
 
     /// The original set of attributes on the DeclSpec.
-    llvm::SmallVector<AttributeList*, 2> savedAttrs;
+    SmallVector<AttributeList*, 2> savedAttrs;
 
     /// A list of attributes to diagnose the uselessness of when the
     /// processing is complete.
-    llvm::SmallVector<AttributeList*, 2> ignoredTypeAttrs;
+    SmallVector<AttributeList*, 2> ignoredTypeAttrs;
 
   public:
     TypeProcessingState(Sema &sema, Declarator &declarator)
@@ -183,7 +183,7 @@ namespace {
     /// Diagnose all the ignored type attributes, given that the
     /// declarator worked out to the given type.
     void diagnoseIgnoredTypeAttrs(QualType type) const {
-      for (llvm::SmallVectorImpl<AttributeList*>::const_iterator
+      for (SmallVectorImpl<AttributeList*>::const_iterator
              i = ignoredTypeAttrs.begin(), e = ignoredTypeAttrs.end();
            i != e; ++i)
         diagnoseBadTypeAttribute(getSema(), **i, type);
@@ -664,7 +664,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
       // each struct declaration and type name."
       // FIXME: Does Microsoft really have the implicit int extension in C++?
       if (S.getLangOptions().CPlusPlus &&
-          !S.getLangOptions().Microsoft) {
+          !S.getLangOptions().MicrosoftExt) {
         S.Diag(DeclLoc, diag::err_missing_type_specifier)
           << DS.getSourceRange();
 
@@ -711,6 +711,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     }
     break;
   }
+  case DeclSpec::TST_half: Result = Context.HalfTy; break;
   case DeclSpec::TST_float: Result = Context.FloatTy; break;
   case DeclSpec::TST_double:
     if (DS.getTypeSpecWidth() == DeclSpec::TSW_long)
@@ -856,6 +857,16 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     Result = Context.UnknownAnyTy;
     break;
 
+  case DeclSpec::TST_atomic:
+    Result = S.GetTypeFromParser(DS.getRepAsType());
+    assert(!Result.isNull() && "Didn't get a type for _Atomic?");
+    Result = S.BuildAtomicType(Result, DS.getTypeSpecTypeLoc());
+    if (Result.isNull()) {
+      Result = Context.IntTy;
+      declarator.setInvalidType(true);
+    }
+    break; 
+
   case DeclSpec::TST_error:
     Result = Context.IntTy;
     declarator.setInvalidType(true);
@@ -1038,6 +1049,11 @@ static QualType inferARCLifetimeForPointee(Sema &S, QualType type,
   } else if (type->isObjCARCImplicitlyUnretainedType()) {
     implicitLifetime = Qualifiers::OCL_ExplicitNone;
 
+  // If we are in an unevaluated context, like sizeof, assume ExplicitNone and
+  // don't give error.
+  } else if (S.ExprEvalContexts.back().Context == Sema::Unevaluated) {
+    implicitLifetime = Qualifiers::OCL_ExplicitNone;
+
   // If that failed, give an error and recover using __autoreleasing.
   } else {
     // These types can show up in private ivars in system headers, so
@@ -1419,13 +1435,26 @@ QualType Sema::BuildFunctionType(QualType T,
       << T->isFunctionType() << T;
     return QualType();
   }
-       
+
+  // Functions cannot return half FP.
+  if (T->isHalfType()) {
+    Diag(Loc, diag::err_parameters_retval_cannot_have_fp16_type) << 1 <<
+      FixItHint::CreateInsertion(Loc, "*");
+    return QualType();
+  }
+
   bool Invalid = false;
   for (unsigned Idx = 0; Idx < NumParamTypes; ++Idx) {
+    // FIXME: Loc is too inprecise here, should use proper locations for args.
     QualType ParamType = Context.getAdjustedParameterType(ParamTypes[Idx]);
     if (ParamType->isVoidType()) {
       Diag(Loc, diag::err_param_with_void_type);
       Invalid = true;
+    } else if (ParamType->isHalfType()) {
+      // Disallow half FP arguments.
+      Diag(Loc, diag::err_parameters_retval_cannot_have_fp16_type) << 0 <<
+        FixItHint::CreateInsertion(Loc, "*");
+      Invalid = true;
     }
 
     ParamTypes[Idx] = ParamType;
@@ -1492,7 +1521,7 @@ QualType Sema::BuildMemberPointerType(QualType T, QualType Class,
   // type. In such cases, the compiler makes a worst-case assumption.
   // We make no such assumption right now, so emit an error if the
   // class isn't a complete type.
-  if (Context.Target.getCXXABI() == CXXABI_Microsoft &&
+  if (Context.getTargetInfo().getCXXABI() == CXXABI_Microsoft &&
       RequireCompleteType(Loc, Class, diag::err_incomplete_type))
     return QualType();
 
@@ -1745,9 +1774,10 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
 
     switch (D.getContext()) {
     case Declarator::KNRTypeListContext:
-      assert(0 && "K&R type lists aren't allowed in C++");
+      llvm_unreachable("K&R type lists aren't allowed in C++");
       break;
-    case Declarator::ObjCPrototypeContext:
+    case Declarator::ObjCParameterContext:
+    case Declarator::ObjCResultContext:
     case Declarator::PrototypeContext:
       Error = 0; // Function prototype
       break;
@@ -1755,7 +1785,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
       if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_static)
         break;
       switch (cast<TagDecl>(SemaRef.CurContext)->getTagKind()) {
-      case TTK_Enum: assert(0 && "unhandled tag kind"); break;
+      case TTK_Enum: llvm_unreachable("unhandled tag kind");
       case TTK_Struct: Error = 1; /* Struct member */ break;
       case TTK_Union:  Error = 2; /* Union member */ break;
       case TTK_Class:  Error = 3; /* Class member */ break;
@@ -1825,7 +1855,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
   }
 
   if (SemaRef.getLangOptions().CPlusPlus &&
-      OwnedTagDecl && OwnedTagDecl->isDefinition()) {
+      OwnedTagDecl && OwnedTagDecl->isCompleteDefinition()) {
     // Check the contexts where C++ forbids the declaration of a new class
     // or enumeration in a type-specifier-seq.
     switch (D.getContext()) {
@@ -1856,7 +1886,8 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
         << SemaRef.Context.getTypeDeclType(OwnedTagDecl);
       break;
     case Declarator::PrototypeContext:
-    case Declarator::ObjCPrototypeContext:
+    case Declarator::ObjCParameterContext:
+    case Declarator::ObjCResultContext:
     case Declarator::KNRTypeListContext:
       // C++ [dcl.fct]p6:
       //   Types shall not be defined in return or parameter types.
@@ -1916,7 +1947,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
     state.setCurrentChunkIndex(chunkIndex);
     DeclaratorChunk &DeclType = D.getTypeObject(chunkIndex);
     switch (DeclType.Kind) {
-    default: assert(0 && "Unknown decltype!");
+    default: llvm_unreachable("Unknown decltype!");
     case DeclaratorChunk::Paren:
       T = S.BuildParenType(T);
       break;
@@ -2045,6 +2076,15 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         D.setInvalidType(true);
       }
 
+      // Do not allow returning half FP value.
+      // FIXME: This really should be in BuildFunctionType.
+      if (T->isHalfType()) {
+        S.Diag(D.getIdentifierLoc(),
+             diag::err_parameters_retval_cannot_have_fp16_type) << 1
+          << FixItHint::CreateInsertion(D.getIdentifierLoc(), "*");
+        D.setInvalidType(true);
+      }
+
       // cv-qualifiers on return types are pointless except when the type is a
       // class type in C++.
       if (isa<PointerType>(T) && T.getLocalCVRQualifiers() &&
@@ -2077,7 +2117,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         // C++ [dcl.fct]p6:
         //   Types shall not be defined in return or parameter types.
         TagDecl *Tag = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
-        if (Tag->isDefinition())
+        if (Tag->isCompleteDefinition())
           S.Diag(Tag->getLocation(), diag::err_type_defined_in_result_type)
             << Context.getTypeDeclType(Tag);
       }
@@ -2127,10 +2167,10 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         
         // Otherwise, we have a function with an argument list that is
         // potentially variadic.
-        llvm::SmallVector<QualType, 16> ArgTys;
+        SmallVector<QualType, 16> ArgTys;
         ArgTys.reserve(FTI.NumArgs);
 
-        llvm::SmallVector<bool, 16> ConsumedArguments;
+        SmallVector<bool, 16> ConsumedArguments;
         ConsumedArguments.reserve(FTI.NumArgs);
         bool HasAnyConsumedArguments = false;
 
@@ -2168,6 +2208,13 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
               // Do not add 'void' to the ArgTys list.
               break;
             }
+          } else if (ArgTy->isHalfType()) {
+            // Disallow half FP arguments.
+            // FIXME: This really should be in BuildFunctionType.
+            S.Diag(Param->getLocation(),
+               diag::err_parameters_retval_cannot_have_fp16_type) << 0
+            << FixItHint::CreateInsertion(Param->getLocation(), "*");
+            D.setInvalidType();
           } else if (!FTI.hasPrototype) {
             if (ArgTy->isPromotableIntegerType()) {
               ArgTy = Context.getPromotedIntegerType(ArgTy);
@@ -2192,7 +2239,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         if (HasAnyConsumedArguments)
           EPI.ConsumedArguments = ConsumedArguments.data();
 
-        llvm::SmallVector<QualType, 4> Exceptions;
+        SmallVector<QualType, 4> Exceptions;
         EPI.ExceptionSpecType = FTI.getExceptionSpecType();
         if (FTI.getExceptionSpecType() == EST_Dynamic) {
           Exceptions.reserve(FTI.NumExceptions);
@@ -2320,6 +2367,20 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
       FreeFunction = (DC && !DC->isRecord());
     }
 
+    // C++0x [dcl.constexpr]p8: A constexpr specifier for a non-static member
+    // function that is not a constructor declares that function to be const.
+    if (D.getDeclSpec().isConstexprSpecified() && !FreeFunction &&
+        D.getName().getKind() != UnqualifiedId::IK_ConstructorName &&
+        D.getName().getKind() != UnqualifiedId::IK_ConstructorTemplateId &&
+        !(FnTy->getTypeQuals() & DeclSpec::TQ_const)) {
+      // Rebuild function type adding a 'const' qualifier.
+      FunctionProtoType::ExtProtoInfo EPI = FnTy->getExtProtoInfo();
+      EPI.TypeQuals |= DeclSpec::TQ_const;
+      T = Context.getFunctionType(FnTy->getResultType(), 
+                                  FnTy->arg_type_begin(),
+                                  FnTy->getNumArgs(), EPI);
+    }
+
     // C++0x [dcl.fct]p6:
     //   A ref-qualifier shall only be part of the function type for a
     //   non-static member function, the function type to which a pointer to
@@ -2457,13 +2518,17 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
       // it expands those parameter packs.
       if (T->containsUnexpandedParameterPack())
         T = Context.getPackExpansionType(T, llvm::Optional<unsigned>());
-      else if (!LangOpts.CPlusPlus0x)
-        S.Diag(D.getEllipsisLoc(), diag::ext_variadic_templates);
+      else
+        S.Diag(D.getEllipsisLoc(),
+               LangOpts.CPlusPlus0x
+                 ? diag::warn_cxx98_compat_variadic_templates
+                 : diag::ext_variadic_templates);
       break;
     
     case Declarator::FileContext:
     case Declarator::KNRTypeListContext:
-    case Declarator::ObjCPrototypeContext: // FIXME: special diagnostic here?
+    case Declarator::ObjCParameterContext:  // FIXME: special diagnostic here?
+    case Declarator::ObjCResultContext:     // FIXME: special diagnostic here?
     case Declarator::TypeNameContext:
     case Declarator::CXXNewContext:
     case Declarator::AliasDeclContext:
@@ -2850,6 +2915,14 @@ namespace {
     void VisitTagTypeLoc(TagTypeLoc TL) {
       TL.setNameLoc(DS.getTypeSpecTypeNameLoc());
     }
+    void VisitAtomicTypeLoc(AtomicTypeLoc TL) {
+      TL.setKWLoc(DS.getTypeSpecTypeLoc());
+      TL.setParensRange(DS.getTypeofParensRange());
+      
+      TypeSourceInfo *TInfo = 0;
+      Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
+      TL.getValueLoc().initializeFullCopy(TInfo->getTypeLoc());
+    }
 
     void VisitTypeLoc(TypeLoc TL) {
       // FIXME: add other typespec types and change this to an assert.
@@ -3030,7 +3103,7 @@ ParsedType Sema::CreateParsedType(QualType T, TypeSourceInfo *TInfo) {
 
 void LocInfoType::getAsStringInternal(std::string &Str,
                                       const PrintingPolicy &Policy) const {
-  assert(false && "LocInfoType leaked into the type system; an opaque TypeTy*"
+  llvm_unreachable("LocInfoType leaked into the type system; an opaque TypeTy*"
          " was used directly instead of getting the QualType through"
          " GetTypeFromParser");
 }
@@ -3045,6 +3118,12 @@ TypeResult Sema::ActOnTypeName(Scope *S, Declarator &D) {
   if (D.isInvalidType())
     return true;
 
+  // Make sure there are no unused decl attributes on the declarator.
+  // We don't want to do this for ObjC parameters because we're going
+  // to apply them to the actual parameter declaration.
+  if (D.getContext() != Declarator::ObjCParameterContext)
+    checkUnusedDeclAttributes(D);
+
   if (getLangOptions().CPlusPlus) {
     // Check that there are no default arguments (C++ only).
     CheckExtraCXXDefaultArguments(D);
@@ -3053,6 +3132,13 @@ TypeResult Sema::ActOnTypeName(Scope *S, Declarator &D) {
   return CreateParsedType(T, TInfo);
 }
 
+ParsedType Sema::ActOnObjCInstanceType(SourceLocation Loc) {
+  QualType T = Context.getObjCInstanceType();
+  TypeSourceInfo *TInfo = Context.getTrivialTypeSourceInfo(T, Loc);
+  return CreateParsedType(T, TInfo);
+}
+
+
 //===----------------------------------------------------------------------===//
 // Type Attribute Processing
 //===----------------------------------------------------------------------===//
@@ -3064,14 +3150,22 @@ static void HandleAddressSpaceTypeAttribute(QualType &Type,
                                             const AttributeList &Attr, Sema &S){
 
   // If this type is already address space qualified, reject it.
-  // Clause 6.7.3 - Type qualifiers: "No type shall be qualified by qualifiers
-  // for two or more different address spaces."
+  // ISO/IEC TR 18037 S5.3 (amending C99 6.7.3): "No type shall be qualified by
+  // qualifiers for two or more different address spaces."
   if (Type.getAddressSpace()) {
     S.Diag(Attr.getLoc(), diag::err_attribute_address_multiple_qualifiers);
     Attr.setInvalid();
     return;
   }
 
+  // ISO/IEC TR 18037 S5.3 (amending C99 6.7.3): "A function type shall not be
+  // qualified by an address-space qualifier."
+  if (Type->isFunctionType()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_address_function_type);
+    Attr.setInvalid();
+    return;
+  }
+
   // Check the attribute arguments.
   if (Attr.getNumArgs() != 1) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
@@ -3122,15 +3216,18 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     return false;
 
   Sema &S = state.getSema();
+  SourceLocation AttrLoc = attr.getLoc();
+  if (AttrLoc.isMacroID())
+    AttrLoc = S.getSourceManager().getImmediateExpansionRange(AttrLoc).first;
 
   if (type.getQualifiers().getObjCLifetime()) {
-    S.Diag(attr.getLoc(), diag::err_attr_objc_ownership_redundant)
+    S.Diag(AttrLoc, diag::err_attr_objc_ownership_redundant)
       << type;
     return true;
   }
 
   if (!attr.getParameterName()) {
-    S.Diag(attr.getLoc(), diag::err_attribute_argument_n_not_string)
+    S.Diag(AttrLoc, diag::err_attribute_argument_n_not_string)
       << "objc_ownership" << 1;
     attr.setInvalid();
     return true;
@@ -3146,7 +3243,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
   else if (attr.getParameterName()->isStr("autoreleasing"))
     lifetime = Qualifiers::OCL_Autoreleasing;
   else {
-    S.Diag(attr.getLoc(), diag::warn_attribute_type_not_supported)
+    S.Diag(AttrLoc, diag::warn_attribute_type_not_supported)
       << "objc_ownership" << attr.getParameterName();
     attr.setInvalid();
     return true;
@@ -3164,7 +3261,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
 
   // If we have a valid source location for the attribute, use an
   // AttributedType instead.
-  if (attr.getLoc().isValid())
+  if (AttrLoc.isValid())
     type = S.Context.getAttributedType(AttributedType::attr_objc_ownership,
                                        origType, type);
 
@@ -3175,10 +3272,11 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     // Actually, delay this until we know what we're parsing.
     if (S.DelayedDiagnostics.shouldDelayDiagnostics()) {
       S.DelayedDiagnostics.add(
-          sema::DelayedDiagnostic::makeForbiddenType(attr.getLoc(),
+          sema::DelayedDiagnostic::makeForbiddenType(
+              S.getSourceManager().getExpansionLoc(AttrLoc),
               diag::err_arc_weak_no_runtime, type, /*ignored*/ 0));
     } else {
-      S.Diag(attr.getLoc(), diag::err_arc_weak_no_runtime);
+      S.Diag(AttrLoc, diag::err_arc_weak_no_runtime);
     }
 
     attr.setInvalid();
@@ -3194,7 +3292,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     if (const ObjCObjectPointerType *ObjT = T->getAs<ObjCObjectPointerType>()) {
       ObjCInterfaceDecl *Class = ObjT->getInterfaceDecl();
       if (Class->isArcWeakrefUnavailable()) {
-          S.Diag(attr.getLoc(), diag::err_arc_unsupported_weak_class);
+          S.Diag(AttrLoc, diag::err_arc_unsupported_weak_class);
           S.Diag(ObjT->getInterfaceDecl()->getLocation(), 
                  diag::note_class_declared);
       }
@@ -3283,7 +3381,7 @@ namespace {
 
     QualType Original;
     const FunctionType *Fn;
-    llvm::SmallVector<unsigned char /*WrapKind*/, 8> Stack;
+    SmallVector<unsigned char /*WrapKind*/, 8> Stack;
 
     FunctionTypeUnwrapper(Sema &S, QualType T) : Original(T) {
       while (true) {
@@ -3474,7 +3572,7 @@ static bool handleFunctionTypeAttr(TypeProcessingState &state,
     return true;
   }
 
-  if (CCOld != CC_Default) {
+  if (CCOld != (S.LangOpts.MRTD ? CC_X86StdCall : CC_Default)) {
     // Should we diagnose reapplications of the same convention?
     S.Diag(attr.getLoc(), diag::err_attributes_are_not_compatible)
       << FunctionType::getNameForCallConv(CC)
@@ -3718,32 +3816,44 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
     switch (attr.getKind()) {
     default: break;
 
+    case AttributeList::AT_may_alias:
+      // FIXME: This attribute needs to actually be handled, but if we ignore
+      // it it breaks large amounts of Linux software.
+      attr.setUsedAsTypeAttr();
+      break;
     case AttributeList::AT_address_space:
       HandleAddressSpaceTypeAttribute(type, attr, state.getSema());
+      attr.setUsedAsTypeAttr();
       break;
     OBJC_POINTER_TYPE_ATTRS_CASELIST:
       if (!handleObjCPointerTypeAttr(state, attr, type))
         distributeObjCPointerTypeAttr(state, attr, type);
+      attr.setUsedAsTypeAttr();
       break;
     case AttributeList::AT_vector_size:
       HandleVectorSizeAttr(type, attr, state.getSema());
+      attr.setUsedAsTypeAttr();
       break;
     case AttributeList::AT_ext_vector_type:
       if (state.getDeclarator().getDeclSpec().getStorageClassSpec()
             != DeclSpec::SCS_typedef)
         HandleExtVectorTypeAttr(type, attr, state.getSema());
+      attr.setUsedAsTypeAttr();
       break;
     case AttributeList::AT_neon_vector_type:
       HandleNeonVectorTypeAttr(type, attr, state.getSema(),
                                VectorType::NeonVector, "neon_vector_type");
+      attr.setUsedAsTypeAttr();
       break;
     case AttributeList::AT_neon_polyvector_type:
       HandleNeonVectorTypeAttr(type, attr, state.getSema(),
                                VectorType::NeonPolyVector,
                                "neon_polyvector_type");
+      attr.setUsedAsTypeAttr();
       break;
     case AttributeList::AT_opencl_image_access:
       HandleOpenCLImageAccessAttribute(type, attr, state.getSema());
+      attr.setUsedAsTypeAttr();
       break;
 
     case AttributeList::AT_ns_returns_retained:
@@ -3752,6 +3862,8 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       // fallthrough into the function attrs
 
     FUNCTION_TYPE_ATTRS_CASELIST:
+      attr.setUsedAsTypeAttr();
+
       // Never process function type attributes as part of the
       // declaration-specifiers.
       if (isDeclSpec)
@@ -3954,6 +4066,121 @@ bool Sema::RequireCompleteType(SourceLocation Loc, QualType T,
                              std::make_pair(SourceLocation(), PDiag(0)));
 }
 
+/// @brief Ensure that the type T is a literal type.
+///
+/// This routine checks whether the type @p T is a literal type. If @p T is an
+/// incomplete type, an attempt is made to complete it. If @p T is a literal
+/// type, or @p AllowIncompleteType is true and @p T is an incomplete type,
+/// returns false. Otherwise, this routine issues the diagnostic @p PD (giving
+/// it the type @p T), along with notes explaining why the type is not a
+/// literal type, and returns true.
+///
+/// @param Loc  The location in the source that the non-literal type
+/// diagnostic should refer to.
+///
+/// @param T  The type that this routine is examining for literalness.
+///
+/// @param PD The partial diagnostic that will be printed out if T is not a
+/// literal type.
+///
+/// @param AllowIncompleteType If true, an incomplete type will be considered
+/// acceptable.
+///
+/// @returns @c true if @p T is not a literal type and a diagnostic was emitted,
+/// @c false otherwise.
+bool Sema::RequireLiteralType(SourceLocation Loc, QualType T,
+                              const PartialDiagnostic &PD,
+                              bool AllowIncompleteType) {
+  assert(!T->isDependentType() && "type should not be dependent");
+
+  bool Incomplete = RequireCompleteType(Loc, T, 0);
+  if (T->isLiteralType() || (AllowIncompleteType && Incomplete))
+    return false;
+
+  if (PD.getDiagID() == 0)
+    return true;
+
+  Diag(Loc, PD) << T;
+
+  if (T->isVariableArrayType())
+    return true;
+
+  const RecordType *RT = T->getBaseElementTypeUnsafe()->getAs<RecordType>();
+  if (!RT)
+    return true;
+
+  const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+
+  // If the class has virtual base classes, then it's not an aggregate, and
+  // cannot have any constexpr constructors, so is non-literal. This is better
+  // to diagnose than the resulting absence of constexpr constructors.
+  if (RD->getNumVBases()) {
+    Diag(RD->getLocation(), diag::note_non_literal_virtual_base)
+      << RD->isStruct() << RD->getNumVBases();
+    for (CXXRecordDecl::base_class_const_iterator I = RD->vbases_begin(),
+           E = RD->vbases_end(); I != E; ++I)
+      Diag(I->getSourceRange().getBegin(),
+           diag::note_constexpr_virtual_base_here) << I->getSourceRange();
+  } else if (!RD->isAggregate() && !RD->hasConstexprNonCopyMoveConstructor()) {
+    Diag(RD->getLocation(), diag::note_non_literal_no_constexpr_ctors) << RD;
+
+    switch (RD->getTemplateSpecializationKind()) {
+    case TSK_Undeclared:
+    case TSK_ExplicitSpecialization:
+      break;
+
+    case TSK_ImplicitInstantiation:
+    case TSK_ExplicitInstantiationDeclaration:
+    case TSK_ExplicitInstantiationDefinition:
+      // If the base template had constexpr constructors which were
+      // instantiated as non-constexpr constructors, explain why.
+      for (CXXRecordDecl::ctor_iterator I = RD->ctor_begin(),
+           E = RD->ctor_end(); I != E; ++I) {
+        if ((*I)->isCopyConstructor() || (*I)->isMoveConstructor())
+          continue;
+
+        FunctionDecl *Base = (*I)->getInstantiatedFromMemberFunction();
+        if (Base && Base->isConstexpr())
+          CheckConstexprFunctionDecl(*I, CCK_NoteNonConstexprInstantiation);
+      }
+    }
+  } else if (RD->hasNonLiteralTypeFieldsOrBases()) {
+    for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+         E = RD->bases_end(); I != E; ++I) {
+      if (!I->getType()->isLiteralType()) {
+        Diag(I->getSourceRange().getBegin(),
+             diag::note_non_literal_base_class)
+          << RD << I->getType() << I->getSourceRange();
+        return true;
+      }
+    }
+    for (CXXRecordDecl::field_iterator I = RD->field_begin(),
+         E = RD->field_end(); I != E; ++I) {
+      if (!(*I)->getType()->isLiteralType()) {
+        Diag((*I)->getLocation(), diag::note_non_literal_field)
+          << RD << (*I) << (*I)->getType();
+        return true;
+      } else if ((*I)->isMutable()) {
+        Diag((*I)->getLocation(), diag::note_non_literal_mutable_field) << RD;
+        return true;
+      }
+    }
+  } else if (!RD->hasTrivialDestructor()) {
+    // All fields and bases are of literal types, so have trivial destructors.
+    // If this class's destructor is non-trivial it must be user-declared.
+    CXXDestructorDecl *Dtor = RD->getDestructor();
+    assert(Dtor && "class has literal fields and bases but no dtor?");
+    if (!Dtor)
+      return true;
+
+    Diag(Dtor->getLocation(), Dtor->isUserProvided() ?
+         diag::note_non_literal_user_provided_dtor :
+         diag::note_non_literal_nontrivial_dtor) << RD;
+  }
+
+  return true;
+}
+
 /// \brief Retrieve a version of the type 'T' that is elaborated by Keyword
 /// and qualified by the nested-name-specifier contained in SS.
 QualType Sema::getElaboratedType(ElaboratedTypeKeyword Keyword,
@@ -4015,3 +4242,36 @@ QualType Sema::BuildUnaryTransformType(QualType BaseType,
   }
   llvm_unreachable("unknown unary transform type");
 }
+
+QualType Sema::BuildAtomicType(QualType T, SourceLocation Loc) {
+  if (!T->isDependentType()) {
+    int DisallowedKind = -1;
+    if (T->isIncompleteType())
+      // FIXME: It isn't entirely clear whether incomplete atomic types
+      // are allowed or not; for simplicity, ban them for the moment.
+      DisallowedKind = 0;
+    else if (T->isArrayType())
+      DisallowedKind = 1;
+    else if (T->isFunctionType())
+      DisallowedKind = 2;
+    else if (T->isReferenceType())
+      DisallowedKind = 3;
+    else if (T->isAtomicType())
+      DisallowedKind = 4;
+    else if (T.hasQualifiers())
+      DisallowedKind = 5;
+    else if (!T.isTriviallyCopyableType(Context))
+      // Some other non-trivially-copyable type (probably a C++ class)
+      DisallowedKind = 6;
+
+    if (DisallowedKind != -1) {
+      Diag(Loc, diag::err_atomic_specifier_bad_type) << DisallowedKind << T;
+      return QualType();
+    }
+
+    // FIXME: Do we need any handling for ARC here?
+  }
+
+  // Build the pointer type.
+  return Context.getAtomicType(T);
+}
diff --git a/contrib/llvm/tools/clang/lib/Sema/TargetAttributesSema.cpp b/contrib/llvm/tools/clang/lib/Sema/TargetAttributesSema.cpp
index ab697eeed53b..aa0bc08ef7f4 100644
--- a/contrib/llvm/tools/clang/lib/Sema/TargetAttributesSema.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/TargetAttributesSema.cpp
@@ -147,7 +147,8 @@ static void HandleX86ForceAlignArgPointerAttr(Decl *D,
     return;
   }
 
-  D->addAttr(::new (S.Context) X86ForceAlignArgPointerAttr(Attr.getLoc(), S.Context));
+  D->addAttr(::new (S.Context) X86ForceAlignArgPointerAttr(Attr.getRange(),
+                                                           S.Context));
 }
 
 static void HandleDLLImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -168,7 +169,7 @@ static void HandleDLLImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     // Apparently Visual C++ thinks it is okay to not emit a warning
     // in this case, so only emit a warning when -fms-extensions is not
     // specified.
-    if (!S.getLangOptions().Microsoft)
+    if (!S.getLangOptions().MicrosoftExt)
       S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
         << Attr.getName() << 2 /*variable and function*/;
     return;
@@ -236,7 +237,7 @@ namespace {
     X86AttributesSema() { }
     bool ProcessDeclAttribute(Scope *scope, Decl *D,
                               const AttributeList &Attr, Sema &S) const {
-      const llvm::Triple &Triple(S.Context.Target.getTriple());
+      const llvm::Triple &Triple(S.Context.getTargetInfo().getTriple());
       if (Triple.getOS() == llvm::Triple::Win32 ||
           Triple.getOS() == llvm::Triple::MinGW32) {
         switch (Attr.getKind()) {
@@ -247,8 +248,9 @@ namespace {
         default:                          break;
         }
       }
-      if (Attr.getName()->getName() == "force_align_arg_pointer" ||
-          Attr.getName()->getName() == "__force_align_arg_pointer__") {
+      if (Triple.getArch() != llvm::Triple::x86_64 &&
+          (Attr.getName()->getName() == "force_align_arg_pointer" ||
+           Attr.getName()->getName() == "__force_align_arg_pointer__")) {
         HandleX86ForceAlignArgPointerAttr(D, Attr, S);
         return true;
       }
@@ -261,16 +263,16 @@ const TargetAttributesSema &Sema::getTargetAttributesSema() const {
   if (TheTargetAttributesSema)
     return *TheTargetAttributesSema;
 
-  const llvm::Triple &Triple(Context.Target.getTriple());
+  const llvm::Triple &Triple(Context.getTargetInfo().getTriple());
   switch (Triple.getArch()) {
-  default:
-    return *(TheTargetAttributesSema = new TargetAttributesSema);
-
   case llvm::Triple::msp430:
     return *(TheTargetAttributesSema = new MSP430AttributesSema);
   case llvm::Triple::mblaze:
     return *(TheTargetAttributesSema = new MBlazeAttributesSema);
   case llvm::Triple::x86:
+  case llvm::Triple::x86_64:
     return *(TheTargetAttributesSema = new X86AttributesSema);
+  default:
+    return *(TheTargetAttributesSema = new TargetAttributesSema);
   }
 }
diff --git a/contrib/llvm/tools/clang/lib/Sema/TreeTransform.h b/contrib/llvm/tools/clang/lib/Sema/TreeTransform.h
index fa87217821aa..bb49eee2f332 100644
--- a/contrib/llvm/tools/clang/lib/Sema/TreeTransform.h
+++ b/contrib/llvm/tools/clang/lib/Sema/TreeTransform.h
@@ -31,6 +31,7 @@
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Designator.h"
 #include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "TypeLocBuilder.h"
 #include <algorithm>
@@ -244,8 +245,7 @@ class TreeTransform {
   /// must be set.
   bool TryExpandParameterPacks(SourceLocation EllipsisLoc,
                                SourceRange PatternRange,
-                               const UnexpandedParameterPack *Unexpanded,
-                               unsigned NumUnexpanded,
+                             llvm::ArrayRef<UnexpandedParameterPack> Unexpanded,
                                bool &ShouldExpand,
                                bool &RetainExpansion,
                                llvm::Optional<unsigned> &NumExpansions) {
@@ -345,7 +345,7 @@ class TreeTransform {
   ///
   /// \returns true if an error occurred, false otherwise.
   bool TransformExprs(Expr **Inputs, unsigned NumInputs, bool IsCall,
-                      llvm::SmallVectorImpl<Expr *> &Outputs,
+                      SmallVectorImpl<Expr *> &Outputs,
                       bool *ArgChanged = 0);
   
   /// \brief Transform the given declaration, which is referenced from a type
@@ -520,8 +520,8 @@ class TreeTransform {
   bool TransformFunctionTypeParams(SourceLocation Loc,
                                    ParmVarDecl **Params, unsigned NumParams,
                                    const QualType *ParamTypes,
-                                   llvm::SmallVectorImpl<QualType> &PTypes,
-                                   llvm::SmallVectorImpl<ParmVarDecl*> *PVars);
+                                   SmallVectorImpl<QualType> &PTypes,
+                                   SmallVectorImpl<ParmVarDecl*> *PVars);
 
   /// \brief Transforms a single function-type parameter.  Return null
   /// on error.
@@ -905,6 +905,12 @@ class TreeTransform {
                                         NumExpansions);
   }
 
+  /// \brief Build a new atomic type given its value type.
+  ///
+  /// By default, performs semantic analysis when building the atomic type.
+  /// Subclasses may override this routine to provide different behavior.
+  QualType RebuildAtomicType(QualType ValueType, SourceLocation KWLoc);
+
   /// \brief Build a new template name given a nested name specifier, a flag
   /// indicating whether the "template" keyword was provided, and the template
   /// that the template name refers to.
@@ -1181,15 +1187,22 @@ class TreeTransform {
     return getSema().BuildObjCAtThrowStmt(AtLoc, Operand);
   }
   
+  /// \brief Rebuild the operand to an Objective-C @synchronized statement.
+  ///
+  /// By default, performs semantic analysis to build the new statement.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildObjCAtSynchronizedOperand(SourceLocation atLoc,
+                                              Expr *object) {
+    return getSema().ActOnObjCAtSynchronizedOperand(atLoc, object);
+  }
+
   /// \brief Build a new Objective-C @synchronized statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildObjCAtSynchronizedStmt(SourceLocation AtLoc,
-                                                 Expr *Object,
-                                                 Stmt *Body) {
-    return getSema().ActOnObjCAtSynchronizedStmt(AtLoc, Object,
-                                                 Body);
+                                           Expr *Object, Stmt *Body) {
+    return getSema().ActOnObjCAtSynchronizedStmt(AtLoc, Object, Body);
   }
 
   /// \brief Build a new Objective-C @autoreleasepool statement.
@@ -1200,7 +1213,16 @@ class TreeTransform {
                                             Stmt *Body) {
     return getSema().ActOnObjCAutoreleasePoolStmt(AtLoc, Body);
   }
-  
+
+  /// \brief Build the collection operand to a new Objective-C fast
+  /// enumeration statement.
+  ///
+  /// By default, performs semantic analysis to build the new statement.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildObjCForCollectionOperand(SourceLocation forLoc,
+                                             Expr *collection) {
+    return getSema().ActOnObjCForCollectionOperand(forLoc, collection);
+  }
 
   /// \brief Build a new Objective-C fast enumeration statement.
   ///
@@ -1712,8 +1734,7 @@ class TreeTransform {
                                                    SubExpr, RParenLoc);
 
     default:
-      assert(false && "Invalid C++ named cast");
-      break;
+      llvm_unreachable("Invalid C++ named cast");
     }
 
     return ExprError();
@@ -2016,13 +2037,14 @@ class TreeTransform {
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildCXXConstructExpr(QualType T,
-                                           SourceLocation Loc,
-                                           CXXConstructorDecl *Constructor,
-                                           bool IsElidable,
-                                           MultiExprArg Args,
-                                           bool RequiresZeroInit,
+                                     SourceLocation Loc,
+                                     CXXConstructorDecl *Constructor,
+                                     bool IsElidable,
+                                     MultiExprArg Args,
+                                     bool HadMultipleCandidates,
+                                     bool RequiresZeroInit,
                              CXXConstructExpr::ConstructionKind ConstructKind,
-                                           SourceRange ParenRange) {
+                                     SourceRange ParenRange) {
     ASTOwningVector<Expr*> ConvertedArgs(SemaRef);
     if (getSema().CompleteConstructorCall(Constructor, move(Args), Loc, 
                                           ConvertedArgs))
@@ -2030,6 +2052,7 @@ class TreeTransform {
     
     return getSema().BuildCXXConstructExpr(Loc, T, Constructor, IsElidable,
                                            move_arg(ConvertedArgs),
+                                           HadMultipleCandidates,
                                            RequiresZeroInit, ConstructKind,
                                            ParenRange);
   }
@@ -2117,10 +2140,15 @@ class TreeTransform {
   ExprResult RebuildSizeOfPackExpr(SourceLocation OperatorLoc, NamedDecl *Pack, 
                                    SourceLocation PackLoc, 
                                    SourceLocation RParenLoc,
-                                   unsigned Length) {
+                                   llvm::Optional<unsigned> Length) {
+    if (Length)
+      return new (SemaRef.Context) SizeOfPackExpr(SemaRef.Context.getSizeType(), 
+                                                  OperatorLoc, Pack, PackLoc, 
+                                                  RParenLoc, *Length);
+    
     return new (SemaRef.Context) SizeOfPackExpr(SemaRef.Context.getSizeType(), 
                                                 OperatorLoc, Pack, PackLoc, 
-                                                RParenLoc, Length);
+                                                RParenLoc);
   }
                                    
   /// \brief Build a new Objective-C @encode expression.
@@ -2137,7 +2165,7 @@ class TreeTransform {
   /// \brief Build a new Objective-C class message.
   ExprResult RebuildObjCMessageExpr(TypeSourceInfo *ReceiverTypeInfo,
                                           Selector Sel,
-                                          SourceLocation SelectorLoc,
+                                          ArrayRef<SourceLocation> SelectorLocs,
                                           ObjCMethodDecl *Method,
                                           SourceLocation LBracLoc, 
                                           MultiExprArg Args,
@@ -2145,14 +2173,14 @@ class TreeTransform {
     return SemaRef.BuildClassMessage(ReceiverTypeInfo,
                                      ReceiverTypeInfo->getType(),
                                      /*SuperLoc=*/SourceLocation(),
-                                     Sel, Method, LBracLoc, SelectorLoc,
+                                     Sel, Method, LBracLoc, SelectorLocs,
                                      RBracLoc, move(Args));
   }
 
   /// \brief Build a new Objective-C instance message.
   ExprResult RebuildObjCMessageExpr(Expr *Receiver,
                                           Selector Sel,
-                                          SourceLocation SelectorLoc,
+                                          ArrayRef<SourceLocation> SelectorLocs,
                                           ObjCMethodDecl *Method,
                                           SourceLocation LBracLoc, 
                                           MultiExprArg Args,
@@ -2160,7 +2188,7 @@ class TreeTransform {
     return SemaRef.BuildInstanceMessage(Receiver,
                                         Receiver->getType(),
                                         /*SuperLoc=*/SourceLocation(),
-                                        Sel, Method, LBracLoc, SelectorLoc,
+                                        Sel, Method, LBracLoc, SelectorLocs,
                                         RBracLoc, move(Args));
   }
 
@@ -2357,7 +2385,26 @@ class TreeTransform {
                                   llvm::Optional<unsigned> NumExpansions) {
     return getSema().CheckPackExpansion(Pattern, EllipsisLoc, NumExpansions);
   }
-  
+
+  /// \brief Build a new atomic operation expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildAtomicExpr(SourceLocation BuiltinLoc,
+                               MultiExprArg SubExprs,
+                               QualType RetTy,
+                               AtomicExpr::AtomicOp Op,
+                               SourceLocation RParenLoc) {
+    // Just create the expression; there is not any interesting semantic
+    // analysis here because we can't actually build an AtomicExpr until
+    // we are sure it is semantically sound.
+    unsigned NumSubExprs = SubExprs.size();
+    Expr **Subs = (Expr **)SubExprs.release();
+    return new (SemaRef.Context) AtomicExpr(BuiltinLoc, Subs,
+                                            NumSubExprs, RetTy, Op,
+                                            RParenLoc);
+  }
+
 private:
   TypeLoc TransformTypeInObjectScope(TypeLoc TL,
                                      QualType ObjectType,
@@ -2424,7 +2471,7 @@ template<typename Derived>
 bool TreeTransform<Derived>::TransformExprs(Expr **Inputs, 
                                             unsigned NumInputs, 
                                             bool IsCall,
-                                      llvm::SmallVectorImpl<Expr *> &Outputs,
+                                      SmallVectorImpl<Expr *> &Outputs,
                                             bool *ArgChanged) {
   for (unsigned I = 0; I != NumInputs; ++I) {
     // If requested, drop call arguments that need to be dropped.
@@ -2438,7 +2485,7 @@ bool TreeTransform<Derived>::TransformExprs(Expr **Inputs,
     if (PackExpansionExpr *Expansion = dyn_cast<PackExpansionExpr>(Inputs[I])) {
       Expr *Pattern = Expansion->getPattern();
         
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
       assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
       
@@ -2451,8 +2498,7 @@ bool TreeTransform<Derived>::TransformExprs(Expr **Inputs,
       llvm::Optional<unsigned> NumExpansions = OrigNumExpansions;
       if (getDerived().TryExpandParameterPacks(Expansion->getEllipsisLoc(),
                                                Pattern->getSourceRange(),
-                                               Unexpanded.data(),
-                                               Unexpanded.size(),
+                                               Unexpanded,
                                                Expand, RetainExpansion,
                                                NumExpansions))
         return true;
@@ -2522,7 +2568,7 @@ TreeTransform<Derived>::TransformNestedNameSpecifierLoc(
                                                     NestedNameSpecifierLoc NNS,
                                                      QualType ObjectType,
                                              NamedDecl *FirstQualifierInScope) {
-  llvm::SmallVector<NestedNameSpecifierLoc, 4> Qualifiers;
+  SmallVector<NestedNameSpecifierLoc, 4> Qualifiers;
   for (NestedNameSpecifierLoc Qualifier = NNS; Qualifier; 
        Qualifier = Qualifier.getPrefix())
     Qualifiers.push_back(Qualifier);
@@ -2666,8 +2712,7 @@ ::TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo) {
   }
   }
 
-  assert(0 && "Unknown name kind.");
-  return DeclarationNameInfo();
+  llvm_unreachable("Unknown name kind.");
 }
 
 template<typename Derived>
@@ -2887,7 +2932,7 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
   }
 
   case TemplateArgument::Pack: {
-    llvm::SmallVector<TemplateArgument, 4> TransformedArgs;
+    SmallVector<TemplateArgument, 4> TransformedArgs;
     TransformedArgs.reserve(Arg.pack_size());
     for (TemplateArgument::pack_iterator A = Arg.pack_begin(),
                                       AEnd = Arg.pack_end();
@@ -3016,7 +3061,7 @@ bool TreeTransform<Derived>::TransformTemplateArguments(InputIterator First,
         = In.getPackExpansionPattern(Ellipsis, OrigNumExpansions, 
                                      getSema().Context);
       
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
       assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
       
@@ -3027,8 +3072,7 @@ bool TreeTransform<Derived>::TransformTemplateArguments(InputIterator First,
       llvm::Optional<unsigned> NumExpansions = OrigNumExpansions;
       if (getDerived().TryExpandParameterPacks(Ellipsis,
                                                Pattern.getSourceRange(),
-                                               Unexpanded.data(),
-                                               Unexpanded.size(),
+                                               Unexpanded,
                                                Expand, 
                                                RetainExpansion,
                                                NumExpansions))
@@ -3809,8 +3853,8 @@ bool TreeTransform<Derived>::
   TransformFunctionTypeParams(SourceLocation Loc,
                               ParmVarDecl **Params, unsigned NumParams,
                               const QualType *ParamTypes,
-                              llvm::SmallVectorImpl<QualType> &OutParamTypes,
-                              llvm::SmallVectorImpl<ParmVarDecl*> *PVars) {
+                              SmallVectorImpl<QualType> &OutParamTypes,
+                              SmallVectorImpl<ParmVarDecl*> *PVars) {
   int indexAdjustment = 0;
 
   for (unsigned i = 0; i != NumParams; ++i) {
@@ -3821,7 +3865,7 @@ bool TreeTransform<Derived>::
       ParmVarDecl *NewParm = 0;
       if (OldParm->isParameterPack()) {
         // We have a function parameter pack that may need to be expanded.
-        llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+        SmallVector<UnexpandedParameterPack, 2> Unexpanded;
 
         // Find the parameter packs that could be expanded.
         TypeLoc TL = OldParm->getTypeSourceInfo()->getTypeLoc();
@@ -3838,8 +3882,7 @@ bool TreeTransform<Derived>::
         NumExpansions = OrigNumExpansions;
         if (getDerived().TryExpandParameterPacks(ExpansionTL.getEllipsisLoc(),
                                                  Pattern.getSourceRange(),
-                                                 Unexpanded.data(), 
-                                                 Unexpanded.size(),
+                                                 Unexpanded, 
                                                  ShouldExpand, 
                                                  RetainExpansion,
                                                  NumExpansions)) {
@@ -3921,15 +3964,14 @@ bool TreeTransform<Derived>::
                                        = dyn_cast<PackExpansionType>(OldType)) {
       // We have a function parameter pack that may need to be expanded.
       QualType Pattern = Expansion->getPattern();
-      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
       getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
       
       // Determine whether we should expand the parameter packs.
       bool ShouldExpand = false;
       bool RetainExpansion = false;
       if (getDerived().TryExpandParameterPacks(Loc, SourceRange(),
-                                               Unexpanded.data(), 
-                                               Unexpanded.size(),
+                                               Unexpanded, 
                                                ShouldExpand, 
                                                RetainExpansion,
                                                NumExpansions)) {
@@ -4014,8 +4056,8 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
   // parameters before the return type,  since the return type can then refer
   // to the parameters themselves (via decltype, sizeof, etc.).
   //
-  llvm::SmallVector<QualType, 4> ParamTypes;
-  llvm::SmallVector<ParmVarDecl*, 4> ParamDecls;
+  SmallVector<QualType, 4> ParamTypes;
+  SmallVector<ParmVarDecl*, 4> ParamDecls;
   const FunctionProtoType *T = TL.getTypePtr();
 
   QualType ResultType;
@@ -4387,6 +4429,29 @@ QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
   return getDerived().TransformTemplateSpecializationType(TLB, TL, Template);
 }
 
+template<typename Derived>
+QualType TreeTransform<Derived>::TransformAtomicType(TypeLocBuilder &TLB,
+                                                     AtomicTypeLoc TL) {
+  QualType ValueType = getDerived().TransformType(TLB, TL.getValueLoc());
+  if (ValueType.isNull())
+    return QualType();
+
+  QualType Result = TL.getType();
+  if (getDerived().AlwaysRebuild() ||
+      ValueType != TL.getValueLoc().getType()) {
+    Result = getDerived().RebuildAtomicType(ValueType, TL.getKWLoc());
+    if (Result.isNull())
+      return QualType();
+  }
+
+  AtomicTypeLoc NewTL = TLB.push<AtomicTypeLoc>(Result);
+  NewTL.setKWLoc(TL.getKWLoc());
+  NewTL.setLParenLoc(TL.getLParenLoc());
+  NewTL.setRParenLoc(TL.getRParenLoc());
+
+  return Result;
+}
+
 namespace {
   /// \brief Simple iterator that traverses the template arguments in a 
   /// container that provides a \c getArgLoc() member function.
@@ -5256,7 +5321,7 @@ template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
   bool DeclChanged = false;
-  llvm::SmallVector<Decl *, 4> Decls;
+  SmallVector<Decl *, 4> Decls;
   for (DeclStmt::decl_iterator D = S->decl_begin(), DEnd = S->decl_end();
        D != DEnd; ++D) {
     Decl *Transformed = getDerived().TransformDefinition((*D)->getLocation(),
@@ -5283,7 +5348,7 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
   
   ASTOwningVector<Expr*> Constraints(getSema());
   ASTOwningVector<Expr*> Exprs(getSema());
-  llvm::SmallVector<IdentifierInfo *, 4> Names;
+  SmallVector<IdentifierInfo *, 4> Names;
 
   ExprResult AsmString;
   ASTOwningVector<Expr*> Clobbers(getSema());
@@ -5470,6 +5535,11 @@ TreeTransform<Derived>::TransformObjCAtSynchronizedStmt(
   ExprResult Object = getDerived().TransformExpr(S->getSynchExpr());
   if (Object.isInvalid())
     return StmtError();
+  Object =
+    getDerived().RebuildObjCAtSynchronizedOperand(S->getAtSynchronizedLoc(),
+                                                  Object.get());
+  if (Object.isInvalid())
+    return StmtError();
   
   // Transform the body.
   StmtResult Body = getDerived().TransformStmt(S->getSynchBody());
@@ -5519,6 +5589,10 @@ TreeTransform<Derived>::TransformObjCForCollectionStmt(
   ExprResult Collection = getDerived().TransformExpr(S->getCollection());
   if (Collection.isInvalid())
     return StmtError();
+  Collection = getDerived().RebuildObjCForCollectionOperand(S->getForLoc(),
+                                                            Collection.take());
+  if (Collection.isInvalid())
+    return StmtError();
   
   // Transform the body.
   StmtResult Body = getDerived().TransformStmt(S->getBody());
@@ -5813,8 +5887,8 @@ TreeTransform<Derived>::TransformGenericSelectionExpr(GenericSelectionExpr *E) {
   if (ControllingExpr.isInvalid())
     return ExprError();
 
-  llvm::SmallVector<Expr *, 4> AssocExprs;
-  llvm::SmallVector<TypeSourceInfo *, 4> AssocTypes;
+  SmallVector<Expr *, 4> AssocExprs;
+  SmallVector<TypeSourceInfo *, 4> AssocTypes;
   for (unsigned i = 0; i != E->getNumAssocs(); ++i) {
     TypeSourceInfo *TS = E->getAssocTypeSourceInfo(i);
     if (TS) {
@@ -5887,7 +5961,7 @@ TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
   bool ExprChanged = false;
   typedef Sema::OffsetOfComponent Component;
   typedef OffsetOfExpr::OffsetOfNode Node;
-  llvm::SmallVector<Component, 4> Components;
+  SmallVector<Component, 4> Components;
   for (unsigned I = 0, N = E->getNumComponents(); I != N; ++I) {
     const Node &ON = E->getComponent(I);
     Component Comp;
@@ -6902,6 +6976,19 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
       SemaRef.MarkDeclarationReferenced(E->getLocStart(), OperatorNew);
     if (OperatorDelete)
       SemaRef.MarkDeclarationReferenced(E->getLocStart(), OperatorDelete);
+    
+    if (E->isArray() && Constructor && 
+        !E->getAllocatedType()->isDependentType()) {
+      QualType ElementType
+        = SemaRef.Context.getBaseElementType(E->getAllocatedType());
+      if (const RecordType *RecordT = ElementType->getAs<RecordType>()) {
+        CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordT->getDecl());
+        if (CXXDestructorDecl *Destructor = SemaRef.LookupDestructor(Record)) {
+          SemaRef.MarkDeclarationReferenced(E->getLocStart(), Destructor);
+        }
+      }
+    }
+    
     return SemaRef.Owned(E);
   }
 
@@ -7313,6 +7400,7 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
   return getDerived().RebuildCXXConstructExpr(T, /*FIXME:*/E->getLocStart(),
                                               Constructor, E->isElidable(),
                                               move_arg(Args),
+                                              E->hadMultipleCandidates(),
                                               E->requiresZeroInitialization(),
                                               E->getConstructionKind(),
                                               E->getParenRange());
@@ -7640,19 +7728,28 @@ TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
   bool RetainExpansion = false;
   llvm::Optional<unsigned> NumExpansions;
   if (getDerived().TryExpandParameterPacks(E->getOperatorLoc(), E->getPackLoc(), 
-                                           &Unexpanded, 1, 
+                                           Unexpanded,
                                            ShouldExpand, RetainExpansion,
                                            NumExpansions))
     return ExprError();
   
-  if (!ShouldExpand || RetainExpansion)
+  if (RetainExpansion)
     return SemaRef.Owned(E);
+    
+  NamedDecl *Pack = E->getPack();
+  if (!ShouldExpand) {
+    Pack = cast_or_null<NamedDecl>(getDerived().TransformDecl(E->getPackLoc(), 
+                                                              Pack));
+    if (!Pack)
+      return ExprError();
+  }
+
   
   // We now know the length of the parameter pack, so build a new expression
   // that stores that length.
-  return getDerived().RebuildSizeOfPackExpr(E->getOperatorLoc(), E->getPack(), 
+  return getDerived().RebuildSizeOfPackExpr(E->getOperatorLoc(), Pack, 
                                             E->getPackLoc(), E->getRParenLoc(), 
-                                            *NumExpansions);
+                                            NumExpansions);
 }
 
 template<typename Derived>
@@ -7762,9 +7859,11 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
       return SemaRef.Owned(E);
 
     // Build a new class message send.
+    SmallVector<SourceLocation, 16> SelLocs;
+    E->getSelectorLocs(SelLocs);
     return getDerived().RebuildObjCMessageExpr(ReceiverTypeInfo,
                                                E->getSelector(),
-                                               E->getSelectorLoc(),
+                                               SelLocs,
                                                E->getMethodDecl(),
                                                E->getLeftLoc(),
                                                move_arg(Args),
@@ -7785,9 +7884,11 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
     return SemaRef.Owned(E);
   
   // Build a new instance message send.
+  SmallVector<SourceLocation, 16> SelLocs;
+  E->getSelectorLocs(SelLocs);
   return getDerived().RebuildObjCMessageExpr(Receiver.get(),
                                              E->getSelector(),
-                                             E->getSelectorLoc(),
+                                             SelLocs,
                                              E->getMethodDecl(),
                                              E->getLeftLoc(),
                                              move_arg(Args),
@@ -7908,8 +8009,8 @@ TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
   // expression.
   blockScope->CapturesCXXThis = oldBlock->capturesCXXThis();
   
-  llvm::SmallVector<ParmVarDecl*, 4> params;
-  llvm::SmallVector<QualType, 4> paramTypes;
+  SmallVector<ParmVarDecl*, 4> params;
+  SmallVector<QualType, 4> paramTypes;
   
   // Parameter substitution.
   if (getDerived().TransformFunctionTypeParams(E->getCaretLocation(),
@@ -7951,7 +8052,7 @@ TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
 
   // Set the parameters on the block decl.
   if (!params.empty())
-    blockScope->TheDecl->setParams(params.data(), params.size());
+    blockScope->TheDecl->setParams(params);
   
   // If the return type wasn't explicitly set, it will have been marked as a 
   // dependent type (DependentTy); clear out the return type setting so 
@@ -8015,8 +8116,26 @@ TreeTransform<Derived>::TransformBlockDeclRefExpr(BlockDeclRefExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformAsTypeExpr(AsTypeExpr *E) {
-  assert(false && "Cannot transform asType expressions yet");
-  return SemaRef.Owned(E);
+  llvm_unreachable("Cannot transform asType expressions yet");
+}
+
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformAtomicExpr(AtomicExpr *E) {
+  QualType RetTy = getDerived().TransformType(E->getType());
+  bool ArgumentChanged = false;
+  ASTOwningVector<Expr*> SubExprs(SemaRef);
+  SubExprs.reserve(E->getNumSubExprs());
+  if (getDerived().TransformExprs(E->getSubExprs(), E->getNumSubExprs(), false,
+                                  SubExprs, &ArgumentChanged))
+    return ExprError();
+
+  if (!getDerived().AlwaysRebuild() &&
+      !ArgumentChanged)
+    return SemaRef.Owned(E);
+
+  return getDerived().RebuildAtomicExpr(E->getBuiltinLoc(), move_arg(SubExprs),
+                                        RetTy, E->getOp(), E->getRParenLoc());
 }
   
 //===----------------------------------------------------------------------===//
@@ -8238,6 +8357,12 @@ QualType TreeTransform<Derived>::RebuildTemplateSpecializationType(
   return SemaRef.CheckTemplateIdType(Template, TemplateNameLoc, TemplateArgs);
 }
 
+template<typename Derived>
+QualType TreeTransform<Derived>::RebuildAtomicType(QualType ValueType,
+                                                   SourceLocation KWLoc) {
+  return SemaRef.BuildAtomicType(ValueType, KWLoc);
+}
+
 template<typename Derived>
 TemplateName
 TreeTransform<Derived>::RebuildTemplateName(CXXScopeSpec &SS,
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.cpp
index 782e5c6aa79f..445e750834eb 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.cpp
@@ -43,6 +43,7 @@ serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
   case BuiltinType::Long:       ID = PREDEF_TYPE_LONG_ID;       break;
   case BuiltinType::LongLong:   ID = PREDEF_TYPE_LONGLONG_ID;   break;
   case BuiltinType::Int128:     ID = PREDEF_TYPE_INT128_ID;     break;
+  case BuiltinType::Half:       ID = PREDEF_TYPE_HALF_ID;       break;
   case BuiltinType::Float:      ID = PREDEF_TYPE_FLOAT_ID;      break;
   case BuiltinType::Double:     ID = PREDEF_TYPE_DOUBLE_ID;     break;
   case BuiltinType::LongDouble: ID = PREDEF_TYPE_LONGDOUBLE_ID; break;
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.h b/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.h
index 838df13f2d8f..367f57f7153a 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.h
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTCommon.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_SERIALIZATION_LIB_AST_COMMON_H
 
 #include "clang/Serialization/ASTBitCodes.h"
+#include "clang/AST/ASTContext.h"
 
 namespace clang {
 
@@ -31,7 +32,7 @@ enum DeclUpdateKind {
 TypeIdx TypeIdxFromBuiltin(const BuiltinType *BT);
 
 template <typename IdxForTypeTy>
-TypeID MakeTypeID(QualType T, IdxForTypeTy IdxForType) {
+TypeID MakeTypeID(ASTContext &Context, QualType T, IdxForTypeTy IdxForType) {
   if (T.isNull())
     return PREDEF_TYPE_NULL_ID;
 
@@ -46,6 +47,11 @@ TypeID MakeTypeID(QualType T, IdxForTypeTy IdxForType) {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(T.getTypePtr()))
     return TypeIdxFromBuiltin(BT).asTypeID(FastQuals);
 
+  if (T == Context.AutoDeductTy)
+    return TypeIdx(PREDEF_TYPE_AUTO_DEDUCT).asTypeID(FastQuals);
+  if (T == Context.AutoRRefDeductTy)
+    return TypeIdx(PREDEF_TYPE_AUTO_RREF_DEDUCT).asTypeID(FastQuals);
+
   return IdxForType(T).asTypeID(FastQuals);
 }
 
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp
index a4ed5f4da48c..fe1cc30158d1 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp
@@ -13,7 +13,9 @@
 
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
+#include "clang/Serialization/ModuleManager.h"
 #include "ASTCommon.h"
+#include "ASTReaderInternals.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Sema/Sema.h"
@@ -52,6 +54,7 @@
 
 using namespace clang;
 using namespace clang::serialization;
+using namespace clang::serialization::reader;
 
 //===----------------------------------------------------------------------===//
 // PCH validator implementation
@@ -62,103 +65,36 @@ ASTReaderListener::~ASTReaderListener() {}
 bool
 PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
   const LangOptions &PPLangOpts = PP.getLangOptions();
-#define PARSE_LANGOPT_BENIGN(Option)
-#define PARSE_LANGOPT_IMPORTANT(Option, DiagID)                    \
-  if (PPLangOpts.Option != LangOpts.Option) {                      \
-    Reader.Diag(DiagID) << LangOpts.Option << PPLangOpts.Option;   \
-    return true;                                                   \
+  
+#define LANGOPT(Name, Bits, Default, Description)         \
+  if (PPLangOpts.Name != LangOpts.Name) {                 \
+    Reader.Diag(diag::err_pch_langopt_mismatch)           \
+      << Description << LangOpts.Name << PPLangOpts.Name; \
+    return true;                                          \
   }
 
-  PARSE_LANGOPT_BENIGN(Trigraphs);
-  PARSE_LANGOPT_BENIGN(BCPLComment);
-  PARSE_LANGOPT_BENIGN(DollarIdents);
-  PARSE_LANGOPT_BENIGN(AsmPreprocessor);
-  PARSE_LANGOPT_IMPORTANT(GNUMode, diag::warn_pch_gnu_extensions);
-  PARSE_LANGOPT_IMPORTANT(GNUKeywords, diag::warn_pch_gnu_keywords);
-  PARSE_LANGOPT_BENIGN(ImplicitInt);
-  PARSE_LANGOPT_BENIGN(Digraphs);
-  PARSE_LANGOPT_BENIGN(HexFloats);
-  PARSE_LANGOPT_IMPORTANT(C99, diag::warn_pch_c99);
-  PARSE_LANGOPT_IMPORTANT(C1X, diag::warn_pch_c1x);
-  PARSE_LANGOPT_IMPORTANT(Microsoft, diag::warn_pch_microsoft_extensions);
-  PARSE_LANGOPT_BENIGN(MSCVersion);
-  PARSE_LANGOPT_IMPORTANT(CPlusPlus, diag::warn_pch_cplusplus);
-  PARSE_LANGOPT_IMPORTANT(CPlusPlus0x, diag::warn_pch_cplusplus0x);
-  PARSE_LANGOPT_BENIGN(CXXOperatorName);
-  PARSE_LANGOPT_IMPORTANT(ObjC1, diag::warn_pch_objective_c);
-  PARSE_LANGOPT_IMPORTANT(ObjC2, diag::warn_pch_objective_c2);
-  PARSE_LANGOPT_IMPORTANT(ObjCNonFragileABI, diag::warn_pch_nonfragile_abi);
-  PARSE_LANGOPT_IMPORTANT(ObjCNonFragileABI2, diag::warn_pch_nonfragile_abi2);
-  PARSE_LANGOPT_IMPORTANT(AppleKext, diag::warn_pch_apple_kext);
-  PARSE_LANGOPT_IMPORTANT(ObjCDefaultSynthProperties,
-                          diag::warn_pch_objc_auto_properties);
-  PARSE_LANGOPT_BENIGN(ObjCInferRelatedResultType)
-  PARSE_LANGOPT_IMPORTANT(NoConstantCFStrings,
-                          diag::warn_pch_no_constant_cfstrings);
-  PARSE_LANGOPT_BENIGN(PascalStrings);
-  PARSE_LANGOPT_BENIGN(WritableStrings);
-  PARSE_LANGOPT_IMPORTANT(LaxVectorConversions,
-                          diag::warn_pch_lax_vector_conversions);
-  PARSE_LANGOPT_IMPORTANT(AltiVec, diag::warn_pch_altivec);
-  PARSE_LANGOPT_IMPORTANT(Exceptions, diag::warn_pch_exceptions);
-  PARSE_LANGOPT_IMPORTANT(ObjCExceptions, diag::warn_pch_objc_exceptions);
-  PARSE_LANGOPT_IMPORTANT(CXXExceptions, diag::warn_pch_cxx_exceptions);
-  PARSE_LANGOPT_IMPORTANT(SjLjExceptions, diag::warn_pch_sjlj_exceptions);
-  PARSE_LANGOPT_IMPORTANT(MSBitfields, diag::warn_pch_ms_bitfields);
-  PARSE_LANGOPT_IMPORTANT(NeXTRuntime, diag::warn_pch_objc_runtime);
-  PARSE_LANGOPT_IMPORTANT(Freestanding, diag::warn_pch_freestanding);
-  PARSE_LANGOPT_IMPORTANT(NoBuiltin, diag::warn_pch_builtins);
-  PARSE_LANGOPT_IMPORTANT(ThreadsafeStatics,
-                          diag::warn_pch_thread_safe_statics);
-  PARSE_LANGOPT_IMPORTANT(POSIXThreads, diag::warn_pch_posix_threads);
-  PARSE_LANGOPT_IMPORTANT(Blocks, diag::warn_pch_blocks);
-  PARSE_LANGOPT_BENIGN(EmitAllDecls);
-  PARSE_LANGOPT_IMPORTANT(MathErrno, diag::warn_pch_math_errno);
-  PARSE_LANGOPT_BENIGN(getSignedOverflowBehavior());
-  PARSE_LANGOPT_IMPORTANT(HeinousExtensions,
-                          diag::warn_pch_heinous_extensions);
-  // FIXME: Most of the options below are benign if the macro wasn't
-  // used. Unfortunately, this means that a PCH compiled without
-  // optimization can't be used with optimization turned on, even
-  // though the only thing that changes is whether __OPTIMIZE__ was
-  // defined... but if __OPTIMIZE__ never showed up in the header, it
-  // doesn't matter. We could consider making this some special kind
-  // of check.
-  PARSE_LANGOPT_IMPORTANT(Optimize, diag::warn_pch_optimize);
-  PARSE_LANGOPT_IMPORTANT(OptimizeSize, diag::warn_pch_optimize_size);
-  PARSE_LANGOPT_IMPORTANT(Static, diag::warn_pch_static);
-  PARSE_LANGOPT_IMPORTANT(PICLevel, diag::warn_pch_pic_level);
-  PARSE_LANGOPT_IMPORTANT(GNUInline, diag::warn_pch_gnu_inline);
-  PARSE_LANGOPT_IMPORTANT(NoInline, diag::warn_pch_no_inline);
-  PARSE_LANGOPT_IMPORTANT(Deprecated, diag::warn_pch_deprecated);
-  PARSE_LANGOPT_IMPORTANT(AccessControl, diag::warn_pch_access_control);
-  PARSE_LANGOPT_IMPORTANT(CharIsSigned, diag::warn_pch_char_signed);
-  PARSE_LANGOPT_IMPORTANT(ShortWChar, diag::warn_pch_short_wchar);
-  PARSE_LANGOPT_IMPORTANT(ShortEnums, diag::warn_pch_short_enums);
-  if ((PPLangOpts.getGCMode() != 0) != (LangOpts.getGCMode() != 0)) {
-    Reader.Diag(diag::warn_pch_gc_mode)
-      << LangOpts.getGCMode() << PPLangOpts.getGCMode();
-    return true;
-  }
-  PARSE_LANGOPT_BENIGN(getVisibilityMode());
-  PARSE_LANGOPT_IMPORTANT(getStackProtectorMode(),
-                          diag::warn_pch_stack_protector);
-  PARSE_LANGOPT_BENIGN(InstantiationDepth);
-  PARSE_LANGOPT_IMPORTANT(OpenCL, diag::warn_pch_opencl);
-  PARSE_LANGOPT_IMPORTANT(CUDA, diag::warn_pch_cuda);
-  PARSE_LANGOPT_BENIGN(CatchUndefined);
-  PARSE_LANGOPT_BENIGN(DefaultFPContract);
-  PARSE_LANGOPT_IMPORTANT(ElideConstructors, diag::warn_pch_elide_constructors);
-  PARSE_LANGOPT_BENIGN(SpellChecking);
-  PARSE_LANGOPT_IMPORTANT(ObjCAutoRefCount, diag::warn_pch_auto_ref_count);
-  PARSE_LANGOPT_BENIGN(ObjCInferRelatedReturnType);
-#undef PARSE_LANGOPT_IMPORTANT
-#undef PARSE_LANGOPT_BENIGN
+#define VALUE_LANGOPT(Name, Bits, Default, Description) \
+  if (PPLangOpts.Name != LangOpts.Name) {               \
+    Reader.Diag(diag::err_pch_langopt_value_mismatch)   \
+      << Description;                                   \
+  return true;                                          \
+}
 
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  if (PPLangOpts.get##Name() != LangOpts.get##Name()) {      \
+    Reader.Diag(diag::err_pch_langopt_value_mismatch)        \
+      << Description;                                        \
+    return true;                                             \
+  }
+
+#define BENIGN_LANGOPT(Name, Bits, Default, Description)
+#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+#include "clang/Basic/LangOptions.def"
+  
   return false;
 }
 
-bool PCHValidator::ReadTargetTriple(llvm::StringRef Triple) {
+bool PCHValidator::ReadTargetTriple(StringRef Triple) {
   if (Triple == PP.getTargetInfo().getTriple().str())
     return false;
 
@@ -169,14 +105,14 @@ bool PCHValidator::ReadTargetTriple(llvm::StringRef Triple) {
 
 namespace {
   struct EmptyStringRef {
-    bool operator ()(llvm::StringRef r) const { return r.empty(); }
+    bool operator ()(StringRef r) const { return r.empty(); }
   };
   struct EmptyBlock {
     bool operator ()(const PCHPredefinesBlock &r) const {return r.Data.empty();}
   };
 }
 
-static bool EqualConcatenations(llvm::SmallVector<llvm::StringRef, 2> L,
+static bool EqualConcatenations(SmallVector<StringRef, 2> L,
                                 PCHPredefinesBlocks R) {
   // First, sum up the lengths.
   unsigned LL = 0, RL = 0;
@@ -196,7 +132,7 @@ static bool EqualConcatenations(llvm::SmallVector<llvm::StringRef, 2> L,
   R.erase(std::remove_if(R.begin(), R.end(), EmptyBlock()), R.end());
 
   // Do it the hard way. At this point, both vectors must be non-empty.
-  llvm::StringRef LR = L[0], RR = R[0].Data;
+  StringRef LR = L[0], RR = R[0].Data;
   unsigned LI = 0, RI = 0, LN = L.size(), RN = R.size();
   (void) RN;
   for (;;) {
@@ -236,12 +172,12 @@ static bool EqualConcatenations(llvm::SmallVector<llvm::StringRef, 2> L,
   }
 }
 
-static std::pair<FileID, llvm::StringRef::size_type>
-FindMacro(const PCHPredefinesBlocks &Buffers, llvm::StringRef MacroDef) {
-  std::pair<FileID, llvm::StringRef::size_type> Res;
+static std::pair<FileID, StringRef::size_type>
+FindMacro(const PCHPredefinesBlocks &Buffers, StringRef MacroDef) {
+  std::pair<FileID, StringRef::size_type> Res;
   for (unsigned I = 0, N = Buffers.size(); I != N; ++I) {
     Res.second = Buffers[I].Data.find(MacroDef);
-    if (Res.second != llvm::StringRef::npos) {
+    if (Res.second != StringRef::npos) {
       Res.first = Buffers[I].BufferID;
       break;
     }
@@ -250,7 +186,7 @@ FindMacro(const PCHPredefinesBlocks &Buffers, llvm::StringRef MacroDef) {
 }
 
 bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
-                                        llvm::StringRef OriginalFileName,
+                                        StringRef OriginalFileName,
                                         std::string &SuggestedPredefines,
                                         FileManager &FileMgr) {
   // We are in the context of an implicit include, so the predefines buffer will
@@ -261,9 +197,9 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
   PCHInclude += "#include \"";
   PCHInclude += NormalizeDashIncludePath(OriginalFileName, FileMgr);
   PCHInclude += "\"\n";
-  std::pair<llvm::StringRef,llvm::StringRef> Split =
-    llvm::StringRef(PP.getPredefines()).split(PCHInclude.str());
-  llvm::StringRef Left =  Split.first, Right = Split.second;
+  std::pair<StringRef,StringRef> Split =
+    StringRef(PP.getPredefines()).split(PCHInclude.str());
+  StringRef Left =  Split.first, Right = Split.second;
   if (Left == PP.getPredefines()) {
     Error("Missing PCH include entry!");
     return true;
@@ -271,7 +207,7 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
 
   // If the concatenation of all the PCH buffers is equal to the adjusted
   // command line, we're done.
-  llvm::SmallVector<llvm::StringRef, 2> CommandLine;
+  SmallVector<StringRef, 2> CommandLine;
   CommandLine.push_back(Left);
   CommandLine.push_back(Right);
   if (EqualConcatenations(CommandLine, Buffers))
@@ -281,18 +217,18 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
 
   // The predefines buffers are different. Determine what the differences are,
   // and whether they require us to reject the PCH file.
-  llvm::SmallVector<llvm::StringRef, 8> PCHLines;
+  SmallVector<StringRef, 8> PCHLines;
   for (unsigned I = 0, N = Buffers.size(); I != N; ++I)
     Buffers[I].Data.split(PCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
 
-  llvm::SmallVector<llvm::StringRef, 8> CmdLineLines;
+  SmallVector<StringRef, 8> CmdLineLines;
   Left.split(CmdLineLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
 
   // Pick out implicit #includes after the PCH and don't consider them for
   // validation; we will insert them into SuggestedPredefines so that the
   // preprocessor includes them.
   std::string IncludesAfterPCH;
-  llvm::SmallVector<llvm::StringRef, 8> AfterPCHLines;
+  SmallVector<StringRef, 8> AfterPCHLines;
   Right.split(AfterPCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
   for (unsigned i = 0, e = AfterPCHLines.size(); i != e; ++i) {
     if (AfterPCHLines[i].startswith("#include ")) {
@@ -325,7 +261,7 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
 
   // Determine which predefines that were used to build the PCH file are missing
   // from the command line.
-  std::vector<llvm::StringRef> MissingPredefines;
+  std::vector<StringRef> MissingPredefines;
   std::set_difference(PCHLines.begin(), PCHLines.end(),
                       CmdLineLines.begin(), CmdLineLines.end(),
                       std::back_inserter(MissingPredefines));
@@ -333,7 +269,7 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
   bool MissingDefines = false;
   bool ConflictingDefines = false;
   for (unsigned I = 0, N = MissingPredefines.size(); I != N; ++I) {
-    llvm::StringRef Missing = MissingPredefines[I];
+    StringRef Missing = MissingPredefines[I];
     if (Missing.startswith("#include ")) {
       // An -include was specified when generating the PCH; it is included in
       // the PCH, just ignore it.
@@ -351,13 +287,13 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
       = Missing.find_first_of("( \n\r", StartOfMacroName);
     assert(EndOfMacroName != std::string::npos &&
            "Couldn't find the end of the macro name");
-    llvm::StringRef MacroName = Missing.slice(StartOfMacroName, EndOfMacroName);
+    StringRef MacroName = Missing.slice(StartOfMacroName, EndOfMacroName);
 
     // Determine whether this macro was given a different definition on the
     // command line.
     std::string MacroDefStart = "#define " + MacroName.str();
     std::string::size_type MacroDefLen = MacroDefStart.size();
-    llvm::SmallVector<llvm::StringRef, 8>::iterator ConflictPos
+    SmallVector<StringRef, 8>::iterator ConflictPos
       = std::lower_bound(CmdLineLines.begin(), CmdLineLines.end(),
                          MacroDefStart);
     for (; ConflictPos != CmdLineLines.end(); ++ConflictPos) {
@@ -382,12 +318,12 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
           << MacroName;
 
       // Show the definition of this macro within the PCH file.
-      std::pair<FileID, llvm::StringRef::size_type> MacroLoc =
+      std::pair<FileID, StringRef::size_type> MacroLoc =
           FindMacro(Buffers, Missing);
-      assert(MacroLoc.second!=llvm::StringRef::npos && "Unable to find macro!");
+      assert(MacroLoc.second!=StringRef::npos && "Unable to find macro!");
       SourceLocation PCHMissingLoc =
           SourceMgr.getLocForStartOfFile(MacroLoc.first)
-            .getFileLocWithOffset(MacroLoc.second);
+            .getLocWithOffset(MacroLoc.second);
       Reader.Diag(PCHMissingLoc, diag::note_pch_macro_defined_as) << MacroName;
 
       ConflictingDefines = true;
@@ -405,12 +341,12 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
     }
 
     // Show the definition of this macro within the PCH file.
-    std::pair<FileID, llvm::StringRef::size_type> MacroLoc =
+    std::pair<FileID, StringRef::size_type> MacroLoc =
         FindMacro(Buffers, Missing);
-    assert(MacroLoc.second!=llvm::StringRef::npos && "Unable to find macro!");
+    assert(MacroLoc.second!=StringRef::npos && "Unable to find macro!");
     SourceLocation PCHMissingLoc =
         SourceMgr.getLocForStartOfFile(MacroLoc.first)
-          .getFileLocWithOffset(MacroLoc.second);
+          .getLocWithOffset(MacroLoc.second);
     Reader.Diag(PCHMissingLoc, diag::note_using_macro_def_from_pch);
   }
 
@@ -421,12 +357,12 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
   // parameters that were not present when building the PCH
   // file. Extra #defines are okay, so long as the identifiers being
   // defined were not used within the precompiled header.
-  std::vector<llvm::StringRef> ExtraPredefines;
+  std::vector<StringRef> ExtraPredefines;
   std::set_difference(CmdLineLines.begin(), CmdLineLines.end(),
                       PCHLines.begin(), PCHLines.end(),
                       std::back_inserter(ExtraPredefines));
   for (unsigned I = 0, N = ExtraPredefines.size(); I != N; ++I) {
-    llvm::StringRef &Extra = ExtraPredefines[I];
+    StringRef &Extra = ExtraPredefines[I];
     if (!Extra.startswith("#define ")) {
       Reader.Diag(diag::warn_pch_compiler_options_mismatch);
       return true;
@@ -439,7 +375,7 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
       = Extra.find_first_of("( \n\r", StartOfMacroName);
     assert(EndOfMacroName != std::string::npos &&
            "Couldn't find the end of the macro name");
-    llvm::StringRef MacroName = Extra.slice(StartOfMacroName, EndOfMacroName);
+    StringRef MacroName = Extra.slice(StartOfMacroName, EndOfMacroName);
 
     // Check whether this name was used somewhere in the PCH file. If
     // so, defining it as a macro could change behavior, so we reject
@@ -479,440 +415,321 @@ ASTReader::setDeserializationListener(ASTDeserializationListener *Listener) {
 }
 
 
-namespace {
-class ASTSelectorLookupTrait {
-  ASTReader &Reader;
 
-public:
-  struct data_type {
-    SelectorID ID;
-    ObjCMethodList Instance, Factory;
-  };
+unsigned ASTSelectorLookupTrait::ComputeHash(Selector Sel) {
+  return serialization::ComputeHash(Sel);
+}
 
-  typedef Selector external_key_type;
-  typedef external_key_type internal_key_type;
 
-  explicit ASTSelectorLookupTrait(ASTReader &Reader) : Reader(Reader) { }
+std::pair<unsigned, unsigned>
+ASTSelectorLookupTrait::ReadKeyDataLength(const unsigned char*& d) {
+  using namespace clang::io;
+  unsigned KeyLen = ReadUnalignedLE16(d);
+  unsigned DataLen = ReadUnalignedLE16(d);
+  return std::make_pair(KeyLen, DataLen);
+}
 
-  static bool EqualKey(const internal_key_type& a,
-                       const internal_key_type& b) {
-    return a == b;
+ASTSelectorLookupTrait::internal_key_type 
+ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
+  using namespace clang::io;
+  SelectorTable &SelTable = Reader.getContext().Selectors;
+  unsigned N = ReadUnalignedLE16(d);
+  IdentifierInfo *FirstII
+    = Reader.getLocalIdentifier(F, ReadUnalignedLE32(d));
+  if (N == 0)
+    return SelTable.getNullarySelector(FirstII);
+  else if (N == 1)
+    return SelTable.getUnarySelector(FirstII);
+
+  SmallVector<IdentifierInfo *, 16> Args;
+  Args.push_back(FirstII);
+  for (unsigned I = 1; I != N; ++I)
+    Args.push_back(Reader.getLocalIdentifier(F, ReadUnalignedLE32(d)));
+
+  return SelTable.getSelector(N, Args.data());
+}
+
+ASTSelectorLookupTrait::data_type 
+ASTSelectorLookupTrait::ReadData(Selector, const unsigned char* d, 
+                                 unsigned DataLen) {
+  using namespace clang::io;
+
+  data_type Result;
+
+  Result.ID = Reader.getGlobalSelectorID(F, ReadUnalignedLE32(d));
+  unsigned NumInstanceMethods = ReadUnalignedLE16(d);
+  unsigned NumFactoryMethods = ReadUnalignedLE16(d);
+
+  // Load instance methods
+  for (unsigned I = 0; I != NumInstanceMethods; ++I) {
+    if (ObjCMethodDecl *Method
+          = Reader.GetLocalDeclAs<ObjCMethodDecl>(F, ReadUnalignedLE32(d)))
+      Result.Instance.push_back(Method);
   }
 
-  static unsigned ComputeHash(Selector Sel) {
-    return serialization::ComputeHash(Sel);
+  // Load factory methods
+  for (unsigned I = 0; I != NumFactoryMethods; ++I) {
+    if (ObjCMethodDecl *Method
+          = Reader.GetLocalDeclAs<ObjCMethodDecl>(F, ReadUnalignedLE32(d)))
+      Result.Factory.push_back(Method);
   }
 
-  // This hopefully will just get inlined and removed by the optimizer.
-  static const internal_key_type&
-  GetInternalKey(const external_key_type& x) { return x; }
+  return Result;
+}
 
-  static std::pair<unsigned, unsigned>
-  ReadKeyDataLength(const unsigned char*& d) {
-    using namespace clang::io;
-    unsigned KeyLen = ReadUnalignedLE16(d);
-    unsigned DataLen = ReadUnalignedLE16(d);
-    return std::make_pair(KeyLen, DataLen);
-  }
+unsigned ASTIdentifierLookupTrait::ComputeHash(const internal_key_type& a) {
+  return llvm::HashString(StringRef(a.first, a.second));
+}
 
-  internal_key_type ReadKey(const unsigned char* d, unsigned) {
-    using namespace clang::io;
-    SelectorTable &SelTable = Reader.getContext()->Selectors;
-    unsigned N = ReadUnalignedLE16(d);
-    IdentifierInfo *FirstII
-      = Reader.DecodeIdentifierInfo(ReadUnalignedLE32(d));
-    if (N == 0)
-      return SelTable.getNullarySelector(FirstII);
-    else if (N == 1)
-      return SelTable.getUnarySelector(FirstII);
+std::pair<unsigned, unsigned>
+ASTIdentifierLookupTrait::ReadKeyDataLength(const unsigned char*& d) {
+  using namespace clang::io;
+  unsigned DataLen = ReadUnalignedLE16(d);
+  unsigned KeyLen = ReadUnalignedLE16(d);
+  return std::make_pair(KeyLen, DataLen);
+}
 
-    llvm::SmallVector<IdentifierInfo *, 16> Args;
-    Args.push_back(FirstII);
-    for (unsigned I = 1; I != N; ++I)
-      Args.push_back(Reader.DecodeIdentifierInfo(ReadUnalignedLE32(d)));
+std::pair<const char*, unsigned>
+ASTIdentifierLookupTrait::ReadKey(const unsigned char* d, unsigned n) {
+  assert(n >= 2 && d[n-1] == '\0');
+  return std::make_pair((const char*) d, n-1);
+}
 
-    return SelTable.getSelector(N, Args.data());
-  }
+IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
+                                                   const unsigned char* d,
+                                                   unsigned DataLen) {
+  using namespace clang::io;
+  unsigned RawID = ReadUnalignedLE32(d);
+  bool IsInteresting = RawID & 0x01;
 
-  data_type ReadData(Selector, const unsigned char* d, unsigned DataLen) {
-    using namespace clang::io;
+  // Wipe out the "is interesting" bit.
+  RawID = RawID >> 1;
 
-    data_type Result;
-
-    Result.ID = ReadUnalignedLE32(d);
-    unsigned NumInstanceMethods = ReadUnalignedLE16(d);
-    unsigned NumFactoryMethods = ReadUnalignedLE16(d);
-
-    // Load instance methods
-    ObjCMethodList *Prev = 0;
-    for (unsigned I = 0; I != NumInstanceMethods; ++I) {
-      ObjCMethodDecl *Method
-        = cast<ObjCMethodDecl>(Reader.GetDecl(ReadUnalignedLE32(d)));
-      if (!Result.Instance.Method) {
-        // This is the first method, which is the easy case.
-        Result.Instance.Method = Method;
-        Prev = &Result.Instance;
-        continue;
-      }
-
-      ObjCMethodList *Mem =
-        Reader.getSema()->BumpAlloc.Allocate<ObjCMethodList>();
-      Prev->Next = new (Mem) ObjCMethodList(Method, 0);
-      Prev = Prev->Next;
-    }
-
-    // Load factory methods
-    Prev = 0;
-    for (unsigned I = 0; I != NumFactoryMethods; ++I) {
-      ObjCMethodDecl *Method
-        = cast<ObjCMethodDecl>(Reader.GetDecl(ReadUnalignedLE32(d)));
-      if (!Result.Factory.Method) {
-        // This is the first method, which is the easy case.
-        Result.Factory.Method = Method;
-        Prev = &Result.Factory;
-        continue;
-      }
-
-      ObjCMethodList *Mem =
-        Reader.getSema()->BumpAlloc.Allocate<ObjCMethodList>();
-      Prev->Next = new (Mem) ObjCMethodList(Method, 0);
-      Prev = Prev->Next;
-    }
-
-    return Result;
-  }
-};
-
-} // end anonymous namespace
-
-/// \brief The on-disk hash table used for the global method pool.
-typedef OnDiskChainedHashTable<ASTSelectorLookupTrait>
-  ASTSelectorLookupTable;
-
-namespace clang {
-class ASTIdentifierLookupTrait {
-  ASTReader &Reader;
-  ASTReader::PerFileData &F;
-
-  // If we know the IdentifierInfo in advance, it is here and we will
-  // not build a new one. Used when deserializing information about an
-  // identifier that was constructed before the AST file was read.
-  IdentifierInfo *KnownII;
-
-public:
-  typedef IdentifierInfo * data_type;
-
-  typedef const std::pair<const char*, unsigned> external_key_type;
-
-  typedef external_key_type internal_key_type;
-
-  ASTIdentifierLookupTrait(ASTReader &Reader, ASTReader::PerFileData &F,
-                           IdentifierInfo *II = 0)
-    : Reader(Reader), F(F), KnownII(II) { }
-
-  static bool EqualKey(const internal_key_type& a,
-                       const internal_key_type& b) {
-    return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
-                                  : false;
-  }
-
-  static unsigned ComputeHash(const internal_key_type& a) {
-    return llvm::HashString(llvm::StringRef(a.first, a.second));
-  }
-
-  // This hopefully will just get inlined and removed by the optimizer.
-  static const internal_key_type&
-  GetInternalKey(const external_key_type& x) { return x; }
-
-  // This hopefully will just get inlined and removed by the optimizer.
-  static const external_key_type&
-  GetExternalKey(const internal_key_type& x) { return x; }
-
-  static std::pair<unsigned, unsigned>
-  ReadKeyDataLength(const unsigned char*& d) {
-    using namespace clang::io;
-    unsigned DataLen = ReadUnalignedLE16(d);
-    unsigned KeyLen = ReadUnalignedLE16(d);
-    return std::make_pair(KeyLen, DataLen);
-  }
-
-  static std::pair<const char*, unsigned>
-  ReadKey(const unsigned char* d, unsigned n) {
-    assert(n >= 2 && d[n-1] == '\0');
-    return std::make_pair((const char*) d, n-1);
-  }
-
-  IdentifierInfo *ReadData(const internal_key_type& k,
-                           const unsigned char* d,
-                           unsigned DataLen) {
-    using namespace clang::io;
-    IdentID ID = ReadUnalignedLE32(d);
-    bool IsInteresting = ID & 0x01;
-
-    // Wipe out the "is interesting" bit.
-    ID = ID >> 1;
-
-    if (!IsInteresting) {
-      // For uninteresting identifiers, just build the IdentifierInfo
-      // and associate it with the persistent ID.
-      IdentifierInfo *II = KnownII;
-      if (!II)
-        II = &Reader.getIdentifierTable().getOwn(llvm::StringRef(k.first,
-                                                                 k.second));
-      Reader.SetIdentifierInfo(ID, II);
-      II->setIsFromAST();
-      return II;
-    }
-
-    unsigned Bits = ReadUnalignedLE16(d);
-    bool CPlusPlusOperatorKeyword = Bits & 0x01;
-    Bits >>= 1;
-    bool HasRevertedTokenIDToIdentifier = Bits & 0x01;
-    Bits >>= 1;
-    bool Poisoned = Bits & 0x01;
-    Bits >>= 1;
-    bool ExtensionToken = Bits & 0x01;
-    Bits >>= 1;
-    bool hasMacroDefinition = Bits & 0x01;
-    Bits >>= 1;
-    unsigned ObjCOrBuiltinID = Bits & 0x3FF;
-    Bits >>= 10;
-
-    assert(Bits == 0 && "Extra bits in the identifier?");
-    DataLen -= 6;
-
-    // Build the IdentifierInfo itself and link the identifier ID with
-    // the new IdentifierInfo.
+  IdentID ID = Reader.getGlobalIdentifierID(F, RawID);
+  if (!IsInteresting) {
+    // For uninteresting identifiers, just build the IdentifierInfo
+    // and associate it with the persistent ID.
     IdentifierInfo *II = KnownII;
     if (!II)
-      II = &Reader.getIdentifierTable().getOwn(llvm::StringRef(k.first,
-                                                               k.second));
+      II = &Reader.getIdentifierTable().getOwn(StringRef(k.first, k.second));
     Reader.SetIdentifierInfo(ID, II);
-
-    // Set or check the various bits in the IdentifierInfo structure.
-    // Token IDs are read-only.
-    if (HasRevertedTokenIDToIdentifier)
-      II->RevertTokenIDToIdentifier();
-    II->setObjCOrBuiltinID(ObjCOrBuiltinID);
-    assert(II->isExtensionToken() == ExtensionToken &&
-           "Incorrect extension token flag");
-    (void)ExtensionToken;
-    II->setIsPoisoned(Poisoned);
-    assert(II->isCPlusPlusOperatorKeyword() == CPlusPlusOperatorKeyword &&
-           "Incorrect C++ operator keyword flag");
-    (void)CPlusPlusOperatorKeyword;
-
-    // If this identifier is a macro, deserialize the macro
-    // definition.
-    if (hasMacroDefinition) {
-      uint32_t Offset = ReadUnalignedLE32(d);
-      Reader.SetIdentifierIsMacro(II, F, Offset);
-      DataLen -= 4;
-    }
-
-    // Read all of the declarations visible at global scope with this
-    // name.
-    if (Reader.getContext() == 0) return II;
-    if (DataLen > 0) {
-      llvm::SmallVector<uint32_t, 4> DeclIDs;
-      for (; DataLen > 0; DataLen -= 4)
-        DeclIDs.push_back(ReadUnalignedLE32(d));
-      Reader.SetGloballyVisibleDecls(II, DeclIDs);
-    }
-
     II->setIsFromAST();
     return II;
   }
-};
 
-} // end anonymous namespace
+  unsigned Bits = ReadUnalignedLE16(d);
+  bool CPlusPlusOperatorKeyword = Bits & 0x01;
+  Bits >>= 1;
+  bool HasRevertedTokenIDToIdentifier = Bits & 0x01;
+  Bits >>= 1;
+  bool Poisoned = Bits & 0x01;
+  Bits >>= 1;
+  bool ExtensionToken = Bits & 0x01;
+  Bits >>= 1;
+  bool hasMacroDefinition = Bits & 0x01;
+  Bits >>= 1;
+  unsigned ObjCOrBuiltinID = Bits & 0x3FF;
+  Bits >>= 10;
 
-/// \brief The on-disk hash table used to contain information about
-/// all of the identifiers in the program.
-typedef OnDiskChainedHashTable<ASTIdentifierLookupTrait>
-  ASTIdentifierLookupTable;
+  assert(Bits == 0 && "Extra bits in the identifier?");
+  DataLen -= 6;
 
-namespace {
-class ASTDeclContextNameLookupTrait {
-  ASTReader &Reader;
+  // Build the IdentifierInfo itself and link the identifier ID with
+  // the new IdentifierInfo.
+  IdentifierInfo *II = KnownII;
+  if (!II)
+    II = &Reader.getIdentifierTable().getOwn(StringRef(k.first, k.second));
+  Reader.SetIdentifierInfo(ID, II);
 
-public:
-  /// \brief Pair of begin/end iterators for DeclIDs.
-  typedef std::pair<DeclID *, DeclID *> data_type;
+  // Set or check the various bits in the IdentifierInfo structure.
+  // Token IDs are read-only.
+  if (HasRevertedTokenIDToIdentifier)
+    II->RevertTokenIDToIdentifier();
+  II->setObjCOrBuiltinID(ObjCOrBuiltinID);
+  assert(II->isExtensionToken() == ExtensionToken &&
+         "Incorrect extension token flag");
+  (void)ExtensionToken;
+  if (Poisoned)
+    II->setIsPoisoned(true);
+  assert(II->isCPlusPlusOperatorKeyword() == CPlusPlusOperatorKeyword &&
+         "Incorrect C++ operator keyword flag");
+  (void)CPlusPlusOperatorKeyword;
 
-  /// \brief Special internal key for declaration names.
-  /// The hash table creates keys for comparison; we do not create
-  /// a DeclarationName for the internal key to avoid deserializing types.
-  struct DeclNameKey {
-    DeclarationName::NameKind Kind;
-    uint64_t Data;
-    DeclNameKey() : Kind((DeclarationName::NameKind)0), Data(0) { }
-  };
-
-  typedef DeclarationName external_key_type;
-  typedef DeclNameKey internal_key_type;
-
-  explicit ASTDeclContextNameLookupTrait(ASTReader &Reader) : Reader(Reader) { }
-
-  static bool EqualKey(const internal_key_type& a,
-                       const internal_key_type& b) {
-    return a.Kind == b.Kind && a.Data == b.Data;
+  // If this identifier is a macro, deserialize the macro
+  // definition.
+  if (hasMacroDefinition) {
+    // FIXME: Check for conflicts?
+    uint32_t Offset = ReadUnalignedLE32(d);
+    Reader.SetIdentifierIsMacro(II, F, Offset);
+    DataLen -= 4;
   }
 
-  unsigned ComputeHash(const DeclNameKey &Key) const {
-    llvm::FoldingSetNodeID ID;
-    ID.AddInteger(Key.Kind);
-
-    switch (Key.Kind) {
-    case DeclarationName::Identifier:
-    case DeclarationName::CXXLiteralOperatorName:
-      ID.AddString(((IdentifierInfo*)Key.Data)->getName());
-      break;
-    case DeclarationName::ObjCZeroArgSelector:
-    case DeclarationName::ObjCOneArgSelector:
-    case DeclarationName::ObjCMultiArgSelector:
-      ID.AddInteger(serialization::ComputeHash(Selector(Key.Data)));
-      break;
-    case DeclarationName::CXXConstructorName:
-    case DeclarationName::CXXDestructorName:
-    case DeclarationName::CXXConversionFunctionName:
-      ID.AddInteger((TypeID)Key.Data);
-      break;
-    case DeclarationName::CXXOperatorName:
-      ID.AddInteger((OverloadedOperatorKind)Key.Data);
-      break;
-    case DeclarationName::CXXUsingDirective:
-      break;
-    }
-
-    return ID.ComputeHash();
+  // Read all of the declarations visible at global scope with this
+  // name.
+  if (DataLen > 0) {
+    SmallVector<uint32_t, 4> DeclIDs;
+    for (; DataLen > 0; DataLen -= 4)
+      DeclIDs.push_back(Reader.getGlobalDeclID(F, ReadUnalignedLE32(d)));
+    Reader.SetGloballyVisibleDecls(II, DeclIDs);
   }
 
-  internal_key_type GetInternalKey(const external_key_type& Name) const {
-    DeclNameKey Key;
-    Key.Kind = Name.getNameKind();
-    switch (Name.getNameKind()) {
-    case DeclarationName::Identifier:
-      Key.Data = (uint64_t)Name.getAsIdentifierInfo();
-      break;
-    case DeclarationName::ObjCZeroArgSelector:
-    case DeclarationName::ObjCOneArgSelector:
-    case DeclarationName::ObjCMultiArgSelector:
-      Key.Data = (uint64_t)Name.getObjCSelector().getAsOpaquePtr();
-      break;
-    case DeclarationName::CXXConstructorName:
-    case DeclarationName::CXXDestructorName:
-    case DeclarationName::CXXConversionFunctionName:
-      Key.Data = Reader.GetTypeID(Name.getCXXNameType());
-      break;
-    case DeclarationName::CXXOperatorName:
-      Key.Data = Name.getCXXOverloadedOperator();
-      break;
-    case DeclarationName::CXXLiteralOperatorName:
-      Key.Data = (uint64_t)Name.getCXXLiteralIdentifier();
-      break;
-    case DeclarationName::CXXUsingDirective:
-      break;
-    }
+  II->setIsFromAST();
+  return II;
+}
 
-    return Key;
+unsigned 
+ASTDeclContextNameLookupTrait::ComputeHash(const DeclNameKey &Key) const {
+  llvm::FoldingSetNodeID ID;
+  ID.AddInteger(Key.Kind);
+
+  switch (Key.Kind) {
+  case DeclarationName::Identifier:
+  case DeclarationName::CXXLiteralOperatorName:
+    ID.AddString(((IdentifierInfo*)Key.Data)->getName());
+    break;
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    ID.AddInteger(serialization::ComputeHash(Selector(Key.Data)));
+    break;
+  case DeclarationName::CXXOperatorName:
+    ID.AddInteger((OverloadedOperatorKind)Key.Data);
+    break;
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+  case DeclarationName::CXXUsingDirective:
+    break;
   }
 
-  external_key_type GetExternalKey(const internal_key_type& Key) const {
-    ASTContext *Context = Reader.getContext();
-    switch (Key.Kind) {
-    case DeclarationName::Identifier:
-      return DeclarationName((IdentifierInfo*)Key.Data);
+  return ID.ComputeHash();
+}
 
-    case DeclarationName::ObjCZeroArgSelector:
-    case DeclarationName::ObjCOneArgSelector:
-    case DeclarationName::ObjCMultiArgSelector:
-      return DeclarationName(Selector(Key.Data));
-
-    case DeclarationName::CXXConstructorName:
-      return Context->DeclarationNames.getCXXConstructorName(
-                           Context->getCanonicalType(Reader.GetType(Key.Data)));
-
-    case DeclarationName::CXXDestructorName:
-      return Context->DeclarationNames.getCXXDestructorName(
-                           Context->getCanonicalType(Reader.GetType(Key.Data)));
-
-    case DeclarationName::CXXConversionFunctionName:
-      return Context->DeclarationNames.getCXXConversionFunctionName(
-                           Context->getCanonicalType(Reader.GetType(Key.Data)));
-
-    case DeclarationName::CXXOperatorName:
-      return Context->DeclarationNames.getCXXOperatorName(
-                                         (OverloadedOperatorKind)Key.Data);
-
-    case DeclarationName::CXXLiteralOperatorName:
-      return Context->DeclarationNames.getCXXLiteralOperatorName(
-                                                     (IdentifierInfo*)Key.Data);
-
-    case DeclarationName::CXXUsingDirective:
-      return DeclarationName::getUsingDirectiveName();
-    }
-
-    llvm_unreachable("Invalid Name Kind ?");
+ASTDeclContextNameLookupTrait::internal_key_type 
+ASTDeclContextNameLookupTrait::GetInternalKey(
+                                          const external_key_type& Name) const {
+  DeclNameKey Key;
+  Key.Kind = Name.getNameKind();
+  switch (Name.getNameKind()) {
+  case DeclarationName::Identifier:
+    Key.Data = (uint64_t)Name.getAsIdentifierInfo();
+    break;
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    Key.Data = (uint64_t)Name.getObjCSelector().getAsOpaquePtr();
+    break;
+  case DeclarationName::CXXOperatorName:
+    Key.Data = Name.getCXXOverloadedOperator();
+    break;
+  case DeclarationName::CXXLiteralOperatorName:
+    Key.Data = (uint64_t)Name.getCXXLiteralIdentifier();
+    break;
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+  case DeclarationName::CXXUsingDirective:
+    Key.Data = 0;
+    break;
   }
 
-  static std::pair<unsigned, unsigned>
-  ReadKeyDataLength(const unsigned char*& d) {
-    using namespace clang::io;
-    unsigned KeyLen = ReadUnalignedLE16(d);
-    unsigned DataLen = ReadUnalignedLE16(d);
-    return std::make_pair(KeyLen, DataLen);
+  return Key;
+}
+
+ASTDeclContextNameLookupTrait::external_key_type 
+ASTDeclContextNameLookupTrait::GetExternalKey(
+                                          const internal_key_type& Key) const {
+  ASTContext &Context = Reader.getContext();
+  switch (Key.Kind) {
+  case DeclarationName::Identifier:
+    return DeclarationName((IdentifierInfo*)Key.Data);
+
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    return DeclarationName(Selector(Key.Data));
+
+  case DeclarationName::CXXConstructorName:
+    return Context.DeclarationNames.getCXXConstructorName(
+             Context.getCanonicalType(Reader.getLocalType(F, Key.Data)));
+
+  case DeclarationName::CXXDestructorName:
+    return Context.DeclarationNames.getCXXDestructorName(
+             Context.getCanonicalType(Reader.getLocalType(F, Key.Data)));
+
+  case DeclarationName::CXXConversionFunctionName:
+    return Context.DeclarationNames.getCXXConversionFunctionName(
+             Context.getCanonicalType(Reader.getLocalType(F, Key.Data)));
+
+  case DeclarationName::CXXOperatorName:
+    return Context.DeclarationNames.getCXXOperatorName(
+                                       (OverloadedOperatorKind)Key.Data);
+
+  case DeclarationName::CXXLiteralOperatorName:
+    return Context.DeclarationNames.getCXXLiteralOperatorName(
+                                                   (IdentifierInfo*)Key.Data);
+
+  case DeclarationName::CXXUsingDirective:
+    return DeclarationName::getUsingDirectiveName();
   }
 
-  internal_key_type ReadKey(const unsigned char* d, unsigned) {
-    using namespace clang::io;
+  llvm_unreachable("Invalid Name Kind ?");
+}
 
-    DeclNameKey Key;
-    Key.Kind = (DeclarationName::NameKind)*d++;
-    switch (Key.Kind) {
-    case DeclarationName::Identifier:
-      Key.Data = (uint64_t)Reader.DecodeIdentifierInfo(ReadUnalignedLE32(d));
-      break;
-    case DeclarationName::ObjCZeroArgSelector:
-    case DeclarationName::ObjCOneArgSelector:
-    case DeclarationName::ObjCMultiArgSelector:
-      Key.Data =
-         (uint64_t)Reader.DecodeSelector(ReadUnalignedLE32(d)).getAsOpaquePtr();
-      break;
-    case DeclarationName::CXXConstructorName:
-    case DeclarationName::CXXDestructorName:
-    case DeclarationName::CXXConversionFunctionName:
-      Key.Data = ReadUnalignedLE32(d); // TypeID
-      break;
-    case DeclarationName::CXXOperatorName:
-      Key.Data = *d++; // OverloadedOperatorKind
-      break;
-    case DeclarationName::CXXLiteralOperatorName:
-      Key.Data = (uint64_t)Reader.DecodeIdentifierInfo(ReadUnalignedLE32(d));
-      break;
-    case DeclarationName::CXXUsingDirective:
-      break;
-    }
+std::pair<unsigned, unsigned>
+ASTDeclContextNameLookupTrait::ReadKeyDataLength(const unsigned char*& d) {
+  using namespace clang::io;
+  unsigned KeyLen = ReadUnalignedLE16(d);
+  unsigned DataLen = ReadUnalignedLE16(d);
+  return std::make_pair(KeyLen, DataLen);
+}
 
-    return Key;
+ASTDeclContextNameLookupTrait::internal_key_type 
+ASTDeclContextNameLookupTrait::ReadKey(const unsigned char* d, unsigned) {
+  using namespace clang::io;
+
+  DeclNameKey Key;
+  Key.Kind = (DeclarationName::NameKind)*d++;
+  switch (Key.Kind) {
+  case DeclarationName::Identifier:
+    Key.Data = (uint64_t)Reader.getLocalIdentifier(F, ReadUnalignedLE32(d));
+    break;
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    Key.Data =
+       (uint64_t)Reader.getLocalSelector(F, ReadUnalignedLE32(d))
+                   .getAsOpaquePtr();
+    break;
+  case DeclarationName::CXXOperatorName:
+    Key.Data = *d++; // OverloadedOperatorKind
+    break;
+  case DeclarationName::CXXLiteralOperatorName:
+    Key.Data = (uint64_t)Reader.getLocalIdentifier(F, ReadUnalignedLE32(d));
+    break;
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+  case DeclarationName::CXXUsingDirective:
+    Key.Data = 0;
+    break;
   }
 
-  data_type ReadData(internal_key_type, const unsigned char* d,
-                     unsigned DataLen) {
-    using namespace clang::io;
-    unsigned NumDecls = ReadUnalignedLE16(d);
-    DeclID *Start = (DeclID *)d;
-    return std::make_pair(Start, Start + NumDecls);
-  }
-};
+  return Key;
+}
 
-} // end anonymous namespace
+ASTDeclContextNameLookupTrait::data_type 
+ASTDeclContextNameLookupTrait::ReadData(internal_key_type, 
+                                        const unsigned char* d,
+                                      unsigned DataLen) {
+  using namespace clang::io;
+  unsigned NumDecls = ReadUnalignedLE16(d);
+  DeclID *Start = (DeclID *)d;
+  return std::make_pair(Start, Start + NumDecls);
+}
 
-/// \brief The on-disk hash table used for the DeclContext's Name lookup table.
-typedef OnDiskChainedHashTable<ASTDeclContextNameLookupTrait>
-  ASTDeclContextNameLookupTable;
-
-bool ASTReader::ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
+bool ASTReader::ReadDeclContextStorage(Module &M,
+                                       llvm::BitstreamCursor &Cursor,
                                    const std::pair<uint64_t, uint64_t> &Offsets,
                                        DeclContextInfo &Info) {
   SavedStreamPosition SavedPosition(Cursor);
@@ -932,9 +749,6 @@ bool ASTReader::ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
 
     Info.LexicalDecls = reinterpret_cast<const KindDeclIDPair*>(Blob);
     Info.NumLexicalDecls = BlobLen / sizeof(KindDeclIDPair);
-  } else {
-    Info.LexicalDecls = 0;
-    Info.NumLexicalDecls = 0;
   }
 
   // Now the lookup table.
@@ -954,20 +768,18 @@ bool ASTReader::ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
       = ASTDeclContextNameLookupTable::Create(
                     (const unsigned char *)Blob + Record[0],
                     (const unsigned char *)Blob,
-                    ASTDeclContextNameLookupTrait(*this));
-  } else {
-    Info.NameLookupTableData = 0;
+                    ASTDeclContextNameLookupTrait(*this, M));
   }
 
   return false;
 }
 
-void ASTReader::Error(llvm::StringRef Msg) {
+void ASTReader::Error(StringRef Msg) {
   Error(diag::err_fe_pch_malformed, Msg);
 }
 
 void ASTReader::Error(unsigned DiagID,
-                      llvm::StringRef Arg1, llvm::StringRef Arg2) {
+                      StringRef Arg1, StringRef Arg2) {
   if (Diags.isDiagnosticInFlight())
     Diags.SetDelayedDiagnostic(DiagID, Arg1, Arg2);
   else
@@ -990,8 +802,8 @@ bool ASTReader::CheckPredefinesBuffers() {
 
 /// \brief Read the line table in the source manager block.
 /// \returns true if there was an error.
-bool ASTReader::ParseLineTable(PerFileData &F,
-                               llvm::SmallVectorImpl<uint64_t> &Record) {
+bool ASTReader::ParseLineTable(Module &F,
+                               SmallVectorImpl<uint64_t> &Record) {
   unsigned Idx = 0;
   LineTableInfo &LineTable = SourceMgr.getLineTable();
 
@@ -1010,6 +822,9 @@ bool ASTReader::ParseLineTable(PerFileData &F,
   std::vector<LineEntry> Entries;
   while (Idx < Record.size()) {
     int FID = Record[Idx++];
+    assert(FID >= 0 && "Serialized line entries for non-local file.");
+    // Remap FileID from 1-based old view.
+    FID += F.SLocEntryBaseID - 1;
 
     // Extract the line entries
     unsigned NumEntries = Record[Idx++];
@@ -1131,7 +946,7 @@ class ASTStatCache : public FileSystemStatCache {
 
 
 /// \brief Read a source manager block
-ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(PerFileData &F) {
+ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(Module &F) {
   using namespace SrcMgr;
 
   llvm::BitstreamCursor &SLocEntryCursor = F.SLocEntryCursor;
@@ -1188,11 +1003,6 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(PerFileData &F) {
     default:  // Default behavior: ignore.
       break;
 
-    case SM_LINE_TABLE:
-      if (ParseLineTable(F, Record))
-        return Failure;
-      break;
-
     case SM_SLOC_FILE_ENTRY:
     case SM_SLOC_BUFFER_ENTRY:
     case SM_SLOC_EXPANSION_ENTRY:
@@ -1235,38 +1045,20 @@ resolveFileRelativeToOriginalDir(const std::string &Filename,
   return currPCHPath.str();
 }
 
-/// \brief Get a cursor that's correctly positioned for reading the source
-/// location entry with the given ID.
-ASTReader::PerFileData *ASTReader::SLocCursorForID(unsigned ID) {
-  assert(ID != 0 && ID <= TotalNumSLocEntries &&
-         "SLocCursorForID should only be called for real IDs.");
-
-  ID -= 1;
-  PerFileData *F = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    F = Chain[N - I - 1];
-    if (ID < F->LocalNumSLocEntries)
-      break;
-    ID -= F->LocalNumSLocEntries;
-  }
-  assert(F && F->LocalNumSLocEntries > ID && "Chain corrupted");
-
-  F->SLocEntryCursor.JumpToBit(F->SLocOffsets[ID]);
-  return F;
-}
-
 /// \brief Read in the source location entry with the given ID.
-ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
+ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
   if (ID == 0)
     return Success;
 
-  if (ID > TotalNumSLocEntries) {
+  if (unsigned(-ID) - 2 >= getTotalNumSLocs() || ID > 0) {
     Error("source location entry ID out-of-range for AST file");
     return Failure;
   }
 
-  PerFileData *F = SLocCursorForID(ID);
+  Module *F = GlobalSLocEntryMap.find(-ID)->second;
+  F->SLocEntryCursor.JumpToBit(F->SLocEntryOffsets[ID - F->SLocEntryBaseID]);
   llvm::BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
+  unsigned BaseOffset = F->SLocEntryBaseOffset;
 
   ++NumSLocEntriesRead;
   unsigned Code = SLocEntryCursor.ReadCode();
@@ -1326,12 +1118,19 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
       return Failure;
     }
 
-    FileID FID = SourceMgr.createFileID(File, ReadSourceLocation(*F, Record[1]),
+    SourceLocation IncludeLoc = ReadSourceLocation(*F, Record[1]);
+    if (IncludeLoc.isInvalid() && F->Kind != MK_MainFile) {
+      // This is the module's main file.
+      IncludeLoc = getImportLocation(F);
+    }
+    FileID FID = SourceMgr.createFileID(File, IncludeLoc,
                                         (SrcMgr::CharacteristicKind)Record[2],
-                                        ID, Record[0]);
+                                        ID, BaseOffset + Record[0]);
+    SrcMgr::FileInfo &FileInfo =
+          const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile());
+    FileInfo.NumCreatedFIDs = Record[6];
     if (Record[3])
-      const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile())
-        .setHasLineDirectives();
+      FileInfo.setHasLineDirectives();
     
     break;
   }
@@ -1350,14 +1149,15 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
     }
 
     llvm::MemoryBuffer *Buffer
-    = llvm::MemoryBuffer::getMemBuffer(llvm::StringRef(BlobStart, BlobLen - 1),
+    = llvm::MemoryBuffer::getMemBuffer(StringRef(BlobStart, BlobLen - 1),
                                        Name);
-    FileID BufferID = SourceMgr.createFileIDForMemBuffer(Buffer, ID, Offset);
+    FileID BufferID = SourceMgr.createFileIDForMemBuffer(Buffer, ID,
+                                                         BaseOffset + Offset);
 
     if (strcmp(Name, "<built-in>") == 0) {
       PCHPredefinesBlock Block = {
         BufferID,
-        llvm::StringRef(BlobStart, BlobLen - 1)
+        StringRef(BlobStart, BlobLen - 1)
       };
       PCHPredefinesBuffers.push_back(Block);
     }
@@ -1367,12 +1167,12 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
 
   case SM_SLOC_EXPANSION_ENTRY: {
     SourceLocation SpellingLoc = ReadSourceLocation(*F, Record[1]);
-    SourceMgr.createInstantiationLoc(SpellingLoc,
+    SourceMgr.createExpansionLoc(SpellingLoc,
                                      ReadSourceLocation(*F, Record[2]),
                                      ReadSourceLocation(*F, Record[3]),
                                      Record[4],
                                      ID,
-                                     Record[0]);
+                                     BaseOffset + Record[0]);
     break;
   }
   }
@@ -1380,6 +1180,25 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
   return Success;
 }
 
+/// \brief Find the location where the module F is imported.
+SourceLocation ASTReader::getImportLocation(Module *F) {
+  if (F->ImportLoc.isValid())
+    return F->ImportLoc;
+  
+  // Otherwise we have a PCH. It's considered to be "imported" at the first
+  // location of its includer.
+  if (F->ImportedBy.empty() || !F->ImportedBy[0]) {
+    // Main file is the importer. We assume that it is the first entry in the
+    // entry table. We can't ask the manager, because at the time of PCH loading
+    // the main file entry doesn't exist yet.
+    // The very first entry is the invalid instantiation loc, which takes up
+    // offsets 0 and 1.
+    return SourceLocation::getFromRawEncoding(2U);
+  }
+  //return F->Loaders[0]->FirstLoc;
+  return F->ImportedBy[0]->FirstLoc;
+}
+
 /// ReadBlockAbbrevs - Enter a subblock of the specified BlockID with the
 /// specified cursor.  Read the abbreviations that are at the top of the block
 /// and then leave the cursor pointing into the block.
@@ -1403,8 +1222,7 @@ bool ASTReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
   }
 }
 
-PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
-  assert(PP && "Forgot to set Preprocessor ?");
+void ASTReader::ReadMacroRecord(Module &F, uint64_t Offset) {
   llvm::BitstreamCursor &Stream = F.MacroCursor;
 
   // Keep track of where we are in the stream, then jump back there
@@ -1413,21 +1231,21 @@ PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset)
 
   Stream.JumpToBit(Offset);
   RecordData Record;
-  llvm::SmallVector<IdentifierInfo*, 16> MacroArgs;
+  SmallVector<IdentifierInfo*, 16> MacroArgs;
   MacroInfo *Macro = 0;
 
   while (true) {
     unsigned Code = Stream.ReadCode();
     switch (Code) {
     case llvm::bitc::END_BLOCK:
-      return 0;
+      return;
 
     case llvm::bitc::ENTER_SUBBLOCK:
       // No known subblocks, always skip them.
       Stream.ReadSubBlockID();
       if (Stream.SkipBlock()) {
         Error("malformed block record in AST file");
-        return 0;
+        return;
       }
       continue;
 
@@ -1451,50 +1269,55 @@ PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset)
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return 0;
+        return;
 
-      IdentifierInfo *II = DecodeIdentifierInfo(Record[0]);
+      IdentifierInfo *II = getLocalIdentifier(F, Record[0]);
       if (II == 0) {
         Error("macro must have a name in AST file");
-        return 0;
+        return;
       }
       SourceLocation Loc = ReadSourceLocation(F, Record[1]);
       bool isUsed = Record[2];
 
-      MacroInfo *MI = PP->AllocateMacroInfo(Loc);
+      MacroInfo *MI = PP.AllocateMacroInfo(Loc);
       MI->setIsUsed(isUsed);
       MI->setIsFromAST();
 
       unsigned NextIndex = 3;
+      MI->setExportLocation(ReadSourceLocation(F, Record, NextIndex));
+      
       if (RecType == PP_MACRO_FUNCTION_LIKE) {
         // Decode function-like macro info.
-        bool isC99VarArgs = Record[3];
-        bool isGNUVarArgs = Record[4];
+        bool isC99VarArgs = Record[NextIndex++];
+        bool isGNUVarArgs = Record[NextIndex++];
         MacroArgs.clear();
-        unsigned NumArgs = Record[5];
-        NextIndex = 6 + NumArgs;
+        unsigned NumArgs = Record[NextIndex++];
         for (unsigned i = 0; i != NumArgs; ++i)
-          MacroArgs.push_back(DecodeIdentifierInfo(Record[6+i]));
+          MacroArgs.push_back(getLocalIdentifier(F, Record[NextIndex++]));
 
         // Install function-like macro info.
         MI->setIsFunctionLike();
         if (isC99VarArgs) MI->setIsC99Varargs();
         if (isGNUVarArgs) MI->setIsGNUVarargs();
         MI->setArgumentList(MacroArgs.data(), MacroArgs.size(),
-                            PP->getPreprocessorAllocator());
+                            PP.getPreprocessorAllocator());
       }
 
       // Finally, install the macro.
-      PP->setMacroInfo(II, MI);
+      PP.setMacroInfo(II, MI);
 
       // Remember that we saw this macro last so that we add the tokens that
       // form its body to it.
       Macro = MI;
 
-      if (NextIndex + 1 == Record.size() && PP->getPreprocessingRecord()) {
-        // We have a macro definition. Load it now.
-        PP->getPreprocessingRecord()->RegisterMacroDefinition(Macro,
-                                        getMacroDefinition(Record[NextIndex]));
+      if (NextIndex + 1 == Record.size() && PP.getPreprocessingRecord() &&
+          Record[NextIndex]) {
+        // We have a macro definition. Register the association
+        PreprocessedEntityID
+            GlobalID = getGlobalPreprocessedEntityID(F, Record[NextIndex]);
+        PreprocessingRecord &PPRec = *PP.getPreprocessingRecord();
+        PPRec.RegisterMacroDefinition(Macro,
+                            PPRec.getPPEntityID(GlobalID-1, /*isLoaded=*/true));
       }
 
       ++NumMacrosRead;
@@ -1510,7 +1333,7 @@ PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset)
       Tok.startToken();
       Tok.setLocation(ReadSourceLocation(F, Record[0]));
       Tok.setLength(Record[1]);
-      if (IdentifierInfo *II = DecodeIdentifierInfo(Record[2]))
+      if (IdentifierInfo *II = getLocalIdentifier(F, Record[2]))
         Tok.setIdentifierInfo(II);
       Tok.setKind((tok::TokenKind)Record[3]);
       Tok.setFlag((Token::TokenFlags)Record[4]);
@@ -1520,234 +1343,98 @@ PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset)
   }
   }
   
-  return 0;
+  return;
 }
 
-PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
-  assert(PP && "Forgot to set Preprocessor ?");
-  unsigned Code = F.PreprocessorDetailCursor.ReadCode();
-  switch (Code) {
-  case llvm::bitc::END_BLOCK:
-    return 0;
-    
-  case llvm::bitc::ENTER_SUBBLOCK:
-    Error("unexpected subblock record in preprocessor detail block");
-    return 0;
-      
-  case llvm::bitc::DEFINE_ABBREV:
-    Error("unexpected abbrevation record in preprocessor detail block");
-    return 0;
-      
-  default:
-    break;
-  }
-
-  if (!PP->getPreprocessingRecord()) {
-    Error("no preprocessing record");
-    return 0;
-  }
+PreprocessedEntityID 
+ASTReader::getGlobalPreprocessedEntityID(Module &M, unsigned LocalID) const {
+  ContinuousRangeMap<uint32_t, int, 2>::const_iterator 
+    I = M.PreprocessedEntityRemap.find(LocalID - NUM_PREDEF_PP_ENTITY_IDS);
+  assert(I != M.PreprocessedEntityRemap.end() 
+         && "Invalid index into preprocessed entity index remap");
   
-  // Read the record.
-  PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
-  const char *BlobStart = 0;
-  unsigned BlobLen = 0;
-  RecordData Record;
-  PreprocessorDetailRecordTypes RecType =
-    (PreprocessorDetailRecordTypes)F.PreprocessorDetailCursor.ReadRecord(
-                                             Code, Record, BlobStart, BlobLen);
-  switch (RecType) {
-  case PPD_MACRO_EXPANSION: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
-      return PE;
-    
-    MacroExpansion *ME =
-      new (PPRec) MacroExpansion(DecodeIdentifierInfo(Record[3]),
-                                 SourceRange(ReadSourceLocation(F, Record[1]),
-                                             ReadSourceLocation(F, Record[2])),
-                                 getMacroDefinition(Record[4]));
-    PPRec.SetPreallocatedEntity(Record[0], ME);
-    return ME;
-  }
-      
-  case PPD_MACRO_DEFINITION: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
-      return PE;
-    
-    if (Record[1] > MacroDefinitionsLoaded.size()) {
-      Error("out-of-bounds macro definition record");
-      return 0;
-    }
-    
-    // Decode the identifier info and then check again; if the macro is
-    // still defined and associated with the identifier,
-    IdentifierInfo *II = DecodeIdentifierInfo(Record[4]);
-    if (!MacroDefinitionsLoaded[Record[1] - 1]) {
-      MacroDefinition *MD
-        = new (PPRec) MacroDefinition(II,
-                                      ReadSourceLocation(F, Record[5]),
-                                      SourceRange(
-                                            ReadSourceLocation(F, Record[2]),
-                                            ReadSourceLocation(F, Record[3])));
-      
-      PPRec.SetPreallocatedEntity(Record[0], MD);
-      MacroDefinitionsLoaded[Record[1] - 1] = MD;
-      
-      if (DeserializationListener)
-        DeserializationListener->MacroDefinitionRead(Record[1], MD);
-    }
-    
-    return MacroDefinitionsLoaded[Record[1] - 1];
-  }
-      
-  case PPD_INCLUSION_DIRECTIVE: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
-      return PE;
-    
-    const char *FullFileNameStart = BlobStart + Record[3];
-    const FileEntry *File
-      = PP->getFileManager().getFile(llvm::StringRef(FullFileNameStart,
-                                                     BlobLen - Record[3]));
-    
-    // FIXME: Stable encoding
-    InclusionDirective::InclusionKind Kind
-      = static_cast<InclusionDirective::InclusionKind>(Record[5]);
-    InclusionDirective *ID
-      = new (PPRec) InclusionDirective(PPRec, Kind,
-                                       llvm::StringRef(BlobStart, Record[3]),
-                                       Record[4],
-                                       File,
-                                 SourceRange(ReadSourceLocation(F, Record[1]),
-                                             ReadSourceLocation(F, Record[2])));
-    PPRec.SetPreallocatedEntity(Record[0], ID);
-    return ID;
-  }
-  }
-  
-  Error("invalid offset in preprocessor detail block");
-  return 0;
+  return LocalID + I->second;
 }
 
-namespace {
-  /// \brief Trait class used to search the on-disk hash table containing all of
-  /// the header search information.
-  ///
-  /// The on-disk hash table contains a mapping from each header path to 
-  /// information about that header (how many times it has been included, its
-  /// controlling macro, etc.). Note that we actually hash based on the 
-  /// filename, and support "deep" comparisons of file names based on current
-  /// inode numbers, so that the search can cope with non-normalized path names
-  /// and symlinks.
-  class HeaderFileInfoTrait {
-    const char *SearchPath;
-    struct stat SearchPathStatBuf;
-    llvm::Optional<int> SearchPathStatResult;
+unsigned HeaderFileInfoTrait::ComputeHash(const char *path) {
+  return llvm::HashString(llvm::sys::path::filename(path));
+}
     
-    int StatSimpleCache(const char *Path, struct stat *StatBuf) {
-      if (Path == SearchPath) {
-        if (!SearchPathStatResult)
-          SearchPathStatResult = stat(Path, &SearchPathStatBuf);
-        
-        *StatBuf = SearchPathStatBuf;
-        return *SearchPathStatResult;
-      }
-      
-      return stat(Path, StatBuf);
-    }
+HeaderFileInfoTrait::internal_key_type 
+HeaderFileInfoTrait::GetInternalKey(const char *path) { return path; }
     
-  public:
-    typedef const char *external_key_type;
-    typedef const char *internal_key_type;
+bool HeaderFileInfoTrait::EqualKey(internal_key_type a, internal_key_type b) {
+  if (strcmp(a, b) == 0)
+    return true;
+  
+  if (llvm::sys::path::filename(a) != llvm::sys::path::filename(b))
+    return false;
+  
+  // The file names match, but the path names don't. stat() the files to
+  // see if they are the same.      
+  struct stat StatBufA, StatBufB;
+  if (StatSimpleCache(a, &StatBufA) || StatSimpleCache(b, &StatBufB))
+    return false;
+  
+  return StatBufA.st_ino == StatBufB.st_ino;
+}
     
-    typedef HeaderFileInfo data_type;
+std::pair<unsigned, unsigned>
+HeaderFileInfoTrait::ReadKeyDataLength(const unsigned char*& d) {
+  unsigned KeyLen = (unsigned) clang::io::ReadUnalignedLE16(d);
+  unsigned DataLen = (unsigned) *d++;
+  return std::make_pair(KeyLen + 1, DataLen);
+}
     
-    HeaderFileInfoTrait(const char *SearchPath = 0) : SearchPath(SearchPath) { }
-    
-    static unsigned ComputeHash(const char *path) {
-      return llvm::HashString(llvm::sys::path::filename(path));
-    }
-    
-    static internal_key_type GetInternalKey(const char *path) { return path; }
-    
-    bool EqualKey(internal_key_type a, internal_key_type b) {
-      if (strcmp(a, b) == 0)
-        return true;
-      
-      if (llvm::sys::path::filename(a) != llvm::sys::path::filename(b))
-        return false;
-      
-      // The file names match, but the path names don't. stat() the files to
-      // see if they are the same.      
-      struct stat StatBufA, StatBufB;
-      if (StatSimpleCache(a, &StatBufA) || StatSimpleCache(b, &StatBufB))
-        return false;
-      
-      return StatBufA.st_ino == StatBufB.st_ino;
-    }
-    
-    static std::pair<unsigned, unsigned>
-    ReadKeyDataLength(const unsigned char*& d) {
-      unsigned KeyLen = (unsigned) clang::io::ReadUnalignedLE16(d);
-      unsigned DataLen = (unsigned) *d++;
-      return std::make_pair(KeyLen + 1, DataLen);
-    }
-    
-    static internal_key_type ReadKey(const unsigned char *d, unsigned) {
-      return (const char *)d;
-    }
-    
-    static data_type ReadData(const internal_key_type, const unsigned char *d,
+HeaderFileInfoTrait::data_type 
+HeaderFileInfoTrait::ReadData(const internal_key_type, const unsigned char *d,
                               unsigned DataLen) {
-      const unsigned char *End = d + DataLen;
-      using namespace clang::io;
-      HeaderFileInfo HFI;
-      unsigned Flags = *d++;
-      HFI.isImport = (Flags >> 4) & 0x01;
-      HFI.isPragmaOnce = (Flags >> 3) & 0x01;
-      HFI.DirInfo = (Flags >> 1) & 0x03;
-      HFI.Resolved = Flags & 0x01;
-      HFI.NumIncludes = ReadUnalignedLE16(d);
-      HFI.ControllingMacroID = ReadUnalignedLE32(d);
-      assert(End == d && "Wrong data length in HeaderFileInfo deserialization");
-      (void)End;
-      
-      // This HeaderFileInfo was externally loaded.
-      HFI.External = true;
-      return HFI;
-    }
-  };
+  const unsigned char *End = d + DataLen;
+  using namespace clang::io;
+  HeaderFileInfo HFI;
+  unsigned Flags = *d++;
+  HFI.isImport = (Flags >> 5) & 0x01;
+  HFI.isPragmaOnce = (Flags >> 4) & 0x01;
+  HFI.DirInfo = (Flags >> 2) & 0x03;
+  HFI.Resolved = (Flags >> 1) & 0x01;
+  HFI.IndexHeaderMapHeader = Flags & 0x01;
+  HFI.NumIncludes = ReadUnalignedLE16(d);
+  HFI.ControllingMacroID = Reader.getGlobalDeclID(M, ReadUnalignedLE32(d));
+  if (unsigned FrameworkOffset = ReadUnalignedLE32(d)) {
+    // The framework offset is 1 greater than the actual offset, 
+    // since 0 is used as an indicator for "no framework name".
+    StringRef FrameworkName(FrameworkStrings + FrameworkOffset - 1);
+    HFI.Framework = HS->getUniqueFrameworkName(FrameworkName);
+  }
+  
+  assert(End == d && "Wrong data length in HeaderFileInfo deserialization");
+  (void)End;
+        
+  // This HeaderFileInfo was externally loaded.
+  HFI.External = true;
+  return HFI;
 }
 
-/// \brief The on-disk hash table used for the global method pool.
-typedef OnDiskChainedHashTable<HeaderFileInfoTrait>
-  HeaderFileInfoLookupTable;
-
-void ASTReader::SetIdentifierIsMacro(IdentifierInfo *II, PerFileData &F,
-                                     uint64_t Offset) {
+void ASTReader::SetIdentifierIsMacro(IdentifierInfo *II, Module &F,
+                                     uint64_t LocalOffset) {
   // Note that this identifier has a macro definition.
   II->setHasMacroDefinition(true);
   
-  // Adjust the offset based on our position in the chain.
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    if (Chain[I] == &F)
-      break;
-    
-    Offset += Chain[I]->SizeInBits;
-  }
-    
-  UnreadMacroRecordOffsets[II] = Offset;
+  // Adjust the offset to a global offset.
+  UnreadMacroRecordOffsets[II] = F.GlobalBitOffset + LocalOffset;
 }
 
 void ASTReader::ReadDefinedMacros() {
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    PerFileData &F = *Chain[N - I - 1];
-    llvm::BitstreamCursor &MacroCursor = F.MacroCursor;
+  for (ModuleReverseIterator I = ModuleMgr.rbegin(),
+      E = ModuleMgr.rend(); I != E; ++I) {
+    llvm::BitstreamCursor &MacroCursor = (*I)->MacroCursor;
 
     // If there was no preprocessor block, skip this file.
     if (!MacroCursor.getBitStreamReader())
       continue;
 
     llvm::BitstreamCursor Cursor = MacroCursor;
-    Cursor.JumpToBit(F.MacroStartOffset);
+    Cursor.JumpToBit((*I)->MacroStartOffset);
 
     RecordData Record;
     while (true) {
@@ -1780,7 +1467,7 @@ void ASTReader::ReadDefinedMacros() {
 
       case PP_MACRO_OBJECT_LIKE:
       case PP_MACRO_FUNCTION_LIKE:
-        DecodeIdentifierInfo(Record[0]);
+        getLocalIdentifier(**I, Record[0]);
         break;
 
       case PP_TOKEN:
@@ -1798,24 +1485,11 @@ void ASTReader::ReadDefinedMacros() {
 void ASTReader::LoadMacroDefinition(
                      llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos) {
   assert(Pos != UnreadMacroRecordOffsets.end() && "Unknown macro definition");
-  PerFileData *F = 0;
   uint64_t Offset = Pos->second;
   UnreadMacroRecordOffsets.erase(Pos);
   
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    if (Offset < Chain[I]->SizeInBits) {
-      F = Chain[I];
-      break;
-    }
-    
-    Offset -= Chain[I]->SizeInBits;
-  }    
-  if (!F) {
-    Error("Malformed macro record offset");
-    return;
-  }
-  
-  ReadMacroRecord(*F, Offset);
+  RecordLocation Loc = getLocalBitOffset(Offset);
+  ReadMacroRecord(*Loc.F, Loc.Offset);
 }
 
 void ASTReader::LoadMacroDefinition(IdentifierInfo *II) {
@@ -1824,29 +1498,7 @@ void ASTReader::LoadMacroDefinition(IdentifierInfo *II) {
   LoadMacroDefinition(Pos);
 }
 
-MacroDefinition *ASTReader::getMacroDefinition(MacroID ID) {
-  if (ID == 0 || ID > MacroDefinitionsLoaded.size())
-    return 0;
-
-  if (!MacroDefinitionsLoaded[ID - 1]) {
-    unsigned Index = ID - 1;
-    for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-      PerFileData &F = *Chain[N - I - 1];
-      if (Index < F.LocalNumMacroDefinitions) {
-        SavedStreamPosition SavedPosition(F.PreprocessorDetailCursor);  
-        F.PreprocessorDetailCursor.JumpToBit(F.MacroDefinitionOffsets[Index]);
-        LoadPreprocessedEntity(F);
-        break;
-      }
-      Index -= F.LocalNumMacroDefinitions;
-    }
-    assert(MacroDefinitionsLoaded[ID - 1] && "Broken chain");
-  }
-
-  return MacroDefinitionsLoaded[ID - 1];
-}
-
-const FileEntry *ASTReader::getFileEntry(llvm::StringRef filenameStrRef) {
+const FileEntry *ASTReader::getFileEntry(StringRef filenameStrRef) {
   std::string Filename = filenameStrRef;
   MaybeAddSystemRootToFilename(Filename);
   const FileEntry *File = FileMgr.getFile(Filename);
@@ -1873,21 +1525,21 @@ void ASTReader::MaybeAddSystemRootToFilename(std::string &Filename) {
   if (Filename.empty() || llvm::sys::path::is_absolute(Filename))
     return;
 
-  if (isysroot == 0) {
+  if (isysroot.empty()) {
     // If no system root was given, default to '/'
     Filename.insert(Filename.begin(), '/');
     return;
   }
 
-  unsigned Length = strlen(isysroot);
+  unsigned Length = isysroot.size();
   if (isysroot[Length - 1] != '/')
     Filename.insert(Filename.begin(), '/');
 
-  Filename.insert(Filename.begin(), isysroot, isysroot + Length);
+  Filename.insert(Filename.begin(), isysroot.begin(), isysroot.end());
 }
 
 ASTReader::ASTReadResult
-ASTReader::ReadASTBlock(PerFileData &F) {
+ASTReader::ReadASTBlock(Module &F) {
   llvm::BitstreamCursor &Stream = F.Stream;
 
   if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
@@ -1897,7 +1549,6 @@ ASTReader::ReadASTBlock(PerFileData &F) {
 
   // Read all of the records and blocks for the ASt file.
   RecordData Record;
-  bool First = true;
   while (!Stream.AtEndOfStream()) {
     unsigned Code = Stream.ReadCode();
     if (Code == llvm::bitc::END_BLOCK) {
@@ -1934,8 +1585,8 @@ ASTReader::ReadASTBlock(PerFileData &F) {
 
       case PREPROCESSOR_BLOCK_ID:
         F.MacroCursor = Stream;
-        if (PP)
-          PP->setExternalSource(this);
+        if (!PP.getExternalSource())
+          PP.setExternalSource(this);
 
         if (Stream.SkipBlock() ||
             ReadBlockAbbrevs(F.MacroCursor, PREPROCESSOR_BLOCK_ID)) {
@@ -1955,6 +1606,11 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         }
         F.PreprocessorDetailStartOffset
           = F.PreprocessorDetailCursor.GetCurrentBitNo();
+          
+        if (!PP.getPreprocessingRecord())
+          PP.createPreprocessingRecord(true);
+        if (!PP.getPreprocessingRecord()->getExternalSource())
+          PP.getPreprocessingRecord()->SetExternalSource(*this);
         break;
         
       case SOURCE_MANAGER_BLOCK_ID:
@@ -1971,7 +1627,6 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         }
         break;
       }
-      First = false;
       continue;
     }
 
@@ -2005,79 +1660,108 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       break;
     }
 
-    case CHAINED_METADATA: {
-      if (!First) {
-        Error("CHAINED_METADATA is not first record in block");
-        return Failure;
-      }
-      if (Record[0] != VERSION_MAJOR && !DisableValidation) {
-        Diag(Record[0] < VERSION_MAJOR? diag::warn_pch_version_too_old
-                                           : diag::warn_pch_version_too_new);
-        return IgnorePCH;
-      }
+    case IMPORTS: {
+      // Load each of the imported PCH files. 
+      unsigned Idx = 0, N = Record.size();
+      while (Idx < N) {
+        // Read information about the AST file.
+        ModuleKind ImportedKind = (ModuleKind)Record[Idx++];
+        unsigned Length = Record[Idx++];
+        llvm::SmallString<128> ImportedFile(Record.begin() + Idx,
+                                            Record.begin() + Idx + Length);
+        Idx += Length;
 
-      // Load the chained file, which is always a PCH file.
-      switch(ReadASTCore(llvm::StringRef(BlobStart, BlobLen), PCH)) {
-      case Failure: return Failure;
-        // If we have to ignore the dependency, we'll have to ignore this too.
-      case IgnorePCH: return IgnorePCH;
-      case Success: break;
+        // Load the AST file.
+        switch(ReadASTCore(ImportedFile, ImportedKind, &F)) {
+        case Failure: return Failure;
+          // If we have to ignore the dependency, we'll have to ignore this too.
+        case IgnorePCH: return IgnorePCH;
+        case Success: break;
+        }
       }
       break;
     }
 
-    case TYPE_OFFSET:
+    case TYPE_OFFSET: {
       if (F.LocalNumTypes != 0) {
         Error("duplicate TYPE_OFFSET record in AST file");
         return Failure;
       }
       F.TypeOffsets = (const uint32_t *)BlobStart;
       F.LocalNumTypes = Record[0];
+      unsigned LocalBaseTypeIndex = Record[1];
+      F.BaseTypeIndex = getTotalNumTypes();
+        
+      if (F.LocalNumTypes > 0) {
+        // Introduce the global -> local mapping for types within this module.
+        GlobalTypeMap.insert(std::make_pair(getTotalNumTypes(), &F));
+        
+        // Introduce the local -> global mapping for types within this module.
+        F.TypeRemap.insert(std::make_pair(LocalBaseTypeIndex, 
+                             F.BaseTypeIndex - LocalBaseTypeIndex));
+        
+        TypesLoaded.resize(TypesLoaded.size() + F.LocalNumTypes);
+      }
       break;
-
-    case DECL_OFFSET:
+    }
+        
+    case DECL_OFFSET: {
       if (F.LocalNumDecls != 0) {
         Error("duplicate DECL_OFFSET record in AST file");
         return Failure;
       }
       F.DeclOffsets = (const uint32_t *)BlobStart;
       F.LocalNumDecls = Record[0];
+      unsigned LocalBaseDeclID = Record[1];
+      F.BaseDeclID = getTotalNumDecls();
+        
+      if (F.LocalNumDecls > 0) {
+        // Introduce the global -> local mapping for declarations within this 
+        // module.
+        GlobalDeclMap.insert(
+          std::make_pair(getTotalNumDecls() + NUM_PREDEF_DECL_IDS, &F));
+        
+        // Introduce the local -> global mapping for declarations within this
+        // module.
+        F.DeclRemap.insert(std::make_pair(LocalBaseDeclID, 
+                                          F.BaseDeclID - LocalBaseDeclID));
+        
+        DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls);
+      }
       break;
-
+    }
+        
     case TU_UPDATE_LEXICAL: {
-      DeclContextInfo Info = {
-        /* No visible information */ 0,
-        reinterpret_cast<const KindDeclIDPair *>(BlobStart),
-        BlobLen / sizeof(KindDeclIDPair)
-      };
-      DeclContextOffsets[Context ? Context->getTranslationUnitDecl() : 0]
-        .push_back(Info);
+      DeclContext *TU = Context.getTranslationUnitDecl();
+      DeclContextInfo &Info = F.DeclContextInfos[TU];
+      Info.LexicalDecls = reinterpret_cast<const KindDeclIDPair *>(BlobStart);
+      Info.NumLexicalDecls 
+        = static_cast<unsigned int>(BlobLen / sizeof(KindDeclIDPair));
+      TU->setHasExternalLexicalStorage(true);
       break;
     }
 
     case UPDATE_VISIBLE: {
-      serialization::DeclID ID = Record[0];
+      unsigned Idx = 0;
+      serialization::DeclID ID = ReadDeclID(F, Record, Idx);
       void *Table = ASTDeclContextNameLookupTable::Create(
-                        (const unsigned char *)BlobStart + Record[1],
+                        (const unsigned char *)BlobStart + Record[Idx++],
                         (const unsigned char *)BlobStart,
-                        ASTDeclContextNameLookupTrait(*this));
-      if (ID == 1 && Context) { // Is it the TU?
-        DeclContextInfo Info = {
-          Table, /* No lexical inforamtion */ 0, 0
-        };
-        DeclContextOffsets[Context->getTranslationUnitDecl()].push_back(Info);
+                        ASTDeclContextNameLookupTrait(*this, F));
+      if (ID == PREDEF_DECL_TRANSLATION_UNIT_ID) { // Is it the TU?
+        DeclContext *TU = Context.getTranslationUnitDecl();
+        F.DeclContextInfos[TU].NameLookupTableData = Table;
+        TU->setHasExternalVisibleStorage(true);
       } else
-        PendingVisibleUpdates[ID].push_back(Table);
+        PendingVisibleUpdates[ID].push_back(std::make_pair(Table, &F));
       break;
     }
 
     case REDECLS_UPDATE_LATEST: {
       assert(Record.size() % 2 == 0 && "Expected pairs of DeclIDs");
-      for (unsigned i = 0, e = Record.size(); i < e; i += 2) {
-        DeclID First = Record[i], Latest = Record[i+1];
-        assert((FirstLatestDeclIDs.find(First) == FirstLatestDeclIDs.end() ||
-                Latest > FirstLatestDeclIDs[First]) &&
-               "The new latest is supposed to come after the previous latest");
+      for (unsigned i = 0, e = Record.size(); i < e; /* in loop */) {
+        DeclID First = ReadDeclID(F, Record, i);
+        DeclID Latest = ReadDeclID(F, Record, i);
         FirstLatestDeclIDs[First] = Latest;
       }
       break;
@@ -2096,35 +1780,47 @@ ASTReader::ReadASTBlock(PerFileData &F) {
                        (const unsigned char *)F.IdentifierTableData + Record[0],
                        (const unsigned char *)F.IdentifierTableData,
                        ASTIdentifierLookupTrait(*this, F));
-        if (PP)
-          PP->getIdentifierTable().setExternalIdentifierLookup(this);
+        
+        PP.getIdentifierTable().setExternalIdentifierLookup(this);
       }
       break;
 
-    case IDENTIFIER_OFFSET:
+    case IDENTIFIER_OFFSET: {
       if (F.LocalNumIdentifiers != 0) {
         Error("duplicate IDENTIFIER_OFFSET record in AST file");
         return Failure;
       }
       F.IdentifierOffsets = (const uint32_t *)BlobStart;
       F.LocalNumIdentifiers = Record[0];
+      unsigned LocalBaseIdentifierID = Record[1];
+      F.BaseIdentifierID = getTotalNumIdentifiers();
+        
+      if (F.LocalNumIdentifiers > 0) {
+        // Introduce the global -> local mapping for identifiers within this
+        // module.
+        GlobalIdentifierMap.insert(std::make_pair(getTotalNumIdentifiers() + 1, 
+                                                  &F));
+        
+        // Introduce the local -> global mapping for identifiers within this
+        // module.
+        F.IdentifierRemap.insert(
+                            std::make_pair(LocalBaseIdentifierID,
+                              F.BaseIdentifierID - LocalBaseIdentifierID));
+        
+        IdentifiersLoaded.resize(IdentifiersLoaded.size() 
+                                 + F.LocalNumIdentifiers);
+      }
       break;
-
+    }
+        
     case EXTERNAL_DEFINITIONS:
-      // Optimization for the first block.
-      if (ExternalDefinitions.empty())
-        ExternalDefinitions.swap(Record);
-      else
-        ExternalDefinitions.insert(ExternalDefinitions.end(),
-                                   Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        ExternalDefinitions.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
     case SPECIAL_TYPES:
-      // Optimization for the first block
-      if (SpecialTypes.empty())
-        SpecialTypes.swap(Record);
-      else
-        SpecialTypes.insert(SpecialTypes.end(), Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        SpecialTypes.push_back(getGlobalTypeID(F, Record[I]));
       break;
 
     case STATISTICS:
@@ -2135,42 +1831,63 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       break;
 
     case UNUSED_FILESCOPED_DECLS:
-      // Optimization for the first block.
-      if (UnusedFileScopedDecls.empty())
-        UnusedFileScopedDecls.swap(Record);
-      else
-        UnusedFileScopedDecls.insert(UnusedFileScopedDecls.end(),
-                                     Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        UnusedFileScopedDecls.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
     case DELEGATING_CTORS:
-      // Optimization for the first block.
-      if (DelegatingCtorDecls.empty())
-        DelegatingCtorDecls.swap(Record);
-      else
-        DelegatingCtorDecls.insert(DelegatingCtorDecls.end(),
-                                   Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        DelegatingCtorDecls.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
     case WEAK_UNDECLARED_IDENTIFIERS:
-      // Later blocks overwrite earlier ones.
-      WeakUndeclaredIdentifiers.swap(Record);
+      if (Record.size() % 4 != 0) {
+        Error("invalid weak identifiers record");
+        return Failure;
+      }
+        
+      // FIXME: Ignore weak undeclared identifiers from non-original PCH 
+      // files. This isn't the way to do it :)
+      WeakUndeclaredIdentifiers.clear();
+        
+      // Translate the weak, undeclared identifiers into global IDs.
+      for (unsigned I = 0, N = Record.size(); I < N; /* in loop */) {
+        WeakUndeclaredIdentifiers.push_back(
+          getGlobalIdentifierID(F, Record[I++]));
+        WeakUndeclaredIdentifiers.push_back(
+          getGlobalIdentifierID(F, Record[I++]));
+        WeakUndeclaredIdentifiers.push_back(
+          ReadSourceLocation(F, Record, I).getRawEncoding());
+        WeakUndeclaredIdentifiers.push_back(Record[I++]);
+      }
       break;
 
     case LOCALLY_SCOPED_EXTERNAL_DECLS:
-      // Optimization for the first block.
-      if (LocallyScopedExternalDecls.empty())
-        LocallyScopedExternalDecls.swap(Record);
-      else
-        LocallyScopedExternalDecls.insert(LocallyScopedExternalDecls.end(),
-                                          Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        LocallyScopedExternalDecls.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
-    case SELECTOR_OFFSETS:
+    case SELECTOR_OFFSETS: {
       F.SelectorOffsets = (const uint32_t *)BlobStart;
       F.LocalNumSelectors = Record[0];
-      break;
+      unsigned LocalBaseSelectorID = Record[1];
+      F.BaseSelectorID = getTotalNumSelectors();
+        
+      if (F.LocalNumSelectors > 0) {
+        // Introduce the global -> local mapping for selectors within this 
+        // module.
+        GlobalSelectorMap.insert(std::make_pair(getTotalNumSelectors()+1, &F));
+        
+        // Introduce the local -> global mapping for selectors within this 
+        // module.
+        F.SelectorRemap.insert(std::make_pair(LocalBaseSelectorID,
+                                 F.BaseSelectorID - LocalBaseSelectorID));
 
+        SelectorsLoaded.resize(SelectorsLoaded.size() + F.LocalNumSelectors);        
+      }
+      break;
+    }
+        
     case METHOD_POOL:
       F.SelectorLookupTableData = (const unsigned char *)BlobStart;
       if (Record[0])
@@ -2178,12 +1895,19 @@ ASTReader::ReadASTBlock(PerFileData &F) {
           = ASTSelectorLookupTable::Create(
                         F.SelectorLookupTableData + Record[0],
                         F.SelectorLookupTableData,
-                        ASTSelectorLookupTrait(*this));
+                        ASTSelectorLookupTrait(*this, F));
       TotalNumMethodPoolEntries += Record[1];
       break;
 
     case REFERENCED_SELECTOR_POOL:
-      F.ReferencedSelectorsData.swap(Record);
+      if (!Record.empty()) {
+        for (unsigned Idx = 0, N = Record.size() - 1; Idx < N; /* in loop */) {
+          ReferencedSelectorsData.push_back(getGlobalSelectorID(F, 
+                                                                Record[Idx++]));
+          ReferencedSelectorsData.push_back(ReadSourceLocation(F, Record, Idx).
+                                              getRawEncoding());
+        }
+      }
       break;
 
     case PP_COUNTER_VALUE:
@@ -2191,10 +1915,94 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         Listener->ReadCounter(Record[0]);
       break;
 
-    case SOURCE_LOCATION_OFFSETS:
-      F.SLocOffsets = (const uint32_t *)BlobStart;
+    case SOURCE_LOCATION_OFFSETS: {
+      F.SLocEntryOffsets = (const uint32_t *)BlobStart;
       F.LocalNumSLocEntries = Record[0];
-      F.LocalSLocSize = Record[1];
+      unsigned SLocSpaceSize = Record[1];
+      llvm::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
+          SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries,
+                                              SLocSpaceSize);
+      // Make our entry in the range map. BaseID is negative and growing, so
+      // we invert it. Because we invert it, though, we need the other end of
+      // the range.
+      unsigned RangeStart =
+          unsigned(-F.SLocEntryBaseID) - F.LocalNumSLocEntries + 1;
+      GlobalSLocEntryMap.insert(std::make_pair(RangeStart, &F));
+      F.FirstLoc = SourceLocation::getFromRawEncoding(F.SLocEntryBaseOffset);
+
+      // SLocEntryBaseOffset is lower than MaxLoadedOffset and decreasing.
+      assert((F.SLocEntryBaseOffset & (1U << 31U)) == 0);
+      GlobalSLocOffsetMap.insert(
+          std::make_pair(SourceManager::MaxLoadedOffset - F.SLocEntryBaseOffset
+                           - SLocSpaceSize,&F));
+
+      // Initialize the remapping table.
+      // Invalid stays invalid.
+      F.SLocRemap.insert(std::make_pair(0U, 0));
+      // This module. Base was 2 when being compiled.
+      F.SLocRemap.insert(std::make_pair(2U,
+                                  static_cast<int>(F.SLocEntryBaseOffset - 2)));
+      
+      TotalNumSLocEntries += F.LocalNumSLocEntries;
+      break;
+    }
+
+    case MODULE_OFFSET_MAP: {
+      // Additional remapping information.
+      const unsigned char *Data = (const unsigned char*)BlobStart;
+      const unsigned char *DataEnd = Data + BlobLen;
+      
+      // Continuous range maps we may be updating in our module.
+      ContinuousRangeMap<uint32_t, int, 2>::Builder SLocRemap(F.SLocRemap);
+      ContinuousRangeMap<uint32_t, int, 2>::Builder 
+        IdentifierRemap(F.IdentifierRemap);
+      ContinuousRangeMap<uint32_t, int, 2>::Builder 
+        PreprocessedEntityRemap(F.PreprocessedEntityRemap);
+      ContinuousRangeMap<uint32_t, int, 2>::Builder 
+        SelectorRemap(F.SelectorRemap);
+      ContinuousRangeMap<uint32_t, int, 2>::Builder DeclRemap(F.DeclRemap);
+      ContinuousRangeMap<uint32_t, int, 2>::Builder TypeRemap(F.TypeRemap);
+
+      while(Data < DataEnd) {
+        uint16_t Len = io::ReadUnalignedLE16(Data);
+        StringRef Name = StringRef((const char*)Data, Len);
+        Data += Len;
+        Module *OM = ModuleMgr.lookup(Name);
+        if (!OM) {
+          Error("SourceLocation remap refers to unknown module");
+          return Failure;
+        }
+
+        uint32_t SLocOffset = io::ReadUnalignedLE32(Data);
+        uint32_t IdentifierIDOffset = io::ReadUnalignedLE32(Data);
+        uint32_t PreprocessedEntityIDOffset = io::ReadUnalignedLE32(Data);
+        uint32_t SelectorIDOffset = io::ReadUnalignedLE32(Data);
+        uint32_t DeclIDOffset = io::ReadUnalignedLE32(Data);
+        uint32_t TypeIndexOffset = io::ReadUnalignedLE32(Data);
+        
+        // Source location offset is mapped to OM->SLocEntryBaseOffset.
+        SLocRemap.insert(std::make_pair(SLocOffset,
+          static_cast<int>(OM->SLocEntryBaseOffset - SLocOffset)));
+        IdentifierRemap.insert(
+          std::make_pair(IdentifierIDOffset, 
+                         OM->BaseIdentifierID - IdentifierIDOffset));
+        PreprocessedEntityRemap.insert(
+          std::make_pair(PreprocessedEntityIDOffset, 
+            OM->BasePreprocessedEntityID - PreprocessedEntityIDOffset));
+        SelectorRemap.insert(std::make_pair(SelectorIDOffset, 
+                               OM->BaseSelectorID - SelectorIDOffset));
+        DeclRemap.insert(std::make_pair(DeclIDOffset, 
+                                        OM->BaseDeclID - DeclIDOffset));
+        
+        TypeRemap.insert(std::make_pair(TypeIndexOffset, 
+                                    OM->BaseTypeIndex - TypeIndexOffset));
+      }
+      break;
+    }
+
+    case SOURCE_MANAGER_LINE_TABLE:
+      if (ParseLineTable(F, Record))
+        return Failure;
       break;
 
     case FILE_SOURCE_LOCATION_OFFSETS:
@@ -2202,13 +2010,17 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       F.LocalNumSLocFileEntries = Record[0];
       break;
 
-    case SOURCE_LOCATION_PRELOADS:
-      if (PreloadSLocEntries.empty())
-        PreloadSLocEntries.swap(Record);
-      else
-        PreloadSLocEntries.insert(PreloadSLocEntries.end(),
-            Record.begin(), Record.end());
+    case SOURCE_LOCATION_PRELOADS: {
+      // Need to transform from the local view (1-based IDs) to the global view,
+      // which is based off F.SLocEntryBaseID.
+      if (!F.PreloadSLocEntries.empty()) {
+        Error("Multiple SOURCE_LOCATION_PRELOADS records in AST file");
+        return Failure;
+      }
+      
+      F.PreloadSLocEntries.swap(Record);
       break;
+    }
 
     case STAT_CACHE: {
       if (!DisableStatCache) {
@@ -2223,35 +2035,56 @@ ASTReader::ReadASTBlock(PerFileData &F) {
     }
 
     case EXT_VECTOR_DECLS:
-      // Optimization for the first block.
-      if (ExtVectorDecls.empty())
-        ExtVectorDecls.swap(Record);
-      else
-        ExtVectorDecls.insert(ExtVectorDecls.end(),
-                              Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        ExtVectorDecls.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
     case VTABLE_USES:
+      if (Record.size() % 3 != 0) {
+        Error("Invalid VTABLE_USES record");
+        return Failure;
+      }
+        
       // Later tables overwrite earlier ones.
-      VTableUses.swap(Record);
+      // FIXME: Modules will have some trouble with this. This is clearly not
+      // the right way to do this.
+      VTableUses.clear();
+        
+      for (unsigned Idx = 0, N = Record.size(); Idx != N; /* In loop */) {
+        VTableUses.push_back(getGlobalDeclID(F, Record[Idx++]));
+        VTableUses.push_back(
+          ReadSourceLocation(F, Record, Idx).getRawEncoding());
+        VTableUses.push_back(Record[Idx++]);
+      }
       break;
 
     case DYNAMIC_CLASSES:
-      // Optimization for the first block.
-      if (DynamicClasses.empty())
-        DynamicClasses.swap(Record);
-      else
-        DynamicClasses.insert(DynamicClasses.end(),
-                              Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        DynamicClasses.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
     case PENDING_IMPLICIT_INSTANTIATIONS:
-      F.PendingInstantiations.swap(Record);
+      if (PendingInstantiations.size() % 2 != 0) {
+        Error("Invalid PENDING_IMPLICIT_INSTANTIATIONS block");
+        return Failure;
+      }
+        
+      // Later lists of pending instantiations overwrite earlier ones.
+      // FIXME: This is most certainly wrong for modules.
+      PendingInstantiations.clear();
+      for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
+        PendingInstantiations.push_back(getGlobalDeclID(F, Record[I++]));
+        PendingInstantiations.push_back(
+          ReadSourceLocation(F, Record, I).getRawEncoding());
+      }
       break;
 
     case SEMA_DECL_REFS:
       // Later tables overwrite earlier ones.
-      SemaDeclRefs.swap(Record);
+      // FIXME: Modules will have some trouble with this.
+      SemaDeclRefs.clear();
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        SemaDeclRefs.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
     case ORIGINAL_FILE_NAME:
@@ -2274,28 +2107,54 @@ ASTReader::ReadASTBlock(PerFileData &F) {
 
     case VERSION_CONTROL_BRANCH_REVISION: {
       const std::string &CurBranch = getClangFullRepositoryVersion();
-      llvm::StringRef ASTBranch(BlobStart, BlobLen);
-      if (llvm::StringRef(CurBranch) != ASTBranch && !DisableValidation) {
+      StringRef ASTBranch(BlobStart, BlobLen);
+      if (StringRef(CurBranch) != ASTBranch && !DisableValidation) {
         Diag(diag::warn_pch_different_branch) << ASTBranch << CurBranch;
         return IgnorePCH;
       }
       break;
     }
 
-    case MACRO_DEFINITION_OFFSETS:
-      F.MacroDefinitionOffsets = (const uint32_t *)BlobStart;
-      F.NumPreallocatedPreprocessingEntities = Record[0];
-      F.LocalNumMacroDefinitions = Record[1];
-      break;
+    case PPD_ENTITIES_OFFSETS: {
+      F.PreprocessedEntityOffsets = (const PPEntityOffset *)BlobStart;
+      assert(BlobLen % sizeof(PPEntityOffset) == 0);
+      F.NumPreprocessedEntities = BlobLen / sizeof(PPEntityOffset);
 
+      unsigned LocalBasePreprocessedEntityID = Record[0];
+      
+      unsigned StartingID;
+      if (!PP.getPreprocessingRecord())
+        PP.createPreprocessingRecord(true);
+      if (!PP.getPreprocessingRecord()->getExternalSource())
+        PP.getPreprocessingRecord()->SetExternalSource(*this);
+      StartingID 
+        = PP.getPreprocessingRecord()
+            ->allocateLoadedEntities(F.NumPreprocessedEntities);
+      F.BasePreprocessedEntityID = StartingID;
+
+      if (F.NumPreprocessedEntities > 0) {
+        // Introduce the global -> local mapping for preprocessed entities in
+        // this module.
+        GlobalPreprocessedEntityMap.insert(std::make_pair(StartingID, &F));
+       
+        // Introduce the local -> global mapping for preprocessed entities in
+        // this module.
+        F.PreprocessedEntityRemap.insert(
+          std::make_pair(LocalBasePreprocessedEntityID,
+            F.BasePreprocessedEntityID - LocalBasePreprocessedEntityID));
+      }
+
+      break;
+    }
+        
     case DECL_UPDATE_OFFSETS: {
       if (Record.size() % 2 != 0) {
         Error("invalid DECL_UPDATE_OFFSETS block in AST file");
         return Failure;
       }
       for (unsigned I = 0, N = Record.size(); I != N; I += 2)
-        DeclUpdateOffsets[static_cast<DeclID>(Record[I])]
-            .push_back(std::make_pair(&F, Record[I+1]));
+        DeclUpdateOffsets[getGlobalDeclID(F, Record[I])]
+          .push_back(std::make_pair(&F, Record[I+1]));
       break;
     }
 
@@ -2305,8 +2164,22 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         return Failure;
       }
       for (unsigned I = 0, N = Record.size(); I != N; I += 2)
-        ReplacedDecls[static_cast<DeclID>(Record[I])] =
-            std::make_pair(&F, Record[I+1]);
+        ReplacedDecls[getGlobalDeclID(F, Record[I])]
+          = std::make_pair(&F, Record[I+1]);
+      break;
+    }
+
+    case OBJC_CHAINED_CATEGORIES: {
+      if (Record.size() % 3 != 0) {
+        Error("invalid OBJC_CHAINED_CATEGORIES block in AST file");
+        return Failure;
+      }
+      for (unsigned I = 0, N = Record.size(); I != N; I += 3) {
+        serialization::GlobalDeclID GlobID = getGlobalDeclID(F, Record[I]);
+        F.ChainedObjCCategories[GlobID] = std::make_pair(Record[I+1],
+                                                         Record[I+2]);
+        ObjCChainedCategoriesInterfaces.insert(GlobID);
+      }
       break;
     }
         
@@ -2318,6 +2191,7 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       
       F.LocalNumCXXBaseSpecifiers = Record[0];
       F.CXXBaseSpecifiersOffsets = (const uint32_t *)BlobStart;
+      NumCXXBaseSpecifiersLoaded += F.LocalNumCXXBaseSpecifiers;
       break;
     }
 
@@ -2326,31 +2200,42 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         Error("invalid DIAG_USER_MAPPINGS block in AST file");
         return Failure;
       }
-      if (PragmaDiagMappings.empty())
-        PragmaDiagMappings.swap(Record);
+        
+      if (F.PragmaDiagMappings.empty())
+        F.PragmaDiagMappings.swap(Record);
       else
-        PragmaDiagMappings.insert(PragmaDiagMappings.end(),
-                                Record.begin(), Record.end());
+        F.PragmaDiagMappings.insert(F.PragmaDiagMappings.end(),
+                                    Record.begin(), Record.end());
       break;
         
     case CUDA_SPECIAL_DECL_REFS:
       // Later tables overwrite earlier ones.
-      CUDASpecialDeclRefs.swap(Record);
+      // FIXME: Modules will have trouble with this.
+      CUDASpecialDeclRefs.clear();
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        CUDASpecialDeclRefs.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
-    case HEADER_SEARCH_TABLE:
+    case HEADER_SEARCH_TABLE: {
       F.HeaderFileInfoTableData = BlobStart;
       F.LocalNumHeaderFileInfos = Record[1];
+      F.HeaderFileFrameworkStrings = BlobStart + Record[2];
       if (Record[0]) {
         F.HeaderFileInfoTable
           = HeaderFileInfoLookupTable::Create(
                    (const unsigned char *)F.HeaderFileInfoTableData + Record[0],
-                   (const unsigned char *)F.HeaderFileInfoTableData);
-        if (PP)
-          PP->getHeaderSearchInfo().SetExternalSource(this);
+                   (const unsigned char *)F.HeaderFileInfoTableData,
+                   HeaderFileInfoTrait(*this, F, 
+                                       &PP.getHeaderSearchInfo(),
+                                       BlobStart + Record[2]));
+        
+        PP.getHeaderSearchInfo().SetExternalSource(this);
+        if (!PP.getHeaderSearchInfo().getExternalLookup())
+          PP.getHeaderSearchInfo().SetExternalLookup(this);
       }
       break;
-
+    }
+        
     case FP_PRAGMA_OPTIONS:
       // Later tables overwrite earlier ones.
       FPPragmaOptions.swap(Record);
@@ -2362,272 +2247,222 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       break;
 
     case TENTATIVE_DEFINITIONS:
-      // Optimization for the first block.
-      if (TentativeDefinitions.empty())
-        TentativeDefinitions.swap(Record);
-      else
-        TentativeDefinitions.insert(TentativeDefinitions.end(),
-                                    Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        TentativeDefinitions.push_back(getGlobalDeclID(F, Record[I]));
       break;
         
     case KNOWN_NAMESPACES:
-      // Optimization for the first block.
-      if (KnownNamespaces.empty())
-        KnownNamespaces.swap(Record);
-      else
-        KnownNamespaces.insert(KnownNamespaces.end(), 
-                               Record.begin(), Record.end());
+      for (unsigned I = 0, N = Record.size(); I != N; ++I)
+        KnownNamespaces.push_back(getGlobalDeclID(F, Record[I]));
       break;
     }
-    First = false;
   }
   Error("premature end of bitstream in AST file");
   return Failure;
 }
 
-ASTReader::ASTReadResult ASTReader::validateFileEntries() {
-  for (unsigned CI = 0, CN = Chain.size(); CI != CN; ++CI) {
-    PerFileData *F = Chain[CI];
-    llvm::BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
+ASTReader::ASTReadResult ASTReader::validateFileEntries(Module &M) {
+  llvm::BitstreamCursor &SLocEntryCursor = M.SLocEntryCursor;
 
-    for (unsigned i = 0, e = F->LocalNumSLocFileEntries; i != e; ++i) {
-      SLocEntryCursor.JumpToBit(F->SLocFileOffsets[i]);
-      unsigned Code = SLocEntryCursor.ReadCode();
-      if (Code == llvm::bitc::END_BLOCK ||
-          Code == llvm::bitc::ENTER_SUBBLOCK ||
-          Code == llvm::bitc::DEFINE_ABBREV) {
-        Error("incorrectly-formatted source location entry in AST file");
+  for (unsigned i = 0, e = M.LocalNumSLocFileEntries; i != e; ++i) {
+    SLocEntryCursor.JumpToBit(M.SLocFileOffsets[i]);
+    unsigned Code = SLocEntryCursor.ReadCode();
+    if (Code == llvm::bitc::END_BLOCK ||
+        Code == llvm::bitc::ENTER_SUBBLOCK ||
+        Code == llvm::bitc::DEFINE_ABBREV) {
+      Error("incorrectly-formatted source location entry in AST file");
+      return Failure;
+    }
+
+    RecordData Record;
+    const char *BlobStart;
+    unsigned BlobLen;
+    switch (SLocEntryCursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
+    default:
+      Error("incorrectly-formatted source location entry in AST file");
+      return Failure;
+
+    case SM_SLOC_FILE_ENTRY: {
+      StringRef Filename(BlobStart, BlobLen);
+      const FileEntry *File = getFileEntry(Filename);
+
+      if (File == 0) {
+        std::string ErrorStr = "could not find file '";
+        ErrorStr += Filename;
+        ErrorStr += "' referenced by AST file";
+        Error(ErrorStr.c_str());
+        return IgnorePCH;
+      }
+
+      if (Record.size() < 6) {
+        Error("source location entry is incorrect");
         return Failure;
       }
-  
-      RecordData Record;
-      const char *BlobStart;
-      unsigned BlobLen;
-      switch (SLocEntryCursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
-      default:
-        Error("incorrectly-formatted source location entry in AST file");
-        return Failure;
-  
-      case SM_SLOC_FILE_ENTRY: {
-        llvm::StringRef Filename(BlobStart, BlobLen);
-        const FileEntry *File = getFileEntry(Filename);
 
-        if (File == 0) {
-          std::string ErrorStr = "could not find file '";
-          ErrorStr += Filename;
-          ErrorStr += "' referenced by AST file";
-          Error(ErrorStr.c_str());
-          return IgnorePCH;
-        }
-  
-        if (Record.size() < 6) {
-          Error("source location entry is incorrect");
-          return Failure;
-        }
+      // The stat info from the FileEntry came from the cached stat
+      // info of the PCH, so we cannot trust it.
+      struct stat StatBuf;
+      if (::stat(File->getName(), &StatBuf) != 0) {
+        StatBuf.st_size = File->getSize();
+        StatBuf.st_mtime = File->getModificationTime();
+      }
 
-        // The stat info from the FileEntry came from the cached stat
-        // info of the PCH, so we cannot trust it.
-        struct stat StatBuf;
-        if (::stat(File->getName(), &StatBuf) != 0) {
-          StatBuf.st_size = File->getSize();
-          StatBuf.st_mtime = File->getModificationTime();
-        }
-
-        if (((off_t)Record[4] != StatBuf.st_size
+      if (((off_t)Record[4] != StatBuf.st_size
 #if !defined(LLVM_ON_WIN32)
-            // In our regression testing, the Windows file system seems to
-            // have inconsistent modification times that sometimes
-            // erroneously trigger this error-handling path.
-             || (time_t)Record[5] != StatBuf.st_mtime
+          // In our regression testing, the Windows file system seems to
+          // have inconsistent modification times that sometimes
+          // erroneously trigger this error-handling path.
+           || (time_t)Record[5] != StatBuf.st_mtime
 #endif
-            )) {
-          Error(diag::err_fe_pch_file_modified, Filename);
-          return IgnorePCH;
-        }
+          )) {
+        Error(diag::err_fe_pch_file_modified, Filename);
+        return IgnorePCH;
+      }
 
-        break;
-      }
-      }
+      break;
+    }
     }
   }
 
   return Success;
 }
 
+namespace {
+  /// \brief Visitor class used to look up identifirs in an AST file.
+  class IdentifierLookupVisitor {
+    StringRef Name;
+    IdentifierInfo *Found;
+  public:
+    explicit IdentifierLookupVisitor(StringRef Name) : Name(Name), Found() { }
+    
+    static bool visit(Module &M, void *UserData) {
+      IdentifierLookupVisitor *This
+      = static_cast<IdentifierLookupVisitor *>(UserData);
+      
+      ASTIdentifierLookupTable *IdTable
+        = (ASTIdentifierLookupTable *)M.IdentifierLookupTable;
+      if (!IdTable)
+        return false;
+      
+      std::pair<const char*, unsigned> Key(This->Name.begin(), 
+                                           This->Name.size());
+      ASTIdentifierLookupTable::iterator Pos = IdTable->find(Key);
+      if (Pos == IdTable->end())
+        return false;
+      
+      // Dereferencing the iterator has the effect of building the
+      // IdentifierInfo node and populating it with the various
+      // declarations it needs.
+      This->Found = *Pos;
+      return true;
+    }
+    
+    // \brief Retrieve the identifier info found within the module
+    // files.
+    IdentifierInfo *getIdentifierInfo() const { return Found; }
+  };
+}
+
+
 ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
-                                            ASTFileType Type) {
-  switch(ReadASTCore(FileName, Type)) {
+                                            ModuleKind Type) {
+  switch(ReadASTCore(FileName, Type, /*ImportedBy=*/0)) {
   case Failure: return Failure;
   case IgnorePCH: return IgnorePCH;
   case Success: break;
   }
 
   // Here comes stuff that we only do once the entire chain is loaded.
-
-  if (!DisableValidation) {
-    switch(validateFileEntries()) {
-    case Failure: return Failure;
-    case IgnorePCH: return IgnorePCH;
-    case Success: break;
-    }
-  }
-
-  // Allocate space for loaded slocentries, identifiers, decls and types.
-  unsigned TotalNumIdentifiers = 0, TotalNumTypes = 0, TotalNumDecls = 0,
-           TotalNumPreallocatedPreprocessingEntities = 0, TotalNumMacroDefs = 0,
-           TotalNumSelectors = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    TotalNumSLocEntries += Chain[I]->LocalNumSLocEntries;
-    NextSLocOffset += Chain[I]->LocalSLocSize;
-    TotalNumIdentifiers += Chain[I]->LocalNumIdentifiers;
-    TotalNumTypes += Chain[I]->LocalNumTypes;
-    TotalNumDecls += Chain[I]->LocalNumDecls;
-    TotalNumPreallocatedPreprocessingEntities +=
-        Chain[I]->NumPreallocatedPreprocessingEntities;
-    TotalNumMacroDefs += Chain[I]->LocalNumMacroDefinitions;
-    TotalNumSelectors += Chain[I]->LocalNumSelectors;
-  }
-  SourceMgr.PreallocateSLocEntries(this, TotalNumSLocEntries, NextSLocOffset);
-  IdentifiersLoaded.resize(TotalNumIdentifiers);
-  TypesLoaded.resize(TotalNumTypes);
-  DeclsLoaded.resize(TotalNumDecls);
-  MacroDefinitionsLoaded.resize(TotalNumMacroDefs);
-  if (PP) {
-    if (TotalNumIdentifiers > 0)
-      PP->getHeaderSearchInfo().SetExternalLookup(this);
-    if (TotalNumPreallocatedPreprocessingEntities > 0) {
-      if (!PP->getPreprocessingRecord())
-        PP->createPreprocessingRecord(true);
-      PP->getPreprocessingRecord()->SetExternalSource(*this,
-                                     TotalNumPreallocatedPreprocessingEntities);
-    }
-  }
-  SelectorsLoaded.resize(TotalNumSelectors);
-  // Preload SLocEntries.
-  for (unsigned I = 0, N = PreloadSLocEntries.size(); I != N; ++I) {
-    ASTReadResult Result = ReadSLocEntryRecord(PreloadSLocEntries[I]);
-    if (Result != Success)
-      return Result;
-  }
-
+  
   // Check the predefines buffers.
-  if (!DisableValidation && CheckPredefinesBuffers())
+  if (!DisableValidation && Type != MK_Module && Type != MK_Preamble &&
+      // FIXME: CheckPredefinesBuffers also sets the SuggestedPredefines;
+      // if DisableValidation is true, defines that were set on command-line
+      // but not in the PCH file will not be added to SuggestedPredefines.
+      CheckPredefinesBuffers())
     return IgnorePCH;
 
-  if (PP) {
-    // Initialization of keywords and pragmas occurs before the
-    // AST file is read, so there may be some identifiers that were
-    // loaded into the IdentifierTable before we intercepted the
-    // creation of identifiers. Iterate through the list of known
-    // identifiers and determine whether we have to establish
-    // preprocessor definitions or top-level identifier declaration
-    // chains for those identifiers.
-    //
-    // We copy the IdentifierInfo pointers to a small vector first,
-    // since de-serializing declarations or macro definitions can add
-    // new entries into the identifier table, invalidating the
-    // iterators.
-    llvm::SmallVector<IdentifierInfo *, 128> Identifiers;
-    for (IdentifierTable::iterator Id = PP->getIdentifierTable().begin(),
-                                IdEnd = PP->getIdentifierTable().end();
-         Id != IdEnd; ++Id)
-      Identifiers.push_back(Id->second);
-    // We need to search the tables in all files.
-    for (unsigned J = 0, M = Chain.size(); J != M; ++J) {
-      ASTIdentifierLookupTable *IdTable
-        = (ASTIdentifierLookupTable *)Chain[J]->IdentifierLookupTable;
-      // Not all AST files necessarily have identifier tables, only the useful
-      // ones.
-      if (!IdTable)
-        continue;
-      for (unsigned I = 0, N = Identifiers.size(); I != N; ++I) {
-        IdentifierInfo *II = Identifiers[I];
-        // Look in the on-disk hash tables for an entry for this identifier
-        ASTIdentifierLookupTrait Info(*this, *Chain[J], II);
-        std::pair<const char*,unsigned> Key(II->getNameStart(),II->getLength());
-        ASTIdentifierLookupTable::iterator Pos = IdTable->find(Key, &Info);
-        if (Pos == IdTable->end())
-          continue;
-
-        // Dereferencing the iterator has the effect of populating the
-        // IdentifierInfo node with the various declarations it needs.
-        (void)*Pos;
-      }
-    }
+  // Initialization of keywords and pragmas occurs before the
+  // AST file is read, so there may be some identifiers that were
+  // loaded into the IdentifierTable before we intercepted the
+  // creation of identifiers. Iterate through the list of known
+  // identifiers and determine whether we have to establish
+  // preprocessor definitions or top-level identifier declaration
+  // chains for those identifiers.
+  //
+  // We copy the IdentifierInfo pointers to a small vector first,
+  // since de-serializing declarations or macro definitions can add
+  // new entries into the identifier table, invalidating the
+  // iterators.
+  //
+  // FIXME: We need a lazier way to load this information, e.g., by marking
+  // the identifier data as 'dirty', so that it will be looked up in the
+  // AST file(s) if it is uttered in the source. This could save us some
+  // module load time.
+  SmallVector<IdentifierInfo *, 128> Identifiers;
+  for (IdentifierTable::iterator Id = PP.getIdentifierTable().begin(),
+                              IdEnd = PP.getIdentifierTable().end();
+       Id != IdEnd; ++Id)
+    Identifiers.push_back(Id->second);
+  
+  for (unsigned I = 0, N = Identifiers.size(); I != N; ++I) {
+    IdentifierLookupVisitor Visitor(Identifiers[I]->getName());
+    ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor);
   }
 
-  if (Context)
-    InitializeContext(*Context);
+  InitializeContext();
 
   if (DeserializationListener)
     DeserializationListener->ReaderInitialized(this);
 
-  // If this AST file is a precompiled preamble, then set the main file ID of 
-  // the source manager to the file source file from which the preamble was
-  // built. This is the only valid way to use a precompiled preamble.
-  if (Type == Preamble) {
-    if (OriginalFileID.isInvalid()) {
-      SourceLocation Loc
-        = SourceMgr.getLocation(FileMgr.getFile(getOriginalSourceFile()), 1, 1);
-      if (Loc.isValid())
-        OriginalFileID = SourceMgr.getDecomposedLoc(Loc).first;
+  // If this AST file is a precompiled preamble, then set the preamble file ID
+  // of the source manager to the file source file from which the preamble was
+  // built.
+  if (Type == MK_Preamble) {
+    if (!OriginalFileID.isInvalid()) {
+      OriginalFileID = FileID::get(ModuleMgr.getPrimaryModule().SLocEntryBaseID
+                                        + OriginalFileID.getOpaqueValue() - 1);
+      SourceMgr.setPreambleFileID(OriginalFileID);
     }
-    
-    if (!OriginalFileID.isInvalid())
-      SourceMgr.SetPreambleFileID(OriginalFileID);
   }
   
   return Success;
 }
 
-ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
-                                                ASTFileType Type) {
-  PerFileData *Prev = Chain.empty() ? 0 : Chain.back();
-  Chain.push_back(new PerFileData(Type));
-  PerFileData &F = *Chain.back();
-  if (Prev)
-    Prev->NextInSource = &F;
-  else
-    FirstInSource = &F;
-  F.Loaders.push_back(Prev);
+ASTReader::ASTReadResult ASTReader::ReadASTCore(StringRef FileName,
+                                                ModuleKind Type,
+                                                Module *ImportedBy) {
+  Module *M;
+  bool NewModule;
+  std::string ErrorStr;
+  llvm::tie(M, NewModule) = ModuleMgr.addModule(FileName, Type, ImportedBy,
+                                                ErrorStr);
 
-  // Set the AST file name.
-  F.FileName = FileName;
+  if (!M) {
+    // We couldn't load the module.
+    std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
+      + ErrorStr;
+    Error(Msg);
+    return Failure;
+  }
 
+  if (!NewModule) {
+    // We've already loaded this module.
+    return Success;
+  }
+
+  // FIXME: This seems rather a hack. Should CurrentDir be part of the
+  // module?
   if (FileName != "-") {
     CurrentDir = llvm::sys::path::parent_path(FileName);
     if (CurrentDir.empty()) CurrentDir = ".";
   }
 
-  if (!ASTBuffers.empty()) {
-    F.Buffer.reset(ASTBuffers.back());
-    ASTBuffers.pop_back();
-    assert(F.Buffer && "Passed null buffer");
-  } else {
-    // Open the AST file.
-    //
-    // FIXME: This shouldn't be here, we should just take a raw_ostream.
-    std::string ErrStr;
-    llvm::error_code ec;
-    if (FileName == "-") {
-      ec = llvm::MemoryBuffer::getSTDIN(F.Buffer);
-      if (ec)
-        ErrStr = ec.message();
-    } else
-      F.Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrStr));
-    if (!F.Buffer) {
-      Error(ErrStr.c_str());
-      return IgnorePCH;
-    }
-  }
-
-  // Initialize the stream
-  F.StreamFile.init((const unsigned char *)F.Buffer->getBufferStart(),
-                    (const unsigned char *)F.Buffer->getBufferEnd());
+  Module &F = *M;
   llvm::BitstreamCursor &Stream = F.Stream;
   Stream.init(F.StreamFile);
   F.SizeInBits = F.Buffer->getBufferSize() * 8;
-
+  
   // Sniff for the signature.
   if (Stream.Read(8) != 'C' ||
       Stream.Read(8) != 'P' ||
@@ -2668,9 +2503,8 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
         // AST block, skipping subblocks, to see if there are other
         // AST blocks elsewhere.
 
-        // Clear out any preallocated source location entries, so that
-        // the source manager does not try to resolve them later.
-        SourceMgr.ClearPreallocatedSLocEntries();
+        // FIXME: We can't clear loaded slocentries anymore.
+        //SourceMgr.ClearPreallocatedSLocEntries();
 
         // Remove the stat cache.
         if (F.StatCache)
@@ -2687,143 +2521,152 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
       break;
     }
   }
+  
+  // Once read, set the Module bit base offset and update the size in 
+  // bits of all files we've seen.
+  F.GlobalBitOffset = TotalModulesSizeInBits;
+  TotalModulesSizeInBits += F.SizeInBits;
+  GlobalBitOffsetsMap.insert(std::make_pair(F.GlobalBitOffset, &F));
+
+  // Make sure that the files this module was built against are still available.
+  if (!DisableValidation) {
+    switch(validateFileEntries(*M)) {
+    case Failure: return Failure;
+    case IgnorePCH: return IgnorePCH;
+    case Success: break;
+    }
+  }
+  
+  // Preload SLocEntries.
+  for (unsigned I = 0, N = M->PreloadSLocEntries.size(); I != N; ++I) {
+    int Index = int(M->PreloadSLocEntries[I] - 1) + F.SLocEntryBaseID;
+    // Load it through the SourceManager and don't call ReadSLocEntryRecord()
+    // directly because the entry may have already been loaded in which case
+    // calling ReadSLocEntryRecord() directly would trigger an assertion in
+    // SourceManager.
+    SourceMgr.getLoadedSLocEntryByID(Index);
+  }
+
 
   return Success;
 }
 
-void ASTReader::setPreprocessor(Preprocessor &pp) {
-  PP = &pp;
+void ASTReader::InitializeContext() {  
+  // If there's a listener, notify them that we "read" the translation unit.
+  if (DeserializationListener)
+    DeserializationListener->DeclRead(PREDEF_DECL_TRANSLATION_UNIT_ID, 
+                                      Context.getTranslationUnitDecl());
 
-  unsigned TotalNum = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I)
-    TotalNum += Chain[I]->NumPreallocatedPreprocessingEntities;
-  if (TotalNum) {
-    if (!PP->getPreprocessingRecord())
-      PP->createPreprocessingRecord(true);
-    PP->getPreprocessingRecord()->SetExternalSource(*this, TotalNum);
-  }
-}
-
-void ASTReader::InitializeContext(ASTContext &Ctx) {
-  Context = &Ctx;
-  assert(Context && "Passed null context!");
-
-  assert(PP && "Forgot to set Preprocessor ?");
-  PP->getIdentifierTable().setExternalIdentifierLookup(this);
-  PP->getHeaderSearchInfo().SetExternalLookup(this);
-  PP->setExternalSource(this);
-  PP->getHeaderSearchInfo().SetExternalSource(this);
+  // Make sure we load the declaration update records for the translation unit,
+  // if there are any.
+  loadDeclUpdateRecords(PREDEF_DECL_TRANSLATION_UNIT_ID, 
+                        Context.getTranslationUnitDecl());
+  
+  // FIXME: Find a better way to deal with collisions between these
+  // built-in types. Right now, we just ignore the problem.
   
-  // If we have an update block for the TU waiting, we have to add it before
-  // deserializing the decl.
-  DeclContextOffsetsMap::iterator DCU = DeclContextOffsets.find(0);
-  if (DCU != DeclContextOffsets.end()) {
-    // Insertion could invalidate map, so grab vector.
-    DeclContextInfos T;
-    T.swap(DCU->second);
-    DeclContextOffsets.erase(DCU);
-    DeclContextOffsets[Ctx.getTranslationUnitDecl()].swap(T);
-  }
-
-  // Load the translation unit declaration
-  GetTranslationUnitDecl();
-
   // Load the special types.
-  Context->setBuiltinVaListType(
-    GetType(SpecialTypes[SPECIAL_TYPE_BUILTIN_VA_LIST]));
-  if (unsigned Id = SpecialTypes[SPECIAL_TYPE_OBJC_ID])
-    Context->setObjCIdType(GetType(Id));
-  if (unsigned Sel = SpecialTypes[SPECIAL_TYPE_OBJC_SELECTOR])
-    Context->setObjCSelType(GetType(Sel));
-  if (unsigned Proto = SpecialTypes[SPECIAL_TYPE_OBJC_PROTOCOL])
-    Context->setObjCProtoType(GetType(Proto));
-  if (unsigned Class = SpecialTypes[SPECIAL_TYPE_OBJC_CLASS])
-    Context->setObjCClassType(GetType(Class));
-
-  if (unsigned String = SpecialTypes[SPECIAL_TYPE_CF_CONSTANT_STRING])
-    Context->setCFConstantStringType(GetType(String));
-  if (unsigned FastEnum
-        = SpecialTypes[SPECIAL_TYPE_OBJC_FAST_ENUMERATION_STATE])
-    Context->setObjCFastEnumerationStateType(GetType(FastEnum));
-  if (unsigned File = SpecialTypes[SPECIAL_TYPE_FILE]) {
-    QualType FileType = GetType(File);
-    if (FileType.isNull()) {
-      Error("FILE type is NULL");
-      return;
+  if (SpecialTypes.size() > NumSpecialTypeIDs) {
+    if (Context.getBuiltinVaListType().isNull()) {
+      Context.setBuiltinVaListType(
+        GetType(SpecialTypes[SPECIAL_TYPE_BUILTIN_VA_LIST]));
     }
-    if (const TypedefType *Typedef = FileType->getAs<TypedefType>())
-      Context->setFILEDecl(Typedef->getDecl());
-    else {
-      const TagType *Tag = FileType->getAs<TagType>();
-      if (!Tag) {
-        Error("Invalid FILE type in AST file");
+    
+    if (unsigned Proto = SpecialTypes[SPECIAL_TYPE_OBJC_PROTOCOL]) {
+      if (Context.ObjCProtoType.isNull())
+        Context.ObjCProtoType = GetType(Proto);
+    }
+    
+    if (unsigned String = SpecialTypes[SPECIAL_TYPE_CF_CONSTANT_STRING]) {
+      if (!Context.CFConstantStringTypeDecl)
+        Context.setCFConstantStringType(GetType(String));
+    }
+    
+    if (unsigned File = SpecialTypes[SPECIAL_TYPE_FILE]) {
+      QualType FileType = GetType(File);
+      if (FileType.isNull()) {
+        Error("FILE type is NULL");
         return;
       }
-      Context->setFILEDecl(Tag->getDecl());
+      
+      if (!Context.FILEDecl) {
+        if (const TypedefType *Typedef = FileType->getAs<TypedefType>())
+          Context.setFILEDecl(Typedef->getDecl());
+        else {
+          const TagType *Tag = FileType->getAs<TagType>();
+          if (!Tag) {
+            Error("Invalid FILE type in AST file");
+            return;
+          }
+          Context.setFILEDecl(Tag->getDecl());
+        }
+      }
     }
-  }
-  if (unsigned Jmp_buf = SpecialTypes[SPECIAL_TYPE_jmp_buf]) {
-    QualType Jmp_bufType = GetType(Jmp_buf);
-    if (Jmp_bufType.isNull()) {
-      Error("jmp_bug type is NULL");
-      return;
-    }
-    if (const TypedefType *Typedef = Jmp_bufType->getAs<TypedefType>())
-      Context->setjmp_bufDecl(Typedef->getDecl());
-    else {
-      const TagType *Tag = Jmp_bufType->getAs<TagType>();
-      if (!Tag) {
-        Error("Invalid jmp_buf type in AST file");
+    
+    if (unsigned Jmp_buf = SpecialTypes[SPECIAL_TYPE_jmp_buf]) {
+      QualType Jmp_bufType = GetType(Jmp_buf);
+      if (Jmp_bufType.isNull()) {
+        Error("jmp_buf type is NULL");
         return;
       }
-      Context->setjmp_bufDecl(Tag->getDecl());
+      
+      if (!Context.jmp_bufDecl) {
+        if (const TypedefType *Typedef = Jmp_bufType->getAs<TypedefType>())
+          Context.setjmp_bufDecl(Typedef->getDecl());
+        else {
+          const TagType *Tag = Jmp_bufType->getAs<TagType>();
+          if (!Tag) {
+            Error("Invalid jmp_buf type in AST file");
+            return;
+          }
+          Context.setjmp_bufDecl(Tag->getDecl());
+        }
+      }
+    }
+    
+    if (unsigned Sigjmp_buf = SpecialTypes[SPECIAL_TYPE_sigjmp_buf]) {
+      QualType Sigjmp_bufType = GetType(Sigjmp_buf);
+      if (Sigjmp_bufType.isNull()) {
+        Error("sigjmp_buf type is NULL");
+        return;
+      }
+      
+      if (!Context.sigjmp_bufDecl) {
+        if (const TypedefType *Typedef = Sigjmp_bufType->getAs<TypedefType>())
+          Context.setsigjmp_bufDecl(Typedef->getDecl());
+        else {
+          const TagType *Tag = Sigjmp_bufType->getAs<TagType>();
+          assert(Tag && "Invalid sigjmp_buf type in AST file");
+          Context.setsigjmp_bufDecl(Tag->getDecl());
+        }
+      }
+    }
+
+    if (unsigned ObjCIdRedef
+          = SpecialTypes[SPECIAL_TYPE_OBJC_ID_REDEFINITION]) {
+      if (Context.ObjCIdRedefinitionType.isNull())
+        Context.ObjCIdRedefinitionType = GetType(ObjCIdRedef);
+    }
+
+    if (unsigned ObjCClassRedef
+          = SpecialTypes[SPECIAL_TYPE_OBJC_CLASS_REDEFINITION]) {
+      if (Context.ObjCClassRedefinitionType.isNull())
+        Context.ObjCClassRedefinitionType = GetType(ObjCClassRedef);
+    }
+
+    if (unsigned ObjCSelRedef
+          = SpecialTypes[SPECIAL_TYPE_OBJC_SEL_REDEFINITION]) {
+      if (Context.ObjCSelRedefinitionType.isNull())
+        Context.ObjCSelRedefinitionType = GetType(ObjCSelRedef);
     }
   }
-  if (unsigned Sigjmp_buf = SpecialTypes[SPECIAL_TYPE_sigjmp_buf]) {
-    QualType Sigjmp_bufType = GetType(Sigjmp_buf);
-    if (Sigjmp_bufType.isNull()) {
-      Error("sigjmp_buf type is NULL");
-      return;
-    }
-    if (const TypedefType *Typedef = Sigjmp_bufType->getAs<TypedefType>())
-      Context->setsigjmp_bufDecl(Typedef->getDecl());
-    else {
-      const TagType *Tag = Sigjmp_bufType->getAs<TagType>();
-      assert(Tag && "Invalid sigjmp_buf type in AST file");
-      Context->setsigjmp_bufDecl(Tag->getDecl());
-    }
-  }
-  if (unsigned ObjCIdRedef
-        = SpecialTypes[SPECIAL_TYPE_OBJC_ID_REDEFINITION])
-    Context->ObjCIdRedefinitionType = GetType(ObjCIdRedef);
-  if (unsigned ObjCClassRedef
-      = SpecialTypes[SPECIAL_TYPE_OBJC_CLASS_REDEFINITION])
-    Context->ObjCClassRedefinitionType = GetType(ObjCClassRedef);
-  if (unsigned String = SpecialTypes[SPECIAL_TYPE_BLOCK_DESCRIPTOR])
-    Context->setBlockDescriptorType(GetType(String));
-  if (unsigned String
-      = SpecialTypes[SPECIAL_TYPE_BLOCK_EXTENDED_DESCRIPTOR])
-    Context->setBlockDescriptorExtendedType(GetType(String));
-  if (unsigned ObjCSelRedef
-      = SpecialTypes[SPECIAL_TYPE_OBJC_SEL_REDEFINITION])
-    Context->ObjCSelRedefinitionType = GetType(ObjCSelRedef);
-  if (unsigned String = SpecialTypes[SPECIAL_TYPE_NS_CONSTANT_STRING])
-    Context->setNSConstantStringType(GetType(String));
-
-  if (SpecialTypes[SPECIAL_TYPE_INT128_INSTALLED])
-    Context->setInt128Installed();
-
-  if (unsigned AutoDeduct = SpecialTypes[SPECIAL_TYPE_AUTO_DEDUCT])
-    Context->AutoDeductTy = GetType(AutoDeduct);
-  if (unsigned AutoRRefDeduct = SpecialTypes[SPECIAL_TYPE_AUTO_RREF_DEDUCT])
-    Context->AutoRRefDeductTy = GetType(AutoRRefDeduct);
-
-  ReadPragmaDiagnosticMappings(Context->getDiagnostics());
+  
+  ReadPragmaDiagnosticMappings(Context.getDiagnostics());
 
   // If there were any CUDA special declarations, deserialize them.
   if (!CUDASpecialDeclRefs.empty()) {
     assert(CUDASpecialDeclRefs.size() == 1 && "More decl refs than expected!");
-    Context->setcudaConfigureCallDecl(
+    Context.setcudaConfigureCallDecl(
                            cast<FunctionDecl>(GetDecl(CUDASpecialDeclRefs[0])));
   }
 }
@@ -2833,7 +2676,7 @@ void ASTReader::InitializeContext(ASTContext &Ctx) {
 /// file.
 std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName,
                                              FileManager &FileMgr,
-                                             Diagnostic &Diags) {
+                                             DiagnosticsEngine &Diags) {
   // Open the AST file.
   std::string ErrStr;
   llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
@@ -2917,181 +2760,349 @@ std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName,
 ///
 /// \returns true if the listener deems the file unacceptable, false otherwise.
 bool ASTReader::ParseLanguageOptions(
-                             const llvm::SmallVectorImpl<uint64_t> &Record) {
+                             const SmallVectorImpl<uint64_t> &Record) {
   if (Listener) {
     LangOptions LangOpts;
-
-  #define PARSE_LANGOPT(Option)                  \
-      LangOpts.Option = Record[Idx];             \
-      ++Idx
-
     unsigned Idx = 0;
-    PARSE_LANGOPT(Trigraphs);
-    PARSE_LANGOPT(BCPLComment);
-    PARSE_LANGOPT(DollarIdents);
-    PARSE_LANGOPT(AsmPreprocessor);
-    PARSE_LANGOPT(GNUMode);
-    PARSE_LANGOPT(GNUKeywords);
-    PARSE_LANGOPT(ImplicitInt);
-    PARSE_LANGOPT(Digraphs);
-    PARSE_LANGOPT(HexFloats);
-    PARSE_LANGOPT(C99);
-    PARSE_LANGOPT(C1X);
-    PARSE_LANGOPT(Microsoft);
-    PARSE_LANGOPT(CPlusPlus);
-    PARSE_LANGOPT(CPlusPlus0x);
-    PARSE_LANGOPT(CXXOperatorNames);
-    PARSE_LANGOPT(ObjC1);
-    PARSE_LANGOPT(ObjC2);
-    PARSE_LANGOPT(ObjCNonFragileABI);
-    PARSE_LANGOPT(ObjCNonFragileABI2);
-    PARSE_LANGOPT(AppleKext);
-    PARSE_LANGOPT(ObjCDefaultSynthProperties);
-    PARSE_LANGOPT(ObjCInferRelatedResultType);
-    PARSE_LANGOPT(NoConstantCFStrings);
-    PARSE_LANGOPT(PascalStrings);
-    PARSE_LANGOPT(WritableStrings);
-    PARSE_LANGOPT(LaxVectorConversions);
-    PARSE_LANGOPT(AltiVec);
-    PARSE_LANGOPT(Exceptions);
-    PARSE_LANGOPT(ObjCExceptions);
-    PARSE_LANGOPT(CXXExceptions);
-    PARSE_LANGOPT(SjLjExceptions);
-    PARSE_LANGOPT(MSBitfields);
-    PARSE_LANGOPT(NeXTRuntime);
-    PARSE_LANGOPT(Freestanding);
-    PARSE_LANGOPT(NoBuiltin);
-    PARSE_LANGOPT(ThreadsafeStatics);
-    PARSE_LANGOPT(POSIXThreads);
-    PARSE_LANGOPT(Blocks);
-    PARSE_LANGOPT(EmitAllDecls);
-    PARSE_LANGOPT(MathErrno);
-    LangOpts.setSignedOverflowBehavior((LangOptions::SignedOverflowBehaviorTy)
-                                       Record[Idx++]);
-    PARSE_LANGOPT(HeinousExtensions);
-    PARSE_LANGOPT(Optimize);
-    PARSE_LANGOPT(OptimizeSize);
-    PARSE_LANGOPT(Static);
-    PARSE_LANGOPT(PICLevel);
-    PARSE_LANGOPT(GNUInline);
-    PARSE_LANGOPT(NoInline);
-    PARSE_LANGOPT(Deprecated);
-    PARSE_LANGOPT(AccessControl);
-    PARSE_LANGOPT(CharIsSigned);
-    PARSE_LANGOPT(ShortWChar);
-    PARSE_LANGOPT(ShortEnums);
-    LangOpts.setGCMode((LangOptions::GCMode)Record[Idx++]);
-    LangOpts.setVisibilityMode((Visibility)Record[Idx++]);
-    LangOpts.setStackProtectorMode((LangOptions::StackProtectorMode)
-                                   Record[Idx++]);
-    PARSE_LANGOPT(InstantiationDepth);
-    PARSE_LANGOPT(OpenCL);
-    PARSE_LANGOPT(CUDA);
-    PARSE_LANGOPT(CatchUndefined);
-    PARSE_LANGOPT(DefaultFPContract);
-    PARSE_LANGOPT(ElideConstructors);
-    PARSE_LANGOPT(SpellChecking);
-    PARSE_LANGOPT(MRTD);
-    PARSE_LANGOPT(ObjCAutoRefCount);
-  #undef PARSE_LANGOPT
-
+#define LANGOPT(Name, Bits, Default, Description) \
+  LangOpts.Name = Record[Idx++];
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  LangOpts.set##Name(static_cast<LangOptions::Type>(Record[Idx++]));
+#include "clang/Basic/LangOptions.def"
+    
     return Listener->ReadLanguageOptions(LangOpts);
   }
 
   return false;
 }
 
-void ASTReader::ReadPreprocessedEntities() {
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    PerFileData &F = *Chain[I];
-    if (!F.PreprocessorDetailCursor.getBitStreamReader())
-      continue;
+PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) {
+  PreprocessedEntityID PPID = Index+1;
+  GlobalPreprocessedEntityMapType::iterator
+    I = GlobalPreprocessedEntityMap.find(Index);
+  assert(I != GlobalPreprocessedEntityMap.end() && 
+         "Corrupted global preprocessed entity map");
+  Module &M = *I->second;
+  unsigned LocalIndex = Index - M.BasePreprocessedEntityID;
+  const PPEntityOffset &PPOffs = M.PreprocessedEntityOffsets[LocalIndex];
 
-    SavedStreamPosition SavedPosition(F.PreprocessorDetailCursor);  
-    F.PreprocessorDetailCursor.JumpToBit(F.PreprocessorDetailStartOffset);
-    while (LoadPreprocessedEntity(F)) { }
-  }
-}
+  SavedStreamPosition SavedPosition(M.PreprocessorDetailCursor);  
+  M.PreprocessorDetailCursor.JumpToBit(PPOffs.BitOffset);
 
-PreprocessedEntity *ASTReader::ReadPreprocessedEntityAtOffset(uint64_t Offset) {
-  PerFileData *F = 0;  
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    if (Offset < Chain[I]->SizeInBits) {
-      F = Chain[I];
-      break;
-    }
+  unsigned Code = M.PreprocessorDetailCursor.ReadCode();
+  switch (Code) {
+  case llvm::bitc::END_BLOCK:
+    return 0;
     
-    Offset -= Chain[I]->SizeInBits;
+  case llvm::bitc::ENTER_SUBBLOCK:
+    Error("unexpected subblock record in preprocessor detail block");
+    return 0;
+      
+  case llvm::bitc::DEFINE_ABBREV:
+    Error("unexpected abbrevation record in preprocessor detail block");
+    return 0;
+      
+  default:
+    break;
   }
-  
-  if (!F) {
-    Error("Malformed preprocessed entity offset");
+
+  if (!PP.getPreprocessingRecord()) {
+    Error("no preprocessing record");
     return 0;
   }
+  
+  // Read the record.
+  SourceRange Range(ReadSourceLocation(M, PPOffs.Begin),
+                    ReadSourceLocation(M, PPOffs.End));
+  PreprocessingRecord &PPRec = *PP.getPreprocessingRecord();
+  const char *BlobStart = 0;
+  unsigned BlobLen = 0;
+  RecordData Record;
+  PreprocessorDetailRecordTypes RecType =
+    (PreprocessorDetailRecordTypes)M.PreprocessorDetailCursor.ReadRecord(
+                                             Code, Record, BlobStart, BlobLen);
+  switch (RecType) {
+  case PPD_MACRO_EXPANSION: {
+    bool isBuiltin = Record[0];
+    IdentifierInfo *Name = 0;
+    MacroDefinition *Def = 0;
+    if (isBuiltin)
+      Name = getLocalIdentifier(M, Record[1]);
+    else {
+      PreprocessedEntityID
+          GlobalID = getGlobalPreprocessedEntityID(M, Record[1]);
+      Def =cast<MacroDefinition>(PPRec.getLoadedPreprocessedEntity(GlobalID-1));
+    }
 
-  // Keep track of where we are in the stream, then jump back there
-  // after reading this entity.
-  SavedStreamPosition SavedPosition(F->PreprocessorDetailCursor);  
-  F->PreprocessorDetailCursor.JumpToBit(Offset);
-  return LoadPreprocessedEntity(*F);
+    MacroExpansion *ME;
+    if (isBuiltin)
+      ME = new (PPRec) MacroExpansion(Name, Range);
+    else
+      ME = new (PPRec) MacroExpansion(Def, Range);
+
+    return ME;
+  }
+      
+  case PPD_MACRO_DEFINITION: {
+    // Decode the identifier info and then check again; if the macro is
+    // still defined and associated with the identifier,
+    IdentifierInfo *II = getLocalIdentifier(M, Record[0]);
+    MacroDefinition *MD
+      = new (PPRec) MacroDefinition(II, Range);
+
+    if (DeserializationListener)
+      DeserializationListener->MacroDefinitionRead(PPID, MD);
+
+    return MD;
+  }
+      
+  case PPD_INCLUSION_DIRECTIVE: {
+    const char *FullFileNameStart = BlobStart + Record[0];
+    const FileEntry *File
+      = PP.getFileManager().getFile(StringRef(FullFileNameStart,
+                                               BlobLen - Record[0]));
+    
+    // FIXME: Stable encoding
+    InclusionDirective::InclusionKind Kind
+      = static_cast<InclusionDirective::InclusionKind>(Record[2]);
+    InclusionDirective *ID
+      = new (PPRec) InclusionDirective(PPRec, Kind,
+                                       StringRef(BlobStart, Record[0]),
+                                       Record[1],
+                                       File,
+                                       Range);
+    return ID;
+  }
+  }
+  
+  Error("invalid offset in preprocessor detail block");
+  return 0;
+}
+
+/// \brief \arg SLocMapI points at a chunk of a module that contains no
+/// preprocessed entities or the entities it contains are not the ones we are
+/// looking for. Find the next module that contains entities and return the ID
+/// of the first entry.
+PreprocessedEntityID ASTReader::findNextPreprocessedEntity(
+                       GlobalSLocOffsetMapType::const_iterator SLocMapI) const {
+  ++SLocMapI;
+  for (GlobalSLocOffsetMapType::const_iterator
+         EndI = GlobalSLocOffsetMap.end(); SLocMapI != EndI; ++SLocMapI) {
+    Module &M = *SLocMapI->second;
+    if (M.NumPreprocessedEntities)
+      return getGlobalPreprocessedEntityID(M, M.BasePreprocessedEntityID);
+  }
+
+  return getTotalNumPreprocessedEntities();
+}
+
+namespace {
+
+template <unsigned PPEntityOffset::*PPLoc>
+struct PPEntityComp {
+  const ASTReader &Reader;
+  Module &M;
+
+  PPEntityComp(const ASTReader &Reader, Module &M) : Reader(Reader), M(M) { }
+
+  bool operator()(const PPEntityOffset &L, const PPEntityOffset &R) const {
+    SourceLocation LHS = getLoc(L);
+    SourceLocation RHS = getLoc(R);
+    return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  bool operator()(const PPEntityOffset &L, SourceLocation RHS) const {
+    SourceLocation LHS = getLoc(L);
+    return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  bool operator()(SourceLocation LHS, const PPEntityOffset &R) const {
+    SourceLocation RHS = getLoc(R);
+    return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  SourceLocation getLoc(const PPEntityOffset &PPE) const {
+    return Reader.ReadSourceLocation(M, PPE.*PPLoc);
+  }
+};
+
+}
+
+/// \brief Returns the first preprocessed entity ID that ends after \arg BLoc.
+PreprocessedEntityID
+ASTReader::findBeginPreprocessedEntity(SourceLocation BLoc) const {
+  if (SourceMgr.isLocalSourceLocation(BLoc))
+    return getTotalNumPreprocessedEntities();
+
+  GlobalSLocOffsetMapType::const_iterator
+    SLocMapI = GlobalSLocOffsetMap.find(SourceManager::MaxLoadedOffset -
+                                        BLoc.getOffset());
+  assert(SLocMapI != GlobalSLocOffsetMap.end() &&
+         "Corrupted global sloc offset map");
+
+  if (SLocMapI->second->NumPreprocessedEntities == 0)
+    return findNextPreprocessedEntity(SLocMapI);
+
+  Module &M = *SLocMapI->second;
+  typedef const PPEntityOffset *pp_iterator;
+  pp_iterator pp_begin = M.PreprocessedEntityOffsets;
+  pp_iterator pp_end = pp_begin + M.NumPreprocessedEntities;
+
+  size_t Count = M.NumPreprocessedEntities;
+  size_t Half;
+  pp_iterator First = pp_begin;
+  pp_iterator PPI;
+
+  // Do a binary search manually instead of using std::lower_bound because
+  // The end locations of entities may be unordered (when a macro expansion
+  // is inside another macro argument), but for this case it is not important
+  // whether we get the first macro expansion or its containing macro.
+  while (Count > 0) {
+    Half = Count/2;
+    PPI = First;
+    std::advance(PPI, Half);
+    if (SourceMgr.isBeforeInTranslationUnit(ReadSourceLocation(M, PPI->End),
+                                            BLoc)){
+      First = PPI;
+      ++First;
+      Count = Count - Half - 1;
+    } else
+      Count = Half;
+  }
+
+  if (PPI == pp_end)
+    return findNextPreprocessedEntity(SLocMapI);
+
+  return getGlobalPreprocessedEntityID(M,
+                                 M.BasePreprocessedEntityID + (PPI - pp_begin));
+}
+
+/// \brief Returns the first preprocessed entity ID that begins after \arg ELoc.
+PreprocessedEntityID
+ASTReader::findEndPreprocessedEntity(SourceLocation ELoc) const {
+  if (SourceMgr.isLocalSourceLocation(ELoc))
+    return getTotalNumPreprocessedEntities();
+
+  GlobalSLocOffsetMapType::const_iterator
+    SLocMapI = GlobalSLocOffsetMap.find(SourceManager::MaxLoadedOffset -
+                                        ELoc.getOffset());
+  assert(SLocMapI != GlobalSLocOffsetMap.end() &&
+         "Corrupted global sloc offset map");
+
+  if (SLocMapI->second->NumPreprocessedEntities == 0)
+    return findNextPreprocessedEntity(SLocMapI);
+
+  Module &M = *SLocMapI->second;
+  typedef const PPEntityOffset *pp_iterator;
+  pp_iterator pp_begin = M.PreprocessedEntityOffsets;
+  pp_iterator pp_end = pp_begin + M.NumPreprocessedEntities;
+  pp_iterator PPI =
+      std::upper_bound(pp_begin, pp_end, ELoc,
+                       PPEntityComp<&PPEntityOffset::Begin>(*this, M));
+
+  if (PPI == pp_end)
+    return findNextPreprocessedEntity(SLocMapI);
+
+  return getGlobalPreprocessedEntityID(M,
+                                 M.BasePreprocessedEntityID + (PPI - pp_begin));
+}
+
+/// \brief Returns a pair of [Begin, End) indices of preallocated
+/// preprocessed entities that \arg Range encompasses.
+std::pair<unsigned, unsigned>
+    ASTReader::findPreprocessedEntitiesInRange(SourceRange Range) {
+  if (Range.isInvalid())
+    return std::make_pair(0,0);
+  assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
+
+  PreprocessedEntityID BeginID = findBeginPreprocessedEntity(Range.getBegin());
+  PreprocessedEntityID EndID = findEndPreprocessedEntity(Range.getEnd());
+  return std::make_pair(BeginID, EndID);
+}
+
+namespace {
+  /// \brief Visitor used to search for information about a header file.
+  class HeaderFileInfoVisitor {
+    ASTReader &Reader;
+    const FileEntry *FE;
+    
+    llvm::Optional<HeaderFileInfo> HFI;
+    
+  public:
+    HeaderFileInfoVisitor(ASTReader &Reader, const FileEntry *FE)
+      : Reader(Reader), FE(FE) { }
+    
+    static bool visit(Module &M, void *UserData) {
+      HeaderFileInfoVisitor *This
+        = static_cast<HeaderFileInfoVisitor *>(UserData);
+      
+      HeaderFileInfoTrait Trait(This->Reader, M, 
+                                &This->Reader.getPreprocessor().getHeaderSearchInfo(),
+                                M.HeaderFileFrameworkStrings,
+                                This->FE->getName());
+      
+      HeaderFileInfoLookupTable *Table
+        = static_cast<HeaderFileInfoLookupTable *>(M.HeaderFileInfoTable);
+      if (!Table)
+        return false;
+
+      // Look in the on-disk hash table for an entry for this file name.
+      HeaderFileInfoLookupTable::iterator Pos = Table->find(This->FE->getName(),
+                                                            &Trait);
+      if (Pos == Table->end())
+        return false;
+
+      This->HFI = *Pos;
+      return true;
+    }
+    
+    llvm::Optional<HeaderFileInfo> getHeaderFileInfo() const { return HFI; }
+  };
 }
 
 HeaderFileInfo ASTReader::GetHeaderFileInfo(const FileEntry *FE) {
-  HeaderFileInfoTrait Trait(FE->getName());
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    PerFileData &F = *Chain[I];
-    HeaderFileInfoLookupTable *Table
-      = static_cast<HeaderFileInfoLookupTable *>(F.HeaderFileInfoTable);
-    if (!Table)
-      continue;
-    
-    // Look in the on-disk hash table for an entry for this file name.
-    HeaderFileInfoLookupTable::iterator Pos = Table->find(FE->getName(), 
-                                                          &Trait);
-    if (Pos == Table->end())
-      continue;
-
-    HeaderFileInfo HFI = *Pos;
+  HeaderFileInfoVisitor Visitor(*this, FE);
+  ModuleMgr.visit(&HeaderFileInfoVisitor::visit, &Visitor);
+  if (llvm::Optional<HeaderFileInfo> HFI = Visitor.getHeaderFileInfo()) {
     if (Listener)
-      Listener->ReadHeaderFileInfo(HFI, FE->getUID());
-
-    return HFI;
+      Listener->ReadHeaderFileInfo(*HFI, FE->getUID());
+    return *HFI;
   }
   
   return HeaderFileInfo();
 }
 
-void ASTReader::ReadPragmaDiagnosticMappings(Diagnostic &Diag) {
-  unsigned Idx = 0;
-  while (Idx < PragmaDiagMappings.size()) {
-    SourceLocation
-      Loc = SourceLocation::getFromRawEncoding(PragmaDiagMappings[Idx++]);
-    while (1) {
-      assert(Idx < PragmaDiagMappings.size() &&
-             "Invalid data, didn't find '-1' marking end of diag/map pairs");
-      if (Idx >= PragmaDiagMappings.size())
-        break; // Something is messed up but at least avoid infinite loop in
-               // release build.
-      unsigned DiagID = PragmaDiagMappings[Idx++];
-      if (DiagID == (unsigned)-1)
-        break; // no more diag/map pairs for this location.
-      diag::Mapping Map = (diag::Mapping)PragmaDiagMappings[Idx++];
-      Diag.setDiagnosticMapping(DiagID, Map, Loc);
+void ASTReader::ReadPragmaDiagnosticMappings(DiagnosticsEngine &Diag) {
+  for (ModuleIterator I = ModuleMgr.begin(), E = ModuleMgr.end(); I != E; ++I) {
+    Module &F = *(*I);
+    unsigned Idx = 0;
+    while (Idx < F.PragmaDiagMappings.size()) {
+      SourceLocation Loc = ReadSourceLocation(F, F.PragmaDiagMappings[Idx++]);
+      while (1) {
+        assert(Idx < F.PragmaDiagMappings.size() &&
+               "Invalid data, didn't find '-1' marking end of diag/map pairs");
+        if (Idx >= F.PragmaDiagMappings.size()) {
+          break; // Something is messed up but at least avoid infinite loop in
+                 // release build.
+        }
+        unsigned DiagID = F.PragmaDiagMappings[Idx++];
+        if (DiagID == (unsigned)-1) {
+          break; // no more diag/map pairs for this location.
+        }
+        diag::Mapping Map = (diag::Mapping)F.PragmaDiagMappings[Idx++];
+        // The user bit gets set by WritePragmaDiagnosticMappings.
+        Diag.setDiagnosticMapping(DiagID, Map, Loc);
+      }
     }
   }
 }
 
 /// \brief Get the correct cursor and offset for loading a type.
 ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
-  PerFileData *F = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    F = Chain[N - I - 1];
-    if (Index < F->LocalNumTypes)
-      break;
-    Index -= F->LocalNumTypes;
-  }
-  assert(F && F->LocalNumTypes > Index && "Broken chain");
-  return RecordLocation(F, F->TypeOffsets[Index]);
+  GlobalTypeMapType::iterator I = GlobalTypeMap.find(Index);
+  assert(I != GlobalTypeMap.end() && "Corrupted global type map");
+  Module *M = I->second;
+  return RecordLocation(M, M->TypeOffsets[Index - M->BaseTypeIndex]);
 }
 
 /// \brief Read and return the type with the given index..
@@ -3100,7 +3111,7 @@ ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
 /// routine actually reads the record corresponding to the type at the given
 /// location. It is a helper routine for GetType, which deals with reading type
 /// IDs.
-QualType ASTReader::ReadTypeRecord(unsigned Index) {
+QualType ASTReader::readTypeRecord(unsigned Index) {
   RecordLocation Loc = TypeCursorForIndex(Index);
   llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor;
 
@@ -3113,6 +3124,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
   // Note that we are loading a type record.
   Deserializing AType(this);
 
+  unsigned Idx = 0;
   DeclsCursor.JumpToBit(Loc.Offset);
   RecordData Record;
   unsigned Code = DeclsCursor.ReadCode();
@@ -3122,9 +3134,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of extended qualifier type");
       return QualType();
     }
-    QualType Base = GetType(Record[0]);
-    Qualifiers Quals = Qualifiers::fromOpaqueValue(Record[1]);
-    return Context->getQualifiedType(Base, Quals);
+    QualType Base = readType(*Loc.F, Record, Idx);
+    Qualifiers Quals = Qualifiers::fromOpaqueValue(Record[Idx++]);
+    return Context.getQualifiedType(Base, Quals);
   }
 
   case TYPE_COMPLEX: {
@@ -3132,8 +3144,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of complex type");
       return QualType();
     }
-    QualType ElemType = GetType(Record[0]);
-    return Context->getComplexType(ElemType);
+    QualType ElemType = readType(*Loc.F, Record, Idx);
+    return Context.getComplexType(ElemType);
   }
 
   case TYPE_POINTER: {
@@ -3141,8 +3153,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of pointer type");
       return QualType();
     }
-    QualType PointeeType = GetType(Record[0]);
-    return Context->getPointerType(PointeeType);
+    QualType PointeeType = readType(*Loc.F, Record, Idx);
+    return Context.getPointerType(PointeeType);
   }
 
   case TYPE_BLOCK_POINTER: {
@@ -3150,8 +3162,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of block pointer type");
       return QualType();
     }
-    QualType PointeeType = GetType(Record[0]);
-    return Context->getBlockPointerType(PointeeType);
+    QualType PointeeType = readType(*Loc.F, Record, Idx);
+    return Context.getBlockPointerType(PointeeType);
   }
 
   case TYPE_LVALUE_REFERENCE: {
@@ -3159,8 +3171,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of lvalue reference type");
       return QualType();
     }
-    QualType PointeeType = GetType(Record[0]);
-    return Context->getLValueReferenceType(PointeeType, Record[1]);
+    QualType PointeeType = readType(*Loc.F, Record, Idx);
+    return Context.getLValueReferenceType(PointeeType, Record[1]);
   }
 
   case TYPE_RVALUE_REFERENCE: {
@@ -3168,8 +3180,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of rvalue reference type");
       return QualType();
     }
-    QualType PointeeType = GetType(Record[0]);
-    return Context->getRValueReferenceType(PointeeType);
+    QualType PointeeType = readType(*Loc.F, Record, Idx);
+    return Context.getRValueReferenceType(PointeeType);
   }
 
   case TYPE_MEMBER_POINTER: {
@@ -3177,38 +3189,38 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("Incorrect encoding of member pointer type");
       return QualType();
     }
-    QualType PointeeType = GetType(Record[0]);
-    QualType ClassType = GetType(Record[1]);
+    QualType PointeeType = readType(*Loc.F, Record, Idx);
+    QualType ClassType = readType(*Loc.F, Record, Idx);
     if (PointeeType.isNull() || ClassType.isNull())
       return QualType();
     
-    return Context->getMemberPointerType(PointeeType, ClassType.getTypePtr());
+    return Context.getMemberPointerType(PointeeType, ClassType.getTypePtr());
   }
 
   case TYPE_CONSTANT_ARRAY: {
-    QualType ElementType = GetType(Record[0]);
+    QualType ElementType = readType(*Loc.F, Record, Idx);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
     unsigned Idx = 3;
     llvm::APInt Size = ReadAPInt(Record, Idx);
-    return Context->getConstantArrayType(ElementType, Size,
+    return Context.getConstantArrayType(ElementType, Size,
                                          ASM, IndexTypeQuals);
   }
 
   case TYPE_INCOMPLETE_ARRAY: {
-    QualType ElementType = GetType(Record[0]);
+    QualType ElementType = readType(*Loc.F, Record, Idx);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
-    return Context->getIncompleteArrayType(ElementType, ASM, IndexTypeQuals);
+    return Context.getIncompleteArrayType(ElementType, ASM, IndexTypeQuals);
   }
 
   case TYPE_VARIABLE_ARRAY: {
-    QualType ElementType = GetType(Record[0]);
+    QualType ElementType = readType(*Loc.F, Record, Idx);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
     SourceLocation LBLoc = ReadSourceLocation(*Loc.F, Record[3]);
     SourceLocation RBLoc = ReadSourceLocation(*Loc.F, Record[4]);
-    return Context->getVariableArrayType(ElementType, ReadExpr(*Loc.F),
+    return Context.getVariableArrayType(ElementType, ReadExpr(*Loc.F),
                                          ASM, IndexTypeQuals,
                                          SourceRange(LBLoc, RBLoc));
   }
@@ -3219,10 +3231,10 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       return QualType();
     }
 
-    QualType ElementType = GetType(Record[0]);
+    QualType ElementType = readType(*Loc.F, Record, Idx);
     unsigned NumElements = Record[1];
     unsigned VecKind = Record[2];
-    return Context->getVectorType(ElementType, NumElements,
+    return Context.getVectorType(ElementType, NumElements,
                                   (VectorType::VectorKind)VecKind);
   }
 
@@ -3232,9 +3244,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       return QualType();
     }
 
-    QualType ElementType = GetType(Record[0]);
+    QualType ElementType = readType(*Loc.F, Record, Idx);
     unsigned NumElements = Record[1];
-    return Context->getExtVectorType(ElementType, NumElements);
+    return Context.getExtVectorType(ElementType, NumElements);
   }
 
   case TYPE_FUNCTION_NO_PROTO: {
@@ -3242,14 +3254,14 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("incorrect encoding of no-proto function type");
       return QualType();
     }
-    QualType ResultType = GetType(Record[0]);
+    QualType ResultType = readType(*Loc.F, Record, Idx);
     FunctionType::ExtInfo Info(Record[1], Record[2], Record[3],
                                (CallingConv)Record[4], Record[5]);
-    return Context->getFunctionNoProtoType(ResultType, Info);
+    return Context.getFunctionNoProtoType(ResultType, Info);
   }
 
   case TYPE_FUNCTION_PROTO: {
-    QualType ResultType = GetType(Record[0]);
+    QualType ResultType = readType(*Loc.F, Record, Idx);
 
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.ExtInfo = FunctionType::ExtInfo(/*noreturn*/ Record[1],
@@ -3260,9 +3272,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
 
     unsigned Idx = 6;
     unsigned NumParams = Record[Idx++];
-    llvm::SmallVector<QualType, 16> ParamTypes;
+    SmallVector<QualType, 16> ParamTypes;
     for (unsigned I = 0; I != NumParams; ++I)
-      ParamTypes.push_back(GetType(Record[Idx++]));
+      ParamTypes.push_back(readType(*Loc.F, Record, Idx));
 
     EPI.Variadic = Record[Idx++];
     EPI.TypeQuals = Record[Idx++];
@@ -3272,65 +3284,70 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     EPI.ExceptionSpecType = EST;
     if (EST == EST_Dynamic) {
       EPI.NumExceptions = Record[Idx++];
-      llvm::SmallVector<QualType, 2> Exceptions;
+      SmallVector<QualType, 2> Exceptions;
       for (unsigned I = 0; I != EPI.NumExceptions; ++I)
-        Exceptions.push_back(GetType(Record[Idx++]));
+        Exceptions.push_back(readType(*Loc.F, Record, Idx));
       EPI.Exceptions = Exceptions.data();
     } else if (EST == EST_ComputedNoexcept) {
       EPI.NoexceptExpr = ReadExpr(*Loc.F);
     }
-    return Context->getFunctionType(ResultType, ParamTypes.data(), NumParams,
+    return Context.getFunctionType(ResultType, ParamTypes.data(), NumParams,
                                     EPI);
   }
 
-  case TYPE_UNRESOLVED_USING:
-    return Context->getTypeDeclType(
-             cast<UnresolvedUsingTypenameDecl>(GetDecl(Record[0])));
-
+  case TYPE_UNRESOLVED_USING: {
+    unsigned Idx = 0;
+    return Context.getTypeDeclType(
+                  ReadDeclAs<UnresolvedUsingTypenameDecl>(*Loc.F, Record, Idx));
+  }
+      
   case TYPE_TYPEDEF: {
     if (Record.size() != 2) {
       Error("incorrect encoding of typedef type");
       return QualType();
     }
-    TypedefNameDecl *Decl = cast<TypedefNameDecl>(GetDecl(Record[0]));
-    QualType Canonical = GetType(Record[1]);
+    unsigned Idx = 0;
+    TypedefNameDecl *Decl = ReadDeclAs<TypedefNameDecl>(*Loc.F, Record, Idx);
+    QualType Canonical = readType(*Loc.F, Record, Idx);
     if (!Canonical.isNull())
-      Canonical = Context->getCanonicalType(Canonical);
-    return Context->getTypedefType(Decl, Canonical);
+      Canonical = Context.getCanonicalType(Canonical);
+    return Context.getTypedefType(Decl, Canonical);
   }
 
   case TYPE_TYPEOF_EXPR:
-    return Context->getTypeOfExprType(ReadExpr(*Loc.F));
+    return Context.getTypeOfExprType(ReadExpr(*Loc.F));
 
   case TYPE_TYPEOF: {
     if (Record.size() != 1) {
       Error("incorrect encoding of typeof(type) in AST file");
       return QualType();
     }
-    QualType UnderlyingType = GetType(Record[0]);
-    return Context->getTypeOfType(UnderlyingType);
+    QualType UnderlyingType = readType(*Loc.F, Record, Idx);
+    return Context.getTypeOfType(UnderlyingType);
   }
 
   case TYPE_DECLTYPE:
-    return Context->getDecltypeType(ReadExpr(*Loc.F));
+    return Context.getDecltypeType(ReadExpr(*Loc.F));
 
   case TYPE_UNARY_TRANSFORM: {
-    QualType BaseType = GetType(Record[0]);
-    QualType UnderlyingType = GetType(Record[1]);
+    QualType BaseType = readType(*Loc.F, Record, Idx);
+    QualType UnderlyingType = readType(*Loc.F, Record, Idx);
     UnaryTransformType::UTTKind UKind = (UnaryTransformType::UTTKind)Record[2];
-    return Context->getUnaryTransformType(BaseType, UnderlyingType, UKind);
+    return Context.getUnaryTransformType(BaseType, UnderlyingType, UKind);
   }
 
   case TYPE_AUTO:
-    return Context->getAutoType(GetType(Record[0]));
+    return Context.getAutoType(readType(*Loc.F, Record, Idx));
 
   case TYPE_RECORD: {
     if (Record.size() != 2) {
       Error("incorrect encoding of record type");
       return QualType();
     }
-    bool IsDependent = Record[0];
-    QualType T = Context->getRecordType(cast<RecordDecl>(GetDecl(Record[1])));
+    unsigned Idx = 0;
+    bool IsDependent = Record[Idx++];
+    QualType T
+      = Context.getRecordType(ReadDeclAs<RecordDecl>(*Loc.F, Record, Idx));
     const_cast<Type*>(T.getTypePtr())->setDependent(IsDependent);
     return T;
   }
@@ -3340,8 +3357,10 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("incorrect encoding of enum type");
       return QualType();
     }
-    bool IsDependent = Record[0];
-    QualType T = Context->getEnumType(cast<EnumDecl>(GetDecl(Record[1])));
+    unsigned Idx = 0;
+    bool IsDependent = Record[Idx++];
+    QualType T
+      = Context.getEnumType(ReadDeclAs<EnumDecl>(*Loc.F, Record, Idx));
     const_cast<Type*>(T.getTypePtr())->setDependent(IsDependent);
     return T;
   }
@@ -3351,10 +3370,10 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("incorrect encoding of attributed type");
       return QualType();
     }
-    QualType modifiedType = GetType(Record[0]);
-    QualType equivalentType = GetType(Record[1]);
+    QualType modifiedType = readType(*Loc.F, Record, Idx);
+    QualType equivalentType = readType(*Loc.F, Record, Idx);
     AttributedType::Kind kind = static_cast<AttributedType::Kind>(Record[2]);
-    return Context->getAttributedType(kind, modifiedType, equivalentType);
+    return Context.getAttributedType(kind, modifiedType, equivalentType);
   }
 
   case TYPE_PAREN: {
@@ -3362,8 +3381,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("incorrect encoding of paren type");
       return QualType();
     }
-    QualType InnerType = GetType(Record[0]);
-    return Context->getParenType(InnerType);
+    QualType InnerType = readType(*Loc.F, Record, Idx);
+    return Context.getParenType(InnerType);
   }
 
   case TYPE_PACK_EXPANSION: {
@@ -3371,70 +3390,71 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
       Error("incorrect encoding of pack expansion type");
       return QualType();
     }
-    QualType Pattern = GetType(Record[0]);
+    QualType Pattern = readType(*Loc.F, Record, Idx);
     if (Pattern.isNull())
       return QualType();
     llvm::Optional<unsigned> NumExpansions;
     if (Record[1])
       NumExpansions = Record[1] - 1;
-    return Context->getPackExpansionType(Pattern, NumExpansions);
+    return Context.getPackExpansionType(Pattern, NumExpansions);
   }
 
   case TYPE_ELABORATED: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
-    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
-    QualType NamedType = GetType(Record[Idx++]);
-    return Context->getElaboratedType(Keyword, NNS, NamedType);
+    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(*Loc.F, Record, Idx);
+    QualType NamedType = readType(*Loc.F, Record, Idx);
+    return Context.getElaboratedType(Keyword, NNS, NamedType);
   }
 
   case TYPE_OBJC_INTERFACE: {
     unsigned Idx = 0;
-    ObjCInterfaceDecl *ItfD = cast<ObjCInterfaceDecl>(GetDecl(Record[Idx++]));
-    return Context->getObjCInterfaceType(ItfD);
+    ObjCInterfaceDecl *ItfD
+      = ReadDeclAs<ObjCInterfaceDecl>(*Loc.F, Record, Idx);
+    return Context.getObjCInterfaceType(ItfD);
   }
 
   case TYPE_OBJC_OBJECT: {
     unsigned Idx = 0;
-    QualType Base = GetType(Record[Idx++]);
+    QualType Base = readType(*Loc.F, Record, Idx);
     unsigned NumProtos = Record[Idx++];
-    llvm::SmallVector<ObjCProtocolDecl*, 4> Protos;
+    SmallVector<ObjCProtocolDecl*, 4> Protos;
     for (unsigned I = 0; I != NumProtos; ++I)
-      Protos.push_back(cast<ObjCProtocolDecl>(GetDecl(Record[Idx++])));
-    return Context->getObjCObjectType(Base, Protos.data(), NumProtos);
+      Protos.push_back(ReadDeclAs<ObjCProtocolDecl>(*Loc.F, Record, Idx));
+    return Context.getObjCObjectType(Base, Protos.data(), NumProtos);
   }
 
   case TYPE_OBJC_OBJECT_POINTER: {
     unsigned Idx = 0;
-    QualType Pointee = GetType(Record[Idx++]);
-    return Context->getObjCObjectPointerType(Pointee);
+    QualType Pointee = readType(*Loc.F, Record, Idx);
+    return Context.getObjCObjectPointerType(Pointee);
   }
 
   case TYPE_SUBST_TEMPLATE_TYPE_PARM: {
     unsigned Idx = 0;
-    QualType Parm = GetType(Record[Idx++]);
-    QualType Replacement = GetType(Record[Idx++]);
+    QualType Parm = readType(*Loc.F, Record, Idx);
+    QualType Replacement = readType(*Loc.F, Record, Idx);
     return
-      Context->getSubstTemplateTypeParmType(cast<TemplateTypeParmType>(Parm),
+      Context.getSubstTemplateTypeParmType(cast<TemplateTypeParmType>(Parm),
                                             Replacement);
   }
 
   case TYPE_SUBST_TEMPLATE_TYPE_PARM_PACK: {
     unsigned Idx = 0;
-    QualType Parm = GetType(Record[Idx++]);
+    QualType Parm = readType(*Loc.F, Record, Idx);
     TemplateArgument ArgPack = ReadTemplateArgument(*Loc.F, Record, Idx);
-    return Context->getSubstTemplateTypeParmPackType(
+    return Context.getSubstTemplateTypeParmPackType(
                                                cast<TemplateTypeParmType>(Parm),
                                                      ArgPack);
   }
 
   case TYPE_INJECTED_CLASS_NAME: {
-    CXXRecordDecl *D = cast<CXXRecordDecl>(GetDecl(Record[0]));
-    QualType TST = GetType(Record[1]); // probably derivable
+    CXXRecordDecl *D = ReadDeclAs<CXXRecordDecl>(*Loc.F, Record, Idx);
+    QualType TST = readType(*Loc.F, Record, Idx); // probably derivable
     // FIXME: ASTContext::getInjectedClassNameType is not currently suitable
     // for AST reading, too much interdependencies.
     return
-      QualType(new (*Context, TypeAlignment) InjectedClassNameType(D, TST), 0);
+      QualType(new (Context, TypeAlignment) InjectedClassNameType(D, TST), 0);
   }
 
   case TYPE_TEMPLATE_TYPE_PARM: {
@@ -3442,33 +3462,33 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     unsigned Depth = Record[Idx++];
     unsigned Index = Record[Idx++];
     bool Pack = Record[Idx++];
-    TemplateTypeParmDecl *D =
-      cast_or_null<TemplateTypeParmDecl>(GetDecl(Record[Idx++]));
-    return Context->getTemplateTypeParmType(Depth, Index, Pack, D);
+    TemplateTypeParmDecl *D
+      = ReadDeclAs<TemplateTypeParmDecl>(*Loc.F, Record, Idx);
+    return Context.getTemplateTypeParmType(Depth, Index, Pack, D);
   }
 
   case TYPE_DEPENDENT_NAME: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
-    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
-    const IdentifierInfo *Name = this->GetIdentifierInfo(Record, Idx);
-    QualType Canon = GetType(Record[Idx++]);
+    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(*Loc.F, Record, Idx);
+    const IdentifierInfo *Name = this->GetIdentifierInfo(*Loc.F, Record, Idx);
+    QualType Canon = readType(*Loc.F, Record, Idx);
     if (!Canon.isNull())
-      Canon = Context->getCanonicalType(Canon);
-    return Context->getDependentNameType(Keyword, NNS, Name, Canon);
+      Canon = Context.getCanonicalType(Canon);
+    return Context.getDependentNameType(Keyword, NNS, Name, Canon);
   }
 
   case TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
-    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
-    const IdentifierInfo *Name = this->GetIdentifierInfo(Record, Idx);
+    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(*Loc.F, Record, Idx);
+    const IdentifierInfo *Name = this->GetIdentifierInfo(*Loc.F, Record, Idx);
     unsigned NumArgs = Record[Idx++];
-    llvm::SmallVector<TemplateArgument, 8> Args;
+    SmallVector<TemplateArgument, 8> Args;
     Args.reserve(NumArgs);
     while (NumArgs--)
       Args.push_back(ReadTemplateArgument(*Loc.F, Record, Idx));
-    return Context->getDependentTemplateSpecializationType(Keyword, NNS, Name,
+    return Context.getDependentTemplateSpecializationType(Keyword, NNS, Name,
                                                       Args.size(), Args.data());
   }
 
@@ -3476,7 +3496,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     unsigned Idx = 0;
 
     // ArrayType
-    QualType ElementType = GetType(Record[Idx++]);
+    QualType ElementType = readType(*Loc.F, Record, Idx);
     ArrayType::ArraySizeModifier ASM
       = (ArrayType::ArraySizeModifier)Record[Idx++];
     unsigned IndexTypeQuals = Record[Idx++];
@@ -3485,7 +3505,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     Expr *NumElts = ReadExpr(*Loc.F);
     SourceRange Brackets = ReadSourceRange(*Loc.F, Record, Idx);
 
-    return Context->getDependentSizedArrayType(ElementType, NumElts, ASM,
+    return Context.getDependentSizedArrayType(ElementType, NumElts, ASM,
                                                IndexTypeQuals, Brackets);
   }
 
@@ -3493,19 +3513,28 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     unsigned Idx = 0;
     bool IsDependent = Record[Idx++];
     TemplateName Name = ReadTemplateName(*Loc.F, Record, Idx);
-    llvm::SmallVector<TemplateArgument, 8> Args;
+    SmallVector<TemplateArgument, 8> Args;
     ReadTemplateArgumentList(Args, *Loc.F, Record, Idx);
-    QualType Underlying = GetType(Record[Idx++]);
+    QualType Underlying = readType(*Loc.F, Record, Idx);
     QualType T;
     if (Underlying.isNull())
-      T = Context->getCanonicalTemplateSpecializationType(Name, Args.data(),
+      T = Context.getCanonicalTemplateSpecializationType(Name, Args.data(),
                                                           Args.size());
     else
-      T = Context->getTemplateSpecializationType(Name, Args.data(),
+      T = Context.getTemplateSpecializationType(Name, Args.data(),
                                                  Args.size(), Underlying);
     const_cast<Type*>(T.getTypePtr())->setDependent(IsDependent);
     return T;
   }
+
+  case TYPE_ATOMIC: {
+    if (Record.size() != 1) {
+      Error("Incorrect encoding of atomic type");
+      return QualType();
+    }
+    QualType ValueType = readType(*Loc.F, Record, Idx);
+    return Context.getAtomicType(ValueType);
+  }
   }
   // Suppress a GCC warning
   return QualType();
@@ -3513,7 +3542,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
 
 class clang::TypeLocReader : public TypeLocVisitor<TypeLocReader> {
   ASTReader &Reader;
-  ASTReader::PerFileData &F;
+  Module &F;
   llvm::BitstreamCursor &DeclsCursor;
   const ASTReader::RecordData &Record;
   unsigned &Idx;
@@ -3523,8 +3552,13 @@ class clang::TypeLocReader : public TypeLocVisitor<TypeLocReader> {
     return Reader.ReadSourceLocation(F, R, I);
   }
 
+  template<typename T>
+  T *ReadDeclAs(const ASTReader::RecordData &Record, unsigned &Idx) {
+    return Reader.ReadDeclAs<T>(F, Record, Idx);
+  }
+  
 public:
-  TypeLocReader(ASTReader &Reader, ASTReader::PerFileData &F,
+  TypeLocReader(ASTReader &Reader, Module &F,
                 const ASTReader::RecordData &Record, unsigned &Idx)
     : Reader(Reader), F(F), DeclsCursor(F.DeclsCursor), Record(Record), Idx(Idx)
   { }
@@ -3608,7 +3642,7 @@ void TypeLocReader::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
   TL.setLocalRangeEnd(ReadSourceLocation(Record, Idx));
   TL.setTrailingReturn(Record[Idx++]);
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i) {
-    TL.setArg(i, cast_or_null<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
+    TL.setArg(i, ReadDeclAs<ParmVarDecl>(Record, Idx));
   }
 }
 void TypeLocReader::VisitFunctionProtoTypeLoc(FunctionProtoTypeLoc TL) {
@@ -3735,15 +3769,20 @@ void TypeLocReader::VisitObjCObjectTypeLoc(ObjCObjectTypeLoc TL) {
 void TypeLocReader::VisitObjCObjectPointerTypeLoc(ObjCObjectPointerTypeLoc TL) {
   TL.setStarLoc(ReadSourceLocation(Record, Idx));
 }
+void TypeLocReader::VisitAtomicTypeLoc(AtomicTypeLoc TL) {
+  TL.setKWLoc(ReadSourceLocation(Record, Idx));
+  TL.setLParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setRParenLoc(ReadSourceLocation(Record, Idx));
+}
 
-TypeSourceInfo *ASTReader::GetTypeSourceInfo(PerFileData &F,
+TypeSourceInfo *ASTReader::GetTypeSourceInfo(Module &F,
                                              const RecordData &Record,
                                              unsigned &Idx) {
-  QualType InfoTy = GetType(Record[Idx++]);
+  QualType InfoTy = readType(F, Record, Idx);
   if (InfoTy.isNull())
     return 0;
 
-  TypeSourceInfo *TInfo = getContext()->CreateTypeSourceInfo(InfoTy);
+  TypeSourceInfo *TInfo = getContext().CreateTypeSourceInfo(InfoTy);
   TypeLocReader TLR(*this, F, Record, Idx);
   for (TypeLoc TL = TInfo->getTypeLoc(); !TL.isNull(); TL = TL.getNextTypeLoc())
     TLR.Visit(TL);
@@ -3758,41 +3797,47 @@ QualType ASTReader::GetType(TypeID ID) {
     QualType T;
     switch ((PredefinedTypeIDs)Index) {
     case PREDEF_TYPE_NULL_ID: return QualType();
-    case PREDEF_TYPE_VOID_ID: T = Context->VoidTy; break;
-    case PREDEF_TYPE_BOOL_ID: T = Context->BoolTy; break;
+    case PREDEF_TYPE_VOID_ID: T = Context.VoidTy; break;
+    case PREDEF_TYPE_BOOL_ID: T = Context.BoolTy; break;
 
     case PREDEF_TYPE_CHAR_U_ID:
     case PREDEF_TYPE_CHAR_S_ID:
       // FIXME: Check that the signedness of CharTy is correct!
-      T = Context->CharTy;
+      T = Context.CharTy;
       break;
 
-    case PREDEF_TYPE_UCHAR_ID:      T = Context->UnsignedCharTy;     break;
-    case PREDEF_TYPE_USHORT_ID:     T = Context->UnsignedShortTy;    break;
-    case PREDEF_TYPE_UINT_ID:       T = Context->UnsignedIntTy;      break;
-    case PREDEF_TYPE_ULONG_ID:      T = Context->UnsignedLongTy;     break;
-    case PREDEF_TYPE_ULONGLONG_ID:  T = Context->UnsignedLongLongTy; break;
-    case PREDEF_TYPE_UINT128_ID:    T = Context->UnsignedInt128Ty;   break;
-    case PREDEF_TYPE_SCHAR_ID:      T = Context->SignedCharTy;       break;
-    case PREDEF_TYPE_WCHAR_ID:      T = Context->WCharTy;            break;
-    case PREDEF_TYPE_SHORT_ID:      T = Context->ShortTy;            break;
-    case PREDEF_TYPE_INT_ID:        T = Context->IntTy;              break;
-    case PREDEF_TYPE_LONG_ID:       T = Context->LongTy;             break;
-    case PREDEF_TYPE_LONGLONG_ID:   T = Context->LongLongTy;         break;
-    case PREDEF_TYPE_INT128_ID:     T = Context->Int128Ty;           break;
-    case PREDEF_TYPE_FLOAT_ID:      T = Context->FloatTy;            break;
-    case PREDEF_TYPE_DOUBLE_ID:     T = Context->DoubleTy;           break;
-    case PREDEF_TYPE_LONGDOUBLE_ID: T = Context->LongDoubleTy;       break;
-    case PREDEF_TYPE_OVERLOAD_ID:   T = Context->OverloadTy;         break;
-    case PREDEF_TYPE_BOUND_MEMBER:  T = Context->BoundMemberTy;      break;
-    case PREDEF_TYPE_DEPENDENT_ID:  T = Context->DependentTy;        break;
-    case PREDEF_TYPE_UNKNOWN_ANY:   T = Context->UnknownAnyTy;       break;
-    case PREDEF_TYPE_NULLPTR_ID:    T = Context->NullPtrTy;          break;
-    case PREDEF_TYPE_CHAR16_ID:     T = Context->Char16Ty;           break;
-    case PREDEF_TYPE_CHAR32_ID:     T = Context->Char32Ty;           break;
-    case PREDEF_TYPE_OBJC_ID:       T = Context->ObjCBuiltinIdTy;    break;
-    case PREDEF_TYPE_OBJC_CLASS:    T = Context->ObjCBuiltinClassTy; break;
-    case PREDEF_TYPE_OBJC_SEL:      T = Context->ObjCBuiltinSelTy;   break;
+    case PREDEF_TYPE_UCHAR_ID:      T = Context.UnsignedCharTy;     break;
+    case PREDEF_TYPE_USHORT_ID:     T = Context.UnsignedShortTy;    break;
+    case PREDEF_TYPE_UINT_ID:       T = Context.UnsignedIntTy;      break;
+    case PREDEF_TYPE_ULONG_ID:      T = Context.UnsignedLongTy;     break;
+    case PREDEF_TYPE_ULONGLONG_ID:  T = Context.UnsignedLongLongTy; break;
+    case PREDEF_TYPE_UINT128_ID:    T = Context.UnsignedInt128Ty;   break;
+    case PREDEF_TYPE_SCHAR_ID:      T = Context.SignedCharTy;       break;
+    case PREDEF_TYPE_WCHAR_ID:      T = Context.WCharTy;            break;
+    case PREDEF_TYPE_SHORT_ID:      T = Context.ShortTy;            break;
+    case PREDEF_TYPE_INT_ID:        T = Context.IntTy;              break;
+    case PREDEF_TYPE_LONG_ID:       T = Context.LongTy;             break;
+    case PREDEF_TYPE_LONGLONG_ID:   T = Context.LongLongTy;         break;
+    case PREDEF_TYPE_INT128_ID:     T = Context.Int128Ty;           break;
+    case PREDEF_TYPE_HALF_ID:       T = Context.HalfTy;             break;
+    case PREDEF_TYPE_FLOAT_ID:      T = Context.FloatTy;            break;
+    case PREDEF_TYPE_DOUBLE_ID:     T = Context.DoubleTy;           break;
+    case PREDEF_TYPE_LONGDOUBLE_ID: T = Context.LongDoubleTy;       break;
+    case PREDEF_TYPE_OVERLOAD_ID:   T = Context.OverloadTy;         break;
+    case PREDEF_TYPE_BOUND_MEMBER:  T = Context.BoundMemberTy;      break;
+    case PREDEF_TYPE_DEPENDENT_ID:  T = Context.DependentTy;        break;
+    case PREDEF_TYPE_UNKNOWN_ANY:   T = Context.UnknownAnyTy;       break;
+    case PREDEF_TYPE_NULLPTR_ID:    T = Context.NullPtrTy;          break;
+    case PREDEF_TYPE_CHAR16_ID:     T = Context.Char16Ty;           break;
+    case PREDEF_TYPE_CHAR32_ID:     T = Context.Char32Ty;           break;
+    case PREDEF_TYPE_OBJC_ID:       T = Context.ObjCBuiltinIdTy;    break;
+    case PREDEF_TYPE_OBJC_CLASS:    T = Context.ObjCBuiltinClassTy; break;
+    case PREDEF_TYPE_OBJC_SEL:      T = Context.ObjCBuiltinSelTy;   break;
+    case PREDEF_TYPE_AUTO_DEDUCT:   T = Context.getAutoDeductType(); break;
+        
+    case PREDEF_TYPE_AUTO_RREF_DEDUCT: 
+      T = Context.getAutoRRefDeductType(); 
+      break;
     }
 
     assert(!T.isNull() && "Unknown predefined type");
@@ -3802,12 +3847,11 @@ QualType ASTReader::GetType(TypeID ID) {
   Index -= NUM_PREDEF_TYPE_IDS;
   assert(Index < TypesLoaded.size() && "Type index out-of-range");
   if (TypesLoaded[Index].isNull()) {
-    TypesLoaded[Index] = ReadTypeRecord(Index);
+    TypesLoaded[Index] = readTypeRecord(Index);
     if (TypesLoaded[Index].isNull())
       return QualType();
 
     TypesLoaded[Index]->setFromAST();
-    TypeIdxs[TypesLoaded[Index]] = TypeIdx::fromTypeID(ID);
     if (DeserializationListener)
       DeserializationListener->TypeRead(TypeIdx::fromTypeID(ID),
                                         TypesLoaded[Index]);
@@ -3816,37 +3860,28 @@ QualType ASTReader::GetType(TypeID ID) {
   return TypesLoaded[Index].withFastQualifiers(FastQuals);
 }
 
-TypeID ASTReader::GetTypeID(QualType T) const {
-  return MakeTypeID(T,
-              std::bind1st(std::mem_fun(&ASTReader::GetTypeIdx), this));
+QualType ASTReader::getLocalType(Module &F, unsigned LocalID) {
+  return GetType(getGlobalTypeID(F, LocalID));
 }
 
-TypeIdx ASTReader::GetTypeIdx(QualType T) const {
-  if (T.isNull())
-    return TypeIdx();
-  assert(!T.getLocalFastQualifiers());
-
-  TypeIdxMap::const_iterator I = TypeIdxs.find(T);
-  // GetTypeIdx is mostly used for computing the hash of DeclarationNames and
-  // comparing keys of ASTDeclContextNameLookupTable.
-  // If the type didn't come from the AST file use a specially marked index
-  // so that any hash/key comparison fail since no such index is stored
-  // in a AST file.
-  if (I == TypeIdxs.end())
-    return TypeIdx(-1);
-  return I->second;
-}
-
-unsigned ASTReader::getTotalNumCXXBaseSpecifiers() const {
-  unsigned Result = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I)
-    Result += Chain[I]->LocalNumCXXBaseSpecifiers;
+serialization::TypeID 
+ASTReader::getGlobalTypeID(Module &F, unsigned LocalID) const {
+  unsigned FastQuals = LocalID & Qualifiers::FastMask;
+  unsigned LocalIndex = LocalID >> Qualifiers::FastWidth;
   
-  return Result;
+  if (LocalIndex < NUM_PREDEF_TYPE_IDS)
+    return LocalID;
+
+  ContinuousRangeMap<uint32_t, int, 2>::iterator I
+    = F.TypeRemap.find(LocalIndex - NUM_PREDEF_TYPE_IDS);
+  assert(I != F.TypeRemap.end() && "Invalid index into type index remap");
+  
+  unsigned GlobalIndex = LocalIndex + I->second;
+  return (GlobalIndex << Qualifiers::FastWidth) | FastQuals;
 }
 
 TemplateArgumentLocInfo
-ASTReader::GetTemplateArgumentLocInfo(PerFileData &F,
+ASTReader::GetTemplateArgumentLocInfo(Module &F,
                                       TemplateArgument::ArgKind Kind,
                                       const RecordData &Record,
                                       unsigned &Index) {
@@ -3881,7 +3916,7 @@ ASTReader::GetTemplateArgumentLocInfo(PerFileData &F,
 }
 
 TemplateArgumentLoc
-ASTReader::ReadTemplateArgumentLoc(PerFileData &F,
+ASTReader::ReadTemplateArgumentLoc(Module &F,
                                    const RecordData &Record, unsigned &Index) {
   TemplateArgument Arg = ReadTemplateArgument(F, Record, Index);
 
@@ -3897,47 +3932,20 @@ Decl *ASTReader::GetExternalDecl(uint32_t ID) {
   return GetDecl(ID);
 }
 
-uint64_t 
-ASTReader::GetCXXBaseSpecifiersOffset(serialization::CXXBaseSpecifiersID ID) {
-  if (ID == 0)
+uint64_t ASTReader::readCXXBaseSpecifiers(Module &M, const RecordData &Record, 
+                                          unsigned &Idx){
+  if (Idx >= Record.size())
     return 0;
   
-  --ID;
-  uint64_t Offset = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    PerFileData &F = *Chain[N - I - 1];
-
-    if (ID < F.LocalNumCXXBaseSpecifiers)
-      return Offset + F.CXXBaseSpecifiersOffsets[ID];
-    
-    ID -= F.LocalNumCXXBaseSpecifiers;
-    Offset += F.SizeInBits;
-  }
-  
-  assert(false && "CXXBaseSpecifiers not found");
-  return 0;
+  unsigned LocalID = Record[Idx++];
+  return getGlobalBitOffset(M, M.CXXBaseSpecifiersOffsets[LocalID - 1]);
 }
 
 CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
-  // Figure out which AST file contains this offset.
-  PerFileData *F = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    if (Offset < Chain[N - I - 1]->SizeInBits) {
-      F = Chain[N - I - 1];
-      break;
-    }
-    
-    Offset -= Chain[N - I - 1]->SizeInBits;
-  }
-
-  if (!F) {
-    Error("Malformed AST file: C++ base specifiers at impossible offset");
-    return 0;
-  }
-  
-  llvm::BitstreamCursor &Cursor = F->DeclsCursor;
+  RecordLocation Loc = getLocalBitOffset(Offset);
+  llvm::BitstreamCursor &Cursor = Loc.F->DeclsCursor;
   SavedStreamPosition SavedPosition(Cursor);
-  Cursor.JumpToBit(Offset);
+  Cursor.JumpToBit(Loc.Offset);
   ReadingKindTracker ReadingKind(Read_Decl, *this);
   RecordData Record;
   unsigned Code = Cursor.ReadCode();
@@ -3949,35 +3957,72 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
 
   unsigned Idx = 0;
   unsigned NumBases = Record[Idx++];
-  void *Mem = Context->Allocate(sizeof(CXXBaseSpecifier) * NumBases);
+  void *Mem = Context.Allocate(sizeof(CXXBaseSpecifier) * NumBases);
   CXXBaseSpecifier *Bases = new (Mem) CXXBaseSpecifier [NumBases];
   for (unsigned I = 0; I != NumBases; ++I)
-    Bases[I] = ReadCXXBaseSpecifier(*F, Record, Idx);
+    Bases[I] = ReadCXXBaseSpecifier(*Loc.F, Record, Idx);
   return Bases;
 }
 
-TranslationUnitDecl *ASTReader::GetTranslationUnitDecl() {
-  if (!DeclsLoaded[0]) {
-    ReadDeclRecord(0, 1);
-    if (DeserializationListener)
-      DeserializationListener->DeclRead(1, DeclsLoaded[0]);
-  }
+serialization::DeclID 
+ASTReader::getGlobalDeclID(Module &F, unsigned LocalID) const {
+  if (LocalID < NUM_PREDEF_DECL_IDS)
+    return LocalID;
 
-  return cast<TranslationUnitDecl>(DeclsLoaded[0]);
+  ContinuousRangeMap<uint32_t, int, 2>::iterator I
+    = F.DeclRemap.find(LocalID - NUM_PREDEF_DECL_IDS);
+  assert(I != F.DeclRemap.end() && "Invalid index into decl index remap");
+  
+  return LocalID + I->second;
+}
+
+bool ASTReader::isDeclIDFromModule(serialization::GlobalDeclID ID,
+                                   Module &M) const {
+  GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(ID);
+  assert(I != GlobalDeclMap.end() && "Corrupted global declaration map");
+  return &M == I->second;
 }
 
 Decl *ASTReader::GetDecl(DeclID ID) {
-  if (ID == 0)
-    return 0;
+  if (ID < NUM_PREDEF_DECL_IDS) {    
+    switch ((PredefinedDeclIDs)ID) {
+    case PREDEF_DECL_NULL_ID:
+      return 0;
+        
+    case PREDEF_DECL_TRANSLATION_UNIT_ID:
+      return Context.getTranslationUnitDecl();
+        
+    case PREDEF_DECL_OBJC_ID_ID:
+      return Context.getObjCIdDecl();
 
-  if (ID > DeclsLoaded.size()) {
+    case PREDEF_DECL_OBJC_SEL_ID:
+      return Context.getObjCSelDecl();
+
+    case PREDEF_DECL_OBJC_CLASS_ID:
+      return Context.getObjCClassDecl();
+        
+    case PREDEF_DECL_INT_128_ID:
+      return Context.getInt128Decl();
+
+    case PREDEF_DECL_UNSIGNED_INT_128_ID:
+      return Context.getUInt128Decl();
+        
+    case PREDEF_DECL_OBJC_INSTANCETYPE_ID:
+      return Context.getObjCInstanceTypeDecl();
+    }
+    
+    return 0;
+  }
+  
+  unsigned Index = ID - NUM_PREDEF_DECL_IDS;
+
+  if (Index > DeclsLoaded.size()) {
     Error("declaration ID out-of-range for AST file");
     return 0;
   }
-
-  unsigned Index = ID - 1;
-  if (!DeclsLoaded[Index]) {
-    ReadDeclRecord(Index, ID);
+  
+if (!DeclsLoaded[Index]) {
+    ReadDeclRecord(ID);
     if (DeserializationListener)
       DeserializationListener->DeclRead(ID, DeclsLoaded[Index]);
   }
@@ -3985,6 +4030,17 @@ Decl *ASTReader::GetDecl(DeclID ID) {
   return DeclsLoaded[Index];
 }
 
+serialization::DeclID ASTReader::ReadDeclID(Module &F, 
+                                            const RecordData &Record,
+                                            unsigned &Idx) {
+  if (Idx >= Record.size()) {
+    Error("Corrupted AST file");
+    return 0;
+  }
+  
+  return getGlobalDeclID(F, Record[Idx++]);
+}
+
 /// \brief Resolve the offset of a statement into a statement.
 ///
 /// This operation will read a new statement from the external
@@ -3995,49 +4051,137 @@ Stmt *ASTReader::GetExternalDeclStmt(uint64_t Offset) {
   ClearSwitchCaseIDs();
 
   // Offset here is a global offset across the entire chain.
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    PerFileData &F = *Chain[N - I - 1];
-    if (Offset < F.SizeInBits) {
-      // Since we know that this statement is part of a decl, make sure to use
-      // the decl cursor to read it.
-      F.DeclsCursor.JumpToBit(Offset);
-      return ReadStmtFromStream(F);
+  RecordLocation Loc = getLocalBitOffset(Offset);
+  Loc.F->DeclsCursor.JumpToBit(Loc.Offset);
+  return ReadStmtFromStream(*Loc.F);
+}
+
+namespace {
+  class FindExternalLexicalDeclsVisitor {
+    ASTReader &Reader;
+    const DeclContext *DC;
+    bool (*isKindWeWant)(Decl::Kind);
+    
+    SmallVectorImpl<Decl*> &Decls;
+    bool PredefsVisited[NUM_PREDEF_DECL_IDS];
+
+  public:
+    FindExternalLexicalDeclsVisitor(ASTReader &Reader, const DeclContext *DC,
+                                    bool (*isKindWeWant)(Decl::Kind),
+                                    SmallVectorImpl<Decl*> &Decls)
+      : Reader(Reader), DC(DC), isKindWeWant(isKindWeWant), Decls(Decls) 
+    {
+      for (unsigned I = 0; I != NUM_PREDEF_DECL_IDS; ++I)
+        PredefsVisited[I] = false;
     }
-    Offset -= F.SizeInBits;
-  }
-  llvm_unreachable("Broken chain");
+
+    static bool visit(Module &M, bool Preorder, void *UserData) {
+      if (Preorder)
+        return false;
+
+      FindExternalLexicalDeclsVisitor *This
+        = static_cast<FindExternalLexicalDeclsVisitor *>(UserData);
+
+      Module::DeclContextInfosMap::iterator Info
+        = M.DeclContextInfos.find(This->DC);
+      if (Info == M.DeclContextInfos.end() || !Info->second.LexicalDecls)
+        return false;
+
+      // Load all of the declaration IDs
+      for (const KindDeclIDPair *ID = Info->second.LexicalDecls,
+                               *IDE = ID + Info->second.NumLexicalDecls; 
+           ID != IDE; ++ID) {
+        if (This->isKindWeWant && !This->isKindWeWant((Decl::Kind)ID->first))
+          continue;
+
+        // Don't add predefined declarations to the lexical context more
+        // than once.
+        if (ID->second < NUM_PREDEF_DECL_IDS) {
+          if (This->PredefsVisited[ID->second])
+            continue;
+
+          This->PredefsVisited[ID->second] = true;
+        }
+
+        if (Decl *D = This->Reader.GetLocalDecl(M, ID->second)) {
+          if (!This->DC->isDeclInLexicalTraversal(D))
+            This->Decls.push_back(D);
+        }
+      }
+
+      return false;
+    }
+  };
 }
 
 ExternalLoadResult ASTReader::FindExternalLexicalDecls(const DeclContext *DC,
                                          bool (*isKindWeWant)(Decl::Kind),
-                                         llvm::SmallVectorImpl<Decl*> &Decls) {
-  // There might be lexical decls in multiple parts of the chain, for the TU
-  // at least.
-  // DeclContextOffsets might reallocate as we load additional decls below,
-  // so make a copy of the vector.
-  DeclContextInfos Infos = DeclContextOffsets[DC];
-  for (DeclContextInfos::iterator I = Infos.begin(), E = Infos.end();
-       I != E; ++I) {
-    // IDs can be 0 if this context doesn't contain declarations.
-    if (!I->LexicalDecls)
-      continue;
-
-    // Load all of the declaration IDs
-    for (const KindDeclIDPair *ID = I->LexicalDecls,
-                              *IDE = ID + I->NumLexicalDecls; ID != IDE; ++ID) {
-      if (isKindWeWant && !isKindWeWant((Decl::Kind)ID->first))
-        continue;
-
-      Decl *D = GetDecl(ID->second);
-      assert(D && "Null decl in lexical decls");
-      Decls.push_back(D);
-    }
-  }
-
+                                         SmallVectorImpl<Decl*> &Decls) {
+  // There might be lexical decls in multiple modules, for the TU at
+  // least. Walk all of the modules in the order they were loaded.
+  FindExternalLexicalDeclsVisitor Visitor(*this, DC, isKindWeWant, Decls);
+  ModuleMgr.visitDepthFirst(&FindExternalLexicalDeclsVisitor::visit, &Visitor);
   ++NumLexicalDeclContextsRead;
   return ELR_Success;
 }
 
+namespace {
+  /// \brief Module visitor used to perform name lookup into a
+  /// declaration context.
+  class DeclContextNameLookupVisitor {
+    ASTReader &Reader;
+    const DeclContext *DC;
+    DeclarationName Name;
+    SmallVectorImpl<NamedDecl *> &Decls;
+
+  public:
+    DeclContextNameLookupVisitor(ASTReader &Reader, 
+                                 const DeclContext *DC, DeclarationName Name,
+                                 SmallVectorImpl<NamedDecl *> &Decls)
+      : Reader(Reader), DC(DC), Name(Name), Decls(Decls) { }
+
+    static bool visit(Module &M, void *UserData) {
+      DeclContextNameLookupVisitor *This
+        = static_cast<DeclContextNameLookupVisitor *>(UserData);
+
+      // Check whether we have any visible declaration information for
+      // this context in this module.
+      Module::DeclContextInfosMap::iterator Info
+        = M.DeclContextInfos.find(This->DC);
+      if (Info == M.DeclContextInfos.end() || !Info->second.NameLookupTableData)
+        return false;
+
+      // Look for this name within this module.
+      ASTDeclContextNameLookupTable *LookupTable =
+        (ASTDeclContextNameLookupTable*)Info->second.NameLookupTableData;
+      ASTDeclContextNameLookupTable::iterator Pos
+        = LookupTable->find(This->Name);
+      if (Pos == LookupTable->end())
+        return false;
+
+      bool FoundAnything = false;
+      ASTDeclContextNameLookupTrait::data_type Data = *Pos;
+      for (; Data.first != Data.second; ++Data.first) {
+        NamedDecl *ND = This->Reader.GetLocalDeclAs<NamedDecl>(M, *Data.first);
+        if (!ND)
+          continue;
+
+        if (ND->getDeclName() != This->Name) {
+          assert(!This->Name.getCXXNameType().isNull() && 
+                 "Name mismatch without a type");
+          continue;
+        }
+      
+        // Record this declaration.
+        FoundAnything = true;
+        This->Decls.push_back(ND);
+      }
+
+      return FoundAnything;
+    }
+  };
+}
+
 DeclContext::lookup_result
 ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
                                           DeclarationName Name) {
@@ -4047,69 +4191,39 @@ ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
     return DeclContext::lookup_result(DeclContext::lookup_iterator(0),
                                       DeclContext::lookup_iterator(0));
 
-  llvm::SmallVector<NamedDecl *, 64> Decls;
-  // There might be visible decls in multiple parts of the chain, for the TU
-  // and namespaces. For any given name, the last available results replace
-  // all earlier ones. For this reason, we walk in reverse.
-  DeclContextInfos &Infos = DeclContextOffsets[DC];
-  for (DeclContextInfos::reverse_iterator I = Infos.rbegin(), E = Infos.rend();
-       I != E; ++I) {
-    if (!I->NameLookupTableData)
-      continue;
-
-    ASTDeclContextNameLookupTable *LookupTable =
-        (ASTDeclContextNameLookupTable*)I->NameLookupTableData;
-    ASTDeclContextNameLookupTable::iterator Pos = LookupTable->find(Name);
-    if (Pos == LookupTable->end())
-      continue;
-
-    ASTDeclContextNameLookupTrait::data_type Data = *Pos;
-    for (; Data.first != Data.second; ++Data.first)
-      Decls.push_back(cast<NamedDecl>(GetDecl(*Data.first)));
-    break;
-  }
-
+  SmallVector<NamedDecl *, 64> Decls;
+  DeclContextNameLookupVisitor Visitor(*this, DC, Name, Decls);
+  ModuleMgr.visit(&DeclContextNameLookupVisitor::visit, &Visitor);
   ++NumVisibleDeclContextsRead;
-
   SetExternalVisibleDeclsForName(DC, Name, Decls);
   return const_cast<DeclContext*>(DC)->lookup(Name);
 }
 
-void ASTReader::MaterializeVisibleDecls(const DeclContext *DC) {
-  assert(DC->hasExternalVisibleStorage() &&
-         "DeclContext has no visible decls in storage");
+/// \brief Under non-PCH compilation the consumer receives the objc methods
+/// before receiving the implementation, and codegen depends on this.
+/// We simulate this by deserializing and passing to consumer the methods of the
+/// implementation before passing the deserialized implementation decl.
+static void PassObjCImplDeclToConsumer(ObjCImplDecl *ImplD,
+                                       ASTConsumer *Consumer) {
+  assert(ImplD && Consumer);
 
-  llvm::SmallVector<NamedDecl *, 64> Decls;
-  // There might be visible decls in multiple parts of the chain, for the TU
-  // and namespaces.
-  DeclContextInfos &Infos = DeclContextOffsets[DC];
-  for (DeclContextInfos::iterator I = Infos.begin(), E = Infos.end();
-       I != E; ++I) {
-    if (!I->NameLookupTableData)
-      continue;
+  for (ObjCImplDecl::method_iterator
+         I = ImplD->meth_begin(), E = ImplD->meth_end(); I != E; ++I)
+    Consumer->HandleInterestingDecl(DeclGroupRef(*I));
 
-    ASTDeclContextNameLookupTable *LookupTable =
-        (ASTDeclContextNameLookupTable*)I->NameLookupTableData;
-    for (ASTDeclContextNameLookupTable::item_iterator
-           ItemI = LookupTable->item_begin(),
-           ItemEnd = LookupTable->item_end() ; ItemI != ItemEnd; ++ItemI) {
-      ASTDeclContextNameLookupTable::item_iterator::value_type Val
-          = *ItemI;
-      ASTDeclContextNameLookupTrait::data_type Data = Val.second;
-      Decls.clear();
-      for (; Data.first != Data.second; ++Data.first)
-        Decls.push_back(cast<NamedDecl>(GetDecl(*Data.first)));
-      MaterializeVisibleDeclsForName(DC, Val.first, Decls);
-    }
-  }
+  Consumer->HandleInterestingDecl(DeclGroupRef(ImplD));
 }
 
 void ASTReader::PassInterestingDeclsToConsumer() {
   assert(Consumer);
   while (!InterestingDecls.empty()) {
-    DeclGroupRef DG(InterestingDecls.front());
+    Decl *D = InterestingDecls.front();
     InterestingDecls.pop_front();
-    Consumer->HandleInterestingDecl(DG);
+
+    if (ObjCImplDecl *ImplD = dyn_cast<ObjCImplDecl>(D))
+      PassObjCImplDeclToConsumer(ImplD, Consumer);
+    else
+      Consumer->HandleInterestingDecl(DeclGroupRef(D));
   }
 }
 
@@ -4124,6 +4238,7 @@ void ASTReader::StartTranslationUnit(ASTConsumer *Consumer) {
     // passing to the consumer.
     GetDecl(ExternalDefinitions[I]);
   }
+  ExternalDefinitions.clear();
 
   PassInterestingDeclsToConsumer();
 }
@@ -4148,7 +4263,7 @@ void ASTReader::PrintStats() {
 
   std::fprintf(stderr, "  %u stat cache hits\n", NumStatHits);
   std::fprintf(stderr, "  %u stat cache misses\n", NumStatMisses);
-  if (TotalNumSLocEntries)
+  if (unsigned TotalNumSLocEntries = getTotalNumSLocs())
     std::fprintf(stderr, "  %u/%u source location entries read (%f%%)\n",
                  NumSLocEntriesRead, TotalNumSLocEntries,
                  ((float)NumSLocEntriesRead/TotalNumSLocEntries * 100));
@@ -4194,13 +4309,51 @@ void ASTReader::PrintStats() {
     std::fprintf(stderr, "  %u method pool misses\n", NumMethodPoolMisses);
   }
   std::fprintf(stderr, "\n");
+  dump();
+  std::fprintf(stderr, "\n");
+}
+
+template<typename Key, typename Module, unsigned InitialCapacity>
+static void 
+dumpModuleIDMap(StringRef Name,
+                const ContinuousRangeMap<Key, Module *, 
+                                         InitialCapacity> &Map) {
+  if (Map.begin() == Map.end())
+    return;
+  
+  typedef ContinuousRangeMap<Key, Module *, InitialCapacity> MapType;
+  llvm::errs() << Name << ":\n";
+  for (typename MapType::const_iterator I = Map.begin(), IEnd = Map.end(); 
+       I != IEnd; ++I) {
+    llvm::errs() << "  " << I->first << " -> " << I->second->FileName
+      << "\n";
+  }
+}
+
+void ASTReader::dump() {
+  llvm::errs() << "*** PCH/Module Remappings:\n";
+  dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap);
+  dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap);
+  dumpModuleIDMap("Global type map", GlobalTypeMap);
+  dumpModuleIDMap("Global declaration map", GlobalDeclMap);
+  dumpModuleIDMap("Global identifier map", GlobalIdentifierMap);
+  dumpModuleIDMap("Global selector map", GlobalSelectorMap);
+  dumpModuleIDMap("Global preprocessed entity map", 
+                  GlobalPreprocessedEntityMap);
+  
+  llvm::errs() << "\n*** PCH/Modules Loaded:";
+  for (ModuleManager::ModuleConstIterator M = ModuleMgr.begin(), 
+                                       MEnd = ModuleMgr.end();
+       M != MEnd; ++M)
+    (*M)->dump();
 }
 
 /// Return the amount of memory used by memory buffers, breaking down
 /// by heap-backed versus mmap'ed memory.
 void ASTReader::getMemoryBufferSizes(MemoryBufferSizes &sizes) const {
-  for (unsigned i = 0, e = Chain.size(); i != e; ++i)
-    if (llvm::MemoryBuffer *buf = Chain[i]->Buffer.get()) {
+  for (ModuleConstIterator I = ModuleMgr.begin(),
+      E = ModuleMgr.end(); I != E; ++I) {
+    if (llvm::MemoryBuffer *buf = (*I)->Buffer.get()) {
       size_t bytes = buf->getBufferSize();
       switch (buf->getBufferKind()) {
         case llvm::MemoryBuffer::MemoryBuffer_Malloc:
@@ -4211,6 +4364,7 @@ void ASTReader::getMemoryBufferSizes(MemoryBufferSizes &sizes) const {
           break;
       }
     }
+  }
 }
 
 void ASTReader::InitializeSema(Sema &S) {
@@ -4227,114 +4381,14 @@ void ASTReader::InitializeSema(Sema &S) {
   }
   PreloadedDecls.clear();
 
-  // If there were any tentative definitions, deserialize them and add
-  // them to Sema's list of tentative definitions.
-  for (unsigned I = 0, N = TentativeDefinitions.size(); I != N; ++I) {
-    VarDecl *Var = cast<VarDecl>(GetDecl(TentativeDefinitions[I]));
-    SemaObj->TentativeDefinitions.push_back(Var);
-  }
-
-  // If there were any unused file scoped decls, deserialize them and add to
-  // Sema's list of unused file scoped decls.
-  for (unsigned I = 0, N = UnusedFileScopedDecls.size(); I != N; ++I) {
-    DeclaratorDecl *D = cast<DeclaratorDecl>(GetDecl(UnusedFileScopedDecls[I]));
-    SemaObj->UnusedFileScopedDecls.push_back(D);
-  }
-
-  // If there were any delegating constructors, add them to Sema's list
-  for (unsigned I = 0, N = DelegatingCtorDecls.size(); I != N; ++I) {
-    CXXConstructorDecl *D
-     = cast<CXXConstructorDecl>(GetDecl(DelegatingCtorDecls[I]));
-    SemaObj->DelegatingCtorDecls.push_back(D);
-  }
-
-  // If there were any locally-scoped external declarations,
-  // deserialize them and add them to Sema's table of locally-scoped
-  // external declarations.
-  for (unsigned I = 0, N = LocallyScopedExternalDecls.size(); I != N; ++I) {
-    NamedDecl *D = cast<NamedDecl>(GetDecl(LocallyScopedExternalDecls[I]));
-    SemaObj->LocallyScopedExternalDecls[D->getDeclName()] = D;
-  }
-
-  // If there were any ext_vector type declarations, deserialize them
-  // and add them to Sema's vector of such declarations.
-  for (unsigned I = 0, N = ExtVectorDecls.size(); I != N; ++I)
-    SemaObj->ExtVectorDecls.push_back(
-                             cast<TypedefNameDecl>(GetDecl(ExtVectorDecls[I])));
-
-  // FIXME: Do VTable uses and dynamic classes deserialize too much ?
-  // Can we cut them down before writing them ?
-
-  // If there were any dynamic classes declarations, deserialize them
-  // and add them to Sema's vector of such declarations.
-  for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I)
-    SemaObj->DynamicClasses.push_back(
-                               cast<CXXRecordDecl>(GetDecl(DynamicClasses[I])));
-
   // Load the offsets of the declarations that Sema references.
   // They will be lazily deserialized when needed.
   if (!SemaDeclRefs.empty()) {
     assert(SemaDeclRefs.size() == 2 && "More decl refs than expected!");
-    SemaObj->StdNamespace = SemaDeclRefs[0];
-    SemaObj->StdBadAlloc = SemaDeclRefs[1];
-  }
-
-  for (PerFileData *F = FirstInSource; F; F = F->NextInSource) {
-
-    // If there are @selector references added them to its pool. This is for
-    // implementation of -Wselector.
-    if (!F->ReferencedSelectorsData.empty()) {
-      unsigned int DataSize = F->ReferencedSelectorsData.size()-1;
-      unsigned I = 0;
-      while (I < DataSize) {
-        Selector Sel = DecodeSelector(F->ReferencedSelectorsData[I++]);
-        SourceLocation SelLoc = ReadSourceLocation(
-                                    *F, F->ReferencedSelectorsData, I);
-        SemaObj->ReferencedSelectors.insert(std::make_pair(Sel, SelLoc));
-      }
-    }
-  }
-
-  // The special data sets below always come from the most recent PCH,
-  // which is at the front of the chain.
-  PerFileData &F = *Chain.front();
-
-  // If there were any pending implicit instantiations, deserialize them
-  // and add them to Sema's queue of such instantiations.
-  assert(F.PendingInstantiations.size() % 2 == 0 &&
-         "Expected pairs of entries");
-  for (unsigned Idx = 0, N = F.PendingInstantiations.size(); Idx < N;) {
-    ValueDecl *D=cast<ValueDecl>(GetDecl(F.PendingInstantiations[Idx++]));
-    SourceLocation Loc = ReadSourceLocation(F, F.PendingInstantiations,Idx);
-    SemaObj->PendingInstantiations.push_back(std::make_pair(D, Loc));
-  }
-
-  // If there were any weak undeclared identifiers, deserialize them and add to
-  // Sema's list of weak undeclared identifiers.
-  if (!WeakUndeclaredIdentifiers.empty()) {
-    unsigned Idx = 0;
-    for (unsigned I = 0, N = WeakUndeclaredIdentifiers[Idx++]; I != N; ++I) {
-      IdentifierInfo *WeakId = GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
-      IdentifierInfo *AliasId= GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
-      SourceLocation Loc = ReadSourceLocation(F, WeakUndeclaredIdentifiers,Idx);
-      bool Used = WeakUndeclaredIdentifiers[Idx++];
-      Sema::WeakInfo WI(AliasId, Loc);
-      WI.setUsed(Used);
-      SemaObj->WeakUndeclaredIdentifiers.insert(std::make_pair(WeakId, WI));
-    }
-  }
-
-  // If there were any VTable uses, deserialize the information and add it
-  // to Sema's vector and map of VTable uses.
-  if (!VTableUses.empty()) {
-    unsigned Idx = 0;
-    for (unsigned I = 0, N = VTableUses[Idx++]; I != N; ++I) {
-      CXXRecordDecl *Class = cast<CXXRecordDecl>(GetDecl(VTableUses[Idx++]));
-      SourceLocation Loc = ReadSourceLocation(F, VTableUses, Idx);
-      bool DefinitionRequired = VTableUses[Idx++];
-      SemaObj->VTableUses.push_back(std::make_pair(Class, Loc));
-      SemaObj->VTablesUsed[Class] = DefinitionRequired;
-    }
+    if (!SemaObj->StdNamespace)
+      SemaObj->StdNamespace = SemaDeclRefs[0];
+    if (!SemaObj->StdBadAlloc)
+      SemaObj->StdBadAlloc = SemaDeclRefs[1];
   }
 
   if (!FPPragmaOptions.empty()) {
@@ -4352,24 +4406,9 @@ void ASTReader::InitializeSema(Sema &S) {
 }
 
 IdentifierInfo* ASTReader::get(const char *NameStart, const char *NameEnd) {
-  // Try to find this name within our on-disk hash tables. We start with the
-  // most recent one, since that one contains the most up-to-date info.
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    ASTIdentifierLookupTable *IdTable
-        = (ASTIdentifierLookupTable *)Chain[I]->IdentifierLookupTable;
-    if (!IdTable)
-      continue;
-    std::pair<const char*, unsigned> Key(NameStart, NameEnd - NameStart);
-    ASTIdentifierLookupTable::iterator Pos = IdTable->find(Key);
-    if (Pos == IdTable->end())
-      continue;
-
-    // Dereferencing the iterator has the effect of building the
-    // IdentifierInfo node and populating it with the various
-    // declarations it needs.
-    return *Pos;
-  }
-  return 0;
+  IdentifierLookupVisitor Visitor(StringRef(NameStart, NameEnd - NameStart));
+  ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor);
+  return Visitor.getIdentifierInfo();
 }
 
 namespace clang {
@@ -4394,27 +4433,28 @@ namespace clang {
   public:
     explicit ASTIdentifierIterator(const ASTReader &Reader);
 
-    virtual llvm::StringRef Next();
+    virtual StringRef Next();
   };
 }
 
 ASTIdentifierIterator::ASTIdentifierIterator(const ASTReader &Reader)
-  : Reader(Reader), Index(Reader.Chain.size() - 1) {
+  : Reader(Reader), Index(Reader.ModuleMgr.size() - 1) {
   ASTIdentifierLookupTable *IdTable
-    = (ASTIdentifierLookupTable *)Reader.Chain[Index]->IdentifierLookupTable;
+    = (ASTIdentifierLookupTable *)Reader.ModuleMgr[Index].IdentifierLookupTable;
   Current = IdTable->key_begin();
   End = IdTable->key_end();
 }
 
-llvm::StringRef ASTIdentifierIterator::Next() {
+StringRef ASTIdentifierIterator::Next() {
   while (Current == End) {
     // If we have exhausted all of our AST files, we're done.
     if (Index == 0)
-      return llvm::StringRef();
+      return StringRef();
 
     --Index;
     ASTIdentifierLookupTable *IdTable
-      = (ASTIdentifierLookupTable *)Reader.Chain[Index]->IdentifierLookupTable;
+      = (ASTIdentifierLookupTable *)Reader.ModuleMgr[Index].
+        IdentifierLookupTable;
     Current = IdTable->key_begin();
     End = IdTable->key_end();
   }
@@ -4423,43 +4463,103 @@ llvm::StringRef ASTIdentifierIterator::Next() {
   // the next one.
   std::pair<const char*, unsigned> Key = *Current;
   ++Current;
-  return llvm::StringRef(Key.first, Key.second);
+  return StringRef(Key.first, Key.second);
 }
 
 IdentifierIterator *ASTReader::getIdentifiers() const {
   return new ASTIdentifierIterator(*this);
 }
 
-std::pair<ObjCMethodList, ObjCMethodList>
-ASTReader::ReadMethodPool(Selector Sel) {
-  // Find this selector in a hash table. We want to find the most recent entry.
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    PerFileData &F = *Chain[I];
-    if (!F.SelectorLookupTable)
-      continue;
+namespace clang { namespace serialization {
+  class ReadMethodPoolVisitor {
+    ASTReader &Reader;
+    Selector Sel;    
+    llvm::SmallVector<ObjCMethodDecl *, 4> InstanceMethods;
+    llvm::SmallVector<ObjCMethodDecl *, 4> FactoryMethods;
 
-    ASTSelectorLookupTable *PoolTable
-      = (ASTSelectorLookupTable*)F.SelectorLookupTable;
-    ASTSelectorLookupTable::iterator Pos = PoolTable->find(Sel);
-    if (Pos != PoolTable->end()) {
-      ++NumSelectorsRead;
+    /// \brief Build an ObjCMethodList from a vector of Objective-C method
+    /// declarations.
+    ObjCMethodList 
+    buildObjCMethodList(const SmallVectorImpl<ObjCMethodDecl *> &Vec) const
+    {
+      ObjCMethodList List;
+      ObjCMethodList *Prev = 0;
+      for (unsigned I = 0, N = Vec.size(); I != N; ++I) {
+        if (!List.Method) {
+          // This is the first method, which is the easy case.
+          List.Method = Vec[I];
+          Prev = &List;
+          continue;
+        }
+        
+        ObjCMethodList *Mem =
+          Reader.getSema()->BumpAlloc.Allocate<ObjCMethodList>();
+        Prev->Next = new (Mem) ObjCMethodList(Vec[I], 0);
+        Prev = Prev->Next;
+      }
+      
+      return List;
+    }
+    
+  public:
+    ReadMethodPoolVisitor(ASTReader &Reader, Selector Sel)
+      : Reader(Reader), Sel(Sel) { }
+    
+    static bool visit(Module &M, void *UserData) {
+      ReadMethodPoolVisitor *This
+        = static_cast<ReadMethodPoolVisitor *>(UserData);
+      
+      if (!M.SelectorLookupTable)
+        return false;
+      
+      ASTSelectorLookupTable *PoolTable
+        = (ASTSelectorLookupTable*)M.SelectorLookupTable;
+      ASTSelectorLookupTable::iterator Pos = PoolTable->find(This->Sel);
+      if (Pos == PoolTable->end())
+        return false;
+      
+      ++This->Reader.NumSelectorsRead;
       // FIXME: Not quite happy with the statistics here. We probably should
       // disable this tracking when called via LoadSelector.
       // Also, should entries without methods count as misses?
-      ++NumMethodPoolEntriesRead;
+      ++This->Reader.NumMethodPoolEntriesRead;
       ASTSelectorLookupTrait::data_type Data = *Pos;
-      if (DeserializationListener)
-        DeserializationListener->SelectorRead(Data.ID, Sel);
-      return std::make_pair(Data.Instance, Data.Factory);
+      if (This->Reader.DeserializationListener)
+        This->Reader.DeserializationListener->SelectorRead(Data.ID, 
+                                                           This->Sel);
+      
+      This->InstanceMethods.append(Data.Instance.begin(), Data.Instance.end());
+      This->FactoryMethods.append(Data.Factory.begin(), Data.Factory.end());
+      return true;
+    }
+    
+    /// \brief Retrieve the instance methods found by this visitor.
+    ObjCMethodList getInstanceMethods() const { 
+      return buildObjCMethodList(InstanceMethods); 
     }
-  }
 
-  ++NumMethodPoolMisses;
-  return std::pair<ObjCMethodList, ObjCMethodList>();
+    /// \brief Retrieve the instance methods found by this visitor.
+    ObjCMethodList getFactoryMethods() const { 
+      return buildObjCMethodList(FactoryMethods); 
+    }
+  };
+} } // end namespace clang::serialization
+
+std::pair<ObjCMethodList, ObjCMethodList>
+ASTReader::ReadMethodPool(Selector Sel) {
+  ReadMethodPoolVisitor Visitor(*this, Sel);
+  ModuleMgr.visit(&ReadMethodPoolVisitor::visit, &Visitor);
+  std::pair<ObjCMethodList, ObjCMethodList> Result;
+  Result.first = Visitor.getInstanceMethods();
+  Result.second = Visitor.getFactoryMethods();
+  
+  if (!Result.first.Method && !Result.second.Method)
+    ++NumMethodPoolMisses;
+  return Result;
 }
 
 void ASTReader::ReadKnownNamespaces(
-                          llvm::SmallVectorImpl<NamespaceDecl *> &Namespaces) {
+                          SmallVectorImpl<NamespaceDecl *> &Namespaces) {
   Namespaces.clear();
   
   for (unsigned I = 0, N = KnownNamespaces.size(); I != N; ++I) {
@@ -4469,12 +4569,136 @@ void ASTReader::ReadKnownNamespaces(
   }
 }
 
+void ASTReader::ReadTentativeDefinitions(
+                  SmallVectorImpl<VarDecl *> &TentativeDefs) {
+  for (unsigned I = 0, N = TentativeDefinitions.size(); I != N; ++I) {
+    VarDecl *Var = dyn_cast_or_null<VarDecl>(GetDecl(TentativeDefinitions[I]));
+    if (Var)
+      TentativeDefs.push_back(Var);
+  }
+  TentativeDefinitions.clear();
+}
+
+void ASTReader::ReadUnusedFileScopedDecls(
+                               SmallVectorImpl<const DeclaratorDecl *> &Decls) {
+  for (unsigned I = 0, N = UnusedFileScopedDecls.size(); I != N; ++I) {
+    DeclaratorDecl *D
+      = dyn_cast_or_null<DeclaratorDecl>(GetDecl(UnusedFileScopedDecls[I]));
+    if (D)
+      Decls.push_back(D);
+  }
+  UnusedFileScopedDecls.clear();
+}
+
+void ASTReader::ReadDelegatingConstructors(
+                                 SmallVectorImpl<CXXConstructorDecl *> &Decls) {
+  for (unsigned I = 0, N = DelegatingCtorDecls.size(); I != N; ++I) {
+    CXXConstructorDecl *D
+      = dyn_cast_or_null<CXXConstructorDecl>(GetDecl(DelegatingCtorDecls[I]));
+    if (D)
+      Decls.push_back(D);
+  }
+  DelegatingCtorDecls.clear();
+}
+
+void ASTReader::ReadExtVectorDecls(SmallVectorImpl<TypedefNameDecl *> &Decls) {
+  for (unsigned I = 0, N = ExtVectorDecls.size(); I != N; ++I) {
+    TypedefNameDecl *D
+      = dyn_cast_or_null<TypedefNameDecl>(GetDecl(ExtVectorDecls[I]));
+    if (D)
+      Decls.push_back(D);
+  }
+  ExtVectorDecls.clear();
+}
+
+void ASTReader::ReadDynamicClasses(SmallVectorImpl<CXXRecordDecl *> &Decls) {
+  for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I) {
+    CXXRecordDecl *D
+      = dyn_cast_or_null<CXXRecordDecl>(GetDecl(DynamicClasses[I]));
+    if (D)
+      Decls.push_back(D);
+  }
+  DynamicClasses.clear();
+}
+
+void 
+ASTReader::ReadLocallyScopedExternalDecls(SmallVectorImpl<NamedDecl *> &Decls) {
+  for (unsigned I = 0, N = LocallyScopedExternalDecls.size(); I != N; ++I) {
+    NamedDecl *D 
+      = dyn_cast_or_null<NamedDecl>(GetDecl(LocallyScopedExternalDecls[I]));
+    if (D)
+      Decls.push_back(D);
+  }
+  LocallyScopedExternalDecls.clear();
+}
+
+void ASTReader::ReadReferencedSelectors(
+       SmallVectorImpl<std::pair<Selector, SourceLocation> > &Sels) {
+  if (ReferencedSelectorsData.empty())
+    return;
+  
+  // If there are @selector references added them to its pool. This is for
+  // implementation of -Wselector.
+  unsigned int DataSize = ReferencedSelectorsData.size()-1;
+  unsigned I = 0;
+  while (I < DataSize) {
+    Selector Sel = DecodeSelector(ReferencedSelectorsData[I++]);
+    SourceLocation SelLoc
+      = SourceLocation::getFromRawEncoding(ReferencedSelectorsData[I++]);
+    Sels.push_back(std::make_pair(Sel, SelLoc));
+  }
+  ReferencedSelectorsData.clear();
+}
+
+void ASTReader::ReadWeakUndeclaredIdentifiers(
+       SmallVectorImpl<std::pair<IdentifierInfo *, WeakInfo> > &WeakIDs) {
+  if (WeakUndeclaredIdentifiers.empty())
+    return;
+
+  for (unsigned I = 0, N = WeakUndeclaredIdentifiers.size(); I < N; /*none*/) {
+    IdentifierInfo *WeakId 
+      = DecodeIdentifierInfo(WeakUndeclaredIdentifiers[I++]);
+    IdentifierInfo *AliasId 
+      = DecodeIdentifierInfo(WeakUndeclaredIdentifiers[I++]);
+    SourceLocation Loc
+      = SourceLocation::getFromRawEncoding(WeakUndeclaredIdentifiers[I++]);
+    bool Used = WeakUndeclaredIdentifiers[I++];
+    WeakInfo WI(AliasId, Loc);
+    WI.setUsed(Used);
+    WeakIDs.push_back(std::make_pair(WeakId, WI));
+  }
+  WeakUndeclaredIdentifiers.clear();
+}
+
+void ASTReader::ReadUsedVTables(SmallVectorImpl<ExternalVTableUse> &VTables) {
+  for (unsigned Idx = 0, N = VTableUses.size(); Idx < N; /* In loop */) {
+    ExternalVTableUse VT;
+    VT.Record = dyn_cast_or_null<CXXRecordDecl>(GetDecl(VTableUses[Idx++]));
+    VT.Location = SourceLocation::getFromRawEncoding(VTableUses[Idx++]);
+    VT.DefinitionRequired = VTableUses[Idx++];
+    VTables.push_back(VT);
+  }
+  
+  VTableUses.clear();
+}
+
+void ASTReader::ReadPendingInstantiations(
+       SmallVectorImpl<std::pair<ValueDecl *, SourceLocation> > &Pending) {
+  for (unsigned Idx = 0, N = PendingInstantiations.size(); Idx < N;) {
+    ValueDecl *D = cast<ValueDecl>(GetDecl(PendingInstantiations[Idx++]));
+    SourceLocation Loc
+      = SourceLocation::getFromRawEncoding(PendingInstantiations[Idx++]);
+    Pending.push_back(std::make_pair(D, Loc));
+  }  
+  PendingInstantiations.clear();
+}
+
 void ASTReader::LoadSelector(Selector Sel) {
   // It would be complicated to avoid reading the methods anyway. So don't.
   ReadMethodPool(Sel);
 }
 
-void ASTReader::SetIdentifierInfo(unsigned ID, IdentifierInfo *II) {
+void ASTReader::SetIdentifierInfo(IdentifierID ID, IdentifierInfo *II) {
   assert(ID && "Non-zero identifier ID required");
   assert(ID <= IdentifiersLoaded.size() && "identifier ID out of range");
   IdentifiersLoaded[ID - 1] = II;
@@ -4500,7 +4724,7 @@ void ASTReader::SetIdentifierInfo(unsigned ID, IdentifierInfo *II) {
 /// will not be placed onto the pending queue.
 void
 ASTReader::SetGloballyVisibleDecls(IdentifierInfo *II,
-                              const llvm::SmallVectorImpl<uint32_t> &DeclIDs,
+                              const SmallVectorImpl<uint32_t> &DeclIDs,
                                    bool Nonrecursive) {
   if (NumCurrentElementsDeserializing && !Nonrecursive) {
     PendingIdentifierInfos.push_back(PendingIdentifierInfo());
@@ -4529,7 +4753,7 @@ ASTReader::SetGloballyVisibleDecls(IdentifierInfo *II,
   }
 }
 
-IdentifierInfo *ASTReader::DecodeIdentifierInfo(unsigned ID) {
+IdentifierInfo *ASTReader::DecodeIdentifierInfo(IdentifierID ID) {
   if (ID == 0)
     return 0;
 
@@ -4538,21 +4762,13 @@ IdentifierInfo *ASTReader::DecodeIdentifierInfo(unsigned ID) {
     return 0;
   }
 
-  assert(PP && "Forgot to set Preprocessor ?");
   ID -= 1;
   if (!IdentifiersLoaded[ID]) {
-    unsigned Index = ID;
-    const char *Str = 0;
-    for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-      PerFileData *F = Chain[N - I - 1];
-      if (Index < F->LocalNumIdentifiers) {
-         uint32_t Offset = F->IdentifierOffsets[Index];
-         Str = F->IdentifierTableData + Offset;
-         break;
-      }
-      Index -= F->LocalNumIdentifiers;
-    }
-    assert(Str && "Broken Chain");
+    GlobalIdentifierMapType::iterator I = GlobalIdentifierMap.find(ID + 1);
+    assert(I != GlobalIdentifierMap.end() && "Corrupted global identifier map");
+    Module *M = I->second;
+    unsigned Index = ID - M->BaseIdentifierID;
+    const char *Str = M->IdentifierTableData + M->IdentifierOffsets[Index];
 
     // All of the strings in the AST file are preceded by a 16-bit length.
     // Extract that 16-bit length to avoid having to execute strlen().
@@ -4563,7 +4779,7 @@ IdentifierInfo *ASTReader::DecodeIdentifierInfo(unsigned ID) {
     unsigned StrLen = (((unsigned) StrLenPtr[0])
                        | (((unsigned) StrLenPtr[1]) << 8)) - 1;
     IdentifiersLoaded[ID]
-      = &PP->getIdentifierTable().get(llvm::StringRef(Str, StrLen));
+      = &PP.getIdentifierTable().get(StringRef(Str, StrLen));
     if (DeserializationListener)
       DeserializationListener->IdentifierRead(ID + 1, IdentifiersLoaded[ID]);
   }
@@ -4571,11 +4787,31 @@ IdentifierInfo *ASTReader::DecodeIdentifierInfo(unsigned ID) {
   return IdentifiersLoaded[ID];
 }
 
-bool ASTReader::ReadSLocEntry(unsigned ID) {
+IdentifierInfo *ASTReader::getLocalIdentifier(Module &M, unsigned LocalID) {
+  return DecodeIdentifierInfo(getGlobalIdentifierID(M, LocalID));
+}
+
+IdentifierID ASTReader::getGlobalIdentifierID(Module &M, unsigned LocalID) {
+  if (LocalID < NUM_PREDEF_IDENT_IDS)
+    return LocalID;
+  
+  ContinuousRangeMap<uint32_t, int, 2>::iterator I
+    = M.IdentifierRemap.find(LocalID - NUM_PREDEF_IDENT_IDS);
+  assert(I != M.IdentifierRemap.end() 
+         && "Invalid index into identifier index remap");
+  
+  return LocalID + I->second;
+}
+
+bool ASTReader::ReadSLocEntry(int ID) {
   return ReadSLocEntryRecord(ID) != Success;
 }
 
-Selector ASTReader::DecodeSelector(unsigned ID) {
+Selector ASTReader::getLocalSelector(Module &M, unsigned LocalID) {
+  return DecodeSelector(getGlobalSelectorID(M, LocalID));
+}
+
+Selector ASTReader::DecodeSelector(serialization::SelectorID ID) {
   if (ID == 0)
     return Selector();
 
@@ -4586,25 +4822,21 @@ Selector ASTReader::DecodeSelector(unsigned ID) {
 
   if (SelectorsLoaded[ID - 1].getAsOpaquePtr() == 0) {
     // Load this selector from the selector table.
-    unsigned Idx = ID - 1;
-    for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-      PerFileData &F = *Chain[N - I - 1];
-      if (Idx < F.LocalNumSelectors) {
-        ASTSelectorLookupTrait Trait(*this);
-        SelectorsLoaded[ID - 1] =
-           Trait.ReadKey(F.SelectorLookupTableData + F.SelectorOffsets[Idx], 0);
-        if (DeserializationListener)
-          DeserializationListener->SelectorRead(ID, SelectorsLoaded[ID - 1]);
-        break;
-      }
-      Idx -= F.LocalNumSelectors;
-    }
+    GlobalSelectorMapType::iterator I = GlobalSelectorMap.find(ID);
+    assert(I != GlobalSelectorMap.end() && "Corrupted global selector map");
+    Module &M = *I->second;
+    ASTSelectorLookupTrait Trait(*this, M);
+    unsigned Idx = ID - M.BaseSelectorID - NUM_PREDEF_SELECTOR_IDS;
+    SelectorsLoaded[ID - 1] =
+      Trait.ReadKey(M.SelectorLookupTableData + M.SelectorOffsets[Idx], 0);
+    if (DeserializationListener)
+      DeserializationListener->SelectorRead(ID, SelectorsLoaded[ID - 1]);
   }
 
   return SelectorsLoaded[ID - 1];
 }
 
-Selector ASTReader::GetExternalSelector(uint32_t ID) {
+Selector ASTReader::GetExternalSelector(serialization::SelectorID ID) {
   return DecodeSelector(ID);
 }
 
@@ -4613,37 +4845,51 @@ uint32_t ASTReader::GetNumExternalSelectors() {
   return getTotalNumSelectors() + 1;
 }
 
+serialization::SelectorID
+ASTReader::getGlobalSelectorID(Module &M, unsigned LocalID) const {
+  if (LocalID < NUM_PREDEF_SELECTOR_IDS)
+    return LocalID;
+  
+  ContinuousRangeMap<uint32_t, int, 2>::iterator I
+    = M.SelectorRemap.find(LocalID - NUM_PREDEF_SELECTOR_IDS);
+  assert(I != M.SelectorRemap.end() 
+         && "Invalid index into identifier index remap");
+  
+  return LocalID + I->second;
+}
+
 DeclarationName
-ASTReader::ReadDeclarationName(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadDeclarationName(Module &F, 
+                               const RecordData &Record, unsigned &Idx) {
   DeclarationName::NameKind Kind = (DeclarationName::NameKind)Record[Idx++];
   switch (Kind) {
   case DeclarationName::Identifier:
-    return DeclarationName(GetIdentifierInfo(Record, Idx));
+    return DeclarationName(GetIdentifierInfo(F, Record, Idx));
 
   case DeclarationName::ObjCZeroArgSelector:
   case DeclarationName::ObjCOneArgSelector:
   case DeclarationName::ObjCMultiArgSelector:
-    return DeclarationName(GetSelector(Record, Idx));
+    return DeclarationName(ReadSelector(F, Record, Idx));
 
   case DeclarationName::CXXConstructorName:
-    return Context->DeclarationNames.getCXXConstructorName(
-                          Context->getCanonicalType(GetType(Record[Idx++])));
+    return Context.DeclarationNames.getCXXConstructorName(
+                          Context.getCanonicalType(readType(F, Record, Idx)));
 
   case DeclarationName::CXXDestructorName:
-    return Context->DeclarationNames.getCXXDestructorName(
-                          Context->getCanonicalType(GetType(Record[Idx++])));
+    return Context.DeclarationNames.getCXXDestructorName(
+                          Context.getCanonicalType(readType(F, Record, Idx)));
 
   case DeclarationName::CXXConversionFunctionName:
-    return Context->DeclarationNames.getCXXConversionFunctionName(
-                          Context->getCanonicalType(GetType(Record[Idx++])));
+    return Context.DeclarationNames.getCXXConversionFunctionName(
+                          Context.getCanonicalType(readType(F, Record, Idx)));
 
   case DeclarationName::CXXOperatorName:
-    return Context->DeclarationNames.getCXXOperatorName(
+    return Context.DeclarationNames.getCXXOperatorName(
                                        (OverloadedOperatorKind)Record[Idx++]);
 
   case DeclarationName::CXXLiteralOperatorName:
-    return Context->DeclarationNames.getCXXLiteralOperatorName(
-                                       GetIdentifierInfo(Record, Idx));
+    return Context.DeclarationNames.getCXXLiteralOperatorName(
+                                       GetIdentifierInfo(F, Record, Idx));
 
   case DeclarationName::CXXUsingDirective:
     return DeclarationName::getUsingDirectiveName();
@@ -4653,7 +4899,7 @@ ASTReader::ReadDeclarationName(const RecordData &Record, unsigned &Idx) {
   return DeclarationName();
 }
 
-void ASTReader::ReadDeclarationNameLoc(PerFileData &F,
+void ASTReader::ReadDeclarationNameLoc(Module &F,
                                        DeclarationNameLoc &DNLoc,
                                        DeclarationName Name,
                                       const RecordData &Record, unsigned &Idx) {
@@ -4685,72 +4931,73 @@ void ASTReader::ReadDeclarationNameLoc(PerFileData &F,
   }
 }
 
-void ASTReader::ReadDeclarationNameInfo(PerFileData &F,
+void ASTReader::ReadDeclarationNameInfo(Module &F,
                                         DeclarationNameInfo &NameInfo,
                                       const RecordData &Record, unsigned &Idx) {
-  NameInfo.setName(ReadDeclarationName(Record, Idx));
+  NameInfo.setName(ReadDeclarationName(F, Record, Idx));
   NameInfo.setLoc(ReadSourceLocation(F, Record, Idx));
   DeclarationNameLoc DNLoc;
   ReadDeclarationNameLoc(F, DNLoc, NameInfo.getName(), Record, Idx);
   NameInfo.setInfo(DNLoc);
 }
 
-void ASTReader::ReadQualifierInfo(PerFileData &F, QualifierInfo &Info,
+void ASTReader::ReadQualifierInfo(Module &F, QualifierInfo &Info,
                                   const RecordData &Record, unsigned &Idx) {
   Info.QualifierLoc = ReadNestedNameSpecifierLoc(F, Record, Idx);
   unsigned NumTPLists = Record[Idx++];
   Info.NumTemplParamLists = NumTPLists;
   if (NumTPLists) {
-    Info.TemplParamLists = new (*Context) TemplateParameterList*[NumTPLists];
+    Info.TemplParamLists = new (Context) TemplateParameterList*[NumTPLists];
     for (unsigned i=0; i != NumTPLists; ++i)
       Info.TemplParamLists[i] = ReadTemplateParameterList(F, Record, Idx);
   }
 }
 
 TemplateName
-ASTReader::ReadTemplateName(PerFileData &F, const RecordData &Record, 
+ASTReader::ReadTemplateName(Module &F, const RecordData &Record, 
                             unsigned &Idx) {
   TemplateName::NameKind Kind = (TemplateName::NameKind)Record[Idx++];
   switch (Kind) {
   case TemplateName::Template:
-    return TemplateName(cast_or_null<TemplateDecl>(GetDecl(Record[Idx++])));
+      return TemplateName(ReadDeclAs<TemplateDecl>(F, Record, Idx));
 
   case TemplateName::OverloadedTemplate: {
     unsigned size = Record[Idx++];
     UnresolvedSet<8> Decls;
     while (size--)
-      Decls.addDecl(cast<NamedDecl>(GetDecl(Record[Idx++])));
+      Decls.addDecl(ReadDeclAs<NamedDecl>(F, Record, Idx));
 
-    return Context->getOverloadedTemplateName(Decls.begin(), Decls.end());
+    return Context.getOverloadedTemplateName(Decls.begin(), Decls.end());
   }
 
   case TemplateName::QualifiedTemplate: {
-    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
+    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(F, Record, Idx);
     bool hasTemplKeyword = Record[Idx++];
-    TemplateDecl *Template = cast<TemplateDecl>(GetDecl(Record[Idx++]));
-    return Context->getQualifiedTemplateName(NNS, hasTemplKeyword, Template);
+    TemplateDecl *Template = ReadDeclAs<TemplateDecl>(F, Record, Idx);
+    return Context.getQualifiedTemplateName(NNS, hasTemplKeyword, Template);
   }
 
   case TemplateName::DependentTemplate: {
-    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
+    NestedNameSpecifier *NNS = ReadNestedNameSpecifier(F, Record, Idx);
     if (Record[Idx++])  // isIdentifier
-      return Context->getDependentTemplateName(NNS,
-                                               GetIdentifierInfo(Record, Idx));
-    return Context->getDependentTemplateName(NNS,
+      return Context.getDependentTemplateName(NNS,
+                                               GetIdentifierInfo(F, Record, 
+                                                                 Idx));
+    return Context.getDependentTemplateName(NNS,
                                          (OverloadedOperatorKind)Record[Idx++]);
   }
 
   case TemplateName::SubstTemplateTemplateParm: {
     TemplateTemplateParmDecl *param
-      = cast_or_null<TemplateTemplateParmDecl>(GetDecl(Record[Idx++]));
+      = ReadDeclAs<TemplateTemplateParmDecl>(F, Record, Idx);
     if (!param) return TemplateName();
     TemplateName replacement = ReadTemplateName(F, Record, Idx);
-    return Context->getSubstTemplateTemplateParm(param, replacement);
+    return Context.getSubstTemplateTemplateParm(param, replacement);
   }
       
   case TemplateName::SubstTemplateTemplateParmPack: {
     TemplateTemplateParmDecl *Param 
-      = cast_or_null<TemplateTemplateParmDecl>(GetDecl(Record[Idx++]));
+      = ReadDeclAs<TemplateTemplateParmDecl>(F, Record, Idx);
     if (!Param)
       return TemplateName();
     
@@ -4758,28 +5005,27 @@ ASTReader::ReadTemplateName(PerFileData &F, const RecordData &Record,
     if (ArgPack.getKind() != TemplateArgument::Pack)
       return TemplateName();
     
-    return Context->getSubstTemplateTemplateParmPack(Param, ArgPack);
+    return Context.getSubstTemplateTemplateParmPack(Param, ArgPack);
   }
   }
 
-  assert(0 && "Unhandled template name kind!");
-  return TemplateName();
+  llvm_unreachable("Unhandled template name kind!");
 }
 
 TemplateArgument
-ASTReader::ReadTemplateArgument(PerFileData &F,
+ASTReader::ReadTemplateArgument(Module &F,
                                 const RecordData &Record, unsigned &Idx) {
   TemplateArgument::ArgKind Kind = (TemplateArgument::ArgKind)Record[Idx++];
   switch (Kind) {
   case TemplateArgument::Null:
     return TemplateArgument();
   case TemplateArgument::Type:
-    return TemplateArgument(GetType(Record[Idx++]));
+    return TemplateArgument(readType(F, Record, Idx));
   case TemplateArgument::Declaration:
-    return TemplateArgument(GetDecl(Record[Idx++]));
+    return TemplateArgument(ReadDecl(F, Record, Idx));
   case TemplateArgument::Integral: {
     llvm::APSInt Value = ReadAPSInt(Record, Idx);
-    QualType T = GetType(Record[Idx++]);
+    QualType T = readType(F, Record, Idx);
     return TemplateArgument(Value, T);
   }
   case TemplateArgument::Template: 
@@ -4795,40 +5041,39 @@ ASTReader::ReadTemplateArgument(PerFileData &F,
     return TemplateArgument(ReadExpr(F));
   case TemplateArgument::Pack: {
     unsigned NumArgs = Record[Idx++];
-    TemplateArgument *Args = new (*Context) TemplateArgument[NumArgs];
+    TemplateArgument *Args = new (Context) TemplateArgument[NumArgs];
     for (unsigned I = 0; I != NumArgs; ++I)
       Args[I] = ReadTemplateArgument(F, Record, Idx);
     return TemplateArgument(Args, NumArgs);
   }
   }
 
-  assert(0 && "Unhandled template argument kind!");
-  return TemplateArgument();
+  llvm_unreachable("Unhandled template argument kind!");
 }
 
 TemplateParameterList *
-ASTReader::ReadTemplateParameterList(PerFileData &F,
+ASTReader::ReadTemplateParameterList(Module &F,
                                      const RecordData &Record, unsigned &Idx) {
   SourceLocation TemplateLoc = ReadSourceLocation(F, Record, Idx);
   SourceLocation LAngleLoc = ReadSourceLocation(F, Record, Idx);
   SourceLocation RAngleLoc = ReadSourceLocation(F, Record, Idx);
 
   unsigned NumParams = Record[Idx++];
-  llvm::SmallVector<NamedDecl *, 16> Params;
+  SmallVector<NamedDecl *, 16> Params;
   Params.reserve(NumParams);
   while (NumParams--)
-    Params.push_back(cast<NamedDecl>(GetDecl(Record[Idx++])));
+    Params.push_back(ReadDeclAs<NamedDecl>(F, Record, Idx));
 
   TemplateParameterList* TemplateParams =
-    TemplateParameterList::Create(*Context, TemplateLoc, LAngleLoc,
+    TemplateParameterList::Create(Context, TemplateLoc, LAngleLoc,
                                   Params.data(), Params.size(), RAngleLoc);
   return TemplateParams;
 }
 
 void
 ASTReader::
-ReadTemplateArgumentList(llvm::SmallVector<TemplateArgument, 8> &TemplArgs,
-                         PerFileData &F, const RecordData &Record,
+ReadTemplateArgumentList(SmallVector<TemplateArgument, 8> &TemplArgs,
+                         Module &F, const RecordData &Record,
                          unsigned &Idx) {
   unsigned NumTemplateArgs = Record[Idx++];
   TemplArgs.reserve(NumTemplateArgs);
@@ -4837,18 +5082,18 @@ ReadTemplateArgumentList(llvm::SmallVector<TemplateArgument, 8> &TemplArgs,
 }
 
 /// \brief Read a UnresolvedSet structure.
-void ASTReader::ReadUnresolvedSet(UnresolvedSetImpl &Set,
+void ASTReader::ReadUnresolvedSet(Module &F, UnresolvedSetImpl &Set,
                                   const RecordData &Record, unsigned &Idx) {
   unsigned NumDecls = Record[Idx++];
   while (NumDecls--) {
-    NamedDecl *D = cast<NamedDecl>(GetDecl(Record[Idx++]));
+    NamedDecl *D = ReadDeclAs<NamedDecl>(F, Record, Idx);
     AccessSpecifier AS = (AccessSpecifier)Record[Idx++];
     Set.addDecl(D, AS);
   }
 }
 
 CXXBaseSpecifier
-ASTReader::ReadCXXBaseSpecifier(PerFileData &F,
+ASTReader::ReadCXXBaseSpecifier(Module &F,
                                 const RecordData &Record, unsigned &Idx) {
   bool isVirtual = static_cast<bool>(Record[Idx++]);
   bool isBaseOfClass = static_cast<bool>(Record[Idx++]);
@@ -4864,15 +5109,13 @@ ASTReader::ReadCXXBaseSpecifier(PerFileData &F,
 }
 
 std::pair<CXXCtorInitializer **, unsigned>
-ASTReader::ReadCXXCtorInitializers(PerFileData &F, const RecordData &Record,
+ASTReader::ReadCXXCtorInitializers(Module &F, const RecordData &Record,
                                    unsigned &Idx) {
   CXXCtorInitializer **CtorInitializers = 0;
   unsigned NumInitializers = Record[Idx++];
   if (NumInitializers) {
-    ASTContext &C = *getContext();
-
     CtorInitializers
-        = new (C) CXXCtorInitializer*[NumInitializers];
+        = new (Context) CXXCtorInitializer*[NumInitializers];
     for (unsigned i=0; i != NumInitializers; ++i) {
       TypeSourceInfo *BaseClassInfo = 0;
       bool IsBaseVirtual = false;
@@ -4888,15 +5131,15 @@ ASTReader::ReadCXXCtorInitializers(PerFileData &F, const RecordData &Record,
         break;
 
        case CTOR_INITIALIZER_DELEGATING:
-        Target = cast<CXXConstructorDecl>(GetDecl(Record[Idx++]));
+        Target = ReadDeclAs<CXXConstructorDecl>(F, Record, Idx);
         break;
 
        case CTOR_INITIALIZER_MEMBER:
-        Member = cast<FieldDecl>(GetDecl(Record[Idx++]));
+        Member = ReadDeclAs<FieldDecl>(F, Record, Idx);
         break;
 
        case CTOR_INITIALIZER_INDIRECT_MEMBER:
-        IndirectMember = cast<IndirectFieldDecl>(GetDecl(Record[Idx++]));
+        IndirectMember = ReadDeclAs<IndirectFieldDecl>(F, Record, Idx);
         break;
       }
 
@@ -4906,34 +5149,34 @@ ASTReader::ReadCXXCtorInitializers(PerFileData &F, const RecordData &Record,
       SourceLocation RParenLoc = ReadSourceLocation(F, Record, Idx);
       bool IsWritten = Record[Idx++];
       unsigned SourceOrderOrNumArrayIndices;
-      llvm::SmallVector<VarDecl *, 8> Indices;
+      SmallVector<VarDecl *, 8> Indices;
       if (IsWritten) {
         SourceOrderOrNumArrayIndices = Record[Idx++];
       } else {
         SourceOrderOrNumArrayIndices = Record[Idx++];
         Indices.reserve(SourceOrderOrNumArrayIndices);
         for (unsigned i=0; i != SourceOrderOrNumArrayIndices; ++i)
-          Indices.push_back(cast<VarDecl>(GetDecl(Record[Idx++])));
+          Indices.push_back(ReadDeclAs<VarDecl>(F, Record, Idx));
       }
 
       CXXCtorInitializer *BOMInit;
       if (Type == CTOR_INITIALIZER_BASE) {
-        BOMInit = new (C) CXXCtorInitializer(C, BaseClassInfo, IsBaseVirtual,
+        BOMInit = new (Context) CXXCtorInitializer(Context, BaseClassInfo, IsBaseVirtual,
                                              LParenLoc, Init, RParenLoc,
                                              MemberOrEllipsisLoc);
       } else if (Type == CTOR_INITIALIZER_DELEGATING) {
-        BOMInit = new (C) CXXCtorInitializer(C, MemberOrEllipsisLoc, LParenLoc,
+        BOMInit = new (Context) CXXCtorInitializer(Context, MemberOrEllipsisLoc, LParenLoc,
                                              Target, Init, RParenLoc);
       } else if (IsWritten) {
         if (Member)
-          BOMInit = new (C) CXXCtorInitializer(C, Member, MemberOrEllipsisLoc,
+          BOMInit = new (Context) CXXCtorInitializer(Context, Member, MemberOrEllipsisLoc,
                                                LParenLoc, Init, RParenLoc);
         else 
-          BOMInit = new (C) CXXCtorInitializer(C, IndirectMember,
+          BOMInit = new (Context) CXXCtorInitializer(Context, IndirectMember,
                                                MemberOrEllipsisLoc, LParenLoc,
                                                Init, RParenLoc);
       } else {
-        BOMInit = CXXCtorInitializer::Create(C, Member, MemberOrEllipsisLoc,
+        BOMInit = CXXCtorInitializer::Create(Context, Member, MemberOrEllipsisLoc,
                                              LParenLoc, Init, RParenLoc,
                                              Indices.data(), Indices.size());
       }
@@ -4948,7 +5191,8 @@ ASTReader::ReadCXXCtorInitializers(PerFileData &F, const RecordData &Record,
 }
 
 NestedNameSpecifier *
-ASTReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadNestedNameSpecifier(Module &F,
+                                   const RecordData &Record, unsigned &Idx) {
   unsigned N = Record[Idx++];
   NestedNameSpecifier *NNS = 0, *Prev = 0;
   for (unsigned I = 0; I != N; ++I) {
@@ -4956,37 +5200,36 @@ ASTReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
       = (NestedNameSpecifier::SpecifierKind)Record[Idx++];
     switch (Kind) {
     case NestedNameSpecifier::Identifier: {
-      IdentifierInfo *II = GetIdentifierInfo(Record, Idx);
-      NNS = NestedNameSpecifier::Create(*Context, Prev, II);
+      IdentifierInfo *II = GetIdentifierInfo(F, Record, Idx);
+      NNS = NestedNameSpecifier::Create(Context, Prev, II);
       break;
     }
 
     case NestedNameSpecifier::Namespace: {
-      NamespaceDecl *NS = cast<NamespaceDecl>(GetDecl(Record[Idx++]));
-      NNS = NestedNameSpecifier::Create(*Context, Prev, NS);
+      NamespaceDecl *NS = ReadDeclAs<NamespaceDecl>(F, Record, Idx);
+      NNS = NestedNameSpecifier::Create(Context, Prev, NS);
       break;
     }
 
     case NestedNameSpecifier::NamespaceAlias: {
-      NamespaceAliasDecl *Alias
-        = cast<NamespaceAliasDecl>(GetDecl(Record[Idx++]));
-      NNS = NestedNameSpecifier::Create(*Context, Prev, Alias);
+      NamespaceAliasDecl *Alias =ReadDeclAs<NamespaceAliasDecl>(F, Record, Idx);
+      NNS = NestedNameSpecifier::Create(Context, Prev, Alias);
       break;
     }
 
     case NestedNameSpecifier::TypeSpec:
     case NestedNameSpecifier::TypeSpecWithTemplate: {
-      const Type *T = GetType(Record[Idx++]).getTypePtrOrNull();
+      const Type *T = readType(F, Record, Idx).getTypePtrOrNull();
       if (!T)
         return 0;
       
       bool Template = Record[Idx++];
-      NNS = NestedNameSpecifier::Create(*Context, Prev, Template, T);
+      NNS = NestedNameSpecifier::Create(Context, Prev, Template, T);
       break;
     }
 
     case NestedNameSpecifier::Global: {
-      NNS = NestedNameSpecifier::GlobalSpecifier(*Context);
+      NNS = NestedNameSpecifier::GlobalSpecifier(Context);
       // No associated value, and there can't be a prefix.
       break;
     }
@@ -4997,7 +5240,7 @@ ASTReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
 }
 
 NestedNameSpecifierLoc
-ASTReader::ReadNestedNameSpecifierLoc(PerFileData &F, const RecordData &Record, 
+ASTReader::ReadNestedNameSpecifierLoc(Module &F, const RecordData &Record, 
                                       unsigned &Idx) {
   unsigned N = Record[Idx++];
   NestedNameSpecifierLocBuilder Builder;
@@ -5006,24 +5249,23 @@ ASTReader::ReadNestedNameSpecifierLoc(PerFileData &F, const RecordData &Record,
       = (NestedNameSpecifier::SpecifierKind)Record[Idx++];
     switch (Kind) {
     case NestedNameSpecifier::Identifier: {
-      IdentifierInfo *II = GetIdentifierInfo(Record, Idx);      
+      IdentifierInfo *II = GetIdentifierInfo(F, Record, Idx);      
       SourceRange Range = ReadSourceRange(F, Record, Idx);
-      Builder.Extend(*Context, II, Range.getBegin(), Range.getEnd());
+      Builder.Extend(Context, II, Range.getBegin(), Range.getEnd());
       break;
     }
 
     case NestedNameSpecifier::Namespace: {
-      NamespaceDecl *NS = cast<NamespaceDecl>(GetDecl(Record[Idx++]));
+      NamespaceDecl *NS = ReadDeclAs<NamespaceDecl>(F, Record, Idx);
       SourceRange Range = ReadSourceRange(F, Record, Idx);
-      Builder.Extend(*Context, NS, Range.getBegin(), Range.getEnd());
+      Builder.Extend(Context, NS, Range.getBegin(), Range.getEnd());
       break;
     }
 
     case NestedNameSpecifier::NamespaceAlias: {
-      NamespaceAliasDecl *Alias
-        = cast<NamespaceAliasDecl>(GetDecl(Record[Idx++]));
+      NamespaceAliasDecl *Alias =ReadDeclAs<NamespaceAliasDecl>(F, Record, Idx);
       SourceRange Range = ReadSourceRange(F, Record, Idx);
-      Builder.Extend(*Context, Alias, Range.getBegin(), Range.getEnd());
+      Builder.Extend(Context, Alias, Range.getBegin(), Range.getEnd());
       break;
     }
 
@@ -5036,7 +5278,7 @@ ASTReader::ReadNestedNameSpecifierLoc(PerFileData &F, const RecordData &Record,
       SourceLocation ColonColonLoc = ReadSourceLocation(F, Record, Idx);
 
       // FIXME: 'template' keyword location not saved anywhere, so we fake it.
-      Builder.Extend(*Context, 
+      Builder.Extend(Context, 
                      Template? T->getTypeLoc().getBeginLoc() : SourceLocation(),
                      T->getTypeLoc(), ColonColonLoc);
       break;
@@ -5044,17 +5286,17 @@ ASTReader::ReadNestedNameSpecifierLoc(PerFileData &F, const RecordData &Record,
 
     case NestedNameSpecifier::Global: {
       SourceLocation ColonColonLoc = ReadSourceLocation(F, Record, Idx);
-      Builder.MakeGlobal(*Context, ColonColonLoc);
+      Builder.MakeGlobal(Context, ColonColonLoc);
       break;
     }
     }
   }
   
-  return Builder.getWithLocInContext(*Context);
+  return Builder.getWithLocInContext(Context);
 }
 
 SourceRange
-ASTReader::ReadSourceRange(PerFileData &F, const RecordData &Record,
+ASTReader::ReadSourceRange(Module &F, const RecordData &Record,
                            unsigned &Idx) {
   SourceLocation beg = ReadSourceLocation(F, Record, Idx);
   SourceLocation end = ReadSourceLocation(F, Record, Idx);
@@ -5101,10 +5343,11 @@ VersionTuple ASTReader::ReadVersionTuple(const RecordData &Record,
   return VersionTuple(Major, Minor - 1, Subminor - 1);
 }
 
-CXXTemporary *ASTReader::ReadCXXTemporary(const RecordData &Record,
+CXXTemporary *ASTReader::ReadCXXTemporary(Module &F, 
+                                          const RecordData &Record,
                                           unsigned &Idx) {
-  CXXDestructorDecl *Decl = cast<CXXDestructorDecl>(GetDecl(Record[Idx++]));
-  return CXXTemporary::Create(*Context, Decl);
+  CXXDestructorDecl *Decl = ReadDeclAs<CXXDestructorDecl>(F, Record, Idx);
+  return CXXTemporary::Create(Context, Decl);
 }
 
 DiagnosticBuilder ASTReader::Diag(unsigned DiagID) {
@@ -5118,8 +5361,7 @@ DiagnosticBuilder ASTReader::Diag(SourceLocation Loc, unsigned DiagID) {
 /// \brief Retrieve the identifier table associated with the
 /// preprocessor.
 IdentifierTable &ASTReader::getIdentifierTable() {
-  assert(PP && "Forgot to set Preprocessor ?");
-  return PP->getIdentifierTable();
+  return PP.getIdentifierTable();
 }
 
 /// \brief Record that the given ID maps to the given switch-case
@@ -5169,56 +5411,29 @@ void ASTReader::FinishedDeserializing() {
   --NumCurrentElementsDeserializing;
 }
 
-ASTReader::ASTReader(Preprocessor &PP, ASTContext *Context,
-                     const char *isysroot, bool DisableValidation,
+ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context,
+                     StringRef isysroot, bool DisableValidation,
                      bool DisableStatCache)
   : Listener(new PCHValidator(PP, *this)), DeserializationListener(0),
     SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
-    Diags(PP.getDiagnostics()), SemaObj(0), PP(&PP), Context(Context),
-    Consumer(0), isysroot(isysroot), DisableValidation(DisableValidation),
+    Diags(PP.getDiagnostics()), SemaObj(0), PP(PP), Context(Context),
+    Consumer(0), ModuleMgr(FileMgr.getFileSystemOptions()),
+    RelocatablePCH(false), isysroot(isysroot),
+    DisableValidation(DisableValidation),
     DisableStatCache(DisableStatCache), NumStatHits(0), NumStatMisses(0), 
-    NumSLocEntriesRead(0), TotalNumSLocEntries(0), NextSLocOffset(0), 
+    NumSLocEntriesRead(0), TotalNumSLocEntries(0), 
     NumStatementsRead(0), TotalNumStatements(0), NumMacrosRead(0), 
     TotalNumMacros(0), NumSelectorsRead(0), NumMethodPoolEntriesRead(0), 
     NumMethodPoolMisses(0), TotalNumMethodPoolEntries(0), 
     NumLexicalDeclContextsRead(0), TotalLexicalDeclContexts(0), 
-    NumVisibleDeclContextsRead(0), TotalVisibleDeclContexts(0), 
-    NumCurrentElementsDeserializing(0) 
+    NumVisibleDeclContextsRead(0), TotalVisibleDeclContexts(0),
+    TotalModulesSizeInBits(0), NumCurrentElementsDeserializing(0),
+    NumCXXBaseSpecifiersLoaded(0)
 {
-  RelocatablePCH = false;
-}
-
-ASTReader::ASTReader(SourceManager &SourceMgr, FileManager &FileMgr,
-                     Diagnostic &Diags, const char *isysroot,
-                     bool DisableValidation, bool DisableStatCache)
-  : DeserializationListener(0), SourceMgr(SourceMgr), FileMgr(FileMgr),
-    Diags(Diags), SemaObj(0), PP(0), Context(0), Consumer(0),
-    isysroot(isysroot), DisableValidation(DisableValidation), 
-    DisableStatCache(DisableStatCache), NumStatHits(0), NumStatMisses(0), 
-    NumSLocEntriesRead(0), TotalNumSLocEntries(0),
-    NextSLocOffset(0), NumStatementsRead(0), TotalNumStatements(0),
-    NumMacrosRead(0), TotalNumMacros(0), NumSelectorsRead(0),
-    NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0),
-    TotalNumMethodPoolEntries(0), NumLexicalDeclContextsRead(0),
-    TotalLexicalDeclContexts(0), NumVisibleDeclContextsRead(0),
-    TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) {
-  RelocatablePCH = false;
+  SourceMgr.setExternalSLocEntrySource(this);
 }
 
 ASTReader::~ASTReader() {
-  for (unsigned i = 0, e = Chain.size(); i != e; ++i)
-    delete Chain[e - i - 1];
-  // Delete all visible decl lookup tables
-  for (DeclContextOffsetsMap::iterator I = DeclContextOffsets.begin(),
-                                       E = DeclContextOffsets.end();
-       I != E; ++I) {
-    for (DeclContextInfos::iterator J = I->second.begin(), F = I->second.end();
-         J != F; ++J) {
-      if (J->NameLookupTableData)
-        delete static_cast<ASTDeclContextNameLookupTable*>(
-            J->NameLookupTableData);
-    }
-  }
   for (DeclContextVisibleUpdatesPending::iterator
            I = PendingVisibleUpdates.begin(),
            E = PendingVisibleUpdates.end();
@@ -5226,27 +5441,6 @@ ASTReader::~ASTReader() {
     for (DeclContextVisibleUpdates::iterator J = I->second.begin(),
                                              F = I->second.end();
          J != F; ++J)
-      delete static_cast<ASTDeclContextNameLookupTable*>(*J);
+      delete static_cast<ASTDeclContextNameLookupTable*>(J->first);
   }
 }
-
-ASTReader::PerFileData::PerFileData(ASTFileType Ty)
-  : Type(Ty), SizeInBits(0), LocalNumSLocEntries(0), SLocOffsets(0),
-    SLocFileOffsets(0), LocalSLocSize(0),
-    LocalNumIdentifiers(0), IdentifierOffsets(0), IdentifierTableData(0),
-    IdentifierLookupTable(0), LocalNumMacroDefinitions(0),
-    MacroDefinitionOffsets(0), 
-    LocalNumHeaderFileInfos(0), HeaderFileInfoTableData(0),
-    HeaderFileInfoTable(0),
-    LocalNumSelectors(0), SelectorOffsets(0),
-    SelectorLookupTableData(0), SelectorLookupTable(0), LocalNumDecls(0),
-    DeclOffsets(0), LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0),
-    LocalNumTypes(0), TypeOffsets(0), StatCache(0),
-    NumPreallocatedPreprocessingEntities(0), NextInSource(0)
-{}
-
-ASTReader::PerFileData::~PerFileData() {
-  delete static_cast<ASTIdentifierLookupTable *>(IdentifierLookupTable);
-  delete static_cast<HeaderFileInfoLookupTable *>(HeaderFileInfoTable);
-  delete static_cast<ASTSelectorLookupTable *>(SelectorLookupTable);
-}
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp
index 24ab544dcd1c..6cc3f0a70bc1 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -14,6 +14,7 @@
 
 #include "ASTCommon.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Sema/SemaDiagnostic.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclVisitor.h"
@@ -31,7 +32,7 @@ using namespace clang::serialization;
 namespace clang {
   class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
     ASTReader &Reader;
-    ASTReader::PerFileData &F;
+    Module &F;
     llvm::BitstreamCursor &Cursor;
     const DeclID ThisDeclID;
     typedef ASTReader::RecordData RecordData;
@@ -43,23 +44,42 @@ namespace clang {
     DeclID LexicalDeclContextIDForTemplateParmDecl;
 
     uint64_t GetCurrentCursorOffset();
+    
     SourceLocation ReadSourceLocation(const RecordData &R, unsigned &I) {
       return Reader.ReadSourceLocation(F, R, I);
     }
+    
     SourceRange ReadSourceRange(const RecordData &R, unsigned &I) {
       return Reader.ReadSourceRange(F, R, I);
     }
+    
     TypeSourceInfo *GetTypeSourceInfo(const RecordData &R, unsigned &I) {
       return Reader.GetTypeSourceInfo(F, R, I);
     }
+    
+    serialization::DeclID ReadDeclID(const RecordData &R, unsigned &I) {
+      return Reader.ReadDeclID(F, R, I);
+    }
+    
+    Decl *ReadDecl(const RecordData &R, unsigned &I) {
+      return Reader.ReadDecl(F, R, I);
+    }
+
+    template<typename T>
+    T *ReadDeclAs(const RecordData &R, unsigned &I) {
+      return Reader.ReadDeclAs<T>(F, R, I);
+    }
+
     void ReadQualifierInfo(QualifierInfo &Info,
                            const RecordData &R, unsigned &I) {
       Reader.ReadQualifierInfo(F, Info, R, I);
     }
+    
     void ReadDeclarationNameLoc(DeclarationNameLoc &DNLoc, DeclarationName Name,
                                 const RecordData &R, unsigned &I) {
       Reader.ReadDeclarationNameLoc(F, DNLoc, Name, R, I);
     }
+    
     void ReadDeclarationNameInfo(DeclarationNameInfo &NameInfo,
                                 const RecordData &R, unsigned &I) {
       Reader.ReadDeclarationNameInfo(F, NameInfo, R, I);
@@ -72,7 +92,7 @@ namespace clang {
                                      CXXRecordDecl *DefinitionDecl,
                                      const RecordData &Record, unsigned &Idx);
   public:
-    ASTDeclReader(ASTReader &Reader, ASTReader::PerFileData &F,
+    ASTDeclReader(ASTReader &Reader, Module &F,
                   llvm::BitstreamCursor &Cursor, DeclID thisDeclID,
                   const RecordData &Record, unsigned &Idx)
       : Reader(Reader), F(F), Cursor(Cursor), ThisDeclID(thisDeclID),
@@ -82,9 +102,14 @@ namespace clang {
 
     void Visit(Decl *D);
 
-    void UpdateDecl(Decl *D, ASTReader::PerFileData &Module,
+    void UpdateDecl(Decl *D, Module &Module,
                     const RecordData &Record);
 
+    static void setNextObjCCategory(ObjCCategoryDecl *Cat,
+                                    ObjCCategoryDecl *Next) {
+      Cat->NextClassCategory = Next;
+    }
+
     void VisitDecl(Decl *D);
     void VisitTranslationUnitDecl(TranslationUnitDecl *TU);
     void VisitNamedDecl(NamedDecl *ND);
@@ -104,6 +129,8 @@ namespace clang {
                                             ClassTemplateSpecializationDecl *D);
     void VisitClassTemplatePartialSpecializationDecl(
                                      ClassTemplatePartialSpecializationDecl *D);
+    void VisitClassScopeFunctionSpecializationDecl(
+                                       ClassScopeFunctionSpecializationDecl *D);
     void VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D);
     void VisitValueDecl(ValueDecl *VD);
     void VisitEnumConstantDecl(EnumConstantDecl *ECD);
@@ -159,16 +186,7 @@ namespace clang {
 }
 
 uint64_t ASTDeclReader::GetCurrentCursorOffset() {
-  uint64_t Off = 0;
-  for (unsigned I = 0, N = Reader.Chain.size(); I != N; ++I) {
-    ASTReader::PerFileData &F = *Reader.Chain[N - I - 1];
-    if (&Cursor == &F.DeclsCursor) {
-      Off += F.DeclsCursor.GetCurrentBitNo();
-      break;
-    }
-    Off += F.SizeInBits;
-  }
-  return Off;
+  return F.DeclsCursor.GetCurrentBitNo() + F.GlobalBitOffset;
 }
 
 void ASTDeclReader::Visit(Decl *D) {
@@ -211,13 +229,12 @@ void ASTDeclReader::VisitDecl(Decl *D) {
     // parameter immediately, because the template parameter might be
     // used in the formulation of its DeclContext. Use the translation
     // unit DeclContext as a placeholder.
-    DeclContextIDForTemplateParmDecl = Record[Idx++];
-    LexicalDeclContextIDForTemplateParmDecl = Record[Idx++];
-    D->setDeclContext(Reader.getContext()->getTranslationUnitDecl()); 
+    DeclContextIDForTemplateParmDecl = ReadDeclID(Record, Idx);
+    LexicalDeclContextIDForTemplateParmDecl = ReadDeclID(Record, Idx);
+    D->setDeclContext(Reader.getContext().getTranslationUnitDecl()); 
   } else {
-    D->setDeclContext(cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++])));
-    D->setLexicalDeclContext(
-                     cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++])));
+    D->setDeclContext(ReadDeclAs<DeclContext>(Record, Idx));
+    D->setLexicalDeclContext(ReadDeclAs<DeclContext>(Record, Idx));
   }
   D->setLocation(ReadSourceLocation(Record, Idx));
   D->setInvalidDecl(Record[Idx++]);
@@ -230,25 +247,24 @@ void ASTDeclReader::VisitDecl(Decl *D) {
   D->setUsed(Record[Idx++]);
   D->setReferenced(Record[Idx++]);
   D->setAccess((AccessSpecifier)Record[Idx++]);
-  D->setPCHLevel(Record[Idx++] + (F.Type <= ASTReader::PCH));
+  D->FromASTFile = true;
+  D->ModulePrivate = Record[Idx++];
 }
 
 void ASTDeclReader::VisitTranslationUnitDecl(TranslationUnitDecl *TU) {
-  VisitDecl(TU);
-  TU->setAnonymousNamespace(
-                    cast_or_null<NamespaceDecl>(Reader.GetDecl(Record[Idx++])));
+  llvm_unreachable("Translation units are not serialized");
 }
 
 void ASTDeclReader::VisitNamedDecl(NamedDecl *ND) {
   VisitDecl(ND);
-  ND->setDeclName(Reader.ReadDeclarationName(Record, Idx));
+  ND->setDeclName(Reader.ReadDeclarationName(F, Record, Idx));
 }
 
 void ASTDeclReader::VisitTypeDecl(TypeDecl *TD) {
   VisitNamedDecl(TD);
   TD->setLocStart(ReadSourceLocation(Record, Idx));
   // Delay type reading until after we have fully initialized the decl.
-  TypeIDForTypeDecl = Record[Idx++];
+  TypeIDForTypeDecl = Reader.getGlobalTypeID(F, Record[Idx++]);
 }
 
 void ASTDeclReader::VisitTypedefDecl(TypedefDecl *TD) {
@@ -266,16 +282,16 @@ void ASTDeclReader::VisitTagDecl(TagDecl *TD) {
   VisitRedeclarable(TD);
   TD->IdentifierNamespace = Record[Idx++];
   TD->setTagKind((TagDecl::TagKind)Record[Idx++]);
-  TD->setDefinition(Record[Idx++]);
+  TD->setCompleteDefinition(Record[Idx++]);
   TD->setEmbeddedInDeclarator(Record[Idx++]);
+  TD->setFreeStanding(Record[Idx++]);
   TD->setRBraceLoc(ReadSourceLocation(Record, Idx));
   if (Record[Idx++]) { // hasExtInfo
-    TagDecl::ExtInfo *Info = new (*Reader.getContext()) TagDecl::ExtInfo();
+    TagDecl::ExtInfo *Info = new (Reader.getContext()) TagDecl::ExtInfo();
     ReadQualifierInfo(*Info, Record, Idx);
     TD->TypedefNameDeclOrQualifier = Info;
   } else
-    TD->setTypedefNameForAnonDecl(
-                  cast_or_null<TypedefNameDecl>(Reader.GetDecl(Record[Idx++])));
+    TD->setTypedefNameForAnonDecl(ReadDeclAs<TypedefNameDecl>(Record, Idx));
 }
 
 void ASTDeclReader::VisitEnumDecl(EnumDecl *ED) {
@@ -283,15 +299,14 @@ void ASTDeclReader::VisitEnumDecl(EnumDecl *ED) {
   if (TypeSourceInfo *TI = Reader.GetTypeSourceInfo(F, Record, Idx))
     ED->setIntegerTypeSourceInfo(TI);
   else
-    ED->setIntegerType(Reader.GetType(Record[Idx++]));
-  ED->setPromotionType(Reader.GetType(Record[Idx++]));
+    ED->setIntegerType(Reader.readType(F, Record, Idx));
+  ED->setPromotionType(Reader.readType(F, Record, Idx));
   ED->setNumPositiveBits(Record[Idx++]);
   ED->setNumNegativeBits(Record[Idx++]);
   ED->IsScoped = Record[Idx++];
   ED->IsScopedUsingClassTag = Record[Idx++];
   ED->IsFixed = Record[Idx++];
-  ED->setInstantiationOfMemberEnum(
-                         cast_or_null<EnumDecl>(Reader.GetDecl(Record[Idx++])));
+  ED->setInstantiationOfMemberEnum(ReadDeclAs<EnumDecl>(Record, Idx));
 }
 
 void ASTDeclReader::VisitRecordDecl(RecordDecl *RD) {
@@ -303,7 +318,7 @@ void ASTDeclReader::VisitRecordDecl(RecordDecl *RD) {
 
 void ASTDeclReader::VisitValueDecl(ValueDecl *VD) {
   VisitNamedDecl(VD);
-  VD->setType(Reader.GetType(Record[Idx++]));
+  VD->setType(Reader.readType(F, Record, Idx));
 }
 
 void ASTDeclReader::VisitEnumConstantDecl(EnumConstantDecl *ECD) {
@@ -318,7 +333,7 @@ void ASTDeclReader::VisitDeclaratorDecl(DeclaratorDecl *DD) {
   DD->setInnerLocStart(ReadSourceLocation(Record, Idx));
   if (Record[Idx++]) { // hasExtInfo
     DeclaratorDecl::ExtInfo *Info
-        = new (*Reader.getContext()) DeclaratorDecl::ExtInfo();
+        = new (Reader.getContext()) DeclaratorDecl::ExtInfo();
     ReadQualifierInfo(*Info, Record, Idx);
     DD->DeclInfo = Info;
   }
@@ -331,35 +346,35 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   ReadDeclarationNameLoc(FD->DNLoc, FD->getDeclName(), Record, Idx);
   FD->IdentifierNamespace = Record[Idx++];
   switch ((FunctionDecl::TemplatedKind)Record[Idx++]) {
-  default: assert(false && "Unhandled TemplatedKind!");
-    break;
+  default: llvm_unreachable("Unhandled TemplatedKind!");
   case FunctionDecl::TK_NonTemplate:
     break;
   case FunctionDecl::TK_FunctionTemplate:
-    FD->setDescribedFunctionTemplate(
-                     cast<FunctionTemplateDecl>(Reader.GetDecl(Record[Idx++])));
+    FD->setDescribedFunctionTemplate(ReadDeclAs<FunctionTemplateDecl>(Record, 
+                                                                      Idx));
     break;
   case FunctionDecl::TK_MemberSpecialization: {
-    FunctionDecl *InstFD = cast<FunctionDecl>(Reader.GetDecl(Record[Idx++]));
+    FunctionDecl *InstFD = ReadDeclAs<FunctionDecl>(Record, Idx);
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
     SourceLocation POI = ReadSourceLocation(Record, Idx);
-    FD->setInstantiationOfMemberFunction(*Reader.getContext(), InstFD, TSK);
+    FD->setInstantiationOfMemberFunction(Reader.getContext(), InstFD, TSK);
     FD->getMemberSpecializationInfo()->setPointOfInstantiation(POI);
     break;
   }
   case FunctionDecl::TK_FunctionTemplateSpecialization: {
-    FunctionTemplateDecl *Template
-      = cast<FunctionTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+    FunctionTemplateDecl *Template = ReadDeclAs<FunctionTemplateDecl>(Record, 
+                                                                      Idx);
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
     
     // Template arguments.
-    llvm::SmallVector<TemplateArgument, 8> TemplArgs;
+    SmallVector<TemplateArgument, 8> TemplArgs;
     Reader.ReadTemplateArgumentList(TemplArgs, F, Record, Idx);
     
     // Template args as written.
-    llvm::SmallVector<TemplateArgumentLoc, 8> TemplArgLocs;
+    SmallVector<TemplateArgumentLoc, 8> TemplArgLocs;
     SourceLocation LAngleLoc, RAngleLoc;
-    if (Record[Idx++]) {  // TemplateArgumentsAsWritten != 0
+    bool HasTemplateArgumentsAsWritten = Record[Idx++];
+    if (HasTemplateArgumentsAsWritten) {
       unsigned NumTemplateArgLocs = Record[Idx++];
       TemplArgLocs.reserve(NumTemplateArgLocs);
       for (unsigned i=0; i != NumTemplateArgLocs; ++i)
@@ -372,24 +387,24 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     
     SourceLocation POI = ReadSourceLocation(Record, Idx);
 
-    ASTContext &C = *Reader.getContext();
+    ASTContext &C = Reader.getContext();
     TemplateArgumentList *TemplArgList
       = TemplateArgumentList::CreateCopy(C, TemplArgs.data(), TemplArgs.size());
-    TemplateArgumentListInfo *TemplArgsInfo
-      = new (C) TemplateArgumentListInfo(LAngleLoc, RAngleLoc);
+    TemplateArgumentListInfo TemplArgsInfo(LAngleLoc, RAngleLoc);
     for (unsigned i=0, e = TemplArgLocs.size(); i != e; ++i)
-      TemplArgsInfo->addArgument(TemplArgLocs[i]);
+      TemplArgsInfo.addArgument(TemplArgLocs[i]);
     FunctionTemplateSpecializationInfo *FTInfo
         = FunctionTemplateSpecializationInfo::Create(C, FD, Template, TSK,
                                                      TemplArgList,
-                                                     TemplArgsInfo, POI);
+                             HasTemplateArgumentsAsWritten ? &TemplArgsInfo : 0,
+                                                     POI);
     FD->TemplateOrSpecialization = FTInfo;
 
     if (FD->isCanonicalDecl()) { // if canonical add to template's set.
       // The template that contains the specializations set. It's not safe to
       // use getCanonicalDecl on Template since it may still be initializing.
       FunctionTemplateDecl *CanonTemplate
-        = cast<FunctionTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+        = ReadDeclAs<FunctionTemplateDecl>(Record, Idx);
       // Get the InsertPos by FindNodeOrInsertPos() instead of calling
       // InsertNode(FTInfo) directly to avoid the getASTContext() call in
       // FunctionTemplateSpecializationInfo's Profile().
@@ -410,7 +425,7 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     UnresolvedSet<8> TemplDecls;
     unsigned NumTemplates = Record[Idx++];
     while (NumTemplates--)
-      TemplDecls.addDecl(cast<NamedDecl>(Reader.GetDecl(Record[Idx++])));
+      TemplDecls.addDecl(ReadDeclAs<NamedDecl>(Record, Idx));
     
     // Templates args.
     TemplateArgumentListInfo TemplArgs;
@@ -420,7 +435,7 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     TemplArgs.setLAngleLoc(ReadSourceLocation(Record, Idx));
     TemplArgs.setRAngleLoc(ReadSourceLocation(Record, Idx));
     
-    FD->setDependentTemplateSpecialization(*Reader.getContext(),
+    FD->setDependentTemplateSpecialization(Reader.getContext(),
                                            TemplDecls, TemplArgs);
     break;
   }
@@ -442,15 +457,16 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   FD->IsDefaulted = Record[Idx++];
   FD->IsExplicitlyDefaulted = Record[Idx++];
   FD->HasImplicitReturnZero = Record[Idx++];
+  FD->IsConstexpr = Record[Idx++];
   FD->EndRangeLoc = ReadSourceLocation(Record, Idx);
 
   // Read in the parameters.
   unsigned NumParams = Record[Idx++];
-  llvm::SmallVector<ParmVarDecl *, 16> Params;
+  SmallVector<ParmVarDecl *, 16> Params;
   Params.reserve(NumParams);
   for (unsigned I = 0; I != NumParams; ++I)
-    Params.push_back(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
-  FD->setParams(*Reader.getContext(), Params.data(), NumParams);
+    Params.push_back(ReadDeclAs<ParmVarDecl>(Record, Idx));
+  FD->setParams(Reader.getContext(), Params);
 }
 
 void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
@@ -459,77 +475,87 @@ void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
     // In practice, this won't be executed (since method definitions
     // don't occur in header files).
     MD->setBody(Reader.ReadStmt(F));
-    MD->setSelfDecl(cast<ImplicitParamDecl>(Reader.GetDecl(Record[Idx++])));
-    MD->setCmdDecl(cast<ImplicitParamDecl>(Reader.GetDecl(Record[Idx++])));
+    MD->setSelfDecl(ReadDeclAs<ImplicitParamDecl>(Record, Idx));
+    MD->setCmdDecl(ReadDeclAs<ImplicitParamDecl>(Record, Idx));
   }
   MD->setInstanceMethod(Record[Idx++]);
   MD->setVariadic(Record[Idx++]);
   MD->setSynthesized(Record[Idx++]);
   MD->setDefined(Record[Idx++]);
+
+  MD->IsRedeclaration = Record[Idx++];
+  MD->HasRedeclaration = Record[Idx++];
+  if (MD->HasRedeclaration)
+    Reader.getContext().setObjCMethodRedeclaration(MD,
+                                       ReadDeclAs<ObjCMethodDecl>(Record, Idx));
+
   MD->setDeclImplementation((ObjCMethodDecl::ImplementationControl)Record[Idx++]);
   MD->setObjCDeclQualifier((Decl::ObjCDeclQualifier)Record[Idx++]);
   MD->SetRelatedResultType(Record[Idx++]);
-  MD->setNumSelectorArgs(unsigned(Record[Idx++]));
-  MD->setResultType(Reader.GetType(Record[Idx++]));
+  MD->setResultType(Reader.readType(F, Record, Idx));
   MD->setResultTypeSourceInfo(GetTypeSourceInfo(Record, Idx));
   MD->setEndLoc(ReadSourceLocation(Record, Idx));
   unsigned NumParams = Record[Idx++];
-  llvm::SmallVector<ParmVarDecl *, 16> Params;
+  SmallVector<ParmVarDecl *, 16> Params;
   Params.reserve(NumParams);
   for (unsigned I = 0; I != NumParams; ++I)
-    Params.push_back(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
-  MD->setMethodParams(*Reader.getContext(), Params.data(), NumParams,
-                      NumParams);
+    Params.push_back(ReadDeclAs<ParmVarDecl>(Record, Idx));
+
+  MD->SelLocsKind = Record[Idx++];
+  unsigned NumStoredSelLocs = Record[Idx++];
+  SmallVector<SourceLocation, 16> SelLocs;
+  SelLocs.reserve(NumStoredSelLocs);
+  for (unsigned i = 0; i != NumStoredSelLocs; ++i)
+    SelLocs.push_back(ReadSourceLocation(Record, Idx));
+
+  MD->setParamsAndSelLocs(Reader.getContext(), Params, SelLocs);
 }
 
 void ASTDeclReader::VisitObjCContainerDecl(ObjCContainerDecl *CD) {
   VisitNamedDecl(CD);
-  SourceLocation A = ReadSourceLocation(Record, Idx);
-  SourceLocation B = ReadSourceLocation(Record, Idx);
-  CD->setAtEndRange(SourceRange(A, B));
+  CD->setAtStartLoc(ReadSourceLocation(Record, Idx));
+  CD->setAtEndRange(ReadSourceRange(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
   VisitObjCContainerDecl(ID);
-  ID->setTypeForDecl(Reader.GetType(Record[Idx++]).getTypePtrOrNull());
-  ID->setSuperClass(cast_or_null<ObjCInterfaceDecl>
-                       (Reader.GetDecl(Record[Idx++])));
+  ID->setTypeForDecl(Reader.readType(F, Record, Idx).getTypePtrOrNull());
+  ID->setSuperClass(ReadDeclAs<ObjCInterfaceDecl>(Record, Idx));
   
   // Read the directly referenced protocols and their SourceLocations.
   unsigned NumProtocols = Record[Idx++];
-  llvm::SmallVector<ObjCProtocolDecl *, 16> Protocols;
+  SmallVector<ObjCProtocolDecl *, 16> Protocols;
   Protocols.reserve(NumProtocols);
   for (unsigned I = 0; I != NumProtocols; ++I)
-    Protocols.push_back(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
-  llvm::SmallVector<SourceLocation, 16> ProtoLocs;
+    Protocols.push_back(ReadDeclAs<ObjCProtocolDecl>(Record, Idx));
+  SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtocols);
   for (unsigned I = 0; I != NumProtocols; ++I)
     ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   ID->setProtocolList(Protocols.data(), NumProtocols, ProtoLocs.data(),
-                      *Reader.getContext());
+                      Reader.getContext());
   
   // Read the transitive closure of protocols referenced by this class.
   NumProtocols = Record[Idx++];
   Protocols.clear();
   Protocols.reserve(NumProtocols);
   for (unsigned I = 0; I != NumProtocols; ++I)
-    Protocols.push_back(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
+    Protocols.push_back(ReadDeclAs<ObjCProtocolDecl>(Record, Idx));
   ID->AllReferencedProtocols.set(Protocols.data(), NumProtocols,
-                                 *Reader.getContext());
+                                 Reader.getContext());
   
   // Read the ivars.
   unsigned NumIvars = Record[Idx++];
-  llvm::SmallVector<ObjCIvarDecl *, 16> IVars;
+  SmallVector<ObjCIvarDecl *, 16> IVars;
   IVars.reserve(NumIvars);
   for (unsigned I = 0; I != NumIvars; ++I)
-    IVars.push_back(cast<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
-  ID->setCategoryList(
-               cast_or_null<ObjCCategoryDecl>(Reader.GetDecl(Record[Idx++])));
+    IVars.push_back(ReadDeclAs<ObjCIvarDecl>(Record, Idx));
+  ID->setCategoryList(ReadDeclAs<ObjCCategoryDecl>(Record, Idx));
+  
   // We will rebuild this list lazily.
   ID->setIvarList(0);
   ID->setForwardDecl(Record[Idx++]);
   ID->setImplicitInterfaceDecl(Record[Idx++]);
-  ID->setClassLoc(ReadSourceLocation(Record, Idx));
   ID->setSuperClassLoc(ReadSourceLocation(Record, Idx));
   ID->setLocEnd(ReadSourceLocation(Record, Idx));
 }
@@ -548,16 +574,16 @@ void ASTDeclReader::VisitObjCProtocolDecl(ObjCProtocolDecl *PD) {
   PD->setForwardDecl(Record[Idx++]);
   PD->setLocEnd(ReadSourceLocation(Record, Idx));
   unsigned NumProtoRefs = Record[Idx++];
-  llvm::SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
+  SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
   ProtoRefs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
-    ProtoRefs.push_back(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
-  llvm::SmallVector<SourceLocation, 16> ProtoLocs;
+    ProtoRefs.push_back(ReadDeclAs<ObjCProtocolDecl>(Record, Idx));
+  SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
     ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   PD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
-                      *Reader.getContext());
+                      Reader.getContext());
 }
 
 void ASTDeclReader::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *FD) {
@@ -566,57 +592,48 @@ void ASTDeclReader::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *FD) {
 
 void ASTDeclReader::VisitObjCClassDecl(ObjCClassDecl *CD) {
   VisitDecl(CD);
-  unsigned NumClassRefs = Record[Idx++];
-  llvm::SmallVector<ObjCInterfaceDecl *, 16> ClassRefs;
-  ClassRefs.reserve(NumClassRefs);
-  for (unsigned I = 0; I != NumClassRefs; ++I)
-    ClassRefs.push_back(cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
-  llvm::SmallVector<SourceLocation, 16> SLocs;
-  SLocs.reserve(NumClassRefs);
-  for (unsigned I = 0; I != NumClassRefs; ++I)
-    SLocs.push_back(ReadSourceLocation(Record, Idx));
-  CD->setClassList(*Reader.getContext(), ClassRefs.data(), SLocs.data(),
-                   NumClassRefs);
+  ObjCInterfaceDecl *ClassRef = ReadDeclAs<ObjCInterfaceDecl>(Record, Idx);
+  SourceLocation SLoc = ReadSourceLocation(Record, Idx);
+  CD->setClass(Reader.getContext(), ClassRef, SLoc);
 }
 
 void ASTDeclReader::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *FPD) {
   VisitDecl(FPD);
   unsigned NumProtoRefs = Record[Idx++];
-  llvm::SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
+  SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
   ProtoRefs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
-    ProtoRefs.push_back(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
-  llvm::SmallVector<SourceLocation, 16> ProtoLocs;
+    ProtoRefs.push_back(ReadDeclAs<ObjCProtocolDecl>(Record, Idx));
+  SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
     ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   FPD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
-                       *Reader.getContext());
+                       Reader.getContext());
 }
 
 void ASTDeclReader::VisitObjCCategoryDecl(ObjCCategoryDecl *CD) {
   VisitObjCContainerDecl(CD);
-  CD->setClassInterface(cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
+  CD->ClassInterface = ReadDeclAs<ObjCInterfaceDecl>(Record, Idx);
   unsigned NumProtoRefs = Record[Idx++];
-  llvm::SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
+  SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
   ProtoRefs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
-    ProtoRefs.push_back(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
-  llvm::SmallVector<SourceLocation, 16> ProtoLocs;
+    ProtoRefs.push_back(ReadDeclAs<ObjCProtocolDecl>(Record, Idx));
+  SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
     ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   CD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
-                      *Reader.getContext());
-  CD->setNextClassCategory(cast_or_null<ObjCCategoryDecl>(Reader.GetDecl(Record[Idx++])));
+                      Reader.getContext());
+  CD->NextClassCategory = ReadDeclAs<ObjCCategoryDecl>(Record, Idx);
   CD->setHasSynthBitfield(Record[Idx++]);
-  CD->setAtLoc(ReadSourceLocation(Record, Idx));
   CD->setCategoryNameLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *CAD) {
   VisitNamedDecl(CAD);
-  CAD->setClassInterface(cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
+  CAD->setClassInterface(ReadDeclAs<ObjCInterfaceDecl>(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
@@ -631,31 +648,26 @@ void ASTDeclReader::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   // FIXME: stable encoding
   D->setPropertyImplementation(
                             (ObjCPropertyDecl::PropertyControl)Record[Idx++]);
-  D->setGetterName(Reader.ReadDeclarationName(Record, Idx).getObjCSelector());
-  D->setSetterName(Reader.ReadDeclarationName(Record, Idx).getObjCSelector());
-  D->setGetterMethodDecl(
-                 cast_or_null<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++])));
-  D->setSetterMethodDecl(
-                 cast_or_null<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++])));
-  D->setPropertyIvarDecl(
-                   cast_or_null<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
+  D->setGetterName(Reader.ReadDeclarationName(F,Record, Idx).getObjCSelector());
+  D->setSetterName(Reader.ReadDeclarationName(F,Record, Idx).getObjCSelector());
+  D->setGetterMethodDecl(ReadDeclAs<ObjCMethodDecl>(Record, Idx));
+  D->setSetterMethodDecl(ReadDeclAs<ObjCMethodDecl>(Record, Idx));
+  D->setPropertyIvarDecl(ReadDeclAs<ObjCIvarDecl>(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCImplDecl(ObjCImplDecl *D) {
   VisitObjCContainerDecl(D);
-  D->setClassInterface(
-              cast_or_null<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
+  D->setClassInterface(ReadDeclAs<ObjCInterfaceDecl>(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
   VisitObjCImplDecl(D);
-  D->setIdentifier(Reader.GetIdentifierInfo(Record, Idx));
+  D->setIdentifier(Reader.GetIdentifierInfo(F, Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   VisitObjCImplDecl(D);
-  D->setSuperClass(
-              cast_or_null<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
+  D->setSuperClass(ReadDeclAs<ObjCInterfaceDecl>(Record, Idx));
   llvm::tie(D->IvarInitializers, D->NumIvarInitializers)
       = Reader.ReadCXXCtorInitializers(F, Record, Idx);
   D->setHasSynthBitfield(Record[Idx++]);
@@ -665,10 +677,8 @@ void ASTDeclReader::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
 void ASTDeclReader::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
   VisitDecl(D);
   D->setAtLoc(ReadSourceLocation(Record, Idx));
-  D->setPropertyDecl(
-               cast_or_null<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
-  D->PropertyIvarDecl = 
-                   cast_or_null<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++]));
+  D->setPropertyDecl(ReadDeclAs<ObjCPropertyDecl>(Record, Idx));
+  D->PropertyIvarDecl = ReadDeclAs<ObjCIvarDecl>(Record, Idx);
   D->IvarLoc = ReadSourceLocation(Record, Idx);
   D->setGetterCXXConstructor(Reader.ReadExpr(F));
   D->setSetterCXXAssignment(Reader.ReadExpr(F));
@@ -683,9 +693,8 @@ void ASTDeclReader::VisitFieldDecl(FieldDecl *FD) {
   else if (BitWidthOrInitializer == 2)
     FD->setInClassInitializer(Reader.ReadExpr(F));
   if (!FD->getDeclName()) {
-    FieldDecl *Tmpl = cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++]));
-    if (Tmpl)
-      Reader.getContext()->setInstantiatedFromUnnamedFieldDecl(FD, Tmpl);
+    if (FieldDecl *Tmpl = ReadDeclAs<FieldDecl>(Record, Idx))
+      Reader.getContext().setInstantiatedFromUnnamedFieldDecl(FD, Tmpl);
   }
 }
 
@@ -694,10 +703,10 @@ void ASTDeclReader::VisitIndirectFieldDecl(IndirectFieldDecl *FD) {
 
   FD->ChainingSize = Record[Idx++];
   assert(FD->ChainingSize >= 2 && "Anonymous chaining must be >= 2");
-  FD->Chaining = new (*Reader.getContext())NamedDecl*[FD->ChainingSize];
+  FD->Chaining = new (Reader.getContext())NamedDecl*[FD->ChainingSize];
 
   for (unsigned I = 0; I != FD->ChainingSize; ++I)
-    FD->Chaining[I] = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+    FD->Chaining[I] = ReadDeclAs<NamedDecl>(Record, Idx);
 }
 
 void ASTDeclReader::VisitVarDecl(VarDecl *VD) {
@@ -715,10 +724,10 @@ void ASTDeclReader::VisitVarDecl(VarDecl *VD) {
     VD->setInit(Reader.ReadExpr(F));
 
   if (Record[Idx++]) { // HasMemberSpecializationInfo.
-    VarDecl *Tmpl = cast<VarDecl>(Reader.GetDecl(Record[Idx++]));
+    VarDecl *Tmpl = ReadDeclAs<VarDecl>(Record, Idx);
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
     SourceLocation POI = ReadSourceLocation(Record, Idx);
-    Reader.getContext()->setInstantiatedFromStaticDataMember(VD, Tmpl, TSK,POI);
+    Reader.getContext().setInstantiatedFromStaticDataMember(VD, Tmpl, TSK,POI);
   }
 }
 
@@ -756,18 +765,18 @@ void ASTDeclReader::VisitBlockDecl(BlockDecl *BD) {
   BD->setBody(cast_or_null<CompoundStmt>(Reader.ReadStmt(F)));
   BD->setSignatureAsWritten(GetTypeSourceInfo(Record, Idx));
   unsigned NumParams = Record[Idx++];
-  llvm::SmallVector<ParmVarDecl *, 16> Params;
+  SmallVector<ParmVarDecl *, 16> Params;
   Params.reserve(NumParams);
   for (unsigned I = 0; I != NumParams; ++I)
-    Params.push_back(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
-  BD->setParams(Params.data(), NumParams);
+    Params.push_back(ReadDeclAs<ParmVarDecl>(Record, Idx));
+  BD->setParams(Params);
 
   bool capturesCXXThis = Record[Idx++];
   unsigned numCaptures = Record[Idx++];
-  llvm::SmallVector<BlockDecl::Capture, 16> captures;
+  SmallVector<BlockDecl::Capture, 16> captures;
   captures.reserve(numCaptures);
   for (unsigned i = 0; i != numCaptures; ++i) {
-    VarDecl *decl = cast<VarDecl>(Reader.GetDecl(Record[Idx++]));
+    VarDecl *decl = ReadDeclAs<VarDecl>(Record, Idx);
     unsigned flags = Record[Idx++];
     bool byRef = (flags & 1);
     bool nested = (flags & 2);
@@ -775,7 +784,7 @@ void ASTDeclReader::VisitBlockDecl(BlockDecl *BD) {
 
     captures.push_back(BlockDecl::Capture(decl, byRef, nested, copyExpr));
   }
-  BD->setCaptures(*Reader.getContext(), captures.begin(),
+  BD->setCaptures(Reader.getContext(), captures.begin(),
                   captures.end(), capturesCXXThis);
 }
 
@@ -800,9 +809,10 @@ void ASTDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
   D->NextNamespace = Record[Idx++];
 
   bool IsOriginal = Record[Idx++];
+  // FIXME: Modules will likely have trouble with pointing directly at
+  // the original namespace.
   D->OrigOrAnonNamespace.setInt(IsOriginal);
-  D->OrigOrAnonNamespace.setPointer(
-                    cast_or_null<NamespaceDecl>(Reader.GetDecl(Record[Idx++])));
+  D->OrigOrAnonNamespace.setPointer(ReadDeclAs<NamespaceDecl>(Record, Idx));
 }
 
 void ASTDeclReader::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
@@ -810,7 +820,7 @@ void ASTDeclReader::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
   D->NamespaceLoc = ReadSourceLocation(Record, Idx);
   D->IdentLoc = ReadSourceLocation(Record, Idx);
   D->QualifierLoc = Reader.ReadNestedNameSpecifierLoc(F, Record, Idx);
-  D->Namespace = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+  D->Namespace = ReadDeclAs<NamedDecl>(Record, Idx);
 }
 
 void ASTDeclReader::VisitUsingDecl(UsingDecl *D) {
@@ -818,21 +828,19 @@ void ASTDeclReader::VisitUsingDecl(UsingDecl *D) {
   D->setUsingLocation(ReadSourceLocation(Record, Idx));
   D->QualifierLoc = Reader.ReadNestedNameSpecifierLoc(F, Record, Idx);
   ReadDeclarationNameLoc(D->DNLoc, D->getDeclName(), Record, Idx);
-  D->FirstUsingShadow = cast_or_null<UsingShadowDecl>(Reader.GetDecl(Record[Idx++]));
+  D->FirstUsingShadow = ReadDeclAs<UsingShadowDecl>(Record, Idx);
   D->setTypeName(Record[Idx++]);
-  NamedDecl *Pattern = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
-  if (Pattern)
-    Reader.getContext()->setInstantiatedFromUsingDecl(D, Pattern);
+  if (NamedDecl *Pattern = ReadDeclAs<NamedDecl>(Record, Idx))
+    Reader.getContext().setInstantiatedFromUsingDecl(D, Pattern);
 }
 
 void ASTDeclReader::VisitUsingShadowDecl(UsingShadowDecl *D) {
   VisitNamedDecl(D);
-  D->setTargetDecl(cast<NamedDecl>(Reader.GetDecl(Record[Idx++])));
-  D->UsingOrNextShadow = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
-  UsingShadowDecl *Pattern
-      = cast_or_null<UsingShadowDecl>(Reader.GetDecl(Record[Idx++]));
+  D->setTargetDecl(ReadDeclAs<NamedDecl>(Record, Idx));
+  D->UsingOrNextShadow = ReadDeclAs<NamedDecl>(Record, Idx);
+  UsingShadowDecl *Pattern = ReadDeclAs<UsingShadowDecl>(Record, Idx);
   if (Pattern)
-    Reader.getContext()->setInstantiatedFromUsingShadowDecl(D, Pattern);
+    Reader.getContext().setInstantiatedFromUsingShadowDecl(D, Pattern);
 }
 
 void ASTDeclReader::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
@@ -840,8 +848,8 @@ void ASTDeclReader::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   D->UsingLoc = ReadSourceLocation(Record, Idx);
   D->NamespaceLoc = ReadSourceLocation(Record, Idx);
   D->QualifierLoc = Reader.ReadNestedNameSpecifierLoc(F, Record, Idx);
-  D->NominatedNamespace = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
-  D->CommonAncestor = cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++]));
+  D->NominatedNamespace = ReadDeclAs<NamedDecl>(Record, Idx);
+  D->CommonAncestor = ReadDeclAs<DeclContext>(Record, Idx);
 }
 
 void ASTDeclReader::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
@@ -863,7 +871,9 @@ void ASTDeclReader::ReadCXXDefinitionData(
                                    const RecordData &Record, unsigned &Idx) {
   Data.UserDeclaredConstructor = Record[Idx++];
   Data.UserDeclaredCopyConstructor = Record[Idx++];
+  Data.UserDeclaredMoveConstructor = Record[Idx++];
   Data.UserDeclaredCopyAssignment = Record[Idx++];
+  Data.UserDeclaredMoveAssignment = Record[Idx++];
   Data.UserDeclaredDestructor = Record[Idx++];
   Data.Aggregate = Record[Idx++];
   Data.PlainOldData = Record[Idx++];
@@ -877,7 +887,7 @@ void ASTDeclReader::ReadCXXDefinitionData(
   Data.HasPublicFields = Record[Idx++];
   Data.HasMutableFields = Record[Idx++];
   Data.HasTrivialDefaultConstructor = Record[Idx++];
-  Data.HasConstExprNonCopyMoveConstructor = Record[Idx++];
+  Data.HasConstexprNonCopyMoveConstructor = Record[Idx++];
   Data.HasTrivialCopyConstructor = Record[Idx++];
   Data.HasTrivialMoveConstructor = Record[Idx++];
   Data.HasTrivialCopyAssignment = Record[Idx++];
@@ -888,28 +898,31 @@ void ASTDeclReader::ReadCXXDefinitionData(
   Data.UserProvidedDefaultConstructor = Record[Idx++];
   Data.DeclaredDefaultConstructor = Record[Idx++];
   Data.DeclaredCopyConstructor = Record[Idx++];
+  Data.DeclaredMoveConstructor = Record[Idx++];
   Data.DeclaredCopyAssignment = Record[Idx++];
+  Data.DeclaredMoveAssignment = Record[Idx++];
   Data.DeclaredDestructor = Record[Idx++];
+  Data.FailedImplicitMoveConstructor = Record[Idx++];
+  Data.FailedImplicitMoveAssignment = Record[Idx++];
 
   Data.NumBases = Record[Idx++];
   if (Data.NumBases)
-    Data.Bases = Reader.GetCXXBaseSpecifiersOffset(Record[Idx++]);
+    Data.Bases = Reader.readCXXBaseSpecifiers(F, Record, Idx);
   Data.NumVBases = Record[Idx++];
   if (Data.NumVBases)
-    Data.VBases = Reader.GetCXXBaseSpecifiersOffset(Record[Idx++]);
+    Data.VBases = Reader.readCXXBaseSpecifiers(F, Record, Idx);
   
-  Reader.ReadUnresolvedSet(Data.Conversions, Record, Idx);
-  Reader.ReadUnresolvedSet(Data.VisibleConversions, Record, Idx);
+  Reader.ReadUnresolvedSet(F, Data.Conversions, Record, Idx);
+  Reader.ReadUnresolvedSet(F, Data.VisibleConversions, Record, Idx);
   assert(Data.Definition && "Data.Definition should be already set!");
-  Data.FirstFriend
-      = cast_or_null<FriendDecl>(Reader.GetDecl(Record[Idx++]));
+  Data.FirstFriend = ReadDeclAs<FriendDecl>(Record, Idx);
 }
 
 void ASTDeclReader::InitializeCXXDefinitionData(CXXRecordDecl *D,
                                                 CXXRecordDecl *DefinitionDecl,
                                                 const RecordData &Record,
                                                 unsigned &Idx) {
-  ASTContext &C = *Reader.getContext();
+  ASTContext &C = Reader.getContext();
 
   if (D == DefinitionDecl) {
     D->DefinitionData = new (C) struct CXXRecordDecl::DefinitionData(D);
@@ -942,26 +955,24 @@ void ASTDeclReader::InitializeCXXDefinitionData(CXXRecordDecl *D,
 void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   VisitRecordDecl(D);
 
-  CXXRecordDecl *DefinitionDecl
-      = cast_or_null<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
+  CXXRecordDecl *DefinitionDecl = ReadDeclAs<CXXRecordDecl>(Record, Idx);
   InitializeCXXDefinitionData(D, DefinitionDecl, Record, Idx);
 
-  ASTContext &C = *Reader.getContext();
+  ASTContext &C = Reader.getContext();
 
   enum CXXRecKind {
     CXXRecNotTemplate = 0, CXXRecTemplate, CXXRecMemberSpecialization
   };
   switch ((CXXRecKind)Record[Idx++]) {
   default:
-    assert(false && "Out of sync with ASTDeclWriter::VisitCXXRecordDecl?");
+    llvm_unreachable("Out of sync with ASTDeclWriter::VisitCXXRecordDecl?");
   case CXXRecNotTemplate:
     break;
   case CXXRecTemplate:
-    D->TemplateOrInstantiation
-        = cast<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+    D->TemplateOrInstantiation = ReadDeclAs<ClassTemplateDecl>(Record, Idx);
     break;
   case CXXRecMemberSpecialization: {
-    CXXRecordDecl *RD = cast<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
+    CXXRecordDecl *RD = ReadDeclAs<CXXRecordDecl>(Record, Idx);
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
     SourceLocation POI = ReadSourceLocation(Record, Idx);
     MemberSpecializationInfo *MSI = new (C) MemberSpecializationInfo(RD, TSK);
@@ -973,10 +984,8 @@ void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
 
   // Load the key function to avoid deserializing every method so we can
   // compute it.
-  if (D->IsDefinition) {
-    CXXMethodDecl *Key
-        = cast_or_null<CXXMethodDecl>(Reader.GetDecl(Record[Idx++]));
-    if (Key)
+  if (D->IsCompleteDefinition) {
+    if (CXXMethodDecl *Key = ReadDeclAs<CXXMethodDecl>(Record, Idx))
       C.KeyFunctions[D] = Key;
   }
 }
@@ -985,10 +994,10 @@ void ASTDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
   VisitFunctionDecl(D);
   unsigned NumOverridenMethods = Record[Idx++];
   while (NumOverridenMethods--) {
-    CXXMethodDecl *MD = cast<CXXMethodDecl>(Reader.GetDecl(Record[Idx++]));
     // Avoid invariant checking of CXXMethodDecl::addOverriddenMethod,
     // MD may be initializing.
-    Reader.getContext()->addOverriddenMethod(D, MD);
+    if (CXXMethodDecl *MD = ReadDeclAs<CXXMethodDecl>(Record, Idx))
+      Reader.getContext().addOverriddenMethod(D, MD);
   }
 }
 
@@ -1005,7 +1014,7 @@ void ASTDeclReader::VisitCXXDestructorDecl(CXXDestructorDecl *D) {
   VisitCXXMethodDecl(D);
 
   D->ImplicitlyDefined = Record[Idx++];
-  D->OperatorDelete = cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++]));
+  D->OperatorDelete = ReadDeclAs<FunctionDecl>(Record, Idx);
 }
 
 void ASTDeclReader::VisitCXXConversionDecl(CXXConversionDecl *D) {
@@ -1023,7 +1032,7 @@ void ASTDeclReader::VisitFriendDecl(FriendDecl *D) {
   if (Record[Idx++])
     D->Friend = GetTypeSourceInfo(Record, Idx);
   else
-    D->Friend = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+    D->Friend = ReadDeclAs<NamedDecl>(Record, Idx);
   D->NextFriend = Record[Idx++];
   D->UnsupportedFriend = (Record[Idx++] != 0);
   D->FriendLoc = ReadSourceLocation(Record, Idx);
@@ -1037,7 +1046,7 @@ void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
   for (unsigned i = 0; i != NumParams; ++i)
     D->Params[i] = Reader.ReadTemplateParameterList(F, Record, Idx);
   if (Record[Idx++]) // HasFriendDecl
-    D->Friend = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+    D->Friend = ReadDeclAs<NamedDecl>(Record, Idx);
   else
     D->Friend = GetTypeSourceInfo(Record, Idx);
   D->FriendLoc = ReadSourceLocation(Record, Idx);
@@ -1046,8 +1055,7 @@ void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
 void ASTDeclReader::VisitTemplateDecl(TemplateDecl *D) {
   VisitNamedDecl(D);
 
-  NamedDecl *TemplatedDecl
-    = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+  NamedDecl *TemplatedDecl = ReadDeclAs<NamedDecl>(Record, Idx);
   TemplateParameterList* TemplateParams
       = Reader.ReadTemplateParameterList(F, Record, Idx); 
   D->init(TemplatedDecl, TemplateParams);
@@ -1058,8 +1066,8 @@ void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
   // can be used while this is still initializing.
 
   assert(D->CommonOrPrev.isNull() && "getCommonPtr was called earlier on this");
-  DeclID PreviousDeclID = Record[Idx++];
-  DeclID FirstDeclID =  PreviousDeclID ? Record[Idx++] : 0;
+  DeclID PreviousDeclID = ReadDeclID(Record, Idx);
+  DeclID FirstDeclID =  PreviousDeclID ? ReadDeclID(Record, Idx) : 0;
   // We delay loading of the redeclaration chain to avoid deeply nested calls.
   // We temporarily set the first (canonical) declaration as the previous one
   // which is the one that matters and mark the real previous DeclID to be
@@ -1074,9 +1082,9 @@ void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
     if (PreviousDeclID != FirstDeclID)
       Reader.PendingPreviousDecls.push_back(std::make_pair(D, PreviousDeclID));
   } else {
-    D->CommonOrPrev = D->newCommon(*Reader.getContext());
+    D->CommonOrPrev = D->newCommon(Reader.getContext());
     if (RedeclarableTemplateDecl *RTD
-          = cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]))) {
+          = ReadDeclAs<RedeclarableTemplateDecl>(Record, Idx)) {
       assert(RTD->getKind() == D->getKind() &&
              "InstantiatedFromMemberTemplate kind mismatch");
       D->setInstantiatedFromMemberTemplateImpl(RTD);
@@ -1084,8 +1092,8 @@ void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
         D->setMemberSpecialization();
     }
 
-    RedeclarableTemplateDecl *LatestDecl = 
-        cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+    RedeclarableTemplateDecl *LatestDecl
+      = ReadDeclAs<RedeclarableTemplateDecl>(Record, Idx);
   
     // This decl is a first one and the latest declaration that it points to is
     // in the same AST file. However, if this actually needs to point to a
@@ -1095,14 +1103,8 @@ void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
     ASTReader::FirstLatestDeclIDMap::iterator I
         = Reader.FirstLatestDeclIDs.find(ThisDeclID);
     if (I != Reader.FirstLatestDeclIDs.end()) {
-      Decl *NewLatest = Reader.GetDecl(I->second);
-      assert((LatestDecl->getLocation().isInvalid() ||
-              NewLatest->getLocation().isInvalid()  ||
-              !Reader.SourceMgr.isBeforeInTranslationUnit(
-                                                  NewLatest->getLocation(),
-                                                  LatestDecl->getLocation())) &&
-             "The new latest is supposed to come after the previous latest");
-      LatestDecl = cast<RedeclarableTemplateDecl>(NewLatest);
+      if (Decl *NewLatest = Reader.GetDecl(I->second))
+        LatestDecl = cast<RedeclarableTemplateDecl>(NewLatest);
     }
 
     assert(LatestDecl->getKind() == D->getKind() && "Latest kind mismatch");
@@ -1119,27 +1121,27 @@ void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   if (D->getPreviousDeclaration() == 0) {
     // This ClassTemplateDecl owns a CommonPtr; read it to keep track of all of
     // the specializations.
-    llvm::SmallVector<serialization::DeclID, 2> SpecIDs;
+    SmallVector<serialization::DeclID, 2> SpecIDs;
     SpecIDs.push_back(0);
     
     // Specializations.
     unsigned Size = Record[Idx++];
     SpecIDs[0] += Size;
-    SpecIDs.append(Record.begin() + Idx, Record.begin() + Idx + Size);
-    Idx += Size;
+    for (unsigned I = 0; I != Size; ++I)
+      SpecIDs.push_back(ReadDeclID(Record, Idx));
 
     // Partial specializations.
     Size = Record[Idx++];
     SpecIDs[0] += Size;
-    SpecIDs.append(Record.begin() + Idx, Record.begin() + Idx + Size);
-    Idx += Size;
+    for (unsigned I = 0; I != Size; ++I)
+      SpecIDs.push_back(ReadDeclID(Record, Idx));
 
     if (SpecIDs[0]) {
       typedef serialization::DeclID DeclID;
       
       ClassTemplateDecl::Common *CommonPtr = D->getCommonPtr();
       CommonPtr->LazySpecializations
-        = new (*Reader.getContext()) DeclID [SpecIDs.size()];
+        = new (Reader.getContext()) DeclID [SpecIDs.size()];
       memcpy(CommonPtr->LazySpecializations, SpecIDs.data(), 
              SpecIDs.size() * sizeof(DeclID));
     }
@@ -1152,12 +1154,12 @@ void ASTDeclReader::VisitClassTemplateSpecializationDecl(
                                            ClassTemplateSpecializationDecl *D) {
   VisitCXXRecordDecl(D);
   
-  ASTContext &C = *Reader.getContext();
-  if (Decl *InstD = Reader.GetDecl(Record[Idx++])) {
+  ASTContext &C = Reader.getContext();
+  if (Decl *InstD = ReadDecl(Record, Idx)) {
     if (ClassTemplateDecl *CTD = dyn_cast<ClassTemplateDecl>(InstD)) {
       D->SpecializedTemplate = CTD;
     } else {
-      llvm::SmallVector<TemplateArgument, 8> TemplArgs;
+      SmallVector<TemplateArgument, 8> TemplArgs;
       Reader.ReadTemplateArgumentList(TemplArgs, F, Record, Idx);
       TemplateArgumentList *ArgList
         = TemplateArgumentList::CreateCopy(C, TemplArgs.data(), 
@@ -1182,7 +1184,7 @@ void ASTDeclReader::VisitClassTemplateSpecializationDecl(
     D->ExplicitInfo = ExplicitInfo;
   }
 
-  llvm::SmallVector<TemplateArgument, 8> TemplArgs;
+  SmallVector<TemplateArgument, 8> TemplArgs;
   Reader.ReadTemplateArgumentList(TemplArgs, F, Record, Idx);
   D->TemplateArgs = TemplateArgumentList::CreateCopy(C, TemplArgs.data(), 
                                                      TemplArgs.size());
@@ -1190,8 +1192,7 @@ void ASTDeclReader::VisitClassTemplateSpecializationDecl(
   D->SpecializationKind = (TemplateSpecializationKind)Record[Idx++];
   
   if (D->isCanonicalDecl()) { // It's kept in the folding set.
-    ClassTemplateDecl *CanonPattern
-                       = cast<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+    ClassTemplateDecl *CanonPattern = ReadDeclAs<ClassTemplateDecl>(Record,Idx);
     if (ClassTemplatePartialSpecializationDecl *Partial
                        = dyn_cast<ClassTemplatePartialSpecializationDecl>(D)) {
       CanonPattern->getCommonPtr()->PartialSpecializations.InsertNode(Partial);
@@ -1205,7 +1206,7 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   VisitClassTemplateSpecializationDecl(D);
 
-  ASTContext &C = *Reader.getContext();
+  ASTContext &C = Reader.getContext();
   D->TemplateParams = Reader.ReadTemplateParameterList(F, Record, Idx);
 
   unsigned NumArgs = Record[Idx++];
@@ -1221,12 +1222,17 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
   // These are read/set from/to the first declaration.
   if (D->getPreviousDeclaration() == 0) {
     D->InstantiatedFromMember.setPointer(
-        cast_or_null<ClassTemplatePartialSpecializationDecl>(
-                                                Reader.GetDecl(Record[Idx++])));
+      ReadDeclAs<ClassTemplatePartialSpecializationDecl>(Record, Idx));
     D->InstantiatedFromMember.setInt(Record[Idx++]);
   }
 }
 
+void ASTDeclReader::VisitClassScopeFunctionSpecializationDecl(
+                                    ClassScopeFunctionSpecializationDecl *D) {
+  VisitDecl(D);
+  D->Specialization = ReadDeclAs<CXXMethodDecl>(Record, Idx);
+}
+
 void ASTDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   VisitRedeclarableTemplateDecl(D);
 
@@ -1238,7 +1244,7 @@ void ASTDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
     // when reading the specialized FunctionDecl.
     unsigned NumSpecs = Record[Idx++];
     while (NumSpecs--)
-      Reader.GetDecl(Record[Idx++]);
+      (void)ReadDecl(Record, Idx);
   }
 }
 
@@ -1260,7 +1266,7 @@ void ASTDeclReader::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
   if (D->isExpandedParameterPack()) {
     void **Data = reinterpret_cast<void **>(D + 1);
     for (unsigned I = 0, N = D->getNumExpansionTypes(); I != N; ++I) {
-      Data[2*I] = Reader.GetType(Record[Idx++]).getAsOpaquePtr();
+      Data[2*I] = Reader.readType(F, Record, Idx).getAsOpaquePtr();
       Data[2*I + 1] = GetTypeSourceInfo(Record, Idx);
     }
   } else {
@@ -1310,13 +1316,13 @@ void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
   RedeclKind Kind = (RedeclKind)Record[Idx++];
   switch (Kind) {
   default:
-    assert(0 && "Out of sync with ASTDeclWriter::VisitRedeclarable or messed up"
-                " reading");
+    llvm_unreachable("Out of sync with ASTDeclWriter::VisitRedeclarable or"
+                     " messed up reading");
   case NoRedeclaration:
     break;
   case PointsToPrevious: {
-    DeclID PreviousDeclID = Record[Idx++];
-    DeclID FirstDeclID = Record[Idx++];
+    DeclID PreviousDeclID = ReadDeclID(Record, Idx);
+    DeclID FirstDeclID = ReadDeclID(Record, Idx);
     // We delay loading of the redeclaration chain to avoid deeply nested calls.
     // We temporarily set the first (canonical) declaration as the previous one
     // which is the one that matters and mark the real previous DeclID to be
@@ -1330,7 +1336,7 @@ void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
   }
   case PointsToLatest:
     D->RedeclLink = typename Redeclarable<T>::LatestDeclLink(
-                                cast_or_null<T>(Reader.GetDecl(Record[Idx++])));
+                                                   ReadDeclAs<T>(Record, Idx));
     break;
   }
 
@@ -1361,12 +1367,12 @@ void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
 //===----------------------------------------------------------------------===//
 
 /// \brief Reads attributes from the current stream position.
-void ASTReader::ReadAttributes(PerFileData &F, AttrVec &Attrs,
+void ASTReader::ReadAttributes(Module &F, AttrVec &Attrs,
                                const RecordData &Record, unsigned &Idx) {
   for (unsigned i = 0, e = Record[Idx++]; i != e; ++i) {
     Attr *New = 0;
     attr::Kind Kind = (attr::Kind)Record[Idx++];
-    SourceLocation Loc = ReadSourceLocation(F, Record, Idx);
+    SourceRange Range = ReadSourceRange(F, Record, Idx);
 
 #include "clang/Serialization/AttrPCHRead.inc"
 
@@ -1398,33 +1404,47 @@ inline void ASTReader::LoadedDecl(unsigned Index, Decl *D) {
 /// code generation, e.g., inline function definitions, Objective-C
 /// declarations with metadata, etc.
 static bool isConsumerInterestedIn(Decl *D) {
-  if (isa<FileScopeAsmDecl>(D))
+  // An ObjCMethodDecl is never considered as "interesting" because its
+  // implementation container always is.
+
+  if (isa<FileScopeAsmDecl>(D) || 
+      isa<ObjCProtocolDecl>(D) || 
+      isa<ObjCImplDecl>(D))
     return true;
   if (VarDecl *Var = dyn_cast<VarDecl>(D))
     return Var->isFileVarDecl() &&
            Var->isThisDeclarationADefinition() == VarDecl::Definition;
   if (FunctionDecl *Func = dyn_cast<FunctionDecl>(D))
     return Func->doesThisDeclarationHaveABody();
-  return isa<ObjCProtocolDecl>(D) || isa<ObjCImplementationDecl>(D);
+  
+  return false;
 }
 
-/// \brief Get the correct cursor and offset for loading a type.
+/// \brief Get the correct cursor and offset for loading a declaration.
 ASTReader::RecordLocation
-ASTReader::DeclCursorForIndex(unsigned Index, DeclID ID) {
+ASTReader::DeclCursorForID(DeclID ID) {
   // See if there's an override.
   DeclReplacementMap::iterator It = ReplacedDecls.find(ID);
   if (It != ReplacedDecls.end())
     return RecordLocation(It->second.first, It->second.second);
 
-  PerFileData *F = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    F = Chain[N - I - 1];
-    if (Index < F->LocalNumDecls)
-      break;
-    Index -= F->LocalNumDecls;
-  }
-  assert(F && F->LocalNumDecls > Index && "Broken chain");
-  return RecordLocation(F, F->DeclOffsets[Index]);
+  GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID);
+  assert(I != GlobalDeclMap.end() && "Corrupted global declaration map");
+  Module *M = I->second;
+  return RecordLocation(M, 
+           M->DeclOffsets[ID - M->BaseDeclID - NUM_PREDEF_DECL_IDS]);
+}
+
+ASTReader::RecordLocation ASTReader::getLocalBitOffset(uint64_t GlobalOffset) {
+  ContinuousRangeMap<uint64_t, Module*, 4>::iterator I
+    = GlobalBitOffsetsMap.find(GlobalOffset);
+
+  assert(I != GlobalBitOffsetsMap.end() && "Corrupted global bit offsets map");
+  return RecordLocation(I->second, GlobalOffset - I->second->GlobalBitOffset);
+}
+
+uint64_t ASTReader::getGlobalBitOffset(Module &M, uint32_t LocalOffset) {
+  return LocalOffset + M.GlobalBitOffset;
 }
 
 void ASTDeclReader::attachPreviousDecl(Decl *D, Decl *previous) {
@@ -1447,8 +1467,9 @@ void ASTReader::loadAndAttachPreviousDecl(Decl *D, serialization::DeclID ID) {
 }
 
 /// \brief Read the declaration at the given offset from the AST file.
-Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
-  RecordLocation Loc = DeclCursorForIndex(Index, ID);
+Decl *ASTReader::ReadDeclRecord(DeclID ID) {
+  unsigned Index = ID - NUM_PREDEF_DECL_IDS;
+  RecordLocation Loc = DeclCursorForID(ID);
   llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor;
   // Keep track of where we are in the stream, then jump back there
   // after reading this declaration.
@@ -1469,215 +1490,216 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   switch ((DeclCode)DeclsCursor.ReadRecord(Code, Record)) {
   case DECL_CONTEXT_LEXICAL:
   case DECL_CONTEXT_VISIBLE:
-    assert(false && "Record cannot be de-serialized with ReadDeclRecord");
-    break;
-  case DECL_TRANSLATION_UNIT:
-    assert(Index == 0 && "Translation unit must be at index 0");
-    D = Context->getTranslationUnitDecl();
-    break;
+    llvm_unreachable("Record cannot be de-serialized with ReadDeclRecord");
   case DECL_TYPEDEF:
-    D = TypedefDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+    D = TypedefDecl::Create(Context, 0, SourceLocation(), SourceLocation(),
                             0, 0);
     break;
   case DECL_TYPEALIAS:
-    D = TypeAliasDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+    D = TypeAliasDecl::Create(Context, 0, SourceLocation(), SourceLocation(),
                               0, 0);
     break;
   case DECL_ENUM:
-    D = EnumDecl::Create(*Context, Decl::EmptyShell());
+    D = EnumDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_RECORD:
-    D = RecordDecl::Create(*Context, Decl::EmptyShell());
+    D = RecordDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_ENUM_CONSTANT:
-    D = EnumConstantDecl::Create(*Context, 0, SourceLocation(), 0, QualType(),
+    D = EnumConstantDecl::Create(Context, 0, SourceLocation(), 0, QualType(),
                                  0, llvm::APSInt());
     break;
   case DECL_FUNCTION:
-    D = FunctionDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+    D = FunctionDecl::Create(Context, 0, SourceLocation(), SourceLocation(),
                              DeclarationName(), QualType(), 0);
     break;
   case DECL_LINKAGE_SPEC:
-    D = LinkageSpecDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+    D = LinkageSpecDecl::Create(Context, 0, SourceLocation(), SourceLocation(),
                                 (LinkageSpecDecl::LanguageIDs)0,
                                 SourceLocation());
     break;
   case DECL_LABEL:
-    D = LabelDecl::Create(*Context, 0, SourceLocation(), 0);
+    D = LabelDecl::Create(Context, 0, SourceLocation(), 0);
     break;
   case DECL_NAMESPACE:
-    D = NamespaceDecl::Create(*Context, 0, SourceLocation(),
+    D = NamespaceDecl::Create(Context, 0, SourceLocation(),
                               SourceLocation(), 0);
     break;
   case DECL_NAMESPACE_ALIAS:
-    D = NamespaceAliasDecl::Create(*Context, 0, SourceLocation(),
+    D = NamespaceAliasDecl::Create(Context, 0, SourceLocation(),
                                    SourceLocation(), 0, 
                                    NestedNameSpecifierLoc(),
                                    SourceLocation(), 0);
     break;
   case DECL_USING:
-    D = UsingDecl::Create(*Context, 0, SourceLocation(),
+    D = UsingDecl::Create(Context, 0, SourceLocation(),
                           NestedNameSpecifierLoc(), DeclarationNameInfo(), 
                           false);
     break;
   case DECL_USING_SHADOW:
-    D = UsingShadowDecl::Create(*Context, 0, SourceLocation(), 0, 0);
+    D = UsingShadowDecl::Create(Context, 0, SourceLocation(), 0, 0);
     break;
   case DECL_USING_DIRECTIVE:
-    D = UsingDirectiveDecl::Create(*Context, 0, SourceLocation(),
+    D = UsingDirectiveDecl::Create(Context, 0, SourceLocation(),
                                    SourceLocation(), NestedNameSpecifierLoc(),
                                    SourceLocation(), 0, 0);
     break;
   case DECL_UNRESOLVED_USING_VALUE:
-    D = UnresolvedUsingValueDecl::Create(*Context, 0, SourceLocation(),
+    D = UnresolvedUsingValueDecl::Create(Context, 0, SourceLocation(),
                                          NestedNameSpecifierLoc(), 
                                          DeclarationNameInfo());
     break;
   case DECL_UNRESOLVED_USING_TYPENAME:
-    D = UnresolvedUsingTypenameDecl::Create(*Context, 0, SourceLocation(),
+    D = UnresolvedUsingTypenameDecl::Create(Context, 0, SourceLocation(),
                                             SourceLocation(), 
                                             NestedNameSpecifierLoc(),
                                             SourceLocation(),
                                             DeclarationName());
     break;
   case DECL_CXX_RECORD:
-    D = CXXRecordDecl::Create(*Context, Decl::EmptyShell());
+    D = CXXRecordDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_CXX_METHOD:
-    D = CXXMethodDecl::Create(*Context, 0, SourceLocation(),
+    D = CXXMethodDecl::Create(Context, 0, SourceLocation(),
                               DeclarationNameInfo(), QualType(), 0,
-                              false, SC_None, false, SourceLocation());
+                              false, SC_None, false, false, SourceLocation());
     break;
   case DECL_CXX_CONSTRUCTOR:
-    D = CXXConstructorDecl::Create(*Context, Decl::EmptyShell());
+    D = CXXConstructorDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_CXX_DESTRUCTOR:
-    D = CXXDestructorDecl::Create(*Context, Decl::EmptyShell());
+    D = CXXDestructorDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_CXX_CONVERSION:
-    D = CXXConversionDecl::Create(*Context, Decl::EmptyShell());
+    D = CXXConversionDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_ACCESS_SPEC:
-    D = AccessSpecDecl::Create(*Context, Decl::EmptyShell());
+    D = AccessSpecDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_FRIEND:
-    D = FriendDecl::Create(*Context, Decl::EmptyShell());
+    D = FriendDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_FRIEND_TEMPLATE:
-    D = FriendTemplateDecl::Create(*Context, Decl::EmptyShell());
+    D = FriendTemplateDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_CLASS_TEMPLATE:
-    D = ClassTemplateDecl::Create(*Context, Decl::EmptyShell());
+    D = ClassTemplateDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_CLASS_TEMPLATE_SPECIALIZATION:
-    D = ClassTemplateSpecializationDecl::Create(*Context, Decl::EmptyShell());
+    D = ClassTemplateSpecializationDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION:
-    D = ClassTemplatePartialSpecializationDecl::Create(*Context,
+    D = ClassTemplatePartialSpecializationDecl::Create(Context,
                                                        Decl::EmptyShell());
     break;
+  case DECL_CLASS_SCOPE_FUNCTION_SPECIALIZATION:
+    D = ClassScopeFunctionSpecializationDecl::Create(Context,
+                                                     Decl::EmptyShell());
+    break;
   case DECL_FUNCTION_TEMPLATE:
-      D = FunctionTemplateDecl::Create(*Context, Decl::EmptyShell());
+      D = FunctionTemplateDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_TEMPLATE_TYPE_PARM:
-    D = TemplateTypeParmDecl::Create(*Context, Decl::EmptyShell());
+    D = TemplateTypeParmDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_NON_TYPE_TEMPLATE_PARM:
-    D = NonTypeTemplateParmDecl::Create(*Context, 0, SourceLocation(),
+    D = NonTypeTemplateParmDecl::Create(Context, 0, SourceLocation(),
                                         SourceLocation(), 0, 0, 0, QualType(),
                                         false, 0);
     break;
   case DECL_EXPANDED_NON_TYPE_TEMPLATE_PARM_PACK:
-    D = NonTypeTemplateParmDecl::Create(*Context, 0, SourceLocation(),
+    D = NonTypeTemplateParmDecl::Create(Context, 0, SourceLocation(),
                                         SourceLocation(), 0, 0, 0, QualType(),
                                         0, 0, Record[Idx++], 0);
     break;
   case DECL_TEMPLATE_TEMPLATE_PARM:
-    D = TemplateTemplateParmDecl::Create(*Context, 0, SourceLocation(), 0, 0,
+    D = TemplateTemplateParmDecl::Create(Context, 0, SourceLocation(), 0, 0,
                                          false, 0, 0);
     break;
   case DECL_TYPE_ALIAS_TEMPLATE:
-    D = TypeAliasTemplateDecl::Create(*Context, Decl::EmptyShell());
+    D = TypeAliasTemplateDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_STATIC_ASSERT:
-    D = StaticAssertDecl::Create(*Context, 0, SourceLocation(), 0, 0,
+    D = StaticAssertDecl::Create(Context, 0, SourceLocation(), 0, 0,
                                  SourceLocation());
     break;
 
   case DECL_OBJC_METHOD:
-    D = ObjCMethodDecl::Create(*Context, SourceLocation(), SourceLocation(),
+    D = ObjCMethodDecl::Create(Context, SourceLocation(), SourceLocation(),
                                Selector(), QualType(), 0, 0);
     break;
   case DECL_OBJC_INTERFACE:
-    D = ObjCInterfaceDecl::Create(*Context, 0, SourceLocation(), 0);
+    D = ObjCInterfaceDecl::Create(Context, 0, SourceLocation(), 0);
     break;
   case DECL_OBJC_IVAR:
-    D = ObjCIvarDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+    D = ObjCIvarDecl::Create(Context, 0, SourceLocation(), SourceLocation(),
                              0, QualType(), 0, ObjCIvarDecl::None);
     break;
   case DECL_OBJC_PROTOCOL:
-    D = ObjCProtocolDecl::Create(*Context, 0, SourceLocation(), 0);
+    D = ObjCProtocolDecl::Create(Context, 0, 0, SourceLocation(),
+                                 SourceLocation());
     break;
   case DECL_OBJC_AT_DEFS_FIELD:
-    D = ObjCAtDefsFieldDecl::Create(*Context, 0, SourceLocation(),
+    D = ObjCAtDefsFieldDecl::Create(Context, 0, SourceLocation(),
                                     SourceLocation(), 0, QualType(), 0);
     break;
   case DECL_OBJC_CLASS:
-    D = ObjCClassDecl::Create(*Context, 0, SourceLocation());
+    D = ObjCClassDecl::Create(Context, 0, SourceLocation());
     break;
   case DECL_OBJC_FORWARD_PROTOCOL:
-    D = ObjCForwardProtocolDecl::Create(*Context, 0, SourceLocation());
+    D = ObjCForwardProtocolDecl::Create(Context, 0, SourceLocation());
     break;
   case DECL_OBJC_CATEGORY:
-    D = ObjCCategoryDecl::Create(*Context, 0, SourceLocation(), 
-                                 SourceLocation(), SourceLocation(), 0);
+    D = ObjCCategoryDecl::Create(Context, Decl::EmptyShell());
     break;
   case DECL_OBJC_CATEGORY_IMPL:
-    D = ObjCCategoryImplDecl::Create(*Context, 0, SourceLocation(), 0, 0);
+    D = ObjCCategoryImplDecl::Create(Context, 0, 0, 0, SourceLocation(),
+                                     SourceLocation());
     break;
   case DECL_OBJC_IMPLEMENTATION:
-    D = ObjCImplementationDecl::Create(*Context, 0, SourceLocation(), 0, 0);
+    D = ObjCImplementationDecl::Create(Context, 0, 0, 0, SourceLocation(),
+                                       SourceLocation());
     break;
   case DECL_OBJC_COMPATIBLE_ALIAS:
-    D = ObjCCompatibleAliasDecl::Create(*Context, 0, SourceLocation(), 0, 0);
+    D = ObjCCompatibleAliasDecl::Create(Context, 0, SourceLocation(), 0, 0);
     break;
   case DECL_OBJC_PROPERTY:
-    D = ObjCPropertyDecl::Create(*Context, 0, SourceLocation(), 0, SourceLocation(),
+    D = ObjCPropertyDecl::Create(Context, 0, SourceLocation(), 0, SourceLocation(),
                                  0);
     break;
   case DECL_OBJC_PROPERTY_IMPL:
-    D = ObjCPropertyImplDecl::Create(*Context, 0, SourceLocation(),
+    D = ObjCPropertyImplDecl::Create(Context, 0, SourceLocation(),
                                      SourceLocation(), 0,
                                      ObjCPropertyImplDecl::Dynamic, 0,
                                      SourceLocation());
     break;
   case DECL_FIELD:
-    D = FieldDecl::Create(*Context, 0, SourceLocation(), SourceLocation(), 0,
+    D = FieldDecl::Create(Context, 0, SourceLocation(), SourceLocation(), 0,
                           QualType(), 0, 0, false, false);
     break;
   case DECL_INDIRECTFIELD:
-    D = IndirectFieldDecl::Create(*Context, 0, SourceLocation(), 0, QualType(),
+    D = IndirectFieldDecl::Create(Context, 0, SourceLocation(), 0, QualType(),
                                   0, 0);
     break;
   case DECL_VAR:
-    D = VarDecl::Create(*Context, 0, SourceLocation(), SourceLocation(), 0,
+    D = VarDecl::Create(Context, 0, SourceLocation(), SourceLocation(), 0,
                         QualType(), 0, SC_None, SC_None);
     break;
 
   case DECL_IMPLICIT_PARAM:
-    D = ImplicitParamDecl::Create(*Context, 0, SourceLocation(), 0, QualType());
+    D = ImplicitParamDecl::Create(Context, 0, SourceLocation(), 0, QualType());
     break;
 
   case DECL_PARM_VAR:
-    D = ParmVarDecl::Create(*Context, 0, SourceLocation(), SourceLocation(), 0,
+    D = ParmVarDecl::Create(Context, 0, SourceLocation(), SourceLocation(), 0,
                             QualType(), 0, SC_None, SC_None, 0);
     break;
   case DECL_FILE_SCOPE_ASM:
-    D = FileScopeAsmDecl::Create(*Context, 0, 0, SourceLocation(),
+    D = FileScopeAsmDecl::Create(Context, 0, 0, SourceLocation(),
                                  SourceLocation());
     break;
   case DECL_BLOCK:
-    D = BlockDecl::Create(*Context, 0, SourceLocation());
+    D = BlockDecl::Create(Context, 0, SourceLocation());
     break;
   case DECL_CXX_BASE_SPECIFIERS:
     Error("attempt to read a C++ base-specifier record as a declaration");
@@ -1693,16 +1715,13 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   if (DeclContext *DC = dyn_cast<DeclContext>(D)) {
     std::pair<uint64_t, uint64_t> Offsets = Reader.VisitDeclContext(DC);
     if (Offsets.first || Offsets.second) {
-      DC->setHasExternalLexicalStorage(Offsets.first != 0);
-      DC->setHasExternalVisibleStorage(Offsets.second != 0);
-      DeclContextInfo Info;
-      if (ReadDeclContextStorage(DeclsCursor, Offsets, Info))
+      if (Offsets.first != 0)
+        DC->setHasExternalLexicalStorage(true);
+      if (Offsets.second != 0)
+        DC->setHasExternalVisibleStorage(true);
+      if (ReadDeclContextStorage(*Loc.F, DeclsCursor, Offsets, 
+                                 Loc.F->DeclContextInfos[DC]))
         return 0;
-      DeclContextInfos &Infos = DeclContextOffsets[DC];
-      // Reading the TU will happen after reading its lexical update blocks,
-      // so we need to make sure we insert in front. For all other contexts,
-      // the vector is empty here anyway, so there's no loss in efficiency.
-      Infos.insert(Infos.begin(), Info);
     }
 
     // Now add the pending visible updates for this decl context, if it has any.
@@ -1713,20 +1732,32 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
       // storage, even if the original stored version didn't.
       DC->setHasExternalVisibleStorage(true);
       DeclContextVisibleUpdates &U = I->second;
-      DeclContextInfos &Infos = DeclContextOffsets[DC];
-      DeclContextInfo Info;
-      Info.LexicalDecls = 0;
-      Info.NumLexicalDecls = 0;
       for (DeclContextVisibleUpdates::iterator UI = U.begin(), UE = U.end();
            UI != UE; ++UI) {
-        Info.NameLookupTableData = *UI;
-        Infos.push_back(Info);
+        UI->second->DeclContextInfos[DC].NameLookupTableData = UI->first;
       }
       PendingVisibleUpdates.erase(I);
     }
   }
   assert(Idx == Record.size());
 
+  // Load any relevant update records.
+  loadDeclUpdateRecords(ID, D);
+  
+  if (ObjCChainedCategoriesInterfaces.count(ID))
+    loadObjCChainedCategories(ID, cast<ObjCInterfaceDecl>(D));
+  
+  // If we have deserialized a declaration that has a definition the
+  // AST consumer might need to know about, queue it.
+  // We don't pass it to the consumer immediately because we may be in recursive
+  // loading, and some declarations may still be initializing.
+  if (isConsumerInterestedIn(D))
+      InterestingDecls.push_back(D);
+  
+  return D;
+}
+
+void ASTReader::loadDeclUpdateRecords(serialization::DeclID ID, Decl *D) {
   // The declaration may have been modified by files later in the chain.
   // If this is the case, read the record containing the updates from each file
   // and pass it to ASTDeclReader to make the modifications.
@@ -1734,8 +1765,8 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   if (UpdI != DeclUpdateOffsets.end()) {
     FileOffsetsTy &UpdateOffsets = UpdI->second;
     for (FileOffsetsTy::iterator
-           I = UpdateOffsets.begin(), E = UpdateOffsets.end(); I != E; ++I) {
-      PerFileData *F = I->first;
+         I = UpdateOffsets.begin(), E = UpdateOffsets.end(); I != E; ++I) {
+      Module *F = I->first;
       uint64_t Offset = I->second;
       llvm::BitstreamCursor &Cursor = F->DeclsCursor;
       SavedStreamPosition SavedPosition(Cursor);
@@ -1745,45 +1776,166 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
       unsigned RecCode = Cursor.ReadRecord(Code, Record);
       (void)RecCode;
       assert(RecCode == DECL_UPDATES && "Expected DECL_UPDATES record!");
+      
+      unsigned Idx = 0;
+      ASTDeclReader Reader(*this, *F, Cursor, ID, Record, Idx);
       Reader.UpdateDecl(D, *F, Record);
     }
   }
-
-  // If we have deserialized a declaration that has a definition the
-  // AST consumer might need to know about, queue it.
-  // We don't pass it to the consumer immediately because we may be in recursive
-  // loading, and some declarations may still be initializing.
-  if (isConsumerInterestedIn(D))
-    InterestingDecls.push_back(D);
-
-  return D;
 }
 
-void ASTDeclReader::UpdateDecl(Decl *D, ASTReader::PerFileData &Module,
+namespace {
+  /// \brief Given an ObjC interface, goes through the modules and links to the
+  /// interface all the categories for it.
+  class ObjCChainedCategoriesVisitor {
+    ASTReader &Reader;
+    serialization::GlobalDeclID InterfaceID;
+    ObjCInterfaceDecl *Interface;
+    ObjCCategoryDecl *GlobHeadCat, *GlobTailCat;
+    llvm::DenseMap<DeclarationName, ObjCCategoryDecl *> NameCategoryMap;
+
+  public:
+    ObjCChainedCategoriesVisitor(ASTReader &Reader,
+                                 serialization::GlobalDeclID InterfaceID,
+                                 ObjCInterfaceDecl *Interface)
+      : Reader(Reader), InterfaceID(InterfaceID), Interface(Interface),
+        GlobHeadCat(0), GlobTailCat(0) { }
+
+    static bool visit(Module &M, void *UserData) {
+      return static_cast<ObjCChainedCategoriesVisitor *>(UserData)->visit(M);
+    }
+
+    bool visit(Module &M) {
+      if (Reader.isDeclIDFromModule(InterfaceID, M))
+        return true; // We reached the module where the interface originated
+                    // from. Stop traversing the imported modules.
+
+      Module::ChainedObjCCategoriesMap::iterator
+        I = M.ChainedObjCCategories.find(InterfaceID);
+      if (I == M.ChainedObjCCategories.end())
+        return false;
+
+      ObjCCategoryDecl *
+        HeadCat = Reader.GetLocalDeclAs<ObjCCategoryDecl>(M, I->second.first);
+      ObjCCategoryDecl *
+        TailCat = Reader.GetLocalDeclAs<ObjCCategoryDecl>(M, I->second.second);
+
+      addCategories(HeadCat, TailCat);
+      return false;
+    }
+
+    void addCategories(ObjCCategoryDecl *HeadCat,
+                       ObjCCategoryDecl *TailCat = 0) {
+      if (!HeadCat) {
+        assert(!TailCat);
+        return;
+      }
+
+      if (!TailCat) {
+        TailCat = HeadCat;
+        while (TailCat->getNextClassCategory())
+          TailCat = TailCat->getNextClassCategory();
+      }
+
+      if (!GlobHeadCat) {
+        GlobHeadCat = HeadCat;
+        GlobTailCat = TailCat;
+      } else {
+        ASTDeclReader::setNextObjCCategory(GlobTailCat, HeadCat);
+        GlobTailCat = TailCat;
+      }
+
+      llvm::DenseSet<DeclarationName> Checked;
+      for (ObjCCategoryDecl *Cat = HeadCat,
+                            *CatEnd = TailCat->getNextClassCategory();
+             Cat != CatEnd; Cat = Cat->getNextClassCategory()) {
+        if (Checked.count(Cat->getDeclName()))
+          continue;
+        Checked.insert(Cat->getDeclName());
+        checkForDuplicate(Cat);
+      }
+    }
+
+    /// \brief Warns for duplicate categories that come from different modules.
+    void checkForDuplicate(ObjCCategoryDecl *Cat) {
+      DeclarationName Name = Cat->getDeclName();
+      // Find the top category with the same name. We do not want to warn for
+      // duplicates along the established chain because there were already
+      // warnings for them when the module was created. We only want to warn for
+      // duplicates between non-dependent modules:
+      //
+      //   MT     //
+      //  /  \    //
+      // ML  MR   //
+      //
+      // We want to warn for duplicates between ML and MR,not between ML and MT.
+      //
+      // FIXME: We should not warn for duplicates in diamond:
+      //
+      //   MT     //
+      //  /  \    //
+      // ML  MR   //
+      //  \  /    //
+      //   MB     //
+      //
+      // If there are duplicates in ML/MR, there will be warning when creating
+      // MB *and* when importing MB. We should not warn when importing.
+      for (ObjCCategoryDecl *Next = Cat->getNextClassCategory(); Next;
+             Next = Next->getNextClassCategory()) {
+        if (Next->getDeclName() == Name)
+          Cat = Next;
+      }
+
+      ObjCCategoryDecl *&PrevCat = NameCategoryMap[Name];
+      if (!PrevCat)
+        PrevCat = Cat;
+
+      if (PrevCat != Cat) {
+        Reader.Diag(Cat->getLocation(), diag::warn_dup_category_def)
+          << Interface->getDeclName() << Name;
+        Reader.Diag(PrevCat->getLocation(), diag::note_previous_definition);
+      }
+    }
+
+    ObjCCategoryDecl *getHeadCategory() const { return GlobHeadCat; }
+  };
+}
+
+void ASTReader::loadObjCChainedCategories(serialization::GlobalDeclID ID,
+                                          ObjCInterfaceDecl *D) {
+  ObjCChainedCategoriesVisitor Visitor(*this, ID, D);
+  ModuleMgr.visit(ObjCChainedCategoriesVisitor::visit, &Visitor);
+  // Also add the categories that the interface already links to.
+  Visitor.addCategories(D->getCategoryList());
+  D->setCategoryList(Visitor.getHeadCategory());
+}
+
+void ASTDeclReader::UpdateDecl(Decl *D, Module &Module,
                                const RecordData &Record) {
   unsigned Idx = 0;
   while (Idx < Record.size()) {
     switch ((DeclUpdateKind)Record[Idx++]) {
     case UPD_CXX_SET_DEFINITIONDATA: {
       CXXRecordDecl *RD = cast<CXXRecordDecl>(D);
-      CXXRecordDecl *
-          DefinitionDecl = cast<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
+      CXXRecordDecl *DefinitionDecl
+        = Reader.ReadDeclAs<CXXRecordDecl>(Module, Record, Idx);
       assert(!RD->DefinitionData && "DefinitionData is already set!");
       InitializeCXXDefinitionData(RD, DefinitionDecl, Record, Idx);
       break;
     }
 
     case UPD_CXX_ADDED_IMPLICIT_MEMBER:
-      cast<CXXRecordDecl>(D)->addedMember(Reader.GetDecl(Record[Idx++]));
+      cast<CXXRecordDecl>(D)->addedMember(Reader.ReadDecl(Module, Record, Idx));
       break;
 
     case UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION:
       // It will be added to the template's specializations set when loaded.
-      Reader.GetDecl(Record[Idx++]);
+      (void)Reader.ReadDecl(Module, Record, Idx);
       break;
 
     case UPD_CXX_ADDED_ANONYMOUS_NAMESPACE: {
-      NamespaceDecl *Anon = cast<NamespaceDecl>(Reader.GetDecl(Record[Idx++]));
+      NamespaceDecl *Anon
+        = Reader.ReadDeclAs<NamespaceDecl>(Module, Record, Idx);
       // Guard against these being loaded out of original order. Don't use
       // getNextNamespace(), since it tries to access the context and can't in
       // the middle of deserialization.
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderInternals.h b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderInternals.h
new file mode 100644
index 000000000000..99e8be5ee272
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderInternals.h
@@ -0,0 +1,243 @@
+//===--- ASTReaderInternals.h - AST Reader Internals ------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file provides internal definitions used in the AST reader.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_SERIALIZATION_ASTREADER_INTERNALS_H
+#define LLVM_CLANG_SERIALIZATION_ASTREADER_INTERNALS_H
+
+#include "clang/Basic/OnDiskHashTable.h"
+#include "clang/AST/DeclarationName.h"
+#include <utility>
+#include <sys/stat.h>
+
+namespace clang {
+
+class ASTReader;
+class HeaderSearch;
+struct HeaderFileInfo;
+  
+namespace serialization {
+
+class Module;
+
+namespace reader {
+
+/// \brief Class that performs name lookup into a DeclContext stored
+/// in an AST file.
+class ASTDeclContextNameLookupTrait {
+  ASTReader &Reader;
+  Module &F;
+  
+public:
+  /// \brief Pair of begin/end iterators for DeclIDs.
+  ///
+  /// Note that these declaration IDs are local to the module that contains this
+  /// particular lookup t
+  typedef std::pair<DeclID *, DeclID *> data_type;
+
+  /// \brief Special internal key for declaration names.
+  /// The hash table creates keys for comparison; we do not create
+  /// a DeclarationName for the internal key to avoid deserializing types.
+  struct DeclNameKey {
+    DeclarationName::NameKind Kind;
+    uint64_t Data;
+    DeclNameKey() : Kind((DeclarationName::NameKind)0), Data(0) { }
+  };
+
+  typedef DeclarationName external_key_type;
+  typedef DeclNameKey internal_key_type;
+
+  explicit ASTDeclContextNameLookupTrait(ASTReader &Reader, 
+                                         Module &F) 
+    : Reader(Reader), F(F) { }
+
+  static bool EqualKey(const internal_key_type& a,
+                       const internal_key_type& b) {
+    return a.Kind == b.Kind && a.Data == b.Data;
+  }
+
+  unsigned ComputeHash(const DeclNameKey &Key) const;
+  internal_key_type GetInternalKey(const external_key_type& Name) const;
+  external_key_type GetExternalKey(const internal_key_type& Key) const;
+
+  static std::pair<unsigned, unsigned> 
+  ReadKeyDataLength(const unsigned char*& d);
+
+  internal_key_type ReadKey(const unsigned char* d, unsigned);
+
+  data_type ReadData(internal_key_type, const unsigned char* d,
+                     unsigned DataLen);
+};
+
+/// \brief The on-disk hash table used for the DeclContext's Name lookup table.
+typedef OnDiskChainedHashTable<ASTDeclContextNameLookupTrait>
+  ASTDeclContextNameLookupTable;
+
+/// \brief Class that performs lookup for an identifier stored in an AST file.
+class ASTIdentifierLookupTrait {
+  ASTReader &Reader;
+  Module &F;
+  
+  // If we know the IdentifierInfo in advance, it is here and we will
+  // not build a new one. Used when deserializing information about an
+  // identifier that was constructed before the AST file was read.
+  IdentifierInfo *KnownII;
+  
+public:
+  typedef IdentifierInfo * data_type;
+  
+  typedef const std::pair<const char*, unsigned> external_key_type;
+  
+  typedef external_key_type internal_key_type;
+  
+  ASTIdentifierLookupTrait(ASTReader &Reader, Module &F,
+                           IdentifierInfo *II = 0)
+    : Reader(Reader), F(F), KnownII(II) { }
+  
+  static bool EqualKey(const internal_key_type& a,
+                       const internal_key_type& b) {
+    return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
+    : false;
+  }
+  
+  static unsigned ComputeHash(const internal_key_type& a);
+  
+  // This hopefully will just get inlined and removed by the optimizer.
+  static const internal_key_type&
+  GetInternalKey(const external_key_type& x) { return x; }
+  
+  // This hopefully will just get inlined and removed by the optimizer.
+  static const external_key_type&
+  GetExternalKey(const internal_key_type& x) { return x; }
+ 
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char*& d);
+
+  static std::pair<const char*, unsigned>
+  ReadKey(const unsigned char* d, unsigned n);
+  
+  IdentifierInfo *ReadData(const internal_key_type& k,
+                           const unsigned char* d,
+                           unsigned DataLen);
+};
+  
+/// \brief The on-disk hash table used to contain information about
+/// all of the identifiers in the program.
+typedef OnDiskChainedHashTable<ASTIdentifierLookupTrait>
+  ASTIdentifierLookupTable;
+
+/// \brief Class that performs lookup for a selector's entries in the global
+/// method pool stored in an AST file.
+class ASTSelectorLookupTrait {
+  ASTReader &Reader;
+  Module &F;
+  
+public:
+  struct data_type {
+    SelectorID ID;
+    llvm::SmallVector<ObjCMethodDecl *, 2> Instance;
+    llvm::SmallVector<ObjCMethodDecl *, 2> Factory;
+  };
+  
+  typedef Selector external_key_type;
+  typedef external_key_type internal_key_type;
+  
+  ASTSelectorLookupTrait(ASTReader &Reader, Module &F) 
+    : Reader(Reader), F(F) { }
+  
+  static bool EqualKey(const internal_key_type& a,
+                       const internal_key_type& b) {
+    return a == b;
+  }
+  
+  static unsigned ComputeHash(Selector Sel);
+  
+  static const internal_key_type&
+  GetInternalKey(const external_key_type& x) { return x; }
+  
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char*& d);
+  
+  internal_key_type ReadKey(const unsigned char* d, unsigned);
+  data_type ReadData(Selector, const unsigned char* d, unsigned DataLen);
+};
+  
+/// \brief The on-disk hash table used for the global method pool.
+typedef OnDiskChainedHashTable<ASTSelectorLookupTrait>
+  ASTSelectorLookupTable;
+  
+/// \brief Trait class used to search the on-disk hash table containing all of
+/// the header search information.
+///
+/// The on-disk hash table contains a mapping from each header path to 
+/// information about that header (how many times it has been included, its
+/// controlling macro, etc.). Note that we actually hash based on the 
+/// filename, and support "deep" comparisons of file names based on current
+/// inode numbers, so that the search can cope with non-normalized path names
+/// and symlinks.
+class HeaderFileInfoTrait {
+  ASTReader &Reader;
+  Module &M;
+  HeaderSearch *HS;
+  const char *FrameworkStrings;
+  const char *SearchPath;
+  struct stat SearchPathStatBuf;
+  llvm::Optional<int> SearchPathStatResult;
+  
+  int StatSimpleCache(const char *Path, struct stat *StatBuf) {
+    if (Path == SearchPath) {
+      if (!SearchPathStatResult)
+        SearchPathStatResult = stat(Path, &SearchPathStatBuf);
+      
+      *StatBuf = SearchPathStatBuf;
+      return *SearchPathStatResult;
+    }
+    
+    return stat(Path, StatBuf);
+  }
+  
+public:
+  typedef const char *external_key_type;
+  typedef const char *internal_key_type;
+  
+  typedef HeaderFileInfo data_type;
+  
+  HeaderFileInfoTrait(ASTReader &Reader, Module &M, HeaderSearch *HS,
+                      const char *FrameworkStrings,
+                      const char *SearchPath = 0) 
+  : Reader(Reader), M(M), HS(HS), FrameworkStrings(FrameworkStrings), 
+    SearchPath(SearchPath) { }
+  
+  static unsigned ComputeHash(const char *path);
+  static internal_key_type GetInternalKey(const char *path);
+  bool EqualKey(internal_key_type a, internal_key_type b);
+  
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char*& d);
+  
+  static internal_key_type ReadKey(const unsigned char *d, unsigned) {
+    return (const char *)d;
+  }
+  
+  data_type ReadData(const internal_key_type, const unsigned char *d,
+                     unsigned DataLen);
+};
+
+/// \brief The on-disk hash table used for known header files.
+typedef OnDiskChainedHashTable<HeaderFileInfoTrait>
+  HeaderFileInfoLookupTable;
+  
+} // end namespace clang::serialization::reader
+} // end namespace clang::serialization
+} // end namespace clang
+
+
+#endif
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderStmt.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderStmt.cpp
index 14927b9e3bc8..ab07b85bf4c3 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -22,34 +22,51 @@ using namespace clang::serialization;
 namespace clang {
 
   class ASTStmtReader : public StmtVisitor<ASTStmtReader> {
+    typedef ASTReader::RecordData RecordData;
+    
     ASTReader &Reader;
-    ASTReader::PerFileData &F;
+    Module &F;
     llvm::BitstreamCursor &DeclsCursor;
     const ASTReader::RecordData &Record;
     unsigned &Idx;
 
-    SourceLocation ReadSourceLocation(const ASTReader::RecordData &R,
-                                      unsigned &I) {
+    SourceLocation ReadSourceLocation(const RecordData &R, unsigned &I) {
       return Reader.ReadSourceLocation(F, R, I);
     }
-    SourceRange ReadSourceRange(const ASTReader::RecordData &R, unsigned &I) {
+    
+    SourceRange ReadSourceRange(const RecordData &R, unsigned &I) {
       return Reader.ReadSourceRange(F, R, I);
     }
-    TypeSourceInfo *GetTypeSourceInfo(const ASTReader::RecordData &R,
-                                      unsigned &I) {
+    
+    TypeSourceInfo *GetTypeSourceInfo(const RecordData &R, unsigned &I) {
       return Reader.GetTypeSourceInfo(F, R, I);
     }
+    
+    serialization::DeclID ReadDeclID(const RecordData &R, unsigned &I) {
+      return Reader.ReadDeclID(F, R, I);
+    }
+    
+    Decl *ReadDecl(const RecordData &R, unsigned &I) {
+      return Reader.ReadDecl(F, R, I);
+    }
+    
+    template<typename T>
+    T *ReadDeclAs(const RecordData &R, unsigned &I) {
+      return Reader.ReadDeclAs<T>(F, R, I);
+    }
+
     void ReadDeclarationNameLoc(DeclarationNameLoc &DNLoc, DeclarationName Name,
                                 const ASTReader::RecordData &R, unsigned &I) {
       Reader.ReadDeclarationNameLoc(F, DNLoc, Name, R, I);
     }
+    
     void ReadDeclarationNameInfo(DeclarationNameInfo &NameInfo,
                                 const ASTReader::RecordData &R, unsigned &I) {
       Reader.ReadDeclarationNameInfo(F, NameInfo, R, I);
     }
 
   public:
-    ASTStmtReader(ASTReader &Reader, ASTReader::PerFileData &F,
+    ASTStmtReader(ASTReader &Reader, Module &F,
                   llvm::BitstreamCursor &Cursor,
                   const ASTReader::RecordData &Record, unsigned &Idx)
       : Reader(Reader), F(F), DeclsCursor(Cursor), Record(Record), Idx(Idx) { }
@@ -63,7 +80,7 @@ namespace clang {
     static const unsigned NumExprFields = NumStmtFields + 7;
     
     /// \brief Read and initialize a ExplicitTemplateArgumentList structure.
-    void ReadExplicitTemplateArgumentList(ExplicitTemplateArgumentList &ArgList,
+    void ReadExplicitTemplateArgumentList(ASTTemplateArgumentListInfo &ArgList,
                                           unsigned NumTemplateArgs);
 
     void VisitStmt(Stmt *S);
@@ -74,7 +91,7 @@ namespace clang {
 }
 
 void ASTStmtReader::
-ReadExplicitTemplateArgumentList(ExplicitTemplateArgumentList &ArgList,
+ReadExplicitTemplateArgumentList(ASTTemplateArgumentListInfo &ArgList,
                                  unsigned NumTemplateArgs) {
   TemplateArgumentListInfo ArgInfo;
   ArgInfo.setLAngleLoc(ReadSourceLocation(Record, Idx));
@@ -92,16 +109,16 @@ void ASTStmtReader::VisitStmt(Stmt *S) {
 void ASTStmtReader::VisitNullStmt(NullStmt *S) {
   VisitStmt(S);
   S->setSemiLoc(ReadSourceLocation(Record, Idx));
-  S->LeadingEmptyMacro = ReadSourceLocation(Record, Idx);
+  S->HasLeadingEmptyMacro = Record[Idx++];
 }
 
 void ASTStmtReader::VisitCompoundStmt(CompoundStmt *S) {
   VisitStmt(S);
-  llvm::SmallVector<Stmt *, 16> Stmts;
+  SmallVector<Stmt *, 16> Stmts;
   unsigned NumStmts = Record[Idx++];
   while (NumStmts--)
     Stmts.push_back(Reader.ReadSubStmt());
-  S->setStmts(*Reader.getContext(), Stmts.data(), Stmts.size());
+  S->setStmts(Reader.getContext(), Stmts.data(), Stmts.size());
   S->setLBracLoc(ReadSourceLocation(Record, Idx));
   S->setRBracLoc(ReadSourceLocation(Record, Idx));
 }
@@ -130,7 +147,7 @@ void ASTStmtReader::VisitDefaultStmt(DefaultStmt *S) {
 
 void ASTStmtReader::VisitLabelStmt(LabelStmt *S) {
   VisitStmt(S);
-  LabelDecl *LD = cast<LabelDecl>(Reader.GetDecl(Record[Idx++]));
+  LabelDecl *LD = ReadDeclAs<LabelDecl>(Record, Idx);
   LD->setStmt(S);
   S->setDecl(LD);
   S->setSubStmt(Reader.ReadSubStmt());
@@ -139,8 +156,8 @@ void ASTStmtReader::VisitLabelStmt(LabelStmt *S) {
 
 void ASTStmtReader::VisitIfStmt(IfStmt *S) {
   VisitStmt(S);
-  S->setConditionVariable(*Reader.getContext(),
-                          cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setConditionVariable(Reader.getContext(), 
+                          ReadDeclAs<VarDecl>(Record, Idx));
   S->setCond(Reader.ReadSubExpr());
   S->setThen(Reader.ReadSubStmt());
   S->setElse(Reader.ReadSubStmt());
@@ -150,8 +167,8 @@ void ASTStmtReader::VisitIfStmt(IfStmt *S) {
 
 void ASTStmtReader::VisitSwitchStmt(SwitchStmt *S) {
   VisitStmt(S);
-  S->setConditionVariable(*Reader.getContext(),
-                          cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setConditionVariable(Reader.getContext(),
+                          ReadDeclAs<VarDecl>(Record, Idx));
   S->setCond(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
   S->setSwitchLoc(ReadSourceLocation(Record, Idx));
@@ -172,8 +189,9 @@ void ASTStmtReader::VisitSwitchStmt(SwitchStmt *S) {
 
 void ASTStmtReader::VisitWhileStmt(WhileStmt *S) {
   VisitStmt(S);
-  S->setConditionVariable(*Reader.getContext(),
-                          cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setConditionVariable(Reader.getContext(),
+                          ReadDeclAs<VarDecl>(Record, Idx));
+
   S->setCond(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
   S->setWhileLoc(ReadSourceLocation(Record, Idx));
@@ -192,8 +210,8 @@ void ASTStmtReader::VisitForStmt(ForStmt *S) {
   VisitStmt(S);
   S->setInit(Reader.ReadSubStmt());
   S->setCond(Reader.ReadSubExpr());
-  S->setConditionVariable(*Reader.getContext(),
-                          cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setConditionVariable(Reader.getContext(),
+                          ReadDeclAs<VarDecl>(Record, Idx));
   S->setInc(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
   S->setForLoc(ReadSourceLocation(Record, Idx));
@@ -203,7 +221,7 @@ void ASTStmtReader::VisitForStmt(ForStmt *S) {
 
 void ASTStmtReader::VisitGotoStmt(GotoStmt *S) {
   VisitStmt(S);
-  S->setLabel(cast<LabelDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setLabel(ReadDeclAs<LabelDecl>(Record, Idx));
   S->setGotoLoc(ReadSourceLocation(Record, Idx));
   S->setLabelLoc(ReadSourceLocation(Record, Idx));
 }
@@ -229,7 +247,7 @@ void ASTStmtReader::VisitReturnStmt(ReturnStmt *S) {
   VisitStmt(S);
   S->setRetValue(Reader.ReadSubExpr());
   S->setReturnLoc(ReadSourceLocation(Record, Idx));
-  S->setNRVOCandidate(cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setNRVOCandidate(ReadDeclAs<VarDecl>(Record, Idx));
 }
 
 void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
@@ -239,13 +257,13 @@ void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
 
   if (Idx + 1 == Record.size()) {
     // Single declaration
-    S->setDeclGroup(DeclGroupRef(Reader.GetDecl(Record[Idx++])));
+    S->setDeclGroup(DeclGroupRef(ReadDecl(Record, Idx)));
   } else {
-    llvm::SmallVector<Decl *, 16> Decls;
-    Decls.reserve(Record.size() - Idx);
-    for (unsigned N = Record.size(); Idx != N; ++Idx)
-      Decls.push_back(Reader.GetDecl(Record[Idx]));
-    S->setDeclGroup(DeclGroupRef(DeclGroup::Create(*Reader.getContext(),
+    SmallVector<Decl *, 16> Decls;
+    Decls.reserve(Record.size() - Idx);    
+    for (unsigned N = Record.size(); Idx != N; )
+      Decls.push_back(ReadDecl(Record, Idx));
+    S->setDeclGroup(DeclGroupRef(DeclGroup::Create(Reader.getContext(),
                                                    Decls.data(),
                                                    Decls.size())));
   }
@@ -265,21 +283,21 @@ void ASTStmtReader::VisitAsmStmt(AsmStmt *S) {
   S->setAsmString(cast_or_null<StringLiteral>(Reader.ReadSubStmt()));
 
   // Outputs and inputs
-  llvm::SmallVector<IdentifierInfo *, 16> Names;
-  llvm::SmallVector<StringLiteral*, 16> Constraints;
-  llvm::SmallVector<Stmt*, 16> Exprs;
+  SmallVector<IdentifierInfo *, 16> Names;
+  SmallVector<StringLiteral*, 16> Constraints;
+  SmallVector<Stmt*, 16> Exprs;
   for (unsigned I = 0, N = NumOutputs + NumInputs; I != N; ++I) {
-    Names.push_back(Reader.GetIdentifierInfo(Record, Idx));
+    Names.push_back(Reader.GetIdentifierInfo(F, Record, Idx));
     Constraints.push_back(cast_or_null<StringLiteral>(Reader.ReadSubStmt()));
     Exprs.push_back(Reader.ReadSubStmt());
   }
 
   // Constraints
-  llvm::SmallVector<StringLiteral*, 16> Clobbers;
+  SmallVector<StringLiteral*, 16> Clobbers;
   for (unsigned I = 0; I != NumClobbers; ++I)
     Clobbers.push_back(cast_or_null<StringLiteral>(Reader.ReadSubStmt()));
 
-  S->setOutputsAndInputsAndClobbers(*Reader.getContext(),
+  S->setOutputsAndInputsAndClobbers(Reader.getContext(),
                                     Names.data(), Constraints.data(), 
                                     Exprs.data(), NumOutputs, NumInputs, 
                                     Clobbers.data(), NumClobbers);
@@ -287,7 +305,7 @@ void ASTStmtReader::VisitAsmStmt(AsmStmt *S) {
 
 void ASTStmtReader::VisitExpr(Expr *E) {
   VisitStmt(E);
-  E->setType(Reader.GetType(Record[Idx++]));
+  E->setType(Reader.readType(F, Record, Idx));
   E->setTypeDependent(Record[Idx++]);
   E->setValueDependent(Record[Idx++]);
   E->setInstantiationDependent(Record[Idx++]);
@@ -309,6 +327,7 @@ void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
   E->DeclRefExprBits.HasQualifier = Record[Idx++];
   E->DeclRefExprBits.HasFoundDecl = Record[Idx++];
   E->DeclRefExprBits.HasExplicitTemplateArgs = Record[Idx++];
+  E->DeclRefExprBits.HadMultipleCandidates = Record[Idx++];
   unsigned NumTemplateArgs = 0;
   if (E->hasExplicitTemplateArgs())
     NumTemplateArgs = Record[Idx++];
@@ -318,13 +337,13 @@ void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
       = Reader.ReadNestedNameSpecifierLoc(F, Record, Idx);
 
   if (E->hasFoundDecl())
-    E->getInternalFoundDecl() = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+    E->getInternalFoundDecl() = ReadDeclAs<NamedDecl>(Record, Idx);
 
   if (E->hasExplicitTemplateArgs())
     ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
                                      NumTemplateArgs);
 
-  E->setDecl(cast<ValueDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setDecl(ReadDeclAs<ValueDecl>(Record, Idx));
   E->setLocation(ReadSourceLocation(Record, Idx));
   ReadDeclarationNameLoc(E->DNLoc, E->getDecl()->getDeclName(), Record, Idx);
 }
@@ -332,12 +351,12 @@ void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
 void ASTStmtReader::VisitIntegerLiteral(IntegerLiteral *E) {
   VisitExpr(E);
   E->setLocation(ReadSourceLocation(Record, Idx));
-  E->setValue(*Reader.getContext(), Reader.ReadAPInt(Record, Idx));
+  E->setValue(Reader.getContext(), Reader.ReadAPInt(Record, Idx));
 }
 
 void ASTStmtReader::VisitFloatingLiteral(FloatingLiteral *E) {
   VisitExpr(E);
-  E->setValue(*Reader.getContext(), Reader.ReadAPFloat(Record, Idx));
+  E->setValue(Reader.getContext(), Reader.ReadAPFloat(Record, Idx));
   E->setExact(Record[Idx++]);
   E->setLocation(ReadSourceLocation(Record, Idx));
 }
@@ -353,12 +372,12 @@ void ASTStmtReader::VisitStringLiteral(StringLiteral *E) {
   assert(Record[Idx] == E->getNumConcatenated() &&
          "Wrong number of concatenated tokens!");
   ++Idx;
-  E->IsWide = Record[Idx++];
+  E->Kind = static_cast<StringLiteral::StringKind>(Record[Idx++]);
   E->IsPascal = Record[Idx++];
 
   // Read string data
   llvm::SmallString<16> Str(&Record[Idx], &Record[Idx] + Len);
-  E->setString(*Reader.getContext(), Str.str());
+  E->setString(Reader.getContext(), Str.str());
   Idx += Len;
 
   // Read source locations
@@ -370,7 +389,7 @@ void ASTStmtReader::VisitCharacterLiteral(CharacterLiteral *E) {
   VisitExpr(E);
   E->setValue(Record[Idx++]);
   E->setLocation(ReadSourceLocation(Record, Idx));
-  E->setWide(Record[Idx++]);
+  E->setKind(static_cast<CharacterLiteral::CharacterKind>(Record[Idx++]));
 }
 
 void ASTStmtReader::VisitParenExpr(ParenExpr *E) {
@@ -383,7 +402,7 @@ void ASTStmtReader::VisitParenExpr(ParenExpr *E) {
 void ASTStmtReader::VisitParenListExpr(ParenListExpr *E) {
   VisitExpr(E);
   unsigned NumExprs = Record[Idx++];
-  E->Exprs = new (*Reader.getContext()) Stmt*[NumExprs];
+  E->Exprs = new (Reader.getContext()) Stmt*[NumExprs];
   for (unsigned i = 0; i != NumExprs; ++i)
     E->Exprs[i] = Reader.ReadSubStmt();
   E->NumExprs = NumExprs;
@@ -418,18 +437,18 @@ void ASTStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
       break;
         
     case Node::Field:
-      E->setComponent(I, 
-             Node(Start,  
-                  dyn_cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++])),
-                  End));
+      E->setComponent(I, Node(Start, ReadDeclAs<FieldDecl>(Record, Idx), End));
       break;
 
     case Node::Identifier:
-      E->setComponent(I, Node(Start, Reader.GetIdentifier(Record[Idx++]), End));
+      E->setComponent(I, 
+                      Node(Start, 
+                           Reader.GetIdentifierInfo(F, Record, Idx),
+                           End));
       break;
         
     case Node::Base: {
-      CXXBaseSpecifier *Base = new (*Reader.getContext()) CXXBaseSpecifier();
+      CXXBaseSpecifier *Base = new (Reader.getContext()) CXXBaseSpecifier();
       *Base = Reader.ReadCXXBaseSpecifier(F, Record, Idx);
       E->setComponent(I, Node(Base));
       break;
@@ -463,7 +482,7 @@ void ASTStmtReader::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
 
 void ASTStmtReader::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
-  E->setNumArgs(*Reader.getContext(), Record[Idx++]);
+  E->setNumArgs(Reader.getContext(), Record[Idx++]);
   E->setRParenLoc(ReadSourceLocation(Record, Idx));
   E->setCallee(Reader.ReadSubExpr());
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
@@ -509,7 +528,7 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) {
   E->setCastKind((CastExpr::CastKind)Record[Idx++]);
   CastExpr::path_iterator BaseI = E->path_begin();
   while (NumBaseSpecs--) {
-    CXXBaseSpecifier *BaseSpec = new (*Reader.getContext()) CXXBaseSpecifier;
+    CXXBaseSpecifier *BaseSpec = new (Reader.getContext()) CXXBaseSpecifier;
     *BaseSpec = Reader.ReadCXXBaseSpecifier(F, Record, Idx);
     *BaseI++ = BaseSpec;
   }
@@ -525,8 +544,8 @@ void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
 
 void ASTStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
   VisitBinaryOperator(E);
-  E->setComputationLHSType(Reader.GetType(Record[Idx++]));
-  E->setComputationResultType(Reader.GetType(Record[Idx++]));
+  E->setComputationLHSType(Reader.readType(F, Record, Idx));
+  E->setComputationResultType(Reader.readType(F, Record, Idx));
 }
 
 void ASTStmtReader::VisitConditionalOperator(ConditionalOperator *E) {
@@ -578,7 +597,7 @@ void ASTStmtReader::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
 void ASTStmtReader::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
   VisitExpr(E);
   E->setBase(Reader.ReadSubExpr());
-  E->setAccessor(Reader.GetIdentifierInfo(Record, Idx));
+  E->setAccessor(Reader.GetIdentifierInfo(F, Record, Idx));
   E->setAccessorLoc(ReadSourceLocation(Record, Idx));
 }
 
@@ -593,19 +612,18 @@ void ASTStmtReader::VisitInitListExpr(InitListExpr *E) {
     filler = Reader.ReadSubExpr();
     E->ArrayFillerOrUnionFieldInit = filler;
   } else
-    E->ArrayFillerOrUnionFieldInit
-        = cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++]));
+    E->ArrayFillerOrUnionFieldInit = ReadDeclAs<FieldDecl>(Record, Idx);
   E->sawArrayRangeDesignator(Record[Idx++]);
   unsigned NumInits = Record[Idx++];
-  E->reserveInits(*Reader.getContext(), NumInits);
+  E->reserveInits(Reader.getContext(), NumInits);
   if (isArrayFiller) {
     for (unsigned I = 0; I != NumInits; ++I) {
       Expr *init = Reader.ReadSubExpr();
-      E->updateInit(*Reader.getContext(), I, init ? init : filler);
+      E->updateInit(Reader.getContext(), I, init ? init : filler);
     }
   } else {
     for (unsigned I = 0; I != NumInits; ++I)
-      E->updateInit(*Reader.getContext(), I, Reader.ReadSubExpr());
+      E->updateInit(Reader.getContext(), I, Reader.ReadSubExpr());
   }
 }
 
@@ -620,11 +638,11 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
   E->setEqualOrColonLoc(ReadSourceLocation(Record, Idx));
   E->setGNUSyntax(Record[Idx++]);
 
-  llvm::SmallVector<Designator, 4> Designators;
+  SmallVector<Designator, 4> Designators;
   while (Idx < Record.size()) {
     switch ((DesignatorTypes)Record[Idx++]) {
     case DESIG_FIELD_DECL: {
-      FieldDecl *Field = cast<FieldDecl>(Reader.GetDecl(Record[Idx++]));
+      FieldDecl *Field = ReadDeclAs<FieldDecl>(Record, Idx);
       SourceLocation DotLoc
         = ReadSourceLocation(Record, Idx);
       SourceLocation FieldLoc
@@ -636,7 +654,7 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
     }
 
     case DESIG_FIELD_NAME: {
-      const IdentifierInfo *Name = Reader.GetIdentifierInfo(Record, Idx);
+      const IdentifierInfo *Name = Reader.GetIdentifierInfo(F, Record, Idx);
       SourceLocation DotLoc
         = ReadSourceLocation(Record, Idx);
       SourceLocation FieldLoc
@@ -669,7 +687,7 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
     }
     }
   }
-  E->setDesignators(*Reader.getContext(), 
+  E->setDesignators(Reader.getContext(), 
                     Designators.data(), Designators.size());
 }
 
@@ -689,7 +707,7 @@ void ASTStmtReader::VisitAddrLabelExpr(AddrLabelExpr *E) {
   VisitExpr(E);
   E->setAmpAmpLoc(ReadSourceLocation(Record, Idx));
   E->setLabelLoc(ReadSourceLocation(Record, Idx));
-  E->setLabel(cast<LabelDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setLabel(ReadDeclAs<LabelDecl>(Record, Idx));
 }
 
 void ASTStmtReader::VisitStmtExpr(StmtExpr *E) {
@@ -715,23 +733,23 @@ void ASTStmtReader::VisitGNUNullExpr(GNUNullExpr *E) {
 
 void ASTStmtReader::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
   VisitExpr(E);
-  llvm::SmallVector<Expr *, 16> Exprs;
+  SmallVector<Expr *, 16> Exprs;
   unsigned NumExprs = Record[Idx++];
   while (NumExprs--)
     Exprs.push_back(Reader.ReadSubExpr());
-  E->setExprs(*Reader.getContext(), Exprs.data(), Exprs.size());
+  E->setExprs(Reader.getContext(), Exprs.data(), Exprs.size());
   E->setBuiltinLoc(ReadSourceLocation(Record, Idx));
   E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitBlockExpr(BlockExpr *E) {
   VisitExpr(E);
-  E->setBlockDecl(cast_or_null<BlockDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setBlockDecl(ReadDeclAs<BlockDecl>(Record, Idx));
 }
 
 void ASTStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
   VisitExpr(E);
-  E->setDecl(cast<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setDecl(ReadDeclAs<VarDecl>(Record, Idx));
   E->setLocation(ReadSourceLocation(Record, Idx));
   E->setByRef(Record[Idx++]);
   E->setConstQualAdded(Record[Idx++]);
@@ -740,9 +758,9 @@ void ASTStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
 void ASTStmtReader::VisitGenericSelectionExpr(GenericSelectionExpr *E) {
   VisitExpr(E);
   E->NumAssocs = Record[Idx++];
-  E->AssocTypes = new (*Reader.getContext()) TypeSourceInfo*[E->NumAssocs];
+  E->AssocTypes = new (Reader.getContext()) TypeSourceInfo*[E->NumAssocs];
   E->SubExprs =
-   new(*Reader.getContext()) Stmt*[GenericSelectionExpr::END_EXPR+E->NumAssocs];
+   new(Reader.getContext()) Stmt*[GenericSelectionExpr::END_EXPR+E->NumAssocs];
 
   E->SubExprs[GenericSelectionExpr::CONTROLLING] = Reader.ReadSubExpr();
   for (unsigned I = 0, N = E->getNumAssocs(); I != N; ++I) {
@@ -756,6 +774,25 @@ void ASTStmtReader::VisitGenericSelectionExpr(GenericSelectionExpr *E) {
   E->RParenLoc = ReadSourceLocation(Record, Idx);
 }
 
+void ASTStmtReader::VisitAtomicExpr(AtomicExpr *E) {
+  VisitExpr(E);
+  E->setOp(AtomicExpr::AtomicOp(Record[Idx++]));
+  E->setPtr(Reader.ReadSubExpr());
+  E->setOrder(Reader.ReadSubExpr());
+  E->setNumSubExprs(2);
+  if (E->getOp() != AtomicExpr::Load) {
+    E->setVal1(Reader.ReadSubExpr());
+    E->setNumSubExprs(3);
+  }
+  if (E->isCmpXChg()) {
+    E->setOrderFail(Reader.ReadSubExpr());
+    E->setVal2(Reader.ReadSubExpr());
+    E->setNumSubExprs(5);
+  }
+  E->setBuiltinLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
+}
+
 //===----------------------------------------------------------------------===//
 // Objective-C Expressions and Statements
 
@@ -774,21 +811,21 @@ void ASTStmtReader::VisitObjCEncodeExpr(ObjCEncodeExpr *E) {
 
 void ASTStmtReader::VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
   VisitExpr(E);
-  E->setSelector(Reader.GetSelector(Record, Idx));
+  E->setSelector(Reader.ReadSelector(F, Record, Idx));
   E->setAtLoc(ReadSourceLocation(Record, Idx));
   E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
   VisitExpr(E);
-  E->setProtocol(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setProtocol(ReadDeclAs<ObjCProtocolDecl>(Record, Idx));
   E->setAtLoc(ReadSourceLocation(Record, Idx));
   E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   VisitExpr(E);
-  E->setDecl(cast<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setDecl(ReadDeclAs<ObjCIvarDecl>(Record, Idx));
   E->setLocation(ReadSourceLocation(Record, Idx));
   E->setBase(Reader.ReadSubExpr());
   E->setIsArrow(Record[Idx++]);
@@ -799,14 +836,11 @@ void ASTStmtReader::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   VisitExpr(E);
   bool Implicit = Record[Idx++] != 0;
   if (Implicit) {
-    ObjCMethodDecl *Getter =
-      cast<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++]));
-    ObjCMethodDecl *Setter =
-      cast<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++]));
+    ObjCMethodDecl *Getter = ReadDeclAs<ObjCMethodDecl>(Record, Idx);
+    ObjCMethodDecl *Setter = ReadDeclAs<ObjCMethodDecl>(Record, Idx);
     E->setImplicitProperty(Getter, Setter);
   } else {
-    E->setExplicitProperty(
-                      cast<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
+    E->setExplicitProperty(ReadDeclAs<ObjCPropertyDecl>(Record, Idx));
   }
   E->setLocation(ReadSourceLocation(Record, Idx));
   E->setReceiverLocation(ReadSourceLocation(Record, Idx));
@@ -815,11 +849,10 @@ void ASTStmtReader::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
     E->setBase(Reader.ReadSubExpr());
     break;
   case 1:
-    E->setSuperReceiver(Reader.GetType(Record[Idx++]));
+    E->setSuperReceiver(Reader.readType(F, Record, Idx));
     break;
   case 2:
-    E->setClassReceiver(
-                     cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
+    E->setClassReceiver(ReadDeclAs<ObjCInterfaceDecl>(Record, Idx));
     break;
   }
 }
@@ -828,6 +861,8 @@ void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   VisitExpr(E);
   assert(Record[Idx] == E->getNumArgs());
   ++Idx;
+  unsigned NumStoredSelLocs = Record[Idx++];
+  E->SelLocsKind = Record[Idx++]; 
   E->setDelegateInitCall(Record[Idx++]);
   ObjCMessageExpr::ReceiverKind Kind
     = static_cast<ObjCMessageExpr::ReceiverKind>(Record[Idx++]);
@@ -842,7 +877,7 @@ void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
 
   case ObjCMessageExpr::SuperClass:
   case ObjCMessageExpr::SuperInstance: {
-    QualType T = Reader.GetType(Record[Idx++]);
+    QualType T = Reader.readType(F, Record, Idx);
     SourceLocation SuperLoc = ReadSourceLocation(Record, Idx);
     E->setSuper(SuperLoc, T, Kind == ObjCMessageExpr::SuperInstance);
     break;
@@ -852,16 +887,19 @@ void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   assert(Kind == E->getReceiverKind());
 
   if (Record[Idx++])
-    E->setMethodDecl(cast_or_null<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++])));
+    E->setMethodDecl(ReadDeclAs<ObjCMethodDecl>(Record, Idx));
   else
-    E->setSelector(Reader.GetSelector(Record, Idx));
+    E->setSelector(Reader.ReadSelector(F, Record, Idx));
 
   E->LBracLoc = ReadSourceLocation(Record, Idx);
   E->RBracLoc = ReadSourceLocation(Record, Idx);
-  E->SelectorLoc = ReadSourceLocation(Record, Idx);
 
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
     E->setArg(I, Reader.ReadSubExpr());
+
+  SourceLocation *Locs = E->getStoredSelLocs();
+  for (unsigned I = 0; I != NumStoredSelLocs; ++I)
+    Locs[I] = ReadSourceLocation(Record, Idx);
 }
 
 void ASTStmtReader::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
@@ -876,7 +914,7 @@ void ASTStmtReader::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
 void ASTStmtReader::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   VisitStmt(S);
   S->setCatchBody(Reader.ReadSubStmt());
-  S->setCatchParamDecl(cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setCatchParamDecl(ReadDeclAs<VarDecl>(Record, Idx));
   S->setAtCatchLoc(ReadSourceLocation(Record, Idx));
   S->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
@@ -927,7 +965,7 @@ void ASTStmtReader::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
 void ASTStmtReader::VisitCXXCatchStmt(CXXCatchStmt *S) {
   VisitStmt(S);
   S->CatchLoc = ReadSourceLocation(Record, Idx);
-  S->ExceptionDecl = cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++]));
+  S->ExceptionDecl = ReadDeclAs<VarDecl>(Record, Idx);
   S->HandlerBlock = Reader.ReadSubStmt();
 }
 
@@ -963,12 +1001,13 @@ void ASTStmtReader::VisitCXXConstructExpr(CXXConstructExpr *E) {
   VisitExpr(E);
   E->NumArgs = Record[Idx++];
   if (E->NumArgs)
-    E->Args = new (*Reader.getContext()) Stmt*[E->NumArgs];
+    E->Args = new (Reader.getContext()) Stmt*[E->NumArgs];
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
     E->setArg(I, Reader.ReadSubExpr());
-  E->setConstructor(cast<CXXConstructorDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setConstructor(ReadDeclAs<CXXConstructorDecl>(Record, Idx));
   E->setLocation(ReadSourceLocation(Record, Idx));
-  E->setElidable(Record[Idx++]);  
+  E->setElidable(Record[Idx++]);
+  E->setHadMultipleCandidates(Record[Idx++]);
   E->setRequiresZeroInitialization(Record[Idx++]);
   E->setConstructionKind((CXXConstructExpr::ConstructionKind)Record[Idx++]);
   E->ParenRange = ReadSourceRange(Record, Idx);
@@ -1048,15 +1087,15 @@ void ASTStmtReader::VisitCXXThrowExpr(CXXThrowExpr *E) {
 void ASTStmtReader::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   VisitExpr(E);
 
-  assert(Record[Idx] == E->Param.getInt() && "We messed up at creation ?");
+  assert((bool)Record[Idx] == E->Param.getInt() && "We messed up at creation ?");
   ++Idx; // HasOtherExprStored and SubExpr was handled during creation.
-  E->Param.setPointer(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
+  E->Param.setPointer(ReadDeclAs<ParmVarDecl>(Record, Idx));
   E->Loc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTStmtReader::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
   VisitExpr(E);
-  E->setTemporary(Reader.ReadCXXTemporary(Record, Idx));
+  E->setTemporary(Reader.ReadCXXTemporary(F, Record, Idx));
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
@@ -1072,13 +1111,12 @@ void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
   E->Initializer = Record[Idx++];
   E->UsualArrayDeleteWantsSize = Record[Idx++];
   bool isArray = Record[Idx++];
+  E->setHadMultipleCandidates(Record[Idx++]);
   unsigned NumPlacementArgs = Record[Idx++];
   unsigned NumCtorArgs = Record[Idx++];
-  E->setOperatorNew(cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setOperatorDelete(
-                    cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setConstructor(
-               cast_or_null<CXXConstructorDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setOperatorNew(ReadDeclAs<FunctionDecl>(Record, Idx));
+  E->setOperatorDelete(ReadDeclAs<FunctionDecl>(Record, Idx));
+  E->setConstructor(ReadDeclAs<CXXConstructorDecl>(Record, Idx));
   E->AllocatedTypeInfo = GetTypeSourceInfo(Record, Idx);
   SourceRange TypeIdParens;
   TypeIdParens.setBegin(ReadSourceLocation(Record, Idx));
@@ -1089,7 +1127,7 @@ void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
   E->ConstructorLParen = ReadSourceLocation(Record, Idx);
   E->ConstructorRParen = ReadSourceLocation(Record, Idx);
 
-  E->AllocateArgsArray(*Reader.getContext(), isArray, NumPlacementArgs,
+  E->AllocateArgsArray(Reader.getContext(), isArray, NumPlacementArgs,
                        NumCtorArgs);
 
   // Install all the subexpressions.
@@ -1104,7 +1142,7 @@ void ASTStmtReader::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   E->ArrayForm = Record[Idx++];
   E->ArrayFormAsWritten = Record[Idx++];
   E->UsualArrayDeleteWantsSize = Record[Idx++];
-  E->OperatorDelete = cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++]));
+  E->OperatorDelete = ReadDeclAs<FunctionDecl>(Record, Idx);
   E->Argument = Reader.ReadSubExpr();
   E->Loc = ReadSourceLocation(Record, Idx);
 }
@@ -1120,7 +1158,7 @@ void ASTStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   E->ColonColonLoc = ReadSourceLocation(Record, Idx);
   E->TildeLoc = ReadSourceLocation(Record, Idx);
   
-  IdentifierInfo *II = Reader.GetIdentifierInfo(Record, Idx);
+  IdentifierInfo *II = Reader.GetIdentifierInfo(F, Record, Idx);
   if (II)
     E->setDestroyedType(II, ReadSourceLocation(Record, Idx));
   else
@@ -1131,9 +1169,9 @@ void ASTStmtReader::VisitExprWithCleanups(ExprWithCleanups *E) {
   VisitExpr(E);
   unsigned NumTemps = Record[Idx++];
   if (NumTemps) {
-    E->setNumTemporaries(*Reader.getContext(), NumTemps);
+    E->setNumTemporaries(Reader.getContext(), NumTemps);
     for (unsigned i = 0; i != NumTemps; ++i)
-      E->setTemporary(i, Reader.ReadCXXTemporary(Record, Idx));
+      E->setTemporary(i, Reader.ReadCXXTemporary(F, Record, Idx));
   }
   E->setSubExpr(Reader.ReadSubExpr());
 }
@@ -1147,12 +1185,11 @@ ASTStmtReader::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
                                      Record[Idx++]);
 
   E->Base = Reader.ReadSubExpr();
-  E->BaseType = Reader.GetType(Record[Idx++]);
+  E->BaseType = Reader.readType(F, Record, Idx);
   E->IsArrow = Record[Idx++];
   E->OperatorLoc = ReadSourceLocation(Record, Idx);
   E->QualifierLoc = Reader.ReadNestedNameSpecifierLoc(F, Record, Idx);
-  E->FirstQualifierFoundInScope
-                      = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+  E->FirstQualifierFoundInScope = ReadDeclAs<NamedDecl>(Record, Idx);
   ReadDeclarationNameInfo(E->MemberNameInfo, Record, Idx);
 }
 
@@ -1191,11 +1228,11 @@ void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) {
   unsigned NumDecls = Record[Idx++];
   UnresolvedSet<8> Decls;
   for (unsigned i = 0; i != NumDecls; ++i) {
-    NamedDecl *D = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+    NamedDecl *D = ReadDeclAs<NamedDecl>(Record, Idx);
     AccessSpecifier AS = (AccessSpecifier)Record[Idx++];
     Decls.addDecl(D, AS);
   }
-  E->initializeResults(*Reader.getContext(), Decls.begin(), Decls.end());
+  E->initializeResults(Reader.getContext(), Decls.begin(), Decls.end());
 
   ReadDeclarationNameInfo(E->NameInfo, Record, Idx);
   E->QualifierLoc = Reader.ReadNestedNameSpecifierLoc(F, Record, Idx);
@@ -1206,7 +1243,7 @@ void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
   E->IsArrow = Record[Idx++];
   E->HasUnresolvedUsing = Record[Idx++];
   E->Base = Reader.ReadSubExpr();
-  E->BaseType = Reader.GetType(Record[Idx++]);
+  E->BaseType = Reader.readType(F, Record, Idx);
   E->OperatorLoc = ReadSourceLocation(Record, Idx);
 }
 
@@ -1216,7 +1253,7 @@ void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
   if (E->RequiresADL)
     E->StdIsAssociatedNamespace = Record[Idx++];
   E->Overloaded = Record[Idx++];
-  E->NamingClass = cast_or_null<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
+  E->NamingClass = ReadDeclAs<CXXRecordDecl>(Record, Idx);
 }
 
 void ASTStmtReader::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
@@ -1280,14 +1317,13 @@ void ASTStmtReader::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
   E->PackLoc = ReadSourceLocation(Record, Idx);
   E->RParenLoc = ReadSourceLocation(Record, Idx);
   E->Length = Record[Idx++];
-  E->Pack = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+  E->Pack = ReadDeclAs<NamedDecl>(Record, Idx);
 }
 
 void ASTStmtReader::VisitSubstNonTypeTemplateParmExpr(
                                               SubstNonTypeTemplateParmExpr *E) {
   VisitExpr(E);
-  E->Param
-    = cast_or_null<NonTypeTemplateParmDecl>(Reader.GetDecl(Record[Idx++]));
+  E->Param = ReadDeclAs<NonTypeTemplateParmDecl>(Record, Idx);
   E->NameLoc = ReadSourceLocation(Record, Idx);
   E->Replacement = Reader.ReadSubExpr();
 }
@@ -1295,8 +1331,7 @@ void ASTStmtReader::VisitSubstNonTypeTemplateParmExpr(
 void ASTStmtReader::VisitSubstNonTypeTemplateParmPackExpr(
                                           SubstNonTypeTemplateParmPackExpr *E) {
   VisitExpr(E);
-  E->Param
-    = cast_or_null<NonTypeTemplateParmDecl>(Reader.GetDecl(Record[Idx++]));
+  E->Param = ReadDeclAs<NonTypeTemplateParmDecl>(Record, Idx);
   TemplateArgument ArgPack = Reader.ReadTemplateArgument(F, Record, Idx);
   if (ArgPack.getKind() != TemplateArgument::Pack)
     return;
@@ -1377,7 +1412,7 @@ void ASTStmtReader::VisitAsTypeExpr(AsTypeExpr *E) {
 // ASTReader Implementation
 //===----------------------------------------------------------------------===//
 
-Stmt *ASTReader::ReadStmt(PerFileData &F) {
+Stmt *ASTReader::ReadStmt(Module &F) {
   switch (ReadingKind) {
   case Read_Decl:
   case Read_Type:
@@ -1390,7 +1425,7 @@ Stmt *ASTReader::ReadStmt(PerFileData &F) {
   return 0;
 }
 
-Expr *ASTReader::ReadExpr(PerFileData &F) {
+Expr *ASTReader::ReadExpr(Module &F) {
   return cast_or_null<Expr>(ReadStmt(F));
 }
 
@@ -1405,7 +1440,7 @@ Expr *ASTReader::ReadSubExpr() {
 // the stack, with expressions having operands removing those operands from the
 // stack. Evaluation terminates when we see a STMT_STOP record, and
 // the single remaining expression on the stack is our result.
-Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
+Stmt *ASTReader::ReadStmtFromStream(Module &F) {
 
   ReadingKindTracker ReadingKind(Read_Stmt, *this);
   llvm::BitstreamCursor &Cursor = F.DeclsCursor;
@@ -1531,20 +1566,20 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
 
     case EXPR_DECL_REF:
       S = DeclRefExpr::CreateEmpty(
-        *Context,
+        Context,
         /*HasQualifier=*/Record[ASTStmtReader::NumExprFields],
         /*HasFoundDecl=*/Record[ASTStmtReader::NumExprFields + 1],
         /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 2],
         /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 2] ?
-          Record[ASTStmtReader::NumExprFields + 3] : 0);
+          Record[ASTStmtReader::NumExprFields + 4] : 0);
       break;
 
     case EXPR_INTEGER_LITERAL:
-      S = IntegerLiteral::Create(*Context, Empty);
+      S = IntegerLiteral::Create(Context, Empty);
       break;
 
     case EXPR_FLOATING_LITERAL:
-      S = FloatingLiteral::Create(*Context, Empty);
+      S = FloatingLiteral::Create(Context, Empty);
       break;
 
     case EXPR_IMAGINARY_LITERAL:
@@ -1552,7 +1587,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_STRING_LITERAL:
-      S = StringLiteral::CreateEmpty(*Context,
+      S = StringLiteral::CreateEmpty(Context,
                                      Record[ASTStmtReader::NumExprFields + 1]);
       break;
 
@@ -1573,7 +1608,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_OFFSETOF:
-      S = OffsetOfExpr::CreateEmpty(*Context, 
+      S = OffsetOfExpr::CreateEmpty(Context, 
                                     Record[ASTStmtReader::NumExprFields],
                                     Record[ASTStmtReader::NumExprFields + 1]);
       break;
@@ -1587,7 +1622,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_CALL:
-      S = new (Context) CallExpr(*Context, Stmt::CallExprClass, Empty);
+      S = new (Context) CallExpr(Context, Stmt::CallExprClass, Empty);
       break;
 
     case EXPR_MEMBER: {
@@ -1610,25 +1645,29 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
         for (unsigned i = 0; i != NumTemplateArgs; ++i)
           ArgInfo.addArgument(ReadTemplateArgumentLoc(F, Record, Idx));
       }
-      
-      NamedDecl *FoundD = cast_or_null<NamedDecl>(GetDecl(Record[Idx++]));
+
+      bool HadMultipleCandidates = Record[Idx++];
+
+      NamedDecl *FoundD = ReadDeclAs<NamedDecl>(F, Record, Idx);
       AccessSpecifier AS = (AccessSpecifier)Record[Idx++];
       DeclAccessPair FoundDecl = DeclAccessPair::make(FoundD, AS);
 
-      QualType T = GetType(Record[Idx++]);
+      QualType T = readType(F, Record, Idx);
       ExprValueKind VK = static_cast<ExprValueKind>(Record[Idx++]);
       ExprObjectKind OK = static_cast<ExprObjectKind>(Record[Idx++]);
       Expr *Base = ReadSubExpr();
-      ValueDecl *MemberD = cast<ValueDecl>(GetDecl(Record[Idx++]));
+      ValueDecl *MemberD = ReadDeclAs<ValueDecl>(F, Record, Idx);
       SourceLocation MemberLoc = ReadSourceLocation(F, Record, Idx);
       DeclarationNameInfo MemberNameInfo(MemberD->getDeclName(), MemberLoc);
       bool IsArrow = Record[Idx++];
 
-      S = MemberExpr::Create(*Context, Base, IsArrow, QualifierLoc,
+      S = MemberExpr::Create(Context, Base, IsArrow, QualifierLoc,
                              MemberD, FoundDecl, MemberNameInfo,
                              HasExplicitTemplateArgs ? &ArgInfo : 0, T, VK, OK);
       ReadDeclarationNameLoc(F, cast<MemberExpr>(S)->MemberDNLoc,
                              MemberD->getDeclName(), Record, Idx);
+      if (HadMultipleCandidates)
+        cast<MemberExpr>(S)->setHadMultipleCandidates(true);
       break;
     }
 
@@ -1649,12 +1688,12 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_IMPLICIT_CAST:
-      S = ImplicitCastExpr::CreateEmpty(*Context,
+      S = ImplicitCastExpr::CreateEmpty(Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
     case EXPR_CSTYLE_CAST:
-      S = CStyleCastExpr::CreateEmpty(*Context,
+      S = CStyleCastExpr::CreateEmpty(Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
@@ -1667,11 +1706,11 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_INIT_LIST:
-      S = new (Context) InitListExpr(*getContext(), Empty);
+      S = new (Context) InitListExpr(getContext(), Empty);
       break;
 
     case EXPR_DESIGNATED_INIT:
-      S = DesignatedInitExpr::CreateEmpty(*Context,
+      S = DesignatedInitExpr::CreateEmpty(Context,
                                      Record[ASTStmtReader::NumExprFields] - 1);
 
       break;
@@ -1738,8 +1777,9 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       llvm_unreachable("mismatching AST file");
       break;
     case EXPR_OBJC_MESSAGE_EXPR:
-      S = ObjCMessageExpr::CreateEmpty(*Context,
-                                     Record[ASTStmtReader::NumExprFields]);
+      S = ObjCMessageExpr::CreateEmpty(Context,
+                                     Record[ASTStmtReader::NumExprFields],
+                                     Record[ASTStmtReader::NumExprFields + 1]);
       break;
     case EXPR_OBJC_ISA:
       S = new (Context) ObjCIsaExpr(Empty);
@@ -1760,7 +1800,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       S = new (Context) ObjCAtFinallyStmt(Empty);
       break;
     case STMT_OBJC_AT_TRY:
-      S = ObjCAtTryStmt::CreateEmpty(*Context, 
+      S = ObjCAtTryStmt::CreateEmpty(Context, 
                                      Record[ASTStmtReader::NumStmtFields],
                                      Record[ASTStmtReader::NumStmtFields + 1]);
       break;
@@ -1787,7 +1827,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case STMT_CXX_TRY:
-      S = CXXTryStmt::Create(*Context, Empty,
+      S = CXXTryStmt::Create(Context, Empty,
              /*NumHandlers=*/Record[ASTStmtReader::NumStmtFields]);
       break;
 
@@ -1796,11 +1836,11 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_CXX_OPERATOR_CALL:
-      S = new (Context) CXXOperatorCallExpr(*Context, Empty);
+      S = new (Context) CXXOperatorCallExpr(Context, Empty);
       break;
 
     case EXPR_CXX_MEMBER_CALL:
-      S = new (Context) CXXMemberCallExpr(*Context, Empty);
+      S = new (Context) CXXMemberCallExpr(Context, Empty);
       break;
         
     case EXPR_CXX_CONSTRUCT:
@@ -1812,26 +1852,26 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
 
     case EXPR_CXX_STATIC_CAST:
-      S = CXXStaticCastExpr::CreateEmpty(*Context,
+      S = CXXStaticCastExpr::CreateEmpty(Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
     case EXPR_CXX_DYNAMIC_CAST:
-      S = CXXDynamicCastExpr::CreateEmpty(*Context,
+      S = CXXDynamicCastExpr::CreateEmpty(Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
     case EXPR_CXX_REINTERPRET_CAST:
-      S = CXXReinterpretCastExpr::CreateEmpty(*Context,
+      S = CXXReinterpretCastExpr::CreateEmpty(Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
     case EXPR_CXX_CONST_CAST:
-      S = CXXConstCastExpr::CreateEmpty(*Context);
+      S = CXXConstCastExpr::CreateEmpty(Context);
       break;
 
     case EXPR_CXX_FUNCTIONAL_CAST:
-      S = CXXFunctionalCastExpr::CreateEmpty(*Context,
+      S = CXXFunctionalCastExpr::CreateEmpty(Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
@@ -1864,7 +1904,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       bool HasOtherExprStored = Record[ASTStmtReader::NumExprFields];
       if (HasOtherExprStored) {
         Expr *SubExpr = ReadSubExpr();
-        S = CXXDefaultArgExpr::Create(*Context, SourceLocation(), 0, SubExpr);
+        S = CXXDefaultArgExpr::Create(Context, SourceLocation(), 0, SubExpr);
       } else
         S = new (Context) CXXDefaultArgExpr(Empty);
       break;
@@ -1891,7 +1931,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
       
     case EXPR_CXX_DEPENDENT_SCOPE_MEMBER:
-      S = CXXDependentScopeMemberExpr::CreateEmpty(*Context,
+      S = CXXDependentScopeMemberExpr::CreateEmpty(Context,
           /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
                   /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
                                    ? Record[ASTStmtReader::NumExprFields + 1] 
@@ -1899,7 +1939,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
       
     case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF:
-      S = DependentScopeDeclRefExpr::CreateEmpty(*Context,
+      S = DependentScopeDeclRefExpr::CreateEmpty(Context,
           /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
                   /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
                                    ? Record[ASTStmtReader::NumExprFields + 1] 
@@ -1907,12 +1947,12 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
       
     case EXPR_CXX_UNRESOLVED_CONSTRUCT:
-      S = CXXUnresolvedConstructExpr::CreateEmpty(*Context,
+      S = CXXUnresolvedConstructExpr::CreateEmpty(Context,
                               /*NumArgs=*/Record[ASTStmtReader::NumExprFields]);
       break;
       
     case EXPR_CXX_UNRESOLVED_MEMBER:
-      S = UnresolvedMemberExpr::CreateEmpty(*Context,
+      S = UnresolvedMemberExpr::CreateEmpty(Context,
           /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
                   /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
                                    ? Record[ASTStmtReader::NumExprFields + 1] 
@@ -1920,7 +1960,7 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
       break;
       
     case EXPR_CXX_UNRESOLVED_LOOKUP:
-      S = UnresolvedLookupExpr::CreateEmpty(*Context,
+      S = UnresolvedLookupExpr::CreateEmpty(Context,
           /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
                   /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
                                    ? Record[ASTStmtReader::NumExprFields + 1] 
@@ -1983,12 +2023,16 @@ Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
     }
 
     case EXPR_CUDA_KERNEL_CALL:
-      S = new (Context) CUDAKernelCallExpr(*Context, Empty);
+      S = new (Context) CUDAKernelCallExpr(Context, Empty);
       break;
         
     case EXPR_ASTYPE:
       S = new (Context) AsTypeExpr(Empty);
       break;
+
+    case EXPR_ATOMIC:
+      S = new (Context) AtomicExpr(Empty);
+      break;
     }
     
     // We hit a STMT_STOP, so we're done with this expression.
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp
index 39f2fbddf80e..b31262d375fe 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Serialization/ASTWriter.h"
-#include "clang/Serialization/ASTSerializationListener.h"
 #include "ASTCommon.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/IdentifierResolver.h"
@@ -45,21 +44,23 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include <algorithm>
 #include <cstdio>
 #include <string.h>
+#include <utility>
 using namespace clang;
 using namespace clang::serialization;
 
 template <typename T, typename Allocator>
-static llvm::StringRef data(const std::vector<T, Allocator> &v) {
-  if (v.empty()) return llvm::StringRef();
-  return llvm::StringRef(reinterpret_cast<const char*>(&v[0]),
+static StringRef data(const std::vector<T, Allocator> &v) {
+  if (v.empty()) return StringRef();
+  return StringRef(reinterpret_cast<const char*>(&v[0]),
                          sizeof(T) * v.size());
 }
 
 template <typename T>
-static llvm::StringRef data(const llvm::SmallVectorImpl<T> &v) {
-  return llvm::StringRef(reinterpret_cast<const char*>(v.data()),
+static StringRef data(const SmallVectorImpl<T> &v) {
+  return StringRef(reinterpret_cast<const char*>(v.data()),
                          sizeof(T) * v.size());
 }
 
@@ -90,7 +91,7 @@ namespace {
 }
 
 void ASTTypeWriter::VisitBuiltinType(const BuiltinType *T) {
-  assert(false && "Built-in types are never serialized");
+  llvm_unreachable("Built-in types are never serialized");
 }
 
 void ASTTypeWriter::VisitComplexType(const ComplexType *T) {
@@ -304,7 +305,7 @@ void
 ASTTypeWriter::VisitDependentSizedExtVectorType(
                                         const DependentSizedExtVectorType *T) {
   // FIXME: Serialize this type (C++ only)
-  assert(false && "Cannot serialize dependent sized extended vector types");
+  llvm_unreachable("Cannot serialize dependent sized extended vector types");
 }
 
 void
@@ -387,6 +388,12 @@ ASTTypeWriter::VisitObjCObjectPointerType(const ObjCObjectPointerType *T) {
   Code = TYPE_OBJC_OBJECT_POINTER;
 }
 
+void
+ASTTypeWriter::VisitAtomicType(const AtomicType *T) {
+  Writer.AddTypeRef(T->getValueType(), Record);
+  Code = TYPE_ATOMIC;
+}
+
 namespace {
 
 class TypeLocWriter : public TypeLocVisitor<TypeLocWriter> {
@@ -595,6 +602,11 @@ void TypeLocWriter::VisitObjCObjectTypeLoc(ObjCObjectTypeLoc TL) {
 void TypeLocWriter::VisitObjCObjectPointerTypeLoc(ObjCObjectPointerTypeLoc TL) {
   Writer.AddSourceLocation(TL.getStarLoc(), Record);
 }
+void TypeLocWriter::VisitAtomicTypeLoc(AtomicTypeLoc TL) {
+  Writer.AddSourceLocation(TL.getKWLoc(), Record);
+  Writer.AddSourceLocation(TL.getLParenLoc(), Record);
+  Writer.AddSourceLocation(TL.getRParenLoc(), Record);
+}
 
 //===----------------------------------------------------------------------===//
 // ASTWriter Implementation
@@ -762,8 +774,8 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(STAT_CACHE);
   RECORD(EXT_VECTOR_DECLS);
   RECORD(VERSION_CONTROL_BRANCH_REVISION);
-  RECORD(MACRO_DEFINITION_OFFSETS);
-  RECORD(CHAINED_METADATA);
+  RECORD(PPD_ENTITIES_OFFSETS);
+  RECORD(IMPORTS);
   RECORD(REFERENCED_SELECTOR_POOL);
   RECORD(TU_UPDATE_LEXICAL);
   RECORD(REDECLS_UPDATE_LATEST);
@@ -778,11 +790,14 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(DIAG_PRAGMA_MAPPINGS);
   RECORD(CUDA_SPECIAL_DECL_REFS);
   RECORD(HEADER_SEARCH_TABLE);
+  RECORD(ORIGINAL_PCH_DIR);
   RECORD(FP_PRAGMA_OPTIONS);
   RECORD(OPENCL_EXTENSIONS);
   RECORD(DELEGATING_CTORS);
   RECORD(FILE_SOURCE_LOCATION_OFFSETS);
   RECORD(KNOWN_NAMESPACES);
+  RECORD(MODULE_OFFSET_MAP);
+  RECORD(SOURCE_MANAGER_LINE_TABLE);
   
   // SourceManager Block.
   BLOCK(SOURCE_MANAGER_BLOCK);
@@ -790,7 +805,6 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(SM_SLOC_BUFFER_ENTRY);
   RECORD(SM_SLOC_BUFFER_BLOB);
   RECORD(SM_SLOC_EXPANSION_ENTRY);
-  RECORD(SM_LINE_TABLE);
 
   // Preprocessor Block.
   BLOCK(PREPROCESSOR_BLOCK);
@@ -837,7 +851,7 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(TYPE_PACK_EXPANSION);
   RECORD(TYPE_ATTRIBUTED);
   RECORD(TYPE_SUBST_TEMPLATE_TYPE_PARM_PACK);
-  RECORD(DECL_TRANSLATION_UNIT);
+  RECORD(TYPE_ATOMIC);
   RECORD(DECL_TYPEDEF);
   RECORD(DECL_ENUM);
   RECORD(DECL_RECORD);
@@ -916,15 +930,15 @@ void ASTWriter::WriteBlockInfoBlock() {
 /// \returns either the original filename (if it needs no adjustment) or the
 /// adjusted filename (which points into the @p Filename parameter).
 static const char *
-adjustFilenameForRelocatablePCH(const char *Filename, const char *isysroot) {
+adjustFilenameForRelocatablePCH(const char *Filename, StringRef isysroot) {
   assert(Filename && "No file name to adjust?");
 
-  if (!isysroot)
+  if (isysroot.empty())
     return Filename;
 
   // Verify that the filename and the system root have the same prefix.
   unsigned Pos = 0;
-  for (; Filename[Pos] && isysroot[Pos]; ++Pos)
+  for (; Filename[Pos] && Pos < isysroot.size(); ++Pos)
     if (Filename[Pos] != isysroot[Pos])
       return Filename; // Prefixes don't match.
 
@@ -942,34 +956,52 @@ adjustFilenameForRelocatablePCH(const char *Filename, const char *isysroot) {
 }
 
 /// \brief Write the AST metadata (e.g., i686-apple-darwin9).
-void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot,
+void ASTWriter::WriteMetadata(ASTContext &Context, StringRef isysroot,
                               const std::string &OutputFile) {
   using namespace llvm;
 
   // Metadata
-  const TargetInfo &Target = Context.Target;
+  const TargetInfo &Target = Context.getTargetInfo();
   BitCodeAbbrev *MetaAbbrev = new BitCodeAbbrev();
-  MetaAbbrev->Add(BitCodeAbbrevOp(
-                    Chain ? CHAINED_METADATA : METADATA));
+  MetaAbbrev->Add(BitCodeAbbrevOp(METADATA));
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // AST major
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // AST minor
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang major
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang minor
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Relocatable
-  // Target triple or chained PCH name
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Target triple
   unsigned MetaAbbrevCode = Stream.EmitAbbrev(MetaAbbrev);
 
   RecordData Record;
-  Record.push_back(Chain ? CHAINED_METADATA : METADATA);
+  Record.push_back(METADATA);
   Record.push_back(VERSION_MAJOR);
   Record.push_back(VERSION_MINOR);
   Record.push_back(CLANG_VERSION_MAJOR);
   Record.push_back(CLANG_VERSION_MINOR);
-  Record.push_back(isysroot != 0);
-  // FIXME: This writes the absolute path for chained headers.
-  const std::string &BlobStr = Chain ? Chain->getFileName() : Target.getTriple().getTriple();
-  Stream.EmitRecordWithBlob(MetaAbbrevCode, Record, BlobStr);
+  Record.push_back(!isysroot.empty());
+  const std::string &Triple = Target.getTriple().getTriple();
+  Stream.EmitRecordWithBlob(MetaAbbrevCode, Record, Triple);
+
+  if (Chain) {
+    serialization::ModuleManager &Mgr = Chain->getModuleManager();
+    llvm::SmallVector<char, 128> ModulePaths;
+    Record.clear();
+
+    for (ModuleManager::ModuleIterator M = Mgr.begin(), MEnd = Mgr.end();
+         M != MEnd; ++M) {
+      // Skip modules that weren't directly imported.
+      if (!(*M)->isDirectlyImported())
+        continue;
+
+      Record.push_back((unsigned)(*M)->Kind); // FIXME: Stable encoding
+      // FIXME: Write import location, once it matters.
+      // FIXME: This writes the absolute path for AST files we depend on.
+      const std::string &FileName = (*M)->FileName;
+      Record.push_back(FileName.size());
+      Record.append(FileName.begin(), FileName.end());
+    }
+    Stream.EmitRecord(IMPORTS, Record);
+  }
 
   // Original file name and file ID
   SourceManager &SM = Context.getSourceManager();
@@ -1026,94 +1058,11 @@ void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot,
 /// \brief Write the LangOptions structure.
 void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
   RecordData Record;
-  Record.push_back(LangOpts.Trigraphs);
-  Record.push_back(LangOpts.BCPLComment);  // BCPL-style '//' comments.
-  Record.push_back(LangOpts.DollarIdents);  // '$' allowed in identifiers.
-  Record.push_back(LangOpts.AsmPreprocessor);  // Preprocessor in asm mode.
-  Record.push_back(LangOpts.GNUMode);  // True in gnu99 mode false in c99 mode (etc)
-  Record.push_back(LangOpts.GNUKeywords);  // Allow GNU-extension keywords
-  Record.push_back(LangOpts.ImplicitInt);  // C89 implicit 'int'.
-  Record.push_back(LangOpts.Digraphs);  // C94, C99 and C++
-  Record.push_back(LangOpts.HexFloats);  // C99 Hexadecimal float constants.
-  Record.push_back(LangOpts.C99);  // C99 Support
-  Record.push_back(LangOpts.C1X);  // C1X Support
-  Record.push_back(LangOpts.Microsoft);  // Microsoft extensions.
-  // LangOpts.MSCVersion is ignored because all it does it set a macro, which is
-  // already saved elsewhere.
-  Record.push_back(LangOpts.CPlusPlus);  // C++ Support
-  Record.push_back(LangOpts.CPlusPlus0x);  // C++0x Support
-  Record.push_back(LangOpts.CXXOperatorNames);  // Treat C++ operator names as keywords.
-
-  Record.push_back(LangOpts.ObjC1);  // Objective-C 1 support enabled.
-  Record.push_back(LangOpts.ObjC2);  // Objective-C 2 support enabled.
-  Record.push_back(LangOpts.ObjCNonFragileABI);  // Objective-C
-                                                 // modern abi enabled.
-  Record.push_back(LangOpts.ObjCNonFragileABI2); // Objective-C enhanced
-                                                 // modern abi enabled.
-  Record.push_back(LangOpts.AppleKext);          // Apple's kernel extensions ABI
-  Record.push_back(LangOpts.ObjCDefaultSynthProperties); // Objective-C auto-synthesized
-                                                      // properties enabled.
-  Record.push_back(LangOpts.ObjCInferRelatedResultType);
-  Record.push_back(LangOpts.NoConstantCFStrings); // non cfstring generation enabled..
-
-  Record.push_back(LangOpts.PascalStrings);  // Allow Pascal strings
-  Record.push_back(LangOpts.WritableStrings);  // Allow writable strings
-  Record.push_back(LangOpts.LaxVectorConversions);
-  Record.push_back(LangOpts.AltiVec);
-  Record.push_back(LangOpts.Exceptions);  // Support exception handling.
-  Record.push_back(LangOpts.ObjCExceptions);
-  Record.push_back(LangOpts.CXXExceptions);
-  Record.push_back(LangOpts.SjLjExceptions);
-
-  Record.push_back(LangOpts.MSBitfields); // MS-compatible structure layout
-  Record.push_back(LangOpts.NeXTRuntime); // Use NeXT runtime.
-  Record.push_back(LangOpts.Freestanding); // Freestanding implementation
-  Record.push_back(LangOpts.NoBuiltin); // Do not use builtin functions (-fno-builtin)
-
-  // Whether static initializers are protected by locks.
-  Record.push_back(LangOpts.ThreadsafeStatics);
-  Record.push_back(LangOpts.POSIXThreads);
-  Record.push_back(LangOpts.Blocks); // block extension to C
-  Record.push_back(LangOpts.EmitAllDecls); // Emit all declarations, even if
-                                  // they are unused.
-  Record.push_back(LangOpts.MathErrno); // Math functions must respect errno
-                                  // (modulo the platform support).
-
-  Record.push_back(LangOpts.getSignedOverflowBehavior());
-  Record.push_back(LangOpts.HeinousExtensions);
-
-  Record.push_back(LangOpts.Optimize); // Whether __OPTIMIZE__ should be defined.
-  Record.push_back(LangOpts.OptimizeSize); // Whether __OPTIMIZE_SIZE__ should be
-                                  // defined.
-  Record.push_back(LangOpts.Static); // Should __STATIC__ be defined (as
-                                  // opposed to __DYNAMIC__).
-  Record.push_back(LangOpts.PICLevel); // The value for __PIC__, if non-zero.
-
-  Record.push_back(LangOpts.GNUInline); // Should GNU inline semantics be
-                                  // used (instead of C99 semantics).
-  Record.push_back(LangOpts.NoInline); // Should __NO_INLINE__ be defined.
-  Record.push_back(LangOpts.Deprecated); // Should __DEPRECATED be defined.
-  Record.push_back(LangOpts.AccessControl); // Whether C++ access control should
-                                            // be enabled.
-  Record.push_back(LangOpts.CharIsSigned); // Whether char is a signed or
-                                           // unsigned type
-  Record.push_back(LangOpts.ShortWChar);  // force wchar_t to be unsigned short
-  Record.push_back(LangOpts.ShortEnums);  // Should the enum type be equivalent
-                                          // to the smallest integer type with
-                                          // enough room.
-  Record.push_back(LangOpts.getGCMode());
-  Record.push_back(LangOpts.getVisibilityMode());
-  Record.push_back(LangOpts.getStackProtectorMode());
-  Record.push_back(LangOpts.InstantiationDepth);
-  Record.push_back(LangOpts.OpenCL);
-  Record.push_back(LangOpts.CUDA);
-  Record.push_back(LangOpts.CatchUndefined);
-  Record.push_back(LangOpts.DefaultFPContract);
-  Record.push_back(LangOpts.ElideConstructors);
-  Record.push_back(LangOpts.SpellChecking);
-  Record.push_back(LangOpts.MRTD);
-  Record.push_back(LangOpts.ObjCAutoRefCount);
-  Record.push_back(LangOpts.ObjCInferRelatedReturnType);
+#define LANGOPT(Name, Bits, Default, Description) \
+  Record.push_back(LangOpts.Name);
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  Record.push_back(static_cast<unsigned>(LangOpts.get##Name()));
+#include "clang/Basic/LangOptions.def"  
   Stream.EmitRecord(LANGUAGE_OPTIONS, Record);
 }
 
@@ -1136,7 +1085,7 @@ class ASTStatCacheTrait {
   }
 
   std::pair<unsigned,unsigned>
-    EmitKeyDataLength(llvm::raw_ostream& Out, const char *path,
+    EmitKeyDataLength(raw_ostream& Out, const char *path,
                       data_type_ref Data) {
     unsigned StrLen = strlen(path);
     clang::io::Emit16(Out, StrLen);
@@ -1145,11 +1094,11 @@ class ASTStatCacheTrait {
     return std::make_pair(StrLen + 1, DataLen);
   }
 
-  void EmitKey(llvm::raw_ostream& Out, const char *path, unsigned KeyLen) {
+  void EmitKey(raw_ostream& Out, const char *path, unsigned KeyLen) {
     Out.write(path, KeyLen);
   }
 
-  void EmitData(llvm::raw_ostream &Out, key_type_ref,
+  void EmitData(raw_ostream &Out, key_type_ref,
                 data_type_ref Data, unsigned DataLen) {
     using namespace clang::io;
     uint64_t Start = Out.tell(); (void)Start;
@@ -1174,7 +1123,7 @@ void ASTWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
   for (MemorizeStatCalls::iterator Stat = StatCalls.begin(),
                                 StatEnd = StatCalls.end();
        Stat != StatEnd; ++Stat, ++NumStatEntries) {
-    llvm::StringRef Filename = Stat->first();
+    StringRef Filename = Stat->first();
     Generator.insert(Filename.data(), Stat->second);
   }
 
@@ -1222,6 +1171,7 @@ static unsigned CreateSLocFileAbbrev(llvm::BitstreamWriter &Stream) {
   // FileEntry fields.
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // Size
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // Modification time
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // NumCreatedFIDs
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
   return Stream.EmitAbbrev(Abbrev);
 }
@@ -1270,6 +1220,10 @@ namespace {
     ASTWriter &Writer;
     HeaderSearch &HS;
     
+    // Keep track of the framework names we've used during serialization.
+    SmallVector<char, 128> FrameworkStringData;
+    llvm::StringMap<unsigned> FrameworkNameOffset;
+    
   public:
     HeaderFileInfoTrait(ASTWriter &Writer, HeaderSearch &HS) 
       : Writer(Writer), HS(HS) { }
@@ -1289,28 +1243,29 @@ namespace {
     }
     
     std::pair<unsigned,unsigned>
-    EmitKeyDataLength(llvm::raw_ostream& Out, const char *path,
+    EmitKeyDataLength(raw_ostream& Out, const char *path,
                       data_type_ref Data) {
       unsigned StrLen = strlen(path);
       clang::io::Emit16(Out, StrLen);
-      unsigned DataLen = 1 + 2 + 4;
+      unsigned DataLen = 1 + 2 + 4 + 4;
       clang::io::Emit8(Out, DataLen);
       return std::make_pair(StrLen + 1, DataLen);
     }
     
-    void EmitKey(llvm::raw_ostream& Out, const char *path, unsigned KeyLen) {
+    void EmitKey(raw_ostream& Out, const char *path, unsigned KeyLen) {
       Out.write(path, KeyLen);
     }
     
-    void EmitData(llvm::raw_ostream &Out, key_type_ref,
+    void EmitData(raw_ostream &Out, key_type_ref,
                   data_type_ref Data, unsigned DataLen) {
       using namespace clang::io;
       uint64_t Start = Out.tell(); (void)Start;
       
-      unsigned char Flags = (Data.isImport << 4)
-                          | (Data.isPragmaOnce << 3)
-                          | (Data.DirInfo << 1)
-                          | Data.Resolved;
+      unsigned char Flags = (Data.isImport << 5)
+                          | (Data.isPragmaOnce << 4)
+                          | (Data.DirInfo << 2)
+                          | (Data.Resolved << 1)
+                          | Data.IndexHeaderMapHeader;
       Emit8(Out, (uint8_t)Flags);
       Emit16(Out, (uint16_t) Data.NumIncludes);
       
@@ -1318,8 +1273,29 @@ namespace {
         Emit32(Out, (uint32_t)Data.ControllingMacroID);
       else
         Emit32(Out, (uint32_t)Writer.getIdentifierRef(Data.ControllingMacro));
+      
+      unsigned Offset = 0;
+      if (!Data.Framework.empty()) {
+        // If this header refers into a framework, save the framework name.
+        llvm::StringMap<unsigned>::iterator Pos
+          = FrameworkNameOffset.find(Data.Framework);
+        if (Pos == FrameworkNameOffset.end()) {
+          Offset = FrameworkStringData.size() + 1;
+          FrameworkStringData.append(Data.Framework.begin(), 
+                                     Data.Framework.end());
+          FrameworkStringData.push_back(0);
+          
+          FrameworkNameOffset[Data.Framework] = Offset;
+        } else
+          Offset = Pos->second;
+      }
+      Emit32(Out, Offset);
+      
       assert(Out.tell() - Start == DataLen && "Wrong data length");
     }
+    
+    const char *strings_begin() const { return FrameworkStringData.begin(); }
+    const char *strings_end() const { return FrameworkStringData.end(); }
   };
 } // end anonymous namespace
 
@@ -1328,8 +1304,8 @@ namespace {
 /// \param HS The header search structure to save.
 ///
 /// \param Chain Whether we're creating a chained AST file.
-void ASTWriter::WriteHeaderSearch(HeaderSearch &HS, const char* isysroot) {
-  llvm::SmallVector<const FileEntry *, 16> FilesByUID;
+void ASTWriter::WriteHeaderSearch(HeaderSearch &HS, StringRef isysroot) {
+  SmallVector<const FileEntry *, 16> FilesByUID;
   HS.getFileMgr().GetUniqueIDMapping(FilesByUID);
   
   if (FilesByUID.size() > HS.header_file_size())
@@ -1337,7 +1313,7 @@ void ASTWriter::WriteHeaderSearch(HeaderSearch &HS, const char* isysroot) {
   
   HeaderFileInfoTrait GeneratorTrait(*this, HS);
   OnDiskChainedHashTableGenerator<HeaderFileInfoTrait> Generator;  
-  llvm::SmallVector<const char *, 4> SavedStrings;
+  SmallVector<const char *, 4> SavedStrings;
   unsigned NumHeaderSearchEntries = 0;
   for (unsigned UID = 0, LastUID = FilesByUID.size(); UID != LastUID; ++UID) {
     const FileEntry *File = FilesByUID[UID];
@@ -1379,14 +1355,17 @@ void ASTWriter::WriteHeaderSearch(HeaderSearch &HS, const char* isysroot) {
   Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_TABLE));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
   unsigned TableAbbrev = Stream.EmitAbbrev(Abbrev);
   
-  // Write the stat cache
+  // Write the header search table
   RecordData Record;
   Record.push_back(HEADER_SEARCH_TABLE);
   Record.push_back(BucketOffset);
   Record.push_back(NumHeaderSearchEntries);
+  Record.push_back(TableData.size());
+  TableData.append(GeneratorTrait.strings_begin(),GeneratorTrait.strings_end());
   Stream.EmitRecordWithBlob(TableAbbrev, Record, TableData.str());
   
   // Free all of the strings we had to duplicate.
@@ -1404,7 +1383,7 @@ void ASTWriter::WriteHeaderSearch(HeaderSearch &HS, const char* isysroot) {
 /// the files in the AST.
 void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
                                         const Preprocessor &PP,
-                                        const char *isysroot) {
+                                        StringRef isysroot) {
   RecordData Record;
 
   // Enter the source manager block.
@@ -1416,43 +1395,6 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   unsigned SLocBufferBlobAbbrv = CreateSLocBufferBlobAbbrev(Stream);
   unsigned SLocExpansionAbbrv = CreateSLocExpansionAbbrev(Stream);
 
-  // Write the line table.
-  if (SourceMgr.hasLineTable()) {
-    LineTableInfo &LineTable = SourceMgr.getLineTable();
-
-    // Emit the file names
-    Record.push_back(LineTable.getNumFilenames());
-    for (unsigned I = 0, N = LineTable.getNumFilenames(); I != N; ++I) {
-      // Emit the file name
-      const char *Filename = LineTable.getFilename(I);
-      Filename = adjustFilenameForRelocatablePCH(Filename, isysroot);
-      unsigned FilenameLen = Filename? strlen(Filename) : 0;
-      Record.push_back(FilenameLen);
-      if (FilenameLen)
-        Record.insert(Record.end(), Filename, Filename + FilenameLen);
-    }
-
-    // Emit the line entries
-    for (LineTableInfo::iterator L = LineTable.begin(), LEnd = LineTable.end();
-         L != LEnd; ++L) {
-      // Emit the file ID
-      Record.push_back(L->first);
-
-      // Emit the line entries
-      Record.push_back(L->second.size());
-      for (std::vector<LineEntry>::iterator LE = L->second.begin(),
-                                         LEEnd = L->second.end();
-           LE != LEEnd; ++LE) {
-        Record.push_back(LE->FileOffset);
-        Record.push_back(LE->LineNo);
-        Record.push_back(LE->FilenameID);
-        Record.push_back((unsigned)LE->FileKind);
-        Record.push_back(LE->IncludeOffset);
-      }
-    }
-    Stream.EmitRecord(SM_LINE_TABLE, Record);
-  }
-
   // Write out the source location entry table. We skip the first
   // entry, which is always the same dummy entry.
   std::vector<uint32_t> SLocEntryOffsets;
@@ -1460,12 +1402,11 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   // We will go through them in ASTReader::validateFileEntries().
   std::vector<uint32_t> SLocFileEntryOffsets;
   RecordData PreloadSLocs;
-  unsigned BaseSLocID = Chain ? Chain->getTotalNumSLocs() : 0;
-  SLocEntryOffsets.reserve(SourceMgr.sloc_entry_size() - 1 - BaseSLocID);
-  for (unsigned I = BaseSLocID + 1, N = SourceMgr.sloc_entry_size();
+  SLocEntryOffsets.reserve(SourceMgr.local_sloc_entry_size() - 1);
+  for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size();
        I != N; ++I) {
     // Get this source location entry.
-    const SrcMgr::SLocEntry *SLoc = &SourceMgr.getSLocEntry(I);
+    const SrcMgr::SLocEntry *SLoc = &SourceMgr.getLocalSLocEntry(I);
 
     // Record the offset of this source-location entry.
     SLocEntryOffsets.push_back(Stream.GetCurrentBitNo());
@@ -1483,7 +1424,8 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
     Record.clear();
     Record.push_back(Code);
 
-    Record.push_back(SLoc->getOffset());
+    // Starting offset of this entry within this module, so skip the dummy.
+    Record.push_back(SLoc->getOffset() - 2);
     if (SLoc->isFile()) {
       const SrcMgr::FileInfo &File = SLoc->getFile();
       Record.push_back(File.getIncludeLoc().getRawEncoding());
@@ -1502,6 +1444,8 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
         Record.push_back(Content->OrigEntry->getSize());
         Record.push_back(Content->OrigEntry->getModificationTime());
 
+        Record.push_back(File.NumCreatedFIDs);
+
         // Turn the file name into an absolute path, if it isn't already.
         const char *Filename = Content->OrigEntry->getName();
         llvm::SmallString<128> FilePath(Filename);
@@ -1528,27 +1472,29 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
           = Content->getBuffer(PP.getDiagnostics(), PP.getSourceManager());
         const char *Name = Buffer->getBufferIdentifier();
         Stream.EmitRecordWithBlob(SLocBufferAbbrv, Record,
-                                  llvm::StringRef(Name, strlen(Name) + 1));
+                                  StringRef(Name, strlen(Name) + 1));
         Record.clear();
         Record.push_back(SM_SLOC_BUFFER_BLOB);
         Stream.EmitRecordWithBlob(SLocBufferBlobAbbrv, Record,
-                                  llvm::StringRef(Buffer->getBufferStart(),
+                                  StringRef(Buffer->getBufferStart(),
                                                   Buffer->getBufferSize() + 1));
 
-        if (strcmp(Name, "<built-in>") == 0)
-          PreloadSLocs.push_back(BaseSLocID + SLocEntryOffsets.size());
+        if (strcmp(Name, "<built-in>") == 0) {
+          PreloadSLocs.push_back(SLocEntryOffsets.size());
+        }
       }
     } else {
       // The source location entry is a macro expansion.
-      const SrcMgr::InstantiationInfo &Inst = SLoc->getInstantiation();
-      Record.push_back(Inst.getSpellingLoc().getRawEncoding());
-      Record.push_back(Inst.getInstantiationLocStart().getRawEncoding());
-      Record.push_back(Inst.getInstantiationLocEnd().getRawEncoding());
+      const SrcMgr::ExpansionInfo &Expansion = SLoc->getExpansion();
+      Record.push_back(Expansion.getSpellingLoc().getRawEncoding());
+      Record.push_back(Expansion.getExpansionLocStart().getRawEncoding());
+      Record.push_back(Expansion.isMacroArgExpansion() ? 0
+                             : Expansion.getExpansionLocEnd().getRawEncoding());
 
       // Compute the token length for this macro expansion.
-      unsigned NextOffset = SourceMgr.getNextOffset();
+      unsigned NextOffset = SourceMgr.getNextLocalOffset();
       if (I + 1 != N)
-        NextOffset = SourceMgr.getSLocEntry(I + 1).getOffset();
+        NextOffset = SourceMgr.getLocalSLocEntry(I + 1).getOffset();
       Record.push_back(NextOffset - SLoc->getOffset() - 1);
       Stream.EmitRecordWithAbbrev(SLocExpansionAbbrv, Record);
     }
@@ -1565,15 +1511,14 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(SOURCE_LOCATION_OFFSETS));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // next offset
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // total size
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // offsets
   unsigned SLocOffsetsAbbrev = Stream.EmitAbbrev(Abbrev);
 
   Record.clear();
   Record.push_back(SOURCE_LOCATION_OFFSETS);
   Record.push_back(SLocEntryOffsets.size());
-  unsigned BaseOffset = Chain ? Chain->getNextSLocOffset() : 0;
-  Record.push_back(SourceMgr.getNextOffset() - BaseOffset);
+  Record.push_back(SourceMgr.getNextLocalOffset() - 1); // skip dummy
   Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record, data(SLocEntryOffsets));
 
   Abbrev = new BitCodeAbbrev();
@@ -1591,6 +1536,49 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   // Write the source location entry preloads array, telling the AST
   // reader which source locations entries it should load eagerly.
   Stream.EmitRecord(SOURCE_LOCATION_PRELOADS, PreloadSLocs);
+
+  // Write the line table. It depends on remapping working, so it must come
+  // after the source location offsets.
+  if (SourceMgr.hasLineTable()) {
+    LineTableInfo &LineTable = SourceMgr.getLineTable();
+
+    Record.clear();
+    // Emit the file names
+    Record.push_back(LineTable.getNumFilenames());
+    for (unsigned I = 0, N = LineTable.getNumFilenames(); I != N; ++I) {
+      // Emit the file name
+      const char *Filename = LineTable.getFilename(I);
+      Filename = adjustFilenameForRelocatablePCH(Filename, isysroot);
+      unsigned FilenameLen = Filename? strlen(Filename) : 0;
+      Record.push_back(FilenameLen);
+      if (FilenameLen)
+        Record.insert(Record.end(), Filename, Filename + FilenameLen);
+    }
+
+    // Emit the line entries
+    for (LineTableInfo::iterator L = LineTable.begin(), LEnd = LineTable.end();
+         L != LEnd; ++L) {
+      // Only emit entries for local files.
+      if (L->first < 0)
+        continue;
+
+      // Emit the file ID
+      Record.push_back(L->first);
+
+      // Emit the line entries
+      Record.push_back(L->second.size());
+      for (std::vector<LineEntry>::iterator LE = L->second.begin(),
+                                         LEEnd = L->second.end();
+           LE != LEEnd; ++LE) {
+        Record.push_back(LE->FileOffset);
+        Record.push_back(LE->LineNo);
+        Record.push_back(LE->FilenameID);
+        Record.push_back((unsigned)LE->FileKind);
+        Record.push_back(LE->IncludeOffset);
+      }
+    }
+    Stream.EmitRecord(SOURCE_MANAGER_LINE_TABLE, Record);
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -1608,7 +1596,11 @@ static int compareMacroDefinitions(const void *XPtr, const void *YPtr) {
 /// \brief Writes the block containing the serialized form of the
 /// preprocessor.
 ///
-void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
+void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
+  PreprocessingRecord *PPRec = PP.getPreprocessingRecord();
+  if (PPRec)
+    WritePreprocessorDetail(*PPRec);
+
   RecordData Record;
 
   // If the preprocessor __COUNTER__ value has been bumped, remember it.
@@ -1629,17 +1621,18 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
 
   // Loop over all the macro definitions that are live at the end of the file,
   // emitting each to the PP section.
-  PreprocessingRecord *PPRec = PP.getPreprocessingRecord();
 
   // Construct the list of macro definitions that need to be serialized.
-  llvm::SmallVector<std::pair<const IdentifierInfo *, MacroInfo *>, 2> 
+  SmallVector<std::pair<const IdentifierInfo *, MacroInfo *>, 2> 
     MacrosToEmit;
   llvm::SmallPtrSet<const IdentifierInfo*, 4> MacroDefinitionsSeen;
   for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0), 
                                     E = PP.macro_end(Chain == 0);
        I != E; ++I) {
-    MacroDefinitionsSeen.insert(I->first);
-    MacrosToEmit.push_back(std::make_pair(I->first, I->second));
+    if (!IsModule || I->second->isExported()) {
+      MacroDefinitionsSeen.insert(I->first);
+      MacrosToEmit.push_back(std::make_pair(I->first, I->second));
+    }
   }
   
   // Sort the set of macro definitions that need to be serialized by the
@@ -1671,14 +1664,15 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
     // chained PCH, by storing the offset into the original PCH rather than
     // writing the macro definition a second time.
     if (MI->isBuiltinMacro() ||
-        (Chain && Name->isFromAST() && MI->isFromAST()))
+        (Chain && Name->isFromAST() && MI->isFromAST() &&
+        !MI->hasChangedAfterLoad()))
       continue;
 
     AddIdentifierRef(Name, Record);
     MacroOffsets[Name] = Stream.GetCurrentBitNo();
     Record.push_back(MI->getDefinitionLoc().getRawEncoding());
     Record.push_back(MI->isUsed());
-
+    AddSourceLocation(MI->getExportLocation(), Record);
     unsigned Code;
     if (MI->isObjectLike()) {
       Code = PP_MACRO_OBJECT_LIKE;
@@ -1696,7 +1690,7 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
     // If we have a detailed preprocessing record, record the macro definition
     // ID that corresponds to this macro.
     if (PPRec)
-      Record.push_back(getMacroDefinitionID(PPRec->findMacroDefinition(MI)));
+      Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
 
     Stream.EmitRecord(Code, Record);
     Record.clear();
@@ -1725,15 +1719,14 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
     ++NumMacros;
   }
   Stream.ExitBlock();
-
-  if (PPRec)
-    WritePreprocessorDetail(*PPRec);
 }
 
 void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
-  if (PPRec.begin(Chain) == PPRec.end(Chain))
+  if (PPRec.local_begin() == PPRec.local_end())
     return;
-  
+
+  SmallVector<PPEntityOffset, 64> PreprocessedEntityOffsets;
+
   // Enter the preprocessor block.
   Stream.EnterSubblock(PREPROCESSOR_DETAIL_BLOCK_ID, 3);
 
@@ -1746,9 +1739,6 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
   {
     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
     Abbrev->Add(BitCodeAbbrevOp(PPD_INCLUSION_DIRECTIVE));
-    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // index
-    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // start location
-    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // end location
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // filename length
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // in quotes
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // kind
@@ -1756,65 +1746,41 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
     InclusionAbbrev = Stream.EmitAbbrev(Abbrev);
   }
   
-  unsigned IndexBase = Chain ? PPRec.getNumPreallocatedEntities() : 0;
+  unsigned FirstPreprocessorEntityID 
+    = (Chain ? PPRec.getNumLoadedPreprocessedEntities() : 0) 
+    + NUM_PREDEF_PP_ENTITY_IDS;
+  unsigned NextPreprocessorEntityID = FirstPreprocessorEntityID;
   RecordData Record;
-  for (PreprocessingRecord::iterator E = PPRec.begin(Chain),
-                                  EEnd = PPRec.end(Chain);
-       E != EEnd; ++E) {
+  for (PreprocessingRecord::iterator E = PPRec.local_begin(),
+                                  EEnd = PPRec.local_end();
+       E != EEnd; 
+       (void)++E, ++NumPreprocessingRecords, ++NextPreprocessorEntityID) {
     Record.clear();
 
-    if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
-      // Record this macro definition's location.
-      MacroID ID = getMacroDefinitionID(MD);
-      
-      // Don't write the macro definition if it is from another AST file.
-      if (ID < FirstMacroID)
-        continue;
-      
-      // Notify the serialization listener that we're serializing this entity.
-      if (SerializationListener)
-        SerializationListener->SerializedPreprocessedEntity(*E, 
-                                                    Stream.GetCurrentBitNo());
+    PreprocessedEntityOffsets.push_back(PPEntityOffset((*E)->getSourceRange(),
+                                                     Stream.GetCurrentBitNo()));
 
-      unsigned Position = ID - FirstMacroID;
-      if (Position != MacroDefinitionOffsets.size()) {
-        if (Position > MacroDefinitionOffsets.size())
-          MacroDefinitionOffsets.resize(Position + 1);
-        
-        MacroDefinitionOffsets[Position] = Stream.GetCurrentBitNo();
-      } else
-        MacroDefinitionOffsets.push_back(Stream.GetCurrentBitNo());
+    if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
+      // Record this macro definition's ID.
+      MacroDefinitions[MD] = NextPreprocessorEntityID;
       
-      Record.push_back(IndexBase + NumPreprocessingRecords++);
-      Record.push_back(ID);
-      AddSourceLocation(MD->getSourceRange().getBegin(), Record);
-      AddSourceLocation(MD->getSourceRange().getEnd(), Record);
       AddIdentifierRef(MD->getName(), Record);
-      AddSourceLocation(MD->getLocation(), Record);
       Stream.EmitRecord(PPD_MACRO_DEFINITION, Record);
       continue;
     }
 
-    // Notify the serialization listener that we're serializing this entity.
-    if (SerializationListener)
-      SerializationListener->SerializedPreprocessedEntity(*E, 
-                                                    Stream.GetCurrentBitNo());
-
     if (MacroExpansion *ME = dyn_cast<MacroExpansion>(*E)) {
-      Record.push_back(IndexBase + NumPreprocessingRecords++);
-      AddSourceLocation(ME->getSourceRange().getBegin(), Record);
-      AddSourceLocation(ME->getSourceRange().getEnd(), Record);
-      AddIdentifierRef(ME->getName(), Record);
-      Record.push_back(getMacroDefinitionID(ME->getDefinition()));
+      Record.push_back(ME->isBuiltinMacro());
+      if (ME->isBuiltinMacro())
+        AddIdentifierRef(ME->getName(), Record);
+      else
+        Record.push_back(MacroDefinitions[ME->getDefinition()]);
       Stream.EmitRecord(PPD_MACRO_EXPANSION, Record);
       continue;
     }
 
     if (InclusionDirective *ID = dyn_cast<InclusionDirective>(*E)) {
       Record.push_back(PPD_INCLUSION_DIRECTIVE);
-      Record.push_back(IndexBase + NumPreprocessingRecords++);
-      AddSourceLocation(ID->getSourceRange().getBegin(), Record);
-      AddSourceLocation(ID->getSourceRange().getEnd(), Record);
       Record.push_back(ID->getFileName().size());
       Record.push_back(ID->wasInQuotes());
       Record.push_back(static_cast<unsigned>(ID->getKind()));
@@ -1831,40 +1797,39 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
 
   // Write the offsets table for the preprocessing record.
   if (NumPreprocessingRecords > 0) {
+    assert(PreprocessedEntityOffsets.size() == NumPreprocessingRecords);
+
     // Write the offsets table for identifier IDs.
     using namespace llvm;
     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-    Abbrev->Add(BitCodeAbbrevOp(MACRO_DEFINITION_OFFSETS));
-    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of records
-    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macro defs
+    Abbrev->Add(BitCodeAbbrevOp(PPD_ENTITIES_OFFSETS));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first pp entity
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
-    unsigned MacroDefOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
+    unsigned PPEOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
 
     Record.clear();
-    Record.push_back(MACRO_DEFINITION_OFFSETS);
-    Record.push_back(NumPreprocessingRecords);
-    Record.push_back(MacroDefinitionOffsets.size());
-    Stream.EmitRecordWithBlob(MacroDefOffsetAbbrev, Record,
-                              data(MacroDefinitionOffsets));
+    Record.push_back(PPD_ENTITIES_OFFSETS);
+    Record.push_back(FirstPreprocessorEntityID - NUM_PREDEF_PP_ENTITY_IDS);
+    Stream.EmitRecordWithBlob(PPEOffsetAbbrev, Record,
+                              data(PreprocessedEntityOffsets));
   }
 }
 
-void ASTWriter::WritePragmaDiagnosticMappings(const Diagnostic &Diag) {
+void ASTWriter::WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag) {
   RecordData Record;
-  for (Diagnostic::DiagStatePointsTy::const_iterator
+  for (DiagnosticsEngine::DiagStatePointsTy::const_iterator
          I = Diag.DiagStatePoints.begin(), E = Diag.DiagStatePoints.end();
          I != E; ++I) {
-    const Diagnostic::DiagStatePoint &point = *I; 
+    const DiagnosticsEngine::DiagStatePoint &point = *I; 
     if (point.Loc.isInvalid())
       continue;
 
     Record.push_back(point.Loc.getRawEncoding());
-    for (Diagnostic::DiagState::iterator
+    for (DiagnosticsEngine::DiagState::const_iterator
            I = point.State->begin(), E = point.State->end(); I != E; ++I) {
-      unsigned diag = I->first, map = I->second;
-      if (map & 0x10) { // mapping from a diagnostic pragma.
-        Record.push_back(diag);
-        Record.push_back(map & 0x7);
+      if (I->second.isPragma()) {
+        Record.push_back(I->first);
+        Record.push_back(I->second.getMapping());
       }
     }
     Record.push_back(-1); // mark the end of the diag/map pairs for this
@@ -1890,7 +1855,7 @@ void ASTWriter::WriteCXXBaseSpecifiersOffsets() {
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
   unsigned BaseSpecifierOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
   
-  // Write the selector offsets table.
+  // Write the base specifier offsets table.
   Record.clear();
   Record.push_back(CXX_BASE_SPECIFIER_OFFSETS);
   Record.push_back(CXXBaseSpecifiersOffsets.size());
@@ -1964,7 +1929,7 @@ uint64_t ASTWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
   uint64_t Offset = Stream.GetCurrentBitNo();
   RecordData Record;
   Record.push_back(DECL_CONTEXT_LEXICAL);
-  llvm::SmallVector<KindDeclIDPair, 64> Decls;
+  SmallVector<KindDeclIDPair, 64> Decls;
   for (DeclContext::decl_iterator D = DC->decls_begin(), DEnd = DC->decls_end();
          D != DEnd; ++D)
     Decls.push_back(std::make_pair((*D)->getKind(), GetDeclRef(*D)));
@@ -1982,22 +1947,26 @@ void ASTWriter::WriteTypeDeclOffsets() {
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(TYPE_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of types
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base type index
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // types block
   unsigned TypeOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
   Record.clear();
   Record.push_back(TYPE_OFFSET);
   Record.push_back(TypeOffsets.size());
+  Record.push_back(FirstTypeID - NUM_PREDEF_TYPE_IDS);
   Stream.EmitRecordWithBlob(TypeOffsetAbbrev, Record, data(TypeOffsets));
 
   // Write the declaration offsets array
   Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block
   unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
   Record.clear();
   Record.push_back(DECL_OFFSET);
   Record.push_back(DeclOffsets.size());
+  Record.push_back(FirstDeclID - NUM_PREDEF_DECL_IDS);
   Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, data(DeclOffsets));
 }
 
@@ -2027,7 +1996,7 @@ class ASTMethodPoolTrait {
   }
 
   std::pair<unsigned,unsigned>
-    EmitKeyDataLength(llvm::raw_ostream& Out, Selector Sel,
+    EmitKeyDataLength(raw_ostream& Out, Selector Sel,
                       data_type_ref Methods) {
     unsigned KeyLen = 2 + (Sel.getNumArgs()? Sel.getNumArgs() * 4 : 4);
     clang::io::Emit16(Out, KeyLen);
@@ -2044,7 +2013,7 @@ class ASTMethodPoolTrait {
     return std::make_pair(KeyLen, DataLen);
   }
 
-  void EmitKey(llvm::raw_ostream& Out, Selector Sel, unsigned) {
+  void EmitKey(raw_ostream& Out, Selector Sel, unsigned) {
     uint64_t Start = Out.tell();
     assert((Start >> 32) == 0 && "Selector key offset too large");
     Writer.SetSelectorOffset(Sel, Start);
@@ -2057,7 +2026,7 @@ class ASTMethodPoolTrait {
                     Writer.getIdentifierRef(Sel.getIdentifierInfoForSlot(I)));
   }
 
-  void EmitData(llvm::raw_ostream& Out, key_type_ref,
+  void EmitData(raw_ostream& Out, key_type_ref,
                 data_type_ref Methods, unsigned DataLen) {
     uint64_t Start = Out.tell(); (void)Start;
     clang::io::Emit32(Out, Methods.ID);
@@ -2130,12 +2099,12 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
         bool changed = false;
         for (ObjCMethodList *M = &Data.Instance; !changed && M && M->Method;
              M = M->Next) {
-          if (M->Method->getPCHLevel() == 0)
+          if (!M->Method->isFromASTFile())
             changed = true;
         }
         for (ObjCMethodList *M = &Data.Factory; !changed && M && M->Method;
              M = M->Next) {
-          if (M->Method->getPCHLevel() == 0)
+          if (!M->Method->isFromASTFile())
             changed = true;
         }
         if (!changed)
@@ -2177,6 +2146,7 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
     Abbrev = new BitCodeAbbrev();
     Abbrev->Add(BitCodeAbbrevOp(SELECTOR_OFFSETS));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // size
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first ID
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned SelectorOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
 
@@ -2184,6 +2154,7 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
     Record.clear();
     Record.push_back(SELECTOR_OFFSETS);
     Record.push_back(SelectorOffsets.size());
+    Record.push_back(FirstSelectorID - NUM_PREDEF_SELECTOR_IDS);
     Stream.EmitRecordWithBlob(SelectorOffsetAbbrev, Record,
                               data(SelectorOffsets));
   }
@@ -2219,41 +2190,53 @@ namespace {
 class ASTIdentifierTableTrait {
   ASTWriter &Writer;
   Preprocessor &PP;
-
+  bool IsModule;
+  
   /// \brief Determines whether this is an "interesting" identifier
   /// that needs a full IdentifierInfo structure written into the hash
   /// table.
-  static bool isInterestingIdentifier(const IdentifierInfo *II) {
-    return II->isPoisoned() ||
-      II->isExtensionToken() ||
-      II->hasMacroDefinition() ||
-      II->getObjCOrBuiltinID() ||
-      II->getFETokenInfo<void>();
+  bool isInterestingIdentifier(IdentifierInfo *II, MacroInfo *&Macro) {
+    if (II->isPoisoned() ||
+        II->isExtensionToken() ||
+        II->getObjCOrBuiltinID() ||
+        II->getFETokenInfo<void>())
+      return true;
+
+    return hasMacroDefinition(II, Macro);
+  }
+  
+  bool hasMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) {
+    if (!II->hasMacroDefinition())
+      return false;
+    
+    if (Macro || (Macro = PP.getMacroInfo(II)))
+      return !Macro->isBuiltinMacro() && (!IsModule || Macro->isExported());
+    
+    return false;    
   }
 
 public:
-  typedef const IdentifierInfo* key_type;
+  typedef IdentifierInfo* key_type;
   typedef key_type  key_type_ref;
 
   typedef IdentID data_type;
   typedef data_type data_type_ref;
 
-  ASTIdentifierTableTrait(ASTWriter &Writer, Preprocessor &PP)
-    : Writer(Writer), PP(PP) { }
+  ASTIdentifierTableTrait(ASTWriter &Writer, Preprocessor &PP, bool IsModule)
+    : Writer(Writer), PP(PP), IsModule(IsModule) { }
 
   static unsigned ComputeHash(const IdentifierInfo* II) {
     return llvm::HashString(II->getName());
   }
 
   std::pair<unsigned,unsigned>
-    EmitKeyDataLength(llvm::raw_ostream& Out, const IdentifierInfo* II,
-                      IdentID ID) {
+    EmitKeyDataLength(raw_ostream& Out, IdentifierInfo* II, IdentID ID) {
     unsigned KeyLen = II->getLength() + 1;
     unsigned DataLen = 4; // 4 bytes for the persistent ID << 1
-    if (isInterestingIdentifier(II)) {
+    MacroInfo *Macro = 0;
+    if (isInterestingIdentifier(II, Macro)) {
       DataLen += 2; // 2 bytes for builtin ID, flags
-      if (II->hasMacroDefinition() &&
-          !PP.getMacroInfo(const_cast<IdentifierInfo *>(II))->isBuiltinMacro())
+      if (hasMacroDefinition(II, Macro))
         DataLen += 4;
       for (IdentifierResolver::iterator D = IdentifierResolver::begin(II),
                                      DEnd = IdentifierResolver::end();
@@ -2268,7 +2251,7 @@ class ASTIdentifierTableTrait {
     return std::make_pair(KeyLen, DataLen);
   }
 
-  void EmitKey(llvm::raw_ostream& Out, const IdentifierInfo* II,
+  void EmitKey(raw_ostream& Out, const IdentifierInfo* II,
                unsigned KeyLen) {
     // Record the location of the key data.  This is used when generating
     // the mapping from persistent IDs to strings.
@@ -2276,27 +2259,26 @@ class ASTIdentifierTableTrait {
     Out.write(II->getNameStart(), KeyLen);
   }
 
-  void EmitData(llvm::raw_ostream& Out, const IdentifierInfo* II,
+  void EmitData(raw_ostream& Out, IdentifierInfo* II,
                 IdentID ID, unsigned) {
-    if (!isInterestingIdentifier(II)) {
+    MacroInfo *Macro = 0;
+    if (!isInterestingIdentifier(II, Macro)) {
       clang::io::Emit32(Out, ID << 1);
       return;
     }
 
     clang::io::Emit32(Out, (ID << 1) | 0x01);
     uint32_t Bits = 0;
-    bool hasMacroDefinition =
-      II->hasMacroDefinition() &&
-      !PP.getMacroInfo(const_cast<IdentifierInfo *>(II))->isBuiltinMacro();
+    bool HasMacroDefinition = hasMacroDefinition(II, Macro);
     Bits = (uint32_t)II->getObjCOrBuiltinID();
-    Bits = (Bits << 1) | unsigned(hasMacroDefinition);
+    Bits = (Bits << 1) | unsigned(HasMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     clang::io::Emit16(Out, Bits);
 
-    if (hasMacroDefinition)
+    if (HasMacroDefinition)
       clang::io::Emit32(Out, Writer.getMacroOffset(II));
 
     // Emit the declaration IDs in reverse order, because the
@@ -2306,9 +2288,9 @@ class ASTIdentifierTableTrait {
     // adds declarations to the end of the list (so we need to see the
     // struct "status" before the function "status").
     // Only emit declarations that aren't from a chained PCH, though.
-    llvm::SmallVector<Decl *, 16> Decls(IdentifierResolver::begin(II),
+    SmallVector<Decl *, 16> Decls(IdentifierResolver::begin(II),
                                         IdentifierResolver::end());
-    for (llvm::SmallVector<Decl *, 16>::reverse_iterator D = Decls.rbegin(),
+    for (SmallVector<Decl *, 16>::reverse_iterator D = Decls.rbegin(),
                                                       DEnd = Decls.rend();
          D != DEnd; ++D)
       clang::io::Emit32(Out, Writer.getDeclID(*D));
@@ -2321,14 +2303,14 @@ class ASTIdentifierTableTrait {
 /// The identifier table consists of a blob containing string data
 /// (the actual identifiers themselves) and a separate "offsets" index
 /// that maps identifier IDs to locations within the blob.
-void ASTWriter::WriteIdentifierTable(Preprocessor &PP) {
+void ASTWriter::WriteIdentifierTable(Preprocessor &PP, bool IsModule) {
   using namespace llvm;
 
   // Create and write out the blob that contains the identifier
   // strings.
   {
     OnDiskChainedHashTableGenerator<ASTIdentifierTableTrait> Generator;
-    ASTIdentifierTableTrait Trait(*this, PP);
+    ASTIdentifierTableTrait Trait(*this, PP, IsModule);
 
     // Look for any identifiers that were named while processing the
     // headers, but are otherwise not needed. We add these to the hash
@@ -2348,14 +2330,15 @@ void ASTWriter::WriteIdentifierTable(Preprocessor &PP) {
          ID != IDEnd; ++ID) {
       assert(ID->first && "NULL identifier in identifier table");
       if (!Chain || !ID->first->isFromAST())
-        Generator.insert(ID->first, ID->second, Trait);
+        Generator.insert(const_cast<IdentifierInfo *>(ID->first), ID->second, 
+                         Trait);
     }
 
     // Create the on-disk hash table in a buffer.
     llvm::SmallString<4096> IdentifierTable;
     uint32_t BucketOffset;
     {
-      ASTIdentifierTableTrait Trait(*this, PP);
+      ASTIdentifierTableTrait Trait(*this, PP, IsModule);
       llvm::raw_svector_ostream Out(IdentifierTable);
       // Make sure that no bucket is at offset 0
       clang::io::Emit32(Out, 0);
@@ -2380,12 +2363,14 @@ void ASTWriter::WriteIdentifierTable(Preprocessor &PP) {
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of identifiers
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first ID
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
   unsigned IdentifierOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
 
   RecordData Record;
   Record.push_back(IDENTIFIER_OFFSET);
   Record.push_back(IdentifierOffsets.size());
+  Record.push_back(FirstIdentID - NUM_PREDEF_IDENT_IDS);
   Stream.EmitRecordWithBlob(IdentifierOffsetAbbrev, Record,
                             data(IdentifierOffsets));
 }
@@ -2424,7 +2409,6 @@ class ASTDeclContextNameLookupTrait {
     case DeclarationName::CXXConstructorName:
     case DeclarationName::CXXDestructorName:
     case DeclarationName::CXXConversionFunctionName:
-      ID.AddInteger(Writer.GetOrCreateTypeID(Name.getCXXNameType()));
       break;
     case DeclarationName::CXXOperatorName:
       ID.AddInteger(Name.getCXXOverloadedOperator());
@@ -2439,7 +2423,7 @@ class ASTDeclContextNameLookupTrait {
   }
 
   std::pair<unsigned,unsigned>
-    EmitKeyDataLength(llvm::raw_ostream& Out, DeclarationName Name,
+    EmitKeyDataLength(raw_ostream& Out, DeclarationName Name,
                       data_type_ref Lookup) {
     unsigned KeyLen = 1;
     switch (Name.getNameKind()) {
@@ -2447,15 +2431,15 @@ class ASTDeclContextNameLookupTrait {
     case DeclarationName::ObjCZeroArgSelector:
     case DeclarationName::ObjCOneArgSelector:
     case DeclarationName::ObjCMultiArgSelector:
-    case DeclarationName::CXXConstructorName:
-    case DeclarationName::CXXDestructorName:
-    case DeclarationName::CXXConversionFunctionName:
     case DeclarationName::CXXLiteralOperatorName:
       KeyLen += 4;
       break;
     case DeclarationName::CXXOperatorName:
       KeyLen += 1;
       break;
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
     case DeclarationName::CXXUsingDirective:
       break;
     }
@@ -2468,7 +2452,7 @@ class ASTDeclContextNameLookupTrait {
     return std::make_pair(KeyLen, DataLen);
   }
 
-  void EmitKey(llvm::raw_ostream& Out, DeclarationName Name, unsigned) {
+  void EmitKey(raw_ostream& Out, DeclarationName Name, unsigned) {
     using namespace clang::io;
 
     assert(Name.getNameKind() < 0x100 && "Invalid name kind ?");
@@ -2482,11 +2466,6 @@ class ASTDeclContextNameLookupTrait {
     case DeclarationName::ObjCMultiArgSelector:
       Emit32(Out, Writer.getSelectorRef(Name.getObjCSelector()));
       break;
-    case DeclarationName::CXXConstructorName:
-    case DeclarationName::CXXDestructorName:
-    case DeclarationName::CXXConversionFunctionName:
-      Emit32(Out, Writer.getTypeID(Name.getCXXNameType()));
-      break;
     case DeclarationName::CXXOperatorName:
       assert(Name.getCXXOverloadedOperator() < 0x100 && "Invalid operator ?");
       Emit8(Out, Name.getCXXOverloadedOperator());
@@ -2494,12 +2473,15 @@ class ASTDeclContextNameLookupTrait {
     case DeclarationName::CXXLiteralOperatorName:
       Emit32(Out, Writer.getIdentifierRef(Name.getCXXLiteralIdentifier()));
       break;
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
     case DeclarationName::CXXUsingDirective:
       break;
     }
   }
 
-  void EmitData(llvm::raw_ostream& Out, key_type_ref,
+  void EmitData(raw_ostream& Out, key_type_ref,
                 data_type Lookup, unsigned DataLen) {
     uint64_t Start = Out.tell(); (void)Start;
     clang::io::Emit16(Out, Lookup.second - Lookup.first);
@@ -2534,9 +2516,7 @@ uint64_t ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
     return 0;
 
   // Force the DeclContext to build a its name-lookup table.
-  if (DC->hasExternalVisibleStorage())
-    DC->MaterializeVisibleDeclsFromExternalStorage();
-  else
+  if (!DC->hasExternalVisibleStorage())
     DC->lookup(DeclarationName());
 
   // Serialize the contents of the mapping used for lookup. Note that,
@@ -2553,13 +2533,37 @@ uint64_t ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
   ASTDeclContextNameLookupTrait Trait(*this);
 
   // Create the on-disk hash table representation.
+  DeclarationName ConversionName;
+  llvm::SmallVector<NamedDecl *, 4> ConversionDecls;
   for (StoredDeclsMap::iterator D = Map->begin(), DEnd = Map->end();
        D != DEnd; ++D) {
     DeclarationName Name = D->first;
     DeclContext::lookup_result Result = D->second.getLookupResult();
-    Generator.insert(Name, Result, Trait);
+    if (Result.first != Result.second) {
+      if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName) {
+        // Hash all conversion function names to the same name. The actual
+        // type information in conversion function name is not used in the
+        // key (since such type information is not stable across different
+        // modules), so the intended effect is to coalesce all of the conversion
+        // functions under a single key.
+        if (!ConversionName)
+          ConversionName = Name;
+        ConversionDecls.append(Result.first, Result.second);
+        continue;
+      }
+      
+      Generator.insert(Name, Result, Trait);
+    }
   }
 
+  // Add the conversion functions
+  if (!ConversionDecls.empty()) {
+    Generator.insert(ConversionName, 
+                     DeclContext::lookup_result(ConversionDecls.begin(),
+                                                ConversionDecls.end()),
+                     Trait);
+  }
+  
   // Create the on-disk hash table in a buffer.
   llvm::SmallString<4096> LookupTable;
   uint32_t BucketOffset;
@@ -2602,7 +2606,8 @@ void ASTWriter::WriteDeclContextVisibleUpdate(const DeclContext *DC) {
     DeclContext::lookup_result Result = D->second.getLookupResult();
     // For any name that appears in this table, the results are complete, i.e.
     // they overwrite results from previous PCHs. Merging is always a mess.
-    Generator.insert(Name, Result, Trait);
+    if (Result.first != Result.second)
+      Generator.insert(Name, Result, Trait);
   }
 
   // Create the on-disk hash table in a buffer.
@@ -2652,14 +2657,14 @@ void ASTWriter::WriteAttributes(const AttrVec &Attrs, RecordDataImpl &Record) {
   for (AttrVec::const_iterator i = Attrs.begin(), e = Attrs.end(); i != e; ++i){
     const Attr * A = *i;
     Record.push_back(A->getKind()); // FIXME: stable encoding, target attrs
-    AddSourceLocation(A->getLocation(), Record);
+    AddSourceRange(A->getRange(), Record);
 
 #include "clang/Serialization/AttrPCHWrite.inc"
 
   }
 }
 
-void ASTWriter::AddString(llvm::StringRef Str, RecordDataImpl &Record) {
+void ASTWriter::AddString(StringRef Str, RecordDataImpl &Record) {
   Record.push_back(Str.size());
   Record.insert(Record.end(), Str.begin(), Str.end());
 }
@@ -2700,15 +2705,15 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
 }
 
 ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream)
-  : Stream(Stream), Chain(0), SerializationListener(0), 
-    FirstDeclID(1), NextDeclID(FirstDeclID),
+  : Stream(Stream), Context(0), Chain(0), WritingAST(false),
+    FirstDeclID(NUM_PREDEF_DECL_IDS), NextDeclID(FirstDeclID),
     FirstTypeID(NUM_PREDEF_TYPE_IDS), NextTypeID(FirstTypeID),
-    FirstIdentID(1), NextIdentID(FirstIdentID), FirstSelectorID(1),
-    NextSelectorID(FirstSelectorID), FirstMacroID(1), NextMacroID(FirstMacroID),
+    FirstIdentID(NUM_PREDEF_IDENT_IDS), NextIdentID(FirstIdentID), 
+    FirstSelectorID(NUM_PREDEF_SELECTOR_IDS), NextSelectorID(FirstSelectorID),
     CollectedStmts(&StmtsToEmit),
     NumStatements(0), NumMacros(0), NumLexicalDeclContexts(0),
     NumVisibleDeclContexts(0),
-    FirstCXXBaseSpecifiersID(1), NextCXXBaseSpecifiersID(1),
+    NextCXXBaseSpecifiersID(1),
     DeclParmVarAbbrev(0), DeclContextLexicalAbbrev(0),
     DeclContextVisibleLookupAbbrev(0), UpdateVisibleAbbrev(0),
     DeclRefExprAbbrev(0), CharacterLiteralAbbrev(0),
@@ -2721,7 +2726,9 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream)
 
 void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
                          const std::string &OutputFile,
-                         const char *isysroot) {
+                         bool IsModule, StringRef isysroot) {
+  WritingAST = true;
+  
   // Emit the file header.
   Stream.Emit((unsigned)'C', 8);
   Stream.Emit((unsigned)'P', 8);
@@ -2730,30 +2737,53 @@ void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   WriteBlockInfoBlock();
 
-  if (Chain)
-    WriteASTChain(SemaRef, StatCalls, isysroot);
-  else
-    WriteASTCore(SemaRef, StatCalls, isysroot, OutputFile);
+  Context = &SemaRef.Context;
+  WriteASTCore(SemaRef, StatCalls, isysroot, OutputFile, IsModule);
+  Context = 0;
+  
+  WritingAST = false;
+}
+
+template<typename Vector>
+static void AddLazyVectorDecls(ASTWriter &Writer, Vector &Vec,
+                               ASTWriter::RecordData &Record) {
+  for (typename Vector::iterator I = Vec.begin(0, true), E = Vec.end();
+       I != E; ++I)  {
+    Writer.AddDeclRef(*I, Record);
+  }
 }
 
 void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                             const char *isysroot,
-                             const std::string &OutputFile) {
+                             StringRef isysroot,
+                             const std::string &OutputFile, bool IsModule) {
   using namespace llvm;
 
   ASTContext &Context = SemaRef.Context;
   Preprocessor &PP = SemaRef.PP;
 
-  // The translation unit is the first declaration we'll emit.
-  DeclIDs[Context.getTranslationUnitDecl()] = 1;
-  ++NextDeclID;
-  DeclTypesToEmit.push(Context.getTranslationUnitDecl());
-
-  // Make sure that we emit IdentifierInfos (and any attached
-  // declarations) for builtins.
-  {
+  // Set up predefined declaration IDs.
+  DeclIDs[Context.getTranslationUnitDecl()] = PREDEF_DECL_TRANSLATION_UNIT_ID;
+  if (Context.ObjCIdDecl)
+    DeclIDs[Context.ObjCIdDecl] = PREDEF_DECL_OBJC_ID_ID;
+  if (Context.ObjCSelDecl)
+    DeclIDs[Context.ObjCSelDecl] = PREDEF_DECL_OBJC_SEL_ID;
+  if (Context.ObjCClassDecl)
+    DeclIDs[Context.ObjCClassDecl] = PREDEF_DECL_OBJC_CLASS_ID;
+  if (Context.Int128Decl)
+    DeclIDs[Context.Int128Decl] = PREDEF_DECL_INT_128_ID;
+  if (Context.UInt128Decl)
+    DeclIDs[Context.UInt128Decl] = PREDEF_DECL_UNSIGNED_INT_128_ID;
+  if (Context.ObjCInstanceTypeDecl)
+    DeclIDs[Context.ObjCInstanceTypeDecl] = PREDEF_DECL_OBJC_INSTANCETYPE_ID;
+  
+  if (!Chain) {
+    // Make sure that we emit IdentifierInfos (and any attached
+    // declarations) for builtins. We don't need to do this when we're
+    // emitting chained PCH files, because all of the builtins will be
+    // in the original PCH file.
+    // FIXME: Modules won't like this at all.
     IdentifierTable &Table = PP.getIdentifierTable();
-    llvm::SmallVector<const char *, 32> BuiltinNames;
+    SmallVector<const char *, 32> BuiltinNames;
     Context.BuiltinInfo.GetBuiltinNames(BuiltinNames,
                                         Context.getLangOptions().NoBuiltin);
     for (unsigned I = 0, N = BuiltinNames.size(); I != N; ++I)
@@ -2764,24 +2794,24 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   // TentativeDefinitions order.  Generally, this record will be empty for
   // headers.
   RecordData TentativeDefinitions;
-  for (unsigned i = 0, e = SemaRef.TentativeDefinitions.size(); i != e; ++i) {
-    AddDeclRef(SemaRef.TentativeDefinitions[i], TentativeDefinitions);
-  }
-
+  AddLazyVectorDecls(*this, SemaRef.TentativeDefinitions, TentativeDefinitions);
+  
   // Build a record containing all of the file scoped decls in this file.
   RecordData UnusedFileScopedDecls;
-  for (unsigned i=0, e = SemaRef.UnusedFileScopedDecls.size(); i !=e; ++i)
-    AddDeclRef(SemaRef.UnusedFileScopedDecls[i], UnusedFileScopedDecls);
+  AddLazyVectorDecls(*this, SemaRef.UnusedFileScopedDecls, 
+                     UnusedFileScopedDecls);
 
+  // Build a record containing all of the delegating constructors we still need
+  // to resolve.
   RecordData DelegatingCtorDecls;
-  for (unsigned i=0, e = SemaRef.DelegatingCtorDecls.size(); i != e; ++i)
-    AddDeclRef(SemaRef.DelegatingCtorDecls[i], DelegatingCtorDecls);
+  AddLazyVectorDecls(*this, SemaRef.DelegatingCtorDecls, DelegatingCtorDecls);
 
+  // Write the set of weak, undeclared identifiers. We always write the
+  // entire table, since later PCH files in a PCH chain are only interested in
+  // the results at the end of the chain.
   RecordData WeakUndeclaredIdentifiers;
   if (!SemaRef.WeakUndeclaredIdentifiers.empty()) {
-    WeakUndeclaredIdentifiers.push_back(
-                                      SemaRef.WeakUndeclaredIdentifiers.size());
-    for (llvm::DenseMap<IdentifierInfo*,Sema::WeakInfo>::iterator
+    for (llvm::DenseMap<IdentifierInfo*,WeakInfo>::iterator
          I = SemaRef.WeakUndeclaredIdentifiers.begin(),
          E = SemaRef.WeakUndeclaredIdentifiers.end(); I != E; ++I) {
       AddIdentifierRef(I->first, WeakUndeclaredIdentifiers);
@@ -2800,18 +2830,18 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   for (llvm::DenseMap<DeclarationName, NamedDecl *>::iterator
          TD = SemaRef.LocallyScopedExternalDecls.begin(),
          TDEnd = SemaRef.LocallyScopedExternalDecls.end();
-       TD != TDEnd; ++TD)
-    AddDeclRef(TD->second, LocallyScopedExternalDecls);
-
+       TD != TDEnd; ++TD) {
+    if (!TD->second->isFromASTFile())
+      AddDeclRef(TD->second, LocallyScopedExternalDecls);
+  }
+  
   // Build a record containing all of the ext_vector declarations.
   RecordData ExtVectorDecls;
-  for (unsigned I = 0, N = SemaRef.ExtVectorDecls.size(); I != N; ++I)
-    AddDeclRef(SemaRef.ExtVectorDecls[I], ExtVectorDecls);
+  AddLazyVectorDecls(*this, SemaRef.ExtVectorDecls, ExtVectorDecls);
 
   // Build a record containing all of the VTable uses information.
   RecordData VTableUses;
   if (!SemaRef.VTableUses.empty()) {
-    VTableUses.push_back(SemaRef.VTableUses.size());
     for (unsigned I = 0, N = SemaRef.VTableUses.size(); I != N; ++I) {
       AddDeclRef(SemaRef.VTableUses[I].first, VTableUses);
       AddSourceLocation(SemaRef.VTableUses[I].second, VTableUses);
@@ -2821,8 +2851,7 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   // Build a record containing all of dynamic classes declarations.
   RecordData DynamicClasses;
-  for (unsigned I = 0, N = SemaRef.DynamicClasses.size(); I != N; ++I)
-    AddDeclRef(SemaRef.DynamicClasses[I], DynamicClasses);
+  AddLazyVectorDecls(*this, SemaRef.DynamicClasses, DynamicClasses);
 
   // Build a record containing all of pending implicit instantiations.
   RecordData PendingInstantiations;
@@ -2862,37 +2891,118 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Stream.EnterSubblock(AST_BLOCK_ID, 5);
   WriteMetadata(Context, isysroot, OutputFile);
   WriteLanguageOptions(Context.getLangOptions());
-  if (StatCalls && !isysroot)
+  if (StatCalls && isysroot.empty())
     WriteStatCache(*StatCalls);
   WriteSourceManagerBlock(Context.getSourceManager(), PP, isysroot);
-  // Write the record of special types.
+  
+  if (Chain) {
+    // Write the mapping information describing our module dependencies and how
+    // each of those modules were mapped into our own offset/ID space, so that
+    // the reader can build the appropriate mapping to its own offset/ID space.
+    // The map consists solely of a blob with the following format:
+    // *(module-name-len:i16 module-name:len*i8
+    //   source-location-offset:i32
+    //   identifier-id:i32
+    //   preprocessed-entity-id:i32
+    //   macro-definition-id:i32
+    //   selector-id:i32
+    //   declaration-id:i32
+    //   c++-base-specifiers-id:i32
+    //   type-id:i32)
+    // 
+    llvm::BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+    Abbrev->Add(BitCodeAbbrevOp(MODULE_OFFSET_MAP));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+    unsigned ModuleOffsetMapAbbrev = Stream.EmitAbbrev(Abbrev);
+    llvm::SmallString<2048> Buffer;
+    {
+      llvm::raw_svector_ostream Out(Buffer);
+      for (ModuleManager::ModuleConstIterator M = Chain->ModuleMgr.begin(),
+           MEnd = Chain->ModuleMgr.end();
+           M != MEnd; ++M) {
+        StringRef FileName = (*M)->FileName;
+        io::Emit16(Out, FileName.size());
+        Out.write(FileName.data(), FileName.size());
+        io::Emit32(Out, (*M)->SLocEntryBaseOffset);
+        io::Emit32(Out, (*M)->BaseIdentifierID);
+        io::Emit32(Out, (*M)->BasePreprocessedEntityID);
+        io::Emit32(Out, (*M)->BaseSelectorID);
+        io::Emit32(Out, (*M)->BaseDeclID);
+        io::Emit32(Out, (*M)->BaseTypeIndex);
+      }
+    }
+    Record.clear();
+    Record.push_back(MODULE_OFFSET_MAP);
+    Stream.EmitRecordWithBlob(ModuleOffsetMapAbbrev, Record,
+                              Buffer.data(), Buffer.size());
+  }
+
+  // Create a lexical update block containing all of the declarations in the
+  // translation unit that do not come from other AST files.
+  const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
+  SmallVector<KindDeclIDPair, 64> NewGlobalDecls;
+  for (DeclContext::decl_iterator I = TU->noload_decls_begin(),
+                                  E = TU->noload_decls_end();
+       I != E; ++I) {
+    if (!(*I)->isFromASTFile())
+      NewGlobalDecls.push_back(std::make_pair((*I)->getKind(), GetDeclRef(*I)));
+    else if ((*I)->isChangedSinceDeserialization())
+      (void)GetDeclRef(*I); // Make sure it's written, but don't record it.
+  }
+  
+  llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev();
+  Abv->Add(llvm::BitCodeAbbrevOp(TU_UPDATE_LEXICAL));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+  unsigned TuUpdateLexicalAbbrev = Stream.EmitAbbrev(Abv);
   Record.clear();
-
-  AddTypeRef(Context.getBuiltinVaListType(), Record);
-  AddTypeRef(Context.getObjCIdType(), Record);
-  AddTypeRef(Context.getObjCSelType(), Record);
-  AddTypeRef(Context.getObjCProtoType(), Record);
-  AddTypeRef(Context.getObjCClassType(), Record);
-  AddTypeRef(Context.getRawCFConstantStringType(), Record);
-  AddTypeRef(Context.getRawObjCFastEnumerationStateType(), Record);
-  AddTypeRef(Context.getFILEType(), Record);
-  AddTypeRef(Context.getjmp_bufType(), Record);
-  AddTypeRef(Context.getsigjmp_bufType(), Record);
-  AddTypeRef(Context.ObjCIdRedefinitionType, Record);
-  AddTypeRef(Context.ObjCClassRedefinitionType, Record);
-  AddTypeRef(Context.getRawBlockdescriptorType(), Record);
-  AddTypeRef(Context.getRawBlockdescriptorExtendedType(), Record);
-  AddTypeRef(Context.ObjCSelRedefinitionType, Record);
-  AddTypeRef(Context.getRawNSConstantStringType(), Record);
-  Record.push_back(Context.isInt128Installed());
-  AddTypeRef(Context.AutoDeductTy, Record);
-  AddTypeRef(Context.AutoRRefDeductTy, Record);
-  Stream.EmitRecord(SPECIAL_TYPES, Record);
-
+  Record.push_back(TU_UPDATE_LEXICAL);
+  Stream.EmitRecordWithBlob(TuUpdateLexicalAbbrev, Record,
+                            data(NewGlobalDecls));
+  
+  // And a visible updates block for the translation unit.
+  Abv = new llvm::BitCodeAbbrev();
+  Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Fixed, 32));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+  UpdateVisibleAbbrev = Stream.EmitAbbrev(Abv);
+  WriteDeclContextVisibleUpdate(TU);
+  
+  // If the translation unit has an anonymous namespace, and we don't already
+  // have an update block for it, write it as an update block.
+  if (NamespaceDecl *NS = TU->getAnonymousNamespace()) {
+    ASTWriter::UpdateRecord &Record = DeclUpdates[TU];
+    if (Record.empty()) {
+      Record.push_back(UPD_CXX_ADDED_ANONYMOUS_NAMESPACE);
+      Record.push_back(reinterpret_cast<uint64_t>(NS));
+    }
+  }
+  
+  // Resolve any declaration pointers within the declaration updates block and
+  // chained Objective-C categories block to declaration IDs.
+  ResolveDeclUpdatesBlocks();
+  ResolveChainedObjCCategories();
+  
+  // Form the record of special types.
+  RecordData SpecialTypes;
+  AddTypeRef(Context.getBuiltinVaListType(), SpecialTypes);
+  AddTypeRef(Context.ObjCProtoType, SpecialTypes);
+  AddTypeRef(Context.getRawCFConstantStringType(), SpecialTypes);
+  AddTypeRef(Context.getFILEType(), SpecialTypes);
+  AddTypeRef(Context.getjmp_bufType(), SpecialTypes);
+  AddTypeRef(Context.getsigjmp_bufType(), SpecialTypes);
+  AddTypeRef(Context.ObjCIdRedefinitionType, SpecialTypes);
+  AddTypeRef(Context.ObjCClassRedefinitionType, SpecialTypes);
+  AddTypeRef(Context.ObjCSelRedefinitionType, SpecialTypes);
+  
   // Keep writing types and declarations until all types and
   // declarations have been written.
   Stream.EnterSubblock(DECLTYPES_BLOCK_ID, NUM_ALLOWED_ABBREVS_SIZE);
   WriteDeclsBlockAbbrevs();
+  for (DeclsToRewriteTy::iterator I = DeclsToRewrite.begin(), 
+                                  E = DeclsToRewrite.end(); 
+       I != E; ++I)
+    DeclTypesToEmit.push(const_cast<Decl*>(*I));
   while (!DeclTypesToEmit.empty()) {
     DeclOrType DOT = DeclTypesToEmit.front();
     DeclTypesToEmit.pop();
@@ -2903,11 +3013,11 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   }
   Stream.ExitBlock();
 
-  WritePreprocessor(PP);
+  WritePreprocessor(PP, IsModule);
   WriteHeaderSearch(PP.getHeaderSearchInfo(), isysroot);
   WriteSelectors(SemaRef);
   WriteReferencedSelectorsPool(SemaRef);
-  WriteIdentifierTable(PP);
+  WriteIdentifierTable(PP, IsModule);
   WriteFPPragmaOptions(SemaRef.getFPOptions());
   WriteOpenCLExtensions(SemaRef);
 
@@ -2916,6 +3026,21 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   WriteCXXBaseSpecifiersOffsets();
   
+  Stream.EmitRecord(SPECIAL_TYPES, SpecialTypes);
+
+  /// Build a record containing first declarations from a chained PCH and the
+  /// most recent declarations in this AST that they point to.
+  RecordData FirstLatestDeclIDs;
+  for (FirstLatestDeclMap::iterator I = FirstLatestDecls.begin(), 
+                                    E = FirstLatestDecls.end(); 
+       I != E; ++I) {
+    AddDeclRef(I->first, FirstLatestDeclIDs);
+    AddDeclRef(I->second, FirstLatestDeclIDs);
+  }
+  
+  if (!FirstLatestDeclIDs.empty())
+    Stream.EmitRecord(REDECLS_UPDATE_LATEST, FirstLatestDeclIDs);
+
   // Write the record containing external, unnamed definitions.
   if (!ExternalDefinitions.empty())
     Stream.EmitRecord(EXTERNAL_DEFINITIONS, ExternalDefinitions);
@@ -2970,6 +3095,17 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   if (!KnownNamespaces.empty())
     Stream.EmitRecord(KNOWN_NAMESPACES, KnownNamespaces);
   
+  // Write the visible updates to DeclContexts.
+  for (llvm::SmallPtrSet<const DeclContext *, 16>::iterator
+       I = UpdatedDeclContexts.begin(),
+       E = UpdatedDeclContexts.end();
+       I != E; ++I)
+    WriteDeclContextVisibleUpdate(*I);
+
+  WriteDeclUpdatesBlocks();
+  WriteDeclReplacementsBlock();
+  WriteChainedObjCCategories();
+
   // Some simple statistics
   Record.clear();
   Record.push_back(NumStatements);
@@ -2980,256 +3116,34 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Stream.ExitBlock();
 }
 
-void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                              const char *isysroot) {
-  using namespace llvm;
+/// \brief Go through the declaration update blocks and resolve declaration
+/// pointers into declaration IDs.
+void ASTWriter::ResolveDeclUpdatesBlocks() {
+  for (DeclUpdateMap::iterator
+       I = DeclUpdates.begin(), E = DeclUpdates.end(); I != E; ++I) {
+    const Decl *D = I->first;
+    UpdateRecord &URec = I->second;
+    
+    if (DeclsToRewrite.count(D))
+      continue; // The decl will be written completely
 
-  ASTContext &Context = SemaRef.Context;
-  Preprocessor &PP = SemaRef.PP;
-
-  RecordData Record;
-  Stream.EnterSubblock(AST_BLOCK_ID, 5);
-  WriteMetadata(Context, isysroot, "");
-  if (StatCalls && !isysroot)
-    WriteStatCache(*StatCalls);
-  // FIXME: Source manager block should only write new stuff, which could be
-  // done by tracking the largest ID in the chain
-  WriteSourceManagerBlock(Context.getSourceManager(), PP, isysroot);
-
-  // The special types are in the chained PCH.
-
-  // We don't start with the translation unit, but with its decls that
-  // don't come from the chained PCH.
-  const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
-  llvm::SmallVector<KindDeclIDPair, 64> NewGlobalDecls;
-  for (DeclContext::decl_iterator I = TU->noload_decls_begin(),
-                                  E = TU->noload_decls_end();
-       I != E; ++I) {
-    if ((*I)->getPCHLevel() == 0)
-      NewGlobalDecls.push_back(std::make_pair((*I)->getKind(), GetDeclRef(*I)));
-    else if ((*I)->isChangedSinceDeserialization())
-      (void)GetDeclRef(*I); // Make sure it's written, but don't record it.
-  }
-  // We also need to write a lexical updates block for the TU.
-  llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev();
-  Abv->Add(llvm::BitCodeAbbrevOp(TU_UPDATE_LEXICAL));
-  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
-  unsigned TuUpdateLexicalAbbrev = Stream.EmitAbbrev(Abv);
-  Record.clear();
-  Record.push_back(TU_UPDATE_LEXICAL);
-  Stream.EmitRecordWithBlob(TuUpdateLexicalAbbrev, Record,
-                            data(NewGlobalDecls));
-  // And a visible updates block for the DeclContexts.
-  Abv = new llvm::BitCodeAbbrev();
-  Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE));
-  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
-  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Fixed, 32));
-  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
-  UpdateVisibleAbbrev = Stream.EmitAbbrev(Abv);
-  WriteDeclContextVisibleUpdate(TU);
-
-  // Build a record containing all of the new tentative definitions in this
-  // file, in TentativeDefinitions order.
-  RecordData TentativeDefinitions;
-  for (unsigned i = 0, e = SemaRef.TentativeDefinitions.size(); i != e; ++i) {
-    if (SemaRef.TentativeDefinitions[i]->getPCHLevel() == 0)
-      AddDeclRef(SemaRef.TentativeDefinitions[i], TentativeDefinitions);
-  }
-
-  // Build a record containing all of the file scoped decls in this file.
-  RecordData UnusedFileScopedDecls;
-  for (unsigned i=0, e = SemaRef.UnusedFileScopedDecls.size(); i !=e; ++i) {
-    if (SemaRef.UnusedFileScopedDecls[i]->getPCHLevel() == 0)
-      AddDeclRef(SemaRef.UnusedFileScopedDecls[i], UnusedFileScopedDecls);
-  }
-
-  // Build a record containing all of the delegating constructor decls in this
-  // file.
-  RecordData DelegatingCtorDecls;
-  for (unsigned i=0, e = SemaRef.DelegatingCtorDecls.size(); i != e; ++i) {
-    if (SemaRef.DelegatingCtorDecls[i]->getPCHLevel() == 0)
-      AddDeclRef(SemaRef.DelegatingCtorDecls[i], DelegatingCtorDecls);
-  }
-
-  // We write the entire table, overwriting the tables from the chain.
-  RecordData WeakUndeclaredIdentifiers;
-  if (!SemaRef.WeakUndeclaredIdentifiers.empty()) {
-    WeakUndeclaredIdentifiers.push_back(
-                                      SemaRef.WeakUndeclaredIdentifiers.size());
-    for (llvm::DenseMap<IdentifierInfo*,Sema::WeakInfo>::iterator
-         I = SemaRef.WeakUndeclaredIdentifiers.begin(),
-         E = SemaRef.WeakUndeclaredIdentifiers.end(); I != E; ++I) {
-      AddIdentifierRef(I->first, WeakUndeclaredIdentifiers);
-      AddIdentifierRef(I->second.getAlias(), WeakUndeclaredIdentifiers);
-      AddSourceLocation(I->second.getLocation(), WeakUndeclaredIdentifiers);
-      WeakUndeclaredIdentifiers.push_back(I->second.getUsed());
+    unsigned Idx = 0, N = URec.size();
+    while (Idx < N) {
+      switch ((DeclUpdateKind)URec[Idx++]) {
+      case UPD_CXX_SET_DEFINITIONDATA:
+      case UPD_CXX_ADDED_IMPLICIT_MEMBER:
+      case UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION:
+      case UPD_CXX_ADDED_ANONYMOUS_NAMESPACE:
+        URec[Idx] = GetDeclRef(reinterpret_cast<Decl *>(URec[Idx]));
+        ++Idx;
+        break;
+          
+      case UPD_CXX_INSTANTIATED_STATIC_DATA_MEMBER:
+        ++Idx;
+        break;
+      }
     }
   }
-
-  // Build a record containing all of the locally-scoped external
-  // declarations in this header file. Generally, this record will be
-  // empty.
-  RecordData LocallyScopedExternalDecls;
-  // FIXME: This is filling in the AST file in densemap order which is
-  // nondeterminstic!
-  for (llvm::DenseMap<DeclarationName, NamedDecl *>::iterator
-         TD = SemaRef.LocallyScopedExternalDecls.begin(),
-         TDEnd = SemaRef.LocallyScopedExternalDecls.end();
-       TD != TDEnd; ++TD) {
-    if (TD->second->getPCHLevel() == 0)
-      AddDeclRef(TD->second, LocallyScopedExternalDecls);
-  }
-
-  // Build a record containing all of the ext_vector declarations.
-  RecordData ExtVectorDecls;
-  for (unsigned I = 0, N = SemaRef.ExtVectorDecls.size(); I != N; ++I) {
-    if (SemaRef.ExtVectorDecls[I]->getPCHLevel() == 0)
-      AddDeclRef(SemaRef.ExtVectorDecls[I], ExtVectorDecls);
-  }
-
-  // Build a record containing all of the VTable uses information.
-  // We write everything here, because it's too hard to determine whether
-  // a use is new to this part.
-  RecordData VTableUses;
-  if (!SemaRef.VTableUses.empty()) {
-    VTableUses.push_back(SemaRef.VTableUses.size());
-    for (unsigned I = 0, N = SemaRef.VTableUses.size(); I != N; ++I) {
-      AddDeclRef(SemaRef.VTableUses[I].first, VTableUses);
-      AddSourceLocation(SemaRef.VTableUses[I].second, VTableUses);
-      VTableUses.push_back(SemaRef.VTablesUsed[SemaRef.VTableUses[I].first]);
-    }
-  }
-
-  // Build a record containing all of dynamic classes declarations.
-  RecordData DynamicClasses;
-  for (unsigned I = 0, N = SemaRef.DynamicClasses.size(); I != N; ++I)
-    if (SemaRef.DynamicClasses[I]->getPCHLevel() == 0)
-      AddDeclRef(SemaRef.DynamicClasses[I], DynamicClasses);
-
-  // Build a record containing all of pending implicit instantiations.
-  RecordData PendingInstantiations;
-  for (std::deque<Sema::PendingImplicitInstantiation>::iterator
-         I = SemaRef.PendingInstantiations.begin(),
-         N = SemaRef.PendingInstantiations.end(); I != N; ++I) {
-    AddDeclRef(I->first, PendingInstantiations);
-    AddSourceLocation(I->second, PendingInstantiations);
-  }
-  assert(SemaRef.PendingLocalImplicitInstantiations.empty() &&
-         "There are local ones at end of translation unit!");
-
-  // Build a record containing some declaration references.
-  // It's not worth the effort to avoid duplication here.
-  RecordData SemaDeclRefs;
-  if (SemaRef.StdNamespace || SemaRef.StdBadAlloc) {
-    AddDeclRef(SemaRef.getStdNamespace(), SemaDeclRefs);
-    AddDeclRef(SemaRef.getStdBadAlloc(), SemaDeclRefs);
-  }
-
-  Stream.EnterSubblock(DECLTYPES_BLOCK_ID, NUM_ALLOWED_ABBREVS_SIZE);
-  WriteDeclsBlockAbbrevs();
-  for (DeclsToRewriteTy::iterator
-         I = DeclsToRewrite.begin(), E = DeclsToRewrite.end(); I != E; ++I)
-    DeclTypesToEmit.push(const_cast<Decl*>(*I));
-  while (!DeclTypesToEmit.empty()) {
-    DeclOrType DOT = DeclTypesToEmit.front();
-    DeclTypesToEmit.pop();
-    if (DOT.isType())
-      WriteType(DOT.getType());
-    else
-      WriteDecl(Context, DOT.getDecl());
-  }
-  Stream.ExitBlock();
-
-  WritePreprocessor(PP);
-  WriteSelectors(SemaRef);
-  WriteReferencedSelectorsPool(SemaRef);
-  WriteIdentifierTable(PP);
-  WriteFPPragmaOptions(SemaRef.getFPOptions());
-  WriteOpenCLExtensions(SemaRef);
-
-  WriteTypeDeclOffsets();
-  // FIXME: For chained PCH only write the new mappings (we currently
-  // write all of them again).
-  WritePragmaDiagnosticMappings(Context.getDiagnostics());
-
-  WriteCXXBaseSpecifiersOffsets();
-
-  /// Build a record containing first declarations from a chained PCH and the
-  /// most recent declarations in this AST that they point to.
-  RecordData FirstLatestDeclIDs;
-  for (FirstLatestDeclMap::iterator
-        I = FirstLatestDecls.begin(), E = FirstLatestDecls.end(); I != E; ++I) {
-    assert(I->first->getPCHLevel() > I->second->getPCHLevel() &&
-           "Expected first & second to be in different PCHs");
-    AddDeclRef(I->first, FirstLatestDeclIDs);
-    AddDeclRef(I->second, FirstLatestDeclIDs);
-  }
-  if (!FirstLatestDeclIDs.empty())
-    Stream.EmitRecord(REDECLS_UPDATE_LATEST, FirstLatestDeclIDs);
-
-  // Write the record containing external, unnamed definitions.
-  if (!ExternalDefinitions.empty())
-    Stream.EmitRecord(EXTERNAL_DEFINITIONS, ExternalDefinitions);
-
-  // Write the record containing tentative definitions.
-  if (!TentativeDefinitions.empty())
-    Stream.EmitRecord(TENTATIVE_DEFINITIONS, TentativeDefinitions);
-
-  // Write the record containing unused file scoped decls.
-  if (!UnusedFileScopedDecls.empty())
-    Stream.EmitRecord(UNUSED_FILESCOPED_DECLS, UnusedFileScopedDecls);
-
-  // Write the record containing weak undeclared identifiers.
-  if (!WeakUndeclaredIdentifiers.empty())
-    Stream.EmitRecord(WEAK_UNDECLARED_IDENTIFIERS,
-                      WeakUndeclaredIdentifiers);
-
-  // Write the record containing locally-scoped external definitions.
-  if (!LocallyScopedExternalDecls.empty())
-    Stream.EmitRecord(LOCALLY_SCOPED_EXTERNAL_DECLS,
-                      LocallyScopedExternalDecls);
-
-  // Write the record containing ext_vector type names.
-  if (!ExtVectorDecls.empty())
-    Stream.EmitRecord(EXT_VECTOR_DECLS, ExtVectorDecls);
-
-  // Write the record containing VTable uses information.
-  if (!VTableUses.empty())
-    Stream.EmitRecord(VTABLE_USES, VTableUses);
-
-  // Write the record containing dynamic classes declarations.
-  if (!DynamicClasses.empty())
-    Stream.EmitRecord(DYNAMIC_CLASSES, DynamicClasses);
-
-  // Write the record containing pending implicit instantiations.
-  if (!PendingInstantiations.empty())
-    Stream.EmitRecord(PENDING_IMPLICIT_INSTANTIATIONS, PendingInstantiations);
-
-  // Write the record containing declaration references of Sema.
-  if (!SemaDeclRefs.empty())
-    Stream.EmitRecord(SEMA_DECL_REFS, SemaDeclRefs);
-  
-  // Write the delegating constructors.
-  if (!DelegatingCtorDecls.empty())
-    Stream.EmitRecord(DELEGATING_CTORS, DelegatingCtorDecls);
-
-  // Write the updates to DeclContexts.
-  for (llvm::SmallPtrSet<const DeclContext *, 16>::iterator
-           I = UpdatedDeclContexts.begin(),
-           E = UpdatedDeclContexts.end();
-         I != E; ++I)
-    WriteDeclContextVisibleUpdate(*I);
-
-  WriteDeclUpdatesBlocks();
-
-  Record.clear();
-  Record.push_back(NumStatements);
-  Record.push_back(NumMacros);
-  Record.push_back(NumLexicalDeclContexts);
-  Record.push_back(NumVisibleDeclContexts);
-  WriteDeclReplacementsBlock();
-  Stream.EmitRecord(STATISTICS, Record);
-  Stream.ExitBlock();
 }
 
 void ASTWriter::WriteDeclUpdatesBlocks() {
@@ -3261,7 +3175,7 @@ void ASTWriter::WriteDeclReplacementsBlock() {
     return;
 
   RecordData Record;
-  for (llvm::SmallVector<std::pair<DeclID, uint64_t>, 16>::iterator
+  for (SmallVector<std::pair<DeclID, uint64_t>, 16>::iterator
            I = ReplacedDecls.begin(), E = ReplacedDecls.end(); I != E; ++I) {
     Record.push_back(I->first);
     Record.push_back(I->second);
@@ -3269,6 +3183,37 @@ void ASTWriter::WriteDeclReplacementsBlock() {
   Stream.EmitRecord(DECL_REPLACEMENTS, Record);
 }
 
+void ASTWriter::ResolveChainedObjCCategories() {
+  for (SmallVector<ChainedObjCCategoriesData, 16>::iterator
+       I = LocalChainedObjCCategories.begin(),
+       E = LocalChainedObjCCategories.end(); I != E; ++I) {
+    ChainedObjCCategoriesData &Data = *I;
+    Data.InterfaceID = GetDeclRef(Data.Interface);
+    Data.TailCategoryID = GetDeclRef(Data.TailCategory);
+  }
+
+}
+
+void ASTWriter::WriteChainedObjCCategories() {
+  if (LocalChainedObjCCategories.empty())
+    return;
+
+  RecordData Record;
+  for (SmallVector<ChainedObjCCategoriesData, 16>::iterator
+         I = LocalChainedObjCCategories.begin(),
+         E = LocalChainedObjCCategories.end(); I != E; ++I) {
+    ChainedObjCCategoriesData &Data = *I;
+    serialization::DeclID
+        HeadCatID = getDeclID(Data.Interface->getCategoryList());
+    assert(HeadCatID != 0 && "Category not written ?");
+
+    Record.push_back(Data.InterfaceID);
+    Record.push_back(HeadCatID);
+    Record.push_back(Data.TailCategoryID);
+  }
+  Stream.EmitRecord(OBJC_CHAINED_CATEGORIES, Record);
+}
+
 void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record) {
   Record.push_back(Loc.getRawEncoding());
 }
@@ -3307,16 +3252,6 @@ IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   return ID;
 }
 
-MacroID ASTWriter::getMacroDefinitionID(MacroDefinition *MD) {
-  if (MD == 0)
-    return 0;
-
-  MacroID &ID = MacroDefinitions[MD];
-  if (ID == 0)
-    ID = NextMacroID++;
-  return ID;
-}
-
 void ASTWriter::AddSelectorRef(const Selector SelRef, RecordDataImpl &Record) {
   Record.push_back(getSelectorRef(SelRef));
 }
@@ -3416,13 +3351,13 @@ void ASTWriter::AddTypeRef(QualType T, RecordDataImpl &Record) {
   Record.push_back(GetOrCreateTypeID(T));
 }
 
-TypeID ASTWriter::GetOrCreateTypeID(QualType T) {
-  return MakeTypeID(T,
+TypeID ASTWriter::GetOrCreateTypeID( QualType T) {
+  return MakeTypeID(*Context, T,
               std::bind1st(std::mem_fun(&ASTWriter::GetOrCreateTypeIdx), this));
 }
 
 TypeID ASTWriter::getTypeID(QualType T) const {
-  return MakeTypeID(T,
+  return MakeTypeID(*Context, T,
               std::bind1st(std::mem_fun(&ASTWriter::getTypeIdx), this));
 }
 
@@ -3456,6 +3391,8 @@ void ASTWriter::AddDeclRef(const Decl *D, RecordDataImpl &Record) {
 }
 
 DeclID ASTWriter::GetDeclRef(const Decl *D) {
+  assert(WritingAST && "Cannot request a declaration ID before AST writing");
+  
   if (D == 0) {
     return 0;
   }
@@ -3571,7 +3508,7 @@ void ASTWriter::AddNestedNameSpecifier(NestedNameSpecifier *NNS,
                                        RecordDataImpl &Record) {
   // Nested name specifiers usually aren't too long. I think that 8 would
   // typically accommodate the vast majority.
-  llvm::SmallVector<NestedNameSpecifier *, 8> NestedNames;
+  SmallVector<NestedNameSpecifier *, 8> NestedNames;
 
   // Push each of the NNS's onto a stack for serialization in reverse order.
   while (NNS) {
@@ -3614,7 +3551,7 @@ void ASTWriter::AddNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS,
                                           RecordDataImpl &Record) {
   // Nested name specifiers usually aren't too long. I think that 8 would
   // typically accommodate the vast majority.
-  llvm::SmallVector<NestedNameSpecifierLoc , 8> NestedNames;
+  SmallVector<NestedNameSpecifierLoc , 8> NestedNames;
 
   // Push each of the nested-name-specifiers's onto a stack for
   // serialization in reverse order.
@@ -3805,7 +3742,7 @@ void ASTWriter::FlushCXXBaseSpecifiers() {
     Record.clear();
     
     // Record the offset of this base-specifier set.
-    unsigned Index = CXXBaseSpecifiersToWrite[I].ID - FirstCXXBaseSpecifiersID;
+    unsigned Index = CXXBaseSpecifiersToWrite[I].ID - 1;
     if (Index == CXXBaseSpecifiersOffsets.size())
       CXXBaseSpecifiersOffsets.push_back(Stream.GetCurrentBitNo());
     else {
@@ -3871,7 +3808,9 @@ void ASTWriter::AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Rec
   struct CXXRecordDecl::DefinitionData &Data = *D->DefinitionData;
   Record.push_back(Data.UserDeclaredConstructor);
   Record.push_back(Data.UserDeclaredCopyConstructor);
+  Record.push_back(Data.UserDeclaredMoveConstructor);
   Record.push_back(Data.UserDeclaredCopyAssignment);
+  Record.push_back(Data.UserDeclaredMoveAssignment);
   Record.push_back(Data.UserDeclaredDestructor);
   Record.push_back(Data.Aggregate);
   Record.push_back(Data.PlainOldData);
@@ -3885,7 +3824,7 @@ void ASTWriter::AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Rec
   Record.push_back(Data.HasPublicFields);
   Record.push_back(Data.HasMutableFields);
   Record.push_back(Data.HasTrivialDefaultConstructor);
-  Record.push_back(Data.HasConstExprNonCopyMoveConstructor);
+  Record.push_back(Data.HasConstexprNonCopyMoveConstructor);
   Record.push_back(Data.HasTrivialCopyConstructor);
   Record.push_back(Data.HasTrivialMoveConstructor);
   Record.push_back(Data.HasTrivialCopyAssignment);
@@ -3896,8 +3835,12 @@ void ASTWriter::AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Rec
   Record.push_back(Data.UserProvidedDefaultConstructor);
   Record.push_back(Data.DeclaredDefaultConstructor);
   Record.push_back(Data.DeclaredCopyConstructor);
+  Record.push_back(Data.DeclaredMoveConstructor);
   Record.push_back(Data.DeclaredCopyAssignment);
+  Record.push_back(Data.DeclaredMoveAssignment);
   Record.push_back(Data.DeclaredDestructor);
+  Record.push_back(Data.FailedImplicitMoveConstructor);
+  Record.push_back(Data.FailedImplicitMoveAssignment);
 
   Record.push_back(Data.NumBases);
   if (Data.NumBases > 0)
@@ -3918,28 +3861,23 @@ void ASTWriter::AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Rec
 
 void ASTWriter::ReaderInitialized(ASTReader *Reader) {
   assert(Reader && "Cannot remove chain");
-  assert(!Chain && "Cannot replace chain");
+  assert((!Chain || Chain == Reader) && "Cannot replace chain");
   assert(FirstDeclID == NextDeclID &&
          FirstTypeID == NextTypeID &&
          FirstIdentID == NextIdentID &&
          FirstSelectorID == NextSelectorID &&
-         FirstMacroID == NextMacroID &&
-         FirstCXXBaseSpecifiersID == NextCXXBaseSpecifiersID &&
          "Setting chain after writing has started.");
+
   Chain = Reader;
 
-  FirstDeclID += Chain->getTotalNumDecls();
-  FirstTypeID += Chain->getTotalNumTypes();
-  FirstIdentID += Chain->getTotalNumIdentifiers();
-  FirstSelectorID += Chain->getTotalNumSelectors();
-  FirstMacroID += Chain->getTotalNumMacroDefinitions();
-  FirstCXXBaseSpecifiersID += Chain->getTotalNumCXXBaseSpecifiers();
+  FirstDeclID = NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls();
+  FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes();
+  FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers();
+  FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors();
   NextDeclID = FirstDeclID;
   NextTypeID = FirstTypeID;
   NextIdentID = FirstIdentID;
   NextSelectorID = FirstSelectorID;
-  NextMacroID = FirstMacroID;
-  NextCXXBaseSpecifiersID = FirstCXXBaseSpecifiersID;
 }
 
 void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) {
@@ -3967,16 +3905,18 @@ void ASTWriter::SelectorRead(SelectorID ID, Selector S) {
   SelectorIDs[S] = ID;
 }
 
-void ASTWriter::MacroDefinitionRead(serialization::MacroID ID,
+void ASTWriter::MacroDefinitionRead(serialization::PreprocessedEntityID ID,
                                     MacroDefinition *MD) {
+  assert(MacroDefinitions.find(MD) == MacroDefinitions.end());
   MacroDefinitions[MD] = ID;
 }
 
 void ASTWriter::CompletedTagDefinition(const TagDecl *D) {
-  assert(D->isDefinition());
+  assert(D->isCompleteDefinition());
+  assert(!WritingAST && "Already writing the AST!");
   if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
     // We are interested when a PCH decl is modified.
-    if (RD->getPCHLevel() > 0) {
+    if (RD->isFromASTFile()) {
       // A forward reference was mutated into a definition. Rewrite it.
       // FIXME: This happens during template instantiation, should we
       // have created a new definition decl instead ?
@@ -3990,67 +3930,73 @@ void ASTWriter::CompletedTagDefinition(const TagDecl *D) {
         continue;
 
       // We are interested when a PCH decl is modified.
-      if (Redecl->getPCHLevel() > 0) {
+      if (Redecl->isFromASTFile()) {
         UpdateRecord &Record = DeclUpdates[Redecl];
         Record.push_back(UPD_CXX_SET_DEFINITIONDATA);
         assert(Redecl->DefinitionData);
         assert(Redecl->DefinitionData->Definition == D);
-        AddDeclRef(D, Record); // the DefinitionDecl
+        Record.push_back(reinterpret_cast<uint64_t>(D)); // the DefinitionDecl
       }
     }
   }
 }
 void ASTWriter::AddedVisibleDecl(const DeclContext *DC, const Decl *D) {
+  assert(!WritingAST && "Already writing the AST!");
+
   // TU and namespaces are handled elsewhere.
   if (isa<TranslationUnitDecl>(DC) || isa<NamespaceDecl>(DC))
     return;
 
-  if (!(D->getPCHLevel() == 0 && cast<Decl>(DC)->getPCHLevel() > 0))
+  if (!(!D->isFromASTFile() && cast<Decl>(DC)->isFromASTFile()))
     return; // Not a source decl added to a DeclContext from PCH.
 
   AddUpdatedDeclContext(DC);
 }
 
 void ASTWriter::AddedCXXImplicitMember(const CXXRecordDecl *RD, const Decl *D) {
+  assert(!WritingAST && "Already writing the AST!");
   assert(D->isImplicit());
-  if (!(D->getPCHLevel() == 0 && RD->getPCHLevel() > 0))
+  if (!(!D->isFromASTFile() && RD->isFromASTFile()))
     return; // Not a source member added to a class from PCH.
   if (!isa<CXXMethodDecl>(D))
     return; // We are interested in lazily declared implicit methods.
 
   // A decl coming from PCH was modified.
-  assert(RD->isDefinition());
+  assert(RD->isCompleteDefinition());
   UpdateRecord &Record = DeclUpdates[RD];
   Record.push_back(UPD_CXX_ADDED_IMPLICIT_MEMBER);
-  AddDeclRef(D, Record);
+  Record.push_back(reinterpret_cast<uint64_t>(D));
 }
 
 void ASTWriter::AddedCXXTemplateSpecialization(const ClassTemplateDecl *TD,
                                      const ClassTemplateSpecializationDecl *D) {
   // The specializations set is kept in the canonical template.
+  assert(!WritingAST && "Already writing the AST!");
   TD = TD->getCanonicalDecl();
-  if (!(D->getPCHLevel() == 0 && TD->getPCHLevel() > 0))
+  if (!(!D->isFromASTFile() && TD->isFromASTFile()))
     return; // Not a source specialization added to a template from PCH.
 
   UpdateRecord &Record = DeclUpdates[TD];
   Record.push_back(UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION);
-  AddDeclRef(D, Record);
+  Record.push_back(reinterpret_cast<uint64_t>(D));
 }
 
 void ASTWriter::AddedCXXTemplateSpecialization(const FunctionTemplateDecl *TD,
                                                const FunctionDecl *D) {
   // The specializations set is kept in the canonical template.
+  assert(!WritingAST && "Already writing the AST!");
   TD = TD->getCanonicalDecl();
-  if (!(D->getPCHLevel() == 0 && TD->getPCHLevel() > 0))
+  if (!(!D->isFromASTFile() && TD->isFromASTFile()))
     return; // Not a source specialization added to a template from PCH.
 
   UpdateRecord &Record = DeclUpdates[TD];
   Record.push_back(UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION);
-  AddDeclRef(D, Record);
+  Record.push_back(reinterpret_cast<uint64_t>(D));
 }
 
 void ASTWriter::CompletedImplicitDefinition(const FunctionDecl *D) {
-  if (D->getPCHLevel() == 0)
+  assert(!WritingAST && "Already writing the AST!");
+  if (!D->isFromASTFile())
     return; // Declaration not imported from PCH.
 
   // Implicit decl from a PCH was defined.
@@ -4059,7 +4005,8 @@ void ASTWriter::CompletedImplicitDefinition(const FunctionDecl *D) {
 }
 
 void ASTWriter::StaticDataMemberInstantiated(const VarDecl *D) {
-  if (D->getPCHLevel() == 0)
+  assert(!WritingAST && "Already writing the AST!");
+  if (!D->isFromASTFile())
     return;
 
   // Since the actual instantiation is delayed, this really means that we need
@@ -4070,4 +4017,16 @@ void ASTWriter::StaticDataMemberInstantiated(const VarDecl *D) {
       D->getMemberSpecializationInfo()->getPointOfInstantiation(), Record);
 }
 
-ASTSerializationListener::~ASTSerializationListener() { }
+void ASTWriter::AddedObjCCategoryToInterface(const ObjCCategoryDecl *CatD,
+                                             const ObjCInterfaceDecl *IFD) {
+  assert(!WritingAST && "Already writing the AST!");
+  if (!IFD->isFromASTFile())
+    return; // Declaration not imported from PCH.
+  if (CatD->getNextClassCategory() &&
+      !CatD->getNextClassCategory()->isFromASTFile())
+    return; // We already recorded that the tail of a category chain should be
+            // attached to an interface.
+
+  ChainedObjCCategoriesData Data =  { IFD, CatD, 0, 0 };
+  LocalChainedObjCCategories.push_back(Data);
+}
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp
index 2b8349495751..a8243e502e30 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -65,6 +65,8 @@ namespace clang {
                                             ClassTemplateSpecializationDecl *D);
     void VisitClassTemplatePartialSpecializationDecl(
                                      ClassTemplatePartialSpecializationDecl *D);
+    void VisitClassScopeFunctionSpecializationDecl(
+                                       ClassScopeFunctionSpecializationDecl *D);
     void VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D);
     void VisitValueDecl(ValueDecl *D);
     void VisitEnumConstantDecl(EnumConstantDecl *D);
@@ -153,13 +155,11 @@ void ASTDeclWriter::VisitDecl(Decl *D) {
   Record.push_back(D->isUsed(false));
   Record.push_back(D->isReferenced());
   Record.push_back(D->getAccess());
-  Record.push_back(D->getPCHLevel());
+  Record.push_back(D->ModulePrivate);
 }
 
 void ASTDeclWriter::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
-  VisitDecl(D);
-  Writer.AddDeclRef(D->getAnonymousNamespace(), Record);
-  Code = serialization::DECL_TRANSLATION_UNIT;
+  llvm_unreachable("Translation units aren't directly serialized");
 }
 
 void ASTDeclWriter::VisitNamedDecl(NamedDecl *D) {
@@ -180,11 +180,11 @@ void ASTDeclWriter::VisitTypedefDecl(TypedefDecl *D) {
   if (!D->hasAttrs() &&
       !D->isImplicit() &&
       !D->isUsed(false) &&
-      D->getPCHLevel() == 0 &&
       D->RedeclLink.getNext() == D &&
       !D->isInvalidDecl() &&
       !D->isReferenced() &&
       D->getAccess() == AS_none &&
+      !D->isModulePrivate() &&
       D->getDeclName().getNameKind() == DeclarationName::Identifier)
     AbbrevToUse = Writer.getDeclTypedefAbbrev();
 
@@ -202,8 +202,9 @@ void ASTDeclWriter::VisitTagDecl(TagDecl *D) {
   VisitRedeclarable(D);
   Record.push_back(D->getIdentifierNamespace());
   Record.push_back((unsigned)D->getTagKind()); // FIXME: stable encoding
-  Record.push_back(D->isDefinition());
+  Record.push_back(D->isCompleteDefinition());
   Record.push_back(D->isEmbeddedInDeclarator());
+  Record.push_back(D->isFreeStanding());
   Writer.AddSourceLocation(D->getRBraceLoc(), Record);
   Record.push_back(D->hasExtInfo());
   if (D->hasExtInfo())
@@ -228,12 +229,12 @@ void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) {
   if (!D->hasAttrs() &&
       !D->isImplicit() &&
       !D->isUsed(false) &&
-      D->getPCHLevel() == 0 &&
       !D->hasExtInfo() &&
       D->RedeclLink.getNext() == D &&
       !D->isInvalidDecl() &&
       !D->isReferenced() &&
       D->getAccess() == AS_none &&
+      !D->isModulePrivate() &&
       !CXXRecordDecl::classofKind(D->getKind()) &&
       !D->getIntegerTypeSourceInfo() &&
       D->getDeclName().getNameKind() == DeclarationName::Identifier)
@@ -251,12 +252,12 @@ void ASTDeclWriter::VisitRecordDecl(RecordDecl *D) {
   if (!D->hasAttrs() &&
       !D->isImplicit() &&
       !D->isUsed(false) &&
-      D->getPCHLevel() == 0 &&
       !D->hasExtInfo() &&
       D->RedeclLink.getNext() == D &&
       !D->isInvalidDecl() &&
       !D->isReferenced() &&
       D->getAccess() == AS_none &&
+      !D->isModulePrivate() &&
       !CXXRecordDecl::classofKind(D->getKind()) &&
       D->getDeclName().getNameKind() == DeclarationName::Identifier)
     AbbrevToUse = Writer.getDeclRecordAbbrev();
@@ -295,8 +296,7 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   Record.push_back(D->getIdentifierNamespace());
   Record.push_back(D->getTemplatedKind());
   switch (D->getTemplatedKind()) {
-  default: assert(false && "Unhandled TemplatedKind!");
-    break;
+  default: llvm_unreachable("Unhandled TemplatedKind!");
   case FunctionDecl::TK_NonTemplate:
     break;
   case FunctionDecl::TK_FunctionTemplate:
@@ -321,13 +321,14 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
     // Template args as written.
     Record.push_back(FTSInfo->TemplateArgumentsAsWritten != 0);
     if (FTSInfo->TemplateArgumentsAsWritten) {
-      Record.push_back(FTSInfo->TemplateArgumentsAsWritten->size());
-      for (int i=0, e = FTSInfo->TemplateArgumentsAsWritten->size(); i!=e; ++i)
+      Record.push_back(FTSInfo->TemplateArgumentsAsWritten->NumTemplateArgs);
+      for (int i=0, e = FTSInfo->TemplateArgumentsAsWritten->NumTemplateArgs;
+             i!=e; ++i)
         Writer.AddTemplateArgumentLoc((*FTSInfo->TemplateArgumentsAsWritten)[i],
                                       Record);
-      Writer.AddSourceLocation(FTSInfo->TemplateArgumentsAsWritten->getLAngleLoc(),
+      Writer.AddSourceLocation(FTSInfo->TemplateArgumentsAsWritten->LAngleLoc,
                                Record);
-      Writer.AddSourceLocation(FTSInfo->TemplateArgumentsAsWritten->getRAngleLoc(),
+      Writer.AddSourceLocation(FTSInfo->TemplateArgumentsAsWritten->RAngleLoc,
                                Record);
     }
     
@@ -375,6 +376,7 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   Record.push_back(D->isDefaulted());
   Record.push_back(D->isExplicitlyDefaulted());
   Record.push_back(D->hasImplicitReturnZero());
+  Record.push_back(D->isConstexpr());
   Writer.AddSourceLocation(D->getLocEnd(), Record);
 
   Record.push_back(D->param_size());
@@ -400,12 +402,19 @@ void ASTDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   Record.push_back(D->isVariadic());
   Record.push_back(D->isSynthesized());
   Record.push_back(D->isDefined());
+
+  Record.push_back(D->IsRedeclaration);
+  Record.push_back(D->HasRedeclaration);
+  if (D->HasRedeclaration) {
+    assert(Context.getObjCMethodRedeclaration(D));
+    Writer.AddDeclRef(Context.getObjCMethodRedeclaration(D), Record);
+  }
+
   // FIXME: stable encoding for @required/@optional
   Record.push_back(D->getImplementationControl());
   // FIXME: stable encoding for in/out/inout/bycopy/byref/oneway
   Record.push_back(D->getObjCDeclQualifier());
   Record.push_back(D->hasRelatedResultType());
-  Record.push_back(D->getNumSelectorArgs());
   Writer.AddTypeRef(D->getResultType(), Record);
   Writer.AddTypeSourceInfo(D->getResultTypeSourceInfo(), Record);
   Writer.AddSourceLocation(D->getLocEnd(), Record);
@@ -413,11 +422,20 @@ void ASTDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   for (ObjCMethodDecl::param_iterator P = D->param_begin(),
                                    PEnd = D->param_end(); P != PEnd; ++P)
     Writer.AddDeclRef(*P, Record);
+
+  Record.push_back(D->SelLocsKind);
+  unsigned NumStoredSelLocs = D->getNumStoredSelLocs();
+  SourceLocation *SelLocs = D->getStoredSelLocs();
+  Record.push_back(NumStoredSelLocs);
+  for (unsigned i = 0; i != NumStoredSelLocs; ++i)
+    Writer.AddSourceLocation(SelLocs[i], Record);
+
   Code = serialization::DECL_OBJC_METHOD;
 }
 
 void ASTDeclWriter::VisitObjCContainerDecl(ObjCContainerDecl *D) {
   VisitNamedDecl(D);
+  Writer.AddSourceLocation(D->getAtStartLoc(), Record);
   Writer.AddSourceRange(D->getAtEndRange(), Record);
   // Abstract class (no need to define a stable serialization::DECL code).
 }
@@ -454,7 +472,6 @@ void ASTDeclWriter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
   Writer.AddDeclRef(D->getCategoryList(), Record);
   Record.push_back(D->isForwardDecl());
   Record.push_back(D->isImplicitInterfaceDecl());
-  Writer.AddSourceLocation(D->getClassLoc(), Record);
   Writer.AddSourceLocation(D->getSuperClassLoc(), Record);
   Writer.AddSourceLocation(D->getLocEnd(), Record);
   Code = serialization::DECL_OBJC_INTERFACE;
@@ -471,7 +488,7 @@ void ASTDeclWriter::VisitObjCIvarDecl(ObjCIvarDecl *D) {
       !D->isUsed(false) &&
       !D->isInvalidDecl() &&
       !D->isReferenced() &&
-      D->getPCHLevel() == 0 &&
+      !D->isModulePrivate() &&
       !D->getBitWidth() &&
       !D->hasExtInfo() &&
       D->getDeclName())
@@ -502,11 +519,8 @@ void ASTDeclWriter::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *D) {
 
 void ASTDeclWriter::VisitObjCClassDecl(ObjCClassDecl *D) {
   VisitDecl(D);
-  Record.push_back(D->size());
-  for (ObjCClassDecl::iterator I = D->begin(), IEnd = D->end(); I != IEnd; ++I)
-    Writer.AddDeclRef(I->getInterface(), Record);
-  for (ObjCClassDecl::iterator I = D->begin(), IEnd = D->end(); I != IEnd; ++I)
-    Writer.AddSourceLocation(I->getLocation(), Record);
+  Writer.AddDeclRef(D->getForwardInterfaceDecl(), Record);
+  Writer.AddSourceLocation(D->getForwardDecl()->getLocation(), Record);
   Code = serialization::DECL_OBJC_CLASS;
 }
 
@@ -536,7 +550,6 @@ void ASTDeclWriter::VisitObjCCategoryDecl(ObjCCategoryDecl *D) {
     Writer.AddSourceLocation(*PL, Record);
   Writer.AddDeclRef(D->getNextClassCategory(), Record);
   Record.push_back(D->hasSynthBitfield());
-  Writer.AddSourceLocation(D->getAtLoc(), Record);
   Writer.AddSourceLocation(D->getCategoryNameLoc(), Record);
   Code = serialization::DECL_OBJC_CATEGORY;
 }
@@ -612,7 +625,7 @@ void ASTDeclWriter::VisitFieldDecl(FieldDecl *D) {
       !D->isUsed(false) &&
       !D->isInvalidDecl() &&
       !D->isReferenced() &&
-      D->getPCHLevel() == 0 &&
+      !D->isModulePrivate() &&
       !D->getBitWidth() &&
       !D->hasInClassInitializer() &&
       !D->hasExtInfo() &&
@@ -665,7 +678,7 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
       !D->isInvalidDecl() &&
       !D->isReferenced() &&
       D->getAccess() == AS_none &&
-      D->getPCHLevel() == 0 &&
+      !D->isModulePrivate() &&
       D->getDeclName().getNameKind() == DeclarationName::Identifier &&
       !D->hasExtInfo() &&
       D->RedeclLink.getNext() == D &&
@@ -706,7 +719,7 @@ void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) {
       !D->isImplicit() &&
       !D->isUsed(false) &&
       D->getAccess() == AS_none &&
-      D->getPCHLevel() == 0 &&
+      !D->isModulePrivate() &&
       D->getStorageClass() == 0 &&
       !D->hasCXXDirectInitializer() && // Can params have this ever?
       D->getFunctionScopeDepth() == 0 &&
@@ -793,7 +806,7 @@ void ASTDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
   Code = serialization::DECL_NAMESPACE;
 
   if (Writer.hasChain() && !D->isOriginalNamespace() &&
-      D->getOriginalNamespace()->getPCHLevel() > 0) {
+      D->getOriginalNamespace()->isFromASTFile()) {
     NamespaceDecl *NS = D->getOriginalNamespace();
     Writer.AddUpdatedDeclContext(NS);
 
@@ -819,7 +832,7 @@ void ASTDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
     // anonymous namespace.
     Decl *Parent = cast<Decl>(
         D->getParent()->getRedeclContext()->getPrimaryContext());
-    if (Parent->getPCHLevel() > 0) {
+    if (Parent->isFromASTFile() || isa<TranslationUnitDecl>(Parent)) {
       ASTWriter::UpdateRecord &Record = Writer.DeclUpdates[Parent];
       Record.push_back(UPD_CXX_ADDED_ANONYMOUS_NAMESPACE);
       Writer.AddDeclRef(D, Record);
@@ -909,7 +922,7 @@ void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
 
   // Store the key function to avoid deserializing every method so we can
   // compute it.
-  if (D->IsDefinition)
+  if (D->IsCompleteDefinition)
     Writer.AddDeclRef(Context.getKeyFunction(D), Record);
 
   Code = serialization::DECL_CXX_RECORD;
@@ -1015,7 +1028,7 @@ void ASTDeclWriter::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
     // in a chained PCH, keep track of the association with the map so we can
     // update the first decl during AST reading.
     if (First->getMostRecentDeclaration() == D &&
-        First->getPCHLevel() > D->getPCHLevel()) {
+        First->isFromASTFile() && !D->isFromASTFile()) {
       assert(Writer.FirstLatestDecls.find(First)==Writer.FirstLatestDecls.end()
              && "The latest is already set");
       Writer.FirstLatestDecls[First] = D;
@@ -1058,16 +1071,12 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
   llvm::PointerUnion<ClassTemplateDecl *,
                      ClassTemplatePartialSpecializationDecl *> InstFrom
     = D->getSpecializedTemplateOrPartial();
-  Decl *InstFromD;
-  if (InstFrom.is<ClassTemplateDecl *>()) {
-    InstFromD = InstFrom.get<ClassTemplateDecl *>();
+  if (Decl *InstFromD = InstFrom.dyn_cast<ClassTemplateDecl *>()) {
     Writer.AddDeclRef(InstFromD, Record);
   } else {
-    InstFromD = InstFrom.get<ClassTemplatePartialSpecializationDecl *>();
-    Writer.AddDeclRef(InstFromD, Record);
+    Writer.AddDeclRef(InstFrom.get<ClassTemplatePartialSpecializationDecl *>(),
+                      Record);
     Writer.AddTemplateArgumentList(&D->getTemplateInstantiationArgs(), Record);
-    InstFromD = cast<ClassTemplatePartialSpecializationDecl>(InstFromD)->
-                    getSpecializedTemplate();
   }
 
   // Explicit info.
@@ -1110,6 +1119,14 @@ void ASTDeclWriter::VisitClassTemplatePartialSpecializationDecl(
   Code = serialization::DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION;
 }
 
+void ASTDeclWriter::VisitClassScopeFunctionSpecializationDecl(
+                                    ClassScopeFunctionSpecializationDecl *D) {
+  VisitDecl(D);
+  Writer.AddDeclRef(D->getSpecialization(), Record);
+  Code = serialization::DECL_CLASS_SCOPE_FUNCTION_SPECIALIZATION;
+}
+
+
 void ASTDeclWriter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   VisitRedeclarableTemplateDecl(D);
 
@@ -1233,7 +1250,7 @@ void ASTDeclWriter::VisitRedeclarable(Redeclarable<T> *D) {
   // in a chained PCH, keep track of the association with the map so we can
   // update the first decl during AST reading.
   if (ThisDecl != First && First->getMostRecentDeclaration() == ThisDecl &&
-      First->getPCHLevel() > ThisDecl->getPCHLevel()) {
+      First->isFromASTFile() && !ThisDecl->isFromASTFile()) {
     assert(Writer.FirstLatestDecls.find(First) == Writer.FirstLatestDecls.end()
            && "The latest is already set");
     Writer.FirstLatestDecls[First] = ThisDecl;
@@ -1262,7 +1279,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2));  // AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1293,7 +1310,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2));  // AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1327,7 +1344,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(AS_none));                 // C++ AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1339,8 +1356,9 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   // TagDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // IdentifierNamespace
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // getTagKind
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDefinition
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isCompleteDefinition
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // EmbeddedInDeclarator
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsFreeStanding
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // SourceLocation
   Abv->Add(BitCodeAbbrevOp(0));                         // hasExtInfo
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // TypedefNameAnonDecl
@@ -1372,7 +1390,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(AS_none));                 // C++ AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1384,8 +1402,9 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   // TagDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // IdentifierNamespace
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // getTagKind
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDefinition
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isCompleteDefinition
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // EmbeddedInDeclarator
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsFreeStanding
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // SourceLocation
   Abv->Add(BitCodeAbbrevOp(0));                         // hasExtInfo
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // TypedefNameAnonDecl
@@ -1411,7 +1430,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(AS_none));                 // C++ AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1459,7 +1478,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(AS_none));                 // C++ AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1484,7 +1503,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isUsed
   Abv->Add(BitCodeAbbrevOp(0));                       // isReferenced
   Abv->Add(BitCodeAbbrevOp(AS_none));                 // C++ AccessSpecifier
-  Abv->Add(BitCodeAbbrevOp(0));                       // PCH level
+  Abv->Add(BitCodeAbbrevOp(0));                       // ModulePrivate
   // NamedDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // NameKind = Identifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name
@@ -1527,6 +1546,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HasQualifier
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //GetDeclFound
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //ExplicitTemplateArgs
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HadMultipleCandidates
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclRef
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location
   DeclRefExprAbbrev = Stream.EmitAbbrev(Abv);
@@ -1592,8 +1612,11 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
 /// relatively painless since they would presumably only do it for top-level
 /// decls.
 static bool isRequiredDecl(const Decl *D, ASTContext &Context) {
+  // An ObjCMethodDecl is never considered as "required" because its
+  // implementation container always is.
+
   // File scoped assembly or obj-c implementation must be seen.
-  if (isa<FileScopeAsmDecl>(D) || isa<ObjCImplementationDecl>(D))
+  if (isa<FileScopeAsmDecl>(D) || isa<ObjCImplDecl>(D))
     return true;
 
   return Context.DeclMustBeEmitted(D);
@@ -1648,7 +1671,7 @@ void ASTWriter::WriteDecl(ASTContext &Context, Decl *D) {
   if (DC) W.VisitDeclContext(DC, LexicalOffset, VisibleOffset);
 
   if (!W.Code)
-    llvm::report_fatal_error(llvm::StringRef("unexpected declaration kind '") +
+    llvm::report_fatal_error(StringRef("unexpected declaration kind '") +
                             D->getDeclKindName() + "'");
   Stream.EmitRecord(W.Code, Record, W.AbbrevToUse);
 
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTWriterStmt.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTWriterStmt.cpp
index 1d73ed447b4f..7e2d45c5e7f9 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -36,7 +36,7 @@ namespace clang {
       : Writer(Writer), Record(Record) { }
     
     void
-    AddExplicitTemplateArgumentList(const ExplicitTemplateArgumentList &Args);
+    AddExplicitTemplateArgumentList(const ASTTemplateArgumentListInfo &Args);
 
     void VisitStmt(Stmt *S);
 #define STMT(Type, Base) \
@@ -46,7 +46,7 @@ namespace clang {
 }
 
 void ASTStmtWriter::
-AddExplicitTemplateArgumentList(const ExplicitTemplateArgumentList &Args) {
+AddExplicitTemplateArgumentList(const ASTTemplateArgumentListInfo &Args) {
   Writer.AddSourceLocation(Args.LAngleLoc, Record);
   Writer.AddSourceLocation(Args.RAngleLoc, Record);
   for (unsigned i=0; i != Args.NumTemplateArgs; ++i)
@@ -59,7 +59,7 @@ void ASTStmtWriter::VisitStmt(Stmt *S) {
 void ASTStmtWriter::VisitNullStmt(NullStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getSemiLoc(), Record);
-  Writer.AddSourceLocation(S->LeadingEmptyMacro, Record);
+  Record.push_back(S->HasLeadingEmptyMacro);
   Code = serialization::STMT_NULL;
 }
 
@@ -265,6 +265,7 @@ void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) {
   Record.push_back(E->hasQualifier());
   Record.push_back(E->getDecl() != E->getFoundDecl());
   Record.push_back(E->hasExplicitTemplateArgs());
+  Record.push_back(E->hadMultipleCandidates());
 
   if (E->hasExplicitTemplateArgs()) {
     unsigned NumTemplateArgs = E->getNumTemplateArgs();
@@ -324,7 +325,7 @@ void ASTStmtWriter::VisitStringLiteral(StringLiteral *E) {
   VisitExpr(E);
   Record.push_back(E->getByteLength());
   Record.push_back(E->getNumConcatenated());
-  Record.push_back(E->isWide());
+  Record.push_back(E->getKind());
   Record.push_back(E->isPascal());
   // FIXME: String data should be stored as a blob at the end of the
   // StringLiteral. However, we can't do so now because we have no
@@ -340,7 +341,7 @@ void ASTStmtWriter::VisitCharacterLiteral(CharacterLiteral *E) {
   VisitExpr(E);
   Record.push_back(E->getValue());
   Writer.AddSourceLocation(E->getLocation(), Record);
-  Record.push_back(E->isWide());
+  Record.push_back(E->getKind());
 
   AbbrevToUse = Writer.getCharacterLiteralAbbrev();
 
@@ -457,7 +458,9 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) {
     for (unsigned i=0; i != NumTemplateArgs; ++i)
       Writer.AddTemplateArgumentLoc(E->getTemplateArgs()[i], Record);
   }
-  
+
+  Record.push_back(E->hadMultipleCandidates());
+
   DeclAccessPair FoundDecl = E->getFoundDecl();
   Writer.AddDeclRef(FoundDecl.getDecl(), Record);
   Record.push_back(FoundDecl.getAccess());
@@ -733,6 +736,21 @@ void ASTStmtWriter::VisitGenericSelectionExpr(GenericSelectionExpr *E) {
   Code = serialization::EXPR_GENERIC_SELECTION;
 }
 
+void ASTStmtWriter::VisitAtomicExpr(AtomicExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->getOp());
+  Writer.AddStmt(E->getPtr());
+  Writer.AddStmt(E->getOrder());
+  if (E->getOp() != AtomicExpr::Load)
+    Writer.AddStmt(E->getVal1());
+  if (E->isCmpXChg()) {
+    Writer.AddStmt(E->getOrderFail());
+    Writer.AddStmt(E->getVal2());
+  }
+  Writer.AddSourceLocation(E->getBuiltinLoc(), Record);
+  Writer.AddSourceLocation(E->getRParenLoc(), Record);
+}
+
 //===----------------------------------------------------------------------===//
 // Objective-C Expressions and Statements.
 //===----------------------------------------------------------------------===//
@@ -806,6 +824,8 @@ void ASTStmtWriter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
 void ASTStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
+  Record.push_back(E->getNumStoredSelLocs());
+  Record.push_back(E->SelLocsKind);
   Record.push_back(E->isDelegateInitCall());
   Record.push_back((unsigned)E->getReceiverKind()); // FIXME: stable encoding
   switch (E->getReceiverKind()) {
@@ -834,11 +854,15 @@ void ASTStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
     
   Writer.AddSourceLocation(E->getLeftLoc(), Record);
   Writer.AddSourceLocation(E->getRightLoc(), Record);
-  Writer.AddSourceLocation(E->getSelectorLoc(), Record);
 
   for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg)
     Writer.AddStmt(*Arg);
+
+  SourceLocation *Locs = E->getStoredSelLocs();
+  for (unsigned i = 0, e = E->getNumStoredSelLocs(); i != e; ++i)
+    Writer.AddSourceLocation(Locs[i], Record);
+
   Code = serialization::EXPR_OBJC_MESSAGE_EXPR;
 }
 
@@ -952,6 +976,7 @@ void ASTStmtWriter::VisitCXXConstructExpr(CXXConstructExpr *E) {
   Writer.AddDeclRef(E->getConstructor(), Record);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Record.push_back(E->isElidable());
+  Record.push_back(E->hadMultipleCandidates());
   Record.push_back(E->requiresZeroInitialization());
   Record.push_back(E->getConstructionKind()); // FIXME: stable encoding
   Writer.AddSourceRange(E->getParenRange(), Record);
@@ -1071,6 +1096,7 @@ void ASTStmtWriter::VisitCXXNewExpr(CXXNewExpr *E) {
   Record.push_back(E->hasInitializer());
   Record.push_back(E->doesUsualArrayDeleteWantSize());
   Record.push_back(E->isArray());
+  Record.push_back(E->hadMultipleCandidates());
   Record.push_back(E->getNumPlacementArgs());
   Record.push_back(E->getNumConstructorArgs());
   Writer.AddDeclRef(E->getOperatorNew(), Record);
@@ -1142,7 +1168,7 @@ ASTStmtWriter::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
 
   Record.push_back(E->hasExplicitTemplateArgs());
   if (E->hasExplicitTemplateArgs()) {
-    const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
+    const ASTTemplateArgumentListInfo &Args = E->getExplicitTemplateArgs();
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
   }
@@ -1168,7 +1194,7 @@ ASTStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
   // emitted first.
   Record.push_back(E->hasExplicitTemplateArgs());
   if (E->hasExplicitTemplateArgs()) {
-    const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
+    const ASTTemplateArgumentListInfo &Args = E->getExplicitTemplateArgs();
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
   }
@@ -1197,7 +1223,7 @@ void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
   // Don't emit anything here, hasExplicitTemplateArgs() must be emitted first.
   Record.push_back(E->hasExplicitTemplateArgs());
   if (E->hasExplicitTemplateArgs()) {
-    const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
+    const ASTTemplateArgumentListInfo &Args = E->getExplicitTemplateArgs();
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
   }
@@ -1428,7 +1454,7 @@ void ASTWriter::WriteSubStmt(Stmt *S) {
   }
 
   // Redirect ASTWriter::AddStmt to collect sub stmts.
-  llvm::SmallVector<Stmt *, 16> SubStmts;
+  SmallVector<Stmt *, 16> SubStmts;
   CollectedStmts = &SubStmts;
 
   Writer.Code = serialization::STMT_NULL_PTR;
@@ -1440,7 +1466,7 @@ void ASTWriter::WriteSubStmt(Stmt *S) {
     SourceManager &SrcMgr
       = DeclIDs.begin()->first->getASTContext().getSourceManager();
     S->dump(SrcMgr);
-    assert(0 && "Unhandled sub statement writing AST file");
+    llvm_unreachable("Unhandled sub statement writing AST file");
   }
 #endif
 
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ChainedIncludesSource.cpp b/contrib/llvm/tools/clang/lib/Serialization/ChainedIncludesSource.cpp
index 3b7cd23b92ae..5fcf11a36e3c 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ChainedIncludesSource.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ChainedIncludesSource.cpp
@@ -26,17 +26,20 @@
 using namespace clang;
 
 static ASTReader *createASTReader(CompilerInstance &CI,
-                                  llvm::StringRef pchFile,
-                                  llvm::MemoryBuffer **memBufs,
-                                  unsigned numBufs,
+                                  StringRef pchFile,  
+                                  SmallVector<llvm::MemoryBuffer *, 4> &memBufs,
+                                  SmallVector<std::string, 4> &bufNames,
                              ASTDeserializationListener *deserialListener = 0) {
   Preprocessor &PP = CI.getPreprocessor();
   llvm::OwningPtr<ASTReader> Reader;
-  Reader.reset(new ASTReader(PP, &CI.getASTContext(), /*isysroot=*/0,
+  Reader.reset(new ASTReader(PP, CI.getASTContext(), /*isysroot=*/"",
                              /*DisableValidation=*/true));
-  Reader->setASTMemoryBuffers(memBufs, numBufs);
+  for (unsigned ti = 0; ti < bufNames.size(); ++ti) {
+    StringRef sr(bufNames[ti]);
+    Reader->addInMemoryBuffer(sr, memBufs[ti]);
+  }
   Reader->setDeserializationListener(deserialListener);
-  switch (Reader->ReadAST(pchFile, ASTReader::PCH)) {
+  switch (Reader->ReadAST(pchFile, serialization::MK_PCH)) {
   case ASTReader::Success:
     // Set the predefines buffer as suggested by the PCH reader.
     PP.setPredefines(Reader->getSuggestedPredefines());
@@ -62,7 +65,8 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
   llvm::OwningPtr<ChainedIncludesSource> source(new ChainedIncludesSource());
   InputKind IK = CI.getFrontendOpts().Inputs[0].first;
 
-  llvm::SmallVector<llvm::MemoryBuffer *, 4> serialBufs;
+  SmallVector<llvm::MemoryBuffer *, 4> serialBufs;
+  SmallVector<std::string, 4> serialBufNames;
 
   for (unsigned i = 0, e = includes.size(); i != e; ++i) {
     bool firstInclude = (i == 0);
@@ -83,8 +87,8 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
     TextDiagnosticPrinter *DiagClient =
       new TextDiagnosticPrinter(llvm::errs(), DiagnosticOptions());
     llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-    llvm::IntrusiveRefCntPtr<Diagnostic> Diags(new Diagnostic(DiagID,
-                                                              DiagClient));
+    llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+        new DiagnosticsEngine(DiagID, DiagClient));
 
     llvm::OwningPtr<CompilerInstance> Clang(new CompilerInstance());
     Clang->setInvocation(CInvok.take());
@@ -98,16 +102,15 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
                                                  &Clang->getPreprocessor());
     Clang->createASTContext();
 
-    llvm::SmallVector<char, 256> serialAST;
+    SmallVector<char, 256> serialAST;
     llvm::raw_svector_ostream OS(serialAST);
     llvm::OwningPtr<ASTConsumer> consumer;
     consumer.reset(new PCHGenerator(Clang->getPreprocessor(), "-",
-                                    /*Chaining=*/!firstInclude,
-                                    /*isysroot=*/0, &OS));
+                                    /*IsModule=*/false, /*isysroot=*/"", &OS));
     Clang->getASTContext().setASTMutationListener(
                                             consumer->GetASTMutationListener());
     Clang->setASTConsumer(consumer.take());
-    Clang->createSema(/*CompleteTranslationUnit=*/false, 0);
+    Clang->createSema(TU_Prefix, 0);
 
     if (firstInclude) {
       Preprocessor &PP = Clang->getPreprocessor();
@@ -115,19 +118,23 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
                                              PP.getLangOptions());
     } else {
       assert(!serialBufs.empty());
-      llvm::SmallVector<llvm::MemoryBuffer *, 4> bufs;
+      SmallVector<llvm::MemoryBuffer *, 4> bufs;
       for (unsigned si = 0, se = serialBufs.size(); si != se; ++si) {
         bufs.push_back(llvm::MemoryBuffer::getMemBufferCopy(
-                             llvm::StringRef(serialBufs[si]->getBufferStart(),
+                             StringRef(serialBufs[si]->getBufferStart(),
                                              serialBufs[si]->getBufferSize())));
       }
       std::string pchName = includes[i-1];
       llvm::raw_string_ostream os(pchName);
       os << ".pch" << i-1;
       os.flush();
+      
+      serialBufNames.push_back(pchName);
+
       llvm::OwningPtr<ExternalASTSource> Reader;
-      Reader.reset(createASTReader(*Clang, pchName, bufs.data(), bufs.size(),
-                      Clang->getASTConsumer().GetASTDeserializationListener()));
+
+      Reader.reset(createASTReader(*Clang, pchName, bufs, serialBufNames, 
+        Clang->getASTConsumer().GetASTDeserializationListener()));
       if (!Reader)
         return 0;
       Clang->getASTContext().setExternalSource(Reader);
@@ -140,16 +147,16 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
     OS.flush();
     Clang->getDiagnosticClient().EndSourceFile();
     serialBufs.push_back(
-      llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(serialAST.data(),
+      llvm::MemoryBuffer::getMemBufferCopy(StringRef(serialAST.data(),
                                                            serialAST.size())));
     source->CIs.push_back(Clang.take());
   }
 
   assert(!serialBufs.empty());
   std::string pchName = includes.back() + ".pch-final";
+  serialBufNames.push_back(pchName);
   llvm::OwningPtr<ASTReader> Reader;
-  Reader.reset(createASTReader(CI, pchName,
-                               serialBufs.data(), serialBufs.size()));
+  Reader.reset(createASTReader(CI, pchName, serialBufs, serialBufNames));
   if (!Reader)
     return 0;
 
@@ -182,13 +189,10 @@ ChainedIncludesSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
                                                       DeclarationName Name) {
   return getFinalReader().FindExternalVisibleDeclsByName(DC, Name);
 }
-void ChainedIncludesSource::MaterializeVisibleDecls(const DeclContext *DC) {
-  return getFinalReader().MaterializeVisibleDecls(DC);
-}
 ExternalLoadResult 
 ChainedIncludesSource::FindExternalLexicalDecls(const DeclContext *DC,
                                       bool (*isKindWeWant)(Decl::Kind),
-                                      llvm::SmallVectorImpl<Decl*> &Result) {
+                                      SmallVectorImpl<Decl*> &Result) {
   return getFinalReader().FindExternalLexicalDecls(DC, isKindWeWant, Result);
 }
 void ChainedIncludesSource::CompleteType(TagDecl *Tag) {
diff --git a/contrib/llvm/tools/clang/lib/Serialization/GeneratePCH.cpp b/contrib/llvm/tools/clang/lib/Serialization/GeneratePCH.cpp
index b8833ceacc7e..a2534db82cdd 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/GeneratePCH.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/GeneratePCH.cpp
@@ -27,31 +27,29 @@
 using namespace clang;
 
 PCHGenerator::PCHGenerator(const Preprocessor &PP,
-                           const std::string &OutputFile,
-                           bool Chaining,
-                           const char *isysroot,
-                           llvm::raw_ostream *OS)
-  : PP(PP), OutputFile(OutputFile), isysroot(isysroot), Out(OS), SemaPtr(0),
-    StatCalls(0), Stream(Buffer), Writer(Stream), Chaining(Chaining) {
+                           StringRef OutputFile,
+                           bool IsModule,
+                           StringRef isysroot,
+                           raw_ostream *OS)
+  : PP(PP), OutputFile(OutputFile), IsModule(IsModule), 
+    isysroot(isysroot.str()), Out(OS), 
+    SemaPtr(0), StatCalls(0), Stream(Buffer), Writer(Stream) {
   // Install a stat() listener to keep track of all of the stat()
   // calls.
   StatCalls = new MemorizeStatCalls();
-  // If we have a chain, we want new stat calls only, so install the memorizer
-  // *after* the already installed ASTReader's stat cache.
-  PP.getFileManager().addStatCache(StatCalls,
-    /*AtBeginning=*/!Chaining);
+  PP.getFileManager().addStatCache(StatCalls, /*AtBeginning=*/false);
+}
+
+PCHGenerator::~PCHGenerator() {
 }
 
 void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   if (PP.getDiagnostics().hasErrorOccurred())
     return;
-
-  // Set up the serialization listener.
-  Writer.SetSerializationListener(GetASTSerializationListener());
   
   // Emit the PCH file
   assert(SemaPtr && "No Sema?");
-  Writer.WriteAST(*SemaPtr, StatCalls, OutputFile, isysroot);
+  Writer.WriteAST(*SemaPtr, StatCalls, OutputFile, IsModule, isysroot);
 
   // Write the generated bitstream to "Out".
   Out->write((char *)&Buffer.front(), Buffer.size());
@@ -64,13 +62,7 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
 }
 
 ASTMutationListener *PCHGenerator::GetASTMutationListener() {
-  if (Chaining)
-    return &Writer;
-  return 0;
-}
-
-ASTSerializationListener *PCHGenerator::GetASTSerializationListener() {
-  return 0;
+  return &Writer;
 }
 
 ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() {
diff --git a/contrib/llvm/tools/clang/lib/Serialization/Module.cpp b/contrib/llvm/tools/clang/lib/Serialization/Module.cpp
new file mode 100644
index 000000000000..0a721c4365af
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Serialization/Module.cpp
@@ -0,0 +1,109 @@
+//===--- Module.cpp - Module description ------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the Module class, which describes a module that has
+//  been loaded from an AST file.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Serialization/Module.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "ASTReaderInternals.h"
+
+using namespace clang;
+using namespace serialization;
+using namespace reader;
+
+Module::Module(ModuleKind Kind)
+  : Kind(Kind), DirectlyImported(false), SizeInBits(0), 
+    LocalNumSLocEntries(0), SLocEntryBaseID(0),
+    SLocEntryBaseOffset(0), SLocEntryOffsets(0),
+    SLocFileOffsets(0), LocalNumIdentifiers(0), 
+    IdentifierOffsets(0), BaseIdentifierID(0), IdentifierTableData(0),
+    IdentifierLookupTable(0), BasePreprocessedEntityID(0),
+    PreprocessedEntityOffsets(0), NumPreprocessedEntities(0),
+    LocalNumHeaderFileInfos(0), 
+    HeaderFileInfoTableData(0), HeaderFileInfoTable(0),
+    HeaderFileFrameworkStrings(0),
+    LocalNumSelectors(0), SelectorOffsets(0), BaseSelectorID(0),
+    SelectorLookupTableData(0), SelectorLookupTable(0), LocalNumDecls(0),
+    DeclOffsets(0), BaseDeclID(0),
+    LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0),
+    LocalNumTypes(0), TypeOffsets(0), BaseTypeIndex(0), StatCache(0)
+{}
+
+Module::~Module() {
+  for (DeclContextInfosMap::iterator I = DeclContextInfos.begin(),
+       E = DeclContextInfos.end();
+       I != E; ++I) {
+    if (I->second.NameLookupTableData)
+      delete static_cast<ASTDeclContextNameLookupTable*>(
+                                                 I->second.NameLookupTableData);
+  }
+  
+  delete static_cast<ASTIdentifierLookupTable *>(IdentifierLookupTable);
+  delete static_cast<HeaderFileInfoLookupTable *>(HeaderFileInfoTable);
+  delete static_cast<ASTSelectorLookupTable *>(SelectorLookupTable);
+}
+
+template<typename Key, typename Offset, unsigned InitialCapacity>
+static void 
+dumpLocalRemap(StringRef Name,
+               const ContinuousRangeMap<Key, Offset, InitialCapacity> &Map) {
+  if (Map.begin() == Map.end())
+    return;
+  
+  typedef ContinuousRangeMap<Key, Offset, InitialCapacity> MapType;
+  llvm::errs() << "  " << Name << ":\n";
+  for (typename MapType::const_iterator I = Map.begin(), IEnd = Map.end(); 
+       I != IEnd; ++I) {
+    llvm::errs() << "    " << I->first << " -> " << I->second << "\n";
+  }
+}
+
+void Module::dump() {
+  llvm::errs() << "\nModule: " << FileName << "\n";
+  if (!Imports.empty()) {
+    llvm::errs() << "  Imports: ";
+    for (unsigned I = 0, N = Imports.size(); I != N; ++I) {
+      if (I)
+        llvm::errs() << ", ";
+      llvm::errs() << Imports[I]->FileName;
+    }
+    llvm::errs() << "\n";
+  }
+  
+  // Remapping tables.
+  llvm::errs() << "  Base source location offset: " << SLocEntryBaseOffset 
+               << '\n';
+  dumpLocalRemap("Source location offset local -> global map", SLocRemap);
+  
+  llvm::errs() << "  Base identifier ID: " << BaseIdentifierID << '\n'
+               << "  Number of identifiers: " << LocalNumIdentifiers << '\n';
+  dumpLocalRemap("Identifier ID local -> global map", IdentifierRemap);
+  
+  llvm::errs() << "  Base selector ID: " << BaseSelectorID << '\n'
+               << "  Number of selectors: " << LocalNumSelectors << '\n';
+  dumpLocalRemap("Selector ID local -> global map", SelectorRemap);
+  
+  llvm::errs() << "  Base preprocessed entity ID: " << BasePreprocessedEntityID
+               << '\n'  
+               << "  Number of preprocessed entities: " 
+               << NumPreprocessedEntities << '\n';
+  dumpLocalRemap("Preprocessed entity ID local -> global map", 
+                 PreprocessedEntityRemap);
+  
+  llvm::errs() << "  Base type index: " << BaseTypeIndex << '\n'
+               << "  Number of types: " << LocalNumTypes << '\n';
+  dumpLocalRemap("Type index local -> global map", TypeRemap);
+  
+  llvm::errs() << "  Base decl ID: " << BaseDeclID << '\n'
+               << "  Number of decls: " << LocalNumDecls << '\n';
+  dumpLocalRemap("Decl ID local -> global map", DeclRemap);
+}
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ModuleManager.cpp b/contrib/llvm/tools/clang/lib/Serialization/ModuleManager.cpp
new file mode 100644
index 000000000000..c4b1f7199bed
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/Serialization/ModuleManager.cpp
@@ -0,0 +1,253 @@
+//===--- ModuleManager.cpp - Module Manager ---------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ModuleManager class, which manages a set of loaded
+//  modules for the ASTReader.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Serialization/ModuleManager.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/system_error.h"
+
+#ifndef NDEBUG
+#include "llvm/Support/GraphWriter.h"
+#endif
+
+using namespace clang;
+using namespace serialization;
+
+Module *ModuleManager::lookup(StringRef Name) {
+  const FileEntry *Entry = FileMgr.getFile(Name);
+  return Modules[Entry];
+}
+
+llvm::MemoryBuffer *ModuleManager::lookupBuffer(StringRef Name) {
+  const FileEntry *Entry = FileMgr.getFile(Name);
+  return InMemoryBuffers[Entry];
+}
+
+std::pair<Module *, bool>
+ModuleManager::addModule(StringRef FileName, ModuleKind Type, 
+                         Module *ImportedBy, std::string &ErrorStr) {
+  const FileEntry *Entry = FileMgr.getFile(FileName);
+  if (!Entry && FileName != "-") {
+    ErrorStr = "file not found";
+    return std::make_pair(static_cast<Module*>(0), false);
+  }
+  
+  // Check whether we already loaded this module, before 
+  Module *&ModuleEntry = Modules[Entry];
+  bool NewModule = false;
+  if (!ModuleEntry) {
+    // Allocate a new module.
+    Module *New = new Module(Type);
+    New->FileName = FileName.str();
+    Chain.push_back(New);
+    NewModule = true;
+    ModuleEntry = New;
+    
+    // Load the contents of the module
+    if (llvm::MemoryBuffer *Buffer = lookupBuffer(FileName)) {
+      // The buffer was already provided for us.
+      assert(Buffer && "Passed null buffer");
+      New->Buffer.reset(Buffer);
+    } else {
+      // Open the AST file.
+      llvm::error_code ec;
+      if (FileName == "-") {
+        ec = llvm::MemoryBuffer::getSTDIN(New->Buffer);
+        if (ec)
+          ErrorStr = ec.message();
+      } else
+        New->Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrorStr));
+      
+      if (!New->Buffer)
+        return std::make_pair(static_cast<Module*>(0), false);
+    }
+    
+    // Initialize the stream
+    New->StreamFile.init((const unsigned char *)New->Buffer->getBufferStart(),
+                         (const unsigned char *)New->Buffer->getBufferEnd());     }
+  
+  if (ImportedBy) {
+    ModuleEntry->ImportedBy.insert(ImportedBy);
+    ImportedBy->Imports.insert(ModuleEntry);
+  } else {
+    ModuleEntry->DirectlyImported = true;
+  }
+  
+  return std::make_pair(ModuleEntry, NewModule);
+}
+
+void ModuleManager::addInMemoryBuffer(StringRef FileName, 
+                                      llvm::MemoryBuffer *Buffer) {
+  
+  const FileEntry *Entry = FileMgr.getVirtualFile(FileName, 
+                                                  Buffer->getBufferSize(), 0);
+  InMemoryBuffers[Entry] = Buffer;
+}
+
+ModuleManager::ModuleManager(const FileSystemOptions &FSO) : FileMgr(FSO) { }
+
+ModuleManager::~ModuleManager() {
+  for (unsigned i = 0, e = Chain.size(); i != e; ++i)
+    delete Chain[e - i - 1];
+}
+
+void ModuleManager::visit(bool (*Visitor)(Module &M, void *UserData), 
+                          void *UserData) {
+  unsigned N = size();
+  
+  // Record the number of incoming edges for each module. When we
+  // encounter a module with no incoming edges, push it into the queue
+  // to seed the queue.
+  SmallVector<Module *, 4> Queue;
+  Queue.reserve(N);
+  llvm::DenseMap<Module *, unsigned> UnusedIncomingEdges; 
+  for (ModuleIterator M = begin(), MEnd = end(); M != MEnd; ++M) {
+    if (unsigned Size = (*M)->ImportedBy.size())
+      UnusedIncomingEdges[*M] = Size;
+    else
+      Queue.push_back(*M);
+  }
+  
+  llvm::SmallPtrSet<Module *, 4> Skipped;
+  unsigned QueueStart = 0;
+  while (QueueStart < Queue.size()) {
+    Module *CurrentModule = Queue[QueueStart++];
+    
+    // Check whether this module should be skipped.
+    if (Skipped.count(CurrentModule))
+      continue;
+    
+    if (Visitor(*CurrentModule, UserData)) {
+      // The visitor has requested that cut off visitation of any
+      // module that the current module depends on. To indicate this
+      // behavior, we mark all of the reachable modules as having N
+      // incoming edges (which is impossible otherwise).
+      SmallVector<Module *, 4> Stack;
+      Stack.push_back(CurrentModule);
+      Skipped.insert(CurrentModule);
+      while (!Stack.empty()) {
+        Module *NextModule = Stack.back();
+        Stack.pop_back();
+        
+        // For any module that this module depends on, push it on the
+        // stack (if it hasn't already been marked as visited).
+        for (llvm::SetVector<Module *>::iterator 
+             M = NextModule->Imports.begin(),
+             MEnd = NextModule->Imports.end();
+             M != MEnd; ++M) {
+          if (Skipped.insert(*M))
+            Stack.push_back(*M);
+        }
+      }
+      continue;
+    }
+    
+    // For any module that this module depends on, push it on the
+    // stack (if it hasn't already been marked as visited).
+    for (llvm::SetVector<Module *>::iterator M = CurrentModule->Imports.begin(),
+         MEnd = CurrentModule->Imports.end();
+         M != MEnd; ++M) {
+      
+      // Remove our current module as an impediment to visiting the
+      // module we depend on. If we were the last unvisited module
+      // that depends on this particular module, push it into the
+      // queue to be visited.
+      unsigned &NumUnusedEdges = UnusedIncomingEdges[*M];
+      if (NumUnusedEdges && (--NumUnusedEdges == 0))
+        Queue.push_back(*M);
+    }
+  }
+}
+
+/// \brief Perform a depth-first visit of the current module.
+static bool visitDepthFirst(Module &M, 
+                            bool (*Visitor)(Module &M, bool Preorder, 
+                                            void *UserData), 
+                            void *UserData,
+                            llvm::SmallPtrSet<Module *, 4> &Visited) {
+  // Preorder visitation
+  if (Visitor(M, /*Preorder=*/true, UserData))
+    return true;
+  
+  // Visit children
+  for (llvm::SetVector<Module *>::iterator IM = M.Imports.begin(),
+       IMEnd = M.Imports.end();
+       IM != IMEnd; ++IM) {
+    if (!Visited.insert(*IM))
+      continue;
+    
+    if (visitDepthFirst(**IM, Visitor, UserData, Visited))
+      return true;
+  }  
+  
+  // Postorder visitation
+  return Visitor(M, /*Preorder=*/false, UserData);
+}
+
+void ModuleManager::visitDepthFirst(bool (*Visitor)(Module &M, bool Preorder, 
+                                                    void *UserData), 
+                                    void *UserData) {
+  llvm::SmallPtrSet<Module *, 4> Visited;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (!Visited.insert(Chain[I]))
+      continue;
+    
+    if (::visitDepthFirst(*Chain[I], Visitor, UserData, Visited))
+      return;
+  }
+}
+
+#ifndef NDEBUG
+namespace llvm {
+  template<>
+  struct GraphTraits<ModuleManager> {
+    typedef Module NodeType;
+    typedef llvm::SetVector<Module *>::const_iterator ChildIteratorType;
+    typedef ModuleManager::ModuleConstIterator nodes_iterator;
+    
+    static ChildIteratorType child_begin(NodeType *Node) {
+      return Node->Imports.begin();
+    }
+
+    static ChildIteratorType child_end(NodeType *Node) {
+      return Node->Imports.end();
+    }
+    
+    static nodes_iterator nodes_begin(const ModuleManager &Manager) {
+      return Manager.begin();
+    }
+    
+    static nodes_iterator nodes_end(const ModuleManager &Manager) {
+      return Manager.end();
+    }
+  };
+  
+  template<>
+  struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
+    explicit DOTGraphTraits(bool IsSimple = false)
+      : DefaultDOTGraphTraits(IsSimple) { }
+    
+    static bool renderGraphFromBottomUp() {
+      return true;
+    }
+
+    std::string getNodeLabel(Module *M, const ModuleManager&) {
+      return llvm::sys::path::stem(M->FileName);
+    }
+  };
+}
+
+void ModuleManager::viewGraph() {
+  llvm::ViewGraph(*this, "Modules");
+}
+#endif
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/CMakeLists.txt b/contrib/llvm/tools/clang/lib/StaticAnalyzer/CMakeLists.txt
deleted file mode 100644
index 3d1509254f52..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-add_subdirectory(Core)
-add_subdirectory(Checkers)
-add_subdirectory(Frontend)
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
index 8fc6d2a2933e..dc524ba24e06 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
@@ -37,7 +37,7 @@ void AdjustedReturnValueChecker::checkPostStmt(const CallExpr *CE,
   QualType expectedResultTy = CE->getType();
 
   // Fetch the signature of the called function.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   SVal V = state->getSVal(CE);
   
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
index 983427afb62d..cd977bf54c67 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
@@ -80,7 +80,7 @@ void AnalyzerStatsChecker::checkEndAnalysis(ExplodedGraph &G,
 
     if (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)) {
       const NamedDecl *ND = cast<NamedDecl>(D);
-      output << ND;
+      output << *ND;
     }
     else if (isa<BlockDecl>(D)) {
       output << "block(line:" << Loc.getLine() << ":col:" << Loc.getColumn();
@@ -94,7 +94,7 @@ void AnalyzerStatsChecker::checkEndAnalysis(ExplodedGraph &G,
       << (Eng.hasEmptyWorkList() ? "yes" : "no");
 
   B.EmitBasicReport("Analyzer Statistics", "Internal Statistics", output.str(),
-      D->getLocation());
+      PathDiagnosticLocation(D, SM));
 
   // Emit warning for each block we bailed out on
   typedef CoreEngine::BlocksExhausted::const_iterator ExhaustedIterator;
@@ -106,7 +106,8 @@ void AnalyzerStatsChecker::checkEndAnalysis(ExplodedGraph &G,
     const CFGElement &CE = Exit->front();
     if (const CFGStmt *CS = dyn_cast<CFGStmt>(&CE))
       B.EmitBasicReport("Bailout Point", "Internal Statistics", "The analyzer "
-          "stopped analyzing at this point", CS->getStmt()->getLocStart());
+          "stopped analyzing at this point",
+          PathDiagnosticLocation::createBegin(CS->getStmt(), SM, LC));
   }
 }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
index eb9665a4a343..6935c5f1c192 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
@@ -27,11 +27,12 @@ class ArrayBoundChecker :
     public Checker<check::Location> {
   mutable llvm::OwningPtr<BuiltinBug> BT;
 public:
-  void checkLocation(SVal l, bool isLoad, CheckerContext &C) const;
+  void checkLocation(SVal l, bool isLoad, const Stmt* S,
+                     CheckerContext &C) const;
 };
 }
 
-void ArrayBoundChecker::checkLocation(SVal l, bool isLoad,
+void ArrayBoundChecker::checkLocation(SVal l, bool isLoad, const Stmt* LoadS,
                                       CheckerContext &C) const {
   // Check for out of bound array element access.
   const MemRegion *R = l.getAsRegion();
@@ -50,15 +51,15 @@ void ArrayBoundChecker::checkLocation(SVal l, bool isLoad,
   if (Idx.isZeroConstant())
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   // Get the size of the array.
   DefinedOrUnknownSVal NumElements 
     = C.getStoreManager().getSizeInElements(state, ER->getSuperRegion(), 
                                             ER->getValueType());
 
-  const GRState *StInBound = state->assumeInBound(Idx, NumElements, true);
-  const GRState *StOutBound = state->assumeInBound(Idx, NumElements, false);
+  const ProgramState *StInBound = state->assumeInBound(Idx, NumElements, true);
+  const ProgramState *StOutBound = state->assumeInBound(Idx, NumElements, false);
   if (StOutBound && !StInBound) {
     ExplodedNode *N = C.generateSink(StOutBound);
     if (!N)
@@ -73,10 +74,10 @@ void ArrayBoundChecker::checkLocation(SVal l, bool isLoad,
     // reference is outside the range.
 
     // Generate a report for this bug.
-    RangedBugReport *report = 
-      new RangedBugReport(*BT, BT->getDescription(), N);
+    BugReport *report = 
+      new BugReport(*BT, BT->getDescription(), N);
 
-    report->addRange(C.getStmt()->getSourceRange());
+    report->addRange(LoadS->getSourceRange());
     C.EmitReport(report);
     return;
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
index 65a6e633dc8e..6175028a9b20 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
@@ -30,11 +30,12 @@ class ArrayBoundCheckerV2 :
       
   enum OOB_Kind { OOB_Precedes, OOB_Excedes };
   
-  void reportOOB(CheckerContext &C, const GRState *errorState,
+  void reportOOB(CheckerContext &C, const ProgramState *errorState,
                  OOB_Kind kind) const;
       
 public:
-  void checkLocation(SVal l, bool isLoad, CheckerContext &C) const;
+  void checkLocation(SVal l, bool isLoad, const Stmt*S,
+                     CheckerContext &C) const;
 };
 
 // FIXME: Eventually replace RegionRawOffset with this class.
@@ -53,12 +54,12 @@ class RegionRawOffsetV2 {
   NonLoc getByteOffset() const { return cast<NonLoc>(byteOffset); }
   const SubRegion *getRegion() const { return baseRegion; }
   
-  static RegionRawOffsetV2 computeOffset(const GRState *state,
+  static RegionRawOffsetV2 computeOffset(const ProgramState *state,
                                          SValBuilder &svalBuilder,
                                          SVal location);
 
   void dump() const;
-  void dumpToStream(llvm::raw_ostream& os) const;
+  void dumpToStream(raw_ostream &os) const;
 };
 }
 
@@ -79,9 +80,10 @@ static SVal computeExtentBegin(SValBuilder &svalBuilder,
 }
 
 void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
+                                        const Stmt* LoadS,
                                         CheckerContext &checkerContext) const {
 
-  // NOTE: Instead of using GRState::assumeInBound(), we are prototyping
+  // NOTE: Instead of using ProgramState::assumeInBound(), we are prototyping
   // some new logic here that reasons directly about memory region extents.
   // Once that logic is more mature, we can bring it back to assumeInBound()
   // for all clients to use.
@@ -90,8 +92,8 @@ void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
   // memory access is within the extent of the base region.  Since we
   // have some flexibility in defining the base region, we can achieve
   // various levels of conservatism in our buffer overflow checking.
-  const GRState *state = checkerContext.getState();  
-  const GRState *originalState = state;
+  const ProgramState *state = checkerContext.getState();  
+  const ProgramState *originalState = state;
 
   SValBuilder &svalBuilder = checkerContext.getSValBuilder();
   const RegionRawOffsetV2 &rawOffset = 
@@ -116,7 +118,7 @@ void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
     if (!lowerBoundToCheck)
       return;
     
-    const GRState *state_precedesLowerBound, *state_withinLowerBound;
+    const ProgramState *state_precedesLowerBound, *state_withinLowerBound;
     llvm::tie(state_precedesLowerBound, state_withinLowerBound) =
       state->assume(*lowerBoundToCheck);
 
@@ -148,7 +150,7 @@ void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
     if (!upperboundToCheck)
       break;
   
-    const GRState *state_exceedsUpperBound, *state_withinUpperBound;
+    const ProgramState *state_exceedsUpperBound, *state_withinUpperBound;
     llvm::tie(state_exceedsUpperBound, state_withinUpperBound) =
       state->assume(*upperboundToCheck);
   
@@ -168,7 +170,7 @@ void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
 }
 
 void ArrayBoundCheckerV2::reportOOB(CheckerContext &checkerContext,
-                                    const GRState *errorState,
+                                    const ProgramState *errorState,
                                     OOB_Kind kind) const {
   
   ExplodedNode *errorNode = checkerContext.generateSink(errorState);
@@ -187,14 +189,14 @@ void ArrayBoundCheckerV2::reportOOB(CheckerContext &checkerContext,
      << (kind == OOB_Precedes ? "(accessed memory precedes memory block)"
                               : "(access exceeds upper limit of memory block)");
 
-  checkerContext.EmitReport(new RangedBugReport(*BT, os.str(), errorNode));
+  checkerContext.EmitReport(new BugReport(*BT, os.str(), errorNode));
 }
 
 void RegionRawOffsetV2::dump() const {
   dumpToStream(llvm::errs());
 }
 
-void RegionRawOffsetV2::dumpToStream(llvm::raw_ostream& os) const {
+void RegionRawOffsetV2::dumpToStream(raw_ostream &os) const {
   os << "raw_offset_v2{" << getRegion() << ',' << getByteOffset() << '}';
 }
 
@@ -219,7 +221,7 @@ static inline SVal getValue(SVal val, SValBuilder &svalBuilder) {
 
 // Scale a base value by a scaling factor, and return the scaled
 // value as an SVal.  Used by 'computeOffset'.
-static inline SVal scaleValue(const GRState *state,
+static inline SVal scaleValue(const ProgramState *state,
                               NonLoc baseVal, CharUnits scaling,
                               SValBuilder &sb) {
   return sb.evalBinOpNN(state, BO_Mul, baseVal,
@@ -229,7 +231,7 @@ static inline SVal scaleValue(const GRState *state,
 
 // Add an SVal to another, treating unknown and undefined values as
 // summing to UnknownVal.  Used by 'computeOffset'.
-static SVal addValue(const GRState *state, SVal x, SVal y,
+static SVal addValue(const ProgramState *state, SVal x, SVal y,
                      SValBuilder &svalBuilder) {
   // We treat UnknownVals and UndefinedVals the same here because we
   // only care about computing offsets.
@@ -243,7 +245,7 @@ static SVal addValue(const GRState *state, SVal x, SVal y,
 
 /// Compute a raw byte offset from a base region.  Used for array bounds
 /// checking.
-RegionRawOffsetV2 RegionRawOffsetV2::computeOffset(const GRState *state,
+RegionRawOffsetV2 RegionRawOffsetV2::computeOffset(const ProgramState *state,
                                                    SValBuilder &svalBuilder,
                                                    SVal location)
 {
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
index d88a111e9a56..8296eb93c5ae 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
@@ -33,12 +33,12 @@ class AttrNonNullChecker
 
 void AttrNonNullChecker::checkPreStmt(const CallExpr *CE,
                                       CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   // Check if the callee has a 'nonnull' attribute.
   SVal X = state->getSVal(CE->getCallee());
 
-  const FunctionDecl* FD = X.getAsFunctionDecl();
+  const FunctionDecl *FD = X.getAsFunctionDecl();
   if (!FD)
     return;
 
@@ -85,7 +85,7 @@ void AttrNonNullChecker::checkPreStmt(const CallExpr *CE,
     }
 
     ConstraintManager &CM = C.getConstraintManager();
-    const GRState *stateNotNull, *stateNull;
+    const ProgramState *stateNotNull, *stateNull;
     llvm::tie(stateNotNull, stateNull) = CM.assumeDual(state, *DV);
 
     if (stateNull && !stateNotNull) {
@@ -100,16 +100,15 @@ void AttrNonNullChecker::checkPreStmt(const CallExpr *CE,
           BT.reset(new BugType("Argument with 'nonnull' attribute passed null",
                                "API"));
 
-        EnhancedBugReport *R =
-          new EnhancedBugReport(*BT,
-                                "Null pointer passed as an argument to a "
-                                "'nonnull' parameter", errorNode);
+        BugReport *R =
+          new BugReport(*BT, "Null pointer passed as an argument to a "
+                             "'nonnull' parameter", errorNode);
 
         // Highlight the range of the argument that was null.
         const Expr *arg = *I;
         R->addRange(arg->getSourceRange());
-        R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, arg);
-
+        R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(errorNode,
+                                                                   arg));
         // Emit the bug report.
         C.EmitReport(R);
       }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
index 9fc8163ab8b9..08cff0fd4292 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
@@ -20,7 +20,8 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "clang/AST/DeclObjC.h"
@@ -49,7 +50,7 @@ static const char* GetReceiverNameType(const ObjCMessage &msg) {
 }
 
 static bool isReceiverClassOrSuperclass(const ObjCInterfaceDecl *ID,
-                                        llvm::StringRef ClassName) {
+                                        StringRef ClassName) {
   if (ID->getIdentifier()->getName() == ClassName)
     return true;
 
@@ -92,7 +93,7 @@ void NilArgChecker::WarnNilArg(CheckerContext &C,
     os << "Argument to '" << GetReceiverNameType(msg) << "' method '"
        << msg.getSelector().getAsString() << "' cannot be nil";
 
-    RangedBugReport *R = new RangedBugReport(*BT, os.str(), N);
+    BugReport *R = new BugReport(*BT, os.str(), N);
     R->addRange(msg.getArgSourceRange(Arg));
     C.EmitReport(R);
   }
@@ -114,7 +115,7 @@ void NilArgChecker::checkPreObjCMessage(ObjCMessage msg,
     //  lexical comparisons.
     
     std::string NameStr = S.getAsString();
-    llvm::StringRef Name(NameStr);
+    StringRef Name(NameStr);
     assert(!Name.empty());
     
     // FIXME: Checking for initWithFormat: will not work in most cases
@@ -148,7 +149,7 @@ class CFNumberCreateChecker : public Checker< check::PreStmt<CallExpr> > {
   void checkPreStmt(const CallExpr *CE, CheckerContext &C) const;
 
 private:
-  void EmitError(const TypedRegion* R, const Expr* Ex,
+  void EmitError(const TypedRegion* R, const Expr *Ex,
                 uint64_t SourceSize, uint64_t TargetSize, uint64_t NumberKind);
 };
 } // end anonymous namespace
@@ -194,7 +195,7 @@ namespace {
   };
 }
 
-static Optional<uint64_t> GetCFNumberSize(ASTContext& Ctx, uint64_t i) {
+static Optional<uint64_t> GetCFNumberSize(ASTContext &Ctx, uint64_t i) {
   static const unsigned char FixedSize[] = { 8, 16, 32, 64, 32, 64 };
 
   if (i < kCFNumberCharType)
@@ -248,10 +249,10 @@ static const char* GetCFNumberTypeStr(uint64_t i) {
 
 void CFNumberCreateChecker::checkPreStmt(const CallExpr *CE,
                                          CheckerContext &C) const {
-  const Expr* Callee = CE->getCallee();
-  const GRState *state = C.getState();
+  const Expr *Callee = CE->getCallee();
+  const ProgramState *state = C.getState();
   SVal CallV = state->getSVal(Callee);
-  const FunctionDecl* FD = CallV.getAsFunctionDecl();
+  const FunctionDecl *FD = CallV.getAsFunctionDecl();
 
   if (!FD)
     return;
@@ -290,7 +291,7 @@ void CFNumberCreateChecker::checkPreStmt(const CallExpr *CE,
   if (!LV)
     return;
 
-  const TypedRegion* R = dyn_cast<TypedRegion>(LV->stripCasts());
+  const TypedValueRegion* R = dyn_cast<TypedValueRegion>(LV->stripCasts());
   if (!R)
     return;
 
@@ -335,7 +336,7 @@ void CFNumberCreateChecker::checkPreStmt(const CallExpr *CE,
     if (!BT)
       BT.reset(new APIMisuse("Bad use of CFNumberCreate"));
     
-    RangedBugReport *report = new RangedBugReport(*BT, os.str(), N);
+    BugReport *report = new BugReport(*BT, os.str(), N);
     report->addRange(CE->getArg(2)->getSourceRange());
     C.EmitReport(report);
   }
@@ -351,21 +352,21 @@ class CFRetainReleaseChecker : public Checker< check::PreStmt<CallExpr> > {
   mutable IdentifierInfo *Retain, *Release;
 public:
   CFRetainReleaseChecker(): Retain(0), Release(0) {}
-  void checkPreStmt(const CallExpr* CE, CheckerContext& C) const;
+  void checkPreStmt(const CallExpr *CE, CheckerContext &C) const;
 };
 } // end anonymous namespace
 
 
-void CFRetainReleaseChecker::checkPreStmt(const CallExpr* CE,
-                                          CheckerContext& C) const {
+void CFRetainReleaseChecker::checkPreStmt(const CallExpr *CE,
+                                          CheckerContext &C) const {
   // If the CallExpr doesn't have exactly 1 argument just give up checking.
   if (CE->getNumArgs() != 1)
     return;
 
   // Get the function declaration of the callee.
-  const GRState* state = C.getState();
+  const ProgramState *state = C.getState();
   SVal X = state->getSVal(CE->getCallee());
-  const FunctionDecl* FD = X.getAsFunctionDecl();
+  const FunctionDecl *FD = X.getAsFunctionDecl();
 
   if (!FD)
     return;
@@ -400,7 +401,7 @@ void CFRetainReleaseChecker::checkPreStmt(const CallExpr* CE,
   DefinedOrUnknownSVal ArgIsNull = svalBuilder.evalEQ(state, zero, *DefArgVal);
 
   // Are they equal?
-  const GRState *stateTrue, *stateFalse;
+  const ProgramState *stateTrue, *stateFalse;
   llvm::tie(stateTrue, stateFalse) = state->assume(ArgIsNull);
 
   if (stateTrue && !stateFalse) {
@@ -412,9 +413,9 @@ void CFRetainReleaseChecker::checkPreStmt(const CallExpr* CE,
                             ? "Null pointer argument in call to CFRetain"
                             : "Null pointer argument in call to CFRelease";
 
-    EnhancedBugReport *report = new EnhancedBugReport(*BT, description, N);
+    BugReport *report = new BugReport(*BT, description, N);
     report->addRange(Arg->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Arg);
+    report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, Arg));
     C.EmitReport(report);
     return;
   }
@@ -471,7 +472,7 @@ void ClassReleaseChecker::checkPreObjCMessage(ObjCMessage msg,
           "of class '" << Class->getName()
        << "' and not the class directly";
   
-    RangedBugReport *report = new RangedBugReport(*BT, os.str(), N);
+    BugReport *report = new BugReport(*BT, os.str(), N);
     report->addRange(msg.getSourceRange());
     C.EmitReport(report);
   }
@@ -586,7 +587,7 @@ void VariadicMethodTypeChecker::checkPreObjCMessage(ObjCMessage msg,
 
   // Verify that all arguments have Objective-C types.
   llvm::Optional<ExplodedNode*> errorNode;
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   
   for (unsigned I = variadicArgsBegin; I != variadicArgsEnd; ++I) {
     QualType ArgTy = msg.getArgType(I);
@@ -629,7 +630,7 @@ void VariadicMethodTypeChecker::checkPreObjCMessage(ObjCMessage msg,
       << "' should be an Objective-C pointer type, not '" 
       << ArgTy.getAsString() << "'";
 
-    RangedBugReport *R = new RangedBugReport(*BT, os.str(),
+    BugReport *R = new BugReport(*BT, os.str(),
                                              errorNode.getValue());
     R->addRange(msg.getArgSourceRange(I));
     C.EmitReport(R);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 12ac652ba060..a57d03175c5f 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -31,7 +31,7 @@ class BuiltinFunctionChecker : public Checker<eval::Call> {
 
 bool BuiltinFunctionChecker::evalCall(const CallExpr *CE,
                                       CheckerContext &C) const{
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
   const FunctionDecl *FD = L.getAsFunctionDecl();
@@ -57,7 +57,7 @@ bool BuiltinFunctionChecker::evalCall(const CallExpr *CE,
     // FIXME: Refactor into StoreManager itself?
     MemRegionManager& RM = C.getStoreManager().getRegionManager();
     const AllocaRegion* R =
-      RM.getAllocaRegion(CE, C.getNodeBuilder().getCurrentBlockCount(),
+      RM.getAllocaRegion(CE, C.getCurrentBlockCount(),
                          C.getPredecessor()->getLocationContext());
 
     // Set the extent of the region in bytes. This enables us to use the
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
deleted file mode 100644
index e172a529d768..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-set(LLVM_TARGET_DEFINITIONS Checkers.td)
-tablegen(Checkers.inc
-         -gen-clang-sa-checkers
-         -I ${CMAKE_CURRENT_SOURCE_DIR}/../../../include)
-add_custom_target(ClangSACheckers
-  DEPENDS Checkers.inc)
-
-set(LLVM_USED_LIBS clangBasic clangAST)
-
-add_clang_library(clangStaticAnalyzerCheckers
-  AdjustedReturnValueChecker.cpp
-  AnalyzerStatsChecker.cpp
-  ArrayBoundChecker.cpp
-  ArrayBoundCheckerV2.cpp
-  AttrNonNullChecker.cpp
-  BasicObjCFoundationChecks.cpp
-  BuiltinFunctionChecker.cpp
-  CStringChecker.cpp
-  CallAndMessageChecker.cpp
-  CastSizeChecker.cpp
-  CastToStructChecker.cpp
-  CheckObjCDealloc.cpp
-  CheckObjCInstMethSignature.cpp
-  CheckSecuritySyntaxOnly.cpp
-  CheckSizeofPointer.cpp
-  ChrootChecker.cpp
-  ClangSACheckerProvider.cpp
-  DeadStoresChecker.cpp
-  DebugCheckers.cpp
-  DereferenceChecker.cpp
-  DivZeroChecker.cpp
-  ExprEngine.cpp
-  ExperimentalChecks.cpp
-  FixedAddressChecker.cpp
-  IdempotentOperationChecker.cpp
-  LLVMConventionsChecker.cpp
-  MacOSXAPIChecker.cpp
-  MallocChecker.cpp
-  NSAutoreleasePoolChecker.cpp
-  NSErrorChecker.cpp
-  NoReturnFunctionChecker.cpp
-  OSAtomicChecker.cpp
-  ObjCAtSyncChecker.cpp
-  ObjCSelfInitChecker.cpp
-  ObjCUnusedIVarsChecker.cpp
-  PointerArithChecker.cpp
-  PointerSubChecker.cpp
-  PthreadLockChecker.cpp
-  ReturnPointerRangeChecker.cpp
-  ReturnUndefChecker.cpp
-  StackAddrLeakChecker.cpp
-  StreamChecker.cpp
-  UndefBranchChecker.cpp
-  UndefCapturedBlockVarChecker.cpp
-  UndefResultChecker.cpp
-  UndefinedArraySubscriptChecker.cpp
-  UndefinedAssignmentChecker.cpp
-  UnixAPIChecker.cpp
-  UnreachableCodeChecker.cpp
-  VLASizeChecker.cpp
-  )
-
-add_dependencies(clangStaticAnalyzerCheckers
-  clangStaticAnalyzerCore
-  ClangAttrClasses
-  ClangAttrList
-  ClangDeclNodes
-  ClangStmtNodes
-  ClangSACheckers
-  )
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index c5dac5d21626..1625219fc877 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -1,4 +1,4 @@
-//= CStringChecker.h - Checks calls to C string functions ----------*- C++ -*-//
+//= CStringChecker.cpp - Checks calls to C string functions --------*- C++ -*-//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -17,7 +17,7 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace clang;
@@ -40,14 +40,15 @@ class CStringChecker : public Checker< eval::Call,
 
   bool evalCall(const CallExpr *CE, CheckerContext &C) const;
   void checkPreStmt(const DeclStmt *DS, CheckerContext &C) const;
-  void checkLiveSymbols(const GRState *state, SymbolReaper &SR) const;
+  void checkLiveSymbols(const ProgramState *state, SymbolReaper &SR) const;
   void checkDeadSymbols(SymbolReaper &SR, CheckerContext &C) const;
-  bool wantsRegionChangeUpdate(const GRState *state) const;
+  bool wantsRegionChangeUpdate(const ProgramState *state) const;
 
-  const GRState *checkRegionChanges(const GRState *state,
-                                    const StoreManager::InvalidatedSymbols *,
-                                    const MemRegion * const *Begin,
-                                    const MemRegion * const *End) const;
+  const ProgramState *
+    checkRegionChanges(const ProgramState *state,
+                       const StoreManager::InvalidatedSymbols *,
+                       ArrayRef<const MemRegion *> ExplicitRegions,
+                       ArrayRef<const MemRegion *> Regions) const;
 
   typedef void (CStringChecker::*FnCheck)(CheckerContext &,
                                           const CallExpr *) const;
@@ -57,8 +58,10 @@ class CStringChecker : public Checker< eval::Call,
   void evalMemmove(CheckerContext &C, const CallExpr *CE) const;
   void evalBcopy(CheckerContext &C, const CallExpr *CE) const;
   void evalCopyCommon(CheckerContext &C, const CallExpr *CE,
-                      const GRState *state,
-                      const Expr *Size, const Expr *Source, const Expr *Dest,
+                      const ProgramState *state,
+                      const Expr *Size,
+                      const Expr *Source,
+                      const Expr *Dest,
                       bool Restricted = false,
                       bool IsMempcpy = false) const;
 
@@ -66,14 +69,18 @@ class CStringChecker : public Checker< eval::Call,
 
   void evalstrLength(CheckerContext &C, const CallExpr *CE) const;
   void evalstrnLength(CheckerContext &C, const CallExpr *CE) const;
-  void evalstrLengthCommon(CheckerContext &C, const CallExpr *CE, 
+  void evalstrLengthCommon(CheckerContext &C,
+                           const CallExpr *CE, 
                            bool IsStrnlen = false) const;
 
   void evalStrcpy(CheckerContext &C, const CallExpr *CE) const;
   void evalStrncpy(CheckerContext &C, const CallExpr *CE) const;
   void evalStpcpy(CheckerContext &C, const CallExpr *CE) const;
-  void evalStrcpyCommon(CheckerContext &C, const CallExpr *CE, bool returnEnd,
-                        bool isBounded, bool isAppending) const;
+  void evalStrcpyCommon(CheckerContext &C,
+                        const CallExpr *CE,
+                        bool returnEnd,
+                        bool isBounded,
+                        bool isAppending) const;
 
   void evalStrcat(CheckerContext &C, const CallExpr *CE) const;
   void evalStrncat(CheckerContext &C, const CallExpr *CE) const;
@@ -82,64 +89,85 @@ class CStringChecker : public Checker< eval::Call,
   void evalStrncmp(CheckerContext &C, const CallExpr *CE) const;
   void evalStrcasecmp(CheckerContext &C, const CallExpr *CE) const;
   void evalStrncasecmp(CheckerContext &C, const CallExpr *CE) const;
-  void evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
-                        bool isBounded = false, bool ignoreCase = false) const;
+  void evalStrcmpCommon(CheckerContext &C,
+                        const CallExpr *CE,
+                        bool isBounded = false,
+                        bool ignoreCase = false) const;
 
   // Utility methods
-  std::pair<const GRState*, const GRState*>
+  std::pair<const ProgramState*, const ProgramState*>
   static assumeZero(CheckerContext &C,
-                    const GRState *state, SVal V, QualType Ty);
+                    const ProgramState *state, SVal V, QualType Ty);
 
-  static const GRState *setCStringLength(const GRState *state,
-                                         const MemRegion *MR, SVal strLength);
+  static const ProgramState *setCStringLength(const ProgramState *state,
+                                              const MemRegion *MR,
+                                              SVal strLength);
   static SVal getCStringLengthForRegion(CheckerContext &C,
-                                        const GRState *&state,
-                                        const Expr *Ex, const MemRegion *MR,
+                                        const ProgramState *&state,
+                                        const Expr *Ex,
+                                        const MemRegion *MR,
                                         bool hypothetical);
-  SVal getCStringLength(CheckerContext &C, const GRState *&state,
-                        const Expr *Ex, SVal Buf,
+  SVal getCStringLength(CheckerContext &C,
+                        const ProgramState *&state,
+                        const Expr *Ex,
+                        SVal Buf,
                         bool hypothetical = false) const;
 
   const StringLiteral *getCStringLiteral(CheckerContext &C, 
-                                         const GRState *&state,
+                                         const ProgramState *&state,
                                          const Expr *expr,  
                                          SVal val) const;
 
-  static const GRState *InvalidateBuffer(CheckerContext &C,
-                                         const GRState *state,
-                                         const Expr *Ex, SVal V);
+  static const ProgramState *InvalidateBuffer(CheckerContext &C,
+                                              const ProgramState *state,
+                                              const Expr *Ex, SVal V);
 
-  static bool SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
+  static bool SummarizeRegion(raw_ostream &os, ASTContext &Ctx,
                               const MemRegion *MR);
 
   // Re-usable checks
-  const GRState *checkNonNull(CheckerContext &C, const GRState *state,
-                               const Expr *S, SVal l) const;
-  const GRState *CheckLocation(CheckerContext &C, const GRState *state,
-                               const Expr *S, SVal l,
-                               const char *message = NULL) const;
-  const GRState *CheckBufferAccess(CheckerContext &C, const GRState *state,
-                                   const Expr *Size,
-                                   const Expr *FirstBuf,
-                                   const Expr *SecondBuf,
-                                   const char *firstMessage = NULL,
-                                   const char *secondMessage = NULL,
-                                   bool WarnAboutSize = false) const;
-  const GRState *CheckBufferAccess(CheckerContext &C, const GRState *state,
-                                   const Expr *Size, const Expr *Buf,
-                                   const char *message = NULL,
-                                   bool WarnAboutSize = false) const {
+  const ProgramState *checkNonNull(CheckerContext &C,
+                                   const ProgramState *state,
+                                   const Expr *S,
+                                   SVal l) const;
+  const ProgramState *CheckLocation(CheckerContext &C,
+                                    const ProgramState *state,
+                                    const Expr *S,
+                                    SVal l,
+                                    const char *message = NULL) const;
+  const ProgramState *CheckBufferAccess(CheckerContext &C,
+                                        const ProgramState *state,
+                                        const Expr *Size,
+                                        const Expr *FirstBuf,
+                                        const Expr *SecondBuf,
+                                        const char *firstMessage = NULL,
+                                        const char *secondMessage = NULL,
+                                        bool WarnAboutSize = false) const;
+
+  const ProgramState *CheckBufferAccess(CheckerContext &C,
+                                        const ProgramState *state,
+                                        const Expr *Size,
+                                        const Expr *Buf,
+                                        const char *message = NULL,
+                                        bool WarnAboutSize = false) const {
     // This is a convenience override.
     return CheckBufferAccess(C, state, Size, Buf, NULL, message, NULL,
                              WarnAboutSize);
   }
-  const GRState *CheckOverlap(CheckerContext &C, const GRState *state,
-                              const Expr *Size, const Expr *First,
-                              const Expr *Second) const;
-  void emitOverlapBug(CheckerContext &C, const GRState *state,
-                      const Stmt *First, const Stmt *Second) const;
-  const GRState *checkAdditionOverflow(CheckerContext &C, const GRState *state,
-                                       NonLoc left, NonLoc right) const;
+  const ProgramState *CheckOverlap(CheckerContext &C,
+                                   const ProgramState *state,
+                                   const Expr *Size,
+                                   const Expr *First,
+                                   const Expr *Second) const;
+  void emitOverlapBug(CheckerContext &C,
+                      const ProgramState *state,
+                      const Stmt *First,
+                      const Stmt *Second) const;
+
+  const ProgramState *checkAdditionOverflow(CheckerContext &C,
+                                            const ProgramState *state,
+                                            NonLoc left,
+                                            NonLoc right) const;
 };
 
 class CStringLength {
@@ -151,8 +179,8 @@ class CStringLength {
 namespace clang {
 namespace ento {
   template <>
-  struct GRStateTrait<CStringLength> 
-    : public GRStatePartialTrait<CStringLength::EntryMap> {
+  struct ProgramStateTrait<CStringLength> 
+    : public ProgramStatePartialTrait<CStringLength::EntryMap> {
     static void *GDMIndex() { return CStringChecker::getTag(); }
   };
 }
@@ -162,26 +190,26 @@ namespace ento {
 // Individual checks and utility methods.
 //===----------------------------------------------------------------------===//
 
-std::pair<const GRState*, const GRState*>
-CStringChecker::assumeZero(CheckerContext &C, const GRState *state, SVal V,
+std::pair<const ProgramState*, const ProgramState*>
+CStringChecker::assumeZero(CheckerContext &C, const ProgramState *state, SVal V,
                            QualType Ty) {
   DefinedSVal *val = dyn_cast<DefinedSVal>(&V);
   if (!val)
-    return std::pair<const GRState*, const GRState *>(state, state);
+    return std::pair<const ProgramState*, const ProgramState *>(state, state);
 
   SValBuilder &svalBuilder = C.getSValBuilder();
   DefinedOrUnknownSVal zero = svalBuilder.makeZeroVal(Ty);
   return state->assume(svalBuilder.evalEQ(state, *val, zero));
 }
 
-const GRState *CStringChecker::checkNonNull(CheckerContext &C,
-                                            const GRState *state,
+const ProgramState *CStringChecker::checkNonNull(CheckerContext &C,
+                                            const ProgramState *state,
                                             const Expr *S, SVal l) const {
   // If a previous check has failed, propagate the failure.
   if (!state)
     return NULL;
 
-  const GRState *stateNull, *stateNonNull;
+  const ProgramState *stateNull, *stateNonNull;
   llvm::tie(stateNull, stateNonNull) = assumeZero(C, state, l, S->getType());
 
   if (stateNull && !stateNonNull) {
@@ -200,10 +228,10 @@ const GRState *CStringChecker::checkNonNull(CheckerContext &C,
 
     // Generate a report for this bug.
     BuiltinBug *BT = static_cast<BuiltinBug*>(BT_Null.get());
-    EnhancedBugReport *report = new EnhancedBugReport(*BT, os.str(), N);
+    BugReport *report = new BugReport(*BT, os.str(), N);
 
     report->addRange(S->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, S);
+    report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, S));
     C.EmitReport(report);
     return NULL;
   }
@@ -214,8 +242,8 @@ const GRState *CStringChecker::checkNonNull(CheckerContext &C,
 }
 
 // FIXME: This was originally copied from ArrayBoundChecker.cpp. Refactor?
-const GRState *CStringChecker::CheckLocation(CheckerContext &C,
-                                             const GRState *state,
+const ProgramState *CStringChecker::CheckLocation(CheckerContext &C,
+                                             const ProgramState *state,
                                              const Expr *S, SVal l,
                                              const char *warningMsg) const {
   // If a previous check has failed, propagate the failure.
@@ -244,8 +272,8 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
   // Get the index of the accessed element.
   DefinedOrUnknownSVal Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
 
-  const GRState *StInBound = state->assumeInBound(Idx, Size, true);
-  const GRState *StOutBound = state->assumeInBound(Idx, Size, false);
+  const ProgramState *StInBound = state->assumeInBound(Idx, Size, true);
+  const ProgramState *StOutBound = state->assumeInBound(Idx, Size, false);
   if (StOutBound && !StInBound) {
     ExplodedNode *N = C.generateSink(StOutBound);
     if (!N)
@@ -258,9 +286,9 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
     BuiltinBug *BT = static_cast<BuiltinBug*>(BT_Bounds.get());
 
     // Generate a report for this bug.
-    RangedBugReport *report;
+    BugReport *report;
     if (warningMsg) {
-      report = new RangedBugReport(*BT, warningMsg, N);
+      report = new BugReport(*BT, warningMsg, N);
     } else {
       assert(CurrentFunctionDescription);
       assert(CurrentFunctionDescription[0] != '\0');
@@ -270,7 +298,7 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
       os << (char)toupper(CurrentFunctionDescription[0])
          << &CurrentFunctionDescription[1]
          << " accesses out-of-bound array element";
-      report = new RangedBugReport(*BT, os.str(), N);      
+      report = new BugReport(*BT, os.str(), N);      
     }
 
     // FIXME: It would be nice to eventually make this diagnostic more clear,
@@ -287,8 +315,8 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
   return StInBound;
 }
 
-const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
-                                                 const GRState *state,
+const ProgramState *CStringChecker::CheckBufferAccess(CheckerContext &C,
+                                                 const ProgramState *state,
                                                  const Expr *Size,
                                                  const Expr *FirstBuf,
                                                  const Expr *SecondBuf,
@@ -359,8 +387,8 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
   return state;
 }
 
-const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
-                                            const GRState *state,
+const ProgramState *CStringChecker::CheckOverlap(CheckerContext &C,
+                                            const ProgramState *state,
                                             const Expr *Size,
                                             const Expr *First,
                                             const Expr *Second) const {
@@ -372,7 +400,7 @@ const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
   if (!state)
     return NULL;
 
-  const GRState *stateTrue, *stateFalse;
+  const ProgramState *stateTrue, *stateFalse;
 
   // Get the buffer values and make sure they're known locations.
   SVal firstVal = state->getSVal(First);
@@ -470,7 +498,7 @@ const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
   return stateFalse;
 }
 
-void CStringChecker::emitOverlapBug(CheckerContext &C, const GRState *state,
+void CStringChecker::emitOverlapBug(CheckerContext &C, const ProgramState *state,
                                   const Stmt *First, const Stmt *Second) const {
   ExplodedNode *N = C.generateSink(state);
   if (!N)
@@ -480,8 +508,8 @@ void CStringChecker::emitOverlapBug(CheckerContext &C, const GRState *state,
     BT_Overlap.reset(new BugType("Unix API", "Improper arguments"));
 
   // Generate a report for this bug.
-  RangedBugReport *report = 
-    new RangedBugReport(*BT_Overlap,
+  BugReport *report = 
+    new BugReport(*BT_Overlap,
       "Arguments must not be overlapping buffers", N);
   report->addRange(First->getSourceRange());
   report->addRange(Second->getSourceRange());
@@ -489,8 +517,8 @@ void CStringChecker::emitOverlapBug(CheckerContext &C, const GRState *state,
   C.EmitReport(report);
 }
 
-const GRState *CStringChecker::checkAdditionOverflow(CheckerContext &C,
-                                                     const GRState *state,
+const ProgramState *CStringChecker::checkAdditionOverflow(CheckerContext &C,
+                                                     const ProgramState *state,
                                                      NonLoc left,
                                                      NonLoc right) const {
   // If a previous check has failed, propagate the failure.
@@ -521,7 +549,7 @@ const GRState *CStringChecker::checkAdditionOverflow(CheckerContext &C,
     SVal willOverflow = svalBuilder.evalBinOpNN(state, BO_GT, left,
                                                 *maxMinusRightNL, cmpTy);
 
-    const GRState *stateOverflow, *stateOkay;
+    const ProgramState *stateOverflow, *stateOkay;
     llvm::tie(stateOverflow, stateOkay) =
       state->assume(cast<DefinedOrUnknownSVal>(willOverflow));
 
@@ -557,7 +585,7 @@ const GRState *CStringChecker::checkAdditionOverflow(CheckerContext &C,
   return state;
 }
 
-const GRState *CStringChecker::setCStringLength(const GRState *state,
+const ProgramState *CStringChecker::setCStringLength(const ProgramState *state,
                                                 const MemRegion *MR,
                                                 SVal strLength) {
   assert(!strLength.isUndef() && "Attempt to set an undefined string length");
@@ -598,7 +626,7 @@ const GRState *CStringChecker::setCStringLength(const GRState *state,
 }
 
 SVal CStringChecker::getCStringLengthForRegion(CheckerContext &C,
-                                               const GRState *&state,
+                                               const ProgramState *&state,
                                                const Expr *Ex,
                                                const MemRegion *MR,
                                                bool hypothetical) {
@@ -610,7 +638,7 @@ SVal CStringChecker::getCStringLengthForRegion(CheckerContext &C,
   }
   
   // Otherwise, get a new symbol and update the state.
-  unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+  unsigned Count = C.getCurrentBlockCount();
   SValBuilder &svalBuilder = C.getSValBuilder();
   QualType sizeTy = svalBuilder.getContext().getSizeType();
   SVal strLength = svalBuilder.getMetadataSymbolVal(CStringChecker::getTag(),
@@ -622,7 +650,7 @@ SVal CStringChecker::getCStringLengthForRegion(CheckerContext &C,
   return strLength;
 }
 
-SVal CStringChecker::getCStringLength(CheckerContext &C, const GRState *&state,
+SVal CStringChecker::getCStringLength(CheckerContext &C, const ProgramState *&state,
                                       const Expr *Ex, SVal Buf,
                                       bool hypothetical) const {
   const MemRegion *MR = Buf.getAsRegion();
@@ -644,7 +672,7 @@ SVal CStringChecker::getCStringLength(CheckerContext &C, const GRState *&state,
            << "', which is not a null-terminated string";
 
         // Generate a report for this bug.
-        EnhancedBugReport *report = new EnhancedBugReport(*BT_NotCString,
+        BugReport *report = new BugReport(*BT_NotCString,
                                                           os.str(), N);
 
         report->addRange(Ex->getSourceRange());
@@ -705,7 +733,7 @@ SVal CStringChecker::getCStringLength(CheckerContext &C, const GRState *&state,
         os << "not a null-terminated string";
 
       // Generate a report for this bug.
-      EnhancedBugReport *report = new EnhancedBugReport(*BT_NotCString,
+      BugReport *report = new BugReport(*BT_NotCString,
                                                         os.str(), N);
 
       report->addRange(Ex->getSourceRange());
@@ -717,7 +745,7 @@ SVal CStringChecker::getCStringLength(CheckerContext &C, const GRState *&state,
 }
 
 const StringLiteral *CStringChecker::getCStringLiteral(CheckerContext &C,
-  const GRState *&state, const Expr *expr, SVal val) const {
+  const ProgramState *&state, const Expr *expr, SVal val) const {
 
   // Get the memory region pointed to by the val.
   const MemRegion *bufRegion = val.getAsRegion();
@@ -736,8 +764,8 @@ const StringLiteral *CStringChecker::getCStringLiteral(CheckerContext &C,
   return strRegion->getStringLiteral();
 }
 
-const GRState *CStringChecker::InvalidateBuffer(CheckerContext &C,
-                                                const GRState *state,
+const ProgramState *CStringChecker::InvalidateBuffer(CheckerContext &C,
+                                                const ProgramState *state,
                                                 const Expr *E, SVal V) {
   Loc *L = dyn_cast<Loc>(&V);
   if (!L)
@@ -757,8 +785,8 @@ const GRState *CStringChecker::InvalidateBuffer(CheckerContext &C,
     }
 
     // Invalidate this region.
-    unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-    return state->invalidateRegion(R, E, Count, NULL);
+    unsigned Count = C.getCurrentBlockCount();
+    return state->invalidateRegions(R, E, Count);
   }
 
   // If we have a non-region value by chance, just remove the binding.
@@ -767,17 +795,15 @@ const GRState *CStringChecker::InvalidateBuffer(CheckerContext &C,
   return state->unbindLoc(*L);
 }
 
-bool CStringChecker::SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
+bool CStringChecker::SummarizeRegion(raw_ostream &os, ASTContext &Ctx,
                                      const MemRegion *MR) {
-  const TypedRegion *TR = dyn_cast<TypedRegion>(MR);
-  if (!TR)
-    return false;
+  const TypedValueRegion *TVR = dyn_cast<TypedValueRegion>(MR);
 
-  switch (TR->getKind()) {
+  switch (MR->getKind()) {
   case MemRegion::FunctionTextRegionKind: {
-    const FunctionDecl *FD = cast<FunctionTextRegion>(TR)->getDecl();
+    const FunctionDecl *FD = cast<FunctionTextRegion>(MR)->getDecl();
     if (FD)
-      os << "the address of the function '" << FD << "'";
+      os << "the address of the function '" << *FD << '\'';
     else
       os << "the address of a function";
     return true;
@@ -790,16 +816,16 @@ bool CStringChecker::SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
     return true;
   case MemRegion::CXXThisRegionKind:
   case MemRegion::CXXTempObjectRegionKind:
-    os << "a C++ temp object of type " << TR->getValueType().getAsString();
+    os << "a C++ temp object of type " << TVR->getValueType().getAsString();
     return true;
   case MemRegion::VarRegionKind:
-    os << "a variable of type" << TR->getValueType().getAsString();
+    os << "a variable of type" << TVR->getValueType().getAsString();
     return true;
   case MemRegion::FieldRegionKind:
-    os << "a field of type " << TR->getValueType().getAsString();
+    os << "a field of type " << TVR->getValueType().getAsString();
     return true;
   case MemRegion::ObjCIvarRegionKind:
-    os << "an instance variable of type " << TR->getValueType().getAsString();
+    os << "an instance variable of type " << TVR->getValueType().getAsString();
     return true;
   default:
     return false;
@@ -812,7 +838,7 @@ bool CStringChecker::SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
 
 void CStringChecker::evalCopyCommon(CheckerContext &C, 
                                     const CallExpr *CE,
-                                    const GRState *state,
+                                    const ProgramState *state,
                                     const Expr *Size, const Expr *Dest,
                                     const Expr *Source, bool Restricted,
                                     bool IsMempcpy) const {
@@ -822,7 +848,7 @@ void CStringChecker::evalCopyCommon(CheckerContext &C,
   SVal sizeVal = state->getSVal(Size);
   QualType sizeTy = Size->getType();
 
-  const GRState *stateZeroSize, *stateNonZeroSize;
+  const ProgramState *stateZeroSize, *stateNonZeroSize;
   llvm::tie(stateZeroSize, stateNonZeroSize) =
     assumeZero(C, state, sizeVal, sizeTy);
 
@@ -887,7 +913,7 @@ void CStringChecker::evalCopyCommon(CheckerContext &C,
       } else {
         // If we don't know how much we copied, we can at least
         // conjure a return value for later.
-        unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+        unsigned Count = C.getCurrentBlockCount();
         SVal result =
           C.getSValBuilder().getConjuredSymbolVal(NULL, CE, Count);
         state = state->BindExpr(CE, result);
@@ -914,7 +940,7 @@ void CStringChecker::evalMemcpy(CheckerContext &C, const CallExpr *CE) const {
   // void *memcpy(void *restrict dst, const void *restrict src, size_t n);
   // The return value is the address of the destination buffer.
   const Expr *Dest = CE->getArg(0);
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   evalCopyCommon(C, CE, state, CE->getArg(2), Dest, CE->getArg(1), true);
 }
@@ -923,7 +949,7 @@ void CStringChecker::evalMempcpy(CheckerContext &C, const CallExpr *CE) const {
   // void *mempcpy(void *restrict dst, const void *restrict src, size_t n);
   // The return value is a pointer to the byte following the last written byte.
   const Expr *Dest = CE->getArg(0);
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   
   evalCopyCommon(C, CE, state, CE->getArg(2), Dest, CE->getArg(1), true, true);
 }
@@ -932,7 +958,7 @@ void CStringChecker::evalMemmove(CheckerContext &C, const CallExpr *CE) const {
   // void *memmove(void *dst, const void *src, size_t n);
   // The return value is the address of the destination buffer.
   const Expr *Dest = CE->getArg(0);
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   evalCopyCommon(C, CE, state, CE->getArg(2), Dest, CE->getArg(1));
 }
@@ -951,14 +977,14 @@ void CStringChecker::evalMemcmp(CheckerContext &C, const CallExpr *CE) const {
   const Expr *Right = CE->getArg(1);
   const Expr *Size = CE->getArg(2);
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SValBuilder &svalBuilder = C.getSValBuilder();
 
   // See if the size argument is zero.
   SVal sizeVal = state->getSVal(Size);
   QualType sizeTy = Size->getType();
 
-  const GRState *stateZeroSize, *stateNonZeroSize;
+  const ProgramState *stateZeroSize, *stateNonZeroSize;
   llvm::tie(stateZeroSize, stateNonZeroSize) =
     assumeZero(C, state, sizeVal, sizeTy);
 
@@ -981,7 +1007,7 @@ void CStringChecker::evalMemcmp(CheckerContext &C, const CallExpr *CE) const {
 
     // See if they are the same.
     DefinedOrUnknownSVal SameBuf = svalBuilder.evalEQ(state, LV, RV);
-    const GRState *StSameBuf, *StNotSameBuf;
+    const ProgramState *StSameBuf, *StNotSameBuf;
     llvm::tie(StSameBuf, StNotSameBuf) = state->assume(SameBuf);
 
     // If the two arguments might be the same buffer, we know the result is 0,
@@ -1002,7 +1028,7 @@ void CStringChecker::evalMemcmp(CheckerContext &C, const CallExpr *CE) const {
       state = CheckBufferAccess(C, state, Size, Left, Right);
       if (state) {
         // The return value is the comparison result, which we don't know.
-        unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+        unsigned Count = C.getCurrentBlockCount();
         SVal CmpV = svalBuilder.getConjuredSymbolVal(NULL, CE, Count);
         state = state->BindExpr(CE, CmpV);
         C.addTransition(state);
@@ -1026,13 +1052,13 @@ void CStringChecker::evalstrnLength(CheckerContext &C,
 void CStringChecker::evalstrLengthCommon(CheckerContext &C, const CallExpr *CE,
                                          bool IsStrnlen) const {
   CurrentFunctionDescription = "string length function";
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   if (IsStrnlen) {
     const Expr *maxlenExpr = CE->getArg(1);
     SVal maxlenVal = state->getSVal(maxlenExpr);
 
-    const GRState *stateZeroSize, *stateNonZeroSize;
+    const ProgramState *stateZeroSize, *stateNonZeroSize;
     llvm::tie(stateZeroSize, stateNonZeroSize) =
       assumeZero(C, state, maxlenVal, maxlenExpr->getType());
 
@@ -1084,7 +1110,7 @@ void CStringChecker::evalstrLengthCommon(CheckerContext &C, const CallExpr *CE,
     NonLoc *maxlenValNL = dyn_cast<NonLoc>(&maxlenVal);
 
     if (strLengthNL && maxlenValNL) {
-      const GRState *stateStringTooLong, *stateStringNotTooLong;
+      const ProgramState *stateStringTooLong, *stateStringNotTooLong;
 
       // Check if the strLength is greater than the maxlen.
       llvm::tie(stateStringTooLong, stateStringNotTooLong) =
@@ -1108,7 +1134,7 @@ void CStringChecker::evalstrLengthCommon(CheckerContext &C, const CallExpr *CE,
       // no guarantee the full string length will actually be returned.
       // All we know is the return value is the min of the string length
       // and the limit. This is better than nothing.
-      unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+      unsigned Count = C.getCurrentBlockCount();
       result = C.getSValBuilder().getConjuredSymbolVal(NULL, CE, Count);
       NonLoc *resultNL = cast<NonLoc>(&result);
 
@@ -1136,7 +1162,7 @@ void CStringChecker::evalstrLengthCommon(CheckerContext &C, const CallExpr *CE,
     // If we don't know the length of the string, conjure a return
     // value, so it can be used in constraints, at least.
     if (result.isUnknown()) {
-      unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+      unsigned Count = C.getCurrentBlockCount();
       result = C.getSValBuilder().getConjuredSymbolVal(NULL, CE, Count);
     }
   }
@@ -1191,7 +1217,7 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
                                       bool returnEnd, bool isBounded,
                                       bool isAppending) const {
   CurrentFunctionDescription = "string copy function";
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   // Check that the destination is non-null.
   const Expr *Dst = CE->getArg(0);
@@ -1241,7 +1267,7 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
     // If we know both values, we might be able to figure out how much
     // we're copying.
     if (strLengthNL && lenValNL) {
-      const GRState *stateSourceTooLong, *stateSourceNotTooLong;
+      const ProgramState *stateSourceTooLong, *stateSourceNotTooLong;
 
       // Check if the max number to copy is less than the length of the src.
       // If the bound is equal to the source length, strncpy won't null-
@@ -1480,7 +1506,7 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
   // If this is a stpcpy-style copy, but we were unable to check for a buffer
   // overflow, we still need a result. Conjure a return value.
   if (returnEnd && Result.isUnknown()) {
-    unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+    unsigned Count = C.getCurrentBlockCount();
     Result = svalBuilder.getConjuredSymbolVal(NULL, CE, Count);
   }
 
@@ -1514,7 +1540,7 @@ void CStringChecker::evalStrncasecmp(CheckerContext &C,
 void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
                                       bool isBounded, bool ignoreCase) const {
   CurrentFunctionDescription = "string comparison function";
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   // Check that the first string is non-null
   const Expr *s1 = CE->getArg(0);
@@ -1549,7 +1575,7 @@ void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
   // See if they are the same.
   SValBuilder &svalBuilder = C.getSValBuilder();
   DefinedOrUnknownSVal SameBuf = svalBuilder.evalEQ(state, LV, RV);
-  const GRState *StSameBuf, *StNotSameBuf;
+  const ProgramState *StSameBuf, *StNotSameBuf;
   llvm::tie(StSameBuf, StNotSameBuf) = state->assume(SameBuf);
 
   // If the two arguments might be the same buffer, we know the result is 0,
@@ -1575,8 +1601,8 @@ void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
   bool canComputeResult = false;
 
   if (s1StrLiteral && s2StrLiteral) {
-    llvm::StringRef s1StrRef = s1StrLiteral->getString();
-    llvm::StringRef s2StrRef = s2StrLiteral->getString();
+    StringRef s1StrRef = s1StrLiteral->getString();
+    StringRef s2StrRef = s2StrLiteral->getString();
 
     if (isBounded) {
       // Get the max number of characters to compare.
@@ -1598,11 +1624,11 @@ void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
     if (canComputeResult) {
       // Real strcmp stops at null characters.
       size_t s1Term = s1StrRef.find('\0');
-      if (s1Term != llvm::StringRef::npos)
+      if (s1Term != StringRef::npos)
         s1StrRef = s1StrRef.substr(0, s1Term);
 
       size_t s2Term = s2StrRef.find('\0');
-      if (s2Term != llvm::StringRef::npos)
+      if (s2Term != StringRef::npos)
         s2StrRef = s2StrRef.substr(0, s2Term);
 
       // Use StringRef's comparison methods to compute the actual result.
@@ -1624,7 +1650,7 @@ void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
 
   if (!canComputeResult) {
     // Conjure a symbolic value. It's the best we can do.
-    unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+    unsigned Count = C.getCurrentBlockCount();
     SVal resultVal = svalBuilder.getConjuredSymbolVal(NULL, CE, Count);
     state = state->BindExpr(CE, resultVal);
   }
@@ -1640,7 +1666,7 @@ void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
 bool CStringChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
   // Get the callee.  All the functions we care about are C functions
   // with simple identifiers.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   const FunctionDecl *FD = state->getSVal(Callee).getAsFunctionDecl();
 
@@ -1651,7 +1677,7 @@ bool CStringChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
   IdentifierInfo *II = FD->getIdentifier();
   if (!II)   // if no identifier, not a simple C function
     return false;
-  llvm::StringRef Name = II->getName();
+  StringRef Name = II->getName();
   if (Name.startswith("__builtin_"))
     Name = Name.substr(10);
 
@@ -1689,7 +1715,7 @@ bool CStringChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
 
 void CStringChecker::checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {
   // Record string length for char a[] = "abc";
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   for (DeclStmt::const_decl_iterator I = DS->decl_begin(), E = DS->decl_end();
        I != E; ++I) {
@@ -1723,16 +1749,16 @@ void CStringChecker::checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {
   C.addTransition(state);
 }
 
-bool CStringChecker::wantsRegionChangeUpdate(const GRState *state) const {
+bool CStringChecker::wantsRegionChangeUpdate(const ProgramState *state) const {
   CStringLength::EntryMap Entries = state->get<CStringLength>();
   return !Entries.isEmpty();
 }
 
-const GRState *
-CStringChecker::checkRegionChanges(const GRState *state,
+const ProgramState *
+CStringChecker::checkRegionChanges(const ProgramState *state,
                                    const StoreManager::InvalidatedSymbols *,
-                                   const MemRegion * const *Begin,
-                                   const MemRegion * const *End) const {
+                                   ArrayRef<const MemRegion *> ExplicitRegions,
+                                   ArrayRef<const MemRegion *> Regions) const {
   CStringLength::EntryMap Entries = state->get<CStringLength>();
   if (Entries.isEmpty())
     return state;
@@ -1741,8 +1767,9 @@ CStringChecker::checkRegionChanges(const GRState *state,
   llvm::SmallPtrSet<const MemRegion *, 32> SuperRegions;
 
   // First build sets for the changed regions and their super-regions.
-  for ( ; Begin != End; ++Begin) {
-    const MemRegion *MR = *Begin;
+  for (ArrayRef<const MemRegion *>::iterator
+       I = Regions.begin(), E = Regions.end(); I != E; ++I) {
+    const MemRegion *MR = *I;
     Invalidated.insert(MR);
 
     SuperRegions.insert(MR);
@@ -1779,7 +1806,7 @@ CStringChecker::checkRegionChanges(const GRState *state,
   return state->set<CStringLength>(Entries);
 }
 
-void CStringChecker::checkLiveSymbols(const GRState *state,
+void CStringChecker::checkLiveSymbols(const ProgramState *state,
                                       SymbolReaper &SR) const {
   // Mark all symbols in our string length map as valid.
   CStringLength::EntryMap Entries = state->get<CStringLength>();
@@ -1799,7 +1826,7 @@ void CStringChecker::checkDeadSymbols(SymbolReaper &SR,
   if (!SR.hasDeadSymbols())
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   CStringLength::EntryMap Entries = state->get<CStringLength>();
   if (Entries.isEmpty())
     return;
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 6c3dfacec331..4db6ac0181a0 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -16,6 +16,7 @@
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/AST/ParentMap.h"
 #include "clang/Basic/TargetInfo.h"
@@ -47,7 +48,8 @@ class CallAndMessageChecker
   void emitNilReceiverBug(CheckerContext &C, const ObjCMessage &msg,
                           ExplodedNode *N) const;
 
-  void HandleNilReceiver(CheckerContext &C, const GRState *state,
+  void HandleNilReceiver(CheckerContext &C,
+                         const ProgramState *state,
                          ObjCMessage msg) const;
 
   static void LazyInit_BT(const char *desc, llvm::OwningPtr<BugType> &BT) {
@@ -63,9 +65,9 @@ void CallAndMessageChecker::EmitBadCall(BugType *BT, CheckerContext &C,
   if (!N)
     return;
 
-  EnhancedBugReport *R = new EnhancedBugReport(*BT, BT->getName(), N);
-  R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                       bugreporter::GetCalleeExpr(N));
+  BugReport *R = new BugReport(*BT, BT->getName(), N);
+  R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                               bugreporter::GetCalleeExpr(N)));
   C.EmitReport(R);
 }
 
@@ -91,10 +93,10 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
       LazyInit_BT(BT_desc, BT);
 
       // Generate a report for this bug.
-      EnhancedBugReport *R = new EnhancedBugReport(*BT, BT->getName(), N);
+      BugReport *R = new BugReport(*BT, BT->getName(), N);
       R->addRange(argRange);
       if (argEx)
-        R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, argEx);
+        R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, argEx));
       C.EmitReport(R);
     }
     return true;
@@ -105,7 +107,7 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
 
     class FindUninitializedField {
     public:
-      llvm::SmallVector<const FieldDecl *, 10> FieldChain;
+      SmallVector<const FieldDecl *, 10> FieldChain;
     private:
       ASTContext &C;
       StoreManager &StoreMgr;
@@ -116,7 +118,7 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
                              MemRegionManager &mrMgr, Store s)
       : C(c), StoreMgr(storeMgr), MrMgr(mrMgr), store(s) {}
 
-      bool Find(const TypedRegion *R) {
+      bool Find(const TypedValueRegion *R) {
         QualType T = R->getValueType();
         if (const RecordType *RT = T->getAsStructureType()) {
           const RecordDecl *RD = RT->getDecl()->getDefinition();
@@ -157,23 +159,23 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
         os << "Passed-by-value struct argument contains uninitialized data";
 
         if (F.FieldChain.size() == 1)
-          os << " (e.g., field: '" << F.FieldChain[0] << "')";
+          os << " (e.g., field: '" << *F.FieldChain[0] << "')";
         else {
           os << " (e.g., via the field chain: '";
           bool first = true;
-          for (llvm::SmallVectorImpl<const FieldDecl *>::iterator
+          for (SmallVectorImpl<const FieldDecl *>::iterator
                DI = F.FieldChain.begin(), DE = F.FieldChain.end(); DI!=DE;++DI){
             if (first)
               first = false;
             else
               os << '.';
-            os << *DI;
+            os << **DI;
           }
           os << "')";
         }
 
         // Generate a report for this bug.
-        EnhancedBugReport *R = new EnhancedBugReport(*BT, os.str(), N);
+        BugReport *R = new BugReport(*BT, os.str(), N);
         R->addRange(argRange);
 
         // FIXME: enhance track back for uninitialized value for arbitrary
@@ -216,7 +218,7 @@ void CallAndMessageChecker::checkPreStmt(const CallExpr *CE,
 void CallAndMessageChecker::checkPreObjCMessage(ObjCMessage msg,
                                                 CheckerContext &C) const {
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   // FIXME: Handle 'super'?
   if (const Expr *receiver = msg.getInstanceReceiver()) {
@@ -226,11 +228,11 @@ void CallAndMessageChecker::checkPreObjCMessage(ObjCMessage msg,
         if (!BT_msg_undef)
           BT_msg_undef.reset(new BuiltinBug("Receiver in message expression is "
                                             "an uninitialized value"));
-        EnhancedBugReport *R =
-          new EnhancedBugReport(*BT_msg_undef, BT_msg_undef->getName(), N);
+        BugReport *R =
+          new BugReport(*BT_msg_undef, BT_msg_undef->getName(), N);
         R->addRange(receiver->getSourceRange());
-        R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                             receiver);
+        R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                                                   receiver));
         C.EmitReport(R);
       }
       return;
@@ -238,7 +240,7 @@ void CallAndMessageChecker::checkPreObjCMessage(ObjCMessage msg,
       // Bifurcate the state into nil and non-nil ones.
       DefinedOrUnknownSVal receiverVal = cast<DefinedOrUnknownSVal>(recVal);
   
-      const GRState *notNilState, *nilState;
+      const ProgramState *notNilState, *nilState;
       llvm::tie(notNilState, nilState) = state->assume(receiverVal);
   
       // Handle receiver must be nil.
@@ -271,11 +273,11 @@ void CallAndMessageChecker::emitNilReceiverBug(CheckerContext &C,
      << "' is nil and returns a value of type '"
      << msg.getType(C.getASTContext()).getAsString() << "' that will be garbage";
 
-  EnhancedBugReport *report = new EnhancedBugReport(*BT_msg_ret, os.str(), N);
+  BugReport *report = new BugReport(*BT_msg_ret, os.str(), N);
   if (const Expr *receiver = msg.getInstanceReceiver()) {
     report->addRange(receiver->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                              receiver);
+    report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                                                    receiver));
   }
   C.EmitReport(report);
 }
@@ -288,7 +290,7 @@ static bool supportsNilWithFloatRet(const llvm::Triple &triple) {
 }
 
 void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
-                                              const GRState *state,
+                                              const ProgramState *state,
                                               ObjCMessage msg) const {
   ASTContext &Ctx = C.getASTContext();
 
@@ -303,7 +305,7 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
     // have the "use of undefined value" be smarter about where the
     // undefined value came from.
     if (C.getPredecessor()->getParentMap().isConsumedExpr(msg.getOriginExpr())){
-      if (ExplodedNode* N = C.generateSink(state))
+      if (ExplodedNode *N = C.generateSink(state))
         emitNilReceiverBug(C, msg, N);
       return;
     }
@@ -322,13 +324,13 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
     const uint64_t returnTypeSize = Ctx.getTypeSize(CanRetTy);
 
     if (voidPtrSize < returnTypeSize &&
-        !(supportsNilWithFloatRet(Ctx.Target.getTriple()) &&
+        !(supportsNilWithFloatRet(Ctx.getTargetInfo().getTriple()) &&
           (Ctx.FloatTy == CanRetTy ||
            Ctx.DoubleTy == CanRetTy ||
            Ctx.LongDoubleTy == CanRetTy ||
            Ctx.LongLongTy == CanRetTy ||
            Ctx.UnsignedLongLongTy == CanRetTy))) {
-      if (ExplodedNode* N = C.generateSink(state))
+      if (ExplodedNode *N = C.generateSink(state))
         emitNilReceiverBug(C, msg, N);
       return;
     }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
index 585a87d498d3..84a9e6b3c528 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
@@ -44,7 +44,7 @@ void CastSizeChecker::checkPreStmt(const CastExpr *CE,CheckerContext &C) const {
   if (ToPointeeTy->isIncompleteType())
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const MemRegion *R = state->getSVal(E).getAsRegion();
   if (R == 0)
     return;
@@ -72,7 +72,7 @@ void CastSizeChecker::checkPreStmt(const CastExpr *CE,CheckerContext &C) const {
         BT.reset(new BuiltinBug("Cast region with wrong size.",
                             "Cast a region whose size is not a multiple of the"
                             " destination type size."));
-      RangedBugReport *R = new RangedBugReport(*BT, BT->getDescription(),
+      BugReport *R = new BugReport(*BT, BT->getDescription(),
                                                errorNode);
       R->addRange(CE->getSourceRange());
       C.EmitReport(R);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
index 3210b0a4bb61..c85521066711 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
@@ -62,7 +62,7 @@ void CastToStructChecker::checkPreStmt(const CastExpr *CE,
                             "Casting a non-structure type to a structure type "
                             "and accessing a field can lead to memory access "
                             "errors or data corruption."));
-      RangedBugReport *R = new RangedBugReport(*BT,BT->getDescription(), N);
+      BugReport *R = new BugReport(*BT,BT->getDescription(), N);
       R->addRange(CE->getSourceRange());
       C.EmitReport(R);
     }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
index 0c693a0bd1b6..c325bb1517f3 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
@@ -27,9 +27,9 @@
 using namespace clang;
 using namespace ento;
 
-static bool scan_dealloc(Stmt* S, Selector Dealloc) {
+static bool scan_dealloc(Stmt *S, Selector Dealloc) {
 
-  if (ObjCMessageExpr* ME = dyn_cast<ObjCMessageExpr>(S))
+  if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(S))
     if (ME->getSelector() == Dealloc) {
       switch (ME->getReceiverKind()) {
       case ObjCMessageExpr::Instance: return false;
@@ -48,26 +48,26 @@ static bool scan_dealloc(Stmt* S, Selector Dealloc) {
   return false;
 }
 
-static bool scan_ivar_release(Stmt* S, ObjCIvarDecl* ID,
-                              const ObjCPropertyDecl* PD,
+static bool scan_ivar_release(Stmt *S, ObjCIvarDecl *ID,
+                              const ObjCPropertyDecl *PD,
                               Selector Release,
                               IdentifierInfo* SelfII,
-                              ASTContext& Ctx) {
+                              ASTContext &Ctx) {
 
   // [mMyIvar release]
-  if (ObjCMessageExpr* ME = dyn_cast<ObjCMessageExpr>(S))
+  if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(S))
     if (ME->getSelector() == Release)
       if (ME->getInstanceReceiver())
-        if (Expr* Receiver = ME->getInstanceReceiver()->IgnoreParenCasts())
-          if (ObjCIvarRefExpr* E = dyn_cast<ObjCIvarRefExpr>(Receiver))
+        if (Expr *Receiver = ME->getInstanceReceiver()->IgnoreParenCasts())
+          if (ObjCIvarRefExpr *E = dyn_cast<ObjCIvarRefExpr>(Receiver))
             if (E->getDecl() == ID)
               return true;
 
   // [self setMyIvar:nil];
-  if (ObjCMessageExpr* ME = dyn_cast<ObjCMessageExpr>(S))
+  if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(S))
     if (ME->getInstanceReceiver())
-      if (Expr* Receiver = ME->getInstanceReceiver()->IgnoreParenCasts())
-        if (DeclRefExpr* E = dyn_cast<DeclRefExpr>(Receiver))
+      if (Expr *Receiver = ME->getInstanceReceiver()->IgnoreParenCasts())
+        if (DeclRefExpr *E = dyn_cast<DeclRefExpr>(Receiver))
           if (E->getDecl()->getIdentifier() == SelfII)
             if (ME->getMethodDecl() == PD->getSetterMethodDecl() &&
                 ME->getNumArgs() == 1 &&
@@ -78,7 +78,7 @@ static bool scan_ivar_release(Stmt* S, ObjCIvarDecl* ID,
   // self.myIvar = nil;
   if (BinaryOperator* BO = dyn_cast<BinaryOperator>(S))
     if (BO->isAssignmentOp())
-      if (ObjCPropertyRefExpr* PRE =
+      if (ObjCPropertyRefExpr *PRE =
            dyn_cast<ObjCPropertyRefExpr>(BO->getLHS()->IgnoreParenCasts()))
         if (PRE->isExplicitProperty() && PRE->getExplicitProperty() == PD)
             if (BO->getRHS()->isNullPointerConstant(Ctx, 
@@ -96,13 +96,13 @@ static bool scan_ivar_release(Stmt* S, ObjCIvarDecl* ID,
   return false;
 }
 
-static void checkObjCDealloc(const ObjCImplementationDecl* D,
+static void checkObjCDealloc(const ObjCImplementationDecl *D,
                              const LangOptions& LOpts, BugReporter& BR) {
 
-  assert (LOpts.getGCMode() != LangOptions::GCOnly);
+  assert (LOpts.getGC() != LangOptions::GCOnly);
 
-  ASTContext& Ctx = BR.getContext();
-  const ObjCInterfaceDecl* ID = D->getClassInterface();
+  ASTContext &Ctx = BR.getContext();
+  const ObjCInterfaceDecl *ID = D->getClassInterface();
 
   // Does the class contain any ivars that are pointers (or id<...>)?
   // If not, skip the check entirely.
@@ -114,7 +114,7 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
   for (ObjCInterfaceDecl::ivar_iterator I=ID->ivar_begin(), E=ID->ivar_end();
        I!=E; ++I) {
 
-    ObjCIvarDecl* ID = *I;
+    ObjCIvarDecl *ID = *I;
     QualType T = ID->getType();
 
     if (!T->isObjCObjectPointerType() ||
@@ -154,7 +154,7 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
   // Get the "dealloc" selector.
   IdentifierInfo* II = &Ctx.Idents.get("dealloc");
   Selector S = Ctx.Selectors.getSelector(0, &II);
-  ObjCMethodDecl* MD = 0;
+  ObjCMethodDecl *MD = 0;
 
   // Scan the instance methods for "dealloc".
   for (ObjCImplementationDecl::instmeth_iterator I = D->instmeth_begin(),
@@ -166,9 +166,12 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
     }
   }
 
+  PathDiagnosticLocation DLoc =
+    PathDiagnosticLocation::createBegin(D, BR.getSourceManager());
+
   if (!MD) { // No dealloc found.
 
-    const char* name = LOpts.getGCMode() == LangOptions::NonGC
+    const char* name = LOpts.getGC() == LangOptions::NonGC
                        ? "missing -dealloc"
                        : "missing -dealloc (Hybrid MM, non-GC)";
 
@@ -176,14 +179,14 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
     llvm::raw_string_ostream os(buf);
     os << "Objective-C class '" << D << "' lacks a 'dealloc' instance method";
 
-    BR.EmitBasicReport(name, os.str(), D->getLocStart());
+    BR.EmitBasicReport(name, os.str(), DLoc);
     return;
   }
 
   // dealloc found.  Scan for missing [super dealloc].
   if (MD->getBody() && !scan_dealloc(MD->getBody(), S)) {
 
-    const char* name = LOpts.getGCMode() == LangOptions::NonGC
+    const char* name = LOpts.getGC() == LangOptions::NonGC
                        ? "missing [super dealloc]"
                        : "missing [super dealloc] (Hybrid MM, non-GC)";
 
@@ -193,7 +196,7 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
        << "' does not send a 'dealloc' message to its super class"
            " (missing [super dealloc])";
 
-    BR.EmitBasicReport(name, os.str(), D->getLocStart());
+    BR.EmitBasicReport(name, os.str(), DLoc);
     return;
   }
 
@@ -213,7 +216,7 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
     if ((*I)->getPropertyImplementation() != ObjCPropertyImplDecl::Synthesize)
       continue;
 
-    ObjCIvarDecl* ID = (*I)->getPropertyIvarDecl();
+    ObjCIvarDecl *ID = (*I)->getPropertyIvarDecl();
     if (!ID)
       continue;
 
@@ -221,13 +224,13 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
     if (!T->isObjCObjectPointerType()) // Skip non-pointer ivars
       continue;
 
-    const ObjCPropertyDecl* PD = (*I)->getPropertyDecl();
+    const ObjCPropertyDecl *PD = (*I)->getPropertyDecl();
     if (!PD)
       continue;
 
     // ivars cannot be set via read-only properties, so we'll skip them
     if (PD->isReadOnly())
-       continue;
+      continue;
 
     // ivar must be released if and only if the kind of setter was not 'assign'
     bool requiresRelease = PD->getSetterKind() != ObjCPropertyDecl::Assign;
@@ -240,24 +243,27 @@ static void checkObjCDealloc(const ObjCImplementationDecl* D,
       llvm::raw_string_ostream os(buf);
 
       if (requiresRelease) {
-        name = LOpts.getGCMode() == LangOptions::NonGC
+        name = LOpts.getGC() == LangOptions::NonGC
                ? "missing ivar release (leak)"
                : "missing ivar release (Hybrid MM, non-GC)";
 
-        os << "The '" << ID
+        os << "The '" << *ID
            << "' instance variable was retained by a synthesized property but "
               "wasn't released in 'dealloc'";
       } else {
-        name = LOpts.getGCMode() == LangOptions::NonGC
+        name = LOpts.getGC() == LangOptions::NonGC
                ? "extra ivar release (use-after-release)"
                : "extra ivar release (Hybrid MM, non-GC)";
 
-        os << "The '" << ID
+        os << "The '" << *ID
            << "' instance variable was not retained by a synthesized property "
               "but was released in 'dealloc'";
       }
 
-      BR.EmitBasicReport(name, category, os.str(), (*I)->getLocation());
+      PathDiagnosticLocation SDLoc =
+        PathDiagnosticLocation::createBegin((*I), BR.getSourceManager());
+
+      BR.EmitBasicReport(name, category, os.str(), SDLoc);
     }
   }
 }
@@ -272,7 +278,7 @@ class ObjCDeallocChecker : public Checker<
 public:
   void checkASTDecl(const ObjCImplementationDecl *D, AnalysisManager& mgr,
                     BugReporter &BR) const {
-    if (mgr.getLangOptions().getGCMode() == LangOptions::GCOnly)
+    if (mgr.getLangOptions().getGC() == LangOptions::GCOnly)
       return;
     checkObjCDealloc(cast<ObjCImplementationDecl>(D), mgr.getLangOptions(), BR);
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
index fec06a95350e..c076c1e39b08 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
@@ -28,7 +28,7 @@ using namespace clang;
 using namespace ento;
 
 static bool AreTypesCompatible(QualType Derived, QualType Ancestor,
-                               ASTContext& C) {
+                               ASTContext &C) {
 
   // Right now don't compare the compatibility of pointers.  That involves
   // looking at subtyping relationships.  FIXME: Future patch.
@@ -51,36 +51,40 @@ static void CompareReturnTypes(const ObjCMethodDecl *MethDerived,
     llvm::raw_string_ostream os(sbuf);
 
     os << "The Objective-C class '"
-       << MethDerived->getClassInterface()
+       << *MethDerived->getClassInterface()
        << "', which is derived from class '"
-       << MethAncestor->getClassInterface()
+       << *MethAncestor->getClassInterface()
        << "', defines the instance method '"
        << MethDerived->getSelector().getAsString()
        << "' whose return type is '"
        << ResDerived.getAsString()
        << "'.  A method with the same name (same selector) is also defined in "
           "class '"
-       << MethAncestor->getClassInterface()
+       << *MethAncestor->getClassInterface()
        << "' and has a return type of '"
        << ResAncestor.getAsString()
        << "'.  These two types are incompatible, and may result in undefined "
           "behavior for clients of these classes.";
 
+    PathDiagnosticLocation MethDLoc =
+      PathDiagnosticLocation::createBegin(MethDerived,
+                                          BR.getSourceManager());
+
     BR.EmitBasicReport("Incompatible instance method return type",
-                       os.str(), MethDerived->getLocStart());
+                       os.str(), MethDLoc);
   }
 }
 
-static void CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
+static void CheckObjCInstMethSignature(const ObjCImplementationDecl *ID,
                                        BugReporter& BR) {
 
-  const ObjCInterfaceDecl* D = ID->getClassInterface();
-  const ObjCInterfaceDecl* C = D->getSuperClass();
+  const ObjCInterfaceDecl *D = ID->getClassInterface();
+  const ObjCInterfaceDecl *C = D->getSuperClass();
 
   if (!C)
     return;
 
-  ASTContext& Ctx = BR.getContext();
+  ASTContext &Ctx = BR.getContext();
 
   // Build a DenseMap of the methods for quick querying.
   typedef llvm::DenseMap<Selector,ObjCMethodDecl*> MapTy;
@@ -90,7 +94,7 @@ static void CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
   for (ObjCImplementationDecl::instmeth_iterator I=ID->instmeth_begin(),
        E=ID->instmeth_end(); I!=E; ++I) {
 
-    ObjCMethodDecl* M = *I;
+    ObjCMethodDecl *M = *I;
     IMeths[M->getSelector()] = M;
     ++NumMethods;
   }
@@ -101,7 +105,7 @@ static void CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
     for (ObjCInterfaceDecl::instmeth_iterator I=C->instmeth_begin(),
          E=C->instmeth_end(); I!=E; ++I) {
 
-      ObjCMethodDecl* M = *I;
+      ObjCMethodDecl *M = *I;
       Selector S = M->getSelector();
 
       MapTy::iterator MI = IMeths.find(S);
@@ -110,7 +114,7 @@ static void CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
         continue;
 
       --NumMethods;
-      ObjCMethodDecl* MethDerived = MI->second;
+      ObjCMethodDecl *MethDerived = MI->second;
       MI->second = 0;
 
       CompareReturnTypes(MethDerived, M, BR, Ctx, ID);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
index 53810eed1255..bf7ba185b5aa 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
@@ -12,18 +12,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangSACheckers.h"
+#include "clang/Analysis/AnalysisContext.h"
+#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/AST/StmtVisitor.h"
-#include "llvm/Support/raw_ostream.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
 using namespace ento;
 
 static bool isArc4RandomAvailable(const ASTContext &Ctx) {
-  const llvm::Triple &T = Ctx.Target.getTriple();
+  const llvm::Triple &T = Ctx.getTargetInfo().getTriple();
   return T.getVendor() == llvm::Triple::Apple ||
          T.getOS() == llvm::Triple::FreeBSD ||
          T.getOS() == llvm::Triple::NetBSD ||
@@ -34,14 +36,16 @@ static bool isArc4RandomAvailable(const ASTContext &Ctx) {
 namespace {
 class WalkAST : public StmtVisitor<WalkAST> {
   BugReporter &BR;
+  AnalysisContext* AC;
   enum { num_setids = 6 };
   IdentifierInfo *II_setid[num_setids];
 
   const bool CheckRand;
 
 public:
-  WalkAST(BugReporter &br) : BR(br), II_setid(),
-                 CheckRand(isArc4RandomAvailable(BR.getContext())) {}
+  WalkAST(BugReporter &br, AnalysisContext* ac)
+  : BR(br), AC(ac), II_setid(),
+    CheckRand(isArc4RandomAvailable(BR.getContext())) {}
 
   // Statement visitor methods.
   void VisitCallExpr(CallExpr *CE);
@@ -52,7 +56,6 @@ class WalkAST : public StmtVisitor<WalkAST> {
   void VisitChildren(Stmt *S);
 
   // Helpers.
-  IdentifierInfo *getIdentifier(IdentifierInfo *& II, const char *str);
   bool checkCall_strCommon(const CallExpr *CE, const FunctionDecl *FD);
 
   typedef void (WalkAST::*FnCheck)(const CallExpr *,
@@ -67,21 +70,11 @@ class WalkAST : public StmtVisitor<WalkAST> {
   void checkCall_strcat(const CallExpr *CE, const FunctionDecl *FD);
   void checkCall_rand(const CallExpr *CE, const FunctionDecl *FD);
   void checkCall_random(const CallExpr *CE, const FunctionDecl *FD);
+  void checkCall_vfork(const CallExpr *CE, const FunctionDecl *FD);
   void checkUncheckedReturnValue(CallExpr *CE);
 };
 } // end anonymous namespace
 
-//===----------------------------------------------------------------------===//
-// Helper methods.
-//===----------------------------------------------------------------------===//
-
-IdentifierInfo *WalkAST::getIdentifier(IdentifierInfo *& II, const char *str) {
-  if (!II)
-    II = &BR.getContext().Idents.get(str);
-
-  return II;
-}
-
 //===----------------------------------------------------------------------===//
 // AST walking.
 //===----------------------------------------------------------------------===//
@@ -103,7 +96,7 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
   IdentifierInfo *II = FD->getIdentifier();
   if (!II)   // if no identifier, not a simple C function
     return;
-  llvm::StringRef Name = II->getName();
+  StringRef Name = II->getName();
   if (Name.startswith("__builtin_"))
     Name = Name.substr(10);
 
@@ -124,6 +117,7 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
     .Case("rand", &WalkAST::checkCall_rand)
     .Case("rand_r", &WalkAST::checkCall_rand)
     .Case("random", &WalkAST::checkCall_random)
+    .Case("vfork", &WalkAST::checkCall_vfork)
     .Default(NULL);
 
   // If the callee isn't defined, it is not of security concern.
@@ -247,7 +241,7 @@ void WalkAST::checkLoopConditionForFloat(const ForStmt *FS) {
   // referenced the compared variable.
   const DeclRefExpr *drCond = vdLHS == drInc->getDecl() ? drLHS : drRHS;
 
-  llvm::SmallVector<SourceRange, 2> ranges;
+  SmallVector<SourceRange, 2> ranges;
   llvm::SmallString<256> sbuf;
   llvm::raw_svector_ostream os(sbuf);
 
@@ -259,8 +253,11 @@ void WalkAST::checkLoopConditionForFloat(const ForStmt *FS) {
   ranges.push_back(drInc->getSourceRange());
 
   const char *bugType = "Floating point variable used as loop counter";
+
+  PathDiagnosticLocation FSLoc =
+    PathDiagnosticLocation::createBegin(FS, BR.getSourceManager(), AC);
   BR.EmitBasicReport(bugType, "Security", os.str(),
-                     FS->getLocStart(), ranges.data(), ranges.size());
+                     FSLoc, ranges.data(), ranges.size());
 }
 
 //===----------------------------------------------------------------------===//
@@ -290,11 +287,13 @@ void WalkAST::checkCall_gets(const CallExpr *CE, const FunctionDecl *FD) {
 
   // Issue a warning.
   SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
   BR.EmitBasicReport("Potential buffer overflow in call to 'gets'",
                      "Security",
                      "Call to function 'gets' is extremely insecure as it can "
                      "always result in a buffer overflow",
-                     CE->getLocStart(), &R, 1);
+                     CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -326,11 +325,13 @@ void WalkAST::checkCall_getpw(const CallExpr *CE, const FunctionDecl *FD) {
 
   // Issue a warning.
   SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
   BR.EmitBasicReport("Potential buffer overflow in call to 'getpw'",
                      "Security",
                      "The getpw() function is dangerous as it may overflow the "
                      "provided buffer. It is obsoleted by getpwuid().",
-                     CE->getLocStart(), &R, 1);
+                     CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -359,11 +360,13 @@ void WalkAST::checkCall_mktemp(const CallExpr *CE, const FunctionDecl *FD) {
 
   // Issue a waring.
   SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
   BR.EmitBasicReport("Potential insecure temporary file in call 'mktemp'",
                      "Security",
                      "Call to function 'mktemp' is insecure as it always "
                      "creates or uses insecure temporary file.  Use 'mkstemp' instead",
-                     CE->getLocStart(), &R, 1);
+                     CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -378,6 +381,8 @@ void WalkAST::checkCall_strcpy(const CallExpr *CE, const FunctionDecl *FD) {
 
   // Issue a warning.
   SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
   BR.EmitBasicReport("Potential insecure memory buffer bounds restriction in "
                      "call 'strcpy'",
                      "Security",
@@ -385,7 +390,7 @@ void WalkAST::checkCall_strcpy(const CallExpr *CE, const FunctionDecl *FD) {
 		     "provide bounding of the memory buffer. Replace "
 		     "unbounded copy functions with analogous functions that "
 		     "support length arguments such as 'strncpy'. CWE-119.",
-                     CE->getLocStart(), &R, 1);
+                     CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -400,6 +405,8 @@ void WalkAST::checkCall_strcat(const CallExpr *CE, const FunctionDecl *FD) {
 
   // Issue a warning.
   SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
   BR.EmitBasicReport("Potential insecure memory buffer bounds restriction in "
 		     "call 'strcat'",
 		     "Security",
@@ -407,7 +414,7 @@ void WalkAST::checkCall_strcat(const CallExpr *CE, const FunctionDecl *FD) {
 		     "provide bounding of the memory buffer. Replace "
 		     "unbounded copy functions with analogous functions that "
 		     "support length arguments such as 'strncat'. CWE-119.",
-                     CE->getLocStart(), &R, 1);
+                     CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -470,16 +477,18 @@ void WalkAST::checkCall_rand(const CallExpr *CE, const FunctionDecl *FD) {
   // Issue a warning.
   llvm::SmallString<256> buf1;
   llvm::raw_svector_ostream os1(buf1);
-  os1 << '\'' << FD << "' is a poor random number generator";
+  os1 << '\'' << *FD << "' is a poor random number generator";
 
   llvm::SmallString<256> buf2;
   llvm::raw_svector_ostream os2(buf2);
-  os2 << "Function '" << FD
+  os2 << "Function '" << *FD
       << "' is obsolete because it implements a poor random number generator."
       << "  Use 'arc4random' instead";
 
   SourceRange R = CE->getCallee()->getSourceRange();
-  BR.EmitBasicReport(os1.str(), "Security", os2.str(),CE->getLocStart(), &R, 1);
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
+  BR.EmitBasicReport(os1.str(), "Security", os2.str(), CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -502,11 +511,33 @@ void WalkAST::checkCall_random(const CallExpr *CE, const FunctionDecl *FD) {
 
   // Issue a warning.
   SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
   BR.EmitBasicReport("'random' is not a secure random number generator",
                      "Security",
                      "The 'random' function produces a sequence of values that "
                      "an adversary may be able to predict.  Use 'arc4random' "
-                     "instead", CE->getLocStart(), &R, 1);
+                     "instead", CELoc, &R, 1);
+}
+
+//===----------------------------------------------------------------------===//
+// Check: 'vfork' should not be used.
+// POS33-C: Do not use vfork().
+//===----------------------------------------------------------------------===//
+
+void WalkAST::checkCall_vfork(const CallExpr *CE, const FunctionDecl *FD) {
+  // All calls to vfork() are insecure, issue a warning.
+  SourceRange R = CE->getCallee()->getSourceRange();
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
+  BR.EmitBasicReport("Potential insecure implementation-specific behavior in "
+                     "call 'vfork'",
+                     "Security",
+                     "Call to function 'vfork' is insecure as it can lead to "
+                     "denial of service situations in the parent process. "
+                     "Replace calls to vfork with calls to the safer "
+                     "'posix_spawn' function",
+                     CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -557,16 +588,18 @@ void WalkAST::checkUncheckedReturnValue(CallExpr *CE) {
   // Issue a warning.
   llvm::SmallString<256> buf1;
   llvm::raw_svector_ostream os1(buf1);
-  os1 << "Return value is not checked in call to '" << FD << '\'';
+  os1 << "Return value is not checked in call to '" << *FD << '\'';
 
   llvm::SmallString<256> buf2;
   llvm::raw_svector_ostream os2(buf2);
-  os2 << "The return value from the call to '" << FD
-      << "' is not checked.  If an error occurs in '" << FD
+  os2 << "The return value from the call to '" << *FD
+      << "' is not checked.  If an error occurs in '" << *FD
       << "', the following code may execute with unexpected privileges";
 
   SourceRange R = CE->getCallee()->getSourceRange();
-  BR.EmitBasicReport(os1.str(), "Security", os2.str(),CE->getLocStart(), &R, 1);
+  PathDiagnosticLocation CELoc =
+    PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC);
+  BR.EmitBasicReport(os1.str(), "Security", os2.str(), CELoc, &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -578,7 +611,7 @@ class SecuritySyntaxChecker : public Checker<check::ASTCodeBody> {
 public:
   void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
                         BugReporter &BR) const {
-    WalkAST walker(BR);
+    WalkAST walker(BR, mgr.getAnalysisContext(D));
     walker.Visit(D->getBody());
   }
 };
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
index abf53fd3db28..469be05fb71f 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
@@ -13,9 +13,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangSACheckers.h"
+#include "clang/AST/StmtVisitor.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
-#include "clang/AST/StmtVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 
 using namespace clang;
 using namespace ento;
@@ -23,9 +24,10 @@ using namespace ento;
 namespace {
 class WalkAST : public StmtVisitor<WalkAST> {
   BugReporter &BR;
+  AnalysisContext* AC;
 
 public:
-  WalkAST(BugReporter &br) : BR(br) {}
+  WalkAST(BugReporter &br, AnalysisContext* ac) : BR(br), AC(ac) {}
   void VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *E);
   void VisitStmt(Stmt *S) { VisitChildren(S); }
   void VisitChildren(Stmt *S);
@@ -59,11 +61,13 @@ void WalkAST::VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *E) {
       return;
 
     SourceRange R = ArgEx->getSourceRange();
+    PathDiagnosticLocation ELoc =
+      PathDiagnosticLocation::createBegin(E, BR.getSourceManager(), AC);
     BR.EmitBasicReport("Potential unintended use of sizeof() on pointer type",
                        "Logic",
                        "The code calls sizeof() on a pointer type. "
                        "This can produce an unexpected result.",
-                       E->getLocStart(), &R, 1);
+                       ELoc, &R, 1);
   }
 }
 
@@ -76,7 +80,7 @@ class SizeofPointerChecker : public Checker<check::ASTCodeBody> {
 public:
   void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
                         BugReporter &BR) const {
-    WalkAST walker(BR);
+    WalkAST walker(BR, mgr.getAnalysisContext(D));
     walker.Visit(D->getBody());
   }
 };
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Checkers.td b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Checkers.td
index 2c196b597b7d..d53e0b887e75 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Checkers.td
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Checkers.td
@@ -9,42 +9,33 @@
 
 include "clang/StaticAnalyzer/Checkers/CheckerBase.td"
 
-//===----------------------------------------------------------------------===//
-// Groups.
-//===----------------------------------------------------------------------===//
-
-def AllExperimental : CheckerGroup<"all-experimental">;
-
 //===----------------------------------------------------------------------===//
 // Packages.
 //===----------------------------------------------------------------------===//
 
+def Experimental : Package<"experimental">;
+
 def Core : Package<"core">;
 def CoreBuiltin : Package<"builtin">, InPackage<Core>;
 def CoreUninitialized  : Package<"uninitialized">, InPackage<Core>;
-def CoreExperimental : Package<"experimental">, InPackage<Core>,
-  InGroup<AllExperimental>, Hidden;
+def CoreExperimental : Package<"core">, InPackage<Experimental>, Hidden;
 
 def Cplusplus : Package<"cplusplus">;
-def CplusplusExperimental : Package<"experimental">, InPackage<Cplusplus>,
-  InGroup<AllExperimental>, Hidden;
+def CplusplusExperimental : Package<"cplusplus">, InPackage<Experimental>, Hidden;
 
 def DeadCode : Package<"deadcode">;
-def DeadCodeExperimental : Package<"experimental">, InPackage<DeadCode>,
-  InGroup<AllExperimental>, Hidden;
+def DeadCodeExperimental : Package<"deadcode">, InPackage<Experimental>, Hidden;
 
 def Security : Package <"security">;
-def SecurityExperimental : Package<"experimental">, InPackage<Security>,
-  InGroup<AllExperimental>, Hidden;
+def SecurityExperimental : Package<"security">, InPackage<Experimental>, Hidden;
 
 def Unix : Package<"unix">;
-def UnixExperimental : Package<"experimental">, InPackage<Unix>,
-  InGroup<AllExperimental>, Hidden;
+def UnixExperimental : Package<"unix">, InPackage<Experimental>, Hidden;
 
 def OSX : Package<"osx">;
+def OSXExperimental : Package<"osx">, InPackage<Experimental>, Hidden;
 def Cocoa : Package<"cocoa">, InPackage<OSX>;
-def CocoaExperimental : Package<"experimental">, InPackage<Cocoa>,
-  InGroup<AllExperimental>, Hidden;
+def CocoaExperimental : Package<"cocoa">, InPackage<OSXExperimental>, Hidden;
 def CoreFoundation : Package<"coreFoundation">, InPackage<OSX>;
 
 def LLVM : Package<"llvm">;
@@ -220,6 +211,10 @@ def ReturnPointerRangeChecker : Checker<"ReturnPtrRange">,
   HelpText<"Check for an out-of-bound pointer being returned to callers">,
   DescFile<"ReturnPointerRangeChecker.cpp">;
 
+def MallocOverflowSecurityChecker : Checker<"MallocOverflow">,
+  HelpText<"Check for overflows in the arguments to malloc()">,
+  DescFile<"MallocOverflowSecurityChecker.cpp">;
+
 } // end "security.experimental"
 
 //===----------------------------------------------------------------------===//
@@ -274,6 +269,11 @@ def OSAtomicChecker : Checker<"AtomicCAS">,
   HelpText<"Evaluate calls to OSAtomic functions">,
   DescFile<"OSAtomicChecker.cpp">;
 
+def MacOSKeychainAPIChecker : Checker<"SecKeychainAPI">,
+  InPackage<OSX>,
+  HelpText<"Check for proper uses of Secure Keychain APIs">,
+  DescFile<"MacOSKeychainAPIChecker.cpp">;
+
 } // end "macosx"
 
 let ParentPackage = Cocoa in {
@@ -311,6 +311,10 @@ def NSErrorChecker : Checker<"NSError">,
   HelpText<"Check usage of NSError** parameters">,
   DescFile<"NSErrorChecker.cpp">;
 
+def RetainCountChecker : Checker<"RetainCount">,
+  HelpText<"Check for leaks and improper reference count management">,
+  DescFile<"RetainCountChecker.cpp">;
+
 } // end "cocoa"
 
 let ParentPackage = CocoaExperimental in {
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
index 50b57d1ae497..3c9238190da4 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
@@ -16,8 +16,8 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/ImmutableMap.h"
 using namespace clang;
@@ -62,7 +62,7 @@ class ChrootChecker : public Checker<eval::Call, check::PreStmt<CallExpr> > {
 } // end anonymous namespace
 
 bool ChrootChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
   const FunctionDecl *FD = L.getAsFunctionDecl();
@@ -88,8 +88,8 @@ bool ChrootChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
 }
 
 void ChrootChecker::Chroot(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
-  GRStateManager &Mgr = state->getStateManager();
+  const ProgramState *state = C.getState();
+  ProgramStateManager &Mgr = state->getStateManager();
   
   // Once encouter a chroot(), set the enum value ROOT_CHANGED directly in 
   // the GDM.
@@ -98,11 +98,11 @@ void ChrootChecker::Chroot(CheckerContext &C, const CallExpr *CE) const {
 }
 
 void ChrootChecker::Chdir(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
-  GRStateManager &Mgr = state->getStateManager();
+  const ProgramState *state = C.getState();
+  ProgramStateManager &Mgr = state->getStateManager();
 
   // If there are no jail state in the GDM, just return.
-  const void* k = state->FindGDM(ChrootChecker::getTag());
+  const void *k = state->FindGDM(ChrootChecker::getTag());
   if (!k)
     return;
 
@@ -125,7 +125,7 @@ void ChrootChecker::Chdir(CheckerContext &C, const CallExpr *CE) const {
 
 // Check the jail state before any function call except chroot and chdir().
 void ChrootChecker::checkPreStmt(const CallExpr *CE, CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
   const FunctionDecl *FD = L.getAsFunctionDecl();
@@ -143,7 +143,7 @@ void ChrootChecker::checkPreStmt(const CallExpr *CE, CheckerContext &C) const {
     return;
   
   // If jail state is ROOT_CHANGED, generate BugReport.
-  void* const* k = state->FindGDM(ChrootChecker::getTag());
+  void *const* k = state->FindGDM(ChrootChecker::getTag());
   if (k)
     if (isRootChanged((intptr_t) *k))
       if (ExplodedNode *N = C.generateNode()) {
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangCheckers.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangCheckers.cpp
new file mode 100644
index 000000000000..77a5a7226453
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangCheckers.cpp
@@ -0,0 +1,32 @@
+//===--- ClangCheckers.h - Provides builtin checkers ------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Checkers/ClangCheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerRegistry.h"
+
+// FIXME: This is only necessary as long as there are checker registration
+// functions that do additional work besides mgr.registerChecker<CLASS>().
+// The only checkers that currently do this are:
+// - NSAutoreleasePoolChecker
+// - NSErrorChecker
+// - ObjCAtSyncChecker
+// It's probably worth including this information in Checkers.td to minimize
+// boilerplate code.
+#include "ClangSACheckers.h"
+
+using namespace clang;
+using namespace ento;
+
+void ento::registerBuiltinCheckers(CheckerRegistry &registry) {
+#define GET_CHECKERS
+#define CHECKER(FULLNAME,CLASS,DESCFILE,HELPTEXT,GROUPINDEX,HIDDEN)    \
+  registry.addChecker(register##CLASS, FULLNAME, HELPTEXT);
+#include "Checkers.inc"
+#undef GET_CHECKERS
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.cpp
deleted file mode 100644
index 291f8e01f4b9..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.cpp
+++ /dev/null
@@ -1,289 +0,0 @@
-//===--- ClangSACheckerProvider.cpp - Clang SA Checkers Provider ----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines the CheckerProvider for the checkers defined in
-// libclangStaticAnalyzerCheckers.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ClangSACheckerProvider.h"
-#include "ClangSACheckers.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/CheckerProvider.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/ADT/DenseSet.h"
-#include "map"
-
-using namespace clang;
-using namespace ento;
-
-namespace {
-
-/// \brief Provider for all the checkers in libclangStaticAnalyzerCheckers.
-class ClangSACheckerProvider : public CheckerProvider {
-public:
-  virtual void registerCheckers(CheckerManager &checkerMgr,
-                              CheckerOptInfo *checkOpts, unsigned numCheckOpts);
-  virtual void printHelp(llvm::raw_ostream &OS);
-};
-
-}
-
-CheckerProvider *ento::createClangSACheckerProvider() {
-  return new ClangSACheckerProvider();
-}
-
-namespace {
-
-struct StaticCheckerInfoRec {
-  const char *FullName;
-  void (*RegFunc)(CheckerManager &mgr);
-  const char *HelpText;
-  int GroupIndex;
-  bool Hidden;
-};
-
-struct StaticPackageInfoRec {
-  const char *FullName;
-  int GroupIndex;
-  bool Hidden;
-};
-
-struct StaticGroupInfoRec {
-  const char *FullName;
-};
-
-} // end anonymous namespace.
-
-static const StaticPackageInfoRec StaticPackageInfo[] = {
-#define GET_PACKAGES
-#define PACKAGE(FULLNAME, GROUPINDEX, HIDDEN)    \
-  { FULLNAME, GROUPINDEX, HIDDEN },
-#include "Checkers.inc"
-  { 0, -1, 0 }
-#undef PACKAGE
-#undef GET_PACKAGES
-};
-
-static const unsigned NumPackages =   sizeof(StaticPackageInfo)
-                                    / sizeof(StaticPackageInfoRec) - 1;
-
-static const StaticGroupInfoRec StaticGroupInfo[] = {
-#define GET_GROUPS
-#define GROUP(FULLNAME)    \
-  { FULLNAME },
-#include "Checkers.inc"
-  { 0 }
-#undef GROUP
-#undef GET_GROUPS
-};
-
-static const unsigned NumGroups =   sizeof(StaticGroupInfo)
-                                    / sizeof(StaticGroupInfoRec) - 1;
-
-static const StaticCheckerInfoRec StaticCheckerInfo[] = {
-#define GET_CHECKERS
-#define CHECKER(FULLNAME,CLASS,DESCFILE,HELPTEXT,GROUPINDEX,HIDDEN)    \
-  { FULLNAME, register##CLASS, HELPTEXT, GROUPINDEX, HIDDEN },
-#include "Checkers.inc"
-  { 0, 0, 0, -1, 0}
-#undef CHECKER
-#undef GET_CHECKERS
-};
-
-static const unsigned NumCheckers =   sizeof(StaticCheckerInfo)
-                                    / sizeof(StaticCheckerInfoRec) - 1;
-
-namespace {
-
-struct CheckNameOption {
-  const char  *Name;
-  const short *Members;
-  const short *SubGroups;
-  bool Hidden;
-};
-
-} // end anonymous namespace.
-
-#define GET_MEMBER_ARRAYS
-#include "Checkers.inc"
-#undef GET_MEMBER_ARRAYS
-
-// The table of check name options, sorted by name for fast binary lookup.
-static const CheckNameOption CheckNameTable[] = {
-#define GET_CHECKNAME_TABLE
-#include "Checkers.inc"
-#undef GET_CHECKNAME_TABLE
-};
-static const size_t
-        CheckNameTableSize = sizeof(CheckNameTable) / sizeof(CheckNameTable[0]);
-
-static bool CheckNameOptionCompare(const CheckNameOption &LHS,
-                                   const CheckNameOption &RHS) {
-  return strcmp(LHS.Name, RHS.Name) < 0;
-}
-
-static void collectCheckers(const CheckNameOption *checkName,
-                            bool enable,
-                         llvm::DenseSet<const StaticCheckerInfoRec *> &checkers,
-                            bool collectHidden) {
-  if (checkName->Hidden && !collectHidden)
-    return;
-
-  if (const short *member = checkName->Members) {
-    if (enable) {
-      for (; *member != -1; ++member)
-        if (collectHidden || !StaticCheckerInfo[*member].Hidden)
-          checkers.insert(&StaticCheckerInfo[*member]);
-    } else {
-      for (; *member != -1; ++member)
-        checkers.erase(&StaticCheckerInfo[*member]);
-    }
-  }
-
-  // Enable/disable all subgroups along with this one.
-  if (const short *subGroups = checkName->SubGroups) {
-    for (; *subGroups != -1; ++subGroups) {
-      const CheckNameOption *sub = &CheckNameTable[*subGroups];
-      collectCheckers(sub, enable, checkers, collectHidden && !sub->Hidden);
-    }
-  }
-}
-
-static void collectCheckers(CheckerOptInfo &opt,
-                       llvm::DenseSet<const StaticCheckerInfoRec *> &checkers) {
-  const char *optName = opt.getName();
-  CheckNameOption key = { optName, 0, 0, false };
-  const CheckNameOption *found =
-  std::lower_bound(CheckNameTable, CheckNameTable + CheckNameTableSize, key,
-                   CheckNameOptionCompare);
-  if (found == CheckNameTable + CheckNameTableSize ||
-      strcmp(found->Name, optName) != 0)
-    return;  // Check name not found.
-
-  opt.claim();
-  collectCheckers(found, opt.isEnabled(), checkers, /*collectHidden=*/true);
-}
-
-void ClangSACheckerProvider::registerCheckers(CheckerManager &checkerMgr,
-                             CheckerOptInfo *checkOpts, unsigned numCheckOpts) {
-  llvm::DenseSet<const StaticCheckerInfoRec *> enabledCheckers;
-  for (unsigned i = 0; i != numCheckOpts; ++i)
-    collectCheckers(checkOpts[i], enabledCheckers);
-  for (llvm::DenseSet<const StaticCheckerInfoRec *>::iterator
-         I = enabledCheckers.begin(), E = enabledCheckers.end(); I != E; ++I) {
-    (*I)->RegFunc(checkerMgr);
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Printing Help.
-//===----------------------------------------------------------------------===//
-
-static void printPackageOption(llvm::raw_ostream &OS) {
-  // Find the maximum option length.
-  unsigned OptionFieldWidth = 0;
-  for (unsigned i = 0; i != NumPackages; ++i) {
-    // Limit the amount of padding we are willing to give up for alignment.
-    unsigned Length = strlen(StaticPackageInfo[i].FullName);
-    if (Length <= 30)
-      OptionFieldWidth = std::max(OptionFieldWidth, Length);
-  }
-
-  const unsigned InitialPad = 2;
-  for (unsigned i = 0; i != NumPackages; ++i) {
-    const StaticPackageInfoRec &package = StaticPackageInfo[i];
-    const std::string &Option = package.FullName;
-    int Pad = OptionFieldWidth - int(Option.size());
-    OS.indent(InitialPad) << Option;
-
-    if (package.GroupIndex != -1 || package.Hidden) {
-      // Break on long option names.
-      if (Pad < 0) {
-        OS << "\n";
-        Pad = OptionFieldWidth + InitialPad;
-      }
-      OS.indent(Pad + 1) << "[";
-      if (package.GroupIndex != -1) {
-        OS << "Group=" << StaticGroupInfo[package.GroupIndex].FullName;
-        if (package.Hidden)
-          OS << ", ";
-      }
-      if (package.Hidden)
-        OS << "Hidden";
-      OS << "]";
-    }
-
-    OS << "\n";
-  }
-}
-
-typedef std::map<std::string, const StaticCheckerInfoRec *> SortedCheckers;
-
-static void printCheckerOption(llvm::raw_ostream &OS,SortedCheckers &checkers) {
-  // Find the maximum option length.
-  unsigned OptionFieldWidth = 0;
-  for (SortedCheckers::iterator
-         I = checkers.begin(), E = checkers.end(); I != E; ++I) {
-    // Limit the amount of padding we are willing to give up for alignment.
-    unsigned Length = strlen(I->second->FullName);
-    if (Length <= 30)
-      OptionFieldWidth = std::max(OptionFieldWidth, Length);
-  }
-
-  const unsigned InitialPad = 2;
-  for (SortedCheckers::iterator
-         I = checkers.begin(), E = checkers.end(); I != E; ++I) {
-    const std::string &Option = I->first;
-    const StaticCheckerInfoRec &checker = *I->second;
-    int Pad = OptionFieldWidth - int(Option.size());
-    OS.indent(InitialPad) << Option;
-
-    // Break on long option names.
-    if (Pad < 0) {
-      OS << "\n";
-      Pad = OptionFieldWidth + InitialPad;
-    }
-    OS.indent(Pad + 1) << checker.HelpText;
-
-    if (checker.GroupIndex != -1 || checker.Hidden) {
-      OS << "  [";
-      if (checker.GroupIndex != -1) {
-        OS << "Group=" << StaticGroupInfo[checker.GroupIndex].FullName;
-        if (checker.Hidden)
-          OS << ", ";
-      }
-      if (checker.Hidden)
-        OS << "Hidden";
-      OS << "]";
-    }
-
-    OS << "\n";
-  }
-}
-
-void ClangSACheckerProvider::printHelp(llvm::raw_ostream &OS) {
-  OS << "USAGE: -analyzer-checker <CHECKER or PACKAGE or GROUP,...>\n";
-
-  OS << "\nGROUPS:\n";
-  for (unsigned i = 0; i != NumGroups; ++i)
-    OS.indent(2) << StaticGroupInfo[i].FullName << "\n";
-
-  OS << "\nPACKAGES:\n";
-  printPackageOption(OS);
-
-  OS << "\nCHECKERS:\n";
-
-  // Sort checkers according to their full name.
-  SortedCheckers checkers;
-  for (unsigned i = 0; i != NumCheckers; ++i)
-    checkers[StaticCheckerInfo[i].FullName] = &StaticCheckerInfo[i];
-
-  printCheckerOption(OS, checkers);
-}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.h b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.h
deleted file mode 100644
index f6c80119fdc0..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.h
+++ /dev/null
@@ -1,29 +0,0 @@
-//===--- ClangSACheckerProvider.h - Clang SA Checkers Provider --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines the entry point for creating the provider for the checkers defined
-// in libclangStaticAnalyzerCheckers.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_SA_CHECKERS_CLANGSACHECKERPROVIDER_H
-#define LLVM_CLANG_SA_CHECKERS_CLANGSACHECKERPROVIDER_H
-
-namespace clang {
-
-namespace ento {
-  class CheckerProvider;
-
-CheckerProvider *createClangSACheckerProvider();
-
-} // end ento namespace
-
-} // end clang namespace
-
-#endif
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckers.h b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckers.h
index 5524b0f53276..289ce8d8c3ba 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckers.h
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ClangSACheckers.h
@@ -19,6 +19,7 @@ namespace clang {
 
 namespace ento {
 class CheckerManager;
+class CheckerRegistry;
 
 #define GET_CHECKERS
 #define CHECKER(FULLNAME,CLASS,CXXFILE,HELPTEXT,GROUPINDEX,HIDDEN)    \
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
index ec2a88a55681..901af43c5f26 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
@@ -50,7 +50,7 @@ void ReachableCode::computeReachableBlocks() {
   if (!cfg.getNumBlockIDs())
     return;
   
-  llvm::SmallVector<const CFGBlock*, 10> worklist;
+  SmallVector<const CFGBlock*, 10> worklist;
   worklist.push_back(&cfg.getEntry());
   
   while (!worklist.empty()) {
@@ -68,12 +68,13 @@ void ReachableCode::computeReachableBlocks() {
 }
 
 namespace {
-class DeadStoreObs : public LiveVariables::ObserverTy {
+class DeadStoreObs : public LiveVariables::Observer {
   const CFG &cfg;
   ASTContext &Ctx;
   BugReporter& BR;
+  AnalysisContext* AC;
   ParentMap& Parents;
-  llvm::SmallPtrSet<VarDecl*, 20> Escaped;
+  llvm::SmallPtrSet<const VarDecl*, 20> Escaped;
   llvm::OwningPtr<ReachableCode> reachableCode;
   const CFGBlock *currentBlock;
 
@@ -81,14 +82,15 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
 
 public:
   DeadStoreObs(const CFG &cfg, ASTContext &ctx,
-               BugReporter& br, ParentMap& parents,
-               llvm::SmallPtrSet<VarDecl*, 20> &escaped)
-    : cfg(cfg), Ctx(ctx), BR(br), Parents(parents),
+               BugReporter& br, AnalysisContext* ac, ParentMap& parents,
+               llvm::SmallPtrSet<const VarDecl*, 20> &escaped)
+    : cfg(cfg), Ctx(ctx), BR(br), AC(ac), Parents(parents),
       Escaped(escaped), currentBlock(0) {}
 
   virtual ~DeadStoreObs() {}
 
-  void Report(VarDecl* V, DeadStoreKind dsk, SourceLocation L, SourceRange R) {
+  void Report(const VarDecl *V, DeadStoreKind dsk,
+              PathDiagnosticLocation L, SourceRange R) {
     if (Escaped.count(V))
       return;
     
@@ -102,26 +104,25 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
     if (!reachableCode->isReachable(currentBlock))
       return;
 
-    const std::string &name = V->getNameAsString();
-
-    const char* BugType = 0;
-    std::string msg;
+    llvm::SmallString<64> buf;
+    llvm::raw_svector_ostream os(buf);
+    const char *BugType = 0;
 
     switch (dsk) {
       default:
-        assert(false && "Impossible dead store type.");
+        llvm_unreachable("Impossible dead store type.");
 
       case DeadInit:
         BugType = "Dead initialization";
-        msg = "Value stored to '" + name +
-          "' during its initialization is never read";
+        os << "Value stored to '" << *V
+           << "' during its initialization is never read";
         break;
 
       case DeadIncrement:
         BugType = "Dead increment";
       case Standard:
         if (!BugType) BugType = "Dead assignment";
-        msg = "Value stored to '" + name + "' is never read";
+        os << "Value stored to '" << *V << "' is never read";
         break;
 
       case Enclosing:
@@ -131,13 +132,12 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
         return;
     }
 
-    BR.EmitBasicReport(BugType, "Dead store", msg, L, R);
+    BR.EmitBasicReport(BugType, "Dead store", os.str(), L, R);
   }
 
-  void CheckVarDecl(VarDecl* VD, Expr* Ex, Expr* Val,
+  void CheckVarDecl(const VarDecl *VD, const Expr *Ex, const Expr *Val,
                     DeadStoreKind dsk,
-                    const LiveVariables::AnalysisDataTy& AD,
-                    const LiveVariables::ValTy& Live) {
+                    const LiveVariables::LivenessValues &Live) {
 
     if (!VD->hasLocalStorage())
       return;
@@ -146,30 +146,32 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
     if (VD->getType()->getAs<ReferenceType>())
       return;
 
-    if (!Live(VD, AD) && 
-        !(VD->getAttr<UnusedAttr>() || VD->getAttr<BlocksAttr>()))
-      Report(VD, dsk, Ex->getSourceRange().getBegin(),
-             Val->getSourceRange());
+    if (!Live.isLive(VD) && 
+        !(VD->getAttr<UnusedAttr>() || VD->getAttr<BlocksAttr>())) {
+
+      PathDiagnosticLocation ExLoc =
+        PathDiagnosticLocation::createBegin(Ex, BR.getSourceManager(), AC);
+      Report(VD, dsk, ExLoc, Val->getSourceRange());
+    }
   }
 
-  void CheckDeclRef(DeclRefExpr* DR, Expr* Val, DeadStoreKind dsk,
-                    const LiveVariables::AnalysisDataTy& AD,
-                    const LiveVariables::ValTy& Live) {
-    if (VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl()))
-      CheckVarDecl(VD, DR, Val, dsk, AD, Live);
+  void CheckDeclRef(const DeclRefExpr *DR, const Expr *Val, DeadStoreKind dsk,
+                    const LiveVariables::LivenessValues& Live) {
+    if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl()))
+      CheckVarDecl(VD, DR, Val, dsk, Live);
   }
 
-  bool isIncrement(VarDecl* VD, BinaryOperator* B) {
+  bool isIncrement(VarDecl *VD, const BinaryOperator* B) {
     if (B->isCompoundAssignmentOp())
       return true;
 
-    Expr* RHS = B->getRHS()->IgnoreParenCasts();
-    BinaryOperator* BRHS = dyn_cast<BinaryOperator>(RHS);
+    const Expr *RHS = B->getRHS()->IgnoreParenCasts();
+    const BinaryOperator* BRHS = dyn_cast<BinaryOperator>(RHS);
 
     if (!BRHS)
       return false;
 
-    DeclRefExpr *DR;
+    const DeclRefExpr *DR;
 
     if ((DR = dyn_cast<DeclRefExpr>(BRHS->getLHS()->IgnoreParenCasts())))
       if (DR->getDecl() == VD)
@@ -182,9 +184,8 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
     return false;
   }
 
-  virtual void ObserveStmt(Stmt* S, const CFGBlock *block,
-                           const LiveVariables::AnalysisDataTy& AD,
-                           const LiveVariables::ValTy& Live) {
+  virtual void observeStmt(const Stmt *S, const CFGBlock *block,
+                           const LiveVariables::LivenessValues &Live) {
 
     currentBlock = block;
     
@@ -194,10 +195,10 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
 
     // Only cover dead stores from regular assignments.  ++/-- dead stores
     // have never flagged a real bug.
-    if (BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
+    if (const BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
       if (!B->isAssignmentOp()) return; // Skip non-assignments.
 
-      if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(B->getLHS()))
+      if (DeclRefExpr *DR = dyn_cast<DeclRefExpr>(B->getLHS()))
         if (VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
           // Special case: check for assigning null to a pointer.
           //  This is a common form of defensive programming.
@@ -208,10 +209,10 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
               return;
           }
 
-          Expr* RHS = B->getRHS()->IgnoreParenCasts();
+          Expr *RHS = B->getRHS()->IgnoreParenCasts();
           // Special case: self-assignments.  These are often used to shut up
           //  "unused variable" compiler warnings.
-          if (DeclRefExpr* RhsDR = dyn_cast<DeclRefExpr>(RHS))
+          if (DeclRefExpr *RhsDR = dyn_cast<DeclRefExpr>(RHS))
             if (VD == dyn_cast<VarDecl>(RhsDR->getDecl()))
               return;
 
@@ -220,29 +221,29 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
                               ? Enclosing
                               : (isIncrement(VD,B) ? DeadIncrement : Standard);
 
-          CheckVarDecl(VD, DR, B->getRHS(), dsk, AD, Live);
+          CheckVarDecl(VD, DR, B->getRHS(), dsk, Live);
         }
     }
-    else if (UnaryOperator* U = dyn_cast<UnaryOperator>(S)) {
+    else if (const UnaryOperator* U = dyn_cast<UnaryOperator>(S)) {
       if (!U->isIncrementOp() || U->isPrefix())
         return;
 
-      Stmt *parent = Parents.getParentIgnoreParenCasts(U);
+      const Stmt *parent = Parents.getParentIgnoreParenCasts(U);
       if (!parent || !isa<ReturnStmt>(parent))
         return;
 
-      Expr *Ex = U->getSubExpr()->IgnoreParenCasts();
+      const Expr *Ex = U->getSubExpr()->IgnoreParenCasts();
 
-      if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(Ex))
-        CheckDeclRef(DR, U, DeadIncrement, AD, Live);
+      if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Ex))
+        CheckDeclRef(DR, U, DeadIncrement, Live);
     }
-    else if (DeclStmt* DS = dyn_cast<DeclStmt>(S))
+    else if (const DeclStmt *DS = dyn_cast<DeclStmt>(S))
       // Iterate through the decls.  Warn if any initializers are complex
       // expressions that are not live (never used).
-      for (DeclStmt::decl_iterator DI=DS->decl_begin(), DE=DS->decl_end();
+      for (DeclStmt::const_decl_iterator DI=DS->decl_begin(), DE=DS->decl_end();
            DI != DE; ++DI) {
 
-        VarDecl* V = dyn_cast<VarDecl>(*DI);
+        VarDecl *V = dyn_cast<VarDecl>(*DI);
 
         if (!V)
           continue;
@@ -253,7 +254,7 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
           if (V->getType()->getAs<ReferenceType>())
             return;
             
-          if (Expr* E = V->getInit()) {
+          if (Expr *E = V->getInit()) {
             while (ExprWithCleanups *exprClean = dyn_cast<ExprWithCleanups>(E))
               E = exprClean->getSubExpr();
             
@@ -265,7 +266,7 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
             // A dead initialization is a variable that is dead after it
             // is initialized.  We don't flag warnings for those variables
             // marked 'unused'.
-            if (!Live(V, AD) && V->getAttr<UnusedAttr>() == 0) {
+            if (!Live.isLive(V) && V->getAttr<UnusedAttr>() == 0) {
               // Special case: check for initializations with constants.
               //
               //  e.g. : int x = 0;
@@ -296,7 +297,9 @@ class DeadStoreObs : public LiveVariables::ObserverTy {
                     return;
                 }
 
-              Report(V, DeadInit, V->getLocation(), E->getSourceRange());
+              PathDiagnosticLocation Loc =
+                PathDiagnosticLocation::create(V, BR.getSourceManager());
+              Report(V, DeadInit, Loc, E->getSourceRange());
             }
           }
         }
@@ -318,15 +321,15 @@ class FindEscaped : public CFGRecStmtDeclVisitor<FindEscaped>{
 
   CFG& getCFG() { return *cfg; }
 
-  llvm::SmallPtrSet<VarDecl*, 20> Escaped;
+  llvm::SmallPtrSet<const VarDecl*, 20> Escaped;
 
   void VisitUnaryOperator(UnaryOperator* U) {
     // Check for '&'.  Any VarDecl whose value has its address-taken we
     // treat as escaped.
-    Expr* E = U->getSubExpr()->IgnoreParenCasts();
+    Expr *E = U->getSubExpr()->IgnoreParenCasts();
     if (U->getOpcode() == UO_AddrOf)
-      if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(E))
-        if (VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl())) {
+      if (DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E))
+        if (VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
           Escaped.insert(VD);
           return;
         }
@@ -345,13 +348,14 @@ class DeadStoresChecker : public Checker<check::ASTCodeBody> {
 public:
   void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
                         BugReporter &BR) const {
-    if (LiveVariables *L = mgr.getLiveVariables(D)) {
+    if (LiveVariables *L = mgr.getAnalysis<LiveVariables>(D)) {
       CFG &cfg = *mgr.getCFG(D);
+      AnalysisContext *AC = mgr.getAnalysisContext(D);
       ParentMap &pmap = mgr.getParentMap(D);
       FindEscaped FS(&cfg);
       FS.getCFG().VisitBlockStmts(FS);
-      DeadStoreObs A(cfg, BR.getContext(), BR, pmap, FS.Escaped);
-      L->runOnAllBlocks(cfg, &A);
+      DeadStoreObs A(cfg, BR.getContext(), BR, AC, pmap, FS.Escaped);
+      L->runOnAllBlocks(A);
     }
   }
 };
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
index 486b7f7feb1b..d9d569423d3c 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
@@ -28,7 +28,7 @@ class LiveVariablesDumper : public Checker<check::ASTCodeBody> {
 public:
   void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
                         BugReporter &BR) const {
-    if (LiveVariables* L = mgr.getLiveVariables(D)) {
+    if (LiveVariables* L = mgr.getAnalysis<LiveVariables>(D)) {
       L->dumpBlockLiveness(mgr.getSourceManager());
     }
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index baaf8b3aff8a..eeda734a0766 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -29,16 +29,17 @@ class DereferenceChecker
   mutable llvm::OwningPtr<BuiltinBug> BT_undef;
 
 public:
-  void checkLocation(SVal location, bool isLoad, CheckerContext &C) const;
+  void checkLocation(SVal location, bool isLoad, const Stmt* S,
+                     CheckerContext &C) const;
 
-  static void AddDerefSource(llvm::raw_ostream &os,
-                             llvm::SmallVectorImpl<SourceRange> &Ranges,
+  static void AddDerefSource(raw_ostream &os,
+                             SmallVectorImpl<SourceRange> &Ranges,
                              const Expr *Ex, bool loadedFrom = false);
 };
 } // end anonymous namespace
 
-void DereferenceChecker::AddDerefSource(llvm::raw_ostream &os,
-                                     llvm::SmallVectorImpl<SourceRange> &Ranges,
+void DereferenceChecker::AddDerefSource(raw_ostream &os,
+                                        SmallVectorImpl<SourceRange> &Ranges,
                                         const Expr *Ex,
                                         bool loadedFrom) {
   Ex = Ex->IgnoreParenLValueCasts();
@@ -65,7 +66,7 @@ void DereferenceChecker::AddDerefSource(llvm::raw_ostream &os,
   }
 }
 
-void DereferenceChecker::checkLocation(SVal l, bool isLoad,
+void DereferenceChecker::checkLocation(SVal l, bool isLoad, const Stmt* S,
                                        CheckerContext &C) const {
   // Check for dereference of an undefined value.
   if (l.isUndef()) {
@@ -73,10 +74,10 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad,
       if (!BT_undef)
         BT_undef.reset(new BuiltinBug("Dereference of undefined pointer value"));
 
-      EnhancedBugReport *report =
-        new EnhancedBugReport(*BT_undef, BT_undef->getDescription(), N);
-      report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                                bugreporter::GetDerefExpr(N));
+      BugReport *report =
+        new BugReport(*BT_undef, BT_undef->getDescription(), N);
+      report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                        bugreporter::GetDerefExpr(N)));
       C.EmitReport(report);
     }
     return;
@@ -88,9 +89,8 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad,
   if (!isa<Loc>(location))
     return;
 
-  const Stmt *S = C.getStmt();
-  const GRState *state = C.getState();
-  const GRState *notNullState, *nullState;
+  const ProgramState *state = C.getState();
+  const ProgramState *notNullState, *nullState;
   llvm::tie(notNullState, nullState) = state->assume(location);
 
   // The explicit NULL case.
@@ -107,7 +107,7 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad,
         BT_null.reset(new BuiltinBug("Dereference of null pointer"));
 
       llvm::SmallString<100> buf;
-      llvm::SmallVector<SourceRange, 2> Ranges;
+      SmallVector<SourceRange, 2> Ranges;
       
       // Walk through lvalue casts to get the original expression
       // that syntactically caused the load.
@@ -157,15 +157,15 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad,
           break;
       }
 
-      EnhancedBugReport *report =
-        new EnhancedBugReport(*BT_null,
+      BugReport *report =
+        new BugReport(*BT_null,
                               buf.empty() ? BT_null->getDescription():buf.str(),
                               N);
 
-      report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                                bugreporter::GetDerefExpr(N));
+      report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                        bugreporter::GetDerefExpr(N)));
 
-      for (llvm::SmallVectorImpl<SourceRange>::iterator
+      for (SmallVectorImpl<SourceRange>::iterator
             I = Ranges.begin(), E = Ranges.end(); I!=E; ++I)
         report->addRange(*I);
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
index 07fb5aa998be..75b7cc47aa79 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
@@ -52,7 +52,7 @@ void DivZeroChecker::checkPreStmt(const BinaryOperator *B,
 
   // Check for divide by zero.
   ConstraintManager &CM = C.getConstraintManager();
-  const GRState *stateNotZero, *stateZero;
+  const ProgramState *stateNotZero, *stateZero;
   llvm::tie(stateNotZero, stateZero) = CM.assumeDual(C.getState(), *DV);
 
   if (stateZero && !stateNotZero) {
@@ -60,11 +60,11 @@ void DivZeroChecker::checkPreStmt(const BinaryOperator *B,
       if (!BT)
         BT.reset(new BuiltinBug("Division by zero"));
 
-      EnhancedBugReport *R = 
-        new EnhancedBugReport(*BT, BT->getDescription(), N);
+      BugReport *R = 
+        new BugReport(*BT, BT->getDescription(), N);
 
-      R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                           bugreporter::GetDenomExpr(N));
+      R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                   bugreporter::GetDenomExpr(N)));
 
       C.EmitReport(R);
     }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
index d699deecd768..531d87e42631 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
@@ -44,7 +44,7 @@ void FixedAddressChecker::checkPreStmt(const BinaryOperator *B,
   if (!T->isPointerType())
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   SVal RV = state->getSVal(B->getRHS());
 
@@ -57,7 +57,7 @@ void FixedAddressChecker::checkPreStmt(const BinaryOperator *B,
                           "Using a fixed address is not portable because that "
                           "address will probably not be valid in all "
                           "environments or platforms."));
-    RangedBugReport *R = new RangedBugReport(*BT, BT->getDescription(), N);
+    BugReport *R = new BugReport(*BT, BT->getDescription(), N);
     R->addRange(B->getRHS()->getSourceRange());
     C.EmitReport(R);
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
index b0c07fc7d264..5c257e595be3 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
@@ -96,10 +96,9 @@ class IdempotentOperationChecker
 
   // Hash table and related data structures
   struct BinaryOperatorData {
-    BinaryOperatorData() : assumption(Possible), analysisContext(0) {}
+    BinaryOperatorData() : assumption(Possible) {}
 
     Assumption assumption;
-    AnalysisContext *analysisContext;
     ExplodedNodeSet explodedNodes; // Set of ExplodedNodes that refer to a
                                    // BinaryOperator
   };
@@ -118,7 +117,6 @@ void IdempotentOperationChecker::checkPreStmt(const BinaryOperator *B,
   BinaryOperatorData &Data = hash[B];
   Assumption &A = Data.assumption;
   AnalysisContext *AC = C.getCurrentAnalysisContext();
-  Data.analysisContext = AC;
 
   // If we already have visited this node on a path that does not contain an
   // idempotent operation, return immediately.
@@ -143,7 +141,7 @@ void IdempotentOperationChecker::checkPreStmt(const BinaryOperator *B,
         || containsNonLocalVarDecl(RHS);
   }
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   SVal LHSVal = state->getSVal(LHS);
   SVal RHSVal = state->getSVal(RHS);
@@ -351,9 +349,14 @@ void IdempotentOperationChecker::checkEndAnalysis(ExplodedGraph &G,
     // Unpack the hash contents
     const BinaryOperatorData &Data = i->second;
     const Assumption &A = Data.assumption;
-    AnalysisContext *AC = Data.analysisContext;
     const ExplodedNodeSet &ES = Data.explodedNodes;
 
+    // If there are no nodes accosted with the expression, nothing to report.
+    // FIXME: This is possible because the checker does part of processing in
+    // checkPreStmt and part in checkPostStmt.
+    if (ES.begin() == ES.end())
+      continue;
+
     const BinaryOperator *B = i->first;
 
     if (A == Impossible)
@@ -363,6 +366,8 @@ void IdempotentOperationChecker::checkEndAnalysis(ExplodedGraph &G,
     // warning
     if (Eng.hasWorkRemaining()) {
       // If we can trace back
+      AnalysisContext *AC = (*ES.begin())->getLocationContext()
+                                         ->getAnalysisContext();
       if (!pathWasCompletelyAnalyzed(AC,
                                      AC->getCFGStmtMap()->getBlock(B),
                                      Eng.getCoreEngine()))
@@ -407,18 +412,18 @@ void IdempotentOperationChecker::checkEndAnalysis(ExplodedGraph &G,
 
     // Add a report for each ExplodedNode
     for (ExplodedNodeSet::iterator I = ES.begin(), E = ES.end(); I != E; ++I) {
-      EnhancedBugReport *report = new EnhancedBugReport(*BT, os.str(), *I);
+      BugReport *report = new BugReport(*BT, os.str(), *I);
 
       // Add source ranges and visitor hooks
       if (LHSRelevant) {
         const Expr *LHS = i->first->getLHS();
         report->addRange(LHS->getSourceRange());
-        report->addVisitorCreator(bugreporter::registerVarDeclsLastStore, LHS);
+        FindLastStoreBRVisitor::registerStatementVarDecls(*report, LHS);
       }
       if (RHSRelevant) {
         const Expr *RHS = i->first->getRHS();
         report->addRange(i->first->getRHS()->getSourceRange());
-        report->addVisitorCreator(bugreporter::registerVarDeclsLastStore, RHS);
+        FindLastStoreBRVisitor::registerStatementVarDecls(*report, RHS);
       }
 
       BR.EmitReport(report);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IteratorsChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IteratorsChecker.cpp
index de6da4f8c377..fbc57d336f90 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IteratorsChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/IteratorsChecker.cpp
@@ -20,7 +20,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Type.h"
@@ -117,16 +117,28 @@ class IteratorsChecker :
                     CheckerContext &C) const;
 
 private:
-  const GRState *handleAssign(const GRState *state, const Expr *lexp,
-      const Expr *rexp, const LocationContext *LC) const;
-  const GRState *handleAssign(const GRState *state, const MemRegion *MR,
-      const Expr *rexp, const LocationContext *LC) const;
-  const GRState *invalidateIterators(const GRState *state, const MemRegion *MR,
-      const MemberExpr *ME) const;
+  const ProgramState *handleAssign(const ProgramState *state,
+                                   const Expr *lexp,
+                                   const Expr *rexp,
+                                   const LocationContext *LC) const;
+
+  const ProgramState *handleAssign(const ProgramState *state,
+                                   const MemRegion *MR,
+                                   const Expr *rexp,
+                                   const LocationContext *LC) const;
+
+  const ProgramState *invalidateIterators(const ProgramState *state,
+                                          const MemRegion *MR,
+                                          const MemberExpr *ME) const;
+
   void checkExpr(CheckerContext &C, const Expr *E) const;
+
   void checkArgs(CheckerContext &C, const CallExpr *CE) const;
-  const MemRegion *getRegion(const GRState *state, const Expr *E,
-      const LocationContext *LC) const;
+
+  const MemRegion *getRegion(const ProgramState *state,
+                             const Expr *E,
+                             const LocationContext *LC) const;
+
   const DeclRefExpr *getDeclRefExpr(const Expr *E) const;
 };
 
@@ -139,8 +151,8 @@ class IteratorState {
 namespace clang {
   namespace ento {
     template <>
-    struct GRStateTrait<IteratorState> 
-      : public GRStatePartialTrait<IteratorState::EntryMap> {
+    struct ProgramStateTrait<IteratorState> 
+      : public ProgramStatePartialTrait<IteratorState::EntryMap> {
       static void *GDMIndex() { return IteratorsChecker::getTag(); }
     };
   }
@@ -162,7 +174,7 @@ static RefKind getTemplateKind(const NamedDecl *td) {
       || nameSpace->getName() != "std")
     return NoKind;
   
-  llvm::StringRef name = td->getName();
+  StringRef name = td->getName();
   return llvm::StringSwitch<RefKind>(name)
     .Cases("vector", "deque", VectorKind)
     .Default(NoKind);
@@ -215,7 +227,7 @@ static RefKind getTemplateKind(QualType T) {
 
 // Iterate through our map and invalidate any iterators that were
 // initialized fromt the specified instance MemRegion.
-const GRState *IteratorsChecker::invalidateIterators(const GRState *state,
+const ProgramState *IteratorsChecker::invalidateIterators(const ProgramState *state,
                           const MemRegion *MR, const MemberExpr *ME) const {
   IteratorState::EntryMap Map = state->get<IteratorState>();
   if (Map.isEmpty())
@@ -234,7 +246,7 @@ const GRState *IteratorsChecker::invalidateIterators(const GRState *state,
 }
 
 // Handle assigning to an iterator where we don't have the LValue MemRegion.
-const GRState *IteratorsChecker::handleAssign(const GRState *state,
+const ProgramState *IteratorsChecker::handleAssign(const ProgramState *state,
     const Expr *lexp, const Expr *rexp, const LocationContext *LC) const {
   // Skip the cast if present.
   if (const MaterializeTemporaryExpr *M 
@@ -259,7 +271,7 @@ const GRState *IteratorsChecker::handleAssign(const GRState *state,
 }
 
 // handle assigning to an iterator
-const GRState *IteratorsChecker::handleAssign(const GRState *state,
+const ProgramState *IteratorsChecker::handleAssign(const ProgramState *state,
     const MemRegion *MR, const Expr *rexp, const LocationContext *LC) const {
   // Assume unknown until we find something definite.
   state = state->set<IteratorState>(MR, RefState::getUnknown());
@@ -287,7 +299,7 @@ const GRState *IteratorsChecker::handleAssign(const GRState *state,
         return state;
       // Finally, see if it is one of the calls that will create
       // a valid iterator and mark it if so, else mark as Unknown.
-      llvm::StringRef mName = ME->getMemberDecl()->getName();
+      StringRef mName = ME->getMemberDecl()->getName();
       
       if (llvm::StringSwitch<bool>(mName)        
           .Cases("begin", "insert", "erase", true).Default(false)) {
@@ -364,7 +376,7 @@ const DeclRefExpr *IteratorsChecker::getDeclRefExpr(const Expr *E) const {
 }
 
 // Get the MemRegion associated with the expresssion.
-const MemRegion *IteratorsChecker::getRegion(const GRState *state,
+const MemRegion *IteratorsChecker::getRegion(const ProgramState *state,
     const Expr *E, const LocationContext *LC) const {
   const DeclRefExpr *DRE = getDeclRefExpr(E);
   if (!DRE)
@@ -382,7 +394,7 @@ const MemRegion *IteratorsChecker::getRegion(const GRState *state,
 // use those nodes.  We also cannot create multiple nodes at one ProgramPoint
 // with the same tag.
 void IteratorsChecker::checkExpr(CheckerContext &C, const Expr *E) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const MemRegion *MR = getRegion(state, E,
                    C.getPredecessor()->getLocationContext());
   if (!MR)
@@ -410,7 +422,7 @@ void IteratorsChecker::checkExpr(CheckerContext &C, const Expr *E) const {
                     "container to its container";
       }
 
-      EnhancedBugReport *R = new EnhancedBugReport(*BT_Invalid, msg, N);
+      BugReport *R = new BugReport(*BT_Invalid, msg, N);
       R->addRange(getDeclRefExpr(E)->getSourceRange());
       C.EmitReport(R);
     }
@@ -422,7 +434,7 @@ void IteratorsChecker::checkExpr(CheckerContext &C, const Expr *E) const {
         const_cast<IteratorsChecker*>(this)->BT_Undefined =
           new BuiltinBug("Use of iterator that is not defined");
 
-      EnhancedBugReport *R = new EnhancedBugReport(*BT_Undefined,
+      BugReport *R = new BugReport(*BT_Undefined,
                                            BT_Undefined->getDescription(), N);
       R->addRange(getDeclRefExpr(E)->getSourceRange());
       C.EmitReport(R);
@@ -455,7 +467,7 @@ void IteratorsChecker::checkPreStmt(const CXXOperatorCallExpr *OCE,
                                     CheckerContext &C) const
 {
   const LocationContext *LC = C.getPredecessor()->getLocationContext();
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   OverloadedOperatorKind Kind = OCE->getOperator();
   if (Kind == OO_Equal) {
     checkExpr(C, OCE->getArg(1));
@@ -491,8 +503,8 @@ void IteratorsChecker::checkPreStmt(const CXXOperatorCallExpr *OCE,
               new BuiltinBug(
                       "Cannot compare iterators from different containers");
 
-          EnhancedBugReport *R = new EnhancedBugReport(*BT_Incompatible,
-                                         BT_Incompatible->getDescription(), N);
+          BugReport *R = new BugReport(*BT_Incompatible,
+                                        BT_Incompatible->getDescription(), N);
           R->addRange(OCE->getSourceRange());
           C.EmitReport(R);
         }
@@ -505,14 +517,14 @@ void IteratorsChecker::checkPreStmt(const CXXOperatorCallExpr *OCE,
 // uninitialized ones as Undefined.
 void IteratorsChecker::checkPreStmt(const DeclStmt *DS,
                                     CheckerContext &C) const {
-  const Decl* D = *DS->decl_begin();
-  const VarDecl* VD = dyn_cast<VarDecl>(D);
+  const Decl *D = *DS->decl_begin();
+  const VarDecl *VD = dyn_cast<VarDecl>(D);
   // Only care about iterators.
   if (getTemplateKind(VD->getType()) != VectorIteratorKind)
     return;
 
   // Get the MemRegion associated with the iterator and mark it as Undefined.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   Loc VarLoc = state->getLValue(VD, C.getPredecessor()->getLocationContext());
   const MemRegion *MR = VarLoc.getAsRegion();
   if (!MR)
@@ -520,7 +532,7 @@ void IteratorsChecker::checkPreStmt(const DeclStmt *DS,
   state = state->set<IteratorState>(MR, RefState::getUndefined());
 
   // if there is an initializer, handle marking Valid if a proper initializer
-  const Expr* InitEx = VD->getInit();
+  const Expr *InitEx = VD->getInit();
   if (InitEx) {
     // FIXME: This is too syntactic.  Since 'InitEx' will be analyzed first
     // it should resolve to an SVal that we can check for validity
@@ -544,8 +556,8 @@ void IteratorsChecker::checkPreStmt(const DeclStmt *DS,
 
 namespace { struct CalledReserved {}; }
 namespace clang { namespace ento {
-template<> struct GRStateTrait<CalledReserved> 
-    :  public GRStatePartialTrait<llvm::ImmutableSet<const MemRegion*> > {
+template<> struct ProgramStateTrait<CalledReserved> 
+    :  public ProgramStatePartialTrait<llvm::ImmutableSet<const MemRegion*> > {
   static void *GDMIndex() { static int index = 0; return &index; }
 };
 }}
@@ -571,8 +583,8 @@ void IteratorsChecker::checkPreStmt(const CXXMemberCallExpr *MCE,
     return;
   // If we are calling a function that invalidates iterators, mark them
   // appropriately by finding matching instances.
-  const GRState *state = C.getState();
-  llvm::StringRef mName = ME->getMemberDecl()->getName();
+  const ProgramState *state = C.getState();
+  StringRef mName = ME->getMemberDecl()->getName();
   if (llvm::StringSwitch<bool>(mName)
       .Cases("insert", "reserve", "push_back", true)
       .Cases("erase", "pop_back", "clear", "resize", true)
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
index 3d1b5e2d1051..e398fcb32257 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
@@ -32,13 +32,13 @@ static bool IsLLVMStringRef(QualType T) {
   if (!RT)
     return false;
 
-  return llvm::StringRef(QualType(RT, 0).getAsString()) ==
-          "class llvm::StringRef";
+  return StringRef(QualType(RT, 0).getAsString()) ==
+          "class StringRef";
 }
 
 /// Check whether the declaration is semantically inside the top-level
 /// namespace named by ns.
-static bool InNamespace(const Decl *D, llvm::StringRef NS) {
+static bool InNamespace(const Decl *D, StringRef NS) {
   const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(D->getDeclContext());
   if (!ND)
     return false;
@@ -109,7 +109,7 @@ static bool IsSmallVector(QualType T) {
 }
 
 //===----------------------------------------------------------------------===//
-// CHECK: a llvm::StringRef should not be bound to a temporary std::string whose
+// CHECK: a StringRef should not be bound to a temporary std::string whose
 // lifetime is shorter than the StringRef's.
 //===----------------------------------------------------------------------===//
 
@@ -150,7 +150,7 @@ void StringRefCheckerVisitor::VisitVarDecl(VarDecl *VD) {
     return;
 
   // Pattern match for:
-  // llvm::StringRef x = call() (where call returns std::string)
+  // StringRef x = call() (where call returns std::string)
   if (!IsLLVMStringRef(VD->getType()))
     return;
   ExprWithCleanups *Ex1 = dyn_cast<ExprWithCleanups>(Init);
@@ -175,9 +175,10 @@ void StringRefCheckerVisitor::VisitVarDecl(VarDecl *VD) {
   // Okay, badness!  Report an error.
   const char *desc = "StringRef should not be bound to temporary "
                      "std::string that it outlives";
-
+  PathDiagnosticLocation VDLoc =
+    PathDiagnosticLocation::createBegin(VD, BR.getSourceManager());
   BR.EmitBasicReport(desc, "LLVM Conventions", desc,
-                     VD->getLocStart(), Init->getSourceRange());
+                     VDLoc, Init->getSourceRange());
 }
 
 //===----------------------------------------------------------------------===//
@@ -210,7 +211,7 @@ static bool IsPartOfAST(const CXXRecordDecl *R) {
 
 namespace {
 class ASTFieldVisitor {
-  llvm::SmallVector<FieldDecl*, 10> FieldChain;
+  SmallVector<FieldDecl*, 10> FieldChain;
   const CXXRecordDecl *Root;
   BugReporter &BR;
 public:
@@ -260,7 +261,7 @@ void ASTFieldVisitor::ReportError(QualType T) {
   if (FieldChain.size() > 1) {
     os << " via the following chain: ";
     bool isFirst = true;
-    for (llvm::SmallVectorImpl<FieldDecl*>::iterator I=FieldChain.begin(),
+    for (SmallVectorImpl<FieldDecl*>::iterator I=FieldChain.begin(),
          E=FieldChain.end(); I!=E; ++I) {
       if (!isFirst)
         os << '.';
@@ -279,8 +280,10 @@ void ASTFieldVisitor::ReportError(QualType T) {
   // just report warnings when we see an out-of-line method definition for a
   // class, as that heuristic doesn't always work (the complete definition of
   // the class may be in the header file, for example).
+  PathDiagnosticLocation L = PathDiagnosticLocation::createBegin(
+                               FieldChain.front(), BR.getSourceManager());
   BR.EmitBasicReport("AST node allocates heap memory", "LLVM Conventions",
-                     os.str(), FieldChain.front()->getLocStart());
+                     os.str(), L);
 }
 
 //===----------------------------------------------------------------------===//
@@ -294,7 +297,7 @@ class LLVMConventionsChecker : public Checker<
 public:
   void checkASTDecl(const CXXRecordDecl *R, AnalysisManager& mgr,
                     BugReporter &BR) const {
-    if (R->isDefinition())
+    if (R->isCompleteDefinition())
       CheckASTMemory(R, BR);
   }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
new file mode 100644
index 000000000000..2607db80ba8e
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
@@ -0,0 +1,632 @@
+//==--- MacOSKeychainAPIChecker.cpp ------------------------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This checker flags misuses of KeyChainAPI. In particular, the password data
+// allocated/returned by SecKeychainItemCopyContent,
+// SecKeychainFindGenericPassword, SecKeychainFindInternetPassword functions has
+// to be freed using a call to SecKeychainItemFreeContent.
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class MacOSKeychainAPIChecker : public Checker<check::PreStmt<CallExpr>,
+                                               check::PreStmt<ReturnStmt>,
+                                               check::PostStmt<CallExpr>,
+                                               check::EndPath,
+                                               check::DeadSymbols> {
+  mutable llvm::OwningPtr<BugType> BT;
+
+public:
+  /// AllocationState is a part of the checker specific state together with the
+  /// MemRegion corresponding to the allocated data.
+  struct AllocationState {
+    /// The index of the allocator function.
+    unsigned int AllocatorIdx;
+    SymbolRef Region;
+
+    AllocationState(const Expr *E, unsigned int Idx, SymbolRef R) :
+      AllocatorIdx(Idx),
+      Region(R) {}
+
+    bool operator==(const AllocationState &X) const {
+      return (AllocatorIdx == X.AllocatorIdx &&
+              Region == X.Region);
+    }
+
+    void Profile(llvm::FoldingSetNodeID &ID) const {
+      ID.AddInteger(AllocatorIdx);
+      ID.AddPointer(Region);
+    }
+  };
+
+  void checkPreStmt(const CallExpr *S, CheckerContext &C) const;
+  void checkPreStmt(const ReturnStmt *S, CheckerContext &C) const;
+  void checkPostStmt(const CallExpr *S, CheckerContext &C) const;
+  void checkDeadSymbols(SymbolReaper &SR, CheckerContext &C) const;
+  void checkEndPath(EndOfFunctionNodeBuilder &B, ExprEngine &Eng) const;
+
+private:
+  typedef std::pair<SymbolRef, const AllocationState*> AllocationPair;
+  typedef llvm::SmallVector<AllocationPair, 2> AllocationPairVec;
+
+  enum APIKind {
+    /// Denotes functions tracked by this checker.
+    ValidAPI = 0,
+    /// The functions commonly/mistakenly used in place of the given API.
+    ErrorAPI = 1,
+    /// The functions which may allocate the data. These are tracked to reduce
+    /// the false alarm rate.
+    PossibleAPI = 2
+  };
+  /// Stores the information about the allocator and deallocator functions -
+  /// these are the functions the checker is tracking.
+  struct ADFunctionInfo {
+    const char* Name;
+    unsigned int Param;
+    unsigned int DeallocatorIdx;
+    APIKind Kind;
+  };
+  static const unsigned InvalidIdx = 100000;
+  static const unsigned FunctionsToTrackSize = 8;
+  static const ADFunctionInfo FunctionsToTrack[FunctionsToTrackSize];
+  /// The value, which represents no error return value for allocator functions.
+  static const unsigned NoErr = 0;
+
+  /// Given the function name, returns the index of the allocator/deallocator
+  /// function.
+  static unsigned getTrackedFunctionIndex(StringRef Name, bool IsAllocator);
+
+  inline void initBugType() const {
+    if (!BT)
+      BT.reset(new BugType("Improper use of SecKeychain API", "Mac OS API"));
+  }
+
+  void generateDeallocatorMismatchReport(const AllocationPair &AP,
+                                         const Expr *ArgExpr,
+                                         CheckerContext &C) const;
+
+  BugReport *generateAllocatedDataNotReleasedReport(const AllocationPair &AP,
+                                                    ExplodedNode *N) const;
+
+  /// Check if RetSym evaluates to an error value in the current state.
+  bool definitelyReturnedError(SymbolRef RetSym,
+                               const ProgramState *State,
+                               SValBuilder &Builder,
+                               bool noError = false) const;
+
+  /// Check if RetSym evaluates to a NoErr value in the current state.
+  bool definitelyDidnotReturnError(SymbolRef RetSym,
+                                   const ProgramState *State,
+                                   SValBuilder &Builder) const {
+    return definitelyReturnedError(RetSym, State, Builder, true);
+  }
+
+  /// The bug visitor which allows us to print extra diagnostics along the
+  /// BugReport path. For example, showing the allocation site of the leaked
+  /// region.
+  class SecKeychainBugVisitor : public BugReporterVisitor {
+  protected:
+    // The allocated region symbol tracked by the main analysis.
+    SymbolRef Sym;
+
+  public:
+    SecKeychainBugVisitor(SymbolRef S) : Sym(S) {}
+    virtual ~SecKeychainBugVisitor() {}
+
+    void Profile(llvm::FoldingSetNodeID &ID) const {
+      static int X = 0;
+      ID.AddPointer(&X);
+      ID.AddPointer(Sym);
+    }
+
+    PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                   const ExplodedNode *PrevN,
+                                   BugReporterContext &BRC,
+                                   BugReport &BR);
+  };
+};
+}
+
+/// ProgramState traits to store the currently allocated (and not yet freed)
+/// symbols. This is a map from the allocated content symbol to the
+/// corresponding AllocationState.
+typedef llvm::ImmutableMap<SymbolRef,
+                       MacOSKeychainAPIChecker::AllocationState> AllocatedSetTy;
+
+namespace { struct AllocatedData {}; }
+namespace clang { namespace ento {
+template<> struct ProgramStateTrait<AllocatedData>
+    :  public ProgramStatePartialTrait<AllocatedSetTy > {
+  static void *GDMIndex() { static int index = 0; return &index; }
+};
+}}
+
+static bool isEnclosingFunctionParam(const Expr *E) {
+  E = E->IgnoreParenCasts();
+  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
+    const ValueDecl *VD = DRE->getDecl();
+    if (isa<ImplicitParamDecl>(VD) || isa<ParmVarDecl>(VD))
+      return true;
+  }
+  return false;
+}
+
+const MacOSKeychainAPIChecker::ADFunctionInfo
+  MacOSKeychainAPIChecker::FunctionsToTrack[FunctionsToTrackSize] = {
+    {"SecKeychainItemCopyContent", 4, 3, ValidAPI},                       // 0
+    {"SecKeychainFindGenericPassword", 6, 3, ValidAPI},                   // 1
+    {"SecKeychainFindInternetPassword", 13, 3, ValidAPI},                 // 2
+    {"SecKeychainItemFreeContent", 1, InvalidIdx, ValidAPI},              // 3
+    {"SecKeychainItemCopyAttributesAndData", 5, 5, ValidAPI},             // 4
+    {"SecKeychainItemFreeAttributesAndData", 1, InvalidIdx, ValidAPI},    // 5
+    {"free", 0, InvalidIdx, ErrorAPI},                                    // 6
+    {"CFStringCreateWithBytesNoCopy", 1, InvalidIdx, PossibleAPI},        // 7
+};
+
+unsigned MacOSKeychainAPIChecker::getTrackedFunctionIndex(StringRef Name,
+                                                          bool IsAllocator) {
+  for (unsigned I = 0; I < FunctionsToTrackSize; ++I) {
+    ADFunctionInfo FI = FunctionsToTrack[I];
+    if (FI.Name != Name)
+      continue;
+    // Make sure the function is of the right type (allocator vs deallocator).
+    if (IsAllocator && (FI.DeallocatorIdx == InvalidIdx))
+      return InvalidIdx;
+    if (!IsAllocator && (FI.DeallocatorIdx != InvalidIdx))
+      return InvalidIdx;
+
+    return I;
+  }
+  // The function is not tracked.
+  return InvalidIdx;
+}
+
+static SymbolRef getSymbolForRegion(CheckerContext &C,
+                                   const MemRegion *R) {
+  // Implicit casts (ex: void* -> char*) can turn Symbolic region into element
+  // region, if that is the case, get the underlining region.
+  R = R->StripCasts();
+  if (!isa<SymbolicRegion>(R)) {
+      return 0;
+  }
+  return cast<SymbolicRegion>(R)->getSymbol();
+}
+
+static bool isBadDeallocationArgument(const MemRegion *Arg) {
+  if (isa<AllocaRegion>(Arg) ||
+      isa<BlockDataRegion>(Arg) ||
+      isa<TypedRegion>(Arg)) {
+    return true;
+  }
+  return false;
+}
+/// Given the address expression, retrieve the value it's pointing to. Assume
+/// that value is itself an address, and return the corresponding symbol.
+static SymbolRef getAsPointeeSymbol(const Expr *Expr,
+                                    CheckerContext &C) {
+  const ProgramState *State = C.getState();
+  SVal ArgV = State->getSVal(Expr);
+
+  if (const loc::MemRegionVal *X = dyn_cast<loc::MemRegionVal>(&ArgV)) {
+    StoreManager& SM = C.getStoreManager();
+    const MemRegion *V = SM.Retrieve(State->getStore(), *X).getAsRegion();
+    if (V)
+      return getSymbolForRegion(C, V);
+  }
+  return 0;
+}
+
+// When checking for error code, we need to consider the following cases:
+// 1) noErr / [0]
+// 2) someErr / [1, inf]
+// 3) unknown
+// If noError, returns true iff (1).
+// If !noError, returns true iff (2).
+bool MacOSKeychainAPIChecker::definitelyReturnedError(SymbolRef RetSym,
+                                                      const ProgramState *State,
+                                                      SValBuilder &Builder,
+                                                      bool noError) const {
+  DefinedOrUnknownSVal NoErrVal = Builder.makeIntVal(NoErr,
+    Builder.getSymbolManager().getType(RetSym));
+  DefinedOrUnknownSVal NoErr = Builder.evalEQ(State, NoErrVal,
+                                                     nonloc::SymbolVal(RetSym));
+  const ProgramState *ErrState = State->assume(NoErr, noError);
+  if (ErrState == State) {
+    return true;
+  }
+
+  return false;
+}
+
+// Report deallocator mismatch. Remove the region from tracking - reporting a
+// missing free error after this one is redundant.
+void MacOSKeychainAPIChecker::
+  generateDeallocatorMismatchReport(const AllocationPair &AP,
+                                    const Expr *ArgExpr,
+                                    CheckerContext &C) const {
+  const ProgramState *State = C.getState();
+  State = State->remove<AllocatedData>(AP.first);
+  ExplodedNode *N = C.generateNode(State);
+
+  if (!N)
+    return;
+  initBugType();
+  llvm::SmallString<80> sbuf;
+  llvm::raw_svector_ostream os(sbuf);
+  unsigned int PDeallocIdx =
+               FunctionsToTrack[AP.second->AllocatorIdx].DeallocatorIdx;
+
+  os << "Deallocator doesn't match the allocator: '"
+     << FunctionsToTrack[PDeallocIdx].Name << "' should be used.";
+  BugReport *Report = new BugReport(*BT, os.str(), N);
+  Report->addVisitor(new SecKeychainBugVisitor(AP.first));
+  Report->addRange(ArgExpr->getSourceRange());
+  C.EmitReport(Report);
+}
+
+void MacOSKeychainAPIChecker::checkPreStmt(const CallExpr *CE,
+                                           CheckerContext &C) const {
+  const ProgramState *State = C.getState();
+  const Expr *Callee = CE->getCallee();
+  SVal L = State->getSVal(Callee);
+  unsigned idx = InvalidIdx;
+
+  const FunctionDecl *funDecl = L.getAsFunctionDecl();
+  if (!funDecl)
+    return;
+  IdentifierInfo *funI = funDecl->getIdentifier();
+  if (!funI)
+    return;
+  StringRef funName = funI->getName();
+
+  // If it is a call to an allocator function, it could be a double allocation.
+  idx = getTrackedFunctionIndex(funName, true);
+  if (idx != InvalidIdx) {
+    const Expr *ArgExpr = CE->getArg(FunctionsToTrack[idx].Param);
+    if (SymbolRef V = getAsPointeeSymbol(ArgExpr, C))
+      if (const AllocationState *AS = State->get<AllocatedData>(V)) {
+        if (!definitelyReturnedError(AS->Region, State, C.getSValBuilder())) {
+          // Remove the value from the state. The new symbol will be added for
+          // tracking when the second allocator is processed in checkPostStmt().
+          State = State->remove<AllocatedData>(V);
+          ExplodedNode *N = C.generateNode(State);
+          if (!N)
+            return;
+          initBugType();
+          llvm::SmallString<128> sbuf;
+          llvm::raw_svector_ostream os(sbuf);
+          unsigned int DIdx = FunctionsToTrack[AS->AllocatorIdx].DeallocatorIdx;
+          os << "Allocated data should be released before another call to "
+              << "the allocator: missing a call to '"
+              << FunctionsToTrack[DIdx].Name
+              << "'.";
+          BugReport *Report = new BugReport(*BT, os.str(), N);
+          Report->addVisitor(new SecKeychainBugVisitor(V));
+          Report->addRange(ArgExpr->getSourceRange());
+          C.EmitReport(Report);
+        }
+      }
+    return;
+  }
+
+  // Is it a call to one of deallocator functions?
+  idx = getTrackedFunctionIndex(funName, false);
+  if (idx == InvalidIdx)
+    return;
+
+  // Check the argument to the deallocator.
+  const Expr *ArgExpr = CE->getArg(FunctionsToTrack[idx].Param);
+  SVal ArgSVal = State->getSVal(ArgExpr);
+
+  // Undef is reported by another checker.
+  if (ArgSVal.isUndef())
+    return;
+
+  const MemRegion *Arg = ArgSVal.getAsRegion();
+  if (!Arg)
+    return;
+
+  SymbolRef ArgSM = getSymbolForRegion(C, Arg);
+  bool RegionArgIsBad = ArgSM ? false : isBadDeallocationArgument(Arg);
+  // If the argument is coming from the heap, globals, or unknown, do not
+  // report it.
+  if (!ArgSM && !RegionArgIsBad)
+    return;
+
+  // Is the argument to the call being tracked?
+  const AllocationState *AS = State->get<AllocatedData>(ArgSM);
+  if (!AS && FunctionsToTrack[idx].Kind != ValidAPI) {
+    return;
+  }
+  // If trying to free data which has not been allocated yet, report as a bug.
+  // TODO: We might want a more precise diagnostic for double free
+  // (that would involve tracking all the freed symbols in the checker state).
+  if (!AS || RegionArgIsBad) {
+    // It is possible that this is a false positive - the argument might
+    // have entered as an enclosing function parameter.
+    if (isEnclosingFunctionParam(ArgExpr))
+      return;
+
+    ExplodedNode *N = C.generateNode(State);
+    if (!N)
+      return;
+    initBugType();
+    BugReport *Report = new BugReport(*BT,
+        "Trying to free data which has not been allocated.", N);
+    Report->addRange(ArgExpr->getSourceRange());
+    C.EmitReport(Report);
+    return;
+  }
+
+  // Process functions which might deallocate.
+  if (FunctionsToTrack[idx].Kind == PossibleAPI) {
+
+    if (funName == "CFStringCreateWithBytesNoCopy") {
+      const Expr *DeallocatorExpr = CE->getArg(5)->IgnoreParenCasts();
+      // NULL ~ default deallocator, so warn.
+      if (DeallocatorExpr->isNullPointerConstant(C.getASTContext(),
+          Expr::NPC_ValueDependentIsNotNull)) {
+        const AllocationPair AP = std::make_pair(ArgSM, AS);
+        generateDeallocatorMismatchReport(AP, ArgExpr, C);
+        return;
+      }
+      // One of the default allocators, so warn.
+      if (const DeclRefExpr *DE = dyn_cast<DeclRefExpr>(DeallocatorExpr)) {
+        StringRef DeallocatorName = DE->getFoundDecl()->getName();
+        if (DeallocatorName == "kCFAllocatorDefault" ||
+            DeallocatorName == "kCFAllocatorSystemDefault" ||
+            DeallocatorName == "kCFAllocatorMalloc") {
+          const AllocationPair AP = std::make_pair(ArgSM, AS);
+          generateDeallocatorMismatchReport(AP, ArgExpr, C);
+          return;
+        }
+        // If kCFAllocatorNull, which does not deallocate, we still have to
+        // find the deallocator. Otherwise, assume that the user had written a
+        // custom deallocator which does the right thing.
+        if (DE->getFoundDecl()->getName() != "kCFAllocatorNull") {
+          State = State->remove<AllocatedData>(ArgSM);
+          C.addTransition(State);
+          return;
+        }
+      }
+    }
+    return;
+  }
+
+  // The call is deallocating a value we previously allocated, so remove it
+  // from the next state.
+  State = State->remove<AllocatedData>(ArgSM);
+
+  // Check if the proper deallocator is used.
+  unsigned int PDeallocIdx = FunctionsToTrack[AS->AllocatorIdx].DeallocatorIdx;
+  if (PDeallocIdx != idx || (FunctionsToTrack[idx].Kind == ErrorAPI)) {
+    const AllocationPair AP = std::make_pair(ArgSM, AS);
+    generateDeallocatorMismatchReport(AP, ArgExpr, C);
+    return;
+  }
+
+  // If the return status is undefined or is error, report a bad call to free.
+  if (!definitelyDidnotReturnError(AS->Region, State, C.getSValBuilder())) {
+    ExplodedNode *N = C.generateNode(State);
+    if (!N)
+      return;
+    initBugType();
+    BugReport *Report = new BugReport(*BT,
+        "Call to free data when error was returned during allocation.", N);
+    Report->addVisitor(new SecKeychainBugVisitor(ArgSM));
+    Report->addRange(ArgExpr->getSourceRange());
+    C.EmitReport(Report);
+    return;
+  }
+
+  C.addTransition(State);
+}
+
+void MacOSKeychainAPIChecker::checkPostStmt(const CallExpr *CE,
+                                            CheckerContext &C) const {
+  const ProgramState *State = C.getState();
+  const Expr *Callee = CE->getCallee();
+  SVal L = State->getSVal(Callee);
+
+  const FunctionDecl *funDecl = L.getAsFunctionDecl();
+  if (!funDecl)
+    return;
+  IdentifierInfo *funI = funDecl->getIdentifier();
+  if (!funI)
+    return;
+  StringRef funName = funI->getName();
+
+  // If a value has been allocated, add it to the set for tracking.
+  unsigned idx = getTrackedFunctionIndex(funName, true);
+  if (idx == InvalidIdx)
+    return;
+
+  const Expr *ArgExpr = CE->getArg(FunctionsToTrack[idx].Param);
+  // If the argument entered as an enclosing function parameter, skip it to
+  // avoid false positives.
+  if (isEnclosingFunctionParam(ArgExpr))
+    return;
+
+  if (SymbolRef V = getAsPointeeSymbol(ArgExpr, C)) {
+    // If the argument points to something that's not a symbolic region, it
+    // can be:
+    //  - unknown (cannot reason about it)
+    //  - undefined (already reported by other checker)
+    //  - constant (null - should not be tracked,
+    //              other constant will generate a compiler warning)
+    //  - goto (should be reported by other checker)
+
+    // The call return value symbol should stay alive for as long as the
+    // allocated value symbol, since our diagnostics depend on the value
+    // returned by the call. Ex: Data should only be freed if noErr was
+    // returned during allocation.)
+    SymbolRef RetStatusSymbol = State->getSVal(CE).getAsSymbol();
+    C.getSymbolManager().addSymbolDependency(V, RetStatusSymbol);
+
+    // Track the allocated value in the checker state.
+    State = State->set<AllocatedData>(V, AllocationState(ArgExpr, idx,
+                                                         RetStatusSymbol));
+    assert(State);
+    C.addTransition(State);
+  }
+}
+
+void MacOSKeychainAPIChecker::checkPreStmt(const ReturnStmt *S,
+                                           CheckerContext &C) const {
+  const Expr *retExpr = S->getRetValue();
+  if (!retExpr)
+    return;
+
+  // Check  if the value is escaping through the return.
+  const ProgramState *state = C.getState();
+  const MemRegion *V = state->getSVal(retExpr).getAsRegion();
+  if (!V)
+    return;
+  state = state->remove<AllocatedData>(getSymbolForRegion(C, V));
+
+  // Proceed from the new state.
+  C.addTransition(state);
+}
+
+BugReport *MacOSKeychainAPIChecker::
+  generateAllocatedDataNotReleasedReport(const AllocationPair &AP,
+                                         ExplodedNode *N) const {
+  const ADFunctionInfo &FI = FunctionsToTrack[AP.second->AllocatorIdx];
+  initBugType();
+  llvm::SmallString<70> sbuf;
+  llvm::raw_svector_ostream os(sbuf);
+
+  os << "Allocated data is not released: missing a call to '"
+      << FunctionsToTrack[FI.DeallocatorIdx].Name << "'.";
+  BugReport *Report = new BugReport(*BT, os.str(), N);
+  Report->addVisitor(new SecKeychainBugVisitor(AP.first));
+  Report->addRange(SourceRange());
+  return Report;
+}
+
+void MacOSKeychainAPIChecker::checkDeadSymbols(SymbolReaper &SR,
+                                               CheckerContext &C) const {
+  const ProgramState *State = C.getState();
+  AllocatedSetTy ASet = State->get<AllocatedData>();
+  if (ASet.isEmpty())
+    return;
+
+  bool Changed = false;
+  AllocationPairVec Errors;
+  for (AllocatedSetTy::iterator I = ASet.begin(), E = ASet.end(); I != E; ++I) {
+    if (SR.isLive(I->first))
+      continue;
+
+    Changed = true;
+    State = State->remove<AllocatedData>(I->first);
+    // If the allocated symbol is null or if the allocation call might have
+    // returned an error, do not report.
+    if (State->getSymVal(I->first) ||
+        definitelyReturnedError(I->second.Region, State, C.getSValBuilder()))
+      continue;
+    Errors.push_back(std::make_pair(I->first, &I->second));
+  }
+  if (!Changed)
+    return;
+
+  // Generate the new, cleaned up state.
+  ExplodedNode *N = C.generateNode(State);
+  if (!N)
+    return;
+
+  // Generate the error reports.
+  for (AllocationPairVec::iterator I = Errors.begin(), E = Errors.end();
+                                                       I != E; ++I) {
+    C.EmitReport(generateAllocatedDataNotReleasedReport(*I, N));
+  }
+}
+
+// TODO: Remove this after we ensure that checkDeadSymbols are always called.
+void MacOSKeychainAPIChecker::checkEndPath(EndOfFunctionNodeBuilder &B,
+                                           ExprEngine &Eng) const {
+  const ProgramState *state = B.getState();
+  AllocatedSetTy AS = state->get<AllocatedData>();
+  if (AS.isEmpty())
+    return;
+
+  // Anything which has been allocated but not freed (nor escaped) will be
+  // found here, so report it.
+  bool Changed = false;
+  AllocationPairVec Errors;
+  for (AllocatedSetTy::iterator I = AS.begin(), E = AS.end(); I != E; ++I ) {
+    Changed = true;
+    state = state->remove<AllocatedData>(I->first);
+    // If the allocated symbol is null or if error code was returned at
+    // allocation, do not report.
+    if (state->getSymVal(I.getKey()) ||
+        definitelyReturnedError(I->second.Region, state,
+                                Eng.getSValBuilder())) {
+      continue;
+    }
+    Errors.push_back(std::make_pair(I->first, &I->second));
+  }
+
+  // If no change, do not generate a new state.
+  if (!Changed)
+    return;
+
+  ExplodedNode *N = B.generateNode(state);
+  if (!N)
+    return;
+
+  // Generate the error reports.
+  for (AllocationPairVec::iterator I = Errors.begin(), E = Errors.end();
+                                                       I != E; ++I) {
+    Eng.getBugReporter().EmitReport(
+      generateAllocatedDataNotReleasedReport(*I, N));
+  }
+}
+
+
+PathDiagnosticPiece *MacOSKeychainAPIChecker::SecKeychainBugVisitor::VisitNode(
+                                                      const ExplodedNode *N,
+                                                      const ExplodedNode *PrevN,
+                                                      BugReporterContext &BRC,
+                                                      BugReport &BR) {
+  const AllocationState *AS = N->getState()->get<AllocatedData>(Sym);
+  if (!AS)
+    return 0;
+  const AllocationState *ASPrev = PrevN->getState()->get<AllocatedData>(Sym);
+  if (ASPrev)
+    return 0;
+
+  // (!ASPrev && AS) ~ We started tracking symbol in node N, it must be the
+  // allocation site.
+  const CallExpr *CE = cast<CallExpr>(cast<StmtPoint>(N->getLocation())
+                                                            .getStmt());
+  const FunctionDecl *funDecl = CE->getDirectCallee();
+  assert(funDecl && "We do not support indirect function calls as of now.");
+  StringRef funName = funDecl->getName();
+
+  // Get the expression of the corresponding argument.
+  unsigned Idx = getTrackedFunctionIndex(funName, true);
+  assert(Idx != InvalidIdx && "This should be a call to an allocator.");
+  const Expr *ArgExpr = CE->getArg(FunctionsToTrack[Idx].Param);
+  PathDiagnosticLocation Pos(ArgExpr, BRC.getSourceManager(),
+                             N->getLocationContext());
+  return new PathDiagnosticEventPiece(Pos, "Data is allocated here.");
+}
+
+void ento::registerMacOSKeychainAPIChecker(CheckerManager &mgr) {
+  mgr.registerChecker<MacOSKeychainAPIChecker>();
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
index f8d076b54e3d..88d492e8d9a2 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
@@ -20,7 +20,7 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -56,7 +56,7 @@ void MacOSXAPIChecker::CheckDispatchOnce(CheckerContext &C, const CallExpr *CE,
 
   // Check if the first argument is stack allocated.  If so, issue a warning
   // because that's likely to be bad news.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const MemRegion *R = state->getSVal(CE->getArg(0)).getAsRegion();
   if (!R || !isa<StackSpaceRegion>(R->getMemorySpace()))
     return;
@@ -81,7 +81,7 @@ void MacOSXAPIChecker::CheckDispatchOnce(CheckerContext &C, const CallExpr *CE,
   if (isa<VarRegion>(R) && isa<StackLocalsSpaceRegion>(R->getMemorySpace()))
     os << "  Perhaps you intended to declare the variable as 'static'?";
 
-  RangedBugReport *report = new RangedBugReport(*BT_dispatchOnce, os.str(), N);
+  BugReport *report = new BugReport(*BT_dispatchOnce, os.str(), N);
   report->addRange(CE->getArg(0)->getSourceRange());
   C.EmitReport(report);
 }
@@ -94,7 +94,7 @@ void MacOSXAPIChecker::checkPreStmt(const CallExpr *CE,
                                     CheckerContext &C) const {
   // FIXME: This sort of logic is common to several checkers, including
   // UnixAPIChecker, PthreadLockChecker, and CStringChecker.  Should refactor.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   const FunctionDecl *Fn = state->getSVal(Callee).getAsFunctionDecl();
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Makefile b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Makefile
deleted file mode 100644
index 97f46424e472..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/Makefile
+++ /dev/null
@@ -1,24 +0,0 @@
-##===- clang/lib/Checker/Makefile --------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-#
-# This implements analyses built on top of source-level CFGs. 
-#
-##===----------------------------------------------------------------------===##
-
-CLANG_LEVEL := ../../..
-LIBRARYNAME := clangStaticAnalyzerCheckers
-
-BUILT_SOURCES = Checkers.inc
-TABLEGEN_INC_FILES_COMMON = 1
-
-include $(CLANG_LEVEL)/Makefile
-
-$(ObjDir)/Checkers.inc.tmp : Checkers.td $(PROJ_SRC_DIR)/$(CLANG_LEVEL)/include/clang/StaticAnalyzer/Checkers/CheckerBase.td $(TBLGEN) $(ObjDir)/.dir
-	$(Echo) "Building Clang SA Checkers tables with tblgen"
-	$(Verb) $(TableGen) -gen-clang-sa-checkers -I $(PROJ_SRC_DIR)/$(CLANG_LEVEL)/include -o $(call SYSPATH, $@) $<
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 91002158c57f..5631802b7c34 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -17,8 +17,8 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/ImmutableMap.h"
 using namespace clang;
@@ -80,35 +80,37 @@ class MallocChecker : public Checker<eval::Call, check::DeadSymbols, check::EndP
   void checkDeadSymbols(SymbolReaper &SymReaper, CheckerContext &C) const;
   void checkEndPath(EndOfFunctionNodeBuilder &B, ExprEngine &Eng) const;
   void checkPreStmt(const ReturnStmt *S, CheckerContext &C) const;
-  const GRState *evalAssume(const GRState *state, SVal Cond,
+  const ProgramState *evalAssume(const ProgramState *state, SVal Cond,
                             bool Assumption) const;
-  void checkLocation(SVal l, bool isLoad, CheckerContext &C) const;
-  void checkBind(SVal location, SVal val, CheckerContext &C) const;
+  void checkLocation(SVal l, bool isLoad, const Stmt *S,
+                     CheckerContext &C) const;
+  void checkBind(SVal location, SVal val, const Stmt*S,
+                 CheckerContext &C) const;
 
 private:
   static void MallocMem(CheckerContext &C, const CallExpr *CE);
   static void MallocMemReturnsAttr(CheckerContext &C, const CallExpr *CE,
                                    const OwnershipAttr* Att);
-  static const GRState *MallocMemAux(CheckerContext &C, const CallExpr *CE,
+  static const ProgramState *MallocMemAux(CheckerContext &C, const CallExpr *CE,
                                      const Expr *SizeEx, SVal Init,
-                                     const GRState *state) {
+                                     const ProgramState *state) {
     return MallocMemAux(C, CE, state->getSVal(SizeEx), Init, state);
   }
-  static const GRState *MallocMemAux(CheckerContext &C, const CallExpr *CE,
+  static const ProgramState *MallocMemAux(CheckerContext &C, const CallExpr *CE,
                                      SVal SizeEx, SVal Init,
-                                     const GRState *state);
+                                     const ProgramState *state);
 
   void FreeMem(CheckerContext &C, const CallExpr *CE) const;
   void FreeMemAttr(CheckerContext &C, const CallExpr *CE,
                    const OwnershipAttr* Att) const;
-  const GRState *FreeMemAux(CheckerContext &C, const CallExpr *CE,
-                           const GRState *state, unsigned Num, bool Hold) const;
+  const ProgramState *FreeMemAux(CheckerContext &C, const CallExpr *CE,
+                           const ProgramState *state, unsigned Num, bool Hold) const;
 
   void ReallocMem(CheckerContext &C, const CallExpr *CE) const;
   static void CallocMem(CheckerContext &C, const CallExpr *CE);
   
-  static bool SummarizeValue(llvm::raw_ostream& os, SVal V);
-  static bool SummarizeRegion(llvm::raw_ostream& os, const MemRegion *MR);
+  static bool SummarizeValue(raw_ostream &os, SVal V);
+  static bool SummarizeRegion(raw_ostream &os, const MemRegion *MR);
   void ReportBadFree(CheckerContext &C, SVal ArgVal, SourceRange range) const;
 };
 } // end anonymous namespace
@@ -118,15 +120,15 @@ typedef llvm::ImmutableMap<SymbolRef, RefState> RegionStateTy;
 namespace clang {
 namespace ento {
   template <>
-  struct GRStateTrait<RegionState> 
-    : public GRStatePartialTrait<RegionStateTy> {
+  struct ProgramStateTrait<RegionState> 
+    : public ProgramStatePartialTrait<RegionStateTy> {
     static void *GDMIndex() { static int x; return &x; }
   };
 }
 }
 
 bool MallocChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
 
@@ -193,7 +195,7 @@ bool MallocChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
 }
 
 void MallocChecker::MallocMem(CheckerContext &C, const CallExpr *CE) {
-  const GRState *state = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(),
+  const ProgramState *state = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(),
                                       C.getState());
   C.addTransition(state);
 }
@@ -205,21 +207,21 @@ void MallocChecker::MallocMemReturnsAttr(CheckerContext &C, const CallExpr *CE,
 
   OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
   if (I != E) {
-    const GRState *state =
+    const ProgramState *state =
         MallocMemAux(C, CE, CE->getArg(*I), UndefinedVal(), C.getState());
     C.addTransition(state);
     return;
   }
-  const GRState *state = MallocMemAux(C, CE, UnknownVal(), UndefinedVal(),
+  const ProgramState *state = MallocMemAux(C, CE, UnknownVal(), UndefinedVal(),
                                         C.getState());
   C.addTransition(state);
 }
 
-const GRState *MallocChecker::MallocMemAux(CheckerContext &C,  
+const ProgramState *MallocChecker::MallocMemAux(CheckerContext &C,  
                                            const CallExpr *CE,
                                            SVal Size, SVal Init,
-                                           const GRState *state) {
-  unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+                                           const ProgramState *state) {
+  unsigned Count = C.getCurrentBlockCount();
   SValBuilder &svalBuilder = C.getSValBuilder();
 
   // Set the return value.
@@ -247,7 +249,7 @@ const GRState *MallocChecker::MallocMemAux(CheckerContext &C,
 }
 
 void MallocChecker::FreeMem(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = FreeMemAux(C, CE, C.getState(), 0, false);
+  const ProgramState *state = FreeMemAux(C, CE, C.getState(), 0, false);
 
   if (state)
     C.addTransition(state);
@@ -260,15 +262,15 @@ void MallocChecker::FreeMemAttr(CheckerContext &C, const CallExpr *CE,
 
   for (OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
        I != E; ++I) {
-    const GRState *state = FreeMemAux(C, CE, C.getState(), *I,
+    const ProgramState *state = FreeMemAux(C, CE, C.getState(), *I,
                                       Att->getOwnKind() == OwnershipAttr::Holds);
     if (state)
       C.addTransition(state);
   }
 }
 
-const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
-                                         const GRState *state, unsigned Num,
+const ProgramState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
+                                         const ProgramState *state, unsigned Num,
                                          bool Hold) const {
   const Expr *ArgExpr = CE->getArg(Num);
   SVal ArgVal = state->getSVal(ArgExpr);
@@ -281,7 +283,7 @@ const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
 
   // FIXME: Technically using 'Assume' here can result in a path
   //  bifurcation.  In such cases we need to return two states, not just one.
-  const GRState *notNullState, *nullState;
+  const ProgramState *notNullState, *nullState;
   llvm::tie(notNullState, nullState) = state->assume(location);
 
   // The explicit NULL case, no operation is performed.
@@ -364,7 +366,7 @@ const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
   return notNullState->set<RegionState>(Sym, RefState::getReleased(CE));
 }
 
-bool MallocChecker::SummarizeValue(llvm::raw_ostream& os, SVal V) {
+bool MallocChecker::SummarizeValue(raw_ostream &os, SVal V) {
   if (nonloc::ConcreteInt *IntVal = dyn_cast<nonloc::ConcreteInt>(&V))
     os << "an integer (" << IntVal->getValue() << ")";
   else if (loc::ConcreteInt *ConstAddr = dyn_cast<loc::ConcreteInt>(&V))
@@ -377,13 +379,13 @@ bool MallocChecker::SummarizeValue(llvm::raw_ostream& os, SVal V) {
   return true;
 }
 
-bool MallocChecker::SummarizeRegion(llvm::raw_ostream& os,
+bool MallocChecker::SummarizeRegion(raw_ostream &os,
                                     const MemRegion *MR) {
   switch (MR->getKind()) {
   case MemRegion::FunctionTextRegionKind: {
     const FunctionDecl *FD = cast<FunctionTextRegion>(MR)->getDecl();
     if (FD)
-      os << "the address of the function '" << FD << "'";
+      os << "the address of the function '" << *FD << '\'';
     else
       os << "the address of a function";
     return true;
@@ -484,14 +486,14 @@ void MallocChecker::ReportBadFree(CheckerContext &C, SVal ArgVal,
         os << "not memory allocated by malloc()";
     }
     
-    EnhancedBugReport *R = new EnhancedBugReport(*BT_BadFree, os.str(), N);
+    BugReport *R = new BugReport(*BT_BadFree, os.str(), N);
     R->addRange(range);
     C.EmitReport(R);
   }
 }
 
 void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *arg0Expr = CE->getArg(0);
   DefinedOrUnknownSVal arg0Val 
     = cast<DefinedOrUnknownSVal>(state->getSVal(arg0Expr));
@@ -517,7 +519,7 @@ void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) const {
 
   // If the ptr is NULL and the size is not 0, the call is equivalent to 
   // malloc(size).
-  const GRState *stateEqual = state->assume(PtrEQ, true);
+  const ProgramState *stateEqual = state->assume(PtrEQ, true);
   if (stateEqual && state->assume(SizeZero, false)) {
     // Hack: set the NULL symbolic region to released to suppress false warning.
     // In the future we should add more states for allocated regions, e.g., 
@@ -527,28 +529,25 @@ void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) const {
     if (Sym)
       stateEqual = stateEqual->set<RegionState>(Sym, RefState::getReleased(CE));
 
-    const GRState *stateMalloc = MallocMemAux(C, CE, CE->getArg(1), 
+    const ProgramState *stateMalloc = MallocMemAux(C, CE, CE->getArg(1), 
                                               UndefinedVal(), stateEqual);
     C.addTransition(stateMalloc);
   }
 
-  if (const GRState *stateNotEqual = state->assume(PtrEQ, false)) {
+  if (const ProgramState *stateNotEqual = state->assume(PtrEQ, false)) {
     // If the size is 0, free the memory.
-    if (const GRState *stateSizeZero = stateNotEqual->assume(SizeZero, true))
-      if (const GRState *stateFree = 
+    if (const ProgramState *stateSizeZero = stateNotEqual->assume(SizeZero, true))
+      if (const ProgramState *stateFree = 
           FreeMemAux(C, CE, stateSizeZero, 0, false)) {
 
-        // Add the state transition to set input pointer argument to be free.
-        C.addTransition(stateFree);
-
-        // Bind the return value to UndefinedVal because it is now free.
-        C.addTransition(stateFree->BindExpr(CE, UndefinedVal(), true));
+        // Bind the return value to NULL because it is now free.
+        C.addTransition(stateFree->BindExpr(CE, svalBuilder.makeNull(), true));
       }
-    if (const GRState *stateSizeNotZero = stateNotEqual->assume(SizeZero,false))
-      if (const GRState *stateFree = FreeMemAux(C, CE, stateSizeNotZero,
+    if (const ProgramState *stateSizeNotZero = stateNotEqual->assume(SizeZero,false))
+      if (const ProgramState *stateFree = FreeMemAux(C, CE, stateSizeNotZero,
                                                 0, false)) {
         // FIXME: We should copy the content of the original buffer.
-        const GRState *stateRealloc = MallocMemAux(C, CE, CE->getArg(1), 
+        const ProgramState *stateRealloc = MallocMemAux(C, CE, CE->getArg(1), 
                                                    UnknownVal(), stateFree);
         C.addTransition(stateRealloc);
       }
@@ -556,7 +555,7 @@ void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) const {
 }
 
 void MallocChecker::CallocMem(CheckerContext &C, const CallExpr *CE) {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SValBuilder &svalBuilder = C.getSValBuilder();
 
   SVal count = state->getSVal(CE->getArg(0));
@@ -574,33 +573,40 @@ void MallocChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   if (!SymReaper.hasDeadSymbols())
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   RegionStateTy RS = state->get<RegionState>();
   RegionStateTy::Factory &F = state->get_context<RegionState>();
 
+  bool generateReport = false;
+  
   for (RegionStateTy::iterator I = RS.begin(), E = RS.end(); I != E; ++I) {
     if (SymReaper.isDead(I->first)) {
-      if (I->second.isAllocated()) {
-        if (ExplodedNode *N = C.generateNode()) {
-          if (!BT_Leak)
-            BT_Leak.reset(new BuiltinBug("Memory leak",
-                    "Allocated memory never released. Potential memory leak."));
-          // FIXME: where it is allocated.
-          BugReport *R = new BugReport(*BT_Leak, BT_Leak->getDescription(), N);
-          C.EmitReport(R);
-        }
-      }
+      if (I->second.isAllocated())
+        generateReport = true;
 
       // Remove the dead symbol from the map.
       RS = F.remove(RS, I->first);
+
     }
   }
-  C.generateNode(state->set<RegionState>(RS));
+  
+  ExplodedNode *N = C.generateNode(state->set<RegionState>(RS));
+
+  // FIXME: This does not handle when we have multiple leaks at a single
+  // place.
+  if (N && generateReport) {
+    if (!BT_Leak)
+      BT_Leak.reset(new BuiltinBug("Memory leak",
+              "Allocated memory never released. Potential memory leak."));
+    // FIXME: where it is allocated.
+    BugReport *R = new BugReport(*BT_Leak, BT_Leak->getDescription(), N);
+    C.EmitReport(R);
+  }
 }
 
 void MallocChecker::checkEndPath(EndOfFunctionNodeBuilder &B,
                                  ExprEngine &Eng) const {
-  const GRState *state = B.getState();
+  const ProgramState *state = B.getState();
   RegionStateTy M = state->get<RegionState>();
 
   for (RegionStateTy::iterator I = M.begin(), E = M.end(); I != E; ++I) {
@@ -623,7 +629,7 @@ void MallocChecker::checkPreStmt(const ReturnStmt *S, CheckerContext &C) const {
   if (!retExpr)
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   SymbolRef Sym = state->getSVal(retExpr).getAsSymbol();
   if (!Sym)
@@ -640,7 +646,7 @@ void MallocChecker::checkPreStmt(const ReturnStmt *S, CheckerContext &C) const {
   C.addTransition(state);
 }
 
-const GRState *MallocChecker::evalAssume(const GRState *state, SVal Cond, 
+const ProgramState *MallocChecker::evalAssume(const ProgramState *state, SVal Cond, 
                                          bool Assumption) const {
   // If a symblic region is assumed to NULL, set its state to AllocateFailed.
   // FIXME: should also check symbols assumed to non-null.
@@ -657,7 +663,8 @@ const GRState *MallocChecker::evalAssume(const GRState *state, SVal Cond,
 }
 
 // Check if the location is a freed symbolic region.
-void MallocChecker::checkLocation(SVal l, bool isLoad,CheckerContext &C) const {
+void MallocChecker::checkLocation(SVal l, bool isLoad, const Stmt *S,
+                                  CheckerContext &C) const {
   SymbolRef Sym = l.getLocSymbolInBase();
   if (Sym) {
     const RefState *RS = C.getState()->get<RegionState>(Sym);
@@ -675,13 +682,14 @@ void MallocChecker::checkLocation(SVal l, bool isLoad,CheckerContext &C) const {
   }
 }
 
-void MallocChecker::checkBind(SVal location, SVal val,CheckerContext &C) const {
+void MallocChecker::checkBind(SVal location, SVal val,
+                              const Stmt *BindS, CheckerContext &C) const {
   // The PreVisitBind implements the same algorithm as already used by the 
   // Objective C ownership checker: if the pointer escaped from this scope by 
   // assignment, let it go.  However, assigning to fields of a stack-storage 
   // structure does not transfer ownership.
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   DefinedOrUnknownSVal l = cast<DefinedOrUnknownSVal>(location);
 
   // Check for null dereferences.
@@ -694,7 +702,7 @@ void MallocChecker::checkBind(SVal location, SVal val,CheckerContext &C) const {
   if (Sym) {
     if (const RefState *RS = state->get<RegionState>(Sym)) {
       // If ptr is NULL, no operation is performed.
-      const GRState *notNullState, *nullState;
+      const ProgramState *notNullState, *nullState;
       llvm::tie(notNullState, nullState) = state->assume(l);
 
       // Generate a transition for 'nullState' to record the assumption
@@ -724,7 +732,7 @@ void MallocChecker::checkBind(SVal location, SVal val,CheckerContext &C) const {
           // We no longer own this pointer.
           notNullState =
             notNullState->set<RegionState>(Sym,
-                                        RefState::getRelinquished(C.getStmt()));
+                                        RefState::getRelinquished(BindS));
         }
         while (false);
       }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp
new file mode 100644
index 000000000000..cf5501a4ac15
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp
@@ -0,0 +1,268 @@
+// MallocOverflowSecurityChecker.cpp - Check for malloc overflows -*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This checker detects a common memory allocation security flaw.
+// Suppose 'unsigned int n' comes from an untrusted source. If the
+// code looks like 'malloc (n * 4)', and an attacker can make 'n' be
+// say MAX_UINT/4+2, then instead of allocating the correct 'n' 4-byte
+// elements, this will actually allocate only two because of overflow.
+// Then when the rest of the program attempts to store values past the
+// second element, these values will actually overwrite other items in
+// the heap, probably allowing the attacker to execute arbitrary code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/AST/EvaluatedExprVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "llvm/ADT/SmallVector.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+struct MallocOverflowCheck {
+  const BinaryOperator *mulop;
+  const Expr *variable;
+
+  MallocOverflowCheck (const BinaryOperator *m, const Expr *v) 
+    : mulop(m), variable (v)
+  {}
+};
+
+class MallocOverflowSecurityChecker : public Checker<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager &mgr,
+                        BugReporter &BR) const;
+
+  void CheckMallocArgument(
+    llvm::SmallVectorImpl<MallocOverflowCheck> &PossibleMallocOverflows,
+    const Expr *TheArgument, ASTContext &Context) const;
+
+  void OutputPossibleOverflows(
+    llvm::SmallVectorImpl<MallocOverflowCheck> &PossibleMallocOverflows,
+    const Decl *D, BugReporter &BR, AnalysisManager &mgr) const;
+
+};
+} // end anonymous namespace
+
+void MallocOverflowSecurityChecker::CheckMallocArgument(
+  llvm::SmallVectorImpl<MallocOverflowCheck> &PossibleMallocOverflows,
+  const Expr *TheArgument,
+  ASTContext &Context) const {
+
+  /* Look for a linear combination with a single variable, and at least
+   one multiplication.
+   Reject anything that applies to the variable: an explicit cast,
+   conditional expression, an operation that could reduce the range
+   of the result, or anything too complicated :-).  */
+  const Expr * e = TheArgument;
+  const BinaryOperator * mulop = NULL;
+
+  for (;;) {
+    e = e->IgnoreParenImpCasts();
+    if (isa<BinaryOperator>(e)) {
+      const BinaryOperator * binop = dyn_cast<BinaryOperator>(e);
+      BinaryOperatorKind opc = binop->getOpcode();
+      // TODO: ignore multiplications by 1, reject if multiplied by 0.
+      if (mulop == NULL && opc == BO_Mul)
+        mulop = binop;
+      if (opc != BO_Mul && opc != BO_Add && opc != BO_Sub && opc != BO_Shl)
+        return;
+
+      const Expr *lhs = binop->getLHS();
+      const Expr *rhs = binop->getRHS();
+      if (rhs->isEvaluatable(Context))
+        e = lhs;
+      else if ((opc == BO_Add || opc == BO_Mul)
+               && lhs->isEvaluatable(Context))
+        e = rhs;
+      else
+        return;
+    }
+    else if (isa<DeclRefExpr>(e) || isa<MemberExpr>(e))
+      break;
+    else
+      return;
+  }
+
+  if (mulop == NULL)
+    return;
+
+  //  We've found the right structure of malloc argument, now save
+  // the data so when the body of the function is completely available
+  // we can check for comparisons.
+
+  // TODO: Could push this into the innermost scope where 'e' is
+  // defined, rather than the whole function.
+  PossibleMallocOverflows.push_back(MallocOverflowCheck(mulop, e));
+}
+
+namespace {
+// A worker class for OutputPossibleOverflows.
+class CheckOverflowOps :
+  public EvaluatedExprVisitor<CheckOverflowOps> {
+public:
+  typedef llvm::SmallVectorImpl<MallocOverflowCheck> theVecType;
+
+private:
+    theVecType &toScanFor;
+    ASTContext &Context;
+
+    bool isIntZeroExpr(const Expr *E) const {
+      if (!E->getType()->isIntegralOrEnumerationType())
+        return false;
+      llvm::APSInt Result;
+      if (E->EvaluateAsInt(Result, Context))
+        return Result == 0;
+      return false;
+    }
+
+    void CheckExpr(const Expr *E_p) {
+      const Expr *E = E_p->IgnoreParenImpCasts();
+
+      theVecType::iterator i = toScanFor.end();
+      theVecType::iterator e = toScanFor.begin();
+
+      if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E)) {
+        const Decl * EdreD = DR->getDecl();
+        while (i != e) {
+          --i;
+          if (const DeclRefExpr *DR_i = dyn_cast<DeclRefExpr>(i->variable)) {
+            if (DR_i->getDecl() == EdreD)
+              i = toScanFor.erase(i);
+          }
+        }
+      }
+      else if (isa<MemberExpr>(E)) {
+        // No points-to analysis, just look at the member
+        const Decl * EmeMD = dyn_cast<MemberExpr>(E)->getMemberDecl();
+        while (i != e) {
+          --i;
+          if (isa<MemberExpr>(i->variable)) {
+            if (dyn_cast<MemberExpr>(i->variable)->getMemberDecl() == EmeMD)
+              i = toScanFor.erase (i);
+          }
+        }
+      }
+    }
+
+  public:
+    void VisitBinaryOperator(BinaryOperator *E) {
+      if (E->isComparisonOp()) {
+        const Expr * lhs = E->getLHS();
+        const Expr * rhs = E->getRHS();
+        // Ignore comparisons against zero, since they generally don't
+        // protect against an overflow.
+        if (!isIntZeroExpr(lhs) && ! isIntZeroExpr(rhs)) {
+          CheckExpr(lhs);
+          CheckExpr(rhs);
+        }
+      }
+      EvaluatedExprVisitor<CheckOverflowOps>::VisitBinaryOperator(E);
+    }
+
+    /* We specifically ignore loop conditions, because they're typically
+     not error checks.  */
+    void VisitWhileStmt(WhileStmt *S) {
+      return this->Visit(S->getBody());
+    }
+    void VisitForStmt(ForStmt *S) {
+      return this->Visit(S->getBody());
+    }
+    void VisitDoStmt(DoStmt *S) {
+      return this->Visit(S->getBody());
+    }
+
+    CheckOverflowOps(theVecType &v, ASTContext &ctx)
+    : EvaluatedExprVisitor<CheckOverflowOps>(ctx),
+      toScanFor(v), Context(ctx)
+    { }
+  };
+}
+
+// OutputPossibleOverflows - We've found a possible overflow earlier,
+// now check whether Body might contain a comparison which might be
+// preventing the overflow.
+// This doesn't do flow analysis, range analysis, or points-to analysis; it's
+// just a dumb "is there a comparison" scan.  The aim here is to
+// detect the most blatent cases of overflow and educate the
+// programmer.
+void MallocOverflowSecurityChecker::OutputPossibleOverflows(
+  llvm::SmallVectorImpl<MallocOverflowCheck> &PossibleMallocOverflows,
+  const Decl *D, BugReporter &BR, AnalysisManager &mgr) const {
+  // By far the most common case: nothing to check.
+  if (PossibleMallocOverflows.empty())
+    return;
+
+  // Delete any possible overflows which have a comparison.
+  CheckOverflowOps c(PossibleMallocOverflows, BR.getContext());
+  c.Visit(mgr.getAnalysisContext(D)->getBody());
+
+  // Output warnings for all overflows that are left.
+  for (CheckOverflowOps::theVecType::iterator
+       i = PossibleMallocOverflows.begin(),
+       e = PossibleMallocOverflows.end();
+       i != e;
+       ++i) {
+    SourceRange R = i->mulop->getSourceRange();
+    BR.EmitBasicReport("MallocOverflowSecurityChecker",
+      "the computation of the size of the memory allocation may overflow",
+      PathDiagnosticLocation::createOperatorLoc(i->mulop,
+                                                BR.getSourceManager()),
+      &R, 1);
+  }
+}
+
+void MallocOverflowSecurityChecker::checkASTCodeBody(const Decl *D,
+                                             AnalysisManager &mgr,
+                                             BugReporter &BR) const {
+
+  CFG *cfg = mgr.getCFG(D);
+  if (!cfg)
+    return;
+
+  // A list of variables referenced in possibly overflowing malloc operands.
+  llvm::SmallVector<MallocOverflowCheck, 2> PossibleMallocOverflows;
+
+  for (CFG::iterator it = cfg->begin(), ei = cfg->end(); it != ei; ++it) {
+    CFGBlock *block = *it;
+    for (CFGBlock::iterator bi = block->begin(), be = block->end();
+         bi != be; ++bi) {
+      if (const CFGStmt *CS = bi->getAs<CFGStmt>()) {
+        if (const CallExpr *TheCall = dyn_cast<CallExpr>(CS->getStmt())) {
+          // Get the callee.
+          const FunctionDecl *FD = TheCall->getDirectCallee();
+
+          if (!FD)
+            return;
+
+          // Get the name of the callee. If it's a builtin, strip off the prefix.
+          IdentifierInfo *FnInfo = FD->getIdentifier();
+          if (!FnInfo)
+            return;
+
+          if (FnInfo->isStr ("malloc") || FnInfo->isStr ("_MALLOC")) {
+            if (TheCall->getNumArgs() == 1)
+              CheckMallocArgument(PossibleMallocOverflows, TheCall->getArg(0),
+                                  mgr.getASTContext());
+          }
+        }
+      }
+    }
+  }
+
+  OutputPossibleOverflows(PossibleMallocOverflows, D, BR, mgr);
+}
+
+void ento::registerMallocOverflowSecurityChecker(CheckerManager &mgr) {
+  mgr.registerChecker<MallocOverflowSecurityChecker>();
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
index f11db6458cc0..7f74a7d015d6 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
@@ -9,7 +9,7 @@
 //
 //  This file defines a NSAutoreleasePoolChecker, a small checker that warns
 //  about subpar uses of NSAutoreleasePool.  Note that while the check itself
-//  (in it's current form) could be written as a flow-insensitive check, in
+//  (in its current form) could be written as a flow-insensitive check, in
 //  can be potentially enhanced in the future with flow-sensitive information.
 //  It is also a good example of the CheckerVisitor interface. 
 //
@@ -18,9 +18,10 @@
 #include "ClangSACheckers.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Decl.h"
 
@@ -53,7 +54,7 @@ void NSAutoreleasePoolChecker::checkPreObjCMessage(ObjCMessage msg,
   
   if (!PT)
     return;  
-  const ObjCInterfaceDecl* OD = PT->getInterfaceDecl();
+  const ObjCInterfaceDecl *OD = PT->getInterfaceDecl();
   if (!OD)
     return;  
   if (!OD->getIdentifier()->getName().equals("NSAutoreleasePool"))
@@ -66,14 +67,18 @@ void NSAutoreleasePoolChecker::checkPreObjCMessage(ObjCMessage msg,
     return;
                      
   SourceRange R = msg.getSourceRange();
-
+  BugReporter &BR = C.getBugReporter();
+  const LocationContext *LC = C.getPredecessor()->getLocationContext();
+  const SourceManager &SM = BR.getSourceManager();
+  const Expr *E = msg.getMsgOrPropExpr();
+  PathDiagnosticLocation L = PathDiagnosticLocation::createBegin(E, SM, LC);
   C.getBugReporter().EmitBasicReport("Use -drain instead of -release",
     "API Upgrade (Apple)",
     "Use -drain instead of -release when using NSAutoreleasePool "
-    "and garbage collection", R.getBegin(), &R, 1);
+    "and garbage collection", L, &R, 1);
 }
 
 void ento::registerNSAutoreleasePoolChecker(CheckerManager &mgr) {
-  if (mgr.getLangOptions().getGCMode() != LangOptions::NonGC)
+  if (mgr.getLangOptions().getGC() != LangOptions::NonGC)
     mgr.registerChecker<NSAutoreleasePoolChecker>();
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
index a51d8e0d19ae..56789983593e 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
@@ -19,7 +19,7 @@
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Decl.h"
@@ -60,7 +60,7 @@ void NSErrorMethodChecker::checkASTDecl(const ObjCMethodDecl *D,
     II = &D->getASTContext().Idents.get("NSError"); 
 
   bool hasNSError = false;
-  for (ObjCMethodDecl::param_iterator
+  for (ObjCMethodDecl::param_const_iterator
          I = D->param_begin(), E = D->param_end(); I != E; ++I)  {
     if (IsNSError((*I)->getType(), II)) {
       hasNSError = true;
@@ -72,8 +72,10 @@ void NSErrorMethodChecker::checkASTDecl(const ObjCMethodDecl *D,
     const char *err = "Method accepting NSError** "
         "should have a non-void return value to indicate whether or not an "
         "error occurred";
+    PathDiagnosticLocation L =
+      PathDiagnosticLocation::create(D, BR.getSourceManager());
     BR.EmitBasicReport("Bad return type when passing NSError**",
-                       "Coding conventions (Apple)", err, D->getLocation());
+                       "Coding conventions (Apple)", err, L);
   }
 }
 
@@ -118,8 +120,10 @@ void CFErrorFunctionChecker::checkASTDecl(const FunctionDecl *D,
     const char *err = "Function accepting CFErrorRef* "
         "should have a non-void return value to indicate whether or not an "
         "error occurred";
+    PathDiagnosticLocation L =
+      PathDiagnosticLocation::create(D, BR.getSourceManager());
     BR.EmitBasicReport("Bad return type when passing CFErrorRef*",
-                       "Coding conventions (Apple)", err, D->getLocation());
+                       "Coding conventions (Apple)", err, L);
   }
 }
 
@@ -153,7 +157,8 @@ class NSOrCFErrorDerefChecker
   NSOrCFErrorDerefChecker() : NSErrorII(0), CFErrorII(0),
                               ShouldCheckNSError(0), ShouldCheckCFError(0) { }
 
-  void checkLocation(SVal loc, bool isLoad, CheckerContext &C) const;
+  void checkLocation(SVal loc, bool isLoad, const Stmt *S,
+                     CheckerContext &C) const;
   void checkEvent(ImplicitNullDerefEvent event) const;
 };
 }
@@ -166,18 +171,18 @@ typedef llvm::ImmutableMap<SymbolRef, unsigned> ErrorOutFlag;
 namespace clang {
 namespace ento {
   template <>
-  struct GRStateTrait<NSErrorOut> : public GRStatePartialTrait<ErrorOutFlag> {  
+  struct ProgramStateTrait<NSErrorOut> : public ProgramStatePartialTrait<ErrorOutFlag> {  
     static void *GDMIndex() { static int index = 0; return &index; }
   };
   template <>
-  struct GRStateTrait<CFErrorOut> : public GRStatePartialTrait<ErrorOutFlag> {  
+  struct ProgramStateTrait<CFErrorOut> : public ProgramStatePartialTrait<ErrorOutFlag> {  
     static void *GDMIndex() { static int index = 0; return &index; }
   };
 }
 }
 
 template <typename T>
-static bool hasFlag(SVal val, const GRState *state) {
+static bool hasFlag(SVal val, const ProgramState *state) {
   if (SymbolRef sym = val.getAsSymbol())
     if (const unsigned *attachedFlags = state->get<T>(sym))
       return *attachedFlags;
@@ -185,7 +190,7 @@ static bool hasFlag(SVal val, const GRState *state) {
 }
 
 template <typename T>
-static void setFlag(const GRState *state, SVal val, CheckerContext &C) {
+static void setFlag(const ProgramState *state, SVal val, CheckerContext &C) {
   // We tag the symbol that the SVal wraps.
   if (SymbolRef sym = val.getAsSymbol())
     C.addTransition(state->set<T>(sym, true));
@@ -207,6 +212,7 @@ static QualType parameterTypeFromSVal(SVal val, CheckerContext &C) {
 }
 
 void NSOrCFErrorDerefChecker::checkLocation(SVal loc, bool isLoad,
+                                            const Stmt *S,
                                             CheckerContext &C) const {
   if (!isLoad)
     return;
@@ -214,7 +220,7 @@ void NSOrCFErrorDerefChecker::checkLocation(SVal loc, bool isLoad,
     return;
 
   ASTContext &Ctx = C.getASTContext();
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   // If we are loading from NSError**/CFErrorRef* parameter, mark the resulting
   // SVal so that we can later check it when handling the
@@ -247,7 +253,7 @@ void NSOrCFErrorDerefChecker::checkEvent(ImplicitNullDerefEvent event) const {
     return;
 
   SVal loc = event.Location;
-  const GRState *state = event.SinkNode->getState();
+  const ProgramState *state = event.SinkNode->getState();
   BugReporter &BR = *event.BR;
 
   bool isNSError = hasFlag<NSErrorOut>(loc, state);
@@ -277,7 +283,7 @@ void NSOrCFErrorDerefChecker::checkEvent(ImplicitNullDerefEvent event) const {
     bug = new NSErrorDerefBug();
   else
     bug = new CFErrorDerefBug();
-  EnhancedBugReport *report = new EnhancedBugReport(*bug, os.str(),
+  BugReport *report = new BugReport(*bug, os.str(),
                                                     event.SinkNode);
   BR.EmitReport(report);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
index 2d0af9c978dc..81f1924b2c7a 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
@@ -16,23 +16,27 @@
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "llvm/ADT/StringSwitch.h"
+#include <cstdarg>
 
 using namespace clang;
 using namespace ento;
 
 namespace {
 
-class NoReturnFunctionChecker : public Checker< check::PostStmt<CallExpr> > {
+class NoReturnFunctionChecker : public Checker< check::PostStmt<CallExpr>,
+                                                check::PostObjCMessage > {
 public:
   void checkPostStmt(const CallExpr *CE, CheckerContext &C) const;
+  void checkPostObjCMessage(const ObjCMessage &msg, CheckerContext &C) const;
 };
 
 }
 
 void NoReturnFunctionChecker::checkPostStmt(const CallExpr *CE,
                                             CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
 
   bool BuildSinks = getFunctionExtInfo(Callee->getType()).getNoReturn();
@@ -50,7 +54,7 @@ void NoReturnFunctionChecker::checkPostStmt(const CallExpr *CE,
       //  Here are a few hardwired ones.  If this takes too long, we can
       //  potentially cache these results.
       BuildSinks
-        = llvm::StringSwitch<bool>(llvm::StringRef(II->getName()))
+        = llvm::StringSwitch<bool>(StringRef(II->getName()))
             .Case("exit", true)
             .Case("panic", true)
             .Case("error", true)
@@ -73,9 +77,70 @@ void NoReturnFunctionChecker::checkPostStmt(const CallExpr *CE,
   }
 
   if (BuildSinks)
-    C.generateSink(CE);
+    C.generateSink();
 }
 
+static bool END_WITH_NULL isMultiArgSelector(const Selector *Sel, ...) {
+  va_list argp;
+  va_start(argp, Sel);
+
+  unsigned Slot = 0;
+  const char *Arg;
+  while ((Arg = va_arg(argp, const char *))) {
+    if (!Sel->getNameForSlot(Slot).equals(Arg))
+      break; // still need to va_end!
+    ++Slot;
+  }
+
+  va_end(argp);
+
+  // We only succeeded if we made it to the end of the argument list.
+  return (Arg == NULL);
+}
+
+void NoReturnFunctionChecker::checkPostObjCMessage(const ObjCMessage &Msg,
+                                                   CheckerContext &C) const {
+  // HACK: This entire check is to handle two messages in the Cocoa frameworks:
+  // -[NSAssertionHandler
+  //    handleFailureInMethod:object:file:lineNumber:description:]
+  // -[NSAssertionHandler
+  //    handleFailureInFunction:file:lineNumber:description:]
+  // Eventually these should be annotated with __attribute__((noreturn)).
+  // Because ObjC messages use dynamic dispatch, it is not generally safe to
+  // assume certain methods can't return. In cases where it is definitely valid,
+  // see if you can mark the methods noreturn or analyzer_noreturn instead of
+  // adding more explicit checks to this method.
+
+  if (!Msg.isInstanceMessage())
+    return;
+
+  const ObjCInterfaceDecl *Receiver = Msg.getReceiverInterface();
+  if (!Receiver)
+    return;
+  if (!Receiver->getIdentifier()->isStr("NSAssertionHandler"))
+    return;
+
+  Selector Sel = Msg.getSelector();
+  switch (Sel.getNumArgs()) {
+  default:
+    return;
+  case 4:
+    if (!isMultiArgSelector(&Sel, "handleFailureInFunction", "file",
+                            "lineNumber", "description", NULL))
+      return;
+    break;
+  case 5:
+    if (!isMultiArgSelector(&Sel, "handleFailureInMethod", "object", "file",
+                            "lineNumber", "description", NULL))
+      return;
+    break;
+  }
+
+  // If we got here, it's one of the messages we care about.
+  C.generateSink();
+}
+
+
 void ento::registerNoReturnFunctionChecker(CheckerManager &mgr) {
   mgr.registerChecker<NoReturnFunctionChecker>();
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
index 7262bc36404d..f426265f6713 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
@@ -22,22 +22,32 @@ using namespace ento;
 
 namespace {
 
-class OSAtomicChecker : public Checker<eval::Call> {
+class OSAtomicChecker : public Checker<eval::InlineCall> {
 public:
-  bool evalCall(const CallExpr *CE, CheckerContext &C) const;
+  bool inlineCall(const CallExpr *CE, ExprEngine &Eng,
+                  ExplodedNode *Pred, ExplodedNodeSet &Dst) const;
 
 private:
-  static bool evalOSAtomicCompareAndSwap(CheckerContext &C, const CallExpr *CE);
-};
+  bool evalOSAtomicCompareAndSwap(const CallExpr *CE,
+                                  ExprEngine &Eng,
+                                  ExplodedNode *Pred,
+                                  ExplodedNodeSet &Dst) const;
 
+  ExplodedNode *generateNode(const ProgramState *State,
+                             ExplodedNode *Pred, const CallExpr *Statement,
+                             StmtNodeBuilder &B, ExplodedNodeSet &Dst) const;
+};
 }
 
-bool OSAtomicChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
-  const GRState *state = C.getState();
+bool OSAtomicChecker::inlineCall(const CallExpr *CE,
+                                 ExprEngine &Eng,
+                                 ExplodedNode *Pred,
+                                 ExplodedNodeSet &Dst) const {
+  const ProgramState *state = Pred->getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
 
-  const FunctionDecl* FD = L.getAsFunctionDecl();
+  const FunctionDecl *FD = L.getAsFunctionDecl();
   if (!FD)
     return false;
 
@@ -45,24 +55,38 @@ bool OSAtomicChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
   if (!II)
     return false;
   
-  llvm::StringRef FName(II->getName());
+  StringRef FName(II->getName());
 
   // Check for compare and swap.
   if (FName.startswith("OSAtomicCompareAndSwap") ||
       FName.startswith("objc_atomicCompareAndSwap"))
-    return evalOSAtomicCompareAndSwap(C, CE);
+    return evalOSAtomicCompareAndSwap(CE, Eng, Pred, Dst);
 
   // FIXME: Other atomics.
   return false;
 }
 
-bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C, 
-                                                 const CallExpr *CE) {
+ExplodedNode *OSAtomicChecker::generateNode(const ProgramState *State,
+                                            ExplodedNode *Pred,
+                                            const CallExpr *Statement,
+                                            StmtNodeBuilder &B,
+                                            ExplodedNodeSet &Dst) const {
+  ExplodedNode *N = B.generateNode(Statement, State, Pred, this);
+  if (N)
+    Dst.Add(N);
+  return N;
+}
+
+bool OSAtomicChecker::evalOSAtomicCompareAndSwap(const CallExpr *CE,
+                                                 ExprEngine &Eng,
+                                                 ExplodedNode *Pred,
+                                                 ExplodedNodeSet &Dst) const {
   // Not enough arguments to match OSAtomicCompareAndSwap?
   if (CE->getNumArgs() != 3)
     return false;
 
-  ASTContext &Ctx = C.getASTContext();
+  StmtNodeBuilder &Builder = Eng.getBuilder();
+  ASTContext &Ctx = Eng.getContext();
   const Expr *oldValueExpr = CE->getArg(0);
   QualType oldValueType = Ctx.getCanonicalType(oldValueExpr->getType());
 
@@ -87,15 +111,11 @@ bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C,
   if (theValueTypePointee != newValueType)
     return false;
 
-  static unsigned magic_load = 0;
-  static unsigned magic_store = 0;
-
-  const void *OSAtomicLoadTag = &magic_load;
-  const void *OSAtomicStoreTag = &magic_store;
-
+  static SimpleProgramPointTag OSAtomicLoadTag("OSAtomicChecker : Load");
+  static SimpleProgramPointTag OSAtomicStoreTag("OSAtomicChecker : Store");
+  
   // Load 'theValue'.
-  ExprEngine &Engine = C.getEngine();
-  const GRState *state = C.getState();
+  const ProgramState *state = Pred->getState();
   ExplodedNodeSet Tmp;
   SVal location = state->getSVal(theValueExpr);
   // Here we should use the value type of the region as the load type, because
@@ -106,19 +126,19 @@ bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C,
   // LoadTy specifying can be omitted. But we put it here to emphasize the 
   // semantics.
   QualType LoadTy;
-  if (const TypedRegion *TR =
-      dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
+  if (const TypedValueRegion *TR =
+      dyn_cast_or_null<TypedValueRegion>(location.getAsRegion())) {
     LoadTy = TR->getValueType();
   }
-  Engine.evalLoad(Tmp, theValueExpr, C.getPredecessor(), 
-                  state, location, OSAtomicLoadTag, LoadTy);
+  Eng.evalLoad(Tmp, theValueExpr, Pred,
+                  state, location, &OSAtomicLoadTag, LoadTy);
 
   if (Tmp.empty()) {
     // If no nodes were generated, other checkers must generated sinks. But 
     // since the builder state was restored, we set it manually to prevent 
     // auto transition.
     // FIXME: there should be a better approach.
-    C.getNodeBuilder().BuildSinks = true;
+    Builder.BuildSinks = true;
     return true;
   }
  
@@ -126,7 +146,7 @@ bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C,
        I != E; ++I) {
 
     ExplodedNode *N = *I;
-    const GRState *stateLoad = N->getState();
+    const ProgramState *stateLoad = N->getState();
 
     // Use direct bindings from the environment since we are forcing a load
     // from a location that the Environment would typically not be used
@@ -145,12 +165,13 @@ bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C,
     DefinedOrUnknownSVal oldValueVal =
       cast<DefinedOrUnknownSVal>(oldValueVal_untested);
 
-    SValBuilder &svalBuilder = Engine.getSValBuilder();
+    SValBuilder &svalBuilder = Eng.getSValBuilder();
 
     // Perform the comparison.
-    DefinedOrUnknownSVal Cmp = svalBuilder.evalEQ(stateLoad,theValueVal,oldValueVal);
+    DefinedOrUnknownSVal Cmp =
+      svalBuilder.evalEQ(stateLoad,theValueVal,oldValueVal);
 
-    const GRState *stateEqual = stateLoad->assume(Cmp, true);
+    const ProgramState *stateEqual = stateLoad->assume(Cmp, true);
 
     // Were they equal?
     if (stateEqual) {
@@ -159,20 +180,20 @@ bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C,
       SVal val = stateEqual->getSVal(newValueExpr);
 
       // Handle implicit value casts.
-      if (const TypedRegion *R =
-          dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
+      if (const TypedValueRegion *R =
+          dyn_cast_or_null<TypedValueRegion>(location.getAsRegion())) {
         val = svalBuilder.evalCast(val,R->getValueType(), newValueExpr->getType());
       }
 
-      Engine.evalStore(TmpStore, NULL, theValueExpr, N, 
-                       stateEqual, location, val, OSAtomicStoreTag);
+      Eng.evalStore(TmpStore, NULL, theValueExpr, N,
+                       stateEqual, location, val, &OSAtomicStoreTag);
 
       if (TmpStore.empty()) {
         // If no nodes were generated, other checkers must generated sinks. But 
         // since the builder state was restored, we set it manually to prevent 
         // auto transition.
         // FIXME: there should be a better approach.
-        C.getNodeBuilder().BuildSinks = true;
+        Builder.BuildSinks = true;
         return true;
       }
 
@@ -180,24 +201,24 @@ bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C,
       for (ExplodedNodeSet::iterator I2 = TmpStore.begin(),
            E2 = TmpStore.end(); I2 != E2; ++I2) {
         ExplodedNode *predNew = *I2;
-        const GRState *stateNew = predNew->getState();
+        const ProgramState *stateNew = predNew->getState();
         // Check for 'void' return type if we have a bogus function prototype.
         SVal Res = UnknownVal();
         QualType T = CE->getType();
         if (!T->isVoidType())
-          Res = Engine.getSValBuilder().makeTruthVal(true, T);
-        C.generateNode(stateNew->BindExpr(CE, Res), predNew);
+          Res = Eng.getSValBuilder().makeTruthVal(true, T);
+        generateNode(stateNew->BindExpr(CE, Res), predNew, CE, Builder, Dst);
       }
     }
 
     // Were they not equal?
-    if (const GRState *stateNotEqual = stateLoad->assume(Cmp, false)) {
+    if (const ProgramState *stateNotEqual = stateLoad->assume(Cmp, false)) {
       // Check for 'void' return type if we have a bogus function prototype.
       SVal Res = UnknownVal();
       QualType T = CE->getType();
       if (!T->isVoidType())
-        Res = Engine.getSValBuilder().makeTruthVal(false, CE->getType());
-      C.generateNode(stateNotEqual->BindExpr(CE, Res), N);
+        Res = Eng.getSValBuilder().makeTruthVal(false, CE->getType());
+      generateNode(stateNotEqual->BindExpr(CE, Res), N, CE, Builder, Dst);
     }
   }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
index a1180492a937..3e4e07b65057 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
@@ -38,7 +38,7 @@ void ObjCAtSyncChecker::checkPreStmt(const ObjCAtSynchronizedStmt *S,
                                      CheckerContext &C) const {
 
   const Expr *Ex = S->getSynchExpr();
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SVal V = state->getSVal(Ex);
 
   // Uninitialized value used for the mutex?
@@ -47,9 +47,9 @@ void ObjCAtSyncChecker::checkPreStmt(const ObjCAtSynchronizedStmt *S,
       if (!BT_undef)
         BT_undef.reset(new BuiltinBug("Uninitialized value used as mutex "
                                   "for @synchronized"));
-      EnhancedBugReport *report =
-        new EnhancedBugReport(*BT_undef, BT_undef->getDescription(), N);
-      report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Ex);
+      BugReport *report =
+        new BugReport(*BT_undef, BT_undef->getDescription(), N);
+      report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, Ex));
       C.EmitReport(report);
     }
     return;
@@ -59,7 +59,7 @@ void ObjCAtSyncChecker::checkPreStmt(const ObjCAtSynchronizedStmt *S,
     return;
 
   // Check for null mutexes.
-  const GRState *notNullState, *nullState;
+  const ProgramState *notNullState, *nullState;
   llvm::tie(notNullState, nullState) = state->assume(cast<DefinedSVal>(V));
 
   if (nullState) {
@@ -70,10 +70,9 @@ void ObjCAtSyncChecker::checkPreStmt(const ObjCAtSynchronizedStmt *S,
         if (!BT_null)
           BT_null.reset(new BuiltinBug("Nil value used as mutex for @synchronized() "
                                    "(no synchronization will occur)"));
-        EnhancedBugReport *report =
-          new EnhancedBugReport(*BT_null, BT_null->getDescription(), N);
-        report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                                  Ex);
+        BugReport *report =
+          new BugReport(*BT_null, BT_null->getDescription(), N);
+        report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, Ex));
 
         C.EmitReport(report);
         return;
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
index 4c05867631f3..2fb9944afaa7 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
@@ -50,7 +50,8 @@
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/AST/ParentMap.h"
 
@@ -76,7 +77,8 @@ class ObjCSelfInitChecker : public Checker<
   void checkPreStmt(const ReturnStmt *S, CheckerContext &C) const;
   void checkPreStmt(const CallExpr *CE, CheckerContext &C) const;
   void checkPostStmt(const CallExpr *CE, CheckerContext &C) const;
-  void checkLocation(SVal location, bool isLoad, CheckerContext &C) const;
+  void checkLocation(SVal location, bool isLoad, const Stmt *S,
+                     CheckerContext &C) const;
 };
 } // end anonymous namespace
 
@@ -109,11 +111,11 @@ namespace { struct PreCallSelfFlags {}; }
 namespace clang {
 namespace ento {
   template<>
-  struct GRStateTrait<SelfFlag> : public GRStatePartialTrait<SelfFlag> {
-    static void* GDMIndex() { static int index = 0; return &index; }
+  struct ProgramStateTrait<SelfFlag> : public ProgramStatePartialTrait<SelfFlag> {
+    static void *GDMIndex() { static int index = 0; return &index; }
   };
   template <>
-  struct GRStateTrait<CalledInit> : public GRStatePartialTrait<bool> {
+  struct ProgramStateTrait<CalledInit> : public ProgramStatePartialTrait<bool> {
     static void *GDMIndex() { static int index = 0; return &index; }
   };
 
@@ -122,13 +124,13 @@ namespace ento {
   /// object before the call so we can assign them to the new object that 'self'
   /// points to after the call.
   template <>
-  struct GRStateTrait<PreCallSelfFlags> : public GRStatePartialTrait<unsigned> {
+  struct ProgramStateTrait<PreCallSelfFlags> : public ProgramStatePartialTrait<unsigned> {
     static void *GDMIndex() { static int index = 0; return &index; }
   };
 }
 }
 
-static SelfFlagEnum getSelfFlags(SVal val, const GRState *state) {
+static SelfFlagEnum getSelfFlags(SVal val, const ProgramState *state) {
   if (SymbolRef sym = val.getAsSymbol())
     if (const unsigned *attachedFlags = state->get<SelfFlag>(sym))
       return (SelfFlagEnum)*attachedFlags;
@@ -139,7 +141,7 @@ static SelfFlagEnum getSelfFlags(SVal val, CheckerContext &C) {
   return getSelfFlags(val, C.getState());
 }
 
-static void addSelfFlag(const GRState *state, SVal val,
+static void addSelfFlag(const ProgramState *state, SVal val,
                         SelfFlagEnum flag, CheckerContext &C) {
   // We tag the symbol that the SVal wraps.
   if (SymbolRef sym = val.getAsSymbol())
@@ -179,8 +181,8 @@ static void checkForInvalidSelf(const Expr *E, CheckerContext &C,
   if (!N)
     return;
 
-  EnhancedBugReport *report =
-    new EnhancedBugReport(*new InitSelfBug(), errorStr, N);
+  BugReport *report =
+    new BugReport(*new InitSelfBug(), errorStr, N);
   C.EmitReport(report);
 }
 
@@ -197,7 +199,7 @@ void ObjCSelfInitChecker::checkPostObjCMessage(ObjCMessage msg,
 
   if (isInitMessage(msg)) {
     // Tag the return value as the result of an initializer.
-    const GRState *state = C.getState();
+    const ProgramState *state = C.getState();
     
     // FIXME this really should be context sensitive, where we record
     // the current stack frame (for IPA).  Also, we need to clean this
@@ -257,7 +259,7 @@ void ObjCSelfInitChecker::checkPreStmt(const ReturnStmt *S,
 
 void ObjCSelfInitChecker::checkPreStmt(const CallExpr *CE,
                                        CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   for (CallExpr::const_arg_iterator
          I = CE->arg_begin(), E = CE->arg_end(); I != E; ++I) {
     SVal argV = state->getSVal(*I);
@@ -275,7 +277,7 @@ void ObjCSelfInitChecker::checkPreStmt(const CallExpr *CE,
 
 void ObjCSelfInitChecker::checkPostStmt(const CallExpr *CE,
                                         CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   for (CallExpr::const_arg_iterator
          I = CE->arg_begin(), E = CE->arg_end(); I != E; ++I) {
     SVal argV = state->getSVal(*I);
@@ -294,10 +296,11 @@ void ObjCSelfInitChecker::checkPostStmt(const CallExpr *CE,
 }
 
 void ObjCSelfInitChecker::checkLocation(SVal location, bool isLoad,
+                                        const Stmt *S,
                                         CheckerContext &C) const {
   // Tag the result of a load from 'self' so that we can easily know that the
   // value is the object that 'self' points to.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (isSelfVar(location, C))
     addSelfFlag(state, state->getSVal(cast<Loc>(location)), SelfFlag_Self, C);
 }
@@ -315,9 +318,9 @@ static bool shouldRunOnFunctionOrMethod(const NamedDecl *ND) {
 
   // self = [super init] applies only to NSObject subclasses.
   // For instance, NSProxy doesn't implement -init.
-  ASTContext& Ctx = MD->getASTContext();
+  ASTContext &Ctx = MD->getASTContext();
   IdentifierInfo* NSObjectII = &Ctx.Idents.get("NSObject");
-  ObjCInterfaceDecl* ID = MD->getClassInterface()->getSuperClass();
+  ObjCInterfaceDecl *ID = MD->getClassInterface()->getSuperClass();
   for ( ; ID ; ID = ID->getSuperClass()) {
     IdentifierInfo *II = ID->getIdentifier();
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
index d78e5ceb533d..bbc262fe8ef7 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
@@ -29,7 +29,7 @@ using namespace ento;
 enum IVarState { Unused, Used };
 typedef llvm::DenseMap<const ObjCIvarDecl*,IVarState> IvarUsageMap;
 
-static void Scan(IvarUsageMap& M, const Stmt* S) {
+static void Scan(IvarUsageMap& M, const Stmt *S) {
   if (!S)
     return;
 
@@ -51,11 +51,11 @@ static void Scan(IvarUsageMap& M, const Stmt* S) {
     Scan(M, *I);
 }
 
-static void Scan(IvarUsageMap& M, const ObjCPropertyImplDecl* D) {
+static void Scan(IvarUsageMap& M, const ObjCPropertyImplDecl *D) {
   if (!D)
     return;
 
-  const ObjCIvarDecl* ID = D->getPropertyIvarDecl();
+  const ObjCIvarDecl *ID = D->getPropertyIvarDecl();
 
   if (!ID)
     return;
@@ -65,7 +65,7 @@ static void Scan(IvarUsageMap& M, const ObjCPropertyImplDecl* D) {
     I->second = Used;
 }
 
-static void Scan(IvarUsageMap& M, const ObjCContainerDecl* D) {
+static void Scan(IvarUsageMap& M, const ObjCContainerDecl *D) {
   // Scan the methods for accesses.
   for (ObjCContainerDecl::instmeth_iterator I = D->instmeth_begin(),
        E = D->instmeth_end(); I!=E; ++I)
@@ -102,14 +102,14 @@ static void Scan(IvarUsageMap &M, const DeclContext *C, const FileID FID,
 static void checkObjCUnusedIvar(const ObjCImplementationDecl *D,
                                 BugReporter &BR) {
 
-  const ObjCInterfaceDecl* ID = D->getClassInterface();
+  const ObjCInterfaceDecl *ID = D->getClassInterface();
   IvarUsageMap M;
 
   // Iterate over the ivars.
   for (ObjCInterfaceDecl::ivar_iterator I=ID->ivar_begin(),
         E=ID->ivar_end(); I!=E; ++I) {
 
-    const ObjCIvarDecl* ID = *I;
+    const ObjCIvarDecl *ID = *I;
 
     // Ignore ivars that...
     // (a) aren't private
@@ -155,12 +155,14 @@ static void checkObjCUnusedIvar(const ObjCImplementationDecl *D,
     if (I->second == Unused) {
       std::string sbuf;
       llvm::raw_string_ostream os(sbuf);
-      os << "Instance variable '" << I->first << "' in class '" << ID
+      os << "Instance variable '" << *I->first << "' in class '" << *ID
          << "' is never used by the methods in its @implementation "
             "(although it may be used by category methods).";
 
+      PathDiagnosticLocation L =
+        PathDiagnosticLocation::create(I->first, BR.getSourceManager());
       BR.EmitBasicReport("Unused instance variable", "Optimization",
-                         os.str(), I->first->getLocation());
+                         os.str(), L);
     }
 }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
index 7c21acc5bee7..202522b1949c 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
@@ -36,7 +36,7 @@ void PointerArithChecker::checkPreStmt(const BinaryOperator *B,
   if (B->getOpcode() != BO_Sub && B->getOpcode() != BO_Add)
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SVal LV = state->getSVal(B->getLHS());
   SVal RV = state->getSVal(B->getRHS());
 
@@ -56,7 +56,7 @@ void PointerArithChecker::checkPreStmt(const BinaryOperator *B,
                             "Pointer arithmetic done on non-array variables "
                             "means reliance on memory layout, which is "
                             "dangerous."));
-      RangedBugReport *R = new RangedBugReport(*BT, BT->getDescription(), N);
+      BugReport *R = new BugReport(*BT, BT->getDescription(), N);
       R->addRange(B->getSourceRange());
       C.EmitReport(R);
     }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
index 16ede2095eae..924c7f2ad3ea 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
@@ -39,7 +39,7 @@ void PointerSubChecker::checkPreStmt(const BinaryOperator *B,
   if (B->getOpcode() != BO_Sub)
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SVal LV = state->getSVal(B->getLHS());
   SVal RV = state->getSVal(B->getRHS());
 
@@ -64,7 +64,7 @@ void PointerSubChecker::checkPreStmt(const BinaryOperator *B,
       BT.reset(new BuiltinBug("Pointer subtraction", 
                           "Subtraction of two pointers that do not point to "
                           "the same memory chunk may cause incorrect result."));
-    RangedBugReport *R = new RangedBugReport(*BT, BT->getDescription(), N);
+    BugReport *R = new BugReport(*BT, BT->getDescription(), N);
     R->addRange(B->getSourceRange());
     C.EmitReport(R);
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
index 74199bb1f6db..c02b5b14ca64 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
@@ -1,4 +1,4 @@
-//===--- PthreadLockChecker.h - Undefined arguments checker ----*- C++ -*--===//
+//===--- PthreadLockChecker.cpp - Check for locking problems ---*- C++ -*--===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines PthreadLockChecker, a simple lock -> unlock checker.  Eventually
-// this shouldn't be registered with ExprEngineInternalChecks.
+// This defines PthreadLockChecker, a simple lock -> unlock checker.
+// Also handles XNU locks, which behave similarly enough to share code.
 //
 //===----------------------------------------------------------------------===//
 
@@ -16,25 +16,29 @@
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
-#include "llvm/ADT/ImmutableSet.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+#include "llvm/ADT/ImmutableList.h"
 
 using namespace clang;
 using namespace ento;
 
 namespace {
-class PthreadLockChecker
-  : public Checker< check::PostStmt<CallExpr> > {
+class PthreadLockChecker : public Checker< check::PostStmt<CallExpr> > {
+  mutable llvm::OwningPtr<BugType> BT_doublelock;
+  mutable llvm::OwningPtr<BugType> BT_lor;
+  enum LockingSemantics {
+    NotApplicable = 0,
+    PthreadSemantics,
+    XNUSemantics
+  };
 public:
   void checkPostStmt(const CallExpr *CE, CheckerContext &C) const;
     
-  void AcquireLock(CheckerContext &C, const CallExpr *CE,
-                   SVal lock, bool isTryLock) const;
+  void AcquireLock(CheckerContext &C, const CallExpr *CE, SVal lock,
+                   bool isTryLock, enum LockingSemantics semantics) const;
     
-  void ReleaseLock(CheckerContext &C, const CallExpr *CE,
-                    SVal lock) const;
-
+  void ReleaseLock(CheckerContext &C, const CallExpr *CE, SVal lock) const;
 };
 } // end anonymous namespace
 
@@ -42,80 +46,117 @@ class PthreadLockChecker
 namespace { class LockSet {}; }
 namespace clang {
 namespace ento {
-template <> struct GRStateTrait<LockSet> :
-  public GRStatePartialTrait<llvm::ImmutableSet<const MemRegion*> > {
-    static void* GDMIndex() { static int x = 0; return &x; }
+template <> struct ProgramStateTrait<LockSet> :
+  public ProgramStatePartialTrait<llvm::ImmutableList<const MemRegion*> > {
+    static void *GDMIndex() { static int x = 0; return &x; }
 };
-} // end GR namespace
+} // end of ento (ProgramState) namespace
 } // end clang namespace
 
 
 void PthreadLockChecker::checkPostStmt(const CallExpr *CE,
                                        CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
-  const FunctionTextRegion *R =
-    dyn_cast_or_null<FunctionTextRegion>(state->getSVal(Callee).getAsRegion());
-  
-  if (!R)
+  const FunctionDecl *FD = state->getSVal(Callee).getAsFunctionDecl();
+
+  if (!FD)
     return;
-  
-  IdentifierInfo *II = R->getDecl()->getIdentifier();
+
+  // Get the name of the callee.
+  IdentifierInfo *II = FD->getIdentifier();
   if (!II)   // if no identifier, not a simple C function
     return;
-  llvm::StringRef FName = II->getName();
-  
-  if (FName == "pthread_mutex_lock") {
-    if (CE->getNumArgs() != 1)
-      return;
-    AcquireLock(C, CE, state->getSVal(CE->getArg(0)), false);
-  }
-  else if (FName == "pthread_mutex_trylock") {
-    if (CE->getNumArgs() != 1)
-      return;
-    AcquireLock(C, CE, state->getSVal(CE->getArg(0)), true);
-  }  
-  else if (FName == "pthread_mutex_unlock") {
-    if (CE->getNumArgs() != 1)
-      return;
+  StringRef FName = II->getName();
+
+  if (CE->getNumArgs() != 1)
+    return;
+
+  if (FName == "pthread_mutex_lock" ||
+      FName == "pthread_rwlock_rdlock" ||
+      FName == "pthread_rwlock_wrlock")
+    AcquireLock(C, CE, state->getSVal(CE->getArg(0)), false, PthreadSemantics);
+  else if (FName == "lck_mtx_lock" ||
+           FName == "lck_rw_lock_exclusive" ||
+           FName == "lck_rw_lock_shared") 
+    AcquireLock(C, CE, state->getSVal(CE->getArg(0)), false, XNUSemantics);
+  else if (FName == "pthread_mutex_trylock" ||
+           FName == "pthread_rwlock_tryrdlock" ||
+           FName == "pthread_rwlock_tryrwlock")
+    AcquireLock(C, CE, state->getSVal(CE->getArg(0)), true, PthreadSemantics);
+  else if (FName == "lck_mtx_try_lock" ||
+           FName == "lck_rw_try_lock_exclusive" ||
+           FName == "lck_rw_try_lock_shared")
+    AcquireLock(C, CE, state->getSVal(CE->getArg(0)), true, XNUSemantics);
+  else if (FName == "pthread_mutex_unlock" ||
+           FName == "pthread_rwlock_unlock" ||
+           FName == "lck_mtx_unlock" ||
+           FName == "lck_rw_done")
     ReleaseLock(C, CE, state->getSVal(CE->getArg(0)));
-  }
 }
 
 void PthreadLockChecker::AcquireLock(CheckerContext &C, const CallExpr *CE,
-                                     SVal lock, bool isTryLock) const {
+                                     SVal lock, bool isTryLock,
+                                     enum LockingSemantics semantics) const {
   
   const MemRegion *lockR = lock.getAsRegion();
   if (!lockR)
     return;
   
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   
   SVal X = state->getSVal(CE);
   if (X.isUnknownOrUndef())
     return;
   
   DefinedSVal retVal = cast<DefinedSVal>(X);
-  const GRState *lockSucc = state;
-  
-  if (isTryLock) {
-      // Bifurcate the state, and allow a mode where the lock acquisition fails.
-    const GRState *lockFail;
-    llvm::tie(lockFail, lockSucc) = state->assume(retVal);    
-    assert(lockFail && lockSucc);
-    C.addTransition(C.generateNode(CE, lockFail));
+
+  if (state->contains<LockSet>(lockR)) {
+    if (!BT_doublelock)
+      BT_doublelock.reset(new BugType("Double locking", "Lock checker"));
+    ExplodedNode *N = C.generateSink();
+    if (!N)
+      return;
+    BugReport *report = new BugReport(*BT_doublelock,
+                                                      "This lock has already "
+                                                      "been acquired", N);
+    report->addRange(CE->getArg(0)->getSourceRange());
+    C.EmitReport(report);
+    return;
   }
-  else {
-      // Assume that the return value was 0.
+
+  const ProgramState *lockSucc = state;
+  if (isTryLock) {
+    // Bifurcate the state, and allow a mode where the lock acquisition fails.
+    const ProgramState *lockFail;
+    switch (semantics) {
+    case PthreadSemantics:
+      llvm::tie(lockFail, lockSucc) = state->assume(retVal);    
+      break;
+    case XNUSemantics:
+      llvm::tie(lockSucc, lockFail) = state->assume(retVal);    
+      break;
+    default:
+      llvm_unreachable("Unknown tryLock locking semantics");
+      break;
+    }
+    assert(lockFail && lockSucc);
+    C.addTransition(lockFail);
+
+  } else if (semantics == PthreadSemantics) {
+    // Assume that the return value was 0.
     lockSucc = state->assume(retVal, false);
     assert(lockSucc);
+
+  } else {
+    // XNU locking semantics return void on non-try locks
+    assert((semantics == XNUSemantics) && "Unknown locking semantics");
+    lockSucc = state;
   }
   
-    // Record that the lock was acquired.  
+  // Record that the lock was acquired.  
   lockSucc = lockSucc->add<LockSet>(lockR);
-  
-  C.addTransition(lockSucc != state ? C.generateNode(CE, lockSucc) :
-                  C.getPredecessor());
+  C.addTransition(lockSucc);
 }
 
 void PthreadLockChecker::ReleaseLock(CheckerContext &C, const CallExpr *CE,
@@ -125,19 +166,37 @@ void PthreadLockChecker::ReleaseLock(CheckerContext &C, const CallExpr *CE,
   if (!lockR)
     return;
   
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
+  llvm::ImmutableList<const MemRegion*> LS = state->get<LockSet>();
 
-  // Record that the lock was released.  
-  // FIXME: Handle unlocking locks that were never acquired.  This may
-  // require IPA for wrappers.
-  const GRState *unlockState = state->remove<LockSet>(lockR);
-  
-  if (state == unlockState)
+  // FIXME: Better analysis requires IPA for wrappers.
+  // FIXME: check for double unlocks
+  if (LS.isEmpty())
     return;
   
-  C.addTransition(C.generateNode(CE, unlockState));  
+  const MemRegion *firstLockR = LS.getHead();
+  if (firstLockR != lockR) {
+    if (!BT_lor)
+      BT_lor.reset(new BugType("Lock order reversal", "Lock checker"));
+    ExplodedNode *N = C.generateSink();
+    if (!N)
+      return;
+    BugReport *report = new BugReport(*BT_lor,
+                                                      "This was not the most "
+                                                      "recently acquired lock. "
+                                                      "Possible lock order "
+                                                      "reversal", N);
+    report->addRange(CE->getArg(0)->getSourceRange());
+    C.EmitReport(report);
+    return;
+  }
+
+  // Record that the lock was released. 
+  state = state->set<LockSet>(LS.getTail());
+  C.addTransition(state);
 }
 
+
 void ento::registerPthreadLockChecker(CheckerManager &mgr) {
   mgr.registerChecker<PthreadLockChecker>();
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CFRefCount.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
similarity index 59%
rename from contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CFRefCount.cpp
rename to contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
index bf5302920819..93e0fe5b4f96 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CFRefCount.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
@@ -1,4 +1,4 @@
-// CFRefCount.cpp - Transfer functions for tracking simple values -*- C++ -*--//
+//==-- RetainCountChecker.cpp - Checks for leaks and other issues -*- C++ -*--//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,111 +7,56 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines the methods for CFRefCount, which implements
-//  a reference count checker for Core Foundation (Mac OS X).
+//  This file defines the methods for RetainCountChecker, which implements
+//  a reference count checker for Core Foundation and Cocoa on (Mac OS X).
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "ClangSACheckers.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclCXX.h"
-#include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
-#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
-#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableList.h"
 #include "llvm/ADT/ImmutableMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
-#include <stdarg.h>
+#include <cstdarg>
 
 using namespace clang;
 using namespace ento;
-using llvm::StringRef;
 using llvm::StrInStrNoCase;
 
 namespace {
-class InstanceReceiver {
-  ObjCMessage Msg;
-  const LocationContext *LC;
-public:
-  InstanceReceiver() : LC(0) { }
-  InstanceReceiver(const ObjCMessage &msg,
-                   const LocationContext *lc = 0) : Msg(msg), LC(lc) {}
-
-  bool isValid() const {
-    return Msg.isValid() && Msg.isInstanceMessage();
-  }
-  operator bool() const {
-    return isValid();
-  }
-
-  SVal getSValAsScalarOrLoc(const GRState *state) {
-    assert(isValid());
-    // We have an expression for the receiver?  Fetch the value
-    // of that expression.
-    if (const Expr *Ex = Msg.getInstanceReceiver())
-      return state->getSValAsScalarOrLoc(Ex);
-
-    // Otherwise we are sending a message to super.  In this case the
-    // object reference is the same as 'self'.
-    if (const ImplicitParamDecl *SelfDecl = LC->getSelfDecl())
-      return state->getSVal(state->getRegion(SelfDecl, LC));
-
-    return UnknownVal();
-  }
-
-  SourceRange getSourceRange() const {
-    assert(isValid());
-    if (const Expr *Ex = Msg.getInstanceReceiver())
-      return Ex->getSourceRange();
-
-    // Otherwise we are sending a message to super.
-    SourceLocation L = Msg.getSuperLoc();
-    assert(L.isValid());
-    return SourceRange(L, L);
-  }
-};
-}
-
-static const ObjCMethodDecl*
-ResolveToInterfaceMethodDecl(const ObjCMethodDecl *MD) {
-  const ObjCInterfaceDecl *ID = MD->getClassInterface();
-
-  return MD->isInstanceMethod()
-         ? ID->lookupInstanceMethod(MD->getSelector())
-         : ID->lookupClassMethod(MD->getSelector());
-}
-
-namespace {
+/// Wrapper around different kinds of node builder, so that helper functions
+/// can have a common interface.
 class GenericNodeBuilderRefCount {
-  StmtNodeBuilder *SNB;
-  const Stmt *S;
-  const void *tag;
+  CheckerContext *C;
+  const ProgramPointTag *tag;
   EndOfFunctionNodeBuilder *ENB;
 public:
-  GenericNodeBuilderRefCount(StmtNodeBuilder &snb, const Stmt *s,
-                     const void *t)
-  : SNB(&snb), S(s), tag(t), ENB(0) {}
+  GenericNodeBuilderRefCount(CheckerContext &c,
+                             const ProgramPointTag *t)
+  : C(&c), tag(t), ENB(0) {}
 
   GenericNodeBuilderRefCount(EndOfFunctionNodeBuilder &enb)
-  : SNB(0), S(0), tag(0), ENB(&enb) {}
+  : C(0), tag(0), ENB(&enb) {}
 
-  ExplodedNode *MakeNode(const GRState *state, ExplodedNode *Pred) {
-    if (SNB)
-      return SNB->generateNode(PostStmt(S, Pred->getLocationContext(), tag),
-                               state, Pred);
+  ExplodedNode *MakeNode(const ProgramState *state, ExplodedNode *Pred) {
+    if (C)
+      return C->generateNode(state, Pred, tag, false);
 
     assert(ENB);
     return ENB->generateNode(state, Pred);
@@ -125,9 +70,9 @@ class GenericNodeBuilderRefCount {
 
 /// ArgEffect is used to summarize a function/method call's effect on a
 /// particular argument.
-enum ArgEffect { Autorelease, Dealloc, DecRef, DecRefMsg, DoNothing,
+enum ArgEffect { DoNothing, Autorelease, Dealloc, DecRef, DecRefMsg,
                  DecRefBridgedTransfered,
-                 DoNothingByRef, IncRefMsg, IncRef, MakeCollectable, MayEscape,
+                 IncRefMsg, IncRef, MakeCollectable, MayEscape,
                  NewAutoreleasePool, SelfOwn, StopTracking };
 
 namespace llvm {
@@ -148,9 +93,8 @@ namespace {
 ///  respect to its return value.
 class RetEffect {
 public:
-  enum Kind { NoRet, Alias, OwnedSymbol, OwnedAllocatedSymbol,
+  enum Kind { NoRet, OwnedSymbol, OwnedAllocatedSymbol,
               NotOwnedSymbol, GCNotOwnedSymbol, ARCNotOwnedSymbol,
-              ReceiverAlias,
               OwnedWhenTrackedReceiver };
 
   enum ObjKind { CF, ObjC, AnyObj };
@@ -158,36 +102,27 @@ class RetEffect {
 private:
   Kind K;
   ObjKind O;
-  unsigned index;
 
-  RetEffect(Kind k, unsigned idx = 0) : K(k), O(AnyObj), index(idx) {}
-  RetEffect(Kind k, ObjKind o) : K(k), O(o), index(0) {}
+  RetEffect(Kind k, ObjKind o = AnyObj) : K(k), O(o) {}
 
 public:
   Kind getKind() const { return K; }
 
   ObjKind getObjKind() const { return O; }
 
-  unsigned getIndex() const {
-    assert(getKind() == Alias);
-    return index;
-  }
-
   bool isOwned() const {
     return K == OwnedSymbol || K == OwnedAllocatedSymbol ||
            K == OwnedWhenTrackedReceiver;
   }
 
+  bool operator==(const RetEffect &Other) const {
+    return K == Other.K && O == Other.O;
+  }
+
   static RetEffect MakeOwnedWhenTrackedReceiver() {
     return RetEffect(OwnedWhenTrackedReceiver, ObjC);
   }
 
-  static RetEffect MakeAlias(unsigned Idx) {
-    return RetEffect(Alias, Idx);
-  }
-  static RetEffect MakeReceiverAlias() {
-    return RetEffect(ReceiverAlias);
-  }
   static RetEffect MakeOwned(ObjKind o, bool isAllocated = false) {
     return RetEffect(isAllocated ? OwnedAllocatedSymbol : OwnedSymbol, o);
   }
@@ -314,15 +249,15 @@ class RefVal {
     ID.Add(T);
   }
 
-  void print(llvm::raw_ostream& Out) const;
+  void print(raw_ostream &Out) const;
 };
 
-void RefVal::print(llvm::raw_ostream& Out) const {
+void RefVal::print(raw_ostream &Out) const {
   if (!T.isNull())
-    Out << "Tracked Type:" << T.getAsString() << '\n';
+    Out << "Tracked " << T.getAsString() << '/';
 
   switch (getKind()) {
-    default: assert(false);
+    default: llvm_unreachable("Invalid RefVal kind");
     case Owned: {
       Out << "Owned";
       unsigned cnt = getCount();
@@ -406,18 +341,19 @@ typedef llvm::ImmutableMap<SymbolRef, RefVal> RefBindings;
 
 namespace clang {
 namespace ento {
-  template<>
-  struct GRStateTrait<RefBindings> : public GRStatePartialTrait<RefBindings> {
-    static void* GDMIndex() {
-      static int RefBIndex = 0;
-      return &RefBIndex;
-    }
-  };
+template<>
+struct ProgramStateTrait<RefBindings>
+  : public ProgramStatePartialTrait<RefBindings> {
+  static void *GDMIndex() {
+    static int RefBIndex = 0;
+    return &RefBIndex;
+  }
+};
 }
 }
 
 //===----------------------------------------------------------------------===//
-// Summaries
+// Function/Method behavior summaries.
 //===----------------------------------------------------------------------===//
 
 namespace {
@@ -436,19 +372,13 @@ class RetainSummary {
   ArgEffect   Receiver;
 
   /// Ret - The effect on the return value.  Used to indicate if the
-  ///  function/method call returns a new tracked symbol, returns an
-  ///  alias of one of the arguments in the call, and so on.
+  ///  function/method call returns a new tracked symbol.
   RetEffect   Ret;
 
-  /// EndPath - Indicates that execution of this method/function should
-  ///  terminate the simulation of a path.
-  bool EndPath;
-
 public:
   RetainSummary(ArgEffects A, RetEffect R, ArgEffect defaultEff,
-                ArgEffect ReceiverEff, bool endpath = false)
-    : Args(A), DefaultArgEffect(defaultEff), Receiver(ReceiverEff), Ret(R),
-      EndPath(endpath) {}
+                ArgEffect ReceiverEff)
+    : Args(A), DefaultArgEffect(defaultEff), Receiver(ReceiverEff), Ret(R) {}
 
   /// getArg - Return the argument effect on the argument specified by
   ///  idx (starting from 0).
@@ -474,10 +404,6 @@ class RetainSummary {
   /// setRetEffect - Set the effect of the return value of the call.
   void setRetEffect(RetEffect E) { Ret = E; }
 
-  /// isEndPath - Returns true if executing the given method/function should
-  ///  terminate the path.
-  bool isEndPath() const { return EndPath; }
-
   
   /// Sets the effect on the receiver of the message.
   void setReceiverEffect(ArgEffect e) { Receiver = e; }
@@ -485,6 +411,14 @@ class RetainSummary {
   /// getReceiverEffect - Returns the effect on the receiver of the call.
   ///  This is only meaningful if the summary applies to an ObjCMessageExpr*.
   ArgEffect getReceiverEffect() const { return Receiver; }
+
+  /// Test if two retain summaries are identical. Note that merely equivalent
+  /// summaries are not necessarily identical (for example, if an explicit 
+  /// argument effect matches the default effect).
+  bool operator==(const RetainSummary &Other) const {
+    return Args == Other.Args && DefaultArgEffect == Other.DefaultArgEffect &&
+           Receiver == Other.Receiver && Ret == Other.Ret;
+  }
 };
 } // end anonymous namespace
 
@@ -500,10 +434,10 @@ class ObjCSummaryKey {
   ObjCSummaryKey(IdentifierInfo* ii, Selector s)
     : II(ii), S(s) {}
 
-  ObjCSummaryKey(const ObjCInterfaceDecl* d, Selector s)
+  ObjCSummaryKey(const ObjCInterfaceDecl *d, Selector s)
     : II(d ? d->getIdentifier() : 0), S(s) {}
 
-  ObjCSummaryKey(const ObjCInterfaceDecl* d, IdentifierInfo *ii, Selector s)
+  ObjCSummaryKey(const ObjCInterfaceDecl *d, IdentifierInfo *ii, Selector s)
     : II(d ? d->getIdentifier() : ii), S(s) {}
 
   ObjCSummaryKey(Selector s)
@@ -547,19 +481,19 @@ struct isPodLike<ObjCSummaryKey> { static const bool value = true; };
 
 namespace {
 class ObjCSummaryCache {
-  typedef llvm::DenseMap<ObjCSummaryKey, RetainSummary*> MapTy;
+  typedef llvm::DenseMap<ObjCSummaryKey, const RetainSummary *> MapTy;
   MapTy M;
 public:
   ObjCSummaryCache() {}
 
-  RetainSummary* find(const ObjCInterfaceDecl* D, IdentifierInfo *ClsName,
+  const RetainSummary * find(const ObjCInterfaceDecl *D, IdentifierInfo *ClsName,
                 Selector S) {
     // Lookup the method using the decl for the class @interface.  If we
     // have no decl, lookup using the class name.
     return D ? find(D, S) : find(ClsName, S);
   }
 
-  RetainSummary* find(const ObjCInterfaceDecl* D, Selector S) {
+  const RetainSummary * find(const ObjCInterfaceDecl *D, Selector S) {
     // Do a lookup with the (D,S) pair.  If we find a match return
     // the iterator.
     ObjCSummaryKey K(D, S);
@@ -574,7 +508,7 @@ class ObjCSummaryCache {
     // generate initial summaries without having to worry about NSObject
     // being declared.
     // FIXME: We may change this at some point.
-    for (ObjCInterfaceDecl* C=D->getSuperClass() ;; C=C->getSuperClass()) {
+    for (ObjCInterfaceDecl *C=D->getSuperClass() ;; C=C->getSuperClass()) {
       if ((I = M.find(ObjCSummaryKey(C, S))) != M.end())
         break;
 
@@ -584,12 +518,12 @@ class ObjCSummaryCache {
 
     // Cache the summary with original key to make the next lookup faster
     // and return the iterator.
-    RetainSummary *Summ = I->second;
+    const RetainSummary *Summ = I->second;
     M[K] = Summ;
     return Summ;
   }
 
-  RetainSummary* find(IdentifierInfo* II, Selector S) {
+  const RetainSummary * find(IdentifierInfo* II, Selector S) {
     // FIXME: Class method lookup.  Right now we dont' have a good way
     // of going between IdentifierInfo* and the class hierarchy.
     MapTy::iterator I = M.find(ObjCSummaryKey(II, S));
@@ -600,11 +534,11 @@ class ObjCSummaryCache {
     return I == M.end() ? NULL : I->second;
   }
 
-  RetainSummary*& operator[](ObjCSummaryKey K) {
+  const RetainSummary *& operator[](ObjCSummaryKey K) {
     return M[K];
   }
 
-  RetainSummary*& operator[](Selector S) {
+  const RetainSummary *& operator[](Selector S) {
     return M[ ObjCSummaryKey(S) ];
   }
 };
@@ -621,7 +555,7 @@ class RetainSummaryManager {
   //  Typedefs.
   //==-----------------------------------------------------------------==//
 
-  typedef llvm::DenseMap<const FunctionDecl*, RetainSummary*>
+  typedef llvm::DenseMap<const FunctionDecl*, const RetainSummary *>
           FuncSummariesTy;
 
   typedef ObjCSummaryCache ObjCMethodSummariesTy;
@@ -631,11 +565,7 @@ class RetainSummaryManager {
   //==-----------------------------------------------------------------==//
 
   /// Ctx - The ASTContext object for the analyzed ASTs.
-  ASTContext& Ctx;
-
-  /// CFDictionaryCreateII - An IdentifierInfo* representing the indentifier
-  ///  "CFDictionaryCreate".
-  IdentifierInfo* CFDictionaryCreateII;
+  ASTContext &Ctx;
 
   /// GCEnabled - Records whether or not the analyzed code runs in GC mode.
   const bool GCEnabled;
@@ -672,7 +602,7 @@ class RetainSummaryManager {
   RetEffect ObjCInitRetE;
 
   RetainSummary DefaultSummary;
-  RetainSummary* StopSummary;
+  const RetainSummary *StopSummary;
 
   //==-----------------------------------------------------------------==//
   //  Methods.
@@ -687,30 +617,28 @@ class RetainSummaryManager {
 public:
   RetEffect getObjAllocRetEffect() const { return ObjCAllocRetE; }
 
-  RetainSummary *getDefaultSummary() {
-    RetainSummary *Summ = (RetainSummary*) BPAlloc.Allocate<RetainSummary>();
-    return new (Summ) RetainSummary(DefaultSummary);
+  const RetainSummary *getDefaultSummary() {
+    return &DefaultSummary;
   }
+  
+  const RetainSummary * getUnarySummary(const FunctionType* FT,
+                                       UnaryFuncKind func);
 
-  RetainSummary* getUnarySummary(const FunctionType* FT, UnaryFuncKind func);
+  const RetainSummary * getCFSummaryCreateRule(const FunctionDecl *FD);
+  const RetainSummary * getCFSummaryGetRule(const FunctionDecl *FD);
+  const RetainSummary * getCFCreateGetRuleSummary(const FunctionDecl *FD);
 
-  RetainSummary* getCFSummaryCreateRule(const FunctionDecl* FD);
-  RetainSummary* getCFSummaryGetRule(const FunctionDecl* FD);
-  RetainSummary* getCFCreateGetRuleSummary(const FunctionDecl* FD, 
-                                           StringRef FName);
+  const RetainSummary * getPersistentSummary(ArgEffects AE, RetEffect RetEff,
+                                            ArgEffect ReceiverEff = DoNothing,
+                                            ArgEffect DefaultEff = MayEscape);
 
-  RetainSummary* getPersistentSummary(ArgEffects AE, RetEffect RetEff,
-                                      ArgEffect ReceiverEff = DoNothing,
-                                      ArgEffect DefaultEff = MayEscape,
-                                      bool isEndPath = false);
-
-  RetainSummary* getPersistentSummary(RetEffect RE,
-                                      ArgEffect ReceiverEff = DoNothing,
-                                      ArgEffect DefaultEff = MayEscape) {
+  const RetainSummary * getPersistentSummary(RetEffect RE,
+                                            ArgEffect ReceiverEff = DoNothing,
+                                            ArgEffect DefaultEff = MayEscape) {
     return getPersistentSummary(getArgEffects(), RE, ReceiverEff, DefaultEff);
   }
 
-  RetainSummary *getPersistentStopSummary() {
+  const RetainSummary *getPersistentStopSummary() {
     if (StopSummary)
       return StopSummary;
 
@@ -720,35 +648,35 @@ class RetainSummaryManager {
     return StopSummary;
   }
 
-  RetainSummary *getInitMethodSummary(QualType RetTy);
+  const RetainSummary *getInitMethodSummary(QualType RetTy);
 
   void InitializeClassMethodSummaries();
   void InitializeMethodSummaries();
 private:
-  void addNSObjectClsMethSummary(Selector S, RetainSummary *Summ) {
+  void addNSObjectClsMethSummary(Selector S, const RetainSummary *Summ) {
     ObjCClassMethodSummaries[S] = Summ;
   }
 
-  void addNSObjectMethSummary(Selector S, RetainSummary *Summ) {
+  void addNSObjectMethSummary(Selector S, const RetainSummary *Summ) {
     ObjCMethodSummaries[S] = Summ;
   }
 
   void addClassMethSummary(const char* Cls, const char* nullaryName,
-                           RetainSummary *Summ) {
+                           const RetainSummary *Summ) {
     IdentifierInfo* ClsII = &Ctx.Idents.get(Cls);
     Selector S = GetNullarySelector(nullaryName, Ctx);
     ObjCClassMethodSummaries[ObjCSummaryKey(ClsII, S)]  = Summ;
   }
 
   void addInstMethSummary(const char* Cls, const char* nullaryName,
-                          RetainSummary *Summ) {
+                          const RetainSummary *Summ) {
     IdentifierInfo* ClsII = &Ctx.Idents.get(Cls);
     Selector S = GetNullarySelector(nullaryName, Ctx);
     ObjCMethodSummaries[ObjCSummaryKey(ClsII, S)]  = Summ;
   }
 
   Selector generateSelector(va_list argp) {
-    llvm::SmallVector<IdentifierInfo*, 10> II;
+    SmallVector<IdentifierInfo*, 10> II;
 
     while (const char* s = va_arg(argp, const char*))
       II.push_back(&Ctx.Idents.get(s));
@@ -757,47 +685,36 @@ class RetainSummaryManager {
   }
 
   void addMethodSummary(IdentifierInfo *ClsII, ObjCMethodSummariesTy& Summaries,
-                        RetainSummary* Summ, va_list argp) {
+                        const RetainSummary * Summ, va_list argp) {
     Selector S = generateSelector(argp);
     Summaries[ObjCSummaryKey(ClsII, S)] = Summ;
   }
 
-  void addInstMethSummary(const char* Cls, RetainSummary* Summ, ...) {
+  void addInstMethSummary(const char* Cls, const RetainSummary * Summ, ...) {
     va_list argp;
     va_start(argp, Summ);
     addMethodSummary(&Ctx.Idents.get(Cls), ObjCMethodSummaries, Summ, argp);
     va_end(argp);
   }
 
-  void addClsMethSummary(const char* Cls, RetainSummary* Summ, ...) {
+  void addClsMethSummary(const char* Cls, const RetainSummary * Summ, ...) {
     va_list argp;
     va_start(argp, Summ);
     addMethodSummary(&Ctx.Idents.get(Cls),ObjCClassMethodSummaries, Summ, argp);
     va_end(argp);
   }
 
-  void addClsMethSummary(IdentifierInfo *II, RetainSummary* Summ, ...) {
+  void addClsMethSummary(IdentifierInfo *II, const RetainSummary * Summ, ...) {
     va_list argp;
     va_start(argp, Summ);
     addMethodSummary(II, ObjCClassMethodSummaries, Summ, argp);
     va_end(argp);
   }
 
-  void addPanicSummary(const char* Cls, ...) {
-    RetainSummary* Summ = getPersistentSummary(AF.getEmptyMap(),
-                                               RetEffect::MakeNoRet(),
-                                               DoNothing,  DoNothing, true);
-    va_list argp;
-    va_start (argp, Cls);
-    addMethodSummary(&Ctx.Idents.get(Cls), ObjCMethodSummaries, Summ, argp);
-    va_end(argp);
-  }
-
 public:
 
-  RetainSummaryManager(ASTContext& ctx, bool gcenabled, bool usesARC)
+  RetainSummaryManager(ASTContext &ctx, bool gcenabled, bool usesARC)
    : Ctx(ctx),
-     CFDictionaryCreateII(&ctx.Idents.get("CFDictionaryCreate")),
      GCEnabled(gcenabled),
      ARCEnabled(usesARC),
      AF(BPAlloc), ScratchArgs(AF.getEmptyMap()),
@@ -819,31 +736,31 @@ class RetainSummaryManager {
     InitializeMethodSummaries();
   }
 
-  ~RetainSummaryManager();
+  const RetainSummary * getSummary(const FunctionDecl *FD);
 
-  RetainSummary* getSummary(const FunctionDecl* FD);
+  const RetainSummary *getInstanceMethodSummary(const ObjCMessage &msg,
+                                                const ProgramState *state,
+                                                const LocationContext *LC);
 
-  RetainSummary *getInstanceMethodSummary(const ObjCMessage &msg,
-                                          const GRState *state,
-                                          const LocationContext *LC);
-
-  RetainSummary* getInstanceMethodSummary(const ObjCMessage &msg,
-                                          const ObjCInterfaceDecl* ID) {
+  const RetainSummary * getInstanceMethodSummary(const ObjCMessage &msg,
+                                                const ObjCInterfaceDecl *ID) {
     return getInstanceMethodSummary(msg.getSelector(), 0,
                             ID, msg.getMethodDecl(), msg.getType(Ctx));
   }
 
-  RetainSummary* getInstanceMethodSummary(Selector S, IdentifierInfo *ClsName,
-                                          const ObjCInterfaceDecl* ID,
-                                          const ObjCMethodDecl *MD,
-                                          QualType RetTy);
+  const RetainSummary * getInstanceMethodSummary(Selector S,
+                                                IdentifierInfo *ClsName,
+                                                const ObjCInterfaceDecl *ID,
+                                                const ObjCMethodDecl *MD,
+                                                QualType RetTy);
 
-  RetainSummary *getClassMethodSummary(Selector S, IdentifierInfo *ClsName,
-                                       const ObjCInterfaceDecl *ID,
-                                       const ObjCMethodDecl *MD,
-                                       QualType RetTy);
+  const RetainSummary *getClassMethodSummary(Selector S,
+                                             IdentifierInfo *ClsName,
+                                             const ObjCInterfaceDecl *ID,
+                                             const ObjCMethodDecl *MD,
+                                             QualType RetTy);
 
-  RetainSummary *getClassMethodSummary(const ObjCMessage &msg) {
+  const RetainSummary *getClassMethodSummary(const ObjCMessage &msg) {
     const ObjCInterfaceDecl *Class = 0;
     if (!msg.isInstanceMessage())
       Class = msg.getReceiverInterface();
@@ -856,30 +773,26 @@ class RetainSummaryManager {
 
   /// getMethodSummary - This version of getMethodSummary is used to query
   ///  the summary for the current method being analyzed.
-  RetainSummary *getMethodSummary(const ObjCMethodDecl *MD) {
+  const RetainSummary *getMethodSummary(const ObjCMethodDecl *MD) {
     // FIXME: Eventually this should be unneeded.
     const ObjCInterfaceDecl *ID = MD->getClassInterface();
     Selector S = MD->getSelector();
     IdentifierInfo *ClsName = ID->getIdentifier();
     QualType ResultTy = MD->getResultType();
 
-    // Resolve the method decl last.
-    if (const ObjCMethodDecl *InterfaceMD = ResolveToInterfaceMethodDecl(MD))
-      MD = InterfaceMD;
-
     if (MD->isInstanceMethod())
       return getInstanceMethodSummary(S, ClsName, ID, MD, ResultTy);
     else
       return getClassMethodSummary(S, ClsName, ID, MD, ResultTy);
   }
 
-  RetainSummary* getCommonMethodSummary(const ObjCMethodDecl* MD,
-                                        Selector S, QualType RetTy);
+  const RetainSummary * getCommonMethodSummary(const ObjCMethodDecl *MD,
+                                              Selector S, QualType RetTy);
 
-  void updateSummaryFromAnnotations(RetainSummary &Summ,
+  void updateSummaryFromAnnotations(const RetainSummary *&Summ,
                                     const ObjCMethodDecl *MD);
 
-  void updateSummaryFromAnnotations(RetainSummary &Summ,
+  void updateSummaryFromAnnotations(const RetainSummary *&Summ,
                                     const FunctionDecl *FD);
 
   bool isGCEnabled() const { return GCEnabled; }
@@ -888,35 +801,69 @@ class RetainSummaryManager {
   
   bool isARCorGCEnabled() const { return GCEnabled || ARCEnabled; }
 
-  RetainSummary *copySummary(RetainSummary *OldSumm) {
-    RetainSummary *Summ = (RetainSummary*) BPAlloc.Allocate<RetainSummary>();
+  const RetainSummary *copySummary(const RetainSummary *OldSumm) {
+    RetainSummary *Summ = (RetainSummary *) BPAlloc.Allocate<RetainSummary>();
     new (Summ) RetainSummary(*OldSumm);
     return Summ;
   }
 };
 
+// Used to avoid allocating long-term (BPAlloc'd) memory for default retain
+// summaries. If a function or method looks like it has a default summary, but
+// it has annotations, the annotations are added to the stack-based template
+// and then copied into managed memory.
+class RetainSummaryTemplate {
+  RetainSummaryManager &Manager;
+  const RetainSummary *&RealSummary;
+  const RetainSummary *BaseSummary;
+  RetainSummary ScratchSummary;
+  bool Accessed;
+public:
+  RetainSummaryTemplate(const RetainSummary *&real, const RetainSummary &base,
+                        RetainSummaryManager &manager)
+  : Manager(manager),
+    RealSummary(real),
+    BaseSummary(&base),
+    ScratchSummary(base),
+    Accessed(false) {}
+
+  ~RetainSummaryTemplate() {
+    if (Accessed)
+      RealSummary = Manager.copySummary(&ScratchSummary);
+    else if (!RealSummary)
+      RealSummary = BaseSummary;
+  }
+
+  RetainSummary &operator*() {
+    Accessed = true;
+    return ScratchSummary;
+  }
+
+  RetainSummary *operator->() {
+    Accessed = true;
+    return &ScratchSummary;
+  }
+};
+
 } // end anonymous namespace
 
 //===----------------------------------------------------------------------===//
 // Implementation of checker data structures.
 //===----------------------------------------------------------------------===//
 
-RetainSummaryManager::~RetainSummaryManager() {}
-
 ArgEffects RetainSummaryManager::getArgEffects() {
   ArgEffects AE = ScratchArgs;
   ScratchArgs = AF.getEmptyMap();
   return AE;
 }
 
-RetainSummary*
+const RetainSummary *
 RetainSummaryManager::getPersistentSummary(ArgEffects AE, RetEffect RetEff,
                                            ArgEffect ReceiverEff,
-                                           ArgEffect DefaultEff,
-                                           bool isEndPath) {
+                                           ArgEffect DefaultEff) {
   // Create the summary and return it.
-  RetainSummary *Summ = (RetainSummary*) BPAlloc.Allocate<RetainSummary>();
-  new (Summ) RetainSummary(AE, RetEff, DefaultEff, ReceiverEff, isEndPath);
+  RetainSummary *Summ = (RetainSummary *) BPAlloc.Allocate<RetainSummary>();
+  new (Summ) RetainSummary(AE, RetEff, DefaultEff, ReceiverEff);
   return Summ;
 }
 
@@ -924,22 +871,28 @@ RetainSummaryManager::getPersistentSummary(ArgEffects AE, RetEffect RetEff,
 // Summary creation for functions (largely uses of Core Foundation).
 //===----------------------------------------------------------------------===//
 
-static bool isRetain(const FunctionDecl* FD, StringRef FName) {
+static bool isRetain(const FunctionDecl *FD, StringRef FName) {
   return FName.endswith("Retain");
 }
 
-static bool isRelease(const FunctionDecl* FD, StringRef FName) {
+static bool isRelease(const FunctionDecl *FD, StringRef FName) {
   return FName.endswith("Release");
 }
 
-RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
+static bool isMakeCollectable(const FunctionDecl *FD, StringRef FName) {
+  // FIXME: Remove FunctionDecl parameter.
+  // FIXME: Is it really okay if MakeCollectable isn't a suffix?
+  return FName.find("MakeCollectable") != StringRef::npos;
+}
+
+const RetainSummary * RetainSummaryManager::getSummary(const FunctionDecl *FD) {
   // Look up a summary in our cache of FunctionDecls -> Summaries.
   FuncSummariesTy::iterator I = FuncSummaries.find(FD);
   if (I != FuncSummaries.end())
     return I->second;
 
   // No summary?  Generate one.
-  RetainSummary *S = 0;
+  const RetainSummary *S = 0;
 
   do {
     // We generate "stop" summaries for implicitly defined functions.
@@ -1054,10 +1007,10 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
       if (cocoa::isRefType(RetTy, "CF", FName)) {
         if (isRetain(FD, FName))
           S = getUnarySummary(FT, cfretain);
-        else if (FName.find("MakeCollectable") != StringRef::npos)
+        else if (isMakeCollectable(FD, FName))
           S = getUnarySummary(FT, cfmakecollectable);
         else
-          S = getCFCreateGetRuleSummary(FD, FName);
+          S = getCFCreateGetRuleSummary(FD);
 
         break;
       }
@@ -1067,7 +1020,7 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
         if (isRetain(FD, FName))
           S = getUnarySummary(FT, cfretain);
         else
-          S = getCFCreateGetRuleSummary(FD, FName);
+          S = getCFCreateGetRuleSummary(FD);
 
         break;
       }
@@ -1076,7 +1029,7 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
       if (cocoa::isRefType(RetTy, "DADisk") ||
           cocoa::isRefType(RetTy, "DADissenter") ||
           cocoa::isRefType(RetTy, "DASessionRef")) {
-        S = getCFCreateGetRuleSummary(FD, FName);
+        S = getCFCreateGetRuleSummary(FD);
         break;
       }
 
@@ -1121,27 +1074,22 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
   }
   while (0);
 
-  if (!S)
-    S = getDefaultSummary();
-
   // Annotations override defaults.
-  assert(S);
-  updateSummaryFromAnnotations(*S, FD);
+  updateSummaryFromAnnotations(S, FD);
 
   FuncSummaries[FD] = S;
   return S;
 }
 
-RetainSummary*
-RetainSummaryManager::getCFCreateGetRuleSummary(const FunctionDecl* FD,
-                                                StringRef FName) {
-  if (coreFoundation::followsCreateRule(FName))
+const RetainSummary *
+RetainSummaryManager::getCFCreateGetRuleSummary(const FunctionDecl *FD) {
+  if (coreFoundation::followsCreateRule(FD))
     return getCFSummaryCreateRule(FD);
 
   return getCFSummaryGetRule(FD);
 }
 
-RetainSummary*
+const RetainSummary *
 RetainSummaryManager::getUnarySummary(const FunctionType* FT,
                                       UnaryFuncKind func) {
 
@@ -1153,44 +1101,27 @@ RetainSummaryManager::getUnarySummary(const FunctionType* FT,
 
   assert (ScratchArgs.isEmpty());
 
+  ArgEffect Effect;
   switch (func) {
-    case cfretain: {
-      ScratchArgs = AF.add(ScratchArgs, 0, IncRef);
-      return getPersistentSummary(RetEffect::MakeAlias(0),
-                                  DoNothing, DoNothing);
-    }
-
-    case cfrelease: {
-      ScratchArgs = AF.add(ScratchArgs, 0, DecRef);
-      return getPersistentSummary(RetEffect::MakeNoRet(),
-                                  DoNothing, DoNothing);
-    }
-
-    case cfmakecollectable: {
-      ScratchArgs = AF.add(ScratchArgs, 0, MakeCollectable);
-      return getPersistentSummary(RetEffect::MakeAlias(0),DoNothing, DoNothing);
-    }
-
-    default:
-      assert (false && "Not a supported unary function.");
-      return getDefaultSummary();
+    case cfretain: Effect = IncRef; break;
+    case cfrelease: Effect = DecRef; break;
+    case cfmakecollectable: Effect = MakeCollectable; break;
+    default: llvm_unreachable("Not a supported unary function.");
   }
+
+  ScratchArgs = AF.add(ScratchArgs, 0, Effect);
+  return getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, DoNothing);
 }
 
-RetainSummary* 
-RetainSummaryManager::getCFSummaryCreateRule(const FunctionDecl* FD) {
+const RetainSummary * 
+RetainSummaryManager::getCFSummaryCreateRule(const FunctionDecl *FD) {
   assert (ScratchArgs.isEmpty());
 
-  if (FD->getIdentifier() == CFDictionaryCreateII) {
-    ScratchArgs = AF.add(ScratchArgs, 1, DoNothingByRef);
-    ScratchArgs = AF.add(ScratchArgs, 2, DoNothingByRef);
-  }
-
   return getPersistentSummary(RetEffect::MakeOwned(RetEffect::CF, true));
 }
 
-RetainSummary* 
-RetainSummaryManager::getCFSummaryGetRule(const FunctionDecl* FD) {
+const RetainSummary * 
+RetainSummaryManager::getCFSummaryGetRule(const FunctionDecl *FD) {
   assert (ScratchArgs.isEmpty());
   return getPersistentSummary(RetEffect::MakeNotOwned(RetEffect::CF),
                               DoNothing, DoNothing);
@@ -1200,7 +1131,7 @@ RetainSummaryManager::getCFSummaryGetRule(const FunctionDecl* FD) {
 // Summary creation for Selectors.
 //===----------------------------------------------------------------------===//
 
-RetainSummary*
+const RetainSummary *
 RetainSummaryManager::getInitMethodSummary(QualType RetTy) {
   assert(ScratchArgs.isEmpty());
   // 'init' methods conceptually return a newly allocated object and claim
@@ -1213,22 +1144,24 @@ RetainSummaryManager::getInitMethodSummary(QualType RetTy) {
 }
 
 void
-RetainSummaryManager::updateSummaryFromAnnotations(RetainSummary &Summ,
+RetainSummaryManager::updateSummaryFromAnnotations(const RetainSummary *&Summ,
                                                    const FunctionDecl *FD) {
   if (!FD)
     return;
 
+  RetainSummaryTemplate Template(Summ, DefaultSummary, *this);
+
   // Effects on the parameters.
   unsigned parm_idx = 0;
   for (FunctionDecl::param_const_iterator pi = FD->param_begin(), 
          pe = FD->param_end(); pi != pe; ++pi, ++parm_idx) {
     const ParmVarDecl *pd = *pi;
     if (pd->getAttr<NSConsumedAttr>()) {
-      if (!GCEnabled)
-        Summ.addArg(AF, parm_idx, DecRef);      
-    }
-    else if(pd->getAttr<CFConsumedAttr>()) {
-      Summ.addArg(AF, parm_idx, DecRef);      
+      if (!GCEnabled) {
+        Template->addArg(AF, parm_idx, DecRef);      
+      }
+    } else if (pd->getAttr<CFConsumedAttr>()) {
+      Template->addArg(AF, parm_idx, DecRef);      
     }   
   }
   
@@ -1237,83 +1170,84 @@ RetainSummaryManager::updateSummaryFromAnnotations(RetainSummary &Summ,
   // Determine if there is a special return effect for this method.
   if (cocoa::isCocoaObjectRef(RetTy)) {
     if (FD->getAttr<NSReturnsRetainedAttr>()) {
-      Summ.setRetEffect(ObjCAllocRetE);
+      Template->setRetEffect(ObjCAllocRetE);
     }
     else if (FD->getAttr<CFReturnsRetainedAttr>()) {
-      Summ.setRetEffect(RetEffect::MakeOwned(RetEffect::CF, true));
+      Template->setRetEffect(RetEffect::MakeOwned(RetEffect::CF, true));
     }
     else if (FD->getAttr<NSReturnsNotRetainedAttr>()) {
-      Summ.setRetEffect(RetEffect::MakeNotOwned(RetEffect::ObjC));
+      Template->setRetEffect(RetEffect::MakeNotOwned(RetEffect::ObjC));
     }
     else if (FD->getAttr<CFReturnsNotRetainedAttr>()) {
-      Summ.setRetEffect(RetEffect::MakeNotOwned(RetEffect::CF));
+      Template->setRetEffect(RetEffect::MakeNotOwned(RetEffect::CF));
     }
-  }
-  else if (RetTy->getAs<PointerType>()) {
+  } else if (RetTy->getAs<PointerType>()) {
     if (FD->getAttr<CFReturnsRetainedAttr>()) {
-      Summ.setRetEffect(RetEffect::MakeOwned(RetEffect::CF, true));
+      Template->setRetEffect(RetEffect::MakeOwned(RetEffect::CF, true));
     }
     else if (FD->getAttr<CFReturnsNotRetainedAttr>()) {
-      Summ.setRetEffect(RetEffect::MakeNotOwned(RetEffect::CF));
+      Template->setRetEffect(RetEffect::MakeNotOwned(RetEffect::CF));
     }
   }
 }
 
 void
-RetainSummaryManager::updateSummaryFromAnnotations(RetainSummary &Summ,
-                                                  const ObjCMethodDecl *MD) {
+RetainSummaryManager::updateSummaryFromAnnotations(const RetainSummary *&Summ,
+                                                   const ObjCMethodDecl *MD) {
   if (!MD)
     return;
 
+  RetainSummaryTemplate Template(Summ, DefaultSummary, *this);
+
   bool isTrackedLoc = false;
 
   // Effects on the receiver.
   if (MD->getAttr<NSConsumesSelfAttr>()) {
     if (!GCEnabled)
-      Summ.setReceiverEffect(DecRefMsg);      
+      Template->setReceiverEffect(DecRefMsg);      
   }
   
   // Effects on the parameters.
   unsigned parm_idx = 0;
-  for (ObjCMethodDecl::param_iterator pi=MD->param_begin(), pe=MD->param_end();
+  for (ObjCMethodDecl::param_const_iterator
+         pi=MD->param_begin(), pe=MD->param_end();
        pi != pe; ++pi, ++parm_idx) {
     const ParmVarDecl *pd = *pi;
     if (pd->getAttr<NSConsumedAttr>()) {
       if (!GCEnabled)
-        Summ.addArg(AF, parm_idx, DecRef);      
+        Template->addArg(AF, parm_idx, DecRef);      
     }
     else if(pd->getAttr<CFConsumedAttr>()) {
-      Summ.addArg(AF, parm_idx, DecRef);      
+      Template->addArg(AF, parm_idx, DecRef);      
     }   
   }
   
   // Determine if there is a special return effect for this method.
   if (cocoa::isCocoaObjectRef(MD->getResultType())) {
     if (MD->getAttr<NSReturnsRetainedAttr>()) {
-      Summ.setRetEffect(ObjCAllocRetE);
+      Template->setRetEffect(ObjCAllocRetE);
       return;
     }
     if (MD->getAttr<NSReturnsNotRetainedAttr>()) {
-      Summ.setRetEffect(RetEffect::MakeNotOwned(RetEffect::ObjC));
+      Template->setRetEffect(RetEffect::MakeNotOwned(RetEffect::ObjC));
       return;
     }
 
     isTrackedLoc = true;
-  }
-
-  if (!isTrackedLoc)
+  } else {
     isTrackedLoc = MD->getResultType()->getAs<PointerType>() != NULL;
+  }
 
   if (isTrackedLoc) {
     if (MD->getAttr<CFReturnsRetainedAttr>())
-      Summ.setRetEffect(RetEffect::MakeOwned(RetEffect::CF, true));
+      Template->setRetEffect(RetEffect::MakeOwned(RetEffect::CF, true));
     else if (MD->getAttr<CFReturnsNotRetainedAttr>())
-      Summ.setRetEffect(RetEffect::MakeNotOwned(RetEffect::CF));
+      Template->setRetEffect(RetEffect::MakeNotOwned(RetEffect::CF));
   }
 }
 
-RetainSummary*
-RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl* MD,
+const RetainSummary *
+RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl *MD,
                                              Selector S, QualType RetTy) {
 
   if (MD) {
@@ -1322,9 +1256,9 @@ RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl* MD,
     // Delegates are a frequent form of false positives with the retain
     // count checker.
     unsigned i = 0;
-    for (ObjCMethodDecl::param_iterator I = MD->param_begin(),
+    for (ObjCMethodDecl::param_const_iterator I = MD->param_begin(),
          E = MD->param_end(); I != E; ++I, ++i)
-      if (ParmVarDecl *PD = *I) {
+      if (const ParmVarDecl *PD = *I) {
         QualType Ty = Ctx.getCanonicalType(PD->getType());
         if (Ty.getLocalUnqualifiedType() == Ctx.VoidPtrTy)
           ScratchArgs = AF.add(ScratchArgs, i, StopTracking);
@@ -1370,15 +1304,15 @@ RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl* MD,
   return getPersistentSummary(RetEffect::MakeNoRet(), ReceiverEff, MayEscape);
 }
 
-RetainSummary*
+const RetainSummary *
 RetainSummaryManager::getInstanceMethodSummary(const ObjCMessage &msg,
-                                               const GRState *state,
+                                               const ProgramState *state,
                                                const LocationContext *LC) {
 
   // We need the type-information of the tracked receiver object
   // Retrieve it from the state.
   const Expr *Receiver = msg.getInstanceReceiver();
-  const ObjCInterfaceDecl* ID = 0;
+  const ObjCInterfaceDecl *ID = 0;
 
   // FIXME: Is this really working as expected?  There are cases where
   //  we just use the 'ID' from the message expression.
@@ -1410,19 +1344,18 @@ RetainSummaryManager::getInstanceMethodSummary(const ObjCMessage &msg,
 
   // FIXME: The receiver could be a reference to a class, meaning that
   //  we should use the class method.
-  RetainSummary *Summ = getInstanceMethodSummary(msg, ID);
-  return Summ ? Summ : getDefaultSummary();
+  return getInstanceMethodSummary(msg, ID);
 }
 
-RetainSummary*
+const RetainSummary *
 RetainSummaryManager::getInstanceMethodSummary(Selector S,
                                                IdentifierInfo *ClsName,
-                                               const ObjCInterfaceDecl* ID,
+                                               const ObjCInterfaceDecl *ID,
                                                const ObjCMethodDecl *MD,
                                                QualType RetTy) {
 
   // Look up a summary in our summary cache.
-  RetainSummary *Summ = ObjCMethodSummaries.find(ID, ClsName, S);
+  const RetainSummary *Summ = ObjCMethodSummaries.find(ID, ClsName, S);
 
   if (!Summ) {
     assert(ScratchArgs.isEmpty());
@@ -1434,7 +1367,7 @@ RetainSummaryManager::getInstanceMethodSummary(Selector S,
       Summ = getCommonMethodSummary(MD, S, RetTy);
 
     // Annotations override defaults.
-    updateSummaryFromAnnotations(*Summ, MD);
+    updateSummaryFromAnnotations(Summ, MD);
 
     // Memoize the summary.
     ObjCMethodSummaries[ObjCSummaryKey(ID, ClsName, S)] = Summ;
@@ -1443,19 +1376,21 @@ RetainSummaryManager::getInstanceMethodSummary(Selector S,
   return Summ;
 }
 
-RetainSummary*
+const RetainSummary *
 RetainSummaryManager::getClassMethodSummary(Selector S, IdentifierInfo *ClsName,
                                             const ObjCInterfaceDecl *ID,
                                             const ObjCMethodDecl *MD,
                                             QualType RetTy) {
 
   assert(ClsName && "Class name must be specified.");
-  RetainSummary *Summ = ObjCClassMethodSummaries.find(ID, ClsName, S);
+  const RetainSummary *Summ = ObjCClassMethodSummaries.find(ID, ClsName, S);
 
   if (!Summ) {
     Summ = getCommonMethodSummary(MD, S, RetTy);
+
     // Annotations override defaults.
-    updateSummaryFromAnnotations(*Summ, MD);
+    updateSummaryFromAnnotations(Summ, MD);
+
     // Memoize the summary.
     ObjCClassMethodSummaries[ObjCSummaryKey(ID, ClsName, S)] = Summ;
   }
@@ -1465,8 +1400,6 @@ RetainSummaryManager::getClassMethodSummary(Selector S, IdentifierInfo *ClsName,
 
 void RetainSummaryManager::InitializeClassMethodSummaries() {
   assert(ScratchArgs.isEmpty());
-  RetainSummary* Summ = getPersistentSummary(ObjCAllocRetE);
-
   // Create the [NSAssertionHandler currentHander] summary.
   addClassMethSummary("NSAssertionHandler", "currentHandler",
                 getPersistentSummary(RetEffect::MakeNotOwned(RetEffect::ObjC)));
@@ -1482,7 +1415,8 @@ void RetainSummaryManager::InitializeClassMethodSummaries() {
   // used for delegates that can release the object.  When we have better
   // inter-procedural analysis we can potentially do something better.  This
   // workaround is to remove false positives.
-  Summ = getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, StopTracking);
+  const RetainSummary *Summ =
+    getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, StopTracking);
   IdentifierInfo *NSObjectII = &Ctx.Idents.get("NSObject");
   addClsMethSummary(NSObjectII, Summ, "performSelector", "withObject",
                     "afterDelay", NULL);
@@ -1506,7 +1440,7 @@ void RetainSummaryManager::InitializeMethodSummaries() {
 
   // Create the "init" selector.  It just acts as a pass-through for the
   // receiver.
-  RetainSummary *InitSumm = getPersistentSummary(ObjCInitRetE, DecRefMsg);
+  const RetainSummary *InitSumm = getPersistentSummary(ObjCInitRetE, DecRefMsg);
   addNSObjectMethSummary(GetNullarySelector("init", Ctx), InitSumm);
 
   // awakeAfterUsingCoder: behaves basically like an 'init' method.  It
@@ -1515,35 +1449,34 @@ void RetainSummaryManager::InitializeMethodSummaries() {
                          InitSumm);
 
   // The next methods are allocators.
-  RetainSummary *AllocSumm = getPersistentSummary(ObjCAllocRetE);
-  RetainSummary *CFAllocSumm =
+  const RetainSummary *AllocSumm = getPersistentSummary(ObjCAllocRetE);
+  const RetainSummary *CFAllocSumm =
     getPersistentSummary(RetEffect::MakeOwned(RetEffect::CF, true));
 
   // Create the "retain" selector.
-  RetEffect E = RetEffect::MakeReceiverAlias();
-  RetainSummary *Summ = getPersistentSummary(E, IncRefMsg);
+  RetEffect NoRet = RetEffect::MakeNoRet();
+  const RetainSummary *Summ = getPersistentSummary(NoRet, IncRefMsg);
   addNSObjectMethSummary(GetNullarySelector("retain", Ctx), Summ);
 
   // Create the "release" selector.
-  Summ = getPersistentSummary(E, DecRefMsg);
+  Summ = getPersistentSummary(NoRet, DecRefMsg);
   addNSObjectMethSummary(GetNullarySelector("release", Ctx), Summ);
 
   // Create the "drain" selector.
-  Summ = getPersistentSummary(E, isGCEnabled() ? DoNothing : DecRef);
+  Summ = getPersistentSummary(NoRet, isGCEnabled() ? DoNothing : DecRef);
   addNSObjectMethSummary(GetNullarySelector("drain", Ctx), Summ);
 
   // Create the -dealloc summary.
-  Summ = getPersistentSummary(RetEffect::MakeNoRet(), Dealloc);
+  Summ = getPersistentSummary(NoRet, Dealloc);
   addNSObjectMethSummary(GetNullarySelector("dealloc", Ctx), Summ);
 
   // Create the "autorelease" selector.
-  Summ = getPersistentSummary(E, Autorelease);
+  Summ = getPersistentSummary(NoRet, Autorelease);
   addNSObjectMethSummary(GetNullarySelector("autorelease", Ctx), Summ);
 
   // Specially handle NSAutoreleasePool.
   addInstMethSummary("NSAutoreleasePool", "init",
-                     getPersistentSummary(RetEffect::MakeReceiverAlias(),
-                                          NewAutoreleasePool));
+                     getPersistentSummary(NoRet, NewAutoreleasePool));
 
   // For NSWindow, allocated objects are (initially) self-owned.
   // FIXME: For now we opt for false negatives with NSWindow, as these objects
@@ -1551,7 +1484,7 @@ void RetainSummaryManager::InitializeMethodSummaries() {
   //  Thus, we need to track an NSWindow's display status.
   //  This is tracked in <rdar://problem/6062711>.
   //  See also http://llvm.org/bugs/show_bug.cgi?id=3714.
-  RetainSummary *NoTrackYet = getPersistentSummary(RetEffect::MakeNoRet(),
+  const RetainSummary *NoTrackYet = getPersistentSummary(RetEffect::MakeNoRet(),
                                                    StopTracking,
                                                    StopTracking);
 
@@ -1583,13 +1516,6 @@ void RetainSummaryManager::InitializeMethodSummaries() {
   // exit a method.
   addClassMethSummary("NSAutoreleasePool", "alloc", NoTrackYet);
 
-  // Create NSAssertionHandler summaries.
-  addPanicSummary("NSAssertionHandler", "handleFailureInFunction", "file",
-                  "lineNumber", "description", NULL);
-
-  addPanicSummary("NSAssertionHandler", "handleFailureInMethod", "object",
-                  "file", "lineNumber", "description", NULL);
-
   // Create summaries QCRenderer/QCView -createSnapShotImageOfType:
   addInstMethSummary("QCRenderer", AllocSumm,
                      "createSnapshotImageOfType", NULL);
@@ -1623,26 +1549,27 @@ namespace { class AutoreleaseStack {}; }
 
 namespace clang {
 namespace ento {
-template<> struct GRStateTrait<AutoreleaseStack>
-  : public GRStatePartialTrait<ARStack> {
-  static inline void* GDMIndex() { return &AutoRBIndex; }
+template<> struct ProgramStateTrait<AutoreleaseStack>
+  : public ProgramStatePartialTrait<ARStack> {
+  static inline void *GDMIndex() { return &AutoRBIndex; }
 };
 
-template<> struct GRStateTrait<AutoreleasePoolContents>
-  : public GRStatePartialTrait<ARPoolContents> {
-  static inline void* GDMIndex() { return &AutoRCIndex; }
+template<> struct ProgramStateTrait<AutoreleasePoolContents>
+  : public ProgramStatePartialTrait<ARPoolContents> {
+  static inline void *GDMIndex() { return &AutoRCIndex; }
 };
 } // end GR namespace
 } // end clang namespace
 
-static SymbolRef GetCurrentAutoreleasePool(const GRState* state) {
+static SymbolRef GetCurrentAutoreleasePool(const ProgramState *state) {
   ARStack stack = state->get<AutoreleaseStack>();
   return stack.isEmpty() ? SymbolRef() : stack.getHead();
 }
 
-static const GRState * SendAutorelease(const GRState *state,
-                                       ARCounts::Factory &F, SymbolRef sym) {
-
+static const ProgramState *
+SendAutorelease(const ProgramState *state,
+                ARCounts::Factory &F,
+                SymbolRef sym) {
   SymbolRef pool = GetCurrentAutoreleasePool(state);
   const ARCounts *cnts = state->get<AutoreleasePoolContents>(pool);
   ARCounts newCnts(0);
@@ -1657,197 +1584,12 @@ static const GRState * SendAutorelease(const GRState *state,
   return state->set<AutoreleasePoolContents>(pool, newCnts);
 }
 
-//===----------------------------------------------------------------------===//
-// Transfer functions.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class CFRefCount : public TransferFuncs {
-public:
-  class BindingsPrinter : public GRState::Printer {
-  public:
-    virtual void Print(llvm::raw_ostream& Out, const GRState* state,
-                       const char* nl, const char* sep);
-  };
-
-  typedef llvm::DenseMap<const ExplodedNode*, const RetainSummary*>
-    SummaryLogTy;
-
-  RetainSummaryManager Summaries;
-  SummaryLogTy SummaryLog;
-  const LangOptions&   LOpts;
-  ARCounts::Factory    ARCountFactory;
-
-  BugType *useAfterRelease, *releaseNotOwned;
-  BugType *deallocGC, *deallocNotOwned;
-  BugType *leakWithinFunction, *leakAtReturn;
-  BugType *overAutorelease;
-  BugType *returnNotOwnedForOwned;
-  BugReporter *BR;
-
-  const GRState * Update(const GRState * state, SymbolRef sym, RefVal V, ArgEffect E,
-                    RefVal::Kind& hasErr);
-
-  void ProcessNonLeakError(ExplodedNodeSet& Dst,
-                           StmtNodeBuilder& Builder,
-                           const Expr* NodeExpr, SourceRange ErrorRange,
-                           ExplodedNode* Pred,
-                           const GRState* St,
-                           RefVal::Kind hasErr, SymbolRef Sym);
-
-  const GRState * HandleSymbolDeath(const GRState * state, SymbolRef sid, RefVal V,
-                               llvm::SmallVectorImpl<SymbolRef> &Leaked);
-
-  ExplodedNode* ProcessLeaks(const GRState * state,
-                                      llvm::SmallVectorImpl<SymbolRef> &Leaked,
-                                      GenericNodeBuilderRefCount &Builder,
-                                      ExprEngine &Eng,
-                                      ExplodedNode *Pred = 0);
-
-public:
-  CFRefCount(ASTContext& Ctx, bool gcenabled, const LangOptions& lopts)
-    : Summaries(Ctx, gcenabled, (bool)lopts.ObjCAutoRefCount),
-      LOpts(lopts), useAfterRelease(0), releaseNotOwned(0),
-      deallocGC(0), deallocNotOwned(0),
-      leakWithinFunction(0), leakAtReturn(0), overAutorelease(0),
-      returnNotOwnedForOwned(0), BR(0) {}
-
-  virtual ~CFRefCount() {}
-
-  void RegisterChecks(ExprEngine &Eng);
-
-  virtual void RegisterPrinters(std::vector<GRState::Printer*>& Printers) {
-    Printers.push_back(new BindingsPrinter());
-  }
-
-  bool isGCEnabled() const { return Summaries.isGCEnabled(); }
-  bool isARCorGCEnabled() const { return Summaries.isARCorGCEnabled(); }
-  
-  const LangOptions& getLangOptions() const { return LOpts; }
-
-  const RetainSummary *getSummaryOfNode(const ExplodedNode *N) const {
-    SummaryLogTy::const_iterator I = SummaryLog.find(N);
-    return I == SummaryLog.end() ? 0 : I->second;
-  }
-
-  // Calls.
-
-  void evalSummary(ExplodedNodeSet& Dst,
-                   ExprEngine& Eng,
-                   StmtNodeBuilder& Builder,
-                   const Expr* Ex,
-                   const CallOrObjCMessage &callOrMsg,
-                   InstanceReceiver Receiver,
-                   const RetainSummary& Summ,
-                   const MemRegion *Callee,
-                   ExplodedNode* Pred, const GRState *state);
-
-  virtual void evalCall(ExplodedNodeSet& Dst,
-                        ExprEngine& Eng,
-                        StmtNodeBuilder& Builder,
-                        const CallExpr* CE, SVal L,
-                        ExplodedNode* Pred);
-
-
-  virtual void evalObjCMessage(ExplodedNodeSet& Dst,
-                               ExprEngine& Engine,
-                               StmtNodeBuilder& Builder,
-                               ObjCMessage msg,
-                               ExplodedNode* Pred,
-                               const GRState *state);
-  // Stores.
-  virtual void evalBind(StmtNodeBuilderRef& B, SVal location, SVal val);
-
-  // End-of-path.
-
-  virtual void evalEndPath(ExprEngine& Engine,
-                           EndOfFunctionNodeBuilder& Builder);
-
-  virtual void evalDeadSymbols(ExplodedNodeSet& Dst,
-                               ExprEngine& Engine,
-                               StmtNodeBuilder& Builder,
-                               ExplodedNode* Pred,
-                               const GRState* state,
-                               SymbolReaper& SymReaper);
-
-  std::pair<ExplodedNode*, const GRState *>
-  HandleAutoreleaseCounts(const GRState * state, GenericNodeBuilderRefCount Bd,
-                          ExplodedNode* Pred, ExprEngine &Eng,
-                          SymbolRef Sym, RefVal V, bool &stop);
-  // Return statements.
-
-  virtual void evalReturn(ExplodedNodeSet& Dst,
-                          ExprEngine& Engine,
-                          StmtNodeBuilder& Builder,
-                          const ReturnStmt* S,
-                          ExplodedNode* Pred);
-  
-  void evalReturnWithRetEffect(ExplodedNodeSet& Dst,
-                               ExprEngine& Engine,
-                               StmtNodeBuilder& Builder,
-                               const ReturnStmt* S,
-                               ExplodedNode* Pred,
-                               RetEffect RE, RefVal X,
-                               SymbolRef Sym, const GRState *state);
-
-
-  // Assumptions.
-
-  virtual const GRState *evalAssume(const GRState* state, SVal condition,
-                                    bool assumption);
-};
-
-} // end anonymous namespace
-
-static void PrintPool(llvm::raw_ostream &Out, SymbolRef Sym,
-                      const GRState *state) {
-  Out << ' ';
-  if (Sym)
-    Out << Sym->getSymbolID();
-  else
-    Out << "<pool>";
-  Out << ":{";
-
-  // Get the contents of the pool.
-  if (const ARCounts *cnts = state->get<AutoreleasePoolContents>(Sym))
-    for (ARCounts::iterator J=cnts->begin(), EJ=cnts->end(); J != EJ; ++J)
-      Out << '(' << J.getKey() << ',' << J.getData() << ')';
-
-  Out << '}';
-}
-
-void CFRefCount::BindingsPrinter::Print(llvm::raw_ostream& Out,
-                                        const GRState* state,
-                                        const char* nl, const char* sep) {
-
-  RefBindings B = state->get<RefBindings>();
-
-  if (!B.isEmpty())
-    Out << sep << nl;
-
-  for (RefBindings::iterator I=B.begin(), E=B.end(); I!=E; ++I) {
-    Out << (*I).first << " : ";
-    (*I).second.print(Out);
-    Out << nl;
-  }
-
-  // Print the autorelease stack.
-  Out << sep << nl << "AR pool stack:";
-  ARStack stack = state->get<AutoreleaseStack>();
-
-  PrintPool(Out, SymbolRef(), state);  // Print the caller's pool.
-  for (ARStack::iterator I=stack.begin(), E=stack.end(); I!=E; ++I)
-    PrintPool(Out, *I, state);
-
-  Out << nl;
-}
-
 //===----------------------------------------------------------------------===//
 // Error reporting.
 //===----------------------------------------------------------------------===//
-
 namespace {
+  typedef llvm::DenseMap<const ExplodedNode *, const RetainSummary *>
+    SummaryLogTy;
 
   //===-------------===//
   // Bug Descriptions. //
@@ -1855,35 +1597,30 @@ namespace {
 
   class CFRefBug : public BugType {
   protected:
-    CFRefCount& TF;
-
-    CFRefBug(CFRefCount* tf, llvm::StringRef name)
-    : BugType(name, "Memory (Core Foundation/Objective-C)"), TF(*tf) {}
+    CFRefBug(StringRef name)
+    : BugType(name, "Memory (Core Foundation/Objective-C)") {}
   public:
 
-    CFRefCount& getTF() { return TF; }
-
     // FIXME: Eventually remove.
-    virtual const char* getDescription() const = 0;
+    virtual const char *getDescription() const = 0;
 
     virtual bool isLeak() const { return false; }
   };
 
   class UseAfterRelease : public CFRefBug {
   public:
-    UseAfterRelease(CFRefCount* tf)
-    : CFRefBug(tf, "Use-after-release") {}
+    UseAfterRelease() : CFRefBug("Use-after-release") {}
 
-    const char* getDescription() const {
+    const char *getDescription() const {
       return "Reference-counted object is used after it is released";
     }
   };
 
   class BadRelease : public CFRefBug {
   public:
-    BadRelease(CFRefCount* tf) : CFRefBug(tf, "Bad release") {}
+    BadRelease() : CFRefBug("Bad release") {}
 
-    const char* getDescription() const {
+    const char *getDescription() const {
       return "Incorrect decrement of the reference count of an object that is "
              "not owned at this point by the caller";
     }
@@ -1891,8 +1628,8 @@ namespace {
 
   class DeallocGC : public CFRefBug {
   public:
-    DeallocGC(CFRefCount *tf)
-      : CFRefBug(tf, "-dealloc called while using garbage collection") {}
+    DeallocGC()
+    : CFRefBug("-dealloc called while using garbage collection") {}
 
     const char *getDescription() const {
       return "-dealloc called while using garbage collection";
@@ -1901,8 +1638,8 @@ namespace {
 
   class DeallocNotOwned : public CFRefBug {
   public:
-    DeallocNotOwned(CFRefCount *tf)
-      : CFRefBug(tf, "-dealloc sent to non-exclusively owned object") {}
+    DeallocNotOwned()
+    : CFRefBug("-dealloc sent to non-exclusively owned object") {}
 
     const char *getDescription() const {
       return "-dealloc sent to object that may be referenced elsewhere";
@@ -1911,8 +1648,8 @@ namespace {
 
   class OverAutorelease : public CFRefBug {
   public:
-    OverAutorelease(CFRefCount *tf) :
-      CFRefBug(tf, "Object sent -autorelease too many times") {}
+    OverAutorelease()
+    : CFRefBug("Object sent -autorelease too many times") {}
 
     const char *getDescription() const {
       return "Object sent -autorelease too many times";
@@ -1921,8 +1658,8 @@ namespace {
 
   class ReturnedNotOwnedForOwned : public CFRefBug {
   public:
-    ReturnedNotOwnedForOwned(CFRefCount *tf) :
-      CFRefBug(tf, "Method should return an owned object") {}
+    ReturnedNotOwnedForOwned()
+    : CFRefBug("Method should return an owned object") {}
 
     const char *getDescription() const {
       return "Object with a +0 retain count returned to caller where a +1 "
@@ -1933,141 +1670,170 @@ namespace {
   class Leak : public CFRefBug {
     const bool isReturn;
   protected:
-    Leak(CFRefCount* tf, llvm::StringRef name, bool isRet)
-    : CFRefBug(tf, name), isReturn(isRet) {}
+    Leak(StringRef name, bool isRet)
+    : CFRefBug(name), isReturn(isRet) {
+      // Leaks should not be reported if they are post-dominated by a sink.
+      setSuppressOnSink(true);
+    }
   public:
 
-    const char* getDescription() const { return ""; }
+    const char *getDescription() const { return ""; }
 
     bool isLeak() const { return true; }
   };
 
   class LeakAtReturn : public Leak {
   public:
-    LeakAtReturn(CFRefCount* tf, llvm::StringRef name)
-    : Leak(tf, name, true) {}
+    LeakAtReturn(StringRef name)
+    : Leak(name, true) {}
   };
 
   class LeakWithinFunction : public Leak {
   public:
-    LeakWithinFunction(CFRefCount* tf, llvm::StringRef name)
-    : Leak(tf, name, false) {}
+    LeakWithinFunction(StringRef name)
+    : Leak(name, false) {}
   };
 
   //===---------===//
   // Bug Reports.  //
   //===---------===//
 
-  class CFRefReport : public RangedBugReport {
+  class CFRefReportVisitor : public BugReporterVisitor {
   protected:
     SymbolRef Sym;
-    const CFRefCount &TF;
+    const SummaryLogTy &SummaryLog;
+    bool GCEnabled;
+    
   public:
-    CFRefReport(CFRefBug& D, const CFRefCount &tf,
-                ExplodedNode *n, SymbolRef sym)
-      : RangedBugReport(D, D.getDescription(), n), Sym(sym), TF(tf) {}
+    CFRefReportVisitor(SymbolRef sym, bool gcEnabled, const SummaryLogTy &log)
+       : Sym(sym), SummaryLog(log), GCEnabled(gcEnabled) {}
 
-    CFRefReport(CFRefBug& D, const CFRefCount &tf,
-                ExplodedNode *n, SymbolRef sym, llvm::StringRef endText)
-      : RangedBugReport(D, D.getDescription(), endText, n), Sym(sym), TF(tf) {}
-
-    virtual ~CFRefReport() {}
-
-    CFRefBug& getBugType() const {
-      return (CFRefBug&) RangedBugReport::getBugType();
+    virtual void Profile(llvm::FoldingSetNodeID &ID) const {
+      static int x = 0;
+      ID.AddPointer(&x);
+      ID.AddPointer(Sym);
     }
 
-    virtual std::pair<ranges_iterator, ranges_iterator> getRanges() const {
-      if (!getBugType().isLeak())
-        return RangedBugReport::getRanges();
+    virtual PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                           const ExplodedNode *PrevN,
+                                           BugReporterContext &BRC,
+                                           BugReport &BR);
+
+    virtual PathDiagnosticPiece *getEndPath(BugReporterContext &BRC,
+                                            const ExplodedNode *N,
+                                            BugReport &BR);
+  };
+
+  class CFRefLeakReportVisitor : public CFRefReportVisitor {
+  public:
+    CFRefLeakReportVisitor(SymbolRef sym, bool GCEnabled,
+                           const SummaryLogTy &log)
+       : CFRefReportVisitor(sym, GCEnabled, log) {}
+
+    PathDiagnosticPiece *getEndPath(BugReporterContext &BRC,
+                                    const ExplodedNode *N,
+                                    BugReport &BR);
+  };
+
+  class CFRefReport : public BugReport {
+    void addGCModeDescription(const LangOptions &LOpts, bool GCEnabled);
+
+  public:
+    CFRefReport(CFRefBug &D, const LangOptions &LOpts, bool GCEnabled,
+                const SummaryLogTy &Log, ExplodedNode *n, SymbolRef sym,
+                bool registerVisitor = true)
+      : BugReport(D, D.getDescription(), n) {
+      if (registerVisitor)
+        addVisitor(new CFRefReportVisitor(sym, GCEnabled, Log));
+      addGCModeDescription(LOpts, GCEnabled);
+    }
+
+    CFRefReport(CFRefBug &D, const LangOptions &LOpts, bool GCEnabled,
+                const SummaryLogTy &Log, ExplodedNode *n, SymbolRef sym,
+                StringRef endText)
+      : BugReport(D, D.getDescription(), endText, n) {
+      addVisitor(new CFRefReportVisitor(sym, GCEnabled, Log));
+      addGCModeDescription(LOpts, GCEnabled);
+    }
+
+    virtual std::pair<ranges_iterator, ranges_iterator> getRanges() {
+      const CFRefBug& BugTy = static_cast<CFRefBug&>(getBugType());
+      if (!BugTy.isLeak())
+        return BugReport::getRanges();
       else
         return std::make_pair(ranges_iterator(), ranges_iterator());
     }
-
-    SymbolRef getSymbol() const { return Sym; }
-
-    PathDiagnosticPiece* getEndPath(BugReporterContext& BRC,
-                                    const ExplodedNode* N);
-
-    std::pair<const char**,const char**> getExtraDescriptiveText();
-
-    PathDiagnosticPiece* VisitNode(const ExplodedNode* N,
-                                   const ExplodedNode* PrevN,
-                                   BugReporterContext& BRC);
   };
 
   class CFRefLeakReport : public CFRefReport {
-    SourceLocation AllocSite;
     const MemRegion* AllocBinding;
+
   public:
-    CFRefLeakReport(CFRefBug& D, const CFRefCount &tf,
-                    ExplodedNode *n, SymbolRef sym,
-                    ExprEngine& Eng);
+    CFRefLeakReport(CFRefBug &D, const LangOptions &LOpts, bool GCEnabled,
+                    const SummaryLogTy &Log, ExplodedNode *n, SymbolRef sym,
+                    ExprEngine &Eng);
 
-    PathDiagnosticPiece* getEndPath(BugReporterContext& BRC,
-                                    const ExplodedNode* N);
-
-    SourceLocation getLocation() const { return AllocSite; }
+    PathDiagnosticLocation getLocation(const SourceManager &SM) const {
+      assert(Location.isValid());
+      return Location;
+    }
   };
 } // end anonymous namespace
 
+void CFRefReport::addGCModeDescription(const LangOptions &LOpts,
+                                       bool GCEnabled) {
+  const char *GCModeDescription = 0;
 
+  switch (LOpts.getGC()) {
+  case LangOptions::GCOnly:
+    assert(GCEnabled);
+    GCModeDescription = "Code is compiled to only use garbage collection";
+    break;
 
-static const char* Msgs[] = {
-  // GC only
-  "Code is compiled to only use garbage collection",
-  // No GC.
-  "Code is compiled to use reference counts",
-  // Hybrid, with GC.
-  "Code is compiled to use either garbage collection (GC) or reference counts"
-  " (non-GC).  The bug occurs with GC enabled",
-  // Hybrid, without GC
-  "Code is compiled to use either garbage collection (GC) or reference counts"
-  " (non-GC).  The bug occurs in non-GC mode"
-};
+  case LangOptions::NonGC:
+    assert(!GCEnabled);
+    GCModeDescription = "Code is compiled to use reference counts";
+    break;
 
-std::pair<const char**,const char**> CFRefReport::getExtraDescriptiveText() {
-  CFRefCount& TF = static_cast<CFRefBug&>(getBugType()).getTF();
-
-  switch (TF.getLangOptions().getGCMode()) {
-    default:
-      assert(false);
-
-    case LangOptions::GCOnly:
-      assert (TF.isGCEnabled());
-      return std::make_pair(&Msgs[0], &Msgs[0]+1);
-
-    case LangOptions::NonGC:
-      assert (!TF.isGCEnabled());
-      return std::make_pair(&Msgs[1], &Msgs[1]+1);
-
-    case LangOptions::HybridGC:
-      if (TF.isGCEnabled())
-        return std::make_pair(&Msgs[2], &Msgs[2]+1);
-      else
-        return std::make_pair(&Msgs[3], &Msgs[3]+1);
+  case LangOptions::HybridGC:
+    if (GCEnabled) {
+      GCModeDescription = "Code is compiled to use either garbage collection "
+                          "(GC) or reference counts (non-GC).  The bug occurs "
+                          "with GC enabled";
+      break;
+    } else {
+      GCModeDescription = "Code is compiled to use either garbage collection "
+                          "(GC) or reference counts (non-GC).  The bug occurs "
+                          "in non-GC mode";
+      break;
+    }
   }
+
+  assert(GCModeDescription && "invalid/unknown GC mode");
+  addExtraText(GCModeDescription);
 }
 
-static inline bool contains(const llvm::SmallVectorImpl<ArgEffect>& V,
+// FIXME: This should be a method on SmallVector.
+static inline bool contains(const SmallVectorImpl<ArgEffect>& V,
                             ArgEffect X) {
-  for (llvm::SmallVectorImpl<ArgEffect>::const_iterator I=V.begin(), E=V.end();
+  for (SmallVectorImpl<ArgEffect>::const_iterator I=V.begin(), E=V.end();
        I!=E; ++I)
     if (*I == X) return true;
 
   return false;
 }
 
-PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
-                                            const ExplodedNode* PrevN,
-                                            BugReporterContext& BRC) {
+PathDiagnosticPiece *CFRefReportVisitor::VisitNode(const ExplodedNode *N,
+                                                   const ExplodedNode *PrevN,
+                                                   BugReporterContext &BRC,
+                                                   BugReport &BR) {
 
-  if (!isa<PostStmt>(N->getLocation()))
+  if (!isa<StmtPoint>(N->getLocation()))
     return NULL;
 
   // Check if the type state has changed.
-  const GRState *PrevSt = PrevN->getState();
-  const GRState *CurrSt = N->getState();
+  const ProgramState *PrevSt = PrevN->getState();
+  const ProgramState *CurrSt = N->getState();
 
   const RefVal* CurrT = CurrSt->get<RefBindings>(Sym);
   if (!CurrT) return NULL;
@@ -2083,13 +1849,13 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
   // This is the allocation site since the previous node had no bindings
   // for this symbol.
   if (!PrevT) {
-    const Stmt* S = cast<PostStmt>(N->getLocation()).getStmt();
+    const Stmt *S = cast<StmtPoint>(N->getLocation()).getStmt();
 
     if (const CallExpr *CE = dyn_cast<CallExpr>(S)) {
       // Get the name of the callee (if it is available).
       SVal X = CurrSt->getSValAsScalarOrLoc(CE->getCallee());
-      if (const FunctionDecl* FD = X.getAsFunctionDecl())
-        os << "Call to function '" << FD << '\'';
+      if (const FunctionDecl *FD = X.getAsFunctionDecl())
+        os << "Call to function '" << *FD << '\'';
       else
         os << "function call";
     }
@@ -2110,7 +1876,7 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
     if (CurrV.isOwned()) {
       os << "+1 retain count";
 
-      if (static_cast<CFRefBug&>(getBugType()).getTF().isGCEnabled()) {
+      if (GCEnabled) {
         assert(CurrV.getObjKind() == RetEffect::CF);
         os << ".  "
         "Core Foundation objects are not automatically garbage collected.";
@@ -2121,19 +1887,20 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
       os << "+0 retain count";
     }
 
-    PathDiagnosticLocation Pos(S, BRC.getSourceManager());
+    PathDiagnosticLocation Pos(S, BRC.getSourceManager(),
+                                  N->getLocationContext());
     return new PathDiagnosticEventPiece(Pos, os.str());
   }
 
   // Gather up the effects that were performed on the object at this
   // program point
-  llvm::SmallVector<ArgEffect, 2> AEffects;
+  SmallVector<ArgEffect, 2> AEffects;
 
-  if (const RetainSummary *Summ =
-        TF.getSummaryOfNode(BRC.getNodeResolver().getOriginalNode(N))) {
+  const ExplodedNode *OrigNode = BRC.getNodeResolver().getOriginalNode(N);
+  if (const RetainSummary *Summ = SummaryLog.lookup(OrigNode)) {
     // We only have summaries attached to nodes after evaluating CallExpr and
     // ObjCMessageExprs.
-    const Stmt* S = cast<PostStmt>(N->getLocation()).getStmt();
+    const Stmt *S = cast<StmtPoint>(N->getLocation()).getStmt();
 
     if (const CallExpr *CE = dyn_cast<CallExpr>(S)) {
       // Iterate through the parameter expressions and see if the symbol
@@ -2166,7 +1933,7 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
     RefVal PrevV = *PrevT;
 
     // Specially handle -dealloc.
-    if (!TF.isGCEnabled() && contains(AEffects, Dealloc)) {
+    if (!GCEnabled && contains(AEffects, Dealloc)) {
       // Determine if the object's reference count was pushed to zero.
       assert(!(PrevV == CurrV) && "The typestate *must* have changed.");
       // We may not have transitioned to 'release' if we hit an error.
@@ -2181,16 +1948,15 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
     // Specially handle CFMakeCollectable and friends.
     if (contains(AEffects, MakeCollectable)) {
       // Get the name of the function.
-      const Stmt* S = cast<PostStmt>(N->getLocation()).getStmt();
+      const Stmt *S = cast<StmtPoint>(N->getLocation()).getStmt();
       SVal X = CurrSt->getSValAsScalarOrLoc(cast<CallExpr>(S)->getCallee());
-      const FunctionDecl* FD = X.getAsFunctionDecl();
-      const std::string& FName = FD->getNameAsString();
+      const FunctionDecl *FD = X.getAsFunctionDecl();
 
-      if (TF.isGCEnabled()) {
+      if (GCEnabled) {
         // Determine if the object's reference count was pushed to zero.
         assert(!(PrevV == CurrV) && "The typestate *must* have changed.");
 
-        os << "In GC mode a call to '" << FName
+        os << "In GC mode a call to '" << *FD
         <<  "' decrements an object's retain count and registers the "
         "object with the garbage collector. ";
 
@@ -2205,7 +1971,7 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
           << '.';
       }
       else
-        os << "When GC is not enabled a call to '" << FName
+        os << "When GC is not enabled a call to '" << *FD
         << "' has no effect on its argument.";
 
       // Nothing more to say.
@@ -2237,7 +2003,7 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
             os << " The object now has a +" << Count << " retain count.";
 
           if (PrevV.getKind() == RefVal::Released) {
-            assert(TF.isGCEnabled() && CurrV.getCount() > 0);
+            assert(GCEnabled && CurrV.getCount() > 0);
             os << " The object is not eligible for garbage collection until the "
             "retain count reaches 0 again.";
           }
@@ -2262,11 +2028,11 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
       }
 
     // Emit any remaining diagnostics for the argument effects (if any).
-    for (llvm::SmallVectorImpl<ArgEffect>::iterator I=AEffects.begin(),
+    for (SmallVectorImpl<ArgEffect>::iterator I=AEffects.begin(),
          E=AEffects.end(); I != E; ++I) {
 
       // A bunch of things have alternate behavior under GC.
-      if (TF.isGCEnabled())
+      if (GCEnabled)
         switch (*I) {
           default: break;
           case Autorelease:
@@ -2285,15 +2051,16 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
   if (os.str().empty())
     return 0; // We have nothing to say!
 
-  const Stmt* S = cast<PostStmt>(N->getLocation()).getStmt();
-  PathDiagnosticLocation Pos(S, BRC.getSourceManager());
-  PathDiagnosticPiece* P = new PathDiagnosticEventPiece(Pos, os.str());
+  const Stmt *S = cast<StmtPoint>(N->getLocation()).getStmt();
+  PathDiagnosticLocation Pos(S, BRC.getSourceManager(),
+                                N->getLocationContext());
+  PathDiagnosticPiece *P = new PathDiagnosticEventPiece(Pos, os.str());
 
   // Add the range by scanning the children of the statement for any bindings
   // to Sym.
   for (Stmt::const_child_iterator I = S->child_begin(), E = S->child_end();
        I!=E; ++I)
-    if (const Expr* Exp = dyn_cast_or_null<Expr>(*I))
+    if (const Expr *Exp = dyn_cast_or_null<Expr>(*I))
       if (CurrSt->getSValAsScalarOrLoc(Exp).getAsLocSymbol() == Sym) {
         P->addRange(Exp->getSourceRange());
         break;
@@ -2335,16 +2102,16 @@ namespace {
 }
 
 static std::pair<const ExplodedNode*,const MemRegion*>
-GetAllocationSite(GRStateManager& StateMgr, const ExplodedNode* N,
+GetAllocationSite(ProgramStateManager& StateMgr, const ExplodedNode *N,
                   SymbolRef Sym) {
 
   // Find both first node that referred to the tracked symbol and the
   // memory location that value was store to.
-  const ExplodedNode* Last = N;
+  const ExplodedNode *Last = N;
   const MemRegion* FirstBinding = 0;
 
   while (N) {
-    const GRState* St = N->getState();
+    const ProgramState *St = N->getState();
     RefBindings B = St->get<RefBindings>();
 
     if (!B.lookup(Sym))
@@ -2362,17 +2129,19 @@ GetAllocationSite(GRStateManager& StateMgr, const ExplodedNode* N,
 }
 
 PathDiagnosticPiece*
-CFRefReport::getEndPath(BugReporterContext& BRC,
-                        const ExplodedNode* EndN) {
+CFRefReportVisitor::getEndPath(BugReporterContext &BRC,
+                               const ExplodedNode *EndN,
+                               BugReport &BR) {
   // Tell the BugReporterContext to report cases when the tracked symbol is
   // assigned to different variables, etc.
   BRC.addNotableSymbol(Sym);
-  return RangedBugReport::getEndPath(BRC, EndN);
+  return BugReporterVisitor::getDefaultEndPath(BRC, EndN, BR);
 }
 
 PathDiagnosticPiece*
-CFRefLeakReport::getEndPath(BugReporterContext& BRC,
-                            const ExplodedNode* EndN){
+CFRefLeakReportVisitor::getEndPath(BugReporterContext &BRC,
+                                   const ExplodedNode *EndN,
+                                   BugReport &BR) {
 
   // Tell the BugReporterContext to report cases when the tracked symbol is
   // assigned to different variables, etc.
@@ -2381,41 +2150,19 @@ CFRefLeakReport::getEndPath(BugReporterContext& BRC,
   // We are reporting a leak.  Walk up the graph to get to the first node where
   // the symbol appeared, and also get the first VarDecl that tracked object
   // is stored to.
-  const ExplodedNode* AllocNode = 0;
+  const ExplodedNode *AllocNode = 0;
   const MemRegion* FirstBinding = 0;
 
   llvm::tie(AllocNode, FirstBinding) =
     GetAllocationSite(BRC.getStateManager(), EndN, Sym);
 
-  SourceManager& SMgr = BRC.getSourceManager();
+  SourceManager& SM = BRC.getSourceManager();
 
   // Compute an actual location for the leak.  Sometimes a leak doesn't
   // occur at an actual statement (e.g., transition between blocks; end
   // of function) so we need to walk the graph and compute a real location.
-  const ExplodedNode* LeakN = EndN;
-  PathDiagnosticLocation L;
-
-  while (LeakN) {
-    ProgramPoint P = LeakN->getLocation();
-
-    if (const PostStmt *PS = dyn_cast<PostStmt>(&P)) {
-      L = PathDiagnosticLocation(PS->getStmt()->getLocStart(), SMgr);
-      break;
-    }
-    else if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-      if (const Stmt* Term = BE->getSrc()->getTerminator()) {
-        L = PathDiagnosticLocation(Term->getLocStart(), SMgr);
-        break;
-      }
-    }
-
-    LeakN = LeakN->succ_empty() ? 0 : *(LeakN->succ_begin());
-  }
-
-  if (!L.isValid()) {
-    const Decl &D = EndN->getCodeDecl();
-    L = PathDiagnosticLocation(D.getBodyRBrace(), SMgr);
-  }
+  const ExplodedNode *LeakN = EndN;
+  PathDiagnosticLocation L = PathDiagnosticLocation::createEndOfPath(LeakN, SM);
 
   std::string sbuf;
   llvm::raw_string_ostream os(sbuf);
@@ -2454,7 +2201,7 @@ CFRefLeakReport::getEndPath(BugReporterContext& BRC,
     }    
   }
   else if (RV->getKind() == RefVal::ErrorGCLeakReturned) {
-    ObjCMethodDecl& MD = cast<ObjCMethodDecl>(EndN->getCodeDecl());
+    ObjCMethodDecl &MD = cast<ObjCMethodDecl>(EndN->getCodeDecl());
     os << " and returned from method '" << MD.getSelector().getAsString()
        << "' is potentially leaked when using garbage collection.  Callers "
           "of this method do not expect a returned object with a +1 retain "
@@ -2468,10 +2215,11 @@ CFRefLeakReport::getEndPath(BugReporterContext& BRC,
   return new PathDiagnosticEventPiece(L, os.str());
 }
 
-CFRefLeakReport::CFRefLeakReport(CFRefBug& D, const CFRefCount &tf,
-                                 ExplodedNode *n,
-                                 SymbolRef sym, ExprEngine& Eng)
-: CFRefReport(D, tf, n, sym) {
+CFRefLeakReport::CFRefLeakReport(CFRefBug &D, const LangOptions &LOpts,
+                                 bool GCEnabled, const SummaryLogTy &Log, 
+                                 ExplodedNode *n, SymbolRef sym,
+                                 ExprEngine &Eng)
+: CFRefReport(D, LOpts, GCEnabled, Log, n, sym, false) {
 
   // Most bug reports are cached at the location where they occurred.
   // With leaks, we want to unique them by the location where they were
@@ -2481,41 +2229,418 @@ CFRefLeakReport::CFRefLeakReport(CFRefBug& D, const CFRefCount &tf,
   // Note that this is *not* the trimmed graph; we are guaranteed, however,
   // that all ancestor nodes that represent the allocation site have the
   // same SourceLocation.
-  const ExplodedNode* AllocNode = 0;
+  const ExplodedNode *AllocNode = 0;
+
+  const SourceManager& SMgr = Eng.getContext().getSourceManager();
 
   llvm::tie(AllocNode, AllocBinding) =  // Set AllocBinding.
-    GetAllocationSite(Eng.getStateManager(), getErrorNode(), getSymbol());
+    GetAllocationSite(Eng.getStateManager(), getErrorNode(), sym);
 
   // Get the SourceLocation for the allocation site.
   ProgramPoint P = AllocNode->getLocation();
-  AllocSite = cast<PostStmt>(P).getStmt()->getLocStart();
-
+  const Stmt *AllocStmt = cast<PostStmt>(P).getStmt();
+  Location = PathDiagnosticLocation::createBegin(AllocStmt, SMgr,
+                                                  n->getLocationContext());
   // Fill in the description of the bug.
   Description.clear();
   llvm::raw_string_ostream os(Description);
-  SourceManager& SMgr = Eng.getContext().getSourceManager();
-  unsigned AllocLine = SMgr.getInstantiationLineNumber(AllocSite);
+  unsigned AllocLine = SMgr.getExpansionLineNumber(AllocStmt->getLocStart());
   os << "Potential leak ";
-  if (tf.isGCEnabled()) {
+  if (GCEnabled)
     os << "(when using garbage collection) ";
-  }
   os << "of an object allocated on line " << AllocLine;
 
   // FIXME: AllocBinding doesn't get populated for RegionStore yet.
   if (AllocBinding)
     os << " and stored into '" << AllocBinding->getString() << '\'';
+
+  addVisitor(new CFRefLeakReportVisitor(sym, GCEnabled, Log));
 }
 
 //===----------------------------------------------------------------------===//
 // Main checker logic.
 //===----------------------------------------------------------------------===//
 
+namespace {
+class RetainCountChecker
+  : public Checker< check::Bind,
+                    check::DeadSymbols,
+                    check::EndAnalysis,
+                    check::EndPath,
+                    check::PostStmt<BlockExpr>,
+                    check::PostStmt<CastExpr>,
+                    check::PostStmt<CallExpr>,
+                    check::PostStmt<CXXConstructExpr>,
+                    check::PostObjCMessage,
+                    check::PreStmt<ReturnStmt>,
+                    check::RegionChanges,
+                    eval::Assume,
+                    eval::Call > {
+  mutable llvm::OwningPtr<CFRefBug> useAfterRelease, releaseNotOwned;
+  mutable llvm::OwningPtr<CFRefBug> deallocGC, deallocNotOwned;
+  mutable llvm::OwningPtr<CFRefBug> overAutorelease, returnNotOwnedForOwned;
+  mutable llvm::OwningPtr<CFRefBug> leakWithinFunction, leakAtReturn;
+  mutable llvm::OwningPtr<CFRefBug> leakWithinFunctionGC, leakAtReturnGC;
+
+  typedef llvm::DenseMap<SymbolRef, const SimpleProgramPointTag *> SymbolTagMap;
+
+  // This map is only used to ensure proper deletion of any allocated tags.
+  mutable SymbolTagMap DeadSymbolTags;
+
+  mutable llvm::OwningPtr<RetainSummaryManager> Summaries;
+  mutable llvm::OwningPtr<RetainSummaryManager> SummariesGC;
+
+  mutable ARCounts::Factory ARCountFactory;
+
+  mutable SummaryLogTy SummaryLog;
+  mutable bool ShouldResetSummaryLog;
+
+public:  
+  RetainCountChecker() : ShouldResetSummaryLog(false) {}
+
+  virtual ~RetainCountChecker() {
+    DeleteContainerSeconds(DeadSymbolTags);
+  }
+
+  void checkEndAnalysis(ExplodedGraph &G, BugReporter &BR,
+                        ExprEngine &Eng) const {
+    // FIXME: This is a hack to make sure the summary log gets cleared between
+    // analyses of different code bodies.
+    //
+    // Why is this necessary? Because a checker's lifetime is tied to a
+    // translation unit, but an ExplodedGraph's lifetime is just a code body.
+    // Once in a blue moon, a new ExplodedNode will have the same address as an
+    // old one with an associated summary, and the bug report visitor gets very
+    // confused. (To make things worse, the summary lifetime is currently also
+    // tied to a code body, so we get a crash instead of incorrect results.)
+    //
+    // Why is this a bad solution? Because if the lifetime of the ExplodedGraph
+    // changes, things will start going wrong again. Really the lifetime of this
+    // log needs to be tied to either the specific nodes in it or the entire
+    // ExplodedGraph, not to a specific part of the code being analyzed.
+    //
+    // (Also, having stateful local data means that the same checker can't be
+    // used from multiple threads, but a lot of checkers have incorrect
+    // assumptions about that anyway. So that wasn't a priority at the time of
+    // this fix.)
+    //
+    // This happens at the end of analysis, but bug reports are emitted /after/
+    // this point. So we can't just clear the summary log now. Instead, we mark
+    // that the next time we access the summary log, it should be cleared.
+
+    // If we never reset the summary log during /this/ code body analysis,
+    // there were no new summaries. There might still have been summaries from
+    // the /last/ analysis, so clear them out to make sure the bug report
+    // visitors don't get confused.
+    if (ShouldResetSummaryLog)
+      SummaryLog.clear();
+
+    ShouldResetSummaryLog = !SummaryLog.empty();
+  }
+
+  CFRefBug *getLeakWithinFunctionBug(const LangOptions &LOpts,
+                                     bool GCEnabled) const {
+    if (GCEnabled) {
+      if (!leakWithinFunctionGC)
+        leakWithinFunctionGC.reset(new LeakWithinFunction("Leak of object when "
+                                                          "using garbage "
+                                                          "collection"));
+      return leakWithinFunctionGC.get();
+    } else {
+      if (!leakWithinFunction) {
+        if (LOpts.getGC() == LangOptions::HybridGC) {
+          leakWithinFunction.reset(new LeakWithinFunction("Leak of object when "
+                                                          "not using garbage "
+                                                          "collection (GC) in "
+                                                          "dual GC/non-GC "
+                                                          "code"));
+        } else {
+          leakWithinFunction.reset(new LeakWithinFunction("Leak"));
+        }
+      }
+      return leakWithinFunction.get();
+    }
+  }
+
+  CFRefBug *getLeakAtReturnBug(const LangOptions &LOpts, bool GCEnabled) const {
+    if (GCEnabled) {
+      if (!leakAtReturnGC)
+        leakAtReturnGC.reset(new LeakAtReturn("Leak of returned object when "
+                                              "using garbage collection"));
+      return leakAtReturnGC.get();
+    } else {
+      if (!leakAtReturn) {
+        if (LOpts.getGC() == LangOptions::HybridGC) {
+          leakAtReturn.reset(new LeakAtReturn("Leak of returned object when "
+                                              "not using garbage collection "
+                                              "(GC) in dual GC/non-GC code"));
+        } else {
+          leakAtReturn.reset(new LeakAtReturn("Leak of returned object"));
+        }
+      }
+      return leakAtReturn.get();
+    }
+  }
+
+  RetainSummaryManager &getSummaryManager(ASTContext &Ctx,
+                                          bool GCEnabled) const {
+    // FIXME: We don't support ARC being turned on and off during one analysis.
+    // (nor, for that matter, do we support changing ASTContexts)
+    bool ARCEnabled = (bool)Ctx.getLangOptions().ObjCAutoRefCount;
+    if (GCEnabled) {
+      if (!SummariesGC)
+        SummariesGC.reset(new RetainSummaryManager(Ctx, true, ARCEnabled));
+      else
+        assert(SummariesGC->isARCEnabled() == ARCEnabled);
+      return *SummariesGC;
+    } else {
+      if (!Summaries)
+        Summaries.reset(new RetainSummaryManager(Ctx, false, ARCEnabled));
+      else
+        assert(Summaries->isARCEnabled() == ARCEnabled);
+      return *Summaries;
+    }
+  }
+
+  RetainSummaryManager &getSummaryManager(CheckerContext &C) const {
+    return getSummaryManager(C.getASTContext(), C.isObjCGCEnabled());
+  }
+
+  void printState(raw_ostream &Out, const ProgramState *State,
+                  const char *NL, const char *Sep) const;
+
+  void checkBind(SVal loc, SVal val, const Stmt *S, CheckerContext &C) const;
+  void checkPostStmt(const BlockExpr *BE, CheckerContext &C) const;
+  void checkPostStmt(const CastExpr *CE, CheckerContext &C) const;
+
+  void checkPostStmt(const CallExpr *CE, CheckerContext &C) const;
+  void checkPostStmt(const CXXConstructExpr *CE, CheckerContext &C) const;
+  void checkPostObjCMessage(const ObjCMessage &Msg, CheckerContext &C) const;
+  void checkSummary(const RetainSummary &Summ, const CallOrObjCMessage &Call,
+                    CheckerContext &C) const;
+
+  bool evalCall(const CallExpr *CE, CheckerContext &C) const;
+
+  const ProgramState *evalAssume(const ProgramState *state, SVal Cond,
+                                 bool Assumption) const;
+
+  const ProgramState *
+  checkRegionChanges(const ProgramState *state,
+                     const StoreManager::InvalidatedSymbols *invalidated,
+                     ArrayRef<const MemRegion *> ExplicitRegions,
+                     ArrayRef<const MemRegion *> Regions) const;
+                                        
+  bool wantsRegionChangeUpdate(const ProgramState *state) const {
+    return true;
+  }
+
+  void checkPreStmt(const ReturnStmt *S, CheckerContext &C) const;
+  void checkReturnWithRetEffect(const ReturnStmt *S, CheckerContext &C,
+                                ExplodedNode *Pred, RetEffect RE, RefVal X,
+                                SymbolRef Sym, const ProgramState *state) const;
+                                              
+  void checkDeadSymbols(SymbolReaper &SymReaper, CheckerContext &C) const;
+  void checkEndPath(EndOfFunctionNodeBuilder &Builder, ExprEngine &Eng) const;
+
+  const ProgramState *updateSymbol(const ProgramState *state, SymbolRef sym,
+                                   RefVal V, ArgEffect E, RefVal::Kind &hasErr,
+                                   CheckerContext &C) const;
+
+  void processNonLeakError(const ProgramState *St, SourceRange ErrorRange,
+                           RefVal::Kind ErrorKind, SymbolRef Sym,
+                           CheckerContext &C) const;
+
+  const ProgramPointTag *getDeadSymbolTag(SymbolRef sym) const;
+
+  const ProgramState *handleSymbolDeath(const ProgramState *state,
+                                        SymbolRef sid, RefVal V,
+                                      SmallVectorImpl<SymbolRef> &Leaked) const;
+
+  std::pair<ExplodedNode *, const ProgramState *>
+  handleAutoreleaseCounts(const ProgramState *state, 
+                          GenericNodeBuilderRefCount Bd, ExplodedNode *Pred,
+                          ExprEngine &Eng, SymbolRef Sym, RefVal V) const;
+
+  ExplodedNode *processLeaks(const ProgramState *state,
+                             SmallVectorImpl<SymbolRef> &Leaked,
+                             GenericNodeBuilderRefCount &Builder,
+                             ExprEngine &Eng,
+                             ExplodedNode *Pred = 0) const;
+};
+} // end anonymous namespace
+
+namespace {
+class StopTrackingCallback : public SymbolVisitor {
+  const ProgramState *state;
+public:
+  StopTrackingCallback(const ProgramState *st) : state(st) {}
+  const ProgramState *getState() const { return state; }
+
+  bool VisitSymbol(SymbolRef sym) {
+    state = state->remove<RefBindings>(sym);
+    return true;
+  }
+};
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// Handle statements that may have an effect on refcounts.
+//===----------------------------------------------------------------------===//
+
+void RetainCountChecker::checkPostStmt(const BlockExpr *BE,
+                                       CheckerContext &C) const {
+
+  // Scan the BlockDecRefExprs for any object the retain count checker
+  // may be tracking.
+  if (!BE->getBlockDecl()->hasCaptures())
+    return;
+
+  const ProgramState *state = C.getState();
+  const BlockDataRegion *R =
+    cast<BlockDataRegion>(state->getSVal(BE).getAsRegion());
+
+  BlockDataRegion::referenced_vars_iterator I = R->referenced_vars_begin(),
+                                            E = R->referenced_vars_end();
+
+  if (I == E)
+    return;
+
+  // FIXME: For now we invalidate the tracking of all symbols passed to blocks
+  // via captured variables, even though captured variables result in a copy
+  // and in implicit increment/decrement of a retain count.
+  SmallVector<const MemRegion*, 10> Regions;
+  const LocationContext *LC = C.getPredecessor()->getLocationContext();
+  MemRegionManager &MemMgr = C.getSValBuilder().getRegionManager();
+
+  for ( ; I != E; ++I) {
+    const VarRegion *VR = *I;
+    if (VR->getSuperRegion() == R) {
+      VR = MemMgr.getVarRegion(VR->getDecl(), LC);
+    }
+    Regions.push_back(VR);
+  }
+
+  state =
+    state->scanReachableSymbols<StopTrackingCallback>(Regions.data(),
+                                    Regions.data() + Regions.size()).getState();
+  C.addTransition(state);
+}
+
+void RetainCountChecker::checkPostStmt(const CastExpr *CE,
+                                       CheckerContext &C) const {
+  const ObjCBridgedCastExpr *BE = dyn_cast<ObjCBridgedCastExpr>(CE);
+  if (!BE)
+    return;
+  
+  ArgEffect AE = IncRef;
+  
+  switch (BE->getBridgeKind()) {
+    case clang::OBC_Bridge:
+      // Do nothing.
+      return;
+    case clang::OBC_BridgeRetained:
+      AE = IncRef;
+      break;      
+    case clang::OBC_BridgeTransfer:
+      AE = DecRefBridgedTransfered;
+      break;
+  }
+  
+  const ProgramState *state = C.getState();
+  SymbolRef Sym = state->getSVal(CE).getAsLocSymbol();
+  if (!Sym)
+    return;
+  const RefVal* T = state->get<RefBindings>(Sym);
+  if (!T)
+    return;
+
+  RefVal::Kind hasErr = (RefVal::Kind) 0;
+  state = updateSymbol(state, Sym, *T, AE, hasErr, C);
+  
+  if (hasErr) {
+    // FIXME: If we get an error during a bridge cast, should we report it?
+    // Should we assert that there is no error?
+    return;
+  }
+
+  C.generateNode(state);
+}
+
+void RetainCountChecker::checkPostStmt(const CallExpr *CE,
+                                       CheckerContext &C) const {
+  // Get the callee.
+  const ProgramState *state = C.getState();
+  const Expr *Callee = CE->getCallee();
+  SVal L = state->getSVal(Callee);
+
+  RetainSummaryManager &Summaries = getSummaryManager(C);
+  const RetainSummary *Summ = 0;
+
+  // FIXME: Better support for blocks.  For now we stop tracking anything
+  // that is passed to blocks.
+  // FIXME: Need to handle variables that are "captured" by the block.
+  if (dyn_cast_or_null<BlockDataRegion>(L.getAsRegion())) {
+    Summ = Summaries.getPersistentStopSummary();
+  } else if (const FunctionDecl *FD = L.getAsFunctionDecl()) {
+    Summ = Summaries.getSummary(FD);
+  } else if (const CXXMemberCallExpr *me = dyn_cast<CXXMemberCallExpr>(CE)) {
+    if (const CXXMethodDecl *MD = me->getMethodDecl())
+      Summ = Summaries.getSummary(MD);
+  }
+
+  if (!Summ)
+    Summ = Summaries.getDefaultSummary();
+
+  checkSummary(*Summ, CallOrObjCMessage(CE, state), C);
+}
+
+void RetainCountChecker::checkPostStmt(const CXXConstructExpr *CE,
+                                       CheckerContext &C) const {
+  const CXXConstructorDecl *Ctor = CE->getConstructor();
+  if (!Ctor)
+    return;
+
+  RetainSummaryManager &Summaries = getSummaryManager(C);
+  const RetainSummary *Summ = Summaries.getSummary(Ctor);
+
+  // If we didn't get a summary, this constructor doesn't affect retain counts.
+  if (!Summ)
+    return;
+
+  const ProgramState *state = C.getState();
+  checkSummary(*Summ, CallOrObjCMessage(CE, state), C);
+}
+
+void RetainCountChecker::checkPostObjCMessage(const ObjCMessage &Msg, 
+                                              CheckerContext &C) const {
+  const ProgramState *state = C.getState();
+  ExplodedNode *Pred = C.getPredecessor();
+
+  RetainSummaryManager &Summaries = getSummaryManager(C);
+
+  const RetainSummary *Summ;
+  if (Msg.isInstanceMessage()) {
+    const LocationContext *LC = Pred->getLocationContext();
+    Summ = Summaries.getInstanceMethodSummary(Msg, state, LC);
+  } else {
+    Summ = Summaries.getClassMethodSummary(Msg);    
+  }
+
+  // If we didn't get a summary, this message doesn't affect retain counts.
+  if (!Summ)
+    return;
+
+  checkSummary(*Summ, CallOrObjCMessage(Msg, state), C);
+}
+
 /// GetReturnType - Used to get the return type of a message expression or
 ///  function call with the intention of affixing that type to a tracked symbol.
 ///  While the the return type can be queried directly from RetEx, when
 ///  invoking class methods we augment to the return type to be that of
 ///  a pointer to the class (as opposed it just being id).
-static QualType GetReturnType(const Expr* RetE, ASTContext& Ctx) {
+// FIXME: We may be able to do this with related result types instead.
+// This function is probably overestimating.
+static QualType GetReturnType(const Expr *RetE, ASTContext &Ctx) {
   QualType RetTy = RetE->getType();
   // If RetE is not a message expression just return its type.
   // If RetE is a message expression, return its types if it is something
@@ -2536,167 +2661,43 @@ static QualType GetReturnType(const Expr* RetE, ASTContext& Ctx) {
   return RetTy;
 }
 
-
-// HACK: Symbols that have ref-count state that are referenced directly
-//  (not as structure or array elements, or via bindings) by an argument
-//  should not have their ref-count state stripped after we have
-//  done an invalidation pass.
-//
-// FIXME: This is a global to currently share between CFRefCount and
-// RetainReleaseChecker.  Eventually all functionality in CFRefCount should
-// be migrated to RetainReleaseChecker, and we can make this a non-global.
-llvm::DenseSet<SymbolRef> WhitelistedSymbols;
-namespace {
-struct ResetWhiteList {
-  ResetWhiteList() {}
-  ~ResetWhiteList() { WhitelistedSymbols.clear(); } 
-};
-}
-
-void CFRefCount::evalSummary(ExplodedNodeSet& Dst,
-                             ExprEngine& Eng,
-                             StmtNodeBuilder& Builder,
-                             const Expr* Ex,
-                             const CallOrObjCMessage &callOrMsg,
-                             InstanceReceiver Receiver,
-                             const RetainSummary& Summ,
-                             const MemRegion *Callee,
-                             ExplodedNode* Pred, const GRState *state) {
+void RetainCountChecker::checkSummary(const RetainSummary &Summ,
+                                      const CallOrObjCMessage &CallOrMsg,
+                                      CheckerContext &C) const {
+  const ProgramState *state = C.getState();
 
   // Evaluate the effect of the arguments.
   RefVal::Kind hasErr = (RefVal::Kind) 0;
   SourceRange ErrorRange;
   SymbolRef ErrorSym = 0;
 
-  llvm::SmallVector<const MemRegion*, 10> RegionsToInvalidate;
-  
-  // Use RAII to make sure the whitelist is properly cleared.
-  ResetWhiteList resetWhiteList;
+  for (unsigned idx = 0, e = CallOrMsg.getNumArgs(); idx != e; ++idx) {
+    SVal V = CallOrMsg.getArgSVal(idx);
 
-  // Invalidate all instance variables of the receiver of a message.
-  // FIXME: We should be able to do better with inter-procedural analysis.
-  if (Receiver) {
-    SVal V = Receiver.getSValAsScalarOrLoc(state);
     if (SymbolRef Sym = V.getAsLocSymbol()) {
-      if (state->get<RefBindings>(Sym))
-        WhitelistedSymbols.insert(Sym);
-    }
-    if (const MemRegion *region = V.getAsRegion())
-      RegionsToInvalidate.push_back(region);
-  }
-  
-  // Invalidate all instance variables for the callee of a C++ method call.
-  // FIXME: We should be able to do better with inter-procedural analysis.
-  // FIXME: we can probably do better for const versus non-const methods.
-  if (callOrMsg.isCXXCall()) {
-    if (const MemRegion *callee = callOrMsg.getCXXCallee().getAsRegion())
-      RegionsToInvalidate.push_back(callee);
-  }
-  
-  for (unsigned idx = 0, e = callOrMsg.getNumArgs(); idx != e; ++idx) {
-    SVal V = callOrMsg.getArgSValAsScalarOrLoc(idx);
-    SymbolRef Sym = V.getAsLocSymbol();
-
-    if (Sym)
-      if (RefBindings::data_type* T = state->get<RefBindings>(Sym)) {
-        WhitelistedSymbols.insert(Sym);
-        state = Update(state, Sym, *T, Summ.getArg(idx), hasErr);
+      if (RefBindings::data_type *T = state->get<RefBindings>(Sym)) {
+        state = updateSymbol(state, Sym, *T, Summ.getArg(idx), hasErr, C);
         if (hasErr) {
-          ErrorRange = callOrMsg.getArgSourceRange(idx);
+          ErrorRange = CallOrMsg.getArgSourceRange(idx);
           ErrorSym = Sym;
           break;
         }
       }
-
-  tryAgain:
-    if (isa<Loc>(V)) {
-      if (loc::MemRegionVal* MR = dyn_cast<loc::MemRegionVal>(&V)) {
-        if (Summ.getArg(idx) == DoNothingByRef)
-          continue;
-
-        // Invalidate the value of the variable passed by reference.
-        const MemRegion *R = MR->getRegion();
-
-        // Are we dealing with an ElementRegion?  If the element type is
-        // a basic integer type (e.g., char, int) and the underying region
-        // is a variable region then strip off the ElementRegion.
-        // FIXME: We really need to think about this for the general case
-        //   as sometimes we are reasoning about arrays and other times
-        //   about (char*), etc., is just a form of passing raw bytes.
-        //   e.g., void *p = alloca(); foo((char*)p);
-        if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
-          // Checking for 'integral type' is probably too promiscuous, but
-          // we'll leave it in for now until we have a systematic way of
-          // handling all of these cases.  Eventually we need to come up
-          // with an interface to StoreManager so that this logic can be
-          // approriately delegated to the respective StoreManagers while
-          // still allowing us to do checker-specific logic (e.g.,
-          // invalidating reference counts), probably via callbacks.
-          if (ER->getElementType()->isIntegralOrEnumerationType()) {
-            const MemRegion *superReg = ER->getSuperRegion();
-            if (isa<VarRegion>(superReg) || isa<FieldRegion>(superReg) ||
-                isa<ObjCIvarRegion>(superReg))
-              R = cast<TypedRegion>(superReg);
-          }
-          // FIXME: What about layers of ElementRegions?
-        }
-
-        // Mark this region for invalidation.  We batch invalidate regions
-        // below for efficiency.
-        RegionsToInvalidate.push_back(R);
-        continue;
-      }
-      else {
-        // Nuke all other arguments passed by reference.
-        // FIXME: is this necessary or correct? This handles the non-Region
-        //  cases.  Is it ever valid to store to these?
-        state = state->unbindLoc(cast<Loc>(V));
-      }
-    }
-    else if (isa<nonloc::LocAsInteger>(V)) {
-      // If we are passing a location wrapped as an integer, unwrap it and
-      // invalidate the values referred by the location.
-      V = cast<nonloc::LocAsInteger>(V).getLoc();
-      goto tryAgain;
     }
   }
 
-  // Block calls result in all captured values passed-via-reference to be
-  // invalidated.
-  if (const BlockDataRegion *BR = dyn_cast_or_null<BlockDataRegion>(Callee)) {
-    RegionsToInvalidate.push_back(BR);
-  }
-
-  // Invalidate regions we designed for invalidation use the batch invalidation
-  // API.
-
-  // FIXME: We can have collisions on the conjured symbol if the
-  //  expression *I also creates conjured symbols.  We probably want
-  //  to identify conjured symbols by an expression pair: the enclosing
-  //  expression (the context) and the expression itself.  This should
-  //  disambiguate conjured symbols.
-  unsigned Count = Builder.getCurrentBlockCount();
-  StoreManager::InvalidatedSymbols IS;
-
-  // NOTE: Even if RegionsToInvalidate is empty, we must still invalidate
-  //  global variables.
-  // NOTE: RetainReleaseChecker handles the actual invalidation of symbols.
-  state =
-    state->invalidateRegions(RegionsToInvalidate.data(),
-                             RegionsToInvalidate.data() +
-                             RegionsToInvalidate.size(),
-                             Ex, Count, &IS,
-                             /* invalidateGlobals = */
-                             Eng.doesInvalidateGlobals(callOrMsg));
-
   // Evaluate the effect on the message receiver.
-  if (!ErrorRange.isValid() && Receiver) {
-    SymbolRef Sym = Receiver.getSValAsScalarOrLoc(state).getAsLocSymbol();
-    if (Sym) {
-      if (const RefVal* T = state->get<RefBindings>(Sym)) {
-        state = Update(state, Sym, *T, Summ.getReceiverEffect(), hasErr);
+  bool ReceiverIsTracked = false;
+  if (!hasErr && CallOrMsg.isObjCMessage()) {
+    const LocationContext *LC = C.getPredecessor()->getLocationContext();
+    SVal Receiver = CallOrMsg.getInstanceMessageReceiver(LC);
+    if (SymbolRef Sym = Receiver.getAsLocSymbol()) {
+      if (const RefVal *T = state->get<RefBindings>(Sym)) {
+        ReceiverIsTracked = true;
+        state = updateSymbol(state, Sym, *T, Summ.getReceiverEffect(),
+                             hasErr, C);
         if (hasErr) {
-          ErrorRange = Receiver.getSourceRange();
+          ErrorRange = CallOrMsg.getReceiverSourceRange();
           ErrorSym = Sym;
         }
       }
@@ -2705,8 +2706,7 @@ void CFRefCount::evalSummary(ExplodedNodeSet& Dst,
 
   // Process any errors.
   if (hasErr) {
-    ProcessNonLeakError(Dst, Builder, Ex, ErrorRange, Pred, state,
-                        hasErr, ErrorSym);
+    processNonLeakError(state, ErrorRange, hasErr, ErrorSym, C);
     return;
   }
 
@@ -2714,75 +2714,34 @@ void CFRefCount::evalSummary(ExplodedNodeSet& Dst,
   RetEffect RE = Summ.getRetEffect();
 
   if (RE.getKind() == RetEffect::OwnedWhenTrackedReceiver) {
-    bool found = false;
-    if (Receiver) {
-      SVal V = Receiver.getSValAsScalarOrLoc(state);
-      if (SymbolRef Sym = V.getAsLocSymbol())
-        if (state->get<RefBindings>(Sym)) {
-          found = true;
-          RE = Summaries.getObjAllocRetEffect();
-        }
-    } // FIXME: Otherwise, this is a send-to-super instance message.
-    if (!found)
+    if (ReceiverIsTracked)
+      RE = getSummaryManager(C).getObjAllocRetEffect();      
+    else
       RE = RetEffect::MakeNoRet();
   }
 
   switch (RE.getKind()) {
     default:
-      assert (false && "Unhandled RetEffect."); break;
-
-    case RetEffect::NoRet: {
-      // Make up a symbol for the return value (not reference counted).
-      // FIXME: Most of this logic is not specific to the retain/release
-      // checker.
-
-      // FIXME: We eventually should handle structs and other compound types
-      // that are returned by value.
-
-      // Use the result type from callOrMsg as it automatically adjusts
-      // for methods/functions that return references.
-      QualType resultTy = callOrMsg.getResultType(Eng.getContext());
-      if (Loc::isLocType(resultTy) || 
-            (resultTy->isIntegerType() && resultTy->isScalarType())) {
-        unsigned Count = Builder.getCurrentBlockCount();
-        SValBuilder &svalBuilder = Eng.getSValBuilder();
-        SVal X = svalBuilder.getConjuredSymbolVal(NULL, Ex, resultTy, Count);
-        state = state->BindExpr(Ex, X, false);
-      }
+      llvm_unreachable("Unhandled RetEffect."); break;
 
+    case RetEffect::NoRet:
+      // No work necessary.
       break;
-    }
-
-    case RetEffect::Alias: {
-      unsigned idx = RE.getIndex();
-      assert (idx < callOrMsg.getNumArgs());
-      SVal V = callOrMsg.getArgSValAsScalarOrLoc(idx);
-      state = state->BindExpr(Ex, V, false);
-      break;
-    }
-
-    case RetEffect::ReceiverAlias: {
-      assert(Receiver);
-      SVal V = Receiver.getSValAsScalarOrLoc(state);
-      state = state->BindExpr(Ex, V, false);
-      break;
-    }
 
     case RetEffect::OwnedAllocatedSymbol:
     case RetEffect::OwnedSymbol: {
-      unsigned Count = Builder.getCurrentBlockCount();
-      SValBuilder &svalBuilder = Eng.getSValBuilder();
-      SymbolRef Sym = svalBuilder.getConjuredSymbol(Ex, Count);
+      SymbolRef Sym = state->getSVal(CallOrMsg.getOriginExpr()).getAsSymbol();
+      if (!Sym)
+        break;
 
       // Use the result type from callOrMsg as it automatically adjusts
-      // for methods/functions that return references.      
-      QualType resultTy = callOrMsg.getResultType(Eng.getContext());
+      // for methods/functions that return references.
+      QualType ResultTy = CallOrMsg.getResultType(C.getASTContext());
       state = state->set<RefBindings>(Sym, RefVal::makeOwned(RE.getObjKind(),
-                                                            resultTy));
-      state = state->BindExpr(Ex, svalBuilder.makeLoc(Sym), false);
+                                                             ResultTy));
 
       // FIXME: Add a flag to the checker where allocations are assumed to
-      // *not fail.
+      // *not* fail. (The code below is out-of-date, though.)
 #if 0
       if (RE.getKind() == RetEffect::OwnedAllocatedSymbol) {
         bool isFeasible;
@@ -2797,151 +2756,324 @@ void CFRefCount::evalSummary(ExplodedNodeSet& Dst,
     case RetEffect::GCNotOwnedSymbol:
     case RetEffect::ARCNotOwnedSymbol:
     case RetEffect::NotOwnedSymbol: {
-      unsigned Count = Builder.getCurrentBlockCount();
-      SValBuilder &svalBuilder = Eng.getSValBuilder();
-      SymbolRef Sym = svalBuilder.getConjuredSymbol(Ex, Count);
-      QualType RetT = GetReturnType(Ex, svalBuilder.getContext());
+      const Expr *Ex = CallOrMsg.getOriginExpr();
+      SymbolRef Sym = state->getSVal(Ex).getAsSymbol();
+      if (!Sym)
+        break;
+
+      // Use GetReturnType in order to give [NSFoo alloc] the type NSFoo *.
+      QualType ResultTy = GetReturnType(Ex, C.getASTContext());
       state = state->set<RefBindings>(Sym, RefVal::makeNotOwned(RE.getObjKind(),
-                                                               RetT));
-      state = state->BindExpr(Ex, svalBuilder.makeLoc(Sym), false);
+                                                                ResultTy));
       break;
     }
   }
 
-  // Generate a sink node if we are at the end of a path.
-  ExplodedNode *NewNode =
-    Summ.isEndPath() ? Builder.MakeSinkNode(Dst, Ex, Pred, state)
-                     : Builder.MakeNode(Dst, Ex, Pred, state);
+  // This check is actually necessary; otherwise the statement builder thinks
+  // we've hit a previously-found path.
+  // Normally addTransition takes care of this, but we want the node pointer.
+  ExplodedNode *NewNode;
+  if (state == C.getState()) {
+    NewNode = C.getPredecessor();
+  } else {
+    NewNode = C.generateNode(state);
+  }
 
-  // Annotate the edge with summary we used.
-  if (NewNode) SummaryLog[NewNode] = &Summ;
+  // Annotate the node with summary we used.
+  if (NewNode) {
+    // FIXME: This is ugly. See checkEndAnalysis for why it's necessary.
+    if (ShouldResetSummaryLog) {
+      SummaryLog.clear();
+      ShouldResetSummaryLog = false;
+    }
+    SummaryLog[NewNode] = &Summ;
+  }
 }
 
 
-void CFRefCount::evalCall(ExplodedNodeSet& Dst,
-                          ExprEngine& Eng,
-                          StmtNodeBuilder& Builder,
-                          const CallExpr* CE, SVal L,
-                          ExplodedNode* Pred) {
+const ProgramState *
+RetainCountChecker::updateSymbol(const ProgramState *state, SymbolRef sym,
+                                 RefVal V, ArgEffect E, RefVal::Kind &hasErr,
+                                 CheckerContext &C) const {
+  // In GC mode [... release] and [... retain] do nothing.
+  // In ARC mode they shouldn't exist at all, but we just ignore them.
+  bool IgnoreRetainMsg = C.isObjCGCEnabled();
+  if (!IgnoreRetainMsg)
+    IgnoreRetainMsg = (bool)C.getASTContext().getLangOptions().ObjCAutoRefCount;
 
-  RetainSummary *Summ = 0;
+  switch (E) {
+    default: break;
+    case IncRefMsg: E = IgnoreRetainMsg ? DoNothing : IncRef; break;
+    case DecRefMsg: E = IgnoreRetainMsg ? DoNothing : DecRef; break;
+    case MakeCollectable: E = C.isObjCGCEnabled() ? DecRef : DoNothing; break;
+    case NewAutoreleasePool: E = C.isObjCGCEnabled() ? DoNothing :
+                                                      NewAutoreleasePool; break;
+  }
 
-  // FIXME: Better support for blocks.  For now we stop tracking anything
-  // that is passed to blocks.
-  // FIXME: Need to handle variables that are "captured" by the block.
-  if (dyn_cast_or_null<BlockDataRegion>(L.getAsRegion())) {
-    Summ = Summaries.getPersistentStopSummary();
+  // Handle all use-after-releases.
+  if (!C.isObjCGCEnabled() && V.getKind() == RefVal::Released) {
+    V = V ^ RefVal::ErrorUseAfterRelease;
+    hasErr = V.getKind();
+    return state->set<RefBindings>(sym, V);
   }
-  else if (const FunctionDecl* FD = L.getAsFunctionDecl()) {
-    Summ = Summaries.getSummary(FD);
-  }
-  else if (const CXXMemberCallExpr *me = dyn_cast<CXXMemberCallExpr>(CE)) {
-    if (const CXXMethodDecl *MD = me->getMethodDecl())
-      Summ = Summaries.getSummary(MD);
-    else
-      Summ = Summaries.getDefaultSummary();    
-  }
-  else
-    Summ = Summaries.getDefaultSummary();
 
-  assert(Summ);
-  evalSummary(Dst, Eng, Builder, CE,
-              CallOrObjCMessage(CE, Builder.GetState(Pred)),
-              InstanceReceiver(), *Summ,L.getAsRegion(),
-              Pred, Builder.GetState(Pred));
+  switch (E) {
+    case DecRefMsg:
+    case IncRefMsg:
+    case MakeCollectable:
+      llvm_unreachable("DecRefMsg/IncRefMsg/MakeCollectable already converted");
+      return state;
+
+    case Dealloc:
+      // Any use of -dealloc in GC is *bad*.
+      if (C.isObjCGCEnabled()) {
+        V = V ^ RefVal::ErrorDeallocGC;
+        hasErr = V.getKind();
+        break;
+      }
+
+      switch (V.getKind()) {
+        default:
+          llvm_unreachable("Invalid RefVal state for an explicit dealloc.");
+          break;
+        case RefVal::Owned:
+          // The object immediately transitions to the released state.
+          V = V ^ RefVal::Released;
+          V.clearCounts();
+          return state->set<RefBindings>(sym, V);
+        case RefVal::NotOwned:
+          V = V ^ RefVal::ErrorDeallocNotOwned;
+          hasErr = V.getKind();
+          break;
+      }
+      break;
+
+    case NewAutoreleasePool:
+      assert(!C.isObjCGCEnabled());
+      return state->add<AutoreleaseStack>(sym);
+
+    case MayEscape:
+      if (V.getKind() == RefVal::Owned) {
+        V = V ^ RefVal::NotOwned;
+        break;
+      }
+
+      // Fall-through.
+
+    case DoNothing:
+      return state;
+
+    case Autorelease:
+      if (C.isObjCGCEnabled())
+        return state;
+
+      // Update the autorelease counts.
+      state = SendAutorelease(state, ARCountFactory, sym);
+      V = V.autorelease();
+      break;
+
+    case StopTracking:
+      return state->remove<RefBindings>(sym);
+
+    case IncRef:
+      switch (V.getKind()) {
+        default:
+          llvm_unreachable("Invalid RefVal state for a retain.");
+          break;
+        case RefVal::Owned:
+        case RefVal::NotOwned:
+          V = V + 1;
+          break;
+        case RefVal::Released:
+          // Non-GC cases are handled above.
+          assert(C.isObjCGCEnabled());
+          V = (V ^ RefVal::Owned) + 1;
+          break;
+      }
+      break;
+
+    case SelfOwn:
+      V = V ^ RefVal::NotOwned;
+      // Fall-through.
+    case DecRef:
+    case DecRefBridgedTransfered:
+      switch (V.getKind()) {
+        default:
+          // case 'RefVal::Released' handled above.
+          llvm_unreachable("Invalid RefVal state for a release.");
+          break;
+
+        case RefVal::Owned:
+          assert(V.getCount() > 0);
+          if (V.getCount() == 1)
+            V = V ^ (E == DecRefBridgedTransfered ? 
+                      RefVal::NotOwned : RefVal::Released);
+          V = V - 1;
+          break;
+
+        case RefVal::NotOwned:
+          if (V.getCount() > 0)
+            V = V - 1;
+          else {
+            V = V ^ RefVal::ErrorReleaseNotOwned;
+            hasErr = V.getKind();
+          }
+          break;
+
+        case RefVal::Released:
+          // Non-GC cases are handled above.
+          assert(C.isObjCGCEnabled());
+          V = V ^ RefVal::ErrorUseAfterRelease;
+          hasErr = V.getKind();
+          break;
+      }
+      break;
+  }
+  return state->set<RefBindings>(sym, V);
 }
 
-void CFRefCount::evalObjCMessage(ExplodedNodeSet& Dst,
-                                 ExprEngine& Eng,
-                                 StmtNodeBuilder& Builder,
-                                 ObjCMessage msg,
-                                 ExplodedNode* Pred,
-                                 const GRState *state) {
-  RetainSummary *Summ =
-    msg.isInstanceMessage()
-      ? Summaries.getInstanceMethodSummary(msg, state,Pred->getLocationContext())
-      : Summaries.getClassMethodSummary(msg);
+void RetainCountChecker::processNonLeakError(const ProgramState *St,
+                                             SourceRange ErrorRange,
+                                             RefVal::Kind ErrorKind,
+                                             SymbolRef Sym,
+                                             CheckerContext &C) const {
+  ExplodedNode *N = C.generateSink(St);
+  if (!N)
+    return;
 
-  assert(Summ && "RetainSummary is null");
-  evalSummary(Dst, Eng, Builder, msg.getOriginExpr(),
-              CallOrObjCMessage(msg, Builder.GetState(Pred)),
-              InstanceReceiver(msg, Pred->getLocationContext()), *Summ, NULL,
-              Pred, state);
+  CFRefBug *BT;
+  switch (ErrorKind) {
+    default:
+      llvm_unreachable("Unhandled error.");
+      return;
+    case RefVal::ErrorUseAfterRelease:
+      if (!useAfterRelease)
+        useAfterRelease.reset(new UseAfterRelease());
+      BT = &*useAfterRelease;
+      break;
+    case RefVal::ErrorReleaseNotOwned:
+      if (!releaseNotOwned)
+        releaseNotOwned.reset(new BadRelease());
+      BT = &*releaseNotOwned;
+      break;
+    case RefVal::ErrorDeallocGC:
+      if (!deallocGC)
+        deallocGC.reset(new DeallocGC());
+      BT = &*deallocGC;
+      break;
+    case RefVal::ErrorDeallocNotOwned:
+      if (!deallocNotOwned)
+        deallocNotOwned.reset(new DeallocNotOwned());
+      BT = &*deallocNotOwned;
+      break;
+  }
+
+  assert(BT);
+  CFRefReport *report = new CFRefReport(*BT, C.getASTContext().getLangOptions(),
+                                        C.isObjCGCEnabled(), SummaryLog,
+                                        N, Sym);
+  report->addRange(ErrorRange);
+  C.EmitReport(report);
 }
 
-namespace {
-class StopTrackingCallback : public SymbolVisitor {
-  const GRState *state;
-public:
-  StopTrackingCallback(const GRState *st) : state(st) {}
-  const GRState *getState() const { return state; }
+//===----------------------------------------------------------------------===//
+// Handle the return values of retain-count-related functions.
+//===----------------------------------------------------------------------===//
 
-  bool VisitSymbol(SymbolRef sym) {
-    state = state->remove<RefBindings>(sym);
-    return true;
-  }
-};
-} // end anonymous namespace
+bool RetainCountChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
+  // Get the callee. We're only interested in simple C functions.
+  const ProgramState *state = C.getState();
+  const Expr *Callee = CE->getCallee();
+  SVal L = state->getSVal(Callee);
 
+  const FunctionDecl *FD = L.getAsFunctionDecl();
+  if (!FD)
+    return false;
 
-void CFRefCount::evalBind(StmtNodeBuilderRef& B, SVal location, SVal val) {
-  // Are we storing to something that causes the value to "escape"?
-  bool escapes = false;
+  IdentifierInfo *II = FD->getIdentifier();
+  if (!II)
+    return false;
 
-  // A value escapes in three possible cases (this may change):
-  //
-  // (1) we are binding to something that is not a memory region.
-  // (2) we are binding to a memregion that does not have stack storage
-  // (3) we are binding to a memregion with stack storage that the store
-  //     does not understand.
-  const GRState *state = B.getState();
+  // For now, we're only handling the functions that return aliases of their
+  // arguments: CFRetain and CFMakeCollectable (and their families).
+  // Eventually we should add other functions we can model entirely,
+  // such as CFRelease, which don't invalidate their arguments or globals.
+  if (CE->getNumArgs() != 1)
+    return false;
 
-  if (!isa<loc::MemRegionVal>(location))
-    escapes = true;
-  else {
-    const MemRegion* R = cast<loc::MemRegionVal>(location).getRegion();
-    escapes = !R->hasStackStorage();
+  // Get the name of the function.
+  StringRef FName = II->getName();
+  FName = FName.substr(FName.find_first_not_of('_'));
 
-    if (!escapes) {
-      // To test (3), generate a new state with the binding removed.  If it is
-      // the same state, then it escapes (since the store cannot represent
-      // the binding).
-      escapes = (state == (state->bindLoc(cast<Loc>(location), UnknownVal())));
+  // See if it's one of the specific functions we know how to eval.
+  bool canEval = false;
+
+  QualType ResultTy = FD->getResultType();
+  if (ResultTy->isObjCIdType()) {
+    // Handle: id NSMakeCollectable(CFTypeRef)
+    canEval = II->isStr("NSMakeCollectable");
+  } else if (ResultTy->isPointerType()) {
+    // Handle: (CF|CG)Retain
+    //         CFMakeCollectable
+    // It's okay to be a little sloppy here (CGMakeCollectable doesn't exist).
+    if (cocoa::isRefType(ResultTy, "CF", FName) ||
+        cocoa::isRefType(ResultTy, "CG", FName)) {
+      canEval = isRetain(FD, FName) || isMakeCollectable(FD, FName);
     }
   }
+        
+  if (!canEval)
+    return false;
 
-  // If our store can represent the binding and we aren't storing to something
-  // that doesn't have local storage then just return and have the simulation
-  // state continue as is.
-  if (!escapes)
-      return;
+  // Bind the return value.
+  SVal RetVal = state->getSVal(CE->getArg(0));
+  if (RetVal.isUnknown()) {
+    // If the receiver is unknown, conjure a return value.
+    SValBuilder &SVB = C.getSValBuilder();
+    unsigned Count = C.getCurrentBlockCount();
+    SVal RetVal = SVB.getConjuredSymbolVal(0, CE, ResultTy, Count);
+  }
+  state = state->BindExpr(CE, RetVal, false);
 
-  // Otherwise, find all symbols referenced by 'val' that we are tracking
-  // and stop tracking them.
-  B.MakeNode(state->scanReachableSymbols<StopTrackingCallback>(val).getState());
+  // FIXME: This should not be necessary, but otherwise the argument seems to be
+  // considered alive during the next statement.
+  if (const MemRegion *ArgRegion = RetVal.getAsRegion()) {
+    // Save the refcount status of the argument.
+    SymbolRef Sym = RetVal.getAsLocSymbol();
+    RefBindings::data_type *Binding = 0;
+    if (Sym)
+      Binding = state->get<RefBindings>(Sym);
+
+    // Invalidate the argument region.
+    unsigned Count = C.getCurrentBlockCount();
+    state = state->invalidateRegions(ArgRegion, CE, Count);
+
+    // Restore the refcount status of the argument.
+    if (Binding)
+      state = state->set<RefBindings>(Sym, *Binding);
+  }
+
+  C.addTransition(state);
+  return true;
 }
 
- // Return statements.
+//===----------------------------------------------------------------------===//
+// Handle return statements.
+//===----------------------------------------------------------------------===//
 
-void CFRefCount::evalReturn(ExplodedNodeSet& Dst,
-                            ExprEngine& Eng,
-                            StmtNodeBuilder& Builder,
-                            const ReturnStmt* S,
-                            ExplodedNode* Pred) {
-
-  const Expr* RetE = S->getRetValue();
+void RetainCountChecker::checkPreStmt(const ReturnStmt *S,
+                                      CheckerContext &C) const {
+  const Expr *RetE = S->getRetValue();
   if (!RetE)
     return;
 
-  const GRState *state = Builder.GetState(Pred);
+  const ProgramState *state = C.getState();
   SymbolRef Sym = state->getSValAsScalarOrLoc(RetE).getAsLocSymbol();
-
   if (!Sym)
     return;
 
   // Get the reference count binding (if any).
-  const RefVal* T = state->get<RefBindings>(Sym);
-
+  const RefVal *T = state->get<RefBindings>(Sym);
   if (!T)
     return;
 
@@ -2951,7 +3083,7 @@ void CFRefCount::evalReturn(ExplodedNodeSet& Dst,
   switch (X.getKind()) {
     case RefVal::Owned: {
       unsigned cnt = X.getCount();
-      assert (cnt > 0);
+      assert(cnt > 0);
       X.setCount(cnt - 1);
       X = X ^ RefVal::ReturnedOwned;
       break;
@@ -2975,21 +3107,25 @@ void CFRefCount::evalReturn(ExplodedNodeSet& Dst,
 
   // Update the binding.
   state = state->set<RefBindings>(Sym, X);
-  Pred = Builder.MakeNode(Dst, S, Pred, state);
+  ExplodedNode *Pred = C.generateNode(state);
+
+  // At this point we have updated the state properly.
+  // Everything after this is merely checking to see if the return value has
+  // been over- or under-retained.
 
   // Did we cache out?
   if (!Pred)
     return;
 
   // Update the autorelease counts.
-  static unsigned autoreleasetag = 0;
-  GenericNodeBuilderRefCount Bd(Builder, S, &autoreleasetag);
-  bool stop = false;
-  llvm::tie(Pred, state) = HandleAutoreleaseCounts(state , Bd, Pred, Eng, Sym,
-                                                   X, stop);
+  static SimpleProgramPointTag
+         AutoreleaseTag("RetainCountChecker : Autorelease");
+  GenericNodeBuilderRefCount Bd(C, &AutoreleaseTag);
+  llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Bd, Pred,
+                                                   C.getEngine(), Sym, X);
 
   // Did we cache out?
-  if (!Pred || stop)
+  if (!Pred)
     return;
 
   // Get the updated binding.
@@ -2998,36 +3134,37 @@ void CFRefCount::evalReturn(ExplodedNodeSet& Dst,
   X = *T;
 
   // Consult the summary of the enclosing method.
-  Decl const *CD = &Pred->getCodeDecl();
+  RetainSummaryManager &Summaries = getSummaryManager(C);
+  const Decl *CD = &Pred->getCodeDecl();
 
-  if (const ObjCMethodDecl* MD = dyn_cast<ObjCMethodDecl>(CD)) {
-    const RetainSummary &Summ = *Summaries.getMethodSummary(MD);
-    return evalReturnWithRetEffect(Dst, Eng, Builder, S, 
-                                   Pred, Summ.getRetEffect(), X,
-                                   Sym, state);
+  if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(CD)) {
+    // Unlike regular functions, /all/ ObjC methods are assumed to always
+    // follow Cocoa retain-count conventions, not just those with special
+    // names or attributes.
+    const RetainSummary *Summ = Summaries.getMethodSummary(MD);
+    RetEffect RE = Summ ? Summ->getRetEffect() : RetEffect::MakeNoRet();
+    checkReturnWithRetEffect(S, C, Pred, RE, X, Sym, state);
   }
 
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(CD)) {
     if (!isa<CXXMethodDecl>(FD))
       if (const RetainSummary *Summ = Summaries.getSummary(FD))
-        return evalReturnWithRetEffect(Dst, Eng, Builder, S, 
-                                       Pred, Summ->getRetEffect(), X,
-                                       Sym, state);
+        checkReturnWithRetEffect(S, C, Pred, Summ->getRetEffect(), X,
+                                 Sym, state);
   }
 }
 
-void CFRefCount::evalReturnWithRetEffect(ExplodedNodeSet &Dst, 
-                                         ExprEngine &Eng,
-                                         StmtNodeBuilder &Builder,
-                                         const ReturnStmt *S,
-                                         ExplodedNode *Pred,
-                                         RetEffect RE, RefVal X,
-                                         SymbolRef Sym, const GRState *state) {
+void RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
+                                                  CheckerContext &C,
+                                                  ExplodedNode *Pred,
+                                                  RetEffect RE, RefVal X,
+                                                  SymbolRef Sym,
+                                              const ProgramState *state) const {
   // Any leaks or other errors?
   if (X.isReturnedOwned() && X.getCount() == 0) {
     if (RE.getKind() != RetEffect::NoRet) {
       bool hasError = false;
-      if (isGCEnabled() && RE.getObjKind() == RetEffect::ObjC) {
+      if (C.isObjCGCEnabled() && RE.getObjKind() == RetEffect::ObjC) {
         // Things are more complicated with garbage collection.  If the
         // returned object is suppose to be an Objective-C object, we have
         // a leak (as the caller expects a GC'ed object) because no
@@ -3045,46 +3182,88 @@ void CFRefCount::evalReturnWithRetEffect(ExplodedNodeSet &Dst,
 
       if (hasError) {
         // Generate an error node.
-        static int ReturnOwnLeakTag = 0;
         state = state->set<RefBindings>(Sym, X);
-        ExplodedNode *N =
-          Builder.generateNode(PostStmt(S, Pred->getLocationContext(),
-                                        &ReturnOwnLeakTag), state, Pred);
+
+        static SimpleProgramPointTag
+               ReturnOwnLeakTag("RetainCountChecker : ReturnsOwnLeak");
+        ExplodedNode *N = C.generateNode(state, Pred, &ReturnOwnLeakTag);
         if (N) {
+          const LangOptions &LOpts = C.getASTContext().getLangOptions();
+          bool GCEnabled = C.isObjCGCEnabled();
           CFRefReport *report =
-            new CFRefLeakReport(*static_cast<CFRefBug*>(leakAtReturn), *this,
-                                N, Sym, Eng);
-          BR->EmitReport(report);
+            new CFRefLeakReport(*getLeakAtReturnBug(LOpts, GCEnabled),
+                                LOpts, GCEnabled, SummaryLog,
+                                N, Sym, C.getEngine());
+          C.EmitReport(report);
         }
       }
     }
-    return;
-  }
-
-  if (X.isReturnedNotOwned()) {
+  } else if (X.isReturnedNotOwned()) {
     if (RE.isOwned()) {
       // Trying to return a not owned object to a caller expecting an
       // owned object.
-
-      static int ReturnNotOwnedForOwnedTag = 0;
       state = state->set<RefBindings>(Sym, X ^ RefVal::ErrorReturnedNotOwned);
-      if (ExplodedNode *N =
-          Builder.generateNode(PostStmt(S, Pred->getLocationContext(),
-                                        &ReturnNotOwnedForOwnedTag),
-                               state, Pred)) {
-          CFRefReport *report =
-              new CFRefReport(*static_cast<CFRefBug*>(returnNotOwnedForOwned),
-                              *this, N, Sym);
-          BR->EmitReport(report);
+
+      static SimpleProgramPointTag
+             ReturnNotOwnedTag("RetainCountChecker : ReturnNotOwnedForOwned");
+      ExplodedNode *N = C.generateNode(state, Pred, &ReturnNotOwnedTag);
+      if (N) {
+        if (!returnNotOwnedForOwned)
+          returnNotOwnedForOwned.reset(new ReturnedNotOwnedForOwned());
+
+        CFRefReport *report =
+            new CFRefReport(*returnNotOwnedForOwned,
+                            C.getASTContext().getLangOptions(), 
+                            C.isObjCGCEnabled(), SummaryLog, N, Sym);
+        C.EmitReport(report);
       }
     }
   }
 }
 
-// Assumptions.
+//===----------------------------------------------------------------------===//
+// Check various ways a symbol can be invalidated.
+//===----------------------------------------------------------------------===//
 
-const GRState* CFRefCount::evalAssume(const GRState *state,
-                                      SVal Cond, bool Assumption) {
+void RetainCountChecker::checkBind(SVal loc, SVal val, const Stmt *S,
+                                   CheckerContext &C) const {
+  // Are we storing to something that causes the value to "escape"?
+  bool escapes = true;
+
+  // A value escapes in three possible cases (this may change):
+  //
+  // (1) we are binding to something that is not a memory region.
+  // (2) we are binding to a memregion that does not have stack storage
+  // (3) we are binding to a memregion with stack storage that the store
+  //     does not understand.
+  const ProgramState *state = C.getState();
+
+  if (loc::MemRegionVal *regionLoc = dyn_cast<loc::MemRegionVal>(&loc)) {
+    escapes = !regionLoc->getRegion()->hasStackStorage();
+
+    if (!escapes) {
+      // To test (3), generate a new state with the binding added.  If it is
+      // the same state, then it escapes (since the store cannot represent
+      // the binding).
+      escapes = (state == (state->bindLoc(*regionLoc, val)));
+    }
+  }
+
+  // If our store can represent the binding and we aren't storing to something
+  // that doesn't have local storage then just return and have the simulation
+  // state continue as is.
+  if (!escapes)
+      return;
+
+  // Otherwise, find all symbols referenced by 'val' that we are tracking
+  // and stop tracking them.
+  state = state->scanReachableSymbols<StopTrackingCallback>(val).getState();
+  C.addTransition(state);
+}
+
+const ProgramState *RetainCountChecker::evalAssume(const ProgramState *state,
+                                                   SVal Cond,
+                                                   bool Assumption) const {
 
   // FIXME: We may add to the interface of evalAssume the list of symbols
   //  whose assumptions have changed.  For now we just iterate through the
@@ -3098,9 +3277,9 @@ const GRState* CFRefCount::evalAssume(const GRState *state,
     return state;
 
   bool changed = false;
-  RefBindings::Factory& RefBFactory = state->get_context<RefBindings>();
+  RefBindings::Factory &RefBFactory = state->get_context<RefBindings>();
 
-  for (RefBindings::iterator I=B.begin(), E=B.end(); I!=E; ++I) {
+  for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I) {
     // Check if the symbol is null (or equal to any constant).
     // If this is the case, stop tracking the symbol.
     if (state->getSymVal(I.getKey())) {
@@ -3115,161 +3294,48 @@ const GRState* CFRefCount::evalAssume(const GRState *state,
   return state;
 }
 
-const GRState * CFRefCount::Update(const GRState * state, SymbolRef sym,
-                              RefVal V, ArgEffect E,
-                              RefVal::Kind& hasErr) {
+const ProgramState *
+RetainCountChecker::checkRegionChanges(const ProgramState *state,
+                            const StoreManager::InvalidatedSymbols *invalidated,
+                                    ArrayRef<const MemRegion *> ExplicitRegions,
+                                    ArrayRef<const MemRegion *> Regions) const {
+  if (!invalidated)
+    return state;
 
-  // In GC mode [... release] and [... retain] do nothing.
-  switch (E) {
-    default: break;
-    case IncRefMsg: E = isARCorGCEnabled() ? DoNothing : IncRef; break;
-    case DecRefMsg: E = isARCorGCEnabled() ? DoNothing : DecRef; break;
-    case MakeCollectable: E = isGCEnabled() ? DecRef : DoNothing; break;
-    case NewAutoreleasePool: E = isGCEnabled() ? DoNothing :
-                                                 NewAutoreleasePool; break;
+  llvm::SmallPtrSet<SymbolRef, 8> WhitelistedSymbols;
+  for (ArrayRef<const MemRegion *>::iterator I = ExplicitRegions.begin(),
+       E = ExplicitRegions.end(); I != E; ++I) {
+    if (const SymbolicRegion *SR = (*I)->StripCasts()->getAs<SymbolicRegion>())
+      WhitelistedSymbols.insert(SR->getSymbol());
   }
 
-  // Handle all use-after-releases.
-  if (!isGCEnabled() && V.getKind() == RefVal::Released) {
-    V = V ^ RefVal::ErrorUseAfterRelease;
-    hasErr = V.getKind();
-    return state->set<RefBindings>(sym, V);
+  for (StoreManager::InvalidatedSymbols::const_iterator I=invalidated->begin(),
+       E = invalidated->end(); I!=E; ++I) {
+    SymbolRef sym = *I;
+    if (WhitelistedSymbols.count(sym))
+      continue;
+    // Remove any existing reference-count binding.
+    state = state->remove<RefBindings>(sym);
   }
-
-  switch (E) {
-    case DecRefMsg:
-    case IncRefMsg:
-    case MakeCollectable:
-      assert(false &&
-             "DecRefMsg/IncRefMsg/MakeCollectable already transformed");
-      return state;
-      
-    case Dealloc:
-      // Any use of -dealloc in GC is *bad*.
-      if (isGCEnabled()) {
-        V = V ^ RefVal::ErrorDeallocGC;
-        hasErr = V.getKind();
-        break;
-      }
-
-      switch (V.getKind()) {
-        default:
-          assert(false && "Invalid case.");
-        case RefVal::Owned:
-          // The object immediately transitions to the released state.
-          V = V ^ RefVal::Released;
-          V.clearCounts();
-          return state->set<RefBindings>(sym, V);
-        case RefVal::NotOwned:
-          V = V ^ RefVal::ErrorDeallocNotOwned;
-          hasErr = V.getKind();
-          break;
-      }
-      break;
-
-    case NewAutoreleasePool:
-      assert(!isGCEnabled());
-      return state->add<AutoreleaseStack>(sym);
-
-    case MayEscape:
-      if (V.getKind() == RefVal::Owned) {
-        V = V ^ RefVal::NotOwned;
-        break;
-      }
-
-      // Fall-through.
-
-    case DoNothingByRef:
-    case DoNothing:
-      return state;
-
-    case Autorelease:
-      if (isGCEnabled())
-        return state;
-
-      // Update the autorelease counts.
-      state = SendAutorelease(state, ARCountFactory, sym);
-      V = V.autorelease();
-      break;
-
-    case StopTracking:
-      return state->remove<RefBindings>(sym);
-
-    case IncRef:
-      switch (V.getKind()) {
-        default:
-          assert(false);
-
-        case RefVal::Owned:
-        case RefVal::NotOwned:
-          V = V + 1;
-          break;
-        case RefVal::Released:
-          // Non-GC cases are handled above.
-          assert(isGCEnabled());
-          V = (V ^ RefVal::Owned) + 1;
-          break;
-      }
-      break;
-
-    case SelfOwn:
-      V = V ^ RefVal::NotOwned;
-      // Fall-through.
-    case DecRef:
-    case DecRefBridgedTransfered:
-      switch (V.getKind()) {
-        default:
-          // case 'RefVal::Released' handled above.
-          assert (false);
-
-        case RefVal::Owned:
-          assert(V.getCount() > 0);
-          if (V.getCount() == 1)
-            V = V ^ (E == DecRefBridgedTransfered ? 
-                      RefVal::NotOwned : RefVal::Released);
-          V = V - 1;
-          break;
-
-        case RefVal::NotOwned:
-          if (V.getCount() > 0)
-            V = V - 1;
-          else {
-            V = V ^ RefVal::ErrorReleaseNotOwned;
-            hasErr = V.getKind();
-          }
-          break;
-
-        case RefVal::Released:
-          // Non-GC cases are handled above.
-          assert(isGCEnabled());
-          V = V ^ RefVal::ErrorUseAfterRelease;
-          hasErr = V.getKind();
-          break;
-      }
-      break;
-  }
-  return state->set<RefBindings>(sym, V);
+  return state;
 }
 
 //===----------------------------------------------------------------------===//
 // Handle dead symbols and end-of-path.
 //===----------------------------------------------------------------------===//
 
-std::pair<ExplodedNode*, const GRState *>
-CFRefCount::HandleAutoreleaseCounts(const GRState * state, 
-                                    GenericNodeBuilderRefCount Bd,
-                                    ExplodedNode* Pred,
-                                    ExprEngine &Eng,
-                                    SymbolRef Sym, RefVal V, bool &stop) {
-
+std::pair<ExplodedNode *, const ProgramState *>
+RetainCountChecker::handleAutoreleaseCounts(const ProgramState *state, 
+                                            GenericNodeBuilderRefCount Bd,
+                                            ExplodedNode *Pred, ExprEngine &Eng,
+                                            SymbolRef Sym, RefVal V) const {
   unsigned ACnt = V.getAutoreleaseCount();
-  stop = false;
 
   // No autorelease counts?  Nothing to be done.
   if (!ACnt)
     return std::make_pair(Pred, state);
 
-  assert(!isGCEnabled() && "Autorelease counts in GC mode?");
+  assert(!Eng.isObjCGCEnabled() && "Autorelease counts in GC mode?");
   unsigned Cnt = V.getCount();
 
   // FIXME: Handle sending 'autorelease' to already released object.
@@ -3284,48 +3350,54 @@ CFRefCount::HandleAutoreleaseCounts(const GRState * state,
         V = V ^ RefVal::ReturnedNotOwned;
       else
         V = V ^ RefVal::NotOwned;
-    }
-    else {
+    } else {
       V.setCount(Cnt - ACnt);
       V.setAutoreleaseCount(0);
     }
     state = state->set<RefBindings>(Sym, V);
     ExplodedNode *N = Bd.MakeNode(state, Pred);
-    stop = (N == 0);
+    if (N == 0)
+      state = 0;
     return std::make_pair(N, state);
   }
 
   // Woah!  More autorelease counts then retain counts left.
   // Emit hard error.
-  stop = true;
   V = V ^ RefVal::ErrorOverAutorelease;
   state = state->set<RefBindings>(Sym, V);
 
   if (ExplodedNode *N = Bd.MakeNode(state, Pred)) {
     N->markAsSink();
 
-    std::string sbuf;
-    llvm::raw_string_ostream os(sbuf);
+    llvm::SmallString<128> sbuf;
+    llvm::raw_svector_ostream os(sbuf);
     os << "Object over-autoreleased: object was sent -autorelease ";
     if (V.getAutoreleaseCount() > 1)
       os << V.getAutoreleaseCount() << " times ";
     os << "but the object has a +" << V.getCount() << " retain count";
 
+    if (!overAutorelease)
+      overAutorelease.reset(new OverAutorelease());
+
+    const LangOptions &LOpts = Eng.getContext().getLangOptions();
     CFRefReport *report =
-      new CFRefReport(*static_cast<CFRefBug*>(overAutorelease),
-                      *this, N, Sym, os.str());
-    BR->EmitReport(report);
+      new CFRefReport(*overAutorelease, LOpts, /* GCEnabled = */ false,
+                      SummaryLog, N, Sym, os.str());
+    Eng.getBugReporter().EmitReport(report);
   }
 
-  return std::make_pair((ExplodedNode*)0, state);
+  return std::make_pair((ExplodedNode *)0, (const ProgramState *)0);
 }
 
-const GRState *
-CFRefCount::HandleSymbolDeath(const GRState * state, SymbolRef sid, RefVal V,
-                              llvm::SmallVectorImpl<SymbolRef> &Leaked) {
-
-  bool hasLeak = V.isOwned() ||
-  ((V.isNotOwned() || V.isReturnedOwned()) && V.getCount() > 0);
+const ProgramState *
+RetainCountChecker::handleSymbolDeath(const ProgramState *state,
+                                      SymbolRef sid, RefVal V,
+                                    SmallVectorImpl<SymbolRef> &Leaked) const {
+  bool hasLeak = false;
+  if (V.isOwned())
+    hasLeak = true;
+  else if (V.isNotOwned() || V.isReturnedOwned())
+    hasLeak = (V.getCount() > 0);
 
   if (!hasLeak)
     return state->remove<RefBindings>(sid);
@@ -3334,13 +3406,11 @@ CFRefCount::HandleSymbolDeath(const GRState * state, SymbolRef sid, RefVal V,
   return state->set<RefBindings>(sid, V ^ RefVal::ErrorLeak);
 }
 
-ExplodedNode*
-CFRefCount::ProcessLeaks(const GRState * state,
-                         llvm::SmallVectorImpl<SymbolRef> &Leaked,
-                         GenericNodeBuilderRefCount &Builder,
-                         ExprEngine& Eng,
-                         ExplodedNode *Pred) {
-
+ExplodedNode *
+RetainCountChecker::processLeaks(const ProgramState *state,
+                                 SmallVectorImpl<SymbolRef> &Leaked,
+                                 GenericNodeBuilderRefCount &Builder,
+                                 ExprEngine &Eng, ExplodedNode *Pred) const {
   if (Leaked.empty())
     return Pred;
 
@@ -3348,85 +3418,94 @@ CFRefCount::ProcessLeaks(const GRState * state,
   ExplodedNode *N = Builder.MakeNode(state, Pred);
 
   if (N) {
-    for (llvm::SmallVectorImpl<SymbolRef>::iterator
+    for (SmallVectorImpl<SymbolRef>::iterator
          I = Leaked.begin(), E = Leaked.end(); I != E; ++I) {
 
-      CFRefBug *BT = static_cast<CFRefBug*>(Pred ? leakWithinFunction
-                                                 : leakAtReturn);
+      const LangOptions &LOpts = Eng.getContext().getLangOptions();
+      bool GCEnabled = Eng.isObjCGCEnabled();
+      CFRefBug *BT = Pred ? getLeakWithinFunctionBug(LOpts, GCEnabled)
+                          : getLeakAtReturnBug(LOpts, GCEnabled);
       assert(BT && "BugType not initialized.");
-      CFRefLeakReport* report = new CFRefLeakReport(*BT, *this, N, *I, Eng);
-      BR->EmitReport(report);
+
+      CFRefLeakReport *report = new CFRefLeakReport(*BT, LOpts, GCEnabled, 
+                                                    SummaryLog, N, *I, Eng);
+      Eng.getBugReporter().EmitReport(report);
     }
   }
 
   return N;
 }
 
-void CFRefCount::evalEndPath(ExprEngine& Eng,
-                             EndOfFunctionNodeBuilder& Builder) {
-
-  const GRState *state = Builder.getState();
+void RetainCountChecker::checkEndPath(EndOfFunctionNodeBuilder &Builder,
+                                      ExprEngine &Eng) const {
+  const ProgramState *state = Builder.getState();
   GenericNodeBuilderRefCount Bd(Builder);
   RefBindings B = state->get<RefBindings>();
-  ExplodedNode *Pred = 0;
+  ExplodedNode *Pred = Builder.getPredecessor();
 
   for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I) {
-    bool stop = false;
-    llvm::tie(Pred, state) = HandleAutoreleaseCounts(state, Bd, Pred, Eng,
-                                                     (*I).first,
-                                                     (*I).second, stop);
-
-    if (stop)
+    llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Bd, Pred, Eng,
+                                                     I->first, I->second);
+    if (!state)
       return;
   }
 
   B = state->get<RefBindings>();
-  llvm::SmallVector<SymbolRef, 10> Leaked;
+  SmallVector<SymbolRef, 10> Leaked;
 
   for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I)
-    state = HandleSymbolDeath(state, (*I).first, (*I).second, Leaked);
+    state = handleSymbolDeath(state, I->first, I->second, Leaked);
 
-  ProcessLeaks(state, Leaked, Bd, Eng, Pred);
+  processLeaks(state, Leaked, Bd, Eng, Pred);
 }
 
-void CFRefCount::evalDeadSymbols(ExplodedNodeSet& Dst,
-                                 ExprEngine& Eng,
-                                 StmtNodeBuilder& Builder,
-                                 ExplodedNode* Pred,
-                                 const GRState* state,
-                                 SymbolReaper& SymReaper) {
-  const Stmt *S = Builder.getStmt();
+const ProgramPointTag *
+RetainCountChecker::getDeadSymbolTag(SymbolRef sym) const {
+  const SimpleProgramPointTag *&tag = DeadSymbolTags[sym];
+  if (!tag) {
+    llvm::SmallString<64> buf;
+    llvm::raw_svector_ostream out(buf);
+    out << "RetainCountChecker : Dead Symbol : " << sym->getSymbolID();
+    tag = new SimpleProgramPointTag(out.str());
+  }
+  return tag;  
+}
+
+void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
+                                          CheckerContext &C) const {
+  ExprEngine &Eng = C.getEngine();
+  ExplodedNode *Pred = C.getPredecessor();
+
+  const ProgramState *state = C.getState();
   RefBindings B = state->get<RefBindings>();
 
   // Update counts from autorelease pools
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
        E = SymReaper.dead_end(); I != E; ++I) {
     SymbolRef Sym = *I;
-    if (const RefVal* T = B.lookup(Sym)){
+    if (const RefVal *T = B.lookup(Sym)){
       // Use the symbol as the tag.
       // FIXME: This might not be as unique as we would like.
-      GenericNodeBuilderRefCount Bd(Builder, S, Sym);
-      bool stop = false;
-      llvm::tie(Pred, state) = HandleAutoreleaseCounts(state, Bd, Pred, Eng,
-                                                       Sym, *T, stop);
-      if (stop)
+      GenericNodeBuilderRefCount Bd(C, getDeadSymbolTag(Sym));
+      llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Bd, Pred, Eng,
+                                                       Sym, *T);
+      if (!state)
         return;
     }
   }
 
   B = state->get<RefBindings>();
-  llvm::SmallVector<SymbolRef, 10> Leaked;
+  SmallVector<SymbolRef, 10> Leaked;
 
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
        E = SymReaper.dead_end(); I != E; ++I) {
-      if (const RefVal* T = B.lookup(*I))
-        state = HandleSymbolDeath(state, *I, *T, Leaked);
+    if (const RefVal *T = B.lookup(*I))
+      state = handleSymbolDeath(state, *I, *T, Leaked);
   }
 
-  static unsigned LeakPPTag = 0;
   {
-    GenericNodeBuilderRefCount Bd(Builder, S, &LeakPPTag);
-    Pred = ProcessLeaks(state, Leaked, Bd, Eng, Pred);
+    GenericNodeBuilderRefCount Bd(C, this);
+    Pred = processLeaks(state, Leaked, Bd, Eng, Pred);
   }
 
   // Did we cache out?
@@ -3434,256 +3513,76 @@ void CFRefCount::evalDeadSymbols(ExplodedNodeSet& Dst,
     return;
 
   // Now generate a new node that nukes the old bindings.
-  RefBindings::Factory& F = state->get_context<RefBindings>();
+  RefBindings::Factory &F = state->get_context<RefBindings>();
 
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
-       E = SymReaper.dead_end(); I!=E; ++I) B = F.remove(B, *I);
+       E = SymReaper.dead_end(); I != E; ++I)
+    B = F.remove(B, *I);
 
   state = state->set<RefBindings>(B);
-  Builder.MakeNode(Dst, S, Pred, state);
-}
-
-void CFRefCount::ProcessNonLeakError(ExplodedNodeSet& Dst,
-                                     StmtNodeBuilder& Builder,
-                                     const Expr* NodeExpr, 
-                                     SourceRange ErrorRange,
-                                     ExplodedNode* Pred,
-                                     const GRState* St,
-                                     RefVal::Kind hasErr, SymbolRef Sym) {
-  Builder.BuildSinks = true;
-  ExplodedNode *N  = Builder.MakeNode(Dst, NodeExpr, Pred, St);
-
-  if (!N)
-    return;
-
-  CFRefBug *BT = 0;
-
-  switch (hasErr) {
-    default:
-      assert(false && "Unhandled error.");
-      return;
-    case RefVal::ErrorUseAfterRelease:
-      BT = static_cast<CFRefBug*>(useAfterRelease);
-      break;
-    case RefVal::ErrorReleaseNotOwned:
-      BT = static_cast<CFRefBug*>(releaseNotOwned);
-      break;
-    case RefVal::ErrorDeallocGC:
-      BT = static_cast<CFRefBug*>(deallocGC);
-      break;
-    case RefVal::ErrorDeallocNotOwned:
-      BT = static_cast<CFRefBug*>(deallocNotOwned);
-      break;
-  }
-
-  CFRefReport *report = new CFRefReport(*BT, *this, N, Sym);
-  report->addRange(ErrorRange);
-  BR->EmitReport(report);
+  C.generateNode(state, Pred);
 }
 
 //===----------------------------------------------------------------------===//
-// Pieces of the retain/release checker implemented using a CheckerVisitor.
-// More pieces of the retain/release checker will be migrated to this interface
-// (ideally, all of it some day).
+// Debug printing of refcount bindings and autorelease pools.
 //===----------------------------------------------------------------------===//
 
-namespace {
-class RetainReleaseChecker
-  : public Checker< check::PostStmt<BlockExpr>,
-                    check::PostStmt<CastExpr>,
-                    check::RegionChanges > {
-public:
-    bool wantsRegionUpdate;
-    
-    RetainReleaseChecker() : wantsRegionUpdate(true) {}
-    
-    
-    void checkPostStmt(const BlockExpr *BE, CheckerContext &C) const;
-                      
-    void checkPostStmt(const CastExpr *CE, CheckerContext &C) const;
+static void PrintPool(raw_ostream &Out, SymbolRef Sym,
+                      const ProgramState *State) {
+  Out << ' ';
+  if (Sym)
+    Out << Sym->getSymbolID();
+  else
+    Out << "<pool>";
+  Out << ":{";
 
-    const GRState *checkRegionChanges(const GRState *state,
-                            const StoreManager::InvalidatedSymbols *invalidated,
-                                      const MemRegion * const *begin,
-                                      const MemRegion * const *end) const;
-                                          
-    bool wantsRegionChangeUpdate(const GRState *state) const {
-      return wantsRegionUpdate;
-    }
-};
-} // end anonymous namespace
+  // Get the contents of the pool.
+  if (const ARCounts *Cnts = State->get<AutoreleasePoolContents>(Sym))
+    for (ARCounts::iterator I = Cnts->begin(), E = Cnts->end(); I != E; ++I)
+      Out << '(' << I.getKey() << ',' << I.getData() << ')';
 
-const GRState *
-RetainReleaseChecker::checkRegionChanges(const GRState *state,
-                            const StoreManager::InvalidatedSymbols *invalidated,
-                                         const MemRegion * const *begin,
-                                         const MemRegion * const *end) const {
-  if (!invalidated)
-    return state;
-
-  for (StoreManager::InvalidatedSymbols::const_iterator I=invalidated->begin(),
-       E = invalidated->end(); I!=E; ++I) {
-    SymbolRef sym = *I;
-    if (WhitelistedSymbols.count(sym))
-      continue;
-    // Remove any existing reference-count binding.
-    state = state->remove<RefBindings>(sym);
-  }
-  return state;
+  Out << '}';
 }
 
-void RetainReleaseChecker::checkPostStmt(const BlockExpr *BE,
-                                         CheckerContext &C) const {
-
-  // Scan the BlockDecRefExprs for any object the retain/release checker
-  // may be tracking.
-  if (!BE->getBlockDecl()->hasCaptures())
-    return;
-
-  const GRState *state = C.getState();
-  const BlockDataRegion *R =
-    cast<BlockDataRegion>(state->getSVal(BE).getAsRegion());
-
-  BlockDataRegion::referenced_vars_iterator I = R->referenced_vars_begin(),
-                                            E = R->referenced_vars_end();
-
-  if (I == E)
-    return;
-
-  // FIXME: For now we invalidate the tracking of all symbols passed to blocks
-  // via captured variables, even though captured variables result in a copy
-  // and in implicit increment/decrement of a retain count.
-  llvm::SmallVector<const MemRegion*, 10> Regions;
-  const LocationContext *LC = C.getPredecessor()->getLocationContext();
-  MemRegionManager &MemMgr = C.getSValBuilder().getRegionManager();
-
-  for ( ; I != E; ++I) {
-    const VarRegion *VR = *I;
-    if (VR->getSuperRegion() == R) {
-      VR = MemMgr.getVarRegion(VR->getDecl(), LC);
-    }
-    Regions.push_back(VR);
-  }
-
-  state =
-    state->scanReachableSymbols<StopTrackingCallback>(Regions.data(),
-                                    Regions.data() + Regions.size()).getState();
-  C.addTransition(state);
+static bool UsesAutorelease(const ProgramState *state) {
+  // A state uses autorelease if it allocated an autorelease pool or if it has
+  // objects in the caller's autorelease pool.
+  return !state->get<AutoreleaseStack>().isEmpty() ||
+          state->get<AutoreleasePoolContents>(SymbolRef());
 }
 
-void RetainReleaseChecker::checkPostStmt(const CastExpr *CE,
-                                         CheckerContext &C) const {
-  const ObjCBridgedCastExpr *BE = dyn_cast<ObjCBridgedCastExpr>(CE);
-  if (!BE)
-    return;
-  
-  ArgEffect AE = IncRef;
-  
-  switch (BE->getBridgeKind()) {
-    case clang::OBC_Bridge:
-      // Do nothing.
-      return;
-    case clang::OBC_BridgeRetained:
-      AE = IncRef;
-      break;      
-    case clang::OBC_BridgeTransfer:
-      AE = DecRefBridgedTransfered;
-      break;
-  }
-  
-  const GRState *state = C.getState();
-  SymbolRef Sym = state->getSVal(CE).getAsLocSymbol();
-  if (!Sym)
-    return;
-  const RefVal* T = state->get<RefBindings>(Sym);
-  if (!T)
-    return;
+void RetainCountChecker::printState(raw_ostream &Out, const ProgramState *State,
+                                    const char *NL, const char *Sep) const {
 
-  // This is gross.  Once the checker and CFRefCount are unified,
-  // this will go away.
-  CFRefCount &cf = static_cast<CFRefCount&>(C.getEngine().getTF());
-  RefVal::Kind hasErr = (RefVal::Kind) 0;
-  state = cf.Update(state, Sym, *T, AE, hasErr);
-  
-  if (hasErr) {
-    
-    return;
+  RefBindings B = State->get<RefBindings>();
+
+  if (!B.isEmpty())
+    Out << Sep << NL;
+
+  for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+    Out << I->first << " : ";
+    I->second.print(Out);
+    Out << NL;
   }
 
-  C.generateNode(state);
+  // Print the autorelease stack.
+  if (UsesAutorelease(State)) {
+    Out << Sep << NL << "AR pool stack:";
+    ARStack Stack = State->get<AutoreleaseStack>();
+
+    PrintPool(Out, SymbolRef(), State);  // Print the caller's pool.
+    for (ARStack::iterator I = Stack.begin(), E = Stack.end(); I != E; ++I)
+      PrintPool(Out, *I, State);
+
+    Out << NL;
+  }
 }
 
 //===----------------------------------------------------------------------===//
-// Transfer function creation for external clients.
+// Checker registration.
 //===----------------------------------------------------------------------===//
 
-void CFRefCount::RegisterChecks(ExprEngine& Eng) {
-  BugReporter &BR = Eng.getBugReporter();
-
-  useAfterRelease = new UseAfterRelease(this);
-  BR.Register(useAfterRelease);
-
-  releaseNotOwned = new BadRelease(this);
-  BR.Register(releaseNotOwned);
-
-  deallocGC = new DeallocGC(this);
-  BR.Register(deallocGC);
-
-  deallocNotOwned = new DeallocNotOwned(this);
-  BR.Register(deallocNotOwned);
-
-  overAutorelease = new OverAutorelease(this);
-  BR.Register(overAutorelease);
-
-  returnNotOwnedForOwned = new ReturnedNotOwnedForOwned(this);
-  BR.Register(returnNotOwnedForOwned);
-
-  // First register "return" leaks.
-  const char* name = 0;
-
-  if (isGCEnabled())
-    name = "Leak of returned object when using garbage collection";
-  else if (getLangOptions().getGCMode() == LangOptions::HybridGC)
-    name = "Leak of returned object when not using garbage collection (GC) in "
-    "dual GC/non-GC code";
-  else {
-    assert(getLangOptions().getGCMode() == LangOptions::NonGC);
-    name = "Leak of returned object";
-  }
-
-  // Leaks should not be reported if they are post-dominated by a sink.
-  leakAtReturn = new LeakAtReturn(this, name);
-  leakAtReturn->setSuppressOnSink(true);
-  BR.Register(leakAtReturn);
-
-  // Second, register leaks within a function/method.
-  if (isGCEnabled())
-    name = "Leak of object when using garbage collection";
-  else if (getLangOptions().getGCMode() == LangOptions::HybridGC)
-    name = "Leak of object when not using garbage collection (GC) in "
-    "dual GC/non-GC code";
-  else {
-    assert(getLangOptions().getGCMode() == LangOptions::NonGC);
-    name = "Leak";
-  }
-
-  // Leaks should not be reported if they are post-dominated by sinks.
-  leakWithinFunction = new LeakWithinFunction(this, name);
-  leakWithinFunction->setSuppressOnSink(true);
-  BR.Register(leakWithinFunction);
-
-  // Save the reference to the BugReporter.
-  this->BR = &BR;
-
-  // Register the RetainReleaseChecker with the ExprEngine object.
-  // Functionality in CFRefCount will be migrated to RetainReleaseChecker
-  // over time.
-  // FIXME: HACK! Remove TransferFuncs and turn all of CFRefCount into fully
-  // using the checker mechanism.
-  Eng.getCheckerManager().registerChecker<RetainReleaseChecker>();
+void ento::registerRetainCountChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<RetainCountChecker>();
 }
 
-TransferFuncs* ento::MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
-                                         const LangOptions& lopts) {
-  return new CFRefCount(Ctx, GCEnabled, lopts);
-}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
index 1729b25a2979..e761bff85583 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
@@ -33,7 +33,7 @@ class ReturnPointerRangeChecker :
 
 void ReturnPointerRangeChecker::checkPreStmt(const ReturnStmt *RS,
                                              CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   const Expr *RetE = RS->getRetValue();
   if (!RetE)
@@ -58,8 +58,8 @@ void ReturnPointerRangeChecker::checkPreStmt(const ReturnStmt *RS,
     = C.getStoreManager().getSizeInElements(state, ER->getSuperRegion(),
                                            ER->getValueType());
 
-  const GRState *StInBound = state->assumeInBound(Idx, NumElements, true);
-  const GRState *StOutBound = state->assumeInBound(Idx, NumElements, false);
+  const ProgramState *StInBound = state->assumeInBound(Idx, NumElements, true);
+  const ProgramState *StOutBound = state->assumeInBound(Idx, NumElements, false);
   if (StOutBound && !StInBound) {
     ExplodedNode *N = C.generateSink(StOutBound);
 
@@ -78,8 +78,8 @@ void ReturnPointerRangeChecker::checkPreStmt(const ReturnStmt *RS,
     // reference is outside the range.
 
     // Generate a report for this bug.
-    RangedBugReport *report = 
-      new RangedBugReport(*BT, BT->getDescription(), N);
+    BugReport *report = 
+      new BugReport(*BT, BT->getDescription(), N);
 
     report->addRange(RetE->getSourceRange());
     C.EmitReport(report);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
index 7c215b7cf318..e8c8d902a5e5 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
@@ -50,11 +50,11 @@ void ReturnUndefChecker::checkPreStmt(const ReturnStmt *RS,
     BT.reset(new BuiltinBug("Garbage return value",
                             "Undefined or garbage value returned to caller"));
     
-  EnhancedBugReport *report = 
-    new EnhancedBugReport(*BT, BT->getDescription(), N);
+  BugReport *report = 
+    new BugReport(*BT, BT->getDescription(), N);
 
   report->addRange(RetE->getSourceRange());
-  report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, RetE);
+  report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, RetE));
 
   C.EmitReport(report);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
index 73ce359edc3e..91c4b96d695e 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
@@ -17,7 +17,7 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/SmallString.h"
 using namespace clang;
@@ -35,12 +35,12 @@ class StackAddrEscapeChecker : public Checker< check::PreStmt<ReturnStmt>,
 private:
   void EmitStackError(CheckerContext &C, const MemRegion *R,
                       const Expr *RetE) const;
-  static SourceRange GenName(llvm::raw_ostream &os, const MemRegion *R,
+  static SourceRange GenName(raw_ostream &os, const MemRegion *R,
                              SourceManager &SM);
 };
 }
 
-SourceRange StackAddrEscapeChecker::GenName(llvm::raw_ostream &os,
+SourceRange StackAddrEscapeChecker::GenName(raw_ostream &os,
                                           const MemRegion *R,
                                           SourceManager &SM) {
     // Get the base region, stripping away fields and elements.
@@ -50,34 +50,39 @@ SourceRange StackAddrEscapeChecker::GenName(llvm::raw_ostream &os,
   
   // Check if the region is a compound literal.
   if (const CompoundLiteralRegion* CR = dyn_cast<CompoundLiteralRegion>(R)) { 
-    const CompoundLiteralExpr* CL = CR->getLiteralExpr();
+    const CompoundLiteralExpr *CL = CR->getLiteralExpr();
     os << "stack memory associated with a compound literal "
           "declared on line "
-        << SM.getInstantiationLineNumber(CL->getLocStart())
+        << SM.getExpansionLineNumber(CL->getLocStart())
         << " returned to caller";    
     range = CL->getSourceRange();
   }
   else if (const AllocaRegion* AR = dyn_cast<AllocaRegion>(R)) {
-    const Expr* ARE = AR->getExpr();
+    const Expr *ARE = AR->getExpr();
     SourceLocation L = ARE->getLocStart();
     range = ARE->getSourceRange();    
     os << "stack memory allocated by call to alloca() on line "
-       << SM.getInstantiationLineNumber(L);
+       << SM.getExpansionLineNumber(L);
   }
   else if (const BlockDataRegion *BR = dyn_cast<BlockDataRegion>(R)) {
     const BlockDecl *BD = BR->getCodeRegion()->getDecl();
     SourceLocation L = BD->getLocStart();
     range = BD->getSourceRange();
     os << "stack-allocated block declared on line "
-       << SM.getInstantiationLineNumber(L);
+       << SM.getExpansionLineNumber(L);
   }
   else if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
     os << "stack memory associated with local variable '"
        << VR->getString() << '\'';
     range = VR->getDecl()->getSourceRange();
   }
+  else if (const CXXTempObjectRegion *TOR = dyn_cast<CXXTempObjectRegion>(R)) {
+    os << "stack memory associated with temporary object of type '"
+       << TOR->getValueType().getAsString() << '\'';
+    range = TOR->getExpr()->getSourceRange();
+  }
   else {
-    assert(false && "Invalid region in ReturnStackAddressChecker.");
+    llvm_unreachable("Invalid region in ReturnStackAddressChecker.");
   } 
   
   return range;
@@ -99,7 +104,7 @@ void StackAddrEscapeChecker::EmitStackError(CheckerContext &C, const MemRegion *
   llvm::raw_svector_ostream os(buf);
   SourceRange range = GenName(os, R, C.getSourceManager());
   os << " returned to caller";
-  RangedBugReport *report = new RangedBugReport(*BT_returnstack, os.str(), N);
+  BugReport *report = new BugReport(*BT_returnstack, os.str(), N);
   report->addRange(RetE->getSourceRange());
   if (range.isValid())
     report->addRange(range);
@@ -134,7 +139,7 @@ void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
 void StackAddrEscapeChecker::checkEndPath(EndOfFunctionNodeBuilder &B,
                                         ExprEngine &Eng) const {
 
-  const GRState *state = B.getState();
+  const ProgramState *state = B.getState();
 
   // Iterate over all bindings to global variables and see if it contains
   // a memory region in the stack space.
@@ -143,7 +148,7 @@ void StackAddrEscapeChecker::checkEndPath(EndOfFunctionNodeBuilder &B,
     ExprEngine &Eng;
     const StackFrameContext *CurSFC;
   public:
-    llvm::SmallVector<std::pair<const MemRegion*, const MemRegion*>, 10> V;
+    SmallVector<std::pair<const MemRegion*, const MemRegion*>, 10> V;
 
     CallBack(ExprEngine &Eng, const LocationContext *LCtx)
       : Eng(Eng), CurSFC(LCtx->getCurrentStackFrame()) {}
@@ -202,9 +207,9 @@ void StackAddrEscapeChecker::checkEndPath(EndOfFunctionNodeBuilder &B,
                                 Eng.getContext().getSourceManager());
     os << " is still referred to by the global variable '";
     const VarRegion *VR = cast<VarRegion>(cb.V[i].first->getBaseRegion());
-    os << VR->getDecl()->getNameAsString() 
+    os << *VR->getDecl()
        << "' upon returning to the caller.  This will be a dangling reference";
-    RangedBugReport *report = new RangedBugReport(*BT_stackleak, os.str(), N);
+    BugReport *report = new BugReport(*BT_stackleak, os.str(), N);
     if (range.isValid())
       report->addRange(range);
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index 711c672c1448..1d14e9e15e42 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -16,8 +16,8 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/ImmutableMap.h"
 
@@ -96,9 +96,9 @@ class StreamChecker : public Checker<eval::Call,
 
   void OpenFileAux(CheckerContext &C, const CallExpr *CE) const;
   
-  const GRState *CheckNullStream(SVal SV, const GRState *state, 
+  const ProgramState *CheckNullStream(SVal SV, const ProgramState *state, 
                                  CheckerContext &C) const;
-  const GRState *CheckDoubleClose(const CallExpr *CE, const GRState *state, 
+  const ProgramState *CheckDoubleClose(const CallExpr *CE, const ProgramState *state, 
                                  CheckerContext &C) const;
 };
 
@@ -107,15 +107,15 @@ class StreamChecker : public Checker<eval::Call,
 namespace clang {
 namespace ento {
   template <>
-  struct GRStateTrait<StreamState> 
-    : public GRStatePartialTrait<llvm::ImmutableMap<SymbolRef, StreamState> > {
+  struct ProgramStateTrait<StreamState> 
+    : public ProgramStatePartialTrait<llvm::ImmutableMap<SymbolRef, StreamState> > {
     static void *GDMIndex() { static int x; return &x; }
   };
 }
 }
 
 bool StreamChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
   const FunctionDecl *FD = L.getAsFunctionDecl();
@@ -221,8 +221,8 @@ void StreamChecker::Tmpfile(CheckerContext &C, const CallExpr *CE) const {
 }
 
 void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
-  unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+  const ProgramState *state = C.getState();
+  unsigned Count = C.getCurrentBlockCount();
   SValBuilder &svalBuilder = C.getSValBuilder();
   DefinedSVal RetVal =
     cast<DefinedSVal>(svalBuilder.getConjuredSymbolVal(0, CE, Count));
@@ -231,7 +231,7 @@ void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) const {
   ConstraintManager &CM = C.getConstraintManager();
   // Bifurcate the state into two: one with a valid FILE* pointer, the other
   // with a NULL.
-  const GRState *stateNotNull, *stateNull;
+  const ProgramState *stateNotNull, *stateNull;
   llvm::tie(stateNotNull, stateNull) = CM.assumeDual(state, RetVal);
   
   if (SymbolRef Sym = RetVal.getAsSymbol()) {
@@ -247,25 +247,25 @@ void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) const {
 }
 
 void StreamChecker::Fclose(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = CheckDoubleClose(CE, C.getState(), C);
+  const ProgramState *state = CheckDoubleClose(CE, C.getState(), C);
   if (state)
     C.addTransition(state);
 }
 
 void StreamChecker::Fread(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(3)), state, C))
     return;
 }
 
 void StreamChecker::Fwrite(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(3)), state, C))
     return;
 }
 
 void StreamChecker::Fseek(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!(state = CheckNullStream(state->getSVal(CE->getArg(0)), state, C)))
     return;
   // Check the legality of the 'whence' argument of 'fseek'.
@@ -291,61 +291,61 @@ void StreamChecker::Fseek(CheckerContext &C, const CallExpr *CE) const {
 }
 
 void StreamChecker::Ftell(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Rewind(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Fgetpos(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Fsetpos(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Clearerr(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Feof(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Ferror(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
 void StreamChecker::Fileno(CheckerContext &C, const CallExpr *CE) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (!CheckNullStream(state->getSVal(CE->getArg(0)), state, C))
     return;
 }
 
-const GRState *StreamChecker::CheckNullStream(SVal SV, const GRState *state,
+const ProgramState *StreamChecker::CheckNullStream(SVal SV, const ProgramState *state,
                                     CheckerContext &C) const {
   const DefinedSVal *DV = dyn_cast<DefinedSVal>(&SV);
   if (!DV)
     return 0;
 
   ConstraintManager &CM = C.getConstraintManager();
-  const GRState *stateNotNull, *stateNull;
+  const ProgramState *stateNotNull, *stateNull;
   llvm::tie(stateNotNull, stateNull) = CM.assumeDual(state, *DV);
 
   if (!stateNotNull && stateNull) {
@@ -361,8 +361,8 @@ const GRState *StreamChecker::CheckNullStream(SVal SV, const GRState *state,
   return stateNotNull;
 }
 
-const GRState *StreamChecker::CheckDoubleClose(const CallExpr *CE,
-                                               const GRState *state,
+const ProgramState *StreamChecker::CheckDoubleClose(const CallExpr *CE,
+                                               const ProgramState *state,
                                                CheckerContext &C) const {
   SymbolRef Sym = state->getSVal(CE->getArg(0)).getAsSymbol();
   if (!Sym)
@@ -399,7 +399,7 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
          E = SymReaper.dead_end(); I != E; ++I) {
     SymbolRef Sym = *I;
-    const GRState *state = C.getState();
+    const ProgramState *state = C.getState();
     const StreamState *SS = state->get<StreamState>(Sym);
     if (!SS)
       return;
@@ -420,7 +420,7 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
 
 void StreamChecker::checkEndPath(EndOfFunctionNodeBuilder &B,
                                  ExprEngine &Eng) const {
-  const GRState *state = B.getState();
+  const ProgramState *state = B.getState();
   typedef llvm::ImmutableMap<SymbolRef, StreamState> SymMap;
   SymMap M = state->get<StreamState>();
   
@@ -445,7 +445,7 @@ void StreamChecker::checkPreStmt(const ReturnStmt *S, CheckerContext &C) const {
   if (!RetE)
     return;
   
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SymbolRef Sym = state->getSVal(RetE).getAsSymbol();
   
   if (!Sym)
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
index 1fb181591b42..b860b34ff359 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
@@ -27,26 +27,25 @@ class UndefBranchChecker : public Checker<check::BranchCondition> {
   mutable llvm::OwningPtr<BuiltinBug> BT;
 
   struct FindUndefExpr {
-    GRStateManager& VM;
-    const GRState* St;
+    const ProgramState *St;
 
-    FindUndefExpr(GRStateManager& V, const GRState* S) : VM(V), St(S) {}
+    FindUndefExpr(const ProgramState *S) : St(S) {}
 
-    const Expr* FindExpr(const Expr* Ex) {
+    const Expr *FindExpr(const Expr *Ex) {
       if (!MatchesCriteria(Ex))
         return 0;
 
       for (Stmt::const_child_iterator I = Ex->child_begin(), 
                                       E = Ex->child_end();I!=E;++I)
-        if (const Expr* ExI = dyn_cast_or_null<Expr>(*I)) {
-          const Expr* E2 = FindExpr(ExI);
+        if (const Expr *ExI = dyn_cast_or_null<Expr>(*I)) {
+          const Expr *E2 = FindExpr(ExI);
           if (E2) return E2;
         }
 
       return Ex;
     }
 
-    bool MatchesCriteria(const Expr* Ex) { return St->getSVal(Ex).isUndef(); }
+    bool MatchesCriteria(const Expr *Ex) { return St->getSVal(Ex).isUndef(); }
   };
 
 public:
@@ -59,10 +58,10 @@ class UndefBranchChecker : public Checker<check::BranchCondition> {
 void UndefBranchChecker::checkBranchCondition(const Stmt *Condition,
                                               BranchNodeBuilder &Builder,
                                               ExprEngine &Eng) const {
-  const GRState *state = Builder.getState();
+  const ProgramState *state = Builder.getState();
   SVal X = state->getSVal(Condition);
   if (X.isUndef()) {
-    ExplodedNode *N = Builder.generateNode(state, true);
+    ExplodedNode *N = Builder.generateNode(Condition, state);
     if (N) {
       N->markAsSink();
       if (!BT)
@@ -74,33 +73,31 @@ void UndefBranchChecker::checkBranchCondition(const Stmt *Condition,
       // branch condition."  We do a recursive walk of the condition's
       // subexpressions and roughly look for the most nested subexpression
       // that binds to Undefined.  We then highlight that expression's range.
-      BlockEdge B = cast<BlockEdge>(N->getLocation());
-      const Expr* Ex = cast<Expr>(B.getSrc()->getTerminatorCondition());
-      assert (Ex && "Block must have a terminator.");
 
       // Get the predecessor node and check if is a PostStmt with the Stmt
       // being the terminator condition.  We want to inspect the state
       // of that node instead because it will contain main information about
       // the subexpressions.
-      assert (!N->pred_empty());
 
       // Note: any predecessor will do.  They should have identical state,
       // since all the BlockEdge did was act as an error sink since the value
       // had to already be undefined.
+      assert (!N->pred_empty());
+      const Expr *Ex = cast<Expr>(Condition);
       ExplodedNode *PrevN = *N->pred_begin();
       ProgramPoint P = PrevN->getLocation();
-      const GRState* St = N->getState();
+      const ProgramState *St = N->getState();
 
-      if (PostStmt* PS = dyn_cast<PostStmt>(&P))
+      if (PostStmt *PS = dyn_cast<PostStmt>(&P))
         if (PS->getStmt() == Ex)
           St = PrevN->getState();
 
-      FindUndefExpr FindIt(Eng.getStateManager(), St);
+      FindUndefExpr FindIt(St);
       Ex = FindIt.FindExpr(Ex);
 
       // Emit the bug report.
-      EnhancedBugReport *R = new EnhancedBugReport(*BT, BT->getDescription(),N);
-      R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Ex);
+      BugReport *R = new BugReport(*BT, BT->getDescription(), N);
+      R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, Ex));
       R->addRange(Ex->getSourceRange());
 
       Eng.getBugReporter().EmitReport(R);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
index 69958d11f717..2aebed9346d6 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
@@ -55,7 +55,7 @@ UndefCapturedBlockVarChecker::checkPostStmt(const BlockExpr *BE,
   if (!BE->getBlockDecl()->hasCaptures())
     return;
 
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const BlockDataRegion *R =
     cast<BlockDataRegion>(state->getSVal(BE).getAsRegion());
 
@@ -74,8 +74,9 @@ UndefCapturedBlockVarChecker::checkPostStmt(const BlockExpr *BE,
     // Get the VarRegion associated with VD in the local stack frame.
     const LocationContext *LC = C.getPredecessor()->getLocationContext();
     VR = C.getSValBuilder().getRegionManager().getVarRegion(VD, LC);
+    SVal VRVal = state->getSVal(VR);
 
-    if (state->getSVal(VR).isUndef())
+    if (VRVal.isUndef())
       if (ExplodedNode *N = C.generateSink()) {
         if (!BT)
           BT.reset(new BuiltinBug("uninitialized variable captured by block"));
@@ -87,10 +88,10 @@ UndefCapturedBlockVarChecker::checkPostStmt(const BlockExpr *BE,
         os << "Variable '" << VD->getName() 
            << "' is uninitialized when captured by block";
 
-        EnhancedBugReport *R = new EnhancedBugReport(*BT, os.str(), N);
+        BugReport *R = new BugReport(*BT, os.str(), N);
         if (const Expr *Ex = FindBlockDeclRefExpr(BE->getBody(), VD))
           R->addRange(Ex->getSourceRange());
-        R->addVisitorCreator(bugreporter::registerFindLastStore, VR);
+        R->addVisitor(new FindLastStoreBRVisitor(VRVal, VR));
         // need location of block
         C.EmitReport(R);
       }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
index 7fa3804639d6..7ae966865c86 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
@@ -35,7 +35,7 @@ class UndefResultChecker
 
 void UndefResultChecker::checkPostStmt(const BinaryOperator *B,
                                        CheckerContext &C) const {
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   if (state->getSVal(B).isUndef()) {
     // Generate an error node.
     ExplodedNode *N = C.generateSink();
@@ -71,13 +71,13 @@ void UndefResultChecker::checkPostStmt(const BinaryOperator *B,
          << BinaryOperator::getOpcodeStr(B->getOpcode())
          << "' expression is undefined";
     }
-    EnhancedBugReport *report = new EnhancedBugReport(*BT, OS.str(), N);
+    BugReport *report = new BugReport(*BT, OS.str(), N);
     if (Ex) {
       report->addRange(Ex->getSourceRange());
-      report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Ex);
+      report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, Ex));
     }
     else
-      report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, B);
+      report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, B));
     C.EmitReport(report);
   }
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
index e51ab20d2f30..bb6831b78301 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
@@ -40,10 +40,10 @@ UndefinedArraySubscriptChecker::checkPreStmt(const ArraySubscriptExpr *A,
         BT.reset(new BuiltinBug("Array subscript is undefined"));
 
       // Generate a report for this bug.
-      EnhancedBugReport *R = new EnhancedBugReport(*BT, BT->getName(), N);
+      BugReport *R = new BugReport(*BT, BT->getName(), N);
       R->addRange(A->getIdx()->getSourceRange());
-      R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, 
-                           A->getIdx());
+      R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                                                 A->getIdx()));
       C.EmitReport(R);
     }
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
index 28806e3db917..5ca4a9fe46d2 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
@@ -27,11 +27,13 @@ class UndefinedAssignmentChecker
   mutable llvm::OwningPtr<BugType> BT;
 
 public:
-  void checkBind(SVal location, SVal val, CheckerContext &C) const;
+  void checkBind(SVal location, SVal val, const Stmt *S,
+                 CheckerContext &C) const;
 };
 }
 
 void UndefinedAssignmentChecker::checkBind(SVal location, SVal val,
+                                           const Stmt *StoreE,
                                            CheckerContext &C) const {
   if (!val.isUndef())
     return;
@@ -49,11 +51,10 @@ void UndefinedAssignmentChecker::checkBind(SVal location, SVal val,
   // Generate a report for this bug.
   const Expr *ex = 0;
 
-  const Stmt *StoreE = C.getStmt();
   while (StoreE) {
     if (const BinaryOperator *B = dyn_cast<BinaryOperator>(StoreE)) {
       if (B->isCompoundAssignmentOp()) {
-        const GRState *state = C.getState();
+        const ProgramState *state = C.getState();
         if (state->getSVal(B->getLHS()).isUndef()) {
           str = "The left expression of the compound assignment is an "
                 "uninitialized value. The computed value will also be garbage";
@@ -67,17 +68,17 @@ void UndefinedAssignmentChecker::checkBind(SVal location, SVal val,
     }
 
     if (const DeclStmt *DS = dyn_cast<DeclStmt>(StoreE)) {
-      const VarDecl* VD = dyn_cast<VarDecl>(DS->getSingleDecl());
+      const VarDecl *VD = dyn_cast<VarDecl>(DS->getSingleDecl());
       ex = VD->getInit();
     }
 
     break;
   }
 
-  EnhancedBugReport *R = new EnhancedBugReport(*BT, str, N);
+  BugReport *R = new BugReport(*BT, str, N);
   if (ex) {
     R->addRange(ex->getSourceRange());
-    R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, ex);
+    R->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, ex));
   }
   C.EmitReport(R);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index 0ecc391f3e8f..cec286d2f3e9 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -62,7 +62,8 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
   // The definition of O_CREAT is platform specific.  We need a better way
   // of querying this information from the checking environment.
   if (!Val_O_CREAT.hasValue()) {
-    if (C.getASTContext().Target.getTriple().getVendor() == llvm::Triple::Apple)
+    if (C.getASTContext().getTargetInfo().getTriple().getVendor() 
+                                                      == llvm::Triple::Apple)
       Val_O_CREAT = 0x0200;
     else {
       // FIXME: We need a more general way of getting the O_CREAT value.
@@ -73,7 +74,7 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
   }
 
   // Look at the 'oflags' argument for the O_CREAT flag.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
 
   if (CE->getNumArgs() < 2) {
     // The frontend should issue a warning for this case, so this is a sanity
@@ -101,7 +102,7 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
   DefinedSVal maskedFlags = cast<DefinedSVal>(maskedFlagsUC);
 
   // Check if maskedFlags is non-zero.
-  const GRState *trueState, *falseState;
+  const ProgramState *trueState, *falseState;
   llvm::tie(trueState, falseState) = state->assume(maskedFlags);
 
   // Only emit an error if the value of 'maskedFlags' is properly
@@ -116,8 +117,8 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
 
     LazyInitialize(BT_open, "Improper use of 'open'");
 
-    RangedBugReport *report =
-      new RangedBugReport(*BT_open,
+    BugReport *report =
+      new BugReport(*BT_open,
                             "Call to 'open' requires a third argument when "
                             "the 'O_CREAT' flag is set", N);
     report->addRange(oflagsEx->getSourceRange());
@@ -140,7 +141,7 @@ void UnixAPIChecker::CheckPthreadOnce(CheckerContext &C,
 
   // Check if the first argument is stack allocated.  If so, issue a warning
   // because that's likely to be bad news.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const MemRegion *R = state->getSVal(CE->getArg(0)).getAsRegion();
   if (!R || !isa<StackSpaceRegion>(R->getMemorySpace()))
     return;
@@ -163,7 +164,7 @@ void UnixAPIChecker::CheckPthreadOnce(CheckerContext &C,
 
   LazyInitialize(BT_pthreadOnce, "Improper use of 'pthread_once'");
 
-  RangedBugReport *report = new RangedBugReport(*BT_pthreadOnce, os.str(), N);
+  BugReport *report = new BugReport(*BT_pthreadOnce, os.str(), N);
   report->addRange(CE->getArg(0)->getSourceRange());
   C.EmitReport(report);
 }
@@ -182,13 +183,13 @@ void UnixAPIChecker::CheckMallocZero(CheckerContext &C,
     return;
 
   // Check if the allocation size is 0.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   SVal argVal = state->getSVal(CE->getArg(0));
 
   if (argVal.isUnknownOrUndef())
     return;
   
-  const GRState *trueState, *falseState;
+  const ProgramState *trueState, *falseState;
   llvm::tie(trueState, falseState) = state->assume(cast<DefinedSVal>(argVal));
   
   // Is the value perfectly constrained to zero?
@@ -202,12 +203,12 @@ void UnixAPIChecker::CheckMallocZero(CheckerContext &C,
 
     LazyInitialize(BT_mallocZero, "Undefined allocation of 0 bytes");
     
-    EnhancedBugReport *report =
-      new EnhancedBugReport(*BT_mallocZero, "Call to 'malloc' has an allocation"
+    BugReport *report =
+      new BugReport(*BT_mallocZero, "Call to 'malloc' has an allocation"
                                             " size of 0 bytes", N);
     report->addRange(CE->getArg(0)->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
-                              CE->getArg(0));
+    report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N,
+                                                                CE->getArg(0)));
     C.EmitReport(report);
     return;
   }
@@ -225,7 +226,7 @@ void UnixAPIChecker::CheckMallocZero(CheckerContext &C,
 void UnixAPIChecker::checkPreStmt(const CallExpr *CE, CheckerContext &C) const {
   // Get the callee.  All the functions we care about are C functions
   // with simple identifiers.
-  const GRState *state = C.getState();
+  const ProgramState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   const FunctionDecl *Fn = state->getSVal(Callee).getAsFunctionDecl();
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
index b540bce98170..459ee65d19d9 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
@@ -60,11 +60,12 @@ void UnreachableCodeChecker::checkEndAnalysis(ExplodedGraph &G,
 
   CFG *C = 0;
   ParentMap *PM = 0;
+  const LocationContext *LC = 0;
   // Iterate over ExplodedGraph
   for (ExplodedGraph::node_iterator I = G.nodes_begin(), E = G.nodes_end();
       I != E; ++I) {
     const ProgramPoint &P = I->getLocation();
-    const LocationContext *LC = P.getLocationContext();
+    LC = P.getLocationContext();
 
     // Save the CFG if we don't have it already
     if (!C)
@@ -111,22 +112,30 @@ void UnreachableCodeChecker::checkEndAnalysis(ExplodedGraph &G,
     // FIXME: This should be extended to include other unreachable markers,
     // such as llvm_unreachable.
     if (!CB->empty()) {
-      CFGElement First = CB->front();
-      if (const CFGStmt *S = First.getAs<CFGStmt>()) {
-        if (const CallExpr *CE = dyn_cast<CallExpr>(S->getStmt())) {
-          if (CE->isBuiltinCall(Ctx) == Builtin::BI__builtin_unreachable)
-            continue;
-        }
+      bool foundUnreachable = false;
+      for (CFGBlock::const_iterator ci = CB->begin(), ce = CB->end();
+           ci != ce; ++ci) {
+        if (const CFGStmt *S = (*ci).getAs<CFGStmt>())
+          if (const CallExpr *CE = dyn_cast<CallExpr>(S->getStmt())) {
+            if (CE->isBuiltinCall(Ctx) == Builtin::BI__builtin_unreachable) {
+              foundUnreachable = true;
+              break;
+            }
+          }
       }
+      if (foundUnreachable)
+        continue;
     }
 
     // We found a block that wasn't covered - find the statement to report
     SourceRange SR;
+    PathDiagnosticLocation DL;
     SourceLocation SL;
     if (const Stmt *S = getUnreachableStmt(CB)) {
       SR = S->getSourceRange();
-      SL = S->getLocStart();
-      if (SR.isInvalid() || SL.isInvalid())
+      DL = PathDiagnosticLocation::createBegin(S, B.getSourceManager(), LC);
+      SL = DL.asLocation();
+      if (SR.isInvalid() || !SL.isValid())
         continue;
     }
     else
@@ -138,7 +147,7 @@ void UnreachableCodeChecker::checkEndAnalysis(ExplodedGraph &G,
       continue;
 
     B.EmitBasicReport("Unreachable code", "Dead code", "This statement is never"
-        " executed", SL, SR);
+        " executed", DL, SR);
   }
 }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
index 875dce2e994a..b34b97c5b301 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
@@ -48,8 +48,8 @@ void VLASizeChecker::checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {
     return;
 
   // FIXME: Handle multi-dimensional VLAs.
-  const Expr* SE = VLA->getSizeExpr();
-  const GRState *state = C.getState();
+  const Expr *SE = VLA->getSizeExpr();
+  const ProgramState *state = C.getState();
   SVal sizeV = state->getSVal(SE);
 
   if (sizeV.isUndef()) {
@@ -62,10 +62,10 @@ void VLASizeChecker::checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {
       BT_undef.reset(new BuiltinBug("Declared variable-length array (VLA) "
                                     "uses a garbage value as its size"));
 
-    EnhancedBugReport *report =
-      new EnhancedBugReport(*BT_undef, BT_undef->getName(), N);
+    BugReport *report =
+      new BugReport(*BT_undef, BT_undef->getName(), N);
     report->addRange(SE->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, SE);
+    report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, SE));
     C.EmitReport(report);
     return;
   }
@@ -78,19 +78,19 @@ void VLASizeChecker::checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {
   // Check if the size is zero.
   DefinedSVal sizeD = cast<DefinedSVal>(sizeV);
 
-  const GRState *stateNotZero, *stateZero;
+  const ProgramState *stateNotZero, *stateZero;
   llvm::tie(stateNotZero, stateZero) = state->assume(sizeD);
 
   if (stateZero && !stateNotZero) {
-    ExplodedNode* N = C.generateSink(stateZero);
+    ExplodedNode *N = C.generateSink(stateZero);
     if (!BT_zero)
       BT_zero.reset(new BuiltinBug("Declared variable-length array (VLA) has "
                                    "zero size"));
 
-    EnhancedBugReport *report =
-      new EnhancedBugReport(*BT_zero, BT_zero->getName(), N);
+    BugReport *report =
+      new BugReport(*BT_zero, BT_zero->getName(), N);
     report->addRange(SE->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, SE);
+    report->addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, SE));
     C.EmitReport(report);
     return;
   }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AggExprVisitor.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AggExprVisitor.cpp
index 901190d901dd..0936d61784ac 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AggExprVisitor.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AggExprVisitor.cpp
@@ -46,7 +46,7 @@ class AggExprVisitor : public StmtVisitor<AggExprVisitor> {
 void AggExprVisitor::VisitCastExpr(CastExpr *E) {
   switch (E->getCastKind()) {
   default: 
-    assert(0 && "Unhandled cast kind");
+    llvm_unreachable("Unhandled cast kind");
   case CK_NoOp:
   case CK_ConstructorConversion:
   case CK_UserDefinedConversion:
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp
index 5f4f83c0a8dc..17ec70d39b90 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp
@@ -14,6 +14,58 @@
 using namespace clang;
 using namespace ento;
 
+AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
+                                 const LangOptions &lang,
+                                 PathDiagnosticConsumer *pd,
+                                 StoreManagerCreator storemgr,
+                                 ConstraintManagerCreator constraintmgr, 
+                                 CheckerManager *checkerMgr,
+                                 idx::Indexer *idxer,
+                                 unsigned maxnodes, unsigned maxvisit,
+                                 bool vizdot, bool vizubi,
+                                 AnalysisPurgeMode purge,
+                                 bool eager, bool trim,
+                                 bool inlinecall, bool useUnoptimizedCFG,
+                                 bool addImplicitDtors, bool addInitializers,
+                                 bool eagerlyTrimEGraph)
+  : AnaCtxMgr(useUnoptimizedCFG, addImplicitDtors, addInitializers),
+    Ctx(ctx), Diags(diags), LangInfo(lang), PD(pd),
+    CreateStoreMgr(storemgr), CreateConstraintMgr(constraintmgr),
+    CheckerMgr(checkerMgr), Idxer(idxer),
+    AScope(ScopeDecl), MaxNodes(maxnodes), MaxVisit(maxvisit),
+    VisualizeEGDot(vizdot), VisualizeEGUbi(vizubi), PurgeDead(purge),
+    EagerlyAssume(eager), TrimGraph(trim), InlineCall(inlinecall),
+    EagerlyTrimEGraph(eagerlyTrimEGraph)
+{
+  AnaCtxMgr.getCFGBuildOptions().setAllAlwaysAdd();
+}
+
+AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
+                                 AnalysisManager &ParentAM)
+  : AnaCtxMgr(ParentAM.AnaCtxMgr.getUseUnoptimizedCFG(),
+              ParentAM.AnaCtxMgr.getCFGBuildOptions().AddImplicitDtors,
+              ParentAM.AnaCtxMgr.getCFGBuildOptions().AddInitializers),
+    Ctx(ctx), Diags(diags),
+    LangInfo(ParentAM.LangInfo), PD(ParentAM.getPathDiagnosticConsumer()),
+    CreateStoreMgr(ParentAM.CreateStoreMgr),
+    CreateConstraintMgr(ParentAM.CreateConstraintMgr),
+    CheckerMgr(ParentAM.CheckerMgr),
+    Idxer(ParentAM.Idxer),
+    AScope(ScopeDecl),
+    MaxNodes(ParentAM.MaxNodes),
+    MaxVisit(ParentAM.MaxVisit),
+    VisualizeEGDot(ParentAM.VisualizeEGDot),
+    VisualizeEGUbi(ParentAM.VisualizeEGUbi),
+    PurgeDead(ParentAM.PurgeDead),
+    EagerlyAssume(ParentAM.EagerlyAssume),
+    TrimGraph(ParentAM.TrimGraph),
+    InlineCall(ParentAM.InlineCall),
+    EagerlyTrimEGraph(ParentAM.EagerlyTrimEGraph)
+{
+  AnaCtxMgr.getCFGBuildOptions().setAllAlwaysAdd();
+}
+
+
 AnalysisContext *
 AnalysisManager::getAnalysisContextInAnotherTU(const Decl *D) {
   idx::Entity Ent = idx::Entity::get(const_cast<Decl *>(D), 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
index 3050ca3ce3cc..6c748b6ad01a 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
@@ -8,14 +8,13 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines BasicConstraintManager, a class that tracks simple
-//  equality and inequality constraints on symbolic values of GRState.
+//  equality and inequality constraints on symbolic values of ProgramState.
 //
 //===----------------------------------------------------------------------===//
 
 #include "SimpleConstraintManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
@@ -25,7 +24,7 @@ using namespace ento;
 namespace { class ConstNotEq {}; }
 namespace { class ConstEq {}; }
 
-typedef llvm::ImmutableMap<SymbolRef,GRState::IntSetTy> ConstNotEqTy;
+typedef llvm::ImmutableMap<SymbolRef,ProgramState::IntSetTy> ConstNotEqTy;
 typedef llvm::ImmutableMap<SymbolRef,const llvm::APSInt*> ConstEqTy;
 
 static int ConstEqIndex = 0;
@@ -34,13 +33,14 @@ static int ConstNotEqIndex = 0;
 namespace clang {
 namespace ento {
 template<>
-struct GRStateTrait<ConstNotEq> : public GRStatePartialTrait<ConstNotEqTy> {
-  static inline void* GDMIndex() { return &ConstNotEqIndex; }
+struct ProgramStateTrait<ConstNotEq> :
+  public ProgramStatePartialTrait<ConstNotEqTy> {
+  static inline void *GDMIndex() { return &ConstNotEqIndex; }
 };
 
 template<>
-struct GRStateTrait<ConstEq> : public GRStatePartialTrait<ConstEqTy> {
-  static inline void* GDMIndex() { return &ConstEqIndex; }
+struct ProgramStateTrait<ConstEq> : public ProgramStatePartialTrait<ConstEqTy> {
+  static inline void *GDMIndex() { return &ConstEqIndex; }
 };
 }
 }
@@ -50,62 +50,81 @@ namespace {
 // constants and integer variables.
 class BasicConstraintManager
   : public SimpleConstraintManager {
-  GRState::IntSetTy::Factory ISetFactory;
+  ProgramState::IntSetTy::Factory ISetFactory;
 public:
-  BasicConstraintManager(GRStateManager &statemgr, SubEngine &subengine)
+  BasicConstraintManager(ProgramStateManager &statemgr, SubEngine &subengine)
     : SimpleConstraintManager(subengine), 
       ISetFactory(statemgr.getAllocator()) {}
 
-  const GRState *assumeSymNE(const GRState* state, SymbolRef sym,
-                             const llvm::APSInt& V,
-                             const llvm::APSInt& Adjustment);
+  const ProgramState *assumeSymNE(const ProgramState *state,
+                                  SymbolRef sym,
+                                  const llvm::APSInt& V,
+                                  const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymEQ(const GRState* state, SymbolRef sym,
-                             const llvm::APSInt& V,
-                             const llvm::APSInt& Adjustment);
+  const ProgramState *assumeSymEQ(const ProgramState *state,
+                                  SymbolRef sym,
+                                  const llvm::APSInt& V,
+                                  const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymLT(const GRState* state, SymbolRef sym,
-                             const llvm::APSInt& V,
-                             const llvm::APSInt& Adjustment);
+  const ProgramState *assumeSymLT(const ProgramState *state,
+                                  SymbolRef sym,
+                                  const llvm::APSInt& V,
+                                  const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymGT(const GRState* state, SymbolRef sym,
-                             const llvm::APSInt& V,
-                             const llvm::APSInt& Adjustment);
+  const ProgramState *assumeSymGT(const ProgramState *state,
+                                  SymbolRef sym,
+                                  const llvm::APSInt& V,
+                                  const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymGE(const GRState* state, SymbolRef sym,
-                             const llvm::APSInt& V,
-                             const llvm::APSInt& Adjustment);
+  const ProgramState *assumeSymGE(const ProgramState *state,
+                                  SymbolRef sym,
+                                  const llvm::APSInt& V,
+                                  const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymLE(const GRState* state, SymbolRef sym,
-                             const llvm::APSInt& V,
-                             const llvm::APSInt& Adjustment);
+  const ProgramState *assumeSymLE(const ProgramState *state,
+                                  SymbolRef sym,
+                                  const llvm::APSInt& V,
+                                  const llvm::APSInt& Adjustment);
 
-  const GRState* AddEQ(const GRState* state, SymbolRef sym, const llvm::APSInt& V);
+  const ProgramState *AddEQ(const ProgramState *state,
+                            SymbolRef sym,
+                            const llvm::APSInt& V);
 
-  const GRState* AddNE(const GRState* state, SymbolRef sym, const llvm::APSInt& V);
+  const ProgramState *AddNE(const ProgramState *state,
+                            SymbolRef sym,
+                            const llvm::APSInt& V);
 
-  const llvm::APSInt* getSymVal(const GRState* state, SymbolRef sym) const;
-  bool isNotEqual(const GRState* state, SymbolRef sym, const llvm::APSInt& V)
-      const;
-  bool isEqual(const GRState* state, SymbolRef sym, const llvm::APSInt& V)
-      const;
+  const llvm::APSInt* getSymVal(const ProgramState *state,
+                                SymbolRef sym) const;
 
-  const GRState* removeDeadBindings(const GRState* state, SymbolReaper& SymReaper);
+  bool isNotEqual(const ProgramState *state,
+                  SymbolRef sym,
+                  const llvm::APSInt& V) const;
 
-  void print(const GRState* state, llvm::raw_ostream& Out,
-             const char* nl, const char *sep);
+  bool isEqual(const ProgramState *state,
+               SymbolRef sym,
+               const llvm::APSInt& V) const;
+
+  const ProgramState *removeDeadBindings(const ProgramState *state,
+                                         SymbolReaper& SymReaper);
+
+  void print(const ProgramState *state,
+             raw_ostream &Out,
+             const char* nl,
+             const char *sep);
 };
 
 } // end anonymous namespace
 
-ConstraintManager* ento::CreateBasicConstraintManager(GRStateManager& statemgr,
-                                                       SubEngine &subengine) {
+ConstraintManager*
+ento::CreateBasicConstraintManager(ProgramStateManager& statemgr,
+                                   SubEngine &subengine) {
   return new BasicConstraintManager(statemgr, subengine);
 }
 
-
-const GRState*
-BasicConstraintManager::assumeSymNE(const GRState *state, SymbolRef sym,
+const ProgramState*
+BasicConstraintManager::assumeSymNE(const ProgramState *state,
+                                    SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // First, determine if sym == X, where X+Adjustment != V.
@@ -124,8 +143,9 @@ BasicConstraintManager::assumeSymNE(const GRState *state, SymbolRef sym,
   return AddNE(state, sym, Adjusted);
 }
 
-const GRState*
-BasicConstraintManager::assumeSymEQ(const GRState *state, SymbolRef sym,
+const ProgramState*
+BasicConstraintManager::assumeSymEQ(const ProgramState *state,
+                                    SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // First, determine if sym == X, where X+Adjustment != V.
@@ -145,8 +165,9 @@ BasicConstraintManager::assumeSymEQ(const GRState *state, SymbolRef sym,
 }
 
 // The logic for these will be handled in another ConstraintManager.
-const GRState*
-BasicConstraintManager::assumeSymLT(const GRState *state, SymbolRef sym,
+const ProgramState*
+BasicConstraintManager::assumeSymLT(const ProgramState *state,
+                                    SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Is 'V' the smallest possible value?
@@ -159,8 +180,9 @@ BasicConstraintManager::assumeSymLT(const GRState *state, SymbolRef sym,
   return assumeSymNE(state, sym, V, Adjustment);
 }
 
-const GRState*
-BasicConstraintManager::assumeSymGT(const GRState *state, SymbolRef sym,
+const ProgramState*
+BasicConstraintManager::assumeSymGT(const ProgramState *state,
+                                    SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Is 'V' the largest possible value?
@@ -173,8 +195,9 @@ BasicConstraintManager::assumeSymGT(const GRState *state, SymbolRef sym,
   return assumeSymNE(state, sym, V, Adjustment);
 }
 
-const GRState*
-BasicConstraintManager::assumeSymGE(const GRState *state, SymbolRef sym,
+const ProgramState*
+BasicConstraintManager::assumeSymGE(const ProgramState *state,
+                                    SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Reject a path if the value of sym is a constant X and !(X+Adj >= V).
@@ -201,8 +224,9 @@ BasicConstraintManager::assumeSymGE(const GRState *state, SymbolRef sym,
   return state;
 }
 
-const GRState*
-BasicConstraintManager::assumeSymLE(const GRState *state, SymbolRef sym,
+const ProgramState*
+BasicConstraintManager::assumeSymLE(const ProgramState *state,
+                                    SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Reject a path if the value of sym is a constant X and !(X+Adj <= V).
@@ -229,18 +253,20 @@ BasicConstraintManager::assumeSymLE(const GRState *state, SymbolRef sym,
   return state;
 }
 
-const GRState* BasicConstraintManager::AddEQ(const GRState* state, SymbolRef sym,
+const ProgramState *BasicConstraintManager::AddEQ(const ProgramState *state,
+                                                  SymbolRef sym,
                                              const llvm::APSInt& V) {
   // Create a new state with the old binding replaced.
   return state->set<ConstEq>(sym, &state->getBasicVals().getValue(V));
 }
 
-const GRState* BasicConstraintManager::AddNE(const GRState* state, SymbolRef sym,
-                                             const llvm::APSInt& V) {
+const ProgramState *BasicConstraintManager::AddNE(const ProgramState *state,
+                                                  SymbolRef sym,
+                                                  const llvm::APSInt& V) {
 
   // First, retrieve the NE-set associated with the given symbol.
   ConstNotEqTy::data_type* T = state->get<ConstNotEq>(sym);
-  GRState::IntSetTy S = T ? *T : ISetFactory.getEmptySet();
+  ProgramState::IntSetTy S = T ? *T : ISetFactory.getEmptySet();
 
   // Now add V to the NE set.
   S = ISetFactory.add(S, &state->getBasicVals().getValue(V));
@@ -249,13 +275,14 @@ const GRState* BasicConstraintManager::AddNE(const GRState* state, SymbolRef sym
   return state->set<ConstNotEq>(sym, S);
 }
 
-const llvm::APSInt* BasicConstraintManager::getSymVal(const GRState* state,
+const llvm::APSInt* BasicConstraintManager::getSymVal(const ProgramState *state,
                                                       SymbolRef sym) const {
   const ConstEqTy::data_type* T = state->get<ConstEq>(sym);
   return T ? *T : NULL;
 }
 
-bool BasicConstraintManager::isNotEqual(const GRState* state, SymbolRef sym,
+bool BasicConstraintManager::isNotEqual(const ProgramState *state,
+                                        SymbolRef sym,
                                         const llvm::APSInt& V) const {
 
   // Retrieve the NE-set associated with the given symbol.
@@ -265,7 +292,8 @@ bool BasicConstraintManager::isNotEqual(const GRState* state, SymbolRef sym,
   return T ? T->contains(&state->getBasicVals().getValue(V)) : false;
 }
 
-bool BasicConstraintManager::isEqual(const GRState* state, SymbolRef sym,
+bool BasicConstraintManager::isEqual(const ProgramState *state,
+                                     SymbolRef sym,
                                      const llvm::APSInt& V) const {
   // Retrieve the EQ-set associated with the given symbol.
   const ConstEqTy::data_type* T = state->get<ConstEq>(sym);
@@ -275,8 +303,8 @@ bool BasicConstraintManager::isEqual(const GRState* state, SymbolRef sym,
 
 /// Scan all symbols referenced by the constraints. If the symbol is not alive
 /// as marked in LSymbols, mark it as dead in DSymbols.
-const GRState*
-BasicConstraintManager::removeDeadBindings(const GRState* state,
+const ProgramState*
+BasicConstraintManager::removeDeadBindings(const ProgramState *state,
                                            SymbolReaper& SymReaper) {
 
   ConstEqTy CE = state->get<ConstEq>();
@@ -301,7 +329,8 @@ BasicConstraintManager::removeDeadBindings(const GRState* state,
   return state->set<ConstNotEq>(CNE);
 }
 
-void BasicConstraintManager::print(const GRState* state, llvm::raw_ostream& Out,
+void BasicConstraintManager::print(const ProgramState *state,
+                                   raw_ostream &Out,
                                    const char* nl, const char *sep) {
   // Print equality constraints.
 
@@ -324,7 +353,7 @@ void BasicConstraintManager::print(const GRState* state, llvm::raw_ostream& Out,
       Out << nl << " $" << I.getKey() << " : ";
       bool isFirst = true;
 
-      GRState::IntSetTy::iterator J = I.getData().begin(),
+      ProgramState::IntSetTy::iterator J = I.getData().begin(),
                                   EJ = I.getData().end();
 
       for ( ; J != EJ; ++J) {
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicStore.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicStore.cpp
deleted file mode 100644
index 7c9f45a474c1..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicStore.cpp
+++ /dev/null
@@ -1,605 +0,0 @@
-//== BasicStore.cpp - Basic map from Locations to Values --------*- C++ -*--==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defined the BasicStore and BasicStoreManager classes.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/ExprObjC.h"
-#include "clang/Analysis/Analyses/LiveVariables.h"
-#include "clang/Analysis/AnalysisContext.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "llvm/ADT/ImmutableMap.h"
-
-using namespace clang;
-using namespace ento;
-
-typedef llvm::ImmutableMap<const MemRegion*,SVal> BindingsTy;
-
-namespace {
-
-class BasicStoreSubRegionMap : public SubRegionMap {
-public:
-  BasicStoreSubRegionMap() {}
-
-  bool iterSubRegions(const MemRegion* R, Visitor& V) const {
-    return true; // Do nothing.  No subregions.
-  }
-};
-
-class BasicStoreManager : public StoreManager {
-  BindingsTy::Factory VBFactory;
-public:
-  BasicStoreManager(GRStateManager& mgr)
-    : StoreManager(mgr), VBFactory(mgr.getAllocator()) {}
-
-  ~BasicStoreManager() {}
-
-  SubRegionMap *getSubRegionMap(Store store) {
-    return new BasicStoreSubRegionMap();
-  }
-
-  SVal Retrieve(Store store, Loc loc, QualType T = QualType());
-
-  StoreRef invalidateRegion(Store store, const MemRegion *R, const Expr *E,
-                            unsigned Count, InvalidatedSymbols &IS);
-
-  StoreRef invalidateRegions(Store store, const MemRegion * const *Begin,
-                             const MemRegion * const *End, const Expr *E,
-                             unsigned Count, InvalidatedSymbols &IS,
-                             bool invalidateGlobals,
-                             InvalidatedRegions *Regions);
-
-  StoreRef scanForIvars(Stmt *B, const Decl* SelfDecl,
-                        const MemRegion *SelfRegion, Store St);
-
-  StoreRef Bind(Store St, Loc loc, SVal V);
-  StoreRef Remove(Store St, Loc loc);
-  StoreRef getInitialStore(const LocationContext *InitLoc);
-
-  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr*,
-                            const LocationContext*, SVal val) {
-    return StoreRef(store, *this);
-  }
-
-  /// ArrayToPointer - Used by ExprEngine::VistCast to handle implicit
-  ///  conversions between arrays and pointers.
-  SVal ArrayToPointer(Loc Array) { return Array; }
-
-  /// removeDeadBindings - Scans a BasicStore of 'state' for dead values.
-  ///  It updatees the GRState object in place with the values removed.
-  StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
-                              SymbolReaper& SymReaper,
-                          llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
-
-  void iterBindings(Store store, BindingsHandler& f);
-
-  StoreRef BindDecl(Store store, const VarRegion *VR, SVal InitVal) {
-    return BindDeclInternal(store, VR, &InitVal);
-  }
-
-  StoreRef BindDeclWithNoInit(Store store, const VarRegion *VR) {
-    return BindDeclInternal(store, VR, 0);
-  }
-
-  StoreRef BindDeclInternal(Store store, const VarRegion *VR, SVal *InitVal);
-
-  static inline BindingsTy GetBindings(Store store) {
-    return BindingsTy(static_cast<const BindingsTy::TreeTy*>(store));
-  }
-
-  void print(Store store, llvm::raw_ostream& Out, const char* nl,
-             const char *sep);
-
-private:
-  SVal LazyRetrieve(Store store, const TypedRegion *R);
-};
-
-} // end anonymous namespace
-
-
-StoreManager* ento::CreateBasicStoreManager(GRStateManager& StMgr) {
-  return new BasicStoreManager(StMgr);
-}
-
-static bool isHigherOrderRawPtr(QualType T, ASTContext &C) {
-  bool foundPointer = false;
-  while (1) {
-    const PointerType *PT = T->getAs<PointerType>();
-    if (!PT) {
-      if (!foundPointer)
-        return false;
-
-      // intptr_t* or intptr_t**, etc?
-      if (T->isIntegerType() && C.getTypeSize(T) == C.getTypeSize(C.VoidPtrTy))
-        return true;
-
-      QualType X = C.getCanonicalType(T).getUnqualifiedType();
-      return X == C.VoidTy;
-    }
-
-    foundPointer = true;
-    T = PT->getPointeeType();
-  }
-}
-
-SVal BasicStoreManager::LazyRetrieve(Store store, const TypedRegion *R) {
-  const VarRegion *VR = dyn_cast<VarRegion>(R);
-  if (!VR)
-    return UnknownVal();
-
-  const VarDecl *VD = VR->getDecl();
-  QualType T = VD->getType();
-
-  // Only handle simple types that we can symbolicate.
-  if (!SymbolManager::canSymbolicate(T) || !T->isScalarType())
-    return UnknownVal();
-
-  // Globals and parameters start with symbolic values.
-  // Local variables initially are undefined.
-
-  // Non-static globals may have had their values reset by invalidateRegions.
-  const MemSpaceRegion *MS = VR->getMemorySpace();
-  if (isa<NonStaticGlobalSpaceRegion>(MS)) {
-    BindingsTy B = GetBindings(store);
-    // FIXME: Copy-and-pasted from RegionStore.cpp.
-    if (BindingsTy::data_type *Val = B.lookup(MS)) {
-      if (SymbolRef parentSym = Val->getAsSymbol())
-        return svalBuilder.getDerivedRegionValueSymbolVal(parentSym, R);
-
-      if (Val->isZeroConstant())
-        return svalBuilder.makeZeroVal(T);
-
-      if (Val->isUnknownOrUndef())
-        return *Val;
-
-      assert(0 && "Unknown default value.");
-    }
-  }
-
-  if (VR->hasGlobalsOrParametersStorage() ||
-      isa<UnknownSpaceRegion>(VR->getMemorySpace()))
-    return svalBuilder.getRegionValueSymbolVal(R);
-
-  return UndefinedVal();
-}
-
-SVal BasicStoreManager::Retrieve(Store store, Loc loc, QualType T) {
-  if (isa<UnknownVal>(loc))
-    return UnknownVal();
-
-  assert(!isa<UndefinedVal>(loc));
-
-  switch (loc.getSubKind()) {
-
-    case loc::MemRegionKind: {
-      const MemRegion* R = cast<loc::MemRegionVal>(loc).getRegion();
-
-      if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R) ||
-          isa<CXXThisRegion>(R)))
-        return UnknownVal();
-
-      BindingsTy B = GetBindings(store);
-      BindingsTy::data_type *Val = B.lookup(R);
-      const TypedRegion *TR = cast<TypedRegion>(R);
-
-      if (Val)
-        return CastRetrievedVal(*Val, TR, T);
-
-      SVal V = LazyRetrieve(store, TR);
-      return V.isUnknownOrUndef() ? V : CastRetrievedVal(V, TR, T);
-    }
-
-    case loc::ObjCPropRefKind:
-    case loc::ConcreteIntKind:
-      // Support direct accesses to memory.  It's up to individual checkers
-      // to flag an error.
-      return UnknownVal();
-
-    default:
-      assert (false && "Invalid Loc.");
-      break;
-  }
-
-  return UnknownVal();
-}
-
-StoreRef BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
-  if (isa<loc::ConcreteInt>(loc))
-    return StoreRef(store, *this);
-
-  const MemRegion* R = cast<loc::MemRegionVal>(loc).getRegion();
-
-  // Special case: a default symbol assigned to the NonStaticGlobalsSpaceRegion
-  //  that is used to derive other symbols.
-  if (isa<NonStaticGlobalSpaceRegion>(R)) {
-    BindingsTy B = GetBindings(store);
-    return StoreRef(VBFactory.add(B, R, V).getRoot(), *this);
-  }
-
-  // Special case: handle store of pointer values (Loc) to pointers via
-  // a cast to intXX_t*, void*, etc.  This is needed to handle
-  // OSCompareAndSwap32Barrier/OSCompareAndSwap64Barrier.
-  if (isa<Loc>(V) || isa<nonloc::LocAsInteger>(V))
-    if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
-      // FIXME: Should check for index 0.
-      QualType T = ER->getLocationType();
-
-      if (isHigherOrderRawPtr(T, Ctx))
-        R = ER->getSuperRegion();
-    }
-
-  if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R) || isa<CXXThisRegion>(R)))
-    return StoreRef(store, *this);
-
-  const TypedRegion *TyR = cast<TypedRegion>(R);
-
-  // Do not bind to arrays.  We need to explicitly check for this so that
-  // we do not encounter any weirdness of trying to load/store from arrays.
-  if (TyR->isBoundable() && TyR->getValueType()->isArrayType())
-    return StoreRef(store, *this);
-
-  if (nonloc::LocAsInteger *X = dyn_cast<nonloc::LocAsInteger>(&V)) {
-    // Only convert 'V' to a location iff the underlying region type
-    // is a location as well.
-    // FIXME: We are allowing a store of an arbitrary location to
-    // a pointer.  We may wish to flag a type error here if the types
-    // are incompatible.  This may also cause lots of breakage
-    // elsewhere. Food for thought.
-    if (TyR->isBoundable() && Loc::isLocType(TyR->getValueType()))
-      V = X->getLoc();
-  }
-
-  BindingsTy B = GetBindings(store);
-  return StoreRef(V.isUnknown()
-                    ? VBFactory.remove(B, R).getRoot()
-                    : VBFactory.add(B, R, V).getRoot(), *this);
-}
-
-StoreRef BasicStoreManager::Remove(Store store, Loc loc) {
-  switch (loc.getSubKind()) {
-    case loc::MemRegionKind: {
-      const MemRegion* R = cast<loc::MemRegionVal>(loc).getRegion();
-
-      if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R) ||
-          isa<CXXThisRegion>(R)))
-        return StoreRef(store, *this);
-
-      return StoreRef(VBFactory.remove(GetBindings(store), R).getRoot(), *this);
-    }
-    default:
-      assert ("Remove for given Loc type not yet implemented.");
-      return StoreRef(store, *this);
-  }
-}
-
-StoreRef BasicStoreManager::removeDeadBindings(Store store,
-                                               const StackFrameContext *LCtx,
-                                               SymbolReaper& SymReaper,
-                           llvm::SmallVectorImpl<const MemRegion*>& RegionRoots)
-{
-  BindingsTy B = GetBindings(store);
-  typedef SVal::symbol_iterator symbol_iterator;
-
-  // Iterate over the variable bindings.
-  for (BindingsTy::iterator I=B.begin(), E=B.end(); I!=E ; ++I) {
-    if (const VarRegion *VR = dyn_cast<VarRegion>(I.getKey())) {
-      if (SymReaper.isLive(VR))
-        RegionRoots.push_back(VR);
-      else
-        continue;
-    }
-    else if (isa<ObjCIvarRegion>(I.getKey()) ||
-             isa<NonStaticGlobalSpaceRegion>(I.getKey()) ||
-             isa<CXXThisRegion>(I.getKey()))
-      RegionRoots.push_back(I.getKey());
-    else
-      continue;
-
-    // Mark the bindings in the data as live.
-    SVal X = I.getData();
-    for (symbol_iterator SI=X.symbol_begin(), SE=X.symbol_end(); SI!=SE; ++SI)
-      SymReaper.markLive(*SI);
-  }
-
-  // Scan for live variables and live symbols.
-  llvm::SmallPtrSet<const MemRegion*, 10> Marked;
-
-  while (!RegionRoots.empty()) {
-    const MemRegion* MR = RegionRoots.back();
-    RegionRoots.pop_back();
-
-    while (MR) {
-      if (const SymbolicRegion* SymR = dyn_cast<SymbolicRegion>(MR)) {
-        SymReaper.markLive(SymR->getSymbol());
-        break;
-      }
-      else if (isa<VarRegion>(MR) || isa<ObjCIvarRegion>(MR) ||
-               isa<NonStaticGlobalSpaceRegion>(MR) || isa<CXXThisRegion>(MR)) {
-        if (Marked.count(MR))
-          break;
-
-        Marked.insert(MR);
-        SVal X = Retrieve(store, loc::MemRegionVal(MR));
-
-        // FIXME: We need to handle symbols nested in region definitions.
-        for (symbol_iterator SI=X.symbol_begin(),SE=X.symbol_end();SI!=SE;++SI)
-          SymReaper.markLive(*SI);
-
-        if (!isa<loc::MemRegionVal>(X))
-          break;
-
-        const loc::MemRegionVal& LVD = cast<loc::MemRegionVal>(X);
-        RegionRoots.push_back(LVD.getRegion());
-        break;
-      }
-      else if (const SubRegion* R = dyn_cast<SubRegion>(MR))
-        MR = R->getSuperRegion();
-      else
-        break;
-    }
-  }
-
-  // Remove dead variable bindings.
-  StoreRef newStore(store, *this);
-  for (BindingsTy::iterator I=B.begin(), E=B.end(); I!=E ; ++I) {
-    const MemRegion* R = I.getKey();
-
-    if (!Marked.count(R)) {
-      newStore = Remove(newStore.getStore(), svalBuilder.makeLoc(R));
-      SVal X = I.getData();
-
-      for (symbol_iterator SI=X.symbol_begin(), SE=X.symbol_end(); SI!=SE; ++SI)
-        SymReaper.maybeDead(*SI);
-    }
-  }
-
-  return newStore;
-}
-
-StoreRef BasicStoreManager::scanForIvars(Stmt *B, const Decl* SelfDecl,
-                                         const MemRegion *SelfRegion,
-                                         Store St) {
-  
-  StoreRef newStore(St, *this);
-
-  for (Stmt::child_iterator CI=B->child_begin(), CE=B->child_end();
-       CI != CE; ++CI) {
-
-    if (!*CI)
-      continue;
-
-    // Check if the statement is an ivar reference.  We only
-    // care about self.ivar.
-    if (ObjCIvarRefExpr *IV = dyn_cast<ObjCIvarRefExpr>(*CI)) {
-      const Expr *Base = IV->getBase()->IgnoreParenCasts();
-      if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Base)) {
-        if (DR->getDecl() == SelfDecl) {
-          const ObjCIvarRegion *IVR = MRMgr.getObjCIvarRegion(IV->getDecl(),
-                                                         SelfRegion);
-          SVal X = svalBuilder.getRegionValueSymbolVal(IVR);
-          newStore = Bind(newStore.getStore(), svalBuilder.makeLoc(IVR), X);
-        }
-      }
-    }
-    else
-      newStore = scanForIvars(*CI, SelfDecl, SelfRegion, newStore.getStore());
-  }
-
-  return newStore;
-}
-
-StoreRef BasicStoreManager::getInitialStore(const LocationContext *InitLoc) {
-  // The LiveVariables information already has a compilation of all VarDecls
-  // used in the function.  Iterate through this set, and "symbolicate"
-  // any VarDecl whose value originally comes from outside the function.
-  typedef LiveVariables::AnalysisDataTy LVDataTy;
-  LVDataTy& D = InitLoc->getLiveVariables()->getAnalysisData();
-  StoreRef St(VBFactory.getEmptyMap().getRoot(), *this);
-
-  for (LVDataTy::decl_iterator I=D.begin_decl(), E=D.end_decl(); I != E; ++I) {
-    const NamedDecl* ND = I->first;
-
-    // Handle implicit parameters.
-    if (const ImplicitParamDecl* PD = dyn_cast<ImplicitParamDecl>(ND)) {
-      const Decl& CD = *InitLoc->getDecl();
-      if (const ObjCMethodDecl* MD = dyn_cast<ObjCMethodDecl>(&CD)) {
-        if (MD->getSelfDecl() == PD) {
-          // FIXME: Add type constraints (when they become available) to
-          // SelfRegion?  (i.e., it implements MD->getClassInterface()).
-          const VarRegion *VR = MRMgr.getVarRegion(PD, InitLoc);
-          const MemRegion *SelfRegion =
-            svalBuilder.getRegionValueSymbolVal(VR).getAsRegion();
-          assert(SelfRegion);
-          St = Bind(St.getStore(), svalBuilder.makeLoc(VR),
-                    loc::MemRegionVal(SelfRegion));
-          // Scan the method for ivar references.  While this requires an
-          // entire AST scan, the cost should not be high in practice.
-          St = scanForIvars(MD->getBody(), PD, SelfRegion, St.getStore());
-        }
-      }
-    }
-  }
-
-  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(InitLoc->getDecl())) {
-    // For C++ non-static member variables, add a symbolic region for 'this' in
-    // the initial stack frame.
-    if (MD->isInstance()) {
-      QualType ThisT = MD->getThisType(StateMgr.getContext());
-      MemRegionManager &RegMgr = svalBuilder.getRegionManager();
-      const CXXThisRegion *ThisR = RegMgr.getCXXThisRegion(ThisT, InitLoc);
-      SVal ThisV = svalBuilder.getRegionValueSymbolVal(ThisR);
-      St = Bind(St.getStore(), svalBuilder.makeLoc(ThisR), ThisV);
-    }
-  }
-
-  return St;
-}
-
-StoreRef BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
-                                             SVal* InitVal) {
-
-  BasicValueFactory& BasicVals = StateMgr.getBasicVals();
-  const VarDecl *VD = VR->getDecl();
-  StoreRef newStore(store, *this);
-
-  // BasicStore does not model arrays and structs.
-  if (VD->getType()->isArrayType() || VD->getType()->isStructureOrClassType())
-    return newStore;
-  
-  if (VD->hasGlobalStorage()) {
-    // Handle variables with global storage: extern, static, PrivateExtern.
-
-    // FIXME:: static variables may have an initializer, but the second time a
-    // function is called those values may not be current. Currently, a function
-    // will not be called more than once.
-
-    // Static global variables should not be visited here.
-    assert(!(VD->getStorageClass() == SC_Static &&
-             VD->isFileVarDecl()));
-
-    // Process static variables.
-    if (VD->getStorageClass() == SC_Static) {
-      // C99: 6.7.8 Initialization
-      //  If an object that has static storage duration is not initialized
-      //  explicitly, then:
-      //   -if it has pointer type, it is initialized to a null pointer;
-      //   -if it has arithmetic type, it is initialized to (positive or
-      //     unsigned) zero;
-      if (!InitVal) {
-        QualType T = VD->getType();
-        if (Loc::isLocType(T))
-          newStore = Bind(store, loc::MemRegionVal(VR),
-                          loc::ConcreteInt(BasicVals.getValue(0, T)));
-        else if (T->isIntegerType() && T->isScalarType())
-          newStore = Bind(store, loc::MemRegionVal(VR),
-                          nonloc::ConcreteInt(BasicVals.getValue(0, T)));
-      } else {
-          newStore = Bind(store, loc::MemRegionVal(VR), *InitVal);
-      }
-    }
-  } else {
-    // Process local scalar variables.
-    QualType T = VD->getType();
-    // BasicStore only supports scalars.
-    if ((T->isScalarType() || T->isReferenceType()) &&
-        svalBuilder.getSymbolManager().canSymbolicate(T)) {
-      SVal V = InitVal ? *InitVal : UndefinedVal();
-      newStore = Bind(store, loc::MemRegionVal(VR), V);
-    }
-  }
-
-  return newStore;
-}
-
-void BasicStoreManager::print(Store store, llvm::raw_ostream& Out,
-                              const char* nl, const char *sep) {
-
-  BindingsTy B = GetBindings(store);
-  Out << "Variables:" << nl;
-
-  bool isFirst = true;
-
-  for (BindingsTy::iterator I=B.begin(), E=B.end(); I != E; ++I) {
-    if (isFirst)
-      isFirst = false;
-    else
-      Out << nl;
-
-    Out << ' ' << I.getKey() << " : " << I.getData();
-  }
-}
-
-
-void BasicStoreManager::iterBindings(Store store, BindingsHandler& f) {
-  BindingsTy B = GetBindings(store);
-
-  for (BindingsTy::iterator I=B.begin(), E=B.end(); I != E; ++I)
-    if (!f.HandleBinding(*this, store, I.getKey(), I.getData()))
-      return;
-
-}
-
-StoreManager::BindingsHandler::~BindingsHandler() {}
-
-//===----------------------------------------------------------------------===//
-// Binding invalidation.
-//===----------------------------------------------------------------------===//
-
-
-StoreRef BasicStoreManager::invalidateRegions(Store store,
-                                              const MemRegion * const *I,
-                                              const MemRegion * const *End,
-                                              const Expr *E, unsigned Count,
-                                              InvalidatedSymbols &IS,
-                                              bool invalidateGlobals,
-                                              InvalidatedRegions *Regions) {
-  StoreRef newStore(store, *this);
-  
-  if (invalidateGlobals) {
-    BindingsTy B = GetBindings(store);
-    for (BindingsTy::iterator I=B.begin(), End=B.end(); I != End; ++I) {
-      const MemRegion *R = I.getKey();
-      if (isa<NonStaticGlobalSpaceRegion>(R->getMemorySpace()))
-        newStore = invalidateRegion(newStore.getStore(), R, E, Count, IS);
-    }
-  }
-
-  for ( ; I != End ; ++I) {
-    const MemRegion *R = *I;
-    // Don't invalidate globals twice.
-    if (invalidateGlobals) {
-      if (isa<NonStaticGlobalSpaceRegion>(R->getMemorySpace()))
-        continue;
-    }
-    newStore = invalidateRegion(newStore.getStore(), *I, E, Count, IS);
-    if (Regions)
-      Regions->push_back(R);
-  }
-
-  // FIXME: This is copy-and-paste from RegionStore.cpp.
-  if (invalidateGlobals) {
-    // Bind the non-static globals memory space to a new symbol that we will
-    // use to derive the bindings for all non-static globals.
-    const GlobalsSpaceRegion *GS = MRMgr.getGlobalsRegion();
-    SVal V =
-      svalBuilder.getConjuredSymbolVal(/* SymbolTag = */ (void*) GS, E,
-                                  /* symbol type, doesn't matter */ Ctx.IntTy,
-                                  Count);
-
-    newStore = Bind(newStore.getStore(), loc::MemRegionVal(GS), V);
-    if (Regions)
-      Regions->push_back(GS);
-  }
-
-  return newStore;
-}
-
-
-StoreRef BasicStoreManager::invalidateRegion(Store store,
-                                             const MemRegion *R,
-                                             const Expr *E,
-                                             unsigned Count,
-                                             InvalidatedSymbols &IS) {
-  R = R->StripCasts();
-
-  if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R)))
-      return StoreRef(store, *this);
-
-  BindingsTy B = GetBindings(store);
-  if (BindingsTy::data_type *Val = B.lookup(R)) {
-    if (SymbolRef Sym = Val->getAsSymbol())
-      IS.insert(Sym);
-  }
-
-  QualType T = cast<TypedRegion>(R)->getValueType();
-  SVal V = svalBuilder.getConjuredSymbolVal(R, E, T, Count);
-  return Bind(store, loc::MemRegionVal(R), V);
-}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
index 0ed4ff143171..fe96700772d6 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
@@ -27,7 +27,7 @@ void CompoundValData::Profile(llvm::FoldingSetNodeID& ID, QualType T,
 
 void LazyCompoundValData::Profile(llvm::FoldingSetNodeID& ID,
                                   const StoreRef &store,
-                                  const TypedRegion *region) {
+                                  const TypedValueRegion *region) {
   ID.AddPointer(store.getStore());
   ID.AddPointer(region);
 }
@@ -70,7 +70,7 @@ BasicValueFactory::~BasicValueFactory() {
 
 const llvm::APSInt& BasicValueFactory::getValue(const llvm::APSInt& X) {
   llvm::FoldingSetNodeID ID;
-  void* InsertPos;
+  void *InsertPos;
   typedef llvm::FoldingSetNodeWrapper<llvm::APSInt> FoldNodeTy;
 
   X.Profile(ID);
@@ -113,7 +113,7 @@ BasicValueFactory::getCompoundValData(QualType T,
 
   llvm::FoldingSetNodeID ID;
   CompoundValData::Profile(ID, T, Vals);
-  void* InsertPos;
+  void *InsertPos;
 
   CompoundValData* D = CompoundValDataSet.FindNodeOrInsertPos(ID, InsertPos);
 
@@ -128,10 +128,10 @@ BasicValueFactory::getCompoundValData(QualType T,
 
 const LazyCompoundValData*
 BasicValueFactory::getLazyCompoundValData(const StoreRef &store,
-                                          const TypedRegion *region) {
+                                          const TypedValueRegion *region) {
   llvm::FoldingSetNodeID ID;
   LazyCompoundValData::Profile(ID, store, region);
-  void* InsertPos;
+  void *InsertPos;
 
   LazyCompoundValData *D =
     LazyCompoundValDataSet.FindNodeOrInsertPos(ID, InsertPos);
@@ -243,7 +243,7 @@ BasicValueFactory::getPersistentSValWithData(const SVal& V, uintptr_t Data) {
   if (!PersistentSVals) PersistentSVals = new PersistentSValsTy();
 
   llvm::FoldingSetNodeID ID;
-  void* InsertPos;
+  void *InsertPos;
   V.Profile(ID);
   ID.AddPointer((void*) Data);
 
@@ -268,7 +268,7 @@ BasicValueFactory::getPersistentSValPair(const SVal& V1, const SVal& V2) {
   if (!PersistentSValPairs) PersistentSValPairs = new PersistentSValPairsTy();
 
   llvm::FoldingSetNodeID ID;
-  void* InsertPos;
+  void *InsertPos;
   V1.Profile(ID);
   V2.Profile(ID);
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BlockCounter.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BlockCounter.cpp
index ed52b6b012b7..74d761e1ecc1 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BlockCounter.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BlockCounter.cpp
@@ -48,11 +48,11 @@ class CountKey {
 
 typedef llvm::ImmutableMap<CountKey, unsigned> CountMap;
 
-static inline CountMap GetMap(void* D) {
+static inline CountMap GetMap(void *D) {
   return CountMap(static_cast<CountMap::TreeTy*>(D));
 }
 
-static inline CountMap::Factory& GetFactory(void* F) {
+static inline CountMap::Factory& GetFactory(void *F) {
   return *static_cast<CountMap::Factory*>(F);
 }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 8b5d383ed07f..fbbdb040e4f2 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -33,52 +33,31 @@ using namespace clang;
 using namespace ento;
 
 BugReporterVisitor::~BugReporterVisitor() {}
-BugReporterContext::~BugReporterContext() {
-  for (visitor_iterator I = visitor_begin(), E = visitor_end(); I != E; ++I)
-    if ((*I)->isOwnedByReporterContext()) delete *I;
-}
-
-void BugReporterContext::addVisitor(BugReporterVisitor* visitor) {
-  if (!visitor)
-    return;
-
-  llvm::FoldingSetNodeID ID;
-  visitor->Profile(ID);
-  void *InsertPos;
-
-  if (CallbacksSet.FindNodeOrInsertPos(ID, InsertPos)) {
-    delete visitor;
-    return;
-  }
-
-  CallbacksSet.InsertNode(visitor, InsertPos);
-  Callbacks = F.add(visitor, Callbacks);
-}
 
 //===----------------------------------------------------------------------===//
 // Helper routines for walking the ExplodedGraph and fetching statements.
 //===----------------------------------------------------------------------===//
 
-static inline const Stmt* GetStmt(const ProgramPoint &P) {
+static inline const Stmt *GetStmt(const ProgramPoint &P) {
   if (const StmtPoint* SP = dyn_cast<StmtPoint>(&P))
     return SP->getStmt();
-  else if (const BlockEdge* BE = dyn_cast<BlockEdge>(&P))
+  else if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P))
     return BE->getSrc()->getTerminator();
 
   return 0;
 }
 
 static inline const ExplodedNode*
-GetPredecessorNode(const ExplodedNode* N) {
+GetPredecessorNode(const ExplodedNode *N) {
   return N->pred_empty() ? NULL : *(N->pred_begin());
 }
 
 static inline const ExplodedNode*
-GetSuccessorNode(const ExplodedNode* N) {
+GetSuccessorNode(const ExplodedNode *N) {
   return N->succ_empty() ? NULL : *(N->succ_begin());
 }
 
-static const Stmt* GetPreviousStmt(const ExplodedNode* N) {
+static const Stmt *GetPreviousStmt(const ExplodedNode *N) {
   for (N = GetPredecessorNode(N); N; N = GetPredecessorNode(N))
     if (const Stmt *S = GetStmt(N->getLocation()))
       return S;
@@ -86,7 +65,7 @@ static const Stmt* GetPreviousStmt(const ExplodedNode* N) {
   return 0;
 }
 
-static const Stmt* GetNextStmt(const ExplodedNode* N) {
+static const Stmt *GetNextStmt(const ExplodedNode *N) {
   for (N = GetSuccessorNode(N); N; N = GetSuccessorNode(N))
     if (const Stmt *S = GetStmt(N->getLocation())) {
       // Check if the statement is '?' or '&&'/'||'.  These are "merges",
@@ -104,11 +83,6 @@ static const Stmt* GetNextStmt(const ExplodedNode* N) {
         default:
           break;
       }
-
-      // Some expressions don't have locations.
-      if (S->getLocStart().isInvalid())
-        continue;
-
       return S;
     }
 
@@ -116,7 +90,7 @@ static const Stmt* GetNextStmt(const ExplodedNode* N) {
 }
 
 static inline const Stmt*
-GetCurrentOrPreviousStmt(const ExplodedNode* N) {
+GetCurrentOrPreviousStmt(const ExplodedNode *N) {
   if (const Stmt *S = GetStmt(N->getLocation()))
     return S;
 
@@ -124,7 +98,7 @@ GetCurrentOrPreviousStmt(const ExplodedNode* N) {
 }
 
 static inline const Stmt*
-GetCurrentOrNextStmt(const ExplodedNode* N) {
+GetCurrentOrNextStmt(const ExplodedNode *N) {
   if (const Stmt *S = GetStmt(N->getLocation()))
     return S;
 
@@ -145,7 +119,7 @@ class NodeMapClosure : public BugReport::NodeResolver {
   NodeMapClosure(NodeBackMap *m) : M(*m) {}
   ~NodeMapClosure() {}
 
-  const ExplodedNode* getOriginalNode(const ExplodedNode* N) {
+  const ExplodedNode *getOriginalNode(const ExplodedNode *N) {
     NodeBackMap::iterator I = M.find(N);
     return I == M.end() ? 0 : I->second;
   }
@@ -153,25 +127,29 @@ class NodeMapClosure : public BugReport::NodeResolver {
 
 class PathDiagnosticBuilder : public BugReporterContext {
   BugReport *R;
-  PathDiagnosticClient *PDC;
+  PathDiagnosticConsumer *PDC;
   llvm::OwningPtr<ParentMap> PM;
   NodeMapClosure NMC;
 public:
   PathDiagnosticBuilder(GRBugReporter &br,
                         BugReport *r, NodeBackMap *Backmap,
-                        PathDiagnosticClient *pdc)
+                        PathDiagnosticConsumer *pdc)
     : BugReporterContext(br),
-      R(r), PDC(pdc), NMC(Backmap) {
-    addVisitor(R);
-  }
+      R(r), PDC(pdc), NMC(Backmap) {}
 
-  PathDiagnosticLocation ExecutionContinues(const ExplodedNode* N);
+  PathDiagnosticLocation ExecutionContinues(const ExplodedNode *N);
 
-  PathDiagnosticLocation ExecutionContinues(llvm::raw_string_ostream& os,
-                                            const ExplodedNode* N);
+  PathDiagnosticLocation ExecutionContinues(llvm::raw_string_ostream &os,
+                                            const ExplodedNode *N);
+
+  BugReport *getBugReport() { return R; }
 
   Decl const &getCodeDecl() { return R->getErrorNode()->getCodeDecl(); }
 
+  const LocationContext* getLocationContext() {
+    return R->getErrorNode()->getLocationContext();
+  }
+
   ParentMap& getParentMap() { return R->getErrorNode()->getParentMap(); }
 
   const Stmt *getParent(const Stmt *S) {
@@ -182,8 +160,8 @@ class PathDiagnosticBuilder : public BugReporterContext {
 
   PathDiagnosticLocation getEnclosingStmtLocation(const Stmt *S);
 
-  PathDiagnosticClient::PathGenerationScheme getGenerationScheme() const {
-    return PDC ? PDC->getGenerationScheme() : PathDiagnosticClient::Extensive;
+  PathDiagnosticConsumer::PathGenerationScheme getGenerationScheme() const {
+    return PDC ? PDC->getGenerationScheme() : PathDiagnosticConsumer::Extensive;
   }
 
   bool supportsLogicalOpControlFlow() const {
@@ -193,17 +171,17 @@ class PathDiagnosticBuilder : public BugReporterContext {
 } // end anonymous namespace
 
 PathDiagnosticLocation
-PathDiagnosticBuilder::ExecutionContinues(const ExplodedNode* N) {
+PathDiagnosticBuilder::ExecutionContinues(const ExplodedNode *N) {
   if (const Stmt *S = GetNextStmt(N))
-    return PathDiagnosticLocation(S, getSourceManager());
+    return PathDiagnosticLocation(S, getSourceManager(), getLocationContext());
 
-  return FullSourceLoc(N->getLocationContext()->getDecl()->getBodyRBrace(),
-                       getSourceManager());
+  return PathDiagnosticLocation::createDeclEnd(N->getLocationContext(),
+                                               getSourceManager());
 }
 
 PathDiagnosticLocation
-PathDiagnosticBuilder::ExecutionContinues(llvm::raw_string_ostream& os,
-                                          const ExplodedNode* N) {
+PathDiagnosticBuilder::ExecutionContinues(llvm::raw_string_ostream &os,
+                                          const ExplodedNode *N) {
 
   // Slow, but probably doesn't matter.
   if (os.str().empty())
@@ -213,7 +191,7 @@ PathDiagnosticBuilder::ExecutionContinues(llvm::raw_string_ostream& os,
 
   if (Loc.asStmt())
     os << "Execution continues on line "
-       << getSourceManager().getInstantiationLineNumber(Loc.asLocation())
+       << getSourceManager().getExpansionLineNumber(Loc.asLocation())
        << '.';
   else {
     os << "Execution jumps to the end of the ";
@@ -253,9 +231,10 @@ static bool IsNested(const Stmt *S, ParentMap &PM) {
 
 PathDiagnosticLocation
 PathDiagnosticBuilder::getEnclosingStmtLocation(const Stmt *S) {
-  assert(S && "Null Stmt* passed to getEnclosingStmtLocation");
+  assert(S && "Null Stmt *passed to getEnclosingStmtLocation");
   ParentMap &P = getParentMap();
   SourceManager &SMgr = getSourceManager();
+  const LocationContext *LC = getLocationContext();
 
   while (IsNested(S, P)) {
     const Stmt *Parent = P.getParentIgnoreParens(S);
@@ -267,44 +246,44 @@ PathDiagnosticBuilder::getEnclosingStmtLocation(const Stmt *S) {
       case Stmt::BinaryOperatorClass: {
         const BinaryOperator *B = cast<BinaryOperator>(Parent);
         if (B->isLogicalOp())
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
         break;
       }
       case Stmt::CompoundStmtClass:
       case Stmt::StmtExprClass:
-        return PathDiagnosticLocation(S, SMgr);
+        return PathDiagnosticLocation(S, SMgr, LC);
       case Stmt::ChooseExprClass:
         // Similar to '?' if we are referring to condition, just have the edge
         // point to the entire choose expression.
         if (cast<ChooseExpr>(Parent)->getCond() == S)
-          return PathDiagnosticLocation(Parent, SMgr);
+          return PathDiagnosticLocation(Parent, SMgr, LC);
         else
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
       case Stmt::BinaryConditionalOperatorClass:
       case Stmt::ConditionalOperatorClass:
         // For '?', if we are referring to condition, just have the edge point
         // to the entire '?' expression.
         if (cast<AbstractConditionalOperator>(Parent)->getCond() == S)
-          return PathDiagnosticLocation(Parent, SMgr);
+          return PathDiagnosticLocation(Parent, SMgr, LC);
         else
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
       case Stmt::DoStmtClass:
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
       case Stmt::ForStmtClass:
         if (cast<ForStmt>(Parent)->getBody() == S)
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
         break;
       case Stmt::IfStmtClass:
         if (cast<IfStmt>(Parent)->getCond() != S)
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
         break;
       case Stmt::ObjCForCollectionStmtClass:
         if (cast<ObjCForCollectionStmt>(Parent)->getBody() == S)
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
         break;
       case Stmt::WhileStmtClass:
         if (cast<WhileStmt>(Parent)->getCond() != S)
-          return PathDiagnosticLocation(S, SMgr);
+          return PathDiagnosticLocation(S, SMgr, LC);
         break;
       default:
         break;
@@ -322,7 +301,7 @@ PathDiagnosticBuilder::getEnclosingStmtLocation(const Stmt *S) {
       switch (Parent->getStmtClass()) {
         case Stmt::ForStmtClass:
         case Stmt::ObjCForCollectionStmtClass:
-          return PathDiagnosticLocation(Parent, SMgr);
+          return PathDiagnosticLocation(Parent, SMgr, LC);
         default:
           break;
       }
@@ -335,20 +314,20 @@ PathDiagnosticBuilder::getEnclosingStmtLocation(const Stmt *S) {
     if (const ForStmt *FS =
           dyn_cast_or_null<ForStmt>(P.getParentIgnoreParens(S))) {
       if (FS->getInit() == S)
-        return PathDiagnosticLocation(FS, SMgr);
+        return PathDiagnosticLocation(FS, SMgr, LC);
     }
   }
 
-  return PathDiagnosticLocation(S, SMgr);
+  return PathDiagnosticLocation(S, SMgr, LC);
 }
 
 //===----------------------------------------------------------------------===//
 // ScanNotableSymbols: closure-like callback for scanning Store bindings.
 //===----------------------------------------------------------------------===//
 
-static const VarDecl*
-GetMostRecentVarDeclBinding(const ExplodedNode* N,
-                            GRStateManager& VMgr, SVal X) {
+static const VarDecl* GetMostRecentVarDeclBinding(const ExplodedNode *N,
+                                                  ProgramStateManager& VMgr,
+                                                  SVal X) {
 
   for ( ; N ; N = N->pred_empty() ? 0 : *N->pred_begin()) {
 
@@ -357,7 +336,7 @@ GetMostRecentVarDeclBinding(const ExplodedNode* N,
     if (!isa<PostStmt>(P))
       continue;
 
-    const DeclRefExpr* DR = dyn_cast<DeclRefExpr>(cast<PostStmt>(P).getStmt());
+    const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(cast<PostStmt>(P).getStmt());
 
     if (!DR)
       continue;
@@ -367,7 +346,7 @@ GetMostRecentVarDeclBinding(const ExplodedNode* N,
     if (X != Y)
       continue;
 
-    const VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl());
+    const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
 
     if (!VD)
       continue;
@@ -383,19 +362,29 @@ class NotableSymbolHandler
 : public StoreManager::BindingsHandler {
 
   SymbolRef Sym;
-  const GRState* PrevSt;
-  const Stmt* S;
-  GRStateManager& VMgr;
-  const ExplodedNode* Pred;
+  const ProgramState *PrevSt;
+  const Stmt *S;
+  ProgramStateManager& VMgr;
+  const ExplodedNode *Pred;
   PathDiagnostic& PD;
   BugReporter& BR;
 
 public:
 
-  NotableSymbolHandler(SymbolRef sym, const GRState* prevst, const Stmt* s,
-                       GRStateManager& vmgr, const ExplodedNode* pred,
-                       PathDiagnostic& pd, BugReporter& br)
-  : Sym(sym), PrevSt(prevst), S(s), VMgr(vmgr), Pred(pred), PD(pd), BR(br) {}
+  NotableSymbolHandler(SymbolRef sym,
+                       const ProgramState *prevst,
+                       const Stmt *s,
+                       ProgramStateManager& vmgr,
+                       const ExplodedNode *pred,
+                       PathDiagnostic& pd,
+                       BugReporter& br)
+  : Sym(sym),
+    PrevSt(prevst),
+    S(s),
+    VMgr(vmgr),
+    Pred(pred),
+    PD(pd),
+    BR(br) {}
 
   bool HandleBinding(StoreManager& SMgr, Store store, const MemRegion* R,
                      SVal V) {
@@ -422,14 +411,14 @@ class NotableSymbolHandler
         return true;
 
       // What variable did we assign to?
-      DeclRefExpr* DR = dyn_cast<DeclRefExpr>(B->getLHS()->IgnoreParenCasts());
+      DeclRefExpr *DR = dyn_cast<DeclRefExpr>(B->getLHS()->IgnoreParenCasts());
 
       if (!DR)
         return true;
 
       VD = dyn_cast<VarDecl>(DR->getDecl());
     }
-    else if (const DeclStmt* DS = dyn_cast<DeclStmt>(S)) {
+    else if (const DeclStmt *DS = dyn_cast<DeclStmt>(S)) {
       // FIXME: Eventually CFGs won't have DeclStmts.  Right now we
       //  assume that each DeclStmt has a single Decl.  This invariant
       //  holds by construction in the CFG.
@@ -440,19 +429,20 @@ class NotableSymbolHandler
       return true;
 
     // What is the most recently referenced variable with this binding?
-    const VarDecl* MostRecent = GetMostRecentVarDeclBinding(Pred, VMgr, V);
+    const VarDecl *MostRecent = GetMostRecentVarDeclBinding(Pred, VMgr, V);
 
     if (!MostRecent)
       return true;
 
     // Create the diagnostic.
-    FullSourceLoc L(S->getLocStart(), BR.getSourceManager());
-
     if (Loc::isLocType(VD->getType())) {
-      std::string msg = "'" + std::string(VD->getNameAsString()) +
-      "' now aliases '" + MostRecent->getNameAsString() + "'";
-
-      PD.push_front(new PathDiagnosticEventPiece(L, msg));
+      llvm::SmallString<64> buf;
+      llvm::raw_svector_ostream os(buf);
+      os << '\'' << *VD << "' now aliases '" << *MostRecent << '\'';
+      PathDiagnosticLocation L =
+        PathDiagnosticLocation::createBegin(S, BR.getSourceManager(),
+                                                   Pred->getLocationContext());
+      PD.push_front(new PathDiagnosticEventPiece(L, os.str()));
     }
 
     return true;
@@ -460,20 +450,20 @@ class NotableSymbolHandler
 };
 }
 
-static void HandleNotableSymbol(const ExplodedNode* N,
-                                const Stmt* S,
+static void HandleNotableSymbol(const ExplodedNode *N,
+                                const Stmt *S,
                                 SymbolRef Sym, BugReporter& BR,
                                 PathDiagnostic& PD) {
 
-  const ExplodedNode* Pred = N->pred_empty() ? 0 : *N->pred_begin();
-  const GRState* PrevSt = Pred ? Pred->getState() : 0;
+  const ExplodedNode *Pred = N->pred_empty() ? 0 : *N->pred_begin();
+  const ProgramState *PrevSt = Pred ? Pred->getState() : 0;
 
   if (!PrevSt)
     return;
 
   // Look at the region bindings of the current state that map to the
   // specified symbol.  Are any of them not in the previous state?
-  GRStateManager& VMgr = cast<GRBugReporter>(BR).getStateManager();
+  ProgramStateManager& VMgr = cast<GRBugReporter>(BR).getStateManager();
   NotableSymbolHandler H(Sym, PrevSt, S, VMgr, Pred, PD, BR);
   cast<GRBugReporter>(BR).getStateManager().iterBindings(N->getState(), H);
 }
@@ -483,13 +473,13 @@ class ScanNotableSymbols
 : public StoreManager::BindingsHandler {
 
   llvm::SmallSet<SymbolRef, 10> AlreadyProcessed;
-  const ExplodedNode* N;
-  const Stmt* S;
+  const ExplodedNode *N;
+  const Stmt *S;
   GRBugReporter& BR;
   PathDiagnostic& PD;
 
 public:
-  ScanNotableSymbols(const ExplodedNode* n, const Stmt* s,
+  ScanNotableSymbols(const ExplodedNode *n, const Stmt *s,
                      GRBugReporter& br, PathDiagnostic& pd)
   : N(n), S(s), BR(br), PD(pd) {}
 
@@ -526,7 +516,8 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
                                           const ExplodedNode *N) {
 
   SourceManager& SMgr = PDB.getSourceManager();
-  const ExplodedNode* NextNode = N->pred_empty()
+  const LocationContext *LC = PDB.getLocationContext();
+  const ExplodedNode *NextNode = N->pred_empty()
                                         ? NULL : *(N->pred_begin());
   while (NextNode) {
     N = NextNode;
@@ -534,15 +525,17 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
 
     ProgramPoint P = N->getLocation();
 
-    if (const BlockEdge* BE = dyn_cast<BlockEdge>(&P)) {
-      const CFGBlock* Src = BE->getSrc();
-      const CFGBlock* Dst = BE->getDst();
-      const Stmt* T = Src->getTerminator();
+    if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
+      const CFGBlock *Src = BE->getSrc();
+      const CFGBlock *Dst = BE->getDst();
+      const Stmt *T = Src->getTerminator();
 
       if (!T)
         continue;
 
-      FullSourceLoc Start(T->getLocStart(), SMgr);
+      PathDiagnosticLocation Start =
+        PathDiagnosticLocation::createBegin(T, SMgr,
+                                                N->getLocationContext());
 
       switch (T->getStmtClass()) {
         default:
@@ -550,7 +543,7 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
 
         case Stmt::GotoStmtClass:
         case Stmt::IndirectGotoStmtClass: {
-          const Stmt* S = GetNextStmt(N);
+          const Stmt *S = GetNextStmt(N);
 
           if (!S)
             continue;
@@ -560,7 +553,7 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           const PathDiagnosticLocation &End = PDB.getEnclosingStmtLocation(S);
 
           os << "Control jumps to line "
-          << End.asLocation().getInstantiationLineNumber();
+          << End.asLocation().getExpansionLineNumber();
           PD.push_front(new PathDiagnosticControlFlowPiece(Start, End,
                                                            os.str()));
           break;
@@ -571,45 +564,45 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           std::string sbuf;
           llvm::raw_string_ostream os(sbuf);
 
-          if (const Stmt* S = Dst->getLabel()) {
-            PathDiagnosticLocation End(S, SMgr);
+          if (const Stmt *S = Dst->getLabel()) {
+            PathDiagnosticLocation End(S, SMgr, LC);
 
             switch (S->getStmtClass()) {
               default:
                 os << "No cases match in the switch statement. "
                 "Control jumps to line "
-                << End.asLocation().getInstantiationLineNumber();
+                << End.asLocation().getExpansionLineNumber();
                 break;
               case Stmt::DefaultStmtClass:
                 os << "Control jumps to the 'default' case at line "
-                << End.asLocation().getInstantiationLineNumber();
+                << End.asLocation().getExpansionLineNumber();
                 break;
 
               case Stmt::CaseStmtClass: {
                 os << "Control jumps to 'case ";
-                const CaseStmt* Case = cast<CaseStmt>(S);
-                const Expr* LHS = Case->getLHS()->IgnoreParenCasts();
+                const CaseStmt *Case = cast<CaseStmt>(S);
+                const Expr *LHS = Case->getLHS()->IgnoreParenCasts();
 
                 // Determine if it is an enum.
                 bool GetRawInt = true;
 
-                if (const DeclRefExpr* DR = dyn_cast<DeclRefExpr>(LHS)) {
+                if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LHS)) {
                   // FIXME: Maybe this should be an assertion.  Are there cases
                   // were it is not an EnumConstantDecl?
-                  const EnumConstantDecl* D =
+                  const EnumConstantDecl *D =
                     dyn_cast<EnumConstantDecl>(DR->getDecl());
 
                   if (D) {
                     GetRawInt = false;
-                    os << D;
+                    os << *D;
                   }
                 }
 
                 if (GetRawInt)
-                  os << LHS->EvaluateAsInt(PDB.getASTContext());
+                  os << LHS->EvaluateKnownConstInt(PDB.getASTContext());
 
                 os << ":'  at line "
-                << End.asLocation().getInstantiationLineNumber();
+                << End.asLocation().getExpansionLineNumber();
                 break;
               }
             }
@@ -673,14 +666,15 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
 
             if (*(Src->succ_begin()+1) == Dst) {
               os << "false";
-              PathDiagnosticLocation End(B->getLHS(), SMgr);
-              PathDiagnosticLocation Start(B->getOperatorLoc(), SMgr);
+              PathDiagnosticLocation End(B->getLHS(), SMgr, LC);
+              PathDiagnosticLocation Start =
+                PathDiagnosticLocation::createOperatorLoc(B, SMgr);
               PD.push_front(new PathDiagnosticControlFlowPiece(Start, End,
                                                                os.str()));
             }
             else {
               os << "true";
-              PathDiagnosticLocation Start(B->getLHS(), SMgr);
+              PathDiagnosticLocation Start(B->getLHS(), SMgr, LC);
               PathDiagnosticLocation End = PDB.ExecutionContinues(N);
               PD.push_front(new PathDiagnosticControlFlowPiece(Start, End,
                                                                os.str()));
@@ -692,15 +686,16 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
 
             if (*(Src->succ_begin()+1) == Dst) {
               os << "false";
-              PathDiagnosticLocation Start(B->getLHS(), SMgr);
+              PathDiagnosticLocation Start(B->getLHS(), SMgr, LC);
               PathDiagnosticLocation End = PDB.ExecutionContinues(N);
               PD.push_front(new PathDiagnosticControlFlowPiece(Start, End,
                                                                os.str()));
             }
             else {
               os << "true";
-              PathDiagnosticLocation End(B->getLHS(), SMgr);
-              PathDiagnosticLocation Start(B->getOperatorLoc(), SMgr);
+              PathDiagnosticLocation End(B->getLHS(), SMgr, LC);
+              PathDiagnosticLocation Start =
+                PathDiagnosticLocation::createOperatorLoc(B, SMgr);
               PD.push_front(new PathDiagnosticControlFlowPiece(Start, End,
                                                                os.str()));
             }
@@ -781,14 +776,16 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
     }
 
     if (NextNode) {
-      for (BugReporterContext::visitor_iterator I = PDB.visitor_begin(),
-           E = PDB.visitor_end(); I!=E; ++I) {
-        if (PathDiagnosticPiece* p = (*I)->VisitNode(N, NextNode, PDB))
+      // Add diagnostic pieces from custom visitors.
+      BugReport *R = PDB.getBugReport();
+      for (BugReport::visitor_iterator I = R->visitor_begin(),
+           E = R->visitor_end(); I!=E; ++I) {
+        if (PathDiagnosticPiece *p = (*I)->VisitNode(N, NextNode, PDB, *R))
           PD.push_front(p);
       }
     }
 
-    if (const PostStmt* PS = dyn_cast<PostStmt>(&P)) {
+    if (const PostStmt *PS = dyn_cast<PostStmt>(&P)) {
       // Scan the region bindings, and see if a "notable" symbol has a new
       // lval binding.
       ScanNotableSymbols SNS(N, PS->getStmt(), PDB.getBugReporter(), PD);
@@ -882,11 +879,11 @@ class EdgeBuilder {
       }
 
       if (S != Original)
-        L = PathDiagnosticLocation(S, L.getManager());
+        L = PathDiagnosticLocation(S, L.getManager(), PDB.getLocationContext());
     }
 
     if (firstCharOnly)
-      L = PathDiagnosticLocation(L.asLocation());
+      L  = PathDiagnosticLocation::createSingleLocation(L);
 
     return L;
   }
@@ -915,17 +912,14 @@ class EdgeBuilder {
 
   ~EdgeBuilder() {
     while (!CLocs.empty()) popLocation();
-
+    
     // Finally, add an initial edge from the start location of the first
     // statement (if it doesn't already exist).
-    // FIXME: Should handle CXXTryStmt if analyser starts supporting C++.
-    if (const CompoundStmt *CS =
-          dyn_cast_or_null<CompoundStmt>(PDB.getCodeDecl().getBody()))
-      if (!CS->body_empty()) {
-        SourceLocation Loc = (*CS->body_begin())->getLocStart();
-        rawAddEdge(PathDiagnosticLocation(Loc, PDB.getSourceManager()));
-      }
-
+    PathDiagnosticLocation L = PathDiagnosticLocation::createDeclBegin(
+                                                       PDB.getLocationContext(),
+                                                       PDB.getSourceManager());
+    if (L.isValid())
+      rawAddEdge(L);
   }
 
   void addEdge(PathDiagnosticLocation NewLoc, bool alwaysAdd = false);
@@ -974,15 +968,15 @@ bool EdgeBuilder::containsLocation(const PathDiagnosticLocation &Container,
   SourceRange ContaineeR = Containee.asRange();
 
   SourceManager &SM = PDB.getSourceManager();
-  SourceLocation ContainerRBeg = SM.getInstantiationLoc(ContainerR.getBegin());
-  SourceLocation ContainerREnd = SM.getInstantiationLoc(ContainerR.getEnd());
-  SourceLocation ContaineeRBeg = SM.getInstantiationLoc(ContaineeR.getBegin());
-  SourceLocation ContaineeREnd = SM.getInstantiationLoc(ContaineeR.getEnd());
+  SourceLocation ContainerRBeg = SM.getExpansionLoc(ContainerR.getBegin());
+  SourceLocation ContainerREnd = SM.getExpansionLoc(ContainerR.getEnd());
+  SourceLocation ContaineeRBeg = SM.getExpansionLoc(ContaineeR.getBegin());
+  SourceLocation ContaineeREnd = SM.getExpansionLoc(ContaineeR.getEnd());
 
-  unsigned ContainerBegLine = SM.getInstantiationLineNumber(ContainerRBeg);
-  unsigned ContainerEndLine = SM.getInstantiationLineNumber(ContainerREnd);
-  unsigned ContaineeBegLine = SM.getInstantiationLineNumber(ContaineeRBeg);
-  unsigned ContaineeEndLine = SM.getInstantiationLineNumber(ContaineeREnd);
+  unsigned ContainerBegLine = SM.getExpansionLineNumber(ContainerRBeg);
+  unsigned ContainerEndLine = SM.getExpansionLineNumber(ContainerREnd);
+  unsigned ContaineeBegLine = SM.getExpansionLineNumber(ContaineeRBeg);
+  unsigned ContaineeEndLine = SM.getExpansionLineNumber(ContaineeREnd);
 
   assert(ContainerBegLine <= ContainerEndLine);
   assert(ContaineeBegLine <= ContaineeEndLine);
@@ -990,11 +984,11 @@ bool EdgeBuilder::containsLocation(const PathDiagnosticLocation &Container,
   return (ContainerBegLine <= ContaineeBegLine &&
           ContainerEndLine >= ContaineeEndLine &&
           (ContainerBegLine != ContaineeBegLine ||
-           SM.getInstantiationColumnNumber(ContainerRBeg) <=
-           SM.getInstantiationColumnNumber(ContaineeRBeg)) &&
+           SM.getExpansionColumnNumber(ContainerRBeg) <=
+           SM.getExpansionColumnNumber(ContaineeRBeg)) &&
           (ContainerEndLine != ContaineeEndLine ||
-           SM.getInstantiationColumnNumber(ContainerREnd) >=
-           SM.getInstantiationColumnNumber(ContainerREnd)));
+           SM.getExpansionColumnNumber(ContainerREnd) >=
+           SM.getExpansionColumnNumber(ContainerREnd)));
 }
 
 void EdgeBuilder::rawAddEdge(PathDiagnosticLocation NewLoc) {
@@ -1010,8 +1004,8 @@ void EdgeBuilder::rawAddEdge(PathDiagnosticLocation NewLoc) {
     return;
 
   // FIXME: Ignore intra-macro edges for now.
-  if (NewLocClean.asLocation().getInstantiationLoc() ==
-      PrevLocClean.asLocation().getInstantiationLoc())
+  if (NewLocClean.asLocation().getExpansionLoc() ==
+      PrevLocClean.asLocation().getExpansionLoc())
     return;
 
   PD.push_front(new PathDiagnosticControlFlowPiece(NewLocClean, PrevLocClean));
@@ -1099,7 +1093,7 @@ void EdgeBuilder::addContext(const Stmt *S) {
   if (!S)
     return;
 
-  PathDiagnosticLocation L(S, PDB.getSourceManager());
+  PathDiagnosticLocation L(S, PDB.getSourceManager(), PDB.getLocationContext());
 
   while (!CLocs.empty()) {
     const PathDiagnosticLocation &TopContextLoc = CLocs.back();
@@ -1124,8 +1118,9 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
                                             PathDiagnosticBuilder &PDB,
                                             const ExplodedNode *N) {
   EdgeBuilder EB(PD, PDB);
+  const SourceManager& SM = PDB.getSourceManager();
 
-  const ExplodedNode* NextNode = N->pred_empty() ? NULL : *(N->pred_begin());
+  const ExplodedNode *NextNode = N->pred_empty() ? NULL : *(N->pred_begin());
   while (NextNode) {
     N = NextNode;
     NextNode = GetPredecessorNode(N);
@@ -1139,7 +1134,7 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
 
         // Are we jumping to the head of a loop?  Add a special diagnostic.
         if (const Stmt *Loop = BE->getDst()->getLoopTarget()) {
-          PathDiagnosticLocation L(Loop, PDB.getSourceManager());
+          PathDiagnosticLocation L(Loop, SM, PDB.getLocationContext());
           const CompoundStmt *CS = NULL;
 
           if (!Term) {
@@ -1157,9 +1152,8 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
           PD.push_front(p);
 
           if (CS) {
-            PathDiagnosticLocation BL(CS->getRBracLoc(),
-                                      PDB.getSourceManager());
-            BL = PathDiagnosticLocation(BL.asLocation());
+            PathDiagnosticLocation BL =
+              PathDiagnosticLocation::createEndBrace(CS, SM);
             EB.addEdge(BL);
           }
         }
@@ -1188,9 +1182,11 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
     if (!NextNode)
       continue;
 
-    for (BugReporterContext::visitor_iterator I = PDB.visitor_begin(),
-         E = PDB.visitor_end(); I!=E; ++I) {
-      if (PathDiagnosticPiece* p = (*I)->VisitNode(N, NextNode, PDB)) {
+    // Add pieces from custom visitors.
+    BugReport *R = PDB.getBugReport();
+    for (BugReport::visitor_iterator I = R->visitor_begin(),
+                                     E = R->visitor_end(); I!=E; ++I) {
+      if (PathDiagnosticPiece *p = (*I)->VisitNode(N, NextNode, PDB, *R)) {
         const PathDiagnosticLocation &Loc = p->getLocation();
         EB.addEdge(Loc, true);
         PD.push_front(p);
@@ -1211,14 +1207,58 @@ void BugType::FlushReports(BugReporter &BR) {}
 //===----------------------------------------------------------------------===//
 // Methods for BugReport and subclasses.
 //===----------------------------------------------------------------------===//
-BugReport::~BugReport() {}
-RangedBugReport::~RangedBugReport() {}
 
-const Stmt* BugReport::getStmt() const {
+void BugReport::addVisitor(BugReporterVisitor* visitor) {
+  if (!visitor)
+    return;
+
+  llvm::FoldingSetNodeID ID;
+  visitor->Profile(ID);
+  void *InsertPos;
+
+  if (CallbacksSet.FindNodeOrInsertPos(ID, InsertPos)) {
+    delete visitor;
+    return;
+  }
+
+  CallbacksSet.InsertNode(visitor, InsertPos);
+  Callbacks = F.add(visitor, Callbacks);
+}
+
+BugReport::~BugReport() {
+  for (visitor_iterator I = visitor_begin(), E = visitor_end(); I != E; ++I) {
+    delete *I;
+  }
+}
+
+void BugReport::Profile(llvm::FoldingSetNodeID& hash) const {
+  hash.AddPointer(&BT);
+  hash.AddString(Description);
+  if (Location.isValid()) {
+    Location.Profile(hash);
+  } else {
+    assert(ErrorNode);
+    hash.AddPointer(GetCurrentOrPreviousStmt(ErrorNode));
+  }
+
+  for (SmallVectorImpl<SourceRange>::const_iterator I =
+      Ranges.begin(), E = Ranges.end(); I != E; ++I) {
+    const SourceRange range = *I;
+    if (!range.isValid())
+      continue;
+    hash.AddInteger(range.getBegin().getRawEncoding());
+    hash.AddInteger(range.getEnd().getRawEncoding());
+  }
+}
+
+const Stmt *BugReport::getStmt() const {
+  if (!ErrorNode)
+    return 0;
+
   ProgramPoint ProgP = ErrorNode->getLocation();
   const Stmt *S = NULL;
 
-  if (BlockEntrance* BE = dyn_cast<BlockEntrance>(&ProgP)) {
+  if (BlockEntrance *BE = dyn_cast<BlockEntrance>(&ProgP)) {
     CFGBlock &Exit = ProgP.getLocationContext()->getCFG()->getExit();
     if (BE->getBlock() == &Exit)
       S = GetPreviousStmt(ErrorNode);
@@ -1229,61 +1269,47 @@ const Stmt* BugReport::getStmt() const {
   return S;
 }
 
-PathDiagnosticPiece*
-BugReport::getEndPath(BugReporterContext& BRC,
-                      const ExplodedNode* EndPathNode) {
-
-  const Stmt* S = getStmt();
-
-  if (!S)
-    return NULL;
-
-  BugReport::ranges_iterator Beg, End;
-  llvm::tie(Beg, End) = getRanges();
-  PathDiagnosticLocation L(S, BRC.getSourceManager());
-
-  // Only add the statement itself as a range if we didn't specify any
-  // special ranges for this report.
-  PathDiagnosticPiece* P = new PathDiagnosticEventPiece(L, getDescription(),
-                                                        Beg == End);
-
-  for (; Beg != End; ++Beg)
-    P->addRange(*Beg);
-
-  return P;
-}
-
 std::pair<BugReport::ranges_iterator, BugReport::ranges_iterator>
-BugReport::getRanges() const {
-  if (const Expr* E = dyn_cast_or_null<Expr>(getStmt())) {
-    R = E->getSourceRange();
-    assert(R.isValid());
-    return std::make_pair(&R, &R+1);
-  }
-  else
-    return std::make_pair(ranges_iterator(), ranges_iterator());
-}
-
-SourceLocation BugReport::getLocation() const {
-  if (ErrorNode)
-    if (const Stmt* S = GetCurrentOrPreviousStmt(ErrorNode)) {
-      // For member expressions, return the location of the '.' or '->'.
-      if (const MemberExpr *ME = dyn_cast<MemberExpr>(S))
-        return ME->getMemberLoc();
-      // For binary operators, return the location of the operator.
-      if (const BinaryOperator *B = dyn_cast<BinaryOperator>(S))
-        return B->getOperatorLoc();
-
-      return S->getLocStart();
+BugReport::getRanges() {
+    // If no custom ranges, add the range of the statement corresponding to
+    // the error node.
+    if (Ranges.empty()) {
+      if (const Expr *E = dyn_cast_or_null<Expr>(getStmt()))
+        addRange(E->getSourceRange());
+      else
+        return std::make_pair(ranges_iterator(), ranges_iterator());
     }
 
-  return FullSourceLoc();
+    // User-specified absence of range info.
+    if (Ranges.size() == 1 && !Ranges.begin()->isValid())
+      return std::make_pair(ranges_iterator(), ranges_iterator());
+
+    return std::make_pair(Ranges.begin(), Ranges.end());
 }
 
-PathDiagnosticPiece* BugReport::VisitNode(const ExplodedNode* N,
-                                          const ExplodedNode* PrevN,
-                                          BugReporterContext &BRC) {
-  return NULL;
+PathDiagnosticLocation BugReport::getLocation(const SourceManager &SM) const {
+  if (ErrorNode) {
+    assert(!Location.isValid() &&
+     "Either Location or ErrorNode should be specified but not both.");
+
+    if (const Stmt *S = GetCurrentOrPreviousStmt(ErrorNode)) {
+      const LocationContext *LC = ErrorNode->getLocationContext();
+
+      // For member expressions, return the location of the '.' or '->'.
+      if (const MemberExpr *ME = dyn_cast<MemberExpr>(S))
+        return PathDiagnosticLocation::createMemberLoc(ME, SM);
+      // For binary operators, return the location of the operator.
+      if (const BinaryOperator *B = dyn_cast<BinaryOperator>(S))
+        return PathDiagnosticLocation::createOperatorLoc(B, SM);
+
+      return PathDiagnosticLocation::createBegin(S, SM, LC);
+    }
+  } else {
+    assert(Location.isValid());
+    return Location;
+  }
+
+  return PathDiagnosticLocation();
 }
 
 //===----------------------------------------------------------------------===//
@@ -1299,10 +1325,19 @@ BugReporterData::~BugReporterData() {}
 
 ExplodedGraph &GRBugReporter::getGraph() { return Eng.getGraph(); }
 
-GRStateManager&
+ProgramStateManager&
 GRBugReporter::getStateManager() { return Eng.getStateManager(); }
 
-BugReporter::~BugReporter() { FlushReports(); }
+BugReporter::~BugReporter() {
+  FlushReports();
+
+  // Free the bug reports we are tracking.
+  typedef std::vector<BugReportEquivClass *> ContTy;
+  for (ContTy::iterator I = EQClassesVector.begin(), E = EQClassesVector.end();
+       I != E; ++I) {
+    delete *I;
+  }
+}
 
 void BugReporter::FlushReports() {
   if (BugTypes.isEmpty())
@@ -1312,10 +1347,10 @@ void BugReporter::FlushReports() {
   // warnings and new BugTypes.
   // FIXME: Only NSErrorChecker needs BugType's FlushReports.
   // Turn NSErrorChecker into a proper checker and remove this.
-  llvm::SmallVector<const BugType*, 16> bugTypes;
+  SmallVector<const BugType*, 16> bugTypes;
   for (BugTypesTy::iterator I=BugTypes.begin(), E=BugTypes.end(); I!=E; ++I)
     bugTypes.push_back(*I);
-  for (llvm::SmallVector<const BugType*, 16>::iterator
+  for (SmallVector<const BugType*, 16>::iterator
          I = bugTypes.begin(), E = bugTypes.end(); I != E; ++I)
     const_cast<BugType*>(*I)->FlushReports(*this);
 
@@ -1344,7 +1379,7 @@ void BugReporter::FlushReports() {
 static std::pair<std::pair<ExplodedGraph*, NodeBackMap*>,
                  std::pair<ExplodedNode*, unsigned> >
 MakeReportGraph(const ExplodedGraph* G,
-                llvm::SmallVectorImpl<const ExplodedNode*> &nodes) {
+                SmallVectorImpl<const ExplodedNode*> &nodes) {
 
   // Create the trimmed graph.  It will contain the shortest paths from the
   // error nodes to the root.  In the new graph we should only have one
@@ -1390,10 +1425,10 @@ MakeReportGraph(const ExplodedGraph* G,
   llvm::DenseMap<const void*,unsigned> Visited;
 
   unsigned cnt = 0;
-  const ExplodedNode* Root = 0;
+  const ExplodedNode *Root = 0;
 
   while (!WS.empty()) {
-    const ExplodedNode* Node = WS.front();
+    const ExplodedNode *Node = WS.front();
     WS.pop();
 
     if (Visited.find(Node) != Visited.end())
@@ -1426,7 +1461,7 @@ MakeReportGraph(const ExplodedGraph* G,
 
     // Create the equivalent node in the new graph with the same state
     // and location.
-    ExplodedNode* NewN = GNew->getNode(N->getLocation(), N->getState());
+    ExplodedNode *NewN = GNew->getNode(N->getLocation(), N->getState());
 
     // Store the mapping to the original node.
     llvm::DenseMap<const void*, const void*>::iterator IMitr=InverseMap.find(N);
@@ -1495,7 +1530,7 @@ static void CompactPathDiagnostic(PathDiagnostic &PD, const SourceManager& SM) {
     // Determine the instantiation location, which is the location we group
     // related PathDiagnosticPieces.
     SourceLocation InstantiationLoc = Loc.isMacroID() ?
-                                      SM.getInstantiationLoc(Loc) :
+                                      SM.getExpansionLoc(Loc) :
                                       SourceLocation();
 
     if (Loc.isFileID()) {
@@ -1517,7 +1552,7 @@ static void CompactPathDiagnostic(PathDiagnostic &PD, const SourceManager& SM) {
     PathDiagnosticMacroPiece *MacroGroup = 0;
 
     SourceLocation ParentInstantiationLoc = InstantiationLoc.isMacroID() ?
-                                          SM.getInstantiationLoc(Loc) :
+                                          SM.getExpansionLoc(Loc) :
                                           SourceLocation();
 
     // Walk the entire macro stack.
@@ -1537,7 +1572,9 @@ static void CompactPathDiagnostic(PathDiagnostic &PD, const SourceManager& SM) {
 
     if (!MacroGroup || ParentInstantiationLoc == MacroStack.back().second) {
       // Create a new macro group and add it to the stack.
-      PathDiagnosticMacroPiece *NewGroup = new PathDiagnosticMacroPiece(Loc);
+      PathDiagnosticMacroPiece *NewGroup =
+        new PathDiagnosticMacroPiece(
+          PathDiagnosticLocation::createSingleLocation(I->getLocation()));
 
       if (MacroGroup)
         MacroGroup->push_back(NewGroup);
@@ -1569,11 +1606,11 @@ static void CompactPathDiagnostic(PathDiagnostic &PD, const SourceManager& SM) {
 }
 
 void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
-                        llvm::SmallVectorImpl<BugReport *> &bugReports) {
+                        SmallVectorImpl<BugReport *> &bugReports) {
 
   assert(!bugReports.empty());
-  llvm::SmallVector<const ExplodedNode *, 10> errorNodes;
-  for (llvm::SmallVectorImpl<BugReport*>::iterator I = bugReports.begin(),
+  SmallVector<const ExplodedNode *, 10> errorNodes;
+  for (SmallVectorImpl<BugReport*>::iterator I = bugReports.begin(),
     E = bugReports.end(); I != E; ++I) {
       errorNodes.push_back((*I)->getErrorNode());
   }
@@ -1594,22 +1631,36 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
   const ExplodedNode *N = GPair.second.first;
 
   // Start building the path diagnostic...
-  PathDiagnosticBuilder PDB(*this, R, BackMap.get(), getPathDiagnosticClient());
+  PathDiagnosticBuilder PDB(*this, R, BackMap.get(),
+                            getPathDiagnosticConsumer());
 
-  if (PathDiagnosticPiece* Piece = R->getEndPath(PDB, N))
-    PD.push_back(Piece);
+  // Register additional node visitors.
+  R->addVisitor(new NilReceiverBRVisitor());
+  R->addVisitor(new ConditionBRVisitor());
+
+  // Generate the very last diagnostic piece - the piece is visible before 
+  // the trace is expanded.
+  PathDiagnosticPiece *LastPiece = 0;
+  for (BugReport::visitor_iterator I = R->visitor_begin(),
+                                   E = R->visitor_end(); I!=E; ++I) {
+    if (PathDiagnosticPiece *Piece = (*I)->getEndPath(PDB, N, *R)) {
+      assert (!LastPiece &&
+              "There can only be one final piece in a diagnostic.");
+      LastPiece = Piece;
+    }
+  }
+  if (!LastPiece)
+    LastPiece = BugReporterVisitor::getDefaultEndPath(PDB, N, *R);
+  if (LastPiece)
+    PD.push_back(LastPiece);
   else
     return;
 
-  // Register node visitors.
-  R->registerInitialVisitors(PDB, N);
-  bugreporter::registerNilReceiverVisitor(PDB);
-
   switch (PDB.getGenerationScheme()) {
-    case PathDiagnosticClient::Extensive:
+    case PathDiagnosticConsumer::Extensive:
       GenerateExtensivePathDiagnostic(PD, PDB, N);
       break;
-    case PathDiagnosticClient::Minimal:
+    case PathDiagnosticConsumer::Minimal:
       GenerateMinimalPathDiagnostic(PD, PDB, N);
       break;
   }
@@ -1633,6 +1684,7 @@ void BugReporter::EmitReport(BugReport* R) {
   if (!EQ) {
     EQ = new BugReportEquivClass(R);
     EQClasses.InsertNode(EQ, InsertPos);
+    EQClassesVector.push_back(EQ);
   }
   else
     EQ->AddReport(R);
@@ -1655,7 +1707,7 @@ struct FRIEC_WLItem {
 
 static BugReport *
 FindReportInEquivalenceClass(BugReportEquivClass& EQ,
-                             llvm::SmallVectorImpl<BugReport*> &bugReports) {
+                             SmallVectorImpl<BugReport*> &bugReports) {
 
   BugReportEquivClass::iterator I = EQ.begin(), E = EQ.end();
   assert(I != E);
@@ -1667,7 +1719,7 @@ FindReportInEquivalenceClass(BugReportEquivClass& EQ,
   // to 'Nodes'.  Any of the reports will serve as a "representative" report.
   if (!BT.isSuppressOnSink()) {
     for (BugReportEquivClass::iterator I=EQ.begin(), E=EQ.end(); I!=E; ++I) {
-      const ExplodedNode* N = I->getErrorNode();
+      const ExplodedNode *N = I->getErrorNode();
       if (N) {
         R = *I;
         bugReports.push_back(R);
@@ -1691,9 +1743,8 @@ FindReportInEquivalenceClass(BugReportEquivClass& EQ,
     if (!errorNode)
       continue;
     if (errorNode->isSink()) {
-      assert(false &&
+      llvm_unreachable(
            "BugType::isSuppressSink() should not be 'true' for sink end nodes");
-      return 0;
     }
     // No successors?  By definition this nodes isn't post-dominated by a sink.
     if (errorNode->succ_empty()) {
@@ -1706,7 +1757,7 @@ FindReportInEquivalenceClass(BugReportEquivClass& EQ,
     // At this point we know that 'N' is not a sink and it has at least one
     // successor.  Use a DFS worklist to find a non-sink end-of-path node.    
     typedef FRIEC_WLItem WLItem;
-    typedef llvm::SmallVector<WLItem, 10> DFSWorkList;
+    typedef SmallVector<WLItem, 10> DFSWorkList;
     llvm::DenseMap<const ExplodedNode *, unsigned> Visited;
     
     DFSWorkList WL;
@@ -1763,11 +1814,8 @@ class DiagCacheItem : public llvm::FoldingSetNode {
   llvm::FoldingSetNodeID ID;
 public:
   DiagCacheItem(BugReport *R, PathDiagnostic *PD) {
-    ID.AddString(R->getBugType().getName());
-    ID.AddString(R->getBugType().getCategory());
-    ID.AddString(R->getDescription());
-    ID.AddInteger(R->getLocation().getRawEncoding());
-    PD->Profile(ID);    
+    R->Profile(ID);
+    PD->Profile(ID);
   }
   
   void Profile(llvm::FoldingSetNodeID &id) {
@@ -1798,12 +1846,12 @@ static bool IsCachedDiagnostic(BugReport *R, PathDiagnostic *PD) {
 }
 
 void BugReporter::FlushReport(BugReportEquivClass& EQ) {
-  llvm::SmallVector<BugReport*, 10> bugReports;
+  SmallVector<BugReport*, 10> bugReports;
   BugReport *exampleReport = FindReportInEquivalenceClass(EQ, bugReports);
   if (!exampleReport)
     return;
   
-  PathDiagnosticClient* PD = getPathDiagnosticClient();
+  PathDiagnosticConsumer* PD = getPathDiagnosticConsumer();
 
   // FIXME: Make sure we use the 'R' for the path that was actually used.
   // Probably doesn't make a difference in practice.
@@ -1823,47 +1871,50 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
     return;
   
   // Get the meta data.
-  std::pair<const char**, const char**> Meta =
-    exampleReport->getExtraDescriptiveText();
-  for (const char** s = Meta.first; s != Meta.second; ++s)
-    D->addMeta(*s);
+  const BugReport::ExtraTextList &Meta =
+                                  exampleReport->getExtraText();
+  for (BugReport::ExtraTextList::const_iterator i = Meta.begin(),
+                                                e = Meta.end(); i != e; ++i) {
+    D->addMeta(*i);
+  }
 
   // Emit a summary diagnostic to the regular Diagnostics engine.
   BugReport::ranges_iterator Beg, End;
   llvm::tie(Beg, End) = exampleReport->getRanges();
-  Diagnostic &Diag = getDiagnostic();
-  FullSourceLoc L(exampleReport->getLocation(), getSourceManager());
+  DiagnosticsEngine &Diag = getDiagnostic();
   
   // Search the description for '%', as that will be interpretted as a
   // format character by FormatDiagnostics.
-  llvm::StringRef desc = exampleReport->getShortDescription();
+  StringRef desc = exampleReport->getShortDescription();
   unsigned ErrorDiag;
   {
     llvm::SmallString<512> TmpStr;
     llvm::raw_svector_ostream Out(TmpStr);
-    for (llvm::StringRef::iterator I=desc.begin(), E=desc.end(); I!=E; ++I)
+    for (StringRef::iterator I=desc.begin(), E=desc.end(); I!=E; ++I)
       if (*I == '%')
         Out << "%%";
       else
         Out << *I;
     
     Out.flush();
-    ErrorDiag = Diag.getCustomDiagID(Diagnostic::Warning, TmpStr);
+    ErrorDiag = Diag.getCustomDiagID(DiagnosticsEngine::Warning, TmpStr);
   }        
 
   {
-    DiagnosticBuilder diagBuilder = Diag.Report(L, ErrorDiag);
+    DiagnosticBuilder diagBuilder = Diag.Report(
+      exampleReport->getLocation(getSourceManager()).asLocation(), ErrorDiag);
     for (BugReport::ranges_iterator I = Beg; I != End; ++I)
       diagBuilder << *I;
   }
 
-  // Emit a full diagnostic for the path if we have a PathDiagnosticClient.
+  // Emit a full diagnostic for the path if we have a PathDiagnosticConsumer.
   if (!PD)
     return;
 
   if (D->empty()) {
-    PathDiagnosticPiece* piece =
-      new PathDiagnosticEventPiece(L, exampleReport->getDescription());
+    PathDiagnosticPiece *piece = new PathDiagnosticEventPiece(
+                                 exampleReport->getLocation(getSourceManager()),
+                                 exampleReport->getDescription());
 
     for ( ; Beg != End; ++Beg) piece->addRange(*Beg);
     D->push_back(piece);
@@ -1872,27 +1923,26 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
   PD->HandlePathDiagnostic(D.take());
 }
 
-void BugReporter::EmitBasicReport(llvm::StringRef name, llvm::StringRef str,
-                                  SourceLocation Loc,
+void BugReporter::EmitBasicReport(StringRef name, StringRef str,
+                                  PathDiagnosticLocation Loc,
                                   SourceRange* RBeg, unsigned NumRanges) {
   EmitBasicReport(name, "", str, Loc, RBeg, NumRanges);
 }
 
-void BugReporter::EmitBasicReport(llvm::StringRef name,
-                                  llvm::StringRef category,
-                                  llvm::StringRef str, SourceLocation Loc,
+void BugReporter::EmitBasicReport(StringRef name,
+                                  StringRef category,
+                                  StringRef str, PathDiagnosticLocation Loc,
                                   SourceRange* RBeg, unsigned NumRanges) {
 
   // 'BT' is owned by BugReporter.
   BugType *BT = getBugTypeForName(name, category);
-  FullSourceLoc L = getContext().getFullLoc(Loc);
-  RangedBugReport *R = new DiagBugReport(*BT, str, L);
+  BugReport *R = new BugReport(*BT, str, Loc);
   for ( ; NumRanges > 0 ; --NumRanges, ++RBeg) R->addRange(*RBeg);
   EmitReport(R);
 }
 
-BugType *BugReporter::getBugTypeForName(llvm::StringRef name,
-                                        llvm::StringRef category) {
+BugType *BugReporter::getBugTypeForName(StringRef name,
+                                        StringRef category) {
   llvm::SmallString<136> fullDesc;
   llvm::raw_svector_ostream(fullDesc) << name << ":" << category;
   llvm::StringMapEntry<BugType *> &
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 8e31adead188..1abd8baef624 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -11,13 +11,15 @@
 //  enhance the diagnostics reported for a bug.
 //
 //===----------------------------------------------------------------------===//
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitor.h"
 
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
 using namespace ento;
@@ -39,8 +41,6 @@ const Stmt *bugreporter::GetDerefExpr(const ExplodedNode *N) {
     return ME->getBase()->IgnoreParenCasts();
   }
   else if (const ArraySubscriptExpr *AE = dyn_cast<ArraySubscriptExpr>(S)) {
-    // Retrieve the base for arrays since BasicStoreManager doesn't know how
-    // to reason about them.
     return AE->getBase();
   }
 
@@ -73,248 +73,255 @@ const Stmt *bugreporter::GetRetValExpr(const ExplodedNode *N) {
 // Definitions for bug reporter visitors.
 //===----------------------------------------------------------------------===//
 
-namespace {
-class FindLastStoreBRVisitor : public BugReporterVisitor {
-  const MemRegion *R;
-  SVal V;
-  bool satisfied;
-  const ExplodedNode *StoreSite;
-public:
-  FindLastStoreBRVisitor(SVal v, const MemRegion *r)
-  : R(r), V(v), satisfied(false), StoreSite(0) {}
+PathDiagnosticPiece*
+BugReporterVisitor::getEndPath(BugReporterContext &BRC,
+                               const ExplodedNode *EndPathNode,
+                               BugReport &BR) {
+  return 0;
+}
 
-  virtual void Profile(llvm::FoldingSetNodeID &ID) const {
-    static int tag = 0;
-    ID.AddPointer(&tag);
-    ID.AddPointer(R);
-    ID.Add(V);
+PathDiagnosticPiece*
+BugReporterVisitor::getDefaultEndPath(BugReporterContext &BRC,
+                                      const ExplodedNode *EndPathNode,
+                                      BugReport &BR) {
+  const ProgramPoint &PP = EndPathNode->getLocation();
+  PathDiagnosticLocation L;
+
+  if (const BlockEntrance *BE = dyn_cast<BlockEntrance>(&PP)) {
+    const CFGBlock *block = BE->getBlock();
+    if (block->getBlockID() == 0) {
+      L = PathDiagnosticLocation::createDeclEnd(PP.getLocationContext(),
+                                                BRC.getSourceManager());
+    }
   }
 
-  PathDiagnosticPiece* VisitNode(const ExplodedNode *N,
-                                 const ExplodedNode *PrevN,
-                                 BugReporterContext& BRC) {
+  if (!L.isValid()) {
+    const Stmt *S = BR.getStmt();
 
-    if (satisfied)
+    if (!S)
       return NULL;
 
-    if (!StoreSite) {
-      const ExplodedNode *Node = N, *Last = NULL;
+    L = PathDiagnosticLocation(S, BRC.getSourceManager(),
+                                  PP.getLocationContext());
+  }
 
-      for ( ; Node ; Last = Node, Node = Node->getFirstPred()) {
+  BugReport::ranges_iterator Beg, End;
+  llvm::tie(Beg, End) = BR.getRanges();
 
-        if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
-          if (const PostStmt *P = Node->getLocationAs<PostStmt>())
-            if (const DeclStmt *DS = P->getStmtAs<DeclStmt>())
-              if (DS->getSingleDecl() == VR->getDecl()) {
-                Last = Node;
-                break;
-              }
-        }
+  // Only add the statement itself as a range if we didn't specify any
+  // special ranges for this report.
+  PathDiagnosticPiece *P = new PathDiagnosticEventPiece(L,
+      BR.getDescription(),
+      Beg == End);
+  for (; Beg != End; ++Beg)
+    P->addRange(*Beg);
 
-        if (Node->getState()->getSVal(R) != V)
-          break;
-      }
+  return P;
+}
 
-      if (!Node || !Last) {
-        satisfied = true;
-        return NULL;
-      }
 
-      StoreSite = Last;
-    }
+void FindLastStoreBRVisitor ::Profile(llvm::FoldingSetNodeID &ID) const {
+  static int tag = 0;
+  ID.AddPointer(&tag);
+  ID.AddPointer(R);
+  ID.Add(V);
+}
 
-    if (StoreSite != N)
-      return NULL;
+PathDiagnosticPiece *FindLastStoreBRVisitor::VisitNode(const ExplodedNode *N,
+                                                     const ExplodedNode *PrevN,
+                                                     BugReporterContext &BRC,
+                                                     BugReport &BR) {
 
-    satisfied = true;
-    llvm::SmallString<256> sbuf;
-    llvm::raw_svector_ostream os(sbuf);
+  if (satisfied)
+    return NULL;
 
-    if (const PostStmt *PS = N->getLocationAs<PostStmt>()) {
-      if (const DeclStmt *DS = PS->getStmtAs<DeclStmt>()) {
+  if (!StoreSite) {
+    const ExplodedNode *Node = N, *Last = NULL;
 
-        if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
-          os << "Variable '" << VR->getDecl() << "' ";
-        }
-        else
-          return NULL;
+    for ( ; Node ; Last = Node, Node = Node->getFirstPred()) {
 
-        if (isa<loc::ConcreteInt>(V)) {
-          bool b = false;
-          if (R->isBoundable()) {
-            if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
-              if (TR->getValueType()->isObjCObjectPointerType()) {
-                os << "initialized to nil";
-                b = true;
-              }
+      if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
+        if (const PostStmt *P = Node->getLocationAs<PostStmt>())
+          if (const DeclStmt *DS = P->getStmtAs<DeclStmt>())
+            if (DS->getSingleDecl() == VR->getDecl()) {
+              Last = Node;
+              break;
             }
-          }
-
-          if (!b)
-            os << "initialized to a null pointer value";
-        }
-        else if (isa<nonloc::ConcreteInt>(V)) {
-          os << "initialized to " << cast<nonloc::ConcreteInt>(V).getValue();
-        }
-        else if (V.isUndef()) {
-          if (isa<VarRegion>(R)) {
-            const VarDecl *VD = cast<VarDecl>(DS->getSingleDecl());
-            if (VD->getInit())
-              os << "initialized to a garbage value";
-            else
-              os << "declared without an initial value";
-          }
-        }
       }
+
+      if (Node->getState()->getSVal(R) != V)
+        break;
     }
 
-    if (os.str().empty()) {
+    if (!Node || !Last) {
+      satisfied = true;
+      return NULL;
+    }
+
+    StoreSite = Last;
+  }
+
+  if (StoreSite != N)
+    return NULL;
+
+  satisfied = true;
+  llvm::SmallString<256> sbuf;
+  llvm::raw_svector_ostream os(sbuf);
+
+  if (const PostStmt *PS = N->getLocationAs<PostStmt>()) {
+    if (const DeclStmt *DS = PS->getStmtAs<DeclStmt>()) {
+
+      if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
+        os << "Variable '" << *VR->getDecl() << "' ";
+      }
+      else
+        return NULL;
+
       if (isa<loc::ConcreteInt>(V)) {
         bool b = false;
         if (R->isBoundable()) {
-          if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
+          if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(R)) {
             if (TR->getValueType()->isObjCObjectPointerType()) {
-              os << "nil object reference stored to ";
+              os << "initialized to nil";
               b = true;
             }
           }
         }
 
         if (!b)
-          os << "Null pointer value stored to ";
-      }
-      else if (V.isUndef()) {
-        os << "Uninitialized value stored to ";
+          os << "initialized to a null pointer value";
       }
       else if (isa<nonloc::ConcreteInt>(V)) {
-        os << "The value " << cast<nonloc::ConcreteInt>(V).getValue()
-           << " is assigned to ";
+        os << "initialized to " << cast<nonloc::ConcreteInt>(V).getValue();
       }
-      else
-        return NULL;
-
-      if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
-        os << '\'' << VR->getDecl() << '\'';
+      else if (V.isUndef()) {
+        if (isa<VarRegion>(R)) {
+          const VarDecl *VD = cast<VarDecl>(DS->getSingleDecl());
+          if (VD->getInit())
+            os << "initialized to a garbage value";
+          else
+            os << "declared without an initial value";
+        }
       }
-      else
-        return NULL;
+    }
+  }
+
+  if (os.str().empty()) {
+    if (isa<loc::ConcreteInt>(V)) {
+      bool b = false;
+      if (R->isBoundable()) {
+        if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(R)) {
+          if (TR->getValueType()->isObjCObjectPointerType()) {
+            os << "nil object reference stored to ";
+            b = true;
+          }
+        }
+      }
+
+      if (!b)
+        os << "Null pointer value stored to ";
+    }
+    else if (V.isUndef()) {
+      os << "Uninitialized value stored to ";
+    }
+    else if (isa<nonloc::ConcreteInt>(V)) {
+      os << "The value " << cast<nonloc::ConcreteInt>(V).getValue()
+               << " is assigned to ";
+    }
+    else
+      return NULL;
+
+    if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
+      os << '\'' << *VR->getDecl() << '\'';
+    }
+    else
+      return NULL;
+  }
+
+  // Construct a new PathDiagnosticPiece.
+  ProgramPoint P = N->getLocation();
+  PathDiagnosticLocation L =
+    PathDiagnosticLocation::create(P, BRC.getSourceManager());
+  if (!L.isValid())
+    return NULL;
+  return new PathDiagnosticEventPiece(L, os.str());
+}
+
+void TrackConstraintBRVisitor::Profile(llvm::FoldingSetNodeID &ID) const {
+  static int tag = 0;
+  ID.AddPointer(&tag);
+  ID.AddBoolean(Assumption);
+  ID.Add(Constraint);
+}
+
+PathDiagnosticPiece *
+TrackConstraintBRVisitor::VisitNode(const ExplodedNode *N,
+                                    const ExplodedNode *PrevN,
+                                    BugReporterContext &BRC,
+                                    BugReport &BR) {
+  if (isSatisfied)
+    return NULL;
+
+  // Check if in the previous state it was feasible for this constraint
+  // to *not* be true.
+  if (PrevN->getState()->assume(Constraint, !Assumption)) {
+
+    isSatisfied = true;
+
+    // As a sanity check, make sure that the negation of the constraint
+    // was infeasible in the current state.  If it is feasible, we somehow
+    // missed the transition point.
+    if (N->getState()->assume(Constraint, !Assumption))
+      return NULL;
+
+    // We found the transition point for the constraint.  We now need to
+    // pretty-print the constraint. (work-in-progress)
+    std::string sbuf;
+    llvm::raw_string_ostream os(sbuf);
+
+    if (isa<Loc>(Constraint)) {
+      os << "Assuming pointer value is ";
+      os << (Assumption ? "non-null" : "null");
     }
 
-    // FIXME: Refactor this into BugReporterContext.
-    const Stmt *S = 0;
-    ProgramPoint P = N->getLocation();
-
-    if (BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-      const CFGBlock *BSrc = BE->getSrc();
-      S = BSrc->getTerminatorCondition();
-    }
-    else if (PostStmt *PS = dyn_cast<PostStmt>(&P)) {
-      S = PS->getStmt();
-    }
-
-    if (!S)
+    if (os.str().empty())
       return NULL;
 
     // Construct a new PathDiagnosticPiece.
-    PathDiagnosticLocation L(S, BRC.getSourceManager());
+    ProgramPoint P = N->getLocation();
+    PathDiagnosticLocation L =
+      PathDiagnosticLocation::create(P, BRC.getSourceManager());
+    if (!L.isValid())
+      return NULL;
     return new PathDiagnosticEventPiece(L, os.str());
   }
-};
 
-
-static void registerFindLastStore(BugReporterContext& BRC, const MemRegion *R,
-                                  SVal V) {
-  BRC.addVisitor(new FindLastStoreBRVisitor(V, R));
+  return NULL;
 }
 
-class TrackConstraintBRVisitor : public BugReporterVisitor {
-  DefinedSVal Constraint;
-  const bool Assumption;
-  bool isSatisfied;
-public:
-  TrackConstraintBRVisitor(DefinedSVal constraint, bool assumption)
-  : Constraint(constraint), Assumption(assumption), isSatisfied(false) {}
+BugReporterVisitor *
+bugreporter::getTrackNullOrUndefValueVisitor(const ExplodedNode *N,
+                                             const Stmt *S) {
+  if (!S || !N)
+    return 0;
 
-  void Profile(llvm::FoldingSetNodeID &ID) const {
-    static int tag = 0;
-    ID.AddPointer(&tag);
-    ID.AddBoolean(Assumption);
-    ID.Add(Constraint);
-  }
+  ProgramStateManager &StateMgr = N->getState()->getStateManager();
 
-  PathDiagnosticPiece* VisitNode(const ExplodedNode *N,
-                                 const ExplodedNode *PrevN,
-                                 BugReporterContext& BRC) {
-    if (isSatisfied)
-      return NULL;
-
-    // Check if in the previous state it was feasible for this constraint
-    // to *not* be true.
-    if (PrevN->getState()->assume(Constraint, !Assumption)) {
-
-      isSatisfied = true;
-
-      // As a sanity check, make sure that the negation of the constraint
-      // was infeasible in the current state.  If it is feasible, we somehow
-      // missed the transition point.
-      if (N->getState()->assume(Constraint, !Assumption))
-        return NULL;
-
-      // We found the transition point for the constraint.  We now need to
-      // pretty-print the constraint. (work-in-progress)
-      std::string sbuf;
-      llvm::raw_string_ostream os(sbuf);
-
-      if (isa<Loc>(Constraint)) {
-        os << "Assuming pointer value is ";
-        os << (Assumption ? "non-null" : "null");
-      }
-
-      if (os.str().empty())
-        return NULL;
-
-      // FIXME: Refactor this into BugReporterContext.
-      const Stmt *S = 0;
-      ProgramPoint P = N->getLocation();
-
-      if (BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-        const CFGBlock *BSrc = BE->getSrc();
-        S = BSrc->getTerminatorCondition();
-      }
-      else if (PostStmt *PS = dyn_cast<PostStmt>(&P)) {
-        S = PS->getStmt();
-      }
-
-      if (!S)
-        return NULL;
-
-      // Construct a new PathDiagnosticPiece.
-      PathDiagnosticLocation L(S, BRC.getSourceManager());
-      return new PathDiagnosticEventPiece(L, os.str());
+  // Walk through nodes until we get one that matches the statement
+  // exactly.
+  while (N) {
+    const ProgramPoint &pp = N->getLocation();
+    if (const PostStmt *ps = dyn_cast<PostStmt>(&pp)) {
+      if (ps->getStmt() == S)
+        break;
     }
-
-    return NULL;
+    N = N->getFirstPred();
   }
-};
-} // end anonymous namespace
 
-static void registerTrackConstraint(BugReporterContext& BRC,
-                                    DefinedSVal Constraint,
-                                    bool Assumption) {
-  BRC.addVisitor(new TrackConstraintBRVisitor(Constraint, Assumption));
-}
-
-void bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
-                                                const void *data,
-                                                const ExplodedNode* N) {
-
-  const Stmt *S = static_cast<const Stmt*>(data);
-
-  if (!S)
-    return;
-
-  GRStateManager &StateMgr = BRC.getStateManager();
-  const GRState *state = N->getState();
+  if (!N)
+    return 0;
+  
+  const ProgramState *state = N->getState();
 
   // Walk through lvalue-to-rvalue conversions.  
   if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S)) {
@@ -327,7 +334,7 @@ void bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
 
       if (isa<loc::ConcreteInt>(V) || isa<nonloc::ConcreteInt>(V)
           || V.isUndef()) {
-        ::registerFindLastStore(BRC, R, V);
+        return new FindLastStoreBRVisitor(V, R);
       }
     }
   }
@@ -347,94 +354,73 @@ void bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
 
     if (R) {
       assert(isa<SymbolicRegion>(R));
-      registerTrackConstraint(BRC, loc::MemRegionVal(R), false);
+      return new TrackConstraintBRVisitor(loc::MemRegionVal(R), false);
     }
   }
+
+  return 0;
 }
 
-void bugreporter::registerFindLastStore(BugReporterContext& BRC,
-                                        const void *data,
-                                        const ExplodedNode* N) {
+BugReporterVisitor *
+FindLastStoreBRVisitor::createVisitorObject(const ExplodedNode *N,
+                                            const MemRegion *R) {
+  assert(R && "The memory region is null.");
 
-  const MemRegion *R = static_cast<const MemRegion*>(data);
-
-  if (!R)
-    return;
-
-  const GRState *state = N->getState();
+  const ProgramState *state = N->getState();
   SVal V = state->getSVal(R);
-
   if (V.isUnknown())
-    return;
+    return 0;
 
-  BRC.addVisitor(new FindLastStoreBRVisitor(V, R));
+  return new FindLastStoreBRVisitor(V, R);
 }
 
 
-namespace {
-class NilReceiverVisitor : public BugReporterVisitor {
-public:
-  NilReceiverVisitor() {}
+PathDiagnosticPiece *NilReceiverBRVisitor::VisitNode(const ExplodedNode *N,
+                                                     const ExplodedNode *PrevN,
+                                                     BugReporterContext &BRC,
+                                                     BugReport &BR) {
+  const PostStmt *P = N->getLocationAs<PostStmt>();
+  if (!P)
+    return 0;
+  const ObjCMessageExpr *ME = P->getStmtAs<ObjCMessageExpr>();
+  if (!ME)
+    return 0;
+  const Expr *Receiver = ME->getInstanceReceiver();
+  if (!Receiver)
+    return 0;
+  const ProgramState *state = N->getState();
+  const SVal &V = state->getSVal(Receiver);
+  const DefinedOrUnknownSVal *DV = dyn_cast<DefinedOrUnknownSVal>(&V);
+  if (!DV)
+    return 0;
+  state = state->assume(*DV, true);
+  if (state)
+    return 0;
 
-  void Profile(llvm::FoldingSetNodeID &ID) const {
-    static int x = 0;
-    ID.AddPointer(&x);
-  }
-
-  PathDiagnosticPiece* VisitNode(const ExplodedNode *N,
-                                 const ExplodedNode *PrevN,
-                                 BugReporterContext& BRC) {
-
-    const PostStmt *P = N->getLocationAs<PostStmt>();
-    if (!P)
-      return 0;
-    const ObjCMessageExpr *ME = P->getStmtAs<ObjCMessageExpr>();
-    if (!ME)
-      return 0;
-    const Expr *Receiver = ME->getInstanceReceiver();
-    if (!Receiver)
-      return 0;
-    const GRState *state = N->getState();
-    const SVal &V = state->getSVal(Receiver);
-    const DefinedOrUnknownSVal *DV = dyn_cast<DefinedOrUnknownSVal>(&V);
-    if (!DV)
-      return 0;
-    state = state->assume(*DV, true);
-    if (state)
-      return 0;
-
-    // The receiver was nil, and hence the method was skipped.
-    // Register a BugReporterVisitor to issue a message telling us how
-    // the receiver was null.
-    bugreporter::registerTrackNullOrUndefValue(BRC, Receiver, N);
-    // Issue a message saying that the method was skipped.
-    PathDiagnosticLocation L(Receiver, BRC.getSourceManager());
-    return new PathDiagnosticEventPiece(L, "No method actually called "
-                                           "because the receiver is nil");
-  }
-};
-} // end anonymous namespace
-
-void bugreporter::registerNilReceiverVisitor(BugReporterContext &BRC) {
-  BRC.addVisitor(new NilReceiverVisitor());
+  // The receiver was nil, and hence the method was skipped.
+  // Register a BugReporterVisitor to issue a message telling us how
+  // the receiver was null.
+  BR.addVisitor(bugreporter::getTrackNullOrUndefValueVisitor(N, Receiver));
+  // Issue a message saying that the method was skipped.
+  PathDiagnosticLocation L(Receiver, BRC.getSourceManager(),
+                                     N->getLocationContext());
+  return new PathDiagnosticEventPiece(L, "No method actually called "
+      "because the receiver is nil");
 }
 
-// Registers every VarDecl inside a Stmt with a last store vistor.
-void bugreporter::registerVarDeclsLastStore(BugReporterContext &BRC,
-                                                   const void *stmt,
-                                                   const ExplodedNode *N) {
-  const Stmt *S = static_cast<const Stmt *>(stmt);
-
+// Registers every VarDecl inside a Stmt with a last store visitor.
+void FindLastStoreBRVisitor::registerStatementVarDecls(BugReport &BR,
+                                                       const Stmt *S) {
+  const ExplodedNode *N = BR.getErrorNode();
   std::deque<const Stmt *> WorkList;
-
   WorkList.push_back(S);
 
   while (!WorkList.empty()) {
     const Stmt *Head = WorkList.front();
     WorkList.pop_front();
 
-    GRStateManager &StateMgr = BRC.getStateManager();
-    const GRState *state = N->getState();
+    const ProgramState *state = N->getState();
+    ProgramStateManager &StateMgr = state->getStateManager();
 
     if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Head)) {
       if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
@@ -445,7 +431,8 @@ void bugreporter::registerVarDeclsLastStore(BugReporterContext &BRC,
         SVal V = state->getSVal(S);
 
         if (isa<loc::ConcreteInt>(V) || isa<nonloc::ConcreteInt>(V)) {
-          ::registerFindLastStore(BRC, R, V);
+          // Register a new visitor with the BugReport.
+          BR.addVisitor(new FindLastStoreBRVisitor(V, R));
         }
       }
     }
@@ -455,3 +442,248 @@ void bugreporter::registerVarDeclsLastStore(BugReporterContext &BRC,
       WorkList.push_back(*I);
   }
 }
+
+//===----------------------------------------------------------------------===//
+// Visitor that tries to report interesting diagnostics from conditions.
+//===----------------------------------------------------------------------===//
+PathDiagnosticPiece *ConditionBRVisitor::VisitNode(const ExplodedNode *N,
+                                                   const ExplodedNode *Prev,
+                                                   BugReporterContext &BRC,
+                                                   BugReport &BR) {
+  
+  const ProgramPoint &progPoint = N->getLocation();
+
+  const ProgramState *CurrentState = N->getState();
+  const ProgramState *PrevState = Prev->getState();
+  
+  // Compare the GDMs of the state, because that is where constraints
+  // are managed.  Note that ensure that we only look at nodes that
+  // were generated by the analyzer engine proper, not checkers.
+  if (CurrentState->getGDM().getRoot() ==
+      PrevState->getGDM().getRoot())
+    return 0;
+  
+  // If an assumption was made on a branch, it should be caught
+  // here by looking at the state transition.
+  if (const BlockEdge *BE = dyn_cast<BlockEdge>(&progPoint)) {
+    const CFGBlock *srcBlk = BE->getSrc();    
+    if (const Stmt *term = srcBlk->getTerminator())
+      return VisitTerminator(term, N, srcBlk, BE->getDst(), BRC);
+    return 0;
+  }
+  
+  if (const PostStmt *PS = dyn_cast<PostStmt>(&progPoint)) {
+    // FIXME: Assuming that BugReporter is a GRBugReporter is a layering
+    // violation.
+    const std::pair<const ProgramPointTag *, const ProgramPointTag *> &tags =      
+      cast<GRBugReporter>(BRC.getBugReporter()).
+        getEngine().getEagerlyAssumeTags();
+
+    const ProgramPointTag *tag = PS->getTag();
+    if (tag == tags.first)
+      return VisitTrueTest(cast<Expr>(PS->getStmt()), true,
+                           BRC, N->getLocationContext());
+    if (tag == tags.second)
+      return VisitTrueTest(cast<Expr>(PS->getStmt()), false,
+                           BRC, N->getLocationContext());
+                           
+    return 0;
+  }
+    
+  return 0;
+}
+
+PathDiagnosticPiece *
+ConditionBRVisitor::VisitTerminator(const Stmt *Term,
+                                    const ExplodedNode *N,
+                                    const CFGBlock *srcBlk,
+                                    const CFGBlock *dstBlk,
+                                    BugReporterContext &BRC) {
+  const Expr *Cond = 0;
+  
+  switch (Term->getStmtClass()) {
+  default:
+    return 0;
+  case Stmt::IfStmtClass:
+    Cond = cast<IfStmt>(Term)->getCond();
+    break;
+  case Stmt::ConditionalOperatorClass:
+    Cond = cast<ConditionalOperator>(Term)->getCond();
+    break;
+  }      
+
+  assert(Cond);
+  assert(srcBlk->succ_size() == 2);
+  const bool tookTrue = *(srcBlk->succ_begin()) == dstBlk;
+  return VisitTrueTest(Cond->IgnoreParenNoopCasts(BRC.getASTContext()),
+                       tookTrue, BRC, N->getLocationContext());
+}
+
+PathDiagnosticPiece *
+ConditionBRVisitor::VisitTrueTest(const Expr *Cond,
+                                  bool tookTrue,
+                                  BugReporterContext &BRC,
+                                  const LocationContext *LC) {
+  
+  const Expr *Ex = Cond;
+  
+  while (true) {
+    Ex = Ex->IgnoreParens();
+    switch (Ex->getStmtClass()) {
+      default:
+        return 0;
+      case Stmt::BinaryOperatorClass:
+        return VisitTrueTest(Cond, cast<BinaryOperator>(Ex), tookTrue, BRC, LC);
+      case Stmt::DeclRefExprClass:
+        return VisitTrueTest(Cond, cast<DeclRefExpr>(Ex), tookTrue, BRC, LC);
+      case Stmt::UnaryOperatorClass: {
+        const UnaryOperator *UO = cast<UnaryOperator>(Ex);
+        if (UO->getOpcode() == UO_LNot) {
+          tookTrue = !tookTrue;
+          Ex = UO->getSubExpr()->IgnoreParenNoopCasts(BRC.getASTContext());
+          continue;
+        }
+        return 0;
+      }
+    }
+  }
+}
+
+bool ConditionBRVisitor::patternMatch(const Expr *Ex, llvm::raw_ostream &Out,
+                                      BugReporterContext &BRC) {
+  const Expr *OriginalExpr = Ex;
+  Ex = Ex->IgnoreParenCasts();
+
+  if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Ex)) {
+    const bool quotes = isa<VarDecl>(DR->getDecl());
+    if (quotes)
+      Out << '\'';
+    Out << DR->getDecl()->getDeclName().getAsString();
+    if (quotes)
+      Out << '\'';
+    return quotes;
+  }
+  
+  if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Ex)) {
+    QualType OriginalTy = OriginalExpr->getType();
+    if (OriginalTy->isPointerType()) {
+      if (IL->getValue() == 0) {
+        Out << "null";
+        return false;
+      }
+    }
+    else if (OriginalTy->isObjCObjectPointerType()) {
+      if (IL->getValue() == 0) {
+        Out << "nil";
+        return false;
+      }
+    }
+    
+    Out << IL->getValue();
+    return false;
+  }
+  
+  return false;
+}
+
+PathDiagnosticPiece *
+ConditionBRVisitor::VisitTrueTest(const Expr *Cond,
+                                  const BinaryOperator *BExpr,
+                                  const bool tookTrue,
+                                  BugReporterContext &BRC,
+                                  const LocationContext *LC) {
+  
+  bool shouldInvert = false;
+  
+  llvm::SmallString<128> LhsString, RhsString;
+  {
+    llvm::raw_svector_ostream OutLHS(LhsString), OutRHS(RhsString);  
+    const bool isVarLHS = patternMatch(BExpr->getLHS(), OutLHS, BRC);
+    const bool isVarRHS = patternMatch(BExpr->getRHS(), OutRHS, BRC);
+    
+    shouldInvert = !isVarLHS && isVarRHS;    
+  }
+  
+  if (LhsString.empty() || RhsString.empty())
+    return 0;
+
+  // Should we invert the strings if the LHS is not a variable name?
+  
+  llvm::SmallString<256> buf;
+  llvm::raw_svector_ostream Out(buf);
+  Out << "Assuming " << (shouldInvert ? RhsString : LhsString) << " is ";
+
+  // Do we need to invert the opcode?
+  BinaryOperator::Opcode Op = BExpr->getOpcode();
+    
+  if (shouldInvert)
+    switch (Op) {
+      default: break;
+      case BO_LT: Op = BO_GT; break;
+      case BO_GT: Op = BO_LT; break;
+      case BO_LE: Op = BO_GE; break;
+      case BO_GE: Op = BO_LE; break;
+    }
+
+  if (!tookTrue)
+    switch (Op) {
+      case BO_EQ: Op = BO_NE; break;
+      case BO_NE: Op = BO_EQ; break;
+      case BO_LT: Op = BO_GE; break;
+      case BO_GT: Op = BO_LE; break;
+      case BO_LE: Op = BO_GT; break;
+      case BO_GE: Op = BO_LT; break;
+      default:
+        return 0;
+    }
+  
+  switch (BExpr->getOpcode()) {
+    case BO_EQ:
+      Out << "equal to ";
+      break;
+    case BO_NE:
+      Out << "not equal to ";
+      break;
+    default:
+      Out << BinaryOperator::getOpcodeStr(Op) << ' ';
+      break;
+  }
+  
+  Out << (shouldInvert ? LhsString : RhsString);
+
+  PathDiagnosticLocation Loc(Cond, BRC.getSourceManager(), LC);
+  return new PathDiagnosticEventPiece(Loc, Out.str());
+}
+  
+PathDiagnosticPiece *
+ConditionBRVisitor::VisitTrueTest(const Expr *Cond,
+                                  const DeclRefExpr *DR,
+                                  const bool tookTrue,
+                                  BugReporterContext &BRC,
+                                  const LocationContext *LC) {
+
+  const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
+  if (!VD)
+    return 0;
+  
+  llvm::SmallString<256> Buf;
+  llvm::raw_svector_ostream Out(Buf);
+    
+  Out << "Assuming '";
+  VD->getDeclName().printName(Out);
+  Out << "' is ";
+    
+  QualType VDTy = VD->getType();
+  
+  if (VDTy->isPointerType())
+    Out << (tookTrue ? "non-null" : "null");
+  else if (VDTy->isObjCObjectPointerType())
+    Out << (tookTrue ? "non-nil" : "nil");
+  else if (VDTy->isScalarType())
+    Out << (tookTrue ? "not equal to 0" : "0");
+  else
+    return 0;
+  
+  PathDiagnosticLocation Loc(Cond, BRC.getSourceManager(), LC);
+  return new PathDiagnosticEventPiece(Loc, Out.str());
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CMakeLists.txt b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
deleted file mode 100644
index 14c636cf761a..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-set(LLVM_LINK_COMPONENTS support)
-
-set(LLVM_USED_LIBS clangBasic clangLex clangAST clangFrontend clangRewrite)
-
-add_clang_library(clangStaticAnalyzerCore
-  AggExprVisitor.cpp
-  AnalysisManager.cpp
-  BasicConstraintManager.cpp
-  BasicStore.cpp
-  BasicValueFactory.cpp
-  BugReporter.cpp
-  BugReporterVisitors.cpp
-  CFRefCount.cpp
-  Checker.cpp
-  CheckerHelpers.cpp
-  CheckerManager.cpp
-  Environment.cpp
-  ExplodedGraph.cpp
-  FlatStore.cpp
-  BlockCounter.cpp
-  CXXExprEngine.cpp
-  CoreEngine.cpp
-  GRState.cpp
-  HTMLDiagnostics.cpp
-  MemRegion.cpp
-  ObjCMessage.cpp
-  PathDiagnostic.cpp
-  PlistDiagnostics.cpp
-  RangeConstraintManager.cpp
-  RegionStore.cpp
-  SimpleConstraintManager.cpp
-  SimpleSValBuilder.cpp
-  Store.cpp
-  SValBuilder.cpp
-  SVals.cpp
-  SymbolManager.cpp
-  TextPathDiagnostics.cpp
-  )
-
-add_dependencies(clangStaticAnalyzerCore ClangAttrClasses ClangAttrList ClangDeclNodes
-                 ClangStmtNodes)
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Checker.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Checker.cpp
new file mode 100644
index 000000000000..a3bf2c236f6e
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Checker.cpp
@@ -0,0 +1,22 @@
+//== Checker.cpp - Registration mechanism for checkers -----------*- C++ -*--=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines Checker, used to create and register checkers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/Checker.h"
+
+using namespace clang;
+using namespace ento;
+
+StringRef CheckerBase::getTagDescription() const {
+  // FIXME: We want to return the package + name of the checker here.
+  return "A Checker";  
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
index f6fb8f256c01..5356edc752fa 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
@@ -23,8 +23,8 @@ CheckerContext::~CheckerContext() {
   // if we are building sinks or we generated a node and decided to not
   // add it as a transition.
   if (Dst.size() == size && !B.BuildSinks && !B.hasGeneratedNode) {
-    if (ST && ST != B.GetState(Pred)) {
-      static int autoTransitionTag = 0;
+    if (ST && ST != Pred->getState()) {
+      static SimpleProgramPointTag autoTransitionTag("CheckerContext : auto");
       addTransition(ST, &autoTransitionTag);
     }
     else
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index ba7c384e5c36..acacfb0e18c6 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -12,8 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/CheckerProvider.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "clang/Analysis/ProgramPoint.h"
 #include "clang/AST/DeclBase.h"
 
@@ -33,7 +34,8 @@ bool CheckerManager::hasPathSensitiveCheckers() const {
          !DeadSymbolsCheckers.empty()       ||
          !RegionChangesCheckers.empty()     ||
          !EvalAssumeCheckers.empty()        ||
-         !EvalCallCheckers.empty();
+         !EvalCallCheckers.empty()          ||
+         !InlineCallCheckers.empty();
 }
 
 void CheckerManager::finishedCheckerRegistration() {
@@ -122,7 +124,7 @@ static void expandGraphWithCheckers(CHECK_CTX checkCtx,
 
 namespace {
   struct CheckStmtContext {
-    typedef llvm::SmallVectorImpl<CheckerManager::CheckStmtFunc> CheckersTy;
+    typedef SmallVectorImpl<CheckerManager::CheckStmtFunc> CheckersTy;
     bool IsPreVisit;
     const CheckersTy &Checkers;
     const Stmt *S;
@@ -138,9 +140,12 @@ namespace {
     void runChecker(CheckerManager::CheckStmtFunc checkFn,
                     ExplodedNodeSet &Dst, ExplodedNode *Pred) {
       // FIXME: Remove respondsToCallback from CheckerContext;
-      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, checkFn.Checker,
-                       IsPreVisit ? ProgramPoint::PreStmtKind :
-                                    ProgramPoint::PostStmtKind, 0, S);
+      ProgramPoint::Kind K =  IsPreVisit ? ProgramPoint::PreStmtKind :
+                                           ProgramPoint::PostStmtKind;
+      const ProgramPoint &L = ProgramPoint::getProgramPoint(S, K,
+                                Pred->getLocationContext(), checkFn.Checker);
+      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, L, 0);
+
       checkFn(S, C);
     }
   };
@@ -174,10 +179,12 @@ namespace {
 
     void runChecker(CheckerManager::CheckObjCMessageFunc checkFn,
                     ExplodedNodeSet &Dst, ExplodedNode *Pred) {
-      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, checkFn.Checker,
-                       IsPreVisit ? ProgramPoint::PreStmtKind :
-                                    ProgramPoint::PostStmtKind, 0,
-                       Msg.getOriginExpr());
+      ProgramPoint::Kind K =  IsPreVisit ? ProgramPoint::PreStmtKind :
+                                           ProgramPoint::PostStmtKind;
+      const ProgramPoint &L = ProgramPoint::getProgramPoint(Msg.getOriginExpr(),
+                                K, Pred->getLocationContext(), checkFn.Checker);
+      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, L, 0);
+
       checkFn(Msg, C);
     }
   };
@@ -214,10 +221,13 @@ namespace {
 
     void runChecker(CheckerManager::CheckLocationFunc checkFn,
                     ExplodedNodeSet &Dst, ExplodedNode *Pred) {
-      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, checkFn.Checker,
-                       IsLoad ? ProgramPoint::PreLoadKind :
-                       ProgramPoint::PreStoreKind, 0, S);
-      checkFn(Loc, IsLoad, C);
+      ProgramPoint::Kind K =  IsLoad ? ProgramPoint::PreLoadKind :
+                                       ProgramPoint::PreStoreKind;
+      const ProgramPoint &L = ProgramPoint::getProgramPoint(S, K,
+                                Pred->getLocationContext(), checkFn.Checker);
+      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, L, 0);
+
+      checkFn(Loc, IsLoad, S, C);
     }
   };
 }
@@ -249,9 +259,12 @@ namespace {
 
     void runChecker(CheckerManager::CheckBindFunc checkFn,
                     ExplodedNodeSet &Dst, ExplodedNode *Pred) {
-      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, checkFn.Checker,
-                       ProgramPoint::PreStmtKind, 0, S);
-      checkFn(Loc, Val, C);
+      ProgramPoint::Kind K =  ProgramPoint::PreStmtKind;
+      const ProgramPoint &L = ProgramPoint::getProgramPoint(S, K,
+                                Pred->getLocationContext(), checkFn.Checker);
+      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, L, 0);
+
+      checkFn(Loc, Val, S, C);
     }
   };
 }
@@ -293,7 +306,7 @@ void CheckerManager::runCheckersForBranchCondition(const Stmt *condition,
 }
 
 /// \brief Run checkers for live symbols.
-void CheckerManager::runCheckersForLiveSymbols(const GRState *state,
+void CheckerManager::runCheckersForLiveSymbols(const ProgramState *state,
                                                SymbolReaper &SymReaper) {
   for (unsigned i = 0, e = LiveSymbolsCheckers.size(); i != e; ++i)
     LiveSymbolsCheckers[i](state, SymReaper);
@@ -316,8 +329,11 @@ namespace {
 
     void runChecker(CheckerManager::CheckDeadSymbolsFunc checkFn,
                     ExplodedNodeSet &Dst, ExplodedNode *Pred) {
-      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, checkFn.Checker,
-                       ProgramPoint::PostPurgeDeadSymbolsKind, 0, S);
+      ProgramPoint::Kind K = ProgramPoint::PostPurgeDeadSymbolsKind;
+      const ProgramPoint &L = ProgramPoint::getProgramPoint(S, K,
+                                Pred->getLocationContext(), checkFn.Checker);
+      CheckerContext C(Dst, Eng.getBuilder(), Eng, Pred, L, 0);
+
       checkFn(SR, C);
     }
   };
@@ -334,7 +350,7 @@ void CheckerManager::runCheckersForDeadSymbols(ExplodedNodeSet &Dst,
 }
 
 /// \brief True if at least one checker wants to check region changes.
-bool CheckerManager::wantsRegionChangeUpdate(const GRState *state) {
+bool CheckerManager::wantsRegionChangeUpdate(const ProgramState *state) {
   for (unsigned i = 0, e = RegionChangesCheckers.size(); i != e; ++i)
     if (RegionChangesCheckers[i].WantUpdateFn(state))
       return true;
@@ -343,24 +359,25 @@ bool CheckerManager::wantsRegionChangeUpdate(const GRState *state) {
 }
 
 /// \brief Run checkers for region changes.
-const GRState *
-CheckerManager::runCheckersForRegionChanges(const GRState *state,
+const ProgramState *
+CheckerManager::runCheckersForRegionChanges(const ProgramState *state,
                             const StoreManager::InvalidatedSymbols *invalidated,
-                                            const MemRegion * const *Begin,
-                                            const MemRegion * const *End) {
+                                    ArrayRef<const MemRegion *> ExplicitRegions,
+                                          ArrayRef<const MemRegion *> Regions) {
   for (unsigned i = 0, e = RegionChangesCheckers.size(); i != e; ++i) {
     // If any checker declares the state infeasible (or if it starts that way),
     // bail out.
     if (!state)
       return NULL;
-    state = RegionChangesCheckers[i].CheckFn(state, invalidated, Begin, End);
+    state = RegionChangesCheckers[i].CheckFn(state, invalidated, 
+                                             ExplicitRegions, Regions);
   }
   return state;
 }
 
 /// \brief Run checkers for handling assumptions on symbolic values.
-const GRState *
-CheckerManager::runCheckersForEvalAssume(const GRState *state,
+const ProgramState *
+CheckerManager::runCheckersForEvalAssume(const ProgramState *state,
                                          SVal Cond, bool Assumption) {
   for (unsigned i = 0, e = EvalAssumeCheckers.size(); i != e; ++i) {
     // If any checker declares the state infeasible (or if it starts that way),
@@ -379,7 +396,9 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
                                             const CallExpr *CE,
                                             ExprEngine &Eng,
                                             GraphExpander *defaultEval) {
-  if (EvalCallCheckers.empty() && defaultEval == 0) {
+  if (EvalCallCheckers.empty()   &&
+      InlineCallCheckers.empty() &&
+      defaultEval == 0) {
     Dst.insert(Src);
     return;
   }
@@ -389,13 +408,50 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
 
     ExplodedNode *Pred = *NI;
     bool anyEvaluated = false;
+
+    // First, check if any of the InlineCall callbacks can evaluate the call.
+    assert(InlineCallCheckers.size() <= 1 &&
+           "InlineCall is a special hacky callback to allow intrusive"
+           "evaluation of the call (which simulates inlining). It is "
+           "currently only used by OSAtomicChecker and should go away "
+           "at some point.");
+    for (std::vector<InlineCallFunc>::iterator
+           EI = InlineCallCheckers.begin(), EE = InlineCallCheckers.end();
+         EI != EE; ++EI) {
+      ExplodedNodeSet checkDst;
+      bool evaluated = (*EI)(CE, Eng, Pred, checkDst);
+      assert(!(evaluated && anyEvaluated)
+             && "There are more than one checkers evaluating the call");
+      if (evaluated) {
+        anyEvaluated = true;
+        Dst.insert(checkDst);
+#ifdef NDEBUG
+        break; // on release don't check that no other checker also evals.
+#endif
+      }
+    }
+
+#ifdef NDEBUG // on release don't check that no other checker also evals.
+    if (anyEvaluated) {
+      break;
+    }
+#endif
+
+    // Next, check if any of the EvalCall callbacks can evaluate the call.
     for (std::vector<EvalCallFunc>::iterator
            EI = EvalCallCheckers.begin(), EE = EvalCallCheckers.end();
          EI != EE; ++EI) {
       ExplodedNodeSet checkDst;
-      CheckerContext C(checkDst, Eng.getBuilder(), Eng, Pred, EI->Checker,
-                       ProgramPoint::PostStmtKind, 0, CE);
-      bool evaluated = (*EI)(CE, C);
+      ProgramPoint::Kind K = ProgramPoint::PostStmtKind;
+      const ProgramPoint &L = ProgramPoint::getProgramPoint(CE, K,
+                                Pred->getLocationContext(), EI->Checker);
+      bool evaluated = false;
+      { // CheckerContext generates transitions(populates checkDest) on
+        // destruction, so introduce the scope to make sure it gets properly
+        // populated.
+        CheckerContext C(checkDst, Eng.getBuilder(), Eng, Pred, L, 0);
+        evaluated = (*EI)(CE, C);
+      }
       assert(!(evaluated && anyEvaluated)
              && "There are more than one checkers evaluating the call");
       if (evaluated) {
@@ -407,6 +463,7 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
       }
     }
     
+    // If none of the checkers evaluated the call, ask ExprEngine to handle it.
     if (!anyEvaluated) {
       if (defaultEval)
         defaultEval->expandGraph(Dst, Pred);
@@ -425,6 +482,14 @@ void CheckerManager::runCheckersOnEndOfTranslationUnit(
     EndOfTranslationUnitCheckers[i](TU, mgr, BR);
 }
 
+void CheckerManager::runCheckersForPrintState(raw_ostream &Out,
+                                              const ProgramState *State,
+                                              const char *NL, const char *Sep) {
+  for (llvm::DenseMap<CheckerTag, CheckerRef>::iterator
+        I = CheckerTags.begin(), E = CheckerTags.end(); I != E; ++I)
+    I->second->printState(Out, State, NL, Sep);
+}
+
 //===----------------------------------------------------------------------===//
 // Internal registration functions for AST traversing.
 //===----------------------------------------------------------------------===//
@@ -504,6 +569,10 @@ void CheckerManager::_registerForEvalCall(EvalCallFunc checkfn) {
   EvalCallCheckers.push_back(checkfn);
 }
 
+void CheckerManager::_registerForInlineCall(InlineCallFunc checkfn) {
+  InlineCallCheckers.push_back(checkfn);
+}
+
 void CheckerManager::_registerForEndOfTranslationUnit(
                                             CheckEndOfTranslationUnit checkfn) {
   EndOfTranslationUnitCheckers.push_back(checkfn);
@@ -541,8 +610,5 @@ CheckerManager::~CheckerManager() {
     CheckerDtors[i]();
 }
 
-// Anchor for the vtable.
-CheckerProvider::~CheckerProvider() { }
-
 // Anchor for the vtable.
 GraphExpander::~GraphExpander() { }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerRegistry.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerRegistry.cpp
new file mode 100644
index 000000000000..13401acf3da9
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CheckerRegistry.cpp
@@ -0,0 +1,149 @@
+//===--- CheckerRegistry.cpp - Maintains all available checkers -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerRegistry.h"
+#include "clang/StaticAnalyzer/Core/CheckerOptInfo.h"
+
+using namespace clang;
+using namespace ento;
+
+static const char PackageSeparator = '.';
+typedef llvm::DenseSet<const CheckerRegistry::CheckerInfo *> CheckerInfoSet;
+
+
+static bool checkerNameLT(const CheckerRegistry::CheckerInfo &a,
+                          const CheckerRegistry::CheckerInfo &b) {
+  return a.FullName < b.FullName;
+}
+
+static bool isInPackage(const CheckerRegistry::CheckerInfo &checker,
+                        StringRef packageName) {
+  // Does the checker's full name have the package as a prefix?
+  if (!checker.FullName.startswith(packageName))
+    return false;
+
+  // Is the package actually just the name of a specific checker?
+  if (checker.FullName.size() == packageName.size())
+    return true;
+
+  // Is the checker in the package (or a subpackage)?
+  if (checker.FullName[packageName.size()] == PackageSeparator)
+    return true;
+
+  return false;
+}
+
+static void collectCheckers(const CheckerRegistry::CheckerInfoList &checkers,
+                            const llvm::StringMap<size_t> &packageSizes,
+                            CheckerOptInfo &opt, CheckerInfoSet &collected) {
+  // Use a binary search to find the possible start of the package.
+  CheckerRegistry::CheckerInfo packageInfo(NULL, opt.getName(), "");
+  CheckerRegistry::CheckerInfoList::const_iterator e = checkers.end();
+  CheckerRegistry::CheckerInfoList::const_iterator i =
+    std::lower_bound(checkers.begin(), e, packageInfo, checkerNameLT);
+
+  // If we didn't even find a possible package, give up.
+  if (i == e)
+    return;
+
+  // If what we found doesn't actually start the package, give up.
+  if (!isInPackage(*i, opt.getName()))
+    return;
+
+  // There is at least one checker in the package; claim the option.
+  opt.claim();
+
+  // See how large the package is.
+  // If the package doesn't exist, assume the option refers to a single checker.
+  size_t size = 1;
+  llvm::StringMap<size_t>::const_iterator packageSize =
+    packageSizes.find(opt.getName());
+  if (packageSize != packageSizes.end())
+    size = packageSize->getValue();
+
+  // Step through all the checkers in the package.
+  for (e = i+size; i != e; ++i) {
+    if (opt.isEnabled())
+      collected.insert(&*i);
+    else
+      collected.erase(&*i);
+  }
+}
+
+void CheckerRegistry::addChecker(InitializationFunction fn, StringRef name,
+                                 StringRef desc) {
+  Checkers.push_back(CheckerInfo(fn, name, desc));
+
+  // Record the presence of the checker in its packages.
+  StringRef packageName, leafName;
+  llvm::tie(packageName, leafName) = name.rsplit(PackageSeparator);
+  while (!leafName.empty()) {
+    Packages[packageName] += 1;
+    llvm::tie(packageName, leafName) = packageName.rsplit(PackageSeparator);
+  }
+}
+
+void CheckerRegistry::initializeManager(CheckerManager &checkerMgr, 
+                                  SmallVectorImpl<CheckerOptInfo> &opts) const {
+  // Sort checkers for efficient collection.
+  std::sort(Checkers.begin(), Checkers.end(), checkerNameLT);
+
+  // Collect checkers enabled by the options.
+  CheckerInfoSet enabledCheckers;
+  for (SmallVectorImpl<CheckerOptInfo>::iterator
+         i = opts.begin(), e = opts.end(); i != e; ++i) {
+    collectCheckers(Checkers, Packages, *i, enabledCheckers);
+  }
+
+  // Initialize the CheckerManager with all enabled checkers.
+  for (CheckerInfoSet::iterator
+         i = enabledCheckers.begin(), e = enabledCheckers.end(); i != e; ++i) {
+    (*i)->Initialize(checkerMgr);
+  }
+}
+
+void CheckerRegistry::printHelp(llvm::raw_ostream &out,
+                                size_t maxNameChars) const {
+  // FIXME: Alphabetical sort puts 'experimental' in the middle.
+  // Would it be better to name it '~experimental' or something else
+  // that's ASCIIbetically last?
+  std::sort(Checkers.begin(), Checkers.end(), checkerNameLT);
+
+  // FIXME: Print available packages.
+
+  out << "CHECKERS:\n";
+
+  // Find the maximum option length.
+  size_t optionFieldWidth = 0;
+  for (CheckerInfoList::const_iterator i = Checkers.begin(), e = Checkers.end();
+       i != e; ++i) {
+    // Limit the amount of padding we are willing to give up for alignment.
+    //   Package.Name     Description  [Hidden]
+    size_t nameLength = i->FullName.size();
+    if (nameLength <= maxNameChars)
+      optionFieldWidth = std::max(optionFieldWidth, nameLength);
+  }
+
+  const size_t initialPad = 2;
+  for (CheckerInfoList::const_iterator i = Checkers.begin(), e = Checkers.end();
+       i != e; ++i) {
+    out.indent(initialPad) << i->FullName;
+
+    int pad = optionFieldWidth - i->FullName.size();
+
+    // Break on long option names.
+    if (pad < 0) {
+      out << '\n';
+      pad = optionFieldWidth + initialPad;
+    }
+    out.indent(pad + 2) << i->Desc;
+
+    out << '\n';
+  }
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
index 34cd6e8884a0..525219846a93 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
@@ -17,22 +17,12 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/Index/TranslationUnit.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/StmtCXX.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/ADT/DenseMap.h"
-
-using llvm::cast;
-using llvm::isa;
 using namespace clang;
 using namespace ento;
 
-// This should be removed in the future.
-namespace clang {
-namespace ento {
-TransferFuncs* MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
-                                  const LangOptions& lopts);
-}
-}
-
 //===----------------------------------------------------------------------===//
 // Worklist classes for exploration of reachable states.
 //===----------------------------------------------------------------------===//
@@ -41,7 +31,7 @@ WorkList::Visitor::~Visitor() {}
 
 namespace {
 class DFS : public WorkList {
-  llvm::SmallVector<WorkListUnit,20> Stack;
+  SmallVector<WorkListUnit,20> Stack;
 public:
   virtual bool hasWork() const {
     return !Stack.empty();
@@ -59,7 +49,7 @@ class DFS : public WorkList {
   }
   
   virtual bool visitItemsInWorkList(Visitor &V) {
-    for (llvm::SmallVectorImpl<WorkListUnit>::iterator
+    for (SmallVectorImpl<WorkListUnit>::iterator
          I = Stack.begin(), E = Stack.end(); I != E; ++I) {
       if (V.visit(*I))
         return true;
@@ -107,7 +97,7 @@ WorkList *WorkList::makeBFS() { return new BFS(); }
 namespace {
   class BFSBlockDFSContents : public WorkList {
     std::deque<WorkListUnit> Queue;
-    llvm::SmallVector<WorkListUnit,20> Stack;
+    SmallVector<WorkListUnit,20> Stack;
   public:
     virtual bool hasWork() const {
       return !Queue.empty() || !Stack.empty();
@@ -136,7 +126,7 @@ namespace {
       return U;
     }
     virtual bool visitItemsInWorkList(Visitor &V) {
-      for (llvm::SmallVectorImpl<WorkListUnit>::iterator
+      for (SmallVectorImpl<WorkListUnit>::iterator
            I = Stack.begin(), E = Stack.end(); I != E; ++I) {
         if (V.visit(*I))
           return true;
@@ -162,12 +152,12 @@ WorkList* WorkList::makeBFSBlockDFSContents() {
 
 /// ExecuteWorkList - Run the worklist algorithm for a maximum number of steps.
 bool CoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
-                                   const GRState *InitState) {
+                                   const ProgramState *InitState) {
 
   if (G->num_roots() == 0) { // Initialize the analysis by constructing
     // the root if none exists.
 
-    const CFGBlock* Entry = &(L->getCFG()->getEntry());
+    const CFGBlock *Entry = &(L->getCFG()->getEntry());
 
     assert (Entry->empty() &&
             "Entry block must be empty.");
@@ -176,7 +166,7 @@ bool CoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
             "Entry block must have 1 successor.");
 
     // Get the solitary successor.
-    const CFGBlock* Succ = *(Entry->succ_begin());
+    const CFGBlock *Succ = *(Entry->succ_begin());
 
     // Construct an edge representing the
     // starting location in the function.
@@ -208,7 +198,7 @@ bool CoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
     WList->setBlockCounter(WU.getBlockCounter());
 
     // Retrieve the node.
-    ExplodedNode* Node = WU.getNode();
+    ExplodedNode *Node = WU.getNode();
 
     // Dispatch on the location type.
     switch (Node->getLocation().getKind()) {
@@ -246,11 +236,11 @@ bool CoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
 }
 
 void CoreEngine::ExecuteWorkListWithInitialState(const LocationContext *L, 
-                                                   unsigned Steps,
-                                                   const GRState *InitState, 
-                                                   ExplodedNodeSet &Dst) {
+                                                 unsigned Steps,
+                                                 const ProgramState *InitState, 
+                                                 ExplodedNodeSet &Dst) {
   ExecuteWorkList(L, Steps, InitState);
-  for (llvm::SmallVectorImpl<ExplodedNode*>::iterator I = G->EndNodes.begin(), 
+  for (SmallVectorImpl<ExplodedNode*>::iterator I = G->EndNodes.begin(), 
                                            E = G->EndNodes.end(); I != E; ++I) {
     Dst.Add(*I);
   }
@@ -268,9 +258,9 @@ void CoreEngine::HandleCallExit(const CallExit &L, ExplodedNode *Pred) {
   SubEng.processCallExit(Builder);
 }
 
-void CoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
+void CoreEngine::HandleBlockEdge(const BlockEdge &L, ExplodedNode *Pred) {
 
-  const CFGBlock* Blk = L.getDst();
+  const CFGBlock *Blk = L.getDst();
 
   // Check if we are entering the EXIT block.
   if (Blk == &(L.getLocationContext()->getCFG()->getExit())) {
@@ -305,15 +295,15 @@ void CoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
     }
   }
 
-  for (llvm::SmallVectorImpl<ExplodedNode*>::const_iterator
+  for (SmallVectorImpl<ExplodedNode*>::const_iterator
        I = nodeBuilder.sinks().begin(), E = nodeBuilder.sinks().end();
        I != E; ++I) {
     blocksExhausted.push_back(std::make_pair(L, *I));
   }
 }
 
-void CoreEngine::HandleBlockEntrance(const BlockEntrance& L,
-                                       ExplodedNode* Pred) {
+void CoreEngine::HandleBlockEntrance(const BlockEntrance &L,
+                                       ExplodedNode *Pred) {
 
   // Increment the block counter.
   BlockCounter Counter = WList->getBlockCounter();
@@ -324,21 +314,19 @@ void CoreEngine::HandleBlockEntrance(const BlockEntrance& L,
 
   // Process the entrance of the block.
   if (CFGElement E = L.getFirstElement()) {
-    StmtNodeBuilder Builder(L.getBlock(), 0, Pred, this,
-                              SubEng.getStateManager());
+    StmtNodeBuilder Builder(L.getBlock(), 0, Pred, this);
     SubEng.processCFGElement(E, Builder);
   }
   else
     HandleBlockExit(L.getBlock(), Pred);
 }
 
-void CoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
+void CoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode *Pred) {
 
-  if (const Stmt* Term = B->getTerminator()) {
+  if (const Stmt *Term = B->getTerminator()) {
     switch (Term->getStmtClass()) {
       default:
-        assert(false && "Analysis for this terminator not implemented.");
-        break;
+        llvm_unreachable("Analysis for this terminator not implemented.");
 
       case Stmt::BinaryOperatorClass: // '&&' and '||'
         HandleBranch(cast<BinaryOperator>(Term)->getLHS(), Term, B, Pred);
@@ -361,6 +349,10 @@ void CoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
         HandleBranch(cast<DoStmt>(Term)->getCond(), Term, B, Pred);
         return;
 
+      case Stmt::CXXForRangeStmtClass:
+        HandleBranch(cast<CXXForRangeStmt>(Term)->getCond(), Term, B, Pred);
+        return;
+
       case Stmt::ForStmtClass:
         HandleBranch(cast<ForStmt>(Term)->getCond(), Term, B, Pred);
         return;
@@ -422,34 +414,34 @@ void CoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
                Pred->State, Pred);
 }
 
-void CoreEngine::HandleBranch(const Stmt* Cond, const Stmt* Term, 
-                                const CFGBlock * B, ExplodedNode* Pred) {
+void CoreEngine::HandleBranch(const Stmt *Cond, const Stmt *Term, 
+                                const CFGBlock * B, ExplodedNode *Pred) {
   assert(B->succ_size() == 2);
   BranchNodeBuilder Builder(B, *(B->succ_begin()), *(B->succ_begin()+1),
                             Pred, this);
   SubEng.processBranch(Cond, Term, Builder);
 }
 
-void CoreEngine::HandlePostStmt(const CFGBlock* B, unsigned StmtIdx, 
-                                  ExplodedNode* Pred) {
+void CoreEngine::HandlePostStmt(const CFGBlock *B, unsigned StmtIdx, 
+                                  ExplodedNode *Pred) {
   assert (!B->empty());
 
   if (StmtIdx == B->size())
     HandleBlockExit(B, Pred);
   else {
-    StmtNodeBuilder Builder(B, StmtIdx, Pred, this,
-                              SubEng.getStateManager());
+    StmtNodeBuilder Builder(B, StmtIdx, Pred, this);
     SubEng.processCFGElement((*B)[StmtIdx], Builder);
   }
 }
 
 /// generateNode - Utility method to generate nodes, hook up successors,
 ///  and add nodes to the worklist.
-void CoreEngine::generateNode(const ProgramPoint& Loc,
-                              const GRState* State, ExplodedNode* Pred) {
+void CoreEngine::generateNode(const ProgramPoint &Loc,
+                              const ProgramState *State,
+                              ExplodedNode *Pred) {
 
   bool IsNew;
-  ExplodedNode* Node = G->getNode(Loc, State, &IsNew);
+  ExplodedNode *Node = G->getNode(Loc, State, &IsNew);
 
   if (Pred)
     Node->addPredecessor(Pred, *G);  // Link 'Node' with its predecessor.
@@ -463,7 +455,7 @@ void CoreEngine::generateNode(const ProgramPoint& Loc,
 }
 
 ExplodedNode *
-GenericNodeBuilderImpl::generateNodeImpl(const GRState *state,
+GenericNodeBuilderImpl::generateNodeImpl(const ProgramState *state,
                                          ExplodedNode *pred,
                                          ProgramPoint programPoint,
                                          bool asSink) {
@@ -483,14 +475,14 @@ GenericNodeBuilderImpl::generateNodeImpl(const GRState *state,
   return 0;
 }
 
-StmtNodeBuilder::StmtNodeBuilder(const CFGBlock* b, unsigned idx,
-                                     ExplodedNode* N, CoreEngine* e,
-                                     GRStateManager &mgr)
-  : Eng(*e), B(*b), Idx(idx), Pred(N), Mgr(mgr),
+StmtNodeBuilder::StmtNodeBuilder(const CFGBlock *b,
+                                 unsigned idx,
+                                 ExplodedNode *N,
+                                 CoreEngine* e)
+  : Eng(*e), B(*b), Idx(idx), Pred(N),
     PurgingDeadSymbols(false), BuildSinks(false), hasGeneratedNode(false),
     PointKind(ProgramPoint::PostStmtKind), Tag(0) {
   Deferred.insert(N);
-  CleanedState = Pred->getState();
 }
 
 StmtNodeBuilder::~StmtNodeBuilder() {
@@ -499,7 +491,7 @@ StmtNodeBuilder::~StmtNodeBuilder() {
       GenerateAutoTransition(*I);
 }
 
-void StmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
+void StmtNodeBuilder::GenerateAutoTransition(ExplodedNode *N) {
   assert (!N->isSink());
 
   // Check if this node entered a callee.
@@ -526,18 +518,20 @@ void StmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   }
 
   bool IsNew;
-  ExplodedNode* Succ = Eng.G->getNode(Loc, N->State, &IsNew);
+  ExplodedNode *Succ = Eng.G->getNode(Loc, N->State, &IsNew);
   Succ->addPredecessor(N, *Eng.G);
 
   if (IsNew)
     Eng.WList->enqueue(Succ, &B, Idx+1);
 }
 
-ExplodedNode* StmtNodeBuilder::MakeNode(ExplodedNodeSet& Dst, const Stmt* S, 
-                                          ExplodedNode* Pred, const GRState* St,
-                                          ProgramPoint::Kind K) {
+ExplodedNode *StmtNodeBuilder::MakeNode(ExplodedNodeSet &Dst,
+                                        const Stmt *S, 
+                                        ExplodedNode *Pred,
+                                        const ProgramState *St,
+                                        ProgramPoint::Kind K) {
 
-  ExplodedNode* N = generateNode(S, St, Pred, K);
+  ExplodedNode *N = generateNode(S, St, Pred, K);
 
   if (N) {
     if (BuildSinks)
@@ -549,46 +543,24 @@ ExplodedNode* StmtNodeBuilder::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
   return N;
 }
 
-static ProgramPoint GetProgramPoint(const Stmt *S, ProgramPoint::Kind K,
-                                    const LocationContext *LC, const void *tag){
-  switch (K) {
-    default:
-      assert(false && "Unhandled ProgramPoint kind");    
-    case ProgramPoint::PreStmtKind:
-      return PreStmt(S, LC, tag);
-    case ProgramPoint::PostStmtKind:
-      return PostStmt(S, LC, tag);
-    case ProgramPoint::PreLoadKind:
-      return PreLoad(S, LC, tag);
-    case ProgramPoint::PostLoadKind:
-      return PostLoad(S, LC, tag);
-    case ProgramPoint::PreStoreKind:
-      return PreStore(S, LC, tag);
-    case ProgramPoint::PostStoreKind:
-      return PostStore(S, LC, tag);
-    case ProgramPoint::PostLValueKind:
-      return PostLValue(S, LC, tag);
-    case ProgramPoint::PostPurgeDeadSymbolsKind:
-      return PostPurgeDeadSymbols(S, LC, tag);
-  }
-}
-
 ExplodedNode*
-StmtNodeBuilder::generateNodeInternal(const Stmt* S, const GRState* state,
-                                        ExplodedNode* Pred,
-                                        ProgramPoint::Kind K,
-                                        const void *tag) {
+StmtNodeBuilder::generateNodeInternal(const Stmt *S,
+                                      const ProgramState *state,
+                                      ExplodedNode *Pred,
+                                      ProgramPoint::Kind K,
+                                      const ProgramPointTag *tag) {
   
-  const ProgramPoint &L = GetProgramPoint(S, K, Pred->getLocationContext(),tag);
+  const ProgramPoint &L = ProgramPoint::getProgramPoint(S, K,
+                                        Pred->getLocationContext(), tag);
   return generateNodeInternal(L, state, Pred);
 }
 
 ExplodedNode*
 StmtNodeBuilder::generateNodeInternal(const ProgramPoint &Loc,
-                                        const GRState* State,
-                                        ExplodedNode* Pred) {
+                                      const ProgramState *State,
+                                      ExplodedNode *Pred) {
   bool IsNew;
-  ExplodedNode* N = Eng.G->getNode(Loc, State, &IsNew);
+  ExplodedNode *N = Eng.G->getNode(Loc, State, &IsNew);
   N->addPredecessor(Pred, *Eng.G);
   Deferred.erase(Pred);
 
@@ -601,11 +573,11 @@ StmtNodeBuilder::generateNodeInternal(const ProgramPoint &Loc,
 }
 
 // This function generate a new ExplodedNode but not a new branch(block edge).
-ExplodedNode* BranchNodeBuilder::generateNode(const Stmt* Condition,
-                                              const GRState* State) {
+ExplodedNode *BranchNodeBuilder::generateNode(const Stmt *Condition,
+                                              const ProgramState *State) {
   bool IsNew;
   
-  ExplodedNode* Succ 
+  ExplodedNode *Succ 
     = Eng.G->getNode(PostCondition(Condition, Pred->getLocationContext()), State,
                      &IsNew);
   
@@ -619,8 +591,8 @@ ExplodedNode* BranchNodeBuilder::generateNode(const Stmt* Condition,
   return NULL;
 }
 
-ExplodedNode* BranchNodeBuilder::generateNode(const GRState* State,
-                                                bool branch) {
+ExplodedNode *BranchNodeBuilder::generateNode(const ProgramState *State,
+                                              bool branch) {
 
   // If the branch has been marked infeasible we should not generate a node.
   if (!isFeasible(branch))
@@ -628,7 +600,7 @@ ExplodedNode* BranchNodeBuilder::generateNode(const GRState* State,
 
   bool IsNew;
 
-  ExplodedNode* Succ =
+  ExplodedNode *Succ =
     Eng.G->getNode(BlockEdge(Src,branch ? DstT:DstF,Pred->getLocationContext()),
                    State, &IsNew);
 
@@ -657,11 +629,12 @@ BranchNodeBuilder::~BranchNodeBuilder() {
 
 
 ExplodedNode*
-IndirectGotoNodeBuilder::generateNode(const iterator& I, const GRState* St,
-                                        bool isSink) {
+IndirectGotoNodeBuilder::generateNode(const iterator &I,
+                                      const ProgramState *St,
+                                      bool isSink) {
   bool IsNew;
 
-  ExplodedNode* Succ = Eng.G->getNode(BlockEdge(Src, I.getBlock(),
+  ExplodedNode *Succ = Eng.G->getNode(BlockEdge(Src, I.getBlock(),
                                       Pred->getLocationContext()), St, &IsNew);
 
   Succ->addPredecessor(Pred, *Eng.G);
@@ -681,34 +654,38 @@ IndirectGotoNodeBuilder::generateNode(const iterator& I, const GRState* St,
 
 
 ExplodedNode*
-SwitchNodeBuilder::generateCaseStmtNode(const iterator& I, const GRState* St){
+SwitchNodeBuilder::generateCaseStmtNode(const iterator &I,
+                                        const ProgramState *St) {
 
   bool IsNew;
-
-  ExplodedNode* Succ = Eng.G->getNode(BlockEdge(Src, I.getBlock(),
-                                       Pred->getLocationContext()), St, &IsNew);
+  ExplodedNode *Succ = Eng.G->getNode(BlockEdge(Src, I.getBlock(),
+                                      Pred->getLocationContext()),
+                                      St, &IsNew);
   Succ->addPredecessor(Pred, *Eng.G);
-
   if (IsNew) {
     Eng.WList->enqueue(Succ);
     return Succ;
   }
-
   return NULL;
 }
 
 
 ExplodedNode*
-SwitchNodeBuilder::generateDefaultCaseNode(const GRState* St, bool isSink) {
-
+SwitchNodeBuilder::generateDefaultCaseNode(const ProgramState *St,
+                                           bool isSink) {
   // Get the block for the default case.
-  assert (Src->succ_rbegin() != Src->succ_rend());
-  CFGBlock* DefaultBlock = *Src->succ_rbegin();
+  assert(Src->succ_rbegin() != Src->succ_rend());
+  CFGBlock *DefaultBlock = *Src->succ_rbegin();
 
+  // Sanity check for default blocks that are unreachable and not caught
+  // by earlier stages.
+  if (!DefaultBlock)
+    return NULL;
+  
   bool IsNew;
 
-  ExplodedNode* Succ = Eng.G->getNode(BlockEdge(Src, DefaultBlock,
-                                       Pred->getLocationContext()), St, &IsNew);
+  ExplodedNode *Succ = Eng.G->getNode(BlockEdge(Src, DefaultBlock,
+                                      Pred->getLocationContext()), St, &IsNew);
   Succ->addPredecessor(Pred, *Eng.G);
 
   if (IsNew) {
@@ -735,12 +712,13 @@ EndOfFunctionNodeBuilder::~EndOfFunctionNodeBuilder() {
 }
 
 ExplodedNode*
-EndOfFunctionNodeBuilder::generateNode(const GRState* State,
-                                       ExplodedNode* P, const void *tag) {
+EndOfFunctionNodeBuilder::generateNode(const ProgramState *State,
+                                       ExplodedNode *P,
+                                       const ProgramPointTag *tag) {
   hasGeneratedNode = true;
   bool IsNew;
 
-  ExplodedNode* Node = Eng.G->getNode(BlockEntrance(&B,
+  ExplodedNode *Node = Eng.G->getNode(BlockEntrance(&B,
                                Pred->getLocationContext(), tag ? tag : Tag),
                                State, &IsNew);
 
@@ -754,7 +732,7 @@ EndOfFunctionNodeBuilder::generateNode(const GRState* State,
   return NULL;
 }
 
-void EndOfFunctionNodeBuilder::GenerateCallExitNode(const GRState *state) {
+void EndOfFunctionNodeBuilder::GenerateCallExitNode(const ProgramState *state) {
   hasGeneratedNode = true;
   // Create a CallExit node and enqueue it.
   const StackFrameContext *LocCtx
@@ -773,7 +751,7 @@ void EndOfFunctionNodeBuilder::GenerateCallExitNode(const GRState *state) {
 }
                                                 
 
-void CallEnterNodeBuilder::generateNode(const GRState *state) {
+void CallEnterNodeBuilder::generateNode(const ProgramState *state) {
   // Check if the callee is in the same translation unit.
   if (CalleeCtx->getTranslationUnit() != 
       Pred->getLocationContext()->getTranslationUnit()) {
@@ -787,30 +765,13 @@ void CallEnterNodeBuilder::generateNode(const GRState *state) {
 
     // Create a new AnalysisManager with components of the callee's
     // TranslationUnit.
-    // The Diagnostic is actually shared when we create ASTUnits from AST files.
-    AnalysisManager AMgr(TU->getASTContext(), TU->getDiagnostic(), 
-                         OldMgr.getLangOptions(), 
-                         OldMgr.getPathDiagnosticClient(),
-                         OldMgr.getStoreManagerCreator(),
-                         OldMgr.getConstraintManagerCreator(),
-                         OldMgr.getCheckerManager(),
-                         OldMgr.getIndexer(),
-                         OldMgr.getMaxNodes(), OldMgr.getMaxVisit(),
-                         OldMgr.shouldVisualizeGraphviz(),
-                         OldMgr.shouldVisualizeUbigraph(),
-                         OldMgr.shouldPurgeDead(),
-                         OldMgr.shouldEagerlyAssume(),
-                         OldMgr.shouldTrimGraph(),
-                         OldMgr.shouldInlineCall(),
-                     OldMgr.getAnalysisContextManager().getUseUnoptimizedCFG(),
-                     OldMgr.getAnalysisContextManager().getAddImplicitDtors(),
-                     OldMgr.getAnalysisContextManager().getAddInitializers(),
-                     OldMgr.shouldEagerlyTrimExplodedGraph());
-    llvm::OwningPtr<TransferFuncs> TF(MakeCFRefCountTF(AMgr.getASTContext(),
-                                                         /* GCEnabled */ false,
-                                                        AMgr.getLangOptions()));
+    // The Diagnostic is  actually shared when we create ASTUnits from AST files.
+    AnalysisManager AMgr(TU->getASTContext(), TU->getDiagnostic(), OldMgr);
+
     // Create the new engine.
-    ExprEngine NewEng(AMgr, TF.take());
+    // FIXME: This cast isn't really safe.
+    bool GCEnabled = static_cast<ExprEngine&>(Eng.SubEng).isObjCGCEnabled();
+    ExprEngine NewEng(AMgr, GCEnabled);
 
     // Create the new LocationContext.
     AnalysisContext *NewAnaCtx = AMgr.getAnalysisContext(CalleeCtx->getDecl(), 
@@ -823,8 +784,8 @@ void CallEnterNodeBuilder::generateNode(const GRState *state) {
                                                OldLocCtx->getIndex());
 
     // Now create an initial state for the new engine.
-    const GRState *NewState = NewEng.getStateManager().MarshalState(state,
-                                                                    NewLocCtx);
+    const ProgramState *NewState =
+      NewEng.getStateManager().MarshalState(state, NewLocCtx);
     ExplodedNodeSet ReturnNodes;
     NewEng.ExecuteWorkListWithInitialState(NewLocCtx, AMgr.getMaxNodes(), 
                                            NewState, ReturnNodes);
@@ -850,7 +811,7 @@ void CallEnterNodeBuilder::generateNode(const GRState *state) {
     Eng.WList->enqueue(Node);
 }
 
-void CallExitNodeBuilder::generateNode(const GRState *state) {
+void CallExitNodeBuilder::generateNode(const ProgramState *state) {
   // Get the callee's location context.
   const StackFrameContext *LocCtx 
                          = cast<StackFrameContext>(Pred->getLocationContext());
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Environment.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Environment.cpp
index 3961c7b95259..e1b982c08cfc 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Environment.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Environment.cpp
@@ -11,14 +11,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/ExprObjC.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/CFG.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 
 using namespace clang;
 using namespace ento;
 
-SVal Environment::lookupExpr(const Stmt* E) const {
+SVal Environment::lookupExpr(const Stmt *E) const {
   const SVal* X = ExprBindings.lookup(E);
   if (X) {
     SVal V = *X;
@@ -83,9 +84,9 @@ SVal Environment::getSVal(const Stmt *E, SValBuilder& svalBuilder,
       case Stmt::CXXBindTemporaryExprClass:
         E = cast<CXXBindTemporaryExpr>(E)->getSubExpr();
         continue;
-      case Stmt::MaterializeTemporaryExprClass:
-        E = cast<MaterializeTemporaryExpr>(E)->GetTemporaryExpr();
-        continue;
+      case Stmt::ObjCPropertyRefExprClass:
+        return loc::ObjCPropRef(cast<ObjCPropertyRefExpr>(E));
+        
       // Handle all other Stmt* using a lookup.
       default:
         break;
@@ -129,18 +130,6 @@ class MarkLiveCallback : public SymbolVisitor {
 };
 } // end anonymous namespace
 
-static bool isBlockExprInCallers(const Stmt *E, const LocationContext *LC) {
-  const LocationContext *ParentLC = LC->getParent();
-  while (ParentLC) {
-    CFG &C = *ParentLC->getCFG();
-    if (C.isBlkExpr(E))
-      return true;
-    ParentLC = ParentLC->getParent();
-  }
-
-  return false;
-}
-
 // In addition to mapping from Stmt * - > SVals in the Environment, we also
 // maintain a mapping from Stmt * -> SVals (locations) that were used during
 // a load and store.
@@ -158,24 +147,27 @@ static inline bool IsLocation(const Stmt *S) {
 Environment
 EnvironmentManager::removeDeadBindings(Environment Env,
                                        SymbolReaper &SymReaper,
-                                       const GRState *ST,
-                              llvm::SmallVectorImpl<const MemRegion*> &DRoots) {
-
-  CFG &C = *SymReaper.getLocationContext()->getCFG();
+                                       const ProgramState *ST) {
 
   // We construct a new Environment object entirely, as this is cheaper than
   // individually removing all the subexpression bindings (which will greatly
   // outnumber block-level expression bindings).
   Environment NewEnv = getInitialEnvironment();
   
-  llvm::SmallVector<std::pair<const Stmt*, SVal>, 10> deferredLocations;
+  SmallVector<std::pair<const Stmt*, SVal>, 10> deferredLocations;
+
+  MarkLiveCallback CB(SymReaper);
+  ScanReachableSymbols RSScaner(ST, CB);
+
+  llvm::ImmutableMapRef<const Stmt*,SVal>
+    EBMapRef(NewEnv.ExprBindings.getRootWithoutRetain(),
+             F.getTreeFactory());
 
   // Iterate over the block-expr bindings.
   for (Environment::iterator I = Env.begin(), E = Env.end();
        I != E; ++I) {
 
     const Stmt *BlkExpr = I.getKey();
-    
     // For recorded locations (used when evaluating loads and stores), we
     // consider them live only when their associated normal expression is
     // also live.
@@ -185,41 +177,20 @@ EnvironmentManager::removeDeadBindings(Environment Env,
       deferredLocations.push_back(std::make_pair(BlkExpr, I.getData()));
       continue;
     }
-    
     const SVal &X = I.getData();
 
-    // Block-level expressions in callers are assumed always live.
-    if (isBlockExprInCallers(BlkExpr, SymReaper.getLocationContext())) {
-      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, BlkExpr, X);
-
-      if (isa<loc::MemRegionVal>(X)) {
-        const MemRegion* R = cast<loc::MemRegionVal>(X).getRegion();
-        DRoots.push_back(R);
-      }
-
-      // Mark all symbols in the block expr's value live.
-      MarkLiveCallback cb(SymReaper);
-      ST->scanReachableSymbols(X, cb);
-      continue;
-    }
-
-    // Not a block-level expression?
-    if (!C.isBlkExpr(BlkExpr))
-      continue;
-
     if (SymReaper.isLive(BlkExpr)) {
       // Copy the binding to the new map.
-      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, BlkExpr, X);
+      EBMapRef = EBMapRef.add(BlkExpr, X);
 
       // If the block expr's value is a memory region, then mark that region.
       if (isa<loc::MemRegionVal>(X)) {
-        const MemRegion* R = cast<loc::MemRegionVal>(X).getRegion();
-        DRoots.push_back(R);
+        const MemRegion *R = cast<loc::MemRegionVal>(X).getRegion();
+        SymReaper.markLive(R);
       }
 
       // Mark all symbols in the block expr's value live.
-      MarkLiveCallback cb(SymReaper);
-      ST->scanReachableSymbols(X, cb);
+      RSScaner.scan(X);
       continue;
     }
 
@@ -228,17 +199,18 @@ EnvironmentManager::removeDeadBindings(Environment Env,
     // beginning of itself, but we need its UndefinedVal to determine its
     // SVal.
     if (X.isUndef() && cast<UndefinedVal>(X).getData())
-      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, BlkExpr, X);
+      EBMapRef = EBMapRef.add(BlkExpr, X);
   }
   
   // Go through he deferred locations and add them to the new environment if
   // the correspond Stmt* is in the map as well.
-  for (llvm::SmallVectorImpl<std::pair<const Stmt*, SVal> >::iterator
+  for (SmallVectorImpl<std::pair<const Stmt*, SVal> >::iterator
       I = deferredLocations.begin(), E = deferredLocations.end(); I != E; ++I) {
     const Stmt *S = (Stmt*) (((uintptr_t) I->first) & (uintptr_t) ~0x1);
-    if (NewEnv.ExprBindings.lookup(S))
-      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, I->first, I->second);
+    if (EBMapRef.lookup(S))
+      EBMapRef = EBMapRef.add(I->first, I->second);
   }
 
+  NewEnv.ExprBindings = EBMapRef.asImmutableMap();
   return NewEnv;
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
index fa16fead9f1d..5762a21600e4 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/AST/Stmt.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/DenseMap.h"
@@ -93,18 +93,17 @@ void ExplodedGraph::reclaimRecentlyAllocatedNodes() {
     ProgramPoint progPoint = node->getLocation();
     if (!isa<PostStmt>(progPoint))
       continue;
-
     // Condition 4.
     PostStmt ps = cast<PostStmt>(progPoint);
-    if (ps.getTag() || isa<PostStmtCustom>(ps))
+    if (ps.getTag())
       continue;
 
     if (isa<BinaryOperator>(ps.getStmt()))
       continue;
 
     // Conditions 5, 6, and 7.
-    const GRState *state = node->getState();
-    const GRState *pred_state = pred->getState();    
+    const ProgramState *state = node->getState();
+    const ProgramState *pred_state = pred->getState();    
     if (state->store != pred_state->store || state->GDM != pred_state->GDM ||
         progPoint.getLocationContext() != pred->getLocationContext())
       continue;
@@ -134,11 +133,11 @@ void ExplodedGraph::reclaimRecentlyAllocatedNodes() {
 // ExplodedNode.
 //===----------------------------------------------------------------------===//
 
-static inline BumpVector<ExplodedNode*>& getVector(void* P) {
+static inline BumpVector<ExplodedNode*>& getVector(void *P) {
   return *reinterpret_cast<BumpVector<ExplodedNode*>*>(P);
 }
 
-void ExplodedNode::addPredecessor(ExplodedNode* V, ExplodedGraph &G) {
+void ExplodedNode::addPredecessor(ExplodedNode *V, ExplodedGraph &G) {
   assert (!V->isSink());
   Preds.addNode(V, G);
   V->Succs.addNode(this, G);
@@ -153,12 +152,12 @@ void ExplodedNode::NodeGroup::replaceNode(ExplodedNode *node) {
   assert(getKind() == Size1);
 }
 
-void ExplodedNode::NodeGroup::addNode(ExplodedNode* N, ExplodedGraph &G) {
+void ExplodedNode::NodeGroup::addNode(ExplodedNode *N, ExplodedGraph &G) {
   assert((reinterpret_cast<uintptr_t>(N) & Mask) == 0x0);
   assert(!getFlag());
 
   if (getKind() == Size1) {
-    if (ExplodedNode* NOld = getNode()) {
+    if (ExplodedNode *NOld = getNode()) {
       BumpVectorContext &Ctx = G.getNodeAllocator();
       BumpVector<ExplodedNode*> *V = 
         G.getAllocator().Allocate<BumpVector<ExplodedNode*> >();
@@ -215,11 +214,11 @@ ExplodedNode** ExplodedNode::NodeGroup::end() const {
   }
 }
 
-ExplodedNode *ExplodedGraph::getNode(const ProgramPoint& L,
-                                     const GRState* State, bool* IsNew) {
+ExplodedNode *ExplodedGraph::getNode(const ProgramPoint &L,
+                                     const ProgramState *State, bool* IsNew) {
   // Profile 'State' to determine if we already have an existing node.
   llvm::FoldingSetNodeID profile;
-  void* InsertPos = 0;
+  void *InsertPos = 0;
 
   NodeTy::Profile(profile, L, State);
   NodeTy* V = Nodes.FindNodeOrInsertPos(profile, InsertPos);
@@ -285,7 +284,7 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
   typedef llvm::DenseMap<const ExplodedNode*, ExplodedNode*> Pass2Ty;
   Pass2Ty& Pass2 = M->M;
 
-  llvm::SmallVector<const ExplodedNode*, 10> WL1, WL2;
+  SmallVector<const ExplodedNode*, 10> WL1, WL2;
 
   // ===- Pass 1 (reverse DFS) -===
   for (const ExplodedNode* const* I = BeginSources; I != EndSources; ++I) {
@@ -326,7 +325,7 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
 
   // ===- Pass 2 (forward DFS to construct the new graph) -===
   while (!WL2.empty()) {
-    const ExplodedNode* N = WL2.back();
+    const ExplodedNode *N = WL2.back();
     WL2.pop_back();
 
     // Skip this node if we have already processed it.
@@ -335,7 +334,7 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
 
     // Create the corresponding node in the new graph and record the mapping
     // from the old node to the new node.
-    ExplodedNode* NewN = G->getNode(N->getLocation(), N->State, NULL);
+    ExplodedNode *NewN = G->getNode(N->getLocation(), N->State, NULL);
     Pass2[N] = NewN;
 
     // Also record the reverse mapping from the new node to the old node.
@@ -383,7 +382,7 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
 }
 
 ExplodedNode*
-InterExplodedGraphMap::getMappedNode(const ExplodedNode* N) const {
+InterExplodedGraphMap::getMappedNode(const ExplodedNode *N) const {
   llvm::DenseMap<const ExplodedNode*, ExplodedNode*>::const_iterator I =
     M.find(N);
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index ffe5f0b6cdf2..ac9cf0b08d44 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -18,6 +18,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/ParentMap.h"
 #include "clang/AST/StmtObjC.h"
@@ -35,25 +36,13 @@
 
 using namespace clang;
 using namespace ento;
-using llvm::dyn_cast;
-using llvm::dyn_cast_or_null;
-using llvm::cast;
 using llvm::APSInt;
 
-namespace {
-  // Trait class for recording returned expression in the state.
-  struct ReturnExpr {
-    static int TagInt;
-    typedef const Stmt *data_type;
-  };
-  int ReturnExpr::TagInt; 
-}
-
 //===----------------------------------------------------------------------===//
 // Utility functions.
 //===----------------------------------------------------------------------===//
 
-static inline Selector GetNullarySelector(const char* name, ASTContext& Ctx) {
+static inline Selector GetNullarySelector(const char* name, ASTContext &Ctx) {
   IdentifierInfo* II = &Ctx.Idents.get(name);
   return Ctx.Selectors.getSelector(0, &II);
 }
@@ -62,7 +51,7 @@ static inline Selector GetNullarySelector(const char* name, ASTContext& Ctx) {
 // Engine construction and deletion.
 //===----------------------------------------------------------------------===//
 
-ExprEngine::ExprEngine(AnalysisManager &mgr, TransferFuncs *tf)
+ExprEngine::ExprEngine(AnalysisManager &mgr, bool gcEnabled)
   : AMgr(mgr),
     Engine(*this),
     G(Engine.getGraph()),
@@ -75,11 +64,7 @@ ExprEngine::ExprEngine(AnalysisManager &mgr, TransferFuncs *tf)
     EntryNode(NULL), currentStmt(NULL),
     NSExceptionII(NULL), NSExceptionInstanceRaiseSelectors(NULL),
     RaiseSel(GetNullarySelector("raise", getContext())),
-    BR(mgr, *this), TF(tf) {
-
-  // FIXME: Eventually remove the TF object entirely.
-  TF->RegisterChecks(*this);
-  TF->RegisterPrinters(getStateManager().Printers);
+    ObjCGCEnabled(gcEnabled), BR(mgr, *this) {
   
   if (mgr.shouldEagerlyTrimExplodedGraph()) {
     // Enable eager node reclaimation when constructing the ExplodedGraph.  
@@ -96,8 +81,8 @@ ExprEngine::~ExprEngine() {
 // Utility methods.
 //===----------------------------------------------------------------------===//
 
-const GRState* ExprEngine::getInitialState(const LocationContext *InitLoc) {
-  const GRState *state = StateMgr.getInitialState(InitLoc);
+const ProgramState *ExprEngine::getInitialState(const LocationContext *InitLoc) {
+  const ProgramState *state = StateMgr.getInitialState(InitLoc);
 
   // Preconditions.
 
@@ -132,7 +117,7 @@ const GRState* ExprEngine::getInitialState(const LocationContext *InitLoc) {
       if (!Constraint)
         break;
 
-      if (const GRState *newState = state->assume(*Constraint, true))
+      if (const ProgramState *newState = state->assume(*Constraint, true))
         state = newState;
 
       break;
@@ -162,11 +147,11 @@ ExprEngine::doesInvalidateGlobals(const CallOrObjCMessage &callOrMessage) const
   if (callOrMessage.isFunctionCall() && !callOrMessage.isCXXCall()) {
     SVal calleeV = callOrMessage.getFunctionCallee();
     if (const FunctionTextRegion *codeR =
-          llvm::dyn_cast_or_null<FunctionTextRegion>(calleeV.getAsRegion())) {
+          dyn_cast_or_null<FunctionTextRegion>(calleeV.getAsRegion())) {
       
       const FunctionDecl *fd = codeR->getDecl();
       if (const IdentifierInfo *ii = fd->getIdentifier()) {
-        llvm::StringRef fname = ii->getName();
+        StringRef fname = ii->getName();
         if (fname == "strlen")
           return false;
       }
@@ -183,28 +168,27 @@ ExprEngine::doesInvalidateGlobals(const CallOrObjCMessage &callOrMessage) const
 
 /// evalAssume - Called by ConstraintManager. Used to call checker-specific
 ///  logic for handling assumptions on symbolic values.
-const GRState *ExprEngine::processAssume(const GRState *state, SVal cond,
-                                           bool assumption) {
-  state = getCheckerManager().runCheckersForEvalAssume(state, cond, assumption);
-
-  // If the state is infeasible at this point, bail out.
-  if (!state)
-    return NULL;
-
-  return TF->evalAssume(state, cond, assumption);
+const ProgramState *ExprEngine::processAssume(const ProgramState *state,
+                                              SVal cond, bool assumption) {
+  return getCheckerManager().runCheckersForEvalAssume(state, cond, assumption);
 }
 
-bool ExprEngine::wantsRegionChangeUpdate(const GRState* state) {
+bool ExprEngine::wantsRegionChangeUpdate(const ProgramState *state) {
   return getCheckerManager().wantsRegionChangeUpdate(state);
 }
 
-const GRState *
-ExprEngine::processRegionChanges(const GRState *state,
+const ProgramState *
+ExprEngine::processRegionChanges(const ProgramState *state,
                             const StoreManager::InvalidatedSymbols *invalidated,
-                                 const MemRegion * const *Begin,
-                                 const MemRegion * const *End) {
+                                 ArrayRef<const MemRegion *> Explicits,
+                                 ArrayRef<const MemRegion *> Regions) {
   return getCheckerManager().runCheckersForRegionChanges(state, invalidated,
-                                                         Begin, End);
+                                                         Explicits, Regions);
+}
+
+void ExprEngine::printState(raw_ostream &Out, const ProgramState *State,
+                            const char *NL, const char *Sep) {
+  getCheckerManager().runCheckersForPrintState(Out, State, NL, Sep);
 }
 
 void ExprEngine::processEndWorklist(bool hasWorkRemaining) {
@@ -217,7 +201,7 @@ void ExprEngine::processCFGElement(const CFGElement E,
     case CFGElement::Invalid:
       llvm_unreachable("Unexpected CFGElement kind.");
     case CFGElement::Statement:
-      ProcessStmt(E.getAs<CFGStmt>()->getStmt(), builder);
+      ProcessStmt(const_cast<Stmt*>(E.getAs<CFGStmt>()->getStmt()), builder);
       return;
     case CFGElement::Initializer:
       ProcessInitializer(E.getAs<CFGInitializer>()->getInitializer(), builder);
@@ -232,9 +216,12 @@ void ExprEngine::processCFGElement(const CFGElement E,
 }
 
 void ExprEngine::ProcessStmt(const CFGStmt S, StmtNodeBuilder& builder) {
+  // TODO: Use RAII to remove the unnecessary, tagged nodes.
+  //RegisterCreatedNodes registerCreatedNodes(getGraph());
+
   // Reclaim any unnecessary nodes in the ExplodedGraph.
   G.reclaimRecentlyAllocatedNodes();
-  // Recycle any unused states in the GRStateManager.
+  // Recycle any unused states in the ProgramStateManager.
   StateMgr.recycleUnusedStates();
   
   currentStmt = S.getStmt();
@@ -242,74 +229,93 @@ void ExprEngine::ProcessStmt(const CFGStmt S, StmtNodeBuilder& builder) {
                                 currentStmt->getLocStart(),
                                 "Error evaluating statement");
 
+  // A tag to track convenience transitions, which can be removed at cleanup.
+  static SimpleProgramPointTag cleanupTag("ExprEngine : Clean Node");
   Builder = &builder;
   EntryNode = builder.getPredecessor();
 
+  const ProgramState *EntryState = EntryNode->getState();
+  CleanedState = EntryState;
+  ExplodedNode *CleanedNode = 0;
+
   // Create the cleaned state.
   const LocationContext *LC = EntryNode->getLocationContext();
-  SymbolReaper SymReaper(LC, currentStmt, SymMgr);
+  SymbolReaper SymReaper(LC, currentStmt, SymMgr, getStoreManager());
 
-  if (AMgr.shouldPurgeDead()) {
-    const GRState *St = EntryNode->getState();
-    getCheckerManager().runCheckersForLiveSymbols(St, SymReaper);
+  if (AMgr.getPurgeMode() != PurgeNone) {
+    getCheckerManager().runCheckersForLiveSymbols(CleanedState, SymReaper);
 
     const StackFrameContext *SFC = LC->getCurrentStackFrame();
-    CleanedState = StateMgr.removeDeadBindings(St, SFC, SymReaper);
-  } else {
-    CleanedState = EntryNode->getState();
+
+    // Create a state in which dead bindings are removed from the environment
+    // and the store. TODO: The function should just return new env and store,
+    // not a new state.
+    CleanedState = StateMgr.removeDeadBindings(CleanedState, SFC, SymReaper);
   }
 
   // Process any special transfer function for dead symbols.
   ExplodedNodeSet Tmp;
+  if (!SymReaper.hasDeadSymbols()) {
+    // Generate a CleanedNode that has the environment and store cleaned
+    // up. Since no symbols are dead, we can optimize and not clean out
+    // the constraint manager.
+    CleanedNode =
+      Builder->generateNode(currentStmt, CleanedState, EntryNode, &cleanupTag);
+    Tmp.Add(CleanedNode);
 
-  if (!SymReaper.hasDeadSymbols())
-    Tmp.Add(EntryNode);
-  else {
+  } else {
     SaveAndRestore<bool> OldSink(Builder->BuildSinks);
     SaveOr OldHasGen(Builder->hasGeneratedNode);
 
     SaveAndRestore<bool> OldPurgeDeadSymbols(Builder->PurgingDeadSymbols);
     Builder->PurgingDeadSymbols = true;
 
-    // FIXME: This should soon be removed.
-    ExplodedNodeSet Tmp2;
-    getTF().evalDeadSymbols(Tmp2, *this, *Builder, EntryNode,
-                            CleanedState, SymReaper);
-
-    getCheckerManager().runCheckersForDeadSymbols(Tmp, Tmp2,
+    // Call checkers with the non-cleaned state so that they could query the
+    // values of the soon to be dead symbols.
+    ExplodedNodeSet CheckedSet;
+    getCheckerManager().runCheckersForDeadSymbols(CheckedSet, EntryNode,
                                                  SymReaper, currentStmt, *this);
 
-    if (!Builder->BuildSinks && !Builder->hasGeneratedNode)
-      Tmp.Add(EntryNode);
+    // For each node in CheckedSet, generate CleanedNodes that have the
+    // environment, the store, and the constraints cleaned up but have the
+    // user-supplied states as the predecessors.
+    for (ExplodedNodeSet::const_iterator
+          I = CheckedSet.begin(), E = CheckedSet.end(); I != E; ++I) {
+      const ProgramState *CheckerState = (*I)->getState();
+
+      // The constraint manager has not been cleaned up yet, so clean up now.
+      CheckerState = getConstraintManager().removeDeadBindings(CheckerState,
+                                                               SymReaper);
+
+      assert(StateMgr.haveEqualEnvironments(CheckerState, EntryState) &&
+        "Checkers are not allowed to modify the Environment as a part of "
+        "checkDeadSymbols processing.");
+      assert(StateMgr.haveEqualStores(CheckerState, EntryState) &&
+        "Checkers are not allowed to modify the Store as a part of "
+        "checkDeadSymbols processing.");
+
+      // Create a state based on CleanedState with CheckerState GDM and
+      // generate a transition to that state.
+      const ProgramState *CleanedCheckerSt =
+        StateMgr.getPersistentStateWithGDM(CleanedState, CheckerState);
+      ExplodedNode *CleanedNode = Builder->generateNode(currentStmt,
+                                                        CleanedCheckerSt, *I,
+                                                        &cleanupTag);
+      Tmp.Add(CleanedNode);
+    }
   }
 
-  bool HasAutoGenerated = false;
-
   for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
+    // TODO: Remove Dest set, it's no longer needed.
     ExplodedNodeSet Dst;
-
-    // Set the cleaned state.
-    Builder->SetCleanedState(*I == EntryNode ? CleanedState : GetState(*I));
-
     // Visit the statement.
     Visit(currentStmt, *I, Dst);
-
-    // Do we need to auto-generate a node?  We only need to do this to generate
-    // a node with a "cleaned" state; CoreEngine will actually handle
-    // auto-transitions for other cases.
-    if (Dst.size() == 1 && *Dst.begin() == EntryNode
-        && !Builder->hasGeneratedNode && !HasAutoGenerated) {
-      HasAutoGenerated = true;
-      builder.generateNode(currentStmt, GetState(EntryNode), *I);
-    }
   }
 
   // NULL out these variables to cleanup.
   CleanedState = NULL;
   EntryNode = NULL;
-
   currentStmt = 0;
-
   Builder = NULL;
 }
 
@@ -334,7 +340,7 @@ void ExprEngine::ProcessInitializer(const CFGInitializer Init,
 
     for (ExplodedNodeSet::iterator I = Dst.begin(), E = Dst.end(); I != E; ++I){
       ExplodedNode *Pred = *I;
-      const GRState *state = Pred->getState();
+      const ProgramState *state = Pred->getState();
 
       const FieldDecl *FD = BMI->getAnyMember();
 
@@ -391,7 +397,7 @@ void ExprEngine::ProcessImplicitDtor(const CFGImplicitDtor D,
 void ExprEngine::ProcessAutomaticObjDtor(const CFGAutomaticObjDtor dtor,
                                            StmtNodeBuilder &builder) {
   ExplodedNode *pred = builder.getPredecessor();
-  const GRState *state = pred->getState();
+  const ProgramState *state = pred->getState();
   const VarDecl *varDecl = dtor.getVarDecl();
 
   QualType varType = varDecl->getType();
@@ -422,8 +428,8 @@ void ExprEngine::ProcessTemporaryDtor(const CFGTemporaryDtor D,
                                         StmtNodeBuilder &builder) {
 }
 
-void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred, 
-                         ExplodedNodeSet& Dst) {
+void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred, 
+                         ExplodedNodeSet &Dst) {
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
                                 S->getLocStart(),
                                 "Error evaluating statement");
@@ -447,9 +453,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::CXXBindTemporaryExprClass:
     case Stmt::CXXCatchStmtClass:
     case Stmt::CXXDependentScopeMemberExprClass:
-    case Stmt::CXXForRangeStmtClass:
     case Stmt::CXXPseudoDestructorExprClass:
-    case Stmt::CXXTemporaryObjectExprClass:
     case Stmt::CXXThrowExprClass:
     case Stmt::CXXTryStmtClass:
     case Stmt::CXXTypeidExprClass:
@@ -472,7 +476,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     {
       SaveAndRestore<bool> OldSink(Builder->BuildSinks);
       Builder->BuildSinks = true;
-      const ExplodedNode *node = MakeNode(Dst, S, Pred, GetState(Pred));
+      const ExplodedNode *node = MakeNode(Dst, S, Pred, Pred->getState());
       Engine.addAbortedBlock(node, Builder->getBlock());
       break;
     }
@@ -495,6 +499,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::CaseStmtClass:
     case Stmt::CompoundStmtClass:
     case Stmt::ContinueStmtClass:
+    case Stmt::CXXForRangeStmtClass:
     case Stmt::DefaultStmtClass:
     case Stmt::DoStmtClass:
     case Stmt::ForStmtClass:
@@ -511,7 +516,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
 
     case Stmt::GNUNullExprClass: {
       // GNU __null is a pointer-width integer, not an actual pointer.
-      const GRState *state = GetState(Pred);
+      const ProgramState *state = Pred->getState();
       state = state->BindExpr(S, svalBuilder.makeIntValWithPtrWidth(0, false));
       MakeNode(Dst, S, Pred, state);
       break;
@@ -522,11 +527,12 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
 
     case Stmt::ObjCPropertyRefExprClass:
-      VisitObjCPropertyRefExpr(cast<ObjCPropertyRefExpr>(S), Pred, Dst);
+      // Implicitly handled by Environment::getSVal().
+      Dst.Add(Pred);
       break;
 
     case Stmt::ImplicitValueInitExprClass: {
-      const GRState *state = GetState(Pred);
+      const ProgramState *state = Pred->getState();
       QualType ty = cast<ImplicitValueInitExpr>(S)->getType();
       SVal val = svalBuilder.makeZeroVal(ty);
       MakeNode(Dst, S, Pred, state->BindExpr(S, val));
@@ -558,6 +564,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::CUDAKernelCallExprClass:
     case Stmt::OpaqueValueExprClass:
     case Stmt::AsTypeExprClass:
+    case Stmt::AtomicExprClass:
         // Fall through.
 
     // Cases we intentionally don't evaluate, since they don't need
@@ -597,7 +604,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
         break;
       }
       else if (B->getOpcode() == BO_Comma) {
-        const GRState* state = GetState(Pred);
+        const ProgramState *state = Pred->getState();
         MakeNode(Dst, B, Pred, state->BindExpr(B, state->getSVal(B->getRHS())));
         break;
       }
@@ -621,6 +628,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
     }
 
+    case Stmt::CXXTemporaryObjectExprClass:
     case Stmt::CXXConstructExprClass: {
       const CXXConstructExpr *C = cast<CXXConstructExpr>(S);
       // For block-level CXXConstructExpr, we don't have a destination region.
@@ -644,7 +652,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       //        the CFG do not model them as explicit control-flow.
 
     case Stmt::ChooseExprClass: { // __builtin_choose_expr
-      const ChooseExpr* C = cast<ChooseExpr>(S);
+      const ChooseExpr *C = cast<ChooseExpr>(S);
       VisitGuardedExpr(C, C->getLHS(), C->getRHS(), Pred, Dst);
       break;
     }
@@ -687,7 +695,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::CXXConstCastExprClass:
     case Stmt::CXXFunctionalCastExprClass: 
     case Stmt::ObjCBridgedCastExprClass: {
-      const CastExpr* C = cast<CastExpr>(S);
+      const CastExpr *C = cast<CastExpr>(S);
       // Handle the previsit checks.
       ExplodedNodeSet dstPrevisit;
       getCheckerManager().runCheckersForPreStmt(dstPrevisit, Pred, C, *this);
@@ -708,7 +716,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       const MaterializeTemporaryExpr *Materialize
                                             = cast<MaterializeTemporaryExpr>(S);
       if (!Materialize->getType()->isRecordType())
-        CreateCXXTemporaryObject(Materialize->GetTemporaryExpr(), Pred, Dst);
+        CreateCXXTemporaryObject(Materialize, Pred, Dst);
       else
         Visit(Materialize->GetTemporaryExpr(), Pred, Dst);
       break;
@@ -730,7 +738,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
 
     case Stmt::ObjCMessageExprClass:
-      VisitObjCMessageExpr(cast<ObjCMessageExpr>(S), Pred, Dst);
+      VisitObjCMessage(cast<ObjCMessageExpr>(S), Pred, Dst);
       break;
 
     case Stmt::ObjCAtThrowStmtClass: {
@@ -738,7 +746,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       // an abort.
       SaveAndRestore<bool> OldSink(Builder->BuildSinks);
       Builder->BuildSinks = true;
-      MakeNode(Dst, S, Pred, GetState(Pred));
+      MakeNode(Dst, S, Pred, Pred->getState());
       break;
     }
 
@@ -756,7 +764,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
 
     case Stmt::StmtExprClass: {
-      const StmtExpr* SE = cast<StmtExpr>(S);
+      const StmtExpr *SE = cast<StmtExpr>(S);
 
       if (SE->getSubStmt()->body_empty()) {
         // Empty statement expression.
@@ -766,8 +774,8 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
         break;
       }
 
-      if (Expr* LastExpr = dyn_cast<Expr>(*SE->getSubStmt()->body_rbegin())) {
-        const GRState* state = GetState(Pred);
+      if (Expr *LastExpr = dyn_cast<Expr>(*SE->getSubStmt()->body_rbegin())) {
+        const ProgramState *state = Pred->getState();
         MakeNode(Dst, SE, Pred, state->BindExpr(SE, state->getSVal(LastExpr)));
       }
       else
@@ -777,7 +785,7 @@ void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     }
 
     case Stmt::StringLiteralClass: {
-      const GRState* state = GetState(Pred);
+      const ProgramState *state = Pred->getState();
       SVal V = state->getLValue(cast<StringLiteral>(S));
       MakeNode(Dst, S, Pred, state->BindExpr(S, V));
       return;
@@ -811,8 +819,7 @@ void ExprEngine::processCFGBlockEntrance(ExplodedNodeSet &dstNodes,
   if (nodeBuilder.getBlockCounter().getNumVisited(
                        pred->getLocationContext()->getCurrentStackFrame(), 
                        block->getBlockID()) >= AMgr.getMaxVisit()) {
-
-    static int tag = 0;
+    static SimpleProgramPointTag tag("ExprEngine : Block count exceeded");
     nodeBuilder.generateNode(pred->getState(), pred, &tag, true);
   }
 }
@@ -821,11 +828,12 @@ void ExprEngine::processCFGBlockEntrance(ExplodedNodeSet &dstNodes,
 // Generic node creation.
 //===----------------------------------------------------------------------===//
 
-ExplodedNode* ExprEngine::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
-                                     ExplodedNode* Pred, const GRState* St,
-                                     ProgramPoint::Kind K, const void *tag) {
+ExplodedNode *ExprEngine::MakeNode(ExplodedNodeSet &Dst, const Stmt *S,
+                                   ExplodedNode *Pred, const ProgramState *St,
+                                   ProgramPoint::Kind K,
+                                   const ProgramPointTag *tag) {
   assert (Builder && "StmtNodeBuilder not present.");
-  SaveAndRestore<const void*> OldTag(Builder->Tag);
+  SaveAndRestore<const ProgramPointTag*> OldTag(Builder->Tag);
   Builder->Tag = tag;
   return Builder->MakeNode(Dst, S, Pred, St, K);
 }
@@ -834,8 +842,8 @@ ExplodedNode* ExprEngine::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
 // Branch processing.
 //===----------------------------------------------------------------------===//
 
-const GRState* ExprEngine::MarkBranch(const GRState* state,
-                                        const Stmt* Terminator,
+const ProgramState *ExprEngine::MarkBranch(const ProgramState *state,
+                                        const Stmt *Terminator,
                                         bool branchTaken) {
 
   switch (Terminator->getStmtClass()) {
@@ -855,7 +863,7 @@ const GRState* ExprEngine::MarkBranch(const GRState* state,
       // For ||, if we take the false branch, then the value of the whole
       // expression is that of the RHS expression.
 
-      const Expr* Ex = (Op == BO_LAnd && branchTaken) ||
+      const Expr *Ex = (Op == BO_LAnd && branchTaken) ||
                        (Op == BO_LOr && !branchTaken)
                        ? B->getRHS() : B->getLHS();
 
@@ -870,7 +878,7 @@ const GRState* ExprEngine::MarkBranch(const GRState* state,
       // For ?, if branchTaken == true then the value is either the LHS or
       // the condition itself. (GNU extension).
 
-      const Expr* Ex;
+      const Expr *Ex;
 
       if (branchTaken)
         Ex = C->getTrueExpr();
@@ -882,9 +890,9 @@ const GRState* ExprEngine::MarkBranch(const GRState* state,
 
     case Stmt::ChooseExprClass: { // ?:
 
-      const ChooseExpr* C = cast<ChooseExpr>(Terminator);
+      const ChooseExpr *C = cast<ChooseExpr>(Terminator);
 
-      const Expr* Ex = branchTaken ? C->getLHS() : C->getRHS();
+      const Expr *Ex = branchTaken ? C->getLHS() : C->getRHS();
       return state->BindExpr(C, UndefinedVal(Ex));
     }
   }
@@ -895,8 +903,10 @@ const GRState* ExprEngine::MarkBranch(const GRState* state,
 /// integers that promote their values (which are currently not tracked well).
 /// This function returns the SVal bound to Condition->IgnoreCasts if all the
 //  cast(s) did was sign-extend the original value.
-static SVal RecoverCastedSymbol(GRStateManager& StateMgr, const GRState* state,
-                                const Stmt* Condition, ASTContext& Ctx) {
+static SVal RecoverCastedSymbol(ProgramStateManager& StateMgr,
+                                const ProgramState *state,
+                                const Stmt *Condition,
+                                ASTContext &Ctx) {
 
   const Expr *Ex = dyn_cast<Expr>(Condition);
   if (!Ex)
@@ -929,7 +939,7 @@ static SVal RecoverCastedSymbol(GRStateManager& StateMgr, const GRState* state,
   return state->getSVal(Ex);
 }
 
-void ExprEngine::processBranch(const Stmt* Condition, const Stmt* Term,
+void ExprEngine::processBranch(const Stmt *Condition, const Stmt *Term,
                                  BranchNodeBuilder& builder) {
 
   // Check for NULL conditions; e.g. "for(;;)"
@@ -948,7 +958,7 @@ void ExprEngine::processBranch(const Stmt* Condition, const Stmt* Term,
   if (!builder.isFeasible(true) && !builder.isFeasible(false))
     return;
 
-  const GRState* PrevState = builder.getState();
+  const ProgramState *PrevState = builder.getState();
   SVal X = PrevState->getSVal(Condition);
 
   if (X.isUnknownOrUndef()) {
@@ -980,7 +990,7 @@ void ExprEngine::processBranch(const Stmt* Condition, const Stmt* Term,
 
   // Process the true branch.
   if (builder.isFeasible(true)) {
-    if (const GRState *state = PrevState->assume(V, true))
+    if (const ProgramState *state = PrevState->assume(V, true))
       builder.generateNode(MarkBranch(state, Term, true), true);
     else
       builder.markInfeasible(true);
@@ -988,7 +998,7 @@ void ExprEngine::processBranch(const Stmt* Condition, const Stmt* Term,
 
   // Process the false branch.
   if (builder.isFeasible(false)) {
-    if (const GRState *state = PrevState->assume(V, false))
+    if (const ProgramState *state = PrevState->assume(V, false))
       builder.generateNode(MarkBranch(state, Term, false), false);
     else
       builder.markInfeasible(false);
@@ -999,7 +1009,7 @@ void ExprEngine::processBranch(const Stmt* Condition, const Stmt* Term,
 ///  nodes by processing the 'effects' of a computed goto jump.
 void ExprEngine::processIndirectGoto(IndirectGotoNodeBuilder &builder) {
 
-  const GRState *state = builder.getState();
+  const ProgramState *state = builder.getState();
   SVal V = state->getSVal(builder.getTarget());
 
   // Three possibilities:
@@ -1021,8 +1031,7 @@ void ExprEngine::processIndirectGoto(IndirectGotoNodeBuilder &builder) {
       }
     }
 
-    assert(false && "No block with label.");
-    return;
+    llvm_unreachable("No block with label.");
   }
 
   if (isa<loc::ConcreteInt>(V) || isa<UndefinedVal>(V)) {
@@ -1040,31 +1049,9 @@ void ExprEngine::processIndirectGoto(IndirectGotoNodeBuilder &builder) {
     builder.generateNode(I, state);
 }
 
-
-void ExprEngine::VisitGuardedExpr(const Expr* Ex, const Expr* L, 
-                                    const Expr* R,
-                                    ExplodedNode* Pred, ExplodedNodeSet& Dst) {
-
-  assert(Ex == currentStmt &&
-         Pred->getLocationContext()->getCFG()->isBlkExpr(Ex));
-
-  const GRState* state = GetState(Pred);
-  SVal X = state->getSVal(Ex);
-
-  assert (X.isUndef());
-
-  const Expr *SE = (Expr*) cast<UndefinedVal>(X).getData();
-  assert(SE);
-  X = state->getSVal(SE);
-
-  // Make sure that we invalidate the previous binding.
-  MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, X, true));
-}
-
 /// ProcessEndPath - Called by CoreEngine.  Used to generate end-of-path
 ///  nodes when the control reaches the end of a function.
 void ExprEngine::processEndOfFunction(EndOfFunctionNodeBuilder& builder) {
-  getTF().evalEndPath(*this, builder);
   StateMgr.EndPath(builder.getState());
   getCheckerManager().runCheckersForEndPath(builder, *this);
 }
@@ -1073,8 +1060,8 @@ void ExprEngine::processEndOfFunction(EndOfFunctionNodeBuilder& builder) {
 ///  nodes by processing the 'effects' of a switch statement.
 void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
   typedef SwitchNodeBuilder::iterator iterator;
-  const GRState* state = builder.getState();
-  const Expr* CondE = builder.getCondition();
+  const ProgramState *state = builder.getState();
+  const Expr *CondE = builder.getCondition();
   SVal  CondV_untested = state->getSVal(CondE);
 
   if (CondV_untested.isUndef()) {
@@ -1086,7 +1073,7 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
   }
   DefinedOrUnknownSVal CondV = cast<DefinedOrUnknownSVal>(CondV_untested);
 
-  const GRState *DefaultSt = state;
+  const ProgramState *DefaultSt = state;
   
   iterator I = builder.begin(), EI = builder.end();
   bool defaultIsFeasible = I == EI;
@@ -1096,28 +1083,16 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
     if (!I.getBlock())
       continue;
     
-    const CaseStmt* Case = I.getCase();
+    const CaseStmt *Case = I.getCase();
 
     // Evaluate the LHS of the case value.
-    Expr::EvalResult V1;
-    bool b = Case->getLHS()->Evaluate(V1, getContext());
-
-    // Sanity checks.  These go away in Release builds.
-    assert(b && V1.Val.isInt() && !V1.HasSideEffects
-             && "Case condition must evaluate to an integer constant.");
-    (void)b; // silence unused variable warning
-    assert(V1.Val.getInt().getBitWidth() ==
-           getContext().getTypeSize(CondE->getType()));
+    llvm::APSInt V1 = Case->getLHS()->EvaluateKnownConstInt(getContext());
+    assert(V1.getBitWidth() == getContext().getTypeSize(CondE->getType()));
 
     // Get the RHS of the case, if it exists.
-    Expr::EvalResult V2;
-
-    if (const Expr* E = Case->getRHS()) {
-      b = E->Evaluate(V2, getContext());
-      assert(b && V2.Val.isInt() && !V2.HasSideEffects
-             && "Case condition must evaluate to an integer constant.");
-      (void)b; // silence unused variable warning
-    }
+    llvm::APSInt V2;
+    if (const Expr *E = Case->getRHS())
+      V2 = E->EvaluateKnownConstInt(getContext());
     else
       V2 = V1;
 
@@ -1126,12 +1101,12 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
     //  This should be easy once we have "ranges" for NonLVals.
 
     do {
-      nonloc::ConcreteInt CaseVal(getBasicVals().getValue(V1.Val.getInt()));
+      nonloc::ConcreteInt CaseVal(getBasicVals().getValue(V1));
       DefinedOrUnknownSVal Res = svalBuilder.evalEQ(DefaultSt ? DefaultSt : state,
                                                CondV, CaseVal);
 
       // Now "assume" that the case matches.
-      if (const GRState* stateNew = state->assume(Res, true)) {
+      if (const ProgramState *stateNew = state->assume(Res, true)) {
         builder.generateCaseStmtNode(I, stateNew);
 
         // If CondV evaluates to a constant, then we know that this
@@ -1144,7 +1119,7 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
       // Now "assume" that the case doesn't match.  Add this state
       // to the default state (if it is feasible).
       if (DefaultSt) {
-        if (const GRState *stateNew = DefaultSt->assume(Res, false)) {
+        if (const ProgramState *stateNew = DefaultSt->assume(Res, false)) {
           defaultIsFeasible = true;
           DefaultSt = stateNew;
         }
@@ -1155,11 +1130,11 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
       }
 
       // Concretize the next value in the range.
-      if (V1.Val.getInt() == V2.Val.getInt())
+      if (V1 == V2)
         break;
 
-      ++V1.Val.getInt();
-      assert (V1.Val.getInt() <= V2.Val.getInt());
+      ++V1;
+      assert (V1 <= V2);
 
     } while (true);
   }
@@ -1184,120 +1159,16 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
   builder.generateDefaultCaseNode(DefaultSt);
 }
 
-void ExprEngine::processCallEnter(CallEnterNodeBuilder &B) {
-  const GRState *state = B.getState()->enterStackFrame(B.getCalleeContext());
-  B.generateNode(state);
-}
-
-void ExprEngine::processCallExit(CallExitNodeBuilder &B) {
-  const GRState *state = B.getState();
-  const ExplodedNode *Pred = B.getPredecessor();
-  const StackFrameContext *calleeCtx = 
-                            cast<StackFrameContext>(Pred->getLocationContext());
-  const Stmt *CE = calleeCtx->getCallSite();
-
-  // If the callee returns an expression, bind its value to CallExpr.
-  const Stmt *ReturnedExpr = state->get<ReturnExpr>();
-  if (ReturnedExpr) {
-    SVal RetVal = state->getSVal(ReturnedExpr);
-    state = state->BindExpr(CE, RetVal);
-    // Clear the return expr GDM.
-    state = state->remove<ReturnExpr>();
-  }
-
-  // Bind the constructed object value to CXXConstructExpr.
-  if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(CE)) {
-    const CXXThisRegion *ThisR =
-      getCXXThisRegion(CCE->getConstructor()->getParent(), calleeCtx);
-
-    SVal ThisV = state->getSVal(ThisR);
-    // Always bind the region to the CXXConstructExpr.
-    state = state->BindExpr(CCE, ThisV);
-  }
-
-  B.generateNode(state);
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer functions: logical operations ('&&', '||').
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
-                                    ExplodedNodeSet& Dst) {
-
-  assert(B->getOpcode() == BO_LAnd ||
-         B->getOpcode() == BO_LOr);
-
-  assert(B==currentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(B));
-
-  const GRState* state = GetState(Pred);
-  SVal X = state->getSVal(B);
-  assert(X.isUndef());
-
-  const Expr *Ex = (const Expr*) cast<UndefinedVal>(X).getData();
-  assert(Ex);
-
-  if (Ex == B->getRHS()) {
-    X = state->getSVal(Ex);
-
-    // Handle undefined values.
-    if (X.isUndef()) {
-      MakeNode(Dst, B, Pred, state->BindExpr(B, X));
-      return;
-    }
-
-    DefinedOrUnknownSVal XD = cast<DefinedOrUnknownSVal>(X);
-
-    // We took the RHS.  Because the value of the '&&' or '||' expression must
-    // evaluate to 0 or 1, we must assume the value of the RHS evaluates to 0
-    // or 1.  Alternatively, we could take a lazy approach, and calculate this
-    // value later when necessary.  We don't have the machinery in place for
-    // this right now, and since most logical expressions are used for branches,
-    // the payoff is not likely to be large.  Instead, we do eager evaluation.
-    if (const GRState *newState = state->assume(XD, true))
-      MakeNode(Dst, B, Pred,
-               newState->BindExpr(B, svalBuilder.makeIntVal(1U, B->getType())));
-
-    if (const GRState *newState = state->assume(XD, false))
-      MakeNode(Dst, B, Pred,
-               newState->BindExpr(B, svalBuilder.makeIntVal(0U, B->getType())));
-  }
-  else {
-    // We took the LHS expression.  Depending on whether we are '&&' or
-    // '||' we know what the value of the expression is via properties of
-    // the short-circuiting.
-    X = svalBuilder.makeIntVal(B->getOpcode() == BO_LAnd ? 0U : 1U,
-                          B->getType());
-    MakeNode(Dst, B, Pred, state->BindExpr(B, X));
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Transfer functions: Loads and stores.
 //===----------------------------------------------------------------------===//
 
-void ExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
-                                  ExplodedNodeSet &Dst) {
-
-  ExplodedNodeSet Tmp;
-
-  CanQualType T = getContext().getCanonicalType(BE->getType());
-  SVal V = svalBuilder.getBlockPointer(BE->getBlockDecl(), T,
-                                  Pred->getLocationContext());
-
-  MakeNode(Tmp, BE, Pred, GetState(Pred)->BindExpr(BE, V),
-           ProgramPoint::PostLValueKind);
-
-  // Post-visit the BlockExpr.
-  getCheckerManager().runCheckersForPostStmt(Dst, Tmp, BE, *this);
-}
-
 void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
                                         ExplodedNode *Pred,
                                         ExplodedNodeSet &Dst) {
-  const GRState *state = GetState(Pred);
+  const ProgramState *state = Pred->getState();
 
-  if (const VarDecl* VD = dyn_cast<VarDecl>(D)) {
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
     assert(Ex->isLValue());
     SVal V = state->getLValue(VD, Pred->getLocationContext());
 
@@ -1314,13 +1185,13 @@ void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
              ProgramPoint::PostLValueKind);
     return;
   }
-  if (const EnumConstantDecl* ED = dyn_cast<EnumConstantDecl>(D)) {
+  if (const EnumConstantDecl *ED = dyn_cast<EnumConstantDecl>(D)) {
     assert(!Ex->isLValue());
     SVal V = svalBuilder.makeIntVal(ED->getInitVal());
     MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V));
     return;
   }
-  if (const FunctionDecl* FD = dyn_cast<FunctionDecl>(D)) {
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     SVal V = svalBuilder.getFunctionPointer(FD);
     MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V),
              ProgramPoint::PostLValueKind);
@@ -1331,124 +1202,93 @@ void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
 }
 
 /// VisitArraySubscriptExpr - Transfer function for array accesses
-void ExprEngine::VisitLvalArraySubscriptExpr(const ArraySubscriptExpr* A,
-                                             ExplodedNode* Pred,
-                                             ExplodedNodeSet& Dst){
+void ExprEngine::VisitLvalArraySubscriptExpr(const ArraySubscriptExpr *A,
+                                             ExplodedNode *Pred,
+                                             ExplodedNodeSet &Dst){
 
-  const Expr* Base = A->getBase()->IgnoreParens();
-  const Expr* Idx  = A->getIdx()->IgnoreParens();
+  const Expr *Base = A->getBase()->IgnoreParens();
+  const Expr *Idx  = A->getIdx()->IgnoreParens();
   
-  // Evaluate the base.
-  ExplodedNodeSet Tmp;
-  Visit(Base, Pred, Tmp);
 
-  for (ExplodedNodeSet::iterator I1=Tmp.begin(), E1=Tmp.end(); I1!=E1; ++I1) {
-    ExplodedNodeSet Tmp2;
-    Visit(Idx, *I1, Tmp2);     // Evaluate the index.
-    ExplodedNodeSet Tmp3;
-    getCheckerManager().runCheckersForPreStmt(Tmp3, Tmp2, A, *this);
+  ExplodedNodeSet checkerPreStmt;
+  getCheckerManager().runCheckersForPreStmt(checkerPreStmt, Pred, A, *this);
 
-    for (ExplodedNodeSet::iterator I2=Tmp3.begin(),E2=Tmp3.end();I2!=E2; ++I2) {
-      const GRState* state = GetState(*I2);
-      SVal V = state->getLValue(A->getType(), state->getSVal(Idx),
-                                state->getSVal(Base));
-      assert(A->isLValue());
-      MakeNode(Dst, A, *I2, state->BindExpr(A, V), ProgramPoint::PostLValueKind);
-    }
+  for (ExplodedNodeSet::iterator it = checkerPreStmt.begin(),
+                                 ei = checkerPreStmt.end(); it != ei; ++it) {
+    const ProgramState *state = (*it)->getState();
+    SVal V = state->getLValue(A->getType(), state->getSVal(Idx),
+                              state->getSVal(Base));
+    assert(A->isLValue());
+    MakeNode(Dst, A, *it, state->BindExpr(A, V), ProgramPoint::PostLValueKind);
   }
 }
 
 /// VisitMemberExpr - Transfer function for member expressions.
-void ExprEngine::VisitMemberExpr(const MemberExpr* M, ExplodedNode* Pred,
-                                 ExplodedNodeSet& Dst) {
+void ExprEngine::VisitMemberExpr(const MemberExpr *M, ExplodedNode *Pred,
+                                 ExplodedNodeSet &Dst) {
 
-  Expr *baseExpr = M->getBase()->IgnoreParens();
-  ExplodedNodeSet dstBase;
-  Visit(baseExpr, Pred, dstBase);
-
-  FieldDecl *field = dyn_cast<FieldDecl>(M->getMemberDecl());
+  Decl *member = M->getMemberDecl();
+  if (VarDecl *VD = dyn_cast<VarDecl>(member)) {
+    assert(M->isLValue());
+    VisitCommonDeclRefExpr(M, VD, Pred, Dst);
+    return;
+  }
+  
+  FieldDecl *field = dyn_cast<FieldDecl>(member);
   if (!field) // FIXME: skipping member expressions for non-fields
     return;
 
-  for (ExplodedNodeSet::iterator I = dstBase.begin(), E = dstBase.end();
-    I != E; ++I) {
-    const GRState* state = GetState(*I);
-    SVal baseExprVal = state->getSVal(baseExpr);
-    if (isa<nonloc::LazyCompoundVal>(baseExprVal) ||
-        isa<nonloc::CompoundVal>(baseExprVal) ||
-        // FIXME: This can originate by conjuring a symbol for an unknown
-        // temporary struct object, see test/Analysis/fields.c:
-        // (p = getit()).x
-        isa<nonloc::SymbolVal>(baseExprVal)) {
-      MakeNode(Dst, M, *I, state->BindExpr(M, UnknownVal()));
-      continue;
-    }
-
-    // FIXME: Should we insert some assumption logic in here to determine
-    // if "Base" is a valid piece of memory?  Before we put this assumption
-    // later when using FieldOffset lvals (which we no longer have).
-
-    // For all other cases, compute an lvalue.    
-    SVal L = state->getLValue(field, baseExprVal);
-    if (M->isLValue())
-      MakeNode(Dst, M, *I, state->BindExpr(M, L), ProgramPoint::PostLValueKind);
-    else
-      evalLoad(Dst, M, *I, state, L);
+  Expr *baseExpr = M->getBase()->IgnoreParens();
+  const ProgramState *state = Pred->getState();
+  SVal baseExprVal = state->getSVal(baseExpr);
+  if (isa<nonloc::LazyCompoundVal>(baseExprVal) ||
+      isa<nonloc::CompoundVal>(baseExprVal) ||
+      // FIXME: This can originate by conjuring a symbol for an unknown
+      // temporary struct object, see test/Analysis/fields.c:
+      // (p = getit()).x
+      isa<nonloc::SymbolVal>(baseExprVal)) {
+    MakeNode(Dst, M, Pred, state->BindExpr(M, UnknownVal()));
+    return;
   }
+
+  // FIXME: Should we insert some assumption logic in here to determine
+  // if "Base" is a valid piece of memory?  Before we put this assumption
+  // later when using FieldOffset lvals (which we no longer have).
+
+  // For all other cases, compute an lvalue.    
+  SVal L = state->getLValue(field, baseExprVal);
+  if (M->isLValue())
+    MakeNode(Dst, M, Pred, state->BindExpr(M, L), ProgramPoint::PostLValueKind);
+  else
+    evalLoad(Dst, M, Pred, state, L);
 }
 
 /// evalBind - Handle the semantics of binding a value to a specific location.
 ///  This method is used by evalStore and (soon) VisitDeclStmt, and others.
-void ExprEngine::evalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
-                            ExplodedNode* Pred, const GRState* state,
-                            SVal location, SVal Val, bool atDeclInit) {
-
+void ExprEngine::evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE,
+                          ExplodedNode *Pred,
+                          SVal location, SVal Val, bool atDeclInit) {
 
   // Do a previsit of the bind.
-  ExplodedNodeSet CheckedSet, Src;
-  Src.Add(Pred);
-  getCheckerManager().runCheckersForBind(CheckedSet, Src, location, Val, StoreE,
-                                         *this);
+  ExplodedNodeSet CheckedSet;
+  getCheckerManager().runCheckersForBind(CheckedSet, Pred, location, Val,
+                                         StoreE, *this);
 
   for (ExplodedNodeSet::iterator I = CheckedSet.begin(), E = CheckedSet.end();
        I!=E; ++I) {
 
-    if (Pred != *I)
-      state = GetState(*I);
-
-    const GRState* newState = 0;
+    const ProgramState *state = (*I)->getState();
 
     if (atDeclInit) {
       const VarRegion *VR =
         cast<VarRegion>(cast<loc::MemRegionVal>(location).getRegion());
 
-      newState = state->bindDecl(VR, Val);
-    }
-    else {
-      if (location.isUnknown()) {
-        // We know that the new state will be the same as the old state since
-        // the location of the binding is "unknown".  Consequently, there
-        // is no reason to just create a new node.
-        newState = state;
-      }
-      else {
-        // We are binding to a value other than 'unknown'.  Perform the binding
-        // using the StoreManager.
-        newState = state->bindLoc(cast<Loc>(location), Val);
-      }
+      state = state->bindDecl(VR, Val);
+    } else {
+      state = state->bindLoc(location, Val);
     }
 
-    // The next thing to do is check if the TransferFuncs object wants to
-    // update the state based on the new binding.  If the GRTransferFunc object
-    // doesn't do anything, just auto-propagate the current state.
-    
-    // NOTE: We use 'AssignE' for the location of the PostStore if 'AssignE'
-    // is non-NULL.  Checkers typically care about 
-    
-    StmtNodeBuilderRef BuilderRef(Dst, *Builder, *this, *I, newState, StoreE,
-                                    true);
-
-    getTF().evalBind(BuilderRef, location, Val);
+    MakeNode(Dst, StoreE, *I, state);
   }
 }
 
@@ -1460,11 +1300,11 @@ void ExprEngine::evalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
 ///  @param state The current simulation state
 ///  @param location The location to store the value
 ///  @param Val The value to be stored
-void ExprEngine::evalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
-                             const Expr* LocationE,
-                             ExplodedNode* Pred,
-                             const GRState* state, SVal location, SVal Val,
-                             const void *tag) {
+void ExprEngine::evalStore(ExplodedNodeSet &Dst, const Expr *AssignE,
+                             const Expr *LocationE,
+                             ExplodedNode *Pred,
+                             const ProgramState *state, SVal location, SVal Val,
+                             const ProgramPointTag *tag) {
 
   assert(Builder && "StmtNodeBuilder must be defined.");
 
@@ -1474,9 +1314,8 @@ void ExprEngine::evalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
 
   if (isa<loc::ObjCPropRef>(location)) {
     loc::ObjCPropRef prop = cast<loc::ObjCPropRef>(location);
-    ExplodedNodeSet src = Pred;
     return VisitObjCMessage(ObjCPropertySetter(prop.getPropRefExpr(),
-                                               StoreE, Val), src, Dst);
+                                               StoreE, Val), Pred, Dst);
   }
 
   // Evaluate the location (checks for bad dereferences).
@@ -1493,38 +1332,38 @@ void ExprEngine::evalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
                                                    ProgramPoint::PostStoreKind);
 
   for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI)
-    evalBind(Dst, StoreE, *NI, GetState(*NI), location, Val);
+    evalBind(Dst, StoreE, *NI, location, Val);
 }
 
-void ExprEngine::evalLoad(ExplodedNodeSet& Dst, const Expr *Ex,
-                            ExplodedNode* Pred,
-                            const GRState* state, SVal location,
-                            const void *tag, QualType LoadTy) {
+void ExprEngine::evalLoad(ExplodedNodeSet &Dst, const Expr *Ex,
+                            ExplodedNode *Pred,
+                            const ProgramState *state, SVal location,
+                            const ProgramPointTag *tag, QualType LoadTy) {
   assert(!isa<NonLoc>(location) && "location cannot be a NonLoc.");
 
   if (isa<loc::ObjCPropRef>(location)) {
     loc::ObjCPropRef prop = cast<loc::ObjCPropRef>(location);
-    ExplodedNodeSet src = Pred;
     return VisitObjCMessage(ObjCPropertyGetter(prop.getPropRefExpr(), Ex),
-                            src, Dst);
+                            Pred, Dst);
   }
 
   // Are we loading from a region?  This actually results in two loads; one
   // to fetch the address of the referenced value and one to fetch the
   // referenced value.
-  if (const TypedRegion *TR =
-        dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
+  if (const TypedValueRegion *TR =
+        dyn_cast_or_null<TypedValueRegion>(location.getAsRegion())) {
 
     QualType ValTy = TR->getValueType();
     if (const ReferenceType *RT = ValTy->getAs<ReferenceType>()) {
-      static int loadReferenceTag = 0;
+      static SimpleProgramPointTag
+             loadReferenceTag("ExprEngine : Load Reference");
       ExplodedNodeSet Tmp;
       evalLoadCommon(Tmp, Ex, Pred, state, location, &loadReferenceTag,
                      getContext().getPointerType(RT->getPointeeType()));
 
       // Perform the load from the referenced value.
       for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end() ; I!=E; ++I) {
-        state = GetState(*I);
+        state = (*I)->getState();
         location = state->getSVal(Ex);
         evalLoadCommon(Dst, Ex, *I, state, location, tag, LoadTy);
       }
@@ -1535,10 +1374,10 @@ void ExprEngine::evalLoad(ExplodedNodeSet& Dst, const Expr *Ex,
   evalLoadCommon(Dst, Ex, Pred, state, location, tag, LoadTy);
 }
 
-void ExprEngine::evalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
-                                  ExplodedNode* Pred,
-                                  const GRState* state, SVal location,
-                                  const void *tag, QualType LoadTy) {
+void ExprEngine::evalLoadCommon(ExplodedNodeSet &Dst, const Expr *Ex,
+                                  ExplodedNode *Pred,
+                                  const ProgramState *state, SVal location,
+                                  const ProgramPointTag *tag, QualType LoadTy) {
 
   // Evaluate the location (checks for bad dereferences).
   ExplodedNodeSet Tmp;
@@ -1554,7 +1393,7 @@ void ExprEngine::evalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
 
   // Proceed with the load.
   for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI) {
-    state = GetState(*NI);
+    state = (*NI)->getState();
 
     if (location.isUnknown()) {
       // This is important.  We must nuke the old binding.
@@ -1572,9 +1411,9 @@ void ExprEngine::evalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
 }
 
 void ExprEngine::evalLocation(ExplodedNodeSet &Dst, const Stmt *S,
-                                ExplodedNode* Pred,
-                                const GRState* state, SVal location,
-                                const void *tag, bool isLoad) {
+                                ExplodedNode *Pred,
+                                const ProgramState *state, SVal location,
+                                const ProgramPointTag *tag, bool isLoad) {
   // Early checks for performance reason.
   if (location.isUnknown()) {
     Dst.Add(Pred);
@@ -1582,7 +1421,7 @@ void ExprEngine::evalLocation(ExplodedNodeSet &Dst, const Stmt *S,
   }
 
   ExplodedNodeSet Src;
-  if (Builder->GetState(Pred) == state) {
+  if (Pred->getState() == state) {
     Src.Add(Pred);
   } else {
     // Associate this new state with an ExplodedNode.
@@ -1593,7 +1432,11 @@ void ExprEngine::evalLocation(ExplodedNodeSet &Dst, const Stmt *S,
     // "p = 0" is not noted as "Null pointer value stored to 'p'" but
     // instead "int *p" is noted as
     // "Variable 'p' initialized to a null pointer value"
-    ExplodedNode *N = Builder->generateNode(S, state, Pred, this);
+    
+    // FIXME: why is 'tag' not used instead of etag?
+    static SimpleProgramPointTag etag("ExprEngine: Location");
+
+    ExplodedNode *N = Builder->generateNode(S, state, Pred, &etag);
     Src.Add(N ? N : Pred);
   }
   getCheckerManager().runCheckersForLocation(Dst, Src, location, isLoad, S,
@@ -1611,7 +1454,7 @@ bool ExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
   // cases as well.
   
 #if 0
-  const GRState *state = GetState(Pred);
+  const ProgramState *state = Pred->getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
   
@@ -1627,7 +1470,7 @@ bool ExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
     case Stmt::CXXOperatorCallExprClass: {
       const CXXOperatorCallExpr *opCall = cast<CXXOperatorCallExpr>(CE);
       methodDecl = 
-        llvm::dyn_cast_or_null<CXXMethodDecl>(opCall->getCalleeDecl());
+        dyn_cast_or_null<CXXMethodDecl>(opCall->getCalleeDecl());
       break;
     }
     case Stmt::CXXMemberCallExprClass: {
@@ -1676,126 +1519,18 @@ bool ExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
 #endif
 }
 
-void ExprEngine::VisitCallExpr(const CallExpr* CE, ExplodedNode* Pred,
-                               ExplodedNodeSet& dst) {
-
-  // Determine the type of function we're calling (if available).
-  const FunctionProtoType *Proto = NULL;
-  QualType FnType = CE->getCallee()->IgnoreParens()->getType();
-  if (const PointerType *FnTypePtr = FnType->getAs<PointerType>())
-    Proto = FnTypePtr->getPointeeType()->getAs<FunctionProtoType>();
-
-  // Should the first argument be evaluated as an lvalue?
-  bool firstArgumentAsLvalue = false;
-  switch (CE->getStmtClass()) {
-    case Stmt::CXXOperatorCallExprClass:
-      firstArgumentAsLvalue = true;
-      break;
-    default:
-      break;
-  }
-  
-  // Evaluate the arguments.
-  ExplodedNodeSet dstArgsEvaluated;
-  evalArguments(CE->arg_begin(), CE->arg_end(), Proto, Pred, dstArgsEvaluated,
-                firstArgumentAsLvalue);
-
-  // Evaluate the callee.
-  ExplodedNodeSet dstCalleeEvaluated;
-  evalCallee(CE, dstArgsEvaluated, dstCalleeEvaluated);
-
-  // Perform the previsit of the CallExpr.
-  ExplodedNodeSet dstPreVisit;
-  getCheckerManager().runCheckersForPreStmt(dstPreVisit, dstCalleeEvaluated,
-                                            CE, *this);
-    
-  // Now evaluate the call itself.
-  class DefaultEval : public GraphExpander {
-    ExprEngine &Eng;
-    const CallExpr *CE;
-  public:
-    
-    DefaultEval(ExprEngine &eng, const CallExpr *ce)
-      : Eng(eng), CE(ce) {}
-    virtual void expandGraph(ExplodedNodeSet &Dst, ExplodedNode *Pred) {
-      // Should we inline the call?
-      if (Eng.getAnalysisManager().shouldInlineCall() &&
-          Eng.InlineCall(Dst, CE, Pred)) {
-        return;
-      }
-
-      StmtNodeBuilder &Builder = Eng.getBuilder();
-      assert(&Builder && "StmtNodeBuilder must be defined.");
-
-      // Dispatch to the plug-in transfer function.
-      unsigned oldSize = Dst.size();
-      SaveOr OldHasGen(Builder.hasGeneratedNode);
-
-      // Dispatch to transfer function logic to handle the call itself.
-      const Expr* Callee = CE->getCallee()->IgnoreParens();
-      const GRState* state = Eng.GetState(Pred);
-      SVal L = state->getSVal(Callee);
-      Eng.getTF().evalCall(Dst, Eng, Builder, CE, L, Pred);
-
-      // Handle the case where no nodes where generated.  Auto-generate that
-      // contains the updated state if we aren't generating sinks.
-      if (!Builder.BuildSinks && Dst.size() == oldSize &&
-          !Builder.hasGeneratedNode)
-        Eng.MakeNode(Dst, CE, Pred, state);
-    }
-  };
-
-  // Finally, evaluate the function call.  We try each of the checkers
-  // to see if the can evaluate the function call.
-  ExplodedNodeSet dstCallEvaluated;
-  DefaultEval defEval(*this, CE);
-  getCheckerManager().runCheckersForEvalCall(dstCallEvaluated,
-                                             dstPreVisit,
-                                             CE, *this, &defEval);
-
-  // Finally, perform the post-condition check of the CallExpr and store
-  // the created nodes in 'Dst'.
-  getCheckerManager().runCheckersForPostStmt(dst, dstCallEvaluated, CE,
-                                             *this);
+std::pair<const ProgramPointTag *, const ProgramPointTag*>
+ExprEngine::getEagerlyAssumeTags() {
+  static SimpleProgramPointTag
+         EagerlyAssumeTrue("ExprEngine : Eagerly Assume True"),
+         EagerlyAssumeFalse("ExprEngine : Eagerly Assume False");
+  return std::make_pair(&EagerlyAssumeTrue, &EagerlyAssumeFalse);
 }
 
-//===----------------------------------------------------------------------===//
-// Transfer function: Objective-C dot-syntax to access a property.
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitObjCPropertyRefExpr(const ObjCPropertyRefExpr *Ex,
-                                          ExplodedNode *Pred,
-                                          ExplodedNodeSet &Dst) {
-  ExplodedNodeSet dstBase;
-
-  // Visit the receiver (if any).
-  if (Ex->isObjectReceiver())
-    Visit(Ex->getBase(), Pred, dstBase);
-  else
-    dstBase = Pred;
-
-  ExplodedNodeSet dstPropRef;
-
-  // Using the base, compute the lvalue of the instance variable.
-  for (ExplodedNodeSet::iterator I = dstBase.begin(), E = dstBase.end();
-       I!=E; ++I) {
-    ExplodedNode *nodeBase = *I;
-    const GRState *state = GetState(nodeBase);
-    MakeNode(dstPropRef, Ex, *I, state->BindExpr(Ex, loc::ObjCPropRef(Ex)));
-  }
-  
-  Dst.insert(dstPropRef);
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer function: Objective-C ivar references.
-//===----------------------------------------------------------------------===//
-
-static std::pair<const void*,const void*> EagerlyAssumeTag
-  = std::pair<const void*,const void*>(&EagerlyAssumeTag,static_cast<void*>(0));
-
 void ExprEngine::evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
                                      const Expr *Ex) {
+  
+  
   for (ExplodedNodeSet::iterator I=Src.begin(), E=Src.end(); I!=E; ++I) {
     ExplodedNode *Pred = *I;
 
@@ -1808,25 +1543,24 @@ void ExprEngine::evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
       continue;
     }
 
-    const GRState* state = GetState(Pred);
+    const ProgramState *state = Pred->getState();
     SVal V = state->getSVal(Ex);
     if (nonloc::SymExprVal *SEV = dyn_cast<nonloc::SymExprVal>(&V)) {
+      const std::pair<const ProgramPointTag *, const ProgramPointTag*> &tags =
+        getEagerlyAssumeTags();
+
       // First assume that the condition is true.
-      if (const GRState *stateTrue = state->assume(*SEV, true)) {
-        stateTrue = stateTrue->BindExpr(Ex,
-                                        svalBuilder.makeIntVal(1U, Ex->getType()));
-        Dst.Add(Builder->generateNode(PostStmtCustom(Ex,
-                                &EagerlyAssumeTag, Pred->getLocationContext()),
-                                      stateTrue, Pred));
+      if (const ProgramState *StateTrue = state->assume(*SEV, true)) {
+        SVal Val = svalBuilder.makeIntVal(1U, Ex->getType());        
+        StateTrue = StateTrue->BindExpr(Ex, Val);
+        Dst.Add(Builder->generateNode(Ex, StateTrue, Pred, tags.first));
       }
 
       // Next, assume that the condition is false.
-      if (const GRState *stateFalse = state->assume(*SEV, false)) {
-        stateFalse = stateFalse->BindExpr(Ex,
-                                          svalBuilder.makeIntVal(0U, Ex->getType()));
-        Dst.Add(Builder->generateNode(PostStmtCustom(Ex, &EagerlyAssumeTag,
-                                                   Pred->getLocationContext()),
-                                      stateFalse, Pred));
+      if (const ProgramState *StateFalse = state->assume(*SEV, false)) {
+        SVal Val = svalBuilder.makeIntVal(0U, Ex->getType());
+        StateFalse = StateFalse->BindExpr(Ex, Val);
+        Dst.Add(Builder->generateNode(Ex, StateFalse, Pred, tags.second));
       }
     }
     else
@@ -1834,954 +1568,15 @@ void ExprEngine::evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
   }
 }
 
-//===----------------------------------------------------------------------===//
-// Transfer function: Objective-C @synchronized.
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt *S,
-                                               ExplodedNode *Pred,
-                                               ExplodedNodeSet &Dst) {
-
-  // The mutex expression is a CFGElement, so we don't need to explicitly
-  // visit it since it will already be processed.
-
-  // Pre-visit the ObjCAtSynchronizedStmt.
-  ExplodedNodeSet Tmp;
-  Tmp.Add(Pred);
-  getCheckerManager().runCheckersForPreStmt(Dst, Tmp, S, *this);
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer function: Objective-C ivar references.
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitLvalObjCIvarRefExpr(const ObjCIvarRefExpr* Ex, 
-                                          ExplodedNode* Pred,
-                                          ExplodedNodeSet& Dst) {
-
-  // Visit the base expression, which is needed for computing the lvalue
-  // of the ivar.
-  ExplodedNodeSet dstBase;
-  const Expr *baseExpr = Ex->getBase();
-  Visit(baseExpr, Pred, dstBase);
-
-  ExplodedNodeSet dstIvar;
-
-  // Using the base, compute the lvalue of the instance variable.
-  for (ExplodedNodeSet::iterator I = dstBase.begin(), E = dstBase.end();
-       I!=E; ++I) {
-    ExplodedNode *nodeBase = *I;
-    const GRState *state = GetState(nodeBase);
-    SVal baseVal = state->getSVal(baseExpr);
-    SVal location = state->getLValue(Ex->getDecl(), baseVal);
-    MakeNode(dstIvar, Ex, *I, state->BindExpr(Ex, location));
-  }
-
-  // Perform the post-condition check of the ObjCIvarRefExpr and store
-  // the created nodes in 'Dst'.
-  getCheckerManager().runCheckersForPostStmt(Dst, dstIvar, Ex, *this);
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer function: Objective-C fast enumeration 'for' statements.
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt* S,
-                                     ExplodedNode* Pred, ExplodedNodeSet& Dst) {
-
-  // ObjCForCollectionStmts are processed in two places.  This method
-  // handles the case where an ObjCForCollectionStmt* occurs as one of the
-  // statements within a basic block.  This transfer function does two things:
-  //
-  //  (1) binds the next container value to 'element'.  This creates a new
-  //      node in the ExplodedGraph.
-  //
-  //  (2) binds the value 0/1 to the ObjCForCollectionStmt* itself, indicating
-  //      whether or not the container has any more elements.  This value
-  //      will be tested in ProcessBranch.  We need to explicitly bind
-  //      this value because a container can contain nil elements.
-  //
-  // FIXME: Eventually this logic should actually do dispatches to
-  //   'countByEnumeratingWithState:objects:count:' (NSFastEnumeration).
-  //   This will require simulating a temporary NSFastEnumerationState, either
-  //   through an SVal or through the use of MemRegions.  This value can
-  //   be affixed to the ObjCForCollectionStmt* instead of 0/1; when the loop
-  //   terminates we reclaim the temporary (it goes out of scope) and we
-  //   we can test if the SVal is 0 or if the MemRegion is null (depending
-  //   on what approach we take).
-  //
-  //  For now: simulate (1) by assigning either a symbol or nil if the
-  //    container is empty.  Thus this transfer function will by default
-  //    result in state splitting.
-
-  const Stmt* elem = S->getElement();
-  SVal ElementV;
-
-  if (const DeclStmt* DS = dyn_cast<DeclStmt>(elem)) {
-    const VarDecl* ElemD = cast<VarDecl>(DS->getSingleDecl());
-    assert (ElemD->getInit() == 0);
-    ElementV = GetState(Pred)->getLValue(ElemD, Pred->getLocationContext());
-    VisitObjCForCollectionStmtAux(S, Pred, Dst, ElementV);
-    return;
-  }
-
-  ExplodedNodeSet Tmp;
-  Visit(cast<Expr>(elem), Pred, Tmp);
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I!=E; ++I) {
-    const GRState* state = GetState(*I);
-    VisitObjCForCollectionStmtAux(S, *I, Dst, state->getSVal(elem));
-  }
-}
-
-void ExprEngine::VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S,
-                                       ExplodedNode* Pred, ExplodedNodeSet& Dst,
-                                                 SVal ElementV) {
-
-  // Check if the location we are writing back to is a null pointer.
-  const Stmt* elem = S->getElement();
-  ExplodedNodeSet Tmp;
-  evalLocation(Tmp, elem, Pred, GetState(Pred), ElementV, NULL, false);
-
-  if (Tmp.empty())
-    return;
-
-  for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI) {
-    Pred = *NI;
-    const GRState *state = GetState(Pred);
-
-    // Handle the case where the container still has elements.
-    SVal TrueV = svalBuilder.makeTruthVal(1);
-    const GRState *hasElems = state->BindExpr(S, TrueV);
-
-    // Handle the case where the container has no elements.
-    SVal FalseV = svalBuilder.makeTruthVal(0);
-    const GRState *noElems = state->BindExpr(S, FalseV);
-
-    if (loc::MemRegionVal* MV = dyn_cast<loc::MemRegionVal>(&ElementV))
-      if (const TypedRegion* R = dyn_cast<TypedRegion>(MV->getRegion())) {
-        // FIXME: The proper thing to do is to really iterate over the
-        //  container.  We will do this with dispatch logic to the store.
-        //  For now, just 'conjure' up a symbolic value.
-        QualType T = R->getValueType();
-        assert(Loc::isLocType(T));
-        unsigned Count = Builder->getCurrentBlockCount();
-        SymbolRef Sym = SymMgr.getConjuredSymbol(elem, T, Count);
-        SVal V = svalBuilder.makeLoc(Sym);
-        hasElems = hasElems->bindLoc(ElementV, V);
-
-        // Bind the location to 'nil' on the false branch.
-        SVal nilV = svalBuilder.makeIntVal(0, T);
-        noElems = noElems->bindLoc(ElementV, nilV);
-      }
-
-    // Create the new nodes.
-    MakeNode(Dst, S, Pred, hasElems);
-    MakeNode(Dst, S, Pred, noElems);
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer function: Objective-C message expressions.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class ObjCMsgWLItem {
-public:
-  ObjCMessageExpr::const_arg_iterator I;
-  ExplodedNode *N;
-
-  ObjCMsgWLItem(const ObjCMessageExpr::const_arg_iterator &i, ExplodedNode *n)
-    : I(i), N(n) {}
-};
-} // end anonymous namespace
-
-void ExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME, 
-                                        ExplodedNode* Pred,
-                                        ExplodedNodeSet& Dst){
-
-  // Create a worklist to process both the arguments.
-  llvm::SmallVector<ObjCMsgWLItem, 20> WL;
-
-  // But first evaluate the receiver (if any).
-  ObjCMessageExpr::const_arg_iterator AI = ME->arg_begin(), AE = ME->arg_end();
-  if (const Expr *Receiver = ME->getInstanceReceiver()) {
-    ExplodedNodeSet Tmp;
-    Visit(Receiver, Pred, Tmp);
-
-    if (Tmp.empty())
-      return;
-
-    for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I)
-      WL.push_back(ObjCMsgWLItem(AI, *I));
-  }
-  else
-    WL.push_back(ObjCMsgWLItem(AI, Pred));
-
-  // Evaluate the arguments.
-  ExplodedNodeSet ArgsEvaluated;
-  while (!WL.empty()) {
-    ObjCMsgWLItem Item = WL.back();
-    WL.pop_back();
-
-    if (Item.I == AE) {
-      ArgsEvaluated.insert(Item.N);
-      continue;
-    }
-
-    // Evaluate the subexpression.
-    ExplodedNodeSet Tmp;
-
-    // FIXME: [Objective-C++] handle arguments that are references
-    Visit(*Item.I, Item.N, Tmp);
-
-    // Enqueue evaluating the next argument on the worklist.
-    ++(Item.I);
-    for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI)
-      WL.push_back(ObjCMsgWLItem(Item.I, *NI));
-  }
-
-  // Now that the arguments are processed, handle the ObjC message.
-  VisitObjCMessage(ME, ArgsEvaluated, Dst);
-}
-
-void ExprEngine::VisitObjCMessage(const ObjCMessage &msg,
-                                  ExplodedNodeSet &Src, ExplodedNodeSet& Dst) {
-
-  // Handle the previsits checks.
-  ExplodedNodeSet DstPrevisit;
-  getCheckerManager().runCheckersForPreObjCMessage(DstPrevisit, Src, msg,*this);
-
-  // Proceed with evaluate the message expression.
-  ExplodedNodeSet dstEval;
-
-  for (ExplodedNodeSet::iterator DI = DstPrevisit.begin(),
-                                 DE = DstPrevisit.end(); DI != DE; ++DI) {
-
-    ExplodedNode *Pred = *DI;
-    bool RaisesException = false;
-    unsigned oldSize = dstEval.size();
-    SaveAndRestore<bool> OldSink(Builder->BuildSinks);
-    SaveOr OldHasGen(Builder->hasGeneratedNode);
-
-    if (const Expr *Receiver = msg.getInstanceReceiver()) {
-      const GRState *state = GetState(Pred);
-      SVal recVal = state->getSVal(Receiver);
-      if (!recVal.isUndef()) {
-        // Bifurcate the state into nil and non-nil ones.
-        DefinedOrUnknownSVal receiverVal = cast<DefinedOrUnknownSVal>(recVal);
-    
-        const GRState *notNilState, *nilState;
-        llvm::tie(notNilState, nilState) = state->assume(receiverVal);
-    
-        // There are three cases: can be nil or non-nil, must be nil, must be
-        // non-nil. We ignore must be nil, and merge the rest two into non-nil.
-        if (nilState && !notNilState) {
-          dstEval.insert(Pred);
-          continue;
-        }
-    
-        // Check if the "raise" message was sent.
-        assert(notNilState);
-        if (msg.getSelector() == RaiseSel)
-          RaisesException = true;
-    
-        // Check if we raise an exception.  For now treat these as sinks.
-        // Eventually we will want to handle exceptions properly.
-        if (RaisesException)
-          Builder->BuildSinks = true;
-    
-        // Dispatch to plug-in transfer function.
-        evalObjCMessage(dstEval, msg, Pred, notNilState);
-      }
-    }
-    else if (const ObjCInterfaceDecl *Iface = msg.getReceiverInterface()) {
-      IdentifierInfo* ClsName = Iface->getIdentifier();
-      Selector S = msg.getSelector();
-
-      // Check for special instance methods.
-      if (!NSExceptionII) {
-        ASTContext& Ctx = getContext();
-        NSExceptionII = &Ctx.Idents.get("NSException");
-      }
-
-      if (ClsName == NSExceptionII) {
-        enum { NUM_RAISE_SELECTORS = 2 };
-
-        // Lazily create a cache of the selectors.
-        if (!NSExceptionInstanceRaiseSelectors) {
-          ASTContext& Ctx = getContext();
-          NSExceptionInstanceRaiseSelectors =
-            new Selector[NUM_RAISE_SELECTORS];
-          llvm::SmallVector<IdentifierInfo*, NUM_RAISE_SELECTORS> II;
-          unsigned idx = 0;
-
-          // raise:format:
-          II.push_back(&Ctx.Idents.get("raise"));
-          II.push_back(&Ctx.Idents.get("format"));
-          NSExceptionInstanceRaiseSelectors[idx++] =
-            Ctx.Selectors.getSelector(II.size(), &II[0]);
-
-          // raise:format::arguments:
-          II.push_back(&Ctx.Idents.get("arguments"));
-          NSExceptionInstanceRaiseSelectors[idx++] =
-            Ctx.Selectors.getSelector(II.size(), &II[0]);
-        }
-
-        for (unsigned i = 0; i < NUM_RAISE_SELECTORS; ++i)
-          if (S == NSExceptionInstanceRaiseSelectors[i]) {
-            RaisesException = true;
-            break;
-          }
-      }
-
-      // Check if we raise an exception.  For now treat these as sinks.
-      // Eventually we will want to handle exceptions properly.
-      if (RaisesException)
-        Builder->BuildSinks = true;
-
-      // Dispatch to plug-in transfer function.
-      evalObjCMessage(dstEval, msg, Pred, Builder->GetState(Pred));
-    }
-
-    // Handle the case where no nodes where generated.  Auto-generate that
-    // contains the updated state if we aren't generating sinks.
-    if (!Builder->BuildSinks && dstEval.size() == oldSize &&
-        !Builder->hasGeneratedNode)
-      MakeNode(dstEval, msg.getOriginExpr(), Pred, GetState(Pred));
-  }
-
-  // Finally, perform the post-condition check of the ObjCMessageExpr and store
-  // the created nodes in 'Dst'.
-  getCheckerManager().runCheckersForPostObjCMessage(Dst, dstEval, msg, *this);
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer functions: Miscellaneous statements.
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, 
-                           ExplodedNode *Pred, ExplodedNodeSet &Dst) {
-  
-  ExplodedNodeSet S1;
-  Visit(Ex, Pred, S1);
-  ExplodedNodeSet S2;
-  getCheckerManager().runCheckersForPreStmt(S2, S1, CastE, *this);
-  
-  if (CastE->getCastKind() == CK_LValueToRValue ||
-      CastE->getCastKind() == CK_GetObjCProperty) {
-    for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I!=E; ++I) {
-      ExplodedNode *subExprNode = *I;
-      const GRState *state = GetState(subExprNode);
-      evalLoad(Dst, CastE, subExprNode, state, state->getSVal(Ex));
-    }
-    return;
-  }
-  
-  // All other casts.  
-  QualType T = CastE->getType();
-  QualType ExTy = Ex->getType();
-
-  if (const ExplicitCastExpr *ExCast=dyn_cast_or_null<ExplicitCastExpr>(CastE))
-    T = ExCast->getTypeAsWritten();
-
-  for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I) {
-    Pred = *I;
-
-    switch (CastE->getCastKind()) {
-      case CK_LValueToRValue:
-        assert(false && "LValueToRValue casts handled earlier.");
-      case CK_GetObjCProperty:
-        assert(false && "GetObjCProperty casts handled earlier.");
-      case CK_ToVoid:
-        Dst.Add(Pred);
-        continue;
-      // The analyzer doesn't do anything special with these casts,
-      // since it understands retain/release semantics already.
-      case CK_ObjCProduceObject:
-      case CK_ObjCConsumeObject:
-      case CK_ObjCReclaimReturnedObject: // Fall-through.
-      // True no-ops.
-      case CK_NoOp:
-      case CK_FunctionToPointerDecay: {
-        // Copy the SVal of Ex to CastE.
-        const GRState *state = GetState(Pred);
-        SVal V = state->getSVal(Ex);
-        state = state->BindExpr(CastE, V);
-        MakeNode(Dst, CastE, Pred, state);
-        continue;
-      }
-      case CK_Dependent:
-      case CK_ArrayToPointerDecay:
-      case CK_BitCast:
-      case CK_LValueBitCast:
-      case CK_IntegralCast:
-      case CK_NullToPointer:
-      case CK_IntegralToPointer:
-      case CK_PointerToIntegral:
-      case CK_PointerToBoolean:
-      case CK_IntegralToBoolean:
-      case CK_IntegralToFloating:
-      case CK_FloatingToIntegral:
-      case CK_FloatingToBoolean:
-      case CK_FloatingCast:
-      case CK_FloatingRealToComplex:
-      case CK_FloatingComplexToReal:
-      case CK_FloatingComplexToBoolean:
-      case CK_FloatingComplexCast:
-      case CK_FloatingComplexToIntegralComplex:
-      case CK_IntegralRealToComplex:
-      case CK_IntegralComplexToReal:
-      case CK_IntegralComplexToBoolean:
-      case CK_IntegralComplexCast:
-      case CK_IntegralComplexToFloatingComplex:
-      case CK_AnyPointerToObjCPointerCast:
-      case CK_AnyPointerToBlockPointerCast:  
-      case CK_ObjCObjectLValueCast: {
-        // Delegate to SValBuilder to process.
-        const GRState* state = GetState(Pred);
-        SVal V = state->getSVal(Ex);
-        V = svalBuilder.evalCast(V, T, ExTy);
-        state = state->BindExpr(CastE, V);
-        MakeNode(Dst, CastE, Pred, state);
-        continue;
-      }
-      case CK_DerivedToBase:
-      case CK_UncheckedDerivedToBase: {
-        // For DerivedToBase cast, delegate to the store manager.
-        const GRState *state = GetState(Pred);
-        SVal val = state->getSVal(Ex);
-        val = getStoreManager().evalDerivedToBase(val, T);
-        state = state->BindExpr(CastE, val);
-        MakeNode(Dst, CastE, Pred, state);
-        continue;
-      }
-      // Various C++ casts that are not handled yet.
-      case CK_Dynamic:
-      case CK_ToUnion:
-      case CK_BaseToDerived:
-      case CK_NullToMemberPointer:
-      case CK_BaseToDerivedMemberPointer:
-      case CK_DerivedToBaseMemberPointer:
-      case CK_UserDefinedConversion:
-      case CK_ConstructorConversion:
-      case CK_VectorSplat:
-      case CK_MemberPointerToBoolean: {
-        // Recover some path-sensitivty by conjuring a new value.
-        QualType resultType = CastE->getType();
-        if (CastE->isLValue())
-          resultType = getContext().getPointerType(resultType);
-
-        SVal result =
-          svalBuilder.getConjuredSymbolVal(NULL, CastE, resultType,
-                                           Builder->getCurrentBlockCount());
-
-        const GRState *state = GetState(Pred)->BindExpr(CastE, result);
-        MakeNode(Dst, CastE, Pred, state);
-        continue;
-      }
-    }
-  }
-}
-
-void ExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL,
-                                            ExplodedNode* Pred,
-                                            ExplodedNodeSet& Dst) {
-  const InitListExpr* ILE 
-    = cast<InitListExpr>(CL->getInitializer()->IgnoreParens());
-  ExplodedNodeSet Tmp;
-  Visit(ILE, Pred, Tmp);
-
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), EI = Tmp.end(); I!=EI; ++I) {
-    const GRState* state = GetState(*I);
-    SVal ILV = state->getSVal(ILE);
-    const LocationContext *LC = (*I)->getLocationContext();
-    state = state->bindCompoundLiteral(CL, LC, ILV);
-
-    if (CL->isLValue()) {
-      MakeNode(Dst, CL, *I, state->BindExpr(CL, state->getLValue(CL, LC)));
-    }
-    else
-      MakeNode(Dst, CL, *I, state->BindExpr(CL, ILV));
-  }
-}
-
-void ExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
-                                 ExplodedNodeSet& Dst) {
-
-  // The CFG has one DeclStmt per Decl.
-  const Decl* D = *DS->decl_begin();
-
-  if (!D || !isa<VarDecl>(D))
-    return;
-
-  const VarDecl* VD = dyn_cast<VarDecl>(D);
-  const Expr* InitEx = VD->getInit();
-
-  // FIXME: static variables may have an initializer, but the second
-  //  time a function is called those values may not be current.
-  ExplodedNodeSet Tmp;
-
-  if (InitEx)
-    Visit(InitEx, Pred, Tmp);
-  else
-    Tmp.Add(Pred);
-
-  ExplodedNodeSet Tmp2;
-  getCheckerManager().runCheckersForPreStmt(Tmp2, Tmp, DS, *this);
-
-  for (ExplodedNodeSet::iterator I=Tmp2.begin(), E=Tmp2.end(); I!=E; ++I) {
-    ExplodedNode *N = *I;
-    const GRState *state = GetState(N);
-
-    // Decls without InitExpr are not initialized explicitly.
-    const LocationContext *LC = N->getLocationContext();
-
-    if (InitEx) {
-      SVal InitVal = state->getSVal(InitEx);
-
-      // We bound the temp obj region to the CXXConstructExpr. Now recover
-      // the lazy compound value when the variable is not a reference.
-      if (AMgr.getLangOptions().CPlusPlus && VD->getType()->isRecordType() && 
-          !VD->getType()->isReferenceType() && isa<loc::MemRegionVal>(InitVal)){
-        InitVal = state->getSVal(cast<loc::MemRegionVal>(InitVal).getRegion());
-        assert(isa<nonloc::LazyCompoundVal>(InitVal));
-      }
-
-      // Recover some path-sensitivity if a scalar value evaluated to
-      // UnknownVal.
-      if ((InitVal.isUnknown() ||
-          !getConstraintManager().canReasonAbout(InitVal)) &&
-          !VD->getType()->isReferenceType()) {
-        InitVal = svalBuilder.getConjuredSymbolVal(NULL, InitEx,
-                                               Builder->getCurrentBlockCount());
-      }
-
-      evalBind(Dst, DS, *I, state,
-               loc::MemRegionVal(state->getRegion(VD, LC)), InitVal, true);
-    }
-    else {
-      state = state->bindDeclWithNoInit(state->getRegion(VD, LC));
-      MakeNode(Dst, DS, *I, state);
-    }
-  }
-}
-
-namespace {
-  // This class is used by VisitInitListExpr as an item in a worklist
-  // for processing the values contained in an InitListExpr.
-class InitListWLItem {
-public:
-  llvm::ImmutableList<SVal> Vals;
-  ExplodedNode* N;
-  InitListExpr::const_reverse_iterator Itr;
-
-  InitListWLItem(ExplodedNode* n, llvm::ImmutableList<SVal> vals,
-                 InitListExpr::const_reverse_iterator itr)
-  : Vals(vals), N(n), Itr(itr) {}
-};
-}
-
-
-void ExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
-                                     ExplodedNodeSet& Dst) {
-
-  const GRState* state = GetState(Pred);
-  QualType T = getContext().getCanonicalType(E->getType());
-  unsigned NumInitElements = E->getNumInits();
-
-  if (T->isArrayType() || T->isRecordType() || T->isVectorType()) {
-    llvm::ImmutableList<SVal> StartVals = getBasicVals().getEmptySValList();
-
-    // Handle base case where the initializer has no elements.
-    // e.g: static int* myArray[] = {};
-    if (NumInitElements == 0) {
-      SVal V = svalBuilder.makeCompoundVal(T, StartVals);
-      MakeNode(Dst, E, Pred, state->BindExpr(E, V));
-      return;
-    }
-
-    // Create a worklist to process the initializers.
-    llvm::SmallVector<InitListWLItem, 10> WorkList;
-    WorkList.reserve(NumInitElements);
-    WorkList.push_back(InitListWLItem(Pred, StartVals, E->rbegin()));
-    InitListExpr::const_reverse_iterator ItrEnd = E->rend();
-    assert(!(E->rbegin() == E->rend()));
-
-    // Process the worklist until it is empty.
-    while (!WorkList.empty()) {
-      InitListWLItem X = WorkList.back();
-      WorkList.pop_back();
-
-      ExplodedNodeSet Tmp;
-      Visit(*X.Itr, X.N, Tmp);
-
-      InitListExpr::const_reverse_iterator NewItr = X.Itr + 1;
-
-      for (ExplodedNodeSet::iterator NI=Tmp.begin(),NE=Tmp.end();NI!=NE;++NI) {
-        // Get the last initializer value.
-        state = GetState(*NI);
-        SVal InitV = state->getSVal(cast<Expr>(*X.Itr));
-
-        // Construct the new list of values by prepending the new value to
-        // the already constructed list.
-        llvm::ImmutableList<SVal> NewVals =
-          getBasicVals().consVals(InitV, X.Vals);
-
-        if (NewItr == ItrEnd) {
-          // Now we have a list holding all init values. Make CompoundValData.
-          SVal V = svalBuilder.makeCompoundVal(T, NewVals);
-
-          // Make final state and node.
-          MakeNode(Dst, E, *NI, state->BindExpr(E, V));
-        }
-        else {
-          // Still some initializer values to go.  Push them onto the worklist.
-          WorkList.push_back(InitListWLItem(*NI, NewVals, NewItr));
-        }
-      }
-    }
-
-    return;
-  }
-
-  if (Loc::isLocType(T) || T->isIntegerType()) {
-    assert (E->getNumInits() == 1);
-    ExplodedNodeSet Tmp;
-    const Expr* Init = E->getInit(0);
-    Visit(Init, Pred, Tmp);
-    for (ExplodedNodeSet::iterator I=Tmp.begin(), EI=Tmp.end(); I != EI; ++I) {
-      state = GetState(*I);
-      MakeNode(Dst, E, *I, state->BindExpr(E, state->getSVal(Init)));
-    }
-    return;
-  }
-
-  assert(0 && "unprocessed InitListExpr type");
-}
-
-/// VisitUnaryExprOrTypeTraitExpr - Transfer function for sizeof(type).
-void ExprEngine::VisitUnaryExprOrTypeTraitExpr(
-                                          const UnaryExprOrTypeTraitExpr* Ex,
-                                          ExplodedNode* Pred,
-                                          ExplodedNodeSet& Dst) {
-  QualType T = Ex->getTypeOfArgument();
-
-  if (Ex->getKind() == UETT_SizeOf) {
-    if (!T->isIncompleteType() && !T->isConstantSizeType()) {
-      assert(T->isVariableArrayType() && "Unknown non-constant-sized type.");
-
-      // FIXME: Add support for VLA type arguments, not just VLA expressions.
-      // When that happens, we should probably refactor VLASizeChecker's code.
-      if (Ex->isArgumentType()) {
-        Dst.Add(Pred);
-        return;
-      }
-
-      // Get the size by getting the extent of the sub-expression.
-      // First, visit the sub-expression to find its region.
-      const Expr *Arg = Ex->getArgumentExpr();
-      ExplodedNodeSet Tmp;
-      Visit(Arg, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-        const GRState* state = GetState(*I);
-        const MemRegion *MR = state->getSVal(Arg).getAsRegion();
-
-        // If the subexpression can't be resolved to a region, we don't know
-        // anything about its size. Just leave the state as is and continue.
-        if (!MR) {
-          Dst.Add(*I);
-          continue;
-        }
-
-        // The result is the extent of the VLA.
-        SVal Extent = cast<SubRegion>(MR)->getExtent(svalBuilder);
-        MakeNode(Dst, Ex, *I, state->BindExpr(Ex, Extent));
-      }
-
-      return;
-    }
-    else if (T->getAs<ObjCObjectType>()) {
-      // Some code tries to take the sizeof an ObjCObjectType, relying that
-      // the compiler has laid out its representation.  Just report Unknown
-      // for these.
-      Dst.Add(Pred);
-      return;
-    }
-  }
-
-  Expr::EvalResult Result;
-  Ex->Evaluate(Result, getContext());
-  CharUnits amt = CharUnits::fromQuantity(Result.Val.getInt().getZExtValue());
-
-  MakeNode(Dst, Ex, Pred,
-           GetState(Pred)->BindExpr(Ex,
-              svalBuilder.makeIntVal(amt.getQuantity(), Ex->getType())));
-}
-
-void ExprEngine::VisitOffsetOfExpr(const OffsetOfExpr* OOE, 
-                                     ExplodedNode* Pred, ExplodedNodeSet& Dst) {
-  Expr::EvalResult Res;
-  if (OOE->Evaluate(Res, getContext()) && Res.Val.isInt()) {
-    const APSInt &IV = Res.Val.getInt();
-    assert(IV.getBitWidth() == getContext().getTypeSize(OOE->getType()));
-    assert(OOE->getType()->isIntegerType());
-    assert(IV.isSigned() == OOE->getType()->isSignedIntegerOrEnumerationType());
-    SVal X = svalBuilder.makeIntVal(IV);
-    MakeNode(Dst, OOE, Pred, GetState(Pred)->BindExpr(OOE, X));
-    return;
-  }
-  // FIXME: Handle the case where __builtin_offsetof is not a constant.
-  Dst.Add(Pred);
-}
-
-void ExprEngine::VisitUnaryOperator(const UnaryOperator* U, 
-                                      ExplodedNode* Pred,
-                                      ExplodedNodeSet& Dst) {
-
-  switch (U->getOpcode()) {
-
-    default:
-      break;
-
-    case UO_Real: {
-      const Expr* Ex = U->getSubExpr()->IgnoreParens();
-      ExplodedNodeSet Tmp;
-      Visit(Ex, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-
-        // FIXME: We don't have complex SValues yet.
-        if (Ex->getType()->isAnyComplexType()) {
-          // Just report "Unknown."
-          Dst.Add(*I);
-          continue;
-        }
-
-        // For all other types, UO_Real is an identity operation.
-        assert (U->getType() == Ex->getType());
-        const GRState* state = GetState(*I);
-        MakeNode(Dst, U, *I, state->BindExpr(U, state->getSVal(Ex)));
-      }
-
-      return;
-    }
-
-    case UO_Imag: {
-
-      const Expr* Ex = U->getSubExpr()->IgnoreParens();
-      ExplodedNodeSet Tmp;
-      Visit(Ex, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-        // FIXME: We don't have complex SValues yet.
-        if (Ex->getType()->isAnyComplexType()) {
-          // Just report "Unknown."
-          Dst.Add(*I);
-          continue;
-        }
-
-        // For all other types, UO_Imag returns 0.
-        const GRState* state = GetState(*I);
-        SVal X = svalBuilder.makeZeroVal(Ex->getType());
-        MakeNode(Dst, U, *I, state->BindExpr(U, X));
-      }
-
-      return;
-    }
-      
-    case UO_Plus:
-      assert(!U->isLValue());
-      // FALL-THROUGH.
-    case UO_Deref:
-    case UO_AddrOf:
-    case UO_Extension: {
-
-      // Unary "+" is a no-op, similar to a parentheses.  We still have places
-      // where it may be a block-level expression, so we need to
-      // generate an extra node that just propagates the value of the
-      // subexpression.
-
-      const Expr* Ex = U->getSubExpr()->IgnoreParens();
-      ExplodedNodeSet Tmp;
-      Visit(Ex, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-        const GRState* state = GetState(*I);
-        MakeNode(Dst, U, *I, state->BindExpr(U, state->getSVal(Ex)));
-      }
-
-      return;
-    }
-
-    case UO_LNot:
-    case UO_Minus:
-    case UO_Not: {
-      assert (!U->isLValue());
-      const Expr* Ex = U->getSubExpr()->IgnoreParens();
-      ExplodedNodeSet Tmp;
-      Visit(Ex, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-        const GRState* state = GetState(*I);
-
-        // Get the value of the subexpression.
-        SVal V = state->getSVal(Ex);
-
-        if (V.isUnknownOrUndef()) {
-          MakeNode(Dst, U, *I, state->BindExpr(U, V));
-          continue;
-        }
-
-//        QualType DstT = getContext().getCanonicalType(U->getType());
-//        QualType SrcT = getContext().getCanonicalType(Ex->getType());
-//
-//        if (DstT != SrcT) // Perform promotions.
-//          V = evalCast(V, DstT);
-//
-//        if (V.isUnknownOrUndef()) {
-//          MakeNode(Dst, U, *I, BindExpr(St, U, V));
-//          continue;
-//        }
-
-        switch (U->getOpcode()) {
-          default:
-            assert(false && "Invalid Opcode.");
-            break;
-
-          case UO_Not:
-            // FIXME: Do we need to handle promotions?
-            state = state->BindExpr(U, evalComplement(cast<NonLoc>(V)));
-            break;
-
-          case UO_Minus:
-            // FIXME: Do we need to handle promotions?
-            state = state->BindExpr(U, evalMinus(cast<NonLoc>(V)));
-            break;
-
-          case UO_LNot:
-
-            // C99 6.5.3.3: "The expression !E is equivalent to (0==E)."
-            //
-            //  Note: technically we do "E == 0", but this is the same in the
-            //    transfer functions as "0 == E".
-            SVal Result;
-
-            if (isa<Loc>(V)) {
-              Loc X = svalBuilder.makeNull();
-              Result = evalBinOp(state, BO_EQ, cast<Loc>(V), X,
-                                 U->getType());
-            }
-            else {
-              nonloc::ConcreteInt X(getBasicVals().getValue(0, Ex->getType()));
-              Result = evalBinOp(state, BO_EQ, cast<NonLoc>(V), X,
-                                 U->getType());
-            }
-
-            state = state->BindExpr(U, Result);
-
-            break;
-        }
-
-        MakeNode(Dst, U, *I, state);
-      }
-
-      return;
-    }
-  }
-
-  // Handle ++ and -- (both pre- and post-increment).
-  assert (U->isIncrementDecrementOp());
-  ExplodedNodeSet Tmp;
-  const Expr* Ex = U->getSubExpr()->IgnoreParens();
-  Visit(Ex, Pred, Tmp);
-
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I!=E; ++I) {
-
-    const GRState* state = GetState(*I);
-    SVal loc = state->getSVal(Ex);
-
-    // Perform a load.
-    ExplodedNodeSet Tmp2;
-    evalLoad(Tmp2, Ex, *I, state, loc);
-
-    for (ExplodedNodeSet::iterator I2=Tmp2.begin(), E2=Tmp2.end();I2!=E2;++I2) {
-
-      state = GetState(*I2);
-      SVal V2_untested = state->getSVal(Ex);
-
-      // Propagate unknown and undefined values.
-      if (V2_untested.isUnknownOrUndef()) {
-        MakeNode(Dst, U, *I2, state->BindExpr(U, V2_untested));
-        continue;
-      }
-      DefinedSVal V2 = cast<DefinedSVal>(V2_untested);
-
-      // Handle all other values.
-      BinaryOperator::Opcode Op = U->isIncrementOp() ? BO_Add
-                                                     : BO_Sub;
-
-      // If the UnaryOperator has non-location type, use its type to create the
-      // constant value. If the UnaryOperator has location type, create the
-      // constant with int type and pointer width.
-      SVal RHS;
-
-      if (U->getType()->isAnyPointerType())
-        RHS = svalBuilder.makeArrayIndex(1);
-      else
-        RHS = svalBuilder.makeIntVal(1, U->getType());
-
-      SVal Result = evalBinOp(state, Op, V2, RHS, U->getType());
-
-      // Conjure a new symbol if necessary to recover precision.
-      if (Result.isUnknown() || !getConstraintManager().canReasonAbout(Result)){
-        DefinedOrUnknownSVal SymVal =
-          svalBuilder.getConjuredSymbolVal(NULL, Ex,
-                                      Builder->getCurrentBlockCount());
-        Result = SymVal;
-
-        // If the value is a location, ++/-- should always preserve
-        // non-nullness.  Check if the original value was non-null, and if so
-        // propagate that constraint.
-        if (Loc::isLocType(U->getType())) {
-          DefinedOrUnknownSVal Constraint =
-            svalBuilder.evalEQ(state, V2,svalBuilder.makeZeroVal(U->getType()));
-
-          if (!state->assume(Constraint, true)) {
-            // It isn't feasible for the original value to be null.
-            // Propagate this constraint.
-            Constraint = svalBuilder.evalEQ(state, SymVal,
-                                       svalBuilder.makeZeroVal(U->getType()));
-
-
-            state = state->assume(Constraint, false);
-            assert(state);
-          }
-        }
-      }
-
-      // Since the lvalue-to-rvalue conversion is explicit in the AST,
-      // we bind an l-value if the operator is prefix and an lvalue (in C++).
-      if (U->isLValue())
-        state = state->BindExpr(U, loc);
-      else
-        state = state->BindExpr(U, U->isPostfix() ? V2 : Result);
-
-      // Perform the store.
-      evalStore(Dst, NULL, U, *I2, state, loc, Result);
-    }
-  }
-}
-
-void ExprEngine::VisitAsmStmt(const AsmStmt* A, ExplodedNode* Pred,
-                                ExplodedNodeSet& Dst) {
+void ExprEngine::VisitAsmStmt(const AsmStmt *A, ExplodedNode *Pred,
+                                ExplodedNodeSet &Dst) {
   VisitAsmStmtHelperOutputs(A, A->begin_outputs(), A->end_outputs(), Pred, Dst);
 }
 
-void ExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt* A,
+void ExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt *A,
                                              AsmStmt::const_outputs_iterator I,
                                              AsmStmt::const_outputs_iterator E,
-                                     ExplodedNode* Pred, ExplodedNodeSet& Dst) {
+                                     ExplodedNode *Pred, ExplodedNodeSet &Dst) {
   if (I == E) {
     VisitAsmStmtHelperInputs(A, A->begin_inputs(), A->end_inputs(), Pred, Dst);
     return;
@@ -2795,11 +1590,11 @@ void ExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt* A,
     VisitAsmStmtHelperOutputs(A, I, E, *NI, Dst);
 }
 
-void ExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
+void ExprEngine::VisitAsmStmtHelperInputs(const AsmStmt *A,
                                             AsmStmt::const_inputs_iterator I,
                                             AsmStmt::const_inputs_iterator E,
-                                            ExplodedNode* Pred,
-                                            ExplodedNodeSet& Dst) {
+                                            ExplodedNode *Pred,
+                                            ExplodedNodeSet &Dst) {
   if (I == E) {
 
     // We have processed both the inputs and the outputs.  All of the outputs
@@ -2809,7 +1604,7 @@ void ExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
     // which interprets the inline asm and stores proper results in the
     // outputs.
 
-    const GRState* state = GetState(Pred);
+    const ProgramState *state = Pred->getState();
 
     for (AsmStmt::const_outputs_iterator OI = A->begin_outputs(),
                                    OE = A->end_outputs(); OI != OE; ++OI) {
@@ -2834,198 +1629,6 @@ void ExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
     VisitAsmStmtHelperInputs(A, I, E, *NI, Dst);
 }
 
-void ExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
-                                   ExplodedNodeSet &Dst) {
-  ExplodedNodeSet Src;
-  if (const Expr *RetE = RS->getRetValue()) {
-    // Record the returned expression in the state. It will be used in
-    // processCallExit to bind the return value to the call expr.
-    {
-      static int tag = 0;
-      const GRState *state = GetState(Pred);
-      state = state->set<ReturnExpr>(RetE);
-      Pred = Builder->generateNode(RetE, state, Pred, &tag);
-    }
-    // We may get a NULL Pred because we generated a cached node.
-    if (Pred)
-      Visit(RetE, Pred, Src);
-  }
-  else {
-    Src.Add(Pred);
-  }
-
-  ExplodedNodeSet CheckedSet;
-  getCheckerManager().runCheckersForPreStmt(CheckedSet, Src, RS, *this);
-
-  for (ExplodedNodeSet::iterator I = CheckedSet.begin(), E = CheckedSet.end();
-       I != E; ++I) {
-
-    assert(Builder && "StmtNodeBuilder must be defined.");
-
-    Pred = *I;
-    unsigned size = Dst.size();
-
-    SaveAndRestore<bool> OldSink(Builder->BuildSinks);
-    SaveOr OldHasGen(Builder->hasGeneratedNode);
-
-    getTF().evalReturn(Dst, *this, *Builder, RS, Pred);
-
-    // Handle the case where no nodes where generated.
-    if (!Builder->BuildSinks && Dst.size() == size &&
-        !Builder->hasGeneratedNode)
-      MakeNode(Dst, RS, Pred, GetState(Pred));
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Transfer functions: Binary operators.
-//===----------------------------------------------------------------------===//
-
-void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
-                                       ExplodedNode* Pred,
-                                       ExplodedNodeSet& Dst) {
-  ExplodedNodeSet Tmp1;
-  Expr* LHS = B->getLHS()->IgnoreParens();
-  Expr* RHS = B->getRHS()->IgnoreParens();
-
-  Visit(LHS, Pred, Tmp1);
-  ExplodedNodeSet Tmp3;
-
-  for (ExplodedNodeSet::iterator I1=Tmp1.begin(), E1=Tmp1.end(); I1!=E1; ++I1) {
-    SVal LeftV = GetState(*I1)->getSVal(LHS);
-    ExplodedNodeSet Tmp2;
-    Visit(RHS, *I1, Tmp2);
-
-    ExplodedNodeSet CheckedSet;
-    getCheckerManager().runCheckersForPreStmt(CheckedSet, Tmp2, B, *this);
-
-    // With both the LHS and RHS evaluated, process the operation itself.
-
-    for (ExplodedNodeSet::iterator I2=CheckedSet.begin(), E2=CheckedSet.end();
-         I2 != E2; ++I2) {
-
-      const GRState *state = GetState(*I2);
-      SVal RightV = state->getSVal(RHS);
-
-      BinaryOperator::Opcode Op = B->getOpcode();
-
-      if (Op == BO_Assign) {
-        // EXPERIMENTAL: "Conjured" symbols.
-        // FIXME: Handle structs.
-        if (RightV.isUnknown() ||!getConstraintManager().canReasonAbout(RightV))
-        {
-          unsigned Count = Builder->getCurrentBlockCount();
-          RightV = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), Count);
-        }
-
-        SVal ExprVal = B->isLValue() ? LeftV : RightV;
-
-        // Simulate the effects of a "store":  bind the value of the RHS
-        // to the L-Value represented by the LHS.
-        evalStore(Tmp3, B, LHS, *I2, state->BindExpr(B, ExprVal), LeftV,RightV);
-        continue;
-      }
-
-      if (!B->isAssignmentOp()) {
-        // Process non-assignments except commas or short-circuited
-        // logical expressions (LAnd and LOr).
-        SVal Result = evalBinOp(state, Op, LeftV, RightV, B->getType());
-
-        if (Result.isUnknown()) {
-          MakeNode(Tmp3, B, *I2, state);
-          continue;
-        }
-
-        state = state->BindExpr(B, Result);
-
-        MakeNode(Tmp3, B, *I2, state);
-        continue;
-      }
-
-      assert (B->isCompoundAssignmentOp());
-
-      switch (Op) {
-        default:
-          assert(0 && "Invalid opcode for compound assignment.");
-        case BO_MulAssign: Op = BO_Mul; break;
-        case BO_DivAssign: Op = BO_Div; break;
-        case BO_RemAssign: Op = BO_Rem; break;
-        case BO_AddAssign: Op = BO_Add; break;
-        case BO_SubAssign: Op = BO_Sub; break;
-        case BO_ShlAssign: Op = BO_Shl; break;
-        case BO_ShrAssign: Op = BO_Shr; break;
-        case BO_AndAssign: Op = BO_And; break;
-        case BO_XorAssign: Op = BO_Xor; break;
-        case BO_OrAssign:  Op = BO_Or;  break;
-      }
-
-      // Perform a load (the LHS).  This performs the checks for
-      // null dereferences, and so on.
-      ExplodedNodeSet Tmp4;
-      SVal location = state->getSVal(LHS);
-      evalLoad(Tmp4, LHS, *I2, state, location);
-
-      for (ExplodedNodeSet::iterator I4=Tmp4.begin(), E4=Tmp4.end(); I4!=E4;
-           ++I4) {
-        state = GetState(*I4);
-        SVal V = state->getSVal(LHS);
-
-        // Get the computation type.
-        QualType CTy =
-          cast<CompoundAssignOperator>(B)->getComputationResultType();
-        CTy = getContext().getCanonicalType(CTy);
-
-        QualType CLHSTy =
-          cast<CompoundAssignOperator>(B)->getComputationLHSType();
-        CLHSTy = getContext().getCanonicalType(CLHSTy);
-
-        QualType LTy = getContext().getCanonicalType(LHS->getType());
-
-        // Promote LHS.
-        V = svalBuilder.evalCast(V, CLHSTy, LTy);
-
-        // Compute the result of the operation.
-        SVal Result = svalBuilder.evalCast(evalBinOp(state, Op, V, RightV, CTy),
-                                      B->getType(), CTy);
-
-        // EXPERIMENTAL: "Conjured" symbols.
-        // FIXME: Handle structs.
-
-        SVal LHSVal;
-
-        if (Result.isUnknown() ||
-            !getConstraintManager().canReasonAbout(Result)) {
-
-          unsigned Count = Builder->getCurrentBlockCount();
-
-          // The symbolic value is actually for the type of the left-hand side
-          // expression, not the computation type, as this is the value the
-          // LValue on the LHS will bind to.
-          LHSVal = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), LTy, Count);
-
-          // However, we need to convert the symbol to the computation type.
-          Result = svalBuilder.evalCast(LHSVal, CTy, LTy);
-        }
-        else {
-          // The left-hand side may bind to a different value then the
-          // computation type.
-          LHSVal = svalBuilder.evalCast(Result, LTy, CTy);
-        }
-
-        // In C++, assignment and compound assignment operators return an 
-        // lvalue.
-        if (B->isLValue())
-          state = state->BindExpr(B, location);
-        else
-          state = state->BindExpr(B, Result);
-
-        evalStore(Tmp3, B, LHS, *I4, state, location, LHSVal);
-      }
-    }
-  }
-
-  getCheckerManager().runCheckersForPostStmt(Dst, Tmp3, B, *this);
-}
 
 //===----------------------------------------------------------------------===//
 // Visualization.
@@ -3044,7 +1647,7 @@ struct DOTGraphTraits<ExplodedNode*> :
 
   // FIXME: Since we do not cache error nodes in ExprEngine now, this does not
   // work.
-  static std::string getNodeAttributes(const ExplodedNode* N, void*) {
+  static std::string getNodeAttributes(const ExplodedNode *N, void*) {
 
 #if 0
       // FIXME: Replace with a general scheme to tell if the node is
@@ -3065,7 +1668,7 @@ struct DOTGraphTraits<ExplodedNode*> :
     return "";
   }
 
-  static std::string getNodeLabel(const ExplodedNode* N, void*){
+  static std::string getNodeLabel(const ExplodedNode *N, void*){
 
     std::string sbuf;
     llvm::raw_string_ostream Out(sbuf);
@@ -3093,7 +1696,7 @@ struct DOTGraphTraits<ExplodedNode*> :
 
       default: {
         if (StmtPoint *L = dyn_cast<StmtPoint>(&Loc)) {
-          const Stmt* S = L->getStmt();
+          const Stmt *S = L->getStmt();
           SourceLocation SLoc = S->getLocStart();
 
           Out << S->getStmtClassName() << ' ' << (void*) S << ' ';
@@ -3102,9 +1705,9 @@ struct DOTGraphTraits<ExplodedNode*> :
 
           if (SLoc.isFileID()) {
             Out << "\\lline="
-              << GraphPrintSourceManager->getInstantiationLineNumber(SLoc)
+              << GraphPrintSourceManager->getExpansionLineNumber(SLoc)
               << " col="
-              << GraphPrintSourceManager->getInstantiationColumnNumber(SLoc)
+              << GraphPrintSourceManager->getExpansionColumnNumber(SLoc)
               << "\\l";
           }
 
@@ -3141,11 +1744,11 @@ struct DOTGraphTraits<ExplodedNode*> :
           break;
         }
 
-        const BlockEdge& E = cast<BlockEdge>(Loc);
+        const BlockEdge &E = cast<BlockEdge>(Loc);
         Out << "Edge: (B" << E.getSrc()->getBlockID() << ", B"
             << E.getDst()->getBlockID()  << ')';
 
-        if (const Stmt* T = E.getSrc()->getTerminator()) {
+        if (const Stmt *T = E.getSrc()->getTerminator()) {
 
           SourceLocation SLoc = T->getLocStart();
 
@@ -3155,21 +1758,21 @@ struct DOTGraphTraits<ExplodedNode*> :
 
           if (SLoc.isFileID()) {
             Out << "\\lline="
-              << GraphPrintSourceManager->getInstantiationLineNumber(SLoc)
+              << GraphPrintSourceManager->getExpansionLineNumber(SLoc)
               << " col="
-              << GraphPrintSourceManager->getInstantiationColumnNumber(SLoc);
+              << GraphPrintSourceManager->getExpansionColumnNumber(SLoc);
           }
 
           if (isa<SwitchStmt>(T)) {
-            const Stmt* Label = E.getDst()->getLabel();
+            const Stmt *Label = E.getDst()->getLabel();
 
             if (Label) {
-              if (const CaseStmt* C = dyn_cast<CaseStmt>(Label)) {
+              if (const CaseStmt *C = dyn_cast<CaseStmt>(Label)) {
                 Out << "\\lcase ";
                 LangOptions LO; // FIXME.
                 C->getLHS()->printPretty(Out, 0, PrintingPolicy(LO));
 
-                if (const Stmt* RHS = C->getRHS()) {
+                if (const Stmt *RHS = C->getRHS()) {
                   Out << " .. ";
                   RHS->printPretty(Out, 0, PrintingPolicy(LO));
                 }
@@ -3208,11 +1811,17 @@ struct DOTGraphTraits<ExplodedNode*> :
       }
     }
 
-    const GRState *state = N->getState();
+    const ProgramState *state = N->getState();
     Out << "\\|StateID: " << (void*) state
         << " NodeID: " << (void*) N << "\\|";
     state->printDOT(Out, *N->getLocationContext()->getCFG());
-    Out << "\\l";
+
+    Out << "\\l";    
+
+    if (const ProgramPointTag *tag = Loc.getTag()) {
+      Out << "\\|Tag: " << tag->getTagDescription(); 
+      Out << "\\l";
+    }
     return Out.str();
   }
 };
@@ -3221,7 +1830,7 @@ struct DOTGraphTraits<ExplodedNode*> :
 
 #ifndef NDEBUG
 template <typename ITERATOR>
-ExplodedNode* GetGraphNode(ITERATOR I) { return *I; }
+ExplodedNode *GetGraphNode(ITERATOR I) { return *I; }
 
 template <> ExplodedNode*
 GetGraphNode<llvm::DenseMap<ExplodedNode*, Expr*>::iterator>
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
new file mode 100644
index 000000000000..68ccc59ac952
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -0,0 +1,752 @@
+//=-- ExprEngineC.cpp - ExprEngine support for C expressions ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines ExprEngine's support for C expressions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/Analysis/Support/SaveAndRestore.h"
+
+using namespace clang;
+using namespace ento;
+using llvm::APSInt;
+
+void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
+                                     ExplodedNode *Pred,
+                                     ExplodedNodeSet &Dst) {
+
+  Expr *LHS = B->getLHS()->IgnoreParens();
+  Expr *RHS = B->getRHS()->IgnoreParens();
+  
+  // FIXME: Prechecks eventually go in ::Visit().
+  ExplodedNodeSet CheckedSet;
+  ExplodedNodeSet Tmp2;
+  getCheckerManager().runCheckersForPreStmt(CheckedSet, Pred, B, *this);
+    
+  // With both the LHS and RHS evaluated, process the operation itself.    
+  for (ExplodedNodeSet::iterator it=CheckedSet.begin(), ei=CheckedSet.end();
+         it != ei; ++it) {
+      
+    const ProgramState *state = (*it)->getState();
+    SVal LeftV = state->getSVal(LHS);
+    SVal RightV = state->getSVal(RHS);
+      
+    BinaryOperator::Opcode Op = B->getOpcode();
+      
+    if (Op == BO_Assign) {
+      // EXPERIMENTAL: "Conjured" symbols.
+      // FIXME: Handle structs.
+      if (RightV.isUnknown() ||
+          !getConstraintManager().canReasonAbout(RightV)) {
+        unsigned Count = Builder->getCurrentBlockCount();
+        RightV = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), Count);
+      }
+      // Simulate the effects of a "store":  bind the value of the RHS
+      // to the L-Value represented by the LHS.
+      SVal ExprVal = B->isLValue() ? LeftV : RightV;
+      evalStore(Tmp2, B, LHS, *it, state->BindExpr(B, ExprVal), LeftV, RightV);
+      continue;
+    }
+      
+    if (!B->isAssignmentOp()) {
+      // Process non-assignments except commas or short-circuited
+      // logical expressions (LAnd and LOr).
+      SVal Result = evalBinOp(state, Op, LeftV, RightV, B->getType());      
+      if (Result.isUnknown()) {
+        MakeNode(Tmp2, B, *it, state);
+        continue;
+      }        
+
+      state = state->BindExpr(B, Result);      
+      MakeNode(Tmp2, B, *it, state);
+      continue;
+    }
+      
+    assert (B->isCompoundAssignmentOp());
+    
+    switch (Op) {
+      default:
+        llvm_unreachable("Invalid opcode for compound assignment.");
+      case BO_MulAssign: Op = BO_Mul; break;
+      case BO_DivAssign: Op = BO_Div; break;
+      case BO_RemAssign: Op = BO_Rem; break;
+      case BO_AddAssign: Op = BO_Add; break;
+      case BO_SubAssign: Op = BO_Sub; break;
+      case BO_ShlAssign: Op = BO_Shl; break;
+      case BO_ShrAssign: Op = BO_Shr; break;
+      case BO_AndAssign: Op = BO_And; break;
+      case BO_XorAssign: Op = BO_Xor; break;
+      case BO_OrAssign:  Op = BO_Or;  break;
+    }
+      
+    // Perform a load (the LHS).  This performs the checks for
+    // null dereferences, and so on.
+    ExplodedNodeSet Tmp;
+    SVal location = LeftV;
+    evalLoad(Tmp, LHS, *it, state, location);
+    
+    for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I != E;
+         ++I) {
+
+      state = (*I)->getState();
+      SVal V = state->getSVal(LHS);
+      
+      // Get the computation type.
+      QualType CTy =
+        cast<CompoundAssignOperator>(B)->getComputationResultType();
+      CTy = getContext().getCanonicalType(CTy);
+      
+      QualType CLHSTy =
+        cast<CompoundAssignOperator>(B)->getComputationLHSType();
+      CLHSTy = getContext().getCanonicalType(CLHSTy);
+      
+      QualType LTy = getContext().getCanonicalType(LHS->getType());
+      
+      // Promote LHS.
+      V = svalBuilder.evalCast(V, CLHSTy, LTy);
+      
+      // Compute the result of the operation.
+      SVal Result = svalBuilder.evalCast(evalBinOp(state, Op, V, RightV, CTy),
+                                         B->getType(), CTy);
+      
+      // EXPERIMENTAL: "Conjured" symbols.
+      // FIXME: Handle structs.
+      
+      SVal LHSVal;
+      
+      if (Result.isUnknown() ||
+          !getConstraintManager().canReasonAbout(Result)) {
+        
+        unsigned Count = Builder->getCurrentBlockCount();
+        
+        // The symbolic value is actually for the type of the left-hand side
+        // expression, not the computation type, as this is the value the
+        // LValue on the LHS will bind to.
+        LHSVal = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), LTy,
+                                                  Count);
+        
+        // However, we need to convert the symbol to the computation type.
+        Result = svalBuilder.evalCast(LHSVal, CTy, LTy);
+      }
+      else {
+        // The left-hand side may bind to a different value then the
+        // computation type.
+        LHSVal = svalBuilder.evalCast(Result, LTy, CTy);
+      }
+      
+      // In C++, assignment and compound assignment operators return an 
+      // lvalue.
+      if (B->isLValue())
+        state = state->BindExpr(B, location);
+      else
+        state = state->BindExpr(B, Result);
+      
+      evalStore(Tmp2, B, LHS, *I, state, location, LHSVal);
+    }
+  }
+  
+  // FIXME: postvisits eventually go in ::Visit()
+  getCheckerManager().runCheckersForPostStmt(Dst, Tmp2, B, *this);
+}
+
+void ExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
+                                ExplodedNodeSet &Dst) {
+  
+  CanQualType T = getContext().getCanonicalType(BE->getType());
+  SVal V = svalBuilder.getBlockPointer(BE->getBlockDecl(), T,
+                                       Pred->getLocationContext());
+  
+  ExplodedNodeSet Tmp;
+  MakeNode(Tmp, BE, Pred, Pred->getState()->BindExpr(BE, V),
+           ProgramPoint::PostLValueKind);
+  
+  // FIXME: Move all post/pre visits to ::Visit().
+  getCheckerManager().runCheckersForPostStmt(Dst, Tmp, BE, *this);
+}
+
+void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, 
+                           ExplodedNode *Pred, ExplodedNodeSet &Dst) {
+  
+  ExplodedNodeSet dstPreStmt;
+  getCheckerManager().runCheckersForPreStmt(dstPreStmt, Pred, CastE, *this);
+  
+  if (CastE->getCastKind() == CK_LValueToRValue ||
+      CastE->getCastKind() == CK_GetObjCProperty) {
+    for (ExplodedNodeSet::iterator I = dstPreStmt.begin(), E = dstPreStmt.end();
+         I!=E; ++I) {
+      ExplodedNode *subExprNode = *I;
+      const ProgramState *state = subExprNode->getState();
+      evalLoad(Dst, CastE, subExprNode, state, state->getSVal(Ex));
+    }
+    return;
+  }
+  
+  // All other casts.  
+  QualType T = CastE->getType();
+  QualType ExTy = Ex->getType();
+  
+  if (const ExplicitCastExpr *ExCast=dyn_cast_or_null<ExplicitCastExpr>(CastE))
+    T = ExCast->getTypeAsWritten();
+  
+  for (ExplodedNodeSet::iterator I = dstPreStmt.begin(), E = dstPreStmt.end();
+       I != E; ++I) {
+    
+    Pred = *I;
+    
+    switch (CastE->getCastKind()) {
+      case CK_LValueToRValue:
+        llvm_unreachable("LValueToRValue casts handled earlier.");
+      case CK_GetObjCProperty:
+        llvm_unreachable("GetObjCProperty casts handled earlier.");
+      case CK_ToVoid:
+        Dst.Add(Pred);
+        continue;
+        // The analyzer doesn't do anything special with these casts,
+        // since it understands retain/release semantics already.
+      case CK_ARCProduceObject:
+      case CK_ARCConsumeObject:
+      case CK_ARCReclaimReturnedObject:
+      case CK_ARCExtendBlockObject: // Fall-through.
+        // True no-ops.
+      case CK_NoOp:
+      case CK_FunctionToPointerDecay: {
+        // Copy the SVal of Ex to CastE.
+        const ProgramState *state = Pred->getState();
+        SVal V = state->getSVal(Ex);
+        state = state->BindExpr(CastE, V);
+        MakeNode(Dst, CastE, Pred, state);
+        continue;
+      }
+      case CK_Dependent:
+      case CK_ArrayToPointerDecay:
+      case CK_BitCast:
+      case CK_LValueBitCast:
+      case CK_IntegralCast:
+      case CK_NullToPointer:
+      case CK_IntegralToPointer:
+      case CK_PointerToIntegral:
+      case CK_PointerToBoolean:
+      case CK_IntegralToBoolean:
+      case CK_IntegralToFloating:
+      case CK_FloatingToIntegral:
+      case CK_FloatingToBoolean:
+      case CK_FloatingCast:
+      case CK_FloatingRealToComplex:
+      case CK_FloatingComplexToReal:
+      case CK_FloatingComplexToBoolean:
+      case CK_FloatingComplexCast:
+      case CK_FloatingComplexToIntegralComplex:
+      case CK_IntegralRealToComplex:
+      case CK_IntegralComplexToReal:
+      case CK_IntegralComplexToBoolean:
+      case CK_IntegralComplexCast:
+      case CK_IntegralComplexToFloatingComplex:
+      case CK_CPointerToObjCPointerCast:
+      case CK_BlockPointerToObjCPointerCast:
+      case CK_AnyPointerToBlockPointerCast:  
+      case CK_ObjCObjectLValueCast: {
+        // Delegate to SValBuilder to process.
+        const ProgramState *state = Pred->getState();
+        SVal V = state->getSVal(Ex);
+        V = svalBuilder.evalCast(V, T, ExTy);
+        state = state->BindExpr(CastE, V);
+        MakeNode(Dst, CastE, Pred, state);
+        continue;
+      }
+      case CK_DerivedToBase:
+      case CK_UncheckedDerivedToBase: {
+        // For DerivedToBase cast, delegate to the store manager.
+        const ProgramState *state = Pred->getState();
+        SVal val = state->getSVal(Ex);
+        val = getStoreManager().evalDerivedToBase(val, T);
+        state = state->BindExpr(CastE, val);
+        MakeNode(Dst, CastE, Pred, state);
+        continue;
+      }
+        // Various C++ casts that are not handled yet.
+      case CK_Dynamic:
+      case CK_ToUnion:
+      case CK_BaseToDerived:
+      case CK_NullToMemberPointer:
+      case CK_BaseToDerivedMemberPointer:
+      case CK_DerivedToBaseMemberPointer:
+      case CK_UserDefinedConversion:
+      case CK_ConstructorConversion:
+      case CK_VectorSplat:
+      case CK_MemberPointerToBoolean: {
+        // Recover some path-sensitivty by conjuring a new value.
+        QualType resultType = CastE->getType();
+        if (CastE->isLValue())
+          resultType = getContext().getPointerType(resultType);
+        
+        SVal result =
+        svalBuilder.getConjuredSymbolVal(NULL, CastE, resultType,
+                                         Builder->getCurrentBlockCount());
+        
+        const ProgramState *state = Pred->getState()->BindExpr(CastE, result);
+        MakeNode(Dst, CastE, Pred, state);
+        continue;
+      }
+    }
+  }
+}
+
+void ExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr *CL,
+                                          ExplodedNode *Pred,
+                                          ExplodedNodeSet &Dst) {
+  const InitListExpr *ILE 
+    = cast<InitListExpr>(CL->getInitializer()->IgnoreParens());
+  
+  const ProgramState *state = Pred->getState();
+  SVal ILV = state->getSVal(ILE);
+  const LocationContext *LC = Pred->getLocationContext();
+  state = state->bindCompoundLiteral(CL, LC, ILV);
+  
+  if (CL->isLValue())
+    MakeNode(Dst, CL, Pred, state->BindExpr(CL, state->getLValue(CL, LC)));
+  else
+    MakeNode(Dst, CL, Pred, state->BindExpr(CL, ILV));
+}
+
+void ExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
+                               ExplodedNodeSet &Dst) {
+  
+  // FIXME: static variables may have an initializer, but the second
+  //  time a function is called those values may not be current.
+  //  This may need to be reflected in the CFG.
+  
+  // Assumption: The CFG has one DeclStmt per Decl.
+  const Decl *D = *DS->decl_begin();
+  
+  if (!D || !isa<VarDecl>(D))
+    return;
+  
+  // FIXME: all pre/post visits should eventually be handled by ::Visit().
+  ExplodedNodeSet dstPreVisit;
+  getCheckerManager().runCheckersForPreStmt(dstPreVisit, Pred, DS, *this);
+  
+  const VarDecl *VD = dyn_cast<VarDecl>(D);
+  
+  for (ExplodedNodeSet::iterator I = dstPreVisit.begin(), E = dstPreVisit.end();
+       I!=E; ++I) {
+    ExplodedNode *N = *I;
+    const ProgramState *state = N->getState();
+    
+    // Decls without InitExpr are not initialized explicitly.
+    const LocationContext *LC = N->getLocationContext();
+    
+    if (const Expr *InitEx = VD->getInit()) {
+      SVal InitVal = state->getSVal(InitEx);
+      
+      // We bound the temp obj region to the CXXConstructExpr. Now recover
+      // the lazy compound value when the variable is not a reference.
+      if (AMgr.getLangOptions().CPlusPlus && VD->getType()->isRecordType() && 
+          !VD->getType()->isReferenceType() && isa<loc::MemRegionVal>(InitVal)){
+        InitVal = state->getSVal(cast<loc::MemRegionVal>(InitVal).getRegion());
+        assert(isa<nonloc::LazyCompoundVal>(InitVal));
+      }
+      
+      // Recover some path-sensitivity if a scalar value evaluated to
+      // UnknownVal.
+      if ((InitVal.isUnknown() ||
+           !getConstraintManager().canReasonAbout(InitVal)) &&
+          !VD->getType()->isReferenceType()) {
+        InitVal = svalBuilder.getConjuredSymbolVal(NULL, InitEx,
+                                                   Builder->getCurrentBlockCount());
+      }
+      
+      evalBind(Dst, DS, N, state->getLValue(VD, LC), InitVal, true);
+    }
+    else {
+      MakeNode(Dst, DS, N, state->bindDeclWithNoInit(state->getRegion(VD, LC)));
+    }
+  }
+}
+
+void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
+                                  ExplodedNodeSet &Dst) {
+  
+  assert(B->getOpcode() == BO_LAnd ||
+         B->getOpcode() == BO_LOr);
+  
+  const ProgramState *state = Pred->getState();
+  SVal X = state->getSVal(B);
+  assert(X.isUndef());
+  
+  const Expr *Ex = (const Expr*) cast<UndefinedVal>(X).getData();
+  assert(Ex);
+  
+  if (Ex == B->getRHS()) {
+    X = state->getSVal(Ex);
+    
+    // Handle undefined values.
+    if (X.isUndef()) {
+      MakeNode(Dst, B, Pred, state->BindExpr(B, X));
+      return;
+    }
+    
+    DefinedOrUnknownSVal XD = cast<DefinedOrUnknownSVal>(X);
+    
+    // We took the RHS.  Because the value of the '&&' or '||' expression must
+    // evaluate to 0 or 1, we must assume the value of the RHS evaluates to 0
+    // or 1.  Alternatively, we could take a lazy approach, and calculate this
+    // value later when necessary.  We don't have the machinery in place for
+    // this right now, and since most logical expressions are used for branches,
+    // the payoff is not likely to be large.  Instead, we do eager evaluation.
+    if (const ProgramState *newState = state->assume(XD, true))
+      MakeNode(Dst, B, Pred,
+               newState->BindExpr(B, svalBuilder.makeIntVal(1U, B->getType())));
+    
+    if (const ProgramState *newState = state->assume(XD, false))
+      MakeNode(Dst, B, Pred,
+               newState->BindExpr(B, svalBuilder.makeIntVal(0U, B->getType())));
+  }
+  else {
+    // We took the LHS expression.  Depending on whether we are '&&' or
+    // '||' we know what the value of the expression is via properties of
+    // the short-circuiting.
+    X = svalBuilder.makeIntVal(B->getOpcode() == BO_LAnd ? 0U : 1U,
+                               B->getType());
+    MakeNode(Dst, B, Pred, state->BindExpr(B, X));
+  }
+}
+
+void ExprEngine::VisitInitListExpr(const InitListExpr *IE,
+                                   ExplodedNode *Pred,
+                                   ExplodedNodeSet &Dst) {
+
+  const ProgramState *state = Pred->getState();
+  QualType T = getContext().getCanonicalType(IE->getType());
+  unsigned NumInitElements = IE->getNumInits();
+  
+  if (T->isArrayType() || T->isRecordType() || T->isVectorType()) {
+    llvm::ImmutableList<SVal> vals = getBasicVals().getEmptySValList();
+    
+    // Handle base case where the initializer has no elements.
+    // e.g: static int* myArray[] = {};
+    if (NumInitElements == 0) {
+      SVal V = svalBuilder.makeCompoundVal(T, vals);
+      MakeNode(Dst, IE, Pred, state->BindExpr(IE, V));
+      return;
+    }
+    
+    for (InitListExpr::const_reverse_iterator it = IE->rbegin(),
+         ei = IE->rend(); it != ei; ++it) {
+      vals = getBasicVals().consVals(state->getSVal(cast<Expr>(*it)), vals);
+    }
+    
+    MakeNode(Dst, IE, Pred,
+             state->BindExpr(IE, svalBuilder.makeCompoundVal(T, vals)));
+    return;
+  }
+  
+  if (Loc::isLocType(T) || T->isIntegerType()) {
+    assert(IE->getNumInits() == 1);
+    const Expr *initEx = IE->getInit(0);
+    MakeNode(Dst, IE, Pred, state->BindExpr(IE, state->getSVal(initEx)));
+    return;
+  }
+  
+  llvm_unreachable("unprocessed InitListExpr type");
+}
+
+void ExprEngine::VisitGuardedExpr(const Expr *Ex,
+                                  const Expr *L, 
+                                  const Expr *R,
+                                  ExplodedNode *Pred,
+                                  ExplodedNodeSet &Dst) {
+  
+  const ProgramState *state = Pred->getState();
+  SVal X = state->getSVal(Ex);  
+  assert (X.isUndef());  
+  const Expr *SE = (Expr*) cast<UndefinedVal>(X).getData();
+  assert(SE);
+  X = state->getSVal(SE);
+  
+  // Make sure that we invalidate the previous binding.
+  MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, X, true));
+}
+
+void ExprEngine::
+VisitOffsetOfExpr(const OffsetOfExpr *OOE, 
+                  ExplodedNode *Pred, ExplodedNodeSet &Dst) {
+  Expr::EvalResult Res;
+  if (OOE->Evaluate(Res, getContext()) && Res.Val.isInt()) {
+    const APSInt &IV = Res.Val.getInt();
+    assert(IV.getBitWidth() == getContext().getTypeSize(OOE->getType()));
+    assert(OOE->getType()->isIntegerType());
+    assert(IV.isSigned() == OOE->getType()->isSignedIntegerOrEnumerationType());
+    SVal X = svalBuilder.makeIntVal(IV);
+    MakeNode(Dst, OOE, Pred, Pred->getState()->BindExpr(OOE, X));
+    return;
+  }
+  // FIXME: Handle the case where __builtin_offsetof is not a constant.
+  Dst.Add(Pred);
+}
+
+
+void ExprEngine::
+VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *Ex,
+                              ExplodedNode *Pred,
+                              ExplodedNodeSet &Dst) {
+
+  QualType T = Ex->getTypeOfArgument();
+  
+  if (Ex->getKind() == UETT_SizeOf) {
+    if (!T->isIncompleteType() && !T->isConstantSizeType()) {
+      assert(T->isVariableArrayType() && "Unknown non-constant-sized type.");
+      
+      // FIXME: Add support for VLA type arguments and VLA expressions.
+      // When that happens, we should probably refactor VLASizeChecker's code.
+      Dst.Add(Pred);
+      return;
+    }
+    else if (T->getAs<ObjCObjectType>()) {
+      // Some code tries to take the sizeof an ObjCObjectType, relying that
+      // the compiler has laid out its representation.  Just report Unknown
+      // for these.
+      Dst.Add(Pred);
+      return;
+    }
+  }
+  
+  Expr::EvalResult Result;
+  Ex->Evaluate(Result, getContext());
+  CharUnits amt = CharUnits::fromQuantity(Result.Val.getInt().getZExtValue());
+  
+  const ProgramState *state = Pred->getState();
+  state = state->BindExpr(Ex, svalBuilder.makeIntVal(amt.getQuantity(),
+                                                     Ex->getType()));
+  MakeNode(Dst, Ex, Pred, state);
+}
+
+void ExprEngine::VisitUnaryOperator(const UnaryOperator* U, 
+                                    ExplodedNode *Pred,
+                                    ExplodedNodeSet &Dst) {  
+  switch (U->getOpcode()) {
+    default:
+      break;
+    case UO_Real: {
+      const Expr *Ex = U->getSubExpr()->IgnoreParens();
+      ExplodedNodeSet Tmp;
+      Visit(Ex, Pred, Tmp);
+      
+      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
+        
+        // FIXME: We don't have complex SValues yet.
+        if (Ex->getType()->isAnyComplexType()) {
+          // Just report "Unknown."
+          Dst.Add(*I);
+          continue;
+        }
+        
+        // For all other types, UO_Real is an identity operation.
+        assert (U->getType() == Ex->getType());
+        const ProgramState *state = (*I)->getState();
+        MakeNode(Dst, U, *I, state->BindExpr(U, state->getSVal(Ex)));
+      }
+      
+      return;
+    }
+      
+    case UO_Imag: {
+      
+      const Expr *Ex = U->getSubExpr()->IgnoreParens();
+      ExplodedNodeSet Tmp;
+      Visit(Ex, Pred, Tmp);
+      
+      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
+        // FIXME: We don't have complex SValues yet.
+        if (Ex->getType()->isAnyComplexType()) {
+          // Just report "Unknown."
+          Dst.Add(*I);
+          continue;
+        }
+        
+        // For all other types, UO_Imag returns 0.
+        const ProgramState *state = (*I)->getState();
+        SVal X = svalBuilder.makeZeroVal(Ex->getType());
+        MakeNode(Dst, U, *I, state->BindExpr(U, X));
+      }
+      
+      return;
+    }
+      
+    case UO_Plus:
+      assert(!U->isLValue());
+      // FALL-THROUGH.
+    case UO_Deref:
+    case UO_AddrOf:
+    case UO_Extension: {
+      
+      // Unary "+" is a no-op, similar to a parentheses.  We still have places
+      // where it may be a block-level expression, so we need to
+      // generate an extra node that just propagates the value of the
+      // subexpression.
+      
+      const Expr *Ex = U->getSubExpr()->IgnoreParens();
+      ExplodedNodeSet Tmp;
+      Visit(Ex, Pred, Tmp);
+      
+      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
+        const ProgramState *state = (*I)->getState();
+        MakeNode(Dst, U, *I, state->BindExpr(U, state->getSVal(Ex)));
+      }
+      
+      return;
+    }
+      
+    case UO_LNot:
+    case UO_Minus:
+    case UO_Not: {
+      assert (!U->isLValue());
+      const Expr *Ex = U->getSubExpr()->IgnoreParens();
+      ExplodedNodeSet Tmp;
+      Visit(Ex, Pred, Tmp);
+      
+      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
+        const ProgramState *state = (*I)->getState();
+        
+        // Get the value of the subexpression.
+        SVal V = state->getSVal(Ex);
+        
+        if (V.isUnknownOrUndef()) {
+          MakeNode(Dst, U, *I, state->BindExpr(U, V));
+          continue;
+        }
+        
+        switch (U->getOpcode()) {
+          default:
+            llvm_unreachable("Invalid Opcode.");
+            
+          case UO_Not:
+            // FIXME: Do we need to handle promotions?
+            state = state->BindExpr(U, evalComplement(cast<NonLoc>(V)));
+            break;
+            
+          case UO_Minus:
+            // FIXME: Do we need to handle promotions?
+            state = state->BindExpr(U, evalMinus(cast<NonLoc>(V)));
+            break;
+            
+          case UO_LNot:
+            
+            // C99 6.5.3.3: "The expression !E is equivalent to (0==E)."
+            //
+            //  Note: technically we do "E == 0", but this is the same in the
+            //    transfer functions as "0 == E".
+            SVal Result;
+            
+            if (isa<Loc>(V)) {
+              Loc X = svalBuilder.makeNull();
+              Result = evalBinOp(state, BO_EQ, cast<Loc>(V), X,
+                                 U->getType());
+            }
+            else {
+              nonloc::ConcreteInt X(getBasicVals().getValue(0, Ex->getType()));
+              Result = evalBinOp(state, BO_EQ, cast<NonLoc>(V), X,
+                                 U->getType());
+            }
+            
+            state = state->BindExpr(U, Result);
+            
+            break;
+        }
+        
+        MakeNode(Dst, U, *I, state);
+      }
+      
+      return;
+    }
+  }
+  
+  // Handle ++ and -- (both pre- and post-increment).
+  assert (U->isIncrementDecrementOp());
+  ExplodedNodeSet Tmp;
+  const Expr *Ex = U->getSubExpr()->IgnoreParens();
+  Visit(Ex, Pred, Tmp);
+  
+  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I!=E; ++I) {
+    
+    const ProgramState *state = (*I)->getState();
+    SVal loc = state->getSVal(Ex);
+    
+    // Perform a load.
+    ExplodedNodeSet Tmp2;
+    evalLoad(Tmp2, Ex, *I, state, loc);
+    
+    for (ExplodedNodeSet::iterator I2=Tmp2.begin(), E2=Tmp2.end();I2!=E2;++I2) {
+      
+      state = (*I2)->getState();
+      SVal V2_untested = state->getSVal(Ex);
+      
+      // Propagate unknown and undefined values.
+      if (V2_untested.isUnknownOrUndef()) {
+        MakeNode(Dst, U, *I2, state->BindExpr(U, V2_untested));
+        continue;
+      }
+      DefinedSVal V2 = cast<DefinedSVal>(V2_untested);
+      
+      // Handle all other values.
+      BinaryOperator::Opcode Op = U->isIncrementOp() ? BO_Add
+      : BO_Sub;
+      
+      // If the UnaryOperator has non-location type, use its type to create the
+      // constant value. If the UnaryOperator has location type, create the
+      // constant with int type and pointer width.
+      SVal RHS;
+      
+      if (U->getType()->isAnyPointerType())
+        RHS = svalBuilder.makeArrayIndex(1);
+      else
+        RHS = svalBuilder.makeIntVal(1, U->getType());
+      
+      SVal Result = evalBinOp(state, Op, V2, RHS, U->getType());
+      
+      // Conjure a new symbol if necessary to recover precision.
+      if (Result.isUnknown() || !getConstraintManager().canReasonAbout(Result)){
+        DefinedOrUnknownSVal SymVal =
+        svalBuilder.getConjuredSymbolVal(NULL, Ex,
+                                         Builder->getCurrentBlockCount());
+        Result = SymVal;
+        
+        // If the value is a location, ++/-- should always preserve
+        // non-nullness.  Check if the original value was non-null, and if so
+        // propagate that constraint.
+        if (Loc::isLocType(U->getType())) {
+          DefinedOrUnknownSVal Constraint =
+          svalBuilder.evalEQ(state, V2,svalBuilder.makeZeroVal(U->getType()));
+          
+          if (!state->assume(Constraint, true)) {
+            // It isn't feasible for the original value to be null.
+            // Propagate this constraint.
+            Constraint = svalBuilder.evalEQ(state, SymVal,
+                                            svalBuilder.makeZeroVal(U->getType()));
+            
+            
+            state = state->assume(Constraint, false);
+            assert(state);
+          }
+        }
+      }
+      
+      // Since the lvalue-to-rvalue conversion is explicit in the AST,
+      // we bind an l-value if the operator is prefix and an lvalue (in C++).
+      if (U->isLValue())
+        state = state->BindExpr(U, loc);
+      else
+        state = state->BindExpr(U, U->isPostfix() ? V2 : Result);
+      
+      // Perform the store.
+      evalStore(Dst, NULL, U, *I2, state, loc, Result);
+    }
+  }
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CXXExprEngine.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
similarity index 78%
rename from contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CXXExprEngine.cpp
rename to contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index ef7bc2016cdf..acb007490ee1 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/CXXExprEngine.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -1,4 +1,4 @@
-//===- GRCXXExprEngine.cpp - C++ expr evaluation engine ---------*- C++ -*-===//
+//===- ExprEngineCXX.cpp - ExprEngine support for C++ -----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -14,6 +14,7 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
 #include "clang/AST/DeclCXX.h"
 
 using namespace clang;
@@ -36,7 +37,7 @@ void ExprEngine::evalArguments(ConstExprIterator AI, ConstExprIterator AE,
                                  bool FstArgAsLValue) {
 
 
-  llvm::SmallVector<CallExprWLItem, 20> WorkList;
+  SmallVector<CallExprWLItem, 20> WorkList;
   WorkList.reserve(AE - AI);
   WorkList.push_back(CallExprWLItem(AI, Pred));
 
@@ -103,24 +104,22 @@ const CXXThisRegion *ExprEngine::getCXXThisRegion(const CXXMethodDecl *decl,
                     getCXXThisRegion(decl->getThisType(getContext()), frameCtx);
 }
 
-void ExprEngine::CreateCXXTemporaryObject(const Expr *Ex, ExplodedNode *Pred,
-                                            ExplodedNodeSet &Dst) {
-  ExplodedNodeSet Tmp;
-  Visit(Ex, Pred, Tmp);
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I != E; ++I) {
-    const GRState *state = GetState(*I);
+void ExprEngine::CreateCXXTemporaryObject(const MaterializeTemporaryExpr *ME,
+                                          ExplodedNode *Pred,
+                                          ExplodedNodeSet &Dst) {
+  const Expr *tempExpr = ME->GetTemporaryExpr()->IgnoreParens();
+  const ProgramState *state = Pred->getState();
 
-    // Bind the temporary object to the value of the expression. Then bind
-    // the expression to the location of the object.
-    SVal V = state->getSVal(Ex);
+  // Bind the temporary object to the value of the expression. Then bind
+  // the expression to the location of the object.
+  SVal V = state->getSVal(tempExpr);
 
-    const MemRegion *R =
-      svalBuilder.getRegionManager().getCXXTempObjectRegion(Ex,
-                                                   Pred->getLocationContext());
+  const MemRegion *R =
+    svalBuilder.getRegionManager().getCXXTempObjectRegion(ME,
+                                                 Pred->getLocationContext());
 
-    state = state->bindLoc(loc::MemRegionVal(R), V);
-    MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, loc::MemRegionVal(R)));
-  }
+  state = state->bindLoc(loc::MemRegionVal(R), V);
+  MakeNode(Dst, ME, Pred, state->BindExpr(ME, loc::MemRegionVal(R)));
 }
 
 void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E, 
@@ -186,7 +185,7 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E,
 
     for (ExplodedNodeSet::iterator NI = argsEvaluated.begin(),
                                   NE = argsEvaluated.end(); NI != NE; ++NI) {
-      const GRState *state = GetState(*NI);
+      const ProgramState *state = (*NI)->getState();
       // Setup 'this' region, so that the ctor is evaluated on the object pointed
       // by 'Dest'.
       state = state->bindLoc(loc::MemRegionVal(ThisR), loc::MemRegionVal(Dest));
@@ -197,17 +196,6 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E,
 #endif
   
   // Default semantics: invalidate all regions passed as arguments.
-  llvm::SmallVector<const MemRegion*, 10> regionsToInvalidate;
-
-  // FIXME: We can have collisions on the conjured symbol if the
-  //  expression *I also creates conjured symbols.  We probably want
-  //  to identify conjured symbols by an expression pair: the enclosing
-  //  expression (the context) and the expression itself.  This should
-  //  disambiguate conjured symbols.
-  unsigned blockCount = Builder->getCurrentBlockCount();
-  
-  // NOTE: Even if RegionsToInvalidate is empty, we must still invalidate
-  //  global variables.
   ExplodedNodeSet destCall;
 
   for (ExplodedNodeSet::iterator
@@ -215,25 +203,10 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E,
        i != e; ++i)
   {
     ExplodedNode *Pred = *i;
-    const GRState *state = GetState(Pred);
+    const LocationContext *LC = Pred->getLocationContext();
+    const ProgramState *state = Pred->getState();
 
-    // Accumulate list of regions that are invalidated.
-    for (CXXConstructExpr::const_arg_iterator
-          ai = E->arg_begin(), ae = E->arg_end();
-          ai != ae; ++ai)
-    {
-      SVal val = state->getSVal(*ai);
-      if (const MemRegion *region = val.getAsRegion())
-        regionsToInvalidate.push_back(region);
-    }
-    
-    // Invalidate the regions.    
-    state = state->invalidateRegions(regionsToInvalidate.data(),
-                                     regionsToInvalidate.data() +
-                                     regionsToInvalidate.size(),
-                                     E, blockCount, 0,
-                                     /* invalidateGlobals = */ true);
-    
+    state = invalidateArguments(state, CallOrObjCMessage(E, state), LC);
     Builder->MakeNode(destCall, E, Pred, state);
   }
   
@@ -258,7 +231,7 @@ void ExprEngine::VisitCXXDestructor(const CXXDestructorDecl *DD,
 
   CallEnter PP(S, SFC, Pred->getLocationContext());
 
-  const GRState *state = Pred->getState();
+  const ProgramState *state = Pred->getState();
   state = state->bindLoc(loc::MemRegionVal(ThisR), loc::MemRegionVal(Dest));
   ExplodedNode *N = Builder->generateNode(PP, state, Pred);
   if (N)
@@ -279,7 +252,7 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
   if (CNE->isArray()) {
     // FIXME: allocating an array requires simulating the constructors.
     // For now, just return a symbolicated region.
-    const GRState *state = GetState(Pred);
+    const ProgramState *state = Pred->getState();
     state = state->BindExpr(CNE, loc::MemRegionVal(EleReg));
     MakeNode(Dst, CNE, Pred, state);
     return;
@@ -298,12 +271,12 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
   for (ExplodedNodeSet::iterator I = argsEvaluated.begin(), 
                                  E = argsEvaluated.end(); I != E; ++I) {
 
-    const GRState *state = GetState(*I);
+    const ProgramState *state = (*I)->getState();
     
     // Accumulate list of regions that are invalidated.
     // FIXME: Eventually we should unify the logic for constructor
     // processing in one place.
-    llvm::SmallVector<const MemRegion*, 10> regionsToInvalidate;
+    SmallVector<const MemRegion*, 10> regionsToInvalidate;
     for (CXXNewExpr::const_arg_iterator
           ai = CNE->constructor_arg_begin(), ae = CNE->constructor_arg_end();
           ai != ae; ++ai)
@@ -316,17 +289,13 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
     if (ObjTy->isRecordType()) {
       regionsToInvalidate.push_back(EleReg);
       // Invalidate the regions.
-      state = state->invalidateRegions(regionsToInvalidate.data(),
-                                       regionsToInvalidate.data() +
-                                       regionsToInvalidate.size(),
+      state = state->invalidateRegions(regionsToInvalidate,
                                        CNE, blockCount, 0,
                                        /* invalidateGlobals = */ true);
       
     } else {
       // Invalidate the regions.
-      state = state->invalidateRegions(regionsToInvalidate.data(),
-                                       regionsToInvalidate.data() +
-                                       regionsToInvalidate.size(),
+      state = state->invalidateRegions(regionsToInvalidate,
                                        CNE, blockCount, 0,
                                        /* invalidateGlobals = */ true);
 
@@ -351,7 +320,7 @@ void ExprEngine::VisitCXXDeleteExpr(const CXXDeleteExpr *CDE,
   Visit(CDE->getArgument(), Pred, Argevaluated);
   for (ExplodedNodeSet::iterator I = Argevaluated.begin(), 
                                  E = Argevaluated.end(); I != E; ++I) {
-    const GRState *state = GetState(*I);
+    const ProgramState *state = (*I)->getState();
     MakeNode(Dst, CDE, *I, state);
   }
 }
@@ -364,7 +333,7 @@ void ExprEngine::VisitCXXThisExpr(const CXXThisExpr *TE, ExplodedNode *Pred,
                                   getContext().getCanonicalType(TE->getType()),
                                                Pred->getLocationContext());
 
-  const GRState *state = GetState(Pred);
+  const ProgramState *state = Pred->getState();
   SVal V = state->getSVal(loc::MemRegionVal(R));
   MakeNode(Dst, TE, Pred, state->BindExpr(TE, V));
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
new file mode 100644
index 000000000000..6d377b959136
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
@@ -0,0 +1,253 @@
+//=-- ExprEngineCallAndReturn.cpp - Support for call/return -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines ExprEngine's support for calls and returns.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/Analysis/Support/SaveAndRestore.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+  // Trait class for recording returned expression in the state.
+  struct ReturnExpr {
+    static int TagInt;
+    typedef const Stmt *data_type;
+  };
+  int ReturnExpr::TagInt; 
+}
+
+void ExprEngine::processCallEnter(CallEnterNodeBuilder &B) {
+  const ProgramState *state =
+    B.getState()->enterStackFrame(B.getCalleeContext());
+  B.generateNode(state);
+}
+
+void ExprEngine::processCallExit(CallExitNodeBuilder &B) {
+  const ProgramState *state = B.getState();
+  const ExplodedNode *Pred = B.getPredecessor();
+  const StackFrameContext *calleeCtx = 
+    cast<StackFrameContext>(Pred->getLocationContext());
+  const Stmt *CE = calleeCtx->getCallSite();
+  
+  // If the callee returns an expression, bind its value to CallExpr.
+  const Stmt *ReturnedExpr = state->get<ReturnExpr>();
+  if (ReturnedExpr) {
+    SVal RetVal = state->getSVal(ReturnedExpr);
+    state = state->BindExpr(CE, RetVal);
+    // Clear the return expr GDM.
+    state = state->remove<ReturnExpr>();
+  }
+  
+  // Bind the constructed object value to CXXConstructExpr.
+  if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(CE)) {
+    const CXXThisRegion *ThisR =
+    getCXXThisRegion(CCE->getConstructor()->getParent(), calleeCtx);
+    
+    SVal ThisV = state->getSVal(ThisR);
+    // Always bind the region to the CXXConstructExpr.
+    state = state->BindExpr(CCE, ThisV);
+  }
+  
+  B.generateNode(state);
+}
+
+const ProgramState *
+ExprEngine::invalidateArguments(const ProgramState *State,
+                                const CallOrObjCMessage &Call,
+                                const LocationContext *LC) {
+  SmallVector<const MemRegion *, 8> RegionsToInvalidate;
+
+  if (Call.isObjCMessage()) {
+    // Invalidate all instance variables of the receiver of an ObjC message.
+    // FIXME: We should be able to do better with inter-procedural analysis.
+    if (const MemRegion *MR = Call.getInstanceMessageReceiver(LC).getAsRegion())
+      RegionsToInvalidate.push_back(MR);
+
+  } else if (Call.isCXXCall()) {
+    // Invalidate all instance variables for the callee of a C++ method call.
+    // FIXME: We should be able to do better with inter-procedural analysis.
+    // FIXME: We can probably do better for const versus non-const methods.
+    if (const MemRegion *Callee = Call.getCXXCallee().getAsRegion())
+      RegionsToInvalidate.push_back(Callee);
+
+  } else if (Call.isFunctionCall()) {
+    // Block calls invalidate all captured-by-reference values.
+    if (const MemRegion *Callee = Call.getFunctionCallee().getAsRegion()) {
+      if (isa<BlockDataRegion>(Callee))
+        RegionsToInvalidate.push_back(Callee);
+    }
+  }
+
+  for (unsigned idx = 0, e = Call.getNumArgs(); idx != e; ++idx) {
+    SVal V = Call.getArgSVal(idx);
+
+    // If we are passing a location wrapped as an integer, unwrap it and
+    // invalidate the values referred by the location.
+    if (nonloc::LocAsInteger *Wrapped = dyn_cast<nonloc::LocAsInteger>(&V))
+      V = Wrapped->getLoc();
+    else if (!isa<Loc>(V))
+      continue;
+
+    if (const MemRegion *R = V.getAsRegion()) {
+      // Invalidate the value of the variable passed by reference.
+
+      // Are we dealing with an ElementRegion?  If the element type is
+      // a basic integer type (e.g., char, int) and the underying region
+      // is a variable region then strip off the ElementRegion.
+      // FIXME: We really need to think about this for the general case
+      //   as sometimes we are reasoning about arrays and other times
+      //   about (char*), etc., is just a form of passing raw bytes.
+      //   e.g., void *p = alloca(); foo((char*)p);
+      if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
+        // Checking for 'integral type' is probably too promiscuous, but
+        // we'll leave it in for now until we have a systematic way of
+        // handling all of these cases.  Eventually we need to come up
+        // with an interface to StoreManager so that this logic can be
+        // approriately delegated to the respective StoreManagers while
+        // still allowing us to do checker-specific logic (e.g.,
+        // invalidating reference counts), probably via callbacks.
+        if (ER->getElementType()->isIntegralOrEnumerationType()) {
+          const MemRegion *superReg = ER->getSuperRegion();
+          if (isa<VarRegion>(superReg) || isa<FieldRegion>(superReg) ||
+              isa<ObjCIvarRegion>(superReg))
+            R = cast<TypedRegion>(superReg);
+        }
+        // FIXME: What about layers of ElementRegions?
+      }
+
+      // Mark this region for invalidation.  We batch invalidate regions
+      // below for efficiency.
+      RegionsToInvalidate.push_back(R);
+    } else {
+      // Nuke all other arguments passed by reference.
+      // FIXME: is this necessary or correct? This handles the non-Region
+      //  cases.  Is it ever valid to store to these?
+      State = State->unbindLoc(cast<Loc>(V));
+    }
+  }
+
+  // Invalidate designated regions using the batch invalidation API.
+
+  // FIXME: We can have collisions on the conjured symbol if the
+  //  expression *I also creates conjured symbols.  We probably want
+  //  to identify conjured symbols by an expression pair: the enclosing
+  //  expression (the context) and the expression itself.  This should
+  //  disambiguate conjured symbols.
+  assert(Builder && "Invalidating arguments outside of a statement context");
+  unsigned Count = Builder->getCurrentBlockCount();
+  StoreManager::InvalidatedSymbols IS;
+
+  // NOTE: Even if RegionsToInvalidate is empty, we may still invalidate
+  //  global variables.
+  return State->invalidateRegions(RegionsToInvalidate,
+                                  Call.getOriginExpr(), Count,
+                                  &IS, doesInvalidateGlobals(Call));
+
+}
+
+void ExprEngine::VisitCallExpr(const CallExpr *CE, ExplodedNode *Pred,
+                               ExplodedNodeSet &dst) {
+  // Perform the previsit of the CallExpr.
+  ExplodedNodeSet dstPreVisit;
+  getCheckerManager().runCheckersForPreStmt(dstPreVisit, Pred, CE, *this);
+  
+  // Now evaluate the call itself.
+  class DefaultEval : public GraphExpander {
+    ExprEngine &Eng;
+    const CallExpr *CE;
+  public:
+    
+    DefaultEval(ExprEngine &eng, const CallExpr *ce)
+    : Eng(eng), CE(ce) {}
+    virtual void expandGraph(ExplodedNodeSet &Dst, ExplodedNode *Pred) {
+      // Should we inline the call?
+      if (Eng.getAnalysisManager().shouldInlineCall() &&
+          Eng.InlineCall(Dst, CE, Pred)) {
+        return;
+      }
+
+      // First handle the return value.
+      StmtNodeBuilder &Builder = Eng.getBuilder();
+      assert(&Builder && "StmtNodeBuilder must be defined.");
+
+      // Get the callee.
+      const Expr *Callee = CE->getCallee()->IgnoreParens();
+      const ProgramState *state = Pred->getState();
+      SVal L = state->getSVal(Callee);
+
+      // Figure out the result type. We do this dance to handle references.
+      QualType ResultTy;
+      if (const FunctionDecl *FD = L.getAsFunctionDecl())
+        ResultTy = FD->getResultType();
+      else
+        ResultTy = CE->getType();
+
+      if (CE->isLValue())
+        ResultTy = Eng.getContext().getPointerType(ResultTy);
+
+      // Conjure a symbol value to use as the result.
+      SValBuilder &SVB = Eng.getSValBuilder();
+      unsigned Count = Builder.getCurrentBlockCount();
+      SVal RetVal = SVB.getConjuredSymbolVal(0, CE, ResultTy, Count);
+
+      // Generate a new state with the return value set.
+      state = state->BindExpr(CE, RetVal);
+
+      // Invalidate the arguments.
+      const LocationContext *LC = Pred->getLocationContext();
+      state = Eng.invalidateArguments(state, CallOrObjCMessage(CE, state), LC);
+
+      // And make the result node.
+      Eng.MakeNode(Dst, CE, Pred, state);
+    }
+  };
+  
+  // Finally, evaluate the function call.  We try each of the checkers
+  // to see if the can evaluate the function call.
+  ExplodedNodeSet dstCallEvaluated;
+  DefaultEval defEval(*this, CE);
+  getCheckerManager().runCheckersForEvalCall(dstCallEvaluated,
+                                             dstPreVisit,
+                                             CE, *this, &defEval);
+  
+  // Finally, perform the post-condition check of the CallExpr and store
+  // the created nodes in 'Dst'.
+  getCheckerManager().runCheckersForPostStmt(dst, dstCallEvaluated, CE,
+                                             *this);
+}
+
+void ExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
+                                 ExplodedNodeSet &Dst) {
+  ExplodedNodeSet Src;
+  if (const Expr *RetE = RS->getRetValue()) {
+    // Record the returned expression in the state. It will be used in
+    // processCallExit to bind the return value to the call expr.
+    {
+      static SimpleProgramPointTag tag("ExprEngine: ReturnStmt");
+      const ProgramState *state = Pred->getState();
+      state = state->set<ReturnExpr>(RetE);
+      Pred = Builder->generateNode(RetE, state, Pred, &tag);
+    }
+    // We may get a NULL Pred because we generated a cached node.
+    if (Pred)
+      Visit(RetE, Pred, Src);
+  }
+  else {
+    Src.Add(Pred);
+  }
+  
+  getCheckerManager().runCheckersForPreStmt(Dst, Src, RS, *this);
+}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
new file mode 100644
index 000000000000..e0560fdfe460
--- /dev/null
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
@@ -0,0 +1,279 @@
+//=-- ExprEngineObjC.cpp - ExprEngine support for Objective-C ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines ExprEngine's support for Objective-C expressions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
+#include "clang/Analysis/Support/SaveAndRestore.h"
+
+using namespace clang;
+using namespace ento;
+
+void ExprEngine::VisitLvalObjCIvarRefExpr(const ObjCIvarRefExpr *Ex, 
+                                          ExplodedNode *Pred,
+                                          ExplodedNodeSet &Dst) {
+  
+  const ProgramState *state = Pred->getState();
+  SVal baseVal = state->getSVal(Ex->getBase());
+  SVal location = state->getLValue(Ex->getDecl(), baseVal);
+  
+  ExplodedNodeSet dstIvar;
+  MakeNode(dstIvar, Ex, Pred, state->BindExpr(Ex, location));
+  
+  // Perform the post-condition check of the ObjCIvarRefExpr and store
+  // the created nodes in 'Dst'.
+  getCheckerManager().runCheckersForPostStmt(Dst, dstIvar, Ex, *this);
+}
+
+void ExprEngine::VisitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt *S,
+                                             ExplodedNode *Pred,
+                                             ExplodedNodeSet &Dst) {
+  getCheckerManager().runCheckersForPreStmt(Dst, Pred, S, *this);
+}
+
+void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S,
+                                            ExplodedNode *Pred,
+                                            ExplodedNodeSet &Dst) {
+  
+  // ObjCForCollectionStmts are processed in two places.  This method
+  // handles the case where an ObjCForCollectionStmt* occurs as one of the
+  // statements within a basic block.  This transfer function does two things:
+  //
+  //  (1) binds the next container value to 'element'.  This creates a new
+  //      node in the ExplodedGraph.
+  //
+  //  (2) binds the value 0/1 to the ObjCForCollectionStmt* itself, indicating
+  //      whether or not the container has any more elements.  This value
+  //      will be tested in ProcessBranch.  We need to explicitly bind
+  //      this value because a container can contain nil elements.
+  //
+  // FIXME: Eventually this logic should actually do dispatches to
+  //   'countByEnumeratingWithState:objects:count:' (NSFastEnumeration).
+  //   This will require simulating a temporary NSFastEnumerationState, either
+  //   through an SVal or through the use of MemRegions.  This value can
+  //   be affixed to the ObjCForCollectionStmt* instead of 0/1; when the loop
+  //   terminates we reclaim the temporary (it goes out of scope) and we
+  //   we can test if the SVal is 0 or if the MemRegion is null (depending
+  //   on what approach we take).
+  //
+  //  For now: simulate (1) by assigning either a symbol or nil if the
+  //    container is empty.  Thus this transfer function will by default
+  //    result in state splitting.
+  
+  const Stmt *elem = S->getElement();
+  const ProgramState *state = Pred->getState();
+  SVal elementV;
+  
+  if (const DeclStmt *DS = dyn_cast<DeclStmt>(elem)) {
+    const VarDecl *elemD = cast<VarDecl>(DS->getSingleDecl());
+    assert(elemD->getInit() == 0);
+    elementV = state->getLValue(elemD, Pred->getLocationContext());
+  }
+  else {
+    elementV = state->getSVal(elem);
+  }
+  
+  ExplodedNodeSet dstLocation;
+  evalLocation(dstLocation, elem, Pred, state, elementV, NULL, false);
+  
+  if (dstLocation.empty())
+    return;
+  
+  for (ExplodedNodeSet::iterator NI = dstLocation.begin(),
+       NE = dstLocation.end(); NI!=NE; ++NI) {
+    Pred = *NI;
+    const ProgramState *state = Pred->getState();
+    
+    // Handle the case where the container still has elements.
+    SVal TrueV = svalBuilder.makeTruthVal(1);
+    const ProgramState *hasElems = state->BindExpr(S, TrueV);
+    
+    // Handle the case where the container has no elements.
+    SVal FalseV = svalBuilder.makeTruthVal(0);
+    const ProgramState *noElems = state->BindExpr(S, FalseV);
+    
+    if (loc::MemRegionVal *MV = dyn_cast<loc::MemRegionVal>(&elementV))
+      if (const TypedValueRegion *R = 
+          dyn_cast<TypedValueRegion>(MV->getRegion())) {
+        // FIXME: The proper thing to do is to really iterate over the
+        //  container.  We will do this with dispatch logic to the store.
+        //  For now, just 'conjure' up a symbolic value.
+        QualType T = R->getValueType();
+        assert(Loc::isLocType(T));
+        unsigned Count = Builder->getCurrentBlockCount();
+        SymbolRef Sym = SymMgr.getConjuredSymbol(elem, T, Count);
+        SVal V = svalBuilder.makeLoc(Sym);
+        hasElems = hasElems->bindLoc(elementV, V);
+        
+        // Bind the location to 'nil' on the false branch.
+        SVal nilV = svalBuilder.makeIntVal(0, T);
+        noElems = noElems->bindLoc(elementV, nilV);
+      }
+    
+    // Create the new nodes.
+    MakeNode(Dst, S, Pred, hasElems);
+    MakeNode(Dst, S, Pred, noElems);
+  }
+}
+
+void ExprEngine::VisitObjCMessage(const ObjCMessage &msg,
+                                  ExplodedNode *Pred,
+                                  ExplodedNodeSet &Dst) {
+  
+  // Handle the previsits checks.
+  ExplodedNodeSet dstPrevisit;
+  getCheckerManager().runCheckersForPreObjCMessage(dstPrevisit, Pred, 
+                                                   msg, *this);
+  
+  // Proceed with evaluate the message expression.
+  ExplodedNodeSet dstEval;
+  
+  for (ExplodedNodeSet::iterator DI = dstPrevisit.begin(),
+       DE = dstPrevisit.end(); DI != DE; ++DI) {
+    
+    ExplodedNode *Pred = *DI;
+    bool RaisesException = false;
+    SaveAndRestore<bool> OldSink(Builder->BuildSinks);
+    SaveOr OldHasGen(Builder->hasGeneratedNode);
+    
+    if (const Expr *Receiver = msg.getInstanceReceiver()) {
+      const ProgramState *state = Pred->getState();
+      SVal recVal = state->getSVal(Receiver);
+      if (!recVal.isUndef()) {
+        // Bifurcate the state into nil and non-nil ones.
+        DefinedOrUnknownSVal receiverVal = cast<DefinedOrUnknownSVal>(recVal);
+        
+        const ProgramState *notNilState, *nilState;
+        llvm::tie(notNilState, nilState) = state->assume(receiverVal);
+        
+        // There are three cases: can be nil or non-nil, must be nil, must be
+        // non-nil. We ignore must be nil, and merge the rest two into non-nil.
+        if (nilState && !notNilState) {
+          dstEval.insert(Pred);
+          continue;
+        }
+        
+        // Check if the "raise" message was sent.
+        assert(notNilState);
+        if (msg.getSelector() == RaiseSel)
+          RaisesException = true;
+        
+        // Check if we raise an exception.  For now treat these as sinks.
+        // Eventually we will want to handle exceptions properly.
+        if (RaisesException)
+          Builder->BuildSinks = true;
+        
+        // Dispatch to plug-in transfer function.
+        evalObjCMessage(dstEval, msg, Pred, notNilState);
+      }
+    }
+    else if (const ObjCInterfaceDecl *Iface = msg.getReceiverInterface()) {
+      IdentifierInfo* ClsName = Iface->getIdentifier();
+      Selector S = msg.getSelector();
+      
+      // Check for special instance methods.
+      if (!NSExceptionII) {
+        ASTContext &Ctx = getContext();
+        NSExceptionII = &Ctx.Idents.get("NSException");
+      }
+      
+      if (ClsName == NSExceptionII) {
+        enum { NUM_RAISE_SELECTORS = 2 };
+        
+        // Lazily create a cache of the selectors.
+        if (!NSExceptionInstanceRaiseSelectors) {
+          ASTContext &Ctx = getContext();
+          NSExceptionInstanceRaiseSelectors =
+          new Selector[NUM_RAISE_SELECTORS];
+          SmallVector<IdentifierInfo*, NUM_RAISE_SELECTORS> II;
+          unsigned idx = 0;
+          
+          // raise:format:
+          II.push_back(&Ctx.Idents.get("raise"));
+          II.push_back(&Ctx.Idents.get("format"));
+          NSExceptionInstanceRaiseSelectors[idx++] =
+          Ctx.Selectors.getSelector(II.size(), &II[0]);
+          
+          // raise:format::arguments:
+          II.push_back(&Ctx.Idents.get("arguments"));
+          NSExceptionInstanceRaiseSelectors[idx++] =
+          Ctx.Selectors.getSelector(II.size(), &II[0]);
+        }
+        
+        for (unsigned i = 0; i < NUM_RAISE_SELECTORS; ++i)
+          if (S == NSExceptionInstanceRaiseSelectors[i]) {
+            RaisesException = true;
+            break;
+          }
+      }
+      
+      // Check if we raise an exception.  For now treat these as sinks.
+      // Eventually we will want to handle exceptions properly.
+      if (RaisesException)
+        Builder->BuildSinks = true;
+      
+      // Dispatch to plug-in transfer function.
+      evalObjCMessage(dstEval, msg, Pred, Pred->getState());
+    }
+
+    assert(Builder->BuildSinks || Builder->hasGeneratedNode);
+  }
+  
+  // Finally, perform the post-condition check of the ObjCMessageExpr and store
+  // the created nodes in 'Dst'.
+  getCheckerManager().runCheckersForPostObjCMessage(Dst, dstEval, msg, *this);
+}
+
+void ExprEngine::evalObjCMessage(ExplodedNodeSet &Dst, const ObjCMessage &msg, 
+                                 ExplodedNode *Pred,
+                                 const ProgramState *state) {
+  assert (Builder && "StmtNodeBuilder must be defined.");
+
+  // First handle the return value.
+  SVal ReturnValue = UnknownVal();
+
+  // Some method families have known return values.
+  switch (msg.getMethodFamily()) {
+  default:
+    break;
+  case OMF_autorelease:
+  case OMF_retain:
+  case OMF_self: {
+    // These methods return their receivers.
+    const Expr *ReceiverE = msg.getInstanceReceiver();
+    if (ReceiverE)
+      ReturnValue = state->getSVal(ReceiverE);
+    break;
+  }
+  }
+
+  // If we failed to figure out the return value, use a conjured value instead.
+  if (ReturnValue.isUnknown()) {
+    SValBuilder &SVB = getSValBuilder();
+    QualType ResultTy = msg.getResultType(getContext());
+    unsigned Count = Builder->getCurrentBlockCount();
+    const Expr *CurrentE = cast<Expr>(currentStmt);
+    ReturnValue = SVB.getConjuredSymbolVal(NULL, CurrentE, ResultTy, Count);
+  }
+
+  // Bind the return value.
+  state = state->BindExpr(currentStmt, ReturnValue);
+
+  // Invalidate the arguments (and the receiver)
+  const LocationContext *LC = Pred->getLocationContext();
+  state = invalidateArguments(state, CallOrObjCMessage(msg, state), LC);
+
+  // And create the new node.
+  MakeNode(Dst, msg.getOriginExpr(), Pred, state);
+}
+
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/FlatStore.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/FlatStore.cpp
deleted file mode 100644
index ca867aebded4..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/FlatStore.cpp
+++ /dev/null
@@ -1,217 +0,0 @@
-//=== FlatStore.cpp - Flat region-based store model -------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "llvm/ADT/ImmutableIntervalMap.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace clang;
-using namespace ento;
-using llvm::Interval;
-
-// The actual store type.
-typedef llvm::ImmutableIntervalMap<SVal> BindingVal;
-typedef llvm::ImmutableMap<const MemRegion *, BindingVal> RegionBindings;
-
-namespace {
-class FlatStoreManager : public StoreManager {
-  RegionBindings::Factory RBFactory;
-  BindingVal::Factory BVFactory;
-
-public:
-  FlatStoreManager(GRStateManager &mgr) 
-    : StoreManager(mgr), 
-      RBFactory(mgr.getAllocator()), 
-      BVFactory(mgr.getAllocator()) {}
-
-  SVal Retrieve(Store store, Loc L, QualType T);
-  StoreRef Bind(Store store, Loc L, SVal val);
-  StoreRef Remove(Store St, Loc L);
-  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr* cl,
-                            const LocationContext *LC, SVal v);
-
-  StoreRef getInitialStore(const LocationContext *InitLoc) {
-    return StoreRef(RBFactory.getEmptyMap().getRoot(), *this);
-  }
-
-  SubRegionMap *getSubRegionMap(Store store) {
-    return 0;
-  }
-
-  SVal ArrayToPointer(Loc Array);
-  StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
-                              SymbolReaper& SymReaper,
-                          llvm::SmallVectorImpl<const MemRegion*>& RegionRoots){
-    return StoreRef(store, *this);
-  }
-
-  StoreRef BindDecl(Store store, const VarRegion *VR, SVal initVal);
-
-  StoreRef BindDeclWithNoInit(Store store, const VarRegion *VR);
-
-  typedef llvm::DenseSet<SymbolRef> InvalidatedSymbols;
-  
-  StoreRef invalidateRegions(Store store, const MemRegion * const *I,
-                             const MemRegion * const *E, const Expr *Ex,
-                             unsigned Count, InvalidatedSymbols &IS,
-                             bool invalidateGlobals,
-                             InvalidatedRegions *Regions);
-
-  void print(Store store, llvm::raw_ostream& Out, const char* nl, 
-             const char *sep);
-  void iterBindings(Store store, BindingsHandler& f);
-
-private:
-  static RegionBindings getRegionBindings(Store store) {
-    return RegionBindings(static_cast<const RegionBindings::TreeTy*>(store));
-  }
-
-  class RegionInterval {
-  public:
-    const MemRegion *R;
-    Interval I;
-    RegionInterval(const MemRegion *r, int64_t s, int64_t e) : R(r), I(s, e){}
-  };
-
-  RegionInterval RegionToInterval(const MemRegion *R);
-
-  SVal RetrieveRegionWithNoBinding(const MemRegion *R, QualType T);
-};
-} // end anonymous namespace
-
-StoreManager *ento::CreateFlatStoreManager(GRStateManager &StMgr) {
-  return new FlatStoreManager(StMgr);
-}
-
-SVal FlatStoreManager::Retrieve(Store store, Loc L, QualType T) {
-  // For access to concrete addresses, return UnknownVal.  Checks
-  // for null dereferences (and similar errors) are done by checkers, not
-  // the Store.
-  // FIXME: We can consider lazily symbolicating such memory, but we really
-  // should defer this when we can reason easily about symbolicating arrays
-  // of bytes.
-  if (isa<loc::ConcreteInt>(L)) {
-    return UnknownVal();
-  }
-  if (!isa<loc::MemRegionVal>(L)) {
-    return UnknownVal();
-  }
-
-  const MemRegion *R = cast<loc::MemRegionVal>(L).getRegion();
-  RegionInterval RI = RegionToInterval(R);
-  // FIXME: FlatStore should handle regions with unknown intervals.
-  if (!RI.R)
-    return UnknownVal();
-
-  RegionBindings B = getRegionBindings(store);
-  const BindingVal *BV = B.lookup(RI.R);
-  if (BV) {
-    const SVal *V = BVFactory.lookup(*BV, RI.I);
-    if (V)
-      return *V;
-    else
-      return RetrieveRegionWithNoBinding(R, T);
-  }
-  return RetrieveRegionWithNoBinding(R, T);
-}
-
-SVal FlatStoreManager::RetrieveRegionWithNoBinding(const MemRegion *R,
-                                                   QualType T) {
-  if (R->hasStackNonParametersStorage())
-    return UndefinedVal();
-  else
-    return svalBuilder.getRegionValueSymbolVal(cast<TypedRegion>(R));
-}
-
-StoreRef FlatStoreManager::Bind(Store store, Loc L, SVal val) {
-  const MemRegion *R = cast<loc::MemRegionVal>(L).getRegion();
-  RegionBindings B = getRegionBindings(store);
-  const BindingVal *V = B.lookup(R);
-
-  BindingVal BV = BVFactory.getEmptyMap();
-  if (V)
-    BV = *V;
-
-  RegionInterval RI = RegionToInterval(R);
-  // FIXME: FlatStore should handle regions with unknown intervals.
-  if (!RI.R)
-    return StoreRef(B.getRoot(), *this);
-  BV = BVFactory.add(BV, RI.I, val);
-  B = RBFactory.add(B, RI.R, BV);
-  return StoreRef(B.getRoot(), *this);
-}
-
-StoreRef FlatStoreManager::Remove(Store store, Loc L) {
-  return StoreRef(store, *this);
-}
-
-StoreRef FlatStoreManager::BindCompoundLiteral(Store store,
-                                            const CompoundLiteralExpr* cl,
-                                            const LocationContext *LC,
-                                            SVal v) {
-  return StoreRef(store, *this);
-}
-
-SVal FlatStoreManager::ArrayToPointer(Loc Array) {
-  return Array;
-}
-
-StoreRef FlatStoreManager::BindDecl(Store store, const VarRegion *VR, 
-                                    SVal initVal) {
-  return Bind(store, svalBuilder.makeLoc(VR), initVal);
-}
-
-StoreRef FlatStoreManager::BindDeclWithNoInit(Store store, const VarRegion *VR){
-  return StoreRef(store, *this);
-}
-
-StoreRef FlatStoreManager::invalidateRegions(Store store,
-                                             const MemRegion * const *I,
-                                             const MemRegion * const *E,
-                                             const Expr *Ex, unsigned Count,
-                                             InvalidatedSymbols &IS,
-                                             bool invalidateGlobals,
-                                             InvalidatedRegions *Regions) {
-  assert(false && "Not implemented");
-  return StoreRef(store, *this);
-}
-
-void FlatStoreManager::print(Store store, llvm::raw_ostream& Out, 
-                             const char* nl, const char *sep) {
-}
-
-void FlatStoreManager::iterBindings(Store store, BindingsHandler& f) {
-}
-
-FlatStoreManager::RegionInterval 
-FlatStoreManager::RegionToInterval(const MemRegion *R) { 
-  switch (R->getKind()) {
-  case MemRegion::VarRegionKind: {
-    QualType T = cast<VarRegion>(R)->getValueType();
-    int64_t Size = Ctx.getTypeSize(T);
-    return RegionInterval(R, 0, Size-1);
-  }
-
-  case MemRegion::ElementRegionKind: 
-  case MemRegion::FieldRegionKind: {
-    RegionOffset Offset = R->getAsOffset();
-    // We cannot compute offset for all regions, for example, elements
-    // with symbolic offsets.
-    if (!Offset.getRegion())
-      return RegionInterval(0, 0, 0);
-    int64_t Start = Offset.getOffset();
-    int64_t Size = Ctx.getTypeSize(cast<TypedRegion>(R)->getValueType());
-    return RegionInterval(Offset.getRegion(), Start, Start+Size);
-  }
-
-  default:
-    llvm_unreachable("Region kind unhandled.");
-    return RegionInterval(0, 0, 0);
-  }
-}
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index 1ebc28c044fd..0c4e4277e4d6 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
@@ -35,7 +35,7 @@ using namespace ento;
 
 namespace {
 
-class HTMLDiagnostics : public PathDiagnosticClient {
+class HTMLDiagnostics : public PathDiagnosticConsumer {
   llvm::sys::Path Directory, FilePrefix;
   bool createdDir, noDir;
   const Preprocessor &PP;
@@ -45,15 +45,15 @@ class HTMLDiagnostics : public PathDiagnosticClient {
 
   virtual ~HTMLDiagnostics() { FlushDiagnostics(NULL); }
 
-  virtual void FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade);
+  virtual void FlushDiagnostics(SmallVectorImpl<std::string> *FilesMade);
 
-  virtual void HandlePathDiagnostic(const PathDiagnostic* D);
+  virtual void HandlePathDiagnosticImpl(const PathDiagnostic* D);
 
-  virtual llvm::StringRef getName() const {
+  virtual StringRef getName() const {
     return "HTMLDiagnostics";
   }
 
-  unsigned ProcessMacroPiece(llvm::raw_ostream& os,
+  unsigned ProcessMacroPiece(raw_ostream &os,
                              const PathDiagnosticMacroPiece& P,
                              unsigned num);
 
@@ -65,7 +65,7 @@ class HTMLDiagnostics : public PathDiagnosticClient {
                       const char *HighlightEnd = "</span>");
 
   void ReportDiag(const PathDiagnostic& D,
-                  llvm::SmallVectorImpl<std::string> *FilesMade);
+                  SmallVectorImpl<std::string> *FilesMade);
 };
 
 } // end anonymous namespace
@@ -78,8 +78,8 @@ HTMLDiagnostics::HTMLDiagnostics(const std::string& prefix,
   FilePrefix.appendComponent("report");
 }
 
-PathDiagnosticClient*
-ento::createHTMLDiagnosticClient(const std::string& prefix,
+PathDiagnosticConsumer*
+ento::createHTMLDiagnosticConsumer(const std::string& prefix,
                                  const Preprocessor &PP) {
   return new HTMLDiagnostics(prefix, PP);
 }
@@ -88,7 +88,7 @@ ento::createHTMLDiagnosticClient(const std::string& prefix,
 // Report processing.
 //===----------------------------------------------------------------------===//
 
-void HTMLDiagnostics::HandlePathDiagnostic(const PathDiagnostic* D) {
+void HTMLDiagnostics::HandlePathDiagnosticImpl(const PathDiagnostic* D) {
   if (!D)
     return;
 
@@ -102,7 +102,7 @@ void HTMLDiagnostics::HandlePathDiagnostic(const PathDiagnostic* D) {
 }
 
 void
-HTMLDiagnostics::FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade)
+HTMLDiagnostics::FlushDiagnostics(SmallVectorImpl<std::string> *FilesMade)
 {
   while (!BatchedDiags.empty()) {
     const PathDiagnostic* D = BatchedDiags.back();
@@ -115,7 +115,7 @@ HTMLDiagnostics::FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade)
 }
 
 void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
-                                 llvm::SmallVectorImpl<std::string> *FilesMade){
+                                 SmallVectorImpl<std::string> *FilesMade){
   // Create the HTML directory if it is missing.
   if (!createdDir) {
     createdDir = true;
@@ -143,7 +143,7 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
 
   // Verify that the entire path is from the same FileID.
   for (PathDiagnostic::const_iterator I = D.begin(), E = D.end(); I != E; ++I) {
-    FullSourceLoc L = I->getLocation().asLocation().getInstantiationLoc();
+    FullSourceLoc L = I->getLocation().asLocation().getExpansionLoc();
 
     if (FID.isInvalid()) {
       FID = SMgr.getFileID(L);
@@ -154,12 +154,12 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
     for (PathDiagnosticPiece::range_iterator RI=I->ranges_begin(),
                                              RE=I->ranges_end(); RI!=RE; ++RI) {
 
-      SourceLocation L = SMgr.getInstantiationLoc(RI->getBegin());
+      SourceLocation L = SMgr.getExpansionLoc(RI->getBegin());
 
       if (!L.isFileID() || SMgr.getFileID(L) != FID)
         return; // FIXME: Emit a warning?
 
-      L = SMgr.getInstantiationLoc(RI->getEnd());
+      L = SMgr.getExpansionLoc(RI->getEnd());
 
       if (!L.isFileID() || SMgr.getFileID(L) != FID)
         return; // FIXME: Emit a warning?
@@ -221,9 +221,9 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
       << html::EscapeText(Entry->getName())
       << "</td></tr>\n<tr><td class=\"rowname\">Location:</td><td>"
          "<a href=\"#EndPath\">line "
-      << (*D.rbegin()).getLocation().asLocation().getInstantiationLineNumber()
+      << (*D.rbegin()).getLocation().asLocation().getExpansionLineNumber()
       << ", column "
-      << (*D.rbegin()).getLocation().asLocation().getInstantiationColumnNumber()
+      << (*D.rbegin()).getLocation().asLocation().getExpansionColumnNumber()
       << "</a></td></tr>\n"
          "<tr><td class=\"rowname\">Description:</td><td>"
       << D.getDescription() << "</td></tr>\n";
@@ -261,7 +261,7 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
     os << "\n<!-- BUGFILE " << DirName << Entry->getName() << " -->\n";
 
     os << "\n<!-- BUGLINE "
-       << D.back()->getLocation().asLocation().getInstantiationLineNumber()
+       << D.back()->getLocation().asLocation().getExpansionLineNumber()
        << " -->\n";
 
     os << "\n<!-- BUGPATHLENGTH " << D.size() << " -->\n";
@@ -324,7 +324,7 @@ void HTMLDiagnostics::HandlePiece(Rewriter& R, FileID BugFileID,
 
   SourceManager &SM = R.getSourceMgr();
   assert(&Pos.getManager() == &SM && "SourceManagers are different!");
-  std::pair<FileID, unsigned> LPosInfo = SM.getDecomposedInstantiationLoc(Pos);
+  std::pair<FileID, unsigned> LPosInfo = SM.getDecomposedExpansionLoc(Pos);
 
   if (LPosInfo.first != BugFileID)
     return;
@@ -335,7 +335,7 @@ void HTMLDiagnostics::HandlePiece(Rewriter& R, FileID BugFileID,
   // Compute the column number.  Rewind from the current position to the start
   // of the line.
   unsigned ColNo = SM.getColumnNumber(LPosInfo.first, LPosInfo.second);
-  const char *TokInstantiationPtr =Pos.getInstantiationLoc().getCharacterData();
+  const char *TokInstantiationPtr =Pos.getExpansionLoc().getCharacterData();
   const char *LineStart = TokInstantiationPtr-ColNo;
 
   // Compute LineEnd.
@@ -441,9 +441,9 @@ void HTMLDiagnostics::HandlePiece(Rewriter& R, FileID BugFileID,
 
     // Get the name of the macro by relexing it.
     {
-      FullSourceLoc L = MP->getLocation().asLocation().getInstantiationLoc();
+      FullSourceLoc L = MP->getLocation().asLocation().getExpansionLoc();
       assert(L.isFileID());
-      llvm::StringRef BufferInfo = L.getBufferData();
+      StringRef BufferInfo = L.getBufferData();
       const char* MacroName = L.getDecomposedLoc().second + BufferInfo.data();
       Lexer rawLexer(L, PP.getLangOptions(), BufferInfo.begin(),
                      MacroName, BufferInfo.end());
@@ -474,7 +474,7 @@ void HTMLDiagnostics::HandlePiece(Rewriter& R, FileID BugFileID,
   // Insert the new html.
   unsigned DisplayPos = LineEnd - FileStart;
   SourceLocation Loc =
-    SM.getLocForStartOfFile(LPosInfo.first).getFileLocWithOffset(DisplayPos);
+    SM.getLocForStartOfFile(LPosInfo.first).getLocWithOffset(DisplayPos);
 
   R.InsertTextBefore(Loc, os.str());
 
@@ -504,7 +504,7 @@ void HTMLDiagnostics::HandlePiece(Rewriter& R, FileID BugFileID,
 #endif
 }
 
-static void EmitAlphaCounter(llvm::raw_ostream& os, unsigned n) {
+static void EmitAlphaCounter(raw_ostream &os, unsigned n) {
   unsigned x = n % ('z' - 'a');
   n /= 'z' - 'a';
 
@@ -514,7 +514,7 @@ static void EmitAlphaCounter(llvm::raw_ostream& os, unsigned n) {
   os << char('a' + x);
 }
 
-unsigned HTMLDiagnostics::ProcessMacroPiece(llvm::raw_ostream& os,
+unsigned HTMLDiagnostics::ProcessMacroPiece(raw_ostream &os,
                                             const PathDiagnosticMacroPiece& P,
                                             unsigned num) {
 
@@ -549,11 +549,11 @@ void HTMLDiagnostics::HighlightRange(Rewriter& R, FileID BugFileID,
   SourceManager &SM = R.getSourceMgr();
   const LangOptions &LangOpts = R.getLangOpts();
 
-  SourceLocation InstantiationStart = SM.getInstantiationLoc(Range.getBegin());
-  unsigned StartLineNo = SM.getInstantiationLineNumber(InstantiationStart);
+  SourceLocation InstantiationStart = SM.getExpansionLoc(Range.getBegin());
+  unsigned StartLineNo = SM.getExpansionLineNumber(InstantiationStart);
 
-  SourceLocation InstantiationEnd = SM.getInstantiationLoc(Range.getEnd());
-  unsigned EndLineNo = SM.getInstantiationLineNumber(InstantiationEnd);
+  SourceLocation InstantiationEnd = SM.getExpansionLoc(Range.getEnd());
+  unsigned EndLineNo = SM.getExpansionLineNumber(InstantiationEnd);
 
   if (EndLineNo < StartLineNo)
     return;
@@ -563,7 +563,7 @@ void HTMLDiagnostics::HighlightRange(Rewriter& R, FileID BugFileID,
     return;
 
   // Compute the column number of the end.
-  unsigned EndColNo = SM.getInstantiationColumnNumber(InstantiationEnd);
+  unsigned EndColNo = SM.getExpansionColumnNumber(InstantiationEnd);
   unsigned OldEndColNo = EndColNo;
 
   if (EndColNo) {
@@ -575,7 +575,7 @@ void HTMLDiagnostics::HighlightRange(Rewriter& R, FileID BugFileID,
   // selected range.
 
   SourceLocation E =
-    InstantiationEnd.getFileLocWithOffset(EndColNo - OldEndColNo);
+    InstantiationEnd.getLocWithOffset(EndColNo - OldEndColNo);
 
   html::HighlightRange(R, InstantiationStart, E, HighlightStart, HighlightEnd);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Makefile b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Makefile
deleted file mode 100644
index 4aebc163dddc..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- clang/lib/StaticAnalyzer/Core/Makefile --------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-#
-# This implements analyses built on top of source-level CFGs. 
-#
-##===----------------------------------------------------------------------===##
-
-CLANG_LEVEL := ../../..
-LIBRARYNAME := clangStaticAnalyzerCore
-
-include $(CLANG_LEVEL)/Makefile
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index d9e884a4b208..6f92da8f3e5f 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -38,7 +38,7 @@ RegionTy* MemRegionManager::getRegion(const A1 a1) {
 
   llvm::FoldingSetNodeID ID;
   RegionTy::ProfileRegion(ID, a1, superRegion);
-  void* InsertPos;
+  void *InsertPos;
   RegionTy* R = cast_or_null<RegionTy>(Regions.FindNodeOrInsertPos(ID,
                                                                    InsertPos));
 
@@ -56,7 +56,7 @@ RegionTy* MemRegionManager::getSubRegion(const A1 a1,
                                          const MemRegion *superRegion) {
   llvm::FoldingSetNodeID ID;
   RegionTy::ProfileRegion(ID, a1, superRegion);
-  void* InsertPos;
+  void *InsertPos;
   RegionTy* R = cast_or_null<RegionTy>(Regions.FindNodeOrInsertPos(ID,
                                                                    InsertPos));
 
@@ -77,7 +77,7 @@ RegionTy* MemRegionManager::getRegion(const A1 a1, const A2 a2) {
 
   llvm::FoldingSetNodeID ID;
   RegionTy::ProfileRegion(ID, a1, a2, superRegion);
-  void* InsertPos;
+  void *InsertPos;
   RegionTy* R = cast_or_null<RegionTy>(Regions.FindNodeOrInsertPos(ID,
                                                                    InsertPos));
 
@@ -96,7 +96,7 @@ RegionTy* MemRegionManager::getSubRegion(const A1 a1, const A2 a2,
 
   llvm::FoldingSetNodeID ID;
   RegionTy::ProfileRegion(ID, a1, a2, superRegion);
-  void* InsertPos;
+  void *InsertPos;
   RegionTy* R = cast_or_null<RegionTy>(Regions.FindNodeOrInsertPos(ID,
                                                                    InsertPos));
 
@@ -115,7 +115,7 @@ RegionTy* MemRegionManager::getSubRegion(const A1 a1, const A2 a2, const A3 a3,
 
   llvm::FoldingSetNodeID ID;
   RegionTy::ProfileRegion(ID, a1, a2, a3, superRegion);
-  void* InsertPos;
+  void *InsertPos;
   RegionTy* R = cast_or_null<RegionTy>(Regions.FindNodeOrInsertPos(ID,
                                                                    InsertPos));
 
@@ -178,7 +178,7 @@ const StackFrameContext *VarRegion::getStackFrame() const {
 //===----------------------------------------------------------------------===//
 
 DefinedOrUnknownSVal DeclRegion::getExtent(SValBuilder &svalBuilder) const {
-  ASTContext& Ctx = svalBuilder.getContext();
+  ASTContext &Ctx = svalBuilder.getContext();
   QualType T = getDesugaredValueType(Ctx);
 
   if (isa<VariableArrayType>(T))
@@ -250,7 +250,7 @@ void StringRegion::ProfileRegion(llvm::FoldingSetNodeID& ID,
 }
 
 void AllocaRegion::ProfileRegion(llvm::FoldingSetNodeID& ID,
-                                 const Expr* Ex, unsigned cnt,
+                                 const Expr *Ex, unsigned cnt,
                                  const MemRegion *) {
   ID.AddInteger((unsigned) AllocaRegionKind);
   ID.AddPointer(Ex);
@@ -266,7 +266,7 @@ void CompoundLiteralRegion::Profile(llvm::FoldingSetNodeID& ID) const {
 }
 
 void CompoundLiteralRegion::ProfileRegion(llvm::FoldingSetNodeID& ID,
-                                          const CompoundLiteralExpr* CL,
+                                          const CompoundLiteralExpr *CL,
                                           const MemRegion* superRegion) {
   ID.AddInteger((unsigned) CompoundLiteralRegionKind);
   ID.AddPointer(CL);
@@ -285,7 +285,7 @@ void CXXThisRegion::Profile(llvm::FoldingSetNodeID &ID) const {
   CXXThisRegion::ProfileRegion(ID, ThisPointerTy, superRegion);
 }
 
-void DeclRegion::ProfileRegion(llvm::FoldingSetNodeID& ID, const Decl* D,
+void DeclRegion::ProfileRegion(llvm::FoldingSetNodeID& ID, const Decl *D,
                                const MemRegion* superRegion, Kind k) {
   ID.AddInteger((unsigned) k);
   ID.AddPointer(D);
@@ -398,81 +398,82 @@ std::string MemRegion::getString() const {
   return os.str();
 }
 
-void MemRegion::dumpToStream(llvm::raw_ostream& os) const {
+void MemRegion::dumpToStream(raw_ostream &os) const {
   os << "<Unknown Region>";
 }
 
-void AllocaRegion::dumpToStream(llvm::raw_ostream& os) const {
+void AllocaRegion::dumpToStream(raw_ostream &os) const {
   os << "alloca{" << (void*) Ex << ',' << Cnt << '}';
 }
 
-void FunctionTextRegion::dumpToStream(llvm::raw_ostream& os) const {
+void FunctionTextRegion::dumpToStream(raw_ostream &os) const {
   os << "code{" << getDecl()->getDeclName().getAsString() << '}';
 }
 
-void BlockTextRegion::dumpToStream(llvm::raw_ostream& os) const {
+void BlockTextRegion::dumpToStream(raw_ostream &os) const {
   os << "block_code{" << (void*) this << '}';
 }
 
-void BlockDataRegion::dumpToStream(llvm::raw_ostream& os) const {
+void BlockDataRegion::dumpToStream(raw_ostream &os) const {
   os << "block_data{" << BC << '}';
 }
 
-void CompoundLiteralRegion::dumpToStream(llvm::raw_ostream& os) const {
+void CompoundLiteralRegion::dumpToStream(raw_ostream &os) const {
   // FIXME: More elaborate pretty-printing.
   os << "{ " << (void*) CL <<  " }";
 }
 
-void CXXTempObjectRegion::dumpToStream(llvm::raw_ostream &os) const {
-  os << "temp_object";
+void CXXTempObjectRegion::dumpToStream(raw_ostream &os) const {
+  os << "temp_object{" << getValueType().getAsString() << ','
+     << (void*) Ex << '}';
 }
 
-void CXXBaseObjectRegion::dumpToStream(llvm::raw_ostream &os) const {
+void CXXBaseObjectRegion::dumpToStream(raw_ostream &os) const {
   os << "base " << decl->getName();
 }
 
-void CXXThisRegion::dumpToStream(llvm::raw_ostream &os) const {
+void CXXThisRegion::dumpToStream(raw_ostream &os) const {
   os << "this";
 }
 
-void ElementRegion::dumpToStream(llvm::raw_ostream& os) const {
+void ElementRegion::dumpToStream(raw_ostream &os) const {
   os << "element{" << superRegion << ','
      << Index << ',' << getElementType().getAsString() << '}';
 }
 
-void FieldRegion::dumpToStream(llvm::raw_ostream& os) const {
-  os << superRegion << "->" << getDecl();
+void FieldRegion::dumpToStream(raw_ostream &os) const {
+  os << superRegion << "->" << *getDecl();
 }
 
-void NonStaticGlobalSpaceRegion::dumpToStream(llvm::raw_ostream &os) const {
+void NonStaticGlobalSpaceRegion::dumpToStream(raw_ostream &os) const {
   os << "NonStaticGlobalSpaceRegion";
 }
 
-void ObjCIvarRegion::dumpToStream(llvm::raw_ostream& os) const {
-  os << "ivar{" << superRegion << ',' << getDecl() << '}';
+void ObjCIvarRegion::dumpToStream(raw_ostream &os) const {
+  os << "ivar{" << superRegion << ',' << *getDecl() << '}';
 }
 
-void StringRegion::dumpToStream(llvm::raw_ostream& os) const {
+void StringRegion::dumpToStream(raw_ostream &os) const {
   Str->printPretty(os, 0, PrintingPolicy(getContext().getLangOptions()));
 }
 
-void SymbolicRegion::dumpToStream(llvm::raw_ostream& os) const {
+void SymbolicRegion::dumpToStream(raw_ostream &os) const {
   os << "SymRegion{" << sym << '}';
 }
 
-void VarRegion::dumpToStream(llvm::raw_ostream& os) const {
-  os << cast<VarDecl>(D);
+void VarRegion::dumpToStream(raw_ostream &os) const {
+  os << *cast<VarDecl>(D);
 }
 
 void RegionRawOffset::dump() const {
   dumpToStream(llvm::errs());
 }
 
-void RegionRawOffset::dumpToStream(llvm::raw_ostream& os) const {
+void RegionRawOffset::dumpToStream(raw_ostream &os) const {
   os << "raw_offset{" << getRegion() << ',' << getOffset().getQuantity() << '}';
 }
 
-void StaticGlobalSpaceRegion::dumpToStream(llvm::raw_ostream &os) const {
+void StaticGlobalSpaceRegion::dumpToStream(raw_ostream &os) const {
   os << "StaticGlobalsMemSpace{" << CR << '}';
 }
 
@@ -631,7 +632,7 @@ MemRegionManager::getBlockDataRegion(const BlockTextRegion *BC,
 }
 
 const CompoundLiteralRegion*
-MemRegionManager::getCompoundLiteralRegion(const CompoundLiteralExpr* CL,
+MemRegionManager::getCompoundLiteralRegion(const CompoundLiteralExpr *CL,
                                            const LocationContext *LC) {
 
   const MemRegion *sReg = 0;
@@ -650,14 +651,14 @@ MemRegionManager::getCompoundLiteralRegion(const CompoundLiteralExpr* CL,
 const ElementRegion*
 MemRegionManager::getElementRegion(QualType elementType, NonLoc Idx,
                                    const MemRegion* superRegion,
-                                   ASTContext& Ctx){
+                                   ASTContext &Ctx){
 
   QualType T = Ctx.getCanonicalType(elementType).getUnqualifiedType();
 
   llvm::FoldingSetNodeID ID;
   ElementRegion::ProfileRegion(ID, T, Idx, superRegion);
 
-  void* InsertPos;
+  void *InsertPos;
   MemRegion* data = Regions.FindNodeOrInsertPos(ID, InsertPos);
   ElementRegion* R = cast_or_null<ElementRegion>(data);
 
@@ -688,13 +689,13 @@ const SymbolicRegion *MemRegionManager::getSymbolicRegion(SymbolRef sym) {
 }
 
 const FieldRegion*
-MemRegionManager::getFieldRegion(const FieldDecl* d,
+MemRegionManager::getFieldRegion(const FieldDecl *d,
                                  const MemRegion* superRegion){
   return getSubRegion<FieldRegion>(d, superRegion);
 }
 
 const ObjCIvarRegion*
-MemRegionManager::getObjCIvarRegion(const ObjCIvarDecl* d,
+MemRegionManager::getObjCIvarRegion(const ObjCIvarDecl *d,
                                     const MemRegion* superRegion) {
   return getSubRegion<ObjCIvarRegion>(d, superRegion);
 }
@@ -724,7 +725,7 @@ MemRegionManager::getCXXThisRegion(QualType thisPointerTy,
 }
 
 const AllocaRegion*
-MemRegionManager::getAllocaRegion(const Expr* E, unsigned cnt,
+MemRegionManager::getAllocaRegion(const Expr *E, unsigned cnt,
                                   const LocationContext *LC) {
   const StackFrameContext *STC = LC->getCurrentStackFrame();
   assert(STC);
@@ -896,7 +897,7 @@ RegionOffset MemRegion::getAsOffset() const {
     case FieldRegionKind: {
       const FieldRegion *FR = cast<FieldRegion>(R);
       const RecordDecl *RD = FR->getDecl()->getParent();
-      if (!RD->isDefinition())
+      if (!RD->isCompleteDefinition())
         // We cannot compute offset for incomplete type.
         return RegionOffset(0);
       // Get the field number.
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ObjCMessage.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ObjCMessage.cpp
index c00060047498..0974fe877ac2 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ObjCMessage.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ObjCMessage.cpp
@@ -112,18 +112,22 @@ QualType CallOrObjCMessage::getResultType(ASTContext &ctx) const {
   QualType resultTy;
   bool isLVal = false;
 
-  if (CallE) {
-    isLVal = CallE->isLValue();
-    const Expr *Callee = CallE->getCallee();
-    if (const FunctionDecl *FD = State->getSVal(Callee).getAsFunctionDecl())
-      resultTy = FD->getResultType();
-    else
-      resultTy = CallE->getType();
-  }
-  else {
+  if (isObjCMessage()) {
     isLVal = isa<ObjCMessageExpr>(Msg.getOriginExpr()) &&
              Msg.getOriginExpr()->isLValue();
     resultTy = Msg.getResultType(ctx);
+  } else if (const CXXConstructExpr *Ctor =
+              CallE.dyn_cast<const CXXConstructExpr *>()) {
+    resultTy = Ctor->getType();
+  } else {
+    const CallExpr *FunctionCall = CallE.get<const CallExpr *>();
+
+    isLVal = FunctionCall->isLValue();
+    const Expr *Callee = FunctionCall->getCallee();
+    if (const FunctionDecl *FD = State->getSVal(Callee).getAsFunctionDecl())
+      resultTy = FD->getResultType();
+    else
+      resultTy = FunctionCall->getType();
   }
 
   if (isLVal)
@@ -132,25 +136,29 @@ QualType CallOrObjCMessage::getResultType(ASTContext &ctx) const {
   return resultTy;
 }
 
-SVal CallOrObjCMessage::getArgSValAsScalarOrLoc(unsigned i) const {
-  assert(i < getNumArgs());
-  if (CallE) return State->getSValAsScalarOrLoc(CallE->getArg(i));
-  QualType argT = Msg.getArgType(i);
-  if (Loc::isLocType(argT) || argT->isIntegerType())
-    return Msg.getArgSVal(i, State);
-  return UnknownVal();
-}
-
 SVal CallOrObjCMessage::getFunctionCallee() const {
   assert(isFunctionCall());
   assert(!isCXXCall());
-  const Expr *callee = CallE->getCallee()->IgnoreParenCasts();
-  return State->getSVal(callee);
+  const Expr *Fun = CallE.get<const CallExpr *>()->getCallee()->IgnoreParens();
+  return State->getSVal(Fun);
 }
 
 SVal CallOrObjCMessage::getCXXCallee() const {
   assert(isCXXCall());
+  const CallExpr *ActualCall = CallE.get<const CallExpr *>();
   const Expr *callee =
-    cast<CXXMemberCallExpr>(CallE)->getImplicitObjectArgument();
-  return State->getSVal(callee);  
+    cast<CXXMemberCallExpr>(ActualCall)->getImplicitObjectArgument();
+  
+  // FIXME: Will eventually need to cope with member pointers.  This is
+  // a limitation in getImplicitObjectArgument().
+  if (!callee)
+    return UnknownVal();
+  
+  return State->getSVal(callee);
+}
+
+SVal
+CallOrObjCMessage::getInstanceMessageReceiver(const LocationContext *LC) const {
+  assert(isObjCMessage());
+  return Msg.getInstanceReceiverSVal(State, LC);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
index 872bbfe9e160..3a879030da11 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
@@ -12,17 +12,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/ParentMap.h"
 #include "clang/AST/StmtCXX.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/Support/Casting.h"
 
 using namespace clang;
 using namespace ento;
-using llvm::dyn_cast;
-using llvm::isa;
 
 bool PathDiagnosticMacroPiece::containsEvent() const {
   for (const_iterator I = begin(), E = end(); I!=E; ++I) {
@@ -37,14 +36,14 @@ bool PathDiagnosticMacroPiece::containsEvent() const {
   return false;
 }
 
-static llvm::StringRef StripTrailingDots(llvm::StringRef s) {
-  for (llvm::StringRef::size_type i = s.size(); i != 0; --i)
+static StringRef StripTrailingDots(StringRef s) {
+  for (StringRef::size_type i = s.size(); i != 0; --i)
     if (s[i - 1] != '.')
       return s.substr(0, i);
   return "";
 }
 
-PathDiagnosticPiece::PathDiagnosticPiece(llvm::StringRef s,
+PathDiagnosticPiece::PathDiagnosticPiece(StringRef s,
                                          Kind k, DisplayHint hint)
   : str(StripTrailingDots(s)), kind(k), Hint(hint) {}
 
@@ -76,55 +75,161 @@ void PathDiagnostic::resetPath(bool deletePieces) {
 }
 
 
-PathDiagnostic::PathDiagnostic(llvm::StringRef bugtype, llvm::StringRef desc,
-                               llvm::StringRef category)
+PathDiagnostic::PathDiagnostic(StringRef bugtype, StringRef desc,
+                               StringRef category)
   : Size(0),
     BugType(StripTrailingDots(bugtype)),
     Desc(StripTrailingDots(desc)),
     Category(StripTrailingDots(category)) {}
 
-void PathDiagnosticClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                            const DiagnosticInfo &Info) {
-  // Default implementation (Warnings/errors count).
-  DiagnosticClient::HandleDiagnostic(DiagLevel, Info);
-
-  // Create a PathDiagnostic with a single piece.
-
-  PathDiagnostic* D = new PathDiagnostic();
-
-  const char *LevelStr;
-  switch (DiagLevel) {
-  default:
-  case Diagnostic::Ignored: assert(0 && "Invalid diagnostic type");
-  case Diagnostic::Note:    LevelStr = "note: "; break;
-  case Diagnostic::Warning: LevelStr = "warning: "; break;
-  case Diagnostic::Error:   LevelStr = "error: "; break;
-  case Diagnostic::Fatal:   LevelStr = "fatal error: "; break;
-  }
-
-  llvm::SmallString<100> StrC;
-  StrC += LevelStr;
-  Info.FormatDiagnostic(StrC);
-
-  PathDiagnosticPiece *P =
-    new PathDiagnosticEventPiece(FullSourceLoc(Info.getLocation(),
-                                               Info.getSourceManager()),
-                                 StrC.str());
-
-  for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i)
-    P->addRange(Info.getRange(i).getAsRange());
-  for (unsigned i = 0, e = Info.getNumFixItHints(); i != e; ++i)
-    P->addFixItHint(Info.getFixItHint(i));
-  D->push_front(P);
-
-  HandlePathDiagnostic(D);
+void PathDiagnosticConsumer::HandlePathDiagnostic(const PathDiagnostic *D) {
+  // For now this simply forwards to HandlePathDiagnosticImpl.  In the future
+  // we can use this indirection to control for multi-threaded access to
+  // the PathDiagnosticConsumer from multiple bug reporters.
+  HandlePathDiagnosticImpl(D);
 }
 
 //===----------------------------------------------------------------------===//
 // PathDiagnosticLocation methods.
 //===----------------------------------------------------------------------===//
 
-FullSourceLoc PathDiagnosticLocation::asLocation() const {
+static SourceLocation getValidSourceLocation(const Stmt* S,
+                                             LocationOrAnalysisContext LAC) {
+  SourceLocation L = S->getLocStart();
+  assert(!LAC.isNull() && "A valid LocationContext or AnalysisContext should "
+                          "be passed to PathDiagnosticLocation upon creation.");
+
+  // S might be a temporary statement that does not have a location in the
+  // source code, so find an enclosing statement and use it's location.
+  if (!L.isValid()) {
+
+    ParentMap *PM = 0;
+    if (LAC.is<const LocationContext*>())
+      PM = &LAC.get<const LocationContext*>()->getParentMap();
+    else
+      PM = &LAC.get<AnalysisContext*>()->getParentMap();
+
+    while (!L.isValid()) {
+      S = PM->getParent(S);
+      L = S->getLocStart();
+    }
+  }
+
+  return L;
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createBegin(const Decl *D,
+                                      const SourceManager &SM) {
+  return PathDiagnosticLocation(D->getLocStart(), SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createBegin(const Stmt *S,
+                                      const SourceManager &SM,
+                                      LocationOrAnalysisContext LAC) {
+  return PathDiagnosticLocation(getValidSourceLocation(S, LAC),
+                                SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createOperatorLoc(const BinaryOperator *BO,
+                                            const SourceManager &SM) {
+  return PathDiagnosticLocation(BO->getOperatorLoc(), SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createMemberLoc(const MemberExpr *ME,
+                                          const SourceManager &SM) {
+  return PathDiagnosticLocation(ME->getMemberLoc(), SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createBeginBrace(const CompoundStmt *CS,
+                                           const SourceManager &SM) {
+  SourceLocation L = CS->getLBracLoc();
+  return PathDiagnosticLocation(L, SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createEndBrace(const CompoundStmt *CS,
+                                         const SourceManager &SM) {
+  SourceLocation L = CS->getRBracLoc();
+  return PathDiagnosticLocation(L, SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createDeclBegin(const LocationContext *LC,
+                                          const SourceManager &SM) {
+  // FIXME: Should handle CXXTryStmt if analyser starts supporting C++.
+  if (const CompoundStmt *CS =
+        dyn_cast_or_null<CompoundStmt>(LC->getDecl()->getBody()))
+    if (!CS->body_empty()) {
+      SourceLocation Loc = (*CS->body_begin())->getLocStart();
+      return PathDiagnosticLocation(Loc, SM, SingleLocK);
+    }
+
+  return PathDiagnosticLocation();
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createDeclEnd(const LocationContext *LC,
+                                        const SourceManager &SM) {
+  SourceLocation L = LC->getDecl()->getBodyRBrace();
+  return PathDiagnosticLocation(L, SM, SingleLocK);
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::create(const ProgramPoint& P,
+                                 const SourceManager &SMng) {
+
+  const Stmt* S = 0;
+  if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
+    const CFGBlock *BSrc = BE->getSrc();
+    S = BSrc->getTerminatorCondition();
+  }
+  else if (const PostStmt *PS = dyn_cast<PostStmt>(&P)) {
+    S = PS->getStmt();
+  }
+
+  return PathDiagnosticLocation(S, SMng, P.getLocationContext());
+
+  if (!S)
+    return PathDiagnosticLocation();
+}
+
+PathDiagnosticLocation
+  PathDiagnosticLocation::createEndOfPath(const ExplodedNode* N,
+                                          const SourceManager &SM) {
+  assert(N && "Cannot create a location with a null node.");
+
+  const ExplodedNode *NI = N;
+
+  while (NI) {
+    ProgramPoint P = NI->getLocation();
+    const LocationContext *LC = P.getLocationContext();
+    if (const StmtPoint *PS = dyn_cast<StmtPoint>(&P))
+      return PathDiagnosticLocation(PS->getStmt(), SM, LC);
+    else if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
+      const Stmt *Term = BE->getSrc()->getTerminator();
+      assert(Term);
+      return PathDiagnosticLocation(Term, SM, LC);
+    }
+    NI = NI->succ_empty() ? 0 : *(NI->succ_begin());
+  }
+
+  return createDeclEnd(N->getLocationContext(), SM);
+}
+
+PathDiagnosticLocation PathDiagnosticLocation::createSingleLocation(
+                                           const PathDiagnosticLocation &PDL) {
+  FullSourceLoc L = PDL.asLocation();
+  return PathDiagnosticLocation(L, L.getManager(), SingleLocK);
+}
+
+FullSourceLoc
+  PathDiagnosticLocation::genLocation(SourceLocation L,
+                                      LocationOrAnalysisContext LAC) const {
   assert(isValid());
   // Note that we want a 'switch' here so that the compiler can warn us in
   // case we add more cases.
@@ -133,21 +238,23 @@ FullSourceLoc PathDiagnosticLocation::asLocation() const {
     case RangeK:
       break;
     case StmtK:
-      return FullSourceLoc(S->getLocStart(), const_cast<SourceManager&>(*SM));
+      return FullSourceLoc(getValidSourceLocation(S, LAC),
+                           const_cast<SourceManager&>(*SM));
     case DeclK:
       return FullSourceLoc(D->getLocation(), const_cast<SourceManager&>(*SM));
   }
 
-  return FullSourceLoc(R.getBegin(), const_cast<SourceManager&>(*SM));
+  return FullSourceLoc(L, const_cast<SourceManager&>(*SM));
 }
 
-PathDiagnosticRange PathDiagnosticLocation::asRange() const {
+PathDiagnosticRange
+  PathDiagnosticLocation::genRange(LocationOrAnalysisContext LAC) const {
   assert(isValid());
   // Note that we want a 'switch' here so that the compiler can warn us in
   // case we add more cases.
   switch (K) {
     case SingleLocK:
-      return PathDiagnosticRange(R, true);
+      return PathDiagnosticRange(SourceRange(Loc,Loc), true);
     case RangeK:
       break;
     case StmtK: {
@@ -176,12 +283,14 @@ PathDiagnosticRange PathDiagnosticLocation::asRange() const {
         case Stmt::BinaryConditionalOperatorClass:
         case Stmt::ConditionalOperatorClass:
         case Stmt::ObjCForCollectionStmtClass: {
-          SourceLocation L = S->getLocStart();
+          SourceLocation L = getValidSourceLocation(S, LAC);
           return SourceRange(L, L);
         }
       }
-
-      return S->getSourceRange();
+      SourceRange R = S->getSourceRange();
+      if (R.isValid())
+        return R;
+      break;  
     }
     case DeclK:
       if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D))
@@ -196,19 +305,16 @@ PathDiagnosticRange PathDiagnosticLocation::asRange() const {
       }
   }
 
-  return R;
+  return SourceRange(Loc,Loc);
 }
 
 void PathDiagnosticLocation::flatten() {
   if (K == StmtK) {
-    R = asRange();
     K = RangeK;
     S = 0;
     D = 0;
   }
   else if (K == DeclK) {
-    SourceLocation L = D->getLocation();
-    R = SourceRange(L, L);
     K = SingleLocK;
     S = 0;
     D = 0;
@@ -220,22 +326,9 @@ void PathDiagnosticLocation::flatten() {
 //===----------------------------------------------------------------------===//
 
 void PathDiagnosticLocation::Profile(llvm::FoldingSetNodeID &ID) const {
-  ID.AddInteger((unsigned) K);
-  switch (K) {
-    case RangeK:
-      ID.AddInteger(R.getBegin().getRawEncoding());
-      ID.AddInteger(R.getEnd().getRawEncoding());
-      break;      
-    case SingleLocK:
-      ID.AddInteger(R.getBegin().getRawEncoding());
-      break;
-    case StmtK:
-      ID.Add(S);
-      break;
-    case DeclK:
-      ID.Add(D);
-      break;
-  }
+  ID.AddInteger(Range.getBegin().getRawEncoding());
+  ID.AddInteger(Range.getEnd().getRawEncoding());
+  ID.AddInteger(Loc.getRawEncoding());
   return;
 }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
index fbbbd46ac149..5ae95c61f819 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
@@ -22,14 +22,9 @@
 #include "llvm/ADT/SmallVector.h"
 using namespace clang;
 using namespace ento;
-using llvm::cast;
 
 typedef llvm::DenseMap<FileID, unsigned> FIDMap;
 
-namespace clang {
-  class Preprocessor;
-}
-
 namespace {
 struct CompareDiagnostics {
   // Compare if 'X' is "<" than 'Y'.
@@ -43,16 +38,16 @@ struct CompareDiagnostics {
       return false;
     
     // Next, compare by bug type.
-    llvm::StringRef XBugType = X->getBugType();
-    llvm::StringRef YBugType = Y->getBugType();
+    StringRef XBugType = X->getBugType();
+    StringRef YBugType = Y->getBugType();
     if (XBugType < YBugType)
       return true;
     if (XBugType != YBugType)
       return false;
     
     // Next, compare by bug description.
-    llvm::StringRef XDesc = X->getDescription();
-    llvm::StringRef YDesc = Y->getDescription();
+    StringRef XDesc = X->getDescription();
+    StringRef YDesc = Y->getDescription();
     if (XDesc < YDesc)
       return true;
     if (XDesc != YDesc)
@@ -65,23 +60,23 @@ struct CompareDiagnostics {
 }
 
 namespace {
-  class PlistDiagnostics : public PathDiagnosticClient {
+  class PlistDiagnostics : public PathDiagnosticConsumer {
     std::vector<const PathDiagnostic*> BatchedDiags;
     const std::string OutputFile;
     const LangOptions &LangOpts;
-    llvm::OwningPtr<PathDiagnosticClient> SubPD;
+    llvm::OwningPtr<PathDiagnosticConsumer> SubPD;
     bool flushed;
   public:
     PlistDiagnostics(const std::string& prefix, const LangOptions &LangOpts,
-                     PathDiagnosticClient *subPD);
+                     PathDiagnosticConsumer *subPD);
 
     ~PlistDiagnostics() { FlushDiagnostics(NULL); }
 
-    void FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade);
+    void FlushDiagnostics(SmallVectorImpl<std::string> *FilesMade);
     
-    void HandlePathDiagnostic(const PathDiagnostic* D);
+    void HandlePathDiagnosticImpl(const PathDiagnostic* D);
     
-    virtual llvm::StringRef getName() const {
+    virtual StringRef getName() const {
       return "PlistDiagnostics";
     }
 
@@ -94,27 +89,27 @@ namespace {
 
 PlistDiagnostics::PlistDiagnostics(const std::string& output,
                                    const LangOptions &LO,
-                                   PathDiagnosticClient *subPD)
+                                   PathDiagnosticConsumer *subPD)
   : OutputFile(output), LangOpts(LO), SubPD(subPD), flushed(false) {}
 
-PathDiagnosticClient*
-ento::createPlistDiagnosticClient(const std::string& s, const Preprocessor &PP,
-                                  PathDiagnosticClient *subPD) {
+PathDiagnosticConsumer*
+ento::createPlistDiagnosticConsumer(const std::string& s, const Preprocessor &PP,
+                                  PathDiagnosticConsumer *subPD) {
   return new PlistDiagnostics(s, PP.getLangOptions(), subPD);
 }
 
-PathDiagnosticClient::PathGenerationScheme
+PathDiagnosticConsumer::PathGenerationScheme
 PlistDiagnostics::getGenerationScheme() const {
-  if (const PathDiagnosticClient *PD = SubPD.get())
+  if (const PathDiagnosticConsumer *PD = SubPD.get())
     return PD->getGenerationScheme();
 
   return Extensive;
 }
 
-static void AddFID(FIDMap &FIDs, llvm::SmallVectorImpl<FileID> &V,
+static void AddFID(FIDMap &FIDs, SmallVectorImpl<FileID> &V,
                    const SourceManager* SM, SourceLocation L) {
 
-  FileID FID = SM->getFileID(SM->getInstantiationLoc(L));
+  FileID FID = SM->getFileID(SM->getExpansionLoc(L));
   FIDMap::iterator I = FIDs.find(FID);
   if (I != FIDs.end()) return;
   FIDs[FID] = V.size();
@@ -123,23 +118,23 @@ static void AddFID(FIDMap &FIDs, llvm::SmallVectorImpl<FileID> &V,
 
 static unsigned GetFID(const FIDMap& FIDs, const SourceManager &SM,
                        SourceLocation L) {
-  FileID FID = SM.getFileID(SM.getInstantiationLoc(L));
+  FileID FID = SM.getFileID(SM.getExpansionLoc(L));
   FIDMap::const_iterator I = FIDs.find(FID);
   assert(I != FIDs.end());
   return I->second;
 }
 
-static llvm::raw_ostream& Indent(llvm::raw_ostream& o, const unsigned indent) {
+static raw_ostream &Indent(raw_ostream &o, const unsigned indent) {
   for (unsigned i = 0; i < indent; ++i) o << ' ';
   return o;
 }
 
-static void EmitLocation(llvm::raw_ostream& o, const SourceManager &SM,
+static void EmitLocation(raw_ostream &o, const SourceManager &SM,
                          const LangOptions &LangOpts,
                          SourceLocation L, const FIDMap &FM,
                          unsigned indent, bool extend = false) {
 
-  FullSourceLoc Loc(SM.getInstantiationLoc(L), const_cast<SourceManager&>(SM));
+  FullSourceLoc Loc(SM.getExpansionLoc(L), const_cast<SourceManager&>(SM));
 
   // Add in the length of the token, so that we cover multi-char tokens.
   unsigned offset =
@@ -147,22 +142,22 @@ static void EmitLocation(llvm::raw_ostream& o, const SourceManager &SM,
 
   Indent(o, indent) << "<dict>\n";
   Indent(o, indent) << " <key>line</key><integer>"
-                    << Loc.getInstantiationLineNumber() << "</integer>\n";
+                    << Loc.getExpansionLineNumber() << "</integer>\n";
   Indent(o, indent) << " <key>col</key><integer>"
-                    << Loc.getInstantiationColumnNumber() + offset << "</integer>\n";
+                    << Loc.getExpansionColumnNumber() + offset << "</integer>\n";
   Indent(o, indent) << " <key>file</key><integer>"
                     << GetFID(FM, SM, Loc) << "</integer>\n";
   Indent(o, indent) << "</dict>\n";
 }
 
-static void EmitLocation(llvm::raw_ostream& o, const SourceManager &SM,
+static void EmitLocation(raw_ostream &o, const SourceManager &SM,
                          const LangOptions &LangOpts,
                          const PathDiagnosticLocation &L, const FIDMap& FM,
                          unsigned indent, bool extend = false) {
   EmitLocation(o, SM, LangOpts, L.asLocation(), FM, indent, extend);
 }
 
-static void EmitRange(llvm::raw_ostream& o, const SourceManager &SM,
+static void EmitRange(raw_ostream &o, const SourceManager &SM,
                       const LangOptions &LangOpts,
                       PathDiagnosticRange R, const FIDMap &FM,
                       unsigned indent) {
@@ -172,7 +167,7 @@ static void EmitRange(llvm::raw_ostream& o, const SourceManager &SM,
   Indent(o, indent) << "</array>\n";
 }
 
-static llvm::raw_ostream& EmitString(llvm::raw_ostream& o,
+static raw_ostream &EmitString(raw_ostream &o,
                                      const std::string& s) {
   o << "<string>";
   for (std::string::const_iterator I=s.begin(), E=s.end(); I!=E; ++I) {
@@ -190,7 +185,7 @@ static llvm::raw_ostream& EmitString(llvm::raw_ostream& o,
   return o;
 }
 
-static void ReportControlFlow(llvm::raw_ostream& o,
+static void ReportControlFlow(raw_ostream &o,
                               const PathDiagnosticControlFlowPiece& P,
                               const FIDMap& FM,
                               const SourceManager &SM,
@@ -233,7 +228,7 @@ static void ReportControlFlow(llvm::raw_ostream& o,
   Indent(o, indent) << "</dict>\n";
 }
 
-static void ReportEvent(llvm::raw_ostream& o, const PathDiagnosticPiece& P,
+static void ReportEvent(raw_ostream &o, const PathDiagnosticPiece& P,
                         const FIDMap& FM,
                         const SourceManager &SM,
                         const LangOptions &LangOpts,
@@ -280,7 +275,7 @@ static void ReportEvent(llvm::raw_ostream& o, const PathDiagnosticPiece& P,
   Indent(o, indent); o << "</dict>\n";
 }
 
-static void ReportMacro(llvm::raw_ostream& o,
+static void ReportMacro(raw_ostream &o,
                         const PathDiagnosticMacroPiece& P,
                         const FIDMap& FM, const SourceManager &SM,
                         const LangOptions &LangOpts,
@@ -304,7 +299,7 @@ static void ReportMacro(llvm::raw_ostream& o,
   }
 }
 
-static void ReportDiag(llvm::raw_ostream& o, const PathDiagnosticPiece& P,
+static void ReportDiag(raw_ostream &o, const PathDiagnosticPiece& P,
                        const FIDMap& FM, const SourceManager &SM,
                        const LangOptions &LangOpts) {
 
@@ -326,7 +321,7 @@ static void ReportDiag(llvm::raw_ostream& o, const PathDiagnosticPiece& P,
   }
 }
 
-void PlistDiagnostics::HandlePathDiagnostic(const PathDiagnostic* D) {
+void PlistDiagnostics::HandlePathDiagnosticImpl(const PathDiagnostic* D) {
   if (!D)
     return;
 
@@ -342,7 +337,7 @@ void PlistDiagnostics::HandlePathDiagnostic(const PathDiagnostic* D) {
   BatchedDiags.push_back(D);
 }
 
-void PlistDiagnostics::FlushDiagnostics(llvm::SmallVectorImpl<std::string>
+void PlistDiagnostics::FlushDiagnostics(SmallVectorImpl<std::string>
                                         *FilesMade) {
   
   if (flushed)
@@ -358,7 +353,7 @@ void PlistDiagnostics::FlushDiagnostics(llvm::SmallVectorImpl<std::string>
   // Build up a set of FIDs that we use by scanning the locations and
   // ranges of the diagnostics.
   FIDMap FM;
-  llvm::SmallVector<FileID, 10> Fids;
+  SmallVector<FileID, 10> Fids;
   const SourceManager* SM = 0;
 
   if (!BatchedDiags.empty())
@@ -401,7 +396,7 @@ void PlistDiagnostics::FlushDiagnostics(llvm::SmallVectorImpl<std::string>
        " <key>files</key>\n"
        " <array>\n";
 
-  for (llvm::SmallVectorImpl<FileID>::iterator I=Fids.begin(), E=Fids.end();
+  for (SmallVectorImpl<FileID>::iterator I=Fids.begin(), E=Fids.end();
        I!=E; ++I) {
     o << "  ";
     EmitString(o, SM->getFileEntryForID(*I)->getName()) << '\n';
@@ -444,7 +439,7 @@ void PlistDiagnostics::FlushDiagnostics(llvm::SmallVectorImpl<std::string>
     // Output the diagnostic to the sub-diagnostic client, if any.
     if (SubPD) {
       SubPD->HandlePathDiagnostic(OwnedD.take());
-      llvm::SmallVector<std::string, 1> SubFilesMade;
+      SmallVector<std::string, 1> SubFilesMade;
       SubPD->FlushDiagnostics(SubFilesMade);
 
       if (!SubFilesMade.empty()) {
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/GRState.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
similarity index 65%
rename from contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/GRState.cpp
rename to contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
index 0f6ff1ef588c..73788cc42efb 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/GRState.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
@@ -1,4 +1,4 @@
-//= GRState.cpp - Path-Sensitive "State" for tracking values -----*- C++ -*--=//
+//= ProgramState.cpp - Path-Sensitive "State" for tracking values --*- C++ -*--=
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,15 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file implements GRState and GRStateManager.
+//  This file implements ProgramState and ProgramStateManager.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/CFG.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
@@ -25,7 +24,7 @@ using namespace ento;
 // FIXME: Move this elsewhere.
 ConstraintManager::~ConstraintManager() {}
 
-GRState::GRState(GRStateManager *mgr, const Environment& env,
+ProgramState::ProgramState(ProgramStateManager *mgr, const Environment& env,
                  StoreRef st, GenericDataMap gdm)
   : stateMgr(mgr),
     Env(env),
@@ -35,7 +34,7 @@ GRState::GRState(GRStateManager *mgr, const Environment& env,
   stateMgr->getStoreManager().incrementReferenceCount(store);
 }
 
-GRState::GRState(const GRState& RHS)
+ProgramState::ProgramState(const ProgramState &RHS)
     : llvm::FoldingSetNode(),
       stateMgr(RHS.stateMgr),
       Env(RHS.Env),
@@ -45,23 +44,19 @@ GRState::GRState(const GRState& RHS)
   stateMgr->getStoreManager().incrementReferenceCount(store);
 }
 
-GRState::~GRState() {
+ProgramState::~ProgramState() {
   if (store)
     stateMgr->getStoreManager().decrementReferenceCount(store);
 }
 
-GRStateManager::~GRStateManager() {
-  for (std::vector<GRState::Printer*>::iterator I=Printers.begin(),
-        E=Printers.end(); I!=E; ++I)
-    delete *I;
-
+ProgramStateManager::~ProgramStateManager() {
   for (GDMContextsTy::iterator I=GDMContexts.begin(), E=GDMContexts.end();
        I!=E; ++I)
     I->second.second(I->second.first);
 }
 
-const GRState*
-GRStateManager::removeDeadBindings(const GRState* state,
+const ProgramState*
+ProgramStateManager::removeDeadBindings(const ProgramState *state,
                                    const StackFrameContext *LCtx,
                                    SymbolReaper& SymReaper) {
 
@@ -71,23 +66,23 @@ GRStateManager::removeDeadBindings(const GRState* state,
   // those around.  This code more than likely can be made faster, and the
   // frequency of which this method is called should be experimented with
   // for optimum performance.
-  llvm::SmallVector<const MemRegion*, 10> RegionRoots;
-  GRState NewState = *state;
+  ProgramState NewState = *state;
 
-  NewState.Env = EnvMgr.removeDeadBindings(NewState.Env, SymReaper,
-                                           state, RegionRoots);
+  NewState.Env = EnvMgr.removeDeadBindings(NewState.Env, SymReaper, state);
 
   // Clean up the store.
-  NewState.setStore(StoreMgr->removeDeadBindings(NewState.getStore(), LCtx,
-                                                 SymReaper, RegionRoots));
-  state = getPersistentState(NewState);
-  return ConstraintMgr->removeDeadBindings(state, SymReaper);
+  StoreRef newStore = StoreMgr->removeDeadBindings(NewState.getStore(), LCtx,
+                                                   SymReaper);
+  NewState.setStore(newStore);
+  SymReaper.setReapedStore(newStore);
+  
+  return getPersistentState(NewState);
 }
 
-const GRState *GRStateManager::MarshalState(const GRState *state,
+const ProgramState *ProgramStateManager::MarshalState(const ProgramState *state,
                                             const StackFrameContext *InitLoc) {
   // make up an empty state for now.
-  GRState State(this,
+  ProgramState State(this,
                 EnvMgr.getInitialEnvironment(),
                 StoreMgr->getInitialStore(InitLoc),
                 GDMFactory.getEmptyMap());
@@ -95,7 +90,7 @@ const GRState *GRStateManager::MarshalState(const GRState *state,
   return getPersistentState(State);
 }
 
-const GRState *GRState::bindCompoundLiteral(const CompoundLiteralExpr* CL,
+const ProgramState *ProgramState::bindCompoundLiteral(const CompoundLiteralExpr *CL,
                                             const LocationContext *LC,
                                             SVal V) const {
   const StoreRef &newStore = 
@@ -103,21 +98,21 @@ const GRState *GRState::bindCompoundLiteral(const CompoundLiteralExpr* CL,
   return makeWithStore(newStore);
 }
 
-const GRState *GRState::bindDecl(const VarRegion* VR, SVal IVal) const {
+const ProgramState *ProgramState::bindDecl(const VarRegion* VR, SVal IVal) const {
   const StoreRef &newStore =
     getStateManager().StoreMgr->BindDecl(getStore(), VR, IVal);
   return makeWithStore(newStore);
 }
 
-const GRState *GRState::bindDeclWithNoInit(const VarRegion* VR) const {
+const ProgramState *ProgramState::bindDeclWithNoInit(const VarRegion* VR) const {
   const StoreRef &newStore =
     getStateManager().StoreMgr->BindDeclWithNoInit(getStore(), VR);
   return makeWithStore(newStore);
 }
 
-const GRState *GRState::bindLoc(Loc LV, SVal V) const {
-  GRStateManager &Mgr = getStateManager();
-  const GRState *newState = makeWithStore(Mgr.StoreMgr->Bind(getStore(), 
+const ProgramState *ProgramState::bindLoc(Loc LV, SVal V) const {
+  ProgramStateManager &Mgr = getStateManager();
+  const ProgramState *newState = makeWithStore(Mgr.StoreMgr->Bind(getStore(), 
                                                              LV, V));
   const MemRegion *MR = LV.getAsRegion();
   if (MR && Mgr.getOwningEngine())
@@ -126,56 +121,53 @@ const GRState *GRState::bindLoc(Loc LV, SVal V) const {
   return newState;
 }
 
-const GRState *GRState::bindDefault(SVal loc, SVal V) const {
-  GRStateManager &Mgr = getStateManager();
+const ProgramState *ProgramState::bindDefault(SVal loc, SVal V) const {
+  ProgramStateManager &Mgr = getStateManager();
   const MemRegion *R = cast<loc::MemRegionVal>(loc).getRegion();
   const StoreRef &newStore = Mgr.StoreMgr->BindDefault(getStore(), R, V);
-  const GRState *new_state = makeWithStore(newStore);
+  const ProgramState *new_state = makeWithStore(newStore);
   return Mgr.getOwningEngine() ? 
            Mgr.getOwningEngine()->processRegionChange(new_state, R) : 
            new_state;
 }
 
-const GRState *GRState::invalidateRegions(const MemRegion * const *Begin,
-                                          const MemRegion * const *End,
-                                          const Expr *E, unsigned Count,
-                                          StoreManager::InvalidatedSymbols *IS,
-                                          bool invalidateGlobals) const {
+const ProgramState *
+ProgramState::invalidateRegions(ArrayRef<const MemRegion *> Regions,
+                                const Expr *E, unsigned Count,
+                                StoreManager::InvalidatedSymbols *IS,
+                                bool invalidateGlobals) const {
   if (!IS) {
     StoreManager::InvalidatedSymbols invalidated;
-    return invalidateRegionsImpl(Begin, End, E, Count,
-                             invalidated, invalidateGlobals);
+    return invalidateRegionsImpl(Regions, E, Count,
+                                 invalidated, invalidateGlobals);
   }
-  return invalidateRegionsImpl(Begin, End, E, Count, *IS, invalidateGlobals);
+  return invalidateRegionsImpl(Regions, E, Count, *IS, invalidateGlobals);
 }
 
-const GRState *
-GRState::invalidateRegionsImpl(const MemRegion * const *Begin,
-                               const MemRegion * const *End,
-                               const Expr *E, unsigned Count,
-                               StoreManager::InvalidatedSymbols &IS,
-                               bool invalidateGlobals) const {
-  GRStateManager &Mgr = getStateManager();
+const ProgramState *
+ProgramState::invalidateRegionsImpl(ArrayRef<const MemRegion *> Regions,
+                                    const Expr *E, unsigned Count,
+                                    StoreManager::InvalidatedSymbols &IS,
+                                    bool invalidateGlobals) const {
+  ProgramStateManager &Mgr = getStateManager();
   SubEngine* Eng = Mgr.getOwningEngine();
  
   if (Eng && Eng->wantsRegionChangeUpdate(this)) {
-    StoreManager::InvalidatedRegions Regions;
+    StoreManager::InvalidatedRegions Invalidated;
     const StoreRef &newStore
-      = Mgr.StoreMgr->invalidateRegions(getStore(), Begin, End, E, Count, IS,
-                                        invalidateGlobals, &Regions);
-    const GRState *newState = makeWithStore(newStore);
-    return Eng->processRegionChanges(newState, &IS,
-                                     &Regions.front(),
-                                     &Regions.back()+1);
+      = Mgr.StoreMgr->invalidateRegions(getStore(), Regions, E, Count, IS,
+                                        invalidateGlobals, &Invalidated);
+    const ProgramState *newState = makeWithStore(newStore);
+    return Eng->processRegionChanges(newState, &IS, Regions, Invalidated);
   }
 
   const StoreRef &newStore =
-    Mgr.StoreMgr->invalidateRegions(getStore(), Begin, End, E, Count, IS,
+    Mgr.StoreMgr->invalidateRegions(getStore(), Regions, E, Count, IS,
                                     invalidateGlobals, NULL);
   return makeWithStore(newStore);
 }
 
-const GRState *GRState::unbindLoc(Loc LV) const {
+const ProgramState *ProgramState::unbindLoc(Loc LV) const {
   assert(!isa<loc::MemRegionVal>(LV) && "Use invalidateRegion instead.");
 
   Store OldStore = getStore();
@@ -187,20 +179,20 @@ const GRState *GRState::unbindLoc(Loc LV) const {
   return makeWithStore(newStore);
 }
 
-const GRState *GRState::enterStackFrame(const StackFrameContext *frame) const {
+const ProgramState *ProgramState::enterStackFrame(const StackFrameContext *frame) const {
   const StoreRef &new_store =
     getStateManager().StoreMgr->enterStackFrame(this, frame);
   return makeWithStore(new_store);
 }
 
-SVal GRState::getSValAsScalarOrLoc(const MemRegion *R) const {
+SVal ProgramState::getSValAsScalarOrLoc(const MemRegion *R) const {
   // We only want to do fetches from regions that we can actually bind
   // values.  For example, SymbolicRegions of type 'id<...>' cannot
   // have direct bindings (but their can be bindings on their subregions).
   if (!R->isBoundable())
     return UnknownVal();
 
-  if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
+  if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(R)) {
     QualType T = TR->getValueType();
     if (Loc::isLocType(T) || T->isIntegerType())
       return getSVal(R);
@@ -209,7 +201,7 @@ SVal GRState::getSValAsScalarOrLoc(const MemRegion *R) const {
   return UnknownVal();
 }
 
-SVal GRState::getSVal(Loc location, QualType T) const {
+SVal ProgramState::getSVal(Loc location, QualType T) const {
   SVal V = getRawSVal(cast<Loc>(location), T);
 
   // If 'V' is a symbolic value that is *perfectly* constrained to
@@ -246,18 +238,18 @@ SVal GRState::getSVal(Loc location, QualType T) const {
   return V;
 }
 
-const GRState *GRState::BindExpr(const Stmt* S, SVal V, bool Invalidate) const{
+const ProgramState *ProgramState::BindExpr(const Stmt *S, SVal V, bool Invalidate) const{
   Environment NewEnv = getStateManager().EnvMgr.bindExpr(Env, S, V,
                                                          Invalidate);
   if (NewEnv == Env)
     return this;
 
-  GRState NewSt = *this;
+  ProgramState NewSt = *this;
   NewSt.Env = NewEnv;
   return getStateManager().getPersistentState(NewSt);
 }
 
-const GRState *GRState::bindExprAndLocation(const Stmt *S, SVal location,
+const ProgramState *ProgramState::bindExprAndLocation(const Stmt *S, SVal location,
                                             SVal V) const {
   Environment NewEnv =
     getStateManager().EnvMgr.bindExprAndLocation(Env, S, location, V);
@@ -265,12 +257,12 @@ const GRState *GRState::bindExprAndLocation(const Stmt *S, SVal location,
   if (NewEnv == Env)
     return this;
   
-  GRState NewSt = *this;
+  ProgramState NewSt = *this;
   NewSt.Env = NewEnv;
   return getStateManager().getPersistentState(NewSt);
 }
 
-const GRState *GRState::assumeInBound(DefinedOrUnknownSVal Idx,
+const ProgramState *ProgramState::assumeInBound(DefinedOrUnknownSVal Idx,
                                       DefinedOrUnknownSVal UpperBound,
                                       bool Assumption) const {
   if (Idx.isUnknown() || UpperBound.isUnknown())
@@ -279,7 +271,7 @@ const GRState *GRState::assumeInBound(DefinedOrUnknownSVal Idx,
   // Build an expression for 0 <= Idx < UpperBound.
   // This is the same as Idx + MIN < UpperBound + MIN, if overflow is allowed.
   // FIXME: This should probably be part of SValBuilder.
-  GRStateManager &SM = getStateManager();
+  ProgramStateManager &SM = getStateManager();
   SValBuilder &svalBuilder = SM.getSValBuilder();
   ASTContext &Ctx = svalBuilder.getContext();
 
@@ -315,8 +307,8 @@ const GRState *GRState::assumeInBound(DefinedOrUnknownSVal Idx,
   return CM.assume(this, cast<DefinedSVal>(inBound), Assumption);
 }
 
-const GRState* GRStateManager::getInitialState(const LocationContext *InitLoc) {
-  GRState State(this,
+const ProgramState *ProgramStateManager::getInitialState(const LocationContext *InitLoc) {
+  ProgramState State(this,
                 EnvMgr.getInitialEnvironment(),
                 StoreMgr->getInitialStore(InitLoc),
                 GDMFactory.getEmptyMap());
@@ -324,49 +316,57 @@ const GRState* GRStateManager::getInitialState(const LocationContext *InitLoc) {
   return getPersistentState(State);
 }
 
-void GRStateManager::recycleUnusedStates() {
-  for (std::vector<GRState*>::iterator i = recentlyAllocatedStates.begin(),
+void ProgramStateManager::recycleUnusedStates() {
+  for (std::vector<ProgramState*>::iterator i = recentlyAllocatedStates.begin(),
        e = recentlyAllocatedStates.end(); i != e; ++i) {
-    GRState *state = *i;
+    ProgramState *state = *i;
     if (state->referencedByExplodedNode())
       continue;
     StateSet.RemoveNode(state);
     freeStates.push_back(state);
-    state->~GRState();
+    state->~ProgramState();
   }
   recentlyAllocatedStates.clear();
 }
 
-const GRState* GRStateManager::getPersistentState(GRState& State) {
+const ProgramState *ProgramStateManager::getPersistentStateWithGDM(
+                                                     const ProgramState *FromState,
+                                                     const ProgramState *GDMState) {
+  ProgramState NewState = *FromState;
+  NewState.GDM = GDMState->GDM;
+  return getPersistentState(NewState);
+}
+
+const ProgramState *ProgramStateManager::getPersistentState(ProgramState &State) {
 
   llvm::FoldingSetNodeID ID;
   State.Profile(ID);
-  void* InsertPos;
+  void *InsertPos;
 
-  if (GRState* I = StateSet.FindNodeOrInsertPos(ID, InsertPos))
+  if (ProgramState *I = StateSet.FindNodeOrInsertPos(ID, InsertPos))
     return I;
 
-  GRState *newState = 0;
+  ProgramState *newState = 0;
   if (!freeStates.empty()) {
     newState = freeStates.back();
     freeStates.pop_back();    
   }
   else {
-    newState = (GRState*) Alloc.Allocate<GRState>();
+    newState = (ProgramState*) Alloc.Allocate<ProgramState>();
   }
-  new (newState) GRState(State);
+  new (newState) ProgramState(State);
   StateSet.InsertNode(newState, InsertPos);
   recentlyAllocatedStates.push_back(newState);
   return newState;
 }
 
-const GRState* GRState::makeWithStore(const StoreRef &store) const {
-  GRState NewSt = *this;
+const ProgramState *ProgramState::makeWithStore(const StoreRef &store) const {
+  ProgramState NewSt = *this;
   NewSt.setStore(store);
   return getStateManager().getPersistentState(NewSt);
 }
 
-void GRState::setStore(const StoreRef &newStore) {
+void ProgramState::setStore(const StoreRef &newStore) {
   Store newStoreStore = newStore.getStore();
   if (newStoreStore)
     stateMgr->getStoreManager().incrementReferenceCount(newStoreStore);
@@ -384,11 +384,11 @@ static bool IsEnvLoc(const Stmt *S) {
   return (bool) (((uintptr_t) S) & 0x1);
 }
 
-void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
-                    const char* sep) const {
+void ProgramState::print(raw_ostream &Out, CFG &C,
+                         const char *NL, const char *Sep) const {
   // Print the store.
-  GRStateManager &Mgr = getStateManager();
-  Mgr.getStoreManager().print(getStore(), Out, nl, sep);
+  ProgramStateManager &Mgr = getStateManager();
+  Mgr.getStoreManager().print(getStore(), Out, NL, Sep);
 
   // Print Subexpression bindings.
   bool isFirst = true;
@@ -399,10 +399,11 @@ void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
       continue;
 
     if (isFirst) {
-      Out << nl << nl << "Sub-Expressions:" << nl;
+      Out << NL << NL << "Sub-Expressions:" << NL;
       isFirst = false;
+    } else {
+      Out << NL;
     }
-    else { Out << nl; }
 
     Out << " (" << (void*) I.getKey() << ") ";
     LangOptions LO; // FIXME.
@@ -418,10 +419,11 @@ void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
       continue;
 
     if (isFirst) {
-      Out << nl << nl << "Block-level Expressions:" << nl;
+      Out << NL << NL << "Block-level Expressions:" << NL;
       isFirst = false;
+    } else {
+      Out << NL;
     }
-    else { Out << nl; }
 
     Out << " (" << (void*) I.getKey() << ") ";
     LangOptions LO; // FIXME.
@@ -437,10 +439,11 @@ void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
       continue;
     
     if (isFirst) {
-      Out << nl << nl << "Load/store locations:" << nl;
+      Out << NL << NL << "Load/store locations:" << NL;
       isFirst = false;
+    } else {
+      Out << NL;
     }
-    else { Out << nl; }
 
     const Stmt *S = (Stmt*) (((uintptr_t) I.getKey()) & ((uintptr_t) ~0x1));
     
@@ -450,20 +453,17 @@ void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
     Out << " : " << I.getData();
   }
 
-  Mgr.getConstraintManager().print(this, Out, nl, sep);
+  Mgr.getConstraintManager().print(this, Out, NL, Sep);
 
   // Print checker-specific data.
-  for (std::vector<Printer*>::iterator I = Mgr.Printers.begin(),
-                                       E = Mgr.Printers.end(); I != E; ++I) {
-    (*I)->Print(Out, this, nl, sep);
-  }
+  Mgr.getOwningEngine()->printState(Out, this, NL, Sep);
 }
 
-void GRState::printDOT(llvm::raw_ostream& Out, CFG &C) const {
+void ProgramState::printDOT(raw_ostream &Out, CFG &C) const {
   print(Out, C, "\\l", "\\|");
 }
 
-void GRState::printStdErr(CFG &C) const {
+void ProgramState::printStdErr(CFG &C) const {
   print(llvm::errs(), C);
 }
 
@@ -471,13 +471,13 @@ void GRState::printStdErr(CFG &C) const {
 // Generic Data Map.
 //===----------------------------------------------------------------------===//
 
-void* const* GRState::FindGDM(void* K) const {
+void *const* ProgramState::FindGDM(void *K) const {
   return GDM.lookup(K);
 }
 
 void*
-GRStateManager::FindGDMContext(void* K,
-                               void* (*CreateContext)(llvm::BumpPtrAllocator&),
+ProgramStateManager::FindGDMContext(void *K,
+                               void *(*CreateContext)(llvm::BumpPtrAllocator&),
                                void (*DeleteContext)(void*)) {
 
   std::pair<void*, void (*)(void*)>& p = GDMContexts[K];
@@ -489,58 +489,30 @@ GRStateManager::FindGDMContext(void* K,
   return p.first;
 }
 
-const GRState* GRStateManager::addGDM(const GRState* St, void* Key, void* Data){
-  GRState::GenericDataMap M1 = St->getGDM();
-  GRState::GenericDataMap M2 = GDMFactory.add(M1, Key, Data);
+const ProgramState *ProgramStateManager::addGDM(const ProgramState *St, void *Key, void *Data){
+  ProgramState::GenericDataMap M1 = St->getGDM();
+  ProgramState::GenericDataMap M2 = GDMFactory.add(M1, Key, Data);
 
   if (M1 == M2)
     return St;
 
-  GRState NewSt = *St;
+  ProgramState NewSt = *St;
   NewSt.GDM = M2;
   return getPersistentState(NewSt);
 }
 
-const GRState *GRStateManager::removeGDM(const GRState *state, void *Key) {
-  GRState::GenericDataMap OldM = state->getGDM();
-  GRState::GenericDataMap NewM = GDMFactory.remove(OldM, Key);
+const ProgramState *ProgramStateManager::removeGDM(const ProgramState *state, void *Key) {
+  ProgramState::GenericDataMap OldM = state->getGDM();
+  ProgramState::GenericDataMap NewM = GDMFactory.remove(OldM, Key);
 
   if (NewM == OldM)
     return state;
 
-  GRState NewState = *state;
+  ProgramState NewState = *state;
   NewState.GDM = NewM;
   return getPersistentState(NewState);
 }
 
-//===----------------------------------------------------------------------===//
-// Utility.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class ScanReachableSymbols : public SubRegionMap::Visitor  {
-  typedef llvm::DenseSet<const MemRegion*> VisitedRegionsTy;
-
-  VisitedRegionsTy visited;
-  const GRState *state;
-  SymbolVisitor &visitor;
-  llvm::OwningPtr<SubRegionMap> SRM;
-public:
-
-  ScanReachableSymbols(const GRState *st, SymbolVisitor& v)
-    : state(st), visitor(v) {}
-
-  bool scan(nonloc::CompoundVal val);
-  bool scan(SVal val);
-  bool scan(const MemRegion *R);
-
-  // From SubRegionMap::Visitor.
-  bool Visit(const MemRegion* Parent, const MemRegion* SubRegion) {
-    return scan(SubRegion);
-  }
-};
-}
-
 bool ScanReachableSymbols::scan(nonloc::CompoundVal val) {
   for (nonloc::CompoundVal::iterator I=val.begin(), E=val.end(); I!=E; ++I)
     if (!scan(*I))
@@ -549,6 +521,33 @@ bool ScanReachableSymbols::scan(nonloc::CompoundVal val) {
   return true;
 }
 
+bool ScanReachableSymbols::scan(const SymExpr *sym) {
+  unsigned &isVisited = visited[sym];
+  if (isVisited)
+    return true;
+  isVisited = 1;
+  
+  if (const SymbolData *sData = dyn_cast<SymbolData>(sym))
+    if (!visitor.VisitSymbol(sData))
+      return false;
+  
+  switch (sym->getKind()) {
+    case SymExpr::RegionValueKind:
+    case SymExpr::ConjuredKind:
+    case SymExpr::DerivedKind:
+    case SymExpr::ExtentKind:
+    case SymExpr::MetadataKind:
+      break;
+    case SymExpr::SymIntKind:
+      return scan(cast<SymIntExpr>(sym)->getLHS());
+    case SymExpr::SymSymKind: {
+      const SymSymExpr *x = cast<SymSymExpr>(sym);
+      return scan(x->getLHS()) && scan(x->getRHS());
+    }
+  }
+  return true;
+}
+
 bool ScanReachableSymbols::scan(SVal val) {
   if (loc::MemRegionVal *X = dyn_cast<loc::MemRegionVal>(&val))
     return scan(X->getRegion());
@@ -557,7 +556,10 @@ bool ScanReachableSymbols::scan(SVal val) {
     return scan(X->getLoc());
 
   if (SymbolRef Sym = val.getAsSymbol())
-    return visitor.VisitSymbol(Sym);
+    return scan(Sym);
+
+  if (const SymExpr *Sym = val.getAsSymbolicExpression())
+    return scan(Sym);
 
   if (nonloc::CompoundVal *X = dyn_cast<nonloc::CompoundVal>(&val))
     return scan(*X);
@@ -566,10 +568,13 @@ bool ScanReachableSymbols::scan(SVal val) {
 }
 
 bool ScanReachableSymbols::scan(const MemRegion *R) {
-  if (isa<MemSpaceRegion>(R) || visited.count(R))
+  if (isa<MemSpaceRegion>(R))
     return true;
-
-  visited.insert(R);
+  
+  unsigned &isVisited = visited[R];
+  if (isVisited)
+    return true;
+  isVisited = 1;
 
   // If this is a symbolic region, visit the symbol for the region.
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(R))
@@ -593,12 +598,12 @@ bool ScanReachableSymbols::scan(const MemRegion *R) {
   return SRM->iterSubRegions(R, *this);
 }
 
-bool GRState::scanReachableSymbols(SVal val, SymbolVisitor& visitor) const {
+bool ProgramState::scanReachableSymbols(SVal val, SymbolVisitor& visitor) const {
   ScanReachableSymbols S(this, visitor);
   return S.scan(val);
 }
 
-bool GRState::scanReachableSymbols(const SVal *I, const SVal *E,
+bool ProgramState::scanReachableSymbols(const SVal *I, const SVal *E,
                                    SymbolVisitor &visitor) const {
   ScanReachableSymbols S(this, visitor);
   for ( ; I != E; ++I) {
@@ -608,7 +613,7 @@ bool GRState::scanReachableSymbols(const SVal *I, const SVal *E,
   return true;
 }
 
-bool GRState::scanReachableSymbols(const MemRegion * const *I,
+bool ProgramState::scanReachableSymbols(const MemRegion * const *I,
                                    const MemRegion * const *E,
                                    SymbolVisitor &visitor) const {
   ScanReachableSymbols S(this, visitor);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index 389fff5ed593..9337788535ab 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -8,14 +8,13 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines RangeConstraintManager, a class that tracks simple
-//  equality and inequality constraints on symbolic values of GRState.
+//  equality and inequality constraints on symbolic values of ProgramState.
 //
 //===----------------------------------------------------------------------===//
 
 #include "SimpleConstraintManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableSet.h"
@@ -170,7 +169,7 @@ class RangeSet {
     return newRanges;
   }
 
-  void print(llvm::raw_ostream &os) const {
+  void print(raw_ostream &os) const {
     bool isFirst = true;
     os << "{ ";
     for (iterator i = begin(), e = end(); i != e; ++i) {
@@ -196,55 +195,55 @@ typedef llvm::ImmutableMap<SymbolRef,RangeSet> ConstraintRangeTy;
 namespace clang {
 namespace ento {
 template<>
-struct GRStateTrait<ConstraintRange>
-  : public GRStatePartialTrait<ConstraintRangeTy> {
-  static inline void* GDMIndex() { return &ConstraintRangeIndex; }
+struct ProgramStateTrait<ConstraintRange>
+  : public ProgramStatePartialTrait<ConstraintRangeTy> {
+  static inline void *GDMIndex() { return &ConstraintRangeIndex; }
 };
 }
 }
 
 namespace {
 class RangeConstraintManager : public SimpleConstraintManager{
-  RangeSet GetRange(const GRState *state, SymbolRef sym);
+  RangeSet GetRange(const ProgramState *state, SymbolRef sym);
 public:
   RangeConstraintManager(SubEngine &subengine)
     : SimpleConstraintManager(subengine) {}
 
-  const GRState *assumeSymNE(const GRState* state, SymbolRef sym,
+  const ProgramState *assumeSymNE(const ProgramState *state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymEQ(const GRState* state, SymbolRef sym,
+  const ProgramState *assumeSymEQ(const ProgramState *state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymLT(const GRState* state, SymbolRef sym,
+  const ProgramState *assumeSymLT(const ProgramState *state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymGT(const GRState* state, SymbolRef sym,
+  const ProgramState *assumeSymGT(const ProgramState *state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymGE(const GRState* state, SymbolRef sym,
+  const ProgramState *assumeSymGE(const ProgramState *state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState *assumeSymLE(const GRState* state, SymbolRef sym,
+  const ProgramState *assumeSymLE(const ProgramState *state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const llvm::APSInt* getSymVal(const GRState* St, SymbolRef sym) const;
+  const llvm::APSInt* getSymVal(const ProgramState *St, SymbolRef sym) const;
 
   // FIXME: Refactor into SimpleConstraintManager?
-  bool isEqual(const GRState* St, SymbolRef sym, const llvm::APSInt& V) const {
+  bool isEqual(const ProgramState *St, SymbolRef sym, const llvm::APSInt& V) const {
     const llvm::APSInt *i = getSymVal(St, sym);
     return i ? *i == V : false;
   }
 
-  const GRState* removeDeadBindings(const GRState* St, SymbolReaper& SymReaper);
+  const ProgramState *removeDeadBindings(const ProgramState *St, SymbolReaper& SymReaper);
 
-  void print(const GRState* St, llvm::raw_ostream& Out,
+  void print(const ProgramState *St, raw_ostream &Out,
              const char* nl, const char *sep);
 
 private:
@@ -253,12 +252,12 @@ class RangeConstraintManager : public SimpleConstraintManager{
 
 } // end anonymous namespace
 
-ConstraintManager* ento::CreateRangeConstraintManager(GRStateManager&,
+ConstraintManager* ento::CreateRangeConstraintManager(ProgramStateManager&,
                                                     SubEngine &subeng) {
   return new RangeConstraintManager(subeng);
 }
 
-const llvm::APSInt* RangeConstraintManager::getSymVal(const GRState* St,
+const llvm::APSInt* RangeConstraintManager::getSymVal(const ProgramState *St,
                                                       SymbolRef sym) const {
   const ConstraintRangeTy::data_type *T = St->get<ConstraintRange>(sym);
   return T ? T->getConcreteValue() : NULL;
@@ -266,8 +265,8 @@ const llvm::APSInt* RangeConstraintManager::getSymVal(const GRState* St,
 
 /// Scan all symbols referenced by the constraints. If the symbol is not alive
 /// as marked in LSymbols, mark it as dead in DSymbols.
-const GRState*
-RangeConstraintManager::removeDeadBindings(const GRState* state,
+const ProgramState*
+RangeConstraintManager::removeDeadBindings(const ProgramState *state,
                                            SymbolReaper& SymReaper) {
 
   ConstraintRangeTy CR = state->get<ConstraintRange>();
@@ -283,7 +282,7 @@ RangeConstraintManager::removeDeadBindings(const GRState* state,
 }
 
 RangeSet
-RangeConstraintManager::GetRange(const GRState *state, SymbolRef sym) {
+RangeConstraintManager::GetRange(const ProgramState *state, SymbolRef sym) {
   if (ConstraintRangeTy::data_type* V = state->get<ConstraintRange>(sym))
     return *V;
 
@@ -306,8 +305,8 @@ RangeConstraintManager::GetRange(const GRState *state, SymbolRef sym) {
 // As an example, the range [UINT_MAX-1, 3) contains five values: UINT_MAX-1,
 // UINT_MAX, 0, 1, and 2.
 
-const GRState*
-RangeConstraintManager::assumeSymNE(const GRState* state, SymbolRef sym,
+const ProgramState*
+RangeConstraintManager::assumeSymNE(const ProgramState *state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -323,8 +322,8 @@ RangeConstraintManager::assumeSymNE(const GRState* state, SymbolRef sym,
   return New.isEmpty() ? NULL : state->set<ConstraintRange>(sym, New);
 }
 
-const GRState*
-RangeConstraintManager::assumeSymEQ(const GRState* state, SymbolRef sym,
+const ProgramState*
+RangeConstraintManager::assumeSymEQ(const ProgramState *state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   // [Int-Adjustment, Int-Adjustment]
@@ -334,8 +333,8 @@ RangeConstraintManager::assumeSymEQ(const GRState* state, SymbolRef sym,
   return New.isEmpty() ? NULL : state->set<ConstraintRange>(sym, New);
 }
 
-const GRState*
-RangeConstraintManager::assumeSymLT(const GRState* state, SymbolRef sym,
+const ProgramState*
+RangeConstraintManager::assumeSymLT(const ProgramState *state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -355,8 +354,8 @@ RangeConstraintManager::assumeSymLT(const GRState* state, SymbolRef sym,
   return New.isEmpty() ? NULL : state->set<ConstraintRange>(sym, New);
 }
 
-const GRState*
-RangeConstraintManager::assumeSymGT(const GRState* state, SymbolRef sym,
+const ProgramState*
+RangeConstraintManager::assumeSymGT(const ProgramState *state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -376,8 +375,8 @@ RangeConstraintManager::assumeSymGT(const GRState* state, SymbolRef sym,
   return New.isEmpty() ? NULL : state->set<ConstraintRange>(sym, New);
 }
 
-const GRState*
-RangeConstraintManager::assumeSymGE(const GRState* state, SymbolRef sym,
+const ProgramState*
+RangeConstraintManager::assumeSymGE(const ProgramState *state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -398,8 +397,8 @@ RangeConstraintManager::assumeSymGE(const GRState* state, SymbolRef sym,
   return New.isEmpty() ? NULL : state->set<ConstraintRange>(sym, New);
 }
 
-const GRState*
-RangeConstraintManager::assumeSymLE(const GRState* state, SymbolRef sym,
+const ProgramState*
+RangeConstraintManager::assumeSymLE(const ProgramState *state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -424,7 +423,7 @@ RangeConstraintManager::assumeSymLE(const GRState* state, SymbolRef sym,
 // Pretty-printing.
 //===------------------------------------------------------------------------===/
 
-void RangeConstraintManager::print(const GRState* St, llvm::raw_ostream& Out,
+void RangeConstraintManager::print(const ProgramState *St, raw_ostream &Out,
                                    const char* nl, const char *sep) {
 
   ConstraintRangeTy Ranges = St->get<ConstraintRange>();
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RegionStore.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
index 23dd6416a8f1..4b76cf1a3deb 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -20,8 +20,8 @@
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "llvm/ADT/ImmutableList.h"
 #include "llvm/ADT/ImmutableMap.h"
@@ -94,7 +94,7 @@ BindingKey BindingKey::Make(const MemRegion *R, Kind k) {
 
 namespace llvm {
   static inline
-  llvm::raw_ostream& operator<<(llvm::raw_ostream& os, BindingKey K) {
+  raw_ostream &operator<<(raw_ostream &os, BindingKey K) {
     os << '(' << K.getRegion() << ',' << K.getOffset()
        << ',' << (K.isDirect() ? "direct" : "default")
        << ')';
@@ -157,7 +157,7 @@ class RegionStoreSubRegionMap : public SubRegionMap {
     return false;
   }
 
-  void process(llvm::SmallVectorImpl<const SubRegion*> &WL, const SubRegion *R);
+  void process(SmallVectorImpl<const SubRegion*> &WL, const SubRegion *R);
 
   ~RegionStoreSubRegionMap() {}
 
@@ -183,7 +183,7 @@ class RegionStoreSubRegionMap : public SubRegionMap {
 };
 
 void
-RegionStoreSubRegionMap::process(llvm::SmallVectorImpl<const SubRegion*> &WL,
+RegionStoreSubRegionMap::process(SmallVectorImpl<const SubRegion*> &WL,
                                  const SubRegion *R) {
   const MemRegion *superR = R->getSuperRegion();
   if (add(superR, R))
@@ -196,7 +196,7 @@ class RegionStoreManager : public StoreManager {
   RegionBindings::Factory RBFactory;
 
 public:
-  RegionStoreManager(GRStateManager& mgr, const RegionStoreFeatures &f)
+  RegionStoreManager(ProgramStateManager& mgr, const RegionStoreFeatures &f)
     : StoreManager(mgr),
       Features(f),
       RBFactory(mgr.getAllocator()) {}
@@ -236,13 +236,11 @@ class RegionStoreManager : public StoreManager {
   // Binding values to regions.
   //===-------------------------------------------------------------------===//
 
-  StoreRef invalidateRegions(Store store,
-                             const MemRegion * const *Begin,
-                             const MemRegion * const *End,
+  StoreRef invalidateRegions(Store store, ArrayRef<const MemRegion *> Regions,
                              const Expr *E, unsigned Count,
                              InvalidatedSymbols &IS,
                              bool invalidateGlobals,
-                             InvalidatedRegions *Regions);
+                             InvalidatedRegions *Invalidated);
 
 public:   // Made public for helper classes.
 
@@ -278,7 +276,7 @@ class RegionStoreManager : public StoreManager {
     return StoreRef(addBinding(B, R, BindingKey::Default, V).getRootWithoutRetain(), *this);
   }
 
-  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr* CL,
+  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr *CL,
                                const LocationContext *LC, SVal V);
 
   StoreRef BindDecl(Store store, const VarRegion *VR, SVal InitVal);
@@ -288,9 +286,9 @@ class RegionStoreManager : public StoreManager {
   }
 
   /// BindStruct - Bind a compound value to a structure.
-  StoreRef BindStruct(Store store, const TypedRegion* R, SVal V);
+  StoreRef BindStruct(Store store, const TypedValueRegion* R, SVal V);
 
-  StoreRef BindArray(Store store, const TypedRegion* R, SVal V);
+  StoreRef BindArray(Store store, const TypedValueRegion* R, SVal V);
 
   /// KillStruct - Set the entire struct to unknown.
   StoreRef KillStruct(Store store, const TypedRegion* R, SVal DefaultVal);
@@ -307,6 +305,8 @@ class RegionStoreManager : public StoreManager {
   void decrementReferenceCount(Store store) {
     GetRegionBindings(store).manualRelease();
   }
+  
+  bool includedInBindings(Store store, const MemRegion *region) const;
 
   //===------------------------------------------------------------------===//
   // Loading values from regions.
@@ -333,9 +333,9 @@ class RegionStoreManager : public StoreManager {
 
   SVal RetrieveVar(Store store, const VarRegion *R);
 
-  SVal RetrieveLazySymbol(const TypedRegion *R);
+  SVal RetrieveLazySymbol(const TypedValueRegion *R);
 
-  SVal RetrieveFieldOrElementCommon(Store store, const TypedRegion *R,
+  SVal RetrieveFieldOrElementCommon(Store store, const TypedValueRegion *R,
                                     QualType Ty, const MemRegion *superR);
   
   SVal RetrieveLazyBinding(const MemRegion *lazyBindingRegion,
@@ -346,15 +346,16 @@ class RegionStoreManager : public StoreManager {
   /// struct s x, y;
   /// x = y;
   /// y's value is retrieved by this method.
-  SVal RetrieveStruct(Store store, const TypedRegion* R);
+  SVal RetrieveStruct(Store store, const TypedValueRegion* R);
 
-  SVal RetrieveArray(Store store, const TypedRegion* R);
+  SVal RetrieveArray(Store store, const TypedValueRegion* R);
 
   /// Used to lazily generate derived symbols for bindings that are defined
   ///  implicitly by default bindings in a super region.
   Optional<SVal> RetrieveDerivedDefaultValue(RegionBindings B,
                                              const MemRegion *superR,
-                                             const TypedRegion *R, QualType Ty);
+                                             const TypedValueRegion *R, 
+                                             QualType Ty);
 
   /// Get the state and region whose binding this region R corresponds to.
   std::pair<Store, const MemRegion*>
@@ -371,17 +372,17 @@ class RegionStoreManager : public StoreManager {
   /// removeDeadBindings - Scans the RegionStore of 'state' for dead values.
   ///  It returns a new Store with these values removed.
   StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
-                           SymbolReaper& SymReaper,
-                          llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
+                              SymbolReaper& SymReaper);
 
-  StoreRef enterStackFrame(const GRState *state, const StackFrameContext *frame);
+  StoreRef enterStackFrame(const ProgramState *state,
+                           const StackFrameContext *frame);
 
   //===------------------------------------------------------------------===//
   // Region "extents".
   //===------------------------------------------------------------------===//
 
   // FIXME: This method will soon be eliminated; see the note in Store.h.
-  DefinedOrUnknownSVal getSizeInElements(const GRState *state,
+  DefinedOrUnknownSVal getSizeInElements(const ProgramState *state,
                                          const MemRegion* R, QualType EleTy);
 
   //===------------------------------------------------------------------===//
@@ -392,7 +393,7 @@ class RegionStoreManager : public StoreManager {
     return RegionBindings(static_cast<const RegionBindings::TreeTy*>(store));
   }
 
-  void print(Store store, llvm::raw_ostream& Out, const char* nl,
+  void print(Store store, raw_ostream &Out, const char* nl,
              const char *sep);
 
   void iterBindings(Store store, BindingsHandler& f) {
@@ -416,12 +417,12 @@ class RegionStoreManager : public StoreManager {
 // RegionStore creation.
 //===----------------------------------------------------------------------===//
 
-StoreManager *ento::CreateRegionStoreManager(GRStateManager& StMgr) {
+StoreManager *ento::CreateRegionStoreManager(ProgramStateManager& StMgr) {
   RegionStoreFeatures F = maximal_features_tag();
   return new RegionStoreManager(StMgr, F);
 }
 
-StoreManager *ento::CreateFieldsOnlyRegionStoreManager(GRStateManager &StMgr) {
+StoreManager *ento::CreateFieldsOnlyRegionStoreManager(ProgramStateManager &StMgr) {
   RegionStoreFeatures F = minimal_features_tag();
   F.enableFields(true);
   return new RegionStoreManager(StMgr, F);
@@ -433,7 +434,7 @@ RegionStoreManager::getRegionStoreSubRegionMap(Store store) {
   RegionBindings B = GetRegionBindings(store);
   RegionStoreSubRegionMap *M = new RegionStoreSubRegionMap();
 
-  llvm::SmallVector<const SubRegion*, 10> WL;
+  SmallVector<const SubRegion*, 10> WL;
 
   for (RegionBindings::iterator I=B.begin(), E=B.end(); I!=E; ++I)
     if (const SubRegion *R = dyn_cast<SubRegion>(I.getKey().getRegion()))
@@ -461,7 +462,7 @@ class ClusterAnalysis  {
   typedef BumpVector<BindingKey> RegionCluster;
   typedef llvm::DenseMap<const MemRegion *, RegionCluster *> ClusterMap;
   llvm::DenseMap<const RegionCluster*, unsigned> Visited;
-  typedef llvm::SmallVector<std::pair<const MemRegion *, RegionCluster*>, 10>
+  typedef SmallVector<std::pair<const MemRegion *, RegionCluster*>, 10>
     WorkList;
 
   BumpVectorContext BVC;
@@ -477,7 +478,7 @@ class ClusterAnalysis  {
   const bool includeGlobals;
 
 public:
-  ClusterAnalysis(RegionStoreManager &rm, GRStateManager &StateMgr,
+  ClusterAnalysis(RegionStoreManager &rm, ProgramStateManager &StateMgr,
                   RegionBindings b, const bool includeGlobals)
     : RM(rm), Ctx(StateMgr.getContext()),
       svalBuilder(StateMgr.getSValBuilder()),
@@ -590,7 +591,7 @@ class invalidateRegionsWorker : public ClusterAnalysis<invalidateRegionsWorker>
   StoreManager::InvalidatedRegions *Regions;
 public:
   invalidateRegionsWorker(RegionStoreManager &rm,
-                          GRStateManager &stateMgr,
+                          ProgramStateManager &stateMgr,
                           RegionBindings b,
                           const Expr *ex, unsigned count,
                           StoreManager::InvalidatedSymbols &is,
@@ -681,7 +682,7 @@ void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   if (!baseR->isBoundable())
     return;
 
-  const TypedRegion *TR = cast<TypedRegion>(baseR);
+  const TypedValueRegion *TR = cast<TypedValueRegion>(baseR);
   QualType T = TR->getValueType();
 
     // Invalidate the binding.
@@ -718,21 +719,21 @@ void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
 }
 
 StoreRef RegionStoreManager::invalidateRegions(Store store,
-                                               const MemRegion * const *I,
-                                               const MemRegion * const *E,
+                                            ArrayRef<const MemRegion *> Regions,
                                                const Expr *Ex, unsigned Count,
                                                InvalidatedSymbols &IS,
                                                bool invalidateGlobals,
-                                               InvalidatedRegions *Regions) {
+                                              InvalidatedRegions *Invalidated) {
   invalidateRegionsWorker W(*this, StateMgr,
                             RegionStoreManager::GetRegionBindings(store),
-                            Ex, Count, IS, Regions, invalidateGlobals);
+                            Ex, Count, IS, Invalidated, invalidateGlobals);
 
   // Scan the bindings and generate the clusters.
   W.GenerateClusters();
 
-  // Add I .. E to the worklist.
-  for ( ; I != E; ++I)
+  // Add the regions to the worklist.
+  for (ArrayRef<const MemRegion *>::iterator
+       I = Regions.begin(), E = Regions.end(); I != E; ++I)
     W.AddToWorkList(*I);
 
   W.RunWorkList();
@@ -752,8 +753,8 @@ StoreRef RegionStoreManager::invalidateRegions(Store store,
 
     // Even if there are no bindings in the global scope, we still need to
     // record that we touched it.
-    if (Regions)
-      Regions->push_back(GS);
+    if (Invalidated)
+      Invalidated->push_back(GS);
   }
 
   return StoreRef(B.getRootWithoutRetain(), *this);
@@ -763,7 +764,7 @@ StoreRef RegionStoreManager::invalidateRegions(Store store,
 // Extents for regions.
 //===----------------------------------------------------------------------===//
 
-DefinedOrUnknownSVal RegionStoreManager::getSizeInElements(const GRState *state,
+DefinedOrUnknownSVal RegionStoreManager::getSizeInElements(const ProgramState *state,
                                                            const MemRegion *R,
                                                            QualType EleTy) {
   SVal Size = cast<SubRegion>(R)->getExtent(svalBuilder);
@@ -803,7 +804,7 @@ SVal RegionStoreManager::ArrayToPointer(Loc Array) {
     return UnknownVal();
 
   const MemRegion* R = cast<loc::MemRegionVal>(&Array)->getRegion();
-  const TypedRegion* ArrayR = dyn_cast<TypedRegion>(R);
+  const TypedValueRegion* ArrayR = dyn_cast<TypedValueRegion>(R);
 
   if (!ArrayR)
     return UnknownVal();
@@ -852,7 +853,7 @@ Optional<SVal> RegionStoreManager::getDirectBinding(RegionBindings B,
 Optional<SVal> RegionStoreManager::getDefaultBinding(RegionBindings B,
                                                      const MemRegion *R) {
   if (R->isBoundable())
-    if (const TypedRegion *TR = dyn_cast<TypedRegion>(R))
+    if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(R))
       if (TR->getValueType()->isUnionType())
         return UnknownVal();
 
@@ -890,13 +891,12 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   }
 
   if (isa<CodeTextRegion>(MR)) {
-    assert(0 && "Why load from a code text region?");
-    return UnknownVal();
+    llvm_unreachable("Why load from a code text region?");
   }
 
   // FIXME: Perhaps this method should just take a 'const MemRegion*' argument
   //  instead of 'Loc', and have the other Loc cases handled at a higher level.
-  const TypedRegion *R = cast<TypedRegion>(MR);
+  const TypedValueRegion *R = cast<TypedValueRegion>(MR);
   QualType RTy = R->getValueType();
 
   // FIXME: We should eventually handle funny addressing.  e.g.:
@@ -1042,6 +1042,10 @@ SVal RegionStoreManager::RetrieveElement(Store store,
     SVal Idx = R->getIndex();
     if (nonloc::ConcreteInt *CI = dyn_cast<nonloc::ConcreteInt>(&Idx)) {
       int64_t i = CI->getValue().getSExtValue();
+      // Abort on string underrun.  This can be possible by arbitrary
+      // clients of RetrieveElement().
+      if (i < 0)
+        return UndefinedVal();
       int64_t byteLength = Str->getByteLength();
       // Technically, only i == byteLength is guaranteed to be null.
       // However, such overflows should be caught before reaching this point;
@@ -1068,7 +1072,8 @@ SVal RegionStoreManager::RetrieveElement(Store store,
   if (!O.getRegion())
     return UnknownVal();
   
-  if (const TypedRegion *baseR = dyn_cast_or_null<TypedRegion>(O.getRegion())) {
+  if (const TypedValueRegion *baseR = 
+        dyn_cast_or_null<TypedValueRegion>(O.getRegion())) {
     QualType baseT = baseR->getValueType();
     if (baseT->isScalarType()) {
       QualType elemT = R->getElementType();
@@ -1106,7 +1111,7 @@ SVal RegionStoreManager::RetrieveField(Store store,
 Optional<SVal>
 RegionStoreManager::RetrieveDerivedDefaultValue(RegionBindings B,
                                                 const MemRegion *superR,
-                                                const TypedRegion *R,
+                                                const TypedValueRegion *R,
                                                 QualType Ty) {
 
   if (const Optional<SVal> &D = getDefaultBinding(B, superR)) {
@@ -1124,7 +1129,7 @@ RegionStoreManager::RetrieveDerivedDefaultValue(RegionBindings B,
     if (isa<nonloc::LazyCompoundVal>(val))
       return Optional<SVal>();
 
-    assert(0 && "Unknown default value");
+    llvm_unreachable("Unknown default value");
   }
 
   return Optional<SVal>();
@@ -1140,7 +1145,7 @@ SVal RegionStoreManager::RetrieveLazyBinding(const MemRegion *lazyBindingRegion,
 }
                                         
 SVal RegionStoreManager::RetrieveFieldOrElementCommon(Store store,
-                                                      const TypedRegion *R,
+                                                      const TypedValueRegion *R,
                                                       QualType Ty,
                                                       const MemRegion *superR) {
 
@@ -1175,7 +1180,8 @@ SVal RegionStoreManager::RetrieveFieldOrElementCommon(Store store,
     if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
       // Currently we don't reason specially about Clang-style vectors.  Check
       // if superR is a vector and if so return Unknown.
-      if (const TypedRegion *typedSuperR = dyn_cast<TypedRegion>(superR)) {
+      if (const TypedValueRegion *typedSuperR = 
+            dyn_cast<TypedValueRegion>(superR)) {
         if (typedSuperR->getValueType()->isVectorType())
           return UnknownVal();
       }
@@ -1264,22 +1270,41 @@ SVal RegionStoreManager::RetrieveVar(Store store, const VarRegion *R) {
   return UndefinedVal();
 }
 
-SVal RegionStoreManager::RetrieveLazySymbol(const TypedRegion *R) {
+SVal RegionStoreManager::RetrieveLazySymbol(const TypedValueRegion *R) {
   // All other values are symbolic.
   return svalBuilder.getRegionValueSymbolVal(R);
 }
 
-SVal RegionStoreManager::RetrieveStruct(Store store, const TypedRegion* R) {
+SVal RegionStoreManager::RetrieveStruct(Store store, 
+                                        const TypedValueRegion* R) {
   QualType T = R->getValueType();
   assert(T->isStructureOrClassType());
   return svalBuilder.makeLazyCompoundVal(StoreRef(store, *this), R);
 }
 
-SVal RegionStoreManager::RetrieveArray(Store store, const TypedRegion * R) {
+SVal RegionStoreManager::RetrieveArray(Store store, 
+                                       const TypedValueRegion * R) {
   assert(Ctx.getAsConstantArrayType(R->getValueType()));
   return svalBuilder.makeLazyCompoundVal(StoreRef(store, *this), R);
 }
 
+bool RegionStoreManager::includedInBindings(Store store,
+                                            const MemRegion *region) const {
+  RegionBindings B = GetRegionBindings(store);
+  region = region->getBaseRegion();
+  
+  for (RegionBindings::iterator it = B.begin(), ei = B.end(); it != ei; ++it) {
+    const BindingKey &K = it.getKey();
+    if (region == K.getRegion())
+      return true;
+    const SVal &D = it.getData();
+    if (const MemRegion *r = D.getAsRegion())
+      if (r == region)
+        return true;
+  }
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // Binding values to regions.
 //===----------------------------------------------------------------------===//
@@ -1302,14 +1327,14 @@ StoreRef RegionStoreManager::Bind(Store store, Loc L, SVal V) {
   const MemRegion *R = cast<loc::MemRegionVal>(L).getRegion();
 
   // Check if the region is a struct region.
-  if (const TypedRegion* TR = dyn_cast<TypedRegion>(R))
+  if (const TypedValueRegion* TR = dyn_cast<TypedValueRegion>(R))
     if (TR->getValueType()->isStructureOrClassType())
       return BindStruct(store, TR, V);
 
   if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
     if (ER->getIndex().isZeroConstant()) {
-      if (const TypedRegion *superR =
-            dyn_cast<TypedRegion>(ER->getSuperRegion())) {
+      if (const TypedValueRegion *superR =
+            dyn_cast<TypedValueRegion>(ER->getSuperRegion())) {
         QualType superTy = superR->getValueType();
         // For now, just invalidate the fields of the struct/union/class.
         // This is for test rdar_test_7185607 in misc-ps-region-store.m.
@@ -1389,7 +1414,7 @@ StoreRef RegionStoreManager::setImplicitDefaultValue(Store store,
                              V).getRootWithoutRetain(), *this);
 }
 
-StoreRef RegionStoreManager::BindArray(Store store, const TypedRegion* R,
+StoreRef RegionStoreManager::BindArray(Store store, const TypedValueRegion* R,
                                        SVal Init) {
 
   const ArrayType *AT =cast<ArrayType>(Ctx.getCanonicalType(R->getValueType()));
@@ -1448,7 +1473,7 @@ StoreRef RegionStoreManager::BindArray(Store store, const TypedRegion* R,
   return newStore;
 }
 
-StoreRef RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
+StoreRef RegionStoreManager::BindStruct(Store store, const TypedValueRegion* R,
                                         SVal V) {
 
   if (!Features.supportsFields())
@@ -1458,9 +1483,9 @@ StoreRef RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
   assert(T->isStructureOrClassType());
 
   const RecordType* RT = T->getAs<RecordType>();
-  RecordDecl* RD = RT->getDecl();
+  RecordDecl *RD = RT->getDecl();
 
-  if (!RD->isDefinition())
+  if (!RD->isCompleteDefinition())
     return StoreRef(store, *this);
 
   // Handle lazy compound values.
@@ -1611,12 +1636,12 @@ RegionBindings RegionStoreManager::removeBinding(RegionBindings B,
 namespace {
 class removeDeadBindingsWorker :
   public ClusterAnalysis<removeDeadBindingsWorker> {
-  llvm::SmallVector<const SymbolicRegion*, 12> Postponed;
+  SmallVector<const SymbolicRegion*, 12> Postponed;
   SymbolReaper &SymReaper;
   const StackFrameContext *CurrentLCtx;
 
 public:
-  removeDeadBindingsWorker(RegionStoreManager &rm, GRStateManager &stateMgr,
+  removeDeadBindingsWorker(RegionStoreManager &rm, ProgramStateManager &stateMgr,
                            RegionBindings b, SymbolReaper &symReaper,
                            const StackFrameContext *LCtx)
     : ClusterAnalysis<removeDeadBindingsWorker>(rm, stateMgr, b,
@@ -1736,7 +1761,7 @@ bool removeDeadBindingsWorker::UpdatePostponed() {
   // having done a scan.
   bool changed = false;
 
-  for (llvm::SmallVectorImpl<const SymbolicRegion*>::iterator
+  for (SmallVectorImpl<const SymbolicRegion*>::iterator
         I = Postponed.begin(), E = Postponed.end() ; I != E ; ++I) {
     if (const SymbolicRegion *SR = cast_or_null<SymbolicRegion>(*I)) {
       if (SymReaper.isLive(SR->getSymbol())) {
@@ -1751,17 +1776,16 @@ bool removeDeadBindingsWorker::UpdatePostponed() {
 
 StoreRef RegionStoreManager::removeDeadBindings(Store store,
                                                 const StackFrameContext *LCtx,
-                                                SymbolReaper& SymReaper,
-                           llvm::SmallVectorImpl<const MemRegion*>& RegionRoots)
-{
+                                                SymbolReaper& SymReaper) {
   RegionBindings B = GetRegionBindings(store);
   removeDeadBindingsWorker W(*this, StateMgr, B, SymReaper, LCtx);
   W.GenerateClusters();
 
   // Enqueue the region roots onto the worklist.
-  for (llvm::SmallVectorImpl<const MemRegion*>::iterator I=RegionRoots.begin(),
-       E=RegionRoots.end(); I!=E; ++I)
+  for (SymbolReaper::region_iterator I = SymReaper.region_begin(),
+       E = SymReaper.region_end(); I != E; ++I) {
     W.AddToWorkList(*I);
+  }
 
   do W.RunWorkList(); while (W.UpdatePostponed());
 
@@ -1792,7 +1816,7 @@ StoreRef RegionStoreManager::removeDeadBindings(Store store,
 }
 
 
-StoreRef RegionStoreManager::enterStackFrame(const GRState *state,
+StoreRef RegionStoreManager::enterStackFrame(const ProgramState *state,
                                              const StackFrameContext *frame) {
   FunctionDecl const *FD = cast<FunctionDecl>(frame->getDecl());
   FunctionDecl::param_const_iterator PI = FD->param_begin(), 
@@ -1831,7 +1855,7 @@ StoreRef RegionStoreManager::enterStackFrame(const GRState *state,
 // Utility methods.
 //===----------------------------------------------------------------------===//
 
-void RegionStoreManager::print(Store store, llvm::raw_ostream& OS,
+void RegionStoreManager::print(Store store, raw_ostream &OS,
                                const char* nl, const char *sep) {
   RegionBindings B = GetRegionBindings(store);
   OS << "Store (direct and default bindings):" << nl;
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
index 71f2b4abb9c3..ebf7ae2fd4b1 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
@@ -15,7 +15,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h"
 
 using namespace clang;
@@ -70,7 +70,7 @@ SVal SValBuilder::convertToArrayIndex(SVal val) {
 }
 
 DefinedOrUnknownSVal 
-SValBuilder::getRegionValueSymbolVal(const TypedRegion* region) {
+SValBuilder::getRegionValueSymbolVal(const TypedValueRegion* region) {
   QualType T = region->getValueType();
 
   if (!SymbolManager::canSymbolicate(T))
@@ -133,7 +133,7 @@ DefinedSVal SValBuilder::getMetadataSymbolVal(const void *symbolTag,
 
 DefinedOrUnknownSVal
 SValBuilder::getDerivedRegionValueSymbolVal(SymbolRef parentSymbol,
-                                             const TypedRegion *region) {
+                                             const TypedValueRegion *region) {
   QualType T = region->getValueType();
 
   if (!SymbolManager::canSymbolicate(T))
@@ -147,7 +147,7 @@ SValBuilder::getDerivedRegionValueSymbolVal(SymbolRef parentSymbol,
   return nonloc::SymbolVal(sym);
 }
 
-DefinedSVal SValBuilder::getFunctionPointer(const FunctionDecl* func) {
+DefinedSVal SValBuilder::getFunctionPointer(const FunctionDecl *func) {
   return loc::MemRegionVal(MemMgr.getFunctionTextRegion(func));
 }
 
@@ -162,7 +162,7 @@ DefinedSVal SValBuilder::getBlockPointer(const BlockDecl *block,
 
 //===----------------------------------------------------------------------===//
 
-SVal SValBuilder::evalBinOp(const GRState *state, BinaryOperator::Opcode op,
+SVal SValBuilder::evalBinOp(const ProgramState *state, BinaryOperator::Opcode op,
                             SVal lhs, SVal rhs, QualType type) {
 
   if (lhs.isUndef() || rhs.isUndef())
@@ -190,7 +190,7 @@ SVal SValBuilder::evalBinOp(const GRState *state, BinaryOperator::Opcode op,
   return evalBinOpNN(state, op, cast<NonLoc>(lhs), cast<NonLoc>(rhs), type);
 }
 
-DefinedOrUnknownSVal SValBuilder::evalEQ(const GRState *state,
+DefinedOrUnknownSVal SValBuilder::evalEQ(const ProgramState *state,
                                          DefinedOrUnknownSVal lhs,
                                          DefinedOrUnknownSVal rhs) {
   return cast<DefinedOrUnknownSVal>(evalBinOp(state, BO_EQ, lhs, rhs,
@@ -213,8 +213,13 @@ SVal SValBuilder::evalCast(SVal val, QualType castTy, QualType originalTy) {
     return UnknownVal();
   
   // Check for casts from integers to integers.
-  if (castTy->isIntegerType() && originalTy->isIntegerType())
-    return evalCastFromNonLoc(cast<NonLoc>(val), castTy);
+  if (castTy->isIntegerType() && originalTy->isIntegerType()) {
+    if (isa<Loc>(val))
+      // This can be a cast to ObjC property of type int.
+      return evalCastFromLoc(cast<Loc>(val), castTy);
+    else
+      return evalCastFromNonLoc(cast<NonLoc>(val), castTy);
+  }
 
   // Check for casts from pointers to integers.
   if (castTy->isIntegerType() && Loc::isLocType(originalTy))
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SVals.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SVals.cpp
index 4614e349dece..b5980b96938a 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SVals.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SVals.cpp
@@ -12,14 +12,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/Basic/IdentifierTable.h"
-
 using namespace clang;
 using namespace ento;
-using llvm::dyn_cast;
-using llvm::cast;
 using llvm::APSInt;
 
 //===----------------------------------------------------------------------===//
@@ -146,7 +143,7 @@ SVal::symbol_iterator::symbol_iterator(const SymExpr *SE) {
   while (!isa<SymbolData>(itr.back())) expand();
 }
 
-SVal::symbol_iterator& SVal::symbol_iterator::operator++() {
+SVal::symbol_iterator &SVal::symbol_iterator::operator++() {
   assert(!itr.empty() && "attempting to iterate on an 'end' iterator");
   assert(isa<SymbolData>(itr.back()));
   itr.pop_back();
@@ -174,7 +171,7 @@ void SVal::symbol_iterator::expand() {
     return;
   }
 
-  assert(false && "unhandled expansion case");
+  llvm_unreachable("unhandled expansion case");
 }
 
 const void *nonloc::LazyCompoundVal::getStore() const {
@@ -270,7 +267,7 @@ SVal loc::ConcreteInt::evalBinOp(BasicValueFactory& BasicVals,
 
 void SVal::dump() const { dumpToStream(llvm::errs()); }
 
-void SVal::dumpToStream(llvm::raw_ostream& os) const {
+void SVal::dumpToStream(raw_ostream &os) const {
   switch (getBaseKind()) {
     case UnknownKind:
       os << "Unknown";
@@ -289,7 +286,7 @@ void SVal::dumpToStream(llvm::raw_ostream& os) const {
   }
 }
 
-void NonLoc::dumpToStream(llvm::raw_ostream& os) const {
+void NonLoc::dumpToStream(raw_ostream &os) const {
   switch (getSubKind()) {
     case nonloc::ConcreteIntKind: {
       const nonloc::ConcreteInt& C = *cast<nonloc::ConcreteInt>(this);
@@ -344,7 +341,7 @@ void NonLoc::dumpToStream(llvm::raw_ostream& os) const {
   }
 }
 
-void Loc::dumpToStream(llvm::raw_ostream& os) const {
+void Loc::dumpToStream(raw_ostream &os) const {
   switch (getSubKind()) {
     case loc::ConcreteIntKind:
       os << cast<loc::ConcreteInt>(this)->getValue().getZExtValue() << " (Loc)";
@@ -372,7 +369,6 @@ void Loc::dumpToStream(llvm::raw_ostream& os) const {
       break;
     }
     default:
-      assert(false && "Pretty-printing not implemented for this Loc.");
-      break;
+      llvm_unreachable("Pretty-printing not implemented for this Loc.");
   }
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
index 20762e07dd22..79d8b8bf9de3 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
@@ -14,7 +14,7 @@
 
 #include "SimpleConstraintManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 
 namespace clang {
 
@@ -56,7 +56,7 @@ bool SimpleConstraintManager::canReasonAbout(SVal X) const {
   return true;
 }
 
-const GRState *SimpleConstraintManager::assume(const GRState *state,
+const ProgramState *SimpleConstraintManager::assume(const ProgramState *state,
                                                DefinedSVal Cond,
                                                bool Assumption) {
   if (isa<NonLoc>(Cond))
@@ -65,13 +65,13 @@ const GRState *SimpleConstraintManager::assume(const GRState *state,
     return assume(state, cast<Loc>(Cond), Assumption);
 }
 
-const GRState *SimpleConstraintManager::assume(const GRState *state, Loc cond,
+const ProgramState *SimpleConstraintManager::assume(const ProgramState *state, Loc cond,
                                                bool assumption) {
   state = assumeAux(state, cond, assumption);
   return SU.processAssume(state, cond, assumption);
 }
 
-const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
+const ProgramState *SimpleConstraintManager::assumeAux(const ProgramState *state,
                                                   Loc Cond, bool Assumption) {
 
   BasicValueFactory &BasicVals = state->getBasicVals();
@@ -113,7 +113,7 @@ const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
   } // end switch
 }
 
-const GRState *SimpleConstraintManager::assume(const GRState *state,
+const ProgramState *SimpleConstraintManager::assume(const ProgramState *state,
                                                NonLoc cond,
                                                bool assumption) {
   state = assumeAux(state, cond, assumption);
@@ -125,7 +125,7 @@ static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   // the only place it's used. (This code was copied from SimpleSValBuilder.cpp.)
   switch (op) {
   default:
-    assert(false && "Invalid opcode.");
+    llvm_unreachable("Invalid opcode.");
   case BO_LT: return BO_GE;
   case BO_GT: return BO_LE;
   case BO_LE: return BO_GT;
@@ -135,7 +135,7 @@ static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   }
 }
 
-const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
+const ProgramState *SimpleConstraintManager::assumeAux(const ProgramState *state,
                                                   NonLoc Cond,
                                                   bool Assumption) {
 
@@ -152,7 +152,7 @@ const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
 
   switch (Cond.getSubKind()) {
   default:
-    assert(false && "'Assume' not implemented for this NonLoc");
+    llvm_unreachable("'Assume' not implemented for this NonLoc");
 
   case nonloc::SymbolValKind: {
     nonloc::SymbolVal& SV = cast<nonloc::SymbolVal>(Cond);
@@ -202,7 +202,7 @@ const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
   } // end switch
 }
 
-const GRState *SimpleConstraintManager::assumeSymRel(const GRState *state,
+const ProgramState *SimpleConstraintManager::assumeSymRel(const ProgramState *state,
                                                      const SymExpr *LHS,
                                                      BinaryOperator::Opcode op,
                                                      const llvm::APSInt& Int) {
@@ -256,7 +256,7 @@ const GRState *SimpleConstraintManager::assumeSymRel(const GRState *state,
   // be of the same type as the symbol, which is not always correct. Really the
   // comparisons should be performed using the Int's type, then mapped back to
   // the symbol's range of values.
-  GRStateManager &StateMgr = state->getStateManager();
+  ProgramStateManager &StateMgr = state->getStateManager();
   ASTContext &Ctx = StateMgr.getContext();
 
   QualType T = Sym->getType(Ctx);
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.h b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
index a2952afa7351..d4295d42d95e 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
@@ -15,7 +15,7 @@
 #define LLVM_CLANG_GR_SIMPLE_CONSTRAINT_MANAGER_H
 
 #include "clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 
 namespace clang {
 
@@ -33,14 +33,14 @@ class SimpleConstraintManager : public ConstraintManager {
 
   bool canReasonAbout(SVal X) const;
 
-  const GRState *assume(const GRState *state, DefinedSVal Cond,
+  const ProgramState *assume(const ProgramState *state, DefinedSVal Cond,
                         bool Assumption);
 
-  const GRState *assume(const GRState *state, Loc Cond, bool Assumption);
+  const ProgramState *assume(const ProgramState *state, Loc Cond, bool Assumption);
 
-  const GRState *assume(const GRState *state, NonLoc Cond, bool Assumption);
+  const ProgramState *assume(const ProgramState *state, NonLoc Cond, bool Assumption);
 
-  const GRState *assumeSymRel(const GRState *state,
+  const ProgramState *assumeSymRel(const ProgramState *state,
                               const SymExpr *LHS,
                               BinaryOperator::Opcode op,
                               const llvm::APSInt& Int);
@@ -53,27 +53,27 @@ class SimpleConstraintManager : public ConstraintManager {
 
   // Each of these is of the form "$sym+Adj <> V", where "<>" is the comparison
   // operation for the method being invoked.
-  virtual const GRState *assumeSymNE(const GRState *state, SymbolRef sym,
+  virtual const ProgramState *assumeSymNE(const ProgramState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *assumeSymEQ(const GRState *state, SymbolRef sym,
+  virtual const ProgramState *assumeSymEQ(const ProgramState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *assumeSymLT(const GRState *state, SymbolRef sym,
+  virtual const ProgramState *assumeSymLT(const ProgramState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *assumeSymGT(const GRState *state, SymbolRef sym,
+  virtual const ProgramState *assumeSymGT(const ProgramState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *assumeSymLE(const GRState *state, SymbolRef sym,
+  virtual const ProgramState *assumeSymLE(const ProgramState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *assumeSymGE(const GRState *state, SymbolRef sym,
+  virtual const ProgramState *assumeSymGE(const ProgramState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
@@ -81,9 +81,9 @@ class SimpleConstraintManager : public ConstraintManager {
   // Internal implementation.
   //===------------------------------------------------------------------===//
 
-  const GRState *assumeAux(const GRState *state, Loc Cond,bool Assumption);
+  const ProgramState *assumeAux(const ProgramState *state, Loc Cond,bool Assumption);
 
-  const GRState *assumeAux(const GRState *state, NonLoc Cond, bool Assumption);
+  const ProgramState *assumeAux(const ProgramState *state, NonLoc Cond, bool Assumption);
 };
 
 } // end GR namespace
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
index 80c18a335fde..bd63ecf775d3 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 
 using namespace clang;
 using namespace ento;
@@ -25,22 +25,22 @@ class SimpleSValBuilder : public SValBuilder {
 
 public:
   SimpleSValBuilder(llvm::BumpPtrAllocator &alloc, ASTContext &context,
-                    GRStateManager &stateMgr)
+                    ProgramStateManager &stateMgr)
                     : SValBuilder(alloc, context, stateMgr) {}
   virtual ~SimpleSValBuilder() {}
 
   virtual SVal evalMinus(NonLoc val);
   virtual SVal evalComplement(NonLoc val);
-  virtual SVal evalBinOpNN(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpNN(const ProgramState *state, BinaryOperator::Opcode op,
                            NonLoc lhs, NonLoc rhs, QualType resultTy);
-  virtual SVal evalBinOpLL(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpLL(const ProgramState *state, BinaryOperator::Opcode op,
                            Loc lhs, Loc rhs, QualType resultTy);
-  virtual SVal evalBinOpLN(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpLN(const ProgramState *state, BinaryOperator::Opcode op,
                            Loc lhs, NonLoc rhs, QualType resultTy);
 
   /// getKnownValue - evaluates a given SVal. If the SVal has only one possible
   ///  (integer) value, that value is returned. Otherwise, returns NULL.
-  virtual const llvm::APSInt *getKnownValue(const GRState *state, SVal V);
+  virtual const llvm::APSInt *getKnownValue(const ProgramState *state, SVal V);
   
   SVal MakeSymIntVal(const SymExpr *LHS, BinaryOperator::Opcode op,
                      const llvm::APSInt &RHS, QualType resultTy);
@@ -49,7 +49,7 @@ class SimpleSValBuilder : public SValBuilder {
 
 SValBuilder *ento::createSimpleSValBuilder(llvm::BumpPtrAllocator &alloc,
                                            ASTContext &context,
-                                           GRStateManager &stateMgr) {
+                                           ProgramStateManager &stateMgr) {
   return new SimpleSValBuilder(alloc, context, stateMgr);
 }
 
@@ -171,7 +171,7 @@ SVal SimpleSValBuilder::evalComplement(NonLoc X) {
 static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   switch (op) {
   default:
-    assert(false && "Invalid opcode.");
+    llvm_unreachable("Invalid opcode.");
   case BO_LT: return BO_GE;
   case BO_GT: return BO_LE;
   case BO_LE: return BO_GT;
@@ -184,7 +184,7 @@ static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
 static BinaryOperator::Opcode ReverseComparison(BinaryOperator::Opcode op) {
   switch (op) {
   default:
-    assert(false && "Invalid opcode.");
+    llvm_unreachable("Invalid opcode.");
   case BO_LT: return BO_GT;
   case BO_GT: return BO_LT;
   case BO_LE: return BO_GE;
@@ -270,7 +270,7 @@ SVal SimpleSValBuilder::MakeSymIntVal(const SymExpr *LHS,
   return makeNonLoc(LHS, op, RHS, resultTy);
 }
 
-SVal SimpleSValBuilder::evalBinOpNN(const GRState *state,
+SVal SimpleSValBuilder::evalBinOpNN(const ProgramState *state,
                                   BinaryOperator::Opcode op,
                                   NonLoc lhs, NonLoc rhs,
                                   QualType resultTy)  {
@@ -347,8 +347,7 @@ SVal SimpleSValBuilder::evalBinOpNN(const GRState *state,
           break;
         case BO_LAnd:
         case BO_LOr:
-          assert(false && "Logical operators handled by branching logic.");
-          return UnknownVal();
+          llvm_unreachable("Logical operators handled by branching logic.");
         case BO_Assign:
         case BO_MulAssign:
         case BO_DivAssign:
@@ -361,12 +360,10 @@ SVal SimpleSValBuilder::evalBinOpNN(const GRState *state,
         case BO_XorAssign:
         case BO_OrAssign:
         case BO_Comma:
-          assert(false && "'=' and ',' operators handled by ExprEngine.");
-          return UnknownVal();
+          llvm_unreachable("'=' and ',' operators handled by ExprEngine.");
         case BO_PtrMemD:
         case BO_PtrMemI:
-          assert(false && "Pointer arithmetic not handled here.");
-          return UnknownVal();
+          llvm_unreachable("Pointer arithmetic not handled here.");
         case BO_LT:
         case BO_GT:
         case BO_LE:
@@ -539,7 +536,7 @@ SVal SimpleSValBuilder::evalBinOpNN(const GRState *state,
 }
 
 // FIXME: all this logic will change if/when we have MemRegion::getLocation().
-SVal SimpleSValBuilder::evalBinOpLL(const GRState *state,
+SVal SimpleSValBuilder::evalBinOpLL(const ProgramState *state,
                                   BinaryOperator::Opcode op,
                                   Loc lhs, Loc rhs,
                                   QualType resultTy) {
@@ -556,8 +553,7 @@ SVal SimpleSValBuilder::evalBinOpLL(const GRState *state,
   if (lhs == rhs) {
     switch (op) {
     default:
-      assert(false && "Unimplemented operation for two identical values");
-      return UnknownVal();
+      llvm_unreachable("Unimplemented operation for two identical values");
     case BO_Sub:
       return makeZeroVal(resultTy);
     case BO_EQ:
@@ -573,8 +569,7 @@ SVal SimpleSValBuilder::evalBinOpLL(const GRState *state,
 
   switch (lhs.getSubKind()) {
   default:
-    assert(false && "Ordering not implemented for this Loc.");
-    return UnknownVal();
+    llvm_unreachable("Ordering not implemented for this Loc.");
 
   case loc::GotoLabelKind:
     // The only thing we know about labels is that they're non-null.
@@ -827,7 +822,7 @@ SVal SimpleSValBuilder::evalBinOpLL(const GRState *state,
           return makeTruthVal(!leftFirst, resultTy);
       }
 
-      assert(false && "Fields not found in parent record's definition");
+      llvm_unreachable("Fields not found in parent record's definition");
     }
 
     // If we get here, we have no way of comparing the regions.
@@ -836,7 +831,7 @@ SVal SimpleSValBuilder::evalBinOpLL(const GRState *state,
   }
 }
 
-SVal SimpleSValBuilder::evalBinOpLN(const GRState *state,
+SVal SimpleSValBuilder::evalBinOpLN(const ProgramState *state,
                                   BinaryOperator::Opcode op,
                                   Loc lhs, NonLoc rhs, QualType resultTy) {
   
@@ -930,7 +925,7 @@ SVal SimpleSValBuilder::evalBinOpLN(const GRState *state,
   return UnknownVal();  
 }
 
-const llvm::APSInt *SimpleSValBuilder::getKnownValue(const GRState *state,
+const llvm::APSInt *SimpleSValBuilder::getKnownValue(const ProgramState *state,
                                                    SVal V) {
   if (V.isUnknownOrUndef())
     return NULL;
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Store.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Store.cpp
index b936738009ec..48a6f4f60f8a 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Store.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/Store.cpp
@@ -12,17 +12,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/AST/CharUnits.h"
 
 using namespace clang;
 using namespace ento;
 
-StoreManager::StoreManager(GRStateManager &stateMgr)
+StoreManager::StoreManager(ProgramStateManager &stateMgr)
   : svalBuilder(stateMgr.getSValBuilder()), StateMgr(stateMgr),
     MRMgr(svalBuilder.getRegionManager()), Ctx(stateMgr.getContext()) {}
 
-StoreRef StoreManager::enterStackFrame(const GRState *state,
+StoreRef StoreManager::enterStackFrame(const ProgramState *state,
                                        const StackFrameContext *frame) {
   return StoreRef(state->getStore(), *this);
 }
@@ -57,7 +57,7 @@ const ElementRegion *StoreManager::GetElementZeroRegion(const MemRegion *R,
 
 const MemRegion *StoreManager::castRegion(const MemRegion *R, QualType CastToTy) {
 
-  ASTContext& Ctx = StateMgr.getContext();
+  ASTContext &Ctx = StateMgr.getContext();
 
   // Handle casts to Objective-C objects.
   if (CastToTy->isObjCObjectPointerType())
@@ -87,7 +87,7 @@ const MemRegion *StoreManager::castRegion(const MemRegion *R, QualType CastToTy)
 
   // Handle casts from compatible types.
   if (R->isBoundable())
-    if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
+    if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(R)) {
       QualType ObjTy = Ctx.getCanonicalType(TR->getValueType());
       if (CanonPointeeTy == ObjTy)
         return R;
@@ -103,8 +103,7 @@ const MemRegion *StoreManager::castRegion(const MemRegion *R, QualType CastToTy)
     case MemRegion::UnknownSpaceRegionKind:
     case MemRegion::NonStaticGlobalSpaceRegionKind:
     case MemRegion::StaticGlobalSpaceRegionKind: {
-      assert(0 && "Invalid region cast");
-      break;
+      llvm_unreachable("Invalid region cast");
     }
 
     case MemRegion::FunctionTextRegionKind:
@@ -157,7 +156,7 @@ const MemRegion *StoreManager::castRegion(const MemRegion *R, QualType CastToTy)
         // Edge case: we are at 0 bytes off the beginning of baseR.  We
         // check to see if type we are casting to is the same as the base
         // region.  If so, just return the base region.
-        if (const TypedRegion *TR = dyn_cast<TypedRegion>(baseR)) {
+        if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(baseR)) {
           QualType ObjTy = Ctx.getCanonicalType(TR->getValueType());
           QualType CanonPointeeTy = Ctx.getCanonicalType(PointeeTy);
           if (CanonPointeeTy == ObjTy)
@@ -203,15 +202,14 @@ const MemRegion *StoreManager::castRegion(const MemRegion *R, QualType CastToTy)
     }
   }
 
-  assert(0 && "unreachable");
-  return 0;
+  llvm_unreachable("unreachable");
 }
 
 
 /// CastRetrievedVal - Used by subclasses of StoreManager to implement
 ///  implicit casts that arise from loads from regions that are reinterpreted
 ///  as another region.
-SVal StoreManager::CastRetrievedVal(SVal V, const TypedRegion *R,
+SVal StoreManager::CastRetrievedVal(SVal V, const TypedValueRegion *R,
                                     QualType castTy, bool performTestOnly) {
   
   if (castTy.isNull())
@@ -237,7 +235,7 @@ SVal StoreManager::CastRetrievedVal(SVal V, const TypedRegion *R,
   return V;
 }
 
-SVal StoreManager::getLValueFieldOrIvar(const Decl* D, SVal Base) {
+SVal StoreManager::getLValueFieldOrIvar(const Decl *D, SVal Base) {
   if (Base.isUnknownOrUndef())
     return Base;
 
@@ -261,8 +259,7 @@ SVal StoreManager::getLValueFieldOrIvar(const Decl* D, SVal Base) {
     return Base;
 
   default:
-    assert(0 && "Unhandled Base.");
-    return Base;
+    llvm_unreachable("Unhandled Base.");
   }
 
   // NOTE: We must have this check first because ObjCIvarDecl is a subclass
@@ -336,3 +333,6 @@ SVal StoreManager::getLValueElement(QualType elementType, NonLoc Offset,
   return loc::MemRegionVal(MRMgr.getElementRegion(elementType, NewIdx, ArrayR,
                                                   Ctx));
 }
+
+StoreManager::BindingsHandler::~BindingsHandler() {}
+
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SymbolManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SymbolManager.cpp
index c1ca1cfcdc07..b843ab1a903b 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SymbolManager.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SymbolManager.cpp
@@ -15,6 +15,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
@@ -24,11 +25,10 @@ void SymExpr::dump() const {
   dumpToStream(llvm::errs());
 }
 
-static void print(llvm::raw_ostream& os, BinaryOperator::Opcode Op) {
+static void print(raw_ostream &os, BinaryOperator::Opcode Op) {
   switch (Op) {
     default:
-      assert(false && "operator printing not implemented");
-      break;
+      llvm_unreachable("operator printing not implemented");
     case BO_Mul: os << '*'  ; break;
     case BO_Div: os << '/'  ; break;
     case BO_Rem: os << '%'  ; break;
@@ -48,7 +48,7 @@ static void print(llvm::raw_ostream& os, BinaryOperator::Opcode Op) {
   }
 }
 
-void SymIntExpr::dumpToStream(llvm::raw_ostream& os) const {
+void SymIntExpr::dumpToStream(raw_ostream &os) const {
   os << '(';
   getLHS()->dumpToStream(os);
   os << ") ";
@@ -57,7 +57,7 @@ void SymIntExpr::dumpToStream(llvm::raw_ostream& os) const {
   if (getRHS().isUnsigned()) os << 'U';
 }
 
-void SymSymExpr::dumpToStream(llvm::raw_ostream& os) const {
+void SymSymExpr::dumpToStream(raw_ostream &os) const {
   os << '(';
   getLHS()->dumpToStream(os);
   os << ") ";
@@ -66,33 +66,33 @@ void SymSymExpr::dumpToStream(llvm::raw_ostream& os) const {
   os << ')';
 }
 
-void SymbolConjured::dumpToStream(llvm::raw_ostream& os) const {
+void SymbolConjured::dumpToStream(raw_ostream &os) const {
   os << "conj_$" << getSymbolID() << '{' << T.getAsString() << '}';
 }
 
-void SymbolDerived::dumpToStream(llvm::raw_ostream& os) const {
+void SymbolDerived::dumpToStream(raw_ostream &os) const {
   os << "derived_$" << getSymbolID() << '{'
      << getParentSymbol() << ',' << getRegion() << '}';
 }
 
-void SymbolExtent::dumpToStream(llvm::raw_ostream& os) const {
+void SymbolExtent::dumpToStream(raw_ostream &os) const {
   os << "extent_$" << getSymbolID() << '{' << getRegion() << '}';
 }
 
-void SymbolMetadata::dumpToStream(llvm::raw_ostream& os) const {
+void SymbolMetadata::dumpToStream(raw_ostream &os) const {
   os << "meta_$" << getSymbolID() << '{'
      << getRegion() << ',' << T.getAsString() << '}';
 }
 
-void SymbolRegionValue::dumpToStream(llvm::raw_ostream& os) const {
+void SymbolRegionValue::dumpToStream(raw_ostream &os) const {
   os << "reg_$" << getSymbolID() << "<" << R << ">";
 }
 
 const SymbolRegionValue*
-SymbolManager::getRegionValueSymbol(const TypedRegion* R) {
+SymbolManager::getRegionValueSymbol(const TypedValueRegion* R) {
   llvm::FoldingSetNodeID profile;
   SymbolRegionValue::Profile(profile, R);
-  void* InsertPos;
+  void *InsertPos;
   SymExpr *SD = DataSet.FindNodeOrInsertPos(profile, InsertPos);
   if (!SD) {
     SD = (SymExpr*) BPAlloc.Allocate<SymbolRegionValue>();
@@ -105,12 +105,12 @@ SymbolManager::getRegionValueSymbol(const TypedRegion* R) {
 }
 
 const SymbolConjured*
-SymbolManager::getConjuredSymbol(const Stmt* E, QualType T, unsigned Count,
-                                 const void* SymbolTag) {
+SymbolManager::getConjuredSymbol(const Stmt *E, QualType T, unsigned Count,
+                                 const void *SymbolTag) {
 
   llvm::FoldingSetNodeID profile;
   SymbolConjured::Profile(profile, E, T, Count, SymbolTag);
-  void* InsertPos;
+  void *InsertPos;
   SymExpr *SD = DataSet.FindNodeOrInsertPos(profile, InsertPos);
   if (!SD) {
     SD = (SymExpr*) BPAlloc.Allocate<SymbolConjured>();
@@ -124,11 +124,11 @@ SymbolManager::getConjuredSymbol(const Stmt* E, QualType T, unsigned Count,
 
 const SymbolDerived*
 SymbolManager::getDerivedSymbol(SymbolRef parentSymbol,
-                                const TypedRegion *R) {
+                                const TypedValueRegion *R) {
 
   llvm::FoldingSetNodeID profile;
   SymbolDerived::Profile(profile, parentSymbol, R);
-  void* InsertPos;
+  void *InsertPos;
   SymExpr *SD = DataSet.FindNodeOrInsertPos(profile, InsertPos);
   if (!SD) {
     SD = (SymExpr*) BPAlloc.Allocate<SymbolDerived>();
@@ -144,7 +144,7 @@ const SymbolExtent*
 SymbolManager::getExtentSymbol(const SubRegion *R) {
   llvm::FoldingSetNodeID profile;
   SymbolExtent::Profile(profile, R);
-  void* InsertPos;
+  void *InsertPos;
   SymExpr *SD = DataSet.FindNodeOrInsertPos(profile, InsertPos);
   if (!SD) {
     SD = (SymExpr*) BPAlloc.Allocate<SymbolExtent>();
@@ -157,12 +157,12 @@ SymbolManager::getExtentSymbol(const SubRegion *R) {
 }
 
 const SymbolMetadata*
-SymbolManager::getMetadataSymbol(const MemRegion* R, const Stmt* S, QualType T,
-                                 unsigned Count, const void* SymbolTag) {
+SymbolManager::getMetadataSymbol(const MemRegion* R, const Stmt *S, QualType T,
+                                 unsigned Count, const void *SymbolTag) {
 
   llvm::FoldingSetNodeID profile;
   SymbolMetadata::Profile(profile, R, S, T, Count, SymbolTag);
-  void* InsertPos;
+  void *InsertPos;
   SymExpr *SD = DataSet.FindNodeOrInsertPos(profile, InsertPos);
   if (!SD) {
     SD = (SymExpr*) BPAlloc.Allocate<SymbolMetadata>();
@@ -214,11 +214,11 @@ QualType SymbolConjured::getType(ASTContext&) const {
   return T;
 }
 
-QualType SymbolDerived::getType(ASTContext& Ctx) const {
+QualType SymbolDerived::getType(ASTContext &Ctx) const {
   return R->getValueType();
 }
 
-QualType SymbolExtent::getType(ASTContext& Ctx) const {
+QualType SymbolExtent::getType(ASTContext &Ctx) const {
   return Ctx.getSizeType();
 }
 
@@ -226,11 +226,17 @@ QualType SymbolMetadata::getType(ASTContext&) const {
   return T;
 }
 
-QualType SymbolRegionValue::getType(ASTContext& C) const {
+QualType SymbolRegionValue::getType(ASTContext &C) const {
   return R->getValueType();
 }
 
-SymbolManager::~SymbolManager() {}
+SymbolManager::~SymbolManager() {
+  for (SymbolDependTy::const_iterator I = SymbolDependencies.begin(),
+       E = SymbolDependencies.end(); I != E; ++I) {
+    delete I->second;
+  }
+
+}
 
 bool SymbolManager::canSymbolicate(QualType T) {
   T = T.getCanonicalType();
@@ -247,9 +253,53 @@ bool SymbolManager::canSymbolicate(QualType T) {
   return false;
 }
 
+void SymbolManager::addSymbolDependency(const SymbolRef Primary,
+                                        const SymbolRef Dependent) {
+  SymbolDependTy::iterator I = SymbolDependencies.find(Primary);
+  SymbolRefSmallVectorTy *dependencies = 0;
+  if (I == SymbolDependencies.end()) {
+    dependencies = new SymbolRefSmallVectorTy();
+    SymbolDependencies[Primary] = dependencies;
+  } else {
+    dependencies = I->second;
+  }
+  dependencies->push_back(Dependent);
+}
+
+const SymbolRefSmallVectorTy *SymbolManager::getDependentSymbols(
+                                                     const SymbolRef Primary) {
+  SymbolDependTy::const_iterator I = SymbolDependencies.find(Primary);
+  if (I == SymbolDependencies.end())
+    return 0;
+  return I->second;
+}
+
+void SymbolReaper::markDependentsLive(SymbolRef sym) {
+  // Do not mark dependents more then once.
+  SymbolMapTy::iterator LI = TheLiving.find(sym);
+  assert(LI != TheLiving.end() && "The primary symbol is not live.");
+  if (LI->second == HaveMarkedDependents)
+    return;
+  LI->second = HaveMarkedDependents;
+
+  if (const SymbolRefSmallVectorTy *Deps = SymMgr.getDependentSymbols(sym)) {
+    for (SymbolRefSmallVectorTy::const_iterator I = Deps->begin(),
+                                                E = Deps->end(); I != E; ++I) {
+      if (TheLiving.find(*I) != TheLiving.end())
+        continue;
+      markLive(*I);
+    }
+  }
+}
+
 void SymbolReaper::markLive(SymbolRef sym) {
-  TheLiving.insert(sym);
+  TheLiving[sym] = NotProcessed;
   TheDead.erase(sym);
+  markDependentsLive(sym);
+}
+
+void SymbolReaper::markLive(const MemRegion *region) {
+  RegionRoots.insert(region);
 }
 
 void SymbolReaper::markInUse(SymbolRef sym) {
@@ -265,14 +315,17 @@ bool SymbolReaper::maybeDead(SymbolRef sym) {
   return true;
 }
 
-static bool IsLiveRegion(SymbolReaper &Reaper, const MemRegion *MR) {
+bool SymbolReaper::isLiveRegion(const MemRegion *MR) {
+  if (RegionRoots.count(MR))
+    return true;
+  
   MR = MR->getBaseRegion();
 
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(MR))
-    return Reaper.isLive(SR->getSymbol());
+    return isLive(SR->getSymbol());
 
   if (const VarRegion *VR = dyn_cast<VarRegion>(MR))
-    return Reaper.isLive(VR);
+    return isLive(VR, true);
 
   // FIXME: This is a gross over-approximation. What we really need is a way to
   // tell if anything still refers to this region. Unlike SymbolicRegions,
@@ -291,8 +344,10 @@ static bool IsLiveRegion(SymbolReaper &Reaper, const MemRegion *MR) {
 }
 
 bool SymbolReaper::isLive(SymbolRef sym) {
-  if (TheLiving.count(sym))
+  if (TheLiving.count(sym)) {
+    markDependentsLive(sym);
     return true;
+  }
 
   if (const SymbolDerived *derived = dyn_cast<SymbolDerived>(sym)) {
     if (isLive(derived->getParentSymbol())) {
@@ -303,7 +358,7 @@ bool SymbolReaper::isLive(SymbolRef sym) {
   }
 
   if (const SymbolExtent *extent = dyn_cast<SymbolExtent>(sym)) {
-    if (IsLiveRegion(*this, extent->getRegion())) {
+    if (isLiveRegion(extent->getRegion())) {
       markLive(sym);
       return true;
     }
@@ -312,7 +367,7 @@ bool SymbolReaper::isLive(SymbolRef sym) {
 
   if (const SymbolMetadata *metadata = dyn_cast<SymbolMetadata>(sym)) {
     if (MetadataInUse.count(sym)) {
-      if (IsLiveRegion(*this, metadata->getRegion())) {
+      if (isLiveRegion(metadata->getRegion())) {
         markLive(sym);
         MetadataInUse.erase(sym);
         return true;
@@ -326,18 +381,38 @@ bool SymbolReaper::isLive(SymbolRef sym) {
   return isa<SymbolRegionValue>(sym);
 }
 
-bool SymbolReaper::isLive(const Stmt* ExprVal) const {
-  return LCtx->getAnalysisContext()->getRelaxedLiveVariables()->
-      isLive(Loc, ExprVal);
+bool SymbolReaper::isLive(const Stmt *ExprVal) const {
+  return LCtx->getAnalysis<RelaxedLiveVariables>()->isLive(Loc, ExprVal);
 }
 
-bool SymbolReaper::isLive(const VarRegion *VR) const {
+bool SymbolReaper::isLive(const VarRegion *VR, bool includeStoreBindings) const{
   const StackFrameContext *VarContext = VR->getStackFrame();
   const StackFrameContext *CurrentContext = LCtx->getCurrentStackFrame();
 
-  if (VarContext == CurrentContext)
-    return LCtx->getAnalysisContext()->getRelaxedLiveVariables()->
-        isLive(Loc, VR->getDecl());
+  if (VarContext == CurrentContext) {
+    if (LCtx->getAnalysis<RelaxedLiveVariables>()->isLive(Loc, VR->getDecl()))
+      return true;
+
+    if (!includeStoreBindings)
+      return false;
+    
+    unsigned &cachedQuery =
+      const_cast<SymbolReaper*>(this)->includedRegionCache[VR];
+
+    if (cachedQuery) {
+      return cachedQuery == 1;
+    }
+
+    // Query the store to see if the region occurs in any live bindings.
+    if (Store store = reapedStore.getStore()) {
+      bool hasRegion = 
+        reapedStore.getStoreManager().includedInBindings(store, VR);
+      cachedQuery = hasRegion ? 1 : 2;
+      return hasRegion;
+    }
+    
+    return false;
+  }
 
   return VarContext->isParentOf(CurrentContext);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
index 230b6a1087dc..3543f7ff13dc 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/raw_ostream.h"
@@ -23,19 +23,19 @@ namespace {
 
 /// \brief Simple path diagnostic client used for outputting as diagnostic notes
 /// the sequence of events.
-class TextPathDiagnostics : public PathDiagnosticClient {
+class TextPathDiagnostics : public PathDiagnosticConsumer {
   const std::string OutputFile;
-  Diagnostic &Diag;
+  DiagnosticsEngine &Diag;
 
 public:
-  TextPathDiagnostics(const std::string& output, Diagnostic &diag)
+  TextPathDiagnostics(const std::string& output, DiagnosticsEngine &diag)
     : OutputFile(output), Diag(diag) {}
 
-  void HandlePathDiagnostic(const PathDiagnostic* D);
+  void HandlePathDiagnosticImpl(const PathDiagnostic* D);
 
-  void FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade) { }
+  void FlushDiagnostics(SmallVectorImpl<std::string> *FilesMade) { }
   
-  virtual llvm::StringRef getName() const {
+  virtual StringRef getName() const {
     return "TextPathDiagnostics";
   }
 
@@ -47,13 +47,13 @@ class TextPathDiagnostics : public PathDiagnosticClient {
 
 } // end anonymous namespace
 
-PathDiagnosticClient*
-ento::createTextPathDiagnosticClient(const std::string& out,
+PathDiagnosticConsumer*
+ento::createTextPathDiagnosticConsumer(const std::string& out,
                                      const Preprocessor &PP) {
   return new TextPathDiagnostics(out, PP.getDiagnostics());
 }
 
-void TextPathDiagnostics::HandlePathDiagnostic(const PathDiagnostic* D) {
+void TextPathDiagnostics::HandlePathDiagnosticImpl(const PathDiagnostic* D) {
   if (!D)
     return;
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index b8dbb545041e..34a358ff200b 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -25,8 +25,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
-#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
@@ -43,15 +42,15 @@ using namespace ento;
 static ExplodedNode::Auditor* CreateUbiViz();
 
 //===----------------------------------------------------------------------===//
-// Special PathDiagnosticClients.
+// Special PathDiagnosticConsumers.
 //===----------------------------------------------------------------------===//
 
-static PathDiagnosticClient*
-createPlistHTMLDiagnosticClient(const std::string& prefix,
+static PathDiagnosticConsumer*
+createPlistHTMLDiagnosticConsumer(const std::string& prefix,
                                 const Preprocessor &PP) {
-  PathDiagnosticClient *PD =
-    createHTMLDiagnosticClient(llvm::sys::path::parent_path(prefix), PP);
-  return createPlistDiagnosticClient(prefix, PP, PD);
+  PathDiagnosticConsumer *PD =
+    createHTMLDiagnosticConsumer(llvm::sys::path::parent_path(prefix), PP);
+  return createPlistDiagnosticConsumer(prefix, PP, PD);
 }
 
 //===----------------------------------------------------------------------===//
@@ -62,13 +61,14 @@ namespace {
 
 class AnalysisConsumer : public ASTConsumer {
 public:
-  ASTContext* Ctx;
+  ASTContext *Ctx;
   const Preprocessor &PP;
   const std::string OutDir;
   AnalyzerOptions Opts;
+  ArrayRef<std::string> Plugins;
 
   // PD is owned by AnalysisManager.
-  PathDiagnosticClient *PD;
+  PathDiagnosticConsumer *PD;
 
   StoreManagerCreator CreateStoreMgr;
   ConstraintManagerCreator CreateConstraintMgr;
@@ -78,14 +78,14 @@ class AnalysisConsumer : public ASTConsumer {
 
   AnalysisConsumer(const Preprocessor& pp,
                    const std::string& outdir,
-                   const AnalyzerOptions& opts)
-    : Ctx(0), PP(pp), OutDir(outdir),
-      Opts(opts), PD(0) {
+                   const AnalyzerOptions& opts,
+                   ArrayRef<std::string> plugins)
+    : Ctx(0), PP(pp), OutDir(outdir), Opts(opts), Plugins(plugins), PD(0) {
     DigestAnalyzerOptions();
   }
 
   void DigestAnalyzerOptions() {
-    // Create the PathDiagnosticClient.
+    // Create the PathDiagnosticConsumer.
     if (!OutDir.empty()) {
       switch (Opts.AnalysisDiagOpt) {
       default:
@@ -96,13 +96,13 @@ class AnalysisConsumer : public ASTConsumer {
     } else if (Opts.AnalysisDiagOpt == PD_TEXT) {
       // Create the text client even without a specified output file since
       // it just uses diagnostic notes.
-      PD = createTextPathDiagnosticClient("", PP);
+      PD = createTextPathDiagnosticConsumer("", PP);
     }
 
     // Create the analyzer component creators.
     switch (Opts.AnalysisStoreOpt) {
     default:
-      assert(0 && "Unknown store manager.");
+      llvm_unreachable("Unknown store manager.");
 #define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATEFN)           \
       case NAME##Model: CreateStoreMgr = CREATEFN; break;
 #include "clang/Frontend/Analyses.def"
@@ -110,7 +110,7 @@ class AnalysisConsumer : public ASTConsumer {
 
     switch (Opts.AnalysisConstraintsOpt) {
     default:
-      assert(0 && "Unknown store manager.");
+      llvm_unreachable("Unknown store manager.");
 #define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATEFN)     \
       case NAME##Model: CreateConstraintMgr = CREATEFN; break;
 #include "clang/Frontend/Analyses.def"
@@ -128,7 +128,7 @@ class AnalysisConsumer : public ASTConsumer {
 
       if (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)) {
         const NamedDecl *ND = cast<NamedDecl>(D);
-        llvm::errs() << ' ' << ND << '\n';
+        llvm::errs() << ' ' << *ND << '\n';
       }
       else if (isa<BlockDecl>(D)) {
         llvm::errs() << ' ' << "block(line:" << Loc.getLine() << ",col:"
@@ -143,8 +143,8 @@ class AnalysisConsumer : public ASTConsumer {
 
   virtual void Initialize(ASTContext &Context) {
     Ctx = &Context;
-    checkerMgr.reset(registerCheckers(Opts, PP.getLangOptions(),
-                                      PP.getDiagnostics()));
+    checkerMgr.reset(createCheckerManager(Opts, PP.getLangOptions(), Plugins,
+                                          PP.getDiagnostics()));
     Mgr.reset(new AnalysisManager(*Ctx, PP.getDiagnostics(),
                                   PP.getLangOptions(), PD,
                                   CreateStoreMgr, CreateConstraintMgr,
@@ -152,7 +152,7 @@ class AnalysisConsumer : public ASTConsumer {
                                   /* Indexer */ 0, 
                                   Opts.MaxNodes, Opts.MaxLoop,
                                   Opts.VisualizeEGDot, Opts.VisualizeEGUbi,
-                                  Opts.PurgeDead, Opts.EagerlyAssume,
+                                  Opts.AnalysisPurgeOpt, Opts.EagerlyAssume,
                                   Opts.TrimGraph, Opts.InlineCall,
                                   Opts.UnoptimizedCFG, Opts.CFGAddImplicitDtors,
                                   Opts.CFGAddInitializers,
@@ -161,6 +161,7 @@ class AnalysisConsumer : public ASTConsumer {
 
   virtual void HandleTranslationUnit(ASTContext &C);
   void HandleDeclContext(ASTContext &C, DeclContext *dc);
+  void HandleDeclContextDecl(ASTContext &C, Decl *D);
 
   void HandleCode(Decl *D);
 };
@@ -171,61 +172,67 @@ class AnalysisConsumer : public ASTConsumer {
 //===----------------------------------------------------------------------===//
 
 void AnalysisConsumer::HandleDeclContext(ASTContext &C, DeclContext *dc) {
-  BugReporter BR(*Mgr);
   for (DeclContext::decl_iterator I = dc->decls_begin(), E = dc->decls_end();
        I != E; ++I) {
-    Decl *D = *I;
+    HandleDeclContextDecl(C, *I);
+  }
+}
+
+void AnalysisConsumer::HandleDeclContextDecl(ASTContext &C, Decl *D) {
+  { // Handle callbacks for arbitrary decls.
+    BugReporter BR(*Mgr);
     checkerMgr->runCheckersOnASTDecl(D, *Mgr, BR);
+  }
 
-    switch (D->getKind()) {
-      case Decl::Namespace: {
-        HandleDeclContext(C, cast<NamespaceDecl>(D));
-        break;
-      }
-      case Decl::CXXConstructor:
-      case Decl::CXXDestructor:
-      case Decl::CXXConversion:
-      case Decl::CXXMethod:
-      case Decl::Function: {
-        FunctionDecl* FD = cast<FunctionDecl>(D);
-        // We skip function template definitions, as their semantics is
-        // only determined when they are instantiated.
-        if (FD->isThisDeclarationADefinition() &&
-            !FD->isDependentContext()) {
-          if (!Opts.AnalyzeSpecificFunction.empty() &&
-              FD->getDeclName().getAsString() != Opts.AnalyzeSpecificFunction)
-            break;
-          DisplayFunction(FD);
-          HandleCode(FD);
-        }
-        break;
-      }
-       
-      case Decl::ObjCCategoryImpl:
-      case Decl::ObjCImplementation: {
-        ObjCImplDecl* ID = cast<ObjCImplDecl>(*I);
-        HandleCode(ID);
-        
-        for (ObjCContainerDecl::method_iterator MI = ID->meth_begin(), 
-             ME = ID->meth_end(); MI != ME; ++MI) {
-          checkerMgr->runCheckersOnASTDecl(*MI, *Mgr, BR);
-
-          if ((*MI)->isThisDeclarationADefinition()) {
-            if (!Opts.AnalyzeSpecificFunction.empty() &&
-                Opts.AnalyzeSpecificFunction != 
-                  (*MI)->getSelector().getAsString())
-              break;
-            DisplayFunction(*MI);
-            HandleCode(*MI);
-          }
-        }
-        break;
-      }
-        
-      default:
-        break;
+  switch (D->getKind()) {
+    case Decl::Namespace: {
+      HandleDeclContext(C, cast<NamespaceDecl>(D));
+      break;
     }
-  }  
+    case Decl::CXXConstructor:
+    case Decl::CXXDestructor:
+    case Decl::CXXConversion:
+    case Decl::CXXMethod:
+    case Decl::Function: {
+      FunctionDecl *FD = cast<FunctionDecl>(D);
+      // We skip function template definitions, as their semantics is
+      // only determined when they are instantiated.
+      if (FD->isThisDeclarationADefinition() &&
+          !FD->isDependentContext()) {
+        if (!Opts.AnalyzeSpecificFunction.empty() &&
+            FD->getDeclName().getAsString() != Opts.AnalyzeSpecificFunction)
+          break;
+        DisplayFunction(FD);
+        HandleCode(FD);
+      }
+      break;
+    }
+     
+    case Decl::ObjCCategoryImpl:
+    case Decl::ObjCImplementation: {
+      ObjCImplDecl *ID = cast<ObjCImplDecl>(D);
+      HandleCode(ID);
+      
+      for (ObjCContainerDecl::method_iterator MI = ID->meth_begin(), 
+           ME = ID->meth_end(); MI != ME; ++MI) {
+        BugReporter BR(*Mgr);
+        checkerMgr->runCheckersOnASTDecl(*MI, *Mgr, BR);
+
+        if ((*MI)->isThisDeclarationADefinition()) {
+          if (!Opts.AnalyzeSpecificFunction.empty() &&
+              Opts.AnalyzeSpecificFunction != 
+                (*MI)->getSelector().getAsString())
+            continue;
+          DisplayFunction(*MI);
+          HandleCode(*MI);
+        }
+      }
+      break;
+    }
+      
+    default:
+      break;
+  }
 }
 
 void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
@@ -237,14 +244,14 @@ void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
   // After all decls handled, run checkers on the entire TranslationUnit.
   checkerMgr->runCheckersOnEndOfTranslationUnit(TU, *Mgr, BR);
 
-  // Explicitly destroy the PathDiagnosticClient.  This will flush its output.
+  // Explicitly destroy the PathDiagnosticConsumer.  This will flush its output.
   // FIXME: This should be replaced with something that doesn't rely on
-  // side-effects in PathDiagnosticClient's destructor. This is required when
+  // side-effects in PathDiagnosticConsumer's destructor. This is required when
   // used with option -disable-free.
   Mgr.reset(NULL);
 }
 
-static void FindBlocks(DeclContext *D, llvm::SmallVectorImpl<Decl*> &WL) {
+static void FindBlocks(DeclContext *D, SmallVectorImpl<Decl*> &WL) {
   if (BlockDecl *BD = dyn_cast<BlockDecl>(D))
     WL.push_back(BD);
 
@@ -254,20 +261,20 @@ static void FindBlocks(DeclContext *D, llvm::SmallVectorImpl<Decl*> &WL) {
       FindBlocks(DC, WL);
 }
 
-static void ActionObjCMemChecker(AnalysisConsumer &C, AnalysisManager& mgr,
-                                 Decl *D);
+static void RunPathSensitiveChecks(AnalysisConsumer &C, AnalysisManager &mgr,
+                                   Decl *D);
 
 void AnalysisConsumer::HandleCode(Decl *D) {
 
   // Don't run the actions if an error has occurred with parsing the file.
-  Diagnostic &Diags = PP.getDiagnostics();
+  DiagnosticsEngine &Diags = PP.getDiagnostics();
   if (Diags.hasErrorOccurred() || Diags.hasFatalErrorOccurred())
     return;
 
   // Don't run the actions on declarations in header files unless
   // otherwise specified.
   SourceManager &SM = Ctx->getSourceManager();
-  SourceLocation SL = SM.getInstantiationLoc(D->getLocation());
+  SourceLocation SL = SM.getExpansionLoc(D->getLocation());
   if (!Opts.AnalyzeAll && !SM.isFromMainFile(SL))
     return;
 
@@ -275,19 +282,19 @@ void AnalysisConsumer::HandleCode(Decl *D) {
   Mgr->ClearContexts();
 
   // Dispatch on the actions.
-  llvm::SmallVector<Decl*, 10> WL;
+  SmallVector<Decl*, 10> WL;
   WL.push_back(D);
 
   if (D->hasBody() && Opts.AnalyzeNestedBlocks)
     FindBlocks(cast<DeclContext>(D), WL);
 
   BugReporter BR(*Mgr);
-  for (llvm::SmallVectorImpl<Decl*>::iterator WI=WL.begin(), WE=WL.end();
+  for (SmallVectorImpl<Decl*>::iterator WI=WL.begin(), WE=WL.end();
        WI != WE; ++WI)
     if ((*WI)->hasBody()) {
       checkerMgr->runCheckersOnASTBody(*WI, *Mgr, BR);
       if (checkerMgr->hasPathSensitiveCheckers())
-        ActionObjCMemChecker(*this, *Mgr, *WI);
+        RunPathSensitiveChecks(*this, *Mgr, *WI);
     }
 }
 
@@ -295,18 +302,13 @@ void AnalysisConsumer::HandleCode(Decl *D) {
 // Path-sensitive checking.
 //===----------------------------------------------------------------------===//
 
-static void ActionExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
-                               Decl *D,
-                               TransferFuncs* tf) {
-
-  llvm::OwningPtr<TransferFuncs> TF(tf);
-
-  // Construct the analysis engine.  We first query for the LiveVariables
-  // information to see if the CFG is valid.
+static void ActionExprEngine(AnalysisConsumer &C, AnalysisManager &mgr,
+                             Decl *D, bool ObjCGCEnabled) {
+  // Construct the analysis engine.  First check if the CFG is valid.
   // FIXME: Inter-procedural analysis will need to handle invalid CFGs.
-  if (!mgr.getLiveVariables(D))
+  if (!mgr.getCFG(D))
     return;
-  ExprEngine Eng(mgr, TF.take());
+  ExprEngine Eng(mgr, ObjCGCEnabled);
 
   // Set the graph auditor.
   llvm::OwningPtr<ExplodedNode::Auditor> Auditor;
@@ -330,35 +332,25 @@ static void ActionExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
   Eng.getBugReporter().FlushReports();
 }
 
-static void ActionObjCMemCheckerAux(AnalysisConsumer &C, AnalysisManager& mgr,
-                                  Decl *D, bool GCEnabled) {
+static void RunPathSensitiveChecks(AnalysisConsumer &C, AnalysisManager &mgr,
+                                   Decl *D) {
 
-  TransferFuncs* TF = MakeCFRefCountTF(mgr.getASTContext(),
-                                         GCEnabled,
-                                         mgr.getLangOptions());
-
-  ActionExprEngine(C, mgr, D, TF);
-}
-
-static void ActionObjCMemChecker(AnalysisConsumer &C, AnalysisManager& mgr,
-                               Decl *D) {
-
- switch (mgr.getLangOptions().getGCMode()) {
- default:
-   assert (false && "Invalid GC mode.");
- case LangOptions::NonGC:
-   ActionObjCMemCheckerAux(C, mgr, D, false);
-   break;
-
- case LangOptions::GCOnly:
-   ActionObjCMemCheckerAux(C, mgr, D, true);
-   break;
-
- case LangOptions::HybridGC:
-   ActionObjCMemCheckerAux(C, mgr, D, false);
-   ActionObjCMemCheckerAux(C, mgr, D, true);
-   break;
- }
+  switch (mgr.getLangOptions().getGC()) {
+  default:
+    llvm_unreachable("Invalid GC mode.");
+  case LangOptions::NonGC:
+    ActionExprEngine(C, mgr, D, false);
+    break;
+  
+  case LangOptions::GCOnly:
+    ActionExprEngine(C, mgr, D, true);
+    break;
+  
+  case LangOptions::HybridGC:
+    ActionExprEngine(C, mgr, D, false);
+    ActionExprEngine(C, mgr, D, true);
+    break;
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -366,14 +358,13 @@ static void ActionObjCMemChecker(AnalysisConsumer &C, AnalysisManager& mgr,
 //===----------------------------------------------------------------------===//
 
 ASTConsumer* ento::CreateAnalysisConsumer(const Preprocessor& pp,
-                                           const std::string& OutDir,
-                                           const AnalyzerOptions& Opts) {
-  llvm::OwningPtr<AnalysisConsumer> C(new AnalysisConsumer(pp, OutDir, Opts));
-
-  // Last, disable the effects of '-Werror' when using the AnalysisConsumer.
+                                          const std::string& outDir,
+                                          const AnalyzerOptions& opts,
+                                          ArrayRef<std::string> plugins) {
+  // Disable the effects of '-Werror' when using the AnalysisConsumer.
   pp.getDiagnostics().setWarningsAsErrors(false);
 
-  return C.take();
+  return new AnalysisConsumer(pp, outDir, opts, plugins);
 }
 
 //===----------------------------------------------------------------------===//
@@ -383,7 +374,7 @@ ASTConsumer* ento::CreateAnalysisConsumer(const Preprocessor& pp,
 namespace {
 
 class UbigraphViz : public ExplodedNode::Auditor {
-  llvm::OwningPtr<llvm::raw_ostream> Out;
+  llvm::OwningPtr<raw_ostream> Out;
   llvm::sys::Path Dir, Filename;
   unsigned Cntr;
 
@@ -391,12 +382,12 @@ class UbigraphViz : public ExplodedNode::Auditor {
   VMap M;
 
 public:
-  UbigraphViz(llvm::raw_ostream* out, llvm::sys::Path& dir,
+  UbigraphViz(raw_ostream *out, llvm::sys::Path& dir,
               llvm::sys::Path& filename);
 
   ~UbigraphViz();
 
-  virtual void AddEdge(ExplodedNode* Src, ExplodedNode* Dst);
+  virtual void AddEdge(ExplodedNode *Src, ExplodedNode *Dst);
 };
 
 } // end anonymous namespace
@@ -426,7 +417,7 @@ static ExplodedNode::Auditor* CreateUbiViz() {
   return new UbigraphViz(Stream.take(), Dir, Filename);
 }
 
-void UbigraphViz::AddEdge(ExplodedNode* Src, ExplodedNode* Dst) {
+void UbigraphViz::AddEdge(ExplodedNode *Src, ExplodedNode *Dst) {
 
   assert (Src != Dst && "Self-edges are not allowed.");
 
@@ -460,7 +451,7 @@ void UbigraphViz::AddEdge(ExplodedNode* Src, ExplodedNode* Dst) {
        << ", ('arrow','true'), ('oriented', 'true'))\n";
 }
 
-UbigraphViz::UbigraphViz(llvm::raw_ostream* out, llvm::sys::Path& dir,
+UbigraphViz::UbigraphViz(raw_ostream *out, llvm::sys::Path& dir,
                          llvm::sys::Path& filename)
   : Out(out), Dir(dir), Filename(filename), Cntr(0) {
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
index 646fe97564b8..5a16bffeacf3 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CLANG_GR_ANALYSISCONSUMER_H
 #define LLVM_CLANG_GR_ANALYSISCONSUMER_H
 
+#include "clang/Basic/LLVM.h"
 #include <string>
 
 namespace clang {
@@ -22,7 +23,7 @@ namespace clang {
 class AnalyzerOptions;
 class ASTConsumer;
 class Preprocessor;
-class Diagnostic;
+class DiagnosticsEngine;
 
 namespace ento {
 class CheckerManager;
@@ -32,7 +33,8 @@ class CheckerManager;
 /// options.)
 ASTConsumer* CreateAnalysisConsumer(const Preprocessor &pp,
                                     const std::string &output,
-                                    const AnalyzerOptions& Opts);
+                                    const AnalyzerOptions& opts,
+                                    ArrayRef<std::string> plugins);
 
 } // end GR namespace
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
deleted file mode 100644
index cd9ac1b0c7e8..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-set(LLVM_NO_RTTI 1)
-
-set(LLVM_USED_LIBS clangBasic clangLex clangAST clangFrontend clangRewrite)
-
-include_directories( ${CMAKE_CURRENT_BINARY_DIR}/../Checkers )
-
-add_clang_library(clangStaticAnalyzerFrontend
-  AnalysisConsumer.cpp
-  CheckerRegistration.cpp
-  FrontendActions.cpp
-  )
-
-add_dependencies(clangStaticAnalyzerFrontend
-  clangStaticAnalyzerCheckers
-  clangStaticAnalyzerCore
-  ClangAttrClasses
-  ClangAttrList
-  ClangDeclNodes
-  ClangStmtNode
-  )
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
index d7edc7e599df..a59fcad6f819 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
@@ -13,53 +13,121 @@
 
 #include "clang/StaticAnalyzer/Frontend/CheckerRegistration.h"
 #include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
-#include "../Checkers/ClangSACheckerProvider.h"
+#include "clang/StaticAnalyzer/Checkers/ClangCheckers.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/CheckerProvider.h"
+#include "clang/StaticAnalyzer/Core/CheckerOptInfo.h"
+#include "clang/StaticAnalyzer/Core/CheckerRegistry.h"
 #include "clang/Frontend/AnalyzerOptions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Basic/Diagnostic.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
 
 using namespace clang;
 using namespace ento;
+using llvm::sys::DynamicLibrary;
 
-CheckerManager *ento::registerCheckers(const AnalyzerOptions &opts,
-                                       const LangOptions &langOpts,
-                                       Diagnostic &diags) {
+namespace {
+class ClangCheckerRegistry : public CheckerRegistry {
+  typedef void (*RegisterCheckersFn)(CheckerRegistry &);
+
+  static bool isCompatibleAPIVersion(const char *versionString);
+  static void warnIncompatible(DiagnosticsEngine *diags, StringRef pluginPath,
+                               const char *pluginAPIVersion);
+
+public:
+  ClangCheckerRegistry(ArrayRef<std::string> plugins,
+                       DiagnosticsEngine *diags = 0);
+};
+  
+} // end anonymous namespace
+
+ClangCheckerRegistry::ClangCheckerRegistry(ArrayRef<std::string> plugins,
+                                           DiagnosticsEngine *diags) {
+  registerBuiltinCheckers(*this);
+
+  for (ArrayRef<std::string>::iterator i = plugins.begin(), e = plugins.end();
+       i != e; ++i) {
+    // Get access to the plugin.
+    DynamicLibrary lib = DynamicLibrary::getPermanentLibrary(i->c_str());
+
+    // See if it's compatible with this build of clang.
+    const char *pluginAPIVersion =
+      (const char *) lib.getAddressOfSymbol("clang_analyzerAPIVersionString");
+    if (!isCompatibleAPIVersion(pluginAPIVersion)) {
+      warnIncompatible(diags, *i, pluginAPIVersion);
+      continue;
+    }
+
+    // Register its checkers.
+    RegisterCheckersFn registerPluginCheckers =
+      (RegisterCheckersFn) (intptr_t) lib.getAddressOfSymbol(
+                                                      "clang_registerCheckers");
+    if (registerPluginCheckers)
+      registerPluginCheckers(*this);
+  }
+}
+
+bool ClangCheckerRegistry::isCompatibleAPIVersion(const char *versionString) {
+  // If the version string is null, it's not an analyzer plugin.
+  if (versionString == 0)
+    return false;
+
+  // For now, none of the static analyzer API is considered stable.
+  // Versions must match exactly.
+  if (strcmp(versionString, CLANG_ANALYZER_API_VERSION_STRING) == 0)
+    return true;
+
+  return false;
+}
+
+void ClangCheckerRegistry::warnIncompatible(DiagnosticsEngine *diags,
+                                            StringRef pluginPath,
+                                            const char *pluginAPIVersion) {
+  if (!diags)
+    return;
+  if (!pluginAPIVersion)
+    return;
+
+  diags->Report(diag::warn_incompatible_analyzer_plugin_api)
+      << llvm::sys::path::filename(pluginPath);
+  diags->Report(diag::note_incompatible_analyzer_plugin_api)
+      << CLANG_ANALYZER_API_VERSION_STRING
+      << pluginAPIVersion;
+}
+
+
+CheckerManager *ento::createCheckerManager(const AnalyzerOptions &opts,
+                                           const LangOptions &langOpts,
+                                           ArrayRef<std::string> plugins,
+                                           DiagnosticsEngine &diags) {
   llvm::OwningPtr<CheckerManager> checkerMgr(new CheckerManager(langOpts));
 
-  llvm::SmallVector<CheckerOptInfo, 8> checkerOpts;
+  SmallVector<CheckerOptInfo, 8> checkerOpts;
   for (unsigned i = 0, e = opts.CheckersControlList.size(); i != e; ++i) {
     const std::pair<std::string, bool> &opt = opts.CheckersControlList[i];
     checkerOpts.push_back(CheckerOptInfo(opt.first.c_str(), opt.second));
   }
 
-  llvm::OwningPtr<CheckerProvider> provider(createClangSACheckerProvider());
-  provider->registerCheckers(*checkerMgr,
-                             checkerOpts.data(), checkerOpts.size());
-
-  // FIXME: Load CheckerProviders from plugins.
-
+  ClangCheckerRegistry allCheckers(plugins, &diags);
+  allCheckers.initializeManager(*checkerMgr, checkerOpts);
   checkerMgr->finishedCheckerRegistration();
 
   for (unsigned i = 0, e = checkerOpts.size(); i != e; ++i) {
     if (checkerOpts[i].isUnclaimed())
-      diags.Report(diag::warn_unkwown_analyzer_checker)
+      diags.Report(diag::warn_unknown_analyzer_checker)
           << checkerOpts[i].getName();
   }
 
   return checkerMgr.take();
 }
 
-void ento::printCheckerHelp(llvm::raw_ostream &OS) {
-  OS << "OVERVIEW: Clang Static Analyzer Checkers List\n";
-  OS << '\n';
+void ento::printCheckerHelp(raw_ostream &out, ArrayRef<std::string> plugins) {
+  out << "OVERVIEW: Clang Static Analyzer Checkers List\n\n";
+  out << "USAGE: -analyzer-checker <CHECKER or PACKAGE,...>\n\n";
 
-  llvm::OwningPtr<CheckerProvider> provider(createClangSACheckerProvider());
-  provider->printHelp(OS);
-
-  // FIXME: Load CheckerProviders from plugins.
+  ClangCheckerRegistry(plugins).printHelp(out);
 }
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/FrontendActions.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/FrontendActions.cpp
index a59cc6888fdb..85a18ec98ead 100644
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/FrontendActions.cpp
+++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/FrontendActions.cpp
@@ -14,9 +14,10 @@ using namespace clang;
 using namespace ento;
 
 ASTConsumer *AnalysisAction::CreateASTConsumer(CompilerInstance &CI,
-                                               llvm::StringRef InFile) {
+                                               StringRef InFile) {
   return CreateAnalysisConsumer(CI.getPreprocessor(),
                                 CI.getFrontendOpts().OutputFile,
-                                CI.getAnalyzerOpts());
+                                CI.getAnalyzerOpts(),
+                                CI.getFrontendOpts().Plugins);
 }
 
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/Makefile b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/Makefile
deleted file mode 100644
index 2698120d9098..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Frontend/Makefile
+++ /dev/null
@@ -1,19 +0,0 @@
-##===- clang/lib/StaticAnalyzer/Frontend/Makefile ----------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-#
-# Starting point into the static analyzer land for the driver.
-#
-##===----------------------------------------------------------------------===##
-
-CLANG_LEVEL := ../../..
-LIBRARYNAME := clangStaticAnalyzerFrontend
-
-CPP.Flags += -I${PROJ_OBJ_DIR}/../Checkers
-
-include $(CLANG_LEVEL)/Makefile
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Makefile b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Makefile
deleted file mode 100644
index c166f063f925..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Makefile
+++ /dev/null
@@ -1,18 +0,0 @@
-##===- clang/lib/StaticAnalyzer/Makefile -------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-#
-# This implements analyses built on top of source-level CFGs. 
-#
-##===----------------------------------------------------------------------===##
-
-CLANG_LEVEL := ../..
-DIRS := Checkers Frontend
-PARALLEL_DIRS := Core
-
-include $(CLANG_LEVEL)/Makefile
diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/README.txt b/contrib/llvm/tools/clang/lib/StaticAnalyzer/README.txt
deleted file mode 100644
index 1406eca8c85f..000000000000
--- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/README.txt
+++ /dev/null
@@ -1,139 +0,0 @@
-//===----------------------------------------------------------------------===//
-// Clang Static Analyzer
-//===----------------------------------------------------------------------===//
-
-= Library Structure =
-
-The analyzer library has two layers: a (low-level) static analysis
-engine (GRExprEngine.cpp and friends), and some static checkers
-(*Checker.cpp).  The latter are built on top of the former via the
-Checker and CheckerVisitor interfaces (Checker.h and
-CheckerVisitor.h).  The Checker interface is designed to be minimal
-and simple for checker writers, and attempts to isolate them from much
-of the gore of the internal analysis engine.
-
-= How It Works =
-
-The analyzer is inspired by several foundational research papers ([1],
-[2]).  (FIXME: kremenek to add more links)
-
-In a nutshell, the analyzer is basically a source code simulator that
-traces out possible paths of execution.  The state of the program
-(values of variables and expressions) is encapsulated by the state
-(GRState).  A location in the program is called a program point
-(ProgramPoint), and the combination of state and program point is a
-node in an exploded graph (ExplodedGraph).  The term "exploded" comes
-from exploding the control-flow edges in the control-flow graph (CFG).
-
-Conceptually the analyzer does a reachability analysis through the
-ExplodedGraph.  We start at a root node, which has the entry program
-point and initial state, and then simulate transitions by analyzing
-individual expressions.  The analysis of an expression can cause the
-state to change, resulting in a new node in the ExplodedGraph with an
-updated program point and an updated state.  A bug is found by hitting
-a node that satisfies some "bug condition" (basically a violation of a
-checking invariant).
-
-The analyzer traces out multiple paths by reasoning about branches and
-then bifurcating the state: on the true branch the conditions of the
-branch are assumed to be true and on the false branch the conditions
-of the branch are assumed to be false.  Such "assumptions" create
-constraints on the values of the program, and those constraints are
-recorded in the GRState object (and are manipulated by the
-ConstraintManager).  If assuming the conditions of a branch would
-cause the constraints to be unsatisfiable, the branch is considered
-infeasible and that path is not taken.  This is how we get
-path-sensitivity.  We reduce exponential blow-up by caching nodes.  If
-a new node with the same state and program point as an existing node
-would get generated, the path "caches out" and we simply reuse the
-existing node.  Thus the ExplodedGraph is not a DAG; it can contain
-cycles as paths loop back onto each other and cache out.
-
-GRState and ExplodedNodes are basically immutable once created.  Once
-one creates a GRState, you need to create a new one to get a new
-GRState.  This immutability is key since the ExplodedGraph represents
-the behavior of the analyzed program from the entry point.  To
-represent these efficiently, we use functional data structures (e.g.,
-ImmutableMaps) which share data between instances.
-
-Finally, individual Checkers work by also manipulating the analysis
-state.  The analyzer engine talks to them via a visitor interface.
-For example, the PreVisitCallExpr() method is called by GRExprEngine
-to tell the Checker that we are about to analyze a CallExpr, and the
-checker is asked to check for any preconditions that might not be
-satisfied.  The checker can do nothing, or it can generate a new
-GRState and ExplodedNode which contains updated checker state.  If it
-finds a bug, it can tell the BugReporter object about the bug,
-providing it an ExplodedNode which is the last node in the path that
-triggered the problem.
-
-= Notes about C++ =
-
-Since now constructors are seen before the variable that is constructed 
-in the CFG, we create a temporary object as the destination region that 
-is constructed into. See ExprEngine::VisitCXXConstructExpr().
-
-In ExprEngine::processCallExit(), we always bind the object region to the
-evaluated CXXConstructExpr. Then in VisitDeclStmt(), we compute the
-corresponding lazy compound value if the variable is not a reference, and
-bind the variable region to the lazy compound value. If the variable
-is a reference, just use the object region as the initilizer value.
-
-Before entering a C++ method (or ctor/dtor), the 'this' region is bound
-to the object region. In ctors, we synthesize 'this' region with  
-CXXRecordDecl*, which means we do not use type qualifiers. In methods, we
-synthesize 'this' region with CXXMethodDecl*, which has getThisType() 
-taking type qualifiers into account. It does not matter we use qualified
-'this' region in one method and unqualified 'this' region in another
-method, because we only need to ensure the 'this' region is consistent 
-when we synthesize it and create it directly from CXXThisExpr in a single
-method call.
-
-= Working on the Analyzer =
-
-If you are interested in bringing up support for C++ expressions, the
-best place to look is the visitation logic in GRExprEngine, which
-handles the simulation of individual expressions.  There are plenty of
-examples there of how other expressions are handled.
-
-If you are interested in writing checkers, look at the Checker and
-CheckerVisitor interfaces (Checker.h and CheckerVisitor.h).  Also look
-at the files named *Checker.cpp for examples on how you can implement
-these interfaces.
-
-= Debugging the Analyzer =
-
-There are some useful command-line options for debugging.  For example:
-
-$ clang -cc1 -help | grep analyze
- -analyze-function <value>
- -analyzer-display-progress
- -analyzer-viz-egraph-graphviz
- ...
-
-The first allows you to specify only analyzing a specific function.
-The second prints to the console what function is being analyzed.  The
-third generates a graphviz dot file of the ExplodedGraph.  This is
-extremely useful when debugging the analyzer and viewing the
-simulation results.
-
-Of course, viewing the CFG (Control-Flow Graph) is also useful:
-
-$ clang -cc1 -help | grep cfg
- -cfg-add-implicit-dtors Add C++ implicit destructors to CFGs for all analyses
- -cfg-add-initializers   Add C++ initializers to CFGs for all analyses
- -cfg-dump               Display Control-Flow Graphs
- -cfg-view               View Control-Flow Graphs using GraphViz
- -unoptimized-cfg        Generate unoptimized CFGs for all analyses
-
--cfg-dump dumps a textual representation of the CFG to the console,
-and -cfg-view creates a GraphViz representation.
-
-= References =
-
-[1] Precise interprocedural dataflow analysis via graph reachability,
-    T Reps, S Horwitz, and M Sagiv, POPL '95,
-    http://portal.acm.org/citation.cfm?id=199462
-
-[2] A memory model for static analysis of C programs, Z Xu, T
-    Kremenek, and J Zhang, http://lcs.ios.ac.cn/~xzx/memmodel.pdf
diff --git a/contrib/llvm/tools/clang/tools/driver/cc1_main.cpp b/contrib/llvm/tools/clang/tools/driver/cc1_main.cpp
index 9ad4af1436ca..27f79b7e6677 100644
--- a/contrib/llvm/tools/clang/tools/driver/cc1_main.cpp
+++ b/contrib/llvm/tools/clang/tools/driver/cc1_main.cpp
@@ -27,9 +27,9 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetSelect.h"
 #include <cstdio>
 using namespace clang;
 
@@ -38,7 +38,7 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 static void LLVMErrorHandler(void *UserData, const std::string &Message) {
-  Diagnostic &Diags = *static_cast<Diagnostic*>(UserData);
+  DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
 
   Diags.Report(diag::err_fe_error_backend) << Message;
 
@@ -47,7 +47,7 @@ static void LLVMErrorHandler(void *UserData, const std::string &Message) {
 }
 
 // FIXME: Define the need for this testing away.
-static int cc1_test(Diagnostic &Diags,
+static int cc1_test(DiagnosticsEngine &Diags,
                     const char **ArgBegin, const char **ArgEnd) {
   using namespace clang::driver;
 
@@ -83,7 +83,7 @@ static int cc1_test(Diagnostic &Diags,
   Invocation.toArgs(InvocationArgs);
 
   // Dump the converted arguments.
-  llvm::SmallVector<const char*, 32> Invocation2Args;
+  SmallVector<const char*, 32> Invocation2Args;
   llvm::errs() << "invocation argv :";
   for (unsigned i = 0, e = InvocationArgs.size(); i != e; ++i) {
     Invocation2Args.push_back(InvocationArgs[i].c_str());
@@ -118,23 +118,22 @@ int cc1_main(const char **ArgBegin, const char **ArgEnd,
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
 
   // Run clang -cc1 test.
-  if (ArgBegin != ArgEnd && llvm::StringRef(ArgBegin[0]) == "-cc1test") {
-    Diagnostic Diags(DiagID, new TextDiagnosticPrinter(llvm::errs(), 
+  if (ArgBegin != ArgEnd && StringRef(ArgBegin[0]) == "-cc1test") {
+    DiagnosticsEngine Diags(DiagID, new TextDiagnosticPrinter(llvm::errs(), 
                                                        DiagnosticOptions()));
     return cc1_test(Diags, ArgBegin + 1, ArgEnd);
   }
 
   // Initialize targets first, so that --version shows registered targets.
   llvm::InitializeAllTargets();
-  llvm::InitializeAllMCAsmInfos();
-  llvm::InitializeAllMCSubtargetInfos();
+  llvm::InitializeAllTargetMCs();
   llvm::InitializeAllAsmPrinters();
   llvm::InitializeAllAsmParsers();
 
   // Buffer diagnostics from argument parsing so that we can output them using a
   // well formed diagnostic object.
   TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
-  Diagnostic Diags(DiagID, DiagsBuffer);
+  DiagnosticsEngine Diags(DiagID, DiagsBuffer);
   CompilerInvocation::CreateFromArgs(Clang->getInvocation(), ArgBegin, ArgEnd,
                                      Diags);
 
diff --git a/contrib/llvm/tools/clang/tools/driver/cc1as_main.cpp b/contrib/llvm/tools/clang/tools/driver/cc1as_main.cpp
index 358d746986c8..7cc42aa62768 100644
--- a/contrib/llvm/tools/clang/tools/driver/cc1as_main.cpp
+++ b/contrib/llvm/tools/clang/tools/driver/cc1as_main.cpp
@@ -30,8 +30,12 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -39,23 +43,15 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/Timer.h"
-#include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Signals.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/Timer.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
-#include "llvm/Target/TargetAsmBackend.h"
-#include "llvm/Target/TargetAsmInfo.h"
-#include "llvm/Target/TargetAsmParser.h"
 #include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetInstrInfo.h"
-#include "llvm/Target/TargetLowering.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetSelect.h"
-#include "llvm/Target/TargetSubtargetInfo.h"
 using namespace clang;
 using namespace clang::driver;
 using namespace llvm;
@@ -125,7 +121,7 @@ struct AssemblerInvocation {
   }
 
   static void CreateFromArgs(AssemblerInvocation &Res, const char **ArgBegin,
-                             const char **ArgEnd, Diagnostic &Diags);
+                             const char **ArgEnd, DiagnosticsEngine &Diags);
 };
 
 }
@@ -133,7 +129,7 @@ struct AssemblerInvocation {
 void AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
                                          const char **ArgBegin,
                                          const char **ArgEnd,
-                                         Diagnostic &Diags) {
+                                         DiagnosticsEngine &Diags) {
   using namespace clang::driver::cc1asoptions;
   // Parse the arguments.
   OwningPtr<OptTable> OptTbl(createCC1AsOptTable());
@@ -207,7 +203,7 @@ void AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
 }
 
 static formatted_raw_ostream *GetOutputStream(AssemblerInvocation &Opts,
-                                              Diagnostic &Diags,
+                                              DiagnosticsEngine &Diags,
                                               bool Binary) {
   if (Opts.OutputPath.empty())
     Opts.OutputPath = "-";
@@ -230,7 +226,8 @@ static formatted_raw_ostream *GetOutputStream(AssemblerInvocation &Opts,
   return new formatted_raw_ostream(*Out, formatted_raw_ostream::DELETE_STREAM);
 }
 
-static bool ExecuteAssembler(AssemblerInvocation &Opts, Diagnostic &Diags) {
+static bool ExecuteAssembler(AssemblerInvocation &Opts,
+                             DiagnosticsEngine &Diags) {
   // Get the target specific parser.
   std::string Error;
   const Target *TheTarget(TargetRegistry::lookupTarget(Opts.Triple, Error));
@@ -259,64 +256,60 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts, Diagnostic &Diags) {
   OwningPtr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(Opts.Triple));
   assert(MAI && "Unable to create target asm info!");
 
+  OwningPtr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
+  assert(MRI && "Unable to create target register info!");
+
   bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
   formatted_raw_ostream *Out = GetOutputStream(Opts, Diags, IsBinary);
   if (!Out)
     return false;
 
-  // FIXME: We shouldn't need to do this (and link in codegen).
-  OwningPtr<TargetMachine> TM(TheTarget->createTargetMachine(Opts.Triple,
-                                                             "", ""));
-  if (!TM) {
-    Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
-    return false;
-  }
-
-  const TargetAsmInfo *tai = new TargetAsmInfo(*TM);
-  MCContext Ctx(*MAI, tai);
+  // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
+  // MCObjectFileInfo needs a MCContext reference in order to initialize itself.
+  OwningPtr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
+  MCContext Ctx(*MAI, *MRI, MOFI.get());
+  // FIXME: Assembler behavior can change with -static.
+  MOFI->InitMCObjectFileInfo(Opts.Triple,
+                             Reloc::Default, CodeModel::Default, Ctx);
   if (Opts.SaveTemporaryLabels)
     Ctx.setAllowTemporaryLabels(false);
 
   OwningPtr<MCStreamer> Str;
 
-  const TargetLoweringObjectFile &TLOF =
-    TM->getTargetLowering()->getObjFileLowering();
-  const_cast<TargetLoweringObjectFile&>(TLOF).Initialize(Ctx, *TM);
-
-  const MCSubtargetInfo &STI = TM->getSubtarget<MCSubtargetInfo>();
+  OwningPtr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
+  OwningPtr<MCSubtargetInfo>
+    STI(TheTarget->createMCSubtargetInfo(Opts.Triple, "", ""));
 
   // FIXME: There is a bit of code duplication with addPassesToEmitFile.
   if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
     MCInstPrinter *IP =
-      TheTarget->createMCInstPrinter(Opts.OutputAsmVariant, *MAI);
+      TheTarget->createMCInstPrinter(Opts.OutputAsmVariant, *MAI, *STI);
     MCCodeEmitter *CE = 0;
-    TargetAsmBackend *TAB = 0;
+    MCAsmBackend *MAB = 0;
     if (Opts.ShowEncoding) {
-      CE = TheTarget->createCodeEmitter(*TM->getInstrInfo(), STI, Ctx);
-      TAB = TheTarget->createAsmBackend(Opts.Triple);
+      CE = TheTarget->createMCCodeEmitter(*MCII, *STI, Ctx);
+      MAB = TheTarget->createMCAsmBackend(Opts.Triple);
     }
     Str.reset(TheTarget->createAsmStreamer(Ctx, *Out, /*asmverbose*/true,
                                            /*useLoc*/ true,
-                                           /*useCFI*/ true, IP, CE, TAB,
+                                           /*useCFI*/ true, IP, CE, MAB,
                                            Opts.ShowInst));
   } else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
     Str.reset(createNullStreamer(Ctx));
   } else {
     assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
            "Invalid file type!");
-    MCCodeEmitter *CE = TheTarget->createCodeEmitter(*TM->getInstrInfo(),
-                                                     STI, Ctx);
-    TargetAsmBackend *TAB = TheTarget->createAsmBackend(Opts.Triple);
-    Str.reset(TheTarget->createObjectStreamer(Opts.Triple, Ctx, *TAB, *Out,
-                                              CE, Opts.RelaxAll,
-                                              Opts.NoExecStack));
+    MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *STI, Ctx);
+    MCAsmBackend *MAB = TheTarget->createMCAsmBackend(Opts.Triple);
+    Str.reset(TheTarget->createMCObjectStreamer(Opts.Triple, Ctx, *MAB, *Out,
+                                                CE, Opts.RelaxAll,
+                                                Opts.NoExecStack));
     Str.get()->InitSections();
   }
 
-  OwningPtr<MCAsmParser> Parser(createMCAsmParser(*TheTarget, SrcMgr, Ctx,
+  OwningPtr<MCAsmParser> Parser(createMCAsmParser(SrcMgr, Ctx,
                                                   *Str.get(), *MAI));
-  OwningPtr<TargetAsmParser>
-    TAP(TheTarget->createAsmParser(const_cast<MCSubtargetInfo&>(STI), *Parser));
+  OwningPtr<MCTargetAsmParser> TAP(TheTarget->createMCAsmParser(*STI, *Parser));
   if (!TAP) {
     Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
     return false;
@@ -337,7 +330,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts, Diagnostic &Diags) {
 }
 
 static void LLVMErrorHandler(void *UserData, const std::string &Message) {
-  Diagnostic &Diags = *static_cast<Diagnostic*>(UserData);
+  DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
 
   Diags.Report(diag::err_fe_error_backend) << Message;
 
@@ -354,12 +347,7 @@ int cc1as_main(const char **ArgBegin, const char **ArgEnd,
 
   // Initialize targets and assembly printers/parsers.
   InitializeAllTargetInfos();
-  // FIXME: We shouldn't need to initialize the Target(Machine)s.
-  InitializeAllTargets();
-  InitializeAllMCAsmInfos();
-  InitializeAllMCInstrInfos();
-  InitializeAllMCSubtargetInfos();
-  InitializeAllAsmPrinters();
+  InitializeAllTargetMCs();
   InitializeAllAsmParsers();
 
   // Construct our diagnostic client.
@@ -367,7 +355,7 @@ int cc1as_main(const char **ArgBegin, const char **ArgEnd,
     = new TextDiagnosticPrinter(errs(), DiagnosticOptions());
   DiagClient->setPrefix("clang -cc1as");
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  Diagnostic Diags(DiagID, DiagClient);
+  DiagnosticsEngine Diags(DiagID, DiagClient);
 
   // Set an error handler, so that any LLVM backend diagnostics go through our
   // error handler.
diff --git a/contrib/llvm/tools/clang/tools/driver/driver.cpp b/contrib/llvm/tools/clang/tools/driver/driver.cpp
index ca8982619e94..bd1d2a2f5f35 100644
--- a/contrib/llvm/tools/clang/tools/driver/driver.cpp
+++ b/contrib/llvm/tools/clang/tools/driver/driver.cpp
@@ -35,9 +35,9 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/system_error.h"
-#include "llvm/Target/TargetRegistry.h"
-#include "llvm/Target/TargetSelect.h"
 #include <cctype>
 using namespace clang;
 using namespace clang::driver;
@@ -53,7 +53,7 @@ llvm::sys::Path GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
 }
 
 static const char *SaveStringInSet(std::set<std::string> &SavedStrings,
-                                   llvm::StringRef S) {
+                                   StringRef S) {
   return SavedStrings.insert(S).first->c_str();
 }
 
@@ -84,9 +84,9 @@ static const char *SaveStringInSet(std::set<std::string> &SavedStrings,
 /// \param Args - The vector of command line arguments.
 /// \param Edit - The override command to perform.
 /// \param SavedStrings - Set to use for storing string representations.
-static void ApplyOneQAOverride(llvm::raw_ostream &OS,
-                               llvm::SmallVectorImpl<const char*> &Args,
-                               llvm::StringRef Edit,
+static void ApplyOneQAOverride(raw_ostream &OS,
+                               SmallVectorImpl<const char*> &Args,
+                               StringRef Edit,
                                std::set<std::string> &SavedStrings) {
   // This does not need to be efficient.
 
@@ -101,9 +101,9 @@ static void ApplyOneQAOverride(llvm::raw_ostream &OS,
     OS << "### Adding argument " << Str << " at end\n";
     Args.push_back(Str);
   } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
-             Edit.slice(2, Edit.size()-1).find('/') != llvm::StringRef::npos) {
-    llvm::StringRef MatchPattern = Edit.substr(2).split('/').first;
-    llvm::StringRef ReplPattern = Edit.substr(2).split('/').second;
+             Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
+    StringRef MatchPattern = Edit.substr(2).split('/').first;
+    StringRef ReplPattern = Edit.substr(2).split('/').second;
     ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
 
     for (unsigned i = 1, e = Args.size(); i != e; ++i) {
@@ -151,10 +151,10 @@ static void ApplyOneQAOverride(llvm::raw_ostream &OS,
 
 /// ApplyQAOverride - Apply a comma separate list of edits to the
 /// input argument lists. See ApplyOneQAOverride.
-static void ApplyQAOverride(llvm::SmallVectorImpl<const char*> &Args,
+static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
                             const char *OverrideStr,
                             std::set<std::string> &SavedStrings) {
-  llvm::raw_ostream *OS = &llvm::errs();
+  raw_ostream *OS = &llvm::errs();
 
   if (OverrideStr[0] == '#') {
     ++OverrideStr;
@@ -184,7 +184,7 @@ extern int cc1as_main(const char **ArgBegin, const char **ArgEnd,
                       const char *Argv0, void *MainAddr);
 
 static void ExpandArgsFromBuf(const char *Arg,
-                              llvm::SmallVectorImpl<const char*> &ArgVector,
+                              SmallVectorImpl<const char*> &ArgVector,
                               std::set<std::string> &SavedStrings) {
   const char *FName = Arg + 1;
   llvm::OwningPtr<llvm::MemoryBuffer> MemBuf;
@@ -242,7 +242,7 @@ static void ExpandArgsFromBuf(const char *Arg,
 }
 
 static void ExpandArgv(int argc, const char **argv,
-                       llvm::SmallVectorImpl<const char*> &ArgVector,
+                       SmallVectorImpl<const char*> &ArgVector,
                        std::set<std::string> &SavedStrings) {
   for (int i = 0; i < argc; ++i) {
     const char *Arg = argv[i];
@@ -255,7 +255,7 @@ static void ExpandArgv(int argc, const char **argv,
   }
 }
 
-static void ParseProgName(llvm::SmallVectorImpl<const char *> &ArgVector,
+static void ParseProgName(SmallVectorImpl<const char *> &ArgVector,
                           std::set<std::string> &SavedStrings,
                           Driver &TheDriver)
 {
@@ -290,8 +290,8 @@ static void ParseProgName(llvm::SmallVectorImpl<const char *> &ArgVector,
     { "++", true, false },
   };
   std::string ProgName(llvm::sys::path::stem(ArgVector[0]));
-  llvm::StringRef ProgNameRef(ProgName);
-  llvm::StringRef Prefix;
+  StringRef ProgNameRef(ProgName);
+  StringRef Prefix;
 
   for (int Components = 2; Components; --Components) {
     bool FoundMatch = false;
@@ -309,15 +309,15 @@ static void ParseProgName(llvm::SmallVectorImpl<const char *> &ArgVector,
     }
 
     if (FoundMatch) {
-      llvm::StringRef::size_type LastComponent = ProgNameRef.rfind('-',
+      StringRef::size_type LastComponent = ProgNameRef.rfind('-',
         ProgNameRef.size() - strlen(suffixes[i].Suffix));
-      if (LastComponent != llvm::StringRef::npos)
+      if (LastComponent != StringRef::npos)
         Prefix = ProgNameRef.slice(0, LastComponent);
       break;
     }
 
-    llvm::StringRef::size_type LastComponent = ProgNameRef.rfind('-');
-    if (LastComponent == llvm::StringRef::npos)
+    StringRef::size_type LastComponent = ProgNameRef.rfind('-');
+    if (LastComponent == StringRef::npos)
       break;
     ProgNameRef = ProgNameRef.slice(0, LastComponent);
   }
@@ -327,7 +327,7 @@ static void ParseProgName(llvm::SmallVectorImpl<const char *> &ArgVector,
 
   std::string IgnoredError;
   if (llvm::TargetRegistry::lookupTarget(Prefix, IgnoredError)) {
-    llvm::SmallVectorImpl<const char *>::iterator it = ArgVector.begin();
+    SmallVectorImpl<const char *>::iterator it = ArgVector.begin();
     if (it != ArgVector.end())
       ++it;
     ArgVector.insert(it, SaveStringInSet(SavedStrings, Prefix));
@@ -341,13 +341,13 @@ int main(int argc_, const char **argv_) {
   llvm::PrettyStackTraceProgram X(argc_, argv_);
 
   std::set<std::string> SavedStrings;
-  llvm::SmallVector<const char*, 256> argv;
+  SmallVector<const char*, 256> argv;
 
   ExpandArgv(argc_, argv_, argv, SavedStrings);
 
   // Handle -cc1 integrated tools.
-  if (argv.size() > 1 && llvm::StringRef(argv[1]).startswith("-cc1")) {
-    llvm::StringRef Tool = argv[1] + 4;
+  if (argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) {
+    StringRef Tool = argv[1] + 4;
 
     if (Tool == "")
       return cc1_main(argv.data()+2, argv.data()+argv.size(), argv[0],
@@ -363,7 +363,7 @@ int main(int argc_, const char **argv_) {
 
   bool CanonicalPrefixes = true;
   for (int i = 1, size = argv.size(); i < size; ++i) {
-    if (llvm::StringRef(argv[i]) == "-no-canonical-prefixes") {
+    if (StringRef(argv[i]) == "-no-canonical-prefixes") {
       CanonicalPrefixes = false;
       break;
     }
@@ -375,22 +375,15 @@ int main(int argc_, const char **argv_) {
     = new TextDiagnosticPrinter(llvm::errs(), DiagnosticOptions());
   DiagClient->setPrefix(llvm::sys::path::stem(Path.str()));
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  Diagnostic Diags(DiagID, DiagClient);
+  DiagnosticsEngine Diags(DiagID, DiagClient);
 
 #ifdef CLANG_IS_PRODUCTION
   const bool IsProduction = true;
-#  ifdef CLANGXX_IS_PRODUCTION
-  const bool CXXIsProduction = true;
-#  else
-  const bool CXXIsProduction = false;
-#  endif
 #else
   const bool IsProduction = false;
-  const bool CXXIsProduction = false;
 #endif
   Driver TheDriver(Path.str(), llvm::sys::getHostTriple(),
-                   "a.out", IsProduction, CXXIsProduction,
-                   Diags);
+                   "a.out", IsProduction, Diags);
 
   // Attempt to find the original path used to invoke the driver, to determine
   // the installed path. We do this manually, because we want to support that
@@ -458,9 +451,15 @@ int main(int argc_, const char **argv_) {
 
   llvm::OwningPtr<Compilation> C(TheDriver.BuildCompilation(argv));
   int Res = 0;
+  const Command *FailingCommand = 0;
   if (C.get())
-    Res = TheDriver.ExecuteCompilation(*C);
-  
+    Res = TheDriver.ExecuteCompilation(*C, FailingCommand);
+
+  // If result status is < 0, then the driver command signalled an error.
+  // In this case, generate additional diagnostic information if possible.
+  if (Res < 0)
+    TheDriver.generateCompilationDiagnostics(*C, FailingCommand);
+
   // If any timers were active but haven't been destroyed yet, print their
   // results now.  This happens in -disable-free mode.
   llvm::TimerGroup::printAll(llvm::errs());
diff --git a/contrib/llvm/tools/clang/utils/TableGen/CMakeLists.txt b/contrib/llvm/tools/clang/utils/TableGen/CMakeLists.txt
new file mode 100644
index 000000000000..75a616700405
--- /dev/null
+++ b/contrib/llvm/tools/clang/utils/TableGen/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(LLVM_REQUIRES_EH 1)
+set(LLVM_REQUIRES_RTTI 1)
+
+add_tablegen(clang-tblgen CLANG
+  ClangASTNodesEmitter.cpp
+  ClangAttrEmitter.cpp
+  ClangDiagnosticsEmitter.cpp
+  ClangSACheckersEmitter.cpp
+  NeonEmitter.cpp
+  OptParserEmitter.cpp
+  TableGen.cpp
+  )
diff --git a/contrib/llvm/utils/TableGen/ClangASTNodesEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/ClangASTNodesEmitter.cpp
similarity index 100%
rename from contrib/llvm/utils/TableGen/ClangASTNodesEmitter.cpp
rename to contrib/llvm/tools/clang/utils/TableGen/ClangASTNodesEmitter.cpp
diff --git a/contrib/llvm/utils/TableGen/ClangASTNodesEmitter.h b/contrib/llvm/tools/clang/utils/TableGen/ClangASTNodesEmitter.h
similarity index 96%
rename from contrib/llvm/utils/TableGen/ClangASTNodesEmitter.h
rename to contrib/llvm/tools/clang/utils/TableGen/ClangASTNodesEmitter.h
index 712333bd2d25..edd9316544ea 100644
--- a/contrib/llvm/utils/TableGen/ClangASTNodesEmitter.h
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangASTNodesEmitter.h
@@ -14,8 +14,8 @@
 #ifndef CLANGAST_EMITTER_H
 #define CLANGAST_EMITTER_H
 
-#include "TableGenBackend.h"
-#include "Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include "llvm/TableGen/Record.h"
 #include <string>
 #include <cctype>
 #include <map>
diff --git a/contrib/llvm/utils/TableGen/ClangAttrEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/ClangAttrEmitter.cpp
similarity index 93%
rename from contrib/llvm/utils/TableGen/ClangAttrEmitter.cpp
rename to contrib/llvm/tools/clang/utils/TableGen/ClangAttrEmitter.cpp
index 26bd8786a49c..5c236be559f1 100644
--- a/contrib/llvm/utils/TableGen/ClangAttrEmitter.cpp
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -12,8 +12,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangAttrEmitter.h"
-#include "Record.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/TableGen/Record.h"
 #include <algorithm>
 #include <cctype>
 
@@ -27,7 +27,9 @@ getValueAsListOfStrings(Record &R, StringRef FieldName) {
   std::vector<StringRef> Strings;
   Strings.reserve(List->getSize());
 
-  for (ListInit::iterator i = List->begin(), e = List->end(); i != e; ++i) {
+  for (ListInit::const_iterator i = List->begin(), e = List->end();
+       i != e;
+       ++i) {
     assert(*i && "Got a null element in a ListInit");
     if (StringInit *S = dynamic_cast<StringInit *>(*i))
       Strings.push_back(S->getValue());
@@ -40,18 +42,19 @@ getValueAsListOfStrings(Record &R, StringRef FieldName) {
   return Strings;
 }
 
-std::string ReadPCHRecord(StringRef type) {
+static std::string ReadPCHRecord(StringRef type) {
   return StringSwitch<std::string>(type)
-    .EndsWith("Decl *", "cast_or_null<" + std::string(type, 0, type.size()-1) +
-              ">(GetDecl(Record[Idx++]))")
-    .Case("QualType", "GetType(Record[Idx++])")
+    .EndsWith("Decl *", "GetLocalDeclAs<" 
+              + std::string(type, 0, type.size()-1) + ">(F, Record[Idx++])")
+    .Case("QualType", "getLocalType(F, Record[Idx++])")
     .Case("Expr *", "ReadSubExpr()")
-    .Case("IdentifierInfo *", "GetIdentifierInfo(Record, Idx)")
+    .Case("IdentifierInfo *", "GetIdentifierInfo(F, Record, Idx)")
+    .Case("SourceLocation", "ReadSourceLocation(F, Record, Idx)")
     .Default("Record[Idx++]");
 }
 
 // Assumes that the way to get the value is SA->getname()
-std::string WritePCHRecord(StringRef type, StringRef name) {
+static std::string WritePCHRecord(StringRef type, StringRef name) {
   return StringSwitch<std::string>(type)
     .EndsWith("Decl *", "AddDeclRef(" + std::string(name) +
                         ", Record);\n")
@@ -59,6 +62,8 @@ std::string WritePCHRecord(StringRef type, StringRef name) {
     .Case("Expr *", "AddStmt(" + std::string(name) + ");\n")
     .Case("IdentifierInfo *", 
           "AddIdentifierRef(" + std::string(name) + ", Record);\n")
+    .Case("SourceLocation", 
+          "AddSourceLocation(" + std::string(name) + ", Record);\n")
     .Default("Record.push_back(" + std::string(name) + ");\n");
 }
 
@@ -234,7 +239,7 @@ namespace {
       OS << "  assert(!is" << getUpperName() << "Dependent());\n";
       OS << "  if (is" << getLowerName() << "Expr)\n";
       OS << "    return (" << getLowerName() << "Expr ? " << getLowerName()
-         << "Expr->EvaluateAsInt(Ctx).getZExtValue() : 16)"
+         << "Expr->EvaluateKnownConstInt(Ctx).getZExtValue() : 16)"
          << "* Ctx.getCharWidth();\n";
       OS << "  else\n";
       OS << "    return 0; // FIXME\n";
@@ -485,8 +490,12 @@ static Argument *createArgument(Record &Arg, StringRef Attr,
     Ptr = new SimpleArgument(Arg, Attr, "QualType");
   else if (ArgName == "UnsignedArgument")
     Ptr = new SimpleArgument(Arg, Attr, "unsigned");
+  else if (ArgName == "SourceLocArgument")
+    Ptr = new SimpleArgument(Arg, Attr, "SourceLocation");
   else if (ArgName == "VariadicUnsignedArgument")
     Ptr = new VariadicArgument(Arg, Attr, "unsigned");
+  else if (ArgName == "VariadicExprArgument")
+    Ptr = new VariadicArgument(Arg, Attr, "Expr *");
   else if (ArgName == "VersionArgument")
     Ptr = new VersionArgument(Arg, Attr);
 
@@ -536,7 +545,7 @@ void ClangAttrClassEmitter::run(raw_ostream &OS) {
     ae = Args.end();
 
     OS << "\n public:\n";
-    OS << "  " << R.getName() << "Attr(SourceLocation L, ASTContext &Ctx\n";
+    OS << "  " << R.getName() << "Attr(SourceRange R, ASTContext &Ctx\n";
     
     for (ai = Args.begin(); ai != ae; ++ai) {
       OS << "              , ";
@@ -545,7 +554,7 @@ void ClangAttrClassEmitter::run(raw_ostream &OS) {
     }
     
     OS << "             )\n";
-    OS << "    : " << SuperName << "(attr::" << R.getName() << ", L)\n";
+    OS << "    : " << SuperName << "(attr::" << R.getName() << ", R)\n";
 
     for (ai = Args.begin(); ai != ae; ++ai) {
       OS << "              , ";
@@ -697,7 +706,7 @@ void ClangAttrPCHReadEmitter::run(raw_ostream &OS) {
       Args.push_back(A);
       A->writePCHReadDecls(OS);
     }
-    OS << "    New = new (*Context) " << R.getName() << "Attr(Loc, *Context";
+    OS << "    New = new (Context) " << R.getName() << "Attr(Range, Context";
     for (ri = Args.begin(), re = Args.end(); ri != re; ++ri) {
       OS << ", ";
       (*ri)->writePCHReadArgs(OS);
@@ -754,3 +763,26 @@ void ClangAttrSpellingListEmitter::run(raw_ostream &OS) {
   }
 
 }
+
+void ClangAttrLateParsedListEmitter::run(raw_ostream &OS) {
+  OS << "// This file is generated by TableGen. Do not edit.\n\n";
+
+  std::vector<Record*> Attrs = Records.getAllDerivedDefinitions("Attr");
+
+  for (std::vector<Record*>::iterator I = Attrs.begin(), E = Attrs.end();
+       I != E; ++I) {
+    Record &Attr = **I;
+
+    bool LateParsed = Attr.getValueAsBit("LateParsed");
+
+    if (LateParsed) {
+      std::vector<StringRef> Spellings =
+        getValueAsListOfStrings(Attr, "Spellings");
+
+      for (std::vector<StringRef>::const_iterator I = Spellings.begin(),
+           E = Spellings.end(); I != E; ++I) {
+        OS << ".Case(\"" << (*I) << "\", " << LateParsed << ")\n";
+      }
+    }
+  }
+}
diff --git a/contrib/llvm/utils/TableGen/ClangAttrEmitter.h b/contrib/llvm/tools/clang/utils/TableGen/ClangAttrEmitter.h
similarity index 87%
rename from contrib/llvm/utils/TableGen/ClangAttrEmitter.h
rename to contrib/llvm/tools/clang/utils/TableGen/ClangAttrEmitter.h
index af870098a842..5acca560f013 100644
--- a/contrib/llvm/utils/TableGen/ClangAttrEmitter.h
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangAttrEmitter.h
@@ -14,7 +14,7 @@
 #ifndef CLANGATTR_EMITTER_H
 #define CLANGATTR_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
@@ -96,6 +96,19 @@ class ClangAttrSpellingListEmitter : public TableGenBackend {
   void run(raw_ostream &OS);
 };
 
+/// ClangAttrLateParsedListEmitter emits the LateParsed property for attributes
+/// for clang.
+class ClangAttrLateParsedListEmitter : public TableGenBackend {
+  RecordKeeper &Records;
+
+ public:
+  explicit ClangAttrLateParsedListEmitter(RecordKeeper &R)
+    : Records(R)
+    {}
+
+  void run(raw_ostream &OS);
+};
+
 }
 
 #endif
diff --git a/contrib/llvm/utils/TableGen/ClangDiagnosticsEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
similarity index 96%
rename from contrib/llvm/utils/TableGen/ClangDiagnosticsEmitter.cpp
rename to contrib/llvm/tools/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index 0a48e75681fc..da2fb70b4a57 100644
--- a/contrib/llvm/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangDiagnosticsEmitter.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/ADT/DenseSet.h"
@@ -149,7 +149,7 @@ void ClangDiagsDefsEmitter::run(raw_ostream &OS) {
     // Filter by component.
     if (!Component.empty() && Component != R.getValueAsString("Component"))
       continue;
-    
+
     OS << "DIAG(" << R.getName() << ", ";
     OS << R.getValueAsDef("Class")->getName();
     OS << ", diag::" << R.getValueAsDef("DefaultMapping")->getName();
@@ -178,6 +178,22 @@ void ClangDiagsDefsEmitter::run(raw_ostream &OS) {
     else
       OS << ", false";
 
+    // FIXME: This condition is just to avoid temporary revlock, it can be
+    // removed.
+    if (R.getValue("WarningNoWerror")) {
+      // Default warning has no Werror bit.
+      if (R.getValueAsBit("WarningNoWerror"))
+        OS << ", true";
+      else
+        OS << ", false";
+  
+      // Default warning show in system header bit.
+      if (R.getValueAsBit("WarningShowInSystemHeader"))
+        OS << ", true";
+      else
+        OS << ", false";
+    }
+  
     // Category number.
     OS << ", " << CategoryIDs.getID(getDiagnosticCategory(&R, DGParentMap));
 
diff --git a/contrib/llvm/utils/TableGen/ClangDiagnosticsEmitter.h b/contrib/llvm/tools/clang/utils/TableGen/ClangDiagnosticsEmitter.h
similarity index 97%
rename from contrib/llvm/utils/TableGen/ClangDiagnosticsEmitter.h
rename to contrib/llvm/tools/clang/utils/TableGen/ClangDiagnosticsEmitter.h
index 1e4c8b70c26d..73d3c4dc0e97 100644
--- a/contrib/llvm/utils/TableGen/ClangDiagnosticsEmitter.h
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangDiagnosticsEmitter.h
@@ -14,7 +14,7 @@
 #ifndef CLANGDIAGS_EMITTER_H
 #define CLANGDIAGS_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/utils/TableGen/ClangSACheckersEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/ClangSACheckersEmitter.cpp
similarity index 99%
rename from contrib/llvm/utils/TableGen/ClangSACheckersEmitter.cpp
rename to contrib/llvm/tools/clang/utils/TableGen/ClangSACheckersEmitter.cpp
index 97739c6b3f4c..423b68a64842 100644
--- a/contrib/llvm/utils/TableGen/ClangSACheckersEmitter.cpp
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangSACheckersEmitter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangSACheckersEmitter.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/DenseSet.h"
 #include <map>
 #include <string>
diff --git a/contrib/llvm/utils/TableGen/ClangSACheckersEmitter.h b/contrib/llvm/tools/clang/utils/TableGen/ClangSACheckersEmitter.h
similarity index 94%
rename from contrib/llvm/utils/TableGen/ClangSACheckersEmitter.h
rename to contrib/llvm/tools/clang/utils/TableGen/ClangSACheckersEmitter.h
index 6bd163547329..5a0e14811124 100644
--- a/contrib/llvm/utils/TableGen/ClangSACheckersEmitter.h
+++ b/contrib/llvm/tools/clang/utils/TableGen/ClangSACheckersEmitter.h
@@ -14,7 +14,7 @@
 #ifndef CLANGSACHECKERS_EMITTER_H
 #define CLANGSACHECKERS_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/Makefile b/contrib/llvm/tools/clang/utils/TableGen/Makefile
similarity index 58%
rename from contrib/llvm/lib/Target/ARM/MCTargetDesc/Makefile
rename to contrib/llvm/tools/clang/utils/TableGen/Makefile
index 448ed9df2bff..9790efc061ef 100644
--- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/Makefile
+++ b/contrib/llvm/tools/clang/utils/TableGen/Makefile
@@ -1,4 +1,4 @@
-##===- lib/Target/ARM/TargetDesc/Makefile ------------------*- Makefile -*-===##
+##===- utils/TableGen/Makefile -----------------------------*- Makefile -*-===##
 #
 #                     The LLVM Compiler Infrastructure
 #
@@ -8,9 +8,12 @@
 ##===----------------------------------------------------------------------===##
 
 LEVEL = ../../../..
-LIBRARYNAME = LLVMARMDesc
+TOOLNAME = clang-tblgen
+USEDLIBS = LLVMTableGen.a LLVMSupport.a
+REQUIRES_EH := 1
+REQUIRES_RTTI := 1
 
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+# This tool has no plugins, optimize startup time.
+TOOL_NO_EXPORTS = 1
 
 include $(LEVEL)/Makefile.common
diff --git a/contrib/llvm/utils/TableGen/NeonEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp
similarity index 98%
rename from contrib/llvm/utils/TableGen/NeonEmitter.cpp
rename to contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp
index ef9774438f7b..66845ccacb32 100644
--- a/contrib/llvm/utils/TableGen/NeonEmitter.cpp
+++ b/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp
@@ -24,7 +24,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "NeonEmitter.h"
-#include "Error.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -485,6 +485,7 @@ static bool UseMacro(const std::string &proto) {
 /// defined as a macro should be accessed directly instead of being first
 /// assigned to a local temporary.
 static bool MacroArgUsedDirectly(const std::string &proto, unsigned i) {
+  // True for constant ints (i), pointers (p) and const pointers (c).
   return (proto[i] == 'i' || proto[i] == 'p' || proto[i] == 'c');
 }
 
@@ -525,24 +526,16 @@ static std::string GenMacroLocals(const std::string &proto, StringRef typestr) {
   for (unsigned i = 1, e = proto.size(); i != e; ++i, ++arg) {
     // Do not create a temporary for an immediate argument.
     // That would defeat the whole point of using a macro!
-    if (proto[i] == 'i')
+    // FIXME: For other (non-immediate) arguments that are used directly, a
+    // local temporary (or some other method) is still needed to get the
+    // correct type checking, even if that temporary is not used for anything.
+    // This is omitted for now because it turns out the the use of
+    // "__extension__" in the macro disables any warnings from the pointer
+    // assignment.
+    if (MacroArgUsedDirectly(proto, i))
       continue;
     generatedLocal = true;
 
-    // For other (non-immediate) arguments that are used directly, a local
-    // temporary is still needed to get the correct type checking, even though
-    // that temporary is not used for anything.
-    if (MacroArgUsedDirectly(proto, i)) {
-      s += TypeString(proto[i], typestr) + " __";
-      s.push_back(arg);
-      s += "_ = (__";
-      s.push_back(arg);
-      s += "); (void)__";
-      s.push_back(arg);
-      s += "_; ";
-      continue;
-    }
-
     s += TypeString(proto[i], typestr) + " __";
     s.push_back(arg);
     s += " = (";
@@ -1172,7 +1165,7 @@ void NeonEmitter::run(raw_ostream &OS) {
     }
   }
 
-  OS << "#define __ai static __attribute__((__always_inline__))\n\n";
+  OS<<"#define __ai static __attribute__((__always_inline__, __nodebug__))\n\n";
 
   std::vector<Record*> RV = Records.getAllDerivedDefinitions("Inst");
 
diff --git a/contrib/llvm/utils/TableGen/NeonEmitter.h b/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.h
similarity index 98%
rename from contrib/llvm/utils/TableGen/NeonEmitter.h
rename to contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.h
index 12e4e8679908..708ad3c06a0f 100644
--- a/contrib/llvm/utils/TableGen/NeonEmitter.h
+++ b/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.h
@@ -16,8 +16,8 @@
 #ifndef NEON_EMITTER_H
 #define NEON_EMITTER_H
 
-#include "Record.h"
-#include "TableGenBackend.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 
diff --git a/contrib/llvm/utils/TableGen/OptParserEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/OptParserEmitter.cpp
similarity index 99%
rename from contrib/llvm/utils/TableGen/OptParserEmitter.cpp
rename to contrib/llvm/tools/clang/utils/TableGen/OptParserEmitter.cpp
index 431026c669c3..dea22d388612 100644
--- a/contrib/llvm/utils/TableGen/OptParserEmitter.cpp
+++ b/contrib/llvm/tools/clang/utils/TableGen/OptParserEmitter.cpp
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "OptParserEmitter.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/STLExtras.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/utils/TableGen/OptParserEmitter.h b/contrib/llvm/tools/clang/utils/TableGen/OptParserEmitter.h
similarity index 96%
rename from contrib/llvm/utils/TableGen/OptParserEmitter.h
rename to contrib/llvm/tools/clang/utils/TableGen/OptParserEmitter.h
index 241a3f2b4a08..ca667caf439c 100644
--- a/contrib/llvm/utils/TableGen/OptParserEmitter.h
+++ b/contrib/llvm/tools/clang/utils/TableGen/OptParserEmitter.h
@@ -10,7 +10,7 @@
 #ifndef UTILS_TABLEGEN_OPTPARSEREMITTER_H
 #define UTILS_TABLEGEN_OPTPARSEREMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
   /// OptParserEmitter - This tablegen backend takes an input .td file
diff --git a/contrib/llvm/tools/clang/utils/TableGen/TableGen.cpp b/contrib/llvm/tools/clang/utils/TableGen/TableGen.cpp
new file mode 100644
index 000000000000..5c016d32ddd7
--- /dev/null
+++ b/contrib/llvm/tools/clang/utils/TableGen/TableGen.cpp
@@ -0,0 +1,176 @@
+//===- TableGen.cpp - Top-Level TableGen implementation for Clang ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the main function for Clang's TableGen.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangASTNodesEmitter.h"
+#include "ClangAttrEmitter.h"
+#include "ClangDiagnosticsEmitter.h"
+#include "ClangSACheckersEmitter.h"
+#include "NeonEmitter.h"
+#include "OptParserEmitter.h"
+
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/PrettyStackTrace.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Main.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenAction.h"
+
+using namespace llvm;
+
+enum ActionType {
+  GenClangAttrClasses,
+  GenClangAttrImpl,
+  GenClangAttrList,
+  GenClangAttrPCHRead,
+  GenClangAttrPCHWrite,
+  GenClangAttrSpellingList,
+  GenClangAttrLateParsedList,
+  GenClangDiagsDefs,
+  GenClangDiagGroups,
+  GenClangDiagsIndexName,
+  GenClangDeclNodes,
+  GenClangStmtNodes,
+  GenClangSACheckers,
+  GenOptParserDefs, GenOptParserImpl,
+  GenArmNeon,
+  GenArmNeonSema,
+  GenArmNeonTest
+};
+
+namespace {
+  cl::opt<ActionType>
+  Action(cl::desc("Action to perform:"),
+         cl::values(clEnumValN(GenOptParserDefs, "gen-opt-parser-defs",
+                               "Generate option definitions"),
+                    clEnumValN(GenOptParserImpl, "gen-opt-parser-impl",
+                               "Generate option parser implementation"),
+                    clEnumValN(GenClangAttrClasses, "gen-clang-attr-classes",
+                               "Generate clang attribute clases"),
+                    clEnumValN(GenClangAttrImpl, "gen-clang-attr-impl",
+                               "Generate clang attribute implementations"),
+                    clEnumValN(GenClangAttrList, "gen-clang-attr-list",
+                               "Generate a clang attribute list"),
+                    clEnumValN(GenClangAttrPCHRead, "gen-clang-attr-pch-read",
+                               "Generate clang PCH attribute reader"),
+                    clEnumValN(GenClangAttrPCHWrite, "gen-clang-attr-pch-write",
+                               "Generate clang PCH attribute writer"),
+                    clEnumValN(GenClangAttrSpellingList,
+                               "gen-clang-attr-spelling-list",
+                               "Generate a clang attribute spelling list"),
+                    clEnumValN(GenClangAttrLateParsedList,
+                               "gen-clang-attr-late-parsed-list",
+                               "Generate a clang attribute LateParsed list"),
+                    clEnumValN(GenClangDiagsDefs, "gen-clang-diags-defs",
+                               "Generate Clang diagnostics definitions"),
+                    clEnumValN(GenClangDiagGroups, "gen-clang-diag-groups",
+                               "Generate Clang diagnostic groups"),
+                    clEnumValN(GenClangDiagsIndexName,
+                               "gen-clang-diags-index-name",
+                               "Generate Clang diagnostic name index"),
+                    clEnumValN(GenClangDeclNodes, "gen-clang-decl-nodes",
+                               "Generate Clang AST declaration nodes"),
+                    clEnumValN(GenClangStmtNodes, "gen-clang-stmt-nodes",
+                               "Generate Clang AST statement nodes"),
+                    clEnumValN(GenClangSACheckers, "gen-clang-sa-checkers",
+                               "Generate Clang Static Analyzer checkers"),
+                    clEnumValN(GenArmNeon, "gen-arm-neon",
+                               "Generate arm_neon.h for clang"),
+                    clEnumValN(GenArmNeonSema, "gen-arm-neon-sema",
+                               "Generate ARM NEON sema support for clang"),
+                    clEnumValN(GenArmNeonTest, "gen-arm-neon-test",
+                               "Generate ARM NEON tests for clang"),
+                    clEnumValEnd));
+
+  cl::opt<std::string>
+  ClangComponent("clang-component",
+                 cl::desc("Only use warnings from specified component"),
+                 cl::value_desc("component"), cl::Hidden);
+}
+
+class ClangTableGenAction : public TableGenAction {
+public:
+  bool operator()(raw_ostream &OS, RecordKeeper &Records) {
+    switch (Action) {
+    case GenClangAttrClasses:
+      ClangAttrClassEmitter(Records).run(OS);
+      break;
+    case GenClangAttrImpl:
+      ClangAttrImplEmitter(Records).run(OS);
+      break;
+    case GenClangAttrList:
+      ClangAttrListEmitter(Records).run(OS);
+      break;
+    case GenClangAttrPCHRead:
+      ClangAttrPCHReadEmitter(Records).run(OS);
+      break;
+    case GenClangAttrPCHWrite:
+      ClangAttrPCHWriteEmitter(Records).run(OS);
+      break;
+    case GenClangAttrSpellingList:
+      ClangAttrSpellingListEmitter(Records).run(OS);
+      break;
+    case GenClangAttrLateParsedList:
+      ClangAttrLateParsedListEmitter(Records).run(OS);
+      break;
+    case GenClangDiagsDefs:
+      ClangDiagsDefsEmitter(Records, ClangComponent).run(OS);
+      break;
+    case GenClangDiagGroups:
+      ClangDiagGroupsEmitter(Records).run(OS);
+      break;
+    case GenClangDiagsIndexName:
+      ClangDiagsIndexNameEmitter(Records).run(OS);
+      break;
+    case GenClangDeclNodes:
+      ClangASTNodesEmitter(Records, "Decl", "Decl").run(OS);
+      ClangDeclContextEmitter(Records).run(OS);
+      break;
+    case GenClangStmtNodes:
+      ClangASTNodesEmitter(Records, "Stmt", "").run(OS);
+      break;
+    case GenClangSACheckers:
+      ClangSACheckersEmitter(Records).run(OS);
+      break;
+    case GenOptParserDefs:
+      OptParserEmitter(Records, true).run(OS);
+      break;
+    case GenOptParserImpl:
+      OptParserEmitter(Records, false).run(OS);
+      break;
+    case GenArmNeon:
+      NeonEmitter(Records).run(OS);
+      break;
+    case GenArmNeonSema:
+      NeonEmitter(Records).runHeader(OS);
+      break;
+    case GenArmNeonTest:
+      NeonEmitter(Records).runTests(OS);
+      break;
+    default:
+      assert(1 && "Invalid Action");
+      return true;
+    }
+
+    return false;
+  }
+};
+
+int main(int argc, char **argv) {
+  sys::PrintStackTraceOnErrorSignal();
+  PrettyStackTraceProgram X(argc, argv);
+  cl::ParseCommandLineOptions(argc, argv);
+
+  ClangTableGenAction Action;
+  return TableGenMain(argv[0], Action);
+}
diff --git a/contrib/llvm/utils/TableGen/ARMDecoderEmitter.cpp b/contrib/llvm/utils/TableGen/ARMDecoderEmitter.cpp
index 8a5dc8ba1543..145b96df98e2 100644
--- a/contrib/llvm/utils/TableGen/ARMDecoderEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/ARMDecoderEmitter.cpp
@@ -18,10 +18,10 @@
 
 #include "ARMDecoderEmitter.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Record.h"
 
 #include <vector>
 #include <map>
@@ -41,7 +41,7 @@ using namespace llvm;
   ENTRY(ARM_FORMAT_BRFRM,          2) \
   ENTRY(ARM_FORMAT_BRMISCFRM,      3) \
   ENTRY(ARM_FORMAT_DPFRM,          4) \
-  ENTRY(ARM_FORMAT_DPSOREGFRM,     5) \
+  ENTRY(ARM_FORMAT_DPSOREGREGFRM,     5) \
   ENTRY(ARM_FORMAT_LDFRM,          6) \
   ENTRY(ARM_FORMAT_STFRM,          7) \
   ENTRY(ARM_FORMAT_LDMISCFRM,      8) \
@@ -77,7 +77,8 @@ using namespace llvm;
   ENTRY(ARM_FORMAT_N3RegVecSh,    38) \
   ENTRY(ARM_FORMAT_NVecExtract,   39) \
   ENTRY(ARM_FORMAT_NVecMulScalar, 40) \
-  ENTRY(ARM_FORMAT_NVTBL,         41)
+  ENTRY(ARM_FORMAT_NVTBL,         41) \
+  ENTRY(ARM_FORMAT_DPSOREGIMMFRM, 42)
 
 // ARM instruction format specifies the encoding used by the instruction.
 #define ENTRY(n, v) n = v,
@@ -1614,15 +1615,6 @@ ARMDEBackend::populateInstruction(const CodeGenInstruction &CGI,
     if (!thumbInstruction(Form))
       return false;
 
-    // A8.6.189 STM / STMIA / STMEA -- Encoding T1
-    // There's only STMIA_UPD for Thumb1.
-    if (Name == "tSTMIA")
-      return false;
-
-    // On Darwin R9 is call-clobbered.  Ignore the non-Darwin counterparts.
-    if (Name == "tBL" || Name == "tBLXi" || Name == "tBLXr")
-      return false;
-
     // A8.6.25 BX.  Use the generic tBX_Rm, ignore tBX_RET and tBX_RET_vararg.
     if (Name == "tBX_RET" || Name == "tBX_RET_vararg")
       return false;
@@ -1654,14 +1646,12 @@ ARMDEBackend::populateInstruction(const CodeGenInstruction &CGI,
 
     // Resolve conflicts:
     //
-    //   tBfar conflicts with tBLr9
     //   t2LDMIA_RET conflict with t2LDM (ditto)
     //   tMOVCCi conflicts with tMOVi8
     //   tMOVCCr conflicts with tMOVgpr2gpr
     //   tLDRcp conflicts with tLDRspi
     //   t2MOVCCi16 conflicts with tMOVi16
-    if (Name == "tBfar" ||
-        Name == "t2LDMIA_RET" ||
+    if (Name == "t2LDMIA_RET" ||
         Name == "tMOVCCi" || Name == "tMOVCCr" ||
         Name == "tLDRcp" || 
         Name == "t2MOVCCi16")
diff --git a/contrib/llvm/utils/TableGen/ARMDecoderEmitter.h b/contrib/llvm/utils/TableGen/ARMDecoderEmitter.h
index 1faeb91fae8a..486f899354f4 100644
--- a/contrib/llvm/utils/TableGen/ARMDecoderEmitter.h
+++ b/contrib/llvm/utils/TableGen/ARMDecoderEmitter.h
@@ -15,9 +15,8 @@
 #ifndef ARMDECODEREMITTER_H
 #define ARMDECODEREMITTER_H
 
-#include "TableGenBackend.h"
-
 #include "llvm/Support/DataTypes.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index a6a4fecd3015..8b86c23d0632 100644
--- a/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -98,8 +98,6 @@
 
 #include "AsmMatcherEmitter.h"
 #include "CodeGenTarget.h"
-#include "Error.h"
-#include "Record.h"
 #include "StringMatcher.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/PointerUnion.h"
@@ -109,6 +107,8 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include <map>
 #include <set>
 using namespace llvm;
@@ -914,17 +914,17 @@ void AsmMatcherInfo::
 BuildRegisterClasses(SmallPtrSet<Record*, 16> &SingletonRegisters) {
   const std::vector<CodeGenRegister*> &Registers =
     Target.getRegBank().getRegisters();
-  const std::vector<CodeGenRegisterClass> &RegClassList =
-    Target.getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RegClassList =
+    Target.getRegBank().getRegClasses();
 
   // The register sets used for matching.
   std::set< std::set<Record*> > RegisterSets;
 
   // Gather the defined sets.
-  for (std::vector<CodeGenRegisterClass>::const_iterator it =
+  for (ArrayRef<CodeGenRegisterClass*>::const_iterator it =
        RegClassList.begin(), ie = RegClassList.end(); it != ie; ++it)
-    RegisterSets.insert(std::set<Record*>(it->getOrder().begin(),
-                                          it->getOrder().end()));
+    RegisterSets.insert(std::set<Record*>(
+        (*it)->getOrder().begin(), (*it)->getOrder().end()));
 
   // Add any required singleton sets.
   for (SmallPtrSet<Record*, 16>::iterator it = SingletonRegisters.begin(),
@@ -996,18 +996,23 @@ BuildRegisterClasses(SmallPtrSet<Record*, 16> &SingletonRegisters) {
   }
 
   // Name the register classes which correspond to a user defined RegisterClass.
-  for (std::vector<CodeGenRegisterClass>::const_iterator
+  for (ArrayRef<CodeGenRegisterClass*>::const_iterator
        it = RegClassList.begin(), ie = RegClassList.end(); it != ie; ++it) {
-    ClassInfo *CI = RegisterSetClasses[std::set<Record*>(it->getOrder().begin(),
-                                                         it->getOrder().end())];
+    const CodeGenRegisterClass &RC = **it;
+    // Def will be NULL for non-user defined register classes.
+    Record *Def = RC.getDef();
+    if (!Def)
+      continue;
+    ClassInfo *CI = RegisterSetClasses[std::set<Record*>(RC.getOrder().begin(),
+                                                         RC.getOrder().end())];
     if (CI->ValueName.empty()) {
-      CI->ClassName = it->getName();
-      CI->Name = "MCK_" + it->getName();
-      CI->ValueName = it->getName();
+      CI->ClassName = RC.getName();
+      CI->Name = "MCK_" + RC.getName();
+      CI->ValueName = RC.getName();
     } else
-      CI->ValueName = CI->ValueName + "," + it->getName();
+      CI->ValueName = CI->ValueName + "," + RC.getName();
 
-    RegisterClassClasses.insert(std::make_pair(it->TheDef, CI));
+    RegisterClassClasses.insert(std::make_pair(Def, CI));
   }
 
   // Populate the map for individual registers.
@@ -2172,21 +2177,14 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "\n#ifdef GET_ASSEMBLER_HEADER\n";
   OS << "#undef GET_ASSEMBLER_HEADER\n";
   OS << "  // This should be included into the middle of the declaration of\n";
-  OS << "  // your subclasses implementation of TargetAsmParser.\n";
+  OS << "  // your subclasses implementation of MCTargetAsmParser.\n";
   OS << "  unsigned ComputeAvailableFeatures(uint64_t FeatureBits) const;\n";
-  OS << "  enum MatchResultTy {\n";
-  OS << "    Match_ConversionFail,\n";
-  OS << "    Match_InvalidOperand,\n";
-  OS << "    Match_MissingFeature,\n";
-  OS << "    Match_MnemonicFail,\n";
-  OS << "    Match_Success\n";
-  OS << "  };\n";
   OS << "  bool ConvertToMCInst(unsigned Kind, MCInst &Inst, "
      << "unsigned Opcode,\n"
      << "                       const SmallVectorImpl<MCParsedAsmOperand*> "
      << "&Operands);\n";
   OS << "  bool MnemonicIsValid(StringRef Mnemonic);\n";
-  OS << "  MatchResultTy MatchInstructionImpl(\n";
+  OS << "  unsigned MatchInstructionImpl(\n";
   OS << "    const SmallVectorImpl<MCParsedAsmOperand*> &Operands,\n";
   OS << "    MCInst &Inst, unsigned &ErrorInfo);\n";
 
@@ -2328,7 +2326,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "}\n\n";
 
   // Finally, build the match function.
-  OS << Target.getName() << ClassName << "::MatchResultTy "
+  OS << "unsigned "
      << Target.getName() << ClassName << "::\n"
      << "MatchInstructionImpl(const SmallVectorImpl<MCParsedAsmOperand*>"
      << " &Operands,\n";
@@ -2355,7 +2353,9 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "  }\n\n";
 
   OS << "  // Some state to try to produce better error messages.\n";
-  OS << "  bool HadMatchOtherThanFeatures = false;\n\n";
+  OS << "  bool HadMatchOtherThanFeatures = false;\n";
+  OS << "  bool HadMatchOtherThanPredicate = false;\n";
+  OS << "  unsigned RetCode = Match_InvalidOperand;\n";
   OS << "  // Set ErrorInfo to the operand that mismatches if it is\n";
   OS << "  // wrong for all instances of the instruction.\n";
   OS << "  ErrorInfo = ~0U;\n";
@@ -2411,6 +2411,18 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "      return Match_ConversionFail;\n";
   OS << "\n";
 
+  // Verify the instruction with the target-specific match predicate function.
+  OS << "    // We have a potential match. Check the target predicate to\n"
+     << "    // handle any context sensitive constraints.\n"
+     << "    unsigned MatchResult;\n"
+     << "    if ((MatchResult = checkTargetMatchPredicate(Inst)) !="
+     << " Match_Success) {\n"
+     << "      Inst.clear();\n"
+     << "      RetCode = MatchResult;\n"
+     << "      HadMatchOtherThanPredicate = true;\n"
+     << "      continue;\n"
+     << "    }\n\n";
+
   // Call the post-processing function, if used.
   std::string InsnCleanupFn =
     AsmParser->getValueAsString("AsmParserInstCleanup");
@@ -2421,8 +2433,9 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "  }\n\n";
 
   OS << "  // Okay, we had no match.  Try to return a useful error code.\n";
-  OS << "  if (HadMatchOtherThanFeatures) return Match_MissingFeature;\n";
-  OS << "  return Match_InvalidOperand;\n";
+  OS << "  if (HadMatchOtherThanPredicate || !HadMatchOtherThanFeatures)";
+  OS << " return RetCode;\n";
+  OS << "  return Match_MissingFeature;\n";
   OS << "}\n\n";
 
   if (Info.OperandMatchInfo.size())
diff --git a/contrib/llvm/utils/TableGen/AsmMatcherEmitter.h b/contrib/llvm/utils/TableGen/AsmMatcherEmitter.h
index c13adf3dc535..e04ac103a401 100644
--- a/contrib/llvm/utils/TableGen/AsmMatcherEmitter.h
+++ b/contrib/llvm/utils/TableGen/AsmMatcherEmitter.h
@@ -15,7 +15,7 @@
 #ifndef ASMMATCHER_EMITTER_H
 #define ASMMATCHER_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 
 namespace llvm {
diff --git a/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp b/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp
index f44f050ef048..3123e11f774f 100644
--- a/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -14,13 +14,13 @@
 
 #include "AsmWriterEmitter.h"
 #include "AsmWriterInst.h"
-#include "Error.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
 #include "StringToOffsetTable.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include <algorithm>
 using namespace llvm;
 
@@ -703,8 +703,8 @@ void AsmWriterEmitter::EmitRegIsInRegClass(raw_ostream &O) {
   CodeGenTarget Target(Records);
 
   // Enumerate the register classes.
-  const std::vector<CodeGenRegisterClass> &RegisterClasses =
-    Target.getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RegisterClasses =
+    Target.getRegBank().getRegClasses();
 
   O << "namespace { // Register classes\n";
   O << "  enum RegClass {\n";
@@ -712,7 +712,7 @@ void AsmWriterEmitter::EmitRegIsInRegClass(raw_ostream &O) {
   // Emit the register enum value for each RegisterClass.
   for (unsigned I = 0, E = RegisterClasses.size(); I != E; ++I) {
     if (I != 0) O << ",\n";
-    O << "    RC_" << RegisterClasses[I].TheDef->getName();
+    O << "    RC_" << RegisterClasses[I]->getName();
   }
 
   O << "\n  };\n";
@@ -729,10 +729,10 @@ void AsmWriterEmitter::EmitRegIsInRegClass(raw_ostream &O) {
   O << "  default: break;\n";
 
   for (unsigned I = 0, E = RegisterClasses.size(); I != E; ++I) {
-    const CodeGenRegisterClass &RC = RegisterClasses[I];
+    const CodeGenRegisterClass &RC = *RegisterClasses[I];
 
     // Give the register class a legal C name if it's anonymous.
-    std::string Name = RC.TheDef->getName();
+    std::string Name = RC.getName();
     O << "  case RC_" << Name << ":\n";
   
     // Emit the register list now.
diff --git a/contrib/llvm/utils/TableGen/AsmWriterEmitter.h b/contrib/llvm/utils/TableGen/AsmWriterEmitter.h
index 84c925b66e8c..731e31cc746e 100644
--- a/contrib/llvm/utils/TableGen/AsmWriterEmitter.h
+++ b/contrib/llvm/utils/TableGen/AsmWriterEmitter.h
@@ -15,7 +15,7 @@
 #ifndef ASMWRITER_EMITTER_H
 #define ASMWRITER_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <map>
 #include <vector>
 #include <cassert>
diff --git a/contrib/llvm/utils/TableGen/AsmWriterInst.cpp b/contrib/llvm/utils/TableGen/AsmWriterInst.cpp
index fdf447f2aaf3..350a2ccfcc23 100644
--- a/contrib/llvm/utils/TableGen/AsmWriterInst.cpp
+++ b/contrib/llvm/utils/TableGen/AsmWriterInst.cpp
@@ -13,8 +13,8 @@
 
 #include "AsmWriterInst.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/TableGen/Record.h"
 
 using namespace llvm;
 
diff --git a/contrib/llvm/utils/TableGen/CallingConvEmitter.cpp b/contrib/llvm/utils/TableGen/CallingConvEmitter.cpp
index c51afd82a37a..fcdaa082fb2a 100644
--- a/contrib/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -13,8 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "CallingConvEmitter.h"
-#include "Record.h"
 #include "CodeGenTarget.h"
+#include "llvm/TableGen/Record.h"
 using namespace llvm;
 
 void CallingConvEmitter::run(raw_ostream &O) {
diff --git a/contrib/llvm/utils/TableGen/CallingConvEmitter.h b/contrib/llvm/utils/TableGen/CallingConvEmitter.h
index 431c33bcc0df..7bddd6c93eff 100644
--- a/contrib/llvm/utils/TableGen/CallingConvEmitter.h
+++ b/contrib/llvm/utils/TableGen/CallingConvEmitter.h
@@ -15,7 +15,7 @@
 #ifndef CALLINGCONV_EMITTER_H
 #define CALLINGCONV_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 
 namespace llvm {
diff --git a/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp b/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp
index d828dfc25db7..c5a152665b06 100644
--- a/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -15,7 +15,7 @@
 
 #include "CodeEmitterGen.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -41,19 +41,23 @@ void CodeEmitterGen::reverseBits(std::vector<Record*> &Insts) {
     BitsInit *BI = R->getValueAsBitsInit("Inst");
 
     unsigned numBits = BI->getNumBits();
-    BitsInit *NewBI = new BitsInit(numBits);
+ 
+    SmallVector<Init *, 16> NewBits(numBits);
+ 
     for (unsigned bit = 0, end = numBits / 2; bit != end; ++bit) {
       unsigned bitSwapIdx = numBits - bit - 1;
       Init *OrigBit = BI->getBit(bit);
       Init *BitSwap = BI->getBit(bitSwapIdx);
-      NewBI->setBit(bit, BitSwap);
-      NewBI->setBit(bitSwapIdx, OrigBit);
+      NewBits[bit]        = BitSwap;
+      NewBits[bitSwapIdx] = OrigBit;
     }
     if (numBits % 2) {
       unsigned middle = (numBits + 1) / 2;
-      NewBI->setBit(middle, BI->getBit(middle));
+      NewBits[middle] = BI->getBit(middle);
     }
 
+    BitsInit *NewBI = BitsInit::get(NewBits);
+
     // Update the bits in reversed order so that emitInstrOpBits will get the
     // correct endianness.
     R->getValue("Inst")->setValue(NewBI);
diff --git a/contrib/llvm/utils/TableGen/CodeEmitterGen.h b/contrib/llvm/utils/TableGen/CodeEmitterGen.h
index a874d970feac..7f6ee2a1b40f 100644
--- a/contrib/llvm/utils/TableGen/CodeEmitterGen.h
+++ b/contrib/llvm/utils/TableGen/CodeEmitterGen.h
@@ -14,7 +14,7 @@
 #ifndef CODEMITTERGEN_H
 #define CODEMITTERGEN_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <vector>
 #include <string>
 
diff --git a/contrib/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/contrib/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index 072893fe5abd..dbf166262bb6 100644
--- a/contrib/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/contrib/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -13,8 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenDAGPatterns.h"
-#include "Error.h"
-#include "Record.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
@@ -1747,9 +1747,10 @@ TreePatternNode *TreePattern::ParseTreePattern(Init *TheInit, StringRef OpName){
     // TreePatternNode of its own.  For example:
     ///   (foo GPR, imm) -> (foo GPR, (imm))
     if (R->isSubClassOf("SDNode") || R->isSubClassOf("PatFrag"))
-      return ParseTreePattern(new DagInit(DI, "",
-                          std::vector<std::pair<Init*, std::string> >()),
-                              OpName);
+      return ParseTreePattern(
+        DagInit::get(DI, "",
+                     std::vector<std::pair<Init*, std::string> >()),
+        OpName);
 
     // Input argument?
     TreePatternNode *Res = new TreePatternNode(DI, 1);
@@ -1771,7 +1772,7 @@ TreePatternNode *TreePattern::ParseTreePattern(Init *TheInit, StringRef OpName){
 
   if (BitsInit *BI = dynamic_cast<BitsInit*>(TheInit)) {
     // Turn this into an IntInit.
-    Init *II = BI->convertInitializerTo(new IntRecTy());
+    Init *II = BI->convertInitializerTo(IntRecTy::get());
     if (II == 0 || !dynamic_cast<IntInit*>(II))
       error("Bits value must be constants!");
     return ParseTreePattern(II, OpName);
@@ -1860,7 +1861,7 @@ TreePatternNode *TreePattern::ParseTreePattern(Init *TheInit, StringRef OpName){
     else // Otherwise, no chain.
       Operator = getDAGPatterns().get_intrinsic_wo_chain_sdnode();
 
-    TreePatternNode *IIDNode = new TreePatternNode(new IntInit(IID), 1);
+    TreePatternNode *IIDNode = new TreePatternNode(IntInit::get(IID), 1);
     Children.insert(Children.begin(), IIDNode);
   }
 
@@ -2180,7 +2181,7 @@ void CodeGenDAGPatterns::ParseDefaultOperands() {
 
   // Find some SDNode.
   assert(!SDNodes.empty() && "No SDNodes parsed?");
-  Init *SomeSDNode = new DefInit(SDNodes.begin()->first);
+  Init *SomeSDNode = DefInit::get(SDNodes.begin()->first);
 
   for (unsigned iter = 0; iter != 2; ++iter) {
     for (unsigned i = 0, e = DefaultOps[iter].size(); i != e; ++i) {
@@ -2192,7 +2193,7 @@ void CodeGenDAGPatterns::ParseDefaultOperands() {
       for (unsigned op = 0, e = DefaultInfo->getNumArgs(); op != e; ++op)
         Ops.push_back(std::make_pair(DefaultInfo->getArg(op),
                                      DefaultInfo->getArgName(op)));
-      DagInit *DI = new DagInit(SomeSDNode, "", Ops);
+      DagInit *DI = DagInit::get(SomeSDNode, "", Ops);
 
       // Create a TreePattern to parse this.
       TreePattern P(DefaultOps[iter][i], DI, false, *this);
@@ -2828,6 +2829,12 @@ void CodeGenDAGPatterns::InferInstructionFlags() {
     InstInfo.isBitcast = IsBitcast;
     InstInfo.hasSideEffects = HasSideEffects;
     InstInfo.Operands.isVariadic = IsVariadic;
+
+    // Sanity checks.
+    if (InstInfo.isReMaterializable && InstInfo.hasSideEffects)
+      throw TGError(InstInfo.TheDef->getLoc(), "The instruction " +
+                    InstInfo.TheDef->getName() +
+                    " is rematerializable AND has unmodeled side effects?");
   }
 }
 
diff --git a/contrib/llvm/utils/TableGen/CodeGenInstruction.cpp b/contrib/llvm/utils/TableGen/CodeGenInstruction.cpp
index a52ce86c485a..53d499f39553 100644
--- a/contrib/llvm/utils/TableGen/CodeGenInstruction.cpp
+++ b/contrib/llvm/utils/TableGen/CodeGenInstruction.cpp
@@ -13,8 +13,8 @@
 
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
-#include "Error.h"
-#include "Record.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/STLExtras.h"
@@ -267,8 +267,9 @@ static void ParseConstraints(const std::string &CStr, CGIOperandList &Ops) {
 
 void CGIOperandList::ProcessDisableEncoding(std::string DisableEncoding) {
   while (1) {
-    std::string OpName;
-    tie(OpName, DisableEncoding) = getToken(DisableEncoding, " ,\t");
+    std::pair<StringRef, StringRef> P = getToken(DisableEncoding, " ,\t");
+    std::string OpName = P.first;
+    DisableEncoding = P.second;
     if (OpName.empty()) break;
 
     // Figure out which operand this is.
@@ -308,6 +309,7 @@ CodeGenInstruction::CodeGenInstruction(Record *R) : TheDef(R), Operands(R) {
   isReMaterializable = R->getValueAsBit("isReMaterializable");
   hasDelaySlot = R->getValueAsBit("hasDelaySlot");
   usesCustomInserter = R->getValueAsBit("usesCustomInserter");
+  hasPostISelHook = R->getValueAsBit("hasPostISelHook");
   hasCtrlDep   = R->getValueAsBit("hasCtrlDep");
   isNotDuplicable = R->getValueAsBit("isNotDuplicable");
   hasSideEffects = R->getValueAsBit("hasSideEffects");
@@ -423,6 +425,13 @@ bool CodeGenInstAlias::tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo,
 
   // Handle explicit registers.
   if (ADI && ADI->getDef()->isSubClassOf("Register")) {
+    if (InstOpRec->isSubClassOf("OptionalDefOperand")) {
+      DagInit *DI = InstOpRec->getValueAsDag("MIOperandInfo");
+      // The operand info should only have a single (register) entry. We
+      // want the register class of it.
+      InstOpRec = dynamic_cast<DefInit*>(DI->getArg(0))->getDef();
+    }
+
     if (InstOpRec->isSubClassOf("RegisterOperand"))
       InstOpRec = InstOpRec->getValueAsDef("RegClass");
 
@@ -431,8 +440,8 @@ bool CodeGenInstAlias::tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo,
 
     if (!T.getRegisterClass(InstOpRec)
         .contains(T.getRegBank().getReg(ADI->getDef())))
-      throw TGError(Loc, "fixed register " +ADI->getDef()->getName()
-                    + " is not a member of the " + InstOpRec->getName() +
+      throw TGError(Loc, "fixed register " + ADI->getDef()->getName() +
+                    " is not a member of the " + InstOpRec->getName() +
                     " register class!");
 
     if (!Result->getArgName(AliasOpNo).empty())
diff --git a/contrib/llvm/utils/TableGen/CodeGenInstruction.h b/contrib/llvm/utils/TableGen/CodeGenInstruction.h
index 8d7669aca98c..468277aa96c2 100644
--- a/contrib/llvm/utils/TableGen/CodeGenInstruction.h
+++ b/contrib/llvm/utils/TableGen/CodeGenInstruction.h
@@ -233,6 +233,7 @@ namespace llvm {
     bool isReMaterializable;
     bool hasDelaySlot;
     bool usesCustomInserter;
+    bool hasPostISelHook;
     bool hasCtrlDep;
     bool isNotDuplicable;
     bool hasSideEffects;
diff --git a/contrib/llvm/utils/TableGen/CodeGenRegisters.cpp b/contrib/llvm/utils/TableGen/CodeGenRegisters.cpp
index 1acf3a85b600..8de461527955 100644
--- a/contrib/llvm/utils/TableGen/CodeGenRegisters.cpp
+++ b/contrib/llvm/utils/TableGen/CodeGenRegisters.cpp
@@ -14,8 +14,9 @@
 
 #include "CodeGenRegisters.h"
 #include "CodeGenTarget.h"
-#include "Error.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 
 using namespace llvm;
@@ -182,8 +183,8 @@ struct TupleExpander : SetTheory::Expander {
 
     // Precompute some types.
     Record *RegisterCl = Def->getRecords().getClass("Register");
-    RecTy *RegisterRecTy = new RecordRecTy(RegisterCl);
-    StringInit *BlankName = new StringInit("");
+    RecTy *RegisterRecTy = RecordRecTy::get(RegisterCl);
+    StringInit *BlankName = StringInit::get("");
 
     // Zip them up.
     for (unsigned n = 0; n != Length; ++n) {
@@ -195,7 +196,7 @@ struct TupleExpander : SetTheory::Expander {
         Record *Reg = Lists[i][n];
         if (i) Name += '_';
         Name += Reg->getName();
-        Tuple.push_back(new DefInit(Reg));
+        Tuple.push_back(DefInit::get(Reg));
         CostPerUse = std::max(CostPerUse,
                               unsigned(Reg->getValueAsInt("CostPerUse")));
       }
@@ -216,7 +217,7 @@ struct TupleExpander : SetTheory::Expander {
 
         // Replace the sub-register list with Tuple.
         if (RV.getName() == "SubRegs")
-          RV.setValue(new ListInit(Tuple, RegisterRecTy));
+          RV.setValue(ListInit::get(Tuple, RegisterRecTy));
 
         // Provide a blank AsmName. MC hacks are required anyway.
         if (RV.getName() == "AsmName")
@@ -224,7 +225,7 @@ struct TupleExpander : SetTheory::Expander {
 
         // CostPerUse is aggregated from all Tuple members.
         if (RV.getName() == "CostPerUse")
-          RV.setValue(new IntInit(CostPerUse));
+          RV.setValue(IntInit::get(CostPerUse));
 
         // Copy fields from the RegisterTuples def.
         if (RV.getName() == "SubRegIndices" ||
@@ -255,7 +256,7 @@ struct TupleExpander : SetTheory::Expander {
 //===----------------------------------------------------------------------===//
 
 CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R)
-  : TheDef(R) {
+  : TheDef(R), Name(R->getName()), EnumValue(-1) {
   // Rename anonymous register classes.
   if (R->getName().size() > 9 && R->getName()[9] == '.') {
     static unsigned AnonCounter = 0;
@@ -272,18 +273,22 @@ CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R)
   }
   assert(!VTs.empty() && "RegisterClass must contain at least one ValueType!");
 
+  // Allocation order 0 is the full set. AltOrders provides others.
+  const SetTheory::RecVec *Elements = RegBank.getSets().expand(R);
+  ListInit *AltOrders = R->getValueAsListInit("AltOrders");
+  Orders.resize(1 + AltOrders->size());
+
   // Default allocation order always contains all registers.
-  Elements = RegBank.getSets().expand(R);
-  for (unsigned i = 0, e = Elements->size(); i != e; ++i)
+  for (unsigned i = 0, e = Elements->size(); i != e; ++i) {
+    Orders[0].push_back((*Elements)[i]);
     Members.insert(RegBank.getReg((*Elements)[i]));
+  }
 
   // Alternative allocation orders may be subsets.
-  ListInit *Alts = R->getValueAsListInit("AltOrders");
-  AltOrders.resize(Alts->size());
   SetTheory::RecSet Order;
-  for (unsigned i = 0, e = Alts->size(); i != e; ++i) {
-    RegBank.getSets().evaluate(Alts->getElement(i), Order);
-    AltOrders[i].append(Order.begin(), Order.end());
+  for (unsigned i = 0, e = AltOrders->size(); i != e; ++i) {
+    RegBank.getSets().evaluate(AltOrders->getElement(i), Order);
+    Orders[1 + i].append(Order.begin(), Order.end());
     // Verify that all altorder members are regclass members.
     while (!Order.empty()) {
       CodeGenRegister *Reg = RegBank.getReg(Order.back());
@@ -328,10 +333,71 @@ CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R)
   AltOrderSelect = R->getValueAsCode("AltOrderSelect");
 }
 
+// Create an inferred register class that was missing from the .td files.
+// Most properties will be inherited from the closest super-class after the
+// class structure has been computed.
+CodeGenRegisterClass::CodeGenRegisterClass(StringRef Name, Key Props)
+  : Members(*Props.Members),
+    TheDef(0),
+    Name(Name),
+    EnumValue(-1),
+    SpillSize(Props.SpillSize),
+    SpillAlignment(Props.SpillAlignment),
+    CopyCost(0),
+    Allocatable(true) {
+}
+
+// Compute inherited propertied for a synthesized register class.
+void CodeGenRegisterClass::inheritProperties(CodeGenRegBank &RegBank) {
+  assert(!getDef() && "Only synthesized classes can inherit properties");
+  assert(!SuperClasses.empty() && "Synthesized class without super class");
+
+  // The last super-class is the smallest one.
+  CodeGenRegisterClass &Super = *SuperClasses.back();
+
+  // Most properties are copied directly.
+  // Exceptions are members, size, and alignment
+  Namespace = Super.Namespace;
+  VTs = Super.VTs;
+  CopyCost = Super.CopyCost;
+  Allocatable = Super.Allocatable;
+  AltOrderSelect = Super.AltOrderSelect;
+
+  // Copy all allocation orders, filter out foreign registers from the larger
+  // super-class.
+  Orders.resize(Super.Orders.size());
+  for (unsigned i = 0, ie = Super.Orders.size(); i != ie; ++i)
+    for (unsigned j = 0, je = Super.Orders[i].size(); j != je; ++j)
+      if (contains(RegBank.getReg(Super.Orders[i][j])))
+        Orders[i].push_back(Super.Orders[i][j]);
+}
+
 bool CodeGenRegisterClass::contains(const CodeGenRegister *Reg) const {
   return Members.count(Reg);
 }
 
+namespace llvm {
+  raw_ostream &operator<<(raw_ostream &OS, const CodeGenRegisterClass::Key &K) {
+    OS << "{ S=" << K.SpillSize << ", A=" << K.SpillAlignment;
+    for (CodeGenRegister::Set::const_iterator I = K.Members->begin(),
+         E = K.Members->end(); I != E; ++I)
+      OS << ", " << (*I)->getName();
+    return OS << " }";
+  }
+}
+
+// This is a simple lexicographical order that can be used to search for sets.
+// It is not the same as the topological order provided by TopoOrderRC.
+bool CodeGenRegisterClass::Key::
+operator<(const CodeGenRegisterClass::Key &B) const {
+  assert(Members && B.Members);
+  if (*Members != *B.Members)
+    return *Members < *B.Members;
+  if (SpillSize != B.SpillSize)
+    return SpillSize < B.SpillSize;
+  return SpillAlignment < B.SpillAlignment;
+}
+
 // Returns true if RC is a strict subclass.
 // RC is a sub-class of this class if it is a valid replacement for any
 // instruction operand where a register of this classis required. It must
@@ -341,16 +407,101 @@ bool CodeGenRegisterClass::contains(const CodeGenRegister *Reg) const {
 // 2. The RC spill size must not be smaller than our spill size.
 // 3. RC spill alignment must be compatible with ours.
 //
-bool CodeGenRegisterClass::hasSubClass(const CodeGenRegisterClass *RC) const {
-  return SpillAlignment && RC->SpillAlignment % SpillAlignment == 0 &&
-    SpillSize <= RC->SpillSize &&
-    std::includes(Members.begin(), Members.end(),
-                  RC->Members.begin(), RC->Members.end(),
+static bool testSubClass(const CodeGenRegisterClass *A,
+                         const CodeGenRegisterClass *B) {
+  return A->SpillAlignment && B->SpillAlignment % A->SpillAlignment == 0 &&
+    A->SpillSize <= B->SpillSize &&
+    std::includes(A->getMembers().begin(), A->getMembers().end(),
+                  B->getMembers().begin(), B->getMembers().end(),
                   CodeGenRegister::Less());
 }
 
-const std::string &CodeGenRegisterClass::getName() const {
-  return TheDef->getName();
+/// Sorting predicate for register classes.  This provides a topological
+/// ordering that arranges all register classes before their sub-classes.
+///
+/// Register classes with the same registers, spill size, and alignment form a
+/// clique.  They will be ordered alphabetically.
+///
+static int TopoOrderRC(const void *PA, const void *PB) {
+  const CodeGenRegisterClass *A = *(const CodeGenRegisterClass* const*)PA;
+  const CodeGenRegisterClass *B = *(const CodeGenRegisterClass* const*)PB;
+  if (A == B)
+    return 0;
+
+  // Order by descending set size.  Note that the classes' allocation order may
+  // not have been computed yet.  The Members set is always vaild.
+  if (A->getMembers().size() > B->getMembers().size())
+    return -1;
+  if (A->getMembers().size() < B->getMembers().size())
+    return 1;
+
+  // Order by ascending spill size.
+  if (A->SpillSize < B->SpillSize)
+    return -1;
+  if (A->SpillSize > B->SpillSize)
+    return 1;
+
+  // Order by ascending spill alignment.
+  if (A->SpillAlignment < B->SpillAlignment)
+    return -1;
+  if (A->SpillAlignment > B->SpillAlignment)
+    return 1;
+
+  // Finally order by name as a tie breaker.
+  return A->getName() < B->getName();
+}
+
+std::string CodeGenRegisterClass::getQualifiedName() const {
+  if (Namespace.empty())
+    return getName();
+  else
+    return Namespace + "::" + getName();
+}
+
+// Compute sub-classes of all register classes.
+// Assume the classes are ordered topologically.
+void CodeGenRegisterClass::computeSubClasses(CodeGenRegBank &RegBank) {
+  ArrayRef<CodeGenRegisterClass*> RegClasses = RegBank.getRegClasses();
+
+  // Visit backwards so sub-classes are seen first.
+  for (unsigned rci = RegClasses.size(); rci; --rci) {
+    CodeGenRegisterClass &RC = *RegClasses[rci - 1];
+    RC.SubClasses.resize(RegClasses.size());
+    RC.SubClasses.set(RC.EnumValue);
+
+    // Normally, all subclasses have IDs >= rci, unless RC is part of a clique.
+    for (unsigned s = rci; s != RegClasses.size(); ++s) {
+      if (RC.SubClasses.test(s))
+        continue;
+      CodeGenRegisterClass *SubRC = RegClasses[s];
+      if (!testSubClass(&RC, SubRC))
+        continue;
+      // SubRC is a sub-class. Grap all its sub-classes so we won't have to
+      // check them again.
+      RC.SubClasses |= SubRC->SubClasses;
+    }
+
+    // Sweep up missed clique members.  They will be immediately preceeding RC.
+    for (unsigned s = rci - 1; s && testSubClass(&RC, RegClasses[s - 1]); --s)
+      RC.SubClasses.set(s - 1);
+  }
+
+  // Compute the SuperClasses lists from the SubClasses vectors.
+  for (unsigned rci = 0; rci != RegClasses.size(); ++rci) {
+    const BitVector &SC = RegClasses[rci]->getSubClasses();
+    for (int s = SC.find_first(); s >= 0; s = SC.find_next(s)) {
+      if (unsigned(s) == rci)
+        continue;
+      RegClasses[s]->SuperClasses.push_back(RegClasses[rci]);
+    }
+  }
+
+  // With the class hierarchy in place, let synthesized register classes inherit
+  // properties from their closest super-class. The iteration order here can
+  // propagate properties down multiple levels.
+  for (unsigned rci = 0; rci != RegClasses.size(); ++rci)
+    if (!RegClasses[rci]->getDef())
+      RegClasses[rci]->inheritProperties(RegBank);
 }
 
 //===----------------------------------------------------------------------===//
@@ -385,14 +536,29 @@ CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records) : Records(Records) {
       getReg((*TupRegs)[j]);
   }
 
+  // Precompute all sub-register maps now all the registers are known.
+  // This will create Composite entries for all inferred sub-register indices.
+  for (unsigned i = 0, e = Registers.size(); i != e; ++i)
+    Registers[i]->getSubRegs(*this);
+
   // Read in register class definitions.
   std::vector<Record*> RCs = Records.getAllDerivedDefinitions("RegisterClass");
   if (RCs.empty())
     throw std::string("No 'RegisterClass' subclasses defined!");
 
+  // Allocate user-defined register classes.
   RegClasses.reserve(RCs.size());
   for (unsigned i = 0, e = RCs.size(); i != e; ++i)
-    RegClasses.push_back(CodeGenRegisterClass(*this, RCs[i]));
+    addToMaps(new CodeGenRegisterClass(*this, RCs[i]));
+
+  // Infer missing classes to create a full algebra.
+  computeInferredRegisterClasses();
+
+  // Order register classes topologically and assign enum values.
+  array_pod_sort(RegClasses.begin(), RegClasses.end(), TopoOrderRC);
+  for (unsigned i = 0, e = RegClasses.size(); i != e; ++i)
+    RegClasses[i]->EnumValue = i;
+  CodeGenRegisterClass::computeSubClasses(*this);
 }
 
 CodeGenRegister *CodeGenRegBank::getReg(Record *Def) {
@@ -404,11 +570,19 @@ CodeGenRegister *CodeGenRegBank::getReg(Record *Def) {
   return Reg;
 }
 
-CodeGenRegisterClass *CodeGenRegBank::getRegClass(Record *Def) {
-  if (Def2RC.empty())
-    for (unsigned i = 0, e = RegClasses.size(); i != e; ++i)
-      Def2RC[RegClasses[i].TheDef] = &RegClasses[i];
+void CodeGenRegBank::addToMaps(CodeGenRegisterClass *RC) {
+  RegClasses.push_back(RC);
 
+  if (Record *Def = RC->getDef())
+    Def2RC.insert(std::make_pair(Def, RC));
+
+  // Duplicate classes are rejected by insert().
+  // That's OK, we only care about the properties handled by CGRC::Key.
+  CodeGenRegisterClass::Key K(*RC);
+  Key2RC.insert(std::make_pair(K, RC));
+}
+
+CodeGenRegisterClass *CodeGenRegBank::getRegClass(Record *Def) {
   if (CodeGenRegisterClass *RC = Def2RC[Def])
     return RC;
 
@@ -425,7 +599,6 @@ Record *CodeGenRegBank::getCompositeSubRegIndex(Record *A, Record *B,
   // None exists, synthesize one.
   std::string Name = A->getName() + "_then_" + B->getName();
   Comp = new Record(Name, SMLoc(), Records);
-  Records.addDef(Comp);
   SubRegIndices.push_back(Comp);
   return Comp;
 }
@@ -438,11 +611,6 @@ unsigned CodeGenRegBank::getSubRegIndexNo(Record *idx) {
 }
 
 void CodeGenRegBank::computeComposites() {
-  // Precompute all sub-register maps. This will create Composite entries for
-  // all inferred sub-register indices.
-  for (unsigned i = 0, e = Registers.size(); i != e; ++i)
-    Registers[i]->getSubRegs(*this);
-
   for (unsigned i = 0, e = Registers.size(); i != e; ++i) {
     CodeGenRegister *Reg1 = Registers[i];
     const CodeGenRegister::SubRegMap &SRM1 = Reg1->getSubRegs();
@@ -571,6 +739,64 @@ void CodeGenRegBank::computeDerivedInfo() {
   computeComposites();
 }
 
+// Infer missing register classes.
+//
+// For every register class RC, make sure that the set of registers in RC with
+// a given SubIxx sub-register form a register class.
+void CodeGenRegBank::computeInferredRegisterClasses() {
+  // When this function is called, the register classes have not been sorted
+  // and assigned EnumValues yet.  That means getSubClasses(),
+  // getSuperClasses(), and hasSubClass() functions are defunct.
+
+  // Map SubRegIndex to register set.
+  typedef std::map<Record*, CodeGenRegister::Set, LessRecord> SubReg2SetMap;
+
+  // Visit all register classes, including the ones being added by the loop.
+  for (unsigned rci = 0; rci != RegClasses.size(); ++rci) {
+    CodeGenRegisterClass &RC = *RegClasses[rci];
+
+    // Compute the set of registers supporting each SubRegIndex.
+    SubReg2SetMap SRSets;
+    for (CodeGenRegister::Set::const_iterator RI = RC.getMembers().begin(),
+         RE = RC.getMembers().end(); RI != RE; ++RI) {
+      const CodeGenRegister::SubRegMap &SRM = (*RI)->getSubRegs();
+      for (CodeGenRegister::SubRegMap::const_iterator I = SRM.begin(),
+           E = SRM.end(); I != E; ++I)
+        SRSets[I->first].insert(*RI);
+    }
+
+    // Find matching classes for all SRSets entries.  Iterate in SubRegIndex
+    // numerical order to visit synthetic indices last.
+    for (unsigned sri = 0, sre = SubRegIndices.size(); sri != sre; ++sri) {
+      Record *SubIdx = SubRegIndices[sri];
+      SubReg2SetMap::const_iterator I = SRSets.find(SubIdx);
+      // Unsupported SubRegIndex. Skip it.
+      if (I == SRSets.end())
+        continue;
+      // In most cases, all RC registers support the SubRegIndex.
+      if (I->second.size() == RC.getMembers().size()) {
+        RC.setSubClassWithSubReg(SubIdx, &RC);
+        continue;
+      }
+
+      // This is a real subset.  See if we have a matching class.
+      CodeGenRegisterClass::Key K(&I->second, RC.SpillSize, RC.SpillAlignment);
+      RCKeyMap::const_iterator FoundI = Key2RC.find(K);
+      if (FoundI != Key2RC.end()) {
+        RC.setSubClassWithSubReg(SubIdx, FoundI->second);
+        continue;
+      }
+
+      // Class doesn't exist.
+      CodeGenRegisterClass *NewRC =
+        new CodeGenRegisterClass(RC.getName() + "_with_" +
+                                 I->first->getName(), K);
+      addToMaps(NewRC);
+      RC.setSubClassWithSubReg(SubIdx, NewRC);
+    }
+  }
+}
+
 /// getRegisterClassForRegister - Find the register class that contains the
 /// specified physical register.  If the register is not in a register class,
 /// return null. If the register is in multiple classes, and the classes have a
@@ -579,10 +805,10 @@ void CodeGenRegBank::computeDerivedInfo() {
 const CodeGenRegisterClass*
 CodeGenRegBank::getRegClassForRegister(Record *R) {
   const CodeGenRegister *Reg = getReg(R);
-  const std::vector<CodeGenRegisterClass> &RCs = getRegClasses();
+  ArrayRef<CodeGenRegisterClass*> RCs = getRegClasses();
   const CodeGenRegisterClass *FoundRC = 0;
   for (unsigned i = 0, e = RCs.size(); i != e; ++i) {
-    const CodeGenRegisterClass &RC = RCs[i];
+    const CodeGenRegisterClass &RC = *RCs[i];
     if (!RC.contains(Reg))
       continue;
 
diff --git a/contrib/llvm/utils/TableGen/CodeGenRegisters.h b/contrib/llvm/utils/TableGen/CodeGenRegisters.h
index 5edbf475659b..4fc34b092260 100644
--- a/contrib/llvm/utils/TableGen/CodeGenRegisters.h
+++ b/contrib/llvm/utils/TableGen/CodeGenRegisters.h
@@ -15,10 +15,11 @@
 #ifndef CODEGEN_REGISTERS_H
 #define CODEGEN_REGISTERS_H
 
-#include "Record.h"
 #include "SetTheory.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SetVector.h"
 #include <cstdlib>
@@ -69,6 +70,7 @@ namespace llvm {
     struct Less {
       bool operator()(const CodeGenRegister *A,
                       const CodeGenRegister *B) const {
+        assert(A && B);
         return A->EnumValue < B->EnumValue;
       }
     };
@@ -85,10 +87,25 @@ namespace llvm {
 
   class CodeGenRegisterClass {
     CodeGenRegister::Set Members;
-    const std::vector<Record*> *Elements;
-    std::vector<SmallVector<Record*, 16> > AltOrders;
-  public:
+    // Allocation orders. Order[0] always contains all registers in Members.
+    std::vector<SmallVector<Record*, 16> > Orders;
+    // Bit mask of sub-classes including this, indexed by their EnumValue.
+    BitVector SubClasses;
+    // List of super-classes, topologocally ordered to have the larger classes
+    // first.  This is the same as sorting by EnumValue.
+    SmallVector<CodeGenRegisterClass*, 4> SuperClasses;
     Record *TheDef;
+    std::string Name;
+
+    // For a synthesized class, inherit missing properties from the nearest
+    // super-class.
+    void inheritProperties(CodeGenRegBank&);
+
+    // Map SubRegIndex -> sub-class
+    DenseMap<Record*, CodeGenRegisterClass*> SubClassWithSubReg;
+
+  public:
+    unsigned EnumValue;
     std::string Namespace;
     std::vector<MVT::SimpleValueType> VTs;
     unsigned SpillSize;
@@ -99,7 +116,12 @@ namespace llvm {
     DenseMap<Record*,Record*> SubRegClasses;
     std::string AltOrderSelect;
 
-    const std::string &getName() const;
+    // Return the Record that defined this class, or NULL if the class was
+    // created by TableGen.
+    Record *getDef() const { return TheDef; }
+
+    const std::string &getName() const { return Name; }
+    std::string getQualifiedName() const;
     const std::vector<MVT::SimpleValueType> &getValueTypes() const {return VTs;}
     unsigned getNumValueTypes() const { return VTs.size(); }
 
@@ -122,22 +144,77 @@ namespace llvm {
     // 2. The RC spill size must not be smaller than our spill size.
     // 3. RC spill alignment must be compatible with ours.
     //
-    bool hasSubClass(const CodeGenRegisterClass *RC) const;
+    bool hasSubClass(const CodeGenRegisterClass *RC) const {
+      return SubClasses.test(RC->EnumValue);
+    }
+
+    // getSubClassWithSubReg - Returns the largest sub-class where all
+    // registers have a SubIdx sub-register.
+    CodeGenRegisterClass *getSubClassWithSubReg(Record *SubIdx) const {
+      return SubClassWithSubReg.lookup(SubIdx);
+    }
+
+    void setSubClassWithSubReg(Record *SubIdx, CodeGenRegisterClass *SubRC) {
+      SubClassWithSubReg[SubIdx] = SubRC;
+    }
+
+    // getSubClasses - Returns a constant BitVector of subclasses indexed by
+    // EnumValue.
+    // The SubClasses vector includs an entry for this class.
+    const BitVector &getSubClasses() const { return SubClasses; }
+
+    // getSuperClasses - Returns a list of super classes ordered by EnumValue.
+    // The array does not include an entry for this class.
+    ArrayRef<CodeGenRegisterClass*> getSuperClasses() const {
+      return SuperClasses;
+    }
 
     // Returns an ordered list of class members.
     // The order of registers is the same as in the .td file.
     // No = 0 is the default allocation order, No = 1 is the first alternative.
     ArrayRef<Record*> getOrder(unsigned No = 0) const {
-      if (No == 0)
-        return *Elements;
-      else
-        return AltOrders[No - 1];
+        return Orders[No];
     }
 
     // Return the total number of allocation orders available.
-    unsigned getNumOrders() const { return 1 + AltOrders.size(); }
+    unsigned getNumOrders() const { return Orders.size(); }
+
+    // Get the set of registers.  This set contains the same registers as
+    // getOrder(0).
+    const CodeGenRegister::Set &getMembers() const { return Members; }
 
     CodeGenRegisterClass(CodeGenRegBank&, Record *R);
+
+    // A key representing the parts of a register class used for forming
+    // sub-classes.  Note the ordering provided by this key is not the same as
+    // the topological order used for the EnumValues.
+    struct Key {
+      const CodeGenRegister::Set *Members;
+      unsigned SpillSize;
+      unsigned SpillAlignment;
+
+      Key(const Key &O)
+        : Members(O.Members),
+          SpillSize(O.SpillSize),
+          SpillAlignment(O.SpillAlignment) {}
+
+      Key(const CodeGenRegister::Set *M, unsigned S = 0, unsigned A = 0)
+        : Members(M), SpillSize(S), SpillAlignment(A) {}
+
+      Key(const CodeGenRegisterClass &RC)
+        : Members(&RC.getMembers()),
+          SpillSize(RC.SpillSize),
+          SpillAlignment(RC.SpillAlignment) {}
+
+      // Lexicographical order of (Members, SpillSize, SpillAlignment).
+      bool operator<(const Key&) const;
+    };
+
+    // Create a non-user defined register class.
+    CodeGenRegisterClass(StringRef Name, Key Props);
+
+    // Called by CodeGenRegBank::CodeGenRegBank().
+    static void computeSubClasses(CodeGenRegBank&);
   };
 
   // CodeGenRegBank - Represent a target's registers and the relations between
@@ -151,8 +228,17 @@ namespace llvm {
     std::vector<CodeGenRegister*> Registers;
     DenseMap<Record*, CodeGenRegister*> Def2Reg;
 
-    std::vector<CodeGenRegisterClass> RegClasses;
+    // Register classes.
+    std::vector<CodeGenRegisterClass*> RegClasses;
     DenseMap<Record*, CodeGenRegisterClass*> Def2RC;
+    typedef std::map<CodeGenRegisterClass::Key, CodeGenRegisterClass*> RCKeyMap;
+    RCKeyMap Key2RC;
+
+    // Add RC to *2RC maps.
+    void addToMaps(CodeGenRegisterClass*);
+
+    // Infer missing register classes.
+    void computeInferredRegisterClasses();
 
     // Composite SubRegIndex instances.
     // Map (SubRegIndex, SubRegIndex) -> SubRegIndex.
@@ -184,7 +270,7 @@ namespace llvm {
     // Find a register from its Record def.
     CodeGenRegister *getReg(Record*);
 
-    const std::vector<CodeGenRegisterClass> &getRegClasses() {
+    ArrayRef<CodeGenRegisterClass*> getRegClasses() const {
       return RegClasses;
     }
 
diff --git a/contrib/llvm/utils/TableGen/CodeGenTarget.cpp b/contrib/llvm/utils/TableGen/CodeGenTarget.cpp
index 929791c3182a..4a7bad7e6d85 100644
--- a/contrib/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/contrib/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -16,7 +16,7 @@
 
 #include "CodeGenTarget.h"
 #include "CodeGenIntrinsics.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/CommandLine.h"
@@ -184,9 +184,9 @@ std::vector<MVT::SimpleValueType> CodeGenTarget::
 getRegisterVTs(Record *R) const {
   const CodeGenRegister *Reg = getRegBank().getReg(R);
   std::vector<MVT::SimpleValueType> Result;
-  const std::vector<CodeGenRegisterClass> &RCs = getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RCs = getRegBank().getRegClasses();
   for (unsigned i = 0, e = RCs.size(); i != e; ++i) {
-    const CodeGenRegisterClass &RC = RCs[i];
+    const CodeGenRegisterClass &RC = *RCs[i];
     if (RC.contains(Reg)) {
       const std::vector<MVT::SimpleValueType> &InVTs = RC.getValueTypes();
       Result.insert(Result.end(), InVTs.begin(), InVTs.end());
@@ -201,10 +201,10 @@ getRegisterVTs(Record *R) const {
 
 
 void CodeGenTarget::ReadLegalValueTypes() const {
-  const std::vector<CodeGenRegisterClass> &RCs = getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RCs = getRegBank().getRegClasses();
   for (unsigned i = 0, e = RCs.size(); i != e; ++i)
-    for (unsigned ri = 0, re = RCs[i].VTs.size(); ri != re; ++ri)
-      LegalValueTypes.push_back(RCs[i].VTs[ri]);
+    for (unsigned ri = 0, re = RCs[i]->VTs.size(); ri != re; ++ri)
+      LegalValueTypes.push_back(RCs[i]->VTs[ri]);
 
   // Remove duplicates.
   std::sort(LegalValueTypes.begin(), LegalValueTypes.end());
diff --git a/contrib/llvm/utils/TableGen/CodeGenTarget.h b/contrib/llvm/utils/TableGen/CodeGenTarget.h
index 143daedae837..730216c331b4 100644
--- a/contrib/llvm/utils/TableGen/CodeGenTarget.h
+++ b/contrib/llvm/utils/TableGen/CodeGenTarget.h
@@ -19,7 +19,7 @@
 
 #include "CodeGenRegisters.h"
 #include "CodeGenInstruction.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
@@ -107,10 +107,6 @@ class CodeGenTarget {
     return RegAltNameIndices;
   }
 
-  const std::vector<CodeGenRegisterClass> &getRegisterClasses() const {
-    return getRegBank().getRegClasses();
-  }
-
   const CodeGenRegisterClass &getRegisterClass(Record *R) const {
     return *getRegBank().getRegClass(R);
   }
diff --git a/contrib/llvm/utils/TableGen/DAGISelEmitter.cpp b/contrib/llvm/utils/TableGen/DAGISelEmitter.cpp
index d66ae96cbc97..7db90034999f 100644
--- a/contrib/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -13,7 +13,7 @@
 
 #include "DAGISelEmitter.h"
 #include "DAGISelMatcher.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/Support/Debug.h"
 using namespace llvm;
 
diff --git a/contrib/llvm/utils/TableGen/DAGISelEmitter.h b/contrib/llvm/utils/TableGen/DAGISelEmitter.h
index 35ab55034308..9c9fe4273fed 100644
--- a/contrib/llvm/utils/TableGen/DAGISelEmitter.h
+++ b/contrib/llvm/utils/TableGen/DAGISelEmitter.h
@@ -14,7 +14,7 @@
 #ifndef DAGISEL_EMITTER_H
 #define DAGISEL_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include "CodeGenDAGPatterns.h"
 
 namespace llvm {
diff --git a/contrib/llvm/utils/TableGen/DAGISelMatcher.cpp b/contrib/llvm/utils/TableGen/DAGISelMatcher.cpp
index b12e1015c33b..1367e8dd6e86 100644
--- a/contrib/llvm/utils/TableGen/DAGISelMatcher.cpp
+++ b/contrib/llvm/utils/TableGen/DAGISelMatcher.cpp
@@ -10,7 +10,7 @@
 #include "DAGISelMatcher.h"
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/StringExtras.h"
 using namespace llvm;
diff --git a/contrib/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/contrib/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index acb0135422e8..3b65b2a6de0c 100644
--- a/contrib/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -13,7 +13,7 @@
 
 #include "DAGISelMatcher.h"
 #include "CodeGenDAGPatterns.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringMap.h"
diff --git a/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index c5897c72d365..49ad956f8866 100644
--- a/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -10,7 +10,7 @@
 #include "DAGISelMatcher.h"
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenRegisters.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -26,10 +26,10 @@ static MVT::SimpleValueType getRegisterValueType(Record *R,
   bool FoundRC = false;
   MVT::SimpleValueType VT = MVT::Other;
   const CodeGenRegister *Reg = T.getRegBank().getReg(R);
-  const std::vector<CodeGenRegisterClass> &RCs = T.getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RCs = T.getRegBank().getRegClasses();
 
   for (unsigned rc = 0, e = RCs.size(); rc != e; ++rc) {
-    const CodeGenRegisterClass &RC = RCs[rc];
+    const CodeGenRegisterClass &RC = *RCs[rc];
     if (!RC.contains(Reg))
       continue;
 
diff --git a/contrib/llvm/utils/TableGen/DisassemblerEmitter.cpp b/contrib/llvm/utils/TableGen/DisassemblerEmitter.cpp
index 07313d19e874..ff314e9c4f2f 100644
--- a/contrib/llvm/utils/TableGen/DisassemblerEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/DisassemblerEmitter.cpp
@@ -9,12 +9,12 @@
 
 #include "DisassemblerEmitter.h"
 #include "CodeGenTarget.h"
-#include "Error.h"
-#include "Record.h"
 #include "X86DisassemblerTables.h"
 #include "X86RecognizableInstr.h"
 #include "ARMDecoderEmitter.h"
 #include "FixedLenDecoderEmitter.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 
 using namespace llvm;
 using namespace llvm::X86Disassembler;
@@ -128,12 +128,16 @@ void DisassemblerEmitter::run(raw_ostream &OS) {
     return;
   }
 
-  // Fixed-instruction-length targets use a common disassembler.
-  // ARM use its own implementation for now.
-  if (Target.getName() == "ARM") {
-    ARMDecoderEmitter(Records).run(OS);
+  // ARM and Thumb have a CHECK() macro to deal with DecodeStatuses.
+  if (Target.getName() == "ARM" ||
+      Target.getName() == "Thumb") {
+    FixedLenDecoderEmitter(Records,
+                           "ARM",
+                           "if (!Check(S, ", ")) return MCDisassembler::Fail;",
+                           "S", "MCDisassembler::Fail",
+                           "  MCDisassembler::DecodeStatus S = MCDisassembler::Success;\n(void)S;").run(OS);
     return;
-  }  
+  }
 
-  FixedLenDecoderEmitter(Records).run(OS);
+  FixedLenDecoderEmitter(Records, Target.getName()).run(OS);
 }
diff --git a/contrib/llvm/utils/TableGen/DisassemblerEmitter.h b/contrib/llvm/utils/TableGen/DisassemblerEmitter.h
index 7229d81649ee..63ee55264ad8 100644
--- a/contrib/llvm/utils/TableGen/DisassemblerEmitter.h
+++ b/contrib/llvm/utils/TableGen/DisassemblerEmitter.h
@@ -10,7 +10,7 @@
 #ifndef DISASSEMBLEREMITTER_H
 #define DISASSEMBLEREMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/utils/TableGen/EDEmitter.cpp b/contrib/llvm/utils/TableGen/EDEmitter.cpp
index 2f9814aee484..abef70e31897 100644
--- a/contrib/llvm/utils/TableGen/EDEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/EDEmitter.cpp
@@ -17,8 +17,8 @@
 
 #include "AsmWriterInst.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
 
+#include "llvm/TableGen/Record.h"
 #include "llvm/MC/EDInstInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
@@ -256,12 +256,15 @@ static int X86TypeFromOpName(LiteralConstantEmitter *type,
   REG("GR8");
   REG("GR8_NOREX");
   REG("GR16");
+  REG("GR16_NOAX");
   REG("GR32");
+  REG("GR32_NOAX");
   REG("GR32_NOREX");
   REG("GR32_TC");
   REG("FR32");
   REG("RFP32");
   REG("GR64");
+  REG("GR64_NOAX");
   REG("GR64_TC");
   REG("FR64");
   REG("VR64");
@@ -279,6 +282,7 @@ static int X86TypeFromOpName(LiteralConstantEmitter *type,
   IMM("i16i8imm");
   IMM("i32imm");
   IMM("i32i8imm");
+  IMM("u32u8imm");
   IMM("i64imm");
   IMM("i64i8imm");
   IMM("i64i32imm");
@@ -554,6 +558,8 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
                              const std::string &name) {
   REG("GPR");
   REG("rGPR");
+  REG("GPRnopc");
+  REG("GPRsp");
   REG("tcGPR");
   REG("cc_out");
   REG("s_cc_out");
@@ -575,6 +581,7 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
   IMM("nohash_imm");
   IMM("p_imm");
   IMM("c_imm");
+  IMM("coproc_option_imm");
   IMM("imod_op");
   IMM("iflags_op");
   IMM("cpinst_operand");
@@ -587,26 +594,40 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
   IMM("neg_zero");
   IMM("imm0_31");
   IMM("imm0_31_m1");
+  IMM("imm1_16");
+  IMM("imm1_32");
   IMM("nModImm");
   IMM("imm0_7");
   IMM("imm0_15");
   IMM("imm0_255");
   IMM("imm0_4095");
   IMM("imm0_65535");
+  IMM("imm0_65535_expr");
+  IMM("imm24b");
+  IMM("pkh_lsl_amt");
+  IMM("pkh_asr_amt");
   IMM("jt2block_operand");
-  IMM("t_imm_s4");
+  IMM("t_imm0_1020s4");
+  IMM("t_imm0_508s4");
   IMM("pclabel");
   IMM("adrlabel");
   IMM("t_adrlabel");
   IMM("t2adrlabel");
   IMM("shift_imm");
-  IMM("ssat_imm");
+  IMM("t2_shift_imm");
   IMM("neon_vcvt_imm32");
   IMM("shr_imm8");
   IMM("shr_imm16");
   IMM("shr_imm32");
   IMM("shr_imm64");
   IMM("t2ldrlabel");
+  IMM("postidx_imm8");
+  IMM("postidx_imm8s4");
+  IMM("imm_sr");
+  IMM("imm1_31");
+  IMM("VectorIndex8");
+  IMM("VectorIndex16");
+  IMM("VectorIndex32");
 
   MISC("brtarget", "kOperandTypeARMBranchTarget");                // ?
   MISC("uncondbrtarget", "kOperandTypeARMBranchTarget");           // ?
@@ -617,11 +638,14 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
 
   MISC("br_target", "kOperandTypeARMBranchTarget");                // ?
   MISC("bl_target", "kOperandTypeARMBranchTarget");                // ?
+  MISC("blx_target", "kOperandTypeARMBranchTarget");                // ?
 
   MISC("t_bltarget", "kOperandTypeARMBranchTarget");              // ?
   MISC("t_blxtarget", "kOperandTypeARMBranchTarget");             // ?
-  MISC("so_reg", "kOperandTypeARMSoReg");                         // R, R, I
-  MISC("shift_so_reg", "kOperandTypeARMSoReg");                   // R, R, I
+  MISC("so_reg_imm", "kOperandTypeARMSoRegReg");                         // R, R, I
+  MISC("so_reg_reg", "kOperandTypeARMSoRegImm");                         // R, R, I
+  MISC("shift_so_reg_reg", "kOperandTypeARMSoRegReg");                   // R, R, I
+  MISC("shift_so_reg_imm", "kOperandTypeARMSoRegImm");                   // R, R, I
   MISC("t2_so_reg", "kOperandTypeThumb2SoReg");                   // R, I
   MISC("so_imm", "kOperandTypeARMSoImm");                         // I
   MISC("rot_imm", "kOperandTypeARMRotImm");                       // I
@@ -631,8 +655,10 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
   MISC("it_pred", "kOperandTypeARMPredicate");                    // I
   MISC("addrmode_imm12", "kOperandTypeAddrModeImm12");            // R, I
   MISC("ldst_so_reg", "kOperandTypeLdStSOReg");                   // R, R, I
+  MISC("postidx_reg", "kOperandTypeARMAddrMode3Offset");          // R, I
   MISC("addrmode2", "kOperandTypeARMAddrMode2");                  // R, R, I
-  MISC("am2offset", "kOperandTypeARMAddrMode2Offset");            // R, I
+  MISC("am2offset_reg", "kOperandTypeARMAddrMode2Offset");        // R, I
+  MISC("am2offset_imm", "kOperandTypeARMAddrMode2Offset");        // R, I
   MISC("addrmode3", "kOperandTypeARMAddrMode3");                  // R, R, I
   MISC("am3offset", "kOperandTypeARMAddrMode3Offset");            // R, I
   MISC("ldstm_mode", "kOperandTypeARMLdStmMode");                 // I
@@ -642,17 +668,20 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
   MISC("addrmode6dup", "kOperandTypeARMAddrMode6");               // R, R, I, I
   MISC("addrmode6oneL32", "kOperandTypeARMAddrMode6");            // R, R, I, I
   MISC("addrmodepc", "kOperandTypeARMAddrModePC");                // R, I
-  MISC("addrmode7", "kOperandTypeARMAddrMode7");                  // R
+  MISC("addr_offset_none", "kOperandTypeARMAddrMode7");           // R
   MISC("reglist", "kOperandTypeARMRegisterList");                 // I, R, ...
   MISC("dpr_reglist", "kOperandTypeARMDPRRegisterList");          // I, R, ...
   MISC("spr_reglist", "kOperandTypeARMSPRRegisterList");          // I, R, ...
   MISC("it_mask", "kOperandTypeThumbITMask");                     // I
   MISC("t2addrmode_reg", "kOperandTypeThumb2AddrModeReg");        // R
+  MISC("t2addrmode_posimm8", "kOperandTypeThumb2AddrModeImm8");   // R, I
+  MISC("t2addrmode_negimm8", "kOperandTypeThumb2AddrModeImm8");   // R, I
   MISC("t2addrmode_imm8", "kOperandTypeThumb2AddrModeImm8");      // R, I
   MISC("t2am_imm8_offset", "kOperandTypeThumb2AddrModeImm8Offset");//I
   MISC("t2addrmode_imm12", "kOperandTypeThumb2AddrModeImm12");    // R, I
   MISC("t2addrmode_so_reg", "kOperandTypeThumb2AddrModeSoReg");   // R, R, I
   MISC("t2addrmode_imm8s4", "kOperandTypeThumb2AddrModeImm8s4");  // R, I
+  MISC("t2addrmode_imm0_1020s4", "kOperandTypeThumb2AddrModeImm8s4");  // R, I
   MISC("t2am_imm8s4_offset", "kOperandTypeThumb2AddrModeImm8s4Offset");
                                                                   // R, I
   MISC("tb_addrmode", "kOperandTypeARMTBAddrMode");               // I
@@ -665,6 +694,8 @@ static int ARMFlagFromOpName(LiteralConstantEmitter *type,
   MISC("t_addrmode_rr", "kOperandTypeThumbAddrModeRR");           // R, R
   MISC("t_addrmode_sp", "kOperandTypeThumbAddrModeSP");           // R, I
   MISC("t_addrmode_pc", "kOperandTypeThumbAddrModePC");           // R, I
+  MISC("addrmode_tbb", "kOperandTypeThumbAddrModeRR");            // R, R
+  MISC("addrmode_tbh", "kOperandTypeThumbAddrModeRR");            // R, R
 
   return 1;
 }
@@ -772,11 +803,6 @@ static void populateInstInfo(CompoundConstantEmitter &infoArray,
   for (index = 0; index < numInstructions; ++index) {
     const CodeGenInstruction& inst = *numberedInstructions[index];
 
-    // We don't need to do anything for pseudo-instructions, as we'll never
-    // see them here. We'll only see real instructions.
-    if (inst.isPseudo)
-      continue;
-
     CompoundConstantEmitter *infoStruct = new CompoundConstantEmitter;
     infoArray.addEntry(infoStruct);
 
@@ -809,15 +835,20 @@ static void populateInstInfo(CompoundConstantEmitter &infoArray,
 
     unsigned numSyntaxes = 0;
 
-    if (target.getName() == "X86") {
-      X86PopulateOperands(operandTypes, inst);
-      X86ExtractSemantics(*instType, operandFlags, inst);
-      numSyntaxes = 2;
-    }
-    else if (target.getName() == "ARM") {
-      ARMPopulateOperands(operandTypes, inst);
-      ARMExtractSemantics(*instType, operandTypes, operandFlags, inst);
-      numSyntaxes = 1;
+    // We don't need to do anything for pseudo-instructions, as we'll never
+    // see them here. We'll only see real instructions.
+    // We still need to emit null initializers for everything.
+    if (!inst.isPseudo) {
+      if (target.getName() == "X86") {
+        X86PopulateOperands(operandTypes, inst);
+        X86ExtractSemantics(*instType, operandFlags, inst);
+        numSyntaxes = 2;
+      }
+      else if (target.getName() == "ARM") {
+        ARMPopulateOperands(operandTypes, inst);
+        ARMExtractSemantics(*instType, operandTypes, operandFlags, inst);
+        numSyntaxes = 1;
+      }
     }
 
     CompoundConstantEmitter *operandOrderArray = new CompoundConstantEmitter;
@@ -850,7 +881,8 @@ static void emitCommonEnums(raw_ostream &o, unsigned int &i) {
   operandTypes.addEntry("kOperandTypeX86EffectiveAddress");
   operandTypes.addEntry("kOperandTypeX86PCRelative");
   operandTypes.addEntry("kOperandTypeARMBranchTarget");
-  operandTypes.addEntry("kOperandTypeARMSoReg");
+  operandTypes.addEntry("kOperandTypeARMSoRegReg");
+  operandTypes.addEntry("kOperandTypeARMSoRegImm");
   operandTypes.addEntry("kOperandTypeARMSoImm");
   operandTypes.addEntry("kOperandTypeARMRotImm");
   operandTypes.addEntry("kOperandTypeARMSoImm2Part");
diff --git a/contrib/llvm/utils/TableGen/EDEmitter.h b/contrib/llvm/utils/TableGen/EDEmitter.h
index e30373fed2eb..f26837554722 100644
--- a/contrib/llvm/utils/TableGen/EDEmitter.h
+++ b/contrib/llvm/utils/TableGen/EDEmitter.h
@@ -16,7 +16,7 @@
 #ifndef SEMANTIC_INFO_EMITTER_H
 #define SEMANTIC_INFO_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
   
diff --git a/contrib/llvm/utils/TableGen/FastISelEmitter.cpp b/contrib/llvm/utils/TableGen/FastISelEmitter.cpp
index f54e8df40f2d..9fdc2e33a546 100644
--- a/contrib/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -18,8 +18,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "FastISelEmitter.h"
-#include "Error.h"
-#include "Record.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/VectorExtras.h"
 #include "llvm/Support/Debug.h"
@@ -504,7 +504,7 @@ void FastISelMap::collectPatterns(CodeGenDAGPatterns &CGP) {
 
     std::vector<std::string>* PhysRegInputs = new std::vector<std::string>();
     if (InstPatNode->getOperator()->getName() == "imm" ||
-        InstPatNode->getOperator()->getName() == "fpimmm")
+        InstPatNode->getOperator()->getName() == "fpimm")
       PhysRegInputs->push_back("");
     else {
       // Compute the PhysRegs used by the given pattern, and check that
diff --git a/contrib/llvm/utils/TableGen/FastISelEmitter.h b/contrib/llvm/utils/TableGen/FastISelEmitter.h
index ce4e77e6f8f2..4f75ac1fd9d2 100644
--- a/contrib/llvm/utils/TableGen/FastISelEmitter.h
+++ b/contrib/llvm/utils/TableGen/FastISelEmitter.h
@@ -14,8 +14,8 @@
 #ifndef FASTISEL_EMITTER_H
 #define FASTISEL_EMITTER_H
 
-#include "TableGenBackend.h"
 #include "CodeGenDAGPatterns.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
index c9dcb01de008..02b966a21431 100644
--- a/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
@@ -16,7 +16,7 @@
 
 #include "FixedLenDecoderEmitter.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -84,11 +84,8 @@ static BitsInit &getBitsField(const Record &def, const char *str) {
 // Forward declaration.
 class FilterChooser;
 
-// FIXME: Possibly auto-detected?
-#define BIT_WIDTH 32
-
 // Representation of the instruction to work on.
-typedef bit_value_t insn_t[BIT_WIDTH];
+typedef std::vector<bit_value_t> insn_t;
 
 /// Filter - Filter works with FilterChooser to produce the decoding tree for
 /// the ISA.
@@ -230,7 +227,7 @@ class FilterChooser {
 
   // Array of bit values passed down from our parent.
   // Set to all BIT_UNFILTERED's for Parent == NULL.
-  bit_value_t FilterBitValues[BIT_WIDTH];
+  std::vector<bit_value_t> FilterBitValues;
 
   // Links to the FilterChooser above us in the decoding tree.
   FilterChooser *Parent;
@@ -238,21 +235,29 @@ class FilterChooser {
   // Index of the best filter from Filters.
   int BestIndex;
 
+  // Width of instructions
+  unsigned BitWidth;
+
+  // Parent emitter
+  const FixedLenDecoderEmitter *Emitter;
+
 public:
   FilterChooser(const FilterChooser &FC) :
     AllInstructions(FC.AllInstructions), Opcodes(FC.Opcodes),
-      Operands(FC.Operands), Filters(FC.Filters), Parent(FC.Parent),
-      BestIndex(FC.BestIndex) {
-    memcpy(FilterBitValues, FC.FilterBitValues, sizeof(FilterBitValues));
-  }
+      Operands(FC.Operands), Filters(FC.Filters),
+      FilterBitValues(FC.FilterBitValues), Parent(FC.Parent),
+    BestIndex(FC.BestIndex), BitWidth(FC.BitWidth),
+    Emitter(FC.Emitter) { }
 
   FilterChooser(const std::vector<const CodeGenInstruction*> &Insts,
                 const std::vector<unsigned> &IDs,
-    std::map<unsigned, std::vector<OperandInfo> > &Ops) :
+    std::map<unsigned, std::vector<OperandInfo> > &Ops,
+                unsigned BW,
+                const FixedLenDecoderEmitter *E) :
       AllInstructions(Insts), Opcodes(IDs), Operands(Ops), Filters(),
-      Parent(NULL), BestIndex(-1) {
-    for (unsigned i = 0; i < BIT_WIDTH; ++i)
-      FilterBitValues[i] = BIT_UNFILTERED;
+      Parent(NULL), BestIndex(-1), BitWidth(BW), Emitter(E) {
+    for (unsigned i = 0; i < BitWidth; ++i)
+      FilterBitValues.push_back(BIT_UNFILTERED);
 
     doFilter();
   }
@@ -260,13 +265,12 @@ class FilterChooser {
   FilterChooser(const std::vector<const CodeGenInstruction*> &Insts,
                 const std::vector<unsigned> &IDs,
         std::map<unsigned, std::vector<OperandInfo> > &Ops,
-                bit_value_t (&ParentFilterBitValues)[BIT_WIDTH],
+                std::vector<bit_value_t> &ParentFilterBitValues,
                 FilterChooser &parent) :
       AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
-      Filters(), Parent(&parent), BestIndex(-1) {
-    for (unsigned i = 0; i < BIT_WIDTH; ++i)
-      FilterBitValues[i] = ParentFilterBitValues[i];
-
+      Filters(), FilterBitValues(ParentFilterBitValues),
+      Parent(&parent), BestIndex(-1), BitWidth(parent.BitWidth),
+      Emitter(parent.Emitter) {
     doFilter();
   }
 
@@ -274,15 +278,15 @@ class FilterChooser {
   bool isTopLevel() { return Parent == NULL; }
 
   // Emit the top level typedef and decodeInstruction() function.
-  void emitTop(raw_ostream &o, unsigned Indentation);
+  void emitTop(raw_ostream &o, unsigned Indentation, std::string Namespace);
 
 protected:
   // Populates the insn given the uid.
   void insnWithID(insn_t &Insn, unsigned Opcode) const {
     BitsInit &Bits = getBitsField(*AllInstructions[Opcode]->TheDef, "Inst");
 
-    for (unsigned i = 0; i < BIT_WIDTH; ++i)
-      Insn[i] = bitFromBits(Bits, i);
+    for (unsigned i = 0; i < BitWidth; ++i)
+      Insn.push_back(bitFromBits(Bits, i));
   }
 
   // Returns the record name.
@@ -300,7 +304,7 @@ class FilterChooser {
 
   /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
   /// filter array as a series of chars.
-  void dumpFilterArray(raw_ostream &o, bit_value_t (&filter)[BIT_WIDTH]);
+  void dumpFilterArray(raw_ostream &o, std::vector<bit_value_t> & filter);
 
   /// dumpStack - dumpStack traverses the filter chooser chain and calls
   /// dumpFilterArray on each filter chooser up to the top level one.
@@ -326,6 +330,10 @@ class FilterChooser {
       std::vector<unsigned> &EndBits, std::vector<uint64_t> &FieldVals,
       insn_t &Insn);
 
+  // Emits code to check the Predicates member of an instruction are true.
+  // Returns true if predicate matches were emitted, false otherwise.
+  bool emitPredicateMatch(raw_ostream &o, unsigned &Indentation,unsigned Opc);
+
   // Emits code to decode the singleton.  Return true if we have matched all the
   // well-known bits.
   bool emitSingletonDecoder(raw_ostream &o, unsigned &Indentation,unsigned Opc);
@@ -333,6 +341,9 @@ class FilterChooser {
   // Emits code to decode the singleton, and then to decode the rest.
   void emitSingletonDecoder(raw_ostream &o, unsigned &Indentation,Filter &Best);
 
+  void emitBinaryParser(raw_ostream &o , unsigned &Indentation,
+                        OperandInfo &OpInfo);
+
   // Assign a single filter and run with it.
   void runSingleFilter(FilterChooser &owner, unsigned startBit, unsigned numBit,
       bool mixed);
@@ -375,7 +386,7 @@ Filter::Filter(const Filter &f) :
 Filter::Filter(FilterChooser &owner, unsigned startBit, unsigned numBits,
     bool mixed) : Owner(&owner), StartBit(startBit), NumBits(numBits),
                   Mixed(mixed) {
-  assert(StartBit + NumBits - 1 < BIT_WIDTH);
+  assert(StartBit + NumBits - 1 < Owner->BitWidth);
 
   NumFiltered = 0;
   LastOpcFiltered = 0;
@@ -427,9 +438,8 @@ Filter::~Filter() {
 void Filter::recurse() {
   std::map<uint64_t, std::vector<unsigned> >::const_iterator mapIterator;
 
-  bit_value_t BitValueArray[BIT_WIDTH];
   // Starts by inheriting our parent filter chooser's filter bit values.
-  memcpy(BitValueArray, Owner->FilterBitValues, sizeof(BitValueArray));
+  std::vector<bit_value_t> BitValueArray(Owner->FilterBitValues);
 
   unsigned bitIndex;
 
@@ -493,8 +503,9 @@ void Filter::emit(raw_ostream &o, unsigned &Indentation) {
 
   o << StartBit << "} ...\n";
 
-  o.indent(Indentation) << "switch (fieldFromInstruction(insn, "
-                        << StartBit << ", " << NumBits << ")) {\n";
+  o.indent(Indentation) << "switch (fieldFromInstruction" << Owner->BitWidth
+                        << "(insn, " << StartBit << ", "
+                        << NumBits << ")) {\n";
 
   std::map<unsigned, FilterChooser*>::iterator filterIterator;
 
@@ -559,68 +570,21 @@ unsigned Filter::usefulness() const {
 //////////////////////////////////
 
 // Emit the top level typedef and decodeInstruction() function.
-void FilterChooser::emitTop(raw_ostream &o, unsigned Indentation) {
-  switch (BIT_WIDTH) {
-  case 8:
-    o.indent(Indentation) << "typedef uint8_t field_t;\n";
-    break;
-  case 16:
-    o.indent(Indentation) << "typedef uint16_t field_t;\n";
-    break;
-  case 32:
-    o.indent(Indentation) << "typedef uint32_t field_t;\n";
-    break;
-  case 64:
-    o.indent(Indentation) << "typedef uint64_t field_t;\n";
-    break;
-  default:
-    assert(0 && "Unexpected instruction size!");
-  }
-
-  o << '\n';
-
-  o.indent(Indentation) << "static field_t " <<
-    "fieldFromInstruction(field_t insn, unsigned startBit, unsigned numBits)\n";
-
-  o.indent(Indentation) << "{\n";
-
-  ++Indentation; ++Indentation;
-  o.indent(Indentation) << "assert(startBit + numBits <= " << BIT_WIDTH
-                        << " && \"Instruction field out of bounds!\");\n";
-  o << '\n';
-  o.indent(Indentation) << "field_t fieldMask;\n";
-  o << '\n';
-  o.indent(Indentation) << "if (numBits == " << BIT_WIDTH << ")\n";
-
-  ++Indentation; ++Indentation;
-  o.indent(Indentation) << "fieldMask = (field_t)-1;\n";
-  --Indentation; --Indentation;
-
-  o.indent(Indentation) << "else\n";
-
-  ++Indentation; ++Indentation;
-  o.indent(Indentation) << "fieldMask = ((1 << numBits) - 1) << startBit;\n";
-  --Indentation; --Indentation;
-
-  o << '\n';
-  o.indent(Indentation) << "return (insn & fieldMask) >> startBit;\n";
-  --Indentation; --Indentation;
-
-  o.indent(Indentation) << "}\n";
-
-  o << '\n';
-
+void FilterChooser::emitTop(raw_ostream &o, unsigned Indentation,
+                            std::string Namespace) {
   o.indent(Indentation) <<
-    "static bool decodeInstruction(MCInst &MI, field_t insn, "
-    "uint64_t Address, const void *Decoder) {\n";
-  o.indent(Indentation) << "  unsigned tmp = 0;\n";
+    "static MCDisassembler::DecodeStatus decode" << Namespace << "Instruction" << BitWidth
+    << "(MCInst &MI, uint" << BitWidth << "_t insn, uint64_t Address, "
+    << "const void *Decoder, const MCSubtargetInfo &STI) {\n";
+  o.indent(Indentation) << "  unsigned tmp = 0;\n  (void)tmp;\n" << Emitter->Locals << "\n";
+  o.indent(Indentation) << "  uint64_t Bits = STI.getFeatureBits();\n";
 
   ++Indentation; ++Indentation;
   // Emits code to decode the instructions.
   emit(o, Indentation);
 
   o << '\n';
-  o.indent(Indentation) << "return false;\n";
+  o.indent(Indentation) << "return " << Emitter->ReturnFail << ";\n";
   --Indentation; --Indentation;
 
   o.indent(Indentation) << "}\n";
@@ -651,10 +615,10 @@ bool FilterChooser::fieldFromInsn(uint64_t &Field, insn_t &Insn,
 /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
 /// filter array as a series of chars.
 void FilterChooser::dumpFilterArray(raw_ostream &o,
-                                    bit_value_t (&filter)[BIT_WIDTH]) {
+                                    std::vector<bit_value_t> &filter) {
   unsigned bitIndex;
 
-  for (bitIndex = BIT_WIDTH; bitIndex > 0; bitIndex--) {
+  for (bitIndex = BitWidth; bitIndex > 0; bitIndex--) {
     switch (filter[bitIndex - 1]) {
     case BIT_UNFILTERED:
       o << ".";
@@ -727,7 +691,7 @@ unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits,
   int State = 0;
   int Val = -1;
 
-  for (unsigned i = 0; i < BIT_WIDTH; ++i) {
+  for (unsigned i = 0; i < BitWidth; ++i) {
     Val = Value(Insn[i]);
     bool Filtered = PositionFiltered(i);
     switch (State) {
@@ -761,7 +725,7 @@ unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits,
   }
   // If we are still in Island after the loop, do some housekeeping.
   if (State == 2) {
-    EndBits.push_back(BIT_WIDTH - 1);
+    EndBits.push_back(BitWidth - 1);
     FieldVals.push_back(FieldVal);
     ++Num;
   }
@@ -771,6 +735,70 @@ unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits,
   return Num;
 }
 
+void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation,
+                         OperandInfo &OpInfo) {
+  std::string &Decoder = OpInfo.Decoder;
+
+  if (OpInfo.numFields() == 1) {
+    OperandInfo::iterator OI = OpInfo.begin();
+    o.indent(Indentation) << "  tmp = fieldFromInstruction" << BitWidth
+                            << "(insn, " << OI->Base << ", " << OI->Width
+                            << ");\n";
+  } else {
+    o.indent(Indentation) << "  tmp = 0;\n";
+    for (OperandInfo::iterator OI = OpInfo.begin(), OE = OpInfo.end();
+         OI != OE; ++OI) {
+      o.indent(Indentation) << "  tmp |= (fieldFromInstruction" << BitWidth
+                            << "(insn, " << OI->Base << ", " << OI->Width
+                            << ") << " << OI->Offset << ");\n";
+    }
+  }
+
+  if (Decoder != "")
+    o.indent(Indentation) << "  " << Emitter->GuardPrefix << Decoder
+                          << "(MI, tmp, Address, Decoder)" << Emitter->GuardPostfix << "\n";
+  else
+    o.indent(Indentation) << "  MI.addOperand(MCOperand::CreateImm(tmp));\n";
+
+}
+
+static void emitSinglePredicateMatch(raw_ostream &o, StringRef str,
+                                     std::string PredicateNamespace) {
+  if (str[0] == '!')
+    o << "!(Bits & " << PredicateNamespace << "::"
+      << str.slice(1,str.size()) << ")";
+  else
+    o << "(Bits & " << PredicateNamespace << "::" << str << ")";
+}
+
+bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation,
+                                           unsigned Opc) {
+  ListInit *Predicates = AllInstructions[Opc]->TheDef->getValueAsListInit("Predicates");
+  for (unsigned i = 0; i < Predicates->getSize(); ++i) {
+    Record *Pred = Predicates->getElementAsRecord(i);
+    if (!Pred->getValue("AssemblerMatcherPredicate"))
+      continue;
+
+    std::string P = Pred->getValueAsString("AssemblerCondString");
+
+    if (!P.length())
+      continue;
+
+    if (i != 0)
+      o << " && ";
+
+    StringRef SR(P);
+    std::pair<StringRef, StringRef> pairs = SR.split(',');
+    while (pairs.second.size()) {
+      emitSinglePredicateMatch(o, pairs.first, Emitter->PredicateNamespace);
+      o << " && ";
+      pairs = pairs.second.split(',');
+    }
+    emitSinglePredicateMatch(o, pairs.first, Emitter->PredicateNamespace);
+  }
+  return Predicates->getSize() > 0;
+}
+
 // Emits code to decode the singleton.  Return true if we have matched all the
 // well-known bits.
 bool FilterChooser::emitSingletonDecoder(raw_ostream &o, unsigned &Indentation,
@@ -789,33 +817,27 @@ bool FilterChooser::emitSingletonDecoder(raw_ostream &o, unsigned &Indentation,
 
   // If we have matched all the well-known bits, just issue a return.
   if (Size == 0) {
-    o.indent(Indentation) << "{\n";
+    o.indent(Indentation) << "if (";
+    if (!emitPredicateMatch(o, Indentation, Opc))
+      o << "1";
+    o << ") {\n";
     o.indent(Indentation) << "  MI.setOpcode(" << Opc << ");\n";
     std::vector<OperandInfo>& InsnOperands = Operands[Opc];
     for (std::vector<OperandInfo>::iterator
          I = InsnOperands.begin(), E = InsnOperands.end(); I != E; ++I) {
       // If a custom instruction decoder was specified, use that.
-      if (I->FieldBase == ~0U && I->FieldLength == ~0U) {
-        o.indent(Indentation) << "  " << I->Decoder
-                              << "(MI, insn, Address, Decoder);\n";
+      if (I->numFields() == 0 && I->Decoder.size()) {
+        o.indent(Indentation) << "  " << Emitter->GuardPrefix << I->Decoder
+                              << "(MI, insn, Address, Decoder)" << Emitter->GuardPostfix << "\n";
         break;
       }
 
-      o.indent(Indentation)
-        << "  tmp = fieldFromInstruction(insn, " << I->FieldBase
-        << ", " << I->FieldLength << ");\n";
-      if (I->Decoder != "") {
-        o.indent(Indentation) << "  " << I->Decoder
-                              << "(MI, tmp, Address, Decoder);\n";
-      } else {
-        o.indent(Indentation)
-          << "  MI.addOperand(MCOperand::CreateImm(tmp));\n";
-      }
+      emitBinaryParser(o, Indentation, *I);
     }
 
-    o.indent(Indentation) << "  return true; // " << nameWithID(Opc)
+    o.indent(Indentation) << "  return " << Emitter->ReturnOK << "; // " << nameWithID(Opc)
                           << '\n';
-    o.indent(Indentation) << "}\n";
+    o.indent(Indentation) << "}\n"; // Closing predicate block.
     return true;
   }
 
@@ -827,16 +849,21 @@ bool FilterChooser::emitSingletonDecoder(raw_ostream &o, unsigned &Indentation,
   for (I = Size; I != 0; --I) {
     o << "Inst{" << EndBits[I-1] << '-' << StartBits[I-1] << "} ";
     if (I > 1)
-      o << "&& ";
+      o << " && ";
     else
       o << "for singleton decoding...\n";
   }
 
   o.indent(Indentation) << "if (";
+  if (emitPredicateMatch(o, Indentation, Opc)) {
+    o << " &&\n";
+    o.indent(Indentation+4);
+  }
 
   for (I = Size; I != 0; --I) {
     NumBits = EndBits[I-1] - StartBits[I-1] + 1;
-    o << "fieldFromInstruction(insn, " << StartBits[I-1] << ", " << NumBits
+    o << "fieldFromInstruction" << BitWidth << "(insn, "
+      << StartBits[I-1] << ", " << NumBits
       << ") == " << FieldVals[I-1];
     if (I > 1)
       o << " && ";
@@ -848,24 +875,15 @@ bool FilterChooser::emitSingletonDecoder(raw_ostream &o, unsigned &Indentation,
   for (std::vector<OperandInfo>::iterator
        I = InsnOperands.begin(), E = InsnOperands.end(); I != E; ++I) {
     // If a custom instruction decoder was specified, use that.
-    if (I->FieldBase == ~0U && I->FieldLength == ~0U) {
-      o.indent(Indentation) << "  " << I->Decoder
-                            << "(MI, insn, Address, Decoder);\n";
+    if (I->numFields() == 0 && I->Decoder.size()) {
+      o.indent(Indentation) << "  " << Emitter->GuardPrefix << I->Decoder
+                            << "(MI, insn, Address, Decoder)" << Emitter->GuardPostfix << "\n";
       break;
     }
 
-    o.indent(Indentation)
-      << "  tmp = fieldFromInstruction(insn, " << I->FieldBase
-      << ", " << I->FieldLength << ");\n";
-    if (I->Decoder != "") {
-      o.indent(Indentation) << "  " << I->Decoder
-                            << "(MI, tmp, Address, Decoder);\n";
-    } else {
-      o.indent(Indentation)
-        << "  MI.addOperand(MCOperand::CreateImm(tmp));\n";
-    }
+    emitBinaryParser(o, Indentation, *I);
   }
-  o.indent(Indentation) << "  return true; // " << nameWithID(Opc)
+  o.indent(Indentation) << "  return " << Emitter->ReturnOK << "; // " << nameWithID(Opc)
                         << '\n';
   o.indent(Indentation) << "}\n";
 
@@ -965,23 +983,23 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   // (MIXED) ------ . ----> (MIXED)
   // (FILTERED)---- . ----> (FILTERED)
 
-  bitAttr_t bitAttrs[BIT_WIDTH];
+  std::vector<bitAttr_t> bitAttrs;
 
   // FILTERED bit positions provide no entropy and are not worthy of pursuing.
   // Filter::recurse() set either BIT_TRUE or BIT_FALSE for each position.
-  for (BitIndex = 0; BitIndex < BIT_WIDTH; ++BitIndex)
+  for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex)
     if (FilterBitValues[BitIndex] == BIT_TRUE ||
         FilterBitValues[BitIndex] == BIT_FALSE)
-      bitAttrs[BitIndex] = ATTR_FILTERED;
+      bitAttrs.push_back(ATTR_FILTERED);
     else
-      bitAttrs[BitIndex] = ATTR_NONE;
+      bitAttrs.push_back(ATTR_NONE);
 
   for (InsnIndex = 0; InsnIndex < numInstructions; ++InsnIndex) {
     insn_t insn;
 
     insnWithID(insn, Opcodes[InsnIndex]);
 
-    for (BitIndex = 0; BitIndex < BIT_WIDTH; ++BitIndex) {
+    for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) {
       switch (bitAttrs[BitIndex]) {
       case ATTR_NONE:
         if (insn[BitIndex] == BIT_UNSET)
@@ -1027,7 +1045,7 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   bitAttr_t RA = ATTR_NONE;
   unsigned StartBit = 0;
 
-  for (BitIndex = 0; BitIndex < BIT_WIDTH; BitIndex++) {
+  for (BitIndex = 0; BitIndex < BitWidth; BitIndex++) {
     bitAttr_t bitAttr = bitAttrs[BitIndex];
 
     assert(bitAttr != ATTR_NONE && "Bit without attributes");
@@ -1216,8 +1234,9 @@ bool FilterChooser::emit(raw_ostream &o, unsigned &Indentation) {
   return true;
 }
 
-bool FixedLenDecoderEmitter::populateInstruction(const CodeGenInstruction &CGI,
-                                                 unsigned Opc){
+static bool populateInstruction(const CodeGenInstruction &CGI,
+                                unsigned Opc,
+                      std::map<unsigned, std::vector<OperandInfo> >& Operands){
   const Record &Def = *CGI.TheDef;
   // If all the bit positions are not specified; do not decode this instruction.
   // We are bound to fail!  For proper disassembly, the well-known encoding bits
@@ -1239,7 +1258,7 @@ bool FixedLenDecoderEmitter::populateInstruction(const CodeGenInstruction &CGI,
   // of trying to auto-generate the decoder.
   std::string InstDecoder = Def.getValueAsString("DecoderMethod");
   if (InstDecoder != "") {
-    InsnOperands.push_back(OperandInfo(~0U, ~0U, InstDecoder));
+    InsnOperands.push_back(OperandInfo(InstDecoder));
     Operands[Opc] = InsnOperands;
     return true;
   }
@@ -1259,72 +1278,97 @@ bool FixedLenDecoderEmitter::populateInstruction(const CodeGenInstruction &CGI,
   for (unsigned i = 0; i < In->getNumArgs(); ++i)
     InOutOperands.push_back(std::make_pair(In->getArg(i), In->getArgName(i)));
 
+  // Search for tied operands, so that we can correctly instantiate
+  // operands that are not explicitly represented in the encoding.
+  std::map<std::string, std::string> TiedNames;
+  for (unsigned i = 0; i < CGI.Operands.size(); ++i) {
+    int tiedTo = CGI.Operands[i].getTiedRegister();
+    if (tiedTo != -1) {
+      TiedNames[InOutOperands[i].second] = InOutOperands[tiedTo].second;
+      TiedNames[InOutOperands[tiedTo].second] = InOutOperands[i].second;
+    }
+  }
+
   // For each operand, see if we can figure out where it is encoded.
   for (std::vector<std::pair<Init*, std::string> >::iterator
        NI = InOutOperands.begin(), NE = InOutOperands.end(); NI != NE; ++NI) {
-    unsigned PrevBit = ~0;
-    unsigned Base = ~0;
-    unsigned PrevPos = ~0;
     std::string Decoder = "";
 
+    // At this point, we can locate the field, but we need to know how to
+    // interpret it.  As a first step, require the target to provide callbacks
+    // for decoding register classes.
+    // FIXME: This need to be extended to handle instructions with custom
+    // decoder methods, and operands with (simple) MIOperandInfo's.
+    TypedInit *TI = dynamic_cast<TypedInit*>(NI->first);
+    RecordRecTy *Type = dynamic_cast<RecordRecTy*>(TI->getType());
+    Record *TypeRecord = Type->getRecord();
+    bool isReg = false;
+    if (TypeRecord->isSubClassOf("RegisterOperand"))
+      TypeRecord = TypeRecord->getValueAsDef("RegClass");
+    if (TypeRecord->isSubClassOf("RegisterClass")) {
+      Decoder = "Decode" + TypeRecord->getName() + "RegisterClass";
+      isReg = true;
+    }
+
+    RecordVal *DecoderString = TypeRecord->getValue("DecoderMethod");
+    StringInit *String = DecoderString ?
+      dynamic_cast<StringInit*>(DecoderString->getValue()) : 0;
+    if (!isReg && String && String->getValue() != "")
+      Decoder = String->getValue();
+
+    OperandInfo OpInfo(Decoder);
+    unsigned Base = ~0U;
+    unsigned Width = 0;
+    unsigned Offset = 0;
+
     for (unsigned bi = 0; bi < Bits.getNumBits(); ++bi) {
+      VarInit *Var = 0;
       VarBitInit *BI = dynamic_cast<VarBitInit*>(Bits.getBit(bi));
-      if (!BI) continue;
+      if (BI)
+        Var = dynamic_cast<VarInit*>(BI->getVariable());
+      else
+        Var = dynamic_cast<VarInit*>(Bits.getBit(bi));
 
-      VarInit *Var = dynamic_cast<VarInit*>(BI->getVariable());
-      assert(Var);
-      unsigned CurrBit = BI->getBitNum();
-      if (Var->getName() != NI->second) continue;
-
-      // Figure out the lowest bit of the value, and the width of the field.
-      // Deliberately don't try to handle cases where the field is scattered,
-      // or where not all bits of the the field are explicit.
-      if (Base == ~0U && PrevBit == ~0U && PrevPos == ~0U) {
-        if (CurrBit == 0)
-          Base = bi;
-        else
-          continue;
+      if (!Var) {
+        if (Base != ~0U) {
+          OpInfo.addField(Base, Width, Offset);
+          Base = ~0U;
+          Width = 0;
+          Offset = 0;
+        }
+        continue;
       }
 
-      if ((PrevPos != ~0U && bi-1 != PrevPos) ||
-          (CurrBit != ~0U && CurrBit-1 != PrevBit)) {
-        PrevBit = ~0;
-        Base = ~0;
-        PrevPos = ~0;
+      if (Var->getName() != NI->second &&
+          Var->getName() != TiedNames[NI->second]) {
+        if (Base != ~0U) {
+          OpInfo.addField(Base, Width, Offset);
+          Base = ~0U;
+          Width = 0;
+          Offset = 0;
+        }
+        continue;
       }
 
-      PrevPos = bi;
-      PrevBit = CurrBit;
-
-      // At this point, we can locate the field, but we need to know how to
-      // interpret it.  As a first step, require the target to provide callbacks
-      // for decoding register classes.
-      // FIXME: This need to be extended to handle instructions with custom
-      // decoder methods, and operands with (simple) MIOperandInfo's.
-      TypedInit *TI = dynamic_cast<TypedInit*>(NI->first);
-      RecordRecTy *Type = dynamic_cast<RecordRecTy*>(TI->getType());
-      Record *TypeRecord = Type->getRecord();
-      bool isReg = false;
-      if (TypeRecord->isSubClassOf("RegisterOperand"))
-        TypeRecord = TypeRecord->getValueAsDef("RegClass");
-      if (TypeRecord->isSubClassOf("RegisterClass")) {
-        Decoder = "Decode" + TypeRecord->getName() + "RegisterClass";
-        isReg = true;
+      if (Base == ~0U) {
+        Base = bi;
+        Width = 1;
+        Offset = BI ? BI->getBitNum() : 0;
+      } else if (BI && BI->getBitNum() != Offset + Width) {
+        OpInfo.addField(Base, Width, Offset);
+        Base = bi;
+        Width = 1;
+        Offset = BI->getBitNum();
+      } else {
+        ++Width;
       }
-
-      RecordVal *DecoderString = TypeRecord->getValue("DecoderMethod");
-      StringInit *String = DecoderString ?
-        dynamic_cast<StringInit*>(DecoderString->getValue()) :
-        0;
-      if (!isReg && String && String->getValue() != "")
-        Decoder = String->getValue();
     }
 
-    if (Base != ~0U) {
-      InsnOperands.push_back(OperandInfo(Base, PrevBit+1, Decoder));
-      DEBUG(errs() << "ENCODED OPERAND: $" << NI->second << " @ ("
-                   << utostr(Base+PrevBit) << ", " << utostr(Base) << ")\n");
-    }
+    if (Base != ~0U)
+      OpInfo.addField(Base, Width, Offset);
+
+    if (OpInfo.numFields() > 0)
+      InsnOperands.push_back(OpInfo);
   }
 
   Operands[Opc] = InsnOperands;
@@ -1351,16 +1395,43 @@ bool FixedLenDecoderEmitter::populateInstruction(const CodeGenInstruction &CGI,
   return true;
 }
 
-void FixedLenDecoderEmitter::populateInstructions() {
-  for (unsigned i = 0, e = NumberedInstructions.size(); i < e; ++i) {
-    Record *R = NumberedInstructions[i]->TheDef;
-    if (R->getValueAsString("Namespace") == "TargetOpcode" ||
-        R->getValueAsBit("isPseudo"))
-      continue;
+static void emitHelper(llvm::raw_ostream &o, unsigned BitWidth) {
+  unsigned Indentation = 0;
+  std::string WidthStr = "uint" + utostr(BitWidth) + "_t";
 
-    if (populateInstruction(*NumberedInstructions[i], i))
-      Opcodes.push_back(i);
-  }
+  o << '\n';
+
+  o.indent(Indentation) << "static " << WidthStr <<
+    " fieldFromInstruction" << BitWidth <<
+    "(" << WidthStr <<" insn, unsigned startBit, unsigned numBits)\n";
+
+  o.indent(Indentation) << "{\n";
+
+  ++Indentation; ++Indentation;
+  o.indent(Indentation) << "assert(startBit + numBits <= " << BitWidth
+                        << " && \"Instruction field out of bounds!\");\n";
+  o << '\n';
+  o.indent(Indentation) << WidthStr << " fieldMask;\n";
+  o << '\n';
+  o.indent(Indentation) << "if (numBits == " << BitWidth << ")\n";
+
+  ++Indentation; ++Indentation;
+  o.indent(Indentation) << "fieldMask = (" << WidthStr << ")-1;\n";
+  --Indentation; --Indentation;
+
+  o.indent(Indentation) << "else\n";
+
+  ++Indentation; ++Indentation;
+  o.indent(Indentation) << "fieldMask = ((1 << numBits) - 1) << startBit;\n";
+  --Indentation; --Indentation;
+
+  o << '\n';
+  o.indent(Indentation) << "return (insn & fieldMask) >> startBit;\n";
+  --Indentation; --Indentation;
+
+  o.indent(Indentation) << "}\n";
+
+  o << '\n';
 }
 
 // Emits disassembler code for instruction decoding.
@@ -1372,10 +1443,47 @@ void FixedLenDecoderEmitter::run(raw_ostream &o)
   o << '\n';
   o << "namespace llvm {\n\n";
 
+  // Parameterize the decoders based on namespace and instruction width.
   NumberedInstructions = Target.getInstructionsByEnumValue();
-  populateInstructions();
-  FilterChooser FC(NumberedInstructions, Opcodes, Operands);
-  FC.emitTop(o, 0);
+  std::map<std::pair<std::string, unsigned>,
+           std::vector<unsigned> > OpcMap;
+  std::map<unsigned, std::vector<OperandInfo> > Operands;
+
+  for (unsigned i = 0; i < NumberedInstructions.size(); ++i) {
+    const CodeGenInstruction *Inst = NumberedInstructions[i];
+    Record *Def = Inst->TheDef;
+    unsigned Size = Def->getValueAsInt("Size");
+    if (Def->getValueAsString("Namespace") == "TargetOpcode" ||
+        Def->getValueAsBit("isPseudo") ||
+        Def->getValueAsBit("isAsmParserOnly") ||
+        Def->getValueAsBit("isCodeGenOnly"))
+      continue;
+
+    std::string DecoderNamespace = Def->getValueAsString("DecoderNamespace");
+
+    if (Size) {
+      if (populateInstruction(*Inst, i, Operands)) {
+        OpcMap[std::make_pair(DecoderNamespace, Size)].push_back(i);
+      }
+    }
+  }
+
+  std::set<unsigned> Sizes;
+  for (std::map<std::pair<std::string, unsigned>,
+                std::vector<unsigned> >::iterator
+       I = OpcMap.begin(), E = OpcMap.end(); I != E; ++I) {
+    // If we haven't visited this instruction width before, emit the
+    // helper method to extract fields.
+    if (!Sizes.count(I->first.second)) {
+      emitHelper(o, 8*I->first.second);
+      Sizes.insert(I->first.second);
+    }
+
+    // Emit the decoder for this namespace+width combination.
+    FilterChooser FC(NumberedInstructions, I->second, Operands,
+                     8*I->first.second, this);
+    FC.emitTop(o, 0, I->first.first);
+  }
 
   o << "\n} // End llvm namespace \n";
 }
diff --git a/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.h b/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.h
index d46a495540ea..2df5448aa8d1 100644
--- a/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.h
+++ b/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.h
@@ -16,26 +16,51 @@
 #define FixedLenDECODEREMITTER_H
 
 #include "CodeGenTarget.h"
-#include "TableGenBackend.h"
 
+#include "llvm/TableGen/TableGenBackend.h"
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
 
+struct EncodingField {
+  unsigned Base, Width, Offset;
+  EncodingField(unsigned B, unsigned W, unsigned O)
+    : Base(B), Width(W), Offset(O) { }
+};
+
 struct OperandInfo {
-  unsigned FieldBase;
-  unsigned FieldLength;
+  std::vector<EncodingField> Fields;
   std::string Decoder;
 
-  OperandInfo(unsigned FB, unsigned FL, std::string D)
-    : FieldBase(FB), FieldLength(FL), Decoder(D) { }
+  OperandInfo(std::string D)
+    : Decoder(D) { }
+
+  void addField(unsigned Base, unsigned Width, unsigned Offset) {
+    Fields.push_back(EncodingField(Base, Width, Offset));
+  }
+
+  unsigned numFields() { return Fields.size(); }
+
+  typedef std::vector<EncodingField>::iterator iterator;
+
+  iterator begin() { return Fields.begin(); }
+  iterator end()   { return Fields.end();   }
 };
 
 class FixedLenDecoderEmitter : public TableGenBackend {
 public:
-  FixedLenDecoderEmitter(RecordKeeper &R) :
+  FixedLenDecoderEmitter(RecordKeeper &R,
+                         std::string PredicateNamespace,
+                         std::string GPrefix  = "if (",
+                         std::string GPostfix = " == MCDisassembler::Fail) return MCDisassembler::Fail;",
+                         std::string ROK      = "MCDisassembler::Success",
+                         std::string RFail    = "MCDisassembler::Fail",
+                         std::string L        = "") :
     Records(R), Target(R),
-    NumberedInstructions(Target.getInstructionsByEnumValue()) {}
+    NumberedInstructions(Target.getInstructionsByEnumValue()),
+    PredicateNamespace(PredicateNamespace),
+    GuardPrefix(GPrefix), GuardPostfix(GPostfix),
+    ReturnOK(ROK), ReturnFail(RFail), Locals(L) {}
 
   // run - Output the code emitter
   void run(raw_ostream &o);
@@ -46,9 +71,11 @@ class FixedLenDecoderEmitter : public TableGenBackend {
   std::vector<const CodeGenInstruction*> NumberedInstructions;
   std::vector<unsigned> Opcodes;
   std::map<unsigned, std::vector<OperandInfo> > Operands;
-
-  bool populateInstruction(const CodeGenInstruction &CGI, unsigned Opc);
-  void populateInstructions();
+public:
+  std::string PredicateNamespace;
+  std::string GuardPrefix, GuardPostfix;
+  std::string ReturnOK, ReturnFail;
+  std::string Locals;
 };
 
 } // end llvm namespace
diff --git a/contrib/llvm/utils/TableGen/InstrEnumEmitter.cpp b/contrib/llvm/utils/TableGen/InstrEnumEmitter.cpp
index aa596892f52f..5981afde0e7e 100644
--- a/contrib/llvm/utils/TableGen/InstrEnumEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/InstrEnumEmitter.cpp
@@ -14,7 +14,7 @@
 
 #include "InstrEnumEmitter.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include <cstdio>
 using namespace llvm;
 
diff --git a/contrib/llvm/utils/TableGen/InstrEnumEmitter.h b/contrib/llvm/utils/TableGen/InstrEnumEmitter.h
index 89f8b659d702..c29a30938d34 100644
--- a/contrib/llvm/utils/TableGen/InstrEnumEmitter.h
+++ b/contrib/llvm/utils/TableGen/InstrEnumEmitter.h
@@ -15,7 +15,7 @@
 #ifndef INSTRENUM_EMITTER_H
 #define INSTRENUM_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
 
diff --git a/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp b/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 5ebaf174655b..8341724a73e4 100644
--- a/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -14,7 +14,7 @@
 
 #include "InstrInfoEmitter.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include <algorithm>
 using namespace llvm;
@@ -268,6 +268,7 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
      << Inst.TheDef->getName() << "\", 0";
 
   // Emit all of the target indepedent flags...
+  if (Inst.isPseudo)           OS << "|(1<<MCID::Pseudo)";
   if (Inst.isReturn)           OS << "|(1<<MCID::Return)";
   if (Inst.isBranch)           OS << "|(1<<MCID::Branch)";
   if (Inst.isIndirectBranch)   OS << "|(1<<MCID::IndirectBranch)";
@@ -288,6 +289,7 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
   if (Inst.isNotDuplicable)    OS << "|(1<<MCID::NotDuplicable)";
   if (Inst.Operands.hasOptionalDef) OS << "|(1<<MCID::HasOptionalDef)";
   if (Inst.usesCustomInserter) OS << "|(1<<MCID::UsesCustomInserter)";
+  if (Inst.hasPostISelHook)    OS << "|(1<<MCID::HasPostISelHook)";
   if (Inst.Operands.isVariadic)OS << "|(1<<MCID::Variadic)";
   if (Inst.hasSideEffects)     OS << "|(1<<MCID::UnmodeledSideEffects)";
   if (Inst.isAsCheapAsAMove)   OS << "|(1<<MCID::CheapAsAMove)";
diff --git a/contrib/llvm/utils/TableGen/InstrInfoEmitter.h b/contrib/llvm/utils/TableGen/InstrInfoEmitter.h
index 165ce423ab60..1461e2c5f7cf 100644
--- a/contrib/llvm/utils/TableGen/InstrInfoEmitter.h
+++ b/contrib/llvm/utils/TableGen/InstrInfoEmitter.h
@@ -15,8 +15,8 @@
 #ifndef INSTRINFO_EMITTER_H
 #define INSTRINFO_EMITTER_H
 
-#include "TableGenBackend.h"
 #include "CodeGenDAGPatterns.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <vector>
 #include <map>
 
diff --git a/contrib/llvm/utils/TableGen/IntrinsicEmitter.cpp b/contrib/llvm/utils/TableGen/IntrinsicEmitter.cpp
index e5e7cea1200f..782b89ede2e7 100644
--- a/contrib/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -13,8 +13,8 @@
 
 #include "CodeGenTarget.h"
 #include "IntrinsicEmitter.h"
-#include "Record.h"
 #include "StringMatcher.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include <algorithm>
 using namespace llvm;
diff --git a/contrib/llvm/utils/TableGen/IntrinsicEmitter.h b/contrib/llvm/utils/TableGen/IntrinsicEmitter.h
index b1efecb92eea..eb6379cc7414 100644
--- a/contrib/llvm/utils/TableGen/IntrinsicEmitter.h
+++ b/contrib/llvm/utils/TableGen/IntrinsicEmitter.h
@@ -15,7 +15,7 @@
 #define INTRINSIC_EMITTER_H
 
 #include "CodeGenIntrinsics.h"
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 namespace llvm {
   class IntrinsicEmitter : public TableGenBackend {
diff --git a/contrib/llvm/utils/TableGen/LLVMCConfigurationEmitter.cpp b/contrib/llvm/utils/TableGen/LLVMCConfigurationEmitter.cpp
deleted file mode 100644
index cd0cbeb1c6b3..000000000000
--- a/contrib/llvm/utils/TableGen/LLVMCConfigurationEmitter.cpp
+++ /dev/null
@@ -1,3134 +0,0 @@
-//===- LLVMCConfigurationEmitter.cpp - Generate LLVMC config ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This tablegen backend is responsible for emitting LLVMC configuration code.
-//
-//===----------------------------------------------------------------------===//
-
-#include "LLVMCConfigurationEmitter.h"
-#include "Record.h"
-
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringSet.h"
-
-#include <algorithm>
-#include <cassert>
-#include <functional>
-#include <stdexcept>
-#include <string>
-#include <typeinfo>
-
-
-using namespace llvm;
-
-namespace {
-
-//===----------------------------------------------------------------------===//
-/// Typedefs
-
-typedef std::vector<Record*> RecordVector;
-typedef std::vector<const DagInit*> DagVector;
-typedef std::vector<std::string> StrVector;
-
-//===----------------------------------------------------------------------===//
-/// Constants
-
-// Indentation.
-const unsigned TabWidth = 4;
-const unsigned Indent1  = TabWidth*1;
-const unsigned Indent2  = TabWidth*2;
-const unsigned Indent3  = TabWidth*3;
-const unsigned Indent4  = TabWidth*4;
-
-// Default help string.
-const char * const DefaultHelpString = "NO HELP MESSAGE PROVIDED";
-
-// Name for the "sink" option.
-const char * const SinkOptionName = "SinkOption";
-
-//===----------------------------------------------------------------------===//
-/// Helper functions
-
-/// Id - An 'identity' function object.
-struct Id {
-  template<typename T0>
-  void operator()(const T0&) const {
-  }
-  template<typename T0, typename T1>
-  void operator()(const T0&, const T1&) const {
-  }
-  template<typename T0, typename T1, typename T2>
-  void operator()(const T0&, const T1&, const T2&) const {
-  }
-};
-
-int InitPtrToInt(const Init* ptr) {
-  const IntInit& val = dynamic_cast<const IntInit&>(*ptr);
-  return val.getValue();
-}
-
-bool InitPtrToBool(const Init* ptr) {
-  bool ret = false;
-  const DefInit& val = dynamic_cast<const DefInit&>(*ptr);
-  const std::string& str = val.getAsString();
-
-  if (str == "true") {
-    ret = true;
-  }
-  else if (str == "false") {
-    ret = false;
-  }
-  else {
-    throw "Incorrect boolean value: '" + str +
-      "': must be either 'true' or 'false'";
-  }
-
-  return ret;
-}
-
-const std::string& InitPtrToString(const Init* ptr) {
-  const StringInit& val = dynamic_cast<const StringInit&>(*ptr);
-  return val.getValue();
-}
-
-const ListInit& InitPtrToList(const Init* ptr) {
-  const ListInit& val = dynamic_cast<const ListInit&>(*ptr);
-  return val;
-}
-
-const DagInit& InitPtrToDag(const Init* ptr) {
-  const DagInit& val = dynamic_cast<const DagInit&>(*ptr);
-  return val;
-}
-
-const std::string GetOperatorName(const DagInit& D) {
-  return D.getOperator()->getAsString();
-}
-
-/// CheckBooleanConstant - Check that the provided value is a boolean constant.
-void CheckBooleanConstant(const Init* I) {
-  InitPtrToBool(I);
-}
-
-// CheckNumberOfArguments - Ensure that the number of args in d is
-// greater than or equal to min_arguments, otherwise throw an exception.
-void CheckNumberOfArguments (const DagInit& d, unsigned minArgs) {
-  if (d.getNumArgs() < minArgs)
-    throw GetOperatorName(d) + ": too few arguments!";
-}
-
-// EscapeVariableName - Escape commas and other symbols not allowed
-// in the C++ variable names. Makes it possible to use options named
-// like "Wa," (useful for prefix options).
-std::string EscapeVariableName (const std::string& Var) {
-  std::string ret;
-  for (unsigned i = 0; i != Var.size(); ++i) {
-    char cur_char = Var[i];
-    if (cur_char == ',') {
-      ret += "_comma_";
-    }
-    else if (cur_char == '+') {
-      ret += "_plus_";
-    }
-    else if (cur_char == '-') {
-      ret += "_dash_";
-    }
-    else {
-      ret.push_back(cur_char);
-    }
-  }
-  return ret;
-}
-
-/// EscapeQuotes - Replace '"' with '\"'.
-std::string EscapeQuotes (const std::string& Var) {
-  std::string ret;
-  for (unsigned i = 0; i != Var.size(); ++i) {
-    char cur_char = Var[i];
-    if (cur_char == '"') {
-      ret += "\\\"";
-    }
-    else {
-      ret.push_back(cur_char);
-    }
-  }
-  return ret;
-}
-
-/// OneOf - Does the input string contain this character?
-bool OneOf(const char* lst, char c) {
-  while (*lst) {
-    if (*lst++ == c)
-      return true;
-  }
-  return false;
-}
-
-template <class I, class S>
-void CheckedIncrement(I& P, I E, S ErrorString) {
-  ++P;
-  if (P == E)
-    throw ErrorString;
-}
-
-//===----------------------------------------------------------------------===//
-/// Back-end specific code
-
-
-/// OptionType - One of six different option types. See the
-/// documentation for detailed description of differences.
-namespace OptionType {
-
-  enum OptionType { Alias, Switch, SwitchList,
-                    Parameter, ParameterList, Prefix, PrefixList };
-
-  bool IsAlias(OptionType t) {
-    return (t == Alias);
-  }
-
-  bool IsList (OptionType t) {
-    return (t == SwitchList || t == ParameterList || t == PrefixList);
-  }
-
-  bool IsSwitch (OptionType t) {
-    return (t == Switch);
-  }
-
-  bool IsSwitchList (OptionType t) {
-    return (t == SwitchList);
-  }
-
-  bool IsParameter (OptionType t) {
-    return (t == Parameter || t == Prefix);
-  }
-
-}
-
-OptionType::OptionType stringToOptionType(const std::string& T) {
-  if (T == "alias_option")
-    return OptionType::Alias;
-  else if (T == "switch_option")
-    return OptionType::Switch;
-  else if (T == "switch_list_option")
-    return OptionType::SwitchList;
-  else if (T == "parameter_option")
-    return OptionType::Parameter;
-  else if (T == "parameter_list_option")
-    return OptionType::ParameterList;
-  else if (T == "prefix_option")
-    return OptionType::Prefix;
-  else if (T == "prefix_list_option")
-    return OptionType::PrefixList;
-  else
-    throw "Unknown option type: " + T + '!';
-}
-
-namespace OptionDescriptionFlags {
-  enum OptionDescriptionFlags { Required = 0x1, Hidden = 0x2,
-                                ReallyHidden = 0x4, OneOrMore = 0x8,
-                                Optional = 0x10, CommaSeparated = 0x20,
-                                ForwardNotSplit = 0x40, ZeroOrMore = 0x80 };
-}
-
-/// OptionDescription - Represents data contained in a single
-/// OptionList entry.
-struct OptionDescription {
-  OptionType::OptionType Type;
-  std::string Name;
-  unsigned Flags;
-  std::string Help;
-  unsigned MultiVal;
-  Init* InitVal;
-
-  OptionDescription(OptionType::OptionType t = OptionType::Switch,
-                    const std::string& n = "",
-                    const std::string& h = DefaultHelpString)
-    : Type(t), Name(n), Flags(0x0), Help(h), MultiVal(1), InitVal(0)
-  {}
-
-  /// GenTypeDeclaration - Returns the C++ variable type of this
-  /// option.
-  const char* GenTypeDeclaration() const;
-
-  /// GenVariableName - Returns the variable name used in the
-  /// generated C++ code.
-  std::string GenVariableName() const
-  { return "autogenerated::" + GenOptionType() + EscapeVariableName(Name); }
-
-  /// GenPlainVariableName - Returns the variable name without the namespace
-  /// prefix.
-  std::string GenPlainVariableName() const
-  { return GenOptionType() + EscapeVariableName(Name); }
-
-  /// Merge - Merge two option descriptions.
-  void Merge (const OptionDescription& other);
-
-  /// CheckConsistency - Check that the flags are consistent.
-  void CheckConsistency() const;
-
-  // Misc convenient getters/setters.
-
-  bool isAlias() const;
-
-  bool isMultiVal() const;
-
-  bool isCommaSeparated() const;
-  void setCommaSeparated();
-
-  bool isForwardNotSplit() const;
-  void setForwardNotSplit();
-
-  bool isRequired() const;
-  void setRequired();
-
-  bool isOneOrMore() const;
-  void setOneOrMore();
-
-  bool isZeroOrMore() const;
-  void setZeroOrMore();
-
-  bool isOptional() const;
-  void setOptional();
-
-  bool isHidden() const;
-  void setHidden();
-
-  bool isReallyHidden() const;
-  void setReallyHidden();
-
-  bool isSwitch() const
-  { return OptionType::IsSwitch(this->Type); }
-
-  bool isSwitchList() const
-  { return OptionType::IsSwitchList(this->Type); }
-
-  bool isParameter() const
-  { return OptionType::IsParameter(this->Type); }
-
-  bool isList() const
-  { return OptionType::IsList(this->Type); }
-
-  bool isParameterList() const
-  { return (OptionType::IsList(this->Type)
-            && !OptionType::IsSwitchList(this->Type)); }
-
-private:
-
-  // GenOptionType - Helper function used by GenVariableName().
-  std::string GenOptionType() const;
-};
-
-void OptionDescription::CheckConsistency() const {
-  unsigned i = 0;
-
-  i += this->isRequired();
-  i += this->isOptional();
-  i += this->isOneOrMore();
-  i += this->isZeroOrMore();
-
-  if (i > 1) {
-    throw "Only one of (required), (optional), (one_or_more) or "
-      "(zero_or_more) properties is allowed!";
-  }
-}
-
-void OptionDescription::Merge (const OptionDescription& other)
-{
-  if (other.Type != Type)
-    throw "Conflicting definitions for the option " + Name + "!";
-
-  if (Help == other.Help || Help == DefaultHelpString)
-    Help = other.Help;
-  else if (other.Help != DefaultHelpString) {
-    llvm::errs() << "Warning: several different help strings"
-      " defined for option " + Name + "\n";
-  }
-
-  Flags |= other.Flags;
-}
-
-bool OptionDescription::isAlias() const {
-  return OptionType::IsAlias(this->Type);
-}
-
-bool OptionDescription::isMultiVal() const {
-  return MultiVal > 1;
-}
-
-bool OptionDescription::isCommaSeparated() const {
-  return Flags & OptionDescriptionFlags::CommaSeparated;
-}
-void OptionDescription::setCommaSeparated() {
-  Flags |= OptionDescriptionFlags::CommaSeparated;
-}
-
-bool OptionDescription::isForwardNotSplit() const {
-  return Flags & OptionDescriptionFlags::ForwardNotSplit;
-}
-void OptionDescription::setForwardNotSplit() {
-  Flags |= OptionDescriptionFlags::ForwardNotSplit;
-}
-
-bool OptionDescription::isRequired() const {
-  return Flags & OptionDescriptionFlags::Required;
-}
-void OptionDescription::setRequired() {
-  Flags |= OptionDescriptionFlags::Required;
-}
-
-bool OptionDescription::isOneOrMore() const {
-  return Flags & OptionDescriptionFlags::OneOrMore;
-}
-void OptionDescription::setOneOrMore() {
-  Flags |= OptionDescriptionFlags::OneOrMore;
-}
-
-bool OptionDescription::isZeroOrMore() const {
-  return Flags & OptionDescriptionFlags::ZeroOrMore;
-}
-void OptionDescription::setZeroOrMore() {
-  Flags |= OptionDescriptionFlags::ZeroOrMore;
-}
-
-bool OptionDescription::isOptional() const {
-  return Flags & OptionDescriptionFlags::Optional;
-}
-void OptionDescription::setOptional() {
-  Flags |= OptionDescriptionFlags::Optional;
-}
-
-bool OptionDescription::isHidden() const {
-  return Flags & OptionDescriptionFlags::Hidden;
-}
-void OptionDescription::setHidden() {
-  Flags |= OptionDescriptionFlags::Hidden;
-}
-
-bool OptionDescription::isReallyHidden() const {
-  return Flags & OptionDescriptionFlags::ReallyHidden;
-}
-void OptionDescription::setReallyHidden() {
-  Flags |= OptionDescriptionFlags::ReallyHidden;
-}
-
-const char* OptionDescription::GenTypeDeclaration() const {
-  switch (Type) {
-  case OptionType::Alias:
-    return "cl::alias";
-  case OptionType::PrefixList:
-  case OptionType::ParameterList:
-    return "cl::list<std::string>";
-  case OptionType::Switch:
-    return "cl::opt<bool>";
-  case OptionType::SwitchList:
-    return "cl::list<bool>";
-  case OptionType::Parameter:
-  case OptionType::Prefix:
-  default:
-    return "cl::opt<std::string>";
-  }
-}
-
-std::string OptionDescription::GenOptionType() const {
-  switch (Type) {
-  case OptionType::Alias:
-    return "Alias_";
-  case OptionType::PrefixList:
-  case OptionType::ParameterList:
-    return "List_";
-  case OptionType::Switch:
-    return "Switch_";
-  case OptionType::SwitchList:
-    return "SwitchList_";
-  case OptionType::Prefix:
-  case OptionType::Parameter:
-  default:
-    return "Parameter_";
-  }
-}
-
-/// OptionDescriptions - An OptionDescription array plus some helper
-/// functions.
-class OptionDescriptions {
-  typedef StringMap<OptionDescription> container_type;
-
-  /// Descriptions - A list of OptionDescriptions.
-  container_type Descriptions;
-
-public:
-  /// FindOption - exception-throwing wrapper for find().
-  const OptionDescription& FindOption(const std::string& OptName) const;
-
-  // Wrappers for FindOption that throw an exception in case the option has a
-  // wrong type.
-  const OptionDescription& FindSwitch(const std::string& OptName) const;
-  const OptionDescription& FindParameter(const std::string& OptName) const;
-  const OptionDescription& FindParameterList(const std::string& OptName) const;
-  const OptionDescription&
-  FindListOrParameter(const std::string& OptName) const;
-  const OptionDescription&
-  FindParameterListOrParameter(const std::string& OptName) const;
-
-  /// insertDescription - Insert new OptionDescription into
-  /// OptionDescriptions list
-  void InsertDescription (const OptionDescription& o);
-
-  // Support for STL-style iteration
-  typedef container_type::const_iterator const_iterator;
-  const_iterator begin() const { return Descriptions.begin(); }
-  const_iterator end() const { return Descriptions.end(); }
-};
-
-const OptionDescription&
-OptionDescriptions::FindOption(const std::string& OptName) const {
-  const_iterator I = Descriptions.find(OptName);
-  if (I != Descriptions.end())
-    return I->second;
-  else
-    throw OptName + ": no such option!";
-}
-
-const OptionDescription&
-OptionDescriptions::FindSwitch(const std::string& OptName) const {
-  const OptionDescription& OptDesc = this->FindOption(OptName);
-  if (!OptDesc.isSwitch())
-    throw OptName + ": incorrect option type - should be a switch!";
-  return OptDesc;
-}
-
-const OptionDescription&
-OptionDescriptions::FindParameterList(const std::string& OptName) const {
-  const OptionDescription& OptDesc = this->FindOption(OptName);
-  if (!OptDesc.isList() || OptDesc.isSwitchList())
-    throw OptName + ": incorrect option type - should be a parameter list!";
-  return OptDesc;
-}
-
-const OptionDescription&
-OptionDescriptions::FindParameter(const std::string& OptName) const {
-  const OptionDescription& OptDesc = this->FindOption(OptName);
-  if (!OptDesc.isParameter())
-    throw OptName + ": incorrect option type - should be a parameter!";
-  return OptDesc;
-}
-
-const OptionDescription&
-OptionDescriptions::FindListOrParameter(const std::string& OptName) const {
-  const OptionDescription& OptDesc = this->FindOption(OptName);
-  if (!OptDesc.isList() && !OptDesc.isParameter())
-    throw OptName
-      + ": incorrect option type - should be a list or parameter!";
-  return OptDesc;
-}
-
-const OptionDescription&
-OptionDescriptions::FindParameterListOrParameter
-(const std::string& OptName) const {
-  const OptionDescription& OptDesc = this->FindOption(OptName);
-  if ((!OptDesc.isList() && !OptDesc.isParameter()) || OptDesc.isSwitchList())
-    throw OptName
-      + ": incorrect option type - should be a parameter list or parameter!";
-  return OptDesc;
-}
-
-void OptionDescriptions::InsertDescription (const OptionDescription& o) {
-  container_type::iterator I = Descriptions.find(o.Name);
-  if (I != Descriptions.end()) {
-    OptionDescription& D = I->second;
-    D.Merge(o);
-  }
-  else {
-    Descriptions[o.Name] = o;
-  }
-}
-
-/// HandlerTable - A base class for function objects implemented as
-/// 'tables of handlers'.
-template <typename Handler>
-class HandlerTable {
-protected:
-  // Implementation details.
-
-  /// HandlerMap - A map from property names to property handlers
-  typedef StringMap<Handler> HandlerMap;
-
-  static HandlerMap Handlers_;
-  static bool staticMembersInitialized_;
-
-public:
-
-  Handler GetHandler (const std::string& HandlerName) const {
-    typename HandlerMap::iterator method = Handlers_.find(HandlerName);
-
-    if (method != Handlers_.end()) {
-      Handler h = method->second;
-      return h;
-    }
-    else {
-      throw "No handler found for property " + HandlerName + "!";
-    }
-  }
-
-  void AddHandler(const char* Property, Handler H) {
-    Handlers_[Property] = H;
-  }
-
-};
-
-template <class Handler, class FunctionObject>
-Handler GetHandler(FunctionObject* Obj, const DagInit& Dag) {
-  const std::string& HandlerName = GetOperatorName(Dag);
-  return Obj->GetHandler(HandlerName);
-}
-
-template <class FunctionObject>
-void InvokeDagInitHandler(FunctionObject* Obj, Init* I) {
-  typedef void (FunctionObject::*Handler) (const DagInit&);
-
-  const DagInit& Dag = InitPtrToDag(I);
-  Handler h = GetHandler<Handler>(Obj, Dag);
-
-  ((Obj)->*(h))(Dag);
-}
-
-template <class FunctionObject>
-void InvokeDagInitHandler(const FunctionObject* const Obj,
-                          const Init* I, unsigned IndentLevel, raw_ostream& O)
-{
-  typedef void (FunctionObject::*Handler)
-    (const DagInit&, unsigned IndentLevel, raw_ostream& O) const;
-
-  const DagInit& Dag = InitPtrToDag(I);
-  Handler h = GetHandler<Handler>(Obj, Dag);
-
-  ((Obj)->*(h))(Dag, IndentLevel, O);
-}
-
-template <typename H>
-typename HandlerTable<H>::HandlerMap HandlerTable<H>::Handlers_;
-
-template <typename H>
-bool HandlerTable<H>::staticMembersInitialized_ = false;
-
-
-/// CollectOptionProperties - Function object for iterating over an
-/// option property list.
-class CollectOptionProperties;
-typedef void (CollectOptionProperties::* CollectOptionPropertiesHandler)
-(const DagInit&);
-
-class CollectOptionProperties
-: public HandlerTable<CollectOptionPropertiesHandler>
-{
-private:
-
-  /// optDescs_ - OptionDescriptions table. This is where the
-  /// information is stored.
-  OptionDescription& optDesc_;
-
-public:
-
-  explicit CollectOptionProperties(OptionDescription& OD)
-    : optDesc_(OD)
-  {
-    if (!staticMembersInitialized_) {
-      AddHandler("help", &CollectOptionProperties::onHelp);
-      AddHandler("hidden", &CollectOptionProperties::onHidden);
-      AddHandler("init", &CollectOptionProperties::onInit);
-      AddHandler("multi_val", &CollectOptionProperties::onMultiVal);
-      AddHandler("one_or_more", &CollectOptionProperties::onOneOrMore);
-      AddHandler("zero_or_more", &CollectOptionProperties::onZeroOrMore);
-      AddHandler("really_hidden", &CollectOptionProperties::onReallyHidden);
-      AddHandler("required", &CollectOptionProperties::onRequired);
-      AddHandler("optional", &CollectOptionProperties::onOptional);
-      AddHandler("comma_separated", &CollectOptionProperties::onCommaSeparated);
-      AddHandler("forward_not_split",
-                 &CollectOptionProperties::onForwardNotSplit);
-
-      staticMembersInitialized_ = true;
-    }
-  }
-
-  /// operator() - Just forwards to the corresponding property
-  /// handler.
-  void operator() (Init* I) {
-    InvokeDagInitHandler(this, I);
-  }
-
-private:
-
-  /// Option property handlers --
-  /// Methods that handle option properties such as (help) or (hidden).
-
-  void onHelp (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    optDesc_.Help = EscapeQuotes(InitPtrToString(d.getArg(0)));
-  }
-
-  void onHidden (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-    optDesc_.setHidden();
-  }
-
-  void onReallyHidden (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-    optDesc_.setReallyHidden();
-  }
-
-  void onCommaSeparated (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-    if (!optDesc_.isParameterList())
-      throw "'comma_separated' is valid only on parameter list options!";
-    optDesc_.setCommaSeparated();
-  }
-
-  void onForwardNotSplit (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-    if (!optDesc_.isParameter())
-      throw "'forward_not_split' is valid only for parameter options!";
-    optDesc_.setForwardNotSplit();
-  }
-
-  void onRequired (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-
-    optDesc_.setRequired();
-    optDesc_.CheckConsistency();
-  }
-
-  void onInit (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    Init* i = d.getArg(0);
-    const std::string& str = i->getAsString();
-
-    bool correct = optDesc_.isParameter() && dynamic_cast<StringInit*>(i);
-    correct |= (optDesc_.isSwitch() && (str == "true" || str == "false"));
-
-    if (!correct)
-      throw "Incorrect usage of the 'init' option property!";
-
-    optDesc_.InitVal = i;
-  }
-
-  void onOneOrMore (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-
-    optDesc_.setOneOrMore();
-    optDesc_.CheckConsistency();
-  }
-
-  void onZeroOrMore (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-
-    if (optDesc_.isList())
-      llvm::errs() << "Warning: specifying the 'zero_or_more' property "
-        "on a list option has no effect.\n";
-
-    optDesc_.setZeroOrMore();
-    optDesc_.CheckConsistency();
-  }
-
-  void onOptional (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-
-    if (!optDesc_.isList())
-      llvm::errs() << "Warning: specifying the 'optional' property"
-        "on a non-list option has no effect.\n";
-
-    optDesc_.setOptional();
-    optDesc_.CheckConsistency();
-  }
-
-  void onMultiVal (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    int val = InitPtrToInt(d.getArg(0));
-    if (val < 2)
-      throw "Error in the 'multi_val' property: "
-        "the value must be greater than 1!";
-    if (!optDesc_.isParameterList())
-      throw "The multi_val property is valid only on list options!";
-    optDesc_.MultiVal = val;
-  }
-
-};
-
-/// AddOption - A function object that is applied to every option
-/// description. Used by CollectOptionDescriptions.
-class AddOption {
-private:
-  OptionDescriptions& OptDescs_;
-
-public:
-  explicit AddOption(OptionDescriptions& OD) : OptDescs_(OD)
-  {}
-
-  void operator()(const Init* i) {
-    const DagInit& d = InitPtrToDag(i);
-    CheckNumberOfArguments(d, 1);
-
-    const OptionType::OptionType Type =
-      stringToOptionType(GetOperatorName(d));
-    const std::string& Name = InitPtrToString(d.getArg(0));
-
-    OptionDescription OD(Type, Name);
-
-    CheckNumberOfArguments(d, 2);
-
-    // Alias option store the aliased option name in the 'Help' field and do not
-    // have any properties.
-    if (OD.isAlias()) {
-      OD.Help = InitPtrToString(d.getArg(1));
-    }
-    else {
-      processOptionProperties(d, OD);
-    }
-
-    // Switch options are ZeroOrMore by default.
-    if (OD.isSwitch()) {
-      if (!(OD.isOptional() || OD.isOneOrMore() || OD.isRequired()))
-        OD.setZeroOrMore();
-    }
-
-    OptDescs_.InsertDescription(OD);
-  }
-
-private:
-  /// processOptionProperties - Go through the list of option
-  /// properties and call a corresponding handler for each.
-  static void processOptionProperties (const DagInit& d, OptionDescription& o) {
-    CheckNumberOfArguments(d, 2);
-    DagInit::const_arg_iterator B = d.arg_begin();
-    // Skip the first argument: it's always the option name.
-    ++B;
-    std::for_each(B, d.arg_end(), CollectOptionProperties(o));
-  }
-
-};
-
-/// CollectOptionDescriptions - Collects option properties from all
-/// OptionLists.
-void CollectOptionDescriptions (const RecordVector& V,
-                                OptionDescriptions& OptDescs)
-{
-  // For every OptionList:
-  for (RecordVector::const_iterator B = V.begin(), E = V.end(); B!=E; ++B)
-  {
-    // Throws an exception if the value does not exist.
-    ListInit* PropList = (*B)->getValueAsListInit("options");
-
-    // For every option description in this list: invoke AddOption.
-    std::for_each(PropList->begin(), PropList->end(), AddOption(OptDescs));
-  }
-}
-
-// Tool information record
-
-namespace ToolFlags {
-  enum ToolFlags { Join = 0x1, Sink = 0x2 };
-}
-
-struct ToolDescription : public RefCountedBase<ToolDescription> {
-  std::string Name;
-  Init* CmdLine;
-  Init* Actions;
-  StrVector InLanguage;
-  std::string InFileOption;
-  std::string OutFileOption;
-  StrVector OutLanguage;
-  std::string OutputSuffix;
-  unsigned Flags;
-  const Init* OnEmpty;
-
-  // Various boolean properties
-  void setSink()      { Flags |= ToolFlags::Sink; }
-  bool isSink() const { return Flags & ToolFlags::Sink; }
-  void setJoin()      { Flags |= ToolFlags::Join; }
-  bool isJoin() const { return Flags & ToolFlags::Join; }
-
-  // Default ctor here is needed because StringMap can only store
-  // DefaultConstructible objects
-  ToolDescription (const std::string &n = "")
-    : Name(n), CmdLine(0), Actions(0), OutFileOption("-o"),
-      Flags(0), OnEmpty(0)
-  {}
-};
-
-/// ToolDescriptions - A list of Tool information records.
-typedef std::vector<IntrusiveRefCntPtr<ToolDescription> > ToolDescriptions;
-
-
-/// CollectToolProperties - Function object for iterating over a list of
-/// tool property records.
-
-class CollectToolProperties;
-typedef void (CollectToolProperties::* CollectToolPropertiesHandler)
-(const DagInit&);
-
-class CollectToolProperties : public HandlerTable<CollectToolPropertiesHandler>
-{
-private:
-
-  /// toolDesc_ - Properties of the current Tool. This is where the
-  /// information is stored.
-  ToolDescription& toolDesc_;
-
-public:
-
-  explicit CollectToolProperties (ToolDescription& d)
-    : toolDesc_(d)
-  {
-    if (!staticMembersInitialized_) {
-
-      AddHandler("actions", &CollectToolProperties::onActions);
-      AddHandler("command", &CollectToolProperties::onCommand);
-      AddHandler("in_language", &CollectToolProperties::onInLanguage);
-      AddHandler("join", &CollectToolProperties::onJoin);
-      AddHandler("out_language", &CollectToolProperties::onOutLanguage);
-
-      AddHandler("out_file_option", &CollectToolProperties::onOutFileOption);
-      AddHandler("in_file_option", &CollectToolProperties::onInFileOption);
-
-      AddHandler("output_suffix", &CollectToolProperties::onOutputSuffix);
-      AddHandler("sink", &CollectToolProperties::onSink);
-      AddHandler("works_on_empty", &CollectToolProperties::onWorksOnEmpty);
-
-      staticMembersInitialized_ = true;
-    }
-  }
-
-  void operator() (Init* I) {
-    InvokeDagInitHandler(this, I);
-  }
-
-private:
-
-  /// Property handlers --
-  /// Functions that extract information about tool properties from
-  /// DAG representation.
-
-  void onActions (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    Init* Case = d.getArg(0);
-    if (typeid(*Case) != typeid(DagInit) ||
-        GetOperatorName(static_cast<DagInit&>(*Case)) != "case")
-      throw "The argument to (actions) should be a 'case' construct!";
-    toolDesc_.Actions = Case;
-  }
-
-  void onCommand (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    toolDesc_.CmdLine = d.getArg(0);
-  }
-
-  /// onInOutLanguage - Common implementation of on{In,Out}Language().
-  void onInOutLanguage (const DagInit& d, StrVector& OutVec) {
-    CheckNumberOfArguments(d, 1);
-
-    // Copy strings to the output vector.
-    for (unsigned i = 0, NumArgs = d.getNumArgs(); i < NumArgs; ++i) {
-      OutVec.push_back(InitPtrToString(d.getArg(i)));
-    }
-
-    // Remove duplicates.
-    std::sort(OutVec.begin(), OutVec.end());
-    StrVector::iterator newE = std::unique(OutVec.begin(), OutVec.end());
-    OutVec.erase(newE, OutVec.end());
-  }
-
-
-  void onInLanguage (const DagInit& d) {
-    this->onInOutLanguage(d, toolDesc_.InLanguage);
-  }
-
-  void onJoin (const DagInit& d) {
-    bool isReallyJoin = false;
-
-    if (d.getNumArgs() == 0) {
-      isReallyJoin = true;
-    }
-    else {
-      Init* I = d.getArg(0);
-      isReallyJoin = InitPtrToBool(I);
-    }
-
-    // Is this *really* a join tool? We allow (join false) for generating two
-    // tool descriptions from a single generic one.
-    // TOFIX: come up with a cleaner solution.
-    if (isReallyJoin) {
-      toolDesc_.setJoin();
-    }
-  }
-
-  void onOutLanguage (const DagInit& d) {
-    this->onInOutLanguage(d, toolDesc_.OutLanguage);
-  }
-
-  void onOutFileOption (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    toolDesc_.OutFileOption = InitPtrToString(d.getArg(0));
-  }
-
-  void onInFileOption (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    toolDesc_.InFileOption = InitPtrToString(d.getArg(0));
-  }
-
-  void onOutputSuffix (const DagInit& d) {
-    CheckNumberOfArguments(d, 1);
-    toolDesc_.OutputSuffix = InitPtrToString(d.getArg(0));
-  }
-
-  void onSink (const DagInit& d) {
-    CheckNumberOfArguments(d, 0);
-    toolDesc_.setSink();
-  }
-
-  void onWorksOnEmpty (const DagInit& d) {
-    toolDesc_.OnEmpty = d.getArg(0);
-  }
-
-};
-
-/// CollectToolDescriptions - Gather information about tool properties
-/// from the parsed TableGen data (basically a wrapper for the
-/// CollectToolProperties function object).
-void CollectToolDescriptions (const RecordVector& Tools,
-                              ToolDescriptions& ToolDescs)
-{
-  // Iterate over a properties list of every Tool definition
-  for (RecordVector::const_iterator B = Tools.begin(),
-         E = Tools.end(); B!=E; ++B) {
-    const Record* T = *B;
-    // Throws an exception if the value does not exist.
-    ListInit* PropList = T->getValueAsListInit("properties");
-
-    IntrusiveRefCntPtr<ToolDescription>
-      ToolDesc(new ToolDescription(T->getName()));
-
-    std::for_each(PropList->begin(), PropList->end(),
-                  CollectToolProperties(*ToolDesc));
-    ToolDescs.push_back(ToolDesc);
-  }
-}
-
-/// FillInEdgeVector - Merge all compilation graph definitions into
-/// one single edge list.
-void FillInEdgeVector(const RecordVector& CompilationGraphs,
-                      DagVector& Out) {
-  for (RecordVector::const_iterator B = CompilationGraphs.begin(),
-         E = CompilationGraphs.end(); B != E; ++B) {
-    const ListInit* Edges = (*B)->getValueAsListInit("edges");
-
-    for (ListInit::const_iterator B = Edges->begin(),
-           E = Edges->end(); B != E; ++B) {
-      Out.push_back(&InitPtrToDag(*B));
-    }
-  }
-}
-
-/// NotInGraph - Helper function object for FilterNotInGraph.
-struct NotInGraph {
-private:
-  const llvm::StringSet<>& ToolsInGraph_;
-
-public:
-  NotInGraph(const llvm::StringSet<>& ToolsInGraph)
-  : ToolsInGraph_(ToolsInGraph)
-  {}
-
-  bool operator()(const IntrusiveRefCntPtr<ToolDescription>& x) {
-    return (ToolsInGraph_.count(x->Name) == 0);
-  }
-};
-
-/// FilterNotInGraph - Filter out from ToolDescs all Tools not
-/// mentioned in the compilation graph definition.
-void FilterNotInGraph (const DagVector& EdgeVector,
-                       ToolDescriptions& ToolDescs) {
-
-  // List all tools mentioned in the graph.
-  llvm::StringSet<> ToolsInGraph;
-
-  for (DagVector::const_iterator B = EdgeVector.begin(),
-         E = EdgeVector.end(); B != E; ++B) {
-
-    const DagInit* Edge = *B;
-    const std::string& NodeA = InitPtrToString(Edge->getArg(0));
-    const std::string& NodeB = InitPtrToString(Edge->getArg(1));
-
-    if (NodeA != "root")
-      ToolsInGraph.insert(NodeA);
-    ToolsInGraph.insert(NodeB);
-  }
-
-  // Filter ToolPropertiesList.
-  ToolDescriptions::iterator new_end =
-    std::remove_if(ToolDescs.begin(), ToolDescs.end(),
-                   NotInGraph(ToolsInGraph));
-  ToolDescs.erase(new_end, ToolDescs.end());
-}
-
-/// FillInToolToLang - Fills in two tables that map tool names to
-/// input & output language names.  Helper function used by TypecheckGraph().
-void FillInToolToLang (const ToolDescriptions& ToolDescs,
-                       StringMap<StringSet<> >& ToolToInLang,
-                       StringMap<StringSet<> >& ToolToOutLang) {
-  for (ToolDescriptions::const_iterator B = ToolDescs.begin(),
-         E = ToolDescs.end(); B != E; ++B) {
-    const ToolDescription& D = *(*B);
-    for (StrVector::const_iterator B = D.InLanguage.begin(),
-           E = D.InLanguage.end(); B != E; ++B)
-      ToolToInLang[D.Name].insert(*B);
-    for (StrVector::const_iterator B = D.OutLanguage.begin(),
-           E = D.OutLanguage.end(); B != E; ++B)
-      ToolToOutLang[D.Name].insert(*B);
-  }
-}
-
-/// Intersect - Is set intersection non-empty?
-bool Intersect (const StringSet<>& S1, const StringSet<>& S2) {
-  for (StringSet<>::const_iterator B = S1.begin(), E = S1.end(); B != E; ++B) {
-    if (S2.count(B->first()) != 0)
-      return true;
-  }
-  return false;
-}
-
-/// TypecheckGraph - Check that names for output and input languages
-/// on all edges do match.
-void TypecheckGraph (const DagVector& EdgeVector,
-                     const ToolDescriptions& ToolDescs) {
-  StringMap<StringSet<> > ToolToInLang;
-  StringMap<StringSet<> > ToolToOutLang;
-
-  FillInToolToLang(ToolDescs, ToolToInLang, ToolToOutLang);
-
-  for (DagVector::const_iterator B = EdgeVector.begin(),
-         E = EdgeVector.end(); B != E; ++B) {
-    const DagInit* Edge = *B;
-    const std::string& NodeA = InitPtrToString(Edge->getArg(0));
-    const std::string& NodeB = InitPtrToString(Edge->getArg(1));
-    StringMap<StringSet<> >::iterator IA = ToolToOutLang.find(NodeA);
-    StringMap<StringSet<> >::iterator IB = ToolToInLang.find(NodeB);
-
-    if (NodeB == "root")
-      throw "Edges back to the root are not allowed!";
-
-    if (NodeA != "root") {
-      if (IA == ToolToOutLang.end())
-        throw NodeA + ": no output language defined!";
-      if (IB == ToolToInLang.end())
-        throw NodeB + ": no input language defined!";
-
-      if (!Intersect(IA->second, IB->second)) {
-        throw "Edge " + NodeA + "->" + NodeB
-          + ": output->input language mismatch";
-      }
-    }
-  }
-}
-
-/// WalkCase - Walks the 'case' expression DAG and invokes
-/// TestCallback on every test, and StatementCallback on every
-/// statement. Handles 'case' nesting, but not the 'and' and 'or'
-/// combinators (that is, they are passed directly to TestCallback).
-/// TestCallback must have type 'void TestCallback(const DagInit*, unsigned
-/// IndentLevel, bool FirstTest)'.
-/// StatementCallback must have type 'void StatementCallback(const Init*,
-/// unsigned IndentLevel)'.
-template <typename F1, typename F2>
-void WalkCase(const Init* Case, F1 TestCallback, F2 StatementCallback,
-              unsigned IndentLevel = 0)
-{
-  const DagInit& d = InitPtrToDag(Case);
-
-  // Error checks.
-  if (GetOperatorName(d) != "case")
-    throw "WalkCase should be invoked only on 'case' expressions!";
-
-  if (d.getNumArgs() < 2)
-    throw "There should be at least one clause in the 'case' expression:\n"
-      + d.getAsString();
-
-  // Main loop.
-  bool even = false;
-  const unsigned numArgs = d.getNumArgs();
-  unsigned i = 1;
-  for (DagInit::const_arg_iterator B = d.arg_begin(), E = d.arg_end();
-       B != E; ++B) {
-    Init* arg = *B;
-
-    if (!even)
-    {
-      // Handle test.
-      const DagInit& Test = InitPtrToDag(arg);
-
-      if (GetOperatorName(Test) == "default" && (i+1 != numArgs))
-        throw "The 'default' clause should be the last in the "
-          "'case' construct!";
-      if (i == numArgs)
-        throw "Case construct handler: no corresponding action "
-          "found for the test " + Test.getAsString() + '!';
-
-      TestCallback(Test, IndentLevel, (i == 1));
-    }
-    else
-    {
-      if (dynamic_cast<DagInit*>(arg)
-          && GetOperatorName(static_cast<DagInit&>(*arg)) == "case") {
-        // Nested 'case'.
-        WalkCase(arg, TestCallback, StatementCallback, IndentLevel + Indent1);
-      }
-
-      // Handle statement.
-      StatementCallback(arg, IndentLevel);
-    }
-
-    ++i;
-    even = !even;
-  }
-}
-
-/// ExtractOptionNames - A helper function object used by
-/// CheckForSuperfluousOptions() to walk the 'case' DAG.
-class ExtractOptionNames {
-  llvm::StringSet<>& OptionNames_;
-
-  void processDag(const Init* Statement) {
-    const DagInit& Stmt = InitPtrToDag(Statement);
-    const std::string& ActionName = GetOperatorName(Stmt);
-    if (ActionName == "forward" || ActionName == "forward_as" ||
-        ActionName == "forward_value" ||
-        ActionName == "forward_transformed_value" ||
-        ActionName == "parameter_equals" || ActionName == "element_in_list") {
-      CheckNumberOfArguments(Stmt, 1);
-
-      Init* Arg = Stmt.getArg(0);
-      if (typeid(*Arg) == typeid(StringInit))
-        OptionNames_.insert(InitPtrToString(Arg));
-    }
-    else if (ActionName == "any_switch_on" || ActionName == "switch_on" ||
-             ActionName == "any_not_empty" || ActionName == "any_empty" ||
-             ActionName == "not_empty" || ActionName == "empty") {
-      for (unsigned i = 0, NumArgs = Stmt.getNumArgs(); i < NumArgs; ++i) {
-        Init* Arg = Stmt.getArg(i);
-        if (typeid(*Arg) == typeid(StringInit))
-          OptionNames_.insert(InitPtrToString(Arg));
-      }
-    }
-    else if (ActionName == "and" || ActionName == "or" || ActionName == "not") {
-      for (unsigned i = 0, NumArgs = Stmt.getNumArgs(); i < NumArgs; ++i) {
-        this->processDag(Stmt.getArg(i));
-      }
-    }
-  }
-
-public:
-  ExtractOptionNames(llvm::StringSet<>& OptionNames) : OptionNames_(OptionNames)
-  {}
-
-  void operator()(const Init* Statement) {
-    // Statement is either a dag, or a list of dags.
-    if (typeid(*Statement) == typeid(ListInit)) {
-      const ListInit& DagList = *static_cast<const ListInit*>(Statement);
-      for (ListInit::const_iterator B = DagList.begin(), E = DagList.end();
-           B != E; ++B)
-        this->processDag(*B);
-    }
-    else {
-      this->processDag(Statement);
-    }
-  }
-
-  void operator()(const DagInit& Test, unsigned, bool) {
-    this->operator()(&Test);
-  }
-  void operator()(const Init* Statement, unsigned) {
-    this->operator()(Statement);
-  }
-};
-
-/// IsOptionalEdge - Validate that the 'optional_edge' has proper structure.
-bool IsOptionalEdge (const DagInit& Edg) {
-  return (GetOperatorName(Edg) == "optional_edge") && (Edg.getNumArgs() > 2);
-}
-
-/// CheckForSuperfluousOptions - Check that there are no side
-/// effect-free options (specified only in the OptionList). Otherwise,
-/// output a warning.
-void CheckForSuperfluousOptions (const DagVector& EdgeVector,
-                                 const ToolDescriptions& ToolDescs,
-                                 const OptionDescriptions& OptDescs) {
-  llvm::StringSet<> nonSuperfluousOptions;
-
-  // Add all options mentioned in the ToolDesc.Actions to the set of
-  // non-superfluous options.
-  for (ToolDescriptions::const_iterator B = ToolDescs.begin(),
-         E = ToolDescs.end(); B != E; ++B) {
-    const ToolDescription& TD = *(*B);
-    ExtractOptionNames Callback(nonSuperfluousOptions);
-    if (TD.Actions)
-      WalkCase(TD.Actions, Callback, Callback);
-  }
-
-  // Add all options mentioned in the 'case' clauses of the
-  // OptionalEdges of the compilation graph to the set of
-  // non-superfluous options.
-  for (DagVector::const_iterator B = EdgeVector.begin(),
-         E = EdgeVector.end(); B != E; ++B) {
-    const DagInit& Edge = **B;
-    if (IsOptionalEdge(Edge)) {
-      const DagInit& Weight = InitPtrToDag(Edge.getArg(2));
-      WalkCase(&Weight, ExtractOptionNames(nonSuperfluousOptions), Id());
-    }
-  }
-
-  // Check that all options in OptDescs belong to the set of
-  // non-superfluous options.
-  for (OptionDescriptions::const_iterator B = OptDescs.begin(),
-         E = OptDescs.end(); B != E; ++B) {
-    const OptionDescription& Val = B->second;
-    if (!nonSuperfluousOptions.count(Val.Name)
-        && Val.Type != OptionType::Alias)
-      llvm::errs() << "Warning: option '-" << Val.Name << "' has no effect! "
-        "Probable cause: this option is specified only in the OptionList.\n";
-  }
-}
-
-/// EmitCaseTest0Args - Helper function used by EmitCaseConstructHandler().
-bool EmitCaseTest0Args(const std::string& TestName, raw_ostream& O) {
-  if (TestName == "single_input_file") {
-    O << "InputFilenames.size() == 1";
-    return true;
-  }
-  else if (TestName == "multiple_input_files") {
-    O << "InputFilenames.size() > 1";
-    return true;
-  }
-
-  return false;
-}
-
-/// EmitMultipleArgumentTest - Helper function used by
-/// EmitCaseTestMultipleArgs()
-template <typename F>
-void EmitMultipleArgumentTest(const DagInit& D, const char* LogicOp,
-                              F Callback, raw_ostream& O)
-{
-  for (unsigned i = 0, NumArgs = D.getNumArgs(); i < NumArgs; ++i) {
-    if (i != 0)
-       O << ' ' << LogicOp << ' ';
-    Callback(InitPtrToString(D.getArg(i)), O);
-  }
-}
-
-// Callbacks for use with EmitMultipleArgumentTest
-
-class EmitSwitchOn {
-  const OptionDescriptions& OptDescs_;
-public:
-  EmitSwitchOn(const OptionDescriptions& OptDescs) : OptDescs_(OptDescs)
-  {}
-
-  void operator()(const std::string& OptName, raw_ostream& O) const {
-    const OptionDescription& OptDesc = OptDescs_.FindSwitch(OptName);
-    O << OptDesc.GenVariableName();
-  }
-};
-
-class EmitEmptyTest {
-  bool EmitNegate_;
-  const OptionDescriptions& OptDescs_;
-public:
-  EmitEmptyTest(bool EmitNegate, const OptionDescriptions& OptDescs)
-    : EmitNegate_(EmitNegate), OptDescs_(OptDescs)
-  {}
-
-  void operator()(const std::string& OptName, raw_ostream& O) const {
-    const char* Neg = (EmitNegate_ ? "!" : "");
-    if (OptName == "o") {
-      O << Neg << "OutputFilename.empty()";
-    }
-    else if (OptName == "save-temps") {
-      O << Neg << "(SaveTemps == SaveTempsEnum::Unset)";
-    }
-    else {
-      const OptionDescription& OptDesc = OptDescs_.FindListOrParameter(OptName);
-      O << Neg << OptDesc.GenVariableName() << ".empty()";
-    }
-  }
-};
-
-
-/// EmitCaseTestMultipleArgs - Helper function used by EmitCaseTest1Arg()
-bool EmitCaseTestMultipleArgs (const std::string& TestName,
-                               const DagInit& d,
-                               const OptionDescriptions& OptDescs,
-                               raw_ostream& O) {
-  if (TestName == "any_switch_on") {
-    EmitMultipleArgumentTest(d, "||", EmitSwitchOn(OptDescs), O);
-    return true;
-  }
-  else if (TestName == "switch_on") {
-    EmitMultipleArgumentTest(d, "&&", EmitSwitchOn(OptDescs), O);
-    return true;
-  }
-  else if (TestName == "any_not_empty") {
-    EmitMultipleArgumentTest(d, "||", EmitEmptyTest(true, OptDescs), O);
-    return true;
-  }
-  else if (TestName == "any_empty") {
-    EmitMultipleArgumentTest(d, "||", EmitEmptyTest(false, OptDescs), O);
-    return true;
-  }
-  else if (TestName == "not_empty") {
-    EmitMultipleArgumentTest(d, "&&", EmitEmptyTest(true, OptDescs), O);
-    return true;
-  }
-  else if (TestName == "empty") {
-    EmitMultipleArgumentTest(d, "&&", EmitEmptyTest(false, OptDescs), O);
-    return true;
-  }
-
-  return false;
-}
-
-/// EmitCaseTest1Arg - Helper function used by EmitCaseTest1OrMoreArgs()
-bool EmitCaseTest1Arg (const std::string& TestName,
-                       const DagInit& d,
-                       const OptionDescriptions& OptDescs,
-                       raw_ostream& O) {
-  const std::string& Arg = InitPtrToString(d.getArg(0));
-
-  if (TestName == "input_languages_contain") {
-    O << "InLangs.count(\"" << Arg << "\") != 0";
-    return true;
-  }
-  else if (TestName == "in_language") {
-    // This works only for single-argument Tool::GenerateAction. Join
-    // tools can process several files in different languages simultaneously.
-
-    // TODO: make this work with Edge::Weight (if possible).
-    O << "LangMap.GetLanguage(inFile) == \"" << Arg << '\"';
-    return true;
-  }
-
-  return false;
-}
-
-/// EmitCaseTest1OrMoreArgs - Helper function used by
-/// EmitCaseConstructHandler()
-bool EmitCaseTest1OrMoreArgs(const std::string& TestName,
-                             const DagInit& d,
-                             const OptionDescriptions& OptDescs,
-                             raw_ostream& O) {
-  CheckNumberOfArguments(d, 1);
-  return EmitCaseTest1Arg(TestName, d, OptDescs, O) ||
-    EmitCaseTestMultipleArgs(TestName, d, OptDescs, O);
-}
-
-/// EmitCaseTest2Args - Helper function used by EmitCaseConstructHandler().
-bool EmitCaseTest2Args(const std::string& TestName,
-                       const DagInit& d,
-                       unsigned IndentLevel,
-                       const OptionDescriptions& OptDescs,
-                       raw_ostream& O) {
-  CheckNumberOfArguments(d, 2);
-  const std::string& OptName = InitPtrToString(d.getArg(0));
-  const std::string& OptArg = InitPtrToString(d.getArg(1));
-
-  if (TestName == "parameter_equals") {
-    const OptionDescription& OptDesc = OptDescs.FindParameter(OptName);
-    O << OptDesc.GenVariableName() << " == \"" << OptArg << "\"";
-    return true;
-  }
-  else if (TestName == "element_in_list") {
-    const OptionDescription& OptDesc = OptDescs.FindParameterList(OptName);
-    const std::string& VarName = OptDesc.GenVariableName();
-    O << "std::find(" << VarName << ".begin(),\n";
-    O.indent(IndentLevel + Indent1)
-      << VarName << ".end(), \""
-      << OptArg << "\") != " << VarName << ".end()";
-    return true;
-  }
-
-  return false;
-}
-
-// Forward declaration.
-// EmitLogicalOperationTest and EmitCaseTest are mutually recursive.
-void EmitCaseTest(const DagInit& d, unsigned IndentLevel,
-                  const OptionDescriptions& OptDescs,
-                  raw_ostream& O);
-
-/// EmitLogicalOperationTest - Helper function used by
-/// EmitCaseConstructHandler.
-void EmitLogicalOperationTest(const DagInit& d, const char* LogicOp,
-                              unsigned IndentLevel,
-                              const OptionDescriptions& OptDescs,
-                              raw_ostream& O) {
-  O << '(';
-  for (unsigned i = 0, NumArgs = d.getNumArgs(); i < NumArgs; ++i) {
-    const DagInit& InnerTest = InitPtrToDag(d.getArg(i));
-    EmitCaseTest(InnerTest, IndentLevel, OptDescs, O);
-    if (i != NumArgs - 1) {
-      O << ")\n";
-      O.indent(IndentLevel + Indent1) << ' ' << LogicOp << " (";
-    }
-    else {
-      O << ')';
-    }
-  }
-}
-
-void EmitLogicalNot(const DagInit& d, unsigned IndentLevel,
-                    const OptionDescriptions& OptDescs, raw_ostream& O)
-{
-  CheckNumberOfArguments(d, 1);
-  const DagInit& InnerTest = InitPtrToDag(d.getArg(0));
-  O << "! (";
-  EmitCaseTest(InnerTest, IndentLevel, OptDescs, O);
-  O << ")";
-}
-
-/// EmitCaseTest - Helper function used by EmitCaseConstructHandler.
-void EmitCaseTest(const DagInit& d, unsigned IndentLevel,
-                  const OptionDescriptions& OptDescs,
-                  raw_ostream& O) {
-  const std::string& TestName = GetOperatorName(d);
-
-  if (TestName == "and")
-    EmitLogicalOperationTest(d, "&&", IndentLevel, OptDescs, O);
-  else if (TestName == "or")
-    EmitLogicalOperationTest(d, "||", IndentLevel, OptDescs, O);
-  else if (TestName == "not")
-    EmitLogicalNot(d, IndentLevel, OptDescs, O);
-  else if (EmitCaseTest0Args(TestName, O))
-    return;
-  else if (EmitCaseTest1OrMoreArgs(TestName, d, OptDescs, O))
-    return;
-  else if (EmitCaseTest2Args(TestName, d, IndentLevel, OptDescs, O))
-    return;
-  else
-    throw "Unknown test '" + TestName + "' used in the 'case' construct!";
-}
-
-
-/// EmitCaseTestCallback - Callback used by EmitCaseConstructHandler.
-class EmitCaseTestCallback {
-  bool EmitElseIf_;
-  const OptionDescriptions& OptDescs_;
-  raw_ostream& O_;
-public:
-
-  EmitCaseTestCallback(bool EmitElseIf,
-                       const OptionDescriptions& OptDescs, raw_ostream& O)
-    : EmitElseIf_(EmitElseIf), OptDescs_(OptDescs), O_(O)
-  {}
-
-  void operator()(const DagInit& Test, unsigned IndentLevel, bool FirstTest)
-  {
-    if (GetOperatorName(Test) == "default") {
-      O_.indent(IndentLevel) << "else {\n";
-    }
-    else {
-      O_.indent(IndentLevel)
-        << ((!FirstTest && EmitElseIf_) ? "else if (" : "if (");
-      EmitCaseTest(Test, IndentLevel, OptDescs_, O_);
-      O_ << ") {\n";
-    }
-  }
-};
-
-/// EmitCaseStatementCallback - Callback used by EmitCaseConstructHandler.
-template <typename F>
-class EmitCaseStatementCallback {
-  F Callback_;
-  raw_ostream& O_;
-public:
-
-  EmitCaseStatementCallback(F Callback, raw_ostream& O)
-    : Callback_(Callback), O_(O)
-  {}
-
-  void operator() (const Init* Statement, unsigned IndentLevel) {
-    // Is this a nested 'case'?
-    bool IsCase = dynamic_cast<const DagInit*>(Statement) &&
-      GetOperatorName(static_cast<const DagInit&>(*Statement)) == "case";
-
-    // If so, ignore it, it is handled by our caller, WalkCase.
-    if (!IsCase) {
-      if (typeid(*Statement) == typeid(ListInit)) {
-        const ListInit& DagList = *static_cast<const ListInit*>(Statement);
-        for (ListInit::const_iterator B = DagList.begin(), E = DagList.end();
-             B != E; ++B)
-          Callback_(*B, (IndentLevel + Indent1), O_);
-      }
-      else {
-        Callback_(Statement, (IndentLevel + Indent1), O_);
-      }
-    }
-    O_.indent(IndentLevel) << "}\n";
-  }
-
-};
-
-/// EmitCaseConstructHandler - Emit code that handles the 'case'
-/// construct. Takes a function object that should emit code for every case
-/// clause. Implemented on top of WalkCase.
-/// Callback's type is void F(const Init* Statement, unsigned IndentLevel,
-/// raw_ostream& O).
-/// EmitElseIf parameter controls the type of condition that is emitted ('if
-/// (..) {..} else if (..) {} .. else {..}' vs. 'if (..) {..} if(..)  {..}
-/// .. else {..}').
-template <typename F>
-void EmitCaseConstructHandler(const Init* Case, unsigned IndentLevel,
-                              F Callback, bool EmitElseIf,
-                              const OptionDescriptions& OptDescs,
-                              raw_ostream& O) {
-  WalkCase(Case, EmitCaseTestCallback(EmitElseIf, OptDescs, O),
-           EmitCaseStatementCallback<F>(Callback, O), IndentLevel);
-}
-
-/// TokenizeCmdLine - converts from
-/// "$CALL(HookName, 'Arg1', 'Arg2')/path -arg1 -arg2" to
-/// ["$CALL(", "HookName", "Arg1", "Arg2", ")/path", "-arg1", "-arg2"].
-void TokenizeCmdLine(const std::string& CmdLine, StrVector& Out) {
-  const char* Delimiters = " \t\n\v\f\r";
-  enum TokenizerState
-  { Normal, SpecialCommand, InsideSpecialCommand, InsideQuotationMarks }
-  cur_st  = Normal;
-
-  if (CmdLine.empty())
-    return;
-  Out.push_back("");
-
-  std::string::size_type B = CmdLine.find_first_not_of(Delimiters),
-    E = CmdLine.size();
-
-  for (; B != E; ++B) {
-    char cur_ch = CmdLine[B];
-
-    switch (cur_st) {
-    case Normal:
-      if (cur_ch == '$') {
-        cur_st = SpecialCommand;
-        break;
-      }
-      if (OneOf(Delimiters, cur_ch)) {
-        // Skip whitespace
-        B = CmdLine.find_first_not_of(Delimiters, B);
-        if (B == std::string::npos) {
-          B = E-1;
-          continue;
-        }
-        --B;
-        Out.push_back("");
-        continue;
-      }
-      break;
-
-
-    case SpecialCommand:
-      if (OneOf(Delimiters, cur_ch)) {
-        cur_st = Normal;
-        Out.push_back("");
-        continue;
-      }
-      if (cur_ch == '(') {
-        Out.push_back("");
-        cur_st = InsideSpecialCommand;
-        continue;
-      }
-      break;
-
-    case InsideSpecialCommand:
-      if (OneOf(Delimiters, cur_ch)) {
-        continue;
-      }
-      if (cur_ch == '\'') {
-        cur_st = InsideQuotationMarks;
-        Out.push_back("");
-        continue;
-      }
-      if (cur_ch == ')') {
-        cur_st = Normal;
-        Out.push_back("");
-      }
-      if (cur_ch == ',') {
-        continue;
-      }
-
-      break;
-
-    case InsideQuotationMarks:
-      if (cur_ch == '\'') {
-        cur_st = InsideSpecialCommand;
-        continue;
-      }
-      break;
-    }
-
-    Out.back().push_back(cur_ch);
-  }
-}
-
-/// SubstituteCall - Given "$CALL(HookName, [Arg1 [, Arg2 [...]]])", output
-/// "hooks::HookName([Arg1 [, Arg2 [, ...]]])". Helper function used by
-/// SubstituteSpecialCommands().
-StrVector::const_iterator
-SubstituteCall (StrVector::const_iterator Pos,
-                StrVector::const_iterator End,
-                bool IsJoin, raw_ostream& O)
-{
-  const char* errorMessage = "Syntax error in $CALL invocation!";
-  CheckedIncrement(Pos, End, errorMessage);
-  const std::string& CmdName = *Pos;
-
-  if (CmdName == ")")
-    throw "$CALL invocation: empty argument list!";
-
-  O << "hooks::";
-  O << CmdName << "(";
-
-
-  bool firstIteration = true;
-  while (true) {
-    CheckedIncrement(Pos, End, errorMessage);
-    const std::string& Arg = *Pos;
-    assert(Arg.size() != 0);
-
-    if (Arg[0] == ')')
-      break;
-
-    if (firstIteration)
-      firstIteration = false;
-    else
-      O << ", ";
-
-    if (Arg == "$INFILE") {
-      if (IsJoin)
-        throw "$CALL(Hook, $INFILE) can't be used with a Join tool!";
-      else
-        O << "inFile.c_str()";
-    }
-    else {
-      O << '"' << Arg << '"';
-    }
-  }
-
-  O << ')';
-
-  return Pos;
-}
-
-/// SubstituteEnv - Given '$ENV(VAR_NAME)', output 'getenv("VAR_NAME")'. Helper
-/// function used by SubstituteSpecialCommands().
-StrVector::const_iterator
-SubstituteEnv (StrVector::const_iterator Pos,
-               StrVector::const_iterator End, raw_ostream& O)
-{
-  const char* errorMessage = "Syntax error in $ENV invocation!";
-  CheckedIncrement(Pos, End, errorMessage);
-  const std::string& EnvName = *Pos;
-
-  if (EnvName == ")")
-    throw "$ENV invocation: empty argument list!";
-
-  O << "checkCString(std::getenv(\"";
-  O << EnvName;
-  O << "\"))";
-
-  CheckedIncrement(Pos, End, errorMessage);
-
-  return Pos;
-}
-
-/// SubstituteSpecialCommands - Given an invocation of $CALL or $ENV, output
-/// handler code. Helper function used by EmitCmdLineVecFill().
-StrVector::const_iterator
-SubstituteSpecialCommands (StrVector::const_iterator Pos,
-                           StrVector::const_iterator End,
-                           bool IsJoin, raw_ostream& O)
-{
-
-  const std::string& cmd = *Pos;
-
-  // Perform substitution.
-  if (cmd == "$CALL") {
-    Pos = SubstituteCall(Pos, End, IsJoin, O);
-  }
-  else if (cmd == "$ENV") {
-    Pos = SubstituteEnv(Pos, End, O);
-  }
-  else {
-    throw "Unknown special command: " + cmd;
-  }
-
-  // Handle '$CMD(ARG)/additional/text'.
-  const std::string& Leftover = *Pos;
-  assert(Leftover.at(0) == ')');
-  if (Leftover.size() != 1)
-    O << " + std::string(\"" << (Leftover.c_str() + 1) << "\")";
-
-  return Pos;
-}
-
-/// EmitCmdLineVecFill - Emit code that fills in the command line
-/// vector. Helper function used by EmitGenerateActionMethod().
-void EmitCmdLineVecFill(const Init* CmdLine, const std::string& ToolName,
-                        bool IsJoin, unsigned IndentLevel,
-                        raw_ostream& O) {
-  StrVector StrVec;
-  TokenizeCmdLine(InitPtrToString(CmdLine), StrVec);
-
-  if (StrVec.empty())
-    throw "Tool '" + ToolName + "' has empty command line!";
-
-  StrVector::const_iterator B = StrVec.begin(), E = StrVec.end();
-
-  // Emit the command itself.
-  assert(!StrVec[0].empty());
-  O.indent(IndentLevel) << "cmd = ";
-  if (StrVec[0][0] == '$') {
-    B = SubstituteSpecialCommands(B, E, IsJoin, O);
-    ++B;
-  }
-  else {
-    O << '"' << StrVec[0] << '"';
-    ++B;
-  }
-  O << ";\n";
-
-  // Go through the command arguments.
-  assert(B <= E);
-  for (; B != E; ++B) {
-    const std::string& cmd = *B;
-
-    assert(!cmd.empty());
-    O.indent(IndentLevel);
-
-    if (cmd.at(0) == '$') {
-      O << "vec.push_back(std::make_pair(0, ";
-      B = SubstituteSpecialCommands(B, E, IsJoin, O);
-      O << "));\n";
-    }
-    else {
-      O << "vec.push_back(std::make_pair(0, \"" << cmd << "\"));\n";
-    }
-  }
-
-}
-
-/// EmitForEachListElementCycleHeader - Emit common code for iterating through
-/// all elements of a list. Helper function used by
-/// EmitForwardOptionPropertyHandlingCode.
-void EmitForEachListElementCycleHeader (const OptionDescription& D,
-                                        unsigned IndentLevel,
-                                        raw_ostream& O) {
-  unsigned IndentLevel1 = IndentLevel + Indent1;
-
-  O.indent(IndentLevel)
-    << "for (" << D.GenTypeDeclaration()
-    << "::iterator B = " << D.GenVariableName() << ".begin(),\n";
-  O.indent(IndentLevel)
-    << "E = " << D.GenVariableName() << ".end(); B != E;) {\n";
-  O.indent(IndentLevel1) << "unsigned pos = " << D.GenVariableName()
-                         << ".getPosition(B - " << D.GenVariableName()
-                         << ".begin());\n";
-}
-
-/// EmitForwardOptionPropertyHandlingCode - Helper function used to
-/// implement EmitActionHandler. Emits code for
-/// handling the (forward) and (forward_as) option properties.
-void EmitForwardOptionPropertyHandlingCode (const OptionDescription& D,
-                                            unsigned IndentLevel,
-                                            const std::string& NewName,
-                                            raw_ostream& O) {
-  const std::string& Name = NewName.empty()
-    ? ("-" + D.Name)
-    : NewName;
-  unsigned IndentLevel1 = IndentLevel + Indent1;
-
-  switch (D.Type) {
-  case OptionType::Switch:
-    O.indent(IndentLevel)
-      << "vec.push_back(std::make_pair(" << D.GenVariableName()
-      << ".getPosition(), \"" << Name << "\"));\n";
-    break;
-  case OptionType::Parameter:
-    O.indent(IndentLevel) << "vec.push_back(std::make_pair("
-                          << D.GenVariableName()
-                          <<".getPosition(), \"" << Name;
-
-    if (!D.isForwardNotSplit()) {
-      O << "\"));\n";
-      O.indent(IndentLevel) << "vec.push_back(std::make_pair("
-                            << D.GenVariableName() << ".getPosition(), "
-                            << D.GenVariableName() << "));\n";
-    }
-    else {
-      O << "=\" + " << D.GenVariableName() << "));\n";
-    }
-    break;
-  case OptionType::Prefix:
-    O.indent(IndentLevel) << "vec.push_back(std::make_pair("
-                          << D.GenVariableName() << ".getPosition(), \""
-                          << Name << "\" + "
-                          << D.GenVariableName() << "));\n";
-    break;
-  case OptionType::PrefixList:
-    EmitForEachListElementCycleHeader(D, IndentLevel, O);
-    O.indent(IndentLevel1) << "vec.push_back(std::make_pair(pos, \""
-                           << Name << "\" + " << "*B));\n";
-    O.indent(IndentLevel1) << "++B;\n";
-
-    for (int i = 1, j = D.MultiVal; i < j; ++i) {
-      O.indent(IndentLevel1) << "vec.push_back(std::make_pair(pos, *B));\n";
-      O.indent(IndentLevel1) << "++B;\n";
-    }
-
-    O.indent(IndentLevel) << "}\n";
-    break;
-  case OptionType::ParameterList:
-    EmitForEachListElementCycleHeader(D, IndentLevel, O);
-    O.indent(IndentLevel1) << "vec.push_back(std::make_pair(pos, \""
-                           << Name << "\"));\n";
-
-    for (int i = 0, j = D.MultiVal; i < j; ++i) {
-      O.indent(IndentLevel1) << "vec.push_back(std::make_pair(pos, *B));\n";
-      O.indent(IndentLevel1) << "++B;\n";
-    }
-
-    O.indent(IndentLevel) << "}\n";
-    break;
-  case OptionType::SwitchList:
-    EmitForEachListElementCycleHeader(D, IndentLevel, O);
-    O.indent(IndentLevel1) << "vec.push_back(std::make_pair(pos, \""
-                           << Name << "\"));\n";
-    O.indent(IndentLevel1) << "++B;\n";
-    O.indent(IndentLevel) << "}\n";
-    break;
-  case OptionType::Alias:
-  default:
-    throw "Aliases are not allowed in tool option descriptions!";
-  }
-}
-
-/// ActionHandlingCallbackBase - Base class of EmitActionHandlersCallback and
-/// EmitPreprocessOptionsCallback.
-struct ActionHandlingCallbackBase
-{
-
-  void onErrorDag(const DagInit& d,
-                  unsigned IndentLevel, raw_ostream& O) const
-  {
-    O.indent(IndentLevel)
-      << "PrintError(\""
-      << (d.getNumArgs() >= 1 ? InitPtrToString(d.getArg(0)) : "Unknown error!")
-      << "\");\n";
-    O.indent(IndentLevel) << "return 1;\n";
-  }
-
-  void onWarningDag(const DagInit& d,
-                    unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(d, 1);
-    O.indent(IndentLevel) << "llvm::errs() << \""
-                          << InitPtrToString(d.getArg(0)) << "\";\n";
-  }
-
-};
-
-/// EmitActionHandlersCallback - Emit code that handles actions. Used by
-/// EmitGenerateActionMethod() as an argument to EmitCaseConstructHandler().
-class EmitActionHandlersCallback;
-
-typedef void (EmitActionHandlersCallback::* EmitActionHandlersCallbackHandler)
-(const DagInit&, unsigned, raw_ostream&) const;
-
-class EmitActionHandlersCallback :
-  public ActionHandlingCallbackBase,
-  public HandlerTable<EmitActionHandlersCallbackHandler>
-{
-  typedef EmitActionHandlersCallbackHandler Handler;
-
-  const OptionDescriptions& OptDescs;
-
-  /// EmitHookInvocation - Common code for hook invocation from actions. Used by
-  /// onAppendCmd and onOutputSuffix.
-  void EmitHookInvocation(const std::string& Str,
-                          const char* BlockOpen, const char* BlockClose,
-                          unsigned IndentLevel, raw_ostream& O) const
-  {
-    StrVector Out;
-    TokenizeCmdLine(Str, Out);
-
-    for (StrVector::const_iterator B = Out.begin(), E = Out.end();
-         B != E; ++B) {
-      const std::string& cmd = *B;
-
-      O.indent(IndentLevel) << BlockOpen;
-
-      if (cmd.at(0) == '$')
-        B = SubstituteSpecialCommands(B, E,  /* IsJoin = */ true, O);
-      else
-        O << '"' << cmd << '"';
-
-      O << BlockClose;
-    }
-  }
-
-  void onAppendCmd (const DagInit& Dag,
-                    unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 1);
-    this->EmitHookInvocation(InitPtrToString(Dag.getArg(0)),
-                             "vec.push_back(std::make_pair(65536, ", "));\n",
-                             IndentLevel, O);
-  }
-
-  void onForward (const DagInit& Dag,
-                  unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 1);
-    const std::string& Name = InitPtrToString(Dag.getArg(0));
-    EmitForwardOptionPropertyHandlingCode(OptDescs.FindOption(Name),
-                                          IndentLevel, "", O);
-  }
-
-  void onForwardAs (const DagInit& Dag,
-                    unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 2);
-    const std::string& Name = InitPtrToString(Dag.getArg(0));
-    const std::string& NewName = InitPtrToString(Dag.getArg(1));
-    EmitForwardOptionPropertyHandlingCode(OptDescs.FindOption(Name),
-                                          IndentLevel, NewName, O);
-  }
-
-  void onForwardValue (const DagInit& Dag,
-                       unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 1);
-    const std::string& Name = InitPtrToString(Dag.getArg(0));
-    const OptionDescription& D = OptDescs.FindParameterListOrParameter(Name);
-
-    if (D.isSwitchList()) {
-      throw std::runtime_error
-        ("forward_value is not allowed with switch_list");
-    }
-
-    if (D.isParameter()) {
-      O.indent(IndentLevel) << "vec.push_back(std::make_pair("
-                            << D.GenVariableName() << ".getPosition(), "
-                            << D.GenVariableName() << "));\n";
-    }
-    else {
-      O.indent(IndentLevel) << "for (" << D.GenTypeDeclaration()
-                            << "::iterator B = " << D.GenVariableName()
-                            << ".begin(), \n";
-      O.indent(IndentLevel + Indent1) << " E = " << D.GenVariableName()
-                                      << ".end(); B != E; ++B)\n";
-      O.indent(IndentLevel) << "{\n";
-      O.indent(IndentLevel + Indent1)
-        << "unsigned pos = " << D.GenVariableName()
-        << ".getPosition(B - " << D.GenVariableName()
-        << ".begin());\n";
-      O.indent(IndentLevel + Indent1)
-        << "vec.push_back(std::make_pair(pos, *B));\n";
-      O.indent(IndentLevel) << "}\n";
-    }
-  }
-
-  void onForwardTransformedValue (const DagInit& Dag,
-                                  unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 2);
-    const std::string& Name = InitPtrToString(Dag.getArg(0));
-    const std::string& Hook = InitPtrToString(Dag.getArg(1));
-    const OptionDescription& D = OptDescs.FindParameterListOrParameter(Name);
-
-    O.indent(IndentLevel) << "vec.push_back(std::make_pair("
-                          << D.GenVariableName() << ".getPosition("
-                          << (D.isList() ? "0" : "") << "), "
-                          << "hooks::" << Hook << "(" << D.GenVariableName()
-                          << (D.isParameter() ? ".c_str()" : "") << ")));\n";
-  }
-
-  void onNoOutFile (const DagInit& Dag,
-                    unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 0);
-    O.indent(IndentLevel) << "no_out_file = true;\n";
-  }
-
-  void onOutputSuffix (const DagInit& Dag,
-                       unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(Dag, 1);
-    this->EmitHookInvocation(InitPtrToString(Dag.getArg(0)),
-                             "output_suffix = ", ";\n", IndentLevel, O);
-  }
-
-  void onStopCompilation (const DagInit& Dag,
-                          unsigned IndentLevel, raw_ostream& O) const
-  {
-    O.indent(IndentLevel) << "stop_compilation = true;\n";
-  }
-
-
-  void onUnpackValues (const DagInit& Dag,
-                       unsigned IndentLevel, raw_ostream& O) const
-  {
-    throw "'unpack_values' is deprecated. "
-      "Use 'comma_separated' + 'forward_value' instead!";
-  }
-
- public:
-
-  explicit EmitActionHandlersCallback(const OptionDescriptions& OD)
-    : OptDescs(OD)
-  {
-    if (!staticMembersInitialized_) {
-      AddHandler("error", &EmitActionHandlersCallback::onErrorDag);
-      AddHandler("warning", &EmitActionHandlersCallback::onWarningDag);
-      AddHandler("append_cmd", &EmitActionHandlersCallback::onAppendCmd);
-      AddHandler("forward", &EmitActionHandlersCallback::onForward);
-      AddHandler("forward_as", &EmitActionHandlersCallback::onForwardAs);
-      AddHandler("forward_value", &EmitActionHandlersCallback::onForwardValue);
-      AddHandler("forward_transformed_value",
-                 &EmitActionHandlersCallback::onForwardTransformedValue);
-      AddHandler("no_out_file",
-                 &EmitActionHandlersCallback::onNoOutFile);
-      AddHandler("output_suffix", &EmitActionHandlersCallback::onOutputSuffix);
-      AddHandler("stop_compilation",
-                 &EmitActionHandlersCallback::onStopCompilation);
-      AddHandler("unpack_values",
-                 &EmitActionHandlersCallback::onUnpackValues);
-
-
-      staticMembersInitialized_ = true;
-    }
-  }
-
-  void operator()(const Init* I,
-                  unsigned IndentLevel, raw_ostream& O) const
-  {
-    InvokeDagInitHandler(this, I, IndentLevel, O);
-  }
-};
-
-void EmitGenerateActionMethodHeader(const ToolDescription& D,
-                                    bool IsJoin, bool Naked,
-                                    raw_ostream& O)
-{
-  O.indent(Indent1) << "int GenerateAction(Action& Out,\n";
-
-  if (IsJoin)
-    O.indent(Indent2) << "const PathVector& inFiles,\n";
-  else
-    O.indent(Indent2) << "const sys::Path& inFile,\n";
-
-  O.indent(Indent2) << "const bool HasChildren,\n";
-  O.indent(Indent2) << "const llvm::sys::Path& TempDir,\n";
-  O.indent(Indent2) << "const InputLanguagesSet& InLangs,\n";
-  O.indent(Indent2) << "const LanguageMap& LangMap) const\n";
-  O.indent(Indent1) << "{\n";
-
-  if (!Naked) {
-    O.indent(Indent2) << "std::string cmd;\n";
-    O.indent(Indent2) << "std::string out_file;\n";
-    O.indent(Indent2)
-      << "std::vector<std::pair<unsigned, std::string> > vec;\n";
-    O.indent(Indent2) << "bool stop_compilation = !HasChildren;\n";
-    O.indent(Indent2) << "bool no_out_file = false;\n";
-    O.indent(Indent2) << "std::string output_suffix(\""
-                      << D.OutputSuffix << "\");\n";
-  }
-}
-
-// EmitGenerateActionMethod - Emit either a normal or a "join" version of the
-// Tool::GenerateAction() method.
-void EmitGenerateActionMethod (const ToolDescription& D,
-                               const OptionDescriptions& OptDescs,
-                               bool IsJoin, raw_ostream& O) {
-
-  EmitGenerateActionMethodHeader(D, IsJoin, /* Naked = */ false, O);
-
-  if (!D.CmdLine)
-    throw "Tool " + D.Name + " has no cmd_line property!";
-
-  // Process the 'command' property.
-  O << '\n';
-  EmitCmdLineVecFill(D.CmdLine, D.Name, IsJoin, Indent2, O);
-  O << '\n';
-
-  // Process the 'actions' list of this tool.
-  if (D.Actions)
-    EmitCaseConstructHandler(D.Actions, Indent2,
-                             EmitActionHandlersCallback(OptDescs),
-                             false, OptDescs, O);
-  O << '\n';
-
-  // Input file (s)
-  if (!D.InFileOption.empty()) {
-    O.indent(Indent2)
-      << "vec.push_back(std::make_pair(InputFilenames.getPosition(0), \""
-      << D.InFileOption << "\");\n";
-  }
-
-  if (IsJoin) {
-    O.indent(Indent2)
-      << "for (PathVector::const_iterator B = inFiles.begin(),\n";
-    O.indent(Indent3) << "E = inFiles.end(); B != E; ++B)\n";
-    O.indent(Indent2) << "{\n";
-    O.indent(Indent3) << "vec.push_back(std::make_pair("
-                      << "InputFilenames.getPosition(B - inFiles.begin()), "
-                      << "B->str()));\n";
-    O.indent(Indent2) << "}\n";
-  }
-  else {
-    O.indent(Indent2) << "vec.push_back(std::make_pair("
-                      << "InputFilenames.getPosition(0), inFile.str()));\n";
-  }
-
-  // Output file
-  O.indent(Indent2) << "if (!no_out_file) {\n";
-  if (!D.OutFileOption.empty())
-    O.indent(Indent3) << "vec.push_back(std::make_pair(65536, \""
-                      << D.OutFileOption << "\"));\n";
-
-  O.indent(Indent3) << "out_file = this->OutFilename("
-                    << (IsJoin ? "sys::Path(),\n" : "inFile,\n");
-  O.indent(Indent4) <<
-    "TempDir, stop_compilation, output_suffix.c_str()).str();\n\n";
-  O.indent(Indent3) << "vec.push_back(std::make_pair(65536, out_file));\n";
-
-  O.indent(Indent2) << "}\n\n";
-
-  // Handle the Sink property.
-  std::string SinkOption("autogenerated::");
-  SinkOption += SinkOptionName;
-  if (D.isSink()) {
-    O.indent(Indent2) << "if (!" << SinkOption << ".empty()) {\n";
-    O.indent(Indent3) << "for (cl::list<std::string>::iterator B = "
-                      << SinkOption << ".begin(), E = " << SinkOption
-                      << ".end(); B != E; ++B)\n";
-    O.indent(Indent4) << "vec.push_back(std::make_pair(" << SinkOption
-                      << ".getPosition(B - " << SinkOption
-                      <<  ".begin()), *B));\n";
-    O.indent(Indent2) << "}\n";
-  }
-
-  O.indent(Indent2) << "Out.Construct(cmd, this->SortArgs(vec), "
-                    << "stop_compilation, out_file);\n";
-  O.indent(Indent2) << "return 0;\n";
-  O.indent(Indent1) << "}\n\n";
-}
-
-/// EmitGenerateActionMethods - Emit two GenerateAction() methods for
-/// a given Tool class.
-void EmitGenerateActionMethods (const ToolDescription& ToolDesc,
-                                const OptionDescriptions& OptDescs,
-                                raw_ostream& O) {
-  if (!ToolDesc.isJoin()) {
-    EmitGenerateActionMethodHeader(ToolDesc, /* IsJoin = */ true,
-                                   /* Naked = */ true, O);
-    O.indent(Indent2) << "PrintError(\"" << ToolDesc.Name
-                      << " is not a Join tool!\");\n";
-    O.indent(Indent2) << "return -1;\n";
-    O.indent(Indent1) << "}\n\n";
-  }
-  else {
-    EmitGenerateActionMethod(ToolDesc, OptDescs, true, O);
-  }
-
-  EmitGenerateActionMethod(ToolDesc, OptDescs, false, O);
-}
-
-/// EmitInOutLanguageMethods - Emit the [Input,Output]Language()
-/// methods for a given Tool class.
-void EmitInOutLanguageMethods (const ToolDescription& D, raw_ostream& O) {
-  O.indent(Indent1) << "const char** InputLanguages() const {\n";
-  O.indent(Indent2) << "return InputLanguages_;\n";
-  O.indent(Indent1) << "}\n\n";
-
-  O.indent(Indent1) << "const char** OutputLanguages() const {\n";
-  O.indent(Indent2) << "return OutputLanguages_;\n";
-  O.indent(Indent1) << "}\n\n";
-}
-
-/// EmitNameMethod - Emit the Name() method for a given Tool class.
-void EmitNameMethod (const ToolDescription& D, raw_ostream& O) {
-  O.indent(Indent1) << "const char* Name() const {\n";
-  O.indent(Indent2) << "return \"" << D.Name << "\";\n";
-  O.indent(Indent1) << "}\n\n";
-}
-
-/// EmitIsJoinMethod - Emit the IsJoin() method for a given Tool
-/// class.
-void EmitIsJoinMethod (const ToolDescription& D, raw_ostream& O) {
-  O.indent(Indent1) << "bool IsJoin() const {\n";
-  if (D.isJoin())
-    O.indent(Indent2) << "return true;\n";
-  else
-    O.indent(Indent2) << "return false;\n";
-  O.indent(Indent1) << "}\n\n";
-}
-
-/// EmitWorksOnEmptyCallback - Callback used by EmitWorksOnEmptyMethod in
-/// conjunction with EmitCaseConstructHandler.
-void EmitWorksOnEmptyCallback (const Init* Value,
-                               unsigned IndentLevel, raw_ostream& O) {
-  CheckBooleanConstant(Value);
-  O.indent(IndentLevel) << "return " << Value->getAsString() << ";\n";
-}
-
-/// EmitWorksOnEmptyMethod - Emit the WorksOnEmpty() method for a given Tool
-/// class.
-void EmitWorksOnEmptyMethod (const ToolDescription& D,
-                             const OptionDescriptions& OptDescs,
-                             raw_ostream& O)
-{
-  O.indent(Indent1) << "bool WorksOnEmpty() const {\n";
-  if (D.OnEmpty == 0)
-    O.indent(Indent2) << "return false;\n";
-  else
-    EmitCaseConstructHandler(D.OnEmpty, Indent2, EmitWorksOnEmptyCallback,
-                             /*EmitElseIf = */ true, OptDescs, O);
-  O.indent(Indent1) << "}\n\n";
-}
-
-/// EmitStrArray - Emit definition of a 'const char**' static member
-/// variable. Helper used by EmitStaticMemberDefinitions();
-void EmitStrArray(const std::string& Name, const std::string& VarName,
-                  const StrVector& StrVec, raw_ostream& O) {
-  O << "const char* " << Name << "::" << VarName << "[] = {";
-  for (StrVector::const_iterator B = StrVec.begin(), E = StrVec.end();
-       B != E; ++B)
-    O << '\"' << *B << "\", ";
-  O << "0};\n";
-}
-
-/// EmitStaticMemberDefinitions - Emit static member definitions for a
-/// given Tool class.
-void EmitStaticMemberDefinitions(const ToolDescription& D, raw_ostream& O) {
-  if (D.InLanguage.empty())
-    throw "Tool " + D.Name + " has no 'in_language' property!";
-  if (D.OutLanguage.empty())
-    throw "Tool " + D.Name + " has no 'out_language' property!";
-
-  EmitStrArray(D.Name, "InputLanguages_", D.InLanguage, O);
-  EmitStrArray(D.Name, "OutputLanguages_", D.OutLanguage, O);
-  O << '\n';
-}
-
-/// EmitToolClassDefinition - Emit a Tool class definition.
-void EmitToolClassDefinition (const ToolDescription& D,
-                              const OptionDescriptions& OptDescs,
-                              raw_ostream& O) {
-  if (D.Name == "root")
-    return;
-
-  // Header
-  O << "class " << D.Name << " : public ";
-  if (D.isJoin())
-    O << "JoinTool";
-  else
-    O << "Tool";
-
-  O << " {\nprivate:\n";
-  O.indent(Indent1) << "static const char* InputLanguages_[];\n";
-  O.indent(Indent1) << "static const char* OutputLanguages_[];\n\n";
-
-  O << "public:\n";
-  EmitNameMethod(D, O);
-  EmitInOutLanguageMethods(D, O);
-  EmitIsJoinMethod(D, O);
-  EmitWorksOnEmptyMethod(D, OptDescs, O);
-  EmitGenerateActionMethods(D, OptDescs, O);
-
-  // Close class definition
-  O << "};\n";
-
-  EmitStaticMemberDefinitions(D, O);
-
-}
-
-/// EmitOptionDefinitions - Iterate over a list of option descriptions
-/// and emit registration code.
-void EmitOptionDefinitions (const OptionDescriptions& descs,
-                            bool HasSink, raw_ostream& O)
-{
-  std::vector<OptionDescription> Aliases;
-
-  // Emit static cl::Option variables.
-  for (OptionDescriptions::const_iterator B = descs.begin(),
-         E = descs.end(); B!=E; ++B) {
-    const OptionDescription& val = B->second;
-
-    if (val.Type == OptionType::Alias) {
-      Aliases.push_back(val);
-      continue;
-    }
-
-    O << val.GenTypeDeclaration() << ' '
-      << val.GenPlainVariableName();
-
-    O << "(\"" << val.Name << "\"\n";
-
-    if (val.Type == OptionType::Prefix || val.Type == OptionType::PrefixList)
-      O << ", cl::Prefix";
-
-    if (val.isRequired()) {
-      if (val.isList() && !val.isMultiVal())
-        O << ", cl::OneOrMore";
-      else
-        O << ", cl::Required";
-    }
-
-    if (val.isOptional())
-        O << ", cl::Optional";
-
-    if (val.isOneOrMore())
-        O << ", cl::OneOrMore";
-
-    if (val.isZeroOrMore())
-        O << ", cl::ZeroOrMore";
-
-    if (val.isReallyHidden())
-      O << ", cl::ReallyHidden";
-    else if (val.isHidden())
-      O << ", cl::Hidden";
-
-    if (val.isCommaSeparated())
-      O << ", cl::CommaSeparated";
-
-    if (val.MultiVal > 1)
-      O << ", cl::multi_val(" << val.MultiVal << ')';
-
-    if (val.InitVal) {
-      const std::string& str = val.InitVal->getAsString();
-      O << ", cl::init(" << str << ')';
-    }
-
-    if (!val.Help.empty())
-      O << ", cl::desc(\"" << val.Help << "\")";
-
-    O << ");\n\n";
-  }
-
-  // Emit the aliases (they should go after all the 'proper' options).
-  for (std::vector<OptionDescription>::const_iterator
-         B = Aliases.begin(), E = Aliases.end(); B != E; ++B) {
-    const OptionDescription& val = *B;
-
-    O << val.GenTypeDeclaration() << ' '
-      << val.GenPlainVariableName()
-      << "(\"" << val.Name << '\"';
-
-    const OptionDescription& D = descs.FindOption(val.Help);
-    O << ", cl::aliasopt(" << D.GenVariableName() << ")";
-
-    O << ", cl::desc(\"" << "An alias for -" + val.Help  << "\"));\n";
-  }
-
-  // Emit the sink option.
-  if (HasSink)
-    O << "cl::list<std::string> " << SinkOptionName << "(cl::Sink);\n";
-
-  O << '\n';
-}
-
-/// EmitPreprocessOptionsCallback - Helper function passed to
-/// EmitCaseConstructHandler() by EmitPreprocessOptions().
-
-class EmitPreprocessOptionsCallback;
-
-typedef void
-(EmitPreprocessOptionsCallback::* EmitPreprocessOptionsCallbackHandler)
-(const DagInit&, unsigned, raw_ostream&) const;
-
-class EmitPreprocessOptionsCallback :
-  public ActionHandlingCallbackBase,
-  public HandlerTable<EmitPreprocessOptionsCallbackHandler>
-{
-  typedef EmitPreprocessOptionsCallbackHandler Handler;
-  typedef void
-  (EmitPreprocessOptionsCallback::* HandlerImpl)
-  (const Init*, unsigned, raw_ostream&) const;
-
-  const OptionDescriptions& OptDescs_;
-
-  void onEachArgument(const DagInit& d, HandlerImpl h,
-                      unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(d, 1);
-
-    for (unsigned i = 0, NumArgs = d.getNumArgs(); i < NumArgs; ++i) {
-      ((this)->*(h))(d.getArg(i), IndentLevel, O);
-    }
-  }
-
-  void onUnsetOptionImpl(const Init* I,
-                         unsigned IndentLevel, raw_ostream& O) const
-  {
-    const std::string& OptName = InitPtrToString(I);
-    const OptionDescription& OptDesc = OptDescs_.FindOption(OptName);
-
-    if (OptDesc.isSwitch()) {
-      O.indent(IndentLevel) << OptDesc.GenVariableName() << " = false;\n";
-    }
-    else if (OptDesc.isParameter()) {
-      O.indent(IndentLevel) << OptDesc.GenVariableName() << " = \"\";\n";
-    }
-    else if (OptDesc.isList()) {
-      O.indent(IndentLevel) << OptDesc.GenVariableName() << ".clear();\n";
-    }
-    else {
-      throw "Can't apply 'unset_option' to alias option '" + OptName + "'!";
-    }
-  }
-
-  void onUnsetOption(const DagInit& d,
-                     unsigned IndentLevel, raw_ostream& O) const
-  {
-    this->onEachArgument(d, &EmitPreprocessOptionsCallback::onUnsetOptionImpl,
-                         IndentLevel, O);
-  }
-
-  void onSetOptionImpl(const DagInit& D,
-                       unsigned IndentLevel, raw_ostream& O) const {
-    CheckNumberOfArguments(D, 2);
-
-    const std::string& OptName = InitPtrToString(D.getArg(0));
-    const OptionDescription& OptDesc = OptDescs_.FindOption(OptName);
-    const Init* Value = D.getArg(1);
-
-    if (OptDesc.isList()) {
-      const ListInit& List = InitPtrToList(Value);
-
-      O.indent(IndentLevel) << OptDesc.GenVariableName() << ".clear();\n";
-      for (ListInit::const_iterator B = List.begin(), E = List.end();
-           B != E; ++B) {
-        const Init* CurElem = *B;
-        if (OptDesc.isSwitchList())
-          CheckBooleanConstant(CurElem);
-
-        O.indent(IndentLevel)
-          << OptDesc.GenVariableName() << ".push_back(\""
-          << (OptDesc.isSwitchList() ? CurElem->getAsString()
-              : InitPtrToString(CurElem))
-          << "\");\n";
-      }
-    }
-    else if (OptDesc.isSwitch()) {
-      CheckBooleanConstant(Value);
-      O.indent(IndentLevel) << OptDesc.GenVariableName()
-                            << " = " << Value->getAsString() << ";\n";
-    }
-    else if (OptDesc.isParameter()) {
-      const std::string& Str = InitPtrToString(Value);
-      O.indent(IndentLevel) << OptDesc.GenVariableName()
-                            << " = \"" << Str << "\";\n";
-    }
-    else {
-      throw "Can't apply 'set_option' to alias option '" + OptName + "'!";
-    }
-  }
-
-  void onSetSwitch(const Init* I,
-                   unsigned IndentLevel, raw_ostream& O) const {
-    const std::string& OptName = InitPtrToString(I);
-    const OptionDescription& OptDesc = OptDescs_.FindOption(OptName);
-
-    if (OptDesc.isSwitch())
-      O.indent(IndentLevel) << OptDesc.GenVariableName() << " = true;\n";
-    else
-      throw "set_option: -" + OptName + " is not a switch option!";
-  }
-
-  void onSetOption(const DagInit& d,
-                   unsigned IndentLevel, raw_ostream& O) const
-  {
-    CheckNumberOfArguments(d, 1);
-
-    // 2-argument form: (set_option "A", true), (set_option "B", "C"),
-    // (set_option "D", ["E", "F"])
-    if (d.getNumArgs() == 2) {
-      const OptionDescription& OptDesc =
-        OptDescs_.FindOption(InitPtrToString(d.getArg(0)));
-      const Init* Opt2 = d.getArg(1);
-
-      if (!OptDesc.isSwitch() || typeid(*Opt2) != typeid(StringInit)) {
-        this->onSetOptionImpl(d, IndentLevel, O);
-        return;
-      }
-    }
-
-    // Multiple argument form: (set_option "A"), (set_option "B", "C", "D")
-    this->onEachArgument(d, &EmitPreprocessOptionsCallback::onSetSwitch,
-                         IndentLevel, O);
-  }
-
-public:
-
-  EmitPreprocessOptionsCallback(const OptionDescriptions& OptDescs)
-  : OptDescs_(OptDescs)
-  {
-    if (!staticMembersInitialized_) {
-      AddHandler("error", &EmitPreprocessOptionsCallback::onErrorDag);
-      AddHandler("warning", &EmitPreprocessOptionsCallback::onWarningDag);
-      AddHandler("unset_option", &EmitPreprocessOptionsCallback::onUnsetOption);
-      AddHandler("set_option", &EmitPreprocessOptionsCallback::onSetOption);
-
-      staticMembersInitialized_ = true;
-    }
-  }
-
-  void operator()(const Init* I,
-                  unsigned IndentLevel, raw_ostream& O) const
-  {
-    InvokeDagInitHandler(this, I, IndentLevel, O);
-  }
-
-};
-
-/// EmitPreprocessOptions - Emit the PreprocessOptions() function.
-void EmitPreprocessOptions (const RecordKeeper& Records,
-                            const OptionDescriptions& OptDecs, raw_ostream& O)
-{
-  O << "int PreprocessOptions () {\n";
-
-  const RecordVector& OptionPreprocessors =
-    Records.getAllDerivedDefinitions("OptionPreprocessor");
-
-  for (RecordVector::const_iterator B = OptionPreprocessors.begin(),
-         E = OptionPreprocessors.end(); B!=E; ++B) {
-    DagInit* Case = (*B)->getValueAsDag("preprocessor");
-    EmitCaseConstructHandler(Case, Indent1,
-                             EmitPreprocessOptionsCallback(OptDecs),
-                             false, OptDecs, O);
-  }
-
-  O << '\n';
-  O.indent(Indent1) << "return 0;\n";
-  O << "}\n\n";
-}
-
-class DoEmitPopulateLanguageMap;
-typedef void (DoEmitPopulateLanguageMap::* DoEmitPopulateLanguageMapHandler)
-(const DagInit& D);
-
-class DoEmitPopulateLanguageMap
-: public HandlerTable<DoEmitPopulateLanguageMapHandler>
-{
-private:
-  raw_ostream& O_;
-
-public:
-
-  explicit DoEmitPopulateLanguageMap (raw_ostream& O) : O_(O) {
-    if (!staticMembersInitialized_) {
-      AddHandler("lang_to_suffixes",
-                 &DoEmitPopulateLanguageMap::onLangToSuffixes);
-
-      staticMembersInitialized_ = true;
-    }
-  }
-
-  void operator() (Init* I) {
-    InvokeDagInitHandler(this, I);
-  }
-
-private:
-
-  void onLangToSuffixes (const DagInit& d) {
-    CheckNumberOfArguments(d, 2);
-
-    const std::string& Lang = InitPtrToString(d.getArg(0));
-    Init* Suffixes = d.getArg(1);
-
-    // Second argument to lang_to_suffixes is either a single string...
-    if (typeid(*Suffixes) == typeid(StringInit)) {
-      O_.indent(Indent1) << "langMap[\"" << InitPtrToString(Suffixes)
-                         << "\"] = \"" << Lang << "\";\n";
-    }
-    // ...or a list of strings.
-    else {
-      const ListInit& Lst = InitPtrToList(Suffixes);
-      assert(Lst.size() != 0);
-      for (ListInit::const_iterator B = Lst.begin(), E = Lst.end();
-           B != E; ++B) {
-        O_.indent(Indent1) << "langMap[\"" << InitPtrToString(*B)
-                           << "\"] = \"" << Lang << "\";\n";
-      }
-    }
-  }
-
-};
-
-/// EmitPopulateLanguageMap - Emit the PopulateLanguageMap() function.
-void EmitPopulateLanguageMap (const RecordKeeper& Records, raw_ostream& O)
-{
-  O << "int PopulateLanguageMap (LanguageMap& langMap) {\n";
-
-  // For each LanguageMap:
-  const RecordVector& LangMaps =
-    Records.getAllDerivedDefinitions("LanguageMap");
-
-  // Call DoEmitPopulateLanguageMap.
-  for (RecordVector::const_iterator B = LangMaps.begin(),
-         E = LangMaps.end(); B!=E; ++B) {
-    ListInit* LangMap = (*B)->getValueAsListInit("map");
-    std::for_each(LangMap->begin(), LangMap->end(),
-                  DoEmitPopulateLanguageMap(O));
-  }
-
-  O << '\n';
-  O.indent(Indent1) << "return 0;\n";
-  O << "}\n\n";
-}
-
-/// EmitEdgePropertyHandlerCallback - Emits code that handles edge
-/// properties. Helper function passed to EmitCaseConstructHandler() by
-/// EmitEdgeClass().
-void EmitEdgePropertyHandlerCallback (const Init* i, unsigned IndentLevel,
-                                      raw_ostream& O) {
-  const DagInit& d = InitPtrToDag(i);
-  const std::string& OpName = GetOperatorName(d);
-
-  if (OpName == "inc_weight") {
-    O.indent(IndentLevel) << "ret += ";
-  }
-  else if (OpName == "error") {
-    CheckNumberOfArguments(d, 1);
-    O.indent(IndentLevel) << "PrintError(\""
-                          << InitPtrToString(d.getArg(0))
-                          << "\");\n";
-    O.indent(IndentLevel) << "return -1;\n";
-    return;
-  }
-  else {
-    throw "Unknown operator in edge properties list: '" + OpName + "'!"
-      "\nOnly 'inc_weight', 'dec_weight' and 'error' are allowed.";
-  }
-
-  if (d.getNumArgs() > 0)
-    O << InitPtrToInt(d.getArg(0)) << ";\n";
-  else
-    O << "2;\n";
-
-}
-
-/// EmitEdgeClass - Emit a single Edge# class.
-void EmitEdgeClass (unsigned N, const std::string& Target,
-                    const DagInit& Case, const OptionDescriptions& OptDescs,
-                    raw_ostream& O) {
-
-  // Class constructor.
-  O << "class Edge" << N << ": public Edge {\n"
-    << "public:\n";
-  O.indent(Indent1) << "Edge" << N << "() : Edge(\"" << Target
-                    << "\") {}\n\n";
-
-  // Function Weight().
-  O.indent(Indent1)
-    << "int Weight(const InputLanguagesSet& InLangs) const {\n";
-  O.indent(Indent2) << "unsigned ret = 0;\n";
-
-  // Handle the 'case' construct.
-  EmitCaseConstructHandler(&Case, Indent2, EmitEdgePropertyHandlerCallback,
-                           false, OptDescs, O);
-
-  O.indent(Indent2) << "return ret;\n";
-  O.indent(Indent1) << "}\n\n};\n\n";
-}
-
-/// EmitEdgeClasses - Emit Edge* classes that represent graph edges.
-void EmitEdgeClasses (const DagVector& EdgeVector,
-                      const OptionDescriptions& OptDescs,
-                      raw_ostream& O) {
-  int i = 0;
-  for (DagVector::const_iterator B = EdgeVector.begin(),
-         E = EdgeVector.end(); B != E; ++B) {
-    const DagInit& Edge = **B;
-    const std::string& Name = GetOperatorName(Edge);
-
-    if (Name == "optional_edge") {
-      assert(IsOptionalEdge(Edge));
-      const std::string& NodeB = InitPtrToString(Edge.getArg(1));
-
-      const DagInit& Weight = InitPtrToDag(Edge.getArg(2));
-      EmitEdgeClass(i, NodeB, Weight, OptDescs, O);
-    }
-    else if (Name != "edge") {
-      throw "Unknown edge class: '" + Name + "'!";
-    }
-
-    ++i;
-  }
-}
-
-/// EmitPopulateCompilationGraph - Emit the PopulateCompilationGraph() function.
-void EmitPopulateCompilationGraph (const DagVector& EdgeVector,
-                                   const ToolDescriptions& ToolDescs,
-                                   raw_ostream& O)
-{
-  O << "int PopulateCompilationGraph (CompilationGraph& G) {\n";
-
-  for (ToolDescriptions::const_iterator B = ToolDescs.begin(),
-         E = ToolDescs.end(); B != E; ++B)
-    O.indent(Indent1) << "G.insertNode(new " << (*B)->Name << "());\n";
-
-  O << '\n';
-
-  // Insert edges.
-
-  int i = 0;
-  for (DagVector::const_iterator B = EdgeVector.begin(),
-         E = EdgeVector.end(); B != E; ++B) {
-    const DagInit& Edge = **B;
-    const std::string& NodeA = InitPtrToString(Edge.getArg(0));
-    const std::string& NodeB = InitPtrToString(Edge.getArg(1));
-
-    O.indent(Indent1) << "if (int ret = G.insertEdge(\"" << NodeA << "\", ";
-
-    if (IsOptionalEdge(Edge))
-      O << "new Edge" << i << "()";
-    else
-      O << "new SimpleEdge(\"" << NodeB << "\")";
-
-    O << "))\n";
-    O.indent(Indent2) << "return ret;\n";
-
-    ++i;
-  }
-
-  O << '\n';
-  O.indent(Indent1) << "return 0;\n";
-  O << "}\n\n";
-}
-
-/// HookInfo - Information about the hook type and number of arguments.
-struct HookInfo {
-
-  // A hook can either have a single parameter of type std::vector<std::string>,
-  // or NumArgs parameters of type const char*.
-  enum HookType { ListHook, ArgHook };
-
-  HookType Type;
-  unsigned NumArgs;
-
-  HookInfo() : Type(ArgHook), NumArgs(1)
-  {}
-
-  HookInfo(HookType T) : Type(T), NumArgs(1)
-  {}
-
-  HookInfo(unsigned N) : Type(ArgHook), NumArgs(N)
-  {}
-};
-
-typedef llvm::StringMap<HookInfo> HookInfoMap;
-
-/// ExtractHookNames - Extract the hook names from all instances of
-/// $CALL(HookName) in the provided command line string/action. Helper
-/// function used by FillInHookNames().
-class ExtractHookNames {
-  HookInfoMap& HookNames_;
-  const OptionDescriptions& OptDescs_;
-public:
-  ExtractHookNames(HookInfoMap& HookNames, const OptionDescriptions& OptDescs)
-    : HookNames_(HookNames), OptDescs_(OptDescs)
-  {}
-
-  void onAction (const DagInit& Dag) {
-    const std::string& Name = GetOperatorName(Dag);
-
-    if (Name == "forward_transformed_value") {
-      CheckNumberOfArguments(Dag, 2);
-      const std::string& OptName = InitPtrToString(Dag.getArg(0));
-      const std::string& HookName = InitPtrToString(Dag.getArg(1));
-      const OptionDescription& D =
-        OptDescs_.FindParameterListOrParameter(OptName);
-
-      HookNames_[HookName] = HookInfo(D.isList() ? HookInfo::ListHook
-                                      : HookInfo::ArgHook);
-    }
-    else if (Name == "append_cmd" || Name == "output_suffix") {
-      CheckNumberOfArguments(Dag, 1);
-      this->onCmdLine(InitPtrToString(Dag.getArg(0)));
-    }
-  }
-
-  void onCmdLine(const std::string& Cmd) {
-    StrVector cmds;
-    TokenizeCmdLine(Cmd, cmds);
-
-    for (StrVector::const_iterator B = cmds.begin(), E = cmds.end();
-         B != E; ++B) {
-      const std::string& cmd = *B;
-
-      if (cmd == "$CALL") {
-        unsigned NumArgs = 0;
-        CheckedIncrement(B, E, "Syntax error in $CALL invocation!");
-        const std::string& HookName = *B;
-
-        if (HookName.at(0) == ')')
-          throw "$CALL invoked with no arguments!";
-
-        while (++B != E && B->at(0) != ')') {
-          ++NumArgs;
-        }
-
-        HookInfoMap::const_iterator H = HookNames_.find(HookName);
-
-        if (H != HookNames_.end() && H->second.NumArgs != NumArgs &&
-            H->second.Type != HookInfo::ArgHook)
-          throw "Overloading of hooks is not allowed. Overloaded hook: "
-            + HookName;
-        else
-          HookNames_[HookName] = HookInfo(NumArgs);
-      }
-    }
-  }
-
-  void operator()(const Init* Arg) {
-
-    // We're invoked on an action (either a dag or a dag list).
-    if (typeid(*Arg) == typeid(DagInit)) {
-      const DagInit& Dag = InitPtrToDag(Arg);
-      this->onAction(Dag);
-      return;
-    }
-    else if (typeid(*Arg) == typeid(ListInit)) {
-      const ListInit& List = InitPtrToList(Arg);
-      for (ListInit::const_iterator B = List.begin(), E = List.end(); B != E;
-           ++B) {
-        const DagInit& Dag = InitPtrToDag(*B);
-        this->onAction(Dag);
-      }
-      return;
-    }
-
-    // We're invoked on a command line string.
-    this->onCmdLine(InitPtrToString(Arg));
-  }
-
-  void operator()(const Init* Statement, unsigned) {
-    this->operator()(Statement);
-  }
-};
-
-/// FillInHookNames - Actually extract the hook names from all command
-/// line strings. Helper function used by EmitHookDeclarations().
-void FillInHookNames(const ToolDescriptions& ToolDescs,
-                     const OptionDescriptions& OptDescs,
-                     HookInfoMap& HookNames)
-{
-  // For all tool descriptions:
-  for (ToolDescriptions::const_iterator B = ToolDescs.begin(),
-         E = ToolDescs.end(); B != E; ++B) {
-    const ToolDescription& D = *(*B);
-
-    // Look for 'forward_transformed_value' in 'actions'.
-    if (D.Actions)
-      WalkCase(D.Actions, Id(), ExtractHookNames(HookNames, OptDescs));
-
-    // Look for hook invocations in 'cmd_line'.
-    if (!D.CmdLine)
-      continue;
-    if (dynamic_cast<StringInit*>(D.CmdLine))
-      // This is a string.
-      ExtractHookNames(HookNames, OptDescs).operator()(D.CmdLine);
-    else
-      // This is a 'case' construct.
-      WalkCase(D.CmdLine, Id(), ExtractHookNames(HookNames, OptDescs));
-  }
-}
-
-/// EmitHookDeclarations - Parse CmdLine fields of all the tool
-/// property records and emit hook function declaration for each
-/// instance of $CALL(HookName).
-void EmitHookDeclarations(const ToolDescriptions& ToolDescs,
-                          const OptionDescriptions& OptDescs, raw_ostream& O) {
-  HookInfoMap HookNames;
-
-  FillInHookNames(ToolDescs, OptDescs, HookNames);
-  if (HookNames.empty())
-    return;
-
-  for (HookInfoMap::const_iterator B = HookNames.begin(),
-         E = HookNames.end(); B != E; ++B) {
-    StringRef HookName = B->first();
-    const HookInfo &Info = B->second;
-
-    O.indent(Indent1) << "std::string " << HookName << "(";
-
-    if (Info.Type == HookInfo::ArgHook) {
-      for (unsigned i = 0, j = Info.NumArgs; i < j; ++i) {
-        O << "const char* Arg" << i << (i+1 == j ? "" : ", ");
-      }
-    }
-    else {
-      O << "const std::vector<std::string>& Arg";
-    }
-
-    O <<");\n";
-  }
-}
-
-/// EmitIncludes - Emit necessary #include directives and some
-/// additional declarations.
-void EmitIncludes(raw_ostream& O) {
-  O << "#include \"llvm/CompilerDriver/BuiltinOptions.h\"\n"
-    << "#include \"llvm/CompilerDriver/CompilationGraph.h\"\n"
-    << "#include \"llvm/CompilerDriver/Error.h\"\n"
-    << "#include \"llvm/CompilerDriver/Tool.h\"\n\n"
-
-    << "#include \"llvm/Support/CommandLine.h\"\n"
-    << "#include \"llvm/Support/raw_ostream.h\"\n\n"
-
-    << "#include <algorithm>\n"
-    << "#include <cstdlib>\n"
-    << "#include <iterator>\n"
-    << "#include <stdexcept>\n\n"
-
-    << "using namespace llvm;\n"
-    << "using namespace llvmc;\n\n"
-
-    << "inline const char* checkCString(const char* s)\n"
-    << "{ return s == NULL ? \"\" : s; }\n\n";
-}
-
-
-/// DriverData - Holds all information about the driver.
-struct DriverData {
-  OptionDescriptions OptDescs;
-  ToolDescriptions ToolDescs;
-  DagVector Edges;
-  bool HasSink;
-};
-
-/// HasSink - Go through the list of tool descriptions and check if
-/// there are any with the 'sink' property set.
-bool HasSink(const ToolDescriptions& ToolDescs) {
-  for (ToolDescriptions::const_iterator B = ToolDescs.begin(),
-         E = ToolDescs.end(); B != E; ++B)
-    if ((*B)->isSink())
-      return true;
-
-  return false;
-}
-
-/// CollectDriverData - Collect compilation graph edges, tool properties and
-/// option properties from the parse tree.
-void CollectDriverData (const RecordKeeper& Records, DriverData& Data) {
-  // Collect option properties.
-  const RecordVector& OptionLists =
-    Records.getAllDerivedDefinitions("OptionList");
-  CollectOptionDescriptions(OptionLists, Data.OptDescs);
-
-  // Collect tool properties.
-  const RecordVector& Tools = Records.getAllDerivedDefinitions("Tool");
-  CollectToolDescriptions(Tools, Data.ToolDescs);
-  Data.HasSink = HasSink(Data.ToolDescs);
-
-  // Collect compilation graph edges.
-  const RecordVector& CompilationGraphs =
-    Records.getAllDerivedDefinitions("CompilationGraph");
-  FillInEdgeVector(CompilationGraphs, Data.Edges);
-}
-
-/// CheckDriverData - Perform some sanity checks on the collected data.
-void CheckDriverData(DriverData& Data) {
-  // Filter out all tools not mentioned in the compilation graph.
-  FilterNotInGraph(Data.Edges, Data.ToolDescs);
-
-  // Typecheck the compilation graph.
-  // TODO: use a genuine graph representation instead of a vector and check for
-  // multiple edges.
-  TypecheckGraph(Data.Edges, Data.ToolDescs);
-
-  // Check that there are no options without side effects (specified
-  // only in the OptionList).
-  CheckForSuperfluousOptions(Data.Edges, Data.ToolDescs, Data.OptDescs);
-}
-
-void EmitDriverCode(const DriverData& Data, 
-                    raw_ostream& O, RecordKeeper &Records) {
-  // Emit file header.
-  EmitIncludes(O);
-
-  // Emit global option registration code.
-  O << "namespace llvmc {\n"
-    << "namespace autogenerated {\n\n";
-  EmitOptionDefinitions(Data.OptDescs, Data.HasSink, O);
-  O << "} // End namespace autogenerated.\n"
-    << "} // End namespace llvmc.\n\n";
-
-  // Emit hook declarations.
-  O << "namespace hooks {\n";
-  EmitHookDeclarations(Data.ToolDescs, Data.OptDescs, O);
-  O << "} // End namespace hooks.\n\n";
-
-  O << "namespace {\n\n";
-  O << "using namespace llvmc::autogenerated;\n\n";
-
-  // Emit Tool classes.
-  for (ToolDescriptions::const_iterator B = Data.ToolDescs.begin(),
-         E = Data.ToolDescs.end(); B!=E; ++B)
-    EmitToolClassDefinition(*(*B), Data.OptDescs, O);
-
-  // Emit Edge# classes.
-  EmitEdgeClasses(Data.Edges, Data.OptDescs, O);
-
-  O << "} // End anonymous namespace.\n\n";
-
-  O << "namespace llvmc {\n";
-  O << "namespace autogenerated {\n\n";
-
-  // Emit PreprocessOptions() function.
-  EmitPreprocessOptions(Records, Data.OptDescs, O);
-
-  // Emit PopulateLanguageMap() function
-  // (language map maps from file extensions to language names).
-  EmitPopulateLanguageMap(Records, O);
-
-  // Emit PopulateCompilationGraph() function.
-  EmitPopulateCompilationGraph(Data.Edges, Data.ToolDescs, O);
-
-  O << "} // End namespace autogenerated.\n";
-  O << "} // End namespace llvmc.\n\n";
-
-  // EOF
-}
-
-
-// End of anonymous namespace
-}
-
-/// run - The back-end entry point.
-void LLVMCConfigurationEmitter::run (raw_ostream &O) {
-  try {
-    DriverData Data;
-
-    CollectDriverData(Records, Data);
-    CheckDriverData(Data);
-
-    this->EmitSourceFileHeader("llvmc-based driver: auto-generated code", O);
-    EmitDriverCode(Data, O, Records);
-
-  } catch (std::exception& Error) {
-    throw Error.what() + std::string(" - usually this means a syntax error.");
-  }
-}
diff --git a/contrib/llvm/utils/TableGen/LLVMCConfigurationEmitter.h b/contrib/llvm/utils/TableGen/LLVMCConfigurationEmitter.h
deleted file mode 100644
index 0f2ff3719678..000000000000
--- a/contrib/llvm/utils/TableGen/LLVMCConfigurationEmitter.h
+++ /dev/null
@@ -1,34 +0,0 @@
-//===- LLVMCConfigurationEmitter.cpp - Generate LLVMCC config ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This tablegen backend is responsible for emitting LLVMCC configuration code.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_UTILS_TABLEGEN_LLVMCCONF_EMITTER_H
-#define LLVM_UTILS_TABLEGEN_LLVMCCONF_EMITTER_H
-
-#include "TableGenBackend.h"
-
-namespace llvm {
-
-  /// LLVMCConfigurationEmitter - TableGen backend that generates
-  /// configuration code for LLVMC.
-  class LLVMCConfigurationEmitter : public TableGenBackend {
-    RecordKeeper &Records;
-  public:
-    explicit LLVMCConfigurationEmitter(RecordKeeper &records) : 
-      Records(records) {}
-
-    // run - Output the asmwriter, returning true on failure.
-    void run(raw_ostream &o);
-  };
-}
-
-#endif //LLVM_UTILS_TABLEGEN_LLVMCCONF_EMITTER_H
diff --git a/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index db33c1f7f686..c685527a140c 100644
--- a/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -8,10 +8,10 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "pseudo-lowering"
-#include "Error.h"
 #include "CodeGenInstruction.h"
 #include "PseudoLoweringEmitter.h"
-#include "Record.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.h b/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.h
index 2749280e6a9d..325bc8be146f 100644
--- a/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.h
+++ b/contrib/llvm/utils/TableGen/PseudoLoweringEmitter.h
@@ -12,7 +12,7 @@
 
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallVector.h"
 
diff --git a/contrib/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/contrib/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 65d4a9b02dfd..b0f4ffc84e08 100644
--- a/contrib/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -16,7 +16,8 @@
 #include "RegisterInfoEmitter.h"
 #include "CodeGenTarget.h"
 #include "CodeGenRegisters.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Format.h"
@@ -39,6 +40,9 @@ RegisterInfoEmitter::runEnums(raw_ostream &OS,
 
   OS << "namespace llvm {\n\n";
 
+  OS << "class MCRegisterClass;\n"
+     << "extern MCRegisterClass " << Namespace << "MCRegisterClasses[];\n\n";
+
   if (!Namespace.empty())
     OS << "namespace " << Namespace << " {\n";
   OS << "enum {\n  NoRegister,\n";
@@ -53,8 +57,7 @@ RegisterInfoEmitter::runEnums(raw_ostream &OS,
   if (!Namespace.empty())
     OS << "}\n";
 
-  const std::vector<CodeGenRegisterClass> &RegisterClasses =
-    Target.getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RegisterClasses = Bank.getRegClasses();
   if (!RegisterClasses.empty()) {
     OS << "\n// Register classes\n";
     if (!Namespace.empty())
@@ -62,7 +65,7 @@ RegisterInfoEmitter::runEnums(raw_ostream &OS,
     OS << "enum {\n";
     for (unsigned i = 0, e = RegisterClasses.size(); i != e; ++i) {
       if (i) OS << ",\n";
-      OS << "  " << RegisterClasses[i].getName() << "RegClassID";
+      OS << "  " << RegisterClasses[i]->getName() << "RegClassID";
       OS << " = " << i;
     }
     OS << "\n  };\n";
@@ -91,6 +94,147 @@ RegisterInfoEmitter::runEnums(raw_ostream &OS,
   OS << "#endif // GET_REGINFO_ENUM\n\n";
 }
 
+void
+RegisterInfoEmitter::EmitRegMapping(raw_ostream &OS,
+                                    const std::vector<CodeGenRegister*> &Regs,
+                                    bool isCtor) {
+
+  // Collect all information about dwarf register numbers
+  typedef std::map<Record*, std::vector<int64_t>, LessRecord> DwarfRegNumsMapTy;
+  DwarfRegNumsMapTy DwarfRegNums;
+
+  // First, just pull all provided information to the map
+  unsigned maxLength = 0;
+  for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
+    Record *Reg = Regs[i]->TheDef;
+    std::vector<int64_t> RegNums = Reg->getValueAsListOfInts("DwarfNumbers");
+    maxLength = std::max((size_t)maxLength, RegNums.size());
+    if (DwarfRegNums.count(Reg))
+      errs() << "Warning: DWARF numbers for register " << getQualifiedName(Reg)
+             << "specified multiple times\n";
+    DwarfRegNums[Reg] = RegNums;
+  }
+
+  if (!maxLength)
+    return;
+
+  // Now we know maximal length of number list. Append -1's, where needed
+  for (DwarfRegNumsMapTy::iterator
+       I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I)
+    for (unsigned i = I->second.size(), e = maxLength; i != e; ++i)
+      I->second.push_back(-1);
+
+  // Emit reverse information about the dwarf register numbers.
+  for (unsigned j = 0; j < 2; ++j) {
+    OS << "  switch (";
+    if (j == 0)
+      OS << "DwarfFlavour";
+    else
+      OS << "EHFlavour";
+    OS << ") {\n"
+     << "  default:\n"
+     << "    assert(0 && \"Unknown DWARF flavour\");\n"
+     << "    break;\n";
+
+    for (unsigned i = 0, e = maxLength; i != e; ++i) {
+      OS << "  case " << i << ":\n";
+      for (DwarfRegNumsMapTy::iterator
+             I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) {
+        int DwarfRegNo = I->second[i];
+        if (DwarfRegNo < 0)
+          continue;
+        OS << "    ";
+        if (!isCtor)
+          OS << "RI->";
+        OS << "mapDwarfRegToLLVMReg(" << DwarfRegNo << ", "
+           << getQualifiedName(I->first) << ", ";
+        if (j == 0)
+          OS << "false";
+        else
+          OS << "true";
+        OS << " );\n";
+      }
+      OS << "    break;\n";
+    }
+    OS << "  }\n";
+  }
+
+  for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
+    Record *Reg = Regs[i]->TheDef;
+    const RecordVal *V = Reg->getValue("DwarfAlias");
+    if (!V || !V->getValue())
+      continue;
+
+    DefInit *DI = dynamic_cast<DefInit*>(V->getValue());
+    Record *Alias = DI->getDef();
+    DwarfRegNums[Reg] = DwarfRegNums[Alias];
+  }
+
+  // Emit information about the dwarf register numbers.
+  for (unsigned j = 0; j < 2; ++j) {
+    OS << "  switch (";
+    if (j == 0)
+      OS << "DwarfFlavour";
+    else
+      OS << "EHFlavour";
+    OS << ") {\n"
+       << "  default:\n"
+       << "    assert(0 && \"Unknown DWARF flavour\");\n"
+       << "    break;\n";
+
+    for (unsigned i = 0, e = maxLength; i != e; ++i) {
+      OS << "  case " << i << ":\n";
+      // Sort by name to get a stable order.
+      for (DwarfRegNumsMapTy::iterator
+             I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) {
+        int RegNo = I->second[i];
+        OS << "    ";
+        if (!isCtor)
+          OS << "RI->";
+        OS << "mapLLVMRegToDwarfReg(" << getQualifiedName(I->first) << ", "
+           <<  RegNo << ", ";
+        if (j == 0)
+          OS << "false";
+        else
+          OS << "true";
+        OS << " );\n";
+      }
+      OS << "    break;\n";
+    }
+    OS << "  }\n";
+  }
+}
+
+// Print a BitVector as a sequence of hex numbers using a little-endian mapping.
+// Width is the number of bits per hex number.
+static void printBitVectorAsHex(raw_ostream &OS,
+                                const BitVector &Bits,
+                                unsigned Width) {
+  assert(Width <= 32 && "Width too large");
+  unsigned Digits = (Width + 3) / 4;
+  for (unsigned i = 0, e = Bits.size(); i < e; i += Width) {
+    unsigned Value = 0;
+    for (unsigned j = 0; j != Width && i + j != e; ++j)
+      Value |= Bits.test(i + j) << j;
+    OS << format("0x%0*x, ", Digits, Value);
+  }
+}
+
+// Helper to emit a set of bits into a constant byte array.
+class BitVectorEmitter {
+  BitVector Values;
+public:
+  void add(unsigned v) {
+    if (v >= Values.size())
+      Values.resize(((v/8)+1)*8); // Round up to the next byte.
+    Values[v] = true;
+  }
+
+  void print(raw_ostream &OS) {
+    printBitVectorAsHex(OS, Values, 8);
+  }
+};
+
 //
 // runMCDesc - Print out MC register descriptions.
 //
@@ -186,11 +330,73 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
   }
   OS << "};\n\n";      // End of register descriptors...
 
+  ArrayRef<CodeGenRegisterClass*> RegisterClasses = RegBank.getRegClasses();
+
+  // Loop over all of the register classes... emitting each one.
+  OS << "namespace {     // Register classes...\n";
+
+  // Emit the register enum value arrays for each RegisterClass
+  for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
+    const CodeGenRegisterClass &RC = *RegisterClasses[rc];
+    ArrayRef<Record*> Order = RC.getOrder();
+
+    // Give the register class a legal C name if it's anonymous.
+    std::string Name = RC.getName();
+
+    // Emit the register list now.
+    OS << "  // " << Name << " Register Class...\n"
+       << "  static const unsigned " << Name
+       << "[] = {\n    ";
+    for (unsigned i = 0, e = Order.size(); i != e; ++i) {
+      Record *Reg = Order[i];
+      OS << getQualifiedName(Reg) << ", ";
+    }
+    OS << "\n  };\n\n";
+
+    OS << "  // " << Name << " Bit set.\n"
+       << "  static const unsigned char " << Name
+       << "Bits[] = {\n    ";
+    BitVectorEmitter BVE;
+    for (unsigned i = 0, e = Order.size(); i != e; ++i) {
+      Record *Reg = Order[i];
+      BVE.add(Target.getRegBank().getReg(Reg)->EnumValue);
+    }
+    BVE.print(OS);
+    OS << "\n  };\n\n";
+
+  }
+  OS << "}\n\n";
+
+  OS << "MCRegisterClass " << TargetName << "MCRegisterClasses[] = {\n";
+
+  for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
+    const CodeGenRegisterClass &RC = *RegisterClasses[rc];
+    OS << "  MCRegisterClass(" << RC.getQualifiedName() + "RegClassID" << ", "
+       << '\"' << RC.getName() << "\", "
+       << RC.SpillSize/8 << ", "
+       << RC.SpillAlignment/8 << ", "
+       << RC.CopyCost << ", "
+       << RC.Allocatable << ", "
+       << RC.getName() << ", " << RC.getName() << " + "
+       << RC.getOrder().size() << ", "
+       << RC.getName() << "Bits, sizeof(" << RC.getName() << "Bits)"
+       << "),\n";
+  }
+
+  OS << "};\n\n";
+
   // MCRegisterInfo initialization routine.
   OS << "static inline void Init" << TargetName
-     << "MCRegisterInfo(MCRegisterInfo *RI) {\n";
+     << "MCRegisterInfo(MCRegisterInfo *RI, unsigned RA, "
+     << "unsigned DwarfFlavour = 0, unsigned EHFlavour = 0) {\n";
   OS << "  RI->InitMCRegisterInfo(" << TargetName << "RegDesc, "
-     << Regs.size()+1 << ");\n}\n\n";
+     << Regs.size()+1 << ", RA, " << TargetName << "MCRegisterClasses, "
+     << RegisterClasses.size() << ");\n\n";
+
+  EmitRegMapping(OS, Regs, false);
+
+  OS << "}\n\n";
+
 
   OS << "} // End llvm namespace \n";
   OS << "#endif // GET_REGINFO_MC_DESC\n\n";
@@ -213,17 +419,15 @@ RegisterInfoEmitter::runTargetHeader(raw_ostream &OS, CodeGenTarget &Target,
   OS << "namespace llvm {\n\n";
 
   OS << "struct " << ClassName << " : public TargetRegisterInfo {\n"
-     << "  explicit " << ClassName << "();\n"
-     << "  virtual int getDwarfRegNumFull(unsigned RegNum, "
-     << "unsigned Flavour) const;\n"
-     << "  virtual int getLLVMRegNumFull(unsigned DwarfRegNum, "
-     << "unsigned Flavour) const;\n"
-     << "  virtual int getDwarfRegNum(unsigned RegNum, bool isEH) const = 0;\n"
+     << "  explicit " << ClassName
+     << "(unsigned RA, unsigned D = 0, unsigned E = 0);\n"
      << "  virtual bool needsStackRealignment(const MachineFunction &) const\n"
      << "     { return false; }\n"
      << "  unsigned getSubReg(unsigned RegNo, unsigned Index) const;\n"
      << "  unsigned getSubRegIndex(unsigned RegNo, unsigned SubRegNo) const;\n"
      << "  unsigned composeSubRegIndices(unsigned, unsigned) const;\n"
+     << "  const TargetRegisterClass *"
+        "getSubClassWithSubReg(const TargetRegisterClass*, unsigned) const;\n"
      << "};\n\n";
 
   const std::vector<Record*> &SubRegIndices = RegBank.getSubRegIndices();
@@ -241,15 +445,14 @@ RegisterInfoEmitter::runTargetHeader(raw_ostream &OS, CodeGenTarget &Target,
       OS << "}\n";
   }
 
-  const std::vector<CodeGenRegisterClass> &RegisterClasses =
-    Target.getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RegisterClasses = RegBank.getRegClasses();
 
   if (!RegisterClasses.empty()) {
-    OS << "namespace " << RegisterClasses[0].Namespace
+    OS << "namespace " << RegisterClasses[0]->Namespace
        << " { // Register classes\n";
 
     for (unsigned i = 0, e = RegisterClasses.size(); i != e; ++i) {
-      const CodeGenRegisterClass &RC = RegisterClasses[i];
+      const CodeGenRegisterClass &RC = *RegisterClasses[i];
       const std::string &Name = RC.getName();
 
       // Output the register class definition.
@@ -285,42 +488,30 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   OS << "namespace llvm {\n\n";
 
+  // Get access to MCRegisterClass data.
+  OS << "extern MCRegisterClass " << Target.getName()
+    << "MCRegisterClasses[];\n";
+
   // Start out by emitting each of the register classes.
-  const std::vector<CodeGenRegisterClass> &RegisterClasses =
-    Target.getRegisterClasses();
+  ArrayRef<CodeGenRegisterClass*> RegisterClasses = RegBank.getRegClasses();
 
   // Collect all registers belonging to any allocatable class.
   std::set<Record*> AllocatableRegs;
 
-  // Loop over all of the register classes... emitting each one.
-  OS << "namespace {     // Register classes...\n";
-
-  // Emit the register enum value arrays for each RegisterClass
+  // Collect allocatable registers.
   for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
-    const CodeGenRegisterClass &RC = RegisterClasses[rc];
+    const CodeGenRegisterClass &RC = *RegisterClasses[rc];
     ArrayRef<Record*> Order = RC.getOrder();
 
-    // Collect allocatable registers.
     if (RC.Allocatable)
       AllocatableRegs.insert(Order.begin(), Order.end());
-
-    // Give the register class a legal C name if it's anonymous.
-    std::string Name = RC.getName();
-
-    // Emit the register list now.
-    OS << "  // " << Name << " Register Class...\n"
-       << "  static const unsigned " << Name
-       << "[] = {\n    ";
-    for (unsigned i = 0, e = Order.size(); i != e; ++i) {
-      Record *Reg = Order[i];
-      OS << getQualifiedName(Reg) << ", ";
-    }
-    OS << "\n  };\n\n";
   }
 
+  OS << "namespace {     // Register classes...\n";
+
   // Emit the ValueType arrays for each RegisterClass
   for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
-    const CodeGenRegisterClass &RC = RegisterClasses[rc];
+    const CodeGenRegisterClass &RC = *RegisterClasses[rc];
 
     // Give the register class a legal C name if it's anonymous.
     std::string Name = RC.getName() + "VTs";
@@ -338,65 +529,39 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   // Now that all of the structs have been emitted, emit the instances.
   if (!RegisterClasses.empty()) {
-    OS << "namespace " << RegisterClasses[0].Namespace
+    OS << "namespace " << RegisterClasses[0]->Namespace
        << " {   // Register class instances\n";
     for (unsigned i = 0, e = RegisterClasses.size(); i != e; ++i)
-      OS << "  " << RegisterClasses[i].getName()  << "Class\t"
-         << RegisterClasses[i].getName() << "RegClass;\n";
+      OS << "  " << RegisterClasses[i]->getName()  << "Class\t"
+         << RegisterClasses[i]->getName() << "RegClass;\n";
 
-    std::map<unsigned, std::set<unsigned> > SuperClassMap;
     std::map<unsigned, std::set<unsigned> > SuperRegClassMap;
-    OS << "\n";
+
+    OS << "\n  static const TargetRegisterClass* const "
+      << "NullRegClasses[] = { NULL };\n\n";
 
     unsigned NumSubRegIndices = RegBank.getSubRegIndices().size();
 
     if (NumSubRegIndices) {
-      // Emit the sub-register classes for each RegisterClass
+      // Compute the super-register classes for each RegisterClass
       for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
-        const CodeGenRegisterClass &RC = RegisterClasses[rc];
-        std::vector<Record*> SRC(NumSubRegIndices);
+        const CodeGenRegisterClass &RC = *RegisterClasses[rc];
         for (DenseMap<Record*,Record*>::const_iterator
              i = RC.SubRegClasses.begin(),
              e = RC.SubRegClasses.end(); i != e; ++i) {
-          // Build SRC array.
-          unsigned idx = RegBank.getSubRegIndexNo(i->first);
-          SRC.at(idx-1) = i->second;
-
           // Find the register class number of i->second for SuperRegClassMap.
-          for (unsigned rc2 = 0, e2 = RegisterClasses.size(); rc2 != e2; ++rc2) {
-            const CodeGenRegisterClass &RC2 =  RegisterClasses[rc2];
-            if (RC2.TheDef == i->second) {
-              SuperRegClassMap[rc2].insert(rc);
-              break;
-            }
-          }
+          const CodeGenRegisterClass *RC2 = RegBank.getRegClass(i->second);
+          assert(RC2 && "Invalid register class in SubRegClasses");
+          SuperRegClassMap[RC2->EnumValue].insert(rc);
         }
-
-        // Give the register class a legal C name if it's anonymous.
-        std::string Name = RC.TheDef->getName();
-
-        OS << "  // " << Name
-           << " Sub-register Classes...\n"
-           << "  static const TargetRegisterClass* const "
-           << Name << "SubRegClasses[] = {\n    ";
-
-        for (unsigned idx = 0; idx != NumSubRegIndices; ++idx) {
-          if (idx)
-            OS << ", ";
-          if (SRC[idx])
-            OS << "&" << getQualifiedName(SRC[idx]) << "RegClass";
-          else
-            OS << "0";
-        }
-        OS << "\n  };\n\n";
       }
 
       // Emit the super-register classes for each RegisterClass
       for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
-        const CodeGenRegisterClass &RC = RegisterClasses[rc];
+        const CodeGenRegisterClass &RC = *RegisterClasses[rc];
 
         // Give the register class a legal C name if it's anonymous.
-        std::string Name = RC.TheDef->getName();
+        std::string Name = RC.getName();
 
         OS << "  // " << Name
            << " Super-register Classes...\n"
@@ -409,10 +574,10 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
         if (I != SuperRegClassMap.end()) {
           for (std::set<unsigned>::iterator II = I->second.begin(),
                  EE = I->second.end(); II != EE; ++II) {
-            const CodeGenRegisterClass &RC2 = RegisterClasses[*II];
+            const CodeGenRegisterClass &RC2 = *RegisterClasses[*II];
             if (!Empty)
               OS << ", ";
-            OS << "&" << getQualifiedName(RC2.TheDef) << "RegClass";
+            OS << "&" << RC2.getQualifiedName() << "RegClass";
             Empty = false;
           }
         }
@@ -420,97 +585,51 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
         OS << (!Empty ? ", " : "") << "NULL";
         OS << "\n  };\n\n";
       }
-    } else {
-      // No subregindices in this target
-      OS << "  static const TargetRegisterClass* const "
-         << "NullRegClasses[] = { NULL };\n\n";
     }
 
     // Emit the sub-classes array for each RegisterClass
     for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
-      const CodeGenRegisterClass &RC = RegisterClasses[rc];
+      const CodeGenRegisterClass &RC = *RegisterClasses[rc];
 
       // Give the register class a legal C name if it's anonymous.
-      std::string Name = RC.TheDef->getName();
+      std::string Name = RC.getName();
 
-      OS << "  // " << Name
-         << " Register Class sub-classes...\n"
-         << "  static const TargetRegisterClass* const "
-         << Name << "Subclasses[] = {\n    ";
-
-      bool Empty = true;
-      for (unsigned rc2 = 0, e2 = RegisterClasses.size(); rc2 != e2; ++rc2) {
-        const CodeGenRegisterClass &RC2 = RegisterClasses[rc2];
-
-        // Sub-classes are used to determine if a virtual register can be used
-        // as an instruction operand, or if it must be copied first.
-        if (rc == rc2 || !RC.hasSubClass(&RC2)) continue;
-
-        if (!Empty) OS << ", ";
-        OS << "&" << getQualifiedName(RC2.TheDef) << "RegClass";
-        Empty = false;
-
-        std::map<unsigned, std::set<unsigned> >::iterator SCMI =
-          SuperClassMap.find(rc2);
-        if (SCMI == SuperClassMap.end()) {
-          SuperClassMap.insert(std::make_pair(rc2, std::set<unsigned>()));
-          SCMI = SuperClassMap.find(rc2);
-        }
-        SCMI->second.insert(rc);
-      }
-
-      OS << (!Empty ? ", " : "") << "NULL";
-      OS << "\n  };\n\n";
+      OS << "  static const unsigned " << Name << "SubclassMask[] = { ";
+      printBitVectorAsHex(OS, RC.getSubClasses(), 32);
+      OS << "};\n\n";
     }
 
+    // Emit NULL terminated super-class lists.
     for (unsigned rc = 0, e = RegisterClasses.size(); rc != e; ++rc) {
-      const CodeGenRegisterClass &RC = RegisterClasses[rc];
+      const CodeGenRegisterClass &RC = *RegisterClasses[rc];
+      ArrayRef<CodeGenRegisterClass*> Supers = RC.getSuperClasses();
 
-      // Give the register class a legal C name if it's anonymous.
-      std::string Name = RC.TheDef->getName();
+      // Skip classes without supers.  We can reuse NullRegClasses.
+      if (Supers.empty())
+        continue;
 
-      OS << "  // " << Name
-         << " Register Class super-classes...\n"
-         << "  static const TargetRegisterClass* const "
-         << Name << "Superclasses[] = {\n    ";
-
-      bool Empty = true;
-      std::map<unsigned, std::set<unsigned> >::iterator I =
-        SuperClassMap.find(rc);
-      if (I != SuperClassMap.end()) {
-        for (std::set<unsigned>::iterator II = I->second.begin(),
-               EE = I->second.end(); II != EE; ++II) {
-          const CodeGenRegisterClass &RC2 = RegisterClasses[*II];
-          if (!Empty) OS << ", ";
-          OS << "&" << getQualifiedName(RC2.TheDef) << "RegClass";
-          Empty = false;
-        }
-      }
-
-      OS << (!Empty ? ", " : "") << "NULL";
-      OS << "\n  };\n\n";
+      OS << "  static const TargetRegisterClass* const "
+         << RC.getName() << "Superclasses[] = {\n";
+      for (unsigned i = 0; i != Supers.size(); ++i)
+        OS << "    &" << Supers[i]->getQualifiedName() << "RegClass,\n";
+      OS << "    NULL\n  };\n\n";
     }
 
     // Emit methods.
     for (unsigned i = 0, e = RegisterClasses.size(); i != e; ++i) {
-      const CodeGenRegisterClass &RC = RegisterClasses[i];
+      const CodeGenRegisterClass &RC = *RegisterClasses[i];
       OS << RC.getName() << "Class::" << RC.getName()
-         << "Class()  : TargetRegisterClass("
-         << RC.getName() + "RegClassID" << ", "
-         << '\"' << RC.getName() << "\", "
+         << "Class()  : TargetRegisterClass(&"
+         << Target.getName() << "MCRegisterClasses["
+         << RC.getName() + "RegClassID" << "], "
          << RC.getName() + "VTs" << ", "
-         << RC.getName() + "Subclasses" << ", "
-         << RC.getName() + "Superclasses" << ", "
-         << (NumSubRegIndices ? RC.getName() + "Sub" : std::string("Null"))
-         << "RegClasses, "
-         << (NumSubRegIndices ? RC.getName() + "Super" : std::string("Null"))
-         << "RegClasses, "
-         << RC.SpillSize/8 << ", "
-         << RC.SpillAlignment/8 << ", "
-         << RC.CopyCost << ", "
-         << RC.Allocatable << ", "
-         << RC.getName() << ", " << RC.getName() << " + "
-         << RC.getOrder().size()
+         << RC.getName() + "SubclassMask" << ", ";
+      if (RC.getSuperClasses().empty())
+        OS << "NullRegClasses, ";
+      else
+        OS << RC.getName() + "Superclasses, ";
+      OS << (NumSubRegIndices ? RC.getName() + "Super" : std::string("Null"))
+         << "RegClasses"
          << ") {}\n";
       if (!RC.AltOrderSelect.empty()) {
         OS << "\nstatic inline unsigned " << RC.getName()
@@ -525,10 +644,12 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
             OS << (elem ? ", " : " ") << getQualifiedName(Elems[elem]);
           OS << " };\n";
         }
-        OS << "  static const ArrayRef<unsigned> Order[] = {\n"
-           << "    ArrayRef<unsigned>(" << RC.getName();
+        OS << "  const MCRegisterClass &MCR = " << Target.getName()
+           << "MCRegisterClasses[" << RC.getQualifiedName() + "RegClassID];"
+           << "  static const ArrayRef<unsigned> Order[] = {\n"
+           << "    makeArrayRef(MCR.begin(), MCR.getNumRegs()";
         for (unsigned oi = 1, oe = RC.getNumOrders(); oi != oe; ++oi)
-          OS << "),\n    ArrayRef<unsigned>(AltOrder" << oi;
+          OS << "),\n    makeArrayRef(AltOrder" << oi;
         OS << ")\n  };\n  const unsigned Select = " << RC.getName()
            << "AltOrderSelect(MF);\n  assert(Select < " << RC.getNumOrders()
            << ");\n  return Order[Select];\n}\n";
@@ -541,7 +662,7 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
   OS << "\nnamespace {\n";
   OS << "  const TargetRegisterClass* const RegisterClasses[] = {\n";
   for (unsigned i = 0, e = RegisterClasses.size(); i != e; ++i)
-    OS << "    &" << getQualifiedName(RegisterClasses[i].TheDef)
+    OS << "    &" << RegisterClasses[i]->getQualifiedName()
        << "RegClass,\n";
   OS << "  };\n";
   OS << "}\n";       // End of anonymous namespace...
@@ -658,106 +779,59 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
   }
   OS << "  }\n}\n\n";
 
+  // Emit getSubClassWithSubReg.
+  OS << "const TargetRegisterClass *" << ClassName
+     << "::getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx)"
+        " const {\n";
+  if (SubRegIndices.empty()) {
+    OS << "  assert(Idx == 0 && \"Target has no sub-registers\");\n"
+       << "  return RC;\n";
+  } else {
+    // Use the smallest type that can hold a regclass ID with room for a
+    // sentinel.
+    if (RegisterClasses.size() < UINT8_MAX)
+      OS << "  static const uint8_t Table[";
+    else if (RegisterClasses.size() < UINT16_MAX)
+      OS << "  static const uint16_t Table[";
+    else
+      throw "Too many register classes.";
+    OS << RegisterClasses.size() << "][" << SubRegIndices.size() << "] = {\n";
+    for (unsigned rci = 0, rce = RegisterClasses.size(); rci != rce; ++rci) {
+      const CodeGenRegisterClass &RC = *RegisterClasses[rci];
+      OS << "    {\t// " << RC.getName() << "\n";
+      for (unsigned sri = 0, sre = SubRegIndices.size(); sri != sre; ++sri) {
+        Record *Idx = SubRegIndices[sri];
+        if (CodeGenRegisterClass *SRC = RC.getSubClassWithSubReg(Idx))
+          OS << "      " << SRC->EnumValue + 1 << ",\t// " << Idx->getName()
+             << " -> " << SRC->getName() << "\n";
+        else
+          OS << "      0,\t// " << Idx->getName() << "\n";
+      }
+      OS << "    },\n";
+    }
+    OS << "  };\n  assert(RC && \"Missing regclass\");\n"
+       << "  if (!Idx) return RC;\n  --Idx;\n"
+       << "  assert(Idx < " << SubRegIndices.size() << " && \"Bad subreg\");\n"
+       << "  unsigned TV = Table[RC->getID()][Idx];\n"
+       << "  return TV ? getRegClass(TV - 1) : 0;\n";
+  }
+  OS << "}\n\n";
+
   // Emit the constructor of the class...
   OS << "extern MCRegisterDesc " << TargetName << "RegDesc[];\n";
 
   OS << ClassName << "::" << ClassName
-     << "()\n"
+     << "(unsigned RA, unsigned DwarfFlavour, unsigned EHFlavour)\n"
      << "  : TargetRegisterInfo(" << TargetName << "RegInfoDesc"
      << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() <<",\n"
      << "                 " << TargetName << "SubRegIndexTable) {\n"
      << "  InitMCRegisterInfo(" << TargetName << "RegDesc, "
-     << Regs.size()+1 << ");\n"
-     << "}\n\n";
+     << Regs.size()+1 << ", RA, " << TargetName << "MCRegisterClasses, "
+     << RegisterClasses.size() << ");\n\n";
 
-  // Collect all information about dwarf register numbers
-  typedef std::map<Record*, std::vector<int64_t>, LessRecord> DwarfRegNumsMapTy;
-  DwarfRegNumsMapTy DwarfRegNums;
+  EmitRegMapping(OS, Regs, true);
 
-  // First, just pull all provided information to the map
-  unsigned maxLength = 0;
-  for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
-    Record *Reg = Regs[i]->TheDef;
-    std::vector<int64_t> RegNums = Reg->getValueAsListOfInts("DwarfNumbers");
-    maxLength = std::max((size_t)maxLength, RegNums.size());
-    if (DwarfRegNums.count(Reg))
-      errs() << "Warning: DWARF numbers for register " << getQualifiedName(Reg)
-             << "specified multiple times\n";
-    DwarfRegNums[Reg] = RegNums;
-  }
-
-  // Now we know maximal length of number list. Append -1's, where needed
-  for (DwarfRegNumsMapTy::iterator
-       I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I)
-    for (unsigned i = I->second.size(), e = maxLength; i != e; ++i)
-      I->second.push_back(-1);
-
-  // Emit reverse information about the dwarf register numbers.
-  OS << "int " << ClassName << "::getLLVMRegNumFull(unsigned DwarfRegNum, "
-     << "unsigned Flavour) const {\n"
-     << "  switch (Flavour) {\n"
-     << "  default:\n"
-     << "    assert(0 && \"Unknown DWARF flavour\");\n"
-     << "    return -1;\n";
-
-  for (unsigned i = 0, e = maxLength; i != e; ++i) {
-    OS << "  case " << i << ":\n"
-       << "    switch (DwarfRegNum) {\n"
-       << "    default:\n"
-       << "      assert(0 && \"Invalid DwarfRegNum\");\n"
-       << "      return -1;\n";
-
-    for (DwarfRegNumsMapTy::iterator
-           I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) {
-      int DwarfRegNo = I->second[i];
-      if (DwarfRegNo >= 0)
-        OS << "    case " <<  DwarfRegNo << ":\n"
-           << "      return " << getQualifiedName(I->first) << ";\n";
-    }
-    OS << "    };\n";
-  }
-
-  OS << "  };\n}\n\n";
-
-  for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
-    Record *Reg = Regs[i]->TheDef;
-    const RecordVal *V = Reg->getValue("DwarfAlias");
-    if (!V || !V->getValue())
-      continue;
-
-    DefInit *DI = dynamic_cast<DefInit*>(V->getValue());
-    Record *Alias = DI->getDef();
-    DwarfRegNums[Reg] = DwarfRegNums[Alias];
-  }
-
-  // Emit information about the dwarf register numbers.
-  OS << "int " << ClassName << "::getDwarfRegNumFull(unsigned RegNum, "
-     << "unsigned Flavour) const {\n"
-     << "  switch (Flavour) {\n"
-     << "  default:\n"
-     << "    assert(0 && \"Unknown DWARF flavour\");\n"
-     << "    return -1;\n";
-
-  for (unsigned i = 0, e = maxLength; i != e; ++i) {
-    OS << "  case " << i << ":\n"
-       << "    switch (RegNum) {\n"
-       << "    default:\n"
-       << "      assert(0 && \"Invalid RegNum\");\n"
-       << "      return -1;\n";
-
-    // Sort by name to get a stable order.
-
-
-    for (DwarfRegNumsMapTy::iterator
-           I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) {
-      int RegNo = I->second[i];
-      OS << "    case " << getQualifiedName(I->first) << ":\n"
-         << "      return " << RegNo << ";\n";
-    }
-    OS << "    };\n";
-  }
-
-  OS << "  };\n}\n\n";
+  OS << "}\n\n";
 
   OS << "} // End llvm namespace \n";
   OS << "#endif // GET_REGINFO_TARGET_DESC\n\n";
diff --git a/contrib/llvm/utils/TableGen/RegisterInfoEmitter.h b/contrib/llvm/utils/TableGen/RegisterInfoEmitter.h
index 2c01b5c3bd06..0fd4d079ebc0 100644
--- a/contrib/llvm/utils/TableGen/RegisterInfoEmitter.h
+++ b/contrib/llvm/utils/TableGen/RegisterInfoEmitter.h
@@ -16,11 +16,13 @@
 #ifndef REGISTER_INFO_EMITTER_H
 #define REGISTER_INFO_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include <vector>
 
 namespace llvm {
 
 class CodeGenRegBank;
+struct CodeGenRegister;
 class CodeGenTarget;
 
 class RegisterInfoEmitter : public TableGenBackend {
@@ -44,6 +46,11 @@ class RegisterInfoEmitter : public TableGenBackend {
 
   // run - Output the register file description.
   void run(raw_ostream &o);
+
+private:
+  void EmitRegMapping(raw_ostream &o,
+                      const std::vector<CodeGenRegister*> &Regs, bool isCtor);
+  void EmitRegClasses(raw_ostream &OS, CodeGenTarget &Target);
 };
 
 } // End llvm namespace
diff --git a/contrib/llvm/utils/TableGen/SetTheory.cpp b/contrib/llvm/utils/TableGen/SetTheory.cpp
index 21ac09cb6625..bef73f33effe 100644
--- a/contrib/llvm/utils/TableGen/SetTheory.cpp
+++ b/contrib/llvm/utils/TableGen/SetTheory.cpp
@@ -13,8 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "SetTheory.h"
-#include "Error.h"
-#include "Record.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/Support/Format.h"
 
 using namespace llvm;
diff --git a/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp b/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp
index 978e91a1d6c0..103a4032b02a 100644
--- a/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -13,7 +13,7 @@
 
 #include "SubtargetEmitter.h"
 #include "CodeGenTarget.h"
-#include "Record.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Debug.h"
 #include <algorithm>
diff --git a/contrib/llvm/utils/TableGen/SubtargetEmitter.h b/contrib/llvm/utils/TableGen/SubtargetEmitter.h
index b239f3dda76d..ff01274bd1a7 100644
--- a/contrib/llvm/utils/TableGen/SubtargetEmitter.h
+++ b/contrib/llvm/utils/TableGen/SubtargetEmitter.h
@@ -14,7 +14,7 @@
 #ifndef SUBTARGET_EMITTER_H
 #define SUBTARGET_EMITTER_H
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include <vector>
 #include <map>
diff --git a/contrib/llvm/utils/TableGen/TableGen.cpp b/contrib/llvm/utils/TableGen/TableGen.cpp
index e8eacb841d03..eacfdf6fed39 100644
--- a/contrib/llvm/utils/TableGen/TableGen.cpp
+++ b/contrib/llvm/utils/TableGen/TableGen.cpp
@@ -1,4 +1,4 @@
-//===- TableGen.cpp - Top-Level TableGen implementation -------------------===//
+//===- TableGen.cpp - Top-Level TableGen implementation for LLVM ----------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,48 +7,34 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// TableGen is a tool which can be used to build up a description of something,
-// then invoke one or more "tablegen backends" to emit information about the
-// description in some predefined format.  In practice, this is used by the LLVM
-// code generators to automate generation of a code generator through a
-// high-level description of the target.
+// This file contains the main function for LLVM's TableGen.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AsmMatcherEmitter.h"
 #include "AsmWriterEmitter.h"
 #include "CallingConvEmitter.h"
-#include "ClangASTNodesEmitter.h"
-#include "ClangAttrEmitter.h"
-#include "ClangDiagnosticsEmitter.h"
-#include "ClangSACheckersEmitter.h"
 #include "CodeEmitterGen.h"
 #include "DAGISelEmitter.h"
 #include "DisassemblerEmitter.h"
 #include "EDEmitter.h"
-#include "Error.h"
 #include "FastISelEmitter.h"
 #include "InstrInfoEmitter.h"
 #include "IntrinsicEmitter.h"
-#include "LLVMCConfigurationEmitter.h"
-#include "NeonEmitter.h"
-#include "OptParserEmitter.h"
 #include "PseudoLoweringEmitter.h"
-#include "Record.h"
 #include "RegisterInfoEmitter.h"
 #include "ARMDecoderEmitter.h"
 #include "SubtargetEmitter.h"
 #include "SetTheory.h"
-#include "TGParser.h"
-#include "llvm/ADT/OwningPtr.h"
+
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/Signals.h"
-#include "llvm/Support/system_error.h"
-#include <algorithm>
-#include <cstdio>
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Main.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenAction.h"
+
 using namespace llvm;
 
 enum ActionType {
@@ -62,29 +48,12 @@ enum ActionType {
   GenDisassembler,
   GenPseudoLowering,
   GenCallingConv,
-  GenClangAttrClasses,
-  GenClangAttrImpl,
-  GenClangAttrList,
-  GenClangAttrPCHRead,
-  GenClangAttrPCHWrite,
-  GenClangAttrSpellingList,
-  GenClangDiagsDefs,
-  GenClangDiagGroups,
-  GenClangDiagsIndexName,
-  GenClangDeclNodes,
-  GenClangStmtNodes,
-  GenClangSACheckers,
   GenDAGISel,
   GenFastISel,
-  GenOptParserDefs, GenOptParserImpl,
   GenSubtarget,
   GenIntrinsic,
   GenTgtIntrinsic,
-  GenLLVMCConf,
   GenEDInfo,
-  GenArmNeon,
-  GenArmNeonSema,
-  GenArmNeonTest,
   PrintEnums,
   PrintSets
 };
@@ -116,52 +85,14 @@ namespace {
                                "Generate a DAG instruction selector"),
                     clEnumValN(GenFastISel, "gen-fast-isel",
                                "Generate a \"fast\" instruction selector"),
-                    clEnumValN(GenOptParserDefs, "gen-opt-parser-defs",
-                               "Generate option definitions"),
-                    clEnumValN(GenOptParserImpl, "gen-opt-parser-impl",
-                               "Generate option parser implementation"),
                     clEnumValN(GenSubtarget, "gen-subtarget",
                                "Generate subtarget enumerations"),
                     clEnumValN(GenIntrinsic, "gen-intrinsic",
                                "Generate intrinsic information"),
                     clEnumValN(GenTgtIntrinsic, "gen-tgt-intrinsic",
                                "Generate target intrinsic information"),
-                    clEnumValN(GenClangAttrClasses, "gen-clang-attr-classes",
-                               "Generate clang attribute clases"),
-                    clEnumValN(GenClangAttrImpl, "gen-clang-attr-impl",
-                               "Generate clang attribute implementations"),
-                    clEnumValN(GenClangAttrList, "gen-clang-attr-list",
-                               "Generate a clang attribute list"),
-                    clEnumValN(GenClangAttrPCHRead, "gen-clang-attr-pch-read",
-                               "Generate clang PCH attribute reader"),
-                    clEnumValN(GenClangAttrPCHWrite, "gen-clang-attr-pch-write",
-                               "Generate clang PCH attribute writer"),
-                    clEnumValN(GenClangAttrSpellingList,
-                               "gen-clang-attr-spelling-list",
-                               "Generate a clang attribute spelling list"),
-                    clEnumValN(GenClangDiagsDefs, "gen-clang-diags-defs",
-                               "Generate Clang diagnostics definitions"),
-                    clEnumValN(GenClangDiagGroups, "gen-clang-diag-groups",
-                               "Generate Clang diagnostic groups"),
-                    clEnumValN(GenClangDiagsIndexName,
-                               "gen-clang-diags-index-name",
-                               "Generate Clang diagnostic name index"),
-                    clEnumValN(GenClangDeclNodes, "gen-clang-decl-nodes",
-                               "Generate Clang AST declaration nodes"),
-                    clEnumValN(GenClangStmtNodes, "gen-clang-stmt-nodes",
-                               "Generate Clang AST statement nodes"),
-                    clEnumValN(GenClangSACheckers, "gen-clang-sa-checkers",
-                               "Generate Clang Static Analyzer checkers"),
-                    clEnumValN(GenLLVMCConf, "gen-llvmc",
-                               "Generate LLVMC configuration library"),
                     clEnumValN(GenEDInfo, "gen-enhanced-disassembly-info",
                                "Generate enhanced disassembly info"),
-                    clEnumValN(GenArmNeon, "gen-arm-neon",
-                               "Generate arm_neon.h for clang"),
-                    clEnumValN(GenArmNeonSema, "gen-arm-neon-sema",
-                               "Generate ARM NEON sema support for clang"),
-                    clEnumValN(GenArmNeonTest, "gen-arm-neon-test",
-                               "Generate ARM NEON tests for clang"),
                     clEnumValN(PrintEnums, "print-enums",
                                "Print enum values for a class"),
                     clEnumValN(PrintSets, "print-sets",
@@ -171,198 +102,66 @@ namespace {
   cl::opt<std::string>
   Class("class", cl::desc("Print Enum list for this class"),
         cl::value_desc("class name"));
-
-  cl::opt<std::string>
-  OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"),
-                 cl::init("-"));
-
-  cl::opt<std::string>
-  DependFilename("d", cl::desc("Dependency filename"), cl::value_desc("filename"),
-                 cl::init(""));
-
-  cl::opt<std::string>
-  InputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
-
-  cl::list<std::string>
-  IncludeDirs("I", cl::desc("Directory of include files"),
-              cl::value_desc("directory"), cl::Prefix);
-
-  cl::opt<std::string>
-  ClangComponent("clang-component",
-                 cl::desc("Only use warnings from specified component"),
-                 cl::value_desc("component"), cl::Hidden);
 }
 
-
-int main(int argc, char **argv) {
-  RecordKeeper Records;
-
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-  cl::ParseCommandLineOptions(argc, argv);
-
-
-  try {
-    // Parse the input file.
-    OwningPtr<MemoryBuffer> File;
-    if (error_code ec = MemoryBuffer::getFileOrSTDIN(InputFilename.c_str(), File)) {
-      errs() << "Could not open input file '" << InputFilename << "': "
-             << ec.message() <<"\n";
-      return 1;
-    }
-    MemoryBuffer *F = File.take();
-
-    // Tell SrcMgr about this buffer, which is what TGParser will pick up.
-    SrcMgr.AddNewSourceBuffer(F, SMLoc());
-
-    // Record the location of the include directory so that the lexer can find
-    // it later.
-    SrcMgr.setIncludeDirs(IncludeDirs);
-
-    TGParser Parser(SrcMgr, Records);
-
-    if (Parser.ParseFile())
-      return 1;
-
-    std::string Error;
-    tool_output_file Out(OutputFilename.c_str(), Error);
-    if (!Error.empty()) {
-      errs() << argv[0] << ": error opening " << OutputFilename
-        << ":" << Error << "\n";
-      return 1;
-    }
-    if (!DependFilename.empty()) {
-      if (OutputFilename == "-") {
-        errs() << argv[0] << ": the option -d must be used together with -o\n";
-        return 1;
-      }
-      tool_output_file DepOut(DependFilename.c_str(), Error);
-      if (!Error.empty()) {
-        errs() << argv[0] << ": error opening " << DependFilename
-          << ":" << Error << "\n";
-        return 1;
-      }
-      DepOut.os() << DependFilename << ":";
-      const std::vector<std::string> &Dependencies = Parser.getDependencies();
-      for (std::vector<std::string>::const_iterator I = Dependencies.begin(),
-                                                          E = Dependencies.end();
-           I != E; ++I) {
-        DepOut.os() << " " << (*I);
-      }
-      DepOut.os() << "\n";
-      DepOut.keep();
-    }
-
+class LLVMTableGenAction : public TableGenAction {
+public:
+  bool operator()(raw_ostream &OS, RecordKeeper &Records) {
     switch (Action) {
     case PrintRecords:
-      Out.os() << Records;           // No argument, dump all contents
+      OS << Records;           // No argument, dump all contents
       break;
     case GenEmitter:
-      CodeEmitterGen(Records).run(Out.os());
+      CodeEmitterGen(Records).run(OS);
       break;
     case GenRegisterInfo:
-      RegisterInfoEmitter(Records).run(Out.os());
+      RegisterInfoEmitter(Records).run(OS);
       break;
     case GenInstrInfo:
-      InstrInfoEmitter(Records).run(Out.os());
+      InstrInfoEmitter(Records).run(OS);
       break;
     case GenCallingConv:
-      CallingConvEmitter(Records).run(Out.os());
+      CallingConvEmitter(Records).run(OS);
       break;
     case GenAsmWriter:
-      AsmWriterEmitter(Records).run(Out.os());
+      AsmWriterEmitter(Records).run(OS);
       break;
     case GenARMDecoder:
-      ARMDecoderEmitter(Records).run(Out.os());
+      ARMDecoderEmitter(Records).run(OS);
       break;
     case GenAsmMatcher:
-      AsmMatcherEmitter(Records).run(Out.os());
-      break;
-    case GenClangAttrClasses:
-      ClangAttrClassEmitter(Records).run(Out.os());
-      break;
-    case GenClangAttrImpl:
-      ClangAttrImplEmitter(Records).run(Out.os());
-      break;
-    case GenClangAttrList:
-      ClangAttrListEmitter(Records).run(Out.os());
-      break;
-    case GenClangAttrPCHRead:
-      ClangAttrPCHReadEmitter(Records).run(Out.os());
-      break;
-    case GenClangAttrPCHWrite:
-      ClangAttrPCHWriteEmitter(Records).run(Out.os());
-      break;
-    case GenClangAttrSpellingList:
-      ClangAttrSpellingListEmitter(Records).run(Out.os());
-      break;
-    case GenClangDiagsDefs:
-      ClangDiagsDefsEmitter(Records, ClangComponent).run(Out.os());
-      break;
-    case GenClangDiagGroups:
-      ClangDiagGroupsEmitter(Records).run(Out.os());
-      break;
-    case GenClangDiagsIndexName:
-      ClangDiagsIndexNameEmitter(Records).run(Out.os());
-      break;
-    case GenClangDeclNodes:
-      ClangASTNodesEmitter(Records, "Decl", "Decl").run(Out.os());
-      ClangDeclContextEmitter(Records).run(Out.os());
-      break;
-    case GenClangStmtNodes:
-      ClangASTNodesEmitter(Records, "Stmt", "").run(Out.os());
-      break;
-    case GenClangSACheckers:
-      ClangSACheckersEmitter(Records).run(Out.os());
+      AsmMatcherEmitter(Records).run(OS);
       break;
     case GenDisassembler:
-      DisassemblerEmitter(Records).run(Out.os());
+      DisassemblerEmitter(Records).run(OS);
       break;
     case GenPseudoLowering:
-      PseudoLoweringEmitter(Records).run(Out.os());
-      break;
-    case GenOptParserDefs:
-      OptParserEmitter(Records, true).run(Out.os());
-      break;
-    case GenOptParserImpl:
-      OptParserEmitter(Records, false).run(Out.os());
+      PseudoLoweringEmitter(Records).run(OS);
       break;
     case GenDAGISel:
-      DAGISelEmitter(Records).run(Out.os());
+      DAGISelEmitter(Records).run(OS);
       break;
     case GenFastISel:
-      FastISelEmitter(Records).run(Out.os());
+      FastISelEmitter(Records).run(OS);
       break;
     case GenSubtarget:
-      SubtargetEmitter(Records).run(Out.os());
+      SubtargetEmitter(Records).run(OS);
       break;
     case GenIntrinsic:
-      IntrinsicEmitter(Records).run(Out.os());
+      IntrinsicEmitter(Records).run(OS);
       break;
     case GenTgtIntrinsic:
-      IntrinsicEmitter(Records, true).run(Out.os());
-      break;
-    case GenLLVMCConf:
-      LLVMCConfigurationEmitter(Records).run(Out.os());
+      IntrinsicEmitter(Records, true).run(OS);
       break;
     case GenEDInfo:
-      EDEmitter(Records).run(Out.os());
-      break;
-    case GenArmNeon:
-      NeonEmitter(Records).run(Out.os());
-      break;
-    case GenArmNeonSema:
-      NeonEmitter(Records).runHeader(Out.os());
-      break;
-    case GenArmNeonTest:
-      NeonEmitter(Records).runTests(Out.os());
+      EDEmitter(Records).run(OS);
       break;
     case PrintEnums:
     {
       std::vector<Record*> Recs = Records.getAllDerivedDefinitions(Class);
       for (unsigned i = 0, e = Recs.size(); i != e; ++i)
-        Out.os() << Recs[i]->getName() << ", ";
-      Out.os() << "\n";
+        OS << Recs[i]->getName() << ", ";
+      OS << "\n";
       break;
     }
     case PrintSets:
@@ -371,33 +170,29 @@ int main(int argc, char **argv) {
       Sets.addFieldExpander("Set", "Elements");
       std::vector<Record*> Recs = Records.getAllDerivedDefinitions("Set");
       for (unsigned i = 0, e = Recs.size(); i != e; ++i) {
-        Out.os() << Recs[i]->getName() << " = [";
+        OS << Recs[i]->getName() << " = [";
         const std::vector<Record*> *Elts = Sets.expand(Recs[i]);
         assert(Elts && "Couldn't expand Set instance");
         for (unsigned ei = 0, ee = Elts->size(); ei != ee; ++ei)
-          Out.os() << ' ' << (*Elts)[ei]->getName();
-        Out.os() << " ]\n";
+          OS << ' ' << (*Elts)[ei]->getName();
+        OS << " ]\n";
       }
       break;
     }
     default:
       assert(1 && "Invalid Action");
-      return 1;
+      return true;
     }
 
-    // Declare success.
-    Out.keep();
-    return 0;
-
-  } catch (const TGError &Error) {
-    PrintError(Error);
-  } catch (const std::string &Error) {
-    PrintError(Error);
-  } catch (const char *Error) {
-    PrintError(Error);
-  } catch (...) {
-    errs() << argv[0] << ": Unknown unexpected exception occurred.\n";
+    return false;
   }
+};
 
-  return 1;
+int main(int argc, char **argv) {
+  sys::PrintStackTraceOnErrorSignal();
+  PrettyStackTraceProgram X(argc, argv);
+  cl::ParseCommandLineOptions(argc, argv);
+
+  LLVMTableGenAction Action;
+  return TableGenMain(argv[0], Action);
 }
diff --git a/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp b/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp
index 74310593d29d..e8c9a4897321 100644
--- a/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp
+++ b/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp
@@ -17,7 +17,7 @@
 #include "X86DisassemblerShared.h"
 #include "X86DisassemblerTables.h"
 
-#include "TableGenBackend.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
@@ -32,26 +32,32 @@ using namespace X86Disassembler;
 /// @param parent - The class that may be the superset
 /// @return       - True if child is a subset of parent, false otherwise.
 static inline bool inheritsFrom(InstructionContext child,
-                                InstructionContext parent) {
+                                InstructionContext parent,
+                                bool VEX_LIG = false) {
   if (child == parent)
     return true;
   
   switch (parent) {
   case IC:
-    return true;
+    return(inheritsFrom(child, IC_64BIT) ||
+           inheritsFrom(child, IC_OPSIZE) ||
+           inheritsFrom(child, IC_XD) ||
+           inheritsFrom(child, IC_XS));
   case IC_64BIT:
     return(inheritsFrom(child, IC_64BIT_REXW)   ||
            inheritsFrom(child, IC_64BIT_OPSIZE) ||
            inheritsFrom(child, IC_64BIT_XD)     ||
            inheritsFrom(child, IC_64BIT_XS));
   case IC_OPSIZE:
-    return(inheritsFrom(child, IC_64BIT_OPSIZE));
+    return inheritsFrom(child, IC_64BIT_OPSIZE);
   case IC_XD:
-    return(inheritsFrom(child, IC_64BIT_XD) ||
-           inheritsFrom(child, IC_VEX_XD));
+    return inheritsFrom(child, IC_64BIT_XD);
   case IC_XS:
-    return(inheritsFrom(child, IC_64BIT_XS) ||
-           inheritsFrom(child, IC_VEX_XS));
+    return inheritsFrom(child, IC_64BIT_XS);
+  case IC_XD_OPSIZE:
+    return inheritsFrom(child, IC_64BIT_XD_OPSIZE);
+  case IC_XS_OPSIZE:
+    return inheritsFrom(child, IC_64BIT_XS_OPSIZE);
   case IC_64BIT_REXW:
     return(inheritsFrom(child, IC_64BIT_REXW_XS) ||
            inheritsFrom(child, IC_64BIT_REXW_XD) ||
@@ -62,42 +68,37 @@ static inline bool inheritsFrom(InstructionContext child,
     return(inheritsFrom(child, IC_64BIT_REXW_XD));
   case IC_64BIT_XS:
     return(inheritsFrom(child, IC_64BIT_REXW_XS));
+  case IC_64BIT_XD_OPSIZE:
+  case IC_64BIT_XS_OPSIZE:
+    return false;
   case IC_64BIT_REXW_XD:
-    return false;
   case IC_64BIT_REXW_XS:
-    return false;
   case IC_64BIT_REXW_OPSIZE:
     return false;
   case IC_VEX:
-    return(inheritsFrom(child, IC_VEX_XS) ||
-           inheritsFrom(child, IC_VEX_XD) ||
-           inheritsFrom(child, IC_VEX_L) ||
-           inheritsFrom(child, IC_VEX_W) ||
-           inheritsFrom(child, IC_VEX_OPSIZE));
+    return inheritsFrom(child, IC_VEX_W) ||
+           (VEX_LIG && inheritsFrom(child, IC_VEX_L));
   case IC_VEX_XS:
-    return(inheritsFrom(child, IC_VEX_L_XS) ||
-           inheritsFrom(child, IC_VEX_W_XS));
+    return inheritsFrom(child, IC_VEX_W_XS) ||
+           (VEX_LIG && inheritsFrom(child, IC_VEX_L_XS));
   case IC_VEX_XD:
-    return(inheritsFrom(child, IC_VEX_L_XD) ||
-           inheritsFrom(child, IC_VEX_W_XD));
-  case IC_VEX_L:
-    return(inheritsFrom(child, IC_VEX_L_XS) ||
-           inheritsFrom(child, IC_VEX_L_XD));
-  case IC_VEX_L_XS:
-    return false;
-  case IC_VEX_L_XD:
-    return false;
-  case IC_VEX_W:
-    return(inheritsFrom(child, IC_VEX_W_XS) ||
-           inheritsFrom(child, IC_VEX_W_XD) ||
-           inheritsFrom(child, IC_VEX_W_OPSIZE));
-  case IC_VEX_W_XS:
-    return false;
-  case IC_VEX_W_XD:
-    return false;
+    return inheritsFrom(child, IC_VEX_W_XD) ||
+           (VEX_LIG && inheritsFrom(child, IC_VEX_L_XD));
   case IC_VEX_OPSIZE:
-    return inheritsFrom(child, IC_VEX_W_OPSIZE);
+    return inheritsFrom(child, IC_VEX_W_OPSIZE) ||
+           (VEX_LIG && inheritsFrom(child, IC_VEX_L_OPSIZE));
+  case IC_VEX_W:
+  case IC_VEX_W_XS:
+  case IC_VEX_W_XD:
+  case IC_VEX_W_OPSIZE:
+    return false;
+  case IC_VEX_L:
+  case IC_VEX_L_XS:
+  case IC_VEX_L_XD:
+  case IC_VEX_L_OPSIZE:
+    return false;
   default:
+    llvm_unreachable("Unknown instruction class");
     return false;
   }
 }
@@ -515,6 +516,8 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, uint32_t &i) const {
       o << "IC_VEX_XD";
     else if ((index & ATTR_VEX) && (index & ATTR_XS))
       o << "IC_VEX_XS";
+    else if (index & ATTR_VEX)
+      o << "IC_VEX";
     else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS))
       o << "IC_64BIT_REXW_XS";
     else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XD))
@@ -522,6 +525,10 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, uint32_t &i) const {
     else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && 
              (index & ATTR_OPSIZE))
       o << "IC_64BIT_REXW_OPSIZE";
+    else if ((index & ATTR_64BIT) && (index & ATTR_XD) && (index & ATTR_OPSIZE))
+      o << "IC_64BIT_XD_OPSIZE";
+    else if ((index & ATTR_64BIT) && (index & ATTR_XS) && (index & ATTR_OPSIZE))
+      o << "IC_64BIT_XS_OPSIZE";
     else if ((index & ATTR_64BIT) && (index & ATTR_XS))
       o << "IC_64BIT_XS";
     else if ((index & ATTR_64BIT) && (index & ATTR_XD))
@@ -532,14 +539,16 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, uint32_t &i) const {
       o << "IC_64BIT_REXW";
     else if ((index & ATTR_64BIT))
       o << "IC_64BIT";
+    else if ((index & ATTR_XS) && (index & ATTR_OPSIZE))
+      o << "IC_XS_OPSIZE";
+    else if ((index & ATTR_XD) && (index & ATTR_OPSIZE))
+      o << "IC_XD_OPSIZE";
     else if (index & ATTR_XS)
       o << "IC_XS";
     else if (index & ATTR_XD)
       o << "IC_XD";
     else if (index & ATTR_OPSIZE)
       o << "IC_OPSIZE";
-    else if (index & ATTR_VEX)
-      o << "IC_VEX";
     else
       o << "IC";
 
@@ -616,8 +625,11 @@ void DisassemblerTables::setTableFields(ModRMDecision     &decision,
         if(newInfo.filtered)
           continue; // filtered instructions get lowest priority
         
-        if(previousInfo.name == "NOOP")
-          continue; // special case for XCHG32ar and NOOP
+        if(previousInfo.name == "NOOP" && (newInfo.name == "XCHG16ar" ||
+                                           newInfo.name == "XCHG32ar" ||
+                                           newInfo.name == "XCHG32ar64" ||
+                                           newInfo.name == "XCHG64ar"))
+          continue; // special case for XCHG*ar and NOOP
 
         if (outranks(previousInfo.insnContext, newInfo.insnContext))
           continue;
@@ -643,14 +655,19 @@ void DisassemblerTables::setTableFields(OpcodeType          type,
                                         InstructionContext  insnContext,
                                         uint8_t             opcode,
                                         const ModRMFilter   &filter,
-                                        InstrUID            uid) {
+                                        InstrUID            uid,
+                                        bool                is32bit,
+                                        bool                ignoresVEX_L) {
   unsigned index;
   
   ContextDecision &decision = *Tables[type];
 
   for (index = 0; index < IC_max; ++index) {
+    if (is32bit && inheritsFrom((InstructionContext)index, IC_64BIT))
+      continue;
+
     if (inheritsFrom((InstructionContext)index, 
-                     InstructionSpecifiers[uid].insnContext))
+                     InstructionSpecifiers[uid].insnContext, ignoresVEX_L))
       setTableFields(decision.opcodeDecisions[index].modRMDecisions[opcode], 
                      filter,
                      uid,
diff --git a/contrib/llvm/utils/TableGen/X86DisassemblerTables.h b/contrib/llvm/utils/TableGen/X86DisassemblerTables.h
index d16ebfca419e..e148cd26e809 100644
--- a/contrib/llvm/utils/TableGen/X86DisassemblerTables.h
+++ b/contrib/llvm/utils/TableGen/X86DisassemblerTables.h
@@ -260,11 +260,15 @@ class DisassemblerTables {
   /// @param filter       - The ModRMFilter that decides which ModR/M byte values
   ///                       correspond to the desired instruction.
   /// @param uid          - The unique ID of the instruction.
+  /// @param is32bit      - Instructon is only 32-bit
+  /// @param ignoresVEX_L - Instruction ignores VEX.L
   void setTableFields(OpcodeType type,
                       InstructionContext insnContext,
                       uint8_t opcode,
                       const ModRMFilter &filter,
-                      InstrUID uid);  
+                      InstrUID uid,
+                      bool is32bit,
+                      bool ignoresVEX_L);  
   
   /// specForUID - Returns the instruction specifier for a given unique
   ///   instruction ID.  Used when resolving collisions.
diff --git a/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp b/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp
index ea3bb700b27d..cae823749245 100644
--- a/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -68,7 +68,7 @@ namespace X86Local {
     DC = 7, DD = 8, DE = 9, DF = 10,
     XD = 11,  XS = 12,
     T8 = 13,  P_TA = 14,
-    A6 = 15,  A7 = 16
+    A6 = 15,  A7 = 16, TF = 17
   };
 }
 
@@ -217,6 +217,7 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables,
   HasVEXPrefix     = Rec->getValueAsBit("hasVEXPrefix");
   HasVEX_4VPrefix  = Rec->getValueAsBit("hasVEX_4VPrefix");
   HasVEX_WPrefix   = Rec->getValueAsBit("hasVEX_WPrefix");
+  IgnoresVEX_L     = Rec->getValueAsBit("ignoresVEX_L");
   HasLockPrefix    = Rec->getValueAsBit("hasLockPrefix");
   IsCodeGenOnly    = Rec->getValueAsBit("isCodeGenOnly");
   
@@ -225,15 +226,21 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables,
   
   Operands = &insn.Operands.OperandList;
   
-  IsSSE            = HasOpSizePrefix && (Name.find("16") == Name.npos);
+  IsSSE            = (HasOpSizePrefix && (Name.find("16") == Name.npos)) ||
+                     (Name.find("CRC32") != Name.npos);
   HasFROperands    = hasFROperands();
   HasVEX_LPrefix   = has256BitOperands() || Rec->getValueAsBit("hasVEX_L");
   
   // Check for 64-bit inst which does not require REX
+  Is32Bit = false;
   Is64Bit = false;
   // FIXME: Is there some better way to check for In64BitMode?
   std::vector<Record*> Predicates = Rec->getValueAsListOfDefs("Predicates");
   for (unsigned i = 0, e = Predicates.size(); i != e; ++i) {
+    if (Predicates[i]->getName().find("32Bit") != Name.npos) {
+      Is32Bit = true;
+      break;
+    }
     if (Predicates[i]->getName().find("64Bit") != Name.npos) {
       Is64Bit = true;
       break;
@@ -249,6 +256,8 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables,
              Rec->getName() == "REX64_PREFIX" ||
              Rec->getName().find("VMREAD64") != Name.npos ||
              Rec->getName().find("VMWRITE64") != Name.npos ||
+             Rec->getName().find("INVEPT64") != Name.npos ||
+             Rec->getName().find("INVVPID64") != Name.npos ||
              Rec->getName().find("MOV64") != Name.npos || 
              Rec->getName().find("PUSH64") != Name.npos ||
              Rec->getName().find("POP64") != Name.npos;
@@ -257,7 +266,7 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables,
 }
   
 void RecognizableInstr::processInstr(DisassemblerTables &tables,
-                                   const CodeGenInstruction &insn,
+	const CodeGenInstruction &insn,
                                    InstrUID uid)
 {
   // Ignore "asm parser only" instructions.
@@ -276,7 +285,9 @@ InstructionContext RecognizableInstr::insnContext() const {
   InstructionContext insnContext;
 
   if (HasVEX_4VPrefix || HasVEXPrefix) {
-    if (HasOpSizePrefix && HasVEX_LPrefix)
+    if (HasVEX_LPrefix && HasVEX_WPrefix)
+      llvm_unreachable("Don't support VEX.L and VEX.W together");
+    else if (HasOpSizePrefix && HasVEX_LPrefix)
       insnContext = IC_VEX_L_OPSIZE;
     else if (HasOpSizePrefix && HasVEX_WPrefix)
       insnContext = IC_VEX_W_OPSIZE;
@@ -303,13 +314,19 @@ InstructionContext RecognizableInstr::insnContext() const {
   } else if (Is64Bit || HasREX_WPrefix) {
     if (HasREX_WPrefix && HasOpSizePrefix)
       insnContext = IC_64BIT_REXW_OPSIZE;
+    else if (HasOpSizePrefix &&
+             (Prefix == X86Local::XD || Prefix == X86Local::TF))
+      insnContext = IC_64BIT_XD_OPSIZE;
+    else if (HasOpSizePrefix && Prefix == X86Local::XS)
+      insnContext = IC_64BIT_XS_OPSIZE;
     else if (HasOpSizePrefix)
       insnContext = IC_64BIT_OPSIZE;
     else if (HasREX_WPrefix && Prefix == X86Local::XS)
       insnContext = IC_64BIT_REXW_XS;
-    else if (HasREX_WPrefix && Prefix == X86Local::XD)
+    else if (HasREX_WPrefix &&
+             (Prefix == X86Local::XD || Prefix == X86Local::TF))
       insnContext = IC_64BIT_REXW_XD;
-    else if (Prefix == X86Local::XD)
+    else if (Prefix == X86Local::XD || Prefix == X86Local::TF)
       insnContext = IC_64BIT_XD;
     else if (Prefix == X86Local::XS)
       insnContext = IC_64BIT_XS;
@@ -318,11 +335,16 @@ InstructionContext RecognizableInstr::insnContext() const {
     else
       insnContext = IC_64BIT;
   } else {
-    if (HasOpSizePrefix)
+    if (HasOpSizePrefix &&
+        (Prefix == X86Local::XD || Prefix == X86Local::TF))
+      insnContext = IC_XD_OPSIZE;
+    else if (HasOpSizePrefix && Prefix == X86Local::XS)
+      insnContext = IC_XS_OPSIZE;
+    else if (HasOpSizePrefix)
       insnContext = IC_OPSIZE;
-    else if (Prefix == X86Local::XD)
+    else if (Prefix == X86Local::XD || Prefix == X86Local::TF)
       insnContext = IC_XD;
-    else if (Prefix == X86Local::XS)
+    else if (Prefix == X86Local::XS || Prefix == X86Local::REP)
       insnContext = IC_XS;
     else
       insnContext = IC;
@@ -342,20 +364,13 @@ RecognizableInstr::filter_ret RecognizableInstr::filter() const {
     return FILTER_STRONG;
   
   if (Form == X86Local::Pseudo ||
-      IsCodeGenOnly)
+      (IsCodeGenOnly && Name.find("_REV") == Name.npos))
     return FILTER_STRONG;
   
   if (Form == X86Local::MRMInitReg)
     return FILTER_STRONG;
     
     
-  // TEMPORARY pending bug fixes
-
-  if (Name.find("VMOVDQU") != Name.npos ||
-      Name.find("VMOVDQA") != Name.npos ||
-      Name.find("VROUND") != Name.npos)
-    return FILTER_STRONG;
-    
   // Filter out artificial instructions
     
   if (Name.find("TAILJMP") != Name.npos    ||
@@ -401,7 +416,7 @@ RecognizableInstr::filter_ret RecognizableInstr::filter() const {
   // Filter out alternate forms of AVX instructions
   if (Name.find("_alt") != Name.npos ||
       Name.find("XrYr") != Name.npos ||
-      Name.find("r64r") != Name.npos ||
+      (Name.find("r64r") != Name.npos && Name.find("r64r64") == Name.npos) ||
       Name.find("_64mr") != Name.npos ||
       Name.find("Xrr") != Name.npos ||
       Name.find("rr64") != Name.npos)
@@ -623,20 +638,43 @@ void RecognizableInstr::emitInstructionSpecifier(DisassemblerTables &tables) {
   case X86Local::MRMDestReg:
     // Operand 1 is a register operand in the R/M field.
     // Operand 2 is a register operand in the Reg/Opcode field.
+    // - In AVX, there is a register operand in the VEX.vvvv field here -
     // Operand 3 (optional) is an immediate.
-    assert(numPhysicalOperands >= 2 && numPhysicalOperands <= 3 &&
-           "Unexpected number of operands for MRMDestRegFrm");
+    if (HasVEX_4VPrefix)
+      assert(numPhysicalOperands >= 3 && numPhysicalOperands <= 4 &&
+             "Unexpected number of operands for MRMDestRegFrm with VEX_4V");
+    else
+      assert(numPhysicalOperands >= 2 && numPhysicalOperands <= 3 &&
+             "Unexpected number of operands for MRMDestRegFrm");
+  
     HANDLE_OPERAND(rmRegister)
+
+    if (HasVEX_4VPrefix)
+      // FIXME: In AVX, the register below becomes the one encoded
+      // in ModRMVEX and the one above the one in the VEX.VVVV field
+      HANDLE_OPERAND(vvvvRegister)
+          
     HANDLE_OPERAND(roRegister)
     HANDLE_OPTIONAL(immediate)
     break;
   case X86Local::MRMDestMem:
     // Operand 1 is a memory operand (possibly SIB-extended)
     // Operand 2 is a register operand in the Reg/Opcode field.
+    // - In AVX, there is a register operand in the VEX.vvvv field here -
     // Operand 3 (optional) is an immediate.
-    assert(numPhysicalOperands >= 2 && numPhysicalOperands <= 3 &&
-           "Unexpected number of operands for MRMDestMemFrm");
+    if (HasVEX_4VPrefix)
+      assert(numPhysicalOperands >= 3 && numPhysicalOperands <= 4 &&
+             "Unexpected number of operands for MRMDestMemFrm with VEX_4V");
+    else
+      assert(numPhysicalOperands >= 2 && numPhysicalOperands <= 3 &&
+             "Unexpected number of operands for MRMDestMemFrm");
     HANDLE_OPERAND(memory)
+
+    if (HasVEX_4VPrefix)
+      // FIXME: In AVX, the register below becomes the one encoded
+      // in ModRMVEX and the one above the one in the VEX.VVVV field
+      HANDLE_OPERAND(vvvvRegister)
+          
     HANDLE_OPERAND(roRegister)
     HANDLE_OPTIONAL(immediate)
     break;
@@ -805,6 +843,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
     opcodeToSet = Opcode;
     break;
   case X86Local::T8:
+  case X86Local::TF:
     opcodeType = THREEBYTE_38;
     if (needsModRMForDecode(Form))
       filter = new ModFilter(isRegFormat(Form));
@@ -855,6 +894,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
     }
     opcodeToSet = 0xd8 + (Prefix - X86Local::D8);
     break;
+  case X86Local::REP:
   default:
     opcodeType = ONEBYTE;
     switch (Opcode) {
@@ -926,7 +966,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
                               insnContext(), 
                               currentOpcode, 
                               *filter, 
-                              UID);
+                              UID, Is32Bit, IgnoresVEX_L);
     
       Spec->modifierType = MODIFIER_OPCODE;
       Spec->modifierBase = opcodeToSet;
@@ -936,14 +976,14 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
                             insnContext(), 
                             opcodeToSet, 
                             *filter, 
-                            UID);
+                            UID, Is32Bit, IgnoresVEX_L);
     }
   } else {
     tables.setTableFields(opcodeType,
                           insnContext(),
                           opcodeToSet,
                           *filter,
-                          UID);
+                          UID, Is32Bit, IgnoresVEX_L);
     
     Spec->modifierType = MODIFIER_NONE;
     Spec->modifierBase = opcodeToSet;
@@ -984,6 +1024,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("i32mem",              TYPE_Mv)
   TYPE("i32imm",              TYPE_IMMv)
   TYPE("i32i8imm",            TYPE_IMM32)
+  TYPE("u32u8imm",            TYPE_IMM32)
   TYPE("GR32",                TYPE_Rv)
   TYPE("i64mem",              TYPE_Mv)
   TYPE("i64i32imm",           TYPE_IMM64)
@@ -1029,6 +1070,9 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("offset32",            TYPE_MOFFS32)
   TYPE("offset64",            TYPE_MOFFS64)
   TYPE("VR256",               TYPE_XMM256)
+  TYPE("GR16_NOAX",           TYPE_Rv)
+  TYPE("GR32_NOAX",           TYPE_Rv)
+  TYPE("GR64_NOAX",           TYPE_R64)
   errs() << "Unhandled type string " << s << "\n";
   llvm_unreachable("Unhandled type string");
 }
@@ -1044,6 +1088,7 @@ OperandEncoding RecognizableInstr::immediateEncodingFromString
     ENCODING("i16imm",        ENCODING_IW)
   }
   ENCODING("i32i8imm",        ENCODING_IB)
+  ENCODING("u32u8imm",        ENCODING_IB)
   ENCODING("SSECC",           ENCODING_IB)
   ENCODING("i16imm",          ENCODING_Iv)
   ENCODING("i16i8imm",        ENCODING_IB)
@@ -1097,6 +1142,8 @@ OperandEncoding RecognizableInstr::roRegisterEncodingFromString
 OperandEncoding RecognizableInstr::vvvvRegisterEncodingFromString
   (const std::string &s,
    bool hasOpSizePrefix) {
+  ENCODING("GR32",            ENCODING_VVVV)
+  ENCODING("GR64",            ENCODING_VVVV)
   ENCODING("FR32",            ENCODING_VVVV)
   ENCODING("FR64",            ENCODING_VVVV)
   ENCODING("VR128",           ENCODING_VVVV)
@@ -1169,6 +1216,9 @@ OperandEncoding RecognizableInstr::opcodeModifierEncodingFromString
   ENCODING("GR64",            ENCODING_RO)
   ENCODING("GR16",            ENCODING_Rv)
   ENCODING("GR8",             ENCODING_RB)
+  ENCODING("GR16_NOAX",       ENCODING_Rv)
+  ENCODING("GR32_NOAX",       ENCODING_Rv)
+  ENCODING("GR64_NOAX",       ENCODING_RO)
   errs() << "Unhandled opcode modifier encoding " << s << "\n";
   llvm_unreachable("Unhandled opcode modifier encoding");
 }
diff --git a/contrib/llvm/utils/TableGen/X86RecognizableInstr.h b/contrib/llvm/utils/TableGen/X86RecognizableInstr.h
index 677d9f01554e..44415978273f 100644
--- a/contrib/llvm/utils/TableGen/X86RecognizableInstr.h
+++ b/contrib/llvm/utils/TableGen/X86RecognizableInstr.h
@@ -20,8 +20,8 @@
 #include "X86DisassemblerTables.h"
 
 #include "CodeGenTarget.h"
-#include "Record.h"
 
+#include "llvm/TableGen/Record.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -60,12 +60,16 @@ class RecognizableInstr {
   bool HasVEX_WPrefix;
   /// Inferred from the operands; indicates whether the L bit in the VEX prefix is set
   bool HasVEX_LPrefix;
+  // The ignoreVEX_L field from the record
+  bool IgnoresVEX_L;
   /// The hasLockPrefix field from the record
   bool HasLockPrefix;
   /// The isCodeGenOnly filed from the record
   bool IsCodeGenOnly;
-  // Whether the instruction has the predicate "Mode64Bit"
+  // Whether the instruction has the predicate "In64BitMode"
   bool Is64Bit;
+  // Whether the instruction has the predicate "In32BitMode"
+  bool Is32Bit;
   
   /// The instruction name as listed in the tables
   std::string Name;
diff --git a/contrib/sendmail/src/main.c b/contrib/sendmail/src/main.c
index 4d2318fb4355..e14ffa01c10d 100644
--- a/contrib/sendmail/src/main.c
+++ b/contrib/sendmail/src/main.c
@@ -109,8 +109,8 @@ GIDSET_T	InitialGidSet[NGROUPS_MAX];
 #if SASL
 static sasl_callback_t srvcallbacks[] =
 {
-	{	SASL_CB_VERIFYFILE,	&safesaslfile,	NULL	},
-	{	SASL_CB_PROXY_POLICY,	&proxy_policy,	NULL	},
+	{	SASL_CB_VERIFYFILE,	(sasl_callback_ft)&safesaslfile,	NULL	},
+	{	SASL_CB_PROXY_POLICY,	(sasl_callback_ft)&proxy_policy,	NULL	},
 	{	SASL_CB_LIST_END,	NULL,		NULL	}
 };
 #endif /* SASL */
diff --git a/contrib/sendmail/src/sendmail.h b/contrib/sendmail/src/sendmail.h
index 39ba16cf0cc6..ecb3fa3812f1 100644
--- a/contrib/sendmail/src/sendmail.h
+++ b/contrib/sendmail/src/sendmail.h
@@ -133,10 +133,15 @@ SM_UNUSED(static char SmailId[]) = "@(#)$Id: sendmail.h,v 8.1089 2011/03/15 23:1
 
 # if SASL == 2 || SASL >= 20000
 #  include <sasl/sasl.h>
+#  include <sasl/saslplug.h>
 #  include <sasl/saslutil.h>
+#  if SASL_VERSION_FULL < 0x020119
+typedef int (*sasl_callback_ft)(void);
+#  endif
 # else /* SASL == 2 || SASL >= 20000 */
 #  include <sasl.h>
 #  include <saslutil.h>
+typedef int (*sasl_callback_ft)(void);
 # endif /* SASL == 2 || SASL >= 20000 */
 # if defined(SASL_VERSION_MAJOR) && defined(SASL_VERSION_MINOR) && defined(SASL_VERSION_STEP)
 #  define SASL_VERSION (SASL_VERSION_MAJOR * 10000)  + (SASL_VERSION_MINOR * 100) + SASL_VERSION_STEP
diff --git a/contrib/sendmail/src/usersmtp.c b/contrib/sendmail/src/usersmtp.c
index 0de839e4221a..c5da51257a97 100644
--- a/contrib/sendmail/src/usersmtp.c
+++ b/contrib/sendmail/src/usersmtp.c
@@ -524,15 +524,15 @@ static int attemptauth	__P((MAILER *, MCI *, ENVELOPE *, SASL_AI_T *));
 
 static sasl_callback_t callbacks[] =
 {
-	{	SASL_CB_GETREALM,	&saslgetrealm,	NULL	},
+	{	SASL_CB_GETREALM,	(sasl_callback_ft)&saslgetrealm,	NULL	},
 #define CB_GETREALM_IDX	0
-	{	SASL_CB_PASS,		&getsecret,	NULL	},
+	{	SASL_CB_PASS,		(sasl_callback_ft)&getsecret,	NULL	},
 #define CB_PASS_IDX	1
-	{	SASL_CB_USER,		&getsimple,	NULL	},
+	{	SASL_CB_USER,		(sasl_callback_ft)&getsimple,	NULL	},
 #define CB_USER_IDX	2
-	{	SASL_CB_AUTHNAME,	&getsimple,	NULL	},
+	{	SASL_CB_AUTHNAME,	(sasl_callback_ft)&getsimple,	NULL	},
 #define CB_AUTHNAME_IDX	3
-	{	SASL_CB_VERIFYFILE,	&safesaslfile,	NULL	},
+	{	SASL_CB_VERIFYFILE,	(sasl_callback_ft)&safesaslfile,	NULL	},
 #define CB_SAFESASL_IDX	4
 	{	SASL_CB_LIST_END,	NULL,		NULL	}
 };
diff --git a/contrib/smbfs/mount_smbfs/mount_smbfs.8 b/contrib/smbfs/mount_smbfs/mount_smbfs.8
index 85ee6e3efefb..05b96e896c5e 100644
--- a/contrib/smbfs/mount_smbfs/mount_smbfs.8
+++ b/contrib/smbfs/mount_smbfs/mount_smbfs.8
@@ -1,6 +1,6 @@
 .\" $Id: mount_smbfs.8,v 1.10 2002/04/16 02:47:41 bp Exp $
 .\" $FreeBSD$
-.Dd January 21, 2008
+.Dd September 17, 2011
 .Dt MOUNT_SMBFS 8
 .Os
 .Sh NAME
@@ -157,11 +157,11 @@ mount_smbfs -I samba.mydomain.com //guest@samba/public /smb/public
 mount_smbfs -I 192.168.20.3 -E koi8-r:cp866 //guest@samba/tmp /smb/tmp
 .Ed
 .Pp
-It is possible to use
+It is also possible to use
 .Xr fstab 5
-for smbfs mounts:
+for smbfs mounts (the example below doesn't prompt for a password):
 .Pp
-.Dl "//guest@samba/public    /smb/public     smbfs  rw,noauto 0   0"
+.Dl "//guest@samba/public    /smb/public     smbfs  rw,noauto,-N 0   0"
 .Sh AUTHORS
 .An Boris Popov Aq bp@butya.kz ,
 .Aq bp@FreeBSD.org
diff --git a/contrib/tzcode/stdtime/localtime.c b/contrib/tzcode/stdtime/localtime.c
index 80b70acb9ff8..b1981b614300 100644
--- a/contrib/tzcode/stdtime/localtime.c
+++ b/contrib/tzcode/stdtime/localtime.c
@@ -380,13 +380,16 @@ register const int	doextend;
 	int		fid;
 	int		stored;
 	int		nread;
+	int		res;
 	union {
 		struct tzhead	tzhead;
 		char		buf[2 * sizeof(struct tzhead) +
 					2 * sizeof *sp +
 					4 * TZ_MAX_TIMES];
-	} u;
+	} *u;
 
+	u = NULL;
+	res = -1;
 	sp->goback = sp->goahead = FALSE;
 
 	/* XXX The following is from OpenBSD, and I'm not sure it is correct */
@@ -406,16 +409,24 @@ register const int	doextend;
 		** to hold the longest file name string that the implementation
 		** guarantees can be opened."
 		*/
-		char		fullname[FILENAME_MAX + 1];
+		char		*fullname;
+
+		fullname = malloc(FILENAME_MAX + 1);
+		if (fullname == NULL)
+			goto out;
 
 		if (name[0] == ':')
 			++name;
 		doaccess = name[0] == '/';
 		if (!doaccess) {
-			if ((p = TZDIR) == NULL)
+			if ((p = TZDIR) == NULL) {
+				free(fullname);
 				return -1;
-			if ((strlen(p) + 1 + strlen(name) + 1) >= sizeof fullname)
+			}
+			if (strlen(p) + 1 + strlen(name) >= FILENAME_MAX) {
+				free(fullname);
 				return -1;
+			}
 			(void) strcpy(fullname, p);
 			(void) strcat(fullname, "/");
 			(void) strcat(fullname, name);
@@ -426,37 +437,45 @@ register const int	doextend;
 				doaccess = TRUE;
 			name = fullname;
 		}
-		if (doaccess && access(name, R_OK) != 0)
+		if (doaccess && access(name, R_OK) != 0) {
+			free(fullname);
 		     	return -1;
-		if ((fid = _open(name, OPEN_MODE)) == -1)
+		}
+		if ((fid = _open(name, OPEN_MODE)) == -1) {
+			free(fullname);
 			return -1;
+		}
 		if ((_fstat(fid, &stab) < 0) || !S_ISREG(stab.st_mode)) {
+			free(fullname);
 			_close(fid);
 			return -1;
 		}
 	}
-	nread = _read(fid, u.buf, sizeof u.buf);
+	u = malloc(sizeof(*u));
+	if (u == NULL)
+		goto out;
+	nread = _read(fid, u->buf, sizeof u->buf);
 	if (_close(fid) < 0 || nread <= 0)
-		return -1;
+		goto out;
 	for (stored = 4; stored <= 8; stored *= 2) {
 		int		ttisstdcnt;
 		int		ttisgmtcnt;
 
-		ttisstdcnt = (int) detzcode(u.tzhead.tzh_ttisstdcnt);
-		ttisgmtcnt = (int) detzcode(u.tzhead.tzh_ttisgmtcnt);
-		sp->leapcnt = (int) detzcode(u.tzhead.tzh_leapcnt);
-		sp->timecnt = (int) detzcode(u.tzhead.tzh_timecnt);
-		sp->typecnt = (int) detzcode(u.tzhead.tzh_typecnt);
-		sp->charcnt = (int) detzcode(u.tzhead.tzh_charcnt);
-		p = u.tzhead.tzh_charcnt + sizeof u.tzhead.tzh_charcnt;
+		ttisstdcnt = (int) detzcode(u->tzhead.tzh_ttisstdcnt);
+		ttisgmtcnt = (int) detzcode(u->tzhead.tzh_ttisgmtcnt);
+		sp->leapcnt = (int) detzcode(u->tzhead.tzh_leapcnt);
+		sp->timecnt = (int) detzcode(u->tzhead.tzh_timecnt);
+		sp->typecnt = (int) detzcode(u->tzhead.tzh_typecnt);
+		sp->charcnt = (int) detzcode(u->tzhead.tzh_charcnt);
+		p = u->tzhead.tzh_charcnt + sizeof u->tzhead.tzh_charcnt;
 		if (sp->leapcnt < 0 || sp->leapcnt > TZ_MAX_LEAPS ||
 			sp->typecnt <= 0 || sp->typecnt > TZ_MAX_TYPES ||
 			sp->timecnt < 0 || sp->timecnt > TZ_MAX_TIMES ||
 			sp->charcnt < 0 || sp->charcnt > TZ_MAX_CHARS ||
 			(ttisstdcnt != sp->typecnt && ttisstdcnt != 0) ||
 			(ttisgmtcnt != sp->typecnt && ttisgmtcnt != 0))
-				return -1;
-		if (nread - (p - u.buf) <
+				goto out;
+		if (nread - (p - u->buf) <
 			sp->timecnt * stored +		/* ats */
 			sp->timecnt +			/* types */
 			sp->typecnt * 6 +		/* ttinfos */
@@ -464,7 +483,7 @@ register const int	doextend;
 			sp->leapcnt * (stored + 4) +	/* lsinfos */
 			ttisstdcnt +			/* ttisstds */
 			ttisgmtcnt)			/* ttisgmts */
-				return -1;
+				goto out;
 		for (i = 0; i < sp->timecnt; ++i) {
 			sp->ats[i] = (stored == 4) ?
 				detzcode(p) : detzcode64(p);
@@ -473,7 +492,7 @@ register const int	doextend;
 		for (i = 0; i < sp->timecnt; ++i) {
 			sp->types[i] = (unsigned char) *p++;
 			if (sp->types[i] >= sp->typecnt)
-				return -1;
+				goto out;
 		}
 		for (i = 0; i < sp->typecnt; ++i) {
 			struct ttinfo *	ttisp;
@@ -483,11 +502,11 @@ register const int	doextend;
 			p += 4;
 			ttisp->tt_isdst = (unsigned char) *p++;
 			if (ttisp->tt_isdst != 0 && ttisp->tt_isdst != 1)
-				return -1;
+				goto out;
 			ttisp->tt_abbrind = (unsigned char) *p++;
 			if (ttisp->tt_abbrind < 0 ||
 				ttisp->tt_abbrind > sp->charcnt)
-					return -1;
+					goto out;
 		}
 		for (i = 0; i < sp->charcnt; ++i)
 			sp->chars[i] = *p++;
@@ -512,7 +531,7 @@ register const int	doextend;
 				ttisp->tt_ttisstd = *p++;
 				if (ttisp->tt_ttisstd != TRUE &&
 					ttisp->tt_ttisstd != FALSE)
-						return -1;
+						goto out;
 			}
 		}
 		for (i = 0; i < sp->typecnt; ++i) {
@@ -525,7 +544,7 @@ register const int	doextend;
 				ttisp->tt_ttisgmt = *p++;
 				if (ttisp->tt_ttisgmt != TRUE &&
 					ttisp->tt_ttisgmt != FALSE)
-						return -1;
+						goto out;
 			}
 		}
 		/*
@@ -558,11 +577,11 @@ register const int	doextend;
 		/*
 		** If this is an old file, we're done.
 		*/
-		if (u.tzhead.tzh_version[0] == '\0')
+		if (u->tzhead.tzh_version[0] == '\0')
 			break;
-		nread -= p - u.buf;
+		nread -= p - u->buf;
 		for (i = 0; i < nread; ++i)
-			u.buf[i] = p[i];
+			u->buf[i] = p[i];
 		/*
 		** If this is a narrow integer time_t system, we're done.
 		*/
@@ -570,39 +589,43 @@ register const int	doextend;
 			break;
 	}
 	if (doextend && nread > 2 &&
-		u.buf[0] == '\n' && u.buf[nread - 1] == '\n' &&
+		u->buf[0] == '\n' && u->buf[nread - 1] == '\n' &&
 		sp->typecnt + 2 <= TZ_MAX_TYPES) {
-			struct state	ts;
+			struct state	*ts;
 			register int	result;
 
-			u.buf[nread - 1] = '\0';
-			result = tzparse(&u.buf[1], &ts, FALSE);
-			if (result == 0 && ts.typecnt == 2 &&
-				sp->charcnt + ts.charcnt <= TZ_MAX_CHARS) {
+			ts = malloc(sizeof(*ts));
+			if (ts == NULL)
+				goto out;
+			u->buf[nread - 1] = '\0';
+			result = tzparse(&u->buf[1], ts, FALSE);
+			if (result == 0 && ts->typecnt == 2 &&
+				sp->charcnt + ts->charcnt <= TZ_MAX_CHARS) {
 					for (i = 0; i < 2; ++i)
-						ts.ttis[i].tt_abbrind +=
+						ts->ttis[i].tt_abbrind +=
 							sp->charcnt;
-					for (i = 0; i < ts.charcnt; ++i)
+					for (i = 0; i < ts->charcnt; ++i)
 						sp->chars[sp->charcnt++] =
-							ts.chars[i];
+							ts->chars[i];
 					i = 0;
-					while (i < ts.timecnt &&
-						ts.ats[i] <=
+					while (i < ts->timecnt &&
+						ts->ats[i] <=
 						sp->ats[sp->timecnt - 1])
 							++i;
-					while (i < ts.timecnt &&
+					while (i < ts->timecnt &&
 					    sp->timecnt < TZ_MAX_TIMES) {
 						sp->ats[sp->timecnt] =
-							ts.ats[i];
+							ts->ats[i];
 						sp->types[sp->timecnt] =
 							sp->typecnt +
-							ts.types[i];
+							ts->types[i];
 						++sp->timecnt;
 						++i;
 					}
-					sp->ttis[sp->typecnt++] = ts.ttis[0];
-					sp->ttis[sp->typecnt++] = ts.ttis[1];
+					sp->ttis[sp->typecnt++] = ts->ttis[0];
+					sp->ttis[sp->typecnt++] = ts->ttis[1];
 			}
+			free(ts);
 	}
 	if (sp->timecnt > 1) {
 		for (i = 1; i < sp->timecnt; ++i)
@@ -620,7 +643,10 @@ register const int	doextend;
 					break;
 		}
 	}
-	return 0;
+	res = 0;
+out:
+	free(u);
+	return (res);
 }
 
 static int
diff --git a/contrib/tzdata/africa b/contrib/tzdata/africa
index 085c6a760e21..87f57c650a8b 100644
--- a/contrib/tzdata/africa
+++ b/contrib/tzdata/africa
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)africa	8.31
+# @(#)africa	8.33
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -58,7 +58,7 @@
 # I invented the following abbreviations; corrections are welcome!
 #	 2:00	WAST	West Africa Summer Time
 #	 2:30	BEAT	British East Africa Time (no longer used)
-#	 2:44:45 BEAUT	British East Africa Unified Time (no longer used)
+#	 2:45	BEAUT	British East Africa Unified Time (no longer used)
 #	 3:00	CAST	Central Africa Summer Time (no longer used)
 #	 3:00	SAST	South Africa Summer Time (no longer used)
 #	 3:00	EAT	East Africa Time
@@ -396,7 +396,7 @@ Zone	Africa/Bissau	-1:02:20 -	LMT	1911 May 26
 Zone	Africa/Nairobi	2:27:16	-	LMT	1928 Jul
 			3:00	-	EAT	1930
 			2:30	-	BEAT	1940
-			2:44:45	-	BEAUT	1960
+			2:45	-	BEAUT	1960
 			3:00	-	EAT
 
 # Lesotho
@@ -957,6 +957,11 @@ Zone	Africa/Khartoum	2:10:08 -	LMT	1931
 			2:00	Sudan	CA%sT	2000 Jan 15 12:00
 			3:00	-	EAT
 
+# South Sudan
+Zone	Africa/Juba	2:06:24 -	LMT	1931
+			2:00	Sudan	CA%sT	2000 Jan 15 12:00
+			3:00	-	EAT
+
 # Swaziland
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Africa/Mbabane	2:04:24 -	LMT	1903 Mar
@@ -966,7 +971,7 @@ Zone	Africa/Mbabane	2:04:24 -	LMT	1903 Mar
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone Africa/Dar_es_Salaam 2:37:08 -	LMT	1931
 			3:00	-	EAT	1948
-			2:44:45	-	BEAUT	1961
+			2:45	-	BEAUT	1961
 			3:00	-	EAT
 
 # Togo
@@ -1092,7 +1097,7 @@ Zone	Africa/Tunis	0:40:44 -	LMT	1881 May 12
 Zone	Africa/Kampala	2:09:40 -	LMT	1928 Jul
 			3:00	-	EAT	1930
 			2:30	-	BEAT	1948
-			2:44:45	-	BEAUT	1957
+			2:45	-	BEAUT	1957
 			3:00	-	EAT
 
 # Zambia
diff --git a/contrib/tzdata/asia b/contrib/tzdata/asia
index 1fc61f0b4910..925cc36f0fb1 100644
--- a/contrib/tzdata/asia
+++ b/contrib/tzdata/asia
@@ -1,4 +1,4 @@
-# @(#)asia	8.65
+# @(#)asia	8.69
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -2194,7 +2194,47 @@ Zone	Asia/Karachi	4:28:12 -	LMT	1907
 # http://www.timeanddate.com/news/time/westbank-gaza-end-dst-2010.html
 # </a>
 
+# From Steffen Thorsen (2011-08-26):
+# Gaza and the West Bank did go back to standard time in the beginning of
+# August, and will now enter daylight saving time again on 2011-08-30
+# 00:00 (so two periods of DST in 2011). The pause was because of
+# Ramadan.
+#
+# <a href="http://www.maannews.net/eng/ViewDetails.aspx?ID=416217">
+# http://www.maannews.net/eng/ViewDetails.aspx?ID=416217
+# </a>
+# Additional info:
+# <a href="http://www.timeanddate.com/news/time/palestine-dst-2011.html">
+# http://www.timeanddate.com/news/time/palestine-dst-2011.html
+# </a>
+
+# From Alexander Krivenyshev (2011-08-27):
+# According to the article in The Jerusalem Post:
+# "...Earlier this month, the Palestinian government in the West Bank decided to
+# move to standard time for 30 days, during Ramadan. The Palestinians in the
+# Gaza Strip accepted the change and also moved their clocks one hour back.
+# The Hamas government said on Saturday that it won't observe summertime after
+# the Muslim feast of Id al-Fitr, which begins on Tuesday..."
+# ...
+# <a href="http://www.jpost.com/MiddleEast/Article.aspx?id=235650">
+# http://www.jpost.com/MiddleEast/Article.aspx?id=235650
+# </a>
+# or
+# <a href="http://www.worldtimezone.com/dst_news/dst_news_gazastrip05.html">
+# http://www.worldtimezone.com/dst_news/dst_news_gazastrip05.html
+# </a>
 # The rules for Egypt are stolen from the `africa' file.
+
+# From Steffen Thorsen (2011-09-30):
+# West Bank did end Daylight Saving Time this morning/midnight (2011-09-30 
+# 00:00).
+# So West Bank and Gaza now have the same time again.
+#
+# Many sources, including:
+# <a href="http://www.maannews.net/eng/ViewDetails.aspx?ID=424808">
+# http://www.maannews.net/eng/ViewDetails.aspx?ID=424808
+# </a>
+
 # Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
 Rule EgyptAsia	1957	only	-	May	10	0:00	1:00	S
 Rule EgyptAsia	1957	1958	-	Oct	 1	0:00	0	-
@@ -2210,19 +2250,37 @@ Rule Palestine	2005	only	-	Oct	 4	2:00	0	-
 Rule Palestine	2006	2008	-	Apr	 1	0:00	1:00	S
 Rule Palestine	2006	only	-	Sep	22	0:00	0	-
 Rule Palestine	2007	only	-	Sep	Thu>=8	2:00	0	-
-Rule Palestine	2008	only	-	Aug	lastFri	2:00	0	-
+Rule Palestine	2008	only	-	Aug	lastFri	0:00	0	-
 Rule Palestine	2009	only	-	Mar	lastFri	0:00	1:00	S
-Rule Palestine	2010	max	-	Mar	lastSat	0:01	1:00	S
-Rule Palestine	2009	max	-	Sep	Fri>=1	2:00	0	-
+Rule Palestine	2009	only	-	Sep	Fri>=1	2:00	0	-
+Rule Palestine	2010	only	-	Mar	lastSat	0:01	1:00	S
 Rule Palestine	2010	only	-	Aug	11	0:00	0	-
 
+# From Arthur David Olson (2011-09-20):
+# 2011 transitions per http://www.timeanddate.com as of 2011-09-20.
+
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Asia/Gaza	2:17:52	-	LMT	1900 Oct
 			2:00	Zion	EET	1948 May 15
 			2:00 EgyptAsia	EE%sT	1967 Jun  5
 			2:00	Zion	I%sT	1996
 			2:00	Jordan	EE%sT	1999
-			2:00 Palestine	EE%sT
+			2:00 Palestine	EE%sT	2011 Apr  2 12:01
+			2:00	1:00	EEST	2011 Aug  1
+			2:00	-	EET
+
+Zone	Asia/Hebron	2:20:23	-	LMT	1900 Oct
+			2:00	Zion	EET	1948 May 15
+			2:00 EgyptAsia	EE%sT	1967 Jun  5
+			2:00	Zion	I%sT	1996
+			2:00	Jordan	EE%sT	1999
+			2:00 Palestine	EE%sT	2008 Aug
+			2:00 	1:00	EEST	2008 Sep
+			2:00 Palestine	EE%sT	2011 Apr  1 12:01
+			2:00	1:00	EEST	2011 Aug  1
+			2:00	-	EET	2011 Aug 30
+			2:00	1:00	EEST	2011 Sep 30 3:00
+			2:00	-	EET
 
 # Paracel Is
 # no information
diff --git a/contrib/tzdata/australasia b/contrib/tzdata/australasia
index 6949172dc31a..930beed32ed5 100644
--- a/contrib/tzdata/australasia
+++ b/contrib/tzdata/australasia
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)australasia	8.23
+# @(#)australasia	8.28
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -296,6 +296,18 @@ Zone	Indian/Cocos	6:27:40	-	LMT	1900
 # http://www.worldtimezone.com/dst_news/dst_news_fiji04.html
 # </a>
 
+# From Steffen Thorsen (2011-10-03):
+# Now the dates have been confirmed, and at least our start date 
+# assumption was correct (end date was one week wrong).
+#
+# <a href="http://www.fiji.gov.fj/index.php?option=com_content&view=article&id=4966:daylight-saving-starts-in-fiji&catid=71:press-releases&Itemid=155">
+# www.fiji.gov.fj/index.php?option=com_content&view=article&id=4966:daylight-saving-starts-in-fiji&catid=71:press-releases&Itemid=155
+# </a>
+# which says
+# Members of the public are reminded to change their time to one hour in 
+# advance at 2am to 3am on October 23, 2011 and one hour back at 3am to 
+# 2am on February 26 next year.
+
 # Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
 Rule	Fiji	1998	1999	-	Nov	Sun>=1	2:00	1:00	S
 Rule	Fiji	1999	2000	-	Feb	lastSun	3:00	0	-
@@ -303,6 +315,8 @@ Rule	Fiji	2009	only	-	Nov	29	2:00	1:00	S
 Rule	Fiji	2010	only	-	Mar	lastSun	3:00	0	-
 Rule	Fiji	2010	only	-	Oct	24	2:00	1:00	S
 Rule	Fiji	2011	only	-	Mar	Sun>=1	3:00	0	-
+Rule	Fiji	2011	only	-	Oct	23	2:00	1:00	S
+Rule	Fiji	2012	only	-	Feb	26	3:00	0	-
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Pacific/Fiji	11:53:40 -	LMT	1915 Oct 26	# Suva
 			12:00	Fiji	FJ%sT	# Fiji Time
@@ -499,7 +513,7 @@ Zone Pacific/Pago_Pago	 12:37:12 -	LMT	1879 Jul  5
 # http://www.parliament.gov.ws/documents/acts/Daylight%20Saving%20Act%20%202009%20%28English%29%20-%20Final%207-7-091.pdf
 # </a>
 
-# From Raymond Hughes (2010-10-07):
+# From Laupue Raymond Hughes (2010-10-07):
 # Please see
 # <a href="http://www.mcil.gov.ws">
 # http://www.mcil.gov.ws
@@ -509,7 +523,7 @@ Zone Pacific/Pago_Pago	 12:37:12 -	LMT	1879 Jul  5
 # to 01:00am and First Sunday April 2011 (03/04/11) - adjust clocks
 # backwards from 1:00am to 12:00am"
 
-# From Raymond Hughes (2011-03-07)
+# From Laupue Raymond Hughes (2011-03-07):
 # I believe this will be posted shortly on the website
 # <a href="http://www.mcil.gov.ws">
 # www.mcil.gov.ws
@@ -529,12 +543,74 @@ Zone Pacific/Pago_Pago	 12:37:12 -	LMT	1879 Jul  5
 # Margaret Fruean ACTING CHIEF EXECUTIVE OFFICER MINISTRY OF COMMERCE,
 # INDUSTRY AND LABOUR 28th February 2011
 
+# From David Zuelke (2011-05-09):
+# Subject: Samoa to move timezone from east to west of international date line
+# 
+# <a href="http://www.morningstar.co.uk/uk/markets/newsfeeditem.aspx?id=138501958347963">
+# http://www.morningstar.co.uk/uk/markets/newsfeeditem.aspx?id=138501958347963
+# </a>
+
+# From Mark Sim-Smith (2011-08-17):
+# I have been in contact with Leilani Tuala Warren from the Samoa Law
+# Reform Commission, and she has sent me a copy of the Bill that she
+# confirmed has been passed...Most of the sections are about maps rather
+# than the time zone change, but I'll paste the relevant bits below. But
+# the essence is that at midnight 29 Dec (UTC-11 I suppose), Samoa
+# changes from UTC-11 to UTC+13:
+#
+# International Date Line Bill 2011
+#
+# AN ACT to provide for the change to standard time in Samoa and to make
+# consequential amendments to the position of the International Date
+# Line, and for related purposes.
+#
+# BE IT ENACTED by the Legislative Assembly of Samoa in Parliament
+# assembled as follows:
+#
+# 1. Short title and commencement-(1) This Act may be cited as the
+# International Date Line Act 2011. (2) Except for section 5(3) this Act
+# commences at 12 o'clock midnight, on Thursday 29th December 2011. (3)
+# Section 5(3) commences on the date of assent by the Head of State.
+#
+# [snip]
+#
+# 3. Interpretation - [snip] "Samoa standard time" in this Act and any
+# other statute of Samoa which refers to 'Samoa standard time' means the
+# time 13 hours in advance of Co-ordinated Universal Time.
+#
+# 4. Samoa standard time - (1) Upon the commencement of this Act, Samoa
+# standard time shall be set at 13 hours in advance of Co-ordinated
+# Universal Time for the whole of Samoa. (2) All references to Samoa's
+# time zone and to Samoa standard time in Samoa in all legislation and
+# instruments after the commencement of this Act shall be references to
+# Samoa standard time as provided for in this Act. (3) Nothing in this
+# Act affects the provisions of the Daylight Saving Act 2009, except that
+# it defines Samoa standard time....
+
+# From Laupue Raymond Hughes (2011-09-02):
+# <a href="http://www.mcil.gov.ws/mcil_publications.html">
+# http://www.mcil.gov.ws/mcil_publications.html
+# </a>
+#
+# here is the official website publication for Samoa DST and dateline change
+#
+# DST
+# Year	End	Time	Start	Time
+# 2011	- - -	- - -	24 September	3:00am to 4:00am
+# 2012	01 April	4:00am to 3:00am	- - -	- - -
+#
+# Dateline Change skip Friday 30th Dec 2011
+# Thursday 29th December 2011	23:59:59 Hours
+# Saturday 31st December 2011	00:00:00 Hours
 Zone Pacific/Apia	 12:33:04 -	LMT	1879 Jul  5
 			-11:26:56 -	LMT	1911
 			-11:30	-	SAMT	1950		# Samoa Time
 			-11:00	-	WST	2010 Sep 26
 			-11:00	1:00	WSDT	2011 Apr 2 4:00
-			-11:00	-	WST
+			-11:00	-	WST	2011 Sep 24 3:00
+			-11:00	1:00	WSDT	2011 Dec 30
+			 13:00	1:00	WSDT	2012 Apr 1 4:00
+			 13:00	-	WST
 
 # Solomon Is
 # excludes Bougainville, for which see Papua New Guinea
@@ -1206,7 +1282,7 @@ Zone	Pacific/Wallis	12:15:20 -	LMT	1901
 # Lord Howe Island Board (controlling authority for the Island) is
 # seeking the community's views on various options for summer time
 # arrangements on the Island, e.g. advance clocks by 1 full hour
-# instead of only 30 minutes.  Dependant on the wishes of residents
+# instead of only 30 minutes.  [Dependent] on the wishes of residents
 # the Board may approach the NSW government to change the existing
 # arrangements.  The starting date for summer time on the Island will
 # however always coincide with the rest of NSW.
@@ -1332,7 +1408,7 @@ Zone	Pacific/Wallis	12:15:20 -	LMT	1901
 
 # From Paul Eggert (1996-01-22):
 # Today's _Wall Street Journal_ (page 1) reports that Kiribati
-# ``declared it the same day throught the country as of Jan. 1, 1995''
+# ``declared it the same day [throughout] the country as of Jan. 1, 1995''
 # as part of the competition to be first into the 21st century.
 
 
diff --git a/contrib/tzdata/backward b/contrib/tzdata/backward
index f1f95a894151..b8e591165118 100644
--- a/contrib/tzdata/backward
+++ b/contrib/tzdata/backward
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)backward	8.9
+# @(#)backward	8.10
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -67,7 +67,6 @@ Link	America/Havana		Cuba
 Link	Africa/Cairo		Egypt
 Link	Europe/Dublin		Eire
 Link	Europe/London		Europe/Belfast
-Link	Europe/Chisinau		Europe/Tiraspol
 Link	Europe/London		GB
 Link	Europe/London		GB-Eire
 Link	Etc/GMT			GMT+0
diff --git a/contrib/tzdata/europe b/contrib/tzdata/europe
index aab6833114ee..22bdd3648b7e 100644
--- a/contrib/tzdata/europe
+++ b/contrib/tzdata/europe
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)europe	8.33
+# @(#)europe	8.39
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -561,9 +561,9 @@ Rule	Russia	1985	1991	-	Mar	lastSun	 2:00s	1:00	S
 #
 Rule	Russia	1992	only	-	Mar	lastSat	 23:00	1:00	S
 Rule	Russia	1992	only	-	Sep	lastSat	 23:00	0	-
-Rule	Russia	1993	max	-	Mar	lastSun	 2:00s	1:00	S
+Rule	Russia	1993	2010	-	Mar	lastSun	 2:00s	1:00	S
 Rule	Russia	1993	1995	-	Sep	lastSun	 2:00s	0	-
-Rule	Russia	1996	max	-	Oct	lastSun	 2:00s	0	-
+Rule	Russia	1996	2010	-	Oct	lastSun	 2:00s	0	-
 
 # From Alexander Krivenyshev (2011-06-14):
 # According to Kremlin press service, Russian President Dmitry Medvedev
@@ -583,7 +583,6 @@ Rule	Russia	1996	max	-	Oct	lastSun	 2:00s	0	-
 # From Arthur David Olson (2011-06-15):
 # Take "abolishing daylight saving time" to mean that time is now considered
 # to be standard.
-# At least for now, keep the "old" Russia rules for the benefit of Belarus.
 
 # These are for backward compatibility with older versions.
 
@@ -689,6 +688,23 @@ Zone	Europe/Vienna	1:05:20 -	LMT	1893 Apr
 			1:00	EU	CE%sT
 
 # Belarus
+# From Yauhen Kharuzhy (2011-09-16):
+# By latest Belarus government act Europe/Minsk timezone was changed to
+# GMT+3 without DST (was GMT+2 with DST).
+#
+# Sources (Russian language):
+# 1.
+# <a href="http://www.belta.by/ru/all_news/society/V-Belarusi-otmenjaetsja-perexod-na-sezonnoe-vremja_i_572952.html">
+# http://www.belta.by/ru/all_news/society/V-Belarusi-otmenjaetsja-perexod-na-sezonnoe-vremja_i_572952.html
+# </a>
+# 2.
+# <a href="http://naviny.by/rubrics/society/2011/09/16/ic_articles_116_175144/">
+# http://naviny.by/rubrics/society/2011/09/16/ic_articles_116_175144/
+# </a>
+# 3.
+# <a href="http://news.tut.by/society/250578.html">
+# http://news.tut.by/society/250578.html
+# </a>
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Europe/Minsk	1:50:16 -	LMT	1880
 			1:50	-	MMT	1924 May 2 # Minsk Mean Time
@@ -700,7 +716,8 @@ Zone	Europe/Minsk	1:50:16 -	LMT	1880
 			2:00	1:00	EEST	1991 Sep 29 2:00s
 			2:00	-	EET	1992 Mar 29 0:00s
 			2:00	1:00	EEST	1992 Sep 27 0:00s
-			2:00	Russia	EE%sT
+			2:00	Russia	EE%sT	2011 Mar 27 2:00s
+			3:00	-	FET # Further-eastern European Time
 
 # Belgium
 #
@@ -1638,6 +1655,29 @@ Zone	Europe/Malta	0:58:04 -	LMT	1893 Nov  2 0:00s # Valletta
 # But [two people] separately reported via
 # Jesper Norgaard that as of 2001-01-24 Tiraspol was like Chisinau.
 # The Tiraspol entry has therefore been removed for now.
+#
+# From Alexander Krivenyshev (2011-10-17):
+# Pridnestrovian Moldavian Republic (PMR, also known as
+# "Pridnestrovie") has abolished seasonal clock change (no transition
+# to the Winter Time).
+#
+# News (in Russian):
+# <a href="http://www.kyivpost.ua/russia/news/pridnestrove-otkazalos-ot-perehoda-na-zimnee-vremya-30954.html">
+# http://www.kyivpost.ua/russia/news/pridnestrove-otkazalos-ot-perehoda-na-zimnee-vremya-30954.html
+# </a>
+#
+# <a href="http://www.allmoldova.com/moldova-news/1249064116.html">
+# http://www.allmoldova.com/moldova-news/1249064116.html
+# </a>
+#
+# The substance of this change (reinstatement of the Tiraspol entry)
+# is from a patch from Petr Machata (2011-10-17)
+#
+# From Tim Parenti (2011-10-19)
+# In addition, being situated at +4651+2938 would give Tiraspol
+# a pre-1880 LMT offset of 1:58:32.
+#
+# (which agrees with the earlier entry that had been removed)
 
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Europe/Chisinau	1:55:20 -	LMT	1880
@@ -1654,6 +1694,21 @@ Zone	Europe/Chisinau	1:55:20 -	LMT	1880
 # See Romania commentary for the guessed 1997 transition to EU rules.
 			2:00	EU	EE%sT
 
+Zone	Europe/Tiraspol	1:58:32 -	LMT	1880
+			1:55	-	CMT	1918 Feb 15 # Chisinau MT
+			1:44:24	-	BMT	1931 Jul 24 # Bucharest MT
+			2:00	Romania	EE%sT	1940 Aug 15
+			2:00	1:00	EEST	1941 Jul 17
+			1:00	C-Eur	CE%sT	1944 Aug 24
+			3:00	Russia	MSK/MSD	1990
+			3:00	-	MSK	1990 May 6
+			2:00	-	EET	1991
+			2:00	Russia	EE%sT	1992
+			2:00	E-Eur	EE%sT	1997
+# See Romania commentary for the guessed 1997 transition to EU rules.
+			2:00	EU	EE%sT	2011 Mar lastSun 1:00u
+			3:00	-	FET # Further-eastern European Time
+
 # Monaco
 # Shanks & Pottenger give 0:09:20 for Paris Mean Time; go with Howse's
 # more precise 0:09:21.
@@ -2034,7 +2089,7 @@ Zone Europe/Kaliningrad	 1:22:00 -	LMT	1893 Apr
 			 2:00	Poland	CE%sT	1946
 			 3:00	Russia	MSK/MSD	1991 Mar 31 2:00s
 			 2:00	Russia	EE%sT	2011 Mar 27 2:00s
-			 3:00	-	EET
+			 3:00	-	FET # Further-eastern European Time
 #
 # From Oscar van Vlijmen (2001-08-25): [This region consists of]
 # Respublika Adygeya, Arkhangel'skaya oblast',
@@ -2189,7 +2244,7 @@ Zone Asia/Irkutsk	 6:57:20 -	LMT	1880
 # [parts of] Respublika Sakha (Yakutiya), Chitinskaya oblast'.
 
 # From Oscar van Vlijmen (2009-11-29):
-# ...some regions of RUssia were merged with others since 2005...
+# ...some regions of [Russia] were merged with others since 2005...
 # Some names were changed, no big deal, except for one instance: a new name.
 # YAK/YAKST: UTC+9 Zabajkal'skij kraj.
 
@@ -2613,6 +2668,50 @@ Link	Europe/Istanbul	Asia/Istanbul	# Istanbul is in both continents.
 # of March at 3am the time is changing to 4am and each last Sunday of
 # October the time at 4am is changing to 3am"
 
+# From Alexander Krivenyshev (2011-09-20):
+# On September 20, 2011 the deputies of the Verkhovna Rada agreed to
+# abolish the transfer clock to winter time.
+#
+# Bill number 8330 of MP from the Party of Regions Oleg Nadoshi got
+# approval from 266 deputies.
+#
+# Ukraine abolishes transter back to the winter time (in Russian)
+# <a href="http://news.mail.ru/politics/6861560/">
+# http://news.mail.ru/politics/6861560/
+# </a>
+#
+# The Ukrainians will no longer change the clock (in Russian)
+# <a href="http://www.segodnya.ua/news/14290482.html">
+# http://www.segodnya.ua/news/14290482.html
+# </a>
+#
+# Deputies cancelled the winter time (in Russian)
+# <a href="http://www.pravda.com.ua/rus/news/2011/09/20/6600616/">
+# http://www.pravda.com.ua/rus/news/2011/09/20/6600616/
+# </a>
+#
+# From Philip Pizzey (2011-10-18):
+# Today my Ukrainian colleagues have informed me that the
+# Ukrainian parliament have decided that they will go to winter
+# time this year after all.
+#
+# From Udo Schwedt (2011-10-18):
+# As far as I understand, the recent change to the Ukranian time zone 
+# (Europe/Kiev) to introduce permanent daylight saving time (similar
+# to Russia) was reverted today:
+#
+# <a href="http://portal.rada.gov.ua/rada/control/en/publish/article/info_left?art_id=287324&cat_id=105995">
+# http://portal.rada.gov.ua/rada/control/en/publish/article/info_left?art_id=287324&cat_id=105995
+# </a>
+#
+# Also reported by Alexander Bokovoy (2011-10-18) who also noted:
+# The law documents themselves are at
+#
+# <a href="http://w1.c1.rada.gov.ua/pls/zweb_n/webproc4_1?id=&pf3511=41484">
+# http://w1.c1.rada.gov.ua/pls/zweb_n/webproc4_1?id=&pf3511=41484
+# </a>
+
+
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 # Most of Ukraine since 1970 has been like Kiev.
 # "Kyiv" is the transliteration of the Ukrainian name, but
diff --git a/contrib/tzdata/northamerica b/contrib/tzdata/northamerica
index 0dcdafb9d822..68a7cd5191dd 100644
--- a/contrib/tzdata/northamerica
+++ b/contrib/tzdata/northamerica
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)northamerica	8.42
+# @(#)northamerica	8.50
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -468,6 +468,10 @@ Zone America/Los_Angeles -7:52:58 -	LMT	1883 Nov 18 12:07:02
 # own time. I asked about daylight saving; they said it wasn't used. I
 # did not inquire about practices in the past.
 
+# From Arthur David Olson (2011-08-17):
+# For lack of better information, assume that Metlakatla's
+# abandonment of use of daylight saving resulted from the 1983 vote.
+
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone America/Juneau	 15:02:19 -	LMT	1867 Oct 18
 			 -8:57:41 -	LMT	1900 Aug 20 12:00
@@ -479,7 +483,7 @@ Zone America/Juneau	 15:02:19 -	LMT	1867 Oct 18
 			 -8:00	US	P%sT	1983 Oct 30 2:00
 			 -9:00	US	Y%sT	1983 Nov 30
 			 -9:00	US	AK%sT
-Zone America/Sitka	 -14:58:47 -	LMT	1867 Oct 18
+Zone America/Sitka	 14:58:47 -	LMT	1867 Oct 18
 			 -9:01:13 -	LMT	1900 Aug 20 12:00
 			 -8:00	-	PST	1942
 			 -8:00	US	P%sT	1946
@@ -493,7 +497,7 @@ Zone America/Metlakatla	 15:13:42 -	LMT	1867 Oct 18
 			 -8:00	US	P%sT	1946
 			 -8:00	-	PST	1969
 			 -8:00	US	P%sT	1983 Oct 30 2:00
-			 -8:00	US	MeST
+			 -8:00	-	MeST
 Zone America/Yakutat	 14:41:05 -	LMT	1867 Oct 18
 			 -9:18:55 -	LMT	1900 Aug 20 12:00
 			 -9:00	-	YST	1942
@@ -593,8 +597,8 @@ Zone Pacific/Honolulu	-10:31:26 -	LMT	1896 Jan 13 12:00 #Schmitt&Cox
 			-10:30	-	HST	1933 Apr 30 2:00 #Laws 1933
 			-10:30	1:00	HDT	1933 May 21 12:00 #Laws 1933+12
 			-10:30	-	HST	1942 Feb 09 2:00 #Schmitt&Cox+2
-			-10:30	1:00	HDT	1945 Sep 30 2:00 #Schmitt&Fox+2
-			-10:30	US	H%sT	1947 Jun  8 2:00 #Schmitt&Fox+2
+			-10:30	1:00	HDT	1945 Sep 30 2:00 #Schmitt&Cox+2
+			-10:30	-	HST	1947 Jun  8 2:00 #Schmitt&Cox+2
 			-10:00	-	HST
 
 # Now we turn to US areas that have diverged from the consensus since 1970.
@@ -1163,12 +1167,29 @@ Rule	StJohns	1960	1986	-	Oct	lastSun	2:00	0	S
 # From Paul Eggert (2000-10-02):
 # INMS (2000-09-12) says that, since 1988 at least, Newfoundland switches
 # at 00:01 local time.  For now, assume it started in 1987.
+
+# From Michael Pelley (2011-09-12):
+# We received today, Monday, September 12, 2011, notification that the
+# changes to the Newfoundland Standard Time Act have been proclaimed.
+# The change in the Act stipulates that the change from Daylight Savings
+# Time to Standard Time and from Standard Time to Daylight Savings Time
+# now occurs at 2:00AM.
+# ...
+# <a href="http://www.assembly.nl.ca/legislation/sr/annualstatutes/2011/1106.chp.htm">
+# http://www.assembly.nl.ca/legislation/sr/annualstatutes/2011/1106.chp.htm
+# </a>
+# ...
+# MICHAEL PELLEY  |  Manager of Enterprise Architecture - Solution Delivery
+# Office of the Chief Information Officer
+# Executive Council
+# Government of Newfoundland & Labrador
+
 Rule	StJohns	1987	only	-	Apr	Sun>=1	0:01	1:00	D
 Rule	StJohns	1987	2006	-	Oct	lastSun	0:01	0	S
 Rule	StJohns	1988	only	-	Apr	Sun>=1	0:01	2:00	DD
 Rule	StJohns	1989	2006	-	Apr	Sun>=1	0:01	1:00	D
-Rule	StJohns	2007	max	-	Mar	Sun>=8	0:01	1:00	D
-Rule	StJohns	2007	max	-	Nov	Sun>=1	0:01	0	S
+Rule	StJohns	2007	2011	-	Mar	Sun>=8	0:01	1:00	D
+Rule	StJohns	2007	2010	-	Nov	Sun>=1	0:01	0	S
 #
 # St John's has an apostrophe, but Posix file names can't have apostrophes.
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
@@ -1178,7 +1199,8 @@ Zone America/St_Johns	-3:30:52 -	LMT	1884
 			-3:30:52 StJohns N%sT	1935 Mar 30
 			-3:30	StJohns	N%sT	1942 May 11
 			-3:30	Canada	N%sT	1946
-			-3:30	StJohns	N%sT
+			-3:30	StJohns	N%sT	2011 Nov
+			-3:30	Canada	N%sT
 
 # most of east Labrador
 
@@ -1192,7 +1214,8 @@ Zone America/Goose_Bay	-4:01:40 -	LMT	1884 # Happy Valley-Goose Bay
 			-3:30	StJohns	N%sT	1942 May 11
 			-3:30	Canada	N%sT	1946
 			-3:30	StJohns	N%sT	1966 Mar 15 2:00
-			-4:00	StJohns	A%sT
+			-4:00	StJohns	A%sT	2011 Nov
+			-4:00	Canada	A%sT
 
 
 # west Labrador, Nova Scotia, Prince Edward I
@@ -1924,20 +1947,69 @@ Zone America/Dawson_Creek -8:00:56 -	LMT	1884
 # daylight saving....
 # http://www.nnsl.com/frames/newspapers/2006-11/nov13_06none.html
 
-# From Chris Walton (2007-03-14):
-# Today I phoned the "hamlet office" to find out what Resolute was doing with
-# its clocks.
+# From Chris Walton (2011-03-21):
+# Back in 2007 I initiated the creation of a new "zone file" for Resolute
+# Bay. Resolute Bay is a small community located about 900km north of
+# the Arctic Circle. The zone file was required because Resolute Bay had
+# decided to use UTC-5 instead of UTC-6 for the winter of 2006-2007.
 #
-# The individual that answered the phone confirmed that the clocks did not
-# move at the end of daylight saving on October 29/2006.  He also told me that
-# the clocks did not move this past weekend (March 11/2007)....
-
-# From Chris Walton (2008-11-13):
-# ...the residents of Resolute believe that they are changing "time zones"
-# twice a year.  In winter months, local time is qualified with "Eastern
-# Time" which is really "Eastern Standard Time (UTC-5)".  In summer
-# months, local time is qualified with "Central Time" which is really
-# "Central Daylight Time (UTC-5)"...
+# According to new information which I received last week, Resolute Bay
+# went back to using UTC-6 in the winter of 2007-2008...
+#
+# On March 11/2007 most of Canada went onto daylight saving. On March
+# 14/2007 I phoned the Resolute Bay hamlet office to do a "time check." I
+# talked to somebody that was both knowledgeable and helpful. I was able
+# to confirm that Resolute Bay was still operating on UTC-5. It was
+# explained to me that Resolute Bay had been on the Eastern Time zone
+# (EST) in the winter, and was now back on the Central Time zone (CDT).
+# i.e. the time zone had changed twice in the last year but the clocks
+# had not moved. The residents had to know which time zone they were in
+# so they could follow the correct TV schedule...
+#
+# On Nov 02/2008 most of Canada went onto standard time. On Nov 03/2008 I
+# phoned the Resolute Bay hamlet office...[D]ue to the challenging nature
+# of the phone call, I decided to seek out an alternate source of
+# information. I found an e-mail address for somebody by the name of
+# Stephanie Adams whose job was listed as "Inns North Support Officer for
+# Arctic Co-operatives." I was under the impression that Stephanie lived
+# and worked in Resolute Bay...
+#
+# On March 14/2011 I phoned the hamlet office again. I was told that
+# Resolute Bay had been using Central Standard Time over the winter of
+# 2010-2011 and that the clocks had therefore been moved one hour ahead
+# on March 13/2011. The person I talked to was aware that Resolute Bay
+# had previously experimented with Eastern Standard Time but he could not
+# tell me when the practice had stopped.
+#
+# On March 17/2011 I searched the Web to find an e-mail address of
+# somebody that might be able to tell me exactly when Resolute Bay went
+# off Eastern Standard Time. I stumbled on the name "Aziz Kheraj." Aziz
+# used to be the mayor of Resolute Bay and he apparently owns half the
+# businesses including "South Camp Inn." This website has some info on
+# Aziz:
+# <a href="http://www.uphere.ca/node/493">
+# http://www.uphere.ca/node/493
+# </a>
+#
+# I sent Aziz an e-mail asking when Resolute Bay had stopped using
+# Eastern Standard Time.
+#
+# Aziz responded quickly with this: "hi, The time was not changed for the
+# 1 year only, the following year, the community went back to the old way
+# of "spring ahead-fall behind" currently we are zulu plus 5 hrs and in
+# the winter Zulu plus 6 hrs"
+#
+# This of course conflicted with everything I had ascertained in November 2008.
+#
+# I sent Aziz a copy of my 2008 e-mail exchange with Stephanie. Aziz
+# responded with this: "Hi, Stephanie lives in Winnipeg. I live here, You
+# may want to check with the weather office in Resolute Bay or do a
+# search on the weather through Env. Canada. web site"
+#
+# If I had realized the Stephanie did not live in Resolute Bay I would
+# never have contacted her.  I now believe that all the information I
+# obtained in November 2008 should be ignored...
+# I apologize for reporting incorrect information in 2008.
 
 # Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
 Rule	NT_YK	1918	only	-	Apr	14	2:00	1:00	D
@@ -1965,14 +2037,12 @@ Zone America/Iqaluit	0	-	zzz	1942 Aug # Frobisher Bay est.
 			-6:00	Canada	C%sT	2000 Oct 29 2:00
 			-5:00	Canada	E%sT
 # aka Qausuittuq
-# Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
-Rule	Resolute 2006	max	-	Nov	Sun>=1	2:00	0	ES
-Rule	Resolute 2007	max	-	Mar	Sun>=8	2:00	0	CD
 Zone America/Resolute	0	-	zzz	1947 Aug 31 # Resolute founded
 			-6:00	NT_YK	C%sT	2000 Oct 29 2:00
 			-5:00	-	EST	2001 Apr  1 3:00
 			-6:00	Canada	C%sT	2006 Oct 29 2:00
-			-5:00	Resolute	%sT
+			-5:00	-	EST	2007 Mar 11 3:00
+			-6:00	Canada	C%sT
 # aka Kangiqiniq
 Zone America/Rankin_Inlet 0	-	zzz	1957 # Rankin Inlet founded
 			-6:00	NT_YK	C%sT	2000 Oct 29 2:00
diff --git a/contrib/tzdata/southamerica b/contrib/tzdata/southamerica
index 90f531d712f7..6012005c4850 100644
--- a/contrib/tzdata/southamerica
+++ b/contrib/tzdata/southamerica
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)southamerica	8.50
+# @(#)southamerica	8.52
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -797,6 +797,39 @@ Zone	America/La_Paz	-4:32:36 -	LMT	1890
 # <a href="http://www.timeanddate.com/news/time/brazil-dst-2008-2009.html">
 # http://www.timeanddate.com/news/time/brazil-dst-2008-2009.html
 # </a>
+#
+# From Alexander Krivenyshev (2011-10-04):
+# State Bahia will return to Daylight savings time this year after 8 years off.
+# The announcement was made by Governor Jaques Wagner in an interview to a 
+# television station in Salvador. 
+
+# In Portuguese:
+# <a href="http://g1.globo.com/bahia/noticia/2011/10/governador-jaques-wagner-confirma-horario-de-verao-na-bahia.html">
+# http://g1.globo.com/bahia/noticia/2011/10/governador-jaques-wagner-confirma-horario-de-verao-na-bahia.html
+# </a> and
+# <a href="http://noticias.terra.com.br/brasil/noticias/0,,OI5390887-EI8139,00-Bahia+volta+a+ter+horario+de+verao+apos+oito+anos.html">
+# http://noticias.terra.com.br/brasil/noticias/0,,OI5390887-EI8139,00-Bahia+volta+a+ter+horario+de+verao+apos+oito+anos.html
+# </a>
+
+# From Guilherme Bernardes Rodrigues (2011-10-07):
+# There is news in the media, however there is still no decree about it.
+# I just send a e-mail to Zulmira Brandão at
+# <a href="http://pcdsh01.on.br/">http://pcdsh01.on.br/</a> the
+# oficial agency about time in Brazil, and she confirmed that the old rule is
+# still in force.
+
+# From Guilherme Bernardes Rodrigues (2011-10-14)
+# It's official, the President signed a decree that includes Bahia in summer
+# time.
+#	 [ and in a second message (same day): ]
+# I found the decree.
+#
+# DECRETO No- 7.584, DE 13 DE OUTUBRO DE 2011
+# Link :
+# <a href="http://www.in.gov.br/visualiza/index.jsp?data=13/10/2011&jornal=1000&pagina=6&totalArquivos=6">
+# http://www.in.gov.br/visualiza/index.jsp?data=13/10/2011&jornal=1000&pagina=6&totalArquivos=6
+# </a>
+
 
 # Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
 # Decree <a href="http://pcdsh01.on.br/HV20466.htm">20,466</a> (1931-10-01)
@@ -1034,7 +1067,8 @@ Zone America/Maceio	-2:22:52 -	LMT	1914
 # of America/Salvador.
 Zone America/Bahia	-2:34:04 -	LMT	1914
 			-3:00	Brazil	BR%sT	2003 Sep 24
-			-3:00	-	BRT
+			-3:00	-	BRT	2011 Oct 16
+			-3:00	Brazil	BR%sT
 #
 # Goias (GO), Distrito Federal (DF), Minas Gerais (MG),
 # Espirito Santo (ES), Rio de Janeiro (RJ), Sao Paulo (SP), Parana (PR),
diff --git a/contrib/tzdata/zone.tab b/contrib/tzdata/zone.tab
index 251776449fd4..02166ee96163 100644
--- a/contrib/tzdata/zone.tab
+++ b/contrib/tzdata/zone.tab
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)zone.tab	8.45
+# @(#)zone.tab	8.50
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 #
@@ -120,7 +120,7 @@ CA	+4901-08816	America/Nipigon	Eastern Time - Ontario & Quebec - places that did
 CA	+4823-08915	America/Thunder_Bay	Eastern Time - Thunder Bay, Ontario
 CA	+6344-06828	America/Iqaluit	Eastern Time - east Nunavut - most locations
 CA	+6608-06544	America/Pangnirtung	Eastern Time - Pangnirtung, Nunavut
-CA	+744144-0944945	America/Resolute	Eastern Standard Time - Resolute, Nunavut
+CA	+744144-0944945	America/Resolute	Central Standard Time - Resolute, Nunavut
 CA	+484531-0913718	America/Atikokan	Eastern Standard Time - Atikokan, Ontario and Southampton I, Nunavut
 CA	+624900-0920459	America/Rankin_Inlet	Central Time - central Nunavut
 CA	+4953-09709	America/Winnipeg	Central Time - Manitoba & west Ontario
@@ -257,7 +257,8 @@ LV	+5657+02406	Europe/Riga
 LY	+3254+01311	Africa/Tripoli
 MA	+3339-00735	Africa/Casablanca
 MC	+4342+00723	Europe/Monaco
-MD	+4700+02850	Europe/Chisinau
+MD	+4700+02850	Europe/Chisinau	most locations
+MD	+4651+02938	Europe/Tiraspol	Pridnestrovie
 ME	+4226+01916	Europe/Podgorica
 MF	+1804-06305	America/Marigot
 MG	-1855+04731	Indian/Antananarivo
@@ -319,7 +320,8 @@ PL	+5215+02100	Europe/Warsaw
 PM	+4703-05620	America/Miquelon
 PN	-2504-13005	Pacific/Pitcairn
 PR	+182806-0660622	America/Puerto_Rico
-PS	+3130+03428	Asia/Gaza
+PS	+3130+03428	Asia/Gaza	Gaza Strip
+PS	+313200+0350542	Asia/Hebron	West Bank
 PT	+3843-00908	Europe/Lisbon	mainland
 PT	+3238-01654	Atlantic/Madeira	Madeira Islands
 PT	+3744-02540	Atlantic/Azores	Azores
@@ -361,6 +363,7 @@ SM	+4355+01228	Europe/San_Marino
 SN	+1440-01726	Africa/Dakar
 SO	+0204+04522	Africa/Mogadishu
 SR	+0550-05510	America/Paramaribo
+SS	+0451+03136	Africa/Juba
 ST	+0020+00644	Africa/Sao_Tome
 SV	+1342-08912	America/El_Salvador
 SX	+180305-0630250	America/Lower_Princes
diff --git a/crypto/openssh/ChangeLog b/crypto/openssh/ChangeLog
index f51d17e03756..ee6460d4dd8d 100644
--- a/crypto/openssh/ChangeLog
+++ b/crypto/openssh/ChangeLog
@@ -1,13 +1,463 @@
-20110403
- - (djm) [README contrib/caldera/openssh.spec contrib/redhat/openssh.spec]
-   [contrib/suse/openssh.spec] Prepare for 5.8p2 release.
- - (djm) [version.h] crank version
- - Release 5.8p2
+20110906
+ - (djm) [README version.h] Correct version
+ - (djm) [contrib/redhat/openssh.spec] Correct restorcon => restorecon
+ - (djm) Respin OpenSSH-5.9p1 release
+
+20110905
+ - (djm) [README contrib/caldera/openssh.spec contrib/redhat/openssh.spec]
+   [contrib/suse/openssh.spec] Update version numbers.
+
+20110904
+ - (djm) [regress/connect-privsep.sh regress/test-exec.sh] demote fatal
+   regress errors for the sandbox to warnings. ok tim dtucker
+ - (dtucker) [ssh-keygen.c ssh-pkcs11.c] Bug #1929: add null implementations
+   ofsh-pkcs11.cpkcs_init and pkcs_terminate for building without dlopen
+   support.
+
+20110829
+ - (djm) [openbsd-compat/port-linux.c] Suppress logging when attempting
+   to switch SELinux context away from unconfined_t, based on patch from
+   Jan Chadima; bz#1919 ok dtucker@
+
+20110827
+ - (dtucker) [auth-skey.c] Add log.h to fix build --with-skey.
+
+20110818
+ - (tim) [configure.ac] Typo in error message spotted by Andy Tsouladze
+
+20110817
+ - (tim) [mac.c myproposal.h] Wrap SHA256 and SHA512 in ifdefs for
+   OpenSSL 0.9.7. ok djm
+ - (djm) [ openbsd-compat/bsd-cygwin_util.c openbsd-compat/bsd-cygwin_util.h]
+   binary_pipe is no longer required on Cygwin; patch from Corinna Vinschen
+ - (djm) [configure.ac] error out if the host lacks the necessary bits for
+   an explicitly requested sandbox type
+ - (djm) [contrib/ssh-copy-id] Missing backlslash; spotted by
+   bisson AT archlinux.org
+ - (djm) OpenBSD CVS Sync
+   - dtucker@cvs.openbsd.org 2011/06/03 05:35:10
+     [regress/cfgmatch.sh]
+     use OBJ to find test configs, patch from Tim Rice
+   - markus@cvs.openbsd.org 2011/06/30 22:44:43
+     [regress/connect-privsep.sh]
+     test with sandbox enabled; ok djm@
+   - djm@cvs.openbsd.org 2011/08/02 01:23:41
+     [regress/cipher-speed.sh regress/try-ciphers.sh]
+     add SHA256/SHA512 based HMAC modes
+ - (djm) [regress/cipher-speed.sh regress/try-ciphers.sh] disable HMAC-SHA2
+   MAC tests for platforms that hack EVP_SHA2 support
+
+20110812
+ - (dtucker) [openbsd-compat/port-linux.c] Bug 1924: Improve selinux context
+   change error by reporting old and new context names  Patch from
+   jchadima at redhat.
+ - (djm) [contrib/redhat/openssh.spec contrib/redhat/sshd.init]
+   [contrib/suse/openssh.spec contrib/suse/rc.sshd] Updated RHEL and SLES
+   init scrips from imorgan AT nas.nasa.gov; bz#1920
+ - (djm) [contrib/ssh-copy-id] Fix failure for cases where the path to the
+   identify file contained whitespace. bz#1828 patch from gwenael.lambrouin
+   AT gmail.com; ok dtucker@
+
+20110807
+ - (dtucker) OpenBSD CVS Sync
+   - jmc@cvs.openbsd.org 2008/06/26 06:59:39
+     [moduli.5]
+     tweak previous;
+   - sobrado@cvs.openbsd.org 2009/10/28 08:56:54
+     [moduli.5]
+     "Diffie-Hellman" is the usual spelling for the cryptographic protocol
+     first published by Whitfield Diffie and Martin Hellman in 1976.
+     ok jmc@
+   - jmc@cvs.openbsd.org 2010/10/14 20:41:28
+     [moduli.5]
+     probabalistic -> probabilistic; from naddy
+   - dtucker@cvs.openbsd.org 2011/08/07 12:55:30
+     [sftp.1]
+     typo, fix from Laurent Gautrot
+
+20110805
+ - OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/06/23 23:35:42
+     [monitor.c]
+     ignore EINTR errors from poll()
+   - tedu@cvs.openbsd.org 2011/07/06 18:09:21
+     [authfd.c]
+     bzero the agent address.  the kernel was for a while very cranky about
+     these things.  evne though that's fixed, always good to initialize
+     memory.  ok deraadt djm
+   - djm@cvs.openbsd.org 2011/07/29 14:42:45
+     [sandbox-systrace.c]
+     fail open(2) with EPERM rather than SIGKILLing the whole process. libc
+     will call open() to do strerror() when NLS is enabled;
+     feedback and ok markus@
+   - markus@cvs.openbsd.org 2011/08/01 19:18:15
+     [gss-serv.c]
+     prevent post-auth resource exhaustion (int overflow leading to 4GB malloc);
+     report Adam Zabrock; ok djm@, deraadt@
+   - djm@cvs.openbsd.org 2011/08/02 01:22:11
+     [mac.c myproposal.h ssh.1 ssh_config.5 sshd.8 sshd_config.5]
+     Add new SHA256 and SHA512 based HMAC modes from
+     http://www.ietf.org/id/draft-dbider-sha2-mac-for-ssh-02.txt
+     Patch from mdb AT juniper.net; feedback and ok markus@
+   - djm@cvs.openbsd.org 2011/08/02 23:13:01
+     [version.h]
+     crank now, release later
+   - djm@cvs.openbsd.org 2011/08/02 23:15:03
+     [ssh.c]
+     typo in comment
+
+20110624
+ - (djm) [configure.ac Makefile.in sandbox-darwin.c] Add a sandbox for
+   Darwin/OS X using sandbox_init() + setrlimit(); feedback and testing
+   markus@
+
+20110623
+ - OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/06/22 21:47:28
+     [servconf.c]
+     reuse the multistate option arrays to pretty-print options for "sshd -T"
+   - djm@cvs.openbsd.org 2011/06/22 21:57:01
+     [servconf.c servconf.h sshd.c sshd_config.5]
+     [configure.ac Makefile.in]
+     introduce sandboxing of the pre-auth privsep child using systrace(4).
+     
+     This introduces a new "UsePrivilegeSeparation=sandbox" option for
+     sshd_config that applies mandatory restrictions on the syscalls the
+     privsep child can perform. This prevents a compromised privsep child
+     from being used to attack other hosts (by opening sockets and proxying)
+     or probing local kernel attack surface.
+     
+     The sandbox is implemented using systrace(4) in unsupervised "fast-path"
+     mode, where a list of permitted syscalls is supplied. Any syscall not
+     on the list results in SIGKILL being sent to the privsep child. Note
+     that this requires a kernel with the new SYSTR_POLICY_KILL option.
+     
+     UsePrivilegeSeparation=sandbox will become the default in the future
+     so please start testing it now.
+     
+     feedback dtucker@; ok markus@
+   - djm@cvs.openbsd.org 2011/06/22 22:08:42
+     [channels.c channels.h clientloop.c clientloop.h mux.c ssh.c]
+     hook up a channel confirm callback to warn the user then requested X11
+     forwarding was refused by the server; ok markus@
+   - djm@cvs.openbsd.org 2011/06/23 09:34:13
+     [sshd.c ssh-sandbox.h sandbox.h sandbox-rlimit.c sandbox-systrace.c]
+     [sandbox-null.c]
+     rename sandbox.h => ssh-sandbox.h to make things easier for portable
+ - (djm) [sandbox-null.c] Dummy sandbox for platforms that don't support
+   setrlimit(2)
+
+20110620
+ - OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/06/04 00:10:26
+     [ssh_config.5]
+     explain IdentifyFile's semantics a little better, prompted by bz#1898
+     ok dtucker jmc
+   - markus@cvs.openbsd.org 2011/06/14 22:49:18
+     [authfile.c]
+     make sure key_parse_public/private_rsa1() no longer consumes its input
+     buffer.  fixes ssh-add for passphrase-protected ssh1-keys;
+     noted by naddy@; ok djm@
+   - djm@cvs.openbsd.org 2011/06/17 21:44:31
+     [log.c log.h monitor.c monitor.h monitor_wrap.c monitor_wrap.h sshd.c]
+     make the pre-auth privsep slave log via a socketpair shared with the
+     monitor rather than /var/empty/dev/log; ok dtucker@ deraadt@ markus@
+   - djm@cvs.openbsd.org 2011/06/17 21:46:16
+     [sftp-server.c]
+     the protocol version should be unsigned; bz#1913 reported by mb AT
+     smartftp.com
+   - djm@cvs.openbsd.org 2011/06/17 21:47:35
+     [servconf.c]
+     factor out multi-choice option parsing into a parse_multistate label
+     and some support structures; ok dtucker@
+   - djm@cvs.openbsd.org 2011/06/17 21:57:25
+     [clientloop.c]
+     setproctitle for a mux master that has been gracefully stopped;
+     bz#1911 from Bert.Wesarg AT googlemail.com
+
+20110603
+ - (dtucker) [README version.h contrib/caldera/openssh.spec
+   contrib/redhat/openssh.spec contrib/suse/openssh.spec] Pull the version
+   bumps from the 5.8p2 branch into HEAD.  ok djm.
+ - (tim) [configure.ac defines.h] Run test program to detect system mail
+   directory. Add --with-maildir option to override. Fixed OpenServer 6
+   getting it wrong. Fixed many systems having MAIL=/var/mail//username
+   ok dtucker
+ - (dtucker) [monitor.c] Remove the !HAVE_SOCKETPAIR case.  We use socketpair
+   unconditionally in other places and the survey data we have does not show
+   any systems that use it.  "nuke it" djm@
+ - (djm) [configure.ac] enable setproctitle emulation for OS X
+ - (djm) OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/06/03 00:54:38
+     [ssh.c]
+     bz#1883 - setproctitle() to identify mux master; patch from Bert.Wesarg
+     AT googlemail.com; ok dtucker@
+     NB. includes additional portability code to enable setproctitle emulation
+     on platforms that don't support it.
+   - dtucker@cvs.openbsd.org 2011/06/03 01:37:40
+     [ssh-agent.c]
+     Check current parent process ID against saved one to determine if the parent
+     has exited, rather than attempting to send a zero signal, since the latter
+     won't work if the parent has changed privs.  bz#1905, patch from Daniel Kahn
+     Gillmor, ok djm@
+    - dtucker@cvs.openbsd.org 2011/05/31 02:01:58
+     [regress/dynamic-forward.sh]
+     back out revs 1.6 and 1.5 since it's not reliable
+   - dtucker@cvs.openbsd.org 2011/05/31 02:03:34
+     [regress/dynamic-forward.sh]
+     work around startup and teardown races; caught by deraadt
+   - dtucker@cvs.openbsd.org 2011/06/03 00:29:52
+     [regress/dynamic-forward.sh]
+     Retry establishing the port forwarding after a small delay, should make
+     the tests less flaky when the previous test is slow to shut down and free
+     up the port.
+ - (tim) [regress/cfgmatch.sh] Build/test out of tree fix.
+
+20110529
+ - (djm) OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/05/23 03:30:07
+     [auth-rsa.c auth.c auth.h auth2-pubkey.c monitor.c monitor_wrap.c]
+     [pathnames.h servconf.c servconf.h sshd.8 sshd_config sshd_config.5]
+     allow AuthorizedKeysFile to specify multiple files, separated by spaces.
+     Bring back authorized_keys2 as a default search path (to avoid breaking
+     existing users of this file), but override this in sshd_config so it will
+     be no longer used on fresh installs. Maybe in 2015 we can remove it
+     entierly :)
+     
+     feedback and ok markus@ dtucker@
+   - djm@cvs.openbsd.org 2011/05/23 03:33:38
+     [auth.c]
+     make secure_filename() spam debug logs less
+   - djm@cvs.openbsd.org 2011/05/23 03:52:55
+     [sshconnect.c]
+     remove extra newline
+   - jmc@cvs.openbsd.org 2011/05/23 07:10:21
+     [sshd.8 sshd_config.5]
+     tweak previous; ok djm
+   - djm@cvs.openbsd.org 2011/05/23 07:24:57
+     [authfile.c]
+     read in key comments for v.2 keys (though note that these are not
+     passed over the agent protocol); bz#439, based on patch from binder
+     AT arago.de; ok markus@
+   - djm@cvs.openbsd.org 2011/05/24 07:15:47
+     [readconf.c readconf.h ssh.c ssh_config.5 sshconnect.c sshconnect2.c]
+     Remove undocumented legacy options UserKnownHostsFile2 and
+     GlobalKnownHostsFile2 by making UserKnownHostsFile/GlobalKnownHostsFile
+     accept multiple paths per line and making their defaults include
+     known_hosts2; ok markus
+   - djm@cvs.openbsd.org 2011/05/23 03:31:31
+     [regress/cfgmatch.sh]
+     include testing of multiple/overridden AuthorizedKeysFiles
+     refactor to simply daemon start/stop and get rid of racy constructs
+
+20110520
+ - (djm) [session.c] call setexeccon() before executing passwd for pw
+   changes; bz#1891 reported by jchadima AT redhat.com; ok dtucker@
+ - (djm) [aclocal.m4 configure.ac] since gcc-4.x ignores all -Wno-options
+   options, we should corresponding -W-option when trying to determine
+   whether it is accepted.  Also includes a warning fix on the program
+   fragment uses (bad main() return type).
+   bz#1900 and bz#1901 reported by g.esp AT free.fr; ok dtucker@
+ - (djm) [servconf.c] remove leftover droppings of AuthorizedKeysFile2
+ - OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/05/15 08:09:01
+     [authfd.c monitor.c serverloop.c]
+     use FD_CLOEXEC consistently; patch from zion AT x96.org
+   - djm@cvs.openbsd.org 2011/05/17 07:13:31
+     [key.c]
+     fatal() if asked to generate a legacy ECDSA cert (these don't exist)
+     and fix the regress test that was trying to generate them :)
+   - djm@cvs.openbsd.org 2011/05/20 00:55:02
+     [servconf.c]
+     the options TrustedUserCAKeys, RevokedKeysFile, AuthorizedKeysFile
+     and AuthorizedPrincipalsFile were not being correctly applied in
+     Match blocks, despite being overridable there; ok dtucker@
+   - dtucker@cvs.openbsd.org 2011/05/20 02:00:19
+     [servconf.c]
+     Add comment documenting what should be after the preauth check.  ok djm
+   - djm@cvs.openbsd.org 2011/05/20 03:25:45
+     [monitor.c monitor_wrap.c servconf.c servconf.h]
+     use a macro to define which string options to copy between configs
+     for Match. This avoids problems caused by forgetting to keep three
+     code locations in perfect sync and ordering
+     
+     "this is at once beautiful and horrible" + ok dtucker@
+   - djm@cvs.openbsd.org 2011/05/17 07:13:31
+     [regress/cert-userkey.sh]
+     fatal() if asked to generate a legacy ECDSA cert (these don't exist)
+     and fix the regress test that was trying to generate them :)
+   - djm@cvs.openbsd.org 2011/05/20 02:43:36
+     [cert-hostkey.sh]
+     another attempt to generate a v00 ECDSA key that broke the test
+     ID sync only - portable already had this somehow
+   - dtucker@cvs.openbsd.org 2011/05/20 05:19:50
+     [dynamic-forward.sh]
+     Prevent races in dynamic forwarding test; ok djm
+   - dtucker@cvs.openbsd.org 2011/05/20 06:32:30
+     [dynamic-forward.sh]
+     fix dumb error in dynamic-forward test
+
+20110515
+ - (djm) OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/05/05 05:12:08
+     [mux.c]
+     gracefully fall back when ControlPath is too large for a
+     sockaddr_un. ok markus@ as part of a larger diff
+   - dtucker@cvs.openbsd.org 2011/05/06 01:03:35
+     [sshd_config]
+     clarify language about overriding defaults.  bz#1892, from Petr Cerny
+   - djm@cvs.openbsd.org 2011/05/06 01:09:53
+     [sftp.1]
+     mention that IPv6 addresses must be enclosed in square brackets;
+     bz#1845
+   - djm@cvs.openbsd.org 2011/05/06 02:05:41
+     [sshconnect2.c]
+     fix memory leak; bz#1849 ok dtucker@
+   - djm@cvs.openbsd.org 2011/05/06 21:14:05
+     [packet.c packet.h]
+     set traffic class for IPv6 traffic as we do for IPv4 TOS;
+     patch from lionel AT mamane.lu via Colin Watson in bz#1855;
+     ok markus@
+   - djm@cvs.openbsd.org 2011/05/06 21:18:02
+     [ssh.c ssh_config.5]
+     add a %L expansion (short-form of the local host name) for ControlPath;
+     sync some more expansions with LocalCommand; ok markus@
+   - djm@cvs.openbsd.org 2011/05/06 21:31:38
+     [readconf.c ssh_config.5]
+     support negated Host matching, e.g.
+     
+     Host *.example.org !c.example.org
+        User mekmitasdigoat
+     
+     Will match "a.example.org", "b.example.org", but not "c.example.org"
+     ok markus@
+   - djm@cvs.openbsd.org 2011/05/06 21:34:32
+     [clientloop.c mux.c readconf.c readconf.h ssh.c ssh_config.5]
+     Add a RequestTTY ssh_config option to allow configuration-based
+     control over tty allocation (like -t/-T); ok markus@
+   - djm@cvs.openbsd.org 2011/05/06 21:38:58
+     [ssh.c]
+     fix dropping from previous diff
+   - djm@cvs.openbsd.org 2011/05/06 22:20:10
+     [PROTOCOL.mux]
+     fix numbering; from bert.wesarg AT googlemail.com
+   - jmc@cvs.openbsd.org 2011/05/07 23:19:39
+     [ssh_config.5]
+     - tweak previous
+     - come consistency fixes
+     ok djm
+   - jmc@cvs.openbsd.org 2011/05/07 23:20:25
+     [ssh.1]
+     +.It RequestTTY
+   - djm@cvs.openbsd.org 2011/05/08 12:52:01
+     [PROTOCOL.mux clientloop.c clientloop.h mux.c]
+     improve our behaviour when TTY allocation fails: if we are in
+     RequestTTY=auto mode (the default), then do not treat at TTY
+     allocation error as fatal but rather just restore the local TTY
+     to cooked mode and continue. This is more graceful on devices that
+     never allocate TTYs.
+     
+     If RequestTTY is set to "yes" or "force", then failure to allocate
+     a TTY is fatal.
+     
+     ok markus@
+   - djm@cvs.openbsd.org 2011/05/10 05:46:46
+     [authfile.c]
+     despam debug() logs by detecting that we are trying to load a private key
+     in key_try_load_public() and returning early; ok markus@
+   - djm@cvs.openbsd.org 2011/05/11 04:47:06
+     [auth.c auth.h auth2-pubkey.c pathnames.h servconf.c servconf.h]
+     remove support for authorized_keys2; it is a relic from the early days
+     of protocol v.2 support and has been undocumented for many years;
+     ok markus@
+   - djm@cvs.openbsd.org 2011/05/13 00:05:36
+     [authfile.c]
+     warn on unexpected key type in key_parse_private_type()
+ - (djm) [packet.c] unbreak portability #endif
+
+20110510
+ - (dtucker) [openbsd-compat/openssl-compat.{c,h}] Bug #1882: fix
+   --with-ssl-engine which was broken with the change from deprecated
+   SSLeay_add_all_algorithms().  ok djm
+
+20110506
+ - (dtucker) [openbsd-compat/regress/closefromtest.c] Bug #1875: add prototype
+   for closefrom() in test code.  Report from Dan Wallis via Gentoo.
+
+20110505
+ - (djm) [defines.h] Move up include of netinet/ip.h for IPTOS
+   definitions. From des AT des.no
+ - (djm) [Makefile.in WARNING.RNG aclocal.m4 buildpkg.sh.in configure.ac]
+   [entropy.c ssh-add.c ssh-agent.c ssh-keygen.c ssh-keyscan.c]
+   [ssh-keysign.c ssh-pkcs11-helper.c ssh-rand-helper.8 ssh-rand-helper.c]
+   [ssh.c ssh_prng_cmds.in sshd.c contrib/aix/buildbff.sh]
+   [regress/README.regress] Remove ssh-rand-helper and all its
+   tentacles. PRNGd seeding has been rolled into entropy.c directly.
+   Thanks to tim@ for testing on affected platforms.
+ - OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/03/10 02:52:57
+     [auth2-gss.c auth2.c auth.h]
+     allow GSSAPI authentication to detect when a server-side failure causes
+     authentication failure and don't count such failures against MaxAuthTries;
+     bz#1244 from simon AT sxw.org.uk; ok markus@ before lock
+   - okan@cvs.openbsd.org 2011/03/15 10:36:02
+     [ssh-keyscan.c]
+     use timerclear macro
+     ok djm@
+   - stevesk@cvs.openbsd.org 2011/03/23 15:16:22
+     [ssh-keygen.1 ssh-keygen.c]
+     Add -A option.  For each of the key types (rsa1, rsa, dsa and ecdsa)
+     for which host keys do not exist, generate the host keys with the
+     default key file path, an empty passphrase, default bits for the key
+     type, and default comment.  This will be used by /etc/rc to generate
+     new host keys.  Idea from deraadt.
+     ok deraadt
+   - stevesk@cvs.openbsd.org 2011/03/23 16:24:56
+     [ssh-keygen.1]
+     -q not used in /etc/rc now so remove statement.
+   - stevesk@cvs.openbsd.org 2011/03/23 16:50:04
+     [ssh-keygen.c]
+     remove -d, documentation removed >10 years ago; ok markus
+   - jmc@cvs.openbsd.org 2011/03/24 15:29:30
+     [ssh-keygen.1]
+     zap trailing whitespace;
+   - stevesk@cvs.openbsd.org 2011/03/24 22:14:54
+     [ssh-keygen.c]
+     use strcasecmp() for "clear" cert permission option also; ok djm
+   - stevesk@cvs.openbsd.org 2011/03/29 18:54:17
+     [misc.c misc.h servconf.c]
+     print ipqos friendly string for sshd -T; ok markus
+     # sshd -Tf sshd_config|grep ipqos
+     ipqos lowdelay throughput
+   - djm@cvs.openbsd.org 2011/04/12 04:23:50
+     [ssh-keygen.c]
+     fix -Wshadow
+   - djm@cvs.openbsd.org 2011/04/12 05:32:49
+     [sshd.c]
+     exit with 0 status on SIGTERM; bz#1879
+   - djm@cvs.openbsd.org 2011/04/13 04:02:48
+     [ssh-keygen.1]
+     improve wording; bz#1861
+   - djm@cvs.openbsd.org 2011/04/13 04:09:37
+     [ssh-keygen.1]
+     mention valid -b sizes for ECDSA keys; bz#1862
+   - djm@cvs.openbsd.org 2011/04/17 22:42:42
+     [PROTOCOL.mux clientloop.c clientloop.h mux.c ssh.1 ssh.c]
+     allow graceful shutdown of multiplexing: request that a mux server
+     removes its listener socket and refuse future multiplexing requests;
+     ok markus@
+   - djm@cvs.openbsd.org 2011/04/18 00:46:05
+     [ssh-keygen.c]
+     certificate options are supposed to be packed in lexical order of
+     option name (though we don't actually enforce this at present).
+     Move one up that was out of sequence
+   - djm@cvs.openbsd.org 2011/05/04 21:15:29
+     [authfile.c authfile.h ssh-add.c]
+     allow "ssh-add - < key"; feedback and ok markus@
+ - (tim) [configure.ac] Add AC_LANG_SOURCE to OPENSSH_CHECK_CFLAG_COMPILE
+   so autoreconf 2.68 is happy.
+ - (tim) [defines.h] Deal with platforms that do not have S_IFSOCK ok djm@
 
-20110329
- - (djm) [entropy.c] closefrom() before running ssh-rand-helper; leftover fds
-   noticed by tmraz AT redhat.com
- 
 20110221
  - (dtucker) [contrib/cygwin/ssh-host-config] From Corinna: revamp of the
    Cygwin-specific service installer script ssh-host-config.  The actual
@@ -19,6 +469,13 @@
    The new script also is more thorough to inform the user why the
    script failed.  Patch from vinschen at redhat com.
 
+20110218
+ - OpenBSD CVS Sync
+   - djm@cvs.openbsd.org 2011/02/16 00:31:14
+     [ssh-keysign.c]
+     make hostbased auth with ECDSA keys work correctly. Based on patch
+     by harvey.eneman AT oracle.com in bz#1858; ok markus@ (pre-lock)
+
 20110206
  - (dtucker) [openbsd-compat/port-linux.c] Bug #1851: fix syntax error in
    selinux code.  Patch from Leonardo Chiquitto 
@@ -46,6 +503,14 @@
    succeeded before using its result. Patch from cjwatson AT debian.org;
    bz#1851
 
+20110127
+ - (tim) [config.guess config.sub] Sync with upstream.
+ - (tim) [configure.ac] Consistent M4 quoting throughout, updated obsolete
+   AC_TRY_COMPILE with AC_COMPILE_IFELSE, updated obsolete AC_TRY_LINK with
+   AC_LINK_IFELSE, updated obsolete AC_TRY_RUN with AC_RUN_IFELSE, misc white
+   space changes for consistency/readability. Makes autoconf 2.68 happy.
+   "Nice work" djm
+
 20110125
  - (djm) [configure.ac Makefile.in ssh.c openbsd-compat/port-linux.c
    openbsd-compat/port-linux.h] Move SELinux-specific code from ssh.c to
@@ -1256,4 +1721,3 @@
      (use "ssh-keygen -t v00 -s ca_key ..." to generate a v00 certificate)
      
      ok markus@
-
diff --git a/crypto/openssh/INSTALL b/crypto/openssh/INSTALL
index 09dfd666d993..0031dea85964 100644
--- a/crypto/openssh/INSTALL
+++ b/crypto/openssh/INSTALL
@@ -16,9 +16,7 @@ The remaining items are optional.
 
 NB. If you operating system supports /dev/random, you should configure
 OpenSSL to use it. OpenSSH relies on OpenSSL's direct support of
-/dev/random, or failing that, either prngd or egd.  If you don't have
-any of these you will have to rely on ssh-rand-helper, which is inferior
-to a good kernel-based solution or prngd.
+/dev/random, or failing that, either prngd or egd
 
 PRNGD:
 
@@ -262,4 +260,4 @@ Please refer to the "reporting bugs" section of the webpage at
 http://www.openssh.com/
 
 
-$Id: INSTALL,v 1.85 2010/02/11 22:34:22 djm Exp $
+$Id: INSTALL,v 1.86 2011/05/05 03:48:37 djm Exp $
diff --git a/crypto/openssh/PROTOCOL.mux b/crypto/openssh/PROTOCOL.mux
index 2a5817bd7f8c..9ad2566026b7 100644
--- a/crypto/openssh/PROTOCOL.mux
+++ b/crypto/openssh/PROTOCOL.mux
@@ -73,6 +73,13 @@ non-multiplexed ssh(1) connection. Two additional cases that the
 client must cope with are it receiving a signal itself and the
 server disconnecting without sending an exit message.
 
+A master may also send a MUX_S_TTY_ALLOC_FAIL before MUX_S_EXIT_MESSAGE
+if remote TTY allocation was unsuccessful. The client may use this to
+return its local tty to "cooked" mode.
+
+	uint32	MUX_S_TTY_ALLOC_FAIL
+	uint32	session id
+
 3. Health checks
 
 The client may request a health check/PID report from a server:
@@ -149,10 +156,21 @@ The client then sends its standard input and output file descriptors
 
 The contents of "reserved" are currently ignored.
 
-A server may reply with a MUX_S_SESSION_OPEED, a MUX_S_PERMISSION_DENIED
+A server may reply with a MUX_S_SESSION_OPENED, a MUX_S_PERMISSION_DENIED
 or a MUX_S_FAILURE.
 
-8. Status messages
+8. Requesting shutdown of mux listener
+
+A client may request the master to stop accepting new multiplexing requests
+and remove its listener socket.
+
+	uint32	MUX_C_STOP_LISTENING
+	uint32	request id
+
+A server may reply with a MUX_S_OK, a MUX_S_PERMISSION_DENIED or a
+MUX_S_FAILURE.
+
+9. Status messages
 
 The MUX_S_OK message is empty:
 
@@ -169,7 +187,7 @@ The MUX_S_PERMISSION_DENIED and MUX_S_FAILURE include a reason:
 	uint32	client request id
 	string	reason
 
-9. Protocol numbers
+10. Protocol numbers
 
 #define MUX_MSG_HELLO		0x00000001
 #define MUX_C_NEW_SESSION	0x10000002
@@ -178,6 +196,7 @@ The MUX_S_PERMISSION_DENIED and MUX_S_FAILURE include a reason:
 #define MUX_C_OPEN_FWD		0x10000006
 #define MUX_C_CLOSE_FWD		0x10000007
 #define MUX_C_NEW_STDIO_FWD	0x10000008
+#define MUX_C_STOP_LISTENING	0x10000009
 #define MUX_S_OK		0x80000001
 #define MUX_S_PERMISSION_DENIED	0x80000002
 #define MUX_S_FAILURE		0x80000003
@@ -185,6 +204,7 @@ The MUX_S_PERMISSION_DENIED and MUX_S_FAILURE include a reason:
 #define MUX_S_ALIVE		0x80000005
 #define MUX_S_SESSION_OPENED	0x80000006
 #define MUX_S_REMOTE_PORT	0x80000007
+#define MUX_S_TTY_ALLOC_FAIL	0x80000008
 
 #define MUX_FWD_LOCAL	1
 #define MUX_FWD_REMOTE	2
@@ -192,12 +212,10 @@ The MUX_S_PERMISSION_DENIED and MUX_S_FAILURE include a reason:
 
 XXX TODO
 XXX extended status (e.g. report open channels / forwards)
-XXX graceful close (delete listening socket, but keep existing sessions active)
 XXX lock (maybe)
 XXX watch in/out traffic (pre/post crypto)
 XXX inject packet (what about replies)
 XXX server->client error/warning notifications
-XXX port0 rfwd (need custom response message)
 XXX send signals via mux
 
-$OpenBSD: PROTOCOL.mux,v 1.4 2011/01/31 21:42:15 djm Exp $
+$OpenBSD: PROTOCOL.mux,v 1.7 2011/05/08 12:52:01 djm Exp $
diff --git a/crypto/openssh/README b/crypto/openssh/README
index 960abf9895e7..e26654b2f796 100644
--- a/crypto/openssh/README
+++ b/crypto/openssh/README
@@ -1,4 +1,4 @@
-See http://www.openssh.com/txt/release-5.8p2 for the release notes.
+See http://www.openssh.com/txt/release-5.9 for the release notes.
 
 - A Japanese translation of this document and of the OpenSSH FAQ is
 - available at http://www.unixuser.org/~haruyama/security/openssh/index.html
@@ -62,4 +62,4 @@ References -
 [6] http://www.openbsd.org/cgi-bin/man.cgi?query=style&sektion=9
 [7] http://www.openssh.com/faq.html
 
-$Id: README,v 1.75.4.2 2011/05/03 00:04:21 djm Exp $
+$Id: README,v 1.77.2.2 2011/09/06 23:11:20 djm Exp $
diff --git a/crypto/openssh/WARNING.RNG b/crypto/openssh/WARNING.RNG
deleted file mode 100644
index 97da74ff79bc..000000000000
--- a/crypto/openssh/WARNING.RNG
+++ /dev/null
@@ -1,95 +0,0 @@
-This document contains a description of portable OpenSSH's random
-number collection code. An alternate reading of this text could
-well be titled "Why I should pressure my system vendor to supply
-/dev/random in their OS".
-
-Why is this important? OpenSSH depends on good, unpredictable numbers
-for generating keys, performing digital signatures and forming
-cryptographic challenges. If the random numbers that it uses are
-predictable, then the strength of the whole system is compromised.
-
-A particularly pernicious problem arises with DSA keys (used by the
-ssh2 protocol). Performing a DSA signature (which is required for
-authentication), entails the use of a 160 bit random number.  If an
-attacker can predict this number, then they can deduce your *private*
-key and impersonate you or your hosts.
-
-If you are using the builtin random number support (configure will
-tell you if this is the case), then read this document in its entirety.
-Alternately, you can use Lutz Jaenicke's PRNGd - a small daemon which
-collects random numbers and makes them available by a socket.
-
-Please also request that your OS vendor provides a kernel-based random
-number collector (/dev/random) in future versions of your operating
-systems by default.
-
-On to the description...
-
-The portable OpenSSH contains random number collection support for
-systems which lack a kernel entropy pool (/dev/random).
-
-This collector (as of 3.1 and beyond) comes as an external application
-that allows the local admin to decide on how to implement entropy
-collection.
-
-The default entropy collector operates by executing the programs listed
-in ($etcdir)/ssh_prng_cmds, reading their output and adding it to the
-PRNG supplied by OpenSSL (which is hash-based). It also stirs in the
-output of several system calls and timings from the execution of the
-programs that it runs.
-
-The ssh_prng_cmds file also specifies a 'rate' for each program. This
-represents the number of bits of randomness per byte of output from
-the specified program.
-
-The random number code will also read and save a seed file to
-~/.ssh/prng_seed. This contents of this file are added to the random
-number generator at startup. The goal here is to maintain as much
-randomness between sessions as possible.
-
-The default entropy collection code has two main problems:
-
-1. It is slow.
-
-Executing each program in the list can take a large amount of time,
-especially on slower machines. Additionally some program can take a
-disproportionate time to execute.
-
-Tuning the random helper can be done by running ./ssh-random-helper in
-very verbose mode ("-vvv") and identifying the commands that are taking
-excessive amounts of time or hanging altogher.  Any problem commands can
-be modified or removed from ssh_prng_cmds.
-
-The default entropy collector will timeout programs which take too long
-to execute, the actual timeout used can be adjusted with the
---with-entropy-timeout configure option. OpenSSH will not try to
-re-execute programs which have not been found, have had a non-zero
-exit status or have timed out more than a couple of times.
-
-2. Estimating the real 'rate' of program outputs is non-trivial
-
-The shear volume of the task is problematic: there are currently
-around 50 commands in the ssh_prng_cmds list, portable OpenSSH
-supports at least 12 different OSs. That is already 600 sets of data
-to be analysed, without taking into account the numerous differences
-between versions of each OS.
-
-On top of this, the different commands can produce varying amounts of
-usable data depending on how busy the machine is, how long it has been
-up and various other factors.
-
-To make matters even more complex, some of the commands are reporting
-largely the same data as other commands (eg. the various "ps" calls).
-
-
-How to avoid the default entropy code?
-
-The best way is to read the OpenSSL documentation and recompile OpenSSL
-to use prngd or egd.  Some platforms (like earily solaris) have 3rd
-party /dev/random devices that can be also used for this task.
-
-If you are forced to use ssh-rand-helper consider still downloading
-prngd/egd and configure OpenSSH using --with-prngd-port=xx or
---with-prngd-socket=xx (refer to INSTALL for more information).
-
-$Id: WARNING.RNG,v 1.8 2005/05/26 01:47:54 djm Exp $
diff --git a/crypto/openssh/aclocal.m4 b/crypto/openssh/aclocal.m4
index b68a47080ffc..9bdea5ec23bb 100644
--- a/crypto/openssh/aclocal.m4
+++ b/crypto/openssh/aclocal.m4
@@ -1,8 +1,26 @@
-dnl $Id: aclocal.m4,v 1.6 2005/09/19 16:33:39 tim Exp $
+dnl $Id: aclocal.m4,v 1.8 2011/05/20 01:45:25 djm Exp $
 dnl
 dnl OpenSSH-specific autoconf macros
 dnl
 
+dnl OSSH_CHECK_CFLAG_COMPILE(check_flag[, define_flag])
+dnl Check that $CC accepts a flag 'check_flag'. If it is supported append
+dnl 'define_flag' to $CFLAGS. If 'define_flag' is not specified, then append
+dnl 'check_flag'.
+AC_DEFUN([OSSH_CHECK_CFLAG_COMPILE], [{
+	AC_MSG_CHECKING([if $CC supports $1])
+	saved_CFLAGS="$CFLAGS"
+	CFLAGS="$CFLAGS $1"
+	_define_flag="$2"
+	test "x$_define_flag" = "x" && _define_flag="$1"
+	AC_COMPILE_IFELSE([AC_LANG_SOURCE([[int main(void) { return 0; }]])],
+		[ AC_MSG_RESULT([yes])
+		  CFLAGS="$saved_CFLAGS $_define_flag"],
+		[ AC_MSG_RESULT([no])
+		  CFLAGS="$saved_CFLAGS" ]
+	)
+}])
+
 
 dnl OSSH_CHECK_HEADER_FOR_FIELD(field, header, symbol)
 dnl Does AC_EGREP_HEADER on 'header' for the string 'field'
@@ -33,16 +51,6 @@ AC_DEFUN(OSSH_CHECK_HEADER_FOR_FIELD, [
 	fi
 ])
 
-dnl OSSH_PATH_ENTROPY_PROG(variablename, command):
-dnl Tidiness function, sets 'undef' if not found, and does the AC_SUBST
-AC_DEFUN(OSSH_PATH_ENTROPY_PROG, [
-	AC_PATH_PROG($1, $2)
-	if test -z "[$]$1" ; then
-		$1="undef"
-	fi
-	AC_SUBST($1)
-])
-
 dnl Check for socklen_t: historically on BSD it is an int, and in
 dnl POSIX 1g it is a type of its own, but some platforms use different
 dnl types for the argument to getsockopt, getpeername, etc.  So we
diff --git a/crypto/openssh/audit-linux.c b/crypto/openssh/audit-linux.c
index 20174d700a5d..b3ee2f4daed8 100644
--- a/crypto/openssh/audit-linux.c
+++ b/crypto/openssh/audit-linux.c
@@ -1,4 +1,4 @@
-/* $Id$ */
+/* $Id: audit-linux.c,v 1.1 2011/01/17 10:15:30 dtucker Exp $ */
 
 /*
  * Copyright 2010 Red Hat, Inc.  All rights reserved.
diff --git a/crypto/openssh/auth-pam.c b/crypto/openssh/auth-pam.c
index fc79dee80e81..675006e6fb25 100644
--- a/crypto/openssh/auth-pam.c
+++ b/crypto/openssh/auth-pam.c
@@ -45,7 +45,7 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* Based on $FreeBSD$ */
+/* Based on $FreeBSD: src/crypto/openssh/auth2-pam-freebsd.c,v 1.11 2003/03/31 13:48:18 des Exp $ */
 #include "includes.h"
 
 #include <sys/types.h>
diff --git a/crypto/openssh/auth-rsa.c b/crypto/openssh/auth-rsa.c
index 4edaab056aa0..4ab46cd5163a 100644
--- a/crypto/openssh/auth-rsa.c
+++ b/crypto/openssh/auth-rsa.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: auth-rsa.c,v 1.79 2010/12/03 23:55:27 djm Exp $ */
+/* $OpenBSD: auth-rsa.c,v 1.80 2011/05/23 03:30:07 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -160,44 +160,27 @@ auth_rsa_challenge_dialog(Key *key)
 	return (success);
 }
 
-/*
- * check if there's user key matching client_n,
- * return key if login is allowed, NULL otherwise
- */
-
-int
-auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
+static int
+rsa_key_allowed_in_file(struct passwd *pw, char *file,
+    const BIGNUM *client_n, Key **rkey)
 {
-	char line[SSH_MAX_PUBKEY_BYTES], *file;
+	char line[SSH_MAX_PUBKEY_BYTES];
 	int allowed = 0;
 	u_int bits;
 	FILE *f;
 	u_long linenum = 0;
 	Key *key;
 
-	/* Temporarily use the user's uid. */
-	temporarily_use_uid(pw);
-
-	/* The authorized keys. */
-	file = authorized_keys_file(pw);
 	debug("trying public RSA key file %s", file);
-	f = auth_openkeyfile(file, pw, options.strict_modes);
-	if (!f) {
-		xfree(file);
-		restore_uid();
-		return (0);
-	}
-
-	/* Flag indicating whether the key is allowed. */
-	allowed = 0;
-
-	key = key_new(KEY_RSA1);
+	if ((f = auth_openkeyfile(file, pw, options.strict_modes)) == NULL)
+		return 0;
 
 	/*
 	 * Go though the accepted keys, looking for the current key.  If
 	 * found, perform a challenge-response dialog to verify that the
 	 * user really has the corresponding private key.
 	 */
+	key = key_new(KEY_RSA1);
 	while (read_keyfile_line(f, file, line, sizeof(line), &linenum) != -1) {
 		char *cp;
 		char *key_options;
@@ -235,7 +218,10 @@ auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
 		}
 		/* cp now points to the comment part. */
 
-		/* Check if the we have found the desired key (identified by its modulus). */
+		/*
+		 * Check if the we have found the desired key (identified
+		 * by its modulus).
+		 */
 		if (BN_cmp(key->rsa->n, client_n) != 0)
 			continue;
 
@@ -264,11 +250,7 @@ auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
 		break;
 	}
 
-	/* Restore the privileged uid. */
-	restore_uid();
-
 	/* Close the file. */
-	xfree(file);
 	fclose(f);
 
 	/* return key if allowed */
@@ -276,7 +258,33 @@ auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
 		*rkey = key;
 	else
 		key_free(key);
-	return (allowed);
+
+	return allowed;
+}
+
+/*
+ * check if there's user key matching client_n,
+ * return key if login is allowed, NULL otherwise
+ */
+
+int
+auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
+{
+	char *file;
+	u_int i, allowed = 0;
+
+	temporarily_use_uid(pw);
+
+	for (i = 0; !allowed && i < options.num_authkeys_files; i++) {
+		file = expand_authorized_keys(
+		    options.authorized_keys_files[i], pw);
+		allowed = rsa_key_allowed_in_file(pw, file, client_n, rkey);
+		xfree(file);
+	}
+
+	restore_uid();
+
+	return allowed;
 }
 
 /*
diff --git a/crypto/openssh/auth-skey.c b/crypto/openssh/auth-skey.c
index cb43dba48521..3536ec8fad42 100644
--- a/crypto/openssh/auth-skey.c
+++ b/crypto/openssh/auth-skey.c
@@ -39,6 +39,7 @@
 #include "hostfile.h"
 #include "auth.h"
 #include "ssh-gss.h"
+#include "log.h"
 #include "monitor_wrap.h"
 
 static void *
diff --git a/crypto/openssh/auth.c b/crypto/openssh/auth.c
index 23c39e58734c..893dc7bc67bc 100644
--- a/crypto/openssh/auth.c
+++ b/crypto/openssh/auth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: auth.c,v 1.91 2010/11/29 23:45:51 djm Exp $ */
+/* $OpenBSD: auth.c,v 1.94 2011/05/23 03:33:38 djm Exp $ */
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
  *
@@ -332,7 +332,7 @@ auth_root_allowed(char *method)
  *
  * This returns a buffer allocated by xmalloc.
  */
-static char *
+char *
 expand_authorized_keys(const char *filename, struct passwd *pw)
 {
 	char *file, ret[MAXPATHLEN];
@@ -355,18 +355,6 @@ expand_authorized_keys(const char *filename, struct passwd *pw)
 	return (xstrdup(ret));
 }
 
-char *
-authorized_keys_file(struct passwd *pw)
-{
-	return expand_authorized_keys(options.authorized_keys_file, pw);
-}
-
-char *
-authorized_keys_file2(struct passwd *pw)
-{
-	return expand_authorized_keys(options.authorized_keys_file2, pw);
-}
-
 char *
 authorized_principals_file(struct passwd *pw)
 {
@@ -469,7 +457,6 @@ secure_filename(FILE *f, const char *file, struct passwd *pw,
 		}
 		strlcpy(buf, cp, sizeof(buf));
 
-		debug3("secure_filename: checking '%s'", buf);
 		if (stat(buf, &st) < 0 ||
 		    (st.st_uid != 0 && st.st_uid != uid) ||
 		    (st.st_mode & 022) != 0) {
@@ -479,11 +466,9 @@ secure_filename(FILE *f, const char *file, struct passwd *pw,
 		}
 
 		/* If are past the homedir then we can stop */
-		if (comparehome && strcmp(homedir, buf) == 0) {
-			debug3("secure_filename: terminating check at '%s'",
-			    buf);
+		if (comparehome && strcmp(homedir, buf) == 0)
 			break;
-		}
+
 		/*
 		 * dirname should always complete with a "/" path,
 		 * but we can be paranoid and check for "." too
diff --git a/crypto/openssh/auth.h b/crypto/openssh/auth.h
index 77317aee6dbf..0d786c4d53c4 100644
--- a/crypto/openssh/auth.h
+++ b/crypto/openssh/auth.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: auth.h,v 1.66 2010/05/07 11:30:29 djm Exp $ */
+/* $OpenBSD: auth.h,v 1.69 2011/05/23 03:30:07 djm Exp $ */
 
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
@@ -53,6 +53,7 @@ struct Authctxt {
 	int		 valid;		/* user exists and is allowed to login */
 	int		 attempt;
 	int		 failures;
+	int		 server_caused_failure; 
 	int		 force_pwchange;
 	char		*user;		/* username sent by the client */
 	char		*service;
@@ -167,8 +168,7 @@ char	*get_challenge(Authctxt *);
 int	verify_response(Authctxt *, const char *);
 void	abandon_challenge_response(Authctxt *);
 
-char	*authorized_keys_file(struct passwd *);
-char	*authorized_keys_file2(struct passwd *);
+char	*expand_authorized_keys(const char *, struct passwd *pw);
 char	*authorized_principals_file(struct passwd *);
 
 FILE	*auth_openkeyfile(const char *, struct passwd *, int);
diff --git a/crypto/openssh/auth2-gss.c b/crypto/openssh/auth2-gss.c
index 0e08d889cd09..0d59b21776dc 100644
--- a/crypto/openssh/auth2-gss.c
+++ b/crypto/openssh/auth2-gss.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: auth2-gss.c,v 1.16 2007/10/29 00:52:45 dtucker Exp $ */
+/* $OpenBSD: auth2-gss.c,v 1.17 2011/03/10 02:52:57 djm Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Simon Wilkinson. All rights reserved.
@@ -102,6 +102,7 @@ userauth_gssapi(Authctxt *authctxt)
 
 	if (!present) {
 		xfree(doid);
+		authctxt->server_caused_failure = 1;
 		return (0);
 	}
 
@@ -109,6 +110,7 @@ userauth_gssapi(Authctxt *authctxt)
 		if (ctxt != NULL)
 			ssh_gssapi_delete_ctx(&ctxt);
 		xfree(doid);
+		authctxt->server_caused_failure = 1;
 		return (0);
 	}
 
diff --git a/crypto/openssh/auth2-pubkey.c b/crypto/openssh/auth2-pubkey.c
index 7d2141355994..137887ecd0dc 100644
--- a/crypto/openssh/auth2-pubkey.c
+++ b/crypto/openssh/auth2-pubkey.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: auth2-pubkey.c,v 1.27 2010/11/20 05:12:38 deraadt Exp $ */
+/* $OpenBSD: auth2-pubkey.c,v 1.29 2011/05/23 03:30:07 djm Exp $ */
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
  *
@@ -436,7 +436,7 @@ user_cert_trusted_ca(struct passwd *pw, Key *key)
 int
 user_key_allowed(struct passwd *pw, Key *key)
 {
-	int success;
+	u_int success, i;
 	char *file;
 
 	if (auth_key_is_revoked(key))
@@ -448,16 +448,13 @@ user_key_allowed(struct passwd *pw, Key *key)
 	if (success)
 		return success;
 
-	file = authorized_keys_file(pw);
-	success = user_key_allowed2(pw, key, file);
-	xfree(file);
-	if (success)
-		return success;
+	for (i = 0; !success && i < options.num_authkeys_files; i++) {
+		file = expand_authorized_keys(
+		    options.authorized_keys_files[i], pw);
+		success = user_key_allowed2(pw, key, file);
+		xfree(file);
+	}
 
-	/* try suffix "2" for backward compat, too */
-	file = authorized_keys_file2(pw);
-	success = user_key_allowed2(pw, key, file);
-	xfree(file);
 	return success;
 }
 
diff --git a/crypto/openssh/auth2.c b/crypto/openssh/auth2.c
index 827f034e4fd2..2dc9122fb51f 100644
--- a/crypto/openssh/auth2.c
+++ b/crypto/openssh/auth2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: auth2.c,v 1.122 2010/08/31 09:58:37 djm Exp $ */
+/* $OpenBSD: auth2.c,v 1.123 2011/03/10 02:52:57 djm Exp $ */
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
  *
@@ -304,6 +304,7 @@ input_userauth_request(int type, u_int32_t seq, void *ctxt)
 #endif
 
 	authctxt->postponed = 0;
+	authctxt->server_caused_failure = 0;
 
 	/* try to authenticate user */
 	m = authmethod_lookup(method);
@@ -376,7 +377,8 @@ userauth_finish(Authctxt *authctxt, int authenticated, char *method)
 	} else {
 
 		/* Allow initial try of "none" auth without failure penalty */
-		if (authctxt->attempt > 1 || strcmp(method, "none") != 0)
+		if (!authctxt->server_caused_failure &&
+		    (authctxt->attempt > 1 || strcmp(method, "none") != 0))
 			authctxt->failures++;
 		if (authctxt->failures >= options.max_authtries) {
 #ifdef SSH_AUDIT_EVENTS
diff --git a/crypto/openssh/authfd.c b/crypto/openssh/authfd.c
index c11c3f5a828b..f037e838b0ee 100644
--- a/crypto/openssh/authfd.c
+++ b/crypto/openssh/authfd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: authfd.c,v 1.84 2010/08/31 11:54:45 djm Exp $ */
+/* $OpenBSD: authfd.c,v 1.86 2011/07/06 18:09:21 tedu Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -102,6 +102,7 @@ ssh_get_authentication_socket(void)
 	if (!authsocket)
 		return -1;
 
+	bzero(&sunaddr, sizeof(sunaddr));
 	sunaddr.sun_family = AF_UNIX;
 	strlcpy(sunaddr.sun_path, authsocket, sizeof(sunaddr.sun_path));
 
@@ -110,7 +111,7 @@ ssh_get_authentication_socket(void)
 		return -1;
 
 	/* close on exec */
-	if (fcntl(sock, F_SETFD, 1) == -1) {
+	if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) {
 		close(sock);
 		return -1;
 	}
diff --git a/crypto/openssh/authfile.c b/crypto/openssh/authfile.c
index f2aec267a5a6..1d7e53cd19a1 100644
--- a/crypto/openssh/authfile.c
+++ b/crypto/openssh/authfile.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: authfile.c,v 1.87 2010/11/29 18:57:04 markus Exp $ */
+/* $OpenBSD: authfile.c,v 1.92 2011/06/14 22:49:18 markus Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -69,6 +69,8 @@
 #include "misc.h"
 #include "atomicio.h"
 
+#define MAX_KEY_FILE_SIZE	(1024 * 1024)
+
 /* Version identification string for SSH v1 identity files. */
 static const char authfile_id_string[] =
     "SSH PRIVATE KEY FILE FORMAT 1.1\n";
@@ -277,6 +279,7 @@ static Key *
 key_parse_public_rsa1(Buffer *blob, char **commentp)
 {
 	Key *pub;
+	Buffer copy;
 
 	/* Check that it is at least big enough to contain the ID string. */
 	if (buffer_len(blob) < sizeof(authfile_id_string)) {
@@ -293,31 +296,33 @@ key_parse_public_rsa1(Buffer *blob, char **commentp)
 		debug3("Incorrect RSA1 identifier");
 		return NULL;
 	}
-	buffer_consume(blob, sizeof(authfile_id_string));
+	buffer_init(&copy);
+	buffer_append(&copy, buffer_ptr(blob), buffer_len(blob));
+	buffer_consume(&copy, sizeof(authfile_id_string));
 
 	/* Skip cipher type and reserved data. */
-	(void) buffer_get_char(blob);	/* cipher type */
-	(void) buffer_get_int(blob);		/* reserved */
+	(void) buffer_get_char(&copy);		/* cipher type */
+	(void) buffer_get_int(&copy);		/* reserved */
 
 	/* Read the public key from the buffer. */
-	(void) buffer_get_int(blob);
+	(void) buffer_get_int(&copy);
 	pub = key_new(KEY_RSA1);
-	buffer_get_bignum(blob, pub->rsa->n);
-	buffer_get_bignum(blob, pub->rsa->e);
+	buffer_get_bignum(&copy, pub->rsa->n);
+	buffer_get_bignum(&copy, pub->rsa->e);
 	if (commentp)
-		*commentp = buffer_get_string(blob, NULL);
+		*commentp = buffer_get_string(&copy, NULL);
 	/* The encrypted private part is not parsed by this function. */
-	buffer_clear(blob);
+	buffer_free(&copy);
 
 	return pub;
 }
 
-/* Load the contents of a key file into a buffer */
-static int
+/* Load a key from a fd into a buffer */
+int
 key_load_file(int fd, const char *filename, Buffer *blob)
 {
+	u_char buf[1024];
 	size_t len;
-	u_char *cp;
 	struct stat st;
 
 	if (fstat(fd, &st) < 0) {
@@ -325,30 +330,45 @@ key_load_file(int fd, const char *filename, Buffer *blob)
 		    filename == NULL ? "" : filename,
 		    filename == NULL ? "" : " ",
 		    strerror(errno));
-		close(fd);
 		return 0;
 	}
-	if (st.st_size > 1*1024*1024) {
+	if ((st.st_mode & (S_IFSOCK|S_IFCHR|S_IFIFO)) == 0 &&
+	    st.st_size > MAX_KEY_FILE_SIZE) {
+ toobig:
 		error("%s: key file %.200s%stoo large", __func__,
 		    filename == NULL ? "" : filename,
 		    filename == NULL ? "" : " ");
-		close(fd);
 		return 0;
 	}
-	len = (size_t)st.st_size;		/* truncated */
-
 	buffer_init(blob);
-	cp = buffer_append_space(blob, len);
-
-	if (atomicio(read, fd, cp, len) != len) {
-		debug("%s: read from key file %.200s%sfailed: %.100s", __func__,
-		    filename == NULL ? "" : filename,
-		    filename == NULL ? "" : " ",
-		    strerror(errno));
+	for (;;) {
+		if ((len = atomicio(read, fd, buf, sizeof(buf))) == 0) {
+			if (errno == EPIPE)
+				break;
+			debug("%s: read from key file %.200s%sfailed: %.100s",
+			    __func__, filename == NULL ? "" : filename,
+			    filename == NULL ? "" : " ", strerror(errno));
+			buffer_clear(blob);
+			bzero(buf, sizeof(buf));
+			return 0;
+		}
+		buffer_append(blob, buf, len);
+		if (buffer_len(blob) > MAX_KEY_FILE_SIZE) {
+			buffer_clear(blob);
+			bzero(buf, sizeof(buf));
+			goto toobig;
+		}
+	}
+	bzero(buf, sizeof(buf));
+	if ((st.st_mode & (S_IFSOCK|S_IFCHR|S_IFIFO)) == 0 &&
+	    st.st_size != buffer_len(blob)) {
+		debug("%s: key file %.200s%schanged size while reading",
+		    __func__, filename == NULL ? "" : filename,
+		    filename == NULL ? "" : " ");
 		buffer_clear(blob);
-		close(fd);
 		return 0;
 	}
+
 	return 1;
 }
 
@@ -403,6 +423,7 @@ key_parse_private_rsa1(Buffer *blob, const char *passphrase, char **commentp)
 	CipherContext ciphercontext;
 	Cipher *cipher;
 	Key *prv = NULL;
+	Buffer copy;
 
 	/* Check that it is at least big enough to contain the ID string. */
 	if (buffer_len(blob) < sizeof(authfile_id_string)) {
@@ -419,41 +440,44 @@ key_parse_private_rsa1(Buffer *blob, const char *passphrase, char **commentp)
 		debug3("Incorrect RSA1 identifier");
 		return NULL;
 	}
-	buffer_consume(blob, sizeof(authfile_id_string));
+	buffer_init(&copy);
+	buffer_append(&copy, buffer_ptr(blob), buffer_len(blob));
+	buffer_consume(&copy, sizeof(authfile_id_string));
 
 	/* Read cipher type. */
-	cipher_type = buffer_get_char(blob);
-	(void) buffer_get_int(blob);	/* Reserved data. */
+	cipher_type = buffer_get_char(&copy);
+	(void) buffer_get_int(&copy);	/* Reserved data. */
 
 	/* Read the public key from the buffer. */
-	(void) buffer_get_int(blob);
+	(void) buffer_get_int(&copy);
 	prv = key_new_private(KEY_RSA1);
 
-	buffer_get_bignum(blob, prv->rsa->n);
-	buffer_get_bignum(blob, prv->rsa->e);
+	buffer_get_bignum(&copy, prv->rsa->n);
+	buffer_get_bignum(&copy, prv->rsa->e);
 	if (commentp)
-		*commentp = buffer_get_string(blob, NULL);
+		*commentp = buffer_get_string(&copy, NULL);
 	else
-		(void)buffer_get_string_ptr(blob, NULL);
+		(void)buffer_get_string_ptr(&copy, NULL);
 
 	/* Check that it is a supported cipher. */
 	cipher = cipher_by_number(cipher_type);
 	if (cipher == NULL) {
 		debug("Unsupported RSA1 cipher %d", cipher_type);
+		buffer_free(&copy);
 		goto fail;
 	}
 	/* Initialize space for decrypted data. */
 	buffer_init(&decrypted);
-	cp = buffer_append_space(&decrypted, buffer_len(blob));
+	cp = buffer_append_space(&decrypted, buffer_len(&copy));
 
 	/* Rest of the buffer is encrypted.  Decrypt it using the passphrase. */
 	cipher_set_key_string(&ciphercontext, cipher, passphrase,
 	    CIPHER_DECRYPT);
 	cipher_crypt(&ciphercontext, cp,
-	    buffer_ptr(blob), buffer_len(blob));
+	    buffer_ptr(&copy), buffer_len(&copy));
 	cipher_cleanup(&ciphercontext);
 	memset(&ciphercontext, 0, sizeof(ciphercontext));
-	buffer_clear(blob);
+	buffer_free(&copy);
 
 	check1 = buffer_get_char(&decrypted);
 	check2 = buffer_get_char(&decrypted);
@@ -606,7 +630,7 @@ key_perm_ok(int fd, const char *filename)
 		error("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
 		error("Permissions 0%3.3o for '%s' are too open.",
 		    (u_int)st.st_mode & 0777, filename);
-		error("It is recommended that your private key files are NOT accessible by others.");
+		error("It is required that your private key files are NOT accessible by others.");
 		error("This private key will be ignored.");
 		return 0;
 	}
@@ -626,6 +650,7 @@ key_parse_private_type(Buffer *blob, int type, const char *passphrase,
 	case KEY_UNSPEC:
 		return key_parse_private_pem(blob, type, passphrase, commentp);
 	default:
+		error("%s: cannot parse key type %d", __func__, type);
 		break;
 	}
 	return NULL;
@@ -669,12 +694,35 @@ key_load_private_type(int type, const char *filename, const char *passphrase,
 	return ret;
 }
 
+Key *
+key_parse_private(Buffer *buffer, const char *filename,
+    const char *passphrase, char **commentp)
+{
+	Key *pub, *prv;
+
+	/* it's a SSH v1 key if the public key part is readable */
+	pub = key_parse_public_rsa1(buffer, commentp);
+	if (pub == NULL) {
+		prv = key_parse_private_type(buffer, KEY_UNSPEC,
+		    passphrase, NULL);
+		/* use the filename as a comment for PEM */
+		if (commentp && prv)
+			*commentp = xstrdup(filename);
+	} else {
+		key_free(pub);
+		/* key_parse_public_rsa1() has already loaded the comment */
+		prv = key_parse_private_type(buffer, KEY_RSA1, passphrase,
+		    NULL);
+	}
+	return prv;
+}
+
 Key *
 key_load_private(const char *filename, const char *passphrase,
     char **commentp)
 {
-	Key *pub, *prv;
-	Buffer buffer, pubcopy;
+	Key *prv;
+	Buffer buffer;
 	int fd;
 
 	fd = open(filename, O_RDONLY);
@@ -697,23 +745,7 @@ key_load_private(const char *filename, const char *passphrase,
 	}
 	close(fd);
 
-	buffer_init(&pubcopy);
-	buffer_append(&pubcopy, buffer_ptr(&buffer), buffer_len(&buffer));
-	/* it's a SSH v1 key if the public key part is readable */
-	pub = key_parse_public_rsa1(&pubcopy, commentp);
-	buffer_free(&pubcopy);
-	if (pub == NULL) {
-		prv = key_parse_private_type(&buffer, KEY_UNSPEC,
-		    passphrase, NULL);
-		/* use the filename as a comment for PEM */
-		if (commentp && prv)
-			*commentp = xstrdup(filename);
-	} else {
-		key_free(pub);
-		/* key_parse_public_rsa1() has already loaded the comment */
-		prv = key_parse_private_type(&buffer, KEY_RSA1, passphrase,
-		    NULL);
-	}
+	prv = key_parse_private(&buffer, filename, passphrase, commentp);
 	buffer_free(&buffer);
 	return prv;
 }
@@ -737,13 +769,19 @@ key_try_load_public(Key *k, const char *filename, char **commentp)
 			case '\0':
 				continue;
 			}
+			/* Abort loading if this looks like a private key */
+			if (strncmp(cp, "-----BEGIN", 10) == 0)
+				break;
 			/* Skip leading whitespace. */
 			for (; *cp && (*cp == ' ' || *cp == '\t'); cp++)
 				;
 			if (*cp) {
 				if (key_read(k, &cp) == 1) {
-					if (commentp)
-						*commentp=xstrdup(filename);
+					cp[strcspn(cp, "\r\n")] = '\0';
+					if (commentp) {
+						*commentp = xstrdup(*cp ?
+						    cp : filename);
+					}
 					fclose(f);
 					return 1;
 				}
diff --git a/crypto/openssh/authfile.h b/crypto/openssh/authfile.h
index 6745dc062be1..78349beb5615 100644
--- a/crypto/openssh/authfile.h
+++ b/crypto/openssh/authfile.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: authfile.h,v 1.15 2010/08/04 05:42:47 djm Exp $ */
+/* $OpenBSD: authfile.h,v 1.16 2011/05/04 21:15:29 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -16,9 +16,11 @@
 #define AUTHFILE_H
 
 int	 key_save_private(Key *, const char *, const char *, const char *);
+int	 key_load_file(int, const char *, Buffer *);
 Key	*key_load_cert(const char *);
 Key	*key_load_public(const char *, char **);
 Key	*key_load_public_type(int, const char *, char **);
+Key	*key_parse_private(Buffer *, const char *, const char *, char **);
 Key	*key_load_private(const char *, const char *, char **);
 Key	*key_load_private_cert(int, const char *, const char *, int *);
 Key	*key_load_private_type(int, const char *, const char *, char **, int *);
diff --git a/crypto/openssh/channels.c b/crypto/openssh/channels.c
index bf7072a1c0e6..4cff8f2098cb 100644
--- a/crypto/openssh/channels.c
+++ b/crypto/openssh/channels.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: channels.c,v 1.310 2010/11/24 01:24:14 djm Exp $ */
+/* $OpenBSD: channels.c,v 1.311 2011/06/22 22:08:42 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -3638,7 +3638,7 @@ deny_input_open(int type, u_int32_t seq, void *ctxt)
  */
 void
 x11_request_forwarding_with_spoofing(int client_session_id, const char *disp,
-    const char *proto, const char *data)
+    const char *proto, const char *data, int want_reply)
 {
 	u_int data_len = (u_int) strlen(data) / 2;
 	u_int i, value;
@@ -3691,7 +3691,7 @@ x11_request_forwarding_with_spoofing(int client_session_id, const char *disp,
 
 	/* Send the request packet. */
 	if (compat20) {
-		channel_request_start(client_session_id, "x11-req", 0);
+		channel_request_start(client_session_id, "x11-req", want_reply);
 		packet_put_char(0);	/* XXX bool single connection */
 	} else {
 		packet_start(SSH_CMSG_X11_REQUEST_FORWARDING);
diff --git a/crypto/openssh/channels.h b/crypto/openssh/channels.h
index a967a7e1aa9e..6ee993ad8948 100644
--- a/crypto/openssh/channels.h
+++ b/crypto/openssh/channels.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: channels.h,v 1.104 2010/05/14 23:29:23 djm Exp $ */
+/* $OpenBSD: channels.h,v 1.105 2011/06/22 22:08:42 djm Exp $ */
 /* $FreeBSD$ */
 
 /*
@@ -278,7 +278,7 @@ int	 x11_connect_display(void);
 int	 x11_create_display_inet(int, int, int, u_int *, int **);
 void     x11_input_open(int, u_int32_t, void *);
 void	 x11_request_forwarding_with_spoofing(int, const char *, const char *,
-	     const char *);
+	     const char *, int);
 void	 deny_input_open(int, u_int32_t, void *);
 
 /* agent forwarding */
diff --git a/crypto/openssh/clientloop.c b/crypto/openssh/clientloop.c
index 05487f266d6a..ba2218a3c8f8 100644
--- a/crypto/openssh/clientloop.c
+++ b/crypto/openssh/clientloop.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: clientloop.c,v 1.231 2011/01/16 12:05:59 djm Exp $ */
+/* $OpenBSD: clientloop.c,v 1.236 2011/06/22 22:08:42 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -131,9 +131,6 @@ extern int muxserver_sock; /* XXX use mux_client_cleanup() instead */
  */
 extern char *host;
 
-/* Force TTY allocation */
-extern int force_tty_flag;
-
 /*
  * Flag to indicate that we have received a window change signal which has
  * not yet been processed.  This will cause a message indicating the new
@@ -180,7 +177,8 @@ struct escape_filter_ctx {
 /* Context for channel confirmation replies */
 struct channel_reply_ctx {
 	const char *request_type;
-	int id, do_close;
+	int id;
+	enum confirm_action action;
 };
 
 /* Global request success/failure callbacks */
@@ -266,10 +264,10 @@ static void
 set_control_persist_exit_time(void)
 {
 	if (muxserver_sock == -1 || !options.control_persist
-	    || options.control_persist_timeout == 0)
+	    || options.control_persist_timeout == 0) {
 		/* not using a ControlPersist timeout */
 		control_persist_exit_time = 0;
-	else if (channel_still_open()) {
+	} else if (channel_still_open()) {
 		/* some client connections are still open */
 		if (control_persist_exit_time > 0)
 			debug2("%s: cancel scheduled exit", __func__);
@@ -663,7 +661,7 @@ client_suspend_self(Buffer *bin, Buffer *bout, Buffer *berr)
 		atomicio(vwrite, fileno(stderr), buffer_ptr(berr),
 		    buffer_len(berr));
 
-	leave_raw_mode(force_tty_flag);
+	leave_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 
 	/*
 	 * Free (and clear) the buffer to reduce the amount of data that gets
@@ -684,7 +682,7 @@ client_suspend_self(Buffer *bin, Buffer *bout, Buffer *berr)
 	buffer_init(bout);
 	buffer_init(berr);
 
-	enter_raw_mode(force_tty_flag);
+	enter_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 }
 
 static void
@@ -743,6 +741,15 @@ client_status_confirm(int type, Channel *c, void *ctx)
 	char errmsg[256];
 	int tochan;
 
+	/*
+	 * If a TTY was explicitly requested, then a failure to allocate
+	 * one is fatal.
+	 */
+	if (cr->action == CONFIRM_TTY &&
+	    (options.request_tty == REQUEST_TTY_FORCE ||
+	    options.request_tty == REQUEST_TTY_YES))
+		cr->action = CONFIRM_CLOSE;
+
 	/* XXX supress on mux _client_ quietmode */
 	tochan = options.log_level >= SYSLOG_LEVEL_ERROR &&
 	    c->ctl_chan != -1 && c->extended_usage == CHAN_EXTENDED_WRITE;
@@ -760,14 +767,27 @@ client_status_confirm(int type, Channel *c, void *ctx)
 			    cr->request_type, c->self);
 		}
 		/* If error occurred on primary session channel, then exit */
-		if (cr->do_close && c->self == session_ident)
+		if (cr->action == CONFIRM_CLOSE && c->self == session_ident)
 			fatal("%s", errmsg);
-		/* If error occurred on mux client, append to their stderr */
-		if (tochan)
-			buffer_append(&c->extended, errmsg, strlen(errmsg));
-		else
+		/*
+		 * If error occurred on mux client, append to
+		 * their stderr.
+		 */
+		if (tochan) {
+			buffer_append(&c->extended, errmsg,
+			    strlen(errmsg));
+		} else
 			error("%s", errmsg);
-		if (cr->do_close) {
+		if (cr->action == CONFIRM_TTY) {
+			/*
+			 * If a TTY allocation error occurred, then arrange
+			 * for the correct TTY to leave raw mode.
+			 */
+			if (c->self == session_ident)
+				leave_raw_mode(0);
+			else
+				mux_tty_alloc_failed(c);
+		} else if (cr->action == CONFIRM_CLOSE) {
 			chan_read_failed(c);
 			chan_write_failed(c);
 		}
@@ -781,13 +801,14 @@ client_abandon_status_confirm(Channel *c, void *ctx)
 	xfree(ctx);
 }
 
-static void
-client_expect_confirm(int id, const char *request, int do_close)
+void
+client_expect_confirm(int id, const char *request,
+    enum confirm_action action)
 {
 	struct channel_reply_ctx *cr = xmalloc(sizeof(*cr));
 
 	cr->request_type = request;
-	cr->do_close = do_close;
+	cr->action = action;
 
 	channel_register_status_confirm(id, client_status_confirm,
 	    client_abandon_status_confirm, cr);
@@ -827,7 +848,7 @@ process_cmdline(void)
 	bzero(&fwd, sizeof(fwd));
 	fwd.listen_host = fwd.connect_host = NULL;
 
-	leave_raw_mode(force_tty_flag);
+	leave_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 	handler = signal(SIGINT, SIG_IGN);
 	cmd = s = read_passphrase("\r\nssh> ", RP_ECHO);
 	if (s == NULL)
@@ -931,7 +952,7 @@ process_cmdline(void)
 
 out:
 	signal(SIGINT, handler);
-	enter_raw_mode(force_tty_flag);
+	enter_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 	if (cmd)
 		xfree(cmd);
 	if (fwd.listen_host != NULL)
@@ -1050,7 +1071,8 @@ process_escapes(Channel *c, Buffer *bin, Buffer *bout, Buffer *berr,
 				 * more new connections).
 				 */
 				/* Restore tty modes. */
-				leave_raw_mode(force_tty_flag);
+				leave_raw_mode(
+				    options.request_tty == REQUEST_TTY_FORCE);
 
 				/* Stop listening for new connections. */
 				channel_stop_listening();
@@ -1345,7 +1367,7 @@ client_channel_closed(int id, void *arg)
 {
 	channel_cancel_cleanup(id);
 	session_closed = 1;
-	leave_raw_mode(force_tty_flag);
+	leave_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 }
 
 /*
@@ -1416,18 +1438,21 @@ client_loop(int have_pty, int escape_char_arg, int ssh2_chan_id)
 	signal(SIGWINCH, window_change_handler);
 
 	if (have_pty)
-		enter_raw_mode(force_tty_flag);
+		enter_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 
 	if (compat20) {
 		session_ident = ssh2_chan_id;
-		if (escape_char_arg != SSH_ESCAPECHAR_NONE)
-			channel_register_filter(session_ident,
-			    client_simple_escape_filter, NULL,
-			    client_filter_cleanup,
-			    client_new_escape_filter_ctx(escape_char_arg));
-		if (session_ident != -1)
+		if (session_ident != -1) {
+			if (escape_char_arg != SSH_ESCAPECHAR_NONE) {
+				channel_register_filter(session_ident,
+				    client_simple_escape_filter, NULL,
+				    client_filter_cleanup,
+				    client_new_escape_filter_ctx(
+				    escape_char_arg));
+			}
 			channel_register_cleanup(session_ident,
 			    client_channel_closed, 0);
+		}
 	} else {
 		/* Check if we should immediately send eof on stdin. */
 		client_check_initial_eof_on_stdin();
@@ -1557,7 +1582,7 @@ client_loop(int have_pty, int escape_char_arg, int ssh2_chan_id)
 	channel_free_all();
 
 	if (have_pty)
-		leave_raw_mode(force_tty_flag);
+		leave_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 
 	/* restore blocking io */
 	if (!isatty(fileno(stdin)))
@@ -2000,7 +2025,7 @@ client_session2_setup(int id, int want_tty, int want_subsystem,
 			memset(&ws, 0, sizeof(ws));
 
 		channel_request_start(id, "pty-req", 1);
-		client_expect_confirm(id, "PTY allocation", 1);
+		client_expect_confirm(id, "PTY allocation", CONFIRM_TTY);
 		packet_put_cstring(term != NULL ? term : "");
 		packet_put_int((u_int)ws.ws_col);
 		packet_put_int((u_int)ws.ws_row);
@@ -2059,18 +2084,18 @@ client_session2_setup(int id, int want_tty, int want_subsystem,
 			debug("Sending subsystem: %.*s",
 			    len, (u_char*)buffer_ptr(cmd));
 			channel_request_start(id, "subsystem", 1);
-			client_expect_confirm(id, "subsystem", 1);
+			client_expect_confirm(id, "subsystem", CONFIRM_CLOSE);
 		} else {
 			debug("Sending command: %.*s",
 			    len, (u_char*)buffer_ptr(cmd));
 			channel_request_start(id, "exec", 1);
-			client_expect_confirm(id, "exec", 1);
+			client_expect_confirm(id, "exec", CONFIRM_CLOSE);
 		}
 		packet_put_string(buffer_ptr(cmd), buffer_len(cmd));
 		packet_send();
 	} else {
 		channel_request_start(id, "shell", 1);
-		client_expect_confirm(id, "shell", 1);
+		client_expect_confirm(id, "shell", CONFIRM_CLOSE);
 		packet_send();
 	}
 }
@@ -2140,11 +2165,26 @@ client_init_dispatch(void)
 		client_init_dispatch_15();
 }
 
+void
+client_stop_mux(void)
+{
+	if (options.control_path != NULL && muxserver_sock != -1)
+		unlink(options.control_path);
+	/*
+	 * If we are in persist mode, signal that we should close when all
+	 * active channels are closed.
+	 */
+	if (options.control_persist) {
+		session_closed = 1;
+		setproctitle("[stopped mux]");
+	}
+}
+
 /* client specific fatal cleanup */
 void
 cleanup_exit(int i)
 {
-	leave_raw_mode(force_tty_flag);
+	leave_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 	leave_non_blocking();
 	if (options.control_path != NULL && muxserver_sock != -1)
 		unlink(options.control_path);
diff --git a/crypto/openssh/clientloop.h b/crypto/openssh/clientloop.h
index 52115db6ec8d..a259b5e14b21 100644
--- a/crypto/openssh/clientloop.h
+++ b/crypto/openssh/clientloop.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: clientloop.h,v 1.25 2010/06/25 23:15:36 djm Exp $ */
+/* $OpenBSD: clientloop.h,v 1.28 2011/06/22 22:08:42 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -45,6 +45,7 @@ void	 client_global_request_reply_fwd(int, u_int32_t, void *);
 void	 client_session2_setup(int, int, int, const char *, struct termios *,
 	    int, Buffer *, char **);
 int	 client_request_tun_fwd(int, int, int);
+void	 client_stop_mux(void);
 
 /* Escape filter for protocol 2 sessions */
 void	*client_new_escape_filter_ctx(int);
@@ -55,6 +56,10 @@ int	 client_simple_escape_filter(Channel *, char *, int);
 typedef void global_confirm_cb(int, u_int32_t seq, void *);
 void	 client_register_global_confirm(global_confirm_cb *, void *);
 
+/* Channel request confirmation callbacks */
+enum confirm_action { CONFIRM_WARN = 0, CONFIRM_CLOSE, CONFIRM_TTY };
+void client_expect_confirm(int, const char *, enum confirm_action);
+
 /* Multiplexing protocol version */
 #define SSHMUX_VER			4
 
@@ -64,7 +69,10 @@ void	 client_register_global_confirm(global_confirm_cb *, void *);
 #define SSHMUX_COMMAND_TERMINATE	3	/* Ask master to exit */
 #define SSHMUX_COMMAND_STDIO_FWD	4	/* Open stdio fwd (ssh -W) */
 #define SSHMUX_COMMAND_FORWARD		5	/* Forward only, no command */
+#define SSHMUX_COMMAND_STOP		6	/* Disable mux but not conn */
 
 void	muxserver_listen(void);
 void	muxclient(const char *);
 void	mux_exit_message(Channel *, int);
+void	mux_tty_alloc_failed(Channel *);
+
diff --git a/crypto/openssh/config.guess b/crypto/openssh/config.guess
index c2246a4f7f4c..78553c4ea07e 100755
--- a/crypto/openssh/config.guess
+++ b/crypto/openssh/config.guess
@@ -1,10 +1,10 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
 #   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
-#   Free Software Foundation, Inc.
+#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+#   2011 Free Software Foundation, Inc.
 
-timestamp='2009-12-30'
+timestamp='2011-01-23'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -57,7 +57,7 @@ GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free
+2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free
 Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
@@ -270,7 +270,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	# A Xn.n version is an unreleased experimental baselevel.
 	# 1.2 uses "1.2" for uname -r.
 	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
-	exit ;;
+	# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
+	exitcode=$?
+	trap '' 0
+	exit $exitcode ;;
     Alpha\ *:Windows_NT*:*)
 	# How do we know it's Interix rather than the generic POSIX subsystem?
 	# Should we change UNAME_MACHINE based on the output of uname instead
@@ -552,7 +555,7 @@ EOF
 		echo rs6000-ibm-aix3.2
 	fi
 	exit ;;
-    *:AIX:*:[456])
+    *:AIX:*:[4567])
 	IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
 	if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then
 		IBM_ARCH=rs6000
@@ -968,6 +971,9 @@ EOF
     sparc:Linux:*:* | sparc64:Linux:*:*)
 	echo ${UNAME_MACHINE}-unknown-linux-gnu
 	exit ;;
+    tile*:Linux:*:*)
+	echo ${UNAME_MACHINE}-tilera-linux-gnu
+	exit ;;
     vax:Linux:*:*)
 	echo ${UNAME_MACHINE}-dec-linux-gnu
 	exit ;;
@@ -1231,6 +1237,9 @@ EOF
     *:QNX:*:4*)
 	echo i386-pc-qnx
 	exit ;;
+    NEO-?:NONSTOP_KERNEL:*:*)
+	echo neo-tandem-nsk${UNAME_RELEASE}
+	exit ;;
     NSE-?:NONSTOP_KERNEL:*:*)
 	echo nse-tandem-nsk${UNAME_RELEASE}
 	exit ;;
diff --git a/crypto/openssh/config.h b/crypto/openssh/config.h
index de529cb60217..49f4ec3e2dac 100644
--- a/crypto/openssh/config.h
+++ b/crypto/openssh/config.h
@@ -126,9 +126,6 @@
 /* Enable for PKCS#11 support */
 #define ENABLE_PKCS11 /**/
 
-/* Builtin PRNG command timeout */
-#define ENTROPY_TIMEOUT_MSEC 200
-
 /* File names may not contain backslash characters */
 /* #undef FILESYSTEM_NO_BACKSLASH */
 
@@ -754,6 +751,9 @@
 /* Define to 1 if you have the `recvmsg' function. */
 #define HAVE_RECVMSG 1
 
+/* sys/resource.h has RLIMIT_NPROC */
+#define HAVE_RLIMIT_NPROC /**/
+
 /* Define to 1 if you have the <rpc/types.h> header file. */
 #define HAVE_RPC_TYPES_H 1
 
@@ -766,6 +766,12 @@
 /* Define to 1 if you have the `RSA_get_default_method' function. */
 #define HAVE_RSA_GET_DEFAULT_METHOD 1
 
+/* Define to 1 if you have the <sandbox.h> header file. */
+/* #undef HAVE_SANDBOX_H */
+
+/* Define to 1 if you have the `sandbox_init' function. */
+/* #undef HAVE_SANDBOX_INIT */
+
 /* define if you have sa_family_t data type */
 #define HAVE_SA_FAMILY_T 1
 
@@ -1263,7 +1269,7 @@
    from environment and PATH */
 #define LOGIN_PROGRAM_FALLBACK "/usr/bin/login"
 
-/* Set this to your mail directory if you don't have maillock.h */
+/* Set this to your mail directory if you do not have _PATH_MAILDIR */
 /* #undef MAIL_DIRECTORY */
 
 /* Define on *nto-qnx systems */
@@ -1336,6 +1342,18 @@
 /* read(1) can return 0 for a non-closed fd */
 /* #undef PTY_ZEROREAD */
 
+/* Sandbox using Darwin sandbox_init(3) */
+/* #undef SANDBOX_DARWIN */
+
+/* no privsep sandboxing */
+/* #undef SANDBOX_NULL */
+
+/* Sandbox using setrlimit(2) */
+#define SANDBOX_RLIMIT 1
+
+/* Sandbox using systrace(4) */
+/* #undef SANDBOX_SYSTRACE */
+
 /* Define if your platform breaks doing a seteuid before a setuid */
 /* #undef SETEUID_BREAKS_SETUID */
 
@@ -1469,10 +1487,14 @@
 
 /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
    significant byte first (like Motorola and SPARC, unlike Intel). */
-#if defined __BIG_ENDIAN__
-# define WORDS_BIGENDIAN 1
-#elif ! defined __LITTLE_ENDIAN__
-/* # undef WORDS_BIGENDIAN */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+#  define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+/* #  undef WORDS_BIGENDIAN */
+# endif
 #endif
 
 /* Define if xauth is found in your path */
diff --git a/crypto/openssh/config.h.in b/crypto/openssh/config.h.in
index e5c9379c85f9..baf0011a74b7 100644
--- a/crypto/openssh/config.h.in
+++ b/crypto/openssh/config.h.in
@@ -1,5 +1,8 @@
 /* config.h.in.  Generated from configure.ac by autoheader.  */
 
+/* Define if building universal (internal helper macro) */
+#undef AC_APPLE_UNIVERSAL_BUILD
+
 /* Define if you have a getaddrinfo that fails for the all-zeros IPv6 address
    */
 #undef AIX_GETNAMEINFO_HACK
@@ -122,9 +125,6 @@
 /* Enable for PKCS#11 support */
 #undef ENABLE_PKCS11
 
-/* Builtin PRNG command timeout */
-#undef ENTROPY_TIMEOUT_MSEC
-
 /* File names may not contain backslash characters */
 #undef FILESYSTEM_NO_BACKSLASH
 
@@ -750,6 +750,9 @@
 /* Define to 1 if you have the `recvmsg' function. */
 #undef HAVE_RECVMSG
 
+/* sys/resource.h has RLIMIT_NPROC */
+#undef HAVE_RLIMIT_NPROC
+
 /* Define to 1 if you have the <rpc/types.h> header file. */
 #undef HAVE_RPC_TYPES_H
 
@@ -762,6 +765,12 @@
 /* Define to 1 if you have the `RSA_get_default_method' function. */
 #undef HAVE_RSA_GET_DEFAULT_METHOD
 
+/* Define to 1 if you have the <sandbox.h> header file. */
+#undef HAVE_SANDBOX_H
+
+/* Define to 1 if you have the `sandbox_init' function. */
+#undef HAVE_SANDBOX_INIT
+
 /* define if you have sa_family_t data type */
 #undef HAVE_SA_FAMILY_T
 
@@ -948,13 +957,13 @@
 /* define if you have struct sockaddr_in6 data type */
 #undef HAVE_STRUCT_SOCKADDR_IN6
 
-/* Define to 1 if `sin6_scope_id' is member of `struct sockaddr_in6'. */
+/* Define to 1 if `sin6_scope_id' is a member of `struct sockaddr_in6'. */
 #undef HAVE_STRUCT_SOCKADDR_IN6_SIN6_SCOPE_ID
 
 /* define if you have struct sockaddr_storage data type */
 #undef HAVE_STRUCT_SOCKADDR_STORAGE
 
-/* Define to 1 if `st_blksize' is member of `struct stat'. */
+/* Define to 1 if `st_blksize' is a member of `struct stat'. */
 #undef HAVE_STRUCT_STAT_ST_BLKSIZE
 
 /* Define to 1 if the system has the type `struct timespec'. */
@@ -1259,7 +1268,7 @@
    from environment and PATH */
 #undef LOGIN_PROGRAM_FALLBACK
 
-/* Set this to your mail directory if you don't have maillock.h */
+/* Set this to your mail directory if you do not have _PATH_MAILDIR */
 #undef MAIL_DIRECTORY
 
 /* Define on *nto-qnx systems */
@@ -1307,6 +1316,9 @@
 /* Define to the one symbol short name of this package. */
 #undef PACKAGE_TARNAME
 
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION
 
@@ -1329,6 +1341,18 @@
 /* read(1) can return 0 for a non-closed fd */
 #undef PTY_ZEROREAD
 
+/* Sandbox using Darwin sandbox_init(3) */
+#undef SANDBOX_DARWIN
+
+/* no privsep sandboxing */
+#undef SANDBOX_NULL
+
+/* Sandbox using setrlimit(2) */
+#undef SANDBOX_RLIMIT
+
+/* Sandbox using systrace(4) */
+#undef SANDBOX_SYSTRACE
+
 /* Define if your platform breaks doing a seteuid before a setuid */
 #undef SETEUID_BREAKS_SETUID
 
@@ -1460,9 +1484,17 @@
 /* Define if you want SELinux support. */
 #undef WITH_SELINUX
 
-/* Define to 1 if your processor stores words with the most significant byte
-   first (like Motorola and SPARC, unlike Intel and VAX). */
-#undef WORDS_BIGENDIAN
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+   significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+#  define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+#  undef WORDS_BIGENDIAN
+# endif
+#endif
 
 /* Define if xauth is found in your path */
 #undef XAUTH_PATH
diff --git a/crypto/openssh/defines.h b/crypto/openssh/defines.h
index b67cf467dcf1..e4ccc5407bda 100644
--- a/crypto/openssh/defines.h
+++ b/crypto/openssh/defines.h
@@ -25,7 +25,7 @@
 #ifndef _DEFINES_H
 #define _DEFINES_H
 
-/* $Id: defines.h,v 1.165 2011/05/05 01:19:15 djm Exp $ */
+/* $Id: defines.h,v 1.167 2011/06/03 01:17:49 tim Exp $ */
 
 
 /* Constants */
@@ -131,6 +131,10 @@ enum
 # define O_NONBLOCK      00004	/* Non Blocking Open */
 #endif
 
+#ifndef S_IFSOCK
+# define S_IFSOCK 0
+#endif /* S_IFSOCK */
+
 #ifndef S_ISDIR
 # define S_ISDIR(mode)	(((mode) & (_S_IFMT)) == (_S_IFDIR))
 #endif /* S_ISDIR */
@@ -385,18 +389,15 @@ struct winsize {
 # define _PATH_DEVNULL "/dev/null"
 #endif
 
-#ifndef MAIL_DIRECTORY
-# define MAIL_DIRECTORY "/var/spool/mail"
-#endif
+/* user may have set a different path */
+#if defined(_PATH_MAILDIR) && defined(MAIL_DIRECTORY)
+# undef _PATH_MAILDIR MAILDIR
+#endif /* defined(_PATH_MAILDIR) && defined(MAIL_DIRECTORY) */
 
-#ifndef MAILDIR
-# define MAILDIR MAIL_DIRECTORY
+#ifdef MAIL_DIRECTORY
+# define _PATH_MAILDIR MAIL_DIRECTORY
 #endif
 
-#if !defined(_PATH_MAILDIR) && defined(MAILDIR)
-# define _PATH_MAILDIR MAILDIR
-#endif /* !defined(_PATH_MAILDIR) && defined(MAILDIR) */
-
 #ifndef _PATH_NOLOGIN
 # define _PATH_NOLOGIN "/etc/nologin"
 #endif
diff --git a/crypto/openssh/entropy.c b/crypto/openssh/entropy.c
index a650351da93d..2d6d3ec5251c 100644
--- a/crypto/openssh/entropy.c
+++ b/crypto/openssh/entropy.c
@@ -25,19 +25,19 @@
 #include "includes.h"
 
 #include <sys/types.h>
-#include <sys/wait.h>
-
-#ifdef HAVE_SYS_STAT_H
-# include <sys/stat.h>
+#include <sys/socket.h>
+#ifdef HAVE_SYS_UN_H
+# include <sys/un.h>
 #endif
 
-#ifdef HAVE_FCNTL_H
-# include <fcntl.h>
-#endif
-#include <stdarg.h>
-#include <string.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <errno.h>
 #include <signal.h>
+#include <string.h>
 #include <unistd.h>
+#include <stddef.h> /* for offsetof */
 
 #include <openssl/rand.h>
 #include <openssl/crypto.h>
@@ -54,119 +54,128 @@
 /*
  * Portable OpenSSH PRNG seeding:
  * If OpenSSL has not "internally seeded" itself (e.g. pulled data from
- * /dev/random), then we execute a "ssh-rand-helper" program which
- * collects entropy and writes it to stdout. The child program must
- * write at least RANDOM_SEED_SIZE bytes. The child is run with stderr
- * attached, so error/debugging output should be visible.
- *
- * XXX: we should tell the child how many bytes we need.
+ * /dev/random), then collect RANDOM_SEED_SIZE bytes of randomness from
+ * PRNGd.
  */
-
 #ifndef OPENSSL_PRNG_ONLY
+
 #define RANDOM_SEED_SIZE 48
-static uid_t original_uid, original_euid;
-#endif
 
-void
-seed_rng(void)
+/*
+ * Collect 'len' bytes of entropy into 'buf' from PRNGD/EGD daemon
+ * listening either on 'tcp_port', or via Unix domain socket at *
+ * 'socket_path'.
+ * Either a non-zero tcp_port or a non-null socket_path must be
+ * supplied.
+ * Returns 0 on success, -1 on error
+ */
+int
+get_random_bytes_prngd(unsigned char *buf, int len,
+    unsigned short tcp_port, char *socket_path)
 {
-#ifndef OPENSSL_PRNG_ONLY
-	int devnull;
-	int p[2];
-	pid_t pid;
-	int ret;
-	unsigned char buf[RANDOM_SEED_SIZE];
-	mysig_t old_sigchld;
+	int fd, addr_len, rval, errors;
+	u_char msg[2];
+	struct sockaddr_storage addr;
+	struct sockaddr_in *addr_in = (struct sockaddr_in *)&addr;
+	struct sockaddr_un *addr_un = (struct sockaddr_un *)&addr;
+	mysig_t old_sigpipe;
 
-	if (RAND_status() == 1) {
-		debug3("RNG is ready, skipping seeding");
-		return;
+	/* Sanity checks */
+	if (socket_path == NULL && tcp_port == 0)
+		fatal("You must specify a port or a socket");
+	if (socket_path != NULL &&
+	    strlen(socket_path) >= sizeof(addr_un->sun_path))
+		fatal("Random pool path is too long");
+	if (len <= 0 || len > 255)
+		fatal("Too many bytes (%d) to read from PRNGD", len);
+
+	memset(&addr, '\0', sizeof(addr));
+
+	if (tcp_port != 0) {
+		addr_in->sin_family = AF_INET;
+		addr_in->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+		addr_in->sin_port = htons(tcp_port);
+		addr_len = sizeof(*addr_in);
+	} else {
+		addr_un->sun_family = AF_UNIX;
+		strlcpy(addr_un->sun_path, socket_path,
+		    sizeof(addr_un->sun_path));
+		addr_len = offsetof(struct sockaddr_un, sun_path) +
+		    strlen(socket_path) + 1;
 	}
 
-	debug3("Seeding PRNG from %s", SSH_RAND_HELPER);
+	old_sigpipe = mysignal(SIGPIPE, SIG_IGN);
 
-	if ((devnull = open("/dev/null", O_RDWR)) == -1)
-		fatal("Couldn't open /dev/null: %s", strerror(errno));
-	if (pipe(p) == -1)
-		fatal("pipe: %s", strerror(errno));
+	errors = 0;
+	rval = -1;
+reopen:
+	fd = socket(addr.ss_family, SOCK_STREAM, 0);
+	if (fd == -1) {
+		error("Couldn't create socket: %s", strerror(errno));
+		goto done;
+	}
 
-	old_sigchld = signal(SIGCHLD, SIG_DFL);
-	if ((pid = fork()) == -1)
-		fatal("Couldn't fork: %s", strerror(errno));
-	if (pid == 0) {
-		dup2(devnull, STDIN_FILENO);
-		dup2(p[1], STDOUT_FILENO);
-		/* Keep stderr open for errors */
-		close(p[0]);
-		close(p[1]);
-		close(devnull);
-		closefrom(STDERR_FILENO + 1);
-
-		if (original_uid != original_euid &&
-		    ( seteuid(getuid()) == -1 ||
-		      setuid(original_uid) == -1) ) {
-			fprintf(stderr, "(rand child) setuid(%li): %s\n",
-			    (long int)original_uid, strerror(errno));
-			_exit(1);
+	if (connect(fd, (struct sockaddr*)&addr, addr_len) == -1) {
+		if (tcp_port != 0) {
+			error("Couldn't connect to PRNGD port %d: %s",
+			    tcp_port, strerror(errno));
+		} else {
+			error("Couldn't connect to PRNGD socket \"%s\": %s",
+			    addr_un->sun_path, strerror(errno));
 		}
-
-		execl(SSH_RAND_HELPER, "ssh-rand-helper", NULL);
-		fprintf(stderr, "(rand child) Couldn't exec '%s': %s\n",
-		    SSH_RAND_HELPER, strerror(errno));
-		_exit(1);
+		goto done;
 	}
 
-	close(devnull);
-	close(p[1]);
+	/* Send blocking read request to PRNGD */
+	msg[0] = 0x02;
+	msg[1] = len;
 
-	memset(buf, '\0', sizeof(buf));
-	ret = atomicio(read, p[0], buf, sizeof(buf));
-	if (ret == -1)
-		fatal("Couldn't read from ssh-rand-helper: %s",
+	if (atomicio(vwrite, fd, msg, sizeof(msg)) != sizeof(msg)) {
+		if (errno == EPIPE && errors < 10) {
+			close(fd);
+			errors++;
+			goto reopen;
+		}
+		error("Couldn't write to PRNGD socket: %s",
 		    strerror(errno));
-	if (ret != sizeof(buf))
-		fatal("ssh-rand-helper child produced insufficient data");
+		goto done;
+	}
 
-	close(p[0]);
-
-	if (waitpid(pid, &ret, 0) == -1)
-		fatal("Couldn't wait for ssh-rand-helper completion: %s",
+	if (atomicio(read, fd, buf, len) != (size_t)len) {
+		if (errno == EPIPE && errors < 10) {
+			close(fd);
+			errors++;
+			goto reopen;
+		}
+		error("Couldn't read from PRNGD socket: %s",
 		    strerror(errno));
-	signal(SIGCHLD, old_sigchld);
+		goto done;
+	}
 
-	/* We don't mind if the child exits upon a SIGPIPE */
-	if (!WIFEXITED(ret) &&
-	    (!WIFSIGNALED(ret) || WTERMSIG(ret) != SIGPIPE))
-		fatal("ssh-rand-helper terminated abnormally");
-	if (WEXITSTATUS(ret) != 0)
-		fatal("ssh-rand-helper exit with exit status %d", ret);
-
-	RAND_add(buf, sizeof(buf), sizeof(buf));
-	memset(buf, '\0', sizeof(buf));
-
-#endif /* OPENSSL_PRNG_ONLY */
-	if (RAND_status() != 1)
-		fatal("PRNG is not seeded");
+	rval = 0;
+done:
+	mysignal(SIGPIPE, old_sigpipe);
+	if (fd != -1)
+		close(fd);
+	return rval;
 }
 
-void
-init_rng(void)
+static int
+seed_from_prngd(unsigned char *buf, size_t bytes)
 {
-	/*
-	 * OpenSSL version numbers: MNNFFPPS: major minor fix patch status
-	 * We match major, minor, fix and status (not patch)
-	 */
-	if ((SSLeay() ^ OPENSSL_VERSION_NUMBER) & ~0xff0L)
-		fatal("OpenSSL version mismatch. Built against %lx, you "
-		    "have %lx", (u_long)OPENSSL_VERSION_NUMBER, SSLeay());
-
-#ifndef OPENSSL_PRNG_ONLY
-	original_uid = getuid();
-	original_euid = geteuid();
+#ifdef PRNGD_PORT
+	debug("trying egd/prngd port %d", PRNGD_PORT);
+	if (get_random_bytes_prngd(buf, bytes, PRNGD_PORT, NULL) == 0)
+		return 0;
 #endif
+#ifdef PRNGD_SOCKET
+	debug("trying egd/prngd socket %s", PRNGD_SOCKET);
+	if (get_random_bytes_prngd(buf, bytes, 0, PRNGD_SOCKET) == 0)
+		return 0;
+#endif
+	return -1;
 }
 
-#ifndef OPENSSL_PRNG_ONLY
 void
 rexec_send_rng_seed(Buffer *m)
 {
@@ -192,4 +201,34 @@ rexec_recv_rng_seed(Buffer *m)
 		RAND_add(buf, len, len);
 	}
 }
+#endif /* OPENSSL_PRNG_ONLY */
+
+void
+seed_rng(void)
+{
+#ifndef OPENSSL_PRNG_ONLY
+	unsigned char buf[RANDOM_SEED_SIZE];
 #endif
+	/*
+	 * OpenSSL version numbers: MNNFFPPS: major minor fix patch status
+	 * We match major, minor, fix and status (not patch)
+	 */
+	if ((SSLeay() ^ OPENSSL_VERSION_NUMBER) & ~0xff0L)
+		fatal("OpenSSL version mismatch. Built against %lx, you "
+		    "have %lx", (u_long)OPENSSL_VERSION_NUMBER, SSLeay());
+
+#ifndef OPENSSL_PRNG_ONLY
+	if (RAND_status() == 1) {
+		debug3("RNG is ready, skipping seeding");
+		return;
+	}
+
+	if (seed_from_prngd(buf, sizeof(buf)) == -1)
+		fatal("Could not obtain seed from PRNGd");
+	RAND_add(buf, sizeof(buf), sizeof(buf));
+	memset(buf, '\0', sizeof(buf));
+
+#endif /* OPENSSL_PRNG_ONLY */
+	if (RAND_status() != 1)
+		fatal("PRNG is not seeded");
+}
diff --git a/crypto/openssh/gss-serv.c b/crypto/openssh/gss-serv.c
index 2ec7ea19c2da..c719c1306424 100644
--- a/crypto/openssh/gss-serv.c
+++ b/crypto/openssh/gss-serv.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: gss-serv.c,v 1.22 2008/05/08 12:02:23 djm Exp $ */
+/* $OpenBSD: gss-serv.c,v 1.23 2011/08/01 19:18:15 markus Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Simon Wilkinson. All rights reserved.
@@ -229,6 +229,8 @@ ssh_gssapi_parse_ename(Gssctxt *ctx, gss_buffer_t ename, gss_buffer_t name)
 	name->length = get_u32(tok+offset);
 	offset += 4;
 
+	if (UINT_MAX - offset < name->length)
+		return GSS_S_FAILURE;
 	if (ename->length < offset+name->length)
 		return GSS_S_FAILURE;
 
diff --git a/crypto/openssh/key.c b/crypto/openssh/key.c
index e3a305e66eae..498cf5a60f9b 100644
--- a/crypto/openssh/key.c
+++ b/crypto/openssh/key.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: key.c,v 1.96 2011/02/04 00:44:21 djm Exp $ */
+/* $OpenBSD: key.c,v 1.97 2011/05/17 07:13:31 djm Exp $ */
 /*
  * read_bignum():
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -1817,6 +1817,9 @@ key_to_certified(Key *k, int legacy)
 		k->type = legacy ? KEY_DSA_CERT_V00 : KEY_DSA_CERT;
 		return 0;
 	case KEY_ECDSA:
+		if (legacy)
+			fatal("%s: legacy ECDSA certificates are not supported",
+			    __func__);
 		k->cert = cert_new();
 		k->type = KEY_ECDSA_CERT;
 		return 0;
diff --git a/crypto/openssh/log.c b/crypto/openssh/log.c
index 4a8239b931de..ad5a10b4785d 100644
--- a/crypto/openssh/log.c
+++ b/crypto/openssh/log.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: log.c,v 1.41 2008/06/10 04:50:25 dtucker Exp $ */
+/* $OpenBSD: log.c,v 1.42 2011/06/17 21:44:30 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -56,6 +56,8 @@ static LogLevel log_level = SYSLOG_LEVEL_INFO;
 static int log_on_stderr = 1;
 static int log_facility = LOG_AUTH;
 static char *argv0;
+static log_handler_fn *log_handler;
+static void *log_handler_ctx;
 
 extern char *__progname;
 
@@ -260,6 +262,9 @@ log_init(char *av0, LogLevel level, SyslogFacility facility, int on_stderr)
 		exit(1);
 	}
 
+	log_handler = NULL;
+	log_handler_ctx = NULL;
+
 	log_on_stderr = on_stderr;
 	if (on_stderr)
 		return;
@@ -326,6 +331,23 @@ log_init(char *av0, LogLevel level, SyslogFacility facility, int on_stderr)
 
 #define MSGBUFSIZ 1024
 
+void
+set_log_handler(log_handler_fn *handler, void *ctx)
+{
+	log_handler = handler;
+	log_handler_ctx = ctx;
+}
+
+void
+do_log2(LogLevel level, const char *fmt,...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	do_log(level, fmt, args);
+	va_end(args);
+}
+
 void
 do_log(LogLevel level, const char *fmt, va_list args)
 {
@@ -337,6 +359,7 @@ do_log(LogLevel level, const char *fmt, va_list args)
 	char *txt = NULL;
 	int pri = LOG_INFO;
 	int saved_errno = errno;
+	log_handler_fn *tmp_handler;
 
 	if (level > log_level)
 		return;
@@ -375,7 +398,7 @@ do_log(LogLevel level, const char *fmt, va_list args)
 		pri = LOG_ERR;
 		break;
 	}
-	if (txt != NULL) {
+	if (txt != NULL && log_handler == NULL) {
 		snprintf(fmtbuf, sizeof(fmtbuf), "%s: %s", txt, fmt);
 		vsnprintf(msgbuf, sizeof(msgbuf), fmtbuf, args);
 	} else {
@@ -383,7 +406,13 @@ do_log(LogLevel level, const char *fmt, va_list args)
 	}
 	strnvis(fmtbuf, msgbuf, sizeof(fmtbuf),
 	    log_on_stderr ? LOG_STDERR_VIS : LOG_SYSLOG_VIS);
-	if (log_on_stderr) {
+	if (log_handler != NULL) {
+		/* Avoid recursion */
+		tmp_handler = log_handler;
+		log_handler = NULL;
+		tmp_handler(level, fmtbuf, log_handler_ctx);
+		log_handler = tmp_handler;
+	} else if (log_on_stderr) {
 		snprintf(msgbuf, sizeof msgbuf, "%s\r\n", fmtbuf);
 		write(STDERR_FILENO, msgbuf, strlen(msgbuf));
 	} else {
diff --git a/crypto/openssh/log.h b/crypto/openssh/log.h
index 650582791113..1b8d2142b58b 100644
--- a/crypto/openssh/log.h
+++ b/crypto/openssh/log.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: log.h,v 1.17 2008/06/13 00:12:02 dtucker Exp $ */
+/* $OpenBSD: log.h,v 1.18 2011/06/17 21:44:30 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -46,6 +46,8 @@ typedef enum {
 	SYSLOG_LEVEL_NOT_SET = -1
 }       LogLevel;
 
+typedef void (log_handler_fn)(LogLevel, const char *, void *);
+
 void     log_init(char *, LogLevel, SyslogFacility, int);
 
 SyslogFacility	log_facility_number(char *);
@@ -64,6 +66,10 @@ void     debug(const char *, ...) __attribute__((format(printf, 1, 2)));
 void     debug2(const char *, ...) __attribute__((format(printf, 1, 2)));
 void     debug3(const char *, ...) __attribute__((format(printf, 1, 2)));
 
+
+void	 set_log_handler(log_handler_fn *, void *);
+void	 do_log2(LogLevel, const char *, ...)
+    __attribute__((format(printf, 2, 3)));
 void	 do_log(LogLevel, const char *, va_list);
 void	 cleanup_exit(int) __attribute__((noreturn));
 #endif
diff --git a/crypto/openssh/mac.c b/crypto/openssh/mac.c
index fabc3ed661da..eef50f48d6d3 100644
--- a/crypto/openssh/mac.c
+++ b/crypto/openssh/mac.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: mac.c,v 1.15 2008/06/13 00:51:47 dtucker Exp $ */
+/* $OpenBSD: mac.c,v 1.16 2011/08/02 01:22:11 djm Exp $ */
 /*
  * Copyright (c) 2001 Markus Friedl.  All rights reserved.
  *
@@ -57,6 +57,12 @@ struct {
 } macs[] = {
 	{ "hmac-sha1",			SSH_EVP, EVP_sha1, 0, -1, -1 },
 	{ "hmac-sha1-96",		SSH_EVP, EVP_sha1, 96, -1, -1 },
+#ifdef HAVE_EVP_SHA256
+	{ "hmac-sha2-256",		SSH_EVP, EVP_sha256, 0, -1, -1 },
+	{ "hmac-sha2-256-96",		SSH_EVP, EVP_sha256, 96, -1, -1 },
+	{ "hmac-sha2-512",		SSH_EVP, EVP_sha512, 0, -1, -1 },
+	{ "hmac-sha2-512-96",		SSH_EVP, EVP_sha512, 96, -1, -1 },
+#endif
 	{ "hmac-md5",			SSH_EVP, EVP_md5, 0, -1, -1 },
 	{ "hmac-md5-96",		SSH_EVP, EVP_md5, 96, -1, -1 },
 	{ "hmac-ripemd160",		SSH_EVP, EVP_ripemd160, 0, -1, -1 },
diff --git a/crypto/openssh/misc.c b/crypto/openssh/misc.c
index 1fc3ae74cb85..b334b25dd267 100644
--- a/crypto/openssh/misc.c
+++ b/crypto/openssh/misc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: misc.c,v 1.84 2010/11/21 01:01:13 djm Exp $ */
+/* $OpenBSD: misc.c,v 1.85 2011/03/29 18:54:17 stevesk Exp $ */
 /* $FreeBSD$ */
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
@@ -986,6 +986,19 @@ parse_ipqos(const char *cp)
 	return val;
 }
 
+const char *
+iptos2str(int iptos)
+{
+	int i;
+	static char iptos_str[sizeof "0xff"];
+
+	for (i = 0; ipqos[i].name != NULL; i++) {
+		if (ipqos[i].value == iptos)
+			return ipqos[i].name;
+	}
+	snprintf(iptos_str, sizeof iptos_str, "0x%02x", iptos);
+	return iptos_str;
+}
 void
 sock_set_v6only(int s)
 {
diff --git a/crypto/openssh/misc.h b/crypto/openssh/misc.h
index 19149ed3fc20..5d3d0bd0fb37 100644
--- a/crypto/openssh/misc.h
+++ b/crypto/openssh/misc.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: misc.h,v 1.47 2010/11/21 01:01:13 djm Exp $ */
+/* $OpenBSD: misc.h,v 1.48 2011/03/29 18:54:17 stevesk Exp $ */
 /* $FreeBSD$ */
 
 /*
@@ -91,6 +91,7 @@ void bandwidth_limit_init(struct bwlimit *, u_int64_t, size_t);
 void bandwidth_limit(struct bwlimit *, size_t);
 
 int parse_ipqos(const char *);
+const char *iptos2str(int);
 void mktemp_proto(char *, size_t);
 
 /* readpass.c */
diff --git a/crypto/openssh/moduli.5 b/crypto/openssh/moduli.5
index 1abaa3a686b8..13c0b22fbae6 100644
--- a/crypto/openssh/moduli.5
+++ b/crypto/openssh/moduli.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: moduli.5,v 1.12 2008/06/26 05:57:54 djm Exp $
+.\"	$OpenBSD: moduli.5,v 1.15 2010/10/14 20:41:28 jmc Exp $
 .\"
 .\" Copyright (c) 2008 Damien Miller <djm@mindrot.org>
 .\"
@@ -13,16 +13,16 @@
 .\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-.Dd June 26, 2008
+.Dd October 14, 2010
 .Dt MODULI 5
 .Os
 .Sh NAME
 .Nm moduli
-.Nd Diffie Hellman moduli
+.Nd Diffie-Hellman moduli
 .Sh DESCRIPTION
 The
 .Pa /etc/moduli
-file contains prime numbers and generators for use by 
+file contains prime numbers and generators for use by
 .Xr sshd 8
 in the Diffie-Hellman Group Exchange key exchange method.
 .Pp
@@ -31,24 +31,23 @@ New moduli may be generated with
 using a two-step process.
 An initial
 .Em candidate generation
-pass, using 
+pass, using
 .Ic ssh-keygen -G ,
 calculates numbers that are likely to be useful.
 A second
 .Em primality testing
 pass, using
-.Ic ssh-keygen -T
+.Ic ssh-keygen -T ,
 provides a high degree of assurance that the numbers are prime and are
-safe for use in Diffie Hellman operations by
+safe for use in Diffie-Hellman operations by
 .Xr sshd 8 .
 This
 .Nm
 format is used as the output from each pass.
 .Pp
 The file consists of newline-separated records, one per modulus,
-containing seven space separated fields.
+containing seven space-separated fields.
 These fields are as follows:
-.Pp
 .Bl -tag -width Description -offset indent
 .It timestamp
 The time that the modulus was last processed as YYYYMMDDHHMMSS.
@@ -58,7 +57,7 @@ Supported types are:
 .Pp
 .Bl -tag -width 0x00 -compact
 .It 0
-Unknown, not tested
+Unknown, not tested.
 .It 2
 "Safe" prime; (p-1)/2 is also prime.
 .It 4
@@ -68,7 +67,7 @@ Sophie Germain; (p+1)*2 is also prime.
 Moduli candidates initially produced by
 .Xr ssh-keygen 1
 are Sophie Germain primes (type 4).
-Futher primality testing with
+Further primality testing with
 .Xr ssh-keygen 1
 produces safe prime moduli (type 2) that are ready for use in
 .Xr sshd 8 .
@@ -79,13 +78,13 @@ has been subjected to represented as a bitmask of the following values:
 .Pp
 .Bl -tag -width 0x00 -compact
 .It 0x00
-Not tested
+Not tested.
 .It 0x01
-Composite number - not prime.
+Composite number \(en not prime.
 .It 0x02
-Sieve of Eratosthenes
+Sieve of Eratosthenes.
 .It 0x04
-Probabalistic Miller-Rabin primality tests.
+Probabilistic Miller-Rabin primality tests.
 .El
 .Pp
 The
@@ -95,8 +94,8 @@ Subsequent
 .Xr ssh-keygen 1
 primality tests are Miller-Rabin tests (flag 0x04).
 .It trials
-Decimal number indicating of primaility trials that have been performed
-on the modulus.
+Decimal number indicating the number of primality trials
+that have been performed on the modulus.
 .It size
 Decimal number indicating the size of the prime in bits.
 .It generator
@@ -105,18 +104,17 @@ The recommended generator for use with this modulus (hexadecimal).
 The modulus itself in hexadecimal.
 .El
 .Pp
-When performing Diffie Hellman Group Exchange,
+When performing Diffie-Hellman Group Exchange,
 .Xr sshd 8
 first estimates the size of the modulus required to produce enough
-Diffie Hellman output to sufficiently key the selected symmetric cipher.
+Diffie-Hellman output to sufficiently key the selected symmetric cipher.
 .Xr sshd 8
 then randomly selects a modulus from
 .Fa /etc/moduli
 that best meets the size requirement.
-.Pp
 .Sh SEE ALSO
 .Xr ssh-keygen 1 ,
-.Xr sshd 8 ,
+.Xr sshd 8
 .Rs
 .%R RFC 4419
 .%T "Diffie-Hellman Group Exchange for the Secure Shell (SSH) Transport Layer Protocol"
diff --git a/crypto/openssh/monitor.c b/crypto/openssh/monitor.c
index 29d987c70fdc..a166fed2eec7 100644
--- a/crypto/openssh/monitor.c
+++ b/crypto/openssh/monitor.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: monitor.c,v 1.110 2010/09/09 10:45:45 djm Exp $ */
+/* $OpenBSD: monitor.c,v 1.115 2011/06/23 23:35:42 djm Exp $ */
 /*
  * Copyright 2002 Niels Provos <provos@citi.umich.edu>
  * Copyright 2002 Markus Friedl <markus@openbsd.org>
@@ -44,6 +44,13 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#ifdef HAVE_POLL_H
+#include <poll.h>
+#else
+# ifdef HAVE_SYS_POLL_H
+#  include <sys/poll.h>
+# endif
+#endif
 
 #ifdef SKEY
 #include <skey.h>
@@ -52,6 +59,7 @@
 #include <openssl/dh.h>
 
 #include "openbsd-compat/sys-queue.h"
+#include "atomicio.h"
 #include "xmalloc.h"
 #include "ssh.h"
 #include "key.h"
@@ -179,6 +187,8 @@ int mm_answer_audit_event(int, Buffer *);
 int mm_answer_audit_command(int, Buffer *);
 #endif
 
+static int monitor_read_log(struct monitor *);
+
 static Authctxt *authctxt;
 static BIGNUM *ssh1_challenge = NULL;	/* used for ssh1 rsa auth */
 
@@ -346,6 +356,10 @@ monitor_child_preauth(Authctxt *_authctxt, struct monitor *pmonitor)
 
 	debug3("preauth child monitor started");
 
+	close(pmonitor->m_recvfd);
+	close(pmonitor->m_log_sendfd);
+	pmonitor->m_log_sendfd = pmonitor->m_recvfd = -1;
+
 	authctxt = _authctxt;
 	memset(authctxt, 0, sizeof(*authctxt));
 
@@ -405,6 +419,10 @@ monitor_child_preauth(Authctxt *_authctxt, struct monitor *pmonitor)
 #endif
 	}
 
+	/* Drain any buffered messages from the child */
+	while (pmonitor->m_log_recvfd != -1 && monitor_read_log(pmonitor) == 0)
+		;
+
 	if (!authctxt->valid)
 		fatal("%s: authenticated invalid user", __func__);
 	if (strcmp(auth_method, "unknown") == 0)
@@ -414,6 +432,10 @@ monitor_child_preauth(Authctxt *_authctxt, struct monitor *pmonitor)
 	    __func__, authctxt->user);
 
 	mm_get_keystate(pmonitor);
+
+	close(pmonitor->m_sendfd);
+	close(pmonitor->m_log_recvfd);
+	pmonitor->m_sendfd = pmonitor->m_log_recvfd = -1;
 }
 
 static void
@@ -431,6 +453,9 @@ monitor_child_handler(int sig)
 void
 monitor_child_postauth(struct monitor *pmonitor)
 {
+	close(pmonitor->m_recvfd);
+	pmonitor->m_recvfd = -1;
+
 	monitor_set_child_handler(pmonitor->m_pid);
 	signal(SIGHUP, &monitor_child_handler);
 	signal(SIGTERM, &monitor_child_handler);
@@ -454,6 +479,9 @@ monitor_child_postauth(struct monitor *pmonitor)
 
 	for (;;)
 		monitor_read(pmonitor, mon_dispatch, NULL);
+
+	close(pmonitor->m_sendfd);
+	pmonitor->m_sendfd = -1;
 }
 
 void
@@ -465,6 +493,52 @@ monitor_sync(struct monitor *pmonitor)
 	}
 }
 
+static int
+monitor_read_log(struct monitor *pmonitor)
+{
+	Buffer logmsg;
+	u_int len, level;
+	char *msg;
+
+	buffer_init(&logmsg);
+
+	/* Read length */
+	buffer_append_space(&logmsg, 4);
+	if (atomicio(read, pmonitor->m_log_recvfd,
+	    buffer_ptr(&logmsg), buffer_len(&logmsg)) != buffer_len(&logmsg)) {
+		if (errno == EPIPE) {
+			debug("%s: child log fd closed", __func__);
+			close(pmonitor->m_log_recvfd);
+			pmonitor->m_log_recvfd = -1;
+			return -1;
+		}
+		fatal("%s: log fd read: %s", __func__, strerror(errno));
+	}
+	len = buffer_get_int(&logmsg);
+	if (len <= 4 || len > 8192)
+		fatal("%s: invalid log message length %u", __func__, len);
+
+	/* Read severity, message */
+	buffer_clear(&logmsg);
+	buffer_append_space(&logmsg, len);
+	if (atomicio(read, pmonitor->m_log_recvfd,
+	    buffer_ptr(&logmsg), buffer_len(&logmsg)) != buffer_len(&logmsg))
+		fatal("%s: log fd read: %s", __func__, strerror(errno));
+
+	/* Log it */
+	level = buffer_get_int(&logmsg);
+	msg = buffer_get_string(&logmsg, NULL);
+	if (log_level_name(level) == NULL)
+		fatal("%s: invalid log level %u (corrupted message?)",
+		    __func__, level);
+	do_log2(level, "%s [preauth]", msg);
+
+	buffer_free(&logmsg);
+	xfree(msg);
+
+	return 0;
+}
+
 int
 monitor_read(struct monitor *pmonitor, struct mon_table *ent,
     struct mon_table **pent)
@@ -472,6 +546,30 @@ monitor_read(struct monitor *pmonitor, struct mon_table *ent,
 	Buffer m;
 	int ret;
 	u_char type;
+	struct pollfd pfd[2];
+
+	for (;;) {
+		bzero(&pfd, sizeof(pfd));
+		pfd[0].fd = pmonitor->m_sendfd;
+		pfd[0].events = POLLIN;
+		pfd[1].fd = pmonitor->m_log_recvfd;
+		pfd[1].events = pfd[1].fd == -1 ? 0 : POLLIN;
+		if (poll(pfd, pfd[1].fd == -1 ? 1 : 2, -1) == -1) {
+			if (errno == EINTR || errno == EAGAIN)
+				continue;
+			fatal("%s: poll: %s", __func__, strerror(errno));
+		}
+		if (pfd[1].revents) {
+			/*
+			 * Drain all log messages before processing next
+			 * monitor request.
+			 */
+			monitor_read_log(pmonitor);
+			continue;
+		}
+		if (pfd[0].revents)
+			break;  /* Continues below */
+	}
 
 	buffer_init(&m);
 
@@ -632,6 +730,7 @@ mm_answer_pwnamallow(int sock, Buffer *m)
 	char *username;
 	struct passwd *pwent;
 	int allowed = 0;
+	u_int i;
 
 	debug3("%s", __func__);
 
@@ -671,8 +770,20 @@ mm_answer_pwnamallow(int sock, Buffer *m)
 
  out:
 	buffer_put_string(m, &options, sizeof(options));
-	if (options.banner != NULL)
-		buffer_put_cstring(m, options.banner);
+
+#define M_CP_STROPT(x) do { \
+		if (options.x != NULL) \
+			buffer_put_cstring(m, options.x); \
+	} while (0)
+#define M_CP_STRARRAYOPT(x, nx) do { \
+		for (i = 0; i < options.nx; i++) \
+			buffer_put_cstring(m, options.x[i]); \
+	} while (0)
+	/* See comment in servconf.h */
+	COPY_MATCH_STRING_OPTS();
+#undef M_CP_STROPT
+#undef M_CP_STRARRAYOPT
+	
 	debug3("%s: sending MONITOR_ANS_PWNAM: %d", __func__, allowed);
 	mm_request_send(sock, MONITOR_ANS_PWNAM, m);
 
@@ -684,7 +795,6 @@ mm_answer_pwnamallow(int sock, Buffer *m)
 		monitor_permit(mon_dispatch, MONITOR_REQ_AUTHSERV, 1);
 		monitor_permit(mon_dispatch, MONITOR_REQ_AUTH2_READ_BANNER, 1);
 	}
-
 #ifdef USE_PAM
 	if (options.use_pam)
 		monitor_permit(mon_dispatch, MONITOR_REQ_PAM_START, 1);
@@ -1834,22 +1944,31 @@ mm_init_compression(struct mm_master *mm)
 /* XXX */
 
 #define FD_CLOSEONEXEC(x) do { \
-	if (fcntl(x, F_SETFD, 1) == -1) \
+	if (fcntl(x, F_SETFD, FD_CLOEXEC) == -1) \
 		fatal("fcntl(%d, F_SETFD)", x); \
 } while (0)
 
 static void
-monitor_socketpair(int *pair)
+monitor_openfds(struct monitor *mon, int do_logfds)
 {
-#ifdef HAVE_SOCKETPAIR
+	int pair[2];
+
 	if (socketpair(AF_UNIX, SOCK_STREAM, 0, pair) == -1)
-		fatal("%s: socketpair", __func__);
-#else
-	fatal("%s: UsePrivilegeSeparation=yes not supported",
-	    __func__);
-#endif
+		fatal("%s: socketpair: %s", __func__, strerror(errno));
 	FD_CLOSEONEXEC(pair[0]);
 	FD_CLOSEONEXEC(pair[1]);
+	mon->m_recvfd = pair[0];
+	mon->m_sendfd = pair[1];
+
+	if (do_logfds) {
+		if (pipe(pair) == -1)
+			fatal("%s: pipe: %s", __func__, strerror(errno));
+		FD_CLOSEONEXEC(pair[0]);
+		FD_CLOSEONEXEC(pair[1]);
+		mon->m_log_recvfd = pair[0];
+		mon->m_log_sendfd = pair[1];
+	} else
+		mon->m_log_recvfd = mon->m_log_sendfd = -1;
 }
 
 #define MM_MEMSIZE	65536
@@ -1858,14 +1977,10 @@ struct monitor *
 monitor_init(void)
 {
 	struct monitor *mon;
-	int pair[2];
 
 	mon = xcalloc(1, sizeof(*mon));
 
-	monitor_socketpair(pair);
-
-	mon->m_recvfd = pair[0];
-	mon->m_sendfd = pair[1];
+	monitor_openfds(mon, 1);
 
 	/* Used to share zlib space across processes */
 	if (options.compression) {
@@ -1882,12 +1997,7 @@ monitor_init(void)
 void
 monitor_reinit(struct monitor *mon)
 {
-	int pair[2];
-
-	monitor_socketpair(pair);
-
-	mon->m_recvfd = pair[0];
-	mon->m_sendfd = pair[1];
+	monitor_openfds(mon, 0);
 }
 
 #ifdef GSSAPI
diff --git a/crypto/openssh/monitor.h b/crypto/openssh/monitor.h
index a8a2c0c199ff..5e7d552fb6a5 100644
--- a/crypto/openssh/monitor.h
+++ b/crypto/openssh/monitor.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: monitor.h,v 1.15 2008/11/04 08:22:13 djm Exp $ */
+/* $OpenBSD: monitor.h,v 1.16 2011/06/17 21:44:31 djm Exp $ */
 
 /*
  * Copyright 2002 Niels Provos <provos@citi.umich.edu>
@@ -72,6 +72,8 @@ struct mm_master;
 struct monitor {
 	int			 m_recvfd;
 	int			 m_sendfd;
+	int			 m_log_recvfd;
+	int			 m_log_sendfd;
 	struct mm_master	*m_zback;
 	struct mm_master	*m_zlib;
 	struct Kex		**m_pkex;
diff --git a/crypto/openssh/monitor_wrap.c b/crypto/openssh/monitor_wrap.c
index 1a5dda56158e..1f60658e9839 100644
--- a/crypto/openssh/monitor_wrap.c
+++ b/crypto/openssh/monitor_wrap.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: monitor_wrap.c,v 1.70 2010/08/31 11:54:45 djm Exp $ */
+/* $OpenBSD: monitor_wrap.c,v 1.73 2011/06/17 21:44:31 djm Exp $ */
 /*
  * Copyright 2002 Niels Provos <provos@citi.umich.edu>
  * Copyright 2002 Markus Friedl <markus@openbsd.org>
@@ -88,6 +88,32 @@ extern struct monitor *pmonitor;
 extern Buffer loginmsg;
 extern ServerOptions options;
 
+void
+mm_log_handler(LogLevel level, const char *msg, void *ctx)
+{
+	Buffer log_msg;
+	struct monitor *mon = (struct monitor *)ctx;
+
+	if (mon->m_log_sendfd == -1)
+		fatal("%s: no log channel", __func__);
+
+	buffer_init(&log_msg);
+	/*
+	 * Placeholder for packet length. Will be filled in with the actual
+	 * packet length once the packet has been constucted. This saves
+	 * fragile math.
+	 */
+	buffer_put_int(&log_msg, 0);
+
+	buffer_put_int(&log_msg, level);
+	buffer_put_cstring(&log_msg, msg);
+	put_u32(buffer_ptr(&log_msg), buffer_len(&log_msg) - 4);
+	if (atomicio(vwrite, mon->m_log_sendfd, buffer_ptr(&log_msg),
+	    buffer_len(&log_msg)) != buffer_len(&log_msg))
+		fatal("%s: write: %s", __func__, strerror(errno));
+	buffer_free(&log_msg);
+}
+
 int
 mm_is_monitor(void)
 {
@@ -211,7 +237,7 @@ mm_getpwnamallow(const char *username)
 {
 	Buffer m;
 	struct passwd *pw;
-	u_int len;
+	u_int len, i;
 	ServerOptions *newopts;
 
 	debug3("%s entering", __func__);
@@ -245,8 +271,20 @@ mm_getpwnamallow(const char *username)
 	newopts = buffer_get_string(&m, &len);
 	if (len != sizeof(*newopts))
 		fatal("%s: option block size mismatch", __func__);
-	if (newopts->banner != NULL)
-		newopts->banner = buffer_get_string(&m, NULL);
+
+#define M_CP_STROPT(x) do { \
+		if (newopts->x != NULL) \
+			newopts->x = buffer_get_string(&m, NULL); \
+	} while (0)
+#define M_CP_STRARRAYOPT(x, nx) do { \
+		for (i = 0; i < newopts->nx; i++) \
+			newopts->x[i] = buffer_get_string(&m, NULL); \
+	} while (0)
+	/* See comment in servconf.h */
+	COPY_MATCH_STRING_OPTS();
+#undef M_CP_STROPT
+#undef M_CP_STRARRAYOPT
+
 	copy_set_server_options(&options, newopts, 1);
 	xfree(newopts);
 
diff --git a/crypto/openssh/monitor_wrap.h b/crypto/openssh/monitor_wrap.h
index de2d16f66276..0c7f2e3848ad 100644
--- a/crypto/openssh/monitor_wrap.h
+++ b/crypto/openssh/monitor_wrap.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: monitor_wrap.h,v 1.22 2009/03/05 07:18:19 djm Exp $ */
+/* $OpenBSD: monitor_wrap.h,v 1.23 2011/06/17 21:44:31 djm Exp $ */
 
 /*
  * Copyright 2002 Niels Provos <provos@citi.umich.edu>
@@ -37,6 +37,7 @@ struct monitor;
 struct mm_master;
 struct Authctxt;
 
+void mm_log_handler(LogLevel, const char *, void *);
 int mm_is_monitor(void);
 DH *mm_choose_dh(int, int, int);
 int mm_key_sign(Key *, u_char **, u_int *, u_char *, u_int);
diff --git a/crypto/openssh/mux.c b/crypto/openssh/mux.c
index e370462db5a2..add0e26b1b91 100644
--- a/crypto/openssh/mux.c
+++ b/crypto/openssh/mux.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: mux.c,v 1.24 2011/01/13 21:54:53 djm Exp $ */
+/* $OpenBSD: mux.c,v 1.29 2011/06/22 22:08:42 djm Exp $ */
 /*
  * Copyright (c) 2002-2008 Damien Miller <djm@openbsd.org>
  *
@@ -87,7 +87,6 @@
 
 /* from ssh.c */
 extern int tty_flag;
-extern int force_tty_flag;
 extern Options options;
 extern int stdin_null_flag;
 extern char *host;
@@ -146,6 +145,7 @@ struct mux_master_state {
 #define MUX_C_OPEN_FWD		0x10000006
 #define MUX_C_CLOSE_FWD		0x10000007
 #define MUX_C_NEW_STDIO_FWD	0x10000008
+#define MUX_C_STOP_LISTENING	0x10000009
 #define MUX_S_OK		0x80000001
 #define MUX_S_PERMISSION_DENIED	0x80000002
 #define MUX_S_FAILURE		0x80000003
@@ -153,6 +153,7 @@ struct mux_master_state {
 #define MUX_S_ALIVE		0x80000005
 #define MUX_S_SESSION_OPENED	0x80000006
 #define MUX_S_REMOTE_PORT	0x80000007
+#define MUX_S_TTY_ALLOC_FAIL	0x80000008
 
 /* type codes for MUX_C_OPEN_FWD and MUX_C_CLOSE_FWD */
 #define MUX_FWD_LOCAL   1
@@ -168,6 +169,7 @@ static int process_mux_terminate(u_int, Channel *, Buffer *, Buffer *);
 static int process_mux_open_fwd(u_int, Channel *, Buffer *, Buffer *);
 static int process_mux_close_fwd(u_int, Channel *, Buffer *, Buffer *);
 static int process_mux_stdio_fwd(u_int, Channel *, Buffer *, Buffer *);
+static int process_mux_stop_listening(u_int, Channel *, Buffer *, Buffer *);
 
 static const struct {
 	u_int type;
@@ -180,6 +182,7 @@ static const struct {
 	{ MUX_C_OPEN_FWD, process_mux_open_fwd },
 	{ MUX_C_CLOSE_FWD, process_mux_close_fwd },
 	{ MUX_C_NEW_STDIO_FWD, process_mux_stdio_fwd },
+	{ MUX_C_STOP_LISTENING, process_mux_stop_listening },
 	{ 0, NULL }
 };
 
@@ -915,6 +918,39 @@ process_mux_stdio_fwd(u_int rid, Channel *c, Buffer *m, Buffer *r)
 	return 0;
 }
 
+static int
+process_mux_stop_listening(u_int rid, Channel *c, Buffer *m, Buffer *r)
+{
+	debug("%s: channel %d: stop listening", __func__, c->self);
+
+	if (options.control_master == SSHCTL_MASTER_ASK ||
+	    options.control_master == SSHCTL_MASTER_AUTO_ASK) {
+		if (!ask_permission("Disable further multiplexing on shared "
+		    "connection to %s? ", host)) {
+			debug2("%s: stop listen refused by user", __func__);
+			buffer_put_int(r, MUX_S_PERMISSION_DENIED);
+			buffer_put_int(r, rid);
+			buffer_put_cstring(r, "Permission denied");
+			return 0;
+		}
+	}
+
+	if (mux_listener_channel != NULL) {
+		channel_free(mux_listener_channel);
+		client_stop_mux();
+		xfree(options.control_path);
+		options.control_path = NULL;
+		mux_listener_channel = NULL;
+		muxserver_sock = -1;
+	}
+
+	/* prepare reply */
+	buffer_put_int(r, MUX_S_OK);
+	buffer_put_int(r, rid);
+
+	return 0;
+}
+
 /* Channel callbacks fired on read/write from mux slave fd */
 static int
 mux_master_read_cb(Channel *c)
@@ -1019,6 +1055,27 @@ mux_exit_message(Channel *c, int exitval)
 	buffer_free(&m);
 }
 
+void
+mux_tty_alloc_failed(Channel *c)
+{
+	Buffer m;
+	Channel *mux_chan;
+
+	debug3("%s: channel %d: TTY alloc failed", __func__, c->self);
+
+	if ((mux_chan = channel_by_id(c->ctl_chan)) == NULL)
+		fatal("%s: channel %d missing mux channel %d",
+		    __func__, c->self, c->ctl_chan);
+
+	/* Append exit message packet to control socket output queue */
+	buffer_init(&m);
+	buffer_put_int(&m, MUX_S_TTY_ALLOC_FAIL);
+	buffer_put_int(&m, c->self);
+
+	buffer_put_string(&mux_chan->output, buffer_ptr(&m), buffer_len(&m));
+	buffer_free(&m);
+}
+
 /* Prepare a mux master to listen on a Unix domain socket. */
 void
 muxserver_listen(void)
@@ -1059,21 +1116,25 @@ muxserver_listen(void)
 	    strlen(options.control_path) + 1;
 
 	if (strlcpy(addr.sun_path, options.control_path,
-	    sizeof(addr.sun_path)) >= sizeof(addr.sun_path))
-		fatal("ControlPath too long");
+	    sizeof(addr.sun_path)) >= sizeof(addr.sun_path)) {
+		error("ControlPath \"%s\" too long for Unix domain socket",
+		    options.control_path);
+		goto disable_mux_master;
+	}
 
 	if ((muxserver_sock = socket(PF_UNIX, SOCK_STREAM, 0)) < 0)
 		fatal("%s socket(): %s", __func__, strerror(errno));
 
 	old_umask = umask(0177);
 	if (bind(muxserver_sock, (struct sockaddr *)&addr, sun_len) == -1) {
-		muxserver_sock = -1;
 		if (errno == EINVAL || errno == EADDRINUSE) {
 			error("ControlSocket %s already exists, "
 			    "disabling multiplexing", options.control_path);
  disable_mux_master:
-			close(muxserver_sock);
-			muxserver_sock = -1;
+			if (muxserver_sock != -1) {
+				close(muxserver_sock);
+				muxserver_sock = -1;
+			}
 			xfree(options.control_path);
 			options.control_path = NULL;
 			options.control_master = SSHCTL_MASTER_NO;
@@ -1153,8 +1214,10 @@ mux_session_confirm(int id, int success, void *arg)
 		/* Request forwarding with authentication spoofing. */
 		debug("Requesting X11 forwarding with authentication "
 		    "spoofing.");
-		x11_request_forwarding_with_spoofing(id, display, proto, data);
-		/* XXX wait for reply */
+		x11_request_forwarding_with_spoofing(id, display, proto,
+		    data, 1);
+		client_expect_confirm(id, "X11 forwarding", CONFIRM_WARN);
+		/* XXX exit_on_forward_failure */
 	}
 
 	if (cctx->want_agent_fwd && options.forward_agent) {
@@ -1573,7 +1636,7 @@ mux_client_request_session(int fd)
 	char *e, *term;
 	u_int i, rid, sid, esid, exitval, type, exitval_seen;
 	extern char **environ;
-	int devnull;
+	int devnull, rawmode;
 
 	debug3("%s: entering", __func__);
 
@@ -1669,8 +1732,9 @@ mux_client_request_session(int fd)
 	signal(SIGTERM, control_client_sighandler);
 	signal(SIGWINCH, control_client_sigrelay);
 
+	rawmode = tty_flag;
 	if (tty_flag)
-		enter_raw_mode(force_tty_flag);
+		enter_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 
 	/*
 	 * Stick around until the controlee closes the client_fd.
@@ -1684,22 +1748,35 @@ mux_client_request_session(int fd)
 		if (mux_client_read_packet(fd, &m) != 0)
 			break;
 		type = buffer_get_int(&m);
-		if (type != MUX_S_EXIT_MESSAGE) {
+		switch (type) {
+		case MUX_S_TTY_ALLOC_FAIL:
+			if ((esid = buffer_get_int(&m)) != sid)
+				fatal("%s: tty alloc fail on unknown session: "
+				    "my id %u theirs %u",
+				    __func__, sid, esid);
+			leave_raw_mode(options.request_tty ==
+			    REQUEST_TTY_FORCE);
+			rawmode = 0;
+			continue;
+		case MUX_S_EXIT_MESSAGE:
+			if ((esid = buffer_get_int(&m)) != sid)
+				fatal("%s: exit on unknown session: "
+				    "my id %u theirs %u",
+				    __func__, sid, esid);
+			if (exitval_seen)
+				fatal("%s: exitval sent twice", __func__);
+			exitval = buffer_get_int(&m);
+			exitval_seen = 1;
+			continue;
+		default:
 			e = buffer_get_string(&m, NULL);
 			fatal("%s: master returned error: %s", __func__, e);
 		}
-		if ((esid = buffer_get_int(&m)) != sid)
-			fatal("%s: exit on unknown session: my id %u theirs %u",
-			    __func__, sid, esid);
-		debug("%s: master session id: %u", __func__, sid);
-		if (exitval_seen)
-			fatal("%s: exitval sent twice", __func__);
-		exitval = buffer_get_int(&m);
-		exitval_seen = 1;
 	}
 
 	close(fd);
-	leave_raw_mode(force_tty_flag);
+	if (rawmode)
+		leave_raw_mode(options.request_tty == REQUEST_TTY_FORCE);
 
 	if (muxclient_terminate) {
 		debug2("Exiting on signal %d", muxclient_terminate);
@@ -1813,6 +1890,50 @@ mux_client_request_stdio_fwd(int fd)
 	fatal("%s: master returned unexpected message %u", __func__, type);
 }
 
+static void
+mux_client_request_stop_listening(int fd)
+{
+	Buffer m;
+	char *e;
+	u_int type, rid;
+
+	debug3("%s: entering", __func__);
+
+	buffer_init(&m);
+	buffer_put_int(&m, MUX_C_STOP_LISTENING);
+	buffer_put_int(&m, muxclient_request_id);
+
+	if (mux_client_write_packet(fd, &m) != 0)
+		fatal("%s: write packet: %s", __func__, strerror(errno));
+
+	buffer_clear(&m);
+
+	/* Read their reply */
+	if (mux_client_read_packet(fd, &m) != 0)
+		fatal("%s: read from master failed: %s",
+		    __func__, strerror(errno));
+
+	type = buffer_get_int(&m);
+	if ((rid = buffer_get_int(&m)) != muxclient_request_id)
+		fatal("%s: out of sequence reply: my id %u theirs %u",
+		    __func__, muxclient_request_id, rid);
+	switch (type) {
+	case MUX_S_OK:
+		break;
+	case MUX_S_PERMISSION_DENIED:
+		e = buffer_get_string(&m, NULL);
+		fatal("Master refused stop listening request: %s", e);
+	case MUX_S_FAILURE:
+		e = buffer_get_string(&m, NULL);
+		fatal("%s: stop listening request failed: %s", __func__, e);
+	default:
+		fatal("%s: unexpected response from master 0x%08x",
+		    __func__, type);
+	}
+	buffer_free(&m);
+	muxclient_request_id++;
+}
+
 /* Multiplex client main loop. */
 void
 muxclient(const char *path)
@@ -1906,6 +2027,10 @@ muxclient(const char *path)
 	case SSHMUX_COMMAND_STDIO_FWD:
 		mux_client_request_stdio_fwd(sock);
 		exit(0);
+	case SSHMUX_COMMAND_STOP:
+		mux_client_request_stop_listening(sock);
+		fprintf(stderr, "Stop listening request sent.\r\n");
+		exit(0);
 	default:
 		fatal("unrecognised muxclient_command %d", muxclient_command);
 	}
diff --git a/crypto/openssh/myproposal.h b/crypto/openssh/myproposal.h
index 30948c2f9e24..e552c5b621a2 100644
--- a/crypto/openssh/myproposal.h
+++ b/crypto/openssh/myproposal.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: myproposal.h,v 1.27 2010/09/01 22:42:13 djm Exp $ */
+/* $OpenBSD: myproposal.h,v 1.28 2011/08/02 01:22:11 djm Exp $ */
 /* $FreeBSD$ */
 
 /*
@@ -80,10 +80,25 @@
 #define KEX_ENCRYPT_INCLUDE_NONE KEX_DEFAULT_ENCRYPT \
 	",none"
 #endif
+#ifdef HAVE_EVP_SHA256
+#define	SHA2_HMAC_MODES \
+	"hmac-sha2-256," \
+	"hmac-sha2-256-96," \
+	"hmac-sha2-512," \
+	"hmac-sha2-512-96,"
+#else
+# define SHA2_HMAC_MODES
+#endif
 #define	KEX_DEFAULT_MAC \
-	"hmac-md5,hmac-sha1,umac-64@openssh.com,hmac-ripemd160," \
+	"hmac-md5," \
+	"hmac-sha1," \
+	"umac-64@openssh.com," \
+	SHA2_HMAC_MODES \
+	"hmac-ripemd160," \
 	"hmac-ripemd160@openssh.com," \
-	"hmac-sha1-96,hmac-md5-96"
+	"hmac-sha1-96," \
+	"hmac-md5-96"
+
 #define	KEX_DEFAULT_COMP	"none,zlib@openssh.com,zlib"
 #define	KEX_DEFAULT_LANG	""
 
diff --git a/crypto/openssh/openbsd-compat/bsd-cygwin_util.c b/crypto/openssh/openbsd-compat/bsd-cygwin_util.c
index e9fa3a0e244a..9eedc88d2c1c 100644
--- a/crypto/openssh/openbsd-compat/bsd-cygwin_util.c
+++ b/crypto/openssh/openbsd-compat/bsd-cygwin_util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2001, Corinna Vinschen <vinschen@cygnus.com>
+ * Copyright (c) 2000, 2001, 2011 Corinna Vinschen <vinschen@redhat.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -34,9 +34,6 @@
 #if defined(open) && open == binary_open
 # undef open
 #endif
-#if defined(pipe) && open == binary_pipe
-# undef pipe
-#endif
 
 #include <sys/types.h>
 
@@ -59,18 +56,6 @@ binary_open(const char *filename, int flags, ...)
 	return (open(filename, flags | O_BINARY, mode));
 }
 
-int 
-binary_pipe(int fd[2])
-{
-	int ret = pipe(fd);
-
-	if (!ret) {
-		setmode(fd[0], O_BINARY);
-		setmode(fd[1], O_BINARY);
-	}
-	return (ret);
-}
-
 int
 check_ntsec(const char *filename)
 {
diff --git a/crypto/openssh/openbsd-compat/bsd-cygwin_util.h b/crypto/openssh/openbsd-compat/bsd-cygwin_util.h
index 39b8eb788df0..48f64b740690 100644
--- a/crypto/openssh/openbsd-compat/bsd-cygwin_util.h
+++ b/crypto/openssh/openbsd-compat/bsd-cygwin_util.h
@@ -1,7 +1,7 @@
-/* $Id: bsd-cygwin_util.h,v 1.12 2009/03/08 00:40:28 dtucker Exp $ */
+/* $Id: bsd-cygwin_util.h,v 1.13 2011/08/17 01:31:09 djm Exp $ */
 
 /*
- * Copyright (c) 2000, 2001, Corinna Vinschen <vinschen@cygnus.com>
+ * Copyright (c) 2000, 2001, 2011 Corinna Vinschen <vinschen@redhat.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -41,13 +41,11 @@
 #include <io.h>
 
 int binary_open(const char *, int , ...);
-int binary_pipe(int fd[2]);
 int check_ntsec(const char *);
 char **fetch_windows_environment(void);
 void free_windows_environment(char **);
 
 #define open binary_open
-#define pipe binary_pipe
 
 #endif /* HAVE_CYGWIN */
 
diff --git a/crypto/openssh/openbsd-compat/openssl-compat.c b/crypto/openssh/openbsd-compat/openssl-compat.c
index b617fdf19451..5189cab61694 100644
--- a/crypto/openssh/openbsd-compat/openssl-compat.c
+++ b/crypto/openssh/openbsd-compat/openssl-compat.c
@@ -1,4 +1,4 @@
-/* $Id: openssl-compat.c,v 1.13 2011/01/21 22:37:06 dtucker Exp $ */
+/* $Id: openssl-compat.c,v 1.14 2011/05/10 01:13:38 dtucker Exp $ */
 
 /*
  * Copyright (c) 2005 Darren Tucker <dtucker@zip.com.au>
@@ -134,9 +134,9 @@ RSA_get_default_method(void)
 
 #ifdef	USE_OPENSSL_ENGINE
 void
-ssh_SSLeay_add_all_algorithms(void)
+ssh_OpenSSL_add_all_algorithms(void)
 {
-	SSLeay_add_all_algorithms();
+	OpenSSL_add_all_algorithms();
 
 	/* Enable use of crypto hardware */
 	ENGINE_load_builtin_engines();
diff --git a/crypto/openssh/openbsd-compat/openssl-compat.h b/crypto/openssh/openbsd-compat/openssl-compat.h
index 6d4f3f215c6d..c5fc24eb4b71 100644
--- a/crypto/openssh/openbsd-compat/openssl-compat.h
+++ b/crypto/openssh/openbsd-compat/openssl-compat.h
@@ -1,4 +1,4 @@
-/* $Id: openssl-compat.h,v 1.18 2011/01/21 22:37:06 dtucker Exp $ */
+/* $Id: openssl-compat.h,v 1.19 2011/05/10 01:13:38 dtucker Exp $ */
 
 /*
  * Copyright (c) 2005 Darren Tucker <dtucker@zip.com.au>
@@ -106,10 +106,10 @@ RSA_METHOD *RSA_get_default_method(void);
 #  endif
 
 # ifdef USE_OPENSSL_ENGINE
-#  ifdef SSLeay_add_all_algorithms
-#   undef SSLeay_add_all_algorithms
+#  ifdef OpenSSL_add_all_algorithms
+#   undef OpenSSL_add_all_algorithms
 #  endif
-#  define SSLeay_add_all_algorithms()  ssh_SSLeay_add_all_algorithms()
+#  define OpenSSL_add_all_algorithms()  ssh_OpenSSL_add_all_algorithms()
 # endif
 
 # ifndef HAVE_BN_IS_PRIME_EX
@@ -129,6 +129,6 @@ int ssh_EVP_CipherInit(EVP_CIPHER_CTX *, const EVP_CIPHER *, unsigned char *,
     unsigned char *, int);
 int ssh_EVP_Cipher(EVP_CIPHER_CTX *, char *, char *, int);
 int ssh_EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *);
-void ssh_SSLeay_add_all_algorithms(void);
+void ssh_OpenSSL_add_all_algorithms(void);
 #endif	/* SSH_DONT_OVERLOAD_OPENSSL_FUNCS */
 
diff --git a/crypto/openssh/openbsd-compat/port-linux.c b/crypto/openssh/openbsd-compat/port-linux.c
index fe6fc2d84a2d..ea8dff40f706 100644
--- a/crypto/openssh/openbsd-compat/port-linux.c
+++ b/crypto/openssh/openbsd-compat/port-linux.c
@@ -1,4 +1,4 @@
-/* $Id: port-linux.c,v 1.11.4.3 2011/02/06 02:24:17 dtucker Exp $ */
+/* $Id: port-linux.c,v 1.16 2011/08/29 06:09:57 djm Exp $ */
 
 /*
  * Copyright (c) 2005 Daniel Walsh <dwalsh@redhat.com>
@@ -38,6 +38,10 @@
 #include <selinux/flask.h>
 #include <selinux/get_context_list.h>
 
+#ifndef SSH_SELINUX_UNCONFINED_TYPE
+# define SSH_SELINUX_UNCONFINED_TYPE ":unconfined_t:"
+#endif
+
 /* Wrapper around is_selinux_enabled() to log its return value once only */
 int
 ssh_selinux_enabled(void)
@@ -177,12 +181,13 @@ ssh_selinux_change_context(const char *newname)
 {
 	int len, newlen;
 	char *oldctx, *newctx, *cx;
+	void (*switchlog) (const char *fmt,...) = logit;
 
 	if (!ssh_selinux_enabled())
 		return;
 
 	if (getcon((security_context_t *)&oldctx) < 0) {
-		logit("%s: getcon failed with %s", __func__, strerror (errno));
+		logit("%s: getcon failed with %s", __func__, strerror(errno));
 		return;
 	}
 	if ((cx = index(oldctx, ':')) == NULL || (cx = index(cx + 1, ':')) ==
@@ -191,6 +196,14 @@ ssh_selinux_change_context(const char *newname)
 		return;
 	}
 
+	/*
+	 * Check whether we are attempting to switch away from an unconfined
+	 * security context.
+	 */
+	if (strncmp(cx, SSH_SELINUX_UNCONFINED_TYPE,
+	    sizeof(SSH_SELINUX_UNCONFINED_TYPE) - 1) == 0)
+		switchlog = debug3;
+
 	newlen = strlen(oldctx) + strlen(newname) + 1;
 	newctx = xmalloc(newlen);
 	len = cx - oldctx + 1;
@@ -198,10 +211,11 @@ ssh_selinux_change_context(const char *newname)
 	strlcpy(newctx + len, newname, newlen - len);
 	if ((cx = index(cx + 1, ':')))
 		strlcat(newctx, cx, newlen);
-	debug3("%s: setting context from '%s' to '%s'", __func__, oldctx,
-	    newctx);
+	debug3("%s: setting context from '%s' to '%s'", __func__,
+	    oldctx, newctx);
 	if (setcon(newctx) < 0)
-		logit("%s: setcon failed with %s", __func__, strerror (errno));
+		switchlog("%s: setcon %s from %s failed with %s", __func__,
+		    newctx, oldctx, strerror(errno));
 	xfree(oldctx);
 	xfree(newctx);
 }
diff --git a/crypto/openssh/openbsd-compat/port-linux.h b/crypto/openssh/openbsd-compat/port-linux.h
index c2f6184001c1..e3d1004aadd9 100644
--- a/crypto/openssh/openbsd-compat/port-linux.h
+++ b/crypto/openssh/openbsd-compat/port-linux.h
@@ -1,4 +1,4 @@
-/* $Id: port-linux.h,v 1.4.10.1 2011/02/04 00:42:21 djm Exp $ */
+/* $Id: port-linux.h,v 1.5 2011/01/25 01:16:18 djm Exp $ */
 
 /*
  * Copyright (c) 2006 Damien Miller <djm@openbsd.org>
diff --git a/crypto/openssh/openbsd-compat/sha2.c b/crypto/openssh/openbsd-compat/sha2.c
old mode 100755
new mode 100644
diff --git a/crypto/openssh/openbsd-compat/sha2.h b/crypto/openssh/openbsd-compat/sha2.h
old mode 100755
new mode 100644
diff --git a/crypto/openssh/packet.c b/crypto/openssh/packet.c
index 6f2bdc390af9..8e8a297f9b41 100644
--- a/crypto/openssh/packet.c
+++ b/crypto/openssh/packet.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: packet.c,v 1.172 2010/11/13 23:27:50 djm Exp $ */
+/* $OpenBSD: packet.c,v 1.173 2011/05/06 21:14:05 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -426,10 +426,8 @@ packet_set_state(int mode, u_int32_t seqnr, u_int64_t blocks, u_int32_t packets,
 	state->bytes = bytes;
 }
 
-/* returns 1 if connection is via ipv4 */
-
-int
-packet_connection_is_ipv4(void)
+static int
+packet_connection_af(void)
 {
 	struct sockaddr_storage to;
 	socklen_t tolen = sizeof(to);
@@ -443,9 +441,9 @@ packet_connection_is_ipv4(void)
 #ifdef IPV4_IN_IPV6
 	if (to.ss_family == AF_INET6 &&
 	    IN6_IS_ADDR_V4MAPPED(&((struct sockaddr_in6 *)&to)->sin6_addr))
-		return 1;
+		return AF_INET;
 #endif
-	return 0;
+	return to.ss_family;
 }
 
 /* Sets the connection into non-blocking mode. */
@@ -1756,16 +1754,30 @@ packet_not_very_much_data_to_write(void)
 static void
 packet_set_tos(int tos)
 {
-#if defined(IP_TOS) && !defined(IP_TOS_IS_BROKEN)
-	if (!packet_connection_is_on_socket() ||
-	    !packet_connection_is_ipv4())
+#ifndef IP_TOS_IS_BROKEN
+	if (!packet_connection_is_on_socket())
 		return;
-	debug3("%s: set IP_TOS 0x%02x", __func__, tos);
-	if (setsockopt(active_state->connection_in, IPPROTO_IP, IP_TOS, &tos,
-	    sizeof(tos)) < 0)
-		error("setsockopt IP_TOS %d: %.100s:",
-		    tos, strerror(errno));
-#endif
+	switch (packet_connection_af()) {
+# ifdef IP_TOS
+	case AF_INET:
+		debug3("%s: set IP_TOS 0x%02x", __func__, tos);
+		if (setsockopt(active_state->connection_in,
+		    IPPROTO_IP, IP_TOS, &tos, sizeof(tos)) < 0)
+			error("setsockopt IP_TOS %d: %.100s:",
+			    tos, strerror(errno));
+		break;
+# endif /* IP_TOS */
+# ifdef IPV6_TCLASS
+	case AF_INET6:
+		debug3("%s: set IPV6_TCLASS 0x%02x", __func__, tos);
+		if (setsockopt(active_state->connection_in,
+		    IPPROTO_IPV6, IPV6_TCLASS, &tos, sizeof(tos)) < 0)
+			error("setsockopt IPV6_TCLASS %d: %.100s:",
+			    tos, strerror(errno));
+		break;
+# endif /* IPV6_TCLASS */
+	}
+#endif /* IP_TOS_IS_BROKEN */
 }
 
 /* Informs that the current session is interactive.  Sets IP flags for that. */
diff --git a/crypto/openssh/packet.h b/crypto/openssh/packet.h
index 16e33cdcd53f..7cd0c8b6fcf5 100644
--- a/crypto/openssh/packet.h
+++ b/crypto/openssh/packet.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: packet.h,v 1.55 2010/11/13 23:27:50 djm Exp $ */
+/* $OpenBSD: packet.h,v 1.56 2011/05/06 21:14:05 djm Exp $ */
 /* $FreeBSD$ */
 
 /*
@@ -96,7 +96,6 @@ int      packet_have_data_to_write(void);
 int      packet_not_very_much_data_to_write(void);
 
 int	 packet_connection_is_on_socket(void);
-int	 packet_connection_is_ipv4(void);
 int	 packet_remaining(void);
 void	 packet_send_ignore(int);
 void	 packet_add_padding(u_char);
diff --git a/crypto/openssh/pathnames.h b/crypto/openssh/pathnames.h
index 1b6e789b4b24..813af9a49349 100644
--- a/crypto/openssh/pathnames.h
+++ b/crypto/openssh/pathnames.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: pathnames.h,v 1.20 2010/08/31 11:54:45 djm Exp $ */
+/* $OpenBSD: pathnames.h,v 1.22 2011/05/23 03:30:07 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
diff --git a/crypto/openssh/readconf.c b/crypto/openssh/readconf.c
index 17a93a641241..c6c3354d2775 100644
--- a/crypto/openssh/readconf.c
+++ b/crypto/openssh/readconf.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: readconf.c,v 1.190 2010/11/13 23:27:50 djm Exp $ */
+/* $OpenBSD: readconf.c,v 1.193 2011/05/24 07:15:47 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -138,7 +138,7 @@ typedef enum {
 	oHashKnownHosts,
 	oTunnel, oTunnelDevice, oLocalCommand, oPermitLocalCommand,
 	oVisualHostKey, oUseRoaming, oZeroKnowledgePasswordAuthentication,
-	oKexAlgorithms, oIPQoS,
+	oKexAlgorithms, oIPQoS, oRequestTTY,
 	oHPNDisabled, oHPNBufferSize, oTcpRcvBufPoll, oTcpRcvBuf,
 #ifdef NONE_CIPHER_ENABLED
 	oNoneEnabled, oNoneSwitch,
@@ -202,9 +202,9 @@ static struct {
 	{ "host", oHost },
 	{ "escapechar", oEscapeChar },
 	{ "globalknownhostsfile", oGlobalKnownHostsFile },
-	{ "globalknownhostsfile2", oGlobalKnownHostsFile2 },	/* obsolete */
+	{ "globalknownhostsfile2", oDeprecated },
 	{ "userknownhostsfile", oUserKnownHostsFile },
-	{ "userknownhostsfile2", oUserKnownHostsFile2 },	/* obsolete */
+	{ "userknownhostsfile2", oDeprecated }, 
 	{ "connectionattempts", oConnectionAttempts },
 	{ "batchmode", oBatchMode },
 	{ "checkhostip", oCheckHostIP },
@@ -254,6 +254,7 @@ static struct {
 #endif
 	{ "kexalgorithms", oKexAlgorithms },
 	{ "ipqos", oIPQoS },
+	{ "requesttty", oRequestTTY },
 	{ "hpndisabled", oHPNDisabled },
 	{ "hpnbuffersize", oHPNBufferSize },
 	{ "tcprcvbufpoll", oTcpRcvBufPoll },
@@ -262,7 +263,6 @@ static struct {
 	{ "noneenabled", oNoneEnabled },
 	{ "noneswitch", oNoneSwitch },
 #endif
-
 	{ "versionaddendum", oVersionAddendum },
 	{ NULL, oBadOption }
 };
@@ -383,8 +383,10 @@ process_config_line(Options *options, const char *host,
 		    char *line, const char *filename, int linenum,
 		    int *activep)
 {
-	char *s, **charptr, *endofnumber, *keyword, *arg, *arg2, fwdarg[256];
-	int opcode, *intptr, value, value2, scale;
+	char *s, **charptr, *endofnumber, *keyword, *arg, *arg2;
+	char **cpptr, fwdarg[256];
+	u_int *uintptr, max_entries = 0;
+	int negated, opcode, *intptr, value, value2, scale;
 	LogLevel *log_level_ptr;
 	long long orig, val64;
 	size_t len;
@@ -627,26 +629,33 @@ process_config_line(Options *options, const char *host,
 parse_string:
 		arg = strdelim(&s);
 		if (!arg || *arg == '\0')
-			fatal("%.200s line %d: Missing argument.", filename, linenum);
+			fatal("%.200s line %d: Missing argument.",
+			    filename, linenum);
 		if (*activep && *charptr == NULL)
 			*charptr = xstrdup(arg);
 		break;
 
 	case oGlobalKnownHostsFile:
-		charptr = &options->system_hostfile;
-		goto parse_string;
+		cpptr = (char **)&options->system_hostfiles;
+		uintptr = &options->num_system_hostfiles;
+		max_entries = SSH_MAX_HOSTS_FILES;
+parse_char_array:
+		if (*activep && *uintptr == 0) {
+			while ((arg = strdelim(&s)) != NULL && *arg != '\0') {
+				if ((*uintptr) >= max_entries)
+					fatal("%s line %d: "
+					    "too many authorized keys files.",
+					    filename, linenum);
+				cpptr[(*uintptr)++] = xstrdup(arg);
+			}
+		}
+		return 0;
 
 	case oUserKnownHostsFile:
-		charptr = &options->user_hostfile;
-		goto parse_string;
-
-	case oGlobalKnownHostsFile2:
-		charptr = &options->system_hostfile2;
-		goto parse_string;
-
-	case oUserKnownHostsFile2:
-		charptr = &options->user_hostfile2;
-		goto parse_string;
+		cpptr = (char **)&options->user_hostfiles;
+		uintptr = &options->num_user_hostfiles;
+		max_entries = SSH_MAX_HOSTS_FILES;
+		goto parse_char_array;
 
 	case oHostName:
 		charptr = &options->hostname;
@@ -823,12 +832,28 @@ process_config_line(Options *options, const char *host,
 
 	case oHost:
 		*activep = 0;
-		while ((arg = strdelim(&s)) != NULL && *arg != '\0')
+		arg2 = NULL;
+		while ((arg = strdelim(&s)) != NULL && *arg != '\0') {
+			negated = *arg == '!';
+			if (negated)
+				arg++;
 			if (match_pattern(host, arg)) {
-				debug("Applying options for %.100s", arg);
+				if (negated) {
+					debug("%.200s line %d: Skipping Host "
+					    "block because of negated match "
+					    "for %.100s", filename, linenum,
+					    arg);
+					*activep = 0;
+					break;
+				}
+				if (!*activep)
+					arg2 = arg; /* logged below */
 				*activep = 1;
-				break;
 			}
+		}
+		if (*activep)
+			debug("%.200s line %d: Applying options for %.100s",
+			    filename, linenum, arg2);
 		/* Avoid garbage check below, as strdelim is done. */
 		return 0;
 
@@ -1027,11 +1052,24 @@ process_config_line(Options *options, const char *host,
 		intptr = &options->use_roaming;
 		goto parse_flag;
 
-	case oVersionAddendum:
-		ssh_version_set_addendum(strtok(s, "\n"));
-		do {
-			arg = strdelim(&s);
-		} while (arg != NULL && *arg != '\0');
+	case oRequestTTY:
+		arg = strdelim(&s);
+		if (!arg || *arg == '\0')
+			fatal("%s line %d: missing argument.",
+			    filename, linenum);
+		intptr = &options->request_tty;
+		if (strcasecmp(arg, "yes") == 0)
+			value = REQUEST_TTY_YES;
+		else if (strcasecmp(arg, "no") == 0)
+			value = REQUEST_TTY_NO;
+		else if (strcasecmp(arg, "force") == 0)
+			value = REQUEST_TTY_FORCE;
+		else if (strcasecmp(arg, "auto") == 0)
+			value = REQUEST_TTY_AUTO;
+		else
+			fatal("Unsupported RequestTTY \"%s\"", arg);
+		if (*activep && *intptr == -1)
+			*intptr = value;
 		break;
 
 	case oHPNDisabled:
@@ -1075,6 +1113,13 @@ process_config_line(Options *options, const char *host,
 	        }
 #endif
 
+	case oVersionAddendum:
+		ssh_version_set_addendum(strtok(s, "\n"));
+		do {
+			arg = strdelim(&s);
+		} while (arg != NULL && *arg != '\0');
+		break;
+
 	case oDeprecated:
 		debug("%s line %d: Deprecated option \"%s\"",
 		    filename, linenum, keyword);
@@ -1199,10 +1244,8 @@ initialize_options(Options * options)
 	options->proxy_command = NULL;
 	options->user = NULL;
 	options->escape_char = -1;
-	options->system_hostfile = NULL;
-	options->user_hostfile = NULL;
-	options->system_hostfile2 = NULL;
-	options->user_hostfile2 = NULL;
+	options->num_system_hostfiles = 0;
+	options->num_user_hostfiles = 0;
 	options->local_forwards = NULL;
 	options->num_local_forwards = 0;
 	options->remote_forwards = NULL;
@@ -1235,6 +1278,7 @@ initialize_options(Options * options)
 	options->zero_knowledge_password_authentication = -1;
 	options->ip_qos_interactive = -1;
 	options->ip_qos_bulk = -1;
+	options->request_tty = -1;
 	options->hpn_disabled = -1;
 	options->hpn_buffer_size = -1;
 	options->tcp_rcv_buf_poll = -1;
@@ -1349,14 +1393,18 @@ fill_default_options(Options * options)
 	}
 	if (options->escape_char == -1)
 		options->escape_char = '~';
-	if (options->system_hostfile == NULL)
-		options->system_hostfile = _PATH_SSH_SYSTEM_HOSTFILE;
-	if (options->user_hostfile == NULL)
-		options->user_hostfile = _PATH_SSH_USER_HOSTFILE;
-	if (options->system_hostfile2 == NULL)
-		options->system_hostfile2 = _PATH_SSH_SYSTEM_HOSTFILE2;
-	if (options->user_hostfile2 == NULL)
-		options->user_hostfile2 = _PATH_SSH_USER_HOSTFILE2;
+	if (options->num_system_hostfiles == 0) {
+		options->system_hostfiles[options->num_system_hostfiles++] =
+		    xstrdup(_PATH_SSH_SYSTEM_HOSTFILE);
+		options->system_hostfiles[options->num_system_hostfiles++] =
+		    xstrdup(_PATH_SSH_SYSTEM_HOSTFILE2);
+	}
+	if (options->num_user_hostfiles == 0) {
+		options->user_hostfiles[options->num_user_hostfiles++] =
+		    xstrdup(_PATH_SSH_USER_HOSTFILE);
+		options->user_hostfiles[options->num_user_hostfiles++] =
+		    xstrdup(_PATH_SSH_USER_HOSTFILE2);
+	}
 	if (options->log_level == SYSLOG_LEVEL_NOT_SET)
 		options->log_level = SYSLOG_LEVEL_INFO;
 	if (options->clear_forwardings == 1)
@@ -1401,6 +1449,8 @@ fill_default_options(Options * options)
 		options->ip_qos_interactive = IPTOS_LOWDELAY;
 	if (options->ip_qos_bulk == -1)
 		options->ip_qos_bulk = IPTOS_THROUGHPUT;
+	if (options->request_tty == -1)
+		options->request_tty = REQUEST_TTY_AUTO;
 	/* options->local_command should not be set by default */
 	/* options->proxy_command should not be set by default */
 	/* options->user will be set in the main program if appropriate */
diff --git a/crypto/openssh/readconf.h b/crypto/openssh/readconf.h
index 195f6e74d62a..af89bb484ea0 100644
--- a/crypto/openssh/readconf.h
+++ b/crypto/openssh/readconf.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: readconf.h,v 1.88 2010/11/13 23:27:50 djm Exp $ */
+/* $OpenBSD: readconf.h,v 1.90 2011/05/24 07:15:47 djm Exp $ */
 /* $FreeBSD$ */
 
 /*
@@ -28,7 +28,8 @@ typedef struct {
 }       Forward;
 /* Data structure for representing option data. */
 
-#define MAX_SEND_ENV	256
+#define MAX_SEND_ENV		256
+#define SSH_MAX_HOSTS_FILES	256
 
 typedef struct {
 	int     forward_agent;	/* Forward authentication agent. */
@@ -84,10 +85,10 @@ typedef struct {
 	char   *user;		/* User to log in as. */
 	int     escape_char;	/* Escape character; -2 = none */
 
-	char   *system_hostfile;/* Path for /etc/ssh/ssh_known_hosts. */
-	char   *user_hostfile;	/* Path for $HOME/.ssh/known_hosts. */
-	char   *system_hostfile2;
-	char   *user_hostfile2;
+	u_int	num_system_hostfiles;	/* Paths for /etc/ssh/ssh_known_hosts */
+	char   *system_hostfiles[SSH_MAX_HOSTS_FILES];
+	u_int	num_user_hostfiles;	/* Path for $HOME/.ssh/known_hosts */
+	char   *user_hostfiles[SSH_MAX_HOSTS_FILES];
 	char   *preferred_authentications;
 	char   *bind_address;	/* local socket address for connection to sshd */
 	char   *pkcs11_provider; /* PKCS#11 provider */
@@ -132,6 +133,7 @@ typedef struct {
 	int	visual_host_key;
 
 	int	use_roaming;
+	int	request_tty;
 
 	int	hpn_disabled;	/* Switch to disable HPN buffer management. */
 	int	hpn_buffer_size;	/* User definable size for HPN buffer
@@ -152,6 +154,11 @@ typedef struct {
 #define SSHCTL_MASTER_ASK	3
 #define SSHCTL_MASTER_AUTO_ASK	4
 
+#define REQUEST_TTY_AUTO	0
+#define REQUEST_TTY_NO		1
+#define REQUEST_TTY_YES		2
+#define REQUEST_TTY_FORCE	3
+
 void     initialize_options(Options *);
 void     fill_default_options(Options *);
 int	 read_config_file(const char *, const char *, Options *, int);
diff --git a/crypto/openssh/sandbox-darwin.c b/crypto/openssh/sandbox-darwin.c
new file mode 100644
index 000000000000..69901ef143a5
--- /dev/null
+++ b/crypto/openssh/sandbox-darwin.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "includes.h"
+
+#ifdef SANDBOX_DARWIN
+
+#include <sys/types.h>
+
+#include <sandbox.h>
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "log.h"
+#include "sandbox.h"
+#include "xmalloc.h"
+
+/* Darwin/OS X sandbox */
+
+struct ssh_sandbox {
+	pid_t child_pid;
+};
+
+struct ssh_sandbox *
+ssh_sandbox_init(void)
+{
+	struct ssh_sandbox *box;
+
+	/*
+	 * Strictly, we don't need to maintain any state here but we need
+	 * to return non-NULL to satisfy the API.
+	 */
+	debug3("%s: preparing Darwin sandbox", __func__);
+	box = xcalloc(1, sizeof(*box));
+	box->child_pid = 0;
+
+	return box;
+}
+
+void
+ssh_sandbox_child(struct ssh_sandbox *box)
+{
+	char *errmsg;
+	struct rlimit rl_zero;
+
+	debug3("%s: starting Darwin sandbox", __func__);
+	if (sandbox_init(kSBXProfilePureComputation, SANDBOX_NAMED,
+	    &errmsg) == -1)
+		fatal("%s: sandbox_init: %s", __func__, errmsg);
+
+	/*
+	 * The kSBXProfilePureComputation still allows sockets, so
+	 * we must disable these using rlimit.
+	 */
+	rl_zero.rlim_cur = rl_zero.rlim_max = 0;
+	if (setrlimit(RLIMIT_FSIZE, &rl_zero) == -1)
+		fatal("%s: setrlimit(RLIMIT_FSIZE, { 0, 0 }): %s",
+			__func__, strerror(errno));
+	if (setrlimit(RLIMIT_NOFILE, &rl_zero) == -1)
+		fatal("%s: setrlimit(RLIMIT_NOFILE, { 0, 0 }): %s",
+			__func__, strerror(errno));
+	if (setrlimit(RLIMIT_NPROC, &rl_zero) == -1)
+		fatal("%s: setrlimit(RLIMIT_NPROC, { 0, 0 }): %s",
+			__func__, strerror(errno));
+}
+
+void
+ssh_sandbox_parent_finish(struct ssh_sandbox *box)
+{
+	free(box);
+	debug3("%s: finished", __func__);
+}
+
+void
+ssh_sandbox_parent_preauth(struct ssh_sandbox *box, pid_t child_pid)
+{
+	box->child_pid = child_pid;
+}
+
+#endif /* SANDBOX_DARWIN */
diff --git a/crypto/openssh/sandbox-null.c b/crypto/openssh/sandbox-null.c
new file mode 100644
index 000000000000..29fa9669fd85
--- /dev/null
+++ b/crypto/openssh/sandbox-null.c
@@ -0,0 +1,72 @@
+/* $OpenBSD$ */
+/*
+ * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "includes.h"
+
+#ifdef SANDBOX_NULL
+
+#include <sys/types.h>
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "log.h"
+#include "ssh-sandbox.h"
+#include "xmalloc.h"
+
+/* dummy sandbox */
+
+struct ssh_sandbox {
+	int junk;
+};
+
+struct ssh_sandbox *
+ssh_sandbox_init(void)
+{
+	struct ssh_sandbox *box;
+
+	/*
+	 * Strictly, we don't need to maintain any state here but we need
+	 * to return non-NULL to satisfy the API.
+	 */
+	box = xcalloc(1, sizeof(*box));
+	return box;
+}
+
+void
+ssh_sandbox_child(struct ssh_sandbox *box)
+{
+	/* Nothing to do here */
+}
+
+void
+ssh_sandbox_parent_finish(struct ssh_sandbox *box)
+{
+	free(box);
+}
+
+void
+ssh_sandbox_parent_preauth(struct ssh_sandbox *box, pid_t child_pid)
+{
+	/* Nothing to do here */
+}
+
+#endif /* SANDBOX_NULL */
diff --git a/crypto/openssh/sandbox-rlimit.c b/crypto/openssh/sandbox-rlimit.c
new file mode 100644
index 000000000000..761e9284f838
--- /dev/null
+++ b/crypto/openssh/sandbox-rlimit.c
@@ -0,0 +1,93 @@
+/* $OpenBSD: sandbox-rlimit.c,v 1.3 2011/06/23 09:34:13 djm Exp $ */
+/*
+ * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "includes.h"
+
+#ifdef SANDBOX_RLIMIT
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "log.h"
+#include "ssh-sandbox.h"
+#include "xmalloc.h"
+
+/* Minimal sandbox that sets zero nfiles, nprocs and filesize rlimits */
+
+struct ssh_sandbox {
+	pid_t child_pid;
+};
+
+struct ssh_sandbox *
+ssh_sandbox_init(void)
+{
+	struct ssh_sandbox *box;
+
+	/*
+	 * Strictly, we don't need to maintain any state here but we need
+	 * to return non-NULL to satisfy the API.
+	 */
+	debug3("%s: preparing rlimit sandbox", __func__);
+	box = xcalloc(1, sizeof(*box));
+	box->child_pid = 0;
+
+	return box;
+}
+
+void
+ssh_sandbox_child(struct ssh_sandbox *box)
+{
+	struct rlimit rl_zero;
+
+	rl_zero.rlim_cur = rl_zero.rlim_max = 0;
+
+	if (setrlimit(RLIMIT_FSIZE, &rl_zero) == -1)
+		fatal("%s: setrlimit(RLIMIT_FSIZE, { 0, 0 }): %s",
+			__func__, strerror(errno));
+	if (setrlimit(RLIMIT_NOFILE, &rl_zero) == -1)
+		fatal("%s: setrlimit(RLIMIT_NOFILE, { 0, 0 }): %s",
+			__func__, strerror(errno));
+#ifdef HAVE_RLIMIT_NPROC
+	if (setrlimit(RLIMIT_NPROC, &rl_zero) == -1)
+		fatal("%s: setrlimit(RLIMIT_NPROC, { 0, 0 }): %s",
+			__func__, strerror(errno));
+#endif
+}
+
+void
+ssh_sandbox_parent_finish(struct ssh_sandbox *box)
+{
+	free(box);
+	debug3("%s: finished", __func__);
+}
+
+void
+ssh_sandbox_parent_preauth(struct ssh_sandbox *box, pid_t child_pid)
+{
+	box->child_pid = child_pid;
+}
+
+#endif /* SANDBOX_RLIMIT */
diff --git a/crypto/openssh/sandbox-systrace.c b/crypto/openssh/sandbox-systrace.c
new file mode 100644
index 000000000000..5a39f4fe146f
--- /dev/null
+++ b/crypto/openssh/sandbox-systrace.c
@@ -0,0 +1,198 @@
+/* $OpenBSD: sandbox-systrace.c,v 1.4 2011/07/29 14:42:45 djm Exp $ */
+/*
+ * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "includes.h"
+
+#ifdef SANDBOX_SYSTRACE
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/ioctl.h>
+#include <sys/syscall.h>
+#include <sys/socket.h>
+
+#include <dev/systrace.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "atomicio.h"
+#include "log.h"
+#include "ssh-sandbox.h"
+#include "xmalloc.h"
+
+struct sandbox_policy {
+	int syscall;
+	int action;
+};
+
+/* Permitted syscalls in preauth. Unlisted syscalls get SYSTR_POLICY_KILL */
+static const struct sandbox_policy preauth_policy[] = {
+	{ SYS_open, SYSTR_POLICY_NEVER },
+
+	{ SYS___sysctl, SYSTR_POLICY_PERMIT },
+	{ SYS_close, SYSTR_POLICY_PERMIT },
+	{ SYS_exit, SYSTR_POLICY_PERMIT },
+	{ SYS_getpid, SYSTR_POLICY_PERMIT },
+	{ SYS_gettimeofday, SYSTR_POLICY_PERMIT },
+	{ SYS_madvise, SYSTR_POLICY_PERMIT },
+	{ SYS_mmap, SYSTR_POLICY_PERMIT },
+	{ SYS_mprotect, SYSTR_POLICY_PERMIT },
+	{ SYS_poll, SYSTR_POLICY_PERMIT },
+	{ SYS_munmap, SYSTR_POLICY_PERMIT },
+	{ SYS_read, SYSTR_POLICY_PERMIT },
+	{ SYS_select, SYSTR_POLICY_PERMIT },
+	{ SYS_sigprocmask, SYSTR_POLICY_PERMIT },
+	{ SYS_write, SYSTR_POLICY_PERMIT },
+	{ -1, -1 }
+};
+
+struct ssh_sandbox {
+	int child_sock;
+	int parent_sock;
+	int systrace_fd;
+	pid_t child_pid;
+};
+
+struct ssh_sandbox *
+ssh_sandbox_init(void)
+{
+	struct ssh_sandbox *box;
+	int s[2];
+
+	debug3("%s: preparing systrace sandbox", __func__);
+	box = xcalloc(1, sizeof(*box));
+	if (socketpair(AF_UNIX, SOCK_STREAM, 0, s) == -1)
+		fatal("%s: socketpair: %s", __func__, strerror(errno));
+	box->child_sock = s[0];
+	box->parent_sock = s[1];
+	box->systrace_fd = -1;
+	box->child_pid = 0;
+
+	return box;
+}
+
+void
+ssh_sandbox_child(struct ssh_sandbox *box)
+{
+	char whatever = 0;
+
+	close(box->parent_sock);
+	/* Signal parent that we are ready */
+	debug3("%s: ready", __func__);
+	if (atomicio(vwrite, box->child_sock, &whatever, 1) != 1)
+		fatal("%s: write: %s", __func__, strerror(errno));
+	/* Wait for parent to signal for us to go */
+	if (atomicio(read, box->child_sock, &whatever, 1) != 1)
+		fatal("%s: read: %s", __func__, strerror(errno));
+	debug3("%s: started", __func__);
+	close(box->child_sock);
+}
+
+static void
+ssh_sandbox_parent(struct ssh_sandbox *box, pid_t child_pid,
+    const struct sandbox_policy *allowed_syscalls)
+{
+	int dev_systrace, i, j, found;
+	char whatever = 0;
+	struct systrace_policy policy;
+
+	debug3("%s: wait for child %ld", __func__, (long)child_pid);
+	box->child_pid = child_pid;
+	close(box->child_sock);
+	/* Wait for child to signal that it is ready */
+	if (atomicio(read, box->parent_sock, &whatever, 1) != 1)
+		fatal("%s: read: %s", __func__, strerror(errno));
+	debug3("%s: child %ld ready", __func__, (long)child_pid);
+
+	/* Set up systracing of child */
+	if ((dev_systrace = open("/dev/systrace", O_RDONLY)) == -1)
+		fatal("%s: open(\"/dev/systrace\"): %s", __func__,
+		    strerror(errno));
+	if (ioctl(dev_systrace, STRIOCCLONE, &box->systrace_fd) == -1)
+		fatal("%s: ioctl(STRIOCCLONE, %d): %s", __func__,
+		    dev_systrace, strerror(errno));
+	close(dev_systrace);
+	debug3("%s: systrace attach, fd=%d", __func__, box->systrace_fd);
+	if (ioctl(box->systrace_fd, STRIOCATTACH, &child_pid) == -1)
+		fatal("%s: ioctl(%d, STRIOCATTACH, %d): %s", __func__,
+		    box->systrace_fd, child_pid, strerror(errno));
+
+	/* Allocate and assign policy */
+	bzero(&policy, sizeof(policy));
+	policy.strp_op = SYSTR_POLICY_NEW;
+	policy.strp_maxents = SYS_MAXSYSCALL;
+	if (ioctl(box->systrace_fd, STRIOCPOLICY, &policy) == -1)
+		fatal("%s: ioctl(%d, STRIOCPOLICY (new)): %s", __func__,
+		    box->systrace_fd, strerror(errno));
+
+	policy.strp_op = SYSTR_POLICY_ASSIGN;
+	policy.strp_pid = box->child_pid;
+	if (ioctl(box->systrace_fd, STRIOCPOLICY, &policy) == -1)
+		fatal("%s: ioctl(%d, STRIOCPOLICY (assign)): %s",
+		    __func__, box->systrace_fd, strerror(errno));
+
+	/* Set per-syscall policy */
+	for (i = 0; i < SYS_MAXSYSCALL; i++) {
+		found = 0;
+		for (j = 0; allowed_syscalls[j].syscall != -1; j++) {
+			if (allowed_syscalls[j].syscall == i) {
+				found = 1;
+				break;
+			}
+		}
+		policy.strp_op = SYSTR_POLICY_MODIFY;
+		policy.strp_code = i;
+		policy.strp_policy = found ?
+		    allowed_syscalls[j].action : SYSTR_POLICY_KILL;
+		if (found)
+			debug3("%s: policy: enable syscall %d", __func__, i);
+		if (ioctl(box->systrace_fd, STRIOCPOLICY, &policy) == -1)
+			fatal("%s: ioctl(%d, STRIOCPOLICY (modify)): %s",
+			    __func__, box->systrace_fd, strerror(errno));
+	}
+
+	/* Signal the child to start running */
+	debug3("%s: start child %ld", __func__, (long)child_pid);
+	if (atomicio(vwrite, box->parent_sock, &whatever, 1) != 1)
+		fatal("%s: write: %s", __func__, strerror(errno));
+	close(box->parent_sock);
+}
+
+void
+ssh_sandbox_parent_finish(struct ssh_sandbox *box)
+{
+	/* Closing this before the child exits will terminate it */
+	close(box->systrace_fd);
+
+	free(box);
+	debug3("%s: finished", __func__);
+}
+
+void
+ssh_sandbox_parent_preauth(struct ssh_sandbox *box, pid_t child_pid)
+{
+	ssh_sandbox_parent(box, child_pid, preauth_policy);
+}
+
+#endif /* SANDBOX_SYSTRACE */
diff --git a/crypto/openssh/servconf.c b/crypto/openssh/servconf.c
index 96761e7251de..1d88bbf8b9e6 100644
--- a/crypto/openssh/servconf.c
+++ b/crypto/openssh/servconf.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: servconf.c,v 1.213 2010/11/13 23:27:50 djm Exp $ */
+/* $OpenBSD: servconf.c,v 1.222 2011/06/22 21:57:01 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -129,8 +129,7 @@ initialize_server_options(ServerOptions *options)
 	options->use_dns = -1;
 	options->client_alive_interval = -1;
 	options->client_alive_count_max = -1;
-	options->authorized_keys_file = NULL;
-	options->authorized_keys_file2 = NULL;
+	options->num_authkeys_files = 0;
 	options->num_accept_env = 0;
 	options->permit_tun = -1;
 	options->num_permitted_opens = -1;
@@ -273,15 +272,12 @@ fill_default_server_options(ServerOptions *options)
 		options->client_alive_interval = 0;
 	if (options->client_alive_count_max == -1)
 		options->client_alive_count_max = 3;
-	if (options->authorized_keys_file2 == NULL) {
-		/* authorized_keys_file2 falls back to authorized_keys_file */
-		if (options->authorized_keys_file != NULL)
-			options->authorized_keys_file2 = xstrdup(options->authorized_keys_file);
-		else
-			options->authorized_keys_file2 = xstrdup(_PATH_SSH_USER_PERMITTED_KEYS2);
+	if (options->num_authkeys_files == 0) {
+		options->authorized_keys_files[options->num_authkeys_files++] =
+		    xstrdup(_PATH_SSH_USER_PERMITTED_KEYS);
+		options->authorized_keys_files[options->num_authkeys_files++] =
+		    xstrdup(_PATH_SSH_USER_PERMITTED_KEYS2);
 	}
-	if (options->authorized_keys_file == NULL)
-		options->authorized_keys_file = xstrdup(_PATH_SSH_USER_PERMITTED_KEYS);
 	if (options->permit_tun == -1)
 		options->permit_tun = SSH_TUNMODE_NO;
 	if (options->zero_knowledge_password_authentication == -1)
@@ -324,7 +320,7 @@ fill_default_server_options(ServerOptions *options)
 
 	/* Turn privilege separation on by default */
 	if (use_privsep == -1)
-		use_privsep = 1;
+		use_privsep = PRIVSEP_ON;
 
 #ifndef HAVE_MMAP
 	if (use_privsep && options->compression == 1) {
@@ -361,7 +357,7 @@ typedef enum {
 	sMaxStartups, sMaxAuthTries, sMaxSessions,
 	sBanner, sUseDNS, sHostbasedAuthentication,
 	sHostbasedUsesNameFromPacketOnly, sClientAliveInterval,
-	sClientAliveCountMax, sAuthorizedKeysFile, sAuthorizedKeysFile2,
+	sClientAliveCountMax, sAuthorizedKeysFile,
 	sGssAuthentication, sGssCleanupCreds, sAcceptEnv, sPermitTunnel,
 	sMatch, sPermitOpen, sForceCommand, sChrootDirectory,
 	sUsePrivilegeSeparation, sAllowAgentForwarding,
@@ -483,7 +479,7 @@ static struct {
 	{ "clientaliveinterval", sClientAliveInterval, SSHCFG_GLOBAL },
 	{ "clientalivecountmax", sClientAliveCountMax, SSHCFG_GLOBAL },
 	{ "authorizedkeysfile", sAuthorizedKeysFile, SSHCFG_ALL },
-	{ "authorizedkeysfile2", sAuthorizedKeysFile2, SSHCFG_ALL },
+	{ "authorizedkeysfile2", sDeprecated, SSHCFG_ALL },
 	{ "useprivilegeseparation", sUsePrivilegeSeparation, SSHCFG_GLOBAL},
 	{ "acceptenv", sAcceptEnv, SSHCFG_GLOBAL },
 	{ "permittunnel", sPermitTunnel, SSHCFG_ALL },
@@ -727,6 +723,43 @@ match_cfg_line(char **condition, int line, const char *user, const char *host,
 
 #define WHITESPACE " \t\r\n"
 
+/* Multistate option parsing */
+struct multistate {
+	char *key;
+	int value;
+};
+static const struct multistate multistate_addressfamily[] = {
+	{ "inet",			AF_INET },
+	{ "inet6",			AF_INET6 },
+	{ "any",			AF_UNSPEC },
+	{ NULL, -1 }
+};
+static const struct multistate multistate_permitrootlogin[] = {
+	{ "without-password",		PERMIT_NO_PASSWD },
+	{ "forced-commands-only",	PERMIT_FORCED_ONLY },
+	{ "yes",			PERMIT_YES },
+	{ "no",				PERMIT_NO },
+	{ NULL, -1 }
+};
+static const struct multistate multistate_compression[] = {
+	{ "delayed",			COMP_DELAYED },
+	{ "yes",			COMP_ZLIB },
+	{ "no",				COMP_NONE },
+	{ NULL, -1 }
+};
+static const struct multistate multistate_gatewayports[] = {
+	{ "clientspecified",		2 },
+	{ "yes",			1 },
+	{ "no",				0 },
+	{ NULL, -1 }
+};
+static const struct multistate multistate_privsep[] = {
+	{ "sandbox",			PRIVSEP_SANDBOX },
+	{ "yes",			PRIVSEP_ON },
+	{ "no",				PRIVSEP_OFF },
+	{ NULL, -1 }
+};
+
 int
 process_server_config_line(ServerOptions *options, char *line,
     const char *filename, int linenum, int *activep, const char *user,
@@ -740,6 +773,7 @@ process_server_config_line(ServerOptions *options, char *line,
 	int port;
 	u_int i, flags = 0;
 	size_t len;
+	const struct multistate *multistate_ptr;
 
 	cp = line;
 	if ((arg = strdelim(&cp)) == NULL)
@@ -855,24 +889,27 @@ process_server_config_line(ServerOptions *options, char *line,
 		break;
 
 	case sAddressFamily:
+		intptr = &options->address_family;
+		multistate_ptr = multistate_addressfamily;
+		if (options->listen_addrs != NULL)
+			fatal("%s line %d: address family must be specified "
+			    "before ListenAddress.", filename, linenum);
+ parse_multistate:
 		arg = strdelim(&cp);
 		if (!arg || *arg == '\0')
-			fatal("%s line %d: missing address family.",
+			fatal("%s line %d: missing argument.",
 			    filename, linenum);
-		intptr = &options->address_family;
-		if (options->listen_addrs != NULL)
-			fatal("%s line %d: address family must be specified before "
-			    "ListenAddress.", filename, linenum);
-		if (strcasecmp(arg, "inet") == 0)
-			value = AF_INET;
-		else if (strcasecmp(arg, "inet6") == 0)
-			value = AF_INET6;
-		else if (strcasecmp(arg, "any") == 0)
-			value = AF_UNSPEC;
-		else
-			fatal("%s line %d: unsupported address family \"%s\".",
+		value = -1;
+		for (i = 0; multistate_ptr[i].key != NULL; i++) {
+			if (strcasecmp(arg, multistate_ptr[i].key) == 0) {
+				value = multistate_ptr[i].value;
+				break;
+			}
+		}
+		if (value == -1)
+			fatal("%s line %d: unsupported option \"%s\".",
 			    filename, linenum, arg);
-		if (*intptr == -1)
+		if (*activep && *intptr == -1)
 			*intptr = value;
 		break;
 
@@ -911,27 +948,8 @@ process_server_config_line(ServerOptions *options, char *line,
 
 	case sPermitRootLogin:
 		intptr = &options->permit_root_login;
-		arg = strdelim(&cp);
-		if (!arg || *arg == '\0')
-			fatal("%s line %d: missing yes/"
-			    "without-password/forced-commands-only/no "
-			    "argument.", filename, linenum);
-		value = 0;	/* silence compiler */
-		if (strcmp(arg, "without-password") == 0)
-			value = PERMIT_NO_PASSWD;
-		else if (strcmp(arg, "forced-commands-only") == 0)
-			value = PERMIT_FORCED_ONLY;
-		else if (strcmp(arg, "yes") == 0)
-			value = PERMIT_YES;
-		else if (strcmp(arg, "no") == 0)
-			value = PERMIT_NO;
-		else
-			fatal("%s line %d: Bad yes/"
-			    "without-password/forced-commands-only/no "
-			    "argument: %s", filename, linenum, arg);
-		if (*activep && *intptr == -1)
-			*intptr = value;
-		break;
+		multistate_ptr = multistate_permitrootlogin;
+		goto parse_multistate;
 
 	case sIgnoreRhosts:
 		intptr = &options->ignore_rhosts;
@@ -1062,43 +1080,13 @@ process_server_config_line(ServerOptions *options, char *line,
 
 	case sCompression:
 		intptr = &options->compression;
-		arg = strdelim(&cp);
-		if (!arg || *arg == '\0')
-			fatal("%s line %d: missing yes/no/delayed "
-			    "argument.", filename, linenum);
-		value = 0;	/* silence compiler */
-		if (strcmp(arg, "delayed") == 0)
-			value = COMP_DELAYED;
-		else if (strcmp(arg, "yes") == 0)
-			value = COMP_ZLIB;
-		else if (strcmp(arg, "no") == 0)
-			value = COMP_NONE;
-		else
-			fatal("%s line %d: Bad yes/no/delayed "
-			    "argument: %s", filename, linenum, arg);
-		if (*intptr == -1)
-			*intptr = value;
-		break;
+		multistate_ptr = multistate_compression;
+		goto parse_multistate;
 
 	case sGatewayPorts:
 		intptr = &options->gateway_ports;
-		arg = strdelim(&cp);
-		if (!arg || *arg == '\0')
-			fatal("%s line %d: missing yes/no/clientspecified "
-			    "argument.", filename, linenum);
-		value = 0;	/* silence compiler */
-		if (strcmp(arg, "clientspecified") == 0)
-			value = 2;
-		else if (strcmp(arg, "yes") == 0)
-			value = 1;
-		else if (strcmp(arg, "no") == 0)
-			value = 0;
-		else
-			fatal("%s line %d: Bad yes/no/clientspecified "
-			    "argument: %s", filename, linenum, arg);
-		if (*activep && *intptr == -1)
-			*intptr = value;
-		break;
+		multistate_ptr = multistate_gatewayports;
+		goto parse_multistate;
 
 	case sUseDNS:
 		intptr = &options->use_dns;
@@ -1136,7 +1124,8 @@ process_server_config_line(ServerOptions *options, char *line,
 
 	case sUsePrivilegeSeparation:
 		intptr = &use_privsep;
-		goto parse_flag;
+		multistate_ptr = multistate_privsep;
+		goto parse_multistate;
 
 	case sAllowUsers:
 		while ((arg = strdelim(&cp)) && *arg != '\0') {
@@ -1302,14 +1291,22 @@ process_server_config_line(ServerOptions *options, char *line,
 	 * AuthorizedKeysFile	/etc/ssh_keys/%u
 	 */
 	case sAuthorizedKeysFile:
-		charptr = &options->authorized_keys_file;
-		goto parse_tilde_filename;
-	case sAuthorizedKeysFile2:
-		charptr = &options->authorized_keys_file2;
-		goto parse_tilde_filename;
+		if (*activep && options->num_authkeys_files == 0) {
+			while ((arg = strdelim(&cp)) && *arg != '\0') {
+				if (options->num_authkeys_files >=
+				    MAX_AUTHKEYS_FILES)
+					fatal("%s line %d: "
+					    "too many authorized keys files.",
+					    filename, linenum);
+				options->authorized_keys_files[
+				    options->num_authkeys_files++] =
+				    tilde_expand_filename(arg, getuid());
+			}
+		}
+		return 0;
+
 	case sAuthorizedPrincipalsFile:
 		charptr = &options->authorized_principals_file;
- parse_tilde_filename:
 		arg = strdelim(&cp);
 		if (!arg || *arg == '\0')
 			fatal("%s line %d: missing file name.",
@@ -1553,6 +1550,12 @@ parse_server_match_config(ServerOptions *options, const char *user,
 		dst->n = src->n; \
 	} \
 } while(0)
+#define M_CP_STRARRAYOPT(n, num_n) do {\
+	if (src->num_n != 0) { \
+		for (dst->num_n = 0; dst->num_n < src->num_n; dst->num_n++) \
+			dst->n[dst->num_n] = xstrdup(src->n[dst->num_n]); \
+	} \
+} while(0)
 
 /*
  * Copy any supported values that are set.
@@ -1588,20 +1591,23 @@ copy_set_server_options(ServerOptions *dst, ServerOptions *src, int preauth)
 	M_CP_INTOPT(ip_qos_interactive);
 	M_CP_INTOPT(ip_qos_bulk);
 
-	M_CP_STROPT(banner);
+	/* See comment in servconf.h */
+	COPY_MATCH_STRING_OPTS();
+
+	/*
+	 * The only things that should be below this point are string options
+	 * which are only used after authentication.
+	 */
 	if (preauth)
 		return;
+
 	M_CP_STROPT(adm_forced_command);
 	M_CP_STROPT(chroot_directory);
-	M_CP_STROPT(trusted_user_ca_keys);
-	M_CP_STROPT(revoked_keys_file);
-	M_CP_STROPT(authorized_keys_file);
-	M_CP_STROPT(authorized_keys_file2);
-	M_CP_STROPT(authorized_principals_file);
 }
 
 #undef M_CP_INTOPT
 #undef M_CP_STROPT
+#undef M_CP_STRARRAYOPT
 
 void
 parse_server_config(ServerOptions *options, const char *filename, Buffer *conf,
@@ -1626,32 +1632,35 @@ parse_server_config(ServerOptions *options, const char *filename, Buffer *conf,
 		    filename, bad_options);
 }
 
+static const char *
+fmt_multistate_int(int val, const struct multistate *m)
+{
+	u_int i;
+
+	for (i = 0; m[i].key != NULL; i++) {
+		if (m[i].value == val)
+			return m[i].key;
+	}
+	return "UNKNOWN";
+}
+
 static const char *
 fmt_intarg(ServerOpCodes code, int val)
 {
-	if (code == sAddressFamily) {
-		switch (val) {
-		case AF_INET:
-			return "inet";
-		case AF_INET6:
-			return "inet6";
-		case AF_UNSPEC:
-			return "any";
-		default:
-			return "UNKNOWN";
-		}
-	}
-	if (code == sPermitRootLogin) {
-		switch (val) {
-		case PERMIT_NO_PASSWD:
-			return "without-password";
-		case PERMIT_FORCED_ONLY:
-			return "forced-commands-only";
-		case PERMIT_YES:
-			return "yes";
-		}
-	}
-	if (code == sProtocol) {
+	if (val == -1)
+		return "unset";
+	switch (code) {
+	case sAddressFamily:
+		return fmt_multistate_int(val, multistate_addressfamily);
+	case sPermitRootLogin:
+		return fmt_multistate_int(val, multistate_permitrootlogin);
+	case sGatewayPorts:
+		return fmt_multistate_int(val, multistate_gatewayports);
+	case sCompression:
+		return fmt_multistate_int(val, multistate_compression);
+	case sUsePrivilegeSeparation:
+		return fmt_multistate_int(val, multistate_privsep);
+	case sProtocol:
 		switch (val) {
 		case SSH_PROTO_1:
 			return "1";
@@ -1662,20 +1671,16 @@ fmt_intarg(ServerOpCodes code, int val)
 		default:
 			return "UNKNOWN";
 		}
+	default:
+		switch (val) {
+		case 0:
+			return "no";
+		case 1:
+			return "yes";
+		default:
+			return "UNKNOWN";
+		}
 	}
-	if (code == sGatewayPorts && val == 2)
-		return "clientspecified";
-	if (code == sCompression && val == COMP_DELAYED)
-		return "delayed";
-	switch (val) {
-	case -1:
-		return "unset";
-	case 0:
-		return "no";
-	case 1:
-		return "yes";
-	}
-	return "UNKNOWN";
 }
 
 static const char *
@@ -1715,7 +1720,18 @@ dump_cfg_strarray(ServerOpCodes code, u_int count, char **vals)
 	u_int i;
 
 	for (i = 0; i < count; i++)
-		printf("%s %s\n", lookup_opcode_name(code),  vals[i]);
+		printf("%s %s\n", lookup_opcode_name(code), vals[i]);
+}
+
+static void
+dump_cfg_strarray_oneline(ServerOpCodes code, u_int count, char **vals)
+{
+	u_int i;
+
+	printf("%s", lookup_opcode_name(code));
+	for (i = 0; i < count; i++)
+		printf(" %s",  vals[i]);
+	printf("\n");
 }
 
 void
@@ -1813,8 +1829,6 @@ dump_config(ServerOptions *o)
 	dump_cfg_string(sCiphers, o->ciphers);
 	dump_cfg_string(sMacs, o->macs);
 	dump_cfg_string(sBanner, o->banner);
-	dump_cfg_string(sAuthorizedKeysFile, o->authorized_keys_file);
-	dump_cfg_string(sAuthorizedKeysFile2, o->authorized_keys_file2);
 	dump_cfg_string(sForceCommand, o->adm_forced_command);
 	dump_cfg_string(sChrootDirectory, o->chroot_directory);
 	dump_cfg_string(sTrustedUserCAKeys, o->trusted_user_ca_keys);
@@ -1827,6 +1841,8 @@ dump_config(ServerOptions *o)
 	dump_cfg_string(sLogFacility, log_facility_name(o->log_facility));
 
 	/* string array arguments */
+	dump_cfg_strarray_oneline(sAuthorizedKeysFile, o->num_authkeys_files,
+	    o->authorized_keys_files);
 	dump_cfg_strarray(sHostKeyFile, o->num_host_key_files,
 	     o->host_key_files);
 	dump_cfg_strarray(sHostKeyFile, o->num_host_cert_files,
@@ -1852,7 +1868,8 @@ dump_config(ServerOptions *o)
 		}
 	dump_cfg_string(sPermitTunnel, s);
 
-	printf("ipqos 0x%02x 0x%02x\n", o->ip_qos_interactive, o->ip_qos_bulk);
+	printf("ipqos %s ", iptos2str(o->ip_qos_interactive));
+	printf("%s\n", iptos2str(o->ip_qos_bulk));
 
 	channel_print_adm_permitted_opens();
 }
diff --git a/crypto/openssh/servconf.h b/crypto/openssh/servconf.h
index cfa8e4b082b0..847b92c1447f 100644
--- a/crypto/openssh/servconf.h
+++ b/crypto/openssh/servconf.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: servconf.h,v 1.95 2010/11/13 23:27:50 djm Exp $ */
+/* $OpenBSD: servconf.h,v 1.99 2011/06/22 21:57:01 djm Exp $ */
 /* $FreeBSD$ */
 
 /*
@@ -28,6 +28,7 @@
 #define MAX_HOSTCERTS		256	/* Max # host certificates. */
 #define MAX_ACCEPT_ENV		256	/* Max # of env vars. */
 #define MAX_MATCH_GROUPS	256	/* Max # of groups for Match. */
+#define MAX_AUTHKEYS_FILES	256	/* Max # of authorized_keys files. */
 
 /* permit_root_login */
 #define	PERMIT_NOT_SET		-1
@@ -36,6 +37,11 @@
 #define	PERMIT_NO_PASSWD	2
 #define	PERMIT_YES		3
 
+/* use_privsep */
+#define PRIVSEP_OFF		0
+#define PRIVSEP_ON		1
+#define PRIVSEP_SANDBOX		2
+
 #define DEFAULT_AUTH_FAIL_MAX	6	/* Default for MaxAuthTries */
 #define DEFAULT_SESSIONS_MAX	10	/* Default for MaxSessions */
 
@@ -146,8 +152,8 @@ typedef struct {
 					 * disconnect the session
 					 */
 
-	char   *authorized_keys_file;	/* File containing public keys */
-	char   *authorized_keys_file2;
+	u_int num_authkeys_files;	/* Files containing public keys */
+	char   *authorized_keys_files[MAX_AUTHKEYS_FILES];
 
 	char   *adm_forced_command;
 
@@ -172,6 +178,20 @@ typedef struct {
 #endif
 }       ServerOptions;
 
+/*
+ * These are string config options that must be copied between the
+ * Match sub-config and the main config, and must be sent from the
+ * privsep slave to the privsep master. We use a macro to ensure all
+ * the options are copied and the copies are done in the correct order.
+ */
+#define COPY_MATCH_STRING_OPTS() do { \
+		M_CP_STROPT(banner); \
+		M_CP_STROPT(trusted_user_ca_keys); \
+		M_CP_STROPT(revoked_keys_file); \
+		M_CP_STROPT(authorized_principals_file); \
+		M_CP_STRARRAYOPT(authorized_keys_files, num_authkeys_files); \
+	} while (0)
+
 void	 initialize_server_options(ServerOptions *);
 void	 fill_default_server_options(ServerOptions *);
 int	 process_server_config_line(ServerOptions *, char *, const char *, int,
diff --git a/crypto/openssh/serverloop.c b/crypto/openssh/serverloop.c
index 7606937cca3d..e74c2c726abd 100644
--- a/crypto/openssh/serverloop.c
+++ b/crypto/openssh/serverloop.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: serverloop.c,v 1.159 2009/05/28 16:50:16 andreas Exp $ */
+/* $OpenBSD: serverloop.c,v 1.160 2011/05/15 08:09:01 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -132,8 +132,8 @@ notify_setup(void)
 {
 	if (pipe(notify_pipe) < 0) {
 		error("pipe(notify_pipe) failed %s", strerror(errno));
-	} else if ((fcntl(notify_pipe[0], F_SETFD, 1) == -1) ||
-	    (fcntl(notify_pipe[1], F_SETFD, 1) == -1)) {
+	} else if ((fcntl(notify_pipe[0], F_SETFD, FD_CLOEXEC) == -1) ||
+	    (fcntl(notify_pipe[1], F_SETFD, FD_CLOEXEC) == -1)) {
 		error("fcntl(notify_pipe, F_SETFD) failed %s", strerror(errno));
 		close(notify_pipe[0]);
 		close(notify_pipe[1]);
diff --git a/crypto/openssh/session.c b/crypto/openssh/session.c
index b04e629974b1..0f008134166f 100644
--- a/crypto/openssh/session.c
+++ b/crypto/openssh/session.c
@@ -98,6 +98,10 @@ __RCSID("$FreeBSD$");
 #include <kafs.h>
 #endif
 
+#ifdef WITH_SELINUX
+#include <selinux/selinux.h>
+#endif
+
 #define IS_INTERNAL_SFTP(c) \
 	(!strncmp(c, INTERNAL_SFTP_NAME, sizeof(INTERNAL_SFTP_NAME) - 1) && \
 	 (c[sizeof(INTERNAL_SFTP_NAME) - 1] == '\0' || \
@@ -1547,6 +1551,9 @@ do_pwchange(Session *s)
 	if (s->ttyfd != -1) {
 		fprintf(stderr,
 		    "You must change your password now and login again!\n");
+#ifdef WITH_SELINUX
+		setexeccon(NULL);
+#endif
 #ifdef PASSWD_NEEDS_USERNAME
 		execl(_PATH_PASSWD_PROG, "passwd", s->pw->pw_name,
 		    (char *)NULL);
diff --git a/crypto/openssh/sftp-server.c b/crypto/openssh/sftp-server.c
index b268d08835f5..9d01c7d79819 100644
--- a/crypto/openssh/sftp-server.c
+++ b/crypto/openssh/sftp-server.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-server.c,v 1.93 2010/12/04 00:18:01 djm Exp $ */
+/* $OpenBSD: sftp-server.c,v 1.94 2011/06/17 21:46:16 djm Exp $ */
 /*
  * Copyright (c) 2000-2004 Markus Friedl.  All rights reserved.
  *
@@ -68,7 +68,7 @@ Buffer iqueue;
 Buffer oqueue;
 
 /* Version of client */
-int version;
+u_int version;
 
 /* Disable writes */
 int readonly;
@@ -522,7 +522,7 @@ process_init(void)
 	Buffer msg;
 
 	version = get_int();
-	verbose("received client version %d", version);
+	verbose("received client version %u", version);
 	buffer_init(&msg);
 	buffer_put_char(&msg, SSH2_FXP_VERSION);
 	buffer_put_int(&msg, SSH2_FILEXFER_VERSION);
diff --git a/crypto/openssh/sftp.1 b/crypto/openssh/sftp.1
index 416948291804..389c8e8ee520 100644
--- a/crypto/openssh/sftp.1
+++ b/crypto/openssh/sftp.1
@@ -1,4 +1,4 @@
-.\" $OpenBSD: sftp.1,v 1.88 2010/12/04 00:18:01 djm Exp $
+.\" $OpenBSD: sftp.1,v 1.90 2011/08/07 12:55:30 dtucker Exp $
 .\" $FreeBSD$
 .\"
 .\" Copyright (c) 2001 Damien Miller.  All rights reserved.
@@ -23,7 +23,7 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd December 4, 2010
+.Dd August 7, 2011
 .Dt SFTP 1
 .Os
 .Sh NAME
@@ -90,6 +90,10 @@ to obviate the need to enter a password at connection time (see
 and
 .Xr ssh-keygen 1
 for details).
+.Pp
+Since some usage formats use colon characters to delimit host names from path
+names, IPv6 addresses must be enclosed in square brackets to avoid ambiguity.
+.Pp
 The options are as follows:
 .Bl -tag -width Ds
 .It Fl 1
@@ -481,7 +485,7 @@ is specified, then
 .Ar remote-path
 must specify a directory.
 .Pp
-If ether the
+If either the
 .Fl P
 or
 .Fl p
diff --git a/crypto/openssh/ssh-add.c b/crypto/openssh/ssh-add.c
index 125d6645b19e..6d5e2a95757f 100644
--- a/crypto/openssh/ssh-add.c
+++ b/crypto/openssh/ssh-add.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-add.c,v 1.100 2010/08/31 12:33:38 djm Exp $ */
+/* $OpenBSD: ssh-add.c,v 1.101 2011/05/04 21:15:29 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -145,8 +145,12 @@ add_file(AuthenticationConnection *ac, const char *filename)
 	char *comment = NULL;
 	char msg[1024], *certpath;
 	int fd, perms_ok, ret = -1;
+	Buffer keyblob;
 
-	if ((fd = open(filename, O_RDONLY)) < 0) {
+	if (strcmp(filename, "-") == 0) {
+		fd = STDIN_FILENO;
+		filename = "(stdin)";
+	} else if ((fd = open(filename, O_RDONLY)) < 0) {
 		perror(filename);
 		return -1;
 	}
@@ -155,18 +159,28 @@ add_file(AuthenticationConnection *ac, const char *filename)
 	 * Since we'll try to load a keyfile multiple times, permission errors
 	 * will occur multiple times, so check perms first and bail if wrong.
 	 */
-	perms_ok = key_perm_ok(fd, filename);
-	close(fd);
-	if (!perms_ok)
+	if (fd != STDIN_FILENO) {
+		perms_ok = key_perm_ok(fd, filename);
+		if (!perms_ok) {
+			close(fd);
+			return -1;
+		}
+	}
+	buffer_init(&keyblob);
+	if (!key_load_file(fd, filename, &keyblob)) {
+		buffer_free(&keyblob);
+		close(fd);
 		return -1;
+	}
+	close(fd);
 
 	/* At first, try empty passphrase */
-	private = key_load_private(filename, "", &comment);
+	private = key_parse_private(&keyblob, filename, "", &comment);
 	if (comment == NULL)
 		comment = xstrdup(filename);
 	/* try last */
 	if (private == NULL && pass != NULL)
-		private = key_load_private(filename, pass, NULL);
+		private = key_parse_private(&keyblob, filename, pass, NULL);
 	if (private == NULL) {
 		/* clear passphrase since it did not work */
 		clear_pass();
@@ -177,9 +191,11 @@ add_file(AuthenticationConnection *ac, const char *filename)
 			if (strcmp(pass, "") == 0) {
 				clear_pass();
 				xfree(comment);
+				buffer_free(&keyblob);
 				return -1;
 			}
-			private = key_load_private(filename, pass, &comment);
+			private = key_parse_private(&keyblob, filename, pass,
+			    &comment);
 			if (private != NULL)
 				break;
 			clear_pass();
@@ -187,6 +203,7 @@ add_file(AuthenticationConnection *ac, const char *filename)
 			    "Bad passphrase, try again for %.200s: ", comment);
 		}
 	}
+	buffer_free(&keyblob);
 
 	if (ssh_add_identity_constrained(ac, private, comment, lifetime,
 	    confirm)) {
@@ -372,7 +389,6 @@ main(int argc, char **argv)
 	sanitise_stdfd();
 
 	__progname = ssh_get_progname(argv[0]);
-	init_rng();
 	seed_rng();
 
 	OpenSSL_add_all_algorithms();
diff --git a/crypto/openssh/ssh-agent.1 b/crypto/openssh/ssh-agent.1
index 7f4942f2f39b..b14461570b16 100644
--- a/crypto/openssh/ssh-agent.1
+++ b/crypto/openssh/ssh-agent.1
@@ -44,7 +44,7 @@
 .Sh SYNOPSIS
 .Nm ssh-agent
 .Op Fl c | s
-.Op Fl d
+.Op Fl dx
 .Op Fl a Ar bind_address
 .Op Fl t Ar life
 .Op Ar command Op Ar arg ...
@@ -103,6 +103,8 @@ A lifetime specified for an identity with
 .Xr ssh-add 1
 overrides this value.
 Without this option the default maximum lifetime is forever.
+.It Fl x
+Exit after the last client has disconnected.
 .El
 .Pp
 If a commandline is given, this is executed as a subprocess of the agent.
diff --git a/crypto/openssh/ssh-agent.c b/crypto/openssh/ssh-agent.c
index d8af4e460530..9d4e0c9c6c72 100644
--- a/crypto/openssh/ssh-agent.c
+++ b/crypto/openssh/ssh-agent.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-agent.c,v 1.171 2010/11/21 01:01:13 djm Exp $ */
+/* $OpenBSD: ssh-agent.c,v 1.172 2011/06/03 01:37:40 dtucker Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -138,15 +138,34 @@ extern char *__progname;
 /* Default lifetime (0 == forever) */
 static int lifetime = 0;
 
+/*
+ * Client connection count; incremented in new_socket() and decremented in
+ * close_socket().  When it reaches 0, ssh-agent will exit.  Since it is
+ * normally initialized to 1, it will never reach 0.  However, if the -x
+ * option is specified, it is initialized to 0 in main(); in that case,
+ * ssh-agent will exit as soon as it has had at least one client but no
+ * longer has any.
+ */
+static int xcount = 1;
+
 static void
 close_socket(SocketEntry *e)
 {
+	int last = 0;
+
+	if (e->type == AUTH_CONNECTION) {
+		debug("xcount %d -> %d", xcount, xcount - 1);
+		if (--xcount == 0)
+			last = 1;
+	}
 	close(e->fd);
 	e->fd = -1;
 	e->type = AUTH_UNUSED;
 	buffer_free(&e->input);
 	buffer_free(&e->output);
 	buffer_free(&e->request);
+	if (last)
+		cleanup_exit(0);
 }
 
 static void
@@ -901,6 +920,10 @@ new_socket(sock_type type, int fd)
 {
 	u_int i, old_alloc, new_alloc;
 
+	if (type == AUTH_CONNECTION) {
+		debug("xcount %d -> %d", xcount, xcount + 1);
+		++xcount;
+	}
 	set_nonblock(fd);
 
 	if (fd > max_fd)
@@ -1098,7 +1121,11 @@ cleanup_handler(int sig)
 static void
 check_parent_exists(void)
 {
-	if (parent_pid != -1 && kill(parent_pid, 0) < 0) {
+	/*
+	 * If our parent has exited then getppid() will return (pid_t)1,
+	 * so testing for that should be safe.
+	 */
+	if (parent_pid != -1 && getppid() != parent_pid) {
 		/* printf("Parent has died - Authentication agent exiting.\n"); */
 		cleanup_socket();
 		_exit(2);
@@ -1117,6 +1144,7 @@ usage(void)
 	fprintf(stderr, "  -d          Debug mode.\n");
 	fprintf(stderr, "  -a socket   Bind agent socket to given name.\n");
 	fprintf(stderr, "  -t life     Default identity lifetime (seconds).\n");
+	fprintf(stderr, "  -x          Exit when the last client disconnects.\n");
 	exit(1);
 }
 
@@ -1156,10 +1184,9 @@ main(int ac, char **av)
 	OpenSSL_add_all_algorithms();
 
 	__progname = ssh_get_progname(av[0]);
-	init_rng();
 	seed_rng();
 
-	while ((ch = getopt(ac, av, "cdksa:t:")) != -1) {
+	while ((ch = getopt(ac, av, "cdksa:t:x")) != -1) {
 		switch (ch) {
 		case 'c':
 			if (s_flag)
@@ -1188,6 +1215,9 @@ main(int ac, char **av)
 				usage();
 			}
 			break;
+		case 'x':
+			xcount = 0;
+			break;
 		default:
 			usage();
 		}
@@ -1347,8 +1377,7 @@ main(int ac, char **av)
 	if (ac > 0)
 		parent_alive_interval = 10;
 	idtab_init();
-	if (!d_flag)
-		signal(SIGINT, SIG_IGN);
+	signal(SIGINT, d_flag ? cleanup_handler : SIG_IGN);
 	signal(SIGPIPE, SIG_IGN);
 	signal(SIGHUP, cleanup_handler);
 	signal(SIGTERM, cleanup_handler);
diff --git a/crypto/openssh/ssh-keygen.1 b/crypto/openssh/ssh-keygen.1
index 2e8727ba1860..6c55c2271283 100644
--- a/crypto/openssh/ssh-keygen.1
+++ b/crypto/openssh/ssh-keygen.1
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: ssh-keygen.1,v 1.101 2010/10/28 18:33:28 jmc Exp $
+.\"	$OpenBSD: ssh-keygen.1,v 1.106 2011/04/13 04:09:37 djm Exp $
 .\"	$FreeBSD$
 .\"
 .\" Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -36,7 +36,7 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd October 28, 2010
+.Dd April 13, 2011
 .Dt SSH-KEYGEN 1
 .Os
 .Sh NAME
@@ -118,6 +118,8 @@
 .Nm ssh-keygen
 .Fl L
 .Op Fl f Ar input_keyfile
+.Nm ssh-keygen
+.Fl A
 .Ek
 .Sh DESCRIPTION
 .Nm
@@ -174,9 +176,8 @@ The passphrase can be changed later by using the
 option.
 .Pp
 There is no way to recover a lost passphrase.
-If the passphrase is
-lost or forgotten, a new key must be generated and copied to the
-corresponding public key to other machines.
+If the passphrase is lost or forgotten, a new key must be generated
+and the corresponding public key copied to other machines.
 .Pp
 For RSA1 keys,
 there is also a comment field in the key file that is only for
@@ -193,6 +194,13 @@ should be placed to be activated.
 .Pp
 The options are as follows:
 .Bl -tag -width Ds
+.It Fl A
+For each of the key types (rsa1, rsa, dsa and ecdsa) for which host keys
+do not exist, generate the host keys with the default key file path,
+an empty passphrase, default bits for the key type, and default comment.
+This is used by
+.Pa /etc/rc
+to generate new host keys.
 .It Fl a Ar trials
 Specifies the number of primality tests to perform when screening DH-GEX
 candidates using the
@@ -205,6 +213,12 @@ Specifies the number of bits in the key to create.
 For RSA keys, the minimum size is 768 bits and the default is 2048 bits.
 Generally, 2048 bits is considered sufficient.
 DSA keys must be exactly 1024 bits as specified by FIPS 186-2.
+For ECDSA keys, the
+.Fl b
+flag determines they key length by selecting from one of three elliptic
+curve sizes: 256, 384 or 521 bits.
+Attempting to use bit lengths other than these three values for ECDSA keys
+will fail.
 .It Fl C Ar comment
 Provides a new comment.
 .It Fl c
@@ -394,9 +408,6 @@ new passphrase.
 .It Fl q
 Silence
 .Nm ssh-keygen .
-Used by
-.Pa /etc/rc
-when creating a new key.
 .It Fl R Ar hostname
 Removes all keys belonging to
 .Ar hostname
diff --git a/crypto/openssh/ssh-keygen.c b/crypto/openssh/ssh-keygen.c
index c95e4ab2987f..4b6218b10f88 100644
--- a/crypto/openssh/ssh-keygen.c
+++ b/crypto/openssh/ssh-keygen.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-keygen.c,v 1.205 2011/01/11 06:13:10 djm Exp $ */
+/* $OpenBSD: ssh-keygen.c,v 1.210 2011/04/18 00:46:05 djm Exp $ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
  * Copyright (c) 1994 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -49,10 +49,7 @@
 #include "hostfile.h"
 #include "dns.h"
 #include "ssh2.h"
-
-#ifdef ENABLE_PKCS11
 #include "ssh-pkcs11.h"
-#endif
 
 /* Number of bits in the RSA/DSA key.  This value can be set on the command line. */
 #define DEFAULT_BITS		2048
@@ -159,6 +156,38 @@ char hostname[MAXHOSTNAMELEN];
 int gen_candidates(FILE *, u_int32_t, u_int32_t, BIGNUM *);
 int prime_test(FILE *, FILE *, u_int32_t, u_int32_t);
 
+static void
+type_bits_valid(int type, u_int32_t *bitsp)
+{
+	u_int maxbits;
+
+	if (type == KEY_UNSPEC) {
+		fprintf(stderr, "unknown key type %s\n", key_type_name);
+		exit(1);
+	}
+	if (*bitsp == 0) {
+		if (type == KEY_DSA)
+			*bitsp = DEFAULT_BITS_DSA;
+		else if (type == KEY_ECDSA)
+			*bitsp = DEFAULT_BITS_ECDSA;
+		else
+			*bitsp = DEFAULT_BITS;
+	}
+	maxbits = (type == KEY_DSA) ?
+	    OPENSSL_DSA_MAX_MODULUS_BITS : OPENSSL_RSA_MAX_MODULUS_BITS;
+	if (*bitsp > maxbits) {
+		fprintf(stderr, "key bits exceeds maximum %d\n", maxbits);
+		exit(1);
+	}
+	if (type == KEY_DSA && *bitsp != 1024)
+		fatal("DSA keys must be 1024 bits");
+	else if (type != KEY_ECDSA && *bitsp < 768)
+		fatal("Key must at least be 768 bits");
+	else if (type == KEY_ECDSA && key_ecdsa_bits_to_nid(*bitsp) == -1)
+		fatal("Invalid ECDSA key length - valid lengths are "
+		    "256, 384 or 521 bits");
+}
+
 static void
 ask_filename(struct passwd *pw, const char *prompt)
 {
@@ -817,6 +846,98 @@ do_fingerprint(struct passwd *pw)
 	exit(0);
 }
 
+static void
+do_gen_all_hostkeys(struct passwd *pw)
+{
+	struct {
+		char *key_type;
+		char *key_type_display;
+		char *path;
+	} key_types[] = {
+		{ "rsa1", "RSA1", _PATH_HOST_KEY_FILE },
+		{ "rsa", "RSA" ,_PATH_HOST_RSA_KEY_FILE },
+		{ "dsa", "DSA", _PATH_HOST_DSA_KEY_FILE },
+		{ "ecdsa", "ECDSA",_PATH_HOST_ECDSA_KEY_FILE },
+		{ NULL, NULL, NULL }
+	};
+
+	int first = 0;
+	struct stat st;
+	Key *private, *public;
+	char comment[1024];
+	int i, type, fd;
+	FILE *f;
+
+	for (i = 0; key_types[i].key_type; i++) {
+		if (stat(key_types[i].path, &st) == 0)
+			continue;
+		if (errno != ENOENT) {
+			printf("Could not stat %s: %s", key_types[i].path,
+			    strerror(errno));
+			first = 0;
+			continue;
+		}
+
+		if (first == 0) {
+			first = 1;
+			printf("%s: generating new host keys: ", __progname);
+		}
+		printf("%s ", key_types[i].key_type_display);
+		fflush(stdout);
+		arc4random_stir();
+		type = key_type_from_name(key_types[i].key_type);
+		strlcpy(identity_file, key_types[i].path, sizeof(identity_file));
+		bits = 0;
+		type_bits_valid(type, &bits);
+		private = key_generate(type, bits);
+		if (private == NULL) {
+			fprintf(stderr, "key_generate failed\n");
+			first = 0;
+			continue;
+		}
+		public  = key_from_private(private);
+		snprintf(comment, sizeof comment, "%s@%s", pw->pw_name,
+		    hostname);
+		if (!key_save_private(private, identity_file, "", comment)) {
+			printf("Saving the key failed: %s.\n", identity_file);
+			key_free(private);
+			key_free(public);
+			first = 0;
+			continue;
+		}
+		key_free(private);
+		arc4random_stir();
+		strlcat(identity_file, ".pub", sizeof(identity_file));
+		fd = open(identity_file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+		if (fd == -1) {
+			printf("Could not save your public key in %s\n",
+			    identity_file);
+			key_free(public);
+			first = 0;
+			continue;
+		}
+		f = fdopen(fd, "w");
+		if (f == NULL) {
+			printf("fdopen %s failed\n", identity_file);
+			key_free(public);
+			first = 0;
+			continue;
+		}
+		if (!key_write(public, f)) {
+			fprintf(stderr, "write key failed\n");
+			key_free(public);
+			first = 0;
+			continue;
+		}
+		fprintf(f, " %s\n", comment);
+		fclose(f);
+		key_free(public);
+
+	}
+	if (first != 0)
+		printf("\n");
+}
+
 static void
 printhost(FILE *f, const char *name, Key *public, int ca, int hash)
 {
@@ -1329,6 +1450,9 @@ prepare_options_buf(Buffer *c, int which)
 	if ((which & OPTIONS_CRITICAL) != 0 &&
 	    certflags_command != NULL)
 		add_string_option(c, "force-command", certflags_command);
+	if ((which & OPTIONS_EXTENSIONS) != 0 &&
+	    (certflags_flags & CERTOPT_X_FWD) != 0)
+		add_flag_option(c, "permit-X11-forwarding");
 	if ((which & OPTIONS_EXTENSIONS) != 0 &&
 	    (certflags_flags & CERTOPT_AGENT_FWD) != 0)
 		add_flag_option(c, "permit-agent-forwarding");
@@ -1341,9 +1465,6 @@ prepare_options_buf(Buffer *c, int which)
 	if ((which & OPTIONS_EXTENSIONS) != 0 &&
 	    (certflags_flags & CERTOPT_USER_RC) != 0)
 		add_flag_option(c, "permit-user-rc");
-	if ((which & OPTIONS_EXTENSIONS) != 0 &&
-	    (certflags_flags & CERTOPT_X_FWD) != 0)
-		add_flag_option(c, "permit-X11-forwarding");
 	if ((which & OPTIONS_CRITICAL) != 0 &&
 	    certflags_src_addr != NULL)
 		add_string_option(c, "source-address", certflags_src_addr);
@@ -1593,7 +1714,7 @@ add_cert_option(char *opt)
 {
 	char *val;
 
-	if (strcmp(opt, "clear") == 0)
+	if (strcasecmp(opt, "clear") == 0)
 		certflags_flags = 0;
 	else if (strcasecmp(opt, "no-x11-forwarding") == 0)
 		certflags_flags &= ~CERTOPT_X_FWD;
@@ -1745,6 +1866,7 @@ usage(void)
 {
 	fprintf(stderr, "usage: %s [options]\n", __progname);
 	fprintf(stderr, "Options:\n");
+	fprintf(stderr, "  -A          Generate non-existent host keys for all key types.\n");
 	fprintf(stderr, "  -a trials   Number of trials for screening DH-GEX moduli.\n");
 	fprintf(stderr, "  -B          Show bubblebabble digest of key file.\n");
 	fprintf(stderr, "  -b bits     Number of bits in the key to create.\n");
@@ -1799,9 +1921,9 @@ main(int argc, char **argv)
 	struct passwd *pw;
 	struct stat st;
 	int opt, type, fd;
-	u_int maxbits;
 	u_int32_t memory = 0, generator_wanted = 0, trials = 100;
 	int do_gen_candidates = 0, do_screen_candidates = 0;
+	int gen_all_hostkeys = 0;
 	BIGNUM *start = NULL;
 	FILE *f;
 	const char *errstr;
@@ -1817,7 +1939,6 @@ main(int argc, char **argv)
 	OpenSSL_add_all_algorithms();
 	log_init(argv[0], SYSLOG_LEVEL_INFO, SYSLOG_FACILITY_USER, 1);
 
-	init_rng();
 	seed_rng();
 
 	/* we need this for the home * directory.  */
@@ -1831,9 +1952,12 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
-	while ((opt = getopt(argc, argv, "degiqpclBHLhvxXyF:b:f:t:D:I:P:m:N:n:"
+	while ((opt = getopt(argc, argv, "AegiqpclBHLhvxXyF:b:f:t:D:I:P:m:N:n:"
 	    "O:C:r:g:R:T:G:M:S:s:a:V:W:z:")) != -1) {
 		switch (opt) {
+		case 'A':
+			gen_all_hostkeys = 1;
+			break;
 		case 'b':
 			bits = (u_int32_t)strtonum(optarg, 256, 32768, &errstr);
 			if (errstr)
@@ -1928,9 +2052,6 @@ main(int argc, char **argv)
 		case 'y':
 			print_public = 1;
 			break;
-		case 'd':
-			key_type_name = "dsa";
-			break;
 		case 's':
 			ca_key_path = optarg;
 			break;
@@ -2109,37 +2230,19 @@ main(int argc, char **argv)
 		return (0);
 	}
 
+	if (gen_all_hostkeys) {
+		do_gen_all_hostkeys(pw);
+		return (0);
+	}
+
 	arc4random_stir();
 
 	if (key_type_name == NULL)
 		key_type_name = "rsa";
 
 	type = key_type_from_name(key_type_name);
-	if (type == KEY_UNSPEC) {
-		fprintf(stderr, "unknown key type %s\n", key_type_name);
-		exit(1);
-	}
-	if (bits == 0) {
-		if (type == KEY_DSA)
-			bits = DEFAULT_BITS_DSA;
-		else if (type == KEY_ECDSA)
-			bits = DEFAULT_BITS_ECDSA;
-		else
-			bits = DEFAULT_BITS;
-	}
-	maxbits = (type == KEY_DSA) ?
-	    OPENSSL_DSA_MAX_MODULUS_BITS : OPENSSL_RSA_MAX_MODULUS_BITS;
-	if (bits > maxbits) {
-		fprintf(stderr, "key bits exceeds maximum %d\n", maxbits);
-		exit(1);
-	}
-	if (type == KEY_DSA && bits != 1024)
-		fatal("DSA keys must be 1024 bits");
-	else if (type != KEY_ECDSA && bits < 768)
-		fatal("Key must at least be 768 bits");
-	else if (type == KEY_ECDSA && key_ecdsa_bits_to_nid(bits) == -1)
-		fatal("Invalid ECDSA key length - valid lengths are "
-		    "256, 384 or 521 bits");
+	type_bits_valid(type, &bits);
+
 	if (!quiet)
 		printf("Generating public/private %s key pair.\n", key_type_name);
 	private = key_generate(type, bits);
diff --git a/crypto/openssh/ssh-keyscan.c b/crypto/openssh/ssh-keyscan.c
index 25d7ac66f957..b085dd41703d 100644
--- a/crypto/openssh/ssh-keyscan.c
+++ b/crypto/openssh/ssh-keyscan.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-keyscan.c,v 1.84 2011/01/04 20:44:13 otto Exp $ */
+/* $OpenBSD: ssh-keyscan.c,v 1.85 2011/03/15 10:36:02 okan Exp $ */
 /*
  * Copyright 1995, 1996 by David Mazieres <dm@lcs.mit.edu>.
  *
@@ -535,7 +535,7 @@ conloop(void)
 			seltime.tv_sec--;
 		}
 	} else
-		seltime.tv_sec = seltime.tv_usec = 0;
+		timerclear(&seltime);
 
 	r = xcalloc(read_wait_nfdset, sizeof(fd_mask));
 	e = xcalloc(read_wait_nfdset, sizeof(fd_mask));
@@ -620,7 +620,6 @@ main(int argc, char **argv)
 	extern char *optarg;
 
 	__progname = ssh_get_progname(argv[0]);
-	init_rng();
 	seed_rng();
 	TAILQ_INIT(&tq);
 
diff --git a/crypto/openssh/ssh-keysign.c b/crypto/openssh/ssh-keysign.c
index d05156005cf8..1deb7e14152c 100644
--- a/crypto/openssh/ssh-keysign.c
+++ b/crypto/openssh/ssh-keysign.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-keysign.c,v 1.35 2010/08/31 12:33:38 djm Exp $ */
+/* $OpenBSD: ssh-keysign.c,v 1.36 2011/02/16 00:31:14 djm Exp $ */
 /*
  * Copyright (c) 2002 Markus Friedl.  All rights reserved.
  *
@@ -150,9 +150,10 @@ main(int argc, char **argv)
 {
 	Buffer b;
 	Options options;
-	Key *keys[2], *key = NULL;
+#define NUM_KEYTYPES 3
+	Key *keys[NUM_KEYTYPES], *key = NULL;
 	struct passwd *pw;
-	int key_fd[2], i, found, version = 2, fd;
+	int key_fd[NUM_KEYTYPES], i, found, version = 2, fd;
 	u_char *signature, *data;
 	char *host;
 	u_int slen, dlen;
@@ -165,8 +166,10 @@ main(int argc, char **argv)
 	if (fd > 2)
 		close(fd);
 
-	key_fd[0] = open(_PATH_HOST_RSA_KEY_FILE, O_RDONLY);
-	key_fd[1] = open(_PATH_HOST_DSA_KEY_FILE, O_RDONLY);
+	i = 0;
+	key_fd[i++] = open(_PATH_HOST_DSA_KEY_FILE, O_RDONLY);
+	key_fd[i++] = open(_PATH_HOST_ECDSA_KEY_FILE, O_RDONLY);
+	key_fd[i++] = open(_PATH_HOST_RSA_KEY_FILE, O_RDONLY);
 
 	original_real_uid = getuid();	/* XXX readconf.c needs this */
 	if ((pw = getpwuid(original_real_uid)) == NULL)
@@ -175,7 +178,6 @@ main(int argc, char **argv)
 
 	permanently_set_uid(pw);
 
-	init_rng();
 	seed_rng();
 	arc4random_stir();
 
@@ -191,7 +193,11 @@ main(int argc, char **argv)
 		fatal("ssh-keysign not enabled in %s",
 		    _PATH_HOST_CONFIG_FILE);
 
-	if (key_fd[0] == -1 && key_fd[1] == -1)
+	for (i = found = 0; i < NUM_KEYTYPES; i++) {
+		if (key_fd[i] != -1)
+			found = 1;
+	}
+	if (found == 0)
 		fatal("could not open any host key");
 
 	OpenSSL_add_all_algorithms();
@@ -200,7 +206,7 @@ main(int argc, char **argv)
 	RAND_seed(rnd, sizeof(rnd));
 
 	found = 0;
-	for (i = 0; i < 2; i++) {
+	for (i = 0; i < NUM_KEYTYPES; i++) {
 		keys[i] = NULL;
 		if (key_fd[i] == -1)
 			continue;
@@ -230,7 +236,7 @@ main(int argc, char **argv)
 	xfree(host);
 
 	found = 0;
-	for (i = 0; i < 2; i++) {
+	for (i = 0; i < NUM_KEYTYPES; i++) {
 		if (keys[i] != NULL &&
 		    key_equal_public(key, keys[i])) {
 			found = 1;
diff --git a/crypto/openssh/ssh-pkcs11-helper.c b/crypto/openssh/ssh-pkcs11-helper.c
index 8e3f57acef95..cd33515f62f9 100644
--- a/crypto/openssh/ssh-pkcs11-helper.c
+++ b/crypto/openssh/ssh-pkcs11-helper.c
@@ -280,7 +280,6 @@ main(int argc, char **argv)
 	TAILQ_INIT(&pkcs11_keylist);
 	pkcs11_init(0);
 
-	init_rng();
 	seed_rng();
 	__progname = ssh_get_progname(argv[0]);
 
diff --git a/crypto/openssh/ssh-pkcs11.c b/crypto/openssh/ssh-pkcs11.c
index 286c232c7266..1f4c1c8e4f4c 100644
--- a/crypto/openssh/ssh-pkcs11.c
+++ b/crypto/openssh/ssh-pkcs11.c
@@ -590,4 +590,18 @@ pkcs11_add_provider(char *provider_id, char *pin, Key ***keyp)
 	return (-1);
 }
 
+#else
+
+int
+pkcs11_init(int interactive)
+{
+	return (0);
+}
+
+void
+pkcs11_terminate(void)
+{
+	return;
+}
+
 #endif /* ENABLE_PKCS11 */
diff --git a/crypto/openssh/ssh-rand-helper.8 b/crypto/openssh/ssh-rand-helper.8
deleted file mode 100644
index af5a72fcbb9b..000000000000
--- a/crypto/openssh/ssh-rand-helper.8
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" $Id: ssh-rand-helper.8,v 1.3 2007/01/22 01:44:53 djm Exp $
-.\"
-.\" Copyright (c) 2002 Damien Miller.  All rights reserved.
-.\"
-.\" Redistribution and use in source and binary forms, with or without
-.\" modification, are permitted provided that the following conditions
-.\" are met:
-.\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-.\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
-.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.\"
-.Dd April 14, 2002
-.Dt SSH-RAND-HELPER 8
-.Os
-.Sh NAME
-.Nm ssh-rand-helper
-.Nd random number gatherer for OpenSSH
-.Sh SYNOPSIS
-.Nm ssh-rand-hlper
-.Op Fl vxXh
-.Op Fl b Ar bytes
-.Sh DESCRIPTION
-.Nm
-is a small helper program used by
-.Xr ssh 1 ,
-.Xr ssh-add 1 ,
-.Xr ssh-agent 1 ,
-.Xr ssh-keygen 1 ,
-.Xr ssh-keyscan 1
-and
-.Xr sshd 8
-to gather random numbers of cryptographic quality if the
-.Xr openssl 4
-library has not been configured to provide them itself.
-.Pp
-Normally
-.Nm
-will generate a strong random seed and provide it to the calling
-program via standard output. If standard output is a tty,
-.Nm
-will instead print the seed in hexidecimal format unless told otherwise.
-.Pp
-.Nm
-will by default gather random numbers from the system commands listed
-in
-.Pa /etc/ssh/ssh_prng_cmds .
-The output of each of the commands listed will be hashed and used to
-generate a random seed for the calling program.
-.Nm
-will also store seed files in
-.Pa ~/.ssh/prng_seed
-between executions.
-.Pp
-Alternately,
-.Nm
-may be configured at build time to collect random numbers from a
-EGD/PRNGd server via a unix domain or localhost tcp socket.
-.Pp
-This program is not intended to be run by the end-user, so the few
-commandline options are for debugging purposes only.
-.Bl -tag -width Ds
-.It Fl b Ar bytes
-Specify the number of random bytes to include in the output.
-.It Fl x
-Output a hexidecimal instead of a binary seed.
-.It Fl X
-Force output of a binary seed, even if standard output is a tty
-.It Fl v
-Turn on debugging message. Multiple
-.Fl v
-options will increase the debugging level.
-.It Fl h
-Display a summary of options.
-.El
-.Sh AUTHORS
-Damien Miller <djm@mindrot.org>
-.Sh SEE ALSO
-.Xr ssh 1 ,
-.Xr ssh-add 1 ,
-.Xr ssh-keygen 1 ,
-.Xr sshd 8
diff --git a/crypto/openssh/ssh-rand-helper.c b/crypto/openssh/ssh-rand-helper.c
deleted file mode 100644
index fa5070499e53..000000000000
--- a/crypto/openssh/ssh-rand-helper.c
+++ /dev/null
@@ -1,932 +0,0 @@
-/*
- * Copyright (c) 2001-2002 Damien Miller.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "includes.h"
-
-#include <sys/types.h>
-#include <sys/resource.h>
-#include <sys/stat.h>
-#include <sys/wait.h>
-#include <sys/socket.h>
-
-#include <stdarg.h>
-#include <stddef.h>
-#include <string.h>
-
-#include <netinet/in.h>
-#include <arpa/inet.h>
-
-#ifdef HAVE_SYS_UN_H
-# include <sys/un.h>
-#endif
-
-#include <errno.h>
-#include <fcntl.h>
-#include <pwd.h>
-#include <signal.h>
-#include <time.h>
-#include <unistd.h>
-
-#include <openssl/rand.h>
-#include <openssl/sha.h>
-#include <openssl/crypto.h>
-
-/* SunOS 4.4.4 needs this */
-#ifdef HAVE_FLOATINGPOINT_H
-# include <floatingpoint.h>
-#endif /* HAVE_FLOATINGPOINT_H */
-
-#include "misc.h"
-#include "xmalloc.h"
-#include "atomicio.h"
-#include "pathnames.h"
-#include "log.h"
-
-/* Number of bytes we write out */
-#define OUTPUT_SEED_SIZE	48
-
-/* Length of on-disk seedfiles */
-#define SEED_FILE_SIZE		1024
-
-/* Maximum number of command-line arguments to read from file */
-#define NUM_ARGS		10
-
-/* Minimum number of usable commands to be considered sufficient */
-#define MIN_ENTROPY_SOURCES	16
-
-/* Path to on-disk seed file (relative to user's home directory */
-#ifndef SSH_PRNG_SEED_FILE
-# define SSH_PRNG_SEED_FILE      _PATH_SSH_USER_DIR"/prng_seed"
-#endif
-
-/* Path to PRNG commands list */
-#ifndef SSH_PRNG_COMMAND_FILE
-# define SSH_PRNG_COMMAND_FILE   SSHDIR "/ssh_prng_cmds"
-#endif
-
-extern char *__progname;
-
-#define WHITESPACE " \t\n"
-
-#ifndef RUSAGE_SELF
-# define RUSAGE_SELF 0
-#endif
-#ifndef RUSAGE_CHILDREN
-# define RUSAGE_CHILDREN 0
-#endif
-
-#if !defined(PRNGD_SOCKET) && !defined(PRNGD_PORT)
-# define USE_SEED_FILES
-#endif
-
-typedef struct {
-	/* Proportion of data that is entropy */
-	double rate;
-	/* Counter goes positive if this command times out */
-	unsigned int badness;
-	/* Increases by factor of two each timeout */
-	unsigned int sticky_badness;
-	/* Path to executable */
-	char *path;
-	/* argv to pass to executable */
-	char *args[NUM_ARGS]; /* XXX: arbitrary limit */
-	/* full command string (debug) */
-	char *cmdstring;
-} entropy_cmd_t;
-
-/* slow command timeouts (all in milliseconds) */
-/* static int entropy_timeout_default = ENTROPY_TIMEOUT_MSEC; */
-static int entropy_timeout_current = ENTROPY_TIMEOUT_MSEC;
-
-/* this is initialised from a file, by prng_read_commands() */
-static entropy_cmd_t *entropy_cmds = NULL;
-
-/* Prototypes */
-double stir_from_system(void);
-double stir_from_programs(void);
-double stir_gettimeofday(double entropy_estimate);
-double stir_clock(double entropy_estimate);
-double stir_rusage(int who, double entropy_estimate);
-double hash_command_output(entropy_cmd_t *src, unsigned char *hash);
-int get_random_bytes_prngd(unsigned char *buf, int len,
-    unsigned short tcp_port, char *socket_path);
-
-/*
- * Collect 'len' bytes of entropy into 'buf' from PRNGD/EGD daemon
- * listening either on 'tcp_port', or via Unix domain socket at *
- * 'socket_path'.
- * Either a non-zero tcp_port or a non-null socket_path must be
- * supplied.
- * Returns 0 on success, -1 on error
- */
-int
-get_random_bytes_prngd(unsigned char *buf, int len,
-    unsigned short tcp_port, char *socket_path)
-{
-	int fd, addr_len, rval, errors;
-	u_char msg[2];
-	struct sockaddr_storage addr;
-	struct sockaddr_in *addr_in = (struct sockaddr_in *)&addr;
-	struct sockaddr_un *addr_un = (struct sockaddr_un *)&addr;
-	mysig_t old_sigpipe;
-
-	/* Sanity checks */
-	if (socket_path == NULL && tcp_port == 0)
-		fatal("You must specify a port or a socket");
-	if (socket_path != NULL &&
-	    strlen(socket_path) >= sizeof(addr_un->sun_path))
-		fatal("Random pool path is too long");
-	if (len <= 0 || len > 255)
-		fatal("Too many bytes (%d) to read from PRNGD", len);
-
-	memset(&addr, '\0', sizeof(addr));
-
-	if (tcp_port != 0) {
-		addr_in->sin_family = AF_INET;
-		addr_in->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-		addr_in->sin_port = htons(tcp_port);
-		addr_len = sizeof(*addr_in);
-	} else {
-		addr_un->sun_family = AF_UNIX;
-		strlcpy(addr_un->sun_path, socket_path,
-		    sizeof(addr_un->sun_path));
-		addr_len = offsetof(struct sockaddr_un, sun_path) +
-		    strlen(socket_path) + 1;
-	}
-
-	old_sigpipe = mysignal(SIGPIPE, SIG_IGN);
-
-	errors = 0;
-	rval = -1;
-reopen:
-	fd = socket(addr.ss_family, SOCK_STREAM, 0);
-	if (fd == -1) {
-		error("Couldn't create socket: %s", strerror(errno));
-		goto done;
-	}
-
-	if (connect(fd, (struct sockaddr*)&addr, addr_len) == -1) {
-		if (tcp_port != 0) {
-			error("Couldn't connect to PRNGD port %d: %s",
-			    tcp_port, strerror(errno));
-		} else {
-			error("Couldn't connect to PRNGD socket \"%s\": %s",
-			    addr_un->sun_path, strerror(errno));
-		}
-		goto done;
-	}
-
-	/* Send blocking read request to PRNGD */
-	msg[0] = 0x02;
-	msg[1] = len;
-
-	if (atomicio(vwrite, fd, msg, sizeof(msg)) != sizeof(msg)) {
-		if (errno == EPIPE && errors < 10) {
-			close(fd);
-			errors++;
-			goto reopen;
-		}
-		error("Couldn't write to PRNGD socket: %s",
-		    strerror(errno));
-		goto done;
-	}
-
-	if (atomicio(read, fd, buf, len) != (size_t)len) {
-		if (errno == EPIPE && errors < 10) {
-			close(fd);
-			errors++;
-			goto reopen;
-		}
-		error("Couldn't read from PRNGD socket: %s",
-		    strerror(errno));
-		goto done;
-	}
-
-	rval = 0;
-done:
-	mysignal(SIGPIPE, old_sigpipe);
-	if (fd != -1)
-		close(fd);
-	return rval;
-}
-
-static int
-seed_from_prngd(unsigned char *buf, size_t bytes)
-{
-#ifdef PRNGD_PORT
-	debug("trying egd/prngd port %d", PRNGD_PORT);
-	if (get_random_bytes_prngd(buf, bytes, PRNGD_PORT, NULL) == 0)
-		return 0;
-#endif
-#ifdef PRNGD_SOCKET
-	debug("trying egd/prngd socket %s", PRNGD_SOCKET);
-	if (get_random_bytes_prngd(buf, bytes, 0, PRNGD_SOCKET) == 0)
-		return 0;
-#endif
-	return -1;
-}
-
-double
-stir_gettimeofday(double entropy_estimate)
-{
-	struct timeval tv;
-
-	if (gettimeofday(&tv, NULL) == -1)
-		fatal("Couldn't gettimeofday: %s", strerror(errno));
-
-	RAND_add(&tv, sizeof(tv), entropy_estimate);
-
-	return entropy_estimate;
-}
-
-double
-stir_clock(double entropy_estimate)
-{
-#ifdef HAVE_CLOCK
-	clock_t c;
-
-	c = clock();
-	RAND_add(&c, sizeof(c), entropy_estimate);
-
-	return entropy_estimate;
-#else /* _HAVE_CLOCK */
-	return 0;
-#endif /* _HAVE_CLOCK */
-}
-
-double
-stir_rusage(int who, double entropy_estimate)
-{
-#ifdef HAVE_GETRUSAGE
-	struct rusage ru;
-
-	if (getrusage(who, &ru) == -1)
-		return 0;
-
-	RAND_add(&ru, sizeof(ru), entropy_estimate);
-
-	return entropy_estimate;
-#else /* _HAVE_GETRUSAGE */
-	return 0;
-#endif /* _HAVE_GETRUSAGE */
-}
-
-static int
-timeval_diff(struct timeval *t1, struct timeval *t2)
-{
-	int secdiff, usecdiff;
-
-	secdiff = t2->tv_sec - t1->tv_sec;
-	usecdiff = (secdiff*1000000) + (t2->tv_usec - t1->tv_usec);
-	return (int)(usecdiff / 1000);
-}
-
-double
-hash_command_output(entropy_cmd_t *src, unsigned char *hash)
-{
-	char buf[8192];
-	fd_set rdset;
-	int bytes_read, cmd_eof, error_abort, msec_elapsed, p[2];
-	int status, total_bytes_read;
-	static int devnull = -1;
-	pid_t pid;
-	SHA_CTX sha;
-	struct timeval tv_start, tv_current;
-
-	debug3("Reading output from \'%s\'", src->cmdstring);
-
-	if (devnull == -1) {
-		devnull = open("/dev/null", O_RDWR);
-		if (devnull == -1)
-			fatal("Couldn't open /dev/null: %s",
-			    strerror(errno));
-	}
-
-	if (pipe(p) == -1)
-		fatal("Couldn't open pipe: %s", strerror(errno));
-
-	(void)gettimeofday(&tv_start, NULL); /* record start time */
-
-	switch (pid = fork()) {
-		case -1: /* Error */
-			close(p[0]);
-			close(p[1]);
-			fatal("Couldn't fork: %s", strerror(errno));
-			/* NOTREACHED */
-		case 0: /* Child */
-			dup2(devnull, STDIN_FILENO);
-			dup2(p[1], STDOUT_FILENO);
-			dup2(p[1], STDERR_FILENO);
-			close(p[0]);
-			close(p[1]);
-			close(devnull);
-
-			execv(src->path, (char**)(src->args));
-
-			debug("(child) Couldn't exec '%s': %s",
-			    src->cmdstring, strerror(errno));
-			_exit(-1);
-		default: /* Parent */
-			break;
-	}
-
-	RAND_add(&pid, sizeof(&pid), 0.0);
-
-	close(p[1]);
-
-	/* Hash output from child */
-	SHA1_Init(&sha);
-
-	cmd_eof = error_abort = msec_elapsed = total_bytes_read = 0;
-	while (!error_abort && !cmd_eof) {
-		int ret;
-		struct timeval tv;
-		int msec_remaining;
-
-		(void) gettimeofday(&tv_current, 0);
-		msec_elapsed = timeval_diff(&tv_start, &tv_current);
-		if (msec_elapsed >= entropy_timeout_current) {
-			error_abort=1;
-			continue;
-		}
-		msec_remaining = entropy_timeout_current - msec_elapsed;
-
-		FD_ZERO(&rdset);
-		FD_SET(p[0], &rdset);
-		tv.tv_sec = msec_remaining / 1000;
-		tv.tv_usec = (msec_remaining % 1000) * 1000;
-
-		ret = select(p[0] + 1, &rdset, NULL, NULL, &tv);
-
-		RAND_add(&tv, sizeof(tv), 0.0);
-
-		switch (ret) {
-		case 0:
-			/* timer expired */
-			error_abort = 1;
-			kill(pid, SIGINT);
-			break;
-		case 1:
-			/* command input */
-			do {
-				bytes_read = read(p[0], buf, sizeof(buf));
-			} while (bytes_read == -1 && errno == EINTR);
-			RAND_add(&bytes_read, sizeof(&bytes_read), 0.0);
-			if (bytes_read == -1) {
-				error_abort = 1;
-				break;
-			} else if (bytes_read) {
-				SHA1_Update(&sha, buf, bytes_read);
-				total_bytes_read += bytes_read;
-			} else {
-				cmd_eof = 1;
-			}
-			break;
-		case -1:
-		default:
-			/* error */
-			debug("Command '%s': select() failed: %s",
-			    src->cmdstring, strerror(errno));
-			error_abort = 1;
-			break;
-		}
-	}
-
-	SHA1_Final(hash, &sha);
-
-	close(p[0]);
-
-	debug3("Time elapsed: %d msec", msec_elapsed);
-
-	if (waitpid(pid, &status, 0) == -1) {
-		error("Couldn't wait for child '%s' completion: %s",
-		    src->cmdstring, strerror(errno));
-		return 0.0;
-	}
-
-	RAND_add(&status, sizeof(&status), 0.0);
-
-	if (error_abort) {
-		/*
-		 * Closing p[0] on timeout causes the entropy command to
-		 * SIGPIPE. Take whatever output we got, and mark this
-		 * command as slow
-		 */
-		debug2("Command '%s' timed out", src->cmdstring);
-		src->sticky_badness *= 2;
-		src->badness = src->sticky_badness;
-		return total_bytes_read;
-	}
-
-	if (WIFEXITED(status)) {
-		if (WEXITSTATUS(status) == 0) {
-			return total_bytes_read;
-		} else {
-			debug2("Command '%s' exit status was %d",
-			    src->cmdstring, WEXITSTATUS(status));
-			src->badness = src->sticky_badness = 128;
-			return 0.0;
-		}
-	} else if (WIFSIGNALED(status)) {
-		debug2("Command '%s' returned on uncaught signal %d !",
-		    src->cmdstring, status);
-		src->badness = src->sticky_badness = 128;
-		return 0.0;
-	} else
-		return 0.0;
-}
-
-double
-stir_from_system(void)
-{
-	double total_entropy_estimate;
-	long int i;
-
-	total_entropy_estimate = 0;
-
-	i = getpid();
-	RAND_add(&i, sizeof(i), 0.5);
-	total_entropy_estimate += 0.1;
-
-	i = getppid();
-	RAND_add(&i, sizeof(i), 0.5);
-	total_entropy_estimate += 0.1;
-
-	i = getuid();
-	RAND_add(&i, sizeof(i), 0.0);
-	i = getgid();
-	RAND_add(&i, sizeof(i), 0.0);
-
-	total_entropy_estimate += stir_gettimeofday(1.0);
-	total_entropy_estimate += stir_clock(0.5);
-	total_entropy_estimate += stir_rusage(RUSAGE_SELF, 2.0);
-
-	return total_entropy_estimate;
-}
-
-double
-stir_from_programs(void)
-{
-	int c;
-	double entropy, total_entropy;
-	unsigned char hash[SHA_DIGEST_LENGTH];
-
-	total_entropy = 0;
-	for(c = 0; entropy_cmds[c].path != NULL; c++) {
-		if (!entropy_cmds[c].badness) {
-			/* Hash output from command */
-			entropy = hash_command_output(&entropy_cmds[c],
-			    hash);
-
-			/* Scale back estimate by command's rate */
-			entropy *= entropy_cmds[c].rate;
-
-			/* Upper bound of entropy is SHA_DIGEST_LENGTH */
-			if (entropy > SHA_DIGEST_LENGTH)
-				entropy = SHA_DIGEST_LENGTH;
-
-			/* Stir it in */
-			RAND_add(hash, sizeof(hash), entropy);
-
-			debug3("Got %0.2f bytes of entropy from '%s'",
-			    entropy, entropy_cmds[c].cmdstring);
-
-			total_entropy += entropy;
-
-			/* Execution time should be a bit unpredictable */
-			total_entropy += stir_gettimeofday(0.05);
-			total_entropy += stir_clock(0.05);
-			total_entropy += stir_rusage(RUSAGE_SELF, 0.1);
-			total_entropy += stir_rusage(RUSAGE_CHILDREN, 0.1);
-		} else {
-			debug2("Command '%s' disabled (badness %d)",
-			    entropy_cmds[c].cmdstring,
-			    entropy_cmds[c].badness);
-
-			if (entropy_cmds[c].badness > 0)
-				entropy_cmds[c].badness--;
-		}
-	}
-
-	return total_entropy;
-}
-
-/*
- * prng seedfile functions
- */
-int
-prng_check_seedfile(char *filename)
-{
-	struct stat st;
-
-	/*
-	 * XXX raceable: eg replace seed between this stat and subsequent
-	 * open. Not such a problem because we don't really trust the
-	 * seed file anyway.
-	 * XXX: use secure path checking as elsewhere in OpenSSH
-	 */
-	if (lstat(filename, &st) == -1) {
-		/* Give up on hard errors */
-		if (errno != ENOENT)
-			debug("WARNING: Couldn't stat random seed file "
-			    "\"%.100s\": %s", filename, strerror(errno));
-		return 0;
-	}
-
-	/* regular file? */
-	if (!S_ISREG(st.st_mode))
-		fatal("PRNG seedfile %.100s is not a regular file",
-		    filename);
-
-	/* mode 0600, owned by root or the current user? */
-	if (((st.st_mode & 0177) != 0) || !(st.st_uid == getuid())) {
-		debug("WARNING: PRNG seedfile %.100s must be mode 0600, "
-		    "owned by uid %li", filename, (long int)getuid());
-		return 0;
-	}
-
-	return 1;
-}
-
-void
-prng_write_seedfile(void)
-{
-	int fd, save_errno;
-	unsigned char seed[SEED_FILE_SIZE];
-	char filename[MAXPATHLEN], tmpseed[MAXPATHLEN];
-	struct passwd *pw;
-	mode_t old_umask;
-
-	pw = getpwuid(getuid());
-	if (pw == NULL)
-		fatal("Couldn't get password entry for current user "
-		    "(%li): %s", (long int)getuid(), strerror(errno));
-
-	/* Try to ensure that the parent directory is there */
-	snprintf(filename, sizeof(filename), "%.512s/%s", pw->pw_dir,
-	    _PATH_SSH_USER_DIR);
-	if (mkdir(filename, 0700) < 0 && errno != EEXIST)
-		fatal("mkdir %.200s: %s", filename, strerror(errno));
-
-	snprintf(filename, sizeof(filename), "%.512s/%s", pw->pw_dir,
-	    SSH_PRNG_SEED_FILE);
-
-	strlcpy(tmpseed, filename, sizeof(tmpseed));
-	if (strlcat(tmpseed, ".XXXXXXXXXX", sizeof(tmpseed)) >=
-	    sizeof(tmpseed))
-		fatal("PRNG seed filename too long");
-
-	if (RAND_bytes(seed, sizeof(seed)) <= 0)
-		fatal("PRNG seed extraction failed");
-
-	/* Don't care if the seed doesn't exist */
-	prng_check_seedfile(filename);
-
-	old_umask = umask(0177);
-
-	if ((fd = mkstemp(tmpseed)) == -1) {
-		debug("WARNING: couldn't make temporary PRNG seedfile %.100s "
-		    "(%.100s)", tmpseed, strerror(errno));
-	} else {
-		debug("writing PRNG seed to file %.100s", tmpseed);
-		if (atomicio(vwrite, fd, &seed, sizeof(seed)) < sizeof(seed)) {
-			save_errno = errno;
-			close(fd);
-			unlink(tmpseed);
-			fatal("problem writing PRNG seedfile %.100s "
-			    "(%.100s)", filename, strerror(save_errno));
-		}
-		close(fd);
-		debug("moving temporary PRNG seed to file %.100s", filename);
-		if (rename(tmpseed, filename) == -1) {
-			save_errno = errno;
-			unlink(tmpseed);
-			fatal("problem renaming PRNG seedfile from %.100s "
-			    "to %.100s (%.100s)", tmpseed, filename,
-			    strerror(save_errno));
-		}
-	}
-	umask(old_umask);
-}
-
-void
-prng_read_seedfile(void)
-{
-	int fd;
-	char seed[SEED_FILE_SIZE], filename[MAXPATHLEN];
-	struct passwd *pw;
-
-	pw = getpwuid(getuid());
-	if (pw == NULL)
-		fatal("Couldn't get password entry for current user "
-		    "(%li): %s", (long int)getuid(), strerror(errno));
-
-	snprintf(filename, sizeof(filename), "%.512s/%s", pw->pw_dir,
-		SSH_PRNG_SEED_FILE);
-
-	debug("loading PRNG seed from file %.100s", filename);
-
-	if (!prng_check_seedfile(filename)) {
-		verbose("Random seed file not found or invalid, ignoring.");
-		return;
-	}
-
-	/* open the file and read in the seed */
-	fd = open(filename, O_RDONLY);
-	if (fd == -1)
-		fatal("could not open PRNG seedfile %.100s (%.100s)",
-		    filename, strerror(errno));
-
-	if (atomicio(read, fd, &seed, sizeof(seed)) < sizeof(seed)) {
-		verbose("invalid or short read from PRNG seedfile "
-		    "%.100s - ignoring", filename);
-		memset(seed, '\0', sizeof(seed));
-	}
-	close(fd);
-
-	/* stir in the seed, with estimated entropy zero */
-	RAND_add(&seed, sizeof(seed), 0.0);
-}
-
-
-/*
- * entropy command initialisation functions
- */
-int
-prng_read_commands(char *cmdfilename)
-{
-	char cmd[SEED_FILE_SIZE], *cp, line[1024], path[SEED_FILE_SIZE];
-	double est;
-	entropy_cmd_t *entcmd;
-	FILE *f;
-	int cur_cmd, linenum, num_cmds, arg;
-
-	if ((f = fopen(cmdfilename, "r")) == NULL) {
-		fatal("couldn't read entropy commands file %.100s: %.100s",
-		    cmdfilename, strerror(errno));
-	}
-
-	num_cmds = 64;
-	entcmd = xcalloc(num_cmds, sizeof(entropy_cmd_t));
-
-	/* Read in file */
-	cur_cmd = linenum = 0;
-	while (fgets(line, sizeof(line), f)) {
-		linenum++;
-
-		/* Skip leading whitespace, blank lines and comments */
-		cp = line + strspn(line, WHITESPACE);
-		if ((*cp == 0) || (*cp == '#'))
-			continue; /* done with this line */
-
-		/*
-		 * The first non-whitespace char should be a double quote
-		 * delimiting the commandline
-		 */
-		if (*cp != '"') {
-			error("bad entropy command, %.100s line %d",
-			    cmdfilename, linenum);
-			continue;
-		}
-
-		/*
-		 * First token, command args (incl. argv[0]) in double
-		 * quotes
-		 */
-		cp = strtok(cp, "\"");
-		if (cp == NULL) {
-			error("missing or bad command string, %.100s "
-			    "line %d -- ignored", cmdfilename, linenum);
-			continue;
-		}
-		strlcpy(cmd, cp, sizeof(cmd));
-
-		/* Second token, full command path */
-		if ((cp = strtok(NULL, WHITESPACE)) == NULL) {
-			error("missing command path, %.100s "
-			    "line %d -- ignored", cmdfilename, linenum);
-			continue;
-		}
-
-		/* Did configure mark this as dead? */
-		if (strncmp("undef", cp, 5) == 0)
-			continue;
-
-		strlcpy(path, cp, sizeof(path));
-
-		/* Third token, entropy rate estimate for this command */
-		if ((cp = strtok(NULL, WHITESPACE)) == NULL) {
-			error("missing entropy estimate, %.100s "
-			    "line %d -- ignored", cmdfilename, linenum);
-			continue;
-		}
-		est = strtod(cp, NULL);
-
-		/* end of line */
-		if ((cp = strtok(NULL, WHITESPACE)) != NULL) {
-			error("garbage at end of line %d in %.100s "
-			    "-- ignored", linenum, cmdfilename);
-			continue;
-		}
-
-		/* save the command for debug messages */
-		entcmd[cur_cmd].cmdstring = xstrdup(cmd);
-
-		/* split the command args */
-		cp = strtok(cmd, WHITESPACE);
-		arg = 0;
-		do {
-			entcmd[cur_cmd].args[arg] = xstrdup(cp);
-			arg++;
-		} while(arg < NUM_ARGS && (cp = strtok(NULL, WHITESPACE)));
-
-		if (strtok(NULL, WHITESPACE))
-			error("ignored extra commands (max %d), %.100s "
-			    "line %d", NUM_ARGS, cmdfilename, linenum);
-
-		/* Copy the command path and rate estimate */
-		entcmd[cur_cmd].path = xstrdup(path);
-		entcmd[cur_cmd].rate = est;
-
-		/* Initialise other values */
-		entcmd[cur_cmd].sticky_badness = 1;
-
-		cur_cmd++;
-
-		/*
-		 * If we've filled the array, reallocate it twice the size
-		 * Do this now because even if this we're on the last
-		 * command we need another slot to mark the last entry
-		 */
-		if (cur_cmd == num_cmds) {
-			num_cmds *= 2;
-			entcmd = xrealloc(entcmd, num_cmds,
-			    sizeof(entropy_cmd_t));
-		}
-	}
-
-	/* zero the last entry */
-	memset(&entcmd[cur_cmd], '\0', sizeof(entropy_cmd_t));
-
-	/* trim to size */
-	entropy_cmds = xrealloc(entcmd, (cur_cmd + 1),
-	    sizeof(entropy_cmd_t));
-
-	debug("Loaded %d entropy commands from %.100s", cur_cmd,
-	    cmdfilename);
-
-	fclose(f);
-	return cur_cmd < MIN_ENTROPY_SOURCES ? -1 : 0;
-}
-
-void
-usage(void)
-{
-	fprintf(stderr, "Usage: %s [options]\n", __progname);
-	fprintf(stderr, "  -v          Verbose; display verbose debugging messages.\n");
-	fprintf(stderr, "              Multiple -v increases verbosity.\n");
-	fprintf(stderr, "  -x          Force output in hexadecimal (for debugging)\n");
-	fprintf(stderr, "  -X          Force output in binary\n");
-	fprintf(stderr, "  -b bytes    Number of bytes to output (default %d)\n",
-	    OUTPUT_SEED_SIZE);
-}
-
-int
-main(int argc, char **argv)
-{
-	unsigned char *buf;
-	int ret, ch, debug_level, output_hex, bytes;
-	extern char *optarg;
-	extern int optind;
-	LogLevel ll;
-
-	__progname = ssh_get_progname(argv[0]);
-	log_init(argv[0], SYSLOG_LEVEL_INFO, SYSLOG_FACILITY_USER, 1);
-
-	ll = SYSLOG_LEVEL_INFO;
-	debug_level = output_hex = 0;
-	bytes = OUTPUT_SEED_SIZE;
-
-	/* Don't write binary data to a tty, unless we are forced to */
-	if (isatty(STDOUT_FILENO))
-		output_hex = 1;
-
-	while ((ch = getopt(argc, argv, "vxXhb:")) != -1) {
-		switch (ch) {
-		case 'v':
-			if (debug_level < 3)
-				ll = SYSLOG_LEVEL_DEBUG1 + debug_level++;
-			break;
-		case 'x':
-			output_hex = 1;
-			break;
-		case 'X':
-			output_hex = 0;
-			break;
-		case 'b':
-			if ((bytes = atoi(optarg)) <= 0)
-				fatal("Invalid number of output bytes");
-			break;
-		case 'h':
-			usage();
-			exit(0);
-		default:
-			error("Invalid commandline option");
-			usage();
-			exit(1);
-		}
-	}
-	log_init(argv[0], ll, SYSLOG_FACILITY_USER, 1);
-
-	if (argc != optind) {
-		error("Unexpected commandline arguments.");
-		usage();
-		exit(1);
-	}
-
-#ifdef USE_SEED_FILES
-	prng_read_seedfile();
-#endif
-
-	buf = xmalloc(bytes);
-
-	/*
-	 * Seed the RNG from wherever we can
-	 */
-
-	/* Take whatever is on the stack, but don't credit it */
-	RAND_add(buf, bytes, 0);
-
-	debug("Seeded RNG with %i bytes from system calls",
-	    (int)stir_from_system());
-
-	/* try prngd, fall back to commands if prngd fails or not configured */
-	if (seed_from_prngd(buf, bytes) == 0) {
-		RAND_add(buf, bytes, bytes);
-	} else {
-		/* Read in collection commands */
-		if (prng_read_commands(SSH_PRNG_COMMAND_FILE) == -1)
-			fatal("PRNG initialisation failed -- exiting.");
-		debug("Seeded RNG with %i bytes from programs",
-		    (int)stir_from_programs());
-	}
-
-#ifdef USE_SEED_FILES
-	prng_write_seedfile();
-#endif
-
-	/*
-	 * Write the seed to stdout
-	 */
-
-	if (!RAND_status())
-		fatal("Not enough entropy in RNG");
-
-	if (RAND_bytes(buf, bytes) <= 0)
-		fatal("Couldn't extract entropy from PRNG");
-
-	if (output_hex) {
-		for(ret = 0; ret < bytes; ret++)
-			printf("%02x", (unsigned char)(buf[ret]));
-		printf("\n");
-	} else
-		ret = atomicio(vwrite, STDOUT_FILENO, buf, bytes);
-
-	memset(buf, '\0', bytes);
-	xfree(buf);
-
-	return ret == bytes ? 0 : 1;
-}
-
-/*
- * We may attempt to re-seed during mkstemp if we are using the one in the
- * compat library (via mkstemp -> _gettemp -> arc4random -> seed_rng) so we
- * need our own seed_rng().  We must also check that we have enough entropy.
- */
-void
-seed_rng(void)
-{
-	if (!RAND_status())
-		fatal("Not enough entropy in RNG");
-}
diff --git a/crypto/openssh/ssh-sandbox.h b/crypto/openssh/ssh-sandbox.h
new file mode 100644
index 000000000000..dfecd5aa00a5
--- /dev/null
+++ b/crypto/openssh/ssh-sandbox.h
@@ -0,0 +1,23 @@
+/* $OpenBSD: ssh-sandbox.h,v 1.1 2011/06/23 09:34:13 djm Exp $ */
+/*
+ * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+struct ssh_sandbox;
+
+struct ssh_sandbox *ssh_sandbox_init(void);
+void ssh_sandbox_child(struct ssh_sandbox *);
+void ssh_sandbox_parent_finish(struct ssh_sandbox *);
+void ssh_sandbox_parent_preauth(struct ssh_sandbox *, pid_t);
diff --git a/crypto/openssh/ssh.1 b/crypto/openssh/ssh.1
index 4bdb30ef6eaa..12eb28179ffa 100644
--- a/crypto/openssh/ssh.1
+++ b/crypto/openssh/ssh.1
@@ -33,9 +33,9 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.\" $OpenBSD: ssh.1,v 1.316 2010/11/18 15:01:00 jmc Exp $
+.\" $OpenBSD: ssh.1,v 1.320 2011/08/02 01:22:11 djm Exp $
 .\" $FreeBSD$
-.Dd November 18, 2010
+.Dd August 2, 2011
 .Dt SSH 1
 .Os
 .Sh NAME
@@ -393,9 +393,11 @@ Valid commands are:
 .Dq check
 (check that the master process is running),
 .Dq forward
-(request forwardings without command execution) and
+(request forwardings without command execution),
 .Dq exit
-(request the master to exit).
+(request the master to exit), and
+.Dq stop
+(request the master to stop accepting further multiplexing requests).
 .It Fl o Ar option
 Can be used to give options in the format used in the configuration file.
 This is useful for specifying options for which there is no separate
@@ -455,6 +457,7 @@ For full details of the options listed below, and their possible values, see
 .It PubkeyAuthentication
 .It RekeyLimit
 .It RemoteForward
+.It RequestTTY
 .It RhostsRSAAuthentication
 .It RSAAuthentication
 .It SendEnv
@@ -666,7 +669,9 @@ Both protocols support similar authentication methods,
 but protocol 2 is the default since
 it provides additional mechanisms for confidentiality
 (the traffic is encrypted using AES, 3DES, Blowfish, CAST128, or Arcfour)
-and integrity (hmac-md5, hmac-sha1, umac-64, hmac-ripemd160).
+and integrity (hmac-md5, hmac-sha1,
+hmac-sha2-256, hmac-sha2-512,
+umac-64, hmac-ripemd160).
 Protocol 1 lacks a strong mechanism for ensuring the
 integrity of the connection.
 .Pp
diff --git a/crypto/openssh/ssh.c b/crypto/openssh/ssh.c
index 1d21f9336154..7fdae3a71355 100644
--- a/crypto/openssh/ssh.c
+++ b/crypto/openssh/ssh.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh.c,v 1.356 2011/01/06 22:23:53 djm Exp $ */
+/* $OpenBSD: ssh.c,v 1.364 2011/08/02 23:15:03 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -113,13 +113,16 @@ __RCSID("$FreeBSD$");
 
 extern char *__progname;
 
+/* Saves a copy of argv for setproctitle emulation */
+#ifndef HAVE_SETPROCTITLE
+static char **saved_av;
+#endif
+
 /* Flag indicating whether debug mode is on.  May be set on the command line. */
 int debug_flag = 0;
 
-/* Flag indicating whether a tty should be allocated */
+/* Flag indicating whether a tty should be requested */
 int tty_flag = 0;
-int no_tty_flag = 0;
-int force_tty_flag = 0;
 
 /* don't exec a shell */
 int no_shell_flag = 0;
@@ -137,7 +140,7 @@ int stdin_null_flag = 0;
 int need_controlpersist_detach = 0;
 
 /* Copies of flags for ControlPersist foreground slave */
-int ostdin_null_flag, ono_shell_flag, ono_tty_flag, otty_flag;
+int ostdin_null_flag, ono_shell_flag, otty_flag, orequest_tty;
 
 /*
  * Flag indicating that ssh should fork after authentication.  This is useful
@@ -216,6 +219,20 @@ static void main_sigchld_handler(int);
 void muxclient(const char *);
 void muxserver_listen(void);
 
+/* ~/ expand a list of paths. NB. assumes path[n] is heap-allocated. */
+static void
+tilde_expand_paths(char **paths, u_int num_paths)
+{
+	u_int i;
+	char *cp;
+
+	for (i = 0; i < num_paths; i++) {
+		cp = tilde_expand_filename(paths[i], original_real_uid);
+		xfree(paths[i]);
+		paths[i] = cp;
+	}
+}
+
 /*
  * Main program for the ssh client.
  */
@@ -224,11 +241,13 @@ main(int ac, char **av)
 {
 	int i, r, opt, exit_status, use_syslog;
 	char *p, *cp, *line, *argv0, buf[MAXPATHLEN], *host_arg;
+	char thishost[NI_MAXHOST], shorthost[NI_MAXHOST], portstr[NI_MAXSERV];
 	struct stat st;
 	struct passwd *pw;
 	int dummy, timeout_ms;
 	extern int optind, optreset;
 	extern char *optarg;
+
 	struct servent *sp;
 	Forward fwd;
 
@@ -236,7 +255,17 @@ main(int ac, char **av)
 	sanitise_stdfd();
 
 	__progname = ssh_get_progname(av[0]);
-	init_rng();
+
+#ifndef HAVE_SETPROCTITLE
+	/* Prepare for later setproctitle emulation */
+	/* Save argv so it isn't clobbered by setproctitle() emulation */
+	saved_av = xcalloc(ac + 1, sizeof(*saved_av));
+	for (i = 0; i < ac; i++)
+		saved_av[i] = xstrdup(av[i]);
+	saved_av[i] = NULL;
+	compat_init_setproctitle(ac, av);
+	av = saved_av;
+#endif
 
 	/*
 	 * Discard other fds that are hanging around. These can cause problem
@@ -348,6 +377,8 @@ main(int ac, char **av)
 				muxclient_command = SSHMUX_COMMAND_FORWARD;
 			else if (strcmp(optarg, "exit") == 0)
 				muxclient_command = SSHMUX_COMMAND_TERMINATE;
+			else if (strcmp(optarg, "stop") == 0)
+				muxclient_command = SSHMUX_COMMAND_STOP;
 			else
 				fatal("Invalid multiplex command.");
 			break;
@@ -389,9 +420,10 @@ main(int ac, char **av)
 #endif
 			break;
 		case 't':
-			if (tty_flag)
-				force_tty_flag = 1;
-			tty_flag = 1;
+			if (options.request_tty == REQUEST_TTY_YES)
+				options.request_tty = REQUEST_TTY_FORCE;
+			else
+				options.request_tty = REQUEST_TTY_YES;
 			break;
 		case 'v':
 			if (debug_flag == 0) {
@@ -434,7 +466,7 @@ main(int ac, char **av)
 				    optarg);
 				exit(255);
 			}
-			no_tty_flag = 1;
+			options.request_tty = REQUEST_TTY_NO;
 			no_shell_flag = 1;
 			options.clear_forwardings = 1;
 			options.exit_on_forward_failure = 1;
@@ -543,10 +575,10 @@ main(int ac, char **av)
 			break;
 		case 'N':
 			no_shell_flag = 1;
-			no_tty_flag = 1;
+			options.request_tty = REQUEST_TTY_NO;
 			break;
 		case 'T':
-			no_tty_flag = 1;
+			options.request_tty = REQUEST_TTY_NO;
 #ifdef	NONE_CIPHER_ENABLED
 			/*
 			 * Ensure that the user does not try to backdoor a
@@ -615,6 +647,10 @@ main(int ac, char **av)
 	/* Initialize the command to execute on remote host. */
 	buffer_init(&command);
 
+	if (options.request_tty == REQUEST_TTY_YES ||
+	    options.request_tty == REQUEST_TTY_FORCE)
+		tty_flag = 1;
+
 	/*
 	 * Save the command to execute on the remote host in a buffer. There
 	 * is no limit on the length of the command, except by the maximum
@@ -622,7 +658,7 @@ main(int ac, char **av)
 	 */
 	if (!ac) {
 		/* No command specified - execute shell on a tty. */
-		tty_flag = 1;
+		tty_flag = options.request_tty != REQUEST_TTY_NO;
 		if (subsystem_flag) {
 			fprintf(stderr,
 			    "You must specify a subsystem to invoke.\n");
@@ -645,13 +681,14 @@ main(int ac, char **av)
 
 	/* Allocate a tty by default if no command specified. */
 	if (buffer_len(&command) == 0)
-		tty_flag = 1;
+		tty_flag = options.request_tty != REQUEST_TTY_NO;
 
 	/* Force no tty */
-	if (no_tty_flag || muxclient_command != 0)
+	if (options.request_tty == REQUEST_TTY_NO || muxclient_command != 0)
 		tty_flag = 0;
 	/* Do not allocate a tty if stdin is not a tty. */
-	if ((!isatty(fileno(stdin)) || stdin_null_flag) && !force_tty_flag) {
+	if ((!isatty(fileno(stdin)) || stdin_null_flag) &&
+	    options.request_tty != REQUEST_TTY_FORCE) {
 		if (tty_flag)
 			logit("Pseudo-terminal will not be allocated because "
 			    "stdin is not a terminal.");
@@ -680,7 +717,7 @@ main(int ac, char **av)
 		if (r > 0 && (size_t)r < sizeof(buf))
 			(void)read_config_file(buf, host, &options, 1);
 
-		/* Read systemwide configuration file after use config. */
+		/* Read systemwide configuration file after user config. */
 		(void)read_config_file(_PATH_HOST_CONFIG_FILE, host,
 		    &options, 0);
 	}
@@ -711,17 +748,19 @@ main(int ac, char **av)
 		    "h", host, (char *)NULL);
 	}
 
-	if (options.local_command != NULL) {
-		char thishost[NI_MAXHOST];
+	if (gethostname(thishost, sizeof(thishost)) == -1)
+		fatal("gethostname: %s", strerror(errno));
+	strlcpy(shorthost, thishost, sizeof(shorthost));
+	shorthost[strcspn(thishost, ".")] = '\0';
+	snprintf(portstr, sizeof(portstr), "%d", options.port);
 
-		if (gethostname(thishost, sizeof(thishost)) == -1)
-			fatal("gethostname: %s", strerror(errno));
-		snprintf(buf, sizeof(buf), "%d", options.port);
+	if (options.local_command != NULL) {
 		debug3("expanding LocalCommand: %s", options.local_command);
 		cp = options.local_command;
 		options.local_command = percent_expand(cp, "d", pw->pw_dir,
 		    "h", host, "l", thishost, "n", host_arg, "r", options.user,
-		    "p", buf, "u", pw->pw_name, (char *)NULL);
+		    "p", portstr, "u", pw->pw_name, "L", shorthost,
+		    (char *)NULL);
 		debug3("expanded LocalCommand: %s", options.local_command);
 		xfree(cp);
 	}
@@ -762,16 +801,13 @@ main(int ac, char **av)
 	}
 
 	if (options.control_path != NULL) {
-		char thishost[NI_MAXHOST];
-
-		if (gethostname(thishost, sizeof(thishost)) == -1)
-			fatal("gethostname: %s", strerror(errno));
-		snprintf(buf, sizeof(buf), "%d", options.port);
 		cp = tilde_expand_filename(options.control_path,
 		    original_real_uid);
 		xfree(options.control_path);
-		options.control_path = percent_expand(cp, "p", buf, "h", host,
-		    "r", options.user, "l", thishost, (char *)NULL);
+		options.control_path = percent_expand(cp, "h", host,
+		    "l", thishost, "n", host_arg, "r", options.user,
+		    "p", portstr, "u", pw->pw_name, "L", shorthost,
+		    (char *)NULL);
 		xfree(cp);
 	}
 	if (muxclient_command != 0 && options.control_path == NULL)
@@ -892,15 +928,9 @@ main(int ac, char **av)
 	load_public_identity_files();
 
 	/* Expand ~ in known host file names. */
-	/* XXX mem-leaks: */
-	options.system_hostfile =
-	    tilde_expand_filename(options.system_hostfile, original_real_uid);
-	options.user_hostfile =
-	    tilde_expand_filename(options.user_hostfile, original_real_uid);
-	options.system_hostfile2 =
-	    tilde_expand_filename(options.system_hostfile2, original_real_uid);
-	options.user_hostfile2 =
-	    tilde_expand_filename(options.user_hostfile2, original_real_uid);
+	tilde_expand_paths(options.system_hostfiles,
+	    options.num_system_hostfiles);
+	tilde_expand_paths(options.user_hostfiles, options.num_user_hostfiles);
 
 	signal(SIGPIPE, SIG_IGN); /* ignore SIGPIPE early */
 	signal(SIGCHLD, main_sigchld_handler);
@@ -973,8 +1003,7 @@ control_persist_detach(void)
 		/* Parent: set up mux slave to connect to backgrounded master */
 		debug2("%s: background process is %ld", __func__, (long)pid);
 		stdin_null_flag = ostdin_null_flag;
-		no_shell_flag = ono_shell_flag;
-		no_tty_flag = ono_tty_flag;
+		options.request_tty = orequest_tty;
 		tty_flag = otty_flag;
  		close(muxserver_sock);
  		muxserver_sock = -1;
@@ -993,6 +1022,7 @@ control_persist_detach(void)
 		if (devnull > STDERR_FILENO)
 			close(devnull);
 	}
+	setproctitle("%s [mux]", options.control_path);
 }
 
 /* Do fork() after authentication. Used by "ssh -f" */
@@ -1237,8 +1267,8 @@ ssh_session(void)
 		/* Request forwarding with authentication spoofing. */
 		debug("Requesting X11 forwarding with authentication "
 		    "spoofing.");
-		x11_request_forwarding_with_spoofing(0, display, proto, data);
-
+		x11_request_forwarding_with_spoofing(0, display, proto,
+		    data, 0);
 		/* Read response from the server. */
 		type = packet_read();
 		if (type == SSH_SMSG_SUCCESS) {
@@ -1336,9 +1366,11 @@ ssh_session2_setup(int id, int success, void *arg)
 		/* Request forwarding with authentication spoofing. */
 		debug("Requesting X11 forwarding with authentication "
 		    "spoofing.");
-		x11_request_forwarding_with_spoofing(id, display, proto, data);
+		x11_request_forwarding_with_spoofing(id, display, proto,
+		    data, 1);
+		client_expect_confirm(id, "X11 forwarding", CONFIRM_WARN);
+		/* XXX exit_on_forward_failure */
 		interactive = 1;
-		/* XXX wait for reply */
 	}
 
 	check_agent_present();
@@ -1461,11 +1493,10 @@ ssh_session2(void)
  	if (options.control_persist && muxserver_sock != -1) {
 		ostdin_null_flag = stdin_null_flag;
 		ono_shell_flag = no_shell_flag;
-		ono_tty_flag = no_tty_flag;
+		orequest_tty = options.request_tty;
 		otty_flag = tty_flag;
  		stdin_null_flag = 1;
  		no_shell_flag = 1;
- 		no_tty_flag = 1;
  		tty_flag = 0;
 		if (!fork_after_authentication_flag)
 			need_controlpersist_detach = 1;
diff --git a/crypto/openssh/ssh_config b/crypto/openssh/ssh_config
index 1ff198b85e97..4f7b7a125e67 100644
--- a/crypto/openssh/ssh_config
+++ b/crypto/openssh/ssh_config
@@ -46,4 +46,4 @@
 #   PermitLocalCommand no
 #   VisualHostKey no
 #   ProxyCommand ssh -q -W %h:%p gateway.example.com
-#   VersionAddendum FreeBSD-20110503
+#   VersionAddendum FreeBSD-20111001
diff --git a/crypto/openssh/ssh_config.5 b/crypto/openssh/ssh_config.5
index f0754c3432e9..0df39e9de4de 100644
--- a/crypto/openssh/ssh_config.5
+++ b/crypto/openssh/ssh_config.5
@@ -33,9 +33,9 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.\" $OpenBSD: ssh_config.5,v 1.146 2010/12/08 04:02:47 djm Exp $
+.\" $OpenBSD: ssh_config.5,v 1.153 2011/08/02 01:22:11 djm Exp $
 .\" $FreeBSD$
-.Dd December 8, 2010
+.Dd August 2, 2011
 .Dt SSH_CONFIG 5
 .Os
 .Sh NAME
@@ -113,6 +113,15 @@ The host is the
 argument given on the command line (i.e. the name is not converted to
 a canonicalized host name before matching).
 .Pp
+A pattern entry may be negated by prefixing it with an exclamation mark
+.Pq Sq !\& .
+If a negated entry is matched, then the
+.Cm Host
+entry is ignored, regardless of whether any other patterns on the line
+match.
+Negated matches are therefore useful to provide exceptions for wildcard
+matches.
+.Pp
 See
 .Sx PATTERNS
 for more information on patterns.
@@ -306,14 +315,22 @@ section above or the string
 .Dq none
 to disable connection sharing.
 In the path,
+.Ql %L
+will be substituted by the first component of the local host name,
 .Ql %l
-will be substituted by the local host name,
+will be substituted by the local host name (including any domain name),
 .Ql %h
 will be substituted by the target host name,
+.Ql %n
+will be substituted by the original target host name
+specified on the command line,
 .Ql %p
-the port, and
+the port,
 .Ql %r
-by the remote login username.
+by the remote login username, and
+.Ql %u
+by the username of the user running
+.Xr ssh 1 .
 It is recommended that any
 .Cm ControlPath
 used for opportunistic connection sharing include
@@ -501,9 +518,11 @@ or
 The default is
 .Dq no .
 .It Cm GlobalKnownHostsFile
-Specifies a file to use for the global
-host key database instead of
-.Pa /etc/ssh/ssh_known_hosts .
+Specifies one or more files to use for the global
+host key database, separated by whitespace.
+The default is
+.Pa /etc/ssh/ssh_known_hosts ,
+.Pa /etc/ssh/ssh_known_hosts2 .
 .It Cm GSSAPIAuthentication
 Specifies whether user authentication based on GSSAPI is allowed.
 The default is
@@ -570,7 +589,7 @@ Specifies the real host name to log into.
 This can be used to specify nicknames or abbreviations for hosts.
 If the hostname contains the character sequence
 .Ql %h ,
-then this will be replaced with the host name specified on the commandline
+then this will be replaced with the host name specified on the command line
 (this is useful for manipulating unqualified names).
 The default is the name given on the command line.
 Numeric IP addresses are also permitted (both on the command line and in
@@ -630,6 +649,10 @@ escape characters:
 It is possible to have
 multiple identity files specified in configuration files; all these
 identities will be tried in sequence.
+Multiple
+.Cm IdentityFile
+directives will add to the list of identities tried (this behaviour
+differs from that of other configuration directives).
 .It Cm IPQoS
 Specifies the IPv4 type-of-service or DSCP class for connections.
 Accepted values are
@@ -771,7 +794,9 @@ Multiple algorithms must be comma-separated.
 The default is:
 .Bd -literal -offset indent
 hmac-md5,hmac-sha1,umac-64@openssh.com,
-hmac-ripemd160,hmac-sha1-96,hmac-md5-96
+hmac-ripemd160,hmac-sha1-96,hmac-md5-96,
+hmac-sha2-256,hmac-sha2-256-96,hmac-sha2-512,
+hmac-sha2-512-96
 .Ed
 .It Cm NoHostAuthenticationForLocalhost
 This option can be used if the home directory is shared across machines.
@@ -944,6 +969,23 @@ will only succeed if the server's
 .Cm GatewayPorts
 option is enabled (see
 .Xr sshd_config 5 ) .
+.It Cm RequestTTY
+Specifies whether to request a pseudo-tty for the session.
+The argument may be one of:
+.Dq no
+(never request a TTY),
+.Dq yes
+(always request a TTY when standard input is a TTY),
+.Dq force
+(always request a TTY) or
+.Dq auto
+(request a TTY when opening a login session).
+This option mirrors the
+.Fl t
+and
+.Fl T
+flags for
+.Xr ssh 1 .
 .It Cm RhostsRSAAuthentication
 Specifies whether to try rhosts based authentication with RSA host
 authentication.
@@ -1138,9 +1180,11 @@ This can be useful when a different user name is used on different machines.
 This saves the trouble of
 having to remember to give the user name on the command line.
 .It Cm UserKnownHostsFile
-Specifies a file to use for the user
-host key database instead of
-.Pa ~/.ssh/known_hosts .
+Specifies one or more files to use for the user
+host key database, separated by whitespace.
+The default is
+.Pa ~/.ssh/known_hosts ,
+.Pa ~/.ssh/known_hosts2 .
 .It Cm VerifyHostKeyDNS
 Specifies whether to verify the remote key using DNS and SSHFP resource
 records.
@@ -1173,7 +1217,7 @@ in
 Specifies a string to append to the regular version string to identify
 OS- or site-specific modifications.
 The default is
-.Dq FreeBSD-20110503 .
+.Dq FreeBSD-20111001 .
 .It Cm VisualHostKey
 If this flag is set to
 .Dq yes ,
diff --git a/crypto/openssh/ssh_namespace.h b/crypto/openssh/ssh_namespace.h
index d8d39e19870d..030cff765070 100644
--- a/crypto/openssh/ssh_namespace.h
+++ b/crypto/openssh/ssh_namespace.h
@@ -7,7 +7,7 @@
  *
  * A list of symbols which need munging is obtained as follows:
  *
- * nm libssh.a | awk '/[0-9a-z] [A-Z] / && $3 !~ /^ssh_/ { print "#define" $3 "\t\tssh_" $3 }'
+ * nm libssh.a | awk '/[0-9a-z] [A-Z] / && $3 !~ /^ssh_/ { print "#define " $3 "\t\tssh_" $3 }'
  *
  * $FreeBSD$
  */
@@ -58,6 +58,7 @@
 #define buffer_get_int64			ssh_buffer_get_int64
 #define buffer_get_int64_ret			ssh_buffer_get_int64_ret
 #define buffer_get_int_ret			ssh_buffer_get_int_ret
+#define buffer_get_max_len			ssh_buffer_get_max_len
 #define buffer_get_ret				ssh_buffer_get_ret
 #define buffer_get_short			ssh_buffer_get_short
 #define buffer_get_short_ret			ssh_buffer_get_short_ret
@@ -139,6 +140,7 @@
 #define channel_send_window_changes		ssh_channel_send_window_changes
 #define channel_set_af				ssh_channel_set_af
 #define channel_set_fds				ssh_channel_set_fds
+#define channel_set_hpn				ssh_channel_set_hpn
 #define channel_setup_local_fwd_listener	ssh_channel_setup_local_fwd_listener
 #define channel_setup_remote_fwd_listener	ssh_channel_setup_remote_fwd_listener
 #define channel_still_open			ssh_channel_still_open
@@ -199,6 +201,7 @@
 #define dispatch_run				ssh_dispatch_run
 #define dispatch_set				ssh_dispatch_set
 #define do_log					ssh_do_log
+#define do_log2					ssh_do_log2
 #define dump_base64				ssh_dump_base64
 #define enable_compat13				ssh_enable_compat13
 #define enable_compat20				ssh_enable_compat20
@@ -236,6 +239,7 @@
 #define incoming_stream				ssh_incoming_stream
 #define init_hostkeys				ssh_init_hostkeys
 #define init_rng				ssh_init_rng
+#define iptos2str				ssh_iptos2str
 #define ipv64_normalise_mapped			ssh_ipv64_normalise_mapped
 #define kex_derive_keys				ssh_kex_derive_keys
 #define kex_dh_hash				ssh_kex_dh_hash
@@ -280,6 +284,7 @@
 #define key_in_file				ssh_key_in_file
 #define key_is_cert				ssh_key_is_cert
 #define key_load_cert				ssh_key_load_cert
+#define key_load_file				ssh_key_load_file
 #define key_load_private			ssh_key_load_private
 #define key_load_private_cert			ssh_key_load_private_cert
 #define key_load_private_pem			ssh_key_load_private_pem
@@ -289,6 +294,7 @@
 #define key_names_valid2			ssh_key_names_valid2
 #define key_new					ssh_key_new
 #define key_new_private				ssh_key_new_private
+#define key_parse_private			ssh_key_parse_private
 #define key_perm_ok				ssh_key_perm_ok
 #define key_read				ssh_key_read
 #define key_save_private			ssh_key_save_private
@@ -433,11 +439,13 @@
 #define sanitise_stdfd				ssh_sanitise_stdfd
 #define scan_scaled				ssh_scan_scaled
 #define seed_rng				ssh_seed_rng
+#define set_log_handler				ssh_set_log_handler
 #define set_newkeys				ssh_set_newkeys
 #define set_nodelay				ssh_set_nodelay
 #define set_nonblock				ssh_set_nonblock
 #define shadow_pw				ssh_shadow_pw
 #define sigdie					ssh_sigdie
+#define sock_get_rcvbuf				ssh_sock_get_rcvbuf
 #define sock_set_v6only				ssh_sock_set_v6only
 #define ssh1_3des_iv				ssh_ssh1_3des_iv
 #define start_progress_meter			ssh_start_progress_meter
diff --git a/crypto/openssh/sshconnect.c b/crypto/openssh/sshconnect.c
index 0615b44f1496..042f2a624312 100644
--- a/crypto/openssh/sshconnect.c
+++ b/crypto/openssh/sshconnect.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sshconnect.c,v 1.232 2011/01/16 11:50:36 djm Exp $ */
+/* $OpenBSD: sshconnect.c,v 1.234 2011/05/24 07:15:47 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -712,28 +712,30 @@ get_hostfile_hostname_ipaddr(char *hostname, struct sockaddr *hostaddr,
 
 /*
  * check whether the supplied host key is valid, return -1 if the key
- * is not valid. the user_hostfile will not be updated if 'readonly' is true.
+ * is not valid. user_hostfile[0] will not be updated if 'readonly' is true.
  */
 #define RDRW	0
 #define RDONLY	1
 #define ROQUIET	2
 static int
 check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
-    Key *host_key, int readonly, char *user_hostfile,
-    char *system_hostfile)
+    Key *host_key, int readonly,
+    char **user_hostfiles, u_int num_user_hostfiles,
+    char **system_hostfiles, u_int num_system_hostfiles)
 {
-	Key *raw_key = NULL;
-	const char *type;
-	char *ip = NULL, *host = NULL;
-	char hostline[1000], *hostp, *fp, *ra;
 	HostStatus host_status;
 	HostStatus ip_status;
-	int r, want_cert = key_is_cert(host_key), host_ip_differ = 0;
-	int local = sockaddr_is_local(hostaddr);
+	Key *raw_key = NULL;
+	char *ip = NULL, *host = NULL;
+	char hostline[1000], *hostp, *fp, *ra;
 	char msg[1024];
-	int len, cancelled_forwarding = 0;
-	struct hostkeys *host_hostkeys, *ip_hostkeys;
+	const char *type;
 	const struct hostkey_entry *host_found, *ip_found;
+	int len, cancelled_forwarding = 0;
+	int local = sockaddr_is_local(hostaddr);
+	int r, want_cert = key_is_cert(host_key), host_ip_differ = 0;
+	struct hostkeys *host_hostkeys, *ip_hostkeys;
+	u_int i;
 
 	/*
 	 * Force accepting of the host key for loopback/localhost. The
@@ -765,14 +767,18 @@ check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
 		options.check_host_ip = 0;
 
 	host_hostkeys = init_hostkeys();
-	load_hostkeys(host_hostkeys, host, user_hostfile);
-	load_hostkeys(host_hostkeys, host, system_hostfile);
+	for (i = 0; i < num_user_hostfiles; i++)
+		load_hostkeys(host_hostkeys, host, user_hostfiles[i]);
+	for (i = 0; i < num_system_hostfiles; i++)
+		load_hostkeys(host_hostkeys, host, system_hostfiles[i]);
 
 	ip_hostkeys = NULL;
 	if (!want_cert && options.check_host_ip) {
 		ip_hostkeys = init_hostkeys();
-		load_hostkeys(ip_hostkeys, ip, user_hostfile);
-		load_hostkeys(ip_hostkeys, ip, system_hostfile);
+		for (i = 0; i < num_user_hostfiles; i++)
+			load_hostkeys(ip_hostkeys, ip, user_hostfiles[i]);
+		for (i = 0; i < num_system_hostfiles; i++)
+			load_hostkeys(ip_hostkeys, ip, system_hostfiles[i]);
 	}
 
  retry:
@@ -817,11 +823,12 @@ check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
 				logit("%s host key for IP address "
 				    "'%.128s' not in list of known hosts.",
 				    type, ip);
-			else if (!add_host_to_hostfile(user_hostfile, ip,
+			else if (!add_host_to_hostfile(user_hostfiles[0], ip,
 			    host_key, options.hash_known_hosts))
 				logit("Failed to add the %s host key for IP "
 				    "address '%.128s' to the list of known "
-				    "hosts (%.30s).", type, ip, user_hostfile);
+				    "hosts (%.30s).", type, ip,
+				    user_hostfiles[0]);
 			else
 				logit("Warning: Permanently added the %s host "
 				    "key for IP address '%.128s' to the list "
@@ -840,7 +847,8 @@ check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
 		    port != SSH_DEFAULT_PORT) {
 			debug("checking without port identifier");
 			if (check_host_key(hostname, hostaddr, 0, host_key,
-			    ROQUIET, user_hostfile, system_hostfile) == 0) {
+			    ROQUIET, user_hostfiles, num_user_hostfiles,
+			    system_hostfiles, num_system_hostfiles) == 0) {
 				debug("found matching key w/out port");
 				break;
 			}
@@ -905,25 +913,25 @@ check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
 			hostp = hostline;
 			if (options.hash_known_hosts) {
 				/* Add hash of host and IP separately */
-				r = add_host_to_hostfile(user_hostfile, host,
-				    host_key, options.hash_known_hosts) &&
-				    add_host_to_hostfile(user_hostfile, ip,
+				r = add_host_to_hostfile(user_hostfiles[0],
+				    host, host_key, options.hash_known_hosts) &&
+				    add_host_to_hostfile(user_hostfiles[0], ip,
 				    host_key, options.hash_known_hosts);
 			} else {
 				/* Add unhashed "host,ip" */
-				r = add_host_to_hostfile(user_hostfile,
+				r = add_host_to_hostfile(user_hostfiles[0],
 				    hostline, host_key,
 				    options.hash_known_hosts);
 			}
 		} else {
-			r = add_host_to_hostfile(user_hostfile, host, host_key,
-			    options.hash_known_hosts);
+			r = add_host_to_hostfile(user_hostfiles[0], host,
+			    host_key, options.hash_known_hosts);
 			hostp = host;
 		}
 
 		if (!r)
 			logit("Failed to add the host to the list of known "
-			    "hosts (%.500s).", user_hostfile);
+			    "hosts (%.500s).", user_hostfiles[0]);
 		else
 			logit("Warning: Permanently added '%.200s' (%s) to the "
 			    "list of known hosts.", hostp, type);
@@ -984,7 +992,7 @@ check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
 		/* The host key has changed. */
 		warn_changed_key(host_key);
 		error("Add correct host key in %.100s to get rid of this message.",
-		    user_hostfile);
+		    user_hostfiles[0]);
 		error("Offending %s key in %s:%lu", key_type(host_found->key),
 		    host_found->file, host_found->line);
 
@@ -1129,7 +1137,6 @@ check_host_key(char *hostname, struct sockaddr *hostaddr, u_short port,
 int
 verify_host_key(char *host, struct sockaddr *hostaddr, Key *host_key)
 {
-	struct stat st;
 	int flags = 0;
 	char *fp;
 
@@ -1140,7 +1147,6 @@ verify_host_key(char *host, struct sockaddr *hostaddr, Key *host_key)
 	/* XXX certs are not yet supported for DNS */
 	if (!key_is_cert(host_key) && options.verify_host_key_dns &&
 	    verify_host_key_dns(host, hostaddr, host_key, &flags) == 0) {
-
 		if (flags & DNS_VERIFY_FOUND) {
 
 			if (options.verify_host_key_dns == 1 &&
@@ -1158,16 +1164,9 @@ verify_host_key(char *host, struct sockaddr *hostaddr, Key *host_key)
 		}
 	}
 
-	/* return ok if the key can be found in an old keyfile */
-	if (stat(options.system_hostfile2, &st) == 0 ||
-	    stat(options.user_hostfile2, &st) == 0) {
-		if (check_host_key(host, hostaddr, options.port, host_key,
-		    RDONLY, options.user_hostfile2,
-		    options.system_hostfile2) == 0)
-			return 0;
-	}
-	return check_host_key(host, hostaddr, options.port, host_key,
-	    RDRW, options.user_hostfile, options.system_hostfile);
+	return check_host_key(host, hostaddr, options.port, host_key, RDRW,
+	    options.user_hostfiles, options.num_user_hostfiles,
+	    options.system_hostfiles, options.num_system_hostfiles);
 }
 
 /*
diff --git a/crypto/openssh/sshconnect2.c b/crypto/openssh/sshconnect2.c
index 7352276fd95f..f451644d6916 100644
--- a/crypto/openssh/sshconnect2.c
+++ b/crypto/openssh/sshconnect2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sshconnect2.c,v 1.186 2010/11/29 23:45:51 djm Exp $ */
+/* $OpenBSD: sshconnect2.c,v 1.188 2011/05/24 07:15:47 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
@@ -120,14 +120,15 @@ order_hostkeyalgs(char *host, struct sockaddr *hostaddr, u_short port)
 	size_t maxlen;
 	struct hostkeys *hostkeys;
 	int ktype;
+	u_int i;
 
 	/* Find all hostkeys for this hostname */
 	get_hostfile_hostname_ipaddr(host, hostaddr, port, &hostname, NULL);
 	hostkeys = init_hostkeys();
-	load_hostkeys(hostkeys, hostname, options.user_hostfile2);
-	load_hostkeys(hostkeys, hostname, options.system_hostfile2);
-	load_hostkeys(hostkeys, hostname, options.user_hostfile);
-	load_hostkeys(hostkeys, hostname, options.system_hostfile);
+	for (i = 0; i < options.num_user_hostfiles; i++)
+		load_hostkeys(hostkeys, hostname, options.user_hostfiles[i]);
+	for (i = 0; i < options.num_system_hostfiles; i++)
+		load_hostkeys(hostkeys, hostname, options.system_hostfiles[i]);
 
 	oavail = avail = xstrdup(KEX_DEFAULT_PK_ALG);
 	maxlen = strlen(avail) + 1;
@@ -1922,9 +1923,12 @@ authmethod_get(char *authlist)
 		    authmethod_is_enabled(current)) {
 			debug3("authmethod_is_enabled %s", name);
 			debug("Next authentication method: %s", name);
+			xfree(name);
 			return current;
 		}
 	}
+	if (name != NULL)
+		xfree(name);
 }
 
 static char *
diff --git a/crypto/openssh/sshd.8 b/crypto/openssh/sshd.8
index 314300d48e31..70d34c715661 100644
--- a/crypto/openssh/sshd.8
+++ b/crypto/openssh/sshd.8
@@ -33,9 +33,9 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.\" $OpenBSD: sshd.8,v 1.260 2010/10/28 18:33:28 jmc Exp $
+.\" $OpenBSD: sshd.8,v 1.263 2011/08/02 01:22:11 djm Exp $
 .\" $FreeBSD$
-.Dd October 28, 2010
+.Dd August 2, 2011
 .Dt SSHD 8
 .Os
 .Sh NAME
@@ -315,7 +315,8 @@ The client selects the encryption algorithm
 to use from those offered by the server.
 Additionally, session integrity is provided
 through a cryptographic message authentication code
-(hmac-md5, hmac-sha1, umac-64 or hmac-ripemd160).
+(hmac-md5, hmac-sha1, umac-64, hmac-ripemd160,
+hmac-sha2-256 or hmac-sha2-512).
 .Pp
 Finally, the server and the client enter an authentication dialog.
 The client tries to authenticate itself using
@@ -465,10 +466,12 @@ is run, and if that
 does not exist either, xauth is used to add the cookie.
 .Sh AUTHORIZED_KEYS FILE FORMAT
 .Cm AuthorizedKeysFile
-specifies the file containing public keys for
+specifies the files containing public keys for
 public key authentication;
 if none is specified, the default is
-.Pa ~/.ssh/authorized_keys .
+.Pa ~/.ssh/authorized_keys
+and
+.Pa ~/.ssh/authorized_keys2 .
 Each line of the file contains one
 key (empty lines and lines starting with a
 .Ql #
diff --git a/crypto/openssh/sshd.c b/crypto/openssh/sshd.c
index a6c9943e4fa4..070ef0c1750e 100644
--- a/crypto/openssh/sshd.c
+++ b/crypto/openssh/sshd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sshd.c,v 1.381 2011/01/11 06:13:10 djm Exp $ */
+/* $OpenBSD: sshd.c,v 1.385 2011/06/23 09:34:13 djm Exp $ */
 /* $FreeBSD$ */
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -130,6 +130,7 @@ __RCSID("$FreeBSD$");
 #endif
 #include "monitor_wrap.h"
 #include "roaming.h"
+#include "ssh-sandbox.h"
 #include "version.h"
 
 #ifdef LIBWRAP
@@ -636,42 +637,62 @@ privsep_preauth(Authctxt *authctxt)
 {
 	int status;
 	pid_t pid;
+	struct ssh_sandbox *box = NULL;
 
 	/* Set up unprivileged child process to deal with network data */
 	pmonitor = monitor_init();
 	/* Store a pointer to the kex for later rekeying */
 	pmonitor->m_pkex = &xxx_kex;
 
+	if (use_privsep == PRIVSEP_SANDBOX)
+		box = ssh_sandbox_init();
 	pid = fork();
 	if (pid == -1) {
 		fatal("fork of unprivileged child failed");
 	} else if (pid != 0) {
 		debug2("Network child is on pid %ld", (long)pid);
 
-		close(pmonitor->m_recvfd);
+		if (box != NULL)
+			ssh_sandbox_parent_preauth(box, pid);
 		pmonitor->m_pid = pid;
 		monitor_child_preauth(authctxt, pmonitor);
-		close(pmonitor->m_sendfd);
 
 		/* Sync memory */
 		monitor_sync(pmonitor);
 
 		/* Wait for the child's exit status */
-		while (waitpid(pid, &status, 0) < 0)
+		while (waitpid(pid, &status, 0) < 0) {
 			if (errno != EINTR)
-				break;
-		return (1);
+				fatal("%s: waitpid: %s", __func__,
+				    strerror(errno));
+		}
+		if (WIFEXITED(status)) {
+			if (WEXITSTATUS(status) != 0)
+				fatal("%s: preauth child exited with status %d",
+				    __func__, WEXITSTATUS(status));
+		} else if (WIFSIGNALED(status))
+			fatal("%s: preauth child terminated by signal %d",
+			    __func__, WTERMSIG(status));
+		if (box != NULL)
+			ssh_sandbox_parent_finish(box);
+		return 1;
 	} else {
 		/* child */
-
 		close(pmonitor->m_sendfd);
+		close(pmonitor->m_log_recvfd);
+
+		/* Arrange for logging to be sent to the monitor */
+		set_log_handler(mm_log_handler, pmonitor);
 
 		/* Demote the child */
 		if (getuid() == 0 || geteuid() == 0)
 			privsep_preauth_child();
 		setproctitle("%s", "[net]");
+		if (box != NULL)
+			ssh_sandbox_child(box);
+
+		return 0;
 	}
-	return (0);
 }
 
 static void
@@ -697,7 +718,6 @@ privsep_postauth(Authctxt *authctxt)
 		fatal("fork of unprivileged child failed");
 	else if (pmonitor->m_pid != 0) {
 		verbose("User child is on pid %ld", (long)pmonitor->m_pid);
-		close(pmonitor->m_recvfd);
 		buffer_clear(&loginmsg);
 		monitor_child_postauth(pmonitor);
 
@@ -705,7 +725,10 @@ privsep_postauth(Authctxt *authctxt)
 		exit(0);
 	}
 
+	/* child */
+
 	close(pmonitor->m_sendfd);
+	pmonitor->m_sendfd = -1;
 
 	/* Demote the private keys to public keys. */
 	demote_sensitive_data();
@@ -1134,7 +1157,7 @@ server_accept_loop(int *sock_in, int *sock_out, int *newsock, int *config_s)
 			    (int) received_sigterm);
 			close_listen_socks();
 			unlink(options.pid_file);
-			exit(255);
+			exit(received_sigterm == SIGTERM ? 0 : 255);
 		}
 		if (key_used && key_do_regen) {
 			generate_ephemeral_server_key();
@@ -1325,7 +1348,6 @@ main(int ac, char **av)
 	(void)set_auth_parameters(ac, av);
 #endif
 	__progname = ssh_get_progname(av[0]);
-	init_rng();
 
 	/* Save argv. Duplicate so setproctitle emulation doesn't clobber it */
 	saved_argc = ac;
diff --git a/crypto/openssh/sshd_config b/crypto/openssh/sshd_config
index f5079513bbb6..31c44b2a4928 100644
--- a/crypto/openssh/sshd_config
+++ b/crypto/openssh/sshd_config
@@ -1,4 +1,4 @@
-#	$OpenBSD: sshd_config,v 1.82 2010/09/06 17:10:19 naddy Exp $
+#	$OpenBSD: sshd_config,v 1.84 2011/05/23 03:30:07 djm Exp $
 #	$FreeBSD$
 
 # This is the sshd server system-wide configuration file.  See
@@ -8,13 +8,13 @@
 
 # The strategy used for options in the default sshd_config shipped with
 # OpenSSH is to specify options with their default value where
-# possible, but leave them commented.  Uncommented options change a
+# possible, but leave them commented.  Uncommented options override the
 # default value.
 
 # Note that some of FreeBSD's defaults differ from OpenBSD's, and
 # FreeBSD has a few additional options.
 
-#VersionAddendum FreeBSD-20110503
+#VersionAddendum FreeBSD-20111001
 
 #Port 22
 #AddressFamily any
@@ -50,7 +50,10 @@
 
 #RSAAuthentication yes
 #PubkeyAuthentication yes
-#AuthorizedKeysFile	.ssh/authorized_keys
+
+# The default is to check both .ssh/authorized_keys and .ssh/authorized_keys2
+# but this is overridden so installations will only check .ssh/authorized_keys
+AuthorizedKeysFile	.ssh/authorized_keys
 
 # For this to work you will also need host keys in /etc/ssh/ssh_known_hosts
 #RhostsRSAAuthentication no
diff --git a/crypto/openssh/sshd_config.5 b/crypto/openssh/sshd_config.5
index 1f8f7d2ce267..894ad3a3275c 100644
--- a/crypto/openssh/sshd_config.5
+++ b/crypto/openssh/sshd_config.5
@@ -33,9 +33,9 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.\" $OpenBSD: sshd_config.5,v 1.131 2010/12/08 04:02:47 djm Exp $
+.\" $OpenBSD: sshd_config.5,v 1.135 2011/08/02 01:22:11 djm Exp $
 .\" $FreeBSD$
-.Dd December 8, 2010
+.Dd August 2, 2011
 .Dt SSHD_CONFIG 5
 .Os
 .Sh NAME
@@ -169,8 +169,9 @@ After expansion,
 .Cm AuthorizedKeysFile
 is taken to be an absolute path or one relative to the user's home
 directory.
+Multiple files may be listed, separated by whitespace.
 The default is
-.Dq .ssh/authorized_keys .
+.Dq .ssh/authorized_keys .ssh/authorized_keys2 .
 .It Cm AuthorizedPrincipalsFile
 Specifies a file that lists principal names that are accepted for
 certificate authentication.
@@ -655,7 +656,9 @@ Multiple algorithms must be comma-separated.
 The default is:
 .Bd -literal -offset indent
 hmac-md5,hmac-sha1,umac-64@openssh.com,
-hmac-ripemd160,hmac-sha1-96,hmac-md5-96
+hmac-ripemd160,hmac-sha1-96,hmac-md5-96,
+hmac-sha2-256,hmac-sha256-96,hmac-sha2-512,
+hmac-sha2-512-96
 .Ed
 .It Cm Match
 Introduces a conditional block.
@@ -1082,11 +1085,17 @@ The goal of privilege separation is to prevent privilege
 escalation by containing any corruption within the unprivileged processes.
 The default is
 .Dq yes .
+If
+.Cm UsePrivilegeSeparation
+is set to
+.Dq sandbox
+then the pre-authentication unprivileged process is subject to additional
+restrictions.
 .It Cm VersionAddendum
 Specifies a string to append to the regular version string to identify
 OS- or site-specific modifications.
 The default is
-.Dq FreeBSD-20110503 .
+.Dq FreeBSD-20111001 .
 .It Cm X11DisplayOffset
 Specifies the first display number available for
 .Xr sshd 8 Ns 's
diff --git a/crypto/openssh/version.h b/crypto/openssh/version.h
index a15489290926..3afcb715db5a 100644
--- a/crypto/openssh/version.h
+++ b/crypto/openssh/version.h
@@ -1,10 +1,9 @@
-/* $OpenBSD: version.h,v 1.61 2011/02/04 00:44:43 djm Exp $ */
+/* $OpenBSD: version.h,v 1.62 2011/08/02 23:13:01 djm Exp $ */
 /* $FreeBSD$ */
 
 #ifndef SSH_VERSION
-
-#define SSH_VERSION_BASE        "OpenSSH_5.8p2"
-#define SSH_VERSION_ADDENDUM    "FreeBSD-20110503"
+#define SSH_VERSION_BASE        "OpenSSH_5.9p1"
+#define SSH_VERSION_ADDENDUM    "FreeBSD-20111001"
 #define SSH_VERSION_HPN		"_hpn13v11"
 #define SSH_VERSION		SSH_VERSION_BASE SSH_VERSION_HPN " " SSH_VERSION_ADDENDUM
 #define SSH_RELEASE             (ssh_version_get())
diff --git a/etc/defaults/periodic.conf b/etc/defaults/periodic.conf
index f6c42df47d9a..8d3d0f486719 100644
--- a/etc/defaults/periodic.conf
+++ b/etc/defaults/periodic.conf
@@ -147,6 +147,12 @@ daily_status_pkg_changes_enable="NO"			# Show package changes
 daily_queuerun_enable="YES"				# Run mail queue
 daily_submit_queuerun="YES"				# Also submit queue
 
+# 800.scrub-zfs
+daily_scrub_zfs_enable="NO"
+daily_scrub_zfs_pools=""			# empty string selects all pools
+daily_scrub_zfs_default_threshold="30"		# days between scrubs
+#daily_scrub_zfs_${poolname}_threshold="30"	# pool specific threshold
+
 # 999.local
 daily_local="/etc/daily.local"				# Local scripts
 
diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf
index 6946eea03194..b9e78ce0f11a 100644
--- a/etc/defaults/rc.conf
+++ b/etc/defaults/rc.conf
@@ -383,6 +383,7 @@ bsnmpd_flags=""			# Flags for bsnmpd.
 ### Network routing options: ###
 defaultrouter="NO"		# Set to default gateway (or NO).
 static_arp_pairs=""		# Set to static ARP list (or leave empty).
+static_ndp_pairs=""		# Set to static NDP list (or leave empty).
 static_routes=""		# Set to static route list (or leave empty).
 natm_static_routes=""		# Set to static route list for NATM (or leave empty).
 gateway_enable="NO"		# Set to YES if this host will be a gateway.
diff --git a/etc/devd.conf b/etc/devd.conf
index 807ec05ffb2b..27abc1f597a7 100644
--- a/etc/devd.conf
+++ b/etc/devd.conf
@@ -8,10 +8,10 @@
 # NB: device-name is shorthand for 'match device-name'
 
 options {
-	# Each directory directive adds a directory the list of directories
-	# that we scan for files.  Files are read-in in the order that they
-	# are returned from readdir(3).  The rule-sets are combined to
-	# create a DFA that's used to match events to actions.
+	# Each "directory" directive adds a directory to the list of
+	# directories that we scan for files.  Files are loaded in the order
+	# that they are returned from readdir(3).  The rule-sets are combined
+	# to create a DFA that's used to match events to actions.
 	directory "/etc/devd";
 	directory "/usr/local/etc/devd";
 	pid-file "/var/run/devd.pid";
@@ -32,18 +32,19 @@ options {
 # script is called pccard_ether.
 #
 # NB: DETACH events are ignored; the kernel should handle all cleanup
-#     (routes, arp cache) if you need to do something beware of races
-#     against immediate create of a device w/ the same name; e.g.
+#     (routes, arp cache).  Beware of races against immediate create
+#     of a device with the same name; e.g.
 #	ifconfig bridge0 destroy; ifconfig bridge0 create
 #
 notify 0 {
 	match "system"		"IFNET";
+	match "subsystem"	"!usbus[0-9]+";
 	match "type"		"ATTACH";
 	action "/etc/pccard_ether $subsystem start";
 };
 
 #
-# Try to start dhclient on Ethernet like interfaces when the link comes
+# Try to start dhclient on Ethernet-like interfaces when the link comes
 # up.  Only devices that are configured to support DHCP will actually
 # run it.  No link down rule exists because dhclient automatically exits
 # when the link goes down.
@@ -87,7 +88,7 @@ detach 100 {
 	device-name "ed50";
 };
 
-# When a USB Bluetooth dongle appears activate it
+# When a USB Bluetooth dongle appears, activate it
 attach 100 {
 	device-name "ubt[0-9]+";
 	action "/etc/rc.d/bluetooth quietstart $device-name";
@@ -125,7 +126,7 @@ detach 100 {
 };
 
 # Firmware download into the ActiveWire board. After the firmware download is
-# done the device detaches and reappears as something new and shiny
+# done, the device detaches and reappears as something new and shiny
 # automatically.
 attach 100 {
 	match "vendor"	"0x0854";
@@ -265,12 +266,11 @@ notify 10 {
 
 /* EXAMPLES TO END OF FILE
 
-# The following might be an example of something that a vendor might
-# install if you were to add their device.  This might reside in
-# /usr/local/etc/devd/deqna.conf.  A deqna is, in this hypothetical
-# example, a pccard ethernet-like device.  Students of history may
-# know other devices by this name, and will get the in-jokes in this
-# entry.
+# An example of something that a vendor might install if you were to
+# add their device.  This might reside in /usr/local/etc/devd/deqna.conf.
+# A deqna is, in this hypothetical example, a pccard ethernet-like device.
+# Students of history may know other devices by this name, and will get
+# the in-jokes in this entry.
 nomatch 10 {
 	match "bus" "pccard[0-9]+";
 	match "manufacturer" "0x1234";
@@ -288,7 +288,7 @@ detach 10 {
 
 # Examples of notify hooks.  A notify is a generic way for a kernel
 # subsystem to send event notification to userland.
-#
+
 # Here are some examples of ACPI notify handlers.  ACPI subsystems that
 # generate notifies include the AC adapter, power/sleep buttons,
 # control method batteries, lid switch, and thermal zones.
@@ -316,7 +316,7 @@ notify 10 {
 };
 
 # This example works around a memory leak in PostgreSQL, restarting
-# it when "user:pgsql:swap:devctl=1G" rctl(8) rule gets triggered.
+# it when the "user:pgsql:swap:devctl=1G" rctl(8) rule gets triggered.
 notify 0 {
 	match "system"		"RCTL";
 	match "rule"		"user:70:swap:.*";
diff --git a/etc/devd/Makefile b/etc/devd/Makefile
index 8d7246abba35..433436b0890c 100644
--- a/etc/devd/Makefile
+++ b/etc/devd/Makefile
@@ -1,6 +1,14 @@
 # $FreeBSD$
 
-FILES=	asus.conf uath.conf usb.conf
+FILES=	uath.conf usb.conf
+
+.if ${MACHINE} == "powerpc"
+FILES+=	apple.conf
+.endif
+
+.if ${MACHINE} == "amd64" || ${MACHINE} == "i386"
+FILES+=	asus.conf
+.endif
 
 NO_OBJ=
 FILESDIR=	/etc/devd
diff --git a/etc/devd/apple.conf b/etc/devd/apple.conf
new file mode 100644
index 000000000000..a57a8e06a640
--- /dev/null
+++ b/etc/devd/apple.conf
@@ -0,0 +1,46 @@
+# $FreeBSD$
+#
+# PowerPC Apple specific devd events
+
+# Keyboard power key
+notify 0 {
+	match "system"		"PMU";
+	match "subsystem"	"Button";
+	match "notify"		"0x0";
+	action			"shutdown -p now";
+};
+
+
+# The next blocks enable volume hotkeys that can be found on Apple laptops
+notify 0 {
+	match "system"		"PMU";
+	match "subsystem"	"keys";
+	match "type"		"mute";
+	action			"mixer 0";
+};
+
+notify 0 {
+	match "system"		"PMU";
+	match "subsystem"	"keys";
+	match "type"		"volume";
+	match "notify"		"down";
+	action			"mixer vol -10";
+};
+
+notify 0 {
+	match "system"		"PMU";
+	match "subsystem"	"keys";
+	match "type"		"volume";
+	match "notify"		"up";
+	action			"mixer vol +10";
+};
+
+# Eject key
+notify 0 {
+	match "system"		"PMU";
+	match "subsystem"	"keys";
+	match "type"		"eject";
+	action			"camcontrol eject cd0";
+};
+
+
diff --git a/etc/devd/usb.conf b/etc/devd/usb.conf
index 568be15fba6b..22cb7ad0b3dd 100644
--- a/etc/devd/usb.conf
+++ b/etc/devd/usb.conf
@@ -561,6 +561,14 @@ nomatch 32 {
 	action "kldload uipaq";
 };
 
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x04e8";
+	match "product" "0x2018";
+	action "kldload if_run";
+};
+
 nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
@@ -741,7 +749,7 @@ nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
 	match "vendor" "0x050d";
-	match "product" "(0x8053|0x805c|0x815c|0x825a)";
+	match "product" "(0x8053|0x805c|0x815c|0x825a|0x825b)";
 	action "kldload if_run";
 };
 
@@ -1877,7 +1885,23 @@ nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
 	match "vendor" "0x0af0";
-	match "product" "(0x5000|0x6000|0x6050|0x6100|0x6150|0x6200|0x6250|0x6300|0x6350|0x6500|0x6501|0x6600|0x6601|0x6701|0x6721|0x6741|0x6761|0x6800|0x6901)";
+	match "product" "(0x5000|0x6000|0x6050|0x6100|0x6150|0x6200|0x6250|0x6300|0x6350|0x6500|0x6501|0x6600|0x6601|0x6701)";
+	action "kldload u3g";
+};
+
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x0af0";
+	match "product" "0x6711";
+	action "kldload uhso";
+};
+
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x0af0";
+	match "product" "(0x6721|0x6741|0x6761|0x6800|0x6901)";
 	action "kldload u3g";
 };
 
@@ -2069,7 +2093,7 @@ nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
 	match "vendor" "0x0b95";
-	match "product" "(0x1720|0x1780|0x7720|0x772a)";
+	match "product" "(0x1720|0x1780|0x7720|0x772a|0x772b)";
 	action "kldload if_axe";
 };
 
@@ -2497,6 +2521,14 @@ nomatch 32 {
 	action "kldload ugensa";
 };
 
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x0f3d";
+	match "product" "0x68a3";
+	action "kldload usie";
+};
+
 nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
@@ -2789,10 +2821,34 @@ nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
 	match "vendor" "0x1199";
-	match "product" "(0x0220|0x0224|0x0fff|0x6802|0x6803|0x6804|0x6805|0x6808|0x6809|0x6812|0x6813|0x6815|0x6816|0x6820|0x6821|0x6822|0x6832|0x6833|0x6834|0x6835|0x6838|0x6839|0x683a|0x683b|0x683c|0x683d|0x683e|0x6850|0x6851|0x6852|0x6853|0x6855|0x6856|0x6859|0x685a|0x6880|0x6890|0x6891|0x6892|0x6893|0x68a3)";
+	match "product" "(0x0220|0x0224|0x0fff)";
 	action "kldload u3g";
 };
 
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x1199";
+	match "product" "0x0fff";
+	action "kldload usie";
+};
+
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x1199";
+	match "product" "(0x6802|0x6803|0x6804|0x6805|0x6808|0x6809|0x6812|0x6813|0x6815|0x6816|0x6820|0x6821|0x6822|0x6832|0x6833|0x6834|0x6835|0x6838|0x6839|0x683a|0x683b|0x683c|0x683d|0x683e|0x6850|0x6851|0x6852|0x6853|0x6855|0x6856|0x6859|0x685a|0x6880|0x6890|0x6891|0x6892|0x6893|0x68a3)";
+	action "kldload u3g";
+};
+
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x1199";
+	match "product" "0x68a3";
+	action "kldload usie";
+};
+
 nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
@@ -2869,7 +2925,7 @@ nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
 	match "vendor" "0x12d1";
-	match "product" "(0x1001|0x1003|0x1004|0x1401|0x1402|0x1403|0x1404|0x1405|0x1406|0x1407|0x1408|0x1409|0x140a|0x140b|0x140c|0x140d|0x140e|0x140f|0x1410|0x1411|0x1412|0x1413|0x1414|0x1415|0x1416|0x1417|0x1418|0x1419|0x141a|0x141b|0x141c|0x141d|0x141e|0x141f|0x1420|0x1421|0x1422|0x1423|0x1424|0x1425|0x1426|0x1427|0x1428|0x1429|0x142a|0x142b|0x142c|0x142d|0x142e|0x142f|0x1430|0x1431|0x1432|0x1433|0x1434|0x1435|0x1436|0x1437|0x1438|0x1439|0x143a|0x143b|0x143c|0x143d|0x143e|0x143f|0x1446|0x1465|0x14ac|0x1520)";
+	match "product" "(0x1001|0x1003|0x1004|0x1401|0x1402|0x1403|0x1404|0x1405|0x1406|0x1407|0x1408|0x1409|0x140a|0x140b|0x140c|0x140d|0x140e|0x140f|0x1410|0x1411|0x1412|0x1413|0x1414|0x1415|0x1416|0x1417|0x1418|0x1419|0x141a|0x141b|0x141c|0x141d|0x141e|0x141f|0x1420|0x1421|0x1422|0x1423|0x1424|0x1425|0x1426|0x1427|0x1428|0x1429|0x142a|0x142b|0x142c|0x142d|0x142e|0x142f|0x1430|0x1431|0x1432|0x1433|0x1434|0x1435|0x1436|0x1437|0x1438|0x1439|0x143a|0x143b|0x143c|0x143d|0x143e|0x143f|0x1446|0x1465|0x14ac|0x1520|0x1c05|0x1c0b)";
 	action "kldload u3g";
 };
 
@@ -2969,6 +3025,14 @@ nomatch 32 {
 	action "kldload if_zyd";
 };
 
+nomatch 32 {
+	match "bus" "uhub[0-9]+";
+	match "mode" "host";
+	match "vendor" "0x13b1";
+	match "product" "0x002f";
+	action "kldload if_run";
+};
+
 nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
@@ -3669,7 +3733,7 @@ nomatch 32 {
 	match "bus" "uhub[0-9]+";
 	match "mode" "host";
 	match "vendor" "0x1c9e";
-	match "product" "(0x6061|0x9603|0xf000)";
+	match "product" "(0x6061|0x9603|0x9605|0xf000)";
 	action "kldload u3g";
 };
 
@@ -4263,5 +4327,5 @@ nomatch 32 {
 	action "kldload umass";
 };
 
-# 1634 USB entries processed
+# 1645 USB entries processed
 
diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist
index f960c5243e2d..5ec72395db49 100644
--- a/etc/mtree/BSD.include.dist
+++ b/etc/mtree/BSD.include.dist
@@ -96,6 +96,8 @@
         ..
         bktr
         ..
+        ciss
+        ..
         firewire
         ..
         hwpmc
diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
index 9ddf4cfdb586..87497b149fa7 100644
--- a/etc/mtree/BSD.usr.dist
+++ b/etc/mtree/BSD.usr.dist
@@ -93,6 +93,10 @@
                 intel_wpi
                 ..
             ..
+            llvm
+                clang
+                ..
+            ..
             ncurses
             ..
             ntp
diff --git a/etc/network.subr b/etc/network.subr
index 579c7018635e..c1faf59bfa3c 100644
--- a/etc/network.subr
+++ b/etc/network.subr
@@ -297,6 +297,10 @@ dhcpif()
 	local _tmpargs _arg
 	_tmpargs=`_ifconfig_getargs $1`
 
+	if noafif $1; then
+		return 1
+	fi
+
 	for _arg in $_tmpargs; do
 		case $_arg in
 		[Dd][Hh][Cc][Pp])
@@ -322,6 +326,10 @@ syncdhcpif()
 	local _tmpargs _arg
 	_tmpargs=`_ifconfig_getargs $1`
 
+	if noafif $1; then
+		return 1
+	fi
+
 	for _arg in $_tmpargs; do
 		case $_arg in
 		[Nn][Oo][Ss][Yy][Nn][Cc][Dd][Hh][Cc][Pp])
@@ -524,7 +532,7 @@ ipv4_up()
 
 	# Add 127.0.0.1/8 to lo0 unless otherwise specified.
 	if [ "${_if}" = "lo0" ]; then
-		ifconfig_args=`ifconfig_getargs ${_if}`
+		ifconfig_args=`get_if_var ${_if} ifconfig_IF`
 		if [ -z "${ifconfig_args}" ]; then
 			ifconfig ${_if} inet 127.0.0.1/8 alias
 		fi
@@ -548,7 +556,7 @@ ipv6_up()
 	fi
 
 	ifalias_up ${_if} inet6 && _ret=0
-	ipv6_prefix_hostid_addr_up ${_if} && _ret=0
+	ipv6_prefix_hostid_addr_common ${_if} alias && _ret=0
 	ipv6_accept_rtadv_up ${_if} && _ret=0
 
 	# wait for DAD
@@ -604,6 +612,7 @@ ipv6_down()
 	fi
 
 	ipv6_accept_rtadv_down ${_if} && _ret=0
+	ipv6_prefix_hostid_addr_common ${_if} -alias && _ret=0
 	ifalias_down ${_if} inet6 && _ret=0
 
 	inetList="`ifconfig ${_if} | grep 'inet6 ' | tr "\n" "$_ifs"`"
@@ -851,12 +860,14 @@ ifalias_ipv6_down()
 	return $_ret
 }
 
-# ipv6_prefix_hostid_addr_up if
-#	add IPv6 prefix + hostid addr to the interface $if
-ipv6_prefix_hostid_addr_up()
+# ipv6_prefix_hostid_addr_common if action
+#	Add or remove IPv6 prefix + hostid addr on the interface $if
+#
+ipv6_prefix_hostid_addr_common()
 {
-	local _if prefix laddr hostid j address
+	local _if _action prefix laddr hostid j address
 	_if=$1
+	_action=$2
 	prefix=`get_if_var ${_if} ipv6_prefix_IF`
 
 	if [ -n "${prefix}" ]; then
@@ -866,13 +877,13 @@ ipv6_prefix_hostid_addr_up()
 
 		for j in ${prefix}; do
 			address=$j\:${hostid}
-			ifconfig ${_if} inet6 ${address} prefixlen 64 alias
+			ifconfig ${_if} inet6 ${address} prefixlen 64 ${_action}
 
 			# if I am a router, add subnet router
 			# anycast address (RFC 2373).
 			if checkyesno ipv6_gateway_enable; then
 				ifconfig ${_if} inet6 $j:: prefixlen 64 \
-					alias anycast
+					${_action} anycast
 			fi
 		done
 	fi
diff --git a/etc/periodic/daily/220.backup-pkgdb b/etc/periodic/daily/220.backup-pkgdb
index a6b50e5b2767..82bf0b3dce6c 100755
--- a/etc/periodic/daily/220.backup-pkgdb
+++ b/etc/periodic/daily/220.backup-pkgdb
@@ -25,7 +25,7 @@ case "$daily_backup_pkgdb_enable" in
 	then
 	    install -d -o root -g wheel -m 750 $bak || {
 		echo '$daily_backup_pkgdb_enable is enabled but' \
-		    "$daily_backup_pkgdb_dbdir doesn't exist" ;
+		    "$daily_backup_pkgdb_dir doesn't exist" ;
 		exit 2 ; }
 	fi
 
diff --git a/etc/portsnap.conf b/etc/portsnap.conf
index bacf14453036..c20944550149 100644
--- a/etc/portsnap.conf
+++ b/etc/portsnap.conf
@@ -32,3 +32,4 @@ KEYPRINT=9b5feee6d69f170e3dd0a2c8e469ddbd64f13f978f2f3aede40c98633216c330
 # List of INDEX files to build and the DESCRIBE file to use for each
 INDEX INDEX-7 DESCRIBE.7
 INDEX INDEX-8 DESCRIBE.8
+INDEX INDEX-9 DESCRIBE.9
diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile
index fc09cafc4bb6..0a0ff0aef4c6 100644
--- a/etc/rc.d/Makefile
+++ b/etc/rc.d/Makefile
@@ -32,7 +32,7 @@ FILES=	DAEMON FILESYSTEMS LOGIN NETWORKING SERVERS \
 	random rarpd rctl resolv rfcomm_pppd_server root \
 	route6d routed routing rpcbind rtadvd rtsold rwho \
 	savecore sdpd securelevel sendmail \
-	serial sppp statd static_arp stf swap1 \
+	serial sppp statd static_arp static_ndp stf swap1 \
 	syscons sysctl syslogd \
 	timed tmp \
 	ugidfw \
diff --git a/etc/rc.d/NETWORKING b/etc/rc.d/NETWORKING
index 8da2498ce3aa..71cf26dbc697 100755
--- a/etc/rc.d/NETWORKING
+++ b/etc/rc.d/NETWORKING
@@ -5,7 +5,8 @@
 
 # PROVIDE: NETWORKING NETWORK
 # REQUIRE: netif netoptions routing ppp ipfw stf faith
-# REQUIRE: defaultroute routed mrouted route6d mroute6d resolv
+# REQUIRE: defaultroute routed mrouted route6d mroute6d resolv bridge
+# REQUIRE: static_arp static_ndp
 
 #	This is a dummy dependency, for services which require networking
 #	to be operational before starting.
diff --git a/etc/rc.d/bridge b/etc/rc.d/bridge
index 272703899c32..2c3bfd0de7ec 100755
--- a/etc/rc.d/bridge
+++ b/etc/rc.d/bridge
@@ -26,7 +26,7 @@
 #
 
 # PROVIDE: bridge
-# REQUIRE: netif
+# REQUIRE: netif faith ppp stf
 # KEYWORD: nojail
 
 . /etc/rc.subr
diff --git a/etc/rc.d/dhclient b/etc/rc.d/dhclient
index c7e4b4df4531..f842e12b9753 100755
--- a/etc/rc.d/dhclient
+++ b/etc/rc.d/dhclient
@@ -9,55 +9,39 @@
 . /etc/rc.subr
 . /etc/network.subr
 
+ifn="$2"
+
 name="dhclient"
 rcvar=
-start_cmd="dhclient_start"
-stop_cmd="dhclient_stop"
+pidfile="/var/run/${name}.${ifn}.pid"
+start_precmd="dhclient_prestart"
 
-dhclient_start()
+dhclient_prestart()
 {
-	# prevent unnecessary restarts
-	# XXX: dhclient had better create a pidfile
-	if [ -x /bin/pgrep ]; then
-		pids=`/bin/pgrep -f "dhclient: $ifn(\$| .*)"`
-		if [ -n "$pids" ]; then
-			sleep 1
-			pids=`/bin/pgrep -f "dhclient: $ifn(\$| .*)"`
-			if [ -n "$pids" ]; then
-				exit 0
-			fi
-		elif [ -e /var/run/dhclient.pid ]; then
-			if [ -n "`pgrep -F /var/run/dhclient.pid`" ]; then
-				exit 0
-			fi
-		fi
-	fi
-
-	# Override for $ifn specific flags (see rc.subr for $flags setting)
-	specific=`get_if_var $ifn dhclient_flags_IF`
+	# Interface-specific flags (see rc.subr for $flags setting)
+	specific=$(get_if_var $ifn dhclient_flags_IF)
 	if [ -z "$flags" -a -n "$specific" ]; then
 		rc_flags=$specific
 	fi
 
-	background_dhclient=`get_if_var $ifn background_dhclient_IF $background_dhclient`
+	background_dhclient=$(get_if_var $ifn background_dhclient_IF $background_dhclient)
 	if checkyesno background_dhclient; then
 		rc_flags="${rc_flags} -b"
 	fi
 
-	${dhclient_program} ${rc_flags} $ifn
+	rc_flags="${rc_flags} ${ifn}"
 }
 
-dhclient_stop()
-{
-	ifconfig $ifn down	# cause dhclient to die
-}
-
-ifn="$2"
-
 load_rc_config $name
 load_rc_config network
 
-if ! dhcpif $ifn; then
+if [ -z $ifn ] ; then
+	# only complain if a command was specified but no interface
+	if [ -n "$1" ] ; then
+		echo 1>&2 "$0: no interface specified"
+		return 1
+	fi
+elif [ -z "${rc_force}" ] && ! dhcpif $ifn; then
 	return 1
 fi
 
diff --git a/etc/rc.d/resolv b/etc/rc.d/resolv
index c34209cd3701..aa7921fa1fc6 100755
--- a/etc/rc.d/resolv
+++ b/etc/rc.d/resolv
@@ -28,7 +28,7 @@
 #
 
 # PROVIDE: resolv
-# REQUIRE: netif
+# REQUIRE: netif var
 # KEYWORD: nojail
 
 . /etc/rc.subr
@@ -41,17 +41,17 @@ load_rc_config $name
 # if the info is available via dhcp/kenv
 # build the resolv.conf
 #
-if [ ! -e /etc/resolv.conf -a \
-    -n "`/bin/kenv dhcp.domain-name-servers 2> /dev/null`" ]; then
-	> /etc/resolv.conf
-
+if [ -n "`/bin/kenv dhcp.domain-name-servers 2> /dev/null`" ]; then
+	interface="`/bin/kenv boot.netif.name`"
+	(
 	if [ -n "`/bin/kenv dhcp.domain-name 2> /dev/null`" ]; then
-		echo domain `/bin/kenv dhcp.domain-name` > /etc/resolv.conf
+		echo domain `/bin/kenv dhcp.domain-name`
 	fi
 
 	set -- `/bin/kenv dhcp.domain-name-servers`
 	for ns in `IFS=','; echo $*`; do
-		echo nameserver $ns >> /etc/resolv.conf;
+		echo nameserver $ns
 	done
+	) | /sbin/resolvconf -a ${interface}:dhcp4
 fi
 
diff --git a/release/scripts/mkpkgindex.sh b/etc/rc.d/static_ndp
old mode 100644
new mode 100755
similarity index 60%
rename from release/scripts/mkpkgindex.sh
rename to etc/rc.d/static_ndp
index b70a258257df..314adbf7c565
--- a/release/scripts/mkpkgindex.sh
+++ b/etc/rc.d/static_ndp
@@ -1,7 +1,6 @@
-#! /bin/sh
-# ex:ts=8
-
-# Copyright (c) 2003 David E. O'Brien
+#!/bin/sh
+#
+# Copyright (c) 2011  Xin Li <delphij@FreeBSD.org>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -25,33 +24,51 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
+# Configure static NDP table
+#
 # $FreeBSD$
+#
 
-# Creates an INDEX file suitable for an ISO distribution image from a master
-# INDEX file.  The generated INDEX file contains only the packages in the
-# supplied directory.
+# PROVIDE: static_ndp
+# REQUIRE: netif
+# KEYWORD: nojail
 
-case $# in
-	3) PKG_EXT="tbz" ;;
-	4) PKG_EXT=$4 ;;
-	*)
-	echo `basename $0` "<master index file> <output index file> <pkg dir> [pkg ext]"
-	exit 1
-	;;
-esac
+. /etc/rc.subr
+. /etc/network.subr
 
-PKG_LIST=$(basename `ls $3/*.${PKG_EXT}` | sed -e "s/\.${PKG_EXT}$//")
-REGEX=$(echo ${PKG_LIST} | sed \
-	-e 's/ /|/g' \
-	-e 's/\./\\\./g' \
-	-e 's/\+/\\\+/g' \
-	-e 's/\^/\\\^/g')
+name="static_ndp"
+start_cmd="static_ndp_start"
+stop_cmd="static_ndp_stop"
 
-egrep "^(${REGEX})" $1 > $2
+static_ndp_start()
+{
+	local e ndp_args
 
-if [ $(echo ${PKG_LIST} | wc -w) != $(wc -l < $2) ]; then
-	echo "ERROR: some packages not found in $1"
-	exit 1
-else
-	exit 0
-fi
+	if [ -n "${static_ndp_pairs}" ]; then
+		echo -n 'Binding static NDP pair(s):'
+		for e in ${static_ndp_pairs}; do
+			echo -n " ${e}"
+			eval ndp_args=\$static_ndp_${e}
+			ndp -s ${ndp_args} >/dev/null 2>&1
+		done
+		echo '.'
+	fi
+}
+
+static_ndp_stop()
+{
+	local e ndp_args
+
+	if [ -n "${static_ndp_pairs}" ]; then
+		echo -n 'Unbinding static NDP pair(s):'
+		for e in ${static_ndp_pairs}; do
+			echo -n " ${e}"
+			eval ndp_args=\$static_ndp_${e}
+			ndp -d ${ndp_args%%[ 	]*} > /dev/null 2>&1
+		done
+		echo '.'
+	fi
+}
+
+load_rc_config $name
+run_rc_command "$1"
diff --git a/gnu/usr.bin/binutils/objcopy/objcopy.1 b/gnu/usr.bin/binutils/objcopy/objcopy.1
index bf4501711c64..66f051d3b86f 100644
--- a/gnu/usr.bin/binutils/objcopy/objcopy.1
+++ b/gnu/usr.bin/binutils/objcopy/objcopy.1
@@ -738,8 +738,8 @@ distribution and the second a debugging information file which is only
 needed if debugging abilities are required.  The suggested procedure
 to create these files is as follows:
 .RS 4
-.IP "1.<Link the executable as normal.  Assuming that is is called>" 4
-.IX Item "1.<Link the executable as normal.  Assuming that is is called>"
+.IP "1.<Link the executable as normal.  Assuming that it is called>" 4
+.IX Item "1.<Link the executable as normal.  Assuming that it is called>"
 \&\f(CW\*(C`foo\*(C'\fR then...
 .ie n .IP "1.<Run ""objcopy \-\-only\-keep\-debug foo foo.dbg"" to>" 4
 .el .IP "1.<Run \f(CWobjcopy \-\-only\-keep\-debug foo foo.dbg\fR to>" 4
diff --git a/gnu/usr.bin/binutils/strip/strip.1 b/gnu/usr.bin/binutils/strip/strip.1
index 5eb714dd88a5..2b77c33c1ba6 100644
--- a/gnu/usr.bin/binutils/strip/strip.1
+++ b/gnu/usr.bin/binutils/strip/strip.1
@@ -300,8 +300,8 @@ distribution and the second a debugging information file which is only
 needed if debugging abilities are required.  The suggested procedure
 to create these files is as follows:
 .RS 4
-.IP "1.<Link the executable as normal.  Assuming that is is called>" 4
-.IX Item "1.<Link the executable as normal.  Assuming that is is called>"
+.IP "1.<Link the executable as normal.  Assuming that it is called>" 4
+.IX Item "1.<Link the executable as normal.  Assuming that it is called>"
 \&\f(CW\*(C`foo\*(C'\fR then...
 .ie n .IP "1.<Run ""objcopy \-\-only\-keep\-debug foo foo.dbg"" to>" 4
 .el .IP "1.<Run \f(CWobjcopy \-\-only\-keep\-debug foo foo.dbg\fR to>" 4
diff --git a/gnu/usr.bin/cc/Makefile.ver b/gnu/usr.bin/cc/Makefile.ver
index c3b8c4a6f99d..c7f460560ff1 100644
--- a/gnu/usr.bin/cc/Makefile.ver
+++ b/gnu/usr.bin/cc/Makefile.ver
@@ -4,7 +4,7 @@ BASEVER!=	cat ${GCCDIR}/BASE-VER
 DATESTAMP!=	cat ${GCCDIR}/DATESTAMP
 DEVPHASE!=	cat ${GCCDIR}/DEV-PHASE
 
-version.o:	version.c
+version.o: version.c ${GCCDIR}/BASE-VER ${GCCDIR}/DATESTAMP ${GCCDIR}/DEV-PHASE
 	${CC} ${CFLAGS} -DBASEVER=\"${BASEVER}\" \
 		-DDATESTAMP=\"\ ${DATESTAMP}\" \
 		-DDEVPHASE=\"\ ${DEVPHASE}\" -c ${.IMPSRC}
diff --git a/gnu/usr.bin/cc/cc_tools/freebsd-native.h b/gnu/usr.bin/cc/cc_tools/freebsd-native.h
index db4dd89ff53f..30cd9b1b2e35 100644
--- a/gnu/usr.bin/cc/cc_tools/freebsd-native.h
+++ b/gnu/usr.bin/cc/cc_tools/freebsd-native.h
@@ -7,8 +7,8 @@
 #define FREEBSD_NATIVE 1
 
 /* Fake out gcc/config/freebsd<version>.h.  */
-#define	FBSD_MAJOR	8
-#define	FBSD_CC_VER	800001		/* form like __FreeBSD_version */
+#define	FBSD_MAJOR	10
+#define	FBSD_CC_VER	1000001		/* form like __FreeBSD_version */
 
 #undef SYSTEM_INCLUDE_DIR		/* We don't need one for now. */
 #undef TOOL_INCLUDE_DIR			/* We don't need one for now. */
diff --git a/gnu/usr.bin/groff/tmac/mdoc.local b/gnu/usr.bin/groff/tmac/mdoc.local
index 8c356f5656ba..4436c6095404 100644
--- a/gnu/usr.bin/groff/tmac/mdoc.local
+++ b/gnu/usr.bin/groff/tmac/mdoc.local
@@ -71,12 +71,13 @@
 .ds doc-volume-as-arm      arm
 .
 .\" Default .Os value
-.ds doc-default-operating-system FreeBSD\~9.0
+.ds doc-default-operating-system FreeBSD\~10.0
 .
 .\" FreeBSD releases not found in doc-common
 .ds doc-operating-system-FreeBSD-7.4    7.4
 .ds doc-operating-system-FreeBSD-8.3    8.3
 .ds doc-operating-system-FreeBSD-9.0    9.0
+.ds doc-operating-system-FreeBSD-10.0   10.0
 .
 .\" Definitions not (yet) in doc-syms
 .
diff --git a/include/Makefile b/include/Makefile
index 7c8ab1a735df..3e6705c5e3f2 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -39,7 +39,7 @@ LDIRS=	bsm cam geom net net80211 netatalk netgraph netinet netinet6 \
 	sys vm
 
 LSUBDIRS=	cam/ata cam/scsi \
-	dev/acpica dev/an dev/bktr dev/firewire dev/hwpmc \
+	dev/acpica dev/an dev/bktr dev/ciss dev/firewire dev/hwpmc \
 	dev/ic dev/iicbus ${_dev_ieee488} dev/io dev/lmc dev/mfi dev/ofw \
 	dev/pbio ${_dev_powermac_nvram} dev/ppbus dev/smbus \
 	dev/speaker dev/usb dev/utopia dev/vkbd dev/wi \
diff --git a/include/complex.h b/include/complex.h
index f37b003e0d50..414230472d2b 100644
--- a/include/complex.h
+++ b/include/complex.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2001-2008 The FreeBSD Project.
+ * Copyright (c) 2001-2011 The FreeBSD Project.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -49,6 +49,10 @@ long double	cabsl(long double complex);
 double		carg(double complex);
 float		cargf(float complex);
 long double	cargl(long double complex);
+double complex	ccos(double complex);
+float complex	ccosf(float complex);
+double complex	ccosh(double complex);
+float complex	ccoshf(float complex);
 double complex	cexp(double complex);
 float complex	cexpf(float complex);
 double		cimag(double complex) __pure2;
@@ -65,10 +69,18 @@ long double complex
 double		creal(double complex) __pure2;
 float		crealf(float complex) __pure2;
 long double	creall(long double complex) __pure2;
+double complex	csin(double complex);
+float complex	csinf(float complex);
+double complex	csinh(double complex);
+float complex	csinhf(float complex);
 double complex	csqrt(double complex);
 float complex	csqrtf(float complex);
 long double complex
 		csqrtl(long double complex);
+double complex	ctan(double complex);
+float complex	ctanf(float complex);
+double complex	ctanh(double complex);
+float complex	ctanhf(float complex);
 
 __END_DECLS
 
diff --git a/include/ieeefp.h b/include/ieeefp.h
index e4f943d38916..3f64555d5a14 100644
--- a/include/ieeefp.h
+++ b/include/ieeefp.h
@@ -12,15 +12,4 @@
 #include <sys/cdefs.h>
 #include <machine/ieeefp.h>
 
-#if !defined(_IEEEFP_INLINED_)
-__BEGIN_DECLS
-extern fp_rnd_t    fpgetround(void);
-extern fp_rnd_t    fpsetround(fp_rnd_t);
-extern fp_except_t fpgetmask(void);
-extern fp_except_t fpsetmask(fp_except_t);
-extern fp_except_t fpgetsticky(void);
-extern fp_except_t fpsetsticky(fp_except_t);
-__END_DECLS
-#endif /* !_IEEEFP_INLINED_ */
-
 #endif /* _IEEEFP_H_ */
diff --git a/include/iso646.h b/include/iso646.h
index d38bf4be6a2e..a11101e525ce 100644
--- a/include/iso646.h
+++ b/include/iso646.h
@@ -29,6 +29,8 @@
 #ifndef _ISO646_H_
 #define	_ISO646_H_
 
+#ifndef __cplusplus
+
 #define	and	&&
 #define	and_eq	&=
 #define	bitand	&
@@ -41,4 +43,6 @@
 #define	xor	^
 #define	xor_eq	^=
 
+#endif /* !__cplusplus */
+
 #endif /* !_ISO646_H_ */
diff --git a/kerberos5/lib/libasn1/Makefile b/kerberos5/lib/libasn1/Makefile
index 4a9c21ebfd2a..cf91a4bc8d7f 100644
--- a/kerberos5/lib/libasn1/Makefile
+++ b/kerberos5/lib/libasn1/Makefile
@@ -1,7 +1,10 @@
 # $FreeBSD$
 
 LIB=	asn1
+LDFLAGS=	-Wl,--no-undefined
 INCS=	asn1_err.h heim_asn1.h
+LDADD=	-lcom_err -lroken
+DPADD=	${LIBCOM_ERR} ${LIBROKEN}
 
 SRCS=	asn1_err.c \
 	asn1_err.h \
diff --git a/kerberos5/lib/libgssapi_krb5/Makefile b/kerberos5/lib/libgssapi_krb5/Makefile
index 6c2f4217b3c7..943e87572632 100644
--- a/kerberos5/lib/libgssapi_krb5/Makefile
+++ b/kerberos5/lib/libgssapi_krb5/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 
 LIB=	gssapi_krb5
-LDFLAGS= -Wl,-Bsymbolic
+LDFLAGS= -Wl,-Bsymbolic -Wl,--no-undefined
 LDADD=	-lgssapi -lkrb5 -lhx509 -lcrypto -lroken -lasn1 -lcom_err -lcrypt
 DPADD=	${LIBGSSAPI} ${LIBKRB5} ${LIBHX509} ${LIBCRYPTO} ${LIBROKEN} ${LIBASN1} \
 	${LIBCOM_ERR} ${LIBCRYPT}
diff --git a/kerberos5/lib/libgssapi_ntlm/Makefile b/kerberos5/lib/libgssapi_ntlm/Makefile
index cbecc2d5ee93..98b47f716c24 100644
--- a/kerberos5/lib/libgssapi_ntlm/Makefile
+++ b/kerberos5/lib/libgssapi_ntlm/Makefile
@@ -1,9 +1,9 @@
 # $FreeBSD$
 
 LIB=	gssapi_ntlm
-LDFLAGS= -Wl,-Bsymbolic
-LDADD=	-lkrb5 -lhx509 -lheimntlm -lroken
-DPADD=	${LIBKRB5} ${LIBHX509} ${LIBHEIMNTLM} ${LIBROKEN}
+LDFLAGS= -Wl,-Bsymbolic -Wl,--no-undefined
+LDADD=	-lcrypto -lgssapi -lkrb5 -lhx509 -lheimntlm -lroken
+DPADD=	${LIBCRYPTO} ${LIBGSSAPI} ${LIBKRB5} ${LIBHX509} ${LIBHEIMNTLM} ${LIBROKEN}
 
 SRCS=	accept_sec_context.c \
 	acquire_cred.c \
diff --git a/kerberos5/lib/libgssapi_spnego/Makefile b/kerberos5/lib/libgssapi_spnego/Makefile
index 32348ecf1ba5..ce44dbee2a7b 100644
--- a/kerberos5/lib/libgssapi_spnego/Makefile
+++ b/kerberos5/lib/libgssapi_spnego/Makefile
@@ -1,9 +1,9 @@
 # $FreeBSD$
 
 LIB=	gssapi_spnego
-LDFLAGS= -Wl,-Bsymbolic
-LDADD=	-lgssapi -lasn1
-DPADD=	${LIBGSSAPI} ${LIBASN1}
+LDFLAGS= -Wl,-Bsymbolic -Wl,--no-undefined
+LDADD=	-lgssapi -lasn1 -lroken
+DPADD=	${LIBGSSAPI} ${LIBASN1} ${LIBROKEN}
 
 SRCS=	accept_sec_context.c \
 	compat.c \
diff --git a/kerberos5/lib/libhdb/Makefile b/kerberos5/lib/libhdb/Makefile
index 75465d3768d5..653690a1ff46 100644
--- a/kerberos5/lib/libhdb/Makefile
+++ b/kerberos5/lib/libhdb/Makefile
@@ -1,6 +1,9 @@
 # $FreeBSD$
 
 LIB=	hdb
+LDFLAGS=	-Wl,--no-undefined
+LDADD=	-lasn1 -lcom_err -lkrb5 -lroken
+DPADD=	${LIBASN1} ${LIBCOM_ERR} ${LIBKRB5} ${LIBROKEN}
 
 INCS=	hdb-private.h \
 	hdb-protos.h \
diff --git a/kerberos5/lib/libheimntlm/Makefile b/kerberos5/lib/libheimntlm/Makefile
index e223258a50fa..ba7c922d642e 100644
--- a/kerberos5/lib/libheimntlm/Makefile
+++ b/kerberos5/lib/libheimntlm/Makefile
@@ -1,6 +1,9 @@
 # $FreeBSD$
 
 LIB=	heimntlm
+LDFLAGS=	-Wl,--no-undefined
+LDADD=	-lcrypto -lkrb5
+DPADD=	${LIBCRYPTO} ${LIBKRB5}
 SRCS=	ntlm.c
 INCS=	heimntlm.h heimntlm-protos.h
 CFLAGS+=-I${KRB5DIR}/lib/ntlm
diff --git a/kerberos5/lib/libhx509/Makefile b/kerberos5/lib/libhx509/Makefile
index e94831aaca73..9238a74bc96d 100644
--- a/kerberos5/lib/libhx509/Makefile
+++ b/kerberos5/lib/libhx509/Makefile
@@ -1,7 +1,10 @@
 # $FreeBSD$
 
 LIB=	hx509
+LDFLAGS=	-Wl,--no-undefined
 VERSION_MAP= ${KRB5DIR}/lib/hx509/version-script.map
+LDADD=	-lasn1 -lcom_err -lcrypto -lroken
+DPADD=	${LIBASN1} ${LIBCOM_ERR} ${LIBCRYPTO} ${LIBROKEN}
 
 INCS=	hx509-private.h \
 	hx509-protos.h \
diff --git a/kerberos5/lib/libkadm5clnt/Makefile b/kerberos5/lib/libkadm5clnt/Makefile
index 339086686613..dd93847e335f 100644
--- a/kerberos5/lib/libkadm5clnt/Makefile
+++ b/kerberos5/lib/libkadm5clnt/Makefile
@@ -1,6 +1,9 @@
 # $FreeBSD$
 
 LIB=	kadm5clnt
+LDFLAGS=	-Wl,--no-undefined
+LDADD=	-lcom_err -lkrb5 -lroken
+DPADD=	${LIBCOM_ERR} ${LIBKRB5} ${LIBROKEN}
 
 INCS=	admin.h \
 	kadm5-private.h \
diff --git a/kerberos5/lib/libkadm5srv/Makefile b/kerberos5/lib/libkadm5srv/Makefile
index c0be47763673..43bd6e07cc75 100644
--- a/kerberos5/lib/libkadm5srv/Makefile
+++ b/kerberos5/lib/libkadm5srv/Makefile
@@ -1,6 +1,9 @@
 # $FreeBSD$
 
 LIB=	kadm5srv
+LDFLAGS=	-Wl,--no-undefined
+LDADD=	-lcom_err -lhdb -lkrb5 -lroken
+DPADD=	${LIBCOM_ERR} ${LIBHDB} ${LIBKRB5} ${LIBROKEN}
 VERSION_MAP= ${KRB5DIR}/lib/kadm5/version-script.map
 
 SRCS=	acl.c \
diff --git a/kerberos5/lib/libkafs5/Makefile b/kerberos5/lib/libkafs5/Makefile
index e0e0b3036a5e..962a900bd9cf 100644
--- a/kerberos5/lib/libkafs5/Makefile
+++ b/kerberos5/lib/libkafs5/Makefile
@@ -1,6 +1,9 @@
 # $FreeBSD$
 
 LIB=	kafs5
+LDFLAGS=	-Wl,--no-undefined
+LDADD=	-lasn1 -lkrb5 -lroken
+DPADD=	${LIBASN1} ${LIBKRB5} ${LIBROKEN}
 INCS=	kafs.h
 MAN=	kafs5.3
 
diff --git a/kerberos5/lib/libkrb5/Makefile b/kerberos5/lib/libkrb5/Makefile
index 29653388b1c6..71175cdad518 100644
--- a/kerberos5/lib/libkrb5/Makefile
+++ b/kerberos5/lib/libkrb5/Makefile
@@ -1,7 +1,10 @@
 # $FreeBSD$
 
 LIB=	krb5
+LDFLAGS=	-Wl,--no-undefined
 VERSION_MAP= ${KRB5DIR}/lib/krb5/version-script.map
+LDADD=	-lasn1 -lcom_err -lcrypt -lcrypto -lhx509 -lroken
+DPADD=	${LIBASN1} ${LIBCOM_ERR} ${LIBCRYPT} ${LIBCRYPTO} ${LIBHX509} ${LIBROKEN}
 
 INCS=	heim_err.h \
 	heim_threads.h \
diff --git a/kerberos5/lib/libroken/Makefile b/kerberos5/lib/libroken/Makefile
index 92f38a8447b8..c88e1ce9f7e1 100644
--- a/kerberos5/lib/libroken/Makefile
+++ b/kerberos5/lib/libroken/Makefile
@@ -1,6 +1,8 @@
 # $FreeBSD$
 
 LIB=	roken
+LDADD=	-lcrypt
+DPADD=	${LIBCRYPT}
 INCS=	roken.h roken-common.h
 
 SRCS=	base64.c \
diff --git a/lib/Makefile b/lib/Makefile
index 9a898860f053..6537b5a234ee 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -70,7 +70,6 @@ SUBDIR=	${SUBDIR_ORDERED} \
 	${_libefi} \
 	libexpat \
 	libfetch \
-	libftpio \
 	libgeom \
 	${_libgpib} \
 	${_libgssapi} \
@@ -188,18 +187,15 @@ _libproc=	libproc
 _librtld_db=	librtld_db
 .endif
 
+.if ${MACHINE_CPUARCH} == "amd64"
+_libvmmapi=	libvmmapi
+.endif
+
 .if ${MACHINE_CPUARCH} == "ia64"
 _libefi=	libefi
 _libsmb=	libsmb
 .endif
 
-.if ${MACHINE_CPUARCH} == "amd64"
-.if ${MK_NCP} != "no"
-_libncp=	libncp
-.endif
-_libvmmapi=	libvmmapi
-.endif
-
 .if ${MACHINE_CPUARCH} == "powerpc"
 _libsmb=	libsmb
 .endif
diff --git a/lib/clang/Makefile b/lib/clang/Makefile
index abcc11314dae..8f7d18a31213 100644
--- a/lib/clang/Makefile
+++ b/lib/clang/Makefile
@@ -35,6 +35,7 @@ SUBDIR=	libclanganalysis \
 	libllvmscalaropts \
 	libllvmselectiondag \
 	libllvmsupport \
+	libllvmtablegen \
 	libllvmtarget \
 	libllvmtransformutils \
 	\
diff --git a/lib/clang/clang.build.mk b/lib/clang/clang.build.mk
index 17b47c332179..ceaeb05582b1 100644
--- a/lib/clang/clang.build.mk
+++ b/lib/clang/clang.build.mk
@@ -14,8 +14,7 @@ CFLAGS+= -O1
 .endif
 
 TARGET_ARCH?=	${MACHINE_ARCH}
-# XXX: 8.0, to keep __FreeBSD_cc_version happy
-CFLAGS+=-DLLVM_HOSTTRIPLE=\"${TARGET_ARCH:C/amd64/x86_64/}-unknown-freebsd9.0\"
+CFLAGS+=-DLLVM_HOSTTRIPLE=\"${TARGET_ARCH:C/amd64/x86_64/}-unknown-freebsd10.0\"
 
 .ifndef LLVM_REQUIRES_EH
 CXXFLAGS+=-fno-exceptions
@@ -35,6 +34,7 @@ CFLAGS+=-DCLANG_PREFIX=\"${TOOLS_PREFIX}\"
 .PATH:	${LLVM_SRCS}/${SRCDIR}
 
 TBLGEN?=tblgen
+CLANG_TBLGEN?=clang-tblgen
 TBLINC+=-I ${LLVM_SRCS}/include -I ${LLVM_SRCS}/lib/Target
 
 Intrinsics.inc.h: ${LLVM_SRCS}/include/llvm/Intrinsics.td
@@ -64,82 +64,83 @@ ${arch:T}Gen${hdr:H:C/$/.inc.h/}: ${LLVM_SRCS}/lib/Target/${arch:H}/${arch:T}.td
 . endfor
 .endfor
 
-ARMGenDecoderTables.inc.h: ${LLVM_SRCS}/lib/Target/ARM/ARM.td
-	${TBLGEN} -I ${LLVM_SRCS}/lib/Target/ARM ${TBLINC} \
-	    -gen-arm-decoder -o ${.TARGET} ${.ALLSRC}
-
 Attrs.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
 	    -gen-clang-attr-classes -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include ${.ALLSRC}
 
 AttrImpl.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
 	    -gen-clang-attr-impl -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include ${.ALLSRC}
 
+AttrLateParsed.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
+	    -gen-clang-attr-late-parsed-list -o ${.TARGET} \
+	    -I ${CLANG_SRCS}/include ${.ALLSRC}
+
 AttrList.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
 	    -gen-clang-attr-list -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include ${.ALLSRC}
 
 AttrPCHRead.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Serialization \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Serialization \
 	    ${TBLINC} -gen-clang-attr-pch-read -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include ${.ALLSRC}
 
 AttrPCHWrite.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Serialization \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Serialization \
 	    ${TBLINC} -gen-clang-attr-pch-write -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include ${.ALLSRC}
 
 AttrSpellings.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Lex ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Lex ${TBLINC} \
 	    -gen-clang-attr-spelling-list -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include ${.ALLSRC}
 
 DeclNodes.inc.h: ${CLANG_SRCS}/include/clang/Basic/DeclNodes.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
 	    -gen-clang-decl-nodes -o ${.TARGET} ${.ALLSRC}
 
 StmtNodes.inc.h: ${CLANG_SRCS}/include/clang/Basic/StmtNodes.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/AST ${TBLINC} \
 	    -gen-clang-stmt-nodes -o ${.TARGET} ${.ALLSRC}
 
 arm_neon.inc.h: ${CLANG_SRCS}/include/clang/Basic/arm_neon.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
 	    -gen-arm-neon-sema -o ${.TARGET} ${.ALLSRC}
 
 DiagnosticGroups.inc.h: ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
 	    -gen-clang-diag-groups -o ${.TARGET} ${.ALLSRC}
 
 DiagnosticIndexName.inc.h: ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
 	    -gen-clang-diags-index-name -o ${.TARGET} ${.ALLSRC}
 
 .for hdr in AST Analysis Common Driver Frontend Lex Parse Sema
 Diagnostic${hdr}Kinds.inc.h: ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Basic ${TBLINC} \
 	    -gen-clang-diags-defs -clang-component=${hdr} \
 	    -o ${.TARGET} ${.ALLSRC}
 .endfor
 
 Options.inc.h: ${CLANG_SRCS}/include/clang/Driver/Options.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Driver ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Driver ${TBLINC} \
 	    -gen-opt-parser-defs -o ${.TARGET} ${.ALLSRC}
 
 CC1Options.inc.h: ${CLANG_SRCS}/include/clang/Driver/CC1Options.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Driver ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Driver ${TBLINC} \
 	    -gen-opt-parser-defs -o ${.TARGET} ${.ALLSRC}
 
 CC1AsOptions.inc.h: ${CLANG_SRCS}/include/clang/Driver/CC1AsOptions.td
-	${TBLGEN} -I ${CLANG_SRCS}/include/clang/Driver ${TBLINC} \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/include/clang/Driver ${TBLINC} \
 	    -gen-opt-parser-defs -o ${.TARGET} ${.ALLSRC}
 
 Checkers.inc.h: ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers/Checkers.td \
 	    ${CLANG_SRCS}/include/clang/StaticAnalyzer/Checkers/CheckerBase.td
-	${TBLGEN} -I ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers \
+	${CLANG_TBLGEN} -I ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers \
 	    ${TBLINC} -gen-clang-sa-checkers -o ${.TARGET} \
 	    -I ${CLANG_SRCS}/include \
 	    ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers/Checkers.td
diff --git a/lib/clang/include/ARMGenDecoderTables.inc b/lib/clang/include/ARMGenDecoderTables.inc
deleted file mode 100644
index b09685cd40d5..000000000000
--- a/lib/clang/include/ARMGenDecoderTables.inc
+++ /dev/null
@@ -1,2 +0,0 @@
-/* $FreeBSD$ */
-#include "ARMGenDecoderTables.inc.h"
diff --git a/lib/clang/include/ARMGenDisassemblerTables.inc b/lib/clang/include/ARMGenDisassemblerTables.inc
new file mode 100644
index 000000000000..7ea91ee1149d
--- /dev/null
+++ b/lib/clang/include/ARMGenDisassemblerTables.inc
@@ -0,0 +1,2 @@
+/* $FreeBSD$ */
+#include "ARMGenDisassemblerTables.inc.h"
diff --git a/lib/clang/include/clang/Basic/Version.inc b/lib/clang/include/clang/Basic/Version.inc
index 3e3b44cbf19a..1ab3e3eb8c79 100644
--- a/lib/clang/include/clang/Basic/Version.inc
+++ b/lib/clang/include/clang/Basic/Version.inc
@@ -5,6 +5,6 @@
 #define	CLANG_VERSION_MINOR	0
 
 #define	CLANG_VENDOR		"FreeBSD "
-#define	CLANG_VENDOR_SUFFIX	" 20110717"
+#define	CLANG_VENDOR_SUFFIX	" 20111021"
 
-#define	SVN_REVISION		"135360"
+#define	SVN_REVISION		"142614"
diff --git a/lib/clang/include/clang/Parse/AttrLateParsed.inc b/lib/clang/include/clang/Parse/AttrLateParsed.inc
new file mode 100644
index 000000000000..1282d9e188ff
--- /dev/null
+++ b/lib/clang/include/clang/Parse/AttrLateParsed.inc
@@ -0,0 +1,2 @@
+/* $FreeBSD$ */
+#include "AttrLateParsed.inc.h"
diff --git a/lib/clang/include/llvm/Config/config.h b/lib/clang/include/llvm/Config/config.h
index f2645717a53e..5b6761c2eeaa 100644
--- a/lib/clang/include/llvm/Config/config.h
+++ b/lib/clang/include/llvm/Config/config.h
@@ -5,6 +5,9 @@
 #ifndef CONFIG_H
 #define CONFIG_H
 
+/* Bug report URL. */
+#define BUG_REPORT_URL "http://llvm.org/bugs/"
+
 /* Relative directory for resource files */
 #define CLANG_RESOURCE_DIR ""
 
@@ -301,16 +304,16 @@
 #define HAVE_PRINTF_A 1
 
 /* Have pthread_getspecific */
-#define HAVE_PTHREAD_GETSPECIFIC 1
+/* #undef HAVE_PTHREAD_GETSPECIFIC */
 
 /* Define to 1 if you have the <pthread.h> header file. */
-#define HAVE_PTHREAD_H 1
+/* #undef HAVE_PTHREAD_H */
 
 /* Have pthread_mutex_lock */
-#define HAVE_PTHREAD_MUTEX_LOCK 1
+/* #undef HAVE_PTHREAD_MUTEX_LOCK */
 
 /* Have pthread_rwlock_init */
-#define HAVE_PTHREAD_RWLOCK_INIT 1
+/* #undef HAVE_PTHREAD_RWLOCK_INIT */
 
 /* Define to 1 if srand48/lrand48/drand48 exist in <stdlib.h> */
 #define HAVE_RAND48 1
@@ -548,6 +551,9 @@
 /* Installation directory for config files */
 /* #undef LLVM_ETCDIR */
 
+/* Has gcc/MSVC atomic intrinsics */
+#define LLVM_HAS_ATOMICS 1
+
 /* Host triple we were built on */
 /* #undef LLVM_HOSTTRIPLE */
 
@@ -563,9 +569,6 @@
 /* Installation directory for man pages */
 /* #undef LLVM_MANDIR */
 
-/* Build multithreading support into LLVM */
-/* #undef LLVM_MULTITHREADED */
-
 /* LLVM architecture name for the native architecture, if available */
 #define LLVM_NATIVE_ARCH X86
 
@@ -575,15 +578,15 @@
 /* LLVM name for the native AsmPrinter init function, if available */
 #define LLVM_NATIVE_ASMPRINTER LLVMInitializeX86AsmPrinter
 
-/* LLVM name for the native MCAsmInfo init function, if available */
-#define LLVM_NATIVE_MCASMINFO LLVMInitializeX86MCAsmInfo
-
 /* LLVM name for the native Target init function, if available */
 #define LLVM_NATIVE_TARGET LLVMInitializeX86Target
 
 /* LLVM name for the native TargetInfo init function, if available */
 #define LLVM_NATIVE_TARGETINFO LLVMInitializeX86TargetInfo
 
+/* LLVM name for the native target MC init function, if available */
+#define LLVM_NATIVE_TARGETMC LLVMInitializeX86TargetMC
+
 /* Define if this is Unixish platform */
 #define LLVM_ON_UNIX 1
 
@@ -651,13 +654,13 @@
 #define PACKAGE_NAME "llvm"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING "llvm 3.0svn"
+#define PACKAGE_STRING "llvm 3.0"
 
 /* Define to the one symbol short name of this package. */
 #define PACKAGE_TARNAME "-llvm-"
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION "3.0svn"
+#define PACKAGE_VERSION "3.0"
 
 /* Define as the return type of signal handlers (`int' or `void'). */
 #define RETSIGTYPE void
diff --git a/lib/clang/include/llvm/Config/llvm-config.h b/lib/clang/include/llvm/Config/llvm-config.h
index e76d10fa9169..05e17ca1fd90 100644
--- a/lib/clang/include/llvm/Config/llvm-config.h
+++ b/lib/clang/include/llvm/Config/llvm-config.h
@@ -33,6 +33,9 @@
 /* Installation directory for config files */
 /* #undef LLVM_ETCDIR */
 
+/* Has gcc/MSVC atomic intrinsics */
+#define LLVM_HAS_ATOMICS 1
+
 /* Host triple we were built on */
 /* #undef LLVM_HOSTTRIPLE */
 
@@ -48,26 +51,23 @@
 /* Installation directory for man pages */
 /* #undef LLVM_MANDIR */
 
-/* Build multithreading support into LLVM */
-/* #undef LLVM_MULTITHREADED */
-
 /* LLVM architecture name for the native architecture, if available */
 #define LLVM_NATIVE_ARCH X86
 
+/* LLVM name for the native AsmParser init function, if available */
+#define LLVM_NATIVE_ASMPARSER LLVMInitializeX86AsmParser
+
+/* LLVM name for the native AsmPrinter init function, if available */
+#define LLVM_NATIVE_ASMPRINTER LLVMInitializeX86AsmPrinter
+
 /* LLVM name for the native Target init function, if available */
 #define LLVM_NATIVE_TARGET LLVMInitializeX86Target
 
 /* LLVM name for the native TargetInfo init function, if available */
 #define LLVM_NATIVE_TARGETINFO LLVMInitializeX86TargetInfo
 
-/* LLVM name for the native MCAsmInfo init function, if available */
-#define LLVM_NATIVE_MCASMINFO LLVMInitializeX86MCAsmInfo
-
-/* LLVM name for the native AsmPrinter init function, if available */
-#define LLVM_NATIVE_ASMPRINTER LLVMInitializeX86AsmPrinter
-
-/* LLVM name for the native AsmPrinter init function, if available */
-#define LLVM_NATIVE_ASMPARSER LLVMInitializeX86AsmParser
+/* LLVM name for the native target MC init function, if available */
+#define LLVM_NATIVE_TARGETMC LLVMInitializeX86TargetMC
 
 /* Define if this is Unixish platform */
 #define LLVM_ON_UNIX 1
@@ -99,6 +99,9 @@
 /* Define to path to twopi program if found or 'echo twopi' otherwise */
 /* #undef LLVM_PATH_TWOPI */
 
+/* Define to path to xdot.py program if found or 'echo xdot.py' otherwise */
+/* #undef LLVM_PATH_XDOT_PY */
+
 /* Installation prefix directory */
 /* #undef LLVM_PREFIX */
 
diff --git a/lib/clang/libclanganalysis/Makefile b/lib/clang/libclanganalysis/Makefile
index d0df7798838a..f5fbc6270d3a 100644
--- a/lib/clang/libclanganalysis/Makefile
+++ b/lib/clang/libclanganalysis/Makefile
@@ -11,9 +11,11 @@ SRCS=	AnalysisContext.cpp \
 	FormatString.cpp \
 	LiveVariables.cpp \
 	PrintfFormatString.cpp \
+	ProgramPoint.cpp \
 	PseudoConstantAnalysis.cpp \
 	ReachableCode.cpp \
 	ScanfFormatString.cpp \
+	ThreadSafety.cpp \
 	UninitializedValues.cpp
 
 TGHDRS=	AttrList \
diff --git a/lib/clang/libclangarcmigrate/Makefile b/lib/clang/libclangarcmigrate/Makefile
index c1e4511503bb..957c3bade6a0 100644
--- a/lib/clang/libclangarcmigrate/Makefile
+++ b/lib/clang/libclangarcmigrate/Makefile
@@ -6,6 +6,8 @@ SRCDIR=	tools/clang/lib/ARCMigrate
 SRCS=	ARCMT.cpp \
 	ARCMTActions.cpp \
 	FileRemapper.cpp \
+	PlistReporter.cpp \
+	TransAPIUses.cpp \
 	TransARCAssign.cpp \
 	TransAutoreleasePool.cpp \
 	TransBlockObjCVariable.cpp \
diff --git a/lib/clang/libclangast/Makefile b/lib/clang/libclangast/Makefile
index 61b44fb22828..61f68811c5cb 100644
--- a/lib/clang/libclangast/Makefile
+++ b/lib/clang/libclangast/Makefile
@@ -35,6 +35,7 @@ SRCS=	APValue.cpp \
 	ParentMap.cpp \
 	RecordLayout.cpp \
 	RecordLayoutBuilder.cpp \
+	SelectorLocationsKind.cpp \
 	Stmt.cpp \
 	StmtDumper.cpp \
 	StmtIterator.cpp \
@@ -45,7 +46,9 @@ SRCS=	APValue.cpp \
 	TemplateName.cpp \
 	Type.cpp \
 	TypeLoc.cpp \
-	TypePrinter.cpp
+	TypePrinter.cpp \
+	VTTBuilder.cpp \
+	VTableBuilder.cpp
 
 TGHDRS=	AttrImpl \
 	AttrList \
diff --git a/lib/clang/libclangbasic/Makefile b/lib/clang/libclangbasic/Makefile
index 49e35339b696..dec03c585abd 100644
--- a/lib/clang/libclangbasic/Makefile
+++ b/lib/clang/libclangbasic/Makefile
@@ -10,6 +10,7 @@ SRCS=	Builtins.cpp \
 	FileManager.cpp \
 	FileSystemStatCache.cpp \
 	IdentifierTable.cpp \
+	LangOptions.cpp \
 	SourceLocation.cpp \
 	SourceManager.cpp \
 	TargetInfo.cpp \
diff --git a/lib/clang/libclangcodegen/Makefile b/lib/clang/libclangcodegen/Makefile
index d1798ca1bb15..467b2d27ff0e 100644
--- a/lib/clang/libclangcodegen/Makefile
+++ b/lib/clang/libclangcodegen/Makefile
@@ -6,6 +6,8 @@ SRCDIR=	tools/clang/lib/CodeGen
 SRCS=	BackendUtil.cpp \
 	CGBlocks.cpp \
 	CGBuiltin.cpp \
+	CGCUDANV.cpp \
+	CGCUDARuntime.cpp \
 	CGCXX.cpp \
 	CGCXXABI.cpp \
 	CGCall.cpp \
@@ -25,6 +27,7 @@ SRCS=	BackendUtil.cpp \
 	CGObjCGNU.cpp \
 	CGObjCMac.cpp \
 	CGObjCRuntime.cpp \
+	CGOpenCLRuntime.cpp \
 	CGRTTI.cpp \
 	CGRecordLayoutBuilder.cpp \
 	CGStmt.cpp \
diff --git a/lib/clang/libclangfrontend/Makefile b/lib/clang/libclangfrontend/Makefile
index 20879008edd4..d8495a290fb8 100644
--- a/lib/clang/libclangfrontend/Makefile
+++ b/lib/clang/libclangfrontend/Makefile
@@ -23,7 +23,7 @@ SRCS=	ASTConsumers.cpp \
 	PrintPreprocessedOutput.cpp \
 	TextDiagnosticBuffer.cpp \
 	TextDiagnosticPrinter.cpp \
-	VerifyDiagnosticsClient.cpp \
+	VerifyDiagnosticConsumer.cpp \
 	Warnings.cpp
 
 TGHDRS=	AttrList \
diff --git a/lib/clang/libclangparse/Makefile b/lib/clang/libclangparse/Makefile
index ee272fe31ba2..6bd0b5bf2508 100644
--- a/lib/clang/libclangparse/Makefile
+++ b/lib/clang/libclangparse/Makefile
@@ -17,7 +17,8 @@ SRCS=	ParseAST.cpp \
 	ParseTentative.cpp \
 	Parser.cpp
 
-TGHDRS=	AttrList \
+TGHDRS=	AttrLateParsed \
+	AttrList \
 	Attrs \
 	DeclNodes \
 	DiagnosticCommonKinds \
diff --git a/lib/clang/libclangsema/Makefile b/lib/clang/libclangsema/Makefile
index 3c03773a4c1d..57f5ccb8d2c2 100644
--- a/lib/clang/libclangsema/Makefile
+++ b/lib/clang/libclangsema/Makefile
@@ -10,12 +10,13 @@ SRCS=	AnalysisBasedWarnings.cpp \
 	DelayedDiagnostic.cpp \
 	IdentifierResolver.cpp \
 	JumpDiagnostics.cpp \
+	MultiInitializer.cpp \
 	Scope.cpp \
 	Sema.cpp \
 	SemaAccess.cpp \
 	SemaAttr.cpp \
-	SemaCXXCast.cpp \
 	SemaCXXScopeSpec.cpp \
+	SemaCast.cpp \
 	SemaChecking.cpp \
 	SemaCodeComplete.cpp \
 	SemaDecl.cpp \
@@ -27,6 +28,7 @@ SRCS=	AnalysisBasedWarnings.cpp \
 	SemaExpr.cpp \
 	SemaExprCXX.cpp \
 	SemaExprObjC.cpp \
+	SemaFixItUtils.cpp \
 	SemaInit.cpp \
 	SemaLookup.cpp \
 	SemaObjCProperty.cpp \
diff --git a/lib/clang/libclangserialization/Makefile b/lib/clang/libclangserialization/Makefile
index 75a00312e701..a4e0f36532a6 100644
--- a/lib/clang/libclangserialization/Makefile
+++ b/lib/clang/libclangserialization/Makefile
@@ -11,7 +11,9 @@ SRCS=	ASTCommon.cpp \
 	ASTWriterDecl.cpp \
 	ASTWriterStmt.cpp \
 	ChainedIncludesSource.cpp \
-	GeneratePCH.cpp
+	GeneratePCH.cpp \
+	Module.cpp \
+	ModuleManager.cpp
 
 TGHDRS=	AttrList \
 	AttrPCHRead \
diff --git a/lib/clang/libclangstaticanalyzercheckers/Makefile b/lib/clang/libclangstaticanalyzercheckers/Makefile
index adc78822e7fc..44f7d58ca7b1 100644
--- a/lib/clang/libclangstaticanalyzercheckers/Makefile
+++ b/lib/clang/libclangstaticanalyzercheckers/Makefile
@@ -19,7 +19,7 @@ SRCS=	AdjustedReturnValueChecker.cpp \
 	CheckSecuritySyntaxOnly.cpp \
 	CheckSizeofPointer.cpp \
 	ChrootChecker.cpp \
-	ClangSACheckerProvider.cpp \
+	ClangCheckers.cpp \
 	DeadStoresChecker.cpp \
 	DebugCheckers.cpp \
 	DereferenceChecker.cpp \
@@ -28,8 +28,10 @@ SRCS=	AdjustedReturnValueChecker.cpp \
 	IdempotentOperationChecker.cpp \
 	IteratorsChecker.cpp \
 	LLVMConventionsChecker.cpp \
+	MacOSKeychainAPIChecker.cpp \
 	MacOSXAPIChecker.cpp \
 	MallocChecker.cpp \
+	MallocOverflowSecurityChecker.cpp \
 	NSAutoreleasePoolChecker.cpp \
 	NSErrorChecker.cpp \
 	NoReturnFunctionChecker.cpp \
@@ -40,6 +42,7 @@ SRCS=	AdjustedReturnValueChecker.cpp \
 	PointerArithChecker.cpp \
 	PointerSubChecker.cpp \
 	PthreadLockChecker.cpp \
+	RetainCountChecker.cpp \
 	ReturnPointerRangeChecker.cpp \
 	ReturnUndefChecker.cpp \
 	StackAddrEscapeChecker.cpp \
diff --git a/lib/clang/libclangstaticanalyzercore/Makefile b/lib/clang/libclangstaticanalyzercore/Makefile
index 76863dc2961f..9a33751e244d 100644
--- a/lib/clang/libclangstaticanalyzercore/Makefile
+++ b/lib/clang/libclangstaticanalyzercore/Makefile
@@ -6,27 +6,29 @@ SRCDIR=	tools/clang/lib/StaticAnalyzer/Core
 SRCS=	AggExprVisitor.cpp \
 	AnalysisManager.cpp \
 	BasicConstraintManager.cpp \
-	BasicStore.cpp \
 	BasicValueFactory.cpp \
 	BlockCounter.cpp \
 	BugReporter.cpp \
 	BugReporterVisitors.cpp \
-	CFRefCount.cpp \
-	CXXExprEngine.cpp \
+	Checker.cpp \
 	CheckerContext.cpp \
 	CheckerHelpers.cpp \
 	CheckerManager.cpp \
+	CheckerRegistry.cpp \
 	CoreEngine.cpp \
 	Environment.cpp \
 	ExplodedGraph.cpp \
 	ExprEngine.cpp \
-	FlatStore.cpp \
-	GRState.cpp \
+	ExprEngineC.cpp \
+	ExprEngineCXX.cpp \
+	ExprEngineCallAndReturn.cpp \
+	ExprEngineObjC.cpp \
 	HTMLDiagnostics.cpp \
 	MemRegion.cpp \
 	ObjCMessage.cpp \
 	PathDiagnostic.cpp \
 	PlistDiagnostics.cpp \
+	ProgramState.cpp \
 	RangeConstraintManager.cpp \
 	RegionStore.cpp \
 	SValBuilder.cpp \
diff --git a/lib/clang/libllvmarmcodegen/Makefile b/lib/clang/libllvmarmcodegen/Makefile
index 0110781a9597..29579bc32ab5 100644
--- a/lib/clang/libllvmarmcodegen/Makefile
+++ b/lib/clang/libllvmarmcodegen/Makefile
@@ -3,8 +3,7 @@
 LIB=	llvmarmcodegen
 
 SRCDIR=	lib/Target/ARM
-SRCS=	ARMAsmBackend.cpp \
-	ARMAsmPrinter.cpp \
+SRCS=	ARMAsmPrinter.cpp \
 	ARMBaseInstrInfo.cpp \
 	ARMBaseRegisterInfo.cpp \
 	ARMCodeEmitter.cpp \
@@ -21,17 +20,13 @@ SRCS=	ARMAsmBackend.cpp \
 	ARMInstrInfo.cpp \
 	ARMJITInfo.cpp \
 	ARMLoadStoreOptimizer.cpp \
-	ARMMCCodeEmitter.cpp \
-	ARMMCExpr.cpp \
 	ARMMCInstLower.cpp \
-	ARMMachObjectWriter.cpp \
 	ARMRegisterInfo.cpp \
 	ARMSelectionDAGInfo.cpp \
 	ARMSubtarget.cpp \
 	ARMTargetMachine.cpp \
 	ARMTargetObjectFile.cpp \
 	MLxExpansionPass.cpp \
-	NEONMoveFix.cpp \
 	Thumb1FrameLowering.cpp \
 	Thumb1InstrInfo.cpp \
 	Thumb1RegisterInfo.cpp \
diff --git a/lib/clang/libllvmarmdesc/Makefile b/lib/clang/libllvmarmdesc/Makefile
index 0074ec250bfc..e732074ddeab 100644
--- a/lib/clang/libllvmarmdesc/Makefile
+++ b/lib/clang/libllvmarmdesc/Makefile
@@ -3,10 +3,16 @@
 LIB=	llvmarmdesc
 
 SRCDIR=	lib/Target/ARM/MCTargetDesc
-SRCS=	ARMMCAsmInfo.cpp \
+SRCS=	ARMAsmBackend.cpp \
+	ARMMachObjectWriter.cpp \
+	ARMMCAsmInfo.cpp \
+	ARMMCCodeEmitter.cpp \
+	ARMMCExpr.cpp \
 	ARMMCTargetDesc.cpp
+CFLAGS+= -I${LLVM_SRCS}/${SRCDIR}/..
 
 TGHDRS=	ARMGenInstrInfo \
+	ARMGenMCCodeEmitter \
 	ARMGenRegisterInfo \
 	ARMGenSubtargetInfo
 
diff --git a/lib/clang/libllvmarmdisassembler/Makefile b/lib/clang/libllvmarmdisassembler/Makefile
index c0648bb11b3e..a3d16f78416b 100644
--- a/lib/clang/libllvmarmdisassembler/Makefile
+++ b/lib/clang/libllvmarmdisassembler/Makefile
@@ -4,9 +4,9 @@ LIB=	llvmarmdisassembler
 
 SRCDIR=	lib/Target/ARM/Disassembler
 INCDIR=	lib/Target/ARM
-SRCS=	ARMDisassemblerCore.cpp
+SRCS=	ARMDisassembler.cpp
 
-TGHDRS=	ARMGenDecoderTables \
+TGHDRS=	ARMGenDisassemblerTables \
 	ARMGenEDInfo \
 	ARMGenInstrInfo \
 	ARMGenRegisterInfo \
diff --git a/lib/clang/libllvmcodegen/Makefile b/lib/clang/libllvmcodegen/Makefile
index 702c4760d693..0d593b662a48 100644
--- a/lib/clang/libllvmcodegen/Makefile
+++ b/lib/clang/libllvmcodegen/Makefile
@@ -17,7 +17,9 @@ SRCS=	AggressiveAntiDepBreaker.cpp \
 	ELFCodeEmitter.cpp \
 	ELFWriter.cpp \
 	EdgeBundles.cpp \
+	ExecutionDepsFix.cpp \
 	ExpandISelPseudos.cpp \
+	ExpandPostRAPseudos.cpp \
 	GCMetadata.cpp \
 	GCMetadataPrinter.cpp \
 	GCStrategy.cpp \
@@ -27,16 +29,18 @@ SRCS=	AggressiveAntiDepBreaker.cpp \
 	IntrinsicLowering.cpp \
 	LLVMTargetMachine.cpp \
 	LatencyPriorityQueue.cpp \
+	LexicalScopes.cpp \
 	LiveDebugVariables.cpp \
 	LiveInterval.cpp \
 	LiveIntervalAnalysis.cpp \
 	LiveIntervalUnion.cpp \
 	LiveStackAnalysis.cpp \
 	LiveVariables.cpp \
+	LiveRangeCalc.cpp \
 	LiveRangeEdit.cpp \
 	LocalStackSlotAllocation.cpp \
-	LowerSubregs.cpp \
 	MachineBasicBlock.cpp \
+	MachineBranchProbabilityInfo.cpp \
 	MachineCSE.cpp \
 	MachineDominators.cpp \
 	MachineFunction.cpp \
diff --git a/lib/clang/libllvmipo/Makefile b/lib/clang/libllvmipo/Makefile
index 062ba5bdd48b..42cc2fa93252 100644
--- a/lib/clang/libllvmipo/Makefile
+++ b/lib/clang/libllvmipo/Makefile
@@ -16,9 +16,9 @@ SRCS=	ArgumentPromotion.cpp \
 	Inliner.cpp \
 	Internalize.cpp \
 	LoopExtractor.cpp \
-	LowerSetJmp.cpp \
 	MergeFunctions.cpp \
 	PartialInlining.cpp \
+	PassManagerBuilder.cpp \
 	PruneEH.cpp \
 	StripDeadPrototypes.cpp \
 	StripSymbols.cpp
diff --git a/lib/clang/libllvmmc/Makefile b/lib/clang/libllvmmc/Makefile
index 435cdf41cf1c..5fef440114b9 100644
--- a/lib/clang/libllvmmc/Makefile
+++ b/lib/clang/libllvmmc/Makefile
@@ -4,11 +4,13 @@ LIB=	llvmmc
 
 SRCDIR=	lib/MC
 SRCS=	ELFObjectWriter.cpp \
+	MCAsmBackend.cpp \
 	MCAsmInfo.cpp \
 	MCAsmInfoCOFF.cpp \
 	MCAsmInfoDarwin.cpp \
 	MCAsmStreamer.cpp \
 	MCAssembler.cpp \
+	MCCodeGenInfo.cpp \
 	MCCodeEmitter.cpp \
 	MCContext.cpp \
 	MCDwarf.cpp \
@@ -18,10 +20,12 @@ SRCS=	ELFObjectWriter.cpp \
 	MCExpr.cpp \
 	MCInst.cpp \
 	MCInstPrinter.cpp \
+	MCInstrAnalysis.cpp \
 	MCLoggingStreamer.cpp \
 	MCMachOStreamer.cpp \
 	MCMachObjectTargetWriter.cpp \
 	MCNullStreamer.cpp \
+	MCObjectFileInfo.cpp \
 	MCObjectStreamer.cpp \
 	MCObjectWriter.cpp \
 	MCPureStreamer.cpp \
@@ -32,10 +36,10 @@ SRCS=	ELFObjectWriter.cpp \
 	MCStreamer.cpp \
 	MCSubtargetInfo.cpp \
 	MCSymbol.cpp \
+	MCTargetAsmLexer.cpp \
 	MCWin64EH.cpp \
 	MachObjectWriter.cpp \
 	SubtargetFeature.cpp \
-	TargetAsmBackend.cpp \
 	WinCOFFObjectWriter.cpp \
 	WinCOFFStreamer.cpp
 
diff --git a/lib/clang/libllvmmcparser/Makefile b/lib/clang/libllvmmcparser/Makefile
index 4f2f8bfbc76b..48deadfee43f 100644
--- a/lib/clang/libllvmmcparser/Makefile
+++ b/lib/clang/libllvmmcparser/Makefile
@@ -11,6 +11,6 @@ SRCS=	AsmLexer.cpp \
 	MCAsmLexer.cpp \
 	MCAsmParser.cpp \
 	MCAsmParserExtension.cpp \
-	TargetAsmParser.cpp
+	MCTargetAsmParser.cpp
 
 .include "../clang.lib.mk"
diff --git a/lib/clang/libllvmmipscodegen/Makefile b/lib/clang/libllvmmipscodegen/Makefile
index 1651831e6e77..a70c0720a6b0 100644
--- a/lib/clang/libllvmmipscodegen/Makefile
+++ b/lib/clang/libllvmmipscodegen/Makefile
@@ -4,6 +4,7 @@ LIB=	llvmmipscodegen
 
 SRCDIR=	lib/Target/Mips
 SRCS=	MipsAsmPrinter.cpp \
+	MipsCodeEmitter.cpp \
 	MipsDelaySlotFiller.cpp \
 	MipsEmitGPRestore.cpp \
 	MipsExpandPseudo.cpp \
@@ -11,6 +12,7 @@ SRCS=	MipsAsmPrinter.cpp \
 	MipsISelDAGToDAG.cpp \
 	MipsISelLowering.cpp \
 	MipsInstrInfo.cpp \
+	MipsJITInfo.cpp \
 	MipsMCInstLower.cpp \
 	MipsMCSymbolRefExpr.cpp \
 	MipsRegisterInfo.cpp \
diff --git a/lib/clang/libllvmmipsdesc/Makefile b/lib/clang/libllvmmipsdesc/Makefile
index 41472090759a..23136030f434 100644
--- a/lib/clang/libllvmmipsdesc/Makefile
+++ b/lib/clang/libllvmmipsdesc/Makefile
@@ -3,8 +3,11 @@
 LIB=	llvmmipsdesc
 
 SRCDIR=	lib/Target/Mips/MCTargetDesc
-SRCS=	MipsMCAsmInfo.cpp \
+SRCS=	MipsAsmBackend.cpp \
+	MipsMCAsmInfo.cpp \
+	MipsMCCodeEmitter.cpp \
 	MipsMCTargetDesc.cpp
+CFLAGS+= -I${LLVM_SRCS}/${SRCDIR}/..
 
 TGHDRS=	MipsGenInstrInfo \
 	MipsGenRegisterInfo \
diff --git a/lib/clang/libllvmpowerpccodegen/Makefile b/lib/clang/libllvmpowerpccodegen/Makefile
index 08663e8807d9..aec0e1eaf625 100644
--- a/lib/clang/libllvmpowerpccodegen/Makefile
+++ b/lib/clang/libllvmpowerpccodegen/Makefile
@@ -3,8 +3,7 @@
 LIB=	llvmpowerpccodegen
 
 SRCDIR=	lib/Target/PowerPC
-SRCS=	PPCAsmBackend.cpp \
-	PPCAsmPrinter.cpp \
+SRCS=	PPCAsmPrinter.cpp \
 	PPCBranchSelector.cpp \
 	PPCCodeEmitter.cpp \
 	PPCFrameLowering.cpp \
@@ -13,9 +12,7 @@ SRCS=	PPCAsmBackend.cpp \
 	PPCISelLowering.cpp \
 	PPCInstrInfo.cpp \
 	PPCJITInfo.cpp \
-	PPCMCCodeEmitter.cpp \
 	PPCMCInstLower.cpp \
-	PPCPredicates.cpp \
 	PPCRegisterInfo.cpp \
 	PPCSelectionDAGInfo.cpp \
 	PPCSubtarget.cpp \
diff --git a/lib/clang/libllvmpowerpcdesc/Makefile b/lib/clang/libllvmpowerpcdesc/Makefile
index 9207da29992a..c45abecf076a 100644
--- a/lib/clang/libllvmpowerpcdesc/Makefile
+++ b/lib/clang/libllvmpowerpcdesc/Makefile
@@ -3,10 +3,15 @@
 LIB=	llvmpowerpcdesc
 
 SRCDIR=	lib/Target/PowerPC/MCTargetDesc
-SRCS=	PPCMCAsmInfo.cpp \
-	PPCMCTargetDesc.cpp
+SRCS=	PPCAsmBackend.cpp \
+	PPCMCAsmInfo.cpp \
+	PPCMCCodeEmitter.cpp \
+	PPCMCTargetDesc.cpp \
+	PPCPredicates.cpp
+CFLAGS+= -I${LLVM_SRCS}/${SRCDIR}/..
 
 TGHDRS=	PPCGenInstrInfo \
+	PPCGenMCCodeEmitter \
 	PPCGenRegisterInfo \
 	PPCGenSubtargetInfo
 
diff --git a/lib/clang/libllvmscalaropts/Makefile b/lib/clang/libllvmscalaropts/Makefile
index 81998a4fb679..c376b685d591 100644
--- a/lib/clang/libllvmscalaropts/Makefile
+++ b/lib/clang/libllvmscalaropts/Makefile
@@ -29,7 +29,6 @@ SRCS=	ADCE.cpp \
 	SimplifyCFGPass.cpp \
 	SimplifyLibCalls.cpp \
 	Sink.cpp \
-	TailDuplication.cpp \
 	TailRecursionElimination.cpp
 
 TGHDRS=	Intrinsics
diff --git a/lib/clang/libllvmtablegen/Makefile b/lib/clang/libllvmtablegen/Makefile
new file mode 100644
index 000000000000..272aa883e7ab
--- /dev/null
+++ b/lib/clang/libllvmtablegen/Makefile
@@ -0,0 +1,14 @@
+# $FreeBSD$
+
+LIB=	llvmtablegen
+
+SRCDIR=	lib/TableGen
+SRCS=	Error.cpp \
+	Main.cpp \
+	Record.cpp \
+	TableGenBackend.cpp \
+	TGLexer.cpp \
+	TGParser.cpp
+LLVM_REQUIRES_EH=
+
+.include "../clang.lib.mk"
diff --git a/lib/clang/libllvmtarget/Makefile b/lib/clang/libllvmtarget/Makefile
index 493b7c1f6f3f..8c16f84e4dbb 100644
--- a/lib/clang/libllvmtarget/Makefile
+++ b/lib/clang/libllvmtarget/Makefile
@@ -5,8 +5,6 @@ LIB=	llvmtarget
 SRCDIR=	lib/Target
 SRCS=	Mangler.cpp \
 	Target.cpp \
-	TargetAsmInfo.cpp \
-	TargetAsmLexer.cpp \
 	TargetData.cpp \
 	TargetELFWriterInfo.cpp \
 	TargetFrameLowering.cpp \
diff --git a/lib/clang/libllvmtransformutils/Makefile b/lib/clang/libllvmtransformutils/Makefile
index 50dae17971ac..18eecda5d62c 100644
--- a/lib/clang/libllvmtransformutils/Makefile
+++ b/lib/clang/libllvmtransformutils/Makefile
@@ -25,6 +25,7 @@ SRCS=	AddrModeMatcher.cpp \
 	PromoteMemoryToRegister.cpp \
 	SSAUpdater.cpp \
 	SimplifyCFG.cpp \
+	SimplifyIndVar.cpp \
 	UnifyFunctionExitNodes.cpp \
 	ValueMapper.cpp
 
diff --git a/lib/clang/libllvmx86codegen/Makefile b/lib/clang/libllvmx86codegen/Makefile
index 69645019a763..5dc45ef58c61 100644
--- a/lib/clang/libllvmx86codegen/Makefile
+++ b/lib/clang/libllvmx86codegen/Makefile
@@ -3,9 +3,7 @@
 LIB=	llvmx86codegen
 
 SRCDIR=	lib/Target/X86
-SRCS=	SSEDomainFix.cpp \
-	X86AsmBackend.cpp \
-	X86AsmPrinter.cpp \
+SRCS=	X86AsmPrinter.cpp \
 	X86COFFMachineModuleInfo.cpp \
 	X86CodeEmitter.cpp \
 	X86ELFWriterInfo.cpp \
@@ -16,14 +14,13 @@ SRCS=	SSEDomainFix.cpp \
 	X86ISelLowering.cpp \
 	X86InstrInfo.cpp \
 	X86JITInfo.cpp \
-	X86MCCodeEmitter.cpp \
 	X86MCInstLower.cpp \
-	X86MachObjectWriter.cpp \
 	X86RegisterInfo.cpp \
 	X86SelectionDAGInfo.cpp \
 	X86Subtarget.cpp \
 	X86TargetMachine.cpp \
-	X86TargetObjectFile.cpp
+	X86TargetObjectFile.cpp \
+	X86VZeroUpper.cpp \
 
 TGHDRS=	Intrinsics \
 	X86GenCallingConv \
diff --git a/lib/clang/libllvmx86desc/Makefile b/lib/clang/libllvmx86desc/Makefile
index 44b20a5813c9..badb0f2a78b2 100644
--- a/lib/clang/libllvmx86desc/Makefile
+++ b/lib/clang/libllvmx86desc/Makefile
@@ -3,8 +3,12 @@
 LIB=	llvmx86desc
 
 SRCDIR=	lib/Target/X86/MCTargetDesc
-SRCS=	X86MCAsmInfo.cpp \
+SRCS=	X86AsmBackend.cpp \
+	X86MachObjectWriter.cpp \
+	X86MCAsmInfo.cpp \
+	X86MCCodeEmitter.cpp \
 	X86MCTargetDesc.cpp
+CFLAGS+= -I${LLVM_SRCS}/${SRCDIR}/..
 
 TGHDRS=	X86GenInstrInfo \
 	X86GenRegisterInfo \
diff --git a/lib/clang/libllvmx86disassembler/Makefile b/lib/clang/libllvmx86disassembler/Makefile
index b9055fc4b400..1d1b3b651620 100644
--- a/lib/clang/libllvmx86disassembler/Makefile
+++ b/lib/clang/libllvmx86disassembler/Makefile
@@ -8,6 +8,7 @@ SRCS=	X86Disassembler.cpp
 
 TGHDRS=	X86GenDisassemblerTables \
 	X86GenEDInfo \
+	X86GenInstrInfo \
 	X86GenRegisterInfo
 
 .include "../clang.lib.mk"
diff --git a/lib/libarchive/archive_hash.h b/lib/libarchive/archive_hash.h
index dc63d94ec1c2..43f398c5f1e5 100644
--- a/lib/libarchive/archive_hash.h
+++ b/lib/libarchive/archive_hash.h
@@ -79,6 +79,13 @@ typedef MD5_CTX archive_md5_ctx;
 #  define archive_md5_init(ctx)			MD5Init(ctx)
 #  define archive_md5_final(ctx, buf)		MD5Final(buf, ctx)
 #  define archive_md5_update(ctx, buf, n)	MD5Update(ctx, buf, n)
+#elif defined(ARCHIVE_HASH_MD5_LIBMD)
+#  include <md5.h>
+#  define ARCHIVE_HAS_MD5
+typedef MD5_CTX archive_md5_ctx;
+#  define archive_md5_init(ctx)			MD5Init(ctx)
+#  define archive_md5_final(ctx, buf)		MD5Final(buf, ctx)
+#  define archive_md5_update(ctx, buf, n)	MD5Update(ctx, buf, n)
 #elif defined(ARCHIVE_HASH_MD5_LIBSYSTEM)
 #  include <CommonCrypto/CommonDigest.h>
 #  define ARCHIVE_HAS_MD5
@@ -125,6 +132,13 @@ typedef SHA1_CTX archive_sha1_ctx;
 #  define archive_sha1_init(ctx)		SHA1Init(ctx)
 #  define archive_sha1_final(ctx, buf)		SHA1Final(buf, ctx)
 #  define archive_sha1_update(ctx, buf, n)	SHA1Update(ctx, buf, n)
+#elif defined(ARCHIVE_HASH_SHA1_LIBMD)
+#  include <sha.h>
+#  define ARCHIVE_HAS_SHA1
+typedef SHA1_CTX archive_sha1_ctx;
+#  define archive_sha1_init(ctx)		SHA1_Init(ctx)
+#  define archive_sha1_final(ctx, buf)		SHA1_Final(buf, ctx)
+#  define archive_sha1_update(ctx, buf, n)	SHA1_Update(ctx, buf, n)
 #elif defined(ARCHIVE_HASH_SHA1_LIBSYSTEM)
 #  include <CommonCrypto/CommonDigest.h>
 #  define ARCHIVE_HAS_SHA1
@@ -169,6 +183,13 @@ typedef SHA2_CTX archive_sha256_ctx;
 #  define archive_sha256_init(ctx)		SHA256Init(ctx)
 #  define archive_sha256_final(ctx, buf)	SHA256Final(buf, ctx)
 #  define archive_sha256_update(ctx, buf, n)	SHA256Update(ctx, buf, n)
+#elif defined(ARCHIVE_HASH_SHA256_LIBMD)
+#  include <sha256.h>
+#  define ARCHIVE_HAS_SHA256
+typedef SHA256_CTX archive_sha256_ctx;
+#  define archive_sha256_init(ctx)		SHA256_Init(ctx)
+#  define archive_sha256_final(ctx, buf)	SHA256_Final(buf, ctx)
+#  define archive_sha256_update(ctx, buf, n)	SHA256_Update(ctx, buf, n)
 #elif defined(ARCHIVE_HASH_SHA256_LIBSYSTEM)
 #  include <CommonCrypto/CommonDigest.h>
 #  define ARCHIVE_HAS_SHA256
@@ -257,6 +278,13 @@ typedef SHA2_CTX archive_sha512_ctx;
 #  define archive_sha512_init(ctx)		SHA512Init(ctx)
 #  define archive_sha512_final(ctx, buf)	SHA512Final(buf, ctx)
 #  define archive_sha512_update(ctx, buf, n)	SHA512Update(ctx, buf, n)
+#elif defined(ARCHIVE_HASH_SHA512_LIBMD)
+#  include <sha512.h>
+#  define ARCHIVE_HAS_SHA512
+typedef SHA512_CTX archive_sha512_ctx;
+#  define archive_sha512_init(ctx)		SHA512_Init(ctx)
+#  define archive_sha512_final(ctx, buf)	SHA512_Final(buf, ctx)
+#  define archive_sha512_update(ctx, buf, n)	SHA512_Update(ctx, buf, n)
 #elif defined(ARCHIVE_HASH_SHA512_LIBSYSTEM)
 #  include <CommonCrypto/CommonDigest.h>
 #  define ARCHIVE_HAS_SHA512
diff --git a/lib/libarchive/archive_read_disk.3 b/lib/libarchive/archive_read_disk.3
index 290d3a1430b5..638c1585b9a5 100644
--- a/lib/libarchive/archive_read_disk.3
+++ b/lib/libarchive/archive_read_disk.3
@@ -133,16 +133,16 @@ object is destroyed or when new lookup functions are registered.
 This convenience function installs a standard set of user
 and group name lookup functions.
 These functions use
-.Xr getpwid 3
+.Xr getpwuid 3
 and
-.Xr getgrid 3
+.Xr getgrgid 3
 to convert ids to names, defaulting to NULL if the names cannot
 be looked up.
 These functions also implement a simple memory cache to reduce
 the number of calls to
-.Xr getpwid 3
+.Xr getpwuid 3
 and
-.Xr getgrid 3 .
+.Xr getgrgid 3 .
 .It Fn archive_read_disk_entry_from_file
 Populates a
 .Tn struct archive_entry
@@ -281,9 +281,9 @@ library was written by
 The
 .Dq standard
 user name and group name lookup functions are not the defaults because
-.Xr getgrid 3
+.Xr getgrgid 3
 and
-.Xr getpwid 3
+.Xr getpwuid 3
 are sometimes too large for particular applications.
 The current design allows the application author to use a more
 compact implementation when appropriate.
diff --git a/lib/libarchive/config_freebsd.h b/lib/libarchive/config_freebsd.h
index d1cb4620a5ae..a39a1fba4745 100644
--- a/lib/libarchive/config_freebsd.h
+++ b/lib/libarchive/config_freebsd.h
@@ -176,8 +176,8 @@
 #define	ARCHIVE_HASH_SHA384_OPENSSL 1
 #define	ARCHIVE_HASH_SHA512_OPENSSL 1
 #else
-#define	ARCHIVE_HASH_MD5_LIBC 1
-#define	ARCHIVE_HASH_SHA1_LIBC 1
-#define	ARCHIVE_HASH_SHA256_LIBC 1
-#define	ARCHIVE_HASH_SHA512_LIBC 1
+#define	ARCHIVE_HASH_MD5_LIBMD 1
+#define	ARCHIVE_HASH_SHA1_LIBMD 1
+#define	ARCHIVE_HASH_SHA256_LIBMD 1
+#define	ARCHIVE_HASH_SHA512_LIBMD 1
 #endif
diff --git a/lib/libc/Versions.def b/lib/libc/Versions.def
index da0ca6f80632..bbd59a4e780b 100644
--- a/lib/libc/Versions.def
+++ b/lib/libc/Versions.def
@@ -19,6 +19,10 @@ FBSD_1.1 {
 FBSD_1.2 {
 } FBSD_1.1;
 
+# This version was first added to 10.0-current.
+FBSD_1.3 {
+} FBSD_1.2;
+
 # This is our private namespace.  Any global interfaces that are
 # strictly for use only by other FreeBSD applications and libraries
 # are listed here.  We use a separate namespace so we can write
@@ -26,4 +30,4 @@ FBSD_1.2 {
 #
 # Please do NOT increment the version of this namespace.
 FBSDprivate_1.0 {
-} FBSD_1.2;
+} FBSD_1.3;
diff --git a/lib/libc/amd64/Symbol.map b/lib/libc/amd64/Symbol.map
index ab8578150c26..9997cc2e0ab3 100644
--- a/lib/libc/amd64/Symbol.map
+++ b/lib/libc/amd64/Symbol.map
@@ -26,7 +26,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	rfork_thread;
 	setjmp;
 	longjmp;
diff --git a/lib/libc/amd64/gen/Makefile.inc b/lib/libc/amd64/gen/Makefile.inc
index 38fe7e1a8ce3..30fb05f89cb7 100644
--- a/lib/libc/amd64/gen/Makefile.inc
+++ b/lib/libc/amd64/gen/Makefile.inc
@@ -2,7 +2,7 @@
 # $FreeBSD$
 
 SRCS+=	_setjmp.S _set_tp.c rfork_thread.S setjmp.S sigsetjmp.S \
-	fabs.S modf.S \
+	fabs.S \
 	infinity.c ldexp.c makecontext.c signalcontext.c \
 	flt_rounds.c fpgetmask.c fpsetmask.c fpgetprec.c fpsetprec.c \
 	fpgetround.c fpsetround.c fpgetsticky.c
diff --git a/lib/libc/arm/Symbol.map b/lib/libc/arm/Symbol.map
index 6b7e6e4bc473..2cc515783074 100644
--- a/lib/libc/arm/Symbol.map
+++ b/lib/libc/arm/Symbol.map
@@ -19,7 +19,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	setjmp;
 	longjmp;
 	sigsetjmp;
diff --git a/lib/libc/arm/gen/Makefile.inc b/lib/libc/arm/gen/Makefile.inc
index bbe225727889..fb9d88559127 100644
--- a/lib/libc/arm/gen/Makefile.inc
+++ b/lib/libc/arm/gen/Makefile.inc
@@ -2,5 +2,5 @@
 # $FreeBSD$
 
 SRCS+=	_ctx_start.S _setjmp.S _set_tp.c alloca.S fabs.c \
-	infinity.c ldexp.c makecontext.c modf.c \
+	infinity.c ldexp.c makecontext.c \
 	setjmp.S signalcontext.c sigsetjmp.S divsi3.S
diff --git a/lib/libc/arm/gen/modf.c b/lib/libc/arm/gen/modf.c
deleted file mode 100644
index cb5aeac2ea46..000000000000
--- a/lib/libc/arm/gen/modf.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 1994, 1995 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- * 
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- *
- *	$NetBSD: modf.c,v 1.1 1995/02/10 17:50:25 cgd Exp $
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <errno.h>
-#include <math.h>
-#include <machine/ieee.h>
-
-/*
- * double modf(double val, double *iptr)
- * returns: f and i such that |f| < 1.0, (f + i) = val, and
- *	sign(f) == sign(i) == sign(val).
- *
- * Beware signedness when doing subtraction, and also operand size!
- */
-double
-modf(val, iptr)
-	double val, *iptr;
-{
-	union doub {
-		double v;
-		struct ieee_double s;
-	} u, v;
-	u_int64_t frac;
-
-	/*
-	 * If input is Inf or NaN, return it and leave i alone.
-	 */
-	u.v = val;
-	if (u.s.dbl_exp == DBL_EXP_INFNAN)
-		return (u.v);
-
-	/*
-	 * If input can't have a fractional part, return
-	 * (appropriately signed) zero, and make i be the input.
-	 */
-	if ((int)u.s.dbl_exp - DBL_EXP_BIAS > DBL_FRACBITS - 1) {
-		*iptr = u.v;
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		return (v.v);
-	}
-
-	/*
-	 * If |input| < 1.0, return it, and set i to the appropriately
-	 * signed zero.
-	 */
-	if (u.s.dbl_exp < DBL_EXP_BIAS) {
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		*iptr = v.v;
-		return (u.v);
-	}
-
-	/*
-	 * There can be a fractional part of the input.
-	 * If you look at the math involved for a few seconds, it's
-	 * plain to see that the integral part is the input, with the
-	 * low (DBL_FRACBITS - (exponent - DBL_EXP_BIAS)) bits zeroed,
-	 * the fractional part is the part with the rest of the
-	 * bits zeroed.  Just zeroing the high bits to get the
-	 * fractional part would yield a fraction in need of
-	 * normalization.  Therefore, we take the easy way out, and
-	 * just use subtraction to get the fractional part.
-	 */
-	v.v = u.v;
-	/* Zero the low bits of the fraction, the sleazy way. */
-	frac = ((u_int64_t)v.s.dbl_frach << 32) + v.s.dbl_fracl;
-	frac >>= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	frac <<= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	v.s.dbl_fracl = frac & 0xffffffff;
-	v.s.dbl_frach = frac >> 32;
-	*iptr = v.v;
-
-	u.v -= v.v;
-	u.s.dbl_sign = v.s.dbl_sign;
-	return (u.v);
-}
diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc
index d55bc4610587..d3ccf0ae9cdd 100644
--- a/lib/libc/gen/Makefile.inc
+++ b/lib/libc/gen/Makefile.inc
@@ -35,6 +35,8 @@ SRCS+=  __getosreldate.c __xuname.c \
 	usleep.c utime.c utxdb.c valloc.c vis.c wait.c wait3.c waitpid.c \
 	wordexp.c
 
+MISRCS+=modf.c
+
 CANCELPOINTS_SRCS=sem.c sem_new.c
 .for src in ${CANCELPOINTS_SRCS}
 SRCS+=cancelpoints_${src}
diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map
index 32cf92c3e57f..c62e016510cb 100644
--- a/lib/libc/gen/Symbol.map
+++ b/lib/libc/gen/Symbol.map
@@ -213,6 +213,7 @@ FBSD_1.0 {
 	ldexp;
 	lockf;
 	lrand48;
+	modf;
 	mrand48;
 	nftw;
 	nice;
diff --git a/lib/libc/gen/ctermid.3 b/lib/libc/gen/ctermid.3
index 3402d1d5b0f7..d737e5e0d3a1 100644
--- a/lib/libc/gen/ctermid.3
+++ b/lib/libc/gen/ctermid.3
@@ -28,7 +28,7 @@
 .\"     @(#)ctermid.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd June 4, 1993
+.Dd October 1, 2011
 .Dt CTERMID 3
 .Os
 .Sh NAME
@@ -77,7 +77,8 @@ pointer,
 .Dv NULL
 is returned.
 .Pp
-The current implementation simply returns
+If no suitable lookup of the controlling terminal name can be performed,
+this implementation returns
 .Ql /dev/tty .
 .Sh RETURN VALUES
 Upon successful completion, a
diff --git a/lib/libc/gen/ctermid.c b/lib/libc/gen/ctermid.c
index f88b1f578198..8af1cb2f3d30 100644
--- a/lib/libc/gen/ctermid.c
+++ b/lib/libc/gen/ctermid.c
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 1990, 1993
- *	The Regents of the University of California.  All rights reserved.
+ * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org>
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -10,14 +10,11 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -27,31 +24,47 @@
  * SUCH DAMAGE.
  */
 
-#if defined(LIBC_SCCS) && !defined(lint)
-static char sccsid[] = "@(#)ctermid.c	8.1 (Berkeley) 6/4/93";
-#endif /* LIBC_SCCS and not lint */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <stdio.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/sysctl.h>
+
+#include <errno.h>
 #include <paths.h>
+#include <stdio.h>
 #include <string.h>
 
+#define	LEN_PATH_DEV	(sizeof(_PATH_DEV) - 1)
+
 char *
 ctermid(char *s)
 {
-	static char def[] = _PATH_TTY;
+	static char def[sizeof(_PATH_DEV) + SPECNAMELEN];
+	struct stat sb;
+	size_t dlen;
+	int sverrno;
 
-	if (s) {
-		bcopy(def, s, sizeof(_PATH_TTY));
-		return(s);
-	}
-	return(def);
+	if (s == NULL) {
+		s = def;
+		dlen = sizeof(def) - LEN_PATH_DEV;
+	} else
+		dlen = L_ctermid - LEN_PATH_DEV;
+	strcpy(s, _PATH_TTY);
+
+	/* Attempt to perform a lookup of the actual TTY pathname. */
+	sverrno = errno;
+	if (stat(_PATH_TTY, &sb) == 0 && S_ISCHR(sb.st_mode))
+		(void)sysctlbyname("kern.devname", s + LEN_PATH_DEV,
+		    &dlen, &sb.st_rdev, sizeof(sb.st_rdev));
+	errno = sverrno;
+	return (s);
 }
 
-
 char *
 ctermid_r(char *s)
 {
-	return (s) ? ctermid(s) : NULL;
+
+	return (s != NULL ? ctermid(s) : NULL);
 }
diff --git a/lib/libc/gen/devname.c b/lib/libc/gen/devname.c
index 65a690fcdcc3..da0b923573ad 100644
--- a/lib/libc/gen/devname.c
+++ b/lib/libc/gen/devname.c
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 
 #include <stdio.h>
+#include <stdint.h>
 #include <string.h>
 #include <sys/param.h>
 #include <sys/stat.h>
@@ -60,8 +61,8 @@ devname_r(dev_t dev, mode_t type, char *buf, int len)
 	}
 
 	/* Finally just format it */
-	snprintf(buf, len, "#%c:%d:0x%x",
-	    S_ISCHR(type) ? 'C' : 'B', major(dev), minor(dev));
+	snprintf(buf, len, "#%c:%#jx",
+	    S_ISCHR(type) ? 'C' : 'B', (uintmax_t)dev);
 	return (buf);
 }
 
diff --git a/lib/libc/gen/getutxent.3 b/lib/libc/gen/getutxent.3
index 19205556a876..d25aacff7b43 100644
--- a/lib/libc/gen/getutxent.3
+++ b/lib/libc/gen/getutxent.3
@@ -243,7 +243,7 @@ or
 .Dv LOGIN_PROCESS
 and whose
 .Fa ut_line
-is equal to the the same field in
+is equal to the same field in
 .Fa line .
 .Pp
 The
diff --git a/lib/libc/gen/modf.c b/lib/libc/gen/modf.c
new file mode 100644
index 000000000000..d67b44121aaa
--- /dev/null
+++ b/lib/libc/gen/modf.c
@@ -0,0 +1,138 @@
+/* @(#)s_modf.c 5.1 93/09/24 */
+/*
+ * ====================================================
+ * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+ *
+ * Developed at SunPro, a Sun Microsystems, Inc. business.
+ * Permission to use, copy, modify, and distribute this
+ * software is freely granted, provided that this notice
+ * is preserved.
+ * ====================================================
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * modf(double x, double *iptr)
+ * return fraction part of x, and return x's integral part in *iptr.
+ * Method:
+ *	Bit twiddling.
+ *
+ * Exception:
+ *	No exception.
+ */
+
+#include <sys/types.h>
+#include <machine/endian.h>
+#include <math.h>
+
+/* Bit fiddling routines copied from msun/src/math_private.h,v 1.15 */
+
+#if BYTE_ORDER == BIG_ENDIAN
+
+typedef union
+{
+  double value;
+  struct
+  {
+    u_int32_t msw;
+    u_int32_t lsw;
+  } parts;
+} ieee_double_shape_type;
+
+#endif
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+
+typedef union
+{
+  double value;
+  struct
+  {
+    u_int32_t lsw;
+    u_int32_t msw;
+  } parts;
+} ieee_double_shape_type;
+
+#endif
+
+/* Get two 32 bit ints from a double.  */
+
+#define EXTRACT_WORDS(ix0,ix1,d)				\
+do {								\
+  ieee_double_shape_type ew_u;					\
+  ew_u.value = (d);						\
+  (ix0) = ew_u.parts.msw;					\
+  (ix1) = ew_u.parts.lsw;					\
+} while (0)
+
+/* Get the more significant 32 bit int from a double.  */
+
+#define GET_HIGH_WORD(i,d)					\
+do {								\
+  ieee_double_shape_type gh_u;					\
+  gh_u.value = (d);						\
+  (i) = gh_u.parts.msw;						\
+} while (0)
+
+/* Set a double from two 32 bit ints.  */
+
+#define INSERT_WORDS(d,ix0,ix1)					\
+do {								\
+  ieee_double_shape_type iw_u;					\
+  iw_u.parts.msw = (ix0);					\
+  iw_u.parts.lsw = (ix1);					\
+  (d) = iw_u.value;						\
+} while (0)
+
+static const double one = 1.0;
+
+double
+modf(double x, double *iptr)
+{
+	int32_t i0,i1,j0;
+	u_int32_t i;
+	EXTRACT_WORDS(i0,i1,x);
+	j0 = ((i0>>20)&0x7ff)-0x3ff;	/* exponent of x */
+	if(j0<20) {			/* integer part in high x */
+	    if(j0<0) {			/* |x|<1 */
+	        INSERT_WORDS(*iptr,i0&0x80000000,0);	/* *iptr = +-0 */
+		return x;
+	    } else {
+		i = (0x000fffff)>>j0;
+		if(((i0&i)|i1)==0) {		/* x is integral */
+		    u_int32_t high;
+		    *iptr = x;
+		    GET_HIGH_WORD(high,x);
+		    INSERT_WORDS(x,high&0x80000000,0);	/* return +-0 */
+		    return x;
+		} else {
+		    INSERT_WORDS(*iptr,i0&(~i),0);
+		    return x - *iptr;
+		}
+	    }
+	} else if (j0>51) {		/* no fraction part */
+	    u_int32_t high;
+	    if (j0 == 0x400) {		/* inf/NaN */
+		*iptr = x;
+		return 0.0 / x;
+	    }
+	    *iptr = x*one;
+	    GET_HIGH_WORD(high,x);
+	    INSERT_WORDS(x,high&0x80000000,0);	/* return +-0 */
+	    return x;
+	} else {			/* fraction part in low x */
+	    i = ((u_int32_t)(0xffffffff))>>(j0-20);
+	    if((i1&i)==0) { 		/* x is integral */
+	        u_int32_t high;
+		*iptr = x;
+		GET_HIGH_WORD(high,x);
+		INSERT_WORDS(x,high&0x80000000,0);	/* return +-0 */
+		return x;
+	    } else {
+	        INSERT_WORDS(*iptr,i0,i1&(~i));
+		return x - *iptr;
+	    }
+	}
+}
diff --git a/lib/libc/i386/Symbol.map b/lib/libc/i386/Symbol.map
index 9fd9dc2051d4..8a0180228a3c 100644
--- a/lib/libc/i386/Symbol.map
+++ b/lib/libc/i386/Symbol.map
@@ -20,7 +20,6 @@ FBSD_1.0 {
 	__nan;
 	__infinity;
 	makecontext;
-	modf;
 	rfork_thread;
 	setjmp;
 	longjmp;
diff --git a/lib/libc/i386/gen/Makefile.inc b/lib/libc/i386/gen/Makefile.inc
index 89319c85fd7d..45e69cad1d0f 100644
--- a/lib/libc/i386/gen/Makefile.inc
+++ b/lib/libc/i386/gen/Makefile.inc
@@ -2,5 +2,5 @@
 # $FreeBSD$
 
 SRCS+=	_ctx_start.S _setjmp.S _set_tp.c fabs.S \
-	flt_rounds.c infinity.c ldexp.c makecontext.c modf.S \
+	flt_rounds.c infinity.c ldexp.c makecontext.c \
 	rfork_thread.S setjmp.S signalcontext.c sigsetjmp.S
diff --git a/lib/libc/i386/gen/modf.S b/lib/libc/i386/gen/modf.S
deleted file mode 100644
index 33c211fabbb8..000000000000
--- a/lib/libc/i386/gen/modf.S
+++ /dev/null
@@ -1,87 +0,0 @@
-/*-
- * Copyright (c) 1990 The Regents of the University of California.
- * All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * Sean Eric Fagan.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#if defined(LIBC_SCCS) && !defined(lint)
-	.asciz "@(#)modf.s	5.5 (Berkeley) 3/18/91"
-#endif /* LIBC_SCCS and not lint */
-#include <machine/asm.h>
-__FBSDID("$FreeBSD$");
-
-/*
- * modf(value, iptr): return fractional part of value, and stores the
- * integral part into iptr (a pointer to double).
- *
- * Written by Sean Eric Fagan (sef@kithrup.COM)
- * Sun Mar 11 20:27:30 PST 1990
- */
-
-/* With CHOP mode on, frndint behaves as TRUNC does.  Useful. */
-
-ENTRY(modf)
-	pushl %ebp
-	movl %esp,%ebp
-	/* Check for Inf/NaN */
-	movl 12(%ebp),%eax
-	andl $0x7fffffff,%eax
-	cmpl $0x7ff00000,%eax
-	jae 1f
-	/* Finite value */
-	subl $16,%esp
-	fnstcw -12(%ebp)
-	movw -12(%ebp),%dx
-	orw $3072,%dx
-	movw %dx,-16(%ebp)
-	fldcw -16(%ebp)
-	fldl 8(%ebp)
-	frndint
-	fstpl -8(%ebp)
-	fldcw -12(%ebp)
-	movl 16(%ebp),%eax
-	movl -8(%ebp),%edx
-	movl -4(%ebp),%ecx
-	movl %edx,(%eax)
-	movl %ecx,4(%eax)
-	fldl 8(%ebp)
-	fsubl -8(%ebp)
-	leave
-	ret
-	/* Inf/NaN handling */
-1:	fldl 8(%ebp)
-	movl 16(%ebp),%edx
-	fstl (%edx)
-	fldz
-	fdivp	/* return +/- 0 for +/- Inf, NaN for NaN */
-	leave
-	ret
-END(modf)
-
-	.section .note.GNU-stack,"",%progbits
diff --git a/lib/libc/ia64/Symbol.map b/lib/libc/ia64/Symbol.map
index a730922a8f35..7a31a512f5ca 100644
--- a/lib/libc/ia64/Symbol.map
+++ b/lib/libc/ia64/Symbol.map
@@ -23,7 +23,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	setjmp;
 	longjmp;
 	sigsetjmp;
diff --git a/lib/libc/ia64/gen/Makefile.inc b/lib/libc/ia64/gen/Makefile.inc
index 5358ea8c8e37..685ab86c9f46 100644
--- a/lib/libc/ia64/gen/Makefile.inc
+++ b/lib/libc/ia64/gen/Makefile.inc
@@ -3,7 +3,7 @@
 SRCS+=	__divdf3.S __divdi3.S __divsf3.S __divsi3.S __moddi3.S __modsi3.S \
 	__udivdi3.S __udivsi3.S __umoddi3.S __umodsi3.S _mcount.S _set_tp.c \
 	_setjmp.S fabs.S flt_rounds.c fpgetmask.c fpgetround.c fpsetmask.c \
-	fpsetround.c infinity.c ldexp.c makecontext.c modf.c setjmp.S \
+	fpsetround.c infinity.c ldexp.c makecontext.c setjmp.S \
 	signalcontext.c sigsetjmp.S
 
 # The following may go away if function _Unwind_FindTableEntry()
diff --git a/lib/libc/ia64/gen/modf.c b/lib/libc/ia64/gen/modf.c
deleted file mode 100644
index 13e61827b178..000000000000
--- a/lib/libc/ia64/gen/modf.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/*	$NetBSD: modf.c,v 1.1 1995/02/10 17:50:25 cgd Exp $	*/
-
-/*
- * Copyright (c) 1994, 1995 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- * 
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <machine/ieee.h>
-#include <math.h>
-
-/*
- * double modf(double val, double *iptr)
- * returns: f and i such that |f| < 1.0, (f + i) = val, and
- *	sign(f) == sign(i) == sign(val).
- *
- * Beware signedness when doing subtraction, and also operand size!
- */
-double
-modf(val, iptr)
-	double val, *iptr;
-{
-	union doub {
-		double v;
-		struct ieee_double s;
-	} u, v;
-	u_int64_t frac;
-
-	/*
-	 * If input is Inf or NaN, return it and leave i alone.
-	 */
-	u.v = val;
-	if (u.s.dbl_exp == DBL_EXP_INFNAN)
-		return (u.v);
-
-	/*
-	 * If input can't have a fractional part, return
-	 * (appropriately signed) zero, and make i be the input.
-	 */
-	if ((int)u.s.dbl_exp - DBL_EXP_BIAS > DBL_FRACBITS - 1) {
-		*iptr = u.v;
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		return (v.v);
-	}
-
-	/*
-	 * If |input| < 1.0, return it, and set i to the appropriately
-	 * signed zero.
-	 */
-	if (u.s.dbl_exp < DBL_EXP_BIAS) {
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		*iptr = v.v;
-		return (u.v);
-	}
-
-	/*
-	 * There can be a fractional part of the input.
-	 * If you look at the math involved for a few seconds, it's
-	 * plain to see that the integral part is the input, with the
-	 * low (DBL_FRACBITS - (exponent - DBL_EXP_BIAS)) bits zeroed,
-	 * the fractional part is the part with the rest of the
-	 * bits zeroed.  Just zeroing the high bits to get the
-	 * fractional part would yield a fraction in need of
-	 * normalization.  Therefore, we take the easy way out, and
-	 * just use subtraction to get the fractional part.
-	 */
-	v.v = u.v;
-	/* Zero the low bits of the fraction, the sleazy way. */
-	frac = ((u_int64_t)v.s.dbl_frach << 32) + v.s.dbl_fracl;
-	frac >>= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	frac <<= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	v.s.dbl_fracl = frac & 0xffffffff;
-	v.s.dbl_frach = frac >> 32;
-	*iptr = v.v;
-
-	u.v -= v.v;
-	u.s.dbl_sign = v.s.dbl_sign;
-	return (u.v);
-}
diff --git a/lib/libc/iconv/citrus_none.c b/lib/libc/iconv/citrus_none.c
index 1f77ad06d40d..9ec4bd360cb8 100644
--- a/lib/libc/iconv/citrus_none.c
+++ b/lib/libc/iconv/citrus_none.c
@@ -190,7 +190,6 @@ _citrus_NONE_stdenc_wctomb(struct _citrus_stdenc * __restrict ce __unused,
     void * __restrict pspriv __unused, size_t * __restrict nresult,
     struct iconv_hooks *hooks __unused)
 {
-	int ret;
 
 	if ((wc & ~0xFFU) != 0) {
 		*nresult = (size_t)-1;
@@ -198,7 +197,7 @@ _citrus_NONE_stdenc_wctomb(struct _citrus_stdenc * __restrict ce __unused,
 	}
 	if (n == 0) {
 		*nresult = (size_t)-1;
-		ret = E2BIG;
+		return (E2BIG);
 	}
 
 	*nresult = 1;
diff --git a/lib/libc/locale/isspace.3 b/lib/libc/locale/isspace.3
index f1e08d6cdfee..eab6cfcd5633 100644
--- a/lib/libc/locale/isspace.3
+++ b/lib/libc/locale/isspace.3
@@ -47,16 +47,16 @@
 .Sh DESCRIPTION
 The
 .Fn isspace
-function tests for the white-space characters.
+function tests for white-space characters.
 For any locale, this includes the following standard characters:
 .Pp
 .Bl -column \&`\et''___ \&``\et''___ \&``\et''___ \&``\et''___ \&``\et''___ \&``\et''___
 .It "\&``\et''\t``\en''\t``\ev''\t``\ef''\t``\er''\t`` ''"
 .El
 .Pp
-In the "C" locale
+In the "C" locale,
 .Fn isspace
-successful test is limited to this characters only.
+returns non-zero for these characters only.
 The value of the argument must be representable as an
 .Vt "unsigned char"
 or the value of
diff --git a/lib/libc/mips/Symbol.map b/lib/libc/mips/Symbol.map
index b47855178755..d7fbd0adc5d0 100644
--- a/lib/libc/mips/Symbol.map
+++ b/lib/libc/mips/Symbol.map
@@ -1,5 +1,5 @@
-/* 
- * $FreeBSD$ 
+/*
+ * $FreeBSD$
  */
 
 /*
@@ -18,7 +18,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	setjmp;
 	longjmp;
 	sigsetjmp;
diff --git a/lib/libc/mips/gen/Makefile.inc b/lib/libc/mips/gen/Makefile.inc
index 1fafb012bdc3..8ee69d051ddb 100644
--- a/lib/libc/mips/gen/Makefile.inc
+++ b/lib/libc/mips/gen/Makefile.inc
@@ -1,7 +1,7 @@
 #	$NetBSD: Makefile.inc,v 1.27 2005/10/07 17:16:40 tsutsui Exp $
 # $FreeBSD$
 
-SRCS+=	infinity.c fabs.c ldexp.c modf.c
+SRCS+=	infinity.c fabs.c ldexp.c
 
 # SRCS+=	flt_rounds.c fpgetmask.c fpgetround.c fpgetsticky.c fpsetmask.c \
 #	fpsetround.c fpsetsticky.c
diff --git a/lib/libc/mips/gen/modf.S b/lib/libc/mips/gen/modf.S
deleted file mode 100644
index 5eee3f1fb576..000000000000
--- a/lib/libc/mips/gen/modf.S
+++ /dev/null
@@ -1,82 +0,0 @@
-/*	$NetBSD: modf.S,v 1.10 2003/08/07 16:42:15 agc Exp $	*/
-
-/*-
- * Copyright (c) 1991, 1993, 1995
- *	The Regents of the University of California.  All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * Ralph Campbell.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <machine/asm.h>
-__FBSDID("$FreeBSD$");
-
-#if defined(LIBC_SCCS) && !defined(lint)
-	ASMSTR("from: @(#)modf.s	8.1 (Berkeley) 6/4/93")
-	ASMSTR("$NetBSD: modf.S,v 1.10 2003/08/07 16:42:15 agc Exp $")
-#endif /* LIBC_SCCS and not lint */
-
-#ifdef __ABICALLS__
-	.abicalls
-#endif
-
-/*
- * double modf(val, iptr)
- *	double val, *iptr;
- * returns: xxx and n (in *iptr) where val == n.xxx
- */
-LEAF(modf)
-#ifdef __ABICALLS__
-	.set	noreorder
-	.cpload	t9
-	.set	reorder
-#endif
-	cfc1	t0, $31			# get the control register
-	li.d	$f2, 4503599627370496e0 # f2 <- 2^52
-
-	or	t1, t0, 0x3		# set rounding mode to round to zero
-	xor	t1, t1, 0x2		#  (i.e., 01)
-	ctc1	t1, $31
-
-	mov.d	$f0, $f12		# f0 <- f12
-	abs.d	$f4, $f12		# f4 <- |f12|
-	c.olt.d $f4, $f2		# f4 ? < f2 
-	bc1f	1f			# leave f0 alone if Nan, infinity 
-					# or >=2^52
-	c.eq.d	$f12,$f4		# was f12 positive ?
-	add.d	$f4,$f2,$f4		# round off to integer
-	bc1f	2f			# No -> will have to negate result
-	sub.d	$f0,$f4,$f2		# Remove fudge factor
-	j	1f			# integer fraction got
-2:	
-	sub.d	$f0,$f2,$f4		# Remove fudge factor and negate
-1:	
-	ctc1	t0, $31			# restore old rounding mode
-	s.d	$f0, 0(a2)		# save the integer part
-	sub.d	$f0, $f12, $f0		# subtract val - integer part
-	j	ra
-END(modf)
diff --git a/lib/libc/mips/gen/modf.c b/lib/libc/mips/gen/modf.c
deleted file mode 100644
index cb5aeac2ea46..000000000000
--- a/lib/libc/mips/gen/modf.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 1994, 1995 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- * 
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- *
- *	$NetBSD: modf.c,v 1.1 1995/02/10 17:50:25 cgd Exp $
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <errno.h>
-#include <math.h>
-#include <machine/ieee.h>
-
-/*
- * double modf(double val, double *iptr)
- * returns: f and i such that |f| < 1.0, (f + i) = val, and
- *	sign(f) == sign(i) == sign(val).
- *
- * Beware signedness when doing subtraction, and also operand size!
- */
-double
-modf(val, iptr)
-	double val, *iptr;
-{
-	union doub {
-		double v;
-		struct ieee_double s;
-	} u, v;
-	u_int64_t frac;
-
-	/*
-	 * If input is Inf or NaN, return it and leave i alone.
-	 */
-	u.v = val;
-	if (u.s.dbl_exp == DBL_EXP_INFNAN)
-		return (u.v);
-
-	/*
-	 * If input can't have a fractional part, return
-	 * (appropriately signed) zero, and make i be the input.
-	 */
-	if ((int)u.s.dbl_exp - DBL_EXP_BIAS > DBL_FRACBITS - 1) {
-		*iptr = u.v;
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		return (v.v);
-	}
-
-	/*
-	 * If |input| < 1.0, return it, and set i to the appropriately
-	 * signed zero.
-	 */
-	if (u.s.dbl_exp < DBL_EXP_BIAS) {
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		*iptr = v.v;
-		return (u.v);
-	}
-
-	/*
-	 * There can be a fractional part of the input.
-	 * If you look at the math involved for a few seconds, it's
-	 * plain to see that the integral part is the input, with the
-	 * low (DBL_FRACBITS - (exponent - DBL_EXP_BIAS)) bits zeroed,
-	 * the fractional part is the part with the rest of the
-	 * bits zeroed.  Just zeroing the high bits to get the
-	 * fractional part would yield a fraction in need of
-	 * normalization.  Therefore, we take the easy way out, and
-	 * just use subtraction to get the fractional part.
-	 */
-	v.v = u.v;
-	/* Zero the low bits of the fraction, the sleazy way. */
-	frac = ((u_int64_t)v.s.dbl_frach << 32) + v.s.dbl_fracl;
-	frac >>= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	frac <<= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	v.s.dbl_fracl = frac & 0xffffffff;
-	v.s.dbl_frach = frac >> 32;
-	*iptr = v.v;
-
-	u.v -= v.v;
-	u.s.dbl_sign = v.s.dbl_sign;
-	return (u.v);
-}
diff --git a/lib/libc/net/Symbol.map b/lib/libc/net/Symbol.map
index de44cc576149..39ab2c7640eb 100644
--- a/lib/libc/net/Symbol.map
+++ b/lib/libc/net/Symbol.map
@@ -155,14 +155,14 @@ FBSDprivate_1.0 {
 	__ivaliduser_sa;
 	__check_rhosts_file;
 	__rcmd_errstr;
- 	__nss_compat_getgrnam_r;
- 	__nss_compat_getgrgid_r;
- 	__nss_compat_getgrent_r;
- 	__nss_compat_setgrent;
- 	__nss_compat_endgrent;
- 	__nss_compat_getpwnam_r;
- 	__nss_compat_getpwuid_r;
- 	__nss_compat_getpwent_r;
- 	__nss_compat_setpwent;
- 	__nss_compat_endpwent;
+	__nss_compat_getgrnam_r;
+	__nss_compat_getgrgid_r;
+	__nss_compat_getgrent_r;
+	__nss_compat_setgrent;
+	__nss_compat_endgrent;
+	__nss_compat_getpwnam_r;
+	__nss_compat_getpwuid_r;
+	__nss_compat_getpwent_r;
+	__nss_compat_setpwent;
+	__nss_compat_endpwent;
 };
diff --git a/lib/libc/powerpc/Symbol.map b/lib/libc/powerpc/Symbol.map
index 5b2402459d77..14b48114a497 100644
--- a/lib/libc/powerpc/Symbol.map
+++ b/lib/libc/powerpc/Symbol.map
@@ -24,7 +24,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	setjmp;
 	longjmp;
 	sigsetjmp;
diff --git a/lib/libc/powerpc/gen/Makefile.inc b/lib/libc/powerpc/gen/Makefile.inc
index 4b381c3d9382..3bfabdadd585 100644
--- a/lib/libc/powerpc/gen/Makefile.inc
+++ b/lib/libc/powerpc/gen/Makefile.inc
@@ -2,7 +2,7 @@
 
 SRCS += _ctx_start.S fabs.S flt_rounds.c fpgetmask.c fpgetround.c \
 	fpgetsticky.c fpsetmask.c fpsetround.c \
-	infinity.c ldexp.c makecontext.c modf.c _setjmp.S \
+	infinity.c ldexp.c makecontext.c _setjmp.S \
 	setjmp.S sigsetjmp.S signalcontext.c syncicache.c \
 	_set_tp.c
 
diff --git a/lib/libc/powerpc/gen/modf.c b/lib/libc/powerpc/gen/modf.c
deleted file mode 100644
index 612c5067b38e..000000000000
--- a/lib/libc/powerpc/gen/modf.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 1994, 1995 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- * 
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- *
- *	$NetBSD: modf.c,v 1.1 1995/02/10 17:50:25 cgd Exp $
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <machine/ieee.h>
-#include <errno.h>
-#include <math.h>
-
-/*
- * double modf(double val, double *iptr)
- * returns: f and i such that |f| < 1.0, (f + i) = val, and
- *	sign(f) == sign(i) == sign(val).
- *
- * Beware signedness when doing subtraction, and also operand size!
- */
-double
-modf(val, iptr)
-	double val, *iptr;
-{
-	union doub {
-		double v;
-		struct ieee_double s;
-	} u, v;
-	u_int64_t frac;
-
-	/*
-	 * If input is Inf or NaN, return it and leave i alone.
-	 */
-	u.v = val;
-	if (u.s.dbl_exp == DBL_EXP_INFNAN)
-		return (u.v);
-
-	/*
-	 * If input can't have a fractional part, return
-	 * (appropriately signed) zero, and make i be the input.
-	 */
-	if ((int)u.s.dbl_exp - DBL_EXP_BIAS > DBL_FRACBITS - 1) {
-		*iptr = u.v;
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		return (v.v);
-	}
-
-	/*
-	 * If |input| < 1.0, return it, and set i to the appropriately
-	 * signed zero.
-	 */
-	if (u.s.dbl_exp < DBL_EXP_BIAS) {
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		*iptr = v.v;
-		return (u.v);
-	}
-
-	/*
-	 * There can be a fractional part of the input.
-	 * If you look at the math involved for a few seconds, it's
-	 * plain to see that the integral part is the input, with the
-	 * low (DBL_FRACBITS - (exponent - DBL_EXP_BIAS)) bits zeroed,
-	 * the fractional part is the part with the rest of the
-	 * bits zeroed.  Just zeroing the high bits to get the
-	 * fractional part would yield a fraction in need of
-	 * normalization.  Therefore, we take the easy way out, and
-	 * just use subtraction to get the fractional part.
-	 */
-	v.v = u.v;
-	/* Zero the low bits of the fraction, the sleazy way. */
-	frac = ((u_int64_t)v.s.dbl_frach << 32) + v.s.dbl_fracl;
-	frac >>= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	frac <<= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	v.s.dbl_fracl = frac & 0xffffffff;
-	v.s.dbl_frach = frac >> 32;
-	*iptr = v.v;
-
-	u.v -= v.v;
-	u.s.dbl_sign = v.s.dbl_sign;
-	return (u.v);
-}
diff --git a/lib/libc/powerpc64/Symbol.map b/lib/libc/powerpc64/Symbol.map
index 0314d0d38651..018a19382ecd 100644
--- a/lib/libc/powerpc64/Symbol.map
+++ b/lib/libc/powerpc64/Symbol.map
@@ -24,7 +24,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	setjmp;
 	longjmp;
 	sigsetjmp;
diff --git a/lib/libc/powerpc64/gen/Makefile.inc b/lib/libc/powerpc64/gen/Makefile.inc
index 4b381c3d9382..3bfabdadd585 100644
--- a/lib/libc/powerpc64/gen/Makefile.inc
+++ b/lib/libc/powerpc64/gen/Makefile.inc
@@ -2,7 +2,7 @@
 
 SRCS += _ctx_start.S fabs.S flt_rounds.c fpgetmask.c fpgetround.c \
 	fpgetsticky.c fpsetmask.c fpsetround.c \
-	infinity.c ldexp.c makecontext.c modf.c _setjmp.S \
+	infinity.c ldexp.c makecontext.c _setjmp.S \
 	setjmp.S sigsetjmp.S signalcontext.c syncicache.c \
 	_set_tp.c
 
diff --git a/lib/libc/powerpc64/gen/modf.c b/lib/libc/powerpc64/gen/modf.c
deleted file mode 100644
index 612c5067b38e..000000000000
--- a/lib/libc/powerpc64/gen/modf.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 1994, 1995 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- * 
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- *
- *	$NetBSD: modf.c,v 1.1 1995/02/10 17:50:25 cgd Exp $
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <machine/ieee.h>
-#include <errno.h>
-#include <math.h>
-
-/*
- * double modf(double val, double *iptr)
- * returns: f and i such that |f| < 1.0, (f + i) = val, and
- *	sign(f) == sign(i) == sign(val).
- *
- * Beware signedness when doing subtraction, and also operand size!
- */
-double
-modf(val, iptr)
-	double val, *iptr;
-{
-	union doub {
-		double v;
-		struct ieee_double s;
-	} u, v;
-	u_int64_t frac;
-
-	/*
-	 * If input is Inf or NaN, return it and leave i alone.
-	 */
-	u.v = val;
-	if (u.s.dbl_exp == DBL_EXP_INFNAN)
-		return (u.v);
-
-	/*
-	 * If input can't have a fractional part, return
-	 * (appropriately signed) zero, and make i be the input.
-	 */
-	if ((int)u.s.dbl_exp - DBL_EXP_BIAS > DBL_FRACBITS - 1) {
-		*iptr = u.v;
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		return (v.v);
-	}
-
-	/*
-	 * If |input| < 1.0, return it, and set i to the appropriately
-	 * signed zero.
-	 */
-	if (u.s.dbl_exp < DBL_EXP_BIAS) {
-		v.v = 0.0;
-		v.s.dbl_sign = u.s.dbl_sign;
-		*iptr = v.v;
-		return (u.v);
-	}
-
-	/*
-	 * There can be a fractional part of the input.
-	 * If you look at the math involved for a few seconds, it's
-	 * plain to see that the integral part is the input, with the
-	 * low (DBL_FRACBITS - (exponent - DBL_EXP_BIAS)) bits zeroed,
-	 * the fractional part is the part with the rest of the
-	 * bits zeroed.  Just zeroing the high bits to get the
-	 * fractional part would yield a fraction in need of
-	 * normalization.  Therefore, we take the easy way out, and
-	 * just use subtraction to get the fractional part.
-	 */
-	v.v = u.v;
-	/* Zero the low bits of the fraction, the sleazy way. */
-	frac = ((u_int64_t)v.s.dbl_frach << 32) + v.s.dbl_fracl;
-	frac >>= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	frac <<= DBL_FRACBITS - (u.s.dbl_exp - DBL_EXP_BIAS);
-	v.s.dbl_fracl = frac & 0xffffffff;
-	v.s.dbl_frach = frac >> 32;
-	*iptr = v.v;
-
-	u.v -= v.v;
-	u.s.dbl_sign = v.s.dbl_sign;
-	return (u.v);
-}
diff --git a/lib/libc/sparc64/Symbol.map b/lib/libc/sparc64/Symbol.map
index 73d0ed51c4ec..4f90486daf4a 100644
--- a/lib/libc/sparc64/Symbol.map
+++ b/lib/libc/sparc64/Symbol.map
@@ -6,7 +6,7 @@
  * This only needs to contain symbols that are not listed in
  * symbol maps from other parts of libc (i.e., not found in
  * stdlib/Symbol.map, string/Symbol.map, sys/Symbol.map, ...).
- */ 
+ */
 FBSD_1.0 {
 	/* PSEUDO syscalls */
 	_exit;
@@ -24,7 +24,6 @@ FBSD_1.0 {
 	__infinity;
 	__nan;
 	makecontext;
-	modf;
 	setjmp;
 	longjmp;
 	sigsetjmp;
diff --git a/lib/libc/sparc64/gen/Makefile.inc b/lib/libc/sparc64/gen/Makefile.inc
index 979c1822ad6c..f82f22588a74 100644
--- a/lib/libc/sparc64/gen/Makefile.inc
+++ b/lib/libc/sparc64/gen/Makefile.inc
@@ -2,5 +2,5 @@
 
 SRCS+=	_ctx_start.S _setjmp.S fabs.S fixunsdfsi.S flt_rounds.c fpgetmask.c \
 	fpgetround.c fpgetsticky.c fpsetmask.c fpsetround.c \
-	infinity.c ldexp.c makecontext.c modf.S \
+	infinity.c ldexp.c makecontext.c \
 	signalcontext.c setjmp.S sigsetjmp.S _set_tp.c
diff --git a/lib/libc/sparc64/gen/modf.S b/lib/libc/sparc64/gen/modf.S
deleted file mode 100644
index cec2f94b303f..000000000000
--- a/lib/libc/sparc64/gen/modf.S
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Copyright (c) 1992, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * This software was developed by the Computer Systems Engineering group
- * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
- * contributed to Berkeley.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * from: Header: modf.s,v 1.3 92/06/20 00:00:54 torek Exp
- */
-
-#if defined(LIBC_SCCS) && !defined(lint)
-	.asciz "@(#)modf.s	8.1 (Berkeley) 6/4/93"
-#if 0
-	.asciz "$NetBSD: modf.S,v 1.2 2000/07/23 07:12:22 eeh Exp $"
-#endif
-#endif /* LIBC_SCCS and not lint */
-#include <machine/asm.h>
-__FBSDID("$FreeBSD$");
-
-#include "assym.s"
-
-/*
- * double modf(double val, double *iptr)
- *
- * Returns the fractional part of `val', storing the integer part of
- * `val' in *iptr.  Both *iptr and the return value have the same sign
- * as `val'.
- *
- * Method:
- *
- * We use the fpu's normalization hardware to compute the integer portion
- * of the double precision argument.  Sun IEEE double precision numbers
- * have 52 bits of mantissa, 11 bits of exponent, and one bit of sign,
- * with the sign occupying bit 31 of word 0, and the exponent bits 30:20
- * of word 0.  Thus, values >= 2^52 are by definition integers.
- *
- * If we take a value that is in the range [+0..2^52) and add 2^52, all
- * of the fractional bits fall out and all of the integer bits are summed
- * with 2^52.  If we then subtract 2^52, we get those integer bits back.
- * This must be done with rounding set to `towards 0' or `towards -inf'.
- * `Toward -inf' fails when the value is 0 (we get -0 back)....
- *
- * Note that this method will work anywhere, but is machine dependent in
- * various aspects.
- *
- * Stack usage:
- *	4@[%fp + SPOFF - 4]	saved %fsr
- *	4@[%fp + SPOFF - 8]	new %fsr with rounding set to `towards 0'
- *	8@[%fp + SPOFF - 16]	space for moving between %i and %f registers
- * Register usage:
- *	%f0:f1		double val;
- *	%l0		scratch
- *	%l1		sign bit (0x80000000)
- *	%i1		double *iptr;
- *	%f2:f3		`magic number' 2^52, in fpu registers
- *	%f4:f5		double v, in fpu registers
- *	%f6:f7		double temp.
- */
-
-	.align	8
-.Lmagic:
-	.word	0x43300000	! sign = 0, exponent = 52 + 1023, mantissa = 0
-	.word	0		! (i.e., .double 0r4503599627370496e+00)
-
-.L0:
-	.word	0		! 0.0
-	.word	0
-
-ENTRY(modf)
-	save	%sp, -CCFSZ - 16, %sp
-	PIC_PROLOGUE(%l6, %l7)
-
-	/*
-	 * First, compute v = abs(val)
-	 */
-	fabsd	%f0, %f4		! %f4:f5 = v
-	fcmped	%fcc1, %f0, %f4		! %fcc1 = (val == abs(val))
-	SET(.Lmagic, %l7, %l0)
-	ldd	[%l0], %f2
-
-	/*
-	 * Is %f4:f5 >= %f2:f3 ?  If so, it is all integer bits.
-	 * It is probably less, though.
-	 */
-	fcmped	%f4, %f2
-	fbuge	.Lbig			! if >= (or unordered), go out
-	 nop
-
-	/*
-	 * v < 2^52, so add 2^52, then subtract 2^52, but do it all
-	 * with rounding set towards zero.  We leave any enabled
-	 * traps enabled, but change the rounding mode.  This might
-	 * not be so good.  Oh well....
-	 */
-	st	%fsr, [%fp + SPOFF - 4]	! %l5 = current FSR mode
-	set	FSR_RD_MASK, %l3	! %l3 = rounding direction mask
-	ld	[%fp + SPOFF - 4], %l5
-	set	FSR_RD_RD_Z, %l4
-	andn	%l5, %l3, %l6
-	or	%l6, %l4, %l6		! round towards zero, please
-	and	%l5, %l3, %l5		! save original rounding mode
-	st	%l6, [%fp + SPOFF - 8]
-	ld	[%fp + SPOFF - 8], %fsr
-
-	faddd	%f4, %f2, %f4		! %f4:f5 += 2^52
-	fsubd	%f4, %f2, %f4		! %f4:f5 -= 2^52
-
-	/*
-	 * Restore %fsr, but leave exceptions accrued.
-	 */
-	st	%fsr, [%fp + SPOFF - 4]
-	ld	[%fp + SPOFF - 4], %l6
-	andn	%l6, %l3, %l6		! %l6 = %fsr & ~FSR_RD_MASK;
-	or	%l5, %l6, %l5		! %l5 |= %l6;
-	st	%l5, [%fp + SPOFF - 4]
-	ld	[%fp + SPOFF - 4], %fsr	! restore %fsr, leaving accrued stuff
-
-	/*
-	 * Now insert the original sign in %f4:f5.
-	 * %fcc1 should still have the results of (val == abs(val))
-	 * from above, so we use a conditional move on %fcc1 to: 
-	 *
-	 *	%f4 = (val == abs(val)) ? %f4 : -%f4
-	 *
-	 */
-	fnegd	%f4, %f6
-	fmovdnz	%fcc1, %f6, %f4
-1:
-
-	/*
-	 * The value in %f4:f5 is now the integer portion of the original
-	 * argument.  We need to store this in *ival (%i1), subtract it
-	 * from the original value argument (%d0), and return the result.
-	 */
-	std	%f4, [%i1]		! *ival = %f4:f5;
-	fsubd	%f0, %f4, %f0		! %f0:f1 -= %f4:f5;
-	ret
-	restore
-
-.Lbig:
-	/*
-	 * We get here if the original comparison of %f4:f5 (v) to
-	 * %f2:f3 (2^52) came out `greater or unordered'.  In this
-	 * case the integer part is the original value, and the
-	 * fractional part is 0.
-	 */
-	SET(.L0, %l7, %l0)
-	std	%f0, [%i1]		! *ival = val;
-	ldd	[%l0], %f0		! return 0.0;
-	ret
-	restore
-END(modf)
diff --git a/lib/libc/stdio/flags.c b/lib/libc/stdio/flags.c
index 729b4d5638e6..0b6b0755bd3b 100644
--- a/lib/libc/stdio/flags.c
+++ b/lib/libc/stdio/flags.c
@@ -80,11 +80,30 @@ __sflags(mode, optr)
 		return (0);
 	}
 
-	/* [rwa]\+ or [rwa]b\+ means read and write */
-	if (*mode == '+' || (*mode == 'b' && mode[1] == '+')) {
+	/* 'b' (binary) is ignored */
+	if (*mode == 'b')
+		mode++;
+
+	/* [rwa][b]\+ means read and write */
+	if (*mode == '+') {
+		mode++;
 		ret = __SRW;
 		m = O_RDWR;
 	}
+
+	/* 'b' (binary) can appear here, too -- and is ignored again */
+	if (*mode == 'b')
+		mode++;
+
+	/* 'x' means exclusive (fail if the file exists) */
+	if (*mode == 'x') {
+		if (m == O_RDONLY) {
+			errno = EINVAL;
+			return (0);
+		}
+		o |= O_EXCL;
+	}
+
 	*optr = m | o;
 	return (ret);
 }
diff --git a/lib/libc/stdio/fopen.3 b/lib/libc/stdio/fopen.3
index 5c1f2b11e666..64d033e13ed5 100644
--- a/lib/libc/stdio/fopen.3
+++ b/lib/libc/stdio/fopen.3
@@ -32,7 +32,7 @@
 .\"     @(#)fopen.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd January 26, 2003
+.Dd October 17, 2011
 .Dt FOPEN 3
 .Os
 .Sh NAME
@@ -60,45 +60,51 @@ and associates a stream with it.
 .Pp
 The argument
 .Fa mode
-points to a string beginning with one of the following
-sequences (Additional characters may follow these sequences.):
+points to a string beginning with one of the following letters:
 .Bl -tag -width indent
 .It Dq Li r
-Open text file for reading.
-The stream is positioned at the beginning of the file.
-.It Dq Li r+
-Open for reading and writing.
+Open for reading.
 The stream is positioned at the beginning of the file.
+Fail if the file does not exist.
 .It Dq Li w
-Truncate to zero length or create text file for writing.
-The stream is positioned at the beginning of the file.
-.It Dq Li w+
-Open for reading and writing.
-The file is created if it does not exist, otherwise it is truncated.
+Open for writing.
 The stream is positioned at the beginning of the file.
+Create the file if it does not exist.
 .It Dq Li a
 Open for writing.
-The file is created if it does not exist.
-The stream is positioned at the end of the file.
-Subsequent writes to the file will always end up at the then current
-end of file, irrespective of any intervening
-.Xr fseek 3
-or similar.
-.It Dq Li a+
-Open for reading and writing.
-The file is created if it does not exist.
 The stream is positioned at the end of the file.
 Subsequent writes to the file will always end up at the then current
 end of file, irrespective of any intervening
 .Xr fseek 3
 or similar.
+Create the file if it does not exist.
 .El
 .Pp
+An optional
+.Dq Li +
+following
+.Dq Li r ,
+.Dq Li w ,
+or
+.Dq Li a
+opens the file for both reading and writing.
+An optional
+.Dq Li x
+following
+.Dq Li w
+or
+.Dq Li w+
+causes the
+.Fn fopen
+call to fail if the file already exists.
+.Pp
 The
 .Fa mode
-string can also include the letter ``b'' either as last character or
-as a character between the characters in any of the two-character strings
-described above.
+string can also include the letter
+.Dq Li b
+after either the
+.Dq Li +
+or the first letter.
 This is strictly for compatibility with
 .St -isoC
 and has no effect; the ``b'' is ignored.
@@ -135,6 +141,9 @@ function associates a stream with the existing file descriptor,
 .Fa fildes .
 The mode
 of the stream must be compatible with the mode of the file descriptor.
+The
+.Dq Li x
+mode option is ignored.
 When the stream is closed via
 .Xr fclose 3 ,
 .Fa fildes
@@ -165,29 +174,12 @@ attempts to re-open the file associated with
 with a new mode.
 The new mode must be compatible with the mode that the stream was originally
 opened with:
-.Bl -bullet -offset indent
-.It
-Streams originally opened with mode
-.Dq Li r
-can only be reopened with that same mode.
-.It
-Streams originally opened with mode
-.Dq Li a
-can be reopened with the same mode, or mode
-.Dq Li w .
-.It
-Streams originally opened with mode
-.Dq Li w
-can be reopened with the same mode, or mode
-.Dq Li a .
-.It
-Streams originally opened with mode
-.Dq Li r+ ,
-.Dq Li w+ ,
-or
-.Dq Li a+
-can be reopened with any mode.
-.El
+Streams open for reading can only be re-opened for reading,
+streams open for writing can only be re-opened for writing,
+and streams open for reading and writing can be re-opened in any mode.
+The
+.Dq Li x
+mode option is not meaningful in this context.
 .Pp
 The primary use of the
 .Fn freopen
diff --git a/lib/libc/sys/ktrace.2 b/lib/libc/sys/ktrace.2
index aaf6192628ec..8ab4a33a2112 100644
--- a/lib/libc/sys/ktrace.2
+++ b/lib/libc/sys/ktrace.2
@@ -28,7 +28,7 @@
 .\"     @(#)ktrace.2	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd July 13, 2008
+.Dd October 10, 2011
 .Dt KTRACE 2
 .Os
 .Sh NAME
@@ -67,9 +67,9 @@ The
 argument specifies the requested ktrace operation.
 The defined operations are:
 .Bl -column KTRFLAG_DESCENDXXX -offset indent
-.It "KTROP_SET	Enable trace points specified in"
+.It "KTROP_SET		Enable trace points specified in"
 .Fa trpoints .
-.It "KTROP_CLEAR	Disable trace points specified in
+.It "KTROP_CLEAR	Disable trace points specified in"
 .Fa trpoints .
 .It "KTROP_CLEARFILE	Stop all tracing."
 .It "KTRFLAG_DESCEND	The tracing change should apply to the"
@@ -80,7 +80,7 @@ The
 .Fa trpoints
 argument specifies the trace points of interest.
 The defined trace points are:
-.Bl -column KTRFAC_SYSCALLXXX -offset indent
+.Bl -column KTRFAC_PROCCTORXXX -offset indent
 .It "KTRFAC_SYSCALL	Trace system calls."
 .It "KTRFAC_SYSRET	Trace return values from system calls."
 .It "KTRFAC_NAMEI	Trace name lookup operations."
@@ -88,6 +88,12 @@ The defined trace points are:
 generate much output).
 .It "KTRFAC_PSIG	Trace posted signals."
 .It "KTRFAC_CSW	Trace context switch points."
+.It "KTRFAC_USER	Trace application-specific events."
+.It "KTRFAC_STRUCT	Trace certain data structures."
+.It "KTRFAC_SYSCTL	Trace sysctls."
+.It "KTRFAC_PROCCTOR	Trace process construction."
+.It "KTRFAC_PROCDTOR	Trace process destruction."
+.It "KTRFAC_CAPFAIL	Trace capability failures."
 .It "KTRFAC_INHERIT	Inherit tracing to future children."
 .El
 .Pp
@@ -96,12 +102,12 @@ followed by a trace point specific structure.
 The generic header is:
 .Bd -literal
 struct ktr_header {
-	int	ktr_len;		/* length of buf */
-	short	ktr_type;		/* trace record type */
-	pid_t	ktr_pid;		/* process id */
-	char	ktr_comm[MAXCOMLEN+1];	/* command name */
-	struct	timeval ktr_time;	/* timestamp */
-	intptr_t	ktr_tid;	/* was ktr_buffer */
+	int		ktr_len;		/* length of buf */
+	short		ktr_type;		/* trace record type */
+	pid_t		ktr_pid;		/* process id */
+	char		ktr_comm[MAXCOMLEN+1];	/* command name */
+	struct timeval	ktr_time;		/* timestamp */
+	intptr_t	ktr_tid;		/* was ktr_buffer */
 };
 .Ed
 .Pp
diff --git a/lib/libc/sys/ptrace.2 b/lib/libc/sys/ptrace.2
index 1b23c9b7893b..43592283a93e 100644
--- a/lib/libc/sys/ptrace.2
+++ b/lib/libc/sys/ptrace.2
@@ -2,7 +2,7 @@
 .\"	$NetBSD: ptrace.2,v 1.2 1995/02/27 12:35:37 cgd Exp $
 .\"
 .\" This file is in the public domain.
-.Dd January 23, 2011
+.Dd October 5, 2011
 .Dt PTRACE 2
 .Os
 .Sh NAME
@@ -599,11 +599,3 @@ The
 .Fn ptrace
 function appeared in
 .At v7 .
-.Sh BUGS
-The
-.Dv PL_FLAG_FORKED ,
-.Dv PL_FLAG_SCE ,
-.Dv PL_FLAG_SCX
-and
-.Dv PL_FLAG_EXEC
-are not implemented for MIPS and ARM architectures.
diff --git a/lib/libc/sys/timer_create.2 b/lib/libc/sys/timer_create.2
index 3fb415ca0a85..316cea34ea95 100644
--- a/lib/libc/sys/timer_create.2
+++ b/lib/libc/sys/timer_create.2
@@ -151,7 +151,7 @@ memory protection fault.
 .Sh SEE ALSO
 .Xr clock_getres 2 ,
 .Xr timer_delete 2 ,
-.Xr timer_getoverun 2 ,
+.Xr timer_getoverrun 2 ,
 .Xr siginfo 3
 .Sh STANDARDS
 The
diff --git a/lib/libelf/elf_update.3 b/lib/libelf/elf_update.3
index 7f6f4f09ba55..8803730b7e3e 100644
--- a/lib/libelf/elf_update.3
+++ b/lib/libelf/elf_update.3
@@ -66,7 +66,7 @@ responsibility for the layout of the ELF object.
 If this flag is not set, the ELF library will compute the layout of the
 file from its associated section descriptors.
 .Pp
-It is the application's responsibility to manage the the following
+It is the application's responsibility to manage the following
 structure members in the ELF file:
 .Bl -tag -width indent
 .It "Executable Header"
diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index fefc4c5defd6..0d5f2d6448ee 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h
index 05bee0360993..35e0dfd0a7c8 100644
--- a/lib/libfetch/common.h
+++ b/lib/libfetch/common.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index aaf1a8da4084..40d66bfd59f8 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -1,5 +1,5 @@
 .\"-
-.\" Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+.\" Copyright (c) 1998-2011 Dag-Erling Smørgrav
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 26, 2010
+.Dd September 27, 2011
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -318,9 +318,19 @@ and
 implement the FTP protocol as described in RFC959.
 .Pp
 If the
+.Ql P
+(not passive) flag is specified, an active (rather than passive)
+connection will be attempted.
+.Pp
+The
 .Ql p
-(passive) flag is specified, a passive (rather than active) connection
-will be attempted.
+flag is supported for compatibility with earlier versions where active
+connections were the default.
+It has precedence over the
+.Ql P
+flag, so if both are specified,
+.Nm
+will use a passive connection.
 .Pp
 If the
 .Ql l
@@ -475,9 +485,11 @@ connections will be bound.
 .It Ev FTP_LOGIN
 Default FTP login if none was provided in the URL.
 .It Ev FTP_PASSIVE_MODE
-If set to anything but
+If set to
 .Ql no ,
-forces the FTP code to use passive mode.
+forces the FTP code to use active mode.
+If set to any other value, forces passive mode even if the application
+requested active mode.
 .It Ev FTP_PASSWORD
 Default FTP password if the remote server requests one and none was
 provided in the URL.
diff --git a/lib/libfetch/fetch.c b/lib/libfetch/fetch.c
index 5044fe301bc6..f4c4a707d526 100644
--- a/lib/libfetch/fetch.c
+++ b/lib/libfetch/fetch.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h
index e620a01c600d..be494821261e 100644
--- a/lib/libfetch/fetch.h
+++ b/lib/libfetch/fetch.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c
index dffaad8278fb..8569ff3ebd56 100644
--- a/lib/libfetch/file.c
+++ b/lib/libfetch/file.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 04ae07e9e189..4e650b93c0ab 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
  *
  * Major Changelog:
  *
- * Dag-Erling Coïdan Smørgrav
+ * Dag-Erling Smørgrav
  * 9 Jun 1998
  *
  * Incorporated into libfetch
@@ -633,13 +633,12 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file,
 
 	/* check flags */
 	low = CHECK_FLAG('l');
-	pasv = CHECK_FLAG('p');
+	pasv = CHECK_FLAG('p') || !CHECK_FLAG('P');
 	verbose = CHECK_FLAG('v');
 
 	/* passive mode */
-	if (!pasv)
-		pasv = ((s = getenv("FTP_PASSIVE_MODE")) != NULL &&
-		    strncasecmp(s, "no", 2) != 0);
+	if ((s = getenv("FTP_PASSIVE_MODE")) != NULL)
+		pasv = (strncasecmp(s, "no", 2) != 0);
 
 	/* isolate filename */
 	filename = ftp_filename(file, &filenamelen, &type);
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index 309cbf261c39..b33d6f10dbc7 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 2000-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/lib/libftpio/Makefile b/lib/libftpio/Makefile
deleted file mode 100644
index 2082ec57b03b..000000000000
--- a/lib/libftpio/Makefile
+++ /dev/null
@@ -1,34 +0,0 @@
-# $FreeBSD$
-
-.include <bsd.own.mk>
-
-LIB=		ftpio
-SHLIB_MAJOR=	8
-
-SRCS=		ftpio.c ftperr.c
-INCS=		ftpio.h
-CFLAGS+=	-I${.CURDIR} -Wall
-MAN=		ftpio.3
-CLEANFILES=	ftperr.c
-
-.if ${MK_INET6_SUPPORT} != "no"
-CFLAGS+=	-DINET6
-.endif
-
-WARNS?=		2
-
-ftperr.c:	ftp.errors
-	@echo '#include <stdio.h>' > ${.TARGET}
-	@echo '#include "ftpio.h"' >> ${.TARGET}
-	@echo "struct ftperr ftpErrList[] = {" \ >>  ${.TARGET}
-	@cat ${.ALLSRC} \
-	  | grep -v ^# \
-	  | sort \
-	  | while read NUM STRING; do \
-	    echo "  { $${NUM}, \"$${STRING}\" },"; \
-	  done >> ${.TARGET}
-	@echo "};" >> ${.TARGET}
-	@echo -n "int const ftpErrListLength = " >> ${.TARGET}
-	@echo "sizeof(ftpErrList) / sizeof(*ftpErrList);" >> ${.TARGET}
-
-.include <bsd.lib.mk>
diff --git a/lib/libftpio/ftp.errors b/lib/libftpio/ftp.errors
deleted file mode 100644
index c03eeacae329..000000000000
--- a/lib/libftpio/ftp.errors
+++ /dev/null
@@ -1,45 +0,0 @@
-# $FreeBSD$
-#
-# This list is taken from RFC 959.
-# It probably needs a going over.
-#
-110 Restart marker reply
-120 Service ready in a few minutes
-125 Data connection already open; transfer starting
-150 File status okay; about to open data connection
-200 Command okay
-202 Command not implemented, superfluous at this site
-211 System status, or system help reply
-212 Directory status
-213 File status
-214 Help message
-215 Set system type
-220 Service ready for new user
-221 Service closing control connection
-225 Data connection open; no transfer in progress
-226 Requested file action successful
-227 Entering Passive Mode
-229 Entering Extended Passive Mode
-230 User logged in, proceed
-250 Requested file action okay, completed
-257 File/directory created
-331 User name okay, need password
-332 Need account for login
-350 Requested file action pending further information
-421 Service not available, closing control connection
-425 Can't open data connection
-426 Connection closed; transfer aborted
-450 File unavailable (e.g., file busy)
-451 Requested action aborted: local error in processing
-452 Insufficient storage space in system
-500 Syntax error, command unrecognized
-501 Syntax error in parameters or arguments
-502 Command not implemented
-503 Bad sequence of commands
-504 Command not implemented for that parameter
-530 Not logged in
-532 Need account for storing files
-550 File unavailable (e.g., file not found, no access)
-551 Requested action aborted. Page type unknown
-552 Exceeded storage allocation
-553 File name not allowed
diff --git a/lib/libftpio/ftpio.3 b/lib/libftpio/ftpio.3
deleted file mode 100644
index 73a10728ffdb..000000000000
--- a/lib/libftpio/ftpio.3
+++ /dev/null
@@ -1,260 +0,0 @@
-.\" Copyright (c) 1996 Jordan Hubbard <jkh@FreeBSD.org>
-.\" All rights reserved.
-.\"
-.\" Redistribution and use in source and binary forms, with or without
-.\" modification, are permitted provided that the following conditions
-.\" are met:
-.\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
-.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-.\" SUCH DAMAGE.
-.\"
-.\" $FreeBSD$
-.\"
-.Dd June 17, 1996
-.Dt FTPIO 3
-.Os
-.Sh NAME
-.Nm ftpLogin ,
-.Nm ftpChdir ,
-.Nm ftpErrno ,
-.Nm ftpGetModtime ,
-.Nm ftpGetSize ,
-.Nm ftpGet ,
-.Nm ftpPut ,
-.Nm ftpBinary ,
-.Nm ftpPassive ,
-.Nm ftpVerbose ,
-.Nm ftpGetURL ,
-.Nm ftpPutURL ,
-.Nm ftpLoginAf ,
-.Nm ftpGetURLAf ,
-.Nm ftpPutURLAf
-.Nd FTPIO user library
-.Sh SYNOPSIS
-.In ftpio.h
-.Ft FILE *
-.Fn ftpLogin "char *host" "char *user" "char *passwd" "int ftp_port" "int verbose" "int *retcode"
-.Ft int
-.Fn ftpChdir "FILE *stream" "char *dirname"
-.Ft int
-.Fn ftpErrno "FILE *stream"
-.Ft const char *
-.Fn ftpErrString "int errno"
-.Ft time_t
-.Fn ftpGetModtime "FILE *stream" "char *file"
-.Ft off_t
-.Fn ftpGetSize "FILE *stream" "char *file"
-.Ft FILE *
-.Fn ftpGet "FILE *stream" "char *file" "off_t *seekto"
-.Ft FILE *
-.Fn ftpPut "FILE *stream" "char *file"
-.Ft int
-.Fn ftpAscii "FILE *stream"
-.Ft int
-.Fn ftpBinary "FILE *stream"
-.Ft int
-.Fn ftpPassive "FILE *stream" "int status"
-.Ft void
-.Fn ftpVerbose "FILE *stream" "int status"
-.Ft FILE *
-.Fn ftpGetURL "char *url" "char *user" "char *passwd" "int *retcode"
-.Ft FILE *
-.Fn ftpPutURL "char *url" "char *user" "char *passwd" "int *retcode"
-.Ft FILE *
-.Fn ftpLoginAf "char *host" "int af" "char *user" "char *passwd" "int ftp_port" "int verbose" "int *retcode"
-.Ft FILE *
-.Fn ftpGetURLAf "char *url" "int af" "char *user" "char *passwd" "int *retcode"
-.Ft FILE *
-.Fn ftpPutURLAf "char *url" "int af" "char *user" "char *passwd" "int *retcode"
-.Sh DESCRIPTION
-These functions implement a high-level library for managing FTP connections.
-.Pp
-The
-.Fn ftpLogin
-function attempts to log in using the supplied
-.Fa user ,
-.Fa passwd ,
-.Fa ftp_port
-(if passed as 0,
-.Fa ftp_port
-defaults to the standard ftp port of 21) and
-.Fa verbose
-fields.
-If it is successful, a
-standard stream descriptor is returned which should be passed to
-subsequent FTP operations.
-On failure, NULL is returned and
-.Fa retcode
-will have the error code returned by the foreign server.
-.Pp
-The
-.Fn ftpChdir
-function attempts to issue a server CD command to the directory named in
-.Fa dir .
-On success, zero is returned.
-On failure, the error code from the server.
-.Pp
-The
-.Fn ftpErrno
-function returns the server failure code for the last operation (useful for
-seeing more about what happened if you are familiar with FTP error codes).
-The
-.Fn ftpErrString
-function returns a human readable version of the supplied server failure code.
-.Pp
-The
-.Fn ftpGet
-function attempts to retrieve the file named by the
-.Fa file
-argument (which is assumed to be relative to the FTP server's current directory,
-see
-.Fn ftpChdir )
-and returns a new FILE* pointer for the file or NULL on failure.
-If
-.Fa seekto
-is non-NULL, the contents of the integer it points to will be used
-as a restart point for the file, that is to say that the stream
-returned will point
-.Fa *seekto
-bytes into the file gotten (this is handy for restarting failed
-transfers efficiently).
-If the seek operation fails, the value
-of
-.Fa *seekto
-will be zero'd.
-.Pp
-The
-.Fn ftpGetModtime
-function returns the last modification time of the file named by the
-.Fa file
-argument.
-If the file could not be opened or stat'd, 0 is returned.
-.Pp
-The
-.Fn ftpGetSize
-function returns the size in bytes of the file named by the
-.Fa file
-argument.
-If the file could not be opened or stat'd, -1 is returned.
-.Pp
-The
-.Fn ftpPut
-function attempts to create a new file named by the
-.Fa file
-argument (which is assumed to be relative to the FTP server's current directory,
-see
-.Fn ftpChdir )
-and returns a new
-.Fa stream
-pointer for the file or NULL on failure.
-.Pp
-The
-.Fn ftpAscii
-function sets
-.Tn ASCII
-mode for the current server connection named by
-.Fa stream .
-.Pp
-The
-.Fn ftpBinary
-function sets binary mode for the current server connection named by
-.Fa stream .
-.Pp
-The
-.Fn ftpPassive
-function sets passive mode (for firewalls) for the current server connection
-named by
-.Fa stream
-to boolean value
-.Fa status .
-.Pp
-The
-.Fn ftpVerbose
-function sets the verbosity mode for the current server connection named by
-.Fa stream
-to boolean value
-.Fa status .
-.Pp
-The
-.Fn ftpGetURL
-function attempts to retrieve the file named by the supplied
-.Fa URL
-and can be considered equivalent to the combined
-.Fn ftpLogin ,
-.Fn ftpChdir
-and
-.Fn ftpGet
-operations except that no server
-.Fa stream
-is ever returned - the connection to the server closes when
-the file has been completely read.
-Use the lower-level routines
-if multiple gets are required as it will be far more efficient.
-.Pp
-The
-.Fn ftpPutURL
-function attempts to create the file named by the supplied
-.Fa URL
-and can be considered equivalent to the combined
-.Fn ftpLogin ,
-.Fn ftpChdir
-and
-.Fn ftpPut
-operations except that no server stream is ever returned - the connection
-to the server closes when the file has been completely written.
-Use the
-lower-level routines if multiple puts are required as it will be far more
-efficient.
-.Pp
-The
-.Fn ftpLoginAf ,
-.Fn ftpGetURLAf ,
-.Fn ftpPutURLAf
-functions are same as
-.Fn ftpLogin ,
-.Fn ftpGetURL ,
-.Fn ftpPutURL
-except that they are able to specify address family
-.Fa af .
-.Sh ENVIRONMENT
-.Bl -tag -width FTP_PASSIVE_MODE -offset 3n
-.It Ev FTP_TIMEOUT
-Maximum time, in seconds, to wait for a response
-from the peer before aborting an
-.Tn FTP
-connection.
-.It Ev FTP_PASSIVE_MODE
-If defined, forces the use of passive mode, unless equal
-to ``NO'' or ``no'' in which case active mode is forced.
-If defined, the setting of this variable always overrides any calls to
-.Fn ftpPassive .
-.El
-.Sh HISTORY
-Started life as Poul-Henning Kamp's ftp driver for the system installation
-utility, later significantly mutated into a more general form as an
-extension of stdio by Jordan Hubbard.
-Also incorporates some ideas and
-extensions from Jean-Marc Zucconi.
-.Sh AUTHORS
-.An Jordan Hubbard ,
-.An Poul-Henning Kamp
-and
-.An Jean-Marc Zucconi
-.Sh BUGS
-I am sure you can get this thing's internal state machine confused if
-you really work at it, but so far it has proven itself pretty robust in
-all my tests.
diff --git a/lib/libftpio/ftpio.c b/lib/libftpio/ftpio.c
deleted file mode 100644
index a5c9343c7976..000000000000
--- a/lib/libftpio/ftpio.c
+++ /dev/null
@@ -1,1069 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- *
- * Major Changelog:
- *
- * Jordan K. Hubbard
- * 17 Jan 1996
- *
- * Turned inside out. Now returns xfers as new file ids, not as a special
- * `state' of FTP_t
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-
-#include <arpa/inet.h>
-
-#include <ctype.h>
-#include <errno.h>
-#include <ftpio.h>
-#include <netdb.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#define SUCCESS		 0
-#define FAILURE		-1
-
-#ifndef TRUE
-#define TRUE	(1)
-#define FALSE	(0)
-#endif
-
-/* How to see by a given code whether or not the connection has timed out */
-#define FTP_TIMEOUT(code)	(FtpTimedOut || code == FTP_TIMED_OUT)
-
-/* Internal routines - deal only with internal FTP_t type */
-static FTP_t	ftp_new(void);
-static void	check_passive(FILE *fp);
-static int	ftp_read_method(void *n, char *buf, int nbytes);
-static int	ftp_write_method(void *n, const char *buf, int nbytes);
-static int	ftp_close_method(void *n);
-static int	writes(int fd, char *s);
-static __inline char *get_a_line(FTP_t ftp);
-static int	get_a_number(FTP_t ftp, char **q);
-static int	botch(char *func, char *botch_state);
-static int	cmd(FTP_t ftp, const char *fmt, ...);
-static int	ftp_login_session(FTP_t ftp, char *host, int af, char *user, char *passwd, int port, int verbose);
-static int	ftp_file_op(FTP_t ftp, char *operation, char *file, FILE **fp, char *mode, off_t *seekto);
-static int	ftp_close(FTP_t ftp);
-static int	get_url_info(char *url_in, char *host_ret, int *port_ret, char *name_ret);
-static void	ftp_timeout(int sig);
-static void	ftp_set_timeout(void);
-static void	ftp_clear_timeout(void);
-static void	ai_unmapped(struct addrinfo *);
-
-
-/* Global status variable - ick */
-int FtpTimedOut;
-
-/* FTP happy status codes */
-#define FTP_GENERALLY_HAPPY	200
-#define FTP_ASCII_HAPPY		FTP_GENERALLY_HAPPY
-#define FTP_BINARY_HAPPY	FTP_GENERALLY_HAPPY
-#define FTP_PORT_HAPPY		FTP_GENERALLY_HAPPY
-#define FTP_HAPPY_COMMENT	220
-#define FTP_QUIT_HAPPY		221
-#define FTP_TRANSFER_HAPPY	226
-#define FTP_PASSIVE_HAPPY	227
-#define FTP_LPASSIVE_HAPPY	228
-#define FTP_EPASSIVE_HAPPY	229
-#define FTP_CHDIR_HAPPY		250
-
-/* FTP unhappy status codes */
-#define FTP_TIMED_OUT		421
-
-/*
- * XXX
- * gross!  evil!  bad!  We really need an access primitive for cookie in stdio itself.
- * it's too convenient a hook to bury and it's already exported through funopen as it is, so...
- * XXX
- */
-#define fcookie(fp)	((fp)->_cookie)
-
-/* Placeholder in case we want to do any pre-init stuff at some point */ 
-int
-networkInit()
-{
-    return SUCCESS;	/* XXX dummy function for now XXX */
-}
-
-/* Check a return code with some lenience for back-dated garbage that might be in the buffer */
-static int
-check_code(FTP_t ftp, int var, int preferred)
-{
-    ftp->error = 0;
-    while (1) {
-	if (var == preferred)
-	    return 0;
-	else if (var == FTP_TRANSFER_HAPPY)	/* last operation succeeded */
-	    var = get_a_number(ftp, NULL);
-	else if (var == FTP_HAPPY_COMMENT)	/* chit-chat */
-	    var = get_a_number(ftp, NULL);
-	else if (var == FTP_GENERALLY_HAPPY)	/* general success code */
-	    var = get_a_number(ftp, NULL);
-	else {
-	    ftp->error = var;
-	    return 1;
-	}
-    }
-}
-    
-int
-ftpAscii(FILE *fp)
-{
-    FTP_t ftp = fcookie(fp);
-    int i;
-
-    if (!ftp->is_binary)
-	return SUCCESS;
-    i = cmd(ftp, "TYPE A");
-    if (i < 0 || check_code(ftp, i, FTP_ASCII_HAPPY))
-	return i;
-    ftp->is_binary = FALSE;
-    return SUCCESS;
-}
-
-int
-ftpBinary(FILE *fp)
-{
-    FTP_t ftp = fcookie(fp);
-    int i;
-
-    if (ftp->is_binary)
-	return SUCCESS;
-    i = cmd(ftp, "TYPE I");
-    if (i < 0 || check_code(ftp, i, FTP_BINARY_HAPPY))
-	return i;
-    ftp->is_binary = TRUE;
-    return SUCCESS;
-}
-void
-ftpVerbose(FILE *fp, int status)
-{
-    FTP_t ftp = fcookie(fp);
-    ftp->is_verbose = status;
-}
-
-int
-ftpChdir(FILE *fp, char *dir)
-{
-    int i;
-    FTP_t ftp = fcookie(fp);
-
-    i = cmd(ftp, "CWD %s", dir);
-    if (i < 0 || check_code(ftp, i, FTP_CHDIR_HAPPY))
-	return i;
-    return SUCCESS;
-}
-
-int
-ftpErrno(FILE *fp)
-{
-    FTP_t ftp = fcookie(fp);
-    return ftp->error;
-}
-
-const char *
-ftpErrString(int error)
-{
-    int	k;
-
-    if (error == -1)
-	return("connection in wrong state");
-    if (error < 100)
-	/* XXX soon UNIX errnos will catch up with FTP protocol errnos */
-	return strerror(error);
-    for (k = 0; k < ftpErrListLength; k++)
-      if (ftpErrList[k].num == error)
-	return(ftpErrList[k].string);
-    return("Unknown error");
-}
-
-off_t
-ftpGetSize(FILE *fp, char *name)
-{
-    int i;
-    char p[BUFSIZ], *cp, *ep;
-    FTP_t ftp = fcookie(fp);
-    off_t size;
-
-    check_passive(fp);
-    sprintf(p, "SIZE %s\r\n", name);
-    if (ftp->is_verbose)
-	fprintf(stderr, "Sending %s", p);
-    if (writes(ftp->fd_ctrl, p))
-	return (off_t)-1;
-    i = get_a_number(ftp, &cp);
-    if (check_code(ftp, i, 213))
-	return (off_t)-1;
-
-    errno = 0;				/* to check for ERANGE */
-    size = (off_t)strtoq(cp, &ep, 10);
-    if (*ep != '\0' || errno == ERANGE)
-	return (off_t)-1;
-    return size;
-}
-
-time_t
-ftpGetModtime(FILE *fp, char *name)
-{
-    char p[BUFSIZ], *cp;
-    struct tm t;
-    time_t t0 = time (0);
-    FTP_t ftp = fcookie(fp);
-    int i;
-
-    check_passive(fp);
-    sprintf(p, "MDTM %s\r\n", name);
-    if (ftp->is_verbose)
-	fprintf(stderr, "Sending %s", p);
-    if (writes(ftp->fd_ctrl, p))
-	return (time_t)0;
-    i = get_a_number(ftp, &cp);
-    if (check_code(ftp, i, 213))
-	return (time_t)0;
-    while (*cp && !isdigit(*cp))
-	cp++;
-    if (!*cp)
-	return (time_t)0;
-    t0 = localtime (&t0)->tm_gmtoff;
-    sscanf(cp, "%04d%02d%02d%02d%02d%02d", &t.tm_year, &t.tm_mon, &t.tm_mday, &t.tm_hour, &t.tm_min, &t.tm_sec);
-    t.tm_mon--;
-    t.tm_year -= 1900;
-    t.tm_isdst=-1;
-    t.tm_gmtoff = 0;
-    t0 += mktime (&t);
-    return t0;
-}
-
-FILE *
-ftpGet(FILE *fp, char *file, off_t *seekto)
-{
-    FILE *fp2;
-    FTP_t ftp = fcookie(fp);
-
-    check_passive(fp);
-    if (ftpBinary(fp) != SUCCESS)
-	return NULL;
-
-    if (ftp_file_op(ftp, "RETR", file, &fp2, "r", seekto) == SUCCESS)
-	return fp2;
-    return NULL;
-}
-
-/* Returns a standard FILE pointer type representing an open control connection */
-FILE *
-ftpLogin(char *host, char *user, char *passwd, int port, int verbose, int *retcode)
-{
-#ifdef INET6
-    return ftpLoginAf(host, AF_UNSPEC, user, passwd, port, verbose, retcode);
-#else
-    return ftpLoginAf(host, AF_INET, user, passwd, port, verbose, retcode);
-#endif
-}
-
-FILE *
-ftpLoginAf(char *host, int af, char *user, char *passwd, int port, int verbose, int *retcode)
-{
-    FTP_t n;
-    FILE *fp;
-
-    if (retcode)
-	*retcode = 0;
-    if (networkInit() != SUCCESS)
-	return NULL;
-
-    n = ftp_new();
-    fp = NULL;
-    if (n && ftp_login_session(n, host, af, user, passwd, port, verbose) == SUCCESS) {
-	fp = funopen(n, ftp_read_method, ftp_write_method, NULL, ftp_close_method);	/* BSD 4.4 function! */
-    }
-    if (retcode) {
-	if (!n)
-	    *retcode = (FtpTimedOut ? FTP_TIMED_OUT : -1);
-	/* Poor attempt at mapping real errnos to FTP error codes */
-	else switch(n->error) {
-	    case EADDRNOTAVAIL:
-		*retcode = FTP_TIMED_OUT;	/* Actually no such host, but we have no way of saying that. :-( */
-		break;
-
-            case ETIMEDOUT:
-		*retcode = FTP_TIMED_OUT;
-		break;
-
-	    default:
-		*retcode = n->error;
-		break;
-	}
-    }
-    return fp;
-}
-
-FILE *
-ftpPut(FILE *fp, char *file)
-{
-    FILE *fp2;
-    FTP_t ftp = fcookie(fp);
-
-    check_passive(fp);
-    if (ftp_file_op(ftp, "STOR", file, &fp2, "w", NULL) == SUCCESS)
-	return fp2;
-    return NULL;
-}
-
-int
-ftpPassive(FILE *fp, int st)
-{
-    FTP_t ftp = fcookie(fp);
-
-    ftp->is_passive = !!st;	/* normalize "st" to zero or one */
-    return SUCCESS;
-}
-
-FILE *
-ftpGetURL(char *url, char *user, char *passwd, int *retcode)
-{
-#ifdef INET6
-    return ftpGetURLAf(url, AF_UNSPEC, user, passwd, retcode);
-#else
-    return ftpGetURLAf(url, AF_INET, user, passwd, retcode);
-#endif
-}
-
-FILE *
-ftpGetURLAf(char *url, int af, char *user, char *passwd, int *retcode)
-{
-    char host[255], name[255];
-    int port;
-    FILE *fp2;
-    static FILE *fp = NULL;
-    static char *prev_host;
-
-    if (retcode)
-	*retcode = 0;
-    if (get_url_info(url, host, &port, name) == SUCCESS) {
-	if (fp && prev_host) {
-	    if (!strcmp(prev_host, host)) {
-		/* Try to use cached connection */
-		fp2 = ftpGet(fp, name, NULL);
-		if (!fp2) {
-		    /* Connection timed out or was no longer valid */
-		    fclose(fp);
-		    free(prev_host);
-		    prev_host = NULL;
-		}
-		else
-		    return fp2;
-	    }
-	    else {
-		/* It's a different host now, flush old */
-		fclose(fp);
-		free(prev_host);
-		prev_host = NULL;
-	    }
-	}
-	fp = ftpLoginAf(host, af, user, passwd, port, 0, retcode);
-	if (fp) {
-	    fp2 = ftpGet(fp, name, NULL);
-	    if (!fp2) {
-		/* Connection timed out or was no longer valid */
-		if (retcode)
-		    *retcode = ftpErrno(fp);
-		fclose(fp);
-		fp = NULL;
-	    }
-	    else
-		prev_host = strdup(host);
-	    return fp2;
-	}
-    }
-    return NULL;
-}
-
-FILE *
-ftpPutURL(char *url, char *user, char *passwd, int *retcode)
-{
-#ifdef INET6
-    return ftpPutURLAf(url, AF_UNSPEC, user, passwd, retcode);
-#else
-    return ftpPutURLAf(url, AF_INET, user, passwd, retcode);
-#endif
-
-}
-
-FILE *
-ftpPutURLAf(char *url, int af, char *user, char *passwd, int *retcode)
-{
-    char host[255], name[255];
-    int port;
-    static FILE *fp = NULL;
-    FILE *fp2;
-
-    if (retcode)
-	*retcode = 0;
-    if (fp) {	/* Close previous managed connection */
-	fclose(fp);
-	fp = NULL;
-    }
-    if (get_url_info(url, host, &port, name) == SUCCESS) {
-	fp = ftpLoginAf(host, af, user, passwd, port, 0, retcode);
-	if (fp) {
-	    fp2 = ftpPut(fp, name);
-	    if (!fp2) {
-		if (retcode)
-		    *retcode = ftpErrno(fp);
-		fclose(fp);
-		fp = NULL;
-	    }
-	    return fp2;
-	}
-    }
-    return NULL;
-}
-
-/* Internal workhorse function for dissecting URLs.  Takes a URL as the first argument and returns the
-   result of such disection in the host, user, passwd, port and name variables. */
-static int
-get_url_info(char *url_in, char *host_ret, int *port_ret, char *name_ret)
-{
-    char *name, *host, *cp, url[BUFSIZ];
-    int port;
-
-    name = host = NULL;
-    /* XXX add http:// here or somewhere reasonable at some point XXX */
-    if (strncmp("ftp://", url_in, 6) != 0)
-	return FAILURE;
-    /* We like to stomp a lot on the URL string in dissecting it, so copy it first */
-    strncpy(url, url_in, BUFSIZ);
-    host = url + 6;
-    if ((cp = index(host, ':')) != NULL) {
-	*(cp++) = '\0';
-	port = strtol(cp, 0, 0);
-    }
-    else
-	port = 0;	/* use default */
-    if (port_ret)
-	*port_ret = port;
-    
-    if ((name = index(cp ? cp : host, '/')) != NULL)
-	*(name++) = '\0';
-    if (host_ret)
-	strcpy(host_ret, host);
-    if (name && name_ret)
-	strcpy(name_ret, name);
-    return SUCCESS;
-}
-
-static FTP_t
-ftp_new(void)
-{
-    FTP_t ftp;
-
-    ftp = (FTP_t)malloc(sizeof *ftp);
-    if (!ftp)
-	return NULL;
-    memset(ftp, 0, sizeof *ftp);
-    ftp->fd_ctrl = -1;
-    ftp->con_state = init;
-    ftp->is_binary = FALSE;
-    ftp->is_passive = FALSE;
-    ftp->is_verbose = FALSE;
-    ftp->error = 0;
-    return ftp;
-}
-
-static int
-ftp_read_method(void *vp, char *buf, int nbytes)
-{
-    int i, fd;
-    FTP_t n = (FTP_t)vp;
-
-    fd = n->fd_ctrl;
-    i = (fd >= 0) ? read(fd, buf, nbytes) : EOF;
-    return i;
-}
-
-static int
-ftp_write_method(void *vp, const char *buf, int nbytes)
-{
-    int i, fd;
-    FTP_t n = (FTP_t)vp;
-
-    fd = n->fd_ctrl;
-    i = (fd >= 0) ? write(fd, buf, nbytes) : EOF;
-    return i;
-}
-
-static int
-ftp_close_method(void *n)
-{
-    int i;
-
-    i = ftp_close((FTP_t)n);
-    free(n);
-    return i;
-}
-
-/*
- * This function checks whether the FTP_PASSIVE_MODE environment
- * variable is set, and, if so, enforces the desired mode.
- */
-static void
-check_passive(FILE *fp)
-{
-    const char *cp = getenv("FTP_PASSIVE_MODE");
-
-    if (cp != NULL)
-    	ftpPassive(fp, strncasecmp(cp, "no", 2));
-}
-
-static void
-ftp_timeout(int sig)
-{
-    FtpTimedOut = TRUE;
-    /* Debug("ftp_pkg: ftp_timeout called - operation timed out"); */
-}
-
-static void
-ftp_set_timeout(void)
-{
-    struct sigaction new;
-    char *cp;
-    int ival;
-
-    FtpTimedOut = FALSE;
-    sigemptyset(&new.sa_mask);
-    new.sa_flags = 0;
-    new.sa_handler = ftp_timeout;
-    sigaction(SIGALRM, &new, NULL);
-    cp = getenv("FTP_TIMEOUT");
-    if (!cp || !(ival = atoi(cp)))
-	ival = 120;
-    alarm(ival);
-}
-
-static void
-ftp_clear_timeout(void)
-{
-    struct sigaction new;
-
-    alarm(0);
-    sigemptyset(&new.sa_mask);
-    new.sa_flags = 0;
-    new.sa_handler = SIG_DFL;
-    sigaction(SIGALRM, &new, NULL);
-}
-
-static int
-writes(int fd, char *s)
-{
-    int n, i = strlen(s);
-
-    ftp_set_timeout();
-    n = write(fd, s, i);
-    ftp_clear_timeout();
-    if (FtpTimedOut || i != n)
-	return TRUE;
-    return FALSE;
-}
-
-static __inline char *
-get_a_line(FTP_t ftp)
-{
-    static char buf[BUFSIZ];
-    int i,j;
-
-    /* Debug("ftp_pkg: trying to read a line from %d", ftp->fd_ctrl); */
-    for(i = 0; i < BUFSIZ;) {
-	ftp_set_timeout();
-	j = read(ftp->fd_ctrl, buf + i, 1);
-	ftp_clear_timeout();
-	if (FtpTimedOut || j != 1)
-	    return NULL;
-	if (buf[i] == '\r' || buf[i] == '\n') {
-	    if (!i)
-		continue;
-	    buf[i] = '\0';
-	    if (ftp->is_verbose == TRUE)
-		fprintf(stderr, "%s\n",buf+4);
-	    return buf;
-	}
-	i++;
-    }
-    /* Debug("ftp_pkg: read string \"%s\" from %d", buf, ftp->fd_ctrl); */
-    return buf;
-}
-
-static int
-get_a_number(FTP_t ftp, char **q)
-{
-    char *p;
-    int i = -1, j;
-
-    while(1) {
-	p = get_a_line(ftp);
-	if (!p) {
-	    ftp_close(ftp);
-	    if (FtpTimedOut)
-		return FTP_TIMED_OUT;
-	    return FAILURE;
-	}
-	if (!(isdigit(p[0]) && isdigit(p[1]) && isdigit(p[2])))
-	    continue;
-	if (i == -1 && p[3] == '-') {
-	    i = strtol(p, 0, 0);
-	    continue;
-	}
-	if (p[3] != ' ' && p[3] != '\t')
-	    continue;
-	j = strtol(p, 0, 0);
-	if (i == -1) {
-	    if (q) *q = p+4;
-	    /* Debug("ftp_pkg: read reply %d from server (%s)", j, p); */
-	    return j;
-	} else if (j == i) {
-	    if (q) *q = p+4;
-	    /* Debug("ftp_pkg: read reply %d from server (%s)", j, p); */
-	    return j;
-	}
-    }
-}
-
-static int
-ftp_close(FTP_t ftp)
-{
-    int i, rcode;
-
-    rcode = FAILURE;
-    if (ftp->con_state == isopen) {
-	ftp->con_state = quit;
-	/* If last operation timed out, don't try to quit - just close */
-	if (ftp->error != FTP_TIMED_OUT)
-	    i = cmd(ftp, "QUIT");
-	else
-	    i = FTP_QUIT_HAPPY;
-	if (!check_code(ftp, i, FTP_QUIT_HAPPY))
-	    rcode = SUCCESS;
-	close(ftp->fd_ctrl);
-	ftp->fd_ctrl = -1;
-    }
-    else if (ftp->con_state == quit)
-	rcode = SUCCESS;
-    return rcode;
-}
-
-static int
-botch(char *func, char *botch_state)
-{
-    /* Debug("ftp_pkg: botch: %s(%s)", func, botch_state); */
-    return FAILURE;
-}
-
-static int
-cmd(FTP_t ftp, const char *fmt, ...)
-{
-    char p[BUFSIZ];
-    int i;
-
-    va_list ap;
-    va_start(ap, fmt);
-    (void)vsnprintf(p, sizeof p, fmt, ap);
-    va_end(ap);
-
-    if (ftp->con_state == init)
-	return botch("cmd", "open");
-
-    strcat(p, "\r\n");
-    if (ftp->is_verbose)
-	fprintf(stderr, "Sending: %s", p);
-    if (writes(ftp->fd_ctrl, p)) {
-	if (FtpTimedOut)
-	    return FTP_TIMED_OUT;
-	return FAILURE;
-    }
-    while ((i = get_a_number(ftp, NULL)) == FTP_HAPPY_COMMENT);
-    return i;
-}
-
-static int
-ftp_login_session(FTP_t ftp, char *host, int af,
-		  char *user, char *passwd, int port, int verbose)
-{
-    char pbuf[10];
-    struct addrinfo	hints, *res, *res0;
-    int			err;
-    int 		s;
-    int			i;
-
-    if (networkInit() != SUCCESS)
-	return FAILURE;
-
-    if (ftp->con_state != init) {
-	ftp_close(ftp);
-	ftp->error = -1;
-	return FAILURE;
-    }
-
-    if (!user)
-	user = "ftp";
-
-    if (!passwd)
-	passwd = "setup@";
-
-    if (!port)
-	port = 21;
-
-    snprintf(pbuf, sizeof(pbuf), "%d", port);
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = af;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_protocol = 0;
-    err = getaddrinfo(host, pbuf, &hints, &res0);
-    if (err) {
-	ftp->error = 0;
-	return FAILURE;
-    }
-
-    s = -1;
-    for (res = res0; res; res = res->ai_next) {
-	ai_unmapped(res);
-	ftp->addrtype = res->ai_family;
-
-	if ((s = socket(res->ai_family, res->ai_socktype,
-			res->ai_protocol)) < 0)
-	    continue;
-
-	if (connect(s, res->ai_addr, res->ai_addrlen) < 0) {
-	    (void)close(s);
-	    s = -1;
-	    continue;
-	}
-
-	break;
-    }
-    freeaddrinfo(res0);
-    if (s < 0) {
-	ftp->error = errno;
-	return FAILURE;
-    }
-
-    ftp->fd_ctrl = s;
-    ftp->con_state = isopen;
-    ftp->is_verbose = verbose;
-
-    i = cmd(ftp, "USER %s", user);
-    if (i >= 300 && i < 400)
-	i = cmd(ftp, "PASS %s", passwd);
-    if (i >= 299 || i < 0) {
-	ftp_close(ftp);
-	if (i > 0)
-	    ftp->error = i;
-	return FAILURE;
-    }
-    return SUCCESS;
-}
-
-static int
-ftp_file_op(FTP_t ftp, char *operation, char *file, FILE **fp, char *mode, off_t *seekto)
-{
-    socklen_t sinlen;
-    int i,l,s;
-    char *q;
-    unsigned char addr[64];
-    union sockaddr_cmn {
-	struct sockaddr_in sin4;
-	struct sockaddr_in6 sin6;
-    } sin;
-    char *cmdstr;
-
-    if (!fp)
-	return FAILURE;
-    *fp = NULL;
-
-    if (ftp->con_state != isopen)
-	return botch("ftp_file_op", "open");
-
-    if ((s = socket(ftp->addrtype, SOCK_STREAM, 0)) < 0) {
-	ftp->error = errno;
-	return FAILURE;
-    }
-
-    if (ftp->is_passive) {
-	if (ftp->addrtype == AF_INET) {
-            if (ftp->is_verbose)
-		fprintf(stderr, "Sending PASV\n");
-	    if (writes(ftp->fd_ctrl, "PASV\r\n")) {
-		ftp_close(ftp);
-		if (FtpTimedOut)
-		    ftp->error = FTP_TIMED_OUT;
-		return FTP_TIMED_OUT;
-	    }
-	    i = get_a_number(ftp, &q);
-	    if (check_code(ftp, i, FTP_PASSIVE_HAPPY)) {
-		ftp_close(ftp);
-		return i;
-	    }
-	    cmdstr = "PASV";
-	} else {
-            if (ftp->is_verbose)
-		fprintf(stderr, "Sending EPSV\n");
-	    if (writes(ftp->fd_ctrl, "EPSV\r\n")) {
-		ftp_close(ftp);
-		if (FtpTimedOut)
-		    ftp->error = FTP_TIMED_OUT;
-		return FTP_TIMED_OUT;
-	    }
-	    i = get_a_number(ftp, &q);
-	    if (check_code(ftp, i, FTP_EPASSIVE_HAPPY)) {
-		if (ftp->is_verbose)
-		    fprintf(stderr, "Sending LPSV\n");
-		if (writes(ftp->fd_ctrl, "LPSV\r\n")) {
-		    ftp_close(ftp);
-		    if (FtpTimedOut)
-			ftp->error = FTP_TIMED_OUT;
-		    return FTP_TIMED_OUT;
-		}
-		i = get_a_number(ftp, &q);
-		if (check_code(ftp, i, FTP_LPASSIVE_HAPPY)) {
-		    ftp_close(ftp);
-		    return i;
-		}
-		cmdstr = "LPSV";
-	    } else
-		cmdstr = "EPSV";
-	}
-	if (strcmp(cmdstr, "PASV") == 0 || strcmp(cmdstr, "LPSV") == 0) {
-	    while (*q && !isdigit(*q))
-		q++;
-	    if (!*q) {
-		ftp_close(ftp);
-		return FAILURE;
-	    }
-	    q--;
-	    l = (ftp->addrtype == AF_INET ? 6 : 21);
-	    for (i = 0; i < l; i++) {
-		q++;
-		addr[i] = strtol(q, &q, 10);
-	    }
-
-	    sin.sin4.sin_family = ftp->addrtype;
-	    if (ftp->addrtype == AF_INET6) {
-		sin.sin6.sin6_len = sizeof(struct sockaddr_in6);
-		bcopy(addr + 2, (char *)&sin.sin6.sin6_addr, 16);
-		bcopy(addr + 19, (char *)&sin.sin6.sin6_port, 2);
-	    } else {
-		sin.sin4.sin_len = sizeof(struct sockaddr_in);
-		bcopy(addr, (char *)&sin.sin4.sin_addr, 4);
-		bcopy(addr + 4, (char *)&sin.sin4.sin_port, 2);
-	    }
-	} else if (strcmp(cmdstr, "EPSV") == 0) {
-	    int port;
-	    while (*q && *q != '(')		/* ) */
-		q++;
-	    if (!*q) {
-		ftp_close(ftp);
-		return FAILURE;
-	    }
-	    q++;
-	    if (sscanf(q, "%c%c%c%d%c", &addr[0], &addr[1], &addr[2],
-		    &port, &addr[3]) != 5
-	     || addr[0] != addr[1] || addr[0] != addr[2] || addr[0] != addr[3]) {
-		ftp_close(ftp);
-		return FAILURE;
-	    }
-	    sinlen = sizeof(sin);
-	    if (getpeername(ftp->fd_ctrl, (struct sockaddr *)&sin, &sinlen) < 0) {
-		ftp_close(ftp);
-		return FAILURE;
-	    }
-	    switch (sin.sin4.sin_family) {
-	    case AF_INET:
-		sin.sin4.sin_port = htons(port);
-		break;
-	    case AF_INET6:
-		sin.sin6.sin6_port = htons(port);
-		break;
-	    default:
-		ftp_close(ftp);
-		return FAILURE;
-	    }
-	}
-
-	if (connect(s, (struct sockaddr *)&sin, sin.sin4.sin_len) < 0) {
-	    (void)close(s);
-	    return FAILURE;
-	}
-
-	if (seekto && *seekto) {
-	    i = cmd(ftp, "REST %d", *seekto);
-	    if (i < 0 || FTP_TIMEOUT(i)) {
-		close(s);
-		ftp->error = i;
-		*seekto = (off_t)0;
-		return i;
-	    }
-	}
-	i = cmd(ftp, "%s %s", operation, file);
-	if (i < 0 || i > 299) {
-	    close(s);
-	    ftp->error = i;
-	    return i;
-	}
-	*fp = fdopen(s, mode);
-    }
-    else {
-	int fd,portrange;
-
-#ifdef IPV6_PORTRANGE
-	if (ftp->addrtype == AF_INET6) {
-		portrange = IPV6_PORTRANGE_HIGH;
-		if (setsockopt(s, IPPROTO_IPV6, IPV6_PORTRANGE, (char *)
-			       &portrange, sizeof(portrange)) < 0) {
-			close(s);   
-			return FAILURE;
-		}
-	}
-#endif
-#ifdef IP_PORTRANGE
-	if (ftp->addrtype == AF_INET) {
-		portrange = IP_PORTRANGE_HIGH;
-		if (setsockopt(s, IPPROTO_IP, IP_PORTRANGE, (char *)
-			       &portrange, sizeof(portrange)) < 0) {
-			close(s);   
-			return FAILURE;
-		}
-	}
-#endif
-
-	sinlen = sizeof sin;
-	getsockname(ftp->fd_ctrl, (struct sockaddr *)&sin, &sinlen);
-	sin.sin4.sin_port = 0;
-	i = ((struct sockaddr *)&sin)->sa_len;
-	if (bind(s, (struct sockaddr *)&sin, i) < 0) {
-	    close(s);	
-	    return FAILURE;
-	}
-	sinlen = sizeof sin;
-	getsockname(s, (struct sockaddr *)&sin, &sinlen);
-	if (listen(s, 1) < 0) {
-	    close(s);	
-	    return FAILURE;
-	}
-	if (sin.sin4.sin_family == AF_INET) {
-            u_long a;
-	    a = ntohl(sin.sin4.sin_addr.s_addr);
-	    i = cmd(ftp, "PORT %d,%d,%d,%d,%d,%d",
-		    (a                   >> 24) & 0xff,
-		    (a                   >> 16) & 0xff,
-		    (a                   >>  8) & 0xff,
-		    a                           & 0xff,
-		    (ntohs(sin.sin4.sin_port) >>  8) & 0xff,
-		    ntohs(sin.sin4.sin_port)         & 0xff);
-	    if (check_code(ftp, i, FTP_PORT_HAPPY)) {
-		close(s);
-		return i;
-	    }
-	} else {
-#define UC(b)	(((int)b)&0xff)
-	    char *a;
-	    char hname[INET6_ADDRSTRLEN];
-
-	    sin.sin6.sin6_scope_id = 0;
-	    if (getnameinfo((struct sockaddr *)&sin, sin.sin6.sin6_len,
-			    hname, sizeof(hname),
-			    NULL, 0, NI_NUMERICHOST) != 0) {
-		goto try_lprt;
-	    }
-	    i = cmd(ftp, "EPRT |%d|%s|%d|", 2, hname,
-		    htons(sin.sin6.sin6_port));
-	    if (check_code(ftp, i, FTP_PORT_HAPPY)) {
-try_lprt:
-		a = (char *)&sin.sin6.sin6_addr;
-		i = cmd(ftp,
-"LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",
-			6, 16,
-			UC(a[0]),UC(a[1]),UC(a[2]),UC(a[3]),
-			UC(a[4]),UC(a[5]),UC(a[6]),UC(a[7]),
-			UC(a[8]),UC(a[9]),UC(a[10]),UC(a[11]),
-			UC(a[12]),UC(a[13]),UC(a[14]),UC(a[15]),
-			2,
-			(ntohs(sin.sin4.sin_port) >>  8) & 0xff,
-			ntohs(sin.sin4.sin_port)         & 0xff);
-		if (check_code(ftp, i, FTP_PORT_HAPPY)) {
-		    close(s);
-		    return i;
-		}
-	    }
-	}
-	if (seekto && *seekto) {
-	    i = cmd(ftp, "REST %d", *seekto);
-	    if (i < 0 || FTP_TIMEOUT(i)) {
-		close(s);
-		ftp->error = i;
-		return i;
-	    }
-	    else if (i != 350)
-		*seekto = (off_t)0;
-	}
-	i = cmd(ftp, "%s %s", operation, file);
-	if (i < 0 || i > 299) {
-	    close(s);
-	    ftp->error = i;
-	    return FAILURE;
-	}
-	fd = accept(s, 0, 0);
-	if (fd < 0) {
-	    close(s);
-	    ftp->error = 401;
-	    return FAILURE;
-	}
-	close(s);
-	*fp = fdopen(fd, mode);
-    }
-    if (*fp)
-	return SUCCESS;
-    else
-	return FAILURE;
-}
-
-static void
-ai_unmapped(struct addrinfo *ai)
-{
-	struct sockaddr_in6 *sin6;
-	struct sockaddr_in sin;
-
-	if (ai->ai_family != AF_INET6)
-		return;
-	if (ai->ai_addrlen != sizeof(struct sockaddr_in6) ||
-	    sizeof(sin) > ai->ai_addrlen)
-		return;
-	sin6 = (struct sockaddr_in6 *)ai->ai_addr;
-	if (!IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr))
-		return;
-
-	memset(&sin, 0, sizeof(sin));
-	sin.sin_family = AF_INET;
-	sin.sin_len = sizeof(struct sockaddr_in);
-	memcpy(&sin.sin_addr, &sin6->sin6_addr.s6_addr[12],
-	    sizeof(sin.sin_addr));
-	sin.sin_port = sin6->sin6_port;
-
-	ai->ai_family = AF_INET;
-	memcpy(ai->ai_addr, &sin, sin.sin_len);
-	ai->ai_addrlen = sin.sin_len;
-}
diff --git a/lib/libftpio/ftpio.h b/lib/libftpio/ftpio.h
deleted file mode 100644
index 879abea6607c..000000000000
--- a/lib/libftpio/ftpio.h
+++ /dev/null
@@ -1,71 +0,0 @@
-#ifndef _FTP_H_INCLUDE
-#define _FTP_H_INCLUDE
-
-#include <sys/types.h>
-#include <sys/cdefs.h>
-#include <stdio.h>
-#include <time.h>
-
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- *
- * Major Changelog:
- *
- * Jordan K. Hubbard
- * 17 Jan 1996
- *
- * Turned inside out. Now returns xfers as new file ids, not as a special
- * `state' of FTP_t
- *
- * $FreeBSD$
- */
-
-/* Internal housekeeping data structure for FTP sessions */
-typedef struct {
-    enum { init, isopen, quit } con_state;
-    int		fd_ctrl;
-    int		addrtype;
-    char	*host;
-    char	*file;
-    int		error;
-    int		is_binary;
-    int		is_passive;
-    int		is_verbose;
-} *FTP_t;
-
-/* Structure we use to match FTP error codes with readable strings */
-struct ftperr {
-  const int	num;
-  const char	*string;
-};
-
-__BEGIN_DECLS
-extern struct	ftperr ftpErrList[];
-extern int	const ftpErrListLength;
-
-/* Exported routines - deal only with FILE* type */
-extern FILE	*ftpLogin(char *host, char *user, char *passwd,	int port, int verbose, int *retcode);
-extern int	ftpChdir(FILE *fp, char *dir);
-extern int	ftpErrno(FILE *fp);
-extern off_t	ftpGetSize(FILE *fp, char *file);
-extern FILE	*ftpGet(FILE *fp, char *file, off_t *seekto);
-extern FILE	*ftpPut(FILE *fp, char *file);
-extern int	ftpAscii(FILE *fp);
-extern int	ftpBinary(FILE *fp);
-extern int	ftpPassive(FILE *fp, int status);
-extern void	ftpVerbose(FILE *fp, int status);
-extern FILE	*ftpGetURL(char	*url, char *user, char *passwd,	int *retcode);
-extern FILE	*ftpPutURL(char	*url, char *user, char *passwd,	int *retcode);
-extern time_t	ftpGetModtime(FILE *fp, char *s);
-extern const	char *ftpErrString(int error);
-extern FILE	*ftpLoginAf(char *host, int af, char *user, char *passwd,	int port, int verbose, int *retcode);
-extern FILE	*ftpGetURLAf(char *url, int af, char *user, char *passwd, int *retcode);
-extern FILE	*ftpPutURLAf(char *url, int af, char *user, char *passwd, int *retcode);
-__END_DECLS
-
-#endif	/* _FTP_H_INCLUDE */
diff --git a/lib/libgssapi/Symbol.map b/lib/libgssapi/Symbol.map
index 6df8f0c4716d..717f1b9e5af1 100644
--- a/lib/libgssapi/Symbol.map
+++ b/lib/libgssapi/Symbol.map
@@ -1,6 +1,6 @@
 /*
  * $FreeBSD$
- */	
+ */
 
 FBSD_1.1 {
 	GSS_C_NT_ANONYMOUS;
diff --git a/lib/libipsec/ipsec_strerror.3 b/lib/libipsec/ipsec_strerror.3
index 73fe68dcce1f..45e111343d9a 100644
--- a/lib/libipsec/ipsec_strerror.3
+++ b/lib/libipsec/ipsec_strerror.3
@@ -63,7 +63,7 @@ as an underlying function, calling
 .Xr strerror 3
 after
 .Fn ipsec_strerror
-would overwrite the the return value from
+would overwrite the return value from
 .Fn ipsec_strerror
 and make it invalid.
 .\"
diff --git a/lib/libmp/Symbol.map b/lib/libmp/Symbol.map
index 0bd8684bfa9e..df040986e53f 100644
--- a/lib/libmp/Symbol.map
+++ b/lib/libmp/Symbol.map
@@ -1,6 +1,6 @@
 /*
  * $FreeBSD$
- */	
+ */
 
 FBSD_1.1 {
 	mp_gcd;
diff --git a/lib/libncp/ncpl_bind.c b/lib/libncp/ncpl_bind.c
index d7c64ede86c4..d00cfa8376c2 100644
--- a/lib/libncp/ncpl_bind.c
+++ b/lib/libncp/ncpl_bind.c
@@ -85,7 +85,7 @@ ncp_read_property_value(NWCONN_HANDLE connid, int object_type,
 
 int
 ncp_scan_bindery_object(NWCONN_HANDLE connid, u_int32_t last_id,
-	u_int16_t object_type, char *search_string,
+	u_int16_t object_type, const char *search_string,
 	struct ncp_bindery_object *target)
 {
 	int error;
diff --git a/lib/libpam/libpam/Makefile b/lib/libpam/libpam/Makefile
index 8b07458bf4b1..12cd96a28cc3 100644
--- a/lib/libpam/libpam/Makefile
+++ b/lib/libpam/libpam/Makefile
@@ -148,7 +148,15 @@ HEADERS=	security/openpam.h \
 
 ADD_HEADERS=	security/pam_mod_misc.h
 
+#
 # Static modules
+#
+# We build static versions of all modules and of openpam_static.o,
+# then link them all together into openpam_static_modules.o.  None of
+# the modules export any symbols, but they store structures with
+# pointers to their service functions in a linker set which the code
+# in openpam_static.c traverses to locate the individual modules.
+#
 MODULE_DIR=	../modules
 .include "${.CURDIR}/${MODULE_DIR}/modules.inc"
 STATIC_MODULES=	${MODULES:C/.*/${MODULE_DIR}\/&\/lib&.a/}
@@ -159,7 +167,8 @@ CLEANFILES+=	openpam_static.o \
 openpam_static_modules.o: openpam_static.o ${STATIC_MODULES}
 	${LD} -o ${.TARGET} -r --whole-archive ${.ALLSRC}
 
-# Can't put openpam_static.c in SRCS but want .o in .depend.
+# We can't put openpam_static.c in SRCS, but we still want to scan it
+# for dependencies.
 DPSRCS=		openpam_static.c
 
 # Headers
diff --git a/lib/libpam/modules/pam_ssh/pam_ssh.8 b/lib/libpam/modules/pam_ssh/pam_ssh.8
index 07e3176bb66a..ea41455f2e53 100644
--- a/lib/libpam/modules/pam_ssh/pam_ssh.8
+++ b/lib/libpam/modules/pam_ssh/pam_ssh.8
@@ -1,6 +1,6 @@
 .\" Copyright (c) 2001 Mark R V Murray
-.\" All rights reserved.
 .\" Copyright (c) 2001-2003 Networks Associates Technology, Inc.
+.\" Copyright (c) 2004-2011 Dag-Erling Smørgrav
 .\" All rights reserved.
 .\"
 .\" This software was developed for the FreeBSD Project by ThinkSec AS and
@@ -34,7 +34,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 26, 2001
+.Dd October 7, 2011
 .Dt PAM_SSH 8
 .Os
 .Sh NAME
@@ -135,6 +135,8 @@ SSH1 RSA key
 SSH2 RSA key
 .It Pa $HOME/.ssh/id_dsa
 SSH2 DSA key
+.It Pa $HOME/.ssh/id_ecdsa
+SSH2 ECDSA key
 .El
 .Sh SEE ALSO
 .Xr ssh-agent 1 ,
diff --git a/lib/libpam/modules/pam_ssh/pam_ssh.c b/lib/libpam/modules/pam_ssh/pam_ssh.c
index bec9892b52ae..bc5f5223630f 100644
--- a/lib/libpam/modules/pam_ssh/pam_ssh.c
+++ b/lib/libpam/modules/pam_ssh/pam_ssh.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003 Networks Associates Technology, Inc.
+ * Copyright (c) 2004-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * This software was developed for the FreeBSD Project by ThinkSec AS and
@@ -78,6 +79,7 @@ static const char *pam_ssh_keyfiles[] = {
 	".ssh/identity",	/* SSH1 RSA key */
 	".ssh/id_rsa",		/* SSH2 RSA key */
 	".ssh/id_dsa",		/* SSH2 DSA key */
+	".ssh/id_ecdsa",	/* SSH2 ECDSA key */
 	NULL
 };
 
@@ -324,6 +326,7 @@ pam_ssh_add_keys_to_agent(pam_handle_t *pamh)
 
 	/* get a connection to the agent */
 	if ((ac = ssh_get_authentication_connection()) == NULL) {
+		openpam_log(PAM_LOG_DEBUG, "failed to connect to the agent");
 		pam_err = PAM_SYSTEM_ERR;
 		goto end;
 	}
diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 1d86a828120c..f0d5b0c7b5e6 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -2595,6 +2595,12 @@ pmc_flush_logfile(void)
 	return (PMC_CALL(FLUSHLOG,0));
 }
 
+int
+pmc_close_logfile(void)
+{
+	return (PMC_CALL(CLOSELOG,0));
+}
+
 int
 pmc_get_driver_stats(struct pmc_driverstats *ds)
 {
diff --git a/lib/libpmc/pmc.3 b/lib/libpmc/pmc.3
index 2403e64686bb..2431080a32a9 100644
--- a/lib/libpmc/pmc.3
+++ b/lib/libpmc/pmc.3
@@ -322,6 +322,10 @@ to write logged events to.
 Flush all pending log data in
 .Xr hwpmc 4 Ns Ap s
 buffers.
+.It Fn pmc_close_logfile
+Flush all pending log data and close
+.Xr hwpmc 4 Ns Ap s
+side of the stream.
 .It Fn pmc_writelog
 Append arbitrary user data to the current log file.
 .El
diff --git a/lib/libpmc/pmc.core.3 b/lib/libpmc/pmc.core.3
index d32e62a968bc..db4c769c3a22 100644
--- a/lib/libpmc/pmc.core.3
+++ b/lib/libpmc/pmc.core.3
@@ -459,7 +459,7 @@ The number of instruction fetch misses in the instruction cache and
 streaming buffers.
 .It Li ICache_Reads
 .Pq Event 80H , Umask 00H
-The number of instruction fetches from the the instruction cache and
+The number of instruction fetches from the instruction cache and
 streaming buffers counting both cacheable and un-cacheable fetches.
 .It Li IFU_Mem_Stall
 .Pq Event 86H , Umask 00H
diff --git a/lib/libpmc/pmc.h b/lib/libpmc/pmc.h
index 5b1ad07c339a..2be88b121d3c 100644
--- a/lib/libpmc/pmc.h
+++ b/lib/libpmc/pmc.h
@@ -76,6 +76,7 @@ int	pmc_attach(pmc_id_t _pmcid, pid_t _pid);
 int	pmc_capabilities(pmc_id_t _pmc, uint32_t *_caps);
 int	pmc_configure_logfile(int _fd);
 int	pmc_flush_logfile(void);
+int	pmc_close_logfile(void);
 int	pmc_detach(pmc_id_t _pmcid, pid_t _pid);
 int	pmc_disable(int _cpu, int _pmc);
 int	pmc_enable(int _cpu, int _pmc);
diff --git a/lib/libpmc/pmc_configure_logfile.3 b/lib/libpmc/pmc_configure_logfile.3
index a33688c222c0..399f21a7c542 100644
--- a/lib/libpmc/pmc_configure_logfile.3
+++ b/lib/libpmc/pmc_configure_logfile.3
@@ -29,7 +29,8 @@
 .Sh NAME
 .Nm pmc_configure_logfile ,
 .Nm pmc_flush_logfile ,
-.Nm pmc_writelog
+.Nm pmc_writelog ,
+.Nm pmc_close_logfile
 .Nd log file management
 .Sh LIBRARY
 .Lb libpmc
@@ -41,6 +42,8 @@
 .Fn pmc_flush_logfile void
 .Ft int
 .Fn pmc_writelog "uint32_t userdata"
+.Ft int
+.Fn pmc_close_logfile void
 .Sh DESCRIPTION
 The functions manage logging of
 .Xr hwpmc 4
@@ -72,6 +75,12 @@ Function
 will append a log entry containing the value of argument
 .Fa userdata
 to the log file.
+.Pp
+Function
+.Fn pmc_close_logfile
+will flush all pending log data and close
+.Xr hwpmc 4 Ns Ap s
+side of the stream.
 .Sh RETURN VALUES
 .Rv -std
 .Sh ERRORS
diff --git a/lib/libsdp/sdp.h b/lib/libsdp/sdp.h
index acb074a7bb04..35dbea732fa5 100644
--- a/lib/libsdp/sdp.h
+++ b/lib/libsdp/sdp.h
@@ -312,13 +312,13 @@ typedef struct sdp_attr *	sdp_attr_p;
 
 /* Inline versions of get/put byte/short/long. Pointer is advanced */
 #define SDP_GET8(b, cp) { \
-	register uint8_t *t_cp = (uint8_t *)(cp); \
+	const uint8_t *t_cp = (const uint8_t *)(cp); \
 	(b) = *t_cp; \
 	(cp) ++; \
 }
 
 #define SDP_GET16(s, cp) { \
-	register uint8_t *t_cp = (uint8_t *)(cp); \
+	const uint8_t *t_cp = (const uint8_t *)(cp); \
 	(s) = ((uint16_t)t_cp[0] << 8) \
 	    | ((uint16_t)t_cp[1]) \
 	    ; \
@@ -326,7 +326,7 @@ typedef struct sdp_attr *	sdp_attr_p;
 }
 
 #define SDP_GET32(l, cp) { \
-	register uint8_t *t_cp = (uint8_t *)(cp); \
+	const uint8_t *t_cp = (const uint8_t *)(cp); \
 	(l) = ((uint32_t)t_cp[0] << 24) \
 	    | ((uint32_t)t_cp[1] << 16) \
 	    | ((uint32_t)t_cp[2] << 8) \
diff --git a/lib/libulog/Symbol.map b/lib/libulog/Symbol.map
index e0375aee5ab3..0abc40f6cdcc 100644
--- a/lib/libulog/Symbol.map
+++ b/lib/libulog/Symbol.map
@@ -1,6 +1,6 @@
 /*
  * $FreeBSD$
- */	
+ */
 
 FBSD_1.2 {
 	ulog_login;
diff --git a/lib/libusb/Makefile b/lib/libusb/Makefile
index c4396cca0ae5..b76618a09ee1 100644
--- a/lib/libusb/Makefile
+++ b/lib/libusb/Makefile
@@ -40,6 +40,7 @@ CFLAGS+=	-DCOMPAT_32BIT
 MLINKS += libusb.3 libusb_init.3
 MLINKS += libusb.3 libusb_exit.3
 MLINKS += libusb.3 libusb_strerror.3
+MLINKS += libusb.3 libusb_error_name.3
 MLINKS += libusb.3 libusb_set_debug.3
 MLINKS += libusb.3 libusb_get_device_list.3
 MLINKS += libusb.3 libusb_free_device_list.3
@@ -210,3 +211,5 @@ MLINKS += libusb20.3 libusb20_me_get_2.3
 MLINKS += libusb20.3 libusb20_me_encode.3
 MLINKS += libusb20.3 libusb20_me_decode.3
 MLINKS += libusb20.3 libusb20_desc_foreach.3
+MLINKS += libusb20.3 libusb20_strerror.3
+MLINKS += libusb20.3 libusb20_error_name.3
diff --git a/lib/libusb/libusb.3 b/lib/libusb/libusb.3
index 1f52b9ec17ed..7255f2b4d28b 100644
--- a/lib/libusb/libusb.3
+++ b/lib/libusb/libusb.3
@@ -63,6 +63,14 @@ Other libusb routines may not be called after this function.
 Get the ASCII representation of the error given by the
 .Fa code
 argument.
+This function does not return NULL.
+.Pp
+.Ft const char *
+.Fn libusb_error_name "int code"
+Get the ASCII representation of the error enum given by the
+.Fa code
+argument.
+This function does not return NULL.
 .Pp
 .Ft void
 .Fn libusb_set_debug "libusb_context *ctx" "int level"
@@ -215,8 +223,8 @@ code on failure.
 .Ft int
 .Fn libusb_kernel_driver_active "libusb_device_handle *devh" "int interface"
 Determine if a driver is active on a interface.
-Returns 0 if no kernel driver
-is active, 1 if a kernel driver is active, LIBUSB_ERROR_NO_DEVICE
+Returns 0 if no kernel driver is active
+and 1 if a kernel driver is active, LIBUSB_ERROR_NO_DEVICE
 if the device has been disconnected and a LIBUSB_ERROR code on failure.
 .Pp
 .Ft int
@@ -502,7 +510,8 @@ The library is also compliant with LibUSB version 0.1.12.
 .Sh SEE ALSO
 .Xr libusb20 3 ,
 .Xr usb 4 ,
-.Xr usbconfig 8
+.Xr usbconfig 8 ,
+.Xr usbdump 8
 .Pp
 .Pa http://libusb.sourceforge.net/
 .Sh HISTORY
diff --git a/lib/libusb/libusb.h b/lib/libusb/libusb.h
index bbde475842e0..63e3ebc754fc 100644
--- a/lib/libusb/libusb.h
+++ b/lib/libusb/libusb.h
@@ -303,6 +303,7 @@ typedef struct libusb_transfer {
 
 void	libusb_set_debug(libusb_context * ctx, int level);
 const char *libusb_strerror(int code);
+const char *libusb_error_name(int code);
 int	libusb_init(libusb_context ** context);
 void	libusb_exit(struct libusb_context *ctx);
 
diff --git a/lib/libusb/libusb10.c b/lib/libusb/libusb10.c
index b66ccb6eec51..0a7c1e6b157c 100644
--- a/lib/libusb/libusb10.c
+++ b/lib/libusb/libusb10.c
@@ -216,7 +216,6 @@ libusb_get_device_list(libusb_context *ctx, libusb_device ***list)
 			libusb20_be_free(usb_backend);
 			return (LIBUSB_ERROR_NO_MEM);
 		}
-
 		/* get device into libUSB v1.0 list */
 		libusb20_be_dequeue_device(usb_backend, pdev);
 
@@ -718,8 +717,10 @@ libusb_kernel_driver_active(struct libusb20_device *pdev, int interface)
 	if (pdev == NULL)
 		return (LIBUSB_ERROR_INVALID_PARAM);
 
-	return (libusb20_dev_kernel_driver_active(
-	    pdev, interface));
+	if (libusb20_dev_kernel_driver_active(pdev, interface))
+		return (0);		/* no kernel driver is active */
+	else
+		return (1);		/* kernel driver is active */
 }
 
 int
@@ -1449,6 +1450,73 @@ libusb_le16_to_cpu(uint16_t x)
 const char *
 libusb_strerror(int code)
 {
-	/* TODO */
-	return ("Unknown error");
+	switch (code) {
+	case LIBUSB_SUCCESS:
+		return ("Success");
+	case LIBUSB_ERROR_IO:
+		return ("I/O error");
+	case LIBUSB_ERROR_INVALID_PARAM:
+		return ("Invalid parameter");
+	case LIBUSB_ERROR_ACCESS:
+		return ("Permissions error");
+	case LIBUSB_ERROR_NO_DEVICE:
+		return ("No device");
+	case LIBUSB_ERROR_NOT_FOUND:
+		return ("Not found");
+	case LIBUSB_ERROR_BUSY:
+		return ("Device busy");
+	case LIBUSB_ERROR_TIMEOUT:
+		return ("Timeout");
+	case LIBUSB_ERROR_OVERFLOW:
+		return ("Overflow");
+	case LIBUSB_ERROR_PIPE:
+		return ("Pipe error");
+	case LIBUSB_ERROR_INTERRUPTED:
+		return ("Interrupted");
+	case LIBUSB_ERROR_NO_MEM:
+		return ("Out of memory");
+	case LIBUSB_ERROR_NOT_SUPPORTED:
+		return ("Not supported");
+	case LIBUSB_ERROR_OTHER:
+		return ("Other error");
+	default:
+		return ("Unknown error");
+	}
+}
+
+const char *
+libusb_error_name(int code)
+{
+	switch (code) {
+	case LIBUSB_SUCCESS:
+		return ("LIBUSB_SUCCESS");
+	case LIBUSB_ERROR_IO:
+		return ("LIBUSB_ERROR_IO");
+	case LIBUSB_ERROR_INVALID_PARAM:
+		return ("LIBUSB_ERROR_INVALID_PARAM");
+	case LIBUSB_ERROR_ACCESS:
+		return ("LIBUSB_ERROR_ACCESS");
+	case LIBUSB_ERROR_NO_DEVICE:
+		return ("LIBUSB_ERROR_NO_DEVICE");
+	case LIBUSB_ERROR_NOT_FOUND:
+		return ("LIBUSB_ERROR_NOT_FOUND");
+	case LIBUSB_ERROR_BUSY:
+		return ("LIBUSB_ERROR_BUSY");
+	case LIBUSB_ERROR_TIMEOUT:
+		return ("LIBUSB_ERROR_TIMEOUT");
+	case LIBUSB_ERROR_OVERFLOW:
+		return ("LIBUSB_ERROR_OVERFLOW");
+	case LIBUSB_ERROR_PIPE:
+		return ("LIBUSB_ERROR_PIPE");
+	case LIBUSB_ERROR_INTERRUPTED:
+		return ("LIBUSB_ERROR_INTERRUPTED");
+	case LIBUSB_ERROR_NO_MEM:
+		return ("LIBUSB_ERROR_NO_MEM");
+	case LIBUSB_ERROR_NOT_SUPPORTED:
+		return ("LIBUSB_ERROR_NOT_SUPPORTED");
+	case LIBUSB_ERROR_OTHER:
+		return ("LIBUSB_ERROR_OTHER");
+	default:
+		return ("LIBUSB_ERROR_UNKNOWN");
+	}
 }
diff --git a/lib/libusb/libusb20.3 b/lib/libusb/libusb20.3
index 042f8853657f..bd167b239d75 100644
--- a/lib/libusb/libusb20.3
+++ b/lib/libusb/libusb20.3
@@ -212,6 +212,10 @@ USB access library (libusb -lusb)
 .Fn libusb20_me_decode "const void *pdata" "uint16_t len" "void *pdecoded"
 .Ft "const uint8_t *"
 .Fn libusb20_desc_foreach "const struct libusb20_me_struct *me" "const uint8_t *pdesc"
+.Ft "const char *"
+.Fn libusb20_strerror "int code"
+.Ft "const char *"
+.Fn libusb20_error_name "int code"
 .
 .
 .Sh DESCRIPTION
@@ -610,10 +614,9 @@ The file descriptor can be used for polling purposes.
 .Pp
 .
 .Fn libusb20_dev_kernel_driver_active
-returns a non-zero value if a kernel driver is active on
-the given USB interface.
+returns zero if a kernel driver is active on the given USB interface.
 .
-Else zero is returned.
+Else a LIBUSB20_ERROR value is returned.
 .
 .Pp
 .
@@ -996,6 +999,22 @@ The total decoded length is returned.
 The buffer pointer cannot be NULL.
 .
 .
+.Sh USB DEBUGGING
+.Pp
+.Ft const char *
+.Fn libusb20_strerror "int code"
+Get the ASCII representation of the error given by the
+.Fa code
+argument.
+This function does not return NULL.
+.Pp
+.Ft const char *
+.Fn libusb20_error_name "int code"
+Get the ASCII representation of the error enum given by the
+.Fa code
+argument.
+This function does not return NULL.
+.
 .Sh FILES
 .
 .
@@ -1003,7 +1022,8 @@ The buffer pointer cannot be NULL.
 .Sh SEE ALSO
 .Xr usb 4 ,
 .Xr libusb 3 ,
-.Xr usbconfig 8
+.Xr usbconfig 8 ,
+.Xr usbdump 8
 .
 .
 .Sh HISTORY
diff --git a/lib/libusb/libusb20.c b/lib/libusb/libusb20.c
index b35e2bf46eb1..747c16003ce0 100644
--- a/lib/libusb/libusb20.c
+++ b/lib/libusb/libusb20.c
@@ -1244,3 +1244,77 @@ libusb20_be_dequeue_device(struct libusb20_backend *pbe,
 {
 	TAILQ_REMOVE(&(pbe->usb_devs), pdev, dev_entry);
 }
+
+const char *
+libusb20_strerror(int code)
+{
+	switch (code) {
+	case LIBUSB20_SUCCESS:
+		return ("Success");
+	case LIBUSB20_ERROR_IO:
+		return ("I/O error");
+	case LIBUSB20_ERROR_INVALID_PARAM:
+		return ("Invalid parameter");
+	case LIBUSB20_ERROR_ACCESS:
+		return ("Permissions error");
+	case LIBUSB20_ERROR_NO_DEVICE:
+		return ("No device");
+	case LIBUSB20_ERROR_NOT_FOUND:
+		return ("Not found");
+	case LIBUSB20_ERROR_BUSY:
+		return ("Device busy");
+	case LIBUSB20_ERROR_TIMEOUT:
+		return ("Timeout");
+	case LIBUSB20_ERROR_OVERFLOW:
+		return ("Overflow");
+	case LIBUSB20_ERROR_PIPE:
+		return ("Pipe error");
+	case LIBUSB20_ERROR_INTERRUPTED:
+		return ("Interrupted");
+	case LIBUSB20_ERROR_NO_MEM:
+		return ("Out of memory");
+	case LIBUSB20_ERROR_NOT_SUPPORTED:
+		return ("Not supported");
+	case LIBUSB20_ERROR_OTHER:
+		return ("Other error");
+	default:
+		return ("Unknown error");
+	}
+}
+
+const char *
+libusb20_error_name(int code)
+{
+	switch (code) {
+	case LIBUSB20_SUCCESS:
+		return ("LIBUSB20_SUCCESS");
+	case LIBUSB20_ERROR_IO:
+		return ("LIBUSB20_ERROR_IO");
+	case LIBUSB20_ERROR_INVALID_PARAM:
+		return ("LIBUSB20_ERROR_INVALID_PARAM");
+	case LIBUSB20_ERROR_ACCESS:
+		return ("LIBUSB20_ERROR_ACCESS");
+	case LIBUSB20_ERROR_NO_DEVICE:
+		return ("LIBUSB20_ERROR_NO_DEVICE");
+	case LIBUSB20_ERROR_NOT_FOUND:
+		return ("LIBUSB20_ERROR_NOT_FOUND");
+	case LIBUSB20_ERROR_BUSY:
+		return ("LIBUSB20_ERROR_BUSY");
+	case LIBUSB20_ERROR_TIMEOUT:
+		return ("LIBUSB20_ERROR_TIMEOUT");
+	case LIBUSB20_ERROR_OVERFLOW:
+		return ("LIBUSB20_ERROR_OVERFLOW");
+	case LIBUSB20_ERROR_PIPE:
+		return ("LIBUSB20_ERROR_PIPE");
+	case LIBUSB20_ERROR_INTERRUPTED:
+		return ("LIBUSB20_ERROR_INTERRUPTED");
+	case LIBUSB20_ERROR_NO_MEM:
+		return ("LIBUSB20_ERROR_NO_MEM");
+	case LIBUSB20_ERROR_NOT_SUPPORTED:
+		return ("LIBUSB20_ERROR_NOT_SUPPORTED");
+	case LIBUSB20_ERROR_OTHER:
+		return ("LIBUSB20_ERROR_OTHER");
+	default:
+		return ("LIBUSB20_ERROR_UNKNOWN");
+	}
+}
diff --git a/lib/libusb/libusb20.h b/lib/libusb/libusb20.h
index 22a2899bbd80..e4359fcb1549 100644
--- a/lib/libusb/libusb20.h
+++ b/lib/libusb/libusb20.h
@@ -293,6 +293,11 @@ void	libusb20_be_dequeue_device(struct libusb20_backend *pbe, struct libusb20_de
 void	libusb20_be_enqueue_device(struct libusb20_backend *pbe, struct libusb20_device *pdev);
 void	libusb20_be_free(struct libusb20_backend *pbe);
 
+/* USB debugging */
+
+const char *libusb20_strerror(int);
+const char *libusb20_error_name(int);
+
 #if 0
 {					/* style */
 #endif
diff --git a/lib/libusbhid/data.c b/lib/libusbhid/data.c
index 3b90ac6f9560..f607737522d7 100644
--- a/lib/libusbhid/data.c
+++ b/lib/libusbhid/data.c
@@ -32,7 +32,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <assert.h>
 #include <stdlib.h>
+#include <string.h>
+#include <dev/usb/usb_ioctl.h>
 #include "usbhid.h"
+#include "usbvar.h"
 
 int32_t
 hid_get_data(const void *p, const hid_item_t *h)
@@ -114,3 +117,27 @@ hid_set_data(void *p, const hid_item_t *h, int32_t data)
 		buf[offs + i] = (buf[offs + i] & (mask >> (i*8))) |
 		    ((data >> (i*8)) & 0xff);
 }
+
+int
+hid_get_report(int fd, enum hid_kind k, unsigned char *data, unsigned int size)
+{
+	struct usb_gen_descriptor ugd;
+
+	memset(&ugd, 0, sizeof(ugd));
+	ugd.ugd_data = hid_pass_ptr(data);
+	ugd.ugd_maxlen = size;
+	ugd.ugd_report_type = k + 1;
+	return (ioctl(fd, USB_GET_REPORT, &ugd));
+}
+
+int
+hid_set_report(int fd, enum hid_kind k, unsigned char *data, unsigned int size)
+{
+	struct usb_gen_descriptor ugd;
+
+	memset(&ugd, 0, sizeof(ugd));
+	ugd.ugd_data = hid_pass_ptr(data);
+	ugd.ugd_maxlen = size;
+	ugd.ugd_report_type = k + 1;
+	return (ioctl(fd, USB_SET_REPORT, &ugd));
+}
diff --git a/lib/libusbhid/parse.c b/lib/libusbhid/parse.c
index d2b49d40cc9b..f7c2cb195be1 100644
--- a/lib/libusbhid/parse.c
+++ b/lib/libusbhid/parse.c
@@ -43,10 +43,11 @@ __FBSDID("$FreeBSD$");
 #define	MAXUSAGE 100
 #define	MAXPUSH 4
 #define	MAXID 64
+#define	ITEMTYPES 3
 
 struct hid_pos_data {
 	int32_t rid;
-	uint32_t pos;
+	uint32_t pos[ITEMTYPES];
 };
 
 struct hid_data {
@@ -55,6 +56,7 @@ struct hid_data {
 	const uint8_t *p;
 	struct hid_item cur[MAXPUSH];
 	struct hid_pos_data last_pos[MAXID];
+	uint32_t pos[ITEMTYPES];
 	int32_t usages_min[MAXUSAGE];
 	int32_t usages_max[MAXUSAGE];
 	int32_t usage_last;	/* last seen usage */
@@ -92,7 +94,7 @@ hid_clear_local(hid_item_t *c)
 static void
 hid_switch_rid(struct hid_data *s, struct hid_item *c, int32_t next_rID)
 {
-	uint8_t i;
+	uint8_t i, j;
 
 	/* check for same report ID - optimise */
 
@@ -113,7 +115,8 @@ hid_switch_rid(struct hid_data *s, struct hid_item *c, int32_t next_rID)
 	}
 	if (i != MAXID) {
 		s->last_pos[i].rid = c->report_ID;
-		s->last_pos[i].pos = c->pos;
+		for (j = 0; j < ITEMTYPES; j++)
+			s->last_pos[i].pos[j] = s->pos[j];
 	}
 
 	/* store next report ID */
@@ -134,9 +137,12 @@ hid_switch_rid(struct hid_data *s, struct hid_item *c, int32_t next_rID)
 	}
 	if (i != MAXID) {
 		s->last_pos[i].rid = next_rID;
-		c->pos = s->last_pos[i].pos;
-	} else
-		c->pos = 0;	/* Out of RID entries. */
+		for (j = 0; j < ITEMTYPES; j++)
+			s->pos[j] = s->last_pos[i].pos[j];
+	} else {
+		for (j = 0; j < ITEMTYPES; j++)
+			s->pos[j] = 0;	/* Out of RID entries. */
+	}
 }
 
 /*------------------------------------------------------------------------*
@@ -206,7 +212,6 @@ hid_get_item(hid_data_t s, hid_item_t *h)
 {
 	hid_item_t *c;
 	unsigned int bTag, bType, bSize;
-	uint32_t oldpos;
 	int32_t mask;
 	int32_t dval;
 
@@ -240,7 +245,8 @@ hid_get_item(hid_data_t s, hid_item_t *h)
 		 */
 		if (s->kindset & (1 << c->kind)) {
 			*h = *c;
-			c->pos += c->report_size * c->report_count;
+			h->pos = s->pos[c->kind];
+			s->pos[c->kind] += c->report_size * c->report_count;
 			return (1);
 		}
 	}
@@ -406,14 +412,10 @@ hid_get_item(hid_data_t s, hid_item_t *h)
 			case 11:	/* Pop */
 				s->pushlevel --;
 				if (s->pushlevel < MAXPUSH) {
-					/* preserve position */
-					oldpos = c->pos;
 					c = &s->cur[s->pushlevel];
 					/* restore size and count */
 					s->loc_size = c->report_size;
 					s->loc_count = c->report_count;
-					/* set default item location */
-					c->pos = oldpos;
 					c->report_size = 0;
 					c->report_count = 0;
 				}
diff --git a/lib/libusbhid/usbhid.3 b/lib/libusbhid/usbhid.3
index b4951f2d11f2..c34a109d5865 100644
--- a/lib/libusbhid/usbhid.3
+++ b/lib/libusbhid/usbhid.3
@@ -44,7 +44,9 @@
 .Nm hid_usage_in_page ,
 .Nm hid_init ,
 .Nm hid_get_data ,
-.Nm hid_set_data
+.Nm hid_set_data ,
+.Nm hid_get_report ,
+.Nm hid_set_report
 .Nd USB HID access routines
 .Sh LIBRARY
 .Lb libusbhid
@@ -84,6 +86,10 @@
 .Fn hid_get_data "const void *data" "const hid_item_t *h"
 .Ft void
 .Fn hid_set_data "void *buf" "const hid_item_t *h" "int data"
+.Ft int
+.Fn hid_get_report "int fd" "enum hid_kind k" "unsigned char *data" "unsigned int size"
+.Ft int
+.Fn hid_set_report "int fd" "enum hid_kind k" "unsigned char *data" "unsigned int size"
 .Sh DESCRIPTION
 The
 .Nm
@@ -105,6 +111,14 @@ Synchronous HID operation can be enabled or disabled by a call to
 If the second argument is zero synchronous HID operation is disabled.
 Else synchronous HID operation is enabled.
 The function returns a negative value on failure.
+.Pp
+.Fn hid_get_report
+and
+.Fn hid_set_report
+functions allow to synchronously get and set specific report if device
+supports it.
+For devices with multiple report IDs, wanted ID should be provided in the
+first byte of the buffer for both get and set.
 .Ss Descriptor Functions
 The report descriptor ID can be obtained by calling
 .Fn hid_get_report_id .
diff --git a/lib/libusbhid/usbhid.h b/lib/libusbhid/usbhid.h
index 5e098eabad09..231ee1fe8a77 100644
--- a/lib/libusbhid/usbhid.h
+++ b/lib/libusbhid/usbhid.h
@@ -76,6 +76,7 @@ typedef struct hid_item {
 
 #define HID_PAGE(u) (((u) >> 16) & 0xffff)
 #define HID_USAGE(u) ((u) & 0xffff)
+#define HID_HAS_GET_SET_REPORT 1
 
 __BEGIN_DECLS
 
@@ -104,5 +105,9 @@ int hid_parse_usage_page(const char *name);
 /* Extracting/insertion of data, data.c: */
 int32_t hid_get_data(const void *p, const hid_item_t *h);
 void hid_set_data(void *p, const hid_item_t *h, int32_t data);
+int hid_get_report(int fd, enum hid_kind k,
+    unsigned char *data, unsigned int size);
+int hid_set_report(int fd, enum hid_kind k,
+    unsigned char *data, unsigned int size);
 
 __END_DECLS
diff --git a/lib/libutil/pidfile.3 b/lib/libutil/pidfile.3
index cc2b4bb639d4..c42b95b1f838 100644
--- a/lib/libutil/pidfile.3
+++ b/lib/libutil/pidfile.3
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 20, 2008
+.Dd October 16, 2011
 .Dt PIDFILE 3
 .Os
 .Sh NAME
@@ -59,11 +59,14 @@ The
 function opens (or creates) a file specified by the
 .Fa path
 argument and locks it.
-If a file can not be locked, a PID of an already running daemon is returned in
-the
+If
 .Fa pidptr
-argument (if it is not
-.Dv NULL ) .
+argument is not
+.Dv NULL
+and file can not be locked, the function will use it to store a PID of an
+already running daemon or
+.Li -1
+in case daemon did not write its PID yet.
 The function does not write process' PID into the file here, so it can be
 used before
 .Fn fork Ns ing
@@ -162,16 +165,18 @@ function will fail if:
 .It Bq Er EEXIST
 Some process already holds the lock on the given pidfile, meaning that a
 daemon is already running.
+If
+.Fa pidptr
+argument is not
+.Dv NULL
+the function will use it to store a PID of an already running daemon or
+.Li -1
+in case daemon did not write its PID yet.
 .It Bq Er ENAMETOOLONG
 Specified pidfile's name is too long.
 .It Bq Er EINVAL
 Some process already holds the lock on the given pidfile, but PID read
 from there is invalid.
-.It Bq Er EAGAIN
-Some process already holds the lock on the given pidfile, but the file
-is truncated.
-Most likely, the existing daemon is writing new PID into
-the file.
 .El
 .Pp
 The
diff --git a/lib/libutil/pidfile.c b/lib/libutil/pidfile.c
index 6b99936f7aa2..953d1e0438e6 100644
--- a/lib/libutil/pidfile.c
+++ b/lib/libutil/pidfile.c
@@ -119,20 +119,20 @@ pidfile_open(const char *path, mode_t mode, pid_t *pidptr)
 	fd = flopen(pfh->pf_path,
 	    O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK, mode);
 	if (fd == -1) {
-		count = 0;
-		rqtp.tv_sec = 0;
-		rqtp.tv_nsec = 5000000;
 		if (errno == EWOULDBLOCK && pidptr != NULL) {
-		again:
-			errno = pidfile_read(pfh->pf_path, pidptr);
-			if (errno == 0)
-				errno = EEXIST;
-			else if (errno == EAGAIN) {
-				if (++count <= 3) {
-					nanosleep(&rqtp, 0);
-					goto again;
-				}
+			count = 20;
+			rqtp.tv_sec = 0;
+			rqtp.tv_nsec = 5000000;
+			for (;;) {
+				errno = pidfile_read(pfh->pf_path, pidptr);
+				if (errno != EAGAIN || --count == 0)
+					break;
+				nanosleep(&rqtp, 0);
 			}
+			if (errno == EAGAIN)
+				*pidptr = -1;
+			if (errno == 0 || errno == EAGAIN)
+				errno = EEXIST;
 		}
 		free(pfh);
 		return (NULL);
diff --git a/lib/msun/Makefile b/lib/msun/Makefile
index bb99e45a25b2..f1385f13dc63 100644
--- a/lib/msun/Makefile
+++ b/lib/msun/Makefile
@@ -49,7 +49,7 @@ COMMON_SRCS= b_exp.c b_log.c b_tgamma.c \
 	e_pow.c e_powf.c e_rem_pio2.c \
 	e_rem_pio2f.c e_remainder.c e_remainderf.c e_scalb.c e_scalbf.c \
 	e_sinh.c e_sinhf.c e_sqrt.c e_sqrtf.c fenv.c \
-	k_cos.c k_cosf.c k_rem_pio2.c k_sin.c k_sinf.c \
+	k_cos.c k_cosf.c k_exp.c k_expf.c k_rem_pio2.c k_sin.c k_sinf.c \
 	k_tan.c k_tanf.c \
 	s_asinh.c s_asinhf.c s_atan.c s_atanf.c s_carg.c s_cargf.c s_cargl.c \
 	s_cbrt.c s_cbrtf.c s_ceil.c s_ceilf.c \
@@ -101,9 +101,11 @@ COMMON_SRCS+=	e_acosl.c e_asinl.c e_atan2l.c e_fmodl.c \
 .endif
 
 # C99 complex functions
-COMMON_SRCS+=	s_cexp.c s_cexpf.c s_cimag.c s_cimagf.c s_cimagl.c \
+COMMON_SRCS+=	s_ccosh.c s_ccoshf.c s_cexp.c s_cexpf.c \
+	s_cimag.c s_cimagf.c s_cimagl.c \
 	s_conj.c s_conjf.c s_conjl.c \
-	s_cproj.c s_cprojf.c s_creal.c s_crealf.c s_creall.c
+	s_cproj.c s_cprojf.c s_creal.c s_crealf.c s_creall.c \
+	s_csinh.c s_csinhf.c s_ctanh.c s_ctanhf.c
 
 # FreeBSD's C library supplies these functions:
 #COMMON_SRCS+=	s_fabs.c s_frexp.c s_isnan.c s_ldexp.c s_modf.c
@@ -125,7 +127,8 @@ SRCS=	${COMMON_SRCS} ${ARCH_SRCS}
 
 INCS=	fenv.h math.h
 
-MAN=	acos.3 acosh.3 asin.3 asinh.3 atan.3 atan2.3 atanh.3 ceil.3 cexp.3 \
+MAN=	acos.3 acosh.3 asin.3 asinh.3 atan.3 atan2.3 atanh.3 \
+	ceil.3 ccos.3 ccosh.3 cexp.3 \
 	cimag.3 copysign.3 cos.3 cosh.3 csqrt.3 erf.3 exp.3 fabs.3 fdim.3 \
 	feclearexcept.3 feenableexcept.3 fegetenv.3 \
 	fegetround.3 fenv.3 floor.3 \
@@ -143,6 +146,9 @@ MLINKS+=atan.3 atanf.3 atan.3 atanl.3
 MLINKS+=atanh.3 atanhf.3
 MLINKS+=atan2.3 atan2f.3 atan2.3 atan2l.3 \
 	atan2.3 carg.3 atan2.3 cargf.3 atan2.3 cargl.3
+MLINKS+=ccos.3 ccosf.3 ccos.3 csin.3 ccos.3 csinf.3 ccos.3 ctan.3 ccos.3 ctanf.3
+MLINKS+=ccosh.3 ccoshf.3 ccosh.3 csinh.3 ccosh.3 csinhf.3 \
+	ccosh.3 ctanh.3 ccosh.3 ctanhf.3
 MLINKS+=ceil.3 ceilf.3 ceil.3 ceill.3
 MLINKS+=cexp.3 cexpf.3
 MLINKS+=cimag.3 cimagf.3 cimag.3 cimagl.3 \
diff --git a/lib/msun/Symbol.map b/lib/msun/Symbol.map
index bb53ac39f9f4..f3d30003b33a 100644
--- a/lib/msun/Symbol.map
+++ b/lib/msun/Symbol.map
@@ -228,3 +228,25 @@ FBSD_1.2 {
 	log2;
 	log2f;
 };
+
+/* First added in 10.0-CURRENT */
+FBSD_1.3 {
+	feclearexcept;
+	fegetexceptflag;
+	fetestexcept;
+	fegetround;
+	fesetround;
+	fesetenv;
+	csin;
+	csinf;
+	csinh;
+	csinhf;
+	ccos;
+	ccosf;
+	ccosh;
+	ccoshf;
+	ctan;
+	ctanf;
+	ctanh;
+	ctanhf;
+};
diff --git a/lib/msun/amd64/Symbol.map b/lib/msun/amd64/Symbol.map
index f4c13a448322..3da32ffcc990 100644
--- a/lib/msun/amd64/Symbol.map
+++ b/lib/msun/amd64/Symbol.map
@@ -7,8 +7,6 @@ FBSD_1.0 {
 	fegetenv;
 	feholdexcept;
 	feupdateenv;
-	__feenableexcept;
-	__fedisableexcept;
 	feenableexcept;
 	fedisableexcept;
 };
diff --git a/lib/msun/amd64/fenv.c b/lib/msun/amd64/fenv.c
index e12607078f35..8b49b35c3eec 100644
--- a/lib/msun/amd64/fenv.c
+++ b/lib/msun/amd64/fenv.c
@@ -29,7 +29,13 @@
 #include <sys/cdefs.h>
 #include <sys/types.h>
 #include <machine/fpu.h>
-#include <fenv.h>
+
+#define	__fenv_static
+#include "fenv.h"
+
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
 
 const fenv_t __fe_dfl_env = {
 	{ 0xffff0000 | __INITIAL_FPUCW__,
@@ -41,6 +47,9 @@ const fenv_t __fe_dfl_env = {
 	__INITIAL_MXCSR__
 };
 
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+
 int
 fesetexceptflag(const fexcept_t *flagp, int excepts)
 {
@@ -69,6 +78,10 @@ feraiseexcept(int excepts)
 	return (0);
 }
 
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+
 int
 fegetenv(fenv_t *envp)
 {
@@ -98,6 +111,8 @@ feholdexcept(fenv_t *envp)
 	return (0);
 }
 
+extern inline int fesetenv(const fenv_t *__envp);
+
 int
 feupdateenv(const fenv_t *envp)
 {
@@ -120,12 +135,12 @@ __feenableexcept(int mask)
 	mask &= FE_ALL_EXCEPT;
 	__fnstcw(&control);
 	__stmxcsr(&mxcsr);
-	omask = (control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
+	omask = ~(control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
 	control &= ~mask;
 	__fldcw(control);
 	mxcsr &= ~(mask << _SSE_EMASK_SHIFT);
 	__ldmxcsr(mxcsr);
-	return (~omask);
+	return (omask);
 }
 
 int
@@ -137,12 +152,12 @@ __fedisableexcept(int mask)
 	mask &= FE_ALL_EXCEPT;
 	__fnstcw(&control);
 	__stmxcsr(&mxcsr);
-	omask = (control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
+	omask = ~(control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
 	control |= mask;
 	__fldcw(control);
 	mxcsr |= mask << _SSE_EMASK_SHIFT;
 	__ldmxcsr(mxcsr);
-	return (~omask);
+	return (omask);
 }
 
 __weak_reference(__feenableexcept, feenableexcept);
diff --git a/lib/msun/amd64/fenv.h b/lib/msun/amd64/fenv.h
index 875664affd23..b7c9873dd760 100644
--- a/lib/msun/amd64/fenv.h
+++ b/lib/msun/amd64/fenv.h
@@ -32,6 +32,10 @@
 #include <sys/cdefs.h>
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 typedef struct {
 	struct {
 		__uint32_t	__control;
@@ -89,7 +93,7 @@ extern const fenv_t	__fe_dfl_env;
 #define	__ldmxcsr(__csr)	__asm __volatile("ldmxcsr %0" : : "m" (__csr))
 #define	__stmxcsr(__csr)	__asm __volatile("stmxcsr %0" : "=m" (*(__csr)))
 
-static __inline int
+__fenv_static inline int
 feclearexcept(int __excepts)
 {
 	fenv_t __env;
@@ -107,7 +111,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	__uint32_t __mxcsr;
@@ -122,7 +126,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
 int feraiseexcept(int __excepts);
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	__uint32_t __mxcsr;
@@ -133,7 +137,7 @@ fetestexcept(int __excepts)
 	return ((__status | __mxcsr) & __excepts);
 }
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 	__uint16_t __control;
@@ -148,7 +152,7 @@ fegetround(void)
 	return (__control & _ROUND_MASK);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 	__uint32_t __mxcsr;
@@ -173,7 +177,7 @@ fesetround(int __round)
 int fegetenv(fenv_t *__envp);
 int feholdexcept(fenv_t *__envp);
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 
@@ -197,7 +201,8 @@ int feupdateenv(const fenv_t *__envp);
 int feenableexcept(int __mask);
 int fedisableexcept(int __mask);
 
-static __inline int
+/* We currently provide no external definition of fegetexcept(). */
+static inline int
 fegetexcept(void)
 {
 	__uint16_t __control;
diff --git a/lib/msun/arm/Symbol.map b/lib/msun/arm/Symbol.map
index 312117719b42..971112e30c8a 100644
--- a/lib/msun/arm/Symbol.map
+++ b/lib/msun/arm/Symbol.map
@@ -3,3 +3,11 @@
  */
 FBSD_1.0 {
 };
+
+FBSD_1.3 {
+	fesetexceptflag;
+	feraiseexcept;
+	fegetenv;
+	feholdexcept;
+	feupdateenv;
+};
diff --git a/lib/msun/arm/fenv.c b/lib/msun/arm/fenv.c
index a6781e0e715e..a5a5c03d3a74 100644
--- a/lib/msun/arm/fenv.c
+++ b/lib/msun/arm/fenv.c
@@ -26,10 +26,27 @@
  * $FreeBSD$
  */
 
-#include <fenv.h>
+#define	__fenv_static
+#include "fenv.h"
+
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
 
 /*
  * Hopefully the system ID byte is immutable, so it's valid to use
  * this as a default environment.
  */
 const fenv_t __fe_dfl_env = 0;
+
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+extern inline int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
+extern inline int feraiseexcept(int __excepts);
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+extern inline int fegetenv(fenv_t *__envp);
+extern inline int feholdexcept(fenv_t *__envp);
+extern inline int fesetenv(const fenv_t *__envp);
+extern inline int feupdateenv(const fenv_t *__envp);
diff --git a/lib/msun/arm/fenv.h b/lib/msun/arm/fenv.h
index d540ae2d5dbb..f11499806bea 100644
--- a/lib/msun/arm/fenv.h
+++ b/lib/msun/arm/fenv.h
@@ -31,6 +31,10 @@
 
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 typedef	__uint32_t	fenv_t;
 typedef	__uint32_t	fexcept_t;
 
@@ -68,7 +72,7 @@ extern const fenv_t	__fe_dfl_env;
 #define __wfs(__fpsr)
 #endif
 
-static __inline int
+__fenv_static inline int
 feclearexcept(int __excepts)
 {
 	fexcept_t __fpsr;
@@ -79,7 +83,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	fexcept_t __fpsr;
@@ -89,7 +93,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 {
 	fexcept_t __fpsr;
@@ -101,7 +105,7 @@ fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feraiseexcept(int __excepts)
 {
 	fexcept_t __ex = __excepts;
@@ -110,7 +114,7 @@ feraiseexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	fexcept_t __fpsr;
@@ -119,7 +123,7 @@ fetestexcept(int __excepts)
 	return (__fpsr & __excepts);
 }
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 
@@ -131,14 +135,14 @@ fegetround(void)
 	return (-1);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 
 	return (-1);
 }
 
-static __inline int
+__fenv_static inline int
 fegetenv(fenv_t *__envp)
 {
 
@@ -146,7 +150,7 @@ fegetenv(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feholdexcept(fenv_t *__envp)
 {
 	fenv_t __env;
@@ -158,7 +162,7 @@ feholdexcept(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 
@@ -166,7 +170,7 @@ fesetenv(const fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feupdateenv(const fenv_t *__envp)
 {
 	fexcept_t __fpsr;
@@ -179,7 +183,9 @@ feupdateenv(const fenv_t *__envp)
 
 #if __BSD_VISIBLE
 
-static __inline int
+/* We currently provide no external definitions of the functions below. */
+
+static inline int
 feenableexcept(int __mask)
 {
 	fenv_t __old_fpsr, __new_fpsr;
@@ -190,7 +196,7 @@ feenableexcept(int __mask)
 	return ((__old_fpsr >> _FPUSW_SHIFT) & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fedisableexcept(int __mask)
 {
 	fenv_t __old_fpsr, __new_fpsr;
@@ -201,7 +207,7 @@ fedisableexcept(int __mask)
 	return ((__old_fpsr >> _FPUSW_SHIFT) & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fegetexcept(void)
 {
 	fenv_t __fpsr;
diff --git a/lib/msun/bsdsrc/b_exp.c b/lib/msun/bsdsrc/b_exp.c
index 6af1fe119216..4400992795b6 100644
--- a/lib/msun/bsdsrc/b_exp.c
+++ b/lib/msun/bsdsrc/b_exp.c
@@ -74,16 +74,16 @@ __FBSDID("$FreeBSD$");
 
 #include "mathimpl.h"
 
-const static double p1 = 0x1.555555555553ep-3;
-const static double p2 = -0x1.6c16c16bebd93p-9;
-const static double p3 = 0x1.1566aaf25de2cp-14;
-const static double p4 = -0x1.bbd41c5d26bf1p-20;
-const static double p5 = 0x1.6376972bea4d0p-25;
-const static double ln2hi = 0x1.62e42fee00000p-1;
-const static double ln2lo = 0x1.a39ef35793c76p-33;
-const static double lnhuge = 0x1.6602b15b7ecf2p9;
-const static double lntiny = -0x1.77af8ebeae354p9;
-const static double invln2 = 0x1.71547652b82fep0;
+static const double p1 = 0x1.555555555553ep-3;
+static const double p2 = -0x1.6c16c16bebd93p-9;
+static const double p3 = 0x1.1566aaf25de2cp-14;
+static const double p4 = -0x1.bbd41c5d26bf1p-20;
+static const double p5 = 0x1.6376972bea4d0p-25;
+static const double ln2hi = 0x1.62e42fee00000p-1;
+static const double ln2lo = 0x1.a39ef35793c76p-33;
+static const double lnhuge = 0x1.6602b15b7ecf2p9;
+static const double lntiny = -0x1.77af8ebeae354p9;
+static const double invln2 = 0x1.71547652b82fep0;
 
 #if 0
 double exp(x)
diff --git a/lib/msun/i387/Symbol.map b/lib/msun/i387/Symbol.map
index aa8169010705..35e6e18d0259 100644
--- a/lib/msun/i387/Symbol.map
+++ b/lib/msun/i387/Symbol.map
@@ -9,8 +9,6 @@ FBSD_1.0 {
 	fegetenv;
 	feholdexcept;
 	feupdateenv;
-	__feenableexcept;
-	__fedisableexcept;
 	feenableexcept;
 	fedisableexcept;
 };
diff --git a/lib/msun/i387/fenv.c b/lib/msun/i387/fenv.c
index 5c996b36bf6f..6f51ba24a0f7 100644
--- a/lib/msun/i387/fenv.c
+++ b/lib/msun/i387/fenv.c
@@ -29,8 +29,14 @@
 #include <sys/cdefs.h>
 #include <sys/types.h>
 #include <machine/npx.h>
+
+#define	__fenv_static
 #include "fenv.h"
 
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
+
 const fenv_t __fe_dfl_env = {
 	__INITIAL_NPXCW__,
 	0x0000,
@@ -83,6 +89,9 @@ __test_sse(void)
 	return (0);
 }
 
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+
 int
 fesetexceptflag(const fexcept_t *flagp, int excepts)
 {
@@ -114,6 +123,10 @@ feraiseexcept(int excepts)
 	return (0);
 }
 
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+
 int
 fegetenv(fenv_t *envp)
 {
@@ -149,6 +162,8 @@ feholdexcept(fenv_t *envp)
 	return (0);
 }
 
+extern inline int fesetenv(const fenv_t *__envp);
+
 int
 feupdateenv(const fenv_t *envp)
 {
@@ -177,14 +192,14 @@ __feenableexcept(int mask)
 		__stmxcsr(&mxcsr);
 	else
 		mxcsr = 0;
-	omask = (control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
+	omask = ~(control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
 	control &= ~mask;
 	__fldcw(control);
 	if (__HAS_SSE()) {
 		mxcsr &= ~(mask << _SSE_EMASK_SHIFT);
 		__ldmxcsr(mxcsr);
 	}
-	return (~omask);
+	return (omask);
 }
 
 int
@@ -199,14 +214,14 @@ __fedisableexcept(int mask)
 		__stmxcsr(&mxcsr);
 	else
 		mxcsr = 0;
-	omask = (control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
+	omask = ~(control | mxcsr >> _SSE_EMASK_SHIFT) & FE_ALL_EXCEPT;
 	control |= mask;
 	__fldcw(control);
 	if (__HAS_SSE()) {
 		mxcsr |= mask << _SSE_EMASK_SHIFT;
 		__ldmxcsr(mxcsr);
 	}
-	return (~omask);
+	return (omask);
 }
 
 __weak_reference(__feenableexcept, feenableexcept);
diff --git a/lib/msun/i387/fenv.h b/lib/msun/i387/fenv.h
index cc58a92afad0..329dfe1021cb 100644
--- a/lib/msun/i387/fenv.h
+++ b/lib/msun/i387/fenv.h
@@ -32,6 +32,10 @@
 #include <sys/cdefs.h>
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 /*                   
  * To preserve binary compatibility with FreeBSD 5.3, we pack the
  * mxcsr into some reserved fields, rather than changing sizeof(fenv_t).
@@ -110,7 +114,7 @@ extern const fenv_t	__fe_dfl_env;
 #define	__ldmxcsr(__csr)	__asm __volatile("ldmxcsr %0" : : "m" (__csr))
 #define	__stmxcsr(__csr)	__asm __volatile("stmxcsr %0" : "=m" (*(__csr)))
 
-static __inline int
+__fenv_static inline int
 feclearexcept(int __excepts)
 {
 	fenv_t __env;
@@ -131,7 +135,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	__uint32_t __mxcsr;
@@ -149,7 +153,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
 int feraiseexcept(int __excepts);
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	__uint32_t __mxcsr;
@@ -163,7 +167,7 @@ fetestexcept(int __excepts)
 	return ((__status | __mxcsr) & __excepts);
 }
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 	__uint16_t __control;
@@ -178,7 +182,7 @@ fegetround(void)
 	return (__control & _ROUND_MASK);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 	__uint32_t __mxcsr;
@@ -205,7 +209,7 @@ fesetround(int __round)
 int fegetenv(fenv_t *__envp);
 int feholdexcept(fenv_t *__envp);
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 	fenv_t __env = *__envp;
@@ -234,7 +238,8 @@ int feupdateenv(const fenv_t *__envp);
 int feenableexcept(int __mask);
 int fedisableexcept(int __mask);
 
-static __inline int
+/* We currently provide no external definition of fegetexcept(). */
+static inline int
 fegetexcept(void)
 {
 	__uint16_t __control;
diff --git a/lib/msun/ia64/Symbol.map b/lib/msun/ia64/Symbol.map
index 00ed4eef4223..c92e75d6e011 100644
--- a/lib/msun/ia64/Symbol.map
+++ b/lib/msun/ia64/Symbol.map
@@ -4,3 +4,10 @@
 FBSD_1.0 {
 	feupdateenv;
 };
+
+FBSD_1.3 {
+	fesetexceptflag;
+	feraiseexcept;
+	fegetenv;
+	feholdexcept;
+};
diff --git a/lib/msun/ia64/fenv.c b/lib/msun/ia64/fenv.c
index 95a580998e57..09d2d4e92529 100644
--- a/lib/msun/ia64/fenv.c
+++ b/lib/msun/ia64/fenv.c
@@ -27,10 +27,27 @@
  */
 
 #include <sys/types.h>
-#include <fenv.h>
+
+#define	__fenv_static
+#include "fenv.h"
+
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
 
 const fenv_t __fe_dfl_env = 0x0009804c8a70033fULL;
 
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+extern inline int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
+extern inline int feraiseexcept(int __excepts);
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+extern inline int fegetenv(fenv_t *__envp);
+extern inline int feholdexcept(fenv_t *__envp);
+extern inline int fesetenv(const fenv_t *__envp);
+
 /*
  * It doesn't pay to inline feupdateenv() because it includes one of
  * the rare uses of feraiseexcept() where the argument is not a
diff --git a/lib/msun/ia64/fenv.h b/lib/msun/ia64/fenv.h
index 6f0240f7b226..0f001db03183 100644
--- a/lib/msun/ia64/fenv.h
+++ b/lib/msun/ia64/fenv.h
@@ -31,6 +31,10 @@
 
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 typedef	__uint64_t	fenv_t;
 typedef	__uint16_t	fexcept_t;
 
@@ -63,7 +67,7 @@ extern const fenv_t	__fe_dfl_env;
 #define	__stfpsr(__r)	__asm __volatile("mov %0=ar.fpsr" : "=r" (*(__r)))
 #define	__ldfpsr(__r)	__asm __volatile("mov ar.fpsr=%0;;" : : "r" (__r))
 
-static __inline int
+__fenv_static inline int
 feclearexcept(int __excepts)
 {
 	fenv_t __fpsr;
@@ -74,7 +78,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	fenv_t __fpsr;
@@ -84,7 +88,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 {
 	fenv_t __fpsr;
@@ -103,7 +107,7 @@ fesetexceptflag(const fexcept_t *__flagp, int __excepts)
  * figure this out automatically, and there's no way to tell it.
  * We assume that constant arguments will be the common case.
  */
-static __inline int
+__fenv_static inline int
 feraiseexcept(int __excepts)
 {
 	volatile double d;
@@ -138,7 +142,7 @@ feraiseexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	fenv_t __fpsr;
@@ -148,7 +152,7 @@ fetestexcept(int __excepts)
 }
 
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 	fenv_t __fpsr;
@@ -157,7 +161,7 @@ fegetround(void)
 	return (__fpsr & _ROUND_MASK);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 	fenv_t __fpsr;
@@ -171,7 +175,7 @@ fesetround(int __round)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetenv(fenv_t *__envp)
 {
 
@@ -179,7 +183,7 @@ fegetenv(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feholdexcept(fenv_t *__envp)
 {
 	fenv_t __fpsr;
@@ -192,7 +196,7 @@ feholdexcept(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 
@@ -204,7 +208,9 @@ int feupdateenv(const fenv_t *__envp);
 
 #if __BSD_VISIBLE
 
-static __inline int
+/* We currently provide no external definitions of the functions below. */
+
+static inline int
 feenableexcept(int __mask)
 {
 	fenv_t __newfpsr, __oldfpsr;
@@ -215,7 +221,7 @@ feenableexcept(int __mask)
 	return (~__oldfpsr & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fedisableexcept(int __mask)
 {
 	fenv_t __newfpsr, __oldfpsr;
@@ -226,7 +232,7 @@ fedisableexcept(int __mask)
 	return (~__oldfpsr & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fegetexcept(void)
 {
 	fenv_t __fpsr;
diff --git a/lib/msun/man/ccos.3 b/lib/msun/man/ccos.3
new file mode 100644
index 000000000000..cf708c15f20f
--- /dev/null
+++ b/lib/msun/man/ccos.3
@@ -0,0 +1,80 @@
+.\" Copyright (c) 2011 David Schultz <das@FreeBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 17, 2011
+.Dt CCOS 3
+.Os
+.Sh NAME
+.Nm ccos ,
+.Nm ccosf ,
+.Nm csin ,
+.Nm csinf
+.Nm ctan ,
+.Nm ctanf
+.Nd complex trigonometric functions
+.Sh LIBRARY
+.Lb libm
+.Sh SYNOPSIS
+.In complex.h
+.Ft double complex
+.Fn ccos "double complex z"
+.Ft float complex
+.Fn ccosf "float complex z"
+.Ft double complex
+.Fn csin "double complex z"
+.Ft float complex
+.Fn csinf "float complex z"
+.Ft double complex
+.Fn ctan "double complex z"
+.Ft float complex
+.Fn ctanf "float complex z"
+.Sh DESCRIPTION
+The
+.Fn ccos ,
+.Fn csin ,
+and
+.Fn ctan
+functions compute the cosine, sine, and tangent of the complex number
+.Fa z ,
+respectively.
+The
+.Fn ccosf ,
+.Fn csinf ,
+and
+.Fn ctanf
+functions perform the same operations in
+.Fa float
+precision.
+.Sh SEE ALSO
+.Xr ccosh 3 ,
+.Xr complex 3 ,
+.Xr cos 3 ,
+.Xr math 3 ,
+.Xr sin 3 ,
+.Xr tan 3
+.Sh STANDARDS
+These functions conform to
+.St -isoC-99 .
diff --git a/lib/msun/man/ccosh.3 b/lib/msun/man/ccosh.3
new file mode 100644
index 000000000000..01688b5bdb8c
--- /dev/null
+++ b/lib/msun/man/ccosh.3
@@ -0,0 +1,80 @@
+.\" Copyright (c) 2011 David Schultz <das@FreeBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 17, 2011
+.Dt CCOSH 3
+.Os
+.Sh NAME
+.Nm ccosh ,
+.Nm ccoshf ,
+.Nm csinh ,
+.Nm csinhf
+.Nm ctanh ,
+.Nm ctanhf
+.Nd complex hyperbolic functions
+.Sh LIBRARY
+.Lb libm
+.Sh SYNOPSIS
+.In complex.h
+.Ft double complex
+.Fn ccosh "double complex z"
+.Ft float complex
+.Fn ccoshf "float complex z"
+.Ft double complex
+.Fn csinh "double complex z"
+.Ft float complex
+.Fn csinhf "float complex z"
+.Ft double complex
+.Fn ctanh "double complex z"
+.Ft float complex
+.Fn ctanhf "float complex z"
+.Sh DESCRIPTION
+The
+.Fn ccosh ,
+.Fn csinh ,
+and
+.Fn ctanh
+functions compute the hyperbolic cosine, sine, and tangent of the complex number
+.Fa z ,
+respectively.
+The
+.Fn ccoshf ,
+.Fn csinhf ,
+and
+.Fn ctanhf
+functions perform the same operations in
+.Fa float
+precision.
+.Sh SEE ALSO
+.Xr ccos 3 ,
+.Xr complex 3 ,
+.Xr cosh 3 ,
+.Xr math 3 ,
+.Xr sinh 3 ,
+.Xr tanh 3
+.Sh STANDARDS
+These functions conform to
+.St -isoC-99 .
diff --git a/lib/msun/man/complex.3 b/lib/msun/man/complex.3
index 3f7c18c6f77e..a8d9f950e4e5 100644
--- a/lib/msun/man/complex.3
+++ b/lib/msun/man/complex.3
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 7, 2011
+.Dd October 17, 2011
 .Dt COMPLEX 3
 .Os
 .Sh NAME
@@ -86,6 +86,16 @@ conj	compute the complex conjugate
 cproj	compute projection onto Riemann sphere
 creal	compute the real part
 .El
+.\" Section 7.3.5-6 of ISO C99 standard
+.Ss Trigonometric and Hyperbolic Functions
+.Cl
+ccos	cosine
+ccosh	hyperbolic cosine
+csin	sine
+csinh	hyperbolic sine
+ctan	tangent
+ctanh	hyperbolic tangent
+.El
 .Sh SEE ALSO
 .Xr math 3 ,
 .Xr fenv 3 ,
@@ -101,20 +111,14 @@ The
 functions described here conform to
 .St -isoC-99 .
 .Sh BUGS
-The trigonmetric and hyperbolic functions
+The inverse trigonmetric and hyperbolic functions
 .Fn cacos ,
 .Fn cacosh ,
 .Fn casin ,
 .Fn casinh ,
 .Fn catan ,
-.Fn catanh ,
-.Fn ccos ,
-.Fn ccosh ,
-.Fn csin ,
-.Fn csinh ,
-.Fn ctan ,
 and
-.Fn ctanh
+.Fn catanh
 are not implemented.
 .Pp
 The logarithmic functions
diff --git a/lib/msun/man/cos.3 b/lib/msun/man/cos.3
index d0cdf68bf4b4..a430f81e165c 100644
--- a/lib/msun/man/cos.3
+++ b/lib/msun/man/cos.3
@@ -71,6 +71,7 @@ functions return the cosine value.
 .Xr asin 3 ,
 .Xr atan 3 ,
 .Xr atan2 3 ,
+.Xr ccos 3 ,
 .Xr cosh 3 ,
 .Xr math 3 ,
 .Xr sin 3 ,
diff --git a/lib/msun/man/cosh.3 b/lib/msun/man/cosh.3
index 517ec61395dc..96abc5aa7c9f 100644
--- a/lib/msun/man/cosh.3
+++ b/lib/msun/man/cosh.3
@@ -55,6 +55,7 @@ functions compute the hyperbolic cosine of
 .Xr asin 3 ,
 .Xr atan 3 ,
 .Xr atan2 3 ,
+.Xr ccosh 3 ,
 .Xr cos 3 ,
 .Xr math 3 ,
 .Xr sin 3 ,
diff --git a/lib/msun/man/fenv.3 b/lib/msun/man/fenv.3
index aaea5f584086..6fee216ffbfc 100644
--- a/lib/msun/man/fenv.3
+++ b/lib/msun/man/fenv.3
@@ -91,24 +91,35 @@ The following macros expand to bit flags of type
 representing the five standard floating-point exceptions.
 .Bl -tag -width ".Dv FE_DIVBYZERO"
 .It Dv FE_DIVBYZERO
-A divide-by-zero exception occurs when the program attempts to
-divide a finite non-zero number by zero.
+A divide-by-zero exception occurs when the
+.Em exact
+result of a computation is infinite (according to the limit definition).
+For example, dividing a finite non-zero number by zero or computing
+.Fn log 0
+raises a divide-by-zero exception.
 .It Dv FE_INEXACT
-An inexact exception is raised whenever there is a loss of precision
+An inexact exception is raised whenever there is a loss of accuracy
 due to rounding.
 .It Dv FE_INVALID
 Invalid operation exceptions occur when a program attempts to
 perform calculations for which there is no reasonable representable
 answer.
-For instance, subtraction of infinities, division of zero by zero,
-ordered comparison involving \*(Nas, and taking the square root of a
+For instance, subtraction of like-signed infinities, division of zero by zero,
+ordered comparison involving \*(Nas, and taking the real square root of a
 negative number are all invalid operations.
 .It Dv FE_OVERFLOW
-An overflow exception occurs when the magnitude of the result of a
-computation is too large to fit in the destination type.
+In contrast with divide-by-zero,
+an overflow exception occurs when an infinity is produced because
+the magnitude of the exact result is
+.Em finite
+but too large to fit in the destination type.
+For example, computing
+.Li DBL_MAX * 2
+raises an overflow exception.
 .It Dv FE_UNDERFLOW
-Underflow occurs when the result of a computation is too close to zero
-to be represented as a non-zero value in the destination type.
+Underflow occurs when the result of a computation loses precision
+because it is too close to zero.
+The result is a subnormal number or zero.
 .El
 .Pp
 Additionally, the
@@ -183,9 +194,9 @@ as usual, but no
 .Dv SIGFPE
 signals will be generated as a result.
 Non-stop mode is the default, but it may be altered by
-non-standard mechanisms.
-.\" XXX Mention fe[gs]etmask() here after the interface is finalized
-.\" XXX and ready to be officially documented.
+.Fn feenableexcept
+and
+.Fn fedisableexcept .
 The
 .Fn feupdateenv
 function restores a saved environment similarly to
diff --git a/lib/msun/man/ieee.3 b/lib/msun/man/ieee.3
index c99a6282abde..d25a98f1735d 100644
--- a/lib/msun/man/ieee.3
+++ b/lib/msun/man/ieee.3
@@ -183,7 +183,7 @@ Type name:
 .Bd -ragged -offset indent -compact
 On some architectures,
 .Vt long double
-is the the same as
+is the same as
 .Vt double .
 .Ed
 .Pp
diff --git a/lib/msun/man/math.3 b/lib/msun/man/math.3
index 9ffe080454c8..f6b1289a0644 100644
--- a/lib/msun/man/math.3
+++ b/lib/msun/man/math.3
@@ -39,7 +39,21 @@
 .Sh SYNOPSIS
 .In math.h
 .Sh DESCRIPTION
-These functions constitute the C math library.
+The math library includes the following components:
+.Bl -column "<complex.h>" "polymorphic (type-generic) versions of functions" -compact -offset indent
+.In math.h Ta basic routines and real-valued functions
+.In complex.h Ta complex number support
+.In tgmath.h Ta polymorphic (type-generic) versions of functions
+.In fenv.h Ta routines to control rounding and exceptions
+.El
+The rest of this manual page describes the functions provided by
+.In math.h .
+Please consult
+.Xr complex 3 ,
+.Xr tgmath 3 ,
+and
+.Xr fenv 3
+for information on the other components.
 .Sh "LIST OF FUNCTIONS"
 Each of the following
 .Vt double
@@ -200,6 +214,7 @@ so reproducible results cannot be guaranteed across platforms.
 For most of these functions, however, incorrect rounding occurs
 rarely, and then only in very-close-to-halfway cases.
 .Sh SEE ALSO
+.Xr complex 3 ,
 .Xr fenv 3 ,
 .Xr ieee 3 ,
 .Xr tgmath 3
diff --git a/lib/msun/man/sin.3 b/lib/msun/man/sin.3
index 2a34281d150e..c7daf094dd08 100644
--- a/lib/msun/man/sin.3
+++ b/lib/msun/man/sin.3
@@ -70,6 +70,7 @@ functions return the sine value.
 .Xr asin 3 ,
 .Xr atan 3 ,
 .Xr atan2 3 ,
+.Xr csin 3 ,
 .Xr cos 3 ,
 .Xr cosh 3 ,
 .Xr math 3 ,
diff --git a/lib/msun/man/sinh.3 b/lib/msun/man/sinh.3
index a2a2d8ce4439..02944ccc3a6a 100644
--- a/lib/msun/man/sinh.3
+++ b/lib/msun/man/sinh.3
@@ -56,6 +56,7 @@ functions compute the hyperbolic sine of
 .Xr atan2 3 ,
 .Xr cos 3 ,
 .Xr cosh 3 ,
+.Xr csinh 3 ,
 .Xr math 3 ,
 .Xr sin 3 ,
 .Xr tan 3 ,
diff --git a/lib/msun/man/tan.3 b/lib/msun/man/tan.3
index 1577815ad961..f73c61208fa4 100644
--- a/lib/msun/man/tan.3
+++ b/lib/msun/man/tan.3
@@ -73,6 +73,7 @@ functions return the tangent value.
 .Xr atan2 3 ,
 .Xr cos 3 ,
 .Xr cosh 3 ,
+.Xr ctan 3 ,
 .Xr math 3 ,
 .Xr sin 3 ,
 .Xr sinh 3 ,
diff --git a/lib/msun/man/tanh.3 b/lib/msun/man/tanh.3
index 699abe5d94dc..6fb185c418b6 100644
--- a/lib/msun/man/tanh.3
+++ b/lib/msun/man/tanh.3
@@ -65,6 +65,7 @@ functions return the hyperbolic tangent value.
 .Xr atan2 3 ,
 .Xr cos 3 ,
 .Xr cosh 3 ,
+.Xr ctanh 3 ,
 .Xr math 3 ,
 .Xr sin 3 ,
 .Xr sinh 3 ,
diff --git a/lib/msun/mips/Symbol.map b/lib/msun/mips/Symbol.map
index 312117719b42..971112e30c8a 100644
--- a/lib/msun/mips/Symbol.map
+++ b/lib/msun/mips/Symbol.map
@@ -3,3 +3,11 @@
  */
 FBSD_1.0 {
 };
+
+FBSD_1.3 {
+	fesetexceptflag;
+	feraiseexcept;
+	fegetenv;
+	feholdexcept;
+	feupdateenv;
+};
diff --git a/lib/msun/mips/fenv.c b/lib/msun/mips/fenv.c
index a6781e0e715e..a5a5c03d3a74 100644
--- a/lib/msun/mips/fenv.c
+++ b/lib/msun/mips/fenv.c
@@ -26,10 +26,27 @@
  * $FreeBSD$
  */
 
-#include <fenv.h>
+#define	__fenv_static
+#include "fenv.h"
+
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
 
 /*
  * Hopefully the system ID byte is immutable, so it's valid to use
  * this as a default environment.
  */
 const fenv_t __fe_dfl_env = 0;
+
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+extern inline int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
+extern inline int feraiseexcept(int __excepts);
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+extern inline int fegetenv(fenv_t *__envp);
+extern inline int feholdexcept(fenv_t *__envp);
+extern inline int fesetenv(const fenv_t *__envp);
+extern inline int feupdateenv(const fenv_t *__envp);
diff --git a/lib/msun/mips/fenv.h b/lib/msun/mips/fenv.h
index d540ae2d5dbb..f11499806bea 100644
--- a/lib/msun/mips/fenv.h
+++ b/lib/msun/mips/fenv.h
@@ -31,6 +31,10 @@
 
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 typedef	__uint32_t	fenv_t;
 typedef	__uint32_t	fexcept_t;
 
@@ -68,7 +72,7 @@ extern const fenv_t	__fe_dfl_env;
 #define __wfs(__fpsr)
 #endif
 
-static __inline int
+__fenv_static inline int
 feclearexcept(int __excepts)
 {
 	fexcept_t __fpsr;
@@ -79,7 +83,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	fexcept_t __fpsr;
@@ -89,7 +93,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 {
 	fexcept_t __fpsr;
@@ -101,7 +105,7 @@ fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feraiseexcept(int __excepts)
 {
 	fexcept_t __ex = __excepts;
@@ -110,7 +114,7 @@ feraiseexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	fexcept_t __fpsr;
@@ -119,7 +123,7 @@ fetestexcept(int __excepts)
 	return (__fpsr & __excepts);
 }
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 
@@ -131,14 +135,14 @@ fegetround(void)
 	return (-1);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 
 	return (-1);
 }
 
-static __inline int
+__fenv_static inline int
 fegetenv(fenv_t *__envp)
 {
 
@@ -146,7 +150,7 @@ fegetenv(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feholdexcept(fenv_t *__envp)
 {
 	fenv_t __env;
@@ -158,7 +162,7 @@ feholdexcept(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 
@@ -166,7 +170,7 @@ fesetenv(const fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feupdateenv(const fenv_t *__envp)
 {
 	fexcept_t __fpsr;
@@ -179,7 +183,9 @@ feupdateenv(const fenv_t *__envp)
 
 #if __BSD_VISIBLE
 
-static __inline int
+/* We currently provide no external definitions of the functions below. */
+
+static inline int
 feenableexcept(int __mask)
 {
 	fenv_t __old_fpsr, __new_fpsr;
@@ -190,7 +196,7 @@ feenableexcept(int __mask)
 	return ((__old_fpsr >> _FPUSW_SHIFT) & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fedisableexcept(int __mask)
 {
 	fenv_t __old_fpsr, __new_fpsr;
@@ -201,7 +207,7 @@ fedisableexcept(int __mask)
 	return ((__old_fpsr >> _FPUSW_SHIFT) & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fegetexcept(void)
 {
 	fenv_t __fpsr;
diff --git a/lib/msun/powerpc/Symbol.map b/lib/msun/powerpc/Symbol.map
index 312117719b42..971112e30c8a 100644
--- a/lib/msun/powerpc/Symbol.map
+++ b/lib/msun/powerpc/Symbol.map
@@ -3,3 +3,11 @@
  */
 FBSD_1.0 {
 };
+
+FBSD_1.3 {
+	fesetexceptflag;
+	feraiseexcept;
+	fegetenv;
+	feholdexcept;
+	feupdateenv;
+};
diff --git a/lib/msun/powerpc/fenv.c b/lib/msun/powerpc/fenv.c
index 04bbeafc5892..3f4801bbd81b 100644
--- a/lib/msun/powerpc/fenv.c
+++ b/lib/msun/powerpc/fenv.c
@@ -26,6 +26,23 @@
  * $FreeBSD$
  */
 
-#include <fenv.h>
+#define	__fenv_static
+#include "fenv.h"
+
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
 
 const fenv_t __fe_dfl_env = 0x00000000;
+
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+extern inline int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
+extern inline int feraiseexcept(int __excepts);
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+extern inline int fegetenv(fenv_t *__envp);
+extern inline int feholdexcept(fenv_t *__envp);
+extern inline int fesetenv(const fenv_t *__envp);
+extern inline int feupdateenv(const fenv_t *__envp);
diff --git a/lib/msun/powerpc/fenv.h b/lib/msun/powerpc/fenv.h
index c29f8a087009..b96fdeebf4db 100644
--- a/lib/msun/powerpc/fenv.h
+++ b/lib/msun/powerpc/fenv.h
@@ -31,6 +31,10 @@
 
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 typedef	__uint32_t	fenv_t;
 typedef	__uint32_t	fexcept_t;
 
@@ -98,7 +102,7 @@ union __fpscr {
 	} __bits;
 };
 
-static __inline int
+__fenv_static inline int
 feclearexcept(int __excepts)
 {
 	union __fpscr __r;
@@ -111,7 +115,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	union __fpscr __r;
@@ -121,7 +125,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 {
 	union __fpscr __r;
@@ -135,7 +139,7 @@ fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feraiseexcept(int __excepts)
 {
 	union __fpscr __r;
@@ -148,7 +152,7 @@ feraiseexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	union __fpscr __r;
@@ -157,7 +161,7 @@ fetestexcept(int __excepts)
 	return (__r.__bits.__reg & __excepts);
 }
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 	union __fpscr __r;
@@ -166,7 +170,7 @@ fegetround(void)
 	return (__r.__bits.__reg & _ROUND_MASK);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 	union __fpscr __r;
@@ -180,7 +184,7 @@ fesetround(int __round)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetenv(fenv_t *__envp)
 {
 	union __fpscr __r;
@@ -190,7 +194,7 @@ fegetenv(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feholdexcept(fenv_t *__envp)
 {
 	union __fpscr __r;
@@ -202,7 +206,7 @@ feholdexcept(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 	union __fpscr __r;
@@ -212,7 +216,7 @@ fesetenv(const fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feupdateenv(const fenv_t *__envp)
 {
 	union __fpscr __r;
@@ -226,7 +230,9 @@ feupdateenv(const fenv_t *__envp)
 
 #if __BSD_VISIBLE
 
-static __inline int
+/* We currently provide no external definitions of the functions below. */
+
+static inline int
 feenableexcept(int __mask)
 {
 	union __fpscr __r;
@@ -239,7 +245,7 @@ feenableexcept(int __mask)
 	return ((__oldmask & _ENABLE_MASK) << _FPUSW_SHIFT);
 }
 
-static __inline int
+static inline int
 fedisableexcept(int __mask)
 {
 	union __fpscr __r;
@@ -252,7 +258,7 @@ fedisableexcept(int __mask)
 	return ((__oldmask & _ENABLE_MASK) << _FPUSW_SHIFT);
 }
 
-static __inline int
+static inline int
 fegetexcept(void)
 {
 	union __fpscr __r;
diff --git a/lib/msun/sparc64/Symbol.map b/lib/msun/sparc64/Symbol.map
index 312117719b42..971112e30c8a 100644
--- a/lib/msun/sparc64/Symbol.map
+++ b/lib/msun/sparc64/Symbol.map
@@ -3,3 +3,11 @@
  */
 FBSD_1.0 {
 };
+
+FBSD_1.3 {
+	fesetexceptflag;
+	feraiseexcept;
+	fegetenv;
+	feholdexcept;
+	feupdateenv;
+};
diff --git a/lib/msun/sparc64/fenv.c b/lib/msun/sparc64/fenv.c
index 6e8220f7ec0b..c4d3e5de67cc 100644
--- a/lib/msun/sparc64/fenv.c
+++ b/lib/msun/sparc64/fenv.c
@@ -26,7 +26,12 @@
  * $FreeBSD$
  */
 
-#include <fenv.h>
+#define	__fenv_static
+#include "fenv.h"
+
+#ifdef __GNUC_GNU_INLINE__
+#error "This file must be compiled with C99 'inline' semantics"
+#endif
 
 /*
  * The FSR_version field may be different on different
@@ -34,3 +39,15 @@
  * application.  Thus, 0 is valid as the default environment.
  */
 const fenv_t __fe_dfl_env = 0;
+
+extern inline int feclearexcept(int __excepts);
+extern inline int fegetexceptflag(fexcept_t *__flagp, int __excepts);
+extern inline int fesetexceptflag(const fexcept_t *__flagp, int __excepts);
+extern inline int feraiseexcept(int __excepts);
+extern inline int fetestexcept(int __excepts);
+extern inline int fegetround(void);
+extern inline int fesetround(int __round);
+extern inline int fegetenv(fenv_t *__envp);
+extern inline int feholdexcept(fenv_t *__envp);
+extern inline int fesetenv(const fenv_t *__envp);
+extern inline int feupdateenv(const fenv_t *__envp);
diff --git a/lib/msun/sparc64/fenv.h b/lib/msun/sparc64/fenv.h
index 82732991349b..d17361fef453 100644
--- a/lib/msun/sparc64/fenv.h
+++ b/lib/msun/sparc64/fenv.h
@@ -31,6 +31,10 @@
 
 #include <sys/_types.h>
 
+#ifndef	__fenv_static
+#define	__fenv_static	static
+#endif
+
 typedef	__uint64_t	fenv_t;
 typedef	__uint64_t	fexcept_t;
 
@@ -70,7 +74,7 @@ extern const fenv_t	__fe_dfl_env;
 #define	__ldxfsr(__r)	__asm __volatile("ldx %0, %%fsr" : : "m" (__r))
 #define	__stxfsr(__r)	__asm __volatile("stx %%fsr, %0" : "=m" (*(__r)))
 
-static __inline int
+__fenv_static __inline int
 feclearexcept(int __excepts)
 {
 	fexcept_t __r;
@@ -81,7 +85,7 @@ feclearexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
 	fexcept_t __r;
@@ -91,7 +95,7 @@ fegetexceptflag(fexcept_t *__flagp, int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetexceptflag(const fexcept_t *__flagp, int __excepts)
 {
 	fexcept_t __r;
@@ -108,7 +112,7 @@ fesetexceptflag(const fexcept_t *__flagp, int __excepts)
  * inline this function on sparc64 even when the arguments are not
  * compile-time constants.  Perhaps this depends on the register window.
  */
-static __inline int
+__fenv_static inline int
 feraiseexcept(int __excepts)
 {
 	volatile double d;
@@ -143,7 +147,7 @@ feraiseexcept(int __excepts)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fetestexcept(int __excepts)
 {
 	fexcept_t __r;
@@ -152,7 +156,7 @@ fetestexcept(int __excepts)
 	return (__r & __excepts);
 }
 
-static __inline int
+__fenv_static inline int
 fegetround(void)
 {
 	fenv_t __r;
@@ -161,7 +165,7 @@ fegetround(void)
 	return ((__r >> _ROUND_SHIFT) & _ROUND_MASK);
 }
 
-static __inline int
+__fenv_static inline int
 fesetround(int __round)
 {
 	fenv_t __r;
@@ -175,7 +179,7 @@ fesetround(int __round)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fegetenv(fenv_t *__envp)
 {
 
@@ -183,7 +187,7 @@ fegetenv(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feholdexcept(fenv_t *__envp)
 {
 	fenv_t __r;
@@ -195,7 +199,7 @@ feholdexcept(fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 fesetenv(const fenv_t *__envp)
 {
 
@@ -203,7 +207,7 @@ fesetenv(const fenv_t *__envp)
 	return (0);
 }
 
-static __inline int
+__fenv_static inline int
 feupdateenv(const fenv_t *__envp)
 {
 	fexcept_t __r;
@@ -216,7 +220,9 @@ feupdateenv(const fenv_t *__envp)
 
 #if __BSD_VISIBLE
 
-static __inline int
+/* We currently provide no external definitions of the functions below. */
+
+static inline int
 feenableexcept(int __mask)
 {
 	fenv_t __old_r, __new_r;
@@ -227,7 +233,7 @@ feenableexcept(int __mask)
 	return ((__old_r >> _FPUSW_SHIFT) & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fedisableexcept(int __mask)
 {
 	fenv_t __old_r, __new_r;
@@ -238,7 +244,7 @@ fedisableexcept(int __mask)
 	return ((__old_r >> _FPUSW_SHIFT) & FE_ALL_EXCEPT);
 }
 
-static __inline int
+static inline int
 fegetexcept(void)
 {
 	fenv_t __r;
diff --git a/lib/msun/src/e_cosh.c b/lib/msun/src/e_cosh.c
index 11e6590a8399..a36369545d9b 100644
--- a/lib/msun/src/e_cosh.c
+++ b/lib/msun/src/e_cosh.c
@@ -45,7 +45,6 @@ __ieee754_cosh(double x)
 {
 	double t,w;
 	int32_t ix;
-	u_int32_t lx;
 
     /* High word of |x|. */
 	GET_HIGH_WORD(ix,x);
@@ -72,13 +71,8 @@ __ieee754_cosh(double x)
 	if (ix < 0x40862E42)  return half*__ieee754_exp(fabs(x));
 
     /* |x| in [log(maxdouble), overflowthresold] */
-	GET_LOW_WORD(lx,x);
-	if (ix<0x408633CE ||
-	      ((ix==0x408633ce)&&(lx<=(u_int32_t)0x8fb9f87d))) {
-	    w = __ieee754_exp(half*fabs(x));
-	    t = half*w;
-	    return t*w;
-	}
+	if (ix<=0x408633CE)
+	    return __ldexp_exp(fabs(x), -1);
 
     /* |x| > overflowthresold, cosh(x) overflow */
 	return huge*huge;
diff --git a/lib/msun/src/e_coshf.c b/lib/msun/src/e_coshf.c
index 4a1d4994d489..95a0d6ee68ba 100644
--- a/lib/msun/src/e_coshf.c
+++ b/lib/msun/src/e_coshf.c
@@ -51,11 +51,8 @@ __ieee754_coshf(float x)
 	if (ix < 0x42b17217)  return half*__ieee754_expf(fabsf(x));
 
     /* |x| in [log(maxfloat), overflowthresold] */
-	if (ix<=0x42b2d4fc) {
-	    w = __ieee754_expf(half*fabsf(x));
-	    t = half*w;
-	    return t*w;
-	}
+	if (ix<=0x42b2d4fc)
+	    return __ldexp_expf(fabsf(x), -1);
 
     /* |x| > overflowthresold, cosh(x) overflow */
 	return huge*huge;
diff --git a/lib/msun/src/e_exp.c b/lib/msun/src/e_exp.c
index 5b9a10c1a3d6..b47aef583c98 100644
--- a/lib/msun/src/e_exp.c
+++ b/lib/msun/src/e_exp.c
@@ -76,6 +76,8 @@ __FBSDID("$FreeBSD$");
  * to produce the hexadecimal values shown.
  */
 
+#include <float.h>
+
 #include "math.h"
 #include "math_private.h"
 
@@ -133,7 +135,7 @@ __ieee754_exp(double x)	/* default IEEE double exp */
 		hi = x - t*ln2HI[0];	/* t*ln2HI is exact here */
 		lo = t*ln2LO[0];
 	    }
-	    x  = hi - lo;
+	    STRICT_ASSIGN(double, x, hi - lo);
 	} 
 	else if(hx < 0x3e300000)  {	/* when |x|<2**-28 */
 	    if(huge+x>one) return one+x;/* trigger inexact */
diff --git a/lib/msun/src/e_expf.c b/lib/msun/src/e_expf.c
index 502e42138d11..a4790765bcb5 100644
--- a/lib/msun/src/e_expf.c
+++ b/lib/msun/src/e_expf.c
@@ -16,6 +16,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <float.h>
+
 #include "math.h"
 #include "math_private.h"
 
@@ -40,7 +42,7 @@ P2 = -2.7667332906e-3;		/* -0xb55215.0p-32 */
 static volatile float twom100 = 7.8886090522e-31;      /* 2**-100=0x0d800000 */
 
 float
-__ieee754_expf(float x)	/* default IEEE double exp */
+__ieee754_expf(float x)
 {
 	float y,hi=0.0,lo=0.0,c,t,twopk;
 	int32_t k=0,xsb;
@@ -70,7 +72,7 @@ __ieee754_expf(float x)	/* default IEEE double exp */
 		hi = x - t*ln2HI[0];	/* t*ln2HI is exact here */
 		lo = t*ln2LO[0];
 	    }
-	    x  = hi - lo;
+	    STRICT_ASSIGN(float, x, hi - lo);
 	}
 	else if(hx < 0x39000000)  {	/* when |x|<2**-14 */
 	    if(huge+x>one) return one+x;/* trigger inexact */
diff --git a/lib/msun/src/e_hypot.c b/lib/msun/src/e_hypot.c
index fb498c1c4029..2398e9837fec 100644
--- a/lib/msun/src/e_hypot.c
+++ b/lib/msun/src/e_hypot.c
@@ -54,7 +54,7 @@ __FBSDID("$FreeBSD$");
 double
 __ieee754_hypot(double x, double y)
 {
-	double a=x,b=y,t1,t2,y1,y2,w;
+	double a,b,t1,t2,y1,y2,w;
 	int32_t j,k,ha,hb;
 
 	GET_HIGH_WORD(ha,x);
diff --git a/lib/msun/src/e_hypotf.c b/lib/msun/src/e_hypotf.c
index c82c6e7d2ad0..6d083e4d2c3f 100644
--- a/lib/msun/src/e_hypotf.c
+++ b/lib/msun/src/e_hypotf.c
@@ -22,7 +22,7 @@ __FBSDID("$FreeBSD$");
 float
 __ieee754_hypotf(float x, float y)
 {
-	float a=x,b=y,t1,t2,y1,y2,w;
+	float a,b,t1,t2,y1,y2,w;
 	int32_t j,k,ha,hb;
 
 	GET_FLOAT_WORD(ha,x);
diff --git a/lib/msun/src/e_hypotl.c b/lib/msun/src/e_hypotl.c
index 0c899bfd0b8a..7b5ab89562fe 100644
--- a/lib/msun/src/e_hypotl.c
+++ b/lib/msun/src/e_hypotl.c
@@ -21,13 +21,6 @@ __FBSDID("$FreeBSD$");
 #include "math.h"
 #include "math_private.h"
 
-#define	GET_LDBL_EXPSIGN(i, v) do {	\
-	union IEEEl2bits uv;		\
-					\
-	uv.e = v;			\
-	i = uv.xbits.expsign;		\
-} while (0)
-
 #define	GET_LDBL_MAN(h, l, v) do {	\
 	union IEEEl2bits uv;		\
 					\
@@ -36,14 +29,6 @@ __FBSDID("$FreeBSD$");
 	l = uv.bits.manl;		\
 } while (0)
 
-#define	SET_LDBL_EXPSIGN(v, i) do {	\
-	union IEEEl2bits uv;		\
-					\
-	uv.e = v;			\
-	uv.xbits.expsign = i;		\
-	v = uv.e;			\
-} while (0)
-
 #undef GET_HIGH_WORD
 #define	GET_HIGH_WORD(i, v)	GET_LDBL_EXPSIGN(i, v)
 #undef SET_HIGH_WORD
diff --git a/lib/msun/src/e_lgamma_r.c b/lib/msun/src/e_lgamma_r.c
index a587b8f63dcb..1cff592c5d7e 100644
--- a/lib/msun/src/e_lgamma_r.c
+++ b/lib/msun/src/e_lgamma_r.c
@@ -269,7 +269,6 @@ __ieee754_lgamma_r(double x, int *signgamp)
 	}
 	else if(ix<0x40200000) { 			/* x < 8.0 */
 	    i = (int)x;
-	    t = zero;
 	    y = x-(double)i;
 	    p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*s6))))));
 	    q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*r6)))));
diff --git a/lib/msun/src/e_lgammaf_r.c b/lib/msun/src/e_lgammaf_r.c
index 47c6ed073c2a..e2d90ef775bd 100644
--- a/lib/msun/src/e_lgammaf_r.c
+++ b/lib/msun/src/e_lgammaf_r.c
@@ -202,7 +202,6 @@ __ieee754_lgammaf_r(float x, int *signgamp)
 	}
 	else if(ix<0x41000000) { 			/* x < 8.0 */
 	    i = (int)x;
-	    t = zero;
 	    y = x-(float)i;
 	    p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*s6))))));
 	    q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*r6)))));
diff --git a/lib/msun/src/e_log10.c b/lib/msun/src/e_log10.c
index 135f0dcf6d81..104d2574c659 100644
--- a/lib/msun/src/e_log10.c
+++ b/lib/msun/src/e_log10.c
@@ -15,7 +15,11 @@
 __FBSDID("$FreeBSD$");
 
 /*
- * Return the base 10 logarithm of x. See k_log.c for details on the algorithm.
+ * Return the base 10 logarithm of x.  See e_log.c and k_log.h for most
+ * comments.
+ *
+ *    log10(x) = (f - 0.5*f*f + k_log1p(f)) / ln10 + k * log10(2)
+ * in not-quite-routine extra precision.
  */
 
 #include "math.h"
@@ -34,31 +38,50 @@ static const double zero   =  0.0;
 double
 __ieee754_log10(double x)
 {
-	double f,hi,lo,y,z;
+	double f,hfsq,hi,lo,r,val_hi,val_lo,w,y,y2;
 	int32_t i,k,hx;
 	u_int32_t lx;
 
 	EXTRACT_WORDS(hx,lx,x);
 
-        k=0;
-        if (hx < 0x00100000) {                  /* x < 2**-1022  */
-            if (((hx&0x7fffffff)|lx)==0)
-                return -two54/zero;             /* log(+-0)=-inf */
-            if (hx<0) return (x-x)/zero;        /* log(-#) = NaN */
-            k -= 54; x *= two54; /* subnormal number, scale up x */
+	k=0;
+	if (hx < 0x00100000) {			/* x < 2**-1022  */
+	    if (((hx&0x7fffffff)|lx)==0)
+		return -two54/zero;		/* log(+-0)=-inf */
+	    if (hx<0) return (x-x)/zero;	/* log(-#) = NaN */
+	    k -= 54; x *= two54; /* subnormal number, scale up x */
 	    GET_HIGH_WORD(hx,x);
-        }
+	}
 	if (hx >= 0x7ff00000) return x+x;
+	if (hx == 0x3ff00000 && lx == 0)
+	    return zero;			/* log(1) = +0 */
 	k += (hx>>20)-1023;
 	hx &= 0x000fffff;
 	i = (hx+0x95f64)&0x100000;
 	SET_HIGH_WORD(x,hx|(i^0x3ff00000));	/* normalize x or x/2 */
 	k += (i>>20);
 	y = (double)k;
-	f = __kernel_log(x);
-	hi = x = x - 1;
+	f = x - 1.0;
+	hfsq = 0.5*f*f;
+	r = k_log1p(f);
+
+	/* See e_log2.c for most details. */
+	hi = f - hfsq;
 	SET_LOW_WORD(hi,0);
-	lo = x - hi;
-	z = y*log10_2lo + (x+f)*ivln10lo + (lo+f)*ivln10hi + hi*ivln10hi;
-	return  z+y*log10_2hi;
+	lo = (f - hi) - hfsq + r;
+	val_hi = hi*ivln10hi;
+	y2 = y*log10_2hi;
+	val_lo = y*log10_2lo + (lo+hi)*ivln10lo + lo*ivln10hi;
+
+	/*
+	 * Extra precision in for adding y*log10_2hi is not strictly needed
+	 * since there is no very large cancellation near x = sqrt(2) or
+	 * x = 1/sqrt(2), but we do it anyway since it costs little on CPUs
+	 * with some parallelism and it reduces the error for many args.
+	 */
+	w = y2 + val_hi;
+	val_lo += (y2 - w) + val_hi;
+	val_hi = w;
+
+	return val_lo + val_hi;
 }
diff --git a/lib/msun/src/e_log10f.c b/lib/msun/src/e_log10f.c
index 940b8319f804..c8765941c6d0 100644
--- a/lib/msun/src/e_log10f.c
+++ b/lib/msun/src/e_log10f.c
@@ -13,7 +13,7 @@
 __FBSDID("$FreeBSD$");
 
 /*
- * Return the base 10 logarithm of x. See k_log.c for details on the algorithm.
+ * Float version of e_log10.c.  See the latter for most comments.
  */
 
 #include "math.h"
@@ -32,31 +32,40 @@ static const float zero   =  0.0;
 float
 __ieee754_log10f(float x)
 {
-	float f,hi,lo,y,z;
+	float f,hfsq,hi,lo,r,y;
 	int32_t i,k,hx;
 
 	GET_FLOAT_WORD(hx,x);
 
-        k=0;
-        if (hx < 0x00800000) {                  /* x < 2**-126  */
-            if ((hx&0x7fffffff)==0)
-                return -two25/zero;             /* log(+-0)=-inf */
-            if (hx<0) return (x-x)/zero;        /* log(-#) = NaN */
-            k -= 25; x *= two25; /* subnormal number, scale up x */
+	k=0;
+	if (hx < 0x00800000) {			/* x < 2**-126  */
+	    if ((hx&0x7fffffff)==0)
+		return -two25/zero;		/* log(+-0)=-inf */
+	    if (hx<0) return (x-x)/zero;	/* log(-#) = NaN */
+	    k -= 25; x *= two25; /* subnormal number, scale up x */
 	    GET_FLOAT_WORD(hx,x);
-        }
+	}
 	if (hx >= 0x7f800000) return x+x;
+	if (hx == 0x3f800000)
+	    return zero;			/* log(1) = +0 */
 	k += (hx>>23)-127;
 	hx &= 0x007fffff;
 	i = (hx+(0x4afb0d))&0x800000;
 	SET_FLOAT_WORD(x,hx|(i^0x3f800000));	/* normalize x or x/2 */
 	k += (i>>23);
 	y = (float)k;
-	f = __kernel_logf(x);
-	x = x - (float)1.0;
-	GET_FLOAT_WORD(hx,x);
+	f = x - (float)1.0;
+	hfsq = (float)0.5*f*f;
+	r = k_log1pf(f);
+
+	/* See e_log2f.c and e_log2.c for details. */
+	if (sizeof(float_t) > sizeof(float))
+		return (r - hfsq + f) * ((float_t)ivln10lo + ivln10hi) +
+		    y * ((float_t)log10_2lo + log10_2hi);
+	hi = f - hfsq;
+	GET_FLOAT_WORD(hx,hi);
 	SET_FLOAT_WORD(hi,hx&0xfffff000);
-	lo = x - hi;
-	z = y*log10_2lo + (x+f)*ivln10lo + (lo+f)*ivln10hi + hi*ivln10hi;
-	return  z+y*log10_2hi;
+	lo = (f - hi) - hfsq + r;
+	return y*log10_2lo + (lo+hi)*ivln10lo + lo*ivln10hi + hi*ivln10hi +
+	    y*log10_2hi;
 }
diff --git a/lib/msun/src/e_log2.c b/lib/msun/src/e_log2.c
index 6cf3dbc1924d..1fc44a5fedec 100644
--- a/lib/msun/src/e_log2.c
+++ b/lib/msun/src/e_log2.c
@@ -15,7 +15,13 @@
 __FBSDID("$FreeBSD$");
 
 /*
- * Return the base 2 logarithm of x. See k_log.c for details on the algorithm.
+ * Return the base 2 logarithm of x.  See e_log.c and k_log.h for most
+ * comments.
+ *
+ * This reduces x to {k, 1+f} exactly as in e_log.c, then calls the kernel,
+ * then does the combining and scaling steps
+ *    log2(x) = (f - 0.5*f*f + k_log1p(f)) / ln2 + k
+ * in not-quite-routine extra precision.
  */
 
 #include "math.h"
@@ -32,29 +38,73 @@ static const double zero   =  0.0;
 double
 __ieee754_log2(double x)
 {
-	double f,hi,lo;
+	double f,hfsq,hi,lo,r,val_hi,val_lo,w,y;
 	int32_t i,k,hx;
 	u_int32_t lx;
 
 	EXTRACT_WORDS(hx,lx,x);
 
-        k=0;
-        if (hx < 0x00100000) {                  /* x < 2**-1022  */
-            if (((hx&0x7fffffff)|lx)==0)
-                return -two54/zero;             /* log(+-0)=-inf */
-            if (hx<0) return (x-x)/zero;        /* log(-#) = NaN */
-            k -= 54; x *= two54; /* subnormal number, scale up x */
+	k=0;
+	if (hx < 0x00100000) {			/* x < 2**-1022  */
+	    if (((hx&0x7fffffff)|lx)==0)
+		return -two54/zero;		/* log(+-0)=-inf */
+	    if (hx<0) return (x-x)/zero;	/* log(-#) = NaN */
+	    k -= 54; x *= two54; /* subnormal number, scale up x */
 	    GET_HIGH_WORD(hx,x);
-        }
+	}
 	if (hx >= 0x7ff00000) return x+x;
+	if (hx == 0x3ff00000 && lx == 0)
+	    return zero;			/* log(1) = +0 */
 	k += (hx>>20)-1023;
 	hx &= 0x000fffff;
 	i = (hx+0x95f64)&0x100000;
 	SET_HIGH_WORD(x,hx|(i^0x3ff00000));	/* normalize x or x/2 */
 	k += (i>>20);
-	f = __kernel_log(x);
-	hi = x = x - 1;
+	y = (double)k;
+	f = x - 1.0;
+	hfsq = 0.5*f*f;
+	r = k_log1p(f);
+
+	/*
+	 * f-hfsq must (for args near 1) be evaluated in extra precision
+	 * to avoid a large cancellation when x is near sqrt(2) or 1/sqrt(2).
+	 * This is fairly efficient since f-hfsq only depends on f, so can
+	 * be evaluated in parallel with R.  Not combining hfsq with R also
+	 * keeps R small (though not as small as a true `lo' term would be),
+	 * so that extra precision is not needed for terms involving R.
+	 *
+	 * Compiler bugs involving extra precision used to break Dekker's
+	 * theorem for spitting f-hfsq as hi+lo, unless double_t was used
+	 * or the multi-precision calculations were avoided when double_t
+	 * has extra precision.  These problems are now automatically
+	 * avoided as a side effect of the optimization of combining the
+	 * Dekker splitting step with the clear-low-bits step.
+	 *
+	 * y must (for args near sqrt(2) and 1/sqrt(2)) be added in extra
+	 * precision to avoid a very large cancellation when x is very near
+	 * these values.  Unlike the above cancellations, this problem is
+	 * specific to base 2.  It is strange that adding +-1 is so much
+	 * harder than adding +-ln2 or +-log10_2.
+	 *
+	 * This uses Dekker's theorem to normalize y+val_hi, so the
+	 * compiler bugs are back in some configurations, sigh.  And I
+	 * don't want to used double_t to avoid them, since that gives a
+	 * pessimization and the support for avoiding the pessimization
+	 * is not yet available.
+	 *
+	 * The multi-precision calculations for the multiplications are
+	 * routine.
+	 */
+	hi = f - hfsq;
 	SET_LOW_WORD(hi,0);
-	lo = x - hi;
-	return (x+f)*ivln2lo + (lo+f)*ivln2hi + hi*ivln2hi + k;
+	lo = (f - hi) - hfsq + r;
+	val_hi = hi*ivln2hi;
+	val_lo = (lo+hi)*ivln2lo + lo*ivln2hi;
+
+	/* spadd(val_hi, val_lo, y), except for not using double_t: */
+	w = y + val_hi;
+	val_lo += (y - w) + val_hi;
+	val_hi = w;
+
+	return val_lo + val_hi;
 }
diff --git a/lib/msun/src/e_log2f.c b/lib/msun/src/e_log2f.c
index bb308d3e8a13..716634664a42 100644
--- a/lib/msun/src/e_log2f.c
+++ b/lib/msun/src/e_log2f.c
@@ -13,7 +13,7 @@
 __FBSDID("$FreeBSD$");
 
 /*
- * Return the base 2 logarithm of x. See k_log.c for details on the algorithm.
+ * Float version of e_log2.c.  See the latter for most comments.
  */
 
 #include "math.h"
@@ -30,29 +30,52 @@ static const float zero   =  0.0;
 float
 __ieee754_log2f(float x)
 {
-	float f,hi,lo;
+	float f,hfsq,hi,lo,r,y;
 	int32_t i,k,hx;
 
 	GET_FLOAT_WORD(hx,x);
 
-        k=0;
-        if (hx < 0x00800000) {                  /* x < 2**-126  */
-            if ((hx&0x7fffffff)==0)
-                return -two25/zero;             /* log(+-0)=-inf */
-            if (hx<0) return (x-x)/zero;        /* log(-#) = NaN */
-            k -= 25; x *= two25; /* subnormal number, scale up x */
+	k=0;
+	if (hx < 0x00800000) {			/* x < 2**-126  */
+	    if ((hx&0x7fffffff)==0)
+		return -two25/zero;		/* log(+-0)=-inf */
+	    if (hx<0) return (x-x)/zero;	/* log(-#) = NaN */
+	    k -= 25; x *= two25; /* subnormal number, scale up x */
 	    GET_FLOAT_WORD(hx,x);
-        }
+	}
 	if (hx >= 0x7f800000) return x+x;
+	if (hx == 0x3f800000)
+	    return zero;			/* log(1) = +0 */
 	k += (hx>>23)-127;
 	hx &= 0x007fffff;
 	i = (hx+(0x4afb0d))&0x800000;
 	SET_FLOAT_WORD(x,hx|(i^0x3f800000));	/* normalize x or x/2 */
 	k += (i>>23);
-	f = __kernel_logf(x);
-	x = x - (float)1.0;
-	GET_FLOAT_WORD(hx,x);
+	y = (float)k;
+	f = x - (float)1.0;
+	hfsq = (float)0.5*f*f;
+	r = k_log1pf(f);
+
+	/*
+	 * We no longer need to avoid falling into the multi-precision
+	 * calculations due to compiler bugs breaking Dekker's theorem.
+	 * Keep avoiding this as an optimization.  See e_log2.c for more
+	 * details (some details are here only because the optimization
+	 * is not yet available in double precision).
+	 *
+	 * Another compiler bug turned up.  With gcc on i386,
+	 * (ivln2lo + ivln2hi) would be evaluated in float precision
+	 * despite runtime evaluations using double precision.  So we
+	 * must cast one of its terms to float_t.  This makes the whole
+	 * expression have type float_t, so return is forced to waste
+	 * time clobbering its extra precision.
+	 */
+	if (sizeof(float_t) > sizeof(float))
+		return (r - hfsq + f) * ((float_t)ivln2lo + ivln2hi) + y;
+
+	hi = f - hfsq;
+	GET_FLOAT_WORD(hx,hi);
 	SET_FLOAT_WORD(hi,hx&0xfffff000);
-	lo = x - hi;
-	return (x+f)*ivln2lo + (lo+f)*ivln2hi + hi*ivln2hi + k;
+	lo = (f - hi) - hfsq + r;
+	return (lo+hi)*ivln2lo + lo*ivln2hi + hi*ivln2hi + y;
 }
diff --git a/lib/msun/src/e_pow.c b/lib/msun/src/e_pow.c
index 4aa00d572b62..7607a4a72650 100644
--- a/lib/msun/src/e_pow.c
+++ b/lib/msun/src/e_pow.c
@@ -109,6 +109,9 @@ __ieee754_pow(double x, double y)
     /* y==zero: x**0 = 1 */
 	if((iy|ly)==0) return one; 	
 
+    /* x==1: 1**y = 1, even if y is NaN */
+	if (hx==0x3ff00000 && lx == 0) return one;
+
     /* y!=zero: result is NaN if either arg is NaN */
 	if(ix > 0x7ff00000 || ((ix==0x7ff00000)&&(lx!=0)) ||
 	   iy > 0x7ff00000 || ((iy==0x7ff00000)&&(ly!=0))) 
@@ -138,7 +141,7 @@ __ieee754_pow(double x, double y)
 	if(ly==0) { 	
 	    if (iy==0x7ff00000) {	/* y is +-inf */
 	        if(((ix-0x3ff00000)|lx)==0)
-		    return  y - y;	/* inf**+-1 is NaN */
+		    return  one;	/* (-1)**+-inf is NaN */
 	        else if (ix >= 0x3ff00000)/* (|x|>1)**+-inf = inf,0 */
 		    return (hy>=0)? y: zero;
 	        else			/* (|x|<1)**-,+inf = inf,0 */
diff --git a/lib/msun/src/e_powf.c b/lib/msun/src/e_powf.c
index 466ed721538b..5c464788758d 100644
--- a/lib/msun/src/e_powf.c
+++ b/lib/msun/src/e_powf.c
@@ -67,6 +67,9 @@ __ieee754_powf(float x, float y)
     /* y==zero: x**0 = 1 */
 	if(iy==0) return one;
 
+    /* x==1: 1**y = 1, even if y is NaN */
+	if (hx==0x3f800000) return one;
+
     /* y!=zero: result is NaN if either arg is NaN */
 	if(ix > 0x7f800000 ||
 	   iy > 0x7f800000)
@@ -90,7 +93,7 @@ __ieee754_powf(float x, float y)
     /* special value of y */
 	if (iy==0x7f800000) {	/* y is +-inf */
 	    if (ix==0x3f800000)
-	        return  y - y;	/* inf**+-1 is NaN */
+	        return  one;	/* (-1)**+-inf is NaN */
 	    else if (ix > 0x3f800000)/* (|x|>1)**+-inf = inf,0 */
 	        return (hy>=0)? y: zero;
 	    else			/* (|x|<1)**-,+inf = inf,0 */
diff --git a/lib/msun/src/e_sinh.c b/lib/msun/src/e_sinh.c
index afb8e43c99e1..17442d0d152a 100644
--- a/lib/msun/src/e_sinh.c
+++ b/lib/msun/src/e_sinh.c
@@ -40,9 +40,8 @@ static const double one = 1.0, shuge = 1.0e307;
 double
 __ieee754_sinh(double x)
 {
-	double t,w,h;
+	double t,h;
 	int32_t ix,jx;
-	u_int32_t lx;
 
     /* High word of |x|. */
 	GET_HIGH_WORD(jx,x);
@@ -66,12 +65,8 @@ __ieee754_sinh(double x)
 	if (ix < 0x40862E42)  return h*__ieee754_exp(fabs(x));
 
     /* |x| in [log(maxdouble), overflowthresold] */
-	GET_LOW_WORD(lx,x);
-	if (ix<0x408633CE || ((ix==0x408633ce)&&(lx<=(u_int32_t)0x8fb9f87d))) {
-	    w = __ieee754_exp(0.5*fabs(x));
-	    t = h*w;
-	    return t*w;
-	}
+	if (ix<=0x408633CE)
+	    return h*2.0*__ldexp_exp(fabs(x), -1);
 
     /* |x| > overflowthresold, sinh(x) overflow */
 	return x*shuge;
diff --git a/lib/msun/src/e_sinhf.c b/lib/msun/src/e_sinhf.c
index 0f96b2b437e6..1be2dc397d11 100644
--- a/lib/msun/src/e_sinhf.c
+++ b/lib/msun/src/e_sinhf.c
@@ -24,7 +24,7 @@ static const float one = 1.0, shuge = 1.0e37;
 float
 __ieee754_sinhf(float x)
 {
-	float t,w,h;
+	float t,h;
 	int32_t ix,jx;
 
 	GET_FLOAT_WORD(jx,x);
@@ -48,11 +48,8 @@ __ieee754_sinhf(float x)
 	if (ix < 0x42b17217)  return h*__ieee754_expf(fabsf(x));
 
     /* |x| in [logf(maxfloat), overflowthresold] */
-	if (ix<=0x42b2d4fc) {
-	    w = __ieee754_expf((float)0.5*fabsf(x));
-	    t = h*w;
-	    return t*w;
-	}
+	if (ix<=0x42b2d4fc)
+	    return h*2.0F*__ldexp_expf(fabsf(x), -1);
 
     /* |x| > overflowthresold, sinh(x) overflow */
 	return x*shuge;
diff --git a/lib/msun/src/k_exp.c b/lib/msun/src/k_exp.c
new file mode 100644
index 000000000000..f592f69aa009
--- /dev/null
+++ b/lib/msun/src/k_exp.c
@@ -0,0 +1,108 @@
+/*-
+ * Copyright (c) 2011 David Schultz <das@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+
+#include "math.h"
+#include "math_private.h"
+
+static const uint32_t k = 1799;		/* constant for reduction */
+static const double kln2 =  1246.97177782734161156;	/* k * ln2 */
+
+/*
+ * Compute exp(x), scaled to avoid spurious overflow.  An exponent is
+ * returned separately in 'expt'.
+ *
+ * Input:  ln(DBL_MAX) <= x < ln(2 * DBL_MAX / DBL_MIN_DENORM) ~= 1454.91
+ * Output: 2**1023 <= y < 2**1024
+ */
+static double
+__frexp_exp(double x, int *expt)
+{
+	double exp_x;
+	uint32_t hx;
+
+	/*
+	 * We use exp(x) = exp(x - kln2) * 2**k, carefully chosen to
+	 * minimize |exp(kln2) - 2**k|.  We also scale the exponent of
+	 * exp_x to MAX_EXP so that the result can be multiplied by
+	 * a tiny number without losing accuracy due to denormalization.
+	 */
+	exp_x = exp(x - kln2);
+	GET_HIGH_WORD(hx, exp_x);
+	*expt = (hx >> 20) - (0x3ff + 1023) + k;
+	SET_HIGH_WORD(exp_x, (hx & 0xfffff) | ((0x3ff + 1023) << 20));
+	return (exp_x);
+}
+
+/*
+ * __ldexp_exp(x, expt) and __ldexp_cexp(x, expt) compute exp(x) * 2**expt.
+ * They are intended for large arguments (real part >= ln(DBL_MAX))
+ * where care is needed to avoid overflow.
+ *
+ * The present implementation is narrowly tailored for our hyperbolic and
+ * exponential functions.  We assume expt is small (0 or -1), and the caller
+ * has filtered out very large x, for which overflow would be inevitable.
+ */
+
+double
+__ldexp_exp(double x, int expt)
+{
+	double exp_x, scale;
+	int ex_expt;
+
+	exp_x = __frexp_exp(x, &ex_expt);
+	expt += ex_expt;
+	INSERT_WORDS(scale, (0x3ff + expt) << 20, 0);
+	return (exp_x * scale);
+}
+
+double complex
+__ldexp_cexp(double complex z, int expt)
+{
+	double x, y, exp_x, scale1, scale2;
+	int ex_expt, half_expt;
+
+	x = creal(z);
+	y = cimag(z);
+	exp_x = __frexp_exp(x, &ex_expt);
+	expt += ex_expt;
+
+	/*
+	 * Arrange so that scale1 * scale2 == 2**expt.  We use this to
+	 * compensate for scalbn being horrendously slow.
+	 */
+	half_expt = expt / 2;
+	INSERT_WORDS(scale1, (0x3ff + half_expt) << 20, 0);
+	half_expt = expt - half_expt;
+	INSERT_WORDS(scale2, (0x3ff + half_expt) << 20, 0);
+
+	return (cpack(cos(y) * exp_x * scale1 * scale2,
+	    sin(y) * exp_x * scale1 * scale2));
+}
diff --git a/lib/msun/src/k_expf.c b/lib/msun/src/k_expf.c
new file mode 100644
index 000000000000..a860b9f74aed
--- /dev/null
+++ b/lib/msun/src/k_expf.c
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 2011 David Schultz <das@FreeBSD.ORG>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+
+#include "math.h"
+#include "math_private.h"
+
+static const uint32_t k = 235;			/* constant for reduction */
+static const float kln2 =  162.88958740F;	/* k * ln2 */
+
+/*
+ * See k_exp.c for details.
+ *
+ * Input:  ln(FLT_MAX) <= x < ln(2 * FLT_MAX / FLT_MIN_DENORM) ~= 192.7
+ * Output: 2**127 <= y < 2**128
+ */
+static float
+__frexp_expf(float x, int *expt)
+{
+	double exp_x;
+	uint32_t hx;
+
+	exp_x = expf(x - kln2);
+	GET_FLOAT_WORD(hx, exp_x);
+	*expt = (hx >> 23) - (0x7f + 127) + k;
+	SET_FLOAT_WORD(exp_x, (hx & 0x7fffff) | ((0x7f + 127) << 23));
+	return (exp_x);
+}
+
+float
+__ldexp_expf(float x, int expt)
+{
+	float exp_x, scale;
+	int ex_expt;
+
+	exp_x = __frexp_expf(x, &ex_expt);
+	expt += ex_expt;
+	SET_FLOAT_WORD(scale, (0x7f + expt) << 23);
+	return (exp_x * scale);
+}
+
+float complex
+__ldexp_cexpf(float complex z, int expt)
+{
+	float x, y, exp_x, scale1, scale2;
+	int ex_expt, half_expt;
+
+	x = crealf(z);
+	y = cimagf(z);
+	exp_x = __frexp_expf(x, &ex_expt);
+	expt += ex_expt;
+
+	half_expt = expt / 2;
+	SET_FLOAT_WORD(scale1, (0x7f + half_expt) << 23);
+	half_expt = expt - half_expt;
+	SET_FLOAT_WORD(scale2, (0x7f + half_expt) << 23);
+
+	return (cpackf(cosf(y) * exp_x * scale1 * scale2,
+	    sinf(y) * exp_x * scale1 * scale2));
+}
diff --git a/lib/msun/src/k_log.h b/lib/msun/src/k_log.h
index 206355c272ac..aaff8bd9000e 100644
--- a/lib/msun/src/k_log.h
+++ b/lib/msun/src/k_log.h
@@ -14,8 +14,9 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-/* __kernel_log(x)
- * Return log(x) - (x-1) for x in ~[sqrt(2)/2, sqrt(2)].
+/*
+ * k_log1p(f):
+ * Return log(1+f) - f for 1+f in ~[sqrt(2)/2, sqrt(2)].
  *
  * The following describes the overall strategy for computing
  * logarithms in base e.  The argument reduction and adding the final
@@ -80,37 +81,20 @@ Lg6 = 1.531383769920937332e-01,  /* 3FC39A09 D078C69F */
 Lg7 = 1.479819860511658591e-01;  /* 3FC2F112 DF3E5244 */
 
 /*
- * We always inline __kernel_log(), since doing so produces a
+ * We always inline k_log1p(), since doing so produces a
  * substantial performance improvement (~40% on amd64).
  */
 static inline double
-__kernel_log(double x)
+k_log1p(double f)
 {
-	double hfsq,f,s,z,R,w,t1,t2;
-	int32_t hx,i,j;
-	u_int32_t lx;
+	double hfsq,s,z,R,w,t1,t2;
 
-	EXTRACT_WORDS(hx,lx,x);
-
-	f = x-1.0;
-	if((0x000fffff&(2+hx))<3) {	/* -2**-20 <= f < 2**-20 */
-	    if(f==0.0) return 0.0;
-	    return f*f*(0.33333333333333333*f-0.5);
-	}
- 	s = f/(2.0+f); 
+ 	s = f/(2.0+f);
 	z = s*s;
-	hx &= 0x000fffff;
-	i = hx-0x6147a;
 	w = z*z;
-	j = 0x6b851-hx;
-	t1= w*(Lg2+w*(Lg4+w*Lg6)); 
-	t2= z*(Lg1+w*(Lg3+w*(Lg5+w*Lg7))); 
-	i |= j;
+	t1= w*(Lg2+w*(Lg4+w*Lg6));
+	t2= z*(Lg1+w*(Lg3+w*(Lg5+w*Lg7)));
 	R = t2+t1;
-	if (i>0) {
-	    hfsq=0.5*f*f;
-	    return s*(hfsq+R) - hfsq;
-	} else {
-	    return s*(R-f);
-	}
+	hfsq=0.5*f*f;
+	return s*(hfsq+R);
 }
diff --git a/lib/msun/src/k_logf.h b/lib/msun/src/k_logf.h
index d9f0f3ddfde1..71c547e888ae 100644
--- a/lib/msun/src/k_logf.h
+++ b/lib/msun/src/k_logf.h
@@ -13,7 +13,7 @@
 __FBSDID("$FreeBSD$");
 
 /*
- * float version of __kernel_log(x). See k_log.c for details.
+ * Float version of k_log.h.  See the latter for most comments.
  */
 
 static const float
@@ -24,32 +24,16 @@ Lg3 =      0x91e9ee.0p-25,	/* 0.28498786688 */
 Lg4 =      0xf89e26.0p-26;	/* 0.24279078841 */
 
 static inline float
-__kernel_logf(float x)
+k_log1pf(float f)
 {
-	float hfsq,f,s,z,R,w,t1,t2;
-	int32_t ix,i,j;
+	float hfsq,s,z,R,w,t1,t2;
 
-	GET_FLOAT_WORD(ix,x);
-
-	f = x-(float)1.0;
-	if((0x007fffff&(0x8000+ix))<0xc000) {	/* -2**-9 <= f < 2**-9 */
-	    if(f==0.0f) return 0.0f;
-	    return f*f*((float)0.33333333333333333*f-(float)0.5);
-	}
  	s = f/((float)2.0+f);
 	z = s*s;
-	ix &= 0x007fffff;
-	i = ix-(0x6147a<<3);
 	w = z*z;
-	j = (0x6b851<<3)-ix;
 	t1= w*(Lg2+w*Lg4);
 	t2= z*(Lg1+w*Lg3);
-	i |= j;
 	R = t2+t1;
-	if(i>0) {
-	    hfsq=(float)0.5*f*f;
-	    return s*(hfsq+R) - hfsq;
-	} else {
-	    return s*(R-f);
-	}
+	hfsq=(float)0.5*f*f;
+	return s*(hfsq+R);
 }
diff --git a/lib/msun/src/math.h b/lib/msun/src/math.h
index 8ad13ed0b157..69b138e3c6df 100644
--- a/lib/msun/src/math.h
+++ b/lib/msun/src/math.h
@@ -68,14 +68,11 @@ extern const union __nan_un {
 #define	MATH_ERREXCEPT	2
 #define	math_errhandling	MATH_ERREXCEPT
 
-/* XXX We need a <machine/math.h>. */
-#if defined(__ia64__) || defined(__sparc64__)
-#define	FP_FAST_FMA	1
-#endif
+#define	FP_FAST_FMAF	1
 #ifdef __ia64__
+#define	FP_FAST_FMA	1
 #define	FP_FAST_FMAL	1
 #endif
-#define	FP_FAST_FMAF	1
 
 /* Symbolic constants to classify floating point numbers. */
 #define	FP_INFINITE	0x01
diff --git a/lib/msun/src/math_private.h b/lib/msun/src/math_private.h
index d79f808bf0ff..79280e30aa2a 100644
--- a/lib/msun/src/math_private.h
+++ b/lib/msun/src/math_private.h
@@ -58,6 +58,10 @@ typedef union
     u_int32_t msw;
     u_int32_t lsw;
   } parts;
+  struct
+  {
+    u_int64_t w;
+  } xparts;
 } ieee_double_shape_type;
 
 #endif
@@ -72,6 +76,10 @@ typedef union
     u_int32_t lsw;
     u_int32_t msw;
   } parts;
+  struct
+  {
+    u_int64_t w;
+  } xparts;
 } ieee_double_shape_type;
 
 #endif
@@ -86,6 +94,14 @@ do {								\
   (ix1) = ew_u.parts.lsw;					\
 } while (0)
 
+/* Get a 64-bit int from a double. */
+#define EXTRACT_WORD64(ix,d)					\
+do {								\
+  ieee_double_shape_type ew_u;					\
+  ew_u.value = (d);						\
+  (ix) = ew_u.xparts.w;						\
+} while (0)
+
 /* Get the more significant 32 bit int from a double.  */
 
 #define GET_HIGH_WORD(i,d)					\
@@ -114,6 +130,14 @@ do {								\
   (d) = iw_u.value;						\
 } while (0)
 
+/* Set a double from a 64-bit int. */
+#define INSERT_WORD64(d,ix)					\
+do {								\
+  ieee_double_shape_type iw_u;					\
+  iw_u.xparts.w = (ix);						\
+  (d) = iw_u.value;						\
+} while (0)
+
 /* Set the more significant 32 bits of a double from an int.  */
 
 #define SET_HIGH_WORD(d,v)					\
@@ -164,6 +188,25 @@ do {								\
   (d) = sf_u.value;						\
 } while (0)
 
+/* Get expsign as a 16 bit int from a long double.  */
+
+#define	GET_LDBL_EXPSIGN(i,d)					\
+do {								\
+  union IEEEl2bits ge_u;					\
+  ge_u.e = (d);							\
+  (i) = ge_u.xbits.expsign;					\
+} while (0)
+
+/* Set expsign of a long double from a 16 bit int.  */
+
+#define	SET_LDBL_EXPSIGN(d,v)					\
+do {								\
+  union IEEEl2bits se_u;					\
+  se_u.e = (d);							\
+  se_u.xbits.expsign = (v);					\
+  (d) = se_u.e;							\
+} while (0)
+
 #ifdef FLT_EVAL_METHOD
 /*
  * Attempt to get strict C99 semantics for assignment with non-C99 compilers.
@@ -354,6 +397,10 @@ int	__ieee754_rem_pio2(double,double*);
 double	__kernel_sin(double,double,int);
 double	__kernel_cos(double,double);
 double	__kernel_tan(double,double,int);
+double	__ldexp_exp(double,int);
+#ifdef _COMPLEX_H
+double complex __ldexp_cexp(double complex,int);
+#endif
 
 /* float precision kernel functions */
 #ifdef INLINE_REM_PIO2F
@@ -372,6 +419,10 @@ float	__kernel_cosdf(double);
 __inline
 #endif
 float	__kernel_tandf(double,int);
+float	__ldexp_expf(float,int);
+#ifdef _COMPLEX_H
+float complex __ldexp_cexpf(float complex,int);
+#endif
 
 /* long double precision kernel functions */
 long double __kernel_sinl(long double, long double, int);
diff --git a/lib/msun/src/s_ccosh.c b/lib/msun/src/s_ccosh.c
new file mode 100644
index 000000000000..9ea962b4818f
--- /dev/null
+++ b/lib/msun/src/s_ccosh.c
@@ -0,0 +1,155 @@
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Hyperbolic cosine of a complex argument z = x + i y.
+ *
+ * cosh(z) = cosh(x+iy)
+ *         = cosh(x) cos(y) + i sinh(x) sin(y).
+ *
+ * Exceptional values are noted in the comments within the source code.
+ * These values and the return value were taken from n1124.pdf.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+#include <math.h>
+
+#include "math_private.h"
+
+static const double huge = 0x1p1023;
+
+double complex
+ccosh(double complex z)
+{
+	double x, y, h;
+	int32_t hx, hy, ix, iy, lx, ly;
+
+	x = creal(z);
+	y = cimag(z);
+
+	EXTRACT_WORDS(hx, lx, x);
+	EXTRACT_WORDS(hy, ly, y);
+
+	ix = 0x7fffffff & hx;
+	iy = 0x7fffffff & hy;
+
+	/* Handle the nearly-non-exceptional cases where x and y are finite. */
+	if (ix < 0x7ff00000 && iy < 0x7ff00000) {
+		if ((iy | ly) == 0)
+			return (cpack(cosh(x), x * y));
+		if (ix < 0x40360000)	/* small x: normal case */
+			return (cpack(cosh(x) * cos(y), sinh(x) * sin(y)));
+
+		/* |x| >= 22, so cosh(x) ~= exp(|x|) */
+		if (ix < 0x40862e42) {
+			/* x < 710: exp(|x|) won't overflow */
+			h = exp(fabs(x)) * 0.5;
+			return (cpack(h * cos(y), copysign(h, x) * sin(y)));
+		} else if (ix < 0x4096bbaa) {
+			/* x < 1455: scale to avoid overflow */
+			z = __ldexp_cexp(cpack(fabs(x), y), -1);
+			return (cpack(creal(z), cimag(z) * copysign(1, x)));
+		} else {
+			/* x >= 1455: the result always overflows */
+			h = huge * x;
+			return (cpack(h * h * cos(y), h * sin(y)));
+		}
+	}
+
+	/*
+	 * cosh(+-0 +- I Inf) = dNaN + I sign(d(+-0, dNaN))0.
+	 * The sign of 0 in the result is unspecified.  Choice = normally
+	 * the same as dNaN.  Raise the invalid floating-point exception.
+	 *
+	 * cosh(+-0 +- I NaN) = d(NaN) + I sign(d(+-0, NaN))0.
+	 * The sign of 0 in the result is unspecified.  Choice = normally
+	 * the same as d(NaN).
+	 */
+	if ((ix | lx) == 0 && iy >= 0x7ff00000)
+		return (cpack(y - y, copysign(0, x * (y - y))));
+
+	/*
+	 * cosh(+-Inf +- I 0) = +Inf + I (+-)(+-)0.
+	 *
+	 * cosh(NaN +- I 0)   = d(NaN) + I sign(d(NaN, +-0))0.
+	 * The sign of 0 in the result is unspecified.
+	 */
+	if ((iy | ly) == 0 && ix >= 0x7ff00000) {
+		if (((hx & 0xfffff) | lx) == 0)
+			return (cpack(x * x, copysign(0, x) * y));
+		return (cpack(x * x, copysign(0, (x + x) * y)));
+	}
+
+	/*
+	 * cosh(x +- I Inf) = dNaN + I dNaN.
+	 * Raise the invalid floating-point exception for finite nonzero x.
+	 *
+	 * cosh(x + I NaN) = d(NaN) + I d(NaN).
+	 * Optionally raises the invalid floating-point exception for finite
+	 * nonzero x.  Choice = don't raise (except for signaling NaNs).
+	 */
+	if (ix < 0x7ff00000 && iy >= 0x7ff00000)
+		return (cpack(y - y, x * (y - y)));
+
+	/*
+	 * cosh(+-Inf + I NaN)  = +Inf + I d(NaN).
+	 *
+	 * cosh(+-Inf +- I Inf) = +Inf + I dNaN.
+	 * The sign of Inf in the result is unspecified.  Choice = always +.
+	 * Raise the invalid floating-point exception.
+	 *
+	 * cosh(+-Inf + I y)   = +Inf cos(y) +- I Inf sin(y)
+	 */
+	if (ix >= 0x7ff00000 && ((hx & 0xfffff) | lx) == 0) {
+		if (iy >= 0x7ff00000)
+			return (cpack(x * x, x * (y - y)));
+		return (cpack((x * x) * cos(y), x * sin(y)));
+	}
+
+	/*
+	 * cosh(NaN + I NaN)  = d(NaN) + I d(NaN).
+	 *
+	 * cosh(NaN +- I Inf) = d(NaN) + I d(NaN).
+	 * Optionally raises the invalid floating-point exception.
+	 * Choice = raise.
+	 *
+	 * cosh(NaN + I y)    = d(NaN) + I d(NaN).
+	 * Optionally raises the invalid floating-point exception for finite
+	 * nonzero y.  Choice = don't raise (except for signaling NaNs).
+	 */
+	return (cpack((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+double complex
+ccos(double complex z)
+{
+
+	/* ccos(z) = ccosh(I * z) */
+	return (ccosh(cpack(-cimag(z), creal(z))));
+}
diff --git a/lib/msun/src/s_ccoshf.c b/lib/msun/src/s_ccoshf.c
new file mode 100644
index 000000000000..1de9ad443772
--- /dev/null
+++ b/lib/msun/src/s_ccoshf.c
@@ -0,0 +1,104 @@
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Hyperbolic cosine of a complex argument.  See s_ccosh.c for details.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+#include <math.h>
+
+#include "math_private.h"
+
+static const float huge = 0x1p127;
+
+float complex
+ccoshf(float complex z)
+{
+	float x, y, h;
+	int32_t hx, hy, ix, iy;
+
+	x = crealf(z);
+	y = cimagf(z);
+
+	GET_FLOAT_WORD(hx, x);
+	GET_FLOAT_WORD(hy, y);
+
+	ix = 0x7fffffff & hx;
+	iy = 0x7fffffff & hy;
+
+	if (ix < 0x7f800000 && iy < 0x7f800000) {
+		if (iy == 0)
+			return (cpackf(coshf(x), x * y));
+		if (ix < 0x41100000)	/* small x: normal case */
+			return (cpackf(coshf(x) * cosf(y), sinhf(x) * sinf(y)));
+
+		/* |x| >= 9, so cosh(x) ~= exp(|x|) */
+		if (ix < 0x42b17218) {
+			/* x < 88.7: expf(|x|) won't overflow */
+			h = expf(fabsf(x)) * 0.5f;
+			return (cpackf(h * cosf(y), copysignf(h, x) * sinf(y)));
+		} else if (ix < 0x4340b1e7) {
+			/* x < 192.7: scale to avoid overflow */
+			z = __ldexp_cexpf(cpackf(fabsf(x), y), -1);
+			return (cpackf(crealf(z), cimagf(z) * copysignf(1, x)));
+		} else {
+			/* x >= 192.7: the result always overflows */
+			h = huge * x;
+			return (cpackf(h * h * cosf(y), h * sinf(y)));
+		}
+	}
+
+	if (ix == 0 && iy >= 0x7f800000)
+		return (cpackf(y - y, copysignf(0, x * (y - y))));
+
+	if (iy == 0 && ix >= 0x7f800000) {
+		if ((hx & 0x7fffff) == 0)
+			return (cpackf(x * x, copysignf(0, x) * y));
+		return (cpackf(x * x, copysignf(0, (x + x) * y)));
+	}
+
+	if (ix < 0x7f800000 && iy >= 0x7f800000)
+		return (cpackf(y - y, x * (y - y)));
+
+	if (ix >= 0x7f800000 && (hx & 0x7fffff) == 0) {
+		if (iy >= 0x7f800000)
+			return (cpackf(x * x, x * (y - y)));
+		return (cpackf((x * x) * cosf(y), x * sinf(y)));
+	}
+
+	return (cpackf((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+float complex
+ccosf(float complex z)
+{
+
+	return (ccoshf(cpackf(-cimagf(z), crealf(z))));
+}
diff --git a/lib/msun/src/s_cexp.c b/lib/msun/src/s_cexp.c
index ecf0992dc21d..abe178f3169f 100644
--- a/lib/msun/src/s_cexp.c
+++ b/lib/msun/src/s_cexp.c
@@ -34,18 +34,13 @@ __FBSDID("$FreeBSD$");
 
 static const uint32_t
 exp_ovfl  = 0x40862e42,			/* high bits of MAX_EXP * ln2 ~= 710 */
-cexp_ovfl = 0x4096b8e4,			/* (MAX_EXP - MIN_DENORM_EXP) * ln2 */
-k         = 1799;			/* constant for reduction */
-
-static const double
-kln2      =  1246.97177782734161156;	/* k * ln2 */
+cexp_ovfl = 0x4096b8e4;			/* (MAX_EXP - MIN_DENORM_EXP) * ln2 */
 
 double complex
 cexp(double complex z)
 {
 	double x, y, exp_x;
 	uint32_t hx, hy, lx, ly;
-	int scale;
 
 	x = creal(z);
 	y = cimag(z);
@@ -56,8 +51,12 @@ cexp(double complex z)
 	/* cexp(x + I 0) = exp(x) + I 0 */
 	if ((hy | ly) == 0)
 		return (cpack(exp(x), y));
+	EXTRACT_WORDS(hx, lx, x);
+	/* cexp(0 + I y) = cos(y) + I sin(y) */
+	if (((hx & 0x7fffffff) | lx) == 0)
+		return (cpack(cos(y), sin(y)));
+
 	if (hy >= 0x7ff00000) {
-		EXTRACT_WORDS(hx, lx, x);
 		if (lx != 0 || (hx & 0x7fffffff) != 0x7ff00000) {
 			/* cexp(finite|NaN +- I Inf|NaN) = NaN + I NaN */
 			return (cpack(y - y, y - y));
@@ -70,21 +69,12 @@ cexp(double complex z)
 		}
 	}
 
-	GET_HIGH_WORD(hx, x);
 	if (hx >= exp_ovfl && hx <= cexp_ovfl) {
 		/*
 		 * x is between 709.7 and 1454.3, so we must scale to avoid
-		 * overflow in exp(x).  We use exp(x) = exp(x - kln2) * 2**k,
-		 * carefully chosen to minimize |exp(kln2) - 2**k|.  We also
-		 * scale the exponent of exp(x) to MANT_DIG to avoid loss of
-		 * accuracy due to underflow if sin(y) is tiny.
+		 * overflow in exp(x).
 		 */
-		exp_x = exp(x - kln2);
-		GET_HIGH_WORD(hx, exp_x);
-		SET_HIGH_WORD(exp_x, (hx & 0xfffff) | ((0x3ff + 52) << 20));
-		scale = (hx >> 20) - (0x3ff + 52) + k;
-		return (cpack(scalbn(cos(y) * exp_x, scale),
-			scalbn(sin(y) * exp_x, scale)));
+		return (__ldexp_cexp(z, 0));
 	} else {
 		/*
 		 * Cases covered here:
diff --git a/lib/msun/src/s_cexpf.c b/lib/msun/src/s_cexpf.c
index 4ea39312c040..0e30d08c80db 100644
--- a/lib/msun/src/s_cexpf.c
+++ b/lib/msun/src/s_cexpf.c
@@ -34,18 +34,13 @@ __FBSDID("$FreeBSD$");
 
 static const uint32_t
 exp_ovfl  = 0x42b17218,		/* MAX_EXP * ln2 ~= 88.722839355 */
-cexp_ovfl = 0x43400074,		/* (MAX_EXP - MIN_DENORM_EXP) * ln2 */
-k         = 235;		/* constant for reduction */
-
-static const float
-kln2      =  162.88958740f;	/* k * ln2 */
+cexp_ovfl = 0x43400074;		/* (MAX_EXP - MIN_DENORM_EXP) * ln2 */
 
 float complex
 cexpf(float complex z)
 {
 	float x, y, exp_x;
 	uint32_t hx, hy;
-	int scale;
 
 	x = crealf(z);
 	y = cimagf(z);
@@ -57,6 +52,10 @@ cexpf(float complex z)
 	if (hy == 0)
 		return (cpackf(expf(x), y));
 	GET_FLOAT_WORD(hx, x);
+	/* cexp(0 + I y) = cos(y) + I sin(y) */
+	if ((hx & 0x7fffffff) == 0)
+		return (cpackf(cosf(y), sinf(y)));
+
 	if (hy >= 0x7f800000) {
 		if ((hx & 0x7fffffff) != 0x7f800000) {
 			/* cexp(finite|NaN +- I Inf|NaN) = NaN + I NaN */
@@ -73,17 +72,9 @@ cexpf(float complex z)
 	if (hx >= exp_ovfl && hx <= cexp_ovfl) {
 		/*
 		 * x is between 88.7 and 192, so we must scale to avoid
-		 * overflow in expf(x).  We use exp(x) = exp(x - kln2) * 2**k,
-		 * carefully chosen to minimize |exp(kln2) - 2**k|.  We also
-		 * scale the exponent of exp(x) to MANT_DIG to avoid loss of
-		 * accuracy due to underflow if sin(y) is tiny.
+		 * overflow in expf(x).
 		 */
-		exp_x = expf(x - kln2);
-		GET_FLOAT_WORD(hx, exp_x);
-		SET_FLOAT_WORD(exp_x, (hx & 0x7fffff) | ((0x7f + 23) << 23));
-		scale = (hx >> 23) - (0x7f + 23) + k;
-		return (cpackf(scalbnf(cosf(y) * exp_x, scale),
-			scalbnf(sinf(y) * exp_x, scale)));
+		return (__ldexp_cexpf(z, 0));
 	} else {
 		/*
 		 * Cases covered here:
diff --git a/lib/msun/src/s_csinh.c b/lib/msun/src/s_csinh.c
new file mode 100644
index 000000000000..c192f30308e0
--- /dev/null
+++ b/lib/msun/src/s_csinh.c
@@ -0,0 +1,157 @@
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Hyperbolic sine of a complex argument z = x + i y.
+ *
+ * sinh(z) = sinh(x+iy)
+ *         = sinh(x) cos(y) + i cosh(x) sin(y).
+ *
+ * Exceptional values are noted in the comments within the source code.
+ * These values and the return value were taken from n1124.pdf.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+#include <math.h>
+
+#include "math_private.h"
+
+static const double huge = 0x1p1023;
+
+double complex
+csinh(double complex z)
+{
+	double x, y, h;
+	int32_t hx, hy, ix, iy, lx, ly;
+
+	x = creal(z);
+	y = cimag(z);
+
+	EXTRACT_WORDS(hx, lx, x);
+	EXTRACT_WORDS(hy, ly, y);
+
+	ix = 0x7fffffff & hx;
+	iy = 0x7fffffff & hy;
+
+	/* Handle the nearly-non-exceptional cases where x and y are finite. */
+	if (ix < 0x7ff00000 && iy < 0x7ff00000) {
+		if ((iy | ly) == 0)
+			return (cpack(sinh(x), y));
+		if (ix < 0x40360000)	/* small x: normal case */
+			return (cpack(sinh(x) * cos(y), cosh(x) * sin(y)));
+
+		/* |x| >= 22, so cosh(x) ~= exp(|x|) */
+		if (ix < 0x40862e42) {
+			/* x < 710: exp(|x|) won't overflow */
+			h = exp(fabs(x)) * 0.5;
+			return (cpack(copysign(h, x) * cos(y), h * sin(y)));
+		} else if (ix < 0x4096bbaa) {
+			/* x < 1455: scale to avoid overflow */
+			z = __ldexp_cexp(cpack(fabs(x), y), -1);
+			return (cpack(creal(z) * copysign(1, x), cimag(z)));
+		} else {
+			/* x >= 1455: the result always overflows */
+			h = huge * x;
+			return (cpack(h * cos(y), h * h * sin(y)));
+		}
+	}
+
+	/*
+	 * sinh(+-0 +- I Inf) = sign(d(+-0, dNaN))0 + I dNaN.
+	 * The sign of 0 in the result is unspecified.  Choice = normally
+	 * the same as dNaN.  Raise the invalid floating-point exception.
+	 *
+	 * sinh(+-0 +- I NaN) = sign(d(+-0, NaN))0 + I d(NaN).
+	 * The sign of 0 in the result is unspecified.  Choice = normally
+	 * the same as d(NaN).
+	 */
+	if ((ix | lx) == 0 && iy >= 0x7ff00000)
+		return (cpack(copysign(0, x * (y - y)), y - y));
+
+	/*
+	 * sinh(+-Inf +- I 0) = +-Inf + I +-0.
+	 *
+	 * sinh(NaN +- I 0)   = d(NaN) + I +-0.
+	 */
+	if ((iy | ly) == 0 && ix >= 0x7ff00000) {
+		if (((hx & 0xfffff) | lx) == 0)
+			return (cpack(x, y));
+		return (cpack(x, copysign(0, y)));
+	}
+
+	/*
+	 * sinh(x +- I Inf) = dNaN + I dNaN.
+	 * Raise the invalid floating-point exception for finite nonzero x.
+	 *
+	 * sinh(x + I NaN) = d(NaN) + I d(NaN).
+	 * Optionally raises the invalid floating-point exception for finite
+	 * nonzero x.  Choice = don't raise (except for signaling NaNs).
+	 */
+	if (ix < 0x7ff00000 && iy >= 0x7ff00000)
+		return (cpack(y - y, x * (y - y)));
+
+	/*
+	 * sinh(+-Inf + I NaN)  = +-Inf + I d(NaN).
+	 * The sign of Inf in the result is unspecified.  Choice = normally
+	 * the same as d(NaN).
+	 *
+	 * sinh(+-Inf +- I Inf) = +Inf + I dNaN.
+	 * The sign of Inf in the result is unspecified.  Choice = always +.
+	 * Raise the invalid floating-point exception.
+	 *
+	 * sinh(+-Inf + I y)   = +-Inf cos(y) + I Inf sin(y)
+	 */
+	if (ix >= 0x7ff00000 && ((hx & 0xfffff) | lx) == 0) {
+		if (iy >= 0x7ff00000)
+			return (cpack(x * x, x * (y - y)));
+		return (cpack(x * cos(y), INFINITY * sin(y)));
+	}
+
+	/*
+	 * sinh(NaN + I NaN)  = d(NaN) + I d(NaN).
+	 *
+	 * sinh(NaN +- I Inf) = d(NaN) + I d(NaN).
+	 * Optionally raises the invalid floating-point exception.
+	 * Choice = raise.
+	 *
+	 * sinh(NaN + I y)    = d(NaN) + I d(NaN).
+	 * Optionally raises the invalid floating-point exception for finite
+	 * nonzero y.  Choice = don't raise (except for signaling NaNs).
+	 */
+	return (cpack((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+double complex
+csin(double complex z)
+{
+
+	/* csin(z) = -I * csinh(I * z) */
+	z = csinh(cpack(-cimag(z), creal(z)));
+	return (cpack(cimag(z), -creal(z)));
+}
diff --git a/lib/msun/src/s_csinhf.c b/lib/msun/src/s_csinhf.c
new file mode 100644
index 000000000000..c5231251fade
--- /dev/null
+++ b/lib/msun/src/s_csinhf.c
@@ -0,0 +1,105 @@
+/*-
+ * Copyright (c) 2005 Bruce D. Evans and Steven G. Kargl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Hyperbolic sine of a complex argument z.  See s_csinh.c for details.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+#include <math.h>
+
+#include "math_private.h"
+
+static const float huge = 0x1p127;
+
+float complex
+csinhf(float complex z)
+{
+	float x, y, h;
+	int32_t hx, hy, ix, iy;
+
+	x = crealf(z);
+	y = cimagf(z);
+
+	GET_FLOAT_WORD(hx, x);
+	GET_FLOAT_WORD(hy, y);
+
+	ix = 0x7fffffff & hx;
+	iy = 0x7fffffff & hy;
+
+	if (ix < 0x7f800000 && iy < 0x7f800000) {
+		if (iy == 0)
+			return (cpackf(sinhf(x), y));
+		if (ix < 0x41100000)	/* small x: normal case */
+			return (cpackf(sinhf(x) * cosf(y), coshf(x) * sinf(y)));
+
+		/* |x| >= 9, so cosh(x) ~= exp(|x|) */
+		if (ix < 0x42b17218) {
+			/* x < 88.7: expf(|x|) won't overflow */
+			h = expf(fabsf(x)) * 0.5f;
+			return (cpackf(copysignf(h, x) * cosf(y), h * sinf(y)));
+		} else if (ix < 0x4340b1e7) {
+			/* x < 192.7: scale to avoid overflow */
+			z = __ldexp_cexpf(cpackf(fabsf(x), y), -1);
+			return (cpackf(crealf(z) * copysignf(1, x), cimagf(z)));
+		} else {
+			/* x >= 192.7: the result always overflows */
+			h = huge * x;
+			return (cpackf(h * cosf(y), h * h * sinf(y)));
+		}
+	}
+
+	if (ix == 0 && iy >= 0x7f800000)
+		return (cpackf(copysignf(0, x * (y - y)), y - y));
+
+	if (iy == 0 && ix >= 0x7f800000) {
+		if ((hx & 0x7fffff) == 0)
+			return (cpackf(x, y));
+		return (cpackf(x, copysignf(0, y)));
+	}
+
+	if (ix < 0x7f800000 && iy >= 0x7f800000)
+		return (cpackf(y - y, x * (y - y)));
+
+	if (ix >= 0x7f800000 && (hx & 0x7fffff) == 0) {
+		if (iy >= 0x7f800000)
+			return (cpackf(x * x, x * (y - y)));
+		return (cpackf(x * cosf(y), INFINITY * sinf(y)));
+	}
+
+	return (cpackf((x * x) * (y - y), (x + x) * (y - y)));
+}
+
+float complex
+csinf(float complex z)
+{
+
+	z = csinhf(cpackf(-cimagf(z), crealf(z)));
+	return (cpackf(cimagf(z), -crealf(z)));
+}
diff --git a/lib/msun/src/s_ctanh.c b/lib/msun/src/s_ctanh.c
new file mode 100644
index 000000000000..d427e28d7281
--- /dev/null
+++ b/lib/msun/src/s_ctanh.c
@@ -0,0 +1,144 @@
+/*-
+ * Copyright (c) 2011 David Schultz
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Hyperbolic tangent of a complex argument z = x + i y.
+ *
+ * The algorithm is from:
+ *
+ *   W. Kahan.  Branch Cuts for Complex Elementary Functions or Much
+ *   Ado About Nothing's Sign Bit.  In The State of the Art in
+ *   Numerical Analysis, pp. 165 ff.  Iserles and Powell, eds., 1987.
+ *
+ * Method:
+ *
+ *   Let t    = tan(x)
+ *       beta = 1/cos^2(y)
+ *       s    = sinh(x)
+ *       rho  = cosh(x)
+ *
+ *   We have:
+ *
+ *   tanh(z) = sinh(z) / cosh(z)
+ *
+ *             sinh(x) cos(y) + i cosh(x) sin(y)
+ *           = ---------------------------------
+ *             cosh(x) cos(y) + i sinh(x) sin(y)
+ *
+ *             cosh(x) sinh(x) / cos^2(y) + i tan(y)
+ *           = -------------------------------------
+ *                    1 + sinh^2(x) / cos^2(y)
+ *
+ *             beta rho s + i t
+ *           = ----------------
+ *               1 + beta s^2
+ *
+ * Modifications:
+ *
+ *   I omitted the original algorithm's handling of overflow in tan(x) after
+ *   verifying with nearpi.c that this can't happen in IEEE single or double
+ *   precision.  I also handle large x differently.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+#include <math.h>
+
+#include "math_private.h"
+
+double complex
+ctanh(double complex z)
+{
+	double x, y;
+	double t, beta, s, rho, denom;
+	uint32_t hx, ix, lx;
+
+	x = creal(z);
+	y = cimag(z);
+
+	EXTRACT_WORDS(hx, lx, x);
+	ix = hx & 0x7fffffff;
+
+	/*
+	 * ctanh(NaN + i 0) = NaN + i 0
+	 *
+	 * ctanh(NaN + i y) = NaN + i NaN		for y != 0
+	 *
+	 * The imaginary part has the sign of x*sin(2*y), but there's no
+	 * special effort to get this right.
+	 *
+	 * ctanh(+-Inf +- i Inf) = +-1 +- 0
+	 *
+	 * ctanh(+-Inf + i y) = +-1 + 0 sin(2y)		for y finite
+	 *
+	 * The imaginary part of the sign is unspecified.  This special
+	 * case is only needed to avoid a spurious invalid exception when
+	 * y is infinite.
+	 */
+	if (ix >= 0x7ff00000) {
+		if ((ix & 0xfffff) | lx)	/* x is NaN */
+			return (cpack(x, (y == 0 ? y : x * y)));
+		SET_HIGH_WORD(x, hx - 0x40000000);	/* x = copysign(1, x) */
+		return (cpack(x, copysign(0, isinf(y) ? y : sin(y) * cos(y))));
+	}
+
+	/*
+	 * ctanh(x + i NAN) = NaN + i NaN
+	 * ctanh(x +- i Inf) = NaN + i NaN
+	 */
+	if (!isfinite(y))
+		return (cpack(y - y, y - y));
+
+	/*
+	 * ctanh(+-huge + i +-y) ~= +-1 +- i 2sin(2y)/exp(2x), using the
+	 * approximation sinh^2(huge) ~= exp(2*huge) / 4.
+	 * We use a modified formula to avoid spurious overflow.
+	 */
+	if (ix >= 0x40360000) {	/* x >= 22 */
+		double exp_mx = exp(-fabs(x));
+		return (cpack(copysign(1, x),
+		    4 * sin(y) * cos(y) * exp_mx * exp_mx));
+	}
+
+	/* Kahan's algorithm */
+	t = tan(y);
+	beta = 1.0 + t * t;	/* = 1 / cos^2(y) */
+	s = sinh(x);
+	rho = sqrt(1 + s * s);	/* = cosh(x) */
+	denom = 1 + beta * s * s;
+	return (cpack((beta * rho * s) / denom, t / denom));
+}
+
+double complex
+ctan(double complex z)
+{
+
+	/* ctan(z) = -I * ctanh(I * z) */
+	z = ctanh(cpack(-cimag(z), creal(z)));
+	return (cpack(cimag(z), -creal(z)));
+}
diff --git a/lib/msun/src/s_ctanhf.c b/lib/msun/src/s_ctanhf.c
new file mode 100644
index 000000000000..4be28d8526d1
--- /dev/null
+++ b/lib/msun/src/s_ctanhf.c
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2011 David Schultz
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Hyperbolic tangent of a complex argument z.  See s_ctanh.c for details.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <complex.h>
+#include <math.h>
+
+#include "math_private.h"
+
+float complex
+ctanhf(float complex z)
+{
+	float x, y;
+	float t, beta, s, rho, denom;
+	uint32_t hx, ix;
+
+	x = crealf(z);
+	y = cimagf(z);
+
+	GET_FLOAT_WORD(hx, x);
+	ix = hx & 0x7fffffff;
+
+	if (ix >= 0x7f800000) {
+		if (ix & 0x7fffff)
+			return (cpackf(x, (y == 0 ? y : x * y)));
+		SET_FLOAT_WORD(x, hx - 0x40000000);
+		return (cpackf(x,
+		    copysignf(0, isinf(y) ? y : sinf(y) * cosf(y))));
+	}
+
+	if (!isfinite(y))
+		return (cpackf(y - y, y - y));
+
+	if (ix >= 0x41300000) {	/* x >= 11 */
+		float exp_mx = expf(-fabsf(x));
+		return (cpackf(copysignf(1, x),
+		    4 * sinf(y) * cosf(y) * exp_mx * exp_mx));
+	}
+
+	t = tanf(y);
+	beta = 1.0 + t * t;
+	s = sinhf(x);
+	rho = sqrtf(1 + s * s);
+	denom = 1 + beta * s * s;
+	return (cpackf((beta * rho * s) / denom, t / denom));
+}
+
+float complex
+ctanf(float complex z)
+{
+
+	z = ctanhf(cpackf(-cimagf(z), crealf(z)));
+	return (cpackf(cimagf(z), -crealf(z)));
+}
+
diff --git a/lib/msun/src/s_expm1.c b/lib/msun/src/s_expm1.c
index 3de7bfbc837a..5aa1917b5a4c 100644
--- a/lib/msun/src/s_expm1.c
+++ b/lib/msun/src/s_expm1.c
@@ -108,6 +108,8 @@ __FBSDID("$FreeBSD$");
  * to produce the hexadecimal values shown.
  */
 
+#include <float.h>
+
 #include "math.h"
 #include "math_private.h"
 
@@ -135,7 +137,6 @@ expm1(double x)
 
 	GET_HIGH_WORD(hx,x);
 	xsb = hx&0x80000000;		/* sign bit of x */
-	if(xsb==0) y=x; else y= -x;	/* y = |x| */
 	hx &= 0x7fffffff;		/* high word of |x| */
 
     /* filter out huge and non-finite argument */
@@ -169,7 +170,7 @@ expm1(double x)
 		hi = x - t*ln2_hi;	/* t*ln2_hi is exact here */
 		lo = t*ln2_lo;
 	    }
-	    x  = hi - lo;
+	    STRICT_ASSIGN(double, x, hi - lo);
 	    c  = (hi-x)-lo;
 	}
 	else if(hx < 0x3c900000) {  	/* when |x|<2**-54, return x */
diff --git a/lib/msun/src/s_expm1f.c b/lib/msun/src/s_expm1f.c
index 483472ce69b8..fb374943000e 100644
--- a/lib/msun/src/s_expm1f.c
+++ b/lib/msun/src/s_expm1f.c
@@ -16,6 +16,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <float.h>
+
 #include "math.h"
 #include "math_private.h"
 
@@ -44,7 +46,6 @@ expm1f(float x)
 
 	GET_FLOAT_WORD(hx,x);
 	xsb = hx&0x80000000;		/* sign bit of x */
-	if(xsb==0) y=x; else y= -x;	/* y = |x| */
 	hx &= 0x7fffffff;		/* high word of |x| */
 
     /* filter out huge and non-finite argument */
@@ -75,7 +76,7 @@ expm1f(float x)
 		hi = x - t*ln2_hi;	/* t*ln2_hi is exact here */
 		lo = t*ln2_lo;
 	    }
-	    x  = hi - lo;
+	    STRICT_ASSIGN(float, x, hi - lo);
 	    c  = (hi-x)-lo;
 	}
 	else if(hx < 0x33000000) {  	/* when |x|<2**-25, return x */
diff --git a/lib/msun/src/s_fma.c b/lib/msun/src/s_fma.c
index ad1fc4aef49d..dfbd13c46815 100644
--- a/lib/msun/src/s_fma.c
+++ b/lib/msun/src/s_fma.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2005 David Schultz <das@FreeBSD.ORG>
+ * Copyright (c) 2005-2011 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,132 @@ __FBSDID("$FreeBSD$");
 #include <float.h>
 #include <math.h>
 
+#include "math_private.h"
+
+/*
+ * A struct dd represents a floating-point number with twice the precision
+ * of a double.  We maintain the invariant that "hi" stores the 53 high-order
+ * bits of the result.
+ */
+struct dd {
+	double hi;
+	double lo;
+};
+
+/*
+ * Compute a+b exactly, returning the exact result in a struct dd.  We assume
+ * that both a and b are finite, but make no assumptions about their relative
+ * magnitudes.
+ */
+static inline struct dd
+dd_add(double a, double b)
+{
+	struct dd ret;
+	double s;
+
+	ret.hi = a + b;
+	s = ret.hi - a;
+	ret.lo = (a - (ret.hi - s)) + (b - s);
+	return (ret);
+}
+
+/*
+ * Compute a+b, with a small tweak:  The least significant bit of the
+ * result is adjusted into a sticky bit summarizing all the bits that
+ * were lost to rounding.  This adjustment negates the effects of double
+ * rounding when the result is added to another number with a higher
+ * exponent.  For an explanation of round and sticky bits, see any reference
+ * on FPU design, e.g.,
+ *
+ *     J. Coonen.  An Implementation Guide to a Proposed Standard for
+ *     Floating-Point Arithmetic.  Computer, vol. 13, no. 1, Jan 1980.
+ */
+static inline double
+add_adjusted(double a, double b)
+{
+	struct dd sum;
+	uint64_t hibits, lobits;
+
+	sum = dd_add(a, b);
+	if (sum.lo != 0) {
+		EXTRACT_WORD64(hibits, sum.hi);
+		if ((hibits & 1) == 0) {
+			/* hibits += (int)copysign(1.0, sum.hi * sum.lo) */
+			EXTRACT_WORD64(lobits, sum.lo);
+			hibits += 1 - ((hibits ^ lobits) >> 62);
+			INSERT_WORD64(sum.hi, hibits);
+		}
+	}
+	return (sum.hi);
+}
+
+/*
+ * Compute ldexp(a+b, scale) with a single rounding error. It is assumed
+ * that the result will be subnormal, and care is taken to ensure that
+ * double rounding does not occur.
+ */
+static inline double
+add_and_denormalize(double a, double b, int scale)
+{
+	struct dd sum;
+	uint64_t hibits, lobits;
+	int bits_lost;
+
+	sum = dd_add(a, b);
+
+	/*
+	 * If we are losing at least two bits of accuracy to denormalization,
+	 * then the first lost bit becomes a round bit, and we adjust the
+	 * lowest bit of sum.hi to make it a sticky bit summarizing all the
+	 * bits in sum.lo. With the sticky bit adjusted, the hardware will
+	 * break any ties in the correct direction.
+	 *
+	 * If we are losing only one bit to denormalization, however, we must
+	 * break the ties manually.
+	 */
+	if (sum.lo != 0) {
+		EXTRACT_WORD64(hibits, sum.hi);
+		bits_lost = -((int)(hibits >> 52) & 0x7ff) - scale + 1;
+		if (bits_lost != 1 ^ (int)(hibits & 1)) {
+			/* hibits += (int)copysign(1.0, sum.hi * sum.lo) */
+			EXTRACT_WORD64(lobits, sum.lo);
+			hibits += 1 - (((hibits ^ lobits) >> 62) & 2);
+			INSERT_WORD64(sum.hi, hibits);
+		}
+	}
+	return (ldexp(sum.hi, scale));
+}
+
+/*
+ * Compute a*b exactly, returning the exact result in a struct dd.  We assume
+ * that both a and b are normalized, so no underflow or overflow will occur.
+ * The current rounding mode must be round-to-nearest.
+ */
+static inline struct dd
+dd_mul(double a, double b)
+{
+	static const double split = 0x1p27 + 1.0;
+	struct dd ret;
+	double ha, hb, la, lb, p, q;
+
+	p = a * split;
+	ha = a - p;
+	ha += p;
+	la = a - ha;
+
+	p = b * split;
+	hb = b - p;
+	hb += p;
+	lb = b - hb;
+
+	p = ha * hb;
+	q = ha * lb + la * hb;
+
+	ret.hi = p + q;
+	ret.lo = p - ret.hi + q + la * lb;
+	return (ret);
+}
+
 /*
  * Fused multiply-add: Compute x * y + z with a single rounding error.
  *
@@ -48,14 +174,11 @@ __FBSDID("$FreeBSD$");
  * Hardware instructions should be used on architectures that support it,
  * since this implementation will likely be several times slower.
  */
-#if LDBL_MANT_DIG != 113
 double
 fma(double x, double y, double z)
 {
-	static const double split = 0x1p27 + 1.0;
-	double xs, ys, zs;
-	double c, cc, hx, hy, p, q, tx, ty;
-	double r, rr, s;
+	double xs, ys, zs, adj;
+	struct dd xy, r;
 	int oround;
 	int ex, ey, ez;
 	int spread;
@@ -85,41 +208,6 @@ fma(double x, double y, double z)
 	 * will overflow, so we handle these cases specially.  Rounding
 	 * modes other than FE_TONEAREST are painful.
 	 */
-	if (spread > DBL_MANT_DIG * 2) {
-		fenv_t env;
-		feraiseexcept(FE_INEXACT);
-		switch(oround) {
-		case FE_TONEAREST:
-			return (x * y);
-		case FE_TOWARDZERO:
-			if (x > 0.0 ^ y < 0.0 ^ z < 0.0)
-				return (x * y);
-			feholdexcept(&env);
-			r = x * y;
-			if (!fetestexcept(FE_INEXACT))
-				r = nextafter(r, 0);
-			feupdateenv(&env);
-			return (r);
-		case FE_DOWNWARD:
-			if (z > 0.0)
-				return (x * y);
-			feholdexcept(&env);
-			r = x * y;
-			if (!fetestexcept(FE_INEXACT))
-				r = nextafter(r, -INFINITY);
-			feupdateenv(&env);
-			return (r);
-		default:	/* FE_UPWARD */
-			if (z < 0.0)
-				return (x * y);
-			feholdexcept(&env);
-			r = x * y;
-			if (!fetestexcept(FE_INEXACT))
-				r = nextafter(r, INFINITY);
-			feupdateenv(&env);
-			return (r);
-		}
-	}
 	if (spread < -DBL_MANT_DIG) {
 		feraiseexcept(FE_INEXACT);
 		if (!isnormal(z))
@@ -144,63 +232,52 @@ fma(double x, double y, double z)
 				return (z);
 		}
 	}
+	if (spread <= DBL_MANT_DIG * 2)
+		zs = ldexp(zs, -spread);
+	else
+		zs = copysign(DBL_MIN, zs);
 
-	/*
-	 * Use Dekker's algorithm to perform the multiplication and
-	 * subsequent addition in twice the machine precision.
-	 * Arrange so that x * y = c + cc, and x * y + z = r + rr.
-	 */
 	fesetround(FE_TONEAREST);
 
-	p = xs * split;
-	hx = xs - p;
-	hx += p;
-	tx = xs - hx;
-
-	p = ys * split;
-	hy = ys - p;
-	hy += p;
-	ty = ys - hy;
-
-	p = hx * hy;
-	q = hx * ty + tx * hy;
-	c = p + q;
-	cc = p - c + q + tx * ty;
-
-	zs = ldexp(zs, -spread);
-	r = c + zs;
-	s = r - c;
-	rr = (c - (r - s)) + (zs - s) + cc;
+	/*
+	 * Basic approach for round-to-nearest:
+	 *
+	 *     (xy.hi, xy.lo) = x * y		(exact)
+	 *     (r.hi, r.lo)   = xy.hi + z	(exact)
+	 *     adj = xy.lo + r.lo		(inexact; low bit is sticky)
+	 *     result = r.hi + adj		(correctly rounded)
+	 */
+	xy = dd_mul(xs, ys);
+	r = dd_add(xy.hi, zs);
 
 	spread = ex + ey;
-	if (spread + ilogb(r) > -1023) {
-		fesetround(oround);
-		r = r + rr;
-	} else {
+
+	if (r.hi == 0.0) {
 		/*
-		 * The result is subnormal, so we round before scaling to
-		 * avoid double rounding.
+		 * When the addends cancel to 0, ensure that the result has
+		 * the correct sign.
 		 */
-		p = ldexp(copysign(0x1p-1022, r), -spread);
-		c = r + p;
-		s = c - r;
-		cc = (r - (c - s)) + (p - s) + rr;
 		fesetround(oround);
-		r = (c + cc) - p;
+		volatile double vzs = zs; /* XXX gcc CSE bug workaround */
+		return (xy.hi + vzs + ldexp(xy.lo, spread));
 	}
-	return (ldexp(r, spread));
+
+	if (oround != FE_TONEAREST) {
+		/*
+		 * There is no need to worry about double rounding in directed
+		 * rounding modes.
+		 */
+		fesetround(oround);
+		adj = r.lo + xy.lo;
+		return (ldexp(r.hi + adj, spread));
+	}
+
+	adj = add_adjusted(r.lo, xy.lo);
+	if (spread + ilogb(r.hi) > -1023)
+		return (ldexp(r.hi + adj, spread));
+	else
+		return (add_and_denormalize(r.hi, adj, spread));
 }
-#else	/* LDBL_MANT_DIG == 113 */
-/*
- * 113 bits of precision is more than twice the precision of a double,
- * so it is enough to represent the intermediate product exactly.
- */
-double
-fma(double x, double y, double z)
-{
-	return ((long double)x * y + z);
-}
-#endif	/* LDBL_MANT_DIG != 113 */
 
 #if (LDBL_MANT_DIG == 53)
 __weak_reference(fma, fmal);
diff --git a/lib/msun/src/s_fmaf.c b/lib/msun/src/s_fmaf.c
index 7c699e5c60aa..3695823f8314 100644
--- a/lib/msun/src/s_fmaf.c
+++ b/lib/msun/src/s_fmaf.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2005 David Schultz <das@FreeBSD.ORG>
+ * Copyright (c) 2005-2011 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -27,23 +27,43 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <fenv.h>
+
 #include "math.h"
+#include "math_private.h"
 
 /*
  * Fused multiply-add: Compute x * y + z with a single rounding error.
  *
  * A double has more than twice as much precision than a float, so
- * direct double-precision arithmetic suffices.
- *
- * XXX We are relying on the compiler to convert from double to float
- *     using the current rounding mode and with the appropriate
- *     side-effects.  But on at least one platform (gcc 3.4.2/sparc64),
- *     this appears to be too much to ask for.  The precision
- *     reduction should be done manually.
+ * direct double-precision arithmetic suffices, except where double
+ * rounding occurs.
  */
 float
 fmaf(float x, float y, float z)
 {
+	double xy, result;
+	uint32_t hr, lr;
 
-	return ((double)x * y + z);
+	xy = (double)x * y;
+	result = xy + z;
+	EXTRACT_WORDS(hr, lr, result);
+	/* Common case: The double precision result is fine. */
+	if ((lr & 0x1fffffff) != 0x10000000 ||	/* not a halfway case */
+	    (hr & 0x7ff00000) == 0x7ff00000 ||	/* NaN */
+	    result - xy == z ||			/* exact */
+	    fegetround() != FE_TONEAREST)	/* not round-to-nearest */
+		return (result);
+
+	/*
+	 * If result is inexact, and exactly halfway between two float values,
+	 * we need to adjust the low-order bit in the direction of the error.
+	 */
+	fesetround(FE_TOWARDZERO);
+	volatile double vxy = xy;  /* XXX work around gcc CSE bug */
+	double adjusted_result = vxy + z;
+	fesetround(FE_TONEAREST);
+	if (result == adjusted_result)
+		SET_LOW_WORD(adjusted_result, lr + 1);
+	return (adjusted_result);
 }
diff --git a/lib/msun/src/s_fmal.c b/lib/msun/src/s_fmal.c
index 4d5d1141b44b..c2a691350999 100644
--- a/lib/msun/src/s_fmal.c
+++ b/lib/msun/src/s_fmal.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2005 David Schultz <das@FreeBSD.ORG>
+ * Copyright (c) 2005-2011 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,6 +31,128 @@ __FBSDID("$FreeBSD$");
 #include <float.h>
 #include <math.h>
 
+#include "fpmath.h"
+
+/*
+ * A struct dd represents a floating-point number with twice the precision
+ * of a long double.  We maintain the invariant that "hi" stores the high-order
+ * bits of the result.
+ */
+struct dd {
+	long double hi;
+	long double lo;
+};
+
+/*
+ * Compute a+b exactly, returning the exact result in a struct dd.  We assume
+ * that both a and b are finite, but make no assumptions about their relative
+ * magnitudes.
+ */
+static inline struct dd
+dd_add(long double a, long double b)
+{
+	struct dd ret;
+	long double s;
+
+	ret.hi = a + b;
+	s = ret.hi - a;
+	ret.lo = (a - (ret.hi - s)) + (b - s);
+	return (ret);
+}
+
+/*
+ * Compute a+b, with a small tweak:  The least significant bit of the
+ * result is adjusted into a sticky bit summarizing all the bits that
+ * were lost to rounding.  This adjustment negates the effects of double
+ * rounding when the result is added to another number with a higher
+ * exponent.  For an explanation of round and sticky bits, see any reference
+ * on FPU design, e.g.,
+ *
+ *     J. Coonen.  An Implementation Guide to a Proposed Standard for
+ *     Floating-Point Arithmetic.  Computer, vol. 13, no. 1, Jan 1980.
+ */
+static inline long double
+add_adjusted(long double a, long double b)
+{
+	struct dd sum;
+	union IEEEl2bits u;
+
+	sum = dd_add(a, b);
+	if (sum.lo != 0) {
+		u.e = sum.hi;
+		if ((u.bits.manl & 1) == 0)
+			sum.hi = nextafterl(sum.hi, INFINITY * sum.lo);
+	}
+	return (sum.hi);
+}
+
+/*
+ * Compute ldexp(a+b, scale) with a single rounding error. It is assumed
+ * that the result will be subnormal, and care is taken to ensure that
+ * double rounding does not occur.
+ */
+static inline long double
+add_and_denormalize(long double a, long double b, int scale)
+{
+	struct dd sum;
+	int bits_lost;
+	union IEEEl2bits u;
+
+	sum = dd_add(a, b);
+
+	/*
+	 * If we are losing at least two bits of accuracy to denormalization,
+	 * then the first lost bit becomes a round bit, and we adjust the
+	 * lowest bit of sum.hi to make it a sticky bit summarizing all the
+	 * bits in sum.lo. With the sticky bit adjusted, the hardware will
+	 * break any ties in the correct direction.
+	 *
+	 * If we are losing only one bit to denormalization, however, we must
+	 * break the ties manually.
+	 */
+	if (sum.lo != 0) {
+		u.e = sum.hi;
+		bits_lost = -u.bits.exp - scale + 1;
+		if (bits_lost != 1 ^ (int)(u.bits.manl & 1))
+			sum.hi = nextafterl(sum.hi, INFINITY * sum.lo);
+	}
+	return (ldexp(sum.hi, scale));
+}
+
+/*
+ * Compute a*b exactly, returning the exact result in a struct dd.  We assume
+ * that both a and b are normalized, so no underflow or overflow will occur.
+ * The current rounding mode must be round-to-nearest.
+ */
+static inline struct dd
+dd_mul(long double a, long double b)
+{
+#if LDBL_MANT_DIG == 64
+	static const long double split = 0x1p32L + 1.0;
+#elif LDBL_MANT_DIG == 113
+	static const long double split = 0x1p57L + 1.0;
+#endif
+	struct dd ret;
+	long double ha, hb, la, lb, p, q;
+
+	p = a * split;
+	ha = a - p;
+	ha += p;
+	la = a - ha;
+
+	p = b * split;
+	hb = b - p;
+	hb += p;
+	lb = b - hb;
+
+	p = ha * hb;
+	q = ha * lb + la * hb;
+
+	ret.hi = p + q;
+	ret.lo = p - ret.hi + q + la * lb;
+	return (ret);
+}
+
 /*
  * Fused multiply-add: Compute x * y + z with a single rounding error.
  *
@@ -43,14 +165,8 @@ __FBSDID("$FreeBSD$");
 long double
 fmal(long double x, long double y, long double z)
 {
-#if LDBL_MANT_DIG == 64
-	static const long double split = 0x1p32L + 1.0;
-#elif LDBL_MANT_DIG == 113
-	static const long double split = 0x1p57L + 1.0;
-#endif
-	long double xs, ys, zs;
-	long double c, cc, hx, hy, p, q, tx, ty;
-	long double r, rr, s;
+	long double xs, ys, zs, adj;
+	struct dd xy, r;
 	int oround;
 	int ex, ey, ez;
 	int spread;
@@ -80,41 +196,6 @@ fmal(long double x, long double y, long double z)
 	 * will overflow, so we handle these cases specially.  Rounding
 	 * modes other than FE_TONEAREST are painful.
 	 */
-	if (spread > LDBL_MANT_DIG * 2) {
-		fenv_t env;
-		feraiseexcept(FE_INEXACT);
-		switch(oround) {
-		case FE_TONEAREST:
-			return (x * y);
-		case FE_TOWARDZERO:
-			if (x > 0.0 ^ y < 0.0 ^ z < 0.0)
-				return (x * y);
-			feholdexcept(&env);
-			r = x * y;
-			if (!fetestexcept(FE_INEXACT))
-				r = nextafterl(r, 0);
-			feupdateenv(&env);
-			return (r);
-		case FE_DOWNWARD:
-			if (z > 0.0)
-				return (x * y);
-			feholdexcept(&env);
-			r = x * y;
-			if (!fetestexcept(FE_INEXACT))
-				r = nextafterl(r, -INFINITY);
-			feupdateenv(&env);
-			return (r);
-		default:	/* FE_UPWARD */
-			if (z < 0.0)
-				return (x * y);
-			feholdexcept(&env);
-			r = x * y;
-			if (!fetestexcept(FE_INEXACT))
-				r = nextafterl(r, INFINITY);
-			feupdateenv(&env);
-			return (r);
-		}
-	}
 	if (spread < -LDBL_MANT_DIG) {
 		feraiseexcept(FE_INEXACT);
 		if (!isnormal(z))
@@ -139,49 +220,49 @@ fmal(long double x, long double y, long double z)
 				return (z);
 		}
 	}
+	if (spread <= LDBL_MANT_DIG * 2)
+		zs = ldexpl(zs, -spread);
+	else
+		zs = copysignl(LDBL_MIN, zs);
 
-	/*
-	 * Use Dekker's algorithm to perform the multiplication and
-	 * subsequent addition in twice the machine precision.
-	 * Arrange so that x * y = c + cc, and x * y + z = r + rr.
-	 */
 	fesetround(FE_TONEAREST);
 
-	p = xs * split;
-	hx = xs - p;
-	hx += p;
-	tx = xs - hx;
-
-	p = ys * split;
-	hy = ys - p;
-	hy += p;
-	ty = ys - hy;
-
-	p = hx * hy;
-	q = hx * ty + tx * hy;
-	c = p + q;
-	cc = p - c + q + tx * ty;
-
-	zs = ldexpl(zs, -spread);
-	r = c + zs;
-	s = r - c;
-	rr = (c - (r - s)) + (zs - s) + cc;
+	/*
+	 * Basic approach for round-to-nearest:
+	 *
+	 *     (xy.hi, xy.lo) = x * y		(exact)
+	 *     (r.hi, r.lo)   = xy.hi + z	(exact)
+	 *     adj = xy.lo + r.lo		(inexact; low bit is sticky)
+	 *     result = r.hi + adj		(correctly rounded)
+	 */
+	xy = dd_mul(xs, ys);
+	r = dd_add(xy.hi, zs);
 
 	spread = ex + ey;
-	if (spread + ilogbl(r) > -16383) {
-		fesetround(oround);
-		r = r + rr;
-	} else {
+
+	if (r.hi == 0.0) {
 		/*
-		 * The result is subnormal, so we round before scaling to
-		 * avoid double rounding.
+		 * When the addends cancel to 0, ensure that the result has
+		 * the correct sign.
 		 */
-		p = ldexpl(copysignl(0x1p-16382L, r), -spread);
-		c = r + p;
-		s = c - r;
-		cc = (r - (c - s)) + (p - s) + rr;
 		fesetround(oround);
-		r = (c + cc) - p;
+		volatile long double vzs = zs; /* XXX gcc CSE bug workaround */
+		return (xy.hi + vzs + ldexpl(xy.lo, spread));
 	}
-	return (ldexpl(r, spread));
+
+	if (oround != FE_TONEAREST) {
+		/*
+		 * There is no need to worry about double rounding in directed
+		 * rounding modes.
+		 */
+		fesetround(oround);
+		adj = r.lo + xy.lo;
+		return (ldexpl(r.hi + adj, spread));
+	}
+
+	adj = add_adjusted(r.lo, xy.lo);
+	if (spread + ilogbl(r.hi) > -16383)
+		return (ldexpl(r.hi + adj, spread));
+	else
+		return (add_and_denormalize(r.hi, adj, spread));
 }
diff --git a/libexec/rtld-elf/amd64/rtld_start.S b/libexec/rtld-elf/amd64/rtld_start.S
index a63d94cf75cf..da3d156e00f7 100644
--- a/libexec/rtld-elf/amd64/rtld_start.S
+++ b/libexec/rtld-elf/amd64/rtld_start.S
@@ -56,7 +56,7 @@
  * and (2) "reloff", the byte offset of the appropriate relocation entry
  * in the PLT relocation table.
  *
- * We are careful to preserve all registers, even the the caller-save
+ * We are careful to preserve all registers, even the caller-save
  * registers.  That is because this code may be invoked by low-level
  * assembly-language code that is not ABI-compliant.
  *
diff --git a/libexec/rtld-elf/i386/reloc.c b/libexec/rtld-elf/i386/reloc.c
index 9efebb32cd99..c9a6e53ee941 100644
--- a/libexec/rtld-elf/i386/reloc.c
+++ b/libexec/rtld-elf/i386/reloc.c
@@ -213,9 +213,11 @@ reloc_non_plt(Obj_Entry *obj, Obj_Entry *obj_rtld, RtldLockState *lockstate)
 		break;
 
 	    case R_386_TLS_TPOFF:
+	    case R_386_TLS_TPOFF32:
 		{
 		    const Elf_Sym *def;
 		    const Obj_Entry *defobj;
+		    Elf_Addr add;
 
 		    def = find_symdef(ELF_R_SYM(rel->r_info), obj, &defobj,
 		      false, cache, lockstate);
@@ -237,8 +239,11 @@ reloc_non_plt(Obj_Entry *obj, Obj_Entry *obj_rtld, RtldLockState *lockstate)
 			    goto done;
 			}
 		    }
-
-		    *where += (Elf_Addr) (def->st_value - defobj->tlsoffset);
+		    add = (Elf_Addr) (def->st_value - defobj->tlsoffset);
+		    if (ELF_R_TYPE(rel->r_info) == R_386_TLS_TPOFF)
+			*where += add;
+		    else
+			*where -= add;
 		}
 		break;
 
diff --git a/libexec/rtld-elf/i386/rtld_start.S b/libexec/rtld-elf/i386/rtld_start.S
index 482e51a86385..e7df748949f8 100644
--- a/libexec/rtld-elf/i386/rtld_start.S
+++ b/libexec/rtld-elf/i386/rtld_start.S
@@ -63,7 +63,7 @@
  * and (2) "reloff", the byte offset of the appropriate relocation entry
  * in the PLT relocation table.
  *
- * We are careful to preserve all registers, even the the caller-save
+ * We are careful to preserve all registers, even the caller-save
  * registers.  That is because this code may be invoked by low-level
  * assembly-language code that is not ABI-compliant.
  */
diff --git a/libexec/rtld-elf/map_object.c b/libexec/rtld-elf/map_object.c
index 2b33862bc7fb..aca1b2d0d43a 100644
--- a/libexec/rtld-elf/map_object.c
+++ b/libexec/rtld-elf/map_object.c
@@ -215,8 +215,9 @@ map_object(int fd, const char *path, const struct stat *sb)
 	    bss_vlimit = round_page(segs[i]->p_vaddr + segs[i]->p_memsz);
 	    bss_addr = mapbase +  (bss_vaddr - base_vaddr);
 	    if (bss_vlimit > bss_vaddr) {	/* There is something to do */
-		if (mprotect(bss_addr, bss_vlimit - bss_vaddr, data_prot) == -1) {
-		    _rtld_error("%s: mprotect of bss failed: %s", path,
+		if (mmap(bss_addr, bss_vlimit - bss_vaddr, data_prot,
+		    data_flags | MAP_ANON, -1, 0) == (caddr_t)-1) {
+		    _rtld_error("%s: mmap of bss failed: %s", path,
 			strerror(errno));
 		    return NULL;
 		}
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 65eadb5b7a5b..36102ba755f7 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -495,8 +495,12 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
        exit (0);
     }
 
-    /* setup TLS for main thread */
-    dbg("initializing initial thread local storage");
+    /*
+     * Processing tls relocations requires having the tls offsets
+     * initialized.  Prepare offsets before starting initial
+     * relocation processing.
+     */
+    dbg("initializing initial thread local storage offsets");
     STAILQ_FOREACH(entry, &list_main, link) {
 	/*
 	 * Allocate all the initial objects out of the static TLS
@@ -504,7 +508,6 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	 */
 	allocate_tls_offset(entry->obj);
     }
-    allocate_initial_tls(obj_list);
 
     if (relocate_objects(obj_main,
       ld_bind_now != NULL && *ld_bind_now != '\0', &obj_rtld, NULL) == -1)
@@ -519,6 +522,14 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
        exit (0);
     }
 
+    /*
+     * Setup TLS for main thread.  This must be done after the
+     * relocations are processed, since tls initialization section
+     * might be the subject for relocations.
+     */
+    dbg("initializing initial thread local storage");
+    allocate_initial_tls(obj_list);
+
     dbg("initializing key program variables");
     set_program_var("__progname", argv[0] != NULL ? basename(argv[0]) : "");
     set_program_var("environ", env);
diff --git a/release/Makefile.inc.docports b/release/Makefile.inc.docports
deleted file mode 100644
index f4b238a07c1a..000000000000
--- a/release/Makefile.inc.docports
+++ /dev/null
@@ -1,89 +0,0 @@
-# $FreeBSD$
-#
-# List of (dependent) ports that are minimally required to be
-# checked out from CVS in order to get ${DOCPORTS} built and
-# installed.  It should follow the dependency list in
-# ports/textproc/docproj/Makefile as much as feasible.
-#
-# Note that these names are /not/ CVS module names but path
-# names, so the required part of the ports infrastructure will
-# be cvs co'ed accordingly.
-#
-# This list of ports should be checked with the output of :
-# make all-depends-list | sed -e 's|^/usr/||' | sed -e 's|$| \\|'
-#
-
-# Get __FreeBSD_version
-.if !defined(OSVERSION)
-OSVERSION!=	/sbin/sysctl -n kern.osreldate
-.endif
-
-MINIMALDOCPORTS= \
-	ports/Mk \
-	ports/Templates \
-	ports/Tools
-
-MINIMALDOCPORTS+= \
-	ports/archivers/unzip \
-	ports/converters/libiconv \
-	ports/devel/autoconf267 \
-	ports/devel/autoconf-wrapper \
-	ports/devel/gettext \
-	ports/devel/gmake \
-	ports/devel/libtool22 \
-	ports/devel/m4 \
-	ports/devel/pkg-config \
-	ports/graphics/gd \
-	ports/graphics/jbigkit \
-	ports/graphics/jpeg \
-	ports/graphics/netpbm \
-	ports/graphics/png \
-	ports/graphics/scr2png \
-	ports/graphics/tiff \
-	ports/misc/help2man \
-	ports/print/freetype2 \
-	ports/print/ghostscript8 \
-	ports/print/ghostscript8-nox11 \
-	ports/print/gsfonts \
-	ports/textproc/docbook-410 \
-	ports/textproc/docbook-xml \
-	ports/textproc/docbook-xsl \
-	ports/textproc/docproj \
-	ports/textproc/dsssl-docbook-modular \
-	ports/textproc/expat2 \
-	ports/textproc/fixrtf \
-	ports/textproc/html \
-	ports/textproc/html2text \
-	ports/textproc/iso8879 \
-	ports/textproc/libxml2 \
-	ports/textproc/libxslt \
-	ports/textproc/linuxdoc \
-	ports/textproc/scr2txt \
-	ports/textproc/sdocbook-xml \
-	ports/textproc/xhtml \
-	ports/textproc/xmlcatmgr \
-	ports/textproc/xmlcharent \
-	ports/www/links1 \
-	ports/www/tidy
-
-.if ${MACHINE_ARCH} != "i386"
-MINIMALDOCPORTS+=	ports/textproc/openjade \
-			ports/textproc/opensp
-.else
-MINIMALDOCPORTS+=	ports/textproc/jade
-.endif
-
-.if ${OSVERSION} < 500000
-MINIMALDOCPORTS+= \
-	ports/converters/p5-MIME-Base64 \
-	ports/devel/p5-File-Spec \
-	ports/devel/p5-File-Temp \
-	ports/devel/p5-Test-Harness \
-	ports/devel/p5-Test-Simple \
-	ports/security/p5-Digest \
-	ports/security/p5-Digest-MD5 \
-	ports/textproc/p5-PodParser
-.else
-MINIMALDOCPORTS+= \
-	ports/lang/perl5.10
-.endif
diff --git a/release/Makefile.sysinstall b/release/Makefile.sysinstall
deleted file mode 100644
index 740ab42aa4a2..000000000000
--- a/release/Makefile.sysinstall
+++ /dev/null
@@ -1,1444 +0,0 @@
-# $FreeBSD$
-#
-# make release [BUILDNAME=somename] CHROOTDIR=/some/dir CVSROOT=/cvs/dir \
-#     [RELEASETAG=tag] [SVNROOT=svn://svn.freebsd.org/base] \
-#     [SVNBRANCH=some/branch]
-#
-# Where "/some/dir" is the pathname of a directory on a some filesystem with
-# at least 1000MB of free space, "somename" is what you want the release to
-# call itself, "/cvs/dir" is where our source repo resides and, optionally,
-# which CVS "tag" name should be used when checking out the sources to build
-# the release (default is HEAD).
-#
-# Please note the support for building from SVN is preliminary and there
-# are still questions about things like how to handle updates of
-# /usr/src on production systems (csup(1) replacement).  It is a work
-# in progress and may change as the other issues get worked out.
-#
-# Please note: the md(4) driver must be present in the kernel
-# (either by being compiled in or available as a kld(4) module),
-# otherwise the target 'release.8' and possibly others will fail.
-#
-# Note: If you add options to this file, please keep release(7) updated!
-#
-# Set these, release builder!
-#
-# Fixed version:
-#BUILDNAME=9.0-CURRENT
-#
-# Automatic SNAP versioning:
-DATE != date +%Y%m%d
-BASE = 9.0
-BUILDNAME?=${BASE}-${DATE}-SNAP
-#
-#CHROOTDIR=/junk/release
-# If this is a -stable snapshot, then set
-#RELEASETAG=RELENG_8
-#
-# To test a release with a source tree containing patches and
-# other work. This tree will get copied instead of getting the
-# src/ tree from a CVS checkout. For "rerelease", this will NOT
-# be copied; cvs update will be used instead.
-#EXTSRCDIR=/usr/src
-#
-# To use a checked-out ports collection directory instead of
-# checking out from a local CVS repository, set this option.
-#EXTPORTSDIR=/usr/ports
-#
-# To use a checked-out doc/ directory instead of
-# checking out from a local CVS repository, set this option.
-#EXTDOCDIR=/usr/doc
-#
-# To add other options to the CVS subcommands (co,up), set
-#CVSCMDARGS="-D '01/01/2002 00:00:00 UTC'"
-#
-# To add other options to the CVS command, set
-#CVSARGS="-lfq"
-#
-# To prefix the CVS command
-#CVSPREFIX="/usr/bin/time"
-#
-# Where the CVS repository is
-#CVSROOT="/home/ncvs"
-#
-# To add other options to the Subversion subcommands (co,up), set
-#SVNCMDARGS="-r '{ 01/01/2002 00:00:00 UTC }'"
-#
-# To prefix the Subversion command
-#SVNPREFIX="/usr/bin/time"
-#
-# Where the Subversion repository is
-#SVNROOT=svn://svn.freebsd.org/base
-#
-# Subversion branch to build for src.  If this is not set then it is
-# automatically computed from RELEASETAG.
-#SVNBRANCH=stable/8
-#
-# Non-zero if ${RELEASETAG} is in the form "RELENG_ver_RELEASE"; we
-# are building an official release.  Otherwise, we are building for
-# a branch.
-.if defined(RELEASETAG)
-ISRELEASE!=	expr "${RELEASETAG}" : '^RELENG_.*_RELEASE$$' || true
-.if ${ISRELEASE} != 0
-# Convert "RELENG_ver_RELEASE" to "RELEASE_ver" for ports and doc trees.
-AUXRELEASETAG!=	echo ${RELEASETAG} | sed -e 's/^RELENG_/RELEASE_/' -e 's/_RELEASE$$//'
-DOCRELEASETAG?=		${AUXRELEASETAG}
-PORTSRELEASETAG?=	${AUXRELEASETAG}
-.endif
-.endif
-
-# Determine the Subversion source branch that corresponds to the requested
-# RELEASETAG.
-.if !defined(SVNBRANCH)
-.if defined(RELEASETAG)
-SVNBRANCH!=	echo "${RELEASETAG}" | awk -f ${.CURDIR}/svnbranch.awk
-.else
-SVNBRANCH=	head
-.endif
-.endif
-
-# If you want to pass flags to the world build such as -j X, use
-# WORLD_FLAGS.  Similarly, you can specify make flags for kernel
-# builds via KERNEL_FLAGS.
-#WORLD_FLAGS=-j4
-#KERNEL_FLAGS=-j4
-
-TARGET_ARCH?=	${MACHINE_ARCH}
-.if ${TARGET_ARCH} == ${MACHINE_ARCH}
-TARGET?=		${MACHINE}
-TARGET_CPUARCH?=	${MACHINE_CPUARCH}
-.else
-TARGET?=		${TARGET_ARCH}
-TARGET_CPUARCH?=	${TARGET_ARCH}
-.endif
-CROSSENV=	TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}
-CROSSMAKE=	${MAKE} ${CROSSENV}
-NATIVEMAKE=	${MAKE} TARGET_ARCH=${MACHINE_ARCH} TARGET=${MACHINE}
-
-# If you are using a local CVS repository with components stored in 
-# non-standard modules, override these on the make commandline or
-# in the environment.
-RELEASESRCMODULE?=	src
-RELEASEDOCMODULE?=	doc
-RELEASEPORTSMODULE?=	ports
-
-# Uncomment this to disable the doc.1 target.  Docs normally require
-# the ports tree, so NOPORTS can be set together with NODOC in order
-# to have neither ports or docs.  If only NOPORTS is set to YES, but
-# docs are still desired, the DOMINIMALDOCPORTS logic below will only
-# install the ports that are minimally required for the release note documentation.  This is
-# intended as a compromise, less disk space is required than for using
-# the entire ports collection (and much less time due to the huge number
-# of directories it would create), but still quite a bit as well as some
-# CPU cycles (some of the programs are C++, and things like ghostscript
-# belong to the required ports nevertheless).
-#
-#NODOC=  YES
-#NOPORTS=  YES
-
-# When retrieving ports using the pkg_add -r method, set your proxies to these.
-# src/release/${arch}/mkisoimages.sh can use these.
-#HTTP_PROXY?=	www.example.com
-#FTP_PROXY?=	ftp.example.com
-
-# When creating ISO images, point ${CD_PACKAGE_TREE} to a directory containing
-# the package split by an earlier invocation of the 'package-split' target.
-#CD_PACKAGE_TREE=	/path/to/pkg
-
-# Extra source tarballs; each argument is a pair of source dir and
-# distribution name.  The dist name should not exceed 7 characters
-# (another "s" for "source" will be prepended).
-#EXTRA_SRC=	games/fortune fortune
-
-# Modify this definition if you want the release notes 
-# and other release documentation in a language other than English.
-RELNOTES_LANG?=	en_US.ISO8859-1
-
-# As an alternative to installing the entire ports collection (which
-# can take a huge amount of time, in particular on slower disks),
-# setting ${MINIMALDOCPORTS} allows to install and build just those
-# ports that are really required for getting the docs up & running.
-.if defined(NOPORTS) && !defined(NODOC)
-DOMINIMALDOCPORTS=	YES
-.include "Makefile.inc.docports"
-RELEASEPORTSMODULE=	${MINIMALDOCPORTS} ports/sysutils/cdrtools
-.endif
-
-# Helper variable
-.if defined(NOPORTS)
-.if !defined(DOMINIMALDOCPORTS) || ${DOMINIMALDOCPORTS} != "YES"
-NOPORTSATALL=	YES
-.endif
-.endif
-
-#
-# Doing 'make index' in /usr/ports requires Perl.
-MAKEINDEXPORTS=	lang/perl5.10
-DOCPORTS=	textproc/docproj
-# Set this to wherever the distfiles required by release procedures.
-.if defined(DOCDISTFILES)
-# Respect DOCDISTFILES which is used before.
-RELEASEDISTFILES?=	${DOCDISTFILES}
-.else
-RELEASEDISTFILES?=	${.CURDIR}/../../ports/distfiles
-.endif
-# Set this to 1 if you want -P to be used for automatic keyboard detection
-# on the boot floppy.  WARNING: Breaks on some Athlon (K7) motherboards.
-AUTO_KEYBOARD_DETECT?= 0
-
-.if !defined(NODOC)
-DIST_DOCS_ARCH_INDEP=	hardware readme relnotes errata
-DIST_DOCS_ARCH_DEP=	
-.endif
-
-# Things which without too much trouble can be considered variables
-# BASE_DISTS are special in that they get full /etc installation sets.
-#
-OTHER_DISTS?=	catpages manpages proflibs dict info
-BASE_DISTS?=	base doc games ${ARCH_DISTS}
-.if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
-ARCH_DISTS?=	lib32
-.endif
-DISTRIBUTIONS?=	${BASE_DISTS} ${OTHER_DISTS}
-
-#
-# Build and package both GENERIC and SMP kernels if the target
-# has both configuration files.  Otherwise only GENERIC is done.
-#
-.if ${TARGET_ARCH} == "powerpc64"
-KERN_GENERIC?=	GENERIC64
-.else
-KERN_GENERIC?=	GENERIC
-.endif
-
-.if exists(${.CURDIR}/../sys/${TARGET}/conf/SMP)
-KERNELS_BASE?=	${KERN_GENERIC} SMP
-.else
-KERNELS_BASE?=	${KERN_GENERIC}
-.endif
-
-# mountpoint for filesystems.
-MNT=			/mnt
-
-# Various floppy image parameters.
-#
-
-.undef MAKE_FLOPPIES
-.if ${TARGET_ARCH} == "i386"
-MAKE_DVD=
-SEPARATE_LIVEFS=
-SPLIT_MFSROOT=
-.if ${TARGET} == "pc98"
-MAKE_FLOPPIES=		true
-SMALLFLOPPYSIZE=	1200
-SMALLFLOPPYSPLITSIZE=	1152
-SMALLFLOPPYLABEL=	fd1200
-SPLIT_MFSROOT=
-.endif
-FLOPPYSIZE=		1440
-FLOPPYSPLITSIZE=	1392
-FLOPPYINODE=		40000
-FLOPPYLABEL=		fd1440
-BOOTINODE=		80000
-MFSSIZE=		4320
-MFSINODE=		8000
-MFSLABEL=		minimum3
-.elif ${TARGET_ARCH} == "sparc64"
-DISKLABEL=		sunlabel
-MAKE_DVD=
-MFSSIZE=		4096
-MFSINODE=		8192
-MFSLABEL=		auto
-MINIROOT=
-SEPARATE_LIVEFS=
-.elif ${TARGET_ARCH} == "ia64"
-DISKLABEL=		""
-MAKE_DVD=
-MFSSIZE=		8192
-MFSINODE=		8192
-MFSLABEL=		auto
-SEPARATE_LIVEFS=
-.elif ${TARGET_ARCH} == "amd64"
-MAKE_DVD=
-FLOPPYSIZE=		1440
-FLOPPYSPLITSIZE=	1392
-FLOPPYINODE=		40000
-FLOPPYLABEL=		fd1440
-BOOTINODE=		80000
-MFSSIZE=		4096
-MFSINODE=		8192
-MFSLABEL=		auto
-SEPARATE_LIVEFS=
-SPLIT_MFSROOT=
-.elif ${TARGET_ARCH} == "powerpc"
-DISKLABEL=		""
-MFSSIZE=		4096
-MFSINODE=		8192
-MFSLABEL=		auto
-SEPARATE_LIVEFS=
-.elif ${TARGET_ARCH} == "powerpc64"
-DISKLABEL=		""
-MFSSIZE=		8192
-MFSINODE=		8192
-MFSLABEL=		auto
-SEPARATE_LIVEFS=
-.endif
-.if defined(NO_FLOPPIES)
-.undef MAKE_FLOPPIES
-.endif
-
-.if exists(/sbin/bsdlabel)
-DISKLABEL?=		bsdlabel
-.else
-DISKLABEL?=		disklabel
-.endif
-
-ZIPPER=			gzip -9 --no-name
-ZIPNSPLIT=		${ZIPPER} -c | split -b 1392k -
-
-# Things which may get you into trouble if you change them
-MTREEFILES=		${.CURDIR}/../etc/mtree
-_R?=			/R
-RD=			${_R}/stage
-RND=			${RD}/release.doc
-FD=			${_R}/ftp
-CD=			${_R}/cdrom
-CD_BOOT=		${CD}/bootonly
-CD_DISC1=		${CD}/disc1
-CD_DISC2=		${CD}/disc2
-.if defined(MAKE_DVD)
-CD_DVD1=		${CD}/dvd1
-.endif
-.if !defined(NODOC)
-CD_DOCS=		${CD}/docs
-.endif
-.if defined(SEPARATE_LIVEFS)
-CD_LIVEFS=		${CD}/livefs
-.else
-CD_LIVEFS=		${CD_DISC1}
-.endif
-_MK?=			${CHROOTDIR}/mk
-
-# Where the bootstrap ports (see DOCPORTS) get installed.
-LOCALDIR=		/usr/local/bin
-
-.if ${TARGET} != ${MACHINE} && ${DISKLABEL} == "bsdlabel"
-DOFS_SH=	${.CURDIR}/scripts/doFS.sh ${DISKLABEL} ${TARGET}
-.else
-DOFS_SH=	${.CURDIR}/scripts/doFS.sh ${DISKLABEL} ""
-.endif
-
-CRUNCH_TARGETS=	boot
-.if ${TARGET_ARCH} == "i386"
-CRUNCH_TARGETS+=fixit
-.if ${TARGET} == "pc98"
-CRUNCH_TARGETS+=fixit-small
-.endif
-.endif
-
-.if defined(MAKE_FLOPPIES)
-EXTRAS= floppies.1
-.if ${TARGET_ARCH} == "i386"
-EXTRAS+= floppies.2
-.endif
-EXTRAS+= floppies.3
-.endif
-EXTRAS+= ftp.1
-.if !defined(NOCDROM)
-EXTRAS+= cdrom.1 cdrom.2 cdrom.3
-.if defined(MAKE_ISOS)
-EXTRAS+= iso.1
-.endif
-BOOTABLE="-b"
-.endif
-
-.if !defined(NODOC)
-DOCREL= doc.1 doc.2
-.endif
-
-.if !make(release) && !make(rerelease) && !make(package-split)
-BINMAKE!=	cd ${.CURDIR}/..; ${MAKE} -V BINMAKE
-WMAKEENV!=	cd ${.CURDIR}/..; \
-		${BINMAKE} ${CROSSENV} -f Makefile.inc1 -V WMAKEENV
-WMAKE=		${WMAKEENV} ${BINMAKE}
-.endif
-
-SMAKE=		${MAKE} -f ${MAKEFILE}
-
-CVS_SRCARGS=	-P
-.if defined(RELEASETAG)
-CVS_SRCARGS+=	-r ${RELEASETAG}
-.endif
-
-CVS_DOCARGS=	-P
-.if defined(DOCRELEASETAG)
-CVS_DOCARGS+=	-r ${DOCRELEASETAG}
-.endif
-
-CVS_PORTSARGS=	-P
-.if defined(PORTSRELEASETAG)
-CVS_PORTSARGS+=	-r ${PORTSRELEASETAG}
-.endif
-
-WORLDDIR?=	${.CURDIR}/..
-
-release rerelease:
-.if !defined(CHROOTDIR) || !defined(BUILDNAME)
-	@echo "To make a release you must set CHROOTDIR and BUILDNAME" && false
-.endif
-.if !defined(NOPORTSATALL) && !defined(EXTPORTSDIR) && !defined(CVSROOT)
-	@echo "Building ports requires CVSROOT or EXTPORTSDIR" && false
-.endif
-.if !defined(NODOC) && !defined(EXTDOCDIR) && !defined(CVSROOT)
-	@echo "Building docs requires CVSROOT or EXTDOCDIR" && false
-.endif
-.if !defined(EXTSRCDIR) && !defined(CVSROOT) && !defined(SVNROOT)
-	@echo "The source tree requires SVNROOT, CVSROOT, or EXTSRCDIR" && false
-.endif
-.if defined(NOPORTSATALL) && !defined(NODOC)
-	@echo "Ports are required for building the release docs.  Either set NODOC or"
-	@echo "unset NOPORTS, or set at least DOMINIMALDOCPORTS to YES!"
-	@exit 1
-.endif
-.if defined(LOCAL_PATCHES) && !empty(LOCAL_PATCHES)
-.for p in ${LOCAL_PATCHES}
-.if !exists(${p})
-	@echo "The patch file ${p} does not exist!"
-	@exit 1
-.endif
-.endfor
-.endif
-.if defined(LOCAL_SCRIPT) && !exists(${LOCAL_SCRIPT})
-	@echo "The local script ${LOCAL_SCRIPT} does not exist!"
-	@exit 1
-.endif
-.if make(release)
-.if exists(${CHROOTDIR})
-# The first command will fail on a handful of files that have their schg
-# flags set.  But it greatly speeds up the next two commands.
-	# NB: clear any vestigial devfs mount, just in case
-	-umount ${CHROOTDIR}/dev > /dev/null 2>&1
-	-rm -rf ${CHROOTDIR} 2>/dev/null
-	-chflags -R 0 ${CHROOTDIR}/.
-	-rm -rf ${CHROOTDIR}
-.endif
-	mkdir -p ${CHROOTDIR}
-	@echo ">>> make release for ${TARGET} started on `LC_ALL=C TZ=GMT date`"
-	cd ${WORLDDIR} && ${NATIVEMAKE} -DWITHOUT_GAMES -DWITHOUT_HTML -DWITHOUT_LIB32 \
-	    -DWITHOUT_MAN -DWITHOUT_NLS -DWITHOUT_PROFILE installworld DESTDIR=${CHROOTDIR}
-	cd ${WORLDDIR} && ${NATIVEMAKE} distribution DESTDIR=${CHROOTDIR}
-	if [ -f /etc/resolv.conf ]; then \
-		cp -p /etc/resolv.conf ${CHROOTDIR}/etc; \
-	fi
-.if defined(EXTLOCALDIR)
-	rm -rf ${CHROOTDIR}/usr/local
-	cd ${CHROOTDIR}/usr && cp -R -H ${EXTLOCALDIR} local
-.endif
-	rm -rf ${CHROOTDIR}/usr/src
-.if defined(EXTSRCDIR)
-	cd ${CHROOTDIR}/usr && \
-	    cp -R -H ${EXTSRCDIR} src
-.elif defined(SVNROOT)
-	cd ${CHROOTDIR}/usr && \
-	    ${SVNPREFIX} svn co ${SVNCMDARGS} ${SVNROOT}/${SVNBRANCH} \
-	    ${RELEASESRCMODULE} 
-.else
-	cd ${CHROOTDIR}/usr && \
-	    ${CVSPREFIX} cvs -R ${CVSARGS} -d ${CVSROOT} \
-	    co ${CVSCMDARGS} ${CVS_SRCARGS} ${RELEASESRCMODULE}
-.endif
-.if defined(LOCAL_PATCHES) && !empty(LOCAL_PATCHES)
-.for p in ${LOCAL_PATCHES}
-	patch -d ${CHROOTDIR}/usr/${RELEASESRCMODULE} ${PATCH_FLAGS} < ${p}
-.endfor
-.endif
-.if defined(LOCAL_SCRIPT)
-	cd ${CHROOTDIR} && env CHROOTDIR=${CHROOTDIR} BUILDNAME=${BUILDNAME} \
-	    RELEASETAG=${RELEASETAG} ${LOCAL_SCRIPT}
-.endif
-	rm -rf ${CHROOTDIR}/usr/ports
-.if !defined(NOPORTSATALL)
-.if defined(EXTPORTSDIR)
-	cd ${CHROOTDIR}/usr && cp -R -H ${EXTPORTSDIR} ports
-	# If there are distfiles downloaded removing them
-	rm -rf ports/distfiles/*
-.else
-	cd ${CHROOTDIR}/usr && ${CVSPREFIX} cvs -R ${CVSARGS} -d ${CVSROOT} \
-	    co ${CVSCMDARGS} ${CVS_PORTSARGS} ${RELEASEPORTSMODULE}
-.endif
-.endif
-.if !defined(NODOC)
-	rm -rf ${CHROOTDIR}/usr/doc
-.if defined(EXTDOCDIR)
-	cd ${CHROOTDIR}/usr && cp -R -H ${EXTDOCDIR} doc
-.else
-	cd ${CHROOTDIR}/usr && ${CVSPREFIX} cvs -R ${CVSARGS} -d ${CVSROOT} \
-	    co ${CVSCMDARGS} ${CVS_DOCARGS} ${RELEASEDOCMODULE}
-.endif
-	if [ -d ${RELEASEDISTFILES}/ ]; then \
-		cp -rp ${RELEASEDISTFILES} ${CHROOTDIR}/usr/ports/distfiles; \
-	else \
-		mkdir -p ${CHROOTDIR}/usr/ports/distfiles; \
-	fi
-.if !defined(NO_PREFETCHDISTFILES)
-	@cd ${.CURDIR} && ${SMAKE} fetch-distfiles
-.endif
-.endif
-.endif
-.if make(rerelease)
-.if !defined(RELEASENOUPDATE) && !defined(EXTSRCDIR)
-.if defined(SVNROOT)
-	cd ${CHROOTDIR}/usr/src && ${SVNPREFIX} svn switch ${SVNCMDARGS} \
-	    ${SVNROOT}/${SVNBRANCH}
-.elif !defined(RELEASETAG)
-	cd ${CHROOTDIR}/usr/src && ${CVSPREFIX} cvs -R ${CVSARGS} -q \
-	    update ${CVSCMDARGS} -P -d -A
-.else
-	cd ${CHROOTDIR}/usr/src && ${CVSPREFIX} cvs -R ${CVSARGS} -q \
-	    update ${CVSCMDARGS} -P -d -r ${RELEASETAG}
-.endif
-	rm -f ${CHROOTDIR}/tmp/.world_done 
-.if !defined(NOPORTS) && !defined(EXTPORTSDIR)
-	cd ${CHROOTDIR}/usr/ports && ${CVSPREFIX} cvs -R ${CVSARGS} -q \
-	    update ${CVSCMDARGS} -P -d
-	rm -f ${CHROOTDIR}/tmp/.skip_ports_index
-.endif
-.if defined(DOMINIMALDOCPORTS) && ${DOMINIMALDOCPORTS} == "YES"
-	for i in ${MINIMALDOCPORTS}; do \
-	    ( cd ${CHROOTDIR}/usr/$$i && ${CVSPREFIX} cvs -R ${CVSARGS} -q \
-		update ${CVSCMDARGS} -P -d ) ; \
-	done
-.endif
-.if !defined(NODOC)
-	cd ${CHROOTDIR}/usr/doc && ${CVSPREFIX} cvs -R ${CVSARGS} -q \
-	    update ${CVSCMDARGS} -P -d
-.endif
-.endif
-.endif
-	# Add version information to those things that need it.
-	if [ ! -f ${CHROOTDIR}/tmp/.world_done ]; then \
-		cd ${CHROOTDIR}/usr/src/sys/conf && \
-		mv newvers.sh foo && \
-		sed "s/^RELEASE=.*/RELEASE=${BUILDNAME}/" foo > newvers.sh && \
-		rm foo; \
-	fi
-	-test -f install.cfg && cp install.cfg ${CHROOTDIR}/usr/src/release
-	echo "#!/bin/sh"			> ${_MK}
-	echo "set -ex"				>> ${_MK}
-	echo "trap 'umount /dev || true' 0"	>> ${_MK}
-	echo "_RELTARGET=\$${1:-doRELEASE}"	>> ${_MK}
-.for var in \
-	AUTO_KEYBOARD_DETECT \
-	BUILDNAME \
-	CD_PACKAGE_TREE \
-	DISTRIBUTIONS \
-	DOMINIMALDOCPORTS \
-	EXTRA_SRC \
-	FTP_PASSIVE_MODE \
-	FTP_PROXY \
-	HTTP_PROXY \
-	KERNELS \
-	KERNELS_BASE \
-	KERNEL_FLAGS \
-	MAKE_DVD \
-	MAKE_FLOPPIES \
-	MAKE_ISOS \
-	NOCDROM \
-	NODOC \
-	NOPORTS \
-	NO_SHARED \
-	NOSRC \
-	NO_CPU_CFLAGS \
-	NO_CPU_COPTFLAGS \
-	NO_FLOPPIES \
-	RELEASETAG \
-	RELNOTES_LANG \
-	SEPARATE_LIVEFS \
-	TARGET \
-	TARGET_ARCH \
-	TARGET_CPUARCH \
-	WORLD_FLAGS
-.if defined(${var})
-	echo "export ${var}=\"${${var}}\""	>> ${_MK}
-.endif
-.endfor
-	# Don't remove this, or the build will fall over!
-	echo "export RELEASEDIR=${_R}"		>> ${_MK}
-	echo "export PATH=/bin:/usr/bin:/sbin:/usr/sbin:${LOCALDIR}"	>> ${_MK}
-	echo "export MANBUILDCAT=YES"		>> ${_MK}
-	# NB: these may fail if the host is running w/o devfs
-	echo "umount /dev >/dev/null 2>&1 || true" >> ${_MK}
-	echo "mount -t devfs devfs /dev >/dev/null 2>&1 || true"	>> ${_MK}
-	echo "if [ ! -c /dev/null ]; then" >> ${_MK}
-	echo "	echo /dev/null is not a device!" >> ${_MK}
-	echo "	exit 1" >> ${_MK}
-	echo "fi" >> ${_MK}
-	echo "if [ -x /etc/rc.d/ldconfig ]; then" >> ${_MK}
-	echo "	/etc/rc.d/ldconfig start"	>> ${_MK}
-	echo "else"				>> ${_MK}
-	echo "	ldconfig /lib /usr/lib /usr/local/lib || true" >> ${_MK}
-	echo "fi"				>> ${_MK}
-	echo "if [ ! -f /tmp/.world_done ]; then" >> ${_MK}
-	echo "	cd /usr/src"			>> ${_MK}
-	echo "	${CROSSMAKE} ${WORLD_FLAGS} -DNO_CLEAN buildworld && \\" >> ${_MK}
-	echo "	touch /tmp/.world_done || exit 1" >> ${_MK}
-	echo "fi"				>> ${_MK}
-	echo "if [ ! -f /tmp/.skip_ports_index ]; then" >> ${_MK}
-	echo "	echo \">>> make index started on \`LC_ALL=C TZ=GMT date\`\"" >> ${_MK}
-	echo "	for i in ${MAKEINDEXPORTS}"	>> ${_MK}
-	echo "	do"			>> ${_MK}
-	echo "		cd /usr/ports/\$${i}"	>> ${_MK}
-	echo "		env -i HTTP_PROXY=$${HTTP_PROXY} FTP_PROXY=$${FTP_PROXY} FTP_PASSIVE_MODE=$${FTP_PASSIVE_MODE:-no} PATH=$${PATH} \\" >> ${_MK}
-	echo "		make all install clean BATCH=yes FORCE_PKG_REGISTER=yes" >> ${_MK}
-	echo "	done"				>> ${_MK}
-	echo "	cd /usr/ports"			>> ${_MK}
-	echo "	rm -f INDEX*"			>> ${_MK}
-	echo "	make index -DINDEX_PRISTINE"	>> ${_MK}
-	echo "	rm -f INDEX*.tmp"		>> ${_MK}
-	echo "	touch /tmp/.skip_ports_index"	>> ${_MK}
-	echo "	echo \">>> make index finished on \`LC_ALL=C TZ=GMT date\`\"" >> ${_MK}
-	echo "fi"				>> ${_MK}
-	echo "cd /usr/src/release"		>> ${_MK}
-	echo "make obj"				>> ${_MK}
-	echo "if [ -f Makefile.sysinstall ]; then" >> ${_MK}
-	echo "  make -f Makefile.sysinstall \$${_RELTARGET}" >> ${_MK}
-	echo "else"				>> ${_MK}
-	echo "  make \$${_RELTARGET}"		>> ${_MK}
-	echo "fi"				>> ${_MK}
-	echo "echo \">>> make ${.TARGET} for ${TARGET} finished on \`LC_ALL=C TZ=GMT date\`\"" >> ${_MK}
-	chmod 755 ${_MK}
-.if defined(NOPORTS)
-	touch ${CHROOTDIR}/tmp/.skip_ports_index
-.endif
-	# Ensure md.ko is loaded if md(4) is not statically compiled into 
-	# the kernel
-	-mdconfig 2>/dev/null
-	env -i /usr/sbin/chroot `dirname ${_MK}` /`basename ${_MK}`
-
-clean:
-	rm -rf ${CRUNCH_TARGETS:S/$/_crunch/} release.[0-8] ${EXTRAS}
-
-fetch-distfiles:
-	@for i in ${MAKEINDEXPORTS}; do \
-		cd ${CHROOTDIR}/usr/ports/$$i && \
-			make PORTSDIR=${CHROOTDIR}/usr/ports BATCH=yes \
-			checksum-recursive ; \
-	done
-	@for i in ${DOCPORTS}; do \
-		cd ${CHROOTDIR}/usr/ports/$$i && \
-			make PORTSDIR=${CHROOTDIR}/usr/ports BATCH=yes \
-			WITHOUT_X11=yes JADETEX=no WITHOUT_PYTHON=yes \
-			checksum-recursive ; \
-	done
-
-# Clean out ${_R} and make the directory structure.
-release.1:
-	mkdir -p ${_R}
-	-rm -rf ${_R}/* 2> /dev/null
-	-chflags -R noschg ${_R}/.
-	rm -rf ${_R}/*
-	mkdir ${RD}
-.if defined(MAKE_FLOPPIES)
-	mkdir ${RD}/floppies
-.endif
-	mkdir ${RD}/trees
-	for i in ${DISTRIBUTIONS}; do \
-		mkdir ${RD}/trees/$$i && \
-		mtree -deU -f ${MTREEFILES}/BSD.root.dist \
-		    -p ${RD}/trees/$$i > /dev/null && \
-		mtree -deU -f ${MTREEFILES}/BSD.usr.dist \
-		    -p ${RD}/trees/$$i/usr > /dev/null && \
-		mtree -deU -f ${MTREEFILES}/BSD.include.dist \
-		    -p ${RD}/trees/$$i/usr/include > /dev/null; \
-	done
-	mkdir ${RD}/kernels
-	for i in ${KERNELS_BASE} ${KERNELS}; do \
-		mkdir -p ${RD}/kernels/$${i}; \
-	done
-	touch ${.TARGET}
-
-# Install the system into the various distributions.
-release.2:
-	cd ${.CURDIR}/.. && ${CROSSMAKE} distrib-dirs DESTDIR=${RD}/trees/base
-	cd ${.CURDIR}/.. && ${CROSSMAKE} ${WORLD_FLAGS} distributeworld \
-	    DISTDIR=${RD}/trees
-	sh ${.CURDIR}/scripts/mm-mtree.sh -F "${CROSSENV}" -D "${RD}/trees/base"
-	touch ${.TARGET}
-
-# Make and install the generic kernel(s).
-release.3:
-.for kernel in ${KERNELS_BASE} ${KERNELS}
-	cd ${.CURDIR}/..; \
-	${CROSSMAKE} ${KERNEL_FLAGS} \
- 	    KERNCONF=${kernel} kernel \
- 	    DESTDIR=${RD}/kernels KODIR=/${kernel}
-.endfor
-	touch ${.TARGET}
-
-# Make and install the three crunched binaries which live on the floppies.
-# You are not supposed to like this :-)
-release.4:
-	rm -rf ${RD}/crunch
-	mkdir -p ${RD}/crunch
-.for j in ${CRUNCH_TARGETS}
-.if exists(${.CURDIR}/${TARGET}/${j}_crunch.conf)
-	rm -rf ${j}_crunch
-	mkdir ${j}_crunch
-	cd ${j}_crunch; ${WMAKEENV} MAKEFLAGS="-m ${.CURDIR}/../share/mk" \
-	    NO_WERROR= crunchgen -o \
-	    ${.CURDIR}/${TARGET}/${j}_crunch.conf
-	cd ${j}_crunch; ${WMAKE} -f ${j}_crunch.mk subclean
-	cd ${.CURDIR}/..; ${BINMAKE} -f Makefile.inc1 _build-tools
-	cd ${j}_crunch; CFLAGS="-Os -pipe" ${WMAKE} -f ${j}_crunch.mk \
-	    -DNO_CPU_CFLAGS all
-	${WMAKEENV} strip -R .comment ${j}_crunch/${j}_crunch
-	mv ${j}_crunch/${j}_crunch ${RD}/crunch/${j}
-.endif
-.endfor
-	touch ${.TARGET}
-
-#
-# --==## Fix up the distributions. ##==--
-#
-release.5:
-	# Create any "synthetic dists" now.
-	@for i in ${DISTRIBUTIONS}; do \
-		if [ -f ${.CURDIR}/scripts/$${i}-make.sh ]; then \
-			echo -n "Running $$i dist creation script... "; \
-			env RD=${RD} sh ${.CURDIR}/scripts/$${i}-make.sh || echo "$$i distribution script returned bad status."; \
-			echo "Done."; \
-		fi \
-	done \
-
-	# Remove all the directories we don't need.
-	-cd ${RD}/trees && \
-		(find ${OTHER_DISTS} -path '*/var/empty' | xargs chflags noschg; \
-		find ${OTHER_DISTS} -depth -type d -empty -delete)
-	touch ${.TARGET}
-
-#
-# --==## Package up the tarballs from assembled trees ##==--
-#
-release.6:
-	rm -rf ${RD}/dists
-	mkdir -p ${RD}/dists
-	@for i in ${DISTRIBUTIONS} ; \
-	do \
-		if [ -d ${RD}/trees/$${i} ] ; then \
-			cd ${.CURDIR} && $(SMAKE) doTARBALL \
-				SD=${RD}/trees/$${i} \
-				TN=$$i TD=$$i ARG="." && \
-			echo "$${i} distribution is finished."; \
-		fi ; \
-	done
-	@for i in ${KERNELS_BASE} ${KERNELS} ; \
-	do \
-		if [ -d ${RD}/kernels/$${i} ] ; then \
-			cd ${.CURDIR} && $(SMAKE) doTARBALL \
-				SD=${RD}/kernels \
-				TN=$$i TD=kernels ARG="$$i" && \
-			echo "$${i} distribution is finished."; \
-		fi ; \
-	done
-.if !defined(NOPORTS)
-	# XXX: Inline stripped version of doTARBALL
-	@rm -rf ${RD}/dists/ports/ports*
-	@mkdir -p ${RD}/dists/ports
-	@echo rolling ports/ports tarball
-	@tar --exclude CVS --exclude .svn --exclude 'ports/distfiles/*' \
-	  -czf ${RD}/dists/ports/ports.tgz -C /usr ports
-	@cp ${.CURDIR}/scripts/ports-install.sh ${RD}/dists/ports/install.sh
-	@(cd ${RD}/dists/ports; \
-	  rm -f CHECKSUM.MD5 CHECKSUM.SHA256; \
-	  md5 * > .CHECKSUM.MD5; \
-	  sha256 * > .CHECKSUM.SHA256; \
-	  mv .CHECKSUM.MD5 CHECKSUM.MD5; \
-	  mv .CHECKSUM.SHA256 CHECKSUM.SHA256)
-	@echo "ports distribution is finished."
-.endif
-	touch ${.TARGET}
-
-
-#
-# --==## Make source dists ##==--
-#
-release.7:
-.if !defined(NOSRC)
-	@cd ${.CURDIR} && $(SMAKE) doTARBALL SD=/usr/src \
-		TD=src TN=sbase ARG="[A-Z]*"
-	@for i in `cd /usr/src && echo [a-z]*` ; do \
-		if [ -d /usr/src/$$i ] ; then \
-			cd ${.CURDIR} && $(SMAKE) doTARBALL \
-				TN=`echo s$$i | tr -d '.' | \
-				    sed -e 's/usr/u/' \
-					-e 's/kerberos5/krb5/'` \
-				SD=/usr/src TD=src ARG="$$i" ; \
-		fi ; \
-	done
-.if defined(EXTRA_SRC)
-	@set ${EXTRA_SRC} && \
-	while [ $$# -ge 2 ] ; do \
-		if [ -d /usr/src/$$1 ] ; then \
-			cd ${.CURDIR} && $(SMAKE) doTARBALL \
-				SD=/usr/src TN="s$$2" TD=src ARG="$$1" ; \
-		fi && shift && shift ; \
-	done
-.endif
-	(cd ${RD}/dists/src; \
-	  rm -f CHECKSUM.MD5 CHECKSUM.SHA256; \
-	  md5 * > .CHECKSUM.MD5; \
-	  sha256 * > .CHECKSUM.SHA256; \
-	  mv .CHECKSUM.MD5 CHECKSUM.MD5; \
-	  mv .CHECKSUM.SHA256 CHECKSUM.SHA256)
-	@echo "src distribution is finished."
-.endif
-	touch ${.TARGET}
-
-# Build the memory root filesystem.
-release.8:
-	cp ${RD}/trees/base/etc/disktab /etc
-	rm -rf ${RD}/mfsfd
-	mkdir ${RD}/mfsfd
-	cd ${RD}/mfsfd && \
-		mkdir -p etc/defaults dev mnt stand/etc/defaults stand/help \
-		var/empty
-	@cd ${.CURDIR} && $(SMAKE) installCRUNCH CRUNCH=boot \
-		DIR=${RD}/mfsfd/stand ZIP=false
-	( cd ${RD}/mfsfd && \
-	  for dir in bin sbin ; do \
-		ln -sf /stand $$dir; \
-	  done )
-	cp ${RD}/trees/base/sbin/dhclient-script ${RD}/mfsfd/stand
-	cp ${.CURDIR}/../etc/master.passwd ${RD}/mfsfd/etc/master.passwd
-	cp ${RD}/trees/base/etc/*pwd.db ${RD}/mfsfd/etc/
-	( for F in defaults/rc.conf netconfig protocols ; do \
-		sed -e '/^#.*$$/d' -e 's/[:space:]*#.*$$//g' \
-		${RD}/trees/base/etc/$$F > ${RD}/mfsfd/stand/etc/$$F ; \
-	  done )
-	grep -E '^(ftp|nameserver|domain|sunrpc|cmd|nfsd)[^-\w]' \
-	    ${RD}/trees/base/etc/services | \
-	    sed -e '/^#.*$$/d' -e 's/[:space:]*#.*$$//g' \
-	    > ${RD}/mfsfd/stand/etc/services
-	grep 'operator' ${RD}/trees/base/etc/group \
-	    > ${RD}/mfsfd/stand/etc/group
-	ln ${RD}/mfsfd/stand/etc/services ${RD}/mfsfd/etc/services
-	ln ${RD}/mfsfd/stand/etc/group ${RD}/mfsfd/etc/group
-	ln ${RD}/mfsfd/stand/etc/netconfig ${RD}/mfsfd/etc/netconfig
-	cp ${RD}/trees/base/COPYRIGHT ${RD}/mfsfd/stand/help/COPYRIGHT.hlp
-.if !defined(NODOC)
-	@for i in ${DIST_DOCS_ARCH_INDEP}; do \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/article.txt \
-	      ${RD}/mfsfd/stand/help/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-	done
-	@for i in ${DIST_DOCS_ARCH_DEP}; do \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.txt \
-	      ${RD}/mfsfd/stand/help/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-	done
-.endif
-	-test -f ${.CURDIR}/install.cfg \
-	    && cp ${.CURDIR}/install.cfg ${RD}/mfsfd
-	@mkdir -p ${RD}/mfsfd/boot
-.if ${TARGET_ARCH} != "ia64" && ${TARGET_CPUARCH} != "powerpc"
-	@cp ${RD}/trees/base/boot/boot* ${RD}/mfsfd/boot
-.endif
-.if ${TARGET} == "i386" || ${TARGET_ARCH} == "amd64"
-	@cp ${RD}/trees/base/boot/mbr ${RD}/mfsfd/boot
-.endif
-	@tar --exclude CVS --exclude .svn -cf - \
-		-C ${.CURDIR}/../usr.sbin/sysinstall help | \
-		tar xf - -C ${RD}/mfsfd/stand
-	@mkdir -p ${RD}/mfsroot
-	sh -e ${DOFS_SH} ${RD}/mfsroot/mfsroot ${RD} ${MNT} \
-	    ${MFSSIZE} ${RD}/mfsfd ${MFSINODE} ${MFSLABEL}
-	@${ZIPPER} -fv ${RD}/mfsroot/mfsroot
-	touch ${.TARGET}
-
-KERNFLOPPYSET=		${RD}/floppyset/kern/kernel.gz
-.if defined(SMALLFLOPPYSIZE)
-SMALLKERNFLOPPYSET=	${RD}/floppyset/kern-small/kernel.gz
-.endif
-.if defined(SPLIT_MFSROOT)
-MFSROOTFLOPPYSET=	${RD}/floppyset/mfsroot/mfsroot.gz
-.if defined(SMALLFLOPPYSIZE)
-SMALLMFSROOTFLOPPYSET=	${RD}/floppyset/mfsroot-small/mfsroot.gz
-.endif
-.endif
-
-# Build boot and install floppies.
-floppies.1:
-	@${ZIPPER} -c ${RD}/kernels/${KERN_GENERIC}/kernel > ${RD}/kernels/kernel.gz
-	@echo "Making the kernel boot floppies..."
-	@cd ${.CURDIR} && ${SMAKE} makeFloppySet FLOPPYBASE=kern \
-	    FLOPPYDESC="Kernel" SPLITFILE=${RD}/kernels/kernel.gz
-.if defined(SMALLFLOPPYSIZE)
-	@echo "Making the small kernel boot floppies..."
-	@cd ${.CURDIR} && ${SMAKE} makeFloppySet FLOPPYBASE=kern-small \
-	    FLOPPYDESC="Kernel" SPLITFILE=${RD}/kernels/kernel.gz \
-	    FDSIZE="SMALL"
-.endif
-.if defined(SPLIT_MFSROOT)
-	@echo "Making the mfsroot boot floppies..."
-	@cd ${.CURDIR} && ${SMAKE} makeFloppySet FLOPPYBASE=mfsroot \
-	    FLOPPYDESC="Memory Filesystem" SPLITFILE=${RD}/mfsroot/mfsroot.gz
-.if defined(SMALLFLOPPYSIZE)
-	@echo "Making the small mfsroot boot floppies..."
-	@cd ${.CURDIR} && ${SMAKE} makeFloppySet FLOPPYBASE=mfsroot-small \
-	    FLOPPYDESC="Memory Filesystem" SPLITFILE=${RD}/mfsroot/mfsroot.gz \
-	    FDSIZE="SMALL"
-.endif
-	@cd ${.CURDIR} && ${SMAKE} buildBootFloppy FSIMAGE="boot" \
-	    KERNFILE="${KERNFLOPPYSET}.split ${KERNFLOPPYSET}.boot" \
-	    MFSROOTFILE="${MFSROOTFLOPPYSET}.split ${MFSROOTFLOPPYSET}.boot"
-.if defined(SMALLFLOPPYSIZE)
-	@cd ${.CURDIR} && ${SMAKE} buildBootFloppy FSIMAGE="boot-small" \
-	    KERNFILE="${SMALLKERNFLOPPYSET}.split ${SMALLKERNFLOPPYSET}.boot" \
-	    MFSROOTFILE="${SMALLMFSROOTFLOPPYSET}.split ${SMALLMFSROOTFLOPPYSET}.boot" \
-	    FDSIZE="SMALL"
-.endif
-.else	# !SPLIT_MFSROOT
-	@cd ${.CURDIR} && ${SMAKE} buildBootFloppy FSIMAGE="boot" \
-	    KERNFILE="${KERNFLOPPYSET}.split ${KERNFLOPPYSET}.boot" \
-	    MFSROOTFILE=${RD}/mfsroot/mfsroot.gz
-.if defined(SMALLFLOPPYSIZE)
-	@cd ${.CURDIR} && ${SMAKE} buildBootFloppy FSIMAGE="boot-small" \
-	    KERNFILE="${SMALLKERNFLOPPYSET}.split ${SMALLKERNFLOPPYSET}.boot" \
-	    MFSROOTFILE=${RD}/mfsroot/mfsroot.gz \
-	    FDSIZE="SMALL"
-.endif
-.endif	# SPLIT_MFSROOT
-	touch ${.TARGET}
-
-# Build fixit floppy.
-floppies.2:
-	@echo "Making fixit floppy."
-	@rm -rf ${RD}/fixitfd
-	@mkdir ${RD}/fixitfd
-	@cd ${RD}/fixitfd && \
-	    mkdir -p dev stand bin sbin etc mnt mnt1 mnt2 mnt3 mnt4 tmp \
-	    usr/share/misc
-	@cp ${RD}/trees/base/etc/spwd.db ${RD}/trees/base/etc/group \
-	    ${RD}/fixitfd/etc
-	@sed -e 's/#.*//' ${RD}/trees/base/etc/protocols \
-	    > ${RD}/fixitfd/etc/protocols
-	@sed -e 's/#.*//' ${RD}/trees/base/usr/share/misc/scsi_modes \
-	    > ${RD}/fixitfd/usr/share/misc/scsi_modes
-	@cp ${.CURDIR}/fixit.profile ${RD}/fixitfd/.profile
-	@cp ${.CURDIR}/fixit.services ${RD}/fixitfd/etc/services
-	@cp ${.CURDIR}/scripts/tar.sh ${RD}/fixitfd/stand/tar
-	@chmod 555 ${RD}/fixitfd/stand/tar
-.if defined(SMALLFLOPPYSIZE)
-	@cd ${.CURDIR} && ${SMAKE} installCRUNCH CRUNCH=fixit-small \
-	    DIR=${RD}/fixitfd/stand ZIP=false
-	@sh -e ${DOFS_SH} ${RD}/floppies/fixit-small.flp ${RD} ${MNT} \
-	    ${SMALLFLOPPYSIZE} ${RD}/fixitfd ${FLOPPYINODE} ${SMALLFLOPPYLABEL}
-	@rm -rf ${RD}/fixitfd/stand
-	@mkdir ${RD}/fixitfd/stand
-.endif
-	@cd ${.CURDIR} && ${SMAKE} installCRUNCH CRUNCH=fixit \
-	    DIR=${RD}/fixitfd/stand ZIP=false
-	@sh -e ${DOFS_SH} ${RD}/floppies/fixit.flp ${RD} ${MNT} \
-	    ${FLOPPYSIZE} ${RD}/fixitfd ${FLOPPYINODE} ${FLOPPYLABEL}
-	touch ${.TARGET}
-
-# Do our last minute floppies directory setup
-floppies.3:
-	@(cd ${RD}/floppies; md5 *.flp > CHECKSUM.MD5)
-	@(cd ${RD}/floppies; sha256 *.flp > CHECKSUM.SHA256)
-	touch ${.TARGET}
-
-#
-# --==## Setup a suitable ftp-area ##==--
-#
-ftp.1:
-	@echo "Setting up FTP distribution area"
-	@mkdir -p ${FD}
-	-@ln -s . ${FD}/${BUILDNAME}
-.if defined(MAKE_FLOPPIES)
-	@cd ${RD} && find floppies -print | cpio -dumpl ${FD}
-.endif
-	@cd ${RD}/dists && find . -print | cpio -dumpl ${FD}
-.if !defined(NODOC)
-	@for i in ${DIST_DOCS_ARCH_INDEP}; do \
-		cp ${RND}/${RELNOTES_LANG}/$$i/article.txt \
-		    ${FD}/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-		cp ${RND}/${RELNOTES_LANG}/$$i/article.html \
-		    ${FD}/`echo $${i} | tr 'a-z' 'A-Z'`.HTM; \
-	done
-	@for i in ${DIST_DOCS_ARCH_DEP}; do \
-		cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.txt \
-		    ${FD}/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-		cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.html \
-		    ${FD}/`echo $${i} | tr 'a-z' 'A-Z'`.HTM; \
-	done
-	@cp ${RND}/${RELNOTES_LANG}/readme/docbook.css ${FD}
-.endif
-	@echo "CD_VERSION = ${BUILDNAME}" > ${FD}/cdrom.inf
-	touch ${.TARGET}
-
-# Build a live filesystem cdrom image
-cdrom.1:
-	@echo "Building CDROM live filesystem image"
-	@mkdir -p ${CD_LIVEFS}
-	@for i in ${DISTRIBUTIONS} ; \
-	do \
-		if [ -d ${RD}/trees/$${i} ] ; then \
-			chflags -R noschg ${RD}/trees/$${i} || true ; \
-			( cd ${RD}/trees/$${i} && \
-			    find . -depth -print | cpio -dumpl ${CD_LIVEFS} ) ; \
-		fi \
-	done
-.if defined(MAKE_DVD)
-	@echo "Building DVD filesystem image as well as CDROM"
-	@mkdir -p ${CD_DVD1}/${BUILDNAME}
-	@for i in ${DISTRIBUTIONS} ; \
-	do \
-		if [ -d ${RD}/trees/$${i} ] ; then \
-			chflags -R noschg ${RD}/trees/$${i} || true ; \
-			( cd ${RD}/trees/$${i} && \
-			    find . -depth -print | cpio -dumpl ${CD_DVD1} ) ; \
-		fi \
-	done
-.endif
-	@echo "Copy GENERIC kernel to boot area"
-	@cp -Rp ${RD}/kernels/${KERN_GENERIC}/ ${CD_LIVEFS}/boot/kernel
-	@rm -f ${CD_LIVEFS}/boot/kernel/*.symbols
-	@rm -f ${CD_LIVEFS}/.profile
-	@cp ${.CURDIR}/fixit.profile ${CD_LIVEFS}/.profile
-	@ln -sf /rescue ${CD_LIVEFS}/stand
-	@echo "Setting up CDROM boot area"
-	@rm -f ${CD_LIVEFS}/boot/loader.conf
-	@cp ${RD}/mfsroot/mfsroot.gz ${CD_LIVEFS}/boot/mfsroot.gz
-	@echo 'mfsroot_load="YES"' > ${CD_LIVEFS}/boot/loader.conf
-	@echo 'mfsroot_type="mfs_root"' >> ${CD_LIVEFS}/boot/loader.conf
-	@echo 'mfsroot_name="/boot/mfsroot"' >> ${CD_LIVEFS}/boot/loader.conf
-.if exists(${RD}/trees/base/boot/device.hints)
-	# Break the link to device.hints so we can modify it
-	@rm -f ${CD_LIVEFS}/boot/device.hints
-	@cp ${RD}/trees/base/boot/device.hints ${CD_LIVEFS}/boot/device.hints
-.endif
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_LIVEFS}/cdrom.inf
-.if defined(MAKE_DVD)
-	@cp -Rp ${RD}/kernels/${KERN_GENERIC}/ ${CD_DVD1}/boot/kernel
-	@rm -f ${CD_DVD1}/boot/kernel/*.symbols
-	@rm -f ${CD_DVD1}/.profile
-	@cp ${.CURDIR}/fixit.profile ${CD_DVD1}/.profile
-	@ln -sf /rescue ${CD_DVD1}/stand
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_DVD1}/cdrom.inf
-	@rm -f ${CD_DVD1}/boot/loader.conf
-	@cp ${RD}/mfsroot/mfsroot.gz ${CD_DVD1}/boot/mfsroot.gz
-	@echo 'mfsroot_load="YES"' > ${CD_DVD1}/boot/loader.conf
-	@echo 'mfsroot_type="mfs_root"' >> ${CD_DVD1}/boot/loader.conf
-	@echo 'mfsroot_name="/boot/mfsroot"' >> ${CD_DVD1}/boot/loader.conf
-.if exists(${RD}/trees/base/boot/device.hints)
-	@rm -f ${CD_DVD1}/boot/device.hints
-	@cp ${RD}/trees/base/boot/device.hints ${CD_DVD1}/boot/device.hints
-.endif
-.endif
-	touch ${.TARGET}
-
-# Build disc1 and disc2 cdrom images
-cdrom.2:
-	@echo "Building CDROM disc1 filesystem image"
-	@mkdir -p ${CD_DISC1}/${BUILDNAME}
-.if defined(MAKE_FLOPPIES)
-	@cd ${RD} && find floppies -print | cpio -dumpl ${CD_DISC1}
-.endif
-	@cd ${RD}/dists && find . -print | cpio -dumpl ${CD_DISC1}/${BUILDNAME}
-.if !defined(NODOC)
-	@for i in ${DIST_DOCS_ARCH_INDEP}; do \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/article.txt \
-	      ${CD_DISC1}/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/article.html \
-	      ${CD_DISC1}/`echo $${i} | tr 'a-z' 'A-Z'`.HTM; \
-	done
-	@for i in ${DIST_DOCS_ARCH_DEP}; do \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.txt \
-	      ${CD_DISC1}/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.html \
-	      ${CD_DISC1}/`echo $${i} | tr 'a-z' 'A-Z'`.HTM; \
-	done
-	@cp ${RND}/${RELNOTES_LANG}/readme/docbook.css ${CD_DISC1}
-.endif
-.if defined(SEPARATE_LIVEFS)
-	@cp -Rp ${CD_LIVEFS}/boot ${CD_DISC1}
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_DISC1}/cdrom.inf
-.endif
-	@echo "CD_VOLUME = 1" >> ${CD_DISC1}/cdrom.inf
-	@echo "Building CDROM disc2 filesystem image"
-	@mkdir -p ${CD_DISC2}
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_DISC2}/cdrom.inf
-	@echo "CD_VOLUME = 2" >> ${CD_DISC2}/cdrom.inf
-.if defined(MAKE_DVD)
-.if defined(MAKE_FLOPPIES)
-	@cd ${RD} && find floppies -print | cpio -dumpl ${CD_DVD1}
-.endif
-	@cd ${RD}/dists && find . -print | cpio -dumpl ${CD_DVD1}/${BUILDNAME}
-.if !defined(NODOC)
-	@for i in ${DIST_DOCS_ARCH_INDEP}; do \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/article.txt \
-	      ${CD_DVD1}/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/article.html \
-	      ${CD_DVD1}/`echo $${i} | tr 'a-z' 'A-Z'`.HTM; \
-	done
-	@for i in ${DIST_DOCS_ARCH_DEP}; do \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.txt \
-	      ${CD_DVD1}/`echo $${i} | tr 'a-z' 'A-Z'`.TXT; \
-	  cp ${RND}/${RELNOTES_LANG}/$$i/${TARGET}/article.html \
-	      ${CD_DVD1}/`echo $${i} | tr 'a-z' 'A-Z'`.HTM; \
-	done
-	@cp ${RND}/${RELNOTES_LANG}/readme/docbook.css ${CD_DVD1}
-.endif
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_DVD1}/cdrom.inf
-	@echo "CD_VOLUME = 1" >> ${CD_DVD1}/cdrom.inf
-.endif
-.if !defined(NODOC)
-	@echo "Building CDROM docs filesystem image"
-	@mkdir -p ${CD_DOCS}
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_DOCS}/cdrom.inf
-	@echo "CD_VOLUME = 3" >> ${CD_DOCS}/cdrom.inf
-.endif
-	touch ${.TARGET}
-
-#
-# --==## Setup a suitable cdrom-area ##==--
-#
-cdrom.3:
-.if defined(CD_BOOT)
-	@echo "Building bootonly CDROM filesystem image"
-	@mkdir -p ${CD_BOOT}
-	@cp -Rp ${CD_LIVEFS}/boot ${CD_BOOT}
-	@echo "CD_VERSION = ${BUILDNAME}" > ${CD_BOOT}/cdrom.inf
-.if defined(MINIROOT)
-	@echo "Building bootonly UFS filesystem image"
-	@mkdir -p ${FD}/miniroot
-	@sh -e ${DOFS_SH} ${FD}/miniroot/miniroot.ufs \
-	    ${RD} ${MNT} 0 ${CD_BOOT} 8192 auto
-	@${ZIPPER} -v ${FD}/miniroot/miniroot.ufs
-.endif
-.endif
-	touch ${.TARGET}
-
-.if make(iso.1)
-.if defined(CD_PACKAGE_TREE)
-.if exists(${CD_PACKAGE_TREE}/disc1)
-CD_DISC1_PKGS=	${CD_PACKAGE_TREE}/disc1
-.endif
-.if exists(${CD_PACKAGE_TREE}/disc2)
-CD_DISC2_PKGS=	${CD_PACKAGE_TREE}/disc2
-.endif
-# scripts/package-trees.sh names all discs according to the "discX"
-# scheme where X is the number of the disc
-.if exists(${CD_PACKAGE_TREE}/disc3)
-CD_DOCS_PKGS=	${CD_PACKAGE_TREE}/disc3
-.endif
-.if exists(${CD_PACKAGE_TREE}/dvd1)
-CD_DVD1_PKGS=	${CD_PACKAGE_TREE}/dvd1
-.endif
-.endif
-.endif
-
-iso.1:
-.if exists(${.CURDIR}/${TARGET}/mkisoimages.sh)
-	@echo "Creating ISO images..."
-.if defined(CD_BOOT)
-	@sh ${.CURDIR}/${TARGET}/mkisoimages.sh ${BOOTABLE} \
-	    FreeBSD_bootonly \
-	    ${CD}/FreeBSD-${BUILDNAME}-${TARGET}-bootonly.iso ${CD_BOOT}
-.endif
-	@sh ${.CURDIR}/${TARGET}/mkisoimages.sh ${BOOTABLE} \
-	    FreeBSD_Install \
-	    ${CD}/FreeBSD-${BUILDNAME}-${TARGET}-disc1.iso ${CD_DISC1} \
-	    ${CD_DISC1_PKGS}
-	@sh ${.CURDIR}/${TARGET}/mkisoimages.sh \
-	    FreeBSD_Packages \
-	    ${CD}/FreeBSD-${BUILDNAME}-${TARGET}-disc2.iso ${CD_DISC2} \
-	    ${CD_DISC2_PKGS}
-.if defined(MAKE_DVD)
-	@sh ${.CURDIR}/${TARGET}/mkisoimages.sh ${BOOTABLE} \
-	    FreeBSD_Install \
-	    ${CD}/FreeBSD-${BUILDNAME}-${TARGET}-dvd1.iso ${CD_DVD1} \
-	    ${CD_DVD1_PKGS}
-.endif
-.if !defined(NODOC)
-	@sh ${.CURDIR}/${TARGET}/mkisoimages.sh \
-	    FreeBSD_Documentation \
-	    ${CD}/FreeBSD-${BUILDNAME}-${TARGET}-disc3.iso ${CD_DOCS} \
-	    ${CD_DOCS_PKGS}
-.endif
-.if defined(SEPARATE_LIVEFS)
-	@sh ${.CURDIR}/${TARGET}/mkisoimages.sh ${BOOTABLE} \
-	    FreeBSD_LiveFS \
-	    ${CD}/FreeBSD-${BUILDNAME}-${TARGET}-livefs.iso ${CD_LIVEFS}
-.endif
-	@echo "Generating MD5 and SHA256 sums..."
-	@(cd ${CD} && md5 *.iso > FreeBSD-${BUILDNAME}-${TARGET}-iso.CHECKSUM.MD5)
-	@(cd ${CD} && sha256 *.iso > FreeBSD-${BUILDNAME}-${TARGET}-iso.CHECKSUM.SHA256)
-	touch ${.TARGET}
-.else
-	@echo "Do not know how to create an ISO for ${TARGET}."
-.endif
-
-#
-# --==## Documentation Project tools required to build the release notes ##==--
-#
-doc.1:
-	@echo "Making docproj tools..."
-	@for i in ${DOCPORTS}; do \
-	    cd /usr/ports/$$i && \
-	    env -i FTP_PASSIVE_MODE=$${FTP_PASSIVE_MODE:-no} PATH=$${PATH} \
-		make all install clean BATCH=yes WITHOUT_X11=yes JADETEX=no \
-		WITHOUT_PYTHON=yes FORCE_PKG_REGISTER=yes; \
-	done
-	touch ${.TARGET}
-
-#
-# --==## RELNOTESng:  Next-generation replacements for *.TXT files ##==--
-#
-doc.2:
-	@echo "Making release documentation..."
-	@cd ${.CURDIR}/doc && make all install clean 'FORMATS=html txt' \
-	    INSTALL_COMPRESSED='' URLS_ABSOLUTE=YES DOCDIR=${RND}
-	touch ${.TARGET}
-
-# Various "subroutine" and other supporting targets.
-
-# RD=
-# SD=
-# TD=
-# ARG=
-doTARBALL:
-.if !defined(SD)
-	@echo "SD undefined in doTARBALL" && exit 1
-.endif
-.if !defined(TD)
-	@echo "TD undefined in doTARBALL" && exit 1
-.endif
-.if !defined(ARG)
-	@echo "ARG undefined in doTARBALL" && exit 1
-.endif
-	@rm -rf ${RD}/dists/${TD}/${TN}*
-	@mkdir -p ${RD}/dists/${TD}
-	@( cd ${SD} && \
-		tn=`echo ${TN} | tr 'A-Z' 'a-z'` && \
-		echo rolling ${TD}/$$tn tarball &&\
-		tar --exclude CVS --exclude .svn --exclude obj --exclude BOOTMFS -cf - ${ARG} | \
-		${ZIPNSPLIT} ${RD}/dists/${TD}/$$tn. && \
-		sh ${.CURDIR}/scripts/info.sh ${RD}/dists/${TD}/$$tn \
-		    > ${RD}/dists/${TD}/$$tn.inf && \
-		if [ -f ${.CURDIR}/scripts/$${TD}-install.sh ]; then \
-			cp -p ${.CURDIR}/scripts/$${TD}-install.sh \
-			    ${RD}/dists/${TD}/install.sh && \
-			chmod +x ${RD}/dists/${TD}/install.sh; \
-		fi && \
-		if [ "${SD}" != "/usr/src" ]; then \
-			mtree -c -i -p ${SD}/${ARG} \
-			  -k gname,md5digest,mode,nlink,uname,size,link,type \
-			  > ${RD}/dists/${TD}/$$tn.mtree ; \
-		else \
-			true; \
-		fi; \
-		( cd ${RD}/dists/${TD}; \
-		rm -f CHECKSUM.MD5 CHECKSUM.SHA256; \
-		md5 * > .CHECKSUM.MD5; \
-		sha256 * > .CHECKSUM.SHA256; \
-		mv .CHECKSUM.MD5 CHECKSUM.MD5; \
-		mv .CHECKSUM.SHA256 CHECKSUM.SHA256) \
-	)
-
-doRELEASE: release.1 release.2 ${DOCREL} release.3 release.4 \
-    release.5 release.6 release.7 release.8 ${EXTRAS}
-	@echo "Release done"
-
-floppies:
-	@rm -f release.4 release.8 floppies.[123]
-	@cd ${.CURDIR} && ${SMAKE} release.4 release.8 floppies.1 floppies.2 \
-	    floppies.3
-	@cd ${RD} && find floppies -print | cpio -dumpl ${FD}
-
-installCRUNCH:
-.if !defined(CRUNCH)
-	@echo "CRUNCH undefined in installCRUNCH" && exit 1
-.endif
-.if !defined(DIR)
-	@echo "DIR undefined in installCRUNCH" && exit 1
-.endif
-.if !defined(ZIP)
-	@echo "ZIP undefined in installCRUNCH" && exit 1
-.endif
-	@if ${ZIP} ; then \
-		${ZIPPER} < ${RD}/crunch/${CRUNCH} > ${DIR}/${CRUNCH}_crunch ; \
-	else \
-		ln -f ${RD}/crunch/${CRUNCH} ${DIR}/${CRUNCH}_crunch ; \
-	fi
-	@chmod 555 ${DIR}/${CRUNCH}_crunch
-	@if [ -f ${.CURDIR}/${TARGET}/${CRUNCH}_crunch.conf ] ; then \
-		for i in `crunchgen -l ${.CURDIR}/${TARGET}/${CRUNCH}_crunch.conf` ; do \
-			ln -f ${DIR}/${CRUNCH}_crunch ${DIR}/$$i ; \
-		done \
-	else \
-		for i in `crunchgen -l ${.CURDIR}/${CRUNCH}_crunch.conf` ; do \
-			ln -f ${DIR}/${CRUNCH}_crunch ${DIR}/$$i ; \
-		done \
-	fi
-
-#
-# --==## Build a floppy set for a splitfs file ##==--
-#
-# FLOPPYBASE - basename of floppy image files
-# FLOPPYDESC - description of floppy set
-# SPLITFILE - filename of the file to split
-# FDSIZE - if specified and "small", small floppy is created
-
-.if make(makeFloppySet)
-SPLITDIR=	${RD}/floppyset/${FLOPPYBASE}
-.if defined(FDSIZE) && ${FDSIZE} == "SMALL"
-FLPSPLITSIZE=	${SMALLFLOPPYSPLITSIZE}
-FLPSIZE=	${SMALLFLOPPYSIZE}
-FLPLABEL=	${SMALLFLOPPYLABEL}
-.else
-FLPSPLITSIZE=	${FLOPPYSPLITSIZE}
-FLPSIZE=	${FLOPPYSIZE}
-FLPLABEL=	${FLOPPYLABEL}
-.endif
-.endif
-
-makeFloppySet:
-.if !defined(FLOPPYBASE)
-	@echo "FLOPPYBASE undefined in ${.TARGET}" && exit 1
-.endif
-.if !defined(FLOPPYDESC)
-	@echo "FLOPPYDESC undefined in ${.TARGET}" && exit 1
-.endif
-.if !defined(SPLITFILE)
-	@echo "SPLITFILE undefined in ${.TARGET}" && exit 1
-.endif
-	sh ${.CURDIR}/scripts/split-file.sh ${SPLITFILE} \
-	    ${RD}/floppyset/${FLOPPYBASE} ${FLPSPLITSIZE} "${FLOPPYDESC}"
-	( splitfile=${SPLITDIR}/`basename ${SPLITFILE}`.split ; \
-	lines=`cat $${splitfile} | wc -l`; \
-	lines=$$(($$lines - 1)) ; \
-	for line in `jot $$lines`; do \
-		file=`head -n $$(($${line} + 1)) $${splitfile} | tail -1 | cut -f 1 -d ' '` ; \
-		sh -e ${DOFS_SH} ${RD}/floppies/${FLOPPYBASE}$${line}.flp \
-		${RD} ${MNT} ${FLPSIZE} ${SPLITDIR}/$${file} \
-		${BOOTINODE} ${FLPLABEL}; \
-	done )
-
-#
-# --==## Build a boot floppy ##==--
-#
-# FSIMAGE - base floppy image name
-# FDSIZE - if specified and "small", small floppy is created
-# KERNFILE - path to kernel split file
-# MFSROOTFILE - path to mfsroot split file
-
-.if make(buildBootFloppy)
-IMAGEDIR=	${RD}/image.${FSIMAGE}
-BOOTDIR=	${RD}/trees/base/boot
-HINTSFILE=	${BOOTDIR}/device.hints
-ACPI_KO=	${RD}/kernels/${KERN_GENERIC}/acpi.ko
-IMAGEFILE=	${RD}/floppies/${FSIMAGE}.flp
-.if defined(FDSIZE) && ${FDSIZE} == "SMALL"
-FLPSIZE=	${SMALLFLOPPYSIZE}
-FLPLABEL=	${SMALLFLOPPYLABEL}
-.else
-FLPSIZE=	${FLOPPYSIZE}
-FLPLABEL=	${FLOPPYLABEL}
-.endif
-.endif
-
-buildBootFloppy:
-.if !defined(FSIMAGE)
-	@echo "FSIMAGE undefined in ${.TARGET}" && exit 1
-.endif
-.if !defined(KERNFILE)
-	@echo "KERNFILE undefined in ${.TARGET}" && exit 1
-.endif
-.if !defined(MFSROOTFILE)
-	@echo "MFSROOTFILE undefined in ${.TARGET}" && exit 1
-.endif
-	@echo "Running ${.TARGET} for ${FSIMAGE}"
-	@rm -rf ${IMAGEDIR}
-	@mkdir ${IMAGEDIR}
-	@echo "Setting up /boot directory for ${FSIMAGE} floppy"
-	@mkdir -p ${IMAGEDIR}/boot
-.if ${TARGET_ARCH} == "i386"
-	@${WMAKEENV} kgzip -v -l ${RD}/trees/base/usr/lib/kgzldr.o -o \
-	    ${IMAGEDIR}/boot/loader ${BOOTDIR}/loader
-.else
-	@cp ${BOOTDIR}/loader ${IMAGEDIR}/boot
-.endif
-	@cp -Rp ${BOOTDIR}/*.4th ${BOOTDIR}/defaults ${BOOTDIR}/loader.help \
-	    ${BOOTDIR}/loader.rc ${IMAGEDIR}/boot
-.if exists(${HINTSFILE})
-	@cp ${HINTSFILE} ${IMAGEDIR}/boot/device.hints
-	@${ZIPPER} ${IMAGEDIR}/boot/device.hints
-.endif
-	@${ZIPPER} ${IMAGEDIR}/boot/*.4th ${IMAGEDIR}/boot/loader.help \
-	    ${IMAGEDIR}/boot/defaults/loader.conf
-	@echo 'bootfile="/kernel"' > ${IMAGEDIR}/boot/loader.conf
-.if exists(${ACPI_KO})
-	@${ZIPPER} -c ${ACPI_KO} > ${IMAGEDIR}/acpi.ko.gz
-	@echo 'acpi_load="YES"' >> ${IMAGEDIR}/boot/loader.conf
-	@echo 'acpi_name="/acpi.ko"' >> ${IMAGEDIR}/boot/loader.conf
-	@echo 'acpi_before="read -p \"Insert boot floppy and press Enter\""' >> ${IMAGEDIR}/boot/loader.conf
-.endif
-	@echo 'mfsroot_load="YES"' >> ${IMAGEDIR}/boot/loader.conf
-	@echo 'mfsroot_type="mfs_root"' >> ${IMAGEDIR}/boot/loader.conf
-	@echo 'mfsroot_name="/mfsroot"' >> ${IMAGEDIR}/boot/loader.conf
-.if !exists(${ACPI_KO})
-	@echo 'mfsroot_before="read -p \"Insert boot floppy and press Enter\""' >> ${IMAGEDIR}/boot/loader.conf
-.endif
-.if defined(SPLIT_MFSROOT)
-	@echo 'mfsroot_after="read -p \"Insert boot floppy and press Enter\""' >> ${IMAGEDIR}/boot/loader.conf
-.endif
-.if ${TARGET_ARCH} == "i386" || ${TARGET_ARCH} == "amd64"
-.if ${AUTO_KEYBOARD_DETECT}
-	@echo "-P" >> ${IMAGEDIR}/boot.config
-.endif
-.endif
-	@rm -f ${IMAGEFILE}
-	@cp ${KERNFILE} ${MFSROOTFILE} ${IMAGEDIR}
-	sh -e ${DOFS_SH} ${IMAGEFILE} ${RD} ${MNT} ${FLPSIZE} ${IMAGEDIR} \
-	    ${BOOTINODE} ${FLPLABEL}
-	@echo "Created ${RD}/floppies/${FSIMAGE}.flp"
-
-.if make(package-split)
-# Targets related to making a package split
-#
-# PKG_COPY instructs the script to copy the actual package files rather than
-#     make hard links
-# PKG_TREE is the path to the package tree to be split
-# PKG_DEST is the path to the destination tree to create the split in
-# PKG_VERBOSE asks for verbose output of the layout process
-# PKG_INDEX is the path to the INDEX file. By default ${PKG_TREE}/INDEX.
-
-.if defined(PKG_COPY)
-PKG_DO_COPY=	cp
-.else
-PKG_DO_COPY=	ln
-.endif
-PKG_WRKIDX=	${PKG_DEST}/INDEX.master
-PKG_ENV?=
-.if defined(TARGET_ARCH)
-PKG_ENV+=	PKG_ARCH=${TARGET_ARCH}
-.endif
-.if defined(PKG_VERBOSE)
-PKG_ENV+=	PKG_VERBOSE=1
-.endif
-PKG_INDEX?=	${PKG_TREE}/INDEX
-
-package-split:
-.if !defined(PKG_TREE)
-	@echo "PKG_TREE must be defined" && exit 1
-.endif
-.if !defined(PKG_DEST)
-	@echo "PKG_DEST must be defined" && exit 1
-.endif
-	@env ${PKG_ENV} python ${.CURDIR}/scripts/package-split.py \
-	    ${PKG_INDEX} ${PKG_WRKIDX}
-	@env ${PKG_ENV} sh ${.CURDIR}/scripts/package-trees.sh ${PKG_DO_COPY} \
-	    ${PKG_WRKIDX} ${PKG_TREE} ${PKG_DEST}
-.endif
-
-.include <bsd.obj.mk>
diff --git a/release/amd64/boot_crunch.conf b/release/amd64/boot_crunch.conf
deleted file mode 100644
index 3330c9315bc5..000000000000
--- a/release/amd64/boot_crunch.conf
+++ /dev/null
@@ -1,44 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH -Dlint
-
-srcdirs /usr/src/bin
-progs hostname
-progs pwd
-progs rm
-progs sh
-progs test
-ln sh -sh
-ln test [
-
-srcdirs /usr/src/sbin
-progs camcontrol
-progs dhclient
-progs fsck_ffs
-progs ifconfig
-progs mount_nfs
-progs newfs
-progs route
-progs rtsol
-progs tunefs
-ln fsck_ffs fsck_4.2bsd
-ln fsck_ffs fsck_ufs
-
-srcdirs /usr/src/usr.bin
-progs cpio
-progs find
-progs minigzip
-progs sed
-ln minigzip gzip
-ln minigzip gunzip
-ln minigzip zcat
-
-srcdirs /usr/src/usr.sbin
-progs arp
-progs ppp
-progs sysinstall
-progs usbconfig
-
-libs -ll -ledit -lutil -lmd -lcrypt -lftpio -lz -lnetgraph
-libs -lodialog -lncurses -ldisk -lcam -lsbuf -lufs -ldevinfo
-libs -lbsdxml -larchive -lcrypto -lbz2 -llzma -lusb -ljail
diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh
index f26b79dbf7f4..2acecb5ee4be 100755
--- a/release/amd64/make-memstick.sh
+++ b/release/amd64/make-memstick.sh
@@ -13,15 +13,11 @@
 PATH=/bin:/usr/bin:/sbin:/usr/sbin
 export PATH
 
-BLOCKSIZE=10240
-
 if [ $# -ne 2 ]; then
   echo "make-memstick.sh /path/to/directory /path/to/image/file"
   exit 1
 fi
 
-tempfile="${2}.$$"
-
 if [ ! -d ${1} ]; then
   echo "${1} must be a directory"
   exit 1
@@ -32,47 +28,21 @@ if [ -e ${2} ]; then
   exit 1
 fi
 
-echo '/dev/gpt/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab
-rm -f ${tempfile}
-makefs -B little ${tempfile} ${1}
+echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab
+makefs -B little -o label=FreeBSD_Install ${2} ${1}
 if [ $? -ne 0 ]; then
   echo "makefs failed"
   exit 1
 fi
 rm ${1}/etc/fstab
 
-#
-# Use $BLOCKSIZE for transfers to improve efficiency.  When calculating
-# how many blocks to transfer "+ 2" is to account for truncation in the
-# division and to provide space for the label.
-#
-
-filesize=`stat -f "%z" ${tempfile}`
-blocks=$(($filesize / ${BLOCKSIZE} + 256))
-dd if=/dev/zero of=${2} bs=${BLOCKSIZE} count=${blocks}
-if [ $? -ne 0 ]; then
-  echo "creation of image file failed"
-  exit 1
-fi
-
 unit=`mdconfig -a -t vnode -f ${2}`
 if [ $? -ne 0 ]; then
   echo "mdconfig failed"
   exit 1
 fi
-
-gpart create -s GPT ${unit}
-gpart add -t freebsd-boot -s 64K ${unit}
-gpart bootcode -b ${1}/boot/pmbr -p ${1}/boot/gptboot -i 1 ${unit}
-gpart add -t freebsd-ufs -l FreeBSD_Install ${unit}
-
-dd if=${tempfile} of=/dev/${unit}p2 bs=$BLOCKSIZE conv=sync
-if [ $? -ne 0 ]; then
-  echo "copying filesystem into image file failed"
-  exit 1
-fi
-
+gpart create -s BSD ${unit}
+gpart bootcode -b ${1}/boot/boot ${unit}
+gpart add -t freebsd-ufs ${unit}
 mdconfig -d -u ${unit}
 
-rm -f ${tempfile}
-
diff --git a/release/fixit.profile b/release/fixit.profile
deleted file mode 100644
index 337b68bf0661..000000000000
--- a/release/fixit.profile
+++ /dev/null
@@ -1,42 +0,0 @@
-:
-# $FreeBSD$
-
-export BLOCKSIZE=K
-export PS1="Fixit# "
-export EDITOR="/mnt2/rescue/vi"
-export PAGER="/mnt2/usr/bin/more"
-export SCSI_MODES="/mnt2/usr/share/misc/scsi_modes"
-# the root MFS doesn't have /dev/nrsa0, pick a better default for mt(1)
-export TAPE=/mnt2/dev/nrsa0
-# make geom(8) utilities find their modules
-export GEOM_LIBRARY_PATH="/mnt2/lib/geom:/lib/geom"
-
-alias ls="ls -F"
-alias ll="ls -l"
-alias m="more -e"
-
-echo '+---------------------------------------------------------------+'
-echo '| You are now running from FreeBSD "fixit" media.               |'
-echo '| ------------------------------------------------------------- |'
-echo "| When you're finished with this shell, please type exit.       |"
-echo '| The fixit media is mounted as /mnt2.                          |'
-echo '|                                                               |'
-echo '| You might want to symlink /mnt/etc/*pwd.db and /mnt/etc/group |'
-echo '| to /etc after mounting a root filesystem from your disk.      |'
-echo '| tar(1) will not restore all permissions correctly otherwise!  |'
-echo '|                                                               |'
-echo '| In order to load kernel modules you might want to add the     |'
-echo '| fixit media to the kern.module_path sysctl variable so that   |'
-echo '| the kernel knows where to find them.                          |'
-echo '|                                                               |'
-echo '| Note: you can use the arrow keys to browse through the        |'
-echo '| command history of this shell.                                |'
-echo '+---------------------------------------------------------------+'
-echo
-echo 'Good Luck!'
-echo
-
-# Make the arrow keys work; everybody will love this.
-set -o emacs 2>/dev/null
-
-cd /
diff --git a/release/fixit.services b/release/fixit.services
deleted file mode 100644
index ff776b2b6265..000000000000
--- a/release/fixit.services
+++ /dev/null
@@ -1,108 +0,0 @@
-#
-# Network services, Internet style
-#
-# Stripped-down version.
-#
-# $FreeBSD$
-#
-# WELL KNOWN PORT NUMBERS
-#
-echo		  7/tcp
-echo		  7/udp
-discard		  9/tcp	   sink null
-discard		  9/udp	   sink null
-systat		 11/tcp	   users	#Active Users
-systat		 11/udp	   users	#Active Users
-daytime		 13/tcp
-daytime		 13/udp
-chargen		 19/tcp	   ttytst source	#Character Generator
-chargen		 19/udp	   ttytst source	#Character Generator
-ftp-data	 20/tcp	   #File Transfer [Default Data]
-ftp-data	 20/udp	   #File Transfer [Default Data]
-ftp		 21/tcp	   #File Transfer [Control]
-ftp		 21/udp	   #File Transfer [Control]
-ssh		 22/tcp    #Secure Shell Login
-ssh		 22/udp    #Secure Shell Login
-telnet		 23/tcp
-telnet		 23/udp
-#		 24/tcp	   any private mail system
-#		 24/udp	   any private mail system
-smtp		 25/tcp	   mail		#Simple Mail Transfer
-smtp		 25/udp	   mail		#Simple Mail Transfer
-time		 37/tcp	   timserver
-time		 37/udp	   timserver
-domain		 53/tcp	   #Domain Name Server
-domain		 53/udp	   #Domain Name Server
-bootps		 67/tcp	   dhcps	#Bootstrap Protocol Server
-bootps		 67/udp	   dhcps	#Bootstrap Protocol Server
-bootpc		 68/tcp	   dhcpc	#Bootstrap Protocol Client
-bootpc		 68/udp	   dhcpc	#Bootstrap Protocol Client
-tftp		 69/tcp	   #Trivial File Transfer
-tftp		 69/udp	   #Trivial File Transfer
-gopher		 70/tcp
-gopher		 70/udp
-finger		 79/tcp
-finger		 79/udp
-http		 80/tcp	   www www-http		#World Wide Web HTTP
-http		 80/udp	   www www-http		#World Wide Web HTTP
-sunrpc		111/tcp	   rpcbind	#SUN Remote Procedure Call
-sunrpc		111/udp	   rpcbind	#SUN Remote Procedure Call
-auth		113/tcp	   ident tap	#Authentication Service
-auth		113/udp	   ident tap	#Authentication Service
-nntp		119/tcp	   usenet	#Network News Transfer Protocol
-nntp		119/udp	   usenet	#Network News Transfer Protocol
-rmt		411/tcp	   #Remote MT Protocol
-rmt		411/udp	   #Remote MT Protocol
-https		443/tcp
-https		443/udp
-#
-# Berkeley-specific services
-#
-exec		512/tcp	   #remote process execution;
-#			   authentication performed using
-#			   passwords and UNIX loppgin names
-biff		512/udp	   comsat	#used by mail system to notify users
-#					of new mail received; currently
-#					receives messages only from
-#					processes on the same machine
-login		513/tcp	   #remote login a la telnet;
-#			   automatic authentication performed
-#			   based on priviledged port numbers
-#			   and distributed data bases which
-#			   identify "authentication domains"
-who		513/udp	   whod		#maintains data bases showing who's
-#					logged in to machines on a local
-#					net and the load average of the
-#					machine
-cmd		514/tcp	   shell	#like exec, but automatic
-#					authentication is performed as for
-#					login server
-syslog		514/udp
-printer		515/tcp	   spooler
-printer		515/udp	   spooler
-talk		517/tcp	   #like tenex link, but across
-#			   machine - unfortunately, doesn't
-#			   use link protocol (this is actually
-#			   just a rendezvous port from which a
-#			   tcp connection is established)
-talk		517/udp	   #like tenex link, but across
-#			   machine - unfortunately, doesn't
-#			   use link protocol (this is actually
-#			   just a rendezvous port from which a
-#			   tcp connection is established)
-ntalk		518/tcp
-ntalk		518/udp
-utime		519/tcp	   unixtime
-utime		519/udp	   unixtime
-efs		520/tcp	   #extended file name server
-router		520/udp	   route routed	   #local routing process (on site);
-#					   uses variant of Xerox NS routing
-#					   information protocol
-uucp		540/tcp	   uucpd
-uucp		540/udp	   uucpd
-uucp-rlogin	541/tcp
-uucp-rlogin	541/udp
-klogin		543/tcp			# Kerberos (v4/v5)
-klogin		543/udp			# Kerberos (v4/v5)
-kshell		544/tcp	   krcmd	# Kerberos (v4/v5)
-kshell		544/udp	   krcmd	# Kerberos (v4/v5)
diff --git a/release/i386/boot_crunch.conf b/release/i386/boot_crunch.conf
deleted file mode 100644
index 3330c9315bc5..000000000000
--- a/release/i386/boot_crunch.conf
+++ /dev/null
@@ -1,44 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH -Dlint
-
-srcdirs /usr/src/bin
-progs hostname
-progs pwd
-progs rm
-progs sh
-progs test
-ln sh -sh
-ln test [
-
-srcdirs /usr/src/sbin
-progs camcontrol
-progs dhclient
-progs fsck_ffs
-progs ifconfig
-progs mount_nfs
-progs newfs
-progs route
-progs rtsol
-progs tunefs
-ln fsck_ffs fsck_4.2bsd
-ln fsck_ffs fsck_ufs
-
-srcdirs /usr/src/usr.bin
-progs cpio
-progs find
-progs minigzip
-progs sed
-ln minigzip gzip
-ln minigzip gunzip
-ln minigzip zcat
-
-srcdirs /usr/src/usr.sbin
-progs arp
-progs ppp
-progs sysinstall
-progs usbconfig
-
-libs -ll -ledit -lutil -lmd -lcrypt -lftpio -lz -lnetgraph
-libs -lodialog -lncurses -ldisk -lcam -lsbuf -lufs -ldevinfo
-libs -lbsdxml -larchive -lcrypto -lbz2 -llzma -lusb -ljail
diff --git a/release/i386/fixit_crunch.conf b/release/i386/fixit_crunch.conf
deleted file mode 100644
index 703ae847a6e7..000000000000
--- a/release/i386/fixit_crunch.conf
+++ /dev/null
@@ -1,49 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH
-
-srcdirs /usr/src/bin
-progs cat
-progs chmod
-progs cp
-progs dd
-progs df
-progs echo
-progs ed
-progs expr
-progs kill
-progs ln
-progs ls
-progs mkdir
-progs mv
-progs rm
-progs rmdir
-progs sleep
-progs sync
-
-srcdirs /usr/src/sbin
-progs bsdlabel
-progs clri
-progs dmesg
-progs fdisk
-progs mknod
-progs mount
-progs mount_cd9660
-progs mount_msdosfs
-progs reboot
-progs restore
-progs swapon
-progs umount
-ln bsdlabel disklabel
-ln restore rrestore
-
-srcdirs /usr/src/usr.bin
-progs ftp
-progs telnet
-
-srcdirs /usr/src/usr.sbin
-progs chown
-progs chroot
-ln chown chgrp
-
-libs -ledit -lgeom -lbsdxml -lsbuf -lkiconv -lkvm -lncurses -lutil
diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh
index f26b79dbf7f4..2acecb5ee4be 100755
--- a/release/i386/make-memstick.sh
+++ b/release/i386/make-memstick.sh
@@ -13,15 +13,11 @@
 PATH=/bin:/usr/bin:/sbin:/usr/sbin
 export PATH
 
-BLOCKSIZE=10240
-
 if [ $# -ne 2 ]; then
   echo "make-memstick.sh /path/to/directory /path/to/image/file"
   exit 1
 fi
 
-tempfile="${2}.$$"
-
 if [ ! -d ${1} ]; then
   echo "${1} must be a directory"
   exit 1
@@ -32,47 +28,21 @@ if [ -e ${2} ]; then
   exit 1
 fi
 
-echo '/dev/gpt/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab
-rm -f ${tempfile}
-makefs -B little ${tempfile} ${1}
+echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab
+makefs -B little -o label=FreeBSD_Install ${2} ${1}
 if [ $? -ne 0 ]; then
   echo "makefs failed"
   exit 1
 fi
 rm ${1}/etc/fstab
 
-#
-# Use $BLOCKSIZE for transfers to improve efficiency.  When calculating
-# how many blocks to transfer "+ 2" is to account for truncation in the
-# division and to provide space for the label.
-#
-
-filesize=`stat -f "%z" ${tempfile}`
-blocks=$(($filesize / ${BLOCKSIZE} + 256))
-dd if=/dev/zero of=${2} bs=${BLOCKSIZE} count=${blocks}
-if [ $? -ne 0 ]; then
-  echo "creation of image file failed"
-  exit 1
-fi
-
 unit=`mdconfig -a -t vnode -f ${2}`
 if [ $? -ne 0 ]; then
   echo "mdconfig failed"
   exit 1
 fi
-
-gpart create -s GPT ${unit}
-gpart add -t freebsd-boot -s 64K ${unit}
-gpart bootcode -b ${1}/boot/pmbr -p ${1}/boot/gptboot -i 1 ${unit}
-gpart add -t freebsd-ufs -l FreeBSD_Install ${unit}
-
-dd if=${tempfile} of=/dev/${unit}p2 bs=$BLOCKSIZE conv=sync
-if [ $? -ne 0 ]; then
-  echo "copying filesystem into image file failed"
-  exit 1
-fi
-
+gpart create -s BSD ${unit}
+gpart bootcode -b ${1}/boot/boot ${unit}
+gpart add -t freebsd-ufs ${unit}
 mdconfig -d -u ${unit}
 
-rm -f ${tempfile}
-
diff --git a/release/ia64/boot_crunch.conf b/release/ia64/boot_crunch.conf
deleted file mode 100644
index 9ab4f712f7e8..000000000000
--- a/release/ia64/boot_crunch.conf
+++ /dev/null
@@ -1,49 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH -Dlint
-
-srcdirs /usr/src/bin
-progs hostname
-progs pwd
-progs rm
-progs sh
-progs test
-ln sh -sh
-ln test [
-
-srcdirs /usr/src/sbin
-progs camcontrol
-progs dhclient
-progs fsck_ffs
-progs geom
-progs ifconfig
-progs mount_msdosfs
-progs mount_nfs
-progs newfs
-progs newfs_msdos
-progs route
-progs rtsol
-progs tunefs
-ln fsck_ffs fsck_4.2bsd
-ln fsck_ffs fsck_ufs
-ln geom glabel
-ln geom gpart
-
-srcdirs /usr/src/usr.bin
-progs cpio
-progs find
-progs minigzip
-progs sed
-ln minigzip gzip
-ln minigzip gunzip
-ln minigzip zcat
-
-srcdirs /usr/src/usr.sbin
-progs arp
-progs ppp
-progs sysinstall
-progs usbconfig
-
-libs -ll -ledit -lutil -lmd -lcrypt -lftpio -lz -lnetgraph
-libs -lodialog -lncurses -ldisk -lcam -lkiconv -lsbuf -lufs -ldevinfo
-libs -lgeom -lbsdxml -larchive -lcrypto -lbz2 -llzma -lusb -ljail
diff --git a/release/pc98/boot_crunch.conf b/release/pc98/boot_crunch.conf
deleted file mode 100644
index cce455388963..000000000000
--- a/release/pc98/boot_crunch.conf
+++ /dev/null
@@ -1,43 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH -Dlint
-
-srcdirs /usr/src/bin
-progs hostname
-progs pwd
-progs rm
-progs sh
-progs test
-ln sh -sh
-ln test [
-
-srcdirs /usr/src/sbin
-progs camcontrol
-progs dhclient
-progs fsck_ffs
-progs ifconfig
-progs mount_nfs
-progs newfs
-progs route
-progs rtsol
-progs tunefs
-ln fsck_ffs fsck_4.2bsd
-ln fsck_ffs fsck_ufs
-
-srcdirs /usr/src/usr.bin
-progs cpio
-progs find
-progs minigzip
-progs sed
-ln minigzip gzip
-ln minigzip gunzip
-ln minigzip zcat
-
-srcdirs /usr/src/usr.sbin
-progs arp
-progs ppp
-progs sysinstall
-
-libs -ll -ledit -lutil -lmd -lcrypt -lftpio -lz -lnetgraph
-libs -lodialog -lncurses -ldisk -lcam -lsbuf -lufs -lbsdxml
-libs -larchive -lcrypto -lbz2 -llzma -ljail
diff --git a/release/pc98/fixit-small_crunch.conf b/release/pc98/fixit-small_crunch.conf
deleted file mode 100644
index 111f1cdcda51..000000000000
--- a/release/pc98/fixit-small_crunch.conf
+++ /dev/null
@@ -1,43 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH
-
-srcdirs /usr/src/bin
-progs cat
-progs chmod
-progs cp
-progs dd
-progs df
-progs echo
-progs ed
-progs expr
-progs kill
-progs ln
-progs ls
-progs mkdir
-progs mv
-progs rm
-progs rmdir
-progs sleep
-progs sync
-
-srcdirs /usr/src/sbin
-progs bsdlabel
-progs clri
-progs dmesg
-progs fdisk
-progs mount
-progs mount_cd9660
-progs mount_msdosfs
-progs reboot
-progs swapon
-progs umount
-ln bsdlabel disklabel
-
-special fdisk srcdir /usr/src/sbin/fdisk_pc98
-
-srcdirs /usr/src/usr.sbin
-progs chown
-ln chown chgrp
-
-libs -lgeom -lbsdxml -lsbuf -lkiconv -lkvm -lncurses -lutil 
diff --git a/release/pc98/fixit_crunch.conf b/release/pc98/fixit_crunch.conf
deleted file mode 100644
index c03db1c18787..000000000000
--- a/release/pc98/fixit_crunch.conf
+++ /dev/null
@@ -1,47 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH
-
-srcdirs /usr/src/bin
-progs cat
-progs chmod
-progs cp
-progs dd
-progs df
-progs echo
-progs ed
-progs expr
-progs kill
-progs ln
-progs ls
-progs mkdir
-progs mv
-progs rm
-progs rmdir
-progs sleep
-progs sync
-
-srcdirs /usr/src/sbin
-progs bsdlabel
-progs clri
-progs dmesg
-progs fdisk
-progs mount
-progs mount_cd9660
-progs mount_msdosfs
-progs reboot
-progs swapon
-progs umount
-ln bsdlabel disklabel
-
-special fdisk srcdir /usr/src/sbin/fdisk_pc98
-
-srcdirs /usr/src/usr.bin
-progs ftp
-progs telnet
-
-srcdirs /usr/src/usr.sbin
-progs chown
-ln chown chgrp
-
-libs -ledit -lgeom -lbsdxml -lsbuf -lkiconv -lkvm -lncurses -lutil
diff --git a/release/powerpc/boot_crunch.conf b/release/powerpc/boot_crunch.conf
deleted file mode 100644
index 4a2f58029908..000000000000
--- a/release/powerpc/boot_crunch.conf
+++ /dev/null
@@ -1,49 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH -Dlint
-
-srcdirs /usr/src/bin
-progs hostname
-progs pwd
-progs rm
-progs sh
-progs test
-ln sh -sh
-ln test [
-
-srcdirs /usr/src/sbin
-progs camcontrol
-progs dhclient
-progs fsck_ffs
-progs geom
-progs ifconfig
-progs mount_msdosfs
-progs mount_nfs
-progs newfs
-progs newfs_msdos
-progs route
-progs rtsol
-progs tunefs
-ln fsck_ffs fsck_4.2bsd
-ln fsck_ffs fsck_ufs
-ln geom glabel
-ln geom gpart
-
-srcdirs /usr/src/usr.bin
-progs cpio
-progs find
-progs minigzip
-progs sed
-ln minigzip gzip
-ln minigzip gunzip
-ln minigzip zcat
-
-srcdirs /usr/src/usr.sbin
-progs arp
-progs ppp
-progs sysinstall
-progs usbconfig
-
-libs -ll -ledit -lutil -lmd -lcrypt -lftpio -lz -lnetgraph
-libs -lodialog -lncurses -ldisk -lcam -lkiconv -lsbuf -lufs
-libs -lgeom -lbsdxml -larchive -lcrypto -lbz2 -llzma -lusb -ljail
diff --git a/release/rc.local b/release/rc.local
index d583381446f5..1816f382612b 100755
--- a/release/rc.local
+++ b/release/rc.local
@@ -8,13 +8,37 @@
 : ${DIALOG_ITEM_HELP=4}
 : ${DIALOG_ESC=255}
 
-TERM=xterm; export TERM # XXX: serial consoles
+kbdcontrol -d >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+	# Syscons: use xterm
+	TERM=xterm
+else
+	# Serial or other console
+	echo
+	echo "Welcome to FreeBSD!"
+	echo
+	echo "Please choose the appropriate terminal type for your system."
+	echo "Common console types are:"
+	echo "   ansi     Standard ANSI terminal"
+	echo "   vt100    VT100 or compatible terminal"
+	echo "   xterm    xterm terminal emulator (or compatible)"
+	echo
+	echo -n "Console type [vt100]: "
+	read TERM
+	TERM=${TERM:-vt100}
+fi
+export TERM
 
 dialog --backtitle "FreeBSD Installer" --title "Welcome" --extra-button --extra-label "Shell" --ok-label "Install" --cancel-label "Live CD" --yesno "Welcome to FreeBSD! Would you like to begin an installation or use the live CD?" 0 0
 
 case $? in
 $DIALOG_OK)	# Install
-	BSDINSTALL_CONFIGCURRENT=yes; export BSDINSTALL_CONFIGCURRENT
+	# If not netbooting, have the installer configure the network
+	dlv=`/sbin/sysctl -n vfs.nfs.diskless_valid 2> /dev/null`
+	if [ ${dlv:=0} -eq 0 -a ! -f /etc/diskless ]; then
+		BSDINSTALL_CONFIGCURRENT=yes; export BSDINSTALL_CONFIGCURRENT
+	fi
+
 	trap true SIGINT	# Ignore cntrl-C here
 	bsdinstall
 	if [ $? -eq 0 ]; then
diff --git a/release/scripts/base-install.sh b/release/scripts/base-install.sh
deleted file mode 100755
index c87a38bf86a9..000000000000
--- a/release/scripts/base-install.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-
-echo "You are about to extract the base distribution into ${DESTDIR:-/} - are you SURE"
-echo -n "you want to do this over your installed system (y/n)? "
-read ans 
-if [ "$ans" = "y" ]; then
-	cat base.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-fi
diff --git a/release/scripts/catpages-install.sh b/release/scripts/catpages-install.sh
deleted file mode 100755
index 2e618ca76eeb..000000000000
--- a/release/scripts/catpages-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat catpages.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/catpages-make.sh b/release/scripts/catpages-make.sh
deleted file mode 100755
index 755057d40ccb..000000000000
--- a/release/scripts/catpages-make.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Move all the catpages out to their own dist, using the base dist as a
-# starting point.  This must precede the manpages dist script.
-if [ -d ${RD}/trees/base/usr/share/man ]; then
-	( cd ${RD}/trees/base/usr/share/man;
-	find cat* whatis | cpio -dumpl ${RD}/trees/catpages/usr/share/man > /dev/null 2>&1) &&
-	rm -rf ${RD}/trees/base/usr/share/man/cat*;
-fi
diff --git a/release/scripts/checkindex.pl b/release/scripts/checkindex.pl
deleted file mode 100644
index 44612dcbee5c..000000000000
--- a/release/scripts/checkindex.pl
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/perl
-# -----------------------------------------------------------------
-#  FreeBSD Release Checking Utility - Package Index Check
-#
-#  This program checks the packages/INDEX file to verify that
-#  the index is in the correct format and that every package
-#  needed by a release is included on the CD.
-#
-#  Copyright(c) 2000 BSDi
-#  Murray Stokely
-# -----------------------------------------------------------------
-# 08 Apr 2000
-#
-# $FreeBSD$
-#
-
-use Getopt::Long;
-
-#
-# Display the usage instructions
-#
-
-sub printHelp {
-    print<<end;
-usage : checkindex -s <sysinstall src dir> <INDEX>
-
-  This program checks the packages INDEX file to verify that the 
-index is correct and that every package needed by sysinstall is
-included in the index.
-
-  Options
-
-     -help                Display usage instructions
-     -s <src dir>         Specify the sysinstall source directory.  Use
-	                  this so to make sure every package referenced
-                          in the code is in your INDEX
-     -newindex            Generate a new index consisting of only those
-                          packages that actually exist in pkgdir/All
-     -depends <pkg>       Lists all packages in the index that depend
-                          on <pkg>.
-
-end
-}
-
-##
-## Attempts to find the value of a variable in C code by backtracking
-## up the source looking for a previous declaration.
-## 
-## This is a bit overkill for the purpose of this script,
-## stick with grepping for likely packages for now.
-
-sub findAssignment($$) {
-	    ## This code deals with the small (5%) number of matches
-	    ## in which package_add refers to a variable rather than
-	    ## a inline string, so we have to find the value of that
-	    ## variable so that we can push it onto the list
-#	    my($fileName,$code) = split(/:/,$match);
-#	    open(FILE,$fileName) || die "Could not open $fileName : $!\n";
-#	    my(@lines) = <FILE>;
-#	    my($cnt)  = 1;
-#	    my($lineMatch) = 0;
-#	    chomp(@lines);
-#	    foreach $line (@lines) {
-#		$lineMatch = $cnt if ($line eq $code);
-#		$cnt++;
-#	    }
-#	    $code =~ /package_add\((\S+)\)/;
-#	    my($varName) = $1;
-#	    print STDERR "$lineMatch of $fileName is wierd\n";
-#	    print STDERR "Trying to find '$varName'\n";
-#	    while ($cnt > 0) {
-#		$cnt--;
-#	    }
-
-
-}
-
-##
-## Returns a list of all the packages referenced in the sysinstall source
-## code
-##
-
-sub getPackages($) {
-    my($srcDir) = $_[0];
-    my(@matches) = `grep package_add $opt_s/*.c`;
-    my(@packages);
-    foreach $match (@matches) {
-	chomp $match;
-	next if ($match =~ m|$opt_s/package.c|);
-	if ($match =~ /package_add\(\"(\S+)\"\)/) {
-	    push(@packages,$1);
-	} elsif ($match =~ /package_add\(char/) {
-	    # function definition or prototype
-	    next;
-	} else {
-	    # package_add(variable or DEFINE)
-	    my(@varMatches) = `grep variable_set2 $opt_s/*.c`;
-	    chomp @varMatches;
-	    foreach $varMatch (@varMatches) {
-		if ($varMatch =~ /variable_set2\(\S+_PACKAGE,\s+\"(\S+)\"/) {
-		    push(@packages,$1);
-		}
-	    }
-	}
-    }
-    @packages;
-}
-
-
-&GetOptions("help","s=s","newindex","depends=s");
-if ($opt_help) {
-    &printHelp;
-} else{
-    my ($indexName) = $ARGV[0];
-    my ($mistakes) = 0;
-    my ($counter)  = 0;
-    print STDERR "Packages Referenced :\n---------------------\n";
-    open(INDEX,$indexName) || die "Could not open $indexName : $!";
-    @index = <INDEX>;
-    close(INDEX);
-
-    ## Check to ensure that every file in the index exists physically.
-    print STDERR "Check to ensure that every file in the index exists physically\n";
-    foreach $line (@index) {
-	chomp $line;
-	($file,$pathto,$prefix,$comment,$descr,$maint,$cats,$junk,$rdeps,$junk) = split(/\|/,$line,10);
-	$DEPENDS{$file} = $rdeps if (-e "All/$file.tgz");
-    }
-
-    if ($opt_newindex) {
-	foreach $pkg (keys %DEPENDS) {
-	    $new = quotemeta $pkg;
-	    @lines = grep(/^$new\|/,@index);
-	    chomp $lines;
-	    ($#lines == 0) || die "Multiple lines for '$pkg' in index!";
-	    printf "%s\n",$lines[0];
-	}
-    } elsif ($opt_depends) {
-	foreach $key (keys %DEPENDS) {
-	    foreach $dependency (split ' ',$DEPENDS{$key}) {
-		if ($opt_depends eq $dependency) {
-		    print "$opt_depends is needed by $key\n";
-		    $counter++;
-		}
-	    }
-	}
-	print "$opt_depends is not needed by any packages in the index!\n"
-	    unless ($counter);
-    } else {
-
-    ## Check to ensure that all the dependencies are there.
-    print "Check to make sure that every dependency of every file exists\n",
-  	"in the Index and physically.\n";
-    foreach $file (keys %DEPENDS) {
-#	print "Checking $file\n";
-	foreach $depend (split(' ',$DEPENDS{$file})) {
-	    unless (-e "All/$depend.tgz") {
-		# instead of a hash counter, make it a hash of arrays
-		# where the arrays are the files depended on.
-		push @{ $MISSING{$depend} }, $file;
-		$mistakes++;
-	    }
-	}
-    }
-
-    ## This makes sure that the index file contains everything
-    ## that sysinstall uses.
-    if ($opt_s) {
-	@packages = getPackages($opt_s);
-	foreach $pkg (@packages) {
-	    unless (grep(/^$pkg/,@index)) {
-		push @{ $MISSING{$pkg} }, "sysinstall";
-		$mistakes++;
-	    }
-	}
-    }
-
-
-    ## If there were mistakes, print out the missing packages.
-    if ($mistakes) {
-	print "--------------------------------------------------------\n",
-	      " Packages Missing : \n",
-              "--------------------------------------------------------\n";
-	foreach $pkg (keys %MISSING) {
-	    @files = @{ $MISSING{$pkg} };
-	    print "$pkg (@files)\n";
-	}
-    } else {
-	print "Everything looks good!\n";
-    }
-}
-}
diff --git a/release/scripts/chkINDEX b/release/scripts/chkINDEX
deleted file mode 100755
index d2b9e0eed8b7..000000000000
--- a/release/scripts/chkINDEX
+++ /dev/null
@@ -1,20 +0,0 @@
-#! /bin/sh
-#
-# $FreeBSD$
-
-usage()
-{
-	echo "$*" >&2
-	echo "This script must be run from the top level package directory" >&2
-	exit 1
-}
-
-[ $# -ge 1 ] && usage "usage: chkINDEX"
-[ -f INDEX ] || usage "INDEX file not found"
-
-echo Checking for missing files
-sed 's,^\([^|]*\)|.*,All/\1.tbz,' INDEX | xargs ls -l >/dev/null
-
-echo Checking for missing dependents
-awk -F'|' '{print $9}' INDEX | sed 's/ /\
-/g' | sort -u | sed -e /^$/d -e 's/\(.*\)/All\/\1.tbz/' | xargs ls -l >/dev/null
diff --git a/release/scripts/commerce-install.sh b/release/scripts/commerce-install.sh
deleted file mode 100755
index 9bf67a338b44..000000000000
--- a/release/scripts/commerce-install.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-echo "Extracting commerce tarball into ${DESTDIR}/usr/local"
-tar --unlink -xpzf commerce.tgz -C ${DESTDIR}/usr/local
-exit 0
diff --git a/release/scripts/dict-install.sh b/release/scripts/dict-install.sh
deleted file mode 100755
index 4f57aaa11ec9..000000000000
--- a/release/scripts/dict-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat dict.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/dict-make.sh b/release/scripts/dict-make.sh
deleted file mode 100755
index c0ca6a5880ed..000000000000
--- a/release/scripts/dict-make.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Move the dict stuff out to its own dist
-if [ -d ${RD}/trees/base/usr/share/dict ]; then
-	tar -cf - -C ${RD}/trees/base/usr/share/dict . |
-		tar -xpf - -C ${RD}/trees/dict/usr/share/dict &&
-	rm -rf ${RD}/trees/base/usr/share/dict;
-fi
-
-for i in birthtoken flowers; do
-	if [ -f ${RD}/trees/base/usr/share/misc/$i ]; then
-		mv ${RD}/trees/base/usr/share/misc/$i \
-			${RD}/trees/dict/usr/share/misc;
-	fi;
-done
diff --git a/release/scripts/doFS.sh b/release/scripts/doFS.sh
deleted file mode 100644
index ec64c95ead45..000000000000
--- a/release/scripts/doFS.sh
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-set -ex
-
-export BLOCKSIZE=512
-
-DISKLABEL=$1; shift
-MACHINE=${1:+"-m $1"}; shift
-FSIMG=$1; shift
-RD=$1 ; shift
-MNT=$1 ; shift
-FSSIZE=$1 ; shift
-FSPROTO=$1 ; shift
-FSINODE=$1 ; shift
-FSLABEL=$1 ; shift
-
-#
-# If we've been told to, compute the required file system size
-# and average inode size automatically.
-#
-if [ ${FSSIZE} -eq 0 -a ${FSLABEL} = "auto" ]; then
-	roundup() echo $((($1+$2-1)-($1+$2-1)%$2))
-	nf=$(find ${FSPROTO} |wc -l)
-	sk=$(du -sk ${FSPROTO} |cut -f1)
-	FSINODE=$(roundup $(($sk*1024/$nf)) ${FSINODE})
-	FSSIZE=$(roundup $(($sk*12/10)) 1024)
-fi
-
-dofs_vn () {
-	if [ "x$VNDEVICE" = "x" ] ; then
-		VNDEVICE=vn0
-	fi
-	u=`expr $VNDEVICE : 'vn\([0-9]*\)' || true`
-	VNDEVICE=vnn$u
-
-	rm -f /dev/*vnn*
-	mknod /dev/rvnn${u} c 43 `expr 65538 + $u '*' 8`
-	mknod /dev/rvnn${u}c c 43 `expr 2 + $u '*' 8`
-	mknod /dev/vnn${u} b 15 `expr 65538 + $u '*' 8`
-	mknod /dev/vnn${u}c b 15 `expr 2 + $u '*' 8`
-
-	umount /dev/${VNDEVICE} 2>/dev/null || true
-	umount ${MNT} 2>/dev/null || true
-	vnconfig -u /dev/r${VNDEVICE} 2>/dev/null || true
-
-	vnconfig -s labels -c /dev/r${VNDEVICE} ${FSIMG}
-
-	trap "umount ${MNT}; vnconfig -u /dev/r${VNDEVICE}; rm -f /dev/*vnn*" EXIT
-
-	disklabel -w ${BOOT} ${VNDEVICE} ${FSLABEL}
-	newfs -i ${FSINODE} -o space -m 0 /dev/r${VNDEVICE}c
-
-	mount /dev/${VNDEVICE}c ${MNT}
-}
-
-dofs_md () {
-	if [ "x${MDDEVICE}" != "x" ] ; then
-		umount /dev/${MDDEVICE} 2>/dev/null || true
-		umount ${MNT} 2>/dev/null || true
-		mdconfig -d -u ${MDDEVICE} 2>/dev/null || true
-	fi
-
-	MDDEVICE=`mdconfig -a -t vnode -f ${FSIMG}`
-	if [ ! -c /dev/${MDDEVICE} ] ; then
-		echo "No /dev/$MDDEVICE" 1>&2
-		exit 1
-	fi
-
-	trap "umount ${MNT}; mdconfig -d -u ${MDDEVICE}" EXIT
-
-	if [ "x${DISKLABEL}" != "x" ] ; then
-		${DISKLABEL} ${MACHINE} -w ${BOOT} ${MDDEVICE} ${FSLABEL}
-	fi
-	newfs -O1 -b 4096 -f 512 -i ${FSINODE} -o space -m 0 /dev/${MDDEVICE}
-	mount /dev/${MDDEVICE} ${MNT}
-}
-
-rm -f ${FSIMG}
-dd of=${FSIMG} if=/dev/zero count=${FSSIZE} bs=1k 2>/dev/null
-
-#
-# We don't have any bootblocks on ia64. Note that -B implies -r,
-# so we have to specifically specify -r when we don't have -B.
-# bsdlabel fails otherwise.
-#
-case `uname -r` in
-4.*)
-	if [ -f "${RD}/trees/base/boot/boot1" ]; then
-		BOOT="-B -b ${RD}/trees/base/boot/boot1"
-		if [ -f "${RD}/trees/base/boot/boot2" ]; then
-			BOOT="${BOOT} -s ${RD}/trees/base/boot/boot2"
-		fi
-	else
-		BOOT="-r"
-	fi
-	dofs_vn
-	;;
-*)
-	if [ -f "${RD}/trees/base/boot/boot" ]; then
-		BOOT="-B -b ${RD}/trees/base/boot/boot"
-	else
-		BOOT="-r"
-	fi
-	dofs_md
-	;;
-esac
-
-if [ -d ${FSPROTO} ]; then
-	(set -e && cd ${FSPROTO} && find . -print | cpio -dump ${MNT})
-else
-	cp -p ${FSPROTO} ${MNT}
-fi
-
-df -ki ${MNT}
-
-set `df -ki ${MNT} | tail -1`
-
-echo "*** File system is ${FSSIZE} K, $4 left"
-echo "***     ${FSINODE} bytes/inode, $7 left"
diff --git a/release/scripts/doc-install.sh b/release/scripts/doc-install.sh
deleted file mode 100755
index 5609720ada37..000000000000
--- a/release/scripts/doc-install.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-echo "You are about to extract the doc distribution into ${DESTDIR:-/} - are you SURE"
-echo -n "you want to do this over your installed system (y/n)? "
-read ans
-if [ "$ans" = "y" ]; then
-	cat doc.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-fi
diff --git a/release/scripts/doc-make.sh b/release/scripts/doc-make.sh
deleted file mode 100755
index 033554007abe..000000000000
--- a/release/scripts/doc-make.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Create the doc dist.
-if [ -d ${RD}/trees/base/usr/share/doc ]; then
-	( cd ${RD}/trees/base/usr/share/doc;
-	find . | cpio -dumpl ${RD}/trees/doc/usr/share/doc ) &&
-	rm -rf ${RD}/trees/base/usr/share/doc
-fi
diff --git a/release/scripts/games-install.sh b/release/scripts/games-install.sh
deleted file mode 100755
index 6c940776b55e..000000000000
--- a/release/scripts/games-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat games.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/info-install.sh b/release/scripts/info-install.sh
deleted file mode 100755
index 30c12bf23850..000000000000
--- a/release/scripts/info-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat info.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/info-make.sh b/release/scripts/info-make.sh
deleted file mode 100755
index 12fd29d93873..000000000000
--- a/release/scripts/info-make.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Move the info files out to their own dist
-if [ -d ${RD}/trees/base/usr/share/info ]; then
-	tar -cf - -C ${RD}/trees/base/usr/share/info . |
-		tar -xpf - -C ${RD}/trees/info/usr/share/info &&
-	rm -rf ${RD}/trees/base/usr/share/info;
-fi
diff --git a/release/scripts/info.sh b/release/scripts/info.sh
deleted file mode 100644
index e25ceb73dbe9..000000000000
--- a/release/scripts/info.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# $ANA: info.sh,v 1.3 1996/06/04 16:25:30 wollman Exp $
-# $FreeBSD$
-#
-
-ls $1.[a-z][a-z] | wc | awk '{ print "Pieces = ",$1 }'
-for FILE in $1.[a-z][a-z]; do
-       PIECE=`echo $FILE | cut -d . -f 2`
-       echo -n "cksum.$PIECE = "
-       cksum $FILE | awk ' { print $1,$2 } '
-done
diff --git a/release/scripts/kernels-install.sh b/release/scripts/kernels-install.sh
deleted file mode 100644
index 0bf9619b9587..000000000000
--- a/release/scripts/kernels-install.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-if [ $# -lt 1 ]; then
-	echo "You must specify which kernel to extract."
-	exit 1
-fi
-
-CONFIG=$1
-BOOT=${DESTDIR}/boot
-KERNEL=$BOOT/$CONFIG
-
-if [ -d $KERNEL ]; then
-	echo "You are about to extract the $CONFIG kernel distribution into $KERNEL - are you SURE"
-	echo -n "you want to do this over your installed system (y/n)? "
-	read ans 
-else
-	# new installation; do not prompt
-	ans=y
-fi
-if [ "$ans" = "y" ]; then
-	if [ -d $KERNEL ]; then
-		sav=$KERNEL.sav
-		if [ -d $sav ]; then
-			# XXX remove stuff w/o a prompt
-			echo "Removing existing $sav"
-			rm -rf $sav
-		fi
-		echo "Saving existing $KERNEL as $sav"
-		mv $KERNEL $sav
-	fi
-	# translate per Makefile:doTARBALL XXX are we sure to have tr?
-	tn=`echo ${CONFIG} | tr 'A-Z' 'a-z'`
-	cat $tn.?? | tar --unlink -xpzf - -C $BOOT
-else
-	echo "Installation of $CONFIG kernel distribution not done." 
-fi
diff --git a/release/scripts/lib32-install.sh b/release/scripts/lib32-install.sh
deleted file mode 100644
index c6f732d0138b..000000000000
--- a/release/scripts/lib32-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat lib32.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/manpages-install.sh b/release/scripts/manpages-install.sh
deleted file mode 100755
index 35eb4b7a17ca..000000000000
--- a/release/scripts/manpages-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat manpages.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/manpages-make.sh b/release/scripts/manpages-make.sh
deleted file mode 100755
index bb1326f3d506..000000000000
--- a/release/scripts/manpages-make.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Move all the manpages out to their own dist, using the base dist as a
-# starting point.
-if [ -d ${RD}/trees/base/usr/share/man ]; then
-	( cd ${RD}/trees/base/usr/share/man;
-	find . | cpio -dumpl ${RD}/trees/manpages/usr/share/man > /dev/null 2>&1) &&
-	rm -rf ${RD}/trees/base/usr/share/man;
-fi
diff --git a/release/scripts/mkpkghier b/release/scripts/mkpkghier
deleted file mode 100755
index c14b62868018..000000000000
--- a/release/scripts/mkpkghier
+++ /dev/null
@@ -1,33 +0,0 @@
-#! /bin/sh
-#
-# $FreeBSD$
-
-usage()
-{
-	echo "$*" >&2
-	echo "This script must be run from the top level package directory" >&2
-	exit 1
-}
-
-[ $# -ge 1 ] && usage "usage: mkpkghier"
-[ -f INDEX ] || usage "INDEX file not found"
-[ -d All ] || usage "All directory not found"
-
-old=$(ls | egrep -v "^INDEX$|^All$|^Latest$")
-
-[ -n "$old" ] && { echo Removing old hierarchy; rm -fr $old; }
-
-echo Creating new hierarchy
-awk -F'|' '{ print $1 " " $7; }' INDEX | while read pkg dirs
-do
-	[ -f All/$pkg.tbz ] || {
-		echo "All/$pkg.tbz: Non-existent" >&2
-		continue
-	}
-	for dir in $dirs
-	do
-		mkdir -p $dir
-		[ -e $dir/$pkg.tbz ] && continue
-		ln -s ../All/$pkg.tbz $dir/$pkg.tbz
-	done
-done
diff --git a/release/scripts/package-split.py b/release/scripts/package-split.py
deleted file mode 100644
index 727262b05386..000000000000
--- a/release/scripts/package-split.py
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/local/bin/python
-#
-# This script generates a master INDEX file for the CD images built by the
-# FreeBSD release engineers.  Each disc is given a list of desired packages.
-# Dependencies of these packages are placed on either the same disc or an
-# earlier disc.  The resulting master INDEX file is then written out.
-#
-# Usage: package-split.py <INDEX> <master INDEX>
-#
-# $FreeBSD$
-
-import os
-import sys
-
-try:
-    arch = os.environ["PKG_ARCH"]
-except:
-    arch = os.uname()[4]
-print "Using arch %s" % (arch)
-
-if 'PKG_VERBOSE' in os.environ:
-    verbose = 1
-else:
-    verbose = 0
-
-if 'PKG_DVD' in os.environ:
-    doing_dvd = 1
-else:
-    doing_dvd = 0
-
-# List of packages for disc1.
-def disc1_packages():
-    pkgs = ['misc/freebsd-doc-bn', 
-	    'misc/freebsd-doc-da',
-	    'misc/freebsd-doc-de',
-	    'misc/freebsd-doc-el',
-	    'misc/freebsd-doc-en',
-	    'misc/freebsd-doc-es',
-	    'misc/freebsd-doc-fr',
-	    'misc/freebsd-doc-hu',
-	    'misc/freebsd-doc-it',
-	    'misc/freebsd-doc-ja',
-	    'misc/freebsd-doc-mn',
-	    'misc/freebsd-doc-nl',
-	    'misc/freebsd-doc-pl',
-	    'misc/freebsd-doc-pt',
-	    'misc/freebsd-doc-ru',
-	    'misc/freebsd-doc-sr',
-	    'misc/freebsd-doc-tr',
-	    'misc/freebsd-doc-zh_cn',
-	    'misc/freebsd-doc-zh_tw']
-
-    if doing_dvd:
-	pkgs.extend(['archivers/unzip',
-	    'astro/xearth',
-	    'devel/gmake',
-	    'devel/imake',
-	    'editors/emacs',
-	    'editors/vim-lite',
-	    'emulators/linux_base-f10',
-	    'emulators/mtools',
-	    'graphics/png',
-	    'graphics/xv',
-	    'irc/xchat',
-	    'lang/perl5.8',
-	    'mail/alpine',
-	    'mail/exim',
-	    'mail/fetchmail',
-	    'mail/mutt',
-	    'mail/popd',
-	    'mail/postfix',
-	    'mail/xfmail',
-	    'net/cvsup-without-gui',
-	    'net/rsync',
-	    'net/samba3',
-	    'news/slrn',
-	    'news/tin',
-	    'ports-mgmt/p5-FreeBSD-Portindex',
-	    'ports-mgmt/portaudit',
-	    'ports-mgmt/portmaster',
-	    'ports-mgmt/portupgrade',
-	    'print/a2ps-letter',
-	    'print/apsfilter',
-	    'print/ghostscript7-nox11',
-	    'print/psutils-letter',
-	    'print/gv',
-	    'shells/bash',
-	    'shells/pdksh',
-	    'shells/zsh',
-	    'security/sudo',
-	    'sysutils/screen',
-	    'www/links',
-	    'www/lynx',
-	    'x11/gnome2',
-	    'x11/kde4',
-	    'x11/rxvt',
-	    'x11/xorg',
-	    'x11-wm/afterstep',
-	    'x11-wm/fvwm2',
-	    'x11-wm/windowmaker'])
-    return pkgs
-
-# The list of desired packages
-def desired_packages():
-    disc1 = disc1_packages()
-    return [disc1]
-
-# Suck the entire INDEX file into a two different dictionaries.  The first
-# dictionary maps port names (origins) to package names.  The second
-# dictionary maps a package name to a list of its dependent packages.
-PACKAGE_COL=0
-ORIGIN_COL=1
-DEPENDS_COL=8
-
-def load_index(index):
-    deps = {}
-    pkgs = {}
-    line_num = 1
-    for line in index:
-        fields = line.split('|')
-        name = fields[PACKAGE_COL]
-        if name in deps:
-            sys.stderr.write('%d: Duplicate package %s\n' % (line_num, name))
-            sys.exit(1)
-        origin = fields[ORIGIN_COL].replace('/usr/ports/', '', 1)
-        if origin in pkgs:
-            sys.stderr.write('%d: Duplicate port %s\n' % (line_num, origin))
-            sys.exit(1)
-        deps[name] = fields[DEPENDS_COL].split()
-        pkgs[origin] = name
-        line_num = line_num + 1
-    return (deps, pkgs)
-
-# Layout the packages on the various CD images.  Here's how it works.  We walk
-# each disc in the list of discs.  Within each disc we walk the list of ports.
-# For each port, we add the package name to a dictionary with the value being
-# the current disc number.  We also add all of the dependent packages.  If
-# a package is already in the dictionary when we go to add it, we just leave
-# the dictionary as it is.  This means that each package ends up on the first
-# disc that either lists it or contains it as a dependency.
-def layout_discs(discs, pkgs, deps):
-    disc_num = 1
-    layout = {}
-    for disc in discs:
-        for port in disc:
-            if port not in pkgs:
-                sys.stderr.write('Disc %d: Unable to find package for %s\n' %
-                                 (disc_num, port))
-                continue
-            pkg = pkgs[port]
-            pkg_list = [pkg] + deps[pkg]
-            for pkg in pkg_list:
-                if pkg not in layout:
-                    if verbose:
-                        print "--> Adding %s to Disc %d" % (pkg, disc_num)
-                    layout[pkg] = disc_num
-        disc_num = disc_num + 1
-    return layout
-
-# Generate a master INDEX file based on the generated layout.  The way this
-# works is that for each INDEX line, we check to see if the package is in the
-# layout.  If it is, we put that INDEX line into the master INDEX and append
-# a new field with the disc number to the line.
-def generate_index(index, layout, master_index):
-    for line in index:
-        pkg = line.split('|')[PACKAGE_COL]
-        if pkg in layout:
-            new_line = '%s|%d\n' % (line.splitlines()[0], layout[pkg])
-            master_index.write(new_line)
-
-# Verify the command line arguments
-if len(sys.argv) != 3:
-    sys.stderr.write('Invalid number of arguments\n')
-    sys.stderr.write('Usage: package-split.py <source INDEX> <master INDEX>\n')
-    sys.exit(1)
-
-print "Loading %s..." % (sys.argv[1])
-index = file(sys.argv[1])
-(deps, pkgs) = load_index(index)
-discs = desired_packages()
-layout = layout_discs(discs, pkgs, deps)
-index.seek(0)
-print "Generating %s..." % (sys.argv[2])
-master_index = file(sys.argv[2], 'w')
-generate_index(index, layout, master_index)
-index.close()
-master_index.close()
diff --git a/release/scripts/package-trees.sh b/release/scripts/package-trees.sh
deleted file mode 100644
index 9a81dac86d17..000000000000
--- a/release/scripts/package-trees.sh
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/bin/sh
-#
-# This script generates the disk layout for the CD images built by the FreeBSD
-# release engineers as dictated by a specified master INDEX file.  Each disc
-# contains the master INDEX, it's assigned list of packages, and the
-# appropriate tree of category symlinks.
-#
-# Usage: package-trees.sh <copy method> <INDEX> <package tree> <destination>
-#
-# $FreeBSD$
-
-# Verify the command line
-if [ $# -ne 4 ]; then
-    echo "Invalid number of arguments"
-    echo "Usage: package-trees.sh <copy method> <INDEX> <tree> <destination>"
-    exit 1
-fi
-
-COPY=$1 ; shift
-INDEX=$1 ; shift
-TREE=$1 ; shift
-DESTDIR=$1 ; shift
-
-# First, determine the highest disc number.
-high_disc=`cut -d '|' -f 14 ${INDEX} | sort -n | tail -1`
-echo "Generating trees for ${high_disc} discs"
-
-# Second, initialize the trees for each disc
-for disc in `jot $high_disc`; do
-    rm -rf ${DESTDIR}/disc${disc}/packages
-    mkdir -p ${DESTDIR}/disc${disc}/packages/All
-    cp ${INDEX} ${DESTDIR}/disc${disc}/packages/INDEX
-done
-
-# Third, run through the INDEX copying each package to its appropriate CD and
-# making the appropriate category symlinks
-while read line; do
-    disc=`echo $line | cut -d '|' -f 14`
-    package=`echo $line | cut -d '|' -f 1`
-    categories=`echo $line | cut -d '|' -f 7`
-    discdir=${DESTDIR}/disc${disc}
-    if [ -n "$PKG_VERBOSE" ]; then
-	echo "--> Copying $package to Disc $disc"
-    fi
-    ${COPY} ${TREE}/All/${package}.tbz ${discdir}/packages/All
-    for cat in ${categories}; do
-        catdir=${discdir}/packages/${cat}
-	mkdir -p ${catdir}
-	ln -s ../All/${package}.tbz ${catdir}
-    done
-done < ${INDEX}
-
-# Fourth, output du info for the relative size of the trees.
-discs=""
-for disc in `jot $high_disc`; do
-    discs="${discs} disc${disc}"
-done
-(cd ${DESTDIR}; du -sh ${discs})
diff --git a/release/scripts/ports-install.sh b/release/scripts/ports-install.sh
deleted file mode 100755
index 0933e7163264..000000000000
--- a/release/scripts/ports-install.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-echo "Extracting ports tarball into ${DESTDIR}/usr"
-tar --unlink -xpzf ports.tgz -C ${DESTDIR}/usr
-exit 0
diff --git a/release/scripts/proflibs-install.sh b/release/scripts/proflibs-install.sh
deleted file mode 100755
index 8e35ae3fd3a2..000000000000
--- a/release/scripts/proflibs-install.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-cat proflibs.?? | tar --unlink -xpzf - -C ${DESTDIR:-/}
-exit 0
diff --git a/release/scripts/proflibs-make.sh b/release/scripts/proflibs-make.sh
deleted file mode 100755
index 5ff447ac9068..000000000000
--- a/release/scripts/proflibs-make.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Move the profiled libraries out to their own dist
-for i in ${RD}/trees/base/usr/lib/*_p.a; do
-	mv $i ${RD}/trees/proflibs/usr/lib
-done
diff --git a/release/scripts/split-file.sh b/release/scripts/split-file.sh
deleted file mode 100755
index 40b8eaa044e4..000000000000
--- a/release/scripts/split-file.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-# Bail if things fail and be verbose about what we are doing
-set -ex
-
-# Arguments are as follows: file destdir chunksize description
-FILE=$1; shift
-DEST=$1; shift
-CHUNK_SIZE=$1; shift
-DESCR=$1; shift
-
-# Make sure we can read the file.
-[ -r ${FILE} ]
-
-# Create clean working area to stick file chunks and list in
-rm -rf ${DEST} || true
-mkdir -p ${DEST}
-
-# Split the file into pieces
-prefix=`basename $FILE`
-dd if=${FILE} bs=16k iseek=1 | split -b ${CHUNK_SIZE}k - ${DEST}/${prefix}.
-
-# Create a special file for the first 16k that gets stuck on the boot
-# floppy
-files=`ls ${DEST}/${prefix}.*`
-first=`echo "${files}" | head -1`
-bootchunk="${DEST}/${prefix}.boot"
-dd if=${FILE} of=${bootchunk} bs=16k count=1
-
-# Create the split index file
-echo `basename ${bootchunk}` "\"Boot floppy\"" > ${DEST}/${prefix}.split
-i=1
-for file in ${files}; do
-	echo `basename ${file}` "\"${DESCR} floppy ${i}\"" >> ${DEST}/${prefix}.split
-	i=$(($i + 1))
-done
diff --git a/release/scripts/src-install.sh b/release/scripts/src-install.sh
deleted file mode 100755
index 90583c24d592..000000000000
--- a/release/scripts/src-install.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-if [ $# -lt 1 ]; then
-	echo "You must specify which components of src to extract"
-	echo "possible subcomponents are:"
-	echo
-	echo "base bin cddl contrib crypto etc games gnu include krb5"
-	echo "lib libexec release rescue sbin secure share sys tools ubin"
-	echo "usbin"
-	echo
-	echo "You may also specify all to extract all subcomponents."
-	exit 1
-fi
-
-if [ "$1" = "all" ]; then
-	dists="base bin cddl contrib crypto etc games gnu include krb5 lib libexec release rescue sbin secure share sys tools ubin usbin"
-else
-	dists="$*"
-fi
-
-echo "Extracting sources into ${DESTDIR}/usr/src..."
-for i in $dists; do
-	echo "  Extracting source component: $i"
-	cat s${i}.?? | tar --unlink -xpzf - -C ${DESTDIR}/usr/src
-done
-echo "Done extracting sources."
-exit 0
diff --git a/release/scripts/tar.sh b/release/scripts/tar.sh
deleted file mode 100644
index e0933f29d778..000000000000
--- a/release/scripts/tar.sh
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/bin/sh -p
-#
-# Simple replacement for tar(1), using cpio(1).
-#
-# Copyright (c) 1996 Joerg Wunsch
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-# IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
-# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# $FreeBSD$
-#
-
-#
-# For use on the fixit floppy.  External programs required:
-# cpio(1), find(1), test(1)
-#
-
-
-archive=${TAPE:-/dev/rsa0}
-blocksize="20"
-device=""
-mode="none"
-verbose=""
-
-usage()
-{
-	echo "usage: tar -{c|t|x} [-v] [-b blocksize] [-f archive] [files...]" 1>&2
-	exit 64		# EX_USAGE
-}
-
-#
-# Prepend a hyphen to the first arg if necessary, so the traditional form
-# ``tar xvf /dev/foobar'' will work, too.  More kludgy legacy forms are not
-# supported however.
-#
-
-if [ $# -lt 1 ] ; then
-	usage
-fi
-
-case "$1" in
-	-*)	break
-		;;
-	*)	tmp="$1"
-		shift
-		set -- -$tmp "$@"
-		;;
-esac
-
-while getopts "ctxvb:f:" option
-do
-	case $option in
-		[ctx])
-			if [ $mode = "none" ] ; then
-				mode=$option
-			else
-				usage
-			fi
-			;;
-		v)
-			verbose="-v"
-			;;
-		b)
-			blocksize="${OPTARG}"
-			;;
-		f)
-			archive="${OPTARG## }"
-			;;
-		*)
-			usage
-			;;
-	esac
-done
-
-shift $(($OPTIND - 1))
-
-if [ "X${archive}" != "X-" ] ; then
-	device="-F ${archive}"
-# else: use stdin or stdout, which is the default for cpio
-fi
-
-case $mode in
-	none)
-		usage
-		;;
-	t)
-		exec cpio -it $verbose $device --block-size="$blocksize" "$@"
-		;;
-	x)
-		exec cpio -idmu $verbose $device --block-size="$blocksize" "$@"
-		;;
-	c)
-		if [ $# -eq 0 ] ; then
-			# use current dir -- slightly bogus
-			set -- "."
-		fi
-		find "$@" -print |\
-		    cpio -o -H ustar $verbose $device --block-size="$blocksize"
-		exit $?
-		;;
-esac
diff --git a/release/scripts/xperimnt-install.sh b/release/scripts/xperimnt-install.sh
deleted file mode 100755
index 394265892f2c..000000000000
--- a/release/scripts/xperimnt-install.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if [ "`id -u`" != "0" ]; then
-	echo "Sorry, this must be done as root."
-	exit 1
-fi
-echo "Extracting xperimnt tarball into ${DESTDIR}/usr/local"
-tar --unlink -xpzf xperimnt.tgz -C ${DESTDIR}/usr/local
-exit 0
diff --git a/release/sparc64/boot_crunch.conf b/release/sparc64/boot_crunch.conf
deleted file mode 100644
index c392fe9fedf6..000000000000
--- a/release/sparc64/boot_crunch.conf
+++ /dev/null
@@ -1,47 +0,0 @@
-# $FreeBSD$
-
-buildopts -DRELEASE_CRUNCH -Dlint
-
-srcdirs /usr/src/bin
-progs hostname
-progs pwd
-progs rm
-progs sh
-progs test
-ln sh -sh
-ln test [
-
-srcdirs /usr/src/sbin
-progs camcontrol
-progs dhclient
-progs fsck_ffs
-progs geom
-progs ifconfig
-progs mount_nfs
-progs newfs
-progs route
-progs rtsol
-progs tunefs
-ln fsck_ffs fsck_4.2bsd
-ln fsck_ffs fsck_ufs
-ln geom glabel
-ln geom gpart
-
-srcdirs /usr/src/usr.bin
-progs cpio
-progs find
-progs minigzip
-progs sed
-ln minigzip gzip
-ln minigzip gunzip
-ln minigzip zcat
-
-srcdirs /usr/src/usr.sbin
-progs arp
-progs ppp
-progs sysinstall
-progs usbconfig
-
-libs -ll -ledit -lutil -lmd -lcrypt -lftpio -lz -lnetgraph
-libs -lodialog -lncurses -ldisk -lcam -lsbuf -lufs -lgeom -lbsdxml
-libs -larchive -lcrypto -lbz2 -llzma -lusb -ljail
diff --git a/release/svnbranch.awk b/release/svnbranch.awk
deleted file mode 100644
index 0fc86200c043..000000000000
--- a/release/svnbranch.awk
+++ /dev/null
@@ -1,28 +0,0 @@
-# $FreeBSD$
-
-BEGIN {
-	FS = "_"
-}
-
-/RELENG_.*_RELEASE/ {
-	if (NF == 5) {
-		printf "release/%s.%s.%s", $2, $3, $4
-		exit
-	}
-}
-
-/RELENG_.*/ {
-	if (NF == 3) {
-		printf "releng/%s.%s", $2, $3
-		exit
-	}
-
-	if (NF == 2) {
-		printf "stable/%s", $2
-		exit
-	}
-}
-
-// {
-	printf "unknown_branch"
-}
diff --git a/sbin/atacontrol/atacontrol.8 b/sbin/atacontrol/atacontrol.8
index 6c446872293d..8f4a4fc94a61 100644
--- a/sbin/atacontrol/atacontrol.8
+++ b/sbin/atacontrol/atacontrol.8
@@ -25,12 +25,19 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 21, 2009
+.Dd October 9, 2011
 .Dt ATACONTROL 8
 .Os
 .Sh NAME
 .Nm atacontrol
 .Nd ATA device driver control program
+.Pp
+This utility was
+.Em deprecated
+in
+.Fx 9.0 .
+See
+.Sx NOTES .
 .Sh SYNOPSIS
 .Nm
 .Aq Ar command
@@ -361,11 +368,17 @@ or syslog logging on it as the disk will be worn out spinning down and
 up all the time.
 .Sh SEE ALSO
 .Xr ata 4
+.Xr cam 4
+.Xr camcontrol 8
 .Sh HISTORY
 The
 .Nm
 utility first appeared in
 .Fx 4.6 .
+.Pp
+.Nm
+was deprecated in
+.Fx 9.0 .
 .Sh AUTHORS
 .An -nosplit
 The
@@ -377,3 +390,16 @@ utility was written by
 This manual page was written by
 .An S\(/oren Schmidt
 .Aq sos@FreeBSD.org .
+.Sh NOTES
+The
+.Nm
+utility was deprecated in
+.Fx 9.0 .
+When
+.Bd -ragged -offset indent
+.Cd "options ATA_CAM"
+.Ed
+.Pp
+is compiled into the kernel, then
+.Xr camcontrol 8
+must be used instead.
diff --git a/sbin/atacontrol/atacontrol.c b/sbin/atacontrol/atacontrol.c
index 7b67821c8eac..4b9b74f0ed15 100644
--- a/sbin/atacontrol/atacontrol.c
+++ b/sbin/atacontrol/atacontrol.c
@@ -378,6 +378,11 @@ main(int argc, char **argv)
 {
 	int fd, mode, channel, array;
 
+	if (feature_present("ata_cam")) {
+		errx(1, "\nATA_CAM option is enabled in kernel.\n"
+		    "Please use camcontrol instead.");
+	}
+
 	if (argc < 2)
 		usage();
 
diff --git a/sbin/atm/atmconfig/Makefile b/sbin/atm/atmconfig/Makefile
index cbf5a9a87d4c..0d64fea0f4f7 100644
--- a/sbin/atm/atmconfig/Makefile
+++ b/sbin/atm/atmconfig/Makefile
@@ -5,6 +5,8 @@
 #
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 PROG=	atmconfig
 .ifndef RESCUE
 SRCS=	${.OBJDIR}/oid.h
@@ -21,6 +23,9 @@ CFLAGS+= -I${.OBJDIR}
 .ifndef RESCUE
 DPADD=	${LIBBSNMP}
 LDADD=	-lbsnmp
+. if ${MK_DYNAMICROOT} == "no" && ${MK_OPENSSL} != "no"
+LDADD+= -lcrypto
+. endif
 .endif
 
 .ifndef RESCUE
diff --git a/sbin/camcontrol/camcontrol.c b/sbin/camcontrol/camcontrol.c
index c408b811f33a..9ec83f5f2bd6 100644
--- a/sbin/camcontrol/camcontrol.c
+++ b/sbin/camcontrol/camcontrol.c
@@ -1907,7 +1907,9 @@ readdefects(struct cam_device *device, int argc, char **argv,
 		int error_code, sense_key, asc, ascq;
 
 		sense = &ccb->csio.sense_data;
-		scsi_extract_sense(sense, &error_code, &sense_key, &asc, &ascq);
+		scsi_extract_sense_len(sense, ccb->csio.sense_len -
+		    ccb->csio.sense_resid, &error_code, &sense_key, &asc,
+		    &ascq, /*show_errors*/ 1);
 
 		/*
 		 * According to the SCSI spec, if the disk doesn't support
@@ -3798,8 +3800,9 @@ scsiformat(struct cam_device *device, int argc, char **argv,
 			int error_code, sense_key, asc, ascq;
 
 			sense = &ccb->csio.sense_data;
-			scsi_extract_sense(sense, &error_code, &sense_key,
-					   &asc, &ascq);
+			scsi_extract_sense_len(sense, ccb->csio.sense_len -
+			    ccb->csio.sense_resid, &error_code, &sense_key,
+			    &asc, &ascq, /*show_errors*/ 1);
 
 			/*
 			 * According to the SCSI-2 and SCSI-3 specs, a
@@ -3810,15 +3813,15 @@ scsiformat(struct cam_device *device, int argc, char **argv,
 			 */
 			if ((sense_key == SSD_KEY_NOT_READY)
 			 && (asc == 0x04) && (ascq == 0x04)) {
-				if ((sense->extra_len >= 10)
-				 && ((sense->sense_key_spec[0] &
-				      SSD_SCS_VALID) != 0)
+				uint8_t sks[3];
+
+				if ((scsi_get_sks(sense, ccb->csio.sense_len -
+				     ccb->csio.sense_resid, sks) == 0)
 				 && (quiet == 0)) {
 					int val;
 					u_int64_t percentage;
 
-					val = scsi_2btoul(
-						&sense->sense_key_spec[1]);
+					val = scsi_2btoul(&sks[1]);
 					percentage = 10000 * val;
 
 					fprintf(stdout,
diff --git a/sbin/devd/devd.cc b/sbin/devd/devd.cc
index 87d0580e26d4..fd3b6e396785 100644
--- a/sbin/devd/devd.cc
+++ b/sbin/devd/devd.cc
@@ -251,7 +251,14 @@ match::match(config &c, const char *var, const char *re)
 	: _var(var)
 {
 	_re = "^";
-	_re.append(c.expand_string(string(re)));
+	if (!c.expand_string(string(re)).empty() &&
+	    c.expand_string(string(re)).at(0) == '!') {
+		_re.append(c.expand_string(string(re)).substr(1));
+		_inv = 1;
+	} else {
+		_re.append(c.expand_string(string(re)));
+		_inv = 0;
+	}
 	_re.append("$");
 	regcomp(&_regex, _re.c_str(), REG_EXTENDED | REG_NOSUB | REG_ICASE);
 }
@@ -268,10 +275,13 @@ match::do_match(config &c)
 	bool retval;
 
 	if (Dflag)
-		fprintf(stderr, "Testing %s=%s against %s\n", _var.c_str(),
-		    value.c_str(), _re.c_str());
+		fprintf(stderr, "Testing %s=%s against %s, invert=%d\n",
+		    _var.c_str(), value.c_str(), _re.c_str(), _inv);
 
 	retval = (regexec(&_regex, value.c_str(), 0, NULL, 0) == 0);
+	if (_inv == 1)
+		retval = (retval == 0) ? 1 : 0;
+
 	return retval;
 }
 
diff --git a/sbin/devd/devd.conf.5 b/sbin/devd/devd.conf.5
index e4dea72fd317..09e902bf188e 100644
--- a/sbin/devd/devd.conf.5
+++ b/sbin/devd/devd.conf.5
@@ -41,7 +41,7 @@
 .\" ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
 .\" SOFTWARE.
 .\"
-.Dd March 8, 2009
+.Dd October 25, 2011
 .Dt DEVD.CONF 5
 .Os
 .Sh NAME
@@ -121,6 +121,10 @@ Creates a regular expression and assigns it to the variable
 .Ar regexp-name .
 The variable is available throughout the rest of
 the configuration file.
+If the string begins with
+.Ql \&! ,
+it matches if the regular expression formed by the rest of the string
+does not match.
 All regular expressions have an implicit
 .Ql ^$
 around them.
diff --git a/sbin/devd/devd.hh b/sbin/devd/devd.hh
index b3c98f3ebe0c..d70d1787e452 100644
--- a/sbin/devd/devd.hh
+++ b/sbin/devd/devd.hh
@@ -92,6 +92,7 @@ public:
 private:
 	std::string _var;
 	std::string _re;
+	bool _inv;
 	regex_t _regex;
 };
 
diff --git a/sbin/dhclient/Makefile b/sbin/dhclient/Makefile
index 434335b5a070..74d1c4d9dea9 100644
--- a/sbin/dhclient/Makefile
+++ b/sbin/dhclient/Makefile
@@ -39,6 +39,8 @@ PROG=	dhclient
 SCRIPTS=dhclient-script
 MAN=	dhclient.8 dhclient.conf.5 dhclient.leases.5 dhcp-options.5 \
 	dhclient-script.8
+DPADD=	${LIBUTIL}
+LDADD=	-lutil
 
 WARNS?=	2
 
diff --git a/sbin/dhclient/dhclient.8 b/sbin/dhclient/dhclient.8
index 58f5ddd3ea05..d2b8f437f7d4 100644
--- a/sbin/dhclient/dhclient.8
+++ b/sbin/dhclient/dhclient.8
@@ -38,7 +38,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 13, 2007
+.Dd October 13, 2011
 .Dt DHCLIENT 8
 .Os
 .Sh NAME
@@ -49,6 +49,7 @@
 .Op Fl bdqu
 .Op Fl c Ar file
 .Op Fl l Ar file
+.Op Fl p Ar file
 .Ar interface
 .Sh DESCRIPTION
 The
@@ -83,6 +84,10 @@ will revert to running in the background.
 Specify an alternate location,
 .Ar file ,
 for the leases file.
+.It Fl p Ar file
+Specify an alternate location for the PID file.
+The default is
+.Pa /var/run/dhclient. Ns Ar interface Ns Pa .pid .
 .It Fl q
 Forces
 .Nm
diff --git a/sbin/dhclient/dhclient.c b/sbin/dhclient/dhclient.c
index afb6d6364a6c..05217301818b 100644
--- a/sbin/dhclient/dhclient.c
+++ b/sbin/dhclient/dhclient.c
@@ -95,6 +95,9 @@ struct iaddr iaddr_broadcast = { 4, { 255, 255, 255, 255 } };
 struct in_addr inaddr_any;
 struct sockaddr_in sockaddr_broadcast;
 
+char *path_dhclient_pidfile;
+struct pidfh *pidfile;
+
 /*
  * ASSERT_STATE() does nothing now; it used to be
  * assert (state_is == state_shouldbe).
@@ -316,6 +319,8 @@ routehandler(struct protocol *p)
 	if (ifi->client->alias)
 		script_write_params("alias_", ifi->client->alias);
 	script_go();
+	if (pidfile != NULL)
+		pidfile_remove(pidfile);
 	exit(1);
 }
 
@@ -327,12 +332,13 @@ main(int argc, char *argv[])
 	int			 pipe_fd[2];
 	int			 immediate_daemon = 0;
 	struct passwd		*pw;
+	pid_t			 otherpid;
 
 	/* Initially, log errors to stderr as well as to syslogd. */
 	openlog(__progname, LOG_PID | LOG_NDELAY, DHCPD_LOG_FACILITY);
 	setlogmask(LOG_UPTO(LOG_DEBUG));
 
-	while ((ch = getopt(argc, argv, "bc:dl:qu")) != -1)
+	while ((ch = getopt(argc, argv, "bc:dl:p:qu")) != -1)
 		switch (ch) {
 		case 'b':
 			immediate_daemon = 1;
@@ -346,6 +352,9 @@ main(int argc, char *argv[])
 		case 'l':
 			path_dhclient_db = optarg;
 			break;
+		case 'p':
+			path_dhclient_pidfile = optarg;
+			break;
 		case 'q':
 			quiet = 1;
 			break;
@@ -362,6 +371,21 @@ main(int argc, char *argv[])
 	if (argc != 1)
 		usage();
 
+	if (path_dhclient_pidfile == NULL) {
+		asprintf(&path_dhclient_pidfile,
+		    "%sdhclient.%s.pid", _PATH_VARRUN, *argv);
+		if (path_dhclient_pidfile == NULL)
+			error("asprintf");
+	}
+	pidfile = pidfile_open(path_dhclient_pidfile, 0600, &otherpid);
+	if (pidfile == NULL) {
+		if (errno == EEXIST)
+			error("dhclient already running, pid: %d.", otherpid);
+		if (errno == EAGAIN)
+			error("dhclient already running.");
+		warning("Cannot open or create pidfile: %m");
+	}
+
 	if ((ifi = calloc(1, sizeof(struct interface_info))) == NULL)
 		error("calloc");
 	if (strlcpy(ifi->name, argv[0], IFNAMSIZ) >= IFNAMSIZ)
@@ -385,6 +409,12 @@ main(int argc, char *argv[])
 
 	read_client_conf();
 
+	/* The next bit is potentially very time-consuming, so write out
+	   the pidfile right away.  We will write it out again with the
+	   correct pid after daemonizing. */
+	if (pidfile != NULL)
+		pidfile_write(pidfile);
+
 	if (!interface_link_status(ifi->name)) {
 		fprintf(stderr, "%s: no link ...", ifi->name);
 		fflush(stderr);
@@ -2298,6 +2328,9 @@ go_daemon(void)
 	if (daemon(1, 0) == -1)
 		error("daemon");
 
+	if (pidfile != NULL)
+		pidfile_write(pidfile);
+
 	/* we are chrooted, daemon(3) fails to open /dev/null */
 	if (nullfd != -1) {
 		dup2(nullfd, STDIN_FILENO);
diff --git a/sbin/dhclient/dhcpd.h b/sbin/dhclient/dhcpd.h
index bd4c9c0b5e15..1334a7c4a055 100644
--- a/sbin/dhclient/dhcpd.h
+++ b/sbin/dhclient/dhcpd.h
@@ -41,7 +41,7 @@
  * $FreeBSD$
  */
 
-#include <sys/types.h>
+#include <sys/param.h>
 
 #include <sys/socket.h>
 #include <sys/sockio.h>
@@ -60,6 +60,7 @@
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <libutil.h>
 #include <limits.h>
 #include <netdb.h>
 #include <paths.h>
@@ -353,6 +354,8 @@ extern int log_perror;
 
 extern struct client_config top_level_config;
 
+extern struct pidfh *pidfile;
+
 void dhcpoffer(struct packet *);
 void dhcpack(struct packet *);
 void dhcpnak(struct packet *);
diff --git a/sbin/dhclient/errwarn.c b/sbin/dhclient/errwarn.c
index de3458322fda..57207807902f 100644
--- a/sbin/dhclient/errwarn.c
+++ b/sbin/dhclient/errwarn.c
@@ -83,6 +83,8 @@ error(char *fmt, ...)
 		fprintf(stderr, "exiting.\n");
 		fflush(stderr);
 	}
+	if (pidfile != NULL)
+		pidfile_remove(pidfile);
 	exit(1);
 }
 
diff --git a/sbin/dump/dump.h b/sbin/dump/dump.h
index efef7f7fadef..2a65865aab3d 100644
--- a/sbin/dump/dump.h
+++ b/sbin/dump/dump.h
@@ -171,9 +171,10 @@ void	putdumptime(void);
     	if (ddatev != NULL) \
 		for (ddp = ddatev[i = 0]; i < nddates; ddp = ddatev[++i])
 
-#define	DUMPOUTFMT	"%-32s %d %s"		/* for printf */
+#define	DUMPFMTLEN	53			/* max device pathname length */
+#define	DUMPOUTFMT	"%-*s %d %s"		/* for printf */
 						/* name, level, ctime(date) */
-#define	DUMPINFMT	"%32s %d %[^\n]\n"	/* inverse for scanf */
+#define	DUMPINFMT	"%s %d %[^\n]\n"	/* inverse for scanf */
 
 void	sig(int signo);
 
diff --git a/sbin/dump/itime.c b/sbin/dump/itime.c
index f3271885fe13..909b8ab1543e 100644
--- a/sbin/dump/itime.c
+++ b/sbin/dump/itime.c
@@ -222,7 +222,10 @@ static void
 dumprecout(FILE *file, const struct dumpdates *what)
 {
 
-	if (fprintf(file, DUMPOUTFMT, what->dd_name,
+	if (strlen(what->dd_name) > DUMPFMTLEN)
+		quit("Name '%s' exceeds DUMPFMTLEN (%d) bytes\n",
+		    what->dd_name, DUMPFMTLEN);
+	if (fprintf(file, DUMPOUTFMT, DUMPFMTLEN, what->dd_name,
 	      what->dd_level, ctime(&what->dd_ddate)) < 0)
 		quit("%s: %s\n", dumpdates, strerror(errno));
 }
diff --git a/sbin/fsck/fsck.8 b/sbin/fsck/fsck.8
index d3f1d4aa7736..3fa113afe0c7 100644
--- a/sbin/fsck/fsck.8
+++ b/sbin/fsck/fsck.8
@@ -41,6 +41,7 @@
 .Op Fl B | F
 .Op Fl T Ar fstype : Ns Ar fsoptions
 .Op Fl t Ar fstype
+.Op Fl c Ar fstab
 .Oo Ar special | node Oc ...
 .Sh DESCRIPTION
 The
@@ -117,6 +118,10 @@ Check if the
 .Dq clean
 flag is set in the superblock and skip file system checks if file system was
 properly dismounted and marked clean.
+.It Fl c Ar fstab
+Specify the
+.Pa fstab
+file to use.
 .It Fl d
 Debugging mode.
 Just print the commands without executing them.
diff --git a/sbin/fsck/fsck.c b/sbin/fsck/fsck.c
index 24c4e13124ae..5e27010c2be9 100644
--- a/sbin/fsck/fsck.c
+++ b/sbin/fsck/fsck.c
@@ -96,6 +96,7 @@ main(int argc, char *argv[])
 	int i, rval = 0;
 	const char *vfstype = NULL;
 	char globopt[3];
+	const char *etc_fstab;
 
 	globopt[0] = '-';
 	globopt[2] = '\0';
@@ -103,7 +104,8 @@ main(int argc, char *argv[])
 	TAILQ_INIT(&selhead);
 	TAILQ_INIT(&opthead);
 
-	while ((i = getopt(argc, argv, "BCdvpfFnyl:t:T:")) != -1)
+	etc_fstab = NULL;
+	while ((i = getopt(argc, argv, "BCdvpfFnyl:t:T:c:")) != -1)
 		switch (i) {
 		case 'B':
 			if (flags & CHECK_BACKGRD)
@@ -160,6 +162,10 @@ main(int argc, char *argv[])
 			vfstype = optarg;
 			break;
 
+		case 'c':
+			etc_fstab = optarg;
+			break;
+
 		case '?':
 		default:
 			usage();
@@ -169,6 +175,9 @@ main(int argc, char *argv[])
 	argc -= optind;
 	argv += optind;
 
+	if (etc_fstab != NULL)
+		setfstab(etc_fstab);
+
 	if (argc == 0)
 		return checkfstab(flags, isok, checkfs);
 
@@ -571,7 +580,7 @@ static void
 usage(void)
 {
 	static const char common[] =
-	    "[-Cdfnpvy] [-B | -F] [-T fstype:fsoptions] [-t fstype]";
+	    "[-Cdfnpvy] [-B | -F] [-T fstype:fsoptions] [-t fstype] [-c fstab]";
 
 	(void)fprintf(stderr, "usage: %s %s [special | node] ...\n",
 	    getprogname(), common);
diff --git a/sbin/fsdb/fsdbutil.c b/sbin/fsdb/fsdbutil.c
index 2c5710a80aa8..7ed78a7f39ee 100644
--- a/sbin/fsdb/fsdbutil.c
+++ b/sbin/fsdb/fsdbutil.c
@@ -126,12 +126,10 @@ printstat(const char *cp, ino_t inum, union dinode *dp)
 	puts("regular file");
 	break;
     case IFBLK:
-	printf("block special (%d,%d)",
-	       major(DIP(dp, di_rdev)), minor(DIP(dp, di_rdev)));
+	printf("block special (%#jx)", (uintmax_t)DIP(dp, di_rdev));
 	break;
     case IFCHR:
-	printf("character special (%d,%d)",
-	       major(DIP(dp, di_rdev)), minor(DIP(dp, di_rdev)));
+	printf("character special (%#jx)", DIP(dp, di_rdev));
 	break;
     case IFLNK:
 	fputs("symlink",stdout);
diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8
index 6c46680c2f6e..d6b607f33123 100644
--- a/sbin/geom/class/eli/geli.8
+++ b/sbin/geom/class/eli/geli.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 9, 2011
+.Dd October 25, 2011
 .Dt GELI 8
 .Os
 .Sh NAME
@@ -60,6 +60,7 @@ utility:
 .Op Fl K Ar newkeyfile
 .Op Fl l Ar keylen
 .Op Fl s Ar sectorsize
+.Op Fl V Ar version
 .Ar prov
 .Nm
 .Cm label - an alias for
@@ -134,6 +135,9 @@ utility:
 .Fl s Ar oldsize
 .Ar prov
 .Nm
+.Cm version
+.Op Ar prov ...
+.Nm
 .Cm clear
 .Op Fl v
 .Ar prov ...
@@ -319,6 +323,15 @@ Change decrypted provider's sector size.
 Increasing sector size allows to increase performance, because we need to
 generate an IV and do encrypt/decrypt for every single sector - less number
 of sectors means less work to do.
+.It Fl V Ar version
+Metadata version to use.
+This option is helpful when creating provider that may be used by older
+.Nm FreeBSD/GELI
+versions.
+Consult the
+.Sx HISTORY
+section to find which metadata version is supported by which FreeBSD version.
+Note that using older metadata version may limit numer of features available.
 .El
 .It Cm attach
 Attach the given provider.
@@ -597,6 +610,18 @@ Additional options include:
 .It Fl s Ar oldsize
 The size of the provider before it was resized.
 .El
+.It Cm version
+If no arguments are given, the
+.Cm version
+subcommand will print the version of
+.Nm
+userland utility as well as the version of the
+.Nm ELI
+GEOM class.
+.Pp
+If GEOM providers are specified, the
+.Cm version
+subcommand will print metadata version used by each of them.
 .It Cm clear
 Clear metadata from the given providers.
 .It Cm dump
@@ -916,5 +941,33 @@ Support for
 .Nm Camellia
 block cipher is implemented by Yoshisato Yanagisawa in
 .Fx 7.0 .
+.Pp
+Highest
+.Nm GELI
+metadata version supported by the given FreeBSD version:
+.Pp
+.Bl -column -offset indent ".Sy FreeBSD" ".Sy version"
+.It Sy FreeBSD Ta Sy GELI
+.It Sy version Ta Sy version
+.Pp
+.It Li 6.0 Ta 0
+.It Li 6.1 Ta 0
+.It Li 6.2 Ta 3
+.It Li 6.3 Ta 3
+.It Li 6.4 Ta 3
+.Pp
+.It Li 7.0 Ta 3
+.It Li 7.1 Ta 3
+.It Li 7.2 Ta 3
+.It Li 7.3 Ta 3
+.It Li 7.4 Ta 3
+.Pp
+.It Li 8.0 Ta 3
+.It Li 8.1 Ta 3
+.It Li 8.2 Ta 5
+.Pp
+.It Li 9.0 Ta 6
+.El
+.Pp
 .Sh AUTHORS
 .An Pawel Jakub Dawidek Aq pjd@FreeBSD.org
diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c
index 25409e7d3c08..f20980024bb7 100644
--- a/sbin/geom/class/eli/geom_eli.c
+++ b/sbin/geom/class/eli/geom_eli.c
@@ -27,27 +27,27 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/types.h>
-#include <sys/sysctl.h>
-
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <readpassphrase.h>
-#include <string.h>
-#include <strings.h>
-#include <libgeom.h>
-#include <paths.h>
-#include <errno.h>
-#include <assert.h>
-
 #include <sys/param.h>
 #include <sys/mman.h>
+#include <sys/sysctl.h>
 #include <sys/resource.h>
 #include <opencrypto/cryptodev.h>
+
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgeom.h>
+#include <paths.h>
+#include <readpassphrase.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+
 #include <geom/eli/g_eli.h>
 #include <geom/eli/pkcs5v2.h>
 
@@ -72,6 +72,7 @@ static void eli_kill(struct gctl_req *req);
 static void eli_backup(struct gctl_req *req);
 static void eli_restore(struct gctl_req *req);
 static void eli_resize(struct gctl_req *req);
+static void eli_version(struct gctl_req *req);
 static void eli_clear(struct gctl_req *req);
 static void eli_dump(struct gctl_req *req);
 
@@ -81,7 +82,7 @@ static int eli_backup_create(struct gctl_req *req, const char *prov,
 /*
  * Available commands:
  *
- * init [-bhPv] [-a aalgo] [-B backupfile] [-e ealgo] [-i iterations] [-l keylen] [-J newpassfile] [-K newkeyfile] prov
+ * init [-bhPv] [-a aalgo] [-B backupfile] [-e ealgo] [-i iterations] [-l keylen] [-J newpassfile] [-K newkeyfile] [-V version] prov
  * label - alias for 'init'
  * attach [-dprv] [-j passfile] [-k keyfile] prov
  * detach [-fl] prov ...
@@ -96,6 +97,7 @@ static int eli_backup_create(struct gctl_req *req, const char *prov,
  * backup [-v] prov file
  * restore [-fv] file prov
  * resize [-v] -s oldsize prov
+ * version [prov ...]
  * clear [-v] prov ...
  * dump [-v] prov ...
  */
@@ -105,29 +107,31 @@ struct g_command class_commands[] = {
 		{ 'a', "aalgo", "", G_TYPE_STRING },
 		{ 'b', "boot", NULL, G_TYPE_BOOL },
 		{ 'B', "backupfile", "", G_TYPE_STRING },
-		{ 'e', "ealgo", GELI_ENC_ALGO, G_TYPE_STRING },
+		{ 'e', "ealgo", "", G_TYPE_STRING },
 		{ 'i', "iterations", "-1", G_TYPE_NUMBER },
 		{ 'J', "newpassfile", G_VAL_OPTIONAL, G_TYPE_STRING | G_TYPE_MULTI },
 		{ 'K', "newkeyfile", G_VAL_OPTIONAL, G_TYPE_STRING | G_TYPE_MULTI },
 		{ 'l', "keylen", "0", G_TYPE_NUMBER },
 		{ 'P', "nonewpassphrase", NULL, G_TYPE_BOOL },
 		{ 's', "sectorsize", "0", G_TYPE_NUMBER },
+		{ 'V', "mdversion", "-1", G_TYPE_NUMBER },
 		G_OPT_SENTINEL
 	    },
-	    "[-bPv] [-a aalgo] [-B backupfile] [-e ealgo] [-i iterations] [-l keylen] [-J newpassfile] [-K newkeyfile] [-s sectorsize] prov"
+	    "[-bPv] [-a aalgo] [-B backupfile] [-e ealgo] [-i iterations] [-l keylen] [-J newpassfile] [-K newkeyfile] [-s sectorsize] [-V version] prov"
 	},
 	{ "label", G_FLAG_VERBOSE, eli_main,
 	    {
 		{ 'a', "aalgo", "", G_TYPE_STRING },
 		{ 'b', "boot", NULL, G_TYPE_BOOL },
 		{ 'B', "backupfile", "", G_TYPE_STRING },
-		{ 'e', "ealgo", GELI_ENC_ALGO, G_TYPE_STRING },
+		{ 'e', "ealgo", "", G_TYPE_STRING },
 		{ 'i', "iterations", "-1", G_TYPE_NUMBER },
 		{ 'J', "newpassfile", G_VAL_OPTIONAL, G_TYPE_STRING | G_TYPE_MULTI },
 		{ 'K', "newkeyfile", G_VAL_OPTIONAL, G_TYPE_STRING | G_TYPE_MULTI },
 		{ 'l', "keylen", "0", G_TYPE_NUMBER },
 		{ 'P', "nonewpassphrase", NULL, G_TYPE_BOOL },
 		{ 's', "sectorsize", "0", G_TYPE_NUMBER },
+		{ 'V', "mdversion", "-1", G_TYPE_NUMBER },
 		G_OPT_SENTINEL
 	    },
 	    "- an alias for 'init'"
@@ -241,6 +245,9 @@ struct g_command class_commands[] = {
 	    },
 	    "[-v] -s oldsize prov"
 	},
+	{ "version", G_FLAG_LOADKLD, eli_main, G_NULL_OPTS,
+	    "[prov ...]"
+	},
 	{ "clear", G_FLAG_VERBOSE, eli_main, G_NULL_OPTS,
 	    "[-v] prov ..."
 	},
@@ -309,6 +316,8 @@ eli_main(struct gctl_req *req, unsigned int flags)
 		eli_restore(req);
 	else if (strcmp(name, "resize") == 0)
 		eli_resize(req);
+	else if (strcmp(name, "version") == 0)
+		eli_version(req);
 	else if (strcmp(name, "dump") == 0)
 		eli_dump(req);
 	else if (strcmp(name, "clear") == 0)
@@ -333,11 +342,10 @@ arc4rand(unsigned char *buf, size_t size)
 		buf[i] = arc4random() % 0xff;
 }
 
-static int
+static bool
 eli_is_attached(const char *prov)
 {
 	char name[MAXPATHLEN];
-	unsigned secsize;
 
 	/*
 	 * Not the best way to do it, but the easiest.
@@ -345,10 +353,7 @@ eli_is_attached(const char *prov)
 	 * by asking about its sectorsize.
 	 */
 	snprintf(name, sizeof(name), "%s%s", prov, G_ELI_SUFFIX);
-	secsize = g_get_sectorsize(name);
-	if (secsize > 0)
-		return (1);
-	return (0);
+	return (g_get_sectorsize(name) > 0);
 }
 
 static int
@@ -598,8 +603,23 @@ eli_metadata_read(struct gctl_req *req, const char *prov,
 			return (-1);
 		}
 	}
-	if (eli_metadata_decode(sector, md) != 0) {
-		gctl_error(req, "MD5 hash mismatch for %s.", prov);
+	error = eli_metadata_decode(sector, md);
+	switch (error) {
+	case 0:
+		break;
+	case EOPNOTSUPP:
+		gctl_error(req,
+		    "Provider's %s metadata version %u is too new.\n"
+		    "geli: The highest supported version is %u.",
+		    prov, (unsigned int)md->md_version, G_ELI_VERSION);
+		return (-1);
+	case EINVAL:
+		gctl_error(req, "Inconsistent provider's %s metadata.", prov);
+		return (-1);
+	default:
+		gctl_error(req,
+		    "Unexpected error while decoding provider's %s metadata: %s.",
+		    prov, strerror(error));
 		return (-1);
 	}
 	return (0);
@@ -654,7 +674,7 @@ eli_init(struct gctl_req *req)
 	unsigned char key[G_ELI_USERKEYLEN];
 	char backfile[MAXPATHLEN];
 	const char *str, *prov;
-	unsigned secsize;
+	unsigned int secsize, version;
 	off_t mediasize;
 	intmax_t val;
 	int error, nargs;
@@ -675,13 +695,30 @@ eli_init(struct gctl_req *req)
 
 	bzero(&md, sizeof(md));
 	strlcpy(md.md_magic, G_ELI_MAGIC, sizeof(md.md_magic));
-	md.md_version = G_ELI_VERSION;
+	val = gctl_get_intmax(req, "mdversion");
+	if (val == -1) {
+		version = G_ELI_VERSION;
+	} else if (val < 0 || val > G_ELI_VERSION) {
+		gctl_error(req,
+		    "Invalid version specified should be between %u and %u.",
+		    G_ELI_VERSION_00, G_ELI_VERSION);
+		return;
+	} else {
+		version = val;
+	}
+	md.md_version = version;
 	md.md_flags = 0;
 	if (gctl_get_int(req, "boot"))
 		md.md_flags |= G_ELI_FLAG_BOOT;
 	md.md_ealgo = CRYPTO_ALGORITHM_MIN - 1;
 	str = gctl_get_ascii(req, "aalgo");
 	if (*str != '\0') {
+		if (version < G_ELI_VERSION_01) {
+			gctl_error(req,
+			    "Data authentication is supported starting from version %u.",
+			    G_ELI_VERSION_01);
+			return;
+		}
 		md.md_aalgo = g_eli_str2aalgo(str);
 		if (md.md_aalgo >= CRYPTO_ALGORITHM_MIN &&
 		    md.md_aalgo <= CRYPTO_ALGORITHM_MAX) {
@@ -707,12 +744,32 @@ eli_init(struct gctl_req *req)
 	if (md.md_ealgo < CRYPTO_ALGORITHM_MIN ||
 	    md.md_ealgo > CRYPTO_ALGORITHM_MAX) {
 		str = gctl_get_ascii(req, "ealgo");
+		if (*str == '\0') {
+			if (version < G_ELI_VERSION_05)
+				str = "aes-cbc";
+			else
+				str = GELI_ENC_ALGO;
+		}
 		md.md_ealgo = g_eli_str2ealgo(str);
 		if (md.md_ealgo < CRYPTO_ALGORITHM_MIN ||
 		    md.md_ealgo > CRYPTO_ALGORITHM_MAX) {
 			gctl_error(req, "Invalid encryption algorithm.");
 			return;
 		}
+		if (md.md_ealgo == CRYPTO_CAMELLIA_CBC &&
+		    version < G_ELI_VERSION_04) {
+			gctl_error(req,
+			    "Camellia-CBC algorithm is supported starting from version %u.",
+			    G_ELI_VERSION_04);
+			return;
+		}
+		if (md.md_ealgo == CRYPTO_AES_XTS &&
+		    version < G_ELI_VERSION_05) {
+			gctl_error(req,
+			    "AES-XTS algorithm is supported starting from version %u.",
+			    G_ELI_VERSION_05);
+			return;
+		}
 	}
 	val = gctl_get_intmax(req, "keylen");
 	md.md_keylen = val;
@@ -1295,66 +1352,53 @@ eli_kill(struct gctl_req *req)
 static int
 eli_backup_create(struct gctl_req *req, const char *prov, const char *file)
 {
-	struct g_eli_metadata md;
 	unsigned char *sector;
 	ssize_t secsize;
-	off_t mediasize;
-	int filefd, provfd, ret;
+	int error, filefd, ret;
 
 	ret = -1;
-	provfd = filefd = -1;
+	filefd = -1;
 	sector = NULL;
 	secsize = 0;
 
-	provfd = g_open(prov, 0);
-	if (provfd == -1) {
-		gctl_error(req, "Cannot open %s: %s.", prov, strerror(errno));
-		goto out;
-	}
-	filefd = open(file, O_WRONLY | O_TRUNC | O_CREAT, 0600);
-	if (filefd == -1) {
-		gctl_error(req, "Cannot open %s: %s.", file, strerror(errno));
-		goto out;
-	}
-
-	mediasize = g_mediasize(provfd);
-	secsize = g_sectorsize(provfd);
-	if (mediasize == -1 || secsize == -1) {
+	secsize = g_get_sectorsize(prov);
+	if (secsize == 0) {
 		gctl_error(req, "Cannot get informations about %s: %s.", prov,
 		    strerror(errno));
 		goto out;
 	}
-
 	sector = malloc(secsize);
 	if (sector == NULL) {
 		gctl_error(req, "Cannot allocate memory.");
 		goto out;
 	}
-
 	/* Read metadata from the provider. */
-	if (pread(provfd, sector, secsize, mediasize - secsize) != secsize) {
-		gctl_error(req, "Cannot read metadata: %s.", strerror(errno));
+	error = g_metadata_read(prov, sector, secsize, G_ELI_MAGIC);
+	if (error != 0) {
+		gctl_error(req, "Unable to read metadata from %s: %s.", prov,
+		    strerror(error));
 		goto out;
 	}
-	/* Check if this is geli provider. */
-	if (eli_metadata_decode(sector, &md) != 0) {
-		gctl_error(req, "MD5 hash mismatch: not a geli provider?");
+
+	filefd = open(file, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+	if (filefd == -1) {
+		gctl_error(req, "Unable to open %s: %s.", file,
+		    strerror(errno));
 		goto out;
 	}
 	/* Write metadata to the destination file. */
 	if (write(filefd, sector, secsize) != secsize) {
-		gctl_error(req, "Cannot write to %s: %s.", file,
+		gctl_error(req, "Unable to write to %s: %s.", file,
 		    strerror(errno));
+		(void)close(filefd);
+		(void)unlink(file);
 		goto out;
 	}
 	(void)fsync(filefd);
+	(void)close(filefd);
 	/* Success. */
 	ret = 0;
 out:
-	if (provfd >= 0)
-		(void)g_close(provfd);
-	if (filefd >= 0)
-		(void)close(filefd);
 	if (sector != NULL) {
 		bzero(sector, secsize);
 		free(sector);
@@ -1384,10 +1428,8 @@ eli_restore(struct gctl_req *req)
 {
 	struct g_eli_metadata md;
 	const char *file, *prov;
-	unsigned char *sector;
-	ssize_t secsize;
 	off_t mediasize;
-	int nargs, filefd, provfd;
+	int nargs;
 
 	nargs = gctl_get_int(req, "nargs");
 	if (nargs != 2) {
@@ -1397,72 +1439,28 @@ eli_restore(struct gctl_req *req)
 	file = gctl_get_ascii(req, "arg0");
 	prov = gctl_get_ascii(req, "arg1");
 
-	provfd = filefd = -1;
-	sector = NULL;
-	secsize = 0;
-
-	filefd = open(file, O_RDONLY);
-	if (filefd == -1) {
-		gctl_error(req, "Cannot open %s: %s.", file, strerror(errno));
-		goto out;
-	}
-	provfd = g_open(prov, 1);
-	if (provfd == -1) {
-		gctl_error(req, "Cannot open %s: %s.", prov, strerror(errno));
-		goto out;
-	}
-
-	mediasize = g_mediasize(provfd);
-	secsize = g_sectorsize(provfd);
-	if (mediasize == -1 || secsize == -1) {
+	/* Read metadata from the backup file. */
+	if (eli_metadata_read(req, file, &md) == -1)
+		return;
+	/* Obtain provider's mediasize. */
+	mediasize = g_get_mediasize(prov);
+	if (mediasize == 0) {
 		gctl_error(req, "Cannot get informations about %s: %s.", prov,
 		    strerror(errno));
-		goto out;
-	}
-
-	sector = malloc(secsize);
-	if (sector == NULL) {
-		gctl_error(req, "Cannot allocate memory.");
-		goto out;
-	}
-
-	/* Read metadata from the backup file. */
-	if (read(filefd, sector, secsize) != secsize) {
-		gctl_error(req, "Cannot read from %s: %s.", file,
-		    strerror(errno));
-		goto out;
-	}
-	/* Check if this file contains geli metadata. */
-	if (eli_metadata_decode(sector, &md) != 0) {
-		gctl_error(req, "MD5 hash mismatch: not a geli backup file?");
-		goto out;
+		return;
 	}
 	/* Check if the provider size has changed since we did the backup. */
 	if (md.md_provsize != (uint64_t)mediasize) {
 		if (gctl_get_int(req, "force")) {
 			md.md_provsize = mediasize;
-			eli_metadata_encode(&md, sector);
 		} else {
 			gctl_error(req, "Provider size mismatch: "
 			    "wrong backup file?");
-			goto out;
+			return;
 		}
 	}
-	/* Write metadata from the provider. */
-	if (pwrite(provfd, sector, secsize, mediasize - secsize) != secsize) {
-		gctl_error(req, "Cannot write metadata: %s.", strerror(errno));
-		goto out;
-	}
-	(void)g_flush(provfd);
-out:
-	if (provfd >= 0)
-		(void)g_close(provfd);
-	if (filefd >= 0)
-		(void)close(filefd);
-	if (sector != NULL) {
-		bzero(sector, secsize);
-		free(sector);
-	}
+	/* Write metadata to the provider. */
+	(void)eli_metadata_store(req, prov, &md);
 }
 
 static void
@@ -1473,7 +1471,7 @@ eli_resize(struct gctl_req *req)
 	unsigned char *sector;
 	ssize_t secsize;
 	off_t mediasize, oldsize;
-	int nargs, provfd;
+	int error, nargs, provfd;
 
 	nargs = gctl_get_int(req, "nargs");
 	if (nargs != 1) {
@@ -1524,8 +1522,23 @@ eli_resize(struct gctl_req *req)
 	}
 
 	/* Check if this sector contains geli metadata. */
-	if (eli_metadata_decode(sector, &md) != 0) {
-		gctl_error(req, "MD5 hash mismatch: no metadata for oldsize.");
+	error = eli_metadata_decode(sector, &md);
+	switch (error) {
+	case 0:
+		break;
+	case EOPNOTSUPP:
+		gctl_error(req,
+		    "Provider's %s metadata version %u is too new.\n"
+		    "geli: The highest supported version is %u.",
+		    prov, (unsigned int)md.md_version, G_ELI_VERSION);
+		goto out;
+	case EINVAL:
+		gctl_error(req, "Inconsistent provider's %s metadata.", prov);
+		goto out;
+	default:
+		gctl_error(req,
+		    "Unexpected error while decoding provider's %s metadata: %s.",
+		    prov, strerror(error));
 		goto out;
 	}
 
@@ -1543,18 +1556,12 @@ eli_resize(struct gctl_req *req)
 	 * it back to the correct place on the provider.
 	 */
 	md.md_provsize = mediasize;
-	eli_metadata_encode(&md, sector);
-	if (pwrite(provfd, sector, secsize, mediasize - secsize) != secsize) {
-		gctl_error(req, "Cannot write metadata: %s.", strerror(errno));
-		goto out;
-	}
-	(void)g_flush(provfd);
-
+	/* Write metadata to the provider. */
+	(void)eli_metadata_store(req, prov, &md);
 	/* Now trash the old metadata. */
-	if (eli_trash_metadata(req, prov, provfd, oldsize - secsize) == -1)
-		goto out;
+	(void)eli_trash_metadata(req, prov, provfd, oldsize - secsize);
 out:
-	if (provfd >= 0)
+	if (provfd != -1)
 		(void)g_close(provfd);
 	if (sector != NULL) {
 		bzero(sector, secsize);
@@ -1562,6 +1569,46 @@ eli_resize(struct gctl_req *req)
 	}
 }
 
+static void
+eli_version(struct gctl_req *req)
+{
+	struct g_eli_metadata md;
+	const char *name;
+	unsigned int version;
+	int error, i, nargs;
+
+	nargs = gctl_get_int(req, "nargs");
+
+	if (nargs == 0) {
+		unsigned int kernver;
+		ssize_t size;
+
+		size = sizeof(kernver);
+		if (sysctlbyname("kern.geom.eli.version", &kernver, &size,
+		    NULL, 0) == -1) {
+			warn("Unable to obtain GELI kernel version");
+		} else {
+			printf("kernel: %u\n", kernver);
+		}
+		printf("userland: %u\n", G_ELI_VERSION);
+		return;
+	}
+
+	for (i = 0; i < nargs; i++) {
+		name = gctl_get_ascii(req, "arg%d", i);
+		error = g_metadata_read(name, (unsigned char *)&md,
+		    sizeof(md), G_ELI_MAGIC);
+		if (error != 0) {
+			warn("%s: Unable to read metadata: %s.", name,
+			    strerror(error));
+			gctl_error(req, "Not fully done.");
+			continue;
+		}
+		version = le32dec(&md.md_version);
+		printf("%s: %u\n", name, version);
+	}
+}
+
 static void
 eli_clear(struct gctl_req *req)
 {
@@ -1591,9 +1638,9 @@ eli_clear(struct gctl_req *req)
 static void
 eli_dump(struct gctl_req *req)
 {
-	struct g_eli_metadata md, tmpmd;
+	struct g_eli_metadata md;
 	const char *name;
-	int error, i, nargs;
+	int i, nargs;
 
 	nargs = gctl_get_int(req, "nargs");
 	if (nargs < 1) {
@@ -1603,17 +1650,7 @@ eli_dump(struct gctl_req *req)
 
 	for (i = 0; i < nargs; i++) {
 		name = gctl_get_ascii(req, "arg%d", i);
-		error = g_metadata_read(name, (unsigned char *)&tmpmd,
-		    sizeof(tmpmd), G_ELI_MAGIC);
-		if (error != 0) {
-			fprintf(stderr, "Cannot read metadata from %s: %s.\n",
-			    name, strerror(error));
-			gctl_error(req, "Not fully done.");
-			continue;
-		}
-		if (eli_metadata_decode((unsigned char *)&tmpmd, &md) != 0) {
-			fprintf(stderr, "MD5 hash mismatch for %s, skipping.\n",
-			    name);
+		if (eli_metadata_read(NULL, name, &md) == -1) {
 			gctl_error(req, "Not fully done.");
 			continue;
 		}
diff --git a/sbin/geom/class/part/gpart.8 b/sbin/geom/class/part/gpart.8
index 046ab26bb445..66c30d63834c 100644
--- a/sbin/geom/class/part/gpart.8
+++ b/sbin/geom/class/part/gpart.8
@@ -371,7 +371,7 @@ Modify a partition from geom
 and further identified by the
 .Fl i Ar index
 option.
-Only the the type and/or label of the partition can be modified.
+Only the type and/or label of the partition can be modified.
 To change the type of a partition, specify the new type with the
 .Fl t Ar type
 option.
diff --git a/sbin/geom/class/raid/graid.8 b/sbin/geom/class/raid/graid.8
index d1c92a2148af..4166371b98cb 100644
--- a/sbin/geom/class/raid/graid.8
+++ b/sbin/geom/class/raid/graid.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 22, 2011
+.Dd October 26, 2011
 .Dt GRAID 8
 .Os
 .Sh NAME
@@ -250,6 +250,9 @@ If you started migration using BIOS or in some other way, make sure to
 complete it there.
 Do not run GEOM RAID class on migrating volumes under pain of possible data
 corruption!
+.Sh 2TiB BARRIERS
+Intel and Promise metadata formats do not support disks above 2TiB.
+NVIDIA metadata format does not support volumes above 2TiB.
 .Sh EXIT STATUS
 Exit status is 0 on success, and non-zero if the command fails.
 .Sh SEE ALSO
diff --git a/sbin/geom/misc/subr.c b/sbin/geom/misc/subr.c
index 1409eba751cb..f7b2764a6f14 100644
--- a/sbin/geom/misc/subr.c
+++ b/sbin/geom/misc/subr.c
@@ -379,10 +379,15 @@ gctl_error(struct gctl_req *req, const char *error, ...)
 {
 	va_list ap;
 
-	if (req->error != NULL)
+	if (req != NULL && req->error != NULL)
 		return;
 	va_start(ap, error);
-	vasprintf(&req->error, error, ap);
+	if (req != NULL) {
+		vasprintf(&req->error, error, ap);
+	} else {
+		vfprintf(stderr, error, ap);
+		fprintf(stderr, "\n");
+	}
 	va_end(ap);
 }
 
diff --git a/sbin/hastctl/hastctl.c b/sbin/hastctl/hastctl.c
index 233b9483ee86..838e11d27fae 100644
--- a/sbin/hastctl/hastctl.c
+++ b/sbin/hastctl/hastctl.c
@@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 #include <sys/sysctl.h>
 
-#include <assert.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -433,19 +432,19 @@ main(int argc, char *argv[])
 	pjdlog_debug_set(debug);
 
 	cfg = yy_config_parse(cfgpath, true);
-	assert(cfg != NULL);
+	PJDLOG_ASSERT(cfg != NULL);
 
 	switch (cmd) {
 	case CMD_CREATE:
 		control_create(argc, argv, mediasize, extentsize, keepdirty);
 		/* NOTREACHED */
-		assert(!"What are we doing here?!");
+		PJDLOG_ABORT("What are we doing here?!");
 		break;
 	case CMD_DUMP:
 		/* Dump metadata from local component of the given resource. */
 		control_dump(argc, argv);
 		/* NOTREACHED */
-		assert(!"What are we doing here?!");
+		PJDLOG_ABORT("What are we doing here?!");
 		break;
 	case CMD_ROLE:
 		/* Change role for the given resources. */
@@ -476,7 +475,7 @@ main(int argc, char *argv[])
 		}
 		break;
 	default:
-		assert(!"Impossible command!");
+		PJDLOG_ABORT("Impossible command!");
 	}
 
 	/* Setup control connection... */
@@ -523,7 +522,7 @@ main(int argc, char *argv[])
 		error = control_status(nv);
 		break;
 	default:
-		assert(!"Impossible command!");
+		PJDLOG_ABORT("Impossible command!");
 	}
 
 	exit(error);
diff --git a/sbin/hastd/activemap.c b/sbin/hastd/activemap.c
index 127b2c89e96d..e388bb2eea34 100644
--- a/sbin/hastd/activemap.c
+++ b/sbin/hastd/activemap.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>	/* powerof2() */
 #include <sys/queue.h>
 
-#include <assert.h>
 #include <bitstring.h>
 #include <errno.h>
 #include <stdint.h>
@@ -41,7 +40,14 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <string.h>
 
-#include <activemap.h>
+#include <pjdlog.h>
+
+#include "activemap.h"
+
+#ifndef	PJDLOG_ASSERT
+#include <assert.h>
+#define	PJDLOG_ASSERT(...)	assert(__VA_ARGS__)
+#endif
 
 #define	ACTIVEMAP_MAGIC	0xac71e4
 struct activemap {
@@ -93,9 +99,9 @@ off2ext(const struct activemap *amp, off_t offset)
 {
 	int extent;
 
-	assert(offset >= 0 && offset < amp->am_mediasize);
+	PJDLOG_ASSERT(offset >= 0 && offset < amp->am_mediasize);
 	extent = (offset >> amp->am_extentshift);
-	assert(extent >= 0 && extent < amp->am_nextents);
+	PJDLOG_ASSERT(extent >= 0 && extent < amp->am_nextents);
 	return (extent);
 }
 
@@ -104,9 +110,9 @@ ext2off(const struct activemap *amp, int extent)
 {
 	off_t offset;
 
-	assert(extent >= 0 && extent < amp->am_nextents);
+	PJDLOG_ASSERT(extent >= 0 && extent < amp->am_nextents);
 	offset = ((off_t)extent << amp->am_extentshift);
-	assert(offset >= 0 && offset < amp->am_mediasize);
+	PJDLOG_ASSERT(offset >= 0 && offset < amp->am_mediasize);
 	return (offset);
 }
 
@@ -122,7 +128,7 @@ ext2reqs(const struct activemap *amp, int ext)
 	if (ext < amp->am_nextents - 1)
 		return (((amp->am_extentsize - 1) / MAXPHYS) + 1);
 
-	assert(ext == amp->am_nextents - 1);
+	PJDLOG_ASSERT(ext == amp->am_nextents - 1);
 	left = amp->am_mediasize % amp->am_extentsize;
 	if (left == 0)
 		left = amp->am_extentsize;
@@ -139,13 +145,13 @@ activemap_init(struct activemap **ampp, uint64_t mediasize, uint32_t extentsize,
 {
 	struct activemap *amp;
 
-	assert(ampp != NULL);
-	assert(mediasize > 0);
-	assert(extentsize > 0);
-	assert(powerof2(extentsize));
-	assert(sectorsize > 0);
-	assert(powerof2(sectorsize));
-	assert(keepdirty > 0);
+	PJDLOG_ASSERT(ampp != NULL);
+	PJDLOG_ASSERT(mediasize > 0);
+	PJDLOG_ASSERT(extentsize > 0);
+	PJDLOG_ASSERT(powerof2(extentsize));
+	PJDLOG_ASSERT(sectorsize > 0);
+	PJDLOG_ASSERT(powerof2(sectorsize));
+	PJDLOG_ASSERT(keepdirty > 0);
 
 	amp = malloc(sizeof(*amp));
 	if (amp == NULL)
@@ -225,10 +231,10 @@ keepdirty_add(struct activemap *amp, int extent)
 	 */
 	if (amp->am_nkeepdirty >= amp->am_nkeepdirty_limit) {
 		kd = TAILQ_LAST(&amp->am_keepdirty, skeepdirty);
-		assert(kd != NULL);
+		PJDLOG_ASSERT(kd != NULL);
 		TAILQ_REMOVE(&amp->am_keepdirty, kd, kd_next);
 		amp->am_nkeepdirty--;
-		assert(amp->am_nkeepdirty > 0);
+		PJDLOG_ASSERT(amp->am_nkeepdirty > 0);
 	}
 	if (kd == NULL)
 		kd = malloc(sizeof(*kd));
@@ -261,7 +267,7 @@ keepdirty_free(struct activemap *amp)
 		amp->am_nkeepdirty--;
 		free(kd);
 	}
-	assert(amp->am_nkeepdirty == 0);
+	PJDLOG_ASSERT(amp->am_nkeepdirty == 0);
 }
 
 /*
@@ -271,7 +277,7 @@ void
 activemap_free(struct activemap *amp)
 {
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	amp->am_magic = 0;
 
@@ -293,8 +299,8 @@ activemap_write_start(struct activemap *amp, off_t offset, off_t length)
 	off_t end;
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
-	assert(length > 0);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(length > 0);
 
 	modified = false;
 	end = offset + length - 1;
@@ -307,7 +313,7 @@ activemap_write_start(struct activemap *amp, off_t offset, off_t length)
 		 * was modified and has to be flushed to disk.
 		 */
 		if (amp->am_memtab[ext]++ == 0) {
-			assert(!bit_test(amp->am_memmap, ext));
+			PJDLOG_ASSERT(!bit_test(amp->am_memmap, ext));
 			bit_set(amp->am_memmap, ext);
 			amp->am_ndirty++;
 		}
@@ -329,8 +335,8 @@ activemap_write_complete(struct activemap *amp, off_t offset, off_t length)
 	off_t end;
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
-	assert(length > 0);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(length > 0);
 
 	modified = false;
 	end = offset + length - 1;
@@ -342,8 +348,8 @@ activemap_write_complete(struct activemap *amp, off_t offset, off_t length)
 		 * By returning true we inform the caller that on-disk bitmap
 		 * was modified and has to be flushed to disk.
 		 */
-		assert(amp->am_memtab[ext] > 0);
-		assert(bit_test(amp->am_memmap, ext));
+		PJDLOG_ASSERT(amp->am_memtab[ext] > 0);
+		PJDLOG_ASSERT(bit_test(amp->am_memmap, ext));
 		if (--amp->am_memtab[ext] == 0) {
 			bit_clear(amp->am_memmap, ext);
 			amp->am_ndirty--;
@@ -365,15 +371,15 @@ activemap_extent_complete(struct activemap *amp, int extent)
 	bool modified;
 	int reqs;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
-	assert(extent >= 0 && extent < amp->am_nextents);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(extent >= 0 && extent < amp->am_nextents);
 
 	modified = false;
 
 	reqs = ext2reqs(amp, extent);
-	assert(amp->am_memtab[extent] >= reqs);
+	PJDLOG_ASSERT(amp->am_memtab[extent] >= reqs);
 	amp->am_memtab[extent] -= reqs;
-	assert(bit_test(amp->am_memmap, extent));
+	PJDLOG_ASSERT(bit_test(amp->am_memmap, extent));
 	if (amp->am_memtab[extent] == 0) {
 		bit_clear(amp->am_memmap, extent);
 		amp->am_ndirty--;
@@ -390,7 +396,7 @@ uint64_t
 activemap_ndirty(const struct activemap *amp)
 {
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	return (amp->am_ndirty);
 }
@@ -403,7 +409,7 @@ bool
 activemap_differ(const struct activemap *amp)
 {
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	return (memcmp(amp->am_diskmap, amp->am_memmap,
 	    amp->am_mapsize) != 0);
@@ -416,7 +422,7 @@ size_t
 activemap_size(const struct activemap *amp)
 {
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	return (amp->am_mapsize);
 }
@@ -429,7 +435,7 @@ size_t
 activemap_ondisk_size(const struct activemap *amp)
 {
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	return (amp->am_diskmapsize);
 }
@@ -442,8 +448,8 @@ activemap_copyin(struct activemap *amp, const unsigned char *buf, size_t size)
 {
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
-	assert(size >= amp->am_mapsize);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(size >= amp->am_mapsize);
 
 	memcpy(amp->am_diskmap, buf, amp->am_mapsize);
 	memcpy(amp->am_memmap, buf, amp->am_mapsize);
@@ -481,8 +487,8 @@ activemap_merge(struct activemap *amp, const unsigned char *buf, size_t size)
 	bitstr_t *remmap = __DECONST(bitstr_t *, buf);
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
-	assert(size >= amp->am_mapsize);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(size >= amp->am_mapsize);
 
 	bit_ffs(remmap, amp->am_nextents, &ext);
 	if (ext == -1) {
@@ -521,7 +527,7 @@ const unsigned char *
 activemap_bitmap(struct activemap *amp, size_t *sizep)
 {
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	if (sizep != NULL)
 		*sizep = amp->am_diskmapsize;
@@ -539,11 +545,11 @@ activemap_calc_ondisk_size(uint64_t mediasize, uint32_t extentsize,
 {
 	uint64_t nextents, mapsize;
 
-	assert(mediasize > 0);
-	assert(extentsize > 0);
-	assert(powerof2(extentsize));
-	assert(sectorsize > 0);
-	assert(powerof2(sectorsize));
+	PJDLOG_ASSERT(mediasize > 0);
+	PJDLOG_ASSERT(extentsize > 0);
+	PJDLOG_ASSERT(powerof2(extentsize));
+	PJDLOG_ASSERT(sectorsize > 0);
+	PJDLOG_ASSERT(powerof2(sectorsize));
 
 	nextents = ((mediasize - 1) / extentsize) + 1;
 	mapsize = sizeof(bitstr_t) * bitstr_size(nextents);
@@ -558,7 +564,7 @@ activemap_sync_rewind(struct activemap *amp)
 {
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	bit_ffs(amp->am_syncmap, amp->am_nextents, &ext);
 	if (ext == -1) {
@@ -581,9 +587,9 @@ activemap_sync_offset(struct activemap *amp, off_t *lengthp, int *syncextp)
 	off_t syncoff, left;
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
-	assert(lengthp != NULL);
-	assert(syncextp != NULL);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(lengthp != NULL);
+	PJDLOG_ASSERT(syncextp != NULL);
 
 	*syncextp = -1;
 
@@ -632,9 +638,10 @@ activemap_sync_offset(struct activemap *amp, off_t *lengthp, int *syncextp)
 	if (left > MAXPHYS)
 		left = MAXPHYS;
 
-	assert(left >= 0 && left <= MAXPHYS);
-	assert(syncoff >= 0 && syncoff < amp->am_mediasize);
-	assert(syncoff + left >= 0 && syncoff + left <= amp->am_mediasize);
+	PJDLOG_ASSERT(left >= 0 && left <= MAXPHYS);
+	PJDLOG_ASSERT(syncoff >= 0 && syncoff < amp->am_mediasize);
+	PJDLOG_ASSERT(syncoff + left >= 0 &&
+	    syncoff + left <= amp->am_mediasize);
 
 	*lengthp = left;
 	return (syncoff);
@@ -651,7 +658,7 @@ activemap_need_sync(struct activemap *amp, off_t offset, off_t length)
 	off_t end;
 	int ext;
 
-	assert(amp->am_magic == ACTIVEMAP_MAGIC);
+	PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC);
 
 	modified = false;
 	end = offset + length - 1;
@@ -659,7 +666,7 @@ activemap_need_sync(struct activemap *amp, off_t offset, off_t length)
 	for (ext = off2ext(amp, offset); ext <= off2ext(amp, end); ext++) {
 		if (bit_test(amp->am_syncmap, ext)) {
 			/* Already marked for synchronization. */
-			assert(bit_test(amp->am_memmap, ext));
+			PJDLOG_ASSERT(bit_test(amp->am_memmap, ext));
 			continue;
 		}
 		bit_set(amp->am_syncmap, ext);
diff --git a/sbin/hastd/control.c b/sbin/hastd/control.c
index 57a1c8a67a8e..93bfbb7cb394 100644
--- a/sbin/hastd/control.c
+++ b/sbin/hastd/control.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/wait.h>
 
-#include <assert.h>
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
@@ -82,8 +81,8 @@ control_set_role_common(struct hastd_config *cfg, struct nv *nvout,
 		nv_add_string(nvout, name, "resource%u", no);
 
 	if (res == NULL) {
-		assert(cfg != NULL);
-		assert(name != NULL);
+		PJDLOG_ASSERT(cfg != NULL);
+		PJDLOG_ASSERT(name != NULL);
 
 		TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
 			if (strcmp(res->hr_name, name) == 0)
@@ -94,7 +93,7 @@ control_set_role_common(struct hastd_config *cfg, struct nv *nvout,
 			return;
 		}
 	}
-	assert(res != NULL);
+	PJDLOG_ASSERT(res != NULL);
 
 	/* Send previous role back. */
 	nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
@@ -222,9 +221,9 @@ control_status(struct hastd_config *cfg, struct nv *nvout,
     struct hast_resource *res, const char *name, unsigned int no)
 {
 
-	assert(cfg != NULL);
-	assert(nvout != NULL);
-	assert(name != NULL);
+	PJDLOG_ASSERT(cfg != NULL);
+	PJDLOG_ASSERT(nvout != NULL);
+	PJDLOG_ASSERT(name != NULL);
 
 	/* Name is always needed. */
 	nv_add_string(nvout, name, "resource%u", no);
@@ -239,7 +238,7 @@ control_status(struct hastd_config *cfg, struct nv *nvout,
 			return;
 		}
 	}
-	assert(res != NULL);
+	PJDLOG_ASSERT(res != NULL);
 	nv_add_string(nvout, res->hr_provname, "provname%u", no);
 	nv_add_string(nvout, res->hr_localpath, "localpath%u", no);
 	nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no);
@@ -267,7 +266,7 @@ control_status(struct hastd_config *cfg, struct nv *nvout,
 
 	switch (res->hr_role) {
 	case HAST_ROLE_PRIMARY:
-		assert(res->hr_workerpid != 0);
+		PJDLOG_ASSERT(res->hr_workerpid != 0);
 		/* FALLTHROUGH */
 	case HAST_ROLE_SECONDARY:
 		if (res->hr_workerpid != 0)
@@ -470,7 +469,7 @@ ctrl_thread(void *arg)
 			 * something related to us has changes, it sends reload
 			 * message to us.
 			 */
-			assert(res->hr_role == HAST_ROLE_PRIMARY);
+			PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY);
 			primary_config_reload(res, nvin);
 			nv_add_int16(nvout, 0, "error");
 			break;
diff --git a/sbin/hastd/ebuf.c b/sbin/hastd/ebuf.c
index 0bccd181cc35..f5908a41a92b 100644
--- a/sbin/hastd/ebuf.c
+++ b/sbin/hastd/ebuf.c
@@ -32,15 +32,21 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 
-#include <assert.h>
 #include <errno.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <strings.h>
 #include <unistd.h>
 
+#include <pjdlog.h>
+
 #include "ebuf.h"
 
+#ifndef	PJDLOG_ASSERT
+#include <assert.h>
+#define	PJDLOG_ASSERT(...)	assert(__VA_ARGS__)
+#endif
+
 #define	EBUF_MAGIC	0xeb0f41c
 struct ebuf {
 	/* Magic to assert the caller uses valid structure. */
@@ -91,7 +97,7 @@ void
 ebuf_free(struct ebuf *eb)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	eb->eb_magic = 0;
 
@@ -103,7 +109,7 @@ int
 ebuf_add_head(struct ebuf *eb, const void *data, size_t size)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	if (size > (size_t)(eb->eb_used - eb->eb_start)) {
 		/*
@@ -113,7 +119,7 @@ ebuf_add_head(struct ebuf *eb, const void *data, size_t size)
 		if (ebuf_head_extend(eb, size) < 0)
 			return (-1);
 	}
-	assert(size <= (size_t)(eb->eb_used - eb->eb_start));
+	PJDLOG_ASSERT(size <= (size_t)(eb->eb_used - eb->eb_start));
 
 	eb->eb_size += size;
 	eb->eb_used -= size;
@@ -130,7 +136,7 @@ int
 ebuf_add_tail(struct ebuf *eb, const void *data, size_t size)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	if (size > (size_t)(eb->eb_end - (eb->eb_used + eb->eb_size))) {
 		/*
@@ -140,7 +146,8 @@ ebuf_add_tail(struct ebuf *eb, const void *data, size_t size)
 		if (ebuf_tail_extend(eb, size) < 0)
 			return (-1);
 	}
-	assert(size <= (size_t)(eb->eb_end - (eb->eb_used + eb->eb_size)));
+	PJDLOG_ASSERT(size <=
+	    (size_t)(eb->eb_end - (eb->eb_used + eb->eb_size)));
 
 	/*
 	 * If data is NULL the caller just wants to reserve space.
@@ -156,8 +163,8 @@ void
 ebuf_del_head(struct ebuf *eb, size_t size)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
-	assert(size <= eb->eb_size);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(size <= eb->eb_size);
 
 	eb->eb_used += size;
 	eb->eb_size -= size;
@@ -167,8 +174,8 @@ void
 ebuf_del_tail(struct ebuf *eb, size_t size)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
-	assert(size <= eb->eb_size);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(size <= eb->eb_size);
 
 	eb->eb_size -= size;
 }
@@ -180,7 +187,7 @@ void *
 ebuf_data(struct ebuf *eb, size_t *sizep)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	if (sizep != NULL)
 		*sizep = eb->eb_size;
@@ -194,7 +201,7 @@ size_t
 ebuf_size(struct ebuf *eb)
 {
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	return (eb->eb_size);
 }
@@ -208,7 +215,7 @@ ebuf_head_extend(struct ebuf *eb, size_t size)
 	unsigned char *newstart, *newused;
 	size_t newsize;
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	newsize = eb->eb_end - eb->eb_start + (PAGE_SIZE / 4) + size;
 
@@ -236,7 +243,7 @@ ebuf_tail_extend(struct ebuf *eb, size_t size)
 	unsigned char *newstart;
 	size_t newsize;
 
-	assert(eb != NULL && eb->eb_magic == EBUF_MAGIC);
+	PJDLOG_ASSERT(eb != NULL && eb->eb_magic == EBUF_MAGIC);
 
 	newsize = eb->eb_end - eb->eb_start + size + ((3 * PAGE_SIZE) / 4);
 
diff --git a/sbin/hastd/event.c b/sbin/hastd/event.c
index d772f692ba2e..d895f670177a 100644
--- a/sbin/hastd/event.c
+++ b/sbin/hastd/event.c
@@ -27,7 +27,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 
 #include "hast.h"
@@ -46,8 +45,8 @@ event_send(const struct hast_resource *res, int event)
 	struct nv *nvin, *nvout;
 	int error;
 
-	assert(res != NULL);
-	assert(event >= EVENT_MIN && event <= EVENT_MAX);
+	PJDLOG_ASSERT(res != NULL);
+	PJDLOG_ASSERT(event >= EVENT_MIN && event <= EVENT_MAX);
 
 	nvin = nvout = NULL;
 
@@ -89,7 +88,7 @@ event_recv(const struct hast_resource *res)
 	uint8_t event;
 	int error;
 
-	assert(res != NULL);
+	PJDLOG_ASSERT(res != NULL);
 
 	nvin = nvout = NULL;
 
diff --git a/sbin/hastd/hast.conf.5 b/sbin/hastd/hast.conf.5
index f53e6e3a69e5..1eb66e3202e5 100644
--- a/sbin/hastd/hast.conf.5
+++ b/sbin/hastd/hast.conf.5
@@ -63,17 +63,21 @@ checksum <algorithm>
 compression <algorithm>
 timeout <seconds>
 exec <path>
+metaflush "on" | "off"
+pidfile <path>
 
 on <node> {
 	# Node section
         control <addr>
         listen <addr>
+        pidfile <path>
 }
 
 on <node> {
 	# Node section
         control <addr>
         listen <addr>
+        pidfile <path>
 }
 
 resource <name> {
@@ -85,12 +89,14 @@ resource <name> {
 	local <path>
 	timeout <seconds>
 	exec <path>
+	metaflush "on" | "off"
 
 	on <node> {
 		# Resource-node section
 		name <name>
 		# Required
 		local <path>
+		metaflush "on" | "off"
 		# Required
 		remote <addr>
 		source <addr>
@@ -100,6 +106,7 @@ resource <name> {
 		name <name>
 		# Required
 		local <path>
+		metaflush "on" | "off"
 		# Required
 		remote <addr>
 		source <addr>
@@ -142,6 +149,14 @@ unix:///var/run/hastctl
 .Pp
 The default value is
 .Pa uds:///var/run/hastctl .
+.It Ic pidfile Aq path
+.Pp
+File in which to store the process ID of the main
+.Xr hastd 8
+process.
+.Pp
+The default value is
+.Pa /var/run/hastd.pid .
 .It Ic listen Aq addr
 .Pp
 Address to listen on in form of:
@@ -318,6 +333,25 @@ It can be one of:
 .Ar secondary ,
 .Ar primary .
 .Pp
+.It Ic metaflush on | off
+.Pp
+When set to
+.Va on ,
+flush write cache of the local provider after every metadata (activemap) update.
+Flushing write cache ensures that provider will not reorder writes and that
+metadata will be properly updated before real data is stored.
+If the local provider does not support flushing write cache (it returns
+.Er EOPNOTSUPP
+on the
+.Cm BIO_FLUSH
+request),
+.Nm hastd
+will disable
+.Ic metaflush
+automatically.
+The default value is
+.Va on .
+.Pp
 .It Ic name Aq name
 .Pp
 GEOM provider name that will appear as
diff --git a/sbin/hastd/hast.h b/sbin/hastd/hast.h
index a62b63a263f6..94e572426dd8 100644
--- a/sbin/hastd/hast.h
+++ b/sbin/hastd/hast.h
@@ -111,11 +111,13 @@ struct hastd_listen {
 
 struct hastd_config {
 	/* Address to communicate with hastctl(8). */
-	char	 hc_controladdr[HAST_ADDRSIZE];
+	char	hc_controladdr[HAST_ADDRSIZE];
 	/* Protocol-specific data. */
 	struct proto_conn *hc_controlconn;
 	/* Incoming control connection. */
 	struct proto_conn *hc_controlin;
+	/* PID file path. */
+	char	hc_pidfile[PATH_MAX];
 	/* List of addresses to listen on. */
 	TAILQ_HEAD(, hastd_listen) hc_listen;
 	/* List of resources. */
@@ -167,6 +169,10 @@ struct hast_resource {
 	off_t	hr_local_mediasize;
 	/* Sector size of local provider. */
 	unsigned int hr_local_sectorsize;
+	/* Is flushing write cache supported by the local provider? */
+	bool	hr_localflush;
+	/* Flush write cache on metadata updates? */
+	int	hr_metaflush;
 
 	/* Descriptor for /dev/ggctl communication. */
 	int	hr_ggatefd;
diff --git a/sbin/hastd/hast_proto.c b/sbin/hastd/hast_proto.c
index 89568914f5f7..439b0f31940f 100644
--- a/sbin/hastd/hast_proto.c
+++ b/sbin/hastd/hast_proto.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/endian.h>
 
-#include <assert.h>
 #include <errno.h>
 #include <strings.h>
 
@@ -158,7 +157,7 @@ hast_proto_recv_hdr(const struct proto_conn *conn, struct nv **nvp)
 	if (ebuf_add_tail(eb, NULL, hdr.size) < 0)
 		goto fail;
 	hptr = ebuf_data(eb, NULL);
-	assert(hptr != NULL);
+	PJDLOG_ASSERT(hptr != NULL);
 	if (proto_recv(conn, hptr, hdr.size) < 0)
 		goto fail;
 	nv = nv_ntoh(eb);
@@ -183,8 +182,8 @@ hast_proto_recv_data(const struct hast_resource *res, struct proto_conn *conn,
 	void *dptr;
 	int ret;
 
-	assert(data != NULL);
-	assert(size > 0);
+	PJDLOG_ASSERT(data != NULL);
+	PJDLOG_ASSERT(size > 0);
 
 	ret = -1;
 	freedata = false;
diff --git a/sbin/hastd/hastd.c b/sbin/hastd/hastd.c
index 6518f0cf57fd..fdb1c5b99081 100644
--- a/sbin/hastd/hastd.c
+++ b/sbin/hastd/hastd.c
@@ -386,6 +386,12 @@ resource_needs_restart(const struct hast_resource *res0,
 			return (true);
 		if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
 			return (true);
+		/*
+		 * When metaflush has changed we don't really need restart,
+		 * but it is just easier this way.
+		 */
+		if (res0->hr_metaflush != res1->hr_metaflush)
+			return (true);
 	}
 	return (false);
 }
@@ -416,6 +422,8 @@ resource_needs_reload(const struct hast_resource *res0,
 		return (true);
 	if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
 		return (true);
+	if (res0->hr_metaflush != res1->hr_metaflush)
+		return (true);
 	return (false);
 }
 
@@ -436,6 +444,7 @@ resource_reload(const struct hast_resource *res)
 	nv_add_int32(nvout, (int32_t)res->hr_compression, "compression");
 	nv_add_int32(nvout, (int32_t)res->hr_timeout, "timeout");
 	nv_add_string(nvout, res->hr_exec, "exec");
+	nv_add_int32(nvout, (int32_t)res->hr_metaflush, "metaflush");
 	if (nv_error(nvout) != 0) {
 		nv_free(nvout);
 		pjdlog_error("Unable to allocate header for reload message.");
@@ -467,11 +476,15 @@ hastd_reload(void)
 	struct hastd_config *newcfg;
 	struct hast_resource *nres, *cres, *tres;
 	struct hastd_listen *nlst, *clst;
+	struct pidfh *newpfh;
 	unsigned int nlisten;
 	uint8_t role;
+	pid_t otherpid;
 
 	pjdlog_info("Reloading configuration...");
 
+	newpfh = NULL;
+
 	newcfg = yy_config_parse(cfgpath, false);
 	if (newcfg == NULL)
 		goto failed;
@@ -515,6 +528,31 @@ hastd_reload(void)
 		pjdlog_error("No addresses to listen on.");
 		goto failed;
 	}
+	/*
+	 * Check if pidfile's path has changed.
+	 */
+	if (strcmp(cfg->hc_pidfile, newcfg->hc_pidfile) != 0) {
+		newpfh = pidfile_open(newcfg->hc_pidfile, 0600, &otherpid);
+		if (newpfh == NULL) {
+			if (errno == EEXIST) {
+				pjdlog_errno(LOG_WARNING,
+				    "Another hastd is already running, pidfile: %s, pid: %jd.",
+				    newcfg->hc_pidfile, (intmax_t)otherpid);
+			} else {
+				pjdlog_errno(LOG_WARNING,
+				    "Unable to open or create pidfile %s",
+				    newcfg->hc_pidfile);
+			}
+		} else if (pidfile_write(newpfh) < 0) {
+			/* Write PID to a file. */
+			pjdlog_errno(LOG_WARNING,
+			    "Unable to write PID to file %s",
+			    newcfg->hc_pidfile);
+		} else {
+			pjdlog_debug(1, "PID stored in %s.",
+			    newcfg->hc_pidfile);
+		}
+	}
 
 	/* No failures from now on. */
 
@@ -530,6 +568,13 @@ hastd_reload(void)
 		strlcpy(cfg->hc_controladdr, newcfg->hc_controladdr,
 		    sizeof(cfg->hc_controladdr));
 	}
+	/*
+	 * Switch to new pidfile.
+	 */
+	(void)pidfile_remove(pfh);
+	pfh = newpfh;
+	(void)strlcpy(cfg->hc_pidfile, newcfg->hc_pidfile,
+	    sizeof(cfg->hc_pidfile));
 	/*
 	 * Switch to new listen addresses. Close all that were removed.
 	 */
@@ -591,12 +636,13 @@ hastd_reload(void)
 	 * recreating it.
 	 *
 	 * We do just reload (send SIGHUP to worker process) if we act as
-	 * PRIMARY, but only if remote address, replication mode, timeout or
-	 * execution path has changed. For those, there is no need to restart
-	 * worker process.
+	 * PRIMARY, but only if remote address, source address, replication
+	 * mode, timeout, execution path or metaflush has changed.
+	 * For those, there is no need to restart worker process.
 	 * If PRIMARY receives SIGHUP, it will reconnect if remote address or
-	 * replication mode has changed or simply set new timeout if only
-	 * timeout has changed.
+	 * source address has changed or it will set new timeout if only timeout
+	 * has changed or it will update metaflush if only metaflush has
+	 * changed.
 	 */
 	TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) {
 		TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) {
@@ -627,6 +673,7 @@ hastd_reload(void)
 			cres->hr_timeout = nres->hr_timeout;
 			strlcpy(cres->hr_exec, nres->hr_exec,
 			    sizeof(cres->hr_exec));
+			cres->hr_metaflush = nres->hr_metaflush;
 			if (cres->hr_workerpid != 0)
 				resource_reload(cres);
 		}
@@ -655,6 +702,8 @@ hastd_reload(void)
 		}
 		yy_config_free(newcfg);
 	}
+	if (newpfh != NULL)
+		(void)pidfile_remove(newpfh);
 	pjdlog_warning("Configuration not reloaded.");
 }
 
@@ -1112,7 +1161,7 @@ main(int argc, char *argv[])
 
 	foreground = false;
 	debuglevel = 0;
-	pidfile = HASTD_PIDFILE;
+	pidfile = NULL;
 
 	for (;;) {
 		int ch;
@@ -1146,20 +1195,43 @@ main(int argc, char *argv[])
 
 	g_gate_load();
 
-	pfh = pidfile_open(pidfile, 0600, &otherpid);
-	if (pfh == NULL) {
-		if (errno == EEXIST) {
-			pjdlog_exitx(EX_TEMPFAIL,
-			    "Another hastd is already running, pid: %jd.",
-			    (intmax_t)otherpid);
+	/*
+	 * When path to the configuration file is relative, obtain full path,
+	 * so we can always find the file, even after daemonizing and changing
+	 * working directory to /.
+	 */
+	if (cfgpath[0] != '/') {
+		const char *newcfgpath;
+
+		newcfgpath = realpath(cfgpath, NULL);
+		if (newcfgpath == NULL) {
+			pjdlog_exit(EX_CONFIG,
+			    "Unable to obtain full path of %s", cfgpath);
 		}
-		/* If we cannot create pidfile from other reasons, only warn. */
-		pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile");
+		cfgpath = newcfgpath;
 	}
 
 	cfg = yy_config_parse(cfgpath, true);
 	PJDLOG_ASSERT(cfg != NULL);
 
+	if (pidfile != NULL) {
+		if (strlcpy(cfg->hc_pidfile, pidfile,
+		    sizeof(cfg->hc_pidfile)) >= sizeof(cfg->hc_pidfile)) {
+			pjdlog_exitx(EX_CONFIG, "Pidfile path is too long.");
+		}
+	}
+	pfh = pidfile_open(cfg->hc_pidfile, 0600, &otherpid);
+	if (pfh == NULL) {
+		if (errno == EEXIST) {
+			pjdlog_exitx(EX_TEMPFAIL,
+			    "Another hastd is already running, pidfile: %s, pid: %jd.",
+			    cfg->hc_pidfile, (intmax_t)otherpid);
+		}
+		/* If we cannot create pidfile for other reasons, only warn. */
+		pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile %s",
+		    cfg->hc_pidfile);
+	}
+
 	/*
 	 * Restore default actions for interesting signals in case parent
 	 * process (like init(8)) decided to ignore some of them (like SIGHUP).
@@ -1207,7 +1279,10 @@ main(int argc, char *argv[])
 		/* Write PID to a file. */
 		if (pidfile_write(pfh) < 0) {
 			pjdlog_errno(LOG_WARNING,
-			    "Unable to write PID to a file");
+			    "Unable to write PID to a file %s",
+			    cfg->hc_pidfile);
+		} else {
+			pjdlog_debug(1, "PID stored in %s.", cfg->hc_pidfile);
 		}
 	}
 
diff --git a/sbin/hastd/hooks.c b/sbin/hastd/hooks.c
index b161c0c0442b..60d48d1384fd 100644
--- a/sbin/hastd/hooks.c
+++ b/sbin/hastd/hooks.c
@@ -35,7 +35,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/wait.h>
 
-#include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
@@ -138,7 +137,7 @@ void
 hook_init(void)
 {
 
-	assert(!hooks_initialized);
+	PJDLOG_ASSERT(!hooks_initialized);
 
 	mtx_init(&hookprocs_lock);
 	TAILQ_INIT(&hookprocs);
@@ -150,12 +149,12 @@ hook_fini(void)
 {
 	struct hookproc *hp;
 
-	assert(hooks_initialized);
+	PJDLOG_ASSERT(hooks_initialized);
 
 	mtx_lock(&hookprocs_lock);
 	while ((hp = TAILQ_FIRST(&hookprocs)) != NULL) {
-		assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
-		assert(hp->hp_pid > 0);
+		PJDLOG_ASSERT(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+		PJDLOG_ASSERT(hp->hp_pid > 0);
 
 		hook_remove(hp);
 		hook_free(hp);
@@ -201,8 +200,8 @@ static void
 hook_add(struct hookproc *hp, pid_t pid)
 {
 
-	assert(hp->hp_magic == HOOKPROC_MAGIC_ALLOCATED);
-	assert(hp->hp_pid == 0);
+	PJDLOG_ASSERT(hp->hp_magic == HOOKPROC_MAGIC_ALLOCATED);
+	PJDLOG_ASSERT(hp->hp_pid == 0);
 
 	hp->hp_pid = pid;
 	mtx_lock(&hookprocs_lock);
@@ -215,9 +214,9 @@ static void
 hook_remove(struct hookproc *hp)
 {
 
-	assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
-	assert(hp->hp_pid > 0);
-	assert(mtx_owned(&hookprocs_lock));
+	PJDLOG_ASSERT(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+	PJDLOG_ASSERT(hp->hp_pid > 0);
+	PJDLOG_ASSERT(mtx_owned(&hookprocs_lock));
 
 	TAILQ_REMOVE(&hookprocs, hp, hp_next);
 	hp->hp_magic = HOOKPROC_MAGIC_ALLOCATED;
@@ -227,8 +226,8 @@ static void
 hook_free(struct hookproc *hp)
 {
 
-	assert(hp->hp_magic == HOOKPROC_MAGIC_ALLOCATED);
-	assert(hp->hp_pid > 0);
+	PJDLOG_ASSERT(hp->hp_magic == HOOKPROC_MAGIC_ALLOCATED);
+	PJDLOG_ASSERT(hp->hp_pid > 0);
 
 	hp->hp_magic = 0;
 	free(hp);
@@ -239,11 +238,11 @@ hook_find(pid_t pid)
 {
 	struct hookproc *hp;
 
-	assert(mtx_owned(&hookprocs_lock));
+	PJDLOG_ASSERT(mtx_owned(&hookprocs_lock));
 
 	TAILQ_FOREACH(hp, &hookprocs, hp_next) {
-		assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
-		assert(hp->hp_pid > 0);
+		PJDLOG_ASSERT(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+		PJDLOG_ASSERT(hp->hp_pid > 0);
 
 		if (hp->hp_pid == pid)
 			break;
@@ -286,7 +285,7 @@ hook_check(void)
 	struct hookproc *hp, *hp2;
 	time_t now;
 
-	assert(hooks_initialized);
+	PJDLOG_ASSERT(hooks_initialized);
 
 	pjdlog_debug(2, "Checking hooks.");
 
@@ -296,8 +295,8 @@ hook_check(void)
 	now = time(NULL);
 	mtx_lock(&hookprocs_lock);
 	TAILQ_FOREACH_SAFE(hp, &hookprocs, hp_next, hp2) {
-		assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
-		assert(hp->hp_pid > 0);
+		PJDLOG_ASSERT(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+		PJDLOG_ASSERT(hp->hp_pid > 0);
 
 		/*
 		 * If process doesn't exists we somehow missed it.
@@ -347,7 +346,7 @@ hook_execv(const char *path, va_list ap)
 	sigset_t mask;
 	pid_t pid;
 
-	assert(hooks_initialized);
+	PJDLOG_ASSERT(hooks_initialized);
 
 	if (path == NULL || path[0] == '\0')
 		return;
@@ -359,7 +358,7 @@ hook_execv(const char *path, va_list ap)
 		if (args[ii] == NULL)
 			break;
 	}
-	assert(ii < sizeof(args) / sizeof(args[0]));
+	PJDLOG_ASSERT(ii < sizeof(args) / sizeof(args[0]));
 
 	hp = hook_alloc(path, args);
 	if (hp == NULL)
diff --git a/sbin/hastd/metadata.c b/sbin/hastd/metadata.c
index de21b53ef50d..ad6c75bbd3c8 100644
--- a/sbin/hastd/metadata.c
+++ b/sbin/hastd/metadata.c
@@ -30,7 +30,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <string.h>
@@ -100,7 +99,7 @@ metadata_read(struct hast_resource *res, bool openrw)
 		goto fail;
 	}
 	buf = ebuf_data(eb, NULL);
-	assert(buf != NULL);
+	PJDLOG_ASSERT(buf != NULL);
 	done = pread(res->hr_localfd, buf, METADATA_SIZE, 0);
 	if (done < 0 || done != METADATA_SIZE) {
 		rerrno = errno;
@@ -197,7 +196,7 @@ metadata_write(struct hast_resource *res)
 		nv_add_uint64(nv, res->hr_primary_localcnt, "localcnt");
 		nv_add_uint64(nv, res->hr_primary_remotecnt, "remotecnt");
 	} else /* if (res->hr_role == HAST_ROLE_SECONDARY) */ {
-		assert(res->hr_role == HAST_ROLE_SECONDARY);
+		PJDLOG_ASSERT(res->hr_role == HAST_ROLE_SECONDARY);
 		nv_add_uint64(nv, res->hr_secondary_localcnt, "localcnt");
 		nv_add_uint64(nv, res->hr_secondary_remotecnt, "remotecnt");
 	}
@@ -208,10 +207,10 @@ metadata_write(struct hast_resource *res)
 	}
 	res->hr_previous_role = res->hr_role;
 	eb = nv_hton(nv);
-	assert(eb != NULL);
+	PJDLOG_ASSERT(eb != NULL);
 	ptr = ebuf_data(eb, &size);
-	assert(ptr != NULL);
-	assert(size < METADATA_SIZE);
+	PJDLOG_ASSERT(ptr != NULL);
+	PJDLOG_ASSERT(size < METADATA_SIZE);
 	bcopy(ptr, buf, size);
 	done = pwrite(res->hr_localfd, buf, METADATA_SIZE, 0);
 	if (done < 0 || done != METADATA_SIZE) {
diff --git a/sbin/hastd/nv.c b/sbin/hastd/nv.c
index 1f994ee8b901..89c174ace0c4 100644
--- a/sbin/hastd/nv.c
+++ b/sbin/hastd/nv.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/endian.h>
 
-#include <assert.h>
 #include <bitstring.h>
 #include <errno.h>
 #include <stdarg.h>
@@ -44,7 +43,17 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 
 #include <ebuf.h>
-#include <nv.h>
+#include <pjdlog.h>
+
+#include "nv.h"
+
+#ifndef	PJDLOG_ASSERT
+#include <assert.h>
+#define	PJDLOG_ASSERT(...)	assert(__VA_ARGS__)
+#endif
+#ifndef	PJDLOG_ABORT
+#define	PJDLOG_ABORT(...)	abort()
+#endif
 
 #define	NV_TYPE_NONE		0
 
@@ -98,8 +107,8 @@ struct nvhdr {
 #define	NVH_SIZE(nvh)	(NVH_HSIZE(nvh) + roundup2(NVH_DSIZE(nvh), 8))
 
 #define	NV_CHECK(nv)	do {						\
-	assert((nv) != NULL);						\
-	assert((nv)->nv_magic == NV_MAGIC);				\
+	PJDLOG_ASSERT((nv) != NULL);					\
+	PJDLOG_ASSERT((nv)->nv_magic == NV_MAGIC);			\
 } while (0)
 
 static void nv_add(struct nv *nv, const unsigned char *value, size_t vsize,
@@ -200,7 +209,7 @@ nv_validate(struct nv *nv, size_t *extrap)
 	}
 
 	NV_CHECK(nv);
-	assert(nv->nv_error == 0);
+	PJDLOG_ASSERT(nv->nv_error == 0);
 
 	/* TODO: Check that names are unique? */
 
@@ -308,7 +317,7 @@ nv_validate(struct nv *nv, size_t *extrap)
 			}
 			break;
 		default:
-			assert(!"invalid condition");
+			PJDLOG_ABORT("invalid condition");
 		}
 		if (error != 0)
 			break;
@@ -338,7 +347,7 @@ nv_hton(struct nv *nv)
 	size_t size;
 
 	NV_CHECK(nv);
-	assert(nv->nv_error == 0);
+	PJDLOG_ASSERT(nv->nv_error == 0);
 
 	ptr = ebuf_data(nv->nv_ebuf, &size);
 	while (size > 0) {
@@ -346,9 +355,9 @@ nv_hton(struct nv *nv)
 		 * Minimum size at this point is size of nvhdr structure,
 		 * one character long name plus terminating '\0'.
 		 */
-		assert(size >= sizeof(*nvh) + 2);
+		PJDLOG_ASSERT(size >= sizeof(*nvh) + 2);
 		nvh = (struct nvhdr *)ptr;
-		assert(NVH_SIZE(nvh) <= size);
+		PJDLOG_ASSERT(NVH_SIZE(nvh) <= size);
 		nv_swap(nvh, false);
 		ptr += NVH_SIZE(nvh);
 		size -= NVH_SIZE(nvh);
@@ -367,7 +376,7 @@ nv_ntoh(struct ebuf *eb)
 	size_t extra;
 	int rerrno;
 
-	assert(eb != NULL);
+	PJDLOG_ASSERT(eb != NULL);
 
 	nv = malloc(sizeof(*nv));
 	if (nv == NULL)
@@ -494,8 +503,8 @@ nv_get_##type(struct nv *nv, const char *namefmt, ...)			\
 	va_end(nameap);							\
 	if (nvh == NULL)						\
 		return (0);						\
-	assert((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_HOST);	\
-	assert(sizeof(value) == nvh->nvh_dsize);			\
+	PJDLOG_ASSERT((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_HOST);\
+	PJDLOG_ASSERT(sizeof(value) == nvh->nvh_dsize);			\
 	bcopy(NVH_DATA(nvh), &value, sizeof(value));			\
 									\
 	return (value);							\
@@ -525,8 +534,8 @@ nv_get_##type##_array(struct nv *nv, size_t *sizep,			\
 	va_end(nameap);							\
 	if (nvh == NULL)						\
 		return (NULL);						\
-	assert((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_HOST);	\
-	assert((nvh->nvh_dsize % sizeof(type##_t)) == 0);		\
+	PJDLOG_ASSERT((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_HOST);\
+	PJDLOG_ASSERT((nvh->nvh_dsize % sizeof(type##_t)) == 0);	\
 	if (sizep != NULL)						\
 		*sizep = nvh->nvh_dsize / sizeof(type##_t);		\
 	return ((type##_t *)(void *)NVH_DATA(nvh));			\
@@ -555,11 +564,11 @@ nv_get_string(struct nv *nv, const char *namefmt, ...)
 	va_end(nameap);
 	if (nvh == NULL)
 		return (NULL);
-	assert((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_HOST);
-	assert(nvh->nvh_dsize >= 1);
+	PJDLOG_ASSERT((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_HOST);
+	PJDLOG_ASSERT(nvh->nvh_dsize >= 1);
 	str = NVH_DATA(nvh);
-	assert(str[nvh->nvh_dsize - 1] == '\0');
-	assert(strlen(str) == nvh->nvh_dsize - 1);
+	PJDLOG_ASSERT(str[nvh->nvh_dsize - 1] == '\0');
+	PJDLOG_ASSERT(strlen(str) == nvh->nvh_dsize - 1);
 	return (str);
 }
 
@@ -602,7 +611,7 @@ nv_assert(struct nv *nv, const char *namefmt, ...)
 	va_list nameap;
 
 	va_start(nameap, namefmt);
-	assert(nv_vexists(nv, namefmt, nameap));
+	PJDLOG_ASSERT(nv_vexists(nv, namefmt, nameap));
 	va_end(nameap);
 }
 
@@ -624,13 +633,13 @@ nv_dump(struct nv *nv)
 	}
 
 	NV_CHECK(nv);
-	assert(nv->nv_error == 0);
+	PJDLOG_ASSERT(nv->nv_error == 0);
 
 	ptr = ebuf_data(nv->nv_ebuf, &size);
 	while (size > 0) {
-		assert(size >= sizeof(*nvh) + 2);
+		PJDLOG_ASSERT(size >= sizeof(*nvh) + 2);
 		nvh = (struct nvhdr *)ptr;
-		assert(size >= NVH_SIZE(nvh));
+		PJDLOG_ASSERT(size >= NVH_SIZE(nvh));
 		swap = ((nvh->nvh_type & NV_ORDER_MASK) == NV_ORDER_NETWORK);
 		dsize = NVH_DSIZE(nvh);
 		data = NVH_DATA(nvh);
@@ -734,7 +743,7 @@ nv_dump(struct nv *nv)
 			printf("(string): %s", (char *)data);
 			break;
 		default:
-			assert(!"invalid condition");
+			PJDLOG_ABORT("invalid condition");
 		}
 		printf("\n");
 		ptr += NVH_SIZE(nvh);
@@ -776,7 +785,7 @@ nv_add(struct nv *nv, const unsigned char *value, size_t vsize, int type,
 
 	/* Add header first. */
 	if (ebuf_add_tail(nv->nv_ebuf, nvh, NVH_HSIZE(nvh)) < 0) {
-		assert(errno != 0);
+		PJDLOG_ASSERT(errno != 0);
 		if (nv->nv_error == 0)
 			nv->nv_error = errno;
 		free(nvh);
@@ -785,7 +794,7 @@ nv_add(struct nv *nv, const unsigned char *value, size_t vsize, int type,
 	free(nvh);
 	/* Add the actual data. */
 	if (ebuf_add_tail(nv->nv_ebuf, value, vsize) < 0) {
-		assert(errno != 0);
+		PJDLOG_ASSERT(errno != 0);
 		if (nv->nv_error == 0)
 			nv->nv_error = errno;
 		return;
@@ -794,9 +803,9 @@ nv_add(struct nv *nv, const unsigned char *value, size_t vsize, int type,
 	vsize = roundup2(vsize, 8) - vsize;
 	if (vsize == 0)
 		return;
-	assert(vsize > 0 && vsize <= sizeof(align));
+	PJDLOG_ASSERT(vsize > 0 && vsize <= sizeof(align));
 	if (ebuf_add_tail(nv->nv_ebuf, align, vsize) < 0) {
-		assert(errno != 0);
+		PJDLOG_ASSERT(errno != 0);
 		if (nv->nv_error == 0)
 			nv->nv_error = errno;
 		return;
@@ -811,7 +820,7 @@ nv_addv(struct nv *nv, const unsigned char *value, size_t vsize, int type,
 	size_t namesize;
 
 	namesize = vsnprintf(name, sizeof(name), namefmt, nameap);
-	assert(namesize > 0 && namesize < sizeof(name));
+	PJDLOG_ASSERT(namesize > 0 && namesize < sizeof(name));
 
 	nv_add(nv, value, vsize, type, name);
 }
@@ -832,14 +841,14 @@ nv_find(struct nv *nv, int type, const char *namefmt, va_list nameap)
 	NV_CHECK(nv);
 
 	namesize = vsnprintf(name, sizeof(name), namefmt, nameap);
-	assert(namesize > 0 && namesize < sizeof(name));
+	PJDLOG_ASSERT(namesize > 0 && namesize < sizeof(name));
 	namesize++;
 
 	ptr = ebuf_data(nv->nv_ebuf, &size);
 	while (size > 0) {
-		assert(size >= sizeof(*nvh) + 2);
+		PJDLOG_ASSERT(size >= sizeof(*nvh) + 2);
 		nvh = (struct nvhdr *)ptr;
-		assert(size >= NVH_SIZE(nvh));
+		PJDLOG_ASSERT(size >= NVH_SIZE(nvh));
 		nv_swap(nvh, true);
 		if (strcmp(nvh->nvh_name, name) == 0) {
 			if (type != NV_TYPE_NONE &&
@@ -927,7 +936,7 @@ nv_swap(struct nvhdr *nvh, bool tohost)
 					    le64toh(*(uint64_t *)(void *)p);
 					break;
 				default:
-					assert(!"invalid condition");
+					PJDLOG_ABORT("invalid condition");
 				}
 			} else {
 				switch (vsize) {
@@ -944,7 +953,7 @@ nv_swap(struct nvhdr *nvh, bool tohost)
 					    htole64(*(uint64_t *)(void *)p);
 					break;
 				default:
-					assert(!"invalid condition");
+					PJDLOG_ABORT("invalid condition");
 				}
 			}
 		}
@@ -952,6 +961,6 @@ nv_swap(struct nvhdr *nvh, bool tohost)
 	case NV_TYPE_STRING:
 		break;
 	default:
-		assert(!"unrecognized type");
+		PJDLOG_ABORT("unrecognized type");
 	}
 }
diff --git a/sbin/hastd/parse.y b/sbin/hastd/parse.y
index 6e8351d243d5..6174a2bb312e 100644
--- a/sbin/hastd/parse.y
+++ b/sbin/hastd/parse.y
@@ -38,7 +38,6 @@
 
 #include <arpa/inet.h>
 
-#include <assert.h>
 #include <err.h>
 #include <errno.h>
 #include <stdio.h>
@@ -61,6 +60,7 @@ static struct hast_resource *curres;
 static bool mynode, hadmynode;
 
 static char depth0_control[HAST_ADDRSIZE];
+static char depth0_pidfile[PATH_MAX];
 static char depth0_listen_tcp4[HAST_ADDRSIZE];
 static char depth0_listen_tcp6[HAST_ADDRSIZE];
 static TAILQ_HEAD(, hastd_listen) depth0_listen;
@@ -69,9 +69,11 @@ static int depth0_checksum;
 static int depth0_compression;
 static int depth0_timeout;
 static char depth0_exec[PATH_MAX];
+static int depth0_metaflush;
 
 static char depth1_provname[PATH_MAX];
 static char depth1_localpath[PATH_MAX];
+static int depth1_metaflush;
 
 extern void yyrestart(FILE *);
 
@@ -192,12 +194,14 @@ yy_config_parse(const char *config, bool exitonerror)
 	depth0_checksum = HAST_CHECKSUM_NONE;
 	depth0_compression = HAST_COMPRESSION_HOLE;
 	strlcpy(depth0_control, HAST_CONTROL, sizeof(depth0_control));
+	strlcpy(depth0_pidfile, HASTD_PIDFILE, sizeof(depth0_pidfile));
 	TAILQ_INIT(&depth0_listen);
 	strlcpy(depth0_listen_tcp4, HASTD_LISTEN_TCP4,
 	    sizeof(depth0_listen_tcp4));
 	strlcpy(depth0_listen_tcp6, HASTD_LISTEN_TCP6,
 	    sizeof(depth0_listen_tcp6));
 	depth0_exec[0] = '\0';
+	depth0_metaflush = 1;
 
 	lconfig = calloc(1, sizeof(*lconfig));
 	if (lconfig == NULL) {
@@ -236,6 +240,10 @@ yy_config_parse(const char *config, bool exitonerror)
 		strlcpy(lconfig->hc_controladdr, depth0_control,
 		    sizeof(lconfig->hc_controladdr));
 	}
+	if (lconfig->hc_pidfile[0] == '\0') {
+		strlcpy(lconfig->hc_pidfile, depth0_pidfile,
+		    sizeof(lconfig->hc_pidfile));
+	}
 	if (!TAILQ_EMPTY(&depth0_listen))
 		TAILQ_CONCAT(&lconfig->hc_listen, &depth0_listen, hl_next);
 	if (TAILQ_EMPTY(&lconfig->hc_listen)) {
@@ -282,9 +290,9 @@ yy_config_parse(const char *config, bool exitonerror)
 		}
 	}
 	TAILQ_FOREACH(curres, &lconfig->hc_resources, hr_next) {
-		assert(curres->hr_provname[0] != '\0');
-		assert(curres->hr_localpath[0] != '\0');
-		assert(curres->hr_remoteaddr[0] != '\0');
+		PJDLOG_ASSERT(curres->hr_provname[0] != '\0');
+		PJDLOG_ASSERT(curres->hr_localpath[0] != '\0');
+		PJDLOG_ASSERT(curres->hr_remoteaddr[0] != '\0');
 
 		if (curres->hr_replication == -1) {
 			/*
@@ -329,6 +337,13 @@ yy_config_parse(const char *config, bool exitonerror)
 			strlcpy(curres->hr_exec, depth0_exec,
 			    sizeof(curres->hr_exec));
 		}
+		if (curres->hr_metaflush == -1) {
+			/*
+			 * Metaflush is not set at resource-level.
+			 * Use global or default setting.
+			 */
+			curres->hr_metaflush = depth0_metaflush;
+		}
 	}
 
 	return (lconfig);
@@ -356,8 +371,8 @@ yy_config_free(struct hastd_config *config)
 }
 %}
 
-%token CONTROL LISTEN PORT REPLICATION CHECKSUM COMPRESSION
-%token TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE SOURCE ON
+%token CONTROL PIDFILE LISTEN PORT REPLICATION CHECKSUM COMPRESSION METAFLUSH
+%token TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE SOURCE ON OFF
 %token FULLSYNC MEMSYNC ASYNC NONE CRC32 SHA256 HOLE LZF
 %token NUM STR OB CB
 
@@ -365,6 +380,7 @@ yy_config_free(struct hastd_config *config)
 %type <num> replication_type
 %type <num> checksum_type
 %type <num> compression_type
+%type <num> boolean
 
 %union
 {
@@ -385,6 +401,8 @@ statements:
 statement:
 	control_statement
 	|
+	pidfile_statement
+	|
 	listen_statement
 	|
 	replication_statement
@@ -397,6 +415,8 @@ statement:
 	|
 	exec_statement
 	|
+	metaflush_statement
+	|
 	node_statement
 	|
 	resource_statement
@@ -426,7 +446,37 @@ control_statement:	CONTROL STR
 			}
 			break;
 		default:
-			assert(!"control at wrong depth level");
+			PJDLOG_ABORT("control at wrong depth level");
+		}
+		free($2);
+	}
+	;
+
+pidfile_statement:	PIDFILE STR
+	{
+		switch (depth) {
+		case 0:
+			if (strlcpy(depth0_pidfile, $2,
+			    sizeof(depth0_pidfile)) >=
+			    sizeof(depth0_pidfile)) {
+				pjdlog_error("pidfile argument is too long.");
+				free($2);
+				return (1);
+			}
+			break;
+		case 1:
+			if (!mynode)
+				break;
+			if (strlcpy(lconfig->hc_pidfile, $2,
+			    sizeof(lconfig->hc_pidfile)) >=
+			    sizeof(lconfig->hc_pidfile)) {
+				pjdlog_error("pidfile argument is too long.");
+				free($2);
+				return (1);
+			}
+			break;
+		default:
+			PJDLOG_ABORT("pidfile at wrong depth level");
 		}
 		free($2);
 	}
@@ -460,7 +510,7 @@ listen_statement:	LISTEN STR
 				free(lst);
 			break;
 		default:
-			assert(!"listen at wrong depth level");
+			PJDLOG_ABORT("listen at wrong depth level");
 		}
 		free($2);
 	}
@@ -473,11 +523,11 @@ replication_statement:	REPLICATION replication_type
 			depth0_replication = $2;
 			break;
 		case 1:
-			if (curres != NULL)
-				curres->hr_replication = $2;
+			PJDLOG_ASSERT(curres != NULL);
+			curres->hr_replication = $2;
 			break;
 		default:
-			assert(!"replication at wrong depth level");
+			PJDLOG_ABORT("replication at wrong depth level");
 		}
 	}
 	;
@@ -497,11 +547,11 @@ checksum_statement:	CHECKSUM checksum_type
 			depth0_checksum = $2;
 			break;
 		case 1:
-			if (curres != NULL)
-				curres->hr_checksum = $2;
+			PJDLOG_ASSERT(curres != NULL);
+			curres->hr_checksum = $2;
 			break;
 		default:
-			assert(!"checksum at wrong depth level");
+			PJDLOG_ABORT("checksum at wrong depth level");
 		}
 	}
 	;
@@ -521,11 +571,11 @@ compression_statement:	COMPRESSION compression_type
 			depth0_compression = $2;
 			break;
 		case 1:
-			if (curres != NULL)
-				curres->hr_compression = $2;
+			PJDLOG_ASSERT(curres != NULL);
+			curres->hr_compression = $2;
 			break;
 		default:
-			assert(!"compression at wrong depth level");
+			PJDLOG_ABORT("compression at wrong depth level");
 		}
 	}
 	;
@@ -549,11 +599,11 @@ timeout_statement:	TIMEOUT NUM
 			depth0_timeout = $2;
 			break;
 		case 1:
-			if (curres != NULL)
-				curres->hr_timeout = $2;
+			PJDLOG_ASSERT(curres != NULL);
+			curres->hr_timeout = $2;
 			break;
 		default:
-			assert(!"timeout at wrong depth level");
+			PJDLOG_ABORT("timeout at wrong depth level");
 		}
 	}
 	;
@@ -570,8 +620,7 @@ exec_statement:		EXEC STR
 			}
 			break;
 		case 1:
-			if (curres == NULL)
-				break;
+			PJDLOG_ASSERT(curres != NULL);
 			if (strlcpy(curres->hr_exec, $2,
 			    sizeof(curres->hr_exec)) >=
 			    sizeof(curres->hr_exec)) {
@@ -581,12 +630,40 @@ exec_statement:		EXEC STR
 			}
 			break;
 		default:
-			assert(!"exec at wrong depth level");
+			PJDLOG_ABORT("exec at wrong depth level");
 		}
 		free($2);
 	}
 	;
 
+metaflush_statement:	METAFLUSH boolean
+	{
+		switch (depth) {
+		case 0:
+			depth0_metaflush = $2;
+			break;
+		case 1:
+			PJDLOG_ASSERT(curres != NULL);
+			depth1_metaflush = $2;
+			break;
+		case 2:
+			if (!mynode)
+				break;
+			PJDLOG_ASSERT(curres != NULL);
+			curres->hr_metaflush = $2;
+			break;
+		default:
+			PJDLOG_ABORT("metaflush at wrong depth level");
+		}
+	}
+	;
+
+boolean:
+	ON		{ $$ = 1; }
+	|
+	OFF		{ $$ = 0; }
+	;
+
 node_statement:		ON node_start OB node_entries CB
 	{
 		mynode = false;
@@ -605,7 +682,7 @@ node_start:	STR
 			mynode = true;
 			break;
 		default:
-			assert(!"invalid isitme() return value");
+			PJDLOG_ABORT("invalid isitme() return value");
 		}
 		free($1);
 	}
@@ -619,6 +696,8 @@ node_entries:
 node_entry:
 	control_statement
 	|
+	pidfile_statement
+	|
 	listen_statement
 	;
 
@@ -640,7 +719,7 @@ resource_statement:	RESOURCE resource_start OB resource_entries CB
 			}
 
 			/*
-			 * Let's see there are some resource-level settings
+			 * Let's see if there are some resource-level settings
 			 * that we can use for node-level settings.
 			 */
 			if (curres->hr_provname[0] == '\0' &&
@@ -662,6 +741,13 @@ resource_statement:	RESOURCE resource_start OB resource_entries CB
 				strlcpy(curres->hr_localpath, depth1_localpath,
 				    sizeof(curres->hr_localpath));
 			}
+			if (curres->hr_metaflush == -1 && depth1_metaflush != -1) {
+				/*
+				 * Metaflush is not set at node-level,
+				 * but is set at resource-level, use it.
+				 */
+				curres->hr_metaflush = depth1_metaflush;
+			}
 
 			/*
 			 * If provider name is not given, use resource name
@@ -715,6 +801,7 @@ resource_start:	STR
 		 */
 		depth1_provname[0] = '\0';
 		depth1_localpath[0] = '\0';
+		depth1_metaflush = -1;
 		hadmynode = false;
 
 		curres = calloc(1, sizeof(*curres));
@@ -741,6 +828,8 @@ resource_start:	STR
 		curres->hr_provname[0] = '\0';
 		curres->hr_localpath[0] = '\0';
 		curres->hr_localfd = -1;
+		curres->hr_localflush = true;
+		curres->hr_metaflush = -1;
 		curres->hr_remoteaddr[0] = '\0';
 		curres->hr_sourceaddr[0] = '\0';
 		curres->hr_ggateunit = -1;
@@ -763,6 +852,8 @@ resource_entry:
 	|
 	exec_statement
 	|
+	metaflush_statement
+	|
 	name_statement
 	|
 	local_statement
@@ -785,7 +876,7 @@ name_statement:		NAME STR
 		case 2:
 			if (!mynode)
 				break;
-			assert(curres != NULL);
+			PJDLOG_ASSERT(curres != NULL);
 			if (strlcpy(curres->hr_provname, $2,
 			    sizeof(curres->hr_provname)) >=
 			    sizeof(curres->hr_provname)) {
@@ -795,7 +886,7 @@ name_statement:		NAME STR
 			}
 			break;
 		default:
-			assert(!"name at wrong depth level");
+			PJDLOG_ABORT("name at wrong depth level");
 		}
 		free($2);
 	}
@@ -816,7 +907,7 @@ local_statement:	LOCAL STR
 		case 2:
 			if (!mynode)
 				break;
-			assert(curres != NULL);
+			PJDLOG_ASSERT(curres != NULL);
 			if (strlcpy(curres->hr_localpath, $2,
 			    sizeof(curres->hr_localpath)) >=
 			    sizeof(curres->hr_localpath)) {
@@ -826,7 +917,7 @@ local_statement:	LOCAL STR
 			}
 			break;
 		default:
-			assert(!"local at wrong depth level");
+			PJDLOG_ABORT("local at wrong depth level");
 		}
 		free($2);
 	}
@@ -851,7 +942,7 @@ resource_node_start:	STR
 				mynode = hadmynode = true;
 				break;
 			default:
-				assert(!"invalid isitme() return value");
+				PJDLOG_ABORT("invalid isitme() return value");
 			}
 		}
 		free($1);
@@ -871,13 +962,15 @@ resource_node_entry:
 	remote_statement
 	|
 	source_statement
+	|
+	metaflush_statement
 	;
 
 remote_statement:	REMOTE remote_str
 	{
-		assert(depth == 2);
+		PJDLOG_ASSERT(depth == 2);
 		if (mynode) {
-			assert(curres != NULL);
+			PJDLOG_ASSERT(curres != NULL);
 			if (strlcpy(curres->hr_remoteaddr, $2,
 			    sizeof(curres->hr_remoteaddr)) >=
 			    sizeof(curres->hr_remoteaddr)) {
@@ -898,9 +991,9 @@ remote_str:
 
 source_statement:	SOURCE STR
 	{
-		assert(depth == 2);
+		PJDLOG_ASSERT(depth == 2);
 		if (mynode) {
-			assert(curres != NULL);
+			PJDLOG_ASSERT(curres != NULL);
 			if (strlcpy(curres->hr_sourceaddr, $2,
 			    sizeof(curres->hr_sourceaddr)) >=
 			    sizeof(curres->hr_sourceaddr)) {
diff --git a/sbin/hastd/pjdlog.c b/sbin/hastd/pjdlog.c
index 16ecf4d9c8cc..bc4018facf4d 100644
--- a/sbin/hastd/pjdlog.c
+++ b/sbin/hastd/pjdlog.c
@@ -148,11 +148,14 @@ pjdlog_printf_render_sockaddr(struct __printf_io *io,
 void
 pjdlog_init(int mode)
 {
+	int saved_errno;
 
 	assert(pjdlog_initialized == PJDLOG_NEVER_INITIALIZED ||
 	    pjdlog_initialized == PJDLOG_NOT_INITIALIZED);
 	assert(mode == PJDLOG_MODE_STD || mode == PJDLOG_MODE_SYSLOG);
 
+	saved_errno = errno;
+
 	if (pjdlog_initialized == PJDLOG_NEVER_INITIALIZED) {
 		__use_xprintf = 1;
 		register_printf_render_std("T");
@@ -171,18 +174,25 @@ pjdlog_init(int mode)
 	bzero(pjdlog_prefix, sizeof(pjdlog_prefix));
 
 	pjdlog_initialized = PJDLOG_INITIALIZED;
+
+	errno = saved_errno;
 }
 
 void
 pjdlog_fini(void)
 {
+	int saved_errno;
 
 	assert(pjdlog_initialized == PJDLOG_INITIALIZED);
 
+	saved_errno = errno;
+
 	if (pjdlog_mode == PJDLOG_MODE_SYSLOG)
 		closelog();
 
 	pjdlog_initialized = PJDLOG_NOT_INITIALIZED;
+
+	errno = saved_errno;
 }
 
 /*
@@ -194,6 +204,7 @@ pjdlog_fini(void)
 void
 pjdlog_mode_set(int mode)
 {
+	int saved_errno;
 
 	assert(pjdlog_initialized == PJDLOG_INITIALIZED);
 	assert(mode == PJDLOG_MODE_STD || mode == PJDLOG_MODE_SYSLOG);
@@ -201,12 +212,16 @@ pjdlog_mode_set(int mode)
 	if (pjdlog_mode == mode)
 		return;
 
+	saved_errno = errno;
+
 	if (mode == PJDLOG_MODE_SYSLOG)
 		openlog(NULL, LOG_PID | LOG_NDELAY, LOG_DAEMON);
 	else /* if (mode == PJDLOG_MODE_STD) */
 		closelog();
 
 	pjdlog_mode = mode;
+
+	errno = saved_errno;
 }
 
 /*
@@ -270,11 +285,16 @@ pjdlog_prefix_set(const char *fmt, ...)
 void
 pjdlogv_prefix_set(const char *fmt, va_list ap)
 {
+	int saved_errno;
 
 	assert(pjdlog_initialized == PJDLOG_INITIALIZED);
 	assert(fmt != NULL);
 
+	saved_errno = errno;
+
 	vsnprintf(pjdlog_prefix, sizeof(pjdlog_prefix), fmt, ap);
+
+	errno = saved_errno;
 }
 
 /*
@@ -329,6 +349,7 @@ void
 pjdlogv_common(int loglevel, int debuglevel, int error, const char *fmt,
     va_list ap)
 {
+	int saved_errno;
 
 	assert(pjdlog_initialized == PJDLOG_INITIALIZED);
 	assert(loglevel == LOG_EMERG || loglevel == LOG_ALERT ||
@@ -342,6 +363,8 @@ pjdlogv_common(int loglevel, int debuglevel, int error, const char *fmt,
 	if (loglevel == LOG_DEBUG && debuglevel > pjdlog_debug_level)
 		return;
 
+	saved_errno = errno;
+
 	switch (pjdlog_mode) {
 	case PJDLOG_MODE_STD:
 	    {
@@ -398,6 +421,8 @@ pjdlogv_common(int loglevel, int debuglevel, int error, const char *fmt,
 	default:
 		assert(!"Invalid mode.");
 	}
+
+	errno = saved_errno;
 }
 
 /*
diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c
index dd414f8bbf43..21e888a84997 100644
--- a/sbin/hastd/primary.c
+++ b/sbin/hastd/primary.c
@@ -293,10 +293,20 @@ hast_activemap_flush(struct hast_resource *res)
 	PJDLOG_ASSERT((size % res->hr_local_sectorsize) == 0);
 	if (pwrite(res->hr_localfd, buf, size, METADATA_SIZE) !=
 	    (ssize_t)size) {
-		KEEP_ERRNO(pjdlog_errno(LOG_ERR,
-		    "Unable to flush activemap to disk"));
+		pjdlog_errno(LOG_ERR, "Unable to flush activemap to disk");
 		return (-1);
 	}
+	if (res->hr_metaflush == 1 && g_flush(res->hr_localfd) == -1) {
+		if (errno == EOPNOTSUPP) {
+			pjdlog_warning("The %s provider doesn't support flushing write cache. Disabling it.",
+			    res->hr_localpath);
+			res->hr_metaflush = 0;
+		} else {
+			pjdlog_errno(LOG_ERR,
+			    "Unable to flush disk cache on activemap update");
+			return (-1);
+		}
+	}
 	return (0);
 }
 
@@ -1203,7 +1213,7 @@ ggate_recv_thread(void *arg)
 				break;
 			}
 			pjdlog_debug(2,
-			    "ggate_recv: (%p) Moving request to the send queues.",
+			    "ggate_recv: (%p) Moving request to the send queue.",
 			    hio);
 			refcount_init(&hio->hio_countdown, ncomps);
 			for (ii = 0; ii < ncomps; ii++)
@@ -1294,8 +1304,15 @@ local_send_thread(void *arg)
 			}
 			break;
 		case BIO_FLUSH:
+			if (!res->hr_localflush) {
+				ret = -1;
+				errno = EOPNOTSUPP;
+				break;
+			}
 			ret = g_flush(res->hr_localfd);
 			if (ret < 0) {
+				if (errno == EOPNOTSUPP)
+					res->hr_localflush = false;
 				hio->hio_errors[ncomp] = errno;
 				reqlog(LOG_WARNING, 0, ggio,
 				    "Local request failed (%s): ",
@@ -1421,8 +1438,7 @@ remote_send_thread(void *arg)
 			length = 0;
 			break;
 		default:
-			PJDLOG_ASSERT(!"invalid condition");
-			abort();
+			PJDLOG_ABORT("invalid condition");
 		}
 		nv = nv_alloc();
 		nv_add_uint8(nv, cmd, "cmd");
@@ -1621,8 +1637,7 @@ remote_recv_thread(void *arg)
 		case BIO_FLUSH:
 			break;
 		default:
-			PJDLOG_ASSERT(!"invalid condition");
-			abort();
+			PJDLOG_ABORT("invalid condition");
 		}
 		hio->hio_errors[ncomp] = 0;
 		nv_free(nv);
@@ -2002,6 +2017,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 	nv_assert(nv, "compression");
 	nv_assert(nv, "timeout");
 	nv_assert(nv, "exec");
+	nv_assert(nv, "metaflush");
 
 	ncomps = HAST_NCOMPONENTS;
 
@@ -2012,6 +2028,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 #define MODIFIED_COMPRESSION	0x10
 #define MODIFIED_TIMEOUT	0x20
 #define MODIFIED_EXEC		0x40
+#define MODIFIED_METAFLUSH	0x80
 	modified = 0;
 
 	vstr = nv_get_string(nv, "remoteaddr");
@@ -2053,6 +2070,11 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 		strlcpy(gres->hr_exec, vstr, sizeof(gres->hr_exec));
 		modified |= MODIFIED_EXEC;
 	}
+	vint = nv_get_int32(nv, "metaflush");
+	if (gres->hr_metaflush != vint) {
+		gres->hr_metaflush = vint;
+		modified |= MODIFIED_METAFLUSH;
+	}
 
 	/*
 	 * Change timeout for connected sockets.
@@ -2102,6 +2124,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 #undef	MODIFIED_COMPRESSION
 #undef	MODIFIED_TIMEOUT
 #undef	MODIFIED_EXEC
+#undef	MODIFIED_METAFLUSH
 
 	pjdlog_info("Configuration reloaded successfully.");
 }
diff --git a/sbin/hastd/proto_tcp.c b/sbin/hastd/proto_tcp.c
index 8f0f157cf020..44eb13ba77a6 100644
--- a/sbin/hastd/proto_tcp.c
+++ b/sbin/hastd/proto_tcp.c
@@ -298,8 +298,7 @@ tcp_connect(void *ctx, int timeout)
 
 	flags = fcntl(tctx->tc_fd, F_GETFL);
 	if (flags == -1) {
-		KEEP_ERRNO(pjdlog_common(LOG_DEBUG, 1, errno,
-		    "fcntl(F_GETFL) failed"));
+		pjdlog_common(LOG_DEBUG, 1, errno, "fcntl(F_GETFL) failed");
 		return (errno);
 	}
 	/*
@@ -308,8 +307,8 @@ tcp_connect(void *ctx, int timeout)
 	 */
 	flags |= O_NONBLOCK;
 	if (fcntl(tctx->tc_fd, F_SETFL, flags) == -1) {
-		KEEP_ERRNO(pjdlog_common(LOG_DEBUG, 1, errno,
-		    "fcntl(F_SETFL, O_NONBLOCK) failed"));
+		pjdlog_common(LOG_DEBUG, 1, errno,
+		    "fcntl(F_SETFL, O_NONBLOCK) failed");
 		return (errno);
 	}
 
diff --git a/sbin/hastd/rangelock.c b/sbin/hastd/rangelock.c
index 02247d6e3b41..5ff1f19daeeb 100644
--- a/sbin/hastd/rangelock.c
+++ b/sbin/hastd/rangelock.c
@@ -32,13 +32,19 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/queue.h>
 
-#include <assert.h>
 #include <stdbool.h>
 #include <stdlib.h>
 #include <unistd.h>
 
+#include <pjdlog.h>
+
 #include "rangelock.h"
 
+#ifndef	PJDLOG_ASSERT
+#include <assert.h>
+#define	PJDLOG_ASSERT(...)	assert(__VA_ARGS__)
+#endif
+
 #define	RANGELOCKS_MAGIC	0x94310c
 struct rangelocks {
 	int	 rls_magic;		/* Magic value. */
@@ -56,7 +62,7 @@ rangelock_init(struct rangelocks **rlsp)
 {
 	struct rangelocks *rls;
 
-	assert(rlsp != NULL);
+	PJDLOG_ASSERT(rlsp != NULL);
 
 	rls = malloc(sizeof(*rls));
 	if (rls == NULL)
@@ -75,7 +81,7 @@ rangelock_free(struct rangelocks *rls)
 {
 	struct rlock *rl;
 
-	assert(rls->rls_magic == RANGELOCKS_MAGIC);
+	PJDLOG_ASSERT(rls->rls_magic == RANGELOCKS_MAGIC);
 
 	rls->rls_magic = 0;
 
@@ -91,7 +97,7 @@ rangelock_add(struct rangelocks *rls, off_t offset, off_t length)
 {
 	struct rlock *rl;
 
-	assert(rls->rls_magic == RANGELOCKS_MAGIC);
+	PJDLOG_ASSERT(rls->rls_magic == RANGELOCKS_MAGIC);
 
 	rl = malloc(sizeof(*rl));
 	if (rl == NULL)
@@ -107,13 +113,13 @@ rangelock_del(struct rangelocks *rls, off_t offset, off_t length)
 {
 	struct rlock *rl;
 
-	assert(rls->rls_magic == RANGELOCKS_MAGIC);
+	PJDLOG_ASSERT(rls->rls_magic == RANGELOCKS_MAGIC);
 
 	TAILQ_FOREACH(rl, &rls->rls_locks, rl_next) {
 		if (rl->rl_start == offset && rl->rl_end == offset + length)
 			break;
 	}
-	assert(rl != NULL);
+	PJDLOG_ASSERT(rl != NULL);
 	TAILQ_REMOVE(&rls->rls_locks, rl, rl_next);
 	free(rl);
 }
@@ -123,7 +129,7 @@ rangelock_islocked(struct rangelocks *rls, off_t offset, off_t length)
 {
 	struct rlock *rl;
 
-	assert(rls->rls_magic == RANGELOCKS_MAGIC);
+	PJDLOG_ASSERT(rls->rls_magic == RANGELOCKS_MAGIC);
 
 	TAILQ_FOREACH(rl, &rls->rls_locks, rl_next) {
 		if (rl->rl_start >= offset && rl->rl_start < offset + length)
diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c
index 7120d3da3832..c761143f9693 100644
--- a/sbin/hastd/secondary.c
+++ b/sbin/hastd/secondary.c
@@ -664,7 +664,7 @@ disk_thread(void *arg)
 	struct hast_resource *res = arg;
 	struct hio *hio;
 	ssize_t ret;
-	bool clear_activemap;
+	bool clear_activemap, logerror;
 
 	clear_activemap = true;
 
@@ -699,8 +699,10 @@ disk_thread(void *arg)
 			free(map);
 			clear_activemap = false;
 			pjdlog_debug(1, "Local activemap cleared.");
+			break;
 		}
 		reqlog(LOG_DEBUG, 2, -1, hio, "disk: (%p) Got request: ", hio);
+		logerror = true;
 		/* Handle the actual request. */
 		switch (hio->hio_cmd) {
 		case HIO_READ:
@@ -735,14 +737,23 @@ disk_thread(void *arg)
 				hio->hio_error = 0;
 			break;
 		case HIO_FLUSH:
+			if (!res->hr_localflush) {
+				ret = -1;
+				hio->hio_error = EOPNOTSUPP;
+				logerror = false;
+				break;
+			}
 			ret = g_flush(res->hr_localfd);
-			if (ret < 0)
+			if (ret < 0) {
+				if (errno == EOPNOTSUPP)
+					res->hr_localflush = false;
 				hio->hio_error = errno;
-			else
+			} else {
 				hio->hio_error = 0;
+			}
 			break;
 		}
-		if (hio->hio_error != 0) {
+		if (logerror && hio->hio_error != 0) {
 			reqlog(LOG_ERR, 0, hio->hio_error, hio,
 			    "Request failed: ");
 		}
@@ -791,8 +802,8 @@ send_thread(void *arg)
 			length = 0;
 			break;
 		default:
-			abort();
-			break;
+			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
+			    hio->hio_cmd);
 		}
 		if (hio->hio_error != 0)
 			nv_add_int16(nvout, hio->hio_error, "error");
diff --git a/sbin/hastd/subr.c b/sbin/hastd/subr.c
index 89ffda0fe2f3..45218bbdf27c 100644
--- a/sbin/hastd/subr.c
+++ b/sbin/hastd/subr.c
@@ -87,14 +87,13 @@ provinfo(struct hast_resource *res, bool dowrite)
 		res->hr_localfd = open(res->hr_localpath,
 		    dowrite ? O_RDWR : O_RDONLY);
 		if (res->hr_localfd < 0) {
-			KEEP_ERRNO(pjdlog_errno(LOG_ERR, "Unable to open %s",
-			    res->hr_localpath));
+			pjdlog_errno(LOG_ERR, "Unable to open %s",
+			    res->hr_localpath);
 			return (-1);
 		}
 	}
 	if (fstat(res->hr_localfd, &sb) < 0) {
-		KEEP_ERRNO(pjdlog_errno(LOG_ERR, "Unable to stat %s",
-		    res->hr_localpath));
+		pjdlog_errno(LOG_ERR, "Unable to stat %s", res->hr_localpath);
 		return (-1);
 	}
 	if (S_ISCHR(sb.st_mode)) {
@@ -103,16 +102,16 @@ provinfo(struct hast_resource *res, bool dowrite)
 		 */
 		if (ioctl(res->hr_localfd, DIOCGMEDIASIZE,
 		    &res->hr_local_mediasize) < 0) {
-			KEEP_ERRNO(pjdlog_errno(LOG_ERR,
+			pjdlog_errno(LOG_ERR,
 			    "Unable obtain provider %s mediasize",
-			    res->hr_localpath));
+			    res->hr_localpath);
 			return (-1);
 		}
 		if (ioctl(res->hr_localfd, DIOCGSECTORSIZE,
 		    &res->hr_local_sectorsize) < 0) {
-			KEEP_ERRNO(pjdlog_errno(LOG_ERR,
+			pjdlog_errno(LOG_ERR,
 			    "Unable obtain provider %s sectorsize",
-			    res->hr_localpath));
+			    res->hr_localpath);
 			return (-1);
 		}
 	} else if (S_ISREG(sb.st_mode)) {
@@ -169,8 +168,8 @@ drop_privs(struct hast_resource *res)
 	pw = getpwnam(HAST_USER);
 	if (pw == NULL) {
 		if (errno != 0) {
-			KEEP_ERRNO(pjdlog_errno(LOG_ERR,
-			    "Unable to find info about '%s' user", HAST_USER));
+			pjdlog_errno(LOG_ERR,
+			    "Unable to find info about '%s' user", HAST_USER);
 			return (-1);
 		} else {
 			pjdlog_error("'%s' user doesn't exist.", HAST_USER);
@@ -201,28 +200,27 @@ drop_privs(struct hast_resource *res)
 		pjdlog_errno(LOG_WARNING,
 		    "Unable to jail to directory to %s", pw->pw_dir);
 		if (chroot(pw->pw_dir) == -1) {
-			KEEP_ERRNO(pjdlog_errno(LOG_ERR,
+			pjdlog_errno(LOG_ERR,
 			    "Unable to change root directory to %s",
-			    pw->pw_dir));
+			    pw->pw_dir);
 			return (-1);
 		}
 	}
 	PJDLOG_VERIFY(chdir("/") == 0);
 	gidset[0] = pw->pw_gid;
 	if (setgroups(1, gidset) == -1) {
-		KEEP_ERRNO(pjdlog_errno(LOG_ERR,
-		    "Unable to set groups to gid %u",
-		    (unsigned int)pw->pw_gid));
+		pjdlog_errno(LOG_ERR, "Unable to set groups to gid %u",
+		    (unsigned int)pw->pw_gid);
 		return (-1);
 	}
 	if (setgid(pw->pw_gid) == -1) {
-		KEEP_ERRNO(pjdlog_errno(LOG_ERR, "Unable to set gid to %u",
-		    (unsigned int)pw->pw_gid));
+		pjdlog_errno(LOG_ERR, "Unable to set gid to %u",
+		    (unsigned int)pw->pw_gid);
 		return (-1);
 	}
 	if (setuid(pw->pw_uid) == -1) {
-		KEEP_ERRNO(pjdlog_errno(LOG_ERR, "Unable to set uid to %u",
-		    (unsigned int)pw->pw_uid));
+		pjdlog_errno(LOG_ERR, "Unable to set uid to %u",
+		    (unsigned int)pw->pw_uid);
 		return (-1);
 	}
 
diff --git a/sbin/hastd/synch.h b/sbin/hastd/synch.h
index cf388d7d0cd6..36e19278921d 100644
--- a/sbin/hastd/synch.h
+++ b/sbin/hastd/synch.h
@@ -32,20 +32,26 @@
 #ifndef	_SYNCH_H_
 #define	_SYNCH_H_
 
-#include <assert.h>
 #include <errno.h>
 #include <pthread.h>
 #include <pthread_np.h>
 #include <stdbool.h>
 #include <time.h>
 
+#include <pjdlog.h>
+
+#ifndef	PJDLOG_ASSERT
+#include <assert.h>
+#define	PJDLOG_ASSERT(...)	assert(__VA_ARGS__)
+#endif
+
 static __inline void
 mtx_init(pthread_mutex_t *lock)
 {
 	int error;
 
 	error = pthread_mutex_init(lock, NULL);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 mtx_destroy(pthread_mutex_t *lock)
@@ -53,7 +59,7 @@ mtx_destroy(pthread_mutex_t *lock)
 	int error;
 
 	error = pthread_mutex_destroy(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 mtx_lock(pthread_mutex_t *lock)
@@ -61,7 +67,7 @@ mtx_lock(pthread_mutex_t *lock)
 	int error;
 
 	error = pthread_mutex_lock(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline bool
 mtx_trylock(pthread_mutex_t *lock)
@@ -69,7 +75,7 @@ mtx_trylock(pthread_mutex_t *lock)
 	int error;
 
 	error = pthread_mutex_trylock(lock);
-	assert(error == 0 || error == EBUSY);
+	PJDLOG_ASSERT(error == 0 || error == EBUSY);
 	return (error == 0);
 }
 static __inline void
@@ -78,7 +84,7 @@ mtx_unlock(pthread_mutex_t *lock)
 	int error;
 
 	error = pthread_mutex_unlock(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline bool
 mtx_owned(pthread_mutex_t *lock)
@@ -93,7 +99,7 @@ rw_init(pthread_rwlock_t *lock)
 	int error;
 
 	error = pthread_rwlock_init(lock, NULL);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 rw_destroy(pthread_rwlock_t *lock)
@@ -101,7 +107,7 @@ rw_destroy(pthread_rwlock_t *lock)
 	int error;
 
 	error = pthread_rwlock_destroy(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 rw_rlock(pthread_rwlock_t *lock)
@@ -109,7 +115,7 @@ rw_rlock(pthread_rwlock_t *lock)
 	int error;
 
 	error = pthread_rwlock_rdlock(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 rw_wlock(pthread_rwlock_t *lock)
@@ -117,7 +123,7 @@ rw_wlock(pthread_rwlock_t *lock)
 	int error;
 
 	error = pthread_rwlock_wrlock(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 rw_unlock(pthread_rwlock_t *lock)
@@ -125,7 +131,7 @@ rw_unlock(pthread_rwlock_t *lock)
 	int error;
 
 	error = pthread_rwlock_unlock(lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 
 static __inline void
@@ -135,13 +141,13 @@ cv_init(pthread_cond_t *cv)
 	int error;
 
 	error = pthread_condattr_init(&attr);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 	error = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 	error = pthread_cond_init(cv, &attr);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 	error = pthread_condattr_destroy(&attr);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 cv_wait(pthread_cond_t *cv, pthread_mutex_t *lock)
@@ -149,7 +155,7 @@ cv_wait(pthread_cond_t *cv, pthread_mutex_t *lock)
 	int error;
 
 	error = pthread_cond_wait(cv, lock);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline bool
 cv_timedwait(pthread_cond_t *cv, pthread_mutex_t *lock, int timeout)
@@ -163,10 +169,10 @@ cv_timedwait(pthread_cond_t *cv, pthread_mutex_t *lock, int timeout)
 	}
 
         error = clock_gettime(CLOCK_MONOTONIC, &ts);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 	ts.tv_sec += timeout;
 	error = pthread_cond_timedwait(cv, lock, &ts);
-	assert(error == 0 || error == ETIMEDOUT);
+	PJDLOG_ASSERT(error == 0 || error == ETIMEDOUT);
 	return (error == ETIMEDOUT);
 }
 static __inline void
@@ -175,7 +181,7 @@ cv_signal(pthread_cond_t *cv)
 	int error;
 
 	error = pthread_cond_signal(cv);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 static __inline void
 cv_broadcast(pthread_cond_t *cv)
@@ -183,6 +189,6 @@ cv_broadcast(pthread_cond_t *cv)
 	int error;
 
 	error = pthread_cond_broadcast(cv);
-	assert(error == 0);
+	PJDLOG_ASSERT(error == 0);
 }
 #endif	/* !_SYNCH_H_ */
diff --git a/sbin/hastd/token.l b/sbin/hastd/token.l
index 67c1e130e69d..0d1507563eed 100644
--- a/sbin/hastd/token.l
+++ b/sbin/hastd/token.l
@@ -46,6 +46,7 @@ int lineno;
 
 %%
 control			{ DP; return CONTROL; }
+pidfile			{ DP; return PIDFILE; }
 listen			{ DP; return LISTEN; }
 port			{ DP; return PORT; }
 replication		{ DP; return REPLICATION; }
@@ -53,12 +54,14 @@ checksum		{ DP; return CHECKSUM; }
 compression		{ DP; return COMPRESSION; }
 timeout			{ DP; return TIMEOUT; }
 exec			{ DP; return EXEC; }
+metaflush		{ DP; return METAFLUSH; }
 resource		{ DP; return RESOURCE; }
 name			{ DP; return NAME; }
 local			{ DP; return LOCAL; }
 remote			{ DP; return REMOTE; }
 source			{ DP; return SOURCE; }
 on			{ DP; return ON; }
+off			{ DP; return OFF; }
 fullsync		{ DP; return FULLSYNC; }
 memsync			{ DP; return MEMSYNC; }
 async			{ DP; return ASYNC; }
diff --git a/sbin/ifconfig/ifconfig.8 b/sbin/ifconfig/ifconfig.8
index cec32c58bcec..280baaa3fd56 100644
--- a/sbin/ifconfig/ifconfig.8
+++ b/sbin/ifconfig/ifconfig.8
@@ -28,7 +28,7 @@
 .\"     From: @(#)ifconfig.8	8.3 (Berkeley) 1/5/94
 .\" $FreeBSD$
 .\"
-.Dd July 3, 2011
+.Dd September 13, 2011
 .Dt IFCONFIG 8
 .Os
 .Sh NAME
@@ -626,12 +626,37 @@ is needed for them:
 .Bl -tag -width indent
 .It Cm accept_rtadv
 Set a flag to enable accepting ICMPv6 Router Advertisement messages.
+The
+.Xr sysctl 8
+variable
+.Va net.inet6.ip6.accept_rtadv
+controls whether this flag is set by default or not.
 .It Cm -accept_rtadv
 Clear a flag
 .Cm accept_rtadv .
+.It Cm no_radr
+Set a flag to control whether routers from which the system accepts
+Router Advertisement messages will be added to the Default Router List
+or not.
+When the
+.Cm accept_rtadv
+flag is disabled, this flag has no effect.
+The
+.Xr sysctl 8
+variable
+.Va net.inet6.ip6.no_radr
+controls whether this flag is set by default or not.
+.It Cm -no_radr
+Clear a flag
+.Cm no_radr .
 .It Cm auto_linklocal
 Set a flag to perform automatic link-local address configuration when
 the interface becomes available.
+The
+.Xr sysctl 8
+variable
+.Va net.inet6.ip6.auto_linklocal
+controls whether this flag is set by default or not.
 .It Cm -auto_linklocal
 Clear a flag
 .Cm auto_linklocal .
diff --git a/sbin/init/init.8 b/sbin/init/init.8
index f8fefcebef1e..264d82286416 100644
--- a/sbin/init/init.8
+++ b/sbin/init/init.8
@@ -147,7 +147,7 @@ When this shell
 dies, either because the user logged out
 or an abnormal termination occurred (a signal),
 .Nm login
-records the logout in the the user accounting
+records the logout in the user accounting
 database (see
 .Xr getutxent 3)
 and terminates.
diff --git a/sbin/swapon/Makefile b/sbin/swapon/Makefile
index 74422811b4a5..6f8e8df2e5a6 100644
--- a/sbin/swapon/Makefile
+++ b/sbin/swapon/Makefile
@@ -8,7 +8,6 @@ LINKS+=	${BINDIR}/swapon ${BINDIR}/swapctl
 MLINKS=	swapon.8 swapoff.8
 MLINKS+=swapon.8 swapctl.8
 
-WARNS?=	3
 DPADD=	${LIBUTIL}
 LDADD=	-lutil
 
diff --git a/sbin/swapon/swapon.8 b/sbin/swapon/swapon.8
index 40dba745c093..5602a9a5ff86 100644
--- a/sbin/swapon/swapon.8
+++ b/sbin/swapon/swapon.8
@@ -35,8 +35,14 @@
 .Nm swapon , swapoff , swapctl
 .Nd "specify devices for paging and swapping"
 .Sh SYNOPSIS
-.Nm swapon Fl aq | Ar
-.Nm swapoff Fl aq | Ar
+.Nm swapon
+.Oo Fl F Ar fstab
+.Oc
+.Fl aq | Ar
+.Nm swapoff
+.Oo Fl F Ar fstab
+.Oc
+.Fl aq | Ar
 .Nm swapctl
 .Op Fl AghklmsU
 .Oo
@@ -99,6 +105,18 @@ must move swapped pages out of the device being removed which could
 lead to high system loads for a period of time, depending on how
 much data has been swapped out to that device.
 .Pp
+Other options supported by both
+.Nm swapon
+and
+.Nm swapoff
+are as follows:
+.Bl -tag -width indent
+.It Fl F Ar fstab
+Specify the
+.Pa fstab
+file to use.
+.El
+.Pp
 The
 .Nm swapctl
 utility exists primarily for those familiar with other
diff --git a/sbin/swapon/swapon.c b/sbin/swapon/swapon.c
index e2cf8cafc0b4..ed13dc584426 100644
--- a/sbin/swapon/swapon.c
+++ b/sbin/swapon/swapon.c
@@ -67,9 +67,10 @@ main(int argc, char **argv)
 {
 	struct fstab *fsp;
 	char *ptr;
-	int stat;
+	int ret;
 	int ch, doall;
 	int sflag = 0, lflag = 0, hflag = 0, qflag = 0;
+	const char *etc_fstab;
 
 	if ((ptr = strrchr(argv[0], '/')) == NULL)
 		ptr = argv[0];
@@ -80,7 +81,8 @@ main(int argc, char **argv)
 	orig_prog = which_prog;
 	
 	doall = 0;
-	while ((ch = getopt(argc, argv, "AadghklmqsU")) != -1) {
+	etc_fstab = NULL;
+	while ((ch = getopt(argc, argv, "AadghklmqsUF:")) != -1) {
 		switch(ch) {
 		case 'A':
 			if (which_prog == SWAPCTL) {
@@ -132,6 +134,9 @@ main(int argc, char **argv)
 				usage();
 			}
 			break;
+		case 'F':
+			etc_fstab = optarg;
+			break;
 		case '?':
 		default:
 			usage();
@@ -139,7 +144,9 @@ main(int argc, char **argv)
 	}
 	argv += optind;
 
-	stat = 0;
+	ret = 0;
+	if (etc_fstab != NULL)
+		setfstab(etc_fstab);
 	if (which_prog == SWAPON || which_prog == SWAPOFF) {
 		if (doall) {
 			while ((fsp = getfsent()) != NULL) {
@@ -148,7 +155,7 @@ main(int argc, char **argv)
 				if (strstr(fsp->fs_mntops, "noauto"))
 					continue;
 				if (swap_on_off(fsp->fs_spec, 1)) {
-					stat = 1;
+					ret = 1;
 				} else {
 					if (!qflag) {
 						printf("%s: %sing %s as swap device\n",
@@ -163,7 +170,7 @@ main(int argc, char **argv)
 			usage();
 		for (; *argv; ++argv) {
 			if (swap_on_off(*argv, 0)) {
-				stat = 1;
+				ret = 1;
 			} else if (orig_prog == SWAPCTL) {
 				printf("%s: %sing %s as swap device\n",
 				    getprogname(), which_prog == SWAPOFF ? "remov" : "add",
@@ -176,7 +183,7 @@ main(int argc, char **argv)
 		else 
 			usage();
 	}
-	exit(stat);
+	exit(ret);
 }
 
 static int
@@ -210,7 +217,7 @@ usage(void)
 	switch(orig_prog) {
 	case SWAPON:
 	case SWAPOFF:
-	    fprintf(stderr, "-aq | file ...\n");
+	    fprintf(stderr, "[-F fstab] -aq | file ...\n");
 	    break;
 	case SWAPCTL:
 	    fprintf(stderr, "[-AghklmsU] [-a file ... | -d file ...]\n");
diff --git a/sbin/tunefs/tunefs.c b/sbin/tunefs/tunefs.c
index aee33fbae831..c219607a6157 100644
--- a/sbin/tunefs/tunefs.c
+++ b/sbin/tunefs/tunefs.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
+#include <time.h>
 #include <unistd.h>
 
 /* the optimization warning string template */
@@ -923,6 +924,7 @@ journal_alloc(int64_t size)
 	ino_t ino;
 	int blks;
 	int mode;
+	time_t utime;
 	int i;
 
 	cgp = &disk.d_cg;
@@ -983,18 +985,26 @@ journal_alloc(int64_t size)
 		 */
 		dp2 = ip;
 		dp1 = ip;
+		time(&utime);
 		if (sblock.fs_magic == FS_UFS1_MAGIC) {
 			bzero(dp1, sizeof(*dp1));
 			dp1->di_size = size;
 			dp1->di_mode = IFREG | IREAD;
 			dp1->di_nlink = 1;
 			dp1->di_flags = SF_IMMUTABLE | SF_NOUNLINK | UF_NODUMP;
+			dp1->di_atime = utime;
+			dp1->di_mtime = utime;
+			dp1->di_ctime = utime;
 		} else {
 			bzero(dp2, sizeof(*dp2));
 			dp2->di_size = size;
 			dp2->di_mode = IFREG | IREAD;
 			dp2->di_nlink = 1;
 			dp2->di_flags = SF_IMMUTABLE | SF_NOUNLINK | UF_NODUMP;
+			dp2->di_atime = utime;
+			dp2->di_mtime = utime;
+			dp2->di_ctime = utime;
+			dp2->di_birthtime = utime;
 		}
 		for (i = 0; i < NDADDR && resid; i++, resid--) {
 			blk = journal_balloc();
diff --git a/secure/lib/libcrypto/man/engine.3 b/secure/lib/libcrypto/man/engine.3
index e9ce28030e7a..215fceded4ef 100644
--- a/secure/lib/libcrypto/man/engine.3
+++ b/secure/lib/libcrypto/man/engine.3
@@ -653,7 +653,7 @@ implemented by ENGINEs should be numbered from. Any command value lower than
 this symbol is considered a \*(L"generic\*(R" command is handled directly by the
 OpenSSL core routines.
 .PP
-It is using these \*(L"core\*(R" control commands that one can discover the the control
+It is using these \*(L"core\*(R" control commands that one can discover the control
 commands implemented by a given \s-1ENGINE\s0, specifically the commands;
 .PP
 .Vb 9
diff --git a/secure/usr.bin/openssl/man/ca.1 b/secure/usr.bin/openssl/man/ca.1
index 9de935d8eec7..aecc18cc11d7 100644
--- a/secure/usr.bin/openssl/man/ca.1
+++ b/secure/usr.bin/openssl/man/ca.1
@@ -205,7 +205,7 @@ section for information on the required format.
 .IP "\fB\-infiles\fR" 4
 .IX Item "-infiles"
 if present this should be the last option, all subsequent arguments
-are assumed to the the names of files containing certificate requests.
+are assumed to the names of files containing certificate requests.
 .IP "\fB\-out filename\fR" 4
 .IX Item "-out filename"
 the output file to output certificates to. The default is standard
diff --git a/secure/usr.bin/openssl/man/dgst.1 b/secure/usr.bin/openssl/man/dgst.1
index 9d33fdfe92d1..d7def5d54918 100644
--- a/secure/usr.bin/openssl/man/dgst.1
+++ b/secure/usr.bin/openssl/man/dgst.1
@@ -184,7 +184,7 @@ the private key password source. For more information about the format of \fBarg
 see the \fB\s-1PASS\s0 \s-1PHRASE\s0 \s-1ARGUMENTS\s0\fR section in \fIopenssl\fR\|(1).
 .IP "\fB\-verify filename\fR" 4
 .IX Item "-verify filename"
-verify the signature using the the public key in \*(L"filename\*(R".
+verify the signature using the public key in \*(L"filename\*(R".
 The output is either \*(L"Verification \s-1OK\s0\*(R" or \*(L"Verification Failure\*(R".
 .IP "\fB\-prverify filename\fR" 4
 .IX Item "-prverify filename"
diff --git a/secure/usr.sbin/sshd/Makefile b/secure/usr.sbin/sshd/Makefile
index 702bcd076382..24d08e7a2e79 100644
--- a/secure/usr.sbin/sshd/Makefile
+++ b/secure/usr.sbin/sshd/Makefile
@@ -16,7 +16,8 @@ SRCS=	sshd.c auth-rhosts.c auth-passwd.c auth-rsa.c auth-rh-rsa.c \
 	auth2-gss.c gss-serv.c gss-serv-krb5.c \
 	loginrec.c auth-pam.c auth-shadow.c auth-sia.c md5crypt.c \
 	sftp-server.c sftp-common.c \
-	roaming_common.c roaming_serv.c
+	roaming_common.c roaming_serv.c \
+	sandbox-null.c sandbox-rlimit.c sandbox-systrace.c sandbox-darwin.c
 
 # gss-genr.c really belongs in libssh; see src/secure/lib/libssh/Makefile
 SRCS+=	gss-genr.c
diff --git a/share/doc/Makefile b/share/doc/Makefile
index b901b93aad53..7eabbd97e9f8 100644
--- a/share/doc/Makefile
+++ b/share/doc/Makefile
@@ -3,12 +3,16 @@
 
 .include <bsd.own.mk>
 
-SUBDIR=	${_bind9} IPv6 legal ${_roffdocs}
+SUBDIR=	${_bind9} IPv6 legal ${_llvm} ${_roffdocs}
 
 .if ${MK_BIND} != "no"
 _bind9=	bind9
 .endif
 
+.if ${MK_CLANG} != "no"
+_llvm=	llvm
+.endif
+
 # FIXME this is not a real solution ...
 .if ${MK_GROFF} != "no"
 _roffdocs=	papers psd smm usd
diff --git a/share/doc/llvm/Makefile b/share/doc/llvm/Makefile
new file mode 100644
index 000000000000..37493bbc2308
--- /dev/null
+++ b/share/doc/llvm/Makefile
@@ -0,0 +1,15 @@
+# $FreeBSD$
+
+SUBDIR=		clang
+
+SRCDIR=		${.CURDIR}/../../../contrib/llvm
+
+.PATH: ${SRCDIR} ${SRCDIR}/lib/Support
+
+NO_OBJ=
+
+FILESGROUPS=	TOP
+TOP=		LICENSE.TXT COPYRIGHT.regex
+TOPDIR=		${DOCDIR}/llvm
+
+.include <bsd.prog.mk>
diff --git a/share/doc/llvm/clang/Makefile b/share/doc/llvm/clang/Makefile
new file mode 100644
index 000000000000..1b26d6a9a60b
--- /dev/null
+++ b/share/doc/llvm/clang/Makefile
@@ -0,0 +1,13 @@
+# $FreeBSD$
+
+SRCDIR=		${.CURDIR}/../../../../contrib/llvm/tools/clang
+
+.PATH: ${SRCDIR}
+
+NO_OBJ=
+
+FILESGROUPS=	TOP
+TOP=		LICENSE.TXT
+TOPDIR=		${DOCDIR}/llvm/clang
+
+.include <bsd.prog.mk>
diff --git a/share/doc/psd/03.iosys/iosys b/share/doc/psd/03.iosys/iosys
index ce63bc25319a..1459f142a9d5 100644
--- a/share/doc/psd/03.iosys/iosys
+++ b/share/doc/psd/03.iosys/iosys
@@ -813,7 +813,7 @@ which have a separate argument
 which indicates read or write.
 .IP B_DONE 10
 This bit is set
-to 0 when a block is handed to the the device strategy
+to 0 when a block is handed to the device strategy
 routine and is turned on when the operation completes,
 whether normally as the result of an error.
 It is also used as part of the return argument of
diff --git a/share/doc/psd/23.rpc/rpc.prog.ms b/share/doc/psd/23.rpc/rpc.prog.ms
index 54fd2d5cb40d..0bc8743de659 100644
--- a/share/doc/psd/23.rpc/rpc.prog.ms
+++ b/share/doc/psd/23.rpc/rpc.prog.ms
@@ -317,7 +317,7 @@ look up the appropriate program, version and procedure numbers
 in a manual, just as you look up the name of a memory allocator
 when you want to allocate memory.
 .LP
-The simplest way of making remote procedure calls is with the the RPC
+The simplest way of making remote procedure calls is with the RPC
 library routine
 .I callrpc()
 It has eight parameters.  The first is the name of the remote server
@@ -1105,7 +1105,7 @@ Note that, after being used, the character array can be freed with
 .I svc_freeargs()
 .I svc_freeargs()
 will not attempt to free any memory if the variable indicating it
-is NULL.  For example, in the the routine
+is NULL.  For example, in the routine
 .I xdr_finalexample (),
 given earlier, if
 .I finalp->string
diff --git a/share/doc/psd/24.xdr/xdr.nts.ms b/share/doc/psd/24.xdr/xdr.nts.ms
index 260c7f37eb05..f7c495d6a6f3 100644
--- a/share/doc/psd/24.xdr/xdr.nts.ms
+++ b/share/doc/psd/24.xdr/xdr.nts.ms
@@ -721,7 +721,7 @@ it must not exceed
 .I maxlength .
 Next
 .I sp
-is dereferenced; if the the value is
+is dereferenced; if the value is
 .I NULL ,
 then a string of the appropriate length is allocated and
 .I *sp
diff --git a/share/doc/psd/27.nfsrpc/nfs.rfc.ms b/share/doc/psd/27.nfsrpc/nfs.rfc.ms
index 13d761908a8e..5a2d9492d39f 100644
--- a/share/doc/psd/27.nfsrpc/nfs.rfc.ms
+++ b/share/doc/psd/27.nfsrpc/nfs.rfc.ms
@@ -961,7 +961,7 @@ call to get more  entries  starting at a
 given point in  the directory.  The  special cookie zero (all  bits
 zero) can be used to get the entries starting  at the  beginning of
 the directory.  The "fileid" field should be the same number as the
-"fileid" in the the  attributes of the  file.  (See the
+"fileid" in the attributes of the file.  (See the
 .I "Basic Data Types"
 section.) 
 The "eof" flag has a value of
diff --git a/share/examples/cvsup/stable-supfile b/share/examples/cvsup/stable-supfile
index 041d56440eca..341eb6b93a5f 100644
--- a/share/examples/cvsup/stable-supfile
+++ b/share/examples/cvsup/stable-supfile
@@ -66,11 +66,11 @@
 *default host=CHANGE_THIS.FreeBSD.org
 *default base=/var/db
 *default prefix=/usr
-# The following line is for 8-stable.  If you want 7-stable, 6-stable,
-# 5-stable, 4-stable, 3-stable, or 2.2-stable, change to "RELENG_7",
-# "RELENG_6", "RELENG_5", "RELENG_4", "RELENG_3", or "RELENG_2_2"
-# respectively.
-*default release=cvs tag=RELENG_8
+# The following line is for 9-stable.  If you want 8-stable, 7-stable,
+# 6-stable, 5-stable, 4-stable, 3-stable, or 2.2-stable, change to "RELENG_8",
+# "RELENG_7", "RELENG_6", "RELENG_5", "RELENG_4", "RELENG_3", or
+# "RELENG_2_2" respectively.
+*default release=cvs tag=RELENG_9
 *default delete use-rel-suffix
 
 # If you seem to be limited by CPU rather than network or disk bandwidth, try
diff --git a/share/examples/scsi_target/scsi_cmds.c b/share/examples/scsi_target/scsi_cmds.c
index 8d5f14b5f550..10295d7c89e4 100644
--- a/share/examples/scsi_target/scsi_cmds.c
+++ b/share/examples/scsi_target/scsi_cmds.c
@@ -242,22 +242,22 @@ tcmd_sense(u_int init_id, struct ccb_scsiio *ctio, u_int8_t flags,
 	       u_int8_t asc, u_int8_t ascq)
 {
 	struct initiator_state *istate;
-	struct scsi_sense_data *sense;
+	struct scsi_sense_data_fixed *sense;
 
 	/* Set our initiator's istate */
 	istate = tcmd_get_istate(init_id);
 	if (istate == NULL)
 		return;
 	istate->pending_ca |= CA_CMD_SENSE; /* XXX set instead of or? */
-	sense = &istate->sense_data;
+	sense = (struct scsi_sense_data_fixed *)&istate->sense_data;
 	bzero(sense, sizeof(*sense));
 	sense->error_code = SSD_CURRENT_ERROR;
 	sense->flags = flags;
 	sense->add_sense_code = asc;
 	sense->add_sense_code_qual = ascq;
 	sense->extra_len =
-		offsetof(struct scsi_sense_data, sense_key_spec[2]) -
-		offsetof(struct scsi_sense_data, extra_len);
+		offsetof(struct scsi_sense_data_fixed, sense_key_spec[2]) -
+		offsetof(struct scsi_sense_data_fixed, extra_len);
 
 	/* Fill out the supplied CTIO */
 	if (ctio != NULL) {
@@ -298,7 +298,7 @@ tcmd_inquiry(struct ccb_accept_tio *atio, struct ccb_scsiio *ctio)
 	struct scsi_inquiry *inq;
 	struct atio_descr *a_descr;
 	struct initiator_state *istate;
-	struct scsi_sense_data *sense;
+	struct scsi_sense_data_fixed *sense;
 
 	a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
 	inq = (struct scsi_inquiry *)a_descr->cdb;
@@ -310,7 +310,7 @@ tcmd_inquiry(struct ccb_accept_tio *atio, struct ccb_scsiio *ctio)
 	 * complain if EVPD or CMDDT is set.
 	 */
 	istate = tcmd_get_istate(ctio->init_id);
-	sense = &istate->sense_data;
+	sense = (struct scsi_sense_data_fixed *)&istate->sense_data;
 	if ((inq->byte2 & SI_EVPD) != 0) {
 		tcmd_illegal_req(atio, ctio);
 		sense->sense_key_spec[0] = SSD_SCS_VALID | SSD_FIELDPTR_CMD |
@@ -376,7 +376,7 @@ static int
 tcmd_req_sense(struct ccb_accept_tio *atio, struct ccb_scsiio *ctio)
 {
 	struct scsi_request_sense *rsense;
-	struct scsi_sense_data *sense;
+	struct scsi_sense_data_fixed *sense;
 	struct initiator_state *istate;
 	size_t dlen;
 	struct atio_descr *a_descr;
@@ -385,7 +385,7 @@ tcmd_req_sense(struct ccb_accept_tio *atio, struct ccb_scsiio *ctio)
 	rsense = (struct scsi_request_sense *)a_descr->cdb;
 	
 	istate = tcmd_get_istate(ctio->init_id);
-	sense = &istate->sense_data;
+	sense = (struct scsi_sense_data_fixed *)&istate->sense_data;
 
 	if (debug) {
 		cdb_debug(a_descr->cdb, "REQ SENSE from %u: ", atio->init_id);
@@ -400,7 +400,7 @@ tcmd_req_sense(struct ccb_accept_tio *atio, struct ccb_scsiio *ctio)
 	}
 
 	bcopy(sense, ctio->data_ptr, sizeof(struct scsi_sense_data));
-	dlen = offsetof(struct scsi_sense_data, extra_len) +
+	dlen = offsetof(struct scsi_sense_data_fixed, extra_len) +
 			sense->extra_len + 1;
 	ctio->dxfer_len = min(dlen, SCSI_CDB6_LEN(rsense->length));
 	ctio->ccb_h.flags |= CAM_DIR_IN | CAM_SEND_STATUS;
@@ -482,7 +482,7 @@ tcmd_rdwr(struct ccb_accept_tio *atio, struct ccb_scsiio *ctio)
 	c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
 
 	/* Command needs to be decoded */
-	if ((a_descr->flags & CAM_DIR_MASK) == CAM_DIR_RESV) {
+	if ((a_descr->flags & CAM_DIR_MASK) == CAM_DIR_BOTH) {
 		if (debug)
 			warnx("Calling rdwr_decode");
 		ret = tcmd_rdwr_decode(atio, ctio);
diff --git a/share/examples/scsi_target/scsi_target.c b/share/examples/scsi_target/scsi_target.c
index 52401e104991..e6ff5c716a95 100644
--- a/share/examples/scsi_target/scsi_target.c
+++ b/share/examples/scsi_target/scsi_target.c
@@ -651,7 +651,7 @@ work_atio(struct ccb_accept_tio *atio)
 	 * receiving this ATIO.
 	 */
 	if (atio->sense_len != 0) {
-		struct scsi_sense_data *sense;
+		struct scsi_sense_data_fixed *sense;
 
 		if (debug) {
 			warnx("ATIO with %u bytes sense received",
@@ -825,9 +825,9 @@ work_inot(struct ccb_immed_notify *inot)
 
 	/* If there is sense data, use it */
 	if (sense != 0) {
-		struct scsi_sense_data *sense;
+		struct scsi_sense_data_fixed *sense;
 
-		sense = &inot->sense_data;
+		sense = (struct scsi_sense_data_fixed *)&inot->sense_data;
 		tcmd_sense(inot->initiator_id, NULL, sense->flags,
 			   sense->add_sense_code, sense->add_sense_code_qual);
 		if (debug)
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 091eb285defd..b38a01c0ef46 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -447,6 +447,7 @@ MAN=	aac.4 \
 	tun.4 \
 	twa.4 \
 	twe.4 \
+	tws.4 \
 	tx.4 \
 	txp.4 \
 	u3g.4 \
diff --git a/share/man/man4/axe.4 b/share/man/man4/axe.4
index 156aa613b4b1..6c68d46e0fcd 100644
--- a/share/man/man4/axe.4
+++ b/share/man/man4/axe.4
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 14, 2011
+.Dd October 25, 2011
 .Dt AXE 4
 .Os
 .Sh NAME
@@ -238,6 +238,7 @@ The driver failed to allocate an mbuf for the receiver ring.
 .Xr netintro 4 ,
 .Xr ng_ether 4 ,
 .Xr rgephy 4 ,
+.Xr vlan 4 ,
 .Xr ifconfig 8
 .Rs
 .%T "ASIX AX88x7x and AX88760 data sheets"
diff --git a/share/man/man4/dc.4 b/share/man/man4/dc.4
index 3a565fc0d7a6..cc6250c2c2ea 100644
--- a/share/man/man4/dc.4
+++ b/share/man/man4/dc.4
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 23, 2011
+.Dd October 24, 2011
 .Dt DC 4
 .Os
 .Sh NAME
@@ -55,7 +55,7 @@ if_dc_load="YES"
 The
 .Nm
 driver provides support for several PCI Fast Ethernet adapters and
-embedded controllers based on the the DEC/Intel 21143 chipset and clones.
+embedded controllers based on the DEC/Intel 21143 chipset and clones.
 .Pp
 All of supported chipsets have the same general register layout, DMA
 descriptor format and method of operation.
@@ -160,6 +160,8 @@ DEC/Intel 21143
 ADMtek AL981 Comet, AN985 Centaur, ADM9511 Centaur II and ADM9513
 Centaur II
 .It
+ALi/ULi M5261 and M5263
+.It
 ASIX Electronics AX88140A and AX88141
 .It
 Conexant LANfinity RS7112 (miniPCI)
@@ -345,6 +347,7 @@ the card should be configured correctly.
 .Xr netintro 4 ,
 .Xr ng_ether 4 ,
 .Xr polling 4 ,
+.Xr vlan 4 ,
 .Xr eeprom 8 ,
 .Xr ifconfig 8
 .Rs
diff --git a/share/man/man4/em.4 b/share/man/man4/em.4
index 842f9ee5853d..8213b98c00d1 100644
--- a/share/man/man4/em.4
+++ b/share/man/man4/em.4
@@ -31,7 +31,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 14, 2010
+.Dd October 11, 2011
 .Dt EM 4
 .Os
 .Sh NAME
@@ -199,12 +199,14 @@ prompt before booting the kernel or stored in
 .Bl -tag -width indent
 .It Va hw.em.rxd
 Number of receive descriptors allocated by the driver.
-The default value is 256.
+The default value is 1024 for adapters newer than 82547,
+and 256 for older ones.
 The 82542 and 82543-based adapters can handle up to 256 descriptors,
 while others can have up to 4096.
 .It Va hw.em.txd
 Number of transmit descriptors allocated by the driver.
-The default value is 256.
+The default value is 1024 for adapters newer than 82547,
+and 256 for older ones.
 The 82542 and 82543-based adapters can handle up to 256 descriptors,
 while others can have up to 4096.
 .It Va hw.em.rx_int_delay
diff --git a/share/man/man4/igb.4 b/share/man/man4/igb.4
index 70db340bcac5..dc727259ea67 100644
--- a/share/man/man4/igb.4
+++ b/share/man/man4/igb.4
@@ -31,7 +31,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 14, 2010
+.Dd October 11, 2011
 .Dt IGB 4
 .Os
 .Sh NAME
@@ -151,11 +151,11 @@ prompt before booting the kernel or stored in
 .Bl -tag -width indent
 .It Va hw.igb.rxd
 Number of receive descriptors allocated by the driver.
-The default value is 256.
+The default value is 1024.
 The minimum is 80, and the maximum is 4096.
 .It Va hw.igb.txd
 Number of transmit descriptors allocated by the driver.
-The default value is 256.
+The default value is 1024.
 The minimum is 80, and the maximum is 4096.
 .It Va hw.igb.enable_aim
 If set to 1, enable Adaptive Interrupt Moderation.
diff --git a/share/man/man4/man4.powerpc/Makefile b/share/man/man4/man4.powerpc/Makefile
index 57196a3b92b1..09d8b70eb1f2 100644
--- a/share/man/man4/man4.powerpc/Makefile
+++ b/share/man/man4/man4.powerpc/Makefile
@@ -2,6 +2,7 @@
 
 MAN=	adb.4 \
 	akbd.4 \
+	abtn.4 \
 	ams.4 \
 	bm.4 \
 	cuda.4 \
diff --git a/share/man/man4/man4.powerpc/abtn.4 b/share/man/man4/man4.powerpc/abtn.4
new file mode 100644
index 000000000000..9de300d8f500
--- /dev/null
+++ b/share/man/man4/man4.powerpc/abtn.4
@@ -0,0 +1,115 @@
+.\"-
+.\" Copyright (c) 2011 Justin Hibbits
+.\" Copyright (c) 2009 Nathan Whitehorn <nwhitehorn@FreeBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+.\" DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+.\" INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+.\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+.\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+.\" ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 16, 2011
+.Dt ABTN 4
+.Os
+.Sh NAME
+.Nm abtn
+.Nd ADB Keyboard Special Keys Driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device adb"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver provides support for the extended Fn keys on Apple notebooks with an ADB
+interface.
+.Sh HARDWARE
+The
+.Nm
+driver supports extended keyboard keys (special F-keys) on the following devices:
+.Pp
+.Bl -bullet -compact
+.It
+Apple iBook Keyboard
+.It
+Apple PowerBook Keyboard
+.El
+.Sh EVENTS
+The
+.Nm
+driver sends events to
+.Xr devd 8
+for the following events under the 
+.Cd PMU
+system, and
+.Cd keys
+subsystem:
+.Pp
+.Bl -bullet -compact
+.It
+.Cd brightness
+- Generates
+.Cd up
+and
+.Cd down
+notify types matching the pressed key.
+.It
+.Cd mute
+.It
+.Cd volume
+- Generates
+.Cd up
+and
+.Cd down
+notify types matching the pressed key.
+.It
+.Cd eject
+.El
+.Pp
+Examples are included in /etc/devd/apple.conf.
+.Sh SEE ALSO
+.Xr adb 4 ,
+.Xr akbd 4 ,
+.Xr cuda 4 ,
+.Xr pmu 4 ,
+.Xr devd 8
+.Sh HISTORY
+The
+.Nm
+device driver first appeared in
+.Nx 5.0
+and was ported to
+.Fx 10.0 .
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+driver was written by
+.An Tsubai Masanari
+for
+.Nx
+and ported to
+.Fx
+by
+.An Justin Hibbits .
diff --git a/share/man/man4/man4.powerpc/akbd.4 b/share/man/man4/man4.powerpc/akbd.4
index aea2fa01f504..d4f3aa167059 100644
--- a/share/man/man4/man4.powerpc/akbd.4
+++ b/share/man/man4/man4.powerpc/akbd.4
@@ -58,7 +58,36 @@ Apple iBook Keyboard
 .It
 Apple PowerBook Keyboard
 .El
+.Sh EVENTS
+The
+.Nm
+driver sends events to
+.Xr devd 8
+for the following events under the 
+.Cd PMU
+system:
+.Pp
+.Bl -bullet -compact
+.It
+Power button - 
+.Cd "Button"
+subsystem, 
+.Cd "pressed"
+type.
+.El
+.Sh SYSCTL VARIABLES
+The
+.Nm
+driver supports the following sysctl variable for configuring the Fn keys:
+.Bl -tag -width indent
+.It Va dev.akbd.%d.fn_keys_function_as_primary
+Set the Fn keys to be their F-key type as default.  A value of 0 causes the 
+F-keys keys to work as special keys by default (
+.Xr abtn 4 )
+and a value of 1 sets them to behave as F-keys by default.
+.El
 .Sh SEE ALSO
+.Xr abtn 4 ,
 .Xr adb 4 ,
 .Xr cuda 4 ,
 .Xr pmu 4
diff --git a/share/man/man4/smp.4 b/share/man/man4/smp.4
index 5d0d983be183..5712dbd0e35e 100644
--- a/share/man/man4/smp.4
+++ b/share/man/man4/smp.4
@@ -120,25 +120,24 @@ tasks on CPUs that are closely grouped together.
 .Pp
 .Fx
 supports hyperthreading on Intel CPU's on the i386 and AMD64 platforms.
-Since using logical CPUs can cause performance penalties under certain loads,
+Because using logical CPUs can cause performance penalties under certain loads,
 the logical CPUs can be disabled by setting the
 .Va machdep.hyperthreading_allowed
 tunable to zero. 
-.Xr cpuset 1 .
 .Sh SEE ALSO
+.Xr cpuset 1 ,
 .Xr mptable 1 ,
+.Xr sched_4bsd 4 ,
+.Xr sched_ule 4 ,
 .Xr loader 8 ,
 .Xr sysctl 8 ,
 .Xr condvar 9 ,
 .Xr msleep 9 ,
 .Xr mtx_pool 9 ,
 .Xr mutex 9 ,
-.Xr sema 9 ,
-.Xr sx 9 ,
 .Xr rwlock 9 ,
-.Xr sched_4bsd 4 ,
-.Xr sched_ule 4 ,
-.Xr cpuset 1
+.Xr sema 9 ,
+.Xr sx 9
 .Sh HISTORY
 The
 .Nm
diff --git a/share/man/man4/tws.4 b/share/man/man4/tws.4
new file mode 100644
index 000000000000..9ce154e1f265
--- /dev/null
+++ b/share/man/man4/tws.4
@@ -0,0 +1,118 @@
+.\"
+.\"Copyright (c) 2010, 2011 iXsystems, Inc.
+.\"All rights reserved.
+.\" written by: Xin LI <delphij@FreeBSD.org>
+.\"
+.\"Redistribution and use in source and binary forms, with or without
+.\"modification, are permitted provided that the following conditions
+.\"are met:
+.\"1. Redistributions of source code must retain the above copyright
+.\"   notice, this list of conditions and the following disclaimer.
+.\"2. Redistributions in binary form must reproduce the above copyright
+.\"   notice, this list of conditions and the following disclaimer in the
+.\"   documentation and/or other materials provided with the distribution.
+.\"
+.\"THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\"ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\"IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\"ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\"FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\"DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\"OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\"HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\"LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\"OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\"SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 4, 2011
+.Dt TWS 4
+.Os
+.Sh NAME
+.Nm tws
+.Nd 3ware 9750 SATA+SAS 6Gb/s RAID controller card driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device scbus"
+.Cd "device tws"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+tws_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver provides support for LSI's 3ware 9750 SATA+SAS 6Gb/s RAID controller cards.
+.Pp
+These controllers feature the LSISAS2108 6Gb/s SAS RAID-on-Chip (ROC)
+and are available in 4- and 8-port configurations, supports RAID levels
+0, 1, 5, 6, 10, 50 and single disk, with 96 SATA and/or SAS hard drives and SSDs.
+.Pp
+For further hardware information, see
+.Pa http://www.lsi.com/.
+.Sh HARDWARE
+The
+.Nm
+driver supports the following SATA/SAS RAID controller:
+.Pp
+.Bl -bullet -compact
+.It
+LSI's 3ware SAS 9750 series
+.El
+.Sh LOADER TUNABLES
+Tunables can be set at the
+.Xr loader 8
+prompt before booting the kernel or stored in
+.Xr loader.conf 5 .
+.Bl -tag -width "hw.tws.use_32bit_sgls"
+.It Va hw.tws.cam_depth
+The maximium queued CAM SIM requests for one controller.
+The default value is 256.
+.It Va hw.tws.enable_msi
+This tunable enables MSI support on the controller if set to a non-zero value.
+The default value is 0.
+.It Va hw.tws.queue_depth
+The maximium queued requests for one controller.
+.It Va hw.tws.use_32bit_sgls
+Limit the driver to use only 32-bit SG elements regardless whether the operating
+system is running in 64-bit mode.
+The default value is 0.
+.El
+.Sh FILES
+.Bl -tag -width ".Pa /dev/tws?" -compact
+.It Pa /dev/da?
+array/logical disk interface
+.It Pa /dev/tws?
+management interface
+.El
+.Sh DIAGNOSTICS
+Whenever the driver encounters a command failure, it prints out an error code in
+the format:
+.Qq Li "ERROR: (<error source>: <error code>):" ,
+followed by a text description of the error.
+There are other error messages and warnings that the
+driver prints out, depending on the kinds of errors that it encounters.
+If the driver is compiled with
+.Dv TWS_DEBUG
+defined, it prints out a whole bunch of debug
+messages.
+.Sh SEE ALSO
+.Xr da 4 ,
+.Xr scsi 4
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+driver was written by
+.An Manjunath Ranganathaiah
+for LSI and this manual page was written by
+.An Xin LI Aq delphij@FreeBSD.org
+for iXsystems, Inc.
diff --git a/share/man/man4/vlan.4 b/share/man/man4/vlan.4
index e0e6152bd67e..4ef65cbf13f0 100644
--- a/share/man/man4/vlan.4
+++ b/share/man/man4/vlan.4
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 30, 2010
+.Dd October 25, 2011
 .Dt VLAN 4
 .Os
 .Sh NAME
@@ -166,6 +166,7 @@ filtering that breaks the Path MTU Discovery mechanism.
 The following interfaces support long frames for
 .Nm
 natively:
+.Xr axe 4 ,
 .Xr bfe 4 ,
 .Xr cas 4 ,
 .Xr dc 4 ,
diff --git a/share/man/man5/rc.conf.5 b/share/man/man5/rc.conf.5
index 009ba3353e73..e4a058ae3947 100644
--- a/share/man/man5/rc.conf.5
+++ b/share/man/man5/rc.conf.5
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd June 19, 2011
+.Dd October 23, 2011
 .Dt RC.CONF 5
 .Os
 .Sh NAME
@@ -1282,6 +1282,13 @@ ifconfig_ed0_name="net0"
 ifconfig_net0="inet 192.0.2.1 netmask 0xffffff00"
 .Ed
 .It Va ipv6_enable
+This variable is deprecated.
+Use
+.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
+and
+.Va ipv6_activate_all_interfaces
+if necessary.
+.Pp
 .Pq Vt bool
 If the variable is
 .Dq Li YES ,
@@ -1292,13 +1299,12 @@ and the
 .Va ipv6_activate_all_interfaces
 is defined as
 .Dq Li YES .
-.Pp
-This variable is deprecated.  Use
-.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
-and
-.Va ipv6_activate_all_interfaces
-if necessary.
 .It Va ipv6_prefer
+This variable is deprecated.
+Use
+.Va ip6addrctl_policy
+instead.
+.Pp
 .Pq Vt bool
 If the variable is
 .Dq Li YES ,
@@ -1311,19 +1317,45 @@ If the variable is
 the default address selection policy table set by
 .Xr ip6addrctl 8
 will be IPv4-preferred.
-.Pp
-This variable is deprecated.  Use
-.Va ip6addrctl_policy
-instead.
 .It Va ipv6_activate_all_interfaces
+.Pq Vt bool
+This controls initial configuration on IPv6-capable
+interfaces with no corresponding
+.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
+variable.
+Note that it is not always necessary to set this variable to
+.Dq YES
+to use IPv6 functionality on
+.Fx .
+In most cases, just configuring
+.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
+variables works.
+.Pp
 If the variable is
 .Dq Li NO ,
-all of interfaces which do not have the corrsponding
+all interfaces which do not have a corresponding
 .Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
 variable will be marked as
 .Dq Li IFDISABLED
-for security reason.  This means only IPv6 functionality on that interface
-is completely disabled.  For more details of
+at creation.
+This means that all of IPv6 functionality on that interface
+is completely disabled to enforce a security policy.
+If the variable is set to
+.Dq YES ,
+the flag will be cleared on all of the interfaces.
+.Pp
+In most cases, just defining an
+.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
+for an IPv6-capable interface should be sufficient.
+However, if an interface is added dynamically
+.Pq by some tunneling protocols such as PPP, for example ,
+it is often difficult to define the variable in advance.
+In such a case, configuring the
+.Dq Li IFDISABLED
+flag can be disabled by setting this variable to
+.Dq YES .
+.Pp
+For more details of the
 .Dq Li IFDISABLED
 flag and keywords
 .Dq Li inet6 ifdisabled ,
@@ -1344,12 +1376,58 @@ This is the IPv6 equivalent of
 .Va network_interfaces .
 Normally manual configuration of this variable is not needed.
 .Pp
+.It Va ipv6_cpe_wanif
+.Pq Vt str
+If the variable is set to an interface name,
+the
+.Xr ifconfig 8
+options
+.Dq inet6 -no_radr accept_rtadv
+will be added to the specified interface automatically before evaluating
+.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6 ,
+and two
+.Xr sysctl 8
+variables
+.Va net.inet6.ip6.rfc6204w3
+and
+.Va net.inet6.ip6.no_radr
+will be set to 1.
+.Pp
+This means the specified interface will accept ICMPv6 Router
+Advertisement messages on that link and add the discovered
+routers into the Default Router List.
+While the other interfaces can still accept RA messages if the
+.Dq inet6 accept_rtadv
+option is specified, adding
+routes into the Default Router List will be disabled by
+.Dq inet6 no_radr
+option by default.
+See
+.Xr ifconfig 8
+for more details.
+.Pp
+Note that ICMPv6 Router Advertisement messages will be
+accepted even when
+.Va net.inet6.ip6.forwarding
+is 1
+.Pq packet fowarding is enabled
+when
+.Va net.inet6.ip6.rfc6204w3
+is set to 1.
+.Pp
+Default is
+.Dq Li NO .
 .It Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
 .Pq Vt str
 IPv6 functionality on an interface should be configured by
 .Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6 ,
 instead of setting ifconfig parameters in
 .Va ifconfig_ Ns Aq Ar interface .
+If this variable is empty, all of IPv6 configurations on the
+specified interface by other variables such as
+.Va ipv6_prefix_ Ns Ao Ar interface Ac
+will be ignored.
+.Pp
 Aliases should be set by
 .Va ifconfig_ Ns Ao Ar interface Ac Ns Va _alias Ns Aq Ar n
 with
@@ -1364,18 +1442,63 @@ Interfaces that have an
 .Dq Li inet6 accept_rtadv
 keyword in
 .Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
-setting will be automatically configured by
-.Xr rtsol 8 .
-Note that this automatic configuration is disabled if the
-.Va ipv6_gateway_enable
-is set to
-.Dq Li YES .
+setting will be automatically configured by SLAAC
+.Pq StateLess Address AutoConfiguration
+described in
+.Rs
+.%T "RFC 4862"
+.Re
+.Pp
+Note that a link-local address will be automatically configured in
+addition to the configured global-scope addresses because the IPv6
+specifications require it on each link.
+The address is calculated from the MAC address by using an algorithm
+defined in
+.Rs
+.%T "RFC 4862"
+.%O "Section 5.3"
+.Re
+.Pp
+If only a link-local address is needed on the interface,
+the following configuration can be used:
+.Bd -literal
+ifconfig_ed0_ipv6="inet6 auto_linklocal"
+.Ed
+.Pp
+A link-local address can also be configured manually.
+This is useful for the default router address of an IPv6 router
+so that it does not change when the network interface
+card is replaced.
+For example:
+.Bd -literal
+ifconfig_ed0_ipv6="inet6 fe80::1 prefixlen 64"
+.Ed
 .It Va ipv6_prefix_ Ns Aq Ar interface
 .Pq Vt str
 If one or more prefixes are defined in
 .Va ipv6_prefix_ Ns Aq Ar interface
 addresses based on each prefix and the EUI-64 interface index will be
 configured on that interface.
+Note that this variable will be ignored when
+.Va ifconfig_ Ns Ao Ar interface Ac Ns _ipv6
+is empty.
+.Pp
+For example, the following configuration
+.Bd -literal
+ipv6_prefix_ed0="2001:db8:1:0 2001:db8:2:0"
+.Ed
+.Pp
+is equivalent to the following:
+.Bd -literal
+ifconfig_ed0_alias0="inet6 2001:db8:1:: eui64 prefixlen 64"
+ifconfig_ed0_alias1="inet6 2001:db8:1:: prefixlen 64 anycast"
+ifconfig_ed0_alias2="inet6 2001:db8:2:: eui64 prefixlen 64"
+ifconfig_ed0_alias3="inet6 2001:db8:2:: prefixlen 64 anycast"
+.Ed
+.Pp
+These Subnet-Router anycast addresses will be added only when
+.Va ipv6_gateway_enable
+is YES.
 .It Va ipv6_default_interface
 .Pq Vt str
 If not set to
@@ -2474,6 +2597,22 @@ For example
 static_arp_pairs="gw"
 static_arp_gw="192.168.1.1 00:01:02:03:04:05"
 .Ed
+.It Va static_ndp_pairs
+.Pq Vt str
+Set to the list of static NDP pairs that are to be added at system
+boot time.
+For each whitespace separated
+.Ar element
+in the value, a
+.Va static_ndp_ Ns Aq Ar element
+variable is assumed to exist whose contents will later be passed to a
+.Dq Nm ndp Cm -s
+operation.
+For example
+.Bd -literal
+static_ndp_pairs="gw"
+static_ndp_gw="2001:db8:3::1 00:01:02:03:04:05"
+.Ed
 .It Va static_routes
 .Pq Vt str
 Set to the list of static routes that are to be added at system
@@ -2628,16 +2767,12 @@ If set to
 run the
 .Xr rtadvd 8
 daemon at boot time.
-.Xr rtadvd 8
-will only run if
-.Va ipv6_gateway_enable
-is also set to
-.Dq Li YES .
 The
 .Xr rtadvd 8
-utility sends router advertisement packets to the interfaces specified in
-.Va rtadvd_interfaces
-and should only be enabled with great care.
+utility sends ICMPv6 Router Advertisement messages to
+the interfaces specified in
+.Va rtadvd_interfaces .
+This should only be enabled with great care.
 You may want to fine-tune
 .Xr rtadvd.conf 5 .
 .It Va rtadvd_interfaces
diff --git a/share/man/man7/build.7 b/share/man/man7/build.7
index b43a961d6777..f16895875fee 100644
--- a/share/man/man7/build.7
+++ b/share/man/man7/build.7
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd June 16, 2011
+.Dd September 29, 2011
 .Dt BUILD 7
 .Os
 .Sh NAME
@@ -238,7 +238,7 @@ This command takes a long time.
 Get updated sources as configured in
 .Xr make.conf 5 .
 .It Cm targets
-Print a list of supported 
+Print a list of supported
 .Va TARGET
 /
 .Va TARGET_ARCH
@@ -399,21 +399,31 @@ It defaults to
 .It Va KERNFAST
 If set, the build target
 .Cm buildkernel
-defaults to setting 
+defaults to setting
 .Va NO_KERNELCLEAN ,
 .Va NO_KERNELCONFIG ,
-.Va NO_KERNELDEPEND 
+.Va NO_KERNELDEPEND
 and
 .Va NO_KERNELOBJ .
 When set to a value other than
 .Cm 1
-then 
-.Va KERNCONF 
-is set to the value of 
+then
+.Va KERNCONF
+is set to the value of
 .Va KERNFAST .
 .It Va LOCAL_DIRS
 If set, this variable supplies a list of additional directories to
 build, relative to the root of the source tree.
+.It Va PORTS_MODULES
+A list of ports with kernel modules that should be built and installed
+as part of the
+.Cm buildkernel
+and
+.Cm installkernel
+process.
+.Bd -literal -offset indent
+make PORTS_MODULES=emulators/kqemu-kmod kernel
+.Ed
 .It Va SUBDIR_OVERRIDE
 Override the default list of sub-directories and only build the
 sub-directory named in this variable.
@@ -442,11 +452,11 @@ output.
 Set this to cross-build for a different architecture.
 If not set,
 .Va TARGET_ARCH
-defaults to the current machine architecture, unless 
+defaults to the current machine architecture, unless
 .Va TARGET
 is also set, in which case it defaults to the appropriate
 value for that platform.
-Typically, one only needs to set 
+Typically, one only needs to set
 .Va TARGET .
 .El
 .Pp
diff --git a/share/man/man7/security.7 b/share/man/man7/security.7
index ed24435af82a..197fe3e14072 100644
--- a/share/man/man7/security.7
+++ b/share/man/man7/security.7
@@ -544,6 +544,12 @@ may not be opened for writing;
 kernel modules (see
 .Xr kld 4 )
 may not be loaded or unloaded.
+The kernel debugger may not be entered using the
+.Va debug.kdb.enter
+sysctl.
+A panic or trap cannot be forced using the
+.Va debug.kdb.panic
+and other sysctl's.
 .It Ic 2
 Highly secure mode \- same as secure mode, plus disks may not be
 opened for writing (except by
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index 7d34c318c8f6..3026ef8041a9 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -972,7 +972,8 @@ MLINKS+=printf.9 log.9 \
 MLINKS+=priv.9 priv_check.9 \
 	priv.9 priv_check_cred.9
 MLINKS+=psignal.9 gsignal.9 \
-	psignal.9 pgsignal.9
+	psignal.9 pgsignal.9 \
+	psignal.9 tdsignal.9
 MLINKS+=random.9 arc4rand.9 \
 	random.9 arc4random.9 \
 	random.9 read_random.9 \
diff --git a/share/man/man9/ifnet.9 b/share/man/man9/ifnet.9
index 05264dc8ea6c..e015fb74e415 100644
--- a/share/man/man9/ifnet.9
+++ b/share/man/man9/ifnet.9
@@ -1252,7 +1252,7 @@ function is called to perform the operation; qq.v.
 The socket's protocol control routine is called to implement the
 requested action.
 .Pp
-.It Dv OSIOGIFADDR
+.It Dv OSIOCGIFADDR
 .It Dv OSIOCGIFDSTADDR
 .It Dv OSIOCGIFBRDADDR
 .It Dv OSIOCGIFNETMASK
diff --git a/share/man/man9/memguard.9 b/share/man/man9/memguard.9
index ed833ba27e74..072d30b0fd6d 100644
--- a/share/man/man9/memguard.9
+++ b/share/man/man9/memguard.9
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 2, 2010
+.Dd October 21, 2011
 .Dt MEMGUARD 9
 .Os
 .Sh NAME
@@ -39,7 +39,6 @@ to help detect tamper-after-free scenarios.
 These problems are more and more common and likely with
 multithreaded kernels where race conditions are more prevalent.
 .Pp
-Currently,
 .Nm
 can take over
 .Fn malloc ,
@@ -47,8 +46,19 @@ can take over
 and
 .Fn free
 for a single malloc type.
+Alternatively
 .Nm
-can also guard all allocations larger than
+can take over
+.Fn uma_zalloc ,
+.Fn uma_zalloc_arg
+and
+.Fn uma_free
+for a single
+.Xr uma
+zone.
+Also
+.Nm
+can guard all allocations larger than
 .Dv PAGE_SIZE ,
 and can guard a random fraction of all allocations.
 There is also a knob to prevent allocations smaller than a specified
@@ -72,7 +82,10 @@ sysctl vm.memguard.desc=<memory_type>
 .Pp
 Where
 .Ar memory_type
-is a short description of the memory type to monitor.
+can be either a short description of the memory type to monitor,
+either name of
+.Xr uma 9
+zone.
 Only allocations from that
 .Ar memory_type
 made after
@@ -86,13 +99,28 @@ will potentially be guarded once the
 .Xr sysctl 8
 is set.
 Existing guarded allocations will still be properly released by
-.Xr free 9 .
+either
+.Xr free 9
+or
+.Xr uma_zfree 9 ,
+depending on what kind of allocation was taken over.
 .Pp
-The short description of a
+To determine short description of a
 .Xr malloc 9
-type is the second argument to
-.Xr MALLOC_DEFINE 9 ,
-so one has to find it in the kernel source.
+type one can either take it from the first column of
+.Xr vmstat 8 Fl m
+output, or to find it in the kernel source.
+It is the second argument to
+.Xr MALLOC_DEFINE 9
+macro.
+To determine name of
+.Xr uma 9
+zone one can either take it from the first column of
+.Xr vmstat 8 Fl z
+output, or to find it in the kernel source.
+It is the first argument to the
+.Xr uma_zcreate 9
+function.
 .Pp
 The
 .Va vm.memguard.divisor
@@ -142,12 +170,24 @@ or larger if
 .Va vm.memguard.options
 has the 2 bit set.
 This option is off by default.
+By default
+.Nm
+doesn't guard those
+.Xr uma 9
+zones that have been initialized with the
+.Dv UMA_ZONE_NOFREE
+flag set, since it can produce false positives on them.
+However, this safety measure can be turned off by setting bit 3
+of the
+.Va vm.memguard.options
+tunable.
 .Sh SEE ALSO
 .Xr sysctl 8 ,
 .Xr vmstat 8 ,
 .Xr contigmalloc 9 ,
 .Xr malloc 9 ,
-.Xr redzone 9
+.Xr redzone 9 ,
+.Xr uma 9
 .Sh HISTORY
 .Nm
 first appeared in
@@ -161,8 +201,20 @@ This manual page was originally written by
 .An Christian Brueffer Aq brueffer@FreeBSD.org .
 Additions have been made by
 .An Matthew Fleming Aq mdf@FreeBSD.org
+and
+.An Gleb Smirnoff Aq glebius@FreeBSD.org
 to both the implementation and the documentation.
 .Sh BUGS
-Currently, it is not possible to override UMA
-.Xr zone 9
-allocations.
+It is not possible to guard allocations that really expect themselves to be
+allocated from
+.Xr uma 9 ,
+utilizing additional interfaces apart from
+.Fn uma_zalloc
+and
+.Fn uma_free ,
+for example
+.Fn uma_find_refcnt .
+For the moment of writing only
+.Xr mbuf 9
+cluster zones belong to that kind of allocations.
+Attempt to guard them would lead to kernel panic.
diff --git a/share/man/man9/psignal.9 b/share/man/man9/psignal.9
index dd282a64493d..4193bcb50b88 100644
--- a/share/man/man9/psignal.9
+++ b/share/man/man9/psignal.9
@@ -28,14 +28,15 @@
 .\"	$NetBSD: psignal.9,v 1.1 1996/06/22 22:57:35 pk Exp $
 .\" $FreeBSD$
 .\"
-.Dd June 22, 1996
+.Dd October 8, 2011
 .Dt PSIGNAL 9
 .Os
 .Sh NAME
 .Nm psignal ,
 .Nm pgsignal ,
-.Nm gsignal
-.Nd post signal to a process or process group
+.Nm gsignal ,
+.Nm tdsignal
+.Nd post signal to a thread, process, or process group
 .Sh SYNOPSIS
 .In sys/types.h
 .In sys/signalvar.h
@@ -45,8 +46,10 @@
 .Fn pgsignal "struct pgrp *pgrp" "int signum" "int checkctty"
 .Ft void
 .Fn gsignal "int pgid" "int signum"
+.Ft void
+.Fn tdsignal "struct thread *td" "int signum"
 .Sh DESCRIPTION
-These functions post a signal to one or more processes.
+These functions post a signal to a thread or one or more processes.
 The argument
 .Fa signum
 common to all three functions should be in the range
@@ -135,6 +138,13 @@ set to zero.
 If
 .Fa pgid
 is zero no action is taken.
+.Pp
+The
+.Fn tdsignal
+function posts signal number
+.Fa signum
+to the thread represented by the thread structure
+.Fa td .
 .Sh SEE ALSO
 .Xr sigaction 2 ,
 .Xr signal 9 ,
diff --git a/share/man/man9/sleepqueue.9 b/share/man/man9/sleepqueue.9
index 3c0e9a5206e9..22971d801787 100644
--- a/share/man/man9/sleepqueue.9
+++ b/share/man/man9/sleepqueue.9
@@ -351,7 +351,7 @@ argument specifies the thread to awaken and the
 argument specifies the wait channel to awaken it from.
 If the thread
 .Fa td
-is not blocked on the the wait channel
+is not blocked on the wait channel
 .Fa wchan
 then this function will not do anything,
 even if the thread is asleep on a different wait channel.
diff --git a/share/misc/committers-doc.dot b/share/misc/committers-doc.dot
index 3754bb74e0a4..278ccb9b08a0 100644
--- a/share/misc/committers-doc.dot
+++ b/share/misc/committers-doc.dot
@@ -79,12 +79,14 @@ ryusuke [label="Ryusuke Suzuki\nryusuke@FreeBSD.org\n2009/12/21"]
 simon [label="Simon L. Nielsen\nsimon@FreeBSD.org\n2003/07/20"]
 taras [label="Taras Korenko\ntaras@FreeBSD.org\n2010/06/25"]
 trhodes [label="Tom Rhodes\ntrhodes@FreeBSD.org\n2002/03/25"]
+wblock [label="Warren Block\nwblock@FreeBSD.org\n2011/09/12"]
 
 # Here are the mentor/mentee relationships.
 # Group together all the mentees for a particular mentor.
 # Keep the list sorted by mentor login.
 
 bcr -> gavin
+bcr -> wblock
 
 blackend -> ale
 
@@ -105,6 +107,8 @@ gabor -> pgj
 gabor -> manolis
 gabor -> taras
 
+gjb -> wblock
+
 hrs -> ryusuke
 
 jesusr -> jcamou
diff --git a/share/misc/committers-ports.dot b/share/misc/committers-ports.dot
index 602ab3bd2889..2569169b2e3e 100644
--- a/share/misc/committers-ports.dot
+++ b/share/misc/committers-ports.dot
@@ -68,6 +68,7 @@ clement [label="Clement Laforet\nclement@FreeBSD.org\n2003/12/17"]
 clsung [label="Cheng-Lung Sung\nclsung@FreeBSD.org\n2004/8/18"]
 cperciva [label="Colin Percival\ncperciva@FreeBSD.org\n2006/01/31"]
 crees [label="Chris Rees\ncrees@FreeBSD.org\n2011/06/11"]
+cs [label="Carlo Strub\ncs@FreeBSD.org\n2011/09/13"]
 culot [label="Frederic Culot\nculot@FreeBSD.org\n2010/10/16"]
 daichi [label="Daichi Goto\ndaichi@FreeBSD.org\n2002/10/17"]
 danfe [label="Alexey Dokuchaev\ndanfe@FreeBSD.org\n2004/08/20"]
@@ -77,6 +78,7 @@ delphij [label="Xin Li\ndelphij@FreeBSD.org\n2006/05/01"]
 demon [label="Dmitry Sivachenko\ndemon@FreeBSD.org\n2000/11/13"]
 dhn [label="Dennis Herrmann\ndhn@FreeBSD.org\n2009/03/03"]
 dryice [label="Dryice Dong Liu\ndryice@FreeBSD.org\n2006/12/25"]
+eadler [label="Eitan Adler\neadler@FreeBSD.org\n2011/08/17"]
 edwin [label="Edwin Groothuis\nedwin@FreeBSD.org\n2002/10/22"]
 ehaupt [label="Emanuel Haupt\nehaupt@FreeBSD.org\n2005/10/03"]
 eik [label="Oliver Eikemeier\neik@FreeBSD.org\n2003/11/12"]
@@ -152,6 +154,7 @@ pgj [label="Gabor Pali\npgj@FreeBSD.org\n2009/04/12"]
 philip [label="Philip Paeps\nphilip@FreeBSD.org\n2005/10/19"]
 pgollucci [label="Philip M. Gollucci\npgollucci@FreeBSD.org\n2008/07/21"]
 rafan [label="Rong-En Fan\nrafan@FreeBSD.org\n2006/06/23"]
+rakuco [label="Raphael Kubo da Costa\nrakuco@FreeBSD.org\n2011/08/22"]
 rene [label="Rene Ladan\nrene@FreeBSD.org\n2010/04/11"]
 rnoland [label="Robert Noland\nrnoland@FreeBSD.org\n2008/07/21"]
 romain [label="Romain Tartiere\nromain@FreeBSD.org\n2010/01/24"]
@@ -208,6 +211,9 @@ arved -> stefan
 
 asami -> obrien
 
+avilla -> rakuco
+
+bapt -> eadler
 bapt -> jlaffaye
 
 beat -> decke
@@ -273,6 +279,7 @@ garga -> wxs
 garga -> xride
 
 glarkin -> avl
+glarkin -> cs
 
 glewis -> hq
 glewis -> jkim
@@ -323,6 +330,7 @@ marcus -> eik
 marcus -> jmallett
 
 makc -> bf
+makc -> rakuco
 
 mat -> thierry
 
@@ -383,6 +391,7 @@ rafan -> chinsan
 rene -> crees
 
 sahil -> culot
+sahil -> eadler
 
 sat -> beech
 
@@ -421,6 +430,7 @@ thierry -> jadawin
 tmclaugh -> itetcu
 tmclaugh -> xride
 
+wen -> cs
 wen -> culot
 wen -> pawel
 
diff --git a/share/misc/committers-src.dot b/share/misc/committers-src.dot
index 2fce08f0e6b9..8f977753d751 100644
--- a/share/misc/committers-src.dot
+++ b/share/misc/committers-src.dot
@@ -70,6 +70,7 @@ onoe [label="Atsushi Onoe\nonoe@FreeBSD.org\n2000/07/21\n2008/11/10"]
 rgrimes [label="Rod Grimes\nrgrimes@FreeBSD.org\n1993/06/12\n2003/03/08"]
 rink [label="Rink Springer\nrink@FreeBSD.org\n2006/01/16\n2010/11/04"]
 robert [label="Robert Drehmel\nrobert@FreeBSD.org\n2001/08/23\n2006/05/13"]
+rmh [label="Robert Millan\nrmh@FreeBSD.org\n2011/09/18"]
 rpaulo [label="Rui Paulo\nrpaulo@FreeBSD.org\n2007/09/25\n2010/12/03"]
 sah [label="Sam Hopkins\nsah@FreeBSD.org\n2004/12/15\n2008/11/10"]
 shafeeq [label="Shafeeq Sinnamohideen\nshafeeq@FreeBSD.org\n2000/06/19\n2006/04/06"]
@@ -140,6 +141,7 @@ gad [label="Garance A. Drosehn\ngad@FreeBSD.org\n2000/10/27"]
 gallatin [label="Andrew Gallatin\ngallatin@FreeBSD.org\n????/??/??"]
 gavin [label="Gavin Atkinson\ngavin@FreeBSD.org\n2009/12/07"]
 gibbs [label="Justin T. Gibbs\ngibbs@FreeBSD.org\n????/??/??"]
+gleb [label="Gleb Kurtsou\ngleb@FreeBSD.org\n2011/09/19"]
 glebius [label="Gleb Smirnoff\nglebius@FreeBSD.org\n2004/07/14"]
 gnn [label="George V. Neville-Neil\ngnn@FreeBSD.org\n2004/10/11"]
 gordon [label="Gordon Tetlow\ngordon@FreeBSD.org\n2002/05/17"]
@@ -152,6 +154,7 @@ iedowse [label="Ian Dowse\niedowse@FreeBSD.org\n2000/12/01"]
 imp [label="Warner Losh\nimp@FreeBSD.org\n1996/09/20"]
 ivoras [label="Ivan Voras\nivoras@FreeBSD.org\n2008/06/10"]
 jamie [label="Jamie Gritton\njamie@FreeBSD.org\n2009/01/28"]
+jceel [label="Jakub Klama\njceel@FreeBSD.org\n2011/09/25"]
 jchandra [label="Jayachandran C.\njchandra@FreeBSD.org\n2010/05/19"]
 jeff [label="Jeff Roberson\njeff@FreeBSD.org\n2002/02/21"]
 jh [label="Jaakko Heinonen\njh@FreeBSD.org\n2009/10/02"]
@@ -184,6 +187,7 @@ mav [label="Alexander Motin\nmav@FreeBSD.org\n2007/04/12"]
 maxim [label="Maxim Konovalov\nmaxim@FreeBSD.org\n2002/02/07"]
 mdf [label="Matthew Fleming\nmdf@FreeBSD.org\n2010/06/04"]
 mdodd [label="Matthew N. Dodd\nmdodd@FreeBSD.org\n1999/07/27"]
+melifaro [label="Alexander V. Chernikov\nmelifaro@FreeBSD.org\n2011/10/04"]
 mjacob [label="Matt Jacob\nmjacob@FreeBSD.org\n1997/08/13"]
 mlaier [label="Max Laier\nmlaier@FreeBSD.org\n2004/02/10"]
 mr [label="Michael Reifenberger\nmr@FreeBSD.org\n2001/09/30"]
@@ -270,6 +274,9 @@ day1 -> alm
 day1 -> dg
 
 adrian -> ray
+adrian -> rmh
+
+ae -> melifaro
 
 andre -> qingli
 
@@ -301,6 +308,7 @@ bz -> anchie
 bz -> jamie
 bz -> syrinx
 
+cognet -> jceel
 cognet -> kevlo
 
 cperciva -> flz
@@ -436,10 +444,12 @@ kan -> kib
 kib -> ae
 kib -> dchagin
 kib -> lulf
+kib -> melifaro
 kib -> pho
 kib -> pluknet
 kib -> rdivacky
 kib -> rmacklem
+kib -> rmh
 kib -> stas
 kib -> tijl
 kib -> trociny
@@ -455,6 +465,8 @@ markm -> sheldonh
 
 mav -> ae
 
+mdf -> gleb
+
 mdodd -> jake
 
 mike -> das
@@ -566,6 +578,8 @@ ume -> tshiozak
 
 wes -> scf
 
+wkoszek -> jceel
+
 wollman -> gad
 
 zml -> mdf
diff --git a/share/misc/iso3166 b/share/misc/iso3166
index 23c9bd65e312..8b7533b79ab8 100644
--- a/share/misc/iso3166
+++ b/share/misc/iso3166
@@ -46,7 +46,7 @@ BJ	BEN	204	Benin
 BM	BMU	060	Bermuda
 BT	BTN	064	Bhutan
 BO	BOL	068	Bolivia, Plurinational State of
-BQ	BES	535	Bonaire, Saint Eustatius and Saba
+BQ	BES	535	Bonaire, Sint Eustatius and Saba
 BA	BIH	070	Bosnia and Herzegovina
 BW	BWA	072	Botswana
 BV	BVT	074	Bouvet Island
@@ -224,11 +224,12 @@ SB	SLB	090	Solomon Islands
 SO	SOM	706	Somalia
 ZA	ZAF	710	South Africa
 GS	SGS	239	South Georgia and the South Sandwich Islands
+SS	SSD	728	South Sudan
 ES	ESP	724	Spain
 LK	LKA	144	Sri Lanka
 SH	SHN	654	Saint Helena, Ascension and Tristan da Cunha
 PM	SPM	666	Saint Pierre and Miquelon
-SD	SDN	736	Sudan
+SD	SDN	729	Sudan
 SR	SUR	740	Suriname
 SJ	SJM	744	Svalbard and Jan Mayen
 SZ	SWZ	748	Swaziland
@@ -547,3 +548,10 @@ ZW	ZWE	716	Zimbabwe
 #  BONAIRE, SAINT EUSTATIUS AND SABA (BQ), CURACAO (CW) and
 #  SINT MAARTEN (DUTCH PART) (SX) added as new entries.
 #  NETHERLANDS ANTILLES (AN) removed.
+#
+# Newsletter VI-9 2011-06-12
+#  Name changes for Fiji and Myanmar as well as other minor corrections
+#
+# Newsletter VI-10 2011-08-09
+#  Code elements for South Sudan.
+#
diff --git a/share/misc/scsi_modes b/share/misc/scsi_modes
index 687bcfa93533..f6116e81485b 100644
--- a/share/misc/scsi_modes
+++ b/share/misc/scsi_modes
@@ -50,19 +50,32 @@
 # ALL DEVICE TYPES
 
 0x0a "Control Mode Page" {
-	{Reserved} *t7
+	{TST} t3
+	{TMF_ONLY} t1
+	{DPICZ} t1
+	{D_SENSE} t1
+	{GLTSD} t1
 	{RLEC} t1
 	{Queue Algorithm Modifier} t4
-	{Reserved} *t2
-	{QErr} t1
+	{NUAR} t1
+	{QErr} t2
 	{DQue} t1
 	{EECA} t1
-	{Reserved} *t4
+	{RAC} t1
+	{UA_INTLCK_CTRL} t2
+	{SWP} t1
 	{RAENP} t1
 	{UAAENP} t1
 	{EAENP} t1
-	{Reserved} *i1
+	{ATO} t1
+	{TAS} t1
+	{ATMPE} t1
+	{RWWP} t1
+	{Reserved} *t1
+	{Autoload Mode} t3
 	{Ready AEN Holdoff Period} i2
+	{Busy Timeout Period} i2
+	{Extended Self-Test Completion Time} i2
 }
 
 0x02 "Disconnect-Reconnect Page" {
diff --git a/share/mk/bsd.dep.mk b/share/mk/bsd.dep.mk
index c452528da947..1d4a2c5ecea1 100644
--- a/share/mk/bsd.dep.mk
+++ b/share/mk/bsd.dep.mk
@@ -125,8 +125,8 @@ depend: beforedepend ${DEPENDFILE} afterdepend
 
 # Different types of sources are compiled with slightly different flags.
 # Split up the sources, and filter out headers and non-applicable flags.
-MKDEP_CFLAGS=	${CFLAGS:M-nostdinc*} ${CFLAGS:M-[BIDU]*}
-MKDEP_CXXFLAGS=	${CXXFLAGS:M-nostdinc*} ${CXXFLAGS:M-[BIDU]*}
+MKDEP_CFLAGS=	${CFLAGS:M-nostdinc*} ${CFLAGS:M-[BIDU]*} ${CFLAGS:M-std=*} ${CFLAGS:M-ansi}
+MKDEP_CXXFLAGS=	${CXXFLAGS:M-nostdinc*} ${CXXFLAGS:M-[BIDU]*} ${CXXFLAGS:M-std=*} ${CXXFLAGS:M-ansi}
 
 DPSRCS+= ${SRCS}
 ${DEPENDFILE}: ${DPSRCS}
diff --git a/share/mk/bsd.own.mk b/share/mk/bsd.own.mk
index ee786199019d..3058970e1f85 100644
--- a/share/mk/bsd.own.mk
+++ b/share/mk/bsd.own.mk
@@ -434,8 +434,9 @@ __DEFAULT_YES_OPTIONS+=CLANG
 .else
 __DEFAULT_NO_OPTIONS+=CLANG
 .endif
-# FDT is needed only for arm and powerpc (and not powerpc64)
-.if ${__T} == "arm" || ${__T} == "armeb" || ${__T} == "powerpc"
+# FDT is needed only for arm, mips and powerpc (and not powerpc64)
+.if ${__T} == "arm" || ${__T} == "armeb" || ${__T} == "powerpc" || \
+    ${__T:Mmips*}
 __DEFAULT_YES_OPTIONS+=FDT
 .else
 __DEFAULT_NO_OPTIONS+=FDT
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 1e7b26c081f4..081880037f10 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -421,7 +421,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
  * MPSAFE
  */
 int
-sigreturn(td, uap)
+sys_sigreturn(td, uap)
 	struct thread *td;
 	struct sigreturn_args /* {
 		const struct __ucontext *sigcntxp;
@@ -517,7 +517,7 @@ int
 freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
 {
  
-	return sigreturn(td, (struct sigreturn_args *)uap);
+	return sys_sigreturn(td, (struct sigreturn_args *)uap);
 }
 #endif
 
@@ -609,7 +609,7 @@ void
 cpu_halt(void)
 {
 	for (;;)
-		__asm__ ("hlt");
+		halt();
 }
 
 void (*cpu_idle_hook)(void) = NULL;	/* ACPI idle hook. */
@@ -630,6 +630,8 @@ cpu_idle_acpi(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_SLEEPING;
+
+	/* See comments in cpu_idle_hlt(). */
 	disable_intr();
 	if (sched_runnable())
 		enable_intr();
@@ -647,9 +649,22 @@ cpu_idle_hlt(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_SLEEPING;
+
 	/*
-	 * We must absolutely guarentee that hlt is the next instruction
-	 * after sti or we introduce a timing window.
+	 * Since we may be in a critical section from cpu_idle(), if
+	 * an interrupt fires during that critical section we may have
+	 * a pending preemption.  If the CPU halts, then that thread
+	 * may not execute until a later interrupt awakens the CPU.
+	 * To handle this race, check for a runnable thread after
+	 * disabling interrupts and immediately return if one is
+	 * found.  Also, we must absolutely guarentee that hlt is
+	 * the next instruction after sti.  This ensures that any
+	 * interrupt that fires after the call to disable_intr() will
+	 * immediately awaken the CPU from hlt.  Finally, please note
+	 * that on x86 this works fine because of interrupts enabled only
+	 * after the instruction following sti takes place, while IF is set
+	 * to 1 immediately, allowing hlt instruction to acknowledge the
+	 * interrupt.
 	 */
 	disable_intr();
 	if (sched_runnable())
@@ -675,11 +690,19 @@ cpu_idle_mwait(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_MWAIT;
-	if (!sched_runnable()) {
-		cpu_monitor(state, 0, 0);
-		if (*state == STATE_MWAIT)
-			cpu_mwait(0, MWAIT_C1);
+
+	/* See comments in cpu_idle_hlt(). */
+	disable_intr();
+	if (sched_runnable()) {
+		enable_intr();
+		*state = STATE_RUNNING;
+		return;
 	}
+	cpu_monitor(state, 0, 0);
+	if (*state == STATE_MWAIT)
+		__asm __volatile("sti; mwait" : : "a" (MWAIT_C1), "c" (0));
+	else
+		enable_intr();
 	*state = STATE_RUNNING;
 }
 
@@ -691,6 +714,12 @@ cpu_idle_spin(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_RUNNING;
+
+	/*
+	 * The sched_runnable() call is racy but as long as there is
+	 * a loop missing it one time will have just a little impact if any
+	 * (and it is much better than missing the check at all).
+	 */
 	for (i = 0; i < 1000; i++) {
 		if (sched_runnable())
 			return;
diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c
index 17eada1b540c..bcd7fa35d9dc 100644
--- a/sys/amd64/amd64/sys_machdep.c
+++ b/sys/amd64/amd64/sys_machdep.c
@@ -203,6 +203,10 @@ sysarch(td, uap)
 
 		case I386_SET_IOPERM:
 		default:
+#ifdef KTRACE
+			if (KTRPOINT(td, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_SYSCALL, 0, 0);
+#endif
 			return (ECAPMODE);
 		}
 	}
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index 16f151fce3ef..9c72a693a0a4 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -674,6 +674,19 @@ trap_pfault(frame, usermode)
 			goto nogo;
 
 		map = &vm->vm_map;
+
+		/*
+		 * When accessing a usermode address, kernel must be
+		 * ready to accept the page fault, and provide a
+		 * handling routine.  Since accessing the address
+		 * without the handler is a bug, do not try to handle
+		 * it normally, and panic immediately.
+		 */
+		if (!usermode && (td->td_intr_nesting_level != 0 ||
+		    PCPU_GET(curpcb)->pcb_onfault == NULL)) {
+			trap_fatal(frame, eva);
+			return (-1);
+		}
 	}
 
 	/*
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC
index 036c1b281230..43c20f3ffd07 100644
--- a/sys/amd64/conf/GENERIC
+++ b/sys/amd64/conf/GENERIC
@@ -152,6 +152,7 @@ device		mlx		# Mylex DAC960 family
 #XXX pointer/int warnings
 #device		pst		# Promise Supertrak SX6000
 device		twe		# 3ware ATA RAID
+device		tws		# LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller
 
 # atkbdc0 controls both the keyboard and the PS/2 mouse
 device		atkbdc		# AT keyboard controller
@@ -331,7 +332,8 @@ device		zyd		# ZyDAS zd1211/zd1211b wireless NICs
 
 # FireWire support
 device		firewire	# FireWire bus code
-device		sbp		# SCSI over FireWire (Requires scbus and da)
+# sbp(4) works for some systems but causes boot failure on others
+#device		sbp		# SCSI over FireWire (Requires scbus and da)
 device		fwe		# Ethernet over FireWire (non-standard!)
 device		fwip		# IP over FireWire (RFC 2734,3146)
 device		dcons		# Dumb console driver
diff --git a/sys/amd64/include/ieeefp.h b/sys/amd64/include/ieeefp.h
index 124449279d02..a40366092d57 100644
--- a/sys/amd64/include/ieeefp.h
+++ b/sys/amd64/include/ieeefp.h
@@ -39,6 +39,8 @@
 #define _MACHINE_IEEEFP_H_
 
 /*
+ * Deprecated historical FPU control interface
+ *
  * IEEE floating point type, constant and function definitions.
  * XXX: {FP,SSE}*FLD and {FP,SSE}*OFF are undocumented pollution.
  */
@@ -287,13 +289,16 @@ __fpgetsticky(void)
 #define	fpsetprec(m)	__fpsetprec(m)
 #define	fpsetround(m)	__fpsetround(m)
 
-/* Suppress prototypes in the MI header. */
-#define	_IEEEFP_INLINED_	1
-
 #else /* !(!__IEEEFP_NOINLINES__ && __GNUCLIKE_ASM) */
 
 /* Augment the userland declarations. */
 __BEGIN_DECLS
+extern fp_rnd_t    fpgetround(void);
+extern fp_rnd_t    fpsetround(fp_rnd_t);
+extern fp_except_t fpgetmask(void);
+extern fp_except_t fpsetmask(fp_except_t);
+extern fp_except_t fpgetsticky(void);
+extern fp_except_t fpsetsticky(fp_except_t);
 fp_prec_t	fpgetprec(void);
 fp_prec_t	fpsetprec(fp_prec_t);
 __END_DECLS
diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h
index 2de7a9e698bf..5207d89acede 100644
--- a/sys/amd64/include/proc.h
+++ b/sys/amd64/include/proc.h
@@ -85,8 +85,6 @@ struct syscall_args {
 	register_t args[8];
 	int narg;
 };
-#define	HAVE_SYSCALL_ARGS_DEF 1
-
 #endif  /* _KERNEL */
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/amd64/linux32/linux32_machdep.c b/sys/amd64/linux32/linux32_machdep.c
index a8ebe7ef1629..abcdd3ebe63e 100644
--- a/sys/amd64/linux32/linux32_machdep.c
+++ b/sys/amd64/linux32/linux32_machdep.c
@@ -658,7 +658,7 @@ linux_mmap_common(struct thread *td, l_uintptr_t addr, l_size_t len, l_int prot,
 		    (void *)bsd_args.addr, (int)bsd_args.len, bsd_args.prot,
 		    bsd_args.flags, bsd_args.fd, (int)bsd_args.pos);
 #endif
-	error = mmap(td, &bsd_args);
+	error = sys_mmap(td, &bsd_args);
 #ifdef DEBUG
 	if (ldebug(mmap))
 		printf("-> %s() return: 0x%x (0x%08x)\n",
@@ -677,7 +677,7 @@ linux_mprotect(struct thread *td, struct linux_mprotect_args *uap)
 	bsd_args.prot = uap->prot;
 	if (bsd_args.prot & (PROT_READ | PROT_WRITE | PROT_EXEC))
 		bsd_args.prot |= PROT_READ | PROT_EXEC;
-	return (mprotect(td, &bsd_args));
+	return (sys_mprotect(td, &bsd_args));
 }
 
 int
@@ -863,7 +863,7 @@ linux_ftruncate64(struct thread *td, struct linux_ftruncate64_args *args)
 
 	sa.fd = args->fd;
 	sa.length = args->length;
-	return ftruncate(td, &sa);
+	return sys_ftruncate(td, &sa);
 }
 
 int
diff --git a/sys/amd64/linux32/linux32_sysent.c b/sys/amd64/linux32/linux32_sysent.c
index c1a864f261f8..333911677cc0 100644
--- a/sys/amd64/linux32/linux32_sysent.c
+++ b/sys/amd64/linux32/linux32_sysent.c
@@ -20,12 +20,12 @@
 struct sysent linux_sysent[] = {
 #define	nosys	linux_nosys
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 0 = setup */
-	{ AS(sys_exit_args), (sy_call_t *)sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
+	{ AS(sys_exit_args), (sy_call_t *)sys_sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
 	{ 0, (sy_call_t *)linux_fork, AUE_FORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 2 = linux_fork */
-	{ AS(read_args), (sy_call_t *)read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
-	{ AS(write_args), (sy_call_t *)write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
+	{ AS(read_args), (sy_call_t *)sys_read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
+	{ AS(write_args), (sy_call_t *)sys_write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
 	{ AS(linux_open_args), (sy_call_t *)linux_open, AUE_OPEN_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = linux_open */
-	{ AS(close_args), (sy_call_t *)close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
+	{ AS(close_args), (sy_call_t *)sys_close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
 	{ AS(linux_waitpid_args), (sy_call_t *)linux_waitpid, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = linux_waitpid */
 	{ AS(linux_creat_args), (sy_call_t *)linux_creat, AUE_CREAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 8 = linux_creat */
 	{ AS(linux_link_args), (sy_call_t *)linux_link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = linux_link */
@@ -55,12 +55,12 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_access_args), (sy_call_t *)linux_access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = linux_access */
 	{ AS(linux_nice_args), (sy_call_t *)linux_nice, AUE_NICE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = linux_nice */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 35 = ftime */
-	{ 0, (sy_call_t *)sync, AUE_SYNC, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
+	{ 0, (sy_call_t *)sys_sync, AUE_SYNC, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
 	{ AS(linux_kill_args), (sy_call_t *)linux_kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = linux_kill */
 	{ AS(linux_rename_args), (sy_call_t *)linux_rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 38 = linux_rename */
 	{ AS(linux_mkdir_args), (sy_call_t *)linux_mkdir, AUE_MKDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = linux_mkdir */
 	{ AS(linux_rmdir_args), (sy_call_t *)linux_rmdir, AUE_RMDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 40 = linux_rmdir */
-	{ AS(dup_args), (sy_call_t *)dup, AUE_DUP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
+	{ AS(dup_args), (sy_call_t *)sys_dup, AUE_DUP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
 	{ AS(linux_pipe_args), (sy_call_t *)linux_pipe, AUE_PIPE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 42 = linux_pipe */
 	{ AS(linux_times_args), (sy_call_t *)linux_times, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = linux_times */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 44 = prof */
@@ -70,22 +70,22 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_signal_args), (sy_call_t *)linux_signal, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 48 = linux_signal */
 	{ 0, (sy_call_t *)linux_geteuid16, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 49 = linux_geteuid16 */
 	{ 0, (sy_call_t *)linux_getegid16, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = linux_getegid16 */
-	{ AS(acct_args), (sy_call_t *)acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
+	{ AS(acct_args), (sy_call_t *)sys_acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
 	{ AS(linux_umount_args), (sy_call_t *)linux_umount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 52 = linux_umount */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 53 = lock */
 	{ AS(linux_ioctl_args), (sy_call_t *)linux_ioctl, AUE_IOCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 54 = linux_ioctl */
 	{ AS(linux_fcntl_args), (sy_call_t *)linux_fcntl, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = linux_fcntl */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 56 = mpx */
-	{ AS(setpgid_args), (sy_call_t *)setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = setpgid */
+	{ AS(setpgid_args), (sy_call_t *)sys_setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = setpgid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 58 = ulimit */
 	{ 0, (sy_call_t *)linux_olduname, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = linux_olduname */
-	{ AS(umask_args), (sy_call_t *)umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
-	{ AS(chroot_args), (sy_call_t *)chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
+	{ AS(umask_args), (sy_call_t *)sys_umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
+	{ AS(chroot_args), (sy_call_t *)sys_chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
 	{ AS(linux_ustat_args), (sy_call_t *)linux_ustat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 62 = linux_ustat */
-	{ AS(dup2_args), (sy_call_t *)dup2, AUE_DUP2, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = dup2 */
+	{ AS(dup2_args), (sy_call_t *)sys_dup2, AUE_DUP2, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = dup2 */
 	{ 0, (sy_call_t *)linux_getppid, AUE_GETPPID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 64 = linux_getppid */
-	{ 0, (sy_call_t *)getpgrp, AUE_GETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 65 = getpgrp */
-	{ 0, (sy_call_t *)setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 66 = setsid */
+	{ 0, (sy_call_t *)sys_getpgrp, AUE_GETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 65 = getpgrp */
+	{ 0, (sy_call_t *)sys_setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 66 = setsid */
 	{ AS(linux_sigaction_args), (sy_call_t *)linux_sigaction, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 67 = linux_sigaction */
 	{ 0, (sy_call_t *)linux_sgetmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 68 = linux_sgetmask */
 	{ AS(linux_ssetmask_args), (sy_call_t *)linux_ssetmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 69 = linux_ssetmask */
@@ -106,17 +106,17 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_lstat_args), (sy_call_t *)linux_lstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 84 = linux_lstat */
 	{ AS(linux_readlink_args), (sy_call_t *)linux_readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = linux_readlink */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 86 = linux_uselib */
-	{ AS(swapon_args), (sy_call_t *)swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = swapon */
+	{ AS(swapon_args), (sy_call_t *)sys_swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = swapon */
 	{ AS(linux_reboot_args), (sy_call_t *)linux_reboot, AUE_REBOOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 88 = linux_reboot */
 	{ AS(linux_readdir_args), (sy_call_t *)linux_readdir, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = linux_readdir */
 	{ AS(linux_mmap_args), (sy_call_t *)linux_mmap, AUE_MMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = linux_mmap */
-	{ AS(munmap_args), (sy_call_t *)munmap, AUE_MUNMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 91 = munmap */
+	{ AS(munmap_args), (sy_call_t *)sys_munmap, AUE_MUNMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 91 = munmap */
 	{ AS(linux_truncate_args), (sy_call_t *)linux_truncate, AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 92 = linux_truncate */
 	{ AS(linux_ftruncate_args), (sy_call_t *)linux_ftruncate, AUE_FTRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 93 = linux_ftruncate */
-	{ AS(fchmod_args), (sy_call_t *)fchmod, AUE_FCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 94 = fchmod */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_FCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = fchown */
+	{ AS(fchmod_args), (sy_call_t *)sys_fchmod, AUE_FCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 94 = fchmod */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_FCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = fchown */
 	{ AS(linux_getpriority_args), (sy_call_t *)linux_getpriority, AUE_GETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 96 = linux_getpriority */
-	{ AS(setpriority_args), (sy_call_t *)setpriority, AUE_SETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = setpriority */
+	{ AS(setpriority_args), (sy_call_t *)sys_setpriority, AUE_SETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = setpriority */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 98 = profil */
 	{ AS(linux_statfs_args), (sy_call_t *)linux_statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 99 = linux_statfs */
 	{ AS(linux_fstatfs_args), (sy_call_t *)linux_fstatfs, AUE_FSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 100 = linux_fstatfs */
@@ -137,7 +137,7 @@ struct sysent linux_sysent[] = {
 	{ 0, (sy_call_t *)linux_swapoff, AUE_SWAPOFF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 115 = linux_swapoff */
 	{ AS(linux_sysinfo_args), (sy_call_t *)linux_sysinfo, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 116 = linux_sysinfo */
 	{ AS(linux_ipc_args), (sy_call_t *)linux_ipc, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 117 = linux_ipc */
-	{ AS(fsync_args), (sy_call_t *)fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = fsync */
+	{ AS(fsync_args), (sy_call_t *)sys_fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = fsync */
 	{ AS(linux_sigreturn_args), (sy_call_t *)linux_sigreturn, AUE_SIGRETURN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 119 = linux_sigreturn */
 	{ AS(linux_clone_args), (sy_call_t *)linux_clone, AUE_RFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 120 = linux_clone */
 	{ AS(linux_setdomainname_args), (sy_call_t *)linux_setdomainname, AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 121 = linux_setdomainname */
@@ -151,8 +151,8 @@ struct sysent linux_sysent[] = {
 	{ 0, (sy_call_t *)linux_delete_module, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 129 = linux_delete_module */
 	{ 0, (sy_call_t *)linux_get_kernel_syms, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 130 = linux_get_kernel_syms */
 	{ 0, (sy_call_t *)linux_quotactl, AUE_QUOTACTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 131 = linux_quotactl */
-	{ AS(getpgid_args), (sy_call_t *)getpgid, AUE_GETPGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = getpgid */
-	{ AS(fchdir_args), (sy_call_t *)fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 133 = fchdir */
+	{ AS(getpgid_args), (sy_call_t *)sys_getpgid, AUE_GETPGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = getpgid */
+	{ AS(fchdir_args), (sy_call_t *)sys_fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 133 = fchdir */
 	{ 0, (sy_call_t *)linux_bdflush, AUE_BDFLUSH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 134 = linux_bdflush */
 	{ AS(linux_sysfs_args), (sy_call_t *)linux_sysfs, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 135 = linux_sysfs */
 	{ AS(linux_personality_args), (sy_call_t *)linux_personality, AUE_PERSONALITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = linux_personality */
@@ -162,22 +162,22 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_llseek_args), (sy_call_t *)linux_llseek, AUE_LSEEK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 140 = linux_llseek */
 	{ AS(linux_getdents_args), (sy_call_t *)linux_getdents, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 141 = linux_getdents */
 	{ AS(linux_select_args), (sy_call_t *)linux_select, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 142 = linux_select */
-	{ AS(flock_args), (sy_call_t *)flock, AUE_FLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 143 = flock */
+	{ AS(flock_args), (sy_call_t *)sys_flock, AUE_FLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 143 = flock */
 	{ AS(linux_msync_args), (sy_call_t *)linux_msync, AUE_MSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 144 = linux_msync */
 	{ AS(linux_readv_args), (sy_call_t *)linux_readv, AUE_READV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 145 = linux_readv */
 	{ AS(linux_writev_args), (sy_call_t *)linux_writev, AUE_WRITEV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 146 = linux_writev */
 	{ AS(linux_getsid_args), (sy_call_t *)linux_getsid, AUE_GETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 147 = linux_getsid */
 	{ AS(linux_fdatasync_args), (sy_call_t *)linux_fdatasync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 148 = linux_fdatasync */
 	{ AS(linux_sysctl_args), (sy_call_t *)linux_sysctl, AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 149 = linux_sysctl */
-	{ AS(mlock_args), (sy_call_t *)mlock, AUE_MLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 150 = mlock */
-	{ AS(munlock_args), (sy_call_t *)munlock, AUE_MUNLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 151 = munlock */
-	{ AS(mlockall_args), (sy_call_t *)mlockall, AUE_MLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 152 = mlockall */
-	{ 0, (sy_call_t *)munlockall, AUE_MUNLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 153 = munlockall */
-	{ AS(sched_setparam_args), (sy_call_t *)sched_setparam, AUE_SCHED_SETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 154 = sched_setparam */
-	{ AS(sched_getparam_args), (sy_call_t *)sched_getparam, AUE_SCHED_GETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 155 = sched_getparam */
+	{ AS(mlock_args), (sy_call_t *)sys_mlock, AUE_MLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 150 = mlock */
+	{ AS(munlock_args), (sy_call_t *)sys_munlock, AUE_MUNLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 151 = munlock */
+	{ AS(mlockall_args), (sy_call_t *)sys_mlockall, AUE_MLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 152 = mlockall */
+	{ 0, (sy_call_t *)sys_munlockall, AUE_MUNLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 153 = munlockall */
+	{ AS(sched_setparam_args), (sy_call_t *)sys_sched_setparam, AUE_SCHED_SETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 154 = sched_setparam */
+	{ AS(sched_getparam_args), (sy_call_t *)sys_sched_getparam, AUE_SCHED_GETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 155 = sched_getparam */
 	{ AS(linux_sched_setscheduler_args), (sy_call_t *)linux_sched_setscheduler, AUE_SCHED_SETSCHEDULER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 156 = linux_sched_setscheduler */
 	{ AS(linux_sched_getscheduler_args), (sy_call_t *)linux_sched_getscheduler, AUE_SCHED_GETSCHEDULER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 157 = linux_sched_getscheduler */
-	{ 0, (sy_call_t *)sched_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = sched_yield */
+	{ 0, (sy_call_t *)sys_sched_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = sched_yield */
 	{ AS(linux_sched_get_priority_max_args), (sy_call_t *)linux_sched_get_priority_max, AUE_SCHED_GET_PRIORITY_MAX, NULL, 0, 0, 0, SY_THR_STATIC },	/* 159 = linux_sched_get_priority_max */
 	{ AS(linux_sched_get_priority_min_args), (sy_call_t *)linux_sched_get_priority_min, AUE_SCHED_GET_PRIORITY_MIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 160 = linux_sched_get_priority_min */
 	{ AS(linux_sched_rr_get_interval_args), (sy_call_t *)linux_sched_rr_get_interval, AUE_SCHED_RR_GET_INTERVAL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = linux_sched_rr_get_interval */
@@ -187,7 +187,7 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_getresuid16_args), (sy_call_t *)linux_getresuid16, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 165 = linux_getresuid16 */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 166 = vm86 */
 	{ 0, (sy_call_t *)linux_query_module, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 167 = linux_query_module */
-	{ AS(poll_args), (sy_call_t *)poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 168 = poll */
+	{ AS(poll_args), (sy_call_t *)sys_poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 168 = poll */
 	{ 0, (sy_call_t *)linux_nfsservctl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 169 = linux_nfsservctl */
 	{ AS(linux_setresgid16_args), (sy_call_t *)linux_setresgid16, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 170 = linux_setresgid16 */
 	{ AS(linux_getresgid16_args), (sy_call_t *)linux_getresgid16, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 171 = linux_getresgid16 */
@@ -220,25 +220,25 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_lchown_args), (sy_call_t *)linux_lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 198 = linux_lchown */
 	{ 0, (sy_call_t *)linux_getuid, AUE_GETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 199 = linux_getuid */
 	{ 0, (sy_call_t *)linux_getgid, AUE_GETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 200 = linux_getgid */
-	{ 0, (sy_call_t *)geteuid, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 201 = geteuid */
-	{ 0, (sy_call_t *)getegid, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = getegid */
-	{ AS(setreuid_args), (sy_call_t *)setreuid, AUE_SETREUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = setreuid */
-	{ AS(setregid_args), (sy_call_t *)setregid, AUE_SETREGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 204 = setregid */
+	{ 0, (sy_call_t *)sys_geteuid, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 201 = geteuid */
+	{ 0, (sy_call_t *)sys_getegid, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = getegid */
+	{ AS(setreuid_args), (sy_call_t *)sys_setreuid, AUE_SETREUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = setreuid */
+	{ AS(setregid_args), (sy_call_t *)sys_setregid, AUE_SETREGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 204 = setregid */
 	{ AS(linux_getgroups_args), (sy_call_t *)linux_getgroups, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 205 = linux_getgroups */
 	{ AS(linux_setgroups_args), (sy_call_t *)linux_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 206 = linux_setgroups */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 207 = fchown */
-	{ AS(setresuid_args), (sy_call_t *)setresuid, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 208 = setresuid */
-	{ AS(getresuid_args), (sy_call_t *)getresuid, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 209 = getresuid */
-	{ AS(setresgid_args), (sy_call_t *)setresgid, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 210 = setresgid */
-	{ AS(getresgid_args), (sy_call_t *)getresgid, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 211 = getresgid */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 207 = fchown */
+	{ AS(setresuid_args), (sy_call_t *)sys_setresuid, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 208 = setresuid */
+	{ AS(getresuid_args), (sy_call_t *)sys_getresuid, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 209 = getresuid */
+	{ AS(setresgid_args), (sy_call_t *)sys_setresgid, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 210 = setresgid */
+	{ AS(getresgid_args), (sy_call_t *)sys_getresgid, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 211 = getresgid */
 	{ AS(linux_chown_args), (sy_call_t *)linux_chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 212 = linux_chown */
-	{ AS(setuid_args), (sy_call_t *)setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 213 = setuid */
-	{ AS(setgid_args), (sy_call_t *)setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 214 = setgid */
+	{ AS(setuid_args), (sy_call_t *)sys_setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 213 = setuid */
+	{ AS(setgid_args), (sy_call_t *)sys_setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 214 = setgid */
 	{ AS(linux_setfsuid_args), (sy_call_t *)linux_setfsuid, AUE_SETFSUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 215 = linux_setfsuid */
 	{ AS(linux_setfsgid_args), (sy_call_t *)linux_setfsgid, AUE_SETFSGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 216 = linux_setfsgid */
 	{ AS(linux_pivot_root_args), (sy_call_t *)linux_pivot_root, AUE_PIVOT_ROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 217 = linux_pivot_root */
 	{ AS(linux_mincore_args), (sy_call_t *)linux_mincore, AUE_MINCORE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 218 = linux_mincore */
-	{ AS(madvise_args), (sy_call_t *)madvise, AUE_MADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 219 = madvise */
+	{ AS(madvise_args), (sy_call_t *)sys_madvise, AUE_MADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 219 = madvise */
 	{ AS(linux_getdents64_args), (sy_call_t *)linux_getdents64, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 220 = linux_getdents64 */
 	{ AS(linux_fcntl64_args), (sy_call_t *)linux_fcntl64, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 221 = linux_fcntl64 */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 222 =  */
diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c
index 5ea5615893f3..6aec18be0428 100644
--- a/sys/arm/arm/elf_machdep.c
+++ b/sys/arm/arm/elf_machdep.c
@@ -76,7 +76,7 @@ struct sysentvec elf32_freebsd_sysvec = {
 	.sv_maxssiz	= NULL,
 	.sv_flags	= SV_ABI_FREEBSD | SV_ILP32,
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
-	.sv_fetch_syscall_args = NULL, /* XXXKIB */
+	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
 	.sv_schedtail	= NULL,
 };
diff --git a/sys/arm/arm/elf_trampoline.c b/sys/arm/arm/elf_trampoline.c
index 4adb0252c3dc..945f653ad832 100644
--- a/sys/arm/arm/elf_trampoline.c
+++ b/sys/arm/arm/elf_trampoline.c
@@ -22,6 +22,13 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+/*
+ * Since we are compiled outside of the normal kernel build process, we
+ * need to include opt_global.h manually.
+ */
+#include "opt_global.h"
+#include "opt_kernname.h"
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 #include <machine/asm.h>
@@ -33,13 +40,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpufunc.h>
 #include <machine/armreg.h>
 
-/*
- * Since we are compiled outside of the normal kernel build process, we
- * need to include opt_global.h manually.
- */
-#include "opt_global.h"
-#include "opt_kernname.h"
-
 extern char kernel_start[];
 extern char kernel_end[];
 
diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c
index 2d26f685105d..a7a792d26789 100644
--- a/sys/arm/arm/machdep.c
+++ b/sys/arm/arm/machdep.c
@@ -312,6 +312,8 @@ cpu_startup(void *dummy)
 	m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_ZERO);
 	pmap_kenter_user(ARM_TP_ADDRESS, VM_PAGE_TO_PHYS(m));
 #endif
+	*(uint32_t *)ARM_RAS_START = 0;
+	*(uint32_t *)ARM_RAS_END = 0xffffffff;
 }
 
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
@@ -605,7 +607,7 @@ set_mcontext(struct thread *td, const mcontext_t *mcp)
  * MPSAFE
  */
 int
-sigreturn(td, uap)
+sys_sigreturn(td, uap)
 	struct thread *td;
 	struct sigreturn_args /* {
 		const struct __ucontext *sigcntxp;
diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c
index 28835ec9362d..ec7fb1bd5e3f 100644
--- a/sys/arm/arm/pmap.c
+++ b/sys/arm/arm/pmap.c
@@ -185,6 +185,9 @@ int pmap_debug_level = 0;
 #endif  /* PMAP_DEBUG */
 
 extern struct pv_addr systempage;
+
+extern int last_fault_code;
+
 /*
  * Internal function prototypes
  */
@@ -2054,9 +2057,8 @@ pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user)
 	 * the TLB.
 	 */
 	if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) {
-		extern int last_fault_code;
 		printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n",
-		    pm, va, ftype);
+		    pm, (u_long)va, ftype);
 		printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n",
 		    l2, l2b, ptep, pl1pd);
 		printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n",
@@ -4012,13 +4014,6 @@ pmap_zero_page_generic(vm_paddr_t phys, int off, int size)
 	char *dstpg;
 #endif
 
-#ifdef DEBUG
-	struct vm_page *pg = PHYS_TO_VM_PAGE(phys);
-
-	if (pg->md.pvh_list != NULL)
-		panic("pmap_zero_page: page has mappings");
-#endif
-
 	if (_arm_bzero && size >= _min_bzero_size &&
 	    _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0)
 		return;
@@ -4290,13 +4285,6 @@ pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst)
 #if 0
 	struct vm_page *src_pg = PHYS_TO_VM_PAGE(src);
 #endif
-#ifdef DEBUG
-	struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst);
-
-	if (dst_pg->md.pvh_list != NULL)
-		panic("pmap_copy_page: dst page has mappings");
-#endif
-
 
 	/*
 	 * Clean the source page.  Hold the source page's lock for
@@ -4342,13 +4330,6 @@ pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst)
 	/* XXX: Only needed for pmap_clean_page(), which is commented out. */
 	struct vm_page *src_pg = PHYS_TO_VM_PAGE(src);
 #endif
-#ifdef DEBUG
-	struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst);
-
-	if (dst_pg->md.pvh_list != NULL)
-		panic("pmap_copy_page: dst page has mappings");
-#endif
-
 
 	/*
 	 * Clean the source page.  Hold the source page's lock for
diff --git a/sys/arm/arm/sys_machdep.c b/sys/arm/arm/sys_machdep.c
index 4c3e350f56a0..5fe7b5c03d74 100644
--- a/sys/arm/arm/sys_machdep.c
+++ b/sys/arm/arm/sys_machdep.c
@@ -122,6 +122,10 @@ sysarch(td, uap)
 			break;
 
 		default:
+#ifdef KTRACE
+			if (KTRPOINT(td, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_SYSCALL, 0, 0);
+#endif
 			return (ECAPMODE);
 		}
 	}
diff --git a/sys/arm/arm/trap.c b/sys/arm/arm/trap.c
index c7c84d0394d3..e56f0ffded86 100644
--- a/sys/arm/arm/trap.c
+++ b/sys/arm/arm/trap.c
@@ -861,98 +861,68 @@ badaddr_read(void *addr, size_t size, void *rptr)
 	return (rv);
 }
 
-#define MAXARGS	8
+int
+cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
+{
+	struct proc *p;
+	register_t *ap;
+	int error;
+
+	sa->code = sa->insn & 0x000fffff;
+	ap = &td->td_frame->tf_r0;
+	if (sa->code == SYS_syscall) {
+		sa->code = *ap++;
+		sa->nap--;
+	} else if (sa->code == SYS___syscall) {
+		sa->code = ap[_QUAD_LOWWORD];
+		sa->nap -= 2;
+		ap += 2;
+	}
+	p = td->td_proc;
+	if (p->p_sysent->sv_mask)
+		sa->code &= p->p_sysent->sv_mask;
+	if (sa->code >= p->p_sysent->sv_size)
+		sa->callp = &p->p_sysent->sv_table[0];
+	else
+		sa->callp = &p->p_sysent->sv_table[sa->code];
+	sa->narg = sa->callp->sy_narg;
+	error = 0;
+	memcpy(sa->args, ap, sa->nap * sizeof(register_t));
+	if (sa->narg > sa->nap) {
+		error = copyin((void *)td->td_frame->tf_usr_sp, sa->args +
+		    sa->nap, (sa->narg - sa->nap) * sizeof(register_t));
+	}
+	if (error == 0) {
+		td->td_retval[0] = 0;
+		td->td_retval[1] = 0;
+	}
+	return (error);
+}
+
+#include "../../kern/subr_syscall.c"
+
 static void
 syscall(struct thread *td, trapframe_t *frame, u_int32_t insn)
 {
-	struct proc *p = td->td_proc;
-	int code, error;
-	u_int nap, nargs;
-	register_t *ap, *args, copyargs[MAXARGS];
-	struct sysent *callp;
+	struct syscall_args sa;
+	int error;
 
-	PCPU_INC(cnt.v_syscall);
-	td->td_pticks = 0;
-	if (td->td_ucred != td->td_proc->p_ucred)
-		cred_update_thread(td);
+	td->td_frame = frame;
+	sa.insn = insn;
 	switch (insn & SWI_OS_MASK) {
 	case 0: /* XXX: we need our own one. */
-		nap = 4;
+		sa.nap = 4;
 		break;
 	default:
 		call_trapsignal(td, SIGILL, 0);
 		userret(td, frame);
 		return;
 	}
-	code = insn & 0x000fffff;                
-	td->td_pticks = 0;
-	ap = &frame->tf_r0;
-	if (code == SYS_syscall) {
-		code = *ap++;
-		
-		nap--;
-	} else if (code == SYS___syscall) {
-		code = ap[_QUAD_LOWWORD];
-		nap -= 2;
-		ap += 2;
-	}
-	if (p->p_sysent->sv_mask)
-		code &= p->p_sysent->sv_mask;
-	if (code >= p->p_sysent->sv_size)
-		callp = &p->p_sysent->sv_table[0];
-	else
-		callp = &p->p_sysent->sv_table[code];
-	nargs = callp->sy_narg;
-	memcpy(copyargs, ap, nap * sizeof(register_t));
-	if (nargs > nap) {
-		error = copyin((void *)frame->tf_usr_sp, copyargs + nap,
-		    (nargs - nap) * sizeof(register_t));
-		if (error)
-			goto bad;
-	}
-	args = copyargs;
-	error = 0;
-#ifdef KTRACE
-	if (KTRPOINT(td, KTR_SYSCALL))
-		ktrsyscall(code, nargs, args);
-#endif
-		
-	CTR4(KTR_SYSC, "syscall enter thread %p pid %d proc %s code %d", td,
-	    td->td_proc->p_pid, td->td_name, code);
-	if (error == 0) {
-		td->td_retval[0] = 0;
-		td->td_retval[1] = 0;
-		STOPEVENT(p, S_SCE, callp->sy_narg);
-		PTRACESTOP_SC(p, td, S_PT_SCE);
-		AUDIT_SYSCALL_ENTER(code, td);
-		error = (*callp->sy_call)(td, args);
-		AUDIT_SYSCALL_EXIT(error, td);
-		KASSERT(td->td_ar == NULL, 
-		    ("returning from syscall with td_ar set!"));
-	}
-bad:
-	cpu_set_syscall_retval(td, error);
 
-	WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
-	    (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
-	KASSERT(td->td_critnest == 0,
-	    ("System call %s returning in a critical section",
-	    (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"));
-	KASSERT(td->td_locks == 0,
-	    ("System call %s returning with %d locks held",
-	    (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???",
-	    td->td_locks));
-
-	userret(td, frame);
-	CTR4(KTR_SYSC, "syscall exit thread %p pid %d proc %s code %d", td,
-	    td->td_proc->p_pid, td->td_name, code);
-	
-	STOPEVENT(p, S_SCX, code);
-	PTRACESTOP_SC(p, td, S_PT_SCX);
-#ifdef KTRACE
-      	if (KTRPOINT(td, KTR_SYSRET))
-		ktrsysret(code, error, td->td_retval[0]);
-#endif
+	error = syscallenter(td, &sa);
+	KASSERT(error != 0 || td->td_ar == NULL,
+	    ("returning from syscall with td_ar set!"));
+	syscallret(td, error, &sa);
 }
 
 void
diff --git a/sys/arm/at91/at91_mci.c b/sys/arm/at91/at91_mci.c
index 17cf9c25efd6..4f88d7e2fb24 100644
--- a/sys/arm/at91/at91_mci.c
+++ b/sys/arm/at91/at91_mci.c
@@ -243,7 +243,7 @@ at91_mci_attach(device_t dev)
 	child = device_add_child(dev, "mmc", 0);
 	device_set_ivars(dev, &sc->host);
 	err = bus_generic_attach(dev);
-out:;
+out:
 	if (err)
 		at91_mci_deactivate(dev);
 	return (err);
diff --git a/sys/arm/at91/at91_pio.c b/sys/arm/at91/at91_pio.c
index 853e58803828..53dfddbf8d10 100644
--- a/sys/arm/at91/at91_pio.c
+++ b/sys/arm/at91/at91_pio.c
@@ -162,7 +162,7 @@ at91_pio_attach(device_t dev)
 		goto out;
 	}
 	sc->cdev->si_drv1 = sc;
-out:;
+out:
 	if (err)
 		at91_pio_deactivate(dev);
 	return (err);
diff --git a/sys/arm/at91/at91_rtc.c b/sys/arm/at91/at91_rtc.c
index e1a8f25979d4..934fd22807e3 100644
--- a/sys/arm/at91/at91_rtc.c
+++ b/sys/arm/at91/at91_rtc.c
@@ -118,7 +118,7 @@ at91_rtc_attach(device_t dev)
 		goto out;
 	}
 	clock_register(dev, 1000000);
-out:;
+out:
 	if (err)
 		at91_rtc_deactivate(dev);
 	return (err);
diff --git a/sys/arm/at91/at91_spi.c b/sys/arm/at91/at91_spi.c
index 2252f00c8d25..e5bf52f4e0fd 100644
--- a/sys/arm/at91/at91_spi.c
+++ b/sys/arm/at91/at91_spi.c
@@ -134,7 +134,7 @@ at91_spi_attach(device_t dev)
 
 	device_add_child(dev, "spibus", -1);
 	bus_generic_attach(dev);
-out:;
+out:
 	if (err)
 		at91_spi_deactivate(dev);
 	return (err);
@@ -259,7 +259,7 @@ at91_spi_transfer(device_t dev, device_t child, struct spi_command *cmd)
 	for (j = 0; j < i; j++)
 		bus_dmamap_unload(sc->dmatag, sc->map[j]);
 	return (err);
-out:;
+out:
 	for (j = 0; j < i; j++)
 		bus_dmamap_unload(sc->dmatag, sc->map[j]);
 	return (EIO);
diff --git a/sys/arm/at91/at91_ssc.c b/sys/arm/at91/at91_ssc.c
index e6b1c7ffed4e..12ae90ae305a 100644
--- a/sys/arm/at91/at91_ssc.c
+++ b/sys/arm/at91/at91_ssc.c
@@ -150,7 +150,7 @@ at91_ssc_attach(device_t dev)
 	WR4(sc, SSC_TFMR,
 	    0x1f | SSC_TFMR_DATDEF | SSC_TFMR_MSFBF | SSC_TFMR_FSOS_NEG_PULSE);
 
-out:;
+out:
 	if (err)
 		at91_ssc_deactivate(dev);
 	return (err);
diff --git a/sys/arm/at91/at91_twi.c b/sys/arm/at91/at91_twi.c
index fdd377a55298..a70fed012230 100644
--- a/sys/arm/at91/at91_twi.c
+++ b/sys/arm/at91/at91_twi.c
@@ -139,7 +139,7 @@ at91_twi_attach(device_t dev)
 		device_printf(dev, "could not allocate iicbus instance\n");
 	/* probe and attach the iicbus */
 	bus_generic_attach(dev);
-out:;
+out:
 	if (err)
 		at91_twi_deactivate(dev);
 	return (err);
@@ -365,7 +365,7 @@ at91_twi_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs)
 		if ((err = at91_twi_wait(sc, TWI_SR_TXCOMP)))
 			break;
 	}
-out:;
+out:
 	if (err) {
 		WR4(sc, TWI_CR, TWI_CR_SWRST);
 		WR4(sc, TWI_CR, TWI_CR_MSEN | TWI_CR_SVDIS);
diff --git a/sys/arm/at91/uart_dev_at91usart.c b/sys/arm/at91/uart_dev_at91usart.c
index f3d21aae5a0f..d1a52bac40aa 100644
--- a/sys/arm/at91/uart_dev_at91usart.c
+++ b/sys/arm/at91/uart_dev_at91usart.c
@@ -406,7 +406,7 @@ at91_usart_bus_attach(struct uart_softc *sc)
 		WR4(&sc->sc_bas, USART_IER, USART_CSR_RXRDY);
 	}
 	WR4(&sc->sc_bas, USART_IER, USART_CSR_RXBRK);
-errout:;
+errout:
 	// XXX bad
 	return (err);
 }
diff --git a/sys/arm/econa/if_ece.c b/sys/arm/econa/if_ece.c
index a7433f24215c..5169101704b3 100644
--- a/sys/arm/econa/if_ece.c
+++ b/sys/arm/econa/if_ece.c
@@ -441,7 +441,7 @@ ece_attach(device_t dev)
 	taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq",
 	    device_get_nameunit(sc->dev));
 
-out:;
+out:
 	if (err)
 		ece_deactivate(dev);
 	if (err && ifp)
diff --git a/sys/arm/include/asmacros.h b/sys/arm/include/asmacros.h
index 5b0f31776700..f06841f8e516 100644
--- a/sys/arm/include/asmacros.h
+++ b/sys/arm/include/asmacros.h
@@ -71,9 +71,8 @@
 	ldr	r0, =ARM_RAS_START;					   \
 	mov	r1, #0;							   \
 	str	r1, [r0];						   \
-	ldr	r0, =ARM_RAS_END;					   \
 	mov	r1, #0xffffffff;					   \
-	str	r1, [r0];
+	str	r1, [r0, #4];
 
 /*
  * PULLFRAME - macro to pull a trap frame from the stack in the current mode
@@ -120,20 +119,19 @@
 	stmia	r0, {r13-r14}^;		/* Push the user mode registers */ \
         mov     r0, r0;                 /* NOP for previous instruction */ \
 	ldr	r5, =ARM_RAS_START;	/* Check if there's any RAS */	   \
-	ldr	r3, [r5];						   \
-	cmp	r3, #0;			/* Is the update needed ? */	   \
-	ldrgt	lr, [r0, #16];						   \
-	ldrgt	r1, =ARM_RAS_END;					   \
-	ldrgt	r4, [r1];		/* Get the end of the RAS */	   \
-	movgt	r2, #0;			/* Reset the magic addresses */	   \
-	strgt	r2, [r5];						   \
-	movgt	r2, #0xffffffff;					   \
-	strgt	r2, [r1];						   \
-	cmpgt	lr, r3;			/* Were we in the RAS ? */	   \
-	cmpgt	r4, lr;							   \
-	strgt	r3, [r0, #16];		/* Yes, update the pc */	   \
-	mrs	r0, spsr_all;		/* Put the SPSR on the stack */	   \
-	str	r0, [sp, #-4]!
+	ldr     r4, [r5, #4];           /* reset it to point at the     */ \
+	cmp     r4, #0xffffffff;        /* end of memory if necessary;  */ \
+	movne   r1, #0xffffffff;        /* leave value in r4 for later  */ \
+	strne   r1, [r5, #4];           /* comparision against PC.      */ \
+	ldr     r3, [r5];               /* Retrieve global RAS_START    */ \
+	cmp     r3, #0;                 /* and reset it if non-zero.    */ \
+	movne   r1, #0;                 /* If non-zero RAS_START and    */ \
+	strne   r1, [r5];               /* PC was lower than RAS_END,   */ \
+	ldrne   r1, [r0, #16];          /* adjust the saved PC so that  */ \
+	cmpne   r4, r1;                 /* execution later resumes at   */ \
+	strhi   r3, [r0, #16];          /* the RAS_START location.      */ \
+	mrs     r0, spsr_all;                                              \
+	str     r0, [sp, #-4]!
 
 /*
  * PULLFRAMEFROMSVCANDEXIT - macro to pull a trap frame from the stack
diff --git a/sys/arm/include/ieeefp.h b/sys/arm/include/ieeefp.h
index 8fc9811ce60f..0b61744b450b 100644
--- a/sys/arm/include/ieeefp.h
+++ b/sys/arm/include/ieeefp.h
@@ -8,6 +8,8 @@
 #ifndef _MACHINE_IEEEFP_H_
 #define _MACHINE_IEEEFP_H_
 
+/* Deprecated historical FPU control interface */
+
 /* FP exception codes */
 #define FP_EXCEPT_INV	0
 #define FP_EXCEPT_DZ	1
diff --git a/sys/arm/include/proc.h b/sys/arm/include/proc.h
index 7032be5a984c..c1bce0019b8a 100644
--- a/sys/arm/include/proc.h
+++ b/sys/arm/include/proc.h
@@ -62,4 +62,14 @@ struct mdproc {
 
 #define	KINFO_PROC_SIZE 792
 
+#define MAXARGS	8
+struct syscall_args {
+	u_int code;
+	struct sysent *callp;
+	register_t args[MAXARGS];
+	int narg;
+	u_int nap;
+	u_int32_t insn;
+};
+
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/arm/include/sysarch.h b/sys/arm/include/sysarch.h
index 12db4df6e248..ba337bd4eb27 100644
--- a/sys/arm/include/sysarch.h
+++ b/sys/arm/include/sysarch.h
@@ -42,9 +42,13 @@
  * The ARM_TP_ADDRESS points to a special purpose page, which is used as local
  * store for the ARM per-thread data and Restartable Atomic Sequences support.
  * Put it just above the "high" vectors' page.
- * the cpu_switch() code assumes ARM_RAS_START is ARM_TP_ADDRESS + 4, and
+ * The cpu_switch() code assumes ARM_RAS_START is ARM_TP_ADDRESS + 4, and
  * ARM_RAS_END is ARM_TP_ADDRESS + 8, so if that ever changes, be sure to
  * update the cpu_switch() (and cpu_throw()) code as well.
+ * In addition, code in arm/include/atomic.h and arm/include/asmacros.h
+ * assumes that ARM_RAS_END is at ARM_RAS_START+4, so be sure to update those
+ * if ARM_RAS_END moves in relation to ARM_RAS_START (look for occurrances
+ * of ldr/str rm,[rn, #4]).
  */
 #define ARM_TP_ADDRESS		(ARM_VECTORS_HIGH + 0x1000)
 #define ARM_RAS_START		(ARM_TP_ADDRESS + 4)
diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h
index dd1ba1c46cf2..f85fa58cfeb3 100644
--- a/sys/arm/include/vmparam.h
+++ b/sys/arm/include/vmparam.h
@@ -141,11 +141,21 @@
 #endif
 
 #define MAXTSIZ 	(16*1024*1024)
+#ifndef DFLDSIZ
 #define DFLDSIZ         (128*1024*1024)
+#endif
+#ifndef MAXDSIZ
 #define MAXDSIZ         (512*1024*1024)
+#endif
+#ifndef DFLSSIZ
 #define DFLSSIZ         (2*1024*1024)
+#endif
+#ifndef MAXSSIZ
 #define MAXSSIZ         (8*1024*1024)
+#endif
+#ifndef SGROWSIZ
 #define SGROWSIZ        (128*1024)
+#endif
 
 #ifdef ARM_USE_SMALL_ALLOC
 #define UMA_MD_SMALL_ALLOC
diff --git a/sys/arm/mv/common.c b/sys/arm/mv/common.c
index 09da857ecfed..4596a20b93d6 100644
--- a/sys/arm/mv/common.c
+++ b/sys/arm/mv/common.c
@@ -49,9 +49,6 @@ __FBSDID("$FreeBSD$");
 
 #define MAX_CPU_WIN	5
 
-#define DEBUG
-#undef DEBUG
-
 #ifdef DEBUG
 #define debugf(fmt, args...) do { printf("%s(): ", __func__);	\
     printf(fmt,##args); } while (0)
diff --git a/sys/arm/mv/mv_machdep.c b/sys/arm/mv/mv_machdep.c
index fdcef9ce9657..fd176925aa0c 100644
--- a/sys/arm/mv/mv_machdep.c
+++ b/sys/arm/mv/mv_machdep.c
@@ -91,9 +91,6 @@ __FBSDID("$FreeBSD$");
 #include <arm/mv/mvvar.h>	/* XXX eventually this should be eliminated */
 #include <arm/mv/mvwin.h>
 
-#define DEBUG
-#undef DEBUG
-
 #ifdef  DEBUG
 #define debugf(fmt, args...) printf(fmt, ##args)
 #else
diff --git a/sys/arm/xscale/ixp425/avila_gpio.c b/sys/arm/xscale/ixp425/avila_gpio.c
index b88bb28f9ca6..c3deda9f6086 100644
--- a/sys/arm/xscale/ixp425/avila_gpio.c
+++ b/sys/arm/xscale/ixp425/avila_gpio.c
@@ -356,5 +356,10 @@ static driver_t gpio_avila_driver = {
 	sizeof(struct avila_gpio_softc),
 };
 static devclass_t gpio_avila_devclass;
+extern devclass_t gpiobus_devclass, gpioc_devclass;
+extern driver_t gpiobus_driver, gpioc_driver;
 
 DRIVER_MODULE(gpio_avila, ixp, gpio_avila_driver, gpio_avila_devclass, 0, 0);
+DRIVER_MODULE(gpiobus, gpio_avila, gpiobus_driver, gpiobus_devclass, 0, 0);
+DRIVER_MODULE(gpioc, gpio_avila, gpioc_driver, gpioc_devclass, 0, 0);
+MODULE_VERSION(gpio_avila, 1);
diff --git a/sys/arm/xscale/ixp425/cambria_gpio.c b/sys/arm/xscale/ixp425/cambria_gpio.c
index 509cdc35b31f..24d6b8666b9c 100644
--- a/sys/arm/xscale/ixp425/cambria_gpio.c
+++ b/sys/arm/xscale/ixp425/cambria_gpio.c
@@ -84,6 +84,7 @@ struct cambria_gpio_softc {
         struct mtx		sc_mtx;
 	struct gpio_pin		sc_pins[GPIO_PINS];
 	uint8_t			sc_latch;
+	uint8_t			sc_val;
 };
 
 struct cambria_gpio_pin {
@@ -95,11 +96,11 @@ struct cambria_gpio_pin {
 extern struct ixp425_softc *ixp425_softc;
 
 static struct cambria_gpio_pin cambria_gpio_pins[GPIO_PINS] = {
-	{ "GPIO0", 0, GPIO_PIN_OUTPUT },
-	{ "GPIO1", 1, GPIO_PIN_OUTPUT },
-	{ "GPIO2", 2, GPIO_PIN_OUTPUT },
-	{ "GPIO3", 3, GPIO_PIN_OUTPUT },
-	{ "GPIO4", 4, GPIO_PIN_OUTPUT },
+	{ "PLD0", 0, GPIO_PIN_OUTPUT },
+	{ "PLD1", 1, GPIO_PIN_OUTPUT },
+	{ "PLD2", 2, GPIO_PIN_OUTPUT },
+	{ "PLD3", 3, GPIO_PIN_OUTPUT },
+	{ "PLD4", 4, GPIO_PIN_OUTPUT },
 };
 
 /*
@@ -309,6 +310,9 @@ cambria_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags)
 {
 	struct cambria_gpio_softc *sc = device_get_softc(dev);
 	int error;
+	uint8_t mask;
+
+	mask = 1 << pin;
 
 	if (pin >= GPIO_PINS)
 		return (EINVAL);
@@ -325,7 +329,16 @@ cambria_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags)
 	GPIO_LOCK(sc);
 	sc->sc_pins[pin].gp_flags = flags;
 
-	sc->sc_latch |= (1 << pin);
+	/*
+	 * Writing a logical one sets the signal high and writing a logical
+	 * zero sets the signal low. To configure a digital I/O signal as an
+	 * input, a logical one must first be written to the data bit to
+	 * three-state the associated output.
+	 */
+	if (flags & GPIO_PIN_INPUT || sc->sc_val & mask)
+		sc->sc_latch |= mask; /* input or output & high */
+	else
+		sc->sc_latch &= ~mask;
 	error = cambria_gpio_write(sc);
 	GPIO_UNLOCK(sc);
 
@@ -337,15 +350,28 @@ cambria_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value)
 {
 	struct cambria_gpio_softc *sc = device_get_softc(dev);
 	int error;
+	uint8_t mask;
 
-	if (pin >= GPIO_PINS || sc->sc_pins[pin].gp_flags != GPIO_PIN_OUTPUT)
+	mask = 1 << pin;
+
+	if (pin >= GPIO_PINS)
 		return (EINVAL);
-
 	GPIO_LOCK(sc);
 	if (value)
-		sc->sc_latch |= (1 << pin);
+		sc->sc_val |= mask;
 	else
-		sc->sc_latch &= ~(1 << pin);
+		sc->sc_val &= ~mask;
+
+	if (sc->sc_pins[pin].gp_flags != GPIO_PIN_OUTPUT) {
+		/* just save, altering the latch will disable input */
+		GPIO_UNLOCK(sc);
+		return (0);
+	}
+
+	if (value)
+		sc->sc_latch |= mask;
+	else
+		sc->sc_latch &= ~mask;
 	error = cambria_gpio_write(sc);
 	GPIO_UNLOCK(sc);
 
@@ -375,14 +401,17 @@ static int
 cambria_gpio_pin_toggle(device_t dev, uint32_t pin)
 {
 	struct cambria_gpio_softc *sc = device_get_softc(dev);
-	int error;
+	int error = 0;
 
-	if (pin >= GPIO_PINS || sc->sc_pins[pin].gp_flags != GPIO_PIN_OUTPUT)
+	if (pin >= GPIO_PINS)
 		return (EINVAL);
 
 	GPIO_LOCK(sc);
-	sc->sc_latch ^= (1 << pin);
-	error = cambria_gpio_write(sc);
+	sc->sc_val ^= (1 << pin);
+	if (sc->sc_pins[pin].gp_flags == GPIO_PIN_OUTPUT) {
+		sc->sc_latch ^= (1 << pin);
+		error = cambria_gpio_write(sc);
+	}
 	GPIO_UNLOCK(sc);
 
 	return (error);
diff --git a/sys/boot/arm/at91/boot2/boot2.c b/sys/boot/arm/at91/boot2/boot2.c
index 97308887e8ba..e585adaf174d 100644
--- a/sys/boot/arm/at91/boot2/boot2.c
+++ b/sys/boot/arm/at91/boot2/boot2.c
@@ -64,7 +64,8 @@ __FBSDID("$FreeBSD$");
 			OPT_SET(RBX_VERBOSE) | \
 			OPT_SET(RBX_GDB))
 
-#define PATH_CONFIG	"/boot.config"
+#define PATH_DOTCONFIG	"/boot.config"
+#define PATH_CONFIG	"/boot/config"
 //#define PATH_KERNEL	"/boot/kernel/kernel"
 #define PATH_KERNEL	"/boot/kernel/kernel.gz.tramp"
 
@@ -160,7 +161,8 @@ main(void)
 	autoboot = 1;
 
 	/* Process configuration file */
-	if ((ino = lookup(PATH_CONFIG)))
+	if ((ino = lookup(PATH_CONFIG)) ||
+	    (ino = lookup(PATH_DOTCONFIG)))
 		fsread(ino, cmd, sizeof(cmd));
 
 	if (*cmd) {
diff --git a/sys/boot/arm/ixp425/boot2/boot2.c b/sys/boot/arm/ixp425/boot2/boot2.c
index 8f99e0d9b9c1..3d356f090651 100644
--- a/sys/boot/arm/ixp425/boot2/boot2.c
+++ b/sys/boot/arm/ixp425/boot2/boot2.c
@@ -62,7 +62,8 @@ __FBSDID("$FreeBSD$");
 			OPT_SET(RBX_VERBOSE) | \
 			OPT_SET(RBX_GDB))
 
-#define PATH_CONFIG	"/boot.config"
+#define PATH_DOTCONFIG	"/boot.config"
+#define PATH_CONFIG	"/boot/config"
 #define PATH_KERNEL	"/boot/kernel/kernel"
 
 extern uint32_t _end;
@@ -86,7 +87,7 @@ static unsigned dsk_start;
 static char cmd[512];
 static char kname[1024];
 static uint32_t opts;
-static int dsk_meta;
+static uint8_t dsk_meta;
 static int bootslice;
 static int bootpart;
 static int disk_layout;
@@ -168,7 +169,8 @@ main(void)
 	autoboot = 1;
 
 	/* Process configuration file */
-	if ((ino = lookup(PATH_CONFIG)))
+	if ((ino = lookup(PATH_CONFIG)) ||
+	    (ino = lookup(PATH_DOTCONFIG)))
 		fsread(ino, cmd, sizeof(cmd));
 
 	if (*cmd) {
diff --git a/sys/boot/common/disk.c b/sys/boot/common/disk.c
index e1de3821d02a..81902c0f79aa 100644
--- a/sys/boot/common/disk.c
+++ b/sys/boot/common/disk.c
@@ -776,10 +776,11 @@ disk_open(struct disk_devdesc *dev)
 
 #ifdef LOADER_GPT_SUPPORT
 	rc = disk_opengpt(dev);
-	if (rc)
+	if (rc == 0)
+		return (0);
 #endif
 #ifdef LOADER_MBR_SUPPORT
-		rc = disk_openmbr(dev);
+	rc = disk_openmbr(dev);
 #endif
 
 	return (rc);
diff --git a/sys/boot/fdt/dts/xlp-basic.dts b/sys/boot/fdt/dts/xlp-basic.dts
new file mode 100644
index 000000000000..808fea818ad1
--- /dev/null
+++ b/sys/boot/fdt/dts/xlp-basic.dts
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2010 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Netlogic Microsystems XLP8xx Device Tree Source.
+ *
+ * $FreeBSD$
+ */
+
+/dts-v1/;
+
+/ {
+	model = "netl,XLP8XX";
+	compatible = "XLP8XX";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	aliases {
+		serial0 = &serial0;
+	};
+
+	soc {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		ranges = <0x0>;
+		bus-frequency = <0>;
+
+		serial0: serial@30100 {
+			compatible = "ns16550";
+			reg = <0x30100 0x200>;
+			reg-shift = <2>;
+			current-speed = <115200>;
+			clock-frequency = <133000000>;
+			interrupts = <9>;
+		};
+
+	};
+
+	chosen {
+		stdin = "serial0";
+		stdout = "serial0";
+		cpumask = <0xffffffff>;
+		bootargs = "-v"; 
+	};
+};
diff --git a/sys/boot/i386/boot2/boot2.c b/sys/boot/i386/boot2/boot2.c
index 93fd8392cb53..360392431be1 100644
--- a/sys/boot/i386/boot2/boot2.c
+++ b/sys/boot/i386/boot2/boot2.c
@@ -24,7 +24,6 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/bootinfo.h>
 #include <machine/elf.h>
-#include <machine/psl.h>
 
 #include <stdarg.h>
 
@@ -75,7 +74,8 @@ __FBSDID("$FreeBSD$");
 			OPT_SET(RBX_GDB ) | OPT_SET(RBX_MUTE) | \
 			OPT_SET(RBX_PAUSE) | OPT_SET(RBX_DUAL))
 
-#define PATH_CONFIG	"/boot.config"
+#define PATH_DOTCONFIG	"/boot.config"
+#define PATH_CONFIG	"/boot/config"
 #define PATH_BOOT3	"/boot/loader"
 #define PATH_KERNEL	"/boot/kernel/kernel"
 
@@ -84,8 +84,6 @@ __FBSDID("$FreeBSD$");
 #define NDEV		3
 #define MEM_BASE	0x12
 #define MEM_EXT 	0x15
-#define V86_CY(x)	((x) & PSL_C)
-#define V86_ZR(x)	((x) & PSL_Z)
 
 #define DRV_HARD	0x80
 #define DRV_MASK	0x7f
@@ -241,7 +239,8 @@ main(void)
 
     autoboot = 1;
 
-    if ((ino = lookup(PATH_CONFIG)))
+    if ((ino = lookup(PATH_CONFIG)) ||
+        (ino = lookup(PATH_DOTCONFIG)))
 	fsread(ino, cmd, sizeof(cmd));
 
     if (*cmd) {
diff --git a/sys/boot/i386/btx/lib/btxv86.h b/sys/boot/i386/btx/lib/btxv86.h
index 1152f8b1d963..27f6b3404248 100644
--- a/sys/boot/i386/btx/lib/btxv86.h
+++ b/sys/boot/i386/btx/lib/btxv86.h
@@ -21,6 +21,7 @@
 #define _BTXV86_H_
 
 #include <sys/types.h>
+#include <machine/psl.h>
 
 #define V86_ADDR   0x10000	/* Segment:offset address */
 #define V86_CALLF  0x20000	/* Emulate far call */
@@ -57,6 +58,9 @@ extern u_int32_t	__args;
 #define	VTOPSEG(va)	(u_int16_t)(VTOP((caddr_t)va) >> 4)
 #define	VTOPOFF(va)	(u_int16_t)(VTOP((caddr_t)va) & 0xf)
 
+#define	V86_CY(x)	((x) & PSL_C)
+#define	V86_ZR(x)	((x) & PSL_Z)
+
 void __exit(int) __attribute__((__noreturn__));
 void __exec(caddr_t, ...);
 
diff --git a/sys/boot/i386/common/cons.c b/sys/boot/i386/common/cons.c
index 9384f43e9f96..97019c654255 100644
--- a/sys/boot/i386/common/cons.c
+++ b/sys/boot/i386/common/cons.c
@@ -27,8 +27,6 @@ __FBSDID("$FreeBSD$");
 #include "util.h"
 #include "cons.h"
 
-#define	V86_ZR(x)	((x) & PSL_Z)
-
 #define SECOND		18	/* Circa that many ticks in a second. */
 
 uint8_t ioctrl = IO_KEYBOARD;
diff --git a/sys/boot/i386/common/drv.c b/sys/boot/i386/common/drv.c
index 11f6628d7cf7..52933d5d5254 100644
--- a/sys/boot/i386/common/drv.c
+++ b/sys/boot/i386/common/drv.c
@@ -19,53 +19,41 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 
-#include <machine/psl.h>
-
 #include <btxv86.h>
 
 #include "rbx.h"
 #include "util.h"
 #include "drv.h"
+#include "edd.h"
 #ifdef USE_XREAD
 #include "xreadorg.h"
 #endif
 
-#define	V86_CY(x)	((x) & PSL_C)
-#define	V86_ZR(x)	((x) & PSL_Z)
-
 #ifdef GPT
+static struct edd_params params;
+
 uint64_t
 drvsize(struct dsk *dskp)
 {
-	unsigned char params[0x42];
-	uint64_t sectors;
-
-	*(uint32_t *)params = sizeof(params);
 
+	params.len = sizeof(struct edd_params);
 	v86.ctl = V86_FLAGS;
 	v86.addr = 0x13;
 	v86.eax = 0x4800;
 	v86.edx = dskp->drive;
-	v86.ds = VTOPSEG(params);
-	v86.esi = VTOPOFF(params);
+	v86.ds = VTOPSEG(&params);
+	v86.esi = VTOPOFF(&params);
 	v86int();
 	if (V86_CY(v86.efl)) {
 		printf("error %u\n", v86.eax >> 8 & 0xff);
 		return (0);
 	}
-	memcpy(&sectors, params + 0x10, sizeof(sectors));
-	return (sectors);
+	return (params.sectors);
 }
 #endif	/* GPT */
 
 #ifndef USE_XREAD
-static struct {
-	uint16_t	len;
-	uint16_t	count;
-	uint16_t	off;
-	uint16_t	seg;
-	uint64_t	lba;
-} packet;
+static struct edd_packet packet;
 #endif
 
 int
@@ -76,7 +64,7 @@ drvread(struct dsk *dskp, void *buf, daddr_t lba, unsigned nblk)
 	if (!OPT_CHECK(RBX_QUIET))
 		printf("%c\b", c = c << 8 | c >> 24);
 #ifndef USE_XREAD
-	packet.len = 0x10;
+	packet.len = sizeof(struct edd_packet);
 	packet.count = nblk;
 	packet.off = VTOPOFF(buf);
 	packet.seg = VTOPSEG(buf);
@@ -110,7 +98,7 @@ int
 drvwrite(struct dsk *dskp, void *buf, daddr_t lba, unsigned nblk)
 {
 
-	packet.len = 0x10;
+	packet.len = sizeof(struct edd_packet);
 	packet.count = nblk;
 	packet.off = VTOPOFF(buf);
 	packet.seg = VTOPSEG(buf);
diff --git a/sys/boot/i386/common/edd.h b/sys/boot/i386/common/edd.h
new file mode 100644
index 000000000000..a33ce8f5f78c
--- /dev/null
+++ b/sys/boot/i386/common/edd.h
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 2011 Advanced Computing Technologies LLC
+ * Written by: John H. Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_EDD_H_
+#define	_EDD_H_
+
+/* Supported interfaces for "Check Extensions Present". */
+#define	EDD_INTERFACE_FIXED_DISK	0x01
+#define	EDD_INTERFACE_EJECT		0x02
+#define	EDD_INTERFACE_EDD		0x04
+
+struct edd_packet {
+	uint16_t	len;
+	uint16_t	count;
+	uint16_t	off;
+	uint16_t	seg;
+	uint64_t	lba;
+};
+
+struct edd_packet_v3 {
+	uint16_t	len;
+	uint16_t	count;
+	uint16_t	off;
+	uint16_t	seg;
+	uint64_t	lba;
+	uint64_t	phys_addr;
+};
+
+struct edd_params {
+	uint16_t	len;
+	uint16_t	flags;
+	uint32_t	cylinders;
+	uint32_t	heads;
+	uint32_t	sectors_per_track;
+	uint64_t	sectors;
+	uint16_t	sector_size;
+	uint16_t	edd_params_seg;
+	uint16_t	edd_params_off;
+};
+
+struct edd_device_path_v3 {
+	uint16_t	key;
+	uint8_t		len;
+	uint8_t		reserved[3];
+	char		host_bus[4];
+	char		interface[8];
+	uint64_t	interface_path;
+	uint64_t	device_path;
+	uint8_t		reserved2[1];
+	uint8_t		checksum;
+};
+
+struct edd_params_v3 {
+	struct edd_params params;
+	struct edd_device_path_v3 device_path;
+};
+
+struct edd_device_path_v4 {
+	uint16_t	key;
+	uint8_t		len;
+	uint8_t		reserved[3];
+	char		host_bus[4];
+	char		interface[8];
+	uint64_t	interface_path;
+	uint64_t	device_path[2];
+	uint8_t		reserved2[1];
+	uint8_t		checksum;
+};
+
+struct edd_params_v4 {
+	struct edd_params params;
+	struct edd_device_path_v4 device_path;
+};
+
+#define	EDD_FLAGS_DMA_BOUNDARY_HANDLING		0x0001
+#define	EDD_FLAGS_REMOVABLE_MEDIA		0x0002
+#define	EDD_FLAGS_WRITE_VERIFY			0x0004
+#define	EDD_FLAGS_MEDIA_CHANGE_NOTIFICATION	0x0008
+#define	EDD_FLAGS_LOCKABLE_MEDIA		0x0010
+#define	EDD_FLAGS_NO_MEDIA_PRESENT		0x0020
+
+#define	EDD_DEVICE_PATH_KEY	0xbedd
+
+#endif /* !_EDD_H_ */
diff --git a/sys/boot/i386/gptboot/gptboot.c b/sys/boot/i386/gptboot/gptboot.c
index 5f0ab7c655cb..8cb413619464 100644
--- a/sys/boot/i386/gptboot/gptboot.c
+++ b/sys/boot/i386/gptboot/gptboot.c
@@ -38,7 +38,8 @@ __FBSDID("$FreeBSD$");
 #include "cons.h"
 #include "gpt.h"
 
-#define PATH_CONFIG	"/boot.config"
+#define PATH_DOTCONFIG  "/boot.config"
+#define PATH_CONFIG	"/boot/config"
 #define PATH_BOOT3	"/boot/loader"
 #define PATH_KERNEL	"/boot/kernel/kernel"
 
@@ -163,8 +164,8 @@ main(void)
 
 	for (;;) {
 		*kname = '\0';
-		ino = lookup(PATH_CONFIG);
-		if (ino > 0)
+		if ((ino = lookup(PATH_CONFIG)) ||
+		    (ino = lookup(PATH_DOTCONFIG)))
 			fsread(ino, cmd, sizeof(cmd));
 
 		if (*cmd != '\0') {
diff --git a/sys/boot/i386/libi386/Makefile b/sys/boot/i386/libi386/Makefile
index 79404712c867..de32c911ef8e 100644
--- a/sys/boot/i386/libi386/Makefile
+++ b/sys/boot/i386/libi386/Makefile
@@ -47,7 +47,8 @@ CFLAGS+= -DTERM_EMU
 # XXX: make alloca() useable
 CFLAGS+= -Dalloca=__builtin_alloca
 
-CFLAGS+=	-I${.CURDIR}/../../common -I${.CURDIR}/../btx/lib \
+CFLAGS+=	-I${.CURDIR}/../../common -I${.CURDIR}/../common \
+		-I${.CURDIR}/../btx/lib \
 		-I${.CURDIR}/../../../contrib/dev/acpica/include \
 		-I${.CURDIR}/../../.. -I.
 # the location of libstand
diff --git a/sys/boot/i386/libi386/bioscd.c b/sys/boot/i386/libi386/bioscd.c
index a3abf34e5863..dd50f7dd15f2 100644
--- a/sys/boot/i386/libi386/bioscd.c
+++ b/sys/boot/i386/libi386/bioscd.c
@@ -42,12 +42,12 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <machine/bootinfo.h>
-#include <machine/psl.h>
 
 #include <stdarg.h>
 
 #include <bootstrap.h>
 #include <btxv86.h>
+#include <edd.h>
 #include "libi386.h"
 
 #define BIOSCD_SECSIZE		2048
@@ -325,9 +325,9 @@ bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 				v86int();
 			}
 
-			packet.len = 0x10;
+			packet.len = sizeof(struct edd_packet);
 			packet.count = x;
-			packet.offset = VTOPOFF(xp);
+			packet.off = VTOPOFF(xp);
 			packet.seg = VTOPSEG(xp);
 			packet.lba = dblk;
 			v86.ctl = V86_FLAGS;
@@ -337,7 +337,7 @@ bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 			v86.ds = VTOPSEG(&packet);
 			v86.esi = VTOPOFF(&packet);
 			v86int();
-			result = (v86.efl & PSL_C);
+			result = V86_CY(v86.efl);
 			if (result == 0)
 				break;
 		}
diff --git a/sys/boot/i386/libi386/biosdisk.c b/sys/boot/i386/libi386/biosdisk.c
index a67a7ef9a882..533ce4466874 100644
--- a/sys/boot/i386/libi386/biosdisk.c
+++ b/sys/boot/i386/libi386/biosdisk.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 
 #include <bootstrap.h>
 #include <btxv86.h>
+#include <edd.h>
 #include "libi386.h"
 
 #define BIOS_NUMDRIVES		0x475
@@ -250,7 +251,7 @@ bd_int13probe(struct bdinfo *bd)
     v86.edx = bd->bd_unit;
     v86int();
     
-    if (!(v86.efl & 0x1) &&				/* carry clear */
+    if (!(V86_CY(v86.efl)) &&				/* carry clear */
 	((v86.edx & 0xff) > ((unsigned)bd->bd_unit & 0x7f))) {	/* unit # OK */
 	if ((v86.ecx & 0x3f) == 0) {			/* absurd sector size */
 		DEBUG("Invalid geometry for unit %d", bd->bd_unit);
@@ -264,11 +265,11 @@ bd_int13probe(struct bdinfo *bd)
 	v86.edx = bd->bd_unit;
 	v86.ebx = 0x55aa;
 	v86int();
-	if (!(v86.efl & 0x1) &&				/* carry clear */
+	if (!(V86_CY(v86.efl)) &&			/* carry clear */
 	    ((v86.ebx & 0xffff) == 0xaa55) &&		/* signature */
-	    (v86.ecx & 0x1)) {				/* packets mode ok */
+	    (v86.ecx & EDD_INTERFACE_FIXED_DISK)) {	/* packets mode ok */
 	    bd->bd_flags |= BD_MODEEDD1;
-	    if((v86.eax & 0xff00) >= 0x3000)
+	    if ((v86.eax & 0xff00) >= 0x3000)
 	        bd->bd_flags |= BD_MODEEDD3;
 	}
 	return(1);
@@ -562,7 +563,7 @@ bd_opendisk(struct open_disk **odp, struct i386_devdesc *dev)
 	return (ENOMEM);
     }
 
-    /* Look up BIOS unit number, intialise open_disk structure */
+    /* Look up BIOS unit number, initalise open_disk structure */
     od->od_dkunit = dev->d_unit;
     od->od_unit = bdinfo[od->od_dkunit].bd_unit;
     od->od_flags = bdinfo[od->od_dkunit].bd_flags;
@@ -1134,9 +1135,9 @@ bd_edd_io(struct open_disk *od, daddr_t dblk, int blks, caddr_t dest, int write)
 {
     static struct edd_packet packet;
 
-    packet.len = 0x10;
+    packet.len = sizeof(struct edd_packet);
     packet.count = blks;
-    packet.offset = VTOPOFF(dest);
+    packet.off = VTOPOFF(dest);
     packet.seg = VTOPSEG(dest);
     packet.lba = dblk;
     v86.ctl = V86_FLAGS;
@@ -1150,7 +1151,7 @@ bd_edd_io(struct open_disk *od, daddr_t dblk, int blks, caddr_t dest, int write)
     v86.ds = VTOPSEG(&packet);
     v86.esi = VTOPOFF(&packet);
     v86int();
-    return (v86.efl & 0x1);
+    return (V86_CY(v86.efl));
 }
 
 static int
@@ -1183,7 +1184,7 @@ bd_chs_io(struct open_disk *od, daddr_t dblk, int blks, caddr_t dest, int write)
     v86.es = VTOPSEG(dest);
     v86.ebx = VTOPOFF(dest);
     v86int();
-    return (v86.efl & 0x1);
+    return (V86_CY(v86.efl));
 }
 
 static int
@@ -1311,7 +1312,7 @@ bd_getgeom(struct open_disk *od)
     v86.edx = od->od_unit;
     v86int();
 
-    if ((v86.efl & 0x1) ||				/* carry set */
+    if ((V86_CY(v86.efl)) ||				/* carry set */
 	((v86.edx & 0xff) <= (unsigned)(od->od_unit & 0x7f)))	/* unit # bad */
 	return(1);
     
@@ -1348,7 +1349,7 @@ bd_getbigeom(int bunit)
     v86.eax = 0x800;
     v86.edx = 0x80 + bunit;
     v86int();
-    if (v86.efl & 0x1)
+    if (V86_CY(v86.efl))
 	return 0x4f010f;
     return ((v86.ecx & 0xc0) << 18) | ((v86.ecx & 0xff00) << 8) |
 	   (v86.edx & 0xff00) | (v86.ecx & 0x3f);
diff --git a/sys/boot/i386/libi386/biosmem.c b/sys/boot/i386/libi386/biosmem.c
index bae68131b87b..f437c6cb2b94 100644
--- a/sys/boot/i386/libi386/biosmem.c
+++ b/sys/boot/i386/libi386/biosmem.c
@@ -61,7 +61,7 @@ bios_getmem(void)
 	v86.es = VTOPSEG(&smap);
 	v86.edi = VTOPOFF(&smap);
 	v86int();
-	if ((v86.efl & 1) || (v86.eax != SMAP_SIG))
+	if ((V86_CY(v86.efl)) || (v86.eax != SMAP_SIG))
 	    break;
 	/* look for a low-memory segment that's large enough */
 	if ((smap.type == SMAP_TYPE_MEMORY) && (smap.base == 0) &&
@@ -108,7 +108,7 @@ bios_getmem(void)
 	v86.addr = 0x15;		/* int 0x15 function 0xe801*/
 	v86.eax = 0xe801;
 	v86int();
-	if (!(v86.efl & 1)) {
+	if (!(V86_CY(v86.efl))) {
 	    bios_extmem = ((v86.ecx & 0xffff) + ((v86.edx & 0xffff) * 64)) * 1024;
 	}
     }
diff --git a/sys/boot/i386/libi386/biospci.c b/sys/boot/i386/libi386/biospci.c
index b69d8c5873ff..ecbe7db118d2 100644
--- a/sys/boot/i386/libi386/biospci.c
+++ b/sys/boot/i386/libi386/biospci.c
@@ -218,7 +218,8 @@ biospci_enumerate(void)
     v86int();
 
     /* Check for OK response */
-    if ((v86.efl & 1) || ((v86.eax & 0xff00) != 0) || (v86.edx != 0x20494350))
+    if (V86_CY(v86.efl) || ((v86.eax & 0xff00) != 0) ||
+	(v86.edx != 0x20494350))
 	return;
 
     biospci_version = v86.ebx & 0xffff;
@@ -295,7 +296,7 @@ biospci_find_devclass(uint32_t class, int index, uint32_t *locator)
 	v86int();
 
 	 /* error */
-	if ((v86.efl & 1) || (v86.eax & 0xff00))
+	if (V86_CY(v86.efl) || (v86.eax & 0xff00))
 		return (-1);
 
 	*locator = v86.ebx;
@@ -317,7 +318,7 @@ biospci_write_config(uint32_t locator, int offset, int width, uint32_t val)
 	v86int();
 
 	 /* error */
-	if ((v86.efl & 1) || (v86.eax & 0xff00))
+	if (V86_CY(v86.efl) || (v86.eax & 0xff00))
 		return (-1);
 
 	return(0);
@@ -334,7 +335,7 @@ biospci_read_config(uint32_t locator, int offset, int width, uint32_t *val)
 	v86int();
 
 	 /* error */
-	if ((v86.efl & 1) || (v86.eax & 0xff00))
+	if (V86_CY(v86.efl) || (v86.eax & 0xff00))
 		return (-1);
 
 	*val = v86.ecx;
diff --git a/sys/boot/i386/libi386/biossmap.c b/sys/boot/i386/libi386/biossmap.c
index 5a788c172495..e95ea64517d0 100644
--- a/sys/boot/i386/libi386/biossmap.c
+++ b/sys/boot/i386/libi386/biossmap.c
@@ -36,14 +36,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/queue.h>
 #include <sys/stddef.h>
 #include <machine/metadata.h>
-#include <machine/psl.h>
 #include <machine/pc/bios.h>
 #include "bootstrap.h"
 #include "libi386.h"
 #include "btxv86.h"
 
-#define	V86_CY(x)	((x) & PSL_C)
-
 struct smap_buf {
 	struct bios_smap	smap;
 	uint32_t		xattr;	/* Extended attribute from ACPI 3.0 */
diff --git a/sys/boot/i386/libi386/libi386.h b/sys/boot/i386/libi386/libi386.h
index ebf1b4ce9494..8cfe9223c3d7 100644
--- a/sys/boot/i386/libi386/libi386.h
+++ b/sys/boot/i386/libi386/libi386.h
@@ -52,14 +52,6 @@ struct i386_devdesc
     } d_kind;
 };
 
-struct edd_packet {
-    uint16_t	len;
-    uint16_t	count;
-    uint16_t	offset;
-    uint16_t	seg;
-    uint64_t	lba;
-};
-                 
 int	i386_getdev(void **vdev, const char *devspec, const char **path);
 char	*i386_fmtdev(void *vdev);
 int	i386_setcurrdev(struct env_var *ev, int flags, const void *value);
diff --git a/sys/boot/i386/libi386/vidconsole.c b/sys/boot/i386/libi386/vidconsole.c
index 1b1e64884552..d95ce8834cf1 100644
--- a/sys/boot/i386/libi386/vidconsole.c
+++ b/sys/boot/i386/libi386/vidconsole.c
@@ -516,7 +516,7 @@ vidc_ischar(void)
     v86.addr = 0x16;
     v86.eax = 0x100;
     v86int();
-    return (!(v86.efl & PSL_Z));
+    return (!V86_ZR(v86.efl));
 }
 
 #if KEYBOARD_PROBE
diff --git a/sys/boot/i386/zfsboot/zfsboot.c b/sys/boot/i386/zfsboot/zfsboot.c
index 1d0077b7a4af..48a29f5a5a5f 100644
--- a/sys/boot/i386/zfsboot/zfsboot.c
+++ b/sys/boot/i386/zfsboot/zfsboot.c
@@ -45,7 +45,8 @@ __FBSDID("$FreeBSD$");
 /* Hint to loader that we came from ZFS */
 #define	KARGS_FLAGS_ZFS		0x4
 
-#define PATH_CONFIG	"/boot.config"
+#define PATH_DOTCONFIG	"/boot.config"
+#define PATH_CONFIG	"/boot/config"
 #define PATH_BOOT3	"/boot/zfsloader"
 #define PATH_KERNEL	"/boot/kernel/kernel"
 
@@ -533,7 +534,8 @@ main(void)
 
     zfs_mount_pool(spa);
 
-    if (zfs_lookup(spa, PATH_CONFIG, &dn) == 0) {
+    if (zfs_lookup(spa, PATH_CONFIG, &dn) == 0 ||
+        zfs_lookup(spa, PATH_DOTCONFIG, &dn) == 0) {
 	off = 0;
 	zfs_read(spa, &dn, &off, cmd, sizeof(cmd));
     }
diff --git a/sys/boot/pc98/boot2/boot2.c b/sys/boot/pc98/boot2/boot2.c
index 6fb400a0322a..2ef4da60b875 100644
--- a/sys/boot/pc98/boot2/boot2.c
+++ b/sys/boot/pc98/boot2/boot2.c
@@ -26,7 +26,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/bootinfo.h>
 #include <machine/cpufunc.h>
 #include <machine/elf.h>
-#include <machine/psl.h>
 
 #include <stdarg.h>
 
@@ -77,15 +76,14 @@ __FBSDID("$FreeBSD$");
 			OPT_SET(RBX_GDB ) | OPT_SET(RBX_MUTE) | \
 			OPT_SET(RBX_PAUSE) | OPT_SET(RBX_DUAL))
 
-#define PATH_CONFIG	"/boot.config"
+#define PATH_DOTCONFIG	"/boot.config"
+#define PATH_CONFIG	"/boot/config"
 #define PATH_BOOT3	"/boot/loader"
 #define PATH_KERNEL	"/boot/kernel/kernel"
 
 #define ARGS		0x900
 #define NOPT		14
 #define NDEV		3
-#define V86_CY(x)	((x) & PSL_C)
-#define V86_ZR(x)	((x) & PSL_Z)
 
 #define DRV_DISK	0xf0
 #define DRV_UNIT	0x0f
@@ -379,7 +377,8 @@ main(void)
 
     autoboot = 1;
 
-    if ((ino = lookup(PATH_CONFIG)))
+    if ((ino = lookup(PATH_CONFIG)) ||
+        (ino = lookup(PATH_DOTCONFIG)))
 	fsread(ino, cmd, sizeof(cmd));
 
     if (*cmd) {
diff --git a/sys/boot/pc98/btx/lib/btxv86.h b/sys/boot/pc98/btx/lib/btxv86.h
index 1152f8b1d963..27f6b3404248 100644
--- a/sys/boot/pc98/btx/lib/btxv86.h
+++ b/sys/boot/pc98/btx/lib/btxv86.h
@@ -21,6 +21,7 @@
 #define _BTXV86_H_
 
 #include <sys/types.h>
+#include <machine/psl.h>
 
 #define V86_ADDR   0x10000	/* Segment:offset address */
 #define V86_CALLF  0x20000	/* Emulate far call */
@@ -57,6 +58,9 @@ extern u_int32_t	__args;
 #define	VTOPSEG(va)	(u_int16_t)(VTOP((caddr_t)va) >> 4)
 #define	VTOPOFF(va)	(u_int16_t)(VTOP((caddr_t)va) & 0xf)
 
+#define	V86_CY(x)	((x) & PSL_C)
+#define	V86_ZR(x)	((x) & PSL_Z)
+
 void __exit(int) __attribute__((__noreturn__));
 void __exec(caddr_t, ...);
 
diff --git a/sys/boot/pc98/libpc98/bioscd.c b/sys/boot/pc98/libpc98/bioscd.c
index 3e1ba712d660..e3df3915c933 100644
--- a/sys/boot/pc98/libpc98/bioscd.c
+++ b/sys/boot/pc98/libpc98/bioscd.c
@@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <machine/bootinfo.h>
-#include <machine/psl.h>
 
 #include <stdarg.h>
 
@@ -325,7 +324,7 @@ bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 			v86.ebp = VTOPOFF(xp);
 			v86.es = VTOPSEG(xp);
 			v86int();
-			result = (v86.efl & PSL_C);
+			result = V86_CY(v86.efl);
 			if (result == 0)
 				break;
 		}
diff --git a/sys/boot/pc98/libpc98/vidconsole.c b/sys/boot/pc98/libpc98/vidconsole.c
index 19fb981aaa1c..7cf81e81a371 100644
--- a/sys/boot/pc98/libpc98/vidconsole.c
+++ b/sys/boot/pc98/libpc98/vidconsole.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 #include <stand.h>
 #include <bootstrap.h>
 #include <btxv86.h>
-#include <machine/psl.h>
 #include <machine/cpufunc.h>
 #include "libi386.h"
 
diff --git a/sys/boot/pc98/loader/main.c b/sys/boot/pc98/loader/main.c
index 391c5e9bc86c..15b132f11f4b 100644
--- a/sys/boot/pc98/loader/main.c
+++ b/sys/boot/pc98/loader/main.c
@@ -35,7 +35,6 @@ __FBSDID("$FreeBSD$");
 #include <stand.h>
 #include <string.h>
 #include <machine/bootinfo.h>
-#include <machine/psl.h>
 #include <sys/param.h>
 #include <sys/reboot.h>
 
diff --git a/sys/boot/powerpc/boot1.chrp/generate-hfs.sh b/sys/boot/powerpc/boot1.chrp/generate-hfs.sh
index ab5694dd8bc8..2ed6f38f1682 100755
--- a/sys/boot/powerpc/boot1.chrp/generate-hfs.sh
+++ b/sys/boot/powerpc/boot1.chrp/generate-hfs.sh
@@ -29,7 +29,7 @@ hmkdir ppc
 hattrib -b ppc
 hcd ppc
 
-# Make two dummy files for the the CHRP boot script and boot1
+# Make two dummy files for the CHRP boot script and boot1
 echo 'Bootinfo START' | dd of=bootinfo.txt.tmp cbs=$CHRPBOOT_SIZE count=1 conv=block
 echo 'Boot1 START' | dd of=boot1.elf.tmp cbs=$BOOT1_SIZE count=1 conv=block
 
diff --git a/sys/boot/zfs/zfsimpl.c b/sys/boot/zfs/zfsimpl.c
index 497667a8e0ef..c97c2dee0f98 100644
--- a/sys/boot/zfs/zfsimpl.c
+++ b/sys/boot/zfs/zfsimpl.c
@@ -347,7 +347,7 @@ vdev_read_phys(vdev_t *vdev, const blkptr_t *bp, void *buf,
 	rc = vdev->v_phys_read(vdev, vdev->v_read_priv, offset, buf, psize);
 	if (rc)
 		return (rc);
-	if (bp && zio_checksum_error(bp, buf, offset))
+	if (bp && zio_checksum_verify(bp, buf))
 		return (EIO);
 
 	return (0);
@@ -543,8 +543,6 @@ vdev_init_from_nvlist(const unsigned char *nvlist, vdev_t *pvdev,
 			vdev->v_state = VDEV_STATE_DEGRADED;
 		else if (isnt_present)
 			vdev->v_state = VDEV_STATE_CANT_OPEN;
-		else
-			vdev->v_state = VDEV_STATE_HEALTHY;
 	}
 
 	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN,
@@ -800,6 +798,7 @@ vdev_probe(vdev_phys_read_t *read, void *read_priv, spa_t **spap)
 	BP_SET_PSIZE(&bp, sizeof(vdev_phys_t));
 	BP_SET_CHECKSUM(&bp, ZIO_CHECKSUM_LABEL);
 	BP_SET_COMPRESS(&bp, ZIO_COMPRESS_OFF);
+	DVA_SET_OFFSET(BP_IDENTITY(&bp), off);
 	ZIO_SET_CHECKSUM(&bp.blk_cksum, off, 0, 0, 0);
 	if (vdev_read_phys(&vtmp, &bp, vdev_label, off, 0))
 		return (EIO);
@@ -912,6 +911,7 @@ vdev_probe(vdev_phys_read_t *read, void *read_priv, spa_t **spap)
 	if (vdev) {
 		vdev->v_phys_read = read;
 		vdev->v_read_priv = read_priv;
+		vdev->v_state = VDEV_STATE_HEALTHY;
 	} else {
 		printf("ZFS: inconsistent nvlist contents\n");
 		return (EIO);
@@ -941,7 +941,7 @@ vdev_probe(vdev_phys_read_t *read, void *read_priv, spa_t **spap)
 		BP_SET_COMPRESS(&bp, ZIO_COMPRESS_OFF);
 		ZIO_SET_CHECKSUM(&bp.blk_cksum, off, 0, 0, 0);
 
-		if (vdev_read_phys(vdev, NULL, upbuf, off, VDEV_UBERBLOCK_SIZE(vdev)))
+		if (vdev_read_phys(vdev, &bp, upbuf, off, 0))
 			continue;
 
 		if (up->ub_magic != UBERBLOCK_MAGIC)
@@ -974,34 +974,39 @@ ilog2(int n)
 }
 
 static int
-zio_read_gang(spa_t *spa, const blkptr_t *bp, const dva_t *dva, void *buf)
+zio_read_gang(spa_t *spa, const blkptr_t *bp, void *buf)
 {
+	blkptr_t gbh_bp;
 	zio_gbh_phys_t zio_gb;
-	vdev_t *vdev;
-	int vdevid;
-	off_t offset;
+	char *pbuf;
 	int i;
 
-	vdevid = DVA_GET_VDEV(dva);
-	offset = DVA_GET_OFFSET(dva);
-	STAILQ_FOREACH(vdev, &spa->spa_vdevs, v_childlink)
-		if (vdev->v_id == vdevid)
-			break;
-	if (!vdev || !vdev->v_read)
-		return (EIO);
-	if (vdev->v_read(vdev, NULL, &zio_gb, offset, SPA_GANGBLOCKSIZE))
+	/* Artificial BP for gang block header. */
+	gbh_bp = *bp;
+	BP_SET_PSIZE(&gbh_bp, SPA_GANGBLOCKSIZE);
+	BP_SET_LSIZE(&gbh_bp, SPA_GANGBLOCKSIZE);
+	BP_SET_CHECKSUM(&gbh_bp, ZIO_CHECKSUM_GANG_HEADER);
+	BP_SET_COMPRESS(&gbh_bp, ZIO_COMPRESS_OFF);
+	for (i = 0; i < SPA_DVAS_PER_BP; i++)
+		DVA_SET_GANG(&gbh_bp.blk_dva[i], 0);
+
+	/* Read gang header block using the artificial BP. */
+	if (zio_read(spa, &gbh_bp, &zio_gb))
 		return (EIO);
 
+	pbuf = buf;
 	for (i = 0; i < SPA_GBH_NBLKPTRS; i++) {
 		blkptr_t *gbp = &zio_gb.zg_blkptr[i];
 
 		if (BP_IS_HOLE(gbp))
 			continue;
-		if (zio_read(spa, gbp, buf))
+		if (zio_read(spa, gbp, pbuf))
 			return (EIO);
-		buf = (char*)buf + BP_GET_PSIZE(gbp);
+		pbuf += BP_GET_PSIZE(gbp);
 	}
- 
+
+	if (zio_checksum_verify(bp, buf))
+		return (EIO);
 	return (0);
 }
 
@@ -1024,46 +1029,41 @@ zio_read(spa_t *spa, const blkptr_t *bp, void *buf)
 		if (!dva->dva_word[0] && !dva->dva_word[1])
 			continue;
 
-		if (DVA_GET_GANG(dva)) {
-			error = zio_read_gang(spa, bp, dva, buf);
-			if (error != 0)
-				continue;
-		} else {
-			vdevid = DVA_GET_VDEV(dva);
-			offset = DVA_GET_OFFSET(dva);
-			STAILQ_FOREACH(vdev, &spa->spa_vdevs, v_childlink) {
-				if (vdev->v_id == vdevid)
-					break;
-			}
-			if (!vdev || !vdev->v_read)
-				continue;
+		vdevid = DVA_GET_VDEV(dva);
+		offset = DVA_GET_OFFSET(dva);
+		STAILQ_FOREACH(vdev, &spa->spa_vdevs, v_childlink) {
+			if (vdev->v_id == vdevid)
+				break;
+		}
+		if (!vdev || !vdev->v_read)
+			continue;
 
-			size = BP_GET_PSIZE(bp);
+		size = BP_GET_PSIZE(bp);
+		if (vdev->v_read == vdev_raidz_read) {
 			align = 1ULL << vdev->v_top->v_ashift;
 			if (P2PHASE(size, align) != 0)
 				size = P2ROUNDUP(size, align);
-			if (size != BP_GET_PSIZE(bp) || cpfunc != ZIO_COMPRESS_OFF)
-				pbuf = zfs_alloc(size);
-			else
-				pbuf = buf;
-
-			error = vdev->v_read(vdev, bp, pbuf, offset, size);
-			if (error == 0) {
-				if (cpfunc != ZIO_COMPRESS_OFF) {
-					error = zio_decompress_data(cpfunc,
-					    pbuf, BP_GET_PSIZE(bp), buf,
-					    BP_GET_LSIZE(bp));
-				} else if (size != BP_GET_PSIZE(bp)) {
-					bcopy(pbuf, buf, BP_GET_PSIZE(bp));
-				}
-			}
-			if (buf != pbuf)
-				zfs_free(pbuf, size);
-			if (error != 0)
-				continue;
 		}
-		error = 0;
-		break;
+		if (size != BP_GET_PSIZE(bp) || cpfunc != ZIO_COMPRESS_OFF)
+			pbuf = zfs_alloc(size);
+		else
+			pbuf = buf;
+
+		if (DVA_GET_GANG(dva))
+			error = zio_read_gang(spa, bp, pbuf);
+		else
+			error = vdev->v_read(vdev, bp, pbuf, offset, size);
+		if (error == 0) {
+			if (cpfunc != ZIO_COMPRESS_OFF)
+				error = zio_decompress_data(cpfunc, pbuf,
+				    BP_GET_PSIZE(bp), buf, BP_GET_LSIZE(bp));
+			else if (size != BP_GET_PSIZE(bp))
+				bcopy(pbuf, buf, BP_GET_PSIZE(bp));
+		}
+		if (buf != pbuf)
+			zfs_free(pbuf, size);
+		if (error == 0)
+			break;
 	}
 	if (error != 0)
 		printf("ZFS: i/o error - all block copies unavailable\n");
diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h
index ed2a89076892..0d51482dac54 100644
--- a/sys/cam/cam_ccb.h
+++ b/sys/cam/cam_ccb.h
@@ -539,7 +539,7 @@ struct ccb_dev_match {
 /*
  * Definitions for the path inquiry CCB fields.
  */
-#define CAM_VERSION	0x15	/* Hex value for current version */
+#define CAM_VERSION	0x16	/* Hex value for current version */
 
 typedef enum {
 	PI_MDP_ABLE	= 0x80,	/* Supports MDP message */
diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c
index 67c2d01125d6..80dba392183d 100644
--- a/sys/cam/cam_periph.c
+++ b/sys/cam/cam_periph.c
@@ -1085,7 +1085,6 @@ camperiphsensedone(struct cam_periph *periph, union ccb *done_ccb)
 	union ccb      *saved_ccb = (union ccb *)done_ccb->ccb_h.saved_ccb_ptr;
 	cam_status	status;
 	int		frozen = 0;
-	u_int		sense_key;
 	int		depth = done_ccb->ccb_h.recovery_depth;
 
 	status = done_ccb->ccb_h.status;
@@ -1101,22 +1100,25 @@ camperiphsensedone(struct cam_periph *periph, union ccb *done_ccb)
 	switch (status) {
 	case CAM_REQ_CMP:
 	{
+		int error_code, sense_key, asc, ascq;
+
+		scsi_extract_sense_len(&saved_ccb->csio.sense_data,
+				       saved_ccb->csio.sense_len -
+				       saved_ccb->csio.sense_resid,
+				       &error_code, &sense_key, &asc, &ascq,
+				       /*show_errors*/ 1);
 		/*
 		 * If we manually retrieved sense into a CCB and got
 		 * something other than "NO SENSE" send the updated CCB
 		 * back to the client via xpt_done() to be processed via
 		 * the error recovery code again.
 		 */
-		sense_key = saved_ccb->csio.sense_data.flags;
-		sense_key &= SSD_KEY;
-		if (sense_key != SSD_KEY_NO_SENSE) {
-			saved_ccb->ccb_h.status |=
-			    CAM_AUTOSNS_VALID;
+		if ((sense_key != -1)
+		 && (sense_key != SSD_KEY_NO_SENSE)) {
+			saved_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 		} else {
-			saved_ccb->ccb_h.status &=
-			    ~CAM_STATUS_MASK;
-			saved_ccb->ccb_h.status |=
-			    CAM_AUTOSENSE_FAIL;
+			saved_ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+			saved_ccb->ccb_h.status |= CAM_AUTOSENSE_FAIL;
 		}
 		saved_ccb->csio.sense_resid = done_ccb->csio.resid;
 		bcopy(saved_ccb, done_ccb, sizeof(union ccb));
@@ -1198,12 +1200,15 @@ camperiphdone(struct cam_periph *periph, union ccb *done_ccb)
 		if (status & CAM_AUTOSNS_VALID) {
 			struct ccb_getdev cgd;
 			struct scsi_sense_data *sense;
-			int    error_code, sense_key, asc, ascq;	
+			int    error_code, sense_key, asc, ascq, sense_len;
 			scsi_sense_action err_action;
 
 			sense = &done_ccb->csio.sense_data;
-			scsi_extract_sense(sense, &error_code, 
-					   &sense_key, &asc, &ascq);
+			sense_len = done_ccb->csio.sense_len -
+				    done_ccb->csio.sense_resid;
+			scsi_extract_sense_len(sense, sense_len, &error_code, 
+					       &sense_key, &asc, &ascq,
+					       /*show_errors*/ 1);
 			/*
 			 * Grab the inquiry data for this device.
 			 */
diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c
index 7361c42f0bd7..5ec287b39b8a 100644
--- a/sys/cam/scsi/scsi_all.c
+++ b/sys/cam/scsi/scsi_all.c
@@ -31,6 +31,8 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/types.h>
+#include <sys/stdint.h>
 
 #ifdef _KERNEL
 #include <opt_scsi.h>
@@ -54,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sbuf.h>
 #ifndef _KERNEL
 #include <camlib.h>
+#include <stddef.h>
 
 #ifndef FALSE
 #define FALSE   0
@@ -608,14 +611,24 @@ scsi_op_desc(u_int16_t opcode, struct scsi_inquiry_data *inq_data)
 	struct op_table_entry *table[2];
 	int num_tables;
 
-	pd_type = SID_TYPE(inq_data);
+	/*
+	 * If we've got inquiry data, use it to determine what type of
+	 * device we're dealing with here.  Otherwise, assume direct
+	 * access.
+	 */
+	if (inq_data == NULL) {
+		pd_type = T_DIRECT;
+		match = NULL;
+	} else {
+		pd_type = SID_TYPE(inq_data);
 
-	match = cam_quirkmatch((caddr_t)inq_data,
-			       (caddr_t)scsi_op_quirk_table,
-			       sizeof(scsi_op_quirk_table)/
-			       sizeof(*scsi_op_quirk_table),
-			       sizeof(*scsi_op_quirk_table),
-			       scsi_inquiry_match);
+		match = cam_quirkmatch((caddr_t)inq_data,
+				       (caddr_t)scsi_op_quirk_table,
+				       sizeof(scsi_op_quirk_table)/
+				       sizeof(*scsi_op_quirk_table),
+				       sizeof(*scsi_op_quirk_table),
+				       scsi_inquiry_match);
+	}
 
 	if (match != NULL) {
 		table[0] = ((struct scsi_op_quirk_entry *)match)->op_table;
@@ -699,7 +712,7 @@ const struct sense_key_table_entry sense_key_table[] =
 	{ SSD_KEY_EQUAL, SS_NOP, "EQUAL" },
 	{ SSD_KEY_VOLUME_OVERFLOW, SS_FATAL|EIO, "VOLUME OVERFLOW" },
 	{ SSD_KEY_MISCOMPARE, SS_NOP, "MISCOMPARE" },
-	{ SSD_KEY_RESERVED, SS_FATAL|EIO, "RESERVED" }
+	{ SSD_KEY_COMPLETED, SS_NOP, "COMPLETED" }
 };
 
 const int sense_key_table_size =
@@ -1062,25 +1075,25 @@ static struct asc_table_entry asc_table[] = {
 	{ SST(0x10, 0x03, SS_RDEF,	/* XXX TBD */
 	    "Logical block reference tag check failed") },
 	/* DT  WRO   BK   */
-	{ SST(0x11, 0x00, SS_RDEF,
+	{ SST(0x11, 0x00, SS_FATAL|EIO,
 	    "Unrecovered read error") },
 	/* DT  WRO   BK   */
-	{ SST(0x11, 0x01, SS_RDEF,
+	{ SST(0x11, 0x01, SS_FATAL|EIO,
 	    "Read retries exhausted") },
 	/* DT  WRO   BK   */
-	{ SST(0x11, 0x02, SS_RDEF,
+	{ SST(0x11, 0x02, SS_FATAL|EIO,
 	    "Error too long to correct") },
 	/* DT  W O   BK   */
-	{ SST(0x11, 0x03, SS_RDEF,
+	{ SST(0x11, 0x03, SS_FATAL|EIO,
 	    "Multiple read errors") },
 	/* D   W O   BK   */
-	{ SST(0x11, 0x04, SS_RDEF,
+	{ SST(0x11, 0x04, SS_FATAL|EIO,
 	    "Unrecovered read error - auto reallocate failed") },
 	/*     WRO   B    */
-	{ SST(0x11, 0x05, SS_RDEF,
+	{ SST(0x11, 0x05, SS_FATAL|EIO,
 	    "L-EC uncorrectable error") },
 	/*     WRO   B    */
-	{ SST(0x11, 0x06, SS_RDEF,
+	{ SST(0x11, 0x06, SS_FATAL|EIO,
 	    "CIRC unrecovered error") },
 	/*     W O   B    */
 	{ SST(0x11, 0x07, SS_RDEF,
@@ -1095,10 +1108,10 @@ static struct asc_table_entry asc_table[] = {
 	{ SST(0x11, 0x0A, SS_RDEF,
 	    "Miscorrected error") },
 	/* D   W O   BK   */
-	{ SST(0x11, 0x0B, SS_RDEF,
+	{ SST(0x11, 0x0B, SS_FATAL|EIO,
 	    "Unrecovered read error - recommend reassignment") },
 	/* D   W O   BK   */
-	{ SST(0x11, 0x0C, SS_RDEF,
+	{ SST(0x11, 0x0C, SS_FATAL|EIO,
 	    "Unrecovered read error - recommend rewrite the data") },
 	/* DT  WRO   B    */
 	{ SST(0x11, 0x0D, SS_RDEF,
@@ -2790,7 +2803,10 @@ scsi_sense_desc(int sense_key, int asc, int ascq,
 			  &sense_entry,
 			  &asc_entry);
 
-	*sense_key_desc = sense_entry->desc;
+	if (sense_entry != NULL)
+		*sense_key_desc = sense_entry->desc;
+	else
+		*sense_key_desc = "Invalid Sense Key";
 
 	if (asc_entry != NULL)
 		*asc_desc = asc_entry->desc;
@@ -2816,10 +2832,12 @@ scsi_error_action(struct ccb_scsiio *csio, struct scsi_inquiry_data *inq_data,
 	int error_code, sense_key, asc, ascq;
 	scsi_sense_action action;
 
-	scsi_extract_sense(&csio->sense_data, &error_code,
-			   &sense_key, &asc, &ascq);
+	scsi_extract_sense_len(&csio->sense_data, csio->sense_len -
+			       csio->sense_resid, &error_code,
+			       &sense_key, &asc, &ascq, /*show_errors*/ 1);
 
-	if (error_code == SSD_DEFERRED_ERROR) {
+	if ((error_code == SSD_DEFERRED_ERROR)
+	 || (error_code == SSD_DESC_DEFERRED_ERROR)) {
 		/*
 		 * XXX dufault@FreeBSD.org
 		 * This error doesn't relate to the command associated
@@ -2857,8 +2875,10 @@ scsi_error_action(struct ccb_scsiio *csio, struct scsi_inquiry_data *inq_data,
 		if (asc_entry != NULL
 		 && (asc != 0 || ascq != 0))
 			action = asc_entry->action;
-		else
+		else if (sense_entry != NULL)
 			action = sense_entry->action;
+		else
+			action = SS_RETRY|SSQ_DECREMENT_COUNT|SSQ_PRINT_SENSE; 
 
 		if (sense_key == SSD_KEY_RECOVERED_ERROR) {
 			/*
@@ -3040,6 +3060,1346 @@ scsi_command_string(struct cam_device *device, struct ccb_scsiio *csio,
 	return(0);
 }
 
+/*
+ * Iterate over sense descriptors.  Each descriptor is passed into iter_func(). 
+ * If iter_func() returns 0, list traversal continues.  If iter_func()
+ * returns non-zero, list traversal is stopped.
+ */
+void
+scsi_desc_iterate(struct scsi_sense_data_desc *sense, u_int sense_len,
+		  int (*iter_func)(struct scsi_sense_data_desc *sense,
+				   u_int, struct scsi_sense_desc_header *,
+				   void *), void *arg)
+{
+	int cur_pos;
+	int desc_len;
+
+	/*
+	 * First make sure the extra length field is present.
+	 */
+	if (SSD_DESC_IS_PRESENT(sense, sense_len, extra_len) == 0)
+		return;
+
+	/*
+	 * The length of data actually returned may be different than the
+	 * extra_len recorded in the sturcture.
+	 */
+	desc_len = sense_len -offsetof(struct scsi_sense_data_desc, sense_desc);
+
+	/*
+	 * Limit this further by the extra length reported, and the maximum
+	 * allowed extra length.
+	 */
+	desc_len = MIN(desc_len, MIN(sense->extra_len, SSD_EXTRA_MAX));
+
+	/*
+	 * Subtract the size of the header from the descriptor length.
+	 * This is to ensure that we have at least the header left, so we
+	 * don't have to check that inside the loop.  This can wind up
+	 * being a negative value.
+	 */
+	desc_len -= sizeof(struct scsi_sense_desc_header);
+
+	for (cur_pos = 0; cur_pos < desc_len;) {
+		struct scsi_sense_desc_header *header;
+
+		header = (struct scsi_sense_desc_header *)
+			&sense->sense_desc[cur_pos];
+
+		/*
+		 * Check to make sure we have the entire descriptor.  We
+		 * don't call iter_func() unless we do.
+		 *
+		 * Note that although cur_pos is at the beginning of the
+		 * descriptor, desc_len already has the header length
+		 * subtracted.  So the comparison of the length in the
+		 * header (which does not include the header itself) to
+		 * desc_len - cur_pos is correct.
+		 */
+		if (header->length > (desc_len - cur_pos)) 
+			break;
+
+		if (iter_func(sense, sense_len, header, arg) != 0)
+			break;
+
+		cur_pos += sizeof(*header) + header->length;
+	}
+}
+
+struct scsi_find_desc_info {
+	uint8_t desc_type;
+	struct scsi_sense_desc_header *header;
+};
+
+static int
+scsi_find_desc_func(struct scsi_sense_data_desc *sense, u_int sense_len,
+		    struct scsi_sense_desc_header *header, void *arg)
+{
+	struct scsi_find_desc_info *desc_info;
+
+	desc_info = (struct scsi_find_desc_info *)arg;
+
+	if (header->desc_type == desc_info->desc_type) {
+		desc_info->header = header;
+
+		/* We found the descriptor, tell the iterator to stop. */
+		return (1);
+	} else
+		return (0);
+}
+
+/*
+ * Given a descriptor type, return a pointer to it if it is in the sense
+ * data and not truncated.  Avoiding truncating sense data will simplify
+ * things significantly for the caller.
+ */
+uint8_t *
+scsi_find_desc(struct scsi_sense_data_desc *sense, u_int sense_len,
+	       uint8_t desc_type)
+{
+	struct scsi_find_desc_info desc_info;
+
+	desc_info.desc_type = desc_type;
+	desc_info.header = NULL;
+
+	scsi_desc_iterate(sense, sense_len, scsi_find_desc_func, &desc_info);
+
+	return ((uint8_t *)desc_info.header);
+}
+
+/*
+ * Fill in SCSI sense data with the specified parameters.  This routine can
+ * fill in either fixed or descriptor type sense data.
+ */
+void
+scsi_set_sense_data_va(struct scsi_sense_data *sense_data,
+		      scsi_sense_data_type sense_format, int current_error,
+		      int sense_key, int asc, int ascq, va_list ap) 
+{
+	int descriptor_sense;
+	scsi_sense_elem_type elem_type;
+
+	/*
+	 * Determine whether to return fixed or descriptor format sense
+	 * data.  If the user specifies SSD_TYPE_NONE for some reason,
+	 * they'll just get fixed sense data.
+	 */
+	if (sense_format == SSD_TYPE_DESC)
+		descriptor_sense = 1;
+	else
+		descriptor_sense = 0;
+
+	/*
+	 * Zero the sense data, so that we don't pass back any garbage data
+	 * to the user.
+	 */
+	memset(sense_data, 0, sizeof(*sense_data));
+
+	if (descriptor_sense != 0) {
+		struct scsi_sense_data_desc *sense;
+
+		sense = (struct scsi_sense_data_desc *)sense_data;
+		/*
+		 * The descriptor sense format eliminates the use of the
+		 * valid bit.
+		 */
+		if (current_error != 0)
+			sense->error_code = SSD_DESC_CURRENT_ERROR;
+		else
+			sense->error_code = SSD_DESC_DEFERRED_ERROR;
+		sense->sense_key = sense_key;
+		sense->add_sense_code = asc;
+		sense->add_sense_code_qual = ascq;
+		/*
+		 * Start off with no extra length, since the above data
+		 * fits in the standard descriptor sense information.
+		 */
+		sense->extra_len = 0;
+		while ((elem_type = (scsi_sense_elem_type)va_arg(ap,
+			scsi_sense_elem_type)) != SSD_ELEM_NONE) {
+			int sense_len, len_to_copy;
+			uint8_t *data;
+
+			if (elem_type >= SSD_ELEM_MAX) {
+				printf("%s: invalid sense type %d\n", __func__,
+				       elem_type);
+				break;
+			}
+
+			sense_len = (int)va_arg(ap, int);
+			len_to_copy = MIN(sense_len, SSD_EXTRA_MAX -
+					  sense->extra_len);
+			data = (uint8_t *)va_arg(ap, uint8_t *);
+
+			/*
+			 * We've already consumed the arguments for this one.
+			 */
+			if (elem_type == SSD_ELEM_SKIP)
+				continue;
+
+			switch (elem_type) {
+			case SSD_ELEM_DESC: {
+
+				/*
+				 * This is a straight descriptor.  All we
+				 * need to do is copy the data in.
+				 */
+				bcopy(data, &sense->sense_desc[
+				      sense->extra_len], len_to_copy);
+				sense->extra_len += len_to_copy;
+				break;
+			}
+			case SSD_ELEM_SKS: {
+				struct scsi_sense_sks sks;
+
+				bzero(&sks, sizeof(sks));
+
+				/*
+				 * This is already-formatted sense key
+				 * specific data.  We just need to fill out
+				 * the header and copy everything in.
+				 */
+				bcopy(data, &sks.sense_key_spec,
+				      MIN(len_to_copy,
+				          sizeof(sks.sense_key_spec)));
+
+				sks.desc_type = SSD_DESC_SKS;
+				sks.length = sizeof(sks) -
+				    offsetof(struct scsi_sense_sks, reserved1);
+				bcopy(&sks,&sense->sense_desc[sense->extra_len],
+				      sizeof(sks));
+				sense->extra_len += sizeof(sks);
+				break;
+			}
+			case SSD_ELEM_INFO:
+			case SSD_ELEM_COMMAND: {
+				struct scsi_sense_command cmd;
+				struct scsi_sense_info info;
+				uint8_t *data_dest;
+				uint8_t *descriptor;
+				int descriptor_size, i, copy_len;
+
+				bzero(&cmd, sizeof(cmd));
+				bzero(&info, sizeof(info));
+
+				/*
+				 * Command or information data.  The
+				 * operate in pretty much the same way.
+				 */
+				if (elem_type == SSD_ELEM_COMMAND) {
+					len_to_copy = MIN(len_to_copy,
+					    sizeof(cmd.command_info));
+					descriptor = (uint8_t *)&cmd;
+					descriptor_size  = sizeof(cmd);
+					data_dest =(uint8_t *)&cmd.command_info;
+					cmd.desc_type = SSD_DESC_COMMAND;
+					cmd.length = sizeof(cmd) -
+					    offsetof(struct scsi_sense_command,
+						     reserved);
+				} else {
+					len_to_copy = MIN(len_to_copy,
+					    sizeof(info.info));
+					descriptor = (uint8_t *)&info;
+					descriptor_size = sizeof(cmd);
+					data_dest = (uint8_t *)&info.info;
+					info.desc_type = SSD_DESC_INFO;
+					info.byte2 = SSD_INFO_VALID;
+					info.length = sizeof(info) -
+					    offsetof(struct scsi_sense_info,
+						     byte2);
+				}
+
+				/*
+				 * Copy this in reverse because the spec
+				 * (SPC-4) says that when 4 byte quantities
+				 * are stored in this 8 byte field, the
+				 * first four bytes shall be 0.
+				 *
+				 * So we fill the bytes in from the end, and
+				 * if we have less than 8 bytes to copy,
+				 * the initial, most significant bytes will
+				 * be 0.
+				 */
+				for (i = sense_len - 1; i >= 0 &&
+				     len_to_copy > 0; i--, len_to_copy--)
+					data_dest[len_to_copy - 1] = data[i];
+
+				/*
+				 * This calculation looks much like the
+				 * initial len_to_copy calculation, but
+				 * we have to do it again here, because
+				 * we're looking at a larger amount that
+				 * may or may not fit.  It's not only the
+				 * data the user passed in, but also the
+				 * rest of the descriptor.
+				 */
+				copy_len = MIN(descriptor_size,
+				    SSD_EXTRA_MAX - sense->extra_len);
+				bcopy(descriptor, &sense->sense_desc[
+				      sense->extra_len], copy_len);
+				sense->extra_len += copy_len;
+				break;
+			}
+			case SSD_ELEM_FRU: {
+				struct scsi_sense_fru fru;
+				int copy_len;
+
+				bzero(&fru, sizeof(fru));
+
+				fru.desc_type = SSD_DESC_FRU;
+				fru.length = sizeof(fru) -
+				    offsetof(struct scsi_sense_fru, reserved);
+				fru.fru = *data;
+
+				copy_len = MIN(sizeof(fru), SSD_EXTRA_MAX -
+					       sense->extra_len);
+				bcopy(&fru, &sense->sense_desc[
+				      sense->extra_len], copy_len);
+				sense->extra_len += copy_len;
+				break;
+			}
+			case SSD_ELEM_STREAM: {
+				struct scsi_sense_stream stream_sense;
+				int copy_len;
+
+				bzero(&stream_sense, sizeof(stream_sense));
+				stream_sense.desc_type = SSD_DESC_STREAM;
+				stream_sense.length = sizeof(stream_sense) -
+				   offsetof(struct scsi_sense_stream, reserved);
+				stream_sense.byte3 = *data;
+
+				copy_len = MIN(sizeof(stream_sense),
+				    SSD_EXTRA_MAX - sense->extra_len);
+				bcopy(&stream_sense, &sense->sense_desc[
+				      sense->extra_len], copy_len);
+				sense->extra_len += copy_len;
+				break;
+			}
+			default:
+				/*
+				 * We shouldn't get here, but if we do, do
+				 * nothing.  We've already consumed the
+				 * arguments above.
+				 */
+				break;
+			}
+		}
+	} else {
+		struct scsi_sense_data_fixed *sense;
+
+		sense = (struct scsi_sense_data_fixed *)sense_data;
+
+		if (current_error != 0)
+			sense->error_code = SSD_CURRENT_ERROR;
+		else
+			sense->error_code = SSD_DEFERRED_ERROR;
+
+		sense->flags = sense_key;
+		sense->add_sense_code = asc;
+		sense->add_sense_code_qual = ascq;
+		/*
+		 * We've set the ASC and ASCQ, so we have 6 more bytes of
+		 * valid data.  If we wind up setting any of the other
+		 * fields, we'll bump this to 10 extra bytes.
+		 */
+		sense->extra_len = 6;
+
+		while ((elem_type = (scsi_sense_elem_type)va_arg(ap,
+			scsi_sense_elem_type)) != SSD_ELEM_NONE) {
+			int sense_len, len_to_copy;
+			uint8_t *data;
+
+			if (elem_type >= SSD_ELEM_MAX) {
+				printf("%s: invalid sense type %d\n", __func__,
+				       elem_type);
+				break;
+			}
+			/*
+			 * If we get in here, just bump the extra length to
+			 * 10 bytes.  That will encompass anything we're
+			 * going to set here.
+			 */
+			sense->extra_len = 10;
+			sense_len = (int)va_arg(ap, int);
+			len_to_copy = MIN(sense_len, SSD_EXTRA_MAX -
+					  sense->extra_len);
+			data = (uint8_t *)va_arg(ap, uint8_t *);
+
+			switch (elem_type) {
+			case SSD_ELEM_SKS:
+				/*
+				 * The user passed in pre-formatted sense
+				 * key specific data.
+				 */
+				bcopy(data, &sense->sense_key_spec[0],
+				      MIN(sizeof(sense->sense_key_spec),
+				      sense_len));
+				break;
+			case SSD_ELEM_INFO:
+			case SSD_ELEM_COMMAND: {
+				uint8_t *data_dest;
+				int i;
+
+				if (elem_type == SSD_ELEM_COMMAND)
+					data_dest = &sense->cmd_spec_info[0];
+				else {
+					data_dest = &sense->info[0];
+					/*
+					 * We're setting the info field, so
+					 * set the valid bit.
+					 */
+					sense->error_code |= SSD_ERRCODE_VALID;
+				}
+
+				/*
+			 	 * Copy this in reverse so that if we have
+				 * less than 4 bytes to fill, the least
+				 * significant bytes will be at the end.
+				 * If we have more than 4 bytes, only the
+				 * least significant bytes will be included.
+				 */
+				for (i = sense_len - 1; i >= 0 &&
+				     len_to_copy > 0; i--, len_to_copy--)
+					data_dest[len_to_copy - 1] = data[i];
+
+				break;
+			}
+			case SSD_ELEM_FRU:
+				sense->fru = *data;
+				break;
+			case SSD_ELEM_STREAM:
+				sense->flags |= *data;
+				break;
+			case SSD_ELEM_DESC:
+			default:
+
+				/*
+				 * If the user passes in descriptor sense,
+				 * we can't handle that in fixed format.
+				 * So just skip it, and any unknown argument
+				 * types.
+				 */
+				break;
+			}
+		}
+	}
+}
+
+void
+scsi_set_sense_data(struct scsi_sense_data *sense_data, 
+		    scsi_sense_data_type sense_format, int current_error,
+		    int sense_key, int asc, int ascq, ...) 
+{
+	va_list ap;
+
+	va_start(ap, ascq);
+	scsi_set_sense_data_va(sense_data, sense_format, current_error,
+			       sense_key, asc, ascq, ap);
+	va_end(ap);
+}
+
+/*
+ * Get sense information for three similar sense data types.
+ */
+int
+scsi_get_sense_info(struct scsi_sense_data *sense_data, u_int sense_len,
+		    uint8_t info_type, uint64_t *info, int64_t *signed_info)
+{
+	scsi_sense_data_type sense_type;
+
+	if (sense_len == 0)
+		goto bailout;
+
+	sense_type = scsi_sense_type(sense_data);
+
+	switch (sense_type) {
+	case SSD_TYPE_DESC: {
+		struct scsi_sense_data_desc *sense;
+		uint8_t *desc;
+
+		sense = (struct scsi_sense_data_desc *)sense_data;
+
+		desc = scsi_find_desc(sense, sense_len, info_type);
+		if (desc == NULL)
+			goto bailout;
+
+		switch (info_type) {
+		case SSD_DESC_INFO: {
+			struct scsi_sense_info *info_desc;
+
+			info_desc = (struct scsi_sense_info *)desc;
+			*info = scsi_8btou64(info_desc->info);
+			if (signed_info != NULL)
+				*signed_info = *info;
+			break;
+		}
+		case SSD_DESC_COMMAND: {
+			struct scsi_sense_command *cmd_desc;
+
+			cmd_desc = (struct scsi_sense_command *)desc;
+
+			*info = scsi_8btou64(cmd_desc->command_info);
+			if (signed_info != NULL)
+				*signed_info = *info;
+			break;
+		}
+		case SSD_DESC_FRU: {
+			struct scsi_sense_fru *fru_desc;
+
+			fru_desc = (struct scsi_sense_fru *)desc;
+
+			*info = fru_desc->fru;
+			if (signed_info != NULL)
+				*signed_info = (int8_t)fru_desc->fru;
+			break;
+		}
+		default:
+			goto bailout;
+			break;
+		}
+		break;
+	}
+	case SSD_TYPE_FIXED: {
+		struct scsi_sense_data_fixed *sense;
+
+		sense = (struct scsi_sense_data_fixed *)sense_data;
+
+		switch (info_type) {
+		case SSD_DESC_INFO: {
+			uint32_t info_val;
+
+			if ((sense->error_code & SSD_ERRCODE_VALID) == 0)
+				goto bailout;
+
+			if (SSD_FIXED_IS_PRESENT(sense, sense_len, info) == 0)
+				goto bailout;
+
+			info_val = scsi_4btoul(sense->info);
+
+			*info = info_val;
+			if (signed_info != NULL)
+				*signed_info = (int32_t)info_val;
+			break;
+		}
+		case SSD_DESC_COMMAND: {
+			uint32_t cmd_val;
+
+			if ((SSD_FIXED_IS_PRESENT(sense, sense_len,
+			     cmd_spec_info) == 0)
+			 || (SSD_FIXED_IS_FILLED(sense, cmd_spec_info) == 0)) 
+				goto bailout;
+
+			cmd_val = scsi_4btoul(sense->cmd_spec_info);
+			if (cmd_val == 0)
+				goto bailout;
+
+			*info = cmd_val;
+			if (signed_info != NULL)
+				*signed_info = (int32_t)cmd_val;
+			break;
+		}
+		case SSD_DESC_FRU:
+			if ((SSD_FIXED_IS_PRESENT(sense, sense_len, fru) == 0)
+			 || (SSD_FIXED_IS_FILLED(sense, fru) == 0))
+				goto bailout;
+
+			if (sense->fru == 0)
+				goto bailout;
+
+			*info = sense->fru;
+			if (signed_info != NULL)
+				*signed_info = (int8_t)sense->fru;
+			break;
+		default:
+			goto bailout;
+			break;
+		}
+		break;
+	}
+	default: 
+		goto bailout;
+		break;
+	}
+
+	return (0);
+bailout:
+	return (1);
+}
+
+int
+scsi_get_sks(struct scsi_sense_data *sense_data, u_int sense_len, uint8_t *sks)
+{
+	scsi_sense_data_type sense_type;
+
+	if (sense_len == 0)
+		goto bailout;
+
+	sense_type = scsi_sense_type(sense_data);
+
+	switch (sense_type) {
+	case SSD_TYPE_DESC: {
+		struct scsi_sense_data_desc *sense;
+		struct scsi_sense_sks *desc;
+
+		sense = (struct scsi_sense_data_desc *)sense_data;
+
+		desc = (struct scsi_sense_sks *)scsi_find_desc(sense, sense_len,
+							       SSD_DESC_SKS);
+		if (desc == NULL)
+			goto bailout;
+
+		/*
+		 * No need to check the SKS valid bit for descriptor sense.
+		 * If the descriptor is present, it is valid.
+		 */
+		bcopy(desc->sense_key_spec, sks, sizeof(desc->sense_key_spec));
+		break;
+	}
+	case SSD_TYPE_FIXED: {
+		struct scsi_sense_data_fixed *sense;
+
+		sense = (struct scsi_sense_data_fixed *)sense_data;
+
+		if ((SSD_FIXED_IS_PRESENT(sense, sense_len, sense_key_spec)== 0)
+		 || (SSD_FIXED_IS_FILLED(sense, sense_key_spec) == 0))
+			goto bailout;
+
+		if ((sense->sense_key_spec[0] & SSD_SCS_VALID) == 0)
+			goto bailout;
+
+		bcopy(sense->sense_key_spec, sks,sizeof(sense->sense_key_spec));
+		break;
+	}
+	default:
+		goto bailout;
+		break;
+	}
+	return (0);
+bailout:
+	return (1);
+}
+
+/*
+ * Provide a common interface for fixed and descriptor sense to detect
+ * whether we have block-specific sense information.  It is clear by the
+ * presence of the block descriptor in descriptor mode, but we have to
+ * infer from the inquiry data and ILI bit in fixed mode.
+ */
+int
+scsi_get_block_info(struct scsi_sense_data *sense_data, u_int sense_len,
+		    struct scsi_inquiry_data *inq_data, uint8_t *block_bits)
+{
+	scsi_sense_data_type sense_type;
+
+	if (inq_data != NULL) {
+		switch (SID_TYPE(inq_data)) {
+		case T_DIRECT:
+		case T_RBC:
+			break;
+		default:
+			goto bailout;
+			break;
+		}
+	}
+
+	sense_type = scsi_sense_type(sense_data);
+
+	switch (sense_type) {
+	case SSD_TYPE_DESC: {
+		struct scsi_sense_data_desc *sense;
+		struct scsi_sense_block *block;
+
+		sense = (struct scsi_sense_data_desc *)sense_data;
+
+		block = (struct scsi_sense_block *)scsi_find_desc(sense,
+		    sense_len, SSD_DESC_BLOCK);
+		if (block == NULL)
+			goto bailout;
+
+		*block_bits = block->byte3;
+		break;
+	}
+	case SSD_TYPE_FIXED: {
+		struct scsi_sense_data_fixed *sense;
+
+		sense = (struct scsi_sense_data_fixed *)sense_data;
+
+		if (SSD_FIXED_IS_PRESENT(sense, sense_len, flags) == 0)
+			goto bailout;
+
+		if ((sense->flags & SSD_ILI) == 0)
+			goto bailout;
+
+		*block_bits = sense->flags & SSD_ILI;
+		break;
+	}
+	default:
+		goto bailout;
+		break;
+	}
+	return (0);
+bailout:
+	return (1);
+}
+
+int
+scsi_get_stream_info(struct scsi_sense_data *sense_data, u_int sense_len,
+		     struct scsi_inquiry_data *inq_data, uint8_t *stream_bits)
+{
+	scsi_sense_data_type sense_type;
+
+	if (inq_data != NULL) {
+		switch (SID_TYPE(inq_data)) {
+		case T_SEQUENTIAL:
+			break;
+		default:
+			goto bailout;
+			break;
+		}
+	}
+
+	sense_type = scsi_sense_type(sense_data);
+
+	switch (sense_type) {
+	case SSD_TYPE_DESC: {
+		struct scsi_sense_data_desc *sense;
+		struct scsi_sense_stream *stream;
+
+		sense = (struct scsi_sense_data_desc *)sense_data;
+
+		stream = (struct scsi_sense_stream *)scsi_find_desc(sense,
+		    sense_len, SSD_DESC_STREAM);
+		if (stream == NULL)
+			goto bailout;
+
+		*stream_bits = stream->byte3;
+		break;
+	}
+	case SSD_TYPE_FIXED: {
+		struct scsi_sense_data_fixed *sense;
+
+		sense = (struct scsi_sense_data_fixed *)sense_data;
+
+		if (SSD_FIXED_IS_PRESENT(sense, sense_len, flags) == 0)
+			goto bailout;
+
+		if ((sense->flags & (SSD_ILI|SSD_EOM|SSD_FILEMARK)) == 0)
+			goto bailout;
+
+		*stream_bits = sense->flags & (SSD_ILI|SSD_EOM|SSD_FILEMARK);
+		break;
+	}
+	default:
+		goto bailout;
+		break;
+	}
+	return (0);
+bailout:
+	return (1);
+}
+
+void
+scsi_info_sbuf(struct sbuf *sb, uint8_t *cdb, int cdb_len,
+	       struct scsi_inquiry_data *inq_data, uint64_t info)
+{
+	sbuf_printf(sb, "Info: %#jx", info);
+}
+
+void
+scsi_command_sbuf(struct sbuf *sb, uint8_t *cdb, int cdb_len,
+		  struct scsi_inquiry_data *inq_data, uint64_t csi)
+{
+	sbuf_printf(sb, "Command Specific Info: %#jx", csi);
+}
+
+
+void
+scsi_progress_sbuf(struct sbuf *sb, uint16_t progress)
+{
+	sbuf_printf(sb, "Progress: %d%% (%d/%d) complete",
+		    (progress * 100) / SSD_SKS_PROGRESS_DENOM,
+		    progress, SSD_SKS_PROGRESS_DENOM);
+}
+
+/*
+ * Returns 1 for failure (i.e. SKS isn't valid) and 0 for success.
+ */
+int
+scsi_sks_sbuf(struct sbuf *sb, int sense_key, uint8_t *sks)
+{
+	if ((sks[0] & SSD_SKS_VALID) == 0)
+		return (1);
+
+	switch (sense_key) {
+	case SSD_KEY_ILLEGAL_REQUEST: {
+		struct scsi_sense_sks_field *field;
+		int bad_command;
+		char tmpstr[40];
+
+		/*Field Pointer*/
+		field = (struct scsi_sense_sks_field *)sks;
+
+		if (field->byte0 & SSD_SKS_FIELD_CMD)
+			bad_command = 1;
+		else
+			bad_command = 0;
+
+		tmpstr[0] = '\0';
+
+		/* Bit pointer is valid */
+		if (field->byte0 & SSD_SKS_BPV)
+			snprintf(tmpstr, sizeof(tmpstr), "bit %d ",
+				 field->byte0 & SSD_SKS_BIT_VALUE);
+
+		sbuf_printf(sb, "%s byte %d %sis invalid",
+			    bad_command ? "Command" : "Data",
+			    scsi_2btoul(field->field), tmpstr);
+		break;
+	}
+	case SSD_KEY_UNIT_ATTENTION: {
+		struct scsi_sense_sks_overflow *overflow;
+
+		overflow = (struct scsi_sense_sks_overflow *)sks;
+
+		/*UA Condition Queue Overflow*/
+		sbuf_printf(sb, "Unit Attention Condition Queue %s",
+			    (overflow->byte0 & SSD_SKS_OVERFLOW_SET) ?
+			    "Overflowed" : "Did Not Overflow??");
+		break;
+	}
+	case SSD_KEY_RECOVERED_ERROR:
+	case SSD_KEY_HARDWARE_ERROR:
+	case SSD_KEY_MEDIUM_ERROR: {
+		struct scsi_sense_sks_retry *retry;
+
+		/*Actual Retry Count*/
+		retry = (struct scsi_sense_sks_retry *)sks;
+
+		sbuf_printf(sb, "Actual Retry Count: %d",
+			    scsi_2btoul(retry->actual_retry_count));
+		break;
+	}
+	case SSD_KEY_NO_SENSE:
+	case SSD_KEY_NOT_READY: {
+		struct scsi_sense_sks_progress *progress;
+		int progress_val;
+
+		/*Progress Indication*/
+		progress = (struct scsi_sense_sks_progress *)sks;
+		progress_val = scsi_2btoul(progress->progress);
+
+		scsi_progress_sbuf(sb, progress_val);
+		break;
+	}
+	case SSD_KEY_COPY_ABORTED: {
+		struct scsi_sense_sks_segment *segment;
+		char tmpstr[40];
+
+		/*Segment Pointer*/
+		segment = (struct scsi_sense_sks_segment *)sks;
+
+		tmpstr[0] = '\0';
+
+		if (segment->byte0 & SSD_SKS_SEGMENT_BPV)
+			snprintf(tmpstr, sizeof(tmpstr), "bit %d ",
+				 segment->byte0 & SSD_SKS_SEGMENT_BITPTR);
+
+		sbuf_printf(sb, "%s byte %d %sis invalid", (segment->byte0 &
+			    SSD_SKS_SEGMENT_SD) ? "Segment" : "Data",
+			    scsi_2btoul(segment->field), tmpstr);
+		break;
+	}
+	default:
+		sbuf_printf(sb, "Sense Key Specific: %#x,%#x", sks[0],
+			    scsi_2btoul(&sks[1]));
+		break;
+	}
+
+	return (0);
+}
+
+void
+scsi_fru_sbuf(struct sbuf *sb, uint64_t fru)
+{
+	sbuf_printf(sb, "Field Replaceable Unit: %d", (int)fru);
+}
+
+void
+scsi_stream_sbuf(struct sbuf *sb, uint8_t stream_bits, uint64_t info)
+{
+	int need_comma;
+
+	need_comma = 0;
+	/*
+	 * XXX KDM this needs more descriptive decoding.
+	 */
+	if (stream_bits & SSD_DESC_STREAM_FM) {
+		sbuf_printf(sb, "Filemark");
+		need_comma = 1;
+	}
+
+	if (stream_bits & SSD_DESC_STREAM_EOM) {
+		sbuf_printf(sb, "%sEOM", (need_comma) ? "," : "");
+		need_comma = 1;
+	}
+
+	if (stream_bits & SSD_DESC_STREAM_ILI)
+		sbuf_printf(sb, "%sILI", (need_comma) ? "," : "");
+
+	sbuf_printf(sb, ": Info: %#jx", (uintmax_t) info);
+}
+
+void
+scsi_block_sbuf(struct sbuf *sb, uint8_t block_bits, uint64_t info)
+{
+	if (block_bits & SSD_DESC_BLOCK_ILI)
+		sbuf_printf(sb, "ILI: residue %#jx", (uintmax_t) info);
+}
+
+void
+scsi_sense_info_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+		     u_int sense_len, uint8_t *cdb, int cdb_len,
+		     struct scsi_inquiry_data *inq_data,
+		     struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_info *info;
+
+	info = (struct scsi_sense_info *)header;
+
+	scsi_info_sbuf(sb, cdb, cdb_len, inq_data, scsi_8btou64(info->info));
+}
+
+void
+scsi_sense_command_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			u_int sense_len, uint8_t *cdb, int cdb_len,
+			struct scsi_inquiry_data *inq_data,
+			struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_command *command;
+
+	command = (struct scsi_sense_command *)header;
+
+	scsi_command_sbuf(sb, cdb, cdb_len, inq_data,
+			  scsi_8btou64(command->command_info));
+}
+
+void
+scsi_sense_sks_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+		    u_int sense_len, uint8_t *cdb, int cdb_len,
+		    struct scsi_inquiry_data *inq_data,
+		    struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_sks *sks;
+	int error_code, sense_key, asc, ascq;
+
+	sks = (struct scsi_sense_sks *)header;
+
+	scsi_extract_sense_len(sense, sense_len, &error_code, &sense_key,
+			       &asc, &ascq, /*show_errors*/ 1);
+
+	scsi_sks_sbuf(sb, sense_key, sks->sense_key_spec);
+}
+
+void
+scsi_sense_fru_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+		    u_int sense_len, uint8_t *cdb, int cdb_len,
+		    struct scsi_inquiry_data *inq_data,
+		    struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_fru *fru;
+
+	fru = (struct scsi_sense_fru *)header;
+
+	scsi_fru_sbuf(sb, (uint64_t)fru->fru);
+}
+
+void
+scsi_sense_stream_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+		       u_int sense_len, uint8_t *cdb, int cdb_len,
+		       struct scsi_inquiry_data *inq_data,
+		       struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_stream *stream;
+	uint64_t info;
+
+	stream = (struct scsi_sense_stream *)header;
+	info = 0;
+
+	scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &info, NULL);
+
+	scsi_stream_sbuf(sb, stream->byte3, info);
+}
+
+void
+scsi_sense_block_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+		      u_int sense_len, uint8_t *cdb, int cdb_len,
+		      struct scsi_inquiry_data *inq_data,
+		      struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_block *block;
+	uint64_t info;
+
+	block = (struct scsi_sense_block *)header;
+	info = 0;
+
+	scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &info, NULL);
+
+	scsi_block_sbuf(sb, block->byte3, info);
+}
+
+void
+scsi_sense_progress_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			 u_int sense_len, uint8_t *cdb, int cdb_len,
+			 struct scsi_inquiry_data *inq_data,
+			 struct scsi_sense_desc_header *header)
+{
+	struct scsi_sense_progress *progress;
+	const char *sense_key_desc;
+	const char *asc_desc;
+	int progress_val;
+
+	progress = (struct scsi_sense_progress *)header;
+
+	/*
+	 * Get descriptions for the sense key, ASC, and ASCQ in the
+	 * progress descriptor.  These could be different than the values
+	 * in the overall sense data.
+	 */
+	scsi_sense_desc(progress->sense_key, progress->add_sense_code,
+			progress->add_sense_code_qual, inq_data,
+			&sense_key_desc, &asc_desc);
+
+	progress_val = scsi_2btoul(progress->progress);
+
+	/*
+	 * The progress indicator is for the operation described by the
+	 * sense key, ASC, and ASCQ in the descriptor.
+	 */
+	sbuf_cat(sb, sense_key_desc);
+	sbuf_printf(sb, " asc:%x,%x (%s): ", progress->add_sense_code, 
+		    progress->add_sense_code_qual, asc_desc);
+	scsi_progress_sbuf(sb, progress_val);
+}
+
+/*
+ * Generic sense descriptor printing routine.  This is used when we have
+ * not yet implemented a specific printing routine for this descriptor.
+ */
+void
+scsi_sense_generic_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			u_int sense_len, uint8_t *cdb, int cdb_len,
+			struct scsi_inquiry_data *inq_data,
+			struct scsi_sense_desc_header *header)
+{
+	int i;
+	uint8_t *buf_ptr;
+
+	sbuf_printf(sb, "Descriptor %#x:", header->desc_type);
+
+	buf_ptr = (uint8_t *)&header[1];
+
+	for (i = 0; i < header->length; i++, buf_ptr++)
+		sbuf_printf(sb, " %02x", *buf_ptr);
+}
+
+/*
+ * Keep this list in numeric order.  This speeds the array traversal.
+ */
+struct scsi_sense_desc_printer {
+	uint8_t desc_type;
+	/*
+	 * The function arguments here are the superset of what is needed
+	 * to print out various different descriptors.  Command and
+	 * information descriptors need inquiry data and command type.
+	 * Sense key specific descriptors need the sense key.
+	 *
+	 * The sense, cdb, and inquiry data arguments may be NULL, but the
+	 * information printed may not be fully decoded as a result.
+	 */
+	void (*print_func)(struct sbuf *sb, struct scsi_sense_data *sense,
+			   u_int sense_len, uint8_t *cdb, int cdb_len,
+			   struct scsi_inquiry_data *inq_data,
+			   struct scsi_sense_desc_header *header);
+} scsi_sense_printers[] = {
+	{SSD_DESC_INFO, scsi_sense_info_sbuf},
+	{SSD_DESC_COMMAND, scsi_sense_command_sbuf},
+	{SSD_DESC_SKS, scsi_sense_sks_sbuf},
+	{SSD_DESC_FRU, scsi_sense_fru_sbuf},
+	{SSD_DESC_STREAM, scsi_sense_stream_sbuf},
+	{SSD_DESC_BLOCK, scsi_sense_block_sbuf},
+	{SSD_DESC_PROGRESS, scsi_sense_progress_sbuf}
+};
+
+void
+scsi_sense_desc_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+		     u_int sense_len, uint8_t *cdb, int cdb_len,
+		     struct scsi_inquiry_data *inq_data,
+		     struct scsi_sense_desc_header *header)
+{
+	int i, found;
+
+	for (i = 0, found = 0; i < (sizeof(scsi_sense_printers) /
+	     sizeof(scsi_sense_printers[0])); i++) {
+		struct scsi_sense_desc_printer *printer;
+
+		printer = &scsi_sense_printers[i];
+
+		/*
+		 * The list is sorted, so quit if we've passed our
+		 * descriptor number.
+		 */
+		if (printer->desc_type > header->desc_type)
+			break;
+
+		if (printer->desc_type != header->desc_type)
+			continue;
+
+		printer->print_func(sb, sense, sense_len, cdb, cdb_len,
+				    inq_data, header);
+
+		return;
+	}
+
+	/*
+	 * No specific printing routine, so use the generic routine.
+	 */
+	scsi_sense_generic_sbuf(sb, sense, sense_len, cdb, cdb_len,
+				inq_data, header);
+}
+
+scsi_sense_data_type
+scsi_sense_type(struct scsi_sense_data *sense_data)
+{
+	switch (sense_data->error_code & SSD_ERRCODE) {
+	case SSD_DESC_CURRENT_ERROR:
+	case SSD_DESC_DEFERRED_ERROR:
+		return (SSD_TYPE_DESC);
+		break;
+	case SSD_CURRENT_ERROR:
+	case SSD_DEFERRED_ERROR:
+		return (SSD_TYPE_FIXED);
+		break;
+	default:
+		break;
+	}
+
+	return (SSD_TYPE_NONE);
+}
+
+struct scsi_print_sense_info {
+	struct sbuf *sb;
+	char *path_str;
+	uint8_t *cdb;
+	int cdb_len;
+	struct scsi_inquiry_data *inq_data;
+};
+
+static int
+scsi_print_desc_func(struct scsi_sense_data_desc *sense, u_int sense_len,
+		     struct scsi_sense_desc_header *header, void *arg)
+{
+	struct scsi_print_sense_info *print_info;
+
+	print_info = (struct scsi_print_sense_info *)arg;
+
+	switch (header->desc_type) {
+	case SSD_DESC_INFO:
+	case SSD_DESC_FRU:
+	case SSD_DESC_COMMAND:
+	case SSD_DESC_SKS:
+	case SSD_DESC_BLOCK:
+	case SSD_DESC_STREAM:
+		/*
+		 * We have already printed these descriptors, if they are
+		 * present.
+		 */
+		break;
+	default: {
+		sbuf_printf(print_info->sb, "%s", print_info->path_str);
+		scsi_sense_desc_sbuf(print_info->sb,
+				     (struct scsi_sense_data *)sense, sense_len,
+				     print_info->cdb, print_info->cdb_len,
+				     print_info->inq_data, header);
+		sbuf_printf(print_info->sb, "\n");
+		break;
+	}
+	}
+
+	/*
+	 * Tell the iterator that we want to see more descriptors if they
+	 * are present.
+	 */
+	return (0);
+}
+
+void
+scsi_sense_only_sbuf(struct scsi_sense_data *sense, u_int sense_len,
+		     struct sbuf *sb, char *path_str,
+		     struct scsi_inquiry_data *inq_data, uint8_t *cdb,
+		     int cdb_len)
+{
+	int error_code, sense_key, asc, ascq;
+
+	sbuf_cat(sb, path_str);
+
+	scsi_extract_sense_len(sense, sense_len, &error_code, &sense_key,
+			       &asc, &ascq, /*show_errors*/ 1);
+
+	sbuf_printf(sb, "SCSI sense: ");
+	switch (error_code) {
+	case SSD_DEFERRED_ERROR:
+	case SSD_DESC_DEFERRED_ERROR:
+		sbuf_printf(sb, "Deferred error: ");
+
+		/* FALLTHROUGH */
+	case SSD_CURRENT_ERROR:
+	case SSD_DESC_CURRENT_ERROR:
+	{
+		struct scsi_sense_data_desc *desc_sense;
+		struct scsi_print_sense_info print_info;
+		const char *sense_key_desc;
+		const char *asc_desc;
+		uint8_t sks[3];
+		uint64_t val;
+		int info_valid;
+
+		/*
+		 * Get descriptions for the sense key, ASC, and ASCQ.  If
+		 * these aren't present in the sense data (i.e. the sense
+		 * data isn't long enough), the -1 values that
+		 * scsi_extract_sense_len() returns will yield default
+		 * or error descriptions.
+		 */
+		scsi_sense_desc(sense_key, asc, ascq, inq_data,
+				&sense_key_desc, &asc_desc);
+
+		/*
+		 * We first print the sense key and ASC/ASCQ.
+		 */
+		sbuf_cat(sb, sense_key_desc);
+		sbuf_printf(sb, " asc:%x,%x (%s)\n", asc, ascq, asc_desc);
+
+		/*
+		 * Get the info field if it is valid.
+		 */
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO,
+					&val, NULL) == 0)
+			info_valid = 1;
+		else
+			info_valid = 0;
+
+		if (info_valid != 0) {
+			uint8_t bits;
+
+			/*
+			 * Determine whether we have any block or stream
+			 * device-specific information.
+			 */
+			if (scsi_get_block_info(sense, sense_len, inq_data,
+						&bits) == 0) {
+				sbuf_cat(sb, path_str);
+				scsi_block_sbuf(sb, bits, val);
+				sbuf_printf(sb, "\n");
+			} else if (scsi_get_stream_info(sense, sense_len,
+							inq_data, &bits) == 0) {
+				sbuf_cat(sb, path_str);
+				scsi_stream_sbuf(sb, bits, val);
+				sbuf_printf(sb, "\n");
+			} else if (val != 0) {
+				/*
+				 * The information field can be valid but 0.
+				 * If the block or stream bits aren't set,
+				 * and this is 0, it isn't terribly useful
+				 * to print it out.
+				 */
+				sbuf_cat(sb, path_str);
+				scsi_info_sbuf(sb, cdb, cdb_len, inq_data, val);
+				sbuf_printf(sb, "\n");
+			}
+		}
+
+		/* 
+		 * Print the FRU.
+		 */
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_FRU,
+					&val, NULL) == 0) {
+			sbuf_cat(sb, path_str);
+			scsi_fru_sbuf(sb, val);
+			sbuf_printf(sb, "\n");
+		}
+
+		/*
+		 * Print any command-specific information.
+		 */
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_COMMAND,
+					&val, NULL) == 0) {
+			sbuf_cat(sb, path_str);
+			scsi_command_sbuf(sb, cdb, cdb_len, inq_data, val);
+			sbuf_printf(sb, "\n");
+		}
+
+		/*
+		 * Print out any sense-key-specific information.
+		 */
+		if (scsi_get_sks(sense, sense_len, sks) == 0) {
+			sbuf_cat(sb, path_str);
+			scsi_sks_sbuf(sb, sense_key, sks);
+			sbuf_printf(sb, "\n");
+		}
+
+		/*
+		 * If this is fixed sense, we're done.  If we have
+		 * descriptor sense, we might have more information
+		 * available.
+		 */
+		if (scsi_sense_type(sense) != SSD_TYPE_DESC)
+			break;
+
+		desc_sense = (struct scsi_sense_data_desc *)sense;
+
+		print_info.sb = sb;
+		print_info.path_str = path_str;
+		print_info.cdb = cdb;
+		print_info.cdb_len = cdb_len;
+		print_info.inq_data = inq_data;
+
+		/*
+		 * Print any sense descriptors that we have not already printed.
+		 */
+		scsi_desc_iterate(desc_sense, sense_len, scsi_print_desc_func,
+				  &print_info);
+		break;
+
+	}
+	case -1:
+		/*
+		 * scsi_extract_sense_len() sets values to -1 if the
+		 * show_errors flag is set and they aren't present in the
+		 * sense data.  This means that sense_len is 0.
+		 */
+		sbuf_printf(sb, "No sense data present\n");
+		break;
+	default: {
+		sbuf_printf(sb, "Error code 0x%x", error_code);
+		if (sense->error_code & SSD_ERRCODE_VALID) {
+			struct scsi_sense_data_fixed *fixed_sense;
+
+			fixed_sense = (struct scsi_sense_data_fixed *)sense;
+
+			if (SSD_FIXED_IS_PRESENT(fixed_sense, sense_len, info)){
+				uint32_t info;
+
+				info = scsi_4btoul(fixed_sense->info);
+
+				sbuf_printf(sb, " at block no. %d (decimal)",
+					    info);
+			}
+		}
+		sbuf_printf(sb, "\n");
+		break;
+	}
+	}
+}
 
 /*
  * scsi_sense_sbuf() returns 0 for success and -1 for failure.
@@ -3059,11 +4419,8 @@ scsi_sense_sbuf(struct cam_device *device, struct ccb_scsiio *csio,
 #ifdef _KERNEL
 	struct	  ccb_getdev *cgd;
 #endif /* _KERNEL */
-	u_int32_t info;
-	int	  error_code;
-	int	  sense_key;
-	int	  asc, ascq;
 	char	  path_str[64];
+	uint8_t	  *cdb;
 
 #ifndef _KERNEL
 	if (device == NULL)
@@ -3161,129 +4518,14 @@ scsi_sense_sbuf(struct cam_device *device, struct ccb_scsiio *csio,
 			sense = &csio->sense_data;
 	}
 
+	if (csio->ccb_h.flags & CAM_CDB_POINTER)
+		cdb = csio->cdb_io.cdb_ptr;
+	else
+		cdb = csio->cdb_io.cdb_bytes;
 
-	sbuf_cat(sb, path_str);
-
-	error_code = sense->error_code & SSD_ERRCODE;
-	sense_key = sense->flags & SSD_KEY;
-
-	sbuf_printf(sb, "SCSI sense: ");
-	switch (error_code) {
-	case SSD_DEFERRED_ERROR:
-		sbuf_printf(sb, "Deferred error: ");
-
-		/* FALLTHROUGH */
-	case SSD_CURRENT_ERROR:
-	{
-		const char *sense_key_desc;
-		const char *asc_desc;
-
-		asc = (sense->extra_len >= 5) ? sense->add_sense_code : 0;
-		ascq = (sense->extra_len >= 6) ? sense->add_sense_code_qual : 0;
-		scsi_sense_desc(sense_key, asc, ascq, inq_data,
-				&sense_key_desc, &asc_desc);
-		sbuf_cat(sb, sense_key_desc);
-
-		info = scsi_4btoul(sense->info);
-		
-		if (sense->error_code & SSD_ERRCODE_VALID) {
-
-			switch (sense_key) {
-			case SSD_KEY_NOT_READY:
-			case SSD_KEY_ILLEGAL_REQUEST:
-			case SSD_KEY_UNIT_ATTENTION:
-			case SSD_KEY_DATA_PROTECT:
-				break;
-			case SSD_KEY_BLANK_CHECK:
-				sbuf_printf(sb, " req sz: %d (decimal)", info);
-				break;
-			default:
-				if (info) {
-					if (sense->flags & SSD_ILI) {
-						sbuf_printf(sb, " ILI (length "
-							"mismatch): %d", info);
-			
-					} else {
-						sbuf_printf(sb, " info:%x", 
-							    info);
-					}
-				}
-			}
-		} else if (info) {
-			sbuf_printf(sb, " info?:%x", info);
-		}
-
-		if (sense->extra_len >= 4) {
-			if (bcmp(sense->cmd_spec_info, "\0\0\0\0", 4)) {
-				sbuf_printf(sb, " csi:%x,%x,%x,%x",
-					    sense->cmd_spec_info[0],
-					    sense->cmd_spec_info[1],
-					    sense->cmd_spec_info[2],
-					    sense->cmd_spec_info[3]);
-			}
-		}
-
-		sbuf_printf(sb, " asc:%x,%x (%s)", asc, ascq, asc_desc);
-
-		if (sense->extra_len >= 7 && sense->fru) {
-			sbuf_printf(sb, " field replaceable unit: %x", 
-				    sense->fru);
-		}
-
-		if ((sense->extra_len >= 10)
-		 && (sense->sense_key_spec[0] & SSD_SCS_VALID) != 0) {
-			switch(sense_key) {
-			case SSD_KEY_ILLEGAL_REQUEST: {
-				int bad_command;
-				char tmpstr2[40];
-
-				if (sense->sense_key_spec[0] & 0x40)
-					bad_command = 1;
-				else
-					bad_command = 0;
-
-				tmpstr2[0] = '\0';
-
-				/* Bit pointer is valid */
-				if (sense->sense_key_spec[0] & 0x08)
-					snprintf(tmpstr2, sizeof(tmpstr2),
-						 "bit %d ",
-						sense->sense_key_spec[0] & 0x7);
-				sbuf_printf(sb, ": %s byte %d %sis invalid",
-					    bad_command ? "Command" : "Data",
-					    scsi_2btoul(
-					    &sense->sense_key_spec[1]),
-					    tmpstr2);
-				break;
-			}
-			case SSD_KEY_RECOVERED_ERROR:
-			case SSD_KEY_HARDWARE_ERROR:
-			case SSD_KEY_MEDIUM_ERROR:
-				sbuf_printf(sb, " actual retry count: %d",
-					    scsi_2btoul(
-					    &sense->sense_key_spec[1]));
-				break;
-			default:
-				sbuf_printf(sb, " sks:%#x,%#x", 
-					    sense->sense_key_spec[0],
-					    scsi_2btoul(
-					    &sense->sense_key_spec[1]));
-				break;
-			}
-		}
-		break;
-
-	}
-	default:
-		sbuf_printf(sb, "Error code 0x%x", sense->error_code);
-		if (sense->error_code & SSD_ERRCODE_VALID) {
-			sbuf_printf(sb, " at block no. %d (decimal)",
-				    info = scsi_4btoul(sense->info));
-		}
-	}
-
-	sbuf_printf(sb, "\n");
-
+	scsi_sense_only_sbuf(sense, csio->sense_len - csio->sense_resid, sb,
+			     path_str, inq_data, cdb, csio->cdb_len);
+			 
 #ifdef _KERNEL
 	xpt_free_ccb((union ccb*)cgd);
 #endif /* _KERNEL/!_KERNEL */
@@ -3354,6 +4596,135 @@ scsi_sense_print(struct cam_device *device, struct ccb_scsiio *csio,
 
 #endif /* _KERNEL/!_KERNEL */
 
+/*
+ * Extract basic sense information.  This is backward-compatible with the
+ * previous implementation.  For new implementations,
+ * scsi_extract_sense_len() is recommended.
+ */
+void
+scsi_extract_sense(struct scsi_sense_data *sense_data, int *error_code,
+		   int *sense_key, int *asc, int *ascq)
+{
+	scsi_extract_sense_len(sense_data, sizeof(*sense_data), error_code,
+			       sense_key, asc, ascq, /*show_errors*/ 0);
+}
+
+/*
+ * Extract basic sense information.  If show_errors is set, sense values
+ * will be set to -1 if they are not present.
+ */
+void
+scsi_extract_sense_len(struct scsi_sense_data *sense_data, u_int sense_len,
+		       int *error_code, int *sense_key, int *asc, int *ascq,
+		       int show_errors)
+{
+	/*
+	 * If we have no length, we have no sense.
+	 */
+	if (sense_len == 0) {
+		if (show_errors == 0) {
+			*error_code = 0;
+			*sense_key = 0;
+			*asc = 0;
+			*ascq = 0;
+		} else {
+			*error_code = -1;
+			*sense_key = -1;
+			*asc = -1;
+			*ascq = -1;
+		}
+		return;
+	}
+
+	*error_code = sense_data->error_code & SSD_ERRCODE;
+
+	switch (*error_code) {
+	case SSD_DESC_CURRENT_ERROR:
+	case SSD_DESC_DEFERRED_ERROR: {
+		struct scsi_sense_data_desc *sense;
+
+		sense = (struct scsi_sense_data_desc *)sense_data;
+
+		if (SSD_DESC_IS_PRESENT(sense, sense_len, sense_key))
+			*sense_key = sense->sense_key & SSD_KEY;
+		else
+			*sense_key = (show_errors) ? -1 : 0;
+
+		if (SSD_DESC_IS_PRESENT(sense, sense_len, add_sense_code))
+			*asc = sense->add_sense_code;
+		else
+			*asc = (show_errors) ? -1 : 0;
+
+		if (SSD_DESC_IS_PRESENT(sense, sense_len, add_sense_code_qual))
+			*ascq = sense->add_sense_code_qual;
+		else
+			*ascq = (show_errors) ? -1 : 0;
+		break;
+	}
+	case SSD_CURRENT_ERROR:
+	case SSD_DEFERRED_ERROR:
+	default: {
+		struct scsi_sense_data_fixed *sense;
+
+		sense = (struct scsi_sense_data_fixed *)sense_data;
+
+		if (SSD_FIXED_IS_PRESENT(sense, sense_len, flags))
+			*sense_key = sense->flags & SSD_KEY;
+		else
+			*sense_key = (show_errors) ? -1 : 0;
+
+		if ((SSD_FIXED_IS_PRESENT(sense, sense_len, add_sense_code))
+		 && (SSD_FIXED_IS_FILLED(sense, add_sense_code)))
+			*asc = sense->add_sense_code;
+		else
+			*asc = (show_errors) ? -1 : 0;
+
+		if ((SSD_FIXED_IS_PRESENT(sense, sense_len,add_sense_code_qual))
+		 && (SSD_FIXED_IS_FILLED(sense, add_sense_code_qual)))
+			*ascq = sense->add_sense_code_qual;
+		else
+			*ascq = (show_errors) ? -1 : 0;
+		break;
+	}
+	}
+}
+
+int
+scsi_get_sense_key(struct scsi_sense_data *sense_data, u_int sense_len,
+		   int show_errors)
+{
+	int error_code, sense_key, asc, ascq;
+
+	scsi_extract_sense_len(sense_data, sense_len, &error_code,
+			       &sense_key, &asc, &ascq, show_errors);
+
+	return (sense_key);
+}
+
+int
+scsi_get_asc(struct scsi_sense_data *sense_data, u_int sense_len,
+	     int show_errors)
+{
+	int error_code, sense_key, asc, ascq;
+
+	scsi_extract_sense_len(sense_data, sense_len, &error_code,
+			       &sense_key, &asc, &ascq, show_errors);
+
+	return (asc);
+}
+
+int
+scsi_get_ascq(struct scsi_sense_data *sense_data, u_int sense_len,
+	      int show_errors)
+{
+	int error_code, sense_key, asc, ascq;
+
+	scsi_extract_sense_len(sense_data, sense_len, &error_code,
+			       &sense_key, &asc, &ascq, show_errors);
+
+	return (ascq);
+}
+
 /*
  * This function currently requires at least 36 bytes, or
  * SHORT_INQUIRY_LENGTH, worth of data to function properly.  If this
diff --git a/sys/cam/scsi/scsi_all.h b/sys/cam/scsi/scsi_all.h
index 93b11d5a6805..3bcc623f1085 100644
--- a/sys/cam/scsi/scsi_all.h
+++ b/sys/cam/scsi/scsi_all.h
@@ -25,6 +25,7 @@
 #define	_SCSI_SCSI_ALL_H 1
 
 #include <sys/cdefs.h>
+#include <machine/stdarg.h>
 
 #ifdef _KERNEL
 /*
@@ -171,7 +172,8 @@ struct scsi_inquiry
 {
 	u_int8_t opcode;
 	u_int8_t byte2;
-#define	SI_EVPD 0x01
+#define	SI_EVPD 	0x01
+#define	SI_CMDDT	0x02
 	u_int8_t page_code;
 	u_int8_t reserved;
 	u_int8_t length;
@@ -200,7 +202,9 @@ struct scsi_mode_sense_6
 #define	SMS_PAGE_CTRL_CHANGEABLE 	0x40
 #define	SMS_PAGE_CTRL_DEFAULT 		0x80
 #define	SMS_PAGE_CTRL_SAVED 		0xC0
-	u_int8_t unused;
+	u_int8_t subpage;
+#define	SMS_SUBPAGE_PAGE_0		0x00
+#define	SMS_SUBPAGE_ALL			0xff
 	u_int8_t length;
 	u_int8_t control;
 };
@@ -209,8 +213,10 @@ struct scsi_mode_sense_10
 {
 	u_int8_t opcode;
 	u_int8_t byte2;		/* same bits as small version */
+#define	SMS10_LLBAA			0x10
 	u_int8_t page; 		/* same bits as small version */
-	u_int8_t unused[4];
+	u_int8_t subpage;
+	u_int8_t unused[3];
 	u_int8_t length[2];
 	u_int8_t control;
 };
@@ -263,6 +269,120 @@ struct scsi_mode_block_descr
 	u_int8_t block_len[3];
 };
 
+struct scsi_per_res_in
+{
+	u_int8_t opcode;
+	u_int8_t action;
+#define	SPRI_RK	0x00
+#define	SPRI_RR	0x01
+#define	SPRI_RC	0x02
+#define	SPRI_RS	0x03
+	u_int8_t reserved[5];
+	u_int8_t length[2];
+	u_int8_t control;
+};
+
+struct scsi_per_res_in_header
+{
+	u_int8_t generation[4];
+	u_int8_t length[4];
+};
+
+struct scsi_per_res_key
+{
+	u_int8_t key[8];
+};
+
+struct scsi_per_res_in_keys
+{
+	struct scsi_per_res_in_header header;
+	struct scsi_per_res_key keys[0];
+};
+
+struct scsi_per_res_cap
+{
+	uint8_t length[2];
+	uint8_t flags1;
+#define	SPRI_CRH	0x10
+#define	SPRI_SIP_C	0x08
+#define	SPRI_ATP_C	0x04
+#define	SPRI_PTPL_C	0x01
+	uint8_t flags2;
+#define	SPRI_TMV	0x80
+#define	SPRI_PTPL_A	0x01
+	uint8_t type_mask[2];
+#define	SPRI_TM_WR_EX_AR	0x8000
+#define	SPRI_TM_EX_AC_RO	0x4000
+#define	SPRI_TM_WR_EX_RO	0x2000
+#define	SPRI_TM_EX_AC		0x0800
+#define	SPRI_TM_WR_EX		0x0200
+#define	SPRI_TM_EX_AC_AR	0x0001
+	uint8_t reserved[2];
+};
+
+struct scsi_per_res_in_rsrv_data
+{
+	uint8_t reservation[8];
+	uint8_t obsolete1[4];
+	uint8_t reserved;
+	uint8_t scopetype;
+#define	SPRT_WE    0x01
+#define	SPRT_EA    0x03
+#define	SPRT_WERO  0x05
+#define	SPRT_EARO  0x06
+#define	SPRT_WEAR  0x07
+#define	SPRT_EAAR  0x08
+	uint8_t obsolete2[2];
+};
+
+struct scsi_per_res_in_rsrv
+{
+	struct scsi_per_res_in_header header;
+	struct scsi_per_res_in_rsrv_data data;
+};
+
+struct scsi_per_res_out
+{
+	u_int8_t opcode;
+	u_int8_t action;
+#define	SPRO_REGISTER		0x00
+#define	SPRO_RESERVE		0x01
+#define	SPRO_RELEASE		0x02
+#define	SPRO_CLEAR		0x03
+#define	SPRO_PREEMPT		0x04
+#define	SPRO_PRE_ABO		0x05
+#define	SPRO_REG_IGNO		0x06
+#define	SPRO_REG_MOVE		0x07
+#define	SPRO_ACTION_MASK	0x1f
+	u_int8_t scope_type;
+#define	SPR_SCOPE_MASK		0xf0
+#define	SPR_LU_SCOPE		0x00
+#define	SPR_TYPE_MASK		0x0f
+#define	SPR_TYPE_WR_EX		0x01
+#define	SPR_TYPE_EX_AC		0x03
+#define	SPR_TYPE_WR_EX_RO	0x05
+#define	SPR_TYPE_EX_AC_RO	0x06
+#define	SPR_TYPE_WR_EX_AR	0x07
+#define	SPR_TYPE_EX_AC_AR	0x08
+	u_int8_t reserved[2];
+	u_int8_t length[4];
+	u_int8_t control;
+};
+
+struct scsi_per_res_out_parms
+{
+	struct scsi_per_res_key res_key;
+	u_int8_t serv_act_res_key[8];
+	u_int8_t obsolete1[4];
+	u_int8_t flags;
+#define	SPR_SPEC_I_PT		0x08
+#define	SPR_ALL_TG_PT		0x04
+#define	SPR_APTPL		0x01
+	u_int8_t reserved1;
+	u_int8_t obsolete2[2];
+};
+
+
 struct scsi_log_sense
 {
 	u_int8_t opcode;
@@ -337,7 +457,16 @@ struct scsi_control_page {
 	u_int8_t page_code;
 	u_int8_t page_length;
 	u_int8_t rlec;
-#define	SCB_RLEC			0x01	/*Report Log Exception Cond*/
+#define	SCP_RLEC			0x01	/*Report Log Exception Cond*/
+#define	SCP_GLTSD			0x02	/*Global Logging target
+						  save disable */
+#define	SCP_DSENSE			0x04	/*Descriptor Sense */
+#define	SCP_DPICZ			0x08	/*Disable Prot. Info Check
+						  if Prot. Field is Zero */
+#define	SCP_TMF_ONLY			0x10	/*TM Functions Only*/
+#define	SCP_TST_MASK			0xE0	/*Task Set Type Mask*/
+#define	SCP_TST_ONE			0x00	/*One Task Set*/
+#define	SCP_TST_SEPARATE		0x20	/*Separate Task Sets*/
 	u_int8_t queue_flags;
 #define	SCP_QUEUE_ALG_MASK		0xF0
 #define	SCP_QUEUE_ALG_RESTRICTED	0x00
@@ -368,6 +497,41 @@ struct scsi_cache_page {
 	u_int8_t max_prefetch_ceil[2];
 };
 
+/*
+ * XXX KDM
+ * Updated version of the cache page, as of SBC.  Update this to SBC-3 and
+ * rationalize the two.
+ */
+struct scsi_caching_page {
+	uint8_t page_code;
+#define	SMS_CACHING_PAGE		0x08
+	uint8_t page_length;
+	uint8_t flags1;
+#define	SCP_IC		0x80
+#define	SCP_ABPF	0x40
+#define	SCP_CAP		0x20
+#define	SCP_DISC	0x10
+#define	SCP_SIZE	0x08
+#define	SCP_WCE		0x04
+#define	SCP_MF		0x02
+#define	SCP_RCD		0x01
+	uint8_t ret_priority;
+	uint8_t disable_pf_transfer_len[2];
+	uint8_t min_prefetch[2];
+	uint8_t max_prefetch[2];
+	uint8_t max_pf_ceiling[2];
+	uint8_t flags2;
+#define	SCP_FSW		0x80
+#define	SCP_LBCSS	0x40
+#define	SCP_DRA		0x20
+#define	SCP_VS1		0x10
+#define	SCP_VS2		0x08
+	uint8_t cache_segments;
+	uint8_t cache_seg_size[2];
+	uint8_t reserved;
+	uint8_t non_cache_seg_size[3];
+};
+
 struct scsi_info_exceptions_page {
 	u_int8_t page_code;
 #define	SIEP_PAGE_SAVABLE		0x80	/* Page is savable */
@@ -406,20 +570,49 @@ struct scsi_reserve
 {
 	u_int8_t opcode;
 	u_int8_t byte2;
-	u_int8_t unused[2];
-	u_int8_t length;
+#define	SR_EXTENT	0x01
+#define	SR_ID_MASK	0x0e
+#define	SR_3RDPTY	0x10
+#define	SR_LUN_MASK	0xe0
+	u_int8_t resv_id;
+	u_int8_t length[2];
 	u_int8_t control;
 };
 
+struct scsi_reserve_10 {
+	uint8_t	opcode;
+	uint8_t	byte2;
+#define	SR10_3RDPTY	0x10
+#define	SR10_LONGID	0x02
+#define	SR10_EXTENT	0x01
+	uint8_t resv_id;
+	uint8_t thirdparty_id;
+	uint8_t reserved[3];
+	uint8_t length[2];
+	uint8_t control;
+};
+
+
 struct scsi_release
 {
 	u_int8_t opcode;
 	u_int8_t byte2;
-	u_int8_t unused[2];
+	u_int8_t resv_id;
+	u_int8_t unused[1];
 	u_int8_t length;
 	u_int8_t control;
 };
 
+struct scsi_release_10 {
+	uint8_t opcode;
+	uint8_t byte2;
+	uint8_t resv_id;
+	uint8_t thirdparty_id;
+	uint8_t reserved[3];
+	uint8_t length[2];
+	uint8_t control;
+};
+
 struct scsi_prevent
 {
 	u_int8_t opcode;
@@ -435,12 +628,60 @@ struct scsi_sync_cache
 {
 	u_int8_t opcode;
 	u_int8_t byte2;
+#define	SSC_IMMED	0x02
+#define	SSC_RELADR	0x01
 	u_int8_t begin_lba[4];
 	u_int8_t reserved;
 	u_int8_t lb_count[2];
 	u_int8_t control;	
 };
 
+struct scsi_sync_cache_16
+{
+	uint8_t opcode;
+	uint8_t byte2;
+	uint8_t begin_lba[8];
+	uint8_t lb_count[4];
+	uint8_t reserved;
+	uint8_t control;
+};
+
+struct scsi_format {
+	uint8_t opcode;
+	uint8_t byte2;
+#define	SF_LONGLIST		0x20
+#define	SF_FMTDATA		0x10
+#define	SF_CMPLIST		0x08
+#define	SF_FORMAT_MASK		0x07
+#define	SF_FORMAT_BLOCK		0x00
+#define	SF_FORMAT_LONG_BLOCK	0x03
+#define	SF_FORMAT_BFI		0x04
+#define	SF_FORMAT_PHYS		0x05
+	uint8_t vendor;
+	uint8_t interleave[2];
+	uint8_t control;
+};
+
+struct scsi_format_header_short {
+	uint8_t reserved;
+#define	SF_DATA_FOV	0x80
+#define	SF_DATA_DPRY	0x40
+#define	SF_DATA_DCRT	0x20
+#define	SF_DATA_STPF	0x10
+#define	SF_DATA_IP	0x08
+#define	SF_DATA_DSP	0x04
+#define	SF_DATA_IMMED	0x02
+#define	SF_DATA_VS	0x01
+	uint8_t byte2;
+	uint8_t defect_list_len[2];
+};
+
+struct scsi_format_header_long {
+	uint8_t reserved;
+	uint8_t byte2;
+	uint8_t reserved2[2];
+	uint8_t defect_list_len[4];
+};
 
 struct scsi_changedef
 {
@@ -459,6 +700,7 @@ struct scsi_read_buffer
 	u_int8_t byte2;
 #define	RWB_MODE		0x07
 #define	RWB_MODE_HDR_DATA	0x00
+#define	RWB_MODE_VENDOR		0x01
 #define	RWB_MODE_DATA		0x02
 #define	RWB_MODE_DOWNLOAD	0x04
 #define	RWB_MODE_DOWNLOAD_SAVE	0x05
@@ -529,6 +771,40 @@ struct scsi_rw_16
 	u_int8_t control;
 };
 
+struct scsi_write_verify_10
+{
+	uint8_t	opcode;
+	uint8_t	byte2;
+#define	SWV_BYTCHK		0x02
+#define	SWV_DPO			0x10
+#define	SWV_WRPROECT_MASK	0xe0
+	uint8_t	addr[4];
+	uint8_t	group;
+	uint8_t length[2];
+	uint8_t	control;
+};
+
+struct scsi_write_verify_12
+{
+	uint8_t	opcode;
+	uint8_t	byte2;
+	uint8_t	addr[4];
+	uint8_t	length[4];
+	uint8_t	group;
+	uint8_t	control;
+};
+
+struct scsi_write_verify_16
+{
+	uint8_t	opcode;
+	uint8_t	byte2;
+	uint8_t	addr[8];
+	uint8_t	length[4];
+	uint8_t	group;
+	uint8_t	control;
+};
+
+
 struct scsi_start_stop_unit
 {
 	u_int8_t opcode;
@@ -538,6 +814,14 @@ struct scsi_start_stop_unit
 	u_int8_t how;
 #define	SSS_START		0x01
 #define	SSS_LOEJ		0x02
+#define	SSS_PC_MASK		0xf0
+#define	SSS_PC_START_VALID	0x00
+#define	SSS_PC_ACTIVE		0x10
+#define	SSS_PC_IDLE		0x20
+#define	SSS_PC_STANDBY		0x30
+#define	SSS_PC_LU_CONTROL	0x70
+#define	SSS_PC_FORCE_IDLE_0	0xa0
+#define	SSS_PC_FORCE_STANDBY_0	0xb0
 	u_int8_t control;
 };
 
@@ -562,6 +846,18 @@ struct ata_pass_12 {
 	u_int8_t control;
 };
 
+struct scsi_maintenance_in
+{
+        uint8_t  opcode;
+        uint8_t  byte2;
+#define SERVICE_ACTION_MASK  0x1f
+#define SA_RPRT_TRGT_GRP     0x0a
+        uint8_t  reserved[4];
+	uint8_t  length[4];
+	uint8_t  reserved1;
+	uint8_t  control;
+};
+
 struct ata_pass_16 {
 	u_int8_t opcode;
 	u_int8_t protocol;
@@ -607,6 +903,7 @@ struct ata_pass_16 {
 #define	READ_10			0x28
 #define	WRITE_10		0x2A
 #define	POSITION_TO_ELEMENT	0x2B
+#define	WRITE_VERIFY_10		0x2E
 #define	SYNCHRONIZE_CACHE	0x35
 #define	READ_DEFECT_DATA_10	0x37
 #define	WRITE_BUFFER            0x3B
@@ -615,10 +912,16 @@ struct ata_pass_16 {
 #define	LOG_SELECT		0x4C
 #define	LOG_SENSE		0x4D
 #define	MODE_SELECT_10		0x55
+#define	RESERVE_10		0x56
+#define	RELEASE_10		0x57
 #define	MODE_SENSE_10		0x5A
+#define	PERSISTENT_RES_IN	0x5E
+#define	PERSISTENT_RES_OUT	0x5F
 #define	ATA_PASS_16		0x85
 #define	READ_16			0x88
 #define	WRITE_16		0x8A
+#define	WRITE_VERIFY_16		0x8E
+#define	SYNCHRONIZE_CACHE_16	0x91
 #define	SERVICE_ACTION_IN	0x9E
 #define	REPORT_LUNS		0xA0
 #define	ATA_PASS_12		0xA1
@@ -627,6 +930,7 @@ struct ata_pass_16 {
 #define	MOVE_MEDIUM     	0xA5
 #define	READ_12			0xA8
 #define	WRITE_12		0xAA
+#define	WRITE_VERIFY_12		0xAE
 #define	READ_ELEMENT_STATUS	0xB8
 
 /* Maintenance In Service Action Codes */
@@ -737,10 +1041,12 @@ struct scsi_inquiry_data
 	u_int8_t response_format;
 #define	SID_AENC	0x80
 #define	SID_TrmIOP	0x40
+#define	SID_NormACA	0x20
+#define	SID_HiSup	0x10
 	u_int8_t additional_length;
 #define	SID_ADDITIONAL_LENGTH(iqd)					\
 	((iqd)->additional_length +					\
-	offsetof(struct scsi_inquiry_data, additional_length) + 1)
+	__offsetof(struct scsi_inquiry_data, additional_length) + 1)
 	u_int8_t spc3_flags;
 #define	SPC3_SID_PROTECT	0x01
 #define	SPC3_SID_3PC		0x08
@@ -750,6 +1056,7 @@ struct scsi_inquiry_data
 #define	SPC3_SID_ACC		0x40
 #define	SPC3_SID_SCCS		0x80
 	u_int8_t spc2_flags;
+#define	SPC2_SID_ADDR16		0x01
 #define	SPC2_SID_MChngr 	0x08
 #define	SPC2_SID_MultiP 	0x10
 #define	SPC2_SID_EncServ	0x40
@@ -809,6 +1116,10 @@ struct scsi_inquiry_data
 	u_int8_t vendor_specific1[SID_VENDOR_SPECIFIC_1_SIZE];
 };
 
+/*
+ * This structure is more suited to initiator operation, because the
+ * maximum number of supported pages is already allocated.
+ */
 struct scsi_vpd_supported_page_list
 {
 	u_int8_t device;
@@ -852,11 +1163,11 @@ struct scsi_vpd_device_id
 	u_int8_t device;
 	u_int8_t page_code;
 #define	SVPD_DEVICE_ID			0x83
-#define	SVPD_DEVICE_ID_MAX_SIZE		0xffff
-#define	SVPD_DEVICE_ID_HDR_LEN		4
-#define	SVPD_DEVICE_ID_DESC_HDR_LEN	4
+#define	SVPD_DEVICE_ID_MAX_SIZE		252
+#define	SVPD_DEVICE_ID_HDR_LEN \
+    __offsetof(struct scsi_vpd_device_id, desc_list)
 	u_int8_t length[2];
-	u_int8_t desc_list[0];
+	u_int8_t desc_list[];
 };
 
 struct scsi_vpd_id_descriptor
@@ -872,11 +1183,13 @@ struct scsi_vpd_id_descriptor
 #define	SVPD_ID_PROTO_SHIFT	4
 #define	SVPD_ID_CODESET_BINARY	0x01
 #define	SVPD_ID_CODESET_ASCII	0x02
+#define	SVPD_ID_CODESET_MASK	0x0f
 	u_int8_t	id_type;
 #define	SVPD_ID_PIV		0x80
 #define	SVPD_ID_ASSOC_LUN	0x00
 #define	SVPD_ID_ASSOC_PORT	0x10
 #define	SVPD_ID_ASSOC_TARGET	0x20
+#define	SVPD_ID_ASSOC_MASK	0x30
 #define	SVPD_ID_TYPE_VENDOR	0x00
 #define	SVPD_ID_TYPE_T10	0x01
 #define	SVPD_ID_TYPE_EUI64	0x02
@@ -889,7 +1202,9 @@ struct scsi_vpd_id_descriptor
 #define	SVPD_ID_TYPE_MASK	0x0f
 	u_int8_t	reserved;
 	u_int8_t	length;
-	u_int8_t	identifier[0];
+#define	SVPD_DEVICE_ID_DESC_HDR_LEN \
+    __offsetof(struct scsi_vpd_id_descriptor, identifier) 
+	u_int8_t	identifier[];
 };
 
 struct scsi_vpd_id_t10
@@ -990,12 +1305,23 @@ struct scsi_vpd_id_scsi_name
 	uint8_t name_string[256];
 };
 
+struct scsi_service_action_in
+{
+	uint8_t opcode;
+	uint8_t service_action;
+	uint8_t action_dependent[13];
+	uint8_t control;
+};
+
 struct scsi_read_capacity
 {
 	u_int8_t opcode;
 	u_int8_t byte2;
+#define	SRC_RELADR	0x01
 	u_int8_t addr[4];
-	u_int8_t unused[3];
+	u_int8_t unused[2];
+	u_int8_t pmi;
+#define	SRC_PMI		0x01
 	u_int8_t control;
 };
 
@@ -1038,18 +1364,11 @@ struct scsi_report_luns
 	uint8_t control;
 };
 
-struct scsi_report_luns_data {
-	u_int8_t length[4];	/* length of LUN inventory, in bytes */
-	u_int8_t reserved[4];	/* unused */
-	/*
-	 * LUN inventory- we only support the type zero form for now.
-	 */
-	struct {
-		u_int8_t lundata[8];
-	} luns[0];
-};
+struct scsi_report_luns_lundata {
+	uint8_t lundata[8];
 #define	RPL_LUNDATA_PERIPH_BUS_MASK	0x3f
 #define	RPL_LUNDATA_FLAT_LUN_MASK	0x3f
+#define	RPL_LUNDATA_FLAT_LUN_BITS	0x06
 #define	RPL_LUNDATA_LUN_TARG_MASK	0x3f
 #define	RPL_LUNDATA_LUN_BUS_MASK	0xe0
 #define	RPL_LUNDATA_LUN_LUN_MASK	0x1f
@@ -1062,6 +1381,16 @@ struct scsi_report_luns_data {
 #define	RPL_LUNDATA_ATYP_FLAT	0x40
 #define	RPL_LUNDATA_ATYP_LUN	0x80
 #define	RPL_LUNDATA_ATYP_EXTLUN	0xc0
+};
+
+struct scsi_report_luns_data {
+	u_int8_t length[4];	/* length of LUN inventory, in bytes */
+	u_int8_t reserved[4];	/* unused */
+	/*
+	 * LUN inventory- we only support the type zero form for now.
+	 */
+	struct scsi_report_luns_lundata luns[0];
+};
 
 struct scsi_target_group
 {
@@ -1103,6 +1432,9 @@ struct scsi_target_port_group_descriptor {
 	uint8_t target_port_group[2];
 	uint8_t reserved;
 	uint8_t status;
+#define TPG_UNAVLBL      0
+#define TPG_SET_BY_STPG  0x01
+#define TPG_IMPLICIT     0x02
 	uint8_t vendor_specific;
 	uint8_t	target_port_count;
 	struct scsi_target_port_descriptor descriptors[];
@@ -1122,7 +1454,48 @@ struct scsi_target_group_data_extended {
 };
 
 
+typedef enum {
+	SSD_TYPE_NONE,
+	SSD_TYPE_FIXED,
+	SSD_TYPE_DESC
+} scsi_sense_data_type;
+
+typedef enum {
+	SSD_ELEM_NONE,
+	SSD_ELEM_SKIP,
+	SSD_ELEM_DESC,
+	SSD_ELEM_SKS,
+	SSD_ELEM_COMMAND,
+	SSD_ELEM_INFO,
+	SSD_ELEM_FRU,
+	SSD_ELEM_STREAM,
+	SSD_ELEM_MAX
+} scsi_sense_elem_type;
+
+
 struct scsi_sense_data
+{
+	uint8_t error_code;
+	/*
+	 * SPC-4 says that the maximum length of sense data is 252 bytes.
+	 * So this structure is exactly 252 bytes log.
+	 */
+#define	SSD_FULL_SIZE 252
+	uint8_t sense_buf[SSD_FULL_SIZE - 1];
+	/*
+	 * XXX KDM is this still a reasonable minimum size?
+	 */
+#define	SSD_MIN_SIZE 18
+	/*
+	 * Maximum value for the extra_len field in the sense data.
+	 */
+#define	SSD_EXTRA_MAX 244
+};
+
+/*
+ * Fixed format sense data.
+ */
+struct scsi_sense_data_fixed
 {
 	u_int8_t error_code;
 #define	SSD_ERRCODE			0x7F
@@ -1147,7 +1520,7 @@ struct scsi_sense_data
 #define		SSD_KEY_EQUAL		0x0c
 #define		SSD_KEY_VOLUME_OVERFLOW	0x0d
 #define		SSD_KEY_MISCOMPARE	0x0e
-#define		SSD_KEY_RESERVED	0x0f			
+#define		SSD_KEY_COMPLETED	0x0f			
 #define	SSD_ILI		0x20
 #define	SSD_EOM		0x40
 #define	SSD_FILEMARK	0x80
@@ -1162,9 +1535,313 @@ struct scsi_sense_data
 #define	SSD_FIELDPTR_CMD	0x40
 #define	SSD_BITPTR_VALID	0x08
 #define	SSD_BITPTR_VALUE	0x07
-#define	SSD_MIN_SIZE 18
 	u_int8_t extra_bytes[14];
-#define	SSD_FULL_SIZE sizeof(struct scsi_sense_data)
+#define	SSD_FIXED_IS_PRESENT(sense, length, field) 			\
+	((length >= (offsetof(struct scsi_sense_data_fixed, field) +	\
+	sizeof(sense->field))) ? 1 :0)
+#define	SSD_FIXED_IS_FILLED(sense, field) 				\
+	((((offsetof(struct scsi_sense_data_fixed, field) +		\
+	sizeof(sense->field)) -						\
+	(offsetof(struct scsi_sense_data_fixed, extra_len) +		\
+	sizeof(sense->extra_len))) <= sense->extra_len) ? 1 : 0)
+};
+
+/*
+ * Descriptor format sense data definitions.
+ * Introduced in SPC-3.
+ */
+struct scsi_sense_data_desc 
+{
+	uint8_t	error_code;
+#define	SSD_DESC_CURRENT_ERROR	0x72
+#define	SSD_DESC_DEFERRED_ERROR	0x73
+	uint8_t sense_key;
+	uint8_t	add_sense_code;
+	uint8_t	add_sense_code_qual;
+	uint8_t	reserved[3];
+	/*
+	 * Note that SPC-4, section 4.5.2.1 says that the extra_len field
+	 * must be less than or equal to 244.
+	 */
+	uint8_t	extra_len;
+	uint8_t	sense_desc[0];
+#define	SSD_DESC_IS_PRESENT(sense, length, field) 			\
+	((length >= (offsetof(struct scsi_sense_data_desc, field) +	\
+	sizeof(sense->field))) ? 1 :0)
+};
+
+struct scsi_sense_desc_header
+{
+	uint8_t desc_type;
+	uint8_t length;
+};
+/*
+ * The information provide in the Information descriptor is device type or
+ * command specific information, and defined in a command standard.
+ *
+ * Note that any changes to the field names or positions in this structure,
+ * even reserved fields, should be accompanied by an examination of the
+ * code in ctl_set_sense() that uses them.
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_info
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_INFO	0x00
+	uint8_t	length;
+	uint8_t	byte2;
+#define	SSD_INFO_VALID	0x80
+	uint8_t	reserved;
+	uint8_t	info[8];
+};
+
+/*
+ * Command-specific information depends on the command for which the
+ * reported condition occured.
+ *
+ * Note that any changes to the field names or positions in this structure,
+ * even reserved fields, should be accompanied by an examination of the
+ * code in ctl_set_sense() that uses them.
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_command
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_COMMAND	0x01
+	uint8_t	length;
+	uint8_t	reserved[2];
+	uint8_t	command_info[8];
+};
+
+/*
+ * Sense key specific descriptor.  The sense key specific data format
+ * depends on the sense key in question.
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_sks
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_SKS		0x02
+	uint8_t	length;
+	uint8_t reserved1[2];
+	uint8_t	sense_key_spec[3];
+#define	SSD_SKS_VALID		0x80
+	uint8_t reserved2;
+};
+
+/*
+ * This is used for the Illegal Request sense key (0x05) only.
+ */
+struct scsi_sense_sks_field
+{
+	uint8_t	byte0;
+#define	SSD_SKS_FIELD_VALID	0x80
+#define	SSD_SKS_FIELD_CMD	0x40
+#define	SSD_SKS_BPV		0x08
+#define	SSD_SKS_BIT_VALUE	0x07
+	uint8_t	field[2];
+};
+
+
+/* 
+ * This is used for the Hardware Error (0x04), Medium Error (0x03) and
+ * Recovered Error (0x01) sense keys.
+ */
+struct scsi_sense_sks_retry
+{
+	uint8_t byte0;
+#define	SSD_SKS_RETRY_VALID	0x80
+	uint8_t actual_retry_count[2];
+};
+
+/*
+ * Used with the NO Sense (0x00) or Not Ready (0x02) sense keys.
+ */
+struct scsi_sense_sks_progress
+{
+	uint8_t byte0;
+#define	SSD_SKS_PROGRESS_VALID	0x80
+	uint8_t progress[2];
+#define	SSD_SKS_PROGRESS_DENOM	0x10000
+};
+
+/*
+ * Used with the Copy Aborted (0x0a) sense key.
+ */
+struct scsi_sense_sks_segment
+{
+	uint8_t byte0;
+#define	SSD_SKS_SEGMENT_VALID	0x80
+#define	SSD_SKS_SEGMENT_SD	0x20
+#define	SSD_SKS_SEGMENT_BPV	0x08
+#define	SSD_SKS_SEGMENT_BITPTR	0x07
+	uint8_t field[2];
+};
+
+/*
+ * Used with the Unit Attention (0x06) sense key.
+ *
+ * This is currently used to indicate that the unit attention condition
+ * queue has overflowed (when the overflow bit is set).
+ */
+struct scsi_sense_sks_overflow
+{
+	uint8_t byte0;
+#define	SSD_SKS_OVERFLOW_VALID	0x80
+#define	SSD_SKS_OVERFLOW_SET	0x01
+	uint8_t	reserved[2];
+};
+
+/*
+ * This specifies which component is associated with the sense data.  There
+ * is no standard meaning for the fru value.
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_fru
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_FRU		0x03
+	uint8_t	length;
+	uint8_t reserved;
+	uint8_t fru;
+};
+
+/*
+ * Used for Stream commands, defined in SSC-4.
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+ 
+struct scsi_sense_stream
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_STREAM		0x04
+	uint8_t	length;
+	uint8_t	reserved;
+	uint8_t	byte3;
+#define	SSD_DESC_STREAM_FM	0x80
+#define	SSD_DESC_STREAM_EOM	0x40
+#define	SSD_DESC_STREAM_ILI	0x20
+};
+
+/*
+ * Used for Block commands, defined in SBC-3.
+ *
+ * This is currently (as of SBC-3) only used for the Incorrect Length
+ * Indication (ILI) bit, which says that the data length requested in the
+ * READ LONG or WRITE LONG command did not match the length of the logical
+ * block.
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_block
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_BLOCK		0x05
+	uint8_t	length;
+	uint8_t	reserved;
+	uint8_t	byte3;
+#define	SSD_DESC_BLOCK_ILI	0x20
+};
+
+/*
+ * Used for Object-Based Storage Devices (OSD-3).
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_osd_objid
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_OSD_OBJID	0x06
+	uint8_t	length;
+	uint8_t	reserved[6];
+	/*
+	 * XXX KDM provide the bit definitions here?  There are a lot of
+	 * them, and we don't have an OSD driver yet.
+	 */
+	uint8_t	not_init_cmds[4];
+	uint8_t	completed_cmds[4];
+	uint8_t	partition_id[8];
+	uint8_t	object_id[8];
+};
+
+/*
+ * Used for Object-Based Storage Devices (OSD-3).
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_osd_integrity
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_OSD_INTEGRITY	0x07
+	uint8_t	length;
+	uint8_t	integ_check_val[32];
+};
+
+/*
+ * Used for Object-Based Storage Devices (OSD-3).
+ *
+ * Maximum descriptors allowed: 1 (as of SPC-4)
+ */
+struct scsi_sense_osd_attr_id
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_OSD_ATTR_ID	0x08
+	uint8_t	length;
+	uint8_t	reserved[2];
+	uint8_t	attr_desc[0];
+};
+
+/*
+ * Used with Sense keys No Sense (0x00) and Not Ready (0x02).
+ *
+ * Maximum descriptors allowed: 32 (as of SPC-4)
+ */
+struct scsi_sense_progress
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_PROGRESS	0x0a
+	uint8_t	length;
+	uint8_t	sense_key;
+	uint8_t	add_sense_code;
+	uint8_t	add_sense_code_qual;
+	uint8_t reserved;
+	uint8_t	progress[2];
+};
+
+/*
+ * This is typically forwarded as the result of an EXTENDED COPY command.
+ *
+ * Maximum descriptors allowed: 2 (as of SPC-4)
+ */
+struct scsi_sense_forwarded
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_FORWARDED	0x0c
+	uint8_t	length;
+	uint8_t	byte2;
+#define	SSD_FORWARDED_FSDT	0x80
+#define	SSD_FORWARDED_SDS_MASK	0x0f
+#define	SSD_FORWARDED_SDS_UNK	0x00
+#define	SSD_FORWARDED_SDS_EXSRC	0x01
+#define	SSD_FORWARDED_SDS_EXDST	0x02
+};
+
+/*
+ * Vendor-specific sense descriptor.  The desc_type field will be in the
+ * range bewteen MIN and MAX inclusive.
+ */
+struct scsi_sense_vendor
+{
+	uint8_t	desc_type;
+#define	SSD_DESC_VENDOR_MIN	0x80
+#define	SSD_DESC_VENDOR_MAX	0xff
+	uint8_t length;
+	uint8_t	data[0];
 };
 
 struct scsi_mode_header_6
@@ -1187,9 +1864,20 @@ struct scsi_mode_header_10
 struct scsi_mode_page_header
 {
 	u_int8_t page_code;
+#define	SMPH_PS		0x80
+#define	SMPH_SPF	0x40
+#define	SMPH_PC_MASK	0x3f
 	u_int8_t page_length;
 };
 
+struct scsi_mode_page_header_sp
+{
+	uint8_t page_code;
+	uint8_t subpage;
+	uint8_t page_length[2];
+};
+
+
 struct scsi_mode_blk_desc
 {
 	u_int8_t density;
@@ -1292,6 +1980,84 @@ scsi_sense_action scsi_error_action(struct ccb_scsiio* csio,
 				    struct scsi_inquiry_data *inq_data,
 				    u_int32_t sense_flags);
 const char *	scsi_status_string(struct ccb_scsiio *csio);
+
+void scsi_desc_iterate(struct scsi_sense_data_desc *sense, u_int sense_len,
+		       int (*iter_func)(struct scsi_sense_data_desc *sense,
+					u_int, struct scsi_sense_desc_header *,
+					void *), void *arg);
+uint8_t *scsi_find_desc(struct scsi_sense_data_desc *sense, u_int sense_len,
+			uint8_t desc_type);
+void scsi_set_sense_data(struct scsi_sense_data *sense_data, 
+			 scsi_sense_data_type sense_format, int current_error,
+			 int sense_key, int asc, int ascq, ...) ;
+void scsi_set_sense_data_va(struct scsi_sense_data *sense_data,
+			    scsi_sense_data_type sense_format,
+			    int current_error, int sense_key, int asc,
+			    int ascq, va_list ap);
+int scsi_get_sense_info(struct scsi_sense_data *sense_data, u_int sense_len,
+			uint8_t info_type, uint64_t *info,
+			int64_t *signed_info);
+int scsi_get_sks(struct scsi_sense_data *sense_data, u_int sense_len,
+		 uint8_t *sks);
+int scsi_get_block_info(struct scsi_sense_data *sense_data, u_int sense_len,
+			struct scsi_inquiry_data *inq_data,
+			uint8_t *block_bits);
+int scsi_get_stream_info(struct scsi_sense_data *sense_data, u_int sense_len,
+			 struct scsi_inquiry_data *inq_data,
+			 uint8_t *stream_bits);
+void scsi_info_sbuf(struct sbuf *sb, uint8_t *cdb, int cdb_len,
+		    struct scsi_inquiry_data *inq_data, uint64_t info);
+void scsi_command_sbuf(struct sbuf *sb, uint8_t *cdb, int cdb_len,
+		       struct scsi_inquiry_data *inq_data, uint64_t csi);
+void scsi_progress_sbuf(struct sbuf *sb, uint16_t progress);
+int scsi_sks_sbuf(struct sbuf *sb, int sense_key, uint8_t *sks);
+void scsi_fru_sbuf(struct sbuf *sb, uint64_t fru);
+void scsi_stream_sbuf(struct sbuf *sb, uint8_t stream_bits, uint64_t info);
+void scsi_block_sbuf(struct sbuf *sb, uint8_t block_bits, uint64_t info);
+void scsi_sense_info_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			  u_int sense_len, uint8_t *cdb, int cdb_len,
+			  struct scsi_inquiry_data *inq_data,
+			  struct scsi_sense_desc_header *header);
+
+void scsi_sense_command_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			     u_int sense_len, uint8_t *cdb, int cdb_len,
+			     struct scsi_inquiry_data *inq_data,
+			     struct scsi_sense_desc_header *header);
+void scsi_sense_sks_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			 u_int sense_len, uint8_t *cdb, int cdb_len,
+			 struct scsi_inquiry_data *inq_data,
+			 struct scsi_sense_desc_header *header);
+void scsi_sense_fru_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			 u_int sense_len, uint8_t *cdb, int cdb_len,
+			 struct scsi_inquiry_data *inq_data,
+			 struct scsi_sense_desc_header *header);
+void scsi_sense_stream_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			    u_int sense_len, uint8_t *cdb, int cdb_len,
+			    struct scsi_inquiry_data *inq_data,
+			    struct scsi_sense_desc_header *header);
+void scsi_sense_block_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			   u_int sense_len, uint8_t *cdb, int cdb_len,
+			   struct scsi_inquiry_data *inq_data,
+			   struct scsi_sense_desc_header *header);
+void scsi_sense_progress_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			      u_int sense_len, uint8_t *cdb, int cdb_len,
+			      struct scsi_inquiry_data *inq_data,
+			      struct scsi_sense_desc_header *header);
+void scsi_sense_generic_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			     u_int sense_len, uint8_t *cdb, int cdb_len,
+			     struct scsi_inquiry_data *inq_data,
+			     struct scsi_sense_desc_header *header);
+void scsi_sense_desc_sbuf(struct sbuf *sb, struct scsi_sense_data *sense,
+			  u_int sense_len, uint8_t *cdb, int cdb_len,
+			  struct scsi_inquiry_data *inq_data,
+			  struct scsi_sense_desc_header *header);
+scsi_sense_data_type scsi_sense_type(struct scsi_sense_data *sense_data);
+
+void scsi_sense_only_sbuf(struct scsi_sense_data *sense, u_int sense_len,
+			  struct sbuf *sb, char *path_str,
+			  struct scsi_inquiry_data *inq_data, uint8_t *cdb,
+			  int cdb_len);
+
 #ifdef _KERNEL
 int		scsi_command_string(struct ccb_scsiio *csio, struct sbuf *sb);
 int		scsi_sense_sbuf(struct ccb_scsiio *csio, struct sbuf *sb,
@@ -1497,31 +2263,29 @@ int		scsi_static_inquiry_match(caddr_t inqbuffer,
 int		scsi_devid_match(uint8_t *rhs, size_t rhs_len,
 				 uint8_t *lhs, size_t lhs_len);
 
-static __inline void scsi_extract_sense(struct scsi_sense_data *sense,
-					int *error_code, int *sense_key,
-					int *asc, int *ascq);
+void scsi_extract_sense(struct scsi_sense_data *sense, int *error_code,
+			int *sense_key, int *asc, int *ascq);
+void scsi_extract_sense_len(struct scsi_sense_data *sense,
+			    u_int sense_len, int *error_code, int *sense_key,
+			    int *asc, int *ascq, int show_errors);
+int scsi_get_sense_key(struct scsi_sense_data *sense, u_int sense_len,
+		       int show_errors);
+int scsi_get_asc(struct scsi_sense_data *sense, u_int sense_len,
+		 int show_errors);
+int scsi_get_ascq(struct scsi_sense_data *sense, u_int sense_len,
+		  int show_errors);
 static __inline void scsi_ulto2b(u_int32_t val, u_int8_t *bytes);
 static __inline void scsi_ulto3b(u_int32_t val, u_int8_t *bytes);
 static __inline void scsi_ulto4b(u_int32_t val, u_int8_t *bytes);
 static __inline void scsi_u64to8b(u_int64_t val, u_int8_t *bytes);
-static __inline u_int32_t scsi_2btoul(u_int8_t *bytes);
-static __inline u_int32_t scsi_3btoul(u_int8_t *bytes);
-static __inline int32_t scsi_3btol(u_int8_t *bytes);
-static __inline u_int32_t scsi_4btoul(u_int8_t *bytes);
-static __inline u_int64_t scsi_8btou64(u_int8_t *bytes);
+static __inline uint32_t scsi_2btoul(const uint8_t *bytes);
+static __inline uint32_t scsi_3btoul(const uint8_t *bytes);
+static __inline int32_t scsi_3btol(const uint8_t *bytes);
+static __inline uint32_t scsi_4btoul(const uint8_t *bytes);
+static __inline uint64_t scsi_8btou64(const uint8_t *bytes);
 static __inline void *find_mode_page_6(struct scsi_mode_header_6 *mode_header);
 static __inline void *find_mode_page_10(struct scsi_mode_header_10 *mode_header);
 
-static __inline void scsi_extract_sense(struct scsi_sense_data *sense,
-					int *error_code, int *sense_key,
-					int *asc, int *ascq)
-{
-	*error_code = sense->error_code & SSD_ERRCODE;
-	*sense_key = sense->flags & SSD_KEY;
-	*asc = (sense->extra_len >= 5) ? sense->add_sense_code : 0;
-	*ascq = (sense->extra_len >= 6) ? sense->add_sense_code_qual : 0;
-}
-
 static __inline void
 scsi_ulto2b(u_int32_t val, u_int8_t *bytes)
 {
@@ -1563,20 +2327,20 @@ scsi_u64to8b(u_int64_t val, u_int8_t *bytes)
 	bytes[7] = val & 0xff;
 }
 
-static __inline u_int32_t
-scsi_2btoul(u_int8_t *bytes)
+static __inline uint32_t
+scsi_2btoul(const uint8_t *bytes)
 {
-	u_int32_t rv;
+	uint32_t rv;
 
 	rv = (bytes[0] << 8) |
 	     bytes[1];
 	return (rv);
 }
 
-static __inline u_int32_t
-scsi_3btoul(u_int8_t *bytes)
+static __inline uint32_t
+scsi_3btoul(const uint8_t *bytes)
 {
-	u_int32_t rv;
+	uint32_t rv;
 
 	rv = (bytes[0] << 16) |
 	     (bytes[1] << 8) |
@@ -1585,9 +2349,9 @@ scsi_3btoul(u_int8_t *bytes)
 }
 
 static __inline int32_t 
-scsi_3btol(u_int8_t *bytes)
+scsi_3btol(const uint8_t *bytes)
 {
-	u_int32_t rc = scsi_3btoul(bytes);
+	uint32_t rc = scsi_3btoul(bytes);
  
 	if (rc & 0x00800000)
 		rc |= 0xff000000;
@@ -1595,10 +2359,10 @@ scsi_3btol(u_int8_t *bytes)
 	return (int32_t) rc;
 }
 
-static __inline u_int32_t
-scsi_4btoul(u_int8_t *bytes)
+static __inline uint32_t
+scsi_4btoul(const uint8_t *bytes)
 {
-	u_int32_t rv;
+	uint32_t rv;
 
 	rv = (bytes[0] << 24) |
 	     (bytes[1] << 16) |
@@ -1608,7 +2372,7 @@ scsi_4btoul(u_int8_t *bytes)
 }
 
 static __inline uint64_t
-scsi_8btou64(uint8_t *bytes)
+scsi_8btou64(const uint8_t *bytes)
 {
         uint64_t rv;
  
diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c
index 91384b76d38d..53d352e14f34 100644
--- a/sys/cam/scsi/scsi_cd.c
+++ b/sys/cam/scsi/scsi_cd.c
@@ -1691,9 +1691,10 @@ cddone(struct cam_periph *periph, union ccb *done_ccb)
 
 				if (have_sense) {
 					sense = &csio->sense_data;
-					scsi_extract_sense(sense, &error_code,
-							   &sense_key, 
-							   &asc, &ascq);
+					scsi_extract_sense_len(sense,
+					    csio->sense_len - csio->sense_resid,
+					    &error_code, &sense_key, &asc,
+					    &ascq, /*show_errors*/ 1);
 				}
 				/*
 				 * Attach to anything that claims to be a
@@ -3126,8 +3127,9 @@ cderror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
 	 && ((ccb->ccb_h.flags & CAM_SENSE_PTR) == 0)) {
 		int sense_key, error_code, asc, ascq;
 
- 		scsi_extract_sense(&ccb->csio.sense_data,
-				   &error_code, &sense_key, &asc, &ascq);
+ 		scsi_extract_sense_len(&ccb->csio.sense_data,
+		    ccb->csio.sense_len - ccb->csio.sense_resid, &error_code,
+		    &sense_key, &asc, &ascq, /*show_errors*/ 1);
 		if (sense_key == SSD_KEY_ILLEGAL_REQUEST)
  			error = cd6byteworkaround(ccb);
 	}
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index a4363186a0b8..ab46cf60a808 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -753,10 +753,10 @@ daclose(struct disk *dp)
 				int asc, ascq;
 				int sense_key, error_code;
 
-				scsi_extract_sense(&ccb->csio.sense_data,
-						   &error_code,
-						   &sense_key, 
-						   &asc, &ascq);
+				scsi_extract_sense_len(&ccb->csio.sense_data,
+				    ccb->csio.sense_len - ccb->csio.sense_resid,
+				    &error_code, &sense_key, &asc, &ascq,
+				    /*show_errors*/ 1);
 				if (sense_key != SSD_KEY_ILLEGAL_REQUEST)
 					scsi_sense_print(&ccb->csio);
 			} else {
@@ -916,10 +916,10 @@ dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t leng
 				int asc, ascq;
 				int sense_key, error_code;
 
-				scsi_extract_sense(&csio.sense_data,
-						   &error_code,
-						   &sense_key, 
-						   &asc, &ascq);
+				scsi_extract_sense_len(&csio.sense_data,
+				    csio.sense_len - csio.sense_resid,
+				    &error_code, &sense_key, &asc, &ascq,
+				    /*show_errors*/ 1);
 				if (sense_key != SSD_KEY_ILLEGAL_REQUEST)
 					scsi_sense_print(&csio);
 			} else {
@@ -1802,9 +1802,10 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 
 				if (have_sense) {
 					sense = &csio->sense_data;
-					scsi_extract_sense(sense, &error_code,
-							   &sense_key, 
-							   &asc, &ascq);
+					scsi_extract_sense_len(sense,
+					    csio->sense_len - csio->sense_resid,
+					    &error_code, &sense_key, &asc,
+					    &ascq, /*show_errors*/ 1);
 				}
 				/*
 				 * Attach to anything that claims to be a
diff --git a/sys/cam/scsi/scsi_low.c b/sys/cam/scsi/scsi_low.c
index 1b2f4f601edb..60253cd22ad5 100644
--- a/sys/cam/scsi/scsi_low.c
+++ b/sys/cam/scsi/scsi_low.c
@@ -2577,12 +2577,16 @@ scsi_low_setup_done(slp, cb)
 #ifdef	SCSI_LOW_DEBUG
 				if (scsi_low_debug & SCSI_LOW_DEBUG_SENSE)
 				{
-					printf("SENSE: [%x][%x][%x][%x][%x]\n",
-					(u_int) cb->ccb_sense.error_code,
-					(u_int) cb->ccb_sense.segment,
-					(u_int) cb->ccb_sense.flags,
-					(u_int) cb->ccb_sense.add_sense_code,
-					(u_int) cb->ccb_sense.add_sense_code_qual);
+					int error_code, sense_key, asc, ascq;
+
+					scsi_extract_sense(&cb->ccb_sense,
+							   &error_code,
+							   &sense_key,
+							   &asc,
+							   &ascq);
+					printf("SENSE: [%x][%x][%x][%x]\n",
+					       error_code, sense_key, asc,
+					       ascq);
 				}
 #endif	/* SCSI_LOW_DEBUG */
 			}
diff --git a/sys/cam/scsi/scsi_sa.c b/sys/cam/scsi/scsi_sa.c
index 0d3ef1a9ee08..9a504b5dd0c9 100644
--- a/sys/cam/scsi/scsi_sa.c
+++ b/sys/cam/scsi/scsi_sa.c
@@ -235,10 +235,10 @@ struct sa_softc {
 	 */
 	struct {
 		struct scsi_sense_data _last_io_sense;
-		u_int32_t _last_io_resid;
+		u_int64_t _last_io_resid;
 		u_int8_t _last_io_cdb[CAM_MAX_CDBLEN];
 		struct scsi_sense_data _last_ctl_sense;
-		u_int32_t _last_ctl_resid;
+		u_int64_t _last_ctl_resid;
 		u_int8_t _last_ctl_cdb[CAM_MAX_CDBLEN];
 #define	last_io_sense	errinfo._last_io_sense
 #define	last_io_resid	errinfo._last_io_resid
@@ -849,8 +849,10 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 			 */
 			if ((periph->flags & CAM_PERIPH_LOCKED) == 0) {
 				error = cam_periph_hold(periph, PRIBIO|PCATCH);
-				 if (error != 0)
+				if (error != 0) {
+					cam_periph_unlock(periph);
 					return (error);
+				}
 				didlockperiph = 1;
 			}
 			break;
@@ -884,12 +886,15 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 			 * access to data structures.
 			 */
 			error = cam_periph_hold(periph, PRIBIO|PCATCH);
-			if (error != 0)
+			if (error != 0) {
+				cam_periph_unlock(periph);
 				return (error);
+			}
 			didlockperiph = 1;
 			break;
 
 		default:
+			cam_periph_unlock(periph);
 			return (EINVAL);
 		}
 	}
@@ -2322,17 +2327,28 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 	struct	sa_softc *softc;
 	struct	ccb_scsiio *csio;
 	struct	scsi_sense_data *sense;
-	u_int32_t resid = 0;
-	int32_t	info = 0;
+	uint64_t resid = 0;
+	int64_t	info = 0;
 	cam_status status;
-	int error_code, sense_key, asc, ascq, error, aqvalid;
+	int error_code, sense_key, asc, ascq, error, aqvalid, stream_valid;
+	int sense_len;
+	uint8_t stream_bits;
 
 	periph = xpt_path_periph(ccb->ccb_h.path);
 	softc = (struct sa_softc *)periph->softc;
 	csio = &ccb->csio;
 	sense = &csio->sense_data;
-	scsi_extract_sense(sense, &error_code, &sense_key, &asc, &ascq);
-	aqvalid = sense->extra_len >= 6;
+	sense_len = csio->sense_len - csio->sense_resid;
+	scsi_extract_sense_len(sense, sense_len, &error_code, &sense_key,
+	    &asc, &ascq, /*show_errors*/ 1);
+	if (asc != -1 && ascq != -1)
+		aqvalid = 1;
+	else
+		aqvalid = 0;
+	if (scsi_get_stream_info(sense, sense_len, NULL, &stream_bits) == 0)
+		stream_valid = 1;
+	else
+		stream_valid = 0;
 	error = 0;
 
 	status = csio->ccb_h.status & CAM_STATUS_MASK;
@@ -2343,9 +2359,8 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 	 * unit.
 	 */
 	if (status == CAM_SCSI_STATUS_ERROR) {
-		if ((sense->error_code & SSD_ERRCODE_VALID) != 0) {
-			info = (int32_t) scsi_4btoul(sense->info);
-			resid = info;
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &resid,
+					&info) == 0) {
 			if ((softc->flags & SA_FLAG_FIXED) != 0)
 				resid *= softc->media_blksize;
 		} else {
@@ -2372,10 +2387,11 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 			softc->last_resid_was_io = 0;
 		}
 		CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("CDB[0]=0x%x Key 0x%x "
-		    "ASC/ASCQ 0x%x/0x%x CAM STATUS 0x%x flags 0x%x resid %d "
+		    "ASC/ASCQ 0x%x/0x%x CAM STATUS 0x%x flags 0x%x resid %jd "
 		    "dxfer_len %d\n", csio->cdb_io.cdb_bytes[0] & 0xff,
 		    sense_key, asc, ascq, status,
-		    sense->flags & ~SSD_KEY_RESERVED, resid, csio->dxfer_len));
+		    (stream_valid) ? stream_bits : 0, (intmax_t)resid,
+		    csio->dxfer_len));
 	} else {
 		CAM_DEBUG(periph->path, CAM_DEBUG_INFO,
 		    ("Cam Status 0x%x\n", status));
@@ -2431,7 +2447,7 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 		if (sense_key == SSD_KEY_VOLUME_OVERFLOW) {
 			csio->resid = resid;
 			error = ENOSPC;
-		} else if (sense->flags & SSD_EOM) {
+		} else if ((stream_valid != 0) && (stream_bits & SSD_EOM)) {
 			softc->flags |= SA_FLAG_EOM_PENDING;
 			/*
 			 * Grotesque as it seems, the few times
@@ -2450,7 +2466,7 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 			} else {
 				error = EIO;
 			}
-		} else if (sense->flags & SSD_FILEMARK) {
+		} else if ((stream_valid != 0) && (stream_bits & SSD_FILEMARK)){
 			if (softc->flags & SA_FLAG_FIXED) {
 				error = -1;
 				softc->flags |= SA_FLAG_EOF_PENDING;
@@ -2470,7 +2486,7 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 	/*
 	 * Incorrect Length usually applies to read, but can apply to writes.
 	 */
-	if (error == 0 && (sense->flags & SSD_ILI)) {
+	if (error == 0 && (stream_valid != 0) && (stream_bits & SSD_ILI)) {
 		if (info < 0) {
 			xpt_print(csio->ccb_h.path, toobig,
 			    csio->dxfer_len - info);
@@ -2485,7 +2501,8 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs)
 			 * Bump the block number if we hadn't seen a filemark.
 			 * Do this independent of errors (we've moved anyway).
 			 */
-			if ((sense->flags & SSD_FILEMARK) == 0) {
+			if ((stream_valid == 0) ||
+			    (stream_bits & SSD_FILEMARK) == 0) {
 				if (softc->blkno != (daddr_t) -1) {
 					softc->blkno++;
 					csio->ccb_h.ccb_pflags |=
diff --git a/sys/cam/scsi/scsi_targ_bh.c b/sys/cam/scsi/scsi_targ_bh.c
index b8c2379ff911..d5887a1c88f2 100644
--- a/sys/cam/scsi/scsi_targ_bh.c
+++ b/sys/cam/scsi/scsi_targ_bh.c
@@ -112,20 +112,20 @@ static struct scsi_inquiry_data no_lun_inq_data =
 	/* version */2, /* format version */2
 };
 
-static struct scsi_sense_data no_lun_sense_data =
+static struct scsi_sense_data_fixed no_lun_sense_data =
 {
 	SSD_CURRENT_ERROR|SSD_ERRCODE_VALID,
 	0,
 	SSD_KEY_NOT_READY, 
 	{ 0, 0, 0, 0 },
-	/*extra_len*/offsetof(struct scsi_sense_data, fru)
-                   - offsetof(struct scsi_sense_data, extra_len),
+	/*extra_len*/offsetof(struct scsi_sense_data_fixed, fru)
+                   - offsetof(struct scsi_sense_data_fixed, extra_len),
 	{ 0, 0, 0, 0 },
 	/* Logical Unit Not Supported */
 	/*ASC*/0x25, /*ASCQ*/0
 };
 
-static const int request_sense_size = offsetof(struct scsi_sense_data, fru);
+static const int request_sense_size = offsetof(struct scsi_sense_data_fixed, fru);
 
 static periph_init_t	targbhinit;
 static void		targbhasync(void *callback_arg, u_int32_t code,
@@ -587,7 +587,9 @@ targbhdone(struct cam_periph *periph, union ccb *done_ccb)
 				 * This needs to have other than a
 				 * no_lun_sense_data response.
 				 */
-				atio->sense_data = no_lun_sense_data;
+				bcopy(&no_lun_sense_data, &atio->sense_data,
+				      min(sizeof(no_lun_sense_data),
+					  sizeof(atio->sense_data)));
 				atio->sense_len = sizeof(no_lun_sense_data);
 				descr->data_resid = 0;
 				descr->data_increment = 0;
@@ -630,7 +632,9 @@ targbhdone(struct cam_periph *periph, union ccb *done_ccb)
 			/* Direction is always relative to the initator */
 			atio->ccb_h.flags &= ~CAM_DIR_MASK;
 			atio->ccb_h.flags |= CAM_DIR_NONE;
-			atio->sense_data = no_lun_sense_data;
+			bcopy(&no_lun_sense_data, &atio->sense_data,
+			      min(sizeof(no_lun_sense_data),
+				  sizeof(atio->sense_data)));
 			atio->sense_len = sizeof (no_lun_sense_data);
 			descr->data_resid = 0;
 			descr->data_increment = 0;
diff --git a/sys/cddl/boot/zfs/zfssubr.c b/sys/cddl/boot/zfs/zfssubr.c
index e76c2738939e..b2d2e13da3ba 100644
--- a/sys/cddl/boot/zfs/zfssubr.c
+++ b/sys/cddl/boot/zfs/zfssubr.c
@@ -181,14 +181,17 @@ zio_checksum_label_verifier(zio_cksum_t *zcp, uint64_t offset)
 }
 
 static int
-zio_checksum_error(const blkptr_t *bp, void *data, uint64_t offset)
+zio_checksum_verify(const blkptr_t *bp, void *data)
 {
-	unsigned int checksum = BP_IS_GANG(bp) ? ZIO_CHECKSUM_GANG_HEADER : BP_GET_CHECKSUM(bp);
-	uint64_t size = BP_GET_PSIZE(bp);
+	uint64_t size;
+	unsigned int checksum;
 	zio_checksum_info_t *ci;
 	zio_cksum_t actual_cksum, expected_cksum, verifier;
 	int byteswap;
 
+	checksum = BP_GET_CHECKSUM(bp);
+	size = BP_GET_PSIZE(bp);
+
 	if (checksum >= ZIO_CHECKSUM_FUNCTIONS)
 		return (EINVAL);
 	ci = &zio_checksum_table[checksum];
@@ -206,7 +209,8 @@ zio_checksum_error(const blkptr_t *bp, void *data, uint64_t offset)
 		if (checksum == ZIO_CHECKSUM_GANG_HEADER)
 			zio_checksum_gang_verifier(&verifier, bp);
 		else if (checksum == ZIO_CHECKSUM_LABEL)
-			zio_checksum_label_verifier(&verifier, offset);
+			zio_checksum_label_verifier(&verifier,
+			    DVA_GET_OFFSET(BP_IDENTITY(bp)));
 		else
 			verifier = bp->blk_cksum;
 
@@ -224,7 +228,6 @@ zio_checksum_error(const blkptr_t *bp, void *data, uint64_t offset)
 			byteswap_uint64_array(&expected_cksum,
 			    sizeof (zio_cksum_t));
 	} else {
-		ASSERT(!BP_IS_GANG(bp));
 		expected_cksum = bp->blk_cksum;
 		ci->ci_func[0](data, size, &actual_cksum);
 	}
@@ -1215,15 +1218,10 @@ static void
 vdev_raidz_map_free(raidz_map_t *rm)
 {
 	int c;
-	size_t size;
 
 	for (c = rm->rm_firstdatacol - 1; c >= 0; c--)
 		zfs_free(rm->rm_col[c].rc_data, rm->rm_col[c].rc_size);
 
-	size = 0;
-	for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++)
-		size += rm->rm_col[c].rc_size;
-
 	zfs_free(rm, offsetof(raidz_map_t, rm_col[rm->rm_scols]));
 }
 
@@ -1245,10 +1243,10 @@ vdev_child(vdev_t *pvd, uint64_t devidx)
  * any ereports we generate can note it.
  */
 static int
-raidz_checksum_verify(const blkptr_t *bp, void *data)
+raidz_checksum_verify(const blkptr_t *bp, void *data, uint64_t size)
 {
 
-	return (zio_checksum_error(bp, data, 0));
+	return (zio_checksum_verify(bp, data));
 }
 
 /*
@@ -1298,7 +1296,7 @@ raidz_parity_verify(raidz_map_t *rm)
  */
 static int
 vdev_raidz_combrec(raidz_map_t *rm, const blkptr_t *bp, void *data,
-    off_t offset, int total_errors, int data_errors)
+    off_t offset, uint64_t bytes, int total_errors, int data_errors)
 {
 	raidz_col_t *rc;
 	void *orig[VDEV_RAIDZ_MAXPARITY];
@@ -1377,7 +1375,7 @@ vdev_raidz_combrec(raidz_map_t *rm, const blkptr_t *bp, void *data,
 			 * success.
 			 */
 			code = vdev_raidz_reconstruct(rm, tgts, n);
-			if (raidz_checksum_verify(bp, data) == 0) {
+			if (raidz_checksum_verify(bp, data, bytes) == 0) {
 				for (i = 0; i < n; i++) {
 					c = tgts[i];
 					rc = &rm->rm_col[c];
@@ -1548,7 +1546,7 @@ vdev_raidz_read(vdev_t *vd, const blkptr_t *bp, void *data,
 	 */
 	if (total_errors <= rm->rm_firstdatacol - parity_untried) {
 		if (data_errors == 0) {
-			if (raidz_checksum_verify(bp, data) == 0) {
+			if (raidz_checksum_verify(bp, data, bytes) == 0) {
 				/*
 				 * If we read parity information (unnecessarily
 				 * as it happens since no reconstruction was
@@ -1593,7 +1591,7 @@ vdev_raidz_read(vdev_t *vd, const blkptr_t *bp, void *data,
 
 			code = vdev_raidz_reconstruct(rm, tgts, n);
 
-			if (raidz_checksum_verify(bp, data) == 0) {
+			if (raidz_checksum_verify(bp, data, bytes) == 0) {
 				/*
 				 * If we read more parity disks than were used
 				 * for reconstruction, confirm that the other
@@ -1633,7 +1631,9 @@ vdev_raidz_read(vdev_t *vd, const blkptr_t *bp, void *data,
 
 	n = 0;
 	for (c = 0; c < rm->rm_cols; c++) {
-		if (rm->rm_col[c].rc_tried)
+		rc = &rm->rm_col[c];
+
+		if (rc->rc_tried)
 			continue;
 
 		cvd = vdev_child(vd, rc->rc_devidx);
@@ -1665,8 +1665,8 @@ vdev_raidz_read(vdev_t *vd, const blkptr_t *bp, void *data,
 	if (total_errors > rm->rm_firstdatacol) {
 		error = EIO;
 	} else if (total_errors < rm->rm_firstdatacol &&
-	    (code = vdev_raidz_combrec(rm, bp, data, offset, total_errors,
-	     data_errors)) != 0) {
+	    (code = vdev_raidz_combrec(rm, bp, data, offset, bytes,
+	     total_errors, data_errors)) != 0) {
 		/*
 		 * If we didn't use all the available parity for the
 		 * combinatorial reconstruction, verify that the remaining
diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
index f7aeecc13b67..ae243fd62dd3 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
@@ -5669,7 +5669,7 @@ dtrace_action_raise(uint64_t sig)
 #else
 	struct proc *p = curproc;
 	PROC_LOCK(p);
-	psignal(p, sig);
+	kern_psignal(p, sig);
 	PROC_UNLOCK(p);
 #endif
 }
@@ -5689,7 +5689,7 @@ dtrace_action_stop(void)
 #else
 	struct proc *p = curproc;
 	PROC_LOCK(p);
-	psignal(p, SIGSTOP);
+	kern_psignal(p, SIGSTOP);
 	PROC_UNLOCK(p);
 #endif
 }
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
index dc9f4823bbfb..7b47f87926bd 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
@@ -627,6 +627,10 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 	} else if (db->db_state == DB_UNCACHED) {
 		spa_t *spa = dn->dn_objset->os_spa;
 
+#ifdef _KERNEL
+		curthread->td_ru.ru_inblock++;
+#endif
+
 		if (zio == NULL)
 			zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
 		dbuf_read_impl(db, zio, &flags);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
index 56e284a6d610..f7d471fa3fc3 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
@@ -397,9 +397,12 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
 			return (EIO);
 		}
 		/* initiate async i/o */
-		if (read) {
+		if (read)
 			(void) dbuf_read(db, zio, dbuf_flags);
-		}
+#ifdef _KERNEL
+		else
+			curthread->td_ru.ru_oublock++;
+#endif
 		dbp[i] = &db->db;
 	}
 	rw_exit(&dn->dn_struct_rwlock);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c
index 81b8436707ac..b4579e278c85 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c
@@ -21,6 +21,9 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
+/*
+ * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ */
 
 #include <sys/dmu.h>
 #include <sys/dmu_impl.h>
@@ -676,6 +679,8 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name)
 	ASSERT3P(dmu_ot[dn->dn_type].ot_byteswap, ==, zap_byteswap);
 
 	if (dn->dn_maxblkid == 0 && !add) {
+		blkptr_t *bp;
+
 		/*
 		 * If there is only one block  (i.e. this is a micro-zap)
 		 * and we are not adding anything, the accounting is simple.
@@ -690,14 +695,13 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name)
 		 * Use max block size here, since we don't know how much
 		 * the size will change between now and the dbuf dirty call.
 		 */
+		bp = &dn->dn_phys->dn_blkptr[0];
 		if (dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset,
-		    &dn->dn_phys->dn_blkptr[0],
-		    dn->dn_phys->dn_blkptr[0].blk_birth)) {
+		    bp, bp->blk_birth))
 			txh->txh_space_tooverwrite += SPA_MAXBLOCKSIZE;
-		} else {
+		else
 			txh->txh_space_towrite += SPA_MAXBLOCKSIZE;
-		}
-		if (dn->dn_phys->dn_blkptr[0].blk_birth)
+		if (!BP_IS_HOLE(bp))
 			txh->txh_space_tounref += SPA_MAXBLOCKSIZE;
 		return;
 	}
@@ -1273,7 +1277,6 @@ dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object)
 {
 	dnode_t *dn;
 	dmu_tx_hold_t *txh;
-	blkptr_t *bp;
 
 	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, object,
 	    THT_SPILL, 0, 0);
@@ -1286,15 +1289,16 @@ dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object)
 	/* If blkptr doesn't exist then add space to towrite */
 	if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) {
 		txh->txh_space_towrite += SPA_MAXBLOCKSIZE;
-		txh->txh_space_tounref = 0;
 	} else {
+		blkptr_t *bp;
+
 		bp = &dn->dn_phys->dn_spill;
 		if (dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset,
 		    bp, bp->blk_birth))
 			txh->txh_space_tooverwrite += SPA_MAXBLOCKSIZE;
 		else
 			txh->txh_space_towrite += SPA_MAXBLOCKSIZE;
-		if (bp->blk_birth)
+		if (!BP_IS_HOLE(bp))
 			txh->txh_space_tounref += SPA_MAXBLOCKSIZE;
 	}
 }
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
index 498352e7fdd8..a306bb645b8c 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
@@ -21,6 +21,8 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #include <sys/dmu_objset.h>
@@ -2461,7 +2463,7 @@ dsl_valid_rename(const char *oldname, void *arg)
 
 #pragma weak dmu_objset_rename = dsl_dataset_rename
 int
-dsl_dataset_rename(char *oldname, const char *newname, boolean_t recursive)
+dsl_dataset_rename(char *oldname, const char *newname, int flags)
 {
 	dsl_dir_t *dd;
 	dsl_dataset_t *ds;
@@ -2481,7 +2483,7 @@ dsl_dataset_rename(char *oldname, const char *newname, boolean_t recursive)
 			    &delta, DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
 
 		if (err == 0)
-			err = dsl_dir_rename(dd, newname);
+			err = dsl_dir_rename(dd, newname, flags);
 		dsl_dir_close(dd, FTAG);
 		return (err);
 	}
@@ -2502,7 +2504,7 @@ dsl_dataset_rename(char *oldname, const char *newname, boolean_t recursive)
 	if (strncmp(oldname, newname, tail - newname) != 0)
 		return (EXDEV);
 
-	if (recursive) {
+	if (flags & ZFS_RENAME_RECURSIVE) {
 		err = dsl_recursive_rename(oldname, newname);
 	} else {
 		err = dsl_dataset_hold(oldname, FTAG, &ds);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
index 013e811d3283..1213445da81b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
@@ -20,6 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -37,6 +39,9 @@
 #include <sys/arc.h>
 #include <sys/sunddi.h>
 #include <sys/zvol.h>
+#ifdef _KERNEL
+#include <sys/zfs_vfsops.h>
+#endif
 #include "zfs_namecheck.h"
 
 static uint64_t dsl_dir_space_towrite(dsl_dir_t *dd);
@@ -1245,6 +1250,7 @@ would_change(dsl_dir_t *dd, int64_t delta, dsl_dir_t *ancestor)
 struct renamearg {
 	dsl_dir_t *newparent;
 	const char *mynewname;
+	boolean_t allowmounted;
 };
 
 static int
@@ -1263,9 +1269,12 @@ dsl_dir_rename_check(void *arg1, void *arg2, dmu_tx_t *tx)
 	 * stats), but any that are present in open context will likely
 	 * be gone by syncing context, so only fail from syncing
 	 * context.
+	 * Don't check if we allow renaming of busy (mounted) dataset.
 	 */
-	if (dmu_tx_is_syncing(tx) && dmu_buf_refcount(dd->dd_dbuf) > 1)
+	if (!ra->allowmounted && dmu_tx_is_syncing(tx) &&
+	    dmu_buf_refcount(dd->dd_dbuf) > 1) {
 		return (EBUSY);
+	}
 
 	/* check for existing name */
 	err = zap_lookup(mos, ra->newparent->dd_phys->dd_child_dir_zapobj,
@@ -1302,7 +1311,7 @@ dsl_dir_rename_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 	objset_t *mos = dp->dp_meta_objset;
 	int err;
 
-	ASSERT(dmu_buf_refcount(dd->dd_dbuf) <= 2);
+	ASSERT(ra->allowmounted || dmu_buf_refcount(dd->dd_dbuf) <= 2);
 
 	if (ra->newparent != dd->dd_parent) {
 		dsl_dir_diduse_space(dd->dd_parent, DD_USED_CHILD,
@@ -1345,6 +1354,7 @@ dsl_dir_rename_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 	ASSERT3U(err, ==, 0);
 	dsl_dir_name(dd, newname);
 #ifdef _KERNEL
+	zfsvfs_update_fromname(oldname, newname);
 	zvol_rename_minors(oldname, newname);
 #endif
 
@@ -1353,7 +1363,7 @@ dsl_dir_rename_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 }
 
 int
-dsl_dir_rename(dsl_dir_t *dd, const char *newname)
+dsl_dir_rename(dsl_dir_t *dd, const char *newname, int flags)
 {
 	struct renamearg ra;
 	int err;
@@ -1375,6 +1385,8 @@ dsl_dir_rename(dsl_dir_t *dd, const char *newname)
 		goto out;
 	}
 
+	ra.allowmounted = !!(flags & ZFS_RENAME_ALLOW_MOUNTED);
+
 	err = dsl_sync_task_do(dd->dd_pool,
 	    dsl_dir_rename_check, dsl_dir_rename_sync, dd, &ra, 3);
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c
index 4db13fd917c3..69374fb7956c 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright 2011 iXsystems, Inc
  */
 
 #include <sys/zfs_context.h>
@@ -605,14 +606,14 @@ sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count,
 		 * and spill buffer.
 		 */
 		if (buftype == SA_BONUS && *index == -1 &&
-		    P2ROUNDUP(*total + hdrsize, 8) >
+		    (*total + P2ROUNDUP(hdrsize, 8)) >
 		    (full_space - sizeof (blkptr_t))) {
 			*index = i;
 			done = B_TRUE;
 		}
 
 next:
-		if (P2ROUNDUP(*total + hdrsize, 8) > full_space &&
+		if ((*total + P2ROUNDUP(hdrsize, 8)) > full_space &&
 		    buftype == SA_BONUS)
 			*will_spill = B_TRUE;
 	}
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
index 22733d070e8b..e4b812755aa7 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -20,6 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DATASET_H
@@ -178,6 +180,12 @@ struct dsl_ds_holdarg {
 	char failed[MAXPATHLEN];
 };
 
+/*
+ * Flags for dsl_dataset_rename().
+ */
+#define	ZFS_RENAME_RECURSIVE		0x01
+#define	ZFS_RENAME_ALLOW_MOUNTED	0x02
+
 #define	dsl_dataset_is_snapshot(ds) \
 	((ds)->ds_phys->ds_num_children != 0)
 
@@ -211,7 +219,7 @@ dsl_syncfunc_t dsl_dataset_destroy_sync;
 dsl_checkfunc_t dsl_dataset_snapshot_check;
 dsl_syncfunc_t dsl_dataset_snapshot_sync;
 dsl_syncfunc_t dsl_dataset_user_hold_sync;
-int dsl_dataset_rename(char *name, const char *newname, boolean_t recursive);
+int dsl_dataset_rename(char *name, const char *newname, int flags);
 int dsl_dataset_promote(const char *name, char *conflsnap);
 int dsl_dataset_clone_swap(dsl_dataset_t *clone, dsl_dataset_t *origin_head,
     boolean_t force);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h
index 2191635dd813..9f2ddc6fc6ac 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h
@@ -20,6 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DIR_H
@@ -131,7 +133,7 @@ int dsl_dir_set_quota(const char *ddname, zprop_source_t source,
     uint64_t quota);
 int dsl_dir_set_reservation(const char *ddname, zprop_source_t source,
     uint64_t reservation);
-int dsl_dir_rename(dsl_dir_t *dd, const char *newname);
+int dsl_dir_rename(dsl_dir_t *dd, const char *newname, int flags);
 int dsl_dir_transfer_possible(dsl_dir_t *sdd, dsl_dir_t *tdd, uint64_t space);
 int dsl_dir_set_reservation_check(void *arg1, void *arg2, dmu_tx_t *tx);
 boolean_t dsl_dir_is_clone(dsl_dir_t *dd);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h
index 0560672fbb73..63cfb0074d7a 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h
@@ -20,6 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 #ifndef	_SYS_FS_ZFS_VFSOPS_H
@@ -153,6 +155,10 @@ extern void zfsvfs_free(zfsvfs_t *zfsvfs);
 extern int zfs_check_global_label(const char *dsname, const char *hexsl);
 extern int zfs_vnode_lock(vnode_t *vp, int flags);
 
+#ifdef _KERNEL
+extern void zfsvfs_update_fromname(const char *oldname, const char *newname);
+#endif
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c
index 7f9b933e7d1d..cfc1687473fe 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
  */
 
 #include <sys/zfs_context.h>
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c
index be3cefced741..832ffdf566e2 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c
@@ -153,17 +153,19 @@ static int
 vdev_file_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
-	vdev_file_t *vf = vd->vdev_tsd;
-	vnode_t *vp = vf->vf_vnode;
+	vdev_file_t *vf;
+	vnode_t *vp;
 	ssize_t resid;
 
-	if (zio->io_type == ZIO_TYPE_IOCTL) {
-		/* XXPOLICY */
-		if (!vdev_readable(vd)) {
-			zio->io_error = ENXIO;
-			return (ZIO_PIPELINE_CONTINUE);
-		}
+	if (!vdev_readable(vd)) {
+		zio->io_error = ENXIO;
+		return (ZIO_PIPELINE_CONTINUE);
+	}
 
+	vf = vd->vdev_tsd;
+	vp = vf->vf_vnode;
+
+	if (zio->io_type == ZIO_TYPE_IOCTL) {
 		switch (zio->io_cmd) {
 		case DKIOCFLUSHWRITECACHE:
 			zio->io_error = VOP_FSYNC(vp, FSYNC | FDSYNC,
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c
index 7372ee74d3ea..a880961173c5 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c
@@ -20,6 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 /*
@@ -785,7 +787,7 @@ zfsctl_snapdir_rename(vnode_t *sdvp, char *snm, vnode_t *tdvp, char *tnm,
 		return (ENOENT);
 	}
 
-	err = dmu_objset_rename(from, to, B_FALSE);
+	err = dmu_objset_rename(from, to, 0);
 	if (err == 0)
 		zfsctl_rename_snap(sdp, sep, tnm);
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
index 17174b29ed9e..37309c18c6b4 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
@@ -20,6 +20,9 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
+ * Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
  */
 
 #include <sys/types.h>
@@ -3261,7 +3264,12 @@ zfs_ioc_rollback(zfs_cmd_t *zc)
 static int
 zfs_ioc_rename(zfs_cmd_t *zc)
 {
-	boolean_t recursive = zc->zc_cookie & 1;
+	int flags = 0;
+
+	if (zc->zc_cookie & 1)
+		flags |= ZFS_RENAME_RECURSIVE;
+	if (zc->zc_cookie & 2)
+		flags |= ZFS_RENAME_ALLOW_MOUNTED;
 
 	zc->zc_value[sizeof (zc->zc_value) - 1] = '\0';
 	if (dataset_namecheck(zc->zc_value, NULL, NULL) != 0 ||
@@ -3273,13 +3281,14 @@ zfs_ioc_rename(zfs_cmd_t *zc)
 	 * in which case the dataset code figures out which snapshots
 	 * to unmount.
 	 */
-	if (!recursive && strchr(zc->zc_name, '@') != NULL &&
+	if (!(flags & ZFS_RENAME_RECURSIVE) &&
+	    strchr(zc->zc_name, '@') != NULL &&
 	    zc->zc_objset_type == DMU_OST_ZFS) {
 		int err = zfs_unmount_snap(zc->zc_name, NULL);
 		if (err)
 			return (err);
 	}
-	return (dmu_objset_rename(zc->zc_name, zc->zc_value, recursive));
+	return (dmu_objset_rename(zc->zc_name, zc->zc_value, flags));
 }
 
 static int
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
index eff9779ad38f..211d73f6699d 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
@@ -20,6 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
+ * All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -2416,3 +2418,35 @@ zfs_get_zplprop(objset_t *os, zfs_prop_t prop, uint64_t *value)
 	}
 	return (error);
 }
+
+#ifdef _KERNEL
+void
+zfsvfs_update_fromname(const char *oldname, const char *newname)
+{
+	char tmpbuf[MAXPATHLEN];
+	struct mount *mp;
+	char *fromname;
+	size_t oldlen;
+
+	oldlen = strlen(oldname);
+
+	mtx_lock(&mountlist_mtx);
+	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
+		fromname = mp->mnt_stat.f_mntfromname;
+		if (strcmp(fromname, oldname) == 0) {
+			(void)strlcpy(fromname, newname,
+			    sizeof(mp->mnt_stat.f_mntfromname));
+			continue;
+		}
+		if (strncmp(fromname, oldname, oldlen) == 0 &&
+		    (fromname[oldlen] == '/' || fromname[oldlen] == '@')) {
+			(void)snprintf(tmpbuf, sizeof(tmpbuf), "%s%s",
+			    newname, fromname + oldlen);
+			(void)strlcpy(fromname, tmpbuf,
+			    sizeof(mp->mnt_stat.f_mntfromname));
+			continue;
+		}
+	}
+	mtx_unlock(&mountlist_mtx);
+}
+#endif
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
index 285881297f6b..acec094a71bb 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
@@ -23,6 +23,7 @@
  */
 
 /* Portions Copyright 2007 Jeremy Teo */
+/* Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org> */
 
 #ifdef _KERNEL
 #include <sys/types.h>
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c
index 61c96becdea3..a14a9fe5858b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c
@@ -26,6 +26,7 @@
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
+/* Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org> */
 
 /*
  * ZFS volume emulation driver.
diff --git a/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c
index 45d87478d6a2..f3cf665f3d8a 100644
--- a/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c
+++ b/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c
@@ -1474,7 +1474,7 @@ fasttrap_getreg(struct regs *rp, uint_t reg)
 	 * If the copy in failed, the process will be in a irrecoverable
 	 * state, and we have no choice but to kill it.
 	 */
-	psignal(ttoproc(curthread), SIGILL);
+	kern_psignal(ttoproc(curthread), SIGILL);
 	return (0);
 }
 
@@ -1593,5 +1593,5 @@ fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value)
 	 * If we couldn't record this register's value, the process is in an
 	 * irrecoverable state and we have no choice but to euthanize it.
 	 */
-	psignal(ttoproc(curthread), SIGILL);
+	kern_psignal(ttoproc(curthread), SIGILL);
 }
diff --git a/sys/cddl/dev/dtrace/dtrace_debug.c b/sys/cddl/dev/dtrace/dtrace_debug.c
index 03af9aa0efd9..7202fe23e75f 100644
--- a/sys/cddl/dev/dtrace/dtrace_debug.c
+++ b/sys/cddl/dev/dtrace/dtrace_debug.c
@@ -31,49 +31,9 @@
 
 #ifdef DEBUG
 
-#if defined(__amd64__)
-static __inline int
-dtrace_cmpset_long(volatile u_long *dst, u_long exp, u_long src)
-{
-	u_char res;
+#include <machine/atomic.h>
 
-	__asm __volatile(
-	"	 lock ; 		"
-	"	cmpxchgq %2,%1 ;	"
-	"       sete	%0 ;		"
-	"1:				"
-	"# dtrace_cmpset_long"
-	: "=a" (res),			/* 0 */
-	  "=m" (*dst)			/* 1 */
-	: "r" (src),			/* 2 */
-	  "a" (exp),			/* 3 */
-	  "m" (*dst)			/* 4 */
-	: "memory");
-
-	return (res);
-}
-#elif defined(__i386__)
-static __inline int
-dtrace_cmpset_long(volatile u_long *dst, u_long exp, u_long src)
-{
-	u_char res;
-
-	__asm __volatile(
-	"        lock ;            	"
-	"       cmpxchgl %2,%1 ;        "
-	"       sete    %0 ;            "
-	"1:                             "
-	"# dtrace_cmpset_long"
-	: "=a" (res),                   /* 0 */
-	  "=m" (*dst)                   /* 1 */
-	: "r" (src),                    /* 2 */
-	  "a" (exp),                    /* 3 */
-	  "m" (*dst)                    /* 4 */
-	: "memory");
-
-	return (res);
-}
-#endif
+#define	dtrace_cmpset_long	atomic_cmpset_long
 
 #define DTRACE_DEBUG_BUFR_SIZE	(32 * 1024)
 
diff --git a/sys/compat/freebsd32/freebsd32_ioctl.c b/sys/compat/freebsd32/freebsd32_ioctl.c
index 1d773caab3bc..81f5c8ed4715 100644
--- a/sys/compat/freebsd32/freebsd32_ioctl.c
+++ b/sys/compat/freebsd32/freebsd32_ioctl.c
@@ -396,7 +396,7 @@ freebsd32_ioctl(struct thread *td, struct freebsd32_ioctl_args *uap)
 		ap.fd = uap->fd;
 		ap.com = uap->com;
 		PTRIN_CP(*uap, ap, data);
-		return ioctl(td, &ap);
+		return sys_ioctl(td, &ap);
 	}
 
 	fdrop(fp, td);
diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c
index 5b6e8950910b..fc2932b923ff 100644
--- a/sys/compat/freebsd32/freebsd32_misc.c
+++ b/sys/compat/freebsd32/freebsd32_misc.c
@@ -425,7 +425,7 @@ freebsd32_mmap_partial(struct thread *td, vm_offset_t start, vm_offset_t end,
 		r.buf = (void *) start;
 		r.nbyte = end - start;
 		r.offset = pos;
-		return (pread(td, &r));
+		return (sys_pread(td, &r));
 	} else {
 		while (start < end) {
 			subyte((void *) start, 0);
@@ -436,6 +436,21 @@ freebsd32_mmap_partial(struct thread *td, vm_offset_t start, vm_offset_t end,
 }
 #endif
 
+int
+freebsd32_mprotect(struct thread *td, struct freebsd32_mprotect_args *uap)
+{
+	struct mprotect_args ap;
+
+	ap.addr = PTRIN(uap->addr);
+	ap.len = uap->len;
+	ap.prot = uap->prot;
+#if defined(__amd64__) || defined(__ia64__)
+	if (i386_read_exec && (ap.prot & PROT_READ) != 0)
+		ap.prot |= PROT_EXEC;
+#endif
+	return (sys_mprotect(td, &ap));
+}
+
 int
 freebsd32_mmap(struct thread *td, struct freebsd32_mmap_args *uap)
 {
@@ -500,7 +515,7 @@ freebsd32_mmap(struct thread *td, struct freebsd32_mmap_args *uap)
 			r.buf = (void *) start;
 			r.nbyte = end - start;
 			r.offset = pos;
-			error = pread(td, &r);
+			error = sys_pread(td, &r);
 			if (error)
 				return (error);
 
@@ -520,6 +535,11 @@ freebsd32_mmap(struct thread *td, struct freebsd32_mmap_args *uap)
 	}
 #endif
 
+#if defined(__amd64__) || defined(__ia64__)
+	if (i386_read_exec && (prot & PROT_READ))
+		prot |= PROT_EXEC;
+#endif
+
 	ap.addr = (void *) addr;
 	ap.len = len;
 	ap.prot = prot;
@@ -527,7 +547,7 @@ freebsd32_mmap(struct thread *td, struct freebsd32_mmap_args *uap)
 	ap.fd = fd;
 	ap.pos = pos;
 
-	return (mmap(td, &ap));
+	return (sys_mmap(td, &ap));
 }
 
 #ifdef COMPAT_FREEBSD6
@@ -1401,7 +1421,7 @@ freebsd32_pread(struct thread *td, struct freebsd32_pread_args *uap)
 	ap.buf = uap->buf;
 	ap.nbyte = uap->nbyte;
 	ap.offset = PAIR32TO64(off_t,uap->offset);
-	return (pread(td, &ap));
+	return (sys_pread(td, &ap));
 }
 
 int
@@ -1413,7 +1433,7 @@ freebsd32_pwrite(struct thread *td, struct freebsd32_pwrite_args *uap)
 	ap.buf = uap->buf;
 	ap.nbyte = uap->nbyte;
 	ap.offset = PAIR32TO64(off_t,uap->offset);
-	return (pwrite(td, &ap));
+	return (sys_pwrite(td, &ap));
 }
 
 #ifdef COMPAT_43
@@ -1425,7 +1445,7 @@ ofreebsd32_lseek(struct thread *td, struct ofreebsd32_lseek_args *uap)
 	nuap.fd = uap->fd;
 	nuap.offset = uap->offset;
 	nuap.whence = uap->whence;
-	return (lseek(td, &nuap));
+	return (sys_lseek(td, &nuap));
 }
 #endif
 
@@ -1439,7 +1459,7 @@ freebsd32_lseek(struct thread *td, struct freebsd32_lseek_args *uap)
 	ap.fd = uap->fd;
 	ap.offset = PAIR32TO64(off_t,uap->offset);
 	ap.whence = uap->whence;
-	error = lseek(td, &ap);
+	error = sys_lseek(td, &ap);
 	/* Expand the quad return into two parts for eax and edx */
 	pos = *(off_t *)(td->td_retval);
 	td->td_retval[RETVAL_LO] = pos & 0xffffffff;	/* %eax */
@@ -1454,7 +1474,7 @@ freebsd32_truncate(struct thread *td, struct freebsd32_truncate_args *uap)
 
 	ap.path = uap->path;
 	ap.length = PAIR32TO64(off_t,uap->length);
-	return (truncate(td, &ap));
+	return (sys_truncate(td, &ap));
 }
 
 int
@@ -1464,7 +1484,7 @@ freebsd32_ftruncate(struct thread *td, struct freebsd32_ftruncate_args *uap)
 
 	ap.fd = uap->fd;
 	ap.length = PAIR32TO64(off_t,uap->length);
-	return (ftruncate(td, &ap));
+	return (sys_ftruncate(td, &ap));
 }
 
 #ifdef COMPAT_43
@@ -1519,7 +1539,7 @@ freebsd6_freebsd32_pread(struct thread *td, struct freebsd6_freebsd32_pread_args
 	ap.buf = uap->buf;
 	ap.nbyte = uap->nbyte;
 	ap.offset = PAIR32TO64(off_t,uap->offset);
-	return (pread(td, &ap));
+	return (sys_pread(td, &ap));
 }
 
 int
@@ -1531,7 +1551,7 @@ freebsd6_freebsd32_pwrite(struct thread *td, struct freebsd6_freebsd32_pwrite_ar
 	ap.buf = uap->buf;
 	ap.nbyte = uap->nbyte;
 	ap.offset = PAIR32TO64(off_t,uap->offset);
-	return (pwrite(td, &ap));
+	return (sys_pwrite(td, &ap));
 }
 
 int
@@ -1544,7 +1564,7 @@ freebsd6_freebsd32_lseek(struct thread *td, struct freebsd6_freebsd32_lseek_args
 	ap.fd = uap->fd;
 	ap.offset = PAIR32TO64(off_t,uap->offset);
 	ap.whence = uap->whence;
-	error = lseek(td, &ap);
+	error = sys_lseek(td, &ap);
 	/* Expand the quad return into two parts for eax and edx */
 	pos = *(off_t *)(td->td_retval);
 	td->td_retval[RETVAL_LO] = pos & 0xffffffff;	/* %eax */
@@ -1559,7 +1579,7 @@ freebsd6_freebsd32_truncate(struct thread *td, struct freebsd6_freebsd32_truncat
 
 	ap.path = uap->path;
 	ap.length = PAIR32TO64(off_t,uap->length);
-	return (truncate(td, &ap));
+	return (sys_truncate(td, &ap));
 }
 
 int
@@ -1569,7 +1589,7 @@ freebsd6_freebsd32_ftruncate(struct thread *td, struct freebsd6_freebsd32_ftrunc
 
 	ap.fd = uap->fd;
 	ap.length = PAIR32TO64(off_t,uap->length);
-	return (ftruncate(td, &ap));
+	return (sys_ftruncate(td, &ap));
 }
 #endif /* COMPAT_FREEBSD6 */
 
@@ -2409,7 +2429,7 @@ freebsd32_cpuset_setid(struct thread *td,
 	ap.id = PAIR32TO64(id_t,uap->id);
 	ap.setid = uap->setid;
 
-	return (cpuset_setid(td, &ap));
+	return (sys_cpuset_setid(td, &ap));
 }
 
 int
@@ -2423,7 +2443,7 @@ freebsd32_cpuset_getid(struct thread *td,
 	ap.id = PAIR32TO64(id_t,uap->id);
 	ap.setid = uap->setid;
 
-	return (cpuset_getid(td, &ap));
+	return (sys_cpuset_getid(td, &ap));
 }
 
 int
@@ -2438,7 +2458,7 @@ freebsd32_cpuset_getaffinity(struct thread *td,
 	ap.cpusetsize = uap->cpusetsize;
 	ap.mask = uap->mask;
 
-	return (cpuset_getaffinity(td, &ap));
+	return (sys_cpuset_getaffinity(td, &ap));
 }
 
 int
@@ -2453,7 +2473,7 @@ freebsd32_cpuset_setaffinity(struct thread *td,
 	ap.cpusetsize = uap->cpusetsize;
 	ap.mask = uap->mask;
 
-	return (cpuset_setaffinity(td, &ap));
+	return (sys_cpuset_setaffinity(td, &ap));
 }
 
 int
@@ -2811,7 +2831,7 @@ freebsd32_posix_fallocate(struct thread *td,
 	struct posix_fallocate_args ap;
 
 	ap.fd = uap->fd;
-	ap.offset = (uap->offsetlo | ((off_t)uap->offsethi << 32));
-	ap.len = (uap->lenlo | ((off_t)uap->lenhi << 32));
-	return (posix_fallocate(td, &ap));
+	ap.offset = PAIR32TO64(off_t, uap->offset);
+	ap.len = PAIR32TO64(off_t, uap->len);
+	return (sys_posix_fallocate(td, &ap));
 }
diff --git a/sys/compat/freebsd32/freebsd32_proto.h b/sys/compat/freebsd32/freebsd32_proto.h
index 21b84e80826d..ae2036a56daf 100644
--- a/sys/compat/freebsd32/freebsd32_proto.h
+++ b/sys/compat/freebsd32/freebsd32_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
  */
 
 #ifndef _FREEBSD32_SYSPROTO_H_
@@ -76,6 +76,11 @@ struct freebsd32_execve_args {
 	char argv_l_[PADL_(u_int32_t *)]; u_int32_t * argv; char argv_r_[PADR_(u_int32_t *)];
 	char envv_l_[PADL_(u_int32_t *)]; u_int32_t * envv; char envv_r_[PADR_(u_int32_t *)];
 };
+struct freebsd32_mprotect_args {
+	char addr_l_[PADL_(const void *)]; const void * addr; char addr_r_[PADR_(const void *)];
+	char len_l_[PADL_(size_t)]; size_t len; char len_r_[PADR_(size_t)];
+	char prot_l_[PADL_(int)]; int prot; char prot_r_[PADR_(int)];
+};
 struct freebsd32_setitimer_args {
 	char which_l_[PADL_(u_int)]; u_int which; char which_r_[PADR_(u_int)];
 	char itv_l_[PADL_(struct itimerval32 *)]; struct itimerval32 * itv; char itv_r_[PADR_(struct itimerval32 *)];
@@ -570,10 +575,10 @@ struct freebsd32_pselect_args {
 };
 struct freebsd32_posix_fallocate_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
-	char offsetlo_l_[PADL_(uint32_t)]; uint32_t offsetlo; char offsetlo_r_[PADR_(uint32_t)];
-	char offsethi_l_[PADL_(uint32_t)]; uint32_t offsethi; char offsethi_r_[PADR_(uint32_t)];
-	char lenlo_l_[PADL_(uint32_t)]; uint32_t lenlo; char lenlo_r_[PADR_(uint32_t)];
-	char lenhi_l_[PADL_(uint32_t)]; uint32_t lenhi; char lenhi_r_[PADR_(uint32_t)];
+	char offset1_l_[PADL_(uint32_t)]; uint32_t offset1; char offset1_r_[PADR_(uint32_t)];
+	char offset2_l_[PADL_(uint32_t)]; uint32_t offset2; char offset2_r_[PADR_(uint32_t)];
+	char len1_l_[PADL_(uint32_t)]; uint32_t len1; char len1_r_[PADR_(uint32_t)];
+	char len2_l_[PADL_(uint32_t)]; uint32_t len2; char len2_r_[PADR_(uint32_t)];
 };
 #if !defined(PAD64_REQUIRED) && defined(__powerpc__)
 #define PAD64_REQUIRED
@@ -585,6 +590,7 @@ int	freebsd32_recvfrom(struct thread *, struct freebsd32_recvfrom_args *);
 int	freebsd32_sigaltstack(struct thread *, struct freebsd32_sigaltstack_args *);
 int	freebsd32_ioctl(struct thread *, struct freebsd32_ioctl_args *);
 int	freebsd32_execve(struct thread *, struct freebsd32_execve_args *);
+int	freebsd32_mprotect(struct thread *, struct freebsd32_mprotect_args *);
 int	freebsd32_setitimer(struct thread *, struct freebsd32_setitimer_args *);
 int	freebsd32_getitimer(struct thread *, struct freebsd32_getitimer_args *);
 int	freebsd32_select(struct thread *, struct freebsd32_select_args *);
@@ -943,6 +949,7 @@ int	freebsd7_freebsd32_shmctl(struct thread *, struct freebsd7_freebsd32_shmctl_
 #define	FREEBSD32_SYS_AUE_freebsd32_execve	AUE_EXECVE
 #define	FREEBSD32_SYS_AUE_ofreebsd32_fstat	AUE_FSTAT
 #define	FREEBSD32_SYS_AUE_ofreebsd32_getpagesize	AUE_NULL
+#define	FREEBSD32_SYS_AUE_freebsd32_mprotect	AUE_MPROTECT
 #define	FREEBSD32_SYS_AUE_freebsd32_setitimer	AUE_SETITIMER
 #define	FREEBSD32_SYS_AUE_freebsd32_getitimer	AUE_GETITIMER
 #define	FREEBSD32_SYS_AUE_freebsd32_select	AUE_SELECT
diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h
index 19f91be17fed..fe4dbc189a8a 100644
--- a/sys/compat/freebsd32/freebsd32_syscall.h
+++ b/sys/compat/freebsd32/freebsd32_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
  */
 
 #define	FREEBSD32_SYS_syscall	0
@@ -80,7 +80,7 @@
 				/* 71 is obsolete ommap */
 #define	FREEBSD32_SYS_vadvise	72
 #define	FREEBSD32_SYS_munmap	73
-#define	FREEBSD32_SYS_mprotect	74
+#define	FREEBSD32_SYS_freebsd32_mprotect	74
 #define	FREEBSD32_SYS_madvise	75
 				/* 76 is obsolete vhangup */
 				/* 77 is obsolete vlimit */
diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c
index 8e67e8b801d4..c9308bbb7ea0 100644
--- a/sys/compat/freebsd32/freebsd32_syscalls.c
+++ b/sys/compat/freebsd32/freebsd32_syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
  */
 
 const char *freebsd32_syscallnames[] = {
@@ -84,7 +84,7 @@ const char *freebsd32_syscallnames[] = {
 	"obs_ommap",			/* 71 = obsolete ommap */
 	"vadvise",			/* 72 = vadvise */
 	"munmap",			/* 73 = munmap */
-	"mprotect",			/* 74 = mprotect */
+	"freebsd32_mprotect",			/* 74 = freebsd32_mprotect */
 	"madvise",			/* 75 = madvise */
 	"obs_vhangup",			/* 76 = obsolete vhangup */
 	"obs_vlimit",			/* 77 = obsolete vlimit */
diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c
index 66f42ac1d1b7..3f54930250c6 100644
--- a/sys/compat/freebsd32/freebsd32_sysent.c
+++ b/sys/compat/freebsd32/freebsd32_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
  */
 
 #include "opt_compat.h"
@@ -48,112 +48,112 @@ struct sysent freebsd32_sysent[] = {
 #define PAD64_REQUIRED
 #endif
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 0 = syscall */
-	{ AS(sys_exit_args), (sy_call_t *)sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
-	{ 0, (sy_call_t *)fork, AUE_FORK, NULL, 0, 0, 0, SY_THR_STATIC },		/* 2 = fork */
-	{ AS(read_args), (sy_call_t *)read, AUE_READ, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
-	{ AS(write_args), (sy_call_t *)write, AUE_WRITE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
-	{ AS(open_args), (sy_call_t *)open, AUE_OPEN_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = open */
-	{ AS(close_args), (sy_call_t *)close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
+	{ AS(sys_exit_args), (sy_call_t *)sys_sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
+	{ 0, (sy_call_t *)sys_fork, AUE_FORK, NULL, 0, 0, 0, SY_THR_STATIC },		/* 2 = fork */
+	{ AS(read_args), (sy_call_t *)sys_read, AUE_READ, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
+	{ AS(write_args), (sy_call_t *)sys_write, AUE_WRITE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
+	{ AS(open_args), (sy_call_t *)sys_open, AUE_OPEN_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = open */
+	{ AS(close_args), (sy_call_t *)sys_close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
 	{ AS(freebsd32_wait4_args), (sy_call_t *)freebsd32_wait4, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = freebsd32_wait4 */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 8 = obsolete old creat */
-	{ AS(link_args), (sy_call_t *)link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
-	{ AS(unlink_args), (sy_call_t *)unlink, AUE_UNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = unlink */
+	{ AS(link_args), (sy_call_t *)sys_link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
+	{ AS(unlink_args), (sy_call_t *)sys_unlink, AUE_UNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = unlink */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 11 = obsolete execv */
-	{ AS(chdir_args), (sy_call_t *)chdir, AUE_CHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = chdir */
-	{ AS(fchdir_args), (sy_call_t *)fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 13 = fchdir */
-	{ AS(mknod_args), (sy_call_t *)mknod, AUE_MKNOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 14 = mknod */
-	{ AS(chmod_args), (sy_call_t *)chmod, AUE_CHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = chmod */
-	{ AS(chown_args), (sy_call_t *)chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = chown */
-	{ AS(obreak_args), (sy_call_t *)obreak, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 17 = break */
+	{ AS(chdir_args), (sy_call_t *)sys_chdir, AUE_CHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = chdir */
+	{ AS(fchdir_args), (sy_call_t *)sys_fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 13 = fchdir */
+	{ AS(mknod_args), (sy_call_t *)sys_mknod, AUE_MKNOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 14 = mknod */
+	{ AS(chmod_args), (sy_call_t *)sys_chmod, AUE_CHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = chmod */
+	{ AS(chown_args), (sy_call_t *)sys_chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = chown */
+	{ AS(obreak_args), (sy_call_t *)sys_obreak, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 17 = break */
 	{ compat4(AS(freebsd4_freebsd32_getfsstat_args),freebsd32_getfsstat), AUE_GETFSSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 18 = freebsd4 freebsd32_getfsstat */
 	{ compat(AS(ofreebsd32_lseek_args),freebsd32_lseek), AUE_LSEEK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 19 = old freebsd32_lseek */
-	{ 0, (sy_call_t *)getpid, AUE_GETPID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 20 = getpid */
-	{ AS(mount_args), (sy_call_t *)mount, AUE_MOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 21 = mount */
-	{ AS(unmount_args), (sy_call_t *)unmount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 22 = unmount */
-	{ AS(setuid_args), (sy_call_t *)setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 23 = setuid */
-	{ 0, (sy_call_t *)getuid, AUE_GETUID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 24 = getuid */
-	{ 0, (sy_call_t *)geteuid, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 25 = geteuid */
-	{ AS(ptrace_args), (sy_call_t *)ptrace, AUE_PTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 26 = ptrace */
+	{ 0, (sy_call_t *)sys_getpid, AUE_GETPID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 20 = getpid */
+	{ AS(mount_args), (sy_call_t *)sys_mount, AUE_MOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 21 = mount */
+	{ AS(unmount_args), (sy_call_t *)sys_unmount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 22 = unmount */
+	{ AS(setuid_args), (sy_call_t *)sys_setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 23 = setuid */
+	{ 0, (sy_call_t *)sys_getuid, AUE_GETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 24 = getuid */
+	{ 0, (sy_call_t *)sys_geteuid, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 25 = geteuid */
+	{ AS(ptrace_args), (sy_call_t *)sys_ptrace, AUE_PTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 26 = ptrace */
 	{ AS(freebsd32_recvmsg_args), (sy_call_t *)freebsd32_recvmsg, AUE_RECVMSG, NULL, 0, 0, 0, SY_THR_STATIC },	/* 27 = freebsd32_recvmsg */
 	{ AS(freebsd32_sendmsg_args), (sy_call_t *)freebsd32_sendmsg, AUE_SENDMSG, NULL, 0, 0, 0, SY_THR_STATIC },	/* 28 = freebsd32_sendmsg */
 	{ AS(freebsd32_recvfrom_args), (sy_call_t *)freebsd32_recvfrom, AUE_RECVFROM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 29 = freebsd32_recvfrom */
-	{ AS(accept_args), (sy_call_t *)accept, AUE_ACCEPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 30 = accept */
-	{ AS(getpeername_args), (sy_call_t *)getpeername, AUE_GETPEERNAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 31 = getpeername */
-	{ AS(getsockname_args), (sy_call_t *)getsockname, AUE_GETSOCKNAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 32 = getsockname */
-	{ AS(access_args), (sy_call_t *)access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = access */
-	{ AS(chflags_args), (sy_call_t *)chflags, AUE_CHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = chflags */
-	{ AS(fchflags_args), (sy_call_t *)fchflags, AUE_FCHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 35 = fchflags */
-	{ 0, (sy_call_t *)sync, AUE_SYNC, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
-	{ AS(kill_args), (sy_call_t *)kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = kill */
+	{ AS(accept_args), (sy_call_t *)sys_accept, AUE_ACCEPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 30 = accept */
+	{ AS(getpeername_args), (sy_call_t *)sys_getpeername, AUE_GETPEERNAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 31 = getpeername */
+	{ AS(getsockname_args), (sy_call_t *)sys_getsockname, AUE_GETSOCKNAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 32 = getsockname */
+	{ AS(access_args), (sy_call_t *)sys_access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = access */
+	{ AS(chflags_args), (sy_call_t *)sys_chflags, AUE_CHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = chflags */
+	{ AS(fchflags_args), (sy_call_t *)sys_fchflags, AUE_FCHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 35 = fchflags */
+	{ 0, (sy_call_t *)sys_sync, AUE_SYNC, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
+	{ AS(kill_args), (sy_call_t *)sys_kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = kill */
 	{ compat(AS(ofreebsd32_stat_args),freebsd32_stat), AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 38 = old freebsd32_stat */
-	{ 0, (sy_call_t *)getppid, AUE_GETPPID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = getppid */
+	{ 0, (sy_call_t *)sys_getppid, AUE_GETPPID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = getppid */
 	{ compat(AS(ofreebsd32_lstat_args),freebsd32_lstat), AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 40 = old freebsd32_lstat */
-	{ AS(dup_args), (sy_call_t *)dup, AUE_DUP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
-	{ 0, (sy_call_t *)pipe, AUE_PIPE, NULL, 0, 0, 0, SY_THR_STATIC },		/* 42 = pipe */
-	{ 0, (sy_call_t *)getegid, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = getegid */
-	{ AS(profil_args), (sy_call_t *)profil, AUE_PROFILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 44 = profil */
-	{ AS(ktrace_args), (sy_call_t *)ktrace, AUE_KTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 45 = ktrace */
+	{ AS(dup_args), (sy_call_t *)sys_dup, AUE_DUP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
+	{ 0, (sy_call_t *)sys_pipe, AUE_PIPE, NULL, 0, 0, 0, SY_THR_STATIC },		/* 42 = pipe */
+	{ 0, (sy_call_t *)sys_getegid, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = getegid */
+	{ AS(profil_args), (sy_call_t *)sys_profil, AUE_PROFILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 44 = profil */
+	{ AS(ktrace_args), (sy_call_t *)sys_ktrace, AUE_KTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 45 = ktrace */
 	{ compat(AS(ofreebsd32_sigaction_args),freebsd32_sigaction), AUE_SIGACTION, NULL, 0, 0, 0, SY_THR_STATIC },	/* 46 = old freebsd32_sigaction */
-	{ 0, (sy_call_t *)getgid, AUE_GETGID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 47 = getgid */
+	{ 0, (sy_call_t *)sys_getgid, AUE_GETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 47 = getgid */
 	{ compat(AS(ofreebsd32_sigprocmask_args),freebsd32_sigprocmask), AUE_SIGPROCMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 48 = old freebsd32_sigprocmask */
-	{ AS(getlogin_args), (sy_call_t *)getlogin, AUE_GETLOGIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 49 = getlogin */
-	{ AS(setlogin_args), (sy_call_t *)setlogin, AUE_SETLOGIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = setlogin */
-	{ AS(acct_args), (sy_call_t *)acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
+	{ AS(getlogin_args), (sy_call_t *)sys_getlogin, AUE_GETLOGIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 49 = getlogin */
+	{ AS(setlogin_args), (sy_call_t *)sys_setlogin, AUE_SETLOGIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = setlogin */
+	{ AS(acct_args), (sy_call_t *)sys_acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
 	{ compat(0,freebsd32_sigpending), AUE_SIGPENDING, NULL, 0, 0, 0, SY_THR_STATIC },	/* 52 = old freebsd32_sigpending */
 	{ AS(freebsd32_sigaltstack_args), (sy_call_t *)freebsd32_sigaltstack, AUE_SIGALTSTACK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 53 = freebsd32_sigaltstack */
 	{ AS(freebsd32_ioctl_args), (sy_call_t *)freebsd32_ioctl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 54 = freebsd32_ioctl */
-	{ AS(reboot_args), (sy_call_t *)reboot, AUE_REBOOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = reboot */
-	{ AS(revoke_args), (sy_call_t *)revoke, AUE_REVOKE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 56 = revoke */
-	{ AS(symlink_args), (sy_call_t *)symlink, AUE_SYMLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = symlink */
-	{ AS(readlink_args), (sy_call_t *)readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = readlink */
+	{ AS(reboot_args), (sy_call_t *)sys_reboot, AUE_REBOOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = reboot */
+	{ AS(revoke_args), (sy_call_t *)sys_revoke, AUE_REVOKE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 56 = revoke */
+	{ AS(symlink_args), (sy_call_t *)sys_symlink, AUE_SYMLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = symlink */
+	{ AS(readlink_args), (sy_call_t *)sys_readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = readlink */
 	{ AS(freebsd32_execve_args), (sy_call_t *)freebsd32_execve, AUE_EXECVE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = freebsd32_execve */
-	{ AS(umask_args), (sy_call_t *)umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
-	{ AS(chroot_args), (sy_call_t *)chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
+	{ AS(umask_args), (sy_call_t *)sys_umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
+	{ AS(chroot_args), (sy_call_t *)sys_chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
 	{ compat(AS(ofreebsd32_fstat_args),freebsd32_fstat), AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 62 = old freebsd32_fstat */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 63 = obsolete ogetkerninfo */
 	{ compat(AS(ofreebsd32_getpagesize_args),freebsd32_getpagesize), AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 64 = old freebsd32_getpagesize */
-	{ AS(msync_args), (sy_call_t *)msync, AUE_MSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 65 = msync */
-	{ 0, (sy_call_t *)vfork, AUE_VFORK, NULL, 0, 0, 0, SY_THR_STATIC },		/* 66 = vfork */
+	{ AS(msync_args), (sy_call_t *)sys_msync, AUE_MSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 65 = msync */
+	{ 0, (sy_call_t *)sys_vfork, AUE_VFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 66 = vfork */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 67 = obsolete vread */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 68 = obsolete vwrite */
-	{ AS(sbrk_args), (sy_call_t *)sbrk, AUE_SBRK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 69 = sbrk */
-	{ AS(sstk_args), (sy_call_t *)sstk, AUE_SSTK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 70 = sstk */
+	{ AS(sbrk_args), (sy_call_t *)sys_sbrk, AUE_SBRK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 69 = sbrk */
+	{ AS(sstk_args), (sy_call_t *)sys_sstk, AUE_SSTK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 70 = sstk */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 71 = obsolete ommap */
-	{ AS(ovadvise_args), (sy_call_t *)ovadvise, AUE_O_VADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 72 = vadvise */
-	{ AS(munmap_args), (sy_call_t *)munmap, AUE_MUNMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 73 = munmap */
-	{ AS(mprotect_args), (sy_call_t *)mprotect, AUE_MPROTECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 74 = mprotect */
-	{ AS(madvise_args), (sy_call_t *)madvise, AUE_MADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 75 = madvise */
+	{ AS(ovadvise_args), (sy_call_t *)sys_ovadvise, AUE_O_VADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 72 = vadvise */
+	{ AS(munmap_args), (sy_call_t *)sys_munmap, AUE_MUNMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 73 = munmap */
+	{ AS(freebsd32_mprotect_args), (sy_call_t *)freebsd32_mprotect, AUE_MPROTECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 74 = freebsd32_mprotect */
+	{ AS(madvise_args), (sy_call_t *)sys_madvise, AUE_MADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 75 = madvise */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 76 = obsolete vhangup */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 77 = obsolete vlimit */
-	{ AS(mincore_args), (sy_call_t *)mincore, AUE_MINCORE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 78 = mincore */
-	{ AS(getgroups_args), (sy_call_t *)getgroups, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 79 = getgroups */
-	{ AS(setgroups_args), (sy_call_t *)setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = setgroups */
-	{ 0, (sy_call_t *)getpgrp, AUE_GETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 81 = getpgrp */
-	{ AS(setpgid_args), (sy_call_t *)setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 82 = setpgid */
+	{ AS(mincore_args), (sy_call_t *)sys_mincore, AUE_MINCORE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 78 = mincore */
+	{ AS(getgroups_args), (sy_call_t *)sys_getgroups, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 79 = getgroups */
+	{ AS(setgroups_args), (sy_call_t *)sys_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = setgroups */
+	{ 0, (sy_call_t *)sys_getpgrp, AUE_GETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 81 = getpgrp */
+	{ AS(setpgid_args), (sy_call_t *)sys_setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 82 = setpgid */
 	{ AS(freebsd32_setitimer_args), (sy_call_t *)freebsd32_setitimer, AUE_SETITIMER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 83 = freebsd32_setitimer */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 84 = obsolete owait */
-	{ AS(swapon_args), (sy_call_t *)swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = swapon */
+	{ AS(swapon_args), (sy_call_t *)sys_swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = swapon */
 	{ AS(freebsd32_getitimer_args), (sy_call_t *)freebsd32_getitimer, AUE_GETITIMER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 86 = freebsd32_getitimer */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 87 = obsolete ogethostname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 88 = obsolete osethostname */
-	{ 0, (sy_call_t *)getdtablesize, AUE_GETDTABLESIZE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = getdtablesize */
-	{ AS(dup2_args), (sy_call_t *)dup2, AUE_DUP2, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = dup2 */
+	{ 0, (sy_call_t *)sys_getdtablesize, AUE_GETDTABLESIZE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = getdtablesize */
+	{ AS(dup2_args), (sy_call_t *)sys_dup2, AUE_DUP2, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = dup2 */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 91 = getdopt */
-	{ AS(fcntl_args), (sy_call_t *)fcntl, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 92 = fcntl */
+	{ AS(fcntl_args), (sy_call_t *)sys_fcntl, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 92 = fcntl */
 	{ AS(freebsd32_select_args), (sy_call_t *)freebsd32_select, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 93 = freebsd32_select */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 94 = setdopt */
-	{ AS(fsync_args), (sy_call_t *)fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = fsync */
-	{ AS(setpriority_args), (sy_call_t *)setpriority, AUE_SETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 96 = setpriority */
-	{ AS(socket_args), (sy_call_t *)socket, AUE_SOCKET, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = socket */
-	{ AS(connect_args), (sy_call_t *)connect, AUE_CONNECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 98 = connect */
+	{ AS(fsync_args), (sy_call_t *)sys_fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = fsync */
+	{ AS(setpriority_args), (sy_call_t *)sys_setpriority, AUE_SETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 96 = setpriority */
+	{ AS(socket_args), (sy_call_t *)sys_socket, AUE_SOCKET, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = socket */
+	{ AS(connect_args), (sy_call_t *)sys_connect, AUE_CONNECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 98 = connect */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 99 = obsolete oaccept */
-	{ AS(getpriority_args), (sy_call_t *)getpriority, AUE_GETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 100 = getpriority */
+	{ AS(getpriority_args), (sy_call_t *)sys_getpriority, AUE_GETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 100 = getpriority */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 101 = obsolete osend */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 102 = obsolete orecv */
 	{ compat(AS(ofreebsd32_sigreturn_args),freebsd32_sigreturn), AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 103 = old freebsd32_sigreturn */
-	{ AS(bind_args), (sy_call_t *)bind, AUE_BIND, NULL, 0, 0, 0, SY_THR_STATIC },	/* 104 = bind */
-	{ AS(setsockopt_args), (sy_call_t *)setsockopt, AUE_SETSOCKOPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 105 = setsockopt */
-	{ AS(listen_args), (sy_call_t *)listen, AUE_LISTEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 106 = listen */
+	{ AS(bind_args), (sy_call_t *)sys_bind, AUE_BIND, NULL, 0, 0, 0, SY_THR_STATIC },	/* 104 = bind */
+	{ AS(setsockopt_args), (sy_call_t *)sys_setsockopt, AUE_SETSOCKOPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 105 = setsockopt */
+	{ AS(listen_args), (sy_call_t *)sys_listen, AUE_LISTEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 106 = listen */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 107 = obsolete vtimes */
 	{ compat(AS(ofreebsd32_sigvec_args),freebsd32_sigvec), AUE_O_SIGVEC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 108 = old freebsd32_sigvec */
 	{ compat(AS(ofreebsd32_sigblock_args),freebsd32_sigblock), AUE_O_SIGBLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 109 = old freebsd32_sigblock */
@@ -165,26 +165,26 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 115 = obsolete vtrace */
 	{ AS(freebsd32_gettimeofday_args), (sy_call_t *)freebsd32_gettimeofday, AUE_GETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 116 = freebsd32_gettimeofday */
 	{ AS(freebsd32_getrusage_args), (sy_call_t *)freebsd32_getrusage, AUE_GETRUSAGE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 117 = freebsd32_getrusage */
-	{ AS(getsockopt_args), (sy_call_t *)getsockopt, AUE_GETSOCKOPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = getsockopt */
+	{ AS(getsockopt_args), (sy_call_t *)sys_getsockopt, AUE_GETSOCKOPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = getsockopt */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 119 = resuba */
 	{ AS(freebsd32_readv_args), (sy_call_t *)freebsd32_readv, AUE_READV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 120 = freebsd32_readv */
 	{ AS(freebsd32_writev_args), (sy_call_t *)freebsd32_writev, AUE_WRITEV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 121 = freebsd32_writev */
 	{ AS(freebsd32_settimeofday_args), (sy_call_t *)freebsd32_settimeofday, AUE_SETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 122 = freebsd32_settimeofday */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_FCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 123 = fchown */
-	{ AS(fchmod_args), (sy_call_t *)fchmod, AUE_FCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 124 = fchmod */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_FCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 123 = fchown */
+	{ AS(fchmod_args), (sy_call_t *)sys_fchmod, AUE_FCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 124 = fchmod */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 125 = obsolete orecvfrom */
-	{ AS(setreuid_args), (sy_call_t *)setreuid, AUE_SETREUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 126 = setreuid */
-	{ AS(setregid_args), (sy_call_t *)setregid, AUE_SETREGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 127 = setregid */
-	{ AS(rename_args), (sy_call_t *)rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 128 = rename */
+	{ AS(setreuid_args), (sy_call_t *)sys_setreuid, AUE_SETREUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 126 = setreuid */
+	{ AS(setregid_args), (sy_call_t *)sys_setregid, AUE_SETREGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 127 = setregid */
+	{ AS(rename_args), (sy_call_t *)sys_rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 128 = rename */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 129 = obsolete otruncate */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 130 = obsolete ftruncate */
-	{ AS(flock_args), (sy_call_t *)flock, AUE_FLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 131 = flock */
-	{ AS(mkfifo_args), (sy_call_t *)mkfifo, AUE_MKFIFO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = mkfifo */
-	{ AS(sendto_args), (sy_call_t *)sendto, AUE_SENDTO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 133 = sendto */
-	{ AS(shutdown_args), (sy_call_t *)shutdown, AUE_SHUTDOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 134 = shutdown */
-	{ AS(socketpair_args), (sy_call_t *)socketpair, AUE_SOCKETPAIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 135 = socketpair */
-	{ AS(mkdir_args), (sy_call_t *)mkdir, AUE_MKDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = mkdir */
-	{ AS(rmdir_args), (sy_call_t *)rmdir, AUE_RMDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 137 = rmdir */
+	{ AS(flock_args), (sy_call_t *)sys_flock, AUE_FLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 131 = flock */
+	{ AS(mkfifo_args), (sy_call_t *)sys_mkfifo, AUE_MKFIFO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = mkfifo */
+	{ AS(sendto_args), (sy_call_t *)sys_sendto, AUE_SENDTO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 133 = sendto */
+	{ AS(shutdown_args), (sy_call_t *)sys_shutdown, AUE_SHUTDOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 134 = shutdown */
+	{ AS(socketpair_args), (sy_call_t *)sys_socketpair, AUE_SOCKETPAIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 135 = socketpair */
+	{ AS(mkdir_args), (sy_call_t *)sys_mkdir, AUE_MKDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = mkdir */
+	{ AS(rmdir_args), (sy_call_t *)sys_rmdir, AUE_RMDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 137 = rmdir */
 	{ AS(freebsd32_utimes_args), (sy_call_t *)freebsd32_utimes, AUE_UTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 138 = freebsd32_utimes */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 139 = obsolete 4.2 sigreturn */
 	{ AS(freebsd32_adjtime_args), (sy_call_t *)freebsd32_adjtime, AUE_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 140 = freebsd32_adjtime */
@@ -194,8 +194,8 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 144 = obsolete getrlimit */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 145 = obsolete setrlimit */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 146 = obsolete killpg */
-	{ 0, (sy_call_t *)setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 147 = setsid */
-	{ AS(quotactl_args), (sy_call_t *)quotactl, AUE_QUOTACTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 148 = quotactl */
+	{ 0, (sy_call_t *)sys_setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 147 = setsid */
+	{ AS(quotactl_args), (sy_call_t *)sys_quotactl, AUE_QUOTACTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 148 = quotactl */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 149 = obsolete oquota */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 150 = obsolete ogetsockname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 151 = sem_lock */
@@ -208,12 +208,12 @@ struct sysent freebsd32_sysent[] = {
 	{ compat4(AS(freebsd4_freebsd32_fstatfs_args),freebsd32_fstatfs), AUE_FSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = freebsd4 freebsd32_fstatfs */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 159 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 160 = lgetfh */
-	{ AS(getfh_args), (sy_call_t *)getfh, AUE_NFS_GETFH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = getfh */
+	{ AS(getfh_args), (sy_call_t *)sys_getfh, AUE_NFS_GETFH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = getfh */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 162 = obsolete getdomainname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 163 = obsolete setdomainname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 164 = obsolete uname */
 	{ AS(freebsd32_sysarch_args), (sy_call_t *)freebsd32_sysarch, AUE_SYSARCH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 165 = freebsd32_sysarch */
-	{ AS(rtprio_args), (sy_call_t *)rtprio, AUE_RTPRIO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 166 = rtprio */
+	{ AS(rtprio_args), (sy_call_t *)sys_rtprio, AUE_RTPRIO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 166 = rtprio */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 167 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 168 = nosys */
 	{ AS(freebsd32_semsys_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 169 = freebsd32_semsys */
@@ -223,14 +223,14 @@ struct sysent freebsd32_sysent[] = {
 	{ compat6(AS(freebsd6_freebsd32_pread_args),freebsd32_pread), AUE_PREAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 173 = freebsd6 freebsd32_pread */
 	{ compat6(AS(freebsd6_freebsd32_pwrite_args),freebsd32_pwrite), AUE_PWRITE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 174 = freebsd6 freebsd32_pwrite */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 175 = nosys */
-	{ AS(ntp_adjtime_args), (sy_call_t *)ntp_adjtime, AUE_NTP_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 176 = ntp_adjtime */
+	{ AS(ntp_adjtime_args), (sy_call_t *)sys_ntp_adjtime, AUE_NTP_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 176 = ntp_adjtime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 177 = sfork */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 178 = getdescriptor */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 179 = setdescriptor */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 180 = nosys */
-	{ AS(setgid_args), (sy_call_t *)setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 181 = setgid */
-	{ AS(setegid_args), (sy_call_t *)setegid, AUE_SETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 182 = setegid */
-	{ AS(seteuid_args), (sy_call_t *)seteuid, AUE_SETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 183 = seteuid */
+	{ AS(setgid_args), (sy_call_t *)sys_setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 181 = setgid */
+	{ AS(setegid_args), (sy_call_t *)sys_setegid, AUE_SETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 182 = setegid */
+	{ AS(seteuid_args), (sy_call_t *)sys_seteuid, AUE_SETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 183 = seteuid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 184 = lfs_bmapv */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 185 = lfs_markv */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 186 = lfs_segclean */
@@ -238,11 +238,11 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(freebsd32_stat_args), (sy_call_t *)freebsd32_stat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 188 = freebsd32_stat */
 	{ AS(freebsd32_fstat_args), (sy_call_t *)freebsd32_fstat, AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 189 = freebsd32_fstat */
 	{ AS(freebsd32_lstat_args), (sy_call_t *)freebsd32_lstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 190 = freebsd32_lstat */
-	{ AS(pathconf_args), (sy_call_t *)pathconf, AUE_PATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 191 = pathconf */
-	{ AS(fpathconf_args), (sy_call_t *)fpathconf, AUE_FPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 192 = fpathconf */
+	{ AS(pathconf_args), (sy_call_t *)sys_pathconf, AUE_PATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 191 = pathconf */
+	{ AS(fpathconf_args), (sy_call_t *)sys_fpathconf, AUE_FPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 192 = fpathconf */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 193 = nosys */
-	{ AS(__getrlimit_args), (sy_call_t *)getrlimit, AUE_GETRLIMIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 194 = getrlimit */
-	{ AS(__setrlimit_args), (sy_call_t *)setrlimit, AUE_SETRLIMIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 195 = setrlimit */
+	{ AS(__getrlimit_args), (sy_call_t *)sys_getrlimit, AUE_GETRLIMIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 194 = getrlimit */
+	{ AS(__setrlimit_args), (sy_call_t *)sys_setrlimit, AUE_SETRLIMIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 195 = setrlimit */
 	{ AS(freebsd32_getdirentries_args), (sy_call_t *)freebsd32_getdirentries, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 196 = freebsd32_getdirentries */
 	{ compat6(AS(freebsd6_freebsd32_mmap_args),freebsd32_mmap), AUE_MMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 197 = freebsd6 freebsd32_mmap */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 198 = __syscall */
@@ -250,13 +250,13 @@ struct sysent freebsd32_sysent[] = {
 	{ compat6(AS(freebsd6_freebsd32_truncate_args),freebsd32_truncate), AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 200 = freebsd6 freebsd32_truncate */
 	{ compat6(AS(freebsd6_freebsd32_ftruncate_args),freebsd32_ftruncate), AUE_FTRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 201 = freebsd6 freebsd32_ftruncate */
 	{ AS(freebsd32_sysctl_args), (sy_call_t *)freebsd32_sysctl, AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = freebsd32_sysctl */
-	{ AS(mlock_args), (sy_call_t *)mlock, AUE_MLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = mlock */
-	{ AS(munlock_args), (sy_call_t *)munlock, AUE_MUNLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 204 = munlock */
-	{ AS(undelete_args), (sy_call_t *)undelete, AUE_UNDELETE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 205 = undelete */
+	{ AS(mlock_args), (sy_call_t *)sys_mlock, AUE_MLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = mlock */
+	{ AS(munlock_args), (sy_call_t *)sys_munlock, AUE_MUNLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 204 = munlock */
+	{ AS(undelete_args), (sy_call_t *)sys_undelete, AUE_UNDELETE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 205 = undelete */
 	{ AS(freebsd32_futimes_args), (sy_call_t *)freebsd32_futimes, AUE_FUTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 206 = freebsd32_futimes */
-	{ AS(getpgid_args), (sy_call_t *)getpgid, AUE_GETPGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 207 = getpgid */
+	{ AS(getpgid_args), (sy_call_t *)sys_getpgid, AUE_GETPGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 207 = getpgid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 208 = newreboot */
-	{ AS(poll_args), (sy_call_t *)poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 209 = poll */
+	{ AS(poll_args), (sy_call_t *)sys_poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 209 = poll */
 	{ AS(nosys_args), (sy_call_t *)lkmnosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 210 = lkmnosys */
 	{ AS(nosys_args), (sy_call_t *)lkmnosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 211 = lkmnosys */
 	{ AS(nosys_args), (sy_call_t *)lkmnosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 212 = lkmnosys */
@@ -297,11 +297,11 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 247 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 248 = ntp_gettime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 249 = nosys */
-	{ AS(minherit_args), (sy_call_t *)minherit, AUE_MINHERIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 250 = minherit */
-	{ AS(rfork_args), (sy_call_t *)rfork, AUE_RFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 251 = rfork */
-	{ AS(openbsd_poll_args), (sy_call_t *)openbsd_poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 252 = openbsd_poll */
-	{ 0, (sy_call_t *)issetugid, AUE_ISSETUGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 253 = issetugid */
-	{ AS(lchown_args), (sy_call_t *)lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 254 = lchown */
+	{ AS(minherit_args), (sy_call_t *)sys_minherit, AUE_MINHERIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 250 = minherit */
+	{ AS(rfork_args), (sy_call_t *)sys_rfork, AUE_RFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 251 = rfork */
+	{ AS(openbsd_poll_args), (sy_call_t *)sys_openbsd_poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 252 = openbsd_poll */
+	{ 0, (sy_call_t *)sys_issetugid, AUE_ISSETUGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 253 = issetugid */
+	{ AS(lchown_args), (sy_call_t *)sys_lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 254 = lchown */
 	{ AS(freebsd32_aio_read_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 255 = freebsd32_aio_read */
 	{ AS(freebsd32_aio_write_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 256 = freebsd32_aio_write */
 	{ AS(freebsd32_lio_listio_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 257 = freebsd32_lio_listio */
@@ -319,15 +319,15 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 269 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 270 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 271 = nosys */
-	{ AS(getdents_args), (sy_call_t *)getdents, AUE_O_GETDENTS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 272 = getdents */
+	{ AS(getdents_args), (sy_call_t *)sys_getdents, AUE_O_GETDENTS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 272 = getdents */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 273 = nosys */
-	{ AS(lchmod_args), (sy_call_t *)lchmod, AUE_LCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 274 = lchmod */
-	{ AS(lchown_args), (sy_call_t *)lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 275 = netbsd_lchown */
+	{ AS(lchmod_args), (sy_call_t *)sys_lchmod, AUE_LCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 274 = lchmod */
+	{ AS(lchown_args), (sy_call_t *)sys_lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 275 = netbsd_lchown */
 	{ AS(freebsd32_lutimes_args), (sy_call_t *)freebsd32_lutimes, AUE_LUTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 276 = freebsd32_lutimes */
-	{ AS(msync_args), (sy_call_t *)msync, AUE_MSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 277 = netbsd_msync */
-	{ AS(nstat_args), (sy_call_t *)nstat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 278 = nstat */
-	{ AS(nfstat_args), (sy_call_t *)nfstat, AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 279 = nfstat */
-	{ AS(nlstat_args), (sy_call_t *)nlstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 280 = nlstat */
+	{ AS(msync_args), (sy_call_t *)sys_msync, AUE_MSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 277 = netbsd_msync */
+	{ AS(nstat_args), (sy_call_t *)sys_nstat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 278 = nstat */
+	{ AS(nfstat_args), (sy_call_t *)sys_nfstat, AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 279 = nfstat */
+	{ AS(nlstat_args), (sy_call_t *)sys_nlstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 280 = nlstat */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 281 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 282 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 283 = nosys */
@@ -345,21 +345,21 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 295 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 296 = nosys */
 	{ compat4(AS(freebsd4_freebsd32_fhstatfs_args),freebsd32_fhstatfs), AUE_FHSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 297 = freebsd4 freebsd32_fhstatfs */
-	{ AS(fhopen_args), (sy_call_t *)fhopen, AUE_FHOPEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 298 = fhopen */
-	{ AS(fhstat_args), (sy_call_t *)fhstat, AUE_FHSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 299 = fhstat */
-	{ AS(modnext_args), (sy_call_t *)modnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 300 = modnext */
+	{ AS(fhopen_args), (sy_call_t *)sys_fhopen, AUE_FHOPEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 298 = fhopen */
+	{ AS(fhstat_args), (sy_call_t *)sys_fhstat, AUE_FHSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 299 = fhstat */
+	{ AS(modnext_args), (sy_call_t *)sys_modnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 300 = modnext */
 	{ AS(freebsd32_modstat_args), (sy_call_t *)freebsd32_modstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 301 = freebsd32_modstat */
-	{ AS(modfnext_args), (sy_call_t *)modfnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 302 = modfnext */
-	{ AS(modfind_args), (sy_call_t *)modfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 303 = modfind */
-	{ AS(kldload_args), (sy_call_t *)kldload, AUE_MODLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 304 = kldload */
-	{ AS(kldunload_args), (sy_call_t *)kldunload, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 305 = kldunload */
-	{ AS(kldfind_args), (sy_call_t *)kldfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 306 = kldfind */
-	{ AS(kldnext_args), (sy_call_t *)kldnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 307 = kldnext */
+	{ AS(modfnext_args), (sy_call_t *)sys_modfnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 302 = modfnext */
+	{ AS(modfind_args), (sy_call_t *)sys_modfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 303 = modfind */
+	{ AS(kldload_args), (sy_call_t *)sys_kldload, AUE_MODLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 304 = kldload */
+	{ AS(kldunload_args), (sy_call_t *)sys_kldunload, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 305 = kldunload */
+	{ AS(kldfind_args), (sy_call_t *)sys_kldfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 306 = kldfind */
+	{ AS(kldnext_args), (sy_call_t *)sys_kldnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 307 = kldnext */
 	{ AS(freebsd32_kldstat_args), (sy_call_t *)freebsd32_kldstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 308 = freebsd32_kldstat */
-	{ AS(kldfirstmod_args), (sy_call_t *)kldfirstmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 309 = kldfirstmod */
-	{ AS(getsid_args), (sy_call_t *)getsid, AUE_GETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 310 = getsid */
-	{ AS(setresuid_args), (sy_call_t *)setresuid, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 311 = setresuid */
-	{ AS(setresgid_args), (sy_call_t *)setresgid, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 312 = setresgid */
+	{ AS(kldfirstmod_args), (sy_call_t *)sys_kldfirstmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 309 = kldfirstmod */
+	{ AS(getsid_args), (sy_call_t *)sys_getsid, AUE_GETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 310 = getsid */
+	{ AS(setresuid_args), (sy_call_t *)sys_setresuid, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 311 = setresuid */
+	{ AS(setresgid_args), (sy_call_t *)sys_setresgid, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 312 = setresgid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 313 = obsolete signanosleep */
 	{ AS(freebsd32_aio_return_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 314 = freebsd32_aio_return */
 	{ AS(freebsd32_aio_suspend_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 315 = freebsd32_aio_suspend */
@@ -368,48 +368,48 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(freebsd32_oaio_read_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 318 = freebsd32_oaio_read */
 	{ AS(freebsd32_oaio_write_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 319 = freebsd32_oaio_write */
 	{ AS(freebsd32_olio_listio_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 320 = freebsd32_olio_listio */
-	{ 0, (sy_call_t *)yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 321 = yield */
+	{ 0, (sy_call_t *)sys_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 321 = yield */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 322 = obsolete thr_sleep */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 323 = obsolete thr_wakeup */
-	{ AS(mlockall_args), (sy_call_t *)mlockall, AUE_MLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 324 = mlockall */
-	{ 0, (sy_call_t *)munlockall, AUE_MUNLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 325 = munlockall */
-	{ AS(__getcwd_args), (sy_call_t *)__getcwd, AUE_GETCWD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 326 = __getcwd */
-	{ AS(sched_setparam_args), (sy_call_t *)sched_setparam, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 327 = sched_setparam */
-	{ AS(sched_getparam_args), (sy_call_t *)sched_getparam, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 328 = sched_getparam */
-	{ AS(sched_setscheduler_args), (sy_call_t *)sched_setscheduler, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 329 = sched_setscheduler */
-	{ AS(sched_getscheduler_args), (sy_call_t *)sched_getscheduler, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 330 = sched_getscheduler */
-	{ 0, (sy_call_t *)sched_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 331 = sched_yield */
-	{ AS(sched_get_priority_max_args), (sy_call_t *)sched_get_priority_max, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 332 = sched_get_priority_max */
-	{ AS(sched_get_priority_min_args), (sy_call_t *)sched_get_priority_min, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 333 = sched_get_priority_min */
-	{ AS(sched_rr_get_interval_args), (sy_call_t *)sched_rr_get_interval, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 334 = sched_rr_get_interval */
-	{ AS(utrace_args), (sy_call_t *)utrace, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 335 = utrace */
+	{ AS(mlockall_args), (sy_call_t *)sys_mlockall, AUE_MLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 324 = mlockall */
+	{ 0, (sy_call_t *)sys_munlockall, AUE_MUNLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 325 = munlockall */
+	{ AS(__getcwd_args), (sy_call_t *)sys___getcwd, AUE_GETCWD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 326 = __getcwd */
+	{ AS(sched_setparam_args), (sy_call_t *)sys_sched_setparam, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 327 = sched_setparam */
+	{ AS(sched_getparam_args), (sy_call_t *)sys_sched_getparam, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 328 = sched_getparam */
+	{ AS(sched_setscheduler_args), (sy_call_t *)sys_sched_setscheduler, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 329 = sched_setscheduler */
+	{ AS(sched_getscheduler_args), (sy_call_t *)sys_sched_getscheduler, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 330 = sched_getscheduler */
+	{ 0, (sy_call_t *)sys_sched_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 331 = sched_yield */
+	{ AS(sched_get_priority_max_args), (sy_call_t *)sys_sched_get_priority_max, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 332 = sched_get_priority_max */
+	{ AS(sched_get_priority_min_args), (sy_call_t *)sys_sched_get_priority_min, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 333 = sched_get_priority_min */
+	{ AS(sched_rr_get_interval_args), (sy_call_t *)sys_sched_rr_get_interval, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 334 = sched_rr_get_interval */
+	{ AS(utrace_args), (sy_call_t *)sys_utrace, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 335 = utrace */
 	{ compat4(AS(freebsd4_freebsd32_sendfile_args),freebsd32_sendfile), AUE_SENDFILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 336 = freebsd4 freebsd32_sendfile */
-	{ AS(kldsym_args), (sy_call_t *)kldsym, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 337 = kldsym */
+	{ AS(kldsym_args), (sy_call_t *)sys_kldsym, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 337 = kldsym */
 	{ AS(freebsd32_jail_args), (sy_call_t *)freebsd32_jail, AUE_JAIL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 338 = freebsd32_jail */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 339 = pioctl */
-	{ AS(sigprocmask_args), (sy_call_t *)sigprocmask, AUE_SIGPROCMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 340 = sigprocmask */
-	{ AS(sigsuspend_args), (sy_call_t *)sigsuspend, AUE_SIGSUSPEND, NULL, 0, 0, 0, SY_THR_STATIC },	/* 341 = sigsuspend */
+	{ AS(sigprocmask_args), (sy_call_t *)sys_sigprocmask, AUE_SIGPROCMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 340 = sigprocmask */
+	{ AS(sigsuspend_args), (sy_call_t *)sys_sigsuspend, AUE_SIGSUSPEND, NULL, 0, 0, 0, SY_THR_STATIC },	/* 341 = sigsuspend */
 	{ compat4(AS(freebsd4_freebsd32_sigaction_args),freebsd32_sigaction), AUE_SIGACTION, NULL, 0, 0, 0, SY_THR_STATIC },	/* 342 = freebsd4 freebsd32_sigaction */
-	{ AS(sigpending_args), (sy_call_t *)sigpending, AUE_SIGPENDING, NULL, 0, 0, 0, SY_THR_STATIC },	/* 343 = sigpending */
+	{ AS(sigpending_args), (sy_call_t *)sys_sigpending, AUE_SIGPENDING, NULL, 0, 0, 0, SY_THR_STATIC },	/* 343 = sigpending */
 	{ compat4(AS(freebsd4_freebsd32_sigreturn_args),freebsd32_sigreturn), AUE_SIGRETURN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 344 = freebsd4 freebsd32_sigreturn */
 	{ AS(freebsd32_sigtimedwait_args), (sy_call_t *)freebsd32_sigtimedwait, AUE_SIGWAIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 345 = freebsd32_sigtimedwait */
 	{ AS(freebsd32_sigwaitinfo_args), (sy_call_t *)freebsd32_sigwaitinfo, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 346 = freebsd32_sigwaitinfo */
-	{ AS(__acl_get_file_args), (sy_call_t *)__acl_get_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 347 = __acl_get_file */
-	{ AS(__acl_set_file_args), (sy_call_t *)__acl_set_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 348 = __acl_set_file */
-	{ AS(__acl_get_fd_args), (sy_call_t *)__acl_get_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 349 = __acl_get_fd */
-	{ AS(__acl_set_fd_args), (sy_call_t *)__acl_set_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 350 = __acl_set_fd */
-	{ AS(__acl_delete_file_args), (sy_call_t *)__acl_delete_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 351 = __acl_delete_file */
-	{ AS(__acl_delete_fd_args), (sy_call_t *)__acl_delete_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 352 = __acl_delete_fd */
-	{ AS(__acl_aclcheck_file_args), (sy_call_t *)__acl_aclcheck_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 353 = __acl_aclcheck_file */
-	{ AS(__acl_aclcheck_fd_args), (sy_call_t *)__acl_aclcheck_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 354 = __acl_aclcheck_fd */
-	{ AS(extattrctl_args), (sy_call_t *)extattrctl, AUE_EXTATTRCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 355 = extattrctl */
-	{ AS(extattr_set_file_args), (sy_call_t *)extattr_set_file, AUE_EXTATTR_SET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 356 = extattr_set_file */
-	{ AS(extattr_get_file_args), (sy_call_t *)extattr_get_file, AUE_EXTATTR_GET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 357 = extattr_get_file */
-	{ AS(extattr_delete_file_args), (sy_call_t *)extattr_delete_file, AUE_EXTATTR_DELETE_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 358 = extattr_delete_file */
+	{ AS(__acl_get_file_args), (sy_call_t *)sys___acl_get_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 347 = __acl_get_file */
+	{ AS(__acl_set_file_args), (sy_call_t *)sys___acl_set_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 348 = __acl_set_file */
+	{ AS(__acl_get_fd_args), (sy_call_t *)sys___acl_get_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 349 = __acl_get_fd */
+	{ AS(__acl_set_fd_args), (sy_call_t *)sys___acl_set_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 350 = __acl_set_fd */
+	{ AS(__acl_delete_file_args), (sy_call_t *)sys___acl_delete_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 351 = __acl_delete_file */
+	{ AS(__acl_delete_fd_args), (sy_call_t *)sys___acl_delete_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 352 = __acl_delete_fd */
+	{ AS(__acl_aclcheck_file_args), (sy_call_t *)sys___acl_aclcheck_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 353 = __acl_aclcheck_file */
+	{ AS(__acl_aclcheck_fd_args), (sy_call_t *)sys___acl_aclcheck_fd, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 354 = __acl_aclcheck_fd */
+	{ AS(extattrctl_args), (sy_call_t *)sys_extattrctl, AUE_EXTATTRCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 355 = extattrctl */
+	{ AS(extattr_set_file_args), (sy_call_t *)sys_extattr_set_file, AUE_EXTATTR_SET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 356 = extattr_set_file */
+	{ AS(extattr_get_file_args), (sy_call_t *)sys_extattr_get_file, AUE_EXTATTR_GET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 357 = extattr_get_file */
+	{ AS(extattr_delete_file_args), (sy_call_t *)sys_extattr_delete_file, AUE_EXTATTR_DELETE_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 358 = extattr_delete_file */
 	{ AS(freebsd32_aio_waitcomplete_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 359 = freebsd32_aio_waitcomplete */
-	{ AS(getresuid_args), (sy_call_t *)getresuid, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 360 = getresuid */
-	{ AS(getresgid_args), (sy_call_t *)getresgid, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 361 = getresgid */
-	{ 0, (sy_call_t *)kqueue, AUE_KQUEUE, NULL, 0, 0, 0, SY_THR_STATIC },		/* 362 = kqueue */
+	{ AS(getresuid_args), (sy_call_t *)sys_getresuid, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 360 = getresuid */
+	{ AS(getresgid_args), (sy_call_t *)sys_getresgid, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 361 = getresgid */
+	{ 0, (sy_call_t *)sys_kqueue, AUE_KQUEUE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 362 = kqueue */
 	{ AS(freebsd32_kevent_args), (sy_call_t *)freebsd32_kevent, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 363 = freebsd32_kevent */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 364 = __cap_get_proc */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 365 = __cap_set_proc */
@@ -418,12 +418,12 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 368 = __cap_set_fd */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 369 = __cap_set_file */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 370 = nosys */
-	{ AS(extattr_set_fd_args), (sy_call_t *)extattr_set_fd, AUE_EXTATTR_SET_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 371 = extattr_set_fd */
-	{ AS(extattr_get_fd_args), (sy_call_t *)extattr_get_fd, AUE_EXTATTR_GET_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 372 = extattr_get_fd */
-	{ AS(extattr_delete_fd_args), (sy_call_t *)extattr_delete_fd, AUE_EXTATTR_DELETE_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 373 = extattr_delete_fd */
-	{ AS(__setugid_args), (sy_call_t *)__setugid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 374 = __setugid */
+	{ AS(extattr_set_fd_args), (sy_call_t *)sys_extattr_set_fd, AUE_EXTATTR_SET_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 371 = extattr_set_fd */
+	{ AS(extattr_get_fd_args), (sy_call_t *)sys_extattr_get_fd, AUE_EXTATTR_GET_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 372 = extattr_get_fd */
+	{ AS(extattr_delete_fd_args), (sy_call_t *)sys_extattr_delete_fd, AUE_EXTATTR_DELETE_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 373 = extattr_delete_fd */
+	{ AS(__setugid_args), (sy_call_t *)sys___setugid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 374 = __setugid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 375 = nfsclnt */
-	{ AS(eaccess_args), (sy_call_t *)eaccess, AUE_EACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 376 = eaccess */
+	{ AS(eaccess_args), (sy_call_t *)sys_eaccess, AUE_EACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 376 = eaccess */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 377 = afs_syscall */
 	{ AS(freebsd32_nmount_args), (sy_call_t *)freebsd32_nmount, AUE_NMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 378 = freebsd32_nmount */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 379 = kse_exit */
@@ -437,15 +437,15 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 387 = __mac_get_file */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 388 = __mac_set_fd */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 389 = __mac_set_file */
-	{ AS(kenv_args), (sy_call_t *)kenv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 390 = kenv */
-	{ AS(lchflags_args), (sy_call_t *)lchflags, AUE_LCHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 391 = lchflags */
-	{ AS(uuidgen_args), (sy_call_t *)uuidgen, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 392 = uuidgen */
+	{ AS(kenv_args), (sy_call_t *)sys_kenv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 390 = kenv */
+	{ AS(lchflags_args), (sy_call_t *)sys_lchflags, AUE_LCHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 391 = lchflags */
+	{ AS(uuidgen_args), (sy_call_t *)sys_uuidgen, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 392 = uuidgen */
 	{ AS(freebsd32_sendfile_args), (sy_call_t *)freebsd32_sendfile, AUE_SENDFILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 393 = freebsd32_sendfile */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 394 = mac_syscall */
-	{ AS(getfsstat_args), (sy_call_t *)getfsstat, AUE_GETFSSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 395 = getfsstat */
-	{ AS(statfs_args), (sy_call_t *)statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 396 = statfs */
-	{ AS(fstatfs_args), (sy_call_t *)fstatfs, AUE_FSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 397 = fstatfs */
-	{ AS(fhstatfs_args), (sy_call_t *)fhstatfs, AUE_FHSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 398 = fhstatfs */
+	{ AS(getfsstat_args), (sy_call_t *)sys_getfsstat, AUE_GETFSSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 395 = getfsstat */
+	{ AS(statfs_args), (sy_call_t *)sys_statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 396 = statfs */
+	{ AS(fstatfs_args), (sy_call_t *)sys_fstatfs, AUE_FSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 397 = fstatfs */
+	{ AS(fhstatfs_args), (sy_call_t *)sys_fhstatfs, AUE_FHSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 398 = fhstatfs */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 399 = nosys */
 	{ AS(ksem_close_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 400 = ksem_close */
 	{ AS(ksem_post_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 401 = ksem_post */
@@ -459,9 +459,9 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 409 = __mac_get_pid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 410 = __mac_get_link */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 411 = __mac_set_link */
-	{ AS(extattr_set_link_args), (sy_call_t *)extattr_set_link, AUE_EXTATTR_SET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 412 = extattr_set_link */
-	{ AS(extattr_get_link_args), (sy_call_t *)extattr_get_link, AUE_EXTATTR_GET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 413 = extattr_get_link */
-	{ AS(extattr_delete_link_args), (sy_call_t *)extattr_delete_link, AUE_EXTATTR_DELETE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 414 = extattr_delete_link */
+	{ AS(extattr_set_link_args), (sy_call_t *)sys_extattr_set_link, AUE_EXTATTR_SET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 412 = extattr_set_link */
+	{ AS(extattr_get_link_args), (sy_call_t *)sys_extattr_get_link, AUE_EXTATTR_GET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 413 = extattr_get_link */
+	{ AS(extattr_delete_link_args), (sy_call_t *)sys_extattr_delete_link, AUE_EXTATTR_DELETE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 414 = extattr_delete_link */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 415 = __mac_execve */
 	{ AS(freebsd32_sigaction_args), (sy_call_t *)freebsd32_sigaction, AUE_SIGACTION, NULL, 0, 0, 0, SY_THR_STATIC },	/* 416 = freebsd32_sigaction */
 	{ AS(freebsd32_sigreturn_args), (sy_call_t *)freebsd32_sigreturn, AUE_SIGRETURN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 417 = freebsd32_sigreturn */
@@ -472,56 +472,56 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(freebsd32_setcontext_args), (sy_call_t *)freebsd32_setcontext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 422 = freebsd32_setcontext */
 	{ AS(freebsd32_swapcontext_args), (sy_call_t *)freebsd32_swapcontext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 423 = freebsd32_swapcontext */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 424 = swapoff */
-	{ AS(__acl_get_link_args), (sy_call_t *)__acl_get_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 425 = __acl_get_link */
-	{ AS(__acl_set_link_args), (sy_call_t *)__acl_set_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 426 = __acl_set_link */
-	{ AS(__acl_delete_link_args), (sy_call_t *)__acl_delete_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 427 = __acl_delete_link */
-	{ AS(__acl_aclcheck_link_args), (sy_call_t *)__acl_aclcheck_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 428 = __acl_aclcheck_link */
-	{ AS(sigwait_args), (sy_call_t *)sigwait, AUE_SIGWAIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 429 = sigwait */
+	{ AS(__acl_get_link_args), (sy_call_t *)sys___acl_get_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 425 = __acl_get_link */
+	{ AS(__acl_set_link_args), (sy_call_t *)sys___acl_set_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 426 = __acl_set_link */
+	{ AS(__acl_delete_link_args), (sy_call_t *)sys___acl_delete_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 427 = __acl_delete_link */
+	{ AS(__acl_aclcheck_link_args), (sy_call_t *)sys___acl_aclcheck_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 428 = __acl_aclcheck_link */
+	{ AS(sigwait_args), (sy_call_t *)sys_sigwait, AUE_SIGWAIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 429 = sigwait */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 430 = thr_create; */
-	{ AS(thr_exit_args), (sy_call_t *)thr_exit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 431 = thr_exit */
-	{ AS(thr_self_args), (sy_call_t *)thr_self, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 432 = thr_self */
-	{ AS(thr_kill_args), (sy_call_t *)thr_kill, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 433 = thr_kill */
+	{ AS(thr_exit_args), (sy_call_t *)sys_thr_exit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 431 = thr_exit */
+	{ AS(thr_self_args), (sy_call_t *)sys_thr_self, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 432 = thr_self */
+	{ AS(thr_kill_args), (sy_call_t *)sys_thr_kill, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 433 = thr_kill */
 	{ AS(freebsd32_umtx_lock_args), (sy_call_t *)freebsd32_umtx_lock, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 434 = freebsd32_umtx_lock */
 	{ AS(freebsd32_umtx_unlock_args), (sy_call_t *)freebsd32_umtx_unlock, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 435 = freebsd32_umtx_unlock */
-	{ AS(jail_attach_args), (sy_call_t *)jail_attach, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 436 = jail_attach */
-	{ AS(extattr_list_fd_args), (sy_call_t *)extattr_list_fd, AUE_EXTATTR_LIST_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 437 = extattr_list_fd */
-	{ AS(extattr_list_file_args), (sy_call_t *)extattr_list_file, AUE_EXTATTR_LIST_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 438 = extattr_list_file */
-	{ AS(extattr_list_link_args), (sy_call_t *)extattr_list_link, AUE_EXTATTR_LIST_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 439 = extattr_list_link */
+	{ AS(jail_attach_args), (sy_call_t *)sys_jail_attach, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 436 = jail_attach */
+	{ AS(extattr_list_fd_args), (sy_call_t *)sys_extattr_list_fd, AUE_EXTATTR_LIST_FD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 437 = extattr_list_fd */
+	{ AS(extattr_list_file_args), (sy_call_t *)sys_extattr_list_file, AUE_EXTATTR_LIST_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 438 = extattr_list_file */
+	{ AS(extattr_list_link_args), (sy_call_t *)sys_extattr_list_link, AUE_EXTATTR_LIST_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 439 = extattr_list_link */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 440 = kse_switchin */
 	{ AS(freebsd32_ksem_timedwait_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 441 = freebsd32_ksem_timedwait */
 	{ AS(freebsd32_thr_suspend_args), (sy_call_t *)freebsd32_thr_suspend, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 442 = freebsd32_thr_suspend */
-	{ AS(thr_wake_args), (sy_call_t *)thr_wake, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 443 = thr_wake */
-	{ AS(kldunloadf_args), (sy_call_t *)kldunloadf, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 444 = kldunloadf */
-	{ AS(audit_args), (sy_call_t *)audit, AUE_AUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 445 = audit */
-	{ AS(auditon_args), (sy_call_t *)auditon, AUE_AUDITON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 446 = auditon */
-	{ AS(getauid_args), (sy_call_t *)getauid, AUE_GETAUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 447 = getauid */
-	{ AS(setauid_args), (sy_call_t *)setauid, AUE_SETAUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 448 = setauid */
-	{ AS(getaudit_args), (sy_call_t *)getaudit, AUE_GETAUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 449 = getaudit */
-	{ AS(setaudit_args), (sy_call_t *)setaudit, AUE_SETAUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 450 = setaudit */
-	{ AS(getaudit_addr_args), (sy_call_t *)getaudit_addr, AUE_GETAUDIT_ADDR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 451 = getaudit_addr */
-	{ AS(setaudit_addr_args), (sy_call_t *)setaudit_addr, AUE_SETAUDIT_ADDR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 452 = setaudit_addr */
-	{ AS(auditctl_args), (sy_call_t *)auditctl, AUE_AUDITCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 453 = auditctl */
+	{ AS(thr_wake_args), (sy_call_t *)sys_thr_wake, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 443 = thr_wake */
+	{ AS(kldunloadf_args), (sy_call_t *)sys_kldunloadf, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 444 = kldunloadf */
+	{ AS(audit_args), (sy_call_t *)sys_audit, AUE_AUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 445 = audit */
+	{ AS(auditon_args), (sy_call_t *)sys_auditon, AUE_AUDITON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 446 = auditon */
+	{ AS(getauid_args), (sy_call_t *)sys_getauid, AUE_GETAUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 447 = getauid */
+	{ AS(setauid_args), (sy_call_t *)sys_setauid, AUE_SETAUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 448 = setauid */
+	{ AS(getaudit_args), (sy_call_t *)sys_getaudit, AUE_GETAUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 449 = getaudit */
+	{ AS(setaudit_args), (sy_call_t *)sys_setaudit, AUE_SETAUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 450 = setaudit */
+	{ AS(getaudit_addr_args), (sy_call_t *)sys_getaudit_addr, AUE_GETAUDIT_ADDR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 451 = getaudit_addr */
+	{ AS(setaudit_addr_args), (sy_call_t *)sys_setaudit_addr, AUE_SETAUDIT_ADDR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 452 = setaudit_addr */
+	{ AS(auditctl_args), (sy_call_t *)sys_auditctl, AUE_AUDITCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 453 = auditctl */
 	{ AS(freebsd32_umtx_op_args), (sy_call_t *)freebsd32_umtx_op, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 454 = freebsd32_umtx_op */
 	{ AS(freebsd32_thr_new_args), (sy_call_t *)freebsd32_thr_new, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 455 = freebsd32_thr_new */
-	{ AS(sigqueue_args), (sy_call_t *)sigqueue, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 456 = sigqueue */
+	{ AS(sigqueue_args), (sy_call_t *)sys_sigqueue, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 456 = sigqueue */
 	{ AS(freebsd32_kmq_open_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 457 = freebsd32_kmq_open */
 	{ AS(freebsd32_kmq_setattr_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 458 = freebsd32_kmq_setattr */
 	{ AS(freebsd32_kmq_timedreceive_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 459 = freebsd32_kmq_timedreceive */
 	{ AS(freebsd32_kmq_timedsend_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 460 = freebsd32_kmq_timedsend */
 	{ AS(kmq_notify_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 461 = kmq_notify */
 	{ AS(kmq_unlink_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 462 = kmq_unlink */
-	{ AS(abort2_args), (sy_call_t *)abort2, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 463 = abort2 */
-	{ AS(thr_set_name_args), (sy_call_t *)thr_set_name, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 464 = thr_set_name */
+	{ AS(abort2_args), (sy_call_t *)sys_abort2, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 463 = abort2 */
+	{ AS(thr_set_name_args), (sy_call_t *)sys_thr_set_name, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 464 = thr_set_name */
 	{ AS(freebsd32_aio_fsync_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 465 = freebsd32_aio_fsync */
-	{ AS(rtprio_thread_args), (sy_call_t *)rtprio_thread, AUE_RTPRIO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 466 = rtprio_thread */
+	{ AS(rtprio_thread_args), (sy_call_t *)sys_rtprio_thread, AUE_RTPRIO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 466 = rtprio_thread */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 467 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 468 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 469 = __getpath_fromfd */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 470 = __getpath_fromaddr */
-	{ AS(sctp_peeloff_args), (sy_call_t *)sctp_peeloff, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 471 = sctp_peeloff */
-	{ AS(sctp_generic_sendmsg_args), (sy_call_t *)sctp_generic_sendmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 472 = sctp_generic_sendmsg */
-	{ AS(sctp_generic_sendmsg_iov_args), (sy_call_t *)sctp_generic_sendmsg_iov, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 473 = sctp_generic_sendmsg_iov */
-	{ AS(sctp_generic_recvmsg_args), (sy_call_t *)sctp_generic_recvmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 474 = sctp_generic_recvmsg */
+	{ AS(sctp_peeloff_args), (sy_call_t *)sys_sctp_peeloff, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 471 = sctp_peeloff */
+	{ AS(sctp_generic_sendmsg_args), (sy_call_t *)sys_sctp_generic_sendmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 472 = sctp_generic_sendmsg */
+	{ AS(sctp_generic_sendmsg_iov_args), (sy_call_t *)sys_sctp_generic_sendmsg_iov, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 473 = sctp_generic_sendmsg_iov */
+	{ AS(sctp_generic_recvmsg_args), (sy_call_t *)sys_sctp_generic_recvmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 474 = sctp_generic_recvmsg */
 #ifdef PAD64_REQUIRED
 	{ AS(freebsd32_pread_args), (sy_call_t *)freebsd32_pread, AUE_PREAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 475 = freebsd32_pread */
 	{ AS(freebsd32_pwrite_args), (sy_call_t *)freebsd32_pwrite, AUE_PWRITE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 476 = freebsd32_pwrite */
@@ -537,10 +537,10 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(freebsd32_truncate_args), (sy_call_t *)freebsd32_truncate, AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 479 = freebsd32_truncate */
 	{ AS(freebsd32_ftruncate_args), (sy_call_t *)freebsd32_ftruncate, AUE_FTRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 480 = freebsd32_ftruncate */
 #endif
-	{ AS(thr_kill2_args), (sy_call_t *)thr_kill2, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 481 = thr_kill2 */
-	{ AS(shm_open_args), (sy_call_t *)shm_open, AUE_SHMOPEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 482 = shm_open */
-	{ AS(shm_unlink_args), (sy_call_t *)shm_unlink, AUE_SHMUNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 483 = shm_unlink */
-	{ AS(cpuset_args), (sy_call_t *)cpuset, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 484 = cpuset */
+	{ AS(thr_kill2_args), (sy_call_t *)sys_thr_kill2, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 481 = thr_kill2 */
+	{ AS(shm_open_args), (sy_call_t *)sys_shm_open, AUE_SHMOPEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 482 = shm_open */
+	{ AS(shm_unlink_args), (sy_call_t *)sys_shm_unlink, AUE_SHMUNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 483 = shm_unlink */
+	{ AS(cpuset_args), (sy_call_t *)sys_cpuset, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 484 = cpuset */
 #ifdef PAD64_REQUIRED
 	{ AS(freebsd32_cpuset_setid_args), (sy_call_t *)freebsd32_cpuset_setid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 485 = freebsd32_cpuset_setid */
 #else
@@ -549,47 +549,47 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(freebsd32_cpuset_getid_args), (sy_call_t *)freebsd32_cpuset_getid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 486 = freebsd32_cpuset_getid */
 	{ AS(freebsd32_cpuset_getaffinity_args), (sy_call_t *)freebsd32_cpuset_getaffinity, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 487 = freebsd32_cpuset_getaffinity */
 	{ AS(freebsd32_cpuset_setaffinity_args), (sy_call_t *)freebsd32_cpuset_setaffinity, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 488 = freebsd32_cpuset_setaffinity */
-	{ AS(faccessat_args), (sy_call_t *)faccessat, AUE_FACCESSAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 489 = faccessat */
-	{ AS(fchmodat_args), (sy_call_t *)fchmodat, AUE_FCHMODAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 490 = fchmodat */
-	{ AS(fchownat_args), (sy_call_t *)fchownat, AUE_FCHOWNAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 491 = fchownat */
+	{ AS(faccessat_args), (sy_call_t *)sys_faccessat, AUE_FACCESSAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 489 = faccessat */
+	{ AS(fchmodat_args), (sy_call_t *)sys_fchmodat, AUE_FCHMODAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 490 = fchmodat */
+	{ AS(fchownat_args), (sy_call_t *)sys_fchownat, AUE_FCHOWNAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 491 = fchownat */
 	{ AS(freebsd32_fexecve_args), (sy_call_t *)freebsd32_fexecve, AUE_FEXECVE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 492 = freebsd32_fexecve */
 	{ AS(freebsd32_fstatat_args), (sy_call_t *)freebsd32_fstatat, AUE_FSTATAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 493 = freebsd32_fstatat */
 	{ AS(freebsd32_futimesat_args), (sy_call_t *)freebsd32_futimesat, AUE_FUTIMESAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 494 = freebsd32_futimesat */
-	{ AS(linkat_args), (sy_call_t *)linkat, AUE_LINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 495 = linkat */
-	{ AS(mkdirat_args), (sy_call_t *)mkdirat, AUE_MKDIRAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 496 = mkdirat */
-	{ AS(mkfifoat_args), (sy_call_t *)mkfifoat, AUE_MKFIFOAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 497 = mkfifoat */
-	{ AS(mknodat_args), (sy_call_t *)mknodat, AUE_MKNODAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 498 = mknodat */
-	{ AS(openat_args), (sy_call_t *)openat, AUE_OPENAT_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 499 = openat */
-	{ AS(readlinkat_args), (sy_call_t *)readlinkat, AUE_READLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 500 = readlinkat */
-	{ AS(renameat_args), (sy_call_t *)renameat, AUE_RENAMEAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 501 = renameat */
-	{ AS(symlinkat_args), (sy_call_t *)symlinkat, AUE_SYMLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 502 = symlinkat */
-	{ AS(unlinkat_args), (sy_call_t *)unlinkat, AUE_UNLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 503 = unlinkat */
-	{ AS(posix_openpt_args), (sy_call_t *)posix_openpt, AUE_POSIX_OPENPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 504 = posix_openpt */
+	{ AS(linkat_args), (sy_call_t *)sys_linkat, AUE_LINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 495 = linkat */
+	{ AS(mkdirat_args), (sy_call_t *)sys_mkdirat, AUE_MKDIRAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 496 = mkdirat */
+	{ AS(mkfifoat_args), (sy_call_t *)sys_mkfifoat, AUE_MKFIFOAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 497 = mkfifoat */
+	{ AS(mknodat_args), (sy_call_t *)sys_mknodat, AUE_MKNODAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 498 = mknodat */
+	{ AS(openat_args), (sy_call_t *)sys_openat, AUE_OPENAT_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 499 = openat */
+	{ AS(readlinkat_args), (sy_call_t *)sys_readlinkat, AUE_READLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 500 = readlinkat */
+	{ AS(renameat_args), (sy_call_t *)sys_renameat, AUE_RENAMEAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 501 = renameat */
+	{ AS(symlinkat_args), (sy_call_t *)sys_symlinkat, AUE_SYMLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 502 = symlinkat */
+	{ AS(unlinkat_args), (sy_call_t *)sys_unlinkat, AUE_UNLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 503 = unlinkat */
+	{ AS(posix_openpt_args), (sy_call_t *)sys_posix_openpt, AUE_POSIX_OPENPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 504 = posix_openpt */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 505 = gssd_syscall */
 	{ AS(freebsd32_jail_get_args), (sy_call_t *)freebsd32_jail_get, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 506 = freebsd32_jail_get */
 	{ AS(freebsd32_jail_set_args), (sy_call_t *)freebsd32_jail_set, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 507 = freebsd32_jail_set */
-	{ AS(jail_remove_args), (sy_call_t *)jail_remove, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 508 = jail_remove */
-	{ AS(closefrom_args), (sy_call_t *)closefrom, AUE_CLOSEFROM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 509 = closefrom */
+	{ AS(jail_remove_args), (sy_call_t *)sys_jail_remove, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 508 = jail_remove */
+	{ AS(closefrom_args), (sy_call_t *)sys_closefrom, AUE_CLOSEFROM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 509 = closefrom */
 	{ AS(freebsd32_semctl_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 510 = freebsd32_semctl */
 	{ AS(freebsd32_msgctl_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 511 = freebsd32_msgctl */
 	{ AS(freebsd32_shmctl_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 512 = freebsd32_shmctl */
-	{ AS(lpathconf_args), (sy_call_t *)lpathconf, AUE_LPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 513 = lpathconf */
-	{ AS(cap_new_args), (sy_call_t *)cap_new, AUE_CAP_NEW, NULL, 0, 0, 0, SY_THR_STATIC },	/* 514 = cap_new */
-	{ AS(cap_getrights_args), (sy_call_t *)cap_getrights, AUE_CAP_GETRIGHTS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 515 = cap_getrights */
-	{ 0, (sy_call_t *)cap_enter, AUE_CAP_ENTER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 516 = cap_enter */
-	{ AS(cap_getmode_args), (sy_call_t *)cap_getmode, AUE_CAP_GETMODE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 517 = cap_getmode */
+	{ AS(lpathconf_args), (sy_call_t *)sys_lpathconf, AUE_LPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 513 = lpathconf */
+	{ AS(cap_new_args), (sy_call_t *)sys_cap_new, AUE_CAP_NEW, NULL, 0, 0, 0, SY_THR_STATIC },	/* 514 = cap_new */
+	{ AS(cap_getrights_args), (sy_call_t *)sys_cap_getrights, AUE_CAP_GETRIGHTS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 515 = cap_getrights */
+	{ 0, (sy_call_t *)sys_cap_enter, AUE_CAP_ENTER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 516 = cap_enter */
+	{ AS(cap_getmode_args), (sy_call_t *)sys_cap_getmode, AUE_CAP_GETMODE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 517 = cap_getmode */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 518 = pdfork */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 519 = pdkill */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 520 = pdgetpid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 521 = pdwait */
 	{ AS(freebsd32_pselect_args), (sy_call_t *)freebsd32_pselect, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 522 = freebsd32_pselect */
-	{ AS(getloginclass_args), (sy_call_t *)getloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 523 = getloginclass */
-	{ AS(setloginclass_args), (sy_call_t *)setloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 524 = setloginclass */
-	{ AS(rctl_get_racct_args), (sy_call_t *)rctl_get_racct, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 525 = rctl_get_racct */
-	{ AS(rctl_get_rules_args), (sy_call_t *)rctl_get_rules, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 526 = rctl_get_rules */
-	{ AS(rctl_get_limits_args), (sy_call_t *)rctl_get_limits, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 527 = rctl_get_limits */
-	{ AS(rctl_add_rule_args), (sy_call_t *)rctl_add_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 528 = rctl_add_rule */
-	{ AS(rctl_remove_rule_args), (sy_call_t *)rctl_remove_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 529 = rctl_remove_rule */
+	{ AS(getloginclass_args), (sy_call_t *)sys_getloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 523 = getloginclass */
+	{ AS(setloginclass_args), (sy_call_t *)sys_setloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 524 = setloginclass */
+	{ AS(rctl_get_racct_args), (sy_call_t *)sys_rctl_get_racct, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 525 = rctl_get_racct */
+	{ AS(rctl_get_rules_args), (sy_call_t *)sys_rctl_get_rules, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 526 = rctl_get_rules */
+	{ AS(rctl_get_limits_args), (sy_call_t *)sys_rctl_get_limits, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 527 = rctl_get_limits */
+	{ AS(rctl_add_rule_args), (sy_call_t *)sys_rctl_add_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 528 = rctl_add_rule */
+	{ AS(rctl_remove_rule_args), (sy_call_t *)sys_rctl_remove_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 529 = rctl_remove_rule */
 	{ AS(freebsd32_posix_fallocate_args), (sy_call_t *)freebsd32_posix_fallocate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 530 = freebsd32_posix_fallocate */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 531 = posix_fadvise */
 };
diff --git a/sys/compat/freebsd32/freebsd32_systrace_args.c b/sys/compat/freebsd32/freebsd32_systrace_args.c
index 39b66fd92b11..a858b27af30f 100644
--- a/sys/compat/freebsd32/freebsd32_systrace_args.c
+++ b/sys/compat/freebsd32/freebsd32_systrace_args.c
@@ -464,9 +464,9 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 2;
 		break;
 	}
-	/* mprotect */
+	/* freebsd32_mprotect */
 	case 74: {
-		struct mprotect_args *p = params;
+		struct freebsd32_mprotect_args *p = params;
 		uarg[0] = (intptr_t) p->addr; /* const void * */
 		uarg[1] = p->len; /* size_t */
 		iarg[2] = p->prot; /* int */
@@ -3027,10 +3027,10 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 530: {
 		struct freebsd32_posix_fallocate_args *p = params;
 		iarg[0] = p->fd; /* int */
-		uarg[1] = p->offsetlo; /* uint32_t */
-		uarg[2] = p->offsethi; /* uint32_t */
-		uarg[3] = p->lenlo; /* uint32_t */
-		uarg[4] = p->lenhi; /* uint32_t */
+		uarg[1] = p->offset1; /* uint32_t */
+		uarg[2] = p->offset2; /* uint32_t */
+		uarg[3] = p->len1; /* uint32_t */
+		uarg[4] = p->len2; /* uint32_t */
 		*n_args = 5;
 		break;
 	}
@@ -3737,7 +3737,7 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
-	/* mprotect */
+	/* freebsd32_mprotect */
 	case 74:
 		switch(ndx) {
 		case 0:
diff --git a/sys/compat/freebsd32/freebsd32_util.h b/sys/compat/freebsd32/freebsd32_util.h
index 0fed7b0272b8..928227513685 100644
--- a/sys/compat/freebsd32/freebsd32_util.h
+++ b/sys/compat/freebsd32/freebsd32_util.h
@@ -88,6 +88,15 @@ SYSCALL32_MODULE(syscallname,                           \
     .syscall_no = FREEBSD32_SYS_##syscallname			\
 }
 
+#define SYSCALL32_INIT_HELPER_COMPAT(syscallname) {		\
+    .new_sysent = {						\
+	.sy_narg = (sizeof(struct syscallname ## _args )	\
+	    / sizeof(register_t)),				\
+	.sy_call = (sy_call_t *)& sys_ ## syscallname,		\
+    },								\
+    .syscall_no = FREEBSD32_SYS_##syscallname			\
+}
+
 int    syscall32_register(int *offset, struct sysent *new_sysent,
 	    struct sysent *old_sysent);
 int    syscall32_deregister(int *offset, struct sysent *old_sysent);
diff --git a/sys/compat/freebsd32/syscalls.master b/sys/compat/freebsd32/syscalls.master
index d4b67ae31cfd..9428b6c20311 100644
--- a/sys/compat/freebsd32/syscalls.master
+++ b/sys/compat/freebsd32/syscalls.master
@@ -172,7 +172,7 @@
 72	AUE_O_VADVISE	NOPROTO	{ int ovadvise(int anom); } vadvise \
 				    ovadvise_args int
 73	AUE_MUNMAP	NOPROTO	{ int munmap(void *addr, size_t len); }
-74	AUE_MPROTECT	NOPROTO	{ int mprotect(const void *addr, \
+74	AUE_MPROTECT	STD	{ int freebsd32_mprotect(const void *addr, \
 				    size_t len, int prot); }
 75	AUE_MADVISE	NOPROTO	{ int madvise(void *addr, size_t len, \
 				    int behav); }
@@ -989,6 +989,6 @@
 				    size_t inbuflen, void *outbufp, \
 				    size_t outbuflen); }
 530	AUE_NULL	STD	{ int freebsd32_posix_fallocate(int fd,\
-				     uint32_t offsetlo, uint32_t offsethi,\
-				     uint32_t lenlo, uint32_t lenhi); }
+				    uint32_t offset1, uint32_t offset2,\
+				    uint32_t len1, uint32_t len2); }
 531	AUE_NULL	UNIMPL	posix_fadvise
diff --git a/sys/compat/linux/linux_emul.c b/sys/compat/linux/linux_emul.c
index 7585c0d0179e..d6b2f71f5cef 100644
--- a/sys/compat/linux/linux_emul.c
+++ b/sys/compat/linux/linux_emul.c
@@ -244,7 +244,7 @@ linux_proc_exit(void *arg __unused, struct proc *p)
 		KASSERT(em != NULL, ("linux_reparent: emuldata not found: %i\n", q->p_pid));
 		PROC_LOCK(q);
 		if ((q->p_flag & P_WEXIT) == 0 && em->pdeath_signal != 0) {
-			psignal(q, em->pdeath_signal);
+			kern_psignal(q, em->pdeath_signal);
 		}
 		PROC_UNLOCK(q);
 		EMUL_UNLOCK(&emul_lock);
@@ -362,7 +362,7 @@ linux_kill_threads(struct thread *td, int sig)
 
 		sp = pfind(em->pid);
 		if ((sp->p_flag & P_WEXIT) == 0)
-			psignal(sp, sig);
+			kern_psignal(sp, sig);
 		PROC_UNLOCK(sp);
 #ifdef DEBUG
 		printf(LMSG("linux_kill_threads: kill PID %d\n"), em->pid);
diff --git a/sys/compat/linux/linux_file.c b/sys/compat/linux/linux_file.c
index e92303287803..9cbc3a2cee04 100644
--- a/sys/compat/linux/linux_file.c
+++ b/sys/compat/linux/linux_file.c
@@ -232,7 +232,7 @@ linux_lseek(struct thread *td, struct linux_lseek_args *args)
     tmp_args.fd = args->fdes;
     tmp_args.offset = (off_t)args->off;
     tmp_args.whence = args->whence;
-    error = lseek(td, &tmp_args);
+    error = sys_lseek(td, &tmp_args);
     return error;
 }
 
@@ -254,7 +254,7 @@ linux_llseek(struct thread *td, struct linux_llseek_args *args)
 	bsd_args.offset = off;
 	bsd_args.whence = args->whence;
 
-	if ((error = lseek(td, &bsd_args)))
+	if ((error = sys_lseek(td, &bsd_args)))
 		return error;
 
 	if ((error = copyout(td->td_retval, args->res, sizeof (off_t))))
@@ -951,7 +951,7 @@ linux_ftruncate(struct thread *td, struct linux_ftruncate_args *args)
 	   
 	nuap.fd = args->fd;
 	nuap.length = args->length;
-	return (ftruncate(td, &nuap));
+	return (sys_ftruncate(td, &nuap));
 }
 
 int
@@ -1021,7 +1021,7 @@ linux_fdatasync(td, uap)
 	struct fsync_args bsd;
 
 	bsd.fd = uap->fd;
-	return fsync(td, &bsd);
+	return sys_fsync(td, &bsd);
 }
 
 int
@@ -1038,7 +1038,7 @@ linux_pread(td, uap)
 	bsd.nbyte = uap->nbyte;
 	bsd.offset = uap->offset;
 
-	error = pread(td, &bsd);
+	error = sys_pread(td, &bsd);
 
 	if (error == 0) {
    	   	/* This seems to violate POSIX but linux does it */
@@ -1065,7 +1065,7 @@ linux_pwrite(td, uap)
 	bsd.buf = uap->buf;
 	bsd.nbyte = uap->nbyte;
 	bsd.offset = uap->offset;
-	return pwrite(td, &bsd);
+	return sys_pwrite(td, &bsd);
 }
 
 int
@@ -1163,7 +1163,7 @@ linux_umount(struct thread *td, struct linux_umount_args *args)
 
 	bsd.path = args->path;
 	bsd.flags = args->flags;	/* XXX correct? */
-	return (unmount(td, &bsd));
+	return (sys_unmount(td, &bsd));
 }
 
 /*
diff --git a/sys/compat/linux/linux_ioctl.c b/sys/compat/linux/linux_ioctl.c
index d021fbada650..15e313639369 100644
--- a/sys/compat/linux/linux_ioctl.c
+++ b/sys/compat/linux/linux_ioctl.c
@@ -803,7 +803,7 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 				wr.fd = args->fd;
 				wr.buf = &c;
 				wr.nbyte = sizeof(c);
-				return (write(td, &wr));
+				return (sys_write(td, &wr));
 			} else
 				return (0);
 		}
@@ -812,7 +812,7 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 			return (EINVAL);
 		}
 		args->arg = 0;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 	}
 
@@ -838,27 +838,27 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_TIOCEXCL:
 		args->cmd = TIOCEXCL;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCNXCL:
 		args->cmd = TIOCNXCL;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCSCTTY:
 		args->cmd = TIOCSCTTY;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCGPGRP:
 		args->cmd = TIOCGPGRP;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCSPGRP:
 		args->cmd = TIOCSPGRP;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	/* LINUX_TIOCOUTQ */
@@ -866,32 +866,32 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_TIOCGWINSZ:
 		args->cmd = TIOCGWINSZ;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCSWINSZ:
 		args->cmd = TIOCSWINSZ;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCMGET:
 		args->cmd = TIOCMGET;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCMBIS:
 		args->cmd = TIOCMBIS;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCMBIC:
 		args->cmd = TIOCMBIC;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCMSET:
 		args->cmd = TIOCMSET;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	/* TIOCGSOFTCAR */
@@ -899,14 +899,14 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_FIONREAD: /* LINUX_TIOCINQ */
 		args->cmd = FIONREAD;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	/* LINUX_TIOCLINUX */
 
 	case LINUX_TIOCCONS:
 		args->cmd = TIOCCONS;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCGSERIAL: {
@@ -932,17 +932,17 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_TIOCPKT:
 		args->cmd = TIOCPKT;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_FIONBIO:
 		args->cmd = FIONBIO;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCNOTTY:
 		args->cmd = TIOCNOTTY;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCSETD: {
@@ -996,17 +996,17 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_FIONCLEX:
 		args->cmd = FIONCLEX;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_FIOCLEX:
 		args->cmd = FIOCLEX;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_FIOASYNC:
 		args->cmd = FIOASYNC;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	/* LINUX_TIOCSERCONFIG */
@@ -1017,12 +1017,12 @@ linux_ioctl_termio(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_TIOCSBRK:
 		args->cmd = TIOCSBRK;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_TIOCCBRK:
 		args->cmd = TIOCCBRK;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 	case LINUX_TIOCGPTN: {
 		int nb;
@@ -1447,22 +1447,22 @@ linux_ioctl_cdrom(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_CDROMPAUSE:
 		args->cmd = CDIOCPAUSE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_CDROMRESUME:
 		args->cmd = CDIOCRESUME;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_CDROMPLAYMSF:
 		args->cmd = CDIOCPLAYMSF;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_CDROMPLAYTRKIND:
 		args->cmd = CDIOCPLAYTRACKS;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_CDROMREADTOCHDR: {
@@ -1501,17 +1501,17 @@ linux_ioctl_cdrom(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_CDROMSTOP:
 		args->cmd = CDIOCSTOP;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_CDROMSTART:
 		args->cmd = CDIOCSTART;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_CDROMEJECT:
 		args->cmd = CDIOCEJECT;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	/* LINUX_CDROMVOLCTRL */
@@ -1555,7 +1555,7 @@ linux_ioctl_cdrom(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_CDROMRESET:
 		args->cmd = CDIOCRESET;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	/* LINUX_CDROMVOLREAD */
@@ -1689,71 +1689,71 @@ linux_ioctl_sound(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_SOUND_MIXER_WRITE_VOLUME:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_VOLUME);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_BASS:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_BASS);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_TREBLE:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_TREBLE);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_SYNTH:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_SYNTH);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_PCM:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_PCM);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_SPEAKER:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_SPEAKER);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_LINE:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_LINE);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_MIC:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_MIC);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_CD:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_CD);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_IMIX:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_IMIX);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_ALTPCM:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_ALTPCM);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_RECLEV:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_RECLEV);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_IGAIN:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_IGAIN);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_OGAIN:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_OGAIN);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_LINE1:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_LINE1);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_LINE2:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_LINE2);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_LINE3:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_LINE3);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_INFO: {
 		/* Key on encoded length */
@@ -1787,167 +1787,167 @@ linux_ioctl_sound(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_SOUND_MIXER_READ_STEREODEVS:
 		args->cmd = SOUND_MIXER_READ_STEREODEVS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_READ_CAPS:
 		args->cmd = SOUND_MIXER_READ_CAPS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_READ_RECMASK:
 		args->cmd = SOUND_MIXER_READ_RECMASK;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_READ_DEVMASK:
 		args->cmd = SOUND_MIXER_READ_DEVMASK;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_MIXER_WRITE_RECSRC:
 		args->cmd = SETDIR(SOUND_MIXER_WRITE_RECSRC);
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_RESET:
 		args->cmd = SNDCTL_DSP_RESET;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SYNC:
 		args->cmd = SNDCTL_DSP_SYNC;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SPEED:
 		args->cmd = SNDCTL_DSP_SPEED;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_STEREO:
 		args->cmd = SNDCTL_DSP_STEREO;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETBLKSIZE: /* LINUX_SNDCTL_DSP_SETBLKSIZE */
 		args->cmd = SNDCTL_DSP_GETBLKSIZE;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SETFMT:
 		args->cmd = SNDCTL_DSP_SETFMT;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_PCM_WRITE_CHANNELS:
 		args->cmd = SOUND_PCM_WRITE_CHANNELS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SOUND_PCM_WRITE_FILTER:
 		args->cmd = SOUND_PCM_WRITE_FILTER;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_POST:
 		args->cmd = SNDCTL_DSP_POST;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SUBDIVIDE:
 		args->cmd = SNDCTL_DSP_SUBDIVIDE;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SETFRAGMENT:
 		args->cmd = SNDCTL_DSP_SETFRAGMENT;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETFMTS:
 		args->cmd = SNDCTL_DSP_GETFMTS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETOSPACE:
 		args->cmd = SNDCTL_DSP_GETOSPACE;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETISPACE:
 		args->cmd = SNDCTL_DSP_GETISPACE;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_NONBLOCK:
 		args->cmd = SNDCTL_DSP_NONBLOCK;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETCAPS:
 		args->cmd = SNDCTL_DSP_GETCAPS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SETTRIGGER: /* LINUX_SNDCTL_GETTRIGGER */
 		args->cmd = SNDCTL_DSP_SETTRIGGER;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETIPTR:
 		args->cmd = SNDCTL_DSP_GETIPTR;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETOPTR:
 		args->cmd = SNDCTL_DSP_GETOPTR;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_SETDUPLEX:
 		args->cmd = SNDCTL_DSP_SETDUPLEX;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_DSP_GETODELAY:
 		args->cmd = SNDCTL_DSP_GETODELAY;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_RESET:
 		args->cmd = SNDCTL_SEQ_RESET;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_SYNC:
 		args->cmd = SNDCTL_SEQ_SYNC;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SYNTH_INFO:
 		args->cmd = SNDCTL_SYNTH_INFO;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_CTRLRATE:
 		args->cmd = SNDCTL_SEQ_CTRLRATE;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_GETOUTCOUNT:
 		args->cmd = SNDCTL_SEQ_GETOUTCOUNT;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_GETINCOUNT:
 		args->cmd = SNDCTL_SEQ_GETINCOUNT;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_PERCMODE:
 		args->cmd = SNDCTL_SEQ_PERCMODE;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_FM_LOAD_INSTR:
 		args->cmd = SNDCTL_FM_LOAD_INSTR;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_TESTMIDI:
 		args->cmd = SNDCTL_SEQ_TESTMIDI;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_RESETSAMPLES:
 		args->cmd = SNDCTL_SEQ_RESETSAMPLES;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_NRSYNTHS:
 		args->cmd = SNDCTL_SEQ_NRSYNTHS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_NRMIDIS:
 		args->cmd = SNDCTL_SEQ_NRMIDIS;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_MIDI_INFO:
 		args->cmd = SNDCTL_MIDI_INFO;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SEQ_TRESHOLD:
 		args->cmd = SNDCTL_SEQ_TRESHOLD;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	case LINUX_SNDCTL_SYNTH_MEMAVL:
 		args->cmd = SNDCTL_SYNTH_MEMAVL;
-		return (ioctl(td, (struct ioctl_args *)args));
+		return (sys_ioctl(td, (struct ioctl_args *)args));
 
 	}
 
@@ -1972,37 +1972,37 @@ linux_ioctl_console(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_KIOCSOUND:
 		args->cmd = KIOCSOUND;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDMKTONE:
 		args->cmd = KDMKTONE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDGETLED:
 		args->cmd = KDGETLED;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDSETLED:
 		args->cmd = KDSETLED;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDSETMODE:
 		args->cmd = KDSETMODE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDGETMODE:
 		args->cmd = KDGETMODE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDGKBMODE:
 		args->cmd = KDGKBMODE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_KDSKBMODE: {
@@ -2028,12 +2028,12 @@ linux_ioctl_console(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_VT_OPENQRY:
 		args->cmd = VT_OPENQRY;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_VT_GETMODE:
 		args->cmd = VT_GETMODE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_VT_SETMODE: {
@@ -2045,28 +2045,28 @@ linux_ioctl_console(struct thread *td, struct linux_ioctl_args *args)
 		if ((error = copyout(&mode, (void *)args->arg, sizeof(mode))))
 			break;
 		args->cmd = VT_SETMODE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 	}
 
 	case LINUX_VT_GETSTATE:
 		args->cmd = VT_GETACTIVE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_VT_RELDISP:
 		args->cmd = VT_RELDISP;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_VT_ACTIVATE:
 		args->cmd = VT_ACTIVATE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	case LINUX_VT_WAITACTIVE:
 		args->cmd = VT_WAITACTIVE;
-		error = (ioctl(td, (struct ioctl_args *)args));
+		error = (sys_ioctl(td, (struct ioctl_args *)args));
 		break;
 
 	default:
@@ -2440,27 +2440,27 @@ linux_ioctl_socket(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_FIOSETOWN:
 		args->cmd = FIOSETOWN;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCSPGRP:
 		args->cmd = SIOCSPGRP;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_FIOGETOWN:
 		args->cmd = FIOGETOWN;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCGPGRP:
 		args->cmd = SIOCGPGRP;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCATMARK:
 		args->cmd = SIOCATMARK;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	/* LINUX_SIOCGSTAMP */
@@ -2476,31 +2476,31 @@ linux_ioctl_socket(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_SIOCGIFADDR:
 		args->cmd = SIOCGIFADDR;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		bsd_to_linux_ifreq((struct ifreq *)args->arg);
 		break;
 
 	case LINUX_SIOCSIFADDR:
 		/* XXX probably doesn't work, included for completeness */
 		args->cmd = SIOCSIFADDR;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCGIFDSTADDR:
 		args->cmd = SIOCGIFDSTADDR;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		bsd_to_linux_ifreq((struct ifreq *)args->arg);
 		break;
 
 	case LINUX_SIOCGIFBRDADDR:
 		args->cmd = SIOCGIFBRDADDR;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		bsd_to_linux_ifreq((struct ifreq *)args->arg);
 		break;
 
 	case LINUX_SIOCGIFNETMASK:
 		args->cmd = SIOCGIFNETMASK;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		bsd_to_linux_ifreq((struct ifreq *)args->arg);
 		break;
 
@@ -2510,12 +2510,12 @@ linux_ioctl_socket(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_SIOCGIFMTU:
 		args->cmd = SIOCGIFMTU;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCSIFMTU:
 		args->cmd = SIOCSIFMTU;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCSIFNAME:
@@ -2532,17 +2532,17 @@ linux_ioctl_socket(struct thread *td, struct linux_ioctl_args *args)
 
 	case LINUX_SIOCADDMULTI:
 		args->cmd = SIOCADDMULTI;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCDELMULTI:
 		args->cmd = SIOCDELMULTI;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCGIFINDEX:
 		args->cmd = SIOCGIFINDEX;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCGIFCOUNT:
@@ -2555,12 +2555,12 @@ linux_ioctl_socket(struct thread *td, struct linux_ioctl_args *args)
 	 */
 	case LINUX_SIOCDEVPRIVATE:
 		args->cmd = SIOCGPRIVATE_0;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 
 	case LINUX_SIOCDEVPRIVATE+1:
 		args->cmd = SIOCGPRIVATE_1;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 	}
 
@@ -2599,7 +2599,7 @@ static int
 linux_ioctl_drm(struct thread *td, struct linux_ioctl_args *args)
 {
 	args->cmd = SETDIR(args->cmd);
-	return ioctl(td, (struct ioctl_args *)args);
+	return sys_ioctl(td, (struct ioctl_args *)args);
 }
 
 static int
@@ -2971,7 +2971,7 @@ linux_ioctl_v4l(struct thread *td, struct linux_ioctl_args *args)
 	default:			return (ENOIOCTL);
 	}
 
-	error = ioctl(td, (struct ioctl_args *)args);
+	error = sys_ioctl(td, (struct ioctl_args *)args);
 	return (error);
 }
 
@@ -2986,15 +2986,15 @@ linux_ioctl_special(struct thread *td, struct linux_ioctl_args *args)
 	switch (args->cmd) {
 	case LINUX_SIOCGIFADDR:
 		args->cmd = SIOCGIFADDR;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 	case LINUX_SIOCSIFADDR:
 		args->cmd = SIOCSIFADDR;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 	case LINUX_SIOCGIFFLAGS:
 		args->cmd = SIOCGIFFLAGS;
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 		break;
 	default:
 		error = ENOIOCTL;
@@ -3295,7 +3295,7 @@ linux_ioctl_v4l2(struct thread *td, struct linux_ioctl_args *args)
 	default:			return (ENOIOCTL);
 	}
 
-	error = ioctl(td, (struct ioctl_args *)args);
+	error = sys_ioctl(td, (struct ioctl_args *)args);
 	return (error);
 }
 
@@ -3411,7 +3411,7 @@ linux_ioctl_fbsd_usb(struct thread *td, struct linux_ioctl_args *args)
 		error = ENOIOCTL;
 	}
 	if (error != ENOIOCTL)
-		error = ioctl(td, (struct ioctl_args *)args);
+		error = sys_ioctl(td, (struct ioctl_args *)args);
 	return (error);
 }
 
diff --git a/sys/compat/linux/linux_ipc.c b/sys/compat/linux/linux_ipc.c
index 36223b1c2d28..f72a02d94f66 100644
--- a/sys/compat/linux/linux_ipc.c
+++ b/sys/compat/linux/linux_ipc.c
@@ -496,7 +496,7 @@ linux_semop(struct thread *td, struct linux_semop_args *args)
 	bsd_args.semid = args->semid;
 	bsd_args.sops = PTRIN(args->tsops);
 	bsd_args.nsops = args->nsops;
-	return (semop(td, &bsd_args));
+	return (sys_semop(td, &bsd_args));
 }
 
 int
@@ -513,7 +513,7 @@ linux_semget(struct thread *td, struct linux_semget_args *args)
 	bsd_args.key = args->key;
 	bsd_args.nsems = args->nsems;
 	bsd_args.semflg = args->semflg;
-	return (semget(td, &bsd_args));
+	return (sys_semget(td, &bsd_args));
 }
 
 int
@@ -661,7 +661,7 @@ linux_msgget(struct thread *td, struct linux_msgget_args *args)
 
 	bsd_args.key = args->key;
 	bsd_args.msgflg = args->msgflg;
-	return (msgget(td, &bsd_args));
+	return (sys_msgget(td, &bsd_args));
 }
 
 int
@@ -753,7 +753,7 @@ linux_shmat(struct thread *td, struct linux_shmat_args *args)
 	bsd_args.shmid = args->shmid;
 	bsd_args.shmaddr = PTRIN(args->shmaddr);
 	bsd_args.shmflg = args->shmflg;
-	if ((error = shmat(td, &bsd_args)))
+	if ((error = sys_shmat(td, &bsd_args)))
 		return (error);
 #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32))
 	addr = td->td_retval[0];
@@ -772,7 +772,7 @@ linux_shmdt(struct thread *td, struct linux_shmdt_args *args)
 	} */ bsd_args;
 
 	bsd_args.shmaddr = PTRIN(args->shmaddr);
-	return (shmdt(td, &bsd_args));
+	return (sys_shmdt(td, &bsd_args));
 }
 
 int
@@ -787,7 +787,7 @@ linux_shmget(struct thread *td, struct linux_shmget_args *args)
 	bsd_args.key = args->key;
 	bsd_args.size = args->size;
 	bsd_args.shmflg = args->shmflg;
-	return (shmget(td, &bsd_args));
+	return (sys_shmget(td, &bsd_args));
 }
 
 int
diff --git a/sys/compat/linux/linux_misc.c b/sys/compat/linux/linux_misc.c
index bbc805dc6628..e7e0525cba49 100644
--- a/sys/compat/linux/linux_misc.c
+++ b/sys/compat/linux/linux_misc.c
@@ -209,7 +209,7 @@ linux_brk(struct thread *td, struct linux_brk_args *args)
 	old = (vm_offset_t)vm->vm_daddr + ctob(vm->vm_dsize);
 	new = (vm_offset_t)args->dsend;
 	tmp.nsize = (char *)new;
-	if (((caddr_t)new > vm->vm_daddr) && !obreak(td, &tmp))
+	if (((caddr_t)new > vm->vm_daddr) && !sys_obreak(td, &tmp))
 		td->td_retval[0] = (long)new;
 	else
 		td->td_retval[0] = (long)old;
@@ -609,7 +609,7 @@ linux_mremap(struct thread *td, struct linux_mremap_args *args)
 		bsd_args.addr =
 		    (caddr_t)((uintptr_t)args->addr + args->new_len);
 		bsd_args.len = args->old_len - args->new_len;
-		error = munmap(td, &bsd_args);
+		error = sys_munmap(td, &bsd_args);
 	}
 
 	td->td_retval[0] = error ? 0 : (uintptr_t)args->addr;
@@ -629,7 +629,7 @@ linux_msync(struct thread *td, struct linux_msync_args *args)
 	bsd_args.len = (uintptr_t)args->len;
 	bsd_args.flags = args->fl & ~LINUX_MS_SYNC;
 
-	return (msync(td, &bsd_args));
+	return (sys_msync(td, &bsd_args));
 }
 
 int
@@ -1085,7 +1085,7 @@ linux_nice(struct thread *td, struct linux_nice_args *args)
 	bsd_args.which = PRIO_PROCESS;
 	bsd_args.who = 0;		/* current process */
 	bsd_args.prio = args->inc;
-	return (setpriority(td, &bsd_args));
+	return (sys_setpriority(td, &bsd_args));
 }
 
 int
@@ -1317,7 +1317,7 @@ linux_sched_setscheduler(struct thread *td,
 
 	bsd.pid = args->pid;
 	bsd.param = (struct sched_param *)args->param;
-	return (sched_setscheduler(td, &bsd));
+	return (sys_sched_setscheduler(td, &bsd));
 }
 
 int
@@ -1333,7 +1333,7 @@ linux_sched_getscheduler(struct thread *td,
 #endif
 
 	bsd.pid = args->pid;
-	error = sched_getscheduler(td, &bsd);
+	error = sys_sched_getscheduler(td, &bsd);
 
 	switch (td->td_retval[0]) {
 	case SCHED_OTHER:
@@ -1374,7 +1374,7 @@ linux_sched_get_priority_max(struct thread *td,
 	default:
 		return (EINVAL);
 	}
-	return (sched_get_priority_max(td, &bsd));
+	return (sys_sched_get_priority_max(td, &bsd));
 }
 
 int
@@ -1401,7 +1401,7 @@ linux_sched_get_priority_min(struct thread *td,
 	default:
 		return (EINVAL);
 	}
-	return (sched_get_priority_min(td, &bsd));
+	return (sys_sched_get_priority_min(td, &bsd));
 }
 
 #define REBOOT_CAD_ON	0x89abcdef
@@ -1454,7 +1454,7 @@ linux_reboot(struct thread *td, struct linux_reboot_args *args)
 	default:
 		return (EINVAL);
 	}
-	return (reboot(td, &bsd_args));
+	return (sys_reboot(td, &bsd_args));
 }
 
 
@@ -1592,7 +1592,7 @@ linux_getsid(struct thread *td, struct linux_getsid_args *args)
 #endif
 
 	bsd.pid = args->pid;
-	return (getsid(td, &bsd));
+	return (sys_getsid(td, &bsd));
 }
 
 int
@@ -1615,7 +1615,7 @@ linux_getpriority(struct thread *td, struct linux_getpriority_args *args)
 
 	bsd_args.which = args->which;
 	bsd_args.who = args->who;
-	error = getpriority(td, &bsd_args);
+	error = sys_getpriority(td, &bsd_args);
 	td->td_retval[0] = 20 - td->td_retval[0];
 	return (error);
 }
@@ -1893,7 +1893,7 @@ linux_sched_getaffinity(struct thread *td,
 	cga.cpusetsize = sizeof(cpuset_t);
 	cga.mask = (cpuset_t *) args->user_mask_ptr;
 
-	if ((error = cpuset_getaffinity(td, &cga)) == 0)
+	if ((error = sys_cpuset_getaffinity(td, &cga)) == 0)
 		td->td_retval[0] = sizeof(cpuset_t);
 
 	return (error);
@@ -1922,5 +1922,5 @@ linux_sched_setaffinity(struct thread *td,
 	csa.cpusetsize = sizeof(cpuset_t);
 	csa.mask = (cpuset_t *) args->user_mask_ptr;
 
-	return (cpuset_setaffinity(td, &csa));
+	return (sys_cpuset_setaffinity(td, &csa));
 }
diff --git a/sys/compat/linux/linux_signal.c b/sys/compat/linux/linux_signal.c
index 82dd2ee427c0..5f70765a9144 100644
--- a/sys/compat/linux/linux_signal.c
+++ b/sys/compat/linux/linux_signal.c
@@ -533,7 +533,7 @@ linux_kill(struct thread *td, struct linux_kill_args *args)
 		tmp.signum = args->signum;
 
 	tmp.pid = args->pid;
-	return (kill(td, &tmp));
+	return (sys_kill(td, &tmp));
 }
 
 static int
diff --git a/sys/compat/linux/linux_socket.c b/sys/compat/linux/linux_socket.c
index 08728a13db19..431c5b977794 100644
--- a/sys/compat/linux/linux_socket.c
+++ b/sys/compat/linux/linux_socket.c
@@ -72,43 +72,29 @@ __FBSDID("$FreeBSD$");
 #include <compat/linux/linux_socket.h>
 #include <compat/linux/linux_util.h>
 
-static int do_sa_get(struct sockaddr **, const struct osockaddr *, int *,
-    struct malloc_type *);
 static int linux_to_bsd_domain(int);
 
 /*
  * Reads a linux sockaddr and does any necessary translation.
  * Linux sockaddrs don't have a length field, only a family.
- */
-static int
-linux_getsockaddr(struct sockaddr **sap, const struct osockaddr *osa, int len)
-{
-	int osalen = len;
-
-	return (do_sa_get(sap, osa, &osalen, M_SONAME));
-}
-
-/*
  * Copy the osockaddr structure pointed to by osa to kernel, adjust
  * family and convert to sockaddr.
  */
 static int
-do_sa_get(struct sockaddr **sap, const struct osockaddr *osa, int *osalen,
-    struct malloc_type *mtype)
+linux_getsockaddr(struct sockaddr **sap, const struct osockaddr *osa, int salen)
 {
-	int error=0, bdom;
 	struct sockaddr *sa;
 	struct osockaddr *kosa;
-	int alloclen;
 #ifdef INET6
-	int oldv6size;
 	struct sockaddr_in6 *sin6;
+	int oldv6size;
 #endif
+	char *name;
+	int bdom, error, hdrlen, namelen;
 
-	if (*osalen < 2 || *osalen > UCHAR_MAX || !osa)
+	if (salen < 2 || salen > UCHAR_MAX || !osa)
 		return (EINVAL);
 
-	alloclen = *osalen;
 #ifdef INET6
 	oldv6size = 0;
 	/*
@@ -116,15 +102,15 @@ do_sa_get(struct sockaddr **sap, const struct osockaddr *osa, int *osalen,
 	 * if it's a v4-mapped address, so reserve the proper space
 	 * for it.
 	 */
-	if (alloclen == sizeof (struct sockaddr_in6) - sizeof (u_int32_t)) {
-		alloclen = sizeof (struct sockaddr_in6);
+	if (salen == sizeof(struct sockaddr_in6) - sizeof(uint32_t)) {
+		salen += sizeof(uint32_t);
 		oldv6size = 1;
 	}
 #endif
 
-	kosa = malloc(alloclen, mtype, M_WAITOK);
+	kosa = malloc(salen, M_SONAME, M_WAITOK);
 
-	if ((error = copyin(osa, kosa, *osalen)))
+	if ((error = copyin(osa, kosa, salen)))
 		goto out;
 
 	bdom = linux_to_bsd_domain(kosa->sa_family);
@@ -141,41 +127,61 @@ do_sa_get(struct sockaddr **sap, const struct osockaddr *osa, int *osalen,
 	 *
 	 * Still accept addresses for which the scope id is not used.
 	 */
-	if (oldv6size && bdom == AF_INET6) {
-		sin6 = (struct sockaddr_in6 *)kosa;
-		if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr) ||
-		    (!IN6_IS_ADDR_LINKLOCAL(&sin6->sin6_addr) &&
-		     !IN6_IS_ADDR_SITELOCAL(&sin6->sin6_addr) &&
-		     !IN6_IS_ADDR_V4COMPAT(&sin6->sin6_addr) &&
-		     !IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) &&
-		     !IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))) {
-			sin6->sin6_scope_id = 0;
-		} else {
-			log(LOG_DEBUG,
-			    "obsolete pre-RFC2553 sockaddr_in6 rejected\n");
+	if (oldv6size) {
+		if (bdom == AF_INET6) {
+			sin6 = (struct sockaddr_in6 *)kosa;
+			if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr) ||
+			    (!IN6_IS_ADDR_LINKLOCAL(&sin6->sin6_addr) &&
+			     !IN6_IS_ADDR_SITELOCAL(&sin6->sin6_addr) &&
+			     !IN6_IS_ADDR_V4COMPAT(&sin6->sin6_addr) &&
+			     !IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) &&
+			     !IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))) {
+				sin6->sin6_scope_id = 0;
+			} else {
+				log(LOG_DEBUG,
+				    "obsolete pre-RFC2553 sockaddr_in6 rejected\n");
+				error = EINVAL;
+				goto out;
+			}
+		} else
+			salen -= sizeof(uint32_t);
+	}
+#endif
+	if (bdom == AF_INET) {
+		if (salen < sizeof(struct sockaddr_in)) {
 			error = EINVAL;
 			goto out;
 		}
-	} else
-#endif
-	if (bdom == AF_INET) {
-		alloclen = sizeof(struct sockaddr_in);
-		if (*osalen < alloclen) {
-			error = EINVAL;
+		salen = sizeof(struct sockaddr_in);
+	}
+
+	if (bdom == AF_LOCAL && salen > sizeof(struct sockaddr_un)) {
+		hdrlen = offsetof(struct sockaddr_un, sun_path);
+		name = ((struct sockaddr_un *)kosa)->sun_path;
+		if (*name == '\0') {
+			/*
+		 	 * Linux abstract namespace starts with a NULL byte.
+			 * XXX We do not support abstract namespace yet.
+			 */
+			namelen = strnlen(name + 1, salen - hdrlen - 1) + 1;
+		} else
+			namelen = strnlen(name, salen - hdrlen);
+		salen = hdrlen + namelen;
+		if (salen > sizeof(struct sockaddr_un)) {
+			error = ENAMETOOLONG;
 			goto out;
 		}
 	}
 
-	sa = (struct sockaddr *) kosa;
+	sa = (struct sockaddr *)kosa;
 	sa->sa_family = bdom;
-	sa->sa_len = alloclen;
+	sa->sa_len = salen;
 
 	*sap = sa;
-	*osalen = alloclen;
 	return (0);
 
 out:
-	free(kosa, mtype);
+	free(kosa, M_SONAME);
 	return (error);
 }
 
@@ -643,7 +649,7 @@ linux_socket(struct thread *td, struct linux_socket_args *args)
 	if (bsd_args.domain == -1)
 		return (EAFNOSUPPORT);
 
-	retval_socket = socket(td, &bsd_args);
+	retval_socket = sys_socket(td, &bsd_args);
 	if (retval_socket)
 		return (retval_socket);
 
@@ -774,7 +780,7 @@ linux_listen(struct thread *td, struct linux_listen_args *args)
 
 	bsd_args.s = args->s;
 	bsd_args.backlog = args->backlog;
-	return (listen(td, &bsd_args));
+	return (sys_listen(td, &bsd_args));
 }
 
 static int
@@ -795,7 +801,7 @@ linux_accept_common(struct thread *td, int s, l_uintptr_t addr,
 	/* XXX: */
 	bsd_args.name = (struct sockaddr * __restrict)PTRIN(addr);
 	bsd_args.anamelen = PTRIN(namelen);/* XXX */
-	error = accept(td, &bsd_args);
+	error = sys_accept(td, &bsd_args);
 	bsd_to_linux_sockaddr((struct sockaddr *)bsd_args.name);
 	if (error) {
 		if (error == EFAULT && namelen != sizeof(struct sockaddr_in))
@@ -874,7 +880,7 @@ linux_getsockname(struct thread *td, struct linux_getsockname_args *args)
 	/* XXX: */
 	bsd_args.asa = (struct sockaddr * __restrict)PTRIN(args->addr);
 	bsd_args.alen = PTRIN(args->namelen);	/* XXX */
-	error = getsockname(td, &bsd_args);
+	error = sys_getsockname(td, &bsd_args);
 	bsd_to_linux_sockaddr((struct sockaddr *)bsd_args.asa);
 	if (error)
 		return (error);
@@ -903,7 +909,7 @@ linux_getpeername(struct thread *td, struct linux_getpeername_args *args)
 	bsd_args.fdes = args->s;
 	bsd_args.asa = (struct sockaddr *)PTRIN(args->addr);
 	bsd_args.alen = (int *)PTRIN(args->namelen);
-	error = getpeername(td, &bsd_args);
+	error = sys_getpeername(td, &bsd_args);
 	bsd_to_linux_sockaddr((struct sockaddr *)bsd_args.asa);
 	if (error)
 		return (error);
@@ -1001,7 +1007,7 @@ linux_send(struct thread *td, struct linux_send_args *args)
 	bsd_args.flags = args->flags;
 	bsd_args.to = NULL;
 	bsd_args.tolen = 0;
-	return sendto(td, &bsd_args);
+	return sys_sendto(td, &bsd_args);
 }
 
 struct linux_recv_args {
@@ -1029,7 +1035,7 @@ linux_recv(struct thread *td, struct linux_recv_args *args)
 	bsd_args.flags = linux_to_bsd_msg_flags(args->flags);
 	bsd_args.from = NULL;
 	bsd_args.fromlenaddr = 0;
-	return (recvfrom(td, &bsd_args));
+	return (sys_recvfrom(td, &bsd_args));
 }
 
 static int
@@ -1091,7 +1097,7 @@ linux_recvfrom(struct thread *td, struct linux_recvfrom_args *args)
 	bsd_args.fromlenaddr = PTRIN(args->fromlen);/* XXX */
 	
 	linux_to_bsd_sockaddr((struct sockaddr *)bsd_args.from, len);
-	error = recvfrom(td, &bsd_args);
+	error = sys_recvfrom(td, &bsd_args);
 	bsd_to_linux_sockaddr((struct sockaddr *)bsd_args.from);
 	
 	if (error)
@@ -1224,9 +1230,9 @@ linux_sendmsg(struct thread *td, struct linux_sendmsg_args *args)
 			cmsg->cmsg_len = CMSG_LEN(datalen);
 
 			error = ENOBUFS;
-			if (!m_append(control, CMSG_HDRSZ, (c_caddr_t) cmsg))
+			if (!m_append(control, CMSG_HDRSZ, (c_caddr_t)cmsg))
 				goto bad;
-			if (!m_append(control, datalen, (c_caddr_t) data))
+			if (!m_append(control, datalen, (c_caddr_t)data))
 				goto bad;
 		} while ((ptr_cmsg = LINUX_CMSG_NXTHDR(&linux_msg, ptr_cmsg)));
 
@@ -1365,7 +1371,7 @@ linux_recvmsg(struct thread *td, struct linux_recvmsg_args *args)
 				 * effect for Linux so no need to worry
 				 * about sockcred
 				 */
-				if (datalen != sizeof (*cmcred)) {
+				if (datalen != sizeof(*cmcred)) {
 					error = EMSGSIZE;
 					goto bad;
 				}
@@ -1438,7 +1444,7 @@ linux_shutdown(struct thread *td, struct linux_shutdown_args *args)
 
 	bsd_args.s = args->s;
 	bsd_args.how = args->how;
-	return (shutdown(td, &bsd_args));
+	return (sys_shutdown(td, &bsd_args));
 }
 
 struct linux_setsockopt_args {
@@ -1507,10 +1513,10 @@ linux_setsockopt(struct thread *td, struct linux_setsockopt_args *args)
 	if (name == IPV6_NEXTHOP) {
 		linux_to_bsd_sockaddr((struct sockaddr *)bsd_args.val,
 			bsd_args.valsize);
-		error = setsockopt(td, &bsd_args);
+		error = sys_setsockopt(td, &bsd_args);
 		bsd_to_linux_sockaddr((struct sockaddr *)bsd_args.val);
 	} else
-		error = setsockopt(td, &bsd_args);
+		error = sys_setsockopt(td, &bsd_args);
 
 	return (error);
 }
@@ -1600,10 +1606,10 @@ linux_getsockopt(struct thread *td, struct linux_getsockopt_args *args)
 	bsd_args.avalsize = PTRIN(args->optlen);
 
 	if (name == IPV6_NEXTHOP) {
-		error = getsockopt(td, &bsd_args);
+		error = sys_getsockopt(td, &bsd_args);
 		bsd_to_linux_sockaddr((struct sockaddr *)bsd_args.val);
 	} else
-		error = getsockopt(td, &bsd_args);
+		error = sys_getsockopt(td, &bsd_args);
 
 	return (error);
 }
diff --git a/sys/compat/linux/linux_uid16.c b/sys/compat/linux/linux_uid16.c
index 6ad674be77fc..31950ffb2286 100644
--- a/sys/compat/linux/linux_uid16.c
+++ b/sys/compat/linux/linux_uid16.c
@@ -113,8 +113,10 @@ linux_setgroups16(struct thread *td, struct linux_setgroups16_args *args)
 		return (EINVAL);
 	linux_gidset = malloc(ngrp * sizeof(*linux_gidset), M_TEMP, M_WAITOK);
 	error = copyin(args->gidset, linux_gidset, ngrp * sizeof(l_gid16_t));
-	if (error)
+	if (error) {
+		free(linux_gidset, M_TEMP);
 		return (error);
+	}
 	newcred = crget();
 	p = td->td_proc;
 	PROC_LOCK(p);
@@ -234,7 +236,7 @@ linux_getegid16(struct thread *td, struct linux_getegid16_args *args)
 {
 	struct getegid_args bsd;
 
-	return (getegid(td, &bsd));
+	return (sys_getegid(td, &bsd));
 }
 
 int
@@ -242,7 +244,7 @@ linux_geteuid16(struct thread *td, struct linux_geteuid16_args *args)
 {
 	struct geteuid_args bsd;
 
-	return (geteuid(td, &bsd));
+	return (sys_geteuid(td, &bsd));
 }
 
 int
@@ -251,7 +253,7 @@ linux_setgid16(struct thread *td, struct linux_setgid16_args *args)
 	struct setgid_args bsd;
 
 	bsd.gid = args->gid;
-	return (setgid(td, &bsd));
+	return (sys_setgid(td, &bsd));
 }
 
 int
@@ -260,7 +262,7 @@ linux_setuid16(struct thread *td, struct linux_setuid16_args *args)
 	struct setuid_args bsd;
 
 	bsd.uid = args->uid;
-	return (setuid(td, &bsd));
+	return (sys_setuid(td, &bsd));
 }
 
 int
@@ -270,7 +272,7 @@ linux_setregid16(struct thread *td, struct linux_setregid16_args *args)
 
 	bsd.rgid = CAST_NOCHG(args->rgid);
 	bsd.egid = CAST_NOCHG(args->egid);
-	return (setregid(td, &bsd));
+	return (sys_setregid(td, &bsd));
 }
 
 int
@@ -280,7 +282,7 @@ linux_setreuid16(struct thread *td, struct linux_setreuid16_args *args)
 
 	bsd.ruid = CAST_NOCHG(args->ruid);
 	bsd.euid = CAST_NOCHG(args->euid);
-	return (setreuid(td, &bsd));
+	return (sys_setreuid(td, &bsd));
 }
 
 int
@@ -291,7 +293,7 @@ linux_setresgid16(struct thread *td, struct linux_setresgid16_args *args)
 	bsd.rgid = CAST_NOCHG(args->rgid);
 	bsd.egid = CAST_NOCHG(args->egid);
 	bsd.sgid = CAST_NOCHG(args->sgid);
-	return (setresgid(td, &bsd));
+	return (sys_setresgid(td, &bsd));
 }
 
 int
@@ -302,5 +304,5 @@ linux_setresuid16(struct thread *td, struct linux_setresuid16_args *args)
 	bsd.ruid = CAST_NOCHG(args->ruid);
 	bsd.euid = CAST_NOCHG(args->euid);
 	bsd.suid = CAST_NOCHG(args->suid);
-	return (setresuid(td, &bsd));
+	return (sys_setresuid(td, &bsd));
 }
diff --git a/sys/compat/svr4/svr4_fcntl.c b/sys/compat/svr4/svr4_fcntl.c
index ce1452a3043b..cbca7e24b1fb 100644
--- a/sys/compat/svr4/svr4_fcntl.c
+++ b/sys/compat/svr4/svr4_fcntl.c
@@ -368,7 +368,7 @@ fd_truncate(td, fd, flp)
 	ft.fd = fd;
 	ft.length = start;
 
-	error = ftruncate(td, &ft);
+	error = sys_ftruncate(td, &ft);
 
 	fdrop(fp, td);
 	return (error);
@@ -476,7 +476,7 @@ svr4_sys_llseek(td, uap)
 #endif
 	ap.whence = uap->whence;
 
-	return lseek(td, &ap);
+	return sys_lseek(td, &ap);
 }
 
 int
diff --git a/sys/compat/svr4/svr4_filio.c b/sys/compat/svr4/svr4_filio.c
index cb7cadae9c4f..967169b7dc97 100644
--- a/sys/compat/svr4/svr4_filio.c
+++ b/sys/compat/svr4/svr4_filio.c
@@ -75,7 +75,7 @@ svr4_sys_poll(td, uap)
      siz = uap->nfds * sizeof(struct pollfd);
      pfd = (struct pollfd *)malloc(siz, M_TEMP, M_WAITOK);
 
-     error = poll(td, (struct poll_args *)uap);
+     error = sys_poll(td, (struct poll_args *)uap);
 
      if ((cerr = copyin(uap->fds, pfd, siz)) != 0) {
        error = cerr;
diff --git a/sys/compat/svr4/svr4_ipc.c b/sys/compat/svr4/svr4_ipc.c
index 658c6d677fdf..f816cb082c8d 100644
--- a/sys/compat/svr4/svr4_ipc.c
+++ b/sys/compat/svr4/svr4_ipc.c
@@ -292,7 +292,7 @@ svr4_semget(td, v)
 	ap.nsems = uap->nsems;
 	ap.semflg = uap->semflg;
 
-	return semget(td, &ap);
+	return sys_semget(td, &ap);
 }
 
 struct svr4_sys_semop_args {
@@ -315,7 +315,7 @@ svr4_semop(td, v)
 	ap.sops = (struct sembuf *) uap->sops;
 	ap.nsops = uap->nsops;
 
-	return semop(td, &ap);
+	return sys_semop(td, &ap);
 }
 
 int
@@ -400,7 +400,7 @@ svr4_msgsnd(td, v)
 	ap.msgsz = uap->msgsz;
 	ap.msgflg = uap->msgflg;
 
-	return msgsnd(td, &ap);
+	return sys_msgsnd(td, &ap);
 }
 
 struct svr4_sys_msgrcv_args {
@@ -426,7 +426,7 @@ svr4_msgrcv(td, v)
 	ap.msgtyp = uap->msgtyp;
 	ap.msgflg = uap->msgflg;
 
-	return msgrcv(td, &ap);
+	return sys_msgrcv(td, &ap);
 }
 	
 struct svr4_sys_msgget_args {
@@ -446,7 +446,7 @@ svr4_msgget(td, v)
 	ap.key = uap->key;
 	ap.msgflg = uap->msgflg;
 
-	return msgget(td, &ap);
+	return sys_msgget(td, &ap);
 }
 
 struct svr4_sys_msgctl_args {
@@ -569,7 +569,7 @@ svr4_shmat(td, v)
 	ap.shmaddr = uap->shmaddr;
 	ap.shmflg = uap->shmflg;
 
-	return shmat(td, &ap);
+	return sys_shmat(td, &ap);
 }
 
 struct svr4_sys_shmdt_args {
@@ -587,7 +587,7 @@ svr4_shmdt(td, v)
 
 	ap.shmaddr = uap->shmaddr;
 
-	return shmdt(td, &ap);
+	return sys_shmdt(td, &ap);
 }
 
 struct svr4_sys_shmget_args {
@@ -609,7 +609,7 @@ svr4_shmget(td, v)
 	ap.size = uap->size;
 	ap.shmflg = uap->shmflg;
 
-	return shmget(td, &ap);
+	return sys_shmget(td, &ap);
 }
 
 struct svr4_sys_shmctl_args {
diff --git a/sys/compat/svr4/svr4_misc.c b/sys/compat/svr4/svr4_misc.c
index c0a74780ee48..8456e104b266 100644
--- a/sys/compat/svr4/svr4_misc.c
+++ b/sys/compat/svr4/svr4_misc.c
@@ -569,7 +569,7 @@ svr4_sys_mmap(td, uap)
 	mm.addr = uap->addr;
 	mm.pos = uap->pos;
 
-	return mmap(td, &mm);
+	return sys_mmap(td, &mm);
 }
 
 int
@@ -602,7 +602,7 @@ svr4_sys_mmap64(td, uap)
 	    mm.addr != 0 && (void *)mm.addr < rp)
 		mm.addr = rp;
 
-	return mmap(td, &mm);
+	return sys_mmap(td, &mm);
 }
 
 
@@ -844,7 +844,7 @@ svr4_sys_break(td, uap)
 	struct obreak_args ap;
 
 	ap.nsize = uap->nsize;
-	return (obreak(td, &ap));
+	return (sys_obreak(td, &ap));
 }
 
 static __inline clock_t
@@ -990,7 +990,7 @@ svr4_sys_pgrpsys(td, uap)
 		 * setsid() for SVR4.  (Under BSD, the difference is that
 		 * a setpgid(0,0) will not create a new session.)
 		 */
-		setsid(td, NULL);
+		sys_setsid(td, NULL);
 		/*FALLTHROUGH*/
 
 	case 0:			/* getpgrp() */
@@ -1013,7 +1013,7 @@ svr4_sys_pgrpsys(td, uap)
 		return 0;
 
 	case 3:			/* setsid() */
-		return setsid(td, NULL);
+		return sys_setsid(td, NULL);
 
 	case 4:			/* getpgid(pid) */
 
@@ -1032,7 +1032,7 @@ svr4_sys_pgrpsys(td, uap)
 
 			sa.pid = uap->pid;
 			sa.pgid = uap->pgid;
-			return setpgid(td, &sa);
+			return sys_setpgid(td, &sa);
 		}
 
 	default:
@@ -1598,7 +1598,7 @@ svr4_sys_memcntl(td, uap)
 			msa.len = uap->len;
 			msa.flags = (int)uap->arg;
 
-			return msync(td, &msa);
+			return sys_msync(td, &msa);
 		}
 	case SVR4_MC_ADVISE:
 		{
@@ -1608,7 +1608,7 @@ svr4_sys_memcntl(td, uap)
 			maa.len = uap->len;
 			maa.behav = (int)uap->arg;
 
-			return madvise(td, &maa);
+			return sys_madvise(td, &maa);
 		}
 	case SVR4_MC_LOCK:
 	case SVR4_MC_UNLOCK:
@@ -1633,11 +1633,11 @@ svr4_sys_nice(td, uap)
 	ap.who = 0;
 	ap.prio = uap->prio;
 
-	if ((error = setpriority(td, &ap)) != 0)
+	if ((error = sys_setpriority(td, &ap)) != 0)
 		return error;
 
 	/* the cast is stupid, but the structures are the same */
-	if ((error = getpriority(td, (struct getpriority_args *)&ap)) != 0)
+	if ((error = sys_getpriority(td, (struct getpriority_args *)&ap)) != 0)
 		return error;
 
 	return 0;
diff --git a/sys/compat/svr4/svr4_signal.c b/sys/compat/svr4/svr4_signal.c
index 0034abd41bfe..3791c05df7d6 100644
--- a/sys/compat/svr4/svr4_signal.c
+++ b/sys/compat/svr4/svr4_signal.c
@@ -519,7 +519,7 @@ svr4_sys_kill(td, uap)
 		return (EINVAL);
 	ka.pid = uap->pid;
 	ka.signum = SVR4_SVR42BSD_SIG(uap->signum);
-	return kill(td, &ka);
+	return sys_kill(td, &ka);
 }
 
 
diff --git a/sys/compat/svr4/svr4_socket.c b/sys/compat/svr4/svr4_socket.c
index 3f3d7c59dceb..038267ca221d 100644
--- a/sys/compat/svr4/svr4_socket.c
+++ b/sys/compat/svr4/svr4_socket.c
@@ -238,5 +238,5 @@ svr4_sys_socket(td, uap)
 	default:
 		return EINVAL;
 	}
-	return socket(td, (struct socket_args *)uap);
+	return sys_socket(td, (struct socket_args *)uap);
 }
diff --git a/sys/compat/svr4/svr4_stat.c b/sys/compat/svr4/svr4_stat.c
index cc84396a9997..b6866425d54c 100644
--- a/sys/compat/svr4/svr4_stat.c
+++ b/sys/compat/svr4/svr4_stat.c
@@ -694,6 +694,6 @@ svr4_sys_fpathconf(td, uap)
 		*retval = 0;
 		return 0;
 	default:
-		return fpathconf(td, (struct fpathconf_args *)uap);
+		return sys_fpathconf(td, (struct fpathconf_args *)uap);
 	}
 }
diff --git a/sys/compat/svr4/svr4_stream.c b/sys/compat/svr4/svr4_stream.c
index a1a42c030116..27014e38cdd6 100644
--- a/sys/compat/svr4/svr4_stream.c
+++ b/sys/compat/svr4/svr4_stream.c
@@ -523,7 +523,7 @@ si_listen(fp, fd, ioc, td)
 	DPRINTF(("SI_LISTEN: fileno %d backlog = %d\n", fd, 5));
 	la.backlog = 5;
 
-	if ((error = listen(td, &la)) != 0) {
+	if ((error = sys_listen(td, &la)) != 0) {
 		DPRINTF(("SI_LISTEN: listen failed %d\n", error));
 		return error;
 	}
@@ -637,7 +637,7 @@ si_shutdown(fp, fd, ioc, td)
 
 	ap.s = fd;
 
-	return shutdown(td, &ap);
+	return sys_shutdown(td, &ap);
 }
 
 
@@ -1056,7 +1056,7 @@ i_fdinsert(fp, td, retval, fd, cmd, dat)
 	d2p.from = st->s_afd;
 	d2p.to = fdi.fd;
 
-	if ((error = dup2(td, &d2p)) != 0) {
+	if ((error = sys_dup2(td, &d2p)) != 0) {
 		DPRINTF(("fdinsert: dup2(%d, %d) failed %d\n", 
 		    st->s_afd, fdi.fd, error));
 		mtx_unlock(&Giant);
@@ -1099,7 +1099,7 @@ _i_bind_rsvd(fp, td, retval, fd, cmd, dat)
 	ap.path = dat;
 	ap.mode = S_IFIFO;
 
-	return mkfifo(td, &ap);
+	return sys_mkfifo(td, &ap);
 }
 
 static int
@@ -1119,7 +1119,7 @@ _i_rele_rsvd(fp, td, retval, fd, cmd, dat)
 	 */
 	ap.path = dat;
 
-	return unlink(td, &ap);
+	return sys_unlink(td, &ap);
 }
 
 static int
@@ -1539,7 +1539,7 @@ svr4_do_putmsg(td, uap, fp)
 				wa.fd = uap->fd;
 				wa.buf = dat.buf;
 				wa.nbyte = dat.len;
-				return write(td, &wa);
+				return sys_write(td, &wa);
 			}
 	                DPRINTF(("putmsg: Invalid inet length %ld\n", sc.len));
 	                return EINVAL;
@@ -1926,7 +1926,7 @@ svr4_do_getmsg(td, uap, fp)
 			ra.fd = uap->fd;
 			ra.buf = dat.buf;
 			ra.nbyte = dat.maxlen;
-			if ((error = read(td, &ra)) != 0) {
+			if ((error = sys_read(td, &ra)) != 0) {
 				mtx_unlock(&Giant);
 			        return error;
 			}
@@ -1995,7 +1995,7 @@ int svr4_sys_send(td, uap)
 	sta.to = NULL;
 	sta.tolen = 0;
 
-	return (sendto(td, &sta));
+	return (sys_sendto(td, &sta));
 }
 
 int svr4_sys_recv(td, uap)
@@ -2011,7 +2011,7 @@ int svr4_sys_recv(td, uap)
 	rfa.from = NULL;
 	rfa.fromlenaddr = NULL;
 
-	return (recvfrom(td, &rfa));
+	return (sys_recvfrom(td, &rfa));
 }
 
 /* 
@@ -2033,6 +2033,6 @@ svr4_sys_sendto(td, uap)
 	sa.tolen = uap->tolen;
 
 	DPRINTF(("calling sendto()\n"));
-	return sendto(td, &sa);
+	return sys_sendto(td, &sa);
 }
 
diff --git a/sys/compat/svr4/svr4_sysent.c b/sys/compat/svr4/svr4_sysent.c
index 3b961c45d358..cf4481efdea6 100644
--- a/sys/compat/svr4/svr4_sysent.c
+++ b/sys/compat/svr4/svr4_sysent.c
@@ -20,30 +20,30 @@
 /* The casts are bogus but will do for now. */
 struct sysent svr4_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 0 = unused */
-	{ AS(sys_exit_args), (sy_call_t *)sys_exit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
-	{ 0, (sy_call_t *)fork, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 2 = fork */
-	{ AS(read_args), (sy_call_t *)read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
-	{ AS(write_args), (sy_call_t *)write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
+	{ AS(sys_exit_args), (sy_call_t *)sys_sys_exit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
+	{ 0, (sy_call_t *)sys_fork, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 2 = fork */
+	{ AS(read_args), (sy_call_t *)sys_read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
+	{ AS(write_args), (sy_call_t *)sys_write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
 	{ AS(svr4_sys_open_args), (sy_call_t *)svr4_sys_open, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = svr4_sys_open */
-	{ AS(close_args), (sy_call_t *)close, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
+	{ AS(close_args), (sy_call_t *)sys_close, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
 	{ AS(svr4_sys_wait_args), (sy_call_t *)svr4_sys_wait, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = svr4_sys_wait */
 	{ AS(svr4_sys_creat_args), (sy_call_t *)svr4_sys_creat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 8 = svr4_sys_creat */
-	{ AS(link_args), (sy_call_t *)link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
-	{ AS(unlink_args), (sy_call_t *)unlink, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = unlink */
+	{ AS(link_args), (sy_call_t *)sys_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
+	{ AS(unlink_args), (sy_call_t *)sys_unlink, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = unlink */
 	{ AS(svr4_sys_execv_args), (sy_call_t *)svr4_sys_execv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 11 = svr4_sys_execv */
-	{ AS(chdir_args), (sy_call_t *)chdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = chdir */
+	{ AS(chdir_args), (sy_call_t *)sys_chdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = chdir */
 	{ AS(svr4_sys_time_args), (sy_call_t *)svr4_sys_time, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 13 = svr4_sys_time */
 	{ AS(svr4_sys_mknod_args), (sy_call_t *)svr4_sys_mknod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 14 = svr4_sys_mknod */
-	{ AS(chmod_args), (sy_call_t *)chmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = chmod */
-	{ AS(chown_args), (sy_call_t *)chown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = chown */
+	{ AS(chmod_args), (sy_call_t *)sys_chmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = chmod */
+	{ AS(chown_args), (sy_call_t *)sys_chown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = chown */
 	{ AS(svr4_sys_break_args), (sy_call_t *)svr4_sys_break, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 17 = svr4_sys_break */
 	{ AS(svr4_sys_stat_args), (sy_call_t *)svr4_sys_stat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 18 = svr4_sys_stat */
-	{ AS(lseek_args), (sy_call_t *)lseek, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 19 = lseek */
-	{ 0, (sy_call_t *)getpid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 20 = getpid */
+	{ AS(lseek_args), (sy_call_t *)sys_lseek, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 19 = lseek */
+	{ 0, (sy_call_t *)sys_getpid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 20 = getpid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 21 = old_mount */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 22 = sysv_umount */
-	{ AS(setuid_args), (sy_call_t *)setuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 23 = setuid */
-	{ 0, (sy_call_t *)getuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 24 = getuid */
+	{ AS(setuid_args), (sy_call_t *)sys_setuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 23 = setuid */
+	{ 0, (sy_call_t *)sys_getuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 24 = getuid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 25 = stime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 26 = ptrace */
 	{ AS(svr4_sys_alarm_args), (sy_call_t *)svr4_sys_alarm, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 27 = svr4_sys_alarm */
@@ -55,18 +55,18 @@ struct sysent svr4_sysent[] = {
 	{ AS(svr4_sys_access_args), (sy_call_t *)svr4_sys_access, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = svr4_sys_access */
 	{ AS(svr4_sys_nice_args), (sy_call_t *)svr4_sys_nice, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = svr4_sys_nice */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 35 = statfs */
-	{ 0, (sy_call_t *)sync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
+	{ 0, (sy_call_t *)sys_sync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
 	{ AS(svr4_sys_kill_args), (sy_call_t *)svr4_sys_kill, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = svr4_sys_kill */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 38 = fstatfs */
 	{ AS(svr4_sys_pgrpsys_args), (sy_call_t *)svr4_sys_pgrpsys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = svr4_sys_pgrpsys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 40 = xenix */
-	{ AS(dup_args), (sy_call_t *)dup, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
-	{ 0, (sy_call_t *)pipe, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 42 = pipe */
+	{ AS(dup_args), (sy_call_t *)sys_dup, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
+	{ 0, (sy_call_t *)sys_pipe, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 42 = pipe */
 	{ AS(svr4_sys_times_args), (sy_call_t *)svr4_sys_times, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = svr4_sys_times */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 44 = profil */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 45 = plock */
-	{ AS(setgid_args), (sy_call_t *)setgid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 46 = setgid */
-	{ 0, (sy_call_t *)getgid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 47 = getgid */
+	{ AS(setgid_args), (sy_call_t *)sys_setgid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 46 = setgid */
+	{ 0, (sy_call_t *)sys_getgid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 47 = getgid */
 	{ AS(svr4_sys_signal_args), (sy_call_t *)svr4_sys_signal, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 48 = svr4_sys_signal */
 	{ AS(svr4_sys_msgsys_args), (sy_call_t *)svr4_sys_msgsys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 49 = svr4_sys_msgsys */
 	{ AS(svr4_sys_sysarch_args), (sy_call_t *)svr4_sys_sysarch, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = svr4_sys_sysarch */
@@ -77,10 +77,10 @@ struct sysent svr4_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 55 = uadmin */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 56 = exch */
 	{ AS(svr4_sys_utssys_args), (sy_call_t *)svr4_sys_utssys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = svr4_sys_utssys */
-	{ AS(fsync_args), (sy_call_t *)fsync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = fsync */
+	{ AS(fsync_args), (sy_call_t *)sys_fsync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = fsync */
 	{ AS(svr4_sys_execve_args), (sy_call_t *)svr4_sys_execve, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = svr4_sys_execve */
-	{ AS(umask_args), (sy_call_t *)umask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
-	{ AS(chroot_args), (sy_call_t *)chroot, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
+	{ AS(umask_args), (sy_call_t *)sys_umask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
+	{ AS(chroot_args), (sy_call_t *)sys_chroot, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
 	{ AS(svr4_sys_fcntl_args), (sy_call_t *)svr4_sys_fcntl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 62 = svr4_sys_fcntl */
 	{ AS(svr4_sys_ulimit_args), (sy_call_t *)svr4_sys_ulimit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = svr4_sys_ulimit */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 64 = reserved */
@@ -98,8 +98,8 @@ struct sysent svr4_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 76 = rdebug */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 77 = rfstop */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 78 = rfsys */
-	{ AS(rmdir_args), (sy_call_t *)rmdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 79 = rmdir */
-	{ AS(mkdir_args), (sy_call_t *)mkdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = mkdir */
+	{ AS(rmdir_args), (sy_call_t *)sys_rmdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 79 = rmdir */
+	{ AS(mkdir_args), (sy_call_t *)sys_mkdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = mkdir */
 	{ AS(svr4_sys_getdents_args), (sy_call_t *)svr4_sys_getdents, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 81 = svr4_sys_getdents */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 82 = libattach */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 83 = libdetach */
@@ -108,12 +108,12 @@ struct sysent svr4_sysent[] = {
 	{ AS(svr4_sys_putmsg_args), (sy_call_t *)svr4_sys_putmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 86 = svr4_sys_putmsg */
 	{ AS(svr4_sys_poll_args), (sy_call_t *)svr4_sys_poll, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = svr4_sys_poll */
 	{ AS(svr4_sys_lstat_args), (sy_call_t *)svr4_sys_lstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 88 = svr4_sys_lstat */
-	{ AS(symlink_args), (sy_call_t *)symlink, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = symlink */
-	{ AS(readlink_args), (sy_call_t *)readlink, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = readlink */
-	{ AS(getgroups_args), (sy_call_t *)getgroups, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 91 = getgroups */
-	{ AS(setgroups_args), (sy_call_t *)setgroups, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 92 = setgroups */
-	{ AS(fchmod_args), (sy_call_t *)fchmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 93 = fchmod */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 94 = fchown */
+	{ AS(symlink_args), (sy_call_t *)sys_symlink, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = symlink */
+	{ AS(readlink_args), (sy_call_t *)sys_readlink, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = readlink */
+	{ AS(getgroups_args), (sy_call_t *)sys_getgroups, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 91 = getgroups */
+	{ AS(setgroups_args), (sy_call_t *)sys_setgroups, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 92 = setgroups */
+	{ AS(fchmod_args), (sy_call_t *)sys_fchmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 93 = fchmod */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 94 = fchown */
 	{ AS(svr4_sys_sigprocmask_args), (sy_call_t *)svr4_sys_sigprocmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = svr4_sys_sigprocmask */
 	{ AS(svr4_sys_sigsuspend_args), (sy_call_t *)svr4_sys_sigsuspend, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 96 = svr4_sys_sigsuspend */
 	{ AS(svr4_sys_sigaltstack_args), (sy_call_t *)svr4_sys_sigaltstack, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = svr4_sys_sigaltstack */
@@ -135,13 +135,13 @@ struct sysent svr4_sysent[] = {
 	{ AS(svr4_sys_pathconf_args), (sy_call_t *)svr4_sys_pathconf, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 113 = svr4_sys_pathconf */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 114 = mincore */
 	{ AS(svr4_sys_mmap_args), (sy_call_t *)svr4_sys_mmap, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 115 = svr4_sys_mmap */
-	{ AS(mprotect_args), (sy_call_t *)mprotect, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 116 = mprotect */
-	{ AS(munmap_args), (sy_call_t *)munmap, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 117 = munmap */
+	{ AS(mprotect_args), (sy_call_t *)sys_mprotect, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 116 = mprotect */
+	{ AS(munmap_args), (sy_call_t *)sys_munmap, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 117 = munmap */
 	{ AS(svr4_sys_fpathconf_args), (sy_call_t *)svr4_sys_fpathconf, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = svr4_sys_fpathconf */
-	{ 0, (sy_call_t *)vfork, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 119 = vfork */
-	{ AS(fchdir_args), (sy_call_t *)fchdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 120 = fchdir */
-	{ AS(readv_args), (sy_call_t *)readv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 121 = readv */
-	{ AS(writev_args), (sy_call_t *)writev, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 122 = writev */
+	{ 0, (sy_call_t *)sys_vfork, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 119 = vfork */
+	{ AS(fchdir_args), (sy_call_t *)sys_fchdir, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 120 = fchdir */
+	{ AS(readv_args), (sy_call_t *)sys_readv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 121 = readv */
+	{ AS(writev_args), (sy_call_t *)sys_writev, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 122 = writev */
 	{ AS(svr4_sys_xstat_args), (sy_call_t *)svr4_sys_xstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 123 = svr4_sys_xstat */
 	{ AS(svr4_sys_lxstat_args), (sy_call_t *)svr4_sys_lxstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 124 = svr4_sys_lxstat */
 	{ AS(svr4_sys_fxstat_args), (sy_call_t *)svr4_sys_fxstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 125 = svr4_sys_fxstat */
@@ -149,18 +149,18 @@ struct sysent svr4_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 127 = clocal */
 	{ AS(svr4_sys_setrlimit_args), (sy_call_t *)svr4_sys_setrlimit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 128 = svr4_sys_setrlimit */
 	{ AS(svr4_sys_getrlimit_args), (sy_call_t *)svr4_sys_getrlimit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 129 = svr4_sys_getrlimit */
-	{ AS(lchown_args), (sy_call_t *)lchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 130 = lchown */
+	{ AS(lchown_args), (sy_call_t *)sys_lchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 130 = lchown */
 	{ AS(svr4_sys_memcntl_args), (sy_call_t *)svr4_sys_memcntl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 131 = svr4_sys_memcntl */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 132 = getpmsg */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 133 = putpmsg */
-	{ AS(rename_args), (sy_call_t *)rename, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 134 = rename */
+	{ AS(rename_args), (sy_call_t *)sys_rename, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 134 = rename */
 	{ AS(svr4_sys_uname_args), (sy_call_t *)svr4_sys_uname, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 135 = svr4_sys_uname */
-	{ AS(setegid_args), (sy_call_t *)setegid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = setegid */
+	{ AS(setegid_args), (sy_call_t *)sys_setegid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = setegid */
 	{ AS(svr4_sys_sysconfig_args), (sy_call_t *)svr4_sys_sysconfig, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 137 = svr4_sys_sysconfig */
-	{ AS(adjtime_args), (sy_call_t *)adjtime, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 138 = adjtime */
+	{ AS(adjtime_args), (sy_call_t *)sys_adjtime, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 138 = adjtime */
 	{ AS(svr4_sys_systeminfo_args), (sy_call_t *)svr4_sys_systeminfo, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 139 = svr4_sys_systeminfo */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 140 = notused */
-	{ AS(seteuid_args), (sy_call_t *)seteuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 141 = seteuid */
+	{ AS(seteuid_args), (sy_call_t *)sys_seteuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 141 = seteuid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 142 = vtrace */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 143 = { */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 144 = sigtimedwait */
@@ -176,8 +176,8 @@ struct sysent svr4_sysent[] = {
 	{ AS(svr4_sys_utimes_args), (sy_call_t *)svr4_sys_utimes, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 154 = svr4_sys_utimes */
 	{ 0, (sy_call_t *)svr4_sys_vhangup, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 155 = svr4_sys_vhangup */
 	{ AS(svr4_sys_gettimeofday_args), (sy_call_t *)svr4_sys_gettimeofday, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 156 = svr4_sys_gettimeofday */
-	{ AS(getitimer_args), (sy_call_t *)getitimer, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 157 = getitimer */
-	{ AS(setitimer_args), (sy_call_t *)setitimer, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = setitimer */
+	{ AS(getitimer_args), (sy_call_t *)sys_getitimer, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 157 = getitimer */
+	{ AS(setitimer_args), (sy_call_t *)sys_setitimer, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = setitimer */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 159 = lwp_create */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 160 = lwp_exit */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 161 = lwp_suspend */
@@ -218,11 +218,11 @@ struct sysent svr4_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 196 = timer_settime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 197 = timer_gettime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 198 = timer_overrun */
-	{ AS(nanosleep_args), (sy_call_t *)nanosleep, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 199 = nanosleep */
+	{ AS(nanosleep_args), (sy_call_t *)sys_nanosleep, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 199 = nanosleep */
 	{ AS(svr4_sys_facl_args), (sy_call_t *)svr4_sys_facl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 200 = svr4_sys_facl */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 201 = door */
-	{ AS(setreuid_args), (sy_call_t *)setreuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = setreuid */
-	{ AS(setregid_args), (sy_call_t *)setregid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = setregid */
+	{ AS(setreuid_args), (sy_call_t *)sys_setreuid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = setreuid */
+	{ AS(setregid_args), (sy_call_t *)sys_setregid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = setregid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 204 = install_utrap */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 205 = signotify */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 206 = schedctl */
@@ -250,22 +250,22 @@ struct sysent svr4_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 228 = whoknows */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 229 = whoknows */
 	{ AS(svr4_sys_socket_args), (sy_call_t *)svr4_sys_socket, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 230 = svr4_sys_socket */
-	{ AS(socketpair_args), (sy_call_t *)socketpair, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 231 = socketpair */
-	{ AS(bind_args), (sy_call_t *)bind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 232 = bind */
-	{ AS(listen_args), (sy_call_t *)listen, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 233 = listen */
-	{ AS(accept_args), (sy_call_t *)accept, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 234 = accept */
-	{ AS(connect_args), (sy_call_t *)connect, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 235 = connect */
-	{ AS(shutdown_args), (sy_call_t *)shutdown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 236 = shutdown */
+	{ AS(socketpair_args), (sy_call_t *)sys_socketpair, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 231 = socketpair */
+	{ AS(bind_args), (sy_call_t *)sys_bind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 232 = bind */
+	{ AS(listen_args), (sy_call_t *)sys_listen, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 233 = listen */
+	{ AS(accept_args), (sy_call_t *)sys_accept, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 234 = accept */
+	{ AS(connect_args), (sy_call_t *)sys_connect, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 235 = connect */
+	{ AS(shutdown_args), (sy_call_t *)sys_shutdown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 236 = shutdown */
 	{ AS(svr4_sys_recv_args), (sy_call_t *)svr4_sys_recv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 237 = svr4_sys_recv */
-	{ AS(recvfrom_args), (sy_call_t *)recvfrom, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 238 = recvfrom */
-	{ AS(recvmsg_args), (sy_call_t *)recvmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 239 = recvmsg */
+	{ AS(recvfrom_args), (sy_call_t *)sys_recvfrom, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 238 = recvfrom */
+	{ AS(recvmsg_args), (sy_call_t *)sys_recvmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 239 = recvmsg */
 	{ AS(svr4_sys_send_args), (sy_call_t *)svr4_sys_send, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 240 = svr4_sys_send */
-	{ AS(sendmsg_args), (sy_call_t *)sendmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 241 = sendmsg */
+	{ AS(sendmsg_args), (sy_call_t *)sys_sendmsg, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 241 = sendmsg */
 	{ AS(svr4_sys_sendto_args), (sy_call_t *)svr4_sys_sendto, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 242 = svr4_sys_sendto */
-	{ AS(getpeername_args), (sy_call_t *)getpeername, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 243 = getpeername */
-	{ AS(getsockname_args), (sy_call_t *)getsockname, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 244 = getsockname */
-	{ AS(getsockopt_args), (sy_call_t *)getsockopt, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 245 = getsockopt */
-	{ AS(setsockopt_args), (sy_call_t *)setsockopt, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 246 = setsockopt */
+	{ AS(getpeername_args), (sy_call_t *)sys_getpeername, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 243 = getpeername */
+	{ AS(getsockname_args), (sy_call_t *)sys_getsockname, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 244 = getsockname */
+	{ AS(getsockopt_args), (sy_call_t *)sys_getsockopt, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 245 = getsockopt */
+	{ AS(setsockopt_args), (sy_call_t *)sys_setsockopt, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 246 = setsockopt */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 247 = sockconfig */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 248 = { */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 249 = { */
diff --git a/sys/conf/Makefile.mips b/sys/conf/Makefile.mips
index ab2b40ac8c76..692807964df4 100644
--- a/sys/conf/Makefile.mips
+++ b/sys/conf/Makefile.mips
@@ -28,6 +28,8 @@ S=	../../..
 .endif
 .include "$S/conf/kern.pre.mk"
 
+INCLUDES+= -I$S/contrib/libfdt
+
 LDSCRIPT_NAME?=ldscript.$M
 SYSTEM_LD:= ${SYSTEM_LD:$S/conf/${LDSCRIPT_NAME}=${LDSCRIPT_NAME}}
 SYSTEM_DEP:= ${SYSTEM_DEP:$S/conf/${LDSCRIPT_NAME}=${LDSCRIPT_NAME}}
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index fd76593362c0..ed944942551d 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -1862,10 +1862,8 @@ device  	bmtphy		# Broadcom BCM5201/BCM5202 and 3Com 3c905C
 device  	brgphy		# Broadcom BCM54xx/57xx 1000baseTX
 device  	ciphy		# Cicada/Vitesse CS/VSC8xxx
 device  	e1000phy	# Marvell 88E1000 1000/100/10-BT
-device  	exphy		# 3Com internal PHY
 device  	gentbi		# Generic 10-bit 1000BASE-{LX,SX} fiber ifaces
 device  	icsphy		# ICS ICS1889-1893
-device  	inphy		# Intel 82553/82555
 device  	ip1000phy	# IC Plus IP1000A/IP1001
 device  	jmphy		# JMicron JMP211/JMP202
 device  	lxtphy		# Level One LXT-970
@@ -1879,7 +1877,6 @@ device  	rdcphy		# RDC Semiconductor R6040
 device  	rgephy		# RealTek 8169S/8110S/8211B/8211C
 device  	rlphy		# RealTek 8139
 device  	rlswitch	# RealTek 8305
-device  	ruephy		# RealTek RTL8150
 device  	smcphy		# SMSC LAN91C111
 device  	tdkphy		# TDK 89Q2120
 device  	tlphy		# Texas Instruments ThunderLAN
diff --git a/sys/conf/files b/sys/conf/files
index 5c5d92d6b163..90e7b76eed33 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1092,6 +1092,7 @@ dev/firewire/sbp_targ.c		optional sbp_targ
 dev/flash/at45d.c		optional at45d
 dev/flash/mx25l.c		optional mx25l
 dev/fxp/if_fxp.c		optional fxp
+dev/fxp/inphy.c			optional fxp
 dev/gem/if_gem.c		optional gem
 dev/gem/if_gem_pci.c		optional gem pci
 dev/gem/if_gem_sbus.c		optional gem sbus
@@ -1418,12 +1419,8 @@ dev/mii/bmtphy.c		optional miibus | bmtphy
 dev/mii/brgphy.c		optional miibus | brgphy
 dev/mii/ciphy.c			optional miibus | ciphy
 dev/mii/e1000phy.c		optional miibus | e1000phy
-# XXX only xl cards?
-dev/mii/exphy.c			optional miibus | exphy
 dev/mii/gentbi.c		optional miibus | gentbi
 dev/mii/icsphy.c		optional miibus | icsphy
-# XXX only fxp cards?
-dev/mii/inphy.c			optional miibus | inphy
 dev/mii/ip1000phy.c		optional miibus | ip1000phy
 dev/mii/jmphy.c			optional miibus | jmphy
 dev/mii/lxtphy.c		optional miibus | lxtphy
@@ -1440,8 +1437,6 @@ dev/mii/rdcphy.c		optional miibus | rdcphy
 dev/mii/rgephy.c		optional miibus | rgephy
 dev/mii/rlphy.c			optional miibus | rlphy
 dev/mii/rlswitch.c		optional rlswitch
-# XXX rue only?
-dev/mii/ruephy.c		optional miibus | ruephy
 dev/mii/smcphy.c		optional miibus | smcphy
 dev/mii/tdkphy.c		optional miibus | tdkphy
 dev/mii/tlphy.c			optional miibus | tlphy
@@ -1833,6 +1828,11 @@ dev/twa/tw_osl_freebsd.c	optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twe/twe.c			optional twe
 dev/twe/twe_freebsd.c		optional twe
+dev/tws/tws.c			optional tws
+dev/tws/tws_cam.c		optional tws
+dev/tws/tws_hdm.c		optional tws
+dev/tws/tws_services.c		optional tws
+dev/tws/tws_user.c		optional tws
 dev/tx/if_tx.c			optional tx
 dev/txp/if_txp.c		optional txp
 dev/uart/uart_bus_acpi.c	optional uart acpi
@@ -1919,6 +1919,7 @@ dev/usb/net/if_mos.c		optional mos
 dev/usb/net/if_rue.c		optional rue
 dev/usb/net/if_udav.c		optional udav
 dev/usb/net/if_usie.c		optional usie
+dev/usb/net/ruephy.c		optional rue
 dev/usb/net/usb_ethernet.c	optional aue | axe | cdce | cue | kue | mos | \
 					 rue | udav
 dev/usb/net/uhso.c		optional uhso
@@ -2057,6 +2058,7 @@ wpi.fw			optional wpifw					\
 dev/xe/if_xe.c			optional xe
 dev/xe/if_xe_pccard.c		optional xe pccard
 dev/xl/if_xl.c			optional xl pci
+dev/xl/xlphy.c			optional xl pci
 fs/coda/coda_fbsd.c		optional vcoda
 fs/coda/coda_psdev.c		optional vcoda
 fs/coda/coda_subr.c		optional vcoda
@@ -2548,6 +2550,7 @@ libkern/strlcpy.c		standard
 libkern/strlen.c		standard
 libkern/strncmp.c		standard
 libkern/strncpy.c		standard
+libkern/strnlen.c		standard
 libkern/strsep.c		standard
 libkern/strspn.c		standard
 libkern/strstr.c		standard
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 05fd8237b3be..588779048277 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -71,6 +71,7 @@ libkern/lshrdi3.c		standard
 libkern/memchr.c		optional	fdt
 libkern/moddi3.c		standard
 libkern/qdivrem.c		standard
+libkern/ucmpdi2.c		standard
 libkern/udivdi3.c		standard
 libkern/umoddi3.c		standard
 #XXX: We can't use these versions, as strcmp.c is included conf/files
diff --git a/sys/conf/files.mips b/sys/conf/files.mips
index 6438c492d077..b5062ae52e1e 100644
--- a/sys/conf/files.mips
+++ b/sys/conf/files.mips
@@ -78,6 +78,7 @@ libkern/ffsl.c			standard
 libkern/fls.c			standard
 libkern/flsl.c			standard
 libkern/lshrdi3.c		standard
+libkern/memchr.c		optional	fdt
 libkern/memmove.c		standard
 libkern/moddi3.c		optional	isa_mips32
 libkern/qdivrem.c		optional	isa_mips32
@@ -108,3 +109,13 @@ dev/hwpmc/hwpmc_mips24k.c	optional hwpmc
 
 dev/rt/if_rt.c			optional 	rt
 dev/nvram2env/nvram2env.c	optional	nvram2env
+
+dev/ofw/openfirm.c		optional	fdt
+dev/ofw/openfirmio.c		optional	fdt
+dev/ofw/ofw_bus_if.m		optional	fdt
+dev/ofw/ofw_if.m		optional	fdt
+dev/ofw/ofw_bus_subr.c		optional	fdt
+dev/ofw/ofw_fdt.c		optional	fdt
+
+dev/fdt/fdt_mips.c		optional	fdt
+
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index 031a1848f800..2760edf14112 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -22,6 +22,7 @@ dev/adb/adb_kbd.c		optional	adb
 dev/adb/adb_mouse.c		optional	adb
 dev/adb/adb_hb_if.m		optional	adb
 dev/adb/adb_if.m		optional	adb
+dev/adb/adb_buttons.c		optional	adb
 dev/agp/agp_apple.c		optional	agp powermac
 dev/cfi/cfi_bus_fdt.c		optional	cfi fdt
 dev/fb/fb.c			optional	sc
diff --git a/sys/conf/makeLINT.sed b/sys/conf/makeLINT.sed
index 83445cfcb705..6b3dab640af3 100644
--- a/sys/conf/makeLINT.sed
+++ b/sys/conf/makeLINT.sed
@@ -1,7 +1,7 @@
 #!/usr/bin/sed -E -n -f
 # $FreeBSD$
 
-/^(machine|ident|device|nodevice|makeoptions|nomakeoption|options|option|nooption|profile|cpu|maxusers)[[:space:]]/ {
+/^(machine|files|ident|(no)?device|(no)?makeoption(s)?|(no)?option(s)?|profile|cpu|maxusers)[[:space:]]/ {
     s/[[:space:]]*#.*$//
     p
 }
diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh
index b7d02e7a780a..36be77f2a4e8 100644
--- a/sys/conf/newvers.sh
+++ b/sys/conf/newvers.sh
@@ -31,8 +31,8 @@
 # $FreeBSD$
 
 TYPE="FreeBSD"
-REVISION="9.0"
-BRANCH="BETA2"
+REVISION="10.0"
+BRANCH="CURRENT"
 if [ "X${BRANCH_OVERRIDE}" != "X" ]; then
 	BRANCH=${BRANCH_OVERRIDE}
 fi
diff --git a/sys/contrib/pf/net/if_pflog.c b/sys/contrib/pf/net/if_pflog.c
index 0893e8df331a..349930ba1df0 100644
--- a/sys/contrib/pf/net/if_pflog.c
+++ b/sys/contrib/pf/net/if_pflog.c
@@ -429,7 +429,7 @@ static moduledata_t pflog_mod = { "pflog", pflog_modevent, 0 };
 
 #define PFLOG_MODVER 1
 
-DECLARE_MODULE(pflog, pflog_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY);
+DECLARE_MODULE(pflog, pflog_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
 MODULE_VERSION(pflog, PFLOG_MODVER);
 MODULE_DEPEND(pflog, pf, PF_MODVER, PF_MODVER, PF_MODVER);
 #endif /* __FreeBSD__ */
diff --git a/sys/contrib/pf/net/if_pfsync.c b/sys/contrib/pf/net/if_pfsync.c
index 0a093fcf2303..80714c264dc1 100644
--- a/sys/contrib/pf/net/if_pfsync.c
+++ b/sys/contrib/pf/net/if_pfsync.c
@@ -493,7 +493,7 @@ pfsync_clone_create(struct if_clone *ifc, int unit)
 	ifp->if_mtu = 1500; /* XXX */
 #ifdef __FreeBSD__
 	callout_init(&sc->sc_tmo, CALLOUT_MPSAFE);
-	callout_init(&sc->sc_bulk_tmo, CALLOUT_MPSAFE);
+	callout_init_mtx(&sc->sc_bulk_tmo, &pf_task_mtx, 0);
 	callout_init(&sc->sc_bulkfail_tmo, CALLOUT_MPSAFE);
 #else
 	ifp->if_hardmtu = MCLBYTES; /* XXX */
@@ -540,7 +540,7 @@ pfsync_clone_destroy(struct ifnet *ifp)
 #ifdef __FreeBSD__
 	EVENTHANDLER_DEREGISTER(ifnet_departure_event, sc->sc_detachtag);
 #endif
-	timeout_del(&sc->sc_bulk_tmo);
+	timeout_del(&sc->sc_bulk_tmo);	/* XXX: need PF_LOCK() before */
 	timeout_del(&sc->sc_tmo);
 #if NCARP > 0
 #ifdef notyet
@@ -714,6 +714,8 @@ pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
 	int pool_flags;
 	int error;
 
+	PF_LOCK_ASSERT();
+
 #ifdef __FreeBSD__
 	if (sp->creatorid == 0 && V_pf_status.debug >= PF_DEBUG_MISC) {
 #else
@@ -760,7 +762,7 @@ pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
 	if (flags & PFSYNC_SI_IOCTL)
 		pool_flags = PR_WAITOK | PR_ZERO;
 	else
-		pool_flags = PR_ZERO;
+		pool_flags = PR_NOWAIT | PR_ZERO;
 
 	if ((st = pool_get(&V_pf_state_pl, pool_flags)) == NULL)
 		goto cleanup;
@@ -854,7 +856,11 @@ pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
 		CLR(st->state_flags, PFSTATE_NOSYNC);
 		if (ISSET(st->state_flags, PFSTATE_ACK)) {
 			pfsync_q_ins(st, PFSYNC_S_IACK);
+#ifdef __FreeBSD__
+			pfsync_sendout();
+#else
 			schednetisr(NETISR_PFSYNC);
+#endif
 		}
 	}
 	CLR(st->state_flags, PFSTATE_ACK);
@@ -1310,7 +1316,11 @@ pfsync_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 			V_pfsyncstats.pfsyncs_stale++;
 
 			pfsync_update_state(st);
+#ifdef __FreeBSD__
+			pfsync_sendout();
+#else
 			schednetisr(NETISR_PFSYNC);
+#endif
 			continue;
 		}
 		pfsync_alloc_scrub_memory(&sp->dst, &st->dst);
@@ -1416,7 +1426,11 @@ pfsync_in_upd_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 			V_pfsyncstats.pfsyncs_stale++;
 
 			pfsync_update_state(st);
+#ifdef __FreeBSD__
+			pfsync_sendout();
+#else
 			schednetisr(NETISR_PFSYNC);
+#endif
 			continue;
 		}
 		pfsync_alloc_scrub_memory(&up->dst, &st->dst);
@@ -1469,7 +1483,9 @@ pfsync_in_ureq(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 			if (ISSET(st->state_flags, PFSTATE_NOSYNC))
 				continue;
 
+			PF_LOCK();
 			pfsync_update_state_req(st);
+			PF_UNLOCK();
 		}
 	}
 
@@ -1558,7 +1574,7 @@ pfsync_in_del_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		pf_unlink_state(st);
 	}
 #ifdef __FreeBSD__
-	PF_LOCK();
+	PF_UNLOCK();
 #endif
 	splx(s);
 
@@ -1955,7 +1971,11 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		ip->ip_hl = sizeof(sc->sc_template) >> 2;
 		ip->ip_tos = IPTOS_LOWDELAY;
 		/* len and id are set later */
+#ifdef __FreeBSD__
+		ip->ip_off = IP_DF;
+#else
 		ip->ip_off = htons(IP_DF);
+#endif
 		ip->ip_ttl = PFSYNC_DFLTTL;
 		ip->ip_p = IPPROTO_PFSYNC;
 		ip->ip_src.s_addr = INADDR_ANY;
@@ -1986,8 +2006,8 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 #endif
 				printf("pfsync: requesting bulk update\n");
 #ifdef __FreeBSD__
-				callout_reset(&sc->sc_bulkfail_tmo, 5 * hz,
-				    pfsync_bulk_fail, V_pfsyncif);
+			callout_reset(&sc->sc_bulkfail_tmo, 5 * hz,
+			    pfsync_bulk_fail, V_pfsyncif);
 #else
 			timeout_add_sec(&sc->sc_bulkfail_tmo, 5);
 #endif
@@ -2138,12 +2158,13 @@ pfsync_sendout(void)
 #endif
 #ifdef __FreeBSD__
 	size_t pktlen;
+	int dummy_error;
 #endif
 	int offset;
 	int q, count = 0;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_NET);
 #endif
@@ -2207,7 +2228,11 @@ pfsync_sendout(void)
 	bcopy(&sc->sc_template, ip, sizeof(*ip));
 	offset = sizeof(*ip);
 
+#ifdef __FreeBSD__
+	ip->ip_len = m->m_pkthdr.len;
+#else
 	ip->ip_len = htons(m->m_pkthdr.len);
+#endif
 	ip->ip_id = htons(ip_randomid());
 
 	/* build the pfsync header */
@@ -2337,35 +2362,22 @@ pfsync_sendout(void)
 #ifdef __FreeBSD__
 	sc->sc_ifp->if_opackets++;
 	sc->sc_ifp->if_obytes += m->m_pkthdr.len;
+	sc->sc_len = PFSYNC_MINPKT;
+
+	IFQ_ENQUEUE(&sc->sc_ifp->if_snd, m, dummy_error);
+	schednetisr(NETISR_PFSYNC);
 #else
 	sc->sc_if.if_opackets++;
 	sc->sc_if.if_obytes += m->m_pkthdr.len;
-#endif
 
-#ifdef __FreeBSD__
-	PF_UNLOCK();
-#endif
 	if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL) == 0)
-#ifdef __FreeBSD__
-	{
-		PF_LOCK();
-#endif
-		V_pfsyncstats.pfsyncs_opackets++;
-#ifdef __FreeBSD__
-	}
-#endif
+		pfsyncstats.pfsyncs_opackets++;
 	else
-#ifdef __FreeBSD__
-	{
-		PF_LOCK();
-#endif
-		V_pfsyncstats.pfsyncs_oerrors++;
-#ifdef __FreeBSD__
-	}
-#endif
+		pfsyncstats.pfsyncs_oerrors++;
 
 	/* start again */
 	sc->sc_len = PFSYNC_MINPKT;
+#endif
 }
 
 void
@@ -2378,7 +2390,7 @@ pfsync_insert_state(struct pf_state *st)
 #endif
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2412,7 +2424,11 @@ pfsync_insert_state(struct pf_state *st)
 	pfsync_q_ins(st, PFSYNC_S_INS);
 
 	if (ISSET(st->state_flags, PFSTATE_ACK))
+#ifdef __FreeBSD__
+		pfsync_sendout();
+#else
 		schednetisr(NETISR_PFSYNC);
+#endif
 	else
 		st->sync_updates = 0;
 }
@@ -2430,7 +2446,7 @@ pfsync_defer(struct pf_state *st, struct mbuf *m)
 	struct pfsync_deferral *pd;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2477,7 +2493,7 @@ pfsync_undefer(struct pfsync_deferral *pd, int drop)
 	int s;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2560,7 +2576,7 @@ pfsync_update_state(struct pf_state *st)
 	int sync = 0;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2609,7 +2625,11 @@ pfsync_update_state(struct pf_state *st)
 
 	if (sync || (time_second - st->pfsync_time) < 2) {
 		pfsync_upds++;
+#ifdef __FreeBSD__
+		pfsync_sendout();
+#else
 		schednetisr(NETISR_PFSYNC);
+#endif
 	}
 }
 
@@ -2625,6 +2645,8 @@ pfsync_request_update(u_int32_t creatorid, u_int64_t id)
 	size_t nlen = sizeof(struct pfsync_upd_req);
 	int s;
 
+	PF_LOCK_ASSERT();
+
 	/*
 	 * this code does nothing to prevent multiple update requests for the
 	 * same state being generated.
@@ -2658,7 +2680,11 @@ pfsync_request_update(u_int32_t creatorid, u_int64_t id)
 	TAILQ_INSERT_TAIL(&sc->sc_upd_req_list, item, ur_entry);
 	sc->sc_len += nlen;
 
+#ifdef __FreeBSD__
+	pfsync_sendout();
+#else
 	schednetisr(NETISR_PFSYNC);
+#endif
 }
 
 void
@@ -2670,6 +2696,8 @@ pfsync_update_state_req(struct pf_state *st)
 	struct pfsync_softc *sc = pfsyncif;
 #endif
 
+	PF_LOCK_ASSERT();
+
 	if (sc == NULL)
 		panic("pfsync_update_state_req: nonexistant instance");
 
@@ -2685,7 +2713,11 @@ pfsync_update_state_req(struct pf_state *st)
 		pfsync_q_del(st);
 	case PFSYNC_S_NONE:
 		pfsync_q_ins(st, PFSYNC_S_UPD);
+#ifdef __FreeBSD__
+		pfsync_sendout();
+#else
 		schednetisr(NETISR_PFSYNC);
+#endif
 		return;
 
 	case PFSYNC_S_INS:
@@ -2710,7 +2742,7 @@ pfsync_delete_state(struct pf_state *st)
 #endif
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2771,7 +2803,7 @@ pfsync_clear_states(u_int32_t creatorid, const char *ifname)
 #endif
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2801,6 +2833,8 @@ pfsync_q_ins(struct pf_state *st, int q)
 	size_t nlen = pfsync_qs[q].len;
 	int s;
 
+	PF_LOCK_ASSERT();
+
 #ifdef __FreeBSD__
 	KASSERT(st->sync_state == PFSYNC_S_NONE,
 		("%s: st->sync_state == PFSYNC_S_NONE", __FUNCTION__));
@@ -2825,13 +2859,7 @@ pfsync_q_ins(struct pf_state *st, int q)
 	if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
 #endif
 		s = splnet();
-#ifdef __FreeBSD__
-		PF_LOCK();
-#endif
 		pfsync_sendout();
-#ifdef __FreeBSD__
-		PF_UNLOCK();
-#endif
 		splx(s);
 
 		nlen = sizeof(struct pfsync_subheader) + pfsync_qs[q].len;
@@ -2888,7 +2916,9 @@ pfsync_update_tdb(struct tdb *t, int output)
 
 		if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
 			s = splnet();
+			PF_LOCK();
 			pfsync_sendout();
+			PF_UNLOCK();
 			splx(s);
 
 			nlen = sizeof(struct pfsync_subheader) +
@@ -2974,16 +3004,6 @@ pfsync_bulk_start(void)
 	struct pfsync_softc *sc = pfsyncif;
 #endif
 
-	sc->sc_ureq_received = time_uptime;
-
-	if (sc->sc_bulk_next == NULL)
-#ifdef __FreeBSD__
-		sc->sc_bulk_next = TAILQ_FIRST(&V_state_list);
-#else
-		sc->sc_bulk_next = TAILQ_FIRST(&state_list);
-#endif
-	sc->sc_bulk_last = sc->sc_bulk_next;
-
 #ifdef __FreeBSD__
 	if (V_pf_status.debug >= PF_DEBUG_MISC)
 #else
@@ -2991,8 +3011,30 @@ pfsync_bulk_start(void)
 #endif
 		printf("pfsync: received bulk update request\n");
 
-	pfsync_bulk_status(PFSYNC_BUS_START);
-	pfsync_bulk_update(sc);
+#ifdef __FreeBSD__
+	PF_LOCK();
+	if (TAILQ_EMPTY(&V_state_list))
+#else
+	if (TAILQ_EMPTY(&state_list))
+#endif
+		pfsync_bulk_status(PFSYNC_BUS_END);
+	else {
+		sc->sc_ureq_received = time_uptime;
+		if (sc->sc_bulk_next == NULL)
+#ifdef __FreeBSD__
+			sc->sc_bulk_next = TAILQ_FIRST(&V_state_list);
+#else
+			sc->sc_bulk_next = TAILQ_FIRST(&state_list);
+#endif
+			sc->sc_bulk_last = sc->sc_bulk_next;
+
+			pfsync_bulk_status(PFSYNC_BUS_START);
+			callout_reset(&sc->sc_bulk_tmo, 1,
+			    pfsync_bulk_update, sc);
+	}
+#ifdef __FreeBSD__
+	PF_UNLOCK();
+#endif
 }
 
 void
@@ -3003,12 +3045,13 @@ pfsync_bulk_update(void *arg)
 	int i = 0;
 	int s;
 
+	PF_LOCK_ASSERT();
+
 	s = splsoftnet();
 #ifdef __FreeBSD__
 	CURVNET_SET(sc->sc_ifp->if_vnet);
-	PF_LOCK();
 #endif
-	do {
+	for (;;) {
 		if (st->sync_state == PFSYNC_S_NONE &&
 		    st->timeout < PFTM_MAX &&
 		    st->pfsync_time <= sc->sc_ureq_received) {
@@ -3024,26 +3067,33 @@ pfsync_bulk_update(void *arg)
 			st = TAILQ_FIRST(&state_list);
 #endif
 
-		if (i > 0 && TAILQ_EMPTY(&sc->sc_qs[PFSYNC_S_UPD])) {
+		if (st == sc->sc_bulk_last) {
+			/* we're done */
+			sc->sc_bulk_next = NULL;
+			sc->sc_bulk_last = NULL;
+			pfsync_bulk_status(PFSYNC_BUS_END);
+			break;
+		}
+
+#ifdef __FreeBSD__
+		if (i > 1 && (sc->sc_ifp->if_mtu - sc->sc_len) <
+#else
+		if (i > 1 && (sc->sc_if.if_mtu - sc->sc_len) <
+#endif
+		    sizeof(struct pfsync_state)) {
+			/* we've filled a packet */
 			sc->sc_bulk_next = st;
 #ifdef __FreeBSD__
 			callout_reset(&sc->sc_bulk_tmo, 1,
-			    pfsync_bulk_fail, sc);
+			    pfsync_bulk_update, sc);
 #else
 			timeout_add(&sc->sc_bulk_tmo, 1);
 #endif
-			goto out;
+			break;
 		}
-	} while (st != sc->sc_bulk_last);
+	}
 
-	/* we're done */
-	sc->sc_bulk_next = NULL;
-	sc->sc_bulk_last = NULL;
-	pfsync_bulk_status(PFSYNC_BUS_END);
-
-out:
 #ifdef __FreeBSD__
-	PF_UNLOCK();
 	CURVNET_RESTORE();
 #endif
 	splx(s);
@@ -3063,6 +3113,8 @@ pfsync_bulk_status(u_int8_t status)
 	struct pfsync_softc *sc = pfsyncif;
 #endif
 
+	PF_LOCK_ASSERT();
+
 	bzero(&r, sizeof(r));
 
 	r.subh.action = PFSYNC_ACT_BUS;
@@ -3096,7 +3148,9 @@ pfsync_bulk_fail(void *arg)
 #else
 		timeout_add_sec(&sc->sc_bulkfail_tmo, 5);
 #endif
+		PF_LOCK();
 		pfsync_request_update(0, 0);
+		PF_UNLOCK();
 	} else {
 		/* Pretend like the transfer was ok */
 		sc->sc_ureq_sent = 0;
@@ -3139,19 +3193,15 @@ pfsync_send_plus(void *plus, size_t pluslen)
 #endif
 	int s;
 
+	PF_LOCK_ASSERT();
+
 #ifdef __FreeBSD__
 	if (sc->sc_len + pluslen > sc->sc_ifp->if_mtu) {
 #else
 	if (sc->sc_len + pluslen > sc->sc_if.if_mtu) {
 #endif
 		s = splnet();
-#ifdef __FreeBSD__
-		PF_LOCK();
-#endif
 		pfsync_sendout();
-#ifdef __FreeBSD__
-		PF_UNLOCK();
-#endif
 		splx(s);
 	}
 
@@ -3159,13 +3209,7 @@ pfsync_send_plus(void *plus, size_t pluslen)
 	sc->sc_len += (sc->sc_pluslen = pluslen);
 
 	s = splnet();
-#ifdef __FreeBSD__
-	PF_LOCK();
-#endif
 	pfsync_sendout();
-#ifdef __FreeBSD__
-	PF_UNLOCK();
-#endif
 	splx(s);
 }
 
@@ -3200,13 +3244,12 @@ pfsync_state_in_use(struct pf_state *st)
 	if (sc == NULL)
 		return (0);
 
-	if (st->sync_state != PFSYNC_S_NONE)
+	if (st->sync_state != PFSYNC_S_NONE ||
+	    st == sc->sc_bulk_next ||
+	    st == sc->sc_bulk_last)
 		return (1);
 
-	if (sc->sc_bulk_next == NULL && sc->sc_bulk_last == NULL)
-		return (0);
-
-	return (1);
+	return (0);
 }
 
 u_int pfsync_ints;
@@ -3245,37 +3288,38 @@ pfsync_timeout(void *arg)
 void
 #ifdef __FreeBSD__
 pfsyncintr(void *arg)
-#else
-pfsyncintr(void)
-#endif
 {
-#ifdef __FreeBSD__
 	struct pfsync_softc *sc = arg;
-#endif
-	int s;
-
-#ifdef __FreeBSD__
-	if (sc == NULL)
-		return;
+	struct mbuf *m;
 
 	CURVNET_SET(sc->sc_ifp->if_vnet);
-#endif
+	pfsync_ints++;
+
+	for (;;) {
+		IF_DEQUEUE(&sc->sc_ifp->if_snd, m);
+		if (m == 0)
+			break;
+
+		if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL)
+		    == 0)
+			V_pfsyncstats.pfsyncs_opackets++;
+		else
+			V_pfsyncstats.pfsyncs_oerrors++;
+	}
+	CURVNET_RESTORE();
+}
+#else
+pfsyncintr(void)
+{
+	int s;
+
 	pfsync_ints++;
 
 	s = splnet();
-#ifdef __FreeBSD__
-	PF_LOCK();
-#endif
 	pfsync_sendout();
-#ifdef __FreeBSD__
-	PF_UNLOCK();
-#endif
 	splx(s);
-
-#ifdef __FreeBSD__
-	CURVNET_RESTORE();
-#endif
 }
+#endif
 
 int
 pfsync_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
@@ -3380,7 +3424,7 @@ vnet_pfsync_uninit(const void *unused)
 }
 
 /* Define startup order. */
-#define	PFSYNC_SYSINIT_ORDER	SI_SUB_PROTO_BEGIN
+#define	PFSYNC_SYSINIT_ORDER	SI_SUB_PROTO_IF
 #define	PFSYNC_MODEVENT_ORDER	(SI_ORDER_FIRST) /* On boot slot in here. */
 #define	PFSYNC_VNET_ORDER	(PFSYNC_MODEVENT_ORDER + 2) /* Later still. */
 
@@ -3430,7 +3474,7 @@ static moduledata_t pfsync_mod = {
 
 #define PFSYNC_MODVER 1
 
-DECLARE_MODULE(pfsync, pfsync_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY);
+DECLARE_MODULE(pfsync, pfsync_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
 MODULE_VERSION(pfsync, PFSYNC_MODVER);
 MODULE_DEPEND(pfsync, pf, PF_MODVER, PF_MODVER, PF_MODVER);
 #endif /* __FreeBSD__ */
diff --git a/sys/contrib/pf/net/pf.c b/sys/contrib/pf/net/pf.c
index 9d4037147663..9b8ed2691ebe 100644
--- a/sys/contrib/pf/net/pf.c
+++ b/sys/contrib/pf/net/pf.c
@@ -1342,42 +1342,42 @@ pf_purge_thread(void *v)
 		tsleep(pf_purge_thread, PWAIT, "pftm", 1 * hz);
 
 #ifdef __FreeBSD__
-	sx_slock(&V_pf_consistency_lock);
-	PF_LOCK();
-	locked = 0;
-
-	if (V_pf_end_threads) {
-		PF_UNLOCK();
-		sx_sunlock(&V_pf_consistency_lock);
-		sx_xlock(&V_pf_consistency_lock);
+		sx_slock(&V_pf_consistency_lock);
 		PF_LOCK();
+		locked = 0;
 
-		pf_purge_expired_states(V_pf_status.states, 1);
-		pf_purge_expired_fragments();
-		pf_purge_expired_src_nodes(1);
-		V_pf_end_threads++;
+		if (V_pf_end_threads) {
+			PF_UNLOCK();
+			sx_sunlock(&V_pf_consistency_lock);
+			sx_xlock(&V_pf_consistency_lock);
+			PF_LOCK();
 
-		sx_xunlock(&V_pf_consistency_lock);
-		PF_UNLOCK();
-		wakeup(pf_purge_thread);
-		kproc_exit(0);
-	}
+			pf_purge_expired_states(V_pf_status.states, 1);
+			pf_purge_expired_fragments();
+			pf_purge_expired_src_nodes(1);
+			V_pf_end_threads++;
+
+			sx_xunlock(&V_pf_consistency_lock);
+			PF_UNLOCK();
+			wakeup(pf_purge_thread);
+			kproc_exit(0);
+		}
 #endif
 		s = splsoftnet();
 
 		/* process a fraction of the state table every second */
 #ifdef __FreeBSD__
-	if (!pf_purge_expired_states(1 + (V_pf_status.states /
-	    V_pf_default_rule.timeout[PFTM_INTERVAL]), 0)) {
-		PF_UNLOCK();
-		sx_sunlock(&V_pf_consistency_lock);
-		sx_xlock(&V_pf_consistency_lock);
-		PF_LOCK();
-		locked = 1;
+		if (!pf_purge_expired_states(1 + (V_pf_status.states /
+		    V_pf_default_rule.timeout[PFTM_INTERVAL]), 0)) {
+			PF_UNLOCK();
+			sx_sunlock(&V_pf_consistency_lock);
+			sx_xlock(&V_pf_consistency_lock);
+			PF_LOCK();
+			locked = 1;
 
-		pf_purge_expired_states(1 + (V_pf_status.states /
-		    V_pf_default_rule.timeout[PFTM_INTERVAL]), 1);
-	}
+			pf_purge_expired_states(1 + (V_pf_status.states /
+			    V_pf_default_rule.timeout[PFTM_INTERVAL]), 1);
+		}
 #else
 		pf_purge_expired_states(1 + (pf_status.states
 		    / pf_default_rule.timeout[PFTM_INTERVAL]));
@@ -1626,8 +1626,8 @@ pf_free_state(struct pf_state *cur)
 
 #if NPFSYNC > 0
 #ifdef __FreeBSD__
-	if (pfsync_state_in_use_ptr != NULL)
-		pfsync_state_in_use_ptr(cur);
+	if (pfsync_state_in_use_ptr != NULL &&
+		pfsync_state_in_use_ptr(cur))
 #else
 	if (pfsync_state_in_use(cur))
 #endif
diff --git a/sys/contrib/pf/net/pf_ioctl.c b/sys/contrib/pf/net/pf_ioctl.c
index 2ca163056b1f..d5a9b9231ddd 100644
--- a/sys/contrib/pf/net/pf_ioctl.c
+++ b/sys/contrib/pf/net/pf_ioctl.c
@@ -266,7 +266,7 @@ static struct cdevsw pf_cdevsw = {
 static volatile VNET_DEFINE(int, pf_pfil_hooked);
 #define V_pf_pfil_hooked	VNET(pf_pfil_hooked)
 VNET_DEFINE(int,		pf_end_threads);
-VNET_DEFINE(struct mtx,		pf_task_mtx);
+struct mtx			pf_task_mtx;
 
 /* pfsync */
 pfsync_state_import_t 		*pfsync_state_import_ptr = NULL;
@@ -287,18 +287,18 @@ SYSCTL_VNET_INT(_debug, OID_AUTO, pfugidhack, CTLFLAG_RW,
 	&VNET_NAME(debug_pfugidhack), 0,
 	"Enable/disable pf user/group rules mpsafe hack");
 
-void
+static void
 init_pf_mutex(void)
 {
 
-	mtx_init(&V_pf_task_mtx, "pf task mtx", NULL, MTX_DEF);
+	mtx_init(&pf_task_mtx, "pf task mtx", NULL, MTX_DEF);
 }
 
-void
+static void
 destroy_pf_mutex(void)
 {
 
-	mtx_destroy(&V_pf_task_mtx);
+	mtx_destroy(&pf_task_mtx);
 }
 void
 init_zone_var(void)
@@ -4259,7 +4259,7 @@ hook_pf(void)
 	struct pfil_head *pfh_inet6;
 #endif
 
-	PF_ASSERT(MA_NOTOWNED);
+	PF_UNLOCK_ASSERT();
 
 	if (V_pf_pfil_hooked)
 		return (0); 
@@ -4300,7 +4300,7 @@ dehook_pf(void)
 	struct pfil_head *pfh_inet6;
 #endif
 
-	PF_ASSERT(MA_NOTOWNED);
+	PF_UNLOCK_ASSERT();
 
 	if (V_pf_pfil_hooked == 0)
 		return (0);
@@ -4381,11 +4381,8 @@ pf_load(void)
 
 	init_zone_var();
 	sx_init(&V_pf_consistency_lock, "pf_statetbl_lock");
-	init_pf_mutex();
-	if (pfattach() < 0) {
-		destroy_pf_mutex();
+	if (pfattach() < 0)
 		return (ENOMEM);
-	}
 
 	return (0);
 }
@@ -4413,14 +4410,13 @@ pf_unload(void)
 	V_pf_end_threads = 1;
 	while (V_pf_end_threads < 2) {
 		wakeup_one(pf_purge_thread);
-		msleep(pf_purge_thread, &V_pf_task_mtx, 0, "pftmo", hz);
+		msleep(pf_purge_thread, &pf_task_mtx, 0, "pftmo", hz);
 	}
 	pfi_cleanup();
 	pf_osfp_flush();
 	pf_osfp_cleanup();
 	cleanup_pf_zone();
 	PF_UNLOCK();
-	destroy_pf_mutex();
 	sx_destroy(&V_pf_consistency_lock);
 	return error;
 }
@@ -4432,10 +4428,12 @@ pf_modevent(module_t mod, int type, void *data)
 
 	switch(type) {
 	case MOD_LOAD:
+		init_pf_mutex();
 		pf_dev = make_dev(&pf_cdevsw, 0, 0, 0, 0600, PF_NAME);
 		break;
 	case MOD_UNLOAD:
 		destroy_dev(pf_dev);
+		destroy_pf_mutex();
 		break;
 	default:
 		error = EINVAL;
@@ -4450,6 +4448,6 @@ static moduledata_t pf_mod = {
 	0
 };
 
-DECLARE_MODULE(pf, pf_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST);
+DECLARE_MODULE(pf, pf_mod, SI_SUB_PSEUDO, SI_ORDER_FIRST);
 MODULE_VERSION(pf, PF_MODVER);
 #endif /* __FreeBSD__ */
diff --git a/sys/contrib/pf/net/pf_table.c b/sys/contrib/pf/net/pf_table.c
index 010dc500593f..d40c95fdece1 100644
--- a/sys/contrib/pf/net/pf_table.c
+++ b/sys/contrib/pf/net/pf_table.c
@@ -906,7 +906,7 @@ pfr_lookup_addr(struct pfr_ktable *kt, struct pfr_addr *ad, int exact)
 		pfr_prepare_network(&mask, ad->pfra_af, ad->pfra_net);
 		s = splsoftnet(); /* rn_lookup makes use of globals */
 #ifdef __FreeBSD__
-		PF_ASSERT(MA_OWNED);
+		PF_LOCK_ASSERT();
 #endif
 		ke = (struct pfr_kentry *)rn_lookup(&sa, &mask, head);
 		splx(s);
@@ -1127,7 +1127,7 @@ pfr_route_kentry(struct pfr_ktable *kt, struct pfr_kentry *ke)
 
 	s = splsoftnet();
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #endif
 	if (KENTRY_NETWORK(ke)) {
 		pfr_prepare_network(&mask, ke->pfrke_af, ke->pfrke_net);
@@ -1166,7 +1166,7 @@ pfr_unroute_kentry(struct pfr_ktable *kt, struct pfr_kentry *ke)
 
 	s = splsoftnet();
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #endif
 	if (KENTRY_NETWORK(ke)) {
 		pfr_prepare_network(&mask, ke->pfrke_af, ke->pfrke_net);
diff --git a/sys/contrib/pf/net/pfvar.h b/sys/contrib/pf/net/pfvar.h
index 2aa4bbf5d2e4..5b47eee0929d 100644
--- a/sys/contrib/pf/net/pfvar.h
+++ b/sys/contrib/pf/net/pfvar.h
@@ -222,7 +222,7 @@ struct pfi_dynaddr {
 #define	PF_NAME		"pf"
 
 #define	PR_NOWAIT	M_NOWAIT
-#define	PR_WAITOK	M_WAIT
+#define	PR_WAITOK	M_WAITOK
 #define	PR_ZERO		M_ZERO
 #define	pool_get(p, f)	uma_zalloc(*(p), (f))
 #define	pool_put(p, o)	uma_zfree(*(p), (o))
@@ -237,33 +237,25 @@ struct pfi_dynaddr {
 		uma_zdestroy(var)
 
 #ifdef __FreeBSD__
-VNET_DECLARE(struct mtx,	 pf_task_mtx);
-#define	V_pf_task_mtx		 VNET(pf_task_mtx)
-
-#define	PF_ASSERT(h)	mtx_assert(&V_pf_task_mtx, (h))
-
-#define	PF_LOCK()	do {				\
-	PF_ASSERT(MA_NOTOWNED);				\
-	mtx_lock(&V_pf_task_mtx);			\
-} while(0)
-#define	PF_UNLOCK()	do {				\
-	PF_ASSERT(MA_OWNED);				\
-	mtx_unlock(&V_pf_task_mtx);			\
-} while(0)
-#else
 extern struct mtx pf_task_mtx;
 
-#define	PF_ASSERT(h)	mtx_assert(&pf_task_mtx, (h))
+#define	PF_LOCK_ASSERT()	mtx_assert(&pf_task_mtx, MA_OWNED)
+#define	PF_UNLOCK_ASSERT()	mtx_assert(&pf_task_mtx, MA_NOTOWNED)
 
 #define	PF_LOCK()	do {				\
-	PF_ASSERT(MA_NOTOWNED);				\
+	PF_UNLOCK_ASSERT();				\
 	mtx_lock(&pf_task_mtx);				\
 } while(0)
 #define	PF_UNLOCK()	do {				\
-	PF_ASSERT(MA_OWNED);				\
+	PF_LOCK_ASSERT();				\
 	mtx_unlock(&pf_task_mtx);			\
 } while(0)
-#endif
+#else
+#define	PF_LOCK_ASSERT()
+#define	PF_UNLOCK_ASSERT()
+#define	PF_LOCK()
+#define	PF_UNLOCK()
+#endif /* __FreeBSD__ */
 
 #define	PF_COPYIN(uaddr, kaddr, len, r)		do {	\
 	PF_UNLOCK();					\
@@ -277,9 +269,6 @@ extern struct mtx pf_task_mtx;
 	PF_LOCK();					\
 } while(0)
 
-extern void init_pf_mutex(void);
-extern void destroy_pf_mutex(void);
-
 #define	PF_MODVER	1
 #define	PFLOG_MODVER	1
 #define	PFSYNC_MODVER	1
diff --git a/sys/dev/aac/aac_cam.c b/sys/dev/aac/aac_cam.c
index 130cd20b7f3c..b5c3b2662b09 100644
--- a/sys/dev/aac/aac_cam.c
+++ b/sys/dev/aac/aac_cam.c
@@ -532,6 +532,7 @@ aac_cam_complete(struct aac_command *cm)
 	union	ccb *ccb;
 	struct 	aac_srb_response *srbr;
 	struct	aac_softc *sc;
+	int	sense_returned;
 
 	sc = cm->cm_sc;
 	fwprintf(sc, HBA_FLAGS_DBG_FUNCTION_ENTRY_B, "");
@@ -556,16 +557,17 @@ aac_cam_complete(struct aac_command *cm)
 
 			/* Take care of autosense */
 			if (srbr->sense_len) {
-				int sense_len, scsi_sense_len;
-
-				scsi_sense_len = sizeof(struct scsi_sense_data);
-				bzero(&ccb->csio.sense_data, scsi_sense_len);
-				sense_len = (srbr->sense_len >
-				    scsi_sense_len) ? scsi_sense_len :
-				    srbr->sense_len;
+				sense_returned = srbr->sense_len;
+				if (sense_returned < ccb->csio.sense_len)
+					ccb->csio.sense_resid =
+					   ccb->csio.sense_len -
+					   sense_returned;
+					else
+					    ccb->csio.sense_resid = 0;
+				bzero(&ccb->csio.sense_data,
+				    sizeof(struct scsi_sense_data));
 				bcopy(&srbr->sense[0], &ccb->csio.sense_data,
-				    srbr->sense_len);
-				ccb->csio.sense_len = sense_len;
+				    min(ccb->csio.sense_len, sense_returned));
 				ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 				// scsi_sense_print(&ccb->csio);
 			}
diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
index 1d0cc2388cdd..5047e61ed135 100644
--- a/sys/dev/acpica/acpi.c
+++ b/sys/dev/acpica/acpi.c
@@ -1238,7 +1238,6 @@ acpi_alloc_resource(device_t bus, device_t child, int type, int *rid,
     struct resource_list_entry *rle;
     struct resource_list *rl;
     struct resource *res;
-    struct rman *rm;
     int isdefault = (start == 0UL && end == ~0UL);
 
     /*
@@ -1291,15 +1290,29 @@ acpi_alloc_resource(device_t bus, device_t child, int type, int *rid,
     } else
 	res = BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type, rid,
 	    start, end, count, flags);
-    if (res != NULL || start + count - 1 != end)
-	return (res);
 
     /*
      * If the first attempt failed and this is an allocation of a
      * specific range, try to satisfy the request via a suballocation
-     * from our system resource regions.  Note that we only handle
-     * memory and I/O port system resources.
+     * from our system resource regions.
      */
+    if (res == NULL && start + count - 1 == end)
+	res = acpi_alloc_sysres(child, type, rid, start, end, count, flags);
+    return (res);
+}
+
+/*
+ * Attempt to allocate a specific resource range from the system
+ * resource ranges.  Note that we only handle memory and I/O port
+ * system resources.
+ */
+struct resource *
+acpi_alloc_sysres(device_t child, int type, int *rid, u_long start, u_long end,
+    u_long count, u_int flags)
+{
+    struct rman *rm;
+    struct resource *res;
+
     switch (type) {
     case SYS_RES_IOPORT:
 	rm = &acpi_rman_io;
@@ -1311,6 +1324,7 @@ acpi_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	return (NULL);
     }
 
+    KASSERT(start + count - 1 == end, ("wildcard resource range"));
     res = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
 	child);
     if (res == NULL)
diff --git a/sys/dev/acpica/acpi_pcib_acpi.c b/sys/dev/acpica/acpi_pcib_acpi.c
index b96ced52f901..3284fc55df7c 100644
--- a/sys/dev/acpica/acpi_pcib_acpi.c
+++ b/sys/dev/acpica/acpi_pcib_acpi.c
@@ -501,6 +501,7 @@ acpi_pcib_acpi_alloc_resource(device_t dev, device_t child, int type, int *rid,
 {
 #ifdef NEW_PCIB
     struct acpi_hpcib_softc *sc;
+    struct resource *res;
 #endif
 
 #if defined(__i386__) || defined(__amd64__)
@@ -509,8 +510,11 @@ acpi_pcib_acpi_alloc_resource(device_t dev, device_t child, int type, int *rid,
 
 #ifdef NEW_PCIB
     sc = device_get_softc(dev);
-    return (pcib_host_res_alloc(&sc->ap_host_res, child, type, rid, start, end,
-	count, flags));
+    res = pcib_host_res_alloc(&sc->ap_host_res, child, type, rid, start, end,
+	count, flags);
+    if (res == NULL && start + count - 1 == end)
+	res = acpi_alloc_sysres(child, type, rid, start, end, count, flags);
+    return (res);
 #else
     return (bus_generic_alloc_resource(dev, child, type, rid, start, end,
 	count, flags));
diff --git a/sys/dev/acpica/acpivar.h b/sys/dev/acpica/acpivar.h
index 8735776f69ff..19f8ca4fc7d3 100644
--- a/sys/dev/acpica/acpivar.h
+++ b/sys/dev/acpica/acpivar.h
@@ -382,6 +382,8 @@ ACPI_STATUS	acpi_lookup_irq_resource(device_t dev, int rid,
 		    struct resource *res, ACPI_RESOURCE *acpi_res);
 ACPI_STATUS	acpi_parse_resources(device_t dev, ACPI_HANDLE handle,
 		    struct acpi_parse_resource_set *set, void *arg);
+struct resource *acpi_alloc_sysres(device_t child, int type, int *rid,
+		    u_long start, u_long end, u_long count, u_int flags);
 
 /* ACPI event handling */
 UINT32		acpi_event_power_button_sleep(void *context);
diff --git a/sys/dev/adb/adb_buttons.c b/sys/dev/adb/adb_buttons.c
new file mode 100644
index 000000000000..92cdda169bae
--- /dev/null
+++ b/sys/dev/adb/adb_buttons.c
@@ -0,0 +1,165 @@
+/*-
+ * Copyright (c) 2011, Justin Hibbits.
+ * Copyright (c) 2002, Miodrag Vallat.
+ * Copyright (C) 1999 Tsubai Masanari.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * OpenBSD: abtn.c,v 1.12 2009/01/10 18:00:59 robert Exp
+ * NetBSD: abtn.c,v 1.1 1999/07/12 17:48:26 tsubai Exp
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/module.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+
+#include <machine/bus.h>
+
+#include <dev/ofw/openfirm.h>
+#include <machine/ofw_machdep.h>
+
+#include <dev/adb/adb.h>
+
+#define ABTN_HANDLER_ID 31
+
+struct abtn_softc {
+	device_t sc_dev;
+
+	int handler_id;
+};
+
+static int abtn_probe(device_t dev);
+static int abtn_attach(device_t dev);
+static u_int abtn_receive_packet(device_t dev, u_char status, 
+    u_char command, u_char reg, int len, u_char *data);
+
+static device_method_t abtn_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,         abtn_probe),
+        DEVMETHOD(device_attach,        abtn_attach),
+        DEVMETHOD(device_shutdown,      bus_generic_shutdown),
+        DEVMETHOD(device_suspend,       bus_generic_suspend),
+        DEVMETHOD(device_resume,        bus_generic_resume),
+
+	/* ADB interface */
+	DEVMETHOD(adb_receive_packet,	abtn_receive_packet),
+
+	{ 0, 0 }
+};
+
+static driver_t abtn_driver = {
+	"abtn",
+	abtn_methods,
+	sizeof(struct abtn_softc),
+};
+
+static devclass_t abtn_devclass;
+
+DRIVER_MODULE(abtn, adb, abtn_driver, abtn_devclass, 0, 0);
+
+static int
+abtn_probe(device_t dev)
+{
+	uint8_t type;
+
+	type = adb_get_device_type(dev);
+
+	if (type != ADB_DEVICE_MISC)
+		return (ENXIO);
+
+	device_set_desc(dev, "ADB Brightness/Volume/Eject Buttons");
+	return (0);
+}
+
+static int
+abtn_attach(device_t dev) 
+{
+	struct abtn_softc *sc;
+
+	sc = device_get_softc(dev);
+	sc->sc_dev = dev;
+
+	sc->handler_id = adb_get_device_handler(dev);
+
+	return 0;
+}
+
+static u_int
+abtn_receive_packet(device_t dev, u_char status, 
+    u_char command, u_char reg, int len, u_char *data)
+{
+	u_int cmd;
+
+	cmd = data[0];
+
+	switch (cmd) {
+	case 0x0a:	/* decrease brightness */
+		if (devctl_process_running())
+			devctl_notify("PMU", "keys", "brightness",
+			    "notify=down");
+		break;
+
+	case 0x09:	/* increase brightness */
+		if (devctl_process_running())
+			devctl_notify("PMU", "keys", "brightness", "notify=up");
+		break;
+
+	case 0x08:	/* mute */
+	case 0x01:	/* mute, AV hardware */
+		if (devctl_process_running())
+			devctl_notify("PMU", "keys", "mute", NULL);
+		break;
+	case 0x07:	/* decrease volume */
+	case 0x02:	/* decrease volume, AV hardware */
+		if (devctl_process_running())
+			devctl_notify("PMU", "keys", "volume", "notify=down");
+		break;
+	case 0x06:	/* increase volume */
+	case 0x03:	/* increase volume, AV hardware */
+		if (devctl_process_running())
+			devctl_notify("PMU", "keys", "volume", "notify=up");
+		break;
+	case 0x0c:	/* mirror display key */
+		/* Need callback to do something with this */
+		break;
+	case 0x0b:	/* eject tray */
+		if (devctl_process_running())
+			devctl_notify("PMU", "keys", "eject", NULL);
+	case 0x7f:	/* numlock */
+		/* Need callback to do something with this */
+		break;
+
+	default:
+#ifdef DEBUG
+		if ((cmd & ~0x7f) == 0)
+			device_printf(dev, "unknown ADB button 0x%x\n", cmd);
+#endif
+		break;
+	}
+	return 0;
+}
+
diff --git a/sys/dev/adb/adb_kbd.c b/sys/dev/adb/adb_kbd.c
index e66b5b21608d..e69c4db47dcb 100644
--- a/sys/dev/adb/adb_kbd.c
+++ b/sys/dev/adb/adb_kbd.c
@@ -35,12 +35,15 @@
 #include <sys/condvar.h>
 #include <sys/callout.h>
 #include <sys/kernel.h>
+#include <sys/sysctl.h>
 
 #include <machine/bus.h>
 
 #include "opt_kbd.h"
 #include <dev/kbd/kbdreg.h>
 #include <dev/kbd/kbdtables.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
@@ -55,6 +58,7 @@ static int adb_kbd_probe(device_t dev);
 static int adb_kbd_attach(device_t dev);
 static int adb_kbd_detach(device_t dev);
 static void akbd_repeat(void *xsc);
+static int adb_fn_keys(SYSCTL_HANDLER_ARGS);
 
 static u_int adb_kbd_receive_packet(device_t dev, u_char status, 
 	u_char command, u_char reg, int len, u_char *data);
@@ -282,6 +286,8 @@ adb_kbd_attach(device_t dev)
 {
 	struct adb_kbd_softc *sc;
 	keyboard_switch_t *sw;
+	uint32_t fkeys;
+	phandle_t handle;
 
 	sw = kbd_get_switch(KBD_DRIVER_NAME);
 	if (sw == NULL) {
@@ -333,6 +339,40 @@ adb_kbd_attach(device_t dev)
 
 	adb_set_autopoll(dev,1);
 
+	handle = OF_finddevice("mac-io/via-pmu/adb/keyboard");
+	if (handle != -1 && OF_getprop(handle, "AAPL,has-embedded-fn-keys",
+	    &fkeys, sizeof(fkeys)) != -1) {
+		static const char *key_names[] = {"F1", "F2", "F3", "F4", "F5",
+		    "F6", "F7", "F8", "F9", "F10", "F11", "F12"};
+		struct sysctl_ctx_list *ctx;
+		struct sysctl_oid *tree;
+		int i;
+
+		if (bootverbose)
+			device_printf(dev, "Keyboard has embedded Fn keys\n");
+
+		for (i = 0; i < 12; i++) {
+			uint32_t keyval;
+			char buf[3];
+			if (OF_getprop(handle, key_names[i], &keyval,
+			    sizeof(keyval)) < 0)
+				continue;
+			buf[0] = 1;
+			buf[1] = i+1;
+			buf[2] = keyval;
+			adb_write_register(dev, 0, 3, buf);
+		}
+		adb_write_register(dev, 1, 2, &(uint16_t){0});
+
+		ctx = device_get_sysctl_ctx(dev);
+		tree = device_get_sysctl_tree(dev);
+
+		SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		    "fn_keys_function_as_primary", CTLTYPE_INT | CTLFLAG_RW, sc,
+		    0, adb_fn_keys, "I",
+		    "Set the Fn keys to be their F-key type as default");
+	}
+
 	return (0);
 }
 
@@ -383,6 +423,13 @@ adb_kbd_receive_packet(device_t dev, u_char status,
 		return (0);
 
 	mtx_lock(&sc->sc_mutex);
+		/* 0x7f is always the power button */
+		if (data[0] == 0x7f && devctl_process_running()) {
+			devctl_notify("PMU", "Button", "pressed", NULL);
+			return (0);
+		} else if (data[0] == 0xff) {
+			return (0);	/* Ignore power button release. */
+		}
 		if ((data[0] & 0x7f) == 57 && sc->buffers < 7) {
 			/* Fake the down/up cycle for caps lock */
 			sc->buffer[sc->buffers++] = data[0] & 0x7f;
@@ -390,7 +437,6 @@ adb_kbd_receive_packet(device_t dev, u_char status,
 		} else {
 			sc->buffer[sc->buffers++] = data[0];
 		}
-
 		if (sc->buffer[sc->buffers-1] < 0xff)
 			sc->last_press = sc->buffer[sc->buffers-1];
 
@@ -814,5 +860,29 @@ akbd_modevent(module_t mod, int type, void *data)
 	return (0);
 }
 
+static int
+adb_fn_keys(SYSCTL_HANDLER_ARGS)
+{
+	struct adb_kbd_softc *sc = arg1;
+	int error;
+	uint16_t is_fn_enabled;
+	unsigned int is_fn_enabled_sysctl;
+
+	adb_read_register(sc->sc_dev, 1, &is_fn_enabled);
+	is_fn_enabled &= 1;
+	is_fn_enabled_sysctl = is_fn_enabled;
+	error = sysctl_handle_int(oidp, &is_fn_enabled_sysctl, 0, req);
+
+	if (error || !req->newptr)
+		return (error);
+
+	is_fn_enabled = is_fn_enabled_sysctl;
+	if (is_fn_enabled != 1 && is_fn_enabled != 0)
+		return (EINVAL);
+
+	adb_write_register(sc->sc_dev, 1, 2, &is_fn_enabled);
+	return (0);
+}
+
 DEV_MODULE(akbd, akbd_modevent, NULL);
 
diff --git a/sys/dev/age/if_age.c b/sys/dev/age/if_age.c
index a1ab8c7b2d13..175dd9764787 100644
--- a/sys/dev/age/if_age.c
+++ b/sys/dev/age/if_age.c
@@ -282,9 +282,9 @@ age_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc->age_miibus);
 
 	mii_pollstat(mii);
-	AGE_UNLOCK(sc);
 	ifmr->ifm_status = mii->mii_media_status;
 	ifmr->ifm_active = mii->mii_media_active;
+	AGE_UNLOCK(sc);
 }
 
 /*
diff --git a/sys/dev/ahci/ahci.c b/sys/dev/ahci/ahci.c
index 7be4cdb1f283..0a20f2c6418b 100644
--- a/sys/dev/ahci/ahci.c
+++ b/sys/dev/ahci/ahci.c
@@ -197,6 +197,7 @@ static struct {
 	{0x91231b4b, 0x11, "Marvell 88SE912x",	AHCI_Q_NOBSYRES|AHCI_Q_ALTSIG},
 	{0x91231b4b, 0x00, "Marvell 88SE912x",	AHCI_Q_EDGEIS|AHCI_Q_SATA2|AHCI_Q_NOBSYRES},
 	{0x91251b4b, 0x00, "Marvell 88SE9125",	AHCI_Q_NOBSYRES},
+	{0x91281b4b, 0x00, "Marvell 88SE9128",	AHCI_Q_NOBSYRES|AHCI_Q_ALTSIG},
 	{0x91721b4b, 0x00, "Marvell 88SE9172",	AHCI_Q_NOBSYRES},
 	{0x91821b4b, 0x00, "Marvell 88SE9182",	AHCI_Q_NOBSYRES},
 	{0x06201103, 0x00, "HighPoint RocketRAID 620",	AHCI_Q_NOBSYRES},
diff --git a/sys/dev/alc/if_alc.c b/sys/dev/alc/if_alc.c
index c13cb188dc17..c73075c6a8fc 100644
--- a/sys/dev/alc/if_alc.c
+++ b/sys/dev/alc/if_alc.c
@@ -349,9 +349,9 @@ alc_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc->alc_miibus);
 
 	mii_pollstat(mii);
-	ALC_UNLOCK(sc);
 	ifmr->ifm_status = mii->mii_media_status;
 	ifmr->ifm_active = mii->mii_media_active;
+	ALC_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/ale/if_ale.c b/sys/dev/ale/if_ale.c
index bdee4d8bb371..e80bfbbfcad1 100644
--- a/sys/dev/ale/if_ale.c
+++ b/sys/dev/ale/if_ale.c
@@ -270,9 +270,9 @@ ale_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc->ale_miibus);
 
 	mii_pollstat(mii);
-	ALE_UNLOCK(sc);
 	ifmr->ifm_status = mii->mii_media_status;
 	ifmr->ifm_active = mii->mii_media_active;
+	ALE_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/asr/asr.c b/sys/dev/asr/asr.c
index ad4143186c5d..6034b6f239bf 100644
--- a/sys/dev/asr/asr.c
+++ b/sys/dev/asr/asr.c
@@ -3071,6 +3071,14 @@ asr_intr(Asr_softc_t *sc)
 				 && (size > ccb->csio.sense_len)) {
 					size = ccb->csio.sense_len;
 				}
+				if (size < ccb->csio.sense_len) {
+					ccb->csio.sense_resid =
+					    ccb->csio.sense_len - size;
+				} else {
+					ccb->csio.sense_resid = 0;
+				}
+				bzero(&(ccb->csio.sense_data),
+				    sizeof(ccb->csio.sense_data));
 				bcopy(Reply->SenseData,
 				      &(ccb->csio.sense_data), size);
 			}
@@ -3566,6 +3574,12 @@ ASR_queue_i(Asr_softc_t	*sc, PI2O_MESSAGE_FRAME	Packet)
 		if (size > sizeof(ccb->csio.sense_data)) {
 			size = sizeof(ccb->csio.sense_data);
 		}
+		if (size < ccb->csio.sense_len) {
+			ccb->csio.sense_resid = ccb->csio.sense_len - size;
+		} else {
+			ccb->csio.sense_resid = 0;
+		}
+		bzero(&(ccb->csio.sense_data), sizeof(ccb->csio.sense_data));
 		bcopy(&(ccb->csio.sense_data), Reply_Ptr->SenseData, size);
 		I2O_SCSI_ERROR_REPLY_MESSAGE_FRAME_setAutoSenseTransferCount(
 		    Reply_Ptr, size);
diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c
index 1ef3305f617c..34453ad3d17b 100644
--- a/sys/dev/ata/ata-all.c
+++ b/sys/dev/ata/ata-all.c
@@ -120,6 +120,9 @@ SYSCTL_INT(_hw_ata, OID_AUTO, wc, CTLFLAG_RDTUN, &ata_wc, 0,
 TUNABLE_INT("hw.ata.setmax", &ata_setmax);
 SYSCTL_INT(_hw_ata, OID_AUTO, setmax, CTLFLAG_RDTUN, &ata_setmax, 0,
 	   "ATA disk set max native address");
+#ifdef ATA_CAM
+FEATURE(ata_cam, "ATA devices are accessed through the cam(4) driver");
+#endif
 
 /*
  * newbus device interface related functions
@@ -837,7 +840,7 @@ ata_boot_attach(void)
 
     mtx_lock(&Giant);       /* newbus suckage it needs Giant */
 
-    /* kick of probe and attach on all channels */
+    /* kick off probe and attach on all channels */
     for (ctlr = 0; ctlr < devclass_get_maxunit(ata_devclass); ctlr++) {
 	if ((ch = devclass_get_softc(ata_devclass, ctlr))) {
 	    ata_identify(ch->dev);
diff --git a/sys/dev/ata/ata-isa.c b/sys/dev/ata/ata-isa.c
index b6c2cb2cfdcd..c7d7d0ddd5d1 100644
--- a/sys/dev/ata/ata-isa.c
+++ b/sys/dev/ata/ata-isa.c
@@ -92,6 +92,7 @@ ata_isa_probe(device_t dev)
     bus_release_resource(dev, SYS_RES_IOPORT, ATA_CTLADDR_RID, ctlio);
     bus_release_resource(dev, SYS_RES_IOPORT, ATA_IOADDR_RID, io);
 
+    device_set_desc(dev, "ATA channel");
     return (ata_probe(dev));
 }
 
diff --git a/sys/dev/ata/ata-pci.c b/sys/dev/ata/ata-pci.c
index 374612ec88b4..b0359e6c9d39 100644
--- a/sys/dev/ata/ata-pci.c
+++ b/sys/dev/ata/ata-pci.c
@@ -544,6 +544,19 @@ ata_pci_dmafini(device_t dev)
     ata_dmafini(dev);
 }
 
+int
+ata_pci_print_child(device_t dev, device_t child)
+{
+	int retval;
+
+	retval = bus_print_child_header(dev, child);
+	retval += printf(" at channel %d",
+	    (int)(intptr_t)device_get_ivars(child));
+	retval += bus_print_child_footer(dev, child);
+
+	return (retval);
+}
+
 int
 ata_pci_child_location_str(device_t dev, device_t child, char *buf,
     size_t buflen)
@@ -574,6 +587,7 @@ static device_method_t ata_pci_methods[] = {
     DEVMETHOD(bus_teardown_intr,        ata_pci_teardown_intr),
     DEVMETHOD(pci_read_config,		ata_pci_read_config),
     DEVMETHOD(pci_write_config,		ata_pci_write_config),
+    DEVMETHOD(bus_print_child,		ata_pci_print_child),
     DEVMETHOD(bus_child_location_str,	ata_pci_child_location_str),
 
     { 0, 0 }
@@ -594,12 +608,10 @@ MODULE_DEPEND(atapci, ata, 1, 1, 1);
 static int
 ata_pcichannel_probe(device_t dev)
 {
-    char buffer[32];
 
     if ((intptr_t)device_get_ivars(dev) < 0)
 	    return (ENXIO);
-    sprintf(buffer, "ATA channel %d", (int)(intptr_t)device_get_ivars(dev));
-    device_set_desc_copy(dev, buffer);
+    device_set_desc(dev, "ATA channel");
 
     return ata_probe(dev);
 }
diff --git a/sys/dev/ata/ata-pci.h b/sys/dev/ata/ata-pci.h
index 3b90498e9617..2aa206584607 100644
--- a/sys/dev/ata/ata-pci.h
+++ b/sys/dev/ata/ata-pci.h
@@ -549,6 +549,7 @@ int ata_pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
 uint32_t ata_pci_read_config(device_t dev, device_t child, int reg, int width);
 void ata_pci_write_config(device_t dev, device_t child, int reg, 
     uint32_t val, int width);
+int ata_pci_print_child(device_t dev, device_t child);
 int ata_pci_child_location_str(device_t dev, device_t child, char *buf,
     size_t buflen);
 struct resource * ata_pci_alloc_resource(device_t dev, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags);
@@ -601,6 +602,7 @@ static device_method_t __CONCAT(dname,_methods)[] = { \
     DEVMETHOD(bus_teardown_intr,        ata_pci_teardown_intr), \
     DEVMETHOD(pci_read_config,		ata_pci_read_config), \
     DEVMETHOD(pci_write_config,		ata_pci_write_config), \
+    DEVMETHOD(bus_print_child,		ata_pci_print_child), \
     DEVMETHOD(bus_child_location_str,	ata_pci_child_location_str), \
     { 0, 0 } \
 }; \
diff --git a/sys/dev/ath/ah_osdep.c b/sys/dev/ath/ah_osdep.c
index cceb9a4a4c83..54dfb731af95 100644
--- a/sys/dev/ath/ah_osdep.c
+++ b/sys/dev/ath/ah_osdep.c
@@ -128,7 +128,7 @@ void
 DO_HALDEBUG(struct ath_hal *ah, u_int mask, const char* fmt, ...)
 {
 	if ((mask == HAL_DEBUG_UNMASKABLE) ||
-	    (ah->ah_config.ah_debug & mask) ||
+	    (ah != NULL && ah->ah_config.ah_debug & mask) ||
 	    (ath_hal_debug & mask)) {
 		__va_list ap;
 		va_start(ap, fmt);
diff --git a/sys/dev/ath/ath_dfs/null/dfs_null.c b/sys/dev/ath/ath_dfs/null/dfs_null.c
index a3bae9145856..75574b4e4ac1 100644
--- a/sys/dev/ath/ath_dfs/null/dfs_null.c
+++ b/sys/dev/ath/ath_dfs/null/dfs_null.c
@@ -128,7 +128,7 @@ ath_dfs_process_radar_event(struct ath_softc *sc,
 }
 
 /*
- * Determine whether the the DFS check task needs to be queued.
+ * Determine whether the DFS check task needs to be queued.
  *
  * This is called in the RX task when the current batch of packets
  * have been received. It will return whether there are any radar
diff --git a/sys/dev/ath/ath_hal/ah.c b/sys/dev/ath/ath_hal/ah.c
index 4055079dc71c..383ae8fa4138 100644
--- a/sys/dev/ath/ath_hal/ah.c
+++ b/sys/dev/ath/ath_hal/ah.c
@@ -659,6 +659,8 @@ ath_hal_getcapability(struct ath_hal *ah, HAL_CAPABILITY_TYPE type,
 		return pCap->halHasRxSelfLinkedTail ? HAL_OK : HAL_ENOTSUPP;
 	case HAL_CAP_LONG_RXDESC_TSF:		/* 32 bit TSF in RX descriptor? */
 		return pCap->halHasLongRxDescTsf ? HAL_OK : HAL_ENOTSUPP;
+	case HAL_CAP_BB_READ_WAR:		/* Baseband read WAR */
+		return pCap->halHasBBReadWar? HAL_OK : HAL_ENOTSUPP;
 	default:
 		return HAL_EINVAL;
 	}
diff --git a/sys/dev/ath/ath_hal/ah.h b/sys/dev/ath/ath_hal/ah.h
index f4254c14fa87..a5d915675a95 100644
--- a/sys/dev/ath/ath_hal/ah.h
+++ b/sys/dev/ath/ath_hal/ah.h
@@ -149,6 +149,7 @@ typedef enum {
 	HAL_CAP_STREAMS		= 239,	/* how many 802.11n spatial streams are available */
 	HAL_CAP_RXDESC_SELFLINK	= 242,	/* support a self-linked tail RX descriptor */
 	HAL_CAP_LONG_RXDESC_TSF	= 243,	/* hardware supports 32bit TSF in RX descriptor */
+	HAL_CAP_BB_READ_WAR	= 244,	/* baseband read WAR */
 } HAL_CAPABILITY_TYPE;
 
 /* 
@@ -1012,8 +1013,12 @@ struct ath_hal {
 	void	  __ahdecl(*ah_set11nRateScenario)(struct ath_hal *,
 	    			struct ath_desc *, u_int, u_int,
 				HAL_11N_RATE_SERIES [], u_int, u_int);
+	void	  __ahdecl(*ah_set11nAggrFirst)(struct ath_hal *,
+				struct ath_desc *, u_int, u_int);
 	void	  __ahdecl(*ah_set11nAggrMiddle)(struct ath_hal *,
 	    			struct ath_desc *, u_int);
+	void	  __ahdecl(*ah_set11nAggrLast)(struct ath_hal *,
+				struct ath_desc *);
 	void	  __ahdecl(*ah_clr11nAggr)(struct ath_hal *,
 	    			struct ath_desc *);
 	void	  __ahdecl(*ah_set11nBurstDuration)(struct ath_hal *,
diff --git a/sys/dev/ath/ath_hal/ah_desc.h b/sys/dev/ath/ath_hal/ah_desc.h
index 4651fc2d7cc0..2ef7b85f4c08 100644
--- a/sys/dev/ath/ath_hal/ah_desc.h
+++ b/sys/dev/ath/ath_hal/ah_desc.h
@@ -50,6 +50,7 @@ struct ath_tx_status {
 /* #define ts_rssi ts_rssi_combined */
 	uint32_t   	ts_ba_low;	/* blockack bitmap low */
 	uint32_t   	ts_ba_high;	/* blockack bitmap high */
+	uint8_t		ts_tid;		/* TID */
 	uint32_t  	ts_evm0;	/* evm bytes */
 	uint32_t   	ts_evm1;
 	uint32_t   	ts_evm2;
diff --git a/sys/dev/ath/ath_hal/ah_internal.h b/sys/dev/ath/ath_hal/ah_internal.h
index 79184bd23650..4378c83321dd 100644
--- a/sys/dev/ath/ath_hal/ah_internal.h
+++ b/sys/dev/ath/ath_hal/ah_internal.h
@@ -209,7 +209,8 @@ typedef struct {
 			hal4kbSplitTransSupport		: 1,
 			halHasRxSelfLinkedTail		: 1,
 			halSupportsFastClock5GHz	: 1,	/* Hardware supports 5ghz fast clock; check eeprom/channel before using */
-			halHasLongRxDescTsf		: 1;
+			halHasLongRxDescTsf		: 1,
+			halHasBBReadWar			: 1;
 	uint32_t	halWirelessModes;
 	uint16_t	halTotalQueues;
 	uint16_t	halKeyCacheSize;
@@ -472,6 +473,8 @@ isBigEndian(void)
 	OS_REG_WRITE(_a, _r, OS_REG_READ(_a, _r) | (_f))
 #define	OS_REG_CLR_BIT(_a, _r, _f) \
 	OS_REG_WRITE(_a, _r, OS_REG_READ(_a, _r) &~ (_f))
+#define OS_REG_IS_BIT_SET(_a, _r, _f) \
+	    ((OS_REG_READ(_a, _r) & (_f)) != 0)
 
 /* Analog register writes may require a delay between each one (eg Merlin?) */
 #define	OS_A_REG_RMW_FIELD(_a, _r, _f, _v) \
@@ -503,26 +506,15 @@ extern	void ath_hal_free(void *);
 extern	int ath_hal_debug;	/* Global debug flags */
 
 /*
- * This is used for global debugging, when ahp doesn't yet have the
- * related debugging state. For example, during probe/attach.
- */
-#define	HALDEBUG_G(_ah, __m, ...) \
-	do {							\
-		if ((__m) == HAL_DEBUG_UNMASKABLE ||		\
-		    ath_hal_debug & (__m)) {			\
-			DO_HALDEBUG((_ah), (__m), __VA_ARGS__);	\
-		}						\
-	} while (0);
-
-/*
- * This is used for local debugging, when ahp isn't NULL and
- * thus may have debug flags set.
+ * The typecast is purely because some callers will pass in
+ * AH_NULL directly rather than using a NULL ath_hal pointer.
  */
 #define	HALDEBUG(_ah, __m, ...) \
 	do {							\
 		if ((__m) == HAL_DEBUG_UNMASKABLE ||		\
 		    ath_hal_debug & (__m) ||			\
-		    (_ah)->ah_config.ah_debug & (__m)) {	\
+		    ((_ah) != NULL &&				\
+		      ((struct ath_hal *) (_ah))->ah_config.ah_debug & (__m))) {	\
 			DO_HALDEBUG((_ah), (__m), __VA_ARGS__);	\
 		}						\
 	} while(0);
@@ -531,7 +523,6 @@ extern	void DO_HALDEBUG(struct ath_hal *ah, u_int mask, const char* fmt, ...)
 	__printflike(3,4);
 #else
 #define HALDEBUG(_ah, __m, ...)
-#define HALDEBUG_G(_ah, __m, ...)
 #endif /* AH_DEBUG */
 
 /*
diff --git a/sys/dev/ath/ath_hal/ah_regdomain.c b/sys/dev/ath/ath_hal/ah_regdomain.c
index 7298c6bee8af..13f3e428d533 100644
--- a/sys/dev/ath/ath_hal/ah_regdomain.c
+++ b/sys/dev/ath/ath_hal/ah_regdomain.c
@@ -169,7 +169,7 @@ isEepromValid(struct ath_hal *ah)
 			if (regDomainPairs[i].regDmnEnum == rd)
 				return AH_TRUE;
 	}
-	HALDEBUG_G(ah, HAL_DEBUG_REGDOMAIN,
+	HALDEBUG(ah, HAL_DEBUG_REGDOMAIN,
 	    "%s: invalid regulatory domain/country code 0x%x\n", __func__, rd);
 	return AH_FALSE;
 }
@@ -613,7 +613,7 @@ ath_hal_mapgsm(int sku, int freq)
 		return 1544 + freq;
 	if (sku == SKU_SR9)
 		return 3344 - freq;
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+	HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 	    "%s: cannot map freq %u unknown gsm sku %u\n",
 	    __func__, freq, sku);
 	return freq;
diff --git a/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c b/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c
index 79c305511e6f..c624e7a954b3 100644
--- a/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c
+++ b/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c
@@ -182,14 +182,14 @@ ar5210Attach(uint16_t devid, HAL_SOFTC sc, HAL_BUS_TAG st, HAL_BUS_HANDLE sh,
 	HAL_STATUS ecode;
 	int i;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH,
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH,
 	    "%s: devid 0x%x sc %p st %p sh %p\n", __func__, devid,
 	    sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp = ath_hal_malloc(sizeof (struct ath_hal_5210));
 	if (ahp == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: no memory for state block\n", __func__);
 		ecode = HAL_ENOMEM;
 		goto bad;
diff --git a/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c b/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c
index a0c42b2bf7c4..e9def4487cda 100644
--- a/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c
+++ b/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c
@@ -201,13 +201,13 @@ ar5211Attach(uint16_t devid, HAL_SOFTC sc,
 	uint16_t eeval;
 	HAL_STATUS ecode;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp = ath_hal_malloc(sizeof (struct ath_hal_5211));
 	if (ahp == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		ecode = HAL_ENOMEM;
 		goto bad;
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5112.c b/sys/dev/ath/ath_hal/ar5212/ar5112.c
index 10030682a2e8..c1920b935fd0 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5112.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5112.c
@@ -611,7 +611,7 @@ getFullPwrTable(uint16_t numPcdacs, uint16_t *pcdacs, int16_t *power, int16_t ma
 	uint16_t    idxR = 1;
 
 	if (numPcdacs < 2) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		     "%s: at least 2 pcdac values needed [%d]\n",
 		     __func__, numPcdacs);
 		return AH_FALSE;
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212.h b/sys/dev/ath/ath_hal/ar5212/ar5212.h
index 40e718e46736..ec9119382dd8 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212.h
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212.h
@@ -252,6 +252,7 @@ struct ath_hal_5212 {
 	uint8_t		ah_macaddr[IEEE80211_ADDR_LEN];
 	uint8_t		ah_bssid[IEEE80211_ADDR_LEN];
 	uint8_t		ah_bssidmask[IEEE80211_ADDR_LEN];
+	uint16_t	ah_assocId;
 
 	/*
 	 * Runtime state.
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c b/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c
index eaceaba766e3..b2a630c37aea 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c
@@ -319,13 +319,13 @@ ar5212Attach(uint16_t devid, HAL_SOFTC sc,
 	uint16_t eeval;
 	HAL_STATUS ecode;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp = ath_hal_malloc(sizeof (struct ath_hal_5212));
 	if (ahp == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_beacon.c b/sys/dev/ath/ath_hal/ar5212/ar5212_beacon.c
index 1b4b342bc4ae..a411d3181f50 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212_beacon.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212_beacon.c
@@ -44,6 +44,7 @@ ar5212GetNextTBTT(struct ath_hal *ah)
 void
 ar5212SetBeaconTimers(struct ath_hal *ah, const HAL_BEACON_TIMERS *bt)
 {
+	struct ath_hal_5212 *ahp = AH5212(ah);
 
 	OS_REG_WRITE(ah, AR_TIMER0, bt->bt_nexttbtt);
 	OS_REG_WRITE(ah, AR_TIMER1, bt->bt_nextdba);
@@ -65,6 +66,7 @@ ar5212SetBeaconTimers(struct ath_hal *ah, const HAL_BEACON_TIMERS *bt)
 		OS_REG_WRITE(ah, AR_BEACON, AR_BEACON_RESET_TSF);
 	}
 	OS_REG_WRITE(ah, AR_BEACON, bt->bt_intval);
+	ahp->ah_beaconInterval = (bt->bt_intval & HAL_BEACON_PERIOD);
 }
 
 /*
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_misc.c b/sys/dev/ath/ath_hal/ar5212/ar5212_misc.c
index 5bcde762a5d4..0ea42d085e5d 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212_misc.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212_misc.c
@@ -218,8 +218,9 @@ ar5212WriteAssocid(struct ath_hal *ah, const uint8_t *bssid, uint16_t assocId)
 {
 	struct ath_hal_5212 *ahp = AH5212(ah);
 
-	/* XXX save bssid for possible re-use on reset */
+	/* save bssid for possible re-use on reset */
 	OS_MEMCPY(ahp->ah_bssid, bssid, IEEE80211_ADDR_LEN);
+	ahp->ah_assocId = assocId;
 	OS_REG_WRITE(ah, AR_BSS_ID0, LE_READ_4(ahp->ah_bssid));
 	OS_REG_WRITE(ah, AR_BSS_ID1, LE_READ_2(ahp->ah_bssid+4) |
 				     ((assocId & 0x3fff)<<AR_BSS_ID1_AID_S));
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_reset.c b/sys/dev/ath/ath_hal/ar5212/ar5212_reset.c
index 8e6341a04c72..67076b845bf1 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212_reset.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212_reset.c
@@ -95,6 +95,16 @@ write_common(struct ath_hal *ah, const HAL_INI_ARRAY *ia,
 
 #define IS_DISABLE_FAST_ADC_CHAN(x) (((x) == 2462) || ((x) == 2467))
 
+/*
+ * XXX NDIS 5.x code had MAX_RESET_WAIT set to 2000 for AP code
+ * and 10 for Client code
+ */
+#define	MAX_RESET_WAIT			10
+
+#define	TX_QUEUEPEND_CHECK		1
+#define	TX_ENABLE_CHECK			2
+#define	RX_ENABLE_CHECK			4
+
 /*
  * Places the device in and out of reset and then places sane
  * values in the registers based on EEPROM config, initialization
@@ -428,9 +438,10 @@ ar5212Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	/* Restore previous antenna */
 	OS_REG_WRITE(ah, AR_DEF_ANTENNA, saveDefAntenna);
 
-	/* then our BSSID */
+	/* then our BSSID and associate id */
 	OS_REG_WRITE(ah, AR_BSS_ID0, LE_READ_4(ahp->ah_bssid));
-	OS_REG_WRITE(ah, AR_BSS_ID1, LE_READ_2(ahp->ah_bssid + 4));
+	OS_REG_WRITE(ah, AR_BSS_ID1, LE_READ_2(ahp->ah_bssid + 4) |
+	    (ahp->ah_assocId & 0x3fff) << AR_BSS_ID1_AID_S);
 
 	/* Restore bmiss rssi & count thresholds */
 	OS_REG_WRITE(ah, AR_RSSI_THR, ahp->ah_rssiThr);
@@ -1104,6 +1115,76 @@ ar5212ResetCalValid(struct ath_hal *ah, const struct ieee80211_channel *chan)
 	return AH_TRUE;
 }
 
+/**************************************************************
+ * ar5212MacStop
+ *
+ * Disables all active QCUs and ensure that the mac is in a
+ * quiessence state.
+ */
+static HAL_BOOL
+ar5212MacStop(struct ath_hal *ah)
+{
+	HAL_BOOL     status;
+	uint32_t    count;
+	uint32_t    pendFrameCount;
+	uint32_t    macStateFlag;
+	uint32_t    queue;
+
+	status = AH_FALSE;
+
+	/* Disable Rx Operation ***********************************/
+	OS_REG_SET_BIT(ah, AR_CR, AR_CR_RXD);
+
+	/* Disable TX Operation ***********************************/
+#ifdef NOT_YET
+	ar5212SetTxdpInvalid(ah);
+#endif
+	OS_REG_SET_BIT(ah, AR_Q_TXD, AR_Q_TXD_M);
+
+	/* Polling operation for completion of disable ************/
+	macStateFlag = TX_ENABLE_CHECK | RX_ENABLE_CHECK;
+
+	for (count = 0; count < MAX_RESET_WAIT; count++) {
+		if (macStateFlag & RX_ENABLE_CHECK) {
+			if (!OS_REG_IS_BIT_SET(ah, AR_CR, AR_CR_RXE)) {
+				macStateFlag &= ~RX_ENABLE_CHECK;
+			}
+		}
+
+		if (macStateFlag & TX_ENABLE_CHECK) {
+			if (!OS_REG_IS_BIT_SET(ah, AR_Q_TXE, AR_Q_TXE_M)) {
+				macStateFlag &= ~TX_ENABLE_CHECK;
+				macStateFlag |= TX_QUEUEPEND_CHECK;
+			}
+		}
+		if (macStateFlag & TX_QUEUEPEND_CHECK) {
+			pendFrameCount = 0;
+			for (queue = 0; queue < AR_NUM_DCU; queue++) {
+				pendFrameCount += OS_REG_READ(ah,
+				    AR_Q0_STS + (queue * 4)) &
+				    AR_Q_STS_PEND_FR_CNT;
+			}
+			if (pendFrameCount == 0) {
+				macStateFlag &= ~TX_QUEUEPEND_CHECK;
+			}
+		}
+		if (macStateFlag == 0) {
+			status = AH_TRUE;
+			break;
+		}
+		OS_DELAY(50);
+	}
+
+	if (status != AH_TRUE) {
+		HALDEBUG(ah, HAL_DEBUG_RESET,
+		    "%s:Failed to stop the MAC state 0x%x\n",
+		    __func__, macStateFlag);
+	}
+
+	return status;
+}
+
+
 /*
  * Write the given reset bit mask into the reset register
  */
@@ -1113,12 +1194,75 @@ ar5212SetResetReg(struct ath_hal *ah, uint32_t resetMask)
 	uint32_t mask = resetMask ? resetMask : ~0;
 	HAL_BOOL rt;
 
-	/* XXX ar5212MacStop & co. */
-
+	/* Never reset the PCIE core */
 	if (AH_PRIVATE(ah)->ah_ispcie) {
 		resetMask &= ~AR_RC_PCI;
 	}
 
+	if (resetMask & (AR_RC_MAC | AR_RC_PCI)) {
+		/*
+		 * To ensure that the driver can reset the
+		 * MAC, wake up the chip
+		 */
+		rt = ar5212SetPowerMode(ah, HAL_PM_AWAKE, AH_TRUE);
+
+		if (rt != AH_TRUE) {
+			return rt;
+		}
+
+		/*
+		 * Disable interrupts
+		 */
+		OS_REG_WRITE(ah, AR_IER, AR_IER_DISABLE);
+		OS_REG_READ(ah, AR_IER);
+
+		if (ar5212MacStop(ah) != AH_TRUE) {
+			/*
+			 * Failed to stop the MAC gracefully; let's be more forceful then
+			 */
+
+			/* need some delay before flush any pending MMR writes */
+			OS_DELAY(15);
+			OS_REG_READ(ah, AR_RXDP);
+
+			resetMask |= AR_RC_MAC | AR_RC_BB;
+			/* _Never_ reset PCI Express core */
+			if (! AH_PRIVATE(ah)->ah_ispcie) {
+				resetMask |= AR_RC_PCI;
+			}
+#if 0
+			/*
+			 * Flush the park address of the PCI controller
+			*/
+			/* Read PCI slot information less than Hainan revision */
+			if (AH_PRIVATE(ah)->ah_bustype == HAL_BUS_TYPE_PCI) {
+				if (!IS_5112_REV5_UP(ah)) {
+#define PCI_COMMON_CONFIG_STATUS    0x06
+					u_int32_t    i;
+					u_int16_t    reg16;
+
+					for (i = 0; i < 32; i++) {
+						ath_hal_read_pci_config_space(ah,
+						    PCI_COMMON_CONFIG_STATUS,
+						    &reg16, sizeof(reg16));
+					}
+				}
+#undef PCI_COMMON_CONFIG_STATUS
+			}
+#endif
+		} else {
+			/*
+			 * MAC stopped gracefully; no need to warm-reset the PCI bus
+			 */
+
+			resetMask &= ~AR_RC_PCI;
+
+			/* need some delay before flush any pending MMR writes */
+			OS_DELAY(15);
+			OS_REG_READ(ah, AR_RXDP);
+		}
+	}
+
 	(void) OS_REG_READ(ah, AR_RXDP);/* flush any pending MMR writes */
 	OS_REG_WRITE(ah, AR_RC, resetMask);
 	OS_DELAY(15);			/* need to wait at least 128 clocks
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_xmit.c b/sys/dev/ath/ath_hal/ar5212/ar5212_xmit.c
index 3c6ddc0035fe..008330aa0965 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212_xmit.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212_xmit.c
@@ -263,6 +263,7 @@ ar5212ReleaseTxQueue(struct ath_hal *ah, u_int q)
  * Assumes:
  *  phwChannel has been set to point to the current channel
  */
+#define	TU_TO_USEC(_tu)		((_tu) << 10)
 HAL_BOOL
 ar5212ResetTxQueue(struct ath_hal *ah, u_int q)
 {
@@ -270,7 +271,7 @@ ar5212ResetTxQueue(struct ath_hal *ah, u_int q)
 	HAL_CAPABILITIES *pCap = &AH_PRIVATE(ah)->ah_caps;
 	const struct ieee80211_channel *chan = AH_PRIVATE(ah)->ah_curchan;
 	HAL_TX_QUEUE_INFO *qi;
-	uint32_t cwMin, chanCwMin, value, qmisc, dmisc;
+	uint32_t cwMin, chanCwMin, qmisc, dmisc;
 
 	if (q >= pCap->halTotalQueues) {
 		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: invalid queue num %u\n",
@@ -409,17 +410,40 @@ ar5212ResetTxQueue(struct ath_hal *ah, u_int q)
 		      |  AR_Q_MISC_CBR_INCR_DIS1
 		      |  AR_Q_MISC_CBR_INCR_DIS0;
 
-		if (!qi->tqi_readyTime) {
+		if (qi->tqi_readyTime) {
+			HALDEBUG(ah, HAL_DEBUG_TXQUEUE,
+			    "%s: using tqi_readyTime\n", __func__);
+			OS_REG_WRITE(ah, AR_QRDYTIMECFG(q),
+			    SM(qi->tqi_readyTime, AR_Q_RDYTIMECFG_INT) |
+			    AR_Q_RDYTIMECFG_ENA);
+		} else {
+			int value;
 			/*
 			 * NB: don't set default ready time if driver
 			 * has explicitly specified something.  This is
 			 * here solely for backwards compatibility.
 			 */
-			value = (ahp->ah_beaconInterval
+			/*
+			 * XXX for now, hard-code a CAB interval of 70%
+			 * XXX of the total beacon interval.
+			 */
+
+			value = (ahp->ah_beaconInterval * 70 / 100)
 				- (ah->ah_config.ah_sw_beacon_response_time -
-					ah->ah_config.ah_dma_beacon_response_time)
-				- ah->ah_config.ah_additional_swba_backoff) * 1024;
-			OS_REG_WRITE(ah, AR_QRDYTIMECFG(q), value | AR_Q_RDYTIMECFG_ENA);
+				+ ah->ah_config.ah_dma_beacon_response_time)
+				- ah->ah_config.ah_additional_swba_backoff;
+			/*
+			 * XXX Ensure it isn't too low - nothing lower
+			 * XXX than 10 TU
+			 */
+			if (value < 10)
+				value = 10;
+			HALDEBUG(ah, HAL_DEBUG_TXQUEUE,
+			    "%s: defaulting to rdytime = %d uS\n",
+			    __func__, value);
+			OS_REG_WRITE(ah, AR_QRDYTIMECFG(q),
+			    SM(TU_TO_USEC(value), AR_Q_RDYTIMECFG_INT) |
+			    AR_Q_RDYTIMECFG_ENA);
 		}
 		dmisc |= SM(AR_D_MISC_ARB_LOCKOUT_CNTRL_GLOBAL,
 			    AR_D_MISC_ARB_LOCKOUT_CNTRL);
@@ -481,6 +505,7 @@ ar5212ResetTxQueue(struct ath_hal *ah, u_int q)
 
 	return AH_TRUE;
 }
+#undef	TU_TO_USEC
 
 /*
  * Get the TXDP for the specified queue
@@ -796,12 +821,14 @@ ar5212FillTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 		 * copy the multi-rate transmit parameters from
 		 * the first frame for processing on completion. 
 		 */
-		ads->ds_ctl0 = 0;
 		ads->ds_ctl1 = segLen;
 #ifdef AH_NEED_DESC_SWAP
+		ads->ds_ctl0 = __bswap32(AR5212DESC_CONST(ds0)->ds_ctl0)
+		    & AR_TxInterReq;
 		ads->ds_ctl2 = __bswap32(AR5212DESC_CONST(ds0)->ds_ctl2);
 		ads->ds_ctl3 = __bswap32(AR5212DESC_CONST(ds0)->ds_ctl3);
 #else
+		ads->ds_ctl0 = AR5212DESC_CONST(ds0)->ds_ctl0 & AR_TxInterReq;
 		ads->ds_ctl2 = AR5212DESC_CONST(ds0)->ds_ctl2;
 		ads->ds_ctl3 = AR5212DESC_CONST(ds0)->ds_ctl3;
 #endif
@@ -809,7 +836,12 @@ ar5212FillTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 		/*
 		 * Intermediate descriptor in a multi-descriptor frame.
 		 */
-		ads->ds_ctl0 = 0;
+#ifdef AH_NEED_DESC_SWAP
+		ads->ds_ctl0 = __bswap32(AR5212DESC_CONST(ds0)->ds_ctl0)
+		    & AR_TxInterReq;
+#else
+		ads->ds_ctl0 = AR5212DESC_CONST(ds0)->ds_ctl0 & AR_TxInterReq;
+#endif
 		ads->ds_ctl1 = segLen | AR_More;
 		ads->ds_ctl2 = 0;
 		ads->ds_ctl3 = 0;
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212reg.h b/sys/dev/ath/ath_hal/ar5212/ar5212reg.h
index 15c1a58f7df1..96dfa9d58061 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212reg.h
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212reg.h
@@ -94,7 +94,9 @@
 #define	AR_QTXDP(_i)	(AR_Q0_TXDP + ((_i)<<2))
 
 #define	AR_Q_TXE	0x0840	/* MAC Transmit Queue enable */
+#define	AR_Q_TXE_M	0x000003FF	/* Mask for TXE (QCU 0-9) */
 #define	AR_Q_TXD	0x0880	/* MAC Transmit Queue disable */
+#define	AR_Q_TXD_M	0x000003FF	/* Mask for TXD (QCU 0-9) */
 
 #define	AR_Q0_CBRCFG	0x08c0	/* MAC CBR configuration */
 #define	AR_Q1_CBRCFG	0x08c4	/* MAC CBR configuration */
diff --git a/sys/dev/ath/ath_hal/ar5312/ar5312_attach.c b/sys/dev/ath/ath_hal/ar5312/ar5312_attach.c
index d1689d59fc61..4ca1a4d0b7df 100644
--- a/sys/dev/ath/ath_hal/ar5312/ar5312_attach.c
+++ b/sys/dev/ath/ath_hal/ar5312/ar5312_attach.c
@@ -71,13 +71,13 @@ ar5312Attach(uint16_t devid, HAL_SOFTC sc,
 	uint16_t eeval;
 	HAL_STATUS ecode;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 		 __func__, sc, st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp = ath_hal_malloc(sizeof (struct ath_hal_5212));
 	if (ahp == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416.h b/sys/dev/ath/ath_hal/ar5416/ar5416.h
index a0f5deead8bb..35d9a4d61722 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416.h
@@ -190,6 +190,7 @@ extern	void ar5416SetLedState(struct ath_hal *ah, HAL_LED_STATE state);
 extern	uint64_t ar5416GetTsf64(struct ath_hal *ah);
 extern	void ar5416SetTsf64(struct ath_hal *ah, uint64_t tsf64);
 extern	void ar5416ResetTsf(struct ath_hal *ah);
+extern	uint32_t ar5416GetCurRssi(struct ath_hal *ah);
 extern	HAL_BOOL ar5416SetAntennaSwitch(struct ath_hal *, HAL_ANT_SETTING);
 extern	HAL_BOOL ar5416SetDecompMask(struct ath_hal *, uint16_t, int);
 extern	void ar5416SetCoverageClass(struct ath_hal *, uint8_t, int);
@@ -338,8 +339,14 @@ extern	u_int ar5416GetGlobalTxTimeout(struct ath_hal *ah);
 extern	void ar5416Set11nRateScenario(struct ath_hal *ah, struct ath_desc *ds,
 		u_int durUpdateEn, u_int rtsctsRate, HAL_11N_RATE_SERIES series[],
 		u_int nseries, u_int flags);
+
+extern void ar5416Set11nAggrFirst(struct ath_hal *ah, struct ath_desc *ds,
+		u_int aggrLen, u_int numDelims);
 extern	void ar5416Set11nAggrMiddle(struct ath_hal *ah, struct ath_desc *ds, u_int numDelims);
+extern void ar5416Set11nAggrLast(struct ath_hal *ah, struct ath_desc *ds);
+
 extern	void ar5416Clr11nAggr(struct ath_hal *ah, struct ath_desc *ds);
+
 extern	void ar5416Set11nBurstDuration(struct ath_hal *ah, struct ath_desc *ds, u_int burstDuration);
 
 extern	const HAL_RATE_TABLE *ar5416GetRateTable(struct ath_hal *, u_int mode);
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
index 12589f28fff2..630efc044157 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
@@ -168,7 +168,9 @@ ar5416InitState(struct ath_hal_5416 *ahp5416, uint16_t devid, HAL_SOFTC sc,
 	ah->ah_setupFirstTxDesc		= ar5416SetupFirstTxDesc;
 	ah->ah_setupLastTxDesc		= ar5416SetupLastTxDesc;
 	ah->ah_set11nRateScenario	= ar5416Set11nRateScenario;
+	ah->ah_set11nAggrFirst		= ar5416Set11nAggrFirst;
 	ah->ah_set11nAggrMiddle		= ar5416Set11nAggrMiddle;
+	ah->ah_set11nAggrLast		= ar5416Set11nAggrLast;
 	ah->ah_clr11nAggr		= ar5416Clr11nAggr;
 	ah->ah_set11nBurstDuration	= ar5416Set11nBurstDuration;
 	ah->ah_get11nExtBusy		= ar5416Get11nExtBusy;
@@ -246,7 +248,7 @@ ar5416Attach(uint16_t devid, HAL_SOFTC sc,
 	HAL_STATUS ecode;
 	HAL_BOOL rfStatus;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
@@ -255,7 +257,7 @@ ar5416Attach(uint16_t devid, HAL_SOFTC sc,
 		sizeof(ar5416Addac)
 	);
 	if (ahp5416 == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
@@ -892,6 +894,12 @@ ar5416FillCapabilityInfo(struct ath_hal *ah)
 	pCap->halEnhancedDfsSupport = AH_FALSE;
 	/* Hardware supports 32 bit TSF values in the RX descriptor */
 	pCap->halHasLongRxDescTsf = AH_TRUE;
+	/*
+	 * BB Read WAR: this is only for AR5008/AR9001 NICs
+	 * It is also set individually in the AR91xx attach functions.
+	 */
+	if (AR_SREV_OWL(ah))
+		pCap->halHasBBReadWar = AH_TRUE;
 
 	if (ath_hal_eepromGetFlag(ah, AR_EEP_RFKILL) &&
 	    ath_hal_eepromGet(ah, AR_EEP_RFSILENT, &ahpriv->ah_rfsilent) == HAL_OK) {
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_beacon.c b/sys/dev/ath/ath_hal/ar5416/ar5416_beacon.c
index 8b61e14658ae..e2bf6c732767 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_beacon.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_beacon.c
@@ -46,6 +46,7 @@ void
 ar5416SetBeaconTimers(struct ath_hal *ah, const HAL_BEACON_TIMERS *bt)
 {
 	uint32_t bperiod;
+	struct ath_hal_5212 *ahp = AH5212(ah);
 
 	OS_REG_WRITE(ah, AR_NEXT_TBTT, TU_TO_USEC(bt->bt_nexttbtt));
 	OS_REG_WRITE(ah, AR_NEXT_DBA, ONE_EIGHTH_TU_TO_USEC(bt->bt_nextdba));
@@ -53,6 +54,7 @@ ar5416SetBeaconTimers(struct ath_hal *ah, const HAL_BEACON_TIMERS *bt)
 	OS_REG_WRITE(ah, AR_NEXT_NDP, TU_TO_USEC(bt->bt_nextatim));
 
 	bperiod = TU_TO_USEC(bt->bt_intval & HAL_BEACON_PERIOD);
+	ahp->ah_beaconInterval = bt->bt_intval & HAL_BEACON_PERIOD;
 	OS_REG_WRITE(ah, AR5416_BEACON_PERIOD, bperiod);
 	OS_REG_WRITE(ah, AR_DBA_PERIOD, bperiod);
 	OS_REG_WRITE(ah, AR_SWBA_PERIOD, bperiod);
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c b/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c
index d51417f4992a..cb5787017492 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c
@@ -642,7 +642,15 @@ ar5416LoadNF(struct ath_hal *ah, const struct ieee80211_channel *chan)
 	OS_REG_SET_BIT(ah, AR_PHY_AGC_CONTROL, AR_PHY_AGC_CONTROL_NF);
 
 	/* Wait for load to complete, should be fast, a few 10s of us. */
-	if (! ar5212WaitNFCalComplete(ah, 1000)) {
+	/*
+	 * XXX For now, don't be so aggressive in waiting for the NF
+	 * XXX load to complete. A very busy 11n RX load will cause this
+	 * XXX to always fail; so just leave it.
+	 * XXX Later on we may wish to split longcal into two parts - one to do
+	 * XXX the initial longcal, and one to load in an updated NF value
+	 * XXX once it's finished - say, by checking it every 500ms.
+	 */
+	if (! ar5212WaitNFCalComplete(ah, 5)) {
 		/*
 		 * We timed out waiting for the noisefloor to load, probably due to an
 		 * in-progress rx. Simply return here and allow the load plenty of time
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c b/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c
index 03d3544a89aa..3bc2cc56e561 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c
@@ -68,6 +68,7 @@ HAL_BOOL
 ar5416GetPendingInterrupts(struct ath_hal *ah, HAL_INT *masked)
 {
 	uint32_t isr, isr0, isr1, sync_cause = 0;
+	HAL_CAPABILITIES *pCap = &AH_PRIVATE(ah)->ah_caps;
 
 	/*
 	 * Verify there's a mac interrupt and the RTC is on.
@@ -110,44 +111,95 @@ ar5416GetPendingInterrupts(struct ath_hal *ah, HAL_INT *masked)
 				mask2 |= HAL_INT_CST;	
 			if (isr2 & AR_ISR_S2_TSFOOR)
 				mask2 |= HAL_INT_TSFOOR;
+
+			/*
+			 * Don't mask out AR_BCNMISC; instead mask
+			 * out what causes it.
+			 */
+			OS_REG_WRITE(ah, AR_ISR_S2, isr2);
+			isr &= ~AR_ISR_BCNMISC;
 		}
 
-		isr = OS_REG_READ(ah, AR_ISR_RAC);
 		if (isr == 0xffffffff) {
 			*masked = 0;
 			return AH_FALSE;
 		}
 
 		*masked = isr & HAL_INT_COMMON;
-		if (isr & (AR_ISR_RXOK | AR_ISR_RXERR))
-			*masked |= HAL_INT_RX;
-		if (isr & (AR_ISR_TXOK | AR_ISR_TXDESC | AR_ISR_TXERR | AR_ISR_TXEOL)) {
-			*masked |= HAL_INT_TX;
-			isr0 = OS_REG_READ(ah, AR_ISR_S0_S);
-			ahp->ah_intrTxqs |= MS(isr0, AR_ISR_S0_QCU_TXOK);
-			ahp->ah_intrTxqs |= MS(isr0, AR_ISR_S0_QCU_TXDESC);
-			isr1 = OS_REG_READ(ah, AR_ISR_S1_S);
-			ahp->ah_intrTxqs |= MS(isr1, AR_ISR_S1_QCU_TXERR);
-			ahp->ah_intrTxqs |= MS(isr1, AR_ISR_S1_QCU_TXEOL);
-		}
 
-		if (AR_SREV_MERLIN(ah) || AR_SREV_KITE(ah)) {
-			uint32_t isr5;
-			isr5 = OS_REG_READ(ah, AR_ISR_S5_S);
-			if (isr5 & AR_ISR_S5_TIM_TIMER)
-				*masked |= HAL_INT_TIM_TIMER;
-		}
-
-		/* Interrupt Mitigation on AR5416 */
-#ifdef	AH_AR5416_INTERRUPT_MITIGATION
 		if (isr & (AR_ISR_RXMINTR | AR_ISR_RXINTM))
 			*masked |= HAL_INT_RX;
 		if (isr & (AR_ISR_TXMINTR | AR_ISR_TXINTM))
 			*masked |= HAL_INT_TX;
+
+		/*
+		 * When doing RX interrupt mitigation, the RXOK bit is set
+		 * in AR_ISR even if the relevant bit in AR_IMR is clear.
+		 * Since this interrupt may be due to another source, don't
+		 * just automatically set HAL_INT_RX if it's set, otherwise
+		 * we could prematurely service the RX queue.
+		 *
+		 * In some cases, the driver can even handle all the RX
+		 * frames just before the mitigation interrupt fires.
+		 * The subsequent RX processing trip will then end up
+		 * processing 0 frames.
+		 */
+#ifdef	AH_AR5416_INTERRUPT_MITIGATION
+		if (isr & AR_ISR_RXERR)
+			*masked |= HAL_INT_RX;
+#else
+		if (isr & (AR_ISR_RXOK | AR_ISR_RXERR))
+			*masked |= HAL_INT_RX;
 #endif
+
+		if (isr & (AR_ISR_TXOK | AR_ISR_TXDESC | AR_ISR_TXERR |
+		    AR_ISR_TXEOL)) {
+			*masked |= HAL_INT_TX;
+
+			isr0 = OS_REG_READ(ah, AR_ISR_S0);
+			OS_REG_WRITE(ah, AR_ISR_S0, isr0);
+			isr1 = OS_REG_READ(ah, AR_ISR_S1);
+			OS_REG_WRITE(ah, AR_ISR_S1, isr1);
+
+			/*
+			 * Don't clear the primary ISR TX bits, clear
+			 * what causes them (S0/S1.)
+			 */
+			isr &= ~(AR_ISR_TXOK | AR_ISR_TXDESC |
+			    AR_ISR_TXERR | AR_ISR_TXEOL);
+
+			ahp->ah_intrTxqs |= MS(isr0, AR_ISR_S0_QCU_TXOK);
+			ahp->ah_intrTxqs |= MS(isr0, AR_ISR_S0_QCU_TXDESC);
+			ahp->ah_intrTxqs |= MS(isr1, AR_ISR_S1_QCU_TXERR);
+			ahp->ah_intrTxqs |= MS(isr1, AR_ISR_S1_QCU_TXEOL);
+		}
+
+		if ((isr & AR_ISR_GENTMR) || (! pCap->halAutoSleepSupport)) {
+			uint32_t isr5;
+			isr5 = OS_REG_READ(ah, AR_ISR_S5);
+			OS_REG_WRITE(ah, AR_ISR_S5, isr5);
+			isr &= ~AR_ISR_GENTMR;
+
+			if (! pCap->halAutoSleepSupport)
+				if (isr5 & AR_ISR_S5_TIM_TIMER)
+					*masked |= HAL_INT_TIM_TIMER;
+		}
 		*masked |= mask2;
 	}
 
+	/*
+	 * Since we're not using AR_ISR_RAC, clear the status bits
+	 * for handled interrupts here. For bits whose interrupt
+	 * source is a secondary register, those bits should've been
+	 * masked out - instead of those bits being written back,
+	 * their source (ie, the secondary status registers) should
+	 * be cleared. That way there are no race conditions with
+	 * new triggers coming in whilst they've been read/cleared.
+	 */
+	OS_REG_WRITE(ah, AR_ISR, isr);
+	/* Flush previous write */
+	OS_REG_READ(ah, AR_ISR);
+
 	if (AR_SREV_HOWL(ah))
 		return AH_TRUE;
 
@@ -216,18 +268,12 @@ ar5416SetInterrupts(struct ath_hal *ah, HAL_INT ints)
 	 * Overwrite default mask if Interrupt mitigation
 	 * is specified for AR5416
 	 */
-	mask = ints & HAL_INT_COMMON;
-	if (ints & HAL_INT_TX)
-		mask |= AR_IMR_TXMINTR | AR_IMR_TXINTM;
 	if (ints & HAL_INT_RX)
 		mask |= AR_IMR_RXERR | AR_IMR_RXMINTR | AR_IMR_RXINTM;
-	if (ints & HAL_INT_TX) {
-		if (ahp->ah_txErrInterruptMask)
-			mask |= AR_IMR_TXERR;
-		if (ahp->ah_txEolInterruptMask)
-			mask |= AR_IMR_TXEOL;
-	}
 #else
+	if (ints & HAL_INT_RX)
+		mask |= AR_IMR_RXOK | AR_IMR_RXERR | AR_IMR_RXDESC;
+#endif
 	if (ints & HAL_INT_TX) {
 		if (ahp->ah_txOkInterruptMask)
 			mask |= AR_IMR_TXOK;
@@ -238,9 +284,6 @@ ar5416SetInterrupts(struct ath_hal *ah, HAL_INT ints)
 		if (ahp->ah_txEolInterruptMask)
 			mask |= AR_IMR_TXEOL;
 	}
-	if (ints & HAL_INT_RX)
-		mask |= AR_IMR_RXOK | AR_IMR_RXERR | AR_IMR_RXDESC;
-#endif
 	if (ints & (HAL_INT_BMISC)) {
 		mask |= AR_IMR_BCNMISC;
 		if (ints & HAL_INT_TIM)
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c b/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c
index 7e61c84930d9..880900f98915 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c
@@ -21,9 +21,7 @@
 #include "ah.h"
 #include "ah_internal.h"
 #include "ah_devid.h"
-#ifdef AH_DEBUG
 #include "ah_desc.h"                    /* NB: for HAL_PHYERR* */
-#endif
 
 #include "ar5416/ar5416.h"
 #include "ar5416/ar5416reg.h"
@@ -145,6 +143,14 @@ ar5416ResetTsf(struct ath_hal *ah)
 	OS_REG_WRITE(ah, AR_RESET_TSF, AR_RESET_TSF_ONCE);	
 }
 
+uint32_t
+ar5416GetCurRssi(struct ath_hal *ah)
+{
+	if (AR_SREV_OWL(ah))
+		return (OS_REG_READ(ah, AR_PHY_CURRENT_RSSI) & 0xff);
+	return (OS_REG_READ(ah, AR9130_PHY_CURRENT_RSSI) & 0xff);
+}
+
 HAL_BOOL
 ar5416SetAntennaSwitch(struct ath_hal *ah, HAL_ANT_SETTING settings)
 {
@@ -278,7 +284,7 @@ ar5416Get11nRxClear(struct ath_hal *ah)
         rxclear |= HAL_RX_CLEAR_CTL_LOW;
     }
     /* extension channel */
-    if (val & AR_DIAG_RXCLEAR_CTL_LOW) {
+    if (val & AR_DIAG_RXCLEAR_EXT_LOW) {
         rxclear |= HAL_RX_CLEAR_EXT_LOW;
     }
     return rxclear;
@@ -757,14 +763,200 @@ ar5416EnableDfs(struct ath_hal *ah, HAL_PHYERR_PARAM *pe)
  * Returns AH_TRUE if the phy error was actually a phy error,
  * AH_FALSE if the phy error wasn't a phy error.
  */
+
+/* Flags for pulse_bw_info */
+#define	PRI_CH_RADAR_FOUND		0x01
+#define	EXT_CH_RADAR_FOUND		0x02
+#define	EXT_CH_RADAR_EARLY_FOUND	0x04
+
 HAL_BOOL
 ar5416ProcessRadarEvent(struct ath_hal *ah, struct ath_rx_status *rxs,
     uint64_t fulltsf, const char *buf, HAL_DFS_EVENT *event)
 {
+	HAL_BOOL doDfsExtCh;
+	HAL_BOOL doDfsEnhanced;
+	HAL_BOOL doDfsCombinedRssi;
+
+	uint8_t rssi = 0, ext_rssi = 0;
+	uint8_t pulse_bw_info = 0, pulse_length_ext = 0, pulse_length_pri = 0;
+	uint32_t dur = 0;
+	int pri_found = 1, ext_found = 0;
+	int early_ext = 0;
+	int is_dc = 0;
+	uint16_t datalen;		/* length from the RX status field */
+
+	/* Check whether the given phy error is a radar event */
+	if ((rxs->rs_phyerr != HAL_PHYERR_RADAR) &&
+	    (rxs->rs_phyerr != HAL_PHYERR_FALSE_RADAR_EXT)) {
+		return AH_FALSE;
+	}
+
+	/* Grab copies of the capabilities; just to make the code clearer */
+	doDfsExtCh = AH_PRIVATE(ah)->ah_caps.halExtChanDfsSupport;
+	doDfsEnhanced = AH_PRIVATE(ah)->ah_caps.halEnhancedDfsSupport;
+	doDfsCombinedRssi = AH_PRIVATE(ah)->ah_caps.halUseCombinedRadarRssi;
+
+	datalen = rxs->rs_datalen;
+
+	/* If hardware supports it, use combined RSSI, else use chain 0 RSSI */
+	if (doDfsCombinedRssi)
+		rssi = (uint8_t) rxs->rs_rssi;
+	else		
+		rssi = (uint8_t) rxs->rs_rssi_ctl[0];
+
+	/* Set this; but only use it if doDfsExtCh is set */
+	ext_rssi = (uint8_t) rxs->rs_rssi_ext[0];
+
+	/* Cap it at 0 if the RSSI is a negative number */
+	if (rssi & 0x80)
+		rssi = 0;
+
+	if (ext_rssi & 0x80)
+		ext_rssi = 0;
+
 	/*
-	 * For now, this isn't implemented.
+	 * Fetch the relevant data from the frame
 	 */
-	return AH_FALSE;
+	if (doDfsExtCh) {
+		if (datalen < 3)
+			return AH_FALSE;
+
+		/* Last three bytes of the frame are of interest */
+		pulse_length_pri = *(buf + datalen - 3);
+		pulse_length_ext = *(buf + datalen - 2);
+		pulse_bw_info = *(buf + datalen - 1);
+		HALDEBUG(ah, HAL_DEBUG_DFS, "%s: rssi=%d, ext_rssi=%d, pulse_length_pri=%d,"
+		    " pulse_length_ext=%d, pulse_bw_info=%x\n",
+		    __func__, rssi, ext_rssi, pulse_length_pri, pulse_length_ext,
+		    pulse_bw_info);
+	} else {
+		/* The pulse width is byte 0 of the data */
+		if (datalen >= 1)
+			dur = ((uint8_t) buf[0]) & 0xff;
+		else
+			dur = 0;
+
+		if (dur == 0 && rssi == 0) {
+			HALDEBUG(ah, HAL_DEBUG_DFS, "%s: dur and rssi are 0\n", __func__);
+			return AH_FALSE;
+		}
+
+		HALDEBUG(ah, HAL_DEBUG_DFS, "%s: rssi=%d, dur=%d\n", __func__, rssi, dur);
+
+		/* Single-channel only */
+		pri_found = 1;
+		ext_found = 0;
+	}
+
+	/*
+	 * If doing extended channel data, pulse_bw_info must
+	 * have one of the flags set.
+	 */
+	if (doDfsExtCh && pulse_bw_info == 0x0)
+		return AH_FALSE;
+		
+	/*
+	 * If the extended channel data is available, calculate
+	 * which to pay attention to.
+	 */
+	if (doDfsExtCh) {
+		/* If pulse is on DC, take the larger duration of the two */
+		if ((pulse_bw_info & EXT_CH_RADAR_FOUND) &&
+		    (pulse_bw_info & PRI_CH_RADAR_FOUND)) {
+			is_dc = 1;
+			if (pulse_length_ext > pulse_length_pri) {
+				dur = pulse_length_ext;
+				pri_found = 0;
+				ext_found = 1;
+			} else {
+				dur = pulse_length_pri;
+				pri_found = 1;
+				ext_found = 0;
+			}
+		} else if (pulse_bw_info & EXT_CH_RADAR_EARLY_FOUND) {
+			dur = pulse_length_ext;
+			pri_found = 0;
+			ext_found = 1;
+			early_ext = 1;
+		} else if (pulse_bw_info & PRI_CH_RADAR_FOUND) {
+			dur = pulse_length_pri;
+			pri_found = 1;
+			ext_found = 0;
+		} else if (pulse_bw_info & EXT_CH_RADAR_FOUND) {
+			dur = pulse_length_ext;
+			pri_found = 0;
+			ext_found = 1;
+		}
+		
+	}
+
+	/*
+	 * For enhanced DFS (Merlin and later), pulse_bw_info has
+	 * implications for selecting the correct RSSI value.
+	 */
+	if (doDfsEnhanced) {
+		switch (pulse_bw_info & 0x03) {
+		case 0:
+			/* No radar? */
+			rssi = 0;
+			break;
+		case PRI_CH_RADAR_FOUND:
+			/* Radar in primary channel */
+			/* Cannot use ctrl channel RSSI if ext channel is stronger */
+			if (ext_rssi >= (rssi + 3)) {
+				rssi = 0;
+			};
+			break;
+		case EXT_CH_RADAR_FOUND:
+			/* Radar in extended channel */
+			/* Cannot use ext channel RSSI if ctrl channel is stronger */
+			if (rssi >= (ext_rssi + 12)) {
+				rssi = 0;
+			} else {
+				rssi = ext_rssi;
+			}
+			break;
+		case (PRI_CH_RADAR_FOUND | EXT_CH_RADAR_FOUND):
+			/* When both are present, use stronger one */
+			if (rssi < ext_rssi)
+				rssi = ext_rssi;
+			break;
+		}
+	}
+
+	/*
+	 * If not doing enhanced DFS, choose the ext channel if
+	 * it is stronger than the main channel
+	 */
+	if (doDfsExtCh && !doDfsEnhanced) {
+		if ((ext_rssi > rssi) && (ext_rssi < 128))
+			rssi = ext_rssi;
+	}
+
+	/*
+	 * XXX what happens if the above code decides the RSSI
+	 * XXX wasn't valid, an sets it to 0?
+	 */
+
+	/*
+	 * Fill out dfs_event structure.
+	 */
+	event->re_full_ts = fulltsf;
+	event->re_ts = rxs->rs_tstamp;
+	event->re_rssi = rssi;
+	event->re_dur = dur;
+
+	event->re_flags = 0;
+	if (pri_found)
+		event->re_flags |= HAL_DFS_EVENT_PRICH;
+	if (ext_found)
+		event->re_flags |= HAL_DFS_EVENT_EXTCH;
+	if (early_ext)
+		event->re_flags |= HAL_DFS_EVENT_EXTEARLY;
+	if (is_dc)
+		event->re_flags |= HAL_DFS_EVENT_ISDC;
+
+	return AH_TRUE;
 }
 
 /*
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_recv.c b/sys/dev/ath/ath_hal/ar5416/ar5416_recv.c
index 289da427eb4b..de4343e98a45 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_recv.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_recv.c
@@ -107,7 +107,6 @@ ar5416SetupRxDesc(struct ath_hal *ah, struct ath_desc *ds,
     uint32_t size, u_int flags)
 {
 	struct ar5416_desc *ads = AR5416DESC(ds);
-	HAL_CAPABILITIES *pCap = &AH_PRIVATE(ah)->ah_caps;
 
 	HALASSERT((size &~ AR_BufLen) == 0);
 
@@ -119,8 +118,7 @@ ar5416SetupRxDesc(struct ath_hal *ah, struct ath_desc *ds,
 	ads->ds_rxstatus8 &= ~AR_RxDone;
 
 	/* clear the rest of the status fields */
-	if (! pCap->halAutoSleepSupport)
-        	OS_MEMZERO(&(ads->u), sizeof(ads->u));
+	OS_MEMZERO(&(ads->u), sizeof(ads->u));
 
 	return AH_TRUE;
 }
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c b/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
index 112d9669540f..97d292e91ab8 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
@@ -278,9 +278,10 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	/* Restore previous antenna */
 	OS_REG_WRITE(ah, AR_DEF_ANTENNA, saveDefAntenna);
 
-	/* then our BSSID */
+	/* then our BSSID and associate id */
 	OS_REG_WRITE(ah, AR_BSS_ID0, LE_READ_4(ahp->ah_bssid));
-	OS_REG_WRITE(ah, AR_BSS_ID1, LE_READ_2(ahp->ah_bssid + 4));
+	OS_REG_WRITE(ah, AR_BSS_ID1, LE_READ_2(ahp->ah_bssid + 4) |
+	    (ahp->ah_assocId & 0x3fff) << AR_BSS_ID1_AID_S);
 
 	/* Restore bmiss rssi & count thresholds */
 	OS_REG_WRITE(ah, AR_RSSI_THR, ahp->ah_rssiThr);
@@ -357,13 +358,33 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	 */
 	OS_REG_WRITE(ah, AR_OBS, 8);
 
-#ifdef	AH_AR5416_INTERRUPT_MITIGATION
+	/*
+	 * Disable the "general" TX/RX mitigation timers.
+	 */
 	OS_REG_WRITE(ah, AR_MIRT, 0);
 
-	OS_REG_RMW_FIELD(ah, AR_RIMT, AR_RIMT_LAST, 500);
-	OS_REG_RMW_FIELD(ah, AR_RIMT, AR_RIMT_FIRST, 2000);
-	OS_REG_RMW_FIELD(ah, AR_TIMT, AR_TIMT_LAST, 300);
-	OS_REG_RMW_FIELD(ah, AR_TIMT, AR_TIMT_FIRST, 750);
+#ifdef	AH_AR5416_INTERRUPT_MITIGATION
+	/*
+	 * This initialises the RX interrupt mitigation timers.
+	 *
+	 * The mitigation timers begin at idle and are triggered
+	 * upon the RXOK of a single frame (or sub-frame, for A-MPDU.)
+	 * Then, the RX mitigation interrupt will fire:
+	 *
+	 * + 250uS after the last RX'ed frame, or
+	 * + 700uS after the first RX'ed frame
+	 *
+	 * Thus, the LAST field dictates the extra latency
+	 * induced by the RX mitigation method and the FIRST
+	 * field dictates how long to delay before firing an
+	 * RX mitigation interrupt.
+	 *
+	 * Please note this only seems to be for RXOK frames;
+	 * not CRC or PHY error frames.
+	 *
+	 */
+	OS_REG_RMW_FIELD(ah, AR_RIMT, AR_RIMT_LAST, 250);
+	OS_REG_RMW_FIELD(ah, AR_RIMT, AR_RIMT_FIRST, 700);
 #endif
 	ar5416InitBB(ah, chan);
 
@@ -610,11 +631,11 @@ ar5416InitIMR(struct ath_hal *ah, HAL_OPMODE opmode)
                         | AR_IMR_BCNMISC;
 
 #ifdef	AH_AR5416_INTERRUPT_MITIGATION
-	ahp->ah_maskReg |= AR_IMR_TXINTM | AR_IMR_RXINTM
-			|  AR_IMR_TXMINTR | AR_IMR_RXMINTR;
+	ahp->ah_maskReg |= AR_IMR_RXINTM | AR_IMR_RXMINTR;
 #else
-	ahp->ah_maskReg |= AR_IMR_TXOK | AR_IMR_RXOK;
+	ahp->ah_maskReg |= AR_IMR_RXOK;
 #endif	
+	ahp->ah_maskReg |= AR_IMR_TXOK;
 
 	if (opmode == HAL_M_HOSTAP)
 		ahp->ah_maskReg |= AR_IMR_MIB;
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_xmit.c b/sys/dev/ath/ath_hal/ar5416/ar5416_xmit.c
index 20791e8a5a50..ad47f0619a81 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_xmit.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_xmit.c
@@ -295,12 +295,14 @@ ar5416FillTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 		 * copy the multi-rate transmit parameters from
 		 * the first frame for processing on completion. 
 		 */
-		ads->ds_ctl0 = 0;
 		ads->ds_ctl1 = segLen;
 #ifdef AH_NEED_DESC_SWAP
+		ads->ds_ctl0 = __bswap32(AR5416DESC_CONST(ds0)->ds_ctl0)
+		    & AR_TxIntrReq;
 		ads->ds_ctl2 = __bswap32(AR5416DESC_CONST(ds0)->ds_ctl2);
 		ads->ds_ctl3 = __bswap32(AR5416DESC_CONST(ds0)->ds_ctl3);
 #else
+		ads->ds_ctl0 = AR5416DESC_CONST(ds0)->ds_ctl0 & AR_TxIntrReq;
 		ads->ds_ctl2 = AR5416DESC_CONST(ds0)->ds_ctl2;
 		ads->ds_ctl3 = AR5416DESC_CONST(ds0)->ds_ctl3;
 #endif
@@ -308,7 +310,12 @@ ar5416FillTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 		/*
 		 * Intermediate descriptor in a multi-descriptor frame.
 		 */
-		ads->ds_ctl0 = 0;
+#ifdef AH_NEED_DESC_SWAP
+		ads->ds_ctl0 = __bswap32(AR5416DESC_CONST(ds0)->ds_ctl0)
+		    & AR_TxIntrReq;
+#else
+		ads->ds_ctl0 = AR5416DESC_CONST(ds0)->ds_ctl0 & AR_TxIntrReq;
+#endif
 		ads->ds_ctl1 = segLen | AR_TxMore;
 		ads->ds_ctl2 = 0;
 		ads->ds_ctl3 = 0;
@@ -318,6 +325,9 @@ ar5416FillTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 	return AH_TRUE;
 }
 
+/*
+ * NB: cipher is no longer used, it's calculated.
+ */
 HAL_BOOL
 ar5416ChainTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 	u_int pktLen,
@@ -347,10 +357,20 @@ ar5416ChainTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 		isaggr = 1;
 	}
 
-	if (!firstSeg) {
-		OS_MEMZERO(ds->ds_hw, AR5416_DESC_TX_CTL_SZ);
-	}
+	/*
+	 * Since this function is called before any of the other
+	 * descriptor setup functions (at least in this particular
+	 * 802.11n aggregation implementation), always bzero() the
+	 * descriptor. Previously this would be done for all but
+	 * the first segment.
+	 * XXX TODO: figure out why; perhaps I'm using this slightly
+	 * XXX incorrectly.
+	 */
+	OS_MEMZERO(ds->ds_hw, AR5416_DESC_TX_CTL_SZ);
 
+	/*
+	 * Note: VEOL should only be for the last descriptor in the chain.
+	 */
 	ads->ds_ctl0 = (pktLen & AR_FrameLen);
 	ads->ds_ctl1 = (type << AR_FrameType_S)
 			| (isaggr ? (AR_IsAggr | AR_MoreAggr) : 0);
@@ -362,7 +382,7 @@ ar5416ChainTxDesc(struct ath_hal *ah, struct ath_desc *ds,
 		ads->ds_ctl0 |= AR_DestIdxValid;
 	}
 
-	ads->ds_ctl6 = SM(ahp->ah_keytype[cipher], AR_EncrType);
+	ads->ds_ctl6 |= SM(ahp->ah_keytype[keyIx], AR_EncrType);
 	if (isaggr) {
 		ads->ds_ctl6 |= SM(delims, AR_PadDelim);
 	}
@@ -504,6 +524,7 @@ ar5416ProcTxDesc(struct ath_hal *ah,
 	/* Update software copies of the HW status */
 	ts->ts_seqnum = MS(ds_txstatus[9], AR_SeqNum);
 	ts->ts_tstamp = AR_SendTimestamp(ds_txstatus);
+	ts->ts_tid = MS(ds_txstatus[9], AR_TxTid);
 
 	ts->ts_status = 0;
 	if (ds_txstatus[1] & AR_ExcessiveRetries)
@@ -691,6 +712,19 @@ ar5416Set11nRateScenario(struct ath_hal *ah, struct ath_desc *ds,
 		     | SM(rtsctsRate, AR_RTSCTSRate);
 }
 
+void
+ar5416Set11nAggrFirst(struct ath_hal *ah, struct ath_desc *ds,
+    u_int aggrLen, u_int numDelims)
+{
+	struct ar5416_desc *ads = AR5416DESC(ds);
+
+	ads->ds_ctl1 |= (AR_IsAggr | AR_MoreAggr);
+
+	ads->ds_ctl6 &= ~(AR_AggrLen | AR_PadDelim);
+	ads->ds_ctl6 |= SM(aggrLen, AR_AggrLen) |
+	    SM(numDelims, AR_PadDelim);
+}
+
 void
 ar5416Set11nAggrMiddle(struct ath_hal *ah, struct ath_desc *ds, u_int numDelims)
 {
@@ -710,6 +744,16 @@ ar5416Set11nAggrMiddle(struct ath_hal *ah, struct ath_desc *ds, u_int numDelims)
 	ds_txstatus[9] &= ~AR_TxDone;
 }
 
+void
+ar5416Set11nAggrLast(struct ath_hal *ah, struct ath_desc *ds)
+{
+	struct ar5416_desc *ads = AR5416DESC(ds);
+
+	ads->ds_ctl1 |= AR_IsAggr;
+	ads->ds_ctl1 &= ~AR_MoreAggr;
+	ads->ds_ctl6 &= ~AR_PadDelim;
+}
+
 void
 ar5416Clr11nAggr(struct ath_hal *ah, struct ath_desc *ds)
 {
@@ -879,6 +923,7 @@ setTxQInterrupts(struct ath_hal *ah, HAL_TX_QUEUE_INFO *qi)
  * Assumes:
  *  phwChannel has been set to point to the current channel
  */
+#define	TU_TO_USEC(_tu)		((_tu) << 10)
 HAL_BOOL
 ar5416ResetTxQueue(struct ath_hal *ah, u_int q)
 {
@@ -886,7 +931,7 @@ ar5416ResetTxQueue(struct ath_hal *ah, u_int q)
 	HAL_CAPABILITIES *pCap = &AH_PRIVATE(ah)->ah_caps;
 	const struct ieee80211_channel *chan = AH_PRIVATE(ah)->ah_curchan;
 	HAL_TX_QUEUE_INFO *qi;
-	uint32_t cwMin, chanCwMin, value, qmisc, dmisc;
+	uint32_t cwMin, chanCwMin, qmisc, dmisc;
 
 	if (q >= pCap->halTotalQueues) {
 		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: invalid queue num %u\n",
@@ -956,7 +1001,8 @@ ar5416ResetTxQueue(struct ath_hal *ah, u_int q)
 		if (qi->tqi_cbrOverflowLimit)
 			qmisc |= AR_Q_MISC_CBR_EXP_CNTR_LIMIT;
 	}
-	if (qi->tqi_readyTime) {
+
+	if (qi->tqi_readyTime && (qi->tqi_type != HAL_TX_QUEUE_CAB)) {
 		OS_REG_WRITE(ah, AR_QRDYTIMECFG(q),
 			  SM(qi->tqi_readyTime, AR_Q_RDYTIMECFG_INT)
 			| AR_Q_RDYTIMECFG_ENA);
@@ -1027,18 +1073,46 @@ ar5416ResetTxQueue(struct ath_hal *ah, u_int q)
 		qmisc |= AR_Q_MISC_FSP_DBA_GATED
 		      |  AR_Q_MISC_CBR_INCR_DIS1
 		      |  AR_Q_MISC_CBR_INCR_DIS0;
-
-		if (!qi->tqi_readyTime) {
+		HALDEBUG(ah, HAL_DEBUG_TXQUEUE, "%s: CAB: tqi_readyTime = %d\n",
+		    __func__, qi->tqi_readyTime);
+		if (qi->tqi_readyTime) {
+			HALDEBUG(ah, HAL_DEBUG_TXQUEUE,
+			    "%s: using tqi_readyTime\n", __func__);
+			OS_REG_WRITE(ah, AR_QRDYTIMECFG(q),
+			    SM(qi->tqi_readyTime, AR_Q_RDYTIMECFG_INT) |
+			    AR_Q_RDYTIMECFG_ENA);
+		} else {
+			int value;
 			/*
 			 * NB: don't set default ready time if driver
 			 * has explicitly specified something.  This is
 			 * here solely for backwards compatibility.
 			 */
-			value = (ahp->ah_beaconInterval
-				- (ah->ah_config.ah_sw_beacon_response_time -
-					ah->ah_config.ah_dma_beacon_response_time)
-				- ah->ah_config.ah_additional_swba_backoff) * 1024;
-			OS_REG_WRITE(ah, AR_QRDYTIMECFG(q), value | AR_Q_RDYTIMECFG_ENA);
+			/*
+			 * XXX for now, hard-code a CAB interval of 70%
+			 * XXX of the total beacon interval.
+			 *
+			 * XXX This keeps Merlin and later based MACs
+			 * XXX quite a bit happier (stops stuck beacons,
+			 * XXX which I gather is because of such a long
+			 * XXX cabq time.)
+			 */
+			value = (ahp->ah_beaconInterval * 70 / 100)
+				- (ah->ah_config.ah_sw_beacon_response_time
+				+ ah->ah_config.ah_dma_beacon_response_time)
+				- ah->ah_config.ah_additional_swba_backoff;
+			/*
+			 * XXX Ensure it isn't too low - nothing lower
+			 * XXX than 10 TU
+			 */
+			if (value < 10)
+				value = 10;
+			HALDEBUG(ah, HAL_DEBUG_TXQUEUE,
+			    "%s: defaulting to rdytime = %d uS\n",
+			    __func__, value);
+			OS_REG_WRITE(ah, AR_QRDYTIMECFG(q),
+			    SM(TU_TO_USEC(value), AR_Q_RDYTIMECFG_INT) |
+			    AR_Q_RDYTIMECFG_ENA);
 		}
 		dmisc |= SM(AR_D_MISC_ARB_LOCKOUT_CNTRL_GLOBAL,
 			    AR_D_MISC_ARB_LOCKOUT_CNTRL);
@@ -1106,3 +1180,4 @@ ar5416ResetTxQueue(struct ath_hal *ah, u_int q)
 
 	return AH_TRUE;
 }
+#undef	TU_TO_USEC
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416desc.h b/sys/dev/ath/ath_hal/ar5416/ar5416desc.h
index 105c5d6080dc..bc57d8914566 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416desc.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416desc.h
@@ -302,6 +302,8 @@ struct ar5416_desc {
 #define AR_FinalTxIdx_S     21
 #define AR_TxStatusRsvd82   0x01800000
 #define AR_PowerMgmt        0x02000000
+#define AR_TxTid            0xf0000000
+#define AR_TxTid_S          28
 #define AR_TxStatusRsvd83   0xfc000000
 
 /***********
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416phy.h b/sys/dev/ath/ath_hal/ar5416/ar5416phy.h
index d7a5e0b22bb7..82e38017c971 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416phy.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416phy.h
@@ -156,8 +156,14 @@
 #define	AR_PHY_CAL_MEAS_0(_i)	(0x9c10 + ((_i) << 12))
 #define	AR_PHY_CAL_MEAS_1(_i)	(0x9c14 + ((_i) << 12))
 #define	AR_PHY_CAL_MEAS_2(_i)	(0x9c18 + ((_i) << 12))
+/* This is AR9130 and later */
 #define	AR_PHY_CAL_MEAS_3(_i)	(0x9c1c + ((_i) << 12))
 
+/*
+ * AR5416 still uses AR_PHY(263) for current RSSI;
+ * AR9130 and later uses AR_PHY(271).
+ */
+#define	AR9130_PHY_CURRENT_RSSI	0x9c3c		/* rssi of current frame rx'd */
 
 #define AR_PHY_CCA          0x9864
 #define AR_PHY_MINCCA_PWR   0x0FF80000
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416reg.h b/sys/dev/ath/ath_hal/ar5416/ar5416reg.h
index c18c26f90faf..8644f0da68a3 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416reg.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416reg.h
@@ -250,6 +250,7 @@
 /* Interrupts */
 #define	AR_ISR_TXMINTR		0x00080000	/* Maximum interrupt tx rate */
 #define	AR_ISR_RXMINTR		0x01000000	/* Maximum interrupt rx rate */
+#define	AR_ISR_GENTMR		0x10000000	/* OR of generic timer bits in S5 */
 #define	AR_ISR_TXINTM		0x40000000	/* Tx int after mitigation */
 #define	AR_ISR_RXINTM		0x80000000	/* Rx int after mitigation */
 
@@ -259,7 +260,13 @@
 
 #define	AR_ISR_S5		0x0098
 #define	AR_ISR_S5_S		0x00d8
-#define	AR_ISR_S5_TIM_TIMER	0x00000010
+#define	AR_ISR_S5_GENTIMER7	0x00000080 // Mask for timer 7 trigger
+#define	AR_ISR_S5_TIM_TIMER	0x00000010 // TIM Timer ISR
+#define	AR_ISR_S5_DTIM_TIMER	0x00000020 // DTIM Timer ISR
+#define	AR_ISR_S5_GENTIMER_TRIG	0x0000FF80 // ISR for generic timer trigger 7-15
+#define	AR_ISR_S5_GENTIMER_TRIG_S	0
+#define	AR_ISR_S5_GENTIMER_THRESH	0xFF800000 // ISR for generic timer threshold 7-15
+#define	AR_ISR_S5_GENTIMER_THRESH_S	16
 
 #define	AR_INTR_SPURIOUS	0xffffffff
 #define	AR_INTR_RTC_IRQ		0x00000001	/* rtc in shutdown state */
diff --git a/sys/dev/ath/ath_hal/ar9001/ar9130_attach.c b/sys/dev/ath/ath_hal/ar9001/ar9130_attach.c
index 518ed85fe852..e880d56fca5c 100644
--- a/sys/dev/ath/ath_hal/ar9001/ar9130_attach.c
+++ b/sys/dev/ath/ath_hal/ar9001/ar9130_attach.c
@@ -78,13 +78,13 @@ ar9130Attach(uint16_t devid, HAL_SOFTC sc,
 	HAL_STATUS ecode;
 	HAL_BOOL rfStatus;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp5416 = ath_hal_malloc(sizeof (struct ath_hal_5416));
 	if (ahp5416 == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
@@ -298,6 +298,8 @@ ar9130FillCapabilityInfo(struct ath_hal *ah)
 	 */
 	pCap->halMbssidAggrSupport = AH_FALSE;
 	pCap->hal4AddrAggrSupport = AH_TRUE;
+	/* BB Read WAR */
+	pCap->halHasBBReadWar = AH_TRUE;
 
 	return AH_TRUE;
 }
diff --git a/sys/dev/ath/ath_hal/ar9001/ar9160_attach.c b/sys/dev/ath/ath_hal/ar9001/ar9160_attach.c
index 2234eb35bfb6..4775ab01b9b0 100644
--- a/sys/dev/ath/ath_hal/ar9001/ar9160_attach.c
+++ b/sys/dev/ath/ath_hal/ar9001/ar9160_attach.c
@@ -123,13 +123,13 @@ ar9160Attach(uint16_t devid, HAL_SOFTC sc,
 	HAL_STATUS ecode;
 	HAL_BOOL rfStatus;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp5416 = ath_hal_malloc(sizeof (struct ath_hal_5416));
 	if (ahp5416 == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
@@ -320,6 +320,8 @@ ar9160FillCapabilityInfo(struct ath_hal *ah)
 	pCap->halAutoSleepSupport = AH_FALSE;	/* XXX? */
 	pCap->halMbssidAggrSupport = AH_TRUE;
 	pCap->hal4AddrAggrSupport = AH_TRUE;
+	/* BB Read WAR */
+	pCap->halHasBBReadWar = AH_TRUE;
 
 	/* AR9160 is a 2x2 stream device */
 	pCap->halTxStreams = 2;
diff --git a/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c b/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c
index 4f4b8baf2b42..62864b5ba4d7 100644
--- a/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c
+++ b/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c
@@ -153,13 +153,13 @@ ar9280Attach(uint16_t devid, HAL_SOFTC sc,
 	int8_t pwr_table_offset;
 	uint8_t pwr;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp9280 = ath_hal_malloc(sizeof (struct ath_hal_9280));
 	if (ahp9280 == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
diff --git a/sys/dev/ath/ath_hal/ar9002/ar9285_attach.c b/sys/dev/ath/ath_hal/ar9002/ar9285_attach.c
index 02c7b9892aa8..0fbe4144b7c5 100644
--- a/sys/dev/ath/ath_hal/ar9002/ar9285_attach.c
+++ b/sys/dev/ath/ath_hal/ar9002/ar9285_attach.c
@@ -118,13 +118,13 @@ ar9285Attach(uint16_t devid, HAL_SOFTC sc,
 	HAL_STATUS ecode;
 	HAL_BOOL rfStatus;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp9285 = ath_hal_malloc(sizeof (struct ath_hal_9285));
 	if (ahp9285 == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
diff --git a/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c b/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c
index 78400f52f95f..5c0de21ccbf1 100644
--- a/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c
+++ b/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c
@@ -119,13 +119,13 @@ ar9287Attach(uint16_t devid, HAL_SOFTC sc,
 	HAL_BOOL rfStatus;
 	int8_t pwr_table_offset;
 
-	HALDEBUG_G(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
 	    __func__, sc, (void*) st, (void*) sh);
 
 	/* NB: memory is returned zero'd */
 	ahp9287 = ath_hal_malloc(sizeof (struct ath_hal_9287));
 	if (ahp9287 == AH_NULL) {
-		HALDEBUG_G(AH_NULL, HAL_DEBUG_ANY,
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
 		    "%s: cannot allocate memory for state block\n", __func__);
 		*status = HAL_ENOMEM;
 		return AH_NULL;
diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c
index af612748f8c2..e1d2c0fb218f 100644
--- a/sys/dev/ath/if_ath.c
+++ b/sys/dev/ath/if_ath.c
@@ -39,6 +39,11 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_ath.h"
+/*
+ * This is needed for register operations which are performed
+ * by the driver - eg, calls to ath_hal_gettsf32().
+ */
+#include "opt_ah.h"
 #include "opt_wlan.h"
 
 #include <sys/param.h>
@@ -2031,10 +2036,6 @@ ath_calcrxfilter(struct ath_softc *sc)
 	if (ic->ic_opmode == IEEE80211_M_MONITOR)
 		rfilt |= HAL_RX_FILTER_CONTROL;
 
-	if (sc->sc_dodfs) {
-		rfilt |= HAL_RX_FILTER_PHYRADAR;
-	}
-
 	/*
 	 * Enable RX of compressed BAR frames only when doing
 	 * 802.11n. Required for A-MPDU.
@@ -2619,9 +2620,10 @@ ath_beacon_generate(struct ath_softc *sc, struct ieee80211vap *vap)
 			sc->sc_stats.ast_cabq_xmit += nmcastq;
 		}
 		/* NB: gated by beacon so safe to start here */
-		ath_hal_txstart(ah, cabq->axq_qnum);
-		ATH_TXQ_UNLOCK(cabq);
+		if (! STAILQ_EMPTY(&(cabq->axq_q)))
+			ath_hal_txstart(ah, cabq->axq_qnum);
 		ATH_TXQ_UNLOCK(&avp->av_mcastq);
+		ATH_TXQ_UNLOCK(cabq);
 	}
 	return bf;
 }
diff --git a/sys/dev/ath/if_ath_pci.c b/sys/dev/ath/if_ath_pci.c
index c9b7eecbc1b8..4824773dc8fd 100644
--- a/sys/dev/ath/if_ath_pci.c
+++ b/sys/dev/ath/if_ath_pci.c
@@ -73,6 +73,33 @@ struct ath_pci_softc {
 
 #define	BS_BAR	0x10
 #define	PCIR_RETRY_TIMEOUT	0x41
+#define	PCIR_CFG_PMCSR		0x48
+
+static void
+ath_pci_setup(device_t dev)
+{
+	/* Override the system latency timer */
+	pci_write_config(dev, PCIR_LATTIMER, 0x80, 1);
+
+	/* If a PCI NIC, force wakeup */
+#ifdef	ATH_PCI_WAKEUP_WAR
+	/* XXX TODO: don't do this for non-PCI (ie, PCIe, Cardbus!) */
+	if (1) {
+		uint16_t pmcsr;
+		pmcsr = pci_read_config(dev, PCIR_CFG_PMCSR, 2);
+		pmcsr |= 3;
+		pci_write_config(dev, PCIR_CFG_PMCSR, pmcsr, 2);
+		pmcsr &= ~3;
+		pci_write_config(dev, PCIR_CFG_PMCSR, pmcsr, 2);
+	}
+#endif
+
+	/*
+	 * Disable retry timeout to keep PCI Tx retries from
+	 * interfering with C3 CPU state.
+	 */
+	pci_write_config(dev, PCIR_RETRY_TIMEOUT, 0, 1);
+}
 
 static int
 ath_pci_probe(device_t dev)
@@ -103,10 +130,9 @@ ath_pci_attach(device_t dev)
 	pci_enable_busmaster(dev);
 
 	/*
-	 * Disable retry timeout to keep PCI Tx retries from
-	 * interfering with C3 CPU state.
+	 * Setup other PCI bus configuration parameters.
 	 */
-	pci_write_config(dev, PCIR_RETRY_TIMEOUT, 0, 1);
+	ath_pci_setup(dev);
 
 	/* 
 	 * Setup memory-mapping of PCI registers.
@@ -190,6 +216,11 @@ ath_pci_detach(device_t dev)
 	/* check if device was removed */
 	sc->sc_invalid = !bus_child_present(dev);
 
+	/*
+	 * Do a config read to clear pre-existing pci error status.
+	 */
+	(void) pci_read_config(dev, PCIR_COMMAND, 4);
+
 	ath_detach(sc);
 
 	bus_generic_detach(dev);
@@ -228,6 +259,11 @@ ath_pci_resume(device_t dev)
 {
 	struct ath_pci_softc *psc = device_get_softc(dev);
 
+	/*
+	 * Suspend/resume resets the PCI configuration space.
+	 */
+	ath_pci_setup(dev);
+
 	ath_resume(&psc->sc_sc);
 
 	return (0);
diff --git a/sys/dev/ath/if_ath_sysctl.c b/sys/dev/ath/if_ath_sysctl.c
index ea6f949bdbe4..9e52f790eb43 100644
--- a/sys/dev/ath/if_ath_sysctl.c
+++ b/sys/dev/ath/if_ath_sysctl.c
@@ -729,6 +729,23 @@ ath_sysctl_stats_attach(struct ath_softc *sc)
 	    &sc->sc_stats.ast_tx_timerexpired, 0, "TX exceeded TX_TIMER register");
 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_desccfgerr", CTLFLAG_RD,
 	    &sc->sc_stats.ast_tx_desccfgerr, 0, "TX Descriptor Cfg Error");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_swretries", CTLFLAG_RD,
+	    &sc->sc_stats.ast_tx_swretries, 0, "TX software retry count");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_swretrymax", CTLFLAG_RD,
+	    &sc->sc_stats.ast_tx_swretrymax, 0, "TX software retry max reached");
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_data_underrun", CTLFLAG_RD,
+	    &sc->sc_stats.ast_tx_data_underrun, 0, "");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_delim_underrun", CTLFLAG_RD,
+	    &sc->sc_stats.ast_tx_delim_underrun, 0, "");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_aggrfail", CTLFLAG_RD,
+	    &sc->sc_stats.ast_tx_aggrfail, 0,
+	    "Number of aggregate TX failures (whole frame)");
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_rx_intr", CTLFLAG_RD,
+	    &sc->sc_stats.ast_rx_intr, 0, "RX interrupts");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "ast_tx_intr", CTLFLAG_RD,
+	    &sc->sc_stats.ast_tx_intr, 0, "TX interrupts");
 
 	/* Attach the RX phy error array */
 	ath_sysctl_stats_attach_rxphyerr(sc, child);
diff --git a/sys/dev/ath/if_athioctl.h b/sys/dev/ath/if_athioctl.h
index 4b50d0d8265f..065a49cb7f7d 100644
--- a/sys/dev/ath/if_athioctl.h
+++ b/sys/dev/ath/if_athioctl.h
@@ -134,7 +134,16 @@ struct ath_stats {
 	u_int32_t	ast_tx_xtxop;	/* tx exceeded TXOP */
 	u_int32_t	ast_tx_timerexpired;	/* tx exceeded TX_TIMER */
 	u_int32_t	ast_tx_desccfgerr;	/* tx desc cfg error */
-	u_int32_t	ast_pad[13];
+	u_int32_t	ast_tx_swretries;	/* software TX retries */
+	u_int32_t	ast_tx_swretrymax;	/* software TX retry max limit reach */
+	u_int32_t	ast_tx_data_underrun;
+	u_int32_t	ast_tx_delim_underrun;
+	u_int32_t	ast_tx_aggrfail;		/* aggregate TX failed in its entirety */
+	u_int32_t	ast_tx_getnobuf;
+	u_int32_t	ast_tx_getbusybuf;
+	u_int32_t	ast_tx_intr;
+	u_int32_t	ast_rx_intr;
+	u_int32_t	ast_pad[4];
 };
 
 #define	SIOCGATHSTATS	_IOWR('i', 137, struct ifreq)
diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h
index 5c468e7f7646..1eaf709fc895 100644
--- a/sys/dev/ath/if_athvar.h
+++ b/sys/dev/ath/if_athvar.h
@@ -64,7 +64,7 @@
 #define	ATH_TXMGTTRY	4		/* xmit attempts for mgt/ctl frames */
 #define	ATH_TXINTR_PERIOD 5		/* max number of batched tx descriptors */
 
-#define	ATH_BEACON_AIFS_DEFAULT	 0	/* default aifs for ap beacon q */
+#define	ATH_BEACON_AIFS_DEFAULT	 1	/* default aifs for ap beacon q */
 #define	ATH_BEACON_CWMIN_DEFAULT 0	/* default cwmin for ap beacon q */
 #define	ATH_BEACON_CWMAX_DEFAULT 0	/* default cwmax for ap beacon q */
 
diff --git a/sys/dev/bce/if_bce.c b/sys/dev/bce/if_bce.c
index c048f3f097da..dae8d5fe16a1 100644
--- a/sys/dev/bce/if_bce.c
+++ b/sys/dev/bce/if_bce.c
@@ -1112,8 +1112,7 @@ bce_attach(device_t dev)
 			DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using MSI "
 			    "interrupt.\n", __FUNCTION__);
 			sc->bce_flags |= BCE_USING_MSI_FLAG;
-			if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-				(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
+			if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)
 				sc->bce_flags |= BCE_ONE_SHOT_MSI_FLAG;
 			sc->bce_irq_rid = 1;
 			sc->bce_intr = bce_intr;
@@ -1730,8 +1729,7 @@ bce_ctx_rd(struct bce_softc *sc, u32 cid_addr, u32 ctx_offset)
 
 	offset = ctx_offset + cid_addr;
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 
 		REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_READ_REQ));
 
@@ -1783,8 +1781,7 @@ bce_ctx_wr(struct bce_softc *sc, u32 cid_addr, u32 ctx_offset, u32 ctx_val)
 		BCE_PRINTF("%s(): Invalid CID address: 0x%08X.\n",
 		    __FUNCTION__, cid_addr));
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 
 		REG_WR(sc, BCE_CTX_CTX_DATA, ctx_val);
 		REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_WRITE_REQ));
@@ -2469,8 +2466,7 @@ bce_init_nvram(struct bce_softc *sc)
 
 	DBENTER(BCE_VERBOSE_NVRAM);
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		sc->bce_flash_info = &flash_5709;
 		goto bce_init_nvram_get_flash_size;
 	}
@@ -3224,8 +3220,7 @@ bce_dma_free(struct bce_softc *sc)
 
 
 	/* Free, unmap and destroy all context memory pages. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		for (i = 0; i < sc->ctx_pages; i++ ) {
 			if (sc->ctx_block[i] != NULL) {
 				bus_dmamem_free(
@@ -3564,8 +3559,7 @@ bce_dma_alloc(device_t dev)
 	    __FUNCTION__, (uintmax_t) sc->stats_block_paddr);
 
 	/* BCM5709 uses host memory as cache for context memory. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		sc->ctx_pages = 0x2000 / BCM_PAGE_SIZE;
 		if (sc->ctx_pages == 0)
 			sc->ctx_pages = 1;
@@ -4206,8 +4200,7 @@ bce_init_rxp_cpu(struct bce_softc *sc)
 	cpu_reg.spad_base = BCE_RXP_SCRATCH;
 	cpu_reg.mips_view_base = 0x8000000;
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
  		fw.ver_major = bce_RXP_b09FwReleaseMajor;
 		fw.ver_minor = bce_RXP_b09FwReleaseMinor;
 		fw.ver_fix = bce_RXP_b09FwReleaseFix;
@@ -4305,8 +4298,7 @@ bce_init_txp_cpu(struct bce_softc *sc)
 	cpu_reg.spad_base = BCE_TXP_SCRATCH;
 	cpu_reg.mips_view_base = 0x8000000;
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		fw.ver_major = bce_TXP_b09FwReleaseMajor;
 		fw.ver_minor = bce_TXP_b09FwReleaseMinor;
 		fw.ver_fix = bce_TXP_b09FwReleaseFix;
@@ -4403,8 +4395,7 @@ bce_init_tpat_cpu(struct bce_softc *sc)
 	cpu_reg.spad_base = BCE_TPAT_SCRATCH;
 	cpu_reg.mips_view_base = 0x8000000;
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		fw.ver_major = bce_TPAT_b09FwReleaseMajor;
 		fw.ver_minor = bce_TPAT_b09FwReleaseMinor;
 		fw.ver_fix = bce_TPAT_b09FwReleaseFix;
@@ -4501,8 +4492,7 @@ bce_init_cp_cpu(struct bce_softc *sc)
 	cpu_reg.spad_base = BCE_CP_SCRATCH;
 	cpu_reg.mips_view_base = 0x8000000;
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		fw.ver_major = bce_CP_b09FwReleaseMajor;
 		fw.ver_minor = bce_CP_b09FwReleaseMinor;
 		fw.ver_fix = bce_CP_b09FwReleaseFix;
@@ -4599,8 +4589,7 @@ bce_init_com_cpu(struct bce_softc *sc)
 	cpu_reg.spad_base = BCE_COM_SCRATCH;
 	cpu_reg.mips_view_base = 0x8000000;
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		fw.ver_major = bce_COM_b09FwReleaseMajor;
 		fw.ver_minor = bce_COM_b09FwReleaseMinor;
 		fw.ver_fix = bce_COM_b09FwReleaseFix;
@@ -4683,8 +4672,7 @@ bce_init_cpus(struct bce_softc *sc)
 {
 	DBENTER(BCE_VERBOSE_RESET);
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 
 		if ((BCE_CHIP_REV(sc) == BCE_CHIP_REV_Ax)) {
 			bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc1,
@@ -4732,8 +4720,7 @@ bce_init_ctx(struct bce_softc *sc)
 	rc = 0;
 	DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		retry_cnt = CTX_INIT_RETRY_COUNT;
 
 		DBPRINT(sc, BCE_INFO_CTX, "Initializing 5709 context.\n");
@@ -4960,8 +4947,7 @@ bce_reset(struct bce_softc *sc, u32 reset_code)
 	DELAY(5);
 
 	/* Disable DMA */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
 		val &= ~BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
 		REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
@@ -4983,8 +4969,7 @@ bce_reset(struct bce_softc *sc, u32 reset_code)
 	val = REG_RD(sc, BCE_MISC_ID);
 
 	/* Chip reset. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		REG_WR(sc, BCE_MISC_COMMAND, BCE_MISC_COMMAND_SW_RESET);
 		REG_RD(sc, BCE_MISC_COMMAND);
 		DELAY(5);
@@ -5113,8 +5098,7 @@ bce_chipinit(struct bce_softc *sc)
 	val |= BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE_256;
 
 	/* Enable bins used on the 5709. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		val |= BCE_MQ_CONFIG_BIN_MQ_MODE;
 		if (BCE_CHIP_ID(sc) == BCE_CHIP_ID_5709_A1)
 			val |= BCE_MQ_CONFIG_HALT_DIS;
@@ -5268,8 +5252,7 @@ bce_blockinit(struct bce_softc *sc)
 	}
 
 	/* Enable DMA */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
 		val |= BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
 		REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
@@ -5293,8 +5276,7 @@ bce_blockinit(struct bce_softc *sc)
 	}
 
 	/* Enable all remaining blocks in the MAC. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)
 		REG_WR(sc, BCE_MISC_ENABLE_SET_BITS,
 		    BCE_MISC_ENABLE_DEFAULT_XI);
 	else
@@ -5565,8 +5547,7 @@ bce_init_tx_context(struct bce_softc *sc)
 	DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
 
 	/* Initialize the context ID for an L2 TX chain. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		/* Set the CID type to support an L2 connection. */
 		val = BCE_L2CTX_TX_TYPE_TYPE_L2_XI |
 		    BCE_L2CTX_TX_TYPE_SIZE_L2_XI;
@@ -5729,8 +5710,7 @@ bce_init_rx_context(struct bce_softc *sc)
 	 * when pause frames can be stopped (the high
 	 * watermark).
 	 */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		u32 lo_water, hi_water;
 
 		if (sc->bce_flags & BCE_USING_TX_FLOW_CONTROL) {
@@ -5764,8 +5744,7 @@ bce_init_rx_context(struct bce_softc *sc)
 	CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_CTX_TYPE, val);
 
 	/* Setup the MQ BIN mapping for l2_ctx_host_bseq. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		val = REG_RD(sc, BCE_MQ_MAP_L2_5);
 		REG_WR(sc, BCE_MQ_MAP_L2_5, val | BCE_MQ_MAP_L2_5_ARM);
 	}
@@ -5983,8 +5962,7 @@ bce_init_pg_chain(struct bce_softc *sc)
 	}
 
 	/* Setup the MQ BIN mapping for host_pg_bidx. */
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)	||
-		(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)
 		REG_WR(sc, BCE_MQ_MAP_L2_3, BCE_MQ_MAP_L2_3_DEFAULT);
 
 	CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_PG_BUF_SIZE, 0);
@@ -9912,8 +9890,7 @@ bce_dump_ctx(struct bce_softc *sc, u16 cid)
 		    "consumer index\n",	CTX_RD(sc, GET_CID_ADDR(cid),
 		    BCE_L2CTX_RX_NX_PG_BDIDX));
 	} else if (cid == TX_CID) {
-		if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-		    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+		if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 			BCE_PRINTF(" 0x%08X - (L2CTX_TX_TYPE_XI) ctx type\n",
 			    CTX_RD(sc, GET_CID_ADDR(cid),
 			    BCE_L2CTX_TX_TYPE_XI));
@@ -10175,8 +10152,7 @@ bce_dump_ftqs(struct bce_softc *sc)
 	    (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_CPQ_VALID_CNT  <<  8) |
 	    (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_MGMQ_VALID_CNT);
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
+	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)
 		val = val |
 		    (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PCSQ_VALID_CNT_XI <<
 		     24);
@@ -10209,8 +10185,7 @@ bce_dump_ftqs(struct bce_softc *sc)
 	BCE_PRINTF(" CS      0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
 	    cmd, ctl, cur_depth, max_depth, valid_cnt);
 
-	if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
-	    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
+	if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 		/* Input queue to the RV2P Command Scheduler */
 		cmd = REG_RD(sc, BCE_RV2PCSR_FTQ_CMD);
 		ctl = REG_RD(sc, BCE_RV2PCSR_FTQ_CTL);
diff --git a/sys/dev/bce/if_bcereg.h b/sys/dev/bce/if_bcereg.h
index f4215db49bee..f74c4d208934 100644
--- a/sys/dev/bce/if_bcereg.h
+++ b/sys/dev/bce/if_bcereg.h
@@ -576,7 +576,6 @@ default: DBPRINT(sc, BCE_INSANE_PHY,					\
 #define BCE_CHIP_NUM_5706		0x57060000
 #define BCE_CHIP_NUM_5708		0x57080000
 #define BCE_CHIP_NUM_5709		0x57090000
-#define BCE_CHIP_NUM_5716		0x57160000
 
 #define BCE_CHIP_REV(sc)		(((sc)->bce_chipid) & 0x0000f000)
 #define BCE_CHIP_REV_Ax			0x00000000
@@ -601,7 +600,6 @@ default: DBPRINT(sc, BCE_INSANE_PHY,					\
 #define BCE_CHIP_ID_5709_B1		0x57091010
 #define BCE_CHIP_ID_5709_B2		0x57091020
 #define BCE_CHIP_ID_5709_C0		0x57092000
-#define BCE_CHIP_ID_5716_C0		0x57162000
 
 #define BCE_CHIP_BOND_ID(sc)		(((sc)->bce_chipid) & 0xf)
 
diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c
index 0f87eb0ba5f5..4deecc8f659a 100644
--- a/sys/dev/bge/if_bge.c
+++ b/sys/dev/bge/if_bge.c
@@ -1309,15 +1309,17 @@ bge_sig_pre_reset(struct bge_softc *sc, int type)
 	 * Some chips don't like this so only do this if ASF is enabled
 	 */
 	if (sc->bge_asf_mode)
-		bge_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
+		bge_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
 
 	if (sc->bge_asf_mode & ASF_NEW_HANDSHAKE) {
 		switch (type) {
 		case BGE_RESET_START:
-			bge_writemem_ind(sc, BGE_SDI_STATUS, 0x1); /* START */
+			bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
+			    BGE_FW_DRV_STATE_START);
 			break;
 		case BGE_RESET_STOP:
-			bge_writemem_ind(sc, BGE_SDI_STATUS, 0x2); /* UNLOAD */
+			bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
+			    BGE_FW_DRV_STATE_UNLOAD);
 			break;
 		}
 	}
@@ -1330,11 +1332,13 @@ bge_sig_post_reset(struct bge_softc *sc, int type)
 	if (sc->bge_asf_mode & ASF_NEW_HANDSHAKE) {
 		switch (type) {
 		case BGE_RESET_START:
-			bge_writemem_ind(sc, BGE_SDI_STATUS, 0x80000001);
+			bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
+			    BGE_FW_DRV_STATE_START_DONE);
 			/* START DONE */
 			break;
 		case BGE_RESET_STOP:
-			bge_writemem_ind(sc, BGE_SDI_STATUS, 0x80000002);
+			bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
+			    BGE_FW_DRV_STATE_UNLOAD_DONE);
 			break;
 		}
 	}
@@ -1347,10 +1351,12 @@ bge_sig_legacy(struct bge_softc *sc, int type)
 	if (sc->bge_asf_mode) {
 		switch (type) {
 		case BGE_RESET_START:
-			bge_writemem_ind(sc, BGE_SDI_STATUS, 0x1); /* START */
+			bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
+			    BGE_FW_DRV_STATE_START);
 			break;
 		case BGE_RESET_STOP:
-			bge_writemem_ind(sc, BGE_SDI_STATUS, 0x2); /* UNLOAD */
+			bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
+			    BGE_FW_DRV_STATE_UNLOAD);
 			break;
 		}
 	}
@@ -1362,12 +1368,12 @@ bge_stop_fw(struct bge_softc *sc)
 	int i;
 
 	if (sc->bge_asf_mode) {
-		bge_writemem_ind(sc, BGE_SOFTWARE_GENCOMM_FW, BGE_FW_PAUSE);
-		CSR_WRITE_4(sc, BGE_CPU_EVENT,
-		    CSR_READ_4(sc, BGE_CPU_EVENT) | (1 << 14));
+		bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB, BGE_FW_PAUSE);
+		CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
+		    CSR_READ_4(sc, BGE_RX_CPU_EVENT) | (1 << 14));
 
 		for (i = 0; i < 100; i++ ) {
-			if (!(CSR_READ_4(sc, BGE_CPU_EVENT) & (1 << 14)))
+			if (!(CSR_READ_4(sc, BGE_RX_CPU_EVENT) & (1 << 14)))
 				break;
 			DELAY(10);
 		}
@@ -2828,7 +2834,6 @@ bge_attach(device_t dev)
 	switch (sc->bge_asicrev) {
 	case BGE_ASICREV_BCM5717:
 	case BGE_ASICREV_BCM5719:
-		sc->bge_flags |= BGE_FLAG_SHORT_DMA_BUG;
 	case BGE_ASICREV_BCM57765:
 		sc->bge_flags |= BGE_FLAG_5717_PLUS | BGE_FLAG_5755_PLUS |
 		    BGE_FLAG_575X_PLUS | BGE_FLAG_5705_PLUS | BGE_FLAG_JUMBO |
@@ -2863,8 +2868,6 @@ bge_attach(device_t dev)
 	case BGE_ASICREV_BCM5752:
 	case BGE_ASICREV_BCM5906:
 		sc->bge_flags |= BGE_FLAG_575X_PLUS;
-		if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
-			sc->bge_flags |= BGE_FLAG_SHORT_DMA_BUG;
 		/* FALLTHROUGH */
 	case BGE_ASICREV_BCM5705:
 		sc->bge_flags |= BGE_FLAG_5705_PLUS;
@@ -2918,15 +2921,26 @@ bge_attach(device_t dev)
 		sc->bge_mi_mode |= BGE_MIMODE_AUTOPOLL;
 
 	/*
-	 * All controllers that are not 5755 or higher have 4GB
-	 * boundary DMA bug.
+	 * All Broadcom controllers have 4GB boundary DMA bug.
 	 * Whenever an address crosses a multiple of the 4GB boundary
 	 * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
 	 * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
 	 * state machine will lockup and cause the device to hang.
 	 */
-	if (BGE_IS_5755_PLUS(sc) == 0)
-		sc->bge_flags |= BGE_FLAG_4G_BNDRY_BUG;
+	sc->bge_flags |= BGE_FLAG_4G_BNDRY_BUG;
+
+	/* BCM5755 or higher and BCM5906 have short DMA bug. */
+	if (BGE_IS_5755_PLUS(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5906)
+		sc->bge_flags |= BGE_FLAG_SHORT_DMA_BUG;
+
+	/*
+	 * BCM5719 cannot handle DMA requests for DMA segments that
+	 * have larger than 4KB in size.  However the maximum DMA
+	 * segment size created in DMA tag is 4KB for TSO, so we
+	 * wouldn't encounter the issue here.
+	 */
+	if (sc->bge_asicrev == BGE_ASICREV_BCM5719)
+		sc->bge_flags |= BGE_FLAG_4K_RDMA_BUG;
 
 	misccfg = CSR_READ_4(sc, BGE_MISC_CFG) & BGE_MISCCFG_BOARD_ID;
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5705) {
@@ -2985,9 +2999,9 @@ bge_attach(device_t dev)
 			sc->bge_flags |= BGE_FLAG_TSO;
 	}
 
-  	/*
+	/*
 	 * Check if this is a PCI-X or PCI Express device.
-  	 */
+	 */
 	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
 		/*
 		 * Found a PCI Express capabilities register, this
@@ -3073,9 +3087,9 @@ bge_attach(device_t dev)
 	}
 
 	sc->bge_asf_mode = 0;
-	if (bge_allow_asf && (bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG)
-	    == BGE_MAGIC_NUMBER)) {
-		if (bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG)
+	if (bge_allow_asf && (bge_readmem_ind(sc, BGE_SRAM_DATA_SIG) ==
+	    BGE_SRAM_DATA_SIG_MAGIC)) {
+		if (bge_readmem_ind(sc, BGE_SRAM_DATA_CFG)
 		    & BGE_HWCFG_ASF) {
 			sc->bge_asf_mode |= ASF_ENABLE;
 			sc->bge_asf_mode |= ASF_STACKUP;
@@ -3189,8 +3203,8 @@ bge_attach(device_t dev)
 	 * by its PCI subsystem ID, as we do below for the SysKonnect
 	 * SK-9D41.
 	 */
-	if (bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER)
-		hwcfg = bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
+	if (bge_readmem_ind(sc, BGE_SRAM_DATA_SIG) == BGE_SRAM_DATA_SIG_MAGIC)
+		hwcfg = bge_readmem_ind(sc, BGE_SRAM_DATA_CFG);
 	else if ((sc->bge_flags & BGE_FLAG_EADDR) &&
 	    (sc->bge_asicrev != BGE_ASICREV_BCM5906)) {
 		if (bge_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
@@ -3425,9 +3439,9 @@ bge_reset(struct bge_softc *sc)
 	/*
 	 * Write the magic number to SRAM at offset 0xB50.
 	 * When firmware finishes its initialization it will
-	 * write ~BGE_MAGIC_NUMBER to the same location.
+	 * write ~BGE_SRAM_FW_MB_MAGIC to the same location.
 	 */
-	bge_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
+	bge_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
 
 	reset = BGE_MISCCFG_RESET_CORE_CLOCKS | BGE_32BITTIME_66MHZ;
 
@@ -3446,7 +3460,8 @@ bge_reset(struct bge_softc *sc)
 	 * Set GPHY Power Down Override to leave GPHY
 	 * powered up in D0 uninitialized.
 	 */
-	if (BGE_IS_5705_PLUS(sc))
+	if (BGE_IS_5705_PLUS(sc) &&
+	    (sc->bge_flags & BGE_FLAG_CPMU_PRESENT) == 0)
 		reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
 
 	/* Issue global reset */
@@ -3551,8 +3566,8 @@ bge_reset(struct bge_softc *sc)
 		 */
 		for (i = 0; i < BGE_TIMEOUT; i++) {
 			DELAY(10);
-			val = bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
-			if (val == ~BGE_MAGIC_NUMBER)
+			val = bge_readmem_ind(sc, BGE_SRAM_FW_MB);
+			if (val == ~BGE_SRAM_FW_MB_MAGIC)
 				break;
 		}
 
@@ -3995,7 +4010,7 @@ bge_intr_task(void *arg, int pending)
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 		/* Check TX ring producer/consumer. */
 		bge_txeof(sc, tx_cons);
-	    	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 			bge_start_locked(ifp);
 	}
 	BGE_UNLOCK(sc);
@@ -4091,12 +4106,12 @@ bge_asf_driver_up(struct bge_softc *sc)
 			sc->bge_asf_count --;
 		else {
 			sc->bge_asf_count = 2;
-			bge_writemem_ind(sc, BGE_SOFTWARE_GENCOMM_FW,
+			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB,
 			    BGE_FW_DRV_ALIVE);
-			bge_writemem_ind(sc, BGE_SOFTWARE_GENNCOMM_FW_LEN, 4);
-			bge_writemem_ind(sc, BGE_SOFTWARE_GENNCOMM_FW_DATA, 3);
-			CSR_WRITE_4(sc, BGE_CPU_EVENT,
-			    CSR_READ_4(sc, BGE_CPU_EVENT) | (1 << 14));
+			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_LEN_MB, 4);
+			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_DATA_MB, 3);
+			CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
+			    CSR_READ_4(sc, BGE_RX_CPU_EVENT) | (1 << 14));
 		}
 	}
 }
@@ -4112,7 +4127,7 @@ bge_tick(void *xsc)
 	/* Synchronize with possible callout reset/stop. */
 	if (callout_pending(&sc->bge_stat_ch) ||
 	    !callout_active(&sc->bge_stat_ch))
-	    	return;
+		return;
 
 	if (BGE_IS_5705_PLUS(sc))
 		bge_stats_update_regs(sc);
@@ -5036,7 +5051,7 @@ bge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 				break;
 			}
 		} else if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > ETHERMTU) {
-			error = EINVAL; 
+			error = EINVAL;
 			break;
 		}
 		BGE_LOCK(sc);
@@ -5797,8 +5812,7 @@ bge_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
 {
 	struct bge_softc *sc;
 	uint16_t *sbdata;
-	int error;
-	int result;
+	int error, result, sbsz;
 	int i, j;
 
 	result = -1;
@@ -5809,14 +5823,21 @@ bge_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
 	if (result == 1) {
 		sc = (struct bge_softc *)arg1;
 
+		if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
+		    sc->bge_chipid != BGE_CHIPID_BCM5700_C0)
+			sbsz = BGE_STATUS_BLK_SZ;
+		else
+			sbsz = 32;
 		sbdata = (uint16_t *)sc->bge_ldata.bge_status_block;
 		printf("Status Block:\n");
-		for (i = 0x0; i < (BGE_STATUS_BLK_SZ / 4); ) {
+		BGE_LOCK(sc);
+		bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
+		    sc->bge_cdata.bge_status_map,
+		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+		for (i = 0x0; i < sbsz / sizeof(uint16_t); ) {
 			printf("%06x:", i);
-			for (j = 0; j < 8; j++) {
-				printf(" %04x", sbdata[i]);
-				i += 4;
-			}
+			for (j = 0; j < 8; j++)
+				printf(" %04x", sbdata[i++]);
 			printf("\n");
 		}
 
@@ -5829,8 +5850,11 @@ bge_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
 			}
 			printf("\n");
 		}
+		BGE_UNLOCK(sc);
 
 		printf("Hardware Flags:\n");
+		if (BGE_IS_5717_PLUS(sc))
+			printf(" - 5717 Plus\n");
 		if (BGE_IS_5755_PLUS(sc))
 			printf(" - 5755 Plus\n");
 		if (BGE_IS_575X_PLUS(sc))
@@ -5920,11 +5944,11 @@ bge_get_eaddr_mem(struct bge_softc *sc, uint8_t ether_addr[])
 {
 	uint32_t mac_addr;
 
-	mac_addr = bge_readmem_ind(sc, 0x0c14);
+	mac_addr = bge_readmem_ind(sc, BGE_SRAM_MAC_ADDR_HIGH_MB);
 	if ((mac_addr >> 16) == 0x484b) {
 		ether_addr[0] = (uint8_t)(mac_addr >> 8);
 		ether_addr[1] = (uint8_t)mac_addr;
-		mac_addr = bge_readmem_ind(sc, 0x0c18);
+		mac_addr = bge_readmem_ind(sc, BGE_SRAM_MAC_ADDR_LOW_MB);
 		ether_addr[2] = (uint8_t)(mac_addr >> 24);
 		ether_addr[3] = (uint8_t)(mac_addr >> 16);
 		ether_addr[4] = (uint8_t)(mac_addr >> 8);
diff --git a/sys/dev/bge/if_bgereg.h b/sys/dev/bge/if_bgereg.h
index 9b830211756f..7973c8647547 100644
--- a/sys/dev/bge/if_bgereg.h
+++ b/sys/dev/bge/if_bgereg.h
@@ -71,12 +71,15 @@
 #define	BGE_STATS_BLOCK_END		0x00000AFF
 #define	BGE_STATUS_BLOCK		0x00000B00
 #define	BGE_STATUS_BLOCK_END		0x00000B4F
-#define	BGE_SOFTWARE_GENCOMM		0x00000B50
-#define	BGE_SOFTWARE_GENCOMM_SIG	0x00000B54
-#define	BGE_SOFTWARE_GENCOMM_NICCFG	0x00000B58
-#define	BGE_SOFTWARE_GENCOMM_FW		0x00000B78
-#define	BGE_SOFTWARE_GENNCOMM_FW_LEN	0x00000B7C
-#define	BGE_SOFTWARE_GENNCOMM_FW_DATA	0x00000B80
+#define	BGE_SRAM_FW_MB			0x00000B50
+#define	BGE_SRAM_DATA_SIG		0x00000B54
+#define	BGE_SRAM_DATA_CFG		0x00000B58
+#define	BGE_SRAM_FW_CMD_MB		0x00000B78
+#define	BGE_SRAM_FW_CMD_LEN_MB		0x00000B7C
+#define	BGE_SRAM_FW_CMD_DATA_MB		0x00000B80
+#define	BGE_SRAM_FW_DRV_STATE_MB	0x00000C04
+#define	BGE_SRAM_MAC_ADDR_HIGH_MB	0x00000C14
+#define	BGE_SRAM_MAC_ADDR_LOW_MB	0x00000C18
 #define	BGE_SOFTWARE_GENCOMM_END	0x00000FFF
 #define	BGE_UNMAPPED			0x00001000
 #define	BGE_UNMAPPED_END		0x00001FFF
@@ -87,9 +90,17 @@
 #define	BGE_SEND_RING_1_TO_4_END	0x00005FFF
 
 /* Firmware interface */
+#define	BGE_SRAM_DATA_SIG_MAGIC		0x4B657654	/* 'KevT' */
 #define	BGE_FW_DRV_ALIVE		0x00000001
 #define	BGE_FW_PAUSE			0x00000002
 
+#define	BGE_FW_DRV_STATE_START		0x00000001
+#define	BGE_FW_DRV_STATE_START_DONE	0x80000001
+#define	BGE_FW_DRV_STATE_UNLOAD		0x00000002
+#define	BGE_FW_DRV_STATE_UNLOAD_DONE	0x80000002
+#define	BGE_FW_DRV_STATE_WOL		0x00000003
+#define	BGE_FW_DRV_STATE_SUSPEND	0x00000004
+
 /* Mappings for internal memory configuration */
 #define	BGE_STD_RX_RINGS		0x00006000
 #define	BGE_STD_RX_RINGS_END		0x00006FFF
@@ -1875,7 +1886,8 @@
 #define	BGE_MODE_CTL			0x6800
 #define	BGE_MISC_CFG			0x6804
 #define	BGE_MISC_LOCAL_CTL		0x6808
-#define	BGE_CPU_EVENT			0x6810
+#define	BGE_RX_CPU_EVENT		0x6810
+#define	BGE_TX_CPU_EVENT		0x6820
 #define	BGE_EE_ADDR			0x6838
 #define	BGE_EE_DATA			0x683C
 #define	BGE_EE_CTL			0x6840
@@ -2052,10 +2064,10 @@
  * This magic number is written to the firmware mailbox at 0xb50
  * before a software reset is issued.  After the internal firmware
  * has completed its initialization it will write the opposite of
- * this value, ~BGE_MAGIC_NUMBER, to the same location, allowing the
- * driver to synchronize with the firmware.
+ * this value, ~BGE_SRAM_FW_MB_MAGIC, to the same location,
+ * allowing the driver to synchronize with the firmware.
  */
-#define	BGE_MAGIC_NUMBER                0x4B657654
+#define	BGE_SRAM_FW_MB_MAGIC	0x4B657654
 
 typedef struct {
 	uint32_t		bge_addr_hi;
@@ -2797,6 +2809,7 @@ struct bge_softc {
 #define	BGE_FLAG_4G_BNDRY_BUG	0x02000000
 #define	BGE_FLAG_RX_ALIGNBUG	0x04000000
 #define	BGE_FLAG_SHORT_DMA_BUG	0x08000000
+#define	BGE_FLAG_4K_RDMA_BUG	0x10000000
 	uint32_t		bge_phy_flags;
 #define	BGE_PHY_NO_WIRESPEED	0x00000001
 #define	BGE_PHY_ADC_BUG		0x00000002
diff --git a/sys/dev/bktr/bktr_core.c b/sys/dev/bktr/bktr_core.c
index dd7d09ea793f..60aa9e39ff25 100644
--- a/sys/dev/bktr/bktr_core.c
+++ b/sys/dev/bktr/bktr_core.c
@@ -918,7 +918,7 @@ common_bktr_intr( void *arg )
 
 		if (bktr->proc != NULL) {
 			PROC_LOCK(bktr->proc);
-			psignal( bktr->proc, bktr->signal);
+			kern_psignal( bktr->proc, bktr->signal);
 			PROC_UNLOCK(bktr->proc);
 		}
 
diff --git a/sys/dev/bwi/bwiphy.c b/sys/dev/bwi/bwiphy.c
index c41c0f854872..d817d2603447 100644
--- a/sys/dev/bwi/bwiphy.c
+++ b/sys/dev/bwi/bwiphy.c
@@ -796,7 +796,7 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 			bwi_tbl_write_2(mac, BWI_PHYTBL_RSSI + i, i);
 
 		/* Fill noise table */
-		for (i = 0; i < sizeof(bwi_phy_noise_11g); ++i) {
+		for (i = 0; i < N(bwi_phy_noise_11g); ++i) {
 			bwi_tbl_write_2(mac, BWI_PHYTBL_NOISE + i,
 					bwi_phy_noise_11g[i]);
 		}
diff --git a/sys/dev/bwi/if_bwi.c b/sys/dev/bwi/if_bwi.c
index 7ec08f716a39..b1555075255a 100644
--- a/sys/dev/bwi/if_bwi.c
+++ b/sys/dev/bwi/if_bwi.c
@@ -118,8 +118,7 @@ static void	bwi_calibrate(void *);
 
 static int	bwi_calc_rssi(struct bwi_softc *, const struct bwi_rxbuf_hdr *);
 static int	bwi_calc_noise(struct bwi_softc *);
-static __inline uint8_t bwi_ofdm_plcp2rate(const uint32_t *);
-static __inline uint8_t bwi_ds_plcp2rate(const struct ieee80211_ds_plcp_hdr *);
+static __inline uint8_t bwi_plcp2rate(uint32_t, enum ieee80211_phymode);
 static void	bwi_rx_radiotap(struct bwi_softc *, struct mbuf *,
 			struct bwi_rxbuf_hdr *, const void *, int, int, int);
 
@@ -221,7 +220,7 @@ static const struct {
 } bwi_bbpid_map[] = {
 	{ 0x4301, 0x4301, 0x4301 },
 	{ 0x4305, 0x4307, 0x4307 },
-	{ 0x4403, 0x4403, 0x4402 },
+	{ 0x4402, 0x4403, 0x4402 },
 	{ 0x4610, 0x4615, 0x4610 },
 	{ 0x4710, 0x4715, 0x4710 },
 	{ 0x4720, 0x4725, 0x4309 }
@@ -2629,7 +2628,7 @@ bwi_rxeof(struct bwi_softc *sc, int end_idx)
 		struct ieee80211_frame_min *wh;
 		struct ieee80211_node *ni;
 		struct mbuf *m;
-		const void *plcp;
+		uint32_t plcp;
 		uint16_t flags2;
 		int buflen, wh_ofs, hdr_extra, rssi, noise, type, rate;
 
@@ -2659,7 +2658,7 @@ bwi_rxeof(struct bwi_softc *sc, int end_idx)
 			goto next;
 		}
 
-		plcp = ((const uint8_t *)(hdr + 1) + hdr_extra);
+	        bcopy((uint8_t *)(hdr + 1) + hdr_extra, &plcp, sizeof(plcp));	
 		rssi = bwi_calc_rssi(sc, hdr);
 		noise = bwi_calc_noise(sc);
 
@@ -2668,13 +2667,13 @@ bwi_rxeof(struct bwi_softc *sc, int end_idx)
 		m_adj(m, sizeof(*hdr) + wh_ofs);
 
 		if (htole16(hdr->rxh_flags1) & BWI_RXH_F1_OFDM)
-			rate = bwi_ofdm_plcp2rate(plcp);
+			rate = bwi_plcp2rate(plcp, IEEE80211_MODE_11G);
 		else
-			rate = bwi_ds_plcp2rate(plcp);
+			rate = bwi_plcp2rate(plcp, IEEE80211_MODE_11B);
 
 		/* RX radio tap */
 		if (ieee80211_radiotap_active(ic))
-			bwi_rx_radiotap(sc, m, hdr, plcp, rate, rssi, noise);
+			bwi_rx_radiotap(sc, m, hdr, &plcp, rate, rssi, noise);
 
 		m_adj(m, -IEEE80211_CRC_LEN);
 
@@ -3802,20 +3801,10 @@ bwi_calc_noise(struct bwi_softc *sc)
 }
 
 static __inline uint8_t
-bwi_ofdm_plcp2rate(const uint32_t *plcp0)
+bwi_plcp2rate(const uint32_t plcp0, enum ieee80211_phymode phymode)
 {
-	uint32_t plcp;
-	uint8_t plcp_rate;
-
-	plcp = le32toh(*plcp0);
-	plcp_rate = __SHIFTOUT(plcp, IEEE80211_OFDM_PLCP_RATE_MASK);
-	return ieee80211_plcp2rate(plcp_rate, IEEE80211_T_OFDM);
-}
-
-static __inline uint8_t
-bwi_ds_plcp2rate(const struct ieee80211_ds_plcp_hdr *hdr)
-{
-	return ieee80211_plcp2rate(hdr->i_signal, IEEE80211_T_DS);
+       uint32_t plcp = le32toh(plcp0) & IEEE80211_OFDM_PLCP_RATE_MASK;
+	return (ieee80211_plcp2rate(plcp, phymode));
 }
 
 static void
diff --git a/sys/dev/bwn/if_bwn.c b/sys/dev/bwn/if_bwn.c
index a79ec113b967..c924a346e6b4 100644
--- a/sys/dev/bwn/if_bwn.c
+++ b/sys/dev/bwn/if_bwn.c
@@ -3213,8 +3213,6 @@ bwn_core_init(struct bwn_mac *mac)
 		bwn_pio_init(mac);
 	else
 		bwn_dma_init(mac);
-	if (error)
-		goto fail1;
 	bwn_wme_init(mac);
 	bwn_spu_setdelay(mac, 1);
 	bwn_bt_enable(mac);
@@ -3230,8 +3228,6 @@ bwn_core_init(struct bwn_mac *mac)
 
 	return (error);
 
-fail1:
-	bwn_chip_exit(mac);
 fail0:
 	siba_powerdown(sc->sc_dev);
 	KASSERT(mac->mac_status == BWN_MAC_STATUS_UNINIT,
diff --git a/sys/dev/bxe/bxe_link.c b/sys/dev/bxe/bxe_link.c
index 8adc87e87fb4..a83f698dfe93 100644
--- a/sys/dev/bxe/bxe_link.c
+++ b/sys/dev/bxe/bxe_link.c
@@ -2586,7 +2586,7 @@ bxe_8726_read_sfp_module_eeprom(struct link_params *params, uint16_t addr,
 
 	if (byte_cnt > 16) {
 		DBPRINT(sc, BXE_VERBOSE_PHY,
-		    "Reading from eeprom is is limited to 0xf\n");
+		    "Reading from eeprom is limited to 0xf\n");
 		return (-EINVAL);
 	}
 	/* Set the read command byte count. */
@@ -2655,7 +2655,7 @@ bxe_8727_read_sfp_module_eeprom(struct link_params *params, uint16_t addr,
 
 	if (byte_cnt > 16) {
 		DBPRINT(sc, BXE_VERBOSE_PHY,
-		    "Reading from eeprom is is limited to 0xf\n");
+		    "Reading from eeprom is limited to 0xf\n");
 		return (-EINVAL);
 	}
 
diff --git a/sys/dev/ciss/ciss.c b/sys/dev/ciss/ciss.c
index e892e5061a91..016d5f3d7997 100644
--- a/sys/dev/ciss/ciss.c
+++ b/sys/dev/ciss/ciss.c
@@ -3249,13 +3249,17 @@ ciss_cam_complete(struct ciss_request *cr)
 	      ce->sense_length, ce->residual_count);
 	bzero(&csio->sense_data, SSD_FULL_SIZE);
 	bcopy(&ce->sense_info[0], &csio->sense_data, ce->sense_length);
-	csio->sense_len = ce->sense_length;
+	if (csio->sense_len > ce->sense_length)
+		csio->sense_resid = csio->sense_len - ce->sense_length;
+	else
+		csio->sense_resid = 0;
 	csio->resid = ce->residual_count;
 	csio->ccb_h.status |= CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
 #ifdef CISS_DEBUG
 	{
 	    struct scsi_sense_data	*sns = (struct scsi_sense_data *)&ce->sense_info[0];
-	    debug(0, "sense key %x", sns->flags & SSD_KEY);
+	    debug(0, "sense key %x", scsi_get_sense_key(sns, csio->sense_len -
+		  csio->sense_resid, /*show_errors*/ 1));
 	}
 #endif
 	break;
diff --git a/sys/dev/coretemp/coretemp.c b/sys/dev/coretemp/coretemp.c
index cb1cbff3100e..54e5a4365e87 100644
--- a/sys/dev/coretemp/coretemp.c
+++ b/sys/dev/coretemp/coretemp.c
@@ -384,6 +384,7 @@ coretemp_get_val_sysctl(SYSCTL_HANDLER_ARGS)
 	}
 
 	if (msr & THERM_STATUS_LOG) {
+		coretemp_clear_thermal_msr(device_get_unit(dev));
 		sc->sc_throttle_log = 1;
 
 		/*
@@ -424,8 +425,10 @@ coretemp_throttle_log_sysctl(SYSCTL_HANDLER_ARGS)
 	msr = coretemp_get_thermal_msr(device_get_unit(dev));
 	sc = device_get_softc(dev);
 
-	if (msr & THERM_STATUS_LOG)
+	if (msr & THERM_STATUS_LOG) {
+		coretemp_clear_thermal_msr(device_get_unit(dev));
 		sc->sc_throttle_log = 1;
+	}
 
 	val = sc->sc_throttle_log;
 
diff --git a/sys/dev/dc/if_dc.c b/sys/dev/dc/if_dc.c
index b9567de09263..184fa8ffa096 100644
--- a/sys/dev/dc/if_dc.c
+++ b/sys/dev/dc/if_dc.c
@@ -224,6 +224,10 @@ static const struct dc_type dc_devs[] = {
 		"Linksys PCMPC200 CardBus 10/100" },
 	{ DC_DEVID(DC_VENDORID_LINKSYS, DC_DEVICEID_PCMPC200_AB09), 0,
 		"Linksys PCMPC200 CardBus 10/100" },
+	{ DC_DEVID(DC_VENDORID_ULI, DC_DEVICEID_M5261), 0,
+		"ULi M5261 FastEthernet" },
+	{ DC_DEVID(DC_VENDORID_ULI, DC_DEVICEID_M5263), 0,
+		"ULi M5263 FastEthernet" },
 	{ 0, 0, NULL }
 };
 
@@ -252,6 +256,7 @@ static void dc_stop(struct dc_softc *);
 static void dc_watchdog(void *);
 static int dc_shutdown(device_t);
 static int dc_ifmedia_upd(struct ifnet *);
+static int dc_ifmedia_upd_locked(struct dc_softc *);
 static void dc_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 
 static int dc_dma_alloc(struct dc_softc *);
@@ -285,6 +290,7 @@ static uint32_t dc_mchash_be(const uint8_t *);
 static void dc_setfilt_21143(struct dc_softc *);
 static void dc_setfilt_asix(struct dc_softc *);
 static void dc_setfilt_admtek(struct dc_softc *);
+static void dc_setfilt_uli(struct dc_softc *);
 static void dc_setfilt_xircom(struct dc_softc *);
 
 static void dc_setfilt(struct dc_softc *);
@@ -827,6 +833,23 @@ dc_miibus_readreg(device_t dev, int phy, int reg)
 		return (0);
 	}
 
+	if (sc->dc_type == DC_TYPE_ULI_M5263) {
+		CSR_WRITE_4(sc, DC_ROM,
+		    ((phy << DC_ULI_PHY_ADDR_SHIFT) & DC_ULI_PHY_ADDR_MASK) |
+		    ((reg << DC_ULI_PHY_REG_SHIFT) & DC_ULI_PHY_REG_MASK) |
+		    DC_ULI_PHY_OP_READ);
+		for (i = 0; i < DC_TIMEOUT; i++) {
+			DELAY(1);
+			rval = CSR_READ_4(sc, DC_ROM);
+			if ((rval & DC_ULI_PHY_OP_DONE) != 0) {
+				return (rval & DC_ULI_PHY_DATA_MASK);
+			}
+		}
+		if (i == DC_TIMEOUT)
+			device_printf(dev, "phy read timed out\n");
+		return (0);
+	}
+
 	if (DC_IS_COMET(sc)) {
 		switch (reg) {
 		case MII_BMCR:
@@ -897,6 +920,16 @@ dc_miibus_writereg(device_t dev, int phy, int reg, int data)
 		return (0);
 	}
 
+	if (sc->dc_type == DC_TYPE_ULI_M5263) {
+		CSR_WRITE_4(sc, DC_ROM,
+		    ((phy << DC_ULI_PHY_ADDR_SHIFT) & DC_ULI_PHY_ADDR_MASK) |
+		    ((reg << DC_ULI_PHY_REG_SHIFT) & DC_ULI_PHY_REG_MASK) |
+		    ((data << DC_ULI_PHY_DATA_SHIFT) & DC_ULI_PHY_DATA_MASK) |
+		    DC_ULI_PHY_OP_WRITE);
+		DELAY(1);
+		return (0);
+	}
+
 	if (DC_IS_COMET(sc)) {
 		switch (reg) {
 		case MII_BMCR:
@@ -1136,6 +1169,8 @@ dc_setfilt_21143(struct dc_softc *sc)
 	sp[41] = DC_SP_MAC(eaddr[2]);
 
 	sframe->dc_status = htole32(DC_TXSTAT_OWN);
+	bus_dmamap_sync(sc->dc_tx_ltag, sc->dc_tx_lmap, BUS_DMASYNC_PREREAD |
+	    BUS_DMASYNC_PREWRITE);
 	bus_dmamap_sync(sc->dc_stag, sc->dc_smap, BUS_DMASYNC_PREWRITE);
 	CSR_WRITE_4(sc, DC_TXSTART, 0xFFFFFFFF);
 
@@ -1281,6 +1316,97 @@ dc_setfilt_asix(struct dc_softc *sc)
 	CSR_WRITE_4(sc, DC_AX_FILTDATA, hashes[1]);
 }
 
+static void
+dc_setfilt_uli(struct dc_softc *sc)
+{
+	uint8_t eaddr[ETHER_ADDR_LEN];
+	struct ifnet *ifp;
+	struct ifmultiaddr *ifma;
+	struct dc_desc *sframe;
+	uint32_t filter, *sp;
+	uint8_t *ma;
+	int i, mcnt;
+
+	ifp = sc->dc_ifp;
+
+	i = sc->dc_cdata.dc_tx_prod;
+	DC_INC(sc->dc_cdata.dc_tx_prod, DC_TX_LIST_CNT);
+	sc->dc_cdata.dc_tx_cnt++;
+	sframe = &sc->dc_ldata.dc_tx_list[i];
+	sp = sc->dc_cdata.dc_sbuf;
+	bzero(sp, DC_SFRAME_LEN);
+
+	sframe->dc_data = htole32(DC_ADDR_LO(sc->dc_saddr));
+	sframe->dc_ctl = htole32(DC_SFRAME_LEN | DC_TXCTL_SETUP |
+	    DC_TXCTL_TLINK | DC_FILTER_PERFECT | DC_TXCTL_FINT);
+
+	sc->dc_cdata.dc_tx_chain[i] = (struct mbuf *)sc->dc_cdata.dc_sbuf;
+
+	/* Set station address. */
+	bcopy(IF_LLADDR(sc->dc_ifp), eaddr, ETHER_ADDR_LEN);
+	*sp++ = DC_SP_MAC(eaddr[1] << 8 | eaddr[0]);
+	*sp++ = DC_SP_MAC(eaddr[3] << 8 | eaddr[2]);
+	*sp++ = DC_SP_MAC(eaddr[5] << 8 | eaddr[4]);
+
+	/* Set broadcast address. */
+	*sp++ = DC_SP_MAC(0xFFFF);
+	*sp++ = DC_SP_MAC(0xFFFF);
+	*sp++ = DC_SP_MAC(0xFFFF);
+
+	/* Extract current filter configuration. */
+	filter = CSR_READ_4(sc, DC_NETCFG);
+	filter &= ~(DC_NETCFG_RX_PROMISC | DC_NETCFG_RX_ALLMULTI);
+
+	/* Now build perfect filters. */
+	mcnt = 0;
+	if_maddr_rlock(ifp);
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+		if (mcnt >= DC_ULI_FILTER_NPERF) {
+			filter |= DC_NETCFG_RX_ALLMULTI;
+			break;
+		}
+		ma = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
+		*sp++ = DC_SP_MAC(ma[1] << 8 | ma[0]);
+		*sp++ = DC_SP_MAC(ma[3] << 8 | ma[2]);
+		*sp++ = DC_SP_MAC(ma[5] << 8 | ma[4]);
+		mcnt++;
+	}
+	if_maddr_runlock(ifp);
+
+	for (; mcnt < DC_ULI_FILTER_NPERF; mcnt++) {
+		*sp++ = DC_SP_MAC(0xFFFF);
+		*sp++ = DC_SP_MAC(0xFFFF);
+		*sp++ = DC_SP_MAC(0xFFFF);
+	}
+
+	if (filter & (DC_NETCFG_TX_ON | DC_NETCFG_RX_ON))
+		CSR_WRITE_4(sc, DC_NETCFG,
+		    filter & ~(DC_NETCFG_TX_ON | DC_NETCFG_RX_ON));
+	if (ifp->if_flags & IFF_PROMISC)
+		filter |= DC_NETCFG_RX_PROMISC | DC_NETCFG_RX_ALLMULTI;
+	if (ifp->if_flags & IFF_ALLMULTI)
+		filter |= DC_NETCFG_RX_ALLMULTI;
+	CSR_WRITE_4(sc, DC_NETCFG,
+	    filter & ~(DC_NETCFG_TX_ON | DC_NETCFG_RX_ON));
+	if (filter & (DC_NETCFG_TX_ON | DC_NETCFG_RX_ON))
+		CSR_WRITE_4(sc, DC_NETCFG, filter);
+
+	sframe->dc_status = htole32(DC_TXSTAT_OWN);
+	bus_dmamap_sync(sc->dc_tx_ltag, sc->dc_tx_lmap, BUS_DMASYNC_PREREAD |
+	    BUS_DMASYNC_PREWRITE);
+	bus_dmamap_sync(sc->dc_stag, sc->dc_smap, BUS_DMASYNC_PREWRITE);
+	CSR_WRITE_4(sc, DC_TXSTART, 0xFFFFFFFF);
+
+	/*
+	 * Wait some time...
+	 */
+	DELAY(1000);
+
+	sc->dc_wdog_timer = 5;
+}
+
 static void
 dc_setfilt_xircom(struct dc_softc *sc)
 {
@@ -1342,6 +1468,8 @@ dc_setfilt_xircom(struct dc_softc *sc)
 	DC_SETBIT(sc, DC_NETCFG, DC_NETCFG_TX_ON);
 	DC_SETBIT(sc, DC_NETCFG, DC_NETCFG_RX_ON);
 	sframe->dc_status = htole32(DC_TXSTAT_OWN);
+	bus_dmamap_sync(sc->dc_tx_ltag, sc->dc_tx_lmap, BUS_DMASYNC_PREREAD |
+	    BUS_DMASYNC_PREWRITE);
 	bus_dmamap_sync(sc->dc_stag, sc->dc_smap, BUS_DMASYNC_PREWRITE);
 	CSR_WRITE_4(sc, DC_TXSTART, 0xFFFFFFFF);
 
@@ -1367,6 +1495,9 @@ dc_setfilt(struct dc_softc *sc)
 	if (DC_IS_ADMTEK(sc))
 		dc_setfilt_admtek(sc);
 
+	if (DC_IS_ULI(sc))
+		dc_setfilt_uli(sc);
+
 	if (DC_IS_XIRCOM(sc))
 		dc_setfilt_xircom(sc);
 }
@@ -1535,7 +1666,7 @@ dc_reset(struct dc_softc *sc)
 	}
 
 	if (DC_IS_ASIX(sc) || DC_IS_ADMTEK(sc) || DC_IS_CONEXANT(sc) ||
-	    DC_IS_XIRCOM(sc) || DC_IS_INTEL(sc)) {
+	    DC_IS_XIRCOM(sc) || DC_IS_INTEL(sc) || DC_IS_ULI(sc)) {
 		DELAY(10000);
 		DC_CLRBIT(sc, DC_BUSCTL, DC_BUSCTL_RESET);
 		i = 0;
@@ -1747,7 +1878,7 @@ dc_read_srom(struct dc_softc *sc, int bits)
 	int size;
 
 	size = DC_ROM_SIZE(bits);
-	sc->dc_srom = malloc(size, M_DEVBUF, M_NOWAIT);
+	sc->dc_srom = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (sc->dc_srom == NULL) {
 		device_printf(sc->dc_dev, "Could not allocate SROM buffer\n");
 		return (ENOMEM);
@@ -2041,7 +2172,8 @@ dc_attach(device_t dev)
 	struct ifnet *ifp;
 	struct dc_mediainfo *m;
 	uint32_t reg, revision;
-	int error, mac_offset, phy, rid, tmp;
+	uint16_t *srom;
+	int error, mac_offset, n, phy, rid, tmp;
 	uint8_t *mac;
 
 	sc = device_get_softc(dev);
@@ -2221,6 +2353,21 @@ dc_attach(device_t dev)
 		if (error != 0)
 			goto fail;
 		break;
+	case DC_DEVID(DC_VENDORID_ULI, DC_DEVICEID_M5261):
+	case DC_DEVID(DC_VENDORID_ULI, DC_DEVICEID_M5263):
+		if (sc->dc_info->dc_devid ==
+		    DC_DEVID(DC_VENDORID_ULI, DC_DEVICEID_M5261))
+			sc->dc_type = DC_TYPE_ULI_M5261;
+		else
+			sc->dc_type = DC_TYPE_ULI_M5263;
+		/* TX buffers should be aligned on 4 byte boundary. */
+		sc->dc_flags |= DC_TX_INTR_ALWAYS | DC_TX_COALESCE |
+		    DC_TX_ALIGN;
+		sc->dc_pmode = DC_PMODE_MII;
+		error = dc_read_srom(sc, sc->dc_romwidth);
+		if (error != 0)
+			goto fail;
+		break;
 	default:
 		device_printf(dev, "unknown device: %x\n",
 		    sc->dc_info->dc_devid);
@@ -2318,6 +2465,33 @@ dc_attach(device_t dev)
 		}
 		bcopy(mac, eaddr, ETHER_ADDR_LEN);
 		break;
+	case DC_TYPE_ULI_M5261:
+	case DC_TYPE_ULI_M5263:
+		srom = (uint16_t *)sc->dc_srom;
+		if (srom == NULL || *srom == 0xFFFF || *srom == 0) {
+			/*
+			 * No valid SROM present, read station address
+			 * from ID Table.
+			 */
+			device_printf(dev,
+			    "Reading station address from ID Table.\n");
+			CSR_WRITE_4(sc, DC_BUSCTL, 0x10000);
+			CSR_WRITE_4(sc, DC_SIARESET, 0x01C0);
+			CSR_WRITE_4(sc, DC_10BTCTRL, 0x0000);
+			CSR_WRITE_4(sc, DC_10BTCTRL, 0x0010);
+			CSR_WRITE_4(sc, DC_10BTCTRL, 0x0000);
+			CSR_WRITE_4(sc, DC_SIARESET, 0x0000);
+			CSR_WRITE_4(sc, DC_SIARESET, 0x01B0);
+			mac = (uint8_t *)eaddr;
+			for (n = 0; n < ETHER_ADDR_LEN; n++)
+				mac[n] = (uint8_t)CSR_READ_4(sc, DC_10BTCTRL);
+			CSR_WRITE_4(sc, DC_SIARESET, 0x0000);
+			CSR_WRITE_4(sc, DC_BUSCTL, 0x0000);
+			DELAY(10);
+		} else
+			dc_read_eeprom(sc, (caddr_t)&eaddr, DC_EE_NODEADDR, 3,
+			    0);
+		break;
 	default:
 		dc_read_eeprom(sc, (caddr_t)&eaddr, DC_EE_NODEADDR, 3, 0);
 		break;
@@ -2970,7 +3144,7 @@ dc_txeof(struct dc_softc *sc)
 	 * Go through our tx list and free mbufs for those
 	 * frames that have been transmitted.
 	 */
-	bus_dmamap_sync(sc->dc_rx_ltag, sc->dc_tx_lmap, BUS_DMASYNC_POSTREAD |
+	bus_dmamap_sync(sc->dc_tx_ltag, sc->dc_tx_lmap, BUS_DMASYNC_POSTREAD |
 	    BUS_DMASYNC_POSTWRITE);
 	setup = 0;
 	for (idx = sc->dc_cdata.dc_tx_cons; idx != sc->dc_cdata.dc_tx_prod;
@@ -3594,7 +3768,7 @@ dc_init_locked(struct dc_softc *sc)
 	/*
 	 * Set cache alignment and burst length.
 	 */
-	if (DC_IS_ASIX(sc) || DC_IS_DAVICOM(sc))
+	if (DC_IS_ASIX(sc) || DC_IS_DAVICOM(sc) || DC_IS_ULI(sc))
 		CSR_WRITE_4(sc, DC_BUSCTL, 0);
 	else
 		CSR_WRITE_4(sc, DC_BUSCTL, DC_BUSCTL_MRME | DC_BUSCTL_MRLE);
@@ -3707,6 +3881,11 @@ dc_init_locked(struct dc_softc *sc)
 	CSR_WRITE_4(sc, DC_IMR, DC_INTRS);
 	CSR_WRITE_4(sc, DC_ISR, 0xFFFFFFFF);
 
+	/* Initialize TX jabber and RX watchdog timer. */
+	if (DC_IS_ULI(sc))
+		CSR_WRITE_4(sc, DC_WATCHDOG, DC_WDOG_JABBERCLK |
+		    DC_WDOG_HOSTUNJAB);
+
 	/* Enable transmitter. */
 	DC_SETBIT(sc, DC_NETCFG, DC_NETCFG_TX_ON);
 
@@ -3736,8 +3915,7 @@ dc_init_locked(struct dc_softc *sc)
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
-	mii_mediachg(mii);
-	dc_setcfg(sc, sc->dc_if_media);
+	dc_ifmedia_upd_locked(sc);
 
 	/* Clear missed frames and overflow counter. */
 	CSR_READ_4(sc, DC_FRAMESDISCARDED);
@@ -3763,25 +3941,37 @@ static int
 dc_ifmedia_upd(struct ifnet *ifp)
 {
 	struct dc_softc *sc;
-	struct mii_data *mii;
-	struct ifmedia *ifm;
+	int error;
 
 	sc = ifp->if_softc;
-	mii = device_get_softc(sc->dc_miibus);
 	DC_LOCK(sc);
-	mii_mediachg(mii);
-	ifm = &mii->mii_media;
-
-	if (DC_IS_INTEL(sc))
-		dc_setcfg(sc, ifm->ifm_media);
-	else if (DC_IS_DAVICOM(sc) &&
-	    IFM_SUBTYPE(ifm->ifm_media) == IFM_HPNA_1)
-		dc_setcfg(sc, ifm->ifm_media);
-	else
-		sc->dc_link = 0;
+	error = dc_ifmedia_upd_locked(sc);
 	DC_UNLOCK(sc);
+	return (error);
+}
 
-	return (0);
+static int
+dc_ifmedia_upd_locked(struct dc_softc *sc)
+{
+	struct mii_data *mii;
+	struct ifmedia *ifm;
+	int error;
+
+	DC_LOCK_ASSERT(sc);
+
+	sc->dc_link = 0;
+	mii = device_get_softc(sc->dc_miibus);
+	error = mii_mediachg(mii);
+	if (error == 0) {
+		ifm = &mii->mii_media;
+		if (DC_IS_INTEL(sc))
+			dc_setcfg(sc, ifm->ifm_media);
+		else if (DC_IS_DAVICOM(sc) &&
+		    IFM_SUBTYPE(ifm->ifm_media) == IFM_HPNA_1)
+			dc_setcfg(sc, ifm->ifm_media);
+	}
+
+	return (error);
 }
 
 /*
diff --git a/sys/dev/dc/if_dcreg.h b/sys/dev/dc/if_dcreg.h
index cc7df2934822..acd80c026d8f 100644
--- a/sys/dev/dc/if_dcreg.h
+++ b/sys/dev/dc/if_dcreg.h
@@ -78,6 +78,8 @@
 #define	DC_TYPE_PNIC		0xA	/* 82c168/82c169 PNIC I */
 #define	DC_TYPE_XIRCOM		0xB	/* Xircom X3201 */
 #define	DC_TYPE_CONEXANT	0xC	/* Conexant LANfinity RS7112 */
+#define	DC_TYPE_ULI_M5261	0xD	/* ALi/ULi M5261 */
+#define	DC_TYPE_ULI_M5263	0xE	/* ALi/ULi M5263 */
 
 #define	DC_IS_MACRONIX(x)			\
 	(x->dc_type == DC_TYPE_98713 ||		\
@@ -88,6 +90,10 @@
 	(x->dc_type == DC_TYPE_AL981 ||		\
 	 x->dc_type == DC_TYPE_AN983)
 
+#define	DC_IS_ULI(x)				\
+	(x->dc_type == DC_TYPE_ULI_M5261 ||	\
+	 x->dc_type == DC_TYPE_ULI_M5263)
+
 #define	DC_IS_INTEL(x)		(x->dc_type == DC_TYPE_21143)
 #define	DC_IS_ASIX(x)		(x->dc_type == DC_TYPE_ASIX)
 #define	DC_IS_COMET(x)		(x->dc_type == DC_TYPE_AL981)
@@ -725,6 +731,23 @@ struct dc_mii_frame {
 
 /* End of CONEXANT specific registers */
 
+/*
+ * ULi M5263 specific registers.
+ */
+#define	DC_ULI_FILTER_NPERF	14
+
+#define	DC_ULI_PHY_DATA_MASK	0x0000FFFF
+#define	DC_ULI_PHY_REG_MASK	0x001F0000
+#define	DC_ULI_PHY_ADDR_MASK	0x03E00000
+#define	DC_ULI_PHY_OP_WRITE	0x04000000
+#define	DC_ULI_PHY_OP_READ	0x08000000
+#define	DC_ULI_PHY_OP_DONE	0x10000000
+
+#define	DC_ULI_PHY_DATA_SHIFT	0
+#define	DC_ULI_PHY_REG_SHIFT	16
+#define	DC_ULI_PHY_ADDR_SHIFT	21
+
+/* End of ULi M5263 specific registers */
 
 struct dc_softc {
 	struct ifnet		*dc_ifp;	/* interface info */
@@ -1032,6 +1055,17 @@ struct dc_softc {
 #define	DC_DEVICEID_PCMPC200_AB08	0xab08
 #define	DC_DEVICEID_PCMPC200_AB09	0xab09
 
+/*
+ * ULi vendor ID.
+ */
+#define	DC_VENDORID_ULI		0x10b9
+
+/*
+ * ULi device IDs.
+ */
+#define	DC_DEVICEID_M5261	0x5261
+#define	DC_DEVICEID_M5263	0x5263
+
 #define	DC_DEVID(vendor, device)	((device) << 16 | (vendor))
 
 /*
diff --git a/sys/dev/e1000/e1000_82575.c b/sys/dev/e1000/e1000_82575.c
index 7257d15c9e53..8407f7cd3b0b 100644
--- a/sys/dev/e1000/e1000_82575.c
+++ b/sys/dev/e1000/e1000_82575.c
@@ -1983,7 +1983,7 @@ static s32 e1000_write_phy_reg_82580(struct e1000_hw *hw, u32 offset, u16 data)
  *  e1000_reset_mdicnfg_82580 - Reset MDICNFG destination and com_mdio bits
  *  @hw: pointer to the HW structure
  *
- *  This resets the the MDICNFG.Destination and MDICNFG.Com_MDIO bits based on
+ *  This resets the MDICNFG.Destination and MDICNFG.Com_MDIO bits based on
  *  the values found in the EEPROM.  This addresses an issue in which these
  *  bits are not restored from EEPROM after reset.
  **/
diff --git a/sys/dev/e1000/if_lem.c b/sys/dev/e1000/if_lem.c
index d4bd1b6a3a02..a9202f0076b9 100644
--- a/sys/dev/e1000/if_lem.c
+++ b/sys/dev/e1000/if_lem.c
@@ -2654,6 +2654,7 @@ lem_setup_transmit_structures(struct adapter *adapter)
 	}
 
 	/* Reset state */
+	adapter->last_hw_offload = 0;
 	adapter->next_avail_tx_desc = 0;
 	adapter->next_tx_to_clean = 0;
 	adapter->num_tx_desc_avail = adapter->num_tx_desc;
diff --git a/sys/dev/esp/esp_sbus.c b/sys/dev/esp/esp_sbus.c
index 6f80006ff68a..62a4592156a6 100644
--- a/sys/dev/esp/esp_sbus.c
+++ b/sys/dev/esp/esp_sbus.c
@@ -26,7 +26,7 @@
  *
  */
 
-/*	$NetBSD: esp_sbus.c,v 1.31 2005/02/27 00:27:48 perry Exp $	*/
+/*	$NetBSD: esp_sbus.c,v 1.51 2009/09/17 16:28:12 tsutsui Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc.
@@ -44,13 +44,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the NetBSD
- *	Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
@@ -158,12 +151,12 @@ MODULE_DEPEND(esp, sbus, 1, 1, 1);
 /*
  * Functions and the switch for the MI code
  */
-static u_char	esp_read_reg(struct ncr53c9x_softc *sc, int reg);
-static void	esp_write_reg(struct ncr53c9x_softc *sc, int reg, u_char v);
+static uint8_t	esp_read_reg(struct ncr53c9x_softc *sc, int reg);
+static void	esp_write_reg(struct ncr53c9x_softc *sc, int reg, uint8_t v);
 static int	esp_dma_isintr(struct ncr53c9x_softc *sc);
 static void	esp_dma_reset(struct ncr53c9x_softc *sc);
 static int	esp_dma_intr(struct ncr53c9x_softc *sc);
-static int	esp_dma_setup(struct ncr53c9x_softc *sc, caddr_t *addr,
+static int	esp_dma_setup(struct ncr53c9x_softc *sc, void **addr,
 		    size_t *len, int datain, size_t *dmasize);
 static void	esp_dma_go(struct ncr53c9x_softc *sc);
 static void	esp_dma_stop(struct ncr53c9x_softc *sc);
@@ -172,7 +165,7 @@ static int	espattach(struct esp_softc *esc,
 		    const struct ncr53c9x_glue *gluep);
 static int	espdetach(struct esp_softc *esc);
 
-static const struct ncr53c9x_glue esp_sbus_glue = {
+static const struct ncr53c9x_glue const esp_sbus_glue = {
 	esp_read_reg,
 	esp_write_reg,
 	esp_dma_isintr,
@@ -718,9 +711,9 @@ espdetach(struct esp_softc *esc)
 static int esp_sbus_debug = 0;
 
 static const struct {
-	char *r_name;
-	int   r_flag;
-} esp__read_regnames [] = {
+	const char *r_name;
+	int r_flag;
+} const esp__read_regnames [] = {
 	{ "TCL", 0},			/* 0/00 */
 	{ "TCM", 0},			/* 1/04 */
 	{ "FIFO", 0},			/* 2/08 */
@@ -739,10 +732,10 @@ static const struct {
 	{ "TCX", 1},			/* f/3c */
 };
 
-static const struct {
-	char *r_name;
-	int   r_flag;
-} esp__write_regnames[] = {
+static const const struct {
+	const char *r_name;
+	int r_flag;
+} const esp__write_regnames[] = {
 	{ "TCL", 1},			/* 0/00 */
 	{ "TCM", 1},			/* 1/04 */
 	{ "FIFO", 0},			/* 2/08 */
@@ -762,11 +755,11 @@ static const struct {
 };
 #endif
 
-static u_char
+static uint8_t
 esp_read_reg(struct ncr53c9x_softc *sc, int reg)
 {
 	struct esp_softc *esc = (struct esp_softc *)sc;
-	u_char v;
+	uint8_t v;
 
 	v = bus_read_1(esc->sc_res, reg * 4);
 
@@ -780,7 +773,7 @@ esp_read_reg(struct ncr53c9x_softc *sc, int reg)
 }
 
 static void
-esp_write_reg(struct ncr53c9x_softc *sc, int reg, u_char v)
+esp_write_reg(struct ncr53c9x_softc *sc, int reg, uint8_t v)
 {
 	struct esp_softc *esc = (struct esp_softc *)sc;
 
@@ -818,8 +811,8 @@ esp_dma_intr(struct ncr53c9x_softc *sc)
 }
 
 static int
-esp_dma_setup(struct ncr53c9x_softc *sc, caddr_t *addr, size_t *len,
-	      int datain, size_t *dmasize)
+esp_dma_setup(struct ncr53c9x_softc *sc, void **addr, size_t *len,
+    int datain, size_t *dmasize)
 {
 	struct esp_softc *esc = (struct esp_softc *)sc;
 
diff --git a/sys/dev/esp/ncr53c9x.c b/sys/dev/esp/ncr53c9x.c
index 9373420da8ba..1d4631868784 100644
--- a/sys/dev/esp/ncr53c9x.c
+++ b/sys/dev/esp/ncr53c9x.c
@@ -26,7 +26,7 @@
  *
  */
 
-/*	$NetBSD: ncr53c9x.c,v 1.125 2007/01/09 12:53:12 itohy Exp $	*/
+/*	$NetBSD: ncr53c9x.c,v 1.143 2011/07/31 18:39:00 jakllsch Exp $	*/
 
 /*-
  * Copyright (c) 1998, 2002 The NetBSD Foundation, Inc.
@@ -43,13 +43,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *        This product includes software developed by the NetBSD
- *        Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
@@ -133,7 +126,7 @@ __FBSDID("$FreeBSD$");
 MODULE_DEPEND(esp, cam, 1, 1, 1);
 
 #ifdef NCR53C9X_DEBUG
-static int ncr53c9x_debug =
+int ncr53c9x_debug =
     NCR_SHOWMISC /* | NCR_SHOWPHASE | NCR_SHOWTRAC | NCR_SHOWCMDS */;
 #endif
 
@@ -167,7 +160,7 @@ static void	ncr53c9x_sched(struct ncr53c9x_softc *sc);
 static void	ncr53c9x_select(struct ncr53c9x_softc *sc,
 		    struct ncr53c9x_ecb *ecb);
 static void	ncr53c9x_watch(void *arg);
-static void	ncr53c9x_wrfifo(struct ncr53c9x_softc *sc, u_char *p,
+static void	ncr53c9x_wrfifo(struct ncr53c9x_softc *sc, uint8_t *p,
 		    int len);
 
 static struct ncr53c9x_ecb	*ncr53c9x_get_ecb(struct ncr53c9x_softc *sc);
@@ -187,13 +180,11 @@ static inline int	ncr53c9x_stp2cpb(struct ncr53c9x_softc *sc,
 		NCR_WRITE_REG((sc), NCR_TCL, (size));			\
 		NCR_WRITE_REG((sc), NCR_TCM, (size) >> 8);		\
 		if ((sc->sc_cfg2 & NCRCFG2_FE) ||			\
-		    (sc->sc_rev == NCR_VARIANT_FAS366)) {		\
+		    (sc->sc_rev == NCR_VARIANT_FAS366))			\
 			NCR_WRITE_REG((sc), NCR_TCH, (size) >> 16);	\
-		}							\
-		if (sc->sc_rev == NCR_VARIANT_FAS366) {			\
+		if (sc->sc_rev == NCR_VARIANT_FAS366)			\
 			NCR_WRITE_REG(sc, NCR_RCH, 0);			\
-		}							\
-} while (0)
+} while (/* CONSTCOND */0)
 
 #ifndef mstohz
 #define	mstohz(ms) \
@@ -551,11 +542,10 @@ ncr53c9x_reset(struct ncr53c9x_softc *sc)
 		NCR_WRITE_REG(sc, NCR_AMDCFG4, sc->sc_cfg4);
 
 #if 0
-	device_printf(sc->sc_dev, "ncr53c9x_reset: revision %d\n",
-	    sc->sc_rev);
-	device_printf(sc->sc_dev, "ncr53c9x_reset: cfg1 0x%x, cfg2 0x%x, "
-	    "cfg3 0x%x, ccf 0x%x, timeout 0x%x\n",
-	    sc->sc_cfg1, sc->sc_cfg2, sc->sc_cfg3, sc->sc_ccf, sc->sc_timeout);
+	device_printf(sc->sc_dev, "%s: revision %d\n", __func__, sc->sc_rev);
+	device_printf(sc->sc_dev, "%s: cfg1 0x%x, cfg2 0x%x, cfg3 0x%x, ccf "
+	    "0x%x, timeout 0x%x\n", __func__, sc->sc_cfg1, sc->sc_cfg2,
+	    sc->sc_cfg3, sc->sc_ccf, sc->sc_timeout);
 #endif
 }
 
@@ -573,7 +563,8 @@ ncr53c9x_clear(struct ncr53c9x_softc *sc, cam_status result)
 	/* Cancel any active commands. */
 	sc->sc_state = NCR_CLEANING;
 	sc->sc_msgify = 0;
-	if ((ecb = sc->sc_nexus) != NULL) {
+	ecb = sc->sc_nexus;
+	if (ecb != NULL) {
 		ecb->ccb->ccb_h.status = result;
 		ncr53c9x_done(sc, ecb);
 	}
@@ -597,7 +588,8 @@ ncr53c9x_clear_target(struct ncr53c9x_softc *sc, int target,
 
 	/* Cancel outstanding disconnected commands on each LUN. */
 	LIST_FOREACH(li, &sc->sc_tinfo[target].luns, link) {
-		if ((ecb = li->untagged) != NULL) {
+		ecb = li->untagged;
+		if (ecb != NULL) {
 			li->untagged = NULL;
 			/*
 			 * XXX should we terminate a command
@@ -607,12 +599,14 @@ ncr53c9x_clear_target(struct ncr53c9x_softc *sc, int target,
 			ecb->ccb->ccb_h.status = result;
 			ncr53c9x_done(sc, ecb);
 		}
-		for (i = 0; i < NCR_TAG_DEPTH; i++)
-			if ((ecb = li->queued[i])) {
+		for (i = 0; i < NCR_TAG_DEPTH; i++) {
+			ecb = li->queued[i];
+			if (ecb != NULL) {
 				li->queued[i] = NULL;
 				ecb->ccb->ccb_h.status = result;
 				ncr53c9x_done(sc, ecb);
 			}
+		}
 		li->used = 0;
 	}
 }
@@ -635,7 +629,7 @@ ncr53c9x_init(struct ncr53c9x_softc *sc, int doreset)
 
 		TAILQ_INIT(&sc->ready_list);
 		sc->sc_nexus = NULL;
-		memset(sc->sc_tinfo, 0, sizeof(sc->sc_tinfo));
+		memset(sc->sc_tinfo, 0, sizeof(*sc->sc_tinfo));
 		for (r = 0; r < sc->sc_ntarg; r++) {
 			LIST_INIT(&sc->sc_tinfo[r].luns);
 		}
@@ -751,7 +745,7 @@ ncr53c9x_stp2cpb(struct ncr53c9x_softc *sc, int period)
 static inline void
 ncr53c9x_setsync(struct ncr53c9x_softc *sc, struct ncr53c9x_tinfo *ti)
 {
-	u_char cfg3, syncoff, synctp;
+	uint8_t cfg3, syncoff, synctp;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
@@ -810,7 +804,7 @@ static void
 ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 {
 	struct ncr53c9x_tinfo *ti;
-	u_char *cmd;
+	uint8_t *cmd;
 	size_t dmasize;
 	int clen, selatn3, selatns;
 	int lun = ecb->ccb->ccb_h.target_lun;
@@ -818,8 +812,8 @@ ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_select(t%d,l%d,cmd:%x,tag:%x,%x)] ",
-	    target, lun, ecb->cmd.cmd.opcode, ecb->tag[0], ecb->tag[1]));
+	NCR_TRACE(("[%s(t%d,l%d,cmd:%x,tag:%x,%x)] ", __func__, target, lun,
+	    ecb->cmd.cmd.opcode, ecb->tag[0], ecb->tag[1]));
 
 	ti = &sc->sc_tinfo[target];
 	sc->sc_state = NCR_SELECTING;
@@ -838,9 +832,8 @@ ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 		NCRCMD(sc, NCRCMD_FLUSH);
 		NCR_WRITE_REG(sc, NCR_SELID, target | NCR_BUSID_HMEXC32 |
 		    NCR_BUSID_HMEENCID);
-	} else {
+	} else
 		NCR_WRITE_REG(sc, NCR_SELID, target);
-	}
 
 	/*
 	 * If we are requesting sense, force a renegotiation if we are
@@ -873,7 +866,7 @@ ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 		selatns = 1;
 	}
 
-	cmd = (u_char *)&ecb->cmd.cmd;
+	cmd = (uint8_t *)&ecb->cmd.cmd;
 
 	if (selatn3) {
 		/* We'll use tags with SELATN3. */
@@ -954,7 +947,7 @@ ncr53c9x_get_ecb(struct ncr53c9x_softc *sc)
 	ecb = TAILQ_FIRST(&sc->free_list);
 	if (ecb) {
 		if (ecb->flags != 0)
-			panic("ecb flags not cleared\n");
+			panic("%s: ecb flags not cleared", __func__);
 		TAILQ_REMOVE(&sc->free_list, ecb, free_links);
 		ecb->flags = ECB_ALLOC;
 		bzero(&ecb->ccb, sizeof(struct ncr53c9x_ecb) -
@@ -990,7 +983,7 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_action %d]", ccb->ccb_h.func_code));
+	NCR_TRACE(("[%s %d]", __func__, ccb->ccb_h.func_code));
 
 	switch (ccb->ccb_h.func_code) {
 	case XPT_RESET_BUS:
@@ -1196,7 +1189,7 @@ ncr53c9x_poll(struct cam_sim *sim)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_poll] "));
+	NCR_TRACE(("[%s] ", __func__));
 
 	if (NCRDMA_ISINTR(sc))
 		ncr53c9x_intr1(sc);
@@ -1260,10 +1253,10 @@ ncr53c9x_sched(struct ncr53c9x_softc *sc)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_sched] "));
+	NCR_TRACE(("[%s] ", __func__));
 
 	if (sc->sc_state != NCR_IDLE)
-		panic("ncr53c9x_sched: not IDLE (state=%d)", sc->sc_state);
+		panic("%s: not IDLE (state=%d)", __func__, sc->sc_state);
 
 	/*
 	 * Find first ecb in ready queue that is for a target/lunit
@@ -1288,10 +1281,9 @@ ncr53c9x_sched(struct ncr53c9x_softc *sc)
 		li = TINFO_LUN(ti, lun);
 		if (li == NULL) {
 			/* Initialize LUN info and add to list. */
-			if ((li = malloc(sizeof(*li),
-			    M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
+			li = malloc(sizeof(*li), M_DEVBUF, M_NOWAIT | M_ZERO);
+			if (li == NULL)
 				continue;
-			}
 			li->lun = lun;
 
 			LIST_INSERT_HEAD(&ti->luns, li, link);
@@ -1338,7 +1330,7 @@ ncr53c9x_sched(struct ncr53c9x_softc *sc)
 			ncr53c9x_select(sc, ecb);
 			break;
 		} else {
-			NCR_TRACE(("%d:%d busy\n",
+			NCR_TRACE(("[%s %d:%d busy] \n", __func__,
 			    ecb->ccb->ccb_h.target_id,
 			    ecb->ccb->ccb_h.target_lun));
 		}
@@ -1356,7 +1348,7 @@ ncr53c9x_sense(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("requesting sense "));
+	NCR_TRACE(("[%s] ", __func__));
 
 	lun = ccb->ccb_h.target_lun;
 	ti = &sc->sc_tinfo[ccb->ccb_h.target_id];
@@ -1367,7 +1359,8 @@ ncr53c9x_sense(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	ss->byte2 = ccb->ccb_h.target_lun << SCSI_CMD_LUN_SHIFT;
 	ss->length = sizeof(struct scsi_sense_data);
 	ecb->clen = sizeof(*ss);
-	ecb->daddr = (char *)&ecb->ccb->csio.sense_data;
+	memset(&ccb->csio.sense_data, 0, sizeof(ccb->csio.sense_data));
+	ecb->daddr = (uint8_t *)&ccb->csio.sense_data;
 	ecb->dleft = sizeof(struct scsi_sense_data);
 	ecb->flags |= ECB_SENSE;
 	ecb->timeout = NCR_SENSE_TIMEOUT;
@@ -1378,9 +1371,9 @@ ncr53c9x_sense(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	ncr53c9x_dequeue(sc, ecb);
 	li->untagged = ecb;	/* Must be executed first to fix C/A. */
 	li->busy = 2;
-	if (ecb == sc->sc_nexus) {
+	if (ecb == sc->sc_nexus)
 		ncr53c9x_select(sc, ecb);
-	} else {
+	else {
 		TAILQ_INSERT_HEAD(&sc->ready_list, ecb, chain);
 		ecb->flags |= ECB_READY;
 		if (sc->sc_state == NCR_IDLE)
@@ -1397,11 +1390,11 @@ ncr53c9x_done(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	union ccb *ccb = ecb->ccb;
 	struct ncr53c9x_linfo *li;
 	struct ncr53c9x_tinfo *ti;
-	int lun;
+	int lun, sense_returned;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_done(status:%x)] ", ccb->ccb_h.status));
+	NCR_TRACE(("[%s(status:%x)] ", __func__, ccb->ccb_h.status));
 
 	ti = &sc->sc_tinfo[ccb->ccb_h.target_id];
 	lun = ccb->ccb_h.target_lun;
@@ -1426,6 +1419,13 @@ ncr53c9x_done(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
 			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR |
 			    CAM_AUTOSNS_VALID;
+			sense_returned = sizeof(ccb->csio.sense_data) -
+			    ecb->dleft;
+			if (sense_returned < ccb->csio.sense_len)
+				ccb->csio.sense_resid = ccb->csio.sense_len -
+				    sense_returned;
+			else
+				ccb->csio.sense_resid = 0;
 		} else if (ecb->stat == SCSI_STATUS_CHECK_COND) {
 			if ((ecb->flags & ECB_SENSE) != 0)
 				ccb->ccb_h.status = CAM_AUTOSENSE_FAIL;
@@ -1449,7 +1449,7 @@ ncr53c9x_done(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	}
 
 #ifdef NCR53C9X_DEBUG
-	if (ncr53c9x_debug & NCR_SHOWTRAC) {
+	if ((ncr53c9x_debug & NCR_SHOWTRAC) != 0) {
 		if (ccb->csio.resid != 0)
 			printf("resid=%d ", ccb->csio.resid);
 		if ((ccb->ccb_h.status & CAM_AUTOSNS_VALID) != 0)
@@ -1502,7 +1502,7 @@ ncr53c9x_dequeue(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	li = TINFO_LUN(ti, lun);
 #ifdef DIAGNOSTIC
 	if (li == NULL || li->lun != lun)
-		panic("ncr53c9x_dequeue: lun %qx for ecb %p does not exist",
+		panic("%s: lun %qx for ecb %p does not exist", __func__,
 		    (long long)lun, ecb);
 #endif
 	if (li->untagged == ecb) {
@@ -1513,9 +1513,9 @@ ncr53c9x_dequeue(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 #ifdef DIAGNOSTIC
 		if (li->queued[ecb->tag[1]] != NULL &&
 		    (li->queued[ecb->tag[1]] != ecb))
-			panic("ncr53c9x_dequeue: slot %d for lun %qx has %p "
-			    "instead of ecb %p\n", ecb->tag[1],
-			    (long long)lun, li->queued[ecb->tag[1]], ecb);
+			panic("%s: slot %d for lun %qx has %p instead of ecb "
+			    "%p", __func__, ecb->tag[1], (long long)lun,
+			    li->queued[ecb->tag[1]], ecb);
 #endif
 		li->queued[ecb->tag[1]] = NULL;
 		li->used--;
@@ -1542,7 +1542,7 @@ ncr53c9x_dequeue(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	NCRCMD(sc, NCRCMD_SETATN);					\
 	sc->sc_flags |= NCR_ATN;					\
 	sc->sc_msgpriq |= (m);						\
-} while (0)
+} while (/* CONSTCOND */0)
 
 static void
 ncr53c9x_flushfifo(struct ncr53c9x_softc *sc)
@@ -1550,7 +1550,7 @@ ncr53c9x_flushfifo(struct ncr53c9x_softc *sc)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[flushfifo] "));
+	NCR_TRACE(("[%s] ", __func__));
 
 	NCRCMD(sc, NCRCMD_FLUSH);
 
@@ -1562,8 +1562,8 @@ ncr53c9x_flushfifo(struct ncr53c9x_softc *sc)
 static int
 ncr53c9x_rdfifo(struct ncr53c9x_softc *sc, int how)
 {
-	u_char *ibuf;
 	int i, n;
+	uint8_t *ibuf;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
@@ -1578,7 +1578,7 @@ ncr53c9x_rdfifo(struct ncr53c9x_softc *sc, int how)
 		break;
 
 	default:
-		panic("ncr53c9x_rdfifo: bad flag");
+		panic("%s: bad flag", __func__);
 		/* NOTREACHED */
 	}
 
@@ -1603,10 +1603,9 @@ ncr53c9x_rdfifo(struct ncr53c9x_softc *sc, int how)
 
 			ncr53c9x_flushfifo(sc);
 		}
-	} else {
+	} else
 		for (i = 0; i < n; i++)
 			ibuf[i] = NCR_READ_REG(sc, NCR_FIFO);
-	}
 
 	sc->sc_imlen += i;
 
@@ -1614,7 +1613,7 @@ ncr53c9x_rdfifo(struct ncr53c9x_softc *sc, int how)
 #ifdef NCR53C9X_DEBUG
 	NCR_TRACE(("\n[rdfifo %s (%d):",
 	    (how == NCR_RDFIFO_START) ? "start" : "cont", (int)sc->sc_imlen));
-	if (ncr53c9x_debug & NCR_SHOWTRAC) {
+	if ((ncr53c9x_debug & NCR_SHOWTRAC) != 0) {
 		for (i = 0; i < sc->sc_imlen; i++)
 			printf(" %02x", sc->sc_imess[i]);
 		printf("]\n");
@@ -1625,7 +1624,7 @@ ncr53c9x_rdfifo(struct ncr53c9x_softc *sc, int how)
 }
 
 static void
-ncr53c9x_wrfifo(struct ncr53c9x_softc *sc, u_char *p, int len)
+ncr53c9x_wrfifo(struct ncr53c9x_softc *sc, uint8_t *p, int len)
 {
 	int i;
 
@@ -1633,7 +1632,7 @@ ncr53c9x_wrfifo(struct ncr53c9x_softc *sc, u_char *p, int len)
 
 #ifdef NCR53C9X_DEBUG
 	NCR_MSGS(("[wrfifo(%d):", len));
-	if (ncr53c9x_debug & NCR_SHOWMSGS) {
+	if ((ncr53c9x_debug & NCR_SHOWMSGS) != 0) {
 		for (i = 0; i < len; i++)
 			printf(" %02x", p[i]);
 		printf("]\n");
@@ -1655,13 +1654,13 @@ ncr53c9x_reselect(struct ncr53c9x_softc *sc, int message, int tagtype,
 	struct ncr53c9x_ecb *ecb = NULL;
 	struct ncr53c9x_linfo *li;
 	struct ncr53c9x_tinfo *ti;
-	u_char lun, selid, target;
+	uint8_t lun, selid, target;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	if (sc->sc_rev == NCR_VARIANT_FAS366) {
+	if (sc->sc_rev == NCR_VARIANT_FAS366)
 		target = sc->sc_selid;
-	} else {
+	else {
 		/*
 		 * The SCSI chip made a snapshot of the data bus
 		 * while the reselection was being negotiated.
@@ -1743,7 +1742,7 @@ ncr53c9x_reselect(struct ncr53c9x_softc *sc, int message, int tagtype,
 #define	MSG_IS2BYTE(m)		(((m) & 0xf0) == 0x20)
 
 static inline int
-__verify_msg_format(u_char *p, int len)
+__verify_msg_format(uint8_t *p, int len)
 {
 
 	if (len == 1 && MSG_IS1BYTE(p[0]))
@@ -1769,12 +1768,12 @@ ncr53c9x_msgin(struct ncr53c9x_softc *sc)
 	struct ncr53c9x_ecb *ecb;
 	struct ncr53c9x_linfo *li;
 	struct ncr53c9x_tinfo *ti;
-	u_char *pb;
+	uint8_t *pb;
 	int lun, plen;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_msgin(curmsglen:%ld)] ", (long)sc->sc_imlen));
+	NCR_TRACE(("[%s(curmsglen:%ld)] ", __func__, (long)sc->sc_imlen));
 
 	if (sc->sc_imlen == 0) {
 		device_printf(sc->sc_dev, "msgin: no msg byte available\n");
@@ -2060,19 +2059,19 @@ ncr53c9x_msgin(struct ncr53c9x_softc *sc)
 			    sc->sc_imess[0]);
 			goto reset;
 		}
-		(void) ncr53c9x_reselect(sc, sc->sc_msgify,
+		(void)ncr53c9x_reselect(sc, sc->sc_msgify,
 		    sc->sc_imess[0], sc->sc_imess[1]);
 		break;
 
 	case NCR_RESELECTED:
-		if (MSG_ISIDENTIFY(sc->sc_imess[1])) {
+		if (MSG_ISIDENTIFY(sc->sc_imess[1]))
 			sc->sc_msgify = sc->sc_imess[1];
-		} else {
+		else {
 			device_printf(sc->sc_dev, "reselect without IDENTIFY;"
 			    " MSG %x; sending DEVICE RESET\n", sc->sc_imess[1]);
 			goto reset;
 		}
-		(void) ncr53c9x_reselect(sc, sc->sc_msgify, 0, 0);
+		(void)ncr53c9x_reselect(sc, sc->sc_msgify, 0, 0);
 		break;
 
 	default:
@@ -2116,8 +2115,8 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
-	NCR_TRACE(("[ncr53c9x_msgout(priq:%x, prevphase:%x)]",
-	    sc->sc_msgpriq, sc->sc_prevphase));
+	NCR_TRACE(("[%s(priq:%x, prevphase:%x)]", __func__, sc->sc_msgpriq,
+	    sc->sc_prevphase));
 
 	/*
 	 * XXX - the NCR_ATN flag is not in sync with the actual ATN
@@ -2130,7 +2129,9 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 		if (sc->sc_prevphase != MESSAGE_OUT_PHASE) {
 		new:
 			NCRCMD(sc, NCRCMD_FLUSH);
-/*			DELAY(1); */
+#if 0
+			DELAY(1);
+#endif
 			sc->sc_msgoutq = 0;
 			sc->sc_omlen = 0;
 		}
@@ -2138,10 +2139,9 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 		if (sc->sc_prevphase == MESSAGE_OUT_PHASE) {
 			ncr53c9x_sched_msgout(sc->sc_msgoutq);
 			goto new;
-		} else {
+		} else
 			device_printf(sc->sc_dev, "at line %d: unexpected "
 			    "MESSAGE OUT phase\n", __LINE__);
-		}
 	}
 
 	if (sc->sc_omlen == 0) {
@@ -2173,20 +2173,18 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 			break;
 
 		case SEND_IDENTIFY:
-			if (sc->sc_state != NCR_CONNECTED) {
+			if (sc->sc_state != NCR_CONNECTED)
 				device_printf(sc->sc_dev, "at line %d: no "
 				    "nexus\n", __LINE__);
-			}
 			ecb = sc->sc_nexus;
 			sc->sc_omess[0] =
 			    MSG_IDENTIFY(ecb->ccb->ccb_h.target_lun, 0);
 			break;
 
 		case SEND_TAG:
-			if (sc->sc_state != NCR_CONNECTED) {
+			if (sc->sc_state != NCR_CONNECTED)
 				device_printf(sc->sc_dev, "at line %d: no "
 				    "nexus\n", __LINE__);
-			}
 			ecb = sc->sc_nexus;
 			sc->sc_omess[0] = ecb->tag[0];
 			sc->sc_omess[1] = ecb->tag[1];
@@ -2241,7 +2239,7 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 	}
 
 #ifdef NCR53C9X_DEBUG
-	if (ncr53c9x_debug & NCR_SHOWMSGS) {
+	if ((ncr53c9x_debug & NCR_SHOWMSGS) != 0) {
 		NCR_MSGS(("<msgout:"));
 		for (i = 0; i < sc->sc_omlen; i++)
 			NCR_MSGS((" %02x", sc->sc_omess[i]));
@@ -2302,7 +2300,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 	struct timeval cur, wait;
 	size_t size;
 	int i, nfifo;
-	u_char msg;
+	uint8_t msg;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
 
@@ -2377,7 +2375,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 				 * while we were trying to select
 				 * another target.
 				 */
-#ifdef DEBUG
+#ifdef NCR53C9X_DEBUG
 				device_printf(sc->sc_dev, "ESP100 work-around "
 				    "activated\n");
 #endif
@@ -2484,7 +2482,9 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 		    sc->sc_espintr,sc->sc_espstat,sc->sc_espstep));
 		if (NCR_READ_REG(sc, NCR_FFLAG) & NCRFIFO_FF) {
 			NCRCMD(sc, NCRCMD_FLUSH);
-/*			DELAY(1); */
+#if 0
+			DELAY(1);
+#endif
 		}
 		/*
 		 * This command must (apparently) be issued within
@@ -2726,7 +2726,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 				if (sc->sc_state == NCR_IDLE &&
 				    sc->sc_espstep == 0)
 					return;
-				panic("ncr53c9x: no nexus");
+				panic("%s: no nexus", __func__);
 			}
 
 			ti = &sc->sc_tinfo[ecb->ccb->ccb_h.target_id];
@@ -2804,8 +2804,8 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 					if (sc->sc_cmdlen == 0)
 						/* Hope for the best... */
 						break;
-				} else if ((NCR_READ_REG(sc, NCR_FFLAG)
-				    & NCRFIFO_FF) == 0) {
+				} else if ((NCR_READ_REG(sc, NCR_FFLAG) &
+				    NCRFIFO_FF) == 0) {
 					/* Hope for the best... */
 					break;
 				}
@@ -2864,7 +2864,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 			/* "Initiate Command Complete Steps" in progress */
 			sc->sc_flags &= ~NCR_ICCS;
 
-			if (!(sc->sc_espintr & NCRINTR_DONE)) {
+			if ((sc->sc_espintr & NCRINTR_DONE) == 0) {
 				device_printf(sc->sc_dev, "ICCS: "
 				    ": [intr %x, stat %x, step %x]\n",
 				    sc->sc_espintr, sc->sc_espstat,
@@ -2904,9 +2904,8 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 	 * Driver is now in state NCR_CONNECTED, i.e. we
 	 * have a current command working the SCSI bus.
 	 */
-	if (sc->sc_state != NCR_CONNECTED || ecb == NULL) {
-		panic("ncr53c9x: no nexus");
-	}
+	if (sc->sc_state != NCR_CONNECTED || ecb == NULL)
+		panic("%s: no nexus", __func__);
 
 	switch (sc->sc_phase) {
 	case MESSAGE_OUT_PHASE:
@@ -2920,7 +2919,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 		NCR_PHASE(("MESSAGE_IN_PHASE "));
 		if ((sc->sc_espintr & NCRINTR_BS) != 0) {
 			if ((sc->sc_rev != NCR_VARIANT_FAS366) ||
-			    !(sc->sc_espstat2 & NCRFAS_STAT2_EMPTY)) {
+			    (sc->sc_espstat2 & NCRFAS_STAT2_EMPTY) == 0) {
 				NCRCMD(sc, NCRCMD_FLUSH);
 			}
 			sc->sc_flags |= NCR_WAITI;
@@ -2937,11 +2936,10 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 			    (sc->sc_prevphase == sc->sc_phase) ?
 			    NCR_RDFIFO_CONTINUE : NCR_RDFIFO_START);
 			ncr53c9x_msgin(sc);
-		} else {
+		} else
 			device_printf(sc->sc_dev, "MSGIN: weird bits: "
 			    "[intr %x, stat %x, step %x]\n",
 			    sc->sc_espintr, sc->sc_espstat, sc->sc_espstep);
-		}
 		sc->sc_prevphase = MESSAGE_IN_PHASE;
 		goto shortcut;	/* i.e. expect data to be ready */
 
@@ -2958,7 +2956,9 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 		    ecb->cmd.cmd.opcode, ecb->clen));
 		if (NCR_READ_REG(sc, NCR_FFLAG) & NCRFIFO_FF) {
 			NCRCMD(sc, NCRCMD_FLUSH);
-/*			DELAY(1);*/
+#if 0
+			DELAY(1);
+#endif
 		}
 		/*
 		 * If we have more messages to send, e.g. WDTR or SDTR
@@ -2973,7 +2973,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 			/* Setup DMA transfer for command. */
 			size = ecb->clen;
 			sc->sc_cmdlen = size;
-			sc->sc_cmdp = (caddr_t)&ecb->cmd.cmd;
+			sc->sc_cmdp = (void *)&ecb->cmd.cmd;
 			NCRDMA_SETUP(sc, &sc->sc_cmdp, &sc->sc_cmdlen,
 			    0, &size);
 			/* Program the SCSI counter. */
@@ -2986,7 +2986,8 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 			NCRCMD(sc, NCRCMD_TRANS | NCRCMD_DMA);
 			NCRDMA_GO(sc);
 		} else {
-			ncr53c9x_wrfifo(sc, (u_char *)&ecb->cmd.cmd, ecb->clen);
+			ncr53c9x_wrfifo(sc, (uint8_t *)&ecb->cmd.cmd,
+			    ecb->clen);
 			NCRCMD(sc, NCRCMD_TRANS);
 		}
 		sc->sc_prevphase = COMMAND_PHASE;
@@ -3167,7 +3168,7 @@ ncr53c9x_callout(void *arg)
 static void
 ncr53c9x_watch(void *arg)
 {
-	struct ncr53c9x_softc *sc = (struct ncr53c9x_softc *)arg;
+	struct ncr53c9x_softc *sc = arg;
 	struct ncr53c9x_linfo *li;
 	struct ncr53c9x_tinfo *ti;
 	time_t old;
diff --git a/sys/dev/esp/ncr53c9xreg.h b/sys/dev/esp/ncr53c9xreg.h
index 008692f5d785..ab0342173f80 100644
--- a/sys/dev/esp/ncr53c9xreg.h
+++ b/sys/dev/esp/ncr53c9xreg.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: ncr53c9xreg.h,v 1.14 2005/02/27 00:27:02 perry Exp $	*/
+/*	$NetBSD: ncr53c9xreg.h,v 1.16 2009/09/07 13:31:44 tsutsui Exp $	*/
 
 /*-
  * Copyright (c) 1994 Peter Galbavy.  All rights reserved.
@@ -43,84 +43,84 @@
 #define	NCR_FIFO	0x02		/* RW - FIFO data		*/
 
 #define	NCR_CMD		0x03		/* RW - Command (2 deep)	*/
-#define  NCRCMD_DMA	0x80		/*	DMA Bit			*/
-#define  NCRCMD_NOP	0x00		/*	No Operation		*/
-#define  NCRCMD_FLUSH	0x01		/*	Flush FIFO		*/
-#define  NCRCMD_RSTCHIP	0x02		/*	Reset Chip		*/
-#define  NCRCMD_RSTSCSI	0x03		/*	Reset SCSI Bus		*/
-#define  NCRCMD_RESEL	0x40		/*	Reselect Sequence	*/
-#define  NCRCMD_SELNATN	0x41		/*	Select without ATN	*/
-#define  NCRCMD_SELATN	0x42		/*	Select with ATN		*/
-#define  NCRCMD_SELATNS	0x43		/*	Select with ATN & Stop	*/
-#define  NCRCMD_ENSEL	0x44		/*	Enable (Re)Selection	*/
-#define  NCRCMD_DISSEL	0x45		/*	Disable (Re)Selection	*/
-#define  NCRCMD_SELATN3	0x46		/*	Select with ATN3	*/
-#define  NCRCMD_RESEL3	0x47		/*	Reselect3 Sequence	*/
-#define  NCRCMD_SNDMSG	0x20		/*	Send Message		*/
-#define  NCRCMD_SNDSTAT	0x21		/*	Send Status		*/
-#define  NCRCMD_SNDDATA	0x22		/*	Send Data		*/
-#define  NCRCMD_DISCSEQ	0x23		/*	Disconnect Sequence	*/
-#define  NCRCMD_TERMSEQ	0x24		/*	Terminate Sequence	*/
-#define  NCRCMD_TCCS	0x25		/*	Target Command Comp Seq	*/
-#define  NCRCMD_DISC	0x27		/*	Disconnect		*/
-#define  NCRCMD_RECMSG	0x28		/*	Receive Message		*/
-#define  NCRCMD_RECCMD	0x29		/*	Receive Command 	*/
-#define  NCRCMD_RECDATA	0x2a		/*	Receive Data		*/
-#define  NCRCMD_RECCSEQ	0x2b		/*	Receive Command Sequence*/
-#define  NCRCMD_ABORT	0x04		/*	Target Abort DMA	*/
-#define  NCRCMD_TRANS	0x10		/*	Transfer Information	*/
-#define  NCRCMD_ICCS	0x11		/*	Initiator Cmd Comp Seq 	*/
-#define  NCRCMD_MSGOK	0x12		/*	Message Accepted	*/
-#define  NCRCMD_TRPAD	0x18		/*	Transfer Pad		*/
-#define  NCRCMD_SETATN	0x1a		/*	Set ATN			*/
-#define  NCRCMD_RSTATN	0x1b		/*	Reset ATN		*/
+#define	 NCRCMD_DMA	0x80		/*	DMA Bit			*/
+#define	 NCRCMD_NOP	0x00		/*	No Operation		*/
+#define	 NCRCMD_FLUSH	0x01		/*	Flush FIFO		*/
+#define	 NCRCMD_RSTCHIP	0x02		/*	Reset Chip		*/
+#define	 NCRCMD_RSTSCSI	0x03		/*	Reset SCSI Bus		*/
+#define	 NCRCMD_RESEL	0x40		/*	Reselect Sequence	*/
+#define	 NCRCMD_SELNATN	0x41		/*	Select without ATN	*/
+#define	 NCRCMD_SELATN	0x42		/*	Select with ATN		*/
+#define	 NCRCMD_SELATNS	0x43		/*	Select with ATN & Stop	*/
+#define	 NCRCMD_ENSEL	0x44		/*	Enable (Re)Selection	*/
+#define	 NCRCMD_DISSEL	0x45		/*	Disable (Re)Selection	*/
+#define	 NCRCMD_SELATN3	0x46		/*	Select with ATN3	*/
+#define	 NCRCMD_RESEL3	0x47		/*	Reselect3 Sequence	*/
+#define	 NCRCMD_SNDMSG	0x20		/*	Send Message		*/
+#define	 NCRCMD_SNDSTAT	0x21		/*	Send Status		*/
+#define	 NCRCMD_SNDDATA	0x22		/*	Send Data		*/
+#define	 NCRCMD_DISCSEQ	0x23		/*	Disconnect Sequence	*/
+#define	 NCRCMD_TERMSEQ	0x24		/*	Terminate Sequence	*/
+#define	 NCRCMD_TCCS	0x25		/*	Target Command Comp Seq	*/
+#define	 NCRCMD_DISC	0x27		/*	Disconnect		*/
+#define	 NCRCMD_RECMSG	0x28		/*	Receive Message		*/
+#define	 NCRCMD_RECCMD	0x29		/*	Receive Command		*/
+#define	 NCRCMD_RECDATA	0x2a		/*	Receive Data		*/
+#define	 NCRCMD_RECCSEQ	0x2b		/*	Receive Command Sequence*/
+#define	 NCRCMD_ABORT	0x04		/*	Target Abort DMA	*/
+#define	 NCRCMD_TRANS	0x10		/*	Transfer Information	*/
+#define	 NCRCMD_ICCS	0x11		/*	Initiator Cmd Comp Seq	*/
+#define	 NCRCMD_MSGOK	0x12		/*	Message Accepted	*/
+#define	 NCRCMD_TRPAD	0x18		/*	Transfer Pad		*/
+#define	 NCRCMD_SETATN	0x1a		/*	Set ATN			*/
+#define	 NCRCMD_RSTATN	0x1b		/*	Reset ATN		*/
 
 #define	NCR_STAT	0x04		/* RO - Status			*/
-#define  NCRSTAT_INT	0x80		/*	Interrupt		*/
-#define  NCRSTAT_GE	0x40		/*	Gross Error		*/
-#define  NCRSTAT_PE	0x20		/*	Parity Error		*/
-#define  NCRSTAT_TC	0x10		/*	Terminal Count		*/
-#define  NCRSTAT_VGC	0x08		/*	Valid Group Code	*/
-#define  NCRSTAT_PHASE	0x07		/*	Phase bits		*/
+#define	 NCRSTAT_INT	0x80		/*	Interrupt		*/
+#define	 NCRSTAT_GE	0x40		/*	Gross Error		*/
+#define	 NCRSTAT_PE	0x20		/*	Parity Error		*/
+#define	 NCRSTAT_TC	0x10		/*	Terminal Count		*/
+#define	 NCRSTAT_VGC	0x08		/*	Valid Group Code	*/
+#define	 NCRSTAT_PHASE	0x07		/*	Phase bits		*/
 
 #define	NCR_SELID	0x04		/* WO - Select/Reselect Bus ID	*/
-#define  NCR_BUSID_HMEXC32	0x40	/*	HME xfer counter is 32bit */
-#define  NCR_BUSID_HMEENCID	0x10 	/*	HME encode reselection ID */
+#define	 NCR_BUSID_HMEXC32	0x40	/*	HME xfer counter is 32bit */
+#define	 NCR_BUSID_HMEENCID	0x10	/*	HME encode reselection ID */
 
 #define	NCR_INTR	0x05		/* RO - Interrupt		*/
-#define  NCRINTR_SBR	0x80		/*	SCSI Bus Reset		*/
-#define  NCRINTR_ILL	0x40		/*	Illegal Command		*/
-#define  NCRINTR_DIS	0x20		/*	Disconnect		*/
-#define  NCRINTR_BS	0x10		/*	Bus Service		*/
-#define  NCRINTR_FC	0x08		/*	Function Complete	*/
-#define  NCRINTR_RESEL	0x04		/*	Reselected		*/
-#define  NCRINTR_SELATN	0x02		/*	Select with ATN		*/
-#define  NCRINTR_SEL	0x01		/*	Selected		*/
+#define	 NCRINTR_SBR	0x80		/*	SCSI Bus Reset		*/
+#define	 NCRINTR_ILL	0x40		/*	Illegal Command		*/
+#define	 NCRINTR_DIS	0x20		/*	Disconnect		*/
+#define	 NCRINTR_BS	0x10		/*	Bus Service		*/
+#define	 NCRINTR_FC	0x08		/*	Function Complete	*/
+#define	 NCRINTR_RESEL	0x04		/*	Reselected		*/
+#define	 NCRINTR_SELATN	0x02		/*	Select with ATN		*/
+#define	 NCRINTR_SEL	0x01		/*	Selected		*/
 
 #define	NCR_TIMEOUT	0x05		/* WO - Select/Reselect Timeout */
 
 #define	NCR_STEP	0x06		/* RO - Sequence Step		*/
-#define  NCRSTEP_MASK	0x07		/*	the last 3 bits		*/
-#define  NCRSTEP_DONE	0x04		/*	command went out	*/
+#define	 NCRSTEP_MASK	0x07		/*	the last 3 bits		*/
+#define	 NCRSTEP_DONE	0x04		/*	command went out	*/
 
 #define	NCR_SYNCTP	0x06		/* WO - Synch Transfer Period	*/
 					/*	Default 5 (53C9X)	*/
 
 #define	NCR_FFLAG	0x07		/* RO - FIFO Flags		*/
-#define  NCRFIFO_SS	0xe0		/*	Sequence Step (Dup)	*/
-#define  NCRFIFO_FF	0x1f		/*	Bytes in FIFO		*/
+#define	 NCRFIFO_SS	0xe0		/*	Sequence Step (Dup)	*/
+#define	 NCRFIFO_FF	0x1f		/*	Bytes in FIFO		*/
 
 #define	NCR_SYNCOFF	0x07		/* WO - Synch Offset		*/
 					/*	0 = ASYNC		*/
 					/*	1 - 15 = SYNC bytes	*/
 
 #define	NCR_CFG1	0x08		/* RW - Configuration #1	*/
-#define  NCRCFG1_SLOW	0x80		/*	Slow Cable Mode		*/
-#define  NCRCFG1_SRR	0x40		/*	SCSI Reset Rep Int Dis	*/
-#define  NCRCFG1_PTEST	0x20		/*	Parity Test Mod		*/
-#define  NCRCFG1_PARENB	0x10		/*	Enable Parity Check	*/
-#define  NCRCFG1_CTEST	0x08		/*	Enable Chip Test	*/
-#define  NCRCFG1_BUSID	0x07		/*	Bus ID			*/
+#define	 NCRCFG1_SLOW	0x80		/*	Slow Cable Mode		*/
+#define	 NCRCFG1_SRR	0x40		/*	SCSI Reset Rep Int Dis	*/
+#define	 NCRCFG1_PTEST	0x20		/*	Parity Test Mod		*/
+#define	 NCRCFG1_PARENB	0x10		/*	Enable Parity Check	*/
+#define	 NCRCFG1_CTEST	0x08		/*	Enable Chip Test	*/
+#define	 NCRCFG1_BUSID	0x07		/*	Bus ID			*/
 
 #define	NCR_CCF		0x09		/* WO -	Clock Conversion Factor	*/
 					/*	0 = 35.01 - 40MHz	*/
@@ -136,24 +136,24 @@
 
 #define	NCR_CFG2	0x0b		/* RW - Configuration #2	*/
 #define	 NCRCFG2_RSVD	0xa0		/*	reserved		*/
-#define  NCRCFG2_FE	0x40		/* 	Features Enable		*/
-#define  NCRCFG2_DREQ	0x10		/* 	DREQ High Impedance	*/
-#define  NCRCFG2_SCSI2	0x08		/* 	SCSI-2 Enable		*/
-#define  NCRCFG2_BPA	0x04		/* 	Target Bad Parity Abort	*/
-#define  NCRCFG2_RPE	0x02		/* 	Register Parity Error	*/
-#define  NCRCFG2_DPE	0x01		/* 	DMA Parity Error	*/
+#define	 NCRCFG2_FE	0x40		/*	Features Enable		*/
+#define	 NCRCFG2_DREQ	0x10		/*	DREQ High Impedance	*/
+#define	 NCRCFG2_SCSI2	0x08		/*	SCSI-2 Enable		*/
+#define	 NCRCFG2_BPA	0x04		/*	Target Bad Parity Abort	*/
+#define	 NCRCFG2_RPE	0x02		/*	Register Parity Error	*/
+#define	 NCRCFG2_DPE	0x01		/*	DMA Parity Error	*/
 
-#define  NCRCFG2_HMEFE	0x10		/*	HME feature enable	*/
+#define	 NCRCFG2_HMEFE	0x10		/*	HME feature enable	*/
 #define	 NCRCFG2_HME32  0x80		/*	HME 32 extended		*/
 
 /* Config #3 only on 53C9X */
 #define	NCR_CFG3	0x0c		/* RW - Configuration #3	*/
 #define	 NCRCFG3_RSVD	0xe0		/*	reserved		*/
-#define  NCRCFG3_IDM	0x10		/*	ID Message Res Check	*/
-#define  NCRCFG3_QTE	0x08		/*	Queue Tag Enable	*/
-#define  NCRCFG3_CDB	0x04		/*	CDB 10-bytes OK		*/
-#define  NCRCFG3_FSCSI	0x02		/*	Fast SCSI		*/
-#define  NCRCFG3_FCLK	0x01		/*	Fast Clock (>25MHz)	*/
+#define	 NCRCFG3_IDM	0x10		/*	ID Message Res Check	*/
+#define	 NCRCFG3_QTE	0x08		/*	Queue Tag Enable	*/
+#define	 NCRCFG3_CDB	0x04		/*	CDB 10-bytes OK		*/
+#define	 NCRCFG3_FSCSI	0x02		/*	Fast SCSI		*/
+#define	 NCRCFG3_FCLK	0x01		/*	Fast Clock (>25MHz)	*/
 
 /*
  * For some unknown reason, the ESP406/FAS408 looks like every
@@ -164,35 +164,35 @@
 
 /* Config #3 different on ESP406/FAS408 */
 #define	NCR_ESPCFG3		0x0c	/* RW - Configuration #3	*/
-#define  NCRESPCFG3_IDM		0x80	/*	ID Message Res Check	*/
-#define  NCRESPCFG3_QTE		0x40	/*	Queue Tag Enable	*/
-#define  NCRESPCFG3_CDB		0x20	/*	CDB 10-bytes OK		*/
-#define  NCRESPCFG3_FSCSI	0x10	/*	Fast SCSI		*/
+#define	 NCRESPCFG3_IDM		0x80	/*	ID Message Res Check	*/
+#define	 NCRESPCFG3_QTE		0x40	/*	Queue Tag Enable	*/
+#define	 NCRESPCFG3_CDB		0x20	/*	CDB 10-bytes OK		*/
+#define	 NCRESPCFG3_FSCSI	0x10	/*	Fast SCSI		*/
 #define	 NCRESPCFG3_SRESB	0x08	/*	Save Residual Byte	*/
-#define  NCRESPCFG3_FCLK	0x04	/*	Fast Clock (>25MHz)	*/
+#define	 NCRESPCFG3_FCLK	0x04	/*	Fast Clock (>25MHz)	*/
 #define	 NCRESPCFG3_ADMA	0x02	/*	Alternate DMA Mode	*/
 #define	 NCRESPCFG3_T8M		0x01	/*	Threshold 8 Mode	*/
 
 /* Config #3 also different on NCR53CF9x/FAS100A/FAS216/FAS236 */
 #define	NCR_F9XCFG3		0x0c	/* RW - Configuration #3	*/
-#define  NCRF9XCFG3_IDM		0x80	/*	ID Message Res Check	*/
-#define  NCRF9XCFG3_QTE		0x40	/*	Queue Tag Enable	*/
-#define  NCRF9XCFG3_CDB		0x20	/*	CDB 10-bytes OK		*/
-#define  NCRF9XCFG3_FSCSI	0x10	/*	Fast SCSI		*/
-#define  NCRF9XCFG3_FCLK	0x08	/*	Fast Clock (>25MHz)	*/
-#define  NCRF9XCFG3_SRESB	0x04	/*	Save Residual Byte	*/
-#define  NCRF9XCFG3_ADMA	0x02	/*	Alternate DMA Mode	*/
-#define  NCRF9XCFG3_T8M		0x01	/*	Threshold 8 Mode	*/
+#define	 NCRF9XCFG3_IDM		0x80	/*	ID Message Res Check	*/
+#define	 NCRF9XCFG3_QTE		0x40	/*	Queue Tag Enable	*/
+#define	 NCRF9XCFG3_CDB		0x20	/*	CDB 10-bytes OK		*/
+#define	 NCRF9XCFG3_FSCSI	0x10	/*	Fast SCSI		*/
+#define	 NCRF9XCFG3_FCLK	0x08	/*	Fast Clock (>25MHz)	*/
+#define	 NCRF9XCFG3_SRESB	0x04	/*	Save Residual Byte	*/
+#define	 NCRF9XCFG3_ADMA	0x02	/*	Alternate DMA Mode	*/
+#define	 NCRF9XCFG3_T8M		0x01	/*	Threshold 8 Mode	*/
 
 /* Config #3 on FAS366 */
-#define  NCRFASCFG3_OBAUTO    	0x80    /*	auto push odd-byte to DMA */
-#define  NCRFASCFG3_EWIDE     	0x40    /* 	Enable Wide-SCSI     */
-#define  NCRFASCFG3_IDBIT3	0x20	/* 	Bit 3 of HME SCSI-ID */
-#define	 NCRFASCFG3_IDRESCHK	0x10	/* 	ID message checking */
-#define	 NCRFASCFG3_QUENB	0x08	/* 	3-byte msg support */
-#define	 NCRFASCFG3_CDB10	0x04	/* 	group 2 scsi-2 support */
-#define	 NCRFASCFG3_FASTSCSI	0x02	/* 	10 MB/S fast scsi mode */
-#define	 NCRFASCFG3_FASTCLK	0x01	/* 	fast clock mode */
+#define	 NCRFASCFG3_OBAUTO	0x80    /*	auto push odd-byte to DMA */
+#define	 NCRFASCFG3_EWIDE	0x40    /*	Enable Wide-SCSI     */
+#define	 NCRFASCFG3_IDBIT3	0x20	/*	Bit 3 of HME SCSI-ID */
+#define	 NCRFASCFG3_IDRESCHK	0x10	/*	ID message checking */
+#define	 NCRFASCFG3_QUENB	0x08	/*	3-byte msg support */
+#define	 NCRFASCFG3_CDB10	0x04	/*	group 2 scsi-2 support */
+#define	 NCRFASCFG3_FASTSCSI	0x02	/*	10 MB/S fast scsi mode */
+#define	 NCRFASCFG3_FASTCLK	0x01	/*	fast clock mode */
 
 /* Config #4 only on ESP406/FAS408 */
 #define	NCR_CFG4	0x0d		/* RW - Configuration #4	*/
@@ -208,8 +208,8 @@
 */
 
 #define	NCR_JMP		0x00		/* RO - Jumper Sense Register	*/
-#define  NCRJMP_RSVD	0xc0		/*	reserved		*/
-#define  NCRJMP_ROMSZ	0x20		/*	ROM Size 1=16K, 0=32K	*/
+#define	 NCRJMP_RSVD	0xc0		/*	reserved		*/
+#define	 NCRJMP_ROMSZ	0x20		/*	ROM Size 1=16K, 0=32K	*/
 #define	 NCRJMP_J4	0x10		/*	Jumper #4		*/
 #define	 NCRJMP_J3	0x08		/*	Jumper #3		*/
 #define	 NCRJMP_J2	0x04		/*	Jumper #2		*/
@@ -218,17 +218,17 @@
 
 #define	NCR_PIOFIFO	0x04		/* WO - PIO FIFO, 4 bytes deep	*/
 
-#define NCR_PSTAT	0x08		/* RW - PIO Status Register	*/
-#define  NCRPSTAT_PERR	0x80		/*	PIO Error		*/
-#define  NCRPSTAT_SIRQ	0x40		/*	Active High of SCSI IRQ */
-#define  NCRPSTAT_ATAI	0x20		/*	ATA IRQ			*/
-#define  NCRPSTAT_FEMPT	0x10		/*	PIO FIFO Empty		*/
-#define  NCRPSTAT_F13	0x08		/*	PIO FIFO 1/3		*/
-#define  NCRPSTAT_F23	0x04		/*	PIO FIFO 2/3		*/
-#define  NCRPSTAT_FFULL	0x02		/*	PIO FIFO Full		*/
-#define  NCRPSTAT_PIOM	0x01		/*	PIO/DMA Mode		*/
+#define	NCR_PSTAT	0x08		/* RW - PIO Status Register	*/
+#define	 NCRPSTAT_PERR	0x80		/*	PIO Error		*/
+#define	 NCRPSTAT_SIRQ	0x40		/*	Active High of SCSI IRQ */
+#define	 NCRPSTAT_ATAI	0x20		/*	ATA IRQ			*/
+#define	 NCRPSTAT_FEMPT	0x10		/*	PIO FIFO Empty		*/
+#define	 NCRPSTAT_F13	0x08		/*	PIO FIFO 1/3		*/
+#define	 NCRPSTAT_F23	0x04		/*	PIO FIFO 2/3		*/
+#define	 NCRPSTAT_FFULL	0x02		/*	PIO FIFO Full		*/
+#define	 NCRPSTAT_PIOM	0x01		/*	PIO/DMA Mode		*/
 
-#define NCR_PIOI	0x0b		/* RW - PIO Interrupt Enable	*/
+#define	NCR_PIOI	0x0b		/* RW - PIO Interrupt Enable	*/
 #define	 NCRPIOI_RSVD	0xe0		/*	reserved		*/
 #define	 NCRPIOI_EMPTY	0x10		/*	IRQ When Empty		*/
 #define	 NCRPIOI_13	0x08		/*	IRQ When 1/3		*/
@@ -239,12 +239,12 @@
 #define	NCR_CFG5	0x0d		/* RW - Configuration #5	*/
 #define	 NCRCFG5_CRS1	0x80		/*	Select Register Set #1	*/
 #define	 NCRCFG5_SRAM	0x40		/*	SRAM Memory Map		*/
-#define  NCRCFG5_AADDR	0x20		/*	Auto Address		*/
-#define  NCRCFG5_PTRINC	0x10		/*	Pointer Increment	*/
-#define  NCRCFG5_LOWPWR	0x08		/*	Low Power Mode		*/
-#define  NCRCFG5_SINT	0x04		/*	SCSI Interrupt Enable	*/
-#define  NCRCFG5_INTP	0x02		/*	INT Polarity		*/
-#define  NCRCFG5_AINT	0x01		/*	ATA Interrupt Enable	*/
+#define	 NCRCFG5_AADDR	0x20		/*	Auto Address		*/
+#define	 NCRCFG5_PTRINC	0x10		/*	Pointer Increment	*/
+#define	 NCRCFG5_LOWPWR	0x08		/*	Low Power Mode		*/
+#define	 NCRCFG5_SINT	0x04		/*	SCSI Interrupt Enable	*/
+#define	 NCRCFG5_INTP	0x02		/*	INT Polarity		*/
+#define	 NCRCFG5_AINT	0x01		/*	ATA Interrupt Enable	*/
 
 #define	NCR_SIGNTR	0x0e		/* RO - Signature		*/
 
@@ -272,13 +272,13 @@
 /*
  * FAS366
  */
-#define NCR_RCL		NCR_TCH	/* Recommand counter low */
-#define NCR_RCH		0xf	/* Recommand counter high */
-#define NCR_UID		NCR_RCL	/* fas366 part-uniq id */
+#define	NCR_RCL		NCR_TCH	/* Recommand counter low */
+#define	NCR_RCH		0xf	/* Recommand counter high */
+#define	NCR_UID		NCR_RCL	/* fas366 part-uniq id */
 
 
 /* status register #2 definitions (read	only) */
-#define NCR_STAT2	NCR_CCF
+#define	NCR_STAT2	NCR_CCF
 #define	NCRFAS_STAT2_SEQCNT   0x01	   /* Sequence counter bit 7-3 enabled */
 #define	NCRFAS_STAT2_FLATCHED 0x02	   /* FIFO flags register latched */
 #define	NCRFAS_STAT2_CLATCHED 0x04	   /* Xfer cntr	& recommand ctr	latched */
diff --git a/sys/dev/esp/ncr53c9xvar.h b/sys/dev/esp/ncr53c9xvar.h
index 7607e5679abe..38285fcd6a10 100644
--- a/sys/dev/esp/ncr53c9xvar.h
+++ b/sys/dev/esp/ncr53c9xvar.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: ncr53c9xvar.h,v 1.46 2005/02/04 02:10:36 perry Exp $	*/
+/*	$NetBSD: ncr53c9xvar.h,v 1.55 2011/07/31 18:39:00 jakllsch Exp $	*/
 
 /*-
  * Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -133,27 +133,27 @@ struct ncr53c9x_ecb {
 	struct callout ch;
 
 	struct {
-		u_char	msg[3];			/* Selection Id msg and tags */
+		uint8_t	msg[3];			/* Selection Id msg and tags */
 		struct scsi_generic cmd;	/* SCSI command block */
 	} cmd;
-	char	*daddr;		/* Saved data pointer */
+	uint8_t	*daddr;		/* Saved data pointer */
 	int	 clen;		/* Size of command in cmd.cmd */
 	int	 dleft;		/* Residue */
-	u_char	 stat;		/* SCSI status byte */
-	u_char	 tag[2];	/* TAG bytes */
-	u_char	 pad[1];
+	uint8_t	 stat;		/* SCSI status byte */
+	uint8_t	 tag[2];	/* TAG bytes */
+	uint8_t	 pad[1];
 
 #if defined(NCR53C9X_DEBUG) && NCR53C9X_DEBUG > 1
 	char trace[1000];
 #endif
 };
 #if defined(NCR53C9X_DEBUG) && NCR53C9X_DEBUG > 1
-#define	ECB_TRACE(ecb, msg, a, b) do { \
-	const char *f = "[" msg "]"; \
-	int n = strlen((ecb)->trace); \
-	if (n < (sizeof((ecb)->trace)-100)) \
-		sprintf((ecb)->trace + n, f, a, b); \
-} while(0)
+#define	ECB_TRACE(ecb, msg, a, b) do {					\
+	const char *f = "[" msg "]";					\
+	int n = strlen((ecb)->trace);					\
+	if (n < (sizeof((ecb)->trace)-100))				\
+		sprintf((ecb)->trace + n, f, a, b);			\
+} while (/* CONSTCOND */0)
 #else
 #define	ECB_TRACE(ecb, msg, a, b)
 #endif
@@ -174,17 +174,17 @@ struct ncr53c9x_linfo {
 	int64_t			lun;
 	LIST_ENTRY(ncr53c9x_linfo) link;
 	time_t			last_used;
-	u_char			used;	/* # slots in use */
-	u_char			avail;	/* where to start scanning */
-	u_char			busy;
+	uint8_t			used;	/* # slots in use */
+	uint8_t			avail;	/* where to start scanning */
+	uint8_t			busy;
 	struct ncr53c9x_ecb	*untagged;
 	struct ncr53c9x_ecb	*queued[NCR_TAG_DEPTH];
 };
 
 struct ncr53c9x_xinfo {
-	u_char	period;
-	u_char	offset;
-	u_char	width;
+	uint8_t	period;
+	uint8_t	offset;
+	uint8_t	width;
 };
 
 struct ncr53c9x_tinfo {
@@ -193,7 +193,7 @@ struct ncr53c9x_tinfo {
 	int	touts;		/* # of timeouts */
 	int	perrs;		/* # of parity errors */
 	int	senses;		/* # of request sense commands sent */
-	u_char	flags;
+	uint8_t	flags;
 #define	T_SYNCHOFF	0x01	/* SYNC mode is permanently off */
 #define	T_RSELECTOFF	0x02	/* RE-SELECT mode is off */
 #define	T_TAG		0x04	/* Turn on TAG QUEUEs */
@@ -206,13 +206,13 @@ struct ncr53c9x_tinfo {
 };
 
 /* Look up a lun in a tinfo */
-#define	TINFO_LUN(t, l) (					\
-	(((l) < NCR_NLUN) && (((t)->lun[(l)]) != NULL))		\
-		? ((t)->lun[(l)])				\
-		: ncr53c9x_lunsearch((t), (int64_t)(l))		\
+#define	TINFO_LUN(t, l) (						\
+	(((l) < NCR_NLUN) && (((t)->lun[(l)]) != NULL))			\
+		? ((t)->lun[(l)])					\
+		: ncr53c9x_lunsearch((t), (int64_t)(l))			\
 )
 
-/* Register a linenumber (for debugging) */
+/* Register a linenumber (for debugging). */
 #define	LOGLINE(p)
 
 #define	NCR_SHOWECBS	0x01
@@ -228,24 +228,51 @@ struct ncr53c9x_tinfo {
 
 #ifdef NCR53C9X_DEBUG
 extern int ncr53c9x_debug;
-#define	NCR_ECBS(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWECBS) printf str;} while (0)
-#define	NCR_MISC(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWMISC) printf str;} while (0)
-#define	NCR_INTS(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWINTS) printf str;} while (0)
-#define	NCR_TRACE(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWTRAC) printf str;} while (0)
-#define	NCR_CMDS(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWCMDS) printf str;} while (0)
-#define	NCR_START(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWSTART) printf str;}while (0)
-#define	NCR_PHASE(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWPHASE) printf str;}while (0)
-#define	NCR_DMA(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWDMA) printf str;}while (0)
-#define	NCR_MSGS(str)	\
-	do {if (ncr53c9x_debug & NCR_SHOWMSGS) printf str;}while (0)
+#define	NCR_ECBS(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWECBS) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_MISC(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWMISC) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_INTS(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWINTS) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_TRACE(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWTRAC) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_CMDS(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWCMDS) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_START(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWSTART) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_PHASE(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWPHASE) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_DMA(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWDMA) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
+#define	NCR_MSGS(str)							\
+	do {								\
+		if ((ncr53c9x_debug & NCR_SHOWMSGS) != 0)		\
+			printf str;					\
+	} while (/* CONSTCOND */0)
 #else
 #define	NCR_ECBS(str)
 #define	NCR_MISC(str)
@@ -267,13 +294,13 @@ struct ncr53c9x_softc;
  */
 struct ncr53c9x_glue {
 	/* Mandatory entry points. */
-	u_char	(*gl_read_reg)(struct ncr53c9x_softc *, int);
-	void	(*gl_write_reg)(struct ncr53c9x_softc *, int, u_char);
+	uint8_t	(*gl_read_reg)(struct ncr53c9x_softc *, int);
+	void	(*gl_write_reg)(struct ncr53c9x_softc *, int, uint8_t);
 	int	(*gl_dma_isintr)(struct ncr53c9x_softc *);
 	void	(*gl_dma_reset)(struct ncr53c9x_softc *);
 	int	(*gl_dma_intr)(struct ncr53c9x_softc *);
-	int	(*gl_dma_setup)(struct ncr53c9x_softc *,
-		    caddr_t *, size_t *, int, size_t *);
+	int	(*gl_dma_setup)(struct ncr53c9x_softc *, void **, size_t *,
+		    int, size_t *);
 	void	(*gl_dma_go)(struct ncr53c9x_softc *);
 	void	(*gl_dma_stop)(struct ncr53c9x_softc *);
 	int	(*gl_dma_isactive)(struct ncr53c9x_softc *);
@@ -294,21 +321,21 @@ struct ncr53c9x_softc {
 	int	sc_cfflags;			/* Copy of config flags */
 
 	/* register defaults */
-	u_char	sc_cfg1;			/* Config 1 */
-	u_char	sc_cfg2;			/* Config 2, not ESP100 */
-	u_char	sc_cfg3;			/* Config 3, ESP200,FAS */
-	u_char	sc_cfg3_fscsi;			/* Chip-specific FSCSI bit */
-	u_char	sc_cfg4;			/* Config 4, only ESP200 */
-	u_char	sc_cfg5;			/* Config 5, only ESP200 */
-	u_char	sc_ccf;				/* Clock Conversion */
-	u_char	sc_timeout;
+	uint8_t	sc_cfg1;			/* Config 1 */
+	uint8_t	sc_cfg2;			/* Config 2, not ESP100 */
+	uint8_t	sc_cfg3;			/* Config 3, ESP200,FAS */
+	uint8_t	sc_cfg3_fscsi;			/* Chip-specific FSCSI bit */
+	uint8_t	sc_cfg4;			/* Config 4, only ESP200 */
+	uint8_t	sc_cfg5;			/* Config 5, only ESP200 */
+	uint8_t	sc_ccf;				/* Clock Conversion */
+	uint8_t	sc_timeout;
 
 	/* register copies, see espreadregs() */
-	u_char	sc_espintr;
-	u_char	sc_espstat;
-	u_char	sc_espstep;
-	u_char	sc_espstat2;
-	u_char	sc_espfflags;
+	uint8_t	sc_espintr;
+	uint8_t	sc_espstat;
+	uint8_t	sc_espstep;
+	uint8_t	sc_espstat2;
+	uint8_t	sc_espfflags;
 
 	/* Lists of command blocks */
 	TAILQ_HEAD(ecb_list, ncr53c9x_ecb) ready_list;
@@ -318,33 +345,33 @@ struct ncr53c9x_softc {
 	struct ncr53c9x_tinfo *sc_tinfo;
 
 	/* Data about the current nexus (updated for every cmd switch) */
-	caddr_t	sc_dp;		/* Current data pointer */
+	void	*sc_dp;		/* Current data pointer */
 	ssize_t	sc_dleft;	/* Data left to transfer */
 
 	/* Adapter state */
 	int	sc_phase;	/* Copy of what bus phase we are in */
 	int	sc_prevphase;	/* Copy of what bus phase we were in */
-	u_char	sc_state;	/* State applicable to the adapter */
-	u_char	sc_flags;	/* See below */
-	u_char	sc_selid;
-	u_char	sc_lastcmd;
+	uint8_t	sc_state;	/* State applicable to the adapter */
+	uint8_t	sc_flags;	/* See below */
+	uint8_t	sc_selid;
+	uint8_t	sc_lastcmd;
 
 	/* Message stuff */
-	u_short sc_msgify;	/* IDENTIFY message associated with nexus */
-	u_short	sc_msgout;	/* What message is on its way out? */
-	u_short	sc_msgpriq;	/* One or more messages to send (encoded) */
-	u_short	sc_msgoutq;	/* What messages have been sent so far? */
+	uint16_t sc_msgify;	/* IDENTIFY message associated with nexus */
+	uint16_t sc_msgout;	/* What message is on its way out? */
+	uint16_t sc_msgpriq;	/* One or more messages to send (encoded) */
+	uint16_t sc_msgoutq;	/* What messages have been sent so far? */
 
-	u_char	*sc_omess;	/* MSGOUT buffer */
+	uint8_t	*sc_omess;	/* MSGOUT buffer */
 	int	sc_omess_self;	/* MSGOUT buffer is self-allocated */
-	caddr_t	sc_omp;		/* Message pointer (for multibyte messages) */
+	void	*sc_omp;	/* Message pointer (for multibyte messages) */
 	size_t	sc_omlen;
-	u_char	*sc_imess;	/* MSGIN buffer */
+	uint8_t	*sc_imess;	/* MSGIN buffer */
 	int	sc_imess_self;	/* MSGIN buffer is self-allocated */
-	caddr_t	sc_imp;		/* Message pointer (for multibyte messages) */
+	void	*sc_imp;	/* Message pointer (for multibyte messages) */
 	size_t	sc_imlen;
 
-	caddr_t	sc_cmdp;	/* Command pointer (for DMAed commands) */
+	void	*sc_cmdp;	/* Command pointer (for DMAed commands) */
 	size_t	sc_cmdlen;	/* Size of command in transit */
 
 	/* Hardware attributes */
@@ -434,10 +461,10 @@ struct ncr53c9x_softc {
 #ifdef NCR53C9X_DEBUG
 #define	NCRCMD(sc, cmd) do {						\
 	if ((ncr53c9x_debug & NCR_SHOWCCMDS) != 0)			\
-		printf("<CMD:0x%x %d>", (unsigned)cmd, __LINE__);	\
+		printf("<CMD:0x%x %d>", (unsigned int)cmd, __LINE__);	\
 	sc->sc_lastcmd = cmd;						\
 	NCR_WRITE_REG(sc, NCR_CMD, cmd);				\
-} while (0)
+} while (/* CONSTCOND */ 0)
 #else
 #define	NCRCMD(sc, cmd)		NCR_WRITE_REG(sc, NCR_CMD, cmd)
 #endif
diff --git a/sys/dev/et/if_et.c b/sys/dev/et/if_et.c
index 9bd68e84a472..539224a4b291 100644
--- a/sys/dev/et/if_et.c
+++ b/sys/dev/et/if_et.c
@@ -518,9 +518,7 @@ et_ifmedia_upd_locked(struct ifnet *ifp)
 
 	LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
 		PHY_RESET(miisc);
-	mii_mediachg(mii);
-
-	return (0);
+	return (mii_mediachg(mii));
 }
 
 static int
@@ -542,9 +540,11 @@ et_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 	struct et_softc *sc = ifp->if_softc;
 	struct mii_data *mii = device_get_softc(sc->sc_miibus);
 
+	ET_LOCK(sc);
 	mii_pollstat(mii);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	ET_UNLOCK(sc);
 }
 
 static void
diff --git a/sys/dev/fb/machfb.c b/sys/dev/fb/machfb.c
index dad41d86ba75..1c3023de4d56 100644
--- a/sys/dev/fb/machfb.c
+++ b/sys/dev/fb/machfb.c
@@ -87,7 +87,9 @@ struct machfb_softc {
 	struct resource		*sc_vmemres;
 	bus_space_tag_t		sc_memt;
 	bus_space_tag_t		sc_regt;
+	bus_space_tag_t		sc_vmemt;
 	bus_space_handle_t	sc_memh;
+	bus_space_handle_t	sc_vmemh;
 	bus_space_handle_t	sc_regh;
 	u_long			sc_mem;
 	u_long			sc_vmem;
@@ -1175,68 +1177,52 @@ machfb_pci_attach(device_t dev)
 	adp = &sc->sc_va;
 	vi = &adp->va_info;
 
-	/*
-	 * Allocate resources regardless of whether we are the console
-	 * and already obtained the bus tag and handle for the framebuffer
-	 * in machfb_configure() or not so the resources are marked as
-	 * taken in the respective RMAN.
-	 */
-
-	/* Enable memory and IO access. */
-	pci_write_config(dev, PCIR_COMMAND,
-	    pci_read_config(dev, PCIR_COMMAND, 2) | PCIM_CMD_PORTEN |
-	    PCIM_CMD_MEMEN, 2);
-
-	/*
-	 * NB: we need to take care that the framebuffer isn't mapped
-	 * in twice as besides wasting resources this isn't possible with
-	 * all MMUs.
-	 */
 	rid = PCIR_BAR(0);
-	if ((sc->sc_memres = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
-	    &rid, 0)) == NULL) {
+	if ((sc->sc_memres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	    RF_ACTIVE)) == NULL) {
 		device_printf(dev, "cannot allocate memory resources\n");
 		return (ENXIO);
 	}
+	sc->sc_memt = rman_get_bustag(sc->sc_memres);
+	sc->sc_memh = rman_get_bushandle(sc->sc_memres);
+	sc->sc_mem = rman_get_start(sc->sc_memres);
+	vi->vi_buffer = sc->sc_memh;
+	vi->vi_buffer_size = rman_get_size(sc->sc_memres);
 	if (OF_getprop(sc->sc_node, "address", &u32, sizeof(u32)) > 0 &&
-		vtophys(u32) == rman_get_bushandle(sc->sc_memres))
+		vtophys(u32) == sc->sc_memh)
 		adp->va_mem_base = u32;
 	else {
-		bus_release_resource(dev, SYS_RES_MEMORY,
-		    rman_get_rid(sc->sc_memres), sc->sc_memres);
-		rid = PCIR_BAR(0);
-		if ((sc->sc_memres = bus_alloc_resource_any(dev,
-		    SYS_RES_MEMORY, &rid, RF_ACTIVE)) == NULL) {
-			device_printf(dev,
-			    "cannot allocate memory resources\n");
-			return (ENXIO);
+		if (bus_space_map(sc->sc_memt, vi->vi_buffer,
+		    vi->vi_buffer_size, BUS_SPACE_MAP_LINEAR,
+		    &sc->sc_memh) != 0) {
+			device_printf(dev, "cannot map memory resources\n");
+			error = ENXIO;
+			goto fail_memres;
 		}
 		adp->va_mem_base =
 		    (vm_offset_t)rman_get_virtual(sc->sc_memres);
 	}
-	sc->sc_memt = rman_get_bustag(sc->sc_memres);
-	sc->sc_memh = rman_get_bushandle(sc->sc_memres);
-	sc->sc_regt = sc->sc_memt;
-	bus_space_subregion(sc->sc_regt, sc->sc_memh, MACH64_REG_OFF,
-	    MACH64_REG_SIZE, &sc->sc_regh);
-	adp->va_mem_size = rman_get_size(sc->sc_memres);
+	adp->va_mem_size = vi->vi_buffer_size;
 	adp->va_buffer = adp->va_mem_base;
 	adp->va_buffer_size = adp->va_mem_size;
-	sc->sc_mem = rman_get_start(sc->sc_memres);
-	vi->vi_buffer = sc->sc_memh;
-	vi->vi_buffer_size = adp->va_buffer_size;
+	sc->sc_regt = sc->sc_memt;
+	if (bus_space_subregion(sc->sc_regt, sc->sc_memh, MACH64_REG_OFF,
+	    MACH64_REG_SIZE, &sc->sc_regh) != 0) {
+		device_printf(dev, "cannot allocate register resources\n");
+		error = ENXIO;
+		goto fail_memmap;
+	}
 
 	/*
 	 * Depending on the firmware version the VGA I/O and/or memory
-	 * resources of the Mach64 chips come up disabled.  We generally
-	 * enable them above (pci(4) actually already did this unless
-	 * pci_enable_io_modes is not set) but this doesn't necessarily
-	 * mean that we get valid ones.  Invalid resources seem to have
-	 * in common that they start at address 0.  We don't allocate
-	 * them in this case in order to avoid warnings in apb(4) and
-	 * crashes when using these invalid resources.  X.Org is aware
-	 * of this and doesn't use the VGA resources in this case (but
-	 * demands them if they are valid).
+	 * resources of the Mach64 chips come up disabled.  These will be
+	 * enabled by pci(4) when activating the resource in question but
+	 * this doesn't necessarily mean that the resource is valid.
+	 * Invalid resources seem to have in common that they start at
+	 * address 0.  We don't allocate the VGA memory in this case in
+	 * order to avoid warnings in apb(4) and crashes when using this
+	 * invalid resources.  X.Org is aware of this and doesn't use the
+	 * VGA memory resource in this case (but demands it if it's valid).
 	 */
 	rid = PCIR_BAR(2);
 	if (bus_get_resource_start(dev, SYS_RES_MEMORY, rid) != 0) {
@@ -1245,21 +1231,31 @@ machfb_pci_attach(device_t dev)
 			device_printf(dev,
 			    "cannot allocate VGA memory resources\n");
 			error = ENXIO;
-			goto fail_memres;
+			goto fail_memmap;
+		}
+		sc->sc_vmemt = rman_get_bustag(sc->sc_vmemres);
+		sc->sc_vmemh = rman_get_bushandle(sc->sc_vmemres);
+		sc->sc_vmem = rman_get_start(sc->sc_vmemres);
+		vi->vi_registers = sc->sc_vmemh;
+		vi->vi_registers_size = rman_get_size(sc->sc_vmemres);
+		if (bus_space_map(sc->sc_vmemt, vi->vi_registers,
+		    vi->vi_registers_size, BUS_SPACE_MAP_LINEAR,
+		    &sc->sc_vmemh) != 0) {
+			device_printf(dev,
+			    "cannot map VGA memory resources\n");
+			error = ENXIO;
+			goto fail_vmemres;
 		}
 		adp->va_registers =
 		    (vm_offset_t)rman_get_virtual(sc->sc_vmemres);
-		adp->va_registers_size = rman_get_size(sc->sc_vmemres);
-		sc->sc_vmem = rman_get_start(sc->sc_vmemres);
-		vi->vi_registers = rman_get_bushandle(sc->sc_vmemres);
-		vi->vi_registers_size = adp->va_registers_size;
+		adp->va_registers_size = vi->vi_registers_size;
 	}
 
 	if (!(sc->sc_flags & MACHFB_CONSOLE)) {
 		if ((sw = vid_get_switch(MACHFB_DRIVER_NAME)) == NULL) {
 			device_printf(dev, "cannot get video switch\n");
 			error = ENODEV;
-			goto fail_vmemres;
+			goto fail_vmemmap;
 		}
 		/*
 		 * During device configuration we don't necessarily probe
@@ -1275,7 +1271,7 @@ machfb_pci_attach(device_t dev)
 				break;
 		if ((error = sw->init(i, adp, 0)) != 0) {
 			device_printf(dev, "cannot initialize adapter\n");
-			goto fail_vmemres;
+			goto fail_vmemmap;
 		}
 	}
 
@@ -1283,8 +1279,8 @@ machfb_pci_attach(device_t dev)
 	 * Test whether the aperture is byte swapped or not, set
 	 * va_window and va_window_size as appropriate.  Note that
 	 * the aperture could be mapped either big or little endian
-	 * on independently of the endianess of the host so this
-	 * has to be a runtime test.
+	 * independently of the endianess of the host so this has
+	 * to be a runtime test.
 	 */
 	p32 = (uint32_t *)adp->va_buffer;
 	u32 = *p32;
@@ -1346,10 +1342,16 @@ machfb_pci_attach(device_t dev)
 
 	return (0);
 
+ fail_vmemmap:
+	if (adp->va_registers != 0)
+		bus_space_unmap(sc->sc_vmemt, sc->sc_vmemh,
+		    vi->vi_registers_size);
  fail_vmemres:
 	if (sc->sc_vmemres != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(sc->sc_vmemres), sc->sc_vmemres);
+ fail_memmap:
+	bus_space_unmap(sc->sc_memt, sc->sc_memh, vi->vi_buffer_size);
  fail_memres:
 	bus_release_resource(dev, SYS_RES_MEMORY,
 	    rman_get_rid(sc->sc_memres), sc->sc_memres);
diff --git a/sys/dev/fdt/fdt_mips.c b/sys/dev/fdt/fdt_mips.c
new file mode 100644
index 000000000000..0e6828eceb07
--- /dev/null
+++ b/sys/dev/fdt/fdt_mips.c
@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2009-2010 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Semihalf under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+
+#include <machine/intr_machdep.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
+
+#include "ofw_bus_if.h"
+#include "fdt_common.h"
+
+struct fdt_fixup_entry fdt_fixup_table[] = {
+	{ NULL, NULL }
+};
+
+fdt_pic_decode_t fdt_pic_table[] = {
+	NULL,
+	NULL,
+	NULL
+};
diff --git a/sys/dev/fdt/fdtbus.c b/sys/dev/fdt/fdtbus.c
index 86a9d176f9fd..a2530cd421d9 100644
--- a/sys/dev/fdt/fdtbus.c
+++ b/sys/dev/fdt/fdtbus.c
@@ -591,6 +591,9 @@ fdtbus_setup_intr(device_t bus, device_t child, struct resource *res,
 #if defined(__powerpc__)
 	err = powerpc_setup_intr(device_get_nameunit(child),
 	    rman_get_start(res), filter, ihand, arg, flags, cookiep);
+#elif defined(__mips__)
+	cpu_establish_hardintr(device_get_nameunit(child), 
+		filter, ihand, arg, rman_get_start(res), flags, cookiep);
 #elif defined(__arm__)
 	arm_setup_irqhandler(device_get_nameunit(child),
 	    filter, ihand, arg, rman_get_start(res), flags, cookiep);
@@ -617,7 +620,6 @@ fdtbus_deactivate_resource(device_t bus, device_t child, int type, int rid,
 	return (rman_deactivate_resource(res));
 }
 
-
 static int
 fdtbus_teardown_intr(device_t bus, device_t child, struct resource *res,
     void *cookie)
@@ -625,6 +627,9 @@ fdtbus_teardown_intr(device_t bus, device_t child, struct resource *res,
 
 #if defined(__powerpc__)
 	return (powerpc_teardown_intr(cookie));
+#elif defined(__mips__)
+	/* mips does not have a teardown yet */
+	return (0);
 #elif defined(__arm__)
 	return (arm_remove_irqhandler(rman_get_start(res), cookie));
 #endif
diff --git a/sys/dev/firewire/sbp.c b/sys/dev/firewire/sbp.c
index 0ac7d142b4b6..2387bf944991 100644
--- a/sys/dev/firewire/sbp.c
+++ b/sys/dev/firewire/sbp.c
@@ -1515,10 +1515,10 @@ static void
 sbp_scsi_status(struct sbp_status *sbp_status, struct sbp_ocb *ocb)
 {
 	struct sbp_cmd_status *sbp_cmd_status;
-	struct scsi_sense_data *sense;
+	struct scsi_sense_data_fixed *sense;
 
 	sbp_cmd_status = (struct sbp_cmd_status *)sbp_status->data;
-	sense = &ocb->ccb->csio.sense_data;
+	sense = (struct scsi_sense_data_fixed *)&ocb->ccb->csio.sense_data;
 
 SBP_DEBUG(0)
 	sbp_print_scsi_cmd(ocb);
diff --git a/sys/dev/firewire/sbp_targ.c b/sys/dev/firewire/sbp_targ.c
index c5b9d2312cac..f04b968b9a18 100644
--- a/sys/dev/firewire/sbp_targ.c
+++ b/sys/dev/firewire/sbp_targ.c
@@ -41,6 +41,7 @@
 #include <sys/types.h>
 #include <sys/conf.h>
 #include <sys/malloc.h>
+#include <sys/endian.h>
 #if __FreeBSD_version < 500000
 #include <sys/devicestat.h>
 #endif
@@ -632,6 +633,12 @@ sbp_targ_send_status(struct orb_info *orbi, union ccb *ccb)
 	{
 		struct sbp_cmd_status *sbp_cmd_status;
 		struct scsi_sense_data *sense;
+		int error_code, sense_key, asc, ascq;
+		uint8_t stream_bits;
+		uint8_t sks[3];
+		uint64_t info;
+		int64_t sinfo;
+		int sense_len;
 
 		if (debug)
 			printf("%s: STATUS %d\n", __func__,
@@ -659,35 +666,75 @@ sbp_targ_send_status(struct orb_info *orbi, union ccb *ccb)
 #endif
 #endif
 
-		if ((sense->error_code & SSD_ERRCODE) == SSD_CURRENT_ERROR)
+		sense_len = ccb->csio.sense_len - ccb->csio.sense_resid;
+		scsi_extract_sense_len(sense, sense_len, &error_code,
+		    &sense_key, &asc, &ascq, /*show_errors*/ 0);
+
+		switch (error_code) {
+		case SSD_CURRENT_ERROR:
+		case SSD_DESC_CURRENT_ERROR:
 			sbp_cmd_status->sfmt = SBP_SFMT_CURR;
-		else
+			break;
+		default:
 			sbp_cmd_status->sfmt = SBP_SFMT_DEFER;
+			break;
+		}
 
-		sbp_cmd_status->valid = (sense->error_code & SSD_ERRCODE_VALID)
-		    ? 1 : 0;
-		sbp_cmd_status->s_key = sense->flags & SSD_KEY;
-		sbp_cmd_status->mark = (sense->flags & SSD_FILEMARK)? 1 : 0;
-		sbp_cmd_status->eom = (sense->flags & SSD_EOM) ? 1 : 0;
-		sbp_cmd_status->ill_len = (sense->flags & SSD_ILI) ? 1 : 0;
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &info,
+					&sinfo) == 0) {
+			uint32_t info_trunc;
+			sbp_cmd_status->valid = 1;
+			info_trunc = info;
 
-		bcopy(&sense->info[0], &sbp_cmd_status->info, 4);
+			sbp_cmd_status->info = htobe32(info_trunc);
+		} else {
+			sbp_cmd_status->valid = 0;
+		}
 
-		if (sense->extra_len <= 6)
-			/* add_sense_code(_qual), info, cmd_spec_info */
-			sbp_status->len = 4;
-		else
-			/* fru, sense_key_spec */
+		sbp_cmd_status->s_key = sense_key;
+
+		if (scsi_get_stream_info(sense, sense_len, NULL,
+					 &stream_bits) == 0) {
+			sbp_cmd_status->mark =
+			    (stream_bits & SSD_FILEMARK) ? 1 : 0;
+			sbp_cmd_status->eom =
+			    (stream_bits & SSD_EOM) ? 1 : 0;
+			sbp_cmd_status->ill_len =
+			    (stream_bits & SSD_ILI) ? 1 : 0;
+		} else {
+			sbp_cmd_status->mark = 0;
+			sbp_cmd_status->eom = 0;
+			sbp_cmd_status->ill_len = 0;
+		}
+
+
+		/* add_sense_code(_qual), info, cmd_spec_info */
+		sbp_status->len = 4;
+
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_COMMAND,
+					&info, &sinfo) == 0) {
+			uint32_t cmdspec_trunc;
+
+			cmdspec_trunc = info;
+
+			sbp_cmd_status->cdb = htobe32(cmdspec_trunc);
+		}
+
+		sbp_cmd_status->s_code = asc;
+		sbp_cmd_status->s_qlfr = ascq;
+
+		if (scsi_get_sense_info(sense, sense_len, SSD_DESC_FRU, &info,
+					&sinfo) == 0) {
+			sbp_cmd_status->fru = (uint8_t)info;
 			sbp_status->len = 5;
-			
-		bcopy(&sense->cmd_spec_info[0], &sbp_cmd_status->cdb, 4);
+		} else {
+			sbp_cmd_status->fru = 0;
+		}
 
-		sbp_cmd_status->s_code = sense->add_sense_code;
-		sbp_cmd_status->s_qlfr = sense->add_sense_code_qual;
-		sbp_cmd_status->fru = sense->fru;
-
-		bcopy(&sense->sense_key_spec[0],
-		    &sbp_cmd_status->s_keydep[0], 3);
+		if (scsi_get_sks(sense, sense_len, sks) == 0) {
+			bcopy(sks, &sbp_cmd_status->s_keydep[0], sizeof(sks));
+			sbp_status->len = 5;
+		}
 
 		break;
 	}
diff --git a/sys/dev/mii/inphy.c b/sys/dev/fxp/inphy.c
similarity index 99%
rename from sys/dev/mii/inphy.c
rename to sys/dev/fxp/inphy.c
index e6a601cbfa8c..587f3fd62555 100644
--- a/sys/dev/mii/inphy.c
+++ b/sys/dev/fxp/inphy.c
@@ -49,7 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mii/miivar.h>
 #include "miidevs.h"
 
-#include <dev/mii/inphyreg.h>
+#include <dev/fxp/inphyreg.h>
 
 #include "miibus_if.h"
 
diff --git a/sys/dev/mii/inphyreg.h b/sys/dev/fxp/inphyreg.h
similarity index 86%
rename from sys/dev/mii/inphyreg.h
rename to sys/dev/fxp/inphyreg.h
index 4b7d1f113051..42c683dbf189 100644
--- a/sys/dev/mii/inphyreg.h
+++ b/sys/dev/fxp/inphyreg.h
@@ -29,7 +29,12 @@
  *	$FreeBSD$
  */
 
-#define MII_INPHY_SCR           0x10    /* status and control register */
-#define SCR_FLOWCTL		0x8000
-#define SCR_S100		0x0002	/* autonegotiated speed */
-#define SCR_FDX			0x0001	/* autonegotiated duplex */
+#ifndef _INPHYREG_H
+#define	_INPHYREG_H
+
+#define	MII_INPHY_SCR		0x10	/* status and control register */
+#define	SCR_FLOWCTL		0x8000
+#define	SCR_S100		0x0002	/* autonegotiated speed */
+#define	SCR_FDX			0x0001	/* autonegotiated duplex */
+
+#endif
diff --git a/sys/dev/gpio/gpioc.c b/sys/dev/gpio/gpioc.c
index 6a4c0c91938d..d48743b2439f 100644
--- a/sys/dev/gpio/gpioc.c
+++ b/sys/dev/gpio/gpioc.c
@@ -61,9 +61,6 @@ static struct cdevsw gpioc_cdevsw = {
 	.d_version	= D_VERSION,
 	.d_ioctl	= gpioc_ioctl,
 	.d_name		= "gpioc",
-#if __FreeBSD_version >= 800039
-	.d_flags	= D_PSEUDO | D_NEEDMINOR
-#endif
 };
 
 struct gpioc_softc {
diff --git a/sys/dev/hptiop/hptiop.c b/sys/dev/hptiop/hptiop.c
index 49ffcefe6c57..c4e242749681 100644
--- a/sys/dev/hptiop/hptiop.c
+++ b/sys/dev/hptiop/hptiop.c
@@ -424,6 +424,13 @@ static void hptiop_request_callback_itl(struct hpt_iop_hba * hba,
 			ccb->ccb_h.status = CAM_BUSY;
 			break;
 		case IOP_RESULT_CHECK_CONDITION:
+			memset(&ccb->csio.sense_data, 0,
+			    sizeof(ccb->csio.sense_data));
+			if (dxfer < ccb->csio.sense_len)
+				ccb->csio.sense_resid = ccb->csio.sense_len -
+				    dxfer;
+			else
+				ccb->csio.sense_resid = 0;
 			if (srb->srb_flag & HPT_SRB_FLAG_HIGH_MEM_ACESS) {/*iop*/
 				bus_space_read_region_1(hba->bar0t, hba->bar0h,
 					index + offsetof(struct hpt_iop_request_scsi_command,
@@ -573,6 +580,13 @@ static void hptiop_request_callback_mv(struct hpt_iop_hba * hba,
 			ccb->ccb_h.status = CAM_BUSY;
 			break;
 		case IOP_RESULT_CHECK_CONDITION:
+			memset(&ccb->csio.sense_data, 0,
+			    sizeof(ccb->csio.sense_data));
+			if (req->dataxfer_length < ccb->csio.sense_len)
+				ccb->csio.sense_resid = ccb->csio.sense_len -
+				    req->dataxfer_length;
+			else
+				ccb->csio.sense_resid = 0;
 			memcpy(&ccb->csio.sense_data, &req->sg_list, 
 				MIN(req->dataxfer_length, sizeof(ccb->csio.sense_data)));
 			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
diff --git a/sys/dev/hwpmc/hwpmc_logging.c b/sys/dev/hwpmc/hwpmc_logging.c
index b85572a3c86f..5f5cd90a3c8f 100644
--- a/sys/dev/hwpmc/hwpmc_logging.c
+++ b/sys/dev/hwpmc/hwpmc_logging.c
@@ -238,7 +238,7 @@ pmclog_get_buffer(struct pmc_owner *po)
 static void
 pmclog_loop(void *arg)
 {
-	int error, last_buffer;
+	int error;
 	struct pmc_owner *po;
 	struct pmclog_buffer *lb;
 	struct proc *p;
@@ -253,7 +253,6 @@ pmclog_loop(void *arg)
 	p = po->po_owner;
 	td = curthread;
 	mycred = td->td_ucred;
-	last_buffer = 0;
 
 	PROC_LOCK(p);
 	ownercred = crhold(p->p_ucred);
@@ -286,14 +285,22 @@ pmclog_loop(void *arg)
 			if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
 				mtx_unlock_spin(&po->po_mtx);
 
+				if (po->po_flags & PMC_PO_SHUTDOWN) {
+					mtx_unlock(&pmc_kthread_mtx);
+					/*
+			 		 * Close the file to get PMCLOG_EOF
+					 * error in pmclog(3).
+					 */
+					fo_close(po->po_file, curthread);
+					mtx_lock(&pmc_kthread_mtx);
+				}
+
 				(void) msleep(po, &pmc_kthread_mtx, PWAIT,
 				    "pmcloop", 0);
 				continue;
 			}
 
 			TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
-			if (po->po_flags & PMC_PO_SHUTDOWN)
-				last_buffer = TAILQ_EMPTY(&po->po_logbuffers);
 			mtx_unlock_spin(&po->po_mtx);
 		}
 
@@ -324,7 +331,7 @@ pmclog_loop(void *arg)
 			/* XXX some errors are recoverable */
 			/* send a SIGIO to the owner and exit */
 			PROC_LOCK(p);
-			psignal(p, SIGIO);
+			kern_psignal(p, SIGIO);
 			PROC_UNLOCK(p);
 
 			mtx_lock(&pmc_kthread_mtx);
@@ -336,14 +343,6 @@ pmclog_loop(void *arg)
 			break;
 		}
 
-		if (last_buffer) {
-			/*
-			 * Close the file to get PMCLOG_EOF error
-			 * in pmclog(3).
-			 */
-			fo_close(po->po_file, curthread);
-		}
-
 		mtx_lock(&pmc_kthread_mtx);
 
 		/* put the used buffer back into the global pool */
@@ -693,6 +692,7 @@ int
 pmclog_flush(struct pmc_owner *po)
 {
 	int error;
+	struct pmclog_buffer *lb;
 
 	PMCDBG(LOG,FLS,1, "po=%p", po);
 
@@ -715,11 +715,38 @@ pmclog_flush(struct pmc_owner *po)
 	}
 
 	/*
-	 * Schedule the current buffer if any.
+	 * Schedule the current buffer if any and not empty.
+	 */
+	mtx_lock_spin(&po->po_mtx);
+	lb = po->po_curbuf;
+	if (lb && lb->plb_ptr != lb->plb_base) {
+		pmclog_schedule_io(po);
+	} else
+		error = ENOBUFS;
+	mtx_unlock_spin(&po->po_mtx);
+
+ error:
+	mtx_unlock(&pmc_kthread_mtx);
+
+	return (error);
+}
+
+int
+pmclog_close(struct pmc_owner *po)
+{
+
+	PMCDBG(LOG,CLO,1, "po=%p", po);
+
+	mtx_lock(&pmc_kthread_mtx);
+
+	/*
+	 * Schedule the current buffer.
 	 */
 	mtx_lock_spin(&po->po_mtx);
 	if (po->po_curbuf)
 		pmclog_schedule_io(po);
+	else
+		wakeup_one(po);
 	mtx_unlock_spin(&po->po_mtx);
 
 	/*
@@ -728,13 +755,11 @@ pmclog_flush(struct pmc_owner *po)
 	 */
 	po->po_flags |= PMC_PO_SHUTDOWN;
 
- error:
 	mtx_unlock(&pmc_kthread_mtx);
 
-	return (error);
+	return (0);
 }
 
-
 void
 pmclog_process_callchain(struct pmc *pm, struct pmc_sample *ps)
 {
diff --git a/sys/dev/hwpmc/hwpmc_mips24k.h b/sys/dev/hwpmc/hwpmc_mips24k.h
index 5944e45f5a21..876bbe87c5c9 100644
--- a/sys/dev/hwpmc/hwpmc_mips24k.h
+++ b/sys/dev/hwpmc/hwpmc_mips24k.h
@@ -35,7 +35,7 @@
 				 PMC_CAP_WRITE | PMC_CAP_INVERT |	\
 				 PMC_CAP_QUALIFIER)
 
-
+#define MIPS24K_PMC_INTERRUPT_ENABLE      0x10 /* Enable interrupts */
 #define MIPS24K_PMC_USER_ENABLE           0x08 /* Count in USER mode */
 #define MIPS24K_PMC_SUPER_ENABLE          0x04 /* Count in SUPERVISOR mode */
 #define MIPS24K_PMC_KERNEL_ENABLE         0x02 /* Count in KERNEL mode */
@@ -43,9 +43,12 @@
 			    MIPS24K_PMC_SUPER_ENABLE |	   \
 			    MIPS24K_PMC_KERNEL_ENABLE)
 
-
-#define	MIPS24K_RELOAD_COUNT_TO_PERFCTR_VALUE(R)	(-(R))
-#define	MIPS24K_PERFCTR_VALUE_TO_RELOAD_COUNT(P)	(-(P))
+/*
+ * Interrupts are posted when bit 31 of the relevant
+ * counter is set.
+ */
+#define	MIPS24K_RELOAD_COUNT_TO_PERFCTR_VALUE(R)	(0x80000000 - (R))
+#define	MIPS24K_PERFCTR_VALUE_TO_RELOAD_COUNT(P)	((P) - 0x80000000)
 
 #define MIPS24K_PMC_SELECT 0x4 /* Which bit position the event starts at. */
 #define MIPS24K_PMC_OFFSET 2   /* Control registers are 0, 2, 4, etc. */
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
index 4cfcea87fa5b..7ca7a47e12c4 100644
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -889,7 +889,7 @@ pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 	if (LIST_EMPTY(&pm->pm_targets)) {
 		p = pm->pm_owner->po_owner;
 		PROC_LOCK(p);
-		psignal(p, SIGIO);
+		kern_psignal(p, SIGIO);
 		PROC_UNLOCK(p);
 
 		PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
@@ -2891,7 +2891,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 			error = pmclog_configure_log(md, po, cl.pm_logfd);
 		} else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
 			pmclog_process_closelog(po);
-			error = pmclog_flush(po);
+			error = pmclog_close(po);
 			if (error == 0) {
 				LIST_FOREACH(pm, &po->po_pmcs, pm_next)
 				    if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
@@ -2907,7 +2907,6 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 	}
 	break;
 
-
 	/*
 	 * Flush a log file.
 	 */
@@ -2927,6 +2926,25 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 	}
 	break;
 
+	/*
+	 * Close a log file.
+	 */
+
+	case PMC_OP_CLOSELOG:
+	{
+		struct pmc_owner *po;
+
+		sx_assert(&pmc_sx, SX_XLOCKED);
+
+		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
+			error = EINVAL;
+			break;
+		}
+
+		error = pmclog_close(po);
+	}
+	break;
+
 	/*
 	 * Retrieve hardware configuration.
 	 */
@@ -4812,7 +4830,7 @@ pmc_cleanup(void)
 				    po->po_owner->p_comm);
 
 				PROC_LOCK(po->po_owner);
-				psignal(po->po_owner, SIGBUS);
+				kern_psignal(po->po_owner, SIGBUS);
 				PROC_UNLOCK(po->po_owner);
 
 				pmc_destroy_owner_descriptor(po);
diff --git a/sys/dev/iicbus/iic.c b/sys/dev/iicbus/iic.c
index 673d635d02b0..d2515559e932 100644
--- a/sys/dev/iicbus/iic.c
+++ b/sys/dev/iicbus/iic.c
@@ -346,11 +346,11 @@ iicioctl(struct cdev *dev, u_long cmd, caddr_t data, int flags, struct thread *t
 
 	case I2CRDWR:
 		buf = malloc(sizeof(*d->msgs) * d->nmsgs, M_TEMP, M_WAITOK);
-		usrbufs = malloc(sizeof(void *) * d->nmsgs, M_TEMP, M_ZERO | M_WAITOK);
 		error = copyin(d->msgs, buf, sizeof(*d->msgs) * d->nmsgs);
 		if (error)
 			break;
 		/* Alloc kernel buffers for userland data, copyin write data */
+		usrbufs = malloc(sizeof(void *) * d->nmsgs, M_TEMP, M_ZERO | M_WAITOK);
 		for (i = 0; i < d->nmsgs; i++) {
 			m = &((struct iic_msg *)buf)[i];
 			usrbufs[i] = m->buf;
diff --git a/sys/dev/iir/iir.c b/sys/dev/iir/iir.c
index b0915c2000bf..2be7c39e87fc 100644
--- a/sys/dev/iir/iir.c
+++ b/sys/dev/iir/iir.c
@@ -1839,13 +1839,20 @@ gdt_sync_event(struct gdt_softc *gdt, int service,
         } else {
             /* error */
             if (gccb->gc_service == GDT_CACHESERVICE) {
+                struct scsi_sense_data *sense;
+
                 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
                 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
                 bzero(&ccb->csio.sense_data, ccb->csio.sense_len);
-                ccb->csio.sense_data.error_code =
-                    SSD_CURRENT_ERROR | SSD_ERRCODE_VALID;
-                ccb->csio.sense_data.flags = SSD_KEY_NOT_READY;
+                sense = &ccb->csio.sense_data;
+                scsi_set_sense_data(sense,
+                                    /*sense_format*/ SSD_TYPE_NONE,
+                                    /*current_error*/ 1,
+                                    /*sense_key*/ SSD_KEY_NOT_READY,
+                                    /*asc*/ 0x4,
+                                    /*ascq*/ 0x01,
+                                    SSD_ELEM_NONE);
 
                 gdt->sc_dvr.size = sizeof(gdt->sc_dvr.eu.sync);
                 gdt->sc_dvr.eu.sync.ionode  = gdt->sc_hanum;
diff --git a/sys/dev/iscsi/initiator/isc_soc.c b/sys/dev/iscsi/initiator/isc_soc.c
index 329f830b6694..7d0219bd8069 100644
--- a/sys/dev/iscsi/initiator/isc_soc.c
+++ b/sys/dev/iscsi/initiator/isc_soc.c
@@ -628,7 +628,7 @@ isc_in(void *vp)
 	    sp->flags, so->so_count, so->so_state, error, sp->proc);
      if((sp->proc != NULL) && sp->signal) {
 	  PROC_LOCK(sp->proc);
-	  psignal(sp->proc, sp->signal);
+	  kern_psignal(sp->proc, sp->signal);
 	  PROC_UNLOCK(sp->proc);
 	  sp->flags |= ISC_SIGNALED;
 	  sdebug(2, "pid=%d signaled(%d)", sp->proc->p_pid, sp->signal);
diff --git a/sys/dev/iscsi/initiator/iscsi_subr.c b/sys/dev/iscsi/initiator/iscsi_subr.c
index 03d9293e7e3c..939bafcf05d3 100644
--- a/sys/dev/iscsi/initiator/iscsi_subr.c
+++ b/sys/dev/iscsi/initiator/iscsi_subr.c
@@ -84,6 +84,7 @@ iscsi_r2t(isc_session_t *sp, pduq_t *opq, pduq_t *pq)
 	       caddr_t		bp = csio->data_ptr;
 
 	       bo = ntohl(r2t->bo);
+	       bp += MIN(bo, edtl - ddtl);
 	       bleft = ddtl;
 
 	       if(sp->opt.maxXmitDataSegmentLength > 0) // danny's RFC
@@ -153,6 +154,7 @@ getSenseData(u_int status, union ccb *ccb, pduq_t *pq)
      scsi_rsp_t		*cmd = &pp->ipdu.scsi_rsp;
      caddr_t		bp;
      int		sense_len, mustfree = 0;
+     int		error_code, sense_key, asc, ascq;
 
      bp = mtod(pq->mp, caddr_t);
      if((sense_len = scsi_2btoul(bp)) == 0)
@@ -174,10 +176,14 @@ getSenseData(u_int status, union ccb *ccb, pduq_t *pq)
      scsi->sense_resid = 0;
      if(cmd->flag & (BIT(1)|BIT(2)))
 	  scsi->sense_resid = ntohl(pp->ipdu.scsi_rsp.rcnt);
+
+     scsi_extract_sense_len(sense, scsi->sense_len - scsi->sense_resid,
+	&error_code, &sense_key, &asc, &ascq, /*show_errors*/ 1);
+
      debug(3, "sense_len=%d rcnt=%d sense_resid=%d dsl=%d error_code=%x flags=%x",
 	   sense_len,
 	   ntohl(pp->ipdu.scsi_rsp.rcnt), scsi->sense_resid,
-	   pp->ds_len, sense->error_code, sense->flags);
+	   pp->ds_len, error_code, sense_key);
 
      if(mustfree)
 	  free(bp, M_ISCSI);
diff --git a/sys/dev/isp/DriverManual.txt b/sys/dev/isp/DriverManual.txt
index ea1968f9f883..0ed5a1b08c4f 100644
--- a/sys/dev/isp/DriverManual.txt
+++ b/sys/dev/isp/DriverManual.txt
@@ -165,7 +165,7 @@ and saves the QLogic HBA from having to write a response queue entry).
 
 The QLogic HBA is an interrupting card, and when servicing an interrupt
 you really only have to check for either a mailbox interrupt or an
-interrupt notification that the the response queue has an entry to
+interrupt notification that the response queue has an entry to
 be dequeued.
 
 4.3 Fibre Channel SCSI out of SCSI
@@ -327,7 +327,7 @@ here in clarifying some of this.
 
 A succesful execution of isp_init will lead to the driver 'registering'
 itself with this platform's SCSI subsystem. One assumed action for this
-is the registry of a function the the SCSI subsystem for this platform
+is the registry of a function the SCSI subsystem for this platform
 will call when it has a SCSI command to run.
 
 The platform specific module function that receives this will do whatever
diff --git a/sys/dev/isp/isp_freebsd.h b/sys/dev/isp/isp_freebsd.h
index 1fb5fc56a89f..5d1b500f046c 100644
--- a/sys/dev/isp/isp_freebsd.h
+++ b/sys/dev/isp/isp_freebsd.h
@@ -438,11 +438,19 @@ default:							\
 #define	XS_SNSP(ccb)		(&(ccb)->sense_data)
 
 #define	XS_SNSLEN(ccb)		\
-	imin((sizeof((ccb)->sense_data)), ccb->sense_len)
+	imin((sizeof((ccb)->sense_data)), ccb->sense_len - ccb->sense_resid)
 
-#define	XS_SNSKEY(ccb)		((ccb)->sense_data.flags & 0xf)
-#define	XS_SNSASC(ccb)		((ccb)->sense_data.add_sense_code)
-#define	XS_SNSASCQ(ccb)		((ccb)->sense_data.add_sense_code_qual)
+#define	XS_SNSKEY(ccb)		(scsi_get_sense_key(&(ccb)->sense_data, \
+				 ccb->sense_len - ccb->sense_resid, 	\
+				 /*show_errors*/ 1))
+
+#define	XS_SNSASC(ccb)		(scsi_get_asc(&(ccb)->sense_data,	\
+				 ccb->sense_len - ccb->sense_resid, 	\
+				 /*show_errors*/ 1))
+
+#define	XS_SNSASCQ(ccb)		(scsi_get_ascq(&(ccb)->sense_data,	\
+				 ccb->sense_len - ccb->sense_resid, 	\
+				 /*show_errors*/ 1))
 #define	XS_TAG_P(ccb)	\
 	(((ccb)->ccb_h.flags & CAM_TAG_ACTION_VALID) && \
 	 (ccb)->tag_action != CAM_TAG_ACTION_NONE)
@@ -476,9 +484,14 @@ default:							\
 #define	XS_INITERR(ccb)		\
 	XS_SETERR(ccb, CAM_REQ_INPROG), (ccb)->ccb_h.spriv_field0 = 0
 
-#define	XS_SAVE_SENSE(xs, sense_ptr, sense_len)		\
-	(xs)->ccb_h.status |= CAM_AUTOSNS_VALID;	\
-	memcpy(&(xs)->sense_data, sense_ptr, imin(XS_SNSLEN(xs), sense_len))
+#define	XS_SAVE_SENSE(xs, sense_ptr, slen)	do {			\
+		(xs)->ccb_h.status |= CAM_AUTOSNS_VALID;		\
+		memset(&(xs)->sense_data, 0, sizeof(&(xs)->sense_data));\
+		memcpy(&(xs)->sense_data, sense_ptr, imin(XS_SNSLEN(xs),\
+		       slen)); 						\
+		if (slen < (xs)->sense_len) 				\
+			(xs)->sense_resid = (xs)->sense_len - slen;	\
+	} while (0);
 
 #define	XS_SENSE_VALID(xs)	(((xs)->ccb_h.status & CAM_AUTOSNS_VALID) != 0)
 
diff --git a/sys/dev/iwn/if_iwn.c b/sys/dev/iwn/if_iwn.c
index ab3dec722c5c..8a02752f30b4 100644
--- a/sys/dev/iwn/if_iwn.c
+++ b/sys/dev/iwn/if_iwn.c
@@ -89,6 +89,7 @@ static const struct iwn_ident iwn_ident_table[] = {
 	{ 0x8086, 0x008b, "Intel(R) Centrino(R) Wireless-N 1030"	 },
 	{ 0x8086, 0x0090, "Intel(R) Centrino(R) Advanced-N 6230"	 },
 	{ 0x8086, 0x0091, "Intel(R) Centrino(R) Advanced-N 6230"	 },
+	{ 0x8086, 0x0896, "Intel(R) Centrino(R) Wireless-N 130"		 },
 	{ 0x8086, 0x4229, "Intel(R) Wireless WiFi Link 4965"		 },
 	{ 0x8086, 0x422b, "Intel(R) Centrino(R) Ultimate-N 6300"	 },
 	{ 0x8086, 0x422c, "Intel(R) Centrino(R) Advanced-N 6200"	 },
@@ -2928,7 +2929,7 @@ iwn_notif_intr(struct iwn_softc *sc)
 			 * reinitialize the sensitivity state machine.
 			 */
 			if (vap->iv_state == IEEE80211_S_RUN &&
-			    (ic->ic_flags & IEEE80211_F_SCAN) != 0) {
+			    (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
 				if (misses > 5)
 					(void)iwn_init_sensitivity(sc);
 				if (misses >= vap->iv_bmissthreshold) {
@@ -5640,6 +5641,7 @@ iwn_ampdu_tx_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
 	sc->qid2tap[qid] = NULL;
 	free(tap->txa_private, M_DEVBUF);
 	tap->txa_private = NULL;
+	sc->sc_addba_stop(ni, tap);
 }
 
 static void
@@ -6952,12 +6954,24 @@ iwn_set_channel(struct ieee80211com *ic)
 	const struct ieee80211_channel *c = ic->ic_curchan;
 	struct ifnet *ifp = ic->ic_ifp;
 	struct iwn_softc *sc = ifp->if_softc;
+	int error;
 
 	IWN_LOCK(sc);
 	sc->sc_rxtap.wr_chan_freq = htole16(c->ic_freq);
 	sc->sc_rxtap.wr_chan_flags = htole16(c->ic_flags);
 	sc->sc_txtap.wt_chan_freq = htole16(c->ic_freq);
 	sc->sc_txtap.wt_chan_flags = htole16(c->ic_flags);
+
+	/*
+	 * Only need to set the channel in Monitor mode. AP scanning and auth
+	 * are already taken care of by their respective firmware commands.
+	 */
+	if (ic->ic_opmode == IEEE80211_M_MONITOR) {
+		error = iwn_config(sc);
+		if (error != 0)
+		device_printf(sc->sc_dev,
+		    "%s: error %d settting channel\n", __func__, error);
+	}
 	IWN_UNLOCK(sc);
 }
 
diff --git a/sys/dev/ksyms/ksyms.c b/sys/dev/ksyms/ksyms.c
index 38e71a397a5b..49298bc21bdd 100644
--- a/sys/dev/ksyms/ksyms.c
+++ b/sys/dev/ksyms/ksyms.c
@@ -74,7 +74,7 @@ static	d_mmap_t	ksyms_mmap;
 
 static struct cdevsw ksyms_cdevsw = {
     .d_version	=	D_VERSION,
-    .d_flags	=	D_PSEUDO | D_TRACKCLOSE,
+    .d_flags	=	D_TRACKCLOSE,
     .d_open	=	ksyms_open,
     .d_close	=	ksyms_close,
     .d_read	=	ksyms_read,
diff --git a/sys/dev/le/lebuffer_sbus.c b/sys/dev/le/lebuffer_sbus.c
index 1f768a11f1fe..1254fc3b5a99 100644
--- a/sys/dev/le/lebuffer_sbus.c
+++ b/sys/dev/le/lebuffer_sbus.c
@@ -81,6 +81,7 @@ static device_method_t lebuffer_methods[] = {
 	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	bus_generic_rl_alloc_resource),
+	DEVMETHOD(bus_adjust_resource,	bus_generic_adjust_resource),
 	DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
diff --git a/sys/dev/lge/if_lge.c b/sys/dev/lge/if_lge.c
index b57d3477f838..c4ddef89774d 100644
--- a/sys/dev/lge/if_lge.c
+++ b/sys/dev/lge/if_lge.c
@@ -110,7 +110,7 @@ __FBSDID("$FreeBSD$");
 /*
  * Various supported device vendors/types and their names.
  */
-static struct lge_type lge_devs[] = {
+static const struct lge_type const lge_devs[] = {
 	{ LGE_VENDORID, LGE_DEVICEID, "Level 1 Gigabit Ethernet" },
 	{ 0, 0, NULL }
 };
@@ -442,7 +442,7 @@ static int
 lge_probe(dev)
 	device_t		dev;
 {
-	struct lge_type		*t;
+	const struct lge_type	*t;
 
 	t = lge_devs;
 
@@ -1430,9 +1430,9 @@ lge_ifmedia_sts(ifp, ifmr)
 	LGE_LOCK(sc);
 	mii = device_get_softc(sc->lge_miibus);
 	mii_pollstat(mii);
-	LGE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	LGE_UNLOCK(sc);
 
 	return;
 }
diff --git a/sys/dev/lge/if_lgereg.h b/sys/dev/lge/if_lgereg.h
index b963d57ab946..bb794c2fad6e 100644
--- a/sys/dev/lge/if_lgereg.h
+++ b/sys/dev/lge/if_lgereg.h
@@ -475,26 +475,9 @@ struct lge_list_data {
 struct lge_type {
 	u_int16_t		lge_vid;
 	u_int16_t		lge_did;
-	char			*lge_name;
+	const char		*lge_name;
 };
 
-struct lge_mii_frame {
-	u_int8_t		mii_stdelim;
-	u_int8_t		mii_opcode;
-	u_int8_t		mii_phyaddr;
-	u_int8_t		mii_regaddr;
-	u_int8_t		mii_turnaround;
-	u_int16_t		mii_data;
-};
-
-/*
- * MII constants
- */
-#define LGE_MII_STARTDELIM	0x01
-#define LGE_MII_READOP		0x02
-#define LGE_MII_WRITEOP		0x01
-#define LGE_MII_TURNAROUND	0x02
-
 #define LGE_JUMBO_FRAMELEN	9018
 #define LGE_JUMBO_MTU		(LGE_JUMBO_FRAMELEN-ETHER_HDR_LEN-ETHER_CRC_LEN)
 #define LGE_JSLOTS		384
diff --git a/sys/dev/mfi/mfi.c b/sys/dev/mfi/mfi.c
index 470362d5baa3..8a3d277e92a7 100644
--- a/sys/dev/mfi/mfi.c
+++ b/sys/dev/mfi/mfi.c
@@ -1174,7 +1174,7 @@ mfi_aen_complete(struct mfi_command *cm)
 			TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
 			    aen_link);
 			PROC_LOCK(mfi_aen_entry->p);
-			psignal(mfi_aen_entry->p, SIGIO);
+			kern_psignal(mfi_aen_entry->p, SIGIO);
 			PROC_UNLOCK(mfi_aen_entry->p);
 			free(mfi_aen_entry, M_MFIBUF);
 		}
@@ -1488,7 +1488,7 @@ mfi_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 	struct mfi_command *cm;
 	union mfi_sgl *sgl;
 	struct mfi_softc *sc;
-	int i, dir;
+	int i, j, first, dir;
 
 	cm = (struct mfi_command *)arg;
 	sc = cm->cm_sc;
@@ -1502,19 +1502,33 @@ mfi_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 		return;
 	}
 
+	j = 0;
+	if (cm->cm_frame->header.cmd == MFI_CMD_STP) {
+		first = cm->cm_stp_len;
+		if ((sc->mfi_flags & MFI_FLAGS_SG64) == 0) {
+			sgl->sg32[j].addr = segs[0].ds_addr;
+			sgl->sg32[j++].len = first;
+		} else {
+			sgl->sg64[j].addr = segs[0].ds_addr;
+			sgl->sg64[j++].len = first;
+		}
+	} else
+		first = 0;
 	if ((sc->mfi_flags & MFI_FLAGS_SG64) == 0) {
 		for (i = 0; i < nsegs; i++) {
-			sgl->sg32[i].addr = segs[i].ds_addr;
-			sgl->sg32[i].len = segs[i].ds_len;
+			sgl->sg32[j].addr = segs[i].ds_addr + first;
+			sgl->sg32[j++].len = segs[i].ds_len - first;
+			first = 0;
 		}
 	} else {
 		for (i = 0; i < nsegs; i++) {
-			sgl->sg64[i].addr = segs[i].ds_addr;
-			sgl->sg64[i].len = segs[i].ds_len;
+			sgl->sg64[j].addr = segs[i].ds_addr + first;
+			sgl->sg64[j++].len = segs[i].ds_len - first;
+			first = 0;
 		}
 		hdr->flags |= MFI_FRAME_SGL64;
 	}
-	hdr->sg_count = nsegs;
+	hdr->sg_count = j;
 
 	dir = 0;
 	if (cm->cm_flags & MFI_CMD_DATAIN) {
@@ -1525,6 +1539,8 @@ mfi_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 		dir |= BUS_DMASYNC_PREWRITE;
 		hdr->flags |= MFI_FRAME_DIR_WRITE;
 	}
+	if (cm->cm_frame->header.cmd == MFI_CMD_STP)
+		dir |= BUS_DMASYNC_PREWRITE;
 	bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap, dir);
 	cm->cm_flags |= MFI_CMD_MAPPED;
 
@@ -1602,7 +1618,8 @@ mfi_complete(struct mfi_softc *sc, struct mfi_command *cm)
 
 	if ((cm->cm_flags & MFI_CMD_MAPPED) != 0) {
 		dir = 0;
-		if (cm->cm_flags & MFI_CMD_DATAIN)
+		if ((cm->cm_flags & MFI_CMD_DATAIN) ||
+		    (cm->cm_frame->header.cmd == MFI_CMD_STP))
 			dir |= BUS_DMASYNC_POSTREAD;
 		if (cm->cm_flags & MFI_CMD_DATAOUT)
 			dir |= BUS_DMASYNC_POSTWRITE;
@@ -1927,7 +1944,8 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td
 	struct mfi_command *cm = NULL;
 	uint32_t context;
 	union mfi_sense_ptr sense_ptr;
-	uint8_t *data = NULL, *temp;
+	uint8_t *data = NULL, *temp, *addr;
+	size_t len;
 	int i;
 	struct mfi_ioc_passthru *iop = (struct mfi_ioc_passthru *)arg;
 #ifdef __amd64__
@@ -2024,6 +2042,21 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td
 		if (cm->cm_flags == 0)
 			cm->cm_flags |= MFI_CMD_DATAIN | MFI_CMD_DATAOUT;
 		cm->cm_len = cm->cm_frame->header.data_len;
+		if (cm->cm_frame->header.cmd == MFI_CMD_STP) {
+#ifdef __amd64__
+			if (cmd == MFI_CMD) {
+#endif
+				/* Native */
+				cm->cm_stp_len = ioc->mfi_sgl[0].iov_len;
+#ifdef __amd64__
+			} else {
+				/* 32bit on 64bit */
+				ioc32 = (struct mfi_ioc_packet32 *)ioc;
+				cm->cm_stp_len = ioc32->mfi_sgl[0].iov_len;
+			}
+#endif
+			cm->cm_len += cm->cm_stp_len;
+		}
 		if (cm->cm_len &&
 		    (cm->cm_flags & (MFI_CMD_DATAIN | MFI_CMD_DATAOUT))) {
 			cm->cm_data = data = malloc(cm->cm_len, M_MFIBUF,
@@ -2040,35 +2073,30 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td
 		cm->cm_frame->header.context = context;
 
 		temp = data;
-		if (cm->cm_flags & MFI_CMD_DATAOUT) {
+		if ((cm->cm_flags & MFI_CMD_DATAOUT) ||
+		    (cm->cm_frame->header.cmd == MFI_CMD_STP)) {
 			for (i = 0; i < ioc->mfi_sge_count; i++) {
 #ifdef __amd64__
 				if (cmd == MFI_CMD) {
-					/* Native */
-					error = copyin(ioc->mfi_sgl[i].iov_base,
-					       temp,
-					       ioc->mfi_sgl[i].iov_len);
-				} else {
-					void *temp_convert;
-					/* 32bit */
-					ioc32 = (struct mfi_ioc_packet32 *)ioc;
-					temp_convert =
-					    PTRIN(ioc32->mfi_sgl[i].iov_base);
-					error = copyin(temp_convert,
-					       temp,
-					       ioc32->mfi_sgl[i].iov_len);
-				}
-#else
-				error = copyin(ioc->mfi_sgl[i].iov_base,
-				       temp,
-				       ioc->mfi_sgl[i].iov_len);
 #endif
+					/* Native */
+					addr = ioc->mfi_sgl[i].iov_base;
+					len = ioc->mfi_sgl[i].iov_len;
+#ifdef __amd64__
+				} else {
+					/* 32bit on 64bit */
+					ioc32 = (struct mfi_ioc_packet32 *)ioc;
+					addr = PTRIN(ioc32->mfi_sgl[i].iov_base);
+					len = ioc32->mfi_sgl[i].iov_len;
+				}
+#endif
+				error = copyin(addr, temp, len);
 				if (error != 0) {
 					device_printf(sc->mfi_dev,
 					    "Copy in failed\n");
 					goto out;
 				}
-				temp = &temp[ioc->mfi_sgl[i].iov_len];
+				temp = &temp[len];
 			}
 		}
 
@@ -2098,35 +2126,30 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td
 		mtx_unlock(&sc->mfi_io_lock);
 
 		temp = data;
-		if (cm->cm_flags & MFI_CMD_DATAIN) {
+		if ((cm->cm_flags & MFI_CMD_DATAIN) ||
+		    (cm->cm_frame->header.cmd == MFI_CMD_STP)) {
 			for (i = 0; i < ioc->mfi_sge_count; i++) {
 #ifdef __amd64__
 				if (cmd == MFI_CMD) {
-					/* Native */
-					error = copyout(temp,
-						ioc->mfi_sgl[i].iov_base,
-						ioc->mfi_sgl[i].iov_len);
-				} else {
-					void *temp_convert;
-					/* 32bit */
-					ioc32 = (struct mfi_ioc_packet32 *)ioc;
-					temp_convert =
-					    PTRIN(ioc32->mfi_sgl[i].iov_base);
-					error = copyout(temp,
-						temp_convert,
-						ioc32->mfi_sgl[i].iov_len);
-				}
-#else
-				error = copyout(temp,
-					ioc->mfi_sgl[i].iov_base,
-					ioc->mfi_sgl[i].iov_len);
 #endif
+					/* Native */
+					addr = ioc->mfi_sgl[i].iov_base;
+					len = ioc->mfi_sgl[i].iov_len;
+#ifdef __amd64__
+				} else {
+					/* 32bit on 64bit */
+					ioc32 = (struct mfi_ioc_packet32 *)ioc;
+					addr = PTRIN(ioc32->mfi_sgl[i].iov_base);
+					len = ioc32->mfi_sgl[i].iov_len;
+				}
+#endif
+				error = copyout(temp, addr, len);
 				if (error != 0) {
 					device_printf(sc->mfi_dev,
 					    "Copy out failed\n");
 					goto out;
 				}
-				temp = &temp[ioc->mfi_sgl[i].iov_len];
+				temp = &temp[len];
 			}
 		}
 
diff --git a/sys/dev/mfi/mfi_cam.c b/sys/dev/mfi/mfi_cam.c
index 747b6c06231d..c49dafff4b29 100644
--- a/sys/dev/mfi/mfi_cam.c
+++ b/sys/dev/mfi/mfi_cam.c
@@ -354,7 +354,13 @@ mfip_done(struct mfi_command *cm)
 
 		ccbh->status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
 		csio->scsi_status = pt->header.scsi_status;
-		sense_len = min(pt->header.sense_len, sizeof(struct scsi_sense_data));
+		if (pt->header.sense_len < csio->sense_len)
+			csio->sense_resid = csio->sense_len -
+			    pt->header.sense_len;
+		else
+			csio->sense_resid = 0;
+		sense_len = min(pt->header.sense_len,
+		    sizeof(struct scsi_sense_data));
 		bzero(&csio->sense_data, sizeof(struct scsi_sense_data));
 		bcopy(&cm->cm_sense->data[0], &csio->sense_data, sense_len);
 		break;
diff --git a/sys/dev/mfi/mfivar.h b/sys/dev/mfi/mfivar.h
index 9ddb62791312..14ea884885bd 100644
--- a/sys/dev/mfi/mfivar.h
+++ b/sys/dev/mfi/mfivar.h
@@ -87,6 +87,7 @@ struct mfi_command {
 	union mfi_sgl		*cm_sg;
 	void			*cm_data;
 	int			cm_len;
+	int			cm_stp_len;
 	int			cm_total_frame_size;
 	int			cm_extra_frames;
 	int			cm_flags;
diff --git a/sys/dev/mly/mly.c b/sys/dev/mly/mly.c
index 90f2819ccbfd..ee73ae9b9736 100644
--- a/sys/dev/mly/mly.c
+++ b/sys/dev/mly/mly.c
@@ -1294,9 +1294,11 @@ mly_complete_event(struct mly_command *mc)
 static void
 mly_process_event(struct mly_softc *sc, struct mly_event *me)
 {
-    struct scsi_sense_data	*ssd = (struct scsi_sense_data *)&me->sense[0];
-    char			*fp, *tp;
-    int				bus, target, event, class, action;
+    struct scsi_sense_data_fixed *ssd;
+    char			 *fp, *tp;
+    int				 bus, target, event, class, action;
+
+    ssd = (struct scsi_sense_data_fixed *)&me->sense[0];
 
     /* 
      * Errors can be reported using vendor-unique sense data.  In this case, the
diff --git a/sys/dev/mps/mps_sas.c b/sys/dev/mps/mps_sas.c
index 2aa4773eef91..28ddca161880 100644
--- a/sys/dev/mps/mps_sas.c
+++ b/sys/dev/mps/mps_sas.c
@@ -1490,7 +1490,6 @@ mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
 	MPI2_SCSI_IO_REPLY *rep;
 	union ccb *ccb;
 	struct mpssas_softc *sassc;
-	u_int sense_len;
 	int dir = 0;
 
 	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
@@ -1666,10 +1665,17 @@ mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
 	}
 
 	if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
-		sense_len = MIN(rep->SenseCount,
-		    sizeof(struct scsi_sense_data));
-		if (sense_len < rep->SenseCount)
-			ccb->csio.sense_resid = rep->SenseCount - sense_len;
+		int sense_len;
+
+		if (rep->SenseCount < ccb->csio.sense_len)
+			ccb->csio.sense_resid = ccb->csio.sense_len -
+				rep->SenseCount;
+		else
+			ccb->csio.sense_resid = 0;
+
+		sense_len = min(rep->SenseCount, ccb->csio.sense_len -
+		    ccb->csio.sense_resid);
+		bzero(&ccb->csio.sense_data, sizeof(&ccb->csio.sense_data));
 		bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
 		ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 	}
diff --git a/sys/dev/mpt/mpt_cam.c b/sys/dev/mpt/mpt_cam.c
index d512458c7582..50134f72fdf0 100644
--- a/sys/dev/mpt/mpt_cam.c
+++ b/sys/dev/mpt/mpt_cam.c
@@ -3167,12 +3167,20 @@ mpt_scsi_reply_frame_handler(struct mpt_softc *mpt, request_t *req,
 
 	if ((sstate & MPI_SCSI_STATE_AUTOSENSE_VALID) != 0
 	 && (ccb->ccb_h.flags & (CAM_SENSE_PHYS | CAM_SENSE_PTR)) == 0) {
+		uint32_t sense_returned;
+
 		ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
-		ccb->csio.sense_resid =
-		    ccb->csio.sense_len - le32toh(scsi_io_reply->SenseCount);
+		
+		sense_returned = le32toh(scsi_io_reply->SenseCount);
+		if (sense_returned < ccb->csio.sense_len)
+			ccb->csio.sense_resid = ccb->csio.sense_len -
+						sense_returned;
+		else
+			ccb->csio.sense_resid = 0;
+
+		bzero(&ccb->csio.sense_data, sizeof(&ccb->csio.sense_data));
 		bcopy(req->sense_vbuf, &ccb->csio.sense_data,
-		    min(ccb->csio.sense_len,
-		    le32toh(scsi_io_reply->SenseCount)));
+		    min(ccb->csio.sense_len, sense_returned));
 	}
 
 	if ((sstate & MPI_SCSI_STATE_QUEUE_TAG_REJECTED) != 0) {
diff --git a/sys/dev/msk/if_msk.c b/sys/dev/msk/if_msk.c
index be4ac5dc8d88..b060949a8592 100644
--- a/sys/dev/msk/if_msk.c
+++ b/sys/dev/msk/if_msk.c
@@ -995,9 +995,9 @@ msk_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc_if->msk_miibus);
 
 	mii_pollstat(mii);
-	MSK_IF_UNLOCK(sc_if);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	MSK_IF_UNLOCK(sc_if);
 }
 
 static int
diff --git a/sys/dev/nfe/if_nfe.c b/sys/dev/nfe/if_nfe.c
index 28a3c0149f29..3c36fe703b7d 100644
--- a/sys/dev/nfe/if_nfe.c
+++ b/sys/dev/nfe/if_nfe.c
@@ -2952,10 +2952,10 @@ nfe_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 	NFE_LOCK(sc);
 	mii = device_get_softc(sc->nfe_miibus);
 	mii_pollstat(mii);
-	NFE_UNLOCK(sc);
 
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	NFE_UNLOCK(sc);
 }
 
 
diff --git a/sys/dev/nge/if_nge.c b/sys/dev/nge/if_nge.c
index dd16606f9272..db20ad29eea4 100644
--- a/sys/dev/nge/if_nge.c
+++ b/sys/dev/nge/if_nge.c
@@ -2417,9 +2417,9 @@ nge_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 	NGE_LOCK(sc);
 	mii = device_get_softc(sc->nge_miibus);
 	mii_pollstat(mii);
-	NGE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	NGE_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/nve/if_nve.c b/sys/dev/nve/if_nve.c
index 7bb5c584788d..1428662e90dc 100644
--- a/sys/dev/nve/if_nve.c
+++ b/sys/dev/nve/if_nve.c
@@ -1202,10 +1202,10 @@ nve_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 	NVE_LOCK(sc);
 	mii = device_get_softc(sc->miibus);
 	mii_pollstat(mii);
-	NVE_UNLOCK(sc);
 
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	NVE_UNLOCK(sc);
 
 	return;
 }
diff --git a/sys/dev/ofw/ofw_fdt.c b/sys/dev/ofw/ofw_fdt.c
index 68d3149de7ab..806f17c513d9 100644
--- a/sys/dev/ofw/ofw_fdt.c
+++ b/sys/dev/ofw/ofw_fdt.c
@@ -109,22 +109,48 @@ ofw_fdt_init(ofw_t ofw, void *data)
 }
 
 /*
- * Device tree functions
+ * Device tree functions.
+ *
+ * We use the offset from fdtp to the node as the 'phandle' in OF interface.
+ *
+ * phandle is a u32 value, therefore we cannot use the pointer to node as
+ * phandle in 64 bit. We also do not use the usual fdt offset as phandle,
+ * as it can be 0, and the OF interface has special meaning for phandle 0.
  */
 
+static phandle_t
+fdt_offset_phandle(int offset)
+{
+	if (offset < 0)
+		return (0);
+	return ((phandle_t)offset + fdt_off_dt_struct(fdtp));
+}
+
 static int
 fdt_phandle_offset(phandle_t p)
 {
-	const char *dt_struct;
+	int pint = (int)p;
+	int dtoff = fdt_off_dt_struct(fdtp);
+
+	if (pint < dtoff)
+		return (-1);
+	return (pint - dtoff);
+}
+
+static int
+fdt_pointer_offset(const void *ptr)
+{
+	uintptr_t dt_struct, p;
 	int offset;
 
-	dt_struct = (const char *)fdtp + fdt_off_dt_struct(fdtp);
+	p = (uintptr_t)ptr;
+	dt_struct = (uintptr_t)fdtp + fdt_off_dt_struct(fdtp);
 
-	if (((const char *)p < dt_struct) ||
-	    (const char *)p > (dt_struct + fdt_size_dt_struct(fdtp)))
+	if ((p < dt_struct) ||
+	    p > (dt_struct + fdt_size_dt_struct(fdtp)))
 		return (-1);
 
-	offset = (const char *)p - dt_struct;
+	offset = p - dt_struct;
 	if (offset < 0)
 		return (-1);
 
@@ -135,15 +161,13 @@ fdt_phandle_offset(phandle_t p)
 static phandle_t
 ofw_fdt_peer(ofw_t ofw, phandle_t node)
 {
-	phandle_t p;
 	int depth, offset;
 
 	if (node == 0) {
 		/* Find root node */
 		offset = fdt_path_offset(fdtp, "/");
-		p = (phandle_t)fdt_offset_ptr(fdtp, offset, sizeof(p));
 
-		return (p);
+		return (fdt_offset_phandle(offset));
 	}
 
 	offset = fdt_phandle_offset(node);
@@ -155,10 +179,8 @@ ofw_fdt_peer(ofw_t ofw, phandle_t node)
 	    offset = fdt_next_node(fdtp, offset, &depth)) {
 		if (depth < 0)
 			return (0);
-		if (depth == 1) {
-			p = (phandle_t)fdt_offset_ptr(fdtp, offset, sizeof(p));
-			return (p);
-		}
+		if (depth == 1)
+			return (fdt_offset_phandle(offset));
 	}
 
 	return (0);
@@ -168,7 +190,6 @@ ofw_fdt_peer(ofw_t ofw, phandle_t node)
 static phandle_t
 ofw_fdt_child(ofw_t ofw, phandle_t node)
 {
-	phandle_t p;
 	int depth, offset;
 
 	offset = fdt_phandle_offset(node);
@@ -180,10 +201,8 @@ ofw_fdt_child(ofw_t ofw, phandle_t node)
 	    offset = fdt_next_node(fdtp, offset, &depth)) {
 		if (depth < 0)
 			return (0);
-		if (depth == 1) {
-			p = (phandle_t)fdt_offset_ptr(fdtp, offset, sizeof(p));
-			return (p);
-		}
+		if (depth == 1)
+			return (fdt_offset_phandle(offset));
 	}
 
 	return (0);
@@ -193,7 +212,6 @@ ofw_fdt_child(ofw_t ofw, phandle_t node)
 static phandle_t
 ofw_fdt_parent(ofw_t ofw, phandle_t node)
 {
-	phandle_t p;
 	int offset, paroffset;
 
 	offset = fdt_phandle_offset(node);
@@ -201,15 +219,13 @@ ofw_fdt_parent(ofw_t ofw, phandle_t node)
 		return (0);
 
 	paroffset = fdt_parent_offset(fdtp, offset);
-	p = (phandle_t)fdt_offset_ptr(fdtp, paroffset, sizeof(phandle_t));
-	return (p);
+	return (fdt_offset_phandle(paroffset));
 }
 
 /* Return the package handle that corresponds to an instance handle. */
 static phandle_t
 ofw_fdt_instance_to_package(ofw_t ofw, ihandle_t instance)
 {
-	phandle_t p;
 	int offset;
 
 	/*
@@ -223,8 +239,7 @@ ofw_fdt_instance_to_package(ofw_t ofw, ihandle_t instance)
 	if (offset < 0)
 		return (-1);
 
-	p = (phandle_t)fdt_offset_ptr(fdtp, offset, sizeof(phandle_t));
-	return (p);
+	return (fdt_offset_phandle(offset));
 }
 
 /* Get the length of a property of a package. */
@@ -343,7 +358,7 @@ ofw_fdt_nextprop(ofw_t ofw, phandle_t package, const char *previous, char *buf,
 	if (prop == NULL)
 		return (-1);
 
-	offset = fdt_phandle_offset((phandle_t)prop);
+	offset = fdt_pointer_offset(prop);
 	rv = fdt_nextprop(offset, buf, size);
 	return (rv);
 }
@@ -374,14 +389,10 @@ ofw_fdt_canon(ofw_t ofw, const char *device, char *buf, size_t len)
 static phandle_t
 ofw_fdt_finddevice(ofw_t ofw, const char *device)
 {
-	phandle_t p;
 	int offset;
 
 	offset = fdt_path_offset(fdtp, device);
-
-	p = (phandle_t)fdt_offset_ptr(fdtp, offset, sizeof(p));
-
-	return (p);
+	return (fdt_offset_phandle(offset));
 }
 
 /* Return the fully qualified pathname corresponding to an instance. */
diff --git a/sys/dev/ppbus/ppb_msq.c b/sys/dev/ppbus/ppb_msq.c
index 89be014e87f6..6de67cecd7d7 100644
--- a/sys/dev/ppbus/ppb_msq.c
+++ b/sys/dev/ppbus/ppb_msq.c
@@ -244,6 +244,7 @@ ppb_MS_init_msq(struct ppb_microseq *msq, int nbparam, ...)
 		}
 	}
 
+	va_end(p_list);
 	return (0);
 }
 
diff --git a/sys/dev/puc/pucdata.c b/sys/dev/puc/pucdata.c
index f675b69b661a..80dff6ea827a 100644
--- a/sys/dev/puc/pucdata.c
+++ b/sys/dev/puc/pucdata.c
@@ -645,6 +645,12 @@ const struct puc_cfg puc_pci_devices[] = {
 	 * As sold by Kouwell <URL:http://www.kouwell.com/>.
 	 * I/O Flex PCI I/O Card Model-223 with 4 serial and 1 parallel ports.
 	 */
+	{
+		0x1415, 0x9501, 0x10fc ,0xc070,
+		"I-O DATA RSA-PCI2/R",
+		DEFAULT_RCLK * 8,
+		PUC_PORT_2S, 0x10, 0, 8,
+	},
 
 	{   0x1415, 0x9501, 0x131f, 0x2050,
 	    "SIIG Cyber 4 PCI 16550",
@@ -733,6 +739,13 @@ const struct puc_cfg puc_pci_devices[] = {
 	 * <URL:http://www.startech.com>
 	 */
 
+	{   0x1415, 0xc138, 0xffff, 0,
+	    "Oxford Semiconductor OXPCIe952 UARTs",
+	    DEFAULT_RCLK * 0x22,
+	    PUC_PORT_NONSTANDARD, 0x10, 0, -1,
+	    .config_function = puc_config_oxford_pcie
+	},
+
 	{   0x1415, 0xc158, 0xffff, 0,
 	    "Oxford Semiconductor OXPCIe952 UARTs",
 	    DEFAULT_RCLK * 0x22,
diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index 3c45cc97d9c6..704276efd08e 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -3214,9 +3214,9 @@ re_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	RL_LOCK(sc);
 	mii_pollstat(mii);
-	RL_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	RL_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/sge/if_sge.c b/sys/dev/sge/if_sge.c
index 03bf900a8fb8..c56760d89746 100644
--- a/sys/dev/sge/if_sge.c
+++ b/sys/dev/sge/if_sge.c
@@ -1746,9 +1746,9 @@ sge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 		return;
 	}
 	mii_pollstat(mii);
-	SGE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	SGE_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/siba/siba_bwn.c b/sys/dev/siba/siba_bwn.c
index 1e0ba0dfa689..a778d49b6150 100644
--- a/sys/dev/siba/siba_bwn.c
+++ b/sys/dev/siba/siba_bwn.c
@@ -206,8 +206,9 @@ siba_bwn_suspend(device_t dev)
 	for (i = 0 ; i < devcnt ; i++) {
 		error = DEVICE_SUSPEND(devlistp[i]);
 		if (error) {
-			for (j = 0; j < i; i++)
+			for (j = 0; j < i; j++)
 				DEVICE_RESUME(devlistp[j]);
+			free(devlistp, M_TEMP);
 			return (error);
 		}
 	}
diff --git a/sys/dev/sis/if_sis.c b/sys/dev/sis/if_sis.c
index 9290ff7591ba..d0be783ba7a3 100644
--- a/sys/dev/sis/if_sis.c
+++ b/sys/dev/sis/if_sis.c
@@ -2237,9 +2237,9 @@ sis_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 	SIS_LOCK(sc);
 	mii = device_get_softc(sc->sis_miibus);
 	mii_pollstat(mii);
-	SIS_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	SIS_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/sound/midi/midi.c b/sys/dev/sound/midi/midi.c
index 53cc5fdea1bc..97492a7f6794 100644
--- a/sys/dev/sound/midi/midi.c
+++ b/sys/dev/sound/midi/midi.c
@@ -562,7 +562,7 @@ midi_in(struct snd_midi *m, MIDI_TYPE *buf, int size)
 		selwakeup(&m->rsel);
 		if (m->async) {
 			PROC_LOCK(m->async);
-			psignal(m->async, SIGIO);
+			kern_psignal(m->async, SIGIO);
 			PROC_UNLOCK(m->async);
 		}
 #if 0
@@ -604,7 +604,7 @@ midi_out(struct snd_midi *m, MIDI_TYPE *buf, int size)
 		selwakeup(&m->wsel);
 		if (m->async) {
 			PROC_LOCK(m->async);
-			psignal(m->async, SIGIO);
+			kern_psignal(m->async, SIGIO);
 			PROC_UNLOCK(m->async);
 		}
 	}
diff --git a/sys/dev/stge/if_stge.c b/sys/dev/stge/if_stge.c
index 9a9ef03181f6..5058a4dfa2ec 100644
--- a/sys/dev/stge/if_stge.c
+++ b/sys/dev/stge/if_stge.c
@@ -199,8 +199,7 @@ static device_method_t stge_methods[] = {
 	DEVMETHOD(miibus_writereg,	stge_miibus_writereg),
 	DEVMETHOD(miibus_statchg,	stge_miibus_statchg),
 
-	{ 0, 0 }
-
+	KOBJMETHOD_END
 };
 
 static driver_t stge_driver = {
diff --git a/sys/dev/sym/sym_hipd.c b/sys/dev/sym/sym_hipd.c
index bc1ddccdcc48..d4e1acae5916 100644
--- a/sys/dev/sym/sym_hipd.c
+++ b/sys/dev/sym/sym_hipd.c
@@ -7154,7 +7154,7 @@ static void sym_complete_error (hcb_p np, ccb_p cp)
 {
 	struct ccb_scsiio *csio;
 	u_int cam_status;
-	int i;
+	int i, sense_returned;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
@@ -7214,11 +7214,15 @@ static void sym_complete_error (hcb_p np, ccb_p cp)
 			 *  Bounce back the sense data to user and
 			 *  fix the residual.
 			 */
-			bzero(&csio->sense_data, csio->sense_len);
+			bzero(&csio->sense_data, sizeof(csio->sense_data));
+			sense_returned = SYM_SNS_BBUF_LEN - csio->sense_resid;
+			if (sense_returned < csio->sense_len)
+				csio->sense_resid = csio->sense_len -
+				    sense_returned;
+			else
+				csio->sense_resid = 0;
 			bcopy(cp->sns_bbuf, &csio->sense_data,
-			      MIN(csio->sense_len, SYM_SNS_BBUF_LEN));
-			csio->sense_resid += csio->sense_len;
-			csio->sense_resid -= SYM_SNS_BBUF_LEN;
+			    MIN(csio->sense_len, sense_returned));
 #if 0
 			/*
 			 *  If the device reports a UNIT ATTENTION condition
diff --git a/sys/dev/syscons/scmouse.c b/sys/dev/syscons/scmouse.c
index 7c1dcc9a99c4..21e6d0830d60 100644
--- a/sys/dev/syscons/scmouse.c
+++ b/sys/dev/syscons/scmouse.c
@@ -811,7 +811,7 @@ sc_mouse_ioctl(struct tty *tp, u_long cmd, caddr_t data, struct thread *td)
 			if (p1)
 			    PROC_UNLOCK(p1);
 		} else {
-		    psignal(cur_scp->mouse_proc, cur_scp->mouse_signal);
+		    kern_psignal(cur_scp->mouse_proc, cur_scp->mouse_signal);
 		    PROC_UNLOCK(cur_scp->mouse_proc);
 		    break;
 		}
@@ -871,7 +871,7 @@ sc_mouse_ioctl(struct tty *tp, u_long cmd, caddr_t data, struct thread *td)
 			if (p1)
 			    PROC_UNLOCK(p1);
 		} else {
-		    psignal(cur_scp->mouse_proc, cur_scp->mouse_signal);
+		    kern_psignal(cur_scp->mouse_proc, cur_scp->mouse_signal);
 		    PROC_UNLOCK(cur_scp->mouse_proc);
 		    break;
 		}
diff --git a/sys/dev/syscons/syscons.c b/sys/dev/syscons/syscons.c
index b8f328ee34c3..227350eb4087 100644
--- a/sys/dev/syscons/syscons.c
+++ b/sys/dev/syscons/syscons.c
@@ -2508,7 +2508,7 @@ signal_vt_rel(scr_stat *scp)
 	return FALSE;
     scp->status |= SWITCH_WAIT_REL;
     PROC_LOCK(scp->proc);
-    psignal(scp->proc, scp->smode.relsig);
+    kern_psignal(scp->proc, scp->smode.relsig);
     PROC_UNLOCK(scp->proc);
     DPRINTF(5, ("sending relsig to %d\n", scp->pid));
     return TRUE;
@@ -2523,7 +2523,7 @@ signal_vt_acq(scr_stat *scp)
 	cnavailable(sc_consptr,  FALSE);
     scp->status |= SWITCH_WAIT_ACQ;
     PROC_LOCK(scp->proc);
-    psignal(scp->proc, scp->smode.acqsig);
+    kern_psignal(scp->proc, scp->smode.acqsig);
     PROC_UNLOCK(scp->proc);
     DPRINTF(5, ("sending acqsig to %d\n", scp->pid));
     return TRUE;
diff --git a/sys/dev/tws/tws.c b/sys/dev/tws/tws.c
new file mode 100644
index 000000000000..750f878f9b5c
--- /dev/null
+++ b/sys/dev/tws/tws.c
@@ -0,0 +1,905 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+*/
+
+
+#include <dev/tws/tws.h>
+#include <dev/tws/tws_services.h>
+#include <dev/tws/tws_hdm.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+
+MALLOC_DEFINE(M_TWS, "twsbuf", "buffers used by tws driver");
+int tws_queue_depth = TWS_MAX_REQS;
+int tws_enable_msi = 0;
+int tws_enable_msix = 0;
+
+
+
+/* externs */
+extern int tws_cam_attach(struct tws_softc *sc);
+extern void tws_cam_detach(struct tws_softc *sc);
+extern int tws_init_ctlr(struct tws_softc *sc);
+extern boolean tws_ctlr_ready(struct tws_softc *sc);
+extern void tws_turn_off_interrupts(struct tws_softc *sc);
+extern void tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req,
+                                u_int8_t q_type );
+extern struct tws_request *tws_q_remove_request(struct tws_softc *sc,
+                                   struct tws_request *req, u_int8_t q_type );
+extern struct tws_request *tws_q_remove_head(struct tws_softc *sc, 
+                                                       u_int8_t q_type );
+extern boolean tws_get_response(struct tws_softc *sc, u_int16_t *req_id);
+extern boolean tws_ctlr_reset(struct tws_softc *sc);
+extern void tws_intr(void *arg);
+extern int tws_use_32bit_sgls;
+
+
+struct tws_request *tws_get_request(struct tws_softc *sc, u_int16_t type);
+int tws_init_connect(struct tws_softc *sc, u_int16_t mc);
+void tws_send_event(struct tws_softc *sc, u_int8_t event);
+uint8_t tws_get_state(struct tws_softc *sc);
+void tws_release_request(struct tws_request *req);
+
+
+
+/* Function prototypes */
+static d_open_t     tws_open;
+static d_close_t    tws_close;
+static d_read_t     tws_read;
+static d_write_t    tws_write;
+extern d_ioctl_t    tws_ioctl;
+
+static int tws_init(struct tws_softc *sc);
+static void tws_dmamap_cmds_load_cbfn(void *arg, bus_dma_segment_t *segs,
+                           int nseg, int error);
+
+static int tws_init_reqs(struct tws_softc *sc, u_int32_t dma_mem_size);
+static int tws_init_aen_q(struct tws_softc *sc);
+static int tws_init_trace_q(struct tws_softc *sc);
+static int tws_setup_irq(struct tws_softc *sc);
+int tws_setup_intr(struct tws_softc *sc, int irqs);
+int tws_teardown_intr(struct tws_softc *sc);
+
+
+/* Character device entry points */
+
+static struct cdevsw tws_cdevsw = {
+    .d_version =    D_VERSION,
+    .d_open =   tws_open,
+    .d_close =  tws_close,
+    .d_read =   tws_read,
+    .d_write =  tws_write,
+    .d_ioctl =  tws_ioctl,
+    .d_name =   "tws",
+};
+
+/*
+ * In the cdevsw routines, we find our softc by using the si_drv1 member
+ * of struct cdev.  We set this variable to point to our softc in our
+ * attach routine when we create the /dev entry.
+ */
+
+int
+tws_open(struct cdev *dev, int oflags, int devtype, d_thread_t *td)
+{
+    struct tws_softc *sc = dev->si_drv1;
+
+    if ( sc ) 
+        TWS_TRACE_DEBUG(sc, "entry", dev, oflags);
+    return (0);
+}
+
+int
+tws_close(struct cdev *dev, int fflag, int devtype, d_thread_t *td)
+{
+    struct tws_softc *sc = dev->si_drv1;
+
+    if ( sc )
+        TWS_TRACE_DEBUG(sc, "entry", dev, fflag);
+    return (0);
+}
+
+int
+tws_read(struct cdev *dev, struct uio *uio, int ioflag)
+{
+    struct tws_softc *sc = dev->si_drv1;
+
+    if ( sc )
+        TWS_TRACE_DEBUG(sc, "entry", dev, ioflag);
+    return (0);
+}
+
+int
+tws_write(struct cdev *dev, struct uio *uio, int ioflag)
+{
+    struct tws_softc *sc = dev->si_drv1;
+
+    if ( sc ) 
+        TWS_TRACE_DEBUG(sc, "entry", dev, ioflag);
+    return (0);
+}
+
+/* PCI Support Functions */
+
+/*
+ * Compare the device ID of this device against the IDs that this driver
+ * supports.  If there is a match, set the description and return success.
+ */
+static int
+tws_probe(device_t dev)
+{
+    static u_int8_t first_ctlr = 1;
+
+    if ((pci_get_vendor(dev) == TWS_VENDOR_ID) &&
+        (pci_get_device(dev) == TWS_DEVICE_ID)) {
+        device_set_desc(dev, "LSI 3ware SAS/SATA Storage Controller");
+        if (first_ctlr) {
+            printf("LSI 3ware device driver for SAS/SATA storage "
+                    "controllers, version: %s\n", TWS_DRIVER_VERSION_STRING);
+            first_ctlr = 0;
+        }
+
+        return(BUS_PROBE_DEFAULT);
+    }
+    return (ENXIO);
+}
+
+/* Attach function is only called if the probe is successful. */
+
+static int
+tws_attach(device_t dev)
+{
+    struct tws_softc *sc = device_get_softc(dev);
+    u_int32_t cmd, bar;
+    int error=0,i;
+
+    /* no tracing yet */
+    /* Look up our softc and initialize its fields. */
+    sc->tws_dev = dev;
+    sc->device_id = pci_get_device(dev);
+    sc->subvendor_id = pci_get_subvendor(dev);
+    sc->subdevice_id = pci_get_subdevice(dev);
+
+    /* Intialize mutexes */
+    mtx_init( &sc->q_lock, "tws_q_lock", NULL, MTX_DEF);
+    mtx_init( &sc->sim_lock,  "tws_sim_lock", NULL, MTX_DEF);
+    mtx_init( &sc->gen_lock,  "tws_gen_lock", NULL, MTX_DEF);
+    mtx_init( &sc->io_lock,  "tws_io_lock", NULL, MTX_DEF);
+
+    if ( tws_init_trace_q(sc) == FAILURE )
+        printf("trace init failure\n");
+    /* send init event */
+    mtx_lock(&sc->gen_lock);
+    tws_send_event(sc, TWS_INIT_START);
+    mtx_unlock(&sc->gen_lock);
+
+
+#if _BYTE_ORDER == _BIG_ENDIAN
+    TWS_TRACE(sc, "BIG endian", 0, 0);
+#endif
+    /* sysctl context setup */
+    sysctl_ctx_init(&sc->tws_clist);
+    sc->tws_oidp = SYSCTL_ADD_NODE(&sc->tws_clist,
+                                   SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
+                                   device_get_nameunit(dev), 
+                                   CTLFLAG_RD, 0, "");
+    if ( sc->tws_oidp == NULL ) {
+        tws_log(sc, SYSCTL_TREE_NODE_ADD);
+        goto attach_fail_1;
+    }
+    SYSCTL_ADD_STRING(&sc->tws_clist, SYSCTL_CHILDREN(sc->tws_oidp),
+                      OID_AUTO, "driver_version", CTLFLAG_RD,
+                      TWS_DRIVER_VERSION_STRING, 0, "TWS driver version");
+
+    cmd = pci_read_config(dev, PCIR_COMMAND, 2);
+    if ( (cmd & PCIM_CMD_PORTEN) == 0) {
+        tws_log(sc, PCI_COMMAND_READ);
+        goto attach_fail_1;
+    }
+    /* Force the busmaster enable bit on. */
+    cmd |= PCIM_CMD_BUSMASTEREN;
+    pci_write_config(dev, PCIR_COMMAND, cmd, 2);
+
+    bar = pci_read_config(dev, TWS_PCI_BAR0, 4);
+    TWS_TRACE_DEBUG(sc, "bar0 ", bar, 0);
+    bar = pci_read_config(dev, TWS_PCI_BAR1, 4);
+    bar = bar & ~TWS_BIT2;
+    TWS_TRACE_DEBUG(sc, "bar1 ", bar, 0);
+ 
+    /* MFA base address is BAR2 register used for 
+     * push mode. Firmware will evatualy move to 
+     * pull mode during witch this needs to change
+     */ 
+#ifndef TWS_PULL_MODE_ENABLE
+    sc->mfa_base = (u_int64_t)pci_read_config(dev, TWS_PCI_BAR2, 4);
+    sc->mfa_base = sc->mfa_base & ~TWS_BIT2;
+    TWS_TRACE_DEBUG(sc, "bar2 ", sc->mfa_base, 0);
+#endif
+
+    /* allocate MMIO register space */ 
+    sc->reg_res_id = TWS_PCI_BAR1; /* BAR1 offset */
+    if ((sc->reg_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
+                                &(sc->reg_res_id), 0, ~0, 1, RF_ACTIVE))
+                                == NULL) {
+        tws_log(sc, ALLOC_MEMORY_RES);
+        goto attach_fail_1;
+    }
+    sc->bus_tag = rman_get_bustag(sc->reg_res);
+    sc->bus_handle = rman_get_bushandle(sc->reg_res);
+
+#ifndef TWS_PULL_MODE_ENABLE
+    /* Allocate bus space for inbound mfa */ 
+    sc->mfa_res_id = TWS_PCI_BAR2; /* BAR2 offset */
+    if ((sc->mfa_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
+                          &(sc->mfa_res_id), 0, ~0, 0x100000, RF_ACTIVE))
+                                == NULL) {
+        tws_log(sc, ALLOC_MEMORY_RES);
+        goto attach_fail_2;
+    }
+    sc->bus_mfa_tag = rman_get_bustag(sc->mfa_res);
+    sc->bus_mfa_handle = rman_get_bushandle(sc->mfa_res);
+#endif
+
+    /* Allocate and register our interrupt. */
+    sc->intr_type = TWS_INTx; /* default */
+
+    if ( tws_enable_msi )
+        sc->intr_type = TWS_MSI;
+    if ( tws_setup_irq(sc) == FAILURE ) {
+        tws_log(sc, ALLOC_MEMORY_RES);
+        goto attach_fail_3;
+    }
+
+    /*
+     * Create a /dev entry for this device.  The kernel will assign us
+     * a major number automatically.  We use the unit number of this
+     * device as the minor number and name the character device
+     * "tws<unit>".
+     */
+    sc->tws_cdev = make_dev(&tws_cdevsw, device_get_unit(dev),
+        UID_ROOT, GID_OPERATOR, S_IRUSR | S_IWUSR, "tws%u", 
+        device_get_unit(dev));
+    sc->tws_cdev->si_drv1 = sc;
+
+    if ( tws_init(sc) == FAILURE ) {
+        tws_log(sc, TWS_INIT_FAILURE);
+        goto attach_fail_4;
+    }
+    if ( tws_init_ctlr(sc) == FAILURE ) {
+        tws_log(sc, TWS_CTLR_INIT_FAILURE);
+        goto attach_fail_4;
+    }
+    if ((error = tws_cam_attach(sc))) {
+        tws_log(sc, TWS_CAM_ATTACH);
+        goto attach_fail_4;
+    }
+    /* send init complete event */
+    mtx_lock(&sc->gen_lock);
+    tws_send_event(sc, TWS_INIT_COMPLETE);
+    mtx_unlock(&sc->gen_lock);
+        
+    TWS_TRACE_DEBUG(sc, "attached successfully", 0, sc->device_id);
+    return(0);
+
+attach_fail_4:
+    tws_teardown_intr(sc);
+    destroy_dev(sc->tws_cdev);
+attach_fail_3:
+    for(i=0;i<sc->irqs;i++) {
+        if ( sc->irq_res[i] ){
+            if (bus_release_resource(sc->tws_dev,
+                 SYS_RES_IRQ, sc->irq_res_id[i], sc->irq_res[i]))
+                TWS_TRACE(sc, "bus irq res", 0, 0);
+        }
+    }
+#ifndef TWS_PULL_MODE_ENABLE
+attach_fail_2: 
+#endif
+    if ( sc->mfa_res ){
+        if (bus_release_resource(sc->tws_dev,
+                 SYS_RES_MEMORY, sc->mfa_res_id, sc->mfa_res))
+            TWS_TRACE(sc, "bus release ", 0, sc->mfa_res_id);
+    }
+    if ( sc->reg_res ){
+        if (bus_release_resource(sc->tws_dev,
+                 SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res))
+            TWS_TRACE(sc, "bus release2 ", 0, sc->reg_res_id);
+    }
+attach_fail_1:
+    mtx_destroy(&sc->q_lock);
+    mtx_destroy(&sc->sim_lock);
+    mtx_destroy(&sc->gen_lock);
+    mtx_destroy(&sc->io_lock);
+    sysctl_ctx_free(&sc->tws_clist);
+    return (ENXIO);
+}
+
+/* Detach device. */
+
+static int
+tws_detach(device_t dev)
+{
+    struct tws_softc *sc = device_get_softc(dev);
+    int i;
+    u_int32_t reg;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+
+    mtx_lock(&sc->gen_lock);
+    tws_send_event(sc, TWS_UNINIT_START);
+    mtx_unlock(&sc->gen_lock);
+
+    /* needs to disable interrupt before detaching from cam */
+    tws_turn_off_interrupts(sc);
+    /* clear door bell */
+    tws_write_reg(sc, TWS_I2O0_HOBDBC, ~0, 4);
+    reg = tws_read_reg(sc, TWS_I2O0_HIMASK, 4);
+    TWS_TRACE_DEBUG(sc, "turn-off-intr", reg, 0);
+    sc->obfl_q_overrun = false;
+    tws_init_connect(sc, 1);
+
+    /* Teardown the state in our softc created in our attach routine. */
+    /* Disconnect the interrupt handler. */
+    tws_teardown_intr(sc);
+
+    /* Release irq resource */
+    for(i=0;i<sc->irqs;i++) {
+        if ( sc->irq_res[i] ){
+            if (bus_release_resource(sc->tws_dev,
+                     SYS_RES_IRQ, sc->irq_res_id[i], sc->irq_res[i]))
+                TWS_TRACE(sc, "bus release irq resource", 
+                                       i, sc->irq_res_id[i]);
+        }
+    }
+    if ( sc->intr_type == TWS_MSI ) {
+        pci_release_msi(sc->tws_dev);
+    }
+
+    tws_cam_detach(sc);
+
+    /* Release memory resource */
+    if ( sc->mfa_res ){
+        if (bus_release_resource(sc->tws_dev,
+                 SYS_RES_MEMORY, sc->mfa_res_id, sc->mfa_res))
+            TWS_TRACE(sc, "bus release mem resource", 0, sc->mfa_res_id);
+    }
+    if ( sc->reg_res ){
+        if (bus_release_resource(sc->tws_dev,
+                 SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res))
+            TWS_TRACE(sc, "bus release mem resource", 0, sc->reg_res_id);
+    }
+
+    free(sc->reqs, M_TWS);
+    free(sc->sense_bufs, M_TWS);
+    free(sc->scan_ccb, M_TWS);
+    free(sc->aen_q.q, M_TWS);
+    free(sc->trace_q.q, M_TWS);
+    mtx_destroy(&sc->q_lock);
+    mtx_destroy(&sc->sim_lock);
+    mtx_destroy(&sc->gen_lock);
+    mtx_destroy(&sc->io_lock);
+    destroy_dev(sc->tws_cdev);
+    sysctl_ctx_free(&sc->tws_clist);
+    return (0);
+}
+
+int
+tws_setup_intr(struct tws_softc *sc, int irqs)
+{
+    int i, error;
+
+    for(i=0;i<irqs;i++) {
+        if (!(sc->intr_handle[i])) {
+            if ((error = bus_setup_intr(sc->tws_dev, sc->irq_res[i],
+                                    INTR_TYPE_CAM | INTR_MPSAFE,
+#if (__FreeBSD_version >= 700000)
+                                    NULL, 
+#endif
+                                    tws_intr, sc, &sc->intr_handle[i]))) {
+                tws_log(sc, SETUP_INTR_RES);
+                return(FAILURE);
+            }
+        }
+    }
+    return(SUCCESS);
+
+}
+
+
+int
+tws_teardown_intr(struct tws_softc *sc)
+{
+    int i, error;
+
+    for(i=0;i<sc->irqs;i++) {
+        if (sc->intr_handle[i]) {
+            error = bus_teardown_intr(sc->tws_dev,
+                                      sc->irq_res[i], sc->intr_handle[i]);
+            sc->intr_handle[i] = NULL;
+        }
+    }
+    return(SUCCESS);
+}
+
+
+static int 
+tws_setup_irq(struct tws_softc *sc)
+{
+    int messages;
+    u_int16_t cmd;
+
+    cmd = pci_read_config(sc->tws_dev, PCIR_COMMAND, 2);
+    switch(sc->intr_type) {
+        case TWS_INTx :
+            cmd = cmd & ~0x0400;
+            pci_write_config(sc->tws_dev, PCIR_COMMAND, cmd, 2);
+            sc->irqs = 1;
+            sc->irq_res_id[0] = 0;
+            sc->irq_res[0] = bus_alloc_resource_any(sc->tws_dev, SYS_RES_IRQ,
+                            &sc->irq_res_id[0], RF_SHAREABLE | RF_ACTIVE);
+            if ( ! sc->irq_res[0] )
+                return(FAILURE);
+            if ( tws_setup_intr(sc, sc->irqs) == FAILURE )
+                return(FAILURE);
+            device_printf(sc->tws_dev, "Using legacy INTx\n");
+            break;
+        case TWS_MSI :
+            cmd = cmd | 0x0400;
+            pci_write_config(sc->tws_dev, PCIR_COMMAND, cmd, 2);
+            sc->irqs = 1;
+            sc->irq_res_id[0] = 1;
+            messages = 1;
+            if (pci_alloc_msi(sc->tws_dev, &messages) != 0 ) {
+                TWS_TRACE(sc, "pci alloc msi fail", 0, messages);
+                return(FAILURE);
+            }
+            sc->irq_res[0] = bus_alloc_resource_any(sc->tws_dev, SYS_RES_IRQ,
+                              &sc->irq_res_id[0], RF_SHAREABLE | RF_ACTIVE);
+              
+            if ( !sc->irq_res[0]  )
+                return(FAILURE);
+            if ( tws_setup_intr(sc, sc->irqs) == FAILURE )
+                return(FAILURE);
+            device_printf(sc->tws_dev, "Using MSI\n");
+            break;
+
+    }
+
+    return(SUCCESS);
+}
+
+static int
+tws_init(struct tws_softc *sc)
+{
+
+    u_int32_t max_sg_elements;
+    u_int32_t dma_mem_size;
+    int error;
+    u_int32_t reg;
+
+    sc->seq_id = 0;
+    if ( tws_queue_depth > TWS_MAX_REQS )
+        tws_queue_depth = TWS_MAX_REQS;
+    if (tws_queue_depth < TWS_RESERVED_REQS+1)
+        tws_queue_depth = TWS_RESERVED_REQS+1;
+    sc->is64bit = (sizeof(bus_addr_t) == 8) ? true : false;
+    max_sg_elements = (sc->is64bit && !tws_use_32bit_sgls) ? 
+                                 TWS_MAX_64BIT_SG_ELEMENTS : 
+                                 TWS_MAX_32BIT_SG_ELEMENTS;
+    dma_mem_size = (sizeof(struct tws_command_packet) * tws_queue_depth) +
+                             (TWS_SECTOR_SIZE) ;
+    if ( bus_dma_tag_create(NULL,                    /* parent */          
+                            TWS_ALIGNMENT,           /* alignment */
+                            0,                       /* boundary */
+                            BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
+                            BUS_SPACE_MAXADDR,       /* highaddr */
+                            NULL, NULL,              /* filter, filterarg */
+                            BUS_SPACE_MAXSIZE,       /* maxsize */
+                            max_sg_elements,         /* numsegs */
+                            BUS_SPACE_MAXSIZE,       /* maxsegsize */
+                            0,                       /* flags */
+                            NULL, NULL,              /* lockfunc, lockfuncarg */
+                            &sc->parent_tag          /* tag */
+                           )) {
+        TWS_TRACE_DEBUG(sc, "DMA parent tag Create fail", max_sg_elements, 
+                                                    sc->is64bit);
+        return(ENOMEM);
+    }
+    /* In bound message frame requires 16byte alignment.
+     * Outbound MF's can live with 4byte alignment - for now just 
+     * use 16 for both.
+     */
+    if ( bus_dma_tag_create(sc->parent_tag,       /* parent */          
+                            TWS_IN_MF_ALIGNMENT,  /* alignment */
+                            0,                    /* boundary */
+                            BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
+                            BUS_SPACE_MAXADDR,    /* highaddr */
+                            NULL, NULL,           /* filter, filterarg */
+                            dma_mem_size,         /* maxsize */
+                            1,                    /* numsegs */
+                            BUS_SPACE_MAXSIZE,    /* maxsegsize */
+                            0,                    /* flags */
+                            NULL, NULL,           /* lockfunc, lockfuncarg */
+                            &sc->cmd_tag          /* tag */
+                           )) {
+        TWS_TRACE_DEBUG(sc, "DMA cmd tag Create fail", max_sg_elements, sc->is64bit);
+        return(ENOMEM);
+    }
+
+    if (bus_dmamem_alloc(sc->cmd_tag, &sc->dma_mem,
+                    BUS_DMA_NOWAIT, &sc->cmd_map)) {
+        TWS_TRACE_DEBUG(sc, "DMA mem alloc fail", max_sg_elements, sc->is64bit);
+        return(ENOMEM);
+    }
+
+    /* if bus_dmamem_alloc succeeds then bus_dmamap_load will succeed */
+    sc->dma_mem_phys=0;
+    error = bus_dmamap_load(sc->cmd_tag, sc->cmd_map, sc->dma_mem,
+                    dma_mem_size, tws_dmamap_cmds_load_cbfn,
+                    &sc->dma_mem_phys, 0);
+
+   /*
+    * Create a dma tag for data buffers; size will be the maximum
+    * possible I/O size (128kB).
+    */
+    if (bus_dma_tag_create(sc->parent_tag,         /* parent */
+                           TWS_ALIGNMENT,          /* alignment */
+                           0,                      /* boundary */
+                           BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
+                           BUS_SPACE_MAXADDR,      /* highaddr */
+                           NULL, NULL,             /* filter, filterarg */
+                           TWS_MAX_IO_SIZE,        /* maxsize */
+                           max_sg_elements,        /* nsegments */
+                           TWS_MAX_IO_SIZE,        /* maxsegsize */
+                           BUS_DMA_ALLOCNOW,       /* flags */
+                           busdma_lock_mutex,      /* lockfunc */
+                           &sc->io_lock,           /* lockfuncarg */
+                           &sc->data_tag           /* tag */)) {
+        TWS_TRACE_DEBUG(sc, "DMA cmd tag Create fail", max_sg_elements, sc->is64bit);
+        return(ENOMEM);
+    }
+
+    sc->reqs = malloc(sizeof(struct tws_request) * tws_queue_depth, M_TWS,
+                      M_WAITOK | M_ZERO);
+    if ( sc->reqs == NULL ) {
+        TWS_TRACE_DEBUG(sc, "malloc failed", 0, sc->is64bit);
+        return(ENOMEM);
+    }
+    sc->sense_bufs = malloc(sizeof(struct tws_sense) * tws_queue_depth, M_TWS,
+                      M_WAITOK | M_ZERO);
+    if ( sc->sense_bufs == NULL ) {
+        TWS_TRACE_DEBUG(sc, "sense malloc failed", 0, sc->is64bit);
+        return(ENOMEM);
+    }
+    sc->scan_ccb = malloc(sizeof(union ccb), M_TWS, M_WAITOK | M_ZERO);
+    if ( sc->scan_ccb == NULL ) {
+        TWS_TRACE_DEBUG(sc, "ccb malloc failed", 0, sc->is64bit);
+        return(ENOMEM);
+    }
+
+    if ( !tws_ctlr_ready(sc) )
+        if( !tws_ctlr_reset(sc) )
+            return(FAILURE);
+    
+    bzero(&sc->stats, sizeof(struct tws_stats));
+    tws_init_qs(sc);
+    tws_turn_off_interrupts(sc);
+
+    /* 
+     * enable pull mode by setting bit1 .
+     * setting bit0 to 1 will enable interrupt coalesing 
+     * will revisit. 
+     */
+
+#ifdef TWS_PULL_MODE_ENABLE
+
+    reg = tws_read_reg(sc, TWS_I2O0_CTL, 4);
+    TWS_TRACE_DEBUG(sc, "i20 ctl", reg, TWS_I2O0_CTL);
+    tws_write_reg(sc, TWS_I2O0_CTL, reg | TWS_BIT1, 4);
+
+#endif
+
+    TWS_TRACE_DEBUG(sc, "dma_mem_phys", sc->dma_mem_phys, TWS_I2O0_CTL);
+    if ( tws_init_reqs(sc, dma_mem_size) == FAILURE )
+        return(FAILURE);
+    if ( tws_init_aen_q(sc) == FAILURE )
+        return(FAILURE);
+
+    return(SUCCESS);
+    
+} 
+
+static int
+tws_init_aen_q(struct tws_softc *sc)
+{
+    sc->aen_q.head=0;
+    sc->aen_q.tail=0;
+    sc->aen_q.depth=256;
+    sc->aen_q.overflow=0;
+    sc->aen_q.q = malloc(sizeof(struct tws_event_packet)*sc->aen_q.depth, 
+                              M_TWS, M_WAITOK | M_ZERO);
+    if ( ! sc->aen_q.q )
+        return(FAILURE);
+    return(SUCCESS);
+}
+
+static int
+tws_init_trace_q(struct tws_softc *sc)
+{
+    sc->trace_q.head=0;
+    sc->trace_q.tail=0;
+    sc->trace_q.depth=256;
+    sc->trace_q.overflow=0;
+    sc->trace_q.q = malloc(sizeof(struct tws_trace_rec)*sc->trace_q.depth,
+                              M_TWS, M_WAITOK | M_ZERO);
+    if ( ! sc->trace_q.q )
+        return(FAILURE);
+    return(SUCCESS);
+}
+
+static int
+tws_init_reqs(struct tws_softc *sc, u_int32_t dma_mem_size)
+{
+
+    struct tws_command_packet *cmd_buf;
+    cmd_buf = (struct tws_command_packet *)sc->dma_mem;
+    int i;
+
+    bzero(cmd_buf, dma_mem_size);
+    TWS_TRACE_DEBUG(sc, "phy cmd", sc->dma_mem_phys, 0);
+    mtx_lock(&sc->q_lock);
+    for ( i=0; i< tws_queue_depth; i++)
+    {
+        if (bus_dmamap_create(sc->data_tag, 0, &sc->reqs[i].dma_map)) {
+            /* log a ENOMEM failure msg here */
+            return(FAILURE);
+        } 
+        sc->reqs[i].cmd_pkt =  &cmd_buf[i];
+
+        sc->sense_bufs[i].hdr = &cmd_buf[i].hdr ;
+        sc->sense_bufs[i].hdr_pkt_phy = sc->dma_mem_phys + 
+                              (i * sizeof(struct tws_command_packet));
+
+        sc->reqs[i].cmd_pkt_phy = sc->dma_mem_phys + 
+                              sizeof(struct tws_command_header) +
+                              (i * sizeof(struct tws_command_packet));
+        sc->reqs[i].request_id = i;
+        sc->reqs[i].sc = sc;
+
+        sc->reqs[i].cmd_pkt->hdr.header_desc.size_header = 128;
+
+        sc->reqs[i].state = TWS_REQ_STATE_FREE;
+        if ( i >= TWS_RESERVED_REQS )
+            tws_q_insert_tail(sc, &sc->reqs[i], TWS_FREE_Q);
+    }
+    mtx_unlock(&sc->q_lock);
+    return(SUCCESS);
+}
+
+static void
+tws_dmamap_cmds_load_cbfn(void *arg, bus_dma_segment_t *segs,
+                           int nseg, int error)
+{
+
+    /* printf("command load done \n"); */
+
+    *((bus_addr_t *)arg) = segs[0].ds_addr;
+}
+
+void
+tws_send_event(struct tws_softc *sc, u_int8_t event)
+{
+    mtx_assert(&sc->gen_lock, MA_OWNED);
+    TWS_TRACE_DEBUG(sc, "received event ", 0, event);
+    switch (event) {
+
+        case TWS_INIT_START:
+            sc->tws_state = TWS_INIT;
+            break;
+
+        case TWS_INIT_COMPLETE:
+            if (sc->tws_state != TWS_INIT) {
+                device_printf(sc->tws_dev, "invalid state transition %d => TWS_ONLINE\n", sc->tws_state);
+            } else {
+                sc->tws_state = TWS_ONLINE;
+            }
+            break;
+
+        case TWS_RESET_START:
+            /* We can transition to reset state from any state except reset*/ 
+            if (sc->tws_state != TWS_RESET) {
+                sc->tws_prev_state = sc->tws_state;
+                sc->tws_state = TWS_RESET;
+            }
+            break;
+
+        case TWS_RESET_COMPLETE:
+            if (sc->tws_state != TWS_RESET) {
+                device_printf(sc->tws_dev, "invalid state transition %d => %d (previous state)\n", sc->tws_state, sc->tws_prev_state);
+            } else {
+                sc->tws_state = sc->tws_prev_state;
+            }
+            break;
+
+        case TWS_SCAN_FAILURE:
+            if (sc->tws_state != TWS_ONLINE) {
+                device_printf(sc->tws_dev, "invalid state transition %d => TWS_OFFLINE\n", sc->tws_state);
+            } else {
+                sc->tws_state = TWS_OFFLINE;
+            }
+            break;
+
+        case TWS_UNINIT_START:
+            if ((sc->tws_state != TWS_ONLINE) && (sc->tws_state != TWS_OFFLINE)) {
+                device_printf(sc->tws_dev, "invalid state transition %d => TWS_UNINIT\n", sc->tws_state);
+            } else {
+                sc->tws_state = TWS_UNINIT;
+            }
+            break;
+    }
+
+}
+
+uint8_t
+tws_get_state(struct tws_softc *sc)
+{
+  
+    return((u_int8_t)sc->tws_state);
+
+}
+
+/* Called during system shutdown after sync. */
+
+static int
+tws_shutdown(device_t dev)
+{
+
+    struct tws_softc *sc = device_get_softc(dev);
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+
+    tws_turn_off_interrupts(sc);
+    tws_init_connect(sc, 1);
+
+    return (0);
+}
+
+/*
+ * Device suspend routine.
+ */
+static int
+tws_suspend(device_t dev)
+{
+    struct tws_softc *sc = device_get_softc(dev);
+
+    if ( sc )
+        TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    return (0);
+}
+
+/*
+ * Device resume routine.
+ */
+static int
+tws_resume(device_t dev)
+{
+
+    struct tws_softc *sc = device_get_softc(dev);
+
+    if ( sc )
+        TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    return (0);
+}
+
+
+struct tws_request *
+tws_get_request(struct tws_softc *sc, u_int16_t type)
+{
+    struct mtx *my_mutex = ((type == TWS_REQ_TYPE_SCSI_IO) ? &sc->q_lock : &sc->gen_lock);
+    struct tws_request *r = NULL;
+
+    mtx_lock(my_mutex);
+
+    if (type == TWS_REQ_TYPE_SCSI_IO) {
+        r = tws_q_remove_head(sc, TWS_FREE_Q);
+    } else {
+        if ( sc->reqs[type].state == TWS_REQ_STATE_FREE ) {
+            r = &sc->reqs[type];
+        }
+    }
+
+    if ( r ) {
+        bzero(&r->cmd_pkt->cmd, sizeof(struct tws_command_apache));
+        r->data = NULL;
+        r->length = 0;
+        r->type = type;
+        r->flags = TWS_DIR_UNKNOWN;
+        r->error_code = TWS_REQ_RET_INVALID;
+        r->cb = NULL;
+        r->ccb_ptr = NULL;
+        r->thandle.callout = NULL;
+        r->next = r->prev = NULL;
+
+        r->state = ((type == TWS_REQ_TYPE_SCSI_IO) ? TWS_REQ_STATE_TRAN : TWS_REQ_STATE_BUSY);
+    }
+
+    mtx_unlock(my_mutex);
+
+    return(r);
+}
+
+void
+tws_release_request(struct tws_request *req)
+{
+
+    struct tws_softc *sc = req->sc;
+
+    TWS_TRACE_DEBUG(sc, "entry", sc, 0);
+    mtx_lock(&sc->q_lock);
+    tws_q_insert_tail(sc, req, TWS_FREE_Q);
+    mtx_unlock(&sc->q_lock);
+}
+
+static device_method_t tws_methods[] = {
+    /* Device interface */
+    DEVMETHOD(device_probe,     tws_probe),
+    DEVMETHOD(device_attach,    tws_attach),
+    DEVMETHOD(device_detach,    tws_detach),
+    DEVMETHOD(device_shutdown,  tws_shutdown),
+    DEVMETHOD(device_suspend,   tws_suspend),
+    DEVMETHOD(device_resume,    tws_resume),
+
+    DEVMETHOD(bus_print_child,      bus_generic_print_child),
+    DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
+    { 0, 0 }
+};
+
+static driver_t tws_driver = {
+        "tws",
+        tws_methods,
+        sizeof(struct tws_softc)
+};
+
+
+static devclass_t tws_devclass;
+
+/* DEFINE_CLASS_0(tws, tws_driver, tws_methods, sizeof(struct tws_softc)); */
+DRIVER_MODULE(tws, pci, tws_driver, tws_devclass, 0, 0);
+MODULE_DEPEND(tws, cam, 1, 1, 1);
+MODULE_DEPEND(tws, pci, 1, 1, 1);
+
+TUNABLE_INT("hw.tws.queue_depth", &tws_queue_depth);
+TUNABLE_INT("hw.tws.enable_msi", &tws_enable_msi);
diff --git a/sys/dev/tws/tws.h b/sys/dev/tws/tws.h
new file mode 100644
index 000000000000..cf9f7275f523
--- /dev/null
+++ b/sys/dev/tws/tws.h
@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>        /* defines used in kernel.h */
+#include <sys/module.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/errno.h>
+#include <sys/kernel.h>       /* types used in module initialization */
+#include <sys/conf.h>         /* cdevsw struct */
+#include <sys/uio.h>          /* uio struct */
+#include <sys/malloc.h>
+#include <sys/bus.h>          /* structs, prototypes for pci bus stuff */
+
+
+#include <machine/bus.h>
+#include <sys/rman.h>
+#include <machine/resource.h>
+
+#include <dev/pci/pcivar.h>   /* For pci_get macros! */
+#include <dev/pci/pcireg.h>
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/stat.h>
+
+
+#define TWS_PULL_MODE_ENABLE 1
+
+MALLOC_DECLARE(M_TWS);
+/* externs */
+extern int tws_queue_depth;
+
+
+#define TWS_DRIVER_VERSION_STRING "10.80.00.003"
+#define TWS_MAX_NUM_UNITS             65 
+#define TWS_MAX_NUM_LUNS              16
+#define TWS_MAX_IRQS                  2
+#define TWS_SCSI_INITIATOR_ID         66
+#define TWS_MAX_IO_SIZE               0x20000 /* 128kB */
+#define TWS_SECTOR_SIZE               0x200
+#define TWS_POLL_TIMEOUT              60
+#define TWS_IO_TIMEOUT                60
+#define TWS_IOCTL_TIMEOUT             60
+#define TWS_RESET_TIMEOUT             60
+
+#define TWS_PCI_BAR0                  0x10
+#define TWS_PCI_BAR1                  0x14
+#define TWS_PCI_BAR2                  0x1C
+
+#define TWS_VENDOR_ID                 0x13C1 
+#define TWS_DEVICE_ID                 0x1010
+
+#define TWS_INVALID_REQID             0xFFFF
+
+/* bus tag related */
+#define TWS_ALIGNMENT                 4
+#define TWS_IN_MF_ALIGNMENT           16
+#define TWS_OUT_MF_ALIGNMENT          4
+
+#define TWS_MAX_32BIT_SG_ELEMENTS     93     /* max 32-bit sg elements */
+#define TWS_MAX_64BIT_SG_ELEMENTS     46     /* max 64-bit sg elements */
+
+#define TWS_MAX_QS                    4
+#define TWS_MAX_REQS                  256
+#define TWS_RESERVED_REQS             4
+
+/* Request states */
+#define TWS_REQ_STATE_FREE            0
+#define TWS_REQ_STATE_BUSY            1
+#define TWS_REQ_STATE_TRAN            2
+#define TWS_REQ_STATE_COMPLETE        3
+
+/* Request types */
+#define TWS_REQ_TYPE_INTERNAL_CMD     0x0
+#define TWS_REQ_TYPE_AEN_FETCH        0x1
+#define TWS_REQ_TYPE_PASSTHRU         0x2
+#define TWS_REQ_TYPE_GETSET_PARAM     0x3
+#define TWS_REQ_TYPE_SCSI_IO          0x4
+
+/* Driver states */
+
+enum tws_states {
+    TWS_INIT=50,
+    TWS_UNINIT,
+    TWS_OFFLINE,
+    TWS_ONLINE,
+    TWS_RESET,
+};
+
+/* events */
+
+enum tws_events {
+    TWS_INIT_START=100,
+    TWS_INIT_COMPLETE,
+    TWS_UNINIT_START,
+    TWS_RESET_START,
+    TWS_RESET_COMPLETE,
+    TWS_SCAN_FAILURE,
+};
+
+enum tws_req_flags {
+    TWS_DIR_UNKNOWN = 0x1,
+    TWS_DIR_IN = 0x2,
+    TWS_DIR_OUT = 0x4,
+    TWS_DIR_NONE = 0x8,
+};
+ 
+enum tws_intrs {
+     TWS_INTx,
+     TWS_MSI,
+     TWS_MSIX,
+};
+
+struct tws_msix_info {
+    int tbl_res_id;
+    bus_space_tag_t tbl_tag;
+    bus_space_handle_t tbl_handle;
+    struct resource *tbl_res;  
+};
+
+struct tws_ioctl_lock {
+    u_int32_t       lock;
+    time_t          timeout;
+};
+ 
+
+#define TWS_TRACE_FNAME_LEN  10
+#define TWS_TRACE_FUNC_LEN   15
+#define TWS_TRACE_DESC_LEN   10
+struct tws_trace_rec {
+    struct timespec ts;
+    char fname[TWS_TRACE_FNAME_LEN];
+    char func[TWS_TRACE_FUNC_LEN];
+    int linenum;
+    char desc[TWS_TRACE_DESC_LEN];
+    u_int64_t val1;
+    u_int64_t val2;
+};
+
+struct tws_circular_q {
+    volatile int16_t head;
+    volatile int16_t tail;
+    u_int16_t depth;
+    u_int8_t  overflow;
+    void *    q;
+};
+ 
+
+
+struct tws_stats {
+    u_int64_t reqs_in;
+    u_int64_t reqs_out;
+    u_int64_t reqs_errored;
+    u_int64_t spurios_intrs;
+    u_int64_t num_intrs;    
+    u_int64_t num_aens;    
+    u_int64_t ioctls;       
+    u_int64_t scsi_ios;
+};
+
+struct tws_init_connect_info {
+    u_int16_t     working_srl;
+    u_int16_t     working_branch;
+    u_int16_t     working_build;
+    u_int16_t     fw_on_ctlr_srl;
+    u_int16_t     fw_on_ctlr_branch;
+    u_int16_t     fw_on_ctlr_build;
+
+};
+
+
+/* ------------ boolean types ------------------- */
+
+typedef enum _boolean { false, true } boolean;
+enum err { SUCCESS, FAILURE };
+
+/* ----------- per instance data ---------------- */
+
+/* The softc holds our per-instance data. */
+struct tws_softc {
+    device_t    tws_dev;                  /* bus device */
+    struct cdev *tws_cdev;                /* controller device */
+    u_int32_t   device_id;                /* device id */
+    u_int32_t   subvendor_id;             /* device id */
+    u_int32_t   subdevice_id;             /* device id */
+    u_int8_t    tws_state;                /* driver state */
+    u_int8_t    tws_prev_state;           /* driver prev state */
+    struct sysctl_ctx_list tws_clist;     /* sysctl context */
+    struct sysctl_oid *tws_oidp;          /* sysctl context */
+    struct resource *reg_res;             /* register interface window */
+    struct resource *mfa_res;             /* mfa interface window */
+    int reg_res_id;                       /* register resource id */
+    int mfa_res_id;                       /* register resource id */
+    bus_space_handle_t bus_handle;        /* bus space handle */
+    bus_space_handle_t bus_mfa_handle;     /* bus space handle */
+    bus_space_tag_t bus_tag;              /* bus space tag */
+    bus_space_tag_t bus_mfa_tag;          /* bus space tag for mfa's */
+    u_int64_t mfa_base;                   /* mfa base address */
+    struct resource *irq_res[TWS_MAX_IRQS];/* interrupt resource */
+    int irq_res_id[TWS_MAX_IRQS];         /* intr resource id */
+    void *intr_handle[TWS_MAX_IRQS];      /* interrupt handle */
+    int irqs;                             /* intrs used */
+    struct tws_msix_info msix;            /* msix info */
+    struct cam_sim *sim;                  /* sim for this controller */
+    struct cam_path *path;                /* Ctlr path to CAM */
+    struct mtx q_lock;                    /* queue lock */
+    struct mtx sim_lock;                  /* sim lock */
+    struct mtx gen_lock;                  /* general driver  lock */
+    struct mtx io_lock;                   /* IO  lock */
+    struct tws_ioctl_lock ioctl_lock;     /* ioctl lock */ 
+    u_int32_t seq_id;                     /* Sequence id */
+    int chan;                             /* wait channel */
+    struct tws_circular_q aen_q;          /* aen q */
+    struct tws_circular_q trace_q;        /* trace q */
+    struct tws_stats stats;               /* I/O stats */
+    struct tws_init_connect_info cinfo;   /* compatibility info */
+    boolean is64bit;                      /* True - 64bit else 32bit */
+    u_int8_t intr_type;                   /* Interrupt type used */
+    bus_dma_tag_t parent_tag;             /* parent DMA tag */
+    bus_dma_tag_t cmd_tag;                /* command DMA tag */
+    bus_dmamap_t cmd_map;                 /* command map */
+    void *dma_mem;                        /* pointer to dmable memory */
+    u_int64_t dma_mem_phys;               /* phy addr */
+    bus_dma_tag_t data_tag;               /* data DMA tag */
+    struct tws_request *reqs;             /* pointer to requests */
+    struct tws_sense *sense_bufs;         /* pointer to sense buffers */
+    boolean obfl_q_overrun;               /* OBFL overrun flag  */
+    union ccb *scan_ccb;                  /* pointer to a ccb */
+    struct tws_request *q_head[TWS_MAX_QS]; /* head pointers to q's */
+    struct tws_request *q_tail[TWS_MAX_QS]; /* tail pointers to q's */
+};
diff --git a/sys/dev/tws/tws_cam.c b/sys/dev/tws/tws_cam.c
new file mode 100644
index 000000000000..df78b99fa7a7
--- /dev/null
+++ b/sys/dev/tws/tws_cam.c
@@ -0,0 +1,1342 @@
+/*
+ * Copyright (c) 2010 LSI Corp. 
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah <manjunath.ranganathaiah@lsi.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/tws/tws.h>
+#include <dev/tws/tws_services.h>
+#include <dev/tws/tws_hdm.h>
+#include <dev/tws/tws_user.h>
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/cam_sim.h>
+#include <cam/cam_xpt_sim.h>
+#include <cam/cam_debug.h>
+#include <cam/cam_periph.h>
+
+#include <cam/scsi/scsi_all.h>
+#include <cam/scsi/scsi_message.h>
+
+static int tws_cam_depth=(TWS_MAX_REQS - TWS_RESERVED_REQS);
+static char tws_sev_str[5][8]={"","ERROR","WARNING","INFO","DEBUG"};
+
+static void  tws_action(struct cam_sim *sim, union ccb *ccb);
+static void  tws_poll(struct cam_sim *sim);
+static void tws_scsi_complete(struct tws_request *req);
+
+
+
+void tws_unmap_request(struct tws_softc *sc, struct tws_request *req);
+int32_t tws_map_request(struct tws_softc *sc, struct tws_request *req);
+int tws_bus_scan(struct tws_softc *sc);
+int tws_cam_attach(struct tws_softc *sc);
+void tws_cam_detach(struct tws_softc *sc);
+void tws_reset(void *arg);
+
+static void tws_reset_cb(void *arg);
+static void tws_reinit(void *arg);
+static int32_t tws_execute_scsi(struct tws_softc *sc, union ccb *ccb);
+static void tws_freeze_simq(struct tws_softc *sc, struct tws_request *req);
+static void tws_dmamap_data_load_cbfn(void *arg, bus_dma_segment_t *segs,
+                            int nseg, int error);
+static void tws_fill_sg_list(struct tws_softc *sc, void *sgl_src, 
+                            void *sgl_dest, u_int16_t num_sgl_entries);
+static void tws_err_complete(struct tws_softc *sc, u_int64_t mfa);
+static void tws_scsi_err_complete(struct tws_request *req, 
+                                               struct tws_command_header *hdr);
+static void tws_passthru_err_complete(struct tws_request *req, 
+                                               struct tws_command_header *hdr);
+
+
+void tws_timeout(void *arg);
+static void tws_intr_attn_aen(struct tws_softc *sc);
+static void tws_intr_attn_error(struct tws_softc *sc);
+static void tws_intr_resp(struct tws_softc *sc);
+void tws_intr(void *arg);
+void tws_cmd_complete(struct tws_request *req);
+void tws_aen_complete(struct tws_request *req);
+int tws_send_scsi_cmd(struct tws_softc *sc, int cmd);
+void tws_getset_param_complete(struct tws_request *req);
+int tws_set_param(struct tws_softc *sc, u_int32_t table_id, u_int32_t param_id,
+              u_int32_t param_size, void *data);
+int tws_get_param(struct tws_softc *sc, u_int32_t table_id, u_int32_t param_id,  
+              u_int32_t param_size, void *data);
+
+
+extern struct tws_request *tws_get_request(struct tws_softc *sc, 
+                                            u_int16_t type);
+extern void *tws_release_request(struct tws_request *req);
+extern int tws_submit_command(struct tws_softc *sc, struct tws_request *req);
+extern boolean tws_get_response(struct tws_softc *sc, 
+                                           u_int16_t *req_id, u_int64_t *mfa);
+extern void tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req,
+                                u_int8_t q_type );
+extern struct tws_request * tws_q_remove_request(struct tws_softc *sc,
+                                   struct tws_request *req, u_int8_t q_type );
+extern void tws_send_event(struct tws_softc *sc, u_int8_t event);
+
+extern struct tws_sense *
+tws_find_sense_from_mfa(struct tws_softc *sc, u_int64_t mfa);
+
+extern void tws_fetch_aen(void *arg);
+extern void tws_disable_db_intr(struct tws_softc *sc);
+extern void tws_enable_db_intr(struct tws_softc *sc);
+extern void tws_passthru_complete(struct tws_request *req);
+extern void tws_aen_synctime_with_host(struct tws_softc *sc);
+extern void tws_circular_aenq_insert(struct tws_softc *sc, 
+                    struct tws_circular_q *cq, struct tws_event_packet *aen);
+extern int tws_use_32bit_sgls;
+extern boolean tws_ctlr_reset(struct tws_softc *sc);
+extern struct tws_request * tws_q_remove_tail(struct tws_softc *sc, 
+                                                           u_int8_t q_type );
+extern void tws_turn_off_interrupts(struct tws_softc *sc);
+extern void tws_turn_on_interrupts(struct tws_softc *sc);
+extern int tws_init_connect(struct tws_softc *sc, u_int16_t mc);
+extern void tws_init_obfl_q(struct tws_softc *sc);
+extern uint8_t tws_get_state(struct tws_softc *sc);
+extern void tws_assert_soft_reset(struct tws_softc *sc);
+extern boolean tws_ctlr_ready(struct tws_softc *sc);
+extern u_int16_t tws_poll4_response(struct tws_softc *sc, u_int64_t *mfa);
+extern int tws_setup_intr(struct tws_softc *sc, int irqs);
+extern int tws_teardown_intr(struct tws_softc *sc);
+
+
+
+int
+tws_cam_attach(struct tws_softc *sc)
+{
+    struct cam_devq *devq;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, sc);
+    /* Create a device queue for sim */
+
+    /* 
+     * if the user sets cam depth to less than 1 
+     * cam may get confused 
+     */
+    if ( tws_cam_depth < 1 )
+        tws_cam_depth = 1;
+    if ( tws_cam_depth > (tws_queue_depth - TWS_RESERVED_REQS)  )
+        tws_cam_depth = tws_queue_depth - TWS_RESERVED_REQS;
+
+    TWS_TRACE_DEBUG(sc, "depths,ctlr,cam", tws_queue_depth, tws_cam_depth);
+
+    if ((devq = cam_simq_alloc(tws_cam_depth)) == NULL) {
+        tws_log(sc, CAM_SIMQ_ALLOC);
+        return(ENOMEM);
+    }
+
+   /*
+    * Create a SIM entry.  Though we can support tws_cam_depth
+    * simultaneous requests, we claim to be able to handle only
+    * (tws_cam_depth), so that we always have reserved  requests
+    * packet available to service ioctls and internal commands.
+    */
+    sc->sim = cam_sim_alloc(tws_action, tws_poll, "tws", sc,
+                      device_get_unit(sc->tws_dev), 
+#if (__FreeBSD_version >= 700000)
+                      &sc->sim_lock,
+#endif
+                      tws_cam_depth, 1, devq);
+                      /* 1, 1, devq); */
+    if (sc->sim == NULL) {
+        cam_simq_free(devq);
+        tws_log(sc, CAM_SIM_ALLOC);
+    }
+    /* Register the bus. */
+    mtx_lock(&sc->sim_lock);
+    if (xpt_bus_register(sc->sim, 
+#if (__FreeBSD_version >= 700000)
+                         sc->tws_dev, 
+#endif
+                         0) != CAM_SUCCESS) {
+        cam_sim_free(sc->sim, TRUE); /* passing true will free the devq */
+        sc->sim = NULL; /* so cam_detach will not try to free it */
+        mtx_unlock(&sc->sim_lock);
+        tws_log(sc, TWS_XPT_BUS_REGISTER);
+        return(ENXIO);
+    }
+    if (xpt_create_path(&sc->path, NULL, cam_sim_path(sc->sim),
+                         CAM_TARGET_WILDCARD,
+                         CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
+        xpt_bus_deregister(cam_sim_path(sc->sim));
+        /* Passing TRUE to cam_sim_free will free the devq as well. */
+        cam_sim_free(sc->sim, TRUE);
+        tws_log(sc, TWS_XPT_CREATE_PATH);
+        mtx_unlock(&sc->sim_lock);
+        return(ENXIO);
+    }
+    mtx_unlock(&sc->sim_lock);
+
+    return(0);
+}
+
+void
+tws_cam_detach(struct tws_softc *sc)
+{
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    mtx_lock(&sc->sim_lock);
+    if (sc->path)
+        xpt_free_path(sc->path);
+    if (sc->sim) {
+        xpt_bus_deregister(cam_sim_path(sc->sim));
+        cam_sim_free(sc->sim, TRUE);
+    }
+    mtx_unlock(&sc->sim_lock);
+}
+
+int
+tws_bus_scan(struct tws_softc *sc)
+{
+    union ccb       *ccb;
+
+    TWS_TRACE_DEBUG(sc, "entry", sc, 0);
+    if (!(sc->sim))
+        return(ENXIO);
+    mtx_assert(&sc->sim_lock, MA_OWNED);
+    if ((ccb = xpt_alloc_ccb()) == NULL)
+		    return(ENOMEM);
+
+    if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, cam_sim_path(sc->sim),
+                  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
+        xpt_free_ccb(ccb);
+        return(EIO);
+    }
+    xpt_rescan(ccb);
+    
+    return(0);
+}
+
+static void
+tws_action(struct cam_sim *sim, union ccb *ccb)
+{
+    struct tws_softc *sc = (struct tws_softc *)cam_sim_softc(sim);
+
+
+    switch( ccb->ccb_h.func_code ) {
+        case XPT_SCSI_IO:   
+        {
+            if ( tws_execute_scsi(sc, ccb) ) 
+                TWS_TRACE_DEBUG(sc, "execute scsi failed", 0, 0);
+            break;
+        }
+        case XPT_ABORT:
+        {
+            TWS_TRACE_DEBUG(sc, "abort i/o", 0, 0);
+            ccb->ccb_h.status = CAM_UA_ABORT;
+            xpt_done(ccb);
+            break;
+        }
+        case XPT_RESET_BUS:
+        {
+            TWS_TRACE_DEBUG(sc, "reset bus", sim, ccb);
+            break;
+        }
+        case XPT_SET_TRAN_SETTINGS:
+        {
+            TWS_TRACE_DEBUG(sc, "set tran settings", sim, ccb);
+            ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
+            xpt_done(ccb);
+
+            break;
+        }
+        case XPT_GET_TRAN_SETTINGS:
+        {
+            TWS_TRACE_DEBUG(sc, "get tran settings", sim, ccb);
+
+#if (__FreeBSD_version >= 700000 )
+            ccb->cts.protocol = PROTO_SCSI;
+            ccb->cts.protocol_version = SCSI_REV_2;
+            ccb->cts.transport = XPORT_SPI;
+            ccb->cts.transport_version = 2;
+
+            ccb->cts.xport_specific.spi.valid = CTS_SPI_VALID_DISC;
+            ccb->cts.xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
+            ccb->cts.proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
+            ccb->cts.proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
+#else
+            ccb->cts.valid = (CCB_TRANS_DISC_VALID | CCB_TRANS_TQ_VALID);
+            ccb->cts.flags &= ~(CCB_TRANS_DISC_ENB | CCB_TRANS_TAG_ENB);
+#endif
+            ccb->ccb_h.status = CAM_REQ_CMP;
+            xpt_done(ccb);
+
+            break;
+        }
+        case XPT_CALC_GEOMETRY:
+        {
+            TWS_TRACE_DEBUG(sc, "calc geometry(ccb,block-size)", ccb, 
+                                          ccb->ccg.block_size);
+            cam_calc_geometry(&ccb->ccg, 1/* extended */);
+            xpt_done(ccb);
+
+            break;
+        }
+        case XPT_PATH_INQ:
+        {
+            TWS_TRACE_DEBUG(sc, "path inquiry", sim, ccb);
+            ccb->cpi.version_num = 1;
+            ccb->cpi.hba_inquiry = 0;
+            ccb->cpi.target_sprt = 0;
+            ccb->cpi.hba_misc = 0;
+            ccb->cpi.hba_eng_cnt = 0;
+            ccb->cpi.max_target = TWS_MAX_NUM_UNITS;
+            ccb->cpi.max_lun = TWS_MAX_NUM_LUNS - 1;
+            ccb->cpi.unit_number = cam_sim_unit(sim);
+            ccb->cpi.bus_id = cam_sim_bus(sim);
+            ccb->cpi.initiator_id = TWS_SCSI_INITIATOR_ID;
+            ccb->cpi.base_transfer_speed = 6000000;
+            strncpy(ccb->cpi.sim_vid, "FreeBSD", SIM_IDLEN);
+            strncpy(ccb->cpi.hba_vid, "3ware", HBA_IDLEN);
+            strncpy(ccb->cpi.dev_name, cam_sim_name(sim), DEV_IDLEN);
+#if (__FreeBSD_version >= 700000 )
+            ccb->cpi.transport = XPORT_SPI;
+            ccb->cpi.transport_version = 2;
+            ccb->cpi.protocol = PROTO_SCSI;
+            ccb->cpi.protocol_version = SCSI_REV_2;
+            ccb->cpi.maxio = TWS_MAX_IO_SIZE;
+#endif
+            ccb->ccb_h.status = CAM_REQ_CMP;
+            xpt_done(ccb);
+
+            break;
+        }
+        default:
+            TWS_TRACE_DEBUG(sc, "default", sim, ccb);
+            ccb->ccb_h.status = CAM_REQ_INVALID;
+            xpt_done(ccb);
+            break;
+    }
+}
+
+static void
+tws_scsi_complete(struct tws_request *req)
+{
+    struct tws_softc *sc = req->sc;
+
+    mtx_lock(&sc->q_lock);
+    tws_q_remove_request(sc, req, TWS_BUSY_Q);
+    mtx_unlock(&sc->q_lock);
+
+    untimeout(tws_timeout, req, req->ccb_ptr->ccb_h.timeout_ch);
+    tws_unmap_request(req->sc, req);
+
+
+    req->ccb_ptr->ccb_h.status = CAM_REQ_CMP;
+    mtx_lock(&sc->sim_lock);
+    xpt_done(req->ccb_ptr);
+    mtx_unlock(&sc->sim_lock);
+
+    mtx_lock(&sc->q_lock);
+    tws_q_insert_tail(sc, req, TWS_FREE_Q);
+    mtx_unlock(&sc->q_lock);
+}
+
+void
+tws_getset_param_complete(struct tws_request *req)
+{
+    struct tws_softc *sc = req->sc;
+
+    TWS_TRACE_DEBUG(sc, "getset complete", req, req->request_id);
+
+    untimeout(tws_timeout, req, req->thandle);
+    tws_unmap_request(sc, req);
+
+    free(req->data, M_TWS);
+
+    req->state = TWS_REQ_STATE_FREE;
+}
+
+void
+tws_aen_complete(struct tws_request *req)
+{
+    struct tws_softc *sc = req->sc;
+    struct tws_command_header *sense;
+    struct tws_event_packet event;
+    u_int16_t aen_code=0;
+
+    TWS_TRACE_DEBUG(sc, "aen complete", 0, req->request_id);
+
+    untimeout(tws_timeout, req, req->thandle);
+    tws_unmap_request(sc, req);
+
+    sense = (struct tws_command_header *)req->data;
+
+    TWS_TRACE_DEBUG(sc,"sense code, key",sense->sense_data[0], 
+                                   sense->sense_data[2]);
+    TWS_TRACE_DEBUG(sc,"sense rid, seve",sense->header_desc.request_id, 
+                                   sense->status_block.res__severity);
+    TWS_TRACE_DEBUG(sc,"sense srcnum, error",sense->status_block.srcnum, 
+                                   sense->status_block.error);
+    TWS_TRACE_DEBUG(sc,"sense shdr, ssense",sense->header_desc.size_header, 
+                                   sense->header_desc.size_sense);
+
+    aen_code = sense->status_block.error;
+
+    switch ( aen_code ) {
+        case TWS_AEN_SYNC_TIME_WITH_HOST :
+            tws_aen_synctime_with_host(sc);
+            break;
+        case TWS_AEN_QUEUE_EMPTY :
+            break;
+        default :
+            bzero(&event, sizeof(struct tws_event_packet));
+            event.sequence_id = sc->seq_id;
+            event.time_stamp_sec = (u_int32_t)TWS_LOCAL_TIME;
+            event.aen_code = sense->status_block.error;
+            event.severity = sense->status_block.res__severity & 0x7;
+            event.event_src = TWS_SRC_CTRL_EVENT;
+            strcpy(event.severity_str, tws_sev_str[event.severity]);
+            event.retrieved = TWS_AEN_NOT_RETRIEVED;
+
+            bcopy(sense->err_specific_desc, event.parameter_data, 
+                                    TWS_ERROR_SPECIFIC_DESC_LEN);
+            event.parameter_data[TWS_ERROR_SPECIFIC_DESC_LEN - 1] = '\0';
+            event.parameter_len = (u_int8_t)strlen(event.parameter_data)+1;
+
+            if ( event.parameter_len < TWS_ERROR_SPECIFIC_DESC_LEN ) {
+                event.parameter_len += ((u_int8_t)strlen(event.parameter_data +
+                                                event.parameter_len) + 1);
+            }
+
+            device_printf(sc->tws_dev, "%s: (0x%02X: 0x%04X): %s: %s\n",
+                event.severity_str,
+                event.event_src,
+                event.aen_code,
+                event.parameter_data + 
+                     (strlen(event.parameter_data) + 1), 
+                event.parameter_data);
+
+            mtx_lock(&sc->gen_lock);
+            tws_circular_aenq_insert(sc, &sc->aen_q, &event);
+            sc->seq_id++;
+            mtx_unlock(&sc->gen_lock);
+            break;
+
+    }
+    
+    free(req->data, M_TWS);
+
+    req->state = TWS_REQ_STATE_FREE;
+
+    if ( aen_code != TWS_AEN_QUEUE_EMPTY ) {
+        /* timeout(tws_fetch_aen, sc, 1);*/
+        sc->stats.num_aens++;
+        tws_fetch_aen((void *)sc);
+    } 
+}
+
+void
+tws_cmd_complete(struct tws_request *req)
+{
+    struct tws_softc *sc = req->sc;
+
+    untimeout(tws_timeout, req, req->ccb_ptr->ccb_h.timeout_ch);
+    tws_unmap_request(sc, req);
+}
+                                   
+static void
+tws_err_complete(struct tws_softc *sc, u_int64_t mfa)
+{
+    struct tws_command_header *hdr;
+    struct tws_sense *sen;
+    struct tws_request *req;
+    u_int16_t req_id;
+    u_int32_t reg, status;
+
+    if ( !mfa ) {
+        TWS_TRACE_DEBUG(sc, "null mfa", 0, mfa);
+        return;
+    } else {
+        /* lookup the sense */
+        sen = tws_find_sense_from_mfa(sc, mfa);
+        if ( sen == NULL ) {
+            TWS_TRACE_DEBUG(sc, "found null req", 0, mfa);
+            return;
+        }
+        hdr = sen->hdr;
+        TWS_TRACE_DEBUG(sc, "sen, hdr", sen, hdr);
+        req_id = hdr->header_desc.request_id;
+        req = &sc->reqs[req_id];
+        TWS_TRACE_DEBUG(sc, "req, id", req, req_id);
+        if ( req->error_code != TWS_REQ_RET_SUBMIT_SUCCESS )
+            TWS_TRACE_DEBUG(sc, "submit failure?", 0, req->error_code);
+    }
+
+    switch (req->type) {
+        case TWS_REQ_TYPE_PASSTHRU :
+            tws_passthru_err_complete(req, hdr);
+            break;
+        case TWS_REQ_TYPE_GETSET_PARAM :
+            tws_getset_param_complete(req);
+            break;
+        case TWS_REQ_TYPE_SCSI_IO :
+            tws_scsi_err_complete(req, hdr);
+            break;
+            
+    }
+
+    mtx_lock(&sc->io_lock);
+    hdr->header_desc.size_header = 128;
+    reg = (u_int32_t)( mfa>>32);
+    tws_write_reg(sc, TWS_I2O0_HOBQPH, reg, 4);
+    reg = (u_int32_t)(mfa);
+    tws_write_reg(sc, TWS_I2O0_HOBQPL, reg, 4);
+
+    status = tws_read_reg(sc, TWS_I2O0_STATUS, 4);
+    if ( status & TWS_BIT13 ) {
+        device_printf(sc->tws_dev,  "OBFL Overrun\n");
+        sc->obfl_q_overrun = true;
+    }
+    mtx_unlock(&sc->io_lock);
+}
+
+static void
+tws_scsi_err_complete(struct tws_request *req, struct tws_command_header *hdr)
+{ 
+    u_int8_t *sense_data;
+    struct tws_softc *sc = req->sc;
+    union ccb *ccb = req->ccb_ptr;
+
+    TWS_TRACE_DEBUG(sc, "sbe, cmd_status", hdr->status_block.error, 
+                                 req->cmd_pkt->cmd.pkt_a.status);
+    if ( hdr->status_block.error == TWS_ERROR_LOGICAL_UNIT_NOT_SUPPORTED ||
+         hdr->status_block.error == TWS_ERROR_UNIT_OFFLINE ) {
+
+        if ( ccb->ccb_h.target_lun ) {
+            TWS_TRACE_DEBUG(sc, "invalid lun error",0,0);
+            ccb->ccb_h.status |= CAM_LUN_INVALID;
+        } else {
+            TWS_TRACE_DEBUG(sc, "invalid target error",0,0);
+            ccb->ccb_h.status |= CAM_TID_INVALID;
+        }
+
+    } else {
+        TWS_TRACE_DEBUG(sc, "scsi status  error",0,0);
+        ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
+        if (((ccb->csio.cdb_io.cdb_bytes[0] == 0x1A) && 
+              (hdr->status_block.error == TWS_ERROR_NOT_SUPPORTED))) {
+            ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
+            TWS_TRACE_DEBUG(sc, "page mode not supported",0,0);
+        }
+    }
+
+    /* if there were no error simply mark complete error */ 
+    if (ccb->ccb_h.status == 0)
+        ccb->ccb_h.status = CAM_REQ_CMP_ERR;
+
+    sense_data = (u_int8_t *)&ccb->csio.sense_data;
+    if (sense_data) {
+        memcpy(sense_data, hdr->sense_data, TWS_SENSE_DATA_LENGTH );
+        ccb->csio.sense_len = TWS_SENSE_DATA_LENGTH;
+        ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
+    }
+    ccb->csio.scsi_status = req->cmd_pkt->cmd.pkt_a.status;
+ 
+    ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
+    mtx_lock(&sc->sim_lock);
+    xpt_done(ccb);
+    mtx_unlock(&sc->sim_lock);
+
+    untimeout(tws_timeout, req, req->ccb_ptr->ccb_h.timeout_ch);
+    tws_unmap_request(req->sc, req);
+    mtx_lock(&sc->q_lock);
+    tws_q_remove_request(sc, req, TWS_BUSY_Q);
+    tws_q_insert_tail(sc, req, TWS_FREE_Q);
+    mtx_unlock(&sc->q_lock);
+}
+
+static void
+tws_passthru_err_complete(struct tws_request *req, 
+                                          struct tws_command_header *hdr)
+{ 
+    TWS_TRACE_DEBUG(req->sc, "entry", hdr, req->request_id);
+    req->error_code = hdr->status_block.error;
+    memcpy(&(req->cmd_pkt->hdr), hdr, sizeof(struct tws_command_header));
+    tws_passthru_complete(req);
+}
+
+static void
+tws_drain_busy_queue(struct tws_softc *sc)
+{
+    struct tws_request *req;
+    union ccb          *ccb;
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+
+    mtx_lock(&sc->q_lock);
+    req = tws_q_remove_tail(sc, TWS_BUSY_Q);
+    mtx_unlock(&sc->q_lock);
+    while ( req ) {
+        TWS_TRACE_DEBUG(sc, "moved to TWS_COMPLETE_Q", 0, req->request_id);
+        untimeout(tws_timeout, req, req->ccb_ptr->ccb_h.timeout_ch);
+
+        req->error_code = TWS_REQ_RET_RESET;
+        ccb = (union ccb *)(req->ccb_ptr);
+
+        ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
+        ccb->ccb_h.status |=  CAM_REQUEUE_REQ;
+        ccb->ccb_h.status |=  CAM_SCSI_BUS_RESET;
+
+        tws_unmap_request(req->sc, req);
+
+        mtx_lock(&sc->sim_lock);
+        xpt_done(req->ccb_ptr);
+        mtx_unlock(&sc->sim_lock);
+
+        mtx_lock(&sc->q_lock);
+        tws_q_insert_tail(sc, req, TWS_FREE_Q);
+        req = tws_q_remove_tail(sc, TWS_BUSY_Q);
+        mtx_unlock(&sc->q_lock);
+    } 
+}
+
+
+static void
+tws_drain_reserved_reqs(struct tws_softc *sc)
+{
+    struct tws_request *r;
+
+    r = &sc->reqs[TWS_REQ_TYPE_AEN_FETCH];
+    if ( r->state != TWS_REQ_STATE_FREE ) {
+        TWS_TRACE_DEBUG(sc, "reset aen req", 0, 0);
+        untimeout(tws_timeout, r, r->thandle);
+        tws_unmap_request(sc, r);
+        free(r->data, M_TWS);
+        r->state = TWS_REQ_STATE_FREE;
+        r->error_code = TWS_REQ_RET_RESET;
+    } 
+
+    r = &sc->reqs[TWS_REQ_TYPE_PASSTHRU];
+    if ( r->state == TWS_REQ_STATE_BUSY ) {
+        TWS_TRACE_DEBUG(sc, "reset passthru req", 0, 0);
+        r->error_code = TWS_REQ_RET_RESET;
+    } 
+
+    r = &sc->reqs[TWS_REQ_TYPE_GETSET_PARAM];
+    if ( r->state != TWS_REQ_STATE_FREE ) {
+        TWS_TRACE_DEBUG(sc, "reset setparam req", 0, 0);
+        untimeout(tws_timeout, r, r->thandle);
+        tws_unmap_request(sc, r);
+        free(r->data, M_TWS);
+        r->state = TWS_REQ_STATE_FREE;
+        r->error_code = TWS_REQ_RET_RESET;
+    } 
+}
+
+static void
+tws_drain_response_queue(struct tws_softc *sc)
+{
+    u_int16_t req_id;
+    u_int64_t mfa;
+    while ( tws_get_response(sc, &req_id, &mfa) );
+}
+
+
+static int32_t
+tws_execute_scsi(struct tws_softc *sc, union ccb *ccb)
+{
+    struct tws_command_packet *cmd_pkt;
+    struct tws_request *req;
+    struct ccb_hdr *ccb_h = &(ccb->ccb_h);
+    struct ccb_scsiio *csio = &(ccb->csio);
+    int error;
+    u_int16_t lun;
+
+    mtx_assert(&sc->sim_lock, MA_OWNED);
+    if (ccb_h->target_id >= TWS_MAX_NUM_UNITS) {
+        TWS_TRACE_DEBUG(sc, "traget id too big", ccb_h->target_id, ccb_h->target_lun);
+        ccb_h->status |= CAM_TID_INVALID;
+        xpt_done(ccb);
+        return(0);
+    }
+    if (ccb_h->target_lun >= TWS_MAX_NUM_LUNS) {
+        TWS_TRACE_DEBUG(sc, "target lun 2 big", ccb_h->target_id, ccb_h->target_lun);
+        ccb_h->status |= CAM_LUN_INVALID;
+        xpt_done(ccb);
+        return(0);
+    }
+
+    if(ccb_h->flags & CAM_CDB_PHYS) {
+        TWS_TRACE_DEBUG(sc, "cdb phy", ccb_h->target_id, ccb_h->target_lun);
+        ccb_h->status = CAM_REQ_INVALID;
+        xpt_done(ccb);
+        return(0);
+    }
+
+    /*
+     * We are going to work on this request.  Mark it as enqueued (though
+     * we don't actually queue it...)
+     */
+    ccb_h->status |= CAM_SIM_QUEUED;
+
+    req = tws_get_request(sc, TWS_REQ_TYPE_SCSI_IO);
+    if ( !req ) {
+        TWS_TRACE_DEBUG(sc, "no reqs", ccb_h->target_id, ccb_h->target_lun);
+        ccb_h->status |= CAM_REQUEUE_REQ;
+        xpt_done(ccb);
+        return(0);
+    }
+
+    if((ccb_h->flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+        if(ccb_h->flags & CAM_DIR_IN)
+            req->flags |= TWS_DIR_IN;
+        if(ccb_h->flags & CAM_DIR_OUT)
+            req->flags |= TWS_DIR_OUT;
+    } else {
+        req->flags = TWS_DIR_NONE; /* no data */
+    }
+
+    req->type = TWS_REQ_TYPE_SCSI_IO;
+    req->cb = tws_scsi_complete;
+
+    cmd_pkt = req->cmd_pkt;
+    /* cmd_pkt->hdr.header_desc.size_header = 128; */
+    cmd_pkt->cmd.pkt_a.res__opcode = TWS_FW_CMD_EXECUTE_SCSI;
+    cmd_pkt->cmd.pkt_a.unit = ccb_h->target_id;
+    cmd_pkt->cmd.pkt_a.status = 0;
+    cmd_pkt->cmd.pkt_a.sgl_offset = 16;
+
+    /* lower nibble */
+    lun = ccb_h->target_lun & 0XF;
+    lun = lun << 12;
+    cmd_pkt->cmd.pkt_a.lun_l4__req_id = lun | req->request_id;
+    /* upper nibble */
+    lun = ccb_h->target_lun & 0XF0;
+    lun = lun << 8;
+    cmd_pkt->cmd.pkt_a.lun_h4__sgl_entries = lun;
+
+#ifdef TWS_DEBUG 
+    if ( csio->cdb_len > 16 ) 
+         TWS_TRACE(sc, "cdb len too big", ccb_h->target_id, csio->cdb_len);
+#endif
+
+    if(ccb_h->flags & CAM_CDB_POINTER)
+        bcopy(csio->cdb_io.cdb_ptr, cmd_pkt->cmd.pkt_a.cdb, csio->cdb_len);
+    else
+        bcopy(csio->cdb_io.cdb_bytes, cmd_pkt->cmd.pkt_a.cdb, csio->cdb_len);
+
+    if (!(ccb_h->flags & CAM_DATA_PHYS)) {
+         /* Virtual data addresses.  Need to convert them... */
+         if (!(ccb_h->flags & CAM_SCATTER_VALID)) {
+             if (csio->dxfer_len > TWS_MAX_IO_SIZE) {
+                 TWS_TRACE(sc, "I/O is big", csio->dxfer_len, 0);
+                 tws_release_request(req);
+                 ccb_h->status = CAM_REQ_TOO_BIG;
+                 xpt_done(ccb);
+                 return(0);
+             }
+
+             req->length = csio->dxfer_len;
+             if (req->length) {
+                 req->data = csio->data_ptr;
+                 /* there is 1 sgl_entrie */
+                 /* cmd_pkt->cmd.pkt_a.lun_h4__sgl_entries |= 1; */
+             }
+         } else {
+             TWS_TRACE_DEBUG(sc, "got sglist", ccb_h->target_id, ccb_h->target_lun);
+             tws_release_request(req);
+             ccb_h->status = CAM_REQ_INVALID;
+             xpt_done(ccb);
+             return(0);
+         }
+    } else {
+         /* Data addresses are physical. */
+         TWS_TRACE_DEBUG(sc, "Phy data addr", ccb_h->target_id, ccb_h->target_lun);
+         tws_release_request(req);
+         ccb_h->status = CAM_REQ_INVALID;
+         ccb_h->status &= ~CAM_SIM_QUEUED;
+         xpt_done(ccb);
+         return(0);
+    }
+    /* save ccb ptr */
+    req->ccb_ptr = ccb;
+    /* 
+     * tws_map_load_data_callback will fill in the SGL,
+     * and submit the I/O.
+     */
+    sc->stats.scsi_ios++;
+    ccb_h->timeout_ch = timeout(tws_timeout, req, (ccb_h->timeout * hz)/1000);
+    error = tws_map_request(sc, req);
+    return(error);
+}
+
+
+int
+tws_send_scsi_cmd(struct tws_softc *sc, int cmd)
+{
+    struct tws_request *req;
+    struct tws_command_packet *cmd_pkt;
+    int error;
+
+    TWS_TRACE_DEBUG(sc, "entry",sc, cmd);
+    req = tws_get_request(sc, TWS_REQ_TYPE_AEN_FETCH);
+
+    if ( req == NULL )
+        return(ENOMEM);
+
+    req->cb = tws_aen_complete;
+
+    cmd_pkt = req->cmd_pkt;
+    cmd_pkt->cmd.pkt_a.res__opcode = TWS_FW_CMD_EXECUTE_SCSI;
+    cmd_pkt->cmd.pkt_a.status = 0;
+    cmd_pkt->cmd.pkt_a.unit = 0;
+    cmd_pkt->cmd.pkt_a.sgl_offset = 16;
+    cmd_pkt->cmd.pkt_a.lun_l4__req_id = req->request_id;
+
+    cmd_pkt->cmd.pkt_a.cdb[0] = (u_int8_t)cmd;
+    cmd_pkt->cmd.pkt_a.cdb[4] = 128;
+
+    req->length = TWS_SECTOR_SIZE;
+    req->data = malloc(TWS_SECTOR_SIZE, M_TWS, M_NOWAIT);
+    if ( req->data == NULL )
+        return(ENOMEM);
+    bzero(req->data, TWS_SECTOR_SIZE);
+    req->flags = TWS_DIR_IN;
+
+    req->thandle = timeout(tws_timeout, req, (TWS_IO_TIMEOUT * hz));
+    error = tws_map_request(sc, req);
+    return(error);
+
+}
+
+int
+tws_set_param(struct tws_softc *sc, u_int32_t table_id, u_int32_t param_id,
+              u_int32_t param_size, void *data)
+{
+    struct tws_request *req;
+    struct tws_command_packet *cmd_pkt;
+    union tws_command_giga *cmd;
+    struct tws_getset_param *param;
+    int error;
+
+    req = tws_get_request(sc, TWS_REQ_TYPE_GETSET_PARAM);
+    if ( req == NULL ) {
+        TWS_TRACE_DEBUG(sc, "null req", 0, 0);
+        return(ENOMEM);
+    }
+
+    req->length = TWS_SECTOR_SIZE;
+    req->data = malloc(TWS_SECTOR_SIZE, M_TWS, M_NOWAIT);
+    if ( req->data == NULL )
+        return(ENOMEM);
+    bzero(req->data, TWS_SECTOR_SIZE);
+    param = (struct tws_getset_param *)req->data;
+
+    req->cb = tws_getset_param_complete;
+    req->flags = TWS_DIR_OUT;
+    cmd_pkt = req->cmd_pkt;
+
+    cmd = &cmd_pkt->cmd.pkt_g;
+    cmd->param.sgl_off__opcode =
+            BUILD_SGL_OFF__OPCODE(2, TWS_FW_CMD_SET_PARAM);
+    cmd->param.request_id = (u_int8_t)req->request_id;
+    cmd->param.host_id__unit = 0;
+    cmd->param.param_count = 1;
+    cmd->param.size = 2; /* map routine will add sgls */
+
+    /* Specify which parameter we want to set. */
+    param->table_id = (table_id | TWS_9K_PARAM_DESCRIPTOR);
+    param->parameter_id = (u_int8_t)(param_id);
+    param->parameter_size_bytes = (u_int16_t)param_size;
+    memcpy(param->data, data, param_size);
+
+    req->thandle = timeout(tws_timeout, req, (TWS_IOCTL_TIMEOUT * hz));
+    error = tws_map_request(sc, req);
+    return(error);
+
+}
+
+int
+tws_get_param(struct tws_softc *sc, u_int32_t table_id, u_int32_t param_id,
+              u_int32_t param_size, void *data)
+{
+    struct tws_request *req;
+    struct tws_command_packet *cmd_pkt;
+    union tws_command_giga *cmd;
+    struct tws_getset_param *param;
+    u_int16_t reqid;
+    u_int64_t mfa;
+    int error = SUCCESS;
+
+
+    req = tws_get_request(sc, TWS_REQ_TYPE_GETSET_PARAM);
+    if ( req == NULL ) {
+        TWS_TRACE_DEBUG(sc, "null req", 0, 0);
+        return(FAILURE);
+    }
+
+    req->length = TWS_SECTOR_SIZE;
+    req->data = malloc(TWS_SECTOR_SIZE, M_TWS, M_NOWAIT);
+    if ( req->data == NULL )
+        return(FAILURE);
+    bzero(req->data, TWS_SECTOR_SIZE);
+    param = (struct tws_getset_param *)req->data;
+
+    req->cb = NULL;
+    req->flags = TWS_DIR_IN;
+    cmd_pkt = req->cmd_pkt;
+
+    cmd = &cmd_pkt->cmd.pkt_g;
+    cmd->param.sgl_off__opcode =
+            BUILD_SGL_OFF__OPCODE(2, TWS_FW_CMD_GET_PARAM);
+    cmd->param.request_id = (u_int8_t)req->request_id;
+    cmd->param.host_id__unit = 0;
+    cmd->param.param_count = 1;
+    cmd->param.size = 2; /* map routine will add sgls */
+
+    /* Specify which parameter we want to set. */
+    param->table_id = (table_id | TWS_9K_PARAM_DESCRIPTOR);
+    param->parameter_id = (u_int8_t)(param_id);
+    param->parameter_size_bytes = (u_int16_t)param_size;
+   
+    error = tws_map_request(sc, req);
+    if (!error) {
+        reqid = tws_poll4_response(sc, &mfa);
+        tws_unmap_request(sc, req);
+
+        if ( reqid == TWS_REQ_TYPE_GETSET_PARAM ) {
+            memcpy(data, param->data, param_size);
+        } else {
+            error = FAILURE;
+        }
+    }
+  
+    free(req->data, M_TWS);
+    req->state = TWS_REQ_STATE_FREE;
+    return(error);
+
+}
+
+void 
+tws_unmap_request(struct tws_softc *sc, struct tws_request *req)
+{
+    if (req->data != NULL) {
+        if ( req->flags & TWS_DIR_IN )
+            bus_dmamap_sync(sc->data_tag, req->dma_map, 
+                                            BUS_DMASYNC_POSTREAD);
+        if ( req->flags & TWS_DIR_OUT )
+            bus_dmamap_sync(sc->data_tag, req->dma_map, 
+                                            BUS_DMASYNC_POSTWRITE);
+        mtx_lock(&sc->io_lock);
+        bus_dmamap_unload(sc->data_tag, req->dma_map);
+        mtx_unlock(&sc->io_lock);
+    }
+}
+
+int32_t
+tws_map_request(struct tws_softc *sc, struct tws_request *req)
+{
+    int32_t error = 0;
+
+
+    /* If the command involves data, map that too. */       
+    if (req->data != NULL) {
+        int my_flags = ((req->type == TWS_REQ_TYPE_SCSI_IO) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT);
+
+        /*
+         * Map the data buffer into bus space and build the SG list.
+         */
+        mtx_lock(&sc->io_lock);
+        error = bus_dmamap_load(sc->data_tag, req->dma_map,
+                                req->data, req->length,
+                                tws_dmamap_data_load_cbfn, req,
+                                my_flags);
+        mtx_unlock(&sc->io_lock);
+
+        if (error == EINPROGRESS) {
+            TWS_TRACE(sc, "in progress", 0, error);
+            tws_freeze_simq(sc, req);
+        } 
+    } else { /* no data involved */
+        error = tws_submit_command(sc, req);
+    }
+    return(error);
+}
+
+
+static void
+tws_dmamap_data_load_cbfn(void *arg, bus_dma_segment_t *segs, 
+                            int nseg, int error)
+{
+    struct tws_request *req = (struct tws_request *)arg;
+    struct tws_softc *sc = req->sc;
+    u_int16_t sgls = nseg;
+    void *sgl_ptr;
+    struct tws_cmd_generic *gcmd;
+
+
+    if ( error == EFBIG ) {
+        TWS_TRACE(sc, "not enough data segs", 0, nseg);
+        req->error_code = error;
+        req->ccb_ptr->ccb_h.status = CAM_REQ_TOO_BIG;
+        return;
+    }
+
+    if ( req->flags & TWS_DIR_IN )
+        bus_dmamap_sync(req->sc->data_tag, req->dma_map, 
+                                            BUS_DMASYNC_PREREAD);
+    if ( req->flags & TWS_DIR_OUT )
+        bus_dmamap_sync(req->sc->data_tag, req->dma_map, 
+                                        BUS_DMASYNC_PREWRITE);
+    if ( segs ) {
+        if ( (req->type == TWS_REQ_TYPE_PASSTHRU && 
+             GET_OPCODE(req->cmd_pkt->cmd.pkt_a.res__opcode) != 
+                            TWS_FW_CMD_EXECUTE_SCSI) ||
+              req->type == TWS_REQ_TYPE_GETSET_PARAM) {
+            gcmd = &req->cmd_pkt->cmd.pkt_g.generic;
+            sgl_ptr = (u_int32_t *)(gcmd) + gcmd->size;
+            gcmd->size += sgls * 
+                          ((req->sc->is64bit && !tws_use_32bit_sgls) ? 4 :2 );
+            tws_fill_sg_list(req->sc, (void *)segs, sgl_ptr, sgls);
+
+        } else {
+            tws_fill_sg_list(req->sc, (void *)segs, 
+                      (void *)req->cmd_pkt->cmd.pkt_a.sg_list, sgls);
+            req->cmd_pkt->cmd.pkt_a.lun_h4__sgl_entries |= sgls ;
+        }
+    }
+
+
+    req->error_code = tws_submit_command(req->sc, req);
+
+}
+
+
+static void
+tws_fill_sg_list(struct tws_softc *sc, void *sgl_src, void *sgl_dest, 
+                          u_int16_t num_sgl_entries)
+{
+    int i;
+
+    if ( sc->is64bit ) {
+        struct tws_sg_desc64 *sgl_s = (struct tws_sg_desc64 *)sgl_src;
+
+        if ( !tws_use_32bit_sgls ) { 
+            struct tws_sg_desc64 *sgl_d = (struct tws_sg_desc64 *)sgl_dest;
+            if ( num_sgl_entries > TWS_MAX_64BIT_SG_ELEMENTS )
+                TWS_TRACE(sc, "64bit sg overflow", num_sgl_entries, 0);
+            for (i = 0; i < num_sgl_entries; i++) {
+                sgl_d[i].address = sgl_s->address;
+                sgl_d[i].length = sgl_s->length;
+                sgl_d[i].flag = 0;
+                sgl_d[i].reserved = 0;
+                sgl_s = (struct tws_sg_desc64 *) (((u_int8_t *)sgl_s) + 
+                                               sizeof(bus_dma_segment_t));
+            }
+        } else {
+            struct tws_sg_desc32 *sgl_d = (struct tws_sg_desc32 *)sgl_dest;
+            if ( num_sgl_entries > TWS_MAX_32BIT_SG_ELEMENTS )
+                TWS_TRACE(sc, "32bit sg overflow", num_sgl_entries, 0);
+            for (i = 0; i < num_sgl_entries; i++) {
+                sgl_d[i].address = sgl_s->address;
+                sgl_d[i].length = sgl_s->length;
+                sgl_d[i].flag = 0;
+                sgl_s = (struct tws_sg_desc64 *) (((u_int8_t *)sgl_s) + 
+                                               sizeof(bus_dma_segment_t));
+            }
+        }
+    } else {
+        struct tws_sg_desc32 *sgl_s = (struct tws_sg_desc32 *)sgl_src;
+        struct tws_sg_desc32 *sgl_d = (struct tws_sg_desc32 *)sgl_dest;
+
+        if ( num_sgl_entries > TWS_MAX_32BIT_SG_ELEMENTS )
+            TWS_TRACE(sc, "32bit sg overflow", num_sgl_entries, 0);
+
+
+        for (i = 0; i < num_sgl_entries; i++) {
+            sgl_d[i].address = sgl_s[i].address;
+            sgl_d[i].length = sgl_s[i].length;
+            sgl_d[i].flag = 0;
+        }
+    }
+}
+ 
+
+void
+tws_intr(void *arg)
+{
+    struct tws_softc *sc = (struct tws_softc *)arg;
+    u_int32_t histat=0, db=0;
+
+    if (!(sc)) {
+        device_printf(sc->tws_dev, "null softc!!!\n");
+        return;
+    }
+
+    if ( tws_get_state(sc) == TWS_RESET ) {
+        return;
+    }
+
+    if ( tws_get_state(sc) != TWS_ONLINE ) {
+        return;
+    }
+
+    sc->stats.num_intrs++;
+    histat = tws_read_reg(sc, TWS_I2O0_HISTAT, 4);
+    if ( histat & TWS_BIT2 ) {
+        TWS_TRACE_DEBUG(sc, "door bell :)", histat, TWS_I2O0_HISTAT);
+        db = tws_read_reg(sc, TWS_I2O0_IOBDB, 4);
+        if ( db & TWS_BIT21 ) {
+            tws_intr_attn_error(sc);
+            return;
+        }
+        if ( db & TWS_BIT18 ) {
+            tws_intr_attn_aen(sc);
+        }
+    }
+
+    if ( histat & TWS_BIT3 ) {
+        tws_intr_resp(sc);
+    }
+}
+
+static void
+tws_intr_attn_aen(struct tws_softc *sc)
+{
+    u_int32_t db=0;
+
+    /* maskoff db intrs untill all the aens are fetched */
+    /* tws_disable_db_intr(sc); */
+    tws_fetch_aen((void *)sc);
+    tws_write_reg(sc, TWS_I2O0_HOBDBC, TWS_BIT18, 4);
+    db = tws_read_reg(sc, TWS_I2O0_IOBDB, 4);
+
+}
+
+static void
+tws_intr_attn_error(struct tws_softc *sc)
+{
+    u_int32_t db=0;
+
+    TWS_TRACE(sc, "attn error", 0, 0);
+    tws_write_reg(sc, TWS_I2O0_HOBDBC, ~0, 4);
+    db = tws_read_reg(sc, TWS_I2O0_IOBDB, 4);
+    device_printf(sc->tws_dev, "Micro controller error.\n");
+    tws_reset(sc);
+}
+
+static void
+tws_intr_resp(struct tws_softc *sc)
+{
+    u_int16_t req_id;
+    u_int64_t mfa;
+
+    while ( tws_get_response(sc, &req_id, &mfa) ) {
+        sc->stats.reqs_out++;
+        if ( req_id == TWS_INVALID_REQID ) {
+            TWS_TRACE_DEBUG(sc, "invalid req_id", mfa, req_id);
+            sc->stats.reqs_errored++;
+            tws_err_complete(sc, mfa);
+            continue;
+        }
+        sc->reqs[req_id].cb(&sc->reqs[req_id]);
+    }
+
+}
+
+
+static void
+tws_poll(struct cam_sim *sim)
+{
+    struct tws_softc *sc = (struct tws_softc *)cam_sim_softc(sim);
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    tws_intr((void *) sc);
+}
+
+void
+tws_timeout(void *arg)
+{
+    struct tws_request *req = (struct tws_request *)arg;
+    struct tws_softc *sc = req->sc;
+
+
+    if ( req->error_code == TWS_REQ_RET_RESET ) {
+        return;
+    }
+
+    mtx_lock(&sc->gen_lock);
+    if ( req->error_code == TWS_REQ_RET_RESET ) {
+        mtx_unlock(&sc->gen_lock);
+        return;
+    }
+
+    if ( tws_get_state(sc) == TWS_RESET ) {
+        mtx_unlock(&sc->gen_lock);
+        return;
+    }
+
+    tws_teardown_intr(sc);
+    xpt_freeze_simq(sc->sim, 1);
+
+    tws_send_event(sc, TWS_RESET_START);
+
+    if (req->type == TWS_REQ_TYPE_SCSI_IO) {
+        device_printf(sc->tws_dev, "I/O Request timed out... Resetting controller\n");
+    } else if (req->type == TWS_REQ_TYPE_PASSTHRU) {
+        device_printf(sc->tws_dev, "IOCTL Request timed out... Resetting controller\n");
+    } else {
+        device_printf(sc->tws_dev, "Internal Request timed out... Resetting controller\n");
+    }
+
+    tws_assert_soft_reset(sc);
+    tws_turn_off_interrupts(sc);
+    tws_reset_cb( (void*) sc );
+    tws_reinit( (void*) sc );
+
+//  device_printf(sc->tws_dev,  "Controller Reset complete!\n");
+    tws_send_event(sc, TWS_RESET_COMPLETE);
+    mtx_unlock(&sc->gen_lock);
+
+    xpt_release_simq(sc->sim, 1);
+    tws_setup_intr(sc, sc->irqs);
+}
+
+void
+tws_reset(void *arg)
+{
+    struct tws_softc *sc = (struct tws_softc *)arg;
+
+    mtx_lock(&sc->gen_lock);
+    if ( tws_get_state(sc) == TWS_RESET ) {
+        mtx_unlock(&sc->gen_lock);
+        return;
+    }
+
+    tws_teardown_intr(sc);
+    xpt_freeze_simq(sc->sim, 1);
+
+    tws_send_event(sc, TWS_RESET_START);
+
+    device_printf(sc->tws_dev,  "Resetting controller\n");
+
+    tws_assert_soft_reset(sc);
+    tws_turn_off_interrupts(sc);
+    tws_reset_cb( (void*) sc );
+    tws_reinit( (void*) sc );
+
+//  device_printf(sc->tws_dev,  "Controller Reset complete!\n");
+    tws_send_event(sc, TWS_RESET_COMPLETE);
+    mtx_unlock(&sc->gen_lock);
+
+    xpt_release_simq(sc->sim, 1);
+    tws_setup_intr(sc, sc->irqs);
+}
+
+static void
+tws_reset_cb(void *arg)
+{
+    struct tws_softc *sc = (struct tws_softc *)arg;
+    time_t endt;
+    int found = 0;
+    u_int32_t reg;
+  
+    if ( tws_get_state(sc) != TWS_RESET ) {
+        return;
+    }
+
+//  device_printf(sc->tws_dev,  "Draining Busy Queue\n");
+    tws_drain_busy_queue(sc);
+//  device_printf(sc->tws_dev,  "Draining Reserved Reqs\n");
+    tws_drain_reserved_reqs(sc);
+//  device_printf(sc->tws_dev,  "Draining Response Queue\n");
+    tws_drain_response_queue(sc);
+
+//  device_printf(sc->tws_dev,  "Looking for controller ready flag...\n");
+    endt = TWS_LOCAL_TIME + TWS_POLL_TIMEOUT;
+    while ((TWS_LOCAL_TIME <= endt) && (!found)) {
+        reg = tws_read_reg(sc, TWS_I2O0_SCRPD3, 4);
+        if ( reg & TWS_BIT13 ) {
+            found = 1;
+//          device_printf(sc->tws_dev,  " ... Got it!\n");
+        }
+    }
+    if ( !found )
+            device_printf(sc->tws_dev,  " ... Controller ready flag NOT found!\n");
+}
+
+static void
+tws_reinit(void *arg)
+{
+    struct tws_softc *sc = (struct tws_softc *)arg;
+    int timeout_val=0;
+    int try=2;
+    int done=0;
+
+
+//  device_printf(sc->tws_dev,  "Waiting for Controller Ready\n");
+    while ( !done && try ) {
+        if ( tws_ctlr_ready(sc) ) {
+            done = 1;
+            break;
+        } else {
+            timeout_val += 5;
+            if ( timeout_val >= TWS_RESET_TIMEOUT ) {
+               timeout_val = 0;
+               if ( try )
+                   tws_assert_soft_reset(sc);
+               try--;
+            }
+            mtx_sleep(sc, &sc->gen_lock, 0, "tws_reinit", 5*hz);
+        }
+    }
+
+    if (!done) {
+        device_printf(sc->tws_dev,  "FAILED to get Controller Ready!\n");
+        return;
+    }
+
+    sc->obfl_q_overrun = false;
+//  device_printf(sc->tws_dev,  "Sending initConnect\n");
+    if ( tws_init_connect(sc, tws_queue_depth) ) {
+        TWS_TRACE_DEBUG(sc, "initConnect failed", 0, sc->is64bit);
+    }
+    tws_init_obfl_q(sc);
+
+    tws_turn_on_interrupts(sc);
+
+    if ( sc->chan ) {
+        sc->chan = 0;
+        wakeup_one((void *)&sc->chan);
+    }
+}
+
+
+static void
+tws_freeze_simq(struct tws_softc *sc, struct tws_request *req)
+{
+    /* Only for IO commands */
+    if (req->type == TWS_REQ_TYPE_SCSI_IO) {
+        union ccb   *ccb = (union ccb *)(req->ccb_ptr);
+
+        xpt_freeze_simq(sc->sim, 1);
+        ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
+        ccb->ccb_h.status |= CAM_REQUEUE_REQ;
+    }
+}
+
+
+TUNABLE_INT("hw.tws.cam_depth", &tws_cam_depth);
diff --git a/sys/dev/tws/tws_hdm.c b/sys/dev/tws/tws_hdm.c
new file mode 100644
index 000000000000..3f6bea0cba9e
--- /dev/null
+++ b/sys/dev/tws/tws_hdm.c
@@ -0,0 +1,535 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+#include <dev/tws/tws.h>
+#include <dev/tws/tws_services.h>
+#include <dev/tws/tws_hdm.h>
+
+
+int tws_use_32bit_sgls=0;
+extern u_int64_t mfa_base;
+extern struct tws_request *tws_get_request(struct tws_softc *sc, 
+                                           u_int16_t type);
+extern void tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req,
+                                u_int8_t q_type );
+extern struct tws_request * tws_q_remove_request(struct tws_softc *sc,
+                                   struct tws_request *req, u_int8_t q_type );
+
+extern void tws_cmd_complete(struct tws_request *req);
+extern void tws_print_stats(void *arg);
+extern int tws_send_scsi_cmd(struct tws_softc *sc, int cmd);
+extern int tws_set_param(struct tws_softc *sc, u_int32_t table_id, 
+           u_int32_t param_id, u_int32_t param_size, void *data);
+extern int tws_get_param(struct tws_softc *sc, u_int32_t table_id,
+            u_int32_t param_id, u_int32_t param_size, void *data);
+extern void tws_reset(void *arg);
+
+int tws_init_connect(struct tws_softc *sc, u_int16_t mc);
+int tws_init_ctlr(struct tws_softc *sc);
+int tws_submit_command(struct tws_softc *sc, struct tws_request *req);
+void tws_nop_cmd(void *arg);
+u_int16_t tws_poll4_response(struct tws_softc *sc, u_int64_t *mfa);
+boolean tws_get_response(struct tws_softc *sc, u_int16_t *req_id, 
+                                               u_int64_t *mfa);
+boolean tws_ctlr_ready(struct tws_softc *sc);
+void tws_turn_on_interrupts(struct tws_softc *sc);
+void tws_turn_off_interrupts(struct tws_softc *sc);
+boolean tws_ctlr_reset(struct tws_softc *sc);
+void tws_assert_soft_reset(struct tws_softc *sc);
+
+int tws_send_generic_cmd(struct tws_softc *sc, u_int8_t opcode);
+void tws_fetch_aen(void *arg);
+void tws_disable_db_intr(struct tws_softc *sc);
+void tws_enable_db_intr(struct tws_softc *sc);
+void tws_aen_synctime_with_host(struct tws_softc *sc);
+void tws_init_obfl_q(struct tws_softc *sc);
+void tws_display_ctlr_info(struct tws_softc *sc);
+
+int 
+tws_init_ctlr(struct tws_softc *sc)
+{
+    u_int64_t reg;
+    u_int32_t regh, regl;
+
+    TWS_TRACE_DEBUG(sc, "entry", sc, sc->is64bit);
+    sc->obfl_q_overrun = false;
+    if ( tws_init_connect(sc, tws_queue_depth) )
+    {
+        TWS_TRACE_DEBUG(sc, "initConnect failed", 0, sc->is64bit);
+        return(FAILURE);
+        
+    }
+
+
+    while( 1 ) {
+        regh = tws_read_reg(sc, TWS_I2O0_IOPOBQPH, 4);
+        regl = tws_read_reg(sc, TWS_I2O0_IOPOBQPL, 4);
+        reg = (((u_int64_t)regh) << 32) | regl;
+        TWS_TRACE_DEBUG(sc, "host outbound clenup",reg, regl);
+        if ( regh == TWS_FIFO_EMPTY32 )
+            break;
+    } 
+
+    tws_init_obfl_q(sc);
+    tws_display_ctlr_info(sc);
+    tws_write_reg(sc, TWS_I2O0_HOBDBC, ~0, 4);
+    tws_turn_on_interrupts(sc);
+    return(SUCCESS);
+}
+
+void
+tws_init_obfl_q(struct tws_softc *sc)
+{
+    int i=0;
+    u_int64_t paddr;
+    u_int32_t paddrh, paddrl, status;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, sc->obfl_q_overrun);
+
+    while ( i < tws_queue_depth ) {
+        paddr = sc->sense_bufs[i].hdr_pkt_phy;
+        paddrh = (u_int32_t)( paddr>>32);
+        paddrl = (u_int32_t) paddr;
+        tws_write_reg(sc, TWS_I2O0_HOBQPH, paddrh, 4);
+        tws_write_reg(sc, TWS_I2O0_HOBQPL, paddrl, 4);
+  
+        status = tws_read_reg(sc, TWS_I2O0_STATUS, 4);
+        if ( status & TWS_BIT13 ) {
+            device_printf(sc->tws_dev,  "OBFL Overrun\n");
+            sc->obfl_q_overrun = true;
+            break;
+        }
+        i++;
+    }
+
+    if ( i == tws_queue_depth )
+        sc->obfl_q_overrun = false;
+}
+
+int
+tws_init_connect(struct tws_softc *sc, u_int16_t mcreadits )
+{
+    struct tws_request *req;
+    struct tws_cmd_init_connect *initc;
+    u_int16_t reqid;
+    u_int64_t mfa;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, mcreadits);
+#if       0
+    req = tws_get_request(sc, TWS_REQ_TYPE_INTERNAL_CMD);
+#else  // 0
+    req = &sc->reqs[TWS_REQ_TYPE_INTERNAL_CMD];
+    bzero(&req->cmd_pkt->cmd, sizeof(struct tws_command_apache));
+    req->data = NULL;
+    req->length = 0;
+    req->type = TWS_REQ_TYPE_INTERNAL_CMD;
+    req->flags = TWS_DIR_UNKNOWN;
+    req->error_code = TWS_REQ_RET_INVALID;
+    req->cb = NULL;
+    req->ccb_ptr = NULL;
+    req->thandle.callout = NULL;
+    req->next = req->prev = NULL;
+    req->state = TWS_REQ_STATE_BUSY;
+#endif // 0
+
+    if ( req == NULL ) {
+        TWS_TRACE_DEBUG(sc, "no requests", 0, 0);
+//      device_printf(sc->tws_dev,  "No requests for initConnect\n");
+        return(FAILURE);
+    }
+
+    tws_swap16(0xbeef); /* just for test */
+    tws_swap32(0xdeadbeef); /* just for test */
+    tws_swap64(0xdeadbeef); /* just for test */
+    initc = &(req->cmd_pkt->cmd.pkt_g.init_connect);
+    /* req->cmd_pkt->hdr.header_desc.size_header = 128; */
+
+    initc->res1__opcode = 
+              BUILD_RES__OPCODE(0, TWS_FW_CMD_INIT_CONNECTION);
+    initc->size = 6;
+    initc->request_id = req->request_id;
+    initc->message_credits = mcreadits;
+    initc->features |= TWS_BIT_EXTEND;
+    if ( sc->is64bit && !tws_use_32bit_sgls )
+        initc->features |= TWS_64BIT_SG_ADDRESSES;
+    /* assuming set features is always on */ 
+
+    initc->size = 6;
+    initc->fw_srl = sc->cinfo.working_srl = TWS_CURRENT_FW_SRL;
+    initc->fw_arch_id = 0;
+    initc->fw_branch = sc->cinfo.working_branch = 0;
+    initc->fw_build = sc->cinfo.working_build = 0;
+
+    req->error_code = tws_submit_command(sc, req);
+    reqid = tws_poll4_response(sc, &mfa);
+    if ( reqid != TWS_INVALID_REQID && reqid == req->request_id ) {
+        sc->cinfo.fw_on_ctlr_srl = initc->fw_srl;
+        sc->cinfo.fw_on_ctlr_branch = initc->fw_branch;
+        sc->cinfo.fw_on_ctlr_build = initc->fw_build;
+        sc->stats.reqs_out++;
+        req->state = TWS_REQ_STATE_FREE;
+    }
+    else {
+        /*
+         * REVISIT::If init connect fails we need to reset the ctlr
+         * and try again? 
+         */ 
+        TWS_TRACE(sc, "unexpected req_id ", reqid, 0);
+        TWS_TRACE(sc, "INITCONNECT FAILED", reqid, 0);
+        return(FAILURE);
+    }
+    return(SUCCESS);
+}
+
+void 
+tws_display_ctlr_info(struct tws_softc *sc)
+{
+
+    uint8_t fw_ver[16], bios_ver[16], ctlr_model[16], num_phys=0;
+    uint32_t error[4];
+
+    error[0] = tws_get_param(sc, TWS_PARAM_PHYS_TABLE,
+                             TWS_PARAM_CONTROLLER_PHYS_COUNT, 1, &num_phys);
+    error[1] = tws_get_param(sc, TWS_PARAM_VERSION_TABLE,
+                             TWS_PARAM_VERSION_FW, 16, fw_ver);
+    error[2] = tws_get_param(sc, TWS_PARAM_VERSION_TABLE,
+                             TWS_PARAM_VERSION_BIOS, 16, bios_ver);
+    error[3] = tws_get_param(sc, TWS_PARAM_VERSION_TABLE,
+                             TWS_PARAM_CTLR_MODEL, 16, ctlr_model);
+
+    if ( !error[0] && !error[1] && !error[2] && !error[3] ) {
+        device_printf( sc->tws_dev, 
+        "Controller details: Model %.16s, %d Phys, Firmware %.16s, BIOS %.16s\n",
+         ctlr_model, num_phys, fw_ver, bios_ver);
+    }
+
+}
+
+int
+tws_send_generic_cmd(struct tws_softc *sc, u_int8_t opcode)
+{
+    struct tws_request *req;
+    struct tws_cmd_generic *cmd;
+
+    TWS_TRACE_DEBUG(sc, "entry", sc, opcode);
+    req = tws_get_request(sc, TWS_REQ_TYPE_INTERNAL_CMD);
+
+    if ( req == NULL ) {
+        TWS_TRACE_DEBUG(sc, "no requests", 0, 0);
+        return(FAILURE);
+    }
+
+    cmd = &(req->cmd_pkt->cmd.pkt_g.generic);
+    bzero(cmd, sizeof(struct tws_cmd_generic));
+    /* req->cmd_pkt->hdr.header_desc.size_header = 128; */
+    req->cb = tws_cmd_complete;
+
+    cmd->sgl_off__opcode = BUILD_RES__OPCODE(0, opcode);
+    cmd->size = 2;
+    cmd->request_id = req->request_id;
+    cmd->host_id__unit = 0;
+    cmd->status = 0;
+    cmd->flags = 0;
+    cmd->count = 0;
+
+    req->error_code = tws_submit_command(sc, req);
+
+    return(SUCCESS);
+
+}
+
+
+int
+tws_submit_command(struct tws_softc *sc, struct tws_request *req)
+{
+    u_int32_t regl, regh;
+    u_int64_t mfa=0;
+    
+    /* 
+     * mfa register  read and write must be in order. 
+     * Get the io_lock to protect against simultinous 
+     * passthru calls 
+     */
+    mtx_lock(&sc->io_lock);
+
+    if ( sc->obfl_q_overrun ) {
+        tws_init_obfl_q(sc);
+    }
+       
+#ifdef TWS_PULL_MODE_ENABLE
+    regh = (u_int32_t)(req->cmd_pkt_phy >> 32);
+    /* regh = regh | TWS_MSG_ACC_MASK; */ 
+    mfa = regh;
+    mfa = mfa << 32;
+    regl = (u_int32_t)req->cmd_pkt_phy;
+    regl = regl | TWS_BIT0;
+    mfa = mfa | regl;
+#else
+    regh = tws_read_reg(sc, TWS_I2O0_HIBQPH, 4);
+    mfa = regh;
+    mfa = mfa << 32;
+    regl = tws_read_reg(sc, TWS_I2O0_HIBQPL, 4);
+    mfa = mfa | regl;
+#endif
+
+    mtx_unlock(&sc->io_lock);
+
+    if ( mfa == TWS_FIFO_EMPTY ) {
+        TWS_TRACE_DEBUG(sc, "inbound fifo empty", mfa, 0);
+
+        /* 
+         * Generaly we should not get here.
+         * If the fifo was empty we can't do any thing much 
+         * retry later 
+         */
+        return(TWS_REQ_RET_PEND_NOMFA);
+
+    }
+
+#ifndef TWS_PULL_MODE_ENABLE
+    for (int i=mfa; i<(sizeof(struct tws_command_packet)+ mfa - 
+                            sizeof( struct tws_command_header)); i++) {
+
+        bus_space_write_1(sc->bus_mfa_tag, sc->bus_mfa_handle,i, 
+                               ((u_int8_t *)&req->cmd_pkt->cmd)[i-mfa]);
+
+    }
+#endif
+
+    if ( req->type == TWS_REQ_TYPE_SCSI_IO ) {
+        mtx_lock(&sc->q_lock);
+        tws_q_insert_tail(sc, req, TWS_BUSY_Q);
+        mtx_unlock(&sc->q_lock);
+    }
+
+    /* 
+     * mfa register  read and write must be in order. 
+     * Get the io_lock to protect against simultinous 
+     * passthru calls 
+     */
+    mtx_lock(&sc->io_lock);
+
+    tws_write_reg(sc, TWS_I2O0_HIBQPH, regh, 4);
+    tws_write_reg(sc, TWS_I2O0_HIBQPL, regl, 4);
+
+    sc->stats.reqs_in++;
+    mtx_unlock(&sc->io_lock);
+    
+    return(TWS_REQ_RET_SUBMIT_SUCCESS);
+
+}
+
+/* 
+ * returns true if the respose was available othewise, false.
+ * In the case of error the arg mfa will contain the address and
+ * req_id will be TWS_INVALID_REQID
+ */
+boolean
+tws_get_response(struct tws_softc *sc, u_int16_t *req_id, u_int64_t *mfa) 
+{
+    u_int64_t out_mfa=0, val=0;
+    struct tws_outbound_response out_res;
+
+    *req_id = TWS_INVALID_REQID;
+    out_mfa = (u_int64_t)tws_read_reg(sc, TWS_I2O0_HOBQPH, 4);
+
+    if ( out_mfa == TWS_FIFO_EMPTY32 ) {
+        return(false);
+
+    }
+    out_mfa = out_mfa << 32;
+    val = tws_read_reg(sc, TWS_I2O0_HOBQPL, 4);
+    out_mfa = out_mfa | val;
+    
+    out_res =  *(struct tws_outbound_response *)&out_mfa;
+
+    if ( !out_res.not_mfa ) {
+        *mfa = out_mfa;
+        return(true);
+    } else {
+        *req_id = out_res.request_id;
+    }
+    
+    return(true);
+}
+
+
+
+
+u_int16_t
+tws_poll4_response(struct tws_softc *sc, u_int64_t *mfa)
+{
+    u_int16_t req_id;
+    time_t endt;
+
+    endt = TWS_LOCAL_TIME + TWS_POLL_TIMEOUT;
+    do {
+        if(tws_get_response(sc, &req_id, mfa)) {
+
+            if ( req_id == TWS_INVALID_REQID ) {
+                TWS_TRACE_DEBUG(sc, "invalid req_id", 0, req_id);
+                return(TWS_INVALID_REQID);
+            }
+            return(req_id);
+        }
+    } while (TWS_LOCAL_TIME <= endt);
+    TWS_TRACE_DEBUG(sc, "poll timeout", 0, 0);
+    return(TWS_INVALID_REQID);
+}
+
+boolean
+tws_ctlr_ready(struct tws_softc *sc)
+{
+    u_int32_t reg;
+
+    reg = tws_read_reg(sc, TWS_I2O0_SCRPD3, 4);
+    if ( reg & TWS_BIT13 )
+        return(true);
+    else
+        return(false);
+}
+
+void
+tws_turn_on_interrupts(struct tws_softc *sc)
+{
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    /* turn on responce and db interrupt only */
+    tws_write_reg(sc, TWS_I2O0_HIMASK, TWS_BIT0, 4);
+
+}
+
+void
+tws_turn_off_interrupts(struct tws_softc *sc)
+{
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    
+    tws_write_reg(sc, TWS_I2O0_HIMASK, ~0, 4);
+
+}
+
+void
+tws_disable_db_intr(struct tws_softc *sc)
+{
+    u_int32_t reg;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    reg = tws_read_reg(sc, TWS_I2O0_HIMASK, 4);
+    reg = reg | TWS_BIT2;
+    tws_write_reg(sc, TWS_I2O0_HIMASK, reg, 4);
+}
+
+void
+tws_enable_db_intr(struct tws_softc *sc)
+{
+    u_int32_t reg;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+    reg = tws_read_reg(sc, TWS_I2O0_HIMASK, 4);
+    reg = reg & ~TWS_BIT2;
+    tws_write_reg(sc, TWS_I2O0_HIMASK, reg, 4);
+}
+
+boolean
+tws_ctlr_reset(struct tws_softc *sc)
+{
+
+    u_int32_t reg;
+    time_t endt;
+    /* int i=0; */
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+
+    tws_assert_soft_reset(sc);
+
+    do {
+        reg = tws_read_reg(sc, TWS_I2O0_SCRPD3, 4);
+    } while ( reg & TWS_BIT13 );
+
+    endt = TWS_LOCAL_TIME + TWS_RESET_TIMEOUT;
+    do {
+        if(tws_ctlr_ready(sc))
+            return(true);
+    } while (TWS_LOCAL_TIME <= endt);
+    return(false);
+
+}
+
+void
+tws_assert_soft_reset(struct tws_softc *sc)
+{
+    u_int32_t reg;
+
+    reg = tws_read_reg(sc, TWS_I2O0_HIBDB, 4);
+    TWS_TRACE_DEBUG(sc, "in bound door bell read ", reg, TWS_I2O0_HIBDB);
+    tws_write_reg(sc, TWS_I2O0_HIBDB, reg | TWS_BIT8, 4);
+
+}
+
+void
+tws_fetch_aen(void *arg)
+{
+    struct tws_softc *sc = (struct tws_softc *)arg;
+    int error = 0;
+
+    TWS_TRACE_DEBUG(sc, "entry", 0, 0);
+
+    if ((error = tws_send_scsi_cmd(sc, 0x03 /* REQUEST_SENSE */))) {
+        TWS_TRACE_DEBUG(sc, "aen fetch send in progress", 0, 0);
+    }
+}
+
+void
+tws_aen_synctime_with_host(struct tws_softc *sc)
+{
+
+    int error;
+    long int sync_time;
+
+    TWS_TRACE_DEBUG(sc, "entry", sc, 0);
+
+    sync_time = (TWS_LOCAL_TIME - (3 * 86400)) % 604800;
+    TWS_TRACE_DEBUG(sc, "sync_time,ts", sync_time, time_second);
+    TWS_TRACE_DEBUG(sc, "utc_offset", utc_offset(), 0);
+    error = tws_set_param(sc, TWS_PARAM_TIME_TABLE, TWS_PARAM_TIME_SCHED_TIME,
+                           4, &sync_time);
+    if ( error )
+        TWS_TRACE_DEBUG(sc, "set param failed", sync_time, error);
+}
+
+TUNABLE_INT("hw.tws.use_32bit_sgls", &tws_use_32bit_sgls);
diff --git a/sys/dev/tws/tws_hdm.h b/sys/dev/tws/tws_hdm.h
new file mode 100644
index 000000000000..5633fdf76bf4
--- /dev/null
+++ b/sys/dev/tws/tws_hdm.h
@@ -0,0 +1,420 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+/* bit's defination */
+
+#define TWS_BIT0                              0x00000001
+#define TWS_BIT1                              0x00000002
+#define TWS_BIT2                              0x00000004
+#define TWS_BIT3                              0x00000008
+#define TWS_BIT4                              0x00000010
+#define TWS_BIT5                              0x00000020
+#define TWS_BIT6                              0x00000040
+#define TWS_BIT7                              0x00000080
+#define TWS_BIT8                              0x00000100
+#define TWS_BIT9                              0x00000200
+#define TWS_BIT10                             0x00000400
+#define TWS_BIT11                             0x00000800
+#define TWS_BIT12                             0x00001000
+#define TWS_BIT13                             0x00002000
+#define TWS_BIT14                             0x00004000
+#define TWS_BIT15                             0x00008000
+#define TWS_BIT16                             0x00010000
+#define TWS_BIT17                             0x00020000
+#define TWS_BIT18                             0x00040000
+#define TWS_BIT19                             0x00080000
+#define TWS_BIT20                             0x00100000
+#define TWS_BIT21                             0x00200000
+#define TWS_BIT22                             0x00400000
+#define TWS_BIT23                             0x00800000
+#define TWS_BIT24                             0x01000000
+#define TWS_BIT25                             0x02000000
+#define TWS_BIT26                             0x04000000
+#define TWS_BIT27                             0x08000000
+#define TWS_BIT28                             0x10000000
+#define TWS_BIT29                             0x20000000
+#define TWS_BIT30                             0x40000000
+#define TWS_BIT31                             0x80000000
+
+#define TWS_SENSE_DATA_LENGTH                 18
+#define TWS_ERROR_SPECIFIC_DESC_LEN           98
+
+/* response codes */
+#define TWS_SENSE_SCSI_CURRENT_ERROR          0x70
+#define TWS_SENSE_SCSI_DEFERRED_ERROR         0x71
+
+#define TWS_SRC_CTRL_ERROR                    3
+#define TWS_SRC_CTRL_EVENT                    4
+#define TWS_SRC_FREEBSD_DRIVER                5
+#define TWS_SRC_FREEBSD_OS                    8
+
+
+enum tws_sense_severity {
+    error = 1,
+    warning ,
+    info,
+    debug,
+};
+
+/*
+ * Some errors of interest (in cmd_hdr->status_block.error) when a command
+ * is completed by the firmware with an error.
+ */
+#define TWS_ERROR_LOGICAL_UNIT_NOT_SUPPORTED    0x010a
+#define TWS_ERROR_NOT_SUPPORTED                 0x010D
+#define TWS_ERROR_UNIT_OFFLINE                  0x0128
+#define TWS_ERROR_MORE_DATA                     0x0231
+
+
+/* AEN codes of interest. */
+#define TWS_AEN_QUEUE_EMPTY                     0x00
+#define TWS_AEN_SOFT_RESET                      0x01
+#define TWS_AEN_SYNC_TIME_WITH_HOST             0x31
+
+
+/* AEN severity */
+#define TWS_SEVERITY_ERROR                      0x1
+#define TWS_SEVERITY_WARNING                    0x2
+#define TWS_SEVERITY_INFO                       0x3
+#define TWS_SEVERITY_DEBUG                      0x4
+
+#define TWS_64BIT_SG_ADDRESSES                  0x00000001
+#define TWS_BIT_EXTEND                          0x00000002
+
+#define TWS_BASE_FW_SRL                         24
+#define TWS_BASE_FW_BRANCH                      0
+#define TWS_BASE_FW_BUILD                       1
+#define TWS_CURRENT_FW_SRL                      41
+
+#define TWS_CURRENT_FW_BRANCH                   8
+#define TWS_CURRENT_FW_BUILD                    4
+#define TWS_CURRENT_ARCH_ID                     0x000A
+
+
+#define TWS_FIFO_EMPTY                          0xFFFFFFFFFFFFFFFFull
+#define TWS_FIFO_EMPTY32                        0xFFFFFFFFull
+
+
+/* Register offsets from base address. */
+#define TWS_CONTROL_REGISTER_OFFSET             0x0
+#define TWS_STATUS_REGISTER_OFFSET              0x4
+#define TWS_COMMAND_QUEUE_OFFSET                0x8
+#define TWS_RESPONSE_QUEUE_OFFSET               0xC
+#define TWS_COMMAND_QUEUE_OFFSET_LOW            0x20
+#define TWS_COMMAND_QUEUE_OFFSET_HIGH           0x24
+#define TWS_LARGE_RESPONSE_QUEUE_OFFSET         0x30
+
+/* I2O offsets */
+#define TWS_I2O0_STATUS                         0x0
+
+#define TWS_I2O0_HIBDB                          0x20
+
+#define TWS_I2O0_HISTAT                         0x30
+#define TWS_I2O0_HIMASK                         0x34
+
+#define TWS_I2O0_HIBQP                          0x40
+#define TWS_I2O0_HOBQP                          0x44
+
+#define TWS_I2O0_CTL                            0x74
+
+#define TWS_I2O0_IOBDB                          0x9C
+#define TWS_I2O0_HOBDBC                         0xA0
+
+#define TWS_I2O0_SCRPD3                         0xBC
+
+#define TWS_I2O0_HIBQPL                         0xC0 /* 64bit inb port low */
+#define TWS_I2O0_HIBQPH                         0xC4 /* 64bit inb port high */
+#define TWS_I2O0_HOBQPL                         0xC8 /* 64bit out port low */
+#define TWS_I2O0_HOBQPH                         0xCC /* 64bit out port high */
+
+/* IOP related */
+#define TWS_I2O0_IOPOBQPL                       0xD8 /* OBFL */
+#define TWS_I2O0_IOPOBQPH                       0xDC /* OBFH */
+#define TWS_I2O0_SRC_ADDRH                      0xF8 /* Msg ASA */
+
+#define TWS_MSG_ACC_MASK                        0x20000000
+#define TWS_32BIT_MASK                          0xFFFFFFFF
+
+/* revisit */
+#define TWS_FW_CMD_NOP                     0x0
+#define TWS_FW_CMD_INIT_CONNECTION         0x01
+#define TWS_FW_CMD_EXECUTE_SCSI            0x10
+
+#define TWS_FW_CMD_ATA_PASSTHROUGH         0x11
+#define TWS_FW_CMD_GET_PARAM               0x12
+#define TWS_FW_CMD_SET_PARAM               0x13
+
+
+#define BUILD_SGL_OFF__OPCODE(sgl_off, opcode)  \
+        ((sgl_off << 5) & 0xE0) | (opcode & 0x1F)       /* 3:5 */
+
+#define BUILD_RES__OPCODE(res, opcode)          \
+        ((res << 5) & 0xE0) | (opcode & 0x1F)           /* 3:5 */
+
+#define GET_OPCODE(sgl_off__opcode)     \
+        (sgl_off__opcode & 0x1F)                        /* 3:5 */
+
+
+
+/* end revisit */
+
+
+/* Table #'s and id's of parameters of interest in firmware's param table. */
+#define TWS_PARAM_VERSION_TABLE         0x0402
+#define TWS_PARAM_VERSION_FW            3       /* firmware version [16] */
+#define TWS_PARAM_VERSION_BIOS          4       /* BIOSs version [16] */
+#define TWS_PARAM_CTLR_MODEL            8       /* Controller model [16] */
+
+#define TWS_PARAM_CONTROLLER_TABLE      0x0403
+#define TWS_PARAM_CONTROLLER_PORT_COUNT 3       /* number of ports [1] */
+
+#define TWS_PARAM_TIME_TABLE            0x40A
+#define TWS_PARAM_TIME_SCHED_TIME       0x3
+
+#define TWS_PARAM_PHYS_TABLE            0x0001 
+#define TWS_PARAM_CONTROLLER_PHYS_COUNT 2       /* number of phys */
+
+#define TWS_9K_PARAM_DESCRIPTOR         0x8000
+
+
+/* ----------- request  ------------- */
+
+
+#pragma pack(1)
+
+struct tws_cmd_init_connect {
+    u_int8_t        res1__opcode;   /* 3:5 */
+    u_int8_t        size;
+    u_int8_t        request_id;
+    u_int8_t        res2;
+    u_int8_t        status;
+    u_int8_t        flags;
+    u_int16_t       message_credits;
+    u_int32_t       features;
+    u_int16_t       fw_srl;
+    u_int16_t       fw_arch_id;
+    u_int16_t       fw_branch;
+    u_int16_t       fw_build;
+    u_int32_t       result;
+};
+
+/* Structure for downloading firmware onto the controller. */
+struct tws_cmd_download_firmware {
+    u_int8_t        sgl_off__opcode;/* 3:5 */
+    u_int8_t        size;
+    u_int8_t        request_id;
+    u_int8_t        unit;
+    u_int8_t        status;
+    u_int8_t        flags;
+    u_int16_t       param;
+    u_int8_t        sgl[1];
+};
+
+/* Structure for hard resetting the controller. */
+struct tws_cmd_reset_firmware {
+    u_int8_t        res1__opcode;   /* 3:5 */
+    u_int8_t        size;
+    u_int8_t        request_id;
+    u_int8_t        unit;
+    u_int8_t        status;
+    u_int8_t        flags;
+    u_int8_t        res2;
+    u_int8_t        param;
+};
+
+
+/* Structure for sending get/set param commands. */
+struct tws_cmd_param {
+    u_int8_t        sgl_off__opcode;/* 3:5 */
+    u_int8_t        size;
+    u_int8_t        request_id;
+    u_int8_t        host_id__unit;  /* 4:4 */
+    u_int8_t        status;
+    u_int8_t        flags;
+    u_int16_t       param_count;
+    u_int8_t        sgl[1];
+};
+
+/* Generic command packet. */
+struct tws_cmd_generic {
+    u_int8_t        sgl_off__opcode;/* 3:5 */
+    u_int8_t        size;
+    u_int8_t        request_id;
+    u_int8_t        host_id__unit;  /* 4:4 */
+    u_int8_t        status;
+    u_int8_t        flags;
+    u_int16_t       count;  /* block cnt, parameter cnt, message credits */
+};
+
+
+
+
+/* Command packet header. */
+struct tws_command_header {
+    u_int8_t        sense_data[TWS_SENSE_DATA_LENGTH];
+    struct { /* status block - additional sense data */
+        u_int16_t       srcnum;
+        u_int8_t        reserved;
+        u_int8_t        status;
+        u_int16_t       error;
+        u_int8_t        res__srcid;     /* 4:4 */
+        u_int8_t        res__severity;  /* 5:3 */
+    } status_block;
+    u_int8_t        err_specific_desc[TWS_ERROR_SPECIFIC_DESC_LEN];
+    struct { /* sense buffer descriptor */
+        u_int8_t        size_header;
+        u_int16_t       request_id;
+        u_int8_t        size_sense;
+    } header_desc;
+};
+
+/* Command - 1024 byte size including header (128+24+896)*/
+union tws_command_giga {
+    struct tws_cmd_init_connect       init_connect;
+    struct tws_cmd_download_firmware  download_fw;
+    struct tws_cmd_reset_firmware     reset_fw;
+    struct tws_cmd_param              param;
+    struct tws_cmd_generic            generic;
+    u_int8_t        padding[1024 - sizeof(struct tws_command_header)];
+};
+    
+/* driver command pkt - 1024 byte size including header(128+24+744+128) */
+/* h/w & f/w supported command size excluding header 768 */
+struct tws_command_apache {
+    u_int8_t        res__opcode;    /* 3:5 */
+    u_int8_t        unit;
+    u_int16_t       lun_l4__req_id; /* 4:12 */
+    u_int8_t        status;
+    u_int8_t        sgl_offset;     /* offset (in bytes) to sg_list, 
+                                     from the end of sgl_entries */
+    u_int16_t       lun_h4__sgl_entries;
+    u_int8_t        cdb[16];
+    u_int8_t        sg_list[744];   /* 768 - 24 */
+    u_int8_t        padding[128];   /* make it 1024 bytes */
+};
+
+struct tws_command_packet {
+    struct tws_command_header hdr;
+    union {
+        union tws_command_giga pkt_g;
+        struct tws_command_apache pkt_a;
+    } cmd;
+};
+
+/* Structure describing payload for get/set param commands. */
+struct tws_getset_param {
+    u_int16_t       table_id;
+    u_int8_t        parameter_id;
+    u_int8_t        reserved;
+    u_int16_t       parameter_size_bytes;
+    u_int16_t       parameter_actual_size_bytes;
+    u_int8_t        data[1];
+};
+
+struct tws_outbound_response {
+    u_int32_t     not_mfa   :1;   /* 1 if the structure is valid else MFA */
+    u_int32_t     reserved  :7;   /* reserved bits */
+    u_int32_t     status    :8;   /* should be 0 */
+    u_int32_t     request_id:16;  /* request id */
+};
+
+
+/* Scatter/Gather list entry with 32 bit addresses. */
+struct tws_sg_desc32 {
+    u_int32_t     address;
+    u_int32_t     length  :24;
+    u_int32_t     flag    :8;
+};
+
+/* Scatter/Gather list entry with 64 bit addresses. */
+struct tws_sg_desc64 {
+    u_int64_t     address;
+    u_int64_t     length   :32;
+    u_int64_t     reserved :24;
+    u_int64_t     flag     :8;
+};
+
+/*
+ * Packet that describes an AEN/error generated by the controller,
+ * shared with user
+ */
+struct tws_event_packet {
+    u_int32_t       sequence_id;
+    u_int32_t       time_stamp_sec;
+    u_int16_t       aen_code;
+    u_int8_t        severity;
+    u_int8_t        retrieved;
+    u_int8_t        repeat_count;
+    u_int8_t        parameter_len;
+    u_int8_t        parameter_data[TWS_ERROR_SPECIFIC_DESC_LEN];
+    u_int32_t       event_src;
+    u_int8_t        severity_str[20];
+};
+
+
+
+#pragma pack()
+
+struct tws_sense {
+    struct tws_command_header *hdr;
+    u_int64_t  hdr_pkt_phy;
+};
+
+struct tws_request {
+    struct tws_command_packet *cmd_pkt; /* command pkt */  
+    u_int64_t    cmd_pkt_phy;    /* cmd pkt physical address */       
+    void         *data;          /* ptr to data being passed to fw */
+    u_int32_t    length;         /* length of data being passed to fw */
+
+    u_int32_t    state;          /* request state */
+    u_int32_t    type;           /* request type */
+    u_int32_t    flags;          /* request flags */
+
+    u_int32_t    error_code;     /* error during request processing */
+
+    u_int32_t    request_id;     /* request id for tracking with fw */
+    void         (*cb)(struct tws_request *);      /* callback func */
+    bus_dmamap_t dma_map;        /* dma map */
+    union ccb    *ccb_ptr;       /* pointer to ccb */
+    struct callout_handle thandle; /* handle to req timeout */
+    struct tws_softc *sc;        /* pointer back to ctlr softc */
+
+    struct tws_request *next;    /* pointer to next request */
+    struct tws_request *prev;    /* pointer to prev request */
+};
+
+
diff --git a/sys/dev/tws/tws_services.c b/sys/dev/tws/tws_services.c
new file mode 100644
index 000000000000..07200b595d15
--- /dev/null
+++ b/sys/dev/tws/tws_services.c
@@ -0,0 +1,401 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/tws/tws.h>
+#include <dev/tws/tws_hdm.h>
+#include <dev/tws/tws_services.h>
+#include <sys/time.h>
+
+void tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req, 
+                                u_int8_t q_type );
+struct tws_request * tws_q_remove_request(struct tws_softc *sc, 
+                                struct tws_request *req, u_int8_t q_type );
+struct tws_request *tws_q_remove_head(struct tws_softc *sc, u_int8_t q_type );
+void tws_q_insert_head(struct tws_softc *sc, struct tws_request *req,
+                                u_int8_t q_type );
+struct tws_request * tws_q_remove_tail(struct tws_softc *sc, u_int8_t q_type );
+void tws_print_stats(void *arg);
+
+struct tws_sense *tws_find_sense_from_mfa(struct tws_softc *sc, u_int64_t mfa);
+
+
+
+struct error_desc array[] = {
+    { "Cannot add sysctl tree node", 0x2000, ERROR,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Register window not available", 0x2001, ERROR,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Can't allocate register window", 0x2002, ERROR,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Can't allocate interrupt", 0x2003, ERROR,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Can't set up interrupt", 0x2004, ERROR,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Couldn't intialize CAM", 0x2007, ERROR,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Couldn't create SIM device queue", 0x2100, ENOMEM,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Unable to  create SIM entry", 0x2101, ENOMEM,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Unable to  register the bus", 0x2102, ENXIO,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Unable to  create the path", 0x2103, ENXIO,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Bus scan request to CAM failed", 0x2104, ENXIO,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Unable to intialize the driver", 0x2008, ENXIO,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+    { "Unable to intialize the controller", 0x2009, ENXIO,
+       "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
+};
+
+void
+tws_trace(const char *file, const char *fun, int linenum,
+          struct tws_softc *sc, char *desc, u_int64_t val1, u_int64_t val2)
+{ 
+
+
+    struct tws_trace_rec *rec = (struct tws_trace_rec *)sc->trace_q.q;
+    volatile u_int16_t head, tail;
+    char fmt[256];
+
+    head = sc->trace_q.head;
+    tail = sc->trace_q.tail;
+/*
+    getnanotime(&rec[tail].ts);
+*/
+    strncpy(rec[tail].fname, file, TWS_TRACE_FNAME_LEN);
+    strncpy(rec[tail].func, fun, TWS_TRACE_FUNC_LEN);
+    rec[tail].linenum = linenum;
+    strncpy(rec[tail].desc, desc, TWS_TRACE_DESC_LEN);
+    rec[tail].val1 = val1;
+    rec[tail].val2 = val2;
+
+    tail = (tail+1) % sc->trace_q.depth;
+
+    if ( head == tail ) { 
+        sc->trace_q.overflow = 1;
+        sc->trace_q.head = (head+1) % sc->trace_q.depth;
+    }
+    sc->trace_q.tail = tail;
+
+/*
+    tws_circular_q_insert(sc, &sc->trace_q, 
+                              &rec, sizeof(struct tws_trace_rec));
+*/
+    if ( sc->is64bit )
+        strcpy(fmt, "%05d:%s::%s :%s: 0x%016lx : 0x%016lx \n");
+    else
+        strcpy(fmt, "%05d:%s::%s :%s: 0x%016llx : 0x%016llx \n");
+
+/*  
+    printf("%05d:%s::%s :%s: 0x%016llx : 0x%016llx \n", 
+            linenum, file, fun, desc, val1, val2);
+*/
+    printf(fmt, linenum, file, fun, desc, val1, val2);
+}
+
+void
+tws_log(struct tws_softc *sc, int index)
+{
+    device_printf((sc)->tws_dev, array[index].fmt,
+                    array[index].error_str,
+                    array[index].error_code,
+                    array[index].severity_level,
+                    array[index].desc );
+}
+
+/* ----------- swap functions ----------- */
+
+
+u_int16_t 
+tws_swap16(u_int16_t val)
+{
+    return((val << 8) | (val >> 8));
+}
+
+u_int32_t 
+tws_swap32(u_int32_t val)
+{
+    return(((val << 24) | ((val << 8) & (0xFF0000)) | 
+           ((val >> 8) & (0xFF00)) | (val >> 24)));
+}
+
+
+u_int64_t 
+tws_swap64(u_int64_t val)
+{
+    return((((u_int64_t)(tws_swap32(((u_int32_t *)(&(val)))[1]))) << 32) |
+           ((u_int32_t)(tws_swap32(((u_int32_t *)(&(val)))[0]))));
+}
+
+
+/* ----------- reg access ----------- */
+
+
+void
+tws_write_reg(struct tws_softc *sc, int offset, 
+                  u_int32_t value, int size)
+{
+    bus_space_tag_t         bus_tag = sc->bus_tag;
+    bus_space_handle_t      bus_handle = sc->bus_handle;
+
+    if (size == 4)
+        bus_space_write_4(bus_tag, bus_handle, offset, value);
+    else 
+        if (size == 2)
+            bus_space_write_2(bus_tag, bus_handle, offset, 
+                                     (u_int16_t)value);
+        else
+            bus_space_write_1(bus_tag, bus_handle, offset, (u_int8_t)value);
+}
+
+u_int32_t
+tws_read_reg(struct tws_softc *sc, int offset, int size)
+{
+    bus_space_tag_t bus_tag = sc->bus_tag;
+    bus_space_handle_t bus_handle = sc->bus_handle;
+
+    if (size == 4)
+        return((u_int32_t)bus_space_read_4(bus_tag, bus_handle, offset));
+    else if (size == 2)
+            return((u_int32_t)bus_space_read_2(bus_tag, bus_handle, offset));
+         else
+            return((u_int32_t)bus_space_read_1(bus_tag, bus_handle, offset));
+}
+
+/* --------------------- Q service --------------------- */
+
+/* 
+ * intialize q  pointers with null.
+ */
+void
+tws_init_qs(struct tws_softc *sc)
+{
+
+    mtx_lock(&sc->q_lock);
+    for(int i=0;i<TWS_MAX_QS;i++) {
+        sc->q_head[i] = NULL;
+        sc->q_tail[i] = NULL;
+    }
+    mtx_unlock(&sc->q_lock);
+
+}
+
+/* called with lock held */
+static void
+tws_insert2_empty_q(struct tws_softc *sc, struct tws_request *req, 
+                                u_int8_t q_type )
+{
+
+    mtx_assert(&sc->q_lock, MA_OWNED);
+    req->next = req->prev = NULL;
+    sc->q_head[q_type] = sc->q_tail[q_type] = req;
+
+}
+
+/* called with lock held */
+void
+tws_q_insert_head(struct tws_softc *sc, struct tws_request *req, 
+                                u_int8_t q_type )
+{
+
+    mtx_assert(&sc->q_lock, MA_OWNED);
+    if ( sc->q_head[q_type] == NULL ) {
+        tws_insert2_empty_q(sc, req, q_type);
+    } else {
+        req->next = sc->q_head[q_type];
+        req->prev = NULL;
+        sc->q_head[q_type]->prev = req;
+        sc->q_head[q_type] = req;
+    }
+
+}
+
+/* called with lock held */
+void
+tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req, 
+                                u_int8_t q_type )
+{
+
+    mtx_assert(&sc->q_lock, MA_OWNED);
+    if ( sc->q_tail[q_type] == NULL ) {
+        tws_insert2_empty_q(sc, req, q_type);
+    } else {
+        req->prev = sc->q_tail[q_type];
+        req->next = NULL;
+        sc->q_tail[q_type]->next = req;
+        sc->q_tail[q_type] = req;
+    }
+
+}
+
+/* called with lock held */
+struct tws_request *
+tws_q_remove_head(struct tws_softc *sc, u_int8_t q_type )
+{
+
+    struct tws_request *r;
+
+    mtx_assert(&sc->q_lock, MA_OWNED);
+    r = sc->q_head[q_type];
+    if ( !r ) 
+        return(NULL);
+    if ( r->next == NULL &&  r->prev == NULL ) {
+        /* last element  */
+        sc->q_head[q_type] = sc->q_tail[q_type] = NULL;
+    } else {
+        sc->q_head[q_type] = r->next;
+        r->next->prev = NULL;
+        r->next = NULL;
+        r->prev = NULL;
+    }
+    return(r);
+}
+
+/* called with lock held */
+struct tws_request *
+tws_q_remove_tail(struct tws_softc *sc, u_int8_t q_type )
+{
+
+    struct tws_request *r;
+
+    mtx_assert(&sc->q_lock, MA_OWNED);
+    r = sc->q_tail[q_type];
+    if ( !r ) 
+        return(NULL);
+    if ( r->next == NULL &&  r->prev == NULL ) {
+        /* last element  */
+        sc->q_head[q_type] = sc->q_tail[q_type] = NULL;
+    } else {
+        sc->q_tail[q_type] = r->prev;
+        r->prev->next = NULL;
+        r->next = NULL;
+        r->prev = NULL;
+    }
+    return(r);
+}
+
+/* returns removed request if successful. return NULL otherwise */ 
+/* called with lock held */
+struct tws_request *
+tws_q_remove_request(struct tws_softc *sc, struct tws_request *req,
+                                 u_int8_t q_type )
+{
+
+    struct tws_request *r;
+
+    mtx_assert(&sc->q_lock, MA_OWNED);
+    if ( req == NULL ) {
+        TWS_TRACE_DEBUG(sc, "null req", 0, q_type);
+        return(NULL);
+    }
+
+    if ( req == sc->q_head[q_type] )
+        return(tws_q_remove_head(sc, q_type));
+    if ( req == sc->q_tail[q_type] )
+        return(tws_q_remove_tail(sc, q_type));
+
+
+    /* The given node is not at head or tail.
+     * It's in the middle and there are more than
+     * 2 elements on the q.
+     */
+
+    if ( req->next == NULL || req->prev == NULL ) {
+        TWS_TRACE_DEBUG(sc, "invalid req", 0, q_type);
+        return(NULL);
+    }
+
+/* debug only */
+    r = sc->q_head[q_type];
+    while ( r ) {
+        if ( req == r )
+            break;
+        r = r->next;
+    } 
+
+    if ( !r ) {
+        TWS_TRACE_DEBUG(sc, "req not in q", 0, req->request_id);
+        return(NULL);
+    }
+/* debug end */
+
+    req->prev->next = r->next;
+    req->next->prev = r->prev;
+    req->next = NULL;
+    req->prev = NULL;
+    return(req);
+}
+
+struct tws_sense *
+tws_find_sense_from_mfa(struct tws_softc *sc, u_int64_t mfa)
+{
+    struct tws_sense *s;
+    int i;
+    TWS_TRACE_DEBUG(sc, "entry",sc,mfa);
+
+    i = (mfa - sc->dma_mem_phys) / sizeof(struct tws_command_packet);
+    if ( i>= 0 && i<tws_queue_depth) {
+        s = &sc->sense_bufs[i];
+        if ( mfa == s->hdr_pkt_phy )
+            return(s);
+    }
+
+    TWS_TRACE_DEBUG(sc, "return null",0,mfa);
+    return(NULL);
+
+}
+
+/* --------------------- Q service end --------------------- */
+/* --------------------- misc service start --------------------- */
+
+
+void
+tws_print_stats(void *arg)
+{
+
+    struct tws_softc *sc = (struct tws_softc *)arg;
+     
+    TWS_TRACE(sc, "reqs(in, out)", sc->stats.reqs_in, sc->stats.reqs_out);
+    TWS_TRACE(sc, "reqs(err, intrs)", sc->stats.reqs_errored
+                                      , sc->stats.num_intrs);
+    TWS_TRACE(sc, "reqs(ioctls, scsi)", sc->stats.ioctls
+                                      , sc->stats.scsi_ios);
+    timeout(tws_print_stats, sc, 300*hz);
+
+}
+/* --------------------- misc service end --------------------- */
diff --git a/sys/dev/tws/tws_services.h b/sys/dev/tws/tws_services.h
new file mode 100644
index 000000000000..643d72033917
--- /dev/null
+++ b/sys/dev/tws/tws_services.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+/* #define TWS_DEBUG on */
+
+void tws_trace(const char *file, const char *fun, int linenum,
+         struct tws_softc *sc,  char *desc, u_int64_t val1, u_int64_t val2);
+void tws_log(struct tws_softc *sc, int index);
+u_int32_t tws_read_reg(struct tws_softc *sc, 
+                  int offset, int size);
+void tws_write_reg(struct tws_softc *sc, int offset,
+                  u_int32_t value, int size);
+
+u_int16_t tws_swap16(u_int16_t val);
+u_int32_t tws_swap32(u_int32_t val);
+u_int64_t tws_swap64(u_int64_t val);
+
+void tws_init_qs(struct tws_softc *sc);
+
+
+
+/* ----------------- trace ----------------- */
+
+#define TWS_TRACE_ON on /* Alawys on - use wisely to trace errors */
+
+#ifdef TWS_DEBUG
+    #define TWS_TRACE_DEBUG_ON on
+#endif
+
+#ifdef TWS_TRACE_DEBUG_ON
+    #define TWS_TRACE_DEBUG(sc, desc, val1, val2) \
+            tws_trace(__FILE__, __func__, __LINE__, sc, desc, \
+                                   (u_int64_t)val1, (u_int64_t)val2)
+#else
+    #define TWS_TRACE_DEBUG(sc, desc, val1, val2)
+#endif
+
+#ifdef TWS_TRACE_ON
+    #define TWS_TRACE(sc, desc, val1, val2) \
+            tws_trace(__FILE__, __func__, __LINE__, sc, desc, \
+                                   (u_int64_t)val1, (u_int64_t)val2)
+#else
+    #define TWS_TRACE(sc, desc, val1, val2)
+#endif
+
+/* ---------------- logging ---------------- */
+
+
+/* ---------------- logging ---------------- */
+enum error_index {
+    SYSCTL_TREE_NODE_ADD,
+    PCI_COMMAND_READ,
+    ALLOC_MEMORY_RES,
+    ALLOC_IRQ_RES,
+    SETUP_INTR_RES,
+    TWS_CAM_ATTACH,
+    CAM_SIMQ_ALLOC,
+    CAM_SIM_ALLOC,
+    TWS_XPT_BUS_REGISTER,
+    TWS_XPT_CREATE_PATH,
+    TWS_BUS_SCAN_REQ,
+    TWS_INIT_FAILURE,
+    TWS_CTLR_INIT_FAILURE,
+};
+
+enum severity {
+    ERROR = 1,
+    WARNING,
+    INFO,
+#if 0
+    DEBUG,
+#endif
+};
+
+struct error_desc {
+    char desc[256];
+    u_int32_t error_code;
+    int severity_level;
+    char *fmt;
+    char *error_str;
+};
+
+extern struct error_desc array[];
+/* ----------- q services ------------- */
+
+#define TWS_FREE_Q        0
+#define TWS_PENDING_Q     1
+#define TWS_BUSY_Q        2
+#define TWS_COMPLETE_Q    3
+
+/* req return codes */
+#define TWS_REQ_RET_SUBMIT_SUCCESS 0
+#define TWS_REQ_RET_PEND_NOMFA     1
+#define TWS_REQ_RET_RESET          2
+#define TWS_REQ_RET_INVALID   0xdead
+
+
+/* ------------------------ */
+#if (__FreeBSD_version >= 700000)
+#include <sys/clock.h>
+#define TWS_LOCAL_TIME (time_second - utc_offset())
+#else
+#include <machine/clock.h>
+#define TWS_LOCAL_TIME (time_second - (tz_minuteswest * 60) -   \
+                  (wall_cmos_clock ? adjkerntz : 0))
+#endif
+
diff --git a/sys/dev/tws/tws_user.c b/sys/dev/tws/tws_user.c
new file mode 100644
index 000000000000..98b8a9be3252
--- /dev/null
+++ b/sys/dev/tws/tws_user.c
@@ -0,0 +1,379 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/tws/tws.h>
+#include <dev/tws/tws_services.h>
+#include <dev/tws/tws_hdm.h>
+#include <dev/tws/tws_user.h>
+
+
+int tws_ioctl(struct cdev *dev, long unsigned int cmd, caddr_t buf, int flags, 
+                                                    d_thread_t *proc);
+void tws_passthru_complete(struct tws_request *req);
+extern void tws_circular_aenq_insert(struct tws_softc *sc,
+                    struct tws_circular_q *cq, struct tws_event_packet *aen);
+
+
+static int tws_passthru(struct tws_softc *sc, void *buf);
+static int tws_ioctl_aen(struct tws_softc *sc, u_long cmd, void *buf);
+
+extern int tws_bus_scan(struct tws_softc *sc);
+extern struct tws_request *tws_get_request(struct tws_softc *sc, 
+                                           u_int16_t type);
+extern int32_t tws_map_request(struct tws_softc *sc, struct tws_request *req);
+extern void tws_unmap_request(struct tws_softc *sc, struct tws_request *req);
+extern uint8_t tws_get_state(struct tws_softc *sc);
+extern void tws_timeout(void *arg);
+
+int
+tws_ioctl(struct cdev *dev, u_long cmd, caddr_t buf, int flags, 
+                                                    d_thread_t *proc)
+{
+    struct tws_softc *sc = (struct tws_softc *)(dev->si_drv1);
+    int error;
+
+    TWS_TRACE_DEBUG(sc, "entry", sc, cmd);
+    sc->stats.ioctls++;
+    switch(cmd) {
+        case TWS_IOCTL_FIRMWARE_PASS_THROUGH :
+            error = tws_passthru(sc, (void *)buf);
+            break;
+        case TWS_IOCTL_SCAN_BUS :
+            TWS_TRACE_DEBUG(sc, "scan-bus", 0, 0);
+            mtx_lock(&sc->sim_lock);
+            error = tws_bus_scan(sc);
+            mtx_unlock(&sc->sim_lock);
+            break;
+        default :
+            TWS_TRACE_DEBUG(sc, "ioctl-aen", cmd, buf);
+            error = tws_ioctl_aen(sc, cmd, (void *)buf);
+            break;
+
+    }
+    return(error);
+}
+
+static int 
+tws_passthru(struct tws_softc *sc, void *buf)
+{
+    struct tws_request *req;
+    struct tws_ioctl_no_data_buf *ubuf = (struct tws_ioctl_no_data_buf *)buf;
+    int error;
+    u_int16_t lun4;
+
+    if ( tws_get_state(sc) != TWS_ONLINE) {
+        return(EBUSY);
+    }
+
+    do {
+        req = tws_get_request(sc, TWS_REQ_TYPE_PASSTHRU);
+        if ( !req ) {
+            sc->chan = 1;
+            error = tsleep((void *)&sc->chan,  0, 
+                                   "tws_sleep", TWS_IOCTL_TIMEOUT*hz);
+            if ( error == EWOULDBLOCK ) {
+                return(ETIMEDOUT);
+            }
+        } else {
+            break;
+        }
+    }while(1);
+
+    req->length = ubuf->driver_pkt.buffer_length;
+    TWS_TRACE_DEBUG(sc, "datal,rid", req->length, req->request_id);
+    if ( req->length ) {
+        req->data = malloc(req->length, M_TWS, M_WAITOK | M_ZERO);
+        if ( !req->data ) {
+            TWS_TRACE_DEBUG(sc, "malloc failed", 0, req->request_id);
+            req->state = TWS_REQ_STATE_FREE;
+            ubuf->driver_pkt.os_status = ENOMEM;
+            if ( sc->chan ) {
+                sc->chan = 0;
+                wakeup_one((void *)&sc->chan);
+            }
+            return(ENOMEM);
+        }
+        bzero(req->data, req->length);
+        error = copyin(ubuf->pdata, req->data, req->length);
+    }
+    req->flags = TWS_DIR_IN | TWS_DIR_OUT;
+    req->cb = tws_passthru_complete;
+
+    memcpy(&req->cmd_pkt->cmd, &ubuf->cmd_pkt.cmd, 
+                              sizeof(struct tws_command_apache));
+
+    if ( GET_OPCODE(req->cmd_pkt->cmd.pkt_a.res__opcode) == 
+                                               TWS_FW_CMD_EXECUTE_SCSI ) { 
+        lun4 = req->cmd_pkt->cmd.pkt_a.lun_l4__req_id & 0xF000;
+        req->cmd_pkt->cmd.pkt_a.lun_l4__req_id = lun4 | req->request_id;
+    } else {
+        req->cmd_pkt->cmd.pkt_g.generic.request_id = (u_int8_t) req->request_id;
+
+    }
+
+    error = tws_map_request(sc, req);
+    if (error) {
+        ubuf->driver_pkt.os_status = error;
+        goto out;
+    }
+
+//==================================================================================================
+    mtx_lock(&sc->gen_lock);
+    error = mtx_sleep(req, &sc->gen_lock, 0, "tws_passthru", TWS_IOCTL_TIMEOUT*hz);
+    mtx_unlock(&sc->gen_lock);
+    if (( req->state != TWS_REQ_STATE_COMPLETE ) && ( error == EWOULDBLOCK )) {
+            TWS_TRACE_DEBUG(sc, "msleep timeout", error, req->request_id);
+            tws_timeout((void*) req);
+    }
+
+    if ( req->error_code == TWS_REQ_RET_RESET ) {
+        error = EBUSY;
+        req->error_code = EBUSY;
+        TWS_TRACE_DEBUG(sc, "ioctl reset", error, req->request_id);
+    }
+
+    tws_unmap_request(sc, req);
+
+    memcpy(&ubuf->cmd_pkt.hdr, &req->cmd_pkt->hdr, sizeof(struct tws_command_apache));
+    memcpy(&ubuf->cmd_pkt.cmd, &req->cmd_pkt->cmd, sizeof(struct tws_command_apache));
+    if ( !error && req->length ) {
+        error = copyout(req->data, ubuf->pdata, req->length);
+    }
+//==================================================================================================
+
+out:
+    free(req->data, M_TWS);
+
+    if ( error ) 
+        TWS_TRACE_DEBUG(sc, "errored", error, 0);
+
+    if ( req->error_code != TWS_REQ_RET_SUBMIT_SUCCESS )
+        ubuf->driver_pkt.os_status = error;
+
+    req->state = TWS_REQ_STATE_FREE;
+
+    if ( sc->chan && (tws_get_state(sc) == TWS_ONLINE) ) {
+        sc->chan = 0;
+        wakeup_one((void *)&sc->chan);
+    }
+    return(error);
+}
+
+void 
+tws_passthru_complete(struct tws_request *req)
+{
+    req->state = TWS_REQ_STATE_COMPLETE;
+    wakeup_one(req);
+
+}
+
+static void 
+tws_retrive_aen(struct tws_softc *sc, u_long cmd, 
+                            struct tws_ioctl_packet *ubuf)
+{
+    u_int16_t index=0;
+    struct tws_event_packet eventp, *qp;
+
+    if ( sc->aen_q.head == sc->aen_q.tail ) {
+        ubuf->driver_pkt.status = TWS_AEN_NO_EVENTS;
+        return;
+    }
+    
+    ubuf->driver_pkt.status = 0;
+
+    /* 
+     * once this flag is set cli will not display alarms 
+     * needs a revisit from tools?
+     */
+    if ( sc->aen_q.overflow ) {
+        ubuf->driver_pkt.status = TWS_AEN_OVERFLOW;
+        sc->aen_q.overflow = 0; /* reset */
+    }
+
+    qp = (struct tws_event_packet *)sc->aen_q.q;
+
+    switch (cmd) {
+        case TWS_IOCTL_GET_FIRST_EVENT :
+            index = sc->aen_q.head;
+            break;
+        case TWS_IOCTL_GET_LAST_EVENT :
+            /* index = tail-1 */ 
+            index = (sc->aen_q.depth + sc->aen_q.tail - 1) % sc->aen_q.depth;
+            break;
+        case TWS_IOCTL_GET_NEXT_EVENT :
+            memcpy(&eventp, ubuf->data_buf, sizeof(struct tws_event_packet));
+            index = sc->aen_q.head;
+            do {
+                if ( qp[index].sequence_id == 
+                           (eventp.sequence_id + 1) )
+                    break;
+                index  = (index+1) % sc->aen_q.depth;
+            }while ( index != sc->aen_q.tail );
+            if ( index == sc->aen_q.tail ) {
+                ubuf->driver_pkt.status = TWS_AEN_NO_EVENTS;
+                return;
+            }
+            break;
+        case TWS_IOCTL_GET_PREVIOUS_EVENT :
+            memcpy(&eventp, ubuf->data_buf, sizeof(struct tws_event_packet));
+            index = sc->aen_q.head;
+            do {
+                if ( qp[index].sequence_id == 
+                           (eventp.sequence_id - 1) )
+                    break;
+                index  = (index+1) % sc->aen_q.depth;
+            }while ( index != sc->aen_q.tail );
+            if ( index == sc->aen_q.tail ) {
+                ubuf->driver_pkt.status = TWS_AEN_NO_EVENTS;
+                return;
+            }
+            break;
+        default :
+            TWS_TRACE_DEBUG(sc, "not a valid event", sc, cmd);
+            ubuf->driver_pkt.status = TWS_AEN_NO_EVENTS;
+            return;
+    }
+
+    memcpy(ubuf->data_buf, &qp[index], 
+                           sizeof(struct tws_event_packet));
+    qp[index].retrieved = TWS_AEN_RETRIEVED;
+
+    return;
+
+}
+
+static int 
+tws_ioctl_aen(struct tws_softc *sc, u_long cmd, void *buf)
+{
+
+    struct tws_ioctl_packet *ubuf = (struct tws_ioctl_packet *)buf;
+    struct tws_compatibility_packet cpkt;
+    struct tws_lock_packet lpkt;
+    time_t ctime;
+
+    mtx_lock(&sc->gen_lock);
+    ubuf->driver_pkt.status = 0;
+    switch(cmd) {
+        case TWS_IOCTL_GET_FIRST_EVENT :
+        case TWS_IOCTL_GET_LAST_EVENT :
+        case TWS_IOCTL_GET_NEXT_EVENT :
+        case TWS_IOCTL_GET_PREVIOUS_EVENT :
+            tws_retrive_aen(sc,cmd,ubuf);
+            break;
+        case TWS_IOCTL_GET_LOCK :
+            ctime = TWS_LOCAL_TIME;
+            memcpy(&lpkt, ubuf->data_buf, sizeof(struct tws_lock_packet));
+            if ( (sc->ioctl_lock.lock == TWS_IOCTL_LOCK_FREE) ||
+                 (lpkt.force_flag) ||
+                 (ctime >= sc->ioctl_lock.timeout) ) {
+                sc->ioctl_lock.lock = TWS_IOCTL_LOCK_HELD;
+                sc->ioctl_lock.timeout = ctime + (lpkt.timeout_msec / 1000);
+                lpkt.time_remaining_msec = lpkt.timeout_msec;
+            }  else {
+                lpkt.time_remaining_msec = (u_int32_t)
+                          ((sc->ioctl_lock.timeout - ctime) * 1000);
+                ubuf->driver_pkt.status = TWS_IOCTL_LOCK_ALREADY_HELD;
+
+            }
+            break;
+        case TWS_IOCTL_RELEASE_LOCK :
+            if (sc->ioctl_lock.lock == TWS_IOCTL_LOCK_FREE) {
+                ubuf->driver_pkt.status = TWS_IOCTL_LOCK_NOT_HELD;
+            } else {
+                sc->ioctl_lock.lock = TWS_IOCTL_LOCK_FREE;
+                ubuf->driver_pkt.status = 0;
+            }
+            break;
+        case TWS_IOCTL_GET_COMPATIBILITY_INFO :
+            TWS_TRACE_DEBUG(sc, "get comp info", sc, cmd);
+
+            memcpy( cpkt.driver_version, TWS_DRIVER_VERSION_STRING,
+                                         sizeof(TWS_DRIVER_VERSION_STRING));
+            cpkt.working_srl = sc->cinfo.working_srl;
+            cpkt.working_branch = sc->cinfo.working_branch;
+            cpkt.working_build = sc->cinfo.working_build;
+            cpkt.driver_srl_high = TWS_CURRENT_FW_SRL;
+            cpkt.driver_branch_high = TWS_CURRENT_FW_BRANCH;
+            cpkt.driver_build_high = TWS_CURRENT_FW_BUILD;
+            cpkt.driver_srl_low = TWS_BASE_FW_SRL;
+            cpkt.driver_branch_low = TWS_BASE_FW_BRANCH;
+            cpkt.driver_build_low = TWS_BASE_FW_BUILD;
+            cpkt.fw_on_ctlr_srl = sc->cinfo.fw_on_ctlr_srl;
+            cpkt.fw_on_ctlr_branch = sc->cinfo.fw_on_ctlr_branch;
+            cpkt.fw_on_ctlr_build = sc->cinfo.fw_on_ctlr_build;
+            ubuf->driver_pkt.status = 0;
+            int len = sizeof(struct tws_compatibility_packet);
+            if ( ubuf->driver_pkt.buffer_length < len )
+                len = ubuf->driver_pkt.buffer_length;
+            memcpy(ubuf->data_buf, &cpkt, len);
+
+            break;
+        default :
+            TWS_TRACE_DEBUG(sc, "not valid cmd", cmd, 
+                           TWS_IOCTL_GET_COMPATIBILITY_INFO);
+            break;
+
+    }
+    mtx_unlock(&sc->gen_lock);
+    return(SUCCESS);
+
+}
+
+void
+tws_circular_aenq_insert(struct tws_softc *sc, struct tws_circular_q *cq,
+struct tws_event_packet *aen)
+{
+
+    struct tws_event_packet *q = (struct tws_event_packet *)cq->q;
+    volatile u_int16_t head, tail;
+    u_int8_t retr;
+    mtx_assert(&sc->gen_lock, MA_OWNED);
+
+    head = cq->head;
+    tail = cq->tail;
+    retr = q[tail].retrieved;
+
+    memcpy(&q[tail], aen, sizeof(struct tws_event_packet));
+    tail = (tail+1) % cq->depth;
+
+    if ( head == tail ) { /* q is full */
+        if ( retr != TWS_AEN_RETRIEVED )
+            cq->overflow = 1;
+        cq->head = (head+1) % cq->depth;
+    }
+    cq->tail = tail;
+
+}
diff --git a/sys/dev/tws/tws_user.h b/sys/dev/tws/tws_user.h
new file mode 100644
index 000000000000..5ae29bc2d2da
--- /dev/null
+++ b/sys/dev/tws/tws_user.h
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2010, LSI Corp.
+ * All rights reserved.
+ * Author : Manjunath Ranganathaiah
+ * Support: freebsdraid@lsi.com
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of the <ORGANIZATION> nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define TWS_AEN_NOT_RETRIEVED        0x1
+#define TWS_AEN_RETRIEVED            0x2
+
+#define TWS_AEN_NO_EVENTS            0x1003  /* No more events */
+#define TWS_AEN_OVERFLOW             0x1004  /* AEN overflow occurred */
+
+#define TWS_IOCTL_LOCK_NOT_HELD      0x1001   /* Not locked */
+#define TWS_IOCTL_LOCK_ALREADY_HELD  0x1002   /* Already locked */
+
+#define TWS_IOCTL_LOCK_HELD          0x1
+#define TWS_IOCTL_LOCK_FREE          0x0
+
+#pragma pack(1)
+
+/* Structure used to handle GET/RELEASE LOCK ioctls. */
+struct tws_lock_packet {
+    u_int32_t       timeout_msec;
+    u_int32_t       time_remaining_msec;
+    u_int32_t       force_flag;
+};
+
+/* Structure used to handle GET COMPATIBILITY INFO ioctl. */
+struct tws_compatibility_packet {
+    u_int8_t    driver_version[32];/* driver version */
+    u_int16_t   working_srl;    /* driver & firmware negotiated srl */
+    u_int16_t   working_branch; /* branch # of the firmware that the
+                                    driver is compatible with */
+    u_int16_t   working_build;  /* build # of the firmware that the
+                                        driver is compatible with */
+    u_int16_t   driver_srl_high;/* highest driver supported srl */
+    u_int16_t   driver_branch_high;/* highest driver supported branch */
+    u_int16_t   driver_build_high;/* highest driver supported build */
+    u_int16_t   driver_srl_low;/* lowest driver supported srl */
+    u_int16_t   driver_branch_low;/* lowest driver supported branch */
+    u_int16_t   driver_build_low;/* lowest driver supported build */
+    u_int16_t   fw_on_ctlr_srl; /* srl of running firmware */
+    u_int16_t   fw_on_ctlr_branch;/* branch # of running firmware */
+    u_int16_t   fw_on_ctlr_build;/* build # of running firmware */
+};
+
+
+/* Driver understandable part of the ioctl packet built by the API. */
+struct tws_driver_packet {
+    u_int32_t       control_code;
+    u_int32_t       status;
+    u_int32_t       unique_id;
+    u_int32_t       sequence_id;
+    u_int32_t       os_status;
+    u_int32_t       buffer_length;
+};
+
+/* ioctl packet built by the API. */
+struct tws_ioctl_packet {
+    struct tws_driver_packet      driver_pkt;
+    char                          padding[488];
+    struct tws_command_packet     cmd_pkt;
+    char                          data_buf[1];
+};
+
+#pragma pack()
+
+
+#pragma pack(1)
+/*
+ * We need the structure below to ensure that the first byte of
+ * data_buf is not overwritten by the kernel, after we return
+ * from the ioctl call.  Note that cmd_pkt has been reduced
+ * to an array of 1024 bytes even though it's actually 2048 bytes
+ * in size.  This is because, we don't expect requests from user
+ * land requiring 2048 (273 sg elements) byte cmd pkts.
+ */
+struct tws_ioctl_no_data_buf {
+    struct tws_driver_packet     driver_pkt;
+    void                         *pdata; /* points to data_buf */
+    char                         padding[488 - sizeof(void *)];
+    struct tws_command_packet    cmd_pkt;
+};
+
+#pragma pack()
+
+
+#include <sys/ioccom.h> 
+
+#pragma pack(1)
+
+struct tws_ioctl_with_payload {
+    struct tws_driver_packet     driver_pkt;
+    char                         padding[488];
+    struct tws_command_packet    cmd_pkt;
+    union {
+        struct tws_event_packet               event_pkt;
+        struct tws_lock_packet                lock_pkt;
+        struct tws_compatibility_packet       compat_pkt;
+        char                                  data_buf[1];
+    } payload;
+};
+
+#pragma pack()
+
+/* ioctl cmds */
+
+#define TWS_IOCTL_SCAN_BUS                            \
+        _IO('T', 200)
+#define TWS_IOCTL_FIRMWARE_PASS_THROUGH               \
+        _IOWR('T', 202, struct tws_ioctl_no_data_buf)
+#define TWS_IOCTL_GET_FIRST_EVENT                     \
+        _IOWR('T', 203, struct tws_ioctl_with_payload)
+#define TWS_IOCTL_GET_LAST_EVENT                      \
+        _IOWR('T', 204, struct tws_ioctl_with_payload)
+#define TWS_IOCTL_GET_NEXT_EVENT                      \
+        _IOWR('T', 205, struct tws_ioctl_with_payload)
+#define TWS_IOCTL_GET_PREVIOUS_EVENT                  \
+        _IOWR('T', 206, struct tws_ioctl_with_payload)
+#define TWS_IOCTL_GET_LOCK                            \
+        _IOWR('T', 207, struct tws_ioctl_with_payload)
+#define TWS_IOCTL_RELEASE_LOCK                        \
+        _IOWR('T', 208, struct tws_ioctl_with_payload)
+#define TWS_IOCTL_GET_COMPATIBILITY_INFO              \
+        _IOWR('T', 209, struct tws_ioctl_with_payload)
+
diff --git a/sys/dev/usb/controller/xhci.c b/sys/dev/usb/controller/xhci.c
index f6c8c0e24b4f..0df4c4d174e9 100644
--- a/sys/dev/usb/controller/xhci.c
+++ b/sys/dev/usb/controller/xhci.c
@@ -3048,7 +3048,9 @@ xhci_roothub_exec(struct usb_device *udev,
 		}
 		port = XHCI_PORTSC(index);
 
-		v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR;
+		v = XREAD4(sc, oper, port);
+		i = XHCI_PS_PLS_GET(v);
+		v &= ~XHCI_PS_CLEAR;
 
 		switch (value) {
 		case UHF_C_BH_PORT_RESET:
@@ -3082,6 +3084,17 @@ xhci_roothub_exec(struct usb_device *udev,
 			XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3));
 			break;
 		case UHF_PORT_SUSPEND:
+
+			/* U3 -> U15 */
+			if (i == 3) {
+				XWRITE4(sc, oper, port, v |
+				    XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS);
+			}
+
+			/* wait 20ms for resume sequence to complete */
+			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
+
+			/* U0 */
 			XWRITE4(sc, oper, port, v |
 			    XHCI_PS_PLS_SET(0) | XHCI_PS_LWS);
 			break;
diff --git a/sys/dev/usb/controller/xhcireg.h b/sys/dev/usb/controller/xhcireg.h
index 71c5c284a17f..8be502ef3137 100644
--- a/sys/dev/usb/controller/xhcireg.h
+++ b/sys/dev/usb/controller/xhcireg.h
@@ -133,7 +133,7 @@
 #define	XHCI_PS_WOE		0x08000000	/* RW - wake on over-current enable */
 #define	XHCI_PS_DR		0x40000000	/* RO - device removable */
 #define	XHCI_PS_WPR		0x80000000U	/* RW - warm port reset */
-#define	XHCI_PS_CLEAR		0x80FF00F7U	/* command bits */
+#define	XHCI_PS_CLEAR		0x80FF01FFU	/* command bits */
 
 #define	XHCI_PORTPMSC(n)	(0x3F4 + (0x10 * (n)))	/* XHCI status and control */
 #define	XHCI_PM3_U1TO_GET(x)	(((x) >> 0) & 0xFF)	/* RW - U1 timeout */
diff --git a/sys/dev/usb/input/uhid.c b/sys/dev/usb/input/uhid.c
index 1b810e321a0c..47231423c2e2 100644
--- a/sys/dev/usb/input/uhid.c
+++ b/sys/dev/usb/input/uhid.c
@@ -566,8 +566,10 @@ uhid_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr,
 		default:
 			return (EINVAL);
 		}
+		if (id != 0)
+			copyin(ugd->ugd_data, &id, 1);
 		error = uhid_get_report(sc, ugd->ugd_report_type, id,
-		    NULL, ugd->ugd_data, size);
+		    NULL, ugd->ugd_data, imin(ugd->ugd_maxlen, size));
 		break;
 
 	case USB_SET_REPORT:
@@ -592,8 +594,10 @@ uhid_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr,
 		default:
 			return (EINVAL);
 		}
+		if (id != 0)
+			copyin(ugd->ugd_data, &id, 1);
 		error = uhid_set_report(sc, ugd->ugd_report_type, id,
-		    NULL, ugd->ugd_data, size);
+		    NULL, ugd->ugd_data, imin(ugd->ugd_maxlen, size));
 		break;
 
 	case USB_GET_REPORT_ID:
diff --git a/sys/dev/usb/net/if_aue.c b/sys/dev/usb/net/if_aue.c
index 86bfe20b432a..adf47f41ff07 100644
--- a/sys/dev/usb/net/if_aue.c
+++ b/sys/dev/usb/net/if_aue.c
@@ -1032,9 +1032,9 @@ aue_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	AUE_LOCK(sc);
 	mii_pollstat(mii);
-	AUE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	AUE_UNLOCK(sc);
 }
 
 /*
diff --git a/sys/dev/usb/net/if_axe.c b/sys/dev/usb/net/if_axe.c
index a440294ae345..62adda47b6e5 100644
--- a/sys/dev/usb/net/if_axe.c
+++ b/sys/dev/usb/net/if_axe.c
@@ -76,24 +76,30 @@ __FBSDID("$FreeBSD$");
  * http://www.asix.com.tw/FrootAttach/datasheet/AX88772_datasheet_Rev10.pdf
  */
 
-#include <sys/stdint.h>
-#include <sys/stddef.h>
 #include <sys/param.h>
-#include <sys/queue.h>
-#include <sys/types.h>
 #include <sys/systm.h>
-#include <sys/kernel.h>
 #include <sys/bus.h>
-#include <sys/module.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
 #include <sys/condvar.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <sys/sx.h>
-#include <sys/unistd.h>
-#include <sys/callout.h>
-#include <sys/malloc.h>
-#include <sys/priv.h>
+
+#include <net/if.h>
+#include <net/ethernet.h>
+#include <net/if_types.h>
+#include <net/if_media.h>
+#include <net/if_vlan_var.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/miivar.h>
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
@@ -122,6 +128,8 @@ __FBSDID("$FreeBSD$");
  */
 #define AXE_178_MAX_FRAME_BURST	1
 
+#define	AXE_CSUM_FEATURES	(CSUM_IP | CSUM_TCP | CSUM_UDP)
+
 #ifdef USB_DEBUG
 static int axe_debug = 0;
 
@@ -186,6 +194,7 @@ static uether_fn_t axe_tick;
 static uether_fn_t axe_setmulti;
 static uether_fn_t axe_setpromisc;
 
+static int	axe_attach_post_sub(struct usb_ether *);
 static int	axe_ifmedia_upd(struct ifnet *);
 static void	axe_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 static int	axe_cmd(struct axe_softc *, int, int, int, void *);
@@ -195,6 +204,11 @@ static void	axe_ax88772_phywake(struct axe_softc *);
 static void	axe_ax88772a_init(struct axe_softc *);
 static void	axe_ax88772b_init(struct axe_softc *);
 static int	axe_get_phyno(struct axe_softc *, int);
+static int	axe_ioctl(struct ifnet *, u_long, caddr_t);
+static int	axe_rx_frame(struct usb_ether *, struct usb_page_cache *, int);
+static int	axe_rxeof(struct usb_ether *, struct usb_page_cache *,
+		    unsigned int offset, unsigned int, struct axe_csum_hdr *);
+static void	axe_csum_cfg(struct usb_ether *);
 
 static const struct usb_config axe_config[AXE_N_TRANSFER] = {
 
@@ -266,6 +280,7 @@ MODULE_VERSION(axe, 1);
 
 static const struct usb_ether_methods axe_ue_methods = {
 	.ue_attach_post = axe_attach_post,
+	.ue_attach_post_sub = axe_attach_post_sub,
 	.ue_start = axe_start,
 	.ue_init = axe_init,
 	.ue_stop = axe_stop,
@@ -304,9 +319,6 @@ axe_miibus_readreg(device_t dev, int phy, int reg)
 	uint16_t val;
 	int locked;
 
-	if (sc->sc_phyno != phy)
-		return (0);
-
 	locked = mtx_owned(&sc->sc_mtx);
 	if (!locked)
 		AXE_LOCK(sc);
@@ -338,10 +350,6 @@ axe_miibus_writereg(device_t dev, int phy, int reg, int val)
 	int locked;
 
 	val = htole32(val);
-
-	if (sc->sc_phyno != phy)
-		return (0);
-
 	locked = mtx_owned(&sc->sc_mtx);
 	if (!locked)
 		AXE_LOCK(sc);
@@ -372,7 +380,7 @@ axe_miibus_statchg(device_t dev)
 	if (mii == NULL || ifp == NULL ||
 	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 		goto done;
- 
+
 	sc->sc_flags &= ~AXE_FLAG_LINK;
 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 	    (IFM_ACTIVE | IFM_AVALID)) {
@@ -390,14 +398,23 @@ axe_miibus_statchg(device_t dev)
 			break;
 		}
 	}
- 
+
 	/* Lost link, do nothing. */
 	if ((sc->sc_flags & AXE_FLAG_LINK) == 0)
 		goto done;
- 
+
 	val = 0;
-	if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0)
+	if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) {
 		val |= AXE_MEDIA_FULL_DUPLEX;
+		if (AXE_IS_178_FAMILY(sc)) {
+			if ((IFM_OPTIONS(mii->mii_media_active) &
+			    IFM_ETH_TXPAUSE) != 0)
+				val |= AXE_178_MEDIA_TXFLOW_CONTROL_EN;
+			if ((IFM_OPTIONS(mii->mii_media_active) &
+			    IFM_ETH_RXPAUSE) != 0)
+				val |= AXE_178_MEDIA_RXFLOW_CONTROL_EN;
+		}
+	}
 	if (AXE_IS_178_FAMILY(sc)) {
 		val |= AXE_178_MEDIA_RX_EN | AXE_178_MEDIA_MAGIC;
 		if ((sc->sc_flags & AXE_FLAG_178) != 0)
@@ -452,9 +469,9 @@ axe_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	AXE_LOCK(sc);
 	mii_pollstat(mii);
-	AXE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	AXE_UNLOCK(sc);
 }
 
 static void
@@ -845,6 +862,53 @@ axe_attach_post(struct usb_ether *ue)
 		axe_cmd(sc, AXE_CMD_READ_IPG012, 0, 0, sc->sc_ipgs);
 }
 
+static int
+axe_attach_post_sub(struct usb_ether *ue)
+{
+	struct axe_softc *sc;
+	struct ifnet *ifp;
+	u_int adv_pause;
+	int error;
+
+	sc = uether_getsc(ue);
+	ifp = ue->ue_ifp;
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_start = uether_start;
+	ifp->if_ioctl = axe_ioctl;
+	ifp->if_init = uether_init;
+	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
+	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
+	IFQ_SET_READY(&ifp->if_snd);
+
+	if (AXE_IS_178_FAMILY(sc))
+		ifp->if_capabilities |= IFCAP_VLAN_MTU;
+	if (sc->sc_flags & AXE_FLAG_772B) {
+		ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_RXCSUM;
+		ifp->if_hwassist = AXE_CSUM_FEATURES;
+		/*
+		 * Checksum offloading of AX88772B also works with VLAN
+		 * tagged frames but there is no way to take advantage
+		 * of the feature because vlan(4) assumes
+		 * IFCAP_VLAN_HWTAGGING is prerequisite condition to
+		 * support checksum offloading with VLAN. VLAN hardware
+		 * tagging support of AX88772B is very limited so it's
+		 * not possible to announce IFCAP_VLAN_HWTAGGING.
+		 */
+	}
+	ifp->if_capenable = ifp->if_capabilities;
+	if (sc->sc_flags & (AXE_FLAG_772A | AXE_FLAG_772B | AXE_FLAG_178))
+		adv_pause = MIIF_DOPAUSE;
+	else
+		adv_pause = 0;
+	mtx_lock(&Giant);
+	error = mii_attach(ue->ue_dev, &ue->ue_miibus, ifp,
+	    uether_ifmedia_upd, ue->ue_methods->ue_mii_sts,
+	    BMSR_DEFCAPMASK, sc->sc_phyno, MII_OFFSET_ANY, adv_pause);
+	mtx_unlock(&Giant);
+
+	return (error);
+}
+
 /*
  * Probe for a AX88172 chip.
  */
@@ -930,52 +994,15 @@ axe_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error)
 {
 	struct axe_softc *sc = usbd_xfer_softc(xfer);
 	struct usb_ether *ue = &sc->sc_ue;
-	struct ifnet *ifp = uether_getifp(ue);
-	struct axe_sframe_hdr hdr;
 	struct usb_page_cache *pc;
-	int err, pos, len;
 	int actlen;
 
 	usbd_xfer_status(xfer, &actlen, NULL, NULL, NULL);
 
 	switch (USB_GET_STATE(xfer)) {
 	case USB_ST_TRANSFERRED:
-		pos = 0;
-		len = 0;
-		err = 0;
-
 		pc = usbd_xfer_get_frame(xfer, 0);
-		if (AXE_IS_178_FAMILY(sc)) {
-			while (pos < actlen) {
-				if ((pos + sizeof(hdr)) > actlen) {
-					/* too little data */
-					err = EINVAL;
-					break;
-				}
-				usbd_copy_out(pc, pos, &hdr, sizeof(hdr));
-
-				if ((hdr.len ^ hdr.ilen) != 0xFFFF) {
-					/* we lost sync */
-					err = EINVAL;
-					break;
-				}
-				pos += sizeof(hdr);
-
-				len = le16toh(hdr.len);
-				if ((pos + len) > actlen) {
-					/* invalid length */
-					err = EINVAL;
-					break;
-				}
-				uether_rxbuf(ue, pc, pos, len);
-
-				pos += len + (len % 2);
-			}
-		} else
-			uether_rxbuf(ue, pc, 0, actlen);
-
-		if (err != 0)
-			ifp->if_ierrors++;
+		axe_rx_frame(ue, pc, actlen);
 
 		/* FALLTHROUGH */
 	case USB_ST_SETUP:
@@ -998,6 +1025,131 @@ axe_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error)
 	}
 }
 
+static int
+axe_rx_frame(struct usb_ether *ue, struct usb_page_cache *pc, int actlen)
+{
+	struct axe_softc *sc;
+	struct axe_sframe_hdr hdr;
+	struct axe_csum_hdr csum_hdr;
+	int error, len, pos;
+
+	sc = uether_getsc(ue);
+	pos = 0;
+	len = 0;
+	error = 0;
+	if ((sc->sc_flags & AXE_FLAG_STD_FRAME) != 0) {
+		while (pos < actlen) {
+			if ((pos + sizeof(hdr)) > actlen) {
+				/* too little data */
+				error = EINVAL;
+				break;
+			}
+			usbd_copy_out(pc, pos, &hdr, sizeof(hdr));
+
+			if ((hdr.len ^ hdr.ilen) != sc->sc_lenmask) {
+				/* we lost sync */
+				error = EINVAL;
+				break;
+			}
+			pos += sizeof(hdr);
+			len = le16toh(hdr.len);
+			if (pos + len > actlen) {
+				/* invalid length */
+				error = EINVAL;
+				break;
+			}
+			axe_rxeof(ue, pc, pos, len, NULL);
+			pos += len + (len % 2);
+		}
+	} else if ((sc->sc_flags & AXE_FLAG_CSUM_FRAME) != 0) {
+		while (pos < actlen) {
+			if ((pos + sizeof(csum_hdr)) > actlen) {
+				/* too little data */
+				error = EINVAL;
+				break;
+			}
+			usbd_copy_out(pc, pos, &csum_hdr, sizeof(csum_hdr));
+
+			csum_hdr.len = le16toh(csum_hdr.len);
+			csum_hdr.ilen = le16toh(csum_hdr.ilen);
+			csum_hdr.cstatus = le16toh(csum_hdr.cstatus);
+			if ((AXE_CSUM_RXBYTES(csum_hdr.len) ^
+			    AXE_CSUM_RXBYTES(csum_hdr.ilen)) !=
+			    sc->sc_lenmask) {
+				/* we lost sync */
+				error = EINVAL;
+				break;
+			}
+			/*
+			 * Get total transferred frame length including
+			 * checksum header.  The length should be multiple
+			 * of 4.
+			 */
+			len = sizeof(csum_hdr) + AXE_CSUM_RXBYTES(csum_hdr.len);
+			len = (len + 3) & ~3;
+			if (pos + len > actlen) {
+				/* invalid length */
+				error = EINVAL;
+				break;
+			}
+			axe_rxeof(ue, pc, pos + sizeof(csum_hdr),
+			    AXE_CSUM_RXBYTES(csum_hdr.len), &csum_hdr);
+			pos += len;
+		}
+	} else
+		axe_rxeof(ue, pc, 0, actlen, NULL);
+
+	if (error != 0)
+		ue->ue_ifp->if_ierrors++;
+	return (error);
+}
+
+static int
+axe_rxeof(struct usb_ether *ue, struct usb_page_cache *pc, unsigned int offset,
+    unsigned int len, struct axe_csum_hdr *csum_hdr)
+{
+	struct ifnet *ifp = ue->ue_ifp;
+	struct mbuf *m;
+
+	if (len < ETHER_HDR_LEN || len > MCLBYTES - ETHER_ALIGN) {
+		ifp->if_ierrors++;
+		return (EINVAL);
+	}
+
+	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+	if (m == NULL) {
+		ifp->if_iqdrops++;
+		return (ENOMEM);
+	}
+	m->m_len = m->m_pkthdr.len = MCLBYTES;
+	m_adj(m, ETHER_ALIGN);
+
+	usbd_copy_out(pc, offset, mtod(m, uint8_t *), len);
+
+	ifp->if_ipackets++;
+	m->m_pkthdr.rcvif = ifp;
+	m->m_pkthdr.len = m->m_len = len;
+
+	if (csum_hdr != NULL && csum_hdr->cstatus & AXE_CSUM_HDR_L3_TYPE_IPV4) {
+		if ((csum_hdr->cstatus & (AXE_CSUM_HDR_L4_CSUM_ERR |
+		    AXE_CSUM_HDR_L3_CSUM_ERR)) == 0) {
+			m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED |
+			    CSUM_IP_VALID;
+			if ((csum_hdr->cstatus & AXE_CSUM_HDR_L4_TYPE_MASK) ==
+			    AXE_CSUM_HDR_L4_TYPE_TCP ||
+			    (csum_hdr->cstatus & AXE_CSUM_HDR_L4_TYPE_MASK) ==
+			    AXE_CSUM_HDR_L4_TYPE_UDP) {
+				m->m_pkthdr.csum_flags |=
+				    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+				m->m_pkthdr.csum_data = 0xffff;
+			}
+		}
+	}
+
+	_IF_ENQUEUE(&ue->ue_rxq, m);
+	return (0);
+}
+
 #if ((AXE_BULK_BUF_SIZE >= 0x10000) || (AXE_BULK_BUF_SIZE < (MCLBYTES+4)))
 #error "Please update axe_bulk_write_callback()!"
 #endif
@@ -1040,6 +1192,21 @@ axe_bulk_write_callback(struct usb_xfer *xfer, usb_error_t error)
 			if (AXE_IS_178_FAMILY(sc)) {
 				hdr.len = htole16(m->m_pkthdr.len);
 				hdr.ilen = ~hdr.len;
+				/*
+				 * If upper stack computed checksum, driver
+				 * should tell controller not to insert
+				 * computed checksum for checksum offloading
+				 * enabled controller.
+				 */
+				if (ifp->if_capabilities & IFCAP_TXCSUM) {
+					if ((m->m_pkthdr.csum_flags &
+					    AXE_CSUM_FEATURES) != 0)
+						hdr.len |= htole16(
+						    AXE_TX_CSUM_PSEUDO_HDR);
+					else
+						hdr.len |= htole16(
+						    AXE_TX_CSUM_DIS);
+				}
 				usbd_copy_in(pc, pos, &hdr, sizeof(hdr));
 				pos += sizeof(hdr);
 				usbd_m_copy_in(pc, pos, m, 0, m->m_pkthdr.len);
@@ -1130,6 +1297,34 @@ axe_start(struct usb_ether *ue)
 	usbd_transfer_start(sc->sc_xfer[AXE_BULK_DT_WR]);
 }
 
+static void
+axe_csum_cfg(struct usb_ether *ue)
+{
+	struct axe_softc *sc;
+	struct ifnet *ifp;
+	uint16_t csum1, csum2;
+
+	sc = uether_getsc(ue);
+	AXE_LOCK_ASSERT(sc, MA_OWNED);
+
+	if ((sc->sc_flags & AXE_FLAG_772B) != 0) {
+		ifp = uether_getifp(ue);
+		csum1 = 0;
+		csum2 = 0;
+		if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
+			csum1 |= AXE_TXCSUM_IP | AXE_TXCSUM_TCP |
+			    AXE_TXCSUM_UDP;
+		axe_cmd(sc, AXE_772B_CMD_WRITE_TXCSUM, csum2, csum1, NULL);
+		csum1 = 0;
+		csum2 = 0;
+		if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
+			csum1 |= AXE_RXCSUM_IP | AXE_RXCSUM_IPVE |
+			    AXE_RXCSUM_TCP | AXE_RXCSUM_UDP | AXE_RXCSUM_ICMP |
+			    AXE_RXCSUM_IGMP;
+		axe_cmd(sc, AXE_772B_CMD_WRITE_RXCSUM, csum2, csum1, NULL);
+	}
+}
+
 static void
 axe_init(struct usb_ether *ue)
 {
@@ -1147,27 +1342,40 @@ axe_init(struct usb_ether *ue)
 
 	axe_reset(sc);
 
-	/* Set MAC address. */
-	if (AXE_IS_178_FAMILY(sc))
+	/* Set MAC address and transmitter IPG values. */
+	if (AXE_IS_178_FAMILY(sc)) {
 		axe_cmd(sc, AXE_178_CMD_WRITE_NODEID, 0, 0, IF_LLADDR(ifp));
-	else
-		axe_cmd(sc, AXE_172_CMD_WRITE_NODEID, 0, 0, IF_LLADDR(ifp));
-
-	/* Set transmitter IPG values */
-	if (AXE_IS_178_FAMILY(sc))
 		axe_cmd(sc, AXE_178_CMD_WRITE_IPG012, sc->sc_ipgs[2],
 		    (sc->sc_ipgs[1] << 8) | (sc->sc_ipgs[0]), NULL);
-	else {
+	} else {
+		axe_cmd(sc, AXE_172_CMD_WRITE_NODEID, 0, 0, IF_LLADDR(ifp));
 		axe_cmd(sc, AXE_172_CMD_WRITE_IPG0, 0, sc->sc_ipgs[0], NULL);
 		axe_cmd(sc, AXE_172_CMD_WRITE_IPG1, 0, sc->sc_ipgs[1], NULL);
 		axe_cmd(sc, AXE_172_CMD_WRITE_IPG2, 0, sc->sc_ipgs[2], NULL);
 	}
 
-	/* AX88772B uses different maximum frame burst configuration. */
-	if (sc->sc_flags & AXE_FLAG_772B)
+	if (AXE_IS_178_FAMILY(sc)) {
+		sc->sc_flags &= ~(AXE_FLAG_STD_FRAME | AXE_FLAG_CSUM_FRAME);
+		if ((sc->sc_flags & AXE_FLAG_772B) != 0)
+			sc->sc_lenmask = AXE_CSUM_HDR_LEN_MASK;
+		else
+			sc->sc_lenmask = AXE_HDR_LEN_MASK;
+		if ((sc->sc_flags & AXE_FLAG_772B) != 0 &&
+		    (ifp->if_capenable & IFCAP_RXCSUM) != 0)
+			sc->sc_flags |= AXE_FLAG_CSUM_FRAME;
+		else
+			sc->sc_flags |= AXE_FLAG_STD_FRAME;
+	}
+
+	/* Configure TX/RX checksum offloading. */
+	axe_csum_cfg(ue);
+
+	if (sc->sc_flags & AXE_FLAG_772B) {
+		/* AX88772B uses different maximum frame burst configuration. */
 		axe_cmd(sc, AXE_772B_CMD_RXCTL_WRITE_CFG,
 		    ax88772b_mfb_table[AX88772B_MFB_16K].threshold,
 		    ax88772b_mfb_table[AX88772B_MFB_16K].byte_cnt, NULL);
+	}
 
 	/* Enable receiver, set RX mode. */
 	rxmode = (AXE_RXCMD_MULTICAST | AXE_RXCMD_ENABLE);
@@ -1175,11 +1383,17 @@ axe_init(struct usb_ether *ue)
 		if (sc->sc_flags & AXE_FLAG_772B) {
 			/*
 			 * Select RX header format type 1.  Aligning IP
-			 * header on 4 byte boundary is not needed
+			 * header on 4 byte boundary is not needed when
+			 * checksum offloading feature is not used
 			 * because we always copy the received frame in
-			 * RX handler.
+			 * RX handler.  When RX checksum offloading is
+			 * active, aligning IP header is required to
+			 * reflect actual frame length including RX
+			 * header size.
 			 */
 			rxmode |= AXE_772B_RXCMD_HDR_TYPE_1;
+			if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
+				rxmode |= AXE_772B_RXCMD_IPHDR_ALIGN;
 		} else {
 			/*
 			 * Default Rx buffer size is too small to get
@@ -1208,7 +1422,6 @@ axe_init(struct usb_ether *ue)
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	/* Switch to selected media. */
 	axe_ifmedia_upd(ifp);
-	axe_start(ue);
 }
 
 static void
@@ -1250,3 +1463,45 @@ axe_stop(struct usb_ether *ue)
 	usbd_transfer_stop(sc->sc_xfer[AXE_BULK_DT_WR]);
 	usbd_transfer_stop(sc->sc_xfer[AXE_BULK_DT_RD]);
 }
+
+static int
+axe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+	struct usb_ether *ue = ifp->if_softc;
+	struct axe_softc *sc;
+	struct ifreq *ifr;
+	int error, mask, reinit;
+
+	sc = uether_getsc(ue);
+	ifr = (struct ifreq *)data;
+	error = 0;
+	reinit = 0;
+	if (cmd == SIOCSIFCAP) {
+		AXE_LOCK(sc);
+		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+		if ((mask & IFCAP_TXCSUM) != 0 &&
+		    (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+			if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
+				ifp->if_hwassist |= AXE_CSUM_FEATURES;
+			else
+				ifp->if_hwassist &= ~AXE_CSUM_FEATURES;
+			reinit++;
+		}
+		if ((mask & IFCAP_RXCSUM) != 0 &&
+		    (ifp->if_capabilities & IFCAP_RXCSUM) != 0) {
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+			reinit++;
+		}
+		if (reinit > 0 && ifp->if_drv_flags & IFF_DRV_RUNNING)
+			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+		else
+			reinit = 0;
+		AXE_UNLOCK(sc);
+		if (reinit > 0)
+			uether_init(ue);
+	} else
+		error = uether_ioctl(ifp, cmd, data);
+
+	return (error);
+}
diff --git a/sys/dev/usb/net/if_axereg.h b/sys/dev/usb/net/if_axereg.h
index 6e08997b369f..ab2b28d24fe9 100644
--- a/sys/dev/usb/net/if_axereg.h
+++ b/sys/dev/usb/net/if_axereg.h
@@ -97,6 +97,10 @@
 #define	AXE_CMD_WRITE_VLAN_CTRL			0x4028
 
 #define	AXE_772B_CMD_RXCTL_WRITE_CFG		0x012A
+#define	AXE_772B_CMD_READ_RXCSUM		0x002B
+#define	AXE_772B_CMD_WRITE_RXCSUM		0x012C
+#define	AXE_772B_CMD_READ_TXCSUM		0x002D
+#define	AXE_772B_CMD_WRITE_TXCSUM		0x012E
 
 #define	AXE_SW_RESET_CLEAR			0x00
 #define	AXE_SW_RESET_RR				0x01
@@ -199,6 +203,40 @@
 #define	AXE_VLAN_CTRL_VID1_MASK		0x00000FFF
 #define	AXE_VLAN_CTRL_VID2_MASK		0x0FFF0000
 
+#define	AXE_RXCSUM_IP			0x0001
+#define	AXE_RXCSUM_IPVE			0x0002
+#define	AXE_RXCSUM_IPV6E		0x0004
+#define	AXE_RXCSUM_TCP			0x0008
+#define	AXE_RXCSUM_UDP			0x0010
+#define	AXE_RXCSUM_ICMP			0x0020
+#define	AXE_RXCSUM_IGMP			0x0040
+#define	AXE_RXCSUM_ICMP6		0x0080
+#define	AXE_RXCSUM_TCPV6		0x0100
+#define	AXE_RXCSUM_UDPV6		0x0200
+#define	AXE_RXCSUM_ICMPV6		0x0400
+#define	AXE_RXCSUM_IGMPV6		0x0800
+#define	AXE_RXCSUM_ICMP6V6		0x1000
+#define	AXE_RXCSUM_FOPC			0x8000
+
+#define	AXE_RXCSUM_64TE			0x0100
+#define	AXE_RXCSUM_PPPOE		0x0200
+#define	AXE_RXCSUM_RPCE			0x8000
+
+#define	AXE_TXCSUM_IP			0x0001
+#define	AXE_TXCSUM_TCP			0x0002
+#define	AXE_TXCSUM_UDP			0x0004
+#define	AXE_TXCSUM_ICMP			0x0008
+#define	AXE_TXCSUM_IGMP			0x0010
+#define	AXE_TXCSUM_ICMP6		0x0020
+#define	AXE_TXCSUM_TCPV6		0x0100
+#define	AXE_TXCSUM_UDPV6		0x0200
+#define	AXE_TXCSUM_ICMPV6		0x0400
+#define	AXE_TXCSUM_IGMPV6		0x0800
+#define	AXE_TXCSUM_ICMP6V6		0x1000
+
+#define	AXE_TXCSUM_64TE			0x0001
+#define	AXE_TXCSUM_PPPOE		0x0002
+
 #define	AXE_BULK_BUF_SIZE	16384	/* bytes */
 
 #define	AXE_CTL_READ		0x01
@@ -227,9 +265,62 @@ struct ax88772b_mfb {
 
 struct axe_sframe_hdr {
 	uint16_t len;
+#define	AXE_HDR_LEN_MASK	0xFFFF
 	uint16_t ilen;
 } __packed;
 
+#define	AXE_TX_CSUM_PSEUDO_HDR	0x4000
+#define	AXE_TX_CSUM_DIS		0x8000
+
+/*
+ * When RX checksum offloading is enabled, AX88772B uses new RX header
+ * format and it's not compatible with previous RX header format.  In
+ * addition, IP header align option should be enabled to get correct
+ * frame size including RX header.  Total transferred size including
+ * the RX header is multiple of 4 and controller will pad necessary
+ * bytes if the length is not multiple of 4.
+ * This driver does not enable partial checksum feature which will
+ * compute 16bit checksum from 14th byte to the end of the frame.  If
+ * this feature is enabled, computed checksum value is embedded into
+ * RX header which in turn means it uses different RX header format.
+ */
+struct axe_csum_hdr {
+	uint16_t len;
+#define	AXE_CSUM_HDR_LEN_MASK		0x07FF
+#define	AXE_CSUM_HDR_CRC_ERR		0x1000
+#define	AXE_CSUM_HDR_MII_ERR		0x2000
+#define	AXE_CSUM_HDR_RUNT		0x4000
+#define	AXE_CSUM_HDR_BMCAST		0x8000
+	uint16_t ilen;
+	uint16_t cstatus;
+#define	AXE_CSUM_HDR_VLAN_MASK		0x0007
+#define	AXE_CSUM_HDR_VLAN_STRIP		0x0008
+#define	AXE_CSUM_HDR_VLAN_PRI_MASK	0x0070
+#define	AXE_CSUM_HDR_L4_CSUM_ERR	0x0100
+#define	AXE_CSUM_HDR_L3_CSUM_ERR	0x0200
+#define	AXE_CSUM_HDR_L4_TYPE_UDP	0x0400
+#define	AXE_CSUM_HDR_L4_TYPE_ICMP	0x0800
+#define	AXE_CSUM_HDR_L4_TYPE_IGMP	0x0C00
+#define	AXE_CSUM_HDR_L4_TYPE_TCP	0x1000
+#define	AXE_CSUM_HDR_L4_TYPE_TCPV6	0x1400
+#define	AXE_CSUM_HDR_L4_TYPE_MASK	0x1C00
+#define	AXE_CSUM_HDR_L3_TYPE_IPV4	0x2000
+#define	AXE_CSUM_HDR_L3_TYPE_IPV6	0x4000
+
+#ifdef AXE_APPEND_PARTIAL_CSUM
+	/*
+	 * These members present only when partial checksum
+	 * offloading is enabled.  The checksum value is simple
+	 * 16bit sum of received frame starting at offset 14 of
+	 * the frame to the end of the frame excluding FCS bytes.
+	 */
+	uint16_t csum_value;
+	uint16_t dummy;
+#endif
+} __packed;
+
+#define	AXE_CSUM_RXBYTES(x)	((x) & AXE_CSUM_HDR_LEN_MASK)
+
 #define	GET_MII(sc)		uether_getmii(&(sc)->sc_ue)
 
 /* The interrupt endpoint is currently unused by the ASIX part. */
@@ -247,6 +338,8 @@ struct axe_softc {
 
 	int			sc_flags;
 #define	AXE_FLAG_LINK		0x0001
+#define	AXE_FLAG_STD_FRAME	0x0010
+#define	AXE_FLAG_CSUM_FRAME	0x0020
 #define	AXE_FLAG_772		0x1000	/* AX88772 */
 #define	AXE_FLAG_772A		0x2000	/* AX88772A */
 #define	AXE_FLAG_772B		0x4000	/* AX88772B */
@@ -255,6 +348,7 @@ struct axe_softc {
 	uint8_t			sc_ipgs[3];
 	uint8_t			sc_phyaddrs[2];
 	uint16_t		sc_pwrcfg;
+	uint16_t		sc_lenmask;
 	int			sc_tx_bufsz;
 };
 
diff --git a/sys/dev/usb/net/if_mos.c b/sys/dev/usb/net/if_mos.c
index 0cbea915afbd..5fbbb393ddec 100644
--- a/sys/dev/usb/net/if_mos.c
+++ b/sys/dev/usb/net/if_mos.c
@@ -551,10 +551,10 @@ mos_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	MOS_LOCK(sc);
 	mii_pollstat(mii);
-	MOS_UNLOCK(sc);
 
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	MOS_UNLOCK(sc);
 }
 
 static void
diff --git a/sys/dev/usb/net/if_rue.c b/sys/dev/usb/net/if_rue.c
index 3e3a151da207..c874c1db5424 100644
--- a/sys/dev/usb/net/if_rue.c
+++ b/sys/dev/usb/net/if_rue.c
@@ -889,9 +889,9 @@ rue_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	RUE_LOCK(sc);
 	mii_pollstat(mii);
-	RUE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	RUE_UNLOCK(sc);
 }
 
 static void
diff --git a/sys/dev/usb/net/if_udav.c b/sys/dev/usb/net/if_udav.c
index ea3512773c22..e9a0504dfb50 100644
--- a/sys/dev/usb/net/if_udav.c
+++ b/sys/dev/usb/net/if_udav.c
@@ -751,9 +751,9 @@ udav_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	UDAV_LOCK(sc);
 	mii_pollstat(mii);
-	UDAV_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	UDAV_UNLOCK(sc);
 }
 
 static void
diff --git a/sys/dev/mii/ruephy.c b/sys/dev/usb/net/ruephy.c
similarity index 99%
rename from sys/dev/mii/ruephy.c
rename to sys/dev/usb/net/ruephy.c
index a0d54417d8c8..64150a58bf65 100644
--- a/sys/dev/mii/ruephy.c
+++ b/sys/dev/usb/net/ruephy.c
@@ -48,8 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mii/miivar.h>
 #include "miidevs.h"
 
-#include <machine/bus.h>
-#include <dev/mii/ruephyreg.h>
+#include <dev/usb/net/ruephyreg.h>
 
 #include "miibus_if.h"
 
diff --git a/sys/dev/mii/ruephyreg.h b/sys/dev/usb/net/ruephyreg.h
similarity index 88%
rename from sys/dev/mii/ruephyreg.h
rename to sys/dev/usb/net/ruephyreg.h
index 5f3911bebd5d..01d3cc172246 100644
--- a/sys/dev/mii/ruephyreg.h
+++ b/sys/dev/usb/net/ruephyreg.h
@@ -27,12 +27,12 @@
  */
 
 #ifndef _RUEPHYREG_H_
-#define _RUEPHYREG_H_
+#define	_RUEPHYREG_H_
 
-#define RUEPHY_MII_MSR		0x0137	/* B, R/W */
-#define RUEPHY_MSR_RXFCE	0x40
-#define RUEPHY_MSR_DUPLEX	0x10
-#define RUEPHY_MSR_SPEED100	0x08
-#define RUEPHY_MSR_LINK		0x04
+#define	RUEPHY_MII_MSR		0x0137	/* B, R/W */
+#define	RUEPHY_MSR_RXFCE	0x40
+#define	RUEPHY_MSR_DUPLEX	0x10
+#define	RUEPHY_MSR_SPEED100	0x08
+#define	RUEPHY_MSR_LINK		0x04
 
 #endif	/* _RUEPHYREG_H_ */
diff --git a/sys/dev/usb/net/usb_ethernet.c b/sys/dev/usb/net/usb_ethernet.c
index 7fbd9efa1c1b..bc73d703c743 100644
--- a/sys/dev/usb/net/usb_ethernet.c
+++ b/sys/dev/usb/net/usb_ethernet.c
@@ -24,24 +24,32 @@
  * SUCH DAMAGE.
  */
 
-#include <sys/stdint.h>
-#include <sys/stddef.h>
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include <sys/param.h>
-#include <sys/queue.h>
-#include <sys/types.h>
 #include <sys/systm.h>
-#include <sys/kernel.h>
 #include <sys/bus.h>
-#include <sys/module.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
 #include <sys/condvar.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <sys/sx.h>
-#include <sys/unistd.h>
-#include <sys/callout.h>
-#include <sys/malloc.h>
-#include <sys/priv.h>
+
+#include <net/if.h>
+#include <net/ethernet.h>
+#include <net/if_types.h>
+#include <net/if_media.h>
+#include <net/if_vlan_var.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/miivar.h>
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
@@ -197,42 +205,53 @@ ue_attach_post_task(struct usb_proc_msg *_task)
 	usb_callout_init_mtx(&ue->ue_watchdog, ue->ue_mtx, 0);
 	sysctl_ctx_init(&ue->ue_sysctl_ctx);
 
+	error = 0;
 	ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
 		device_printf(ue->ue_dev, "could not allocate ifnet\n");
-		goto error;
+		goto fail;
 	}
 
 	ifp->if_softc = ue;
 	if_initname(ifp, "ue", ue->ue_unit);
-	ifp->if_mtu = ETHERMTU;
-	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
-	if (ue->ue_methods->ue_ioctl != NULL)
-		ifp->if_ioctl = ue->ue_methods->ue_ioctl;
-	else
-		ifp->if_ioctl = uether_ioctl;
-	ifp->if_start = ue_start;
-	ifp->if_init = ue_init;
-	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
-	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
-	IFQ_SET_READY(&ifp->if_snd);
-	ue->ue_ifp = ifp;
+	if (ue->ue_methods->ue_attach_post_sub != NULL) {
+		ue->ue_ifp = ifp;
+		error = ue->ue_methods->ue_attach_post_sub(ue);
+	} else {
+		ifp->if_mtu = ETHERMTU;
+		ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+		if (ue->ue_methods->ue_ioctl != NULL)
+			ifp->if_ioctl = ue->ue_methods->ue_ioctl;
+		else
+			ifp->if_ioctl = uether_ioctl;
+		ifp->if_start = ue_start;
+		ifp->if_init = ue_init;
+		IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
+		ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
+		IFQ_SET_READY(&ifp->if_snd);
+		ue->ue_ifp = ifp;
 
-	if (ue->ue_methods->ue_mii_upd != NULL && 
-	    ue->ue_methods->ue_mii_sts != NULL) {
-		mtx_lock(&Giant);	/* device_xxx() depends on this */
-		error = mii_attach(ue->ue_dev, &ue->ue_miibus, ifp,
-		    ue_ifmedia_upd, ue->ue_methods->ue_mii_sts,
-		    BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0);
-		mtx_unlock(&Giant);
-		if (error) {
-			device_printf(ue->ue_dev, "attaching PHYs failed\n");
-			goto error;
+		if (ue->ue_methods->ue_mii_upd != NULL &&
+		    ue->ue_methods->ue_mii_sts != NULL) {
+			/* device_xxx() depends on this */
+			mtx_lock(&Giant);
+			error = mii_attach(ue->ue_dev, &ue->ue_miibus, ifp,
+			    ue_ifmedia_upd, ue->ue_methods->ue_mii_sts,
+			    BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0);
+			mtx_unlock(&Giant);
 		}
 	}
 
+	if (error) {
+		device_printf(ue->ue_dev, "attaching PHYs failed\n");
+		goto fail;
+	}
+
 	if_printf(ifp, "<USB Ethernet> on %s\n", device_get_nameunit(ue->ue_dev));
 	ether_ifattach(ifp, ue->ue_eaddr);
+	/* Tell upper layer we support VLAN oversized frames. */
+	if (ifp->if_capabilities & IFCAP_VLAN_MTU)
+		ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 
 	snprintf(num, sizeof(num), "%u", ue->ue_unit);
 	ue->ue_sysctl_oid = SYSCTL_ADD_NODE(&ue->ue_sysctl_ctx,
@@ -246,7 +265,7 @@ ue_attach_post_task(struct usb_proc_msg *_task)
 	UE_LOCK(ue);
 	return;
 
-error:
+fail:
 	free_unr(ueunit, ue->ue_unit);
 	if (ue->ue_ifp != NULL) {
 		if_free(ue->ue_ifp);
@@ -307,6 +326,13 @@ uether_is_gone(struct usb_ether *ue)
 	return (usb_proc_is_gone(&ue->ue_tq));
 }
 
+void
+uether_init(void *arg)
+{
+
+	ue_init(arg);
+}
+
 static void
 ue_init(void *arg)
 {
@@ -352,6 +378,13 @@ ue_stop_task(struct usb_proc_msg *_task)
 	ue->ue_methods->ue_stop(ue);
 }
 
+void
+uether_start(struct ifnet *ifp)
+{
+
+	ue_start(ifp);
+}
+
 static void
 ue_start(struct ifnet *ifp)
 {
@@ -385,6 +418,13 @@ ue_setmulti_task(struct usb_proc_msg *_task)
 	ue->ue_methods->ue_setmulti(ue);
 }
 
+int
+uether_ifmedia_upd(struct ifnet *ifp)
+{
+
+	return (ue_ifmedia_upd(ifp));
+}
+
 static int
 ue_ifmedia_upd(struct ifnet *ifp)
 {
diff --git a/sys/dev/usb/net/usb_ethernet.h b/sys/dev/usb/net/usb_ethernet.h
index 51a81076b374..af7cad437f97 100644
--- a/sys/dev/usb/net/usb_ethernet.h
+++ b/sys/dev/usb/net/usb_ethernet.h
@@ -22,6 +22,8 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
+ * $FreeBSD$
  */
 
 #ifndef _USB_ETHERNET_H_
@@ -66,7 +68,7 @@ struct usb_ether_methods {
 	void			(*ue_mii_sts)(struct ifnet *,
 				    struct ifmediareq *);
 	int			(*ue_ioctl)(struct ifnet *, u_long, caddr_t);
-
+	int			(*ue_attach_post_sub)(struct usb_ether *);
 };
 
 struct usb_ether_cfg_task {
@@ -110,6 +112,8 @@ struct mii_data *uether_getmii(struct usb_ether *);
 void		*uether_getsc(struct usb_ether *);
 int		uether_ifattach(struct usb_ether *);
 void		uether_ifdetach(struct usb_ether *);
+int		uether_ifmedia_upd(struct ifnet *);
+void		uether_init(void *);
 int		uether_ioctl(struct ifnet *, u_long, caddr_t);
 struct mbuf	*uether_newbuf(void);
 int		uether_rxmbuf(struct usb_ether *, struct mbuf *, 
@@ -119,4 +123,5 @@ int		uether_rxbuf(struct usb_ether *,
 		    unsigned int, unsigned int);
 void		uether_rxflush(struct usb_ether *);
 uint8_t		uether_is_gone(struct usb_ether *);
+void		uether_start(struct ifnet *);
 #endif					/* _USB_ETHERNET_H_ */
diff --git a/sys/dev/usb/quirk/usb_quirk.c b/sys/dev/usb/quirk/usb_quirk.c
index d3093f7ec1fd..128fe2a6193d 100644
--- a/sys/dev/usb/quirk/usb_quirk.c
+++ b/sys/dev/usb/quirk/usb_quirk.c
@@ -457,6 +457,14 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = {
 	USB_QUIRK(ROLAND, SD80, 0x0000, 0xffff, UQ_AU_VENDOR_CLASS),
 	USB_QUIRK(ROLAND, UA700, 0x0000, 0xffff, UQ_AU_VENDOR_CLASS),
 	USB_QUIRK(MEDELI, DD305, 0x0000, 0xffff, UQ_SINGLE_CMD_MIDI, UQ_MATCH_VENDOR_ONLY),
+
+	/*
+	 * Quirks for manufacturers which USB devices does not respond
+	 * after issuing non-supported commands:
+	 */
+	USB_QUIRK(FEIYA, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MATCH_VENDOR_ONLY),
+	USB_QUIRK(REALTEK, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MATCH_VENDOR_ONLY),
+	USB_QUIRK(INITIO, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MATCH_VENDOR_ONLY),
 };
 #undef USB_QUIRK_VP
 #undef USB_QUIRK
diff --git a/sys/dev/usb/serial/usb_serial.c b/sys/dev/usb/serial/usb_serial.c
index b0ff3bf17bd8..61885eaa147a 100644
--- a/sys/dev/usb/serial/usb_serial.c
+++ b/sys/dev/usb/serial/usb_serial.c
@@ -289,6 +289,9 @@ ucom_detach(struct ucom_super_softc *ssc, struct ucom_softc *sc)
 {
 	uint32_t subunit;
 
+	if (ssc->sc_subunits == 0)
+		return;		/* not initialized */
+
 	usb_proc_drain(&ssc->sc_tq);
 
 	for (subunit = 0; subunit < ssc->sc_subunits; subunit++) {
diff --git a/sys/dev/usb/storage/umass.c b/sys/dev/usb/storage/umass.c
index 41e98d50c856..b932cfaa44f8 100644
--- a/sys/dev/usb/storage/umass.c
+++ b/sys/dev/usb/storage/umass.c
@@ -2344,14 +2344,14 @@ umass_cam_action(struct cam_sim *sim, union ccb *ccb)
 					 */
 					if ((sc->sc_quirks & (NO_INQUIRY_EVPD | NO_INQUIRY)) &&
 					    (sc->sc_transfer.cmd_data[1] & SI_EVPD)) {
-						struct scsi_sense_data *sense;
 
-						sense = &ccb->csio.sense_data;
-						bzero(sense, sizeof(*sense));
-						sense->error_code = SSD_CURRENT_ERROR;
-						sense->flags = SSD_KEY_ILLEGAL_REQUEST;
-						sense->add_sense_code = 0x24;
-						sense->extra_len = 10;
+						scsi_set_sense_data(&ccb->csio.sense_data,
+							/*sense_format*/ SSD_TYPE_NONE,
+							/*current_error*/ 1,
+							/*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
+							/*asc*/ 0x24,
+							/*ascq*/ 0x00,
+							/*extra args*/ SSD_ELEM_NONE);
 						ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
 						ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR |
 						    CAM_AUTOSNS_VALID;
@@ -2631,12 +2631,17 @@ umass_cam_sense_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
     uint8_t status)
 {
 	uint8_t *cmd;
-	uint8_t key;
 
 	switch (status) {
 	case STATUS_CMD_OK:
 	case STATUS_CMD_UNKNOWN:
-	case STATUS_CMD_FAILED:
+	case STATUS_CMD_FAILED: {
+		int key, sense_len;
+
+		ccb->csio.sense_resid = residue;
+		sense_len = ccb->csio.sense_len - ccb->csio.sense_resid;
+		key = scsi_get_sense_key(&ccb->csio.sense_data, sense_len,
+					 /*show_errors*/ 1);
 
 		if (ccb->csio.ccb_h.flags & CAM_CDB_POINTER) {
 			cmd = (uint8_t *)(ccb->csio.cdb_io.cdb_ptr);
@@ -2644,8 +2649,6 @@ umass_cam_sense_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 			cmd = (uint8_t *)(ccb->csio.cdb_io.cdb_bytes);
 		}
 
-		key = (ccb->csio.sense_data.flags & SSD_KEY);
-
 		/*
 		 * Getting sense data always succeeds (apart from wire
 		 * failures):
@@ -2704,7 +2707,7 @@ umass_cam_sense_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 		}
 		xpt_done(ccb);
 		break;
-
+	}
 	default:
 		DPRINTF(sc, UDMASS_SCSI, "Autosense failed, "
 		    "status %d\n", status);
diff --git a/sys/dev/usb/usb.h b/sys/dev/usb/usb.h
index 89bc2c44a2a8..6a9b126633e8 100644
--- a/sys/dev/usb/usb.h
+++ b/sys/dev/usb/usb.h
@@ -686,6 +686,7 @@ struct usb_port_status {
 #define	UPS_PORT_LS_HOT_RST	0x09
 #define	UPS_PORT_LS_COMP_MODE	0x0A
 #define	UPS_PORT_LS_LOOPBACK	0x0B
+#define	UPS_PORT_LS_RESUME	0x0F
 #define	UPS_PORT_POWER			0x0100
 #define	UPS_LOW_SPEED			0x0200
 #define	UPS_HIGH_SPEED			0x0400
diff --git a/sys/dev/usb/usb_dev.c b/sys/dev/usb/usb_dev.c
index 990bda22e4c6..e5d98fe9ea1d 100644
--- a/sys/dev/usb/usb_dev.c
+++ b/sys/dev/usb/usb_dev.c
@@ -768,7 +768,7 @@ usb_fifo_close(struct usb_fifo *f, int fflags)
 	/* check if a thread wants SIGIO */
 	if (f->async_p != NULL) {
 		PROC_LOCK(f->async_p);
-		psignal(f->async_p, SIGIO);
+		kern_psignal(f->async_p, SIGIO);
 		PROC_UNLOCK(f->async_p);
 		f->async_p = NULL;
 	}
@@ -1582,7 +1582,7 @@ usb_fifo_wakeup(struct usb_fifo *f)
 	}
 	if (f->async_p != NULL) {
 		PROC_LOCK(f->async_p);
-		psignal(f->async_p, SIGIO);
+		kern_psignal(f->async_p, SIGIO);
 		PROC_UNLOCK(f->async_p);
 	}
 }
diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c
index 12898ec2642c..5795d56bf2d9 100644
--- a/sys/dev/usb/usb_hub.c
+++ b/sys/dev/usb/usb_hub.c
@@ -611,6 +611,7 @@ uhub_suspend_resume_port(struct uhub_softc *sc, uint8_t portno)
 		switch (UPS_PORT_LINK_STATE_GET(sc->sc_st.port_status)) {
 		case UPS_PORT_LS_U0:
 		case UPS_PORT_LS_U1:
+		case UPS_PORT_LS_RESUME:
 			is_suspend = 0;
 			break;
 		default:
diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c
index d4c2408db7c9..cd7b2e2925eb 100644
--- a/sys/dev/usb/usb_transfer.c
+++ b/sys/dev/usb/usb_transfer.c
@@ -2417,8 +2417,9 @@ usbd_transfer_start_cb(void *arg)
 #if USB_HAVE_PF
 	usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
 #endif
-	/* start the transfer */
-	(ep->methods->start) (xfer);
+	/* start USB transfer, if no error */
+	if (xfer->error == 0)
+		(ep->methods->start) (xfer);
 
 	xfer->flags_int.can_cancel_immed = 1;
 
@@ -2597,8 +2598,9 @@ usbd_pipe_start(struct usb_xfer_queue *pq)
 #if USB_HAVE_PF
 	usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
 #endif
-	/* start USB transfer */
-	(ep->methods->start) (xfer);
+	/* start USB transfer, if no error */
+	if (xfer->error == 0)
+		(ep->methods->start) (xfer);
 
 	xfer->flags_int.can_cancel_immed = 1;
 
diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs
index 6a729a2ef61b..5a4d3f41d173 100644
--- a/sys/dev/usb/usbdevs
+++ b/sys/dev/usb/usbdevs
@@ -606,6 +606,7 @@ vendor BALTECH		0x13ad	Baltech
 vendor CISCOLINKSYS	0x13b1	Cisco-Linksys
 vendor SHARK		0x13d2	Shark
 vendor AZUREWAVE	0x13d3	AsureWave
+vendor INITIO		0x13fd	Initio Corporation
 vendor EMTEC		0x13fe	Emtec
 vendor NOVATEL		0x1410	Novatel Wireless
 vendor MERLIN		0x1416	Merlin
@@ -1144,6 +1145,7 @@ product BELKIN RT2870_1		0x8053	RT2870
 product BELKIN RT2870_2		0x805c	RT2870
 product BELKIN F5D8053V3	0x815c	F5D8053 v3
 product BELKIN F5D8055		0x825a	F5D8055
+product BELKIN F5D8055V2	0x825b	F5D8055 v2
 product BELKIN F5D9050V3	0x905b	F5D9050 ver 3 Wireless Adapter
 product BELKIN2 F5U002		0x0002	F5U002 Parallel printer
 product BELKIN F6D4050V1	0x935a	F6D4050 v1
@@ -1227,8 +1229,9 @@ product CISCOLINKSYS HU200TS	0x001a	HU200TS Wireless Adapter
 product CISCOLINKSYS WUSB54GC	0x0020	WUSB54GC
 product CISCOLINKSYS WUSB54GR	0x0023	WUSB54GR
 product CISCOLINKSYS WUSBF54G	0x0024	WUSBF54G
-product	CISCOLINKSYS2 RT3070	0x4001	RT3070
-product	CISCOLINKSYS3 RT3070	0x0101	RT3070
+product CISCOLINKSYS AE1000	0x002f	AE1000
+product CISCOLINKSYS2 RT3070	0x4001	RT3070
+product CISCOLINKSYS3 RT3070	0x0101	RT3070
 
 /* Clipsal products */
 product CLIPSAL 5500PCU		0x0303	5500PCU C-Bus
@@ -1569,6 +1572,7 @@ product EXTENDED XTNDACCESS	0x0100	XTNDAccess IrDA
 product FALCOM TWIST		0x0001	USB GSM/GPRS Modem
 
 /* FEIYA products */
+product FEIYA DUMMY		0x0000	Dummy product
 product FEIYA 5IN1		0x1132	5-in-1 Card Reader
 product FEIYA AC110		0x6300	AC-110 Card Reader
 
@@ -1886,6 +1890,10 @@ product IBM USBCDROMDRIVE	0x4427	USB CD-ROM Drive
 /* Imagination Technologies products */
 product IMAGINATION DBX1	0x2107	DBX1 DSP core
 
+/* Initio Corporation products */
+product INITIO DUMMY		0x0000	Dummy product
+product INITIO INIC_1610P	0x1e40	USB to SATA Bridge
+
 /* Inside Out Networks products */
 product INSIDEOUT EDGEPORT4	0x0001	EdgePort/4 serial ports
 
@@ -2780,6 +2788,7 @@ product RATOC REXUSB60F		0xb020	USB serial adapter REX-USB60F
 
 /* ReakTek products */
 /* Green House and CompUSA OEM this part */
+product REALTEK DUMMY		0x0000	Dummy product
 product REALTEK USB20CRW	0x0158	USB20CRW Card Reader
 product REALTEK USBKR100	0x8150	USBKR100 USB Ethernet
 product REALTEK RTL8187		0x8187	RTL8187 Wireless Adapter
@@ -2833,6 +2842,7 @@ product SAGEM XG760A		0x004a	XG-760A
 product SAGEM XG76NA		0x0062	XG-76NA
 
 /* Samsung products */
+product SAMSUNG WIS09ABGN	0x2018	WIS09ABGN Wireless LAN adapter
 product SAMSUNG ML6060		0x3008	ML-6060 laser printer
 product SAMSUNG YP_U2		0x5050	YP-U2 MP3 Player
 product SAMSUNG YP_U4		0x5092	YP-U4 MP3 Player
diff --git a/sys/dev/usb/wlan/if_rum.c b/sys/dev/usb/wlan/if_rum.c
index 5a69792aeeaa..c51d48546235 100644
--- a/sys/dev/usb/wlan/if_rum.c
+++ b/sys/dev/usb/wlan/if_rum.c
@@ -1407,20 +1407,27 @@ rum_write_multi(struct rum_softc *sc, uint16_t reg, void *buf, size_t len)
 {
 	struct usb_device_request req;
 	usb_error_t error;
+	int offset;
 
 	req.bmRequestType = UT_WRITE_VENDOR_DEVICE;
 	req.bRequest = RT2573_WRITE_MULTI_MAC;
 	USETW(req.wValue, 0);
-	USETW(req.wIndex, reg);
-	USETW(req.wLength, len);
 
-	error = rum_do_request(sc, &req, buf);
-	if (error != 0) {
-		device_printf(sc->sc_dev,
-		    "could not multi write MAC register: %s\n",
-		    usbd_errstr(error));
+	/* write at most 64 bytes at a time */
+	for (offset = 0; offset < len; offset += 64) {
+		USETW(req.wIndex, reg + offset);
+		USETW(req.wLength, MIN(len - offset, 64));
+
+		error = rum_do_request(sc, &req, (char *)buf + offset);
+		if (error != 0) {
+			device_printf(sc->sc_dev,
+			    "could not multi write MAC register: %s\n",
+			    usbd_errstr(error));
+			return (error);
+		}
 	}
-	return (error);
+
+	return (USB_ERR_NORMAL_COMPLETION);
 }
 
 static void
diff --git a/sys/dev/usb/wlan/if_run.c b/sys/dev/usb/wlan/if_run.c
index 47823b9d8ec5..a0ac26add0aa 100644
--- a/sys/dev/usb/wlan/if_run.c
+++ b/sys/dev/usb/wlan/if_run.c
@@ -144,9 +144,11 @@ static const STRUCT_USB_HOST_ID run_devs[] = {
     RUN_DEV(AZUREWAVE,		RT3070_3),
     RUN_DEV(BELKIN,		F5D8053V3),
     RUN_DEV(BELKIN,		F5D8055),
+    RUN_DEV(BELKIN,		F5D8055V2),
     RUN_DEV(BELKIN,		F6D4050V1),
     RUN_DEV(BELKIN,		RT2870_1),
     RUN_DEV(BELKIN,		RT2870_2),
+    RUN_DEV(CISCOLINKSYS,	AE1000),
     RUN_DEV(CISCOLINKSYS2,	RT3070),
     RUN_DEV(CISCOLINKSYS3,	RT3070),
     RUN_DEV(CONCEPTRONIC2,	RT2870_1),
@@ -247,6 +249,7 @@ static const STRUCT_USB_HOST_ID run_devs[] = {
     RUN_DEV(RALINK,		RT3370),
     RUN_DEV(RALINK,		RT3572),
     RUN_DEV(RALINK,		RT8070),
+    RUN_DEV(SAMSUNG,		WIS09ABGN),
     RUN_DEV(SAMSUNG2,		RT2870_1),
     RUN_DEV(SENAO,		RT2870_1),
     RUN_DEV(SENAO,		RT2870_2),
diff --git a/sys/dev/vge/if_vge.c b/sys/dev/vge/if_vge.c
index 952970219603..f4ce30ab2a23 100644
--- a/sys/dev/vge/if_vge.c
+++ b/sys/dev/vge/if_vge.c
@@ -2173,9 +2173,9 @@ vge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 		return;
 	}
 	mii_pollstat(mii);
-	VGE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	VGE_UNLOCK(sc);
 }
 
 static void
diff --git a/sys/dev/vkbd/vkbd.c b/sys/dev/vkbd/vkbd.c
index 42fca400c1d6..81418f2d4334 100644
--- a/sys/dev/vkbd/vkbd.c
+++ b/sys/dev/vkbd/vkbd.c
@@ -158,7 +158,7 @@ static int		vkbd_data_read(vkbd_state_t *, int);
 
 static struct cdevsw	vkbd_dev_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_PSEUDO | D_NEEDGIANT | D_NEEDMINOR,
+	.d_flags =	D_NEEDGIANT | D_NEEDMINOR,
 	.d_open =	vkbd_dev_open,
 	.d_close =	vkbd_dev_close,
 	.d_read =	vkbd_dev_read,
diff --git a/sys/dev/vr/if_vr.c b/sys/dev/vr/if_vr.c
index 1c2aac2a8b1f..0bcef0ed9d48 100644
--- a/sys/dev/vr/if_vr.c
+++ b/sys/dev/vr/if_vr.c
@@ -114,12 +114,12 @@ MODULE_DEPEND(vr, miibus, 1, 1, 1);
 #define VR_Q_CSUM		(1<<1)
 #define VR_Q_CAM		(1<<2)
 
-static struct vr_type {
+static const struct vr_type {
 	u_int16_t		vr_vid;
 	u_int16_t		vr_did;
 	int			vr_quirks;
-	char			*vr_name;
-} vr_devs[] = {
+	const char		*vr_name;
+} const vr_devs[] = {
 	{ VIA_VENDORID, VIA_DEVICEID_RHINE,
 	    VR_Q_NEEDALIGN,
 	    "VIA VT3043 Rhine I 10/100BaseTX" },
@@ -195,11 +195,11 @@ static void vr_setwol(struct vr_softc *);
 static void vr_clrwol(struct vr_softc *);
 static int vr_sysctl_stats(SYSCTL_HANDLER_ARGS);
 
-static struct vr_tx_threshold_table {
+static const struct vr_tx_threshold_table {
 	int tx_cfg;
 	int bcr_cfg;
 	int value;
-} vr_tx_threshold_tables[] = {
+} const vr_tx_threshold_tables[] = {
 	{ VR_TXTHRESH_64BYTES, VR_BCR1_TXTHRESH64BYTES,	64 },
 	{ VR_TXTHRESH_128BYTES, VR_BCR1_TXTHRESH128BYTES, 128 },
 	{ VR_TXTHRESH_256BYTES, VR_BCR1_TXTHRESH256BYTES, 256 },
@@ -557,10 +557,10 @@ vr_reset(const struct vr_softc *sc)
  * Probe for a VIA Rhine chip. Check the PCI vendor and device
  * IDs against our list and return a match or NULL
  */
-static struct vr_type *
+static const struct vr_type *
 vr_match(device_t dev)
 {
-	struct vr_type	*t = vr_devs;
+	const struct vr_type	*t = vr_devs;
 
 	for (t = vr_devs; t->vr_name != NULL; t++)
 		if ((pci_get_vendor(dev) == t->vr_vid) &&
@@ -576,7 +576,7 @@ vr_match(device_t dev)
 static int
 vr_probe(device_t dev)
 {
-	struct vr_type	*t;
+	const struct vr_type	*t;
 
 	t = vr_match(dev);
 	if (t != NULL) {
@@ -595,7 +595,7 @@ vr_attach(device_t dev)
 {
 	struct vr_softc		*sc;
 	struct ifnet		*ifp;
-	struct vr_type		*t;
+	const struct vr_type	*t;
 	uint8_t			eaddr[ETHER_ADDR_LEN];
 	int			error, rid;
 	int			i, phy, pmc;
@@ -2155,9 +2155,9 @@ vr_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 		return;
 	}
 	mii_pollstat(mii);
-	VR_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	VR_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/vte/if_vte.c b/sys/dev/vte/if_vte.c
index 1b38dec1076b..5e491543ce3f 100644
--- a/sys/dev/vte/if_vte.c
+++ b/sys/dev/vte/if_vte.c
@@ -284,9 +284,9 @@ vte_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc->vte_miibus);
 
 	mii_pollstat(mii);
-	VTE_UNLOCK(sc);
 	ifmr->ifm_status = mii->mii_media_status;
 	ifmr->ifm_active = mii->mii_media_active;
+	VTE_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/dev/vxge/include/vxgehal-config.h b/sys/dev/vxge/include/vxgehal-config.h
index f2d8ed363f77..7f5c6dbeec31 100644
--- a/sys/dev/vxge/include/vxgehal-config.h
+++ b/sys/dev/vxge/include/vxgehal-config.h
@@ -1665,7 +1665,7 @@ typedef struct vxge_hal_mrpcim_config_t {
  *		the interrupt is triggered.
  * @rtimer_event_sf: Restriction Timer Event Scale Factor. A scale factor that
  *		is to be applied to the current event count before it is added
- *		to the restriction timer value when the the restriction timer
+ *		to the restriction timer value when the restriction timer
  *		is started.
  *		The scale factor is applied as a right or left shift to multiply
  *		or divide by the event count. The programmable values are as
diff --git a/sys/dev/vxge/include/vxgehal-ll.h b/sys/dev/vxge/include/vxgehal-ll.h
index 9fc416fa9c31..85a95d44a74f 100644
--- a/sys/dev/vxge/include/vxgehal-ll.h
+++ b/sys/dev/vxge/include/vxgehal-ll.h
@@ -1590,7 +1590,7 @@ vxge_hal_ring_rxd_post(
  * vxge_hal_ring_is_next_rxd_completed - Check if the next rxd is completed
  * @vpath_handle: Virtual Path handle.
  *
- * Checks if the the _next_	completed descriptor is	in host	memory
+ * Checks if the _next_	completed descriptor is	in host	memory
  *
  * Returns: VXGE_HAL_OK - success.
  * VXGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed	descriptors
diff --git a/sys/dev/vxge/vxgehal/vxgehal-ring.c b/sys/dev/vxge/vxgehal/vxgehal-ring.c
index bcfa078e778a..1cefb3d000e9 100644
--- a/sys/dev/vxge/vxgehal/vxgehal-ring.c
+++ b/sys/dev/vxge/vxgehal/vxgehal-ring.c
@@ -1339,7 +1339,7 @@ vxge_hal_ring_rxd_post_post_db(
  * vxge_hal_ring_is_next_rxd_completed - Check if the next rxd is completed
  * @vpath_handle: Virtual Path handle.
  *
- * Checks if the the _next_	completed descriptor is	in host	memory
+ * Checks if the _next_	completed descriptor is	in host	memory
  *
  * Returns: VXGE_HAL_OK - success.
  * VXGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed	descriptors
diff --git a/sys/dev/xen/blkfront/blkfront.c b/sys/dev/xen/blkfront/blkfront.c
index 286831351a6d..fb530f3f13f3 100644
--- a/sys/dev/xen/blkfront/blkfront.c
+++ b/sys/dev/xen/blkfront/blkfront.c
@@ -77,11 +77,8 @@ static int blkfront_detach(device_t);
 static int setup_blkring(struct xb_softc *);
 static void blkif_int(void *);
 static void blkfront_initialize(struct xb_softc *);
-#if 0
-static void blkif_recover(struct xb_softc *);
-#endif
 static int blkif_completion(struct xb_command *);
-static void blkif_free(struct xb_softc *, int);
+static void blkif_free(struct xb_softc *);
 static void blkif_queue_cb(void *, bus_dma_segment_t *, int, int);
 
 MALLOC_DEFINE(M_XENBLOCKFRONT, "xbd", "Xen Block Front driver data");
@@ -452,9 +449,6 @@ blkfront_attach(device_t dev)
 	sc->vdevice = vdevice;
 	sc->connected = BLKIF_STATE_DISCONNECTED;
 
-	/* Front end dir is a number, which is used as the id. */
-	sc->handle = strtoul(strrchr(xenbus_get_node(dev),'/')+1, NULL, 0);
-
 	/* Wait for backend device to publish its protocol capabilities. */
 	xenbus_set_state(dev, XenbusStateInitialising);
 
@@ -465,29 +459,40 @@ static int
 blkfront_suspend(device_t dev)
 {
 	struct xb_softc *sc = device_get_softc(dev);
+	int retval;
+	int saved_state;
 
 	/* Prevent new requests being issued until we fix things up. */
 	mtx_lock(&sc->xb_io_lock);
+	saved_state = sc->connected;
 	sc->connected = BLKIF_STATE_SUSPENDED;
+
+	/* Wait for outstanding I/O to drain. */
+	retval = 0;
+	while (TAILQ_EMPTY(&sc->cm_busy) == 0) {
+		if (msleep(&sc->cm_busy, &sc->xb_io_lock,
+			   PRIBIO, "blkf_susp", 30 * hz) == EWOULDBLOCK) {
+			retval = EBUSY;
+			break;
+		}
+	}
 	mtx_unlock(&sc->xb_io_lock);
 
-	return (0);
+	if (retval != 0)
+		sc->connected = saved_state;
+
+	return (retval);
 }
 
 static int
 blkfront_resume(device_t dev)
 {
-#if 0
 	struct xb_softc *sc = device_get_softc(dev);
 
 	DPRINTK("blkfront_resume: %s\n", xenbus_get_node(dev));
 
-/* XXX This can't work!!! */
-	blkif_free(sc, 1);
+	blkif_free(sc);
 	blkfront_initialize(sc);
-	if (sc->connected == BLKIF_STATE_SUSPENDED)
-		blkif_recover(sc);
-#endif
 	return (0);
 }
 
@@ -499,8 +504,10 @@ blkfront_initialize(struct xb_softc *sc)
 	int error;
 	int i;
 
-	if (xenbus_get_state(sc->xb_dev) != XenbusStateInitialising)
-                return;
+	if (xenbus_get_state(sc->xb_dev) != XenbusStateInitialising) {
+		/* Initialization has already been performed. */
+		return;
+	}
 
 	/*
 	 * Protocol defaults valid even if negotiation for a
@@ -593,8 +600,10 @@ blkfront_initialize(struct xb_softc *sc)
 	sc->shadow = malloc(sizeof(*sc->shadow) * sc->max_requests,
 			    M_XENBLOCKFRONT, M_NOWAIT|M_ZERO);
 	if (sc->shadow == NULL) {
+		bus_dma_tag_destroy(sc->xb_io_dmat);
 		xenbus_dev_fatal(sc->xb_dev, error,
 				 "Cannot allocate request structures\n");
+		return;
 	}
 
 	for (i = 0; i < sc->max_requests; i++) {
@@ -755,10 +764,10 @@ blkfront_backend_changed(device_t dev, XenbusState backend_state)
 		break;
 
 	case XenbusStateInitWait:
+	case XenbusStateInitialised:
 		blkfront_initialize(sc);
 		break;
 
-	case XenbusStateInitialised:
 	case XenbusStateConnected:
 		blkfront_initialize(sc);
 		blkfront_connect(sc);
@@ -775,7 +784,7 @@ blkfront_backend_changed(device_t dev, XenbusState backend_state)
 }
 
 /* 
-** Invoked when the backend is finally 'ready' (and has told produced 
+** Invoked when the backend is finally 'ready' (and has published
 ** the details about the physical device - #sectors, size, etc). 
 */
 static void 
@@ -809,13 +818,15 @@ blkfront_connect(struct xb_softc *sc)
 	if (!err || feature_barrier)
 		sc->xb_flags |= XB_BARRIER;
 
-	device_printf(dev, "%juMB <%s> at %s",
-	    (uintmax_t) sectors / (1048576 / sector_size),
-	    device_get_desc(dev),
-	    xenbus_get_node(dev));
-	bus_print_child_footer(device_get_parent(dev), dev);
+	if (sc->xb_disk == NULL) {
+		device_printf(dev, "%juMB <%s> at %s",
+		    (uintmax_t) sectors / (1048576 / sector_size),
+		    device_get_desc(dev),
+		    xenbus_get_node(dev));
+		bus_print_child_footer(device_get_parent(dev), dev);
 
-	xlvbd_add(sc, sectors, sc->vdevice, binfo, sector_size);
+		xlvbd_add(sc, sectors, sc->vdevice, binfo, sector_size);
+	}
 
 	(void)xenbus_set_state(dev, XenbusStateConnected); 
 
@@ -825,7 +836,6 @@ blkfront_connect(struct xb_softc *sc)
 	xb_startio(sc);
 	sc->xb_flags |= XB_READY;
 	mtx_unlock(&sc->xb_io_lock);
-	
 }
 
 /**
@@ -859,7 +869,7 @@ blkfront_detach(device_t dev)
 
 	DPRINTK("blkfront_remove: %s removed\n", xenbus_get_node(dev));
 
-	blkif_free(sc, 0);
+	blkif_free(sc);
 	mtx_destroy(&sc->xb_io_lock);
 
 	return 0;
@@ -1140,6 +1150,9 @@ xb_startio(struct xb_softc *sc)
 
 	mtx_assert(&sc->xb_io_lock, MA_OWNED);
 
+	if (sc->connected != BLKIF_STATE_CONNECTED)
+		return;
+
 	while (RING_FREE_REQUESTS(&sc->ring) >= sc->max_request_blocks) {
 		if (sc->xb_flags & XB_FROZEN)
 			break;
@@ -1174,7 +1187,7 @@ blkif_int(void *xsc)
 
 	mtx_lock(&sc->xb_io_lock);
 
-	if (unlikely(sc->connected != BLKIF_STATE_CONNECTED)) {
+	if (unlikely(sc->connected == BLKIF_STATE_DISCONNECTED)) {
 		mtx_unlock(&sc->xb_io_lock);
 		return;
 	}
@@ -1232,19 +1245,21 @@ blkif_int(void *xsc)
 
 	xb_startio(sc);
 
+	if (unlikely(sc->connected == BLKIF_STATE_SUSPENDED))
+		wakeup(&sc->cm_busy);
+
 	mtx_unlock(&sc->xb_io_lock);
 }
 
 static void 
-blkif_free(struct xb_softc *sc, int suspend)
+blkif_free(struct xb_softc *sc)
 {
 	uint8_t *sring_page_ptr;
 	int i;
 	
 	/* Prevent new requests being issued until we fix things up. */
 	mtx_lock(&sc->xb_io_lock);
-	sc->connected = suspend ? 
-		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED; 
+	sc->connected = BLKIF_STATE_DISCONNECTED; 
 	mtx_unlock(&sc->xb_io_lock);
 
 	/* Free resources associated with old device channel. */
@@ -1276,6 +1291,12 @@ blkif_free(struct xb_softc *sc, int suspend)
 		}
 		free(sc->shadow, M_XENBLOCKFRONT);
 		sc->shadow = NULL;
+
+		bus_dma_tag_destroy(sc->xb_io_dmat);
+		
+		xb_initq_free(sc);
+		xb_initq_ready(sc);
+		xb_initq_complete(sc);
 	}
 		
 	if (sc->irq) {
@@ -1292,21 +1313,6 @@ blkif_completion(struct xb_command *s)
 	return (BLKIF_SEGS_TO_BLOCKS(s->nseg));
 }
 
-#if 0
-static void 
-blkif_recover(struct xb_softc *sc)
-{
-	/*
-	 * XXX The whole concept of not quiescing and completing all i/o
-	 * during suspend, and then hoping to recover and replay the
-	 * resulting abandoned I/O during resume, is laughable.  At best,
-	 * it invalidates the i/o ordering rules required by just about
-	 * every filesystem, and at worst it'll corrupt data.  The code
-	 * has been removed until further notice.
-	 */
-}
-#endif
-
 /* ** Driver registration ** */
 static device_method_t blkfront_methods[] = { 
 	/* Device interface */ 
diff --git a/sys/dev/xen/blkfront/block.h b/sys/dev/xen/blkfront/block.h
index 6235e515afef..6eabcf4e0723 100644
--- a/sys/dev/xen/blkfront/block.h
+++ b/sys/dev/xen/blkfront/block.h
@@ -142,7 +142,6 @@ struct xb_softc {
 #define XB_READY	(1 << 2)	/* Is ready */
 #define XB_FROZEN	(1 << 3)	/* Waiting for resources */
 	int			vdevice;
-	blkif_vdev_t		handle;
 	int			connected;
 	u_int			ring_pages;
 	uint32_t		max_requests;
diff --git a/sys/dev/xen/control/control.c b/sys/dev/xen/control/control.c
index 301d4e827c6e..63ece1ef05f4 100644
--- a/sys/dev/xen/control/control.c
+++ b/sys/dev/xen/control/control.c
@@ -115,6 +115,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/reboot.h>
 #include <sys/rman.h>
+#include <sys/sched.h>
 #include <sys/taskqueue.h>
 #include <sys/types.h>
 #include <sys/vnode.h>
@@ -201,6 +202,8 @@ xctrl_suspend()
 	int i, j, k, fpp;
 	unsigned long max_pfn, start_info_mfn;
 
+	EVENTHANDLER_INVOKE(power_suspend);
+
 #ifdef SMP
 	struct thread *td;
 	cpuset_t map;
@@ -221,7 +224,13 @@ xctrl_suspend()
 		stop_cpus(map);
 #endif
 
+	/*
+	 * Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
+	 * drivers need this.
+	 */
+	mtx_lock(&Giant);
 	if (DEVICE_SUSPEND(root_bus) != 0) {
+		mtx_unlock(&Giant);
 		printf("xen_suspend: device_suspend failed\n");
 #ifdef SMP
 		if (!CPU_EMPTY(&map))
@@ -229,6 +238,7 @@ xctrl_suspend()
 #endif
 		return;
 	}
+	mtx_unlock(&Giant);
 
 	local_irq_disable();
 
@@ -283,11 +293,14 @@ xctrl_suspend()
 		vcpu_prepare(i);
 
 #endif
+
 	/* 
 	 * Only resume xenbus /after/ we've prepared our VCPUs; otherwise
 	 * the VCPU hotplug callback can race with our vcpu_prepare
 	 */
+	mtx_lock(&Giant);
 	DEVICE_RESUME(root_bus);
+	mtx_unlock(&Giant);
 
 #ifdef SMP
 	thread_lock(curthread);
@@ -296,6 +309,7 @@ xctrl_suspend()
 	if (!CPU_EMPTY(&map))
 		restart_cpus(map);
 #endif
+	EVENTHANDLER_INVOKE(power_resume);
 }
 
 static void
@@ -322,39 +336,47 @@ xctrl_suspend()
 {
 	int suspend_cancelled;
 
+	EVENTHANDLER_INVOKE(power_suspend);
+
+	/*
+	 * Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
+	 * drivers need this.
+	 */
+	mtx_lock(&Giant);
 	if (DEVICE_SUSPEND(root_bus)) {
+		mtx_unlock(&Giant);
 		printf("xen_suspend: device_suspend failed\n");
 		return;
 	}
-
-	/*
-	 * Make sure we don't change cpus or switch to some other
-	 * thread. for the duration.
-	 */
-	critical_enter();
+	mtx_unlock(&Giant);
 
 	/*
 	 * Prevent any races with evtchn_interrupt() handler.
 	 */
-	irq_suspend();
 	disable_intr();
+	irq_suspend();
 
 	suspend_cancelled = HYPERVISOR_suspend(0);
-	if (!suspend_cancelled)
+	if (suspend_cancelled)
+		irq_resume();
+	else
 		xenpci_resume();
 
 	/*
 	 * Re-enable interrupts and put the scheduler back to normal.
 	 */
 	enable_intr();
-	critical_exit();
 
 	/*
 	 * FreeBSD really needs to add DEVICE_SUSPEND_CANCEL or
 	 * similar.
 	 */
+	mtx_lock(&Giant);
 	if (!suspend_cancelled)
 		DEVICE_RESUME(root_bus);
+	mtx_unlock(&Giant);
+
+	EVENTHANDLER_INVOKE(power_resume);
 }
 #endif
 
diff --git a/sys/dev/xen/netfront/netfront.c b/sys/dev/xen/netfront/netfront.c
index c694514053ec..8d6444a9f763 100644
--- a/sys/dev/xen/netfront/netfront.c
+++ b/sys/dev/xen/netfront/netfront.c
@@ -92,7 +92,8 @@ __FBSDID("$FreeBSD$");
 
 #include "xenbus_if.h"
 
-#define XN_CSUM_FEATURES	(CSUM_TCP | CSUM_UDP | CSUM_TSO)
+/* Features supported by all backends.  TSO and LRO can be negotiated */
+#define XN_CSUM_FEATURES	(CSUM_TCP | CSUM_UDP)
 
 #define NET_TX_RING_SIZE __RING_SIZE((netif_tx_sring_t *)0, PAGE_SIZE)
 #define NET_RX_RING_SIZE __RING_SIZE((netif_rx_sring_t *)0, PAGE_SIZE)
@@ -159,6 +160,8 @@ static int  xn_ioctl(struct ifnet *, u_long, caddr_t);
 static void xn_ifinit_locked(struct netfront_info *);
 static void xn_ifinit(void *);
 static void xn_stop(struct netfront_info *);
+static void xn_query_features(struct netfront_info *np);
+static int  xn_configure_features(struct netfront_info *np);
 #ifdef notyet
 static void xn_watchdog(struct ifnet *);
 #endif
@@ -174,7 +177,7 @@ static int talk_to_backend(device_t dev, struct netfront_info *info);
 static int create_netdev(device_t dev);
 static void netif_disconnect_backend(struct netfront_info *info);
 static int setup_device(device_t dev, struct netfront_info *info);
-static void end_access(int ref, void *page);
+static void free_ring(int *ref, void *ring_ptr_ref);
 
 static int  xn_ifmedia_upd(struct ifnet *ifp);
 static void xn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr);
@@ -261,6 +264,7 @@ struct netfront_info {
 	u_int irq;
 	u_int copying_receiver;
 	u_int carrier;
+	u_int maxfrags;
 		
 	/* Receive-ring batched refills. */
 #define RX_MIN_TARGET 32
@@ -405,11 +409,33 @@ xen_net_read_mac(device_t dev, uint8_t mac[])
 {
 	int error, i;
 	char *s, *e, *macstr;
+	const char *path;
 
-	error = xs_read(XST_NIL, xenbus_get_node(dev), "mac", NULL,
-	    (void **) &macstr);
-	if (error)
+	path = xenbus_get_node(dev);
+	error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr);
+	if (error == ENOENT) {
+		/*
+		 * Deal with missing mac XenStore nodes on devices with
+		 * HVM emulation (the 'ioemu' configuration attribute)
+		 * enabled.
+		 *
+		 * The HVM emulator may execute in a stub device model
+		 * domain which lacks the permission, only given to Dom0,
+		 * to update the guest's XenStore tree.  For this reason,
+		 * the HVM emulator doesn't even attempt to write the
+		 * front-side mac node, even when operating in Dom0.
+		 * However, there should always be a mac listed in the
+		 * backend tree.  Fallback to this version if our query
+		 * of the front side XenStore location doesn't find
+		 * anything.
+		 */
+		path = xenbus_get_otherend_path(dev);
+		error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr);
+	}
+	if (error != 0) {
+		xenbus_dev_fatal(dev, error, "parsing %s/mac", path);
 		return (error);
+	}
 
 	s = macstr;
 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
@@ -450,7 +476,7 @@ netfront_attach(device_t dev)
 	err = create_netdev(dev);
 	if (err) {
 		xenbus_dev_fatal(dev, err, "creating netdev");
-		return err;
+		return (err);
 	}
 
 #if __FreeBSD_version >= 700000
@@ -460,9 +486,21 @@ netfront_attach(device_t dev)
 	    &xn_enable_lro, 0, "Large Receive Offload");
 #endif
 
-	return 0;
+	return (0);
 }
 
+static int
+netfront_suspend(device_t dev)
+{
+	struct netfront_info *info = device_get_softc(dev);
+
+	XN_RX_LOCK(info);
+	XN_TX_LOCK(info);
+	netfront_carrier_off(info);
+	XN_TX_UNLOCK(info);
+	XN_RX_UNLOCK(info);
+	return (0);
+}
 
 /**
  * We are reconnecting to the backend, due to a suspend/resume, or a backend
@@ -749,10 +787,7 @@ netif_release_tx_bufs(struct netfront_info *np)
 		 */
 		if (((uintptr_t)m) <= NET_TX_RING_SIZE)
 			continue;
-		gnttab_grant_foreign_access_ref(np->grant_tx_ref[i],
-		    xenbus_get_otherend_id(np->xbdev),
-		    virt_to_mfn(mtod(m, vm_offset_t)),
-		    GNTMAP_readonly);
+		gnttab_end_foreign_access_ref(np->grant_tx_ref[i]);
 		gnttab_release_grant_reference(&np->gref_tx_head,
 		    np->grant_tx_ref[i]);
 		np->grant_tx_ref[i] = GRANT_REF_INVALID;
@@ -761,7 +796,7 @@ netif_release_tx_bufs(struct netfront_info *np)
 		if (np->xn_cdata.xn_tx_chain_cnt < 0) {
 			panic("netif_release_tx_bufs: tx_chain_cnt must be >= 0");
 		}
-		m_freem(m);
+		m_free(m);
 	}
 }
 
@@ -1494,7 +1529,7 @@ xn_assemble_tx_request(struct netfront_info *sc, struct mbuf *m_head)
 	 * deal with nfrags > MAX_TX_REQ_FRAGS, which is a quirk of
 	 * the Linux network stack.
 	 */
-	if (nfrags > MAX_TX_REQ_FRAGS) {
+	if (nfrags > sc->maxfrags) {
 		m = m_defrag(m_head, M_DONTWAIT);
 		if (!m) {
 			/*
@@ -1911,6 +1946,8 @@ network_connect(struct netfront_info *np)
 		return (error);
 	
 	/* Step 1: Reinitialise variables. */
+	xn_query_features(np);
+	xn_configure_features(np);
 	netif_release_tx_bufs(np);
 
 	/* Step 2: Rebuild the RX buffer freelist and the RX ring itself. */
@@ -1978,6 +2015,67 @@ show_device(struct netfront_info *sc)
 #endif
 }
 
+static void
+xn_query_features(struct netfront_info *np)
+{
+	int val;
+
+	device_printf(np->xbdev, "backend features:");
+
+	if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
+		"feature-sg", NULL, "%d", &val) < 0)
+		val = 0;
+
+	np->maxfrags = 1;
+	if (val) {
+		np->maxfrags = MAX_TX_REQ_FRAGS;
+		printf(" feature-sg");
+	}
+
+	if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
+		"feature-gso-tcpv4", NULL, "%d", &val) < 0)
+		val = 0;
+
+	np->xn_ifp->if_capabilities &= ~(IFCAP_TSO4|IFCAP_LRO);
+	if (val) {
+		np->xn_ifp->if_capabilities |= IFCAP_TSO4|IFCAP_LRO;
+		printf(" feature-gso-tcp4");
+	}
+
+	printf("\n");
+}
+
+static int
+xn_configure_features(struct netfront_info *np)
+{
+	int err;
+
+	err = 0;
+#if __FreeBSD_version >= 700000
+	if ((np->xn_ifp->if_capenable & IFCAP_LRO) != 0)
+		tcp_lro_free(&np->xn_lro);
+#endif
+    	np->xn_ifp->if_capenable =
+	    np->xn_ifp->if_capabilities & ~(IFCAP_LRO|IFCAP_TSO4);
+	np->xn_ifp->if_hwassist &= ~CSUM_TSO;
+#if __FreeBSD_version >= 700000
+	if (xn_enable_lro && (np->xn_ifp->if_capabilities & IFCAP_LRO) != 0) {
+		err = tcp_lro_init(&np->xn_lro);
+		if (err) {
+			device_printf(np->xbdev, "LRO initialization failed\n");
+		} else {
+			np->xn_lro.ifp = np->xn_ifp;
+			np->xn_ifp->if_capenable |= IFCAP_LRO;
+		}
+	}
+	if ((np->xn_ifp->if_capabilities & IFCAP_TSO4) != 0) {
+		np->xn_ifp->if_capenable |= IFCAP_TSO4;
+		np->xn_ifp->if_hwassist |= CSUM_TSO;
+	}
+#endif
+	return (err);
+}
+
 /** Create a network device.
  * @param handle device handle
  */
@@ -2002,7 +2100,7 @@ create_netdev(device_t dev)
 	np->rx_target     = RX_MIN_TARGET;
 	np->rx_min_target = RX_MIN_TARGET;
 	np->rx_max_target = RX_MAX_TARGET;
-	
+
 	/* Initialise {tx,rx}_skbs to be a free chain containing every entry. */
 	for (i = 0; i <= NET_TX_RING_SIZE; i++) {
 		np->tx_mbufs[i] = (void *) ((u_long) i+1);
@@ -2032,11 +2130,8 @@ create_netdev(device_t dev)
 	}
 	
 	err = xen_net_read_mac(dev, np->mac);
-	if (err) {
-		xenbus_dev_fatal(dev, err, "parsing %s/mac",
-		    xenbus_get_node(dev));
+	if (err)
 		goto out;
-	}
 	
 	/* Set up ifnet structure */
 	ifp = np->xn_ifp = if_alloc(IFT_ETHER);
@@ -2055,19 +2150,6 @@ create_netdev(device_t dev)
 	
     	ifp->if_hwassist = XN_CSUM_FEATURES;
     	ifp->if_capabilities = IFCAP_HWCSUM;
-#if __FreeBSD_version >= 700000
-	ifp->if_capabilities |= IFCAP_TSO4;
-	if (xn_enable_lro) {
-		int err = tcp_lro_init(&np->xn_lro);
-		if (err) {
-			device_printf(dev, "LRO initialization failed\n");
-			goto exit;
-		}
-		np->xn_lro.ifp = ifp;
-		ifp->if_capabilities |= IFCAP_LRO;
-	}
-#endif
-    	ifp->if_capenable = ifp->if_capabilities;
 	
     	ether_ifattach(ifp, np->mac);
     	callout_init(&np->xn_stat_ch, CALLOUT_MPSAFE);
@@ -2078,8 +2160,7 @@ create_netdev(device_t dev)
 exit:
 	gnttab_free_grant_references(np->gref_tx_head);
 out:
-	panic("do something smart");
-
+	return (err);
 }
 
 /**
@@ -2133,12 +2214,8 @@ netif_disconnect_backend(struct netfront_info *info)
 	XN_TX_UNLOCK(info);
 	XN_RX_UNLOCK(info);
 
-	end_access(info->tx_ring_ref, info->tx.sring);
-	end_access(info->rx_ring_ref, info->rx.sring);
-	info->tx_ring_ref = GRANT_REF_INVALID;
-	info->rx_ring_ref = GRANT_REF_INVALID;
-	info->tx.sring = NULL;
-	info->rx.sring = NULL;
+	free_ring(&info->tx_ring_ref, &info->tx.sring);
+	free_ring(&info->rx_ring_ref, &info->rx.sring);
 
 	if (info->irq)
 		unbind_from_irqhandler(info->irq);
@@ -2146,12 +2223,17 @@ netif_disconnect_backend(struct netfront_info *info)
 	info->irq = 0;
 }
 
-
 static void
-end_access(int ref, void *page)
+free_ring(int *ref, void *ring_ptr_ref)
 {
-	if (ref != GRANT_REF_INVALID)
-		gnttab_end_foreign_access(ref, page);
+	void **ring_ptr_ptr = ring_ptr_ref;
+
+	if (*ref != GRANT_REF_INVALID) {
+		/* This API frees the associated storage. */
+		gnttab_end_foreign_access(*ref, *ring_ptr_ptr);
+		*ref = GRANT_REF_INVALID;
+	}
+	*ring_ptr_ptr = NULL;
 }
 
 static int
@@ -2174,7 +2256,7 @@ static device_method_t netfront_methods[] = {
 	DEVMETHOD(device_attach,        netfront_attach), 
 	DEVMETHOD(device_detach,        netfront_detach), 
 	DEVMETHOD(device_shutdown,      bus_generic_shutdown), 
-	DEVMETHOD(device_suspend,       bus_generic_suspend), 
+	DEVMETHOD(device_suspend,       netfront_suspend), 
 	DEVMETHOD(device_resume,        netfront_resume), 
  
 	/* Xenbus interface */
diff --git a/sys/dev/mii/exphy.c b/sys/dev/xl/xlphy.c
similarity index 86%
rename from sys/dev/mii/exphy.c
rename to sys/dev/xl/xlphy.c
index 04add674fcc4..e5ac9fce4b29 100644
--- a/sys/dev/mii/exphy.c
+++ b/sys/dev/xl/xlphy.c
@@ -77,30 +77,30 @@ __FBSDID("$FreeBSD$");
 
 #include "miibus_if.h"
 
-static int exphy_probe(device_t);
-static int exphy_attach(device_t);
+static int xlphy_probe(device_t);
+static int xlphy_attach(device_t);
 
-static device_method_t exphy_methods[] = {
+static device_method_t xlphy_methods[] = {
 	/* device interface */
-	DEVMETHOD(device_probe,		exphy_probe),
-	DEVMETHOD(device_attach,	exphy_attach),
+	DEVMETHOD(device_probe,		xlphy_probe),
+	DEVMETHOD(device_attach,	xlphy_attach),
 	DEVMETHOD(device_detach,	mii_phy_detach),
 	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
 	{ 0, 0 }
 };
 
-static devclass_t exphy_devclass;
+static devclass_t xlphy_devclass;
 
-static driver_t exphy_driver = {
+static driver_t xlphy_driver = {
 	"xlphy",
-	exphy_methods,
+	xlphy_methods,
 	sizeof(struct mii_softc)
 };
 
-DRIVER_MODULE(xlphy, miibus, exphy_driver, exphy_devclass, 0, 0);
+DRIVER_MODULE(xlphy, miibus, xlphy_driver, xlphy_devclass, 0, 0);
 
-static int	exphy_service(struct mii_softc *, struct mii_data *, int);
-static void	exphy_reset(struct mii_softc *);
+static int	xlphy_service(struct mii_softc *, struct mii_data *, int);
+static void	xlphy_reset(struct mii_softc *);
 
 /*
  * Some 3Com internal PHYs report zero for OUI and model, others use
@@ -109,42 +109,42 @@ static void	exphy_reset(struct mii_softc *);
  * handled fine by ukphy(4); they can be isolated and don't require
  * special treatment after reset.
  */
-static const struct mii_phydesc exphys[] = {
+static const struct mii_phydesc xlphys[] = {
 	{ 0, 0, "3Com internal media interface" },
 	MII_PHY_DESC(xxBROADCOM, 3C905C),
 	MII_PHY_END
 };
 
-static const struct mii_phy_funcs exphy_funcs = {
-	exphy_service,
+static const struct mii_phy_funcs xlphy_funcs = {
+	xlphy_service,
 	ukphy_status,
-	exphy_reset
+	xlphy_reset
 };
 
 static int
-exphy_probe(device_t dev)
+xlphy_probe(device_t dev)
 {
 
 	if (strcmp(device_get_name(device_get_parent(device_get_parent(dev))),
 	    "xl") == 0)
-		return (mii_phy_dev_probe(dev, exphys, BUS_PROBE_DEFAULT));
+		return (mii_phy_dev_probe(dev, xlphys, BUS_PROBE_DEFAULT));
 	return (ENXIO);
 }
 
 static int
-exphy_attach(device_t dev)
+xlphy_attach(device_t dev)
 {
 
 	/*
 	 * The 3Com PHY can never be isolated.
 	 */
 	mii_phy_dev_attach(dev, MIIF_NOISOLATE | MIIF_NOMANPAUSE,
-	    &exphy_funcs, 1);
+	    &xlphy_funcs, 1);
 	return (0);
 }
 
 static int
-exphy_service(struct mii_softc *sc, struct mii_data *mii, int cmd)
+xlphy_service(struct mii_softc *sc, struct mii_data *mii, int cmd)
 {
 
 	switch (cmd) {
@@ -184,7 +184,7 @@ exphy_service(struct mii_softc *sc, struct mii_data *mii, int cmd)
 }
 
 static void
-exphy_reset(struct mii_softc *sc)
+xlphy_reset(struct mii_softc *sc)
 {
 
 	mii_phy_reset(sc);
diff --git a/sys/fs/devfs/devfs_devs.c b/sys/fs/devfs/devfs_devs.c
index a2d3222dda93..a111527a68b4 100644
--- a/sys/fs/devfs/devfs_devs.c
+++ b/sys/fs/devfs/devfs_devs.c
@@ -171,8 +171,7 @@ devfs_free(struct cdev *cdev)
 	cdp = cdev2priv(cdev);
 	if (cdev->si_cred != NULL)
 		crfree(cdev->si_cred);
-	if (cdp->cdp_inode > 0)
-		free_unr(devfs_inos, cdp->cdp_inode);
+	devfs_free_cdp_inode(cdp->cdp_inode);
 	if (cdp->cdp_maxdirent > 0) 
 		free(cdp->cdp_dirents, M_DEVFS2);
 	free(cdp, M_CDEVP);
@@ -394,7 +393,7 @@ devfs_delete(struct devfs_mount *dm, struct devfs_dirent *de, int flags)
 	mac_devfs_destroy(de);
 #endif
 	if (de->de_inode > DEVFS_ROOTINO) {
-		free_unr(devfs_inos, de->de_inode);
+		devfs_free_cdp_inode(de->de_inode);
 		de->de_inode = 0;
 	}
 	if (DEVFS_DE_DROP(de))
@@ -685,6 +684,21 @@ devfs_destroy(struct cdev *dev)
 	devfs_generation++;
 }
 
+ino_t
+devfs_alloc_cdp_inode(void)
+{
+
+	return (alloc_unr(devfs_inos));
+}
+
+void
+devfs_free_cdp_inode(ino_t ino)
+{
+
+	if (ino > 0)
+		free_unr(devfs_inos, ino);
+}
+
 static void
 devfs_devs_init(void *junk __unused)
 {
diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index 42ace8204f22..442bc54ac8a8 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -3230,7 +3230,7 @@ nfsrv_backupstable(void)
 		    procp->p_stats->p_start.tv_usec ==
 		    nfsd_master_start.tv_usec &&
 		    strcmp(procp->p_comm, nfsd_master_comm) == 0)
-			psignal(procp, SIGUSR2);
+			kern_psignal(procp, SIGUSR2);
 		else
 			nfsd_master_proc = NULL;
 
diff --git a/sys/fs/nullfs/null_vfsops.c b/sys/fs/nullfs/null_vfsops.c
index e39dd8f892c4..8c00f870d1ab 100644
--- a/sys/fs/nullfs/null_vfsops.c
+++ b/sys/fs/nullfs/null_vfsops.c
@@ -104,11 +104,10 @@ nullfs_mount(struct mount *mp)
 		return (EINVAL);
 
 	/*
-	 * Unlock lower node to avoid deadlock.
-	 * (XXX) VOP_ISLOCKED is needed?
+	 * Unlock lower node to avoid possible deadlock.
 	 */
 	if ((mp->mnt_vnodecovered->v_op == &null_vnodeops) &&
-		VOP_ISLOCKED(mp->mnt_vnodecovered)) {
+	    VOP_ISLOCKED(mp->mnt_vnodecovered) == LK_EXCLUSIVE) {
 		VOP_UNLOCK(mp->mnt_vnodecovered, 0);
 		isvnunlocked = 1;
 	}
@@ -120,7 +119,7 @@ nullfs_mount(struct mount *mp)
 	/*
 	 * Re-lock vnode.
 	 */
-	if (isvnunlocked && !VOP_ISLOCKED(mp->mnt_vnodecovered))
+	if (isvnunlocked)
 		vn_lock(mp->mnt_vnodecovered, LK_EXCLUSIVE | LK_RETRY);
 
 	if (error)
@@ -158,7 +157,6 @@ nullfs_mount(struct mount *mp)
 	 * Make sure the node alias worked
 	 */
 	if (error) {
-		VOP_UNLOCK(vp, 0);
 		vrele(lowerrootvp);
 		free(xmp, M_NULLFSMNT);	/* XXX */
 		return (error);
@@ -244,10 +242,7 @@ nullfs_root(mp, flags, vpp)
 	vp = MOUNTTONULLMOUNT(mp)->nullm_rootvp;
 	VREF(vp);
 
-#ifdef NULLFS_DEBUG
-	if (VOP_ISLOCKED(vp))
-		panic("root vnode is locked.\n");
-#endif
+	ASSERT_VOP_UNLOCKED(vp, "root vnode is locked");
 	vn_lock(vp, flags | LK_RETRY);
 	*vpp = vp;
 	return 0;
diff --git a/sys/fs/portalfs/portal_vnops.c b/sys/fs/portalfs/portal_vnops.c
index 7cd5267698b7..f9ef08a50db1 100644
--- a/sys/fs/portalfs/portal_vnops.c
+++ b/sys/fs/portalfs/portal_vnops.c
@@ -240,8 +240,13 @@ portal_open(ap)
 	 * This may require access to a global namespace (e.g. an IP address);
 	 * disallow it entirely, as we do open(2).
 	 */
-	if (IN_CAPABILITY_MODE(td))
+	if (IN_CAPABILITY_MODE(td)) {
+#ifdef KTRACE
+		if (KTRPOINT(td, KTR_CAPFAIL))
+			ktrcapfail(CAPFAIL_SYSCALL, 0, 0);
+#endif
 		return (ECAPMODE);
+	}
 #endif
 
 	/*
diff --git a/sys/fs/procfs/procfs_ctl.c b/sys/fs/procfs/procfs_ctl.c
index 5980ce179ece..dc267f64073e 100644
--- a/sys/fs/procfs/procfs_ctl.c
+++ b/sys/fs/procfs/procfs_ctl.c
@@ -147,7 +147,7 @@ procfs_control(struct thread *td, struct proc *p, int op)
 			p->p_oppid = p->p_pptr->p_pid;
 			proc_reparent(p, td->td_proc);
 		}
-		psignal(p, SIGSTOP);
+		kern_psignal(p, SIGSTOP);
 out:
 		PROC_UNLOCK(p);
 		sx_xunlock(&proctree_lock);
@@ -348,7 +348,7 @@ procfs_doprocctl(PFS_FILL_ARGS)
 				thread_unsuspend(p);
 				PROC_SUNLOCK(p);
 			} else
-				psignal(p, nm->nm_val);
+				kern_psignal(p, nm->nm_val);
 			PROC_UNLOCK(p);
 			error = 0;
 		}
diff --git a/sys/fs/procfs/procfs_ioctl.c b/sys/fs/procfs/procfs_ioctl.c
index 4b45af67009e..522dd9c57cfb 100644
--- a/sys/fs/procfs/procfs_ioctl.c
+++ b/sys/fs/procfs/procfs_ioctl.c
@@ -188,10 +188,10 @@ procfs_ioctl(PFS_IOCTL_ARGS)
 			thread_unsuspend(p);
 			PROC_SUNLOCK(p);
 		} else if (sig)
-			psignal(p, sig);
+			kern_psignal(p, sig);
 #else
 		if (sig)
-			psignal(p, sig);
+			kern_psignal(p, sig);
 		p->p_step = 0;
 		wakeup(&p->p_step);
 #endif
diff --git a/sys/fs/unionfs/union_vnops.c b/sys/fs/unionfs/union_vnops.c
index 11671cb08ae0..19ae7d2e7f3d 100644
--- a/sys/fs/unionfs/union_vnops.c
+++ b/sys/fs/unionfs/union_vnops.c
@@ -748,7 +748,7 @@ unionfs_access(struct vop_access_args *ap)
 						return (error);
 				}
 			}
-			accmode &= ~VWRITE;
+			accmode &= ~(VWRITE | VAPPEND);
 			accmode |= VREAD; /* will copy to upper */
 		}
 		error = VOP_ACCESS(lvp, accmode, ap->a_cred, td);
diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index 6941470856ab..b5e4f7b6d696 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -475,7 +475,8 @@ g_eli_worker(void *arg)
 			}
 			while (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
 				if (sc->sc_inflight > 0) {
-					G_ELI_DEBUG(0, "inflight=%d", sc->sc_inflight);
+					G_ELI_DEBUG(0, "inflight=%d",
+					    sc->sc_inflight);
 					/*
 					 * We still have inflight BIOs, so
 					 * sleep and retry.
@@ -712,16 +713,19 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 	else
 		gp->access = g_std_access;
 
+	sc->sc_version = md->md_version;
 	sc->sc_inflight = 0;
 	sc->sc_crypto = G_ELI_CRYPTO_UNKNOWN;
 	sc->sc_flags = md->md_flags;
 	/* Backward compatibility. */
-	if (md->md_version < 4)
+	if (md->md_version < G_ELI_VERSION_04)
 		sc->sc_flags |= G_ELI_FLAG_NATIVE_BYTE_ORDER;
-	if (md->md_version < 5)
+	if (md->md_version < G_ELI_VERSION_05)
 		sc->sc_flags |= G_ELI_FLAG_SINGLE_KEY;
-	if (md->md_version < 6 && (sc->sc_flags & G_ELI_FLAG_AUTH) != 0)
+	if (md->md_version < G_ELI_VERSION_06 &&
+	    (sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
 		sc->sc_flags |= G_ELI_FLAG_FIRST_KEY;
+	}
 	sc->sc_ealgo = md->md_ealgo;
 	sc->sc_nkey = nkey;
 
@@ -1132,7 +1136,8 @@ g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 			/* Try again. */
 			continue;
 		} else if (error > 0) {
-			G_ELI_DEBUG(0, "Cannot decrypt Master Key for %s (error=%d).",
+			G_ELI_DEBUG(0,
+			    "Cannot decrypt Master Key for %s (error=%d).",
 			    pp->name, error);
 			g_eli_keyfiles_clear(pp->name);
 			return (NULL);
@@ -1206,6 +1211,7 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 		sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
 		    sc->sc_nkey);
 	}
+	sbuf_printf(sb, "%s<Version>%u</Version>\n", indent, sc->sc_version);
 	sbuf_printf(sb, "%s<Crypto>", indent);
 	switch (sc->sc_crypto) {
 	case G_ELI_CRYPTO_HW:
@@ -1226,8 +1232,8 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 	}
 	sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent,
 	    sc->sc_ekeylen);
-	sbuf_printf(sb, "%s<EncryptionAlgorithm>%s</EncryptionAlgorithm>\n", indent,
-	    g_eli_algo2str(sc->sc_ealgo));
+	sbuf_printf(sb, "%s<EncryptionAlgorithm>%s</EncryptionAlgorithm>\n",
+	    indent, g_eli_algo2str(sc->sc_ealgo));
 	sbuf_printf(sb, "%s<State>%s</State>\n", indent,
 	    (sc->sc_flags & G_ELI_FLAG_SUSPEND) ? "SUSPENDED" : "ACTIVE");
 }
diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h
index 68f92e5ad578..0b48db24e62f 100644
--- a/sys/geom/eli/g_eli.h
+++ b/sys/geom/eli/g_eli.h
@@ -43,6 +43,7 @@
 #include <sys/tree.h>
 #include <geom/geom.h>
 #else
+#include <assert.h>
 #include <stdio.h>
 #include <string.h>
 #include <strings.h>
@@ -166,6 +167,7 @@ struct g_eli_worker {
 
 struct g_eli_softc {
 	struct g_geom	*sc_geom;
+	u_int		 sc_version;
 	u_int		 sc_crypto;
 	uint8_t		 sc_mkey[G_ELI_DATAIVKEYLEN];
 	uint8_t		 sc_ekey[G_ELI_DATAKEYLEN];
@@ -217,14 +219,28 @@ struct g_eli_metadata {
 } __packed;
 #ifndef _OpenSSL_
 static __inline void
-eli_metadata_encode(struct g_eli_metadata *md, u_char *data)
+eli_metadata_encode_v0(struct g_eli_metadata *md, u_char **datap)
 {
-	MD5_CTX ctx;
 	u_char *p;
 
-	p = data;
-	bcopy(md->md_magic, p, sizeof(md->md_magic)); p += sizeof(md->md_magic);
-	le32enc(p, md->md_version);	p += sizeof(md->md_version);
+	p = *datap;
+	le32enc(p, md->md_flags);	p += sizeof(md->md_flags);
+	le16enc(p, md->md_ealgo);	p += sizeof(md->md_ealgo);
+	le16enc(p, md->md_keylen);	p += sizeof(md->md_keylen);
+	le64enc(p, md->md_provsize);	p += sizeof(md->md_provsize);
+	le32enc(p, md->md_sectorsize);	p += sizeof(md->md_sectorsize);
+	*p = md->md_keys;		p += sizeof(md->md_keys);
+	le32enc(p, md->md_iterations);	p += sizeof(md->md_iterations);
+	bcopy(md->md_salt, p, sizeof(md->md_salt)); p += sizeof(md->md_salt);
+	bcopy(md->md_mkeys, p, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
+	*datap = p;
+}
+static __inline void
+eli_metadata_encode_v1v2v3v4v5v6(struct g_eli_metadata *md, u_char **datap)
+{
+	u_char *p;
+
+	p = *datap;
 	le32enc(p, md->md_flags);	p += sizeof(md->md_flags);
 	le16enc(p, md->md_ealgo);	p += sizeof(md->md_ealgo);
 	le16enc(p, md->md_keylen);	p += sizeof(md->md_keylen);
@@ -235,6 +251,39 @@ eli_metadata_encode(struct g_eli_metadata *md, u_char *data)
 	le32enc(p, md->md_iterations);	p += sizeof(md->md_iterations);
 	bcopy(md->md_salt, p, sizeof(md->md_salt)); p += sizeof(md->md_salt);
 	bcopy(md->md_mkeys, p, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
+	*datap = p;
+}
+static __inline void
+eli_metadata_encode(struct g_eli_metadata *md, u_char *data)
+{
+	MD5_CTX ctx;
+	u_char *p;
+
+	p = data;
+	bcopy(md->md_magic, p, sizeof(md->md_magic));
+	p += sizeof(md->md_magic);
+	le32enc(p, md->md_version);
+	p += sizeof(md->md_version);
+	switch (md->md_version) {
+	case G_ELI_VERSION_00:
+		eli_metadata_encode_v0(md, &p);
+		break;
+	case G_ELI_VERSION_01:
+	case G_ELI_VERSION_02:
+	case G_ELI_VERSION_03:
+	case G_ELI_VERSION_04:
+	case G_ELI_VERSION_05:
+	case G_ELI_VERSION_06:
+		eli_metadata_encode_v1v2v3v4v5v6(md, &p);
+		break;
+	default:
+#ifdef _KERNEL
+		panic("%s: Unsupported version %u.", __func__,
+		    (u_int)md->md_version);
+#else
+		assert(!"Unsupported metadata version.");
+#endif
+	}
 	MD5Init(&ctx);
 	MD5Update(&ctx, data, p - data);
 	MD5Final(md->md_hash, &ctx);
@@ -294,6 +343,8 @@ eli_metadata_decode(const u_char *data, struct g_eli_metadata *md)
 	int error;
 
 	bcopy(data, md->md_magic, sizeof(md->md_magic));
+	if (strcmp(md->md_magic, G_ELI_MAGIC) != 0)
+		return (EINVAL);
 	md->md_version = le32dec(data + sizeof(md->md_magic));
 	switch (md->md_version) {
 	case G_ELI_VERSION_00:
@@ -308,7 +359,7 @@ eli_metadata_decode(const u_char *data, struct g_eli_metadata *md)
 		error = eli_metadata_decode_v1v2v3v4v5v6(data, md);
 		break;
 	default:
-		error = EINVAL;
+		error = EOPNOTSUPP;
 		break;
 	}
 	return (error);
diff --git a/sys/geom/geom_disk.c b/sys/geom/geom_disk.c
index 3072895fdc9e..6b6c5ae83bde 100644
--- a/sys/geom/geom_disk.c
+++ b/sys/geom/geom_disk.c
@@ -109,6 +109,7 @@ DECLARE_GEOM_CLASS(g_disk_class, g_disk);
 static void __inline
 g_disk_lock_giant(struct disk *dp)
 {
+
 	if (dp->d_flags & DISKFLAG_NEEDSGIANT)
 		mtx_lock(&Giant);
 }
@@ -116,6 +117,7 @@ g_disk_lock_giant(struct disk *dp)
 static void __inline
 g_disk_unlock_giant(struct disk *dp)
 {
+
 	if (dp->d_flags & DISKFLAG_NEEDSGIANT)
 		mtx_unlock(&Giant);
 }
@@ -251,9 +253,9 @@ g_disk_done(struct bio *bp)
 	if (bp2->bio_error == 0)
 		bp2->bio_error = bp->bio_error;
 	bp2->bio_completed += bp->bio_completed;
-	if ((bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) &&
-	    (sc = bp2->bio_to->geom->softc) &&
-	    (dp = sc->dp)) {
+	if ((bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) != 0 &&
+	    (sc = bp2->bio_to->geom->softc) != NULL &&
+	    (dp = sc->dp) != NULL) {
 		devstat_end_transaction_bio(dp->d_devstat, bp);
 	}
 	g_destroy_bio(bp);
@@ -282,7 +284,7 @@ g_disk_ioctl(struct g_provider *pp, u_long cmd, void * data, int fflag, struct t
 	g_disk_lock_giant(dp);
 	error = dp->d_ioctl(dp, cmd, data, fflag, td);
 	g_disk_unlock_giant(dp);
-	return(error);
+	return (error);
 }
 
 static void
@@ -303,7 +305,7 @@ g_disk_start(struct bio *bp)
 	switch(bp->bio_cmd) {
 	case BIO_DELETE:
 		if (!(dp->d_flags & DISKFLAG_CANDELETE)) {
-			error = 0;
+			error = EOPNOTSUPP;
 			break;
 		}
 		/* fall-through */
@@ -392,8 +394,8 @@ g_disk_start(struct bio *bp)
 		g_trace(G_T_TOPOLOGY, "g_disk_flushcache(%s)",
 		    bp->bio_to->name);
 		if (!(dp->d_flags & DISKFLAG_CANFLUSHCACHE)) {
-			g_io_deliver(bp, ENODEV);
-			return;
+			error = EOPNOTSUPP;
+			break;
 		}
 		bp2 = g_clone_bio(bp);
 		if (bp2 == NULL) {
@@ -537,17 +539,16 @@ g_disk_ident_adjust(char *ident, size_t size)
 }
 
 struct disk *
-disk_alloc()
+disk_alloc(void)
 {
-	struct disk *dp;
 
-	dp = g_malloc(sizeof *dp, M_WAITOK | M_ZERO);
-	return (dp);
+	return (g_malloc(sizeof(struct disk), M_WAITOK | M_ZERO));
 }
 
 void
 disk_create(struct disk *dp, int version)
 {
+
 	if (version != DISK_VERSION_00 && version != DISK_VERSION_01) {
 		printf("WARNING: Attempt to add disk %s%d %s",
 		    dp->d_name, dp->d_unit,
@@ -637,4 +638,3 @@ sysctl_disks(SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_kern, OID_AUTO, disks,
     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
     sysctl_disks, "A", "names of available disks");
-
diff --git a/sys/geom/part/g_part_gpt.c b/sys/geom/part/g_part_gpt.c
index 667a8d6cb6c0..b64f125c7c3c 100644
--- a/sys/geom/part/g_part_gpt.c
+++ b/sys/geom/part/g_part_gpt.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2002, 2005, 2006, 2007 Marcel Moolenaar
+ * Copyright (c) 2002, 2005-2007, 2011 Marcel Moolenaar
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,6 +79,7 @@ struct g_part_gpt_table {
 	struct gpt_hdr		*hdr;
 	quad_t			lba[GPT_ELT_COUNT];
 	enum gpt_state		state[GPT_ELT_COUNT];
+	int			bootcamp;
 };
 
 struct g_part_gpt_entry {
@@ -178,41 +179,165 @@ static struct uuid gpt_uuid_unused = GPT_ENT_TYPE_UNUSED;
 static struct g_part_uuid_alias {
 	struct uuid *uuid;
 	int alias;
+	int mbrtype;
 } gpt_uuid_alias_match[] = {
-	{ &gpt_uuid_apple_boot,		G_PART_ALIAS_APPLE_BOOT },
-	{ &gpt_uuid_apple_hfs,		G_PART_ALIAS_APPLE_HFS },
-	{ &gpt_uuid_apple_label,	G_PART_ALIAS_APPLE_LABEL },
-	{ &gpt_uuid_apple_raid,		G_PART_ALIAS_APPLE_RAID },
-	{ &gpt_uuid_apple_raid_offline,	G_PART_ALIAS_APPLE_RAID_OFFLINE },
-	{ &gpt_uuid_apple_tv_recovery,	G_PART_ALIAS_APPLE_TV_RECOVERY },
-	{ &gpt_uuid_apple_ufs,		G_PART_ALIAS_APPLE_UFS },
-	{ &gpt_uuid_bios_boot,		G_PART_ALIAS_BIOS_BOOT },
-	{ &gpt_uuid_efi, 		G_PART_ALIAS_EFI },
-	{ &gpt_uuid_freebsd,		G_PART_ALIAS_FREEBSD },
-	{ &gpt_uuid_freebsd_boot, 	G_PART_ALIAS_FREEBSD_BOOT },
-	{ &gpt_uuid_freebsd_swap,	G_PART_ALIAS_FREEBSD_SWAP },
-	{ &gpt_uuid_freebsd_ufs,	G_PART_ALIAS_FREEBSD_UFS },
-	{ &gpt_uuid_freebsd_vinum,	G_PART_ALIAS_FREEBSD_VINUM },
-	{ &gpt_uuid_freebsd_zfs,	G_PART_ALIAS_FREEBSD_ZFS },
-	{ &gpt_uuid_linux_data,		G_PART_ALIAS_LINUX_DATA },
-	{ &gpt_uuid_linux_lvm,		G_PART_ALIAS_LINUX_LVM },
-	{ &gpt_uuid_linux_raid,		G_PART_ALIAS_LINUX_RAID },
-	{ &gpt_uuid_linux_swap,		G_PART_ALIAS_LINUX_SWAP },
-	{ &gpt_uuid_mbr,		G_PART_ALIAS_MBR },
-	{ &gpt_uuid_ms_basic_data,	G_PART_ALIAS_MS_BASIC_DATA },
-	{ &gpt_uuid_ms_ldm_data,	G_PART_ALIAS_MS_LDM_DATA },
-	{ &gpt_uuid_ms_ldm_metadata,	G_PART_ALIAS_MS_LDM_METADATA },
-	{ &gpt_uuid_ms_reserved,	G_PART_ALIAS_MS_RESERVED },
-	{ &gpt_uuid_netbsd_ccd,		G_PART_ALIAS_NETBSD_CCD },
-	{ &gpt_uuid_netbsd_cgd,		G_PART_ALIAS_NETBSD_CGD },
-	{ &gpt_uuid_netbsd_ffs,		G_PART_ALIAS_NETBSD_FFS },
-	{ &gpt_uuid_netbsd_lfs,		G_PART_ALIAS_NETBSD_LFS },
-	{ &gpt_uuid_netbsd_raid,	G_PART_ALIAS_NETBSD_RAID },
-	{ &gpt_uuid_netbsd_swap,	G_PART_ALIAS_NETBSD_SWAP },
-
-	{ NULL, 0 }
+	{ &gpt_uuid_apple_boot,		G_PART_ALIAS_APPLE_BOOT,	 0xab },
+	{ &gpt_uuid_apple_hfs,		G_PART_ALIAS_APPLE_HFS,		 0xaf },
+	{ &gpt_uuid_apple_label,	G_PART_ALIAS_APPLE_LABEL,	 0 },
+	{ &gpt_uuid_apple_raid,		G_PART_ALIAS_APPLE_RAID,	 0 },
+	{ &gpt_uuid_apple_raid_offline,	G_PART_ALIAS_APPLE_RAID_OFFLINE, 0 },
+	{ &gpt_uuid_apple_tv_recovery,	G_PART_ALIAS_APPLE_TV_RECOVERY,	 0 },
+	{ &gpt_uuid_apple_ufs,		G_PART_ALIAS_APPLE_UFS,		 0 },
+	{ &gpt_uuid_bios_boot,		G_PART_ALIAS_BIOS_BOOT,		 0 },
+	{ &gpt_uuid_efi, 		G_PART_ALIAS_EFI,		 0xee },
+	{ &gpt_uuid_freebsd,		G_PART_ALIAS_FREEBSD,		 0xa5 },
+	{ &gpt_uuid_freebsd_boot, 	G_PART_ALIAS_FREEBSD_BOOT,	 0 },
+	{ &gpt_uuid_freebsd_swap,	G_PART_ALIAS_FREEBSD_SWAP,	 0 },
+	{ &gpt_uuid_freebsd_ufs,	G_PART_ALIAS_FREEBSD_UFS,	 0 },
+	{ &gpt_uuid_freebsd_vinum,	G_PART_ALIAS_FREEBSD_VINUM,	 0 },
+	{ &gpt_uuid_freebsd_zfs,	G_PART_ALIAS_FREEBSD_ZFS,	 0 },
+	{ &gpt_uuid_linux_data,		G_PART_ALIAS_LINUX_DATA,	 0x0b },
+	{ &gpt_uuid_linux_lvm,		G_PART_ALIAS_LINUX_LVM,		 0 },
+	{ &gpt_uuid_linux_raid,		G_PART_ALIAS_LINUX_RAID,	 0 },
+	{ &gpt_uuid_linux_swap,		G_PART_ALIAS_LINUX_SWAP,	 0 },
+	{ &gpt_uuid_mbr,		G_PART_ALIAS_MBR,		 0 },
+	{ &gpt_uuid_ms_basic_data,	G_PART_ALIAS_MS_BASIC_DATA,	 0x0b },
+	{ &gpt_uuid_ms_ldm_data,	G_PART_ALIAS_MS_LDM_DATA,	 0 },
+	{ &gpt_uuid_ms_ldm_metadata,	G_PART_ALIAS_MS_LDM_METADATA,	 0 },
+	{ &gpt_uuid_ms_reserved,	G_PART_ALIAS_MS_RESERVED,	 0 },
+	{ &gpt_uuid_netbsd_ccd,		G_PART_ALIAS_NETBSD_CCD,	 0 },
+	{ &gpt_uuid_netbsd_cgd,		G_PART_ALIAS_NETBSD_CGD,	 0 },
+	{ &gpt_uuid_netbsd_ffs,		G_PART_ALIAS_NETBSD_FFS,	 0 },
+	{ &gpt_uuid_netbsd_lfs,		G_PART_ALIAS_NETBSD_LFS,	 0 },
+	{ &gpt_uuid_netbsd_raid,	G_PART_ALIAS_NETBSD_RAID,	 0 },
+	{ &gpt_uuid_netbsd_swap,	G_PART_ALIAS_NETBSD_SWAP,	 0 },
+	{ NULL, 0, 0 }
 };
 
+static int
+gpt_write_mbr_entry(u_char *mbr, int idx, int typ, quad_t start,
+    quad_t end)
+{
+
+	if (typ == 0 || start > UINT32_MAX || end > UINT32_MAX)
+		return (EINVAL);
+
+	mbr += DOSPARTOFF + idx * DOSPARTSIZE;
+	mbr[0] = 0;
+	if (start == 1) {
+		/*
+		 * Treat the PMBR partition specially to maximize
+		 * interoperability with BIOSes.
+		 */
+		mbr[1] = mbr[3] = 0;
+		mbr[2] = 2;
+	} else
+		mbr[1] = mbr[2] = mbr[3] = 0xff;
+	mbr[4] = typ;
+	mbr[5] = mbr[6] = mbr[7] = 0xff;
+	le32enc(mbr + 8, (uint32_t)start);
+	le32enc(mbr + 12, (uint32_t)(end - start + 1));
+	return (0);
+}
+
+static int
+gpt_map_type(struct uuid *t)
+{
+	struct g_part_uuid_alias *uap;
+
+	for (uap = &gpt_uuid_alias_match[0]; uap->uuid; uap++) {
+		if (EQUUID(t, uap->uuid))
+			return (uap->mbrtype);
+	}
+	return (0);
+}
+
+/*
+ * Under Boot Camp the PMBR partition (type 0xEE) doesn't cover the
+ * whole disk anymore. Rather, it covers the GPT table and the EFI
+ * system partition only. This way the HFS+ partition and any FAT
+ * partitions can be added to the MBR without creating an overlap.
+ */
+static int
+gpt_is_bootcamp(struct g_part_gpt_table *table, const char *provname)
+{
+	uint8_t *p;
+
+	p = table->mbr + DOSPARTOFF;
+	if (p[4] != 0xee || le32dec(p + 8) != 1)
+		return (0);
+
+	p += DOSPARTSIZE;
+	if (p[4] != 0xaf)
+		return (0);
+
+	printf("GEOM: %s: enabling Boot Camp\n", provname);
+	return (1);
+}
+
+static void
+gpt_update_bootcamp(struct g_part_table *basetable)
+{
+	struct g_part_entry *baseentry;
+	struct g_part_gpt_entry *entry;
+	struct g_part_gpt_table *table;
+	int bootable, error, index, slices, typ;
+
+	table = (struct g_part_gpt_table *)basetable;
+
+	bootable = -1;
+	for (index = 0; index < NDOSPART; index++) {
+		if (table->mbr[DOSPARTOFF + DOSPARTSIZE * index])
+			bootable = index;
+	}
+
+	bzero(table->mbr + DOSPARTOFF, DOSPARTSIZE * NDOSPART);
+	slices = 0;
+	LIST_FOREACH(baseentry, &basetable->gpt_entry, gpe_entry) {
+		if (baseentry->gpe_deleted)
+			continue;
+		index = baseentry->gpe_index - 1;
+		if (index >= NDOSPART)
+			continue;
+
+		entry = (struct g_part_gpt_entry *)baseentry;
+
+		switch (index) {
+		case 0:	/* This must be the EFI system partition. */
+			if (!EQUUID(&entry->ent.ent_type, &gpt_uuid_efi))
+				goto disable;
+			error = gpt_write_mbr_entry(table->mbr, index, 0xee,
+			    1ull, entry->ent.ent_lba_end);
+			break;
+		case 1:	/* This must be the HFS+ partition. */
+			if (!EQUUID(&entry->ent.ent_type, &gpt_uuid_apple_hfs))
+				goto disable;
+			error = gpt_write_mbr_entry(table->mbr, index, 0xaf,
+			    entry->ent.ent_lba_start, entry->ent.ent_lba_end);
+			break;
+		default:
+			typ = gpt_map_type(&entry->ent.ent_type);
+			error = gpt_write_mbr_entry(table->mbr, index, typ,
+			    entry->ent.ent_lba_start, entry->ent.ent_lba_end);
+			break;
+		}
+		if (error)
+			continue;
+
+		if (index == bootable)
+			table->mbr[DOSPARTOFF + DOSPARTSIZE * index] = 0x80;
+		slices |= 1 << index;
+	}
+	if ((slices & 3) == 3)
+		return;
+
+ disable:
+	table->bootcamp = 0;
+	bzero(table->mbr + DOSPARTOFF, DOSPARTSIZE * NDOSPART);
+	gpt_write_mbr_entry(table->mbr, 0, 0xee, 1ull,
+	    MIN(table->lba[GPT_ELT_SECHDR], UINT32_MAX));
+}
+
 static struct gpt_hdr *
 gpt_read_hdr(struct g_part_gpt_table *table, struct g_consumer *cp,
     enum gpt_elt elt)
@@ -457,8 +582,9 @@ g_part_gpt_bootcode(struct g_part_table *basetable, struct g_part_parms *gpp)
 	if (codesz > 0)
 		bcopy(gpp->gpp_codeptr, table->mbr, codesz);
 
-	/* Mark the PMBR active since some BIOS require it */
-	table->mbr[DOSPARTOFF] = 0x80;		/* status */
+	/* Mark the PMBR active since some BIOS require it. */
+	if (!table->bootcamp)
+		table->mbr[DOSPARTOFF] = 0x80;		/* status */
 	return (0);
 }
 
@@ -486,15 +612,7 @@ g_part_gpt_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	last = (pp->mediasize / pp->sectorsize) - 1;
 
 	le16enc(table->mbr + DOSMAGICOFFSET, DOSMAGIC);
-	table->mbr[DOSPARTOFF + 1] = 0x01;		/* shd */
-	table->mbr[DOSPARTOFF + 2] = 0x01;		/* ssect */
-	table->mbr[DOSPARTOFF + 3] = 0x00;		/* scyl */
-	table->mbr[DOSPARTOFF + 4] = 0xee;		/* typ */
-	table->mbr[DOSPARTOFF + 5] = 0xff;		/* ehd */
-	table->mbr[DOSPARTOFF + 6] = 0xff;		/* esect */
-	table->mbr[DOSPARTOFF + 7] = 0xff;		/* ecyl */
-	le32enc(table->mbr + DOSPARTOFF + 8, 1);	/* start */
-	le32enc(table->mbr + DOSPARTOFF + 12, MIN(last, UINT32_MAX));
+	gpt_write_mbr_entry(table->mbr, 0, 0xee, 1, MIN(last, UINT32_MAX));
 
 	/* Allocate space for the header */
 	table->hdr = g_malloc(sizeof(struct gpt_hdr), M_WAITOK | M_ZERO);
@@ -802,6 +920,21 @@ g_part_gpt_read(struct g_part_table *basetable, struct g_consumer *cp)
 	}
 
 	g_free(tbl);
+
+	/*
+	 * Under Mac OS X, the MBR mirrors the first 4 GPT partitions
+	 * if (and only if) any FAT32 or FAT16 partitions have been
+	 * created. This happens irrespective of whether Boot Camp is
+	 * used/enabled, though it's generally understood to be done
+	 * to support legacy Windows under Boot Camp. We refer to this
+	 * mirroring simply as Boot Camp. We try to detect Boot Camp
+	 * so that we can update the MBR if and when GPT changes have
+	 * been made. Note that we do not enable Boot Camp if not
+	 * previously enabled because we can't assume that we're on a
+	 * Mac alongside Mac OS X.
+	 */
+	table->bootcamp = gpt_is_bootcamp(table, pp->name);
+
 	return (0);
 }
 
@@ -816,73 +949,52 @@ g_part_gpt_recover(struct g_part_table *basetable)
 }
 
 static int
-g_part_gpt_setunset(struct g_part_table *table, struct g_part_entry *baseentry,
-    const char *attrib, unsigned int set)
+g_part_gpt_setunset(struct g_part_table *basetable,
+    struct g_part_entry *baseentry, const char *attrib, unsigned int set)
 {
-	struct g_part_entry *iter;
 	struct g_part_gpt_entry *entry;
-	int changed, bootme, bootonce, bootfailed;
+	struct g_part_gpt_table *table;
+	uint64_t attr;
+	int i;
 
-	bootme = bootonce = bootfailed = 0;
+	table = (struct g_part_gpt_table *)basetable;
+	entry = (struct g_part_gpt_entry *)baseentry;
+
+	if (strcasecmp(attrib, "active") == 0) {
+		if (!table->bootcamp || baseentry->gpe_index > NDOSPART)
+			return (EINVAL);
+		for (i = 0; i < NDOSPART; i++) {
+			table->mbr[DOSPARTOFF + i * DOSPARTSIZE] =
+			    (i == baseentry->gpe_index - 1) ? 0x80 : 0;
+		}
+		return (0);
+	}
+
+	attr = 0;
 	if (strcasecmp(attrib, "bootme") == 0) {
-		bootme = 1;
+		attr |= GPT_ENT_ATTR_BOOTME;
 	} else if (strcasecmp(attrib, "bootonce") == 0) {
-		/* BOOTME is set automatically with BOOTONCE, but not unset. */
-		bootonce = 1;
+		attr |= GPT_ENT_ATTR_BOOTONCE;
 		if (set)
-			bootme = 1;
+			attr |= GPT_ENT_ATTR_BOOTME;
 	} else if (strcasecmp(attrib, "bootfailed") == 0) {
 		/*
 		 * It should only be possible to unset BOOTFAILED, but it might
 		 * be useful for test purposes to also be able to set it.
 		 */
-		bootfailed = 1;
+		attr |= GPT_ENT_ATTR_BOOTFAILED;
 	}
-	if (!bootme && !bootonce && !bootfailed)
+	if (attr == 0)
 		return (EINVAL);
 
-	LIST_FOREACH(iter, &table->gpt_entry, gpe_entry) {
-		if (iter->gpe_deleted)
-			continue;
-		if (iter != baseentry)
-			continue;
-		changed = 0;
-		entry = (struct g_part_gpt_entry *)iter;
-		if (set) {
-			if (bootme &&
-			    !(entry->ent.ent_attr & GPT_ENT_ATTR_BOOTME)) {
-				entry->ent.ent_attr |= GPT_ENT_ATTR_BOOTME;
-				changed = 1;
-			}
-			if (bootonce &&
-			    !(entry->ent.ent_attr & GPT_ENT_ATTR_BOOTONCE)) {
-				entry->ent.ent_attr |= GPT_ENT_ATTR_BOOTONCE;
-				changed = 1;
-			}
-			if (bootfailed &&
-			    !(entry->ent.ent_attr & GPT_ENT_ATTR_BOOTFAILED)) {
-				entry->ent.ent_attr |= GPT_ENT_ATTR_BOOTFAILED;
-				changed = 1;
-			}
-		} else {
-			if (bootme &&
-			    (entry->ent.ent_attr & GPT_ENT_ATTR_BOOTME)) {
-				entry->ent.ent_attr &= ~GPT_ENT_ATTR_BOOTME;
-				changed = 1;
-			}
-			if (bootonce &&
-			    (entry->ent.ent_attr & GPT_ENT_ATTR_BOOTONCE)) {
-				entry->ent.ent_attr &= ~GPT_ENT_ATTR_BOOTONCE;
-				changed = 1;
-			}
-			if (bootfailed &&
-			    (entry->ent.ent_attr & GPT_ENT_ATTR_BOOTFAILED)) {
-				entry->ent.ent_attr &= ~GPT_ENT_ATTR_BOOTFAILED;
-				changed = 1;
-			}
-		}
-		if (changed && !iter->gpe_created)
-			iter->gpe_modified = 1;
+	if (set)
+		attr = entry->ent.ent_attr | attr;
+	else
+		attr = entry->ent.ent_attr & ~attr;
+	if (attr != entry->ent.ent_attr) {
+		entry->ent.ent_attr = attr;
+		if (!baseentry->gpe_created)
+			baseentry->gpe_modified = 1;
 	}
 	return (0);
 }
@@ -923,6 +1035,10 @@ g_part_gpt_write(struct g_part_table *basetable, struct g_consumer *cp)
 	tblsz = (table->hdr->hdr_entries * table->hdr->hdr_entsz +
 	    pp->sectorsize - 1) / pp->sectorsize;
 
+	/* Reconstruct the MBR from the GPT if under Boot Camp. */
+	if (table->bootcamp)
+		gpt_update_bootcamp(basetable);
+
 	/* Write the PMBR */
 	buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
 	bcopy(table->mbr, buf, MBRSIZE);
diff --git a/sys/geom/part/g_part_vtoc8.c b/sys/geom/part/g_part_vtoc8.c
index 245d1f94d53a..0c442a736302 100644
--- a/sys/geom/part/g_part_vtoc8.c
+++ b/sys/geom/part/g_part_vtoc8.c
@@ -274,7 +274,8 @@ g_part_vtoc8_dumpto(struct g_part_table *basetable,
 	 */
 	table = (struct g_part_vtoc8_table *)basetable;
 	tag = be16dec(&table->vtoc.part[entry->gpe_index - 1].tag);
-	return ((tag == 0 || tag == VTOC_TAG_FREEBSD_SWAP) ? 1 : 0);
+	return ((tag == 0 || tag == VTOC_TAG_FREEBSD_SWAP ||
+	    tag == VTOC_TAG_SWAP) ? 1 : 0);
 }
 
 static int
diff --git a/sys/geom/raid/md_intel.c b/sys/geom/raid/md_intel.c
index 59d88358a911..2b11d3beba6b 100644
--- a/sys/geom/raid/md_intel.c
+++ b/sys/geom/raid/md_intel.c
@@ -1172,15 +1172,18 @@ g_raid_md_taste_intel(struct g_raid_md_object *md, struct g_class *mp,
 	g_access(cp, -1, 0, 0);
 	if (meta == NULL) {
 		if (g_raid_aggressive_spare) {
-			if (vendor == 0x8086) {
+			if (vendor != 0x8086) {
+				G_RAID_DEBUG(1,
+				    "Intel vendor mismatch 0x%04x != 0x8086",
+				    vendor);
+			} else if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
+				G_RAID_DEBUG(1,
+				    "Intel disk '%s' is too big.", pp->name);
+			} else {
 				G_RAID_DEBUG(1,
 				    "No Intel metadata, forcing spare.");
 				spare = 2;
 				goto search;
-			} else {
-				G_RAID_DEBUG(1,
-				    "Intel vendor mismatch 0x%04x != 0x8086",
-				    vendor);
 			}
 		}
 		return (G_RAID_MD_TASTE_FAIL);
@@ -1194,9 +1197,9 @@ g_raid_md_taste_intel(struct g_raid_md_object *md, struct g_class *mp,
 	}
 	if (meta->disk[disk_pos].sectors !=
 	    (pp->mediasize / pp->sectorsize)) {
-		G_RAID_DEBUG(1, "Intel size mismatch %u != %u",
-		    meta->disk[disk_pos].sectors,
-		    (u_int)(pp->mediasize / pp->sectorsize));
+		G_RAID_DEBUG(1, "Intel size mismatch %ju != %ju",
+		    (off_t)meta->disk[disk_pos].sectors,
+		    (off_t)(pp->mediasize / pp->sectorsize));
 		goto fail1;
 	}
 
@@ -1449,6 +1452,13 @@ g_raid_md_ctl_intel(struct g_raid_md_object *md,
 			cp->private = disk;
 			g_topology_unlock();
 
+			if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
+				gctl_error(req,
+				    "Disk '%s' is too big.", diskname);
+				error = -8;
+				break;
+			}
+
 			error = g_raid_md_get_label(cp,
 			    &pd->pd_disk_meta.serial[0], INTEL_SERIAL_LEN);
 			if (error != 0) {
@@ -1940,6 +1950,14 @@ g_raid_md_ctl_intel(struct g_raid_md_object *md,
 			pp = cp->provider;
 			g_topology_unlock();
 
+			if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
+				gctl_error(req,
+				    "Disk '%s' is too big.", diskname);
+				g_raid_kill_consumer(sc, cp);
+				error = -8;
+				break;
+			}
+
 			/* Read disk serial. */
 			error = g_raid_md_get_label(cp,
 			    &serial[0], INTEL_SERIAL_LEN);
diff --git a/sys/geom/raid/md_nvidia.c b/sys/geom/raid/md_nvidia.c
index c2b300bf598a..0cbab4b4fa32 100644
--- a/sys/geom/raid/md_nvidia.c
+++ b/sys/geom/raid/md_nvidia.c
@@ -1033,7 +1033,7 @@ g_raid_md_ctl_nvidia(struct g_raid_md_object *md,
 	char arg[16];
 	const char *verb, *volname, *levelname, *diskname;
 	int *nargs, *force;
-	off_t size, sectorsize, strip;
+	off_t size, sectorsize, strip, volsize;
 	intmax_t *sizearg, *striparg;
 	int numdisks, i, len, level, qual, update;
 	int error;
@@ -1182,7 +1182,20 @@ g_raid_md_ctl_nvidia(struct g_raid_md_object *md,
 			gctl_error(req, "Size too small.");
 			return (-13);
 		}
-		if (size > 0xffffffffffffllu * sectorsize) {
+
+		if (level == G_RAID_VOLUME_RL_RAID0 ||
+		    level == G_RAID_VOLUME_RL_CONCAT ||
+		    level == G_RAID_VOLUME_RL_SINGLE)
+			volsize = size * numdisks;
+		else if (level == G_RAID_VOLUME_RL_RAID1)
+			volsize = size;
+		else if (level == G_RAID_VOLUME_RL_RAID5)
+			volsize = size * (numdisks - 1);
+		else { /* RAID1E */
+			volsize = ((size * numdisks) / strip / 2) *
+			    strip;
+		}
+		if (volsize > 0xffffffffllu * sectorsize) {
 			gctl_error(req, "Size too big.");
 			return (-14);
 		}
@@ -1196,18 +1209,7 @@ g_raid_md_ctl_nvidia(struct g_raid_md_object *md,
 		vol->v_raid_level_qualifier = G_RAID_VOLUME_RLQ_NONE;
 		vol->v_strip_size = strip;
 		vol->v_disks_count = numdisks;
-		if (level == G_RAID_VOLUME_RL_RAID0 ||
-		    level == G_RAID_VOLUME_RL_CONCAT ||
-		    level == G_RAID_VOLUME_RL_SINGLE)
-			vol->v_mediasize = size * numdisks;
-		else if (level == G_RAID_VOLUME_RL_RAID1)
-			vol->v_mediasize = size;
-		else if (level == G_RAID_VOLUME_RL_RAID5)
-			vol->v_mediasize = size * (numdisks - 1);
-		else { /* RAID1E */
-			vol->v_mediasize = ((size * numdisks) / strip / 2) *
-			    strip;
-		}
+		vol->v_mediasize = volsize;
 		vol->v_sectorsize = sectorsize;
 		g_raid_start_volume(vol);
 
diff --git a/sys/geom/raid/md_promise.c b/sys/geom/raid/md_promise.c
index c47517451bd8..55b4b213f979 100644
--- a/sys/geom/raid/md_promise.c
+++ b/sys/geom/raid/md_promise.c
@@ -121,7 +121,8 @@ struct promise_raid_conf {
 	uint64_t	rebuild_lba64;	/* Per-volume rebuild position. */
 	uint32_t	magic_4;
 	uint32_t	magic_5;
-	uint32_t	filler3[325];
+	uint32_t	total_sectors_high;
+	uint32_t	filler3[324];
 	uint32_t	checksum;
 } __packed;
 
@@ -213,6 +214,7 @@ g_raid_md_promise_print(struct promise_raid_conf *meta)
 	printf("rebuild_lba64       %ju\n", meta->rebuild_lba64);
 	printf("magic_4             0x%08x\n", meta->magic_4);
 	printf("magic_5             0x%08x\n", meta->magic_5);
+	printf("total_sectors_high  0x%08x\n", meta->total_sectors_high);
 	printf("=================================================\n");
 }
 
@@ -867,6 +869,9 @@ g_raid_md_promise_start(struct g_raid_volume *vol)
 	vol->v_strip_size = 512 << meta->stripe_shift; //ZZZ
 	vol->v_disks_count = meta->total_disks;
 	vol->v_mediasize = (off_t)meta->total_sectors * 512; //ZZZ
+	if (meta->total_sectors_high < 256) /* If value looks sane. */
+		vol->v_mediasize |=
+		    ((off_t)meta->total_sectors_high << 32) * 512; //ZZZ
 	vol->v_sectorsize = 512; //ZZZ
 	for (i = 0; i < vol->v_disks_count; i++) {
 		sd = &vol->v_subdisks[i];
@@ -1318,6 +1323,13 @@ g_raid_md_ctl_promise(struct g_raid_md_object *md,
 			cp->private = disk;
 			g_topology_unlock();
 
+			if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
+				gctl_error(req,
+				    "Disk '%s' is too big.", diskname);
+				error = -8;
+				break;
+			}
+
 			/* Read kernel dumping information. */
 			disk->d_kd.offset = 0;
 			disk->d_kd.length = OFF_MAX;
@@ -1609,8 +1621,17 @@ g_raid_md_ctl_promise(struct g_raid_md_object *md,
 				error = -4;
 				break;
 			}
+			pp = cp->provider;
 			g_topology_unlock();
 
+			if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
+				gctl_error(req,
+				    "Disk '%s' is too big.", diskname);
+				g_raid_kill_consumer(sc, cp);
+				error = -8;
+				break;
+			}
+
 			pd = malloc(sizeof(*pd), M_MD_PROMISE, M_WAITOK | M_ZERO);
 
 			disk = g_raid_create_disk(sc);
@@ -1716,6 +1737,8 @@ g_raid_md_write_promise(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 			meta->array_width /= 2;
 		meta->array_number = vol->v_global_id;
 		meta->total_sectors = vol->v_mediasize / vol->v_sectorsize;
+		meta->total_sectors_high =
+		    (vol->v_mediasize / vol->v_sectorsize) >> 32;
 		meta->cylinders = meta->total_sectors / (255 * 63) - 1;
 		meta->heads = 254;
 		meta->sectors = 63;
diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC
index 69e8d9b04292..12bcda423567 100644
--- a/sys/i386/conf/GENERIC
+++ b/sys/i386/conf/GENERIC
@@ -147,6 +147,7 @@ device		iir		# Intel Integrated RAID
 device		ips		# IBM (Adaptec) ServeRAID
 device		mly		# Mylex AcceleRAID/eXtremeRAID
 device		twa		# 3ware 9000 series PATA/SATA RAID
+device		tws		# LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller
 
 # RAID controllers
 device		aac		# Adaptec FSA RAID
@@ -343,7 +344,8 @@ device		zyd		# ZyDAS zd1211/zd1211b wireless NICs
 
 # FireWire support
 device		firewire	# FireWire bus code
-device		sbp		# SCSI over FireWire (Requires scbus and da)
+# sbp(4) works for some systems but causes boot failure on others
+#device		sbp		# SCSI over FireWire (Requires scbus and da)
 device		fwe		# Ethernet over FireWire (non-standard!)
 device		fwip		# IP over FireWire (RFC 2734,3146)
 device		dcons		# Dumb console driver
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index e596b964d992..839dd0dae28e 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -1000,7 +1000,7 @@ freebsd4_sigreturn(td, uap)
  * MPSAFE
  */
 int
-sigreturn(td, uap)
+sys_sigreturn(td, uap)
 	struct thread *td;
 	struct sigreturn_args /* {
 		const struct __ucontext *sigcntxp;
@@ -1222,7 +1222,7 @@ void
 cpu_halt(void)
 {
 	for (;;)
-		__asm__ ("hlt");
+		halt();
 }
 
 #endif
@@ -1245,6 +1245,8 @@ cpu_idle_acpi(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_SLEEPING;
+
+	/* See comments in cpu_idle_hlt(). */
 	disable_intr();
 	if (sched_runnable())
 		enable_intr();
@@ -1263,9 +1265,22 @@ cpu_idle_hlt(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_SLEEPING;
+
 	/*
-	 * We must absolutely guarentee that hlt is the next instruction
-	 * after sti or we introduce a timing window.
+	 * Since we may be in a critical section from cpu_idle(), if
+	 * an interrupt fires during that critical section we may have
+	 * a pending preemption.  If the CPU halts, then that thread
+	 * may not execute until a later interrupt awakens the CPU.
+	 * To handle this race, check for a runnable thread after
+	 * disabling interrupts and immediately return if one is
+	 * found.  Also, we must absolutely guarentee that hlt is
+	 * the next instruction after sti.  This ensures that any
+	 * interrupt that fires after the call to disable_intr() will
+	 * immediately awaken the CPU from hlt.  Finally, please note
+	 * that on x86 this works fine because of interrupts enabled only
+	 * after the instruction following sti takes place, while IF is set
+	 * to 1 immediately, allowing hlt instruction to acknowledge the
+	 * interrupt.
 	 */
 	disable_intr();
 	if (sched_runnable())
@@ -1292,11 +1307,19 @@ cpu_idle_mwait(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_MWAIT;
-	if (!sched_runnable()) {
-		cpu_monitor(state, 0, 0);
-		if (*state == STATE_MWAIT)
-			cpu_mwait(0, MWAIT_C1);
+
+	/* See comments in cpu_idle_hlt(). */
+	disable_intr();
+	if (sched_runnable()) {
+		enable_intr();
+		*state = STATE_RUNNING;
+		return;
 	}
+	cpu_monitor(state, 0, 0);
+	if (*state == STATE_MWAIT)
+		__asm __volatile("sti; mwait" : : "a" (MWAIT_C1), "c" (0));
+	else
+		enable_intr();
 	*state = STATE_RUNNING;
 }
 
@@ -1308,6 +1331,12 @@ cpu_idle_spin(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_RUNNING;
+
+	/*
+	 * The sched_runnable() call is racy but as long as there is
+	 * a loop missing it one time will have just a little impact if any 
+	 * (and it is much better than missing the check at all).
+	 */
 	for (i = 0; i < 1000; i++) {
 		if (sched_runnable())
 			return;
diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c
index 837051649045..beffff2e1747 100644
--- a/sys/i386/i386/sys_machdep.c
+++ b/sys/i386/i386/sys_machdep.c
@@ -130,6 +130,10 @@ sysarch(td, uap)
 
 		case I386_SET_IOPERM:
 		default:
+#ifdef KTRACE
+			if (KTRPOINT(td, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_SYSCALL, 0, 0);
+#endif
 			return (ECAPMODE);
 		}
 	}
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index dddecaa807ff..5006f41be1ec 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -612,7 +612,7 @@ trap(struct trapframe *frame)
 				PCPU_GET(curpcb)->pcb_gs = 0;
 #if 0				
 				PROC_LOCK(p);
-				psignal(p, SIGBUS);
+				kern_psignal(p, SIGBUS);
 				PROC_UNLOCK(p);
 #endif				
 				goto out;
@@ -831,6 +831,11 @@ trap_pfault(frame, usermode, eva)
 			goto nogo;
 
 		map = &vm->vm_map;
+		if (!usermode && (td->td_intr_nesting_level != 0 ||
+		    PCPU_GET(curpcb)->pcb_onfault == NULL)) {
+			trap_fatal(frame, eva);
+			return (-1);
+		}
 	}
 
 	/*
diff --git a/sys/i386/ibcs2/ibcs2_ioctl.c b/sys/i386/ibcs2/ibcs2_ioctl.c
index 3a582ce2f337..83e68cd45197 100644
--- a/sys/i386/ibcs2/ibcs2_ioctl.c
+++ b/sys/i386/ibcs2/ibcs2_ioctl.c
@@ -499,12 +499,12 @@ ibcs2_ioctl(td, uap)
 
 	case IBCS2_TIOCGWINSZ:
 		uap->cmd = TIOCGWINSZ;
-		error = ioctl(td, (struct ioctl_args *)uap);
+		error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_TIOCSWINSZ:
 		uap->cmd = TIOCSWINSZ;
-		error = ioctl(td, (struct ioctl_args *)uap);
+		error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_TIOCGPGRP:
@@ -525,7 +525,7 @@ ibcs2_ioctl(td, uap)
 
 		sa.pid = 0;
 		sa.pgid = (int)uap->data;
-		error = setpgid(td, &sa);
+		error = sys_setpgid(td, &sa);
 		break;
 	    }
 
@@ -565,103 +565,103 @@ ibcs2_ioctl(td, uap)
 	case IBCS2_KDGKBMODE:        /* get keyboard translation mode */
 	        uap->cmd = KDGKBMODE;
 /* printf("ioctl KDGKBMODE = %x\n", uap->cmd);*/
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDSKBMODE:        /* set keyboard translation mode */
 	        uap->cmd = KDSKBMODE;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDMKTONE:        /* sound tone */
 	        uap->cmd = KDMKTONE;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDGETMODE:        /* get text/graphics mode */  
 	        uap->cmd = KDGETMODE;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDSETMODE:       /* set text/graphics mode */
 	        uap->cmd = KDSETMODE;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDSBORDER:       /* set ega color border */
 	        uap->cmd = KDSBORDER;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDGKBSTATE:
 	        uap->cmd = KDGKBSTATE;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDSETRAD:
 	        uap->cmd = KDSETRAD;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDENABIO:       /* enable direct I/O to ports */
 	        uap->cmd = KDENABIO;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDDISABIO:       /* disable direct I/O to ports */
 	        uap->cmd = KDDISABIO;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KIOCSOUND:       /* start sound generation */
 	        uap->cmd = KIOCSOUND;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDGKBTYPE:       /* get keyboard type */
 	        uap->cmd = KDGKBTYPE;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDGETLED:       /* get keyboard LED status */
 	        uap->cmd = KDGETLED;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_KDSETLED:       /* set keyboard LED status */
 	        uap->cmd = KDSETLED;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	    /* Xenix keyboard and display ioctl's from sys/kd.h -- type 'k' */
 	case IBCS2_GETFKEY:      /* Get function key */
 	        uap->cmd = GETFKEY;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_SETFKEY:      /* Set function key */
 	        uap->cmd = SETFKEY;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_GIO_SCRNMAP:      /* Get screen output map table */
 	        uap->cmd = GIO_SCRNMAP;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_PIO_SCRNMAP:      /* Set screen output map table */
 	        uap->cmd = PIO_SCRNMAP;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_GIO_KEYMAP:      /* Get keyboard map table */
 	        uap->cmd = OGIO_KEYMAP;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	case IBCS2_PIO_KEYMAP:      /* Set keyboard map table */
 	        uap->cmd = OPIO_KEYMAP;
-	        error = ioctl(td, (struct ioctl_args *)uap);
+	        error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	    /* socksys */
@@ -672,7 +672,7 @@ ibcs2_ioctl(td, uap)
 	case IBCS2_FIONREAD:
 	case IBCS2_I_NREAD:     /* STREAMS */
 	        uap->cmd = FIONREAD;
-		error = ioctl(td, (struct ioctl_args *)uap);
+		error = sys_ioctl(td, (struct ioctl_args *)uap);
 		break;
 
 	default:
diff --git a/sys/i386/ibcs2/ibcs2_ipc.c b/sys/i386/ibcs2/ibcs2_ipc.c
index c76c0e4d7ed7..4bf3dd1de389 100644
--- a/sys/i386/ibcs2/ibcs2_ipc.c
+++ b/sys/i386/ibcs2/ibcs2_ipc.c
@@ -118,7 +118,7 @@ ibcs2_msgget(struct thread *td, void *v)
 
 	ap.key = uap->key;
 	ap.msgflg = uap->msgflg;
-	return msgget(td, &ap);
+	return sys_msgget(td, &ap);
 }
 
 struct ibcs2_msgctl_args {
@@ -176,7 +176,7 @@ ibcs2_msgrcv(struct thread *td, void *v)
 	ap.msgsz = uap->msgsz;
 	ap.msgtyp = uap->msgtyp;
 	ap.msgflg = uap->msgflg;
-	return (msgrcv(td, &ap));
+	return (sys_msgrcv(td, &ap));
 }
 
 struct ibcs2_msgsnd_args {
@@ -197,7 +197,7 @@ ibcs2_msgsnd(struct thread *td, void *v)
 	ap.msgp = uap->msgp;
 	ap.msgsz = uap->msgsz;
 	ap.msgflg = uap->msgflg;
-	return (msgsnd(td, &ap));
+	return (sys_msgsnd(td, &ap));
 }
 
 int
@@ -363,7 +363,7 @@ ibcs2_semget(struct thread *td, void *v)
 	ap.key = uap->key;
 	ap.nsems = uap->nsems;
 	ap.semflg = uap->semflg;
-	return (semget(td, &ap));
+	return (sys_semget(td, &ap));
 }
 
 struct ibcs2_semop_args {
@@ -382,7 +382,7 @@ ibcs2_semop(struct thread *td, void *v)
 	ap.semid = uap->semid;
 	ap.sops = uap->sops;
 	ap.nsops = uap->nsops;
-	return (semop(td, &ap));
+	return (sys_semop(td, &ap));
 }
 
 int
@@ -459,7 +459,7 @@ ibcs2_shmat(struct thread *td, void *v)
 	ap.shmid = uap->shmid;
 	ap.shmaddr = uap->shmaddr;
 	ap.shmflg = uap->shmflg;
-	return (shmat(td, &ap));
+	return (sys_shmat(td, &ap));
 }
 
 struct ibcs2_shmctl_args {
@@ -514,7 +514,7 @@ ibcs2_shmdt(struct thread *td, void *v)
 	struct shmdt_args ap;
 
 	ap.shmaddr = uap->shmaddr;
-	return (shmdt(td, &ap));
+	return (sys_shmdt(td, &ap));
 }
 
 struct ibcs2_shmget_args {
@@ -533,7 +533,7 @@ ibcs2_shmget(struct thread *td, void *v)
 	ap.key = uap->key;
 	ap.size = uap->size;
 	ap.shmflg = uap->shmflg;
-	return (shmget(td, &ap));
+	return (sys_shmget(td, &ap));
 }
 
 int
diff --git a/sys/i386/ibcs2/ibcs2_isc_sysent.c b/sys/i386/ibcs2/ibcs2_isc_sysent.c
index f4a95f102880..fc5b1baa8c2a 100644
--- a/sys/i386/ibcs2/ibcs2_isc_sysent.c
+++ b/sys/i386/ibcs2/ibcs2_isc_sysent.c
@@ -18,25 +18,25 @@
 
 /* The casts are bogus but will do for now. */
 struct sysent isc_sysent[] = {
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 0 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 1 = isc_setostype */
-	{ AS(ibcs2_rename_args), (sy_call_t *)ibcs2_rename, AUE_RENAME, NULL, 0, 0, 0 },	/* 2 = ibcs2_rename */
-	{ AS(ibcs2_sigaction_args), (sy_call_t *)ibcs2_sigaction, AUE_NULL, NULL, 0, 0, 0 },	/* 3 = ibcs2_sigaction */
-	{ AS(ibcs2_sigprocmask_args), (sy_call_t *)ibcs2_sigprocmask, AUE_NULL, NULL, 0, 0, 0 },	/* 4 = ibcs2_sigprocmask */
-	{ AS(ibcs2_sigpending_args), (sy_call_t *)ibcs2_sigpending, AUE_NULL, NULL, 0, 0, 0 },	/* 5 = ibcs2_sigpending */
-	{ AS(getgroups_args), (sy_call_t *)getgroups, AUE_GETGROUPS, NULL, 0, 0, 0 },	/* 6 = getgroups */
-	{ AS(setgroups_args), (sy_call_t *)setgroups, AUE_SETGROUPS, NULL, 0, 0, 0 },	/* 7 = setgroups */
-	{ AS(ibcs2_pathconf_args), (sy_call_t *)ibcs2_pathconf, AUE_PATHCONF, NULL, 0, 0, 0 },	/* 8 = ibcs2_pathconf */
-	{ AS(ibcs2_fpathconf_args), (sy_call_t *)ibcs2_fpathconf, AUE_FPATHCONF, NULL, 0, 0, 0 },	/* 9 = ibcs2_fpathconf */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 10 = nosys */
-	{ AS(ibcs2_wait_args), (sy_call_t *)ibcs2_wait, AUE_WAIT4, NULL, 0, 0, 0 },	/* 11 = ibcs2_wait */
-	{ 0, (sy_call_t *)setsid, AUE_SETSID, NULL, 0, 0, 0 },		/* 12 = setsid */
-	{ 0, (sy_call_t *)getpid, AUE_GETPID, NULL, 0, 0, 0 },		/* 13 = getpid */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 14 = isc_adduser */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 15 = isc_setuser */
-	{ AS(ibcs2_sysconf_args), (sy_call_t *)ibcs2_sysconf, AUE_NULL, NULL, 0, 0, 0 },	/* 16 = ibcs2_sysconf */
-	{ AS(ibcs2_sigsuspend_args), (sy_call_t *)ibcs2_sigsuspend, AUE_NULL, NULL, 0, 0, 0 },	/* 17 = ibcs2_sigsuspend */
-	{ AS(ibcs2_symlink_args), (sy_call_t *)ibcs2_symlink, AUE_SYMLINK, NULL, 0, 0, 0 },	/* 18 = ibcs2_symlink */
-	{ AS(ibcs2_readlink_args), (sy_call_t *)ibcs2_readlink, AUE_READLINK, NULL, 0, 0, 0 },	/* 19 = ibcs2_readlink */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 20 = isc_getmajor */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 0 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 1 = isc_setostype */
+	{ AS(ibcs2_rename_args), (sy_call_t *)ibcs2_rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 2 = ibcs2_rename */
+	{ AS(ibcs2_sigaction_args), (sy_call_t *)ibcs2_sigaction, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = ibcs2_sigaction */
+	{ AS(ibcs2_sigprocmask_args), (sy_call_t *)ibcs2_sigprocmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = ibcs2_sigprocmask */
+	{ AS(ibcs2_sigpending_args), (sy_call_t *)ibcs2_sigpending, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = ibcs2_sigpending */
+	{ AS(getgroups_args), (sy_call_t *)sys_getgroups, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = getgroups */
+	{ AS(setgroups_args), (sy_call_t *)sys_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = setgroups */
+	{ AS(ibcs2_pathconf_args), (sy_call_t *)ibcs2_pathconf, AUE_PATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 8 = ibcs2_pathconf */
+	{ AS(ibcs2_fpathconf_args), (sy_call_t *)ibcs2_fpathconf, AUE_FPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = ibcs2_fpathconf */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 10 = nosys */
+	{ AS(ibcs2_wait_args), (sy_call_t *)ibcs2_wait, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 11 = ibcs2_wait */
+	{ 0, (sy_call_t *)sys_setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = setsid */
+	{ 0, (sy_call_t *)sys_getpid, AUE_GETPID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 13 = getpid */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 14 = isc_adduser */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 15 = isc_setuser */
+	{ AS(ibcs2_sysconf_args), (sy_call_t *)ibcs2_sysconf, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = ibcs2_sysconf */
+	{ AS(ibcs2_sigsuspend_args), (sy_call_t *)ibcs2_sigsuspend, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 17 = ibcs2_sigsuspend */
+	{ AS(ibcs2_symlink_args), (sy_call_t *)ibcs2_symlink, AUE_SYMLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 18 = ibcs2_symlink */
+	{ AS(ibcs2_readlink_args), (sy_call_t *)ibcs2_readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 19 = ibcs2_readlink */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 20 = isc_getmajor */
 };
diff --git a/sys/i386/ibcs2/ibcs2_misc.c b/sys/i386/ibcs2/ibcs2_misc.c
index a08fdf3914d1..b72d25241d1e 100644
--- a/sys/i386/ibcs2/ibcs2_misc.c
+++ b/sys/i386/ibcs2/ibcs2_misc.c
@@ -240,7 +240,7 @@ ibcs2_umount(td, uap)
 
 	um.path = uap->name;
 	um.flags = 0;
-	return unmount(td, &um);
+	return sys_unmount(td, &um);
 }
 
 int
@@ -497,7 +497,7 @@ ibcs2_read(td, uap)
 	if ((error = getvnode(td->td_proc->p_fd, uap->fd,
 	    CAP_READ | CAP_SEEK, &fp)) != 0) {
 		if (error == EINVAL)
-			return read(td, (struct read_args *)uap);
+			return sys_read(td, (struct read_args *)uap);
 		else
 			return error;
 	}
@@ -510,7 +510,7 @@ ibcs2_read(td, uap)
 	if (vp->v_type != VDIR) {
 		VFS_UNLOCK_GIANT(vfslocked);
 		fdrop(fp, td);
-		return read(td, (struct read_args *)uap);
+		return sys_read(td, (struct read_args *)uap);
 	}
 
 	off = fp->f_offset;
@@ -731,7 +731,7 @@ ibcs2_setuid(td, uap)
 	struct setuid_args sa;
 
 	sa.uid = (uid_t)uap->uid;
-	return setuid(td, &sa);
+	return sys_setuid(td, &sa);
 }
 
 int
@@ -742,7 +742,7 @@ ibcs2_setgid(td, uap)
 	struct setgid_args sa;
 
 	sa.gid = (gid_t)uap->gid;
-	return setgid(td, &sa);
+	return sys_setgid(td, &sa);
 }
 
 int
@@ -782,7 +782,7 @@ ibcs2_fpathconf(td, uap)
 	struct ibcs2_fpathconf_args *uap;
 {
 	uap->name++;	/* iBCS2 _PC_* defines are offset by one */
-        return fpathconf(td, (struct fpathconf_args *)uap);
+        return sys_fpathconf(td, (struct fpathconf_args *)uap);
 }
 
 int
@@ -961,7 +961,7 @@ ibcs2_nice(td, uap)
 	sa.which = PRIO_PROCESS;
 	sa.who = 0;
 	sa.prio = td->td_proc->p_nice + uap->incr;
-	if ((error = setpriority(td, &sa)) != 0)
+	if ((error = sys_setpriority(td, &sa)) != 0)
 		return EPERM;
 	td->td_retval[0] = td->td_proc->p_nice;
 	return 0;
@@ -990,7 +990,7 @@ ibcs2_pgrpsys(td, uap)
 
 		sa.pid = 0;
 		sa.pgid = 0;
-		setpgid(td, &sa);
+		sys_setpgid(td, &sa);
 		PROC_LOCK(p);
 		td->td_retval[0] = p->p_pgrp->pg_id;
 		PROC_UNLOCK(p);
@@ -1003,11 +1003,11 @@ ibcs2_pgrpsys(td, uap)
 
 		sa.pid = uap->pid;
 		sa.pgid = uap->pgid;
-		return setpgid(td, &sa);
+		return sys_setpgid(td, &sa);
 	    }
 
 	case 3:			/* setsid */
-		return setsid(td, NULL);
+		return sys_setsid(td, NULL);
 
 	default:
 		return EINVAL;
@@ -1082,11 +1082,11 @@ ibcs2_uadmin(td, uap)
 		case SCO_AD_PWRDOWN:
 		case SCO_AD_PWRNAP:
 			r.opt = RB_HALT;
-			return (reboot(td, &r));
+			return (sys_reboot(td, &r));
 		case SCO_AD_BOOT:
 		case SCO_AD_IBOOT:
 			r.opt = RB_AUTOBOOT;
-			return (reboot(td, &r));
+			return (sys_reboot(td, &r));
 		}
 		return EINVAL;
 	case SCO_A_REMOUNT:
diff --git a/sys/i386/ibcs2/ibcs2_other.c b/sys/i386/ibcs2/ibcs2_other.c
index 9013af423caf..f688661ded5a 100644
--- a/sys/i386/ibcs2/ibcs2_other.c
+++ b/sys/i386/ibcs2/ibcs2_other.c
@@ -75,7 +75,7 @@ ibcs2_lseek(struct thread *td, register struct ibcs2_lseek_args *uap)
 	largs.fd = uap->fd;
 	largs.offset = uap->offset;
 	largs.whence = uap->whence;
-	error = lseek(td, &largs);
+	error = sys_lseek(td, &largs);
 	return (error);
 }
 
@@ -95,7 +95,7 @@ spx_open(struct thread *td)
 	sock.domain = AF_UNIX;
 	sock.type = SOCK_STREAM;
 	sock.protocol = 0;
-	error = socket(td, &sock);
+	error = sys_socket(td, &sock);
 	if (error)
 		return error;
 	fd = td->td_retval[0];
diff --git a/sys/i386/ibcs2/ibcs2_signal.c b/sys/i386/ibcs2/ibcs2_signal.c
index b2f339338fc6..6776df19e29e 100644
--- a/sys/i386/ibcs2/ibcs2_signal.c
+++ b/sys/i386/ibcs2/ibcs2_signal.c
@@ -437,5 +437,5 @@ ibcs2_kill(td, uap)
 		return (EINVAL);
 	ka.pid = uap->pid;
 	ka.signum = ibcs2_to_bsd_sig[_SIG_IDX(uap->signo)];
-	return kill(td, &ka);
+	return sys_kill(td, &ka);
 }
diff --git a/sys/i386/ibcs2/ibcs2_socksys.c b/sys/i386/ibcs2/ibcs2_socksys.c
index eb0559e0eed3..ce96fc925e99 100644
--- a/sys/i386/ibcs2/ibcs2_socksys.c
+++ b/sys/i386/ibcs2/ibcs2_socksys.c
@@ -81,55 +81,55 @@ ibcs2_socksys(td, uap)
 	passargs = (void *)(realargs + 1);
 	switch (realargs[0]) {
 	case SOCKSYS_ACCEPT:
-		return accept(td, passargs);
+		return sys_accept(td, passargs);
 	case SOCKSYS_BIND:
-		return bind(td, passargs);
+		return sys_bind(td, passargs);
 	case SOCKSYS_CONNECT:
-		return connect(td, passargs);
+		return sys_connect(td, passargs);
 	case SOCKSYS_GETPEERNAME:
-		return getpeername(td, passargs);
+		return sys_getpeername(td, passargs);
 	case SOCKSYS_GETSOCKNAME:
-		return getsockname(td, passargs);
+		return sys_getsockname(td, passargs);
 	case SOCKSYS_GETSOCKOPT:
-		return getsockopt(td, passargs);
+		return sys_getsockopt(td, passargs);
 	case SOCKSYS_LISTEN:
-		return listen(td, passargs);
+		return sys_listen(td, passargs);
 	case SOCKSYS_RECV:
 		realargs[5] = realargs[6] = 0;
 		/* FALLTHROUGH */
 	case SOCKSYS_RECVFROM:
-		return recvfrom(td, passargs);
+		return sys_recvfrom(td, passargs);
 	case SOCKSYS_SEND:
 		realargs[5] = realargs[6] = 0;
 		/* FALLTHROUGH */
 	case SOCKSYS_SENDTO:
-		return sendto(td, passargs);
+		return sys_sendto(td, passargs);
 	case SOCKSYS_SETSOCKOPT:
-		return setsockopt(td, passargs);
+		return sys_setsockopt(td, passargs);
 	case SOCKSYS_SHUTDOWN:
-		return shutdown(td, passargs);
+		return sys_shutdown(td, passargs);
 	case SOCKSYS_SOCKET:
-		return socket(td, passargs);
+		return sys_socket(td, passargs);
 	case SOCKSYS_SELECT:
-		return select(td, passargs);
+		return sys_select(td, passargs);
 	case SOCKSYS_GETIPDOMAIN:
 		return ibcs2_getipdomainname(td, passargs);
 	case SOCKSYS_SETIPDOMAIN:
 		return ibcs2_setipdomainname(td, passargs);
 	case SOCKSYS_ADJTIME:
-		return adjtime(td, passargs);
+		return sys_adjtime(td, passargs);
 	case SOCKSYS_SETREUID:
-		return setreuid(td, passargs);
+		return sys_setreuid(td, passargs);
 	case SOCKSYS_SETREGID:
-		return setregid(td, passargs);
+		return sys_setregid(td, passargs);
 	case SOCKSYS_GETTIME:
-		return gettimeofday(td, passargs);
+		return sys_gettimeofday(td, passargs);
 	case SOCKSYS_SETTIME:
-		return settimeofday(td, passargs);
+		return sys_settimeofday(td, passargs);
 	case SOCKSYS_GETITIMER:
-		return getitimer(td, passargs);
+		return sys_getitimer(td, passargs);
 	case SOCKSYS_SETITIMER:
-		return setitimer(td, passargs);
+		return sys_setitimer(td, passargs);
 
 	default:
 		printf("socksys unknown %08x %08x %08x %08x %08x %08x %08x\n",
diff --git a/sys/i386/ibcs2/ibcs2_sysent.c b/sys/i386/ibcs2/ibcs2_sysent.c
index 6604eea4b958..1fb490597ced 100644
--- a/sys/i386/ibcs2/ibcs2_sysent.c
+++ b/sys/i386/ibcs2/ibcs2_sysent.c
@@ -18,15 +18,15 @@
 /* The casts are bogus but will do for now. */
 struct sysent ibcs2_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 0 = syscall */
-	{ AS(sys_exit_args), (sy_call_t *)sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
-	{ 0, (sy_call_t *)fork, AUE_FORK, NULL, 0, 0, 0, SY_THR_STATIC },		/* 2 = fork */
+	{ AS(sys_exit_args), (sy_call_t *)sys_sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
+	{ 0, (sy_call_t *)sys_fork, AUE_FORK, NULL, 0, 0, 0, SY_THR_STATIC },		/* 2 = fork */
 	{ AS(ibcs2_read_args), (sy_call_t *)ibcs2_read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = ibcs2_read */
-	{ AS(write_args), (sy_call_t *)write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
+	{ AS(write_args), (sy_call_t *)sys_write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
 	{ AS(ibcs2_open_args), (sy_call_t *)ibcs2_open, AUE_OPEN_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = ibcs2_open */
-	{ AS(close_args), (sy_call_t *)close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
+	{ AS(close_args), (sy_call_t *)sys_close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
 	{ AS(ibcs2_wait_args), (sy_call_t *)ibcs2_wait, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = ibcs2_wait */
 	{ AS(ibcs2_creat_args), (sy_call_t *)ibcs2_creat, AUE_CREAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 8 = ibcs2_creat */
-	{ AS(link_args), (sy_call_t *)link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
+	{ AS(link_args), (sy_call_t *)sys_link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
 	{ AS(ibcs2_unlink_args), (sy_call_t *)ibcs2_unlink, AUE_UNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = ibcs2_unlink */
 	{ AS(ibcs2_execv_args), (sy_call_t *)ibcs2_execv, AUE_EXECVE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 11 = ibcs2_execv */
 	{ AS(ibcs2_chdir_args), (sy_call_t *)ibcs2_chdir, AUE_CHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = ibcs2_chdir */
@@ -34,16 +34,16 @@ struct sysent ibcs2_sysent[] = {
 	{ AS(ibcs2_mknod_args), (sy_call_t *)ibcs2_mknod, AUE_MKNOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 14 = ibcs2_mknod */
 	{ AS(ibcs2_chmod_args), (sy_call_t *)ibcs2_chmod, AUE_CHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = ibcs2_chmod */
 	{ AS(ibcs2_chown_args), (sy_call_t *)ibcs2_chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = ibcs2_chown */
-	{ AS(obreak_args), (sy_call_t *)obreak, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 17 = obreak */
+	{ AS(obreak_args), (sy_call_t *)sys_obreak, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 17 = obreak */
 	{ AS(ibcs2_stat_args), (sy_call_t *)ibcs2_stat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 18 = ibcs2_stat */
 	{ AS(ibcs2_lseek_args), (sy_call_t *)ibcs2_lseek, AUE_LSEEK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 19 = ibcs2_lseek */
-	{ 0, (sy_call_t *)getpid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 20 = getpid */
+	{ 0, (sy_call_t *)sys_getpid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 20 = getpid */
 	{ AS(ibcs2_mount_args), (sy_call_t *)ibcs2_mount, AUE_MOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 21 = ibcs2_mount */
 	{ AS(ibcs2_umount_args), (sy_call_t *)ibcs2_umount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 22 = ibcs2_umount */
 	{ AS(ibcs2_setuid_args), (sy_call_t *)ibcs2_setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 23 = ibcs2_setuid */
-	{ 0, (sy_call_t *)getuid, AUE_GETUID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 24 = getuid */
+	{ 0, (sy_call_t *)sys_getuid, AUE_GETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 24 = getuid */
 	{ AS(ibcs2_stime_args), (sy_call_t *)ibcs2_stime, AUE_SETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 25 = ibcs2_stime */
-	{ AS(ptrace_args), (sy_call_t *)ptrace, AUE_PTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 26 = ptrace */
+	{ AS(ptrace_args), (sy_call_t *)sys_ptrace, AUE_PTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 26 = ptrace */
 	{ AS(ibcs2_alarm_args), (sy_call_t *)ibcs2_alarm, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 27 = ibcs2_alarm */
 	{ AS(ibcs2_fstat_args), (sy_call_t *)ibcs2_fstat, AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 28 = ibcs2_fstat */
 	{ 0, (sy_call_t *)ibcs2_pause, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 29 = ibcs2_pause */
@@ -53,18 +53,18 @@ struct sysent ibcs2_sysent[] = {
 	{ AS(ibcs2_access_args), (sy_call_t *)ibcs2_access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = ibcs2_access */
 	{ AS(ibcs2_nice_args), (sy_call_t *)ibcs2_nice, AUE_NICE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = ibcs2_nice */
 	{ AS(ibcs2_statfs_args), (sy_call_t *)ibcs2_statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 35 = ibcs2_statfs */
-	{ 0, (sy_call_t *)sync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
+	{ 0, (sy_call_t *)sys_sync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
 	{ AS(ibcs2_kill_args), (sy_call_t *)ibcs2_kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = ibcs2_kill */
 	{ AS(ibcs2_fstatfs_args), (sy_call_t *)ibcs2_fstatfs, AUE_FSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 38 = ibcs2_fstatfs */
 	{ AS(ibcs2_pgrpsys_args), (sy_call_t *)ibcs2_pgrpsys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = ibcs2_pgrpsys */
 	{ AS(ibcs2_xenix_args), (sy_call_t *)ibcs2_xenix, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 40 = ibcs2_xenix */
-	{ AS(dup_args), (sy_call_t *)dup, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
-	{ 0, (sy_call_t *)pipe, AUE_PIPE, NULL, 0, 0, 0, SY_THR_STATIC },		/* 42 = pipe */
+	{ AS(dup_args), (sy_call_t *)sys_dup, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
+	{ 0, (sy_call_t *)sys_pipe, AUE_PIPE, NULL, 0, 0, 0, SY_THR_STATIC },		/* 42 = pipe */
 	{ AS(ibcs2_times_args), (sy_call_t *)ibcs2_times, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = ibcs2_times */
-	{ AS(profil_args), (sy_call_t *)profil, AUE_PROFILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 44 = profil */
+	{ AS(profil_args), (sy_call_t *)sys_profil, AUE_PROFILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 44 = profil */
 	{ AS(ibcs2_plock_args), (sy_call_t *)ibcs2_plock, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 45 = ibcs2_plock */
 	{ AS(ibcs2_setgid_args), (sy_call_t *)ibcs2_setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 46 = ibcs2_setgid */
-	{ 0, (sy_call_t *)getgid, AUE_GETGID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 47 = getgid */
+	{ 0, (sy_call_t *)sys_getgid, AUE_GETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 47 = getgid */
 	{ AS(ibcs2_sigsys_args), (sy_call_t *)ibcs2_sigsys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 48 = ibcs2_sigsys */
 	{ AS(ibcs2_msgsys_args), (sy_call_t *)ibcs2_msgsys, AUE_MSGSYS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 49 = ibcs2_msgsys */
 	{ AS(ibcs2_sysi86_args), (sy_call_t *)ibcs2_sysi86, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = ibcs2_sysi86 */
@@ -75,10 +75,10 @@ struct sysent ibcs2_sysent[] = {
 	{ AS(ibcs2_uadmin_args), (sy_call_t *)ibcs2_uadmin, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = ibcs2_uadmin */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 56 = nosys */
 	{ AS(ibcs2_utssys_args), (sy_call_t *)ibcs2_utssys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = ibcs2_utssys */
-	{ AS(fsync_args), (sy_call_t *)fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = fsync */
+	{ AS(fsync_args), (sy_call_t *)sys_fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = fsync */
 	{ AS(ibcs2_execve_args), (sy_call_t *)ibcs2_execve, AUE_EXECVE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = ibcs2_execve */
-	{ AS(umask_args), (sy_call_t *)umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
-	{ AS(chroot_args), (sy_call_t *)chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
+	{ AS(umask_args), (sy_call_t *)sys_umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
+	{ AS(chroot_args), (sy_call_t *)sys_chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
 	{ AS(ibcs2_fcntl_args), (sy_call_t *)ibcs2_fcntl, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 62 = ibcs2_fcntl */
 	{ AS(ibcs2_ulimit_args), (sy_call_t *)ibcs2_ulimit, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = ibcs2_ulimit */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 64 = reserved for unix/pc */
@@ -104,7 +104,7 @@ struct sysent ibcs2_sysent[] = {
 	{ AS(ibcs2_sysfs_args), (sy_call_t *)ibcs2_sysfs, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 84 = ibcs2_sysfs */
 	{ AS(ibcs2_getmsg_args), (sy_call_t *)ibcs2_getmsg, AUE_GETMSG, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = ibcs2_getmsg */
 	{ AS(ibcs2_putmsg_args), (sy_call_t *)ibcs2_putmsg, AUE_PUTMSG, NULL, 0, 0, 0, SY_THR_STATIC },	/* 86 = ibcs2_putmsg */
-	{ AS(poll_args), (sy_call_t *)poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = poll */
+	{ AS(poll_args), (sy_call_t *)sys_poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = poll */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 88 = nosys */
 	{ AS(ibcs2_secure_args), (sy_call_t *)ibcs2_secure, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = ibcs2_secure */
 	{ AS(ibcs2_symlink_args), (sy_call_t *)ibcs2_symlink, AUE_SYMLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = ibcs2_symlink */
@@ -120,7 +120,7 @@ struct sysent ibcs2_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 100 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 101 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 102 = nosys */
-	{ AS(sigreturn_args), (sy_call_t *)sigreturn, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 103 = sigreturn */
+	{ AS(sigreturn_args), (sy_call_t *)sys_sigreturn, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 103 = sigreturn */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 104 = nosys */
 	{ 0, (sy_call_t *)ibcs2_isc, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 105 = ibcs2_isc */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 106 = nosys */
diff --git a/sys/i386/ibcs2/ibcs2_xenix.c b/sys/i386/ibcs2/ibcs2_xenix.c
index 5b418c433011..74abbb1ed078 100644
--- a/sys/i386/ibcs2/ibcs2_xenix.c
+++ b/sys/i386/ibcs2/ibcs2_xenix.c
@@ -103,7 +103,7 @@ xenix_chsize(td, uap)
 	DPRINTF(("IBCS2: 'xenix chsize'\n"));
 	sa.fd = uap->fd;
 	sa.length = uap->size;
-	return ftruncate(td, &sa);
+	return sys_ftruncate(td, &sa);
 }
 
 
diff --git a/sys/i386/ibcs2/ibcs2_xenix.h b/sys/i386/ibcs2/ibcs2_xenix.h
index f2a3d6608c74..021772476961 100644
--- a/sys/i386/ibcs2/ibcs2_xenix.h
+++ b/sys/i386/ibcs2/ibcs2_xenix.h
@@ -129,6 +129,12 @@ int	xenix_utsname(struct thread *, struct xenix_utsname_args *);
 
 #endif /* COMPAT_FREEBSD6 */
 
+
+#ifdef COMPAT_FREEBSD7
+
+
+#endif /* COMPAT_FREEBSD7 */
+
 #define	IBCS2_XENIX_AUE_xenix_rdchk	AUE_NULL
 #define	IBCS2_XENIX_AUE_xenix_chsize	AUE_FTRUNCATE
 #define	IBCS2_XENIX_AUE_xenix_ftime	AUE_NULL
diff --git a/sys/i386/ibcs2/ibcs2_xenix_sysent.c b/sys/i386/ibcs2/ibcs2_xenix_sysent.c
index e06feabbbb91..30fddc7936ef 100644
--- a/sys/i386/ibcs2/ibcs2_xenix_sysent.c
+++ b/sys/i386/ibcs2/ibcs2_xenix_sysent.c
@@ -17,61 +17,61 @@
 
 /* The casts are bogus but will do for now. */
 struct sysent xenix_sysent[] = {
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 0 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 1 = xenix_xlocking */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 2 = xenix_creatsem */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 3 = xenix_opensem */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 4 = xenix_sigsem */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 5 = xenix_waitsem */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 6 = xenix_nbwaitsem */
-	{ AS(xenix_rdchk_args), (sy_call_t *)xenix_rdchk, AUE_NULL, NULL, 0, 0, 0 },	/* 7 = xenix_rdchk */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 8 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 9 = nosys */
-	{ AS(xenix_chsize_args), (sy_call_t *)xenix_chsize, AUE_FTRUNCATE, NULL, 0, 0, 0 },	/* 10 = xenix_chsize */
-	{ AS(xenix_ftime_args), (sy_call_t *)xenix_ftime, AUE_NULL, NULL, 0, 0, 0 },	/* 11 = xenix_ftime */
-	{ AS(xenix_nap_args), (sy_call_t *)xenix_nap, AUE_NULL, NULL, 0, 0, 0 },	/* 12 = xenix_nap */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 13 = xenix_sdget */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 14 = xenix_sdfree */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 15 = xenix_sdenter */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 16 = xenix_sdleave */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 17 = xenix_sdgetv */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 18 = xenix_sdwaitv */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 19 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 20 = nosys */
-	{ 0, (sy_call_t *)xenix_scoinfo, AUE_NULL, NULL, 0, 0, 0 },	/* 21 = xenix_scoinfo */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 22 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 23 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 24 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 25 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 26 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 27 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 28 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 29 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 30 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 31 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 32 = xenix_proctl */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 33 = xenix_execseg */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 34 = xenix_unexecseg */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 35 = nosys */
-	{ AS(select_args), (sy_call_t *)select, AUE_SELECT, NULL, 0, 0, 0 },	/* 36 = select */
-	{ AS(xenix_eaccess_args), (sy_call_t *)xenix_eaccess, AUE_EACCESS, NULL, 0, 0, 0 },	/* 37 = xenix_eaccess */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 38 = xenix_paccess */
-	{ AS(ibcs2_sigaction_args), (sy_call_t *)ibcs2_sigaction, AUE_NULL, NULL, 0, 0, 0 },	/* 39 = ibcs2_sigaction */
-	{ AS(ibcs2_sigprocmask_args), (sy_call_t *)ibcs2_sigprocmask, AUE_NULL, NULL, 0, 0, 0 },	/* 40 = ibcs2_sigprocmask */
-	{ AS(ibcs2_sigpending_args), (sy_call_t *)ibcs2_sigpending, AUE_NULL, NULL, 0, 0, 0 },	/* 41 = ibcs2_sigpending */
-	{ AS(ibcs2_sigsuspend_args), (sy_call_t *)ibcs2_sigsuspend, AUE_NULL, NULL, 0, 0, 0 },	/* 42 = ibcs2_sigsuspend */
-	{ AS(ibcs2_getgroups_args), (sy_call_t *)ibcs2_getgroups, AUE_GETGROUPS, NULL, 0, 0, 0 },	/* 43 = ibcs2_getgroups */
-	{ AS(ibcs2_setgroups_args), (sy_call_t *)ibcs2_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0 },	/* 44 = ibcs2_setgroups */
-	{ AS(ibcs2_sysconf_args), (sy_call_t *)ibcs2_sysconf, AUE_NULL, NULL, 0, 0, 0 },	/* 45 = ibcs2_sysconf */
-	{ AS(ibcs2_pathconf_args), (sy_call_t *)ibcs2_pathconf, AUE_PATHCONF, NULL, 0, 0, 0 },	/* 46 = ibcs2_pathconf */
-	{ AS(ibcs2_fpathconf_args), (sy_call_t *)ibcs2_fpathconf, AUE_FPATHCONF, NULL, 0, 0, 0 },	/* 47 = ibcs2_fpathconf */
-	{ AS(ibcs2_rename_args), (sy_call_t *)ibcs2_rename, AUE_RENAME, NULL, 0, 0, 0 },	/* 48 = ibcs2_rename */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 49 = nosys */
-	{ AS(xenix_utsname_args), (sy_call_t *)xenix_utsname, AUE_NULL, NULL, 0, 0, 0 },	/* 50 = xenix_utsname */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 51 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 52 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 53 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0 },			/* 54 = nosys */
-	{ AS(getitimer_args), (sy_call_t *)getitimer, AUE_GETITIMER, NULL, 0, 0, 0 },	/* 55 = getitimer */
-	{ AS(setitimer_args), (sy_call_t *)setitimer, AUE_SETITIMER, NULL, 0, 0, 0 },	/* 56 = setitimer */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 0 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 1 = xenix_xlocking */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 2 = xenix_creatsem */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 3 = xenix_opensem */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 4 = xenix_sigsem */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 5 = xenix_waitsem */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 6 = xenix_nbwaitsem */
+	{ AS(xenix_rdchk_args), (sy_call_t *)xenix_rdchk, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = xenix_rdchk */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 8 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 9 = nosys */
+	{ AS(xenix_chsize_args), (sy_call_t *)xenix_chsize, AUE_FTRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = xenix_chsize */
+	{ AS(xenix_ftime_args), (sy_call_t *)xenix_ftime, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 11 = xenix_ftime */
+	{ AS(xenix_nap_args), (sy_call_t *)xenix_nap, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = xenix_nap */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 13 = xenix_sdget */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 14 = xenix_sdfree */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 15 = xenix_sdenter */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 16 = xenix_sdleave */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 17 = xenix_sdgetv */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 18 = xenix_sdwaitv */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 19 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 20 = nosys */
+	{ 0, (sy_call_t *)xenix_scoinfo, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 21 = xenix_scoinfo */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 22 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 23 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 24 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 25 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 26 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 27 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 28 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 29 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 30 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 31 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 32 = xenix_proctl */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 33 = xenix_execseg */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 34 = xenix_unexecseg */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 35 = nosys */
+	{ AS(select_args), (sy_call_t *)sys_select, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 36 = select */
+	{ AS(xenix_eaccess_args), (sy_call_t *)xenix_eaccess, AUE_EACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = xenix_eaccess */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 38 = xenix_paccess */
+	{ AS(ibcs2_sigaction_args), (sy_call_t *)ibcs2_sigaction, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = ibcs2_sigaction */
+	{ AS(ibcs2_sigprocmask_args), (sy_call_t *)ibcs2_sigprocmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 40 = ibcs2_sigprocmask */
+	{ AS(ibcs2_sigpending_args), (sy_call_t *)ibcs2_sigpending, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = ibcs2_sigpending */
+	{ AS(ibcs2_sigsuspend_args), (sy_call_t *)ibcs2_sigsuspend, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 42 = ibcs2_sigsuspend */
+	{ AS(ibcs2_getgroups_args), (sy_call_t *)ibcs2_getgroups, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = ibcs2_getgroups */
+	{ AS(ibcs2_setgroups_args), (sy_call_t *)ibcs2_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 44 = ibcs2_setgroups */
+	{ AS(ibcs2_sysconf_args), (sy_call_t *)ibcs2_sysconf, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 45 = ibcs2_sysconf */
+	{ AS(ibcs2_pathconf_args), (sy_call_t *)ibcs2_pathconf, AUE_PATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 46 = ibcs2_pathconf */
+	{ AS(ibcs2_fpathconf_args), (sy_call_t *)ibcs2_fpathconf, AUE_FPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 47 = ibcs2_fpathconf */
+	{ AS(ibcs2_rename_args), (sy_call_t *)ibcs2_rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 48 = ibcs2_rename */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 49 = nosys */
+	{ AS(xenix_utsname_args), (sy_call_t *)xenix_utsname, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = xenix_utsname */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 51 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 52 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 53 = nosys */
+	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 54 = nosys */
+	{ AS(getitimer_args), (sy_call_t *)sys_getitimer, AUE_GETITIMER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = getitimer */
+	{ AS(setitimer_args), (sy_call_t *)sys_setitimer, AUE_SETITIMER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 56 = setitimer */
 };
diff --git a/sys/i386/include/ieeefp.h b/sys/i386/include/ieeefp.h
index 65502ac76f21..c676c4541831 100644
--- a/sys/i386/include/ieeefp.h
+++ b/sys/i386/include/ieeefp.h
@@ -39,6 +39,8 @@
 #define _MACHINE_IEEEFP_H_
 
 /*
+ * Deprecated historical FPU control interface
+ *
  * IEEE floating point type, constant and function definitions.
  * XXX: FP*FLD and FP*OFF are undocumented pollution.
  */
@@ -253,7 +255,4 @@ fpresetsticky(fp_except_t _m)
 
 #endif /* __GNUCLIKE_ASM */
 
-/* Suppress prototypes in the MI header. */
-#define	_IEEEFP_INLINED_	1
-
 #endif /* !_MACHINE_IEEEFP_H_ */
diff --git a/sys/i386/include/proc.h b/sys/i386/include/proc.h
index 0d922520eab9..86e86020b99a 100644
--- a/sys/i386/include/proc.h
+++ b/sys/i386/include/proc.h
@@ -83,8 +83,6 @@ struct syscall_args {
 	register_t args[8];
 	int narg;
 };
-#define	HAVE_SYSCALL_ARGS_DEF 1
-
 #endif	/* _KERNEL */
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/i386/linux/linux_machdep.c b/sys/i386/linux/linux_machdep.c
index 57756c4d3e83..396740a848c8 100644
--- a/sys/i386/linux/linux_machdep.c
+++ b/sys/i386/linux/linux_machdep.c
@@ -564,7 +564,7 @@ linux_mmap_common(struct thread *td, l_uintptr_t addr, l_size_t len, l_int prot,
 		    (void *)bsd_args.addr, bsd_args.len, bsd_args.prot,
 		    bsd_args.flags, bsd_args.fd, (int)bsd_args.pos);
 #endif
-	error = mmap(td, &bsd_args);
+	error = sys_mmap(td, &bsd_args);
 #ifdef DEBUG
 	if (ldebug(mmap))
 		printf("-> %s() return: 0x%x (0x%08x)\n",
@@ -583,7 +583,7 @@ linux_mprotect(struct thread *td, struct linux_mprotect_args *uap)
 	bsd_args.prot = uap->prot;
 	if (bsd_args.prot & (PROT_READ | PROT_WRITE | PROT_EXEC))
 		bsd_args.prot |= PROT_READ | PROT_EXEC;
-	return (mprotect(td, &bsd_args));
+	return (sys_mprotect(td, &bsd_args));
 }
 
 int
@@ -844,7 +844,7 @@ linux_ftruncate64(struct thread *td, struct linux_ftruncate64_args *args)
 
 	sa.fd = args->fd;
 	sa.length = args->length;
-	return ftruncate(td, &sa);
+	return sys_ftruncate(td, &sa);
 }
 
 int
@@ -1000,31 +1000,31 @@ linux_get_thread_area(struct thread *td, struct linux_get_thread_area_args *args
 int
 linux_timer_create(struct thread *td, struct linux_timer_create_args *args)
 {
-   	return ktimer_create(td, (struct ktimer_create_args *) args);
+   	return sys_ktimer_create(td, (struct ktimer_create_args *) args);
 }
 
 int
 linux_timer_settime(struct thread *td, struct linux_timer_settime_args *args)
 {
-   	return ktimer_settime(td, (struct ktimer_settime_args *) args);
+   	return sys_ktimer_settime(td, (struct ktimer_settime_args *) args);
 }
 
 int
 linux_timer_gettime(struct thread *td, struct linux_timer_gettime_args *args)
 {
-   	return ktimer_gettime(td, (struct ktimer_gettime_args *) args);
+   	return sys_ktimer_gettime(td, (struct ktimer_gettime_args *) args);
 }
 
 int
 linux_timer_getoverrun(struct thread *td, struct linux_timer_getoverrun_args *args)
 {
-   	return ktimer_getoverrun(td, (struct ktimer_getoverrun_args *) args);
+   	return sys_ktimer_getoverrun(td, (struct ktimer_getoverrun_args *) args);
 }
 
 int
 linux_timer_delete(struct thread *td, struct linux_timer_delete_args *args)
 {
-   	return ktimer_delete(td, (struct ktimer_delete_args *) args);
+   	return sys_ktimer_delete(td, (struct ktimer_delete_args *) args);
 }
 
 /* XXX: this wont work with module - convert it */
@@ -1032,7 +1032,7 @@ int
 linux_mq_open(struct thread *td, struct linux_mq_open_args *args)
 {
 #ifdef P1003_1B_MQUEUE
-   	return kmq_open(td, (struct kmq_open_args *) args);
+   	return sys_kmq_open(td, (struct kmq_open_args *) args);
 #else
 	return (ENOSYS);
 #endif
@@ -1042,7 +1042,7 @@ int
 linux_mq_unlink(struct thread *td, struct linux_mq_unlink_args *args)
 {
 #ifdef P1003_1B_MQUEUE
-   	return kmq_unlink(td, (struct kmq_unlink_args *) args);
+   	return sys_kmq_unlink(td, (struct kmq_unlink_args *) args);
 #else
 	return (ENOSYS);
 #endif
@@ -1052,7 +1052,7 @@ int
 linux_mq_timedsend(struct thread *td, struct linux_mq_timedsend_args *args)
 {
 #ifdef P1003_1B_MQUEUE
-   	return kmq_timedsend(td, (struct kmq_timedsend_args *) args);
+   	return sys_kmq_timedsend(td, (struct kmq_timedsend_args *) args);
 #else
 	return (ENOSYS);
 #endif
@@ -1062,7 +1062,7 @@ int
 linux_mq_timedreceive(struct thread *td, struct linux_mq_timedreceive_args *args)
 {
 #ifdef P1003_1B_MQUEUE
-   	return kmq_timedreceive(td, (struct kmq_timedreceive_args *) args);
+   	return sys_kmq_timedreceive(td, (struct kmq_timedreceive_args *) args);
 #else
 	return (ENOSYS);
 #endif
@@ -1072,7 +1072,7 @@ int
 linux_mq_notify(struct thread *td, struct linux_mq_notify_args *args)
 {
 #ifdef P1003_1B_MQUEUE
-	return kmq_notify(td, (struct kmq_notify_args *) args);
+	return sys_kmq_notify(td, (struct kmq_notify_args *) args);
 #else
 	return (ENOSYS);
 #endif
@@ -1082,7 +1082,7 @@ int
 linux_mq_getsetattr(struct thread *td, struct linux_mq_getsetattr_args *args)
 {
 #ifdef P1003_1B_MQUEUE
-   	return kmq_setattr(td, (struct kmq_setattr_args *) args);
+   	return sys_kmq_setattr(td, (struct kmq_setattr_args *) args);
 #else
 	return (ENOSYS);
 #endif
diff --git a/sys/i386/linux/linux_sysent.c b/sys/i386/linux/linux_sysent.c
index 509df3e570b9..491ead7fa836 100644
--- a/sys/i386/linux/linux_sysent.c
+++ b/sys/i386/linux/linux_sysent.c
@@ -19,12 +19,12 @@
 struct sysent linux_sysent[] = {
 #define	nosys	linux_nosys
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 0 = setup */
-	{ AS(sys_exit_args), (sy_call_t *)sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
+	{ AS(sys_exit_args), (sy_call_t *)sys_sys_exit, AUE_EXIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 1 = exit */
 	{ 0, (sy_call_t *)linux_fork, AUE_FORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 2 = linux_fork */
-	{ AS(read_args), (sy_call_t *)read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
-	{ AS(write_args), (sy_call_t *)write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
+	{ AS(read_args), (sy_call_t *)sys_read, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 3 = read */
+	{ AS(write_args), (sy_call_t *)sys_write, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 4 = write */
 	{ AS(linux_open_args), (sy_call_t *)linux_open, AUE_OPEN_RWTC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 5 = linux_open */
-	{ AS(close_args), (sy_call_t *)close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
+	{ AS(close_args), (sy_call_t *)sys_close, AUE_CLOSE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 6 = close */
 	{ AS(linux_waitpid_args), (sy_call_t *)linux_waitpid, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = linux_waitpid */
 	{ AS(linux_creat_args), (sy_call_t *)linux_creat, AUE_CREAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 8 = linux_creat */
 	{ AS(linux_link_args), (sy_call_t *)linux_link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = linux_link */
@@ -54,12 +54,12 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_access_args), (sy_call_t *)linux_access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = linux_access */
 	{ AS(linux_nice_args), (sy_call_t *)linux_nice, AUE_NICE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = linux_nice */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 35 = ftime */
-	{ 0, (sy_call_t *)sync, AUE_SYNC, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
+	{ 0, (sy_call_t *)sys_sync, AUE_SYNC, NULL, 0, 0, 0, SY_THR_STATIC },		/* 36 = sync */
 	{ AS(linux_kill_args), (sy_call_t *)linux_kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = linux_kill */
 	{ AS(linux_rename_args), (sy_call_t *)linux_rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 38 = linux_rename */
 	{ AS(linux_mkdir_args), (sy_call_t *)linux_mkdir, AUE_MKDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 39 = linux_mkdir */
 	{ AS(linux_rmdir_args), (sy_call_t *)linux_rmdir, AUE_RMDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 40 = linux_rmdir */
-	{ AS(dup_args), (sy_call_t *)dup, AUE_DUP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
+	{ AS(dup_args), (sy_call_t *)sys_dup, AUE_DUP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 41 = dup */
 	{ AS(linux_pipe_args), (sy_call_t *)linux_pipe, AUE_PIPE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 42 = linux_pipe */
 	{ AS(linux_times_args), (sy_call_t *)linux_times, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 43 = linux_times */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 44 = prof */
@@ -69,22 +69,22 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_signal_args), (sy_call_t *)linux_signal, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 48 = linux_signal */
 	{ 0, (sy_call_t *)linux_geteuid16, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 49 = linux_geteuid16 */
 	{ 0, (sy_call_t *)linux_getegid16, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = linux_getegid16 */
-	{ AS(acct_args), (sy_call_t *)acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
+	{ AS(acct_args), (sy_call_t *)sys_acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
 	{ AS(linux_umount_args), (sy_call_t *)linux_umount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 52 = linux_umount */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 53 = lock */
 	{ AS(linux_ioctl_args), (sy_call_t *)linux_ioctl, AUE_IOCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 54 = linux_ioctl */
 	{ AS(linux_fcntl_args), (sy_call_t *)linux_fcntl, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = linux_fcntl */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 56 = mpx */
-	{ AS(setpgid_args), (sy_call_t *)setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = setpgid */
+	{ AS(setpgid_args), (sy_call_t *)sys_setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = setpgid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 58 = ulimit */
 	{ 0, (sy_call_t *)linux_olduname, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = linux_olduname */
-	{ AS(umask_args), (sy_call_t *)umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
-	{ AS(chroot_args), (sy_call_t *)chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
+	{ AS(umask_args), (sy_call_t *)sys_umask, AUE_UMASK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 60 = umask */
+	{ AS(chroot_args), (sy_call_t *)sys_chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
 	{ AS(linux_ustat_args), (sy_call_t *)linux_ustat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 62 = linux_ustat */
-	{ AS(dup2_args), (sy_call_t *)dup2, AUE_DUP2, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = dup2 */
+	{ AS(dup2_args), (sy_call_t *)sys_dup2, AUE_DUP2, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = dup2 */
 	{ 0, (sy_call_t *)linux_getppid, AUE_GETPPID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 64 = linux_getppid */
-	{ 0, (sy_call_t *)getpgrp, AUE_GETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 65 = getpgrp */
-	{ 0, (sy_call_t *)setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },		/* 66 = setsid */
+	{ 0, (sy_call_t *)sys_getpgrp, AUE_GETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 65 = getpgrp */
+	{ 0, (sy_call_t *)sys_setsid, AUE_SETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 66 = setsid */
 	{ AS(linux_sigaction_args), (sy_call_t *)linux_sigaction, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 67 = linux_sigaction */
 	{ 0, (sy_call_t *)linux_sgetmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 68 = linux_sgetmask */
 	{ AS(linux_ssetmask_args), (sy_call_t *)linux_ssetmask, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 69 = linux_ssetmask */
@@ -95,9 +95,9 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_sethostname_args), (sy_call_t *)linux_sethostname, AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 74 = linux_sethostname */
 	{ AS(linux_setrlimit_args), (sy_call_t *)linux_setrlimit, AUE_SETRLIMIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 75 = linux_setrlimit */
 	{ AS(linux_old_getrlimit_args), (sy_call_t *)linux_old_getrlimit, AUE_GETRLIMIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 76 = linux_old_getrlimit */
-	{ AS(getrusage_args), (sy_call_t *)getrusage, AUE_GETRUSAGE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 77 = getrusage */
-	{ AS(gettimeofday_args), (sy_call_t *)gettimeofday, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 78 = gettimeofday */
-	{ AS(settimeofday_args), (sy_call_t *)settimeofday, AUE_SETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 79 = settimeofday */
+	{ AS(getrusage_args), (sy_call_t *)sys_getrusage, AUE_GETRUSAGE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 77 = getrusage */
+	{ AS(gettimeofday_args), (sy_call_t *)sys_gettimeofday, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 78 = gettimeofday */
+	{ AS(settimeofday_args), (sy_call_t *)sys_settimeofday, AUE_SETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 79 = settimeofday */
 	{ AS(linux_getgroups16_args), (sy_call_t *)linux_getgroups16, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = linux_getgroups16 */
 	{ AS(linux_setgroups16_args), (sy_call_t *)linux_setgroups16, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 81 = linux_setgroups16 */
 	{ AS(linux_old_select_args), (sy_call_t *)linux_old_select, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 82 = linux_old_select */
@@ -105,17 +105,17 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_lstat_args), (sy_call_t *)linux_lstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 84 = linux_lstat */
 	{ AS(linux_readlink_args), (sy_call_t *)linux_readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = linux_readlink */
 	{ AS(linux_uselib_args), (sy_call_t *)linux_uselib, AUE_USELIB, NULL, 0, 0, 0, SY_THR_STATIC },	/* 86 = linux_uselib */
-	{ AS(swapon_args), (sy_call_t *)swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = swapon */
+	{ AS(swapon_args), (sy_call_t *)sys_swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 87 = swapon */
 	{ AS(linux_reboot_args), (sy_call_t *)linux_reboot, AUE_REBOOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 88 = linux_reboot */
 	{ AS(linux_readdir_args), (sy_call_t *)linux_readdir, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = linux_readdir */
 	{ AS(linux_mmap_args), (sy_call_t *)linux_mmap, AUE_MMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 90 = linux_mmap */
-	{ AS(munmap_args), (sy_call_t *)munmap, AUE_MUNMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 91 = munmap */
+	{ AS(munmap_args), (sy_call_t *)sys_munmap, AUE_MUNMAP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 91 = munmap */
 	{ AS(linux_truncate_args), (sy_call_t *)linux_truncate, AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 92 = linux_truncate */
 	{ AS(linux_ftruncate_args), (sy_call_t *)linux_ftruncate, AUE_FTRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 93 = linux_ftruncate */
-	{ AS(fchmod_args), (sy_call_t *)fchmod, AUE_FCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 94 = fchmod */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_FCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = fchown */
+	{ AS(fchmod_args), (sy_call_t *)sys_fchmod, AUE_FCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 94 = fchmod */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_FCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 95 = fchown */
 	{ AS(linux_getpriority_args), (sy_call_t *)linux_getpriority, AUE_GETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 96 = linux_getpriority */
-	{ AS(setpriority_args), (sy_call_t *)setpriority, AUE_SETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = setpriority */
+	{ AS(setpriority_args), (sy_call_t *)sys_setpriority, AUE_SETPRIORITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 97 = setpriority */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 98 = profil */
 	{ AS(linux_statfs_args), (sy_call_t *)linux_statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 99 = linux_statfs */
 	{ AS(linux_fstatfs_args), (sy_call_t *)linux_fstatfs, AUE_FSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 100 = linux_fstatfs */
@@ -136,7 +136,7 @@ struct sysent linux_sysent[] = {
 	{ 0, (sy_call_t *)linux_swapoff, AUE_SWAPOFF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 115 = linux_swapoff */
 	{ AS(linux_sysinfo_args), (sy_call_t *)linux_sysinfo, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 116 = linux_sysinfo */
 	{ AS(linux_ipc_args), (sy_call_t *)linux_ipc, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 117 = linux_ipc */
-	{ AS(fsync_args), (sy_call_t *)fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = fsync */
+	{ AS(fsync_args), (sy_call_t *)sys_fsync, AUE_FSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 118 = fsync */
 	{ AS(linux_sigreturn_args), (sy_call_t *)linux_sigreturn, AUE_SIGRETURN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 119 = linux_sigreturn */
 	{ AS(linux_clone_args), (sy_call_t *)linux_clone, AUE_RFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 120 = linux_clone */
 	{ AS(linux_setdomainname_args), (sy_call_t *)linux_setdomainname, AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 121 = linux_setdomainname */
@@ -150,8 +150,8 @@ struct sysent linux_sysent[] = {
 	{ 0, (sy_call_t *)linux_delete_module, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 129 = linux_delete_module */
 	{ 0, (sy_call_t *)linux_get_kernel_syms, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 130 = linux_get_kernel_syms */
 	{ 0, (sy_call_t *)linux_quotactl, AUE_QUOTACTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 131 = linux_quotactl */
-	{ AS(getpgid_args), (sy_call_t *)getpgid, AUE_GETPGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = getpgid */
-	{ AS(fchdir_args), (sy_call_t *)fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 133 = fchdir */
+	{ AS(getpgid_args), (sy_call_t *)sys_getpgid, AUE_GETPGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = getpgid */
+	{ AS(fchdir_args), (sy_call_t *)sys_fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 133 = fchdir */
 	{ 0, (sy_call_t *)linux_bdflush, AUE_BDFLUSH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 134 = linux_bdflush */
 	{ AS(linux_sysfs_args), (sy_call_t *)linux_sysfs, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 135 = linux_sysfs */
 	{ AS(linux_personality_args), (sy_call_t *)linux_personality, AUE_PERSONALITY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = linux_personality */
@@ -161,32 +161,32 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_llseek_args), (sy_call_t *)linux_llseek, AUE_LSEEK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 140 = linux_llseek */
 	{ AS(linux_getdents_args), (sy_call_t *)linux_getdents, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 141 = linux_getdents */
 	{ AS(linux_select_args), (sy_call_t *)linux_select, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 142 = linux_select */
-	{ AS(flock_args), (sy_call_t *)flock, AUE_FLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 143 = flock */
+	{ AS(flock_args), (sy_call_t *)sys_flock, AUE_FLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 143 = flock */
 	{ AS(linux_msync_args), (sy_call_t *)linux_msync, AUE_MSYNC, NULL, 0, 0, 0, SY_THR_STATIC },	/* 144 = linux_msync */
-	{ AS(readv_args), (sy_call_t *)readv, AUE_READV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 145 = readv */
-	{ AS(writev_args), (sy_call_t *)writev, AUE_WRITEV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 146 = writev */
+	{ AS(readv_args), (sy_call_t *)sys_readv, AUE_READV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 145 = readv */
+	{ AS(writev_args), (sy_call_t *)sys_writev, AUE_WRITEV, NULL, 0, 0, 0, SY_THR_STATIC },	/* 146 = writev */
 	{ AS(linux_getsid_args), (sy_call_t *)linux_getsid, AUE_GETSID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 147 = linux_getsid */
 	{ AS(linux_fdatasync_args), (sy_call_t *)linux_fdatasync, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 148 = linux_fdatasync */
 	{ AS(linux_sysctl_args), (sy_call_t *)linux_sysctl, AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 149 = linux_sysctl */
-	{ AS(mlock_args), (sy_call_t *)mlock, AUE_MLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 150 = mlock */
-	{ AS(munlock_args), (sy_call_t *)munlock, AUE_MUNLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 151 = munlock */
-	{ AS(mlockall_args), (sy_call_t *)mlockall, AUE_MLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 152 = mlockall */
-	{ 0, (sy_call_t *)munlockall, AUE_MUNLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 153 = munlockall */
-	{ AS(sched_setparam_args), (sy_call_t *)sched_setparam, AUE_SCHED_SETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 154 = sched_setparam */
-	{ AS(sched_getparam_args), (sy_call_t *)sched_getparam, AUE_SCHED_GETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 155 = sched_getparam */
+	{ AS(mlock_args), (sy_call_t *)sys_mlock, AUE_MLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 150 = mlock */
+	{ AS(munlock_args), (sy_call_t *)sys_munlock, AUE_MUNLOCK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 151 = munlock */
+	{ AS(mlockall_args), (sy_call_t *)sys_mlockall, AUE_MLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 152 = mlockall */
+	{ 0, (sy_call_t *)sys_munlockall, AUE_MUNLOCKALL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 153 = munlockall */
+	{ AS(sched_setparam_args), (sy_call_t *)sys_sched_setparam, AUE_SCHED_SETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 154 = sched_setparam */
+	{ AS(sched_getparam_args), (sy_call_t *)sys_sched_getparam, AUE_SCHED_GETPARAM, NULL, 0, 0, 0, SY_THR_STATIC },	/* 155 = sched_getparam */
 	{ AS(linux_sched_setscheduler_args), (sy_call_t *)linux_sched_setscheduler, AUE_SCHED_SETSCHEDULER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 156 = linux_sched_setscheduler */
 	{ AS(linux_sched_getscheduler_args), (sy_call_t *)linux_sched_getscheduler, AUE_SCHED_GETSCHEDULER, NULL, 0, 0, 0, SY_THR_STATIC },	/* 157 = linux_sched_getscheduler */
-	{ 0, (sy_call_t *)sched_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = sched_yield */
+	{ 0, (sy_call_t *)sys_sched_yield, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 158 = sched_yield */
 	{ AS(linux_sched_get_priority_max_args), (sy_call_t *)linux_sched_get_priority_max, AUE_SCHED_GET_PRIORITY_MAX, NULL, 0, 0, 0, SY_THR_STATIC },	/* 159 = linux_sched_get_priority_max */
 	{ AS(linux_sched_get_priority_min_args), (sy_call_t *)linux_sched_get_priority_min, AUE_SCHED_GET_PRIORITY_MIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 160 = linux_sched_get_priority_min */
-	{ AS(sched_rr_get_interval_args), (sy_call_t *)sched_rr_get_interval, AUE_SCHED_RR_GET_INTERVAL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = sched_rr_get_interval */
+	{ AS(sched_rr_get_interval_args), (sy_call_t *)sys_sched_rr_get_interval, AUE_SCHED_RR_GET_INTERVAL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = sched_rr_get_interval */
 	{ AS(linux_nanosleep_args), (sy_call_t *)linux_nanosleep, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 162 = linux_nanosleep */
 	{ AS(linux_mremap_args), (sy_call_t *)linux_mremap, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 163 = linux_mremap */
 	{ AS(linux_setresuid16_args), (sy_call_t *)linux_setresuid16, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 164 = linux_setresuid16 */
 	{ AS(linux_getresuid16_args), (sy_call_t *)linux_getresuid16, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 165 = linux_getresuid16 */
 	{ 0, (sy_call_t *)linux_vm86, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 166 = linux_vm86 */
 	{ 0, (sy_call_t *)linux_query_module, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 167 = linux_query_module */
-	{ AS(poll_args), (sy_call_t *)poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 168 = poll */
+	{ AS(poll_args), (sy_call_t *)sys_poll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 168 = poll */
 	{ 0, (sy_call_t *)linux_nfsservctl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 169 = linux_nfsservctl */
 	{ AS(linux_setresgid16_args), (sy_call_t *)linux_setresgid16, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 170 = linux_setresgid16 */
 	{ AS(linux_getresgid16_args), (sy_call_t *)linux_getresgid16, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 171 = linux_getresgid16 */
@@ -219,25 +219,25 @@ struct sysent linux_sysent[] = {
 	{ AS(linux_lchown_args), (sy_call_t *)linux_lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 198 = linux_lchown */
 	{ 0, (sy_call_t *)linux_getuid, AUE_GETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 199 = linux_getuid */
 	{ 0, (sy_call_t *)linux_getgid, AUE_GETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 200 = linux_getgid */
-	{ 0, (sy_call_t *)geteuid, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 201 = geteuid */
-	{ 0, (sy_call_t *)getegid, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = getegid */
-	{ AS(setreuid_args), (sy_call_t *)setreuid, AUE_SETREUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = setreuid */
-	{ AS(setregid_args), (sy_call_t *)setregid, AUE_SETREGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 204 = setregid */
+	{ 0, (sy_call_t *)sys_geteuid, AUE_GETEUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 201 = geteuid */
+	{ 0, (sy_call_t *)sys_getegid, AUE_GETEGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 202 = getegid */
+	{ AS(setreuid_args), (sy_call_t *)sys_setreuid, AUE_SETREUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 203 = setreuid */
+	{ AS(setregid_args), (sy_call_t *)sys_setregid, AUE_SETREGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 204 = setregid */
 	{ AS(linux_getgroups_args), (sy_call_t *)linux_getgroups, AUE_GETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 205 = linux_getgroups */
 	{ AS(linux_setgroups_args), (sy_call_t *)linux_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 206 = linux_setgroups */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 207 = fchown */
-	{ AS(setresuid_args), (sy_call_t *)setresuid, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 208 = setresuid */
-	{ AS(getresuid_args), (sy_call_t *)getresuid, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 209 = getresuid */
-	{ AS(setresgid_args), (sy_call_t *)setresgid, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 210 = setresgid */
-	{ AS(getresgid_args), (sy_call_t *)getresgid, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 211 = getresgid */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 207 = fchown */
+	{ AS(setresuid_args), (sy_call_t *)sys_setresuid, AUE_SETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 208 = setresuid */
+	{ AS(getresuid_args), (sy_call_t *)sys_getresuid, AUE_GETRESUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 209 = getresuid */
+	{ AS(setresgid_args), (sy_call_t *)sys_setresgid, AUE_SETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 210 = setresgid */
+	{ AS(getresgid_args), (sy_call_t *)sys_getresgid, AUE_GETRESGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 211 = getresgid */
 	{ AS(linux_chown_args), (sy_call_t *)linux_chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 212 = linux_chown */
-	{ AS(setuid_args), (sy_call_t *)setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 213 = setuid */
-	{ AS(setgid_args), (sy_call_t *)setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 214 = setgid */
+	{ AS(setuid_args), (sy_call_t *)sys_setuid, AUE_SETUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 213 = setuid */
+	{ AS(setgid_args), (sy_call_t *)sys_setgid, AUE_SETGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 214 = setgid */
 	{ AS(linux_setfsuid_args), (sy_call_t *)linux_setfsuid, AUE_SETFSUID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 215 = linux_setfsuid */
 	{ AS(linux_setfsgid_args), (sy_call_t *)linux_setfsgid, AUE_SETFSGID, NULL, 0, 0, 0, SY_THR_STATIC },	/* 216 = linux_setfsgid */
 	{ AS(linux_pivot_root_args), (sy_call_t *)linux_pivot_root, AUE_PIVOT_ROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 217 = linux_pivot_root */
 	{ AS(linux_mincore_args), (sy_call_t *)linux_mincore, AUE_MINCORE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 218 = linux_mincore */
-	{ AS(madvise_args), (sy_call_t *)madvise, AUE_MADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 219 = madvise */
+	{ AS(madvise_args), (sy_call_t *)sys_madvise, AUE_MADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 219 = madvise */
 	{ AS(linux_getdents64_args), (sy_call_t *)linux_getdents64, AUE_GETDIRENTRIES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 220 = linux_getdents64 */
 	{ AS(linux_fcntl64_args), (sy_call_t *)linux_fcntl64, AUE_FCNTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 221 = linux_fcntl64 */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 222 =  */
diff --git a/sys/ia64/conf/GENERIC b/sys/ia64/conf/GENERIC
index eaf3ffd4452c..a9dd567d5528 100644
--- a/sys/ia64/conf/GENERIC
+++ b/sys/ia64/conf/GENERIC
@@ -28,17 +28,11 @@ makeoptions	DEBUG=-g	# Build kernel with debug information.
 options 	AUDIT		# Security event auditing
 options 	CD9660		# ISO 9660 Filesystem
 options 	COMPAT_FREEBSD7	# Compatible with FreeBSD7
-options 	DDB		# Support DDB
-options 	DEADLKRES	# Enable the deadlock resolver
 options 	FFS		# Berkeley Fast Filesystem
-options 	GDB		# Support remote GDB
 options 	GEOM_LABEL	# Provides labelization
 options 	INCLUDE_CONFIG_FILE     # Include this file in kernel
 options 	INET		# InterNETworking
 options 	INET6		# IPv6 communications protocols
-options 	INVARIANTS	# Enable calls of extra sanity checking
-options 	INVARIANT_SUPPORT # required by INVARIANTS
-options 	KDB		# Enable kernel debugger support
 options 	KTRACE		# ktrace(1) syscall trace support
 options 	MAC		# TrustedBSD MAC Framework
 options 	MD_ROOT		# MD usable as root device
@@ -64,9 +58,17 @@ options 	SYSVSHM		# SYSV-style shared memory
 options 	UFS_ACL		# Support for access control lists
 options 	UFS_DIRHASH	# Hash-based directory lookup scheme
 options 	UFS_GJOURNAL	# Enable gjournal-based UFS journaling
+options 	_KPOSIX_PRIORITY_SCHEDULING	# Posix P1003_1B RT extensions
+
+# Debugging for use in -current
+options 	KDB		# Enable kernel debugger support
+options 	DDB		# Support DDB
+options 	GDB		# Support remote GDB
+options 	DEADLKRES	# Enable the deadlock resolver
+options 	INVARIANTS	# Enable calls of extra sanity checking
+options 	INVARIANT_SUPPORT # required by INVARIANTS
 options 	WITNESS		# Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
-options 	_KPOSIX_PRIORITY_SCHEDULING	# Posix P1003_1B RT extensions
 options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # Various "busses"
@@ -175,7 +177,8 @@ device		uvscom		# USB serial support for DDI pocket's PHS
 
 # FireWire support
 device		fwip		# IP over FireWire (RFC 2734,3146)
-device		sbp		# SCSI over FireWire (need scbus & da)
+# sbp(4) works for some systems but causes boot failure on others
+#device		sbp		# SCSI over FireWire (need scbus & da)
 
 # Various (pseudo) devices
 device		ether		# Ethernet support
diff --git a/sys/ia64/ia32/ia32_signal.c b/sys/ia64/ia32/ia32_signal.c
index b8dab9054983..a3d4ace04c08 100644
--- a/sys/ia64/ia32/ia32_signal.c
+++ b/sys/ia64/ia32/ia32_signal.c
@@ -117,14 +117,14 @@ ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 int
 freebsd4_freebsd32_sigreturn(struct thread *td, struct freebsd4_freebsd32_sigreturn_args *uap)
 {
-	return (sigreturn(td, (struct sigreturn_args *)uap));
+	return (sys_sigreturn(td, (struct sigreturn_args *)uap));
 }
 #endif
 
 int
 freebsd32_sigreturn(struct thread *td, struct freebsd32_sigreturn_args *uap)
 {
-	return (sigreturn(td, (struct sigreturn_args *)uap));
+	return (sys_sigreturn(td, (struct sigreturn_args *)uap));
 }
 
 
diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index 2bfd62e4ee4b..2e14a2e17c3e 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -1155,7 +1155,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
  * MPSAFE
  */
 int
-sigreturn(struct thread *td,
+sys_sigreturn(struct thread *td,
 	struct sigreturn_args /* {
 		ucontext_t *sigcntxp;
 	} */ *uap)
@@ -1192,7 +1192,7 @@ int
 freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
 {
 
-	return sigreturn(td, (struct sigreturn_args *)uap);
+	return sys_sigreturn(td, (struct sigreturn_args *)uap);
 }
 #endif
 
diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c
index 37116f5af198..4c289dff2621 100644
--- a/sys/ia64/ia64/pmap.c
+++ b/sys/ia64/ia64/pmap.c
@@ -1486,7 +1486,6 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 	if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
 		panic("pmap_protect: unaligned addresses");
 
-	vm_page_lock_queues();
 	PMAP_LOCK(pmap);
 	oldpmap = pmap_switch(pmap);
 	for ( ; sva < eva; sva += PAGE_SIZE) {
@@ -1514,7 +1513,6 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 		pmap_pte_prot(pmap, pte, prot);
 		pmap_invalidate_page(sva);
 	}
-	vm_page_unlock_queues();
 	pmap_switch(oldpmap);
 	PMAP_UNLOCK(pmap);
 }
diff --git a/sys/ia64/include/ieeefp.h b/sys/ia64/include/ieeefp.h
index 3b0cd4b9b781..7cde91b4814c 100644
--- a/sys/ia64/include/ieeefp.h
+++ b/sys/ia64/include/ieeefp.h
@@ -29,6 +29,8 @@
 #ifndef _MACHINE_IEEEFP_H_
 #define _MACHINE_IEEEFP_H_
 
+/* Deprecated historical FPU control interface */
+
 #include <machine/fpu.h>
 
 typedef int fp_except_t;
@@ -45,4 +47,11 @@ typedef enum {
 	FP_RZ			/* round toward zero */
 } fp_rnd_t;
 
+__BEGIN_DECLS
+extern fp_rnd_t    fpgetround(void);
+extern fp_rnd_t    fpsetround(fp_rnd_t);
+extern fp_except_t fpgetmask(void);
+extern fp_except_t fpsetmask(fp_except_t);
+__END_DECLS
+
 #endif /* !_MACHINE_IEEEFP_H_ */
diff --git a/sys/ia64/include/proc.h b/sys/ia64/include/proc.h
index 8818b9d6bbd0..d7f0b4a7b0be 100644
--- a/sys/ia64/include/proc.h
+++ b/sys/ia64/include/proc.h
@@ -49,7 +49,6 @@ struct syscall_args {
 	register_t args32[8];
 	int narg;
 };
-#define	HAVE_SYSCALL_ARGS_DEF 1
 #endif
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index 45f6d64917a0..8455f4854803 100644
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -123,6 +123,14 @@ SYSCTL_INT(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO,
     nxstack, CTLFLAG_RW, &__elfN(nxstack), 0,
     __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) ": enable non-executable stack");
 
+#if __ELF_WORD_SIZE == 32
+#if defined(__amd64__) || defined(__ia64__)
+int i386_read_exec = 0;
+SYSCTL_INT(_kern_elf32, OID_AUTO, read_exec, CTLFLAG_RW, &i386_read_exec, 0,
+    "enable execution from readable segments");
+#endif
+#endif
+
 static Elf_Brandinfo *elf_brand_list[MAX_BRANDS];
 
 #define	trunc_page_ps(va, ps)	((va) & ~(ps - 1))
@@ -1664,6 +1672,12 @@ __elfN(trans_prot)(Elf_Word flags)
 		prot |= VM_PROT_WRITE;
 	if (flags & PF_R)
 		prot |= VM_PROT_READ;
+#if __ELF_WORD_SIZE == 32
+#if defined(__amd64__) || defined(__ia64__)
+	if (i386_read_exec && (flags & PF_R))
+		prot |= VM_PROT_EXECUTE;
+#endif
+#endif
 	return (prot);
 }
 
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index fc072457c191..2c094f85b8a7 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -765,7 +765,7 @@ start_init(void *dummy)
 		 * Otherwise, return via fork_trampoline() all the way
 		 * to user mode as init!
 		 */
-		if ((error = execve(td, &args)) == 0) {
+		if ((error = sys_execve(td, &args)) == 0) {
 			mtx_unlock(&Giant);
 			return;
 		}
diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c
index d1063f4e83e1..0854cec3c74e 100644
--- a/sys/kern/init_sysent.c
+++ b/sys/kern/init_sysent.c
@@ -35,112 +35,112 @@
 /* The casts are bogus but will do for now. */
 struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 0 = syscall */
-	{ AS(sys_exit_args), (sy_call_t *)sys_exit, AUE_EXIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 1 = exit */
-	{ 0, (sy_call_t *)fork, AUE_FORK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 2 = fork */
-	{ AS(read_args), (sy_call_t *)read, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 3 = read */
-	{ AS(write_args), (sy_call_t *)write, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 4 = write */
-	{ AS(open_args), (sy_call_t *)open, AUE_OPEN_RWTC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 5 = open */
-	{ AS(close_args), (sy_call_t *)close, AUE_CLOSE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 6 = close */
-	{ AS(wait_args), (sy_call_t *)wait4, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = wait4 */
+	{ AS(sys_exit_args), (sy_call_t *)sys_sys_exit, AUE_EXIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 1 = exit */
+	{ 0, (sy_call_t *)sys_fork, AUE_FORK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 2 = fork */
+	{ AS(read_args), (sy_call_t *)sys_read, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 3 = read */
+	{ AS(write_args), (sy_call_t *)sys_write, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 4 = write */
+	{ AS(open_args), (sy_call_t *)sys_open, AUE_OPEN_RWTC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 5 = open */
+	{ AS(close_args), (sy_call_t *)sys_close, AUE_CLOSE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 6 = close */
+	{ AS(wait_args), (sy_call_t *)sys_wait4, AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },	/* 7 = wait4 */
 	{ compat(AS(ocreat_args),creat), AUE_CREAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 8 = old creat */
-	{ AS(link_args), (sy_call_t *)link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
-	{ AS(unlink_args), (sy_call_t *)unlink, AUE_UNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = unlink */
+	{ AS(link_args), (sy_call_t *)sys_link, AUE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 9 = link */
+	{ AS(unlink_args), (sy_call_t *)sys_unlink, AUE_UNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 10 = unlink */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 11 = obsolete execv */
-	{ AS(chdir_args), (sy_call_t *)chdir, AUE_CHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = chdir */
-	{ AS(fchdir_args), (sy_call_t *)fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 13 = fchdir */
-	{ AS(mknod_args), (sy_call_t *)mknod, AUE_MKNOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 14 = mknod */
-	{ AS(chmod_args), (sy_call_t *)chmod, AUE_CHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = chmod */
-	{ AS(chown_args), (sy_call_t *)chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = chown */
-	{ AS(obreak_args), (sy_call_t *)obreak, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 17 = break */
+	{ AS(chdir_args), (sy_call_t *)sys_chdir, AUE_CHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 12 = chdir */
+	{ AS(fchdir_args), (sy_call_t *)sys_fchdir, AUE_FCHDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 13 = fchdir */
+	{ AS(mknod_args), (sy_call_t *)sys_mknod, AUE_MKNOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 14 = mknod */
+	{ AS(chmod_args), (sy_call_t *)sys_chmod, AUE_CHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 15 = chmod */
+	{ AS(chown_args), (sy_call_t *)sys_chown, AUE_CHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 16 = chown */
+	{ AS(obreak_args), (sy_call_t *)sys_obreak, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 17 = break */
 	{ compat4(AS(freebsd4_getfsstat_args),getfsstat), AUE_GETFSSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 18 = freebsd4 getfsstat */
 	{ compat(AS(olseek_args),lseek), AUE_LSEEK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 19 = old lseek */
-	{ 0, (sy_call_t *)getpid, AUE_GETPID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 20 = getpid */
-	{ AS(mount_args), (sy_call_t *)mount, AUE_MOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 21 = mount */
-	{ AS(unmount_args), (sy_call_t *)unmount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 22 = unmount */
-	{ AS(setuid_args), (sy_call_t *)setuid, AUE_SETUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 23 = setuid */
-	{ 0, (sy_call_t *)getuid, AUE_GETUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 24 = getuid */
-	{ 0, (sy_call_t *)geteuid, AUE_GETEUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 25 = geteuid */
-	{ AS(ptrace_args), (sy_call_t *)ptrace, AUE_PTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 26 = ptrace */
-	{ AS(recvmsg_args), (sy_call_t *)recvmsg, AUE_RECVMSG, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 27 = recvmsg */
-	{ AS(sendmsg_args), (sy_call_t *)sendmsg, AUE_SENDMSG, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 28 = sendmsg */
-	{ AS(recvfrom_args), (sy_call_t *)recvfrom, AUE_RECVFROM, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 29 = recvfrom */
-	{ AS(accept_args), (sy_call_t *)accept, AUE_ACCEPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 30 = accept */
-	{ AS(getpeername_args), (sy_call_t *)getpeername, AUE_GETPEERNAME, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 31 = getpeername */
-	{ AS(getsockname_args), (sy_call_t *)getsockname, AUE_GETSOCKNAME, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 32 = getsockname */
-	{ AS(access_args), (sy_call_t *)access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = access */
-	{ AS(chflags_args), (sy_call_t *)chflags, AUE_CHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = chflags */
-	{ AS(fchflags_args), (sy_call_t *)fchflags, AUE_FCHFLAGS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 35 = fchflags */
-	{ 0, (sy_call_t *)sync, AUE_SYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 36 = sync */
-	{ AS(kill_args), (sy_call_t *)kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = kill */
+	{ 0, (sy_call_t *)sys_getpid, AUE_GETPID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 20 = getpid */
+	{ AS(mount_args), (sy_call_t *)sys_mount, AUE_MOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 21 = mount */
+	{ AS(unmount_args), (sy_call_t *)sys_unmount, AUE_UMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 22 = unmount */
+	{ AS(setuid_args), (sy_call_t *)sys_setuid, AUE_SETUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 23 = setuid */
+	{ 0, (sy_call_t *)sys_getuid, AUE_GETUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 24 = getuid */
+	{ 0, (sy_call_t *)sys_geteuid, AUE_GETEUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 25 = geteuid */
+	{ AS(ptrace_args), (sy_call_t *)sys_ptrace, AUE_PTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 26 = ptrace */
+	{ AS(recvmsg_args), (sy_call_t *)sys_recvmsg, AUE_RECVMSG, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 27 = recvmsg */
+	{ AS(sendmsg_args), (sy_call_t *)sys_sendmsg, AUE_SENDMSG, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 28 = sendmsg */
+	{ AS(recvfrom_args), (sy_call_t *)sys_recvfrom, AUE_RECVFROM, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 29 = recvfrom */
+	{ AS(accept_args), (sy_call_t *)sys_accept, AUE_ACCEPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 30 = accept */
+	{ AS(getpeername_args), (sy_call_t *)sys_getpeername, AUE_GETPEERNAME, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 31 = getpeername */
+	{ AS(getsockname_args), (sy_call_t *)sys_getsockname, AUE_GETSOCKNAME, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 32 = getsockname */
+	{ AS(access_args), (sy_call_t *)sys_access, AUE_ACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 33 = access */
+	{ AS(chflags_args), (sy_call_t *)sys_chflags, AUE_CHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 34 = chflags */
+	{ AS(fchflags_args), (sy_call_t *)sys_fchflags, AUE_FCHFLAGS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 35 = fchflags */
+	{ 0, (sy_call_t *)sys_sync, AUE_SYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 36 = sync */
+	{ AS(kill_args), (sy_call_t *)sys_kill, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 37 = kill */
 	{ compat(AS(ostat_args),stat), AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 38 = old stat */
-	{ 0, (sy_call_t *)getppid, AUE_GETPPID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 39 = getppid */
+	{ 0, (sy_call_t *)sys_getppid, AUE_GETPPID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 39 = getppid */
 	{ compat(AS(olstat_args),lstat), AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 40 = old lstat */
-	{ AS(dup_args), (sy_call_t *)dup, AUE_DUP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 41 = dup */
-	{ 0, (sy_call_t *)pipe, AUE_PIPE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 42 = pipe */
-	{ 0, (sy_call_t *)getegid, AUE_GETEGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 43 = getegid */
-	{ AS(profil_args), (sy_call_t *)profil, AUE_PROFILE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 44 = profil */
-	{ AS(ktrace_args), (sy_call_t *)ktrace, AUE_KTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 45 = ktrace */
+	{ AS(dup_args), (sy_call_t *)sys_dup, AUE_DUP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 41 = dup */
+	{ 0, (sy_call_t *)sys_pipe, AUE_PIPE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 42 = pipe */
+	{ 0, (sy_call_t *)sys_getegid, AUE_GETEGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 43 = getegid */
+	{ AS(profil_args), (sy_call_t *)sys_profil, AUE_PROFILE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 44 = profil */
+	{ AS(ktrace_args), (sy_call_t *)sys_ktrace, AUE_KTRACE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 45 = ktrace */
 	{ compat(AS(osigaction_args),sigaction), AUE_SIGACTION, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 46 = old sigaction */
-	{ 0, (sy_call_t *)getgid, AUE_GETGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 47 = getgid */
+	{ 0, (sy_call_t *)sys_getgid, AUE_GETGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 47 = getgid */
 	{ compat(AS(osigprocmask_args),sigprocmask), AUE_SIGPROCMASK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 48 = old sigprocmask */
-	{ AS(getlogin_args), (sy_call_t *)getlogin, AUE_GETLOGIN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 49 = getlogin */
-	{ AS(setlogin_args), (sy_call_t *)setlogin, AUE_SETLOGIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = setlogin */
-	{ AS(acct_args), (sy_call_t *)acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
+	{ AS(getlogin_args), (sy_call_t *)sys_getlogin, AUE_GETLOGIN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 49 = getlogin */
+	{ AS(setlogin_args), (sy_call_t *)sys_setlogin, AUE_SETLOGIN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 50 = setlogin */
+	{ AS(acct_args), (sy_call_t *)sys_acct, AUE_ACCT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 51 = acct */
 	{ compat(0,sigpending), AUE_SIGPENDING, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 52 = old sigpending */
-	{ AS(sigaltstack_args), (sy_call_t *)sigaltstack, AUE_SIGALTSTACK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 53 = sigaltstack */
-	{ AS(ioctl_args), (sy_call_t *)ioctl, AUE_IOCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 54 = ioctl */
-	{ AS(reboot_args), (sy_call_t *)reboot, AUE_REBOOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = reboot */
-	{ AS(revoke_args), (sy_call_t *)revoke, AUE_REVOKE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 56 = revoke */
-	{ AS(symlink_args), (sy_call_t *)symlink, AUE_SYMLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = symlink */
-	{ AS(readlink_args), (sy_call_t *)readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = readlink */
-	{ AS(execve_args), (sy_call_t *)execve, AUE_EXECVE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = execve */
-	{ AS(umask_args), (sy_call_t *)umask, AUE_UMASK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 60 = umask */
-	{ AS(chroot_args), (sy_call_t *)chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
+	{ AS(sigaltstack_args), (sy_call_t *)sys_sigaltstack, AUE_SIGALTSTACK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 53 = sigaltstack */
+	{ AS(ioctl_args), (sy_call_t *)sys_ioctl, AUE_IOCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 54 = ioctl */
+	{ AS(reboot_args), (sy_call_t *)sys_reboot, AUE_REBOOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 55 = reboot */
+	{ AS(revoke_args), (sy_call_t *)sys_revoke, AUE_REVOKE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 56 = revoke */
+	{ AS(symlink_args), (sy_call_t *)sys_symlink, AUE_SYMLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 57 = symlink */
+	{ AS(readlink_args), (sy_call_t *)sys_readlink, AUE_READLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 58 = readlink */
+	{ AS(execve_args), (sy_call_t *)sys_execve, AUE_EXECVE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 59 = execve */
+	{ AS(umask_args), (sy_call_t *)sys_umask, AUE_UMASK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 60 = umask */
+	{ AS(chroot_args), (sy_call_t *)sys_chroot, AUE_CHROOT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 61 = chroot */
 	{ compat(AS(ofstat_args),fstat), AUE_FSTAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 62 = old fstat */
 	{ compat(AS(getkerninfo_args),getkerninfo), AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 63 = old getkerninfo */
 	{ compat(0,getpagesize), AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 64 = old getpagesize */
-	{ AS(msync_args), (sy_call_t *)msync, AUE_MSYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 65 = msync */
-	{ 0, (sy_call_t *)vfork, AUE_VFORK, NULL, 0, 0, 0, SY_THR_STATIC },		/* 66 = vfork */
+	{ AS(msync_args), (sy_call_t *)sys_msync, AUE_MSYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 65 = msync */
+	{ 0, (sy_call_t *)sys_vfork, AUE_VFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 66 = vfork */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 67 = obsolete vread */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 68 = obsolete vwrite */
-	{ AS(sbrk_args), (sy_call_t *)sbrk, AUE_SBRK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 69 = sbrk */
-	{ AS(sstk_args), (sy_call_t *)sstk, AUE_SSTK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 70 = sstk */
+	{ AS(sbrk_args), (sy_call_t *)sys_sbrk, AUE_SBRK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 69 = sbrk */
+	{ AS(sstk_args), (sy_call_t *)sys_sstk, AUE_SSTK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 70 = sstk */
 	{ compat(AS(ommap_args),mmap), AUE_MMAP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 71 = old mmap */
-	{ AS(ovadvise_args), (sy_call_t *)ovadvise, AUE_O_VADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 72 = vadvise */
-	{ AS(munmap_args), (sy_call_t *)munmap, AUE_MUNMAP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 73 = munmap */
-	{ AS(mprotect_args), (sy_call_t *)mprotect, AUE_MPROTECT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 74 = mprotect */
-	{ AS(madvise_args), (sy_call_t *)madvise, AUE_MADVISE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 75 = madvise */
+	{ AS(ovadvise_args), (sy_call_t *)sys_ovadvise, AUE_O_VADVISE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 72 = vadvise */
+	{ AS(munmap_args), (sy_call_t *)sys_munmap, AUE_MUNMAP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 73 = munmap */
+	{ AS(mprotect_args), (sy_call_t *)sys_mprotect, AUE_MPROTECT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 74 = mprotect */
+	{ AS(madvise_args), (sy_call_t *)sys_madvise, AUE_MADVISE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 75 = madvise */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 76 = obsolete vhangup */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 77 = obsolete vlimit */
-	{ AS(mincore_args), (sy_call_t *)mincore, AUE_MINCORE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 78 = mincore */
-	{ AS(getgroups_args), (sy_call_t *)getgroups, AUE_GETGROUPS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 79 = getgroups */
-	{ AS(setgroups_args), (sy_call_t *)setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = setgroups */
-	{ 0, (sy_call_t *)getpgrp, AUE_GETPGRP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 81 = getpgrp */
-	{ AS(setpgid_args), (sy_call_t *)setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 82 = setpgid */
-	{ AS(setitimer_args), (sy_call_t *)setitimer, AUE_SETITIMER, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 83 = setitimer */
+	{ AS(mincore_args), (sy_call_t *)sys_mincore, AUE_MINCORE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 78 = mincore */
+	{ AS(getgroups_args), (sy_call_t *)sys_getgroups, AUE_GETGROUPS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 79 = getgroups */
+	{ AS(setgroups_args), (sy_call_t *)sys_setgroups, AUE_SETGROUPS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 80 = setgroups */
+	{ 0, (sy_call_t *)sys_getpgrp, AUE_GETPGRP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 81 = getpgrp */
+	{ AS(setpgid_args), (sy_call_t *)sys_setpgid, AUE_SETPGRP, NULL, 0, 0, 0, SY_THR_STATIC },	/* 82 = setpgid */
+	{ AS(setitimer_args), (sy_call_t *)sys_setitimer, AUE_SETITIMER, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 83 = setitimer */
 	{ compat(0,wait), AUE_WAIT4, NULL, 0, 0, 0, SY_THR_STATIC },			/* 84 = old wait */
-	{ AS(swapon_args), (sy_call_t *)swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = swapon */
-	{ AS(getitimer_args), (sy_call_t *)getitimer, AUE_GETITIMER, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 86 = getitimer */
+	{ AS(swapon_args), (sy_call_t *)sys_swapon, AUE_SWAPON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 85 = swapon */
+	{ AS(getitimer_args), (sy_call_t *)sys_getitimer, AUE_GETITIMER, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 86 = getitimer */
 	{ compat(AS(gethostname_args),gethostname), AUE_SYSCTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 87 = old gethostname */
 	{ compat(AS(sethostname_args),sethostname), AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 88 = old sethostname */
-	{ 0, (sy_call_t *)getdtablesize, AUE_GETDTABLESIZE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = getdtablesize */
-	{ AS(dup2_args), (sy_call_t *)dup2, AUE_DUP2, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 90 = dup2 */
+	{ 0, (sy_call_t *)sys_getdtablesize, AUE_GETDTABLESIZE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 89 = getdtablesize */
+	{ AS(dup2_args), (sy_call_t *)sys_dup2, AUE_DUP2, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 90 = dup2 */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 91 = getdopt */
-	{ AS(fcntl_args), (sy_call_t *)fcntl, AUE_FCNTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 92 = fcntl */
-	{ AS(select_args), (sy_call_t *)select, AUE_SELECT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 93 = select */
+	{ AS(fcntl_args), (sy_call_t *)sys_fcntl, AUE_FCNTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 92 = fcntl */
+	{ AS(select_args), (sy_call_t *)sys_select, AUE_SELECT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 93 = select */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 94 = setdopt */
-	{ AS(fsync_args), (sy_call_t *)fsync, AUE_FSYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 95 = fsync */
-	{ AS(setpriority_args), (sy_call_t *)setpriority, AUE_SETPRIORITY, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 96 = setpriority */
-	{ AS(socket_args), (sy_call_t *)socket, AUE_SOCKET, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 97 = socket */
-	{ AS(connect_args), (sy_call_t *)connect, AUE_CONNECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 98 = connect */
+	{ AS(fsync_args), (sy_call_t *)sys_fsync, AUE_FSYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 95 = fsync */
+	{ AS(setpriority_args), (sy_call_t *)sys_setpriority, AUE_SETPRIORITY, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 96 = setpriority */
+	{ AS(socket_args), (sy_call_t *)sys_socket, AUE_SOCKET, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 97 = socket */
+	{ AS(connect_args), (sy_call_t *)sys_connect, AUE_CONNECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 98 = connect */
 	{ compat(AS(accept_args),accept), AUE_ACCEPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 99 = old accept */
-	{ AS(getpriority_args), (sy_call_t *)getpriority, AUE_GETPRIORITY, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 100 = getpriority */
+	{ AS(getpriority_args), (sy_call_t *)sys_getpriority, AUE_GETPRIORITY, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 100 = getpriority */
 	{ compat(AS(osend_args),send), AUE_SEND, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 101 = old send */
 	{ compat(AS(orecv_args),recv), AUE_RECV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 102 = old recv */
 	{ compat(AS(osigreturn_args),sigreturn), AUE_SIGRETURN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 103 = old sigreturn */
-	{ AS(bind_args), (sy_call_t *)bind, AUE_BIND, NULL, 0, 0, 0, SY_THR_STATIC },	/* 104 = bind */
-	{ AS(setsockopt_args), (sy_call_t *)setsockopt, AUE_SETSOCKOPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 105 = setsockopt */
-	{ AS(listen_args), (sy_call_t *)listen, AUE_LISTEN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 106 = listen */
+	{ AS(bind_args), (sy_call_t *)sys_bind, AUE_BIND, NULL, 0, 0, 0, SY_THR_STATIC },	/* 104 = bind */
+	{ AS(setsockopt_args), (sy_call_t *)sys_setsockopt, AUE_SETSOCKOPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 105 = setsockopt */
+	{ AS(listen_args), (sy_call_t *)sys_listen, AUE_LISTEN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 106 = listen */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 107 = obsolete vtimes */
 	{ compat(AS(osigvec_args),sigvec), AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 108 = old sigvec */
 	{ compat(AS(osigblock_args),sigblock), AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 109 = old sigblock */
@@ -150,39 +150,39 @@ struct sysent sysent[] = {
 	{ compat(AS(orecvmsg_args),recvmsg), AUE_RECVMSG, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 113 = old recvmsg */
 	{ compat(AS(osendmsg_args),sendmsg), AUE_SENDMSG, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 114 = old sendmsg */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 115 = obsolete vtrace */
-	{ AS(gettimeofday_args), (sy_call_t *)gettimeofday, AUE_GETTIMEOFDAY, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 116 = gettimeofday */
-	{ AS(getrusage_args), (sy_call_t *)getrusage, AUE_GETRUSAGE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 117 = getrusage */
-	{ AS(getsockopt_args), (sy_call_t *)getsockopt, AUE_GETSOCKOPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 118 = getsockopt */
+	{ AS(gettimeofday_args), (sy_call_t *)sys_gettimeofday, AUE_GETTIMEOFDAY, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 116 = gettimeofday */
+	{ AS(getrusage_args), (sy_call_t *)sys_getrusage, AUE_GETRUSAGE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 117 = getrusage */
+	{ AS(getsockopt_args), (sy_call_t *)sys_getsockopt, AUE_GETSOCKOPT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 118 = getsockopt */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 119 = resuba */
-	{ AS(readv_args), (sy_call_t *)readv, AUE_READV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 120 = readv */
-	{ AS(writev_args), (sy_call_t *)writev, AUE_WRITEV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 121 = writev */
-	{ AS(settimeofday_args), (sy_call_t *)settimeofday, AUE_SETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 122 = settimeofday */
-	{ AS(fchown_args), (sy_call_t *)fchown, AUE_FCHOWN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 123 = fchown */
-	{ AS(fchmod_args), (sy_call_t *)fchmod, AUE_FCHMOD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 124 = fchmod */
+	{ AS(readv_args), (sy_call_t *)sys_readv, AUE_READV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 120 = readv */
+	{ AS(writev_args), (sy_call_t *)sys_writev, AUE_WRITEV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 121 = writev */
+	{ AS(settimeofday_args), (sy_call_t *)sys_settimeofday, AUE_SETTIMEOFDAY, NULL, 0, 0, 0, SY_THR_STATIC },	/* 122 = settimeofday */
+	{ AS(fchown_args), (sy_call_t *)sys_fchown, AUE_FCHOWN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 123 = fchown */
+	{ AS(fchmod_args), (sy_call_t *)sys_fchmod, AUE_FCHMOD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 124 = fchmod */
 	{ compat(AS(recvfrom_args),recvfrom), AUE_RECVFROM, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 125 = old recvfrom */
-	{ AS(setreuid_args), (sy_call_t *)setreuid, AUE_SETREUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 126 = setreuid */
-	{ AS(setregid_args), (sy_call_t *)setregid, AUE_SETREGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 127 = setregid */
-	{ AS(rename_args), (sy_call_t *)rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 128 = rename */
+	{ AS(setreuid_args), (sy_call_t *)sys_setreuid, AUE_SETREUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 126 = setreuid */
+	{ AS(setregid_args), (sy_call_t *)sys_setregid, AUE_SETREGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 127 = setregid */
+	{ AS(rename_args), (sy_call_t *)sys_rename, AUE_RENAME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 128 = rename */
 	{ compat(AS(otruncate_args),truncate), AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 129 = old truncate */
 	{ compat(AS(oftruncate_args),ftruncate), AUE_FTRUNCATE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 130 = old ftruncate */
-	{ AS(flock_args), (sy_call_t *)flock, AUE_FLOCK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 131 = flock */
-	{ AS(mkfifo_args), (sy_call_t *)mkfifo, AUE_MKFIFO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = mkfifo */
-	{ AS(sendto_args), (sy_call_t *)sendto, AUE_SENDTO, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 133 = sendto */
-	{ AS(shutdown_args), (sy_call_t *)shutdown, AUE_SHUTDOWN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 134 = shutdown */
-	{ AS(socketpair_args), (sy_call_t *)socketpair, AUE_SOCKETPAIR, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 135 = socketpair */
-	{ AS(mkdir_args), (sy_call_t *)mkdir, AUE_MKDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = mkdir */
-	{ AS(rmdir_args), (sy_call_t *)rmdir, AUE_RMDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 137 = rmdir */
-	{ AS(utimes_args), (sy_call_t *)utimes, AUE_UTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 138 = utimes */
+	{ AS(flock_args), (sy_call_t *)sys_flock, AUE_FLOCK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 131 = flock */
+	{ AS(mkfifo_args), (sy_call_t *)sys_mkfifo, AUE_MKFIFO, NULL, 0, 0, 0, SY_THR_STATIC },	/* 132 = mkfifo */
+	{ AS(sendto_args), (sy_call_t *)sys_sendto, AUE_SENDTO, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 133 = sendto */
+	{ AS(shutdown_args), (sy_call_t *)sys_shutdown, AUE_SHUTDOWN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 134 = shutdown */
+	{ AS(socketpair_args), (sy_call_t *)sys_socketpair, AUE_SOCKETPAIR, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 135 = socketpair */
+	{ AS(mkdir_args), (sy_call_t *)sys_mkdir, AUE_MKDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 136 = mkdir */
+	{ AS(rmdir_args), (sy_call_t *)sys_rmdir, AUE_RMDIR, NULL, 0, 0, 0, SY_THR_STATIC },	/* 137 = rmdir */
+	{ AS(utimes_args), (sy_call_t *)sys_utimes, AUE_UTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 138 = utimes */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 139 = obsolete 4.2 sigreturn */
-	{ AS(adjtime_args), (sy_call_t *)adjtime, AUE_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 140 = adjtime */
+	{ AS(adjtime_args), (sy_call_t *)sys_adjtime, AUE_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 140 = adjtime */
 	{ compat(AS(ogetpeername_args),getpeername), AUE_GETPEERNAME, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 141 = old getpeername */
 	{ compat(0,gethostid), AUE_SYSCTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 142 = old gethostid */
 	{ compat(AS(osethostid_args),sethostid), AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 143 = old sethostid */
 	{ compat(AS(ogetrlimit_args),getrlimit), AUE_GETRLIMIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 144 = old getrlimit */
 	{ compat(AS(osetrlimit_args),setrlimit), AUE_SETRLIMIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 145 = old setrlimit */
 	{ compat(AS(okillpg_args),killpg), AUE_KILLPG, NULL, 0, 0, 0, SY_THR_STATIC },	/* 146 = old killpg */
-	{ 0, (sy_call_t *)setsid, AUE_SETSID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 147 = setsid */
-	{ AS(quotactl_args), (sy_call_t *)quotactl, AUE_QUOTACTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 148 = quotactl */
+	{ 0, (sy_call_t *)sys_setsid, AUE_SETSID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 147 = setsid */
+	{ AS(quotactl_args), (sy_call_t *)sys_quotactl, AUE_QUOTACTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 148 = quotactl */
 	{ compat(0,quota), AUE_O_QUOTA, NULL, 0, 0, 0, SY_THR_STATIC },		/* 149 = old quota */
 	{ compat(AS(getsockname_args),getsockname), AUE_GETSOCKNAME, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 150 = old getsockname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 151 = sem_lock */
@@ -194,13 +194,13 @@ struct sysent sysent[] = {
 	{ compat4(AS(freebsd4_statfs_args),statfs), AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 157 = freebsd4 statfs */
 	{ compat4(AS(freebsd4_fstatfs_args),fstatfs), AUE_FSTATFS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 158 = freebsd4 fstatfs */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 159 = nosys */
-	{ AS(lgetfh_args), (sy_call_t *)lgetfh, AUE_LGETFH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 160 = lgetfh */
-	{ AS(getfh_args), (sy_call_t *)getfh, AUE_NFS_GETFH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = getfh */
+	{ AS(lgetfh_args), (sy_call_t *)sys_lgetfh, AUE_LGETFH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 160 = lgetfh */
+	{ AS(getfh_args), (sy_call_t *)sys_getfh, AUE_NFS_GETFH, NULL, 0, 0, 0, SY_THR_STATIC },	/* 161 = getfh */
 	{ compat4(AS(freebsd4_getdomainname_args),getdomainname), AUE_SYSCTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 162 = freebsd4 getdomainname */
 	{ compat4(AS(freebsd4_setdomainname_args),setdomainname), AUE_SYSCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 163 = freebsd4 setdomainname */
 	{ compat4(AS(freebsd4_uname_args),uname), AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 164 = freebsd4 uname */
 	{ AS(sysarch_args), (sy_call_t *)sysarch, AUE_SYSARCH, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 165 = sysarch */
-	{ AS(rtprio_args), (sy_call_t *)rtprio, AUE_RTPRIO, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 166 = rtprio */
+	{ AS(rtprio_args), (sy_call_t *)sys_rtprio, AUE_RTPRIO, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 166 = rtprio */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 167 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 168 = nosys */
 	{ AS(semsys_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 169 = semsys */
@@ -209,41 +209,41 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 172 = nosys */
 	{ AS(freebsd6_pread_args), (sy_call_t *)freebsd6_pread, AUE_PREAD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 173 = freebsd6_pread */
 	{ AS(freebsd6_pwrite_args), (sy_call_t *)freebsd6_pwrite, AUE_PWRITE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 174 = freebsd6_pwrite */
-	{ AS(setfib_args), (sy_call_t *)setfib, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 175 = setfib */
-	{ AS(ntp_adjtime_args), (sy_call_t *)ntp_adjtime, AUE_NTP_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 176 = ntp_adjtime */
+	{ AS(setfib_args), (sy_call_t *)sys_setfib, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 175 = setfib */
+	{ AS(ntp_adjtime_args), (sy_call_t *)sys_ntp_adjtime, AUE_NTP_ADJTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 176 = ntp_adjtime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 177 = sfork */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 178 = getdescriptor */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 179 = setdescriptor */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 180 = nosys */
-	{ AS(setgid_args), (sy_call_t *)setgid, AUE_SETGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 181 = setgid */
-	{ AS(setegid_args), (sy_call_t *)setegid, AUE_SETEGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 182 = setegid */
-	{ AS(seteuid_args), (sy_call_t *)seteuid, AUE_SETEUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 183 = seteuid */
+	{ AS(setgid_args), (sy_call_t *)sys_setgid, AUE_SETGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 181 = setgid */
+	{ AS(setegid_args), (sy_call_t *)sys_setegid, AUE_SETEGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 182 = setegid */
+	{ AS(seteuid_args), (sy_call_t *)sys_seteuid, AUE_SETEUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 183 = seteuid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 184 = lfs_bmapv */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 185 = lfs_markv */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 186 = lfs_segclean */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 187 = lfs_segwait */
-	{ AS(stat_args), (sy_call_t *)stat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 188 = stat */
-	{ AS(fstat_args), (sy_call_t *)fstat, AUE_FSTAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 189 = fstat */
-	{ AS(lstat_args), (sy_call_t *)lstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 190 = lstat */
-	{ AS(pathconf_args), (sy_call_t *)pathconf, AUE_PATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 191 = pathconf */
-	{ AS(fpathconf_args), (sy_call_t *)fpathconf, AUE_FPATHCONF, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 192 = fpathconf */
+	{ AS(stat_args), (sy_call_t *)sys_stat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 188 = stat */
+	{ AS(fstat_args), (sy_call_t *)sys_fstat, AUE_FSTAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 189 = fstat */
+	{ AS(lstat_args), (sy_call_t *)sys_lstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 190 = lstat */
+	{ AS(pathconf_args), (sy_call_t *)sys_pathconf, AUE_PATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 191 = pathconf */
+	{ AS(fpathconf_args), (sy_call_t *)sys_fpathconf, AUE_FPATHCONF, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 192 = fpathconf */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 193 = nosys */
-	{ AS(__getrlimit_args), (sy_call_t *)getrlimit, AUE_GETRLIMIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 194 = getrlimit */
-	{ AS(__setrlimit_args), (sy_call_t *)setrlimit, AUE_SETRLIMIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 195 = setrlimit */
-	{ AS(getdirentries_args), (sy_call_t *)getdirentries, AUE_GETDIRENTRIES, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 196 = getdirentries */
+	{ AS(__getrlimit_args), (sy_call_t *)sys_getrlimit, AUE_GETRLIMIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 194 = getrlimit */
+	{ AS(__setrlimit_args), (sy_call_t *)sys_setrlimit, AUE_SETRLIMIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 195 = setrlimit */
+	{ AS(getdirentries_args), (sy_call_t *)sys_getdirentries, AUE_GETDIRENTRIES, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 196 = getdirentries */
 	{ AS(freebsd6_mmap_args), (sy_call_t *)freebsd6_mmap, AUE_MMAP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 197 = freebsd6_mmap */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },		/* 198 = __syscall */
 	{ AS(freebsd6_lseek_args), (sy_call_t *)freebsd6_lseek, AUE_LSEEK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 199 = freebsd6_lseek */
 	{ AS(freebsd6_truncate_args), (sy_call_t *)freebsd6_truncate, AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 200 = freebsd6_truncate */
 	{ AS(freebsd6_ftruncate_args), (sy_call_t *)freebsd6_ftruncate, AUE_FTRUNCATE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 201 = freebsd6_ftruncate */
-	{ AS(sysctl_args), (sy_call_t *)__sysctl, AUE_SYSCTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 202 = __sysctl */
-	{ AS(mlock_args), (sy_call_t *)mlock, AUE_MLOCK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 203 = mlock */
-	{ AS(munlock_args), (sy_call_t *)munlock, AUE_MUNLOCK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 204 = munlock */
-	{ AS(undelete_args), (sy_call_t *)undelete, AUE_UNDELETE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 205 = undelete */
-	{ AS(futimes_args), (sy_call_t *)futimes, AUE_FUTIMES, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 206 = futimes */
-	{ AS(getpgid_args), (sy_call_t *)getpgid, AUE_GETPGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 207 = getpgid */
+	{ AS(sysctl_args), (sy_call_t *)sys___sysctl, AUE_SYSCTL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 202 = __sysctl */
+	{ AS(mlock_args), (sy_call_t *)sys_mlock, AUE_MLOCK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 203 = mlock */
+	{ AS(munlock_args), (sy_call_t *)sys_munlock, AUE_MUNLOCK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 204 = munlock */
+	{ AS(undelete_args), (sy_call_t *)sys_undelete, AUE_UNDELETE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 205 = undelete */
+	{ AS(futimes_args), (sy_call_t *)sys_futimes, AUE_FUTIMES, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 206 = futimes */
+	{ AS(getpgid_args), (sy_call_t *)sys_getpgid, AUE_GETPGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 207 = getpgid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 208 = newreboot */
-	{ AS(poll_args), (sy_call_t *)poll, AUE_POLL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 209 = poll */
+	{ AS(poll_args), (sy_call_t *)sys_poll, AUE_POLL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 209 = poll */
 	{ AS(nosys_args), (sy_call_t *)lkmnosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 210 = lkmnosys */
 	{ AS(nosys_args), (sy_call_t *)lkmnosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 211 = lkmnosys */
 	{ AS(nosys_args), (sy_call_t *)lkmnosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 212 = lkmnosys */
@@ -266,15 +266,15 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },		/* 229 = freebsd7 shmctl */
 	{ AS(shmdt_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 230 = shmdt */
 	{ AS(shmget_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 231 = shmget */
-	{ AS(clock_gettime_args), (sy_call_t *)clock_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 232 = clock_gettime */
-	{ AS(clock_settime_args), (sy_call_t *)clock_settime, AUE_CLOCK_SETTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 233 = clock_settime */
-	{ AS(clock_getres_args), (sy_call_t *)clock_getres, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 234 = clock_getres */
-	{ AS(ktimer_create_args), (sy_call_t *)ktimer_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 235 = ktimer_create */
-	{ AS(ktimer_delete_args), (sy_call_t *)ktimer_delete, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 236 = ktimer_delete */
-	{ AS(ktimer_settime_args), (sy_call_t *)ktimer_settime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 237 = ktimer_settime */
-	{ AS(ktimer_gettime_args), (sy_call_t *)ktimer_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 238 = ktimer_gettime */
-	{ AS(ktimer_getoverrun_args), (sy_call_t *)ktimer_getoverrun, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 239 = ktimer_getoverrun */
-	{ AS(nanosleep_args), (sy_call_t *)nanosleep, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 240 = nanosleep */
+	{ AS(clock_gettime_args), (sy_call_t *)sys_clock_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 232 = clock_gettime */
+	{ AS(clock_settime_args), (sy_call_t *)sys_clock_settime, AUE_CLOCK_SETTIME, NULL, 0, 0, 0, SY_THR_STATIC },	/* 233 = clock_settime */
+	{ AS(clock_getres_args), (sy_call_t *)sys_clock_getres, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 234 = clock_getres */
+	{ AS(ktimer_create_args), (sy_call_t *)sys_ktimer_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 235 = ktimer_create */
+	{ AS(ktimer_delete_args), (sy_call_t *)sys_ktimer_delete, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 236 = ktimer_delete */
+	{ AS(ktimer_settime_args), (sy_call_t *)sys_ktimer_settime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 237 = ktimer_settime */
+	{ AS(ktimer_gettime_args), (sy_call_t *)sys_ktimer_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 238 = ktimer_gettime */
+	{ AS(ktimer_getoverrun_args), (sy_call_t *)sys_ktimer_getoverrun, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 239 = ktimer_getoverrun */
+	{ AS(nanosleep_args), (sy_call_t *)sys_nanosleep, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 240 = nanosleep */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 241 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 242 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 243 = nosys */
@@ -282,13 +282,13 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 245 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 246 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 247 = nosys */
-	{ AS(ntp_gettime_args), (sy_call_t *)ntp_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 248 = ntp_gettime */
+	{ AS(ntp_gettime_args), (sy_call_t *)sys_ntp_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 248 = ntp_gettime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 249 = nosys */
-	{ AS(minherit_args), (sy_call_t *)minherit, AUE_MINHERIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 250 = minherit */
-	{ AS(rfork_args), (sy_call_t *)rfork, AUE_RFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 251 = rfork */
-	{ AS(openbsd_poll_args), (sy_call_t *)openbsd_poll, AUE_POLL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 252 = openbsd_poll */
-	{ 0, (sy_call_t *)issetugid, AUE_ISSETUGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 253 = issetugid */
-	{ AS(lchown_args), (sy_call_t *)lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 254 = lchown */
+	{ AS(minherit_args), (sy_call_t *)sys_minherit, AUE_MINHERIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 250 = minherit */
+	{ AS(rfork_args), (sy_call_t *)sys_rfork, AUE_RFORK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 251 = rfork */
+	{ AS(openbsd_poll_args), (sy_call_t *)sys_openbsd_poll, AUE_POLL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 252 = openbsd_poll */
+	{ 0, (sy_call_t *)sys_issetugid, AUE_ISSETUGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 253 = issetugid */
+	{ AS(lchown_args), (sy_call_t *)sys_lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 254 = lchown */
 	{ AS(aio_read_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 255 = aio_read */
 	{ AS(aio_write_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 256 = aio_write */
 	{ AS(lio_listio_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 257 = lio_listio */
@@ -306,15 +306,15 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 269 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 270 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 271 = nosys */
-	{ AS(getdents_args), (sy_call_t *)getdents, AUE_O_GETDENTS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 272 = getdents */
+	{ AS(getdents_args), (sy_call_t *)sys_getdents, AUE_O_GETDENTS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 272 = getdents */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 273 = nosys */
-	{ AS(lchmod_args), (sy_call_t *)lchmod, AUE_LCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 274 = lchmod */
-	{ AS(lchown_args), (sy_call_t *)lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 275 = netbsd_lchown */
-	{ AS(lutimes_args), (sy_call_t *)lutimes, AUE_LUTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 276 = lutimes */
-	{ AS(msync_args), (sy_call_t *)msync, AUE_MSYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 277 = netbsd_msync */
-	{ AS(nstat_args), (sy_call_t *)nstat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 278 = nstat */
-	{ AS(nfstat_args), (sy_call_t *)nfstat, AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 279 = nfstat */
-	{ AS(nlstat_args), (sy_call_t *)nlstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 280 = nlstat */
+	{ AS(lchmod_args), (sy_call_t *)sys_lchmod, AUE_LCHMOD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 274 = lchmod */
+	{ AS(lchown_args), (sy_call_t *)sys_lchown, AUE_LCHOWN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 275 = netbsd_lchown */
+	{ AS(lutimes_args), (sy_call_t *)sys_lutimes, AUE_LUTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 276 = lutimes */
+	{ AS(msync_args), (sy_call_t *)sys_msync, AUE_MSYNC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 277 = netbsd_msync */
+	{ AS(nstat_args), (sy_call_t *)sys_nstat, AUE_STAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 278 = nstat */
+	{ AS(nfstat_args), (sy_call_t *)sys_nfstat, AUE_FSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 279 = nfstat */
+	{ AS(nlstat_args), (sy_call_t *)sys_nlstat, AUE_LSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 280 = nlstat */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 281 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 282 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 283 = nosys */
@@ -323,8 +323,8 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 286 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 287 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 288 = nosys */
-	{ AS(preadv_args), (sy_call_t *)preadv, AUE_PREADV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 289 = preadv */
-	{ AS(pwritev_args), (sy_call_t *)pwritev, AUE_PWRITEV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 290 = pwritev */
+	{ AS(preadv_args), (sy_call_t *)sys_preadv, AUE_PREADV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 289 = preadv */
+	{ AS(pwritev_args), (sy_call_t *)sys_pwritev, AUE_PWRITEV, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 290 = pwritev */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 291 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 292 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 293 = nosys */
@@ -332,21 +332,21 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 295 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 296 = nosys */
 	{ compat4(AS(freebsd4_fhstatfs_args),fhstatfs), AUE_FHSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 297 = freebsd4 fhstatfs */
-	{ AS(fhopen_args), (sy_call_t *)fhopen, AUE_FHOPEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 298 = fhopen */
-	{ AS(fhstat_args), (sy_call_t *)fhstat, AUE_FHSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 299 = fhstat */
-	{ AS(modnext_args), (sy_call_t *)modnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 300 = modnext */
-	{ AS(modstat_args), (sy_call_t *)modstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 301 = modstat */
-	{ AS(modfnext_args), (sy_call_t *)modfnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 302 = modfnext */
-	{ AS(modfind_args), (sy_call_t *)modfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 303 = modfind */
-	{ AS(kldload_args), (sy_call_t *)kldload, AUE_MODLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 304 = kldload */
-	{ AS(kldunload_args), (sy_call_t *)kldunload, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 305 = kldunload */
-	{ AS(kldfind_args), (sy_call_t *)kldfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 306 = kldfind */
-	{ AS(kldnext_args), (sy_call_t *)kldnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 307 = kldnext */
-	{ AS(kldstat_args), (sy_call_t *)kldstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 308 = kldstat */
-	{ AS(kldfirstmod_args), (sy_call_t *)kldfirstmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 309 = kldfirstmod */
-	{ AS(getsid_args), (sy_call_t *)getsid, AUE_GETSID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 310 = getsid */
-	{ AS(setresuid_args), (sy_call_t *)setresuid, AUE_SETRESUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 311 = setresuid */
-	{ AS(setresgid_args), (sy_call_t *)setresgid, AUE_SETRESGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 312 = setresgid */
+	{ AS(fhopen_args), (sy_call_t *)sys_fhopen, AUE_FHOPEN, NULL, 0, 0, 0, SY_THR_STATIC },	/* 298 = fhopen */
+	{ AS(fhstat_args), (sy_call_t *)sys_fhstat, AUE_FHSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 299 = fhstat */
+	{ AS(modnext_args), (sy_call_t *)sys_modnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 300 = modnext */
+	{ AS(modstat_args), (sy_call_t *)sys_modstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 301 = modstat */
+	{ AS(modfnext_args), (sy_call_t *)sys_modfnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 302 = modfnext */
+	{ AS(modfind_args), (sy_call_t *)sys_modfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 303 = modfind */
+	{ AS(kldload_args), (sy_call_t *)sys_kldload, AUE_MODLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 304 = kldload */
+	{ AS(kldunload_args), (sy_call_t *)sys_kldunload, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 305 = kldunload */
+	{ AS(kldfind_args), (sy_call_t *)sys_kldfind, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 306 = kldfind */
+	{ AS(kldnext_args), (sy_call_t *)sys_kldnext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 307 = kldnext */
+	{ AS(kldstat_args), (sy_call_t *)sys_kldstat, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 308 = kldstat */
+	{ AS(kldfirstmod_args), (sy_call_t *)sys_kldfirstmod, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 309 = kldfirstmod */
+	{ AS(getsid_args), (sy_call_t *)sys_getsid, AUE_GETSID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 310 = getsid */
+	{ AS(setresuid_args), (sy_call_t *)sys_setresuid, AUE_SETRESUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 311 = setresuid */
+	{ AS(setresgid_args), (sy_call_t *)sys_setresgid, AUE_SETRESGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 312 = setresgid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 313 = obsolete signanosleep */
 	{ AS(aio_return_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 314 = aio_return */
 	{ AS(aio_suspend_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 315 = aio_suspend */
@@ -355,49 +355,49 @@ struct sysent sysent[] = {
 	{ AS(oaio_read_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 318 = oaio_read */
 	{ AS(oaio_write_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 319 = oaio_write */
 	{ AS(olio_listio_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 320 = olio_listio */
-	{ 0, (sy_call_t *)yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 321 = yield */
+	{ 0, (sy_call_t *)sys_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 321 = yield */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 322 = obsolete thr_sleep */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 323 = obsolete thr_wakeup */
-	{ AS(mlockall_args), (sy_call_t *)mlockall, AUE_MLOCKALL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 324 = mlockall */
-	{ 0, (sy_call_t *)munlockall, AUE_MUNLOCKALL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 325 = munlockall */
-	{ AS(__getcwd_args), (sy_call_t *)__getcwd, AUE_GETCWD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 326 = __getcwd */
-	{ AS(sched_setparam_args), (sy_call_t *)sched_setparam, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 327 = sched_setparam */
-	{ AS(sched_getparam_args), (sy_call_t *)sched_getparam, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 328 = sched_getparam */
-	{ AS(sched_setscheduler_args), (sy_call_t *)sched_setscheduler, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 329 = sched_setscheduler */
-	{ AS(sched_getscheduler_args), (sy_call_t *)sched_getscheduler, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 330 = sched_getscheduler */
-	{ 0, (sy_call_t *)sched_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 331 = sched_yield */
-	{ AS(sched_get_priority_max_args), (sy_call_t *)sched_get_priority_max, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 332 = sched_get_priority_max */
-	{ AS(sched_get_priority_min_args), (sy_call_t *)sched_get_priority_min, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 333 = sched_get_priority_min */
-	{ AS(sched_rr_get_interval_args), (sy_call_t *)sched_rr_get_interval, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 334 = sched_rr_get_interval */
-	{ AS(utrace_args), (sy_call_t *)utrace, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 335 = utrace */
+	{ AS(mlockall_args), (sy_call_t *)sys_mlockall, AUE_MLOCKALL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 324 = mlockall */
+	{ 0, (sy_call_t *)sys_munlockall, AUE_MUNLOCKALL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 325 = munlockall */
+	{ AS(__getcwd_args), (sy_call_t *)sys___getcwd, AUE_GETCWD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 326 = __getcwd */
+	{ AS(sched_setparam_args), (sy_call_t *)sys_sched_setparam, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 327 = sched_setparam */
+	{ AS(sched_getparam_args), (sy_call_t *)sys_sched_getparam, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 328 = sched_getparam */
+	{ AS(sched_setscheduler_args), (sy_call_t *)sys_sched_setscheduler, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 329 = sched_setscheduler */
+	{ AS(sched_getscheduler_args), (sy_call_t *)sys_sched_getscheduler, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 330 = sched_getscheduler */
+	{ 0, (sy_call_t *)sys_sched_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 331 = sched_yield */
+	{ AS(sched_get_priority_max_args), (sy_call_t *)sys_sched_get_priority_max, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 332 = sched_get_priority_max */
+	{ AS(sched_get_priority_min_args), (sy_call_t *)sys_sched_get_priority_min, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 333 = sched_get_priority_min */
+	{ AS(sched_rr_get_interval_args), (sy_call_t *)sys_sched_rr_get_interval, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 334 = sched_rr_get_interval */
+	{ AS(utrace_args), (sy_call_t *)sys_utrace, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 335 = utrace */
 	{ compat4(AS(freebsd4_sendfile_args),sendfile), AUE_SENDFILE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 336 = freebsd4 sendfile */
-	{ AS(kldsym_args), (sy_call_t *)kldsym, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 337 = kldsym */
-	{ AS(jail_args), (sy_call_t *)jail, AUE_JAIL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 338 = jail */
+	{ AS(kldsym_args), (sy_call_t *)sys_kldsym, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 337 = kldsym */
+	{ AS(jail_args), (sy_call_t *)sys_jail, AUE_JAIL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 338 = jail */
 	{ AS(nnpfs_syscall_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 339 = nnpfs_syscall */
-	{ AS(sigprocmask_args), (sy_call_t *)sigprocmask, AUE_SIGPROCMASK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 340 = sigprocmask */
-	{ AS(sigsuspend_args), (sy_call_t *)sigsuspend, AUE_SIGSUSPEND, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 341 = sigsuspend */
+	{ AS(sigprocmask_args), (sy_call_t *)sys_sigprocmask, AUE_SIGPROCMASK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 340 = sigprocmask */
+	{ AS(sigsuspend_args), (sy_call_t *)sys_sigsuspend, AUE_SIGSUSPEND, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 341 = sigsuspend */
 	{ compat4(AS(freebsd4_sigaction_args),sigaction), AUE_SIGACTION, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 342 = freebsd4 sigaction */
-	{ AS(sigpending_args), (sy_call_t *)sigpending, AUE_SIGPENDING, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 343 = sigpending */
+	{ AS(sigpending_args), (sy_call_t *)sys_sigpending, AUE_SIGPENDING, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 343 = sigpending */
 	{ compat4(AS(freebsd4_sigreturn_args),sigreturn), AUE_SIGRETURN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 344 = freebsd4 sigreturn */
-	{ AS(sigtimedwait_args), (sy_call_t *)sigtimedwait, AUE_SIGWAIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 345 = sigtimedwait */
-	{ AS(sigwaitinfo_args), (sy_call_t *)sigwaitinfo, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 346 = sigwaitinfo */
-	{ AS(__acl_get_file_args), (sy_call_t *)__acl_get_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 347 = __acl_get_file */
-	{ AS(__acl_set_file_args), (sy_call_t *)__acl_set_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 348 = __acl_set_file */
-	{ AS(__acl_get_fd_args), (sy_call_t *)__acl_get_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 349 = __acl_get_fd */
-	{ AS(__acl_set_fd_args), (sy_call_t *)__acl_set_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 350 = __acl_set_fd */
-	{ AS(__acl_delete_file_args), (sy_call_t *)__acl_delete_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 351 = __acl_delete_file */
-	{ AS(__acl_delete_fd_args), (sy_call_t *)__acl_delete_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 352 = __acl_delete_fd */
-	{ AS(__acl_aclcheck_file_args), (sy_call_t *)__acl_aclcheck_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 353 = __acl_aclcheck_file */
-	{ AS(__acl_aclcheck_fd_args), (sy_call_t *)__acl_aclcheck_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 354 = __acl_aclcheck_fd */
-	{ AS(extattrctl_args), (sy_call_t *)extattrctl, AUE_EXTATTRCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 355 = extattrctl */
-	{ AS(extattr_set_file_args), (sy_call_t *)extattr_set_file, AUE_EXTATTR_SET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 356 = extattr_set_file */
-	{ AS(extattr_get_file_args), (sy_call_t *)extattr_get_file, AUE_EXTATTR_GET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 357 = extattr_get_file */
-	{ AS(extattr_delete_file_args), (sy_call_t *)extattr_delete_file, AUE_EXTATTR_DELETE_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 358 = extattr_delete_file */
+	{ AS(sigtimedwait_args), (sy_call_t *)sys_sigtimedwait, AUE_SIGWAIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 345 = sigtimedwait */
+	{ AS(sigwaitinfo_args), (sy_call_t *)sys_sigwaitinfo, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 346 = sigwaitinfo */
+	{ AS(__acl_get_file_args), (sy_call_t *)sys___acl_get_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 347 = __acl_get_file */
+	{ AS(__acl_set_file_args), (sy_call_t *)sys___acl_set_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 348 = __acl_set_file */
+	{ AS(__acl_get_fd_args), (sy_call_t *)sys___acl_get_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 349 = __acl_get_fd */
+	{ AS(__acl_set_fd_args), (sy_call_t *)sys___acl_set_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 350 = __acl_set_fd */
+	{ AS(__acl_delete_file_args), (sy_call_t *)sys___acl_delete_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 351 = __acl_delete_file */
+	{ AS(__acl_delete_fd_args), (sy_call_t *)sys___acl_delete_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 352 = __acl_delete_fd */
+	{ AS(__acl_aclcheck_file_args), (sy_call_t *)sys___acl_aclcheck_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 353 = __acl_aclcheck_file */
+	{ AS(__acl_aclcheck_fd_args), (sy_call_t *)sys___acl_aclcheck_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 354 = __acl_aclcheck_fd */
+	{ AS(extattrctl_args), (sy_call_t *)sys_extattrctl, AUE_EXTATTRCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 355 = extattrctl */
+	{ AS(extattr_set_file_args), (sy_call_t *)sys_extattr_set_file, AUE_EXTATTR_SET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 356 = extattr_set_file */
+	{ AS(extattr_get_file_args), (sy_call_t *)sys_extattr_get_file, AUE_EXTATTR_GET_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 357 = extattr_get_file */
+	{ AS(extattr_delete_file_args), (sy_call_t *)sys_extattr_delete_file, AUE_EXTATTR_DELETE_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 358 = extattr_delete_file */
 	{ AS(aio_waitcomplete_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 359 = aio_waitcomplete */
-	{ AS(getresuid_args), (sy_call_t *)getresuid, AUE_GETRESUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 360 = getresuid */
-	{ AS(getresgid_args), (sy_call_t *)getresgid, AUE_GETRESGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 361 = getresgid */
-	{ 0, (sy_call_t *)kqueue, AUE_KQUEUE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 362 = kqueue */
-	{ AS(kevent_args), (sy_call_t *)kevent, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 363 = kevent */
+	{ AS(getresuid_args), (sy_call_t *)sys_getresuid, AUE_GETRESUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 360 = getresuid */
+	{ AS(getresgid_args), (sy_call_t *)sys_getresgid, AUE_GETRESGID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 361 = getresgid */
+	{ 0, (sy_call_t *)sys_kqueue, AUE_KQUEUE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 362 = kqueue */
+	{ AS(kevent_args), (sy_call_t *)sys_kevent, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 363 = kevent */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 364 = __cap_get_proc */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 365 = __cap_set_proc */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 366 = __cap_get_fd */
@@ -405,34 +405,34 @@ struct sysent sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 368 = __cap_set_fd */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 369 = __cap_set_file */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 370 = nosys */
-	{ AS(extattr_set_fd_args), (sy_call_t *)extattr_set_fd, AUE_EXTATTR_SET_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 371 = extattr_set_fd */
-	{ AS(extattr_get_fd_args), (sy_call_t *)extattr_get_fd, AUE_EXTATTR_GET_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 372 = extattr_get_fd */
-	{ AS(extattr_delete_fd_args), (sy_call_t *)extattr_delete_fd, AUE_EXTATTR_DELETE_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 373 = extattr_delete_fd */
-	{ AS(__setugid_args), (sy_call_t *)__setugid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 374 = __setugid */
+	{ AS(extattr_set_fd_args), (sy_call_t *)sys_extattr_set_fd, AUE_EXTATTR_SET_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 371 = extattr_set_fd */
+	{ AS(extattr_get_fd_args), (sy_call_t *)sys_extattr_get_fd, AUE_EXTATTR_GET_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 372 = extattr_get_fd */
+	{ AS(extattr_delete_fd_args), (sy_call_t *)sys_extattr_delete_fd, AUE_EXTATTR_DELETE_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 373 = extattr_delete_fd */
+	{ AS(__setugid_args), (sy_call_t *)sys___setugid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 374 = __setugid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 375 = nfsclnt */
-	{ AS(eaccess_args), (sy_call_t *)eaccess, AUE_EACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 376 = eaccess */
+	{ AS(eaccess_args), (sy_call_t *)sys_eaccess, AUE_EACCESS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 376 = eaccess */
 	{ AS(afs3_syscall_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 377 = afs3_syscall */
-	{ AS(nmount_args), (sy_call_t *)nmount, AUE_NMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 378 = nmount */
+	{ AS(nmount_args), (sy_call_t *)sys_nmount, AUE_NMOUNT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 378 = nmount */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 379 = kse_exit */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 380 = kse_wakeup */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 381 = kse_create */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 382 = kse_thr_interrupt */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 383 = kse_release */
-	{ AS(__mac_get_proc_args), (sy_call_t *)__mac_get_proc, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 384 = __mac_get_proc */
-	{ AS(__mac_set_proc_args), (sy_call_t *)__mac_set_proc, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 385 = __mac_set_proc */
-	{ AS(__mac_get_fd_args), (sy_call_t *)__mac_get_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 386 = __mac_get_fd */
-	{ AS(__mac_get_file_args), (sy_call_t *)__mac_get_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 387 = __mac_get_file */
-	{ AS(__mac_set_fd_args), (sy_call_t *)__mac_set_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 388 = __mac_set_fd */
-	{ AS(__mac_set_file_args), (sy_call_t *)__mac_set_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 389 = __mac_set_file */
-	{ AS(kenv_args), (sy_call_t *)kenv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 390 = kenv */
-	{ AS(lchflags_args), (sy_call_t *)lchflags, AUE_LCHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 391 = lchflags */
-	{ AS(uuidgen_args), (sy_call_t *)uuidgen, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 392 = uuidgen */
-	{ AS(sendfile_args), (sy_call_t *)sendfile, AUE_SENDFILE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 393 = sendfile */
-	{ AS(mac_syscall_args), (sy_call_t *)mac_syscall, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 394 = mac_syscall */
-	{ AS(getfsstat_args), (sy_call_t *)getfsstat, AUE_GETFSSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 395 = getfsstat */
-	{ AS(statfs_args), (sy_call_t *)statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 396 = statfs */
-	{ AS(fstatfs_args), (sy_call_t *)fstatfs, AUE_FSTATFS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 397 = fstatfs */
-	{ AS(fhstatfs_args), (sy_call_t *)fhstatfs, AUE_FHSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 398 = fhstatfs */
+	{ AS(__mac_get_proc_args), (sy_call_t *)sys___mac_get_proc, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 384 = __mac_get_proc */
+	{ AS(__mac_set_proc_args), (sy_call_t *)sys___mac_set_proc, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 385 = __mac_set_proc */
+	{ AS(__mac_get_fd_args), (sy_call_t *)sys___mac_get_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 386 = __mac_get_fd */
+	{ AS(__mac_get_file_args), (sy_call_t *)sys___mac_get_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 387 = __mac_get_file */
+	{ AS(__mac_set_fd_args), (sy_call_t *)sys___mac_set_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 388 = __mac_set_fd */
+	{ AS(__mac_set_file_args), (sy_call_t *)sys___mac_set_file, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 389 = __mac_set_file */
+	{ AS(kenv_args), (sy_call_t *)sys_kenv, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 390 = kenv */
+	{ AS(lchflags_args), (sy_call_t *)sys_lchflags, AUE_LCHFLAGS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 391 = lchflags */
+	{ AS(uuidgen_args), (sy_call_t *)sys_uuidgen, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 392 = uuidgen */
+	{ AS(sendfile_args), (sy_call_t *)sys_sendfile, AUE_SENDFILE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 393 = sendfile */
+	{ AS(mac_syscall_args), (sy_call_t *)sys_mac_syscall, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 394 = mac_syscall */
+	{ AS(getfsstat_args), (sy_call_t *)sys_getfsstat, AUE_GETFSSTAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 395 = getfsstat */
+	{ AS(statfs_args), (sy_call_t *)sys_statfs, AUE_STATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 396 = statfs */
+	{ AS(fstatfs_args), (sy_call_t *)sys_fstatfs, AUE_FSTATFS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 397 = fstatfs */
+	{ AS(fhstatfs_args), (sy_call_t *)sys_fhstatfs, AUE_FHSTATFS, NULL, 0, 0, 0, SY_THR_STATIC },	/* 398 = fhstatfs */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 399 = nosys */
 	{ AS(ksem_close_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 400 = ksem_close */
 	{ AS(ksem_post_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 401 = ksem_post */
@@ -443,127 +443,127 @@ struct sysent sysent[] = {
 	{ AS(ksem_unlink_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 406 = ksem_unlink */
 	{ AS(ksem_getvalue_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 407 = ksem_getvalue */
 	{ AS(ksem_destroy_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 408 = ksem_destroy */
-	{ AS(__mac_get_pid_args), (sy_call_t *)__mac_get_pid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 409 = __mac_get_pid */
-	{ AS(__mac_get_link_args), (sy_call_t *)__mac_get_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 410 = __mac_get_link */
-	{ AS(__mac_set_link_args), (sy_call_t *)__mac_set_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 411 = __mac_set_link */
-	{ AS(extattr_set_link_args), (sy_call_t *)extattr_set_link, AUE_EXTATTR_SET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 412 = extattr_set_link */
-	{ AS(extattr_get_link_args), (sy_call_t *)extattr_get_link, AUE_EXTATTR_GET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 413 = extattr_get_link */
-	{ AS(extattr_delete_link_args), (sy_call_t *)extattr_delete_link, AUE_EXTATTR_DELETE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 414 = extattr_delete_link */
-	{ AS(__mac_execve_args), (sy_call_t *)__mac_execve, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 415 = __mac_execve */
-	{ AS(sigaction_args), (sy_call_t *)sigaction, AUE_SIGACTION, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 416 = sigaction */
-	{ AS(sigreturn_args), (sy_call_t *)sigreturn, AUE_SIGRETURN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 417 = sigreturn */
+	{ AS(__mac_get_pid_args), (sy_call_t *)sys___mac_get_pid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 409 = __mac_get_pid */
+	{ AS(__mac_get_link_args), (sy_call_t *)sys___mac_get_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 410 = __mac_get_link */
+	{ AS(__mac_set_link_args), (sy_call_t *)sys___mac_set_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 411 = __mac_set_link */
+	{ AS(extattr_set_link_args), (sy_call_t *)sys_extattr_set_link, AUE_EXTATTR_SET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 412 = extattr_set_link */
+	{ AS(extattr_get_link_args), (sy_call_t *)sys_extattr_get_link, AUE_EXTATTR_GET_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 413 = extattr_get_link */
+	{ AS(extattr_delete_link_args), (sy_call_t *)sys_extattr_delete_link, AUE_EXTATTR_DELETE_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 414 = extattr_delete_link */
+	{ AS(__mac_execve_args), (sy_call_t *)sys___mac_execve, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 415 = __mac_execve */
+	{ AS(sigaction_args), (sy_call_t *)sys_sigaction, AUE_SIGACTION, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 416 = sigaction */
+	{ AS(sigreturn_args), (sy_call_t *)sys_sigreturn, AUE_SIGRETURN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 417 = sigreturn */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 418 = __xstat */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 419 = __xfstat */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 420 = __xlstat */
-	{ AS(getcontext_args), (sy_call_t *)getcontext, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 421 = getcontext */
-	{ AS(setcontext_args), (sy_call_t *)setcontext, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 422 = setcontext */
-	{ AS(swapcontext_args), (sy_call_t *)swapcontext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 423 = swapcontext */
-	{ AS(swapoff_args), (sy_call_t *)swapoff, AUE_SWAPOFF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 424 = swapoff */
-	{ AS(__acl_get_link_args), (sy_call_t *)__acl_get_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 425 = __acl_get_link */
-	{ AS(__acl_set_link_args), (sy_call_t *)__acl_set_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 426 = __acl_set_link */
-	{ AS(__acl_delete_link_args), (sy_call_t *)__acl_delete_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 427 = __acl_delete_link */
-	{ AS(__acl_aclcheck_link_args), (sy_call_t *)__acl_aclcheck_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 428 = __acl_aclcheck_link */
-	{ AS(sigwait_args), (sy_call_t *)sigwait, AUE_SIGWAIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 429 = sigwait */
-	{ AS(thr_create_args), (sy_call_t *)thr_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 430 = thr_create */
-	{ AS(thr_exit_args), (sy_call_t *)thr_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 431 = thr_exit */
-	{ AS(thr_self_args), (sy_call_t *)thr_self, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 432 = thr_self */
-	{ AS(thr_kill_args), (sy_call_t *)thr_kill, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 433 = thr_kill */
-	{ AS(_umtx_lock_args), (sy_call_t *)_umtx_lock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 434 = _umtx_lock */
-	{ AS(_umtx_unlock_args), (sy_call_t *)_umtx_unlock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 435 = _umtx_unlock */
-	{ AS(jail_attach_args), (sy_call_t *)jail_attach, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 436 = jail_attach */
-	{ AS(extattr_list_fd_args), (sy_call_t *)extattr_list_fd, AUE_EXTATTR_LIST_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 437 = extattr_list_fd */
-	{ AS(extattr_list_file_args), (sy_call_t *)extattr_list_file, AUE_EXTATTR_LIST_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 438 = extattr_list_file */
-	{ AS(extattr_list_link_args), (sy_call_t *)extattr_list_link, AUE_EXTATTR_LIST_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 439 = extattr_list_link */
+	{ AS(getcontext_args), (sy_call_t *)sys_getcontext, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 421 = getcontext */
+	{ AS(setcontext_args), (sy_call_t *)sys_setcontext, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 422 = setcontext */
+	{ AS(swapcontext_args), (sy_call_t *)sys_swapcontext, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 423 = swapcontext */
+	{ AS(swapoff_args), (sy_call_t *)sys_swapoff, AUE_SWAPOFF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 424 = swapoff */
+	{ AS(__acl_get_link_args), (sy_call_t *)sys___acl_get_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 425 = __acl_get_link */
+	{ AS(__acl_set_link_args), (sy_call_t *)sys___acl_set_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 426 = __acl_set_link */
+	{ AS(__acl_delete_link_args), (sy_call_t *)sys___acl_delete_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 427 = __acl_delete_link */
+	{ AS(__acl_aclcheck_link_args), (sy_call_t *)sys___acl_aclcheck_link, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 428 = __acl_aclcheck_link */
+	{ AS(sigwait_args), (sy_call_t *)sys_sigwait, AUE_SIGWAIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 429 = sigwait */
+	{ AS(thr_create_args), (sy_call_t *)sys_thr_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 430 = thr_create */
+	{ AS(thr_exit_args), (sy_call_t *)sys_thr_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 431 = thr_exit */
+	{ AS(thr_self_args), (sy_call_t *)sys_thr_self, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 432 = thr_self */
+	{ AS(thr_kill_args), (sy_call_t *)sys_thr_kill, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 433 = thr_kill */
+	{ AS(_umtx_lock_args), (sy_call_t *)sys__umtx_lock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 434 = _umtx_lock */
+	{ AS(_umtx_unlock_args), (sy_call_t *)sys__umtx_unlock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 435 = _umtx_unlock */
+	{ AS(jail_attach_args), (sy_call_t *)sys_jail_attach, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 436 = jail_attach */
+	{ AS(extattr_list_fd_args), (sy_call_t *)sys_extattr_list_fd, AUE_EXTATTR_LIST_FD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 437 = extattr_list_fd */
+	{ AS(extattr_list_file_args), (sy_call_t *)sys_extattr_list_file, AUE_EXTATTR_LIST_FILE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 438 = extattr_list_file */
+	{ AS(extattr_list_link_args), (sy_call_t *)sys_extattr_list_link, AUE_EXTATTR_LIST_LINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 439 = extattr_list_link */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 440 = kse_switchin */
 	{ AS(ksem_timedwait_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 441 = ksem_timedwait */
-	{ AS(thr_suspend_args), (sy_call_t *)thr_suspend, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 442 = thr_suspend */
-	{ AS(thr_wake_args), (sy_call_t *)thr_wake, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 443 = thr_wake */
-	{ AS(kldunloadf_args), (sy_call_t *)kldunloadf, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 444 = kldunloadf */
-	{ AS(audit_args), (sy_call_t *)audit, AUE_AUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 445 = audit */
-	{ AS(auditon_args), (sy_call_t *)auditon, AUE_AUDITON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 446 = auditon */
-	{ AS(getauid_args), (sy_call_t *)getauid, AUE_GETAUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 447 = getauid */
-	{ AS(setauid_args), (sy_call_t *)setauid, AUE_SETAUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 448 = setauid */
-	{ AS(getaudit_args), (sy_call_t *)getaudit, AUE_GETAUDIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 449 = getaudit */
-	{ AS(setaudit_args), (sy_call_t *)setaudit, AUE_SETAUDIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 450 = setaudit */
-	{ AS(getaudit_addr_args), (sy_call_t *)getaudit_addr, AUE_GETAUDIT_ADDR, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 451 = getaudit_addr */
-	{ AS(setaudit_addr_args), (sy_call_t *)setaudit_addr, AUE_SETAUDIT_ADDR, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 452 = setaudit_addr */
-	{ AS(auditctl_args), (sy_call_t *)auditctl, AUE_AUDITCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 453 = auditctl */
-	{ AS(_umtx_op_args), (sy_call_t *)_umtx_op, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 454 = _umtx_op */
-	{ AS(thr_new_args), (sy_call_t *)thr_new, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 455 = thr_new */
-	{ AS(sigqueue_args), (sy_call_t *)sigqueue, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 456 = sigqueue */
+	{ AS(thr_suspend_args), (sy_call_t *)sys_thr_suspend, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 442 = thr_suspend */
+	{ AS(thr_wake_args), (sy_call_t *)sys_thr_wake, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 443 = thr_wake */
+	{ AS(kldunloadf_args), (sy_call_t *)sys_kldunloadf, AUE_MODUNLOAD, NULL, 0, 0, 0, SY_THR_STATIC },	/* 444 = kldunloadf */
+	{ AS(audit_args), (sy_call_t *)sys_audit, AUE_AUDIT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 445 = audit */
+	{ AS(auditon_args), (sy_call_t *)sys_auditon, AUE_AUDITON, NULL, 0, 0, 0, SY_THR_STATIC },	/* 446 = auditon */
+	{ AS(getauid_args), (sy_call_t *)sys_getauid, AUE_GETAUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 447 = getauid */
+	{ AS(setauid_args), (sy_call_t *)sys_setauid, AUE_SETAUID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 448 = setauid */
+	{ AS(getaudit_args), (sy_call_t *)sys_getaudit, AUE_GETAUDIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 449 = getaudit */
+	{ AS(setaudit_args), (sy_call_t *)sys_setaudit, AUE_SETAUDIT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 450 = setaudit */
+	{ AS(getaudit_addr_args), (sy_call_t *)sys_getaudit_addr, AUE_GETAUDIT_ADDR, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 451 = getaudit_addr */
+	{ AS(setaudit_addr_args), (sy_call_t *)sys_setaudit_addr, AUE_SETAUDIT_ADDR, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 452 = setaudit_addr */
+	{ AS(auditctl_args), (sy_call_t *)sys_auditctl, AUE_AUDITCTL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 453 = auditctl */
+	{ AS(_umtx_op_args), (sy_call_t *)sys__umtx_op, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 454 = _umtx_op */
+	{ AS(thr_new_args), (sy_call_t *)sys_thr_new, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 455 = thr_new */
+	{ AS(sigqueue_args), (sy_call_t *)sys_sigqueue, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 456 = sigqueue */
 	{ AS(kmq_open_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 457 = kmq_open */
 	{ AS(kmq_setattr_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 458 = kmq_setattr */
 	{ AS(kmq_timedreceive_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 459 = kmq_timedreceive */
 	{ AS(kmq_timedsend_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 460 = kmq_timedsend */
 	{ AS(kmq_notify_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 461 = kmq_notify */
 	{ AS(kmq_unlink_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 462 = kmq_unlink */
-	{ AS(abort2_args), (sy_call_t *)abort2, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 463 = abort2 */
-	{ AS(thr_set_name_args), (sy_call_t *)thr_set_name, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 464 = thr_set_name */
+	{ AS(abort2_args), (sy_call_t *)sys_abort2, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 463 = abort2 */
+	{ AS(thr_set_name_args), (sy_call_t *)sys_thr_set_name, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 464 = thr_set_name */
 	{ AS(aio_fsync_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 465 = aio_fsync */
-	{ AS(rtprio_thread_args), (sy_call_t *)rtprio_thread, AUE_RTPRIO, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 466 = rtprio_thread */
+	{ AS(rtprio_thread_args), (sy_call_t *)sys_rtprio_thread, AUE_RTPRIO, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 466 = rtprio_thread */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 467 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 468 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 469 = __getpath_fromfd */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 470 = __getpath_fromaddr */
-	{ AS(sctp_peeloff_args), (sy_call_t *)sctp_peeloff, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 471 = sctp_peeloff */
-	{ AS(sctp_generic_sendmsg_args), (sy_call_t *)sctp_generic_sendmsg, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 472 = sctp_generic_sendmsg */
-	{ AS(sctp_generic_sendmsg_iov_args), (sy_call_t *)sctp_generic_sendmsg_iov, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 473 = sctp_generic_sendmsg_iov */
-	{ AS(sctp_generic_recvmsg_args), (sy_call_t *)sctp_generic_recvmsg, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 474 = sctp_generic_recvmsg */
-	{ AS(pread_args), (sy_call_t *)pread, AUE_PREAD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 475 = pread */
-	{ AS(pwrite_args), (sy_call_t *)pwrite, AUE_PWRITE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 476 = pwrite */
-	{ AS(mmap_args), (sy_call_t *)mmap, AUE_MMAP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 477 = mmap */
-	{ AS(lseek_args), (sy_call_t *)lseek, AUE_LSEEK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 478 = lseek */
-	{ AS(truncate_args), (sy_call_t *)truncate, AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 479 = truncate */
-	{ AS(ftruncate_args), (sy_call_t *)ftruncate, AUE_FTRUNCATE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 480 = ftruncate */
-	{ AS(thr_kill2_args), (sy_call_t *)thr_kill2, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 481 = thr_kill2 */
-	{ AS(shm_open_args), (sy_call_t *)shm_open, AUE_SHMOPEN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 482 = shm_open */
-	{ AS(shm_unlink_args), (sy_call_t *)shm_unlink, AUE_SHMUNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 483 = shm_unlink */
-	{ AS(cpuset_args), (sy_call_t *)cpuset, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 484 = cpuset */
-	{ AS(cpuset_setid_args), (sy_call_t *)cpuset_setid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 485 = cpuset_setid */
-	{ AS(cpuset_getid_args), (sy_call_t *)cpuset_getid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 486 = cpuset_getid */
-	{ AS(cpuset_getaffinity_args), (sy_call_t *)cpuset_getaffinity, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 487 = cpuset_getaffinity */
-	{ AS(cpuset_setaffinity_args), (sy_call_t *)cpuset_setaffinity, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 488 = cpuset_setaffinity */
-	{ AS(faccessat_args), (sy_call_t *)faccessat, AUE_FACCESSAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 489 = faccessat */
-	{ AS(fchmodat_args), (sy_call_t *)fchmodat, AUE_FCHMODAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 490 = fchmodat */
-	{ AS(fchownat_args), (sy_call_t *)fchownat, AUE_FCHOWNAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 491 = fchownat */
-	{ AS(fexecve_args), (sy_call_t *)fexecve, AUE_FEXECVE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 492 = fexecve */
-	{ AS(fstatat_args), (sy_call_t *)fstatat, AUE_FSTATAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 493 = fstatat */
-	{ AS(futimesat_args), (sy_call_t *)futimesat, AUE_FUTIMESAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 494 = futimesat */
-	{ AS(linkat_args), (sy_call_t *)linkat, AUE_LINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 495 = linkat */
-	{ AS(mkdirat_args), (sy_call_t *)mkdirat, AUE_MKDIRAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 496 = mkdirat */
-	{ AS(mkfifoat_args), (sy_call_t *)mkfifoat, AUE_MKFIFOAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 497 = mkfifoat */
-	{ AS(mknodat_args), (sy_call_t *)mknodat, AUE_MKNODAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 498 = mknodat */
-	{ AS(openat_args), (sy_call_t *)openat, AUE_OPENAT_RWTC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 499 = openat */
-	{ AS(readlinkat_args), (sy_call_t *)readlinkat, AUE_READLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 500 = readlinkat */
-	{ AS(renameat_args), (sy_call_t *)renameat, AUE_RENAMEAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 501 = renameat */
-	{ AS(symlinkat_args), (sy_call_t *)symlinkat, AUE_SYMLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 502 = symlinkat */
-	{ AS(unlinkat_args), (sy_call_t *)unlinkat, AUE_UNLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 503 = unlinkat */
-	{ AS(posix_openpt_args), (sy_call_t *)posix_openpt, AUE_POSIX_OPENPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 504 = posix_openpt */
+	{ AS(sctp_peeloff_args), (sy_call_t *)sys_sctp_peeloff, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 471 = sctp_peeloff */
+	{ AS(sctp_generic_sendmsg_args), (sy_call_t *)sys_sctp_generic_sendmsg, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 472 = sctp_generic_sendmsg */
+	{ AS(sctp_generic_sendmsg_iov_args), (sy_call_t *)sys_sctp_generic_sendmsg_iov, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 473 = sctp_generic_sendmsg_iov */
+	{ AS(sctp_generic_recvmsg_args), (sy_call_t *)sys_sctp_generic_recvmsg, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 474 = sctp_generic_recvmsg */
+	{ AS(pread_args), (sy_call_t *)sys_pread, AUE_PREAD, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 475 = pread */
+	{ AS(pwrite_args), (sy_call_t *)sys_pwrite, AUE_PWRITE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 476 = pwrite */
+	{ AS(mmap_args), (sy_call_t *)sys_mmap, AUE_MMAP, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 477 = mmap */
+	{ AS(lseek_args), (sy_call_t *)sys_lseek, AUE_LSEEK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 478 = lseek */
+	{ AS(truncate_args), (sy_call_t *)sys_truncate, AUE_TRUNCATE, NULL, 0, 0, 0, SY_THR_STATIC },	/* 479 = truncate */
+	{ AS(ftruncate_args), (sy_call_t *)sys_ftruncate, AUE_FTRUNCATE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 480 = ftruncate */
+	{ AS(thr_kill2_args), (sy_call_t *)sys_thr_kill2, AUE_KILL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 481 = thr_kill2 */
+	{ AS(shm_open_args), (sy_call_t *)sys_shm_open, AUE_SHMOPEN, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 482 = shm_open */
+	{ AS(shm_unlink_args), (sy_call_t *)sys_shm_unlink, AUE_SHMUNLINK, NULL, 0, 0, 0, SY_THR_STATIC },	/* 483 = shm_unlink */
+	{ AS(cpuset_args), (sy_call_t *)sys_cpuset, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 484 = cpuset */
+	{ AS(cpuset_setid_args), (sy_call_t *)sys_cpuset_setid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 485 = cpuset_setid */
+	{ AS(cpuset_getid_args), (sy_call_t *)sys_cpuset_getid, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 486 = cpuset_getid */
+	{ AS(cpuset_getaffinity_args), (sy_call_t *)sys_cpuset_getaffinity, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 487 = cpuset_getaffinity */
+	{ AS(cpuset_setaffinity_args), (sy_call_t *)sys_cpuset_setaffinity, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 488 = cpuset_setaffinity */
+	{ AS(faccessat_args), (sy_call_t *)sys_faccessat, AUE_FACCESSAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 489 = faccessat */
+	{ AS(fchmodat_args), (sy_call_t *)sys_fchmodat, AUE_FCHMODAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 490 = fchmodat */
+	{ AS(fchownat_args), (sy_call_t *)sys_fchownat, AUE_FCHOWNAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 491 = fchownat */
+	{ AS(fexecve_args), (sy_call_t *)sys_fexecve, AUE_FEXECVE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 492 = fexecve */
+	{ AS(fstatat_args), (sy_call_t *)sys_fstatat, AUE_FSTATAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 493 = fstatat */
+	{ AS(futimesat_args), (sy_call_t *)sys_futimesat, AUE_FUTIMESAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 494 = futimesat */
+	{ AS(linkat_args), (sy_call_t *)sys_linkat, AUE_LINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 495 = linkat */
+	{ AS(mkdirat_args), (sy_call_t *)sys_mkdirat, AUE_MKDIRAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 496 = mkdirat */
+	{ AS(mkfifoat_args), (sy_call_t *)sys_mkfifoat, AUE_MKFIFOAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 497 = mkfifoat */
+	{ AS(mknodat_args), (sy_call_t *)sys_mknodat, AUE_MKNODAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 498 = mknodat */
+	{ AS(openat_args), (sy_call_t *)sys_openat, AUE_OPENAT_RWTC, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 499 = openat */
+	{ AS(readlinkat_args), (sy_call_t *)sys_readlinkat, AUE_READLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 500 = readlinkat */
+	{ AS(renameat_args), (sy_call_t *)sys_renameat, AUE_RENAMEAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 501 = renameat */
+	{ AS(symlinkat_args), (sy_call_t *)sys_symlinkat, AUE_SYMLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 502 = symlinkat */
+	{ AS(unlinkat_args), (sy_call_t *)sys_unlinkat, AUE_UNLINKAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 503 = unlinkat */
+	{ AS(posix_openpt_args), (sy_call_t *)sys_posix_openpt, AUE_POSIX_OPENPT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 504 = posix_openpt */
 	{ AS(gssd_syscall_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 505 = gssd_syscall */
-	{ AS(jail_get_args), (sy_call_t *)jail_get, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 506 = jail_get */
-	{ AS(jail_set_args), (sy_call_t *)jail_set, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 507 = jail_set */
-	{ AS(jail_remove_args), (sy_call_t *)jail_remove, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 508 = jail_remove */
-	{ AS(closefrom_args), (sy_call_t *)closefrom, AUE_CLOSEFROM, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 509 = closefrom */
+	{ AS(jail_get_args), (sy_call_t *)sys_jail_get, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 506 = jail_get */
+	{ AS(jail_set_args), (sy_call_t *)sys_jail_set, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 507 = jail_set */
+	{ AS(jail_remove_args), (sy_call_t *)sys_jail_remove, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 508 = jail_remove */
+	{ AS(closefrom_args), (sy_call_t *)sys_closefrom, AUE_CLOSEFROM, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 509 = closefrom */
 	{ AS(__semctl_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 510 = __semctl */
 	{ AS(msgctl_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 511 = msgctl */
 	{ AS(shmctl_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 512 = shmctl */
-	{ AS(lpathconf_args), (sy_call_t *)lpathconf, AUE_LPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 513 = lpathconf */
-	{ AS(cap_new_args), (sy_call_t *)cap_new, AUE_CAP_NEW, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 514 = cap_new */
-	{ AS(cap_getrights_args), (sy_call_t *)cap_getrights, AUE_CAP_GETRIGHTS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 515 = cap_getrights */
-	{ 0, (sy_call_t *)cap_enter, AUE_CAP_ENTER, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 516 = cap_enter */
-	{ AS(cap_getmode_args), (sy_call_t *)cap_getmode, AUE_CAP_GETMODE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 517 = cap_getmode */
-	{ AS(pdfork_args), (sy_call_t *)pdfork, AUE_PDFORK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 518 = pdfork */
-	{ AS(pdkill_args), (sy_call_t *)pdkill, AUE_PDKILL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 519 = pdkill */
-	{ AS(pdgetpid_args), (sy_call_t *)pdgetpid, AUE_PDGETPID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 520 = pdgetpid */
+	{ AS(lpathconf_args), (sy_call_t *)sys_lpathconf, AUE_LPATHCONF, NULL, 0, 0, 0, SY_THR_STATIC },	/* 513 = lpathconf */
+	{ AS(cap_new_args), (sy_call_t *)sys_cap_new, AUE_CAP_NEW, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 514 = cap_new */
+	{ AS(cap_getrights_args), (sy_call_t *)sys_cap_getrights, AUE_CAP_GETRIGHTS, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 515 = cap_getrights */
+	{ 0, (sy_call_t *)sys_cap_enter, AUE_CAP_ENTER, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 516 = cap_enter */
+	{ AS(cap_getmode_args), (sy_call_t *)sys_cap_getmode, AUE_CAP_GETMODE, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 517 = cap_getmode */
+	{ AS(pdfork_args), (sy_call_t *)sys_pdfork, AUE_PDFORK, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 518 = pdfork */
+	{ AS(pdkill_args), (sy_call_t *)sys_pdkill, AUE_PDKILL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 519 = pdkill */
+	{ AS(pdgetpid_args), (sy_call_t *)sys_pdgetpid, AUE_PDGETPID, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 520 = pdgetpid */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 521 = pdwait4 */
-	{ AS(pselect_args), (sy_call_t *)pselect, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 522 = pselect */
-	{ AS(getloginclass_args), (sy_call_t *)getloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 523 = getloginclass */
-	{ AS(setloginclass_args), (sy_call_t *)setloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 524 = setloginclass */
-	{ AS(rctl_get_racct_args), (sy_call_t *)rctl_get_racct, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 525 = rctl_get_racct */
-	{ AS(rctl_get_rules_args), (sy_call_t *)rctl_get_rules, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 526 = rctl_get_rules */
-	{ AS(rctl_get_limits_args), (sy_call_t *)rctl_get_limits, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 527 = rctl_get_limits */
-	{ AS(rctl_add_rule_args), (sy_call_t *)rctl_add_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 528 = rctl_add_rule */
-	{ AS(rctl_remove_rule_args), (sy_call_t *)rctl_remove_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 529 = rctl_remove_rule */
-	{ AS(posix_fallocate_args), (sy_call_t *)posix_fallocate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 530 = posix_fallocate */
+	{ AS(pselect_args), (sy_call_t *)sys_pselect, AUE_SELECT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 522 = pselect */
+	{ AS(getloginclass_args), (sy_call_t *)sys_getloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 523 = getloginclass */
+	{ AS(setloginclass_args), (sy_call_t *)sys_setloginclass, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 524 = setloginclass */
+	{ AS(rctl_get_racct_args), (sy_call_t *)sys_rctl_get_racct, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 525 = rctl_get_racct */
+	{ AS(rctl_get_rules_args), (sy_call_t *)sys_rctl_get_rules, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 526 = rctl_get_rules */
+	{ AS(rctl_get_limits_args), (sy_call_t *)sys_rctl_get_limits, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 527 = rctl_get_limits */
+	{ AS(rctl_add_rule_args), (sy_call_t *)sys_rctl_add_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 528 = rctl_add_rule */
+	{ AS(rctl_remove_rule_args), (sy_call_t *)sys_rctl_remove_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 529 = rctl_remove_rule */
+	{ AS(posix_fallocate_args), (sy_call_t *)sys_posix_fallocate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 530 = posix_fallocate */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 531 = posix_fadvise */
 };
diff --git a/sys/kern/kern_acct.c b/sys/kern/kern_acct.c
index c7ba51796bf9..8e5479baf8bf 100644
--- a/sys/kern/kern_acct.c
+++ b/sys/kern/kern_acct.c
@@ -193,7 +193,7 @@ SYSCTL_INT(_kern, OID_AUTO, acct_suspended, CTLFLAG_RD, &acct_suspended, 0,
  * implementation done by Mark Tinguely.
  */
 int
-acct(struct thread *td, struct acct_args *uap)
+sys_acct(struct thread *td, struct acct_args *uap)
 {
 	struct nameidata nd;
 	int error, flags, vfslocked;
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 681b64504db7..4f7807362a61 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -842,7 +842,7 @@ watchdog_fire(void)
 	curintr = intrcnt;
 	curname = intrnames;
 	inttotal = 0;
-	nintr = sintrcnt;
+	nintr = sintrcnt / sizeof(u_long);
 
 	printf("interrupt                   total\n");
 	while (--nintr >= 0) {
diff --git a/sys/kern/kern_context.c b/sys/kern/kern_context.c
index 9230857e1c72..70751d02cba8 100644
--- a/sys/kern/kern_context.c
+++ b/sys/kern/kern_context.c
@@ -60,7 +60,7 @@ struct swapcontext_args {
 #endif
 
 int
-getcontext(struct thread *td, struct getcontext_args *uap)
+sys_getcontext(struct thread *td, struct getcontext_args *uap)
 {
 	ucontext_t uc;
 	int ret;
@@ -79,7 +79,7 @@ getcontext(struct thread *td, struct getcontext_args *uap)
 }
 
 int
-setcontext(struct thread *td, struct setcontext_args *uap)
+sys_setcontext(struct thread *td, struct setcontext_args *uap)
 {
 	ucontext_t uc;
 	int ret;	
@@ -100,7 +100,7 @@ setcontext(struct thread *td, struct setcontext_args *uap)
 }
 
 int
-swapcontext(struct thread *td, struct swapcontext_args *uap)
+sys_swapcontext(struct thread *td, struct swapcontext_args *uap)
 {
 	ucontext_t uc;
 	int ret;	
diff --git a/sys/kern/kern_cpuset.c b/sys/kern/kern_cpuset.c
index e1f2801643ac..cee3e815c5ee 100644
--- a/sys/kern/kern_cpuset.c
+++ b/sys/kern/kern_cpuset.c
@@ -848,7 +848,7 @@ struct cpuset_args {
 };
 #endif
 int
-cpuset(struct thread *td, struct cpuset_args *uap)
+sys_cpuset(struct thread *td, struct cpuset_args *uap)
 {
 	struct cpuset *root;
 	struct cpuset *set;
@@ -876,7 +876,7 @@ struct cpuset_setid_args {
 };
 #endif
 int
-cpuset_setid(struct thread *td, struct cpuset_setid_args *uap)
+sys_cpuset_setid(struct thread *td, struct cpuset_setid_args *uap)
 {
 	struct cpuset *set;
 	int error;
@@ -902,7 +902,7 @@ struct cpuset_getid_args {
 	cpusetid_t	*setid;
 #endif
 int
-cpuset_getid(struct thread *td, struct cpuset_getid_args *uap)
+sys_cpuset_getid(struct thread *td, struct cpuset_getid_args *uap)
 {
 	struct cpuset *nset;
 	struct cpuset *set;
@@ -959,7 +959,7 @@ struct cpuset_getaffinity_args {
 };
 #endif
 int
-cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap)
+sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap)
 {
 	struct thread *ttd;
 	struct cpuset *nset;
@@ -1049,7 +1049,7 @@ struct cpuset_setaffinity_args {
 };
 #endif
 int
-cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap)
+sys_cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap)
 {
 	struct cpuset *nset;
 	struct cpuset *set;
diff --git a/sys/kern/kern_ctf.c b/sys/kern/kern_ctf.c
index 758ad81a5398..bdff96ed63ca 100644
--- a/sys/kern/kern_ctf.c
+++ b/sys/kern/kern_ctf.c
@@ -164,8 +164,13 @@ link_elf_ctf_get(linker_file_t lf, linker_ctf_t *lc)
 	 * section names aren't present, then we can't locate the
 	 * .SUNW_ctf section containing the CTF data.
 	 */
-	if (hdr->e_shstrndx == 0 || shdr[hdr->e_shstrndx].sh_type != SHT_STRTAB)
+	if (hdr->e_shstrndx == 0 || shdr[hdr->e_shstrndx].sh_type != SHT_STRTAB) {
+		printf("%s(%d): module %s e_shstrndx is %d, sh_type is %d\n",
+		    __func__, __LINE__, lf->pathname, hdr->e_shstrndx,
+		    shdr[hdr->e_shstrndx].sh_type);
+		error = EFTYPE;
 		goto out;
+	}
 
 	/* Allocate memory to buffer the section header strings. */
 	if ((shstrtab = malloc(shdr[hdr->e_shstrndx].sh_size, M_LINKER,
@@ -187,8 +192,12 @@ link_elf_ctf_get(linker_file_t lf, linker_ctf_t *lc)
 			break;
 
 	/* Check if the CTF section wasn't found. */
-	if (i >= hdr->e_shnum)
+	if (i >= hdr->e_shnum) {
+		printf("%s(%d): module %s has no .SUNW_ctf section\n",
+		    __func__, __LINE__, lf->pathname);
+		error = EFTYPE;
 		goto out;
+	}
 
 	/* Read the CTF header. */
 	if ((error = vn_rdwr(UIO_READ, nd.ni_vp, ctf_hdr, sizeof(ctf_hdr),
@@ -197,12 +206,21 @@ link_elf_ctf_get(linker_file_t lf, linker_ctf_t *lc)
 		goto out;
 
 	/* Check the CTF magic number. (XXX check for big endian!) */
-	if (ctf_hdr[0] != 0xf1 || ctf_hdr[1] != 0xcf)
+	if (ctf_hdr[0] != 0xf1 || ctf_hdr[1] != 0xcf) {
+		printf("%s(%d): module %s has invalid format\n",
+		    __func__, __LINE__, lf->pathname);
+		error = EFTYPE;
 		goto out;
+	}
 
 	/* Check if version 2. */
-	if (ctf_hdr[2] != 2)
+	if (ctf_hdr[2] != 2) {
+		printf("%s(%d): module %s CTF format version is %d "
+		    "(2 expected)\n",
+		    __func__, __LINE__, lf->pathname, ctf_hdr[2]);
+		error = EFTYPE;
 		goto out;
+	}
 
 	/* Check if the data is compressed. */
 	if ((ctf_hdr[3] & 0x1) != 0) {
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 4aaed1f040e8..8b3e20e06914 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -292,7 +292,7 @@ struct getdtablesize_args {
 #endif
 /* ARGSUSED */
 int
-getdtablesize(struct thread *td, struct getdtablesize_args *uap)
+sys_getdtablesize(struct thread *td, struct getdtablesize_args *uap)
 {
 	struct proc *p = td->td_proc;
 	uint64_t lim;
@@ -321,7 +321,7 @@ struct dup2_args {
 #endif
 /* ARGSUSED */
 int
-dup2(struct thread *td, struct dup2_args *uap)
+sys_dup2(struct thread *td, struct dup2_args *uap)
 {
 
 	return (do_dup(td, DUP_FIXED, (int)uap->from, (int)uap->to,
@@ -338,7 +338,7 @@ struct dup_args {
 #endif
 /* ARGSUSED */
 int
-dup(struct thread *td, struct dup_args *uap)
+sys_dup(struct thread *td, struct dup_args *uap)
 {
 
 	return (do_dup(td, 0, (int)uap->fd, 0, td->td_retval));
@@ -356,7 +356,7 @@ struct fcntl_args {
 #endif
 /* ARGSUSED */
 int
-fcntl(struct thread *td, struct fcntl_args *uap)
+sys_fcntl(struct thread *td, struct fcntl_args *uap)
 {
 	struct flock fl;
 	struct oflock ofl;
@@ -1170,7 +1170,7 @@ struct close_args {
 #endif
 /* ARGSUSED */
 int
-close(td, uap)
+sys_close(td, uap)
 	struct thread *td;
 	struct close_args *uap;
 {
@@ -1253,7 +1253,7 @@ struct closefrom_args {
 #endif
 /* ARGSUSED */
 int
-closefrom(struct thread *td, struct closefrom_args *uap)
+sys_closefrom(struct thread *td, struct closefrom_args *uap)
 {
 	struct filedesc *fdp;
 	int fd;
@@ -1317,7 +1317,7 @@ struct fstat_args {
 #endif
 /* ARGSUSED */
 int
-fstat(struct thread *td, struct fstat_args *uap)
+sys_fstat(struct thread *td, struct fstat_args *uap)
 {
 	struct stat ub;
 	int error;
@@ -1361,7 +1361,7 @@ struct nfstat_args {
 #endif
 /* ARGSUSED */
 int
-nfstat(struct thread *td, struct nfstat_args *uap)
+sys_nfstat(struct thread *td, struct nfstat_args *uap)
 {
 	struct nstat nub;
 	struct stat ub;
@@ -1386,7 +1386,7 @@ struct fpathconf_args {
 #endif
 /* ARGSUSED */
 int
-fpathconf(struct thread *td, struct fpathconf_args *uap)
+sys_fpathconf(struct thread *td, struct fpathconf_args *uap)
 {
 	struct file *fp;
 	struct vnode *vp;
@@ -2602,7 +2602,7 @@ struct flock_args {
 #endif
 /* ARGSUSED */
 int
-flock(struct thread *td, struct flock_args *uap)
+sys_flock(struct thread *td, struct flock_args *uap)
 {
 	struct file *fp;
 	struct vnode *vp;
@@ -3182,7 +3182,8 @@ CTASSERT(sizeof(struct kinfo_file) == KINFO_FILE_SIZE);
 
 static int
 export_fd_for_sysctl(void *data, int type, int fd, int fflags, int refcnt,
-    int64_t offset, struct kinfo_file *kif, struct sysctl_req *req)
+    int64_t offset, int fd_is_cap, cap_rights_t fd_cap_rights,
+    struct kinfo_file *kif, struct sysctl_req *req)
 {
 	struct {
 		int	fflag;
@@ -3243,6 +3244,10 @@ export_fd_for_sysctl(void *data, int type, int fd, int fflags, int refcnt,
 	for (i = 0; i < NFFLAGS; i++)
 		if (fflags & fflags_table[i].fflag)
 			kif->kf_flags |=  fflags_table[i].kf_fflag;
+	if (fd_is_cap)
+		kif->kf_flags |= KF_FLAG_CAPABILITY;
+	if (fd_is_cap)
+		kif->kf_cap_rights = fd_cap_rights;
 	kif->kf_fd = fd;
 	kif->kf_type = type;
 	kif->kf_ref_count = refcnt;
@@ -3270,7 +3275,8 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 	int64_t offset;
 	void *data;
 	int error, i, *name;
-	int type, refcnt, fflags;
+	int fd_is_cap, type, refcnt, fflags;
+	cap_rights_t fd_cap_rights;
 
 	name = (int *)arg1;
 	if ((p = pfind((pid_t)name[0])) == NULL)
@@ -3299,13 +3305,13 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 	kif = malloc(sizeof(*kif), M_TEMP, M_WAITOK);
 	if (tracevp != NULL)
 		export_fd_for_sysctl(tracevp, KF_TYPE_VNODE, KF_FD_TYPE_TRACE,
-		    FREAD | FWRITE, -1, -1, kif, req);
+		    FREAD | FWRITE, -1, -1, 0, 0, kif, req);
 	if (textvp != NULL)
 		export_fd_for_sysctl(textvp, KF_TYPE_VNODE, KF_FD_TYPE_TEXT,
-		    FREAD, -1, -1, kif, req);
+		    FREAD, -1, -1, 0, 0, kif, req);
 	if (cttyvp != NULL)
 		export_fd_for_sysctl(cttyvp, KF_TYPE_VNODE, KF_FD_TYPE_CTTY,
-		    FREAD | FWRITE, -1, -1, kif, req);
+		    FREAD | FWRITE, -1, -1, 0, 0, kif, req);
 	if (fdp == NULL)
 		goto fail;
 	FILEDESC_SLOCK(fdp);
@@ -3315,7 +3321,7 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 		data = fdp->fd_cdir;
 		FILEDESC_SUNLOCK(fdp);
 		export_fd_for_sysctl(data, KF_TYPE_VNODE, KF_FD_TYPE_CWD,
-		    FREAD, -1, -1, kif, req);
+		    FREAD, -1, -1, 0, 0, kif, req);
 		FILEDESC_SLOCK(fdp);
 	}
 	/* root directory */
@@ -3324,7 +3330,7 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 		data = fdp->fd_rdir;
 		FILEDESC_SUNLOCK(fdp);
 		export_fd_for_sysctl(data, KF_TYPE_VNODE, KF_FD_TYPE_ROOT,
-		    FREAD, -1, -1, kif, req);
+		    FREAD, -1, -1, 0, 0, kif, req);
 		FILEDESC_SLOCK(fdp);
 	}
 	/* jail directory */
@@ -3333,13 +3339,15 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 		data = fdp->fd_jdir;
 		FILEDESC_SUNLOCK(fdp);
 		export_fd_for_sysctl(data, KF_TYPE_VNODE, KF_FD_TYPE_JAIL,
-		    FREAD, -1, -1, kif, req);
+		    FREAD, -1, -1, 0, 0, kif, req);
 		FILEDESC_SLOCK(fdp);
 	}
 	for (i = 0; i < fdp->fd_nfiles; i++) {
 		if ((fp = fdp->fd_ofiles[i]) == NULL)
 			continue;
 		data = NULL;
+		fd_is_cap = 0;
+		fd_cap_rights = 0;
 
 #ifdef CAPABILITIES
 		/*
@@ -3348,8 +3356,8 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 		 * the capability rights mask.
 		 */
 		if (fp->f_type == DTYPE_CAPABILITY) {
-			kif->kf_flags |= KF_FLAG_CAPABILITY;
-			kif->kf_cap_rights = cap_rights(fp);
+			fd_is_cap = 1;
+			fd_cap_rights = cap_rights(fp);
 			(void)cap_funwrap(fp, 0, &fp);
 		}
 #else /* !CAPABILITIES */
@@ -3428,8 +3436,8 @@ sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
 		oldidx = req->oldidx;
 		if (type == KF_TYPE_VNODE || type == KF_TYPE_FIFO)
 			FILEDESC_SUNLOCK(fdp);
-		error = export_fd_for_sysctl(data, type, i,
-		    fflags, refcnt, offset, kif, req);
+		error = export_fd_for_sysctl(data, type, i, fflags, refcnt,
+		    offset, fd_is_cap, fd_cap_rights, kif, req);
 		if (type == KF_TYPE_VNODE || type == KF_TYPE_FIFO)
 			FILEDESC_SLOCK(fdp);
 		if (error) {
diff --git a/sys/kern/kern_environment.c b/sys/kern/kern_environment.c
index 16760ce90c54..1e3128516190 100644
--- a/sys/kern/kern_environment.c
+++ b/sys/kern/kern_environment.c
@@ -77,7 +77,7 @@ int	dynamic_kenv = 0;
 			    panic("%s: called before SI_SUB_KMEM", __func__)
 
 int
-kenv(td, uap)
+sys_kenv(td, uap)
 	struct thread *td;
 	struct kenv_args /* {
 		int what;
diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index dc11411fd5df..925cbecd0b49 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -679,7 +679,7 @@ filt_usertouch(struct knote *kn, struct kevent *kev, u_long type)
 }
 
 int
-kqueue(struct thread *td, struct kqueue_args *uap)
+sys_kqueue(struct thread *td, struct kqueue_args *uap)
 {
 	struct filedesc *fdp;
 	struct kqueue *kq;
@@ -722,7 +722,7 @@ struct kevent_args {
 };
 #endif
 int
-kevent(struct thread *td, struct kevent_args *uap)
+sys_kevent(struct thread *td, struct kevent_args *uap)
 {
 	struct timespec ts, *tsp;
 	struct kevent_copyops k_ops = { uap,
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 1c424204800f..4545848b6e85 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -192,7 +192,7 @@ struct execve_args {
 #endif
 
 int
-execve(td, uap)
+sys_execve(td, uap)
 	struct thread *td;
 	struct execve_args /* {
 		char *fname;
@@ -218,7 +218,7 @@ struct fexecve_args {
 }
 #endif
 int
-fexecve(struct thread *td, struct fexecve_args *uap)
+sys_fexecve(struct thread *td, struct fexecve_args *uap)
 {
 	int error;
 	struct image_args args;
@@ -242,7 +242,7 @@ struct __mac_execve_args {
 #endif
 
 int
-__mac_execve(td, uap)
+sys___mac_execve(td, uap)
 	struct thread *td;
 	struct __mac_execve_args /* {
 		char *fname;
@@ -777,16 +777,6 @@ do_execve(td, args, mac_p)
 	KNOTE_LOCKED(&p->p_klist, NOTE_EXEC);
 	p->p_flag &= ~P_INEXEC;
 
-	/*
-	 * If tracing the process, trap to the debugger so that
-	 * breakpoints can be set before the program executes.  We
-	 * have to use tdsignal() to deliver the signal to the current
-	 * thread since any other threads in this process will exit if
-	 * execve() succeeds.
-	 */
-	if (p->p_flag & P_TRACED)
-		tdsignal(td, SIGTRAP);
-
 	/* clear "fork but no exec" flag, as we _are_ execing */
 	p->p_acflag &= ~AFORK;
 
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index e5d60942f7c0..49192bfdb796 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -104,7 +104,7 @@ void (*nlminfo_release_p)(struct proc *p);
  * exit -- death of process.
  */
 void
-sys_exit(struct thread *td, struct sys_exit_args *uap)
+sys_sys_exit(struct thread *td, struct sys_exit_args *uap)
 {
 
 	exit1(td, W_EXITCODE(uap->rval, 0));
@@ -224,7 +224,7 @@ exit1(struct thread *td, int rv)
 		q = p->p_peers;
 		while (q != NULL) {
 			PROC_LOCK(q);
-			psignal(q, SIGKILL);
+			kern_psignal(q, SIGKILL);
 			PROC_UNLOCK(q);
 			q = q->p_peers;
 		}
@@ -421,7 +421,7 @@ exit1(struct thread *td, int rv)
 			q->p_flag &= ~(P_TRACED | P_STOPPED_TRACE);
 			FOREACH_THREAD_IN_PROC(q, temp)
 				temp->td_dbgflags &= ~TDB_SUSPEND;
-			psignal(q, SIGKILL);
+			kern_psignal(q, SIGKILL);
 		}
 		PROC_UNLOCK(q);
 	}
@@ -501,12 +501,12 @@ exit1(struct thread *td, int rv)
 			mtx_unlock(&p->p_pptr->p_sigacts->ps_mtx);
 
 		if (p->p_pptr == initproc)
-			psignal(p->p_pptr, SIGCHLD);
+			kern_psignal(p->p_pptr, SIGCHLD);
 		else if (p->p_sigparent != 0) {
 			if (p->p_sigparent == SIGCHLD)
 				childproc_exited(p);
 			else	/* LINUX thread */
-				psignal(p->p_pptr, p->p_sigparent);
+				kern_psignal(p->p_pptr, p->p_sigparent);
 		}
 #ifdef PROCDESC
 	} else
@@ -568,7 +568,7 @@ struct abort2_args {
 #endif
 
 int
-abort2(struct thread *td, struct abort2_args *uap)
+sys_abort2(struct thread *td, struct abort2_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct sbuf *sb;
@@ -656,7 +656,7 @@ owait(struct thread *td, struct owait_args *uap __unused)
  * The dirty work is handled by kern_wait().
  */
 int
-wait4(struct thread *td, struct wait_args *uap)
+sys_wait4(struct thread *td, struct wait_args *uap)
 {
 	struct rusage ru, *rup;
 	int error, status;
@@ -765,12 +765,12 @@ proc_reap(struct thread *td, struct proc *p, int *status, int options,
 	/*
 	 * Destroy resource accounting information associated with the process.
 	 */
-	racct_proc_exit(p);
 #ifdef RACCT
-	PROC_LOCK(p->p_pptr);
-	racct_sub(p->p_pptr, RACCT_NPROC, 1);
-	PROC_UNLOCK(p->p_pptr);
+	PROC_LOCK(p);
+	racct_sub(p, RACCT_NPROC, 1);
+	PROC_UNLOCK(p);
 #endif
+	racct_proc_exit(p);
 
 	/*
 	 * Free credentials, arguments, and sigacts.
@@ -929,25 +929,13 @@ kern_wait(struct thread *td, pid_t pid, int *status, int options,
 void
 proc_reparent(struct proc *child, struct proc *parent)
 {
-#ifdef RACCT
-	int locked;
-#endif
 
 	sx_assert(&proctree_lock, SX_XLOCKED);
 	PROC_LOCK_ASSERT(child, MA_OWNED);
 	if (child->p_pptr == parent)
 		return;
 
-#ifdef RACCT
-	locked = PROC_LOCKED(parent);
-	if (!locked)
-		PROC_LOCK(parent);
-	racct_add_force(parent, RACCT_NPROC, 1);
-	if (!locked)
-		PROC_UNLOCK(parent);
-#endif
 	PROC_LOCK(child->p_pptr);
-	racct_sub(child->p_pptr, RACCT_NPROC, 1);
 	sigqueue_take(child->p_ksi);
 	PROC_UNLOCK(child->p_pptr);
 	LIST_REMOVE(child, p_sibling);
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 32d00550a815..60639c9fcb84 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -102,7 +102,7 @@ struct fork_args {
 
 /* ARGSUSED */
 int
-fork(struct thread *td, struct fork_args *uap)
+sys_fork(struct thread *td, struct fork_args *uap)
 {
 	int error;
 	struct proc *p2;
@@ -117,7 +117,7 @@ fork(struct thread *td, struct fork_args *uap)
 
 /* ARGUSED */
 int
-pdfork(td, uap)
+sys_pdfork(td, uap)
 	struct thread *td;
 	struct pdfork_args *uap;
 {
@@ -145,7 +145,7 @@ pdfork(td, uap)
 
 /* ARGSUSED */
 int
-vfork(struct thread *td, struct vfork_args *uap)
+sys_vfork(struct thread *td, struct vfork_args *uap)
 {
 	int error, flags;
 	struct proc *p2;
@@ -164,7 +164,7 @@ vfork(struct thread *td, struct vfork_args *uap)
 }
 
 int
-rfork(struct thread *td, struct rfork_args *uap)
+sys_rfork(struct thread *td, struct rfork_args *uap)
 {
 	struct proc *p2;
 	int error;
@@ -559,7 +559,7 @@ do_fork(struct thread *td, int flags, struct proc *p2, struct thread *td2,
 			 * to commit suicide.
 			 */
 			PROC_LOCK(p2);
-			psignal(p2, SIGKILL);
+			kern_psignal(p2, SIGKILL);
 			PROC_UNLOCK(p2);
 		} else
 			PROC_UNLOCK(p1->p_leader);
@@ -806,14 +806,6 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 		return (fork_norfproc(td, flags));
 	}
 
-#ifdef RACCT
-	PROC_LOCK(p1);
-	error = racct_add(p1, RACCT_NPROC, 1);
-	PROC_UNLOCK(p1);
-	if (error != 0)
-		return (EAGAIN);
-#endif
-
 #ifdef PROCDESC
 	/*
 	 * If required, create a process descriptor in the parent first; we
@@ -822,14 +814,8 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 	 */
 	if (flags & RFPROCDESC) {
 		error = falloc(td, &fp_procdesc, procdescp, 0);
-		if (error != 0) {
-#ifdef RACCT
-			PROC_LOCK(p1);
-			racct_sub(p1, RACCT_NPROC, 1);
-			PROC_UNLOCK(p1);
-#endif
+		if (error != 0)
 			return (error);
-		}
 	}
 #endif
 
@@ -877,11 +863,6 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 		}
 	} else
 		vm2 = NULL;
-#ifdef MAC
-	mac_proc_init(newproc);
-#endif
-	knlist_init_mtx(&newproc->p_klist, &newproc->p_mtx);
-	STAILQ_INIT(&newproc->p_ktr);
 
 	/*
 	 * XXX: This is ugly; when we copy resource usage, we need to bump
@@ -898,6 +879,12 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 		goto fail1;
 	}
 
+#ifdef MAC
+	mac_proc_init(newproc);
+#endif
+	knlist_init_mtx(&newproc->p_klist, &newproc->p_mtx);
+	STAILQ_INIT(&newproc->p_ktr);
+
 	/* We have to lock the process tree while we look for a pid. */
 	sx_slock(&proctree_lock);
 
@@ -915,19 +902,6 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 		goto fail;
 	}
 
-#ifdef RACCT
-	/*
-	 * After fork, there is exactly one thread running.
-	 */
-	PROC_LOCK(newproc);
-	error = racct_set(newproc, RACCT_NTHR, 1);
-	PROC_UNLOCK(newproc);
-	if (error != 0) {
-		error = EAGAIN;
-		goto fail;
-	}
-#endif
-
 	/*
 	 * Increment the count of procs running with this uid. Don't allow
 	 * a nonprivileged user to exceed their current limit.
@@ -954,12 +928,12 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 		if (flags & RFPROCDESC)
 			procdesc_finit(newproc->p_procdesc, fp_procdesc);
 #endif
+		racct_proc_fork_done(newproc);
 		return (0);
 	}
 
 	error = EAGAIN;
 fail:
-	racct_proc_exit(newproc);
 	sx_sunlock(&proctree_lock);
 	if (ppsratecheck(&lastfail, &curfail, 1))
 		printf("maxproc limit exceeded by uid %i, please see tuning(7) and login.conf(5).\n",
@@ -969,6 +943,7 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 	mac_proc_destroy(newproc);
 #endif
 fail1:
+	racct_proc_exit(newproc);
 	if (vm2 != NULL)
 		vmspace_free(vm2);
 	uma_zfree(proc_zone, newproc);
@@ -977,11 +952,6 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 		fdrop(fp_procdesc, td);
 #endif
 	pause("fork", hz / 2);
-#ifdef RACCT
-	PROC_LOCK(p1);
-	racct_sub(p1, RACCT_NPROC, 1);
-	PROC_UNLOCK(p1);
-#endif
 	return (error);
 }
 
diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
index 8ce8327bde15..37a2c59946fe 100644
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -276,7 +276,7 @@ qcmp_v6(const void *ip1, const void *ip2)
  * };
  */
 int
-jail(struct thread *td, struct jail_args *uap)
+sys_jail(struct thread *td, struct jail_args *uap)
 {
 	uint32_t version;
 	int error;
@@ -489,7 +489,7 @@ kern_jail(struct thread *td, struct jail *j)
  * };
  */
 int
-jail_set(struct thread *td, struct jail_set_args *uap)
+sys_jail_set(struct thread *td, struct jail_set_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -1819,7 +1819,7 @@ kern_jail_set(struct thread *td, struct uio *optuio, int flags)
  * };
  */
 int
-jail_get(struct thread *td, struct jail_get_args *uap)
+sys_jail_get(struct thread *td, struct jail_get_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -2119,7 +2119,7 @@ kern_jail_get(struct thread *td, struct uio *optuio, int flags)
  * };
  */
 int
-jail_remove(struct thread *td, struct jail_remove_args *uap)
+sys_jail_remove(struct thread *td, struct jail_remove_args *uap)
 {
 	struct prison *pr, *cpr, *lpr, *tpr;
 	int descend, error;
@@ -2206,7 +2206,7 @@ prison_remove_one(struct prison *pr)
 		PROC_LOCK(p);
 		if (p->p_state != PRS_NEW && p->p_ucred &&
 		    p->p_ucred->cr_prison == pr)
-			psignal(p, SIGKILL);
+			kern_psignal(p, SIGKILL);
 		PROC_UNLOCK(p);
 	}
 	sx_sunlock(&allproc_lock);
@@ -2221,7 +2221,7 @@ prison_remove_one(struct prison *pr)
  * };
  */
 int
-jail_attach(struct thread *td, struct jail_attach_args *uap)
+sys_jail_attach(struct thread *td, struct jail_attach_args *uap)
 {
 	struct prison *pr;
 	int error;
diff --git a/sys/kern/kern_ktrace.c b/sys/kern/kern_ktrace.c
index 5db5aee717d1..3bb529f7b09c 100644
--- a/sys/kern/kern_ktrace.c
+++ b/sys/kern/kern_ktrace.c
@@ -95,6 +95,7 @@ struct ktr_request {
 	void	*ktr_buffer;
 	union {
 		struct	ktr_proc_ctor ktr_proc_ctor;
+		struct	ktr_cap_fail ktr_cap_fail;
 		struct	ktr_syscall ktr_syscall;
 		struct	ktr_sysret ktr_sysret;
 		struct	ktr_genio ktr_genio;
@@ -117,6 +118,7 @@ static int data_lengths[] = {
 	0,					/* KTR_SYSCTL */
 	sizeof(struct ktr_proc_ctor),		/* KTR_PROCCTOR */
 	0,					/* KTR_PROCDTOR */
+	sizeof(struct ktr_cap_fail),		/* KTR_CAPFAIL */
 };
 
 static STAILQ_HEAD(, ktr_request) ktr_free;
@@ -768,6 +770,27 @@ ktrstruct(name, data, datalen)
 	req->ktr_header.ktr_len = buflen;
 	ktr_submitrequest(curthread, req);
 }
+
+void
+ktrcapfail(type, needed, held)
+	enum ktr_cap_fail_type type;
+	cap_rights_t needed;
+	cap_rights_t held;
+{
+	struct thread *td = curthread;
+	struct ktr_request *req;
+	struct ktr_cap_fail *kcf;
+
+	req = ktr_getrequest(KTR_CAPFAIL);
+	if (req == NULL)
+		return;
+	kcf = &req->ktr_data.ktr_cap_fail;
+	kcf->cap_type = type;
+	kcf->cap_needed = needed;
+	kcf->cap_held = held;
+	ktr_enqueuerequest(td, req);
+	ktrace_exit(td);
+}
 #endif /* KTRACE */
 
 /* Interface and common routines */
@@ -782,7 +805,7 @@ struct ktrace_args {
 #endif
 /* ARGSUSED */
 int
-ktrace(td, uap)
+sys_ktrace(td, uap)
 	struct thread *td;
 	register struct ktrace_args *uap;
 {
@@ -936,7 +959,7 @@ ktrace(td, uap)
 
 /* ARGSUSED */
 int
-utrace(td, uap)
+sys_utrace(td, uap)
 	struct thread *td;
 	register struct utrace_args *uap;
 {
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index 4337c953713e..5c5bfbd005ef 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -1045,7 +1045,7 @@ kern_kldload(struct thread *td, const char *file, int *fileid)
 }
 
 int
-kldload(struct thread *td, struct kldload_args *uap)
+sys_kldload(struct thread *td, struct kldload_args *uap)
 {
 	char *pathname = NULL;
 	int error, fileid;
@@ -1125,14 +1125,14 @@ kern_kldunload(struct thread *td, int fileid, int flags)
 }
 
 int
-kldunload(struct thread *td, struct kldunload_args *uap)
+sys_kldunload(struct thread *td, struct kldunload_args *uap)
 {
 
 	return (kern_kldunload(td, uap->fileid, LINKER_UNLOAD_NORMAL));
 }
 
 int
-kldunloadf(struct thread *td, struct kldunloadf_args *uap)
+sys_kldunloadf(struct thread *td, struct kldunloadf_args *uap)
 {
 
 	if (uap->flags != LINKER_UNLOAD_NORMAL &&
@@ -1142,7 +1142,7 @@ kldunloadf(struct thread *td, struct kldunloadf_args *uap)
 }
 
 int
-kldfind(struct thread *td, struct kldfind_args *uap)
+sys_kldfind(struct thread *td, struct kldfind_args *uap)
 {
 	char *pathname;
 	const char *filename;
@@ -1175,7 +1175,7 @@ kldfind(struct thread *td, struct kldfind_args *uap)
 }
 
 int
-kldnext(struct thread *td, struct kldnext_args *uap)
+sys_kldnext(struct thread *td, struct kldnext_args *uap)
 {
 	linker_file_t lf;
 	int error = 0;
@@ -1212,7 +1212,7 @@ kldnext(struct thread *td, struct kldnext_args *uap)
 }
 
 int
-kldstat(struct thread *td, struct kldstat_args *uap)
+sys_kldstat(struct thread *td, struct kldstat_args *uap)
 {
 	struct kld_file_stat stat;
 	int error, version;
@@ -1274,7 +1274,7 @@ kern_kldstat(struct thread *td, int fileid, struct kld_file_stat *stat)
 }
 
 int
-kldfirstmod(struct thread *td, struct kldfirstmod_args *uap)
+sys_kldfirstmod(struct thread *td, struct kldfirstmod_args *uap)
 {
 	linker_file_t lf;
 	module_t mp;
@@ -1303,7 +1303,7 @@ kldfirstmod(struct thread *td, struct kldfirstmod_args *uap)
 }
 
 int
-kldsym(struct thread *td, struct kldsym_args *uap)
+sys_kldsym(struct thread *td, struct kldsym_args *uap)
 {
 	char *symstr = NULL;
 	c_linker_sym_t sym;
diff --git a/sys/kern/kern_loginclass.c b/sys/kern/kern_loginclass.c
index d98024677ff8..beac93b21277 100644
--- a/sys/kern/kern_loginclass.c
+++ b/sys/kern/kern_loginclass.c
@@ -152,7 +152,7 @@ struct getloginclass_args {
 #endif
 /* ARGSUSED */
 int
-getloginclass(struct thread *td, struct getloginclass_args *uap)
+sys_getloginclass(struct thread *td, struct getloginclass_args *uap)
 {
 	int error = 0;
 	size_t lcnamelen;
@@ -184,7 +184,7 @@ struct setloginclass_args {
 #endif
 /* ARGSUSED */
 int
-setloginclass(struct thread *td, struct setloginclass_args *uap)
+sys_setloginclass(struct thread *td, struct setloginclass_args *uap)
 {
 	struct proc *p = td->td_proc;
 	int error;
diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index 7843caef27ee..ecc45db7e6e6 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -265,8 +265,8 @@ sysctl_kmem_map_free(SYSCTL_HANDLER_ARGS)
 	u_long size;
 
 	vm_map_lock_read(kmem_map);
-	size = kmem_map->root != NULL ?
-	    kmem_map->root->max_free : kmem_map->size;
+	size = kmem_map->root != NULL ? kmem_map->root->max_free :
+	    kmem_map->max_offset - kmem_map->min_offset;
 	vm_map_unlock_read(kmem_map);
 	return (sysctl_handle_long(oidp, &size, 0, req));
 }
@@ -407,6 +407,43 @@ malloc_type_freed(struct malloc_type *mtp, unsigned long size)
 	critical_exit();
 }
 
+/*
+ *	contigmalloc:
+ *
+ *	Allocate a block of physically contiguous memory.
+ *
+ *	If M_NOWAIT is set, this routine will not block and return NULL if
+ *	the allocation fails.
+ */
+void *
+contigmalloc(unsigned long size, struct malloc_type *type, int flags,
+    vm_paddr_t low, vm_paddr_t high, unsigned long alignment,
+    unsigned long boundary)
+{
+	void *ret;
+
+	ret = (void *)kmem_alloc_contig(kernel_map, size, flags, low, high,
+	    alignment, boundary, VM_MEMATTR_DEFAULT);
+	if (ret != NULL)
+		malloc_type_allocated(type, round_page(size));
+	return (ret);
+}
+
+/*
+ *	contigfree:
+ *
+ *	Free a block of memory allocated by contigmalloc.
+ *
+ *	This routine may not block.
+ */
+void
+contigfree(void *addr, unsigned long size, struct malloc_type *type)
+{
+
+	kmem_free(kernel_map, (vm_offset_t)addr, size);
+	malloc_type_freed(type, round_page(size));
+}
+
 /*
  *	malloc:
  *
@@ -458,7 +495,7 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
 		   ("malloc(M_WAITOK) in interrupt context"));
 
 #ifdef DEBUG_MEMGUARD
-	if (memguard_cmp(mtp, size)) {
+	if (memguard_cmp_mtp(mtp, size)) {
 		va = memguard_alloc(size, flags);
 		if (va != NULL)
 			return (va);
diff --git a/sys/kern/kern_module.c b/sys/kern/kern_module.c
index 98c69628019d..3f13373f032f 100644
--- a/sys/kern/kern_module.c
+++ b/sys/kern/kern_module.c
@@ -310,7 +310,7 @@ module_file(module_t mod)
  * Syscalls.
  */
 int
-modnext(struct thread *td, struct modnext_args *uap)
+sys_modnext(struct thread *td, struct modnext_args *uap)
 {
 	module_t mod;
 	int error = 0;
@@ -341,7 +341,7 @@ modnext(struct thread *td, struct modnext_args *uap)
 }
 
 int
-modfnext(struct thread *td, struct modfnext_args *uap)
+sys_modfnext(struct thread *td, struct modfnext_args *uap)
 {
 	module_t mod;
 	int error;
@@ -371,7 +371,7 @@ struct module_stat_v1 {
 };
 
 int
-modstat(struct thread *td, struct modstat_args *uap)
+sys_modstat(struct thread *td, struct modstat_args *uap)
 {
 	module_t mod;
 	modspecific_t data;
@@ -424,7 +424,7 @@ modstat(struct thread *td, struct modstat_args *uap)
 }
 
 int
-modfind(struct thread *td, struct modfind_args *uap)
+sys_modfind(struct thread *td, struct modfind_args *uap)
 {
 	int error = 0;
 	char name[MAXMODNAME];
diff --git a/sys/kern/kern_ntptime.c b/sys/kern/kern_ntptime.c
index 9ca3690f989b..1e56acfca950 100644
--- a/sys/kern/kern_ntptime.c
+++ b/sys/kern/kern_ntptime.c
@@ -274,7 +274,7 @@ struct ntp_gettime_args {
 #endif
 /* ARGSUSED */
 int
-ntp_gettime(struct thread *td, struct ntp_gettime_args *uap)
+sys_ntp_gettime(struct thread *td, struct ntp_gettime_args *uap)
 {	
 	struct ntptimeval ntv;
 
@@ -324,7 +324,7 @@ struct ntp_adjtime_args {
 #endif
 
 int
-ntp_adjtime(struct thread *td, struct ntp_adjtime_args *uap)
+sys_ntp_adjtime(struct thread *td, struct ntp_adjtime_args *uap)
 {
 	struct timex ntv;	/* temporary structure */
 	long freq;		/* frequency ns/s) */
@@ -932,7 +932,7 @@ struct adjtime_args {
 #endif
 /* ARGSUSED */
 int
-adjtime(struct thread *td, struct adjtime_args *uap)
+sys_adjtime(struct thread *td, struct adjtime_args *uap)
 {
 	struct timeval delta, olddelta, *deltap;
 	int error;
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index c84c5ae776db..998e7caaf650 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -608,8 +608,8 @@ orphanpg(pg)
 			PROC_UNLOCK(p);
 			LIST_FOREACH(p, &pg->pg_members, p_pglist) {
 				PROC_LOCK(p);
-				psignal(p, SIGHUP);
-				psignal(p, SIGCONT);
+				kern_psignal(p, SIGHUP);
+				kern_psignal(p, SIGCONT);
 				PROC_UNLOCK(p);
 			}
 			return;
diff --git a/sys/kern/kern_prot.c b/sys/kern/kern_prot.c
index 52db65915fde..cf939844576b 100644
--- a/sys/kern/kern_prot.c
+++ b/sys/kern/kern_prot.c
@@ -99,7 +99,7 @@ struct getpid_args {
 #endif
 /* ARGSUSED */
 int
-getpid(struct thread *td, struct getpid_args *uap)
+sys_getpid(struct thread *td, struct getpid_args *uap)
 {
 	struct proc *p = td->td_proc;
 
@@ -119,7 +119,7 @@ struct getppid_args {
 #endif
 /* ARGSUSED */
 int
-getppid(struct thread *td, struct getppid_args *uap)
+sys_getppid(struct thread *td, struct getppid_args *uap)
 {
 	struct proc *p = td->td_proc;
 
@@ -138,7 +138,7 @@ struct getpgrp_args {
 };
 #endif
 int
-getpgrp(struct thread *td, struct getpgrp_args *uap)
+sys_getpgrp(struct thread *td, struct getpgrp_args *uap)
 {
 	struct proc *p = td->td_proc;
 
@@ -155,7 +155,7 @@ struct getpgid_args {
 };
 #endif
 int
-getpgid(struct thread *td, struct getpgid_args *uap)
+sys_getpgid(struct thread *td, struct getpgid_args *uap)
 {
 	struct proc *p;
 	int error;
@@ -187,7 +187,7 @@ struct getsid_args {
 };
 #endif
 int
-getsid(struct thread *td, struct getsid_args *uap)
+sys_getsid(struct thread *td, struct getsid_args *uap)
 {
 	struct proc *p;
 	int error;
@@ -217,7 +217,7 @@ struct getuid_args {
 #endif
 /* ARGSUSED */
 int
-getuid(struct thread *td, struct getuid_args *uap)
+sys_getuid(struct thread *td, struct getuid_args *uap)
 {
 
 	td->td_retval[0] = td->td_ucred->cr_ruid;
@@ -234,7 +234,7 @@ struct geteuid_args {
 #endif
 /* ARGSUSED */
 int
-geteuid(struct thread *td, struct geteuid_args *uap)
+sys_geteuid(struct thread *td, struct geteuid_args *uap)
 {
 
 	td->td_retval[0] = td->td_ucred->cr_uid;
@@ -248,7 +248,7 @@ struct getgid_args {
 #endif
 /* ARGSUSED */
 int
-getgid(struct thread *td, struct getgid_args *uap)
+sys_getgid(struct thread *td, struct getgid_args *uap)
 {
 
 	td->td_retval[0] = td->td_ucred->cr_rgid;
@@ -270,7 +270,7 @@ struct getegid_args {
 #endif
 /* ARGSUSED */
 int
-getegid(struct thread *td, struct getegid_args *uap)
+sys_getegid(struct thread *td, struct getegid_args *uap)
 {
 
 	td->td_retval[0] = td->td_ucred->cr_groups[0];
@@ -284,7 +284,7 @@ struct getgroups_args {
 };
 #endif
 int
-getgroups(struct thread *td, register struct getgroups_args *uap)
+sys_getgroups(struct thread *td, register struct getgroups_args *uap)
 {
 	gid_t *groups;
 	u_int ngrp;
@@ -334,7 +334,7 @@ struct setsid_args {
 #endif
 /* ARGSUSED */
 int
-setsid(register struct thread *td, struct setsid_args *uap)
+sys_setsid(register struct thread *td, struct setsid_args *uap)
 {
 	struct pgrp *pgrp;
 	int error;
@@ -392,7 +392,7 @@ struct setpgid_args {
 #endif
 /* ARGSUSED */
 int
-setpgid(struct thread *td, register struct setpgid_args *uap)
+sys_setpgid(struct thread *td, register struct setpgid_args *uap)
 {
 	struct proc *curp = td->td_proc;
 	register struct proc *targp;	/* target process */
@@ -492,7 +492,7 @@ struct setuid_args {
 #endif
 /* ARGSUSED */
 int
-setuid(struct thread *td, struct setuid_args *uap)
+sys_setuid(struct thread *td, struct setuid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -606,7 +606,7 @@ struct seteuid_args {
 #endif
 /* ARGSUSED */
 int
-seteuid(struct thread *td, struct seteuid_args *uap)
+sys_seteuid(struct thread *td, struct seteuid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -662,7 +662,7 @@ struct setgid_args {
 #endif
 /* ARGSUSED */
 int
-setgid(struct thread *td, struct setgid_args *uap)
+sys_setgid(struct thread *td, struct setgid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -760,7 +760,7 @@ struct setegid_args {
 #endif
 /* ARGSUSED */
 int
-setegid(struct thread *td, struct setegid_args *uap)
+sys_setegid(struct thread *td, struct setegid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -807,7 +807,7 @@ struct setgroups_args {
 #endif
 /* ARGSUSED */
 int
-setgroups(struct thread *td, struct setgroups_args *uap)
+sys_setgroups(struct thread *td, struct setgroups_args *uap)
 {
 	gid_t *groups = NULL;
 	int error;
@@ -880,7 +880,7 @@ struct setreuid_args {
 #endif
 /* ARGSUSED */
 int
-setreuid(register struct thread *td, struct setreuid_args *uap)
+sys_setreuid(register struct thread *td, struct setreuid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -950,7 +950,7 @@ struct setregid_args {
 #endif
 /* ARGSUSED */
 int
-setregid(register struct thread *td, struct setregid_args *uap)
+sys_setregid(register struct thread *td, struct setregid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -1015,7 +1015,7 @@ struct setresuid_args {
 #endif
 /* ARGSUSED */
 int
-setresuid(register struct thread *td, struct setresuid_args *uap)
+sys_setresuid(register struct thread *td, struct setresuid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -1097,7 +1097,7 @@ struct setresgid_args {
 #endif
 /* ARGSUSED */
 int
-setresgid(register struct thread *td, struct setresgid_args *uap)
+sys_setresgid(register struct thread *td, struct setresgid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -1164,7 +1164,7 @@ struct getresuid_args {
 #endif
 /* ARGSUSED */
 int
-getresuid(register struct thread *td, struct getresuid_args *uap)
+sys_getresuid(register struct thread *td, struct getresuid_args *uap)
 {
 	struct ucred *cred;
 	int error1 = 0, error2 = 0, error3 = 0;
@@ -1191,7 +1191,7 @@ struct getresgid_args {
 #endif
 /* ARGSUSED */
 int
-getresgid(register struct thread *td, struct getresgid_args *uap)
+sys_getresgid(register struct thread *td, struct getresgid_args *uap)
 {
 	struct ucred *cred;
 	int error1 = 0, error2 = 0, error3 = 0;
@@ -1216,7 +1216,7 @@ struct issetugid_args {
 #endif
 /* ARGSUSED */
 int
-issetugid(register struct thread *td, struct issetugid_args *uap)
+sys_issetugid(register struct thread *td, struct issetugid_args *uap)
 {
 	struct proc *p = td->td_proc;
 
@@ -1235,7 +1235,7 @@ issetugid(register struct thread *td, struct issetugid_args *uap)
 }
 
 int
-__setugid(struct thread *td, struct __setugid_args *uap)
+sys___setugid(struct thread *td, struct __setugid_args *uap)
 {
 #ifdef REGRESSION
 	struct proc *p;
@@ -2071,7 +2071,7 @@ struct getlogin_args {
 #endif
 /* ARGSUSED */
 int
-getlogin(struct thread *td, struct getlogin_args *uap)
+sys_getlogin(struct thread *td, struct getlogin_args *uap)
 {
 	int error;
 	char login[MAXLOGNAME];
@@ -2098,7 +2098,7 @@ struct setlogin_args {
 #endif
 /* ARGSUSED */
 int
-setlogin(struct thread *td, struct setlogin_args *uap)
+sys_setlogin(struct thread *td, struct setlogin_args *uap)
 {
 	struct proc *p = td->td_proc;
 	int error;
diff --git a/sys/kern/kern_racct.c b/sys/kern/kern_racct.c
index b12b7c776ab8..a32814f6f8b6 100644
--- a/sys/kern/kern_racct.c
+++ b/sys/kern/kern_racct.c
@@ -261,12 +261,8 @@ racct_alloc_resource(struct racct *racct, int resource,
 	}
 }
 
-/*
- * Increase allocation of 'resource' by 'amount' for process 'p'.
- * Return 0 if it's below limits, or errno, if it's not.
- */
-int
-racct_add(struct proc *p, int resource, uint64_t amount)
+static int
+racct_add_locked(struct proc *p, int resource, uint64_t amount)
 {
 #ifdef RCTL
 	int error;
@@ -282,23 +278,35 @@ racct_add(struct proc *p, int resource, uint64_t amount)
 	 */
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 
-	mtx_lock(&racct_lock);
 #ifdef RCTL
 	error = rctl_enforce(p, resource, amount);
 	if (error && RACCT_IS_DENIABLE(resource)) {
 		SDT_PROBE(racct, kernel, rusage, add_failure, p, resource,
 		    amount, 0, 0);
-		mtx_unlock(&racct_lock);
 		return (error);
 	}
 #endif
 	racct_alloc_resource(p->p_racct, resource, amount);
 	racct_add_cred_locked(p->p_ucred, resource, amount);
-	mtx_unlock(&racct_lock);
 
 	return (0);
 }
 
+/*
+ * Increase allocation of 'resource' by 'amount' for process 'p'.
+ * Return 0 if it's below limits, or errno, if it's not.
+ */
+int
+racct_add(struct proc *p, int resource, uint64_t amount)
+{
+	int error;
+
+	mtx_lock(&racct_lock);
+	error = racct_add_locked(p, resource, amount);
+	mtx_unlock(&racct_lock);
+	return (error);
+}
+
 static void
 racct_add_cred_locked(struct ucred *cred, int resource, uint64_t amount)
 {
@@ -559,6 +567,12 @@ racct_proc_fork(struct proc *parent, struct proc *child)
 	PROC_LOCK(child);
 	mtx_lock(&racct_lock);
 
+#ifdef RCTL
+	error = rctl_proc_fork(parent, child);
+	if (error != 0)
+		goto out;
+#endif
+
 	/*
 	 * Inherit resource usage.
 	 */
@@ -569,32 +583,14 @@ racct_proc_fork(struct proc *parent, struct proc *child)
 
 		error = racct_set_locked(child, i,
 		    parent->p_racct->r_resources[i]);
-		if (error != 0) {
-			/*
-			 * XXX: The only purpose of these two lines is
-			 * to prevent from tripping checks in racct_destroy().
-			 */
-			for (i = 0; i <= RACCT_MAX; i++)
-				racct_set_locked(child, i, 0);
+		if (error != 0)
 			goto out;
-		}
 	}
 
-#ifdef RCTL
-	error = rctl_proc_fork(parent, child);
-	if (error != 0) {
-		/*
-		 * XXX: The only purpose of these two lines is to prevent from
-		 * tripping checks in racct_destroy().
-		 */
-		for (i = 0; i <= RACCT_MAX; i++)
-			racct_set_locked(child, i, 0);
-	}
-#endif
+	error = racct_add_locked(child, RACCT_NPROC, 1);
+	error += racct_add_locked(child, RACCT_NTHR, 1);
 
 out:
-	if (error != 0)
-		racct_destroy_locked(&child->p_racct);
 	mtx_unlock(&racct_lock);
 	PROC_UNLOCK(child);
 	PROC_UNLOCK(parent);
@@ -602,6 +598,24 @@ racct_proc_fork(struct proc *parent, struct proc *child)
 	return (error);
 }
 
+/*
+ * Called at the end of fork1(), to handle rules that require the process
+ * to be fully initialized.
+ */
+void
+racct_proc_fork_done(struct proc *child)
+{
+
+#ifdef RCTL
+	PROC_LOCK(child);
+	mtx_lock(&racct_lock);
+	rctl_enforce(child, RACCT_NPROC, 0);
+	rctl_enforce(child, RACCT_NTHR, 0);
+	mtx_unlock(&racct_lock);
+	PROC_UNLOCK(child);
+#endif
+}
+
 void
 racct_proc_exit(struct proc *p)
 {
@@ -826,6 +840,11 @@ racct_proc_fork(struct proc *parent, struct proc *child)
 	return (0);
 }
 
+void
+racct_proc_fork_done(struct proc *child)
+{
+}
+
 void
 racct_proc_exit(struct proc *p)
 {
diff --git a/sys/kern/kern_rctl.c b/sys/kern/kern_rctl.c
index d60cae44205d..ad3145686390 100644
--- a/sys/kern/kern_rctl.c
+++ b/sys/kern/kern_rctl.c
@@ -312,6 +312,16 @@ rctl_enforce(struct proc *p, int resource, uint64_t amount)
 			if (link->rrl_exceeded != 0)
 				continue;
 
+			/*
+			 * If the process state is not fully initialized yet,
+			 * we can't access most of the required fields, e.g.
+			 * p->p_comm.  This happens when called from fork1().
+			 * Ignore this rule for now; it will be processed just
+			 * after fork, when called from racct_proc_fork_done().
+			 */
+			if (p->p_state != PRS_NORMAL)
+				continue;
+
 			if (!ppsratecheck(&lasttime, &curtime, 10))
 				continue;
 
@@ -335,6 +345,9 @@ rctl_enforce(struct proc *p, int resource, uint64_t amount)
 			if (link->rrl_exceeded != 0)
 				continue;
 
+			if (p->p_state != PRS_NORMAL)
+				continue;
+	
 			buf = malloc(RCTL_LOG_BUFSIZE, M_RCTL, M_NOWAIT);
 			if (buf == NULL) {
 				printf("rctl_enforce: out of memory\n");
@@ -357,27 +370,19 @@ rctl_enforce(struct proc *p, int resource, uint64_t amount)
 			if (link->rrl_exceeded != 0)
 				continue;
 
+			if (p->p_state != PRS_NORMAL)
+				continue;
+
 			KASSERT(rule->rr_action > 0 &&
 			    rule->rr_action <= RCTL_ACTION_SIGNAL_MAX,
 			    ("rctl_enforce: unknown action %d",
 			     rule->rr_action));
 
-			/*
-			 * We're supposed to send a signal, but the process
-			 * is not fully initialized yet, probably because we
-			 * got called from fork1().  For now just deny the
-			 * allocation instead.
-			 */
-			if (p->p_state != PRS_NORMAL) {
-				should_deny = 1;
-				continue;
-			}
-
 			/*
 			 * We're using the fact that RCTL_ACTION_SIG* values
 			 * are equal to their counterparts from sys/signal.h.
 			 */
-			psignal(p, rule->rr_action);
+			kern_psignal(p, rule->rr_action);
 			link->rrl_exceeded = 1;
 			continue;
 		}
@@ -1242,7 +1247,7 @@ rctl_racct_to_sbuf(struct racct *racct, int sloppy)
 }
 
 int
-rctl_get_racct(struct thread *td, struct rctl_get_racct_args *uap)
+sys_rctl_get_racct(struct thread *td, struct rctl_get_racct_args *uap)
 {
 	int error;
 	char *inputstr;
@@ -1338,7 +1343,7 @@ rctl_get_rules_callback(struct racct *racct, void *arg2, void *arg3)
 }
 
 int
-rctl_get_rules(struct thread *td, struct rctl_get_rules_args *uap)
+sys_rctl_get_rules(struct thread *td, struct rctl_get_rules_args *uap)
 {
 	int error;
 	size_t bufsize = RCTL_DEFAULT_BUFSIZE;
@@ -1413,7 +1418,7 @@ rctl_get_rules(struct thread *td, struct rctl_get_rules_args *uap)
 }
 
 int
-rctl_get_limits(struct thread *td, struct rctl_get_limits_args *uap)
+sys_rctl_get_limits(struct thread *td, struct rctl_get_limits_args *uap)
 {
 	int error;
 	size_t bufsize = RCTL_DEFAULT_BUFSIZE;
@@ -1487,7 +1492,7 @@ rctl_get_limits(struct thread *td, struct rctl_get_limits_args *uap)
 }
 
 int
-rctl_add_rule(struct thread *td, struct rctl_add_rule_args *uap)
+sys_rctl_add_rule(struct thread *td, struct rctl_add_rule_args *uap)
 {
 	int error;
 	struct rctl_rule *rule;
@@ -1529,7 +1534,7 @@ rctl_add_rule(struct thread *td, struct rctl_add_rule_args *uap)
 }
 
 int
-rctl_remove_rule(struct thread *td, struct rctl_remove_rule_args *uap)
+sys_rctl_remove_rule(struct thread *td, struct rctl_remove_rule_args *uap)
 {
 	int error;
 	struct rctl_rule *filter;
@@ -1801,35 +1806,35 @@ rctl_init(void)
 #else /* !RCTL */
 
 int
-rctl_get_racct(struct thread *td, struct rctl_get_racct_args *uap)
+sys_rctl_get_racct(struct thread *td, struct rctl_get_racct_args *uap)
 {
 	
 	return (ENOSYS);
 }
 
 int
-rctl_get_rules(struct thread *td, struct rctl_get_rules_args *uap)
+sys_rctl_get_rules(struct thread *td, struct rctl_get_rules_args *uap)
 {
 	
 	return (ENOSYS);
 }
 
 int
-rctl_get_limits(struct thread *td, struct rctl_get_limits_args *uap)
+sys_rctl_get_limits(struct thread *td, struct rctl_get_limits_args *uap)
 {
 	
 	return (ENOSYS);
 }
 
 int
-rctl_add_rule(struct thread *td, struct rctl_add_rule_args *uap)
+sys_rctl_add_rule(struct thread *td, struct rctl_add_rule_args *uap)
 {
 	
 	return (ENOSYS);
 }
 
 int
-rctl_remove_rule(struct thread *td, struct rctl_remove_rule_args *uap)
+sys_rctl_remove_rule(struct thread *td, struct rctl_remove_rule_args *uap)
 {
 	
 	return (ENOSYS);
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 78a25ebaad99..e6e80216d8ef 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -89,7 +89,7 @@ struct getpriority_args {
 };
 #endif
 int
-getpriority(td, uap)
+sys_getpriority(td, uap)
 	struct thread *td;
 	register struct getpriority_args *uap;
 {
@@ -174,7 +174,7 @@ struct setpriority_args {
 };
 #endif
 int
-setpriority(td, uap)
+sys_setpriority(td, uap)
 	struct thread *td;
 	struct setpriority_args *uap;
 {
@@ -284,7 +284,7 @@ struct rtprio_thread_args {
 };
 #endif
 int
-rtprio_thread(struct thread *td, struct rtprio_thread_args *uap)
+sys_rtprio_thread(struct thread *td, struct rtprio_thread_args *uap)
 {
 	struct proc *p;
 	struct rtprio rtp;
@@ -358,7 +358,7 @@ struct rtprio_args {
 };
 #endif
 int
-rtprio(td, uap)
+sys_rtprio(td, uap)
 	struct thread *td;		/* curthread */
 	register struct rtprio_args *uap;
 {
@@ -592,7 +592,7 @@ struct __setrlimit_args {
 };
 #endif
 int
-setrlimit(td, uap)
+sys_setrlimit(td, uap)
 	struct thread *td;
 	register struct __setrlimit_args *uap;
 {
@@ -632,7 +632,7 @@ lim_cb(void *arg)
 		} else {
 			if (p->p_cpulimit < rlim.rlim_max)
 				p->p_cpulimit += 5;
-			psignal(p, SIGXCPU);
+			kern_psignal(p, SIGXCPU);
 		}
 	}
 	if ((p->p_flag & P_WEXIT) == 0)
@@ -771,7 +771,7 @@ struct __getrlimit_args {
 #endif
 /* ARGSUSED */
 int
-getrlimit(td, uap)
+sys_getrlimit(td, uap)
 	struct thread *td;
 	register struct __getrlimit_args *uap;
 {
@@ -950,7 +950,7 @@ struct getrusage_args {
 };
 #endif
 int
-getrusage(td, uap)
+sys_getrusage(td, uap)
 	register struct thread *td;
 	register struct getrusage_args *uap;
 {
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index c1e3dda4fb19..039ca7e18b8a 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -172,7 +172,7 @@ SYSINIT(shutdown_conf, SI_SUB_INTRINSIC, SI_ORDER_ANY, shutdown_conf, NULL);
  */
 /* ARGSUSED */
 int
-reboot(struct thread *td, struct reboot_args *uap)
+sys_reboot(struct thread *td, struct reboot_args *uap)
 {
 	int error;
 
@@ -204,7 +204,7 @@ shutdown_nice(int howto)
 	/* Send a signal to init(8) and have it shutdown the world */
 	if (initproc != NULL) {
 		PROC_LOCK(initproc);
-		psignal(initproc, SIGINT);
+		kern_psignal(initproc, SIGINT);
 		PROC_UNLOCK(initproc);
 	} else {
 		/* No init(8) running, so simply reboot */
@@ -327,7 +327,7 @@ kern_reboot(int howto)
 #ifdef SW_WATCHDOG
 		wdog_kern_pat(WD_LASTVAL);
 #endif
-		sync(curthread, NULL);
+		sys_sync(curthread, NULL);
 
 		/*
 		 * With soft updates, some buffers that are
@@ -355,7 +355,7 @@ kern_reboot(int howto)
 #ifdef SW_WATCHDOG
 			wdog_kern_pat(WD_LASTVAL);
 #endif
-			sync(curthread, NULL);
+			sys_sync(curthread, NULL);
 
 #ifdef PREEMPTION
 			/*
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 26ef0d7f31d4..188bf83bc79c 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -784,7 +784,7 @@ struct sigaction_args {
 };
 #endif
 int
-sigaction(td, uap)
+sys_sigaction(td, uap)
 	struct thread *td;
 	register struct sigaction_args *uap;
 {
@@ -1035,7 +1035,7 @@ struct sigprocmask_args {
 };
 #endif
 int
-sigprocmask(td, uap)
+sys_sigprocmask(td, uap)
 	register struct thread *td;
 	struct sigprocmask_args *uap;
 {
@@ -1080,7 +1080,7 @@ osigprocmask(td, uap)
 #endif /* COMPAT_43 */
 
 int
-sigwait(struct thread *td, struct sigwait_args *uap)
+sys_sigwait(struct thread *td, struct sigwait_args *uap)
 {
 	ksiginfo_t ksi;
 	sigset_t set;
@@ -1094,6 +1094,8 @@ sigwait(struct thread *td, struct sigwait_args *uap)
 
 	error = kern_sigtimedwait(td, set, &ksi, NULL);
 	if (error) {
+		if (error == EINTR && td->td_proc->p_osrel < P_OSREL_SIGWAIT)
+			error = ERESTART;
 		if (error == ERESTART)
 			return (error);
 		td->td_retval[0] = error;
@@ -1106,7 +1108,7 @@ sigwait(struct thread *td, struct sigwait_args *uap)
 }
 
 int
-sigtimedwait(struct thread *td, struct sigtimedwait_args *uap)
+sys_sigtimedwait(struct thread *td, struct sigtimedwait_args *uap)
 {
 	struct timespec ts;
 	struct timespec *timeout;
@@ -1140,7 +1142,7 @@ sigtimedwait(struct thread *td, struct sigtimedwait_args *uap)
 }
 
 int
-sigwaitinfo(struct thread *td, struct sigwaitinfo_args *uap)
+sys_sigwaitinfo(struct thread *td, struct sigwaitinfo_args *uap)
 {
 	ksiginfo_t ksi;
 	sigset_t set;
@@ -1282,7 +1284,7 @@ struct sigpending_args {
 };
 #endif
 int
-sigpending(td, uap)
+sys_sigpending(td, uap)
 	struct thread *td;
 	struct sigpending_args *uap;
 {
@@ -1414,7 +1416,7 @@ struct sigsuspend_args {
 #endif
 /* ARGSUSED */
 int
-sigsuspend(td, uap)
+sys_sigsuspend(td, uap)
 	struct thread *td;
 	struct sigsuspend_args *uap;
 {
@@ -1533,7 +1535,7 @@ struct sigaltstack_args {
 #endif
 /* ARGSUSED */
 int
-sigaltstack(td, uap)
+sys_sigaltstack(td, uap)
 	struct thread *td;
 	register struct sigaltstack_args *uap;
 {
@@ -1660,7 +1662,7 @@ struct kill_args {
 #endif
 /* ARGSUSED */
 int
-kill(struct thread *td, struct kill_args *uap)
+sys_kill(struct thread *td, struct kill_args *uap)
 {
 	ksiginfo_t ksi;
 	struct proc *p;
@@ -1702,7 +1704,7 @@ kill(struct thread *td, struct kill_args *uap)
 }
 
 int
-pdkill(td, uap)
+sys_pdkill(td, uap)
 	struct thread *td;
 	struct pdkill_args *uap;
 {
@@ -1721,7 +1723,7 @@ pdkill(td, uap)
 	AUDIT_ARG_PROCESS(p);
 	error = p_cansignal(td, p, uap->signum);
 	if (error == 0 && uap->signum)
-		psignal(p, uap->signum);
+		kern_psignal(p, uap->signum);
 	PROC_UNLOCK(p);
 	return (error);
 #else
@@ -1764,7 +1766,7 @@ struct sigqueue_args {
 };
 #endif
 int
-sigqueue(struct thread *td, struct sigqueue_args *uap)
+sys_sigqueue(struct thread *td, struct sigqueue_args *uap)
 {
 	ksiginfo_t ksi;
 	struct proc *p;
@@ -1952,7 +1954,7 @@ sigtd(struct proc *p, int sig, int prop)
  * side effects of this unwise possibility.
  */
 void
-psignal(struct proc *p, int sig)
+kern_psignal(struct proc *p, int sig)
 {
 	ksiginfo_t ksi;
 
@@ -2826,7 +2828,7 @@ killproc(p, why)
 	log(LOG_ERR, "pid %d (%s), uid %d, was killed: %s\n", p->p_pid, p->p_comm,
 		p->p_ucred ? p->p_ucred->cr_uid : -1, why);
 	p->p_flag |= P_WKILLED;
-	psignal(p, SIGKILL);
+	kern_psignal(p, SIGKILL);
 }
 
 /*
@@ -3309,7 +3311,7 @@ nosys(td, args)
 	struct proc *p = td->td_proc;
 
 	PROC_LOCK(p);
-	psignal(p, SIGSYS);
+	kern_psignal(p, SIGSYS);
 	PROC_UNLOCK(p);
 	return (ENOSYS);
 }
@@ -3339,7 +3341,7 @@ pgsigio(sigiop, sig, checkctty)
 	if (sigio->sio_pgid > 0) {
 		PROC_LOCK(sigio->sio_proc);
 		if (CANSIGIO(sigio->sio_ucred, sigio->sio_proc->p_ucred))
-			psignal(sigio->sio_proc, sig);
+			kern_psignal(sigio->sio_proc, sig);
 		PROC_UNLOCK(sigio->sio_proc);
 	} else if (sigio->sio_pgid < 0) {
 		struct proc *p;
@@ -3350,7 +3352,7 @@ pgsigio(sigiop, sig, checkctty)
 			if (p->p_state == PRS_NORMAL &&
 			    CANSIGIO(sigio->sio_ucred, p->p_ucred) &&
 			    (checkctty == 0 || (p->p_flag & P_CONTROLT)))
-				psignal(p, sig);
+				kern_psignal(p, sig);
 			PROC_UNLOCK(p);
 		}
 		PGRP_UNLOCK(sigio->sio_pgrp);
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 05fb4a119a3f..a2c26ae10413 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -573,7 +573,7 @@ kern_yield(int prio)
  * General purpose yield system call.
  */
 int
-yield(struct thread *td, struct yield_args *uap)
+sys_yield(struct thread *td, struct yield_args *uap)
 {
 
 	thread_lock(td);
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index 956bf0c873cb..97d03a32ea18 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1530,7 +1530,7 @@ struct sysctl_args {
 };
 #endif
 int
-__sysctl(struct thread *td, struct sysctl_args *uap)
+sys___sysctl(struct thread *td, struct sysctl_args *uap)
 {
 	int error, i, name[CTL_MAXNAME];
 	size_t j;
diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c
index 94e41e285ecf..e997e48534a3 100644
--- a/sys/kern/kern_thr.c
+++ b/sys/kern/kern_thr.c
@@ -97,7 +97,7 @@ static int create_thread(struct thread *td, mcontext_t *ctx,
  * System call interface.
  */
 int
-thr_create(struct thread *td, struct thr_create_args *uap)
+sys_thr_create(struct thread *td, struct thr_create_args *uap)
     /* ucontext_t *ctx, long *id, int flags */
 {
 	ucontext_t ctx;
@@ -112,7 +112,7 @@ thr_create(struct thread *td, struct thr_create_args *uap)
 }
 
 int
-thr_new(struct thread *td, struct thr_new_args *uap)
+sys_thr_new(struct thread *td, struct thr_new_args *uap)
     /* struct thr_param * */
 {
 	struct thr_param param;
@@ -288,7 +288,7 @@ create_thread(struct thread *td, mcontext_t *ctx,
 }
 
 int
-thr_self(struct thread *td, struct thr_self_args *uap)
+sys_thr_self(struct thread *td, struct thr_self_args *uap)
     /* long *id */
 {
 	int error;
@@ -300,7 +300,7 @@ thr_self(struct thread *td, struct thr_self_args *uap)
 }
 
 int
-thr_exit(struct thread *td, struct thr_exit_args *uap)
+sys_thr_exit(struct thread *td, struct thr_exit_args *uap)
     /* long *state */
 {
 	struct proc *p;
@@ -337,7 +337,7 @@ thr_exit(struct thread *td, struct thr_exit_args *uap)
 }
 
 int
-thr_kill(struct thread *td, struct thr_kill_args *uap)
+sys_thr_kill(struct thread *td, struct thr_kill_args *uap)
     /* long id, int sig */
 {
 	ksiginfo_t ksi;
@@ -384,7 +384,7 @@ thr_kill(struct thread *td, struct thr_kill_args *uap)
 }
 
 int
-thr_kill2(struct thread *td, struct thr_kill2_args *uap)
+sys_thr_kill2(struct thread *td, struct thr_kill2_args *uap)
     /* pid_t pid, long id, int sig */
 {
 	ksiginfo_t ksi;
@@ -441,7 +441,7 @@ thr_kill2(struct thread *td, struct thr_kill2_args *uap)
 }
 
 int
-thr_suspend(struct thread *td, struct thr_suspend_args *uap)
+sys_thr_suspend(struct thread *td, struct thr_suspend_args *uap)
 	/* const struct timespec *timeout */
 {
 	struct timespec ts, *tsp;
@@ -506,7 +506,7 @@ kern_thr_suspend(struct thread *td, struct timespec *tsp)
 }
 
 int
-thr_wake(struct thread *td, struct thr_wake_args *uap)
+sys_thr_wake(struct thread *td, struct thr_wake_args *uap)
 	/* long id */
 {
 	struct proc *p;
@@ -530,7 +530,7 @@ thr_wake(struct thread *td, struct thr_wake_args *uap)
 }
 
 int
-thr_set_name(struct thread *td, struct thr_set_name_args *uap)
+sys_thr_set_name(struct thread *td, struct thr_set_name_args *uap)
 {
 	struct proc *p;
 	char name[MAXCOMLEN + 1];
diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c
index a153fbc7749c..4ba6117ed645 100644
--- a/sys/kern/kern_time.c
+++ b/sys/kern/kern_time.c
@@ -172,7 +172,7 @@ struct clock_gettime_args {
 #endif
 /* ARGSUSED */
 int
-clock_gettime(struct thread *td, struct clock_gettime_args *uap)
+sys_clock_gettime(struct thread *td, struct clock_gettime_args *uap)
 {
 	struct timespec ats;
 	int error;
@@ -255,7 +255,7 @@ struct clock_settime_args {
 #endif
 /* ARGSUSED */
 int
-clock_settime(struct thread *td, struct clock_settime_args *uap)
+sys_clock_settime(struct thread *td, struct clock_settime_args *uap)
 {
 	struct timespec ats;
 	int error;
@@ -290,7 +290,7 @@ struct clock_getres_args {
 };
 #endif
 int
-clock_getres(struct thread *td, struct clock_getres_args *uap)
+sys_clock_getres(struct thread *td, struct clock_getres_args *uap)
 {
 	struct timespec ts;
 	int error;
@@ -394,7 +394,7 @@ struct nanosleep_args {
 #endif
 /* ARGSUSED */
 int
-nanosleep(struct thread *td, struct nanosleep_args *uap)
+sys_nanosleep(struct thread *td, struct nanosleep_args *uap)
 {
 	struct timespec rmt, rqt;
 	int error;
@@ -425,7 +425,7 @@ struct gettimeofday_args {
 #endif
 /* ARGSUSED */
 int
-gettimeofday(struct thread *td, struct gettimeofday_args *uap)
+sys_gettimeofday(struct thread *td, struct gettimeofday_args *uap)
 {
 	struct timeval atv;
 	struct timezone rtz;
@@ -451,7 +451,7 @@ struct settimeofday_args {
 #endif
 /* ARGSUSED */
 int
-settimeofday(struct thread *td, struct settimeofday_args *uap)
+sys_settimeofday(struct thread *td, struct settimeofday_args *uap)
 {
 	struct timeval atv, *tvp;
 	struct timezone atz, *tzp;
@@ -523,7 +523,7 @@ struct getitimer_args {
 };
 #endif
 int
-getitimer(struct thread *td, struct getitimer_args *uap)
+sys_getitimer(struct thread *td, struct getitimer_args *uap)
 {
 	struct itimerval aitv;
 	int error;
@@ -575,14 +575,14 @@ struct setitimer_args {
 };
 #endif
 int
-setitimer(struct thread *td, struct setitimer_args *uap)
+sys_setitimer(struct thread *td, struct setitimer_args *uap)
 {
 	struct itimerval aitv, oitv;
 	int error;
 
 	if (uap->itv == NULL) {
 		uap->itv = uap->oitv;
-		return (getitimer(td, (struct getitimer_args *)uap));
+		return (sys_getitimer(td, (struct getitimer_args *)uap));
 	}
 
 	if ((error = copyin(uap->itv, &aitv, sizeof(struct itimerval))))
@@ -660,7 +660,7 @@ realitexpire(void *arg)
 
 	p = (struct proc *)arg;
 	PROC_LOCK(p);
-	psignal(p, SIGALRM);
+	kern_psignal(p, SIGALRM);
 	if (!timevalisset(&p->p_realtimer.it_interval)) {
 		timevalclear(&p->p_realtimer.it_value);
 		if (p->p_flag & P_WEXIT)
@@ -923,7 +923,7 @@ struct ktimer_create_args {
 };
 #endif
 int
-ktimer_create(struct thread *td, struct ktimer_create_args *uap)
+sys_ktimer_create(struct thread *td, struct ktimer_create_args *uap)
 {
 	struct sigevent *evp1, ev;
 	int id;
@@ -1062,7 +1062,7 @@ struct ktimer_delete_args {
 };
 #endif
 int
-ktimer_delete(struct thread *td, struct ktimer_delete_args *uap)
+sys_ktimer_delete(struct thread *td, struct ktimer_delete_args *uap)
 {
 	return (kern_timer_delete(td, uap->timerid));
 }
@@ -1127,7 +1127,7 @@ struct ktimer_settime_args {
 };
 #endif
 int
-ktimer_settime(struct thread *td, struct ktimer_settime_args *uap)
+sys_ktimer_settime(struct thread *td, struct ktimer_settime_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct itimer *it;
@@ -1168,7 +1168,7 @@ struct ktimer_gettime_args {
 };
 #endif
 int
-ktimer_gettime(struct thread *td, struct ktimer_gettime_args *uap)
+sys_ktimer_gettime(struct thread *td, struct ktimer_gettime_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct itimer *it;
@@ -1199,7 +1199,7 @@ struct timer_getoverrun_args {
 };
 #endif
 int
-ktimer_getoverrun(struct thread *td, struct ktimer_getoverrun_args *uap)
+sys_ktimer_getoverrun(struct thread *td, struct ktimer_getoverrun_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct itimer *it;
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 1edff748ffee..dfd6b123928b 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -2885,14 +2885,14 @@ do_sem_wake(struct thread *td, struct _usem *sem)
 }
 
 int
-_umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
+sys__umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
     /* struct umtx *umtx */
 {
 	return _do_lock_umtx(td, uap->umtx, td->td_tid, 0);
 }
 
 int
-_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
+sys__umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
     /* struct umtx *umtx */
 {
 	return do_unlock_umtx(td, uap->umtx, td->td_tid);
@@ -3239,7 +3239,7 @@ static _umtx_op_func op_table[] = {
 };
 
 int
-_umtx_op(struct thread *td, struct _umtx_op_args *uap)
+sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
 {
 	if ((unsigned)uap->op < UMTX_OP_MAX)
 		return (*op_table[uap->op])(td, uap);
diff --git a/sys/kern/kern_uuid.c b/sys/kern/kern_uuid.c
index 8c083f4e1090..693bb2500f3a 100644
--- a/sys/kern/kern_uuid.c
+++ b/sys/kern/kern_uuid.c
@@ -187,7 +187,7 @@ struct uuidgen_args {
 };
 #endif
 int
-uuidgen(struct thread *td, struct uuidgen_args *uap)
+sys_uuidgen(struct thread *td, struct uuidgen_args *uap)
 {
 	struct uuid *store;
 	size_t count;
diff --git a/sys/kern/makesyscalls.sh b/sys/kern/makesyscalls.sh
index 835ffb2ca7c2..d1162b52e455 100644
--- a/sys/kern/makesyscalls.sh
+++ b/sys/kern/makesyscalls.sh
@@ -424,8 +424,16 @@ s/\$//g
 			printf("struct %s {\n\tregister_t dummy;\n};\n",
 			    argalias) > sysarg
 		if (!flag("NOPROTO") && !flag("NODEF")) {
-			printf("%s\t%s(struct thread *, struct %s *)",
-			    rettype, funcname, argalias) > sysdcl
+			if (funcname == "nosys" || funcname == "lkmnosys" ||
+			    funcname == "sysarch" || funcname ~ /^freebsd/ || 
+			    funcname ~ /^linux/ || funcname ~ /^svr4/ || 
+			    funcname ~ /^ibcs2/ || funcname ~ /^xenix/) {				
+				printf("%s\t%s(struct thread *, struct %s *)",
+				    rettype, funcname, argalias) > sysdcl
+			} else {
+				printf("%s\tsys_%s(struct thread *, struct %s *)",
+				    rettype, funcname, argalias) > sysdcl
+			} 
 			printf(";\n") > sysdcl
 			printf("#define\t%sAUE_%s\t%s\n", syscallprefix,
 			    funcalias, auditev) > sysaue
@@ -436,8 +444,16 @@ s/\$//g
 			printf("%s },", "lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT") > sysent
 			column = column + length("lkmressys") + length("AUE_NULL") + 3
 		} else {
-			printf("%s, %s, NULL, 0, 0, %s, %s },", funcname, auditev, flags, thr_flag) > sysent
-			column = column + length(funcname) + length(auditev) + length(flags) + 3
+			if (funcname == "nosys" || funcname == "sysarch" || 
+			    funcname == "lkmnosys" || funcname ~ /^freebsd/ ||
+			    funcname ~ /^linux/ || funcname ~ /^svr4/ ||
+			    funcname ~ /^ibcs2/ || funcname ~ /^xenix/) {
+				printf("%s, %s, NULL, 0, 0, %s, %s },", funcname, auditev, flags, thr_flag) > sysent
+				column = column + length(funcname) + length(auditev) + length(flags) + 3 
+			} else {
+				printf("sys_%s, %s, NULL, 0, 0, %s, %s },", funcname, auditev, flags, thr_flag) > sysent
+				column = column + length(funcname) + length(auditev) + length(flags) + 3 + 4
+			} 
 		} 
 		align_sysent_comment(column)
 		printf("/* %d = %s */\n", syscall, funcalias) > sysent
diff --git a/sys/kern/p1003_1b.c b/sys/kern/p1003_1b.c
index 9620a36bfa10..fb89efc5e00f 100644
--- a/sys/kern/p1003_1b.c
+++ b/sys/kern/p1003_1b.c
@@ -108,7 +108,7 @@ sched_attach(void)
 }
 
 int
-sched_setparam(struct thread *td, struct sched_setparam_args *uap)
+sys_sched_setparam(struct thread *td, struct sched_setparam_args *uap)
 {
 	struct thread *targettd;
 	struct proc *targetp;
@@ -140,7 +140,7 @@ sched_setparam(struct thread *td, struct sched_setparam_args *uap)
 }
 
 int
-sched_getparam(struct thread *td, struct sched_getparam_args *uap)
+sys_sched_getparam(struct thread *td, struct sched_getparam_args *uap)
 {
 	int e;
 	struct sched_param sched_param;
@@ -170,7 +170,7 @@ sched_getparam(struct thread *td, struct sched_getparam_args *uap)
 }
 
 int
-sched_setscheduler(struct thread *td, struct sched_setscheduler_args *uap)
+sys_sched_setscheduler(struct thread *td, struct sched_setscheduler_args *uap)
 {
 	int e;
 	struct sched_param sched_param;
@@ -207,7 +207,7 @@ sched_setscheduler(struct thread *td, struct sched_setscheduler_args *uap)
 }
 
 int
-sched_getscheduler(struct thread *td, struct sched_getscheduler_args *uap)
+sys_sched_getscheduler(struct thread *td, struct sched_getscheduler_args *uap)
 {
 	int e, policy;
 	struct thread *targettd;
@@ -235,7 +235,7 @@ sched_getscheduler(struct thread *td, struct sched_getscheduler_args *uap)
 }
 
 int
-sched_yield(struct thread *td, struct sched_yield_args *uap)
+sys_sched_yield(struct thread *td, struct sched_yield_args *uap)
 {
 
 	sched_relinquish(curthread);
@@ -243,7 +243,7 @@ sched_yield(struct thread *td, struct sched_yield_args *uap)
 }
 
 int
-sched_get_priority_max(struct thread *td,
+sys_sched_get_priority_max(struct thread *td,
     struct sched_get_priority_max_args *uap)
 {
 	int error, prio;
@@ -254,7 +254,7 @@ sched_get_priority_max(struct thread *td,
 }
 
 int
-sched_get_priority_min(struct thread *td,
+sys_sched_get_priority_min(struct thread *td,
     struct sched_get_priority_min_args *uap)
 {
 	int error, prio;
@@ -265,7 +265,7 @@ sched_get_priority_min(struct thread *td,
 }
 
 int
-sched_rr_get_interval(struct thread *td,
+sys_sched_rr_get_interval(struct thread *td,
     struct sched_rr_get_interval_args *uap)
 {
 	struct timespec timespec;
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index f505676933aa..31d3d115a8f4 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -76,7 +76,7 @@ dtrace_vtime_switch_func_t	dtrace_vtime_switch_func;
 #include <machine/cpu.h>
 #include <machine/smp.h>
 
-#if defined(__sparc64__)
+#if defined(__powerpc__) && defined(E500)
 #error "This architecture is not currently compatible with ULE"
 #endif
 
@@ -839,6 +839,7 @@ sched_balance_pair(struct tdq *high, struct tdq *low)
 	int low_load;
 	int moved;
 	int move;
+	int cpu;
 	int diff;
 	int i;
 
@@ -860,10 +861,14 @@ sched_balance_pair(struct tdq *high, struct tdq *low)
 		for (i = 0; i < move; i++)
 			moved += tdq_move(high, low);
 		/*
-		 * IPI the target cpu to force it to reschedule with the new
-		 * workload.
+		 * In case the target isn't the current cpu IPI it to force a
+		 * reschedule with the new workload.
 		 */
-		ipi_cpu(TDQ_ID(low), IPI_PREEMPT);
+		cpu = TDQ_ID(low);
+		sched_pin();
+		if (cpu != PCPU_GET(cpuid))
+			ipi_cpu(cpu, IPI_PREEMPT);
+		sched_unpin();
 	}
 	tdq_unlock_pair(high, low);
 	return (moved);
diff --git a/sys/kern/subr_acl_nfs4.c b/sys/kern/subr_acl_nfs4.c
index ed1f2554a104..0fd21952de95 100644
--- a/sys/kern/subr_acl_nfs4.c
+++ b/sys/kern/subr_acl_nfs4.c
@@ -114,7 +114,6 @@ _acl_denies(const struct acl *aclp, int access_mask, struct ucred *cred,
 	if (denied_explicitly != NULL)
 		*denied_explicitly = 0;
 
-	KASSERT(aclp->acl_cnt > 0, ("aclp->acl_cnt > 0"));
 	KASSERT(aclp->acl_cnt <= ACL_MAX_ENTRIES,
 	    ("aclp->acl_cnt <= ACL_MAX_ENTRIES"));
 
@@ -723,7 +722,6 @@ acl_nfs4_sync_mode_from_acl(mode_t *_mode, const struct acl *aclp)
 	mode_t old_mode = *_mode, mode = 0, seen = 0;
 	const struct acl_entry *entry;
 
-	KASSERT(aclp->acl_cnt > 0, ("aclp->acl_cnt > 0"));
 	KASSERT(aclp->acl_cnt <= ACL_MAX_ENTRIES,
 	    ("aclp->acl_cnt <= ACL_MAX_ENTRIES"));
 
@@ -854,7 +852,6 @@ acl_nfs4_compute_inherited_acl_draft(const struct acl *parent_aclp,
 	struct acl_entry *entry, *copy;
 
 	KASSERT(child_aclp->acl_cnt == 0, ("child_aclp->acl_cnt == 0"));
-	KASSERT(parent_aclp->acl_cnt > 0, ("parent_aclp->acl_cnt > 0"));
 	KASSERT(parent_aclp->acl_cnt <= ACL_MAX_ENTRIES,
 	    ("parent_aclp->acl_cnt <= ACL_MAX_ENTRIES"));
 
@@ -1017,7 +1014,6 @@ acl_nfs4_inherit_entries(const struct acl *parent_aclp,
 	const struct acl_entry *parent_entry;
 	struct acl_entry *entry;
 
-	KASSERT(parent_aclp->acl_cnt > 0, ("parent_aclp->acl_cnt > 0"));
 	KASSERT(parent_aclp->acl_cnt <= ACL_MAX_ENTRIES,
 	    ("parent_aclp->acl_cnt <= ACL_MAX_ENTRIES"));
 
diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c
index e294d70a9952..f46660f38434 100644
--- a/sys/kern/subr_bus.c
+++ b/sys/kern/subr_bus.c
@@ -574,7 +574,7 @@ devctl_queue_data_f(char *data, int flags)
 	p = devsoftc.async_proc;
 	if (p != NULL) {
 		PROC_LOCK(p);
-		psignal(p, SIGIO);
+		kern_psignal(p, SIGIO);
 		PROC_UNLOCK(p);
 	}
 	return;
@@ -2140,6 +2140,11 @@ device_get_children(device_t dev, device_t **devlistp, int *devcountp)
 	TAILQ_FOREACH(child, &dev->children, link) {
 		count++;
 	}
+	if (count == 0) {
+		*devlistp = NULL;
+		*devcountp = 0;
+		return (0);
+	}
 
 	list = malloc(count * sizeof(device_t), M_TEMP, M_NOWAIT|M_ZERO);
 	if (!list)
diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c
index 2ee7dc674dfa..1d23f2173cca 100644
--- a/sys/kern/subr_kdb.c
+++ b/sys/kern/subr_kdb.c
@@ -29,7 +29,6 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_kdb.h"
 #include "opt_stack.h"
-#include "opt_watchdog.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -42,9 +41,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/smp.h>
 #include <sys/stack.h>
 #include <sys/sysctl.h>
-#ifdef SW_WATCHDOG
-#include <sys/watchdog.h>
-#endif
 
 #include <machine/kdb.h>
 #include <machine/pcb.h>
@@ -94,25 +90,30 @@ SYSCTL_PROC(_debug_kdb, OID_AUTO, available, CTLTYPE_STRING | CTLFLAG_RD, NULL,
 SYSCTL_PROC(_debug_kdb, OID_AUTO, current, CTLTYPE_STRING | CTLFLAG_RW, NULL,
     0, kdb_sysctl_current, "A", "currently selected KDB backend");
 
-SYSCTL_PROC(_debug_kdb, OID_AUTO, enter, CTLTYPE_INT | CTLFLAG_RW, NULL, 0,
+SYSCTL_PROC(_debug_kdb, OID_AUTO, enter,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE, NULL, 0,
     kdb_sysctl_enter, "I", "set to enter the debugger");
 
-SYSCTL_PROC(_debug_kdb, OID_AUTO, panic, CTLTYPE_INT | CTLFLAG_RW, NULL, 0,
+SYSCTL_PROC(_debug_kdb, OID_AUTO, panic,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE, NULL, 0,
     kdb_sysctl_panic, "I", "set to panic the kernel");
 
-SYSCTL_PROC(_debug_kdb, OID_AUTO, trap, CTLTYPE_INT | CTLFLAG_RW, NULL, 0,
+SYSCTL_PROC(_debug_kdb, OID_AUTO, trap,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE, NULL, 0,
     kdb_sysctl_trap, "I", "set to cause a page fault via data access");
 
-SYSCTL_PROC(_debug_kdb, OID_AUTO, trap_code, CTLTYPE_INT | CTLFLAG_RW, NULL, 0,
+SYSCTL_PROC(_debug_kdb, OID_AUTO, trap_code,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE, NULL, 0,
     kdb_sysctl_trap_code, "I", "set to cause a page fault via code access");
 
-SYSCTL_INT(_debug_kdb, OID_AUTO, break_to_debugger, CTLTYPE_INT | CTLFLAG_RW |
-    CTLFLAG_TUN, &kdb_break_to_debugger, 0, "Enable break to debugger");
+SYSCTL_INT(_debug_kdb, OID_AUTO, break_to_debugger,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_TUN | CTLFLAG_SECURE,
+    &kdb_break_to_debugger, 0, "Enable break to debugger");
 TUNABLE_INT("debug.kdb.break_to_debugger", &kdb_break_to_debugger);
 
-SYSCTL_INT(_debug_kdb, OID_AUTO, alt_break_to_debugger, CTLTYPE_INT |
-    CTLFLAG_RW | CTLFLAG_TUN, &kdb_alt_break_to_debugger, 0,
-    "Enable alternative break to debugger");
+SYSCTL_INT(_debug_kdb, OID_AUTO, alt_break_to_debugger,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_TUN | CTLFLAG_SECURE,
+    &kdb_alt_break_to_debugger, 0, "Enable alternative break to debugger");
 TUNABLE_INT("debug.kdb.alt_break_to_debugger", &kdb_alt_break_to_debugger);
 
 /*
@@ -591,9 +592,6 @@ kdb_trap(int type, int code, struct trapframe *tf)
 	cpuset_t other_cpus;
 #endif
 	struct kdb_dbbe *be;
-#ifdef SW_WATCHDOG
-	u_int wdoglvt;
-#endif
 	register_t intr;
 	int handled;
 
@@ -607,10 +605,6 @@ kdb_trap(int type, int code, struct trapframe *tf)
 
 	intr = intr_disable();
 
-#ifdef SW_WATCHDOG
-	wdoglvt = wdog_kern_last_timeout();
-	wdog_kern_pat(WD_TO_NEVER);
-#endif
 #ifdef SMP
 	other_cpus = all_cpus;
 	CPU_CLR(PCPU_GET(cpuid), &other_cpus);
@@ -642,9 +636,6 @@ kdb_trap(int type, int code, struct trapframe *tf)
 #ifdef SMP
 	restart_cpus(stopped_cpus);
 #endif
-#ifdef SW_WATCHDOG
-	wdog_kern_pat(wdoglvt);
-#endif
 
 	intr_restore(intr);
 
diff --git a/sys/kern/subr_prf.c b/sys/kern/subr_prf.c
index 48f2dd9ce8af..d8e8e0582a7d 100644
--- a/sys/kern/subr_prf.c
+++ b/sys/kern/subr_prf.c
@@ -467,25 +467,19 @@ putchar(int c, void *arg)
 	struct putchar_arg *ap = (struct putchar_arg*) arg;
 	struct tty *tp = ap->tty;
 	int flags = ap->flags;
-	int putbuf_done = 0;
 
 	/* Don't use the tty code after a panic or while in ddb. */
 	if (kdb_active) {
 		if (c != '\0')
 			cnputc(c);
-	} else {
-		if ((panicstr == NULL) && (flags & TOTTY) && (tp != NULL))
-			tty_putchar(tp, c);
+		return;
+	}
 
-		if (flags & TOCONS) {
-			putbuf(c, ap);
-			putbuf_done = 1;
-		}
-	}
-	if ((flags & TOLOG) && (putbuf_done == 0)) {
-		if (c != '\0')
-			putbuf(c, ap);
-	}
+	if ((flags & TOTTY) && tp != NULL && panicstr == NULL)
+		tty_putchar(tp, c);
+
+	if ((flags & (TOCONS | TOLOG)) && c != '\0')
+		putbuf(c, ap);
 }
 
 /*
diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c
index be98d44d9ed1..7ff54143d250 100644
--- a/sys/kern/subr_prof.c
+++ b/sys/kern/subr_prof.c
@@ -223,8 +223,8 @@ kmstartup(dummy)
 
 	startguprof(p);
 	for (i = 0; i < CALIB_SCALE; i++)
-		MCOUNT_OVERHEAD(profil);
-	mcount_overhead = KCOUNT(p, PC_TO_I(p, profil));
+		MCOUNT_OVERHEAD(sys_profil);
+	mcount_overhead = KCOUNT(p, PC_TO_I(p, sys_profil));
 
 	startguprof(p);
 	for (i = 0; i < CALIB_SCALE; i++)
@@ -404,7 +404,7 @@ struct profil_args {
 #endif
 /* ARGSUSED */
 int
-profil(struct thread *td, struct profil_args *uap)
+sys_profil(struct thread *td, struct profil_args *uap)
 {
 	struct uprof *upp;
 	struct proc *p;
diff --git a/sys/kern/subr_syscall.c b/sys/kern/subr_syscall.c
index cb0d9295ed2b..bba44794cefb 100644
--- a/sys/kern/subr_syscall.c
+++ b/sys/kern/subr_syscall.c
@@ -204,9 +204,17 @@ syscallret(struct thread *td, int error, struct syscall_args *sa __unused)
 	 * is not the case, this code will need to be revisited.
 	 */
 	STOPEVENT(p, S_SCX, sa->code);
-	PTRACESTOP_SC(p, td, S_PT_SCX);
 	if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) {
 		PROC_LOCK(p);
+		/*
+		 * If tracing the execed process, trap to the debugger
+		 * so that breakpoints can be set before the program
+		 * executes.  If debugger requested tracing of syscall
+		 * returns, do it now too.
+		 */
+		if (traced && ((td->td_dbgflags & TDB_EXEC) != 0 ||
+		    (p->p_stops & S_PT_SCX) != 0))
+			ptracestop(td, SIGTRAP);
 		td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK);
 		PROC_UNLOCK(p);
 	}
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index 42adf80f04fe..ba8b66f6589f 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -99,6 +99,8 @@ userret(struct thread *td, struct trapframe *frame)
 
 	CTR3(KTR_SYSC, "userret: thread %p (pid %d, %s)", td, p->p_pid,
             td->td_name);
+	KASSERT((p->p_flag & P_WEXIT) == 0,
+	    ("Exiting process returns to usermode"));
 #if 0
 #ifdef DIAGNOSTIC
 	/* Check that we called signotify() enough. */
@@ -192,12 +194,12 @@ ast(struct trapframe *framep)
 	}
 	if (flags & TDF_ALRMPEND) {
 		PROC_LOCK(p);
-		psignal(p, SIGVTALRM);
+		kern_psignal(p, SIGVTALRM);
 		PROC_UNLOCK(p);
 	}
 	if (flags & TDF_PROFPEND) {
 		PROC_LOCK(p);
-		psignal(p, SIGPROF);
+		kern_psignal(p, SIGPROF);
 		PROC_UNLOCK(p);
 	}
 #ifdef MAC
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c
index 01de5ed9affd..a4cf5a242dd0 100644
--- a/sys/kern/subr_witness.c
+++ b/sys/kern/subr_witness.c
@@ -719,6 +719,18 @@ static int witness_cold = 1;
  */
 static int witness_spin_warn = 0;
 
+/* Trim useless garbage from filenames. */
+static const char *
+fixup_filename(const char *file)
+{
+
+	if (file == NULL)
+		return (NULL);
+	while (strncmp(file, "../", 3) == 0)
+		file += 3;
+	return (file);
+}
+
 /*
  * The WITNESS-enabled diagnostic code.  Note that the witness code does
  * assume that the early boot is single-threaded at least until after this
@@ -924,7 +936,7 @@ witness_ddb_display_descendants(int(*prnt)(const char *fmt, ...),
  	}
  	w->w_displayed = 1;
 	if (w->w_file != NULL && w->w_line != 0)
-		prnt(" -- last acquired @ %s:%d\n", w->w_file,
+		prnt(" -- last acquired @ %s:%d\n", fixup_filename(w->w_file),
 		    w->w_line);
 	else
 		prnt(" -- never acquired\n");
@@ -990,18 +1002,6 @@ witness_ddb_display(int(*prnt)(const char *fmt, ...))
 }
 #endif /* DDB */
 
-/* Trim useless garbage from filenames. */
-static const char *
-fixup_filename(const char *file)
-{
-
-	if (file == NULL)
-		return (NULL);
-	while (strncmp(file, "../", 3) == 0)
-		file += 3;
-	return (file);
-}
-
 int
 witness_defineorder(struct lock_object *lock1, struct lock_object *lock2)
 {
@@ -1053,7 +1053,6 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file,
 	w = lock->lo_witness;
 	class = LOCK_CLASS(lock);
 	td = curthread;
-	file = fixup_filename(file);
 
 	if (class->lc_flags & LC_SLEEPLOCK) {
 
@@ -1064,7 +1063,8 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file,
 		 */
 		if (td->td_critnest != 0 && !kdb_active)
 			panic("blockable sleep lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 
 		/*
 		 * If this is the first lock acquired then just return as
@@ -1102,17 +1102,19 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file,
 		if ((lock1->li_flags & LI_EXCLUSIVE) != 0 &&
 		    (flags & LOP_EXCLUSIVE) == 0) {
 			printf("shared lock of (%s) %s @ %s:%d\n",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 			printf("while exclusively locked from %s:%d\n",
-			    lock1->li_file, lock1->li_line);
+			    fixup_filename(lock1->li_file), lock1->li_line);
 			panic("share->excl");
 		}
 		if ((lock1->li_flags & LI_EXCLUSIVE) == 0 &&
 		    (flags & LOP_EXCLUSIVE) != 0) {
 			printf("exclusive lock of (%s) %s @ %s:%d\n",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 			printf("while share locked from %s:%d\n",
-			    lock1->li_file, lock1->li_line);
+			    fixup_filename(lock1->li_file), lock1->li_line);
 			panic("excl->share");
 		}
 		return;
@@ -1165,11 +1167,12 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file,
 			    "acquiring duplicate lock of same type: \"%s\"\n", 
 			    w->w_name);
 			printf(" 1st %s @ %s:%d\n", plock->li_lock->lo_name,
-			       plock->li_file, plock->li_line);
-			printf(" 2nd %s @ %s:%d\n", lock->lo_name, file, line);
+			    fixup_filename(plock->li_file), plock->li_line);
+			printf(" 2nd %s @ %s:%d\n", lock->lo_name,
+			    fixup_filename(file), line);
 			witness_debugger(1);
-		    } else
-			    mtx_unlock_spin(&w_mtx);
+		} else
+			mtx_unlock_spin(&w_mtx);
 		return;
 	}
 	mtx_assert(&w_mtx, MA_OWNED);
@@ -1307,19 +1310,24 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file,
 			if (i < 0) {
 				printf(" 1st %p %s (%s) @ %s:%d\n",
 				    lock1->li_lock, lock1->li_lock->lo_name,
-				    w1->w_name, lock1->li_file, lock1->li_line);
+				    w1->w_name, fixup_filename(lock1->li_file),
+				    lock1->li_line);
 				printf(" 2nd %p %s (%s) @ %s:%d\n", lock,
-				    lock->lo_name, w->w_name, file, line);
+				    lock->lo_name, w->w_name,
+				    fixup_filename(file), line);
 			} else {
 				printf(" 1st %p %s (%s) @ %s:%d\n",
 				    lock2->li_lock, lock2->li_lock->lo_name,
 				    lock2->li_lock->lo_witness->w_name,
-				    lock2->li_file, lock2->li_line);
+				    fixup_filename(lock2->li_file),
+				    lock2->li_line);
 				printf(" 2nd %p %s (%s) @ %s:%d\n",
 				    lock1->li_lock, lock1->li_lock->lo_name,
-				    w1->w_name, lock1->li_file, lock1->li_line);
+				    w1->w_name, fixup_filename(lock1->li_file),
+				    lock1->li_line);
 				printf(" 3rd %p %s (%s) @ %s:%d\n", lock,
-				    lock->lo_name, w->w_name, file, line);
+				    lock->lo_name, w->w_name,
+				    fixup_filename(file), line);
 			}
 			witness_debugger(1);
 			return;
@@ -1356,7 +1364,6 @@ witness_lock(struct lock_object *lock, int flags, const char *file, int line)
 		return;
 	w = lock->lo_witness;
 	td = curthread;
-	file = fixup_filename(file);
 
 	/* Determine lock list for this lock. */
 	if (LOCK_CLASS(lock)->lc_flags & LC_SLEEPLOCK)
@@ -1413,27 +1420,31 @@ witness_upgrade(struct lock_object *lock, int flags, const char *file, int line)
 	if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL)
 		return;
 	class = LOCK_CLASS(lock);
-	file = fixup_filename(file);
 	if (witness_watch) {
 		if ((lock->lo_flags & LO_UPGRADABLE) == 0)
 			panic("upgrade of non-upgradable lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((class->lc_flags & LC_SLEEPLOCK) == 0)
 			panic("upgrade of non-sleep lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 	}
 	instance = find_instance(curthread->td_sleeplocks, lock);
 	if (instance == NULL)
 		panic("upgrade of unlocked lock (%s) %s @ %s:%d",
-		    class->lc_name, lock->lo_name, file, line);
+		    class->lc_name, lock->lo_name,
+		    fixup_filename(file), line);
 	if (witness_watch) {
 		if ((instance->li_flags & LI_EXCLUSIVE) != 0)
 			panic("upgrade of exclusive lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((instance->li_flags & LI_RECURSEMASK) != 0)
 			panic("upgrade of recursed lock (%s) %s r=%d @ %s:%d",
 			    class->lc_name, lock->lo_name,
-			    instance->li_flags & LI_RECURSEMASK, file, line);
+			    instance->li_flags & LI_RECURSEMASK,
+			    fixup_filename(file), line);
 	}
 	instance->li_flags |= LI_EXCLUSIVE;
 }
@@ -1449,27 +1460,31 @@ witness_downgrade(struct lock_object *lock, int flags, const char *file,
 	if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL)
 		return;
 	class = LOCK_CLASS(lock);
-	file = fixup_filename(file);
 	if (witness_watch) {
 		if ((lock->lo_flags & LO_UPGRADABLE) == 0)
 		panic("downgrade of non-upgradable lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((class->lc_flags & LC_SLEEPLOCK) == 0)
 			panic("downgrade of non-sleep lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 	}
 	instance = find_instance(curthread->td_sleeplocks, lock);
 	if (instance == NULL)
 		panic("downgrade of unlocked lock (%s) %s @ %s:%d",
-		    class->lc_name, lock->lo_name, file, line);
+		    class->lc_name, lock->lo_name,
+		    fixup_filename(file), line);
 	if (witness_watch) {
 		if ((instance->li_flags & LI_EXCLUSIVE) == 0)
 			panic("downgrade of shared lock (%s) %s @ %s:%d",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((instance->li_flags & LI_RECURSEMASK) != 0)
 			panic("downgrade of recursed lock (%s) %s r=%d @ %s:%d",
 			    class->lc_name, lock->lo_name,
-			    instance->li_flags & LI_RECURSEMASK, file, line);
+			    instance->li_flags & LI_RECURSEMASK,
+			    fixup_filename(file), line);
 	}
 	instance->li_flags &= ~LI_EXCLUSIVE;
 }
@@ -1488,7 +1503,6 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
 		return;
 	td = curthread;
 	class = LOCK_CLASS(lock);
-	file = fixup_filename(file);
 
 	/* Find lock instance associated with this lock. */
 	if (class->lc_flags & LC_SLEEPLOCK)
@@ -1511,7 +1525,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
 	 */
 	if (witness_watch > 0)
 		panic("lock (%s) %s not locked @ %s:%d", class->lc_name,
-		    lock->lo_name, file, line);
+		    lock->lo_name, fixup_filename(file), line);
 	else
 		return;
 found:
@@ -1520,16 +1534,17 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
 	if ((instance->li_flags & LI_EXCLUSIVE) != 0 && witness_watch > 0 &&
 	    (flags & LOP_EXCLUSIVE) == 0) {
 		printf("shared unlock of (%s) %s @ %s:%d\n", class->lc_name,
-		    lock->lo_name, file, line);
+		    lock->lo_name, fixup_filename(file), line);
 		printf("while exclusively locked from %s:%d\n",
-		    instance->li_file, instance->li_line);
+		    fixup_filename(instance->li_file), instance->li_line);
 		panic("excl->ushare");
 	}
 	if ((instance->li_flags & LI_EXCLUSIVE) == 0 && witness_watch > 0 &&
 	    (flags & LOP_EXCLUSIVE) != 0) {
 		printf("exclusive unlock of (%s) %s @ %s:%d\n", class->lc_name,
-		    lock->lo_name, file, line);
-		printf("while share locked from %s:%d\n", instance->li_file,
+		    lock->lo_name, fixup_filename(file), line);
+		printf("while share locked from %s:%d\n",
+		    fixup_filename(instance->li_file),
 		    instance->li_line);
 		panic("share->uexcl");
 	}
@@ -1544,7 +1559,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
 	/* The lock is now being dropped, check for NORELEASE flag */
 	if ((instance->li_flags & LI_NORELEASE) != 0 && witness_watch > 0) {
 		printf("forbidden unlock of (%s) %s @ %s:%d\n", class->lc_name,
-		    lock->lo_name, file, line);
+		    lock->lo_name, fixup_filename(file), line);
 		panic("lock marked norelease");
 	}
 
@@ -2074,8 +2089,8 @@ witness_list_lock(struct lock_instance *instance,
 	if (lock->lo_witness->w_name != lock->lo_name)
 		prnt(" (%s)", lock->lo_witness->w_name);
 	prnt(" r = %d (%p) locked @ %s:%d\n",
-	    instance->li_flags & LI_RECURSEMASK, lock, instance->li_file,
-	    instance->li_line);
+	    instance->li_flags & LI_RECURSEMASK, lock,
+	    fixup_filename(instance->li_file), instance->li_line);
 }
 
 #ifdef DDB
@@ -2211,12 +2226,12 @@ witness_assert(struct lock_object *lock, int flags, const char *file, int line)
 		panic("Lock (%s) %s is not sleep or spin!",
 		    class->lc_name, lock->lo_name);
 	}
-	file = fixup_filename(file);
 	switch (flags) {
 	case LA_UNLOCKED:
 		if (instance != NULL)
 			panic("Lock (%s) %s locked @ %s:%d.",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		break;
 	case LA_LOCKED:
 	case LA_LOCKED | LA_RECURSED:
@@ -2229,28 +2244,34 @@ witness_assert(struct lock_object *lock, int flags, const char *file, int line)
 	case LA_XLOCKED | LA_NOTRECURSED:
 		if (instance == NULL) {
 			panic("Lock (%s) %s not locked @ %s:%d.",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 			break;
 		}
 		if ((flags & LA_XLOCKED) != 0 &&
 		    (instance->li_flags & LI_EXCLUSIVE) == 0)
 			panic("Lock (%s) %s not exclusively locked @ %s:%d.",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((flags & LA_SLOCKED) != 0 &&
 		    (instance->li_flags & LI_EXCLUSIVE) != 0)
 			panic("Lock (%s) %s exclusively locked @ %s:%d.",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((flags & LA_RECURSED) != 0 &&
 		    (instance->li_flags & LI_RECURSEMASK) == 0)
 			panic("Lock (%s) %s not recursed @ %s:%d.",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		if ((flags & LA_NOTRECURSED) != 0 &&
 		    (instance->li_flags & LI_RECURSEMASK) != 0)
 			panic("Lock (%s) %s recursed @ %s:%d.",
-			    class->lc_name, lock->lo_name, file, line);
+			    class->lc_name, lock->lo_name,
+			    fixup_filename(file), line);
 		break;
 	default:
-		panic("Invalid lock assertion at %s:%d.", file, line);
+		panic("Invalid lock assertion at %s:%d.",
+		    fixup_filename(file), line);
 
 	}
 #endif	/* INVARIANT_SUPPORT */
diff --git a/sys/kern/sys_capability.c b/sys/kern/sys_capability.c
index 318ffe4e04d2..f1fb1b1ba331 100644
--- a/sys/kern/sys_capability.c
+++ b/sys/kern/sys_capability.c
@@ -52,6 +52,7 @@
  */
 
 #include "opt_capsicum.h"
+#include "opt_ktrace.h"
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
@@ -68,6 +69,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/ucred.h>
+#include <sys/uio.h>
+#include <sys/ktrace.h>
 
 #include <security/audit/audit.h>
 
@@ -82,7 +85,7 @@ FEATURE(security_capability_mode, "Capsicum Capability Mode");
  * System call to enter capability mode for the process.
  */
 int
-cap_enter(struct thread *td, struct cap_enter_args *uap)
+sys_cap_enter(struct thread *td, struct cap_enter_args *uap)
 {
 	struct ucred *newcred, *oldcred;
 	struct proc *p;
@@ -106,7 +109,7 @@ cap_enter(struct thread *td, struct cap_enter_args *uap)
  * System call to query whether the process is in capability mode.
  */
 int
-cap_getmode(struct thread *td, struct cap_getmode_args *uap)
+sys_cap_getmode(struct thread *td, struct cap_getmode_args *uap)
 {
 	u_int i;
 
@@ -117,14 +120,14 @@ cap_getmode(struct thread *td, struct cap_getmode_args *uap)
 #else /* !CAPABILITY_MODE */
 
 int
-cap_enter(struct thread *td, struct cap_enter_args *uap)
+sys_cap_enter(struct thread *td, struct cap_enter_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-cap_getmode(struct thread *td, struct cap_getmode_args *uap)
+sys_cap_getmode(struct thread *td, struct cap_getmode_args *uap)
 {
 
 	return (ENOSYS);
@@ -212,8 +215,13 @@ static int
 cap_check(struct capability *c, cap_rights_t rights)
 {
 
-	if ((c->cap_rights | rights) != c->cap_rights)
+	if ((c->cap_rights | rights) != c->cap_rights) {
+#ifdef KTRACE
+		if (KTRPOINT(curthread, KTR_CAPFAIL))
+			ktrcapfail(CAPFAIL_NOTCAPABLE, rights, c->cap_rights);
+#endif
 		return (ENOTCAPABLE);
+	}
 	return (0);
 }
 
@@ -239,7 +247,7 @@ cap_rights(struct file *fp_cap)
  * file object or an an existing capability.
  */
 int
-cap_new(struct thread *td, struct cap_new_args *uap)
+sys_cap_new(struct thread *td, struct cap_new_args *uap)
 {
 	int error, capfd;
 	int fd = uap->fd;
@@ -269,7 +277,7 @@ cap_new(struct thread *td, struct cap_new_args *uap)
  * System call to query the rights mask associated with a capability.
  */
 int
-cap_getrights(struct thread *td, struct cap_getrights_args *uap)
+sys_cap_getrights(struct thread *td, struct cap_getrights_args *uap)
 {
 	struct capability *cp;
 	struct file *fp;
@@ -306,8 +314,14 @@ kern_capwrap(struct thread *td, struct file *fp, cap_rights_t rights,
 	 */
 	if (fp->f_type == DTYPE_CAPABILITY) {
 		cp_old = fp->f_data;
-		if ((cp_old->cap_rights | rights) != cp_old->cap_rights)
+		if ((cp_old->cap_rights | rights) != cp_old->cap_rights) {
+#ifdef KTRACE
+			if (KTRPOINT(curthread, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_INCREASE,
+				    rights, cp_old->cap_rights);
+#endif
 			return (ENOTCAPABLE);
+		}
 	}
 
 	/*
@@ -513,14 +527,14 @@ capability_chown(struct file *fp, uid_t uid, gid_t gid,
  * into the kernel.
  */
 int
-cap_new(struct thread *td, struct cap_new_args *uap)
+sys_cap_new(struct thread *td, struct cap_new_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-cap_getrights(struct thread *td, struct cap_getrights_args *uap)
+sys_cap_getrights(struct thread *td, struct cap_getrights_args *uap)
 {
 
 	return (ENOSYS);
diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c
index 7b45efa2ff51..8d4ba09e99c5 100644
--- a/sys/kern/sys_generic.c
+++ b/sys/kern/sys_generic.c
@@ -137,7 +137,7 @@ struct read_args {
 };
 #endif
 int
-read(td, uap)
+sys_read(td, uap)
 	struct thread *td;
 	struct read_args *uap;
 {
@@ -170,7 +170,7 @@ struct pread_args {
 };
 #endif
 int
-pread(td, uap)
+sys_pread(td, uap)
 	struct thread *td;
 	struct pread_args *uap;
 {
@@ -201,7 +201,7 @@ freebsd6_pread(td, uap)
 	oargs.buf = uap->buf;
 	oargs.nbyte = uap->nbyte;
 	oargs.offset = uap->offset;
-	return (pread(td, &oargs));
+	return (sys_pread(td, &oargs));
 }
 
 /*
@@ -215,7 +215,7 @@ struct readv_args {
 };
 #endif
 int
-readv(struct thread *td, struct readv_args *uap)
+sys_readv(struct thread *td, struct readv_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -254,7 +254,7 @@ struct preadv_args {
 };
 #endif
 int
-preadv(struct thread *td, struct preadv_args *uap)
+sys_preadv(struct thread *td, struct preadv_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -346,7 +346,7 @@ struct write_args {
 };
 #endif
 int
-write(td, uap)
+sys_write(td, uap)
 	struct thread *td;
 	struct write_args *uap;
 {
@@ -379,7 +379,7 @@ struct pwrite_args {
 };
 #endif
 int
-pwrite(td, uap)
+sys_pwrite(td, uap)
 	struct thread *td;
 	struct pwrite_args *uap;
 {
@@ -410,7 +410,7 @@ freebsd6_pwrite(td, uap)
 	oargs.buf = uap->buf;
 	oargs.nbyte = uap->nbyte;
 	oargs.offset = uap->offset;
-	return (pwrite(td, &oargs));
+	return (sys_pwrite(td, &oargs));
 }
 
 /*
@@ -424,7 +424,7 @@ struct writev_args {
 };
 #endif
 int
-writev(struct thread *td, struct writev_args *uap)
+sys_writev(struct thread *td, struct writev_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -463,7 +463,7 @@ struct pwritev_args {
 };
 #endif
 int
-pwritev(struct thread *td, struct pwritev_args *uap)
+sys_pwritev(struct thread *td, struct pwritev_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -589,7 +589,7 @@ struct ftruncate_args {
 };
 #endif
 int
-ftruncate(td, uap)
+sys_ftruncate(td, uap)
 	struct thread *td;
 	struct ftruncate_args *uap;
 {
@@ -623,7 +623,7 @@ struct ioctl_args {
 #endif
 /* ARGSUSED */
 int
-ioctl(struct thread *td, struct ioctl_args *uap)
+sys_ioctl(struct thread *td, struct ioctl_args *uap)
 {
 	u_long com;
 	int arg, error;
@@ -755,7 +755,7 @@ poll_no_poll(int events)
 }
 
 int
-pselect(struct thread *td, struct pselect_args *uap)
+sys_pselect(struct thread *td, struct pselect_args *uap)
 {
 	struct timespec ts;
 	struct timeval tv, *tvp;
@@ -814,7 +814,7 @@ struct select_args {
 };
 #endif
 int
-select(struct thread *td, struct select_args *uap)
+sys_select(struct thread *td, struct select_args *uap)
 {
 	struct timeval tv, *tvp;
 	int error;
@@ -1178,7 +1178,7 @@ struct poll_args {
 };
 #endif
 int
-poll(td, uap)
+sys_poll(td, uap)
 	struct thread *td;
 	struct poll_args *uap;
 {
@@ -1396,11 +1396,11 @@ struct openbsd_poll_args {
 };
 #endif
 int
-openbsd_poll(td, uap)
+sys_openbsd_poll(td, uap)
 	register struct thread *td;
 	register struct openbsd_poll_args *uap;
 {
-	return (poll(td, (struct poll_args *)uap));
+	return (sys_poll(td, (struct poll_args *)uap));
 }
 
 /*
diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index c44a2c964e49..13e056fa4a3b 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -93,6 +93,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/conf.h>
 #include <sys/fcntl.h>
 #include <sys/file.h>
 #include <sys/filedesc.h>
@@ -224,6 +225,8 @@ static int	pipe_zone_init(void *mem, int size, int flags);
 static void	pipe_zone_fini(void *mem, int size);
 
 static uma_zone_t pipe_zone;
+static struct unrhdr *pipeino_unr;
+static dev_t pipedev_ino;
 
 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
 
@@ -235,6 +238,10 @@ pipeinit(void *dummy __unused)
 	    pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini,
 	    UMA_ALIGN_PTR, 0);
 	KASSERT(pipe_zone != NULL, ("pipe_zone not initialized"));
+	pipeino_unr = new_unrhdr(1, INT32_MAX, NULL);
+	KASSERT(pipeino_unr != NULL, ("pipe fake inodes not initialized"));
+	pipedev_ino = devfs_alloc_cdp_inode();
+	KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized"));
 }
 
 static int
@@ -385,7 +392,7 @@ kern_pipe(struct thread *td, int fildes[2])
 
 /* ARGSUSED */
 int
-pipe(struct thread *td, struct pipe_args *uap)
+sys_pipe(struct thread *td, struct pipe_args *uap)
 {
 	int error;
 	int fildes[2];
@@ -562,6 +569,12 @@ pipe_create(pipe, backing)
 		/* If we're not backing this pipe, no need to do anything. */
 		error = 0;
 	}
+	if (error == 0) {
+		pipe->pipe_ino = alloc_unr(pipeino_unr);
+		if (pipe->pipe_ino == -1)
+			/* pipeclose will clear allocated kva */
+			error = ENOMEM;
+	}
 	return (error);
 }
 
@@ -1408,9 +1421,10 @@ pipe_stat(fp, ub, active_cred, td)
 	ub->st_ctim = pipe->pipe_ctime;
 	ub->st_uid = fp->f_cred->cr_uid;
 	ub->st_gid = fp->f_cred->cr_gid;
+	ub->st_dev = pipedev_ino;
+	ub->st_ino = pipe->pipe_ino;
 	/*
-	 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen.
-	 * XXX (st_dev, st_ino) should be unique.
+	 * Left as 0: st_nlink, st_rdev, st_flags, st_gen.
 	 */
 	return (0);
 }
@@ -1463,6 +1477,7 @@ pipeclose(cpipe)
 {
 	struct pipepair *pp;
 	struct pipe *ppipe;
+	ino_t ino;
 
 	KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL"));
 
@@ -1520,6 +1535,12 @@ pipeclose(cpipe)
 	seldrain(&cpipe->pipe_sel);
 	knlist_destroy(&cpipe->pipe_sel.si_note);
 
+	/*
+	 * Postpone the destroy of the fake inode number allocated for
+	 * our end, until pipe mtx is unlocked.
+	 */
+	ino = cpipe->pipe_ino;
+
 	/*
 	 * If both endpoints are now closed, release the memory for the
 	 * pipe pair.  If not, unlock.
@@ -1532,6 +1553,9 @@ pipeclose(cpipe)
 		uma_zfree(pipe_zone, cpipe->pipe_pair);
 	} else
 		PIPE_UNLOCK(cpipe);
+
+	if (ino > 0)
+		free_unr(pipeino_unr, cpipe->pipe_ino);
 }
 
 /*ARGSUSED*/
diff --git a/sys/kern/sys_procdesc.c b/sys/kern/sys_procdesc.c
index 9993732527cd..08d345f2ef3b 100644
--- a/sys/kern/sys_procdesc.c
+++ b/sys/kern/sys_procdesc.c
@@ -206,7 +206,7 @@ kern_pdgetpid(struct thread *td, int fd, cap_rights_t rights, pid_t *pidp)
  * System call to return the pid of a process given its process descriptor.
  */
 int
-pdgetpid(struct thread *td, struct pdgetpid_args *uap)
+sys_pdgetpid(struct thread *td, struct pdgetpid_args *uap)
 {
 	pid_t pid;
 	int error;
@@ -387,7 +387,7 @@ procdesc_close(struct file *fp, struct thread *td)
 		p->p_sigparent = SIGCHLD;
 		proc_reparent(p, initproc);
 		if ((pd->pd_flags & PD_DAEMON) == 0)
-			psignal(p, SIGKILL);
+			kern_psignal(p, SIGKILL);
 		PROC_UNLOCK(p);
 		sx_xunlock(&proctree_lock);
 	}
@@ -515,7 +515,7 @@ procdesc_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
 #else /* !PROCDESC */
 
 int
-pdgetpid(struct thread *td, struct pdgetpid_args *uap)
+sys_pdgetpid(struct thread *td, struct pdgetpid_args *uap)
 {
 
 	return (ENOSYS);
diff --git a/sys/kern/sys_process.c b/sys/kern/sys_process.c
index ee36b3512f5a..451038036da0 100644
--- a/sys/kern/sys_process.c
+++ b/sys/kern/sys_process.c
@@ -515,7 +515,7 @@ struct ptrace_args {
 #define	COPYOUT(k, u, s)	copyout(k, u, s)
 #endif
 int
-ptrace(struct thread *td, struct ptrace_args *uap)
+sys_ptrace(struct thread *td, struct ptrace_args *uap)
 {
 	/*
 	 * XXX this obfuscation is to reduce stack usage, but the register
@@ -972,7 +972,7 @@ kern_ptrace(struct thread *td, int req, pid_t pid, void *addr, int data)
 			PROC_SUNLOCK(p);
 		} else {
 			if (data)
-				psignal(p, data);
+				kern_psignal(p, data);
 		}
 		break;
 
diff --git a/sys/kern/sysv_msg.c b/sys/kern/sysv_msg.c
index ffd85808f7df..d58cb7e791e8 100644
--- a/sys/kern/sysv_msg.c
+++ b/sys/kern/sysv_msg.c
@@ -81,6 +81,7 @@ static int msginit(void);
 static int msgunload(void);
 static int sysvmsg_modload(struct module *, int, void *);
 
+
 #ifdef MSG_DEBUG
 #define DPRINTF(a)	printf a
 #else
@@ -163,7 +164,7 @@ static struct syscall_helper_data msg_syscalls[] = {
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
 	SYSCALL_INIT_HELPER(msgsys),
-	SYSCALL_INIT_HELPER(freebsd7_msgctl),
+	SYSCALL_INIT_HELPER_COMPAT(freebsd7_msgctl),
 #endif
 	SYSCALL_INIT_LAST
 };
@@ -180,7 +181,7 @@ static struct syscall_helper_data msg32_syscalls[] = {
 	SYSCALL32_INIT_HELPER(freebsd32_msgctl),
 	SYSCALL32_INIT_HELPER(freebsd32_msgsnd),
 	SYSCALL32_INIT_HELPER(freebsd32_msgrcv),
-	SYSCALL32_INIT_HELPER(msgget),
+	SYSCALL32_INIT_HELPER_COMPAT(msgget),
 	SYSCALL32_INIT_HELPER(freebsd32_msgsys),
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
@@ -379,7 +380,7 @@ struct msgctl_args {
 };
 #endif
 int
-msgctl(td, uap)
+sys_msgctl(td, uap)
 	struct thread *td;
 	register struct msgctl_args *uap;
 {
@@ -555,8 +556,9 @@ struct msgget_args {
 	int	msgflg;
 };
 #endif
+
 int
-msgget(td, uap)
+sys_msgget(td, uap)
 	struct thread *td;
 	register struct msgget_args *uap;
 {
@@ -1011,7 +1013,7 @@ kern_msgsnd(td, msqid, msgp, msgsz, msgflg, mtype)
 }
 
 int
-msgsnd(td, uap)
+sys_msgsnd(td, uap)
 	struct thread *td;
 	register struct msgsnd_args *uap;
 {
@@ -1296,7 +1298,7 @@ kern_msgrcv(td, msqid, msgp, msgsz, msgtyp, msgflg, mtype)
 }
 
 int
-msgrcv(td, uap)
+sys_msgrcv(td, uap)
 	struct thread *td;
 	register struct msgrcv_args *uap;
 {
@@ -1356,7 +1358,7 @@ freebsd32_msgsys(struct thread *td, struct freebsd32_msgsys_args *uap)
 		return (freebsd32_msgrcv(td,
 		    (struct freebsd32_msgrcv_args *)&uap->a2));
 	default:
-		return (msgsys(td, (struct msgsys_args *)uap));
+		return (sys_msgsys(td, (struct msgsys_args *)uap));
 	}
 #else
 	return (nosys(td, NULL));
@@ -1494,15 +1496,15 @@ freebsd32_msgrcv(struct thread *td, struct freebsd32_msgrcv_args *uap)
 
 /* XXX casting to (sy_call_t *) is bogus, as usual. */
 static sy_call_t *msgcalls[] = {
-	(sy_call_t *)freebsd7_msgctl, (sy_call_t *)msgget,
-	(sy_call_t *)msgsnd, (sy_call_t *)msgrcv
+	(sy_call_t *)freebsd7_msgctl, (sy_call_t *)sys_msgget,
+	(sy_call_t *)sys_msgsnd, (sy_call_t *)sys_msgrcv
 };
 
 /*
  * Entry point for all MSG calls.
  */
 int
-msgsys(td, uap)
+sys_msgsys(td, uap)
 	struct thread *td;
 	/* XXX actually varargs. */
 	struct msgsys_args /* {
diff --git a/sys/kern/sysv_sem.c b/sys/kern/sysv_sem.c
index 4a4c4790ddec..f9ff2170dc32 100644
--- a/sys/kern/sysv_sem.c
+++ b/sys/kern/sysv_sem.c
@@ -218,7 +218,7 @@ static struct syscall_helper_data sem_syscalls[] = {
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
 	SYSCALL_INIT_HELPER(semsys),
-	SYSCALL_INIT_HELPER(freebsd7___semctl),
+	SYSCALL_INIT_HELPER_COMPAT(freebsd7___semctl),
 #endif
 	SYSCALL_INIT_LAST
 };
@@ -233,8 +233,8 @@ static struct syscall_helper_data sem_syscalls[] = {
 
 static struct syscall_helper_data sem32_syscalls[] = {
 	SYSCALL32_INIT_HELPER(freebsd32_semctl),
-	SYSCALL32_INIT_HELPER(semget),
-	SYSCALL32_INIT_HELPER(semop),
+	SYSCALL32_INIT_HELPER_COMPAT(semget),
+	SYSCALL32_INIT_HELPER_COMPAT(semop),
 	SYSCALL32_INIT_HELPER(freebsd32_semsys),
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
@@ -518,7 +518,7 @@ struct __semctl_args {
 };
 #endif
 int
-__semctl(struct thread *td, struct __semctl_args *uap)
+sys___semctl(struct thread *td, struct __semctl_args *uap)
 {
 	struct semid_ds dsbuf;
 	union semun arg, semun;
@@ -856,7 +856,7 @@ struct semget_args {
 };
 #endif
 int
-semget(struct thread *td, struct semget_args *uap)
+sys_semget(struct thread *td, struct semget_args *uap)
 {
 	int semid, error = 0;
 	int key = uap->key;
@@ -980,7 +980,7 @@ struct semop_args {
 };
 #endif
 int
-semop(struct thread *td, struct semop_args *uap)
+sys_semop(struct thread *td, struct semop_args *uap)
 {
 #define SMALL_SOPS	8
 	struct sembuf small_sops[SMALL_SOPS];
@@ -1382,15 +1382,15 @@ sysctl_sema(SYSCTL_HANDLER_ARGS)
 
 /* XXX casting to (sy_call_t *) is bogus, as usual. */
 static sy_call_t *semcalls[] = {
-	(sy_call_t *)freebsd7___semctl, (sy_call_t *)semget,
-	(sy_call_t *)semop
+	(sy_call_t *)freebsd7___semctl, (sy_call_t *)sys_semget,
+	(sy_call_t *)sys_semop
 };
 
 /*
  * Entry point for all SEM calls.
  */
 int
-semsys(td, uap)
+sys_semsys(td, uap)
 	struct thread *td;
 	/* XXX actually varargs. */
 	struct semsys_args /* {
@@ -1510,7 +1510,7 @@ freebsd32_semsys(struct thread *td, struct freebsd32_semsys_args *uap)
 		return (freebsd7_freebsd32_semctl(td,
 		    (struct freebsd7_freebsd32_semctl_args *)&uap->a2));
 	default:
-		return (semsys(td, (struct semsys_args *)uap));
+		return (sys_semsys(td, (struct semsys_args *)uap));
 	}
 #else
 	return (nosys(td, NULL));
diff --git a/sys/kern/sysv_shm.c b/sys/kern/sysv_shm.c
index 1741a21d9160..2bad48494c3e 100644
--- a/sys/kern/sysv_shm.c
+++ b/sys/kern/sysv_shm.c
@@ -284,7 +284,7 @@ struct shmdt_args {
 };
 #endif
 int
-shmdt(td, uap)
+sys_shmdt(td, uap)
 	struct thread *td;
 	struct shmdt_args *uap;
 {
@@ -434,7 +434,7 @@ kern_shmat(td, shmid, shmaddr, shmflg)
 }
 
 int
-shmat(td, uap)
+sys_shmat(td, uap)
 	struct thread *td;
 	struct shmat_args *uap;
 {
@@ -559,7 +559,7 @@ struct shmctl_args {
 };
 #endif
 int
-shmctl(td, uap)
+sys_shmctl(td, uap)
 	struct thread *td;
 	struct shmctl_args *uap;
 {
@@ -750,7 +750,7 @@ struct shmget_args {
 };
 #endif
 int
-shmget(td, uap)
+sys_shmget(td, uap)
 	struct thread *td;
 	struct shmget_args *uap;
 {
@@ -851,7 +851,7 @@ static struct syscall_helper_data shm_syscalls[] = {
 	SYSCALL_INIT_HELPER(shmget),
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
-	SYSCALL_INIT_HELPER(freebsd7_shmctl),
+	SYSCALL_INIT_HELPER_COMPAT(freebsd7_shmctl),
 #endif
 #if defined(__i386__) && (defined(COMPAT_FREEBSD4) || defined(COMPAT_43))
 	SYSCALL_INIT_HELPER(shmsys),
@@ -868,9 +868,9 @@ static struct syscall_helper_data shm_syscalls[] = {
 #include <compat/freebsd32/freebsd32_util.h>
 
 static struct syscall_helper_data shm32_syscalls[] = {
-	SYSCALL32_INIT_HELPER(shmat),
-	SYSCALL32_INIT_HELPER(shmdt),
-	SYSCALL32_INIT_HELPER(shmget),
+	SYSCALL32_INIT_HELPER_COMPAT(shmat),
+	SYSCALL32_INIT_HELPER_COMPAT(shmdt),
+	SYSCALL32_INIT_HELPER_COMPAT(shmget),
 	SYSCALL32_INIT_HELPER(freebsd32_shmsys),
 	SYSCALL32_INIT_HELPER(freebsd32_shmctl),
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
@@ -1040,13 +1040,13 @@ oshmctl(struct thread *td, struct oshmctl_args *uap)
 
 /* XXX casting to (sy_call_t *) is bogus, as usual. */
 static sy_call_t *shmcalls[] = {
-	(sy_call_t *)shmat, (sy_call_t *)oshmctl,
-	(sy_call_t *)shmdt, (sy_call_t *)shmget,
+	(sy_call_t *)sys_shmat, (sy_call_t *)oshmctl,
+	(sy_call_t *)sys_shmdt, (sy_call_t *)sys_shmget,
 	(sy_call_t *)freebsd7_shmctl
 };
 
 int
-shmsys(td, uap)
+sys_shmsys(td, uap)
 	struct thread *td;
 	/* XXX actually varargs. */
 	struct shmsys_args /* {
diff --git a/sys/kern/tty.c b/sys/kern/tty.c
index b5fcba91b4a5..01ac80900a14 100644
--- a/sys/kern/tty.c
+++ b/sys/kern/tty.c
@@ -1264,7 +1264,7 @@ tty_signal_sessleader(struct tty *tp, int sig)
 	if (tp->t_session != NULL && tp->t_session->s_leader != NULL) {
 		p = tp->t_session->s_leader;
 		PROC_LOCK(p);
-		psignal(p, sig);
+		kern_psignal(p, sig);
 		PROC_UNLOCK(p);
 	}
 }
diff --git a/sys/kern/tty_pts.c b/sys/kern/tty_pts.c
index f2f5c4e71bd8..4c272ad713a7 100644
--- a/sys/kern/tty_pts.c
+++ b/sys/kern/tty_pts.c
@@ -816,7 +816,7 @@ pts_alloc_external(int fflags, struct thread *td, struct file *fp,
 #endif /* PTS_EXTERNAL */
 
 int
-posix_openpt(struct thread *td, struct posix_openpt_args *uap)
+sys_posix_openpt(struct thread *td, struct posix_openpt_args *uap)
 {
 	int error, fd;
 	struct file *fp;
diff --git a/sys/kern/uipc_mqueue.c b/sys/kern/uipc_mqueue.c
index b91b890e8a27..8a3ea793c872 100644
--- a/sys/kern/uipc_mqueue.c
+++ b/sys/kern/uipc_mqueue.c
@@ -2045,7 +2045,7 @@ kern_kmq_open(struct thread *td, const char *upath, int flags, mode_t mode,
  * Syscall to open a message queue.
  */
 int
-kmq_open(struct thread *td, struct kmq_open_args *uap)
+sys_kmq_open(struct thread *td, struct kmq_open_args *uap)
 {
 	struct mq_attr attr;
 	int flags, error;
@@ -2066,7 +2066,7 @@ kmq_open(struct thread *td, struct kmq_open_args *uap)
  * Syscall to unlink a message queue.
  */
 int
-kmq_unlink(struct thread *td, struct kmq_unlink_args *uap)
+sys_kmq_unlink(struct thread *td, struct kmq_unlink_args *uap)
 {
 	char path[MQFS_NAMELEN+1];
 	struct mqfs_node *pn;
@@ -2169,7 +2169,7 @@ kern_kmq_setattr(struct thread *td, int mqd, const struct mq_attr *attr,
 }
 
 int
-kmq_setattr(struct thread *td, struct kmq_setattr_args *uap)
+sys_kmq_setattr(struct thread *td, struct kmq_setattr_args *uap)
 {
 	struct mq_attr attr, oattr;
 	int error;
@@ -2189,7 +2189,7 @@ kmq_setattr(struct thread *td, struct kmq_setattr_args *uap)
 }
 
 int
-kmq_timedreceive(struct thread *td, struct kmq_timedreceive_args *uap)
+sys_kmq_timedreceive(struct thread *td, struct kmq_timedreceive_args *uap)
 {
 	struct mqueue *mq;
 	struct file *fp;
@@ -2215,7 +2215,7 @@ kmq_timedreceive(struct thread *td, struct kmq_timedreceive_args *uap)
 }
 
 int
-kmq_timedsend(struct thread *td, struct kmq_timedsend_args *uap)
+sys_kmq_timedsend(struct thread *td, struct kmq_timedsend_args *uap)
 {
 	struct mqueue *mq;
 	struct file *fp;
@@ -2240,7 +2240,7 @@ kmq_timedsend(struct thread *td, struct kmq_timedsend_args *uap)
 }
 
 int
-kmq_notify(struct thread *td, struct kmq_notify_args *uap)
+sys_kmq_notify(struct thread *td, struct kmq_notify_args *uap)
 {
 	struct sigevent ev;
 	struct filedesc *fdp;
@@ -2770,8 +2770,8 @@ static struct syscall_helper_data mq32_syscalls[] = {
 	SYSCALL32_INIT_HELPER(freebsd32_kmq_setattr),
 	SYSCALL32_INIT_HELPER(freebsd32_kmq_timedsend),
 	SYSCALL32_INIT_HELPER(freebsd32_kmq_timedreceive),
-	SYSCALL32_INIT_HELPER(kmq_notify),
-	SYSCALL32_INIT_HELPER(kmq_unlink),
+	SYSCALL32_INIT_HELPER_COMPAT(kmq_notify),
+	SYSCALL32_INIT_HELPER_COMPAT(kmq_unlink),
 	SYSCALL_INIT_LAST
 };
 #endif
diff --git a/sys/kern/uipc_sem.c b/sys/kern/uipc_sem.c
index f77bf3b9d5a3..c219844f4fa9 100644
--- a/sys/kern/uipc_sem.c
+++ b/sys/kern/uipc_sem.c
@@ -618,7 +618,7 @@ struct ksem_init_args {
 };
 #endif
 int
-ksem_init(struct thread *td, struct ksem_init_args *uap)
+sys_ksem_init(struct thread *td, struct ksem_init_args *uap)
 {
 
 	return (ksem_create(td, NULL, uap->idp, S_IRWXU | S_IRWXG, uap->value,
@@ -635,7 +635,7 @@ struct ksem_open_args {
 };
 #endif
 int
-ksem_open(struct thread *td, struct ksem_open_args *uap)
+sys_ksem_open(struct thread *td, struct ksem_open_args *uap)
 {
 
 	DP((">>> ksem_open start, pid=%d\n", (int)td->td_proc->p_pid));
@@ -652,7 +652,7 @@ struct ksem_unlink_args {
 };
 #endif
 int
-ksem_unlink(struct thread *td, struct ksem_unlink_args *uap)
+sys_ksem_unlink(struct thread *td, struct ksem_unlink_args *uap)
 {
 	char *path;
 	Fnv32_t fnv;
@@ -680,7 +680,7 @@ struct ksem_close_args {
 };
 #endif
 int
-ksem_close(struct thread *td, struct ksem_close_args *uap)
+sys_ksem_close(struct thread *td, struct ksem_close_args *uap)
 {
 	struct ksem *ks;
 	struct file *fp;
@@ -706,7 +706,7 @@ struct ksem_post_args {
 };
 #endif
 int
-ksem_post(struct thread *td, struct ksem_post_args *uap)
+sys_ksem_post(struct thread *td, struct ksem_post_args *uap)
 {
 	struct file *fp;
 	struct ksem *ks;
@@ -744,7 +744,7 @@ struct ksem_wait_args {
 };
 #endif
 int
-ksem_wait(struct thread *td, struct ksem_wait_args *uap)
+sys_ksem_wait(struct thread *td, struct ksem_wait_args *uap)
 {
 
 	return (kern_sem_wait(td, uap->id, 0, NULL));
@@ -757,7 +757,7 @@ struct ksem_timedwait_args {
 };
 #endif
 int
-ksem_timedwait(struct thread *td, struct ksem_timedwait_args *uap)
+sys_ksem_timedwait(struct thread *td, struct ksem_timedwait_args *uap)
 {
 	struct timespec abstime;
 	struct timespec *ts;
@@ -785,7 +785,7 @@ struct ksem_trywait_args {
 };
 #endif
 int
-ksem_trywait(struct thread *td, struct ksem_trywait_args *uap)
+sys_ksem_trywait(struct thread *td, struct ksem_trywait_args *uap)
 {
 
 	return (kern_sem_wait(td, uap->id, 1, NULL));
@@ -862,7 +862,7 @@ struct ksem_getvalue_args {
 };
 #endif
 int
-ksem_getvalue(struct thread *td, struct ksem_getvalue_args *uap)
+sys_ksem_getvalue(struct thread *td, struct ksem_getvalue_args *uap)
 {
 	struct file *fp;
 	struct ksem *ks;
@@ -896,7 +896,7 @@ struct ksem_destroy_args {
 };
 #endif
 int
-ksem_destroy(struct thread *td, struct ksem_destroy_args *uap)
+sys_ksem_destroy(struct thread *td, struct ksem_destroy_args *uap)
 {
 	struct file *fp;
 	struct ksem *ks;
@@ -994,14 +994,14 @@ freebsd32_ksem_timedwait(struct thread *td,
 static struct syscall_helper_data ksem32_syscalls[] = {
 	SYSCALL32_INIT_HELPER(freebsd32_ksem_init),
 	SYSCALL32_INIT_HELPER(freebsd32_ksem_open),
-	SYSCALL32_INIT_HELPER(ksem_unlink),
-	SYSCALL32_INIT_HELPER(ksem_close),
-	SYSCALL32_INIT_HELPER(ksem_post),
-	SYSCALL32_INIT_HELPER(ksem_wait),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_unlink),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_close),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_post),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_wait),
 	SYSCALL32_INIT_HELPER(freebsd32_ksem_timedwait),
-	SYSCALL32_INIT_HELPER(ksem_trywait),
-	SYSCALL32_INIT_HELPER(ksem_getvalue),
-	SYSCALL32_INIT_HELPER(ksem_destroy),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_trywait),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_getvalue),
+	SYSCALL32_INIT_HELPER_COMPAT(ksem_destroy),
 	SYSCALL_INIT_LAST
 };
 #endif
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index f9fc3ca8bf30..8469f67d0d32 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -486,7 +486,7 @@ shm_remove(char *path, Fnv32_t fnv, struct ucred *ucred)
 
 /* System calls. */
 int
-shm_open(struct thread *td, struct shm_open_args *uap)
+sys_shm_open(struct thread *td, struct shm_open_args *uap)
 {
 	struct filedesc *fdp;
 	struct shmfd *shmfd;
@@ -620,7 +620,7 @@ shm_open(struct thread *td, struct shm_open_args *uap)
 }
 
 int
-shm_unlink(struct thread *td, struct shm_unlink_args *uap)
+sys_shm_unlink(struct thread *td, struct shm_unlink_args *uap)
 {
 	char *path;
 	Fnv32_t fnv;
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 0e5efe6f3a9b..3b83e1c5c2e8 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -171,7 +171,7 @@ getsock_cap(struct filedesc *fdp, int fd, cap_rights_t rights,
 #endif
 
 int
-socket(td, uap)
+sys_socket(td, uap)
 	struct thread *td;
 	struct socket_args /* {
 		int	domain;
@@ -210,7 +210,7 @@ socket(td, uap)
 
 /* ARGSUSED */
 int
-bind(td, uap)
+sys_bind(td, uap)
 	struct thread *td;
 	struct bind_args /* {
 		int	s;
@@ -259,7 +259,7 @@ kern_bind(td, fd, sa)
 
 /* ARGSUSED */
 int
-listen(td, uap)
+sys_listen(td, uap)
 	struct thread *td;
 	struct listen_args /* {
 		int	s;
@@ -495,7 +495,7 @@ kern_accept(struct thread *td, int s, struct sockaddr **name,
 }
 
 int
-accept(td, uap)
+sys_accept(td, uap)
 	struct thread *td;
 	struct accept_args *uap;
 {
@@ -516,7 +516,7 @@ oaccept(td, uap)
 
 /* ARGSUSED */
 int
-connect(td, uap)
+sys_connect(td, uap)
 	struct thread *td;
 	struct connect_args /* {
 		int	s;
@@ -664,7 +664,7 @@ kern_socketpair(struct thread *td, int domain, int type, int protocol,
 }
 
 int
-socketpair(struct thread *td, struct socketpair_args *uap)
+sys_socketpair(struct thread *td, struct socketpair_args *uap)
 {
 	int error, sv[2];
 
@@ -834,7 +834,7 @@ kern_sendit(td, s, mp, flags, control, segflg)
 }
 
 int
-sendto(td, uap)
+sys_sendto(td, uap)
 	struct thread *td;
 	struct sendto_args /* {
 		int	s;
@@ -918,7 +918,7 @@ osendmsg(td, uap)
 #endif
 
 int
-sendmsg(td, uap)
+sys_sendmsg(td, uap)
 	struct thread *td;
 	struct sendmsg_args /* {
 		int	s;
@@ -1128,7 +1128,7 @@ recvit(td, s, mp, namelenp)
 }
 
 int
-recvfrom(td, uap)
+sys_recvfrom(td, uap)
 	struct thread *td;
 	struct recvfrom_args /* {
 		int	s;
@@ -1171,7 +1171,7 @@ orecvfrom(td, uap)
 {
 
 	uap->flags |= MSG_COMPAT;
-	return (recvfrom(td, uap));
+	return (sys_recvfrom(td, uap));
 }
 #endif
 
@@ -1238,7 +1238,7 @@ orecvmsg(td, uap)
 #endif
 
 int
-recvmsg(td, uap)
+sys_recvmsg(td, uap)
 	struct thread *td;
 	struct recvmsg_args /* {
 		int	s;
@@ -1273,7 +1273,7 @@ recvmsg(td, uap)
 
 /* ARGSUSED */
 int
-shutdown(td, uap)
+sys_shutdown(td, uap)
 	struct thread *td;
 	struct shutdown_args /* {
 		int	s;
@@ -1297,7 +1297,7 @@ shutdown(td, uap)
 
 /* ARGSUSED */
 int
-setsockopt(td, uap)
+sys_setsockopt(td, uap)
 	struct thread *td;
 	struct setsockopt_args /* {
 		int	s;
@@ -1360,7 +1360,7 @@ kern_setsockopt(td, s, level, name, val, valseg, valsize)
 
 /* ARGSUSED */
 int
-getsockopt(td, uap)
+sys_getsockopt(td, uap)
 	struct thread *td;
 	struct getsockopt_args /* {
 		int	s;
@@ -1519,7 +1519,7 @@ kern_getsockname(struct thread *td, int fd, struct sockaddr **sa,
 }
 
 int
-getsockname(td, uap)
+sys_getsockname(td, uap)
 	struct thread *td;
 	struct getsockname_args *uap;
 {
@@ -1624,7 +1624,7 @@ kern_getpeername(struct thread *td, int fd, struct sockaddr **sa,
 }
 
 int
-getpeername(td, uap)
+sys_getpeername(td, uap)
 	struct thread *td;
 	struct getpeername_args *uap;
 {
@@ -1764,7 +1764,7 @@ sf_buf_mext(void *addr, void *args)
  * specified, write the total number of bytes sent into *sbytes.
  */
 int
-sendfile(struct thread *td, struct sendfile_args *uap)
+sys_sendfile(struct thread *td, struct sendfile_args *uap)
 {
 
 	return (do_sendfile(td, uap, 0));
@@ -2300,7 +2300,7 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
  * XXX: We should make this loadable one day.
  */
 int
-sctp_peeloff(td, uap)
+sys_sctp_peeloff(td, uap)
 	struct thread *td;
 	struct sctp_peeloff_args /* {
 		int	sd;
@@ -2387,7 +2387,7 @@ sctp_peeloff(td, uap)
 }
 
 int
-sctp_generic_sendmsg (td, uap)
+sys_sctp_generic_sendmsg (td, uap)
 	struct thread *td;
 	struct sctp_generic_sendmsg_args /* {
 		int sd, 
@@ -2494,7 +2494,7 @@ sctp_generic_sendmsg (td, uap)
 }
 
 int
-sctp_generic_sendmsg_iov(td, uap)
+sys_sctp_generic_sendmsg_iov(td, uap)
 	struct thread *td;
 	struct sctp_generic_sendmsg_iov_args /* {
 		int sd, 
@@ -2616,7 +2616,7 @@ sctp_generic_sendmsg_iov(td, uap)
 }
 
 int
-sctp_generic_recvmsg(td, uap)
+sys_sctp_generic_recvmsg(td, uap)
 	struct thread *td;
 	struct sctp_generic_recvmsg_args /* {
 		int sd, 
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 3a34f58605b5..801a24434ec7 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -462,6 +462,8 @@ uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	unp = sotounpcb(so);
 	KASSERT(unp != NULL, ("uipc_bind: unp == NULL"));
 
+	if (soun->sun_len > sizeof(struct sockaddr_un))
+		return (EINVAL);
 	namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path);
 	if (namelen <= 0)
 		return (EINVAL);
@@ -1252,6 +1254,8 @@ unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	unp = sotounpcb(so);
 	KASSERT(unp != NULL, ("unp_connect: unp == NULL"));
 
+	if (nam->sa_len > sizeof(struct sockaddr_un))
+		return (EINVAL);
 	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
 	if (len <= 0)
 		return (EINVAL);
diff --git a/sys/kern/vfs_acl.c b/sys/kern/vfs_acl.c
index 9010a5068bfb..2516d5cf8a3b 100644
--- a/sys/kern/vfs_acl.c
+++ b/sys/kern/vfs_acl.c
@@ -324,7 +324,7 @@ vacl_aclcheck(struct thread *td, struct vnode *vp, acl_type_t type,
  * Given a file path, get an ACL for it
  */
 int
-__acl_get_file(struct thread *td, struct __acl_get_file_args *uap)
+sys___acl_get_file(struct thread *td, struct __acl_get_file_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -344,7 +344,7 @@ __acl_get_file(struct thread *td, struct __acl_get_file_args *uap)
  * Given a file path, get an ACL for it; don't follow links.
  */
 int
-__acl_get_link(struct thread *td, struct __acl_get_link_args *uap)
+sys___acl_get_link(struct thread *td, struct __acl_get_link_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -364,7 +364,7 @@ __acl_get_link(struct thread *td, struct __acl_get_link_args *uap)
  * Given a file path, set an ACL for it.
  */
 int
-__acl_set_file(struct thread *td, struct __acl_set_file_args *uap)
+sys___acl_set_file(struct thread *td, struct __acl_set_file_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -384,7 +384,7 @@ __acl_set_file(struct thread *td, struct __acl_set_file_args *uap)
  * Given a file path, set an ACL for it; don't follow links.
  */
 int
-__acl_set_link(struct thread *td, struct __acl_set_link_args *uap)
+sys___acl_set_link(struct thread *td, struct __acl_set_link_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -404,7 +404,7 @@ __acl_set_link(struct thread *td, struct __acl_set_link_args *uap)
  * Given a file descriptor, get an ACL for it.
  */
 int
-__acl_get_fd(struct thread *td, struct __acl_get_fd_args *uap)
+sys___acl_get_fd(struct thread *td, struct __acl_get_fd_args *uap)
 {
 	struct file *fp;
 	int vfslocked, error;
@@ -423,7 +423,7 @@ __acl_get_fd(struct thread *td, struct __acl_get_fd_args *uap)
  * Given a file descriptor, set an ACL for it.
  */
 int
-__acl_set_fd(struct thread *td, struct __acl_set_fd_args *uap)
+sys___acl_set_fd(struct thread *td, struct __acl_set_fd_args *uap)
 {
 	struct file *fp;
 	int vfslocked, error;
@@ -442,7 +442,7 @@ __acl_set_fd(struct thread *td, struct __acl_set_fd_args *uap)
  * Given a file path, delete an ACL from it.
  */
 int
-__acl_delete_file(struct thread *td, struct __acl_delete_file_args *uap)
+sys___acl_delete_file(struct thread *td, struct __acl_delete_file_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -462,7 +462,7 @@ __acl_delete_file(struct thread *td, struct __acl_delete_file_args *uap)
  * Given a file path, delete an ACL from it; don't follow links.
  */
 int
-__acl_delete_link(struct thread *td, struct __acl_delete_link_args *uap)
+sys___acl_delete_link(struct thread *td, struct __acl_delete_link_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -482,7 +482,7 @@ __acl_delete_link(struct thread *td, struct __acl_delete_link_args *uap)
  * Given a file path, delete an ACL from it.
  */
 int
-__acl_delete_fd(struct thread *td, struct __acl_delete_fd_args *uap)
+sys___acl_delete_fd(struct thread *td, struct __acl_delete_fd_args *uap)
 {
 	struct file *fp;
 	int vfslocked, error;
@@ -502,7 +502,7 @@ __acl_delete_fd(struct thread *td, struct __acl_delete_fd_args *uap)
  * Given a file path, check an ACL for it.
  */
 int
-__acl_aclcheck_file(struct thread *td, struct __acl_aclcheck_file_args *uap)
+sys___acl_aclcheck_file(struct thread *td, struct __acl_aclcheck_file_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -522,7 +522,7 @@ __acl_aclcheck_file(struct thread *td, struct __acl_aclcheck_file_args *uap)
  * Given a file path, check an ACL for it; don't follow links.
  */
 int
-__acl_aclcheck_link(struct thread *td, struct __acl_aclcheck_link_args *uap)
+sys___acl_aclcheck_link(struct thread *td, struct __acl_aclcheck_link_args *uap)
 {
 	struct nameidata nd;
 	int vfslocked, error;
@@ -542,7 +542,7 @@ __acl_aclcheck_link(struct thread *td, struct __acl_aclcheck_link_args *uap)
  * Given a file descriptor, check an ACL for it.
  */
 int
-__acl_aclcheck_fd(struct thread *td, struct __acl_aclcheck_fd_args *uap)
+sys___acl_aclcheck_fd(struct thread *td, struct __acl_aclcheck_fd_args *uap)
 {
 	struct file *fp;
 	int vfslocked, error;
diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index aedbdd05385a..7af9f552a417 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -943,7 +943,7 @@ aio_process(struct aiocblist *aiocbe)
 			}
 			if (sigpipe) {
 				PROC_LOCK(aiocbe->userproc);
-				psignal(aiocbe->userproc, SIGPIPE);
+				kern_psignal(aiocbe->userproc, SIGPIPE);
 				PROC_UNLOCK(aiocbe->userproc);
 			}
 		}
@@ -1057,7 +1057,7 @@ aio_daemon(void *_id)
 	aiop->aiothreadflags = 0;
 
 	/* The daemon resides in its own pgrp. */
-	setsid(td, NULL);
+	sys_setsid(td, NULL);
 
 	/*
 	 * Wakeup parent process.  (Parent sleeps to keep from blasting away
@@ -1867,7 +1867,7 @@ kern_aio_return(struct thread *td, struct aiocb *uaiocb, struct aiocb_ops *ops)
 }
 
 int
-aio_return(struct thread *td, struct aio_return_args *uap)
+sys_aio_return(struct thread *td, struct aio_return_args *uap)
 {
 
 	return (kern_aio_return(td, uap->aiocbp, &aiocb_ops));
@@ -1936,7 +1936,7 @@ kern_aio_suspend(struct thread *td, int njoblist, struct aiocb **ujoblist,
 }
 
 int
-aio_suspend(struct thread *td, struct aio_suspend_args *uap)
+sys_aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 {
 	struct timespec ts, *tsp;
 	struct aiocb **ujoblist;
@@ -1966,7 +1966,7 @@ aio_suspend(struct thread *td, struct aio_suspend_args *uap)
  * progress.
  */
 int
-aio_cancel(struct thread *td, struct aio_cancel_args *uap)
+sys_aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct kaioinfo *ki;
@@ -2106,7 +2106,7 @@ kern_aio_error(struct thread *td, struct aiocb *aiocbp, struct aiocb_ops *ops)
 }
 
 int
-aio_error(struct thread *td, struct aio_error_args *uap)
+sys_aio_error(struct thread *td, struct aio_error_args *uap)
 {
 
 	return (kern_aio_error(td, uap->aiocbp, &aiocb_ops));
@@ -2114,7 +2114,7 @@ aio_error(struct thread *td, struct aio_error_args *uap)
 
 /* syscall - asynchronous read from a file (REALTIME) */
 int
-oaio_read(struct thread *td, struct oaio_read_args *uap)
+sys_oaio_read(struct thread *td, struct oaio_read_args *uap)
 {
 
 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
@@ -2122,7 +2122,7 @@ oaio_read(struct thread *td, struct oaio_read_args *uap)
 }
 
 int
-aio_read(struct thread *td, struct aio_read_args *uap)
+sys_aio_read(struct thread *td, struct aio_read_args *uap)
 {
 
 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops));
@@ -2130,7 +2130,7 @@ aio_read(struct thread *td, struct aio_read_args *uap)
 
 /* syscall - asynchronous write to a file (REALTIME) */
 int
-oaio_write(struct thread *td, struct oaio_write_args *uap)
+sys_oaio_write(struct thread *td, struct oaio_write_args *uap)
 {
 
 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
@@ -2138,7 +2138,7 @@ oaio_write(struct thread *td, struct oaio_write_args *uap)
 }
 
 int
-aio_write(struct thread *td, struct aio_write_args *uap)
+sys_aio_write(struct thread *td, struct aio_write_args *uap)
 {
 
 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
@@ -2281,7 +2281,7 @@ kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
 
 /* syscall - list directed I/O (REALTIME) */
 int
-olio_listio(struct thread *td, struct olio_listio_args *uap)
+sys_olio_listio(struct thread *td, struct olio_listio_args *uap)
 {
 	struct aiocb **acb_list;
 	struct sigevent *sigp, sig;
@@ -2318,7 +2318,7 @@ olio_listio(struct thread *td, struct olio_listio_args *uap)
 
 /* syscall - list directed I/O (REALTIME) */
 int
-lio_listio(struct thread *td, struct lio_listio_args *uap)
+sys_lio_listio(struct thread *td, struct lio_listio_args *uap)
 {
 	struct aiocb **acb_list;
 	struct sigevent *sigp, sig;
@@ -2465,7 +2465,7 @@ kern_aio_waitcomplete(struct thread *td, struct aiocb **aiocbp,
 }
 
 int
-aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
+sys_aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 {
 	struct timespec ts, *tsp;
 	int error;
@@ -2498,7 +2498,7 @@ kern_aio_fsync(struct thread *td, int op, struct aiocb *aiocbp,
 }
 
 int
-aio_fsync(struct thread *td, struct aio_fsync_args *uap)
+sys_aio_fsync(struct thread *td, struct aio_fsync_args *uap)
 {
 
 	return (kern_aio_fsync(td, uap->op, uap->aiocbp, &aiocb_ops));
@@ -2836,7 +2836,7 @@ int
 freebsd32_aio_cancel(struct thread *td, struct freebsd32_aio_cancel_args *uap)
 {
 
-	return (aio_cancel(td, (struct aio_cancel_args *)uap));
+	return (sys_aio_cancel(td, (struct aio_cancel_args *)uap));
 }
 
 int
diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index 30fb28bd76a5..ee4bf3ce82df 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -925,7 +925,7 @@ SYSCTL_INT(_debug, OID_AUTO, disablecwd, CTLFLAG_RW, &disablecwd, 0,
 
 /* Implementation of the getcwd syscall. */
 int
-__getcwd(td, uap)
+sys___getcwd(td, uap)
 	struct thread *td;
 	struct __getcwd_args *uap;
 {
diff --git a/sys/kern/vfs_extattr.c b/sys/kern/vfs_extattr.c
index b8b9cdf058bd..7732a94c7696 100644
--- a/sys/kern/vfs_extattr.c
+++ b/sys/kern/vfs_extattr.c
@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
  * Currently this is used only by UFS1 extended attributes.
  */
 int
-extattrctl(td, uap)
+sys_extattrctl(td, uap)
 	struct thread *td;
 	struct extattrctl_args /* {
 		const char *path;
@@ -210,7 +210,7 @@ extattr_set_vp(struct vnode *vp, int attrnamespace, const char *attrname,
 }
 
 int
-extattr_set_fd(td, uap)
+sys_extattr_set_fd(td, uap)
 	struct thread *td;
 	struct extattr_set_fd_args /* {
 		int fd;
@@ -245,7 +245,7 @@ extattr_set_fd(td, uap)
 }
 
 int
-extattr_set_file(td, uap)
+sys_extattr_set_file(td, uap)
 	struct thread *td;
 	struct extattr_set_file_args /* {
 		const char *path;
@@ -282,7 +282,7 @@ extattr_set_file(td, uap)
 }
 
 int
-extattr_set_link(td, uap)
+sys_extattr_set_link(td, uap)
 	struct thread *td;
 	struct extattr_set_link_args /* {
 		const char *path;
@@ -390,7 +390,7 @@ extattr_get_vp(struct vnode *vp, int attrnamespace, const char *attrname,
 }
 
 int
-extattr_get_fd(td, uap)
+sys_extattr_get_fd(td, uap)
 	struct thread *td;
 	struct extattr_get_fd_args /* {
 		int fd;
@@ -425,7 +425,7 @@ extattr_get_fd(td, uap)
 }
 
 int
-extattr_get_file(td, uap)
+sys_extattr_get_file(td, uap)
 	struct thread *td;
 	struct extattr_get_file_args /* {
 		const char *path;
@@ -462,7 +462,7 @@ extattr_get_file(td, uap)
 }
 
 int
-extattr_get_link(td, uap)
+sys_extattr_get_link(td, uap)
 	struct thread *td;
 	struct extattr_get_link_args /* {
 		const char *path;
@@ -542,7 +542,7 @@ extattr_delete_vp(struct vnode *vp, int attrnamespace, const char *attrname,
 }
 
 int
-extattr_delete_fd(td, uap)
+sys_extattr_delete_fd(td, uap)
 	struct thread *td;
 	struct extattr_delete_fd_args /* {
 		int fd;
@@ -575,7 +575,7 @@ extattr_delete_fd(td, uap)
 }
 
 int
-extattr_delete_file(td, uap)
+sys_extattr_delete_file(td, uap)
 	struct thread *td;
 	struct extattr_delete_file_args /* {
 		const char *path;
@@ -608,7 +608,7 @@ extattr_delete_file(td, uap)
 }
 
 int
-extattr_delete_link(td, uap)
+sys_extattr_delete_link(td, uap)
 	struct thread *td;
 	struct extattr_delete_link_args /* {
 		const char *path;
@@ -707,7 +707,7 @@ extattr_list_vp(struct vnode *vp, int attrnamespace, void *data,
 
 
 int
-extattr_list_fd(td, uap)
+sys_extattr_list_fd(td, uap)
 	struct thread *td;
 	struct extattr_list_fd_args /* {
 		int fd;
@@ -735,7 +735,7 @@ extattr_list_fd(td, uap)
 }
 
 int
-extattr_list_file(td, uap)
+sys_extattr_list_file(td, uap)
 	struct thread*td;
 	struct extattr_list_file_args /* {
 		const char *path;
@@ -765,7 +765,7 @@ extattr_list_file(td, uap)
 }
 
 int
-extattr_list_link(td, uap)
+sys_extattr_list_link(td, uap)
 	struct thread*td;
 	struct extattr_list_link_args /* {
 		const char *path;
diff --git a/sys/kern/vfs_lookup.c b/sys/kern/vfs_lookup.c
index a14501704fa9..e2aad7cf3a61 100644
--- a/sys/kern/vfs_lookup.c
+++ b/sys/kern/vfs_lookup.c
@@ -188,8 +188,13 @@ namei(struct nameidata *ndp)
 	 */
 	if (IN_CAPABILITY_MODE(td)) {
 		ndp->ni_strictrelative = 1;
-		if (ndp->ni_dirfd == AT_FDCWD)
+		if (ndp->ni_dirfd == AT_FDCWD) {
+#ifdef KTRACE
+			if (KTRPOINT(td, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_LOOKUP, 0, 0);
+#endif
 			error = ECAPMODE;
+		}
 	}
 #endif
 	if (error) {
@@ -281,8 +286,13 @@ namei(struct nameidata *ndp)
 		if (*(cnp->cn_nameptr) == '/') {
 			vrele(dp);
 			VFS_UNLOCK_GIANT(vfslocked);
-			if (ndp->ni_strictrelative != 0)
+			if (ndp->ni_strictrelative != 0) {
+#ifdef KTRACE
+				if (KTRPOINT(curthread, KTR_CAPFAIL))
+					ktrcapfail(CAPFAIL_LOOKUP, 0, 0);
+#endif
 				return (ENOTCAPABLE);
+			}
 			while (*(cnp->cn_nameptr) == '/') {
 				cnp->cn_nameptr++;
 				ndp->ni_pathlen--;
@@ -644,6 +654,10 @@ lookup(struct nameidata *ndp)
 	 */
 	if (cnp->cn_flags & ISDOTDOT) {
 		if (ndp->ni_strictrelative != 0) {
+#ifdef KTRACE
+			if (KTRPOINT(curthread, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_LOOKUP, 0, 0);
+#endif
 			error = ENOTCAPABLE;
 			goto bad;
 		}
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index 5d6892c5e9b7..e8b89e58b25a 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -365,7 +365,7 @@ vfs_mergeopts(struct vfsoptlist *toopts, struct vfsoptlist *oldopts)
  * Mount a filesystem.
  */
 int
-nmount(td, uap)
+sys_nmount(td, uap)
 	struct thread *td;
 	struct nmount_args /* {
 		struct iovec *iovp;
@@ -682,7 +682,7 @@ struct mount_args {
 #endif
 /* ARGSUSED */
 int
-mount(td, uap)
+sys_mount(td, uap)
 	struct thread *td;
 	struct mount_args /* {
 		char *type;
@@ -1097,7 +1097,7 @@ struct unmount_args {
 #endif
 /* ARGSUSED */
 int
-unmount(td, uap)
+sys_unmount(td, uap)
 	struct thread *td;
 	register struct unmount_args /* {
 		char *path;
@@ -1227,18 +1227,6 @@ dounmount(mp, flags, td)
 		mp->mnt_kern_flag |= MNTK_UNMOUNTF;
 	error = 0;
 	if (mp->mnt_lockref) {
-		if ((flags & MNT_FORCE) == 0) {
-			mp->mnt_kern_flag &= ~(MNTK_UNMOUNT | MNTK_NOINSMNTQ |
-			    MNTK_UNMOUNTF);
-			if (mp->mnt_kern_flag & MNTK_MWAIT) {
-				mp->mnt_kern_flag &= ~MNTK_MWAIT;
-				wakeup(mp);
-			}
-			MNT_IUNLOCK(mp);
-			if (coveredvp)
-				VOP_UNLOCK(coveredvp, 0);
-			return (EBUSY);
-		}
 		mp->mnt_kern_flag |= MNTK_DRAINING;
 		error = msleep(&mp->mnt_lockref, MNT_MTX(mp), PVFS,
 		    "mount drain", 0);
diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c
index ccbcb33f0f95..98e252ed6b7d 100644
--- a/sys/kern/vfs_mountroot.c
+++ b/sys/kern/vfs_mountroot.c
@@ -483,26 +483,27 @@ parse_dir_ask(char **conf)
 	printf("  .               Yield 1 second (for background tasks)\n");
 	printf("  <empty line>    Abort manual input\n");
 
- again:
-	printf("\nmountroot> ");
-	gets(name, sizeof(name), GETS_ECHO);
-	if (name[0] == '\0')
-		return (0);
-	if (name[0] == '?') {
-		printf("\nList of GEOM managed disk devices:\n  ");
-		g_dev_print();
-		goto again;
-	}
-	if (name[0] == '.') {
-		pause("rmask", hz);
-		goto again;
-	}
-	mnt = name;
-	error = parse_mount(&mnt);
-	if (error == -1) {
-		printf("Invalid specification.\n");
-		goto again;
-	}
+	do {
+		error = EINVAL;
+		printf("\nmountroot> ");
+		gets(name, sizeof(name), GETS_ECHO);
+		if (name[0] == '\0')
+			break;
+		if (name[0] == '?' && name[1] == '\0') {
+			printf("\nList of GEOM managed disk devices:\n  ");
+			g_dev_print();
+			continue;
+		}
+		if (name[0] == '.' && name[1] == '\0') {
+			pause("rmask", hz);
+			continue;
+		}
+		mnt = name;
+		error = parse_mount(&mnt);
+		if (error == -1)
+			printf("Invalid file system specification.\n");
+	} while (error != 0);
+
 	return (error);
 }
 
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 325ca99deb32..8101fce0b61e 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -348,6 +348,38 @@ SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
 /*
  * Mark a mount point as busy. Used to synchronize access and to delay
  * unmounting. Eventually, mountlist_mtx is not released on failure.
+ *
+ * vfs_busy() is a custom lock, it can block the caller.
+ * vfs_busy() only sleeps if the unmount is active on the mount point.
+ * For a mountpoint mp, vfs_busy-enforced lock is before lock of any
+ * vnode belonging to mp.
+ *
+ * Lookup uses vfs_busy() to traverse mount points.
+ * root fs			var fs
+ * / vnode lock		A	/ vnode lock (/var)		D
+ * /var vnode lock	B	/log vnode lock(/var/log)	E
+ * vfs_busy lock	C	vfs_busy lock			F
+ *
+ * Within each file system, the lock order is C->A->B and F->D->E.
+ *
+ * When traversing across mounts, the system follows that lock order:
+ *
+ *        C->A->B
+ *              |
+ *              +->F->D->E
+ *
+ * The lookup() process for namei("/var") illustrates the process:
+ *  VOP_LOOKUP() obtains B while A is held
+ *  vfs_busy() obtains a shared lock on F while A and B are held
+ *  vput() releases lock on B
+ *  vput() releases lock on A
+ *  VFS_ROOT() obtains lock on D while shared lock on F is held
+ *  vfs_unbusy() releases shared lock on F
+ *  vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A.
+ *    Attempt to lock A (instead of vp_crossmp) while D is held would
+ *    violate the global order, causing deadlocks.
+ *
+ * dounmount() locks B while F is drained.
  */
 int
 vfs_busy(struct mount *mp, int flags)
@@ -1022,7 +1054,7 @@ getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops,
 	vp->v_tag = tag;
 	vp->v_op = vops;
 	v_incr_usecount(vp);
-	vp->v_data = 0;
+	vp->v_data = NULL;
 #ifdef MAC
 	mac_vnode_init(vp);
 	if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
@@ -3022,7 +3054,7 @@ vfs_sysctl(SYSCTL_HANDLER_ARGS)
 	struct vfsconf *vfsp;
 	struct xvfsconf xvfsp;
 
-	printf("WARNING: userland calling deprecated sysctl, "
+	log(LOG_WARNING, "userland calling deprecated sysctl, "
 	    "please rebuild world\n");
 
 #if 1 || defined(COMPAT_PRELITE2)
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index c0ae0a747649..ec5ad06115cd 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -125,7 +125,7 @@ struct sync_args {
 #endif
 /* ARGSUSED */
 int
-sync(td, uap)
+sys_sync(td, uap)
 	struct thread *td;
 	struct sync_args *uap;
 {
@@ -176,7 +176,7 @@ struct quotactl_args {
 };
 #endif
 int
-quotactl(td, uap)
+sys_quotactl(td, uap)
 	struct thread *td;
 	register struct quotactl_args /* {
 		char *path;
@@ -266,7 +266,7 @@ struct statfs_args {
 };
 #endif
 int
-statfs(td, uap)
+sys_statfs(td, uap)
 	struct thread *td;
 	register struct statfs_args /* {
 		char *path;
@@ -346,7 +346,7 @@ struct fstatfs_args {
 };
 #endif
 int
-fstatfs(td, uap)
+sys_fstatfs(td, uap)
 	struct thread *td;
 	register struct fstatfs_args /* {
 		int fd;
@@ -437,7 +437,7 @@ struct getfsstat_args {
 };
 #endif
 int
-getfsstat(td, uap)
+sys_getfsstat(td, uap)
 	struct thread *td;
 	register struct getfsstat_args /* {
 		struct statfs *buf;
@@ -732,7 +732,7 @@ struct fchdir_args {
 };
 #endif
 int
-fchdir(td, uap)
+sys_fchdir(td, uap)
 	struct thread *td;
 	struct fchdir_args /* {
 		int fd;
@@ -797,7 +797,7 @@ struct chdir_args {
 };
 #endif
 int
-chdir(td, uap)
+sys_chdir(td, uap)
 	struct thread *td;
 	struct chdir_args /* {
 		char *path;
@@ -889,7 +889,7 @@ struct chroot_args {
 };
 #endif
 int
-chroot(td, uap)
+sys_chroot(td, uap)
 	struct thread *td;
 	struct chroot_args /* {
 		char *path;
@@ -1038,7 +1038,7 @@ struct open_args {
 };
 #endif
 int
-open(td, uap)
+sys_open(td, uap)
 	struct thread *td;
 	register struct open_args /* {
 		char *path;
@@ -1059,7 +1059,7 @@ struct openat_args {
 };
 #endif
 int
-openat(struct thread *td, struct openat_args *uap)
+sys_openat(struct thread *td, struct openat_args *uap)
 {
 
 	return (kern_openat(td, uap->fd, uap->path, UIO_USERSPACE, uap->flag,
@@ -1279,7 +1279,7 @@ struct mknod_args {
 };
 #endif
 int
-mknod(td, uap)
+sys_mknod(td, uap)
 	struct thread *td;
 	register struct mknod_args /* {
 		char *path;
@@ -1300,7 +1300,7 @@ struct mknodat_args {
 };
 #endif
 int
-mknodat(struct thread *td, struct mknodat_args *uap)
+sys_mknodat(struct thread *td, struct mknodat_args *uap)
 {
 
 	return (kern_mknodat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode,
@@ -1432,7 +1432,7 @@ struct mkfifo_args {
 };
 #endif
 int
-mkfifo(td, uap)
+sys_mkfifo(td, uap)
 	struct thread *td;
 	register struct mkfifo_args /* {
 		char *path;
@@ -1451,7 +1451,7 @@ struct mkfifoat_args {
 };
 #endif
 int
-mkfifoat(struct thread *td, struct mkfifoat_args *uap)
+sys_mkfifoat(struct thread *td, struct mkfifoat_args *uap)
 {
 
 	return (kern_mkfifoat(td, uap->fd, uap->path, UIO_USERSPACE,
@@ -1533,7 +1533,7 @@ struct link_args {
 };
 #endif
 int
-link(td, uap)
+sys_link(td, uap)
 	struct thread *td;
 	register struct link_args /* {
 		char *path;
@@ -1554,7 +1554,7 @@ struct linkat_args {
 };
 #endif
 int
-linkat(struct thread *td, struct linkat_args *uap)
+sys_linkat(struct thread *td, struct linkat_args *uap)
 {
 	int flag;
 
@@ -1685,7 +1685,7 @@ struct symlink_args {
 };
 #endif
 int
-symlink(td, uap)
+sys_symlink(td, uap)
 	struct thread *td;
 	register struct symlink_args /* {
 		char *path;
@@ -1704,7 +1704,7 @@ struct symlinkat_args {
 };
 #endif
 int
-symlinkat(struct thread *td, struct symlinkat_args *uap)
+sys_symlinkat(struct thread *td, struct symlinkat_args *uap)
 {
 
 	return (kern_symlinkat(td, uap->path1, uap->fd, uap->path2,
@@ -1792,7 +1792,7 @@ kern_symlinkat(struct thread *td, char *path1, int fd, char *path2,
  * Delete a whiteout from the filesystem.
  */
 int
-undelete(td, uap)
+sys_undelete(td, uap)
 	struct thread *td;
 	register struct undelete_args /* {
 		char *path;
@@ -1848,7 +1848,7 @@ struct unlink_args {
 };
 #endif
 int
-unlink(td, uap)
+sys_unlink(td, uap)
 	struct thread *td;
 	struct unlink_args /* {
 		char *path;
@@ -1866,7 +1866,7 @@ struct unlinkat_args {
 };
 #endif
 int
-unlinkat(struct thread *td, struct unlinkat_args *uap)
+sys_unlinkat(struct thread *td, struct unlinkat_args *uap)
 {
 	int flag = uap->flag;
 	int fd = uap->fd;
@@ -1970,7 +1970,7 @@ struct lseek_args {
 };
 #endif
 int
-lseek(td, uap)
+sys_lseek(td, uap)
 	struct thread *td;
 	register struct lseek_args /* {
 		int fd;
@@ -2084,7 +2084,7 @@ olseek(td, uap)
 	nuap.fd = uap->fd;
 	nuap.offset = uap->offset;
 	nuap.whence = uap->whence;
-	return (lseek(td, &nuap));
+	return (sys_lseek(td, &nuap));
 }
 #endif /* COMPAT_43 */
 
@@ -2099,7 +2099,7 @@ freebsd6_lseek(td, uap)
 	ouap.fd = uap->fd;
 	ouap.offset = uap->offset;
 	ouap.whence = uap->whence;
-	return (lseek(td, &ouap));
+	return (sys_lseek(td, &ouap));
 }
 
 /*
@@ -2146,7 +2146,7 @@ struct access_args {
 };
 #endif
 int
-access(td, uap)
+sys_access(td, uap)
 	struct thread *td;
 	register struct access_args /* {
 		char *path;
@@ -2166,7 +2166,7 @@ struct faccessat_args {
 }
 #endif
 int
-faccessat(struct thread *td, struct faccessat_args *uap)
+sys_faccessat(struct thread *td, struct faccessat_args *uap)
 {
 
 	if (uap->flag & ~AT_EACCESS)
@@ -2234,7 +2234,7 @@ struct eaccess_args {
 };
 #endif
 int
-eaccess(td, uap)
+sys_eaccess(td, uap)
 	struct thread *td;
 	register struct eaccess_args /* {
 		char *path;
@@ -2351,7 +2351,7 @@ struct stat_args {
 };
 #endif
 int
-stat(td, uap)
+sys_stat(td, uap)
 	struct thread *td;
 	register struct stat_args /* {
 		char *path;
@@ -2376,7 +2376,7 @@ struct fstatat_args {
 }
 #endif
 int
-fstatat(struct thread *td, struct fstatat_args *uap)
+sys_fstatat(struct thread *td, struct fstatat_args *uap)
 {
 	struct stat sb;
 	int error;
@@ -2453,7 +2453,7 @@ struct lstat_args {
 };
 #endif
 int
-lstat(td, uap)
+sys_lstat(td, uap)
 	struct thread *td;
 	register struct lstat_args /* {
 		char *path;
@@ -2511,7 +2511,7 @@ struct nstat_args {
 };
 #endif
 int
-nstat(td, uap)
+sys_nstat(td, uap)
 	struct thread *td;
 	register struct nstat_args /* {
 		char *path;
@@ -2540,7 +2540,7 @@ struct lstat_args {
 };
 #endif
 int
-nlstat(td, uap)
+sys_nlstat(td, uap)
 	struct thread *td;
 	register struct nlstat_args /* {
 		char *path;
@@ -2569,7 +2569,7 @@ struct pathconf_args {
 };
 #endif
 int
-pathconf(td, uap)
+sys_pathconf(td, uap)
 	struct thread *td;
 	register struct pathconf_args /* {
 		char *path;
@@ -2587,7 +2587,7 @@ struct lpathconf_args {
 };
 #endif
 int
-lpathconf(td, uap)
+sys_lpathconf(td, uap)
 	struct thread *td;
 	register struct lpathconf_args /* {
 		char *path;
@@ -2633,7 +2633,7 @@ struct readlink_args {
 };
 #endif
 int
-readlink(td, uap)
+sys_readlink(td, uap)
 	struct thread *td;
 	register struct readlink_args /* {
 		char *path;
@@ -2654,7 +2654,7 @@ struct readlinkat_args {
 };
 #endif
 int
-readlinkat(struct thread *td, struct readlinkat_args *uap)
+sys_readlinkat(struct thread *td, struct readlinkat_args *uap)
 {
 
 	return (kern_readlinkat(td, uap->fd, uap->path, UIO_USERSPACE,
@@ -2770,7 +2770,7 @@ struct chflags_args {
 };
 #endif
 int
-chflags(td, uap)
+sys_chflags(td, uap)
 	struct thread *td;
 	register struct chflags_args /* {
 		char *path;
@@ -2798,7 +2798,7 @@ chflags(td, uap)
  * Same as chflags() but doesn't follow symlinks.
  */
 int
-lchflags(td, uap)
+sys_lchflags(td, uap)
 	struct thread *td;
 	register struct lchflags_args /* {
 		char *path;
@@ -2832,7 +2832,7 @@ struct fchflags_args {
 };
 #endif
 int
-fchflags(td, uap)
+sys_fchflags(td, uap)
 	struct thread *td;
 	register struct fchflags_args /* {
 		int fd;
@@ -2899,7 +2899,7 @@ struct chmod_args {
 };
 #endif
 int
-chmod(td, uap)
+sys_chmod(td, uap)
 	struct thread *td;
 	register struct chmod_args /* {
 		char *path;
@@ -2919,7 +2919,7 @@ struct fchmodat_args {
 }
 #endif
 int
-fchmodat(struct thread *td, struct fchmodat_args *uap)
+sys_fchmodat(struct thread *td, struct fchmodat_args *uap)
 {
 	int flag = uap->flag;
 	int fd = uap->fd;
@@ -2949,7 +2949,7 @@ struct lchmod_args {
 };
 #endif
 int
-lchmod(td, uap)
+sys_lchmod(td, uap)
 	struct thread *td;
 	register struct lchmod_args /* {
 		char *path;
@@ -2995,7 +2995,7 @@ struct fchmod_args {
 };
 #endif
 int
-fchmod(struct thread *td, struct fchmod_args *uap)
+sys_fchmod(struct thread *td, struct fchmod_args *uap)
 {
 	struct file *fp;
 	int error;
@@ -3054,7 +3054,7 @@ struct chown_args {
 };
 #endif
 int
-chown(td, uap)
+sys_chown(td, uap)
 	struct thread *td;
 	register struct chown_args /* {
 		char *path;
@@ -3076,7 +3076,7 @@ struct fchownat_args {
 };
 #endif
 int
-fchownat(struct thread *td, struct fchownat_args *uap)
+sys_fchownat(struct thread *td, struct fchownat_args *uap)
 {
 	int flag;
 
@@ -3129,7 +3129,7 @@ struct lchown_args {
 };
 #endif
 int
-lchown(td, uap)
+sys_lchown(td, uap)
 	struct thread *td;
 	register struct lchown_args /* {
 		char *path;
@@ -3161,7 +3161,7 @@ struct fchown_args {
 };
 #endif
 int
-fchown(td, uap)
+sys_fchown(td, uap)
 	struct thread *td;
 	register struct fchown_args /* {
 		int fd;
@@ -3268,7 +3268,7 @@ struct utimes_args {
 };
 #endif
 int
-utimes(td, uap)
+sys_utimes(td, uap)
 	struct thread *td;
 	register struct utimes_args /* {
 		char *path;
@@ -3288,7 +3288,7 @@ struct futimesat_args {
 };
 #endif
 int
-futimesat(struct thread *td, struct futimesat_args *uap)
+sys_futimesat(struct thread *td, struct futimesat_args *uap)
 {
 
 	return (kern_utimesat(td, uap->fd, uap->path, UIO_USERSPACE,
@@ -3336,7 +3336,7 @@ struct lutimes_args {
 };
 #endif
 int
-lutimes(td, uap)
+sys_lutimes(td, uap)
 	struct thread *td;
 	register struct lutimes_args /* {
 		char *path;
@@ -3380,7 +3380,7 @@ struct futimes_args {
 };
 #endif
 int
-futimes(td, uap)
+sys_futimes(td, uap)
 	struct thread *td;
 	register struct futimes_args /* {
 		int  fd;
@@ -3429,7 +3429,7 @@ struct truncate_args {
 };
 #endif
 int
-truncate(td, uap)
+sys_truncate(td, uap)
 	struct thread *td;
 	register struct truncate_args /* {
 		char *path;
@@ -3509,7 +3509,7 @@ otruncate(td, uap)
 
 	nuap.path = uap->path;
 	nuap.length = uap->length;
-	return (truncate(td, &nuap));
+	return (sys_truncate(td, &nuap));
 }
 #endif /* COMPAT_43 */
 
@@ -3521,7 +3521,7 @@ freebsd6_truncate(struct thread *td, struct freebsd6_truncate_args *uap)
 
 	ouap.path = uap->path;
 	ouap.length = uap->length;
-	return (truncate(td, &ouap));
+	return (sys_truncate(td, &ouap));
 }
 
 int
@@ -3531,7 +3531,7 @@ freebsd6_ftruncate(struct thread *td, struct freebsd6_ftruncate_args *uap)
 
 	ouap.fd = uap->fd;
 	ouap.length = uap->length;
-	return (ftruncate(td, &ouap));
+	return (sys_ftruncate(td, &ouap));
 }
 
 /*
@@ -3543,7 +3543,7 @@ struct fsync_args {
 };
 #endif
 int
-fsync(td, uap)
+sys_fsync(td, uap)
 	struct thread *td;
 	struct fsync_args /* {
 		int fd;
@@ -3597,7 +3597,7 @@ struct rename_args {
 };
 #endif
 int
-rename(td, uap)
+sys_rename(td, uap)
 	struct thread *td;
 	register struct rename_args /* {
 		char *from;
@@ -3617,7 +3617,7 @@ struct renameat_args {
 };
 #endif
 int
-renameat(struct thread *td, struct renameat_args *uap)
+sys_renameat(struct thread *td, struct renameat_args *uap)
 {
 
 	return (kern_renameat(td, uap->oldfd, uap->old, uap->newfd, uap->new,
@@ -3753,7 +3753,7 @@ struct mkdir_args {
 };
 #endif
 int
-mkdir(td, uap)
+sys_mkdir(td, uap)
 	struct thread *td;
 	register struct mkdir_args /* {
 		char *path;
@@ -3772,7 +3772,7 @@ struct mkdirat_args {
 };
 #endif
 int
-mkdirat(struct thread *td, struct mkdirat_args *uap)
+sys_mkdirat(struct thread *td, struct mkdirat_args *uap)
 {
 
 	return (kern_mkdirat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode));
@@ -3860,7 +3860,7 @@ struct rmdir_args {
 };
 #endif
 int
-rmdir(td, uap)
+sys_rmdir(td, uap)
 	struct thread *td;
 	struct rmdir_args /* {
 		char *path;
@@ -4109,7 +4109,7 @@ struct getdirentries_args {
 };
 #endif
 int
-getdirentries(td, uap)
+sys_getdirentries(td, uap)
 	struct thread *td;
 	register struct getdirentries_args /* {
 		int fd;
@@ -4212,7 +4212,7 @@ struct getdents_args {
 };
 #endif
 int
-getdents(td, uap)
+sys_getdents(td, uap)
 	struct thread *td;
 	register struct getdents_args /* {
 		int fd;
@@ -4225,7 +4225,7 @@ getdents(td, uap)
 	ap.buf = uap->buf;
 	ap.count = uap->count;
 	ap.basep = NULL;
-	return (getdirentries(td, &ap));
+	return (sys_getdirentries(td, &ap));
 }
 
 /*
@@ -4237,7 +4237,7 @@ struct umask_args {
 };
 #endif
 int
-umask(td, uap)
+sys_umask(td, uap)
 	struct thread *td;
 	struct umask_args /* {
 		int newmask;
@@ -4263,7 +4263,7 @@ struct revoke_args {
 };
 #endif
 int
-revoke(td, uap)
+sys_revoke(td, uap)
 	struct thread *td;
 	register struct revoke_args /* {
 		char *path;
@@ -4361,7 +4361,7 @@ struct lgetfh_args {
 };
 #endif
 int
-lgetfh(td, uap)
+sys_lgetfh(td, uap)
 	struct thread *td;
 	register struct lgetfh_args *uap;
 {
@@ -4400,7 +4400,7 @@ struct getfh_args {
 };
 #endif
 int
-getfh(td, uap)
+sys_getfh(td, uap)
 	struct thread *td;
 	register struct getfh_args *uap;
 {
@@ -4446,7 +4446,7 @@ struct fhopen_args {
 };
 #endif
 int
-fhopen(td, uap)
+sys_fhopen(td, uap)
 	struct thread *td;
 	struct fhopen_args /* {
 		const struct fhandle *u_fhp;
@@ -4637,7 +4637,7 @@ struct fhstat_args {
 };
 #endif
 int
-fhstat(td, uap)
+sys_fhstat(td, uap)
 	struct thread *td;
 	register struct fhstat_args /* {
 		struct fhandle *u_fhp;
@@ -4685,7 +4685,7 @@ struct fhstatfs_args {
 };
 #endif
 int
-fhstatfs(td, uap)
+sys_fhstatfs(td, uap)
 	struct thread *td;
 	struct fhstatfs_args /* {
 		struct fhandle *u_fhp;
@@ -4840,7 +4840,7 @@ kern_posix_fallocate(struct thread *td, int fd, off_t offset, off_t len)
 }
 
 int
-posix_fallocate(struct thread *td, struct posix_fallocate_args *uap)
+sys_posix_fallocate(struct thread *td, struct posix_fallocate_args *uap)
 {
 
 	return (kern_posix_fallocate(td, uap->fd, uap->offset, uap->len));
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 92fb0d9335f7..17dc5e7bfb68 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -1356,7 +1356,7 @@ vn_rlimit_fsize(const struct vnode *vp, const struct uio *uio,
 	PROC_LOCK(td->td_proc);
 	if ((uoff_t)uio->uio_offset + uio->uio_resid >
 	    lim_cur(td->td_proc, RLIMIT_FSIZE)) {
-		psignal(td->td_proc, SIGXFSZ);
+		kern_psignal(td->td_proc, SIGXFSZ);
 		PROC_UNLOCK(td->td_proc);
 		return (EFBIG);
 	}
diff --git a/sys/kgssapi/gss_impl.c b/sys/kgssapi/gss_impl.c
index ef6c0410d59e..d66b4a95a2c9 100644
--- a/sys/kgssapi/gss_impl.c
+++ b/sys/kgssapi/gss_impl.c
@@ -86,7 +86,7 @@ kgss_uninit(void *dummy)
 SYSUNINIT(kgss_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, kgss_uninit, NULL);
 
 int
-gssd_syscall(struct thread *td, struct gssd_syscall_args *uap)
+sys_gssd_syscall(struct thread *td, struct gssd_syscall_args *uap)
 {
         struct sockaddr_un sun;
         struct netconfig *nconf;
diff --git a/sys/kgssapi/krb5/krb5_mech.c b/sys/kgssapi/krb5/krb5_mech.c
index 798a80dd1ef8..71f804e64d02 100644
--- a/sys/kgssapi/krb5/krb5_mech.c
+++ b/sys/kgssapi/krb5/krb5_mech.c
@@ -288,6 +288,7 @@ get_keys(struct krb5_context *kc)
 	case ETYPE_DES3_CBC_SHA1:
 	case ETYPE_OLD_DES3_CBC_SHA1:
 		etype = ETYPE_DES3_CBC_SHA1;
+		break;
 
 	default:
 		etype = keydata->kk_type;
diff --git a/sys/libkern/strnlen.c b/sys/libkern/strnlen.c
new file mode 100644
index 000000000000..805aee8483ff
--- /dev/null
+++ b/sys/libkern/strnlen.c
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2009 David Schultz <das@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/libkern.h>
+
+size_t
+strnlen(const char *s, size_t maxlen)
+{
+	size_t len;
+
+	for (len = 0; len < maxlen; len++, s++) {
+		if (!*s)
+			break;
+	}
+	return (len);
+}
diff --git a/sys/mips/atheros/if_arge.c b/sys/mips/atheros/if_arge.c
index 92cc161e329a..567b9ba6f904 100644
--- a/sys/mips/atheros/if_arge.c
+++ b/sys/mips/atheros/if_arge.c
@@ -1199,9 +1199,9 @@ arge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc->arge_miibus);
 	ARGE_LOCK(sc);
 	mii_pollstat(mii);
-	ARGE_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	ARGE_UNLOCK(sc);
 }
 
 struct arge_dmamap_arg {
diff --git a/sys/mips/cavium/asm_octeon.S b/sys/mips/cavium/asm_octeon.S
index 1ef508334897..94ac875a1996 100644
--- a/sys/mips/cavium/asm_octeon.S
+++ b/sys/mips/cavium/asm_octeon.S
@@ -50,12 +50,12 @@ LEAF(octeon_ap_wait)
 	jal	platform_processor_id
 	nop
 
-1:	lld	t0, octeon_ap_boot
+1:	ll	t0, octeon_ap_boot
 	bne	v0, t0, 1b
 	nop
 
 	move	t0, zero
-	scd	t0, octeon_ap_boot
+	sc	t0, octeon_ap_boot
 
 	beqz	t0, 1b
 	nop
diff --git a/sys/mips/cavium/if_octm.c b/sys/mips/cavium/if_octm.c
index a753d663752a..847ad3c0fd88 100644
--- a/sys/mips/cavium/if_octm.c
+++ b/sys/mips/cavium/if_octm.c
@@ -64,8 +64,6 @@
 #include <contrib/octeon-sdk/cvmx-interrupt.h>
 #include <contrib/octeon-sdk/cvmx-mgmt-port.h>
 
-extern cvmx_bootinfo_t *octeon_bootinfo;
-
 struct octm_softc {
 	struct ifnet *sc_ifp;
 	device_t sc_dev;
@@ -179,7 +177,7 @@ octm_attach(device_t dev)
 	 * Set MAC address for this management port.
 	 */
 	mac = 0;
-	memcpy((u_int8_t *)&mac + 2, octeon_bootinfo->mac_addr_base, 6);
+	memcpy((u_int8_t *)&mac + 2, cvmx_sysinfo_get()->mac_addr_base, 6);
 	mac += sc->sc_port;
 	cvmx_mgmt_port_set_mac(sc->sc_port, mac);
 
diff --git a/sys/mips/cavium/octe/ethernet-common.c b/sys/mips/cavium/octe/ethernet-common.c
index 73103cc4725a..839b6f82ecee 100644
--- a/sys/mips/cavium/octe/ethernet-common.c
+++ b/sys/mips/cavium/octe/ethernet-common.c
@@ -45,7 +45,6 @@ __FBSDID("$FreeBSD$");
 #include "ethernet-headers.h"
 
 extern int octeon_is_simulation(void);
-extern cvmx_bootinfo_t *octeon_bootinfo;
 
 
 /**
@@ -270,12 +269,12 @@ void cvm_oct_common_poll(struct ifnet *ifp)
 int cvm_oct_common_init(struct ifnet *ifp)
 {
 	char mac[6] = {
-		octeon_bootinfo->mac_addr_base[0],
-		octeon_bootinfo->mac_addr_base[1],
-		octeon_bootinfo->mac_addr_base[2],
-		octeon_bootinfo->mac_addr_base[3],
-		octeon_bootinfo->mac_addr_base[4],
-		octeon_bootinfo->mac_addr_base[5] };
+		cvmx_sysinfo_get()->mac_addr_base[0],
+		cvmx_sysinfo_get()->mac_addr_base[1],
+		cvmx_sysinfo_get()->mac_addr_base[2],
+		cvmx_sysinfo_get()->mac_addr_base[3],
+		cvmx_sysinfo_get()->mac_addr_base[4],
+		cvmx_sysinfo_get()->mac_addr_base[5] };
 	cvm_oct_private_t *priv = (cvm_oct_private_t *)ifp->if_softc;
 
 	mac[5] += cvm_oct_mac_addr_offset++;
diff --git a/sys/mips/cavium/octe/ethernet.c b/sys/mips/cavium/octe/ethernet.c
index 50b91fcf0a7f..a3a5ae79d3fb 100644
--- a/sys/mips/cavium/octe/ethernet.c
+++ b/sys/mips/cavium/octe/ethernet.c
@@ -76,12 +76,6 @@ TUNABLE_INT("hw.octe.pow_receive_group", &pow_receive_group);
 
 extern int octeon_is_simulation(void);
 
-/**
- * Exported from the kernel so we can determine board information. It is
- * passed by the bootloader to the kernel.
- */
-extern cvmx_bootinfo_t *octeon_bootinfo;
-
 /**
  * Periodic timer to check auto negotiation
  */
@@ -475,7 +469,7 @@ int cvm_oct_init_module(device_t bus)
 
 	if (INTERRUPT_LIMIT) {
 		/* Set the POW timer rate to give an interrupt at most INTERRUPT_LIMIT times per second */
-		cvmx_write_csr(CVMX_POW_WQ_INT_PC, octeon_bootinfo->eclock_hz/(INTERRUPT_LIMIT*16*256)<<8);
+		cvmx_write_csr(CVMX_POW_WQ_INT_PC, cvmx_clock_get_rate(CVMX_CLOCK_CORE)/(INTERRUPT_LIMIT*16*256)<<8);
 
 		/* Enable POW timer interrupt. It will count when there are packets available */
 		cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1ful<<24);
diff --git a/sys/mips/cavium/octeon_ebt3000_cf.c b/sys/mips/cavium/octeon_ebt3000_cf.c
index f5a44c4def55..ae7bb7e7fbba 100644
--- a/sys/mips/cavium/octeon_ebt3000_cf.c
+++ b/sys/mips/cavium/octeon_ebt3000_cf.c
@@ -100,9 +100,6 @@ __FBSDID("$FreeBSD$");
 #define SWAP_SHORT(x)		((x << 8) | (x >> 8))
 #define MODEL_STR_SIZE		40
 
-/* XXX */
-extern cvmx_bootinfo_t *octeon_bootinfo;
-
 /* Globals */
 /*
  * There's three bus types supported by this driver.
@@ -484,7 +481,12 @@ static int cf_cmd_identify (void)
 	drive_param.sec_track   =  SWAP_SHORT (drive_param.u.driveid.current_sectors);
 	drive_param.nr_sectors  = (uint32_t)SWAP_SHORT (drive_param.u.driveid.lba_size_1) |
 	    ((uint32_t)SWAP_SHORT (drive_param.u.driveid.lba_size_2));
-	printf("cf0: <%s> %lld sectors\n", drive_param.model, (long long)drive_param.nr_sectors);
+	if (bootverbose) {
+		printf("    model %s\n", drive_param.model);
+		printf("    heads %d tracks %d sec_tracks %d sectors %d\n",
+			drive_param.heads, drive_param.tracks,
+			drive_param.sec_track, drive_param.nr_sectors);
+	}
 
 	return (0);
 }
@@ -578,7 +580,10 @@ static int cf_wait_busy (void)
 		}
 		break;
 	}
-	if ((status & STATUS_DRQ) == 0) {
+
+	/* DRQ is only for when read data is actually available; check BSY */
+	/* Some vendors do assert DRQ, but not all. Check BSY instead. */
+	if (status & STATUS_BSY) {
 		printf("%s: device not ready (status=%x)\n", __func__, status);
 		return (ENXIO);
 	}
@@ -634,24 +639,25 @@ static int cf_probe (device_t dev)
  * inserted.
  *
  */
-typedef unsigned long long llu;
 static void cf_identify (driver_t *drv, device_t parent)
 {
-        int bus_region;
+	int bus_region;
 	int count = 0;
-        cvmx_mio_boot_reg_cfgx_t cfg;
+	cvmx_mio_boot_reg_cfgx_t cfg;
+
+	uint64_t phys_base = cvmx_sysinfo_get()->compact_flash_common_base_addr;
 
     	if (octeon_is_simulation())
 		return;
 
-	base_addr = cvmx_phys_to_ptr(octeon_bootinfo->compact_flash_common_base_addr);
+	base_addr = cvmx_phys_to_ptr(phys_base);
 
         for (bus_region = 0; bus_region < 8; bus_region++)
         {
                 cfg.u64 = cvmx_read_csr(CVMX_MIO_BOOT_REG_CFGX(bus_region));
-                if (cfg.s.base == octeon_bootinfo->compact_flash_common_base_addr >> 16)
+                if (cfg.s.base == phys_base >> 16)
                 {
-			if (octeon_bootinfo->compact_flash_attribute_base_addr == 0)
+			if (cvmx_sysinfo_get()->compact_flash_attribute_base_addr == 0)
 				bus_type = CF_TRUE_IDE_8;
 			else
 				bus_type = (cfg.s.width) ? CF_16 : CF_8;
diff --git a/sys/mips/cavium/octeon_mp.c b/sys/mips/cavium/octeon_mp.c
index 2bc268e95064..95839779ccaa 100644
--- a/sys/mips/cavium/octeon_mp.c
+++ b/sys/mips/cavium/octeon_mp.c
@@ -43,11 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <contrib/octeon-sdk/cvmx.h>
 #include <contrib/octeon-sdk/cvmx-interrupt.h>
 
-/* XXX */
-extern cvmx_bootinfo_t *octeon_bootinfo;
-
-/* NOTE: this 64-bit mask (and many others) limits MAXCPU to 64 */
-uint64_t octeon_ap_boot = ~0ULL;
+unsigned octeon_ap_boot = ~0;
 
 void
 platform_ipi_send(int cpuid)
@@ -106,7 +102,7 @@ platform_init_ap(int cpuid)
 void
 platform_cpu_mask(cpuset_t *mask)
 {
-	uint64_t core_mask = octeon_bootinfo->core_mask;
+	uint64_t core_mask = cvmx_sysinfo_get()->core_mask;
 	uint64_t i, m;
 
 	CPU_ZERO(mask);
@@ -139,11 +135,11 @@ platform_start_ap(int cpuid)
 	    DELAY(2000);    /* Give it a moment to start */
 	}
 
-	if (atomic_cmpset_64(&octeon_ap_boot, ~0, cpuid) == 0)
+	if (atomic_cmpset_32(&octeon_ap_boot, ~0, cpuid) == 0)
 		return (-1);
 	for (;;) {
 		DELAY(1000);
-		if (atomic_cmpset_64(&octeon_ap_boot, 0, ~0) != 0)
+		if (atomic_cmpset_32(&octeon_ap_boot, 0, ~0) != 0)
 			return (0);
 		printf("Waiting for cpu%d to start\n", cpuid);
 	}
diff --git a/sys/mips/cavium/octeon_pcmap_regs.h b/sys/mips/cavium/octeon_pcmap_regs.h
index f9ee4dfd30fa..61ae38be527a 100644
--- a/sys/mips/cavium/octeon_pcmap_regs.h
+++ b/sys/mips/cavium/octeon_pcmap_regs.h
@@ -297,4 +297,9 @@ extern int octeon_is_simulation(void);
  */
 #define  OCTEON_CHAR_LED_BASE_ADDR	(0x1d020000 | (0x1ffffffffull << 31))
 
+/*
+ * Default FLASH device (physical) base address
+ */
+#define  OCTEON_FLASH_BASE_ADDR		(0x1d040000ull)
+
 #endif /* !OCTEON_PCMAP_REGS_H__ */
diff --git a/sys/mips/conf/XLP b/sys/mips/conf/XLP
index ff40239a3041..286f01fdccf9 100644
--- a/sys/mips/conf/XLP
+++ b/sys/mips/conf/XLP
@@ -93,3 +93,8 @@ device		pci
 
 # Network
 device		ether
+#
+# FDT support 
+#options         FDT
+#options         FDT_DTB_STATIC
+#makeoptions     FDT_DTS_FILE=xlp-basic.dts
diff --git a/sys/mips/conf/XLP64 b/sys/mips/conf/XLP64
index 1d4b575f3a45..0b964a526498 100644
--- a/sys/mips/conf/XLP64
+++ b/sys/mips/conf/XLP64
@@ -95,3 +95,8 @@ device		pci
 
 # Network
 device		ether
+
+# FDT support 
+#options         FDT
+#options         FDT_DTB_STATIC
+#makeoptions     FDT_DTS_FILE=xlp-basic.dts
diff --git a/sys/mips/idt/if_kr.c b/sys/mips/idt/if_kr.c
index 43b8df593783..243968dc297f 100644
--- a/sys/mips/idt/if_kr.c
+++ b/sys/mips/idt/if_kr.c
@@ -945,9 +945,9 @@ kr_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 	mii = device_get_softc(sc->kr_miibus);
 	KR_LOCK(sc);
 	mii_pollstat(mii);
-	KR_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	KR_UNLOCK(sc);
 }
 
 struct kr_dmamap_arg {
diff --git a/sys/mips/include/bus.h b/sys/mips/include/bus.h
index 1745b0feab09..868b22ec1270 100644
--- a/sys/mips/include/bus.h
+++ b/sys/mips/include/bus.h
@@ -724,6 +724,7 @@ extern bus_space_tag_t mips_bus_space_generic;
 #if defined(CPU_RMI) || defined (CPU_NLM)
 extern bus_space_tag_t rmi_bus_space;
 extern bus_space_tag_t rmi_pci_bus_space;
+extern bus_space_tag_t rmi_uart_bus_space;
 #endif
 
 #include <machine/bus_dma.h>
diff --git a/sys/mips/include/fdt.h b/sys/mips/include/fdt.h
new file mode 100644
index 000000000000..3b20a72dfc03
--- /dev/null
+++ b/sys/mips/include/fdt.h
@@ -0,0 +1,57 @@
+/*-
+ * Copyright (c) 2010 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Semihalf under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_FDT_H_
+#define _MACHINE_FDT_H_
+
+#include <machine/bus.h>
+#include <machine/intr_machdep.h>
+
+/* Max interrupt number */
+#if defined(CPU_RMI) || defined(CPU_NLM)
+#define FDT_INTR_MAX	XLR_MAX_INTR
+#else
+#define FDT_INTR_MAX	(NHARD_IRQS + NSOFT_IRQS)
+#endif
+
+/* Map phandle/intpin pair to global IRQ number */ 
+#define	FDT_MAP_IRQ(node, pin)	(pin)
+
+/*
+ * Bus space tag. XXX endianess info needs to be derived from the blob.
+ */
+#if defined(CPU_RMI) || defined(CPU_NLM)
+#define fdtbus_bs_tag	rmi_uart_bus_space
+#else
+#define fdtbus_bs_tag	NULL
+#endif
+
+#endif /* _MACHINE_FDT_H_ */
diff --git a/sys/mips/include/ieeefp.h b/sys/mips/include/ieeefp.h
index c7d9244ae420..fb6c21a47c43 100644
--- a/sys/mips/include/ieeefp.h
+++ b/sys/mips/include/ieeefp.h
@@ -11,6 +11,8 @@
 #ifndef _MACHINE_IEEEFP_H_
 #define	_MACHINE_IEEEFP_H_
 
+/* Deprecated historical FPU control interface */
+
 typedef int fp_except;
 typedef int fp_except_t;
 
diff --git a/sys/mips/include/intr_machdep.h b/sys/mips/include/intr_machdep.h
index f224517645cf..75190f52499c 100644
--- a/sys/mips/include/intr_machdep.h
+++ b/sys/mips/include/intr_machdep.h
@@ -29,6 +29,8 @@
 #ifndef	_MACHINE_INTR_MACHDEP_H_
 #define	_MACHINE_INTR_MACHDEP_H_
 
+#include <machine/atomic.h>
+
 #if defined(CPU_RMI) || defined(CPU_NLM)
 #define XLR_MAX_INTR 64 
 #else
diff --git a/sys/mips/include/md_var.h b/sys/mips/include/md_var.h
index c2a61559ff1a..6f65a0f8f248 100644
--- a/sys/mips/include/md_var.h
+++ b/sys/mips/include/md_var.h
@@ -56,6 +56,7 @@ void MipsSwitchFPState(struct thread *, struct trapframe *);
 u_long	kvtop(void *addr);
 int	is_cacheable_mem(vm_paddr_t addr);
 void	mips_generic_reset(void);
+void	mips_wait(void);
 
 #define	MIPS_DEBUG   0
 
diff --git a/sys/mips/include/ofw_machdep.h b/sys/mips/include/ofw_machdep.h
new file mode 100644
index 000000000000..35afce1ed9c0
--- /dev/null
+++ b/sys/mips/include/ofw_machdep.h
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_OFW_MACHDEP_H_
+#define _MACHINE_OFW_MACHDEP_H_
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/rman.h>
+#include <sys/bus.h>
+#include <dev/ofw/openfirm.h>
+
+typedef	uint32_t	cell_t;
+struct mem_region {
+	vm_offset_t	mr_start;
+	vm_size_t	mr_size;
+};
+
+
+int  OF_decode_addr(phandle_t, int, bus_space_tag_t *, bus_space_handle_t *);
+void OF_getetheraddr(device_t dev, u_char *addr);
+void OF_initial_setup(void *fdt_ptr, void *junk, int (*openfirm)(void *));
+
+#endif /* _MACHINE_OFW_MACHDEP_H_ */
diff --git a/sys/mips/include/param.h b/sys/mips/include/param.h
index 5f25c81c98f9..608d80288d0d 100644
--- a/sys/mips/include/param.h
+++ b/sys/mips/include/param.h
@@ -149,7 +149,6 @@
 
 #define	MAXPAGESIZES		1		/* max supported pagesizes */
 
-#define	BLKDEV_IOSIZE		2048		/* xxx: Why is this 1/2 page? */
 #define	MAXDUMPPGS		1		/* xxx: why is this only one? */
 
 /*
diff --git a/sys/mips/include/proc.h b/sys/mips/include/proc.h
index 11a1f8e17983..8b575ddb53fb 100644
--- a/sys/mips/include/proc.h
+++ b/sys/mips/include/proc.h
@@ -67,11 +67,21 @@ struct mdproc {
 	/* empty */
 };
 
+#ifdef _KERNEL
 struct thread;
 
 void	mips_cpu_switch(struct thread *, struct thread *, struct mtx *);
 void	mips_cpu_throw(struct thread *, struct thread *);
 
+struct syscall_args {
+	u_int code;
+	struct sysent *callp;
+	register_t args[8];
+	int narg;
+	struct trapframe *trapframe;
+};
+#endif
+
 #ifdef __mips_n64
 #define	KINFO_PROC_SIZE 1088
 #else
diff --git a/sys/mips/mips/elf64_machdep.c b/sys/mips/mips/elf64_machdep.c
index 9fa31fa31bbc..ee25ef403cac 100644
--- a/sys/mips/mips/elf64_machdep.c
+++ b/sys/mips/mips/elf64_machdep.c
@@ -80,8 +80,8 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_maxssiz	= NULL,
 	.sv_flags	= SV_ABI_FREEBSD | SV_LP64,
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
-	.sv_fetch_syscall_args = NULL, /* XXXKIB */
-	.sv_syscallnames = NULL,
+	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
+	.sv_syscallnames = syscallnames,
 	.sv_schedtail	= NULL,
 };
 
diff --git a/sys/mips/mips/elf_machdep.c b/sys/mips/mips/elf_machdep.c
index 41611e34dcf6..85ada0be68dc 100644
--- a/sys/mips/mips/elf_machdep.c
+++ b/sys/mips/mips/elf_machdep.c
@@ -80,7 +80,7 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_maxssiz	= NULL,
 	.sv_flags	= SV_ABI_FREEBSD | SV_LP64,
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
-	.sv_fetch_syscall_args = NULL, /* XXXKIB */
+	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
 	.sv_schedtail	= NULL,
 };
@@ -136,7 +136,7 @@ struct sysentvec elf32_freebsd_sysvec = {
 	.sv_maxssiz	= NULL,
 	.sv_flags	= SV_ABI_FREEBSD | SV_ILP32,
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
-	.sv_fetch_syscall_args = NULL, /* XXXKIB */
+	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
 	.sv_schedtail	= NULL,
 };
diff --git a/sys/mips/mips/exception.S b/sys/mips/mips/exception.S
index 729391e1efcf..8b7307c7cdb2 100644
--- a/sys/mips/mips/exception.S
+++ b/sys/mips/mips/exception.S
@@ -557,6 +557,33 @@ NNON_LEAF(MipsUserGenException, CALLFRAME_SIZ, ra)
 	.set	at
 END(MipsUserGenException)
 
+	.set	push
+	.set	noat
+NON_LEAF(mips_wait, CALLFRAME_SIZ, ra)
+	PTR_SUBU        sp, sp, CALLFRAME_SIZ
+	.mask   0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ)
+	REG_S   ra, CALLFRAME_RA(sp)		# save RA
+	mfc0	t0, MIPS_COP_0_STATUS
+	xori	t1, t0, MIPS_SR_INT_IE
+	mtc0	t1, MIPS_COP_0_STATUS
+	COP0_SYNC
+	jal	sched_runnable
+	nop
+	REG_L   ra, CALLFRAME_RA(sp)
+	mfc0	t0, MIPS_COP_0_STATUS
+	ori	t1, t0, MIPS_SR_INT_IE
+	.align 4
+GLOBAL(MipsWaitStart)			# this is 16 byte aligned
+	mtc0	t1, MIPS_COP_0_STATUS
+	bnez	v0, MipsWaitEnd
+	nop
+	wait
+GLOBAL(MipsWaitEnd)			# MipsWaitStart + 16
+	jr	ra
+	PTR_ADDU        sp, sp, CALLFRAME_SIZ
+END(mips_wait)
+	.set	pop
+
 /*----------------------------------------------------------------------------
  *
  * MipsKernIntr --
@@ -578,6 +605,19 @@ NNON_LEAF(MipsKernIntr, KERN_EXC_FRAME_SIZE, ra)
 	.set	noat
 	PTR_SUBU	sp, sp, KERN_EXC_FRAME_SIZE
 	.mask	0x80000000, (CALLFRAME_RA - KERN_EXC_FRAME_SIZE)
+
+/*
+ * Check for getting interrupts just before wait
+ */
+	MFC0	k0, MIPS_COP_0_EXC_PC
+	ori	k0, 0xf
+	xori	k0, 0xf			# 16 byte align
+	PTR_LA	k1, MipsWaitStart
+	bne	k0, k1, 1f
+	nop
+	PTR_ADDU k1, 16			# skip over wait
+	MTC0	k1, MIPS_COP_0_EXC_PC
+1:
 /*
  *  Save CPU state, building 'frame'.
  */
diff --git a/sys/mips/mips/machdep.c b/sys/mips/mips/machdep.c
index f7e52485b7a3..405d457b966b 100644
--- a/sys/mips/mips/machdep.c
+++ b/sys/mips/mips/machdep.c
@@ -163,6 +163,9 @@ extern char MipsTLBMiss[], MipsTLBMissEnd[];
 /* Cache error handler */
 extern char MipsCache[], MipsCacheEnd[];
 
+/* MIPS wait skip region */
+extern char MipsWaitStart[], MipsWaitEnd[];
+
 extern char edata[], end[];
 #ifdef DDB
 extern vm_offset_t ksym_start, ksym_end;
@@ -326,6 +329,12 @@ struct msgbuf *msgbufp=0;
 void
 mips_vector_init(void)
 {
+	/*
+	 * Make sure that the Wait region logic is not been 
+	 * changed
+	 */
+	if (MipsWaitEnd - MipsWaitStart != 16)
+		panic("startup: MIPS wait region not correct");
 	/*
 	 * Copy down exception vector code.
 	 */
@@ -497,7 +506,7 @@ cpu_idle(int busy)
 		critical_enter();
 		cpu_idleclock();
 	}
-	__asm __volatile ("wait");
+	mips_wait();
 	if (!busy) {
 		cpu_activeclock();
 		critical_exit();
diff --git a/sys/mips/mips/mem.c b/sys/mips/mips/mem.c
index 7529a9ed5038..d40c424888af 100644
--- a/sys/mips/mips/mem.c
+++ b/sys/mips/mips/mem.c
@@ -87,6 +87,7 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 
 	GIANT_REQUIRED;
 
+	pmap_page_init(&m);
 	while (uio->uio_resid > 0 && !error) {
 		iov = uio->uio_iov;
 		if (iov->iov_len == 0) {
diff --git a/sys/mips/mips/pm_machdep.c b/sys/mips/mips/pm_machdep.c
index 7e80e2f7634b..d730ccb829d5 100644
--- a/sys/mips/mips/pm_machdep.c
+++ b/sys/mips/mips/pm_machdep.c
@@ -213,7 +213,7 @@ cpu_thread_siginfo(int sig, u_long code, siginfo_t *si)
  * context left by sendsig.
  */
 int
-sigreturn(struct thread *td, struct sigreturn_args *uap)
+sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
 {
 	struct trapframe *regs;
 	ucontext_t *ucp;
diff --git a/sys/mips/mips/trap.c b/sys/mips/mips/trap.c
index c800e715254a..97374a7698af 100644
--- a/sys/mips/mips/trap.c
+++ b/sys/mips/mips/trap.c
@@ -261,6 +261,133 @@ static int emulate_unaligned_access(struct trapframe *frame, int mode);
 
 extern void fswintrberr(void); /* XXX */
 
+int
+cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
+{
+	struct trapframe *locr0 = td->td_frame;
+	struct sysentvec *se;
+	int error, nsaved;
+
+	bzero(sa->args, sizeof(sa->args));
+
+	/* compute next PC after syscall instruction */
+	td->td_pcb->pcb_tpc = sa->trapframe->pc; /* Remember if restart */
+	if (DELAYBRANCH(sa->trapframe->cause))	 /* Check BD bit */
+		locr0->pc = MipsEmulateBranch(locr0, sa->trapframe->pc, 0, 0);
+	else
+		locr0->pc += sizeof(int);
+	sa->code = locr0->v0;
+
+	switch (sa->code) {
+#if defined(__mips_n32) || defined(__mips_n64)
+	case SYS___syscall:
+		/*
+		 * Quads fit in a single register in
+		 * new ABIs.
+		 *
+		 * XXX o64?
+		 */
+#endif
+	case SYS_syscall:
+		/*
+		 * Code is first argument, followed by
+		 * actual args.
+		 */
+		sa->code = locr0->a0;
+		sa->args[0] = locr0->a1;
+		sa->args[1] = locr0->a2;
+		sa->args[2] = locr0->a3;
+		nsaved = 3;
+#if defined(__mips_n32) || defined(__mips_n64)
+		sa->args[3] = locr0->t4;
+		sa->args[4] = locr0->t5;
+		sa->args[5] = locr0->t6;
+		sa->args[6] = locr0->t7;
+		nsaved += 4;
+#endif
+		break;
+
+#if defined(__mips_o32)
+	case SYS___syscall:
+		/*
+		 * Like syscall, but code is a quad, so as
+		 * to maintain quad alignment for the rest
+		 * of the arguments.
+		 */
+		if (_QUAD_LOWWORD == 0)
+			sa->code = locr0->a0;
+		else
+			sa->code = locr0->a1;
+		sa->args[0] = locr0->a2;
+		sa->args[1] = locr0->a3;
+		nsaved = 2;
+		break;
+#endif
+
+	default:
+		sa->args[0] = locr0->a0;
+		sa->args[1] = locr0->a1;
+		sa->args[2] = locr0->a2;
+		sa->args[3] = locr0->a3;
+		nsaved = 4;
+#if defined (__mips_n32) || defined(__mips_n64)
+		sa->args[4] = locr0->t4;
+		sa->args[5] = locr0->t5;
+		sa->args[6] = locr0->t6;
+		sa->args[7] = locr0->t7;
+		nsaved += 4;
+#endif
+		break;
+	}
+#ifdef TRAP_DEBUG
+	if (trap_debug)
+		printf("SYSCALL #%d pid:%u\n", code, p->p_pid);
+#endif
+
+	se = td->td_proc->p_sysent;
+	if (se->sv_mask)
+		sa->code &= se->sv_mask;
+
+	if (sa->code >= se->sv_size)
+		sa->callp = &se->sv_table[0];
+	else
+		sa->callp = &se->sv_table[sa->code];
+
+	sa->narg = sa->callp->sy_narg;
+
+	if (sa->narg > nsaved) {
+#if defined(__mips_n32) || defined(__mips_n64)
+		/*
+		 * XXX
+		 * Is this right for new ABIs?  I think the 4 there
+		 * should be 8, size there are 8 registers to skip,
+		 * not 4, but I'm not certain.
+		 */
+		printf("SYSCALL #%u pid:%u, nargs > nsaved.\n", sa->code,
+		    td->td_proc->p_pid);
+#endif
+		error = copyin((caddr_t)(intptr_t)(locr0->sp +
+		    4 * sizeof(register_t)), (caddr_t)&sa->args[nsaved],
+		   (u_int)(sa->narg - nsaved) * sizeof(register_t));
+		if (error != 0) {
+			locr0->v0 = error;
+			locr0->a3 = 1;
+		}
+	} else
+		error = 0;
+
+	if (error == 0) {
+		td->td_retval[0] = 0;
+		td->td_retval[1] = locr0->v1;
+	}
+
+	return (error);
+}
+
+#undef __FBSDID
+#define __FBSDID(x)
+#include "../../kern/subr_syscall.c"
+
 /*
  * Handle an exception.
  * Called from MipsKernGenException() or MipsUserGenException()
@@ -527,177 +654,19 @@ trap(struct trapframe *trapframe)
 
 	case T_SYSCALL + T_USER:
 		{
-			struct trapframe *locr0 = td->td_frame;
-			struct sysent *callp;
-			unsigned int code;
-			int nargs, nsaved;
-			register_t args[8];
+			struct syscall_args sa;
+			int error;
 
-			bzero(args, sizeof args);
-
-			/*
-			 * note: PCPU_LAZY_INC() can only be used if we can
-			 * afford occassional inaccuracy in the count.
-			 */
-			PCPU_LAZY_INC(cnt.v_syscall);
-			if (td->td_ucred != p->p_ucred)
-				cred_update_thread(td);
-#ifdef KSE
-			if (p->p_flag & P_SA)
-				thread_user_enter(td);
-#endif
-			/* compute next PC after syscall instruction */
-			td->td_pcb->pcb_tpc = trapframe->pc;	/* Remember if restart */
-			if (DELAYBRANCH(trapframe->cause)) {	/* Check BD bit */
-				locr0->pc = MipsEmulateBranch(locr0, trapframe->pc, 0,
-				    0);
-			} else {
-				locr0->pc += sizeof(int);
-			}
-			code = locr0->v0;
-
-			switch (code) {
-#if defined(__mips_n32) || defined(__mips_n64)
-			case SYS___syscall:
-				/*
-				 * Quads fit in a single register in
-				 * new ABIs.
-				 *
-				 * XXX o64?
-				 */
-#endif
-			case SYS_syscall:
-				/*
-				 * Code is first argument, followed by
-				 * actual args.
-				 */
-				code = locr0->a0;
-				args[0] = locr0->a1;
-				args[1] = locr0->a2;
-				args[2] = locr0->a3;
-				nsaved = 3;
-#if defined(__mips_n32) || defined(__mips_n64)
-				args[3] = locr0->t4;
-				args[4] = locr0->t5;
-				args[5] = locr0->t6;
-				args[6] = locr0->t7;
-				nsaved += 4;
-#endif
-				break;
-
-#if defined(__mips_o32)
-			case SYS___syscall:
-				/*
-				 * Like syscall, but code is a quad, so as
-				 * to maintain quad alignment for the rest
-				 * of the arguments.
-				 */
-				if (_QUAD_LOWWORD == 0) {
-					code = locr0->a0;
-				} else {
-					code = locr0->a1;
-				}
-				args[0] = locr0->a2;
-				args[1] = locr0->a3;
-				nsaved = 2;
-				break;
-#endif
-
-			default:
-				args[0] = locr0->a0;
-				args[1] = locr0->a1;
-				args[2] = locr0->a2;
-				args[3] = locr0->a3;
-				nsaved = 4;
-#if defined (__mips_n32) || defined(__mips_n64)
-				args[4] = locr0->t4;
-				args[5] = locr0->t5;
-				args[6] = locr0->t6;
-				args[7] = locr0->t7;
-				nsaved += 4;
-#endif
-			}
-#ifdef TRAP_DEBUG
-			if (trap_debug) {
-				printf("SYSCALL #%d pid:%u\n", code, p->p_pid);
-			}
-#endif
-
-			if (p->p_sysent->sv_mask)
-				code &= p->p_sysent->sv_mask;
-
-			if (code >= p->p_sysent->sv_size)
-				callp = &p->p_sysent->sv_table[0];
-			else
-				callp = &p->p_sysent->sv_table[code];
-
-			nargs = callp->sy_narg;
-
-			if (nargs > nsaved) {
-#if defined(__mips_n32) || defined(__mips_n64)
-				/*
-				 * XXX
-				 * Is this right for new ABIs?  I think the 4 there
-				 * should be 8, size there are 8 registers to skip,
-				 * not 4, but I'm not certain.
-				 */
-				printf("SYSCALL #%u pid:%u, nargs > nsaved.\n", code, p->p_pid);
-#endif
-				i = copyin((caddr_t)(intptr_t)(locr0->sp +
-				    4 * sizeof(register_t)), (caddr_t)&args[nsaved],
-				    (u_int)(nargs - nsaved) * sizeof(register_t));
-				if (i) {
-					locr0->v0 = i;
-					locr0->a3 = 1;
-#ifdef KTRACE
-					if (KTRPOINT(td, KTR_SYSCALL))
-						ktrsyscall(code, nargs, args);
-#endif
-					goto done;
-				}
-			}
-#ifdef TRAP_DEBUG
-			if (trap_debug) {
-				for (i = 0; i < nargs; i++) {
-					printf("args[%d] = %#jx\n", i, (intmax_t)args[i]);
-				}
-			}
-#endif
-#ifdef SYSCALL_TRACING
-			printf("%s(", syscallnames[code]);
-			for (i = 0; i < nargs; i++) {
-				printf("%s%#jx", i == 0 ? "" : ", ", (intmax_t)args[i]);
-			}
-			printf(")\n");
-#endif
-#ifdef KTRACE
-			if (KTRPOINT(td, KTR_SYSCALL))
-				ktrsyscall(code, nargs, args);
-#endif
-			td->td_retval[0] = 0;
-			td->td_retval[1] = locr0->v1;
+			sa.trapframe = trapframe;
+			error = syscallenter(td, &sa);
 
 #if !defined(SMP) && (defined(DDB) || defined(DEBUG))
 			if (trp == trapdebug)
-				trapdebug[TRAPSIZE - 1].code = code;
+				trapdebug[TRAPSIZE - 1].code = sa.code;
 			else
-				trp[-1].code = code;
+				trp[-1].code = sa.code;
 #endif
-			STOPEVENT(p, S_SCE, nargs);
-
-			PTRACESTOP_SC(p, td, S_PT_SCE);
-			i = (*callp->sy_call) (td, args);
-#if 0
-			/*
-			 * Reinitialize proc pointer `p' as it may be
-			 * different if this is a child returning from fork
-			 * syscall.
-			 */
-			td = curthread;
-			locr0 = td->td_frame;
-#endif
-			trapdebug_enter(locr0, -code);
-			cpu_set_syscall_retval(td, i);
+			trapdebug_enter(td->td_frame, -sa.code);
 
 			/*
 			 * The sync'ing of I & D caches for SYS_ptrace() is
@@ -705,38 +674,7 @@ trap(struct trapframe *trapframe)
 			 * instead of being done here under a special check
 			 * for SYS_ptrace().
 			 */
-	done:
-			/*
-			 * Check for misbehavior.
-			 */
-			WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
-			    (code >= 0 && code < SYS_MAXSYSCALL) ?
-			    syscallnames[code] : "???");
-			KASSERT(td->td_critnest == 0,
-			    ("System call %s returning in a critical section",
-			    (code >= 0 && code < SYS_MAXSYSCALL) ?
-			    syscallnames[code] : "???"));
-			KASSERT(td->td_locks == 0,
-			    ("System call %s returning with %d locks held",
-			    (code >= 0 && code < SYS_MAXSYSCALL) ?
-			    syscallnames[code] : "???",
-			    td->td_locks));
-			userret(td, trapframe);
-#ifdef KTRACE
-			if (KTRPOINT(td, KTR_SYSRET))
-				ktrsysret(code, i, td->td_retval[0]);
-#endif
-			/*
-			 * This works because errno is findable through the
-			 * register set.  If we ever support an emulation
-			 * where this is not the case, this code will need
-			 * to be revisited.
-			 */
-			STOPEVENT(p, S_SCX, code);
-
-			PTRACESTOP_SC(p, td, S_PT_SCX);
-
-			mtx_assert(&Giant, MA_NOTOWNED);
+			syscallret(td, error, &sa);
 			return (trapframe->pc);
 		}
 
diff --git a/sys/mips/nlm/bus_space_rmi.c b/sys/mips/nlm/bus_space_rmi.c
index 6bb3fff7f8fb..cbeaa5b6296e 100644
--- a/sys/mips/nlm/bus_space_rmi.c
+++ b/sys/mips/nlm/bus_space_rmi.c
@@ -686,3 +686,87 @@ rmi_bus_space_barrier(void *tag __unused, bus_space_handle_t bsh __unused,
     bus_size_t offset __unused, bus_size_t len __unused, int flags)
 {
 }
+
+/*
+ * need a special bus space for this, because the Netlogic SoC
+ * UART allows only 32 bit access to its registers
+ */
+
+static u_int8_t
+rmi_uart_bus_space_read_1(void *tag, bus_space_handle_t handle,
+    bus_size_t offset)
+{
+	return (u_int8_t)(*(volatile u_int32_t *)(handle + offset));
+}
+
+static void
+rmi_uart_bus_space_write_1(void *tag, bus_space_handle_t handle,
+    bus_size_t offset, u_int8_t value)
+{
+	*(volatile u_int32_t *)(handle + offset) =  value;
+}
+
+static struct bus_space local_rmi_uart_bus_space = {
+	/* cookie */
+	(void *)0,
+
+	/* mapping/unmapping */
+	rmi_bus_space_map,
+	rmi_bus_space_unmap,
+	rmi_bus_space_subregion,
+
+	/* allocation/deallocation */
+	NULL,
+	NULL,
+
+	/* barrier */
+	rmi_bus_space_barrier,
+
+	/* read (single) */
+	rmi_uart_bus_space_read_1, NULL, NULL, NULL,
+
+	/* read multiple */
+	NULL, NULL, NULL, NULL,
+
+	/* read region */
+	NULL, NULL, NULL, NULL,
+
+	/* write (single) */
+	rmi_uart_bus_space_write_1, NULL, NULL, NULL,
+
+	/* write multiple */
+	NULL, NULL, NULL, NULL,
+
+	/* write region */
+	NULL, NULL, NULL, NULL,
+
+	/* set multiple */
+	NULL, NULL, NULL, NULL,
+
+	/* set region */
+	NULL, NULL, NULL, NULL,
+
+	/* copy */
+	NULL, NULL, NULL, NULL,
+
+	/* read (single) stream */
+	NULL, NULL, NULL, NULL,
+
+	/* read multiple stream */
+	NULL, NULL, NULL, NULL,
+
+	/* read region stream */
+	NULL, NULL, NULL, NULL,
+
+	/* write (single) stream */
+	NULL, NULL, NULL, NULL,
+
+	/* write multiple stream */
+	NULL, NULL, NULL, NULL,
+
+	/* write region stream */
+	NULL, NULL, NULL, NULL,
+};
+
+/* generic bus_space tag */
+bus_space_tag_t rmi_uart_bus_space = &local_rmi_uart_bus_space;
diff --git a/sys/mips/nlm/cms.c b/sys/mips/nlm/cms.c
index 043d5884e072..a991add85023 100644
--- a/sys/mips/nlm/cms.c
+++ b/sys/mips/nlm/cms.c
@@ -82,7 +82,7 @@ static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS];
 static struct proc *msgring_proc;	/* all threads are under a proc */
 
 /*
- * The device drivers can register a handler for the the messages sent
+ * The device drivers can register a handler for the messages sent
  * from a station (corresponding to the device).
  */
 struct tx_stn_handler {
diff --git a/sys/mips/nlm/uart_cpu_xlp.c b/sys/mips/nlm/uart_cpu_xlp.c
index 0e476d58c06e..2e6daf45f87e 100644
--- a/sys/mips/nlm/uart_cpu_xlp.c
+++ b/sys/mips/nlm/uart_cpu_xlp.c
@@ -34,6 +34,9 @@
 /*
  * XLRMIPS: This file is hacked from arm/...
  */
+#include "opt_platform.h"
+
+#ifndef	FDT			/* use FDT uart when fdt is enable */
 #include "opt_uart.h"
 
 #include <sys/cdefs.h>
@@ -60,26 +63,6 @@ __FBSDID("$FreeBSD$");
 bus_space_tag_t uart_bus_space_io;
 bus_space_tag_t uart_bus_space_mem;
 
-/*
- * need a special bus space for this, because the Netlogic SoC
- * UART allows only 32 bit access to its registers
- */
-static struct bus_space nlm_uart_bussp;
-
-static u_int8_t
-nlm_uart_bussp_read_1(void *tag, bus_space_handle_t handle,
-    bus_size_t offset)
-{
-	return (u_int8_t)(*(volatile u_int32_t *)(handle + offset));
-}
-
-static void
-nlm_uart_bussp_write_1(void *tag, bus_space_handle_t handle,
-    bus_size_t offset, u_int8_t value)
-{
-	*(volatile u_int32_t *)(handle + offset) =  value;
-}
-
 int
 uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2)
 {
@@ -89,14 +72,9 @@ uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2)
 int
 uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 {
-	/* Create custom bus space */
-	memcpy(&nlm_uart_bussp, rmi_bus_space, sizeof(nlm_uart_bussp));
-	nlm_uart_bussp.bs_r_1 = nlm_uart_bussp_read_1;
-	nlm_uart_bussp.bs_w_1 = nlm_uart_bussp_write_1;
-
 	di->ops = uart_getops(&uart_ns8250_class);
 	di->bas.chan = 0;
-	di->bas.bst = &nlm_uart_bussp;
+	di->bas.bst = rmi_uart_bus_space;
 	di->bas.bsh = nlm_get_uart_regbase(0, 0);
 	
 	di->bas.regshft = 2;
@@ -108,6 +86,7 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 	di->parity = UART_PARITY_NONE;
 
 	uart_bus_space_io = NULL;
-	uart_bus_space_mem = &nlm_uart_bussp;
+	uart_bus_space_mem = rmi_uart_bus_space;
 	return (0);
 }
+#endif
diff --git a/sys/mips/nlm/xlp_machdep.c b/sys/mips/nlm/xlp_machdep.c
index 30430fad53ca..5724df01e587 100644
--- a/sys/mips/nlm/xlp_machdep.c
+++ b/sys/mips/nlm/xlp_machdep.c
@@ -31,6 +31,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
+#include "opt_platform.h"
 
 #include <sys/param.h>
 #include <sys/bus.h>
@@ -85,11 +86,14 @@ __FBSDID("$FreeBSD$");
 #include <mips/nlm/board.h>
 #include <mips/nlm/xlp.h>
 
+#ifdef FDT
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#endif
+
 /* 4KB static data aread to keep a copy of the bootload env until
    the dynamic kenv is setup */
 char boot1_env[4096];
-int xlp_argc;
-char **xlp_argv, **xlp_envp;
 
 uint64_t xlp_cpu_frequency;
 uint64_t xlp_io_base = MIPS_PHYS_TO_KSEG1(XLP_DEFAULT_IO_BASE);
@@ -147,8 +151,13 @@ xlp_parse_mmu_options(void)
 	uint32_t cpu_map = xlp_hw_thread_mask;
 	uint32_t core0_thr_mask, core_thr_mask;
 
-#ifndef SMP /* Uniprocessor! */
-	if (cpu_map != 0x1) {
+#ifdef SMP
+	if (cpu_map == 0)
+		cpu_map = 0xffffffff;
+#else /* Uniprocessor! */
+	if (cpu_map == 0)
+		cpu_map = 0x1;
+	else if (cpu_map != 0x1) {
 		printf("WARNING: Starting uniprocessor kernel on cpumask [0x%lx]!\n"
 		    "WARNING: Other CPUs will be unused.\n", (u_long)cpu_map);
 		cpu_map = 0x1;
@@ -219,45 +228,107 @@ xlp_parse_mmu_options(void)
 	return;
 }
 
-static void 
-xlp_set_boot_flags(void)
+/* Parse cmd line args as env - copied from ar71xx */
+static void
+xlp_parse_bootargs(char *cmdline)
 {
-	char *p;
+	char *n, *v;
 
-	p = getenv("bootflags");
-	if (p == NULL)
-		return;
-
-	for (; p && *p != '\0'; p++) {
-		switch (*p) {
-		case 'd':
-		case 'D':
-			boothowto |= RB_KDB;
-			break;
-		case 'g':
-		case 'G':
-			boothowto |= RB_GDB;
-			break;
-		case 'v':
-		case 'V':
-			boothowto |= RB_VERBOSE;
-			break;
-
-		case 's':	/* single-user (default, supported for sanity) */
-		case 'S':
-			boothowto |= RB_SINGLE;
-			break;
-
-		default:
-			printf("Unrecognized boot flag '%c'.\n", *p);
-			break;
+	while ((v = strsep(&cmdline, " \n")) != NULL) {
+		if (*v == '\0')
+			continue;
+		if (*v == '-') {
+			while (*v != '\0') {
+				v++;
+				switch (*v) {
+				case 'a': boothowto |= RB_ASKNAME; break;
+				case 'd': boothowto |= RB_KDB; break;
+				case 'g': boothowto |= RB_GDB; break;
+				case 's': boothowto |= RB_SINGLE; break;
+				case 'v': boothowto |= RB_VERBOSE; break;
+				}
+			}
+		} else {
+			n = strsep(&v, "=");
+			if (v == NULL)
+				setenv(n, "1");
+			else
+				setenv(n, v);
 		}
 	}
-
-	freeenv(p);
-	return;
 }
 
+#ifdef FDT
+static void
+xlp_bootargs_init(__register_t arg)
+{
+	char	buf[2048]; /* early stack is big enough */
+	void	*dtbp;
+	phandle_t chosen;
+	ihandle_t mask;
+
+	dtbp = (void *)arg;
+#if defined(FDT_DTB_STATIC)
+	/*
+	 * In case the device tree blob was not passed as argument try
+	 * to use the statically embedded one.
+	 */
+	if (dtbp == NULL)
+		dtbp = &fdt_static_dtb;
+#endif
+	if (OF_install(OFW_FDT, 0) == FALSE)
+		while (1);
+	if (OF_init((void *)dtbp) != 0)
+		while (1);
+	if (fdt_immr_addr(xlp_io_base) != 0)
+		while (1);
+	OF_interpret("perform-fixup", 0);
+
+	chosen = OF_finddevice("/chosen");
+	if (OF_getprop(chosen, "cpumask", &mask, sizeof(mask)) == 0)
+		xlp_hw_thread_mask = mask;
+
+	if (OF_getprop(chosen, "bootargs", buf, sizeof(buf)) == 0)
+		xlp_parse_bootargs(buf);
+}
+#else
+/*
+ * arg is a pointer to the environment block, the format of the block is
+ * a=xyz\0b=pqr\0\0
+ */
+static void
+xlp_bootargs_init(__register_t arg)
+{
+	char	buf[2048]; /* early stack is big enough */
+	char	*p, *v, *n;
+	uint32_t mask;
+
+	p = (void *)(intptr_t)arg;
+	while (*p != '\0') {
+		strlcpy(buf, p, sizeof(buf));
+		v = buf;
+		n = strsep(&v, "=");
+		if (v == NULL)
+			setenv(n, "1");
+		else
+			setenv(n, v);
+		p += strlen(p) + 1;
+	}
+
+	/* CPU mask can be passed thru env */
+	if (getenv_uint("cpumask", &mask) != 0)
+		xlp_hw_thread_mask = mask;
+
+	/* command line argument */
+	v = getenv("bootargs");
+	if (v != NULL) {
+		strlcpy(buf, v, sizeof(buf));
+		xlp_parse_bootargs(buf);
+		freeenv(v);
+	}
+}
+#endif
+
 static void
 mips_init(void)
 {
@@ -428,14 +499,6 @@ platform_start(__register_t a0 __unused,
     __register_t a2 __unused,
     __register_t a3 __unused)
 {
-	int i;
-
-	xlp_argc = 1;
-	/*
-	 * argv and envp are passed in array of 32bit pointers
-	 */
-	xlp_argv = NULL;
-	xlp_envp = NULL;
 
 	/* Initialize pcpu stuff */
 	mips_pcpu0_init();
@@ -443,10 +506,8 @@ platform_start(__register_t a0 __unused,
 	/* initialize console so that we have printf */
 	boothowto |= (RB_SERIAL | RB_MULTIPLE);	/* Use multiple consoles */
 
-	/* For now */
-	boothowto |= RB_VERBOSE;
-	boothowto |= RB_SINGLE;
-	bootverbose++;
+	init_static_kenv(boot1_env, sizeof(boot1_env));
+	xlp_bootargs_init(a0);
 
 	/* clockrate used by delay, so initialize it here */
 	xlp_cpu_frequency = xlp_get_cpu_frequency();
@@ -456,32 +517,10 @@ platform_start(__register_t a0 __unused,
 	/* Init console please */
 	cninit();
 
-	/* Environment */
-	printf("Args %#jx %#jx %#jx %#jx:\n", (intmax_t)a0,
-	    (intmax_t)a1, (intmax_t)a2, (intmax_t)a3);
-	xlp_hw_thread_mask = a0;
-	init_static_kenv(boot1_env, sizeof(boot1_env));
-	printf("Environment (from %d args):\n", xlp_argc - 1);
-	if (xlp_argc == 1)
-		printf("\tNone\n");
-	for (i = 1; i < xlp_argc; i++) {
-		char *n, *arg;
-
-		arg = (char *)(intptr_t)xlp_argv[i];
-		printf("\t%s\n", arg);
-		n = strsep(&arg, "=");
-		if (arg == NULL)
-			setenv(n, "1");
-		else
-			setenv(n, arg);
-	}
-
 	/* Early core init and fixes for errata */
 	xlp_setup_core();
 
-	xlp_set_boot_flags();
 	xlp_parse_mmu_options();
-	
 	xlp_mem_init();
 
 	bcopy(XLPResetEntry, (void *)MIPS_RESET_EXC_VEC,
@@ -491,6 +530,7 @@ platform_start(__register_t a0 __unused,
 	 * MIPS generic init 
 	 */
 	mips_init();
+
 	/*
 	 * XLP specific post initialization
  	 * initialize other on chip stuff
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index ca8586c5c12f..f51b560735bc 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -303,6 +303,7 @@ SUBDIR=	${_3dfx} \
 	trm \
 	${_twa} \
 	twe \
+	tws \
 	tx \
 	txp \
 	uart \
diff --git a/sys/modules/fxp/Makefile b/sys/modules/fxp/Makefile
index b80c3ec43558..283b638386a8 100644
--- a/sys/modules/fxp/Makefile
+++ b/sys/modules/fxp/Makefile
@@ -3,6 +3,6 @@
 .PATH: ${.CURDIR}/../../dev/fxp
 
 KMOD=	if_fxp
-SRCS=	if_fxp.c device_if.h bus_if.h pci_if.h miibus_if.h
+SRCS=	device_if.h bus_if.h if_fxp.c inphy.c miibus_if.h miidevs.h pci_if.h
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/ipfw/Makefile b/sys/modules/ipfw/Makefile
index ec9b3cc49c1a..60ab84858c70 100644
--- a/sys/modules/ipfw/Makefile
+++ b/sys/modules/ipfw/Makefile
@@ -8,7 +8,7 @@ KMOD=	ipfw
 SRCS=	ip_fw2.c ip_fw_pfil.c
 SRCS+=	ip_fw_dynamic.c ip_fw_log.c
 SRCS+=	ip_fw_sockopt.c ip_fw_table.c
-SRCS+=	opt_inet6.h opt_ipfw.h opt_ipsec.h
+SRCS+=	opt_inet.h opt_inet6.h opt_ipfw.h opt_ipsec.h
 
 CFLAGS+= -DIPFIREWALL
 CFLAGS+= -I${.CURDIR}/../../contrib/pf
@@ -22,6 +22,10 @@ CFLAGS+= -I${.CURDIR}/../../contrib/pf
 #
 
 .if !defined(KERNBUILDDIR)
+.if ${MK_INET_SUPPORT} != "no"
+opt_inet.h:
+	echo "#define INET 1" > ${.TARGET}
+.endif
 .if ${MK_INET6_SUPPORT} != "no"
 opt_inet6.h:
 	echo "#define INET6 1" > ${.TARGET}
diff --git a/sys/modules/mii/Makefile b/sys/modules/mii/Makefile
index 2dd9811c639f..b0cfc12811f2 100644
--- a/sys/modules/mii/Makefile
+++ b/sys/modules/mii/Makefile
@@ -5,10 +5,10 @@
 KMOD=	miibus
 SRCS=	acphy.c amphy.c atphy.c axphy.c bmtphy.c brgphy.c bus_if.h
 SRCS+=	ciphy.c device_if.h
-SRCS+=	e1000phy.c exphy.c gentbi.c icsphy.c inphy.c ip1000phy.c jmphy.c
+SRCS+=	e1000phy.c gentbi.c icsphy.c ip1000phy.c jmphy.c
 SRCS+=	lxtphy.c miibus_if.c miibus_if.h mii.c miidevs.h mii_physubr.c
 SRCS+=	mlphy.c nsgphy.c nsphy.c nsphyter.c pci_if.h pnaphy.c qsphy.c
-SRCS+=	rdcphy.c rgephy.c rlphy.c ruephy.c tdkphy.c tlphy.c truephy.c
+SRCS+=	rdcphy.c rgephy.c rlphy.c tdkphy.c tlphy.c truephy.c
 SRCS+=	ukphy.c ukphy_subr.c
 SRCS+=	xmphy.c
 
diff --git a/sys/modules/tws/Makefile b/sys/modules/tws/Makefile
new file mode 100644
index 000000000000..af26579279ec
--- /dev/null
+++ b/sys/modules/tws/Makefile
@@ -0,0 +1,10 @@
+# Makefile for tws (LSI 3ware 9750 SAS2/SATA-II RAID PCIe) driver
+# $FreeBSD$
+
+KMOD=	tws
+.PATH:	${.CURDIR}/../../dev/${KMOD}
+ 
+SRCS=	tws.c tws_services.c tws_cam.c tws_hdm.c tws_user.c
+SRCS+=	device_if.h bus_if.h pci_if.h opt_cam.h opt_scsi.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/usb/rue/Makefile b/sys/modules/usb/rue/Makefile
index 258df6f37401..e7e236ee851d 100644
--- a/sys/modules/usb/rue/Makefile
+++ b/sys/modules/usb/rue/Makefile
@@ -30,8 +30,7 @@ S=     ${.CURDIR}/../../..
 .PATH: $S/dev/usb/net
 
 KMOD=	if_rue
-SRCS=	opt_bus.h opt_usb.h device_if.h bus_if.h usb_if.h usbdevs.h \
-	miibus_if.h opt_inet.h \
-	if_rue.c
+SRCS=	bus_if.h device_if.h miibus_if.h miidevs.h if_rue.c opt_bus.h
+SRCS+=	opt_inet.h opt_usb.h ruephy.c usb_if.h usbdevs.h
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/xl/Makefile b/sys/modules/xl/Makefile
index 3898ad49f89e..417f3fcc2710 100644
--- a/sys/modules/xl/Makefile
+++ b/sys/modules/xl/Makefile
@@ -3,7 +3,6 @@
 .PATH: ${.CURDIR}/../../dev/xl
 
 KMOD=	if_xl
-SRCS=	if_xl.c device_if.h bus_if.h pci_if.h
-SRCS+=	miibus_if.h
+SRCS=	bus_if.h device_if.h if_xl.c miibus_if.h miidevs.h pci_if.h xlphy.c
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/zfs/Makefile b/sys/modules/zfs/Makefile
index 21a6ad0a9664..122ce446c8e2 100644
--- a/sys/modules/zfs/Makefile
+++ b/sys/modules/zfs/Makefile
@@ -82,7 +82,6 @@ CFLAGS+=-I${SUNW}/uts/common
 CFLAGS+=-I${.CURDIR}/../..
 CFLAGS+=-I${SUNW}/common/zfs
 CFLAGS+=-I${SUNW}/common
-CFLAGS+=-I${.CURDIR}/../../../include
 CFLAGS+=-DBUILDING_ZFS
 
 .if ${MACHINE_ARCH} == "powerpc64"
diff --git a/sys/net/if.h b/sys/net/if.h
index d1f3883e1edd..4f2dc6f62452 100644
--- a/sys/net/if.h
+++ b/sys/net/if.h
@@ -43,9 +43,11 @@
 /*
  * <net/if.h> does not depend on <sys/time.h> on most other systems.  This
  * helps userland compatibility.  (struct timeval ifi_lastchange)
+ * The same holds for <sys/socket.h>.  (struct sockaddr ifru_addr)
  */
 #ifndef _KERNEL
 #include <sys/time.h>
+#include <sys/socket.h>
 #endif
 
 struct ifnet;
diff --git a/sys/net/if_llatbl.h b/sys/net/if_llatbl.h
index 4b1835332368..2ae27927362d 100644
--- a/sys/net/if_llatbl.h
+++ b/sys/net/if_llatbl.h
@@ -59,6 +59,7 @@ struct llentry {
 	struct rwlock		 lle_lock;
 	struct lltable		 *lle_tbl;
 	struct llentries	 *lle_head;
+	void			(*lle_free)(struct lltable *, struct llentry *);
 	struct mbuf		 *la_hold;
 	int     		 la_numheld;  /* # of packets currently held */
 	time_t			 la_expire;
diff --git a/sys/net/if_tap.c b/sys/net/if_tap.c
index 08c669ab3b89..4ae5a3dc8254 100644
--- a/sys/net/if_tap.c
+++ b/sys/net/if_tap.c
@@ -132,7 +132,7 @@ static struct filterops	tap_write_filterops = {
 
 static struct cdevsw	tap_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_PSEUDO | D_NEEDMINOR,
+	.d_flags =	D_NEEDMINOR,
 	.d_open =	tapopen,
 	.d_close =	tapclose,
 	.d_read =	tapread,
diff --git a/sys/net/if_tun.c b/sys/net/if_tun.c
index c5328848c7b7..57a11f977151 100644
--- a/sys/net/if_tun.c
+++ b/sys/net/if_tun.c
@@ -165,7 +165,7 @@ static struct filterops tun_write_filterops = {
 
 static struct cdevsw tun_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_PSEUDO | D_NEEDMINOR,
+	.d_flags =	D_NEEDMINOR,
 	.d_open =	tunopen,
 	.d_close =	tunclose,
 	.d_read =	tunread,
diff --git a/sys/net/radix.h b/sys/net/radix.h
index 0dc11e98746d..5bacaa3ade8d 100644
--- a/sys/net/radix.h
+++ b/sys/net/radix.h
@@ -105,6 +105,8 @@ typedef int walktree_f_t(struct radix_node *, void *);
 
 struct radix_node_head {
 	struct	radix_node *rnh_treetop;
+	u_int	rnh_gen;		/* generation counter */
+	int	rnh_multipath;		/* multipath capable ? */
 	int	rnh_addrsize;		/* permit, but not require fixed keys */
 	int	rnh_pktsize;		/* permit, but not require fixed keys */
 	struct	radix_node *(*rnh_addaddr)	/* add based on sockaddr */
@@ -131,8 +133,6 @@ struct radix_node_head {
 	void	(*rnh_close)	/* do something when the last ref drops */
 		(struct radix_node *rn, struct radix_node_head *head);
 	struct	radix_node rnh_nodes[3];	/* empty tree for common case */
-	int	rnh_multipath;			/* multipath capable ? */
-	u_int	rnh_spare;			/* route caching */
 #ifdef _KERNEL
 	struct	rwlock rnh_lock;		/* locks entire radix tree */
 #endif
diff --git a/sys/net/raw_cb.h b/sys/net/raw_cb.h
index 35b546c52ecd..1b347e02a8dd 100644
--- a/sys/net/raw_cb.h
+++ b/sys/net/raw_cb.h
@@ -70,9 +70,14 @@ pr_init_t	raw_init;
  * Library routines for raw socket usrreq functions; will always be wrapped
  * so that protocol-specific functions can be handled.
  */
+typedef int (*raw_input_cb_fn)(struct mbuf *, struct sockproto *,
+    struct sockaddr *, struct rawcb *);
+
 int	 raw_attach(struct socket *, int);
 void	 raw_detach(struct rawcb *);
 void	 raw_input(struct mbuf *, struct sockproto *, struct sockaddr *);
+void	 raw_input_ext(struct mbuf *, struct sockproto *, struct sockaddr *,
+	    raw_input_cb_fn);
 
 /*
  * Generic pr_usrreqs entries for raw socket protocols, usually wrapped so
diff --git a/sys/net/raw_usrreq.c b/sys/net/raw_usrreq.c
index 2de3d692163a..df77063863fc 100644
--- a/sys/net/raw_usrreq.c
+++ b/sys/net/raw_usrreq.c
@@ -70,6 +70,14 @@ raw_init(void)
  */
 void
 raw_input(struct mbuf *m0, struct sockproto *proto, struct sockaddr *src)
+{
+
+	return (raw_input_ext(m0, proto, src, NULL));
+}
+
+void
+raw_input_ext(struct mbuf *m0, struct sockproto *proto, struct sockaddr *src,
+    raw_input_cb_fn cb)
 {
 	struct rawcb *rp;
 	struct mbuf *m = m0;
@@ -83,6 +91,8 @@ raw_input(struct mbuf *m0, struct sockproto *proto, struct sockaddr *src)
 		if (rp->rcb_proto.sp_protocol  &&
 		    rp->rcb_proto.sp_protocol != proto->sp_protocol)
 			continue;
+		if (cb != NULL && (*cb)(m, proto, src, rp) != 0)
+			continue;
 		if (last) {
 			struct mbuf *n;
 			n = m_copy(m, 0, (int)M_COPYALL);
diff --git a/sys/net/route.c b/sys/net/route.c
index d42b4c73b40e..85a2c840a0db 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -260,7 +260,7 @@ struct setfib_args {
 };
 #endif
 int
-setfib(struct thread *td, struct setfib_args *uap)
+sys_setfib(struct thread *td, struct setfib_args *uap)
 {
 	if (uap->fibnum < 0 || uap->fibnum >= rt_numfibs)
 		return EINVAL;
@@ -384,7 +384,7 @@ rtalloc1_fib(struct sockaddr *dst, int report, u_long ignflags,
 		 */
 		bzero(&info, sizeof(info));
 		info.rti_info[RTAX_DST] = dst;
-		rt_missmsg(msgtype, &info, 0, err);
+		rt_missmsg_fib(msgtype, &info, 0, err, fibnum);
 	}	
 done:
 	if (newrt)
@@ -609,7 +609,7 @@ rtredirect_fib(struct sockaddr *dst,
 	info.rti_info[RTAX_GATEWAY] = gateway;
 	info.rti_info[RTAX_NETMASK] = netmask;
 	info.rti_info[RTAX_AUTHOR] = src;
-	rt_missmsg(RTM_REDIRECT, &info, flags, error);
+	rt_missmsg_fib(RTM_REDIRECT, &info, flags, error, fibnum);
 	if (ifa != NULL)
 		ifa_free(ifa);
 }
@@ -1025,6 +1025,7 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 	register struct radix_node_head *rnh;
 	struct ifaddr *ifa;
 	struct sockaddr *ndst;
+	struct sockaddr_storage mdst;
 #define senderr(x) { error = x ; goto bad; }
 
 	KASSERT((fibnum < rt_numfibs), ("rtrequest1_fib: bad fibnum"));
@@ -1051,6 +1052,10 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 
 	switch (req) {
 	case RTM_DELETE:
+		if (netmask) {
+			rt_maskedcopy(dst, (struct sockaddr *)&mdst, netmask);
+			dst = (struct sockaddr *)&mdst;
+		}
 #ifdef RADIX_MPATH
 		if (rn_mpath_capable(rnh)) {
 			error = rn_mpath_update(req, info, rnh, ret_nrt);
@@ -1522,7 +1527,7 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
 			}
 			RT_ADDREF(rt);
 			RT_UNLOCK(rt);
-			rt_newaddrmsg(cmd, ifa, error, rt);
+			rt_newaddrmsg_fib(cmd, ifa, error, rt, fibnum);
 			RT_LOCK(rt);
 			RT_REMREF(rt);
 			if (cmd == RTM_DELETE) {
diff --git a/sys/net/route.h b/sys/net/route.h
index 0bc72d7fb8e1..7e045b79305f 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -49,9 +49,13 @@
 struct route {
 	struct	rtentry *ro_rt;
 	struct	llentry *ro_lle;
+	struct	in_ifaddr *ro_ia;
+	int		ro_flags;
 	struct	sockaddr ro_dst;
 };
 
+#define RT_CACHING_CONTEXT	0x1
+
 /*
  * These numbers are used by reliable protocols for determining
  * retransmission behavior and are included in the routing structure.
@@ -365,7 +369,9 @@ void	 rt_ieee80211msg(struct ifnet *, int, void *, size_t);
 void	 rt_ifannouncemsg(struct ifnet *, int);
 void	 rt_ifmsg(struct ifnet *);
 void	 rt_missmsg(int, struct rt_addrinfo *, int, int);
+void	 rt_missmsg_fib(int, struct rt_addrinfo *, int, int, int);
 void	 rt_newaddrmsg(int, struct ifaddr *, int, struct rtentry *);
+void	 rt_newaddrmsg_fib(int, struct ifaddr *, int, struct rtentry *, int);
 void	 rt_newmaddrmsg(int, struct ifmultiaddr *);
 int	 rt_setgate(struct rtentry *, struct sockaddr *, struct sockaddr *);
 void 	 rt_maskedcopy(struct sockaddr *, struct sockaddr *, struct sockaddr *);
diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c
index 999557a7082c..fdd017cd782c 100644
--- a/sys/net/rtsock.c
+++ b/sys/net/rtsock.c
@@ -122,6 +122,13 @@ MALLOC_DEFINE(M_RTABLE, "routetbl", "routing tables");
 static struct	sockaddr route_src = { 2, PF_ROUTE, };
 static struct	sockaddr sa_zero   = { sizeof(sa_zero), AF_INET, };
 
+/*
+ * Used by rtsock/raw_input callback code to decide whether to filter the update
+ * notification to a socket bound to a particular FIB.
+ */
+#define	RTS_FILTER_FIB	M_PROTO8
+#define	RTS_ALLFIBS	-1
+
 static struct {
 	int	ip_count;	/* attached w/ AF_INET */
 	int	ip6_count;	/* attached w/ AF_INET6 */
@@ -196,6 +203,31 @@ rts_init(void)
 }
 SYSINIT(rtsock, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, rts_init, 0);
 
+static int
+raw_input_rts_cb(struct mbuf *m, struct sockproto *proto, struct sockaddr *src,
+    struct rawcb *rp)
+{
+	int fibnum;
+
+	KASSERT(m != NULL, ("%s: m is NULL", __func__));
+	KASSERT(proto != NULL, ("%s: proto is NULL", __func__));
+	KASSERT(rp != NULL, ("%s: rp is NULL", __func__));
+
+	/* No filtering requested. */
+	if ((m->m_flags & RTS_FILTER_FIB) == 0)
+		return (0);
+
+	/* Check if it is a rts and the fib matches the one of the socket. */
+	fibnum = M_GETFIB(m);
+	if (proto->sp_family != PF_ROUTE ||
+	    rp->rcb_socket == NULL ||
+	    rp->rcb_socket->so_fibnum == fibnum)
+		return (0);
+
+	/* Filtering requested and no match, the socket shall be skipped. */
+	return (1);
+}
+
 static void
 rts_input(struct mbuf *m)
 {
@@ -212,7 +244,7 @@ rts_input(struct mbuf *m)
 	} else
 		route_proto.sp_protocol = 0;
 
-	raw_input(m, &route_proto, &route_src);
+	raw_input_ext(m, &route_proto, &route_src, raw_input_rts_cb);
 }
 
 /*
@@ -885,6 +917,8 @@ route_output(struct mbuf *m, struct socket *so)
 			m_adj(m, rtm->rtm_msglen - m->m_pkthdr.len);
 	}
 	if (m) {
+		M_SETFIB(m, so->so_fibnum);
+		m->m_flags |= RTS_FILTER_FIB;
 		if (rp) {
 			/*
 			 * XXX insure we don't get a copy by
@@ -1127,7 +1161,8 @@ rt_msg2(int type, struct rt_addrinfo *rtinfo, caddr_t cp, struct walkarg *w)
  * destination.
  */
 void
-rt_missmsg(int type, struct rt_addrinfo *rtinfo, int flags, int error)
+rt_missmsg_fib(int type, struct rt_addrinfo *rtinfo, int flags, int error,
+    int fibnum)
 {
 	struct rt_msghdr *rtm;
 	struct mbuf *m;
@@ -1138,6 +1173,14 @@ rt_missmsg(int type, struct rt_addrinfo *rtinfo, int flags, int error)
 	m = rt_msg1(type, rtinfo);
 	if (m == NULL)
 		return;
+
+	if (fibnum != RTS_ALLFIBS) {
+		KASSERT(fibnum >= 0 && fibnum < rt_numfibs, ("%s: fibnum out "
+		    "of range 0 <= %d < %d", __func__, fibnum, rt_numfibs));
+		M_SETFIB(m, fibnum);
+		m->m_flags |= RTS_FILTER_FIB;
+	}
+
 	rtm = mtod(m, struct rt_msghdr *);
 	rtm->rtm_flags = RTF_DONE | flags;
 	rtm->rtm_errno = error;
@@ -1145,6 +1188,13 @@ rt_missmsg(int type, struct rt_addrinfo *rtinfo, int flags, int error)
 	rt_dispatch(m, sa);
 }
 
+void
+rt_missmsg(int type, struct rt_addrinfo *rtinfo, int flags, int error)
+{
+
+	rt_missmsg_fib(type, rtinfo, flags, error, RTS_ALLFIBS);
+}
+
 /*
  * This routine is called to generate a message from the routing
  * socket indicating that the status of a network interface has changed.
@@ -1179,7 +1229,8 @@ rt_ifmsg(struct ifnet *ifp)
  * copies of it.
  */
 void
-rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt)
+rt_newaddrmsg_fib(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt,
+    int fibnum)
 {
 	struct rt_addrinfo info;
 	struct sockaddr *sa = NULL;
@@ -1237,10 +1288,24 @@ rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt)
 			rtm->rtm_errno = error;
 			rtm->rtm_addrs = info.rti_addrs;
 		}
+		if (fibnum != RTS_ALLFIBS) {
+			KASSERT(fibnum >= 0 && fibnum < rt_numfibs, ("%s: "
+			    "fibnum out of range 0 <= %d < %d", __func__,
+			     fibnum, rt_numfibs));
+			M_SETFIB(m, fibnum);
+			m->m_flags |= RTS_FILTER_FIB;
+		}
 		rt_dispatch(m, sa);
 	}
 }
 
+void
+rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt)
+{
+
+	rt_newaddrmsg_fib(cmd, ifa, error, rt, RTS_ALLFIBS);
+}
+
 /*
  * This is the analogue to the rt_newaddrmsg which performs the same
  * function but for multicast group memberhips.  This is easier since
diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c
index 8dee3f7f32d4..b4288b77bbb5 100644
--- a/sys/net80211/ieee80211_proto.c
+++ b/sys/net80211/ieee80211_proto.c
@@ -193,7 +193,7 @@ ieee80211_proto_vattach(struct ieee80211vap *vap)
 	vap->iv_rtsthreshold = IEEE80211_RTS_DEFAULT;
 	vap->iv_fragthreshold = IEEE80211_FRAG_DEFAULT;
 	vap->iv_bmiss_max = IEEE80211_BMISS_MAX;
-	callout_init(&vap->iv_swbmiss, CALLOUT_MPSAFE);
+	callout_init_mtx(&vap->iv_swbmiss, IEEE80211_LOCK_OBJ(ic), 0);
 	callout_init(&vap->iv_mgtsend, CALLOUT_MPSAFE);
 	TASK_INIT(&vap->iv_nstate_task, 0, ieee80211_newstate_cb, vap);
 	TASK_INIT(&vap->iv_swbmiss_task, 0, beacon_swmiss, vap);
@@ -1403,7 +1403,7 @@ beacon_miss(void *arg, int npending)
 	struct ieee80211com *ic = arg;
 	struct ieee80211vap *vap;
 
-	/* XXX locking */
+	IEEE80211_LOCK(ic);
 	TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
 		/*
 		 * We only pass events through for sta vap's in RUN state;
@@ -1415,18 +1415,21 @@ beacon_miss(void *arg, int npending)
 		    vap->iv_bmiss != NULL)
 			vap->iv_bmiss(vap);
 	}
+	IEEE80211_UNLOCK(ic);
 }
 
 static void
 beacon_swmiss(void *arg, int npending)
 {
 	struct ieee80211vap *vap = arg;
+	struct ieee80211com *ic = vap->iv_ic;
 
-	if (vap->iv_state != IEEE80211_S_RUN)
-		return;
-
-	/* XXX Call multiple times if npending > zero? */
-	vap->iv_bmiss(vap);
+	IEEE80211_LOCK(ic);
+	if (vap->iv_state == IEEE80211_S_RUN) {
+		/* XXX Call multiple times if npending > zero? */
+		vap->iv_bmiss(vap);
+	}
+	IEEE80211_UNLOCK(ic);
 }
 
 /*
@@ -1440,6 +1443,8 @@ ieee80211_swbmiss(void *arg)
 	struct ieee80211vap *vap = arg;
 	struct ieee80211com *ic = vap->iv_ic;
 
+	IEEE80211_LOCK_ASSERT(ic);
+
 	/* XXX sleep state? */
 	KASSERT(vap->iv_state == IEEE80211_S_RUN,
 	    ("wrong state %d", vap->iv_state));
diff --git a/sys/net80211/ieee80211_sta.c b/sys/net80211/ieee80211_sta.c
index 5444459a1529..97a9dbc39f2f 100644
--- a/sys/net80211/ieee80211_sta.c
+++ b/sys/net80211/ieee80211_sta.c
@@ -109,6 +109,8 @@ sta_beacon_miss(struct ieee80211vap *vap)
 {
 	struct ieee80211com *ic = vap->iv_ic;
 
+	IEEE80211_LOCK_ASSERT(ic);
+
 	KASSERT((ic->ic_flags & IEEE80211_F_SCAN) == 0, ("scanning"));
 	KASSERT(vap->iv_state >= IEEE80211_S_RUN,
 	    ("wrong state %s", ieee80211_state_name[vap->iv_state]));
diff --git a/sys/net80211/ieee80211_tdma.c b/sys/net80211/ieee80211_tdma.c
index 8c191ab92b52..077b9e4e5ed7 100644
--- a/sys/net80211/ieee80211_tdma.c
+++ b/sys/net80211/ieee80211_tdma.c
@@ -286,6 +286,8 @@ tdma_beacon_miss(struct ieee80211vap *vap)
 {
 	struct ieee80211_tdma_state *ts = vap->iv_tdma;
 
+	IEEE80211_LOCK_ASSERT(vap->iv_ic);
+
 	KASSERT((vap->iv_ic->ic_flags & IEEE80211_F_SCAN) == 0, ("scanning"));
 	KASSERT(vap->iv_state == IEEE80211_S_RUN,
 	    ("wrong state %d", vap->iv_state));
diff --git a/sys/netgraph/ng_ipfw.c b/sys/netgraph/ng_ipfw.c
index 4f1bc0ee94d2..117c608dfd7a 100644
--- a/sys/netgraph/ng_ipfw.c
+++ b/sys/netgraph/ng_ipfw.c
@@ -242,7 +242,7 @@ ng_ipfw_rcvdata(hook_p hook, item_p item)
 
 	if (m->m_len < sizeof(struct ip) &&
 	    (m = m_pullup(m, sizeof(struct ip))) == NULL)
-		return (EINVAL);
+		return (ENOBUFS);
 
 	ip = mtod(m, struct ip *);
 
@@ -252,18 +252,14 @@ ng_ipfw_rcvdata(hook_p hook, item_p item)
 #ifdef INET
 		case IPVERSION:
 			ip_input(m);
-			break;
+			return (0);
 #endif
 #ifdef INET6
 		case IPV6_VERSION >> 4:
 			ip6_input(m);
-			break;
+			return (0);
 #endif
-		default:
-			NG_FREE_M(m);
-			return (EINVAL);
 		}
-		return (0);
 	} else {
 		switch (ip->ip_v) {
 #ifdef INET
@@ -277,10 +273,12 @@ ng_ipfw_rcvdata(hook_p hook, item_p item)
 			return (ip6_output(m, NULL, NULL, 0, NULL,
 			    NULL, NULL));
 #endif
-		default:
-			return (EINVAL);
 		}
 	}
+
+	/* unknown IP protocol version */
+	NG_FREE_M(m);
+	return (EPROTONOSUPPORT);
 }
 
 static int
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index 4547e395eba9..c91df4a27444 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -76,11 +76,6 @@ static int	in_ifinit(struct ifnet *,
 	    struct in_ifaddr *, struct sockaddr_in *, int);
 static void	in_purgemaddrs(struct ifnet *);
 
-static VNET_DEFINE(int, subnetsarelocal);
-#define	V_subnetsarelocal		VNET(subnetsarelocal)
-SYSCTL_VNET_INT(_net_inet_ip, OID_AUTO, subnets_are_local, CTLFLAG_RW,
-	&VNET_NAME(subnetsarelocal), 0,
-	"Treat all subnets as directly connected");
 static VNET_DEFINE(int, sameprefixcarponly);
 #define	V_sameprefixcarponly		VNET(sameprefixcarponly)
 SYSCTL_VNET_INT(_net_inet_ip, OID_AUTO, same_prefix_carp_only, CTLFLAG_RW,
@@ -95,9 +90,7 @@ VNET_DECLARE(struct arpstat, arpstat);  /* ARP statistics, see if_arp.h */
 
 /*
  * Return 1 if an internet address is for a ``local'' host
- * (one to which we have a connection).  If subnetsarelocal
- * is true, this includes other subnets of the local net.
- * Otherwise, it includes only the directly-connected (sub)nets.
+ * (one to which we have a connection).
  */
 int
 in_localaddr(struct in_addr in)
@@ -106,19 +99,10 @@ in_localaddr(struct in_addr in)
 	register struct in_ifaddr *ia;
 
 	IN_IFADDR_RLOCK();
-	if (V_subnetsarelocal) {
-		TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
-			if ((i & ia->ia_netmask) == ia->ia_net) {
-				IN_IFADDR_RUNLOCK();
-				return (1);
-			}
-		}
-	} else {
-		TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
-			if ((i & ia->ia_subnetmask) == ia->ia_subnet) {
-				IN_IFADDR_RUNLOCK();
-				return (1);
-			}
+	TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
+		if ((i & ia->ia_subnetmask) == ia->ia_subnet) {
+			IN_IFADDR_RUNLOCK();
+			return (1);
 		}
 	}
 	IN_IFADDR_RUNLOCK();
@@ -845,7 +829,7 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 {
 	register u_long i = ntohl(sin->sin_addr.s_addr);
 	struct sockaddr_in oldaddr;
-	int s = splimp(), flags = RTF_UP, error = 0;
+	int flags = RTF_UP, error = 0;
 
 	oldaddr = ia->ia_addr;
 	if (oldaddr.sin_family == AF_INET)
@@ -865,7 +849,6 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 	if (ifp->if_ioctl != NULL) {
 		error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia);
 		if (error) {
-			splx(s);
 			/* LIST_REMOVE(ia, ia_hash) is done in in_control */
 			ia->ia_addr = oldaddr;
 			IN_IFADDR_WLOCK();
@@ -884,29 +867,24 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 			return (error);
 		}
 	}
-	splx(s);
 	if (scrub) {
 		ia->ia_ifa.ifa_addr = (struct sockaddr *)&oldaddr;
 		in_ifscrub(ifp, ia, LLE_STATIC);
 		ia->ia_ifa.ifa_addr = (struct sockaddr *)&ia->ia_addr;
 	}
-	if (IN_CLASSA(i))
-		ia->ia_netmask = IN_CLASSA_NET;
-	else if (IN_CLASSB(i))
-		ia->ia_netmask = IN_CLASSB_NET;
-	else
-		ia->ia_netmask = IN_CLASSC_NET;
 	/*
-	 * The subnet mask usually includes at least the standard network part,
-	 * but may may be smaller in the case of supernetting.
-	 * If it is set, we believe it.
+	 * Be compatible with network classes, if netmask isn't supplied,
+	 * guess it based on classes.
 	 */
 	if (ia->ia_subnetmask == 0) {
-		ia->ia_subnetmask = ia->ia_netmask;
+		if (IN_CLASSA(i))
+			ia->ia_subnetmask = IN_CLASSA_NET;
+		else if (IN_CLASSB(i))
+			ia->ia_subnetmask = IN_CLASSB_NET;
+		else
+			ia->ia_subnetmask = IN_CLASSC_NET;
 		ia->ia_sockmask.sin_addr.s_addr = htonl(ia->ia_subnetmask);
-	} else
-		ia->ia_netmask &= ia->ia_subnetmask;
-	ia->ia_net = i & ia->ia_netmask;
+	}
 	ia->ia_subnet = i & ia->ia_subnetmask;
 	in_socktrim(&ia->ia_sockmask);
 	/*
@@ -919,10 +897,11 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 	 */
 	ia->ia_ifa.ifa_metric = ifp->if_metric;
 	if (ifp->if_flags & IFF_BROADCAST) {
-		ia->ia_broadaddr.sin_addr.s_addr =
-			htonl(ia->ia_subnet | ~ia->ia_subnetmask);
-		ia->ia_netbroadcast.s_addr =
-			htonl(ia->ia_net | ~ ia->ia_netmask);
+		if (ia->ia_subnetmask == IN_RFC3021_MASK)
+			ia->ia_broadaddr.sin_addr.s_addr = INADDR_BROADCAST;
+		else
+			ia->ia_broadaddr.sin_addr.s_addr =
+			    htonl(ia->ia_subnet | ~ia->ia_subnetmask);
 	} else if (ifp->if_flags & IFF_LOOPBACK) {
 		ia->ia_dstaddr = ia->ia_addr;
 		flags |= RTF_HOST;
@@ -1126,8 +1105,10 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 			RT_LOCK(ia_ro.ro_rt);
 			if (ia_ro.ro_rt->rt_refcnt <= 1)
 				freeit = 1;
-			else
+			else if (flags & LLE_STATIC) {
 				RT_REMREF(ia_ro.ro_rt);
+				target->ia_flags &= ~IFA_RTSELF;
+			}
 			RTFREE_LOCKED(ia_ro.ro_rt);
 		}
 		if (freeit && (flags & LLE_STATIC)) {
@@ -1136,7 +1117,8 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 			if (error == 0)
 				target->ia_flags &= ~IFA_RTSELF;
 		}
-		if (flags & LLE_STATIC)
+		if ((flags & LLE_STATIC) &&
+			!(target->ia_ifp->if_flags & IFF_NOARP))
 			/* remove arp cache */
 			arp_ifscrub(target->ia_ifp, IA_SIN(target)->sin_addr.s_addr);
 	}
@@ -1250,11 +1232,12 @@ in_broadcast(struct in_addr in, struct ifnet *ifp)
 	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
 		if (ifa->ifa_addr->sa_family == AF_INET &&
 		    (in.s_addr == ia->ia_broadaddr.sin_addr.s_addr ||
-		     in.s_addr == ia->ia_netbroadcast.s_addr ||
 		     /*
-		      * Check for old-style (host 0) broadcast.
+		      * Check for old-style (host 0) broadcast, but
+		      * taking into account that RFC 3021 obsoletes it.
 		      */
-		     t == ia->ia_subnet || t == ia->ia_net) &&
+		     (ia->ia_subnetmask != IN_RFC3021_MASK &&
+		     t == ia->ia_subnet)) &&
 		     /*
 		      * Check for an all one subnetmask. These
 		      * only exist when an interface gets a secondary
@@ -1418,30 +1401,67 @@ in_lltable_rtcheck(struct ifnet *ifp, u_int flags, const struct sockaddr *l3addr
 	/* XXX rtalloc1 should take a const param */
 	rt = rtalloc1(__DECONST(struct sockaddr *, l3addr), 0, 0);
 
+	if (rt == NULL)
+		return (EINVAL);
+
 	/*
 	 * If the gateway for an existing host route matches the target L3
-	 * address, allow for ARP to proceed.
+	 * address, which is a special route inserted by some implementation
+	 * such as MANET, and the interface is of the correct type, then
+	 * allow for ARP to proceed.
 	 */
-	if (rt != NULL && (rt->rt_flags & (RTF_HOST|RTF_GATEWAY)) &&
-	    rt->rt_gateway->sa_family == AF_INET &&
-	    memcmp(rt->rt_gateway->sa_data, l3addr->sa_data, 4) == 0) {
-		RTFREE_LOCKED(rt);
-		return (0);
+	if (rt->rt_flags & RTF_GATEWAY) {
+		if (!(rt->rt_flags & RTF_HOST) || !rt->rt_ifp ||
+			rt->rt_ifp->if_type != IFT_ETHER ||
+			  (rt->rt_ifp->if_flags & 
+			   (IFF_NOARP | IFF_STATICARP)) != 0 ||
+			  memcmp(rt->rt_gateway->sa_data, l3addr->sa_data,
+				 sizeof(in_addr_t)) != 0) {
+			RTFREE_LOCKED(rt);
+			return (EINVAL);
+		}
 	}
 
-	if (rt == NULL || (!(flags & LLE_PUB) &&
-			   ((rt->rt_flags & RTF_GATEWAY) || 
-			    (rt->rt_ifp != ifp)))) {
-#ifdef DIAGNOSTIC
-		log(LOG_INFO, "IPv4 address: \"%s\" is not on the network\n",
-		    inet_ntoa(((const struct sockaddr_in *)l3addr)->sin_addr));
-#endif
-		if (rt != NULL)
+	/*
+	 * Make sure that at least the destination address is covered 
+	 * by the route. This is for handling the case where 2 or more 
+	 * interfaces have the same prefix. An incoming packet arrives
+	 * on one interface and the corresponding outgoing packet leaves
+	 * another interface.
+	 */
+	if (!(rt->rt_flags & RTF_HOST) && rt->rt_ifp != ifp) {
+		const char *sa, *mask, *addr, *lim;
+		int len;
+
+		mask = (const char *)rt_mask(rt);
+		/*
+		 * Just being extra cautious to avoid some custom
+		 * code getting into trouble.
+		 */
+		if (mask == NULL) {
 			RTFREE_LOCKED(rt);
-		return (EINVAL);
+			return (EINVAL);
+		}
+
+		sa = (const char *)rt_key(rt);
+		addr = (const char *)l3addr;
+		len = ((const struct sockaddr_in *)l3addr)->sin_len;
+		lim = addr + len;
+
+		for ( ; addr < lim; sa++, mask++, addr++) {
+			if ((*sa ^ *addr) & *mask) {
+#ifdef DIAGNOSTIC
+				log(LOG_INFO, "IPv4 address: \"%s\" is not on the network\n",
+				    inet_ntoa(((const struct sockaddr_in *)l3addr)->sin_addr));
+#endif
+				RTFREE_LOCKED(rt);
+				return (EINVAL);
+			}
+		}
 	}
+
 	RTFREE_LOCKED(rt);
-	return 0;
+	return (0);
 }
 
 /*
diff --git a/sys/netinet/in.h b/sys/netinet/in.h
index d5e429059864..52844f058988 100644
--- a/sys/netinet/in.h
+++ b/sys/netinet/in.h
@@ -392,6 +392,8 @@ __END_DECLS
 
 #define	IN_LOOPBACKNET		127			/* official! */
 
+#define	IN_RFC3021_MASK		(u_int32_t)0xfffffffe
+
 /*
  * Options for use with [gs]etsockopt at the IP level.
  * First word of comment is data type; bool is stored in int.
diff --git a/sys/netinet/in_debug.c b/sys/netinet/in_debug.c
index 7624f1deedfc..0795fa88ef45 100644
--- a/sys/netinet/in_debug.c
+++ b/sys/netinet/in_debug.c
@@ -86,11 +86,8 @@ in_show_in_ifaddr(struct in_ifaddr *ia)
 #define	IA_DB_RPINTF_DPTR(f, e)	db_printf("\t  *%s = " f "\n", #e, *ia->e);
 	db_printf("\tin_ifaddr = %p\n", ia);
 	IA_DB_RPINTF_PTR("%p", ia_ifa);
-	IA_DB_RPINTF("0x%08lx", ia_net);
-	IA_DB_RPINTF("0x%08lx", ia_netmask);
 	IA_DB_RPINTF("0x%08lx", ia_subnet);
 	IA_DB_RPINTF("0x%08lx", ia_subnetmask);
-	IA_DB_RPINTF("0x%08x", ia_netbroadcast.s_addr);
 	IA_DB_RPINTF("%p", ia_hash.le_next);
 	IA_DB_RPINTF("%p", ia_hash.le_prev);
 	IA_DB_RPINTF_DPTR("%p", ia_hash.le_prev);
diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h
index 5be07c151040..3a5e32824a5a 100644
--- a/sys/netinet/in_var.h
+++ b/sys/netinet/in_var.h
@@ -60,12 +60,9 @@ struct in_ifaddr {
 	struct	ifaddr ia_ifa;		/* protocol-independent info */
 #define	ia_ifp		ia_ifa.ifa_ifp
 #define ia_flags	ia_ifa.ifa_flags
-					/* ia_{,sub}net{,mask} in host order */
-	u_long	ia_net;			/* network number of interface */
-	u_long	ia_netmask;		/* mask of net part */
-	u_long	ia_subnet;		/* subnet number, including net */
-	u_long	ia_subnetmask;		/* mask of subnet part */
-	struct	in_addr ia_netbroadcast; /* to recognize net broadcasts */
+					/* ia_subnet{,mask} in host order */
+	u_long	ia_subnet;		/* subnet address */
+	u_long	ia_subnetmask;		/* mask of subnet */
 	LIST_ENTRY(in_ifaddr) ia_hash;	/* entry in bucket of inet addresses */
 	TAILQ_ENTRY(in_ifaddr) ia_link;	/* list of internet addresses */
 	struct	sockaddr_in ia_addr;	/* reserve space for interface name */
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index d16159d7d369..34dced88dfec 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -1424,24 +1424,10 @@ carp_setrun(struct carp_softc *sc, sa_family_t af)
 
 	switch (sc->sc_state) {
 	case INIT:
-		if (carp_opts[CARPCTL_PREEMPT] && !carp_suppress_preempt) {
-			carp_send_ad_locked(sc);
-#ifdef INET
-			carp_send_arp(sc);
-#endif
-#ifdef INET6
-			carp_send_na(sc);
-#endif /* INET6 */
-			CARP_LOG("%s: INIT -> MASTER (preempting)\n",
-			    SC2IFP(sc)->if_xname);
-			carp_set_state(sc, MASTER);
-			carp_setroute(sc, RTM_ADD);
-		} else {
-			CARP_LOG("%s: INIT -> BACKUP\n", SC2IFP(sc)->if_xname);
-			carp_set_state(sc, BACKUP);
-			carp_setroute(sc, RTM_DELETE);
-			carp_setrun(sc, 0);
-		}
+		CARP_LOG("%s: INIT -> BACKUP\n", SC2IFP(sc)->if_xname);
+		carp_set_state(sc, BACKUP);
+		carp_setroute(sc, RTM_DELETE);
+		carp_setrun(sc, 0);
 		break;
 	case BACKUP:
 		callout_stop(&sc->sc_ad_tmo);
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index 67fcb743a9a5..0664acc4f099 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -622,11 +622,6 @@ ip_input(struct mbuf *m)
 				IF_ADDR_UNLOCK(ifp);
 				goto ours;
 			}
-			if (ia->ia_netbroadcast.s_addr == ip->ip_dst.s_addr) {
-				ifa_ref(ifa);
-				IF_ADDR_UNLOCK(ifp);
-				goto ours;
-			}
 #ifdef BOOTP_COMPAT
 			if (IA_SIN(ia)->sin_addr.s_addr == INADDR_ANY) {
 				ifa_ref(ifa);
diff --git a/sys/netinet/ipfw/dummynet.txt b/sys/netinet/ipfw/dummynet.txt
index 0ed6ad15d327..3a1efde59c63 100644
--- a/sys/netinet/ipfw/dummynet.txt
+++ b/sys/netinet/ipfw/dummynet.txt
@@ -325,7 +325,7 @@ we do the following:
 
 dummynet_task()
 ===============
-The dummynet_task() is the the main dummynet processing function and is
+The dummynet_task() is the main dummynet processing function and is
 called every tick. This function first calculate the new current time, then
 it checks if it is the time to wake up object from the system_heap comparing
 the current time and the key of the heap. Two types of object (really the
diff --git a/sys/netinet/ipfw/ip_fw_pfil.c b/sys/netinet/ipfw/ip_fw_pfil.c
index c470b1f2092d..a09ca7982f3c 100644
--- a/sys/netinet/ipfw/ip_fw_pfil.c
+++ b/sys/netinet/ipfw/ip_fw_pfil.c
@@ -31,11 +31,11 @@ __FBSDID("$FreeBSD$");
 #if !defined(KLD_MODULE)
 #include "opt_ipdn.h"
 #include "opt_inet.h"
+#include "opt_inet6.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
 #endif /* KLD_MODULE */
-#include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -154,7 +154,7 @@ ipfw_check_hook(void *arg, struct mbuf **m0, struct ifnet *ifp, int dir,
 		/* next_hop may be set by ipfw_chk */
 		if (args.next_hop == NULL && args.next_hop6 == NULL)
 			break; /* pass */
-#ifndef IPFIREWALL_FORWARD
+#if !defined(IPFIREWALL_FORWARD) || (!defined(INET6) && !defined(INET))
 		ret = EACCES;
 #else
 	    {
@@ -205,7 +205,7 @@ ipfw_check_hook(void *arg, struct mbuf **m0, struct ifnet *ifp, int dir,
 #endif
 		m_tag_prepend(*m0, fwd_tag);
 	    }
-#endif
+#endif /* IPFIREWALL_FORWARD */
 		break;
 
 	case IP_FW_DENY:
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index 1a8e537ac472..dc7d96578905 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -289,6 +289,13 @@ rip_input(struct mbuf *m, int off)
 	last = NULL;
 
 	ifp = m->m_pkthdr.rcvif;
+	/*
+	 * Add back the IP header length which was
+	 * removed by ip_input().  Raw sockets do
+	 * not modify the packet except for some
+	 * byte order swaps.
+	 */
+	ip->ip_len += off;
 
 	hash = INP_PCBHASH_RAW(proto, ip->ip_src.s_addr,
 	    ip->ip_dst.s_addr, V_ripcbinfo.ipi_hashmask);
diff --git a/sys/netinet/sctp.h b/sys/netinet/sctp.h
index 11aa02a0a568..394a7cd5c15a 100644
--- a/sys/netinet/sctp.h
+++ b/sys/netinet/sctp.h
@@ -496,6 +496,7 @@ struct sctp_error_unrecognized_chunk {
 /*
  * PCB Features (in sctp_features bitmask)
  */
+#define SCTP_PCB_FLAGS_DO_NOT_PMTUD     0x00000001
 #define SCTP_PCB_FLAGS_EXT_RCVINFO      0x00000002	/* deprecated */
 #define SCTP_PCB_FLAGS_DONOT_HEARTBEAT  0x00000004
 #define SCTP_PCB_FLAGS_FRAG_INTERLEAVE  0x00000008
diff --git a/sys/netinet/sctp_bsd_addr.c b/sys/netinet/sctp_bsd_addr.c
index 3d004b6b108c..7d15826529e5 100644
--- a/sys/netinet/sctp_bsd_addr.c
+++ b/sys/netinet/sctp_bsd_addr.c
@@ -74,22 +74,6 @@ MALLOC_DEFINE(SCTP_M_MCORE, "sctp_mcore", "sctp mcore queue");
 
 /* Global NON-VNET structure that controls the iterator */
 struct iterator_control sctp_it_ctl;
-static int __sctp_thread_based_iterator_started = 0;
-
-
-static void
-sctp_cleanup_itqueue(void)
-{
-	struct sctp_iterator *it, *nit;
-
-	TAILQ_FOREACH_SAFE(it, &sctp_it_ctl.iteratorhead, sctp_nxt_itr, nit) {
-		if (it->function_atend != NULL) {
-			(*it->function_atend) (it->pointer, it->val);
-		}
-		TAILQ_REMOVE(&sctp_it_ctl.iteratorhead, it, sctp_nxt_itr);
-		SCTP_FREE(it, SCTP_M_ITER);
-	}
-}
 
 
 void
@@ -102,17 +86,11 @@ static void
 sctp_iterator_thread(void *v)
 {
 	SCTP_IPI_ITERATOR_WQ_LOCK();
+	/* In FreeBSD this thread never terminates. */
 	while (1) {
 		msleep(&sctp_it_ctl.iterator_running,
 		    &sctp_it_ctl.ipi_iterator_wq_mtx,
 		    0, "waiting_for_work", 0);
-		if (sctp_it_ctl.iterator_flags & SCTP_ITERATOR_MUST_EXIT) {
-			SCTP_IPI_ITERATOR_WQ_DESTROY();
-			SCTP_ITERATOR_LOCK_DESTROY();
-			sctp_cleanup_itqueue();
-			__sctp_thread_based_iterator_started = 0;
-			kthread_exit();
-		}
 		sctp_iterator_worker();
 	}
 }
@@ -120,21 +98,21 @@ sctp_iterator_thread(void *v)
 void
 sctp_startup_iterator(void)
 {
-	if (__sctp_thread_based_iterator_started) {
+	static int called = 0;
+	int ret;
+
+	if (called) {
 		/* You only get one */
 		return;
 	}
 	/* init the iterator head */
-	__sctp_thread_based_iterator_started = 1;
+	called = 1;
 	sctp_it_ctl.iterator_running = 0;
 	sctp_it_ctl.iterator_flags = 0;
 	sctp_it_ctl.cur_it = NULL;
 	SCTP_ITERATOR_LOCK_INIT();
 	SCTP_IPI_ITERATOR_WQ_INIT();
 	TAILQ_INIT(&sctp_it_ctl.iteratorhead);
-
-	int ret;
-
 	ret = kproc_create(sctp_iterator_thread,
 	    (void *)NULL,
 	    &sctp_it_ctl.thread_proc,
diff --git a/sys/netinet/sctp_constants.h b/sys/netinet/sctp_constants.h
index 4087c8d266d1..5b74056c2af0 100644
--- a/sys/netinet/sctp_constants.h
+++ b/sys/netinet/sctp_constants.h
@@ -508,6 +508,7 @@ __FBSDID("$FreeBSD$");
 
 /* SCTP reachability state for each address */
 #define SCTP_ADDR_REACHABLE		0x001
+#define SCTP_ADDR_NO_PMTUD              0x002
 #define SCTP_ADDR_NOHB			0x004
 #define SCTP_ADDR_BEING_DELETED		0x008
 #define SCTP_ADDR_NOT_IN_ASSOC		0x010
diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c
index b0f288075445..2b36ba4e1f21 100644
--- a/sys/netinet/sctp_pcb.c
+++ b/sys/netinet/sctp_pcb.c
@@ -2786,6 +2786,9 @@ sctp_move_pcb_and_assoc(struct sctp_inpcb *old_inp, struct sctp_inpcb *new_inp,
 			LIST_INSERT_HEAD(&new_inp->sctp_addr_list, laddr,
 			    sctp_nxt_addr);
 			new_inp->laddr_count++;
+			if (oladdr == stcb->asoc.last_used_address) {
+				stcb->asoc.last_used_address = laddr;
+			}
 		}
 	}
 	/*
@@ -2804,6 +2807,7 @@ sctp_move_pcb_and_assoc(struct sctp_inpcb *old_inp, struct sctp_inpcb *new_inp,
 	/* now what about the nets? */
 	TAILQ_FOREACH(net, &stcb->asoc.nets, sctp_next) {
 		net->pmtu_timer.ep = (void *)new_inp;
+		net->hb_timer.ep = (void *)new_inp;
 		net->rxt_timer.ep = (void *)new_inp;
 	}
 	SCTP_INP_WUNLOCK(new_inp);
@@ -4030,11 +4034,16 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 #ifdef INET6
 	net->flowlabel = stcb->asoc.default_flowlabel;
 #endif
-	if (sctp_is_feature_on(stcb->sctp_ep, SCTP_PCB_FLAGS_DONOT_HEARTBEAT)) {
+	if (sctp_stcb_is_feature_on(stcb->sctp_ep, stcb, SCTP_PCB_FLAGS_DONOT_HEARTBEAT)) {
 		net->dest_state |= SCTP_ADDR_NOHB;
 	} else {
 		net->dest_state &= ~SCTP_ADDR_NOHB;
 	}
+	if (sctp_stcb_is_feature_on(stcb->sctp_ep, stcb, SCTP_PCB_FLAGS_DO_NOT_PMTUD)) {
+		net->dest_state |= SCTP_ADDR_NO_PMTUD;
+	} else {
+		net->dest_state &= ~SCTP_ADDR_NO_PMTUD;
+	}
 	net->heart_beat_delay = stcb->asoc.heart_beat_delay;
 	/* Init the timer structure */
 	SCTP_OS_TIMER_INIT(&net->rxt_timer.timer);
diff --git a/sys/netinet/sctp_structs.h b/sys/netinet/sctp_structs.h
index c7dee6956578..eec2daaeef12 100644
--- a/sys/netinet/sctp_structs.h
+++ b/sys/netinet/sctp_structs.h
@@ -185,13 +185,14 @@ struct iterator_control {
 	uint32_t iterator_flags;
 };
 
-#define SCTP_ITERATOR_MUST_EXIT   	0x00000001
-#define SCTP_ITERATOR_STOP_CUR_IT  	0x00000002
-#define SCTP_ITERATOR_STOP_CUR_INP  	0x00000004
+#define SCTP_ITERATOR_STOP_CUR_IT	0x00000004
+#define SCTP_ITERATOR_STOP_CUR_INP	0x00000008
 
 struct sctp_net_route {
 	sctp_rtentry_t *ro_rt;
 	void *ro_lle;
+	void *ro_ia;
+	int ro_flags;
 	union sctp_sockstore _l_addr;	/* remote peer addr */
 	struct sctp_ifa *_s_addr;	/* our selected src addr */
 };
diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c
index 82a8cba37fcb..c6e77cdaa491 100644
--- a/sys/netinet/sctp_usrreq.c
+++ b/sys/netinet/sctp_usrreq.c
@@ -2423,18 +2423,19 @@ sctp_getopt(struct socket *so, int optname, void *optval, size_t *optsize,
 					paddrp->spp_pathmaxrxt = net->failure_threshold;
 					paddrp->spp_pathmtu = net->mtu - ovh;
 					/* get flags for HB */
-					if (net->dest_state & SCTP_ADDR_NOHB)
+					if (net->dest_state & SCTP_ADDR_NOHB) {
 						paddrp->spp_flags |= SPP_HB_DISABLE;
-					else
+					} else {
 						paddrp->spp_flags |= SPP_HB_ENABLE;
+					}
 					/* get flags for PMTU */
-					if (SCTP_OS_TIMER_PENDING(&net->pmtu_timer.timer)) {
+					if (net->dest_state & SCTP_ADDR_NO_PMTUD) {
 						paddrp->spp_flags |= SPP_PMTUD_ENABLE;
 					} else {
 						paddrp->spp_flags |= SPP_PMTUD_DISABLE;
 					}
 					if (net->dscp & 0x01) {
-						paddrp->spp_dscp = net->dscp >> 2;
+						paddrp->spp_dscp = net->dscp & 0xfc;
 						paddrp->spp_flags |= SPP_DSCP;
 					}
 #ifdef INET6
@@ -2449,12 +2450,10 @@ sctp_getopt(struct socket *so, int optname, void *optval, size_t *optsize,
 					 * No destination so return default
 					 * value
 					 */
-					int cnt = 0;
-
 					paddrp->spp_pathmaxrxt = stcb->asoc.def_net_failure;
 					paddrp->spp_pathmtu = sctp_get_frag_point(stcb, &stcb->asoc);
 					if (stcb->asoc.default_dscp & 0x01) {
-						paddrp->spp_dscp = stcb->asoc.default_dscp >> 2;
+						paddrp->spp_dscp = stcb->asoc.default_dscp & 0xfc;
 						paddrp->spp_flags |= SPP_DSCP;
 					}
 #ifdef INET6
@@ -2464,20 +2463,17 @@ sctp_getopt(struct socket *so, int optname, void *optval, size_t *optsize,
 					}
 #endif
 					/* default settings should be these */
-					if (sctp_is_feature_on(stcb->sctp_ep, SCTP_PCB_FLAGS_DONOT_HEARTBEAT)) {
+					if (sctp_stcb_is_feature_on(inp, stcb, SCTP_PCB_FLAGS_DONOT_HEARTBEAT)) {
 						paddrp->spp_flags |= SPP_HB_DISABLE;
 					} else {
 						paddrp->spp_flags |= SPP_HB_ENABLE;
 					}
-					paddrp->spp_hbinterval = stcb->asoc.heart_beat_delay;
-					TAILQ_FOREACH(net, &stcb->asoc.nets, sctp_next) {
-						if (SCTP_OS_TIMER_PENDING(&net->pmtu_timer.timer)) {
-							cnt++;
-						}
-					}
-					if (cnt) {
+					if (sctp_stcb_is_feature_on(inp, stcb, SCTP_PCB_FLAGS_DO_NOT_PMTUD)) {
+						paddrp->spp_flags |= SPP_PMTUD_DISABLE;
+					} else {
 						paddrp->spp_flags |= SPP_PMTUD_ENABLE;
 					}
+					paddrp->spp_hbinterval = stcb->asoc.heart_beat_delay;
 				}
 				paddrp->spp_assoc_id = sctp_get_associd(stcb);
 				SCTP_TCB_UNLOCK(stcb);
@@ -2492,7 +2488,7 @@ sctp_getopt(struct socket *so, int optname, void *optval, size_t *optsize,
 					paddrp->spp_assoc_id = SCTP_FUTURE_ASSOC;
 					/* get inp's default */
 					if (inp->sctp_ep.default_dscp & 0x01) {
-						paddrp->spp_dscp = inp->sctp_ep.default_dscp >> 2;
+						paddrp->spp_dscp = inp->sctp_ep.default_dscp & 0xfc;
 						paddrp->spp_flags |= SPP_DSCP;
 					}
 #ifdef INET6
@@ -2505,14 +2501,16 @@ sctp_getopt(struct socket *so, int optname, void *optval, size_t *optsize,
 					/* can't return this */
 					paddrp->spp_pathmtu = 0;
 
-					/* default behavior, no stcb */
-					paddrp->spp_flags = SPP_PMTUD_ENABLE;
-
 					if (sctp_is_feature_off(inp, SCTP_PCB_FLAGS_DONOT_HEARTBEAT)) {
 						paddrp->spp_flags |= SPP_HB_ENABLE;
 					} else {
 						paddrp->spp_flags |= SPP_HB_DISABLE;
 					}
+					if (sctp_is_feature_off(inp, SCTP_PCB_FLAGS_DO_NOT_PMTUD)) {
+						paddrp->spp_flags |= SPP_PMTUD_ENABLE;
+					} else {
+						paddrp->spp_flags |= SPP_PMTUD_DISABLE;
+					}
 					SCTP_INP_RUNLOCK(inp);
 				} else {
 					SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, EINVAL);
@@ -4651,6 +4649,7 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 							sctp_timer_stop(SCTP_TIMER_TYPE_PATHMTURAISE, inp, stcb, net,
 							    SCTP_FROM_SCTP_USRREQ + SCTP_LOC_10);
 						}
+						net->dest_state |= SCTP_ADDR_NO_PMTUD;
 						if (paddrp->spp_pathmtu > SCTP_DEFAULT_MINSEGMENT) {
 							net->mtu = paddrp->spp_pathmtu + ovh;
 							if (net->mtu < stcb->asoc.smallest_mtu) {
@@ -4659,9 +4658,10 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 						}
 					}
 					if (paddrp->spp_flags & SPP_PMTUD_ENABLE) {
-						if (SCTP_OS_TIMER_PENDING(&net->pmtu_timer.timer)) {
+						if (!SCTP_OS_TIMER_PENDING(&net->pmtu_timer.timer)) {
 							sctp_timer_start(SCTP_TIMER_TYPE_PATHMTURAISE, inp, stcb, net);
 						}
+						net->dest_state &= ~SCTP_ADDR_NO_PMTUD;
 					}
 					if (paddrp->spp_pathmaxrxt) {
 						if (net->dest_state & SCTP_ADDR_PF) {
@@ -4691,7 +4691,7 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 						net->failure_threshold = paddrp->spp_pathmaxrxt;
 					}
 					if (paddrp->spp_flags & SPP_DSCP) {
-						net->dscp = paddrp->spp_dscp << 2;
+						net->dscp = paddrp->spp_dscp & 0xfc;
 						net->dscp |= 0x01;
 					}
 #ifdef INET6
@@ -4754,9 +4754,9 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 							    SCTP_FROM_SCTP_USRREQ + SCTP_LOC_10);
 							sctp_timer_start(SCTP_TIMER_TYPE_HEARTBEAT, inp, stcb, net);
 						}
+						sctp_stcb_feature_off(inp, stcb, SCTP_PCB_FLAGS_DONOT_HEARTBEAT);
 					}
 					if (paddrp->spp_flags & SPP_HB_DISABLE) {
-						/* Turn back on the timer */
 						TAILQ_FOREACH(net, &stcb->asoc.nets, sctp_next) {
 							if (!(net->dest_state & SCTP_ADDR_NOHB)) {
 								net->dest_state |= SCTP_ADDR_NOHB;
@@ -4765,6 +4765,7 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 								}
 							}
 						}
+						sctp_stcb_feature_on(inp, stcb, SCTP_PCB_FLAGS_DONOT_HEARTBEAT);
 					}
 					if ((paddrp->spp_flags & SPP_PMTUD_DISABLE) && (paddrp->spp_pathmtu >= SCTP_SMALLEST_PMTU)) {
 						TAILQ_FOREACH(net, &stcb->asoc.nets, sctp_next) {
@@ -4772,6 +4773,7 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 								sctp_timer_stop(SCTP_TIMER_TYPE_PATHMTURAISE, inp, stcb, net,
 								    SCTP_FROM_SCTP_USRREQ + SCTP_LOC_10);
 							}
+							net->dest_state |= SCTP_ADDR_NO_PMTUD;
 							if (paddrp->spp_pathmtu > SCTP_DEFAULT_MINSEGMENT) {
 								net->mtu = paddrp->spp_pathmtu + ovh;
 								if (net->mtu < stcb->asoc.smallest_mtu) {
@@ -4779,20 +4781,23 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 								}
 							}
 						}
+						sctp_stcb_feature_on(inp, stcb, SCTP_PCB_FLAGS_DO_NOT_PMTUD);
 					}
 					if (paddrp->spp_flags & SPP_PMTUD_ENABLE) {
 						TAILQ_FOREACH(net, &stcb->asoc.nets, sctp_next) {
-							if (SCTP_OS_TIMER_PENDING(&net->pmtu_timer.timer)) {
+							if (!SCTP_OS_TIMER_PENDING(&net->pmtu_timer.timer)) {
 								sctp_timer_start(SCTP_TIMER_TYPE_PATHMTURAISE, inp, stcb, net);
 							}
+							net->dest_state &= ~SCTP_ADDR_NO_PMTUD;
 						}
+						sctp_stcb_feature_off(inp, stcb, SCTP_PCB_FLAGS_DO_NOT_PMTUD);
 					}
 					if (paddrp->spp_flags & SPP_DSCP) {
 						TAILQ_FOREACH(net, &stcb->asoc.nets, sctp_next) {
-							net->dscp = paddrp->spp_dscp << 2;
+							net->dscp = paddrp->spp_dscp & 0xfc;
 							net->dscp |= 0x01;
 						}
-						stcb->asoc.default_dscp = paddrp->spp_dscp << 2;
+						stcb->asoc.default_dscp = paddrp->spp_dscp & 0xfc;
 						stcb->asoc.default_dscp |= 0x01;
 					}
 #ifdef INET6
@@ -4840,8 +4845,13 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 					} else if (paddrp->spp_flags & SPP_HB_DISABLE) {
 						sctp_feature_on(inp, SCTP_PCB_FLAGS_DONOT_HEARTBEAT);
 					}
+					if (paddrp->spp_flags & SPP_PMTUD_ENABLE) {
+						sctp_feature_off(inp, SCTP_PCB_FLAGS_DO_NOT_PMTUD);
+					} else if (paddrp->spp_flags & SPP_PMTUD_DISABLE) {
+						sctp_feature_on(inp, SCTP_PCB_FLAGS_DO_NOT_PMTUD);
+					}
 					if (paddrp->spp_flags & SPP_DSCP) {
-						inp->sctp_ep.default_dscp = paddrp->spp_dscp << 2;
+						inp->sctp_ep.default_dscp = paddrp->spp_dscp & 0xfc;
 						inp->sctp_ep.default_dscp |= 0x01;
 					}
 #ifdef INET6
diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c
index 62fa0d1f2325..3bd4e6fbec0a 100644
--- a/sys/netinet/sctputil.c
+++ b/sys/netinet/sctputil.c
@@ -1292,10 +1292,6 @@ sctp_iterator_work(struct sctp_iterator *it)
 				/* We won't be staying here */
 				SCTP_INP_DECR_REF(it->inp);
 				atomic_add_int(&it->stcb->asoc.refcnt, -1);
-				if (sctp_it_ctl.iterator_flags &
-				    SCTP_ITERATOR_MUST_EXIT) {
-					goto done_with_iterator;
-				}
 				if (sctp_it_ctl.iterator_flags &
 				    SCTP_ITERATOR_STOP_CUR_IT) {
 					sctp_it_ctl.iterator_flags &= ~SCTP_ITERATOR_STOP_CUR_IT;
@@ -1372,9 +1368,6 @@ sctp_iterator_worker(void)
 		sctp_it_ctl.cur_it = NULL;
 		CURVNET_RESTORE();
 		SCTP_IPI_ITERATOR_WQ_LOCK();
-		if (sctp_it_ctl.iterator_flags & SCTP_ITERATOR_MUST_EXIT) {
-			break;
-		}
 		/* sa_ignore FREED_MEMORY */
 	}
 	sctp_it_ctl.iterator_running = 0;
@@ -1668,7 +1661,7 @@ sctp_timeout_handler(void *t)
 		sctp_auditing(4, inp, stcb, net);
 #endif
 		if (!(net->dest_state & SCTP_ADDR_NOHB)) {
-			sctp_timer_start(SCTP_TIMER_TYPE_HEARTBEAT, stcb->sctp_ep, stcb, net);
+			sctp_timer_start(SCTP_TIMER_TYPE_HEARTBEAT, inp, stcb, net);
 			sctp_chunk_output(inp, stcb, SCTP_OUTPUT_FROM_HB_TMR, SCTP_SO_NOT_LOCKED);
 		}
 		break;
@@ -2052,6 +2045,9 @@ sctp_timer_start(int t_type, struct sctp_inpcb *inp, struct sctp_tcb *stcb,
 		if (net == NULL) {
 			return;
 		}
+		if (net->dest_state & SCTP_ADDR_NO_PMTUD) {
+			return;
+		}
 		to_ticks = inp->sctp_ep.sctp_timeoutticks[SCTP_TIMER_PMTU];
 		tmr = &net->pmtu_timer;
 		break;
diff --git a/sys/netinet/tcp.h b/sys/netinet/tcp.h
index 443425f7c721..805a5616dc1c 100644
--- a/sys/netinet/tcp.h
+++ b/sys/netinet/tcp.h
@@ -34,6 +34,7 @@
 #define _NETINET_TCP_H_
 
 #include <sys/cdefs.h>
+#include <sys/types.h>
 
 #if __BSD_VISIBLE
 
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index b1a8b335cabb..128cccd76e90 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -183,6 +183,11 @@ SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, insecure_rst, CTLFLAG_RW,
     &VNET_NAME(tcp_insecure_rst), 0,
     "Follow the old (insecure) criteria for accepting RST packets");
 
+VNET_DEFINE(int, tcp_recvspace) = 1024*64;
+#define	V_tcp_recvspace	VNET(tcp_recvspace)
+SYSCTL_VNET_INT(_net_inet_tcp, TCPCTL_RECVSPACE, tcp_recvspace, CTLFLAG_RW,
+    &VNET_NAME(tcp_recvspace), 0, "Initial receive socket buffer size");
+
 VNET_DEFINE(int, tcp_do_autorcvbuf) = 1;
 #define	V_tcp_do_autorcvbuf	VNET(tcp_do_autorcvbuf)
 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, recvbuf_auto, CTLFLAG_RW,
@@ -301,9 +306,6 @@ cc_conn_init(struct tcpcb *tp)
 	struct hc_metrics_lite metrics;
 	struct inpcb *inp = tp->t_inpcb;
 	int rtt;
-#ifdef INET6
-	int isipv6 = ((inp->inp_vflag & INP_IPV6) != 0) ? 1 : 0;
-#endif
 
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
@@ -337,49 +339,16 @@ cc_conn_init(struct tcpcb *tp)
 	}
 
 	/*
-	 * Set the slow-start flight size depending on whether this
-	 * is a local network or not.
-	 *
-	 * Extend this so we cache the cwnd too and retrieve it here.
-	 * Make cwnd even bigger than RFC3390 suggests but only if we
-	 * have previous experience with the remote host. Be careful
-	 * not make cwnd bigger than remote receive window or our own
-	 * send socket buffer. Maybe put some additional upper bound
-	 * on the retrieved cwnd. Should do incremental updates to
-	 * hostcache when cwnd collapses so next connection doesn't
-	 * overloads the path again.
-	 *
-	 * XXXAO: Initializing the CWND from the hostcache is broken
-	 * and in its current form not RFC conformant.  It is disabled
-	 * until fixed or removed entirely.
+	 * Set the initial slow-start flight size.
 	 *
 	 * RFC3390 says only do this if SYN or SYN/ACK didn't got lost.
-	 * We currently check only in syncache_socket for that.
+	 * XXX: We currently check only in syncache_socket for that.
 	 */
-/* #define TCP_METRICS_CWND */
-#ifdef TCP_METRICS_CWND
-	if (metrics.rmx_cwnd)
-		tp->snd_cwnd = max(tp->t_maxseg, min(metrics.rmx_cwnd / 2,
-		    min(tp->snd_wnd, so->so_snd.sb_hiwat)));
-	else
-#endif
 	if (V_tcp_do_rfc3390)
 		tp->snd_cwnd = min(4 * tp->t_maxseg,
 		    max(2 * tp->t_maxseg, 4380));
-#ifdef INET6
-	else if (isipv6 && in6_localaddr(&inp->in6p_faddr))
-		tp->snd_cwnd = tp->t_maxseg * V_ss_fltsz_local;
-#endif
-#if defined(INET) && defined(INET6)
-	else if (!isipv6 && in_localaddr(inp->inp_faddr))
-		tp->snd_cwnd = tp->t_maxseg * V_ss_fltsz_local;
-#endif
-#ifdef INET
-	else if (in_localaddr(inp->inp_faddr))
-		tp->snd_cwnd = tp->t_maxseg * V_ss_fltsz_local;
-#endif
 	else
-		tp->snd_cwnd = tp->t_maxseg * V_ss_fltsz;
+		tp->snd_cwnd = tp->t_maxseg;
 
 	if (CC_ALGO(tp)->conn_init != NULL)
 		CC_ALGO(tp)->conn_init(tp->ccv);
@@ -948,24 +917,8 @@ tcp_input(struct mbuf *m, int off0)
 		}
 		INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
 
-#ifdef TCP_SIGNATURE
-		tcp_dooptions(&to, optp, optlen,
-		    (thflags & TH_SYN) ? TO_SYN : 0);
-		if (sig_checked == 0) {
-			tp = intotcpcb(inp);
-			if (tp == NULL || tp->t_state == TCPS_CLOSED) {
-				rstreason = BANDLIM_RST_CLOSEDPORT;
-				goto dropwithreset;
-			}
-			if (!tcp_signature_verify_input(m, off0, tlen, optlen,
-			    &to, th, tp->t_flags))
-				goto dropunlock;
-			sig_checked = 1;
-		}
-#else
 		if (thflags & TH_SYN)
 			tcp_dooptions(&to, optp, optlen, TO_SYN);
-#endif
 		/*
 		 * NB: tcp_twcheck unlocks the INP and frees the mbuf.
 		 */
@@ -3533,7 +3486,7 @@ tcp_mss(struct tcpcb *tp, int offer)
 	 */
 	so = inp->inp_socket;
 	SOCKBUF_LOCK(&so->so_snd);
-	if ((so->so_snd.sb_hiwat == tcp_sendspace) && metrics.rmx_sendpipe)
+	if ((so->so_snd.sb_hiwat == V_tcp_sendspace) && metrics.rmx_sendpipe)
 		bufsize = metrics.rmx_sendpipe;
 	else
 		bufsize = so->so_snd.sb_hiwat;
@@ -3550,7 +3503,7 @@ tcp_mss(struct tcpcb *tp, int offer)
 	tp->t_maxseg = mss;
 
 	SOCKBUF_LOCK(&so->so_rcv);
-	if ((so->so_rcv.sb_hiwat == tcp_recvspace) && metrics.rmx_recvpipe)
+	if ((so->so_rcv.sb_hiwat == V_tcp_recvspace) && metrics.rmx_recvpipe)
 		bufsize = metrics.rmx_recvpipe;
 	else
 		bufsize = so->so_rcv.sb_hiwat;
diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c
index 702eed330f15..2db178dac3d4 100644
--- a/sys/netinet/tcp_output.c
+++ b/sys/netinet/tcp_output.c
@@ -89,22 +89,17 @@ SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_RW,
 	&VNET_NAME(path_mtu_discovery), 1,
 	"Enable Path MTU Discovery");
 
-VNET_DEFINE(int, ss_fltsz) = 1;
-SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, CTLFLAG_RW,
-	&VNET_NAME(ss_fltsz), 1,
-	"Slow start flight size");
-
-VNET_DEFINE(int, ss_fltsz_local) = 4;
-SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize,
-	CTLFLAG_RW, &VNET_NAME(ss_fltsz_local), 1,
-	"Slow start flight size for local networks");
-
 VNET_DEFINE(int, tcp_do_tso) = 1;
 #define	V_tcp_do_tso		VNET(tcp_do_tso)
 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW,
 	&VNET_NAME(tcp_do_tso), 0,
 	"Enable TCP Segmentation Offload");
 
+VNET_DEFINE(int, tcp_sendspace) = 1024*32;
+#define	V_tcp_sendspace	VNET(tcp_sendspace)
+SYSCTL_VNET_INT(_net_inet_tcp, TCPCTL_SENDSPACE, tcp_sendspace, CTLFLAG_RW,
+	&VNET_NAME(tcp_sendspace), 0, "Initial send socket buffer size");
+
 VNET_DEFINE(int, tcp_do_autosndbuf) = 1;
 #define	V_tcp_do_autosndbuf	VNET(tcp_do_autosndbuf)
 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, sendbuf_auto, CTLFLAG_RW,
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index 318fc261c8b8..d1303615154d 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -177,7 +177,9 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 	struct tseg_qent *nq;
 	struct tseg_qent *te = NULL;
 	struct socket *so = tp->t_inpcb->inp_socket;
+	char *s = NULL;
 	int flags;
+	struct tseg_qent tqs;
 
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
@@ -215,19 +217,40 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 		TCPSTAT_INC(tcps_rcvmemdrop);
 		m_freem(m);
 		*tlenp = 0;
+		if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
+			log(LOG_DEBUG, "%s; %s: queue limit reached, "
+			    "segment dropped\n", s, __func__);
+			free(s, M_TCPLOG);
+		}
 		return (0);
 	}
 
 	/*
 	 * Allocate a new queue entry. If we can't, or hit the zone limit
 	 * just drop the pkt.
+	 *
+	 * Use a temporary structure on the stack for the missing segment
+	 * when the zone is exhausted. Otherwise we may get stuck.
 	 */
 	te = uma_zalloc(V_tcp_reass_zone, M_NOWAIT);
-	if (te == NULL) {
+	if (te == NULL && th->th_seq != tp->rcv_nxt) {
 		TCPSTAT_INC(tcps_rcvmemdrop);
 		m_freem(m);
 		*tlenp = 0;
+		if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
+			log(LOG_DEBUG, "%s; %s: global zone limit reached, "
+			    "segment dropped\n", s, __func__);
+			free(s, M_TCPLOG);
+		}
 		return (0);
+	} else if (th->th_seq == tp->rcv_nxt) {
+		bzero(&tqs, sizeof(struct tseg_qent));
+		te = &tqs;
+		if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
+			log(LOG_DEBUG, "%s; %s: global zone limit reached, "
+			    "using stack for missing segment\n", s, __func__);
+			free(s, M_TCPLOG);
+		}
 	}
 	tp->t_segqlen++;
 
@@ -304,6 +327,8 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 	if (p == NULL) {
 		LIST_INSERT_HEAD(&tp->t_segq, te, tqe_q);
 	} else {
+		KASSERT(te != &tqs, ("%s: temporary stack based entry not "
+		    "first element in queue", __func__));
 		LIST_INSERT_AFTER(p, te, tqe_q);
 	}
 
@@ -327,7 +352,8 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 			m_freem(q->tqe_m);
 		else
 			sbappendstream_locked(&so->so_rcv, q->tqe_m);
-		uma_zfree(V_tcp_reass_zone, q);
+		if (q != &tqs)
+			uma_zfree(V_tcp_reass_zone, q);
 		tp->t_segqlen--;
 		q = nq;
 	} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 73984c70aff9..1651d6928ba0 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -495,6 +495,13 @@ tcp_timer_rexmt(void * xtp)
 			CURVNET_RESTORE();
 			return;
 		}
+		if (inp->inp_flags & INP_DROPPED) {
+			INP_WUNLOCK(inp);
+			INP_INFO_WUNLOCK(&V_tcbinfo);
+			CURVNET_RESTORE();
+			return;
+		}
+
 		tp = tcp_drop(tp, tp->t_softerror ?
 			      tp->t_softerror : ETIMEDOUT);
 		headlocked = 1;
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 96cb1e4e1b29..7a7e1ce3da89 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -1497,18 +1497,6 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt)
 }
 #undef INP_WLOCK_RECHECK
 
-/*
- * tcp_sendspace and tcp_recvspace are the default send and receive window
- * sizes, respectively.  These are obsolescent (this information should
- * be set by the route).
- */
-u_long	tcp_sendspace = 1024*32;
-SYSCTL_ULONG(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLFLAG_RW,
-    &tcp_sendspace , 0, "Maximum outgoing TCP datagram size");
-u_long	tcp_recvspace = 1024*64;
-SYSCTL_ULONG(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLFLAG_RW,
-    &tcp_recvspace , 0, "Maximum incoming TCP datagram size");
-
 /*
  * Attach TCP protocol to socket, allocating
  * internet protocol control block, tcp control block,
@@ -1522,7 +1510,7 @@ tcp_attach(struct socket *so)
 	int error;
 
 	if (so->so_snd.sb_hiwat == 0 || so->so_rcv.sb_hiwat == 0) {
-		error = soreserve(so, tcp_sendspace, tcp_recvspace);
+		error = soreserve(so, V_tcp_sendspace, V_tcp_recvspace);
 		if (error)
 			return (error);
 	}
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index bbf392f6368a..bc64ac2ba8ef 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -606,9 +606,9 @@ VNET_DECLARE(int, tcp_mssdflt);	/* XXX */
 VNET_DECLARE(int, tcp_minmss);
 VNET_DECLARE(int, tcp_delack_enabled);
 VNET_DECLARE(int, tcp_do_rfc3390);
+VNET_DECLARE(int, tcp_sendspace);
+VNET_DECLARE(int, tcp_recvspace);
 VNET_DECLARE(int, path_mtu_discovery);
-VNET_DECLARE(int, ss_fltsz);
-VNET_DECLARE(int, ss_fltsz_local);
 VNET_DECLARE(int, tcp_do_rfc3465);
 VNET_DECLARE(int, tcp_abc_l_var);
 #define	V_tcb			VNET(tcb)
@@ -618,9 +618,9 @@ VNET_DECLARE(int, tcp_abc_l_var);
 #define	V_tcp_minmss		VNET(tcp_minmss)
 #define	V_tcp_delack_enabled	VNET(tcp_delack_enabled)
 #define	V_tcp_do_rfc3390	VNET(tcp_do_rfc3390)
+#define	V_tcp_sendspace		VNET(tcp_sendspace)
+#define	V_tcp_recvspace		VNET(tcp_recvspace)
 #define	V_path_mtu_discovery	VNET(path_mtu_discovery)
-#define	V_ss_fltsz		VNET(ss_fltsz)
-#define	V_ss_fltsz_local	VNET(ss_fltsz_local)
 #define	V_tcp_do_rfc3465	VNET(tcp_do_rfc3465)
 #define	V_tcp_abc_l_var		VNET(tcp_abc_l_var)
 
@@ -716,8 +716,6 @@ void	 tcp_hc_updatemtu(struct in_conninfo *, u_long);
 void	 tcp_hc_update(struct in_conninfo *, struct hc_metrics_lite *);
 
 extern	struct pr_usrreqs tcp_usrreqs;
-extern	u_long tcp_sendspace;
-extern	u_long tcp_recvspace;
 tcp_seq tcp_new_isn(struct tcpcb *);
 
 void	 tcp_sack_doack(struct tcpcb *, struct tcpopt *, tcp_seq);
diff --git a/sys/netinet/udp_var.h b/sys/netinet/udp_var.h
index 7e7cfba2eeb8..6e5abd05c21b 100644
--- a/sys/netinet/udp_var.h
+++ b/sys/netinet/udp_var.h
@@ -51,6 +51,9 @@ struct udpiphdr {
 #define	ui_ulen		ui_u.uh_ulen
 #define	ui_sum		ui_u.uh_sum
 
+struct inpcb;
+struct mbuf;
+
 typedef void(*udp_tun_func_t)(struct mbuf *, int off, struct inpcb *);
 
 /*
diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index 6a2da07341f6..d22bc43e5e1d 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -2244,10 +2244,6 @@ icmp6_reflect(struct mbuf *m, size_t off)
 		}
 	}
 
-	if ((srcp != NULL) && 
-	    (in6_addrscope(srcp) != in6_addrscope(&ip6->ip6_src)))
-		srcp = NULL;
-
 	if (srcp == NULL) {
 		int e;
 		struct sockaddr_in6 sin6;
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index 9bf7a8f44e5d..1e6cb9481167 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -136,7 +136,89 @@ static void in6_unlink_ifa(struct in6_ifaddr *, struct ifnet *);
 
 int	(*faithprefix_p)(struct in6_addr *);
 
+#define ifa2ia6(ifa)	((struct in6_ifaddr *)(ifa))
+#define ia62ifa(ia6)	(&((ia6)->ia_ifa))
 
+void
+in6_ifaddloop(struct ifaddr *ifa)
+{
+	struct sockaddr_dl gateway;
+	struct sockaddr_in6 mask, addr;
+	struct rtentry rt;
+	struct in6_ifaddr *ia;
+	struct ifnet *ifp;
+	struct llentry *ln;
+
+	ia = ifa2ia6(ifa);
+	ifp = ifa->ifa_ifp;
+	IF_AFDATA_LOCK(ifp);
+	ifa->ifa_rtrequest = NULL;
+
+	/* XXX QL
+	 * we need to report rt_newaddrmsg
+	 */
+	ln = lla_lookup(LLTABLE6(ifp), (LLE_CREATE | LLE_IFADDR |
+	    LLE_EXCLUSIVE), (struct sockaddr *)&ia->ia_addr);
+	IF_AFDATA_UNLOCK(ifp);
+	if (ln != NULL) {
+		ln->la_expire = 0;  /* for IPv6 this means permanent */
+		ln->ln_state = ND6_LLINFO_REACHABLE;
+		/*
+		 * initialize for rtmsg generation
+		 */
+		bzero(&gateway, sizeof(gateway));
+		gateway.sdl_len = sizeof(gateway);
+		gateway.sdl_family = AF_LINK;
+		gateway.sdl_nlen = 0;
+		gateway.sdl_alen = 6;
+		memcpy(gateway.sdl_data, &ln->ll_addr.mac_aligned,
+		    sizeof(ln->ll_addr));
+		LLE_WUNLOCK(ln);
+	}
+
+	bzero(&rt, sizeof(rt));
+	rt.rt_gateway = (struct sockaddr *)&gateway;
+	memcpy(&mask, &ia->ia_prefixmask, sizeof(ia->ia_prefixmask));
+	memcpy(&addr, &ia->ia_addr, sizeof(ia->ia_addr));
+	rt_mask(&rt) = (struct sockaddr *)&mask;
+	rt_key(&rt) = (struct sockaddr *)&addr;
+	rt.rt_flags = RTF_UP | RTF_HOST | RTF_STATIC;
+	rt_newaddrmsg(RTM_ADD, ifa, 0, &rt);
+}
+
+void
+in6_ifremloop(struct ifaddr *ifa)
+{
+	struct sockaddr_dl gateway;
+	struct sockaddr_in6 mask, addr;
+	struct rtentry rt0;
+	struct in6_ifaddr *ia;
+	struct ifnet *ifp;
+
+	ia = ifa2ia6(ifa);
+	ifp = ifa->ifa_ifp;
+	IF_AFDATA_LOCK(ifp);
+	lla_lookup(LLTABLE6(ifp), (LLE_DELETE | LLE_IFADDR),
+	    (struct sockaddr *)&ia->ia_addr);
+	IF_AFDATA_UNLOCK(ifp);
+
+	/*
+	 * initialize for rtmsg generation
+	 */
+	bzero(&gateway, sizeof(gateway));
+	gateway.sdl_len = sizeof(gateway);
+	gateway.sdl_family = AF_LINK;
+	gateway.sdl_nlen = 0;
+	gateway.sdl_alen = ifp->if_addrlen;
+	bzero(&rt0, sizeof(rt0));
+	rt0.rt_gateway = (struct sockaddr *)&gateway;
+	memcpy(&mask, &ia->ia_prefixmask, sizeof(ia->ia_prefixmask));
+	memcpy(&addr, &ia->ia_addr, sizeof(ia->ia_addr));
+	rt_mask(&rt0) = (struct sockaddr *)&mask;
+	rt_key(&rt0) = (struct sockaddr *)&addr;
+	rt0.rt_flags = RTF_HOST | RTF_STATIC;
+	rt_newaddrmsg(RTM_DELETE, ifa, 0, &rt0);
+}
 
 int
 in6_mask2len(struct in6_addr *mask, u_char *lim0)
@@ -174,9 +256,6 @@ in6_mask2len(struct in6_addr *mask, u_char *lim0)
 	return x * 8 + y;
 }
 
-#define ifa2ia6(ifa)	((struct in6_ifaddr *)(ifa))
-#define ia62ifa(ia6)	(&((ia6)->ia_ifa))
-
 #ifdef COMPAT_FREEBSD32
 struct in6_ndifreq32 {
         char ifname[IFNAMSIZ];
@@ -1218,9 +1297,6 @@ in6_purgeaddr(struct ifaddr *ifa)
 	struct in6_ifaddr *ia = (struct in6_ifaddr *) ifa;
 	struct in6_multi_mship *imm;
 	struct sockaddr_in6 mltaddr, mltmask;
-	struct rtentry rt0;
-	struct sockaddr_dl gateway;
-	struct sockaddr_in6 mask, addr;
 	int plen, error;
 	struct rtentry *rt;
 	struct ifaddr *ifa0, *nifa;
@@ -1259,28 +1335,7 @@ in6_purgeaddr(struct ifaddr *ifa)
 	/* stop DAD processing */
 	nd6_dad_stop(ifa);
 
-	IF_AFDATA_LOCK(ifp);
-	lla_lookup(LLTABLE6(ifp), (LLE_DELETE | LLE_IFADDR),
-	    (struct sockaddr *)&ia->ia_addr);
-	IF_AFDATA_UNLOCK(ifp);
-
-	/*
-	 * initialize for rtmsg generation
-	 */
-	bzero(&gateway, sizeof(gateway));
-	gateway.sdl_len = sizeof(gateway);
-	gateway.sdl_family = AF_LINK;
-	gateway.sdl_nlen = 0;
-	gateway.sdl_alen = ifp->if_addrlen;
-	/* */
-	bzero(&rt0, sizeof(rt0));
-	rt0.rt_gateway = (struct sockaddr *)&gateway;
-	memcpy(&mask, &ia->ia_prefixmask, sizeof(ia->ia_prefixmask));
-	memcpy(&addr, &ia->ia_addr, sizeof(ia->ia_addr));
-	rt_mask(&rt0) = (struct sockaddr *)&mask;
-	rt_key(&rt0) = (struct sockaddr *)&addr;
-	rt0.rt_flags = RTF_HOST | RTF_STATIC;
-	rt_newaddrmsg(RTM_DELETE, ifa, 0, &rt0);
+	in6_ifremloop(ifa);
 
 	/*
 	 * leave from multicast groups we have joined for the interface
@@ -1805,14 +1860,17 @@ in6_ifinit(struct ifnet *ifp, struct in6_ifaddr *ia,
 		if (error != 0)
 			return (error);
 		ia->ia_flags |= IFA_ROUTE;
+		/*
+		 * Handle the case for ::1 .
+		 */
+		if (ifp->if_flags & IFF_LOOPBACK)
+			ia->ia_flags |= IFA_RTSELF;
 	}
 
 	/*
 	 * add a loopback route to self
 	 */
-	if (!(ia->ia_flags & IFA_ROUTE)
-	    && (V_nd6_useloopback
-		|| (ifp->if_flags & IFF_LOOPBACK))) {
+	if (!(ia->ia_flags & IFA_RTSELF) && V_nd6_useloopback) {
 		error = ifa_add_loopback_route((struct ifaddr *)ia,
 				       (struct sockaddr *)&ia->ia_addr);
 		if (error == 0)
@@ -1820,46 +1878,8 @@ in6_ifinit(struct ifnet *ifp, struct in6_ifaddr *ia,
 	}
 
 	/* Add ownaddr as loopback rtentry, if necessary (ex. on p2p link). */
-	if (newhost) {
-		struct llentry *ln;
-		struct rtentry rt;
-		struct sockaddr_dl gateway;
-		struct sockaddr_in6 mask, addr;
-
-		IF_AFDATA_LOCK(ifp);
-		ia->ia_ifa.ifa_rtrequest = NULL;
-
-		/* XXX QL
-		 * we need to report rt_newaddrmsg
-		 */
-		ln = lla_lookup(LLTABLE6(ifp), (LLE_CREATE | LLE_IFADDR | LLE_EXCLUSIVE),
-		    (struct sockaddr *)&ia->ia_addr);
-		IF_AFDATA_UNLOCK(ifp);
-		if (ln != NULL) {
-			ln->la_expire = 0;  /* for IPv6 this means permanent */
-			ln->ln_state = ND6_LLINFO_REACHABLE;
-			/*
-			 * initialize for rtmsg generation
-			 */
-			bzero(&gateway, sizeof(gateway));
-			gateway.sdl_len = sizeof(gateway);
-			gateway.sdl_family = AF_LINK;
-			gateway.sdl_nlen = 0;
-			gateway.sdl_alen = 6;
-			memcpy(gateway.sdl_data, &ln->ll_addr.mac_aligned, sizeof(ln->ll_addr));
-			/* */
-			LLE_WUNLOCK(ln);
-		}
-
-		bzero(&rt, sizeof(rt));
-		rt.rt_gateway = (struct sockaddr *)&gateway;
-		memcpy(&mask, &ia->ia_prefixmask, sizeof(ia->ia_prefixmask));
-		memcpy(&addr, &ia->ia_addr, sizeof(ia->ia_addr));
-		rt_mask(&rt) = (struct sockaddr *)&mask;
-		rt_key(&rt) = (struct sockaddr *)&addr;
-		rt.rt_flags = RTF_UP | RTF_HOST | RTF_STATIC;
-		rt_newaddrmsg(RTM_ADD, &ia->ia_ifa, 0, &rt);
-	}
+	if (newhost)
+		in6_ifaddloop(&(ia->ia_ifa));
 
 	return (error);
 }
diff --git a/sys/netinet6/in6.h b/sys/netinet6/in6.h
index 1040d90e94a0..2f986d42b93f 100644
--- a/sys/netinet6/in6.h
+++ b/sys/netinet6/in6.h
@@ -376,6 +376,8 @@ extern const struct in6_addr in6addr_linklocal_allv2routers;
 struct route_in6 {
 	struct	rtentry *ro_rt;
 	struct	llentry *ro_lle;
+	struct	in6_addr *ro_ia6;
+	int		ro_flags;
 	struct	sockaddr_in6 ro_dst;
 };
 #endif
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 85161247d7e6..e4d5172aebed 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -2367,6 +2367,8 @@ copypktopts(struct ip6_pktopts *dst, struct ip6_pktopts *src, int canwait)
 	dst->ip6po_hlim = src->ip6po_hlim;
 	dst->ip6po_tclass = src->ip6po_tclass;
 	dst->ip6po_flags = src->ip6po_flags;
+	dst->ip6po_minmtu = src->ip6po_minmtu;
+	dst->ip6po_prefer_tempaddr = src->ip6po_prefer_tempaddr;
 	if (src->ip6po_pktinfo) {
 		dst->ip6po_pktinfo = malloc(sizeof(*dst->ip6po_pktinfo),
 		    M_IP6OPT, canwait);
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 2b51e4379b3b..ae7cc4a8965b 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -1364,7 +1364,8 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp)
 				    " duplicate.\n");
 			} else {
 				ND_IFINFO(ifp)->flags &= ~ND6_IFF_IFDISABLED;
-				in6_if_up(ifp);
+				if (ifp->if_flags & IFF_UP)
+					in6_if_up(ifp);
 			}
 		} else if (!(ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) &&
 			    (ND.flags & ND6_IFF_IFDISABLED)) {
@@ -1382,35 +1383,37 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp)
 			IF_ADDR_UNLOCK(ifp);
 		}
 
-		if (!(ND_IFINFO(ifp)->flags & ND6_IFF_AUTO_LINKLOCAL) &&
-		    (ND.flags & ND6_IFF_AUTO_LINKLOCAL)) {
-			/* auto_linklocal 0->1 transision */
+		if (ND.flags & ND6_IFF_AUTO_LINKLOCAL) {
+			if (!(ND_IFINFO(ifp)->flags & ND6_IFF_AUTO_LINKLOCAL)) {
+				/* auto_linklocal 0->1 transision */
 
-			/* If no link-local address on ifp, configure */
-			ND_IFINFO(ifp)->flags |= ND6_IFF_AUTO_LINKLOCAL;
-			in6_ifattach(ifp, NULL);
-		} else if ((ND_IFINFO(ifp)->flags & ND6_IFF_AUTO_LINKLOCAL) &&
-		    !(ND.flags & ND6_IFF_IFDISABLED)) {
-			/*
-			 * When the IF already has
-			 * ND6_IFF_AUTO_LINKLOCAL and no link-local
-			 * address is assigned, try to assign one.
-			 */
-			int haslinklocal = 0;
-			
-			IF_ADDR_LOCK(ifp);
-			TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
-				if (ifa->ifa_addr->sa_family != AF_INET6)
-					continue;
-				ia = (struct in6_ifaddr *)ifa;
-				if (IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) {
-					haslinklocal = 1;
-					break;
-				}
-			}
-			IF_ADDR_UNLOCK(ifp);
-			if (!haslinklocal)
+				/* If no link-local address on ifp, configure */
+				ND_IFINFO(ifp)->flags |= ND6_IFF_AUTO_LINKLOCAL;
 				in6_ifattach(ifp, NULL);
+			} else if (!(ND.flags & ND6_IFF_IFDISABLED) &&
+			    ifp->if_flags & IFF_UP) {
+				/*
+				 * When the IF already has
+				 * ND6_IFF_AUTO_LINKLOCAL, no link-local
+				 * address is assigned, and IFF_UP, try to
+				 * assign one.
+				 */
+				int haslinklocal = 0;
+			
+				IF_ADDR_LOCK(ifp);
+				TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+					if (ifa->ifa_addr->sa_family != AF_INET6)
+						continue;
+					ia = (struct in6_ifaddr *)ifa;
+					if (IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) {
+						haslinklocal = 1;
+						break;
+					}
+				}
+				IF_ADDR_UNLOCK(ifp);
+				if (!haslinklocal)
+					in6_ifattach(ifp, NULL);
+			}
 		}
 	}
 		ND_IFINFO(ifp)->flags = ND.flags;
@@ -2042,14 +2045,15 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m0,
 		if (*chain == NULL)
 			*chain = m;
 		else {
-			struct mbuf *m = *chain;
+			struct mbuf *mb;
 
 			/*
 			 * append mbuf to end of deferred chain
 			 */
-			while (m->m_nextpkt != NULL)
-				m = m->m_nextpkt;
-			m->m_nextpkt = m;
+			mb = *chain;
+			while (mb->m_nextpkt != NULL)
+				mb = mb->m_nextpkt;
+			mb->m_nextpkt = m;
 		}
 		return (error);
 	}
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index fb8e37921283..0221c72b44e7 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -1167,11 +1167,11 @@ nd6_dad_find(struct ifaddr *ifa)
 {
 	struct dadq *dp;
 
-	for (dp = V_dadq.tqh_first; dp; dp = dp->dad_list.tqe_next) {
+	TAILQ_FOREACH(dp, &V_dadq, dad_list)
 		if (dp->dad_ifa == ifa)
-			return dp;
-	}
-	return NULL;
+			return (dp);
+
+	return (NULL);
 }
 
 static void
diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c
index aefc0ca99ece..6fe17cd1dd14 100644
--- a/sys/netipsec/key.c
+++ b/sys/netipsec/key.c
@@ -1764,6 +1764,7 @@ key_gather_mbuf(m, mhp, ndeep, nitem, va_alist)
 
 fail:
 	m_freem(result);
+	va_end(ap);
 	return NULL;
 }
 
diff --git a/sys/netncp/ncp_lib.h b/sys/netncp/ncp_lib.h
index 2e8eebe767a5..275b53b5c9a9 100644
--- a/sys/netncp/ncp_lib.h
+++ b/sys/netncp/ncp_lib.h
@@ -174,7 +174,7 @@ int  ncp_get_bindery_object_id(NWCONN_HANDLE, u_int16_t, const char *,
 int  ncp_get_bindery_object_name(NWCONN_HANDLE, u_int32_t,
 		struct ncp_bindery_object *);
 int  ncp_scan_bindery_object(NWCONN_HANDLE, u_int32_t, u_int16_t, 
-		char *, struct ncp_bindery_object *);
+		const char *, struct ncp_bindery_object *);
 int  ncp_read_property_value(NWCONN_HANDLE, int object_type, const char *,
 		int, const char *, struct nw_property *);
 int  ncp_get_encryption_key(NWCONN_HANDLE, char *);
diff --git a/sys/nfs/nfs_nfssvc.c b/sys/nfs/nfs_nfssvc.c
index 4296349f04cb..9680eb7e9708 100644
--- a/sys/nfs/nfs_nfssvc.c
+++ b/sys/nfs/nfs_nfssvc.c
@@ -73,7 +73,7 @@ int (*nfsd_call_nfsd)(struct thread *, struct nfssvc_args *) = NULL;
  * Nfs server psuedo system call for the nfsd's
  */
 int
-nfssvc(struct thread *td, struct nfssvc_args *uap)
+sys_nfssvc(struct thread *td, struct nfssvc_args *uap)
 {
 	int error;
 
diff --git a/sys/nlm/nlm_prot_impl.c b/sys/nlm/nlm_prot_impl.c
index afbb9ddd809c..8a4b3c411fc4 100644
--- a/sys/nlm/nlm_prot_impl.c
+++ b/sys/nlm/nlm_prot_impl.c
@@ -1718,7 +1718,7 @@ nlm_server_main(int addr_count, char **addrs)
 }
 
 int
-nlm_syscall(struct thread *td, struct nlm_syscall_args *uap)
+sys_nlm_syscall(struct thread *td, struct nlm_syscall_args *uap)
 {
 	int error;
 
diff --git a/sys/ofed/drivers/infiniband/ulp/sdp/Kconfig b/sys/ofed/drivers/infiniband/ulp/sdp/Kconfig
index b5fadf4452c3..551454f1bc63 100644
--- a/sys/ofed/drivers/infiniband/ulp/sdp/Kconfig
+++ b/sys/ofed/drivers/infiniband/ulp/sdp/Kconfig
@@ -21,7 +21,7 @@ config INFINIBAND_SDP_DEBUG_DATA
         bool "Sockets Direct Protocol data path debugging"
         depends on INFINIBAND_SDP_DEBUG
         ---help---
-          This option compiles debugging code into the the data path
+          This option compiles debugging code into the data path
           of the SDP driver.  The output can be turned on via the
           data_debug_level module parameter; however, even with output
           turned off, this debugging code will have some performance
diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c
index b42249caea99..c9e1a91eace5 100644
--- a/sys/pc98/pc98/machdep.c
+++ b/sys/pc98/pc98/machdep.c
@@ -935,7 +935,7 @@ freebsd4_sigreturn(td, uap)
  * MPSAFE
  */
 int
-sigreturn(td, uap)
+sys_sigreturn(td, uap)
 	struct thread *td;
 	struct sigreturn_args /* {
 		const struct __ucontext *sigcntxp;
@@ -1117,7 +1117,7 @@ void
 cpu_halt(void)
 {
 	for (;;)
-		__asm__ ("hlt");
+		halt();
 }
 
 static int	idle_mwait = 1;		/* Use MONITOR/MWAIT for short idle. */
@@ -1136,9 +1136,22 @@ cpu_idle_hlt(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_SLEEPING;
+
 	/*
-	 * We must absolutely guarentee that hlt is the next instruction
-	 * after sti or we introduce a timing window.
+	 * Since we may be in a critical section from cpu_idle(), if
+	 * an interrupt fires during that critical section we may have
+	 * a pending preemption.  If the CPU halts, then that thread
+	 * may not execute until a later interrupt awakens the CPU.
+	 * To handle this race, check for a runnable thread after
+	 * disabling interrupts and immediately return if one is
+	 * found.  Also, we must absolutely guarentee that hlt is
+	 * the next instruction after sti.  This ensures that any
+	 * interrupt that fires after the call to disable_intr() will
+	 * immediately awaken the CPU from hlt.  Finally, please note
+	 * that on x86 this works fine because of interrupts enabled only
+	 * after the instruction following sti takes place, while IF is set
+	 * to 1 immediately, allowing hlt instruction to acknowledge the
+	 * interrupt.
 	 */
 	disable_intr();
 	if (sched_runnable())
@@ -1164,11 +1177,19 @@ cpu_idle_mwait(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_MWAIT;
-	if (!sched_runnable()) {
-		cpu_monitor(state, 0, 0);
-		if (*state == STATE_MWAIT)
-			cpu_mwait(0, MWAIT_C1);
+
+	/* See comments in cpu_idle_hlt(). */
+	disable_intr();
+	if (sched_runnable()) {
+		enable_intr();
+		*state = STATE_RUNNING;
+		return;
 	}
+	cpu_monitor(state, 0, 0);
+	if (*state == STATE_MWAIT)
+		__asm __volatile("sti; mwait" : : "a" (MWAIT_C1), "c" (0));
+	else
+		enable_intr();
 	*state = STATE_RUNNING;
 }
 
@@ -1180,6 +1201,12 @@ cpu_idle_spin(int busy)
 
 	state = (int *)PCPU_PTR(monitorbuf);
 	*state = STATE_RUNNING;
+
+	/*
+	 * The sched_runnable() call is racy but as long as there is
+	 * a loop missing it one time will have just a little impact if any 
+	 * (and it is much better than missing the check at all).
+	 */
 	for (i = 0; i < 1000; i++) {
 		if (sched_runnable())
 			return;
diff --git a/sys/pci/if_rl.c b/sys/pci/if_rl.c
index 85250eb21ebb..87c2fc8c8e67 100644
--- a/sys/pci/if_rl.c
+++ b/sys/pci/if_rl.c
@@ -1932,9 +1932,9 @@ rl_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	RL_LOCK(sc);
 	mii_pollstat(mii);
-	RL_UNLOCK(sc);
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
+	RL_UNLOCK(sc);
 }
 
 static int
diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c
index 4d1043a2eea6..712967fc767b 100644
--- a/sys/powerpc/booke/pmap.c
+++ b/sys/powerpc/booke/pmap.c
@@ -1918,7 +1918,6 @@ mmu_booke_protect(mmu_t mmu, pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 	if (prot & VM_PROT_WRITE)
 		return;
 
-	vm_page_lock_queues();
 	PMAP_LOCK(pmap);
 	for (va = sva; va < eva; va += PAGE_SIZE) {
 		if ((pte = pte_find(mmu, pmap, va)) != NULL) {
@@ -1941,7 +1940,6 @@ mmu_booke_protect(mmu_t mmu, pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 		}
 	}
 	PMAP_UNLOCK(pmap);
-	vm_page_unlock_queues();
 }
 
 /*
diff --git a/sys/powerpc/conf/GENERIC b/sys/powerpc/conf/GENERIC
index 367ced5932d7..87e8ae21b2a1 100644
--- a/sys/powerpc/conf/GENERIC
+++ b/sys/powerpc/conf/GENERIC
@@ -171,6 +171,7 @@ device		kue		# Kawasaki LSI USB Ethernet
 
 # FireWire support
 device		firewire	# FireWire bus code
+# sbp(4) works for some systems but causes boot failure on others
 device		sbp		# SCSI over FireWire (Requires scbus and da)
 device		fwe		# Ethernet over FireWire (non-standard!)
 
diff --git a/sys/powerpc/include/ieeefp.h b/sys/powerpc/include/ieeefp.h
index 820fffe1cd9c..58a4fe85a85f 100644
--- a/sys/powerpc/include/ieeefp.h
+++ b/sys/powerpc/include/ieeefp.h
@@ -8,6 +8,8 @@
 #ifndef _MACHINE_IEEEFP_H_
 #define _MACHINE_IEEEFP_H_
 
+/* Deprecated historical FPU control interface */
+
 typedef int fp_except_t;
 #define FP_X_IMP	0x01	/* imprecise (loss of precision) */
 #define FP_X_DZ		0x02	/* divide-by-zero exception */
@@ -22,4 +24,12 @@ typedef enum {
     FP_RM=3			/* round toward negative infinity */
 } fp_rnd_t;
 
+__BEGIN_DECLS
+extern fp_rnd_t    fpgetround(void);
+extern fp_rnd_t    fpsetround(fp_rnd_t);
+extern fp_except_t fpgetmask(void);
+extern fp_except_t fpsetmask(fp_except_t);
+extern fp_except_t fpgetsticky(void);
+__END_DECLS
+
 #endif /* _MACHINE_IEEEFP_H_ */
diff --git a/sys/powerpc/include/proc.h b/sys/powerpc/include/proc.h
index 0842b8327f2f..627fbf2344e4 100644
--- a/sys/powerpc/include/proc.h
+++ b/sys/powerpc/include/proc.h
@@ -60,7 +60,6 @@ struct syscall_args {
 	register_t args[10];
 	int narg;
 };
-#define	HAVE_SYSCALL_ARGS_DEF 1
 #endif
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/powerpc/powerpc/bus_machdep.c b/sys/powerpc/powerpc/bus_machdep.c
index 2d2f6ee8ed93..6372d51ae366 100644
--- a/sys/powerpc/powerpc/bus_machdep.c
+++ b/sys/powerpc/powerpc/bus_machdep.c
@@ -293,6 +293,7 @@ bs_be_ws_1(bus_space_handle_t bsh, bus_size_t ofs, uint8_t val)
 
 	addr = __ppc_ba(bsh, ofs);
 	*addr = val;
+	__asm __volatile("eieio; sync");
 	CTR4(KTR_BE_IO, "%s(bsh=%#x, ofs=%#x, val=%#x)", __func__, bsh, ofs, val);
 }
 
@@ -303,6 +304,7 @@ bs_be_ws_2(bus_space_handle_t bsh, bus_size_t ofs, uint16_t val)
 
 	addr = __ppc_ba(bsh, ofs);
 	*addr = val;
+	__asm __volatile("eieio; sync");
 	CTR4(KTR_BE_IO, "%s(bsh=%#x, ofs=%#x, val=%#x)", __func__, bsh, ofs, val);
 }
 
@@ -313,6 +315,7 @@ bs_be_ws_4(bus_space_handle_t bsh, bus_size_t ofs, uint32_t val)
 
 	addr = __ppc_ba(bsh, ofs);
 	*addr = val;
+	__asm __volatile("eieio; sync");
 	CTR4(KTR_BE_IO, "%s(bsh=%#x, ofs=%#x, val=%#x)", __func__, bsh, ofs, val);
 }
 
@@ -323,6 +326,7 @@ bs_be_ws_8(bus_space_handle_t bsh, bus_size_t ofs, uint64_t val)
 
 	addr = __ppc_ba(bsh, ofs);
 	*addr = val;
+	__asm __volatile("eieio; sync");
 }
 
 static void
@@ -488,6 +492,7 @@ bs_le_rs_1(bus_space_handle_t bsh, bus_size_t ofs)
 
 	addr = __ppc_ba(bsh, ofs);
 	res = *addr;
+	__asm __volatile("eieio; sync");
 	CTR4(KTR_LE_IO, "%s(bsh=%#x, ofs=%#x) = %#x", __func__, bsh, ofs, res);
 	return (res);
 }
@@ -500,6 +505,7 @@ bs_le_rs_2(bus_space_handle_t bsh, bus_size_t ofs)
 
 	addr = __ppc_ba(bsh, ofs);
 	__asm __volatile("lhbrx %0, 0, %1" : "=r"(res) : "r"(addr));
+	__asm __volatile("eieio; sync");
 	CTR4(KTR_LE_IO, "%s(bsh=%#x, ofs=%#x) = %#x", __func__, bsh, ofs, res);
 	return (res);
 }
@@ -512,6 +518,7 @@ bs_le_rs_4(bus_space_handle_t bsh, bus_size_t ofs)
 
 	addr = __ppc_ba(bsh, ofs);
 	__asm __volatile("lwbrx %0, 0, %1" : "=r"(res) : "r"(addr));
+	__asm __volatile("eieio; sync");
 	CTR4(KTR_LE_IO, "%s(bsh=%#x, ofs=%#x) = %#x", __func__, bsh, ofs, res);
 	return (res);
 }
diff --git a/sys/powerpc/powerpc/exec_machdep.c b/sys/powerpc/powerpc/exec_machdep.c
index 107605168980..ec88d9faaea7 100644
--- a/sys/powerpc/powerpc/exec_machdep.c
+++ b/sys/powerpc/powerpc/exec_machdep.c
@@ -312,7 +312,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 }
 
 int
-sigreturn(struct thread *td, struct sigreturn_args *uap)
+sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
 {
 	ucontext_t uc;
 	int error;
@@ -341,7 +341,7 @@ int
 freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
 {
 
-	return sigreturn(td, (struct sigreturn_args *)uap);
+	return sys_sigreturn(td, (struct sigreturn_args *)uap);
 }
 #endif
 
diff --git a/sys/powerpc/ps3/ps3cdrom.c b/sys/powerpc/ps3/ps3cdrom.c
index 319090cacb56..ba3c3adb0f13 100644
--- a/sys/powerpc/ps3/ps3cdrom.c
+++ b/sys/powerpc/ps3/ps3cdrom.c
@@ -506,21 +506,19 @@ ps3cdrom_intr(void *arg)
 
 			if (!ps3cdrom_decode_lv1_status(status, &sense_key,
 			    &asc, &ascq)) {
-				struct scsi_sense_data sense_data;
 
 				CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
 				   ("sense key 0x%02x asc 0x%02x ascq 0x%02x\n",
 				    sense_key, asc, ascq));
 
-				bzero(&sense_data, sizeof(sense_data));
-				sense_data.error_code = SSD_CURRENT_ERROR;
-				sense_data.flags |= sense_key;
-				sense_data.extra_len = 0xa;
-				sense_data.add_sense_code = asc;
-				sense_data.add_sense_code_qual = ascq;
-				ccb->csio.sense_len = sizeof(sense_data);
-				bcopy(&sense_data, &ccb->csio.sense_data,
-				    ccb->csio.sense_len);
+				scsi_set_sense_data(&ccb->csio.sense_data,
+				    /*sense_format*/ SSD_TYPE_NONE,
+				    /*current_error*/ 1,
+				    sense_key,
+				    asc,
+				    ascq,
+				    SSD_ELEM_NONE);
+				ccb->csio.sense_len = SSD_FULL_SIZE;
 				ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR |
 				    CAM_AUTOSNS_VALID;
 			}
@@ -643,8 +641,6 @@ ps3cdrom_transfer(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 	}
 
 	if (err) {
-		struct scsi_sense_data sense_data;
-
 		device_printf(dev, "ATAPI command 0x%02x failed (%d)\n",
 		    cdb[0], err);
 
@@ -653,11 +649,18 @@ ps3cdrom_transfer(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 		xp->x_ccb = NULL;
 		TAILQ_INSERT_TAIL(&sc->sc_free_xferq, xp, x_queue);
 
-		bzero(&sense_data, sizeof(sense_data));
-		sense_data.error_code = SSD_CURRENT_ERROR;
-		sense_data.flags |= SSD_KEY_ILLEGAL_REQUEST;
-		ccb->csio.sense_len = sizeof(sense_data);
-		bcopy(&sense_data, &ccb->csio.sense_data, ccb->csio.sense_len);
+		bzero(&ccb->csio.sense_data, sizeof(ccb->csio.sense_data));
+		/* Invalid field in parameter list */
+		scsi_set_sense_data(&ccb->csio.sense_data,
+				    /*sense_format*/ SSD_TYPE_NONE,
+				    /*current_error*/ 1,
+				    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
+				    /*asc*/ 0x26,
+				    /*ascq*/ 0x00,
+				    SSD_ELEM_NONE);
+
+		ccb->csio.sense_len = SSD_FULL_SIZE;
+		ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
 		ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
 		xpt_done(ccb);
 	} else {
diff --git a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
index 7433b79d2bbe..a4253a4d1446 100644
--- a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
+++ b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
@@ -608,6 +608,22 @@ svc_rpc_gss_release_client(struct svc_rpc_gss_client *client)
 	svc_rpc_gss_destroy_client(client);
 }
 
+/*
+ * Remove a client from our global lists.
+ * Must be called with svc_rpc_gss_lock held.
+ */
+static void
+svc_rpc_gss_forget_client_locked(struct svc_rpc_gss_client *client)
+{
+	struct svc_rpc_gss_client_list *list;
+
+	sx_assert(&svc_rpc_gss_lock, SX_XLOCKED);
+	list = &svc_rpc_gss_client_hash[client->cl_id.ci_id % CLIENT_HASH_SIZE];
+	TAILQ_REMOVE(list, client, cl_link);
+	TAILQ_REMOVE(&svc_rpc_gss_clients, client, cl_alllink);
+	svc_rpc_gss_client_count--;
+}
+
 /*
  * Remove a client from our global lists and free it if we can.
  */
@@ -615,21 +631,30 @@ static void
 svc_rpc_gss_forget_client(struct svc_rpc_gss_client *client)
 {
 	struct svc_rpc_gss_client_list *list;
+	struct svc_rpc_gss_client *tclient;
 
 	list = &svc_rpc_gss_client_hash[client->cl_id.ci_id % CLIENT_HASH_SIZE];
 	sx_xlock(&svc_rpc_gss_lock);
-	TAILQ_REMOVE(list, client, cl_link);
-	TAILQ_REMOVE(&svc_rpc_gss_clients, client, cl_alllink);
-	svc_rpc_gss_client_count--;
+	TAILQ_FOREACH(tclient, list, cl_link) {
+		/*
+		 * Make sure this client has not already been removed
+		 * from the lists by svc_rpc_gss_forget_client() or
+		 * svc_rpc_gss_forget_client_locked().
+		 */
+		if (client == tclient) {
+			svc_rpc_gss_forget_client_locked(client);
+			sx_xunlock(&svc_rpc_gss_lock);
+			svc_rpc_gss_release_client(client);
+			return;
+		}
+	}
 	sx_xunlock(&svc_rpc_gss_lock);
-	svc_rpc_gss_release_client(client);
 }
 
 static void
 svc_rpc_gss_timeout_clients(void)
 {
 	struct svc_rpc_gss_client *client;
-	struct svc_rpc_gss_client *nclient;
 	time_t now = time_uptime;
 
 	rpc_gss_log_debug("in svc_rpc_gss_timeout_clients()");
@@ -638,16 +663,29 @@ svc_rpc_gss_timeout_clients(void)
 	 * First enforce the max client limit. We keep
 	 * svc_rpc_gss_clients in LRU order.
 	 */
-	while (svc_rpc_gss_client_count > CLIENT_MAX)
-		svc_rpc_gss_forget_client(TAILQ_LAST(&svc_rpc_gss_clients,
-			    svc_rpc_gss_client_list));
-	TAILQ_FOREACH_SAFE(client, &svc_rpc_gss_clients, cl_alllink, nclient) {
+	sx_xlock(&svc_rpc_gss_lock);
+	client = TAILQ_LAST(&svc_rpc_gss_clients, svc_rpc_gss_client_list);
+	while (svc_rpc_gss_client_count > CLIENT_MAX && client != NULL) {
+		svc_rpc_gss_forget_client_locked(client);
+		sx_xunlock(&svc_rpc_gss_lock);
+		svc_rpc_gss_release_client(client);
+		sx_xlock(&svc_rpc_gss_lock);
+		client = TAILQ_LAST(&svc_rpc_gss_clients,
+		    svc_rpc_gss_client_list);
+	}
+again:
+	TAILQ_FOREACH(client, &svc_rpc_gss_clients, cl_alllink) {
 		if (client->cl_state == CLIENT_STALE
 		    || now > client->cl_expiration) {
+			svc_rpc_gss_forget_client_locked(client);
+			sx_xunlock(&svc_rpc_gss_lock);
 			rpc_gss_log_debug("expiring client %p", client);
-			svc_rpc_gss_forget_client(client);
+			svc_rpc_gss_release_client(client);
+			sx_xlock(&svc_rpc_gss_lock);
+			goto again;
 		}
 	}
+	sx_xunlock(&svc_rpc_gss_lock);
 }
 
 #ifdef DEBUG
diff --git a/sys/security/audit/audit_pipe.c b/sys/security/audit/audit_pipe.c
index a953eb0200da..dc0df3e8ae4e 100644
--- a/sys/security/audit/audit_pipe.c
+++ b/sys/security/audit/audit_pipe.c
@@ -231,7 +231,7 @@ static d_kqfilter_t	audit_pipe_kqfilter;
 
 static struct cdevsw	audit_pipe_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_PSEUDO | D_NEEDMINOR,
+	.d_flags =	D_NEEDMINOR,
 	.d_open =	audit_pipe_open,
 	.d_close =	audit_pipe_close,
 	.d_read =	audit_pipe_read,
diff --git a/sys/security/audit/audit_syscalls.c b/sys/security/audit/audit_syscalls.c
index 075aac549ac9..bf608746497e 100644
--- a/sys/security/audit/audit_syscalls.c
+++ b/sys/security/audit/audit_syscalls.c
@@ -60,7 +60,7 @@ __FBSDID("$FreeBSD$");
  */
 /* ARGSUSED */
 int
-audit(struct thread *td, struct audit_args *uap)
+sys_audit(struct thread *td, struct audit_args *uap)
 {
 	int error;
 	void * rec;
@@ -154,7 +154,7 @@ audit(struct thread *td, struct audit_args *uap)
  */
 /* ARGSUSED */
 int
-auditon(struct thread *td, struct auditon_args *uap)
+sys_auditon(struct thread *td, struct auditon_args *uap)
 {
 	struct ucred *cred, *newcred, *oldcred;
 	int error;
@@ -560,7 +560,7 @@ auditon(struct thread *td, struct auditon_args *uap)
  */
 /* ARGSUSED */
 int
-getauid(struct thread *td, struct getauid_args *uap)
+sys_getauid(struct thread *td, struct getauid_args *uap)
 {
 	int error;
 
@@ -575,7 +575,7 @@ getauid(struct thread *td, struct getauid_args *uap)
 
 /* ARGSUSED */
 int
-setauid(struct thread *td, struct setauid_args *uap)
+sys_setauid(struct thread *td, struct setauid_args *uap)
 {
 	struct ucred *newcred, *oldcred;
 	au_id_t id;
@@ -615,7 +615,7 @@ setauid(struct thread *td, struct setauid_args *uap)
  */
 /* ARGSUSED */
 int
-getaudit(struct thread *td, struct getaudit_args *uap)
+sys_getaudit(struct thread *td, struct getaudit_args *uap)
 {
 	struct auditinfo ai;
 	struct ucred *cred;
@@ -640,7 +640,7 @@ getaudit(struct thread *td, struct getaudit_args *uap)
 
 /* ARGSUSED */
 int
-setaudit(struct thread *td, struct setaudit_args *uap)
+sys_setaudit(struct thread *td, struct setaudit_args *uap)
 {
 	struct ucred *newcred, *oldcred;
 	struct auditinfo ai;
@@ -683,7 +683,7 @@ setaudit(struct thread *td, struct setaudit_args *uap)
 
 /* ARGSUSED */
 int
-getaudit_addr(struct thread *td, struct getaudit_addr_args *uap)
+sys_getaudit_addr(struct thread *td, struct getaudit_addr_args *uap)
 {
 	int error;
 
@@ -700,7 +700,7 @@ getaudit_addr(struct thread *td, struct getaudit_addr_args *uap)
 
 /* ARGSUSED */
 int
-setaudit_addr(struct thread *td, struct setaudit_addr_args *uap)
+sys_setaudit_addr(struct thread *td, struct setaudit_addr_args *uap)
 {
 	struct ucred *newcred, *oldcred;
 	struct auditinfo_addr aia;
@@ -743,7 +743,7 @@ setaudit_addr(struct thread *td, struct setaudit_addr_args *uap)
  */
 /* ARGSUSED */
 int
-auditctl(struct thread *td, struct auditctl_args *uap)
+sys_auditctl(struct thread *td, struct auditctl_args *uap)
 {
 	struct nameidata nd;
 	struct ucred *cred;
@@ -813,63 +813,63 @@ auditctl(struct thread *td, struct auditctl_args *uap)
 #else /* !AUDIT */
 
 int
-audit(struct thread *td, struct audit_args *uap)
+sys_audit(struct thread *td, struct audit_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-auditon(struct thread *td, struct auditon_args *uap)
+sys_auditon(struct thread *td, struct auditon_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-getauid(struct thread *td, struct getauid_args *uap)
+sys_getauid(struct thread *td, struct getauid_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-setauid(struct thread *td, struct setauid_args *uap)
+sys_setauid(struct thread *td, struct setauid_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-getaudit(struct thread *td, struct getaudit_args *uap)
+sys_getaudit(struct thread *td, struct getaudit_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-setaudit(struct thread *td, struct setaudit_args *uap)
+sys_setaudit(struct thread *td, struct setaudit_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-getaudit_addr(struct thread *td, struct getaudit_addr_args *uap)
+sys_getaudit_addr(struct thread *td, struct getaudit_addr_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-setaudit_addr(struct thread *td, struct setaudit_addr_args *uap)
+sys_setaudit_addr(struct thread *td, struct setaudit_addr_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-auditctl(struct thread *td, struct auditctl_args *uap)
+sys_auditctl(struct thread *td, struct auditctl_args *uap)
 {
 
 	return (ENOSYS);
diff --git a/sys/security/mac/mac_syscalls.c b/sys/security/mac/mac_syscalls.c
index dc275477be50..51392d784faf 100644
--- a/sys/security/mac/mac_syscalls.c
+++ b/sys/security/mac/mac_syscalls.c
@@ -77,7 +77,7 @@ __FBSDID("$FreeBSD$");
 FEATURE(security_mac, "Mandatory Access Control Framework support");
 
 int
-__mac_get_pid(struct thread *td, struct __mac_get_pid_args *uap)
+sys___mac_get_pid(struct thread *td, struct __mac_get_pid_args *uap)
 {
 	char *elements, *buffer;
 	struct mac mac;
@@ -126,7 +126,7 @@ __mac_get_pid(struct thread *td, struct __mac_get_pid_args *uap)
 }
 
 int
-__mac_get_proc(struct thread *td, struct __mac_get_proc_args *uap)
+sys___mac_get_proc(struct thread *td, struct __mac_get_proc_args *uap)
 {
 	char *elements, *buffer;
 	struct mac mac;
@@ -159,7 +159,7 @@ __mac_get_proc(struct thread *td, struct __mac_get_proc_args *uap)
 }
 
 int
-__mac_set_proc(struct thread *td, struct __mac_set_proc_args *uap)
+sys___mac_set_proc(struct thread *td, struct __mac_set_proc_args *uap)
 {
 	struct ucred *newcred, *oldcred;
 	struct label *intlabel;
@@ -220,7 +220,7 @@ __mac_set_proc(struct thread *td, struct __mac_set_proc_args *uap)
 }
 
 int
-__mac_get_fd(struct thread *td, struct __mac_get_fd_args *uap)
+sys___mac_get_fd(struct thread *td, struct __mac_get_fd_args *uap)
 {
 	char *elements, *buffer;
 	struct label *intlabel;
@@ -310,7 +310,7 @@ __mac_get_fd(struct thread *td, struct __mac_get_fd_args *uap)
 }
 
 int
-__mac_get_file(struct thread *td, struct __mac_get_file_args *uap)
+sys___mac_get_file(struct thread *td, struct __mac_get_file_args *uap)
 {
 	char *elements, *buffer;
 	struct nameidata nd;
@@ -363,7 +363,7 @@ __mac_get_file(struct thread *td, struct __mac_get_file_args *uap)
 }
 
 int
-__mac_get_link(struct thread *td, struct __mac_get_link_args *uap)
+sys___mac_get_link(struct thread *td, struct __mac_get_link_args *uap)
 {
 	char *elements, *buffer;
 	struct nameidata nd;
@@ -416,7 +416,7 @@ __mac_get_link(struct thread *td, struct __mac_get_link_args *uap)
 }
 
 int
-__mac_set_fd(struct thread *td, struct __mac_set_fd_args *uap)
+sys___mac_set_fd(struct thread *td, struct __mac_set_fd_args *uap)
 {
 	struct label *intlabel;
 	struct pipe *pipe;
@@ -512,7 +512,7 @@ __mac_set_fd(struct thread *td, struct __mac_set_fd_args *uap)
 }
 
 int
-__mac_set_file(struct thread *td, struct __mac_set_file_args *uap)
+sys___mac_set_file(struct thread *td, struct __mac_set_file_args *uap)
 {
 	struct label *intlabel;
 	struct nameidata nd;
@@ -566,7 +566,7 @@ __mac_set_file(struct thread *td, struct __mac_set_file_args *uap)
 }
 
 int
-__mac_set_link(struct thread *td, struct __mac_set_link_args *uap)
+sys___mac_set_link(struct thread *td, struct __mac_set_link_args *uap)
 {
 	struct label *intlabel;
 	struct nameidata nd;
@@ -620,7 +620,7 @@ __mac_set_link(struct thread *td, struct __mac_set_link_args *uap)
 }
 
 int
-mac_syscall(struct thread *td, struct mac_syscall_args *uap)
+sys_mac_syscall(struct thread *td, struct mac_syscall_args *uap)
 {
 	struct mac_policy_conf *mpc;
 	char target[MAC_MAX_POLICY_NAME];
@@ -659,70 +659,70 @@ mac_syscall(struct thread *td, struct mac_syscall_args *uap)
 #else /* !MAC */
 
 int
-__mac_get_pid(struct thread *td, struct __mac_get_pid_args *uap)
+sys___mac_get_pid(struct thread *td, struct __mac_get_pid_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_get_proc(struct thread *td, struct __mac_get_proc_args *uap)
+sys___mac_get_proc(struct thread *td, struct __mac_get_proc_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_set_proc(struct thread *td, struct __mac_set_proc_args *uap)
+sys___mac_set_proc(struct thread *td, struct __mac_set_proc_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_get_fd(struct thread *td, struct __mac_get_fd_args *uap)
+sys___mac_get_fd(struct thread *td, struct __mac_get_fd_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_get_file(struct thread *td, struct __mac_get_file_args *uap)
+sys___mac_get_file(struct thread *td, struct __mac_get_file_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_get_link(struct thread *td, struct __mac_get_link_args *uap)
+sys___mac_get_link(struct thread *td, struct __mac_get_link_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_set_fd(struct thread *td, struct __mac_set_fd_args *uap)
+sys___mac_set_fd(struct thread *td, struct __mac_set_fd_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_set_file(struct thread *td, struct __mac_set_file_args *uap)
+sys___mac_set_file(struct thread *td, struct __mac_set_file_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-__mac_set_link(struct thread *td, struct __mac_set_link_args *uap)
+sys___mac_set_link(struct thread *td, struct __mac_set_link_args *uap)
 {
 
 	return (ENOSYS);
 }
 
 int
-mac_syscall(struct thread *td, struct mac_syscall_args *uap)
+sys_mac_syscall(struct thread *td, struct mac_syscall_args *uap)
 {
 
 	return (ENOSYS);
diff --git a/sys/security/mac_mls/mac_mls.c b/sys/security/mac_mls/mac_mls.c
index b68790d0259c..73c5df0e426d 100644
--- a/sys/security/mac_mls/mac_mls.c
+++ b/sys/security/mac_mls/mac_mls.c
@@ -1636,9 +1636,6 @@ mls_posixshm_check_open(struct ucred *cred, struct shmfd *shmfd,
 	subj = SLOT(cred->cr_label);
 	obj = SLOT(shmlabel);
 
-	subj = SLOT(cred->cr_label);
-	obj = SLOT(shmlabel);
-
 	if (accmode & (VREAD | VEXEC | VSTAT_PERMS)) {
 		if (!mls_dominate_effective(subj, obj))
 			return (EACCES);
diff --git a/sys/sparc64/central/central.c b/sys/sparc64/central/central.c
index 777660fdfebb..e418a8be458d 100644
--- a/sys/sparc64/central/central.c
+++ b/sys/sparc64/central/central.c
@@ -60,6 +60,7 @@ static device_attach_t central_attach;
 static bus_print_child_t central_print_child;
 static bus_probe_nomatch_t central_probe_nomatch;
 static bus_alloc_resource_t central_alloc_resource;
+static bus_adjust_resource_t central_adjust_resource;
 static bus_get_resource_list_t central_get_resource_list;
 static ofw_bus_get_devinfo_t central_get_devinfo;
 
@@ -79,6 +80,7 @@ static device_method_t central_methods[] = {
 	DEVMETHOD(bus_alloc_resource,	central_alloc_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	central_adjust_resource),
 	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
 	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
@@ -179,6 +181,15 @@ central_attach(device_t dev)
 	return (bus_generic_attach(dev));
 }
 
+static int
+central_adjust_resource(device_t bus __unused, device_t child __unused,
+    int type __unused, struct resource *r __unused, u_long start __unused,
+    u_long end __unused)
+{
+
+	return (ENXIO);
+}
+
 static int
 central_print_child(device_t dev, device_t child)
 {
diff --git a/sys/sparc64/conf/DEFAULTS b/sys/sparc64/conf/DEFAULTS
index 38b2408bf449..c99480c8f86f 100644
--- a/sys/sparc64/conf/DEFAULTS
+++ b/sys/sparc64/conf/DEFAULTS
@@ -19,3 +19,5 @@ options 	GEOM_PART_VTOC8
 
 # Let sunkbd emulate an AT keyboard by default.
 options 	SUNKBD_EMULATE_ATKBD
+
+options 	NEW_PCIB
diff --git a/sys/sparc64/conf/GENERIC b/sys/sparc64/conf/GENERIC
index fed0272734b6..20ecdf7c5d32 100644
--- a/sys/sparc64/conf/GENERIC
+++ b/sys/sparc64/conf/GENERIC
@@ -261,7 +261,8 @@ device		zyd		# ZyDAS zd1211/zd1211b wireless NICs
 
 # FireWire support
 device		firewire	# FireWire bus code
-device		sbp		# SCSI over FireWire (Requires scbus and da)
+# sbp(4) works for some systems but causes boot failure on others
+#device		sbp		# SCSI over FireWire (Requires scbus and da)
 device		fwe		# Ethernet over FireWire (non-standard!)
 device		fwip		# IP over FireWire (RFC 2734,3146)
 device		dcons		# Dumb console driver
diff --git a/sys/sparc64/ebus/ebus.c b/sys/sparc64/ebus/ebus.c
index 317cdfa3b958..42bac06c1919 100644
--- a/sys/sparc64/ebus/ebus.c
+++ b/sys/sparc64/ebus/ebus.c
@@ -136,6 +136,8 @@ static device_attach_t ebus_pci_attach;
 static bus_print_child_t ebus_print_child;
 static bus_probe_nomatch_t ebus_probe_nomatch;
 static bus_alloc_resource_t ebus_alloc_resource;
+static bus_activate_resource_t ebus_activate_resource;
+static bus_adjust_resource_t ebus_adjust_resource;
 static bus_release_resource_t ebus_release_resource;
 static bus_setup_intr_t ebus_setup_intr;
 static bus_get_resource_list_t ebus_get_resource_list;
@@ -161,8 +163,9 @@ static device_method_t ebus_nexus_methods[] = {
 	DEVMETHOD(bus_print_child,	ebus_print_child),
 	DEVMETHOD(bus_probe_nomatch,	ebus_probe_nomatch),
 	DEVMETHOD(bus_alloc_resource,	ebus_alloc_resource),
-	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
+	DEVMETHOD(bus_activate_resource, ebus_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	ebus_adjust_resource),
 	DEVMETHOD(bus_release_resource,	ebus_release_resource),
 	DEVMETHOD(bus_setup_intr,	ebus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
@@ -342,16 +345,10 @@ ebus_pci_attach(device_t dev)
 		eri->eri_res = res;
 		eri->eri_rman.rm_type = RMAN_ARRAY;
 		eri->eri_rman.rm_descr = "EBus range";
-		if (rman_init(&eri->eri_rman) != 0) {
+		if (rman_init_from_resource(&eri->eri_rman, res) != 0) {
 			printf("%s: failed to initialize rman!", __func__);
 			goto fail;
 		}
-		if (rman_manage_region(&eri->eri_rman, rman_get_start(res),
-		    rman_get_end(res)) != 0) {
-			printf("%s: failed to register region!", __func__);
-			rman_fini(&eri->eri_rman);
-			goto fail;
-		}
 	}
 	return (ebus_attach(dev, sc, node));
 
@@ -423,12 +420,10 @@ ebus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	struct resource_list *rl;
 	struct resource_list_entry *rle = NULL;
 	struct resource *res;
-	struct ebus_rinfo *ri;
+	struct ebus_rinfo *eri;
 	struct ebus_nexus_ranges *enr;
-	bus_space_tag_t bt;
-	bus_space_handle_t bh;
 	uint64_t cend, cstart, offset;
-	int i, isdefault, passthrough, ridx, rv;
+	int i, isdefault, passthrough, ridx;
 
 	isdefault = (start == 0UL && end == ~0UL);
 	passthrough = (device_get_parent(child) != bus);
@@ -459,23 +454,17 @@ ebus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 			 */
 			(void)ofw_isa_range_map(sc->sc_range, sc->sc_nrange,
 			    &start, &end, &ridx);
-			ri = &sc->sc_rinfo[ridx];
-			res = rman_reserve_resource(&ri->eri_rman, start, end,
-			    count, flags, child);
+			eri = &sc->sc_rinfo[ridx];
+			res = rman_reserve_resource(&eri->eri_rman, start,
+			    end, count, flags & ~RF_ACTIVE, child);
 			if (res == NULL)
 				return (NULL);
 			rman_set_rid(res, *rid);
-			bt = rman_get_bustag(ri->eri_res);
-			rman_set_bustag(res, bt);
-			rv = bus_space_subregion(bt,
-			    rman_get_bushandle(ri->eri_res),
-			    rman_get_start(res) - rman_get_start(ri->eri_res),
-			    count, &bh);
-			if (rv != 0) {
+			if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(
+			    child, type, *rid, res) != 0) {
 				rman_release_resource(res);
 				return (NULL);
 			}
-			rman_set_bushandle(res, bh);
 		} else {
 			/* Map EBus ranges to nexus ranges. */
 			for (i = 0; i < sc->sc_nrange; i++) {
@@ -496,7 +485,6 @@ ebus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 					break;
 				}
 			}
-
 		}
 		if (!passthrough)
 			rle->res = res;
@@ -508,6 +496,48 @@ ebus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	return (NULL);
 }
 
+static int
+ebus_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *res)
+{
+	struct ebus_softc *sc;
+	struct ebus_rinfo *eri;
+	bus_space_tag_t bt;
+	bus_space_handle_t bh;
+	int i, rv;
+
+	sc = device_get_softc(bus);
+	if ((sc->sc_flags & EBUS_PCI) != 0 && type == SYS_RES_MEMORY) {
+		for (i = 0; i < sc->sc_nrange; i++) {
+			eri = &sc->sc_rinfo[i];
+			if (rman_is_region_manager(res, &eri->eri_rman) != 0) {
+				bt = rman_get_bustag(eri->eri_res);
+				rv = bus_space_subregion(bt,
+				    rman_get_bushandle(eri->eri_res),
+				    rman_get_start(res) -
+				    rman_get_start(eri->eri_res),
+				    rman_get_size(res), &bh);
+				if (rv != 0)
+					return (rv);
+				rman_set_bustag(res, bt);
+				rman_set_bushandle(res, bh);
+				return (rman_activate_resource(res));
+			}
+		}
+		return (EINVAL);
+	}
+	return (bus_generic_activate_resource(bus, child, type, rid, res));
+}
+
+static int
+ebus_adjust_resource(device_t bus __unused, device_t child __unused,
+    int type __unused, struct resource *res __unused, u_long start __unused,
+    u_long end __unused)
+{
+
+	return (ENXIO);
+}
+
 static int
 ebus_release_resource(device_t bus, device_t child, int type, int rid,
     struct resource *res)
@@ -519,13 +549,15 @@ ebus_release_resource(device_t bus, device_t child, int type, int rid,
 
 	passthrough = (device_get_parent(child) != bus);
 	rl = BUS_GET_RESOURCE_LIST(bus, child);
-	switch (type) {
-	case SYS_RES_MEMORY:
-		sc = device_get_softc(bus);
-		if ((sc->sc_flags & EBUS_PCI) == 0)
-			return (resource_list_release(rl, bus, child, type,
-			    rid, res));
-		if ((rv = rman_release_resource(res)) != 0)
+	sc = device_get_softc(bus);
+	if ((sc->sc_flags & EBUS_PCI) != 0 && type == SYS_RES_MEMORY) {
+		if ((rman_get_flags(res) & RF_ACTIVE) != 0 ){
+			rv = bus_deactivate_resource(child, type, rid, res);
+			if (rv != 0)
+				return (rv);
+		}
+		rv = rman_release_resource(res);
+		if (rv != 0)
 			return (rv);
 		if (!passthrough) {
 			rle = resource_list_find(rl, type, rid);
@@ -535,13 +567,9 @@ ebus_release_resource(device_t bus, device_t child, int type, int rid,
 			   ("%s: resource entry is not busy", __func__));
 			rle->res = NULL;
 		}
-		break;
-	case SYS_RES_IRQ:
-		return (resource_list_release(rl, bus, child, type, rid, res));
-	default:
-		panic("%s: unsupported resource type %d", __func__, type);
+		return (0);
 	}
-	return (0);
+	return (resource_list_release(rl, bus, child, type, rid, res));
 }
 
 static int
diff --git a/sys/sparc64/fhc/fhc.c b/sys/sparc64/fhc/fhc.c
index 2b15e5e3adc5..0200bb1ffdb5 100644
--- a/sys/sparc64/fhc/fhc.c
+++ b/sys/sparc64/fhc/fhc.c
@@ -69,6 +69,7 @@ static bus_print_child_t fhc_print_child;
 static bus_probe_nomatch_t fhc_probe_nomatch;
 static bus_setup_intr_t fhc_setup_intr;
 static bus_alloc_resource_t fhc_alloc_resource;
+static bus_adjust_resource_t fhc_adjust_resource;
 static bus_get_resource_list_t fhc_get_resource_list;
 static ofw_bus_get_devinfo_t fhc_get_devinfo;
 
@@ -93,6 +94,7 @@ static device_method_t fhc_methods[] = {
 	DEVMETHOD(bus_alloc_resource,	fhc_alloc_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	fhc_adjust_resource),
 	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
 	DEVMETHOD(bus_setup_intr,	fhc_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
@@ -475,6 +477,15 @@ fhc_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	return (res);
 }
 
+static int
+fhc_adjust_resource(device_t bus __unused, device_t child __unused,
+    int type __unused, struct resource *r __unused, u_long start __unused,
+    u_long end __unused)
+{
+
+	return (ENXIO);
+}
+
 static struct resource_list *
 fhc_get_resource_list(device_t bus, device_t child)
 {
diff --git a/sys/sparc64/include/asmacros.h b/sys/sparc64/include/asmacros.h
index 777b35a4d184..99989177b5ef 100644
--- a/sys/sparc64/include/asmacros.h
+++ b/sys/sparc64/include/asmacros.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2001 Jake Burkholder.
+ * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -49,98 +50,156 @@
 /*
  * Atomically decrement an integer in memory.
  */
-#define	ATOMIC_DEC_INT(r1, r2, r3) \
-	lduw	[r1], r2 ; \
-9:	sub	r2, 1, r3 ; \
-	casa	[r1] ASI_N, r2, r3 ; \
-	cmp	r2, r3 ; \
-	bne,pn	%icc, 9b ; \
+#define	ATOMIC_DEC_INT(r1, r2, r3)					\
+	lduw	[r1], r2 ;						\
+9:	sub	r2, 1, r3 ;						\
+	casa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%icc, 9b ;						\
 	 mov	r3, r2
 
 /*
  * Atomically increment an integer in memory.
  */
-#define	ATOMIC_INC_INT(r1, r2, r3) \
-	lduw	[r1], r2 ; \
-9:	add	r2, 1, r3 ; \
-	casa	[r1] ASI_N, r2, r3 ; \
-	cmp	r2, r3 ; \
-	bne,pn	%icc, 9b ; \
+#define	ATOMIC_INC_INT(r1, r2, r3)					\
+	lduw	[r1], r2 ;						\
+9:	add	r2, 1, r3 ;						\
+	casa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%icc, 9b ;						\
 	 mov	r3, r2
 
 /*
- * Atomically increment an u_long in memory.
+ * Atomically increment a long in memory.
  */
-#define	ATOMIC_INC_ULONG(r1, r2, r3) \
-	ldx	[r1], r2 ; \
-9:	add	r2, 1, r3 ; \
-	casxa	[r1] ASI_N, r2, r3 ; \
-	cmp	r2, r3 ; \
-	bne,pn	%icc, 9b ; \
+#define	ATOMIC_INC_LONG(r1, r2, r3)					\
+	ldx	[r1], r2 ;						\
+9:	add	r2, 1, r3 ;						\
+	casxa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%xcc, 9b ;						\
 	 mov	r3, r2
 
 /*
  * Atomically clear a number of bits of an integer in memory.
  */
-#define	ATOMIC_CLEAR_INT(r1, r2, r3, bits) \
-	lduw	[r1], r2 ; \
-9:	andn	r2, bits, r3 ; \
-	casa	[r1] ASI_N, r2, r3 ; \
-	cmp	r2, r3 ; \
-	bne,pn	%icc, 9b ; \
+#define	ATOMIC_CLEAR_INT(r1, r2, r3, bits)				\
+	lduw	[r1], r2 ;						\
+9:	andn	r2, bits, r3 ;						\
+	casa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%icc, 9b ;						\
 	 mov	r3, r2
 
 /*
- * Atomically clear a number of bits of an u_long in memory.
+ * Atomically clear a number of bits of a long in memory.
  */
-#define	ATOMIC_CLEAR_LONG(r1, r2, r3, bits) \
-	ldx	[r1], r2 ; \
-9:	andn	r2, bits, r3 ; \
-	casxa	[r1] ASI_N, r2, r3 ; \
-	cmp	r2, r3 ; \
-	bne,pn	%icc, 9b ; \
+#define	ATOMIC_CLEAR_LONG(r1, r2, r3, bits)				\
+	ldx	[r1], r2 ;						\
+9:	andn	r2, bits, r3 ;						\
+	casxa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%xcc, 9b ;						\
+	 mov	r3, r2
+
+/*
+ * Atomically load an integer from memory.
+ */
+#define	ATOMIC_LOAD_INT(r1, val)					\
+	clr	val ;							\
+	casa	[r1] ASI_N, %g0, val
+
+/*
+ * Atomically load a long from memory.
+ */
+#define	ATOMIC_LOAD_LONG(r1, val)					\
+	clr	val ;							\
+	casxa	[r1] ASI_N, %g0, val
+
+/*
+ * Atomically set a number of bits of an integer in memory.
+ */
+#define	ATOMIC_SET_INT(r1, r2, r3, bits)				\
+	lduw	[r1], r2 ;						\
+9:	or	r2, bits, r3 ;						\
+	casa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%icc, 9b ;						\
+	 mov	r3, r2
+
+/*
+ * Atomically set a number of bits of a long in memory.
+ */
+#define	ATOMIC_SET_LONG(r1, r2, r3, bits)				\
+	ldx	[r1], r2 ;						\
+9:	or	r2, bits, r3 ;						\
+	casxa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%xcc, 9b ;						\
+	 mov	r3, r2
+
+/*
+ * Atomically store an integer in memory.
+ */
+#define	ATOMIC_STORE_INT(r1, r2, r3, val)				\
+	lduw	[r1], r2 ;						\
+9:	mov	val, r3 ;						\
+	casa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%icc, 9b ;						\
+	 mov	r3, r2
+
+/*
+ * Atomically store a long in memory.
+ */
+#define	ATOMIC_STORE_LONG(r1, r2, r3, val)				\
+	ldx	[r1], r2 ;						\
+9:	mov	val, r3 ;						\
+	casxa	[r1] ASI_N, r2, r3 ;					\
+	cmp	r2, r3 ;						\
+	bne,pn	%xcc, 9b ;						\
 	 mov	r3, r2
 
 #define	PCPU(member)	PCPU_REG + PC_ ## member
-#define	PCPU_ADDR(member, reg) \
+#define	PCPU_ADDR(member, reg)						\
 	add	PCPU_REG, PC_ ## member, reg
 
-#define	DEBUGGER() \
+#define	DEBUGGER()							\
 	ta	%xcc, 1
 
-#define	PANIC(msg, r1) \
-	.sect	.rodata ; \
-9:	.asciz	msg ; \
-	.previous ; \
-	SET(9b, r1, %o0) ; \
-	call	panic ; \
+#define	PANIC(msg, r1)							\
+	.sect	.rodata ;						\
+9:	.asciz	msg ;							\
+	.previous ;							\
+	SET(9b, r1, %o0) ;						\
+	call	panic ;							\
 	 nop
 
 #ifdef INVARIANTS
-#define	KASSERT(r1, msg) \
-	brnz,pt	r1, 8f ; \
-	 nop ; \
-	PANIC(msg, r1) ; \
+#define	KASSERT(r1, msg)						\
+	brnz,pt	r1, 8f ;						\
+	 nop ;								\
+	PANIC(msg, r1) ;						\
 8:
 #else
 #define	KASSERT(r1, msg)
 #endif
 
-#define	PUTS(msg, r1) \
-	.sect	.rodata ; \
-9:	.asciz	msg ; \
-	.previous ; \
-	SET(9b, r1, %o0) ; \
-	call	printf ; \
+#define	PUTS(msg, r1)							\
+	.sect	.rodata ;						\
+9:	.asciz	msg ;							\
+	.previous ;							\
+	SET(9b, r1, %o0) ;						\
+	call	printf ;						\
 	 nop
 
 #define	_ALIGN_DATA	.align 8
 
-#define	DATA(name) \
-	.data ; \
-	_ALIGN_DATA ; \
-	.globl	name ; \
-	.type	name, @object ; \
+#define	DATA(name)							\
+	.data ;								\
+	_ALIGN_DATA ;							\
+	.globl	name ;							\
+	.type	name, @object ;						\
 name:
 
 #define	EMPTY
diff --git a/sys/sparc64/include/atomic.h b/sys/sparc64/include/atomic.h
index d663fbc457ee..a47ce32c81d4 100644
--- a/sys/sparc64/include/atomic.h
+++ b/sys/sparc64/include/atomic.h
@@ -33,6 +33,10 @@
 
 #include <machine/cpufunc.h>
 
+#define	mb()	__asm__ __volatile__ ("membar #MemIssue": : :"memory")
+#define	wmb()	mb()
+#define	rmb()	mb()
+
 /* Userland needs different ASI's. */
 #ifdef _KERNEL
 #define	__ASI_ATOMIC	ASI_N
@@ -40,10 +44,6 @@
 #define	__ASI_ATOMIC	ASI_P
 #endif
 
-#define mb()	__asm__ __volatile__ ("membar #MemIssue": : :"memory")
-#define wmb()	mb()
-#define rmb()	mb()
-
 /*
  * Various simple arithmetic on memory which is atomic in the presence
  * of interrupts and multiple processors.  See atomic(9) for details.
@@ -74,41 +74,44 @@
  *
  * the return value of cas is used to avoid the extra reload.
  *
- * The memory barriers provided by the acq and rel variants are intended
- * to be sufficient for use of relaxed memory ordering.  Due to the
- * suggested assembly syntax of the membar operands containing a #
- * character, they cannot be used in macros.  The cmask and mmask bits
+ * We only include a memory barrier in the rel variants as in total store
+ * order which we use for running the kernel and all of the userland atomic
+ * loads and stores behave as if the were followed by a membar with a mask
+ * of #LoadLoad | #LoadStore | #StoreStore.  In order to be also sufficient
+ * for use of relaxed memory ordering, the atomic_cas() in the acq variants 
+ * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
+ * Due to the suggested assembly syntax of the membar operands containing a
+ * # character, they cannot be used in macros.  The cmask and mmask bits thus
  * are hard coded in machine/cpufunc.h and used here through macros.
- * Hopefully sun will choose not to change the bit numbers.
+ * Hopefully the bit numbers won't change in the future.
  */
 
 #define	itype(sz)	uint ## sz ## _t
 
-#define	atomic_cas_32(p, e, s)	casa(p, e, s, __ASI_ATOMIC)
-#define	atomic_cas_64(p, e, s)	casxa(p, e, s, __ASI_ATOMIC)
+#define	atomic_cas_32(p, e, s)	casa((p), (e), (s), __ASI_ATOMIC)
+#define	atomic_cas_64(p, e, s)	casxa((p), (e), (s), __ASI_ATOMIC)
 
 #define	atomic_cas(p, e, s, sz)						\
-	atomic_cas_ ## sz(p, e, s)
+	atomic_cas_ ## sz((p), (e), (s))
 
 #define	atomic_cas_acq(p, e, s, sz) ({					\
 	itype(sz) v;							\
-	v = atomic_cas(p, e, s, sz);					\
-	membar(LoadLoad | LoadStore);					\
+	v = atomic_cas((p), (e), (s), sz);				\
 	v;								\
 })
 
 #define	atomic_cas_rel(p, e, s, sz) ({					\
 	itype(sz) v;							\
 	membar(LoadStore | StoreStore);					\
-	v = atomic_cas(p, e, s, sz);					\
+	v = atomic_cas((p), (e), (s), sz);				\
 	v;								\
 })
 
 #define	atomic_op(p, op, v, sz) ({					\
 	itype(sz) e, r, s;						\
-	for (e = *(volatile itype(sz) *)p;; e = r) {			\
-		s = e op v;						\
-		r = atomic_cas_ ## sz(p, e, s);				\
+	for (e = *(volatile itype(sz) *)(p);; e = r) {			\
+		s = e op (v);						\
+		r = atomic_cas_ ## sz((p), e, s);			\
 		if (r == e)						\
 			break;						\
 	}								\
@@ -117,32 +120,30 @@
 
 #define	atomic_op_acq(p, op, v, sz) ({					\
 	itype(sz) t;							\
-	t = atomic_op(p, op, v, sz);					\
-	membar(LoadLoad | LoadStore);					\
+	t = atomic_op((p), op, (v), sz);				\
 	t;								\
 })
 
 #define	atomic_op_rel(p, op, v, sz) ({					\
 	itype(sz) t;							\
 	membar(LoadStore | StoreStore);					\
-	t = atomic_op(p, op, v, sz);					\
+	t = atomic_op((p), op, (v), sz);				\
 	t;								\
 })
 
 #define	atomic_load(p, sz)						\
-	atomic_cas(p, 0, 0, sz)
+	atomic_cas((p), 0, 0, sz)
 
 #define	atomic_load_acq(p, sz) ({					\
 	itype(sz) v;							\
-	v = atomic_load(p, sz);						\
-	membar(LoadLoad | LoadStore);					\
+	v = atomic_load((p), sz);					\
 	v;								\
 })
 
 #define	atomic_load_clear(p, sz) ({					\
 	itype(sz) e, r;							\
-	for (e = *(volatile itype(sz) *)p;; e = r) {			\
-		r = atomic_cas(p, e, 0, sz);				\
+	for (e = *(volatile itype(sz) *)(p);; e = r) {			\
+		r = atomic_cas((p), e, 0, sz);				\
 		if (r == e)						\
 			break;						\
 	}								\
@@ -151,8 +152,8 @@
 
 #define	atomic_store(p, v, sz) do {					\
 	itype(sz) e, r;							\
-	for (e = *(volatile itype(sz) *)p;; e = r) {			\
-		r = atomic_cas(p, e, v, sz);				\
+	for (e = *(volatile itype(sz) *)(p);; e = r) {			\
+		r = atomic_cas((p), e, (v), sz);			\
 		if (r == e)						\
 			break;						\
 	}								\
@@ -160,7 +161,7 @@
 
 #define	atomic_store_rel(p, v, sz) do {					\
 	membar(LoadStore | StoreStore);					\
-	atomic_store(p, v, sz);						\
+	atomic_store((p), (v), sz);					\
 } while (0)
 
 #define	ATOMIC_GEN(name, ptype, vtype, atype, sz)			\
@@ -168,109 +169,109 @@
 static __inline vtype							\
 atomic_add_ ## name(volatile ptype p, atype v)				\
 {									\
-	return ((vtype)atomic_op(p, +, v, sz));				\
+	return ((vtype)atomic_op((p), +, (v), sz));			\
 }									\
 static __inline vtype							\
 atomic_add_acq_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_acq(p, +, v, sz));			\
+	return ((vtype)atomic_op_acq((p), +, (v), sz));			\
 }									\
 static __inline vtype							\
 atomic_add_rel_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_rel(p, +, v, sz));			\
+	return ((vtype)atomic_op_rel((p), +, (v), sz));			\
 }									\
 									\
 static __inline vtype							\
 atomic_clear_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op(p, &, ~v, sz));			\
+	return ((vtype)atomic_op((p), &, ~(v), sz));			\
 }									\
 static __inline vtype							\
 atomic_clear_acq_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_acq(p, &, ~v, sz));			\
+	return ((vtype)atomic_op_acq((p), &, ~(v), sz));		\
 }									\
 static __inline vtype							\
 atomic_clear_rel_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_rel(p, &, ~v, sz));			\
+	return ((vtype)atomic_op_rel((p), &, ~(v), sz));		\
 }									\
 									\
 static __inline int							\
 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)		\
 {									\
-	return (((vtype)atomic_cas(p, e, s, sz)) == e);			\
+	return (((vtype)atomic_cas((p), (e), (s), sz)) == (e));		\
 }									\
 static __inline int							\
 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)		\
 {									\
-	return (((vtype)atomic_cas_acq(p, e, s, sz)) == e);		\
+	return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e));	\
 }									\
 static __inline int							\
 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)		\
 {									\
-	return (((vtype)atomic_cas_rel(p, e, s, sz)) == e);		\
+	return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e));	\
 }									\
 									\
 static __inline vtype							\
 atomic_load_ ## name(volatile ptype p)					\
 {									\
-	return ((vtype)atomic_cas(p, 0, 0, sz));			\
+	return ((vtype)atomic_cas((p), 0, 0, sz));			\
 }									\
 static __inline vtype							\
 atomic_load_acq_ ## name(volatile ptype p)				\
 {									\
-	return ((vtype)atomic_cas_acq(p, 0, 0, sz));			\
+	return ((vtype)atomic_cas_acq((p), 0, 0, sz));			\
 }									\
 									\
 static __inline vtype							\
 atomic_readandclear_ ## name(volatile ptype p)				\
 {									\
-	return ((vtype)atomic_load_clear(p, sz));			\
+	return ((vtype)atomic_load_clear((p), sz));			\
 }									\
 									\
 static __inline vtype							\
 atomic_set_ ## name(volatile ptype p, atype v)				\
 {									\
-	return ((vtype)atomic_op(p, |, v, sz));				\
+	return ((vtype)atomic_op((p), |, (v), sz));			\
 }									\
 static __inline vtype							\
 atomic_set_acq_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_acq(p, |, v, sz));			\
+	return ((vtype)atomic_op_acq((p), |, (v), sz));			\
 }									\
 static __inline vtype							\
 atomic_set_rel_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_rel(p, |, v, sz));			\
+	return ((vtype)atomic_op_rel((p), |, (v), sz));			\
 }									\
 									\
 static __inline vtype							\
 atomic_subtract_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op(p, -, v, sz));				\
+	return ((vtype)atomic_op((p), -, (v), sz));			\
 }									\
 static __inline vtype							\
 atomic_subtract_acq_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_acq(p, -, v, sz));			\
+	return ((vtype)atomic_op_acq((p), -, (v), sz));			\
 }									\
 static __inline vtype							\
 atomic_subtract_rel_ ## name(volatile ptype p, atype v)			\
 {									\
-	return ((vtype)atomic_op_rel(p, -, v, sz));			\
+	return ((vtype)atomic_op_rel((p), -, (v), sz));			\
 }									\
 									\
 static __inline void							\
 atomic_store_ ## name(volatile ptype p, vtype v)			\
 {									\
-	atomic_store(p, v, sz);						\
+	atomic_store((p), (v), sz);					\
 }									\
 static __inline void							\
 atomic_store_rel_ ## name(volatile ptype p, vtype v)			\
 {									\
-	atomic_store_rel(p, v, sz);					\
+	atomic_store_rel((p), (v), sz);					\
 }
 
 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
diff --git a/sys/sparc64/include/bus.h b/sys/sparc64/include/bus.h
index 75e598934ec1..c641a25577d6 100644
--- a/sys/sparc64/include/bus.h
+++ b/sys/sparc64/include/bus.h
@@ -122,32 +122,15 @@ static int bus_space_subregion(bus_space_tag_t, bus_space_handle_t,
 /*
  * Map a region of device bus space into CPU virtual address space.
  */
-
-static __inline int bus_space_map(bus_space_tag_t t, bus_addr_t addr,
-    bus_size_t size, int flags, bus_space_handle_t *bshp);
-
-static __inline int
-bus_space_map(bus_space_tag_t t __unused, bus_addr_t addr,
-    bus_size_t size __unused, int flags __unused, bus_space_handle_t *bshp)
-{
-
-	*bshp = addr;
-	return (0);
-}
+int bus_space_map(bus_space_tag_t tag, bus_addr_t address, bus_size_t size,
+    int flags, bus_space_handle_t *handlep);
 
 /*
  * Unmap a region of device bus space.
  */
-static __inline void bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh,
+void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t handle,
     bus_size_t size);
 
-static __inline void
-bus_space_unmap(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused,
-    bus_size_t size __unused)
-{
-
-}
-
 /* This macro finds the first "upstream" implementation of method `f' */
 #define	_BS_CALL(t,f)							\
 	while (t->f == NULL)						\
diff --git a/sys/sparc64/include/bus_private.h b/sys/sparc64/include/bus_private.h
index fb6f8d395cf2..bccd2293cdd8 100644
--- a/sys/sparc64/include/bus_private.h
+++ b/sys/sparc64/include/bus_private.h
@@ -36,10 +36,14 @@
 /*
  * Helpers
  */
-int sparc64_bus_mem_map(bus_space_tag_t, bus_space_handle_t, bus_size_t,
-    int, vm_offset_t, void **);
-int sparc64_bus_mem_unmap(void *, bus_size_t);
-bus_space_handle_t sparc64_fake_bustag(int, bus_addr_t, struct bus_space_tag *);
+int sparc64_bus_mem_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size,
+    int flags, vm_offset_t vaddr, bus_space_handle_t *hp);
+int sparc64_bus_mem_unmap(bus_space_tag_t tag, bus_space_handle_t handle,
+    bus_size_t size);
+bus_space_tag_t sparc64_alloc_bus_tag(void *cookie,
+    struct bus_space_tag *ptag, int type, void *barrier);
+bus_space_handle_t sparc64_fake_bustag(int space, bus_addr_t addr,
+    struct bus_space_tag *ptag);
 
 struct bus_dmamap_res {
 	struct resource		*dr_res;
diff --git a/sys/sparc64/include/ieeefp.h b/sys/sparc64/include/ieeefp.h
index 11264949acee..bb0773730aa0 100644
--- a/sys/sparc64/include/ieeefp.h
+++ b/sys/sparc64/include/ieeefp.h
@@ -7,6 +7,8 @@
 #ifndef _MACHINE_IEEEFP_H_
 #define _MACHINE_IEEEFP_H_
 
+/* Deprecated FPU control interface */
+
 #include <machine/fsr.h>
 
 typedef int fp_except_t;
@@ -23,4 +25,13 @@ typedef enum {
 	FP_RM = FSR_RD_NINF	/* round toward negative infinity */
 } fp_rnd_t;
 
+__BEGIN_DECLS
+extern fp_rnd_t    fpgetround(void);
+extern fp_rnd_t    fpsetround(fp_rnd_t);
+extern fp_except_t fpgetmask(void);
+extern fp_except_t fpsetmask(fp_except_t);
+extern fp_except_t fpgetsticky(void);
+__END_DECLS
+
+
 #endif /* _MACHINE_IEEEFP_H_ */
diff --git a/sys/sparc64/include/proc.h b/sys/sparc64/include/proc.h
index dceea17a33e7..006fcfab84a8 100644
--- a/sys/sparc64/include/proc.h
+++ b/sys/sparc64/include/proc.h
@@ -61,7 +61,6 @@ struct syscall_args {
 	register_t args[8];
 	int narg;
 };
-#define	HAVE_SYSCALL_ARGS_DEF 1
 
 #endif
 
diff --git a/sys/sparc64/pci/apb.c b/sys/sparc64/pci/apb.c
index 87f182109569..7d42b7b0da7f 100644
--- a/sys/sparc64/pci/apb.c
+++ b/sys/sparc64/pci/apb.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
+#include <sys/sysctl.h>
 
 #include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/openfirm.h>
@@ -77,6 +78,7 @@ struct apb_softc {
 static device_probe_t apb_probe;
 static device_attach_t apb_attach;
 static bus_alloc_resource_t apb_alloc_resource;
+static bus_adjust_resource_t apb_adjust_resource;
 
 static device_method_t apb_methods[] = {
 	/* Device interface */
@@ -85,6 +87,8 @@ static device_method_t apb_methods[] = {
 
 	/* Bus interface */
 	DEVMETHOD(bus_alloc_resource,	apb_alloc_resource),
+	DEVMETHOD(bus_adjust_resource,	apb_adjust_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 
 	/* pcib interface */
 	DEVMETHOD(pcib_route_interrupt,	ofw_pcib_gen_route_interrupt),
@@ -158,6 +162,8 @@ static int
 apb_attach(device_t dev)
 {
 	struct apb_softc *sc;
+	struct sysctl_ctx_list *sctx;
+	struct sysctl_oid *soid;
 
 	sc = device_get_softc(dev);
 
@@ -165,12 +171,41 @@ apb_attach(device_t dev)
 	 * Get current bridge configuration.
 	 */
 	sc->sc_bsc.ops_pcib_sc.domain = pci_get_domain(dev);
+	sc->sc_bsc.ops_pcib_sc.secstat =
+	    pci_read_config(dev, PCIR_SECSTAT_1, 2);
+	sc->sc_bsc.ops_pcib_sc.command =
+	    pci_read_config(dev, PCIR_COMMAND, 2);
+	sc->sc_bsc.ops_pcib_sc.pribus =
+	    pci_read_config(dev, PCIR_PRIBUS_1, 1);
 	sc->sc_bsc.ops_pcib_sc.secbus =
 	    pci_read_config(dev, PCIR_SECBUS_1, 1);
 	sc->sc_bsc.ops_pcib_sc.subbus =
 	    pci_read_config(dev, PCIR_SUBBUS_1, 1);
+	sc->sc_bsc.ops_pcib_sc.bridgectl =
+	    pci_read_config(dev, PCIR_BRIDGECTL_1, 2);
+	sc->sc_bsc.ops_pcib_sc.seclat =
+	    pci_read_config(dev, PCIR_SECLAT_1, 1);
 	sc->sc_iomap = pci_read_config(dev, APBR_IOMAP, 1);
 	sc->sc_memmap = pci_read_config(dev, APBR_MEMMAP, 1);
+
+	/*
+	 * Setup SYSCTL reporting nodes.
+	 */
+	sctx = device_get_sysctl_ctx(dev);
+	soid = device_get_sysctl_tree(dev);
+	SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "domain",
+	    CTLFLAG_RD, &sc->sc_bsc.ops_pcib_sc.domain, 0,
+	    "Domain number");
+	SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "pribus",
+	    CTLFLAG_RD, &sc->sc_bsc.ops_pcib_sc.pribus, 0,
+	    "Primary bus number");
+	SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "secbus",
+	    CTLFLAG_RD, &sc->sc_bsc.ops_pcib_sc.secbus, 0,
+	    "Secondary bus number");
+	SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "subbus",
+	    CTLFLAG_RD, &sc->sc_bsc.ops_pcib_sc.subbus, 0,
+	    "Subordinate bus number");
+
 	ofw_pcib_gen_setup(dev);
 
 	if (bootverbose) {
@@ -233,9 +268,9 @@ apb_alloc_resource(device_t dev, device_t child, int type, int *rid,
 			    "%s requested decoded I/O range 0x%lx-0x%lx\n",
 			    device_get_nameunit(child), start, end);
 		break;
-
 	case SYS_RES_MEMORY:
-		if (!apb_checkrange(sc->sc_memmap, APB_MEM_SCALE, start, end)) {
+		if (!apb_checkrange(sc->sc_memmap, APB_MEM_SCALE, start,
+		    end)) {
 			device_printf(dev, "device %s requested unsupported "
 			    "memory range 0x%lx-0x%lx\n",
 			    device_get_nameunit(child), start, end);
@@ -246,9 +281,6 @@ apb_alloc_resource(device_t dev, device_t child, int type, int *rid,
 			    "%s requested decoded memory range 0x%lx-0x%lx\n",
 			    device_get_nameunit(child), start, end);
 		break;
-
-	default:
-		break;
 	}
 
  passup:
@@ -258,3 +290,23 @@ apb_alloc_resource(device_t dev, device_t child, int type, int *rid,
 	return (bus_generic_alloc_resource(dev, child, type, rid, start, end,
 	    count, flags));
 }
+
+static int
+apb_adjust_resource(device_t dev, device_t child, int type,
+    struct resource *r, u_long start, u_long end)
+{
+	struct apb_softc *sc;
+
+	sc = device_get_softc(dev);
+	switch (type) {
+	case SYS_RES_IOPORT:
+		if (!apb_checkrange(sc->sc_iomap, APB_IO_SCALE, start, end))
+			return (ENXIO);
+		break;
+	case SYS_RES_MEMORY:
+		if (!apb_checkrange(sc->sc_memmap, APB_MEM_SCALE, start, end))
+			return (ENXIO);
+		break;
+	}
+	return (bus_generic_adjust_resource(dev, child, type, r, start, end));
+}
diff --git a/sys/sparc64/pci/fire.c b/sys/sparc64/pci/fire.c
index 040289653947..5a1e08af6fa0 100644
--- a/sys/sparc64/pci/fire.c
+++ b/sys/sparc64/pci/fire.c
@@ -85,7 +85,6 @@ __FBSDID("$FreeBSD$");
 
 struct fire_msiqarg;
 
-static bus_space_tag_t fire_alloc_bus_tag(struct fire_softc *sc, int type);
 static const struct fire_desc *fire_get_desc(device_t dev);
 static void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
     bus_dmasync_op_t op);
@@ -113,11 +112,11 @@ static driver_filter_t fire_xcb;
  * Methods
  */
 static bus_activate_resource_t fire_activate_resource;
+static bus_adjust_resource_t fire_adjust_resource;
 static pcib_alloc_msi_t fire_alloc_msi;
 static pcib_alloc_msix_t fire_alloc_msix;
 static bus_alloc_resource_t fire_alloc_resource;
 static device_attach_t fire_attach;
-static bus_deactivate_resource_t fire_deactivate_resource;
 static bus_get_dma_tag_t fire_get_dma_tag;
 static ofw_bus_get_node_t fire_get_node;
 static pcib_map_msi_t fire_map_msi;
@@ -127,7 +126,6 @@ static pcib_read_config_t fire_read_config;
 static bus_read_ivar_t fire_read_ivar;
 static pcib_release_msi_t fire_release_msi;
 static pcib_release_msix_t fire_release_msix;
-static bus_release_resource_t fire_release_resource;
 static pcib_route_interrupt_t fire_route_interrupt;
 static bus_setup_intr_t fire_setup_intr;
 static bus_teardown_intr_t fire_teardown_intr;
@@ -147,9 +145,10 @@ static device_method_t fire_methods[] = {
 	DEVMETHOD(bus_setup_intr,	fire_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	fire_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	fire_alloc_resource),
-	DEVMETHOD(bus_activate_resource,	fire_activate_resource),
-	DEVMETHOD(bus_deactivate_resource,	fire_deactivate_resource),
-	DEVMETHOD(bus_release_resource,	fire_release_resource),
+	DEVMETHOD(bus_activate_resource, fire_activate_resource),
+	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	fire_adjust_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_get_dma_tag,	fire_get_dma_tag),
 
 	/* pcib interface */
@@ -757,13 +756,19 @@ fire_attach(device_t dev)
 	free(range, M_OFWPROP);
 
 	/* Allocate our tags. */
-	sc->sc_pci_memt = fire_alloc_bus_tag(sc, PCI_MEMORY_BUS_SPACE);
-	sc->sc_pci_iot = fire_alloc_bus_tag(sc, PCI_IO_BUS_SPACE);
-	sc->sc_pci_cfgt = fire_alloc_bus_tag(sc, PCI_CONFIG_BUS_SPACE);
+	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, rman_get_bustag(
+	    sc->sc_mem_res[FIRE_PCI]), PCI_IO_BUS_SPACE, NULL);
+	if (sc->sc_pci_iot == NULL)
+		panic("%s: could not allocate PCI I/O tag", __func__);
+	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, rman_get_bustag(
+	    sc->sc_mem_res[FIRE_PCI]), PCI_CONFIG_BUS_SPACE, NULL);
+	if (sc->sc_pci_cfgt == NULL)
+		panic("%s: could not allocate PCI configuration space tag",
+		    __func__);
 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0,
 	    sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr,
 	    0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0)
-		panic("%s: bus_dma_tag_create failed", __func__);
+		panic("%s: could not create PCI DMA tag", __func__);
 	/* Customize the tag. */
 	sc->sc_pci_dmat->dt_cookie = &sc->sc_is;
 	sc->sc_pci_dmat->dt_mt = &sc->sc_dma_methods;
@@ -2015,14 +2020,10 @@ fire_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	struct fire_softc *sc;
 	struct resource *rv;
 	struct rman *rm;
-	bus_space_tag_t bt;
-	bus_space_handle_t bh;
-	int needactivate = flags & RF_ACTIVE;
-
-	flags &= ~RF_ACTIVE;
 
 	sc = device_get_softc(bus);
-	if (type == SYS_RES_IRQ) {
+	switch (type) {
+	case SYS_RES_IRQ:
 		/*
 		 * XXX: Don't accept blank ranges for now, only single
 		 * interrupts.  The other case should not happen with
@@ -2034,38 +2035,28 @@ fire_alloc_resource(device_t bus, device_t child, int type, int *rid,
 			panic("%s: XXX: interrupt range", __func__);
 		if (*rid == 0)
 			start = end = INTMAP_VEC(sc->sc_ign, end);
-		return (BUS_ALLOC_RESOURCE(device_get_parent(bus), child,
-		    type, rid, start, end, count, flags));
-	}
-	switch (type) {
+		return (bus_generic_alloc_resource(bus, child, type, rid,
+		    start, end, count, flags));
 	case SYS_RES_MEMORY:
 		rm = &sc->sc_pci_mem_rman;
-		bt = sc->sc_pci_memt;
-		bh = sc->sc_pci_bh[OFW_PCI_CS_MEM32];
 		break;
 	case SYS_RES_IOPORT:
 		rm = &sc->sc_pci_io_rman;
-		bt = sc->sc_pci_iot;
-		bh = sc->sc_pci_bh[OFW_PCI_CS_IO];
 		break;
 	default:
 		return (NULL);
-		/* NOTREACHED */
 	}
 
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
+	    child);
 	if (rv == NULL)
 		return (NULL);
 	rman_set_rid(rv, *rid);
-	bh += rman_get_start(rv);
-	rman_set_bustag(rv, bt);
-	rman_set_bushandle(rv, bh);
 
-	if (needactivate) {
-		if (bus_activate_resource(child, type, *rid, rv)) {
-			rman_release_resource(rv);
-			return (NULL);
-		}
+	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
+	    *rid, rv) != 0) {
+		rman_release_resource(rv);
+		return (NULL);
 	}
 	return (rv);
 }
@@ -2074,56 +2065,56 @@ static int
 fire_activate_resource(device_t bus, device_t child, int type, int rid,
     struct resource *r)
 {
-	void *p;
-	int error;
+	struct fire_softc *sc;
+	struct bus_space_tag *tag;
 
-	if (type == SYS_RES_IRQ)
-		return (BUS_ACTIVATE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (type == SYS_RES_MEMORY) {
-		/*
-		 * Need to memory-map the device space, as some drivers
-		 * depend on the virtual address being set and usable.
-		 */
-		error = sparc64_bus_mem_map(rman_get_bustag(r),
-		    rman_get_bushandle(r), rman_get_size(r), 0, 0, &p);
-		if (error != 0)
-			return (error);
-		rman_set_virtual(r, p);
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_activate_resource(bus, child, type, rid,
+		    r));
+	case SYS_RES_MEMORY:
+		tag = sparc64_alloc_bus_tag(r, rman_get_bustag(
+		    sc->sc_mem_res[FIRE_PCI]), PCI_MEMORY_BUS_SPACE, NULL);
+		if (tag == NULL)
+			return (ENOMEM);
+		rman_set_bustag(r, tag);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
+		    rman_get_start(r));
+		break;
+	case SYS_RES_IOPORT:
+		rman_set_bustag(r, sc->sc_pci_iot);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
+		    rman_get_start(r));
+		break;
 	}
 	return (rman_activate_resource(r));
 }
 
 static int
-fire_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+fire_adjust_resource(device_t bus, device_t child, int type,
+    struct resource *r, u_long start, u_long end)
 {
+	struct fire_softc *sc;
+	struct rman *rm;
 
-	if (type == SYS_RES_IRQ)
-		return (BUS_DEACTIVATE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (type == SYS_RES_MEMORY) {
-		sparc64_bus_mem_unmap(rman_get_virtual(r), rman_get_size(r));
-		rman_set_virtual(r, NULL);
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_adjust_resource(bus, child, type, r,
+		    start, end));
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_pci_mem_rman;
+		break;
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_pci_io_rman;
+		break;
+	default:
+		return (EINVAL);
 	}
-	return (rman_deactivate_resource(r));
-}
-
-static int
-fire_release_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-	int error;
-
-	if (type == SYS_RES_IRQ)
-		return (BUS_RELEASE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (rman_get_flags(r) & RF_ACTIVE) {
-		error = bus_deactivate_resource(child, type, rid, r);
-		if (error)
-			return (error);
-	}
-	return (rman_release_resource(r));
+	if (rman_is_region_manager(r, rm) == 0)
+		return (EINVAL);
+	return (rman_adjust_resource(r, start, end));
 }
 
 static bus_dma_tag_t
@@ -2145,22 +2136,6 @@ fire_get_node(device_t bus, device_t child __unused)
 	return (sc->sc_node);
 }
 
-static bus_space_tag_t
-fire_alloc_bus_tag(struct fire_softc *sc, int type)
-{
-	bus_space_tag_t bt;
-
-	bt = malloc(sizeof(struct bus_space_tag), M_DEVBUF,
-	    M_NOWAIT | M_ZERO);
-	if (bt == NULL)
-		panic("%s: out of memory", __func__);
-
-	bt->bst_cookie = sc;
-	bt->bst_parent = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]);
-	bt->bst_type = type;
-	return (bt);
-}
-
 static u_int
 fire_get_timecount(struct timecounter *tc)
 {
diff --git a/sys/sparc64/pci/firevar.h b/sys/sparc64/pci/firevar.h
index 58ba419daeea..d414c1adc503 100644
--- a/sys/sparc64/pci/firevar.h
+++ b/sys/sparc64/pci/firevar.h
@@ -47,7 +47,6 @@ struct fire_softc {
 	bus_space_handle_t		sc_pci_bh[FIRE_NRANGE];
 	bus_space_tag_t			sc_pci_cfgt;
 	bus_space_tag_t			sc_pci_iot;
-	bus_space_tag_t			sc_pci_memt;
 	bus_dma_tag_t			sc_pci_dmat;
 
 	device_t			sc_dev;
diff --git a/sys/sparc64/pci/ofw_pcib_subr.c b/sys/sparc64/pci/ofw_pcib_subr.c
index 600102316e10..2ffb32d1ba3b 100644
--- a/sys/sparc64/pci/ofw_pcib_subr.c
+++ b/sys/sparc64/pci/ofw_pcib_subr.c
@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
+#include <sys/rman.h>
 
 #include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/ofw_pci.h>
diff --git a/sys/sparc64/pci/psycho.c b/sys/sparc64/pci/psycho.c
index 2b81f2721787..f7b5d4a31146 100644
--- a/sys/sparc64/pci/psycho.c
+++ b/sys/sparc64/pci/psycho.c
@@ -89,7 +89,6 @@ static void psycho_intr_enable(void *);
 static void psycho_intr_disable(void *);
 static void psycho_intr_assign(void *);
 static void psycho_intr_clear(void *);
-static bus_space_tag_t psycho_alloc_bus_tag(struct psycho_softc *, int);
 
 /* Interrupt handlers */
 static driver_filter_t psycho_ue;
@@ -113,8 +112,7 @@ static bus_read_ivar_t psycho_read_ivar;
 static bus_setup_intr_t psycho_setup_intr;
 static bus_alloc_resource_t psycho_alloc_resource;
 static bus_activate_resource_t psycho_activate_resource;
-static bus_deactivate_resource_t psycho_deactivate_resource;
-static bus_release_resource_t psycho_release_resource;
+static bus_adjust_resource_t psycho_adjust_resource;
 static bus_get_dma_tag_t psycho_get_dma_tag;
 static pcib_maxslots_t psycho_maxslots;
 static pcib_read_config_t psycho_read_config;
@@ -137,9 +135,10 @@ static device_method_t psycho_methods[] = {
 	DEVMETHOD(bus_setup_intr,	psycho_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	psycho_alloc_resource),
-	DEVMETHOD(bus_activate_resource,	psycho_activate_resource),
-	DEVMETHOD(bus_deactivate_resource,	psycho_deactivate_resource),
-	DEVMETHOD(bus_release_resource,	psycho_release_resource),
+	DEVMETHOD(bus_activate_resource, psycho_activate_resource),
+	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	psycho_adjust_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_describe_intr,	bus_generic_describe_intr),
 	DEVMETHOD(bus_get_dma_tag,	psycho_get_dma_tag),
 
@@ -567,13 +566,19 @@ psycho_attach(device_t dev)
 	}
 
 	/* Allocate our tags. */
-	sc->sc_pci_memt = psycho_alloc_bus_tag(sc, PCI_MEMORY_BUS_SPACE);
-	sc->sc_pci_iot = psycho_alloc_bus_tag(sc, PCI_IO_BUS_SPACE);
-	sc->sc_pci_cfgt = psycho_alloc_bus_tag(sc, PCI_CONFIG_BUS_SPACE);
+	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, rman_get_bustag(
+	    sc->sc_mem_res), PCI_IO_BUS_SPACE, NULL);
+	if (sc->sc_pci_iot == NULL)
+		panic("%s: could not allocate PCI I/O tag", __func__);
+	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, rman_get_bustag(
+	    sc->sc_mem_res), PCI_CONFIG_BUS_SPACE, NULL);
+	if (sc->sc_pci_cfgt == NULL)
+		panic("%s: could not allocate PCI configuration space tag",
+		    __func__);
 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0,
 	    sc->sc_is->is_pmaxaddr, ~0, NULL, NULL, sc->sc_is->is_pmaxaddr,
 	    0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0)
-		panic("%s: bus_dma_tag_create failed", __func__);
+		panic("%s: could not create PCI DMA tag", __func__);
 	/* Customize the tag. */
 	sc->sc_pci_dmat->dt_cookie = sc->sc_is;
 	sc->sc_pci_dmat->dt_mt = sc->sc_dma_methods;
@@ -1165,14 +1170,10 @@ psycho_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	struct psycho_softc *sc;
 	struct resource *rv;
 	struct rman *rm;
-	bus_space_tag_t bt;
-	bus_space_handle_t bh;
-	int needactivate = flags & RF_ACTIVE;
-
-	flags &= ~RF_ACTIVE;
 
 	sc = device_get_softc(bus);
-	if (type == SYS_RES_IRQ) {
+	switch (type) {
+	case SYS_RES_IRQ:
 		/*
 		 * XXX: Don't accept blank ranges for now, only single
 		 * interrupts.  The other case should not happen with
@@ -1183,38 +1184,28 @@ psycho_alloc_resource(device_t bus, device_t child, int type, int *rid,
 		if (start != end)
 			panic("%s: XXX: interrupt range", __func__);
 		start = end = INTMAP_VEC(sc->sc_ign, end);
-		return (BUS_ALLOC_RESOURCE(device_get_parent(bus), child,
-		    type, rid, start, end, count, flags));
-	}
-	switch (type) {
+		return (bus_generic_alloc_resource(bus, child, type, rid,
+		     start, end, count, flags));
 	case SYS_RES_MEMORY:
 		rm = &sc->sc_pci_mem_rman;
-		bt = sc->sc_pci_memt;
-		bh = sc->sc_pci_bh[OFW_PCI_CS_MEM32];
 		break;
 	case SYS_RES_IOPORT:
 		rm = &sc->sc_pci_io_rman;
-		bt = sc->sc_pci_iot;
-		bh = sc->sc_pci_bh[OFW_PCI_CS_IO];
 		break;
 	default:
 		return (NULL);
-		/* NOTREACHED */
 	}
 
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
+	    child);
 	if (rv == NULL)
 		return (NULL);
 	rman_set_rid(rv, *rid);
-	bh += rman_get_start(rv);
-	rman_set_bustag(rv, bt);
-	rman_set_bushandle(rv, bh);
 
-	if (needactivate) {
-		if (bus_activate_resource(child, type, *rid, rv)) {
-			rman_release_resource(rv);
-			return (NULL);
-		}
+	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
+	    *rid, rv) != 0) {
+		rman_release_resource(rv);
+		return (NULL);
 	}
 	return (rv);
 }
@@ -1223,56 +1214,56 @@ static int
 psycho_activate_resource(device_t bus, device_t child, int type, int rid,
     struct resource *r)
 {
-	void *p;
-	int error;
+	struct psycho_softc *sc;
+	struct bus_space_tag *tag;
 
-	if (type == SYS_RES_IRQ)
-		return (BUS_ACTIVATE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (type == SYS_RES_MEMORY) {
-		/*
-		 * Need to memory-map the device space, as some drivers
-		 * depend on the virtual address being set and usable.
-		 */
-		error = sparc64_bus_mem_map(rman_get_bustag(r),
-		    rman_get_bushandle(r), rman_get_size(r), 0, 0, &p);
-		if (error != 0)
-			return (error);
-		rman_set_virtual(r, p);
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_activate_resource(bus, child, type, rid,
+		    r));
+	case SYS_RES_MEMORY:
+		tag = sparc64_alloc_bus_tag(r, rman_get_bustag(
+		    sc->sc_mem_res), PCI_MEMORY_BUS_SPACE, NULL);
+		if (tag == NULL)
+			return (ENOMEM);
+		rman_set_bustag(r, tag);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
+		    rman_get_start(r));
+		break;
+	case SYS_RES_IOPORT:
+		rman_set_bustag(r, sc->sc_pci_iot);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
+		    rman_get_start(r));
+		break;
 	}
 	return (rman_activate_resource(r));
 }
 
 static int
-psycho_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+psycho_adjust_resource(device_t bus, device_t child, int type,
+    struct resource *r, u_long start, u_long end)
 {
+	struct psycho_softc *sc;
+	struct rman *rm;
 
-	if (type == SYS_RES_IRQ)
-		return (BUS_DEACTIVATE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (type == SYS_RES_MEMORY) {
-		sparc64_bus_mem_unmap(rman_get_virtual(r), rman_get_size(r));
-		rman_set_virtual(r, NULL);
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_adjust_resource(bus, child, type, r,
+		    start, end));
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_pci_mem_rman;
+		break;
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_pci_io_rman;
+		break;
+	default:
+		return (EINVAL);
 	}
-	return (rman_deactivate_resource(r));
-}
-
-static int
-psycho_release_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-	int error;
-
-	if (type == SYS_RES_IRQ)
-		return (BUS_RELEASE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (rman_get_flags(r) & RF_ACTIVE) {
-		error = bus_deactivate_resource(child, type, rid, r);
-		if (error)
-			return (error);
-	}
-	return (rman_release_resource(r));
+	if (rman_is_region_manager(r, rm) == 0)
+		return (EINVAL);
+	return (rman_adjust_resource(r, start, end));
 }
 
 static bus_dma_tag_t
@@ -1312,19 +1303,3 @@ psycho_setup_device(device_t bus, device_t child)
 		PCICTL_WRITE8(sc, PCR_CS, PCICTL_READ8(sc, PCR_CS) &
 		    ~PCICTL_ARB_PARK);
 }
-
-static bus_space_tag_t
-psycho_alloc_bus_tag(struct psycho_softc *sc, int type)
-{
-	bus_space_tag_t bt;
-
-	bt = malloc(sizeof(struct bus_space_tag), M_DEVBUF,
-	    M_NOWAIT | M_ZERO);
-	if (bt == NULL)
-		panic("%s: out of memory", __func__);
-
-	bt->bst_cookie = sc;
-	bt->bst_parent = rman_get_bustag(sc->sc_mem_res);
-	bt->bst_type = type;
-	return (bt);
-}
diff --git a/sys/sparc64/pci/psychovar.h b/sys/sparc64/pci/psychovar.h
index 117b7385aeb3..5532d16bbe6f 100644
--- a/sys/sparc64/pci/psychovar.h
+++ b/sys/sparc64/pci/psychovar.h
@@ -65,7 +65,6 @@ struct psycho_softc {
 
 	/* Tags for PCI access */
 	bus_space_tag_t			sc_pci_cfgt;
-	bus_space_tag_t			sc_pci_memt;
 	bus_space_tag_t			sc_pci_iot;
 	bus_dma_tag_t			sc_pci_dmat;
 
diff --git a/sys/sparc64/pci/sbbc.c b/sys/sparc64/pci/sbbc.c
index be95f0532114..812b350ea89c 100644
--- a/sys/sparc64/pci/sbbc.c
+++ b/sys/sparc64/pci/sbbc.c
@@ -259,6 +259,9 @@ static const char *sbbc_serengeti_set_console_input(char *new);
 /*
  * SBBC PCI interface
  */
+static bus_activate_resource_t sbbc_bus_activate_resource;
+static bus_adjust_resource_t sbbc_bus_adjust_resource;
+static bus_deactivate_resource_t sbbc_bus_deactivate_resource;
 static bus_alloc_resource_t sbbc_bus_alloc_resource;
 static bus_release_resource_t sbbc_bus_release_resource;
 static bus_get_resource_list_t sbbc_bus_get_resource_list;
@@ -278,9 +281,13 @@ static device_method_t sbbc_pci_methods[] = {
 
 	DEVMETHOD(bus_print_child,	bus_generic_print_child),
 	DEVMETHOD(bus_alloc_resource,	sbbc_bus_alloc_resource),
+	DEVMETHOD(bus_activate_resource,sbbc_bus_activate_resource),
+	DEVMETHOD(bus_deactivate_resource,sbbc_bus_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	sbbc_bus_adjust_resource),
 	DEVMETHOD(bus_release_resource,	sbbc_bus_release_resource),
 	DEVMETHOD(bus_setup_intr,	sbbc_bus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	sbbc_bus_teardown_intr),
+	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
 	DEVMETHOD(bus_get_resource_list, sbbc_bus_get_resource_list),
 
 	/* clock interface */
@@ -333,11 +340,8 @@ sbbc_pci_attach(device_t dev)
 
 	sc = device_get_softc(dev);
 	rid = SBBC_PCI_BAR;
-	/*
-	 * Note that we don't activate the resource so it's not mapped twice
-	 * but only once by the firmware.
-	 */
-	sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 0);
+	sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	    RF_ACTIVE);
 	if (sc->sc_res == NULL) {
 		device_printf(dev, "failed to allocate resources\n");
 		return (ENXIO);
@@ -402,24 +406,52 @@ sbbc_bus_alloc_resource(device_t dev, device_t child __unused, int type,
 	sc = device_get_softc(dev);
 	switch (type) {
 	case SYS_RES_IRQ:
-		return (BUS_ALLOC_RESOURCE(device_get_parent(dev), dev, type,
-		    rid, start, end, count, flags));
+		return (bus_generic_alloc_resource(dev, dev, type, rid, start,
+		    end, count, flags));
 	case SYS_RES_MEMORY:
 		return (sc->sc_res);
 	default:
 		return (NULL);
-		/* NOTREACHED */
 	}
 }
 
+static int
+sbbc_bus_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *res)
+{
+
+	if (type == SYS_RES_MEMORY)
+		return (0);
+	return (bus_generic_activate_resource(bus, child, type, rid, res));
+}
+
+static int
+sbbc_bus_deactivate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *res)
+{
+
+	if (type == SYS_RES_MEMORY)
+		return (0);
+	return (bus_generic_deactivate_resource(bus, child, type, rid, res));
+}
+
+static int
+sbbc_bus_adjust_resource(device_t bus __unused, device_t child __unused,
+    int type __unused, struct resource *res __unused, u_long start __unused,
+    u_long end __unused)
+{
+
+	return (ENXIO);
+}
+
 static int
 sbbc_bus_release_resource(device_t dev, device_t child __unused, int type,
     int rid, struct resource *res)
 {
 
 	if (type == SYS_RES_IRQ)
-		return (BUS_RELEASE_RESOURCE(device_get_parent(dev), dev,
-		    type, rid, res));
+		return (bus_generic_release_resource(dev, dev, type, rid,
+		    res));
 	return (0);
 }
 
@@ -427,7 +459,7 @@ static struct resource_list *
 sbbc_bus_get_resource_list(device_t dev, device_t child __unused)
 {
 
-	return (BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev));
+	return (bus_generic_get_resource_list(dev, dev));
 }
 
 static int
@@ -436,8 +468,8 @@ sbbc_bus_setup_intr(device_t dev, device_t child __unused,
     driver_intr_t *intr, void *arg, void **cookiep)
 {
 
-	return (BUS_SETUP_INTR(device_get_parent(dev), dev, res, flags, filt,
-	    intr, arg, cookiep));
+	return (bus_generic_setup_intr(dev, dev, res, flags, filt, intr, arg,
+	    cookiep));
 }
 
 static int
@@ -445,7 +477,7 @@ sbbc_bus_teardown_intr(device_t dev, device_t child __unused,
     struct resource *res, void *cookie)
 {
 
-	return (BUS_TEARDOWN_INTR(device_get_parent(dev), dev, res, cookie));
+	return (bus_generic_teardown_intr(dev, dev, res, cookie));
 }
 
 /*
diff --git a/sys/sparc64/pci/schizo.c b/sys/sparc64/pci/schizo.c
index 4d1a5a74f88c..b0e754512515 100644
--- a/sys/sparc64/pci/schizo.c
+++ b/sys/sparc64/pci/schizo.c
@@ -91,7 +91,6 @@ static void schizo_intr_clear(void *);
 static int schizo_intr_register(struct schizo_softc *sc, u_int ino);
 static int schizo_get_intrmap(struct schizo_softc *, u_int,
     bus_addr_t *, bus_addr_t *);
-static bus_space_tag_t schizo_alloc_bus_tag(struct schizo_softc *, int);
 static timecounter_get_t schizo_get_timecount;
 
 /* Interrupt handlers */
@@ -113,8 +112,7 @@ static bus_read_ivar_t schizo_read_ivar;
 static bus_setup_intr_t schizo_setup_intr;
 static bus_alloc_resource_t schizo_alloc_resource;
 static bus_activate_resource_t schizo_activate_resource;
-static bus_deactivate_resource_t schizo_deactivate_resource;
-static bus_release_resource_t schizo_release_resource;
+static bus_adjust_resource_t schizo_adjust_resource;
 static bus_get_dma_tag_t schizo_get_dma_tag;
 static pcib_maxslots_t schizo_maxslots;
 static pcib_read_config_t schizo_read_config;
@@ -137,9 +135,10 @@ static device_method_t schizo_methods[] = {
 	DEVMETHOD(bus_setup_intr,	schizo_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	schizo_alloc_resource),
-	DEVMETHOD(bus_activate_resource,	schizo_activate_resource),
-	DEVMETHOD(bus_deactivate_resource,	schizo_deactivate_resource),
-	DEVMETHOD(bus_release_resource,	schizo_release_resource),
+	DEVMETHOD(bus_activate_resource, schizo_activate_resource),
+	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	schizo_adjust_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_get_dma_tag,	schizo_get_dma_tag),
 
 	/* pcib interface */
@@ -151,7 +150,7 @@ static device_method_t schizo_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	schizo_get_node),
 
-	/* ofw_pci interface */ 	 
+	/* ofw_pci interface */
 	DEVMETHOD(ofw_pci_setup_device,	schizo_setup_device),
 
 	KOBJMETHOD_END
@@ -501,7 +500,8 @@ schizo_attach(device_t dev)
 	 * Set up the IOMMU.  Schizo, Tomatillo and XMITS all have
 	 * one per PBM.  Schizo and XMITS additionally have a streaming
 	 * buffer, in Schizo version < 5 (i.e. revision < 2.3) it's
-	 * affected by several errata and basically unusable though.
+	 * affected by several errata though.  However, except for context
+	 * flushes, taking advantage of it should be okay even with those.
 	 */
 	memcpy(&sc->sc_dma_methods, &iommu_dma_methods,
 	    sizeof(sc->sc_dma_methods));
@@ -509,8 +509,7 @@ schizo_attach(device_t dev)
 	sc->sc_is.sis_is.is_flags = IOMMU_PRESERVE_PROM;
 	sc->sc_is.sis_is.is_pmaxaddr = IOMMU_MAXADDR(STX_IOMMU_BITS);
 	sc->sc_is.sis_is.is_sb[0] = sc->sc_is.sis_is.is_sb[1] = 0;
-	if (OF_getproplen(node, "no-streaming-cache") < 0 &&
-	    !(sc->sc_mode == SCHIZO_MODE_SCZ && sc->sc_ver < 5))
+	if (OF_getproplen(node, "no-streaming-cache") < 0)
 		sc->sc_is.sis_is.is_sb[0] = STX_PCI_STRBUF;
 
 #define	TSBCASE(x)							\
@@ -580,14 +579,20 @@ schizo_attach(device_t dev)
 	SLIST_INSERT_HEAD(&schizo_softcs, sc, sc_link);
 
 	/* Allocate our tags. */
-	sc->sc_pci_memt = schizo_alloc_bus_tag(sc, PCI_MEMORY_BUS_SPACE);
-	sc->sc_pci_iot = schizo_alloc_bus_tag(sc, PCI_IO_BUS_SPACE);
-	sc->sc_pci_cfgt = schizo_alloc_bus_tag(sc, PCI_CONFIG_BUS_SPACE);
+	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, rman_get_bustag(
+	    sc->sc_mem_res[STX_PCI]), PCI_IO_BUS_SPACE, NULL);
+	if (sc->sc_pci_iot == NULL)
+		panic("%s: could not allocate PCI I/O tag", __func__);
+	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, rman_get_bustag(
+	    sc->sc_mem_res[STX_PCI]), PCI_CONFIG_BUS_SPACE, NULL);
+	if (sc->sc_pci_cfgt == NULL)
+		panic("%s: could not allocate PCI configuration space tag",
+		    __func__);
 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0,
 	    sc->sc_is.sis_is.is_pmaxaddr, ~0, NULL, NULL,
 	    sc->sc_is.sis_is.is_pmaxaddr, 0xff, 0xffffffff, 0, NULL, NULL,
 	    &sc->sc_pci_dmat) != 0)
-		panic("%s: bus_dma_tag_create failed", __func__);
+		panic("%s: could not create PCI DMA tag", __func__);
 	/* Customize the tag. */
 	sc->sc_pci_dmat->dt_cookie = &sc->sc_is;
 	sc->sc_pci_dmat->dt_mt = &sc->sc_dma_methods;
@@ -1161,7 +1166,7 @@ schizo_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
 
 	if ((op & BUS_DMASYNC_POSTREAD) != 0) {
 		/*
-	 	 * Note that in order to allow this function to be called from
+		 * Note that in order to allow this function to be called from
 		 * filters we would need to use a spin mutex for serialization
 		 * but given that these disable interrupts we have to emulate
 		 * one.
@@ -1333,14 +1338,10 @@ schizo_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	struct schizo_softc *sc;
 	struct resource *rv;
 	struct rman *rm;
-	bus_space_tag_t bt;
-	bus_space_handle_t bh;
-	int needactivate = flags & RF_ACTIVE;
-
-	flags &= ~RF_ACTIVE;
 
 	sc = device_get_softc(bus);
-	if (type == SYS_RES_IRQ) {
+	switch (type) {
+	case SYS_RES_IRQ:
 		/*
 		 * XXX: Don't accept blank ranges for now, only single
 		 * interrupts.  The other case should not happen with
@@ -1351,38 +1352,28 @@ schizo_alloc_resource(device_t bus, device_t child, int type, int *rid,
 		if (start != end)
 			panic("%s: XXX: interrupt range", __func__);
 		start = end = INTMAP_VEC(sc->sc_ign, end);
-		return (BUS_ALLOC_RESOURCE(device_get_parent(bus), child,
-		    type, rid, start, end, count, flags));
-	}
-	switch (type) {
+		return (bus_generic_alloc_resource(bus, child, type, rid,
+		     start, end, count, flags));
 	case SYS_RES_MEMORY:
 		rm = &sc->sc_pci_mem_rman;
-		bt = sc->sc_pci_memt;
-		bh = sc->sc_pci_bh[OFW_PCI_CS_MEM32];
 		break;
 	case SYS_RES_IOPORT:
 		rm = &sc->sc_pci_io_rman;
-		bt = sc->sc_pci_iot;
-		bh = sc->sc_pci_bh[OFW_PCI_CS_IO];
 		break;
 	default:
 		return (NULL);
-		/* NOTREACHED */
 	}
 
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
+	    child);
 	if (rv == NULL)
 		return (NULL);
 	rman_set_rid(rv, *rid);
-	bh += rman_get_start(rv);
-	rman_set_bustag(rv, bt);
-	rman_set_bushandle(rv, bh);
 
-	if (needactivate) {
-		if (bus_activate_resource(child, type, *rid, rv)) {
-			rman_release_resource(rv);
-			return (NULL);
-		}
+	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
+	    *rid, rv) != 0) {
+		rman_release_resource(rv);
+		return (NULL);
 	}
 	return (rv);
 }
@@ -1391,56 +1382,56 @@ static int
 schizo_activate_resource(device_t bus, device_t child, int type, int rid,
     struct resource *r)
 {
-	void *p;
-	int error;
+	struct schizo_softc *sc;
+	struct bus_space_tag *tag;
 
-	if (type == SYS_RES_IRQ)
-		return (BUS_ACTIVATE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (type == SYS_RES_MEMORY) {
-		/*
-		 * Need to memory-map the device space, as some drivers
-		 * depend on the virtual address being set and usable.
-		 */
-		error = sparc64_bus_mem_map(rman_get_bustag(r),
-		    rman_get_bushandle(r), rman_get_size(r), 0, 0, &p);
-		if (error != 0)
-			return (error);
-		rman_set_virtual(r, p);
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_activate_resource(bus, child, type, rid,
+		    r));
+	case SYS_RES_MEMORY:
+		tag = sparc64_alloc_bus_tag(r, rman_get_bustag(
+		    sc->sc_mem_res[STX_PCI]), PCI_MEMORY_BUS_SPACE, NULL);
+		if (tag == NULL)
+			return (ENOMEM);
+		rman_set_bustag(r, tag);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
+		    rman_get_start(r));
+		break;
+	case SYS_RES_IOPORT:
+		rman_set_bustag(r, sc->sc_pci_iot);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
+		    rman_get_start(r));
+		break;
 	}
 	return (rman_activate_resource(r));
 }
 
 static int
-schizo_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+schizo_adjust_resource(device_t bus, device_t child, int type,
+    struct resource *r, u_long start, u_long end)
 {
+	struct schizo_softc *sc;
+	struct rman *rm;
 
-	if (type == SYS_RES_IRQ)
-		return (BUS_DEACTIVATE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (type == SYS_RES_MEMORY) {
-		sparc64_bus_mem_unmap(rman_get_virtual(r), rman_get_size(r));
-		rman_set_virtual(r, NULL);
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_adjust_resource(bus, child, type, r,
+		    start, end));
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_pci_mem_rman;
+		break;
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_pci_io_rman;
+		break;
+	default:
+		return (EINVAL);
 	}
-	return (rman_deactivate_resource(r));
-}
-
-static int
-schizo_release_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-	int error;
-
-	if (type == SYS_RES_IRQ)
-		return (BUS_RELEASE_RESOURCE(device_get_parent(bus), child,
-		    type, rid, r));
-	if (rman_get_flags(r) & RF_ACTIVE) {
-		error = bus_deactivate_resource(child, type, rid, r);
-		if (error)
-			return (error);
-	}
-	return (rman_release_resource(r));
+	if (rman_is_region_manager(r, rm) == 0)
+		return (EINVAL);
+	return (rman_adjust_resource(r, start, end));
 }
 
 static bus_dma_tag_t
@@ -1473,7 +1464,7 @@ schizo_setup_device(device_t bus, device_t child)
 	/*
 	 * Disable bus parking in order to work around a bus hang caused by
 	 * Casinni/Skyhawk combinations.
-	 */ 
+	 */
 	if (OF_getproplen(ofw_bus_get_node(child), "pci-req-removal") >= 0)
 		SCHIZO_PCI_SET(sc, STX_PCI_CTRL, SCHIZO_PCI_READ_8(sc,
 		    STX_PCI_CTRL) & ~STX_PCI_CTRL_ARB_PARK);
@@ -1498,22 +1489,6 @@ schizo_setup_device(device_t bus, device_t child)
 	}
 }
 
-static bus_space_tag_t
-schizo_alloc_bus_tag(struct schizo_softc *sc, int type)
-{
-	bus_space_tag_t bt;
-
-	bt = malloc(sizeof(struct bus_space_tag), M_DEVBUF,
-	    M_NOWAIT | M_ZERO);
-	if (bt == NULL)
-		panic("%s: out of memory", __func__);
-
-	bt->bst_cookie = sc;
-	bt->bst_parent = rman_get_bustag(sc->sc_mem_res[STX_PCI]);
-	bt->bst_type = type;
-	return (bt);
-}
-
 static u_int
 schizo_get_timecount(struct timecounter *tc)
 {
diff --git a/sys/sparc64/pci/schizovar.h b/sys/sparc64/pci/schizovar.h
index 8e18b58d7b15..3fe7cdc353ca 100644
--- a/sys/sparc64/pci/schizovar.h
+++ b/sys/sparc64/pci/schizovar.h
@@ -81,7 +81,6 @@ struct schizo_softc {
 	bus_space_handle_t		sc_pci_bh[STX_NRANGE];
 	bus_space_tag_t			sc_pci_cfgt;
 	bus_space_tag_t			sc_pci_iot;
-	bus_space_tag_t			sc_pci_memt;
 	bus_dma_tag_t			sc_pci_dmat;
 
 	uint32_t			sc_stats_dma_ce;
diff --git a/sys/sparc64/sbus/dma_sbus.c b/sys/sparc64/sbus/dma_sbus.c
index 1310f375a40f..2594243df03c 100644
--- a/sys/sparc64/sbus/dma_sbus.c
+++ b/sys/sparc64/sbus/dma_sbus.c
@@ -118,6 +118,7 @@ static device_method_t dma_methods[] = {
 	DEVMETHOD(bus_alloc_resource,	bus_generic_rl_alloc_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	bus_generic_adjust_resource),
 	DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource),
 	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
diff --git a/sys/sparc64/sbus/lsi64854.c b/sys/sparc64/sbus/lsi64854.c
index 61a77663ba0f..ba9b5d8277de 100644
--- a/sys/sparc64/sbus/lsi64854.c
+++ b/sys/sparc64/sbus/lsi64854.c
@@ -85,7 +85,11 @@ __FBSDID("$FreeBSD$");
 #define LDB_PP		4
 #define LDB_ANY		0xff
 int lsi64854debug = 0;
-#define DPRINTF(a,x) do { if (lsi64854debug & (a)) printf x ; } while (0)
+#define DPRINTF(a,x)							\
+	do {								\
+		if ((lsi64854debug & (a)) != 0)				\
+			printf x;					\
+	} while (/* CONSTCOND */0)
 #else
 #define DPRINTF(a,x)
 #endif
@@ -94,12 +98,12 @@ int lsi64854debug = 0;
 
 static void	lsi64854_reset(struct lsi64854_softc *);
 static void	lsi64854_map_scsi(void *, bus_dma_segment_t *, int, int);
-static int	lsi64854_setup(struct lsi64854_softc *, caddr_t *, size_t *,
+static int	lsi64854_setup(struct lsi64854_softc *, void **, size_t *,
 		    int, size_t *);
 static int	lsi64854_scsi_intr(void *);
 static int	lsi64854_enet_intr(void *);
-static int	lsi64854_setup_pp(struct lsi64854_softc *, caddr_t *, size_t *,
-		    int, size_t *);
+static int	lsi64854_setup_pp(struct lsi64854_softc *, void **,
+		    size_t *, int, size_t *);
 static int	lsi64854_pp_intr(void *);
 
 /*
@@ -235,7 +239,7 @@ lsi64854_detach(struct lsi64854_softc *sc)
 		else							\
 			panic(MSG);					\
 	}								\
-} while (0)
+} while (/* CONSTCOND */0)
 
 #define DMA_DRAIN(sc, dontpanic) do {					\
 	uint32_t csr;							\
@@ -264,7 +268,7 @@ lsi64854_detach(struct lsi64854_softc *sc)
 	 * rev0 & rev1 call this PACKCNT				\
 	 */								\
 	DMAWAIT(sc, L64854_GCSR(sc) & L64854_DRAINING, "DRAINING", dontpanic);\
-} while(0)
+} while (/* CONSTCOND */0)
 
 #define DMA_FLUSH(sc, dontpanic) do {					\
 	uint32_t csr;							\
@@ -279,7 +283,7 @@ lsi64854_detach(struct lsi64854_softc *sc)
 	csr &= ~(L64854_WRITE|L64854_EN_DMA); /* no-ops on ENET */	\
 	csr |= L64854_INVALIDATE;	 	/* XXX FAS ? */		\
 	L64854_SCSR(sc,csr);						\
-} while(0)
+} while (/* CONSTCOND */0)
 
 static void
 lsi64854_reset(struct lsi64854_softc *sc)
@@ -373,7 +377,7 @@ lsi64854_map_scsi(void *arg, bus_dma_segment_t *segs, int nseg, int error)
  * setup a DMA transfer
  */
 static int
-lsi64854_setup(struct lsi64854_softc *sc, caddr_t *addr, size_t *len,
+lsi64854_setup(struct lsi64854_softc *sc, void **addr, size_t *len,
     int datain, size_t *dmasize)
 {
 	long bcnt;
@@ -547,7 +551,7 @@ lsi64854_scsi_intr(void *arg)
 	}
 
 	*sc->sc_dmalen -= trans;
-	*sc->sc_dmaaddr += trans;
+	*sc->sc_dmaaddr = (char *)*sc->sc_dmaaddr + trans;
 
 #if 0	/* this is not normal operation just yet */
 	if (*sc->sc_dmalen == 0 || nsc->sc_phase != nsc->sc_prevphase)
@@ -620,7 +624,7 @@ lsi64854_map_pp(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
  * setup a DMA transfer
  */
 static int
-lsi64854_setup_pp(struct lsi64854_softc *sc, caddr_t *addr, size_t *len,
+lsi64854_setup_pp(struct lsi64854_softc *sc, void **addr, size_t *len,
     int datain, size_t *dmasize)
 {
 	uint32_t csr;
@@ -714,7 +718,7 @@ lsi64854_pp_intr(void *arg)
 	if (trans < 0)				/* transferred < 0? */
 		trans = sc->sc_dmasize;
 	*sc->sc_dmalen -= trans;
-	*sc->sc_dmaaddr += trans;
+	*sc->sc_dmaaddr = (char *)*sc->sc_dmaaddr + trans;
 
 	if (sc->sc_dmasize != 0) {
 		bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap,
diff --git a/sys/sparc64/sbus/lsi64854var.h b/sys/sparc64/sbus/lsi64854var.h
index 209a0c5c8365..c1a8e99cd4ca 100644
--- a/sys/sparc64/sbus/lsi64854var.h
+++ b/sys/sparc64/sbus/lsi64854var.h
@@ -51,12 +51,12 @@ struct lsi64854_softc {
 	bus_dma_tag_t		sc_buffer_dmat;
 	int			sc_datain;
 	size_t			sc_dmasize;
-	caddr_t			*sc_dmaaddr;
+	void			**sc_dmaaddr;
 	size_t			*sc_dmalen;
 
 	void	(*reset)(struct lsi64854_softc *);/* reset routine */
-	int	(*setup)(struct lsi64854_softc *, caddr_t *, size_t *,
-			 int, size_t *);	/* DMA setup */
+	int	(*setup)(struct lsi64854_softc *, void **, size_t *,
+		    int, size_t *);		/* DMA setup */
 	int	(*intr)(void *);		/* interrupt handler */
 
 	u_int 			sc_dmactl;
@@ -78,7 +78,7 @@ struct lsi64854_softc {
 	uint32_t csr = L64854_GCSR(sc);		\
 	csr |= L64854_INT_EN;			\
 	L64854_SCSR(sc, csr);			\
-} while (0)
+} while (/* CONSTCOND */0)
 
 #define DMA_ISINTR(sc)	(L64854_GCSR(sc) & (D_INT_PEND|D_ERR_PEND))
 
@@ -87,7 +87,7 @@ struct lsi64854_softc {
 	csr |= D_EN_DMA;			\
 	L64854_SCSR(sc, csr);			\
 	sc->sc_active = 1;			\
-} while (0)
+} while (/* CONSTCOND */0)
 
 int	lsi64854_attach(struct lsi64854_softc *);
 int	lsi64854_detach(struct lsi64854_softc *);
diff --git a/sys/sparc64/sbus/sbus.c b/sys/sparc64/sbus/sbus.c
index dbe38598265a..f9f78da45599 100644
--- a/sys/sparc64/sbus/sbus.c
+++ b/sys/sparc64/sbus/sbus.c
@@ -108,7 +108,6 @@ struct sbus_rd {
 struct sbus_softc {
 	device_t		sc_dev;
 	bus_dma_tag_t		sc_cdmatag;
-	bus_space_tag_t		sc_cbustag;
 	int			sc_clockfreq;	/* clock frequency (in Hz) */
 	int			sc_nrange;
 	struct sbus_rd		*sc_rd;
@@ -137,9 +136,9 @@ static bus_read_ivar_t sbus_read_ivar;
 static bus_get_resource_list_t sbus_get_resource_list;
 static bus_setup_intr_t sbus_setup_intr;
 static bus_alloc_resource_t sbus_alloc_resource;
-static bus_release_resource_t sbus_release_resource;
 static bus_activate_resource_t sbus_activate_resource;
-static bus_deactivate_resource_t sbus_deactivate_resource;
+static bus_adjust_resource_t sbus_adjust_resource;
+static bus_release_resource_t sbus_release_resource;
 static bus_get_dma_tag_t sbus_get_dma_tag;
 static ofw_bus_get_devinfo_t sbus_get_devinfo;
 
@@ -153,7 +152,6 @@ static void sbus_intr_assign(void *);
 static void sbus_intr_clear(void *);
 static int sbus_find_intrmap(struct sbus_softc *, u_int, bus_addr_t *,
     bus_addr_t *);
-static bus_space_tag_t sbus_alloc_bustag(struct sbus_softc *);
 static driver_intr_t sbus_overtemp;
 static driver_intr_t sbus_pwrfail;
 static int sbus_print_res(struct sbus_devinfo *);
@@ -171,10 +169,11 @@ static device_method_t sbus_methods[] = {
 	DEVMETHOD(bus_probe_nomatch,	sbus_probe_nomatch),
 	DEVMETHOD(bus_read_ivar,	sbus_read_ivar),
 	DEVMETHOD(bus_alloc_resource,	sbus_alloc_resource),
-	DEVMETHOD(bus_activate_resource,	sbus_activate_resource),
-	DEVMETHOD(bus_deactivate_resource,	sbus_deactivate_resource),
+	DEVMETHOD(bus_activate_resource, sbus_activate_resource),
+	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	sbus_adjust_resource),
 	DEVMETHOD(bus_release_resource,	sbus_release_resource),
-	DEVMETHOD(bus_setup_intr, 	sbus_setup_intr),
+	DEVMETHOD(bus_setup_intr,	sbus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
 	DEVMETHOD(bus_get_resource_list, sbus_get_resource_list),
@@ -284,7 +283,6 @@ sbus_attach(device_t dev)
 	if (OF_getprop(node, "interrupts", &prop, sizeof(prop)) == -1)
 		panic("%s: cannot get IGN", __func__);
 	sc->sc_ign = INTIGN(prop);
-	sc->sc_cbustag = sbus_alloc_bustag(sc);
 
 	/*
 	 * Record clock frequency for synchronous SCSI.
@@ -343,7 +341,6 @@ sbus_attach(device_t dev)
 		sc->sc_burst =
 		    (SBUS_BURST64_DEF << SBUS_BURST64_SHIFT) | SBUS_BURST_DEF;
 
-
 	/* initalise the IOMMU */
 
 	/* punch in our copies */
@@ -375,7 +372,7 @@ sbus_attach(device_t dev)
 	sc->sc_cdmatag->dt_cookie = &sc->sc_is;
 	sc->sc_cdmatag->dt_mt = &iommu_dma_methods;
 
- 	/*
+	/*
 	 * Hunt through all the interrupt mapping regs and register our
 	 * interrupt controller for the corresponding interrupt vectors.
 	 * We do this early in order to be able to catch stray interrupts.
@@ -701,12 +698,12 @@ sbus_setup_intr(device_t dev, device_t child, struct resource *ires, int flags,
 	/*
 	 * Make sure the vector is fully specified and we registered
 	 * our interrupt controller for it.
- 	 */
+	 */
 	vec = rman_get_start(ires);
 	if (INTIGN(vec) != sc->sc_ign || intr_vectors[vec].iv_ic != &sbus_ic) {
 		device_printf(dev, "invalid interrupt vector 0x%lx\n", vec);
- 		return (EINVAL);
- 	}
+		return (EINVAL);
+	}
 	return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr,
 	    arg, cookiep));
 }
@@ -721,14 +718,12 @@ sbus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	struct resource_list *rl;
 	struct resource_list_entry *rle;
 	device_t schild;
-	bus_space_handle_t bh;
 	bus_addr_t toffs;
 	bus_size_t tend;
 	int i, slot;
-	int isdefault, needactivate, passthrough;
+	int isdefault, passthrough;
 
 	isdefault = (start == 0UL && end == ~0UL);
-	needactivate = flags & RF_ACTIVE;
 	passthrough = (device_get_parent(child) != bus);
 	rle = NULL;
 	sc = device_get_softc(bus);
@@ -751,7 +746,6 @@ sbus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 			}
 		}
 		rm = NULL;
-		bh = toffs = tend = 0;
 		schild = child;
 		while (device_get_parent(schild) != bus)
 			schild = device_get_parent(schild);
@@ -768,24 +762,21 @@ sbus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 			toffs = start - sc->sc_rd[i].rd_coffset;
 			tend = end - sc->sc_rd[i].rd_coffset;
 			rm = &sc->sc_rd[i].rd_rman;
-			bh = sc->sc_rd[i].rd_bushandle;
 			break;
 		}
 		if (rm == NULL)
 			return (NULL);
-		flags &= ~RF_ACTIVE;
-		rv = rman_reserve_resource(rm, toffs, tend, count, flags,
-		    child);
+
+		rv = rman_reserve_resource(rm, toffs, tend, count, flags &
+		    ~RF_ACTIVE, child);
 		if (rv == NULL)
 			return (NULL);
 		rman_set_rid(rv, *rid);
-		rman_set_bustag(rv, sc->sc_cbustag);
-		rman_set_bushandle(rv, bh + rman_get_start(rv));
-		if (needactivate) {
-			if (bus_activate_resource(child, type, *rid, rv)) {
-				rman_release_resource(rv);
-				return (NULL);
-			}
+
+		if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child,
+		    type, *rid, rv)) {
+			rman_release_resource(rv);
+			return (NULL);
 		}
 		if (!passthrough)
 			rle->res = rv;
@@ -799,41 +790,53 @@ static int
 sbus_activate_resource(device_t bus, device_t child, int type, int rid,
     struct resource *r)
 {
-	void *p;
-	int error;
+	struct sbus_softc *sc;
+	struct bus_space_tag *tag;
+	int i;
 
-	if (type == SYS_RES_IRQ) {
-		return (BUS_ACTIVATE_RESOURCE(device_get_parent(bus),
-		    child, type, rid, r));
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_activate_resource(bus, child, type, rid,
+		    r));
+	case SYS_RES_MEMORY:
+		sc = device_get_softc(bus);
+		for (i = 0; i < sc->sc_nrange; i++) {
+			if (rman_is_region_manager(r,
+			    &sc->sc_rd[i].rd_rman) != 0) {
+				tag = sparc64_alloc_bus_tag(r,
+				    rman_get_bustag(sc->sc_sysio_res),
+				    SBUS_BUS_SPACE, NULL);
+				if (tag == NULL)
+					return (ENOMEM);
+				rman_set_bustag(r, tag);
+				rman_set_bushandle(r,
+				    sc->sc_rd[i].rd_bushandle +
+				    rman_get_start(r));
+				return (rman_activate_resource(r));
+			}
+		}
+		/* FALLTHROUGH */
+	default:
+		return (EINVAL);
 	}
-	if (type == SYS_RES_MEMORY) {
-		/*
-		 * Need to memory-map the device space, as some drivers
-		 * depend on the virtual address being set and usable.
-		 */
-		error = sparc64_bus_mem_map(rman_get_bustag(r),
-		    rman_get_bushandle(r), rman_get_size(r), 0, 0, &p);
-		if (error != 0)
-			return (error);
-		rman_set_virtual(r, p);
-	}
-	return (rman_activate_resource(r));
 }
 
 static int
-sbus_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+sbus_adjust_resource(device_t bus, device_t child, int type,
+    struct resource *r, u_long start, u_long end)
 {
+	struct sbus_softc *sc;
+	int i;
 
-	if (type == SYS_RES_IRQ) {
-		return (BUS_DEACTIVATE_RESOURCE(device_get_parent(bus),
-		    child, type, rid, r));
-	}
 	if (type == SYS_RES_MEMORY) {
-		sparc64_bus_mem_unmap(rman_get_virtual(r), rman_get_size(r));
-		rman_set_virtual(r, NULL);
+		sc = device_get_softc(bus);
+		for (i = 0; i < sc->sc_nrange; i++)
+			if (rman_is_region_manager(r,
+			    &sc->sc_rd[i].rd_rman) != 0)
+				return (rman_adjust_resource(r, start, end));
+		return (EINVAL);
 	}
-	return (rman_deactivate_resource(r));
+	return (bus_generic_adjust_resource(bus, child, type, r, start, end));
 }
 
 static int
@@ -846,23 +849,26 @@ sbus_release_resource(device_t bus, device_t child, int type, int rid,
 
 	passthrough = (device_get_parent(child) != bus);
 	rl = BUS_GET_RESOURCE_LIST(bus, child);
-	if (type == SYS_RES_IRQ)
-		return (resource_list_release(rl, bus, child, type, rid, r));
-	if ((rman_get_flags(r) & RF_ACTIVE) != 0) {
-		error = bus_deactivate_resource(child, type, rid, r);
+	if (type == SYS_RES_MEMORY) {
+		if ((rman_get_flags(r) & RF_ACTIVE) != 0) {
+			error = bus_deactivate_resource(child, type, rid, r);
+			if (error)
+				return (error);
+		}
+		error = rman_release_resource(r);
 		if (error != 0)
 			return (error);
+		if (!passthrough) {
+			rle = resource_list_find(rl, type, rid);
+			KASSERT(rle != NULL,
+			    ("%s: resource entry not found!", __func__));
+			KASSERT(rle->res != NULL,
+			   ("%s: resource entry is not busy", __func__));
+			rle->res = NULL;
+		}
+		return (0);
 	}
-	error = rman_release_resource(r);
-	if (error != 0 || passthrough)
-		return (error);
-	rle = resource_list_find(rl, type, rid);
-	if (rle == NULL)
-		panic("%s: cannot find resource", __func__);
-	if (rle->res == NULL)
-		panic("%s: resource entry is not busy", __func__);
-	rle->res = NULL;
-	return (0);
+	return (resource_list_release(rl, bus, child, type, rid, r));
 }
 
 static bus_dma_tag_t
@@ -918,22 +924,6 @@ sbus_pwrfail(void *arg)
 	shutdown_nice(0);
 }
 
-static bus_space_tag_t
-sbus_alloc_bustag(struct sbus_softc *sc)
-{
-	bus_space_tag_t sbt;
-
-	sbt = (bus_space_tag_t)malloc(sizeof(struct bus_space_tag), M_DEVBUF,
-	    M_NOWAIT | M_ZERO);
-	if (sbt == NULL)
-		panic("%s: out of memory", __func__);
-
-	sbt->bst_cookie = sc;
-	sbt->bst_parent = rman_get_bustag(sc->sc_sysio_res);
-	sbt->bst_type = SBUS_BUS_SPACE;
-	return (sbt);
-}
-
 static int
 sbus_print_res(struct sbus_devinfo *sdi)
 {
diff --git a/sys/sparc64/sparc64/bus_machdep.c b/sys/sparc64/sparc64/bus_machdep.c
index 3e8748f52872..4667e443b280 100644
--- a/sys/sparc64/sparc64/bus_machdep.c
+++ b/sys/sparc64/sparc64/bus_machdep.c
@@ -101,6 +101,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/rman.h>
 #include <sys/smp.h>
 #include <sys/systm.h>
 #include <sys/uio.h>
@@ -665,7 +666,7 @@ nexus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
 		contigfree(vaddr, dmat->dt_maxsize, M_DEVBUF);
 }
 
-struct bus_dma_methods nexus_dma_methods = {
+static struct bus_dma_methods nexus_dma_methods = {
 	nexus_dmamap_create,
 	nexus_dmamap_destroy,
 	nexus_dmamap_load,
@@ -703,22 +704,43 @@ struct bus_dma_tag nexus_dmatag = {
  * Helpers to map/unmap bus memory
  */
 int
-sparc64_bus_mem_map(bus_space_tag_t tag, bus_space_handle_t handle,
-    bus_size_t size, int flags, vm_offset_t vaddr, void **hp)
+bus_space_map(bus_space_tag_t tag, bus_addr_t address, bus_size_t size,
+    int flags, bus_space_handle_t *handlep)
+{
+
+	return (sparc64_bus_mem_map(tag, address, size, flags, 0, handlep));
+}
+
+int
+sparc64_bus_mem_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size,
+    int flags, vm_offset_t vaddr, bus_space_handle_t *hp)
 {
-	vm_offset_t addr;
 	vm_offset_t sva;
 	vm_offset_t va;
 	vm_paddr_t pa;
 	vm_size_t vsz;
 	u_long pm_flags;
 
-	addr = (vm_offset_t)handle;
+	/*
+	 * Given that we use physical access for bus_space(9) there's no need
+	 * need to map anything in unless BUS_SPACE_MAP_LINEAR is requested.
+	 */
+	if ((flags & BUS_SPACE_MAP_LINEAR) == 0) {
+		*hp = addr;
+		return (0);
+	}
+
+	if (tag->bst_cookie == NULL) {
+		printf("%s: resource cookie not set\n", __func__);
+		return (EINVAL);
+	}
+
 	size = round_page(size);
 	if (size == 0) {
 		printf("%s: zero size\n", __func__);
 		return (EINVAL);
 	}
+
 	switch (tag->bst_type) {
 	case PCI_CONFIG_BUS_SPACE:
 	case PCI_IO_BUS_SPACE:
@@ -730,7 +752,7 @@ sparc64_bus_mem_map(bus_space_tag_t tag, bus_space_handle_t handle,
 		break;
 	}
 
-	if (!(flags & BUS_SPACE_MAP_CACHEABLE))
+	if ((flags & BUS_SPACE_MAP_CACHEABLE) == 0)
 		pm_flags |= TD_E;
 
 	if (vaddr != 0L)
@@ -740,9 +762,6 @@ sparc64_bus_mem_map(bus_space_tag_t tag, bus_space_handle_t handle,
 			panic("%s: cannot allocate virtual memory", __func__);
 	}
 
-	/* Preserve page offset. */
-	*hp = (void *)(sva | ((u_long)addr & PAGE_MASK));
-
 	pa = trunc_page(addr);
 	if ((flags & BUS_SPACE_MAP_READONLY) == 0)
 		pm_flags |= TD_W;
@@ -755,17 +774,32 @@ sparc64_bus_mem_map(bus_space_tag_t tag, bus_space_handle_t handle,
 		pa += PAGE_SIZE;
 	} while ((vsz -= PAGE_SIZE) > 0);
 	tlb_range_demap(kernel_pmap, sva, sva + size - 1);
+
+	/* Note: we preserve the page offset. */
+	rman_set_virtual(tag->bst_cookie, (void *)(sva | (addr & PAGE_MASK)));
 	return (0);
 }
 
+void
+bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t handle,
+    bus_size_t size)
+{
+
+	sparc64_bus_mem_unmap(tag, handle, size);
+}
+
 int
-sparc64_bus_mem_unmap(void *bh, bus_size_t size)
+sparc64_bus_mem_unmap(bus_space_tag_t tag, bus_space_handle_t handle,
+    bus_size_t size)
 {
 	vm_offset_t sva;
 	vm_offset_t va;
 	vm_offset_t endva;
 
-	sva = trunc_page((vm_offset_t)bh);
+	if (tag->bst_cookie == NULL ||
+	    (sva = (vm_offset_t)rman_get_virtual(tag->bst_cookie)) == 0)
+		return (0);
+	sva = trunc_page(sva);
 	endva = sva + round_page(size);
 	for (va = sva; va < endva; va += PAGE_SIZE)
 		pmap_kremove_flags(va);
@@ -790,6 +824,25 @@ sparc64_fake_bustag(int space, bus_addr_t addr, struct bus_space_tag *ptag)
 	return (addr);
 }
 
+/*
+ * Allocate a bus tag.
+ */
+bus_space_tag_t
+sparc64_alloc_bus_tag(void *cookie, struct bus_space_tag *ptag, int type,
+    void *barrier)
+{
+	bus_space_tag_t bt;
+
+	bt = malloc(sizeof(struct bus_space_tag), M_DEVBUF, M_NOWAIT);
+	if (bt == NULL)
+		return (NULL);
+	bt->bst_cookie = cookie;
+	bt->bst_parent = ptag;
+	bt->bst_type = type;
+	bt->bst_bus_barrier = barrier;
+	return (bt);
+}
+
 /*
  * Base bus space handlers.
  */
diff --git a/sys/sparc64/sparc64/exception.S b/sys/sparc64/sparc64/exception.S
index 222ddeadd21b..09db4e0797b4 100644
--- a/sys/sparc64/sparc64/exception.S
+++ b/sys/sparc64/sparc64/exception.S
@@ -373,15 +373,15 @@ END(rsf_fatal)
 	_ALIGN_DATA
 	.globl	intrnames, sintrnames
 intrnames:
-	.space	IV_MAX * (MAXCOMLEN + 1)
+	.space	(IV_MAX + PIL_MAX) * (MAXCOMLEN + 1)
 sintrnames:
-	.quad	IV_MAX * (MAXCOMLEN + 1)
+	.quad	(IV_MAX + PIL_MAX) * (MAXCOMLEN + 1)
 
 	.globl	intrcnt, sintrcnt
 intrcnt:
-	.space	IV_MAX * 8
+	.space	(IV_MAX + PIL_MAX) * 8
 sintrcnt:
-	.quad	IV_MAX * 8
+	.quad	(IV_MAX + PIL_MAX) * 8
 
 	.text
 
diff --git a/sys/sparc64/sparc64/genassym.c b/sys/sparc64/sparc64/genassym.c
index 89ec718b94a2..df31805a09ab 100644
--- a/sys/sparc64/sparc64/genassym.c
+++ b/sys/sparc64/sparc64/genassym.c
@@ -43,9 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_page.h>
 #include <vm/vm_map.h>
 
-#ifdef SUN4U
 #include <machine/cache.h>
-#endif
 #include <machine/pcb.h>
 #include <machine/setjmp.h>
 #include <machine/smp.h>
@@ -62,9 +60,7 @@ ASSYM(TAR_VPN_SHIFT, TAR_VPN_SHIFT);
 
 ASSYM(_NCPUBITS, _NCPUBITS);
 
-#ifdef SUN4U
 ASSYM(TLB_DEMAP_ALL, TLB_DEMAP_ALL);
-#endif
 ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT);
 ASSYM(TLB_DEMAP_NUCLEUS, TLB_DEMAP_NUCLEUS);
 ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE);
@@ -82,21 +78,17 @@ ASSYM(PAGE_SIZE_4M, PAGE_SIZE_4M);
 #ifdef SMP
 ASSYM(CSA_PCPU, offsetof(struct cpu_start_args, csa_pcpu));
 ASSYM(CSA_STATE, offsetof(struct cpu_start_args, csa_state));
-#ifdef SUN4U
 ASSYM(CSA_MID, offsetof(struct cpu_start_args, csa_mid));
 ASSYM(CSA_STICK, offsetof(struct cpu_start_args, csa_stick));
 ASSYM(CSA_TICK, offsetof(struct cpu_start_args, csa_tick));
 ASSYM(CSA_TTES, offsetof(struct cpu_start_args, csa_ttes));
 ASSYM(CSA_VER, offsetof(struct cpu_start_args, csa_ver));
 #endif
-#endif
 
-#ifdef SUN4U
 ASSYM(DC_SIZE, offsetof(struct cacheinfo, dc_size));
 ASSYM(DC_LINESIZE, offsetof(struct cacheinfo, dc_linesize));
 ASSYM(IC_SIZE, offsetof(struct cacheinfo, ic_size));
 ASSYM(IC_LINESIZE, offsetof(struct cacheinfo, ic_linesize));
-#endif
 
 ASSYM(KTR_SIZEOF, sizeof(struct ktr_entry));
 ASSYM(KTR_LINE, offsetof(struct ktr_entry, ktr_line));
@@ -112,7 +104,6 @@ ASSYM(KTR_PARM5, offsetof(struct ktr_entry, ktr_parms[4]));
 ASSYM(KTR_PARM6, offsetof(struct ktr_entry, ktr_parms[5]));
 
 ASSYM(TTE_SHIFT, TTE_SHIFT);
-#ifdef SUN4U
 ASSYM(TTE_VPN, offsetof(struct tte, tte_vpn));
 ASSYM(TTE_DATA, offsetof(struct tte, tte_data));
 
@@ -132,7 +123,6 @@ ASSYM(TLB_CXR_PGSZ_MASK, TLB_CXR_PGSZ_MASK);
 ASSYM(TLB_DIRECT_ADDRESS_MASK, TLB_DIRECT_ADDRESS_MASK);
 ASSYM(TLB_DIRECT_TO_TTE_MASK, TLB_DIRECT_TO_TTE_MASK);
 ASSYM(TV_SIZE_BITS, TV_SIZE_BITS);
-#endif
 
 ASSYM(V_INTR, offsetof(struct vmmeter, v_intr));
 
@@ -146,14 +136,12 @@ ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree));
 ASSYM(PC_CNT, offsetof(struct pcpu, pc_cnt));
 ASSYM(PC_SIZEOF, sizeof(struct pcpu));
 
-#ifdef SUN4U
 ASSYM(PC_CACHE, offsetof(struct pcpu, pc_cache));
 ASSYM(PC_MID, offsetof(struct pcpu, pc_mid));
 ASSYM(PC_PMAP, offsetof(struct pcpu, pc_pmap));
 ASSYM(PC_TLB_CTX, offsetof(struct pcpu, pc_tlb_ctx));
 ASSYM(PC_TLB_CTX_MAX, offsetof(struct pcpu, pc_tlb_ctx_max));
 ASSYM(PC_TLB_CTX_MIN, offsetof(struct pcpu, pc_tlb_ctx_min));
-#endif
 
 ASSYM(IR_NEXT, offsetof(struct intr_request, ir_next));
 ASSYM(IR_FUNC, offsetof(struct intr_request, ir_func));
@@ -161,7 +149,7 @@ ASSYM(IR_ARG, offsetof(struct intr_request, ir_arg));
 ASSYM(IR_PRI, offsetof(struct intr_request, ir_pri));
 ASSYM(IR_VEC, offsetof(struct intr_request, ir_vec));
 
-#if defined(SUN4U) && defined(SMP)
+#ifdef SMP
 ASSYM(ICA_PA, offsetof(struct ipi_cache_args, ica_pa));
 
 ASSYM(IRA_MASK, offsetof(struct ipi_rd_args, ira_mask));
@@ -239,14 +227,12 @@ ASSYM(TF_FPRS, offsetof(struct trapframe, tf_fprs));
 ASSYM(TF_FSR, offsetof(struct trapframe, tf_fsr));
 ASSYM(TF_GSR, offsetof(struct trapframe, tf_gsr));
 ASSYM(TF_PIL, offsetof(struct trapframe, tf_pil));
-#ifdef SUN4U
 ASSYM(TF_LEVEL, offsetof(struct trapframe, tf_level));
 ASSYM(TF_SFAR, offsetof(struct trapframe, tf_sfar));
 ASSYM(TF_SFSR, offsetof(struct trapframe, tf_sfsr));
 ASSYM(TF_TAR, offsetof(struct trapframe, tf_tar));
 ASSYM(TF_TYPE, offsetof(struct trapframe, tf_type));
 ASSYM(TF_Y, offsetof(struct trapframe, tf_y));
-#endif
 ASSYM(TF_TNPC, offsetof(struct trapframe, tf_tnpc));
 ASSYM(TF_TPC, offsetof(struct trapframe, tf_tpc));
 ASSYM(TF_TSTATE, offsetof(struct trapframe, tf_tstate));
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index c0cc75da6f5f..272044822873 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -597,11 +597,6 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	wrpr(pil, 0, 0);
 	wrpr(pstate, 0, PSTATE_KERNEL);
 
-	/*
-	 * Finish pmap initialization now that we're ready for mutexes.
-	 */
-	PMAP_LOCK_INIT(kernel_pmap);
-
 	OF_getprop(root, "name", sparc64_model, sizeof(sparc64_model) - 1);
 
 	kdb_init();
@@ -726,7 +721,7 @@ struct sigreturn_args {
  * MPSAFE
  */
 int
-sigreturn(struct thread *td, struct sigreturn_args *uap)
+sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
 {
 	struct proc *p;
 	mcontext_t *mc;
@@ -1020,6 +1015,10 @@ exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
 	tf->tf_out[6] = sp - SPOFF - sizeof(struct frame);
 	tf->tf_tnpc = imgp->entry_addr + 4;
 	tf->tf_tpc = imgp->entry_addr;
+	/*
+	 * While we could adhere to the memory model indicated in the ELF
+	 * header, it turns out that just always using TSO performs best.
+	 */
 	tf->tf_tstate = TSTATE_IE | TSTATE_PEF | TSTATE_MM_TSO;
 
 	td->td_retval[0] = tf->tf_out[0];
diff --git a/sys/sparc64/sparc64/nexus.c b/sys/sparc64/sparc64/nexus.c
index 4c1379e829ef..041f2ff608d6 100644
--- a/sys/sparc64/sparc64/nexus.c
+++ b/sys/sparc64/sparc64/nexus.c
@@ -90,6 +90,7 @@ static bus_teardown_intr_t nexus_teardown_intr;
 static bus_alloc_resource_t nexus_alloc_resource;
 static bus_activate_resource_t nexus_activate_resource;
 static bus_deactivate_resource_t nexus_deactivate_resource;
+static bus_adjust_resource_t nexus_adjust_resource;
 static bus_release_resource_t nexus_release_resource;
 static bus_get_resource_list_t nexus_get_resource_list;
 #ifdef SMP
@@ -122,6 +123,7 @@ static device_method_t nexus_methods[] = {
 	DEVMETHOD(bus_alloc_resource,	nexus_alloc_resource),
 	DEVMETHOD(bus_activate_resource,	nexus_activate_resource),
 	DEVMETHOD(bus_deactivate_resource,	nexus_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	nexus_adjust_resource),
 	DEVMETHOD(bus_release_resource,	nexus_release_resource),
 	DEVMETHOD(bus_setup_intr,	nexus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	nexus_teardown_intr),
@@ -277,22 +279,22 @@ nexus_add_child(device_t dev, u_int order, const char *name, int unit)
 }
 
 static int
-nexus_print_child(device_t dev, device_t child)
+nexus_print_child(device_t bus, device_t child)
 {
 	int rv;
 
-	rv = bus_print_child_header(dev, child);
+	rv = bus_print_child_header(bus, child);
 	rv += nexus_print_res(device_get_ivars(child));
-	rv += bus_print_child_footer(dev, child);
+	rv += bus_print_child_footer(bus, child);
 	return (rv);
 }
 
 static void
-nexus_probe_nomatch(device_t dev, device_t child)
+nexus_probe_nomatch(device_t bus, device_t child)
 {
 	const char *type;
 
-	device_printf(dev, "<%s>", ofw_bus_get_name(child));
+	device_printf(bus, "<%s>", ofw_bus_get_name(child));
 	nexus_print_res(device_get_ivars(child));
 	type = ofw_bus_get_type(child);
 	printf(" type %s (no driver attached)\n",
@@ -300,23 +302,24 @@ nexus_probe_nomatch(device_t dev, device_t child)
 }
 
 static int
-nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
-    driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep)
+nexus_setup_intr(device_t bus __unused, device_t child, struct resource *r,
+    int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg,
+    void **cookiep)
 {
 	int error;
 
-	if (res == NULL)
+	if (r == NULL)
 		panic("%s: NULL interrupt resource!", __func__);
 
-	if ((rman_get_flags(res) & RF_SHAREABLE) == 0)
+	if ((rman_get_flags(r) & RF_SHAREABLE) == 0)
 		flags |= INTR_EXCL;
 
 	/* We depend here on rman_activate_resource() being idempotent. */
-	error = rman_activate_resource(res);
+	error = rman_activate_resource(r);
 	if (error)
 		return (error);
 
-	error = inthand_add(device_get_nameunit(child), rman_get_start(res),
+	error = inthand_add(device_get_nameunit(child), rman_get_start(r),
 	    filt, intr, arg, flags, cookiep);
 
 	/*
@@ -328,7 +331,8 @@ nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
 }
 
 static int
-nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
+nexus_teardown_intr(device_t bus __unused, device_t child __unused,
+    struct resource *r, void *ih)
 {
 
 	inthand_remove(rman_get_start(r), ih);
@@ -337,7 +341,8 @@ nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
 
 #ifdef SMP
 static int
-nexus_bind_intr(device_t dev, device_t child, struct resource *r, int cpu)
+nexus_bind_intr(device_t bus __unused, device_t child __unused,
+    struct resource *r, int cpu)
 {
 
 	return (intr_bind(rman_get_start(r), cpu));
@@ -345,8 +350,8 @@ nexus_bind_intr(device_t dev, device_t child, struct resource *r, int cpu)
 #endif
 
 static int
-nexus_describe_intr(device_t dev, device_t child, struct resource *r,
-    void *cookie, const char *descr)
+nexus_describe_intr(device_t bus __unused, device_t child __unused,
+    struct resource *r, void *cookie, const char *descr)
 {
 
 	return (intr_describe(rman_get_start(r), cookie, descr));
@@ -361,10 +366,9 @@ nexus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 	struct resource *rv;
 	struct resource_list_entry *rle;
 	device_t nexus;
-	int isdefault, needactivate, passthrough;
+	int isdefault, passthrough;
 
 	isdefault = (start == 0UL && end == ~0UL);
-	needactivate = flags & RF_ACTIVE;
 	passthrough = (device_get_parent(child) != bus);
 	nexus = bus;
 	while (strcmp(device_get_name(device_get_parent(nexus)), "root") != 0)
@@ -397,21 +401,16 @@ nexus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 		return (NULL);
 	}
 
-	flags &= ~RF_ACTIVE;
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
+	    child);
 	if (rv == NULL)
 		return (NULL);
 	rman_set_rid(rv, *rid);
-	if (type == SYS_RES_MEMORY) {
-		rman_set_bustag(rv, &nexus_bustag);
-		rman_set_bushandle(rv, rman_get_start(rv));
-	}
 
-	if (needactivate) {
-		if (bus_activate_resource(child, type, *rid, rv) != 0) {
-			rman_release_resource(rv);
-			return (NULL);
-		}
+	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
+	    *rid, rv) != 0) {
+		rman_release_resource(rv);
+		return (NULL);
 	}
 
 	if (!passthrough) {
@@ -425,30 +424,61 @@ nexus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 }
 
 static int
-nexus_activate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+nexus_activate_resource(device_t bus __unused, device_t child __unused,
+    int type, int rid __unused, struct resource *r)
 {
 
-	/* Not much to be done yet... */
+	if (type == SYS_RES_MEMORY) {
+		rman_set_bustag(r, &nexus_bustag);
+		rman_set_bushandle(r, rman_get_start(r));
+	}
 	return (rman_activate_resource(r));
 }
 
 static int
-nexus_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+nexus_deactivate_resource(device_t bus __unused, device_t child __unused,
+    int type __unused, int rid __unused, struct resource *r)
 {
 
-	/* Not much to be done yet... */
 	return (rman_deactivate_resource(r));
 }
 
 static int
-nexus_release_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
+nexus_adjust_resource(device_t bus, device_t child __unused, int type,
+    struct resource *r, u_long start, u_long end)
+{
+	struct nexus_softc *sc;
+	struct rman *rm;
+	device_t nexus;
+
+	nexus = bus;
+	while (strcmp(device_get_name(device_get_parent(nexus)), "root") != 0)
+		nexus = device_get_parent(nexus);
+	sc = device_get_softc(nexus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		rm = &sc->sc_intr_rman;
+		break;
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_mem_rman;
+		break;
+	default:
+		return (EINVAL);
+	}
+	if (rm == NULL)
+		return (ENXIO);
+	if (rman_is_region_manager(r, rm) == 0)
+		return (EINVAL);
+	return (rman_adjust_resource(r, start, end));
+}
+
+static int
+nexus_release_resource(device_t bus __unused, device_t child, int type,
+    int rid, struct resource *r)
 {
 	int error;
 
-	if (rman_get_flags(r) & RF_ACTIVE) {
+	if ((rman_get_flags(r) & RF_ACTIVE) != 0) {
 		error = bus_deactivate_resource(child, type, rid, r);
 		if (error)
 			return (error);
@@ -457,7 +487,7 @@ nexus_release_resource(device_t bus, device_t child, int type, int rid,
 }
 
 static struct resource_list *
-nexus_get_resource_list(device_t dev, device_t child)
+nexus_get_resource_list(device_t bus __unused, device_t child)
 {
 	struct nexus_devinfo *ndi;
 
@@ -466,14 +496,14 @@ nexus_get_resource_list(device_t dev, device_t child)
 }
 
 static bus_dma_tag_t
-nexus_get_dma_tag(device_t bus, device_t child)
+nexus_get_dma_tag(device_t bus __unused, device_t child __unused)
 {
 
 	return (&nexus_dmatag);
 }
 
 static const struct ofw_bus_devinfo *
-nexus_get_devinfo(device_t dev, device_t child)
+nexus_get_devinfo(device_t bus __unused, device_t child)
 {
 	struct nexus_devinfo *ndi;
 
diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c
index 209be106747a..496aff5448dc 100644
--- a/sys/sparc64/sparc64/pmap.c
+++ b/sys/sparc64/sparc64/pmap.c
@@ -100,19 +100,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/tsb.h>
 #include <machine/ver.h>
 
-#define	PMAP_DEBUG
-
-#ifndef	PMAP_SHPGPERPROC
-#define	PMAP_SHPGPERPROC	200
-#endif
-
-/* XXX */
-#include "opt_sched.h"
-#ifndef SCHED_4BSD
-#error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock."
-#endif
-extern struct mtx sched_lock;
-
 /*
  * Virtual address of message buffer
  */
@@ -671,11 +658,9 @@ pmap_bootstrap(u_int cpu_impl)
 
 	/*
 	 * Initialize the kernel pmap (which is statically allocated).
-	 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization
-	 * but sparc64 start up is not ready to initialize mutexes yet.
-	 * It is called in machdep.c.
 	 */
 	pm = kernel_pmap;
+	PMAP_LOCK_INIT(pm);
 	for (i = 0; i < MAXCPU; i++)
 		pm->pm_context[i] = TLB_CTX_KERNEL;
 	CPU_FILL(&pm->pm_active);
@@ -1240,11 +1225,9 @@ pmap_pinit(pmap_t pm)
 	if (pm->pm_tsb_obj == NULL)
 		pm->pm_tsb_obj = vm_object_allocate(OBJT_PHYS, TSB_PAGES);
 
-	mtx_lock_spin(&sched_lock);
 	for (i = 0; i < MAXCPU; i++)
 		pm->pm_context[i] = -1;
 	CPU_ZERO(&pm->pm_active);
-	mtx_unlock_spin(&sched_lock);
 
 	VM_OBJECT_LOCK(pm->pm_tsb_obj);
 	for (i = 0; i < TSB_PAGES; i++) {
@@ -1271,7 +1254,9 @@ pmap_release(pmap_t pm)
 {
 	vm_object_t obj;
 	vm_page_t m;
+#ifdef SMP
 	struct pcpu *pc;
+#endif
 
 	CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p",
 	    pm->pm_context[curcpu], pm->pm_tsb);
@@ -1291,11 +1276,18 @@ pmap_release(pmap_t pm)
 	 * - A process that referenced this pmap ran on a CPU, but we switched
 	 *   to a kernel thread, leaving the pmap pointer unchanged.
 	 */
-	mtx_lock_spin(&sched_lock);
+#ifdef SMP
+	sched_pin();
 	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
-		if (pc->pc_pmap == pm)
-			pc->pc_pmap = NULL;
-	mtx_unlock_spin(&sched_lock);
+		atomic_cmpset_rel_ptr((uintptr_t *)&pc->pc_pmap,
+		    (uintptr_t)pm, (uintptr_t)NULL);
+	sched_unpin();
+#else
+	critical_enter();
+	if (PCPU_GET(pmap) == pm)
+		PCPU_SET(pmap, NULL);
+	critical_exit();
+#endif
 
 	pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES);
 	obj = pm->pm_tsb_obj;
@@ -1425,6 +1417,7 @@ pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp,
 	u_long data;
 	vm_page_t m;
 
+	PMAP_LOCK_ASSERT(pm, MA_OWNED);
 	data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W);
 	if ((data & (TD_PV | TD_W)) == (TD_PV | TD_W)) {
 		m = PHYS_TO_VM_PAGE(TD_PA(data));
@@ -1453,7 +1446,6 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 	if (prot & VM_PROT_WRITE)
 		return;
 
-	vm_page_lock_queues();
 	PMAP_LOCK(pm);
 	if (eva - sva > PMAP_TSB_THRESH) {
 		tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte);
@@ -1465,7 +1457,6 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 		tlb_range_demap(pm, sva, eva - 1);
 	}
 	PMAP_UNLOCK(pm);
-	vm_page_unlock_queues();
 }
 
 /*
@@ -2241,11 +2232,14 @@ pmap_activate(struct thread *td)
 	}
 	PCPU_SET(tlb_ctx, context + 1);
 
-	mtx_lock_spin(&sched_lock);
 	pm->pm_context[curcpu] = context;
+#ifdef SMP
+	CPU_SET_ATOMIC(PCPU_GET(cpuid), &pm->pm_active);
+	atomic_store_ptr((uintptr_t *)PCPU_PTR(pmap), (uintptr_t)pm);
+#else
 	CPU_SET(PCPU_GET(cpuid), &pm->pm_active);
 	PCPU_SET(pmap, pm);
-	mtx_unlock_spin(&sched_lock);
+#endif
 
 	stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb);
 	stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb);
diff --git a/sys/sparc64/sparc64/swtch.S b/sys/sparc64/sparc64/swtch.S
index 75157343cdc8..22db8053f632 100644
--- a/sys/sparc64/sparc64/swtch.S
+++ b/sys/sparc64/sparc64/swtch.S
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2001 Jake Burkholder.
+ * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/tstate.h>
 
 #include "assym.s"
+#include "opt_sched.h"
 
 	.register	%g2, #ignore
 	.register	%g3, #ignore
@@ -66,7 +68,7 @@ ENTRY(cpu_switch)
 	 nop
 	call	savefpctx
 	 add	PCB_REG, PCB_KFP, %o0
-	ba,a	%xcc, 2f
+	ba,a,pt	%xcc, 2f
 	 nop
 
 	/*
@@ -148,7 +150,7 @@ ENTRY(cpu_switch)
 	 * If they are the same we are done.
 	 */
 	cmp	%l2, %l1
-	be,a,pn	%xcc, 7f
+	be,a,pn	%xcc, 8f
 	 nop
 
 	/*
@@ -157,7 +159,7 @@ ENTRY(cpu_switch)
 	 */
 	SET(vmspace0, %i4, %i3)
 	cmp	%i5, %i3
-	be,a,pn	%xcc, 7f
+	be,a,pn	%xcc, 8f
 	 nop
 
 	/*
@@ -180,9 +182,15 @@ ENTRY(cpu_switch)
 	sub	%l3, %l5, %l5
 	mov	1, %l6
 	sllx	%l6, %l5, %l5
+#ifdef SMP
+	add	%l2, %l4, %l4
+	membar	#LoadStore | #StoreStore
+	ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5)
+#else
 	ldx	[%l2 + %l4], %l6
 	andn	%l6, %l5, %l6
 	stx	%l6, [%l2 + %l4]
+#endif
 
 	/*
 	 * Take away its context number.
@@ -194,14 +202,20 @@ ENTRY(cpu_switch)
 
 3:	cmp	%i2, %g0
 	be,pn	%xcc, 4f
-	 lduw	[PCPU(TLB_CTX_MAX)], %i4
-	stx	%i2, [%i0 + TD_LOCK]
+	 add	%i0, TD_LOCK, %l4
+#if defined(SCHED_ULE) && defined(SMP)
+	membar	#LoadStore | #StoreStore
+	ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2)
+#else
+	stx	%i2, [%l4]
+#endif
 
 	/*
 	 * Find a new TLB context.  If we've run out we have to flush all
 	 * user mappings from the TLB and reset the context numbers.
 	 */
 4:	lduw	[PCPU(TLB_CTX)], %i3
+	lduw	[PCPU(TLB_CTX_MAX)], %i4
 	cmp	%i3, %i4
 	bne,a,pt %xcc, 5f
 	 nop
@@ -237,14 +251,24 @@ ENTRY(cpu_switch)
 	sub	%l3, %l5, %l5
 	mov	1, %l6
 	sllx	%l6, %l5, %l5
+#ifdef SMP
+	add	%l1, %l4, %l4
+	ATOMIC_SET_LONG(%l4, %l6, %l7, %l5)
+#else
 	ldx	[%l1 + %l4], %l6
 	or	%l6, %l5, %l6
 	stx	%l6, [%l1 + %l4]
+#endif
 
 	/*
 	 * Make note of the change in pmap.
 	 */
+#ifdef SMP
+	PCPU_ADDR(PMAP, %l4)
+	ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1)
+#else
 	stx	%l1, [PCPU(PMAP)]
+#endif
 
 	/*
 	 * Fiddle the hardware bits.  Set the TSB registers and install the
@@ -264,19 +288,35 @@ ENTRY(cpu_switch)
 	stxa	%i3, [%i5] ASI_DMMU
 	flush	%i4
 
+6:
+#if defined(SCHED_ULE) && defined(SMP)
+	SET(blocked_lock, %l2, %l1)
+	add	%i1, TD_LOCK, %l2
+7:
+	ATOMIC_LOAD_LONG(%l2, %l3)
+	cmp	%l1, %l3
+	be,a,pn	%xcc, 7b
+	 nop
+#endif
+
 	/*
 	 * Done, return and load the new process's window from the stack.
 	 */
-
-6:	ret
-	 restore
-
-7:	cmp	%i2, %g0
-	be,a,pn	%xcc, 6b
-	 nop
-	stx	%i2, [%i0 + TD_LOCK]
 	ret
 	 restore
+
+8:	cmp	%i2, %g0
+	be,pn	%xcc, 6b
+	 add	%i0, TD_LOCK, %l4
+#if defined(SCHED_ULE) && defined(SMP)
+	membar	#LoadStore | #StoreStore
+	ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2)
+	ba,pt	%xcc, 6b
+	 nop
+#else
+	ba,pt	%xcc, 6b
+	 stx	%i2, [%l4]
+#endif
 END(cpu_switch)
 
 ENTRY(savectx)
diff --git a/sys/sparc64/sparc64/sys_machdep.c b/sys/sparc64/sparc64/sys_machdep.c
index 530ff95ed54c..c8e8694af6f3 100644
--- a/sys/sparc64/sparc64/sys_machdep.c
+++ b/sys/sparc64/sparc64/sys_machdep.c
@@ -69,6 +69,10 @@ sysarch(struct thread *td, struct sysarch_args *uap)
 			break;
 
 		default:
+#ifdef KTRACE
+			if (KTRPOINT(td, KTR_CAPFAIL))
+				ktrcapfail(CAPFAIL_SYSCALL, 0, 0);
+#endif
 			return (ECAPMODE);
 		}
 	}
diff --git a/sys/sparc64/sparc64/upa.c b/sys/sparc64/sparc64/upa.c
index ecb11a2db4f3..1891de17e51f 100644
--- a/sys/sparc64/sparc64/upa.c
+++ b/sys/sparc64/sparc64/upa.c
@@ -99,10 +99,11 @@ struct upa_softc {
 
 static device_probe_t upa_probe;
 static device_attach_t upa_attach;
-static bus_alloc_resource_t upa_alloc_resource;
-static bus_setup_intr_t upa_setup_intr;
 static bus_print_child_t upa_print_child;
 static bus_probe_nomatch_t upa_probe_nomatch;
+static bus_alloc_resource_t upa_alloc_resource;
+static bus_adjust_resource_t upa_adjust_resource;
+static bus_setup_intr_t upa_setup_intr;
 static bus_get_resource_list_t upa_get_resource_list;
 static ofw_bus_get_devinfo_t upa_get_devinfo;
 
@@ -130,6 +131,7 @@ static device_method_t upa_methods[] = {
 	DEVMETHOD(bus_alloc_resource,	upa_alloc_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_adjust_resource,	upa_adjust_resource),
 	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
 	DEVMETHOD(bus_setup_intr,	upa_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
@@ -506,6 +508,15 @@ upa_setup_intr(device_t dev, device_t child, struct resource *ires, int flags,
 	    arg, cookiep));
 }
 
+static int
+upa_adjust_resource(device_t bus __unused, device_t child __unused,
+    int type __unused, struct resource *r __unused, u_long start __unused,
+    u_long end __unused)
+{
+
+	return (ENXIO);
+}
+
 static struct resource_list *
 upa_get_resource_list(device_t dev, device_t child)
 {
diff --git a/sys/sys/conf.h b/sys/sys/conf.h
index 08e158267f28..7dfa82eff23c 100644
--- a/sys/sys/conf.h
+++ b/sys/sys/conf.h
@@ -55,15 +55,15 @@ struct cdev {
 	void		*__si_reserved;
 	u_int		si_flags;
 #define	SI_ETERNAL	0x0001	/* never destroyed */
-#define SI_ALIAS	0x0002	/* carrier of alias name */
-#define SI_NAMED	0x0004	/* make_dev{_alias} has been called */
-#define SI_CHEAPCLONE	0x0008	/* can be removed_dev'ed when vnode reclaims */
-#define SI_CHILD	0x0010	/* child of another struct cdev **/
-#define SI_DEVOPEN	0x0020	/* opened by device */
-#define SI_CONSOPEN	0x0040	/* opened by console */
-#define SI_DUMPDEV	0x0080	/* is kernel dumpdev */
-#define SI_CANDELETE	0x0100	/* can do BIO_DELETE */
-#define SI_CLONELIST	0x0200	/* on a clone list */
+#define	SI_ALIAS	0x0002	/* carrier of alias name */
+#define	SI_NAMED	0x0004	/* make_dev{_alias} has been called */
+#define	SI_CHEAPCLONE	0x0008	/* can be removed_dev'ed when vnode reclaims */
+#define	SI_CHILD	0x0010	/* child of another struct cdev **/
+#define	SI_DEVOPEN	0x0020	/* opened by device */
+#define	SI_CONSOPEN	0x0040	/* opened by console */
+#define	SI_DUMPDEV	0x0080	/* is kernel dumpdev */
+#define	SI_CANDELETE	0x0100	/* can do BIO_DELETE */
+#define	SI_CLONELIST	0x0200	/* on a clone list */
 	struct timespec	si_atime;
 	struct timespec	si_ctime;
 	struct timespec	si_mtime;
@@ -90,7 +90,7 @@ struct cdev {
 	char		__si_namebuf[SPECNAMELEN + 1];
 };
 
-#define si_snapdata	__si_u.__sid_snapdata
+#define	si_snapdata	__si_u.__sid_snapdata
 
 #ifdef _KERNEL
 
@@ -159,7 +159,7 @@ typedef int dumper_t(
 #define	D_TTY	0x0004
 #define	D_MEM	0x0008
 
-#ifdef _KERNEL 
+#ifdef _KERNEL
 
 #define	D_TYPEMASK	0xffff
 
@@ -167,24 +167,23 @@ typedef int dumper_t(
  * Flags for d_flags which the drivers can set.
  */
 #define	D_TRACKCLOSE	0x00080000	/* track all closes */
-#define D_MMAP_ANON	0x00100000	/* special treatment in vm_mmap.c */
-#define D_PSEUDO	0x00200000	/* make_dev() can return NULL */
-#define D_NEEDGIANT	0x00400000	/* driver want Giant */
+#define	D_MMAP_ANON	0x00100000	/* special treatment in vm_mmap.c */
+#define	D_NEEDGIANT	0x00400000	/* driver want Giant */
 #define	D_NEEDMINOR	0x00800000	/* driver uses clone_create() */
 
 /*
  * Version numbers.
  */
-#define D_VERSION_00	0x20011966
-#define D_VERSION_01	0x17032005	/* Add d_uid,gid,mode & kind */
-#define D_VERSION_02	0x28042009	/* Add d_mmap_single */
-#define D_VERSION_03	0x17122009	/* d_mmap takes memattr,vm_ooffset_t */
-#define D_VERSION	D_VERSION_03
+#define	D_VERSION_00	0x20011966
+#define	D_VERSION_01	0x17032005	/* Add d_uid,gid,mode & kind */
+#define	D_VERSION_02	0x28042009	/* Add d_mmap_single */
+#define	D_VERSION_03	0x17122009	/* d_mmap takes memattr,vm_ooffset_t */
+#define	D_VERSION	D_VERSION_03
 
 /*
  * Flags used for internal housekeeping
  */
-#define D_INIT		0x80000000	/* cdevsw initialized */
+#define	D_INIT		0x80000000	/* cdevsw initialized */
 
 /*
  * Character device switch table
@@ -229,7 +228,7 @@ struct devsw_module_data {
 	/* Do not initialize fields hereafter */
 };
 
-#define DEV_MODULE(name, evh, arg)					\
+#define	DEV_MODULE(name, evh, arg)					\
 static moduledata_t name##_mod = {					\
     #name,								\
     evh,								\
@@ -240,8 +239,8 @@ DECLARE_MODULE(name, name##_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE)
 
 void clone_setup(struct clonedevs **cdp);
 void clone_cleanup(struct clonedevs **);
-#define CLONE_UNITMASK 0xfffff
-#define CLONE_FLAG0 (CLONE_UNITMASK + 1)
+#define	CLONE_UNITMASK	0xfffff
+#define	CLONE_FLAG0	(CLONE_UNITMASK + 1)
 int clone_create(struct clonedevs **, struct cdevsw *, int *unit, struct cdev **dev, int extra);
 
 int	count_dev(struct cdev *_dev);
@@ -281,8 +280,8 @@ struct cdev *make_dev_alias(struct cdev *_pdev, const char *_fmt, ...)
 int	make_dev_alias_p(int _flags, struct cdev **_cdev, struct cdev *_pdev,
 		const char *_fmt, ...) __printflike(4, 5);
 int	make_dev_physpath_alias(int _flags, struct cdev **_cdev,
-	        struct cdev *_pdev, struct cdev *_old_alias,
-                const char *_physpath);
+		struct cdev *_pdev, struct cdev *_old_alias,
+		const char *_physpath);
 void	dev_lock(void);
 void	dev_unlock(void);
 void	setconf(void);
@@ -301,6 +300,9 @@ int	devfs_set_cdevpriv(void *priv, cdevpriv_dtr_t dtr);
 void	devfs_clear_cdevpriv(void);
 void	devfs_fpdrop(struct file *fp);	/* XXX This is not public KPI */
 
+ino_t	devfs_alloc_cdp_inode(void);
+void	devfs_free_cdp_inode(ino_t ino);
+
 #define		UID_ROOT	0
 #define		UID_BIN		3
 #define		UID_UUCP	66
diff --git a/sys/sys/ktrace.h b/sys/sys/ktrace.h
index edd5c028b4b3..d537c1b49f73 100644
--- a/sys/sys/ktrace.h
+++ b/sys/sys/ktrace.h
@@ -177,6 +177,22 @@ struct ktr_proc_ctor {
  */
 #define KTR_PROCDTOR	11
 
+/*
+ * KTR_CAPFAIL - trace capability check failures
+ */
+#define KTR_CAPFAIL	12
+enum ktr_cap_fail_type {
+	CAPFAIL_NOTCAPABLE,	/* insufficient capabilities in cap_check() */
+	CAPFAIL_INCREASE,	/* attempt to increase capabilities */
+	CAPFAIL_SYSCALL,	/* disallowed system call */
+	CAPFAIL_LOOKUP,		/* disallowed VFS lookup */
+};
+struct ktr_cap_fail {
+	enum ktr_cap_fail_type cap_type;
+	cap_rights_t	cap_needed;
+	cap_rights_t	cap_held;
+};
+
 /*
  * KTR_DROP - If this bit is set in ktr_type, then at least one event
  * between the previous record and this record was dropped.
@@ -198,6 +214,7 @@ struct ktr_proc_ctor {
 #define KTRFAC_SYSCTL	(1<<KTR_SYSCTL)
 #define KTRFAC_PROCCTOR	(1<<KTR_PROCCTOR)
 #define KTRFAC_PROCDTOR	(1<<KTR_PROCDTOR)
+#define KTRFAC_CAPFAIL	(1<<KTR_CAPFAIL)
 
 /*
  * trace flags (also in p_traceflags)
@@ -220,6 +237,7 @@ void	ktrprocexit(struct thread *);
 void	ktrprocfork(struct proc *, struct proc *);
 void	ktruserret(struct thread *);
 void	ktrstruct(const char *, void *, size_t);
+void	ktrcapfail(enum ktr_cap_fail_type, cap_rights_t, cap_rights_t);
 #define ktrsockaddr(s) \
 	ktrstruct("sockaddr", (s), ((struct sockaddr *)(s))->sa_len)
 #define ktrstat(s) \
diff --git a/sys/sys/libkern.h b/sys/sys/libkern.h
index a6e2acda67e3..65405d049b3e 100644
--- a/sys/sys/libkern.h
+++ b/sys/sys/libkern.h
@@ -116,6 +116,7 @@ size_t	 strlen(const char *);
 int	 strncasecmp(const char *, const char *, size_t);
 int	 strncmp(const char *, const char *, size_t);
 char	*strncpy(char * __restrict, const char * __restrict, size_t);
+size_t	 strnlen(const char *, size_t);
 char	*strsep(char **, const char *delim);
 size_t	 strspn(const char *, const char *);
 char	*strstr(const char *, const char *);
diff --git a/sys/sys/msgbuf.h b/sys/sys/msgbuf.h
index 67f80a55196a..ed7d49540f36 100644
--- a/sys/sys/msgbuf.h
+++ b/sys/sys/msgbuf.h
@@ -77,7 +77,7 @@ int	msgbuf_peekbytes(struct msgbuf *mbp, char *buf, int buflen,
 void	msgbuf_reinit(struct msgbuf *mbp, void *ptr, int size);
 
 #ifndef MSGBUF_SIZE
-#define	MSGBUF_SIZE	(32768 * 2)
+#define	MSGBUF_SIZE	(32768 * 3)
 #endif
 #endif /* KERNEL */
 
diff --git a/sys/sys/param.h b/sys/sys/param.h
index 0db0223e3663..57ff3a325dad 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -58,9 +58,10 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 900044	/* Master, propagated to newvers */
+#define __FreeBSD_version 1000000	/* Master, propagated to newvers */
 
 #ifdef _KERNEL
+#define	P_OSREL_SIGWAIT		700000
 #define	P_OSREL_SIGSEGV		700004
 #define	P_OSREL_MAP_ANON	800104
 #endif
diff --git a/sys/sys/pipe.h b/sys/sys/pipe.h
index 2592a8d522c2..20b10924ec7d 100644
--- a/sys/sys/pipe.h
+++ b/sys/sys/pipe.h
@@ -112,6 +112,7 @@ struct pipe {
 	u_int	pipe_state;		/* pipe status info */
 	int	pipe_busy;		/* busy flag, mostly to handle rundown sanely */
 	int	pipe_present;		/* still present? */
+	ino_t	pipe_ino;		/* fake inode for stat(2) */
 };
 
 /*
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
index bcc28983a6c8..8f5f769ec0c6 100644
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -302,7 +302,8 @@ enum pmc_event {
 	__PMC_OP(PMCSETCOUNT, "Set initial count/sampling rate")	\
 	__PMC_OP(PMCSTART, "Start a PMC")				\
 	__PMC_OP(PMCSTOP, "Stop a PMC")					\
-	__PMC_OP(WRITELOG, "Write a cookie to the log file")
+	__PMC_OP(WRITELOG, "Write a cookie to the log file")		\
+	__PMC_OP(CLOSELOG, "Close log file")
 
 
 enum pmc_ops {
@@ -1039,6 +1040,7 @@ extern struct pmc_debugflags pmc_debugflags;
 #define	PMC_DEBUG_MIN_SIO		9 /* schedule i/o */
 #define	PMC_DEBUG_MIN_FLS	       10 /* flush */
 #define	PMC_DEBUG_MIN_SAM	       11 /* sample */
+#define	PMC_DEBUG_MIN_CLO	       12 /* close */
 
 #else
 #define	PMCDBG(M,N,L,F,...)		/* nothing */
diff --git a/sys/sys/pmclog.h b/sys/sys/pmclog.h
index 44c5c6c4c4b4..f2b6480a52c2 100644
--- a/sys/sys/pmclog.h
+++ b/sys/sys/pmclog.h
@@ -243,6 +243,7 @@ int	pmclog_configure_log(struct pmc_mdep *_md, struct pmc_owner *_po,
     int _logfd);
 int	pmclog_deconfigure_log(struct pmc_owner *_po);
 int	pmclog_flush(struct pmc_owner *_po);
+int	pmclog_close(struct pmc_owner *_po);
 void	pmclog_initialize(void);
 void	pmclog_process_callchain(struct pmc *_pm, struct pmc_sample *_ps);
 void	pmclog_process_closelog(struct pmc_owner *po);
diff --git a/sys/sys/posix4.h b/sys/sys/posix4.h
index 34f77f43cef5..25dbeae54266 100644
--- a/sys/sys/posix4.h
+++ b/sys/sys/posix4.h
@@ -48,7 +48,7 @@ struct nosys_args;
 extern int syscall_not_present(struct thread *, const char *, struct nosys_args *);
 
 #define SYSCALL_NOT_PRESENT_GEN(SC) \
-int SC (struct thread *td, struct SC##_args *uap) \
+int sys_ ## SC (struct thread *td, struct SC##_args *uap) \
 { \
 	return syscall_not_present(td, #SC , (struct nosys_args *)uap); \
 }
diff --git a/sys/sys/racct.h b/sys/sys/racct.h
index 119b3334c455..095932c22f9a 100644
--- a/sys/sys/racct.h
+++ b/sys/sys/racct.h
@@ -137,6 +137,7 @@ void	racct_create(struct racct **racctp);
 void	racct_destroy(struct racct **racctp);
 
 int	racct_proc_fork(struct proc *parent, struct proc *child);
+void	racct_proc_fork_done(struct proc *child);
 void	racct_proc_exit(struct proc *p);
 
 void	racct_proc_ucred_changed(struct proc *p, struct ucred *oldcred,
diff --git a/sys/sys/signalvar.h b/sys/sys/signalvar.h
index f5abae686ae0..71685e7e5bd7 100644
--- a/sys/sys/signalvar.h
+++ b/sys/sys/signalvar.h
@@ -337,7 +337,7 @@ int	pksignal(struct proc *p, int sig, ksiginfo_t *ksi);
 void	pgsigio(struct sigio **sigiop, int sig, int checkctty);
 void	pgsignal(struct pgrp *pgrp, int sig, int checkctty, ksiginfo_t *ksi);
 int	postsig(int sig);
-void	psignal(struct proc *p, int sig);
+void	kern_psignal(struct proc *p, int sig);
 int	ptracestop(struct thread *td, int sig);
 void	sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *retmask);
 struct sigacts *sigacts_alloc(void);
diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h
index 90bce215543b..d916cf1388b9 100644
--- a/sys/sys/sysent.h
+++ b/sys/sys/sysent.h
@@ -151,6 +151,10 @@ extern struct sysentvec null_sysvec;
 extern struct sysent sysent[];
 extern const char *syscallnames[];
 
+#if defined(__amd64__) || defined(__ia64__)
+extern int i386_read_exec;
+#endif
+
 #define	NO_SYSCALL (-1)
 
 struct module;
@@ -164,6 +168,14 @@ struct syscall_module_data {
 };
 
 #define	MAKE_SYSENT(syscallname)				\
+static struct sysent syscallname##_sysent = {			\
+	(sizeof(struct syscallname ## _args )			\
+	    / sizeof(register_t)),				\
+	(sy_call_t *)& sys_##syscallname,	       		\
+	SYS_AUE_##syscallname					\
+}
+
+#define	MAKE_SYSENT_COMPAT(syscallname)				\
 static struct sysent syscallname##_sysent = {			\
 	(sizeof(struct syscallname ## _args )			\
 	    / sizeof(register_t)),				\
@@ -204,6 +216,15 @@ struct syscall_helper_data {
 	int registered;
 };
 #define SYSCALL_INIT_HELPER(syscallname) {			\
+    .new_sysent = {						\
+	.sy_narg = (sizeof(struct syscallname ## _args )	\
+	    / sizeof(register_t)),				\
+	.sy_call = (sy_call_t *)& sys_ ## syscallname,		\
+	.sy_auevent = SYS_AUE_##syscallname			\
+    },								\
+    .syscall_no = SYS_##syscallname				\
+}
+#define SYSCALL_INIT_HELPER_COMPAT(syscallname) {		\
     .new_sysent = {						\
 	.sy_narg = (sizeof(struct syscallname ## _args )	\
 	    / sizeof(register_t)),				\
diff --git a/sys/sys/sysproto.h b/sys/sys/sysproto.h
index 9bcb79bd0aec..bb692b8a9b96 100644
--- a/sys/sys/sysproto.h
+++ b/sys/sys/sysproto.h
@@ -1734,381 +1734,381 @@ struct posix_fallocate_args {
 	char len_l_[PADL_(off_t)]; off_t len; char len_r_[PADR_(off_t)];
 };
 int	nosys(struct thread *, struct nosys_args *);
-void	sys_exit(struct thread *, struct sys_exit_args *);
-int	fork(struct thread *, struct fork_args *);
-int	read(struct thread *, struct read_args *);
-int	write(struct thread *, struct write_args *);
-int	open(struct thread *, struct open_args *);
-int	close(struct thread *, struct close_args *);
-int	wait4(struct thread *, struct wait_args *);
-int	link(struct thread *, struct link_args *);
-int	unlink(struct thread *, struct unlink_args *);
-int	chdir(struct thread *, struct chdir_args *);
-int	fchdir(struct thread *, struct fchdir_args *);
-int	mknod(struct thread *, struct mknod_args *);
-int	chmod(struct thread *, struct chmod_args *);
-int	chown(struct thread *, struct chown_args *);
-int	obreak(struct thread *, struct obreak_args *);
-int	getpid(struct thread *, struct getpid_args *);
-int	mount(struct thread *, struct mount_args *);
-int	unmount(struct thread *, struct unmount_args *);
-int	setuid(struct thread *, struct setuid_args *);
-int	getuid(struct thread *, struct getuid_args *);
-int	geteuid(struct thread *, struct geteuid_args *);
-int	ptrace(struct thread *, struct ptrace_args *);
-int	recvmsg(struct thread *, struct recvmsg_args *);
-int	sendmsg(struct thread *, struct sendmsg_args *);
-int	recvfrom(struct thread *, struct recvfrom_args *);
-int	accept(struct thread *, struct accept_args *);
-int	getpeername(struct thread *, struct getpeername_args *);
-int	getsockname(struct thread *, struct getsockname_args *);
-int	access(struct thread *, struct access_args *);
-int	chflags(struct thread *, struct chflags_args *);
-int	fchflags(struct thread *, struct fchflags_args *);
-int	sync(struct thread *, struct sync_args *);
-int	kill(struct thread *, struct kill_args *);
-int	getppid(struct thread *, struct getppid_args *);
-int	dup(struct thread *, struct dup_args *);
-int	pipe(struct thread *, struct pipe_args *);
-int	getegid(struct thread *, struct getegid_args *);
-int	profil(struct thread *, struct profil_args *);
-int	ktrace(struct thread *, struct ktrace_args *);
-int	getgid(struct thread *, struct getgid_args *);
-int	getlogin(struct thread *, struct getlogin_args *);
-int	setlogin(struct thread *, struct setlogin_args *);
-int	acct(struct thread *, struct acct_args *);
-int	sigaltstack(struct thread *, struct sigaltstack_args *);
-int	ioctl(struct thread *, struct ioctl_args *);
-int	reboot(struct thread *, struct reboot_args *);
-int	revoke(struct thread *, struct revoke_args *);
-int	symlink(struct thread *, struct symlink_args *);
-int	readlink(struct thread *, struct readlink_args *);
-int	execve(struct thread *, struct execve_args *);
-int	umask(struct thread *, struct umask_args *);
-int	chroot(struct thread *, struct chroot_args *);
-int	msync(struct thread *, struct msync_args *);
-int	vfork(struct thread *, struct vfork_args *);
-int	sbrk(struct thread *, struct sbrk_args *);
-int	sstk(struct thread *, struct sstk_args *);
-int	ovadvise(struct thread *, struct ovadvise_args *);
-int	munmap(struct thread *, struct munmap_args *);
-int	mprotect(struct thread *, struct mprotect_args *);
-int	madvise(struct thread *, struct madvise_args *);
-int	mincore(struct thread *, struct mincore_args *);
-int	getgroups(struct thread *, struct getgroups_args *);
-int	setgroups(struct thread *, struct setgroups_args *);
-int	getpgrp(struct thread *, struct getpgrp_args *);
-int	setpgid(struct thread *, struct setpgid_args *);
-int	setitimer(struct thread *, struct setitimer_args *);
-int	swapon(struct thread *, struct swapon_args *);
-int	getitimer(struct thread *, struct getitimer_args *);
-int	getdtablesize(struct thread *, struct getdtablesize_args *);
-int	dup2(struct thread *, struct dup2_args *);
-int	fcntl(struct thread *, struct fcntl_args *);
-int	select(struct thread *, struct select_args *);
-int	fsync(struct thread *, struct fsync_args *);
-int	setpriority(struct thread *, struct setpriority_args *);
-int	socket(struct thread *, struct socket_args *);
-int	connect(struct thread *, struct connect_args *);
-int	getpriority(struct thread *, struct getpriority_args *);
-int	bind(struct thread *, struct bind_args *);
-int	setsockopt(struct thread *, struct setsockopt_args *);
-int	listen(struct thread *, struct listen_args *);
-int	gettimeofday(struct thread *, struct gettimeofday_args *);
-int	getrusage(struct thread *, struct getrusage_args *);
-int	getsockopt(struct thread *, struct getsockopt_args *);
-int	readv(struct thread *, struct readv_args *);
-int	writev(struct thread *, struct writev_args *);
-int	settimeofday(struct thread *, struct settimeofday_args *);
-int	fchown(struct thread *, struct fchown_args *);
-int	fchmod(struct thread *, struct fchmod_args *);
-int	setreuid(struct thread *, struct setreuid_args *);
-int	setregid(struct thread *, struct setregid_args *);
-int	rename(struct thread *, struct rename_args *);
-int	flock(struct thread *, struct flock_args *);
-int	mkfifo(struct thread *, struct mkfifo_args *);
-int	sendto(struct thread *, struct sendto_args *);
-int	shutdown(struct thread *, struct shutdown_args *);
-int	socketpair(struct thread *, struct socketpair_args *);
-int	mkdir(struct thread *, struct mkdir_args *);
-int	rmdir(struct thread *, struct rmdir_args *);
-int	utimes(struct thread *, struct utimes_args *);
-int	adjtime(struct thread *, struct adjtime_args *);
-int	setsid(struct thread *, struct setsid_args *);
-int	quotactl(struct thread *, struct quotactl_args *);
-int	nlm_syscall(struct thread *, struct nlm_syscall_args *);
-int	nfssvc(struct thread *, struct nfssvc_args *);
-int	lgetfh(struct thread *, struct lgetfh_args *);
-int	getfh(struct thread *, struct getfh_args *);
+void	sys_sys_exit(struct thread *, struct sys_exit_args *);
+int	sys_fork(struct thread *, struct fork_args *);
+int	sys_read(struct thread *, struct read_args *);
+int	sys_write(struct thread *, struct write_args *);
+int	sys_open(struct thread *, struct open_args *);
+int	sys_close(struct thread *, struct close_args *);
+int	sys_wait4(struct thread *, struct wait_args *);
+int	sys_link(struct thread *, struct link_args *);
+int	sys_unlink(struct thread *, struct unlink_args *);
+int	sys_chdir(struct thread *, struct chdir_args *);
+int	sys_fchdir(struct thread *, struct fchdir_args *);
+int	sys_mknod(struct thread *, struct mknod_args *);
+int	sys_chmod(struct thread *, struct chmod_args *);
+int	sys_chown(struct thread *, struct chown_args *);
+int	sys_obreak(struct thread *, struct obreak_args *);
+int	sys_getpid(struct thread *, struct getpid_args *);
+int	sys_mount(struct thread *, struct mount_args *);
+int	sys_unmount(struct thread *, struct unmount_args *);
+int	sys_setuid(struct thread *, struct setuid_args *);
+int	sys_getuid(struct thread *, struct getuid_args *);
+int	sys_geteuid(struct thread *, struct geteuid_args *);
+int	sys_ptrace(struct thread *, struct ptrace_args *);
+int	sys_recvmsg(struct thread *, struct recvmsg_args *);
+int	sys_sendmsg(struct thread *, struct sendmsg_args *);
+int	sys_recvfrom(struct thread *, struct recvfrom_args *);
+int	sys_accept(struct thread *, struct accept_args *);
+int	sys_getpeername(struct thread *, struct getpeername_args *);
+int	sys_getsockname(struct thread *, struct getsockname_args *);
+int	sys_access(struct thread *, struct access_args *);
+int	sys_chflags(struct thread *, struct chflags_args *);
+int	sys_fchflags(struct thread *, struct fchflags_args *);
+int	sys_sync(struct thread *, struct sync_args *);
+int	sys_kill(struct thread *, struct kill_args *);
+int	sys_getppid(struct thread *, struct getppid_args *);
+int	sys_dup(struct thread *, struct dup_args *);
+int	sys_pipe(struct thread *, struct pipe_args *);
+int	sys_getegid(struct thread *, struct getegid_args *);
+int	sys_profil(struct thread *, struct profil_args *);
+int	sys_ktrace(struct thread *, struct ktrace_args *);
+int	sys_getgid(struct thread *, struct getgid_args *);
+int	sys_getlogin(struct thread *, struct getlogin_args *);
+int	sys_setlogin(struct thread *, struct setlogin_args *);
+int	sys_acct(struct thread *, struct acct_args *);
+int	sys_sigaltstack(struct thread *, struct sigaltstack_args *);
+int	sys_ioctl(struct thread *, struct ioctl_args *);
+int	sys_reboot(struct thread *, struct reboot_args *);
+int	sys_revoke(struct thread *, struct revoke_args *);
+int	sys_symlink(struct thread *, struct symlink_args *);
+int	sys_readlink(struct thread *, struct readlink_args *);
+int	sys_execve(struct thread *, struct execve_args *);
+int	sys_umask(struct thread *, struct umask_args *);
+int	sys_chroot(struct thread *, struct chroot_args *);
+int	sys_msync(struct thread *, struct msync_args *);
+int	sys_vfork(struct thread *, struct vfork_args *);
+int	sys_sbrk(struct thread *, struct sbrk_args *);
+int	sys_sstk(struct thread *, struct sstk_args *);
+int	sys_ovadvise(struct thread *, struct ovadvise_args *);
+int	sys_munmap(struct thread *, struct munmap_args *);
+int	sys_mprotect(struct thread *, struct mprotect_args *);
+int	sys_madvise(struct thread *, struct madvise_args *);
+int	sys_mincore(struct thread *, struct mincore_args *);
+int	sys_getgroups(struct thread *, struct getgroups_args *);
+int	sys_setgroups(struct thread *, struct setgroups_args *);
+int	sys_getpgrp(struct thread *, struct getpgrp_args *);
+int	sys_setpgid(struct thread *, struct setpgid_args *);
+int	sys_setitimer(struct thread *, struct setitimer_args *);
+int	sys_swapon(struct thread *, struct swapon_args *);
+int	sys_getitimer(struct thread *, struct getitimer_args *);
+int	sys_getdtablesize(struct thread *, struct getdtablesize_args *);
+int	sys_dup2(struct thread *, struct dup2_args *);
+int	sys_fcntl(struct thread *, struct fcntl_args *);
+int	sys_select(struct thread *, struct select_args *);
+int	sys_fsync(struct thread *, struct fsync_args *);
+int	sys_setpriority(struct thread *, struct setpriority_args *);
+int	sys_socket(struct thread *, struct socket_args *);
+int	sys_connect(struct thread *, struct connect_args *);
+int	sys_getpriority(struct thread *, struct getpriority_args *);
+int	sys_bind(struct thread *, struct bind_args *);
+int	sys_setsockopt(struct thread *, struct setsockopt_args *);
+int	sys_listen(struct thread *, struct listen_args *);
+int	sys_gettimeofday(struct thread *, struct gettimeofday_args *);
+int	sys_getrusage(struct thread *, struct getrusage_args *);
+int	sys_getsockopt(struct thread *, struct getsockopt_args *);
+int	sys_readv(struct thread *, struct readv_args *);
+int	sys_writev(struct thread *, struct writev_args *);
+int	sys_settimeofday(struct thread *, struct settimeofday_args *);
+int	sys_fchown(struct thread *, struct fchown_args *);
+int	sys_fchmod(struct thread *, struct fchmod_args *);
+int	sys_setreuid(struct thread *, struct setreuid_args *);
+int	sys_setregid(struct thread *, struct setregid_args *);
+int	sys_rename(struct thread *, struct rename_args *);
+int	sys_flock(struct thread *, struct flock_args *);
+int	sys_mkfifo(struct thread *, struct mkfifo_args *);
+int	sys_sendto(struct thread *, struct sendto_args *);
+int	sys_shutdown(struct thread *, struct shutdown_args *);
+int	sys_socketpair(struct thread *, struct socketpair_args *);
+int	sys_mkdir(struct thread *, struct mkdir_args *);
+int	sys_rmdir(struct thread *, struct rmdir_args *);
+int	sys_utimes(struct thread *, struct utimes_args *);
+int	sys_adjtime(struct thread *, struct adjtime_args *);
+int	sys_setsid(struct thread *, struct setsid_args *);
+int	sys_quotactl(struct thread *, struct quotactl_args *);
+int	sys_nlm_syscall(struct thread *, struct nlm_syscall_args *);
+int	sys_nfssvc(struct thread *, struct nfssvc_args *);
+int	sys_lgetfh(struct thread *, struct lgetfh_args *);
+int	sys_getfh(struct thread *, struct getfh_args *);
 int	sysarch(struct thread *, struct sysarch_args *);
-int	rtprio(struct thread *, struct rtprio_args *);
-int	semsys(struct thread *, struct semsys_args *);
-int	msgsys(struct thread *, struct msgsys_args *);
-int	shmsys(struct thread *, struct shmsys_args *);
+int	sys_rtprio(struct thread *, struct rtprio_args *);
+int	sys_semsys(struct thread *, struct semsys_args *);
+int	sys_msgsys(struct thread *, struct msgsys_args *);
+int	sys_shmsys(struct thread *, struct shmsys_args *);
 int	freebsd6_pread(struct thread *, struct freebsd6_pread_args *);
 int	freebsd6_pwrite(struct thread *, struct freebsd6_pwrite_args *);
-int	setfib(struct thread *, struct setfib_args *);
-int	ntp_adjtime(struct thread *, struct ntp_adjtime_args *);
-int	setgid(struct thread *, struct setgid_args *);
-int	setegid(struct thread *, struct setegid_args *);
-int	seteuid(struct thread *, struct seteuid_args *);
-int	stat(struct thread *, struct stat_args *);
-int	fstat(struct thread *, struct fstat_args *);
-int	lstat(struct thread *, struct lstat_args *);
-int	pathconf(struct thread *, struct pathconf_args *);
-int	fpathconf(struct thread *, struct fpathconf_args *);
-int	getrlimit(struct thread *, struct __getrlimit_args *);
-int	setrlimit(struct thread *, struct __setrlimit_args *);
-int	getdirentries(struct thread *, struct getdirentries_args *);
+int	sys_setfib(struct thread *, struct setfib_args *);
+int	sys_ntp_adjtime(struct thread *, struct ntp_adjtime_args *);
+int	sys_setgid(struct thread *, struct setgid_args *);
+int	sys_setegid(struct thread *, struct setegid_args *);
+int	sys_seteuid(struct thread *, struct seteuid_args *);
+int	sys_stat(struct thread *, struct stat_args *);
+int	sys_fstat(struct thread *, struct fstat_args *);
+int	sys_lstat(struct thread *, struct lstat_args *);
+int	sys_pathconf(struct thread *, struct pathconf_args *);
+int	sys_fpathconf(struct thread *, struct fpathconf_args *);
+int	sys_getrlimit(struct thread *, struct __getrlimit_args *);
+int	sys_setrlimit(struct thread *, struct __setrlimit_args *);
+int	sys_getdirentries(struct thread *, struct getdirentries_args *);
 int	freebsd6_mmap(struct thread *, struct freebsd6_mmap_args *);
 int	freebsd6_lseek(struct thread *, struct freebsd6_lseek_args *);
 int	freebsd6_truncate(struct thread *, struct freebsd6_truncate_args *);
 int	freebsd6_ftruncate(struct thread *, struct freebsd6_ftruncate_args *);
-int	__sysctl(struct thread *, struct sysctl_args *);
-int	mlock(struct thread *, struct mlock_args *);
-int	munlock(struct thread *, struct munlock_args *);
-int	undelete(struct thread *, struct undelete_args *);
-int	futimes(struct thread *, struct futimes_args *);
-int	getpgid(struct thread *, struct getpgid_args *);
-int	poll(struct thread *, struct poll_args *);
-int	semget(struct thread *, struct semget_args *);
-int	semop(struct thread *, struct semop_args *);
-int	msgget(struct thread *, struct msgget_args *);
-int	msgsnd(struct thread *, struct msgsnd_args *);
-int	msgrcv(struct thread *, struct msgrcv_args *);
-int	shmat(struct thread *, struct shmat_args *);
-int	shmdt(struct thread *, struct shmdt_args *);
-int	shmget(struct thread *, struct shmget_args *);
-int	clock_gettime(struct thread *, struct clock_gettime_args *);
-int	clock_settime(struct thread *, struct clock_settime_args *);
-int	clock_getres(struct thread *, struct clock_getres_args *);
-int	ktimer_create(struct thread *, struct ktimer_create_args *);
-int	ktimer_delete(struct thread *, struct ktimer_delete_args *);
-int	ktimer_settime(struct thread *, struct ktimer_settime_args *);
-int	ktimer_gettime(struct thread *, struct ktimer_gettime_args *);
-int	ktimer_getoverrun(struct thread *, struct ktimer_getoverrun_args *);
-int	nanosleep(struct thread *, struct nanosleep_args *);
-int	ntp_gettime(struct thread *, struct ntp_gettime_args *);
-int	minherit(struct thread *, struct minherit_args *);
-int	rfork(struct thread *, struct rfork_args *);
-int	openbsd_poll(struct thread *, struct openbsd_poll_args *);
-int	issetugid(struct thread *, struct issetugid_args *);
-int	lchown(struct thread *, struct lchown_args *);
-int	aio_read(struct thread *, struct aio_read_args *);
-int	aio_write(struct thread *, struct aio_write_args *);
-int	lio_listio(struct thread *, struct lio_listio_args *);
-int	getdents(struct thread *, struct getdents_args *);
-int	lchmod(struct thread *, struct lchmod_args *);
-int	lutimes(struct thread *, struct lutimes_args *);
-int	nstat(struct thread *, struct nstat_args *);
-int	nfstat(struct thread *, struct nfstat_args *);
-int	nlstat(struct thread *, struct nlstat_args *);
-int	preadv(struct thread *, struct preadv_args *);
-int	pwritev(struct thread *, struct pwritev_args *);
-int	fhopen(struct thread *, struct fhopen_args *);
-int	fhstat(struct thread *, struct fhstat_args *);
-int	modnext(struct thread *, struct modnext_args *);
-int	modstat(struct thread *, struct modstat_args *);
-int	modfnext(struct thread *, struct modfnext_args *);
-int	modfind(struct thread *, struct modfind_args *);
-int	kldload(struct thread *, struct kldload_args *);
-int	kldunload(struct thread *, struct kldunload_args *);
-int	kldfind(struct thread *, struct kldfind_args *);
-int	kldnext(struct thread *, struct kldnext_args *);
-int	kldstat(struct thread *, struct kldstat_args *);
-int	kldfirstmod(struct thread *, struct kldfirstmod_args *);
-int	getsid(struct thread *, struct getsid_args *);
-int	setresuid(struct thread *, struct setresuid_args *);
-int	setresgid(struct thread *, struct setresgid_args *);
-int	aio_return(struct thread *, struct aio_return_args *);
-int	aio_suspend(struct thread *, struct aio_suspend_args *);
-int	aio_cancel(struct thread *, struct aio_cancel_args *);
-int	aio_error(struct thread *, struct aio_error_args *);
-int	oaio_read(struct thread *, struct oaio_read_args *);
-int	oaio_write(struct thread *, struct oaio_write_args *);
-int	olio_listio(struct thread *, struct olio_listio_args *);
-int	yield(struct thread *, struct yield_args *);
-int	mlockall(struct thread *, struct mlockall_args *);
-int	munlockall(struct thread *, struct munlockall_args *);
-int	__getcwd(struct thread *, struct __getcwd_args *);
-int	sched_setparam(struct thread *, struct sched_setparam_args *);
-int	sched_getparam(struct thread *, struct sched_getparam_args *);
-int	sched_setscheduler(struct thread *, struct sched_setscheduler_args *);
-int	sched_getscheduler(struct thread *, struct sched_getscheduler_args *);
-int	sched_yield(struct thread *, struct sched_yield_args *);
-int	sched_get_priority_max(struct thread *, struct sched_get_priority_max_args *);
-int	sched_get_priority_min(struct thread *, struct sched_get_priority_min_args *);
-int	sched_rr_get_interval(struct thread *, struct sched_rr_get_interval_args *);
-int	utrace(struct thread *, struct utrace_args *);
-int	kldsym(struct thread *, struct kldsym_args *);
-int	jail(struct thread *, struct jail_args *);
-int	nnpfs_syscall(struct thread *, struct nnpfs_syscall_args *);
-int	sigprocmask(struct thread *, struct sigprocmask_args *);
-int	sigsuspend(struct thread *, struct sigsuspend_args *);
-int	sigpending(struct thread *, struct sigpending_args *);
-int	sigtimedwait(struct thread *, struct sigtimedwait_args *);
-int	sigwaitinfo(struct thread *, struct sigwaitinfo_args *);
-int	__acl_get_file(struct thread *, struct __acl_get_file_args *);
-int	__acl_set_file(struct thread *, struct __acl_set_file_args *);
-int	__acl_get_fd(struct thread *, struct __acl_get_fd_args *);
-int	__acl_set_fd(struct thread *, struct __acl_set_fd_args *);
-int	__acl_delete_file(struct thread *, struct __acl_delete_file_args *);
-int	__acl_delete_fd(struct thread *, struct __acl_delete_fd_args *);
-int	__acl_aclcheck_file(struct thread *, struct __acl_aclcheck_file_args *);
-int	__acl_aclcheck_fd(struct thread *, struct __acl_aclcheck_fd_args *);
-int	extattrctl(struct thread *, struct extattrctl_args *);
-int	extattr_set_file(struct thread *, struct extattr_set_file_args *);
-int	extattr_get_file(struct thread *, struct extattr_get_file_args *);
-int	extattr_delete_file(struct thread *, struct extattr_delete_file_args *);
-int	aio_waitcomplete(struct thread *, struct aio_waitcomplete_args *);
-int	getresuid(struct thread *, struct getresuid_args *);
-int	getresgid(struct thread *, struct getresgid_args *);
-int	kqueue(struct thread *, struct kqueue_args *);
-int	kevent(struct thread *, struct kevent_args *);
-int	extattr_set_fd(struct thread *, struct extattr_set_fd_args *);
-int	extattr_get_fd(struct thread *, struct extattr_get_fd_args *);
-int	extattr_delete_fd(struct thread *, struct extattr_delete_fd_args *);
-int	__setugid(struct thread *, struct __setugid_args *);
-int	eaccess(struct thread *, struct eaccess_args *);
-int	afs3_syscall(struct thread *, struct afs3_syscall_args *);
-int	nmount(struct thread *, struct nmount_args *);
-int	__mac_get_proc(struct thread *, struct __mac_get_proc_args *);
-int	__mac_set_proc(struct thread *, struct __mac_set_proc_args *);
-int	__mac_get_fd(struct thread *, struct __mac_get_fd_args *);
-int	__mac_get_file(struct thread *, struct __mac_get_file_args *);
-int	__mac_set_fd(struct thread *, struct __mac_set_fd_args *);
-int	__mac_set_file(struct thread *, struct __mac_set_file_args *);
-int	kenv(struct thread *, struct kenv_args *);
-int	lchflags(struct thread *, struct lchflags_args *);
-int	uuidgen(struct thread *, struct uuidgen_args *);
-int	sendfile(struct thread *, struct sendfile_args *);
-int	mac_syscall(struct thread *, struct mac_syscall_args *);
-int	getfsstat(struct thread *, struct getfsstat_args *);
-int	statfs(struct thread *, struct statfs_args *);
-int	fstatfs(struct thread *, struct fstatfs_args *);
-int	fhstatfs(struct thread *, struct fhstatfs_args *);
-int	ksem_close(struct thread *, struct ksem_close_args *);
-int	ksem_post(struct thread *, struct ksem_post_args *);
-int	ksem_wait(struct thread *, struct ksem_wait_args *);
-int	ksem_trywait(struct thread *, struct ksem_trywait_args *);
-int	ksem_init(struct thread *, struct ksem_init_args *);
-int	ksem_open(struct thread *, struct ksem_open_args *);
-int	ksem_unlink(struct thread *, struct ksem_unlink_args *);
-int	ksem_getvalue(struct thread *, struct ksem_getvalue_args *);
-int	ksem_destroy(struct thread *, struct ksem_destroy_args *);
-int	__mac_get_pid(struct thread *, struct __mac_get_pid_args *);
-int	__mac_get_link(struct thread *, struct __mac_get_link_args *);
-int	__mac_set_link(struct thread *, struct __mac_set_link_args *);
-int	extattr_set_link(struct thread *, struct extattr_set_link_args *);
-int	extattr_get_link(struct thread *, struct extattr_get_link_args *);
-int	extattr_delete_link(struct thread *, struct extattr_delete_link_args *);
-int	__mac_execve(struct thread *, struct __mac_execve_args *);
-int	sigaction(struct thread *, struct sigaction_args *);
-int	sigreturn(struct thread *, struct sigreturn_args *);
-int	getcontext(struct thread *, struct getcontext_args *);
-int	setcontext(struct thread *, struct setcontext_args *);
-int	swapcontext(struct thread *, struct swapcontext_args *);
-int	swapoff(struct thread *, struct swapoff_args *);
-int	__acl_get_link(struct thread *, struct __acl_get_link_args *);
-int	__acl_set_link(struct thread *, struct __acl_set_link_args *);
-int	__acl_delete_link(struct thread *, struct __acl_delete_link_args *);
-int	__acl_aclcheck_link(struct thread *, struct __acl_aclcheck_link_args *);
-int	sigwait(struct thread *, struct sigwait_args *);
-int	thr_create(struct thread *, struct thr_create_args *);
-int	thr_exit(struct thread *, struct thr_exit_args *);
-int	thr_self(struct thread *, struct thr_self_args *);
-int	thr_kill(struct thread *, struct thr_kill_args *);
-int	_umtx_lock(struct thread *, struct _umtx_lock_args *);
-int	_umtx_unlock(struct thread *, struct _umtx_unlock_args *);
-int	jail_attach(struct thread *, struct jail_attach_args *);
-int	extattr_list_fd(struct thread *, struct extattr_list_fd_args *);
-int	extattr_list_file(struct thread *, struct extattr_list_file_args *);
-int	extattr_list_link(struct thread *, struct extattr_list_link_args *);
-int	ksem_timedwait(struct thread *, struct ksem_timedwait_args *);
-int	thr_suspend(struct thread *, struct thr_suspend_args *);
-int	thr_wake(struct thread *, struct thr_wake_args *);
-int	kldunloadf(struct thread *, struct kldunloadf_args *);
-int	audit(struct thread *, struct audit_args *);
-int	auditon(struct thread *, struct auditon_args *);
-int	getauid(struct thread *, struct getauid_args *);
-int	setauid(struct thread *, struct setauid_args *);
-int	getaudit(struct thread *, struct getaudit_args *);
-int	setaudit(struct thread *, struct setaudit_args *);
-int	getaudit_addr(struct thread *, struct getaudit_addr_args *);
-int	setaudit_addr(struct thread *, struct setaudit_addr_args *);
-int	auditctl(struct thread *, struct auditctl_args *);
-int	_umtx_op(struct thread *, struct _umtx_op_args *);
-int	thr_new(struct thread *, struct thr_new_args *);
-int	sigqueue(struct thread *, struct sigqueue_args *);
-int	kmq_open(struct thread *, struct kmq_open_args *);
-int	kmq_setattr(struct thread *, struct kmq_setattr_args *);
-int	kmq_timedreceive(struct thread *, struct kmq_timedreceive_args *);
-int	kmq_timedsend(struct thread *, struct kmq_timedsend_args *);
-int	kmq_notify(struct thread *, struct kmq_notify_args *);
-int	kmq_unlink(struct thread *, struct kmq_unlink_args *);
-int	abort2(struct thread *, struct abort2_args *);
-int	thr_set_name(struct thread *, struct thr_set_name_args *);
-int	aio_fsync(struct thread *, struct aio_fsync_args *);
-int	rtprio_thread(struct thread *, struct rtprio_thread_args *);
-int	sctp_peeloff(struct thread *, struct sctp_peeloff_args *);
-int	sctp_generic_sendmsg(struct thread *, struct sctp_generic_sendmsg_args *);
-int	sctp_generic_sendmsg_iov(struct thread *, struct sctp_generic_sendmsg_iov_args *);
-int	sctp_generic_recvmsg(struct thread *, struct sctp_generic_recvmsg_args *);
-int	pread(struct thread *, struct pread_args *);
-int	pwrite(struct thread *, struct pwrite_args *);
-int	mmap(struct thread *, struct mmap_args *);
-int	lseek(struct thread *, struct lseek_args *);
-int	truncate(struct thread *, struct truncate_args *);
-int	ftruncate(struct thread *, struct ftruncate_args *);
-int	thr_kill2(struct thread *, struct thr_kill2_args *);
-int	shm_open(struct thread *, struct shm_open_args *);
-int	shm_unlink(struct thread *, struct shm_unlink_args *);
-int	cpuset(struct thread *, struct cpuset_args *);
-int	cpuset_setid(struct thread *, struct cpuset_setid_args *);
-int	cpuset_getid(struct thread *, struct cpuset_getid_args *);
-int	cpuset_getaffinity(struct thread *, struct cpuset_getaffinity_args *);
-int	cpuset_setaffinity(struct thread *, struct cpuset_setaffinity_args *);
-int	faccessat(struct thread *, struct faccessat_args *);
-int	fchmodat(struct thread *, struct fchmodat_args *);
-int	fchownat(struct thread *, struct fchownat_args *);
-int	fexecve(struct thread *, struct fexecve_args *);
-int	fstatat(struct thread *, struct fstatat_args *);
-int	futimesat(struct thread *, struct futimesat_args *);
-int	linkat(struct thread *, struct linkat_args *);
-int	mkdirat(struct thread *, struct mkdirat_args *);
-int	mkfifoat(struct thread *, struct mkfifoat_args *);
-int	mknodat(struct thread *, struct mknodat_args *);
-int	openat(struct thread *, struct openat_args *);
-int	readlinkat(struct thread *, struct readlinkat_args *);
-int	renameat(struct thread *, struct renameat_args *);
-int	symlinkat(struct thread *, struct symlinkat_args *);
-int	unlinkat(struct thread *, struct unlinkat_args *);
-int	posix_openpt(struct thread *, struct posix_openpt_args *);
-int	gssd_syscall(struct thread *, struct gssd_syscall_args *);
-int	jail_get(struct thread *, struct jail_get_args *);
-int	jail_set(struct thread *, struct jail_set_args *);
-int	jail_remove(struct thread *, struct jail_remove_args *);
-int	closefrom(struct thread *, struct closefrom_args *);
-int	__semctl(struct thread *, struct __semctl_args *);
-int	msgctl(struct thread *, struct msgctl_args *);
-int	shmctl(struct thread *, struct shmctl_args *);
-int	lpathconf(struct thread *, struct lpathconf_args *);
-int	cap_new(struct thread *, struct cap_new_args *);
-int	cap_getrights(struct thread *, struct cap_getrights_args *);
-int	cap_enter(struct thread *, struct cap_enter_args *);
-int	cap_getmode(struct thread *, struct cap_getmode_args *);
-int	pdfork(struct thread *, struct pdfork_args *);
-int	pdkill(struct thread *, struct pdkill_args *);
-int	pdgetpid(struct thread *, struct pdgetpid_args *);
-int	pselect(struct thread *, struct pselect_args *);
-int	getloginclass(struct thread *, struct getloginclass_args *);
-int	setloginclass(struct thread *, struct setloginclass_args *);
-int	rctl_get_racct(struct thread *, struct rctl_get_racct_args *);
-int	rctl_get_rules(struct thread *, struct rctl_get_rules_args *);
-int	rctl_get_limits(struct thread *, struct rctl_get_limits_args *);
-int	rctl_add_rule(struct thread *, struct rctl_add_rule_args *);
-int	rctl_remove_rule(struct thread *, struct rctl_remove_rule_args *);
-int	posix_fallocate(struct thread *, struct posix_fallocate_args *);
+int	sys___sysctl(struct thread *, struct sysctl_args *);
+int	sys_mlock(struct thread *, struct mlock_args *);
+int	sys_munlock(struct thread *, struct munlock_args *);
+int	sys_undelete(struct thread *, struct undelete_args *);
+int	sys_futimes(struct thread *, struct futimes_args *);
+int	sys_getpgid(struct thread *, struct getpgid_args *);
+int	sys_poll(struct thread *, struct poll_args *);
+int	sys_semget(struct thread *, struct semget_args *);
+int	sys_semop(struct thread *, struct semop_args *);
+int	sys_msgget(struct thread *, struct msgget_args *);
+int	sys_msgsnd(struct thread *, struct msgsnd_args *);
+int	sys_msgrcv(struct thread *, struct msgrcv_args *);
+int	sys_shmat(struct thread *, struct shmat_args *);
+int	sys_shmdt(struct thread *, struct shmdt_args *);
+int	sys_shmget(struct thread *, struct shmget_args *);
+int	sys_clock_gettime(struct thread *, struct clock_gettime_args *);
+int	sys_clock_settime(struct thread *, struct clock_settime_args *);
+int	sys_clock_getres(struct thread *, struct clock_getres_args *);
+int	sys_ktimer_create(struct thread *, struct ktimer_create_args *);
+int	sys_ktimer_delete(struct thread *, struct ktimer_delete_args *);
+int	sys_ktimer_settime(struct thread *, struct ktimer_settime_args *);
+int	sys_ktimer_gettime(struct thread *, struct ktimer_gettime_args *);
+int	sys_ktimer_getoverrun(struct thread *, struct ktimer_getoverrun_args *);
+int	sys_nanosleep(struct thread *, struct nanosleep_args *);
+int	sys_ntp_gettime(struct thread *, struct ntp_gettime_args *);
+int	sys_minherit(struct thread *, struct minherit_args *);
+int	sys_rfork(struct thread *, struct rfork_args *);
+int	sys_openbsd_poll(struct thread *, struct openbsd_poll_args *);
+int	sys_issetugid(struct thread *, struct issetugid_args *);
+int	sys_lchown(struct thread *, struct lchown_args *);
+int	sys_aio_read(struct thread *, struct aio_read_args *);
+int	sys_aio_write(struct thread *, struct aio_write_args *);
+int	sys_lio_listio(struct thread *, struct lio_listio_args *);
+int	sys_getdents(struct thread *, struct getdents_args *);
+int	sys_lchmod(struct thread *, struct lchmod_args *);
+int	sys_lutimes(struct thread *, struct lutimes_args *);
+int	sys_nstat(struct thread *, struct nstat_args *);
+int	sys_nfstat(struct thread *, struct nfstat_args *);
+int	sys_nlstat(struct thread *, struct nlstat_args *);
+int	sys_preadv(struct thread *, struct preadv_args *);
+int	sys_pwritev(struct thread *, struct pwritev_args *);
+int	sys_fhopen(struct thread *, struct fhopen_args *);
+int	sys_fhstat(struct thread *, struct fhstat_args *);
+int	sys_modnext(struct thread *, struct modnext_args *);
+int	sys_modstat(struct thread *, struct modstat_args *);
+int	sys_modfnext(struct thread *, struct modfnext_args *);
+int	sys_modfind(struct thread *, struct modfind_args *);
+int	sys_kldload(struct thread *, struct kldload_args *);
+int	sys_kldunload(struct thread *, struct kldunload_args *);
+int	sys_kldfind(struct thread *, struct kldfind_args *);
+int	sys_kldnext(struct thread *, struct kldnext_args *);
+int	sys_kldstat(struct thread *, struct kldstat_args *);
+int	sys_kldfirstmod(struct thread *, struct kldfirstmod_args *);
+int	sys_getsid(struct thread *, struct getsid_args *);
+int	sys_setresuid(struct thread *, struct setresuid_args *);
+int	sys_setresgid(struct thread *, struct setresgid_args *);
+int	sys_aio_return(struct thread *, struct aio_return_args *);
+int	sys_aio_suspend(struct thread *, struct aio_suspend_args *);
+int	sys_aio_cancel(struct thread *, struct aio_cancel_args *);
+int	sys_aio_error(struct thread *, struct aio_error_args *);
+int	sys_oaio_read(struct thread *, struct oaio_read_args *);
+int	sys_oaio_write(struct thread *, struct oaio_write_args *);
+int	sys_olio_listio(struct thread *, struct olio_listio_args *);
+int	sys_yield(struct thread *, struct yield_args *);
+int	sys_mlockall(struct thread *, struct mlockall_args *);
+int	sys_munlockall(struct thread *, struct munlockall_args *);
+int	sys___getcwd(struct thread *, struct __getcwd_args *);
+int	sys_sched_setparam(struct thread *, struct sched_setparam_args *);
+int	sys_sched_getparam(struct thread *, struct sched_getparam_args *);
+int	sys_sched_setscheduler(struct thread *, struct sched_setscheduler_args *);
+int	sys_sched_getscheduler(struct thread *, struct sched_getscheduler_args *);
+int	sys_sched_yield(struct thread *, struct sched_yield_args *);
+int	sys_sched_get_priority_max(struct thread *, struct sched_get_priority_max_args *);
+int	sys_sched_get_priority_min(struct thread *, struct sched_get_priority_min_args *);
+int	sys_sched_rr_get_interval(struct thread *, struct sched_rr_get_interval_args *);
+int	sys_utrace(struct thread *, struct utrace_args *);
+int	sys_kldsym(struct thread *, struct kldsym_args *);
+int	sys_jail(struct thread *, struct jail_args *);
+int	sys_nnpfs_syscall(struct thread *, struct nnpfs_syscall_args *);
+int	sys_sigprocmask(struct thread *, struct sigprocmask_args *);
+int	sys_sigsuspend(struct thread *, struct sigsuspend_args *);
+int	sys_sigpending(struct thread *, struct sigpending_args *);
+int	sys_sigtimedwait(struct thread *, struct sigtimedwait_args *);
+int	sys_sigwaitinfo(struct thread *, struct sigwaitinfo_args *);
+int	sys___acl_get_file(struct thread *, struct __acl_get_file_args *);
+int	sys___acl_set_file(struct thread *, struct __acl_set_file_args *);
+int	sys___acl_get_fd(struct thread *, struct __acl_get_fd_args *);
+int	sys___acl_set_fd(struct thread *, struct __acl_set_fd_args *);
+int	sys___acl_delete_file(struct thread *, struct __acl_delete_file_args *);
+int	sys___acl_delete_fd(struct thread *, struct __acl_delete_fd_args *);
+int	sys___acl_aclcheck_file(struct thread *, struct __acl_aclcheck_file_args *);
+int	sys___acl_aclcheck_fd(struct thread *, struct __acl_aclcheck_fd_args *);
+int	sys_extattrctl(struct thread *, struct extattrctl_args *);
+int	sys_extattr_set_file(struct thread *, struct extattr_set_file_args *);
+int	sys_extattr_get_file(struct thread *, struct extattr_get_file_args *);
+int	sys_extattr_delete_file(struct thread *, struct extattr_delete_file_args *);
+int	sys_aio_waitcomplete(struct thread *, struct aio_waitcomplete_args *);
+int	sys_getresuid(struct thread *, struct getresuid_args *);
+int	sys_getresgid(struct thread *, struct getresgid_args *);
+int	sys_kqueue(struct thread *, struct kqueue_args *);
+int	sys_kevent(struct thread *, struct kevent_args *);
+int	sys_extattr_set_fd(struct thread *, struct extattr_set_fd_args *);
+int	sys_extattr_get_fd(struct thread *, struct extattr_get_fd_args *);
+int	sys_extattr_delete_fd(struct thread *, struct extattr_delete_fd_args *);
+int	sys___setugid(struct thread *, struct __setugid_args *);
+int	sys_eaccess(struct thread *, struct eaccess_args *);
+int	sys_afs3_syscall(struct thread *, struct afs3_syscall_args *);
+int	sys_nmount(struct thread *, struct nmount_args *);
+int	sys___mac_get_proc(struct thread *, struct __mac_get_proc_args *);
+int	sys___mac_set_proc(struct thread *, struct __mac_set_proc_args *);
+int	sys___mac_get_fd(struct thread *, struct __mac_get_fd_args *);
+int	sys___mac_get_file(struct thread *, struct __mac_get_file_args *);
+int	sys___mac_set_fd(struct thread *, struct __mac_set_fd_args *);
+int	sys___mac_set_file(struct thread *, struct __mac_set_file_args *);
+int	sys_kenv(struct thread *, struct kenv_args *);
+int	sys_lchflags(struct thread *, struct lchflags_args *);
+int	sys_uuidgen(struct thread *, struct uuidgen_args *);
+int	sys_sendfile(struct thread *, struct sendfile_args *);
+int	sys_mac_syscall(struct thread *, struct mac_syscall_args *);
+int	sys_getfsstat(struct thread *, struct getfsstat_args *);
+int	sys_statfs(struct thread *, struct statfs_args *);
+int	sys_fstatfs(struct thread *, struct fstatfs_args *);
+int	sys_fhstatfs(struct thread *, struct fhstatfs_args *);
+int	sys_ksem_close(struct thread *, struct ksem_close_args *);
+int	sys_ksem_post(struct thread *, struct ksem_post_args *);
+int	sys_ksem_wait(struct thread *, struct ksem_wait_args *);
+int	sys_ksem_trywait(struct thread *, struct ksem_trywait_args *);
+int	sys_ksem_init(struct thread *, struct ksem_init_args *);
+int	sys_ksem_open(struct thread *, struct ksem_open_args *);
+int	sys_ksem_unlink(struct thread *, struct ksem_unlink_args *);
+int	sys_ksem_getvalue(struct thread *, struct ksem_getvalue_args *);
+int	sys_ksem_destroy(struct thread *, struct ksem_destroy_args *);
+int	sys___mac_get_pid(struct thread *, struct __mac_get_pid_args *);
+int	sys___mac_get_link(struct thread *, struct __mac_get_link_args *);
+int	sys___mac_set_link(struct thread *, struct __mac_set_link_args *);
+int	sys_extattr_set_link(struct thread *, struct extattr_set_link_args *);
+int	sys_extattr_get_link(struct thread *, struct extattr_get_link_args *);
+int	sys_extattr_delete_link(struct thread *, struct extattr_delete_link_args *);
+int	sys___mac_execve(struct thread *, struct __mac_execve_args *);
+int	sys_sigaction(struct thread *, struct sigaction_args *);
+int	sys_sigreturn(struct thread *, struct sigreturn_args *);
+int	sys_getcontext(struct thread *, struct getcontext_args *);
+int	sys_setcontext(struct thread *, struct setcontext_args *);
+int	sys_swapcontext(struct thread *, struct swapcontext_args *);
+int	sys_swapoff(struct thread *, struct swapoff_args *);
+int	sys___acl_get_link(struct thread *, struct __acl_get_link_args *);
+int	sys___acl_set_link(struct thread *, struct __acl_set_link_args *);
+int	sys___acl_delete_link(struct thread *, struct __acl_delete_link_args *);
+int	sys___acl_aclcheck_link(struct thread *, struct __acl_aclcheck_link_args *);
+int	sys_sigwait(struct thread *, struct sigwait_args *);
+int	sys_thr_create(struct thread *, struct thr_create_args *);
+int	sys_thr_exit(struct thread *, struct thr_exit_args *);
+int	sys_thr_self(struct thread *, struct thr_self_args *);
+int	sys_thr_kill(struct thread *, struct thr_kill_args *);
+int	sys__umtx_lock(struct thread *, struct _umtx_lock_args *);
+int	sys__umtx_unlock(struct thread *, struct _umtx_unlock_args *);
+int	sys_jail_attach(struct thread *, struct jail_attach_args *);
+int	sys_extattr_list_fd(struct thread *, struct extattr_list_fd_args *);
+int	sys_extattr_list_file(struct thread *, struct extattr_list_file_args *);
+int	sys_extattr_list_link(struct thread *, struct extattr_list_link_args *);
+int	sys_ksem_timedwait(struct thread *, struct ksem_timedwait_args *);
+int	sys_thr_suspend(struct thread *, struct thr_suspend_args *);
+int	sys_thr_wake(struct thread *, struct thr_wake_args *);
+int	sys_kldunloadf(struct thread *, struct kldunloadf_args *);
+int	sys_audit(struct thread *, struct audit_args *);
+int	sys_auditon(struct thread *, struct auditon_args *);
+int	sys_getauid(struct thread *, struct getauid_args *);
+int	sys_setauid(struct thread *, struct setauid_args *);
+int	sys_getaudit(struct thread *, struct getaudit_args *);
+int	sys_setaudit(struct thread *, struct setaudit_args *);
+int	sys_getaudit_addr(struct thread *, struct getaudit_addr_args *);
+int	sys_setaudit_addr(struct thread *, struct setaudit_addr_args *);
+int	sys_auditctl(struct thread *, struct auditctl_args *);
+int	sys__umtx_op(struct thread *, struct _umtx_op_args *);
+int	sys_thr_new(struct thread *, struct thr_new_args *);
+int	sys_sigqueue(struct thread *, struct sigqueue_args *);
+int	sys_kmq_open(struct thread *, struct kmq_open_args *);
+int	sys_kmq_setattr(struct thread *, struct kmq_setattr_args *);
+int	sys_kmq_timedreceive(struct thread *, struct kmq_timedreceive_args *);
+int	sys_kmq_timedsend(struct thread *, struct kmq_timedsend_args *);
+int	sys_kmq_notify(struct thread *, struct kmq_notify_args *);
+int	sys_kmq_unlink(struct thread *, struct kmq_unlink_args *);
+int	sys_abort2(struct thread *, struct abort2_args *);
+int	sys_thr_set_name(struct thread *, struct thr_set_name_args *);
+int	sys_aio_fsync(struct thread *, struct aio_fsync_args *);
+int	sys_rtprio_thread(struct thread *, struct rtprio_thread_args *);
+int	sys_sctp_peeloff(struct thread *, struct sctp_peeloff_args *);
+int	sys_sctp_generic_sendmsg(struct thread *, struct sctp_generic_sendmsg_args *);
+int	sys_sctp_generic_sendmsg_iov(struct thread *, struct sctp_generic_sendmsg_iov_args *);
+int	sys_sctp_generic_recvmsg(struct thread *, struct sctp_generic_recvmsg_args *);
+int	sys_pread(struct thread *, struct pread_args *);
+int	sys_pwrite(struct thread *, struct pwrite_args *);
+int	sys_mmap(struct thread *, struct mmap_args *);
+int	sys_lseek(struct thread *, struct lseek_args *);
+int	sys_truncate(struct thread *, struct truncate_args *);
+int	sys_ftruncate(struct thread *, struct ftruncate_args *);
+int	sys_thr_kill2(struct thread *, struct thr_kill2_args *);
+int	sys_shm_open(struct thread *, struct shm_open_args *);
+int	sys_shm_unlink(struct thread *, struct shm_unlink_args *);
+int	sys_cpuset(struct thread *, struct cpuset_args *);
+int	sys_cpuset_setid(struct thread *, struct cpuset_setid_args *);
+int	sys_cpuset_getid(struct thread *, struct cpuset_getid_args *);
+int	sys_cpuset_getaffinity(struct thread *, struct cpuset_getaffinity_args *);
+int	sys_cpuset_setaffinity(struct thread *, struct cpuset_setaffinity_args *);
+int	sys_faccessat(struct thread *, struct faccessat_args *);
+int	sys_fchmodat(struct thread *, struct fchmodat_args *);
+int	sys_fchownat(struct thread *, struct fchownat_args *);
+int	sys_fexecve(struct thread *, struct fexecve_args *);
+int	sys_fstatat(struct thread *, struct fstatat_args *);
+int	sys_futimesat(struct thread *, struct futimesat_args *);
+int	sys_linkat(struct thread *, struct linkat_args *);
+int	sys_mkdirat(struct thread *, struct mkdirat_args *);
+int	sys_mkfifoat(struct thread *, struct mkfifoat_args *);
+int	sys_mknodat(struct thread *, struct mknodat_args *);
+int	sys_openat(struct thread *, struct openat_args *);
+int	sys_readlinkat(struct thread *, struct readlinkat_args *);
+int	sys_renameat(struct thread *, struct renameat_args *);
+int	sys_symlinkat(struct thread *, struct symlinkat_args *);
+int	sys_unlinkat(struct thread *, struct unlinkat_args *);
+int	sys_posix_openpt(struct thread *, struct posix_openpt_args *);
+int	sys_gssd_syscall(struct thread *, struct gssd_syscall_args *);
+int	sys_jail_get(struct thread *, struct jail_get_args *);
+int	sys_jail_set(struct thread *, struct jail_set_args *);
+int	sys_jail_remove(struct thread *, struct jail_remove_args *);
+int	sys_closefrom(struct thread *, struct closefrom_args *);
+int	sys___semctl(struct thread *, struct __semctl_args *);
+int	sys_msgctl(struct thread *, struct msgctl_args *);
+int	sys_shmctl(struct thread *, struct shmctl_args *);
+int	sys_lpathconf(struct thread *, struct lpathconf_args *);
+int	sys_cap_new(struct thread *, struct cap_new_args *);
+int	sys_cap_getrights(struct thread *, struct cap_getrights_args *);
+int	sys_cap_enter(struct thread *, struct cap_enter_args *);
+int	sys_cap_getmode(struct thread *, struct cap_getmode_args *);
+int	sys_pdfork(struct thread *, struct pdfork_args *);
+int	sys_pdkill(struct thread *, struct pdkill_args *);
+int	sys_pdgetpid(struct thread *, struct pdgetpid_args *);
+int	sys_pselect(struct thread *, struct pselect_args *);
+int	sys_getloginclass(struct thread *, struct getloginclass_args *);
+int	sys_setloginclass(struct thread *, struct setloginclass_args *);
+int	sys_rctl_get_racct(struct thread *, struct rctl_get_racct_args *);
+int	sys_rctl_get_rules(struct thread *, struct rctl_get_rules_args *);
+int	sys_rctl_get_limits(struct thread *, struct rctl_get_limits_args *);
+int	sys_rctl_add_rule(struct thread *, struct rctl_add_rule_args *);
+int	sys_rctl_remove_rule(struct thread *, struct rctl_remove_rule_args *);
+int	sys_posix_fallocate(struct thread *, struct posix_fallocate_args *);
 
 #ifdef COMPAT_43
 
diff --git a/sys/teken/demo/Makefile b/sys/teken/demo/Makefile
index ebf40ab30170..1ade690afe22 100644
--- a/sys/teken/demo/Makefile
+++ b/sys/teken/demo/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 
 PROG=	teken_demo
-LDADD=	-lncurses -lteken -lutil
+LDADD=	-lncursesw -lteken -lutil
 MAN=
 
 WARNS?=	6
diff --git a/sys/teken/libteken/Symbol.map b/sys/teken/libteken/Symbol.map
index 9a10aba7acfc..44f7ff875548 100644
--- a/sys/teken/libteken/Symbol.map
+++ b/sys/teken/libteken/Symbol.map
@@ -1,6 +1,6 @@
 /*
  * $FreeBSD$
- */	
+ */
 
 FBSD_1.2 {
 	teken_256to8;
diff --git a/sys/teken/stress/teken_stress.c b/sys/teken/stress/teken_stress.c
index 203c35b12892..a74cd8b36f13 100644
--- a/sys/teken/stress/teken_stress.c
+++ b/sys/teken/stress/teken_stress.c
@@ -99,24 +99,14 @@ int
 main(int argc __unused, char *argv[] __unused)
 {
 	teken_t t;
-	int rnd;
 	unsigned int i, iteration = 0;
 	unsigned char buf[2048];
 
-	rnd = open("/dev/urandom", O_RDONLY);
-	if (rnd < 0) {
-		perror("/dev/urandom");
-		exit(1);
-	}
 
 	teken_init(&t, &tf, NULL);
 
 	for (;;) {
-		if (read(rnd, buf, sizeof buf) != sizeof buf) {
-			perror("read");
-			exit(1);
-		}
-
+		arc4random_buf(buf, sizeof buf);
 		for (i = 0; i < sizeof buf; i++) {
 			if (buf[i] >= 0x80)
 				buf[i] =
diff --git a/sys/teken/teken_subr.h b/sys/teken/teken_subr.h
index f4c78f4ede35..64f4e5456bf2 100644
--- a/sys/teken/teken_subr.h
+++ b/sys/teken/teken_subr.h
@@ -325,7 +325,7 @@ teken_subr_cursor_position(teken_t *t, unsigned int row, unsigned int col)
 {
 
 	t->t_cursor.tp_row = t->t_originreg.ts_begin + row - 1;
-	if (row >= t->t_originreg.ts_end)
+	if (t->t_cursor.tp_row >= t->t_originreg.ts_end)
 		t->t_cursor.tp_row = t->t_originreg.ts_end - 1;
 
 	t->t_cursor.tp_col = col - 1;
@@ -595,20 +595,7 @@ static void
 teken_subr_horizontal_tab(teken_t *t)
 {
 
-	if (t->t_stateflags & TS_CONS25) {
-		teken_subr_cursor_forward_tabulation(t, 1);
-	} else {
-		teken_rect_t tr;
-
-		tr.tr_begin = t->t_cursor;
-		teken_subr_cursor_forward_tabulation(t, 1);
-		tr.tr_end.tp_row = tr.tr_begin.tp_row + 1;
-		tr.tr_end.tp_col = t->t_cursor.tp_col;
-
-		/* Blank region that we skipped. */
-		if (tr.tr_end.tp_col > tr.tr_begin.tp_col)
-			teken_funcs_fill(t, &tr, BLANK, &t->t_curattr);
-	}
+	teken_subr_cursor_forward_tabulation(t, 1);
 }
 
 static void
diff --git a/sys/ufs/ffs/ffs_snapshot.c b/sys/ufs/ffs/ffs_snapshot.c
index c8dd4c66b355..c8d89698e8de 100644
--- a/sys/ufs/ffs/ffs_snapshot.c
+++ b/sys/ufs/ffs/ffs_snapshot.c
@@ -203,7 +203,7 @@ ffs_snapshot(mp, snapfile)
 	ufs2_daddr_t numblks, blkno, *blkp, *snapblklist;
 	int error, cg, snaploc;
 	int i, size, len, loc;
-	int flag;
+	uint64_t flag;
 	struct timespec starttime = {0, 0}, endtime;
 	char saved_nice = 0;
 	long redo = 0, snaplistsize = 0;
@@ -212,7 +212,7 @@ ffs_snapshot(mp, snapfile)
 	struct fs *copy_fs = NULL, *fs;
 	struct thread *td = curthread;
 	struct inode *ip, *xp;
-	struct buf *bp, *nbp, *ibp, *sbp = NULL;
+	struct buf *bp, *nbp, *ibp;
 	struct nameidata nd;
 	struct mount *wrtmp;
 	struct vattr vat;
@@ -460,21 +460,14 @@ ffs_snapshot(mp, snapfile)
 	 * Grab a copy of the superblock and its summary information.
 	 * We delay writing it until the suspension is released below.
 	 */
-	error = bread(vp, lblkno(fs, fs->fs_sblockloc), fs->fs_bsize,
-	    KERNCRED, &sbp);
-	if (error) {
-		brelse(sbp);
-		sbp = NULL;
-		goto out1;
-	}
-	loc = blkoff(fs, fs->fs_sblockloc);
-	copy_fs = (struct fs *)(sbp->b_data + loc);
+	copy_fs = malloc((u_long)fs->fs_bsize, M_UFSMNT, M_WAITOK);
 	bcopy(fs, copy_fs, fs->fs_sbsize);
 	if ((fs->fs_flags & (FS_UNCLEAN | FS_NEEDSFSCK)) == 0)
 		copy_fs->fs_clean = 1;
 	size = fs->fs_bsize < SBLOCKSIZE ? fs->fs_bsize : SBLOCKSIZE;
 	if (fs->fs_sbsize < size)
-		bzero(&sbp->b_data[loc + fs->fs_sbsize], size - fs->fs_sbsize);
+		bzero(&((char *)copy_fs)[fs->fs_sbsize],
+		    size - fs->fs_sbsize);
 	size = blkroundup(fs, fs->fs_cssize);
 	if (fs->fs_contigsumsize > 0)
 		size += fs->fs_ncg * sizeof(int32_t);
@@ -490,8 +483,8 @@ ffs_snapshot(mp, snapfile)
 		    len, KERNCRED, &bp)) != 0) {
 			brelse(bp);
 			free(copy_fs->fs_csp, M_UFSMNT);
-			bawrite(sbp);
-			sbp = NULL;
+			free(copy_fs, M_UFSMNT);
+			copy_fs = NULL;
 			goto out1;
 		}
 		bcopy(bp->b_data, space, (u_int)len);
@@ -606,8 +599,8 @@ ffs_snapshot(mp, snapfile)
 		vdrop(xvp);
 		if (error) {
 			free(copy_fs->fs_csp, M_UFSMNT);
-			bawrite(sbp);
-			sbp = NULL;
+			free(copy_fs, M_UFSMNT);
+			copy_fs = NULL;
 			MNT_VNODE_FOREACH_ABORT(mp, mvp);
 			goto out1;
 		}
@@ -621,8 +614,8 @@ ffs_snapshot(mp, snapfile)
 		error = softdep_journal_lookup(mp, &xvp);
 		if (error) {
 			free(copy_fs->fs_csp, M_UFSMNT);
-			bawrite(sbp);
-			sbp = NULL;
+			free(copy_fs, M_UFSMNT);
+			copy_fs = NULL;
 			goto out1;
 		}
 		xp = VTOI(xvp);
@@ -688,8 +681,8 @@ ffs_snapshot(mp, snapfile)
 	VI_UNLOCK(devvp);
 	ASSERT_VOP_LOCKED(vp, "ffs_snapshot vp");
 out1:
-	KASSERT((sn != NULL && sbp != NULL && error == 0) ||
-		(sn == NULL && sbp == NULL && error != 0),
+	KASSERT((sn != NULL && copy_fs != NULL && error == 0) ||
+		(sn == NULL && copy_fs == NULL && error != 0),
 		("email phk@ and mckusick@"));
 	/*
 	 * Resume operation on filesystem.
@@ -703,7 +696,7 @@ ffs_snapshot(mp, snapfile)
 		    vp->v_mount->mnt_stat.f_mntonname, (long)endtime.tv_sec,
 		    endtime.tv_nsec / 1000000, redo, fs->fs_ncg);
 	}
-	if (sbp == NULL)
+	if (copy_fs == NULL)
 		goto out;
 	/*
 	 * Copy allocation information from all the snapshots in
@@ -793,6 +786,15 @@ ffs_snapshot(mp, snapfile)
 		space = (char *)space + fs->fs_bsize;
 		bawrite(nbp);
 	}
+	error = bread(vp, lblkno(fs, fs->fs_sblockloc), fs->fs_bsize,
+	    KERNCRED, &nbp);
+	if (error) {
+		brelse(nbp);
+	} else {
+		loc = blkoff(fs, fs->fs_sblockloc);
+		bcopy((char *)copy_fs, &nbp->b_data[loc], fs->fs_bsize);
+		bawrite(nbp);
+	}
 	/*
 	 * As this is the newest list, it is the most inclusive, so
 	 * should replace the previous list.
@@ -822,7 +824,8 @@ ffs_snapshot(mp, snapfile)
 		vrele(vp);		/* Drop extra reference */
 done:
 	free(copy_fs->fs_csp, M_UFSMNT);
-	bawrite(sbp);
+	free(copy_fs, M_UFSMNT);
+	copy_fs = NULL;
 out:
 	NDFREE(&nd, NDF_ONLY_PNBUF);
 	if (saved_nice > 0) {
diff --git a/sys/ufs/ffs/ffs_softdep.c b/sys/ufs/ffs/ffs_softdep.c
index 6cf033a43293..6c0e7d758cb6 100644
--- a/sys/ufs/ffs/ffs_softdep.c
+++ b/sys/ufs/ffs/ffs_softdep.c
@@ -12648,7 +12648,7 @@ softdep_request_cleanup(fs, vp, cred, resource)
 				MNT_ILOCK(mp);
 				continue;
 			}
-			(void) ffs_syncvnode(lvp, MNT_WAIT);
+			(void) ffs_syncvnode(lvp, MNT_NOWAIT);
 			vput(lvp);
 			MNT_ILOCK(mp);
 		}
diff --git a/sys/vm/memguard.c b/sys/vm/memguard.c
index c77f27dedd1a..25bea1b7569c 100644
--- a/sys/vm/memguard.c
+++ b/sys/vm/memguard.c
@@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 #include <vm/vm_extern.h>
+#include <vm/uma_int.h>
 #include <vm/memguard.h>
 
 SYSCTL_NODE(_vm, OID_AUTO, memguard, CTLFLAG_RW, NULL, "MemGuard data");
@@ -125,15 +126,17 @@ SYSCTL_ULONG(_vm_memguard, OID_AUTO, fail_kva, CTLFLAG_RD,
 SYSCTL_ULONG(_vm_memguard, OID_AUTO, fail_pgs, CTLFLAG_RD,
     &memguard_fail_pgs, 0, "MemGuard failures due to lack of pages");
 
-#define MG_GUARD	0x001
-#define MG_ALLLARGE	0x002
-static int memguard_options = MG_GUARD;
+#define MG_GUARD_AROUND		0x001
+#define MG_GUARD_ALLLARGE	0x002
+#define MG_GUARD_NOFREE		0x004
+static int memguard_options = MG_GUARD_AROUND;
 TUNABLE_INT("vm.memguard.options", &memguard_options);
 SYSCTL_INT(_vm_memguard, OID_AUTO, options, CTLFLAG_RW,
     &memguard_options, 0,
     "MemGuard options:\n"
     "\t0x001 - add guard pages around each allocation\n"
-    "\t0x002 - always use MemGuard for allocations over a page");
+    "\t0x002 - always use MemGuard for allocations over a page\n"
+    "\t0x004 - guard uma(9) zones with UMA_ZONE_NOFREE flag");
 
 static u_int memguard_minsize;
 static u_long memguard_minsize_reject;
@@ -282,7 +285,7 @@ memguard_alloc(unsigned long req_size, int flags)
 	 * value.
 	 */
 	size_v = size_p;
-	do_guard = (memguard_options & MG_GUARD) != 0;
+	do_guard = (memguard_options & MG_GUARD_AROUND) != 0;
 	if (do_guard)
 		size_v += 2 * PAGE_SIZE;
 
@@ -429,21 +432,32 @@ memguard_realloc(void *addr, unsigned long size, struct malloc_type *mtp,
 	return (newaddr);
 }
 
-int
-memguard_cmp(struct malloc_type *mtp, unsigned long size)
+static int
+memguard_cmp(unsigned long size)
 {
 
 	if (size < memguard_minsize) {
 		memguard_minsize_reject++;
 		return (0);
 	}
-	if ((memguard_options & MG_ALLLARGE) != 0 && size >= PAGE_SIZE)
+	if ((memguard_options & MG_GUARD_ALLLARGE) != 0 && size >= PAGE_SIZE)
 		return (1);
 	if (memguard_frequency > 0 &&
 	    (random() % 100000) < memguard_frequency) {
 		memguard_frequency_hits++;
 		return (1);
 	}
+
+	return (0);
+}
+
+int
+memguard_cmp_mtp(struct malloc_type *mtp, unsigned long size)
+{
+
+	if (memguard_cmp(size))
+		return(1);
+
 #if 1
 	/*
 	 * The safest way of comparsion is to always compare short description
@@ -467,3 +481,21 @@ memguard_cmp(struct malloc_type *mtp, unsigned long size)
 	return (0);
 #endif
 }
+
+int
+memguard_cmp_zone(uma_zone_t zone)
+{
+
+	 if ((memguard_options & MG_GUARD_NOFREE) == 0 &&
+	    zone->uz_flags & UMA_ZONE_NOFREE)
+		return (0);
+
+	if (memguard_cmp(zone->uz_size))
+		return (1);
+
+	/*
+	 * The safest way of comparsion is to always compare zone name,
+	 * but it is also the slowest way.
+	 */
+	return (strcmp(zone->uz_name, vm_memguard_desc) == 0);
+}
diff --git a/sys/vm/memguard.h b/sys/vm/memguard.h
index 25238b420d22..335e2378b522 100644
--- a/sys/vm/memguard.h
+++ b/sys/vm/memguard.h
@@ -40,7 +40,8 @@ void	memguard_init(struct vm_map *);
 void 	*memguard_alloc(unsigned long, int);
 void	*memguard_realloc(void *, unsigned long, struct malloc_type *, int);
 void	memguard_free(void *);
-int	memguard_cmp(struct malloc_type *, unsigned long);
+int	memguard_cmp_mtp(struct malloc_type *, unsigned long);
+int	memguard_cmp_zone(uma_zone_t);
 int	is_memguard_addr(void *);
 #else
 #define	memguard_fudge(size, xxx)	(size)
@@ -48,7 +49,8 @@ int	is_memguard_addr(void *);
 #define	memguard_alloc(size, flags)	NULL
 #define	memguard_realloc(a, s, mtp, f)	NULL
 #define	memguard_free(addr)		do { } while (0)
-#define	memguard_cmp(mtp, size)		0
+#define	memguard_cmp_mtp(mtp, size)	0
+#define	memguard_cmp_zone(zone)		0
 #define	is_memguard_addr(addr)		0
 #endif
 
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index d7da4f919126..058c155176a0 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -2057,7 +2057,7 @@ struct swapon_args {
  */
 /* ARGSUSED */
 int
-swapon(struct thread *td, struct swapon_args *uap)
+sys_swapon(struct thread *td, struct swapon_args *uap)
 {
 	struct vattr attr;
 	struct vnode *vp;
@@ -2199,7 +2199,7 @@ struct swapoff_args {
  */
 /* ARGSUSED */
 int
-swapoff(struct thread *td, struct swapoff_args *uap)
+sys_swapoff(struct thread *td, struct swapoff_args *uap)
 {
 	struct vnode *vp;
 	struct nameidata nd;
diff --git a/sys/vm/uma.h b/sys/vm/uma.h
index 241cabca86a5..fbba22fa9395 100644
--- a/sys/vm/uma.h
+++ b/sys/vm/uma.h
@@ -255,8 +255,8 @@ int uma_zsecond_add(uma_zone_t zone, uma_zone_t master);
  * physical parameters of the request and may not be provided by the consumer.
  */
 #define	UMA_ZONE_INHERIT						\
-    (UMA_ZONE_OFFPAGE | UMA_ZONE_MALLOC | UMA_ZONE_HASH |		\
-    UMA_ZONE_REFCNT | UMA_ZONE_VTOSLAB)
+    (UMA_ZONE_OFFPAGE | UMA_ZONE_MALLOC | UMA_ZONE_NOFREE |		\
+    UMA_ZONE_HASH | UMA_ZONE_REFCNT | UMA_ZONE_VTOSLAB)
 
 /* Definitions for align */
 #define UMA_ALIGN_PTR	(sizeof(void *) - 1)	/* Alignment fit for ptr */
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 8da5b8eea508..9fbea556b212 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
 #include "opt_param.h"
+#include "opt_vm.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -88,6 +89,10 @@ __FBSDID("$FreeBSD$");
 
 #include <ddb/ddb.h>
 
+#ifdef DEBUG_MEMGUARD
+#include <vm/memguard.h>
+#endif
+
 /*
  * This is the zone and keg from which all zones are spawned.  The idea is that
  * even the zone & keg heads are allocated from the allocator, so we use the
@@ -1978,7 +1983,29 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
 		    "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
 	}
-
+#ifdef DEBUG_MEMGUARD
+	if (memguard_cmp_zone(zone)) {
+		item = memguard_alloc(zone->uz_size, flags);
+		if (item != NULL) {
+			/*
+			 * Avoid conflict with the use-after-free
+			 * protecting infrastructure from INVARIANTS.
+			 */
+			if (zone->uz_init != NULL &&
+			    zone->uz_init != mtrash_init &&
+			    zone->uz_init(item, zone->uz_size, flags) != 0)
+				return (NULL);
+			if (zone->uz_ctor != NULL &&
+			    zone->uz_ctor != mtrash_ctor &&
+			    zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
+			    	zone->uz_fini(item, zone->uz_size);
+				return (NULL);
+			}
+			return (item);
+		}
+		/* This is unfortunate but should not be fatal. */
+	}
+#endif
 	/*
 	 * If possible, allocate from the per-CPU cache.  There are two
 	 * requirements for safe access to the per-CPU cache: (1) the thread
@@ -2544,7 +2571,16 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
         /* uma_zfree(..., NULL) does nothing, to match free(9). */
         if (item == NULL)
                 return;
-
+#ifdef DEBUG_MEMGUARD
+	if (is_memguard_addr(item)) {
+		if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor)
+			zone->uz_dtor(item, zone->uz_size, udata);
+		if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini)
+			zone->uz_fini(item, zone->uz_size);
+		memguard_free(item);
+		return;
+	}
+#endif
 	if (zone->uz_dtor)
 		zone->uz_dtor(item, zone->uz_size, udata);
 
diff --git a/sys/vm/vm_contig.c b/sys/vm/vm_contig.c
index 67ebdc3ff53c..dc8779967903 100644
--- a/sys/vm/vm_contig.c
+++ b/sys/vm/vm_contig.c
@@ -65,7 +65,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/lock.h>
-#include <sys/malloc.h>
 #include <sys/mount.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
@@ -334,25 +333,6 @@ contigmapping(vm_map_t map, vm_size_t size, vm_page_t m, vm_memattr_t memattr,
 	return (addr);
 }
 
-void *
-contigmalloc(
-	unsigned long size,	/* should be size_t here and for malloc() */
-	struct malloc_type *type,
-	int flags,
-	vm_paddr_t low,
-	vm_paddr_t high,
-	unsigned long alignment,
-	unsigned long boundary)
-{
-	void *ret;
-
-	ret = (void *)kmem_alloc_contig(kernel_map, size, flags, low, high,
-	    alignment, boundary, VM_MEMATTR_DEFAULT);
-	if (ret != NULL)
-		malloc_type_allocated(type, round_page(size));
-	return (ret);
-}
-
 vm_offset_t
 kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
     vm_paddr_t high, unsigned long alignment, unsigned long boundary,
@@ -382,11 +362,3 @@ kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
 	}
 	return (ret);
 }
-
-void
-contigfree(void *addr, unsigned long size, struct malloc_type *type)
-{
-
-	kmem_free(kernel_map, (vm_offset_t)addr, size);
-	malloc_type_freed(type, round_page(size));
-}
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 1b8ac2fef17b..8c98b26a0146 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -1090,18 +1090,10 @@ vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
 			 * performed through an unmanaged mapping or by a DMA
 			 * operation.
 			 *
-			 * The object lock is not held here.  Therefore, like
-			 * a pmap operation, the page queues lock may be
-			 * required in order to call vm_page_dirty().  See
-			 * vm_page_clear_dirty_mask().
+			 * The object lock is not held here.
+			 * See vm_page_clear_dirty_mask().
 			 */
-#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
 			vm_page_dirty(*mp);
-#else
-			vm_page_lock_queues();
-			vm_page_dirty(*mp);
-			vm_page_unlock_queues();
-#endif
 		}
 	}
 	if (pmap_failed) {
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index ce899e9078c2..e85b681a8694 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -104,7 +104,7 @@ static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
  */
 /* ARGSUSED */
 int
-sbrk(td, uap)
+sys_sbrk(td, uap)
 	struct thread *td;
 	struct sbrk_args *uap;
 {
@@ -123,7 +123,7 @@ struct sstk_args {
  */
 /* ARGSUSED */
 int
-sstk(td, uap)
+sys_sstk(td, uap)
 	struct thread *td;
 	struct sstk_args *uap;
 {
@@ -180,7 +180,7 @@ struct mmap_args {
  * MPSAFE
  */
 int
-mmap(td, uap)
+sys_mmap(td, uap)
 	struct thread *td;
 	struct mmap_args *uap;
 {
@@ -402,7 +402,7 @@ freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap)
 	oargs.flags = uap->flags;
 	oargs.fd = uap->fd;
 	oargs.pos = uap->pos;
-	return (mmap(td, &oargs));
+	return (sys_mmap(td, &oargs));
 }
 
 #ifdef COMPAT_43
@@ -454,7 +454,7 @@ ommap(td, uap)
 		nargs.flags |= MAP_FIXED;
 	nargs.fd = uap->fd;
 	nargs.pos = uap->pos;
-	return (mmap(td, &nargs));
+	return (sys_mmap(td, &nargs));
 }
 #endif				/* COMPAT_43 */
 
@@ -470,7 +470,7 @@ struct msync_args {
  * MPSAFE
  */
 int
-msync(td, uap)
+sys_msync(td, uap)
 	struct thread *td;
 	struct msync_args *uap;
 {
@@ -523,7 +523,7 @@ struct munmap_args {
  * MPSAFE
  */
 int
-munmap(td, uap)
+sys_munmap(td, uap)
 	struct thread *td;
 	struct munmap_args *uap;
 {
@@ -599,7 +599,7 @@ struct mprotect_args {
  * MPSAFE
  */
 int
-mprotect(td, uap)
+sys_mprotect(td, uap)
 	struct thread *td;
 	struct mprotect_args *uap;
 {
@@ -641,7 +641,7 @@ struct minherit_args {
  * MPSAFE
  */
 int
-minherit(td, uap)
+sys_minherit(td, uap)
 	struct thread *td;
 	struct minherit_args *uap;
 {
@@ -683,7 +683,7 @@ struct madvise_args {
  */
 /* ARGSUSED */
 int
-madvise(td, uap)
+sys_madvise(td, uap)
 	struct thread *td;
 	struct madvise_args *uap;
 {
@@ -747,7 +747,7 @@ struct mincore_args {
  */
 /* ARGSUSED */
 int
-mincore(td, uap)
+sys_mincore(td, uap)
 	struct thread *td;
 	struct mincore_args *uap;
 {
@@ -1003,7 +1003,7 @@ struct mlock_args {
  * MPSAFE
  */
 int
-mlock(td, uap)
+sys_mlock(td, uap)
 	struct thread *td;
 	struct mlock_args *uap;
 {
@@ -1067,7 +1067,7 @@ struct mlockall_args {
  * MPSAFE
  */
 int
-mlockall(td, uap)
+sys_mlockall(td, uap)
 	struct thread *td;
 	struct mlockall_args *uap;
 {
@@ -1144,7 +1144,7 @@ struct munlockall_args {
  * MPSAFE
  */
 int
-munlockall(td, uap)
+sys_munlockall(td, uap)
 	struct thread *td;
 	struct munlockall_args *uap;
 {
@@ -1185,7 +1185,7 @@ struct munlock_args {
  * MPSAFE
  */
 int
-munlock(td, uap)
+sys_munlock(td, uap)
 	struct thread *td;
 	struct munlock_args *uap;
 {
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 341c2383abe3..cde138760c0f 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -745,9 +745,9 @@ vm_page_sleep(vm_page_t m, const char *msg)
  *
  *	Set all bits in the page's dirty field.
  *
- *	The object containing the specified page must be locked if the call is
- *	made from the machine-independent layer.  If, however, the call is
- *	made from the pmap layer, then the page queues lock may be required.
+ *	The object containing the specified page must be locked if the
+ *	call is made from the machine-independent layer.
+ *
  *	See vm_page_clear_dirty_mask().
  */
 void
@@ -908,7 +908,7 @@ void
 vm_page_remove(vm_page_t m)
 {
 	vm_object_t object;
-	vm_page_t root;
+	vm_page_t next, prev, root;
 
 	if ((m->oflags & VPO_UNMANAGED) == 0)
 		vm_page_lock_assert(m, MA_OWNED);
@@ -923,15 +923,42 @@ vm_page_remove(vm_page_t m)
 	/*
 	 * Now remove from the object's list of backed pages.
 	 */
-	if (m != object->root)
-		vm_page_splay(m->pindex, object->root);
-	if (m->left == NULL)
-		root = m->right;
-	else {
-		root = vm_page_splay(m->pindex, m->left);
-		root->right = m->right;
+	if ((next = TAILQ_NEXT(m, listq)) != NULL && next->left == m) {
+		/*
+		 * Since the page's successor in the list is also its parent
+		 * in the tree, its right subtree must be empty.
+		 */
+		next->left = m->left;
+		KASSERT(m->right == NULL,
+		    ("vm_page_remove: page %p has right child", m));
+	} else if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
+	    prev->right == m) {
+		/*
+		 * Since the page's predecessor in the list is also its parent
+		 * in the tree, its left subtree must be empty.
+		 */
+		KASSERT(m->left == NULL,
+		    ("vm_page_remove: page %p has left child", m));
+		prev->right = m->right;
+	} else {
+		if (m != object->root)
+			vm_page_splay(m->pindex, object->root);
+		if (m->left == NULL)
+			root = m->right;
+		else if (m->right == NULL)
+			root = m->left;
+		else {
+			/*
+			 * Move the page's successor to the root, because
+			 * pages are usually removed in ascending order.
+			 */
+			if (m->right != next)
+				vm_page_splay(m->pindex, m->right);
+			next->left = m->left;
+			root = next;
+		}
+		object->root = root;
 	}
-	object->root = root;
 	TAILQ_REMOVE(&object->memq, m, listq);
 
 	/*
@@ -2021,7 +2048,7 @@ void
 vm_page_cache(vm_page_t m)
 {
 	vm_object_t object;
-	vm_page_t root;
+	vm_page_t next, prev, root;
 
 	vm_page_lock_assert(m, MA_OWNED);
 	object = m->object;
@@ -2056,15 +2083,42 @@ vm_page_cache(vm_page_t m)
 	 * Remove the page from the object's collection of resident
 	 * pages. 
 	 */
-	if (m != object->root)
-		vm_page_splay(m->pindex, object->root);
-	if (m->left == NULL)
-		root = m->right;
-	else {
-		root = vm_page_splay(m->pindex, m->left);
-		root->right = m->right;
+	if ((next = TAILQ_NEXT(m, listq)) != NULL && next->left == m) {
+		/*
+		 * Since the page's successor in the list is also its parent
+		 * in the tree, its right subtree must be empty.
+		 */
+		next->left = m->left;
+		KASSERT(m->right == NULL,
+		    ("vm_page_cache: page %p has right child", m));
+	} else if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
+	    prev->right == m) {
+		/*
+		 * Since the page's predecessor in the list is also its parent
+		 * in the tree, its left subtree must be empty.
+		 */
+		KASSERT(m->left == NULL,
+		    ("vm_page_cache: page %p has left child", m));
+		prev->right = m->right;
+	} else {
+		if (m != object->root)
+			vm_page_splay(m->pindex, object->root);
+		if (m->left == NULL)
+			root = m->right;
+		else if (m->right == NULL)
+			root = m->left;
+		else {
+			/*
+			 * Move the page's successor to the root, because
+			 * pages are usually removed in ascending order.
+			 */
+			if (m->right != next)
+				vm_page_splay(m->pindex, m->right);
+			next->left = m->left;
+			root = next;
+		}
+		object->root = root;
 	}
-	object->root = root;
 	TAILQ_REMOVE(&object->memq, m, listq);
 	object->resident_page_count--;
 
@@ -2339,44 +2393,52 @@ vm_page_set_valid(vm_page_t m, int base, int size)
 static __inline void
 vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
 {
+	uintptr_t addr;
+#if PAGE_SIZE < 16384
+	int shift;
+#endif
 
 	/*
 	 * If the object is locked and the page is neither VPO_BUSY nor
 	 * PGA_WRITEABLE, then the page's dirty field cannot possibly be
-	 * set by a concurrent pmap operation. 
+	 * set by a concurrent pmap operation.
 	 */
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 	if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
 		m->dirty &= ~pagebits;
 	else {
-#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
 		/*
-		 * On the aforementioned architectures, the page queues lock
-		 * is not required by the following read-modify-write
-		 * operation.  The combination of the object's lock and an
-		 * atomic operation suffice.  Moreover, the pmap layer on
-		 * these architectures can call vm_page_dirty() without
-		 * holding the page queues lock.
+		 * The pmap layer can call vm_page_dirty() without
+		 * holding a distinguished lock.  The combination of
+		 * the object's lock and an atomic operation suffice
+		 * to guarantee consistency of the page dirty field.
+		 *
+		 * For PAGE_SIZE == 32768 case, compiler already
+		 * properly aligns the dirty field, so no forcible
+		 * alignment is needed. Only require existence of
+		 * atomic_clear_64 when page size is 32768.
 		 */
-#if PAGE_SIZE == 4096
-		atomic_clear_char(&m->dirty, pagebits);
-#elif PAGE_SIZE == 8192
-		atomic_clear_short(&m->dirty, pagebits);
+		addr = (uintptr_t)&m->dirty;
+#if PAGE_SIZE == 32768
+#error pagebits too short
+		atomic_clear_64((uint64_t *)addr, pagebits);
 #elif PAGE_SIZE == 16384
-		atomic_clear_int(&m->dirty, pagebits);
-#else
-#error "PAGE_SIZE is not supported."
-#endif
-#else
+		atomic_clear_32((uint32_t *)addr, pagebits);
+#else		/* PAGE_SIZE <= 8192 */
 		/*
-		 * Otherwise, the page queues lock is required to ensure that
-		 * a concurrent pmap operation does not set the page's dirty
-		 * field during the following read-modify-write operation.
+		 * Use a trick to perform a 32-bit atomic on the
+		 * containing aligned word, to not depend on the existence
+		 * of atomic_clear_{8, 16}.
 		 */
-		vm_page_lock_queues();
-		m->dirty &= ~pagebits;
-		vm_page_unlock_queues();
+		shift = addr & (sizeof(uint32_t) - 1);
+#if BYTE_ORDER == BIG_ENDIAN
+		shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
+#else
+		shift *= NBBY;
 #endif
+		addr &= ~(sizeof(uint32_t) - 1);
+		atomic_clear_32((uint32_t *)addr, pagebits << shift);
+#endif		/* PAGE_SIZE */
 	}
 }
 
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 5431d7986a0e..23637bbc834a 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -94,21 +94,21 @@
  *	object that the page belongs to (O), the pool lock for the page (P),
  *	or the lock for either the free or paging queues (Q).  If a field is
  *	annotated below with two of these locks, then holding either lock is
- *	sufficient for read access, but both locks are required for write 
+ *	sufficient for read access, but both locks are required for write
  *	access.
  *
- *	In contrast, the synchronization of accesses to the page's dirty field
- *	is machine dependent (M).  In the machine-independent layer, the lock
- *	on the object that the page belongs to must be held in order to
- *	operate on the field.  However, the pmap layer is permitted to set
- *	all bits within the field without holding that lock.  Therefore, if
- *	the underlying architecture does not support atomic read-modify-write
- *	operations on the field's type, then the machine-independent layer
- *	must also hold the page queues lock when performing read-modify-write
- *	operations and the pmap layer must hold the page queues lock when
- *	setting the field.  In the machine-independent layer, the
- *	implementation of read-modify-write operations on the field is
- *	encapsulated in vm_page_clear_dirty_mask().
+ *	In contrast, the synchronization of accesses to the page's
+ *	dirty field is machine dependent (M).  In the
+ *	machine-independent layer, the lock on the object that the
+ *	page belongs to must be held in order to operate on the field.
+ *	However, the pmap layer is permitted to set all bits within
+ *	the field without holding that lock.  If the underlying
+ *	architecture does not support atomic read-modify-write
+ *	operations on the field's type, then the machine-independent
+ *	layer uses a 32-bit atomic on the aligned 32-bit word that
+ *	contains the dirty field.  In the machine-independent layer,
+ *	the implementation of read-modify-write operations on the
+ *	field is encapsulated in vm_page_clear_dirty_mask().
  */
 
 TAILQ_HEAD(pglist, vm_page);
@@ -139,17 +139,17 @@ struct vm_page {
 	/* so, on normal X86 kernels, they must be at least 8 bits wide */
 	/* In reality, support for 32KB pages is not fully implemented. */
 #if PAGE_SIZE == 4096
-	u_char	valid;			/* map of valid DEV_BSIZE chunks (O) */
-	u_char	dirty;			/* map of dirty DEV_BSIZE chunks (M) */
+	uint8_t	valid;			/* map of valid DEV_BSIZE chunks (O) */
+	uint8_t	dirty;			/* map of dirty DEV_BSIZE chunks (M) */
 #elif PAGE_SIZE == 8192
-	u_short	valid;			/* map of valid DEV_BSIZE chunks (O) */
-	u_short	dirty;			/* map of dirty DEV_BSIZE chunks (M) */
+	uint16_t valid;			/* map of valid DEV_BSIZE chunks (O) */
+	uint16_t dirty;			/* map of dirty DEV_BSIZE chunks (M) */
 #elif PAGE_SIZE == 16384
-	u_int valid;			/* map of valid DEV_BSIZE chunks (O) */
-	u_int dirty;			/* map of dirty DEV_BSIZE chunks (M) */
+	uint32_t valid;			/* map of valid DEV_BSIZE chunks (O) */
+	uint32_t dirty;			/* map of dirty DEV_BSIZE chunks (M) */
 #elif PAGE_SIZE == 32768
-	u_long valid;			/* map of valid DEV_BSIZE chunks (O) */
-	u_long dirty;			/* map of dirty DEV_BSIZE chunks (M) */
+	uint64_t valid;			/* map of valid DEV_BSIZE chunks (O) */
+	uint64_t dirty;			/* map of dirty DEV_BSIZE chunks (M) */
 #endif
 };
 
diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c
index 93fee70b2212..b47455d71b61 100644
--- a/sys/vm/vm_phys.c
+++ b/sys/vm/vm_phys.c
@@ -33,6 +33,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
+#include "opt_vm.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/vm/vm_unix.c b/sys/vm/vm_unix.c
index 6f8e7c85769a..253ab775d8ba 100644
--- a/sys/vm/vm_unix.c
+++ b/sys/vm/vm_unix.c
@@ -36,6 +36,8 @@
  *	@(#)vm_unix.c	8.1 (Berkeley) 6/11/93
  */
 
+#include "opt_compat.h"
+
 /*
  * Traditional sbrk/grow interface to VM
  */
@@ -49,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/racct.h>
 #include <sys/resourcevar.h>
+#include <sys/sysent.h>
 #include <sys/sysproto.h>
 #include <sys/systm.h>
 
@@ -68,14 +71,14 @@ struct obreak_args {
  */
 /* ARGSUSED */
 int
-obreak(td, uap)
+sys_obreak(td, uap)
 	struct thread *td;
 	struct obreak_args *uap;
 {
 	struct vmspace *vm = td->td_proc->p_vmspace;
 	vm_offset_t new, old, base;
 	rlim_t datalim, vmemlim;
-	int rv;
+	int prot, rv;
 	int error = 0;
 	boolean_t do_map_wirefuture;
 
@@ -134,9 +137,16 @@ obreak(td, uap)
 			goto done;
 		}
 		PROC_UNLOCK(td->td_proc);
+#endif
+		prot = VM_PROT_RW;
+#ifdef COMPAT_FREEBSD32
+#if defined(__amd64__) || defined(__ia64__)
+		if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32))
+			prot |= VM_PROT_EXECUTE;
+#endif
 #endif
 		rv = vm_map_insert(&vm->vm_map, NULL, 0, old, new,
-		    VM_PROT_RW, VM_PROT_ALL, 0);
+		    prot, VM_PROT_ALL, 0);
 		if (rv != KERN_SUCCESS) {
 #ifdef RACCT
 			PROC_LOCK(td->td_proc);
@@ -197,7 +207,7 @@ struct ovadvise_args {
  */
 /* ARGSUSED */
 int
-ovadvise(td, uap)
+sys_ovadvise(td, uap)
 	struct thread *td;
 	struct ovadvise_args *uap;
 {
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index cb652f7e186a..e3222cbd026f 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -1150,7 +1150,7 @@ vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount,
 	VM_OBJECT_UNLOCK(object);
 
 	/*
-	 * pageouts are already clustered, use IO_ASYNC t o force a bawrite()
+	 * pageouts are already clustered, use IO_ASYNC to force a bawrite()
 	 * rather then a bdwrite() to prevent paging I/O from saturating 
 	 * the buffer cache.  Dummy-up the sequential heuristic to cause
 	 * large ranges to cluster.  If neither IO_SYNC or IO_ASYNC is set,
diff --git a/sys/x86/acpica/srat.c b/sys/x86/acpica/srat.c
index 4b7df372cba3..cd04f876bf48 100644
--- a/sys/x86/acpica/srat.c
+++ b/sys/x86/acpica/srat.c
@@ -59,6 +59,26 @@ static vm_paddr_t srat_physaddr;
 
 static void	srat_walk_table(acpi_subtable_handler *handler, void *arg);
 
+/*
+ * Returns true if a memory range overlaps with at least one range in
+ * phys_avail[].
+ */
+static int
+overlaps_phys_avail(vm_paddr_t start, vm_paddr_t end)
+{
+	int i;
+
+	for (i = 0; phys_avail[i] != 0 && phys_avail[i + 1] != 0; i += 2) {
+		if (phys_avail[i + 1] < start)
+			continue;
+		if (phys_avail[i] < end)
+			return (1);
+		break;
+	}
+	return (0);
+	
+}
+
 static void
 srat_parse_entry(ACPI_SUBTABLE_HEADER *entry, void *arg)
 {
@@ -111,6 +131,12 @@ srat_parse_entry(ACPI_SUBTABLE_HEADER *entry, void *arg)
 			    "enabled" : "disabled");
 		if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED))
 			break;
+		if (!overlaps_phys_avail(mem->BaseAddress,
+		    mem->BaseAddress + mem->Length)) {
+			printf("SRAT: Ignoring memory at addr %jx\n",
+			    (uintmax_t)mem->BaseAddress);
+			break;
+		}
 		if (num_mem == VM_PHYSSEG_MAX) {
 			printf("SRAT: Too many memory regions\n");
 			*(int *)arg = ENXIO;
diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c
index 44e3a2e93f97..768b6a83ee36 100644
--- a/sys/x86/x86/local_apic.c
+++ b/sys/x86/x86/local_apic.c
@@ -195,10 +195,9 @@ static void	lapic_set_icr_timer(uint32_t value);
 uint32_t	lapic_irr(int num);
 uint32_t	lapic_tmr(int num);
 uint32_t	lapic_isr(int num);
-static uint32_t	lapic_icr_lo(void);
-static void	lapic_set_icr_lo(uint32_t value);
-static uint32_t	lapic_icr_hi(void);
-static void	lapic_set_icr_hi(uint32_t value);
+static uint32_t lapic_icr_lo(void);
+static uint32_t lapic_icr_hi(void);
+static void	lapic_set_icr(uint64_t value);
 static boolean_t lapic_missing(void);
 
 struct pic lapic_pic = { .pic_resume = lapic_resume };
@@ -904,8 +903,6 @@ lapic_isr(int num)
 	}
 }
 
-static uint32_t icr_hi_stashed[MAXCPU];
-
 static uint32_t
 lapic_icr_lo(void)
 {
@@ -916,17 +913,6 @@ lapic_icr_lo(void)
 		return (lapic->icr_lo);
 }
 
-static void
-lapic_set_icr_lo(uint32_t value)
-{
-
-	if (x2apic) {
-		wrmsr(MSR_APIC_ICR,
-		      (uint64_t)icr_hi_stashed[curcpu] << 32 | value);
-	} else
-		lapic->icr_lo = value;
-}
-
 static uint32_t
 lapic_icr_hi(void)
 {
@@ -938,12 +924,15 @@ lapic_icr_hi(void)
 }
 
 static void
-lapic_set_icr_hi(uint32_t value)
+lapic_set_icr(uint64_t value)
 {
+
 	if (x2apic)
-		icr_hi_stashed[curcpu] = value >> APIC_ID_SHIFT; /* XXX */
-	else
-		lapic->icr_hi = value;
+		wrmsr(MSR_APIC_ICR, value);
+	else {
+		lapic->icr_hi = value >> 32;
+		lapic->icr_lo = value;
+	}
 }
 
 static boolean_t
@@ -1790,7 +1779,8 @@ lapic_ipi_wait(int delay)
 void
 lapic_ipi_raw(register_t icrlo, u_int dest)
 {
-	register_t value, saveintr;
+	register_t saveintr;
+	uint32_t hi, lo;
 
 	/* XXX: Need more sanity checking of icrlo? */
 	KASSERT(!lapic_missing(), ("%s called too early", __func__));
@@ -1802,17 +1792,21 @@ lapic_ipi_raw(register_t icrlo, u_int dest)
 	/* Set destination in ICR HI register if it is being used. */
 	saveintr = intr_disable();
 	if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
-		value = lapic_icr_hi();
-		value &= ~APIC_ID_MASK;
-		value |= dest << APIC_ID_SHIFT;
-		lapic_set_icr_hi(value);
-	}
+		if (x2apic) {
+			hi = dest;
+		} else {
+			hi = lapic_icr_hi();
+			hi &= ~APIC_ID_MASK;
+			hi |= dest << APIC_ID_SHIFT;
+		}
+	} else
+		hi = 0;
 
 	/* Program the contents of the IPI and dispatch it. */
-	value = lapic_icr_lo();
-	value &= APIC_ICRLO_RESV_MASK;
-	value |= icrlo;
-	lapic_set_icr_lo(value);
+	lo = lapic_icr_lo();
+	lo &= APIC_ICRLO_RESV_MASK;
+	lo |= icrlo;
+	lapic_set_icr((uint64_t)hi << 32 | lo);
 	intr_restore(saveintr);
 }
 
diff --git a/sys/xen/xenbus/xenbusb.c b/sys/xen/xenbus/xenbusb.c
index cc519c5ea208..3e2d56f13744 100644
--- a/sys/xen/xenbus/xenbusb.c
+++ b/sys/xen/xenbus/xenbusb.c
@@ -773,7 +773,7 @@ xenbusb_resume(device_t dev)
 			ivars = device_get_ivars(kids[i]);
 
 			xs_unregister_watch(&ivars->xd_otherend_watch);
-			ivars->xd_state = XenbusStateInitialising;
+			xenbus_set_state(kids[i], XenbusStateInitialising);
 
 			/*
 			 * Find the new backend details and
@@ -783,16 +783,16 @@ xenbusb_resume(device_t dev)
 			if (error)
 				return (error);
 
-			DEVICE_RESUME(kids[i]);
-
 			statepath = malloc(ivars->xd_otherend_path_len
 			    + strlen("/state") + 1, M_XENBUS, M_WAITOK);
 			sprintf(statepath, "%s/state", ivars->xd_otherend_path);
 
 			free(ivars->xd_otherend_watch.node, M_XENBUS);
 			ivars->xd_otherend_watch.node = statepath;
-			xs_register_watch(&ivars->xd_otherend_watch);
 
+			DEVICE_RESUME(kids[i]);
+
+			xs_register_watch(&ivars->xd_otherend_watch);
 #if 0
 			/*
 			 * Can't do this yet since we are running in
diff --git a/sys/xen/xenbus/xenbusb_back.c b/sys/xen/xenbus/xenbusb_back.c
index 1252abed07f5..351140a165d6 100644
--- a/sys/xen/xenbus/xenbusb_back.c
+++ b/sys/xen/xenbus/xenbusb_back.c
@@ -292,7 +292,7 @@ static device_method_t xenbusb_back_methods[] = {
 	DEVMETHOD(device_detach,        bus_generic_detach), 
 	DEVMETHOD(device_shutdown,      bus_generic_shutdown), 
 	DEVMETHOD(device_suspend,       bus_generic_suspend), 
-	DEVMETHOD(device_resume,        bus_generic_resume), 
+	DEVMETHOD(device_resume,        xenbusb_resume), 
  
 	/* Bus Interface */ 
 	DEVMETHOD(bus_print_child,      xenbusb_print_child),
diff --git a/sys/xen/xenbus/xenbusb_front.c b/sys/xen/xenbus/xenbusb_front.c
index b4e470ed22af..818e7f0a2cc6 100644
--- a/sys/xen/xenbus/xenbusb_front.c
+++ b/sys/xen/xenbus/xenbusb_front.c
@@ -171,7 +171,7 @@ static device_method_t xenbusb_front_methods[] = {
 	DEVMETHOD(device_detach,        bus_generic_detach), 
 	DEVMETHOD(device_shutdown,      bus_generic_shutdown), 
 	DEVMETHOD(device_suspend,       bus_generic_suspend), 
-	DEVMETHOD(device_resume,        bus_generic_resume), 
+	DEVMETHOD(device_resume,        xenbusb_resume), 
  
 	/* Bus Interface */ 
 	DEVMETHOD(bus_print_child,      xenbusb_print_child),
diff --git a/sys/xen/xenstore/xenstore.c b/sys/xen/xenstore/xenstore.c
index ee1bb228246e..658e6e38ea03 100644
--- a/sys/xen/xenstore/xenstore.c
+++ b/sys/xen/xenstore/xenstore.c
@@ -721,8 +721,8 @@ xs_reply_filter(uint32_t request_msg_type,
 	/*
 	 * The count of transactions drops if we attempted
 	 * to end a transaction (even if that attempt fails
-	 * in error), we receive a transaction end acknowledgement
-	 * or if our attempt to begin a transactionfails.
+	 * in error), we receive a transaction end acknowledgement,
+	 * or if our attempt to begin a transaction fails.
 	 */
 	if (request_msg_type == XS_TRANSACTION_END
 	 || (request_reply_error == 0 && reply_msg_type == XS_TRANSACTION_END)
@@ -1194,8 +1194,14 @@ xs_attach(device_t dev)
  * all transactions and individual requests have completed.
  */
 static int
-xs_suspend(device_t dev __unused)
+xs_suspend(device_t dev)
 {
+	int error;
+
+	/* Suspend child Xen devices. */
+	error = bus_generic_suspend(dev);
+	if (error != 0)
+		return (error);
 
 	sx_xlock(&xs.suspend_mutex);
 	sx_xlock(&xs.request_mutex);
@@ -1227,6 +1233,9 @@ xs_resume(device_t dev __unused)
 
 	sx_xunlock(&xs.suspend_mutex);
 
+	/* Resume child Xen devices. */
+	bus_generic_resume(dev);
+
 	return (0);
 }
 
diff --git a/tools/make_libdeps.sh b/tools/make_libdeps.sh
index b0b3f186b1ac..608422aeab8d 100644
--- a/tools/make_libdeps.sh
+++ b/tools/make_libdeps.sh
@@ -52,7 +52,7 @@ sed -E
     -e's;-l(ncurses|termcap)!;lib/ncurses/ncurses;g'
     -e's;-l(gcc)!;gnu/lib/lib\1;g'
     -e's;-lssp_nonshared!;gnu/lib/libssp/libssp_nonshared;g'
-    -e's;-l(asn1|heimntlm|hx509|krb5|roken)!;kerberos5/lib/lib\1;g'
+    -e's;-l(asn1|hdb|heimntlm|hx509|krb5|roken)!;kerberos5/lib/lib\1;g'
     -e's;-l(crypto|ssh|ssl)!;secure/lib/lib\1;g'
     -e's;-l([^!]+)!;lib/lib\1;g'
 "
diff --git a/tools/regression/doat/doat.c b/tools/regression/doat/doat.c
index f5ac795d92df..d71648d8f0ff 100644
--- a/tools/regression/doat/doat.c
+++ b/tools/regression/doat/doat.c
@@ -103,8 +103,9 @@ setup(void)
 {
 	int i, error;
 	struct stat sb;
+	size_t len;
 
-	tests = calloc(NUM_OF_TESTS, sizeof(struct test));
+	tests = calloc(NUM_OF_TESTS + 1, sizeof(struct test));
 	if (tests == NULL) {
 		perror("");
 		exit(0);		
@@ -116,14 +117,16 @@ setup(void)
 		exit(0);
 	}
 
-	absolute_path = realloc(absolute_path, strlen(absolute_path) + 5);
+	len = strlen(absolute_path);
+	absolute_path = realloc(absolute_path,
+	    len + 1 + strlen(relative_path) + 1);
 	if (absolute_path == NULL) {
 		perror("realloc");
 		exit(0);
 	}
 
-	absolute_path[strlen(absolute_path)] = '/';
-	strcpy(absolute_path + strlen(absolute_path), relative_path);
+	absolute_path[len] = '/';
+	strcpy(absolute_path + len + 1, relative_path);
 
 	absolute_file = malloc(strlen(absolute_path) + 1 + strlen(file));
 	bzero(absolute_file, strlen(absolute_path) + 1 + strlen(file));
@@ -145,7 +148,7 @@ setup(void)
 	relative_file[strlen(relative_file)] = '/';
 	strcpy(relative_file + strlen(relative_path), file);
 
-	error = mkdir(relative_path, 666);
+	error = mkdir(relative_path, 0700);
 	dir_exist = (errno == EEXIST);
 	if (error && errno != EEXIST) {
 		perror("tmp");
@@ -154,7 +157,7 @@ setup(void)
 
 	error = stat("tmp/foo", &sb);
 	file_exist = (errno != ENOENT);
-	i = open("tmp/foo", O_RDONLY | O_CREAT);
+	i = open("tmp/foo", O_RDONLY | O_CREAT, 0666);
 	if (i == -1) {
 		perror("foo");
 		exit(0);
diff --git a/tools/regression/geom_eli/resize.t b/tools/regression/geom_eli/resize.t
index f3d07e6dfe55..86ee36475fb5 100644
--- a/tools/regression/geom_eli/resize.t
+++ b/tools/regression/geom_eli/resize.t
@@ -118,6 +118,7 @@ echo ok $i - "fsck says ${md}a.eli is clean," $(echo $(echo "$out" | wc -l)) \
 i=$((i + 1))
 
 geli detach ${md}a.eli
+gpart destroy -F $md >/dev/null
 
 
 # Verify that the man page example works, changing ada0 to $md,
diff --git a/tools/regression/lib/msun/Makefile b/tools/regression/lib/msun/Makefile
index 155fd314e5a6..261b1cba0622 100644
--- a/tools/regression/lib/msun/Makefile
+++ b/tools/regression/lib/msun/Makefile
@@ -1,6 +1,7 @@
 # $FreeBSD$
 
-TESTS=	test-cexp test-conj test-csqrt test-exponential test-fenv test-fma \
+TESTS=	test-cexp test-conj test-csqrt test-ctrig \
+	test-exponential test-fenv test-fma \
 	test-fmaxmin test-ilogb test-invtrig test-logarithm test-lrint \
 	test-lround test-nan test-nearbyint test-next test-rem test-trig
 CFLAGS+= -O0 -lm
diff --git a/tools/regression/lib/msun/test-ctrig.c b/tools/regression/lib/msun/test-ctrig.c
new file mode 100644
index 000000000000..ed7866184304
--- /dev/null
+++ b/tools/regression/lib/msun/test-ctrig.c
@@ -0,0 +1,540 @@
+/*-
+ * Copyright (c) 2008-2011 David Schultz <das@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Tests for csin[h](), ccos[h](), and ctan[h]().
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <assert.h>
+#include <complex.h>
+#include <fenv.h>
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+
+#define	ALL_STD_EXCEPT	(FE_DIVBYZERO | FE_INEXACT | FE_INVALID | \
+			 FE_OVERFLOW | FE_UNDERFLOW)
+#define	OPT_INVALID	(ALL_STD_EXCEPT & ~FE_INVALID)
+#define	OPT_INEXACT	(ALL_STD_EXCEPT & ~FE_INEXACT)
+#define	FLT_ULP()	ldexpl(1.0, 1 - FLT_MANT_DIG)
+#define	DBL_ULP()	ldexpl(1.0, 1 - DBL_MANT_DIG)
+#define	LDBL_ULP()	ldexpl(1.0, 1 - LDBL_MANT_DIG)
+
+#pragma STDC FENV_ACCESS	ON
+#pragma	STDC CX_LIMITED_RANGE	OFF
+
+/*
+ * XXX gcc implements complex multiplication incorrectly. In
+ * particular, it implements it as if the CX_LIMITED_RANGE pragma
+ * were ON. Consequently, we need this function to form numbers
+ * such as x + INFINITY * I, since gcc evalutes INFINITY * I as
+ * NaN + INFINITY * I.
+ */
+static inline long double complex
+cpackl(long double x, long double y)
+{
+	long double complex z;
+
+	__real__ z = x;
+	__imag__ z = y;
+	return (z);
+}
+
+/* Flags that determine whether to check the signs of the result. */
+#define	CS_REAL	1
+#define	CS_IMAG	2
+#define	CS_BOTH	(CS_REAL | CS_IMAG)
+
+#ifdef	DEBUG
+#define	debug(...)	printf(__VA_ARGS__)
+#else
+#define	debug(...)	(void)0
+#endif
+
+/*
+ * Test that a function returns the correct value and sets the
+ * exception flags correctly. The exceptmask specifies which
+ * exceptions we should check. We need to be lenient for several
+ * reasons, but mainly because on some architectures it's impossible
+ * to raise FE_OVERFLOW without raising FE_INEXACT.
+ *
+ * These are macros instead of functions so that assert provides more
+ * meaningful error messages.
+ *
+ * XXX The volatile here is to avoid gcc's bogus constant folding and work
+ *     around the lack of support for the FENV_ACCESS pragma.
+ */
+#define	test_p(func, z, result, exceptmask, excepts, checksign)	do {	\
+	volatile long double complex _d = z;				\
+	debug("  testing %s(%Lg + %Lg I) == %Lg + %Lg I\n", #func,	\
+	    creall(_d), cimagl(_d), creall(result), cimagl(result));	\
+	assert(feclearexcept(FE_ALL_EXCEPT) == 0);			\
+	assert(cfpequal((func)(_d), (result), (checksign)));		\
+	assert(((func), fetestexcept(exceptmask) == (excepts)));	\
+} while (0)
+
+/*
+ * Test within a given tolerance.  The tolerance indicates relative error
+ * in ulps.  If result is 0, however, it measures absolute error in units
+ * of <format>_EPSILON.
+ */
+#define	test_p_tol(func, z, result, tol)			do {	\
+	volatile long double complex _d = z;				\
+	debug("  testing %s(%Lg + %Lg I) ~= %Lg + %Lg I\n", #func,	\
+	    creall(_d), cimagl(_d), creall(result), cimagl(result));	\
+	assert(cfpequal_tol((func)(_d), (result), (tol)));		\
+} while (0)
+
+/* These wrappers apply the identities f(conj(z)) = conj(f(z)). */
+#define	test(func, z, result, exceptmask, excepts, checksign)	do {	\
+	test_p(func, z, result, exceptmask, excepts, checksign);	\
+	test_p(func, conjl(z), conjl(result), exceptmask, excepts, checksign); \
+} while (0)
+#define	test_tol(func, z, result, tol)				do {	\
+	test_p_tol(func, z, result, tol);				\
+	test_p_tol(func, conjl(z), conjl(result), tol);			\
+} while (0)
+
+/* Test the given function in all precisions. */
+#define	testall(func, x, result, exceptmask, excepts, checksign) do {	\
+	test(func, x, result, exceptmask, excepts, checksign);		\
+	test(func##f, x, result, exceptmask, excepts, checksign);	\
+} while (0)
+#define	testall_odd(func, x, result, exceptmask, excepts, checksign) do { \
+	testall(func, x, result, exceptmask, excepts, checksign);	\
+	testall(func, -x, -result, exceptmask, excepts, checksign);	\
+} while (0)
+#define	testall_even(func, x, result, exceptmask, excepts, checksign) do { \
+	testall(func, x, result, exceptmask, excepts, checksign);	\
+	testall(func, -x, result, exceptmask, excepts, checksign);	\
+} while (0)
+
+/*
+ * Test the given function in all precisions, within a given tolerance.
+ * The tolerance is specified in ulps.
+ */
+#define	testall_tol(func, x, result, tol)	       		   do { \
+	test_tol(func, x, result, tol * DBL_ULP());			\
+	test_tol(func##f, x, result, tol * FLT_ULP());			\
+} while (0)
+#define	testall_odd_tol(func, x, result, tol)	       		   do { \
+	test_tol(func, x, result, tol * DBL_ULP());			\
+	test_tol(func, -x, -result, tol * DBL_ULP());			\
+} while (0)
+#define	testall_even_tol(func, x, result, tol)	       		   do { \
+	test_tol(func, x, result, tol * DBL_ULP());			\
+	test_tol(func, -x, result, tol * DBL_ULP());			\
+} while (0)
+
+/*
+ * Determine whether x and y are equal, with two special rules:
+ *	+0.0 != -0.0
+ *	 NaN == NaN
+ * If checksign is 0, we compare the absolute values instead.
+ */
+static int
+fpequal(long double x, long double y, int checksign)
+{
+	if (isnan(x) && isnan(y))
+		return (1);
+	if (checksign)
+		return (x == y && !signbit(x) == !signbit(y));
+	else
+		return (fabsl(x) == fabsl(y));
+}
+
+static int
+fpequal_tol(long double x, long double y, long double tol)
+{
+	fenv_t env;
+	int ret;
+
+	if (isnan(x) && isnan(y))
+		return (1);
+	if (!signbit(x) != !signbit(y) && tol == 0)
+		return (0);
+	if (x == y)
+		return (1);
+	if (tol == 0)
+		return (0);
+
+	/* Hard case: need to check the tolerance. */
+	feholdexcept(&env);
+	/*
+	 * For our purposes here, if y=0, we interpret tol as an absolute
+	 * tolerance. This is to account for roundoff in the input, e.g.,
+	 * cos(Pi/2) ~= 0.
+	 */
+	if (y == 0.0)
+		ret = fabsl(x - y) <= fabsl(tol);
+	else
+		ret = fabsl(x - y) <= fabsl(y * tol);
+	fesetenv(&env);
+	return (ret);
+}
+
+static int
+cfpequal(long double complex x, long double complex y, int checksign)
+{
+	return (fpequal(creal(x), creal(y), checksign & CS_REAL)
+		&& fpequal(cimag(x), cimag(y), checksign & CS_IMAG));
+}
+
+static int
+cfpequal_tol(long double complex x, long double complex y, long double tol)
+{
+	return (fpequal_tol(creal(x), creal(y), tol)
+		&& fpequal_tol(cimag(x), cimag(y), tol));
+}
+
+
+/* Tests for 0 */
+void
+test_zero(void)
+{
+	long double complex zero = cpackl(0.0, 0.0);
+
+	/* csinh(0) = ctanh(0) = 0; ccosh(0) = 1 (no exceptions raised) */
+	testall_odd(csinh, zero, zero, ALL_STD_EXCEPT, 0, CS_BOTH);
+	testall_odd(csin, zero, zero, ALL_STD_EXCEPT, 0, CS_BOTH);
+	testall_even(ccosh, zero, 1.0, ALL_STD_EXCEPT, 0, CS_BOTH);
+	testall_even(ccos, zero, cpackl(1.0, -0.0), ALL_STD_EXCEPT, 0, CS_BOTH);
+	testall_odd(ctanh, zero, zero, ALL_STD_EXCEPT, 0, CS_BOTH);
+	testall_odd(ctan, zero, zero, ALL_STD_EXCEPT, 0, CS_BOTH);
+}
+
+/*
+ * Tests for NaN inputs.
+ */
+void
+test_nan()
+{
+	long double complex nan_nan = cpackl(NAN, NAN);
+	long double complex z;
+
+	/*
+	 * IN		CSINH		CCOSH		CTANH
+	 * NaN,NaN	NaN,NaN		NaN,NaN		NaN,NaN
+	 * finite,NaN	NaN,NaN [inval]	NaN,NaN [inval]	NaN,NaN [inval]
+	 * NaN,finite	NaN,NaN [inval]	NaN,NaN [inval]	NaN,NaN [inval]
+	 * NaN,Inf	NaN,NaN [inval]	NaN,NaN	[inval]	NaN,NaN [inval]
+	 * Inf,NaN	+-Inf,NaN	Inf,NaN		1,+-0
+	 * 0,NaN	+-0,NaN		NaN,+-0		NaN,NaN	[inval]
+	 * NaN,0	NaN,0		NaN,+-0		NaN,0
+	 */
+	z = nan_nan;
+	testall_odd(csinh, z, nan_nan, ALL_STD_EXCEPT, 0, 0);
+	testall_even(ccosh, z, nan_nan, ALL_STD_EXCEPT, 0, 0);
+	testall_odd(ctanh, z, nan_nan, ALL_STD_EXCEPT, 0, 0);
+	testall_odd(csin, z, nan_nan, ALL_STD_EXCEPT, 0, 0);
+	testall_even(ccos, z, nan_nan, ALL_STD_EXCEPT, 0, 0);
+	testall_odd(ctan, z, nan_nan, ALL_STD_EXCEPT, 0, 0);
+
+	z = cpackl(42, NAN);
+	testall_odd(csinh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_even(ccosh, z, nan_nan, OPT_INVALID, 0, 0);
+	/* XXX We allow a spurious inexact exception here. */
+	testall_odd(ctanh, z, nan_nan, OPT_INVALID & ~FE_INEXACT, 0, 0);
+	testall_odd(csin, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_even(ccos, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(ctan, z, nan_nan, OPT_INVALID, 0, 0);
+
+	z = cpackl(NAN, 42);
+	testall_odd(csinh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_even(ccosh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(ctanh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(csin, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_even(ccos, z, nan_nan, OPT_INVALID, 0, 0);
+	/* XXX We allow a spurious inexact exception here. */
+	testall_odd(ctan, z, nan_nan, OPT_INVALID & ~FE_INEXACT, 0, 0);
+
+	z = cpackl(NAN, INFINITY);
+	testall_odd(csinh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_even(ccosh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(ctanh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(csin, z, cpackl(NAN, INFINITY), ALL_STD_EXCEPT, 0, 0);
+	testall_even(ccos, z, cpackl(INFINITY, NAN), ALL_STD_EXCEPT, 0,
+	    CS_IMAG);
+	testall_odd(ctan, z, cpackl(0, 1), ALL_STD_EXCEPT, 0, CS_IMAG);
+
+	z = cpackl(INFINITY, NAN);
+	testall_odd(csinh, z, cpackl(INFINITY, NAN), ALL_STD_EXCEPT, 0, 0);
+	testall_even(ccosh, z, cpackl(INFINITY, NAN), ALL_STD_EXCEPT, 0,
+		     CS_REAL);
+	testall_odd(ctanh, z, cpackl(1, 0), ALL_STD_EXCEPT, 0, CS_REAL);
+	testall_odd(csin, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_even(ccos, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(ctan, z, nan_nan, OPT_INVALID, 0, 0);
+
+	z = cpackl(0, NAN);
+	testall_odd(csinh, z, cpackl(0, NAN), ALL_STD_EXCEPT, 0, 0);
+	testall_even(ccosh, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, 0);
+	testall_odd(ctanh, z, nan_nan, OPT_INVALID, 0, 0);
+	testall_odd(csin, z, cpackl(0, NAN), ALL_STD_EXCEPT, 0, CS_REAL);
+	testall_even(ccos, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, 0);
+	testall_odd(ctan, z, cpackl(0, NAN), ALL_STD_EXCEPT, 0, CS_REAL);
+
+	z = cpackl(NAN, 0);
+	testall_odd(csinh, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, CS_IMAG);
+	testall_even(ccosh, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, 0);
+	testall_odd(ctanh, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, CS_IMAG);
+	testall_odd(csin, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, 0);
+	testall_even(ccos, z, cpackl(NAN, 0), ALL_STD_EXCEPT, 0, 0);
+	testall_odd(ctan, z, nan_nan, OPT_INVALID, 0, 0);
+}
+
+void
+test_inf(void)
+{
+	static const long double finites[] = {
+	    0, M_PI / 4, 3 * M_PI / 4, 5 * M_PI / 4,
+	};
+	long double complex z, c, s;
+	int i;
+
+	/*
+	 * IN		CSINH		CCOSH		CTANH
+	 * Inf,Inf	+-Inf,NaN inval	+-Inf,NaN inval	1,+-0
+	 * Inf,finite	Inf cis(finite)	Inf cis(finite)	1,0 sin(2 finite)
+	 * 0,Inf	+-0,NaN	inval	NaN,+-0 inval	NaN,NaN	inval
+	 * finite,Inf	NaN,NaN inval	NaN,NaN inval	NaN,NaN inval
+	 */
+	z = cpackl(INFINITY, INFINITY);
+	testall_odd(csinh, z, cpackl(INFINITY, NAN),
+		    ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_even(ccosh, z, cpackl(INFINITY, NAN),
+		     ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_odd(ctanh, z, cpackl(1, 0), ALL_STD_EXCEPT, 0, CS_REAL);
+	testall_odd(csin, z, cpackl(NAN, INFINITY),
+		    ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_even(ccos, z, cpackl(INFINITY, NAN),
+		     ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_odd(ctan, z, cpackl(0, 1), ALL_STD_EXCEPT, 0, CS_REAL);
+
+	/* XXX We allow spurious inexact exceptions here (hard to avoid). */
+	for (i = 0; i < sizeof(finites) / sizeof(finites[0]); i++) {
+		z = cpackl(INFINITY, finites[i]);
+		c = INFINITY * cosl(finites[i]);
+		s = finites[i] == 0 ? finites[i] : INFINITY * sinl(finites[i]);
+		testall_odd(csinh, z, cpackl(c, s), OPT_INEXACT, 0, CS_BOTH);
+		testall_even(ccosh, z, cpackl(c, s), OPT_INEXACT, 0, CS_BOTH);
+		testall_odd(ctanh, z, cpackl(1, 0 * sin(finites[i] * 2)),
+			    OPT_INEXACT, 0, CS_BOTH);
+		z = cpackl(finites[i], INFINITY);
+		testall_odd(csin, z, cpackl(s, c), OPT_INEXACT, 0, CS_BOTH);
+		testall_even(ccos, z, cpackl(c, -s), OPT_INEXACT, 0, CS_BOTH);
+		testall_odd(ctan, z, cpackl(0 * sin(finites[i] * 2), 1),
+			    OPT_INEXACT, 0, CS_BOTH);
+	}
+
+	z = cpackl(0, INFINITY);
+	testall_odd(csinh, z, cpackl(0, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_even(ccosh, z, cpackl(NAN, 0), ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_odd(ctanh, z, cpackl(NAN, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+	z = cpackl(INFINITY, 0);
+	testall_odd(csin, z, cpackl(NAN, 0), ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_even(ccos, z, cpackl(NAN, 0), ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_odd(ctan, z, cpackl(NAN, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+
+	z = cpackl(42, INFINITY);
+	testall_odd(csinh, z, cpackl(NAN, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_even(ccosh, z, cpackl(NAN, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+	/* XXX We allow a spurious inexact exception here. */
+	testall_odd(ctanh, z, cpackl(NAN, NAN), OPT_INEXACT, FE_INVALID, 0);
+	z = cpackl(INFINITY, 42);
+	testall_odd(csin, z, cpackl(NAN, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+	testall_even(ccos, z, cpackl(NAN, NAN), ALL_STD_EXCEPT, FE_INVALID, 0);
+	/* XXX We allow a spurious inexact exception here. */
+	testall_odd(ctan, z, cpackl(NAN, NAN), OPT_INEXACT, FE_INVALID, 0);
+}
+
+/* Tests along the real and imaginary axes. */
+void
+test_axes(void)
+{
+	static const long double nums[] = {
+	    M_PI / 4, M_PI / 2, 3 * M_PI / 4,
+	    5 * M_PI / 4, 3 * M_PI / 2, 7 * M_PI / 4,
+	};
+	long double complex z;
+	int i;
+
+	for (i = 0; i < sizeof(nums) / sizeof(nums[0]); i++) {
+		/* Real axis */
+		z = cpackl(nums[i], 0.0);
+		testall_odd_tol(csinh, z, cpackl(sinh(nums[i]), 0), 0);
+		testall_even_tol(ccosh, z, cpackl(cosh(nums[i]), 0), 0);
+		testall_odd_tol(ctanh, z, cpackl(tanh(nums[i]), 0), 1);
+		testall_odd_tol(csin, z, cpackl(sin(nums[i]),
+					    copysign(0, cos(nums[i]))), 0);
+		testall_even_tol(ccos, z, cpackl(cos(nums[i]),
+		    -copysign(0, sin(nums[i]))), 0);
+		testall_odd_tol(ctan, z, cpackl(tan(nums[i]), 0), 1);
+
+		/* Imaginary axis */
+		z = cpackl(0.0, nums[i]);
+		testall_odd_tol(csinh, z, cpackl(copysign(0, cos(nums[i])),
+						 sin(nums[i])), 0);
+		testall_even_tol(ccosh, z, cpackl(cos(nums[i]),
+		    copysign(0, sin(nums[i]))), 0);
+		testall_odd_tol(ctanh, z, cpackl(0, tan(nums[i])), 1);
+		testall_odd_tol(csin, z, cpackl(0, sinh(nums[i])), 0);
+		testall_even_tol(ccos, z, cpackl(cosh(nums[i]), -0.0), 0);
+		testall_odd_tol(ctan, z, cpackl(0, tanh(nums[i])), 1);
+	}
+}
+
+void
+test_small(void)
+{
+	/*
+	 * z =  0.5 + i Pi/4
+	 *     sinh(z) = (sinh(0.5) + i cosh(0.5)) * sqrt(2)/2
+	 *     cosh(z) = (cosh(0.5) + i sinh(0.5)) * sqrt(2)/2
+	 *     tanh(z) = (2cosh(0.5)sinh(0.5) + i) / (2 cosh(0.5)**2 - 1)
+	 * z = -0.5 + i Pi/2
+	 *     sinh(z) = cosh(0.5)
+	 *     cosh(z) = -i sinh(0.5)
+	 *     tanh(z) = -coth(0.5)
+	 * z =  1.0 + i 3Pi/4
+	 *     sinh(z) = (-sinh(1) + i cosh(1)) * sqrt(2)/2
+	 *     cosh(z) = (-cosh(1) + i sinh(1)) * sqrt(2)/2
+	 *     tanh(z) = (2cosh(1)sinh(1) - i) / (2cosh(1)**2 - 1)
+	 */
+	static const struct {
+		long double a, b;
+		long double sinh_a, sinh_b;
+		long double cosh_a, cosh_b;
+		long double tanh_a, tanh_b;
+	} tests[] = {
+		{  0.5L,
+		   0.78539816339744830961566084581987572L,
+		   0.36847002415910435172083660522240710L,
+		   0.79735196663945774996093142586179334L,
+		   0.79735196663945774996093142586179334L,
+		   0.36847002415910435172083660522240710L,
+		   0.76159415595576488811945828260479359L,
+		   0.64805427366388539957497735322615032L },
+		{ -0.5L,
+		   1.57079632679489661923132169163975144L,
+		   0.0L,
+		   1.12762596520638078522622516140267201L,
+		   0.0L,
+		  -0.52109530549374736162242562641149156L,
+		  -2.16395341373865284877000401021802312L,
+		   0.0L },
+		{  1.0L,
+		   2.35619449019234492884698253745962716L,
+		  -0.83099273328405698212637979852748608L,
+		   1.09112278079550143030545602018565236L,
+		  -1.09112278079550143030545602018565236L,
+		   0.83099273328405698212637979852748609L,
+		   0.96402758007581688394641372410092315L,
+		  -0.26580222883407969212086273981988897L }
+	};
+	long double complex z;
+	int i;
+
+	for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) {
+		z = cpackl(tests[i].a, tests[i].b);
+		testall_odd_tol(csinh, z,
+		    cpackl(tests[i].sinh_a, tests[i].sinh_b), 1.1);
+		testall_even_tol(ccosh, z,
+		    cpackl(tests[i].cosh_a, tests[i].cosh_b), 1.1);
+		testall_odd_tol(ctanh, z,
+		    cpackl(tests[i].tanh_a, tests[i].tanh_b), 1.1);
+        }
+}
+
+/* Test inputs that might cause overflow in a sloppy implementation. */
+void
+test_large(void)
+{
+	long double complex z;
+
+	/* tanh() uses a threshold around x=22, so check both sides. */
+	z = cpackl(21, 0.78539816339744830961566084581987572L);
+	testall_odd_tol(ctanh, z,
+	    cpackl(1.0, 1.14990445285871196133287617611468468e-18L), 1);
+	z++;
+	testall_odd_tol(ctanh, z,
+	    cpackl(1.0, 1.55622644822675930314266334585597964e-19L), 1);
+
+	z = cpackl(355, 0.78539816339744830961566084581987572L);
+	testall_odd_tol(ctanh, z,
+	    cpackl(1.0, 8.95257245135025991216632140458264468e-309L), 1);
+	z = cpackl(30, 0x1p1023L);
+	testall_odd_tol(ctanh, z,
+	    cpackl(1.0, -1.62994325413993477997492170229268382e-26L), 1);
+	z = cpackl(1, 0x1p1023L);
+	testall_odd_tol(ctanh, z,
+	    cpackl(0.878606311888306869546254022621986509L,
+		   -0.225462792499754505792678258169527424L), 1);
+
+	z = cpackl(710.6, 0.78539816339744830961566084581987572L);
+	testall_odd_tol(csinh, z,
+	    cpackl(1.43917579766621073533185387499658944e308L,
+		   1.43917579766621073533185387499658944e308L), 1);
+	testall_even_tol(ccosh, z,
+	    cpackl(1.43917579766621073533185387499658944e308L,
+		   1.43917579766621073533185387499658944e308L), 1);
+
+	z = cpackl(1500, 0.78539816339744830961566084581987572L);
+	testall_odd(csinh, z, cpackl(INFINITY, INFINITY), OPT_INEXACT,
+	    FE_OVERFLOW, CS_BOTH);
+	testall_even(ccosh, z, cpackl(INFINITY, INFINITY), OPT_INEXACT,
+	    FE_OVERFLOW, CS_BOTH);
+}
+
+int
+main(int argc, char *argv[])
+{
+
+	printf("1..6\n");
+
+	test_zero();
+	printf("ok 1 - ctrig zero\n");
+
+	test_nan();
+	printf("ok 2 - ctrig nan\n");
+
+	test_inf();
+	printf("ok 3 - ctrig inf\n");
+
+	test_axes();
+	printf("ok 4 - ctrig axes\n");
+
+	test_small();
+	printf("ok 5 - ctrig small\n");
+
+	test_large();
+	printf("ok 6 - ctrig large\n");
+
+	return (0);
+}
diff --git a/tools/regression/lib/msun/test-ctrig.t b/tools/regression/lib/msun/test-ctrig.t
new file mode 100644
index 000000000000..8bdfd03be81b
--- /dev/null
+++ b/tools/regression/lib/msun/test-ctrig.t
@@ -0,0 +1,10 @@
+#!/bin/sh
+# $FreeBSD$
+
+cd `dirname $0`
+
+executable=`basename $0 .t`
+
+make $executable 2>&1 > /dev/null
+
+exec ./$executable
diff --git a/tools/regression/lib/msun/test-fma.c b/tools/regression/lib/msun/test-fma.c
index 00620f85941d..1237a6021778 100644
--- a/tools/regression/lib/msun/test-fma.c
+++ b/tools/regression/lib/msun/test-fma.c
@@ -362,6 +362,116 @@ test_accuracy(void)
 		0x1.d87da3aafda40p70L, 0x1.d87da3aafda3fp70L,
 		0x1.d87da3aafda3fp70L, ALL_STD_EXCEPT, FE_INEXACT);
 #endif
+
+	/* ilogb(x*y) - ilogb(z) = 0 */
+	testrnd(fmaf, 0x1.31ad02p+100, 0x1.2fbf7ap-42, -0x1.c3e106p+58,
+		-0x1.64c27cp+56, -0x1.64c27ap+56, -0x1.64c27cp+56,
+		-0x1.64c27ap+56, ALL_STD_EXCEPT, FE_INEXACT);
+	testrnd(fma, 0x1.31ad012ede8aap+100, 0x1.2fbf79c839067p-42,
+		-0x1.c3e106929056ep+58, -0x1.64c282b970a5fp+56,
+		-0x1.64c282b970a5ep+56, -0x1.64c282b970a5fp+56,
+		-0x1.64c282b970a5ep+56, ALL_STD_EXCEPT, FE_INEXACT);
+#if LDBL_MANT_DIG == 113
+	testrnd(fmal, 0x1.31ad012ede8aa282fa1c19376d16p+100L,
+		 0x1.2fbf79c839066f0f5c68f6d2e814p-42L,
+		-0x1.c3e106929056ec19de72bfe64215p+58L,
+		-0x1.64c282b970a612598fc025ca8cddp+56L,
+		-0x1.64c282b970a612598fc025ca8cddp+56L,
+		-0x1.64c282b970a612598fc025ca8cdep+56L,
+		-0x1.64c282b970a612598fc025ca8cddp+56L,
+		ALL_STD_EXCEPT, FE_INEXACT);
+#elif LDBL_MANT_DIG == 64
+	testrnd(fmal, 0x1.31ad012ede8aa4eap+100L, 0x1.2fbf79c839066aeap-42L,
+		-0x1.c3e106929056e61p+58L, -0x1.64c282b970a60298p+56L,
+		-0x1.64c282b970a60298p+56L, -0x1.64c282b970a6029ap+56L,
+		-0x1.64c282b970a60298p+56L, ALL_STD_EXCEPT, FE_INEXACT);
+#elif LDBL_MANT_DIG == 53
+	testrnd(fmal, 0x1.31ad012ede8aap+100L, 0x1.2fbf79c839067p-42L,
+		-0x1.c3e106929056ep+58L, -0x1.64c282b970a5fp+56L,
+		-0x1.64c282b970a5ep+56L, -0x1.64c282b970a5fp+56L,
+		-0x1.64c282b970a5ep+56L, ALL_STD_EXCEPT, FE_INEXACT);
+#endif
+
+	/* x*y (rounded) ~= -z */
+	/* XXX spurious inexact exceptions */
+	testrnd(fmaf, 0x1.bbffeep-30, -0x1.1d164cp-74, 0x1.ee7296p-104,
+		-0x1.c46ea8p-128, -0x1.c46ea8p-128, -0x1.c46ea8p-128,
+		-0x1.c46ea8p-128, ALL_STD_EXCEPT & ~FE_INEXACT, 0);
+	testrnd(fma, 0x1.bbffeea6fc7d6p-30, 0x1.1d164c6cbf078p-74,
+		-0x1.ee72993aff948p-104, -0x1.71f72ac7d9d8p-159,
+		-0x1.71f72ac7d9d8p-159, -0x1.71f72ac7d9d8p-159,
+		-0x1.71f72ac7d9d8p-159, ALL_STD_EXCEPT & ~FE_INEXACT, 0);
+#if LDBL_MANT_DIG == 113
+	testrnd(fmal, 0x1.bbffeea6fc7d65927d147f437675p-30L,
+		0x1.1d164c6cbf078b7a22607d1cd6a2p-74L,
+		-0x1.ee72993aff94973876031bec0944p-104L,
+		0x1.64e086175b3a2adc36e607058814p-217L,
+		0x1.64e086175b3a2adc36e607058814p-217L,
+		0x1.64e086175b3a2adc36e607058814p-217L,
+		0x1.64e086175b3a2adc36e607058814p-217L,
+		ALL_STD_EXCEPT & ~FE_INEXACT, 0);
+#elif LDBL_MANT_DIG == 64
+	testrnd(fmal, 0x1.bbffeea6fc7d6592p-30L, 0x1.1d164c6cbf078b7ap-74L,
+		-0x1.ee72993aff949736p-104L, 0x1.af190e7a1ee6ad94p-168L,
+		0x1.af190e7a1ee6ad94p-168L, 0x1.af190e7a1ee6ad94p-168L,
+		0x1.af190e7a1ee6ad94p-168L, ALL_STD_EXCEPT & ~FE_INEXACT, 0);
+#elif LDBL_MANT_DIG == 53
+	testrnd(fmal, 0x1.bbffeea6fc7d6p-30L, 0x1.1d164c6cbf078p-74L,
+		-0x1.ee72993aff948p-104L, -0x1.71f72ac7d9d8p-159L,
+		-0x1.71f72ac7d9d8p-159L, -0x1.71f72ac7d9d8p-159L,
+		-0x1.71f72ac7d9d8p-159L, ALL_STD_EXCEPT & ~FE_INEXACT, 0);
+#endif
+}
+
+static void
+test_double_rounding(void)
+{
+
+	/*
+	 *     a =  0x1.8000000000001p0
+	 *     b =  0x1.8000000000001p0
+	 *     c = -0x0.0000000000000000000000000080...1p+1
+	 * a * b =  0x1.2000000000001800000000000080p+1
+	 *
+	 * The correct behavior is to round DOWN to 0x1.2000000000001p+1 in
+	 * round-to-nearest mode.  An implementation that computes a*b+c in
+	 * double+double precision, however, will get 0x1.20000000000018p+1,
+	 * and then round UP.
+	 */
+	fesetround(FE_TONEAREST);
+	test(fma, 0x1.8000000000001p0, 0x1.8000000000001p0,
+	     -0x1.0000000000001p-104, 0x1.2000000000001p+1,
+	     ALL_STD_EXCEPT, FE_INEXACT);
+	fesetround(FE_DOWNWARD);
+	test(fma, 0x1.8000000000001p0, 0x1.8000000000001p0,
+	     -0x1.0000000000001p-104, 0x1.2000000000001p+1,
+	     ALL_STD_EXCEPT, FE_INEXACT);
+	fesetround(FE_UPWARD);
+	test(fma, 0x1.8000000000001p0, 0x1.8000000000001p0,
+	     -0x1.0000000000001p-104, 0x1.2000000000002p+1,
+	     ALL_STD_EXCEPT, FE_INEXACT);
+
+	fesetround(FE_TONEAREST);
+	test(fmaf, 0x1.800002p+0, 0x1.800002p+0, -0x1.000002p-46, 0x1.200002p+1,
+	     ALL_STD_EXCEPT, FE_INEXACT);
+	fesetround(FE_DOWNWARD);
+	test(fmaf, 0x1.800002p+0, 0x1.800002p+0, -0x1.000002p-46, 0x1.200002p+1,
+	     ALL_STD_EXCEPT, FE_INEXACT);
+	fesetround(FE_UPWARD);
+	test(fmaf, 0x1.800002p+0, 0x1.800002p+0, -0x1.000002p-46, 0x1.200004p+1,
+	     ALL_STD_EXCEPT, FE_INEXACT);
+
+	fesetround(FE_TONEAREST);
+#if LDBL_MANT_DIG == 64
+	test(fmal, 0x1.4p+0L, 0x1.0000000000000004p+0L, 0x1p-128L,
+	     0x1.4000000000000006p+0L, ALL_STD_EXCEPT, FE_INEXACT);
+#elif LDBL_MANT_DIG == 113
+	test(fmal, 0x1.8000000000000000000000000001p+0L,
+	     0x1.8000000000000000000000000001p+0L,
+	     -0x1.0000000000000000000000000001p-224L,
+	     0x1.2000000000000000000000000001p+1L, ALL_STD_EXCEPT, FE_INEXACT);
+#endif
+
 }
 
 int
@@ -370,7 +480,7 @@ main(int argc, char *argv[])
 	int rmodes[] = { FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO };
 	int i;
 
-	printf("1..18\n");
+	printf("1..19\n");
 
 	for (i = 0; i < 4; i++) {
 		fesetround(rmodes[i]);
@@ -404,6 +514,9 @@ main(int argc, char *argv[])
 	test_accuracy();
 	printf("ok 18 - fma accuracy\n");
 
+	test_double_rounding();
+	printf("ok 19 - fma double rounding\n");
+
 	/*
 	 * TODO:
 	 * - Tests for subnormals
diff --git a/tools/regression/lib/msun/test-logarithm.c b/tools/regression/lib/msun/test-logarithm.c
index 52f562c5fdb5..258c5141305e 100644
--- a/tools/regression/lib/msun/test-logarithm.c
+++ b/tools/regression/lib/msun/test-logarithm.c
@@ -97,7 +97,7 @@ void
 run_generic_tests(void)
 {
 
-	/* exp(1) == 0, no exceptions raised */
+	/* log(1) == 0, no exceptions raised */
 	testall0(1.0, 0.0, ALL_STD_EXCEPT, 0);
 	testall1(0.0, 0.0, ALL_STD_EXCEPT, 0);
 	testall1(-0.0, -0.0, ALL_STD_EXCEPT, 0);
@@ -142,11 +142,35 @@ run_log2_tests(void)
 	}
 }
 
+void
+run_roundingmode_tests(void)
+{
+
+	/*
+	 * Corner cases in other rounding modes.
+	 */
+	fesetround(FE_DOWNWARD);
+	/* These are still positive per IEEE 754R */
+	testall0(1.0, 0.0, ALL_STD_EXCEPT, 0);
+	testall1(0.0, 0.0, ALL_STD_EXCEPT, 0);
+	fesetround(FE_TOWARDZERO);
+	testall0(1.0, 0.0, ALL_STD_EXCEPT, 0);
+	testall1(0.0, 0.0, ALL_STD_EXCEPT, 0);
+
+	fesetround(FE_UPWARD);
+	testall0(1.0, 0.0, ALL_STD_EXCEPT, 0);
+	testall1(0.0, 0.0, ALL_STD_EXCEPT, 0);
+	/* log1p(-0.0) == -0.0 even when rounding upwards */
+	testall1(-0.0, -0.0, ALL_STD_EXCEPT, 0);
+
+	fesetround(FE_TONEAREST);
+}
+
 int
 main(int argc, char *argv[])
 {
 
-	printf("1..2\n");
+	printf("1..3\n");
 
 	run_generic_tests();
 	printf("ok 1 - logarithm\n");
@@ -154,5 +178,8 @@ main(int argc, char *argv[])
 	run_log2_tests();
 	printf("ok 2 - logarithm\n");
 
+	run_roundingmode_tests();
+	printf("ok 3 - logarithm\n");
+
 	return (0);
 }
diff --git a/tools/regression/lib/msun/test-nearbyint.c b/tools/regression/lib/msun/test-nearbyint.c
index 630ddb987911..7251acb6e0e5 100644
--- a/tools/regression/lib/msun/test-nearbyint.c
+++ b/tools/regression/lib/msun/test-nearbyint.c
@@ -30,7 +30,6 @@
  * TODO:
  * - adapt tests for rint(3)
  * - tests for harder values (more mantissa bits than float)
- * - tests in other rounding modes
  */
 
 #include <sys/cdefs.h>
@@ -44,6 +43,27 @@ __FBSDID("$FreeBSD$");
 #define	ALL_STD_EXCEPT	(FE_DIVBYZERO | FE_INEXACT | FE_INVALID | \
 			 FE_OVERFLOW | FE_UNDERFLOW)
 
+static int testnum;
+
+static const int rmodes[] = {
+	FE_TONEAREST, FE_DOWNWARD, FE_UPWARD, FE_TOWARDZERO,
+};
+
+static const struct {
+	float in;
+	float out[3];	/* one answer per rounding mode except towardzero */
+} tests[] = {
+/* input	output (expected) */
+    { 0.0,	{ 0.0, 0.0, 0.0 }},
+    { 0.5,	{ 0.0, 0.0, 1.0 }},
+    { M_PI,	{ 3.0, 3.0, 4.0 }},
+    { 65536.5,	{ 65536, 65536, 65537 }},
+    { INFINITY,	{ INFINITY, INFINITY, INFINITY }},
+    { NAN,	{ NAN, NAN, NAN }},
+};
+
+static const int ntests = sizeof(tests) / sizeof(tests[0]);
+
 /*
  * Compare d1 and d2 using special rules: NaN == NaN and +0 != -0.
  * Fail an assertion if they differ.
@@ -57,44 +77,106 @@ fpequal(long double d1, long double d2)
 	return (copysignl(1.0, d1) == copysignl(1.0, d2));
 }
 
-static void testit(int testnum, float in, float out)
+/* Get the appropriate result for the current rounding mode. */
+static float
+get_output(int testindex, int rmodeindex, int negative)
 {
+	double out;
 
-    feclearexcept(ALL_STD_EXCEPT);
-    assert(fpequal(out, nearbyintf(in)));
-    assert(fpequal(-out, nearbyintf(-in)));
-    assert(fetestexcept(ALL_STD_EXCEPT) == 0);
-
-    assert(fpequal(out, nearbyint(in)));
-    assert(fpequal(-out, nearbyint(-in)));
-    assert(fetestexcept(ALL_STD_EXCEPT) == 0);
-
-    assert(fpequal(out, nearbyintl(in)));
-    assert(fpequal(-out, nearbyintl(-in)));
-    assert(fetestexcept(ALL_STD_EXCEPT) == 0);
-
-    printf("ok %d\t\t# nearbyint(%g)\n", testnum, in);
+	if (negative) {	/* swap downwards and upwards if input is negative */
+		if (rmodeindex == 1)
+			rmodeindex = 2;
+		else if (rmodeindex == 2)
+			rmodeindex = 1;
+	}
+	if (rmodeindex == 3) /* FE_TOWARDZERO uses the value for downwards */
+		rmodeindex = 1;
+	out = tests[testindex].out[rmodeindex];
+	return (negative ? -out : out);
 }
 
-static const float tests[] = {
-/* input	output (expected) */
-    0.0,	0.0,
-    0.5,	0.0,
-    M_PI,	3,
-    65536.5,	65536,
-    INFINITY,	INFINITY,
-    NAN,	NAN,
-};
+static void
+test_nearby(int testindex)
+{
+	float in, out;
+	int i;
+
+	for (i = 0; i < sizeof(rmodes) / sizeof(rmodes[0]); i++) {
+		fesetround(rmodes[i]);
+		feclearexcept(ALL_STD_EXCEPT);
+
+		in = tests[testindex].in;
+		out = get_output(testindex, i, 0);
+		assert(fpequal(out, nearbyintf(in)));
+		assert(fpequal(out, nearbyint(in)));
+		assert(fpequal(out, nearbyintl(in)));
+		assert(fetestexcept(ALL_STD_EXCEPT) == 0);
+
+		in = -tests[testindex].in;
+		out = get_output(testindex, i, 1);
+		assert(fpequal(out, nearbyintf(in)));
+		assert(fpequal(out, nearbyint(in)));
+		assert(fpequal(out, nearbyintl(in)));
+		assert(fetestexcept(ALL_STD_EXCEPT) == 0);
+	}
+
+	printf("ok %d\t\t# nearbyint(+%g)\n", testnum++, in);
+}
+
+static void
+test_modf(int testindex)
+{
+	float in, out;
+	float ipartf, ipart_expected;
+	double ipart;
+	long double ipartl;
+	int i;
+
+	for (i = 0; i < sizeof(rmodes) / sizeof(rmodes[0]); i++) {
+		fesetround(rmodes[i]);
+		feclearexcept(ALL_STD_EXCEPT);
+
+		in = tests[testindex].in;
+		ipart_expected = tests[testindex].out[1];
+		out = copysignf(
+		    isinf(ipart_expected) ? 0.0 : in - ipart_expected, in);
+		ipartl = ipart = ipartf = 42.0;
+
+		assert(fpequal(out, modff(in, &ipartf)));
+		assert(fpequal(ipart_expected, ipartf));
+		assert(fpequal(out, modf(in, &ipart)));
+		assert(fpequal(ipart_expected, ipart));
+		assert(fpequal(out, modfl(in, &ipartl)));
+		assert(fpequal(ipart_expected, ipartl));
+		assert(fetestexcept(ALL_STD_EXCEPT) == 0);
+
+		in = -in;
+		ipart_expected = -ipart_expected;
+		out = -out;
+		ipartl = ipart = ipartf = 42.0;
+		assert(fpequal(out, modff(in, &ipartf)));
+		assert(fpequal(ipart_expected, ipartf));
+		assert(fpequal(out, modf(in, &ipart)));
+		assert(fpequal(ipart_expected, ipart));
+		assert(fpequal(out, modfl(in, &ipartl)));
+		assert(fpequal(ipart_expected, ipartl));
+		assert(fetestexcept(ALL_STD_EXCEPT) == 0);
+	}
+
+	printf("ok %d\t\t# modf(+%g)\n", testnum++, in);
+}
 
 int
 main(int argc, char *argv[])
 {
-	static const int ntests = sizeof(tests) / sizeof(tests[0]) / 2;
 	int i;
 
-	printf("1..%d\n", ntests);
-	for (i = 0; i < ntests; i++)
-		testit(i + 1, tests[i * 2], tests[i * 2 + 1]);
+	printf("1..%d\n", ntests * 2);
+	testnum = 1;
+	for (i = 0; i < ntests; i++) {
+		test_nearby(i);
+		test_modf(i);
+	}
 
 	return (0);
 }
diff --git a/tools/tools/ath/athstats/athstats.c b/tools/tools/ath/athstats/athstats.c
index 2c9f131722c0..c57b2f3db07d 100644
--- a/tools/tools/ath/athstats/athstats.c
+++ b/tools/tools/ath/athstats/athstats.c
@@ -49,18 +49,10 @@
 #include <unistd.h>
 #include <err.h>
 
-/* Use the system net80211 headers, rather than the kernel tree */
-/*
- * XXX this means that if you build a separate net80211 stack
- * XXX with your kernel and don't install the new/changed headers,
- * XXX this tool may break.
- * XXX -adrian
- */
-#include <net80211/ieee80211_ioctl.h>
-#include <net80211/ieee80211_radiotap.h>
-
 #include "ah.h"
 #include "ah_desc.h"
+#include "net80211/ieee80211_ioctl.h"
+#include "net80211/ieee80211_radiotap.h"
 #include "if_athioctl.h"
 
 #include "athstats.h"
@@ -256,8 +248,46 @@ static const struct fmt athstats[] = {
 	{ 5,	"txrawfail",	"txrawfail",	"raw tx failed 'cuz interface/hw down" },
 #define	S_RX_TOOBIG	AFTER(S_TX_RAW_FAIL)
 	{ 5,	"rx2big",	"rx2big",	"rx failed 'cuz frame too large"  },
+#define	S_RX_AGG	AFTER(S_RX_TOOBIG)
+	{ 5,	"rxagg",	"rxagg",	"A-MPDU sub-frames received" },
+#define	S_RX_HALFGI	AFTER(S_RX_AGG)
+	{ 5,	"rxhalfgi",	"rxhgi",	"Half-GI frames received" },
+#define	S_RX_2040	AFTER(S_RX_HALFGI)
+	{ 6,	"rx2040",	"rx2040",	"40MHz frames received" },
+#define	S_RX_PRE_CRC_ERR	AFTER(S_RX_2040)
+	{ 11,	"rxprecrcerr",	"rxprecrcerr",	"CRC errors for non-last A-MPDU subframes" },
+#define	S_RX_POST_CRC_ERR	AFTER(S_RX_PRE_CRC_ERR)
+	{ 12,	"rxpostcrcerr",	"rxpostcrcerr",	"CRC errors for last subframe in an A-MPDU" },
+#define	S_RX_DECRYPT_BUSY_ERR	AFTER(S_RX_POST_CRC_ERR)
+	{ 10,	"rxdescbusy",	"rxdescbusy",	"Decryption engine busy" },
+#define	S_RX_HI_CHAIN	AFTER(S_RX_DECRYPT_BUSY_ERR)
+	{ 4,	"rxhi",	"rxhi",	"Frames received with RX chain in high power mode" },
+#define	S_TX_HTPROTECT	AFTER(S_RX_HI_CHAIN)
+	{ 7,	"txhtprot",	"txhtprot",	"Frames transmitted with HT Protection" },
+#define	S_RX_QEND	AFTER(S_TX_HTPROTECT)
+	{ 7,	"rxquend",	"rxquend",	"Hit end of RX descriptor queue" },
+#define	S_TX_TIMEOUT	AFTER(S_RX_QEND)
+	{ 4,	"txtimeout",	"TXTX",	"TX Timeout" },
+#define	S_TX_CSTIMEOUT	AFTER(S_TX_TIMEOUT)
+	{ 4,	"csttimeout",	"CSTX",	"Carrier Sense Timeout" },
+#define	S_TX_XTXOP_ERR	AFTER(S_TX_CSTIMEOUT)
+	{ 5,	"xtxoperr",	"TXOPX",	"TXOP exceed" },
+#define	S_TX_TIMEREXPIRED_ERR	AFTER(S_TX_XTXOP_ERR)
+	{ 7,	"texperr",	"texperr",	"TX Timer expired" },
+#define	S_TX_DESCCFG_ERR	AFTER(S_TX_TIMEREXPIRED_ERR)
+	{ 10,	"desccfgerr",	"desccfgerr",	"TX descriptor error" },
+#define	S_TX_SWRETRIES	AFTER(S_TX_DESCCFG_ERR)
+	{ 9,	"txswretry",	"txswretry",	"Number of frames retransmitted in software" },
+#define	S_TX_SWRETRIES_MAX	AFTER(S_TX_SWRETRIES)
+	{ 7,	"txswmax",	"txswmax",	"Number of frames exceeding software retry" },
+#define	S_TX_DATA_UNDERRUN	AFTER(S_TX_SWRETRIES_MAX)
+	{ 5,	"txdataunderrun",	"TXDAU",	"A-MPDU TX FIFO data underrun" },
+#define	S_TX_DELIM_UNDERRUN	AFTER(S_TX_DATA_UNDERRUN)
+	{ 5,	"txdelimunderrun",	"TXDEU",	"A-MPDU TX Delimiter underrun" },
+#define	S_TX_AGGR_FAIL		AFTER(S_TX_DELIM_UNDERRUN)
+	{ 10,	"txaggrfail",	"txaggrfail",	"A-MPDU TX attempt failed" },
 #ifndef __linux__
-#define	S_CABQ_XMIT	AFTER(S_RX_TOOBIG)
+#define	S_CABQ_XMIT	AFTER(S_TX_AGGR_FAIL)
 	{ 5,	"cabxmit",	"cabxmit",	"cabq frames transmitted" },
 #define	S_CABQ_BUSY	AFTER(S_CABQ_XMIT)
 	{ 5,	"cabqbusy",	"cabqbusy",	"cabq xmit overflowed beacon interval" },
@@ -269,7 +299,7 @@ static const struct fmt athstats[] = {
 	{ 5,	"rxbusdma",	"rxbusdma",	"rx setup failed for dma resrcs" },
 #define	S_FF_TXOK	AFTER(S_RX_BUSDMA)
 #else
-#define	S_FF_TXOK	AFTER(S_RX_PHY_UNDERRUN)
+#define	S_FF_TXOK	AFTER(S_TX_AGGR_FAIL)
 #endif
 	{ 5,	"fftxok",	"fftxok",	"fast frames xmit successfully" },
 #define	S_FF_TXERR	AFTER(S_FF_TXOK)
@@ -384,12 +414,16 @@ static const struct fmt athstats[] = {
 	{ 4,	"asignal",	"asig",	"signal of last ack (dBm)" },
 #define	S_RX_SIGNAL	AFTER(S_TX_SIGNAL)
 	{ 4,	"signal",	"sig",	"avg recv signal (dBm)" },
+
 };
 #define	S_PHY_MIN	S_RX_PHY_UNDERRUN
 #define	S_PHY_MAX	S_RX_PHY_CCK_RESTART
 #define	S_LAST		S_ANT_TX0
 #define	S_MAX	S_ANT_RX7+1
 
+/*
+ * XXX fold this into the external HAL definitions! -adrian
+ */
 struct _athstats {
 	struct ath_stats ath;
 #ifdef ATH_SUPPORT_ANI
@@ -723,6 +757,25 @@ ath_get_curstat(struct statfoo *sf, int s, char b[], size_t bs)
 		snprintf(b, bs, "%d",
 			wf->cur.ath.ast_rx_rssi + wf->cur.ath.ast_rx_noise);
 		return 1;
+	case S_RX_AGG:		STAT(rx_agg);
+	case S_RX_HALFGI:	STAT(rx_halfgi);
+	case S_RX_2040:		STAT(rx_2040);
+	case S_RX_PRE_CRC_ERR:	STAT(rx_pre_crc_err);
+	case S_RX_POST_CRC_ERR:	STAT(rx_post_crc_err);
+	case S_RX_DECRYPT_BUSY_ERR:	STAT(rx_decrypt_busy_err);
+	case S_RX_HI_CHAIN:	STAT(rx_hi_rx_chain);
+	case S_TX_HTPROTECT:	STAT(tx_htprotect);
+	case S_RX_QEND:		STAT(rx_hitqueueend);
+	case S_TX_TIMEOUT:	STAT(tx_timeout);
+	case S_TX_CSTIMEOUT:	STAT(tx_cst);
+	case S_TX_XTXOP_ERR:	STAT(tx_xtxop);
+	case S_TX_TIMEREXPIRED_ERR:	STAT(tx_timerexpired);
+	case S_TX_DESCCFG_ERR:	STAT(tx_desccfgerr);
+	case S_TX_SWRETRIES:	STAT(tx_swretries);
+	case S_TX_SWRETRIES_MAX:	STAT(tx_swretrymax);
+	case S_TX_DATA_UNDERRUN:	STAT(tx_data_underrun);
+	case S_TX_DELIM_UNDERRUN:	STAT(tx_delim_underrun);
+	case S_TX_AGGR_FAIL:		STAT(tx_aggrfail);
 	}
 	b[0] = '\0';
 	return 0;
@@ -943,7 +996,27 @@ ath_get_totstat(struct statfoo *sf, int s, char b[], size_t bs)
 		snprintf(b, bs, "%d",
 			wf->total.ath.ast_rx_rssi + wf->total.ath.ast_rx_noise);
 		return 1;
+	case S_RX_AGG:		STAT(rx_agg);
+	case S_RX_HALFGI:	STAT(rx_halfgi);
+	case S_RX_2040:		STAT(rx_2040);
+	case S_RX_PRE_CRC_ERR:	STAT(rx_pre_crc_err);
+	case S_RX_POST_CRC_ERR:	STAT(rx_post_crc_err);
+	case S_RX_DECRYPT_BUSY_ERR:	STAT(rx_decrypt_busy_err);
+	case S_RX_HI_CHAIN:	STAT(rx_hi_rx_chain);
+	case S_TX_HTPROTECT:	STAT(tx_htprotect);
+	case S_RX_QEND:		STAT(rx_hitqueueend);
+	case S_TX_TIMEOUT:	STAT(tx_timeout);
+	case S_TX_CSTIMEOUT:	STAT(tx_cst);
+	case S_TX_XTXOP_ERR:	STAT(tx_xtxop);
+	case S_TX_TIMEREXPIRED_ERR:	STAT(tx_timerexpired);
+	case S_TX_DESCCFG_ERR:	STAT(tx_desccfgerr);
+	case S_TX_SWRETRIES:	STAT(tx_swretries);
+	case S_TX_SWRETRIES_MAX:	STAT(tx_swretrymax);
+	case S_TX_DATA_UNDERRUN:	STAT(tx_data_underrun);
+	case S_TX_DELIM_UNDERRUN:	STAT(tx_delim_underrun);
+	case S_TX_AGGR_FAIL:		STAT(tx_aggrfail);
 	}
+
 	b[0] = '\0';
 	return 0;
 #undef RXANT
diff --git a/tools/tools/nanobsd/gateworks/common b/tools/tools/nanobsd/gateworks/common
index 6e30216adf4f..c23a2bfe8165 100644
--- a/tools/tools/nanobsd/gateworks/common
+++ b/tools/tools/nanobsd/gateworks/common
@@ -4,9 +4,8 @@ NANO_CFGDIR=${NANO_CFGDIR:-${NANO_SRC}/${NANO_TOOLS}/gateworks/cfg}
 test -d ${NANO_CFGDIR} || NANO_CFGDIR=/var/empty
 NANO_PMAKE="make"					# NB: disable -j 3
 
-NANO_ARCH=arm
+NANO_ARCH=armeb
 TARGET_CPUTYPE=xscale; export TARGET_CPUTYPE		# XXX
-TARGET_BIG_ENDIAN=true; export TARGET_BIG_ENDIAN	# XXX
 
 NANO_CUSTOMIZE="cust_allow_ssh_root"
 
diff --git a/tools/tools/sysbuild/sysbuild.sh b/tools/tools/sysbuild/sysbuild.sh
index 93bf5d8270b0..ee772599a8b2 100644
--- a/tools/tools/sysbuild/sysbuild.sh
+++ b/tools/tools/sysbuild/sysbuild.sh
@@ -113,6 +113,17 @@ final_chroot() (
 )
 
 #######################################################################
+# -P is a pretty neat way to clean junk out from your ports dist-files:
+#
+#	mkdir /freebsd/ports/distfiles.old
+#	mv /freebsd/ports/distfiles/* /freebsd/ports/distfiles.old
+#	sh sysbuild.sh -c $yourconfig -P /freebsd/ports/distfiles.old
+#	rm -rf /freebsd/ports/distfiles.old
+#
+# Unfortunately bsd.ports.mk does not attempt to use a hard-link so
+# while this runs you need diskspace for both your old and your "new"
+# distfiles.
+#
 #######################################################################
 
 usage () {
@@ -122,6 +133,7 @@ usage () {
         echo "  -k      suppress buildkernel"
         echo "  -w      suppress buildworld"
         echo "  -p      used cached packages"
+        echo "  -P <dir> prefetch ports"
         echo "  -c      specify config file"
         ) 1>&2
         exit 2
@@ -249,12 +261,13 @@ ports_build() (
 ports_prefetch() (
 	(
 	set +x
-	true > /mnt/_.prefetch
-	echo "Building /tmp/_.plist" >> /mnt/_.prefetch
+	ldir=$1
+	true > /${ldir}/_.prefetch
+	echo "Building /tmp/_.plist" >> /${ldir}/_.prefetch
 
 	ports_recurse . $PORTS_WE_WANT
 
-	echo "Completed /tmp/_.plist" >> /mnt/_.prefetch
+	echo "Completed /tmp/_.plist" >> /${ldir}/_.prefetch
 	# Now checksump/fetch them
 	for p in `cat /tmp/_.plist`
 	do
@@ -262,21 +275,22 @@ ports_prefetch() (
 		(
 			cd $p
 			if make checksum $PORTS_OPTS ; then
-				rm -f /mnt/_.prefetch.$b
-				echo "OK $p" >> /mnt/_.prefetch
+				rm -f /${ldir}/_.prefetch.$b
+				echo "OK $p" >> /${ldir}/_.prefetch
 				exit 0
 			fi
 			make distclean
 			make checksum $PORTS_OPTS || true
 
 			if make checksum $PORTS_OPTS > /dev/null 2>&1 ; then
-				rm -f /mnt/_.prefetch.$b
-				echo "OK $p" >> /mnt/_.prefetch
+				rm -f /${ldir}/_.prefetch.$b
+				echo "OK $p" >> /${ldir}/_.prefetch
 			else
-				echo "BAD $p" >> /mnt/_.prefetch
+				echo "BAD $p" >> /${ldir}/_.prefetch
 			fi
-		) > /mnt/_.prefetch.$b 2>&1
+		) > /${ldir}/_.prefetch.$b 2>&1
 	done
+	echo "Done" >> /${ldir}/_.prefetch
 	) 
 )
 
@@ -284,11 +298,12 @@ ports_prefetch() (
 
 do_world=true
 do_kernel=true
+do_prefetch=false
 use_pkg=""
 c_arg=""
 
 set +e
-args=`getopt bc:hkpw $*`
+args=`getopt bc:hkpP:w $*`
 if [ $? -ne 0 ] ; then
 	usage
 fi
@@ -325,6 +340,12 @@ do
 		shift;
 		use_pkg="-p"
 		;;
+	-P)
+		shift;
+		do_prefetch=true
+		distfile_cache=$1
+		shift;
+		;;
 	-w)
 		shift;
 		do_world=false
@@ -366,6 +387,15 @@ fi
 T0=`date +%s`
 echo $T0 $T0 > /tmp/_sb_log
 
+if $do_prefetch ; then
+	rm -rf /tmp/sysbuild/ports
+	mkdir -p /tmp/sysbuild/ports
+	ln -s ${distfile_cache} /tmp/sysbuild/ports/distfiles
+	export PORTS_OPTS=CD_MOUNTPTS=/tmp/sysbuild
+	ports_prefetch /tmp 
+	exit 0
+fi
+
 log_it Unmount everything
 (
 	( cleanup )
@@ -446,7 +476,7 @@ log_it copy ports config files
 (cd / ; find var/db/ports -print | cpio -dumpv /mnt > /dev/null 2>&1)
 
 log_it "Start prefetch of ports distfiles"
-ports_prefetch &
+ports_prefetch /mnt &
 
 if $do_world ; then
 	(
@@ -558,13 +588,14 @@ if [ "x$SERCONS" != "xfalse" ] ; then
 	sed -i "" -e '/^ttyv[0-8]/s/	on/	off/' /mnt/etc/ttys
 fi
 
-log_it move config files
+log_it move dist config files "(expect warnings)"
 (
 	cd /mnt
 	mkdir root/configfiles_dist
 	find ${CONFIGFILES} -print | cpio -dumpv root/configfiles_dist
 )
 
+log_it copy live config files
 (cd / && find ${CONFIGFILES} -print | cpio -dumpv /mnt)
 
 log_it final_root
diff --git a/tools/tools/zfsboottest/Makefile b/tools/tools/zfsboottest/Makefile
new file mode 100644
index 000000000000..c0c7d92c19e9
--- /dev/null
+++ b/tools/tools/zfsboottest/Makefile
@@ -0,0 +1,29 @@
+# $FreeBSD$
+
+.PATH:		${.CURDIR}/../../../sys/boot/zfs ${.CURDIR}/../../../sys/cddl/boot/zfs
+
+BINDIR?=	/usr/bin
+SCRIPTSDIR?=	/usr/bin
+
+PROG=		zfsboottest
+SCRIPTS=	zfsboottest.sh
+SCRIPTSNAME=	zfsboottest.sh
+NO_MAN=
+
+CFLAGS=	-O1 \
+	-I${.CURDIR}/../../../sys/boot/zfs \
+	-I${.CURDIR}/../../../sys/cddl/boot/zfs \
+	-I. \
+	-fdiagnostics-show-option \
+	-W -Wextra -Wno-sign-compare -Wno-unused-parameter \
+	-Werror
+LDFLAGS+=-lmd
+
+.if ${MACHINE_CPUARCH} == "amd64"
+beforedepend zfsboottest.o: machine
+CLEANFILES+=	machine
+machine:
+	ln -sf ${.CURDIR}/../../../sys/i386/include machine
+.endif
+
+.include <bsd.prog.mk>
diff --git a/sys/boot/zfs/zfstest.c b/tools/tools/zfsboottest/zfsboottest.c
similarity index 59%
rename from sys/boot/zfs/zfstest.c
rename to tools/tools/zfsboottest/zfsboottest.c
index 3e32da57781e..4b29f9494290 100644
--- a/sys/boot/zfs/zfstest.c
+++ b/tools/tools/zfsboottest/zfsboottest.c
@@ -1,5 +1,7 @@
 /*-
  * Copyright (c) 2010 Doug Rabson
+ * Copyright (c) 2011 Andriy Gapon
+ * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -24,14 +26,13 @@
  * SUCH DAMAGE.
  */
 /* $FreeBSD$ */
-/*
- * Compile with 'cc -I. -I../../cddl/boot/zfs zfstest.c -o zfstest'
- */
 
 #include <sys/param.h>
 #include <sys/queue.h>
+#include <err.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <md5.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
@@ -45,15 +46,19 @@
 void
 pager_output(const char *line)
 {
+
 	fprintf(stderr, "%s", line);
 }
 
+#define ZFS_TEST
+#define	printf(...)	 fprintf(stderr, __VA_ARGS__)
 #include "zfsimpl.c"
+#undef printf
 
 static int
 vdev_read(vdev_t *vdev, void *priv, off_t off, void *buf, size_t bytes)
 {
-	int fd = *(int *) priv;
+	int fd = *(int *)priv;
 
 	if (pread(fd, buf, bytes, off) != bytes)
 		return (-1);
@@ -72,7 +77,7 @@ zfs_read(spa_t *spa, dnode_phys_t *dn, void *buf, size_t size, off_t off)
 		n = zp->zp_size - off;
 
 	rc = dnode_read(spa, dn, off, buf, n);
-	if (rc)
+	if (rc != 0)
 		return (-rc);
 
 	return (n);
@@ -81,29 +86,49 @@ zfs_read(spa_t *spa, dnode_phys_t *dn, void *buf, size_t size, off_t off)
 int
 main(int argc, char** argv)
 {
-	char buf[512];
-	int fd[100];
+	char buf[512], hash[33];
+	MD5_CTX ctx;
 	struct stat sb;
 	dnode_phys_t dn;
 	spa_t *spa;
-	int i, n, off;
+	off_t off;
+	ssize_t n;
+	int i, failures, *fd;
 
 	zfs_init();
 	if (argc == 1) {
 		static char *av[] = {
-			"zfstest", "COPYRIGHT",
-			"/dev/da0p2", "/dev/da1p2", "/dev/da2p2",
+			"zfsboottest",
+			"/dev/gpt/system0",
+			"/dev/gpt/system1",
+			"-",
+			"/boot/zfsloader",
+			"/boot/support.4th",
+			"/boot/kernel/kernel",
 			NULL,
 		};
-		argc = 5;
+		argc = sizeof(av) / sizeof(av[0]) - 1;
 		argv = av;
 	}
-	for (i = 2; i < argc; i++) {
-		fd[i] = open(argv[i], O_RDONLY);
-		if (fd[i] < 0)
+	for (i = 1; i < argc; i++) {
+		if (strcmp(argv[i], "-") == 0)
+			break;
+	}
+	fd = malloc(sizeof(fd[0]) * (i - 1));
+	if (fd == NULL)
+		errx(1, "Unable to allocate memory.");
+	for (i = 1; i < argc; i++) {
+		if (strcmp(argv[i], "-") == 0)
+			break;
+		fd[i - 1] = open(argv[i], O_RDONLY);
+		if (fd[i - 1] == -1) {
+			warn("open(%s) failed", argv[i]);
 			continue;
-		if (vdev_probe(vdev_read, &fd[i], NULL) != 0)
-			close(fd[i]);
+		}
+		if (vdev_probe(vdev_read, &fd[i - 1], NULL) != 0) {
+			warnx("vdev_probe(%s) failed", argv[i]);
+			close(fd[i - 1]);
+		}
 	}
 	spa_all_status();
 
@@ -118,27 +143,40 @@ main(int argc, char** argv)
 		exit(1);
 	}
 
-	if (zfs_lookup(spa, argv[1], &dn)) {
-		fprintf(stderr, "can't lookup\n");
-		exit(1);
-	}
-
-	if (zfs_dnode_stat(spa, &dn, &sb)) {
-		fprintf(stderr, "can't stat\n");
-		exit(1);
-	}
-
-
-	off = 0;
-	do {
-		n = zfs_read(spa, &dn, buf, 512, off);
-		if (n < 0) {
-			fprintf(stderr, "zfs_read failed\n");
-			exit(1);
+	printf("\n");
+	for (++i, failures = 0; i < argc; i++) {
+		if (zfs_lookup(spa, argv[i], &dn)) {
+			fprintf(stderr, "%s: can't lookup\n", argv[i]);
+			failures++;
+			continue;
 		}
-		write(1, buf, n);
-		off += n;
-	} while (off < sb.st_size);
 
-	return (0);
+		if (zfs_dnode_stat(spa, &dn, &sb)) {
+			fprintf(stderr, "%s: can't stat\n", argv[i]);
+			failures++;
+			continue;
+		}
+
+		off = 0;
+		MD5Init(&ctx);
+		do {
+			n = sb.st_size - off;
+			n = n > sizeof(buf) ? sizeof(buf) : n;
+			n = zfs_read(spa, &dn, buf, n, off);
+			if (n < 0) {
+				fprintf(stderr, "%s: zfs_read failed\n",
+				    argv[i]);
+				failures++;
+				break;
+			}
+			MD5Update(&ctx, buf, n);
+			off += n;
+		} while (off < sb.st_size);
+		if (off < sb.st_size)
+			continue;
+		MD5End(&ctx, hash);
+		printf("%s %s\n", hash, argv[i]);
+	}
+
+	return (failures == 0 ? 0 : 1);
 }
diff --git a/tools/tools/zfsboottest/zfsboottest.sh b/tools/tools/zfsboottest/zfsboottest.sh
new file mode 100755
index 000000000000..e1bcf8bebfdd
--- /dev/null
+++ b/tools/tools/zfsboottest/zfsboottest.sh
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+# Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+if [ $# -ne 1 ]; then
+	echo "usage: zfsboottest.sh <pool>" >&2
+	exit 1
+fi
+
+which -s zfsboottest
+if [ $? -eq 0 ]; then
+	zfsboottest="zfsboottest"
+else
+	if [ ! -x "/usr/src/tools/tools/zfsboottest/zfsboottest" ]; then
+		echo "Unable to find \"zfsboottest\" utility." >&2
+		exit 1
+	fi
+	zfsboottest="/usr/src/tools/tools/zfsboottest/zfsboottest"
+fi
+
+startdir="/boot"
+
+pool="${1}"
+zpool list "${pool}" >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+	echo "No such pool \"${pool}\"." >&2
+	exit 1
+fi
+bootfs=`zpool get bootfs "${pool}" | tail -1 | awk '{print $3}'`
+if [ "${bootfs}" = "-" ]; then
+	echo "The \"bootfs\" property is not configured for pool \"${pool}\"." >&2
+	exit 1
+fi
+# Dataset's mountpoint property should be set to 'legacy'.
+if [ "`zfs get -H -o value mountpoint ${bootfs}`" != "legacy" ]; then
+	echo "The \"mountpoint\" property of dataset \"${bootfs}\" should be set to \"legacy\"." >&2
+	exit 1
+fi
+mountpoint=`df -t zfs "${bootfs}" 2>/dev/null | tail -1 | awk '{print $6}'`
+if [ -z "${mountpoint}" ]; then
+	echo "The \"${bootfs}\" dataset is not mounted." >&2
+	exit 1
+fi
+if [ ! -d "${mountpoint}${startdir}" ]; then
+	echo "The \"${mountpoint}${startdir}\" directory doesn't exist." >&2
+	exit 1
+fi
+# To be able to mount root ZFS file system we need either /etc/fstab entry
+# or vfs.root.mountfrom variable set in /boot/loader.conf.
+egrep -q '^'"${bootfs}"'[[:space:]]+/[[:space:]]+zfs[[:space:]]+' "${mountpoint}/etc/fstab" 2>/dev/null
+if [ $? -ne 0 ]; then
+	egrep -q 'vfs.root.mountfrom="?'"${bootfs}"'"?[[:space:]]*$' "${mountpoint}/boot/loader.conf" 2>/dev/null
+	if [ $? -ne 0 ]; then
+		echo "To be able to boot from \"${bootfs}\", you need to declare" >&2
+		echo "\"${bootfs}\" as being root file system in ${mountpoint}/etc/fstab" >&2
+		echo "or add \"vfs.root.mountfrom\" variable set to \"${bootfs}\" to" >&2
+		echo "${mountpoint}/boot/loader.conf." >&2
+		exit 1
+	fi
+fi
+vdevs=""
+for vdev in `zpool status "${pool}" | grep ONLINE | awk '{print $1}'`; do
+	vdev="/dev/${vdev#/dev/}"
+	if [ -c "${vdev}" ]; then
+		if [ -z "${vdevs}" ]; then
+			vdevs="${vdev}"
+		else
+			vdevs="${vdevs} ${vdev}"
+		fi
+	fi
+done
+
+list0=`mktemp /tmp/zfsboottest.XXXXXXXXXX`
+if [ $? -ne 0 ]; then
+	echo "Unable to create temporary file." >&2
+	exit 1
+fi
+list1=`mktemp /tmp/zfsboottest.XXXXXXXXXX`
+if [ $? -ne 0 ]; then
+	echo "Unable to create temporary file." >&2
+	rm -f "${list0}"
+	exit 1
+fi
+
+echo "zfsboottest.sh is reading all the files in ${mountpoint}${startdir} using"
+echo "boot code and using file system code."
+echo "It calculates MD5 checksums for all the files and will compare them."
+echo "If all files can be properly read using boot code, it is very likely you"
+echo "will be able to boot from \"${pool}\" pool>:> Good luck!"
+echo
+
+"${zfsboottest}" ${vdevs} - `find "${mountpoint}${startdir}" -type f | sed "s@^${mountpoint}@@"` | egrep '^[0-9a-z]{32} /' | sort -k 2 >"${list0}"
+find "${mountpoint}${startdir}" -type f | xargs md5 -r | sed "s@ ${mountpoint}@ @" | egrep '^[0-9a-z]{32} /' | sort -k 2 >"${list1}"
+
+diff -u "${list0}" "${list1}"
+ec=$?
+
+rm -f "${list0}" "${list1}"
+
+if [ $? -ne 0 ]; then
+	echo >&2
+	echo "You may not be able to boot." >&2
+	exit 1
+fi
+
+echo "OK"
diff --git a/usr.bin/at/Makefile b/usr.bin/at/Makefile
index 75546fc156fa..5e9cfc9302fd 100644
--- a/usr.bin/at/Makefile
+++ b/usr.bin/at/Makefile
@@ -11,7 +11,7 @@ MLINKS=	at.1 batch.1 \
 	at.1 atq.1 \
 	at.1 atrm.1
 
-WARNS?=	1
+NO_WFORMAT=
 
 BINOWN=	root
 BINMODE= 4555
diff --git a/usr.bin/calendar/Makefile b/usr.bin/calendar/Makefile
index 4a5999c93701..24c4bbae7900 100644
--- a/usr.bin/calendar/Makefile
+++ b/usr.bin/calendar/Makefile
@@ -12,8 +12,6 @@ DE_LINKS=       de_DE.ISO8859-15
 FR_LINKS=       fr_FR.ISO8859-15
 TEXTMODE?=	444
 
-WARNS?=		7
-
 beforeinstall:
 	${INSTALL} -o ${BINOWN} -g ${BINGRP} -m ${TEXTMODE} \
 	    ${.CURDIR}/calendars/calendar.* ${DESTDIR}${SHAREDIR}/calendar
diff --git a/usr.bin/calendar/calendars/calendar.freebsd b/usr.bin/calendar/calendars/calendar.freebsd
index bf15da79f815..127cb50baf14 100644
--- a/usr.bin/calendar/calendars/calendar.freebsd
+++ b/usr.bin/calendar/calendars/calendar.freebsd
@@ -22,6 +22,7 @@
 01/16	Ariff Abdullah <ariff@FreeBSD.org> born in Kuala Lumpur, Malaysia, 1978
 01/16	Dmitry Sivachenko <demon@FreeBSD.org> born in Moscow, USSR, 1978
 01/16	Vanilla I. Shu <vanilla@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1978
+01/17	Raphael Kubo da Costa <rakuco@FreeBSD.org> born in Sao Paulo, Sao Paulo, Brazil, 1989
 01/18	Dejan Lesjak <lesi@FreeBSD.org> born in Ljubljana, Slovenia, Yugoslavia, 1977
 01/19	Marshall Kirk McKusick <mckusick@FreeBSD.org> born in Wilmington, Delaware, United States, 1954
 01/19	Ruslan Ermilov <ru@FreeBSD.org> born in Simferopol, USSR, 1974
@@ -47,6 +48,7 @@
 02/02	Michael W Lucas <mwlucas@FreeBSD.org> born in Detroit, Michigan, United States, 1967
 02/02	Dmitry Chagin <dchagin@FreeBSD.org> born in Stalingrad, USSR, 1976
 02/02	Yoichi Nakayama <yoichi@FreeBSD.org> born in Tsu, Mie, Japan, 1976
+02/04	Eitan Adler <eadler@FreeBSD.org> born in West Hempstead, New York, United States, 1991
 02/05	Frank Laszlo <laszlof@FreeBSD.org> born in Howell, Michigan, United States, 1983
 02/10	David Greenman <dg@FreeBSD.org> born in Portland, Oregon, United States, 1968
 02/10	Paul Richards <paul@FreeBSD.org> born in Ammanford, Carmarthenshire, United Kingdom, 1968
@@ -126,6 +128,7 @@
 04/17	Dryice Liu <dryice@FreeBSD.org> born in Jinan, Shandong, China, 1975
 04/22	Joerg Wunsch <joerg@FreeBSD.org> born in Dresden, Sachsen, Germany, 1962
 04/22	Jun Kuriyama <kuriyama@FreeBSD.org> born in Matsue, Shimane, Japan, 1973
+04/22	Jakub Klama <jceel@FreeBSD.org> born in Blachownia, Silesia, Poland, 1989
 04/26	Rene Ladan <rene@FreeBSD.org> born in Geldrop, the Netherlands, 1980
 04/29	Adam Weinberger <adamw@FreeBSD.org> born in Berkeley, California, United States, 1980
 04/29	Eric Anholt <anholt@FreeBSD.org> born in Portland, Oregon, United States, 1983
@@ -272,6 +275,7 @@
 09/17	Maxim Bolotin <mb@FreeBSD.org> born in Rostov-on-Don, Russian Federation, 1976
 09/18	Matthew Fleming <mdf@FreeBSD.org> born in Cleveland, Ohio, United States, 1975
 09/20	Kevin Lo <kevlo@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1972
+09/21	Gleb Kurtsou <gleb@FreeBSD.org> born in Minsk, Belarus, 1984
 09/27	Neil Blakey-Milner <nbm@FreeBSD.org> born in Port Elizabeth, South Africa, 1978
 09/27	Renato Botelho <garga@FreeBSD.org> born in Araras, Sao Paulo, Brazil, 1979
 09/28	Greg Lehey <grog@FreeBSD.org> born in Melbourne, Victoria, Australia, 1948
diff --git a/usr.bin/clang/Makefile b/usr.bin/clang/Makefile
index 0096d42ede83..ef5a0e794f26 100644
--- a/usr.bin/clang/Makefile
+++ b/usr.bin/clang/Makefile
@@ -1,5 +1,5 @@
 # $FreeBSD$
 
-SUBDIR=	clang tblgen
+SUBDIR=	clang clang-tblgen tblgen
 
 .include <bsd.subdir.mk>
diff --git a/usr.bin/clang/clang-tblgen/Makefile b/usr.bin/clang/clang-tblgen/Makefile
new file mode 100644
index 000000000000..aedbeb4b8d57
--- /dev/null
+++ b/usr.bin/clang/clang-tblgen/Makefile
@@ -0,0 +1,18 @@
+# $FreeBSD$
+
+PROG_CXX=clang-tblgen
+NO_MAN=
+
+SRCDIR=	tools/clang/utils/TableGen
+SRCS=	ClangASTNodesEmitter.cpp \
+	ClangAttrEmitter.cpp \
+	ClangDiagnosticsEmitter.cpp \
+	ClangSACheckersEmitter.cpp \
+	NeonEmitter.cpp \
+	OptParserEmitter.cpp \
+	TableGen.cpp
+LLVM_REQUIRES_EH=
+
+LIBDEPS=llvmtablegen llvmsupport
+
+.include "../clang.prog.mk"
diff --git a/usr.bin/clang/clang/Makefile b/usr.bin/clang/clang/Makefile
index 874aeb59a6b9..48f1655cde04 100644
--- a/usr.bin/clang/clang/Makefile
+++ b/usr.bin/clang/clang/Makefile
@@ -9,7 +9,10 @@ SRCS=	cc1_main.cpp \
 	cc1as_main.cpp \
 	driver.cpp
 
-LINKS=	${BINDIR}/clang ${BINDIR}/clang++
+LINKS=	${BINDIR}/clang ${BINDIR}/clang++ \
+	${BINDIR}/clang ${BINDIR}/clang-cpp
+MLINKS=	clang.1 clang++.1 \
+	clang.1 clang-cpp.1
 
 TGHDRS=	CC1AsOptions \
 	CC1Options \
diff --git a/usr.bin/clang/clang/clang.1 b/usr.bin/clang/clang/clang.1
index f32742eb563e..d8884df999f4 100644
--- a/usr.bin/clang/clang/clang.1
+++ b/usr.bin/clang/clang/clang.1
@@ -125,7 +125,7 @@
 .\" ========================================================================
 .\"
 .IX Title "CLANG 1"
-.TH CLANG 1 "2011-04-30" "clang 3.0" "Clang Tools Documentation"
+.TH CLANG 1 "2011-10-17" "clang 3.0" "Clang Tools Documentation"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -456,7 +456,12 @@ Add the specified directory to the search path for include files.
 Add the specified directory to the search path for framework include files.
 .IP "\fB\-nostdinc\fR" 4
 .IX Item "-nostdinc"
-Do not search the standard system directories for include files.
+Do not search the standard system directories or compiler builtin directories
+for include files.
+.IP "\fB\-nostdlibinc\fR" 4
+.IX Item "-nostdlibinc"
+Do not search the standard system directories for include files, but do search
+compiler builting include directories.
 .IP "\fB\-nobuiltininc\fR" 4
 .IX Item "-nobuiltininc"
 Do not search clang's builtin directory for include files.
diff --git a/usr.bin/clang/tblgen/Makefile b/usr.bin/clang/tblgen/Makefile
index 6962406e0724..4eba1068fd5d 100644
--- a/usr.bin/clang/tblgen/Makefile
+++ b/usr.bin/clang/tblgen/Makefile
@@ -8,10 +8,6 @@ SRCS=	ARMDecoderEmitter.cpp \
 	AsmWriterEmitter.cpp \
 	AsmWriterInst.cpp \
 	CallingConvEmitter.cpp \
-	ClangASTNodesEmitter.cpp \
-	ClangAttrEmitter.cpp \
-	ClangDiagnosticsEmitter.cpp \
-	ClangSACheckersEmitter.cpp \
 	CodeEmitterGen.cpp \
 	CodeGenDAGPatterns.cpp \
 	CodeGenInstruction.cpp \
@@ -24,30 +20,22 @@ SRCS=	ARMDecoderEmitter.cpp \
 	DAGISelMatcherOpt.cpp \
 	DisassemblerEmitter.cpp \
 	EDEmitter.cpp \
-	Error.cpp \
 	FastISelEmitter.cpp \
 	FixedLenDecoderEmitter.cpp \
 	InstrEnumEmitter.cpp \
 	InstrInfoEmitter.cpp \
 	IntrinsicEmitter.cpp \
-	LLVMCConfigurationEmitter.cpp \
-	NeonEmitter.cpp \
-	OptParserEmitter.cpp \
 	PseudoLoweringEmitter.cpp \
-	Record.cpp \
 	RegisterInfoEmitter.cpp \
 	SetTheory.cpp \
 	StringMatcher.cpp \
 	SubtargetEmitter.cpp \
-	TGLexer.cpp \
-	TGParser.cpp \
 	TGValueTypes.cpp \
 	TableGen.cpp \
-	TableGenBackend.cpp \
 	X86DisassemblerTables.cpp \
 	X86RecognizableInstr.cpp
 LLVM_REQUIRES_EH=
 
-LIBDEPS=llvmsupport
+LIBDEPS=llvmtablegen llvmsupport
 
 .include "../clang.prog.mk"
diff --git a/usr.bin/compress/doc/NOTES b/usr.bin/compress/doc/NOTES
index d400c9b485ca..29ca3118f2e2 100644
--- a/usr.bin/compress/doc/NOTES
+++ b/usr.bin/compress/doc/NOTES
@@ -51,7 +51,7 @@ comprehensive survey of an area which will remain murky for some time.
 
 Until the dust clears, how you approach ideas which are patented depends
 on how paranoid you are of a legal onslaught.  Arbitrary?  Yes.  But
-the patent bar the the CCPA (Court of Customs and Patent Appeals)
+the patent bar the CCPA (Court of Customs and Patent Appeals)
 thanks you for any uncertainty as they, at least, stand to gain
 from any trouble.
 
diff --git a/usr.bin/compress/zopen.c b/usr.bin/compress/zopen.c
index a729e134de41..8ae3d7d42bba 100644
--- a/usr.bin/compress/zopen.c
+++ b/usr.bin/compress/zopen.c
@@ -486,7 +486,7 @@ zread(void *cookie, char *rbp, int num)
 	block_compress = maxbits & BLOCK_MASK;
 	maxbits &= BIT_MASK;
 	maxmaxcode = 1L << maxbits;
-	if (maxbits > BITS) {
+	if (maxbits > BITS || maxbits < 12) {
 		errno = EFTYPE;
 		return (-1);
 	}
@@ -513,17 +513,28 @@ zread(void *cookie, char *rbp, int num)
 			for (code = 255; code >= 0; code--)
 				tab_prefixof(code) = 0;
 			clear_flg = 1;
-			free_ent = FIRST - 1;
-			if ((code = getcode(zs)) == -1)	/* O, untimely death! */
-				break;
+			free_ent = FIRST;
+			oldcode = -1;
+			continue;
 		}
 		incode = code;
 
-		/* Special case for KwKwK string. */
+		/* Special case for kWkWk string. */
 		if (code >= free_ent) {
+			if (code > free_ent || oldcode == -1) {
+				/* Bad stream. */
+				errno = EINVAL;
+				return (-1);
+			}
 			*stackp++ = finchar;
 			code = oldcode;
 		}
+		/*
+		 * The above condition ensures that code < free_ent.
+		 * The construction of tab_prefixof in turn guarantees that
+		 * each iteration decreases code and therefore stack usage is
+		 * bound by 1 << BITS - 256.
+		 */
 
 		/* Generate output characters in reverse order. */
 		while (code >= 256) {
@@ -540,7 +551,7 @@ middle:		do {
 		} while (stackp > de_stack);
 
 		/* Generate the new entry. */
-		if ((code = free_ent) < maxmaxcode) {
+		if ((code = free_ent) < maxmaxcode && oldcode != -1) {
 			tab_prefixof(code) = (u_short) oldcode;
 			tab_suffixof(code) = finchar;
 			free_ent = code + 1;
diff --git a/usr.bin/csup/diff.c b/usr.bin/csup/diff.c
index 805967689985..bafb458e05be 100644
--- a/usr.bin/csup/diff.c
+++ b/usr.bin/csup/diff.c
@@ -26,11 +26,10 @@
  * $FreeBSD$
  */
 
-#include <sys/limits.h>
-
 #include <assert.h>
 #include <err.h>
 #include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/usr.bin/csup/fixups.c b/usr.bin/csup/fixups.c
index b105a8f7d48f..a83ad1c456e7 100644
--- a/usr.bin/csup/fixups.c
+++ b/usr.bin/csup/fixups.c
@@ -141,7 +141,7 @@ fixups_get(struct fixups *f)
 	fixups_lock(f);
 	while (f->size == 0 && !f->closed)
 		pthread_cond_wait(&f->cond, &f->lock);
-	if (f->closed) {
+	if (f->closed && f->size == 0) {
 		fixups_unlock(f);
 		return (NULL);
 	}
diff --git a/usr.bin/csup/updater.c b/usr.bin/csup/updater.c
index f3f3f27aa8a2..e95ec107bbee 100644
--- a/usr.bin/csup/updater.c
+++ b/usr.bin/csup/updater.c
@@ -238,7 +238,7 @@ updater(void *arg)
 
 	/*
 	 * Make sure to close the fixups even in case of an error,
-	 * so that the lister thread doesn't block indefinitely.
+	 * so that the detailer thread doesn't block indefinitely.
 	 */
 	fixups_close(up->config->fixups);
 	if (!error)
diff --git a/usr.bin/elfdump/elfdump.c b/usr.bin/elfdump/elfdump.c
index 8bd71e838eb7..9acb50390b38 100644
--- a/usr.bin/elfdump/elfdump.c
+++ b/usr.bin/elfdump/elfdump.c
@@ -282,11 +282,12 @@ const char *ei_data[] = {
 	"ELFDATANONE", "ELFDATA2LSB", "ELFDATA2MSB"
 };
 
-const char *ei_abis[] = {
+const char *ei_abis[256] = {
 	"ELFOSABI_SYSV", "ELFOSABI_HPUX", "ELFOSABI_NETBSD", "ELFOSABI_LINUX",
-	"ELFOSABI_HURD", "ELFOSABI_86OPEN", "ELFOSABI_SOLARIS",
-	"ELFOSABI_MONTEREY", "ELFOSABI_IRIX", "ELFOSABI_FREEBSD",
-	"ELFOSABI_TRU64", "ELFOSABI_MODESTO", "ELFOSABI_OPENBSD"
+	"ELFOSABI_HURD", "ELFOSABI_86OPEN", "ELFOSABI_SOLARIS", "ELFOSABI_AIX",
+	"ELFOSABI_IRIX", "ELFOSABI_FREEBSD", "ELFOSABI_TRU64",
+	"ELFOSABI_MODESTO", "ELFOSABI_OPENBSD",
+	[255] = "ELFOSABI_STANDALONE"
 };
 
 const char *p_types[] = {
@@ -931,10 +932,10 @@ elf_get_byte(Elf32_Ehdr *e, void *base, elf_member_t member)
 	val = 0;
 	switch (e->e_ident[EI_CLASS]) {
 	case ELFCLASS32:
-		val = ((char *)base)[elf32_offsets[member]];
+		val = ((uint8_t *)base)[elf32_offsets[member]];
 		break;
 	case ELFCLASS64:
-		val = ((char *)base)[elf64_offsets[member]];
+		val = ((uint8_t *)base)[elf64_offsets[member]];
 		break;
 	case ELFCLASSNONE:
 		errx(1, "invalid class");
diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1
index 0dbbc0b04f15..9069e61d50ce 100644
--- a/usr.bin/fetch/fetch.1
+++ b/usr.bin/fetch/fetch.1
@@ -1,5 +1,5 @@
 .\"-
-.\" Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
+.\" Copyright (c) 2000-2011 Dag-Erling Smørgrav
 .\" All rights reserved.
 .\" Portions Copyright (c) 1999 Massachusetts Institute of Technology; used
 .\" by permission.
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 14, 2008
+.Dd September 27, 2011
 .Dt FETCH 1
 .Os
 .Sh NAME
@@ -165,11 +165,13 @@ directory, with name(s) selected as in the default behaviour.
 .It Fl P
 .It Fl p
 Use passive FTP.
-This is useful if you are behind a firewall which blocks incoming
-connections.
-Try this flag if
-.Nm
-seems to hang when retrieving FTP URLs.
+These flags have no effect, since passive FTP is the default, but are
+provided for compatibility with earlier versions where active FTP was
+the default.
+To force active mode, set the
+.Ev FTP_PASSIVE_MODE
+environment variable to
+.Ql NO .
 .It Fl q
 Quiet mode.
 .It Fl R
diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c
index 7553bd8d1028..8d2667196935 100644
--- a/usr.bin/fetch/fetch.c
+++ b/usr.bin/fetch/fetch.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 2000-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -522,6 +522,12 @@ fetch(char *URL, const char *path)
 				    "does not match remote", path);
 				goto failure_keep;
 			}
+		} else if (url->offset > sb.st_size) {
+			/* gap between what we asked for and what we got */
+			warnx("%s: gap in resume mode", URL);
+			fclose(of);
+			of = NULL;
+			/* picked up again later */
 		} else if (us.size != -1) {
 			if (us.size == sb.st_size)
 				/* nothing to do */
@@ -534,7 +540,7 @@ fetch(char *URL, const char *path)
 				goto failure;
 			}
 			/* we got it, open local file */
-			if ((of = fopen(path, "a")) == NULL) {
+			if ((of = fopen(path, "r+")) == NULL) {
 				warn("%s: fopen()", path);
 				goto failure;
 			}
@@ -551,8 +557,16 @@ fetch(char *URL, const char *path)
 				fclose(of);
 				of = NULL;
 				sb = nsb;
+				/* picked up again later */
 			}
 		}
+		/* seek to where we left off */
+		if (of != NULL && fseeko(of, url->offset, SEEK_SET) != 0) {
+			warn("%s: fseeko()", path);
+			fclose(of);
+			of = NULL;
+			/* picked up again later */
+		}
 	} else if (m_flag && sb.st_size != -1) {
 		/* mirror mode, local file exists */
 		if (sb.st_size == us.size && sb.st_mtime == us.mtime)
diff --git a/usr.bin/find/find.1 b/usr.bin/find/find.1
index 04fe50cccc70..10479f60fddb 100644
--- a/usr.bin/find/find.1
+++ b/usr.bin/find/find.1
@@ -31,7 +31,7 @@
 .\"	@(#)find.1	8.7 (Berkeley) 5/9/95
 .\" $FreeBSD$
 .\"
-.Dd March 17, 2010
+.Dd September 28, 2011
 .Dt FIND 1
 .Os
 .Sh NAME
@@ -507,7 +507,7 @@ This primary always evaluates to true.
 The following information for the current file is written to standard output:
 its inode number, size in 512-byte blocks, file permissions, number of hard
 links, owner, group, size in bytes, last modification time, and pathname.
-If the file is a block or character special file, the major and minor numbers
+If the file is a block or character special file, the device number
 will be displayed instead of the size in bytes.
 If the file is a symbolic link, the pathname of the linked-to file will be
 displayed preceded by
diff --git a/usr.bin/find/ls.c b/usr.bin/find/ls.c
index 8e1b8d3b9e65..44d1852430e1 100644
--- a/usr.bin/find/ls.c
+++ b/usr.bin/find/ls.c
@@ -70,8 +70,7 @@ printlong(char *name, char *accpath, struct stat *sb)
 	    group_from_gid(sb->st_gid, 0));
 
 	if (S_ISCHR(sb->st_mode) || S_ISBLK(sb->st_mode))
-		(void)printf("%3d, %3d ", major(sb->st_rdev),
-		    minor(sb->st_rdev));
+		(void)printf("%#8jx ", (uintmax_t)sb->st_rdev);
 	else
 		(void)printf("%8"PRId64" ", sb->st_size);
 	printtime(sb->st_mtime);
diff --git a/usr.bin/finger/Makefile b/usr.bin/finger/Makefile
index 30e04dfed68e..983392b7a1e0 100644
--- a/usr.bin/finger/Makefile
+++ b/usr.bin/finger/Makefile
@@ -5,6 +5,4 @@ PROG=	finger
 SRCS=	finger.c lprint.c net.c sprint.c util.c
 MAN=	finger.1 finger.conf.5
 
-WARNS?=	2
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/finger/lprint.c b/usr.bin/finger/lprint.c
index 8b7b19fe9a5e..db902df6c5d4 100644
--- a/usr.bin/finger/lprint.c
+++ b/usr.bin/finger/lprint.c
@@ -306,7 +306,7 @@ show_text(const char *directory, const char *file_name, const char *header)
 		return(0);
 
 	/* If short enough, and no newlines, show it on a single line.*/
-	if (sb.st_size <= LINE_LEN - strlen(header) - 5) {
+	if (sb.st_size <= (off_t)(LINE_LEN - strlen(header) - 5)) {
 		nr = read(fd, tbuf, sizeof(tbuf));
 		if (nr <= 0) {
 			(void)close(fd);
diff --git a/usr.bin/fstat/fstat.1 b/usr.bin/fstat/fstat.1
index e1f1c1bed168..7403a8e725ee 100644
--- a/usr.bin/fstat/fstat.1
+++ b/usr.bin/fstat/fstat.1
@@ -28,7 +28,7 @@
 .\"     @(#)fstat.1	8.3 (Berkeley) 2/25/94
 .\" $FreeBSD$
 .\"
-.Dd July 9, 2009
+.Dd September 28, 2011
 .Dt FSTAT 1
 .Os
 .Sh NAME
@@ -142,7 +142,7 @@ pathname that the file system the file resides in is mounted on.
 If the
 .Fl n
 flag is specified, this header is present and is the
-major/minor number of the device that this file resides in.
+number of the device that this file resides in.
 .It Li INUM
 The inode number of the file.
 .It Li MODE
diff --git a/usr.bin/fstat/fstat.c b/usr.bin/fstat/fstat.c
index 531eef23c445..ed853513c4ef 100644
--- a/usr.bin/fstat/fstat.c
+++ b/usr.bin/fstat/fstat.c
@@ -411,7 +411,7 @@ print_pts_info(struct procstat *procstat, struct filestat *fst)
 	}
 	printf("* pseudo-terminal master ");
 	if (nflg || !*pts.devname) {
-		printf("%10d,%-2d", major(pts.dev), minor(pts.dev));
+		printf("%#10jx", (uintmax_t)pts.dev);
 	} else {
 		printf("%10s", pts.devname);
 	}
@@ -441,7 +441,7 @@ print_vnode_info(struct procstat *procstat, struct filestat *fst)
 	}
 
 	if (nflg)
-		printf(" %2d,%-2d", major(vn.vn_fsid), minor(vn.vn_fsid));
+		printf(" %#5jx", (uintmax_t)vn.vn_fsid);
 	else if (vn.vn_mntdir != NULL)
 		(void)printf(" %-8s", vn.vn_mntdir);
 
@@ -457,7 +457,7 @@ print_vnode_info(struct procstat *procstat, struct filestat *fst)
 
 	if (vn.vn_type == PS_FST_VTYPE_VBLK || vn.vn_type == PS_FST_VTYPE_VCHR) {
 		if (nflg || !*vn.vn_devname)
-			printf("  %2d,%-2d", major(vn.vn_dev), minor(vn.vn_dev));
+			printf(" %#6jx", (uintmax_t)vn.vn_dev);
 		else {
 			printf(" %6s", vn.vn_devname);
 		}
diff --git a/usr.bin/grep/Makefile b/usr.bin/grep/Makefile
index f09a7d6f2e9f..d0c24f74240b 100644
--- a/usr.bin/grep/Makefile
+++ b/usr.bin/grep/Makefile
@@ -8,28 +8,62 @@
 PROG=	grep
 .else
 PROG=	bsdgrep
+CLEANFILES+= bsdgrep.1
+
+bsdgrep.1: grep.1
+	cp ${.ALLSRC} ${.TARGET}
 .endif
-SRCS=	fastgrep.c file.c grep.c queue.c util.c
+SRCS=	file.c grep.c queue.c util.c
+
+# Extra files ported backported form some regex improvements
+.PATH: ${.CURDIR}/regex
+SRCS+=	fastmatch.c hashtable.c tre-compile.c tre-fastmatch.c xmalloc.c
+CFLAGS+=-I${.CURDIR}/regex
 
 .if ${MK_BSD_GREP} == "yes"
 LINKS=	${BINDIR}/grep ${BINDIR}/egrep \
 	${BINDIR}/grep ${BINDIR}/fgrep \
 	${BINDIR}/grep ${BINDIR}/zgrep \
 	${BINDIR}/grep ${BINDIR}/zegrep \
-	${BINDIR}/grep ${BINDIR}/zfgrep
+	${BINDIR}/grep ${BINDIR}/zfgrep \
+	${BINDIR}/grep ${BINDIR}/xzgrep \
+	${BINDIR}/grep ${BINDIR}/xzegrep \
+	${BINDIR}/grep ${BINDIR}/xzfgrep \
+	${BINDIR}/grep ${BINDIR}/lzgrep \
+	${BINDIR}/grep ${BINDIR}/lzegrep \
+	${BINDIR}/grep ${BINDIR}/lzfgrep
 
 MLINKS= grep.1 egrep.1 \
 	grep.1 fgrep.1 \
 	grep.1 zgrep.1 \
 	grep.1 zegrep.1 \
-	grep.1 zfgrep.1
+	grep.1 zfgrep.1 \
+	grep.1 xzgrep.1 \
+	grep.1 xzegrep.1 \
+	grep.1 xzfgrep.1 \
+	grep.1 lzgrep.1 \
+	grep.1 lzegrep.1 \
+	grep.1 lzfgrep.1
 .endif
 
-bsdgrep.1: grep.1
-	cp ${.ALLSRC} ${.TARGET}
+LDADD=	-lz -llzma
+DPADD=	${LIBZ} ${LIBLZMA}
 
-LDADD=	-lz -lbz2
-DPADD=	${LIBZ} ${LIBBZ2}
+.if !defined(WITHOUT_BZIP2)
+LDADD+=	-lbz2
+DPADD+=	${LIBBZ2}
+
+.if ${MK_BSD_GREP} == "yes"
+LINKS+= ${BINDIR}/grep ${BINDIR}/bzgrep \
+	${BINDIR}/grep ${BINDIR}/bzegrep \
+	${BINDIR}/grep ${BINDIR}/bzfgrep
+MLINKS+= grep.1 bzgrep.1 \
+	 grep.1 bzegrep.1 \
+	 grep.1 bzfgrep.1
+.endif
+.else
+CFLAGS+= -DWITHOUT_BZIP2
+.endif
 
 .if !defined(WITHOUT_GNU_COMPAT)
 CFLAGS+= -I/usr/include/gnu
diff --git a/usr.bin/grep/fastgrep.c b/usr.bin/grep/fastgrep.c
deleted file mode 100644
index bc8a3aff941d..000000000000
--- a/usr.bin/grep/fastgrep.c
+++ /dev/null
@@ -1,333 +0,0 @@
-/*	$OpenBSD: util.c,v 1.36 2007/10/02 17:59:18 otto Exp $	*/
-/*	$NetBSD: fastgrep.c,v 1.4 2011/02/27 17:33:37 joerg Exp $	*/
-/*	$FreeBSD$ */
-
-/*-
- * Copyright (c) 1999 James Howard and Dag-Erling Coïdan Smørgrav
- * Copyright (C) 2008 Gabor Kovesdan <gabor@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * XXX: This file is a speed up for grep to cover the defects of the
- * regex library.  These optimizations should practically be implemented
- * there keeping this code clean.  This is a future TODO, but for the
- * meantime, we need to use this workaround.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <limits.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-#include <wchar.h>
-#include <wctype.h>
-
-#include "grep.h"
-
-static inline int	grep_cmp(const unsigned char *, const unsigned char *, size_t);
-static inline void	grep_revstr(unsigned char *, int);
-
-void
-fgrepcomp(fastgrep_t *fg, const char *pat)
-{
-	unsigned int i;
-
-	/* Initialize. */
-	fg->len = strlen(pat);
-	fg->bol = false;
-	fg->eol = false;
-	fg->reversed = false;
-
-	fg->pattern = (unsigned char *)grep_strdup(pat);
-
-	/* Preprocess pattern. */
-	for (i = 0; i <= UCHAR_MAX; i++)
-		fg->qsBc[i] = fg->len;
-	for (i = 1; i < fg->len; i++)
-		fg->qsBc[fg->pattern[i]] = fg->len - i;
-}
-
-/*
- * Returns: -1 on failure, 0 on success
- */
-int
-fastcomp(fastgrep_t *fg, const char *pat)
-{
-	unsigned int i;
-	int firstHalfDot = -1;
-	int firstLastHalfDot = -1;
-	int hasDot = 0;
-	int lastHalfDot = 0;
-	int shiftPatternLen;
-
-	/* Initialize. */
-	fg->len = strlen(pat);
-	fg->bol = false;
-	fg->eol = false;
-	fg->reversed = false;
-	fg->word = false;
-
-	/* Remove end-of-line character ('$'). */
-	if (fg->len > 0 && pat[fg->len - 1] == '$') {
-		fg->eol = true;
-		fg->len--;
-	}
-
-	/* Remove beginning-of-line character ('^'). */
-	if (pat[0] == '^') {
-		fg->bol = true;
-		fg->len--;
-		pat++;
-	}
-
-	if (fg->len >= 14 &&
-	    memcmp(pat, "[[:<:]]", 7) == 0 &&
-	    memcmp(pat + fg->len - 7, "[[:>:]]", 7) == 0) {
-		fg->len -= 14;
-		pat += 7;
-		/* Word boundary is handled separately in util.c */
-		fg->word = true;
-	}
-
-	/*
-	 * pat has been adjusted earlier to not include '^', '$' or
-	 * the word match character classes at the beginning and ending
-	 * of the string respectively.
-	 */
-	fg->pattern = grep_malloc(fg->len + 1);
-	memcpy(fg->pattern, pat, fg->len);
-	fg->pattern[fg->len] = '\0';
-
-	/* Look for ways to cheat...er...avoid the full regex engine. */
-	for (i = 0; i < fg->len; i++) {
-		/* Can still cheat? */
-		if (fg->pattern[i] == '.') {
-			hasDot = i;
-			if (i < fg->len / 2) {
-				if (firstHalfDot < 0)
-					/* Closest dot to the beginning */
-					firstHalfDot = i;
-			} else {
-				/* Closest dot to the end of the pattern. */
-				lastHalfDot = i;
-				if (firstLastHalfDot < 0)
-					firstLastHalfDot = i;
-			}
-		} else {
-			/* Free memory and let others know this is empty. */
-			free(fg->pattern);
-			fg->pattern = NULL;
-			return (-1);
-		}
-	}
-
-	/*
-	 * Determine if a reverse search would be faster based on the placement
-	 * of the dots.
-	 */
-	if ((!(lflag || cflag)) && ((!(fg->bol || fg->eol)) &&
-	    ((lastHalfDot) && ((firstHalfDot < 0) ||
-	    ((fg->len - (lastHalfDot + 1)) < (size_t)firstHalfDot)))) &&
-	    !oflag && !color) {
-		fg->reversed = true;
-		hasDot = fg->len - (firstHalfDot < 0 ?
-		    firstLastHalfDot : firstHalfDot) - 1;
-		grep_revstr(fg->pattern, fg->len);
-	}
-
-	/*
-	 * Normal Quick Search would require a shift based on the position the
-	 * next character after the comparison is within the pattern.  With
-	 * wildcards, the position of the last dot effects the maximum shift
-	 * distance.
-	 * The closer to the end the wild card is the slower the search.  A
-	 * reverse version of this algorithm would be useful for wildcards near
-	 * the end of the string.
-	 *
-	 * Examples:
-	 * Pattern	Max shift
-	 * -------	---------
-	 * this		5
-	 * .his		4
-	 * t.is		3
-	 * th.s		2
-	 * thi.		1
-	 */
-
-	/* Adjust the shift based on location of the last dot ('.'). */
-	shiftPatternLen = fg->len - hasDot;
-
-	/* Preprocess pattern. */
-	for (i = 0; i <= (signed)UCHAR_MAX; i++)
-		fg->qsBc[i] = shiftPatternLen;
-	for (i = hasDot + 1; i < fg->len; i++) {
-		fg->qsBc[fg->pattern[i]] = fg->len - i;
-	}
-
-	/*
-	 * Put pattern back to normal after pre-processing to allow for easy
-	 * comparisons later.
-	 */
-	if (fg->reversed)
-		grep_revstr(fg->pattern, fg->len);
-
-	return (0);
-}
-
-int
-grep_search(fastgrep_t *fg, const unsigned char *data, size_t len, regmatch_t *pmatch)
-{
-	unsigned int j;
-	int ret = REG_NOMATCH;
-
-	if (pmatch->rm_so == (ssize_t)len)
-		return (ret);
-
-	if (fg->bol && pmatch->rm_so != 0) {
-		pmatch->rm_so = len;
-		pmatch->rm_eo = len;
-		return (ret);
-	}
-
-	/* No point in going farther if we do not have enough data. */
-	if (len < fg->len)
-		return (ret);
-
-	/* Only try once at the beginning or ending of the line. */
-	if (fg->bol || fg->eol) {
-		/* Simple text comparison. */
-		/* Verify data is >= pattern length before searching on it. */
-		if (len >= fg->len) {
-			/* Determine where in data to start search at. */
-			j = fg->eol ? len - fg->len : 0;
-			if (!((fg->bol && fg->eol) && (len != fg->len)))
-				if (grep_cmp(fg->pattern, data + j,
-				    fg->len) == -1) {
-					pmatch->rm_so = j;
-					pmatch->rm_eo = j + fg->len;
-						ret = 0;
-				}
-		}
-	} else if (fg->reversed) {
-		/* Quick Search algorithm. */
-		j = len;
-		do {
-			if (grep_cmp(fg->pattern, data + j - fg->len,
-				fg->len) == -1) {
-				pmatch->rm_so = j - fg->len;
-				pmatch->rm_eo = j;
-				ret = 0;
-				break;
-			}
-			/* Shift if within bounds, otherwise, we are done. */
-			if (j == fg->len)
-				break;
-			j -= fg->qsBc[data[j - fg->len - 1]];
-		} while (j >= fg->len);
-	} else {
-		/* Quick Search algorithm. */
-		j = pmatch->rm_so;
-		do {
-			if (grep_cmp(fg->pattern, data + j, fg->len) == -1) {
-				pmatch->rm_so = j;
-				pmatch->rm_eo = j + fg->len;
-				ret = 0;
-				break;
-			}
-
-			/* Shift if within bounds, otherwise, we are done. */
-			if (j + fg->len == len)
-				break;
-			else
-				j += fg->qsBc[data[j + fg->len]];
-		} while (j <= (len - fg->len));
-	}
-
-	return (ret);
-}
-
-/*
- * Returns:	i >= 0 on failure (position that it failed)
- *		-1 on success
- */
-static inline int
-grep_cmp(const unsigned char *pat, const unsigned char *data, size_t len)
-{
-	size_t size;
-	wchar_t *wdata, *wpat;
-	unsigned int i;
-
-	if (iflag) {
-		if ((size = mbstowcs(NULL, (const char *)data, 0)) ==
-		    ((size_t) - 1))
-			return (-1);
-
-		wdata = grep_malloc(size * sizeof(wint_t));
-
-		if (mbstowcs(wdata, (const char *)data, size) ==
-		    ((size_t) - 1))
-			return (-1);
-
-		if ((size = mbstowcs(NULL, (const char *)pat, 0)) ==
-		    ((size_t) - 1))
-			return (-1);
-
-		wpat = grep_malloc(size * sizeof(wint_t));
-
-		if (mbstowcs(wpat, (const char *)pat, size) == ((size_t) - 1))
-			return (-1);
-		for (i = 0; i < len; i++) {
-			if ((towlower(wpat[i]) == towlower(wdata[i])) ||
-			    ((grepbehave != GREP_FIXED) && wpat[i] == L'.'))
-				continue;
-			free(wpat);
-			free(wdata);
-				return (i);
-		}
-	} else {
-		for (i = 0; i < len; i++) {
-			if ((pat[i] == data[i]) || ((grepbehave != GREP_FIXED) &&
-			    pat[i] == '.'))
-				continue;
-			return (i);
-		}
-	}
-	return (-1);
-}
-
-static inline void
-grep_revstr(unsigned char *str, int len)
-{
-	int i;
-	char c;
-
-	for (i = 0; i < len / 2; i++) {
-		c = str[i];
-		str[i] = str[len - i - 1];
-		str[len - i - 1] = c;
-	}
-}
diff --git a/usr.bin/grep/file.c b/usr.bin/grep/file.c
index a6216e5f07f7..8cee2c05add1 100644
--- a/usr.bin/grep/file.c
+++ b/usr.bin/grep/file.c
@@ -34,13 +34,14 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
-#include <sys/types.h>
+#include <sys/mman.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 
-#include <bzlib.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <lzma.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
@@ -49,17 +50,25 @@ __FBSDID("$FreeBSD$");
 #include <wctype.h>
 #include <zlib.h>
 
+#ifndef WITHOUT_BZIP2
+#include <bzlib.h>
+#endif
+
 #include "grep.h"
 
 #define	MAXBUFSIZ	(32 * 1024)
 #define	LNBUFBUMP	80
 
 static gzFile gzbufdesc;
+static lzma_stream lstrm = LZMA_STREAM_INIT;
+#ifndef WITHOUT_BZIP2
 static BZFILE* bzbufdesc;
+#endif
 
-static unsigned char buffer[MAXBUFSIZ];
+static unsigned char *buffer;
 static unsigned char *bufpos;
 static size_t bufrem;
+static size_t fsiz;
 
 static unsigned char *lnbuf;
 static size_t lnbuflen;
@@ -68,14 +77,19 @@ static inline int
 grep_refill(struct file *f)
 {
 	ssize_t nr;
-	int bzerr;
+
+	if (filebehave == FILE_MMAP)
+		return (0);
 
 	bufpos = buffer;
 	bufrem = 0;
 
-	if (filebehave == FILE_GZIP)
+	if (filebehave == FILE_GZIP) {
 		nr = gzread(gzbufdesc, buffer, MAXBUFSIZ);
-	else if (filebehave == FILE_BZIP && bzbufdesc != NULL) {
+#ifndef WITHOUT_BZIP2
+	} else if (filebehave == FILE_BZIP && bzbufdesc != NULL) {
+		int bzerr;
+
 		nr = BZ2_bzRead(&bzerr, bzbufdesc, buffer, MAXBUFSIZ);
 		switch (bzerr) {
 		case BZ_OK:
@@ -101,6 +115,37 @@ grep_refill(struct file *f)
 			/* Make sure we exit with an error */
 			nr = -1;
 		}
+#endif
+	} else if ((filebehave == FILE_XZ) || (filebehave == FILE_LZMA)) {
+		lzma_action action = LZMA_RUN;
+		uint8_t in_buf[MAXBUFSIZ];
+		lzma_ret ret;
+
+		ret = (filebehave == FILE_XZ) ?
+		    lzma_stream_decoder(&lstrm, UINT64_MAX,
+		    LZMA_CONCATENATED) :
+		    lzma_alone_decoder(&lstrm, UINT64_MAX);
+
+		if (ret != LZMA_OK)
+			return (-1);
+
+		lstrm.next_out = buffer;
+		lstrm.avail_out = MAXBUFSIZ;
+		lstrm.next_in = in_buf;
+		nr = read(f->fd, in_buf, MAXBUFSIZ);
+
+		if (nr < 0)
+			return (-1);
+		else if (nr == 0)
+			action = LZMA_FINISH;
+
+		lstrm.avail_in = nr;
+		ret = lzma_code(&lstrm, action);
+
+		if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+			return (-1);
+		bufrem = MAXBUFSIZ - lstrm.avail_out;
+		return (0);
 	} else
 		nr = read(f->fd, buffer, MAXBUFSIZ);
 
@@ -186,33 +231,6 @@ grep_fgetln(struct file *f, size_t *lenp)
 	return (NULL);
 }
 
-static inline struct file *
-grep_file_init(struct file *f)
-{
-
-	if (filebehave == FILE_GZIP &&
-	    (gzbufdesc = gzdopen(f->fd, "r")) == NULL)
-		goto error;
-
-	if (filebehave == FILE_BZIP &&
-	    (bzbufdesc = BZ2_bzdopen(f->fd, "r")) == NULL)
-		goto error;
-
-	/* Fill read buffer, also catches errors early */
-	if (grep_refill(f) != 0)
-		goto error;
-
-	/* Check for binary stuff, if necessary */
-	if (binbehave != BINFILE_TEXT && memchr(bufpos, '\0', bufrem) != NULL)
-		f->binary = true;
-
-	return (f);
-error:
-	close(f->fd);
-	free(f);
-	return (NULL);
-}
-
 /*
  * Opens a file for processing.
  */
@@ -227,12 +245,61 @@ grep_open(const char *path)
 		/* Processing stdin implies --line-buffered. */
 		lbflag = true;
 		f->fd = STDIN_FILENO;
-	} else if ((f->fd = open(path, O_RDONLY)) == -1) {
-		free(f);
-		return (NULL);
+	} else if ((f->fd = open(path, O_RDONLY)) == -1)
+		goto error1;
+
+	if (filebehave == FILE_MMAP) {
+		struct stat st;
+
+		if ((fstat(f->fd, &st) == -1) || (st.st_size > OFF_MAX) ||
+		    (!S_ISREG(st.st_mode)))
+			filebehave = FILE_STDIO;
+		else {
+			int flags = MAP_PRIVATE | MAP_NOCORE | MAP_NOSYNC;
+#ifdef MAP_PREFAULT_READ
+			flags |= MAP_PREFAULT_READ;
+#endif
+			fsiz = st.st_size;
+			buffer = mmap(NULL, fsiz, PROT_READ, flags,
+			     f->fd, (off_t)0);
+			if (buffer == MAP_FAILED)
+				filebehave = FILE_STDIO;
+			else {
+				bufrem = st.st_size;
+				bufpos = buffer;
+				madvise(buffer, st.st_size, MADV_SEQUENTIAL);
+			}
+		}
 	}
 
-	return (grep_file_init(f));
+	if ((buffer == NULL) || (buffer == MAP_FAILED))
+		buffer = grep_malloc(MAXBUFSIZ);
+
+	if (filebehave == FILE_GZIP &&
+	    (gzbufdesc = gzdopen(f->fd, "r")) == NULL)
+		goto error2;
+
+#ifndef WITHOUT_BZIP2
+	if (filebehave == FILE_BZIP &&
+	    (bzbufdesc = BZ2_bzdopen(f->fd, "r")) == NULL)
+		goto error2;
+#endif
+
+	/* Fill read buffer, also catches errors early */
+	if (bufrem == 0 && grep_refill(f) != 0)
+		goto error2;
+
+	/* Check for binary stuff, if necessary */
+	if (binbehave != BINFILE_TEXT && memchr(bufpos, '\0', bufrem) != NULL)
+	f->binary = true;
+
+	return (f);
+
+error2:
+	close(f->fd);
+error1:
+	free(f);
+	return (NULL);
 }
 
 /*
@@ -245,6 +312,10 @@ grep_close(struct file *f)
 	close(f->fd);
 
 	/* Reset read buffer and line buffer */
+	if (filebehave == FILE_MMAP) {
+		munmap(buffer, fsiz);
+		buffer = NULL;
+	}
 	bufpos = buffer;
 	bufrem = 0;
 
diff --git a/usr.bin/grep/grep.c b/usr.bin/grep/grep.c
index 73e1d3d3371f..288df901727d 100644
--- a/usr.bin/grep/grep.c
+++ b/usr.bin/grep/grep.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <getopt.h>
 #include <limits.h>
 #include <libgen.h>
@@ -48,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
+#include "fastmatch.h"
 #include "grep.h"
 
 #ifndef WITHOUT_NLS
@@ -81,9 +83,9 @@ bool		 matchall;
 
 /* Searching patterns */
 unsigned int	 patterns, pattern_sz;
-char		**pattern;
+struct pat	*pattern;
 regex_t		*r_pattern;
-fastgrep_t	*fg_pattern;
+fastmatch_t	*fg_pattern;
 
 /* Filename exclusion/inclusion patterns */
 unsigned int	 fpatterns, fpattern_sz;
@@ -104,7 +106,7 @@ bool	 hflag;		/* -h: don't print filename headers */
 bool	 iflag;		/* -i: ignore case */
 bool	 lflag;		/* -l: only show names of files with matches */
 bool	 mflag;		/* -m x: stop reading the files after x matches */
-unsigned long long mcount;	/* count for -m */
+long long mcount;	/* count for -m */
 bool	 nflag;		/* -n: show line numbers in front of matching lines */
 bool	 oflag;		/* -o: print only matching part */
 bool	 qflag;		/* -q: quiet mode (don't output anything) */
@@ -148,15 +150,13 @@ bool	 prev;		/* flag whether or not the previous line matched */
 int	 tail;		/* lines left to print */
 bool	 notfound;	/* file not found */
 
-extern char	*__progname;
-
 /*
  * Prints usage information and returns 2.
  */
 static void
 usage(void)
 {
-	fprintf(stderr, getstr(4), __progname);
+	fprintf(stderr, getstr(4), getprogname());
 	fprintf(stderr, "%s", getstr(5));
 	fprintf(stderr, "%s", getstr(5));
 	fprintf(stderr, "%s", getstr(6));
@@ -164,7 +164,7 @@ usage(void)
 	exit(2);
 }
 
-static const char	*optstr = "0123456789A:B:C:D:EFGHIJLOPSRUVZabcd:e:f:hilm:nopqrsuvwxy";
+static const char	*optstr = "0123456789A:B:C:D:EFGHIJMLOPSRUVZabcd:e:f:hilm:nopqrsuvwxXy";
 
 struct option long_options[] =
 {
@@ -200,6 +200,7 @@ struct option long_options[] =
 	{"files-with-matches",	no_argument,		NULL, 'l'},
 	{"files-without-match", no_argument,            NULL, 'L'},
 	{"max-count",		required_argument,	NULL, 'm'},
+	{"lzma",		no_argument,		NULL, 'M'},
 	{"line-number",		no_argument,		NULL, 'n'},
 	{"only-matching",	no_argument,		NULL, 'o'},
 	{"quiet",		no_argument,		NULL, 'q'},
@@ -212,6 +213,7 @@ struct option long_options[] =
 	{"version",		no_argument,		NULL, 'V'},
 	{"word-regexp",		no_argument,		NULL, 'w'},
 	{"line-regexp",		no_argument,		NULL, 'x'},
+	{"xz",			no_argument,		NULL, 'X'},
 	{"decompress",          no_argument,            NULL, 'Z'},
 	{NULL,			no_argument,		NULL, 0}
 };
@@ -223,23 +225,35 @@ static void
 add_pattern(char *pat, size_t len)
 {
 
+	/* Do not add further pattern is we already match everything */
+	if (matchall)
+	  return;
+
 	/* Check if we can do a shortcut */
-	if (len == 0 || matchall) {
+	if (len == 0) {
 		matchall = true;
+		for (unsigned int i = 0; i < patterns; i++) {
+			free(pattern[i].pat);
+		}
+		pattern = grep_realloc(pattern, sizeof(struct pat));
+		pattern[0].pat = NULL;
+		pattern[0].len = 0;
+		patterns = 1;
 		return;
 	}
 	/* Increase size if necessary */
 	if (patterns == pattern_sz) {
 		pattern_sz *= 2;
 		pattern = grep_realloc(pattern, ++pattern_sz *
-		    sizeof(*pattern));
+		    sizeof(struct pat));
 	}
 	if (len > 0 && pat[len - 1] == '\n')
 		--len;
 	/* pat may not be NUL-terminated */
-	pattern[patterns] = grep_malloc(len + 1);
-	memcpy(pattern[patterns], pat, len);
-	pattern[patterns][len] = '\0';
+	pattern[patterns].pat = grep_malloc(len + 1);
+	memcpy(pattern[patterns].pat, pat, len);
+	pattern[patterns].len = len;
+	pattern[patterns].pat[len] = '\0';
 	++patterns;
 }
 
@@ -285,14 +299,19 @@ add_dpattern(const char *pat, int mode)
 static void
 read_patterns(const char *fn)
 {
+	struct stat st;
 	FILE *f;
 	char *line;
 	size_t len;
 
 	if ((f = fopen(fn, "r")) == NULL)
 		err(2, "%s", fn);
-	while ((line = fgetln(f, &len)) != NULL)
-		add_pattern(line, *line == '\n' ? 0 : len);
+	if ((fstat(fileno(f), &st) == -1) || (S_ISDIR(st.st_mode))) {
+		fclose(f);
+		return;
+	}
+        while ((line = fgetln(f, &len)) != NULL)
+		add_pattern(line, line[0] == '\n' ? 0 : len);
 	if (ferror(f))
 		err(2, "%s", fn);
 	fclose(f);
@@ -312,6 +331,7 @@ main(int argc, char *argv[])
 {
 	char **aargv, **eargv, *eopts;
 	char *ep;
+	const char *pn;
 	unsigned long long l;
 	unsigned int aargc, eargc, i;
 	int c, lastc, needpattern, newarg, prevoptind;
@@ -325,30 +345,27 @@ main(int argc, char *argv[])
 	/* Check what is the program name of the binary.  In this
 	   way we can have all the funcionalities in one binary
 	   without the need of scripting and using ugly hacks. */
-	switch (__progname[0]) {
+	pn = getprogname();
+	if (pn[0] == 'b' && pn[1] == 'z') {
+		filebehave = FILE_BZIP;
+		pn += 2;
+	} else if (pn[0] == 'x' && pn[1] == 'z') {
+		filebehave = FILE_XZ;
+		pn += 2;
+	} else if (pn[0] == 'l' && pn[1] == 'z') {
+		filebehave = FILE_LZMA;
+		pn += 2;
+	} else if (pn[0] == 'z') {
+		filebehave = FILE_GZIP;
+		pn += 1;
+	}
+	switch (pn[0]) {
 	case 'e':
 		grepbehave = GREP_EXTENDED;
 		break;
 	case 'f':
 		grepbehave = GREP_FIXED;
 		break;
-	case 'g':
-		grepbehave = GREP_BASIC;
-		break;
-	case 'z':
-		filebehave = FILE_GZIP;
-		switch(__progname[1]) {
-		case 'e':
-			grepbehave = GREP_EXTENDED;
-			break;
-		case 'f':
-			grepbehave = GREP_FIXED;
-			break;
-		case 'g':
-			grepbehave = GREP_BASIC;
-			break;
-		}
-		break;
 	}
 
 	lastc = '\0';
@@ -490,6 +507,10 @@ main(int argc, char *argv[])
 			cflags |= REG_ICASE;
 			break;
 		case 'J':
+#ifdef WITHOUT_BZIP2
+			errno = EOPNOTSUPP;
+			err(2, "bzip2 support was disabled at compile-time");
+#endif
 			filebehave = FILE_BZIP;
 			break;
 		case 'L':
@@ -503,8 +524,8 @@ main(int argc, char *argv[])
 		case 'm':
 			mflag = true;
 			errno = 0;
-			mcount = strtoull(optarg, &ep, 10);
-			if (((errno == ERANGE) && (mcount == ULLONG_MAX)) ||
+			mcount = strtoll(optarg, &ep, 10);
+			if (((errno == ERANGE) && (mcount == LLONG_MAX)) ||
 			    ((errno == EINVAL) && (mcount == 0)))
 				err(2, NULL);
 			else if (ep[0] != '\0') {
@@ -512,6 +533,9 @@ main(int argc, char *argv[])
 				err(2, NULL);
 			}
 			break;
+		case 'M':
+			filebehave = FILE_LZMA;
+			break;
 		case 'n':
 			nflag = true;
 			break;
@@ -544,10 +568,10 @@ main(int argc, char *argv[])
 			break;
 		case 'u':
 		case MMAP_OPT:
-			/* noop, compatibility */
+			filebehave = FILE_MMAP;
 			break;
 		case 'V':
-			printf(getstr(9), __progname, VERSION);
+			printf(getstr(9), getprogname(), VERSION);
 			exit(0);
 		case 'v':
 			vflag = true;
@@ -560,6 +584,9 @@ main(int argc, char *argv[])
 			xflag = true;
 			cflags &= ~REG_NOSUB;
 			break;
+		case 'X':
+			filebehave = FILE_XZ;
+			break;
 		case 'Z':
 			filebehave = FILE_GZIP;
 			break;
@@ -630,6 +657,10 @@ main(int argc, char *argv[])
 	aargc -= optind;
 	aargv += optind;
 
+	/* Empty pattern file matches nothing */
+	if (!needpattern && (patterns == 0))
+		exit(1);
+
 	/* Fail if we don't have any pattern */
 	if (aargc == 0 && needpattern)
 		usage();
@@ -642,9 +673,12 @@ main(int argc, char *argv[])
 	}
 
 	switch (grepbehave) {
-	case GREP_FIXED:
 	case GREP_BASIC:
 		break;
+	case GREP_FIXED:
+		/* XXX: header mess, REG_LITERAL not defined in gnu/regex.h */
+		cflags |= 0020;
+		break;
 	case GREP_EXTENDED:
 		cflags |= REG_EXTENDED;
 		break;
@@ -655,24 +689,17 @@ main(int argc, char *argv[])
 
 	fg_pattern = grep_calloc(patterns, sizeof(*fg_pattern));
 	r_pattern = grep_calloc(patterns, sizeof(*r_pattern));
-/*
- * XXX: fgrepcomp() and fastcomp() are workarounds for regexec() performance.
- * Optimizations should be done there.
- */
-		/* Check if cheating is allowed (always is for fgrep). */
-	if (grepbehave == GREP_FIXED) {
-		for (i = 0; i < patterns; ++i)
-			fgrepcomp(&fg_pattern[i], pattern[i]);
-	} else {
-		for (i = 0; i < patterns; ++i) {
-			if (fastcomp(&fg_pattern[i], pattern[i])) {
-				/* Fall back to full regex library */
-				c = regcomp(&r_pattern[i], pattern[i], cflags);
-				if (c != 0) {
-					regerror(c, &r_pattern[i], re_error,
-					    RE_ERROR_BUF);
-					errx(2, "%s", re_error);
-				}
+
+	/* Check if cheating is allowed (always is for fgrep). */
+	for (i = 0; i < patterns; ++i) {
+		if (fastncomp(&fg_pattern[i], pattern[i].pat,
+		    pattern[i].len, cflags) != 0) {
+			/* Fall back to full regex library */
+			c = regcomp(&r_pattern[i], pattern[i].pat, cflags);
+			if (c != 0) {
+				regerror(c, &r_pattern[i], re_error,
+				    RE_ERROR_BUF);
+				errx(2, "%s", re_error);
 			}
 		}
 	}
diff --git a/usr.bin/grep/grep.h b/usr.bin/grep/grep.h
index fe15c1781c09..47d4ab937a26 100644
--- a/usr.bin/grep/grep.h
+++ b/usr.bin/grep/grep.h
@@ -36,6 +36,8 @@
 #include <stdio.h>
 #include <zlib.h>
 
+#include "fastmatch.h"
+
 #ifdef WITHOUT_NLS
 #define getstr(n)	 errstr[n]
 #else
@@ -58,8 +60,11 @@ extern const char		*errstr[];
 #define BINFILE_TEXT	2
 
 #define FILE_STDIO	0
-#define FILE_GZIP	1
-#define FILE_BZIP	2
+#define FILE_MMAP	1
+#define FILE_GZIP	2
+#define FILE_BZIP	3
+#define FILE_XZ		4
+#define FILE_LZMA	5
 
 #define DIR_READ	0
 #define DIR_SKIP	1
@@ -90,22 +95,16 @@ struct str {
 	int		 line_no;
 };
 
+struct pat {
+	char		*pat;
+	int		 len;
+};
+
 struct epat {
 	char		*pat;
 	int		 mode;
 };
 
-typedef struct {
-	size_t		 len;
-	unsigned char	*pattern;
-	int		 qsBc[UCHAR_MAX + 1];
-	/* flags */
-	bool		 bol;
-	bool		 eol;
-	bool		 reversed;
-	bool		 word;
-} fastgrep_t;
-
 /* Flags passed to regcomp() and regexec() */
 extern int	 cflags, eflags;
 
@@ -114,7 +113,8 @@ extern bool	 Eflag, Fflag, Gflag, Hflag, Lflag,
 		 bflag, cflag, hflag, iflag, lflag, mflag, nflag, oflag,
 		 qflag, sflag, vflag, wflag, xflag;
 extern bool	 dexclude, dinclude, fexclude, finclude, lbflag, nullflag;
-extern unsigned long long Aflag, Bflag, mcount;
+extern unsigned long long Aflag, Bflag;
+extern long long mcount;
 extern char	*label;
 extern const char *color;
 extern int	 binbehave, devbehave, dirbehave, filebehave, grepbehave, linkbehave;
@@ -122,10 +122,10 @@ extern int	 binbehave, devbehave, dirbehave, filebehave, grepbehave, linkbehave;
 extern bool	 first, matchall, notfound, prev;
 extern int	 tail;
 extern unsigned int dpatterns, fpatterns, patterns;
-extern char    **pattern;
+extern struct pat *pattern;
 extern struct epat *dpattern, *fpattern;
 extern regex_t	*er_pattern, *r_pattern;
-extern fastgrep_t *fg_pattern;
+extern fastmatch_t *fg_pattern;
 
 /* For regex errors  */
 #define RE_ERROR_BUF	512
@@ -150,8 +150,3 @@ void	 clearqueue(void);
 void		 grep_close(struct file *f);
 struct file	*grep_open(const char *path);
 char		*grep_fgetln(struct file *f, size_t *len);
-
-/* fastgrep.c */
-int		 fastcomp(fastgrep_t *, const char *);
-void		 fgrepcomp(fastgrep_t *, const char *);
-int		 grep_search(fastgrep_t *, const unsigned char *, size_t, regmatch_t *);
diff --git a/usr.bin/grep/regex/fastmatch.c b/usr.bin/grep/regex/fastmatch.c
new file mode 100644
index 000000000000..990d847490ad
--- /dev/null
+++ b/usr.bin/grep/regex/fastmatch.c
@@ -0,0 +1,169 @@
+/* $FreeBSD$ */
+
+/*-
+ * Copyright (C) 2011 Gabor Kovesdan <gabor@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "glue.h"
+
+#include <errno.h>
+#include <fastmatch.h>
+#include <regex.h>
+#include <string.h>
+
+#include "tre-fastmatch.h"
+#include "xmalloc.h"
+
+int
+tre_fixncomp(fastmatch_t *preg, const char *regex, size_t n, int cflags)
+{
+  int ret;
+  tre_char_t *wregex;
+  size_t wlen;
+
+  if (n != 0)
+    {
+      ret = tre_convert_pattern(regex, n, &wregex, &wlen);
+      if (ret != REG_OK)
+	return ret;
+      else 
+	ret = tre_compile_literal(preg, wregex, wlen, cflags);
+      tre_free_pattern(wregex);
+      return ret;
+    }
+  else
+    return tre_compile_literal(preg, NULL, 0, cflags);
+}
+
+int
+tre_fastncomp(fastmatch_t *preg, const char *regex, size_t n, int cflags)
+{
+  int ret;
+  tre_char_t *wregex;
+  size_t wlen;
+
+  if (n != 0)
+    {
+      ret = tre_convert_pattern(regex, n, &wregex, &wlen);
+      if (ret != REG_OK)
+	return ret;
+      else
+	ret = (cflags & REG_LITERAL)
+	      ? tre_compile_literal(preg, wregex, wlen, cflags)
+	      : tre_compile_fast(preg, wregex, wlen, cflags);
+      tre_free_pattern(wregex);
+      return ret;
+    }
+  else
+    return tre_compile_literal(preg, NULL, 0, cflags);
+}
+
+
+int
+tre_fixcomp(fastmatch_t *preg, const char *regex, int cflags)
+{
+  return tre_fixncomp(preg, regex, regex ? strlen(regex) : 0, cflags);
+}
+
+int
+tre_fastcomp(fastmatch_t *preg, const char *regex, int cflags)
+{
+  return tre_fastncomp(preg, regex, regex ? strlen(regex) : 0, cflags);
+}
+
+int
+tre_fixwncomp(fastmatch_t *preg, const wchar_t *regex, size_t n, int cflags)
+{
+  return tre_compile_literal(preg, regex, n, cflags);
+}
+
+int
+tre_fastwncomp(fastmatch_t *preg, const wchar_t *regex, size_t n, int cflags)
+{
+  return (cflags & REG_LITERAL) ?
+    tre_compile_literal(preg, regex, n, cflags) :
+    tre_compile_fast(preg, regex, n, cflags);
+}
+
+int
+tre_fixwcomp(fastmatch_t *preg, const wchar_t *regex, int cflags)
+{
+  return tre_fixwncomp(preg, regex, regex ? tre_strlen(regex) : 0, cflags);
+}
+
+int
+tre_fastwcomp(fastmatch_t *preg, const wchar_t *regex, int cflags)
+{
+  return tre_fastwncomp(preg, regex, regex ? tre_strlen(regex) : 0, cflags);
+}
+
+void
+tre_fastfree(fastmatch_t *preg)
+{
+  tre_free_fast(preg);
+}
+
+int
+tre_fastnexec(const fastmatch_t *preg, const char *string, size_t len,
+         size_t nmatch, regmatch_t pmatch[], int eflags)
+{
+  tre_str_type_t type = (TRE_MB_CUR_MAX == 1) ? STR_BYTE : STR_MBS;
+
+  if (eflags & REG_STARTEND)
+    CALL_WITH_OFFSET(tre_match_fast(preg, &string[offset], slen,
+		     type, nmatch, pmatch, eflags));
+  else
+    return tre_match_fast(preg, string, len, type, nmatch,
+      pmatch, eflags);
+}
+
+int
+tre_fastexec(const fastmatch_t *preg, const char *string, size_t nmatch,
+	     regmatch_t pmatch[], int eflags)
+{
+  return tre_fastnexec(preg, string, (size_t)-1, nmatch, pmatch, eflags);
+}
+
+int
+tre_fastwnexec(const fastmatch_t *preg, const wchar_t *string, size_t len,
+          size_t nmatch, regmatch_t pmatch[], int eflags)
+{
+  tre_str_type_t type = STR_WIDE;
+
+  if (eflags & REG_STARTEND)
+    CALL_WITH_OFFSET(tre_match_fast(preg, &string[offset], slen,
+		     type, nmatch, pmatch, eflags));
+  else
+    return tre_match_fast(preg, string, len, type, nmatch,
+      pmatch, eflags);
+}
+
+int
+tre_fastwexec(const fastmatch_t *preg, const wchar_t *string,
+         size_t nmatch, regmatch_t pmatch[], int eflags)
+{
+  return tre_fastwnexec(preg, string, (size_t)-1, nmatch, pmatch, eflags);
+}
+
diff --git a/usr.bin/grep/regex/fastmatch.h b/usr.bin/grep/regex/fastmatch.h
new file mode 100644
index 000000000000..68a50c60e446
--- /dev/null
+++ b/usr.bin/grep/regex/fastmatch.h
@@ -0,0 +1,108 @@
+/* $FreeBSD$ */
+
+#ifndef FASTMATCH_H
+#define FASTMATCH_H 1
+
+#include <limits.h>
+#include <regex.h>
+#include <stdbool.h>
+#include <wchar.h>
+
+typedef struct {
+  size_t	 wlen;
+  size_t	 len;
+  wchar_t	*wpattern;
+  bool		*wescmap;
+  unsigned int	 qsBc[UCHAR_MAX + 1];
+  unsigned int	*bmGs;
+  char		*pattern;
+  bool		*escmap;
+  unsigned int	 defBc;
+  void		*qsBc_table;
+  unsigned int	*sbmGs;
+  const char	*re_endp;
+
+  /* flags */
+  bool		 hasdot;
+  bool		 bol;
+  bool		 eol;
+  bool		 word;
+  bool		 icase;
+  bool		 newline;
+  bool		 nosub;
+  bool		 matchall;
+  bool		 reversed;
+} fastmatch_t;
+
+extern int
+tre_fixcomp(fastmatch_t *preg, const char *regex, int cflags);
+
+extern int
+tre_fastcomp(fastmatch_t *preg, const char *regex, int cflags);
+
+extern int
+tre_fastexec(const fastmatch_t *preg, const char *string, size_t nmatch,
+  regmatch_t pmatch[], int eflags);
+
+extern void
+tre_fastfree(fastmatch_t *preg);
+
+extern int
+tre_fixwcomp(fastmatch_t *preg, const wchar_t *regex, int cflags);
+
+extern int
+tre_fastwcomp(fastmatch_t *preg, const wchar_t *regex, int cflags);
+
+extern int
+tre_fastwexec(const fastmatch_t *preg, const wchar_t *string,
+         size_t nmatch, regmatch_t pmatch[], int eflags);
+
+/* Versions with a maximum length argument and therefore the capability to
+   handle null characters in the middle of the strings. */
+extern int
+tre_fixncomp(fastmatch_t *preg, const char *regex, size_t len, int cflags);
+
+extern int
+tre_fastncomp(fastmatch_t *preg, const char *regex, size_t len, int cflags);
+
+extern int
+tre_fastnexec(const fastmatch_t *preg, const char *string, size_t len,
+  size_t nmatch, regmatch_t pmatch[], int eflags);
+
+extern int
+tre_fixwncomp(fastmatch_t *preg, const wchar_t *regex, size_t len, int cflags);
+
+extern int
+tre_fastwncomp(fastmatch_t *preg, const wchar_t *regex, size_t len, int cflags);
+
+extern int
+tre_fastwnexec(const fastmatch_t *preg, const wchar_t *string, size_t len,
+  size_t nmatch, regmatch_t pmatch[], int eflags);
+
+#define fixncomp	tre_fixncomp
+#define fastncomp	tre_fastncomp
+#define fixcomp		tre_fixcomp
+#define fastcomp	tre_fastcomp
+#define fixwncomp	tre_fixwncomp
+#define fastwncomp	tre_fastwncomp
+#define fixwcomp	tre_fixwcomp
+#define fastwcomp	tre_fastwcomp
+#define fastfree	tre_fastfree
+#define fastnexec	tre_fastnexec
+#define fastexec	tre_fastexec
+#define fastwnexec	tre_fastwnexec
+#define fastwexec	tre_fastwexec
+#define fixcomp		tre_fixcomp
+#define fastcomp	tre_fastcomp
+#define fastexec	tre_fastexec
+#define fastfree	tre_fastfree
+#define fixwcomp	tre_fixwcomp
+#define fastwcomp	tre_fastwcomp
+#define fastwexec	tre_fastwexec
+#define fixncomp	tre_fixncomp
+#define fastncomp	tre_fastncomp
+#define fastnexec	tre_fastnexec
+#define fixwncomp	tre_fixwncomp
+#define fastwncomp	tre_fastwncomp
+#define fastwnexec	tre_fastwnexec
+#endif		/* FASTMATCH_H */
diff --git a/usr.bin/grep/regex/glue.h b/usr.bin/grep/regex/glue.h
new file mode 100644
index 000000000000..2fea4fd542c2
--- /dev/null
+++ b/usr.bin/grep/regex/glue.h
@@ -0,0 +1,67 @@
+/* $FreeBSD$ */
+
+#ifndef GLUE_H
+#define GLUE_H
+
+#include <limits.h>
+#undef RE_DUP_MAX
+#include <regex.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define TRE_WCHAR			1
+#define TRE_MULTIBYTE			1
+#define HAVE_MBSTATE_T			1
+
+#define TRE_CHAR(n) L##n
+#define CHF "%lc"
+
+#define tre_char_t			wchar_t
+#define tre_mbrtowc(pwc, s, n, ps)	(mbrtowc((pwc), (s), (n), (ps)))
+#define tre_strlen			wcslen
+#define tre_isspace			iswspace
+#define tre_isalnum			iswalnum
+
+#define REG_OK				0
+#define REG_LITERAL			0020
+#define REG_WORD			0100
+#define REG_GNU				0400
+
+#define TRE_MB_CUR_MAX			MB_CUR_MAX
+
+#ifndef _GREP_DEBUG
+#define DPRINT(msg)
+#else			
+#define DPRINT(msg) do {printf msg; fflush(stdout);} while(/*CONSTCOND*/0)
+#endif
+
+#define MIN(a,b)			((a > b) ? (b) : (a))
+#define MAX(a,b)			((a > b) ? (a) : (b))
+
+typedef enum { STR_WIDE, STR_BYTE, STR_MBS, STR_USER } tre_str_type_t;
+
+#define CALL_WITH_OFFSET(fn)						\
+  do									\
+    {									\
+      size_t slen = (size_t)(pmatch[0].rm_eo - pmatch[0].rm_so);	\
+      size_t offset = pmatch[0].rm_so;					\
+      int ret;								\
+									\
+      if ((long long)pmatch[0].rm_eo - pmatch[0].rm_so < 0)		\
+	return REG_NOMATCH;						\
+      ret = fn;								\
+      for (unsigned i = 0; (!(eflags & REG_NOSUB) && (i < nmatch)); i++)\
+	{								\
+	  pmatch[i].rm_so += offset;					\
+	  pmatch[i].rm_eo += offset;					\
+	}								\
+      return ret;							\
+    } while (0 /*CONSTCOND*/)
+
+int
+tre_convert_pattern(const char *regex, size_t n, tre_char_t **w,
+    size_t *wn);
+
+void
+tre_free_pattern(tre_char_t *wregex);
+#endif
diff --git a/usr.bin/grep/regex/hashtable.c b/usr.bin/grep/regex/hashtable.c
new file mode 100644
index 000000000000..41ebcd1a0479
--- /dev/null
+++ b/usr.bin/grep/regex/hashtable.c
@@ -0,0 +1,268 @@
+/*      $FreeBSD$       */
+
+/*-
+ * Copyright (C) 2011 Gabor Kovesdan <gabor@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "glue.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "hashtable.h"
+
+
+/*
+ * Return a 32-bit hash of the given buffer.  The init
+ * value should be 0, or the previous hash value to extend
+ * the previous hash.
+ */
+static uint32_t
+hash32_buf(const void *buf, size_t len, uint32_t hash)
+{
+  const unsigned char *p = buf;
+
+  while (len--)
+    hash = HASHSTEP(hash, *p++);
+
+  return hash;
+}
+
+/*
+ * Initializes a hash table that can hold table_size number of entries,
+ * each of which has a key of key_size bytes and a value of value_size
+ * bytes. On successful allocation returns a pointer to the hash table.
+ * Otherwise, returns NULL and sets errno to indicate the error.
+ */
+hashtable
+*hashtable_init(size_t table_size, size_t key_size, size_t value_size)
+{
+  hashtable *tbl;
+
+  DPRINT(("hashtable_init: table_size %zu, key_size %zu, value_size %zu\n",
+	  table_size, key_size, value_size));
+
+  tbl = malloc(sizeof(hashtable));
+  if (tbl == NULL)
+    goto mem1;
+
+  tbl->entries = calloc(sizeof(hashtable_entry *), table_size);
+  if (tbl->entries == NULL)
+    goto mem2;
+
+  tbl->table_size = table_size;
+  tbl->usage = 0;
+  tbl->key_size = key_size;
+  tbl->value_size = value_size;
+
+  return (tbl);
+
+mem2:
+  free(tbl);
+mem1:
+  DPRINT(("hashtable_init: allocation failed\n"));
+  errno = ENOMEM;
+  return (NULL);
+}
+
+/*
+ * Places the key-value pair to the hashtable tbl.
+ * Returns:
+ *   HASH_OK:		if the key was not present in the hash table yet
+ *			but the kay-value pair has been successfully added.
+ *   HASH_UPDATED:	if the value for the key has been updated with the
+ *			new value.
+ *   HASH_FULL:		if the hash table is full and the entry could not
+ *			be added.
+ *   HASH_FAIL:		if an error has occurred and errno has been set to
+ *			indicate the error.
+ */
+int
+hashtable_put(hashtable *tbl, const void *key, const void *value)
+{
+  uint32_t hash = 0;
+
+  if (tbl->table_size == tbl->usage)
+    {
+      DPRINT(("hashtable_put: hashtable is full\n"));
+      return (HASH_FULL);
+    }
+
+  hash = hash32_buf(key, tbl->key_size, hash) % tbl->table_size;
+  DPRINT(("hashtable_put: calculated hash %" PRIu32 "\n", hash));
+
+  /*
+   * On hash collision entries are inserted at the next free space,
+   * so we have to increase the index until we either find an entry
+   * with the same key (and update it) or we find a free space.
+   */
+  for(;;)
+    {
+      if (tbl->entries[hash] == NULL)
+	break;
+      else if (memcmp(tbl->entries[hash]->key, key, tbl->key_size) == 0)
+	{
+	  memcpy(tbl->entries[hash]->value, value, tbl->value_size);
+	  DPRINT(("hashtable_put: effective location is %" PRIu32
+		  ", entry updated\n", hash));
+	  return (HASH_UPDATED);
+	}
+      if (++hash == tbl->table_size)
+	hash = 0;
+    }
+
+  DPRINT(("hashtable_put: effective location is %" PRIu32 "\n", hash));
+
+  tbl->entries[hash] = malloc(sizeof(hashtable_entry));
+  if (tbl->entries[hash] == NULL)
+    {
+      errno = ENOMEM;
+      goto mem1;
+    }
+
+  tbl->entries[hash]->key = malloc(tbl->key_size);
+  if (tbl->entries[hash]->key == NULL)
+    {
+      errno = ENOMEM;
+      goto mem2;
+    }
+
+  tbl->entries[hash]->value = malloc(tbl->value_size);
+  if (tbl->entries[hash]->value == NULL)
+    {
+      errno = ENOMEM;
+      goto mem3;
+    }
+
+  memcpy(tbl->entries[hash]->key, key, tbl->key_size);
+  memcpy(tbl->entries[hash]->value, value, tbl->value_size);
+  tbl->usage++;
+
+  DPRINT(("hashtable_put: entry successfully inserted\n"));
+
+  return (HASH_OK);
+
+mem3:
+  free(tbl->entries[hash]->key);
+mem2:
+  free(tbl->entries[hash]);
+mem1:
+  DPRINT(("hashtable_put: insertion failed\n"));
+  return (HASH_FAIL);
+}
+
+static hashtable_entry
+**hashtable_lookup(const hashtable *tbl, const void *key)
+{
+  uint32_t hash = 0;
+
+  hash = hash32_buf(key, tbl->key_size, hash) % tbl->table_size;
+
+  for (;;)
+    {
+      if (tbl->entries[hash] == NULL)
+	return (NULL);
+      else if (memcmp(key, tbl->entries[hash]->key, tbl->key_size) == 0)
+	{
+	  DPRINT(("hashtable_lookup: entry found at location %" PRIu32 "\n", hash));
+	  return (&tbl->entries[hash]);
+	}
+
+      if (++hash == tbl->table_size)
+	hash = 0;
+    }
+}
+
+/*
+ * Retrieves the value for key from the hash table tbl and places
+ * it to the space indicated by the value argument.
+ * Returns HASH_OK if the value has been found and retrieved or
+ * HASH_NOTFOUND otherwise.
+ */
+int
+hashtable_get(hashtable *tbl, const void *key, void *value)
+{
+  hashtable_entry **entry;
+
+  entry = hashtable_lookup(tbl, key);
+  if (entry == NULL)
+    {
+      DPRINT(("hashtable_get: entry is not available in the hashtable\n"));
+      return (HASH_NOTFOUND);
+    }
+
+  memcpy(value, (*entry)->value, tbl->value_size);
+  DPRINT(("hashtable_get: entry successfully copied into output buffer\n"));
+  return (HASH_OK);
+}
+
+/*
+ * Removes the entry with the specifified key from the hash table
+ * tbl. Returns HASH_OK if the entry has been found and removed
+ * or HASH_NOTFOUND otherwise.
+ */
+int
+hashtable_remove(hashtable *tbl, const void *key)
+{
+  hashtable_entry **entry;
+
+  entry = hashtable_lookup(tbl, key);
+  if (entry == NULL)
+    {
+      DPRINT(("hashtable_remove: entry is not available in the hashtable\n"));
+      return (HASH_NOTFOUND);
+    }
+
+  free((*entry)->key);
+  free((*entry)->value);
+  free(*entry);
+  *entry = NULL;
+
+  tbl->usage--;
+  DPRINT(("hashtable_remove: entry successfully removed\n"));
+  return (HASH_OK);
+}
+
+/*
+ * Frees the resources associated with the hash table tbl.
+ */
+void
+hashtable_free(hashtable *tbl)
+{
+  if (tbl == NULL)
+    return;
+
+  for (unsigned int i = 0; i < tbl->table_size; i++)
+    if ((tbl->entries[i] != NULL))
+      {
+	free(tbl->entries[i]->key);
+	free(tbl->entries[i]->value);
+      }
+
+  free(tbl->entries);
+  DPRINT(("hashtable_free: resources are successfully freed\n"));
+}
diff --git a/usr.bin/grep/regex/hashtable.h b/usr.bin/grep/regex/hashtable.h
new file mode 100644
index 000000000000..f8b1daa510b6
--- /dev/null
+++ b/usr.bin/grep/regex/hashtable.h
@@ -0,0 +1,35 @@
+/* $FreeBSD$ */
+
+#ifndef HASHTABLE_H
+#define HASHTABLE_H 1
+
+#include <sys/types.h>
+
+#define HASH_OK         0
+#define HASH_UPDATED    1
+#define HASH_FAIL       2
+#define HASH_FULL       3
+#define HASH_NOTFOUND   4
+
+#define HASHSTEP(x,c) (((x << 5) + x) + (c))
+
+typedef struct {
+  void *key;
+  void *value;
+} hashtable_entry;
+
+typedef struct {
+  size_t key_size;
+  size_t table_size;
+  size_t usage;
+  size_t value_size;
+  hashtable_entry **entries;
+} hashtable;
+
+void hashtable_free(hashtable *);
+int hashtable_get(hashtable *, const void *, void *);
+hashtable *hashtable_init(size_t, size_t, size_t);
+int hashtable_put(hashtable *, const void *, const void *);
+int hashtable_remove(hashtable *, const void *);
+
+#endif	/* HASHTABLE.H */
diff --git a/usr.bin/grep/regex/tre-compile.c b/usr.bin/grep/regex/tre-compile.c
new file mode 100644
index 000000000000..f037c49d954a
--- /dev/null
+++ b/usr.bin/grep/regex/tre-compile.c
@@ -0,0 +1,103 @@
+/*      $FreeBSD$       */
+
+#include "glue.h"
+
+#include <stdio.h>
+#include <assert.h>
+#include <errno.h>
+#include <regex.h>
+#include <string.h>
+#include <wchar.h>
+
+#include "xmalloc.h"
+
+int
+tre_convert_pattern(const char *regex, size_t n, tre_char_t **w,
+		    size_t *wn)
+{
+#if TRE_WCHAR
+  tre_char_t *wregex;
+  size_t wlen;
+
+  wregex = xmalloc(sizeof(tre_char_t) * (n + 1));
+  if (wregex == NULL)
+    return REG_ESPACE;
+
+  /* If the current locale uses the standard single byte encoding of
+     characters, we don't do a multibyte string conversion.  If we did,
+     many applications which use the default locale would break since
+     the default "C" locale uses the 7-bit ASCII character set, and
+     all characters with the eighth bit set would be considered invalid. */
+#if TRE_MULTIBYTE
+  if (TRE_MB_CUR_MAX == 1)
+#endif /* TRE_MULTIBYTE */
+    {
+      unsigned int i;
+      const unsigned char *str = (const unsigned char *)regex;
+      tre_char_t *wstr = wregex;
+
+      for (i = 0; i < n; i++)
+	*(wstr++) = *(str++);
+      wlen = n;
+    }
+#if TRE_MULTIBYTE
+  else
+    {
+      int consumed;
+      tre_char_t *wcptr = wregex;
+#ifdef HAVE_MBSTATE_T
+      mbstate_t state;
+      memset(&state, '\0', sizeof(state));
+#endif /* HAVE_MBSTATE_T */
+      while (n > 0)
+	{
+	  consumed = tre_mbrtowc(wcptr, regex, n, &state);
+
+	  switch (consumed)
+	    {
+	    case 0:
+	      if (*regex == '\0')
+		consumed = 1;
+	      else
+		{
+		  xfree(wregex);
+		  return REG_BADPAT;
+		}
+	      break;
+	    case -1:
+	      DPRINT(("mbrtowc: error %d: %s.\n", errno, strerror(errno)));
+	      xfree(wregex);
+	      return REG_BADPAT;
+	    case -2:
+	      /* The last character wasn't complete.  Let's not call it a
+		 fatal error. */
+	      consumed = n;
+	      break;
+	    }
+	  regex += consumed;
+	  n -= consumed;
+	  wcptr++;
+	}
+      wlen = wcptr - wregex;
+    }
+#endif /* TRE_MULTIBYTE */
+  wregex[wlen] = L'\0';
+  *w = wregex;
+  *wn = wlen;
+  return REG_OK;
+#else /* !TRE_WCHAR */
+  {
+    *w = (tre_char_t * const *)regex;
+    *wn = n;
+    return REG_OK;
+  }
+#endif /* !TRE_WCHAR */
+}
+
+void
+tre_free_pattern(tre_char_t *wregex)
+{
+#if TRE_WCHAR
+  xfree(wregex);
+#endif
+}
diff --git a/usr.bin/grep/regex/tre-fastmatch.c b/usr.bin/grep/regex/tre-fastmatch.c
new file mode 100644
index 000000000000..6f1aec6bf3b7
--- /dev/null
+++ b/usr.bin/grep/regex/tre-fastmatch.c
@@ -0,0 +1,1042 @@
+/* $FreeBSD$ */
+
+/*-
+ * Copyright (c) 1999 James Howard and Dag-Erling Coïdan Smørgrav
+ * Copyright (C) 2008-2011 Gabor Kovesdan <gabor@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "glue.h"
+
+#include <ctype.h>
+#include <limits.h>
+#include <regex.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#ifdef TRE_WCHAR
+#include <wchar.h>
+#include <wctype.h>
+#endif
+
+#include "hashtable.h"
+#include "tre-fastmatch.h"
+#include "xmalloc.h"
+
+static int	fastcmp(const fastmatch_t *fg, const void *data,
+			tre_str_type_t type);
+
+/*
+ * Clean up if pattern compilation fails.
+ */
+#define FAIL_COMP(errcode)						\
+  {									\
+    if (fg->pattern)							\
+      xfree(fg->pattern);						\
+    if (fg->wpattern)							\
+      xfree(fg->wpattern);						\
+    if (fg->qsBc_table)							\
+      hashtable_free(fg->qsBc_table);					\
+    fg = NULL;								\
+    return errcode;							\
+  }
+
+/*
+ * Skips n characters in the input string and assigns the start
+ * address to startptr. Note: as per IEEE Std 1003.1-2008
+ * matching is based on bit pattern not character representations
+ * so we can handle MB strings as byte sequences just like
+ * SB strings.
+ */
+#define SKIP_CHARS(n)							\
+  switch (type)								\
+    {									\
+      case STR_WIDE:							\
+	startptr = str_wide + n;					\
+	break;								\
+      default:								\
+	startptr = str_byte + n;					\
+    }
+
+/*
+ * Converts the wide string pattern to SB/MB string and stores
+ * it in fg->pattern. Sets fg->len to the byte length of the
+ * converted string.
+ */
+#define STORE_MBS_PAT							\
+  {									\
+    size_t siz;								\
+									\
+    siz = wcstombs(NULL, fg->wpattern, 0);				\
+    if (siz == (size_t)-1)						\
+      return REG_BADPAT;						\
+    fg->len = siz;							\
+    fg->pattern = xmalloc(siz + 1);					\
+    if (fg->pattern == NULL)						\
+      return REG_ESPACE;						\
+    wcstombs(fg->pattern, fg->wpattern, siz);				\
+    fg->pattern[siz] = '\0';						\
+  }									\
+
+#define IS_OUT_OF_BOUNDS						\
+  ((!fg->reversed							\
+    ? ((type == STR_WIDE) ? ((j + fg->wlen) > len)			\
+			  : ((j + fg->len) > len))			\
+    : (j < 0)))
+
+/*
+ * Checks whether the new position after shifting in the input string
+ * is out of the bounds and break out from the loop if so.
+ */
+#define CHECKBOUNDS							\
+  if (IS_OUT_OF_BOUNDS)							\
+    break;								\
+
+/*
+ * Shifts in the input string after a mismatch. The position of the
+ * mismatch is stored in the mismatch variable.
+ */
+#define SHIFT								\
+  CHECKBOUNDS;								\
+									\
+  {									\
+    int r = -1;								\
+    unsigned int bc = 0, gs = 0, ts;					\
+									\
+    switch (type)							\
+      {									\
+	case STR_WIDE:							\
+	  if (!fg->hasdot)						\
+	    {								\
+	      if (u != 0 && (unsigned)mismatch == fg->wlen - 1 - shift)	\
+		mismatch -= u;						\
+	      v = fg->wlen - 1 - mismatch;				\
+	      r = hashtable_get(fg->qsBc_table,				\
+		&str_wide[!fg->reversed ? (size_t)j + fg->wlen		\
+			  : (size_t)j - 1], &bc);			\
+	      gs = fg->bmGs[mismatch];					\
+	    }								\
+	    bc = (r == HASH_OK) ? bc : fg->defBc;			\
+	    DPRINT(("tre_fast_match: mismatch on character" CHF ", "	\
+		    "BC %d, GS %d\n",					\
+		    ((const tre_char_t *)startptr)[mismatch + 1],	\
+		    bc, gs));						\
+            break;							\
+	default:							\
+	  if (!fg->hasdot)						\
+	    {								\
+	      if (u != 0 && (unsigned)mismatch == fg->len - 1 - shift)	\
+		mismatch -= u;						\
+	      v = fg->len - 1 - mismatch;				\
+	      gs = fg->sbmGs[mismatch];					\
+	    }								\
+	  bc = fg->qsBc[((const unsigned char *)str_byte)		\
+			[!fg->reversed ? (size_t)j + fg->len		\
+			 : (size_t)j - 1]];				\
+	  DPRINT(("tre_fast_match: mismatch on character %c, "		\
+		 "BC %d, GS %d\n",					\
+		 ((const unsigned char *)startptr)[mismatch + 1],	\
+		 bc, gs));						\
+      }									\
+    if (fg->hasdot)							\
+      shift = bc;							\
+    else								\
+      {									\
+	ts = (u >= v) ? (u - v) : 0;					\
+	shift = MAX(ts, bc);						\
+	shift = MAX(shift, gs);						\
+	if (shift == gs)						\
+	  u = MIN((type == STR_WIDE ? fg->wlen : fg->len) - shift, v);	\
+	else								\
+	  {								\
+	    if (ts < bc)						\
+	      shift = MAX(shift, u + 1);				\
+	    u = 0;							\
+	  }								\
+      }									\
+      DPRINT(("tre_fast_match: shifting %u characters\n", shift));	\
+      j = !fg->reversed ? j + shift : j - shift;			\
+  }
+
+/*
+ * Normal Quick Search would require a shift based on the position the
+ * next character after the comparison is within the pattern.  With
+ * wildcards, the position of the last dot effects the maximum shift
+ * distance.
+ * The closer to the end the wild card is the slower the search.
+ *
+ * Examples:
+ * Pattern    Max shift
+ * -------    ---------
+ * this               5
+ * .his               4
+ * t.is               3
+ * th.s               2
+ * thi.               1
+ */
+
+/*
+ * Fills in the bad character shift array for SB/MB strings.
+ */
+#define FILL_QSBC							\
+  if (fg->reversed)							\
+    {									\
+      _FILL_QSBC_REVERSED						\
+    }									\
+  else									\
+    {									\
+      _FILL_QSBC							\
+    }
+
+#define _FILL_QSBC							\
+  for (unsigned int i = 0; i <= UCHAR_MAX; i++)				\
+    fg->qsBc[i] = fg->len - hasdot;					\
+  for (unsigned int i = hasdot + 1; i < fg->len; i++)			\
+    {									\
+      fg->qsBc[(unsigned char)fg->pattern[i]] = fg->len - i;		\
+      DPRINT(("BC shift for char %c is %zu\n", fg->pattern[i],		\
+	     fg->len - i));						\
+      if (fg->icase)							\
+	{								\
+	  char c = islower((unsigned char)fg->pattern[i]) ?		\
+		   toupper((unsigned char)fg->pattern[i]) :		\
+		   tolower((unsigned char)fg->pattern[i]);		\
+	  fg->qsBc[(unsigned char)c] = fg->len - i;			\
+	  DPRINT(("BC shift for char %c is %zu\n", c, fg->len - i));	\
+	}								\
+    }
+
+#define _FILL_QSBC_REVERSED						\
+  for (unsigned int i = 0; i <= UCHAR_MAX; i++)				\
+    fg->qsBc[i] = firstdot + 1;						\
+  for (int i = firstdot - 1; i >= 0; i--)				\
+    {									\
+      fg->qsBc[(unsigned char)fg->pattern[i]] = i + 1;			\
+      DPRINT(("Reverse BC shift for char %c is %d\n", fg->pattern[i],	\
+	     i + 1));							\
+      if (fg->icase)							\
+        {								\
+          char c = islower((unsigned char)fg->pattern[i]) ?		\
+		   toupper((unsigned char)fg->pattern[i]) :		\
+		   tolower((unsigned char)fg->pattern[i]);		\
+          fg->qsBc[(unsigned char)c] = i + 1;				\
+	  DPRINT(("Reverse BC shift for char %c is %d\n", c, i + 1));	\
+        }								\
+    }
+
+/*
+ * Fills in the bad character shifts into a hastable for wide strings.
+ * With wide characters it is not possible any more to use a normal
+ * array because there are too many characters and we could not
+ * provide enough memory. Fortunately, we only have to store distinct
+ * values for so many characters as the number of distinct characters
+ * in the pattern, so we can store them in a hashtable and store a
+ * default shift value for the rest.
+ */
+#define FILL_QSBC_WIDE							\
+  if (fg->reversed)							\
+    {									\
+      _FILL_QSBC_WIDE_REVERSED						\
+    }									\
+  else									\
+    {									\
+      _FILL_QSBC_WIDE							\
+    }
+
+#define _FILL_QSBC_WIDE							\
+  /* Adjust the shift based on location of the last dot ('.'). */	\
+  fg->defBc = fg->wlen - whasdot;					\
+									\
+  /* Preprocess pattern. */						\
+  fg->qsBc_table = hashtable_init(fg->wlen * (fg->icase ? 8 : 4),	\
+				  sizeof(tre_char_t), sizeof(int));	\
+  if (!fg->qsBc_table)							\
+    FAIL_COMP(REG_ESPACE);						\
+  for (unsigned int i = whasdot + 1; i < fg->wlen; i++)			\
+    {									\
+      int k = fg->wlen - i;						\
+      int r;								\
+									\
+      r = hashtable_put(fg->qsBc_table, &fg->wpattern[i], &k);		\
+      if ((r == HASH_FAIL) || (r == HASH_FULL))				\
+	FAIL_COMP(REG_ESPACE);						\
+      DPRINT(("BC shift for wide char " CHF " is %d\n", fg->wpattern[i],\
+	     k));							\
+      if (fg->icase)							\
+	{								\
+	  tre_char_t wc = iswlower(fg->wpattern[i]) ?			\
+	    towupper(fg->wpattern[i]) : towlower(fg->wpattern[i]);	\
+	  r = hashtable_put(fg->qsBc_table, &wc, &k);			\
+	  if ((r == HASH_FAIL) || (r == HASH_FULL))			\
+	    FAIL_COMP(REG_ESPACE);					\
+	  DPRINT(("BC shift for wide char " CHF " is %d\n", wc, k));	\
+	}								\
+    }
+
+#define _FILL_QSBC_WIDE_REVERSED					\
+  /* Adjust the shift based on location of the last dot ('.'). */	\
+  fg->defBc = (size_t)wfirstdot;					\
+									\
+  /* Preprocess pattern. */						\
+  fg->qsBc_table = hashtable_init(fg->wlen * (fg->icase ? 8 : 4),	\
+				  sizeof(tre_char_t), sizeof(int));	\
+  if (!fg->qsBc_table)							\
+    FAIL_COMP(REG_ESPACE);						\
+  for (int i = wfirstdot - 1; i >= 0; i--)				\
+    {									\
+      int k = i + 1;							\
+      int r;								\
+									\
+      r = hashtable_put(fg->qsBc_table, &fg->wpattern[i], &k);		\
+      if ((r == HASH_FAIL) || (r == HASH_FULL))				\
+	FAIL_COMP(REG_ESPACE);						\
+      DPRINT(("Reverse BC shift for wide char " CHF " is %d\n",		\
+	     fg->wpattern[i], k));					\
+      if (fg->icase)							\
+	{								\
+	  tre_char_t wc = iswlower(fg->wpattern[i]) ?			\
+	    towupper(fg->wpattern[i]) : towlower(fg->wpattern[i]);	\
+	  r = hashtable_put(fg->qsBc_table, &wc, &k);			\
+	  if ((r == HASH_FAIL) || (r == HASH_FULL))			\
+	    FAIL_COMP(REG_ESPACE);					\
+	  DPRINT(("Reverse BC shift for wide char " CHF " is %d\n", wc,	\
+		 k));							\
+	}								\
+    }
+
+#ifdef _GREP_DEBUG
+#define DPRINT_BMGS(len, fmt_str, sh)					\
+  for (unsigned int i = 0; i < len; i++)				\
+    DPRINT((fmt_str, i, sh[i]));
+#else
+#define DPRINT_BMGS(len, fmt_str, sh)					\
+  do { } while(/*CONSTCOND*/0)
+#endif
+
+/*
+ * Fills in the good suffix table for SB/MB strings.
+ */
+#define FILL_BMGS							\
+  if (!fg->hasdot)							\
+    {									\
+      fg->sbmGs = xmalloc(fg->len * sizeof(int));			\
+      if (!fg->sbmGs)							\
+	return REG_ESPACE;						\
+      if (fg->len == 1)							\
+	fg->sbmGs[0] = 1;						\
+      else								\
+	_FILL_BMGS(fg->sbmGs, fg->pattern, fg->len, false);		\
+      DPRINT_BMGS(fg->len, "GS shift for pos %d is %d\n", fg->sbmGs);	\
+    }
+
+/*
+ * Fills in the good suffix table for wide strings.
+ */
+#define FILL_BMGS_WIDE							\
+  if (!fg->hasdot)							\
+    {									\
+      fg->bmGs = xmalloc(fg->wlen * sizeof(int));			\
+      if (!fg->bmGs)							\
+	return REG_ESPACE;						\
+      if (fg->wlen == 1)						\
+	fg->bmGs[0] = 1;						\
+      else								\
+	_FILL_BMGS(fg->bmGs, fg->wpattern, fg->wlen, true);		\
+      DPRINT_BMGS(fg->wlen, "GS shift (wide) for pos %d is %d\n",	\
+		  fg->bmGs);						\
+    }
+
+#define _FILL_BMGS(arr, pat, plen, wide)				\
+  {									\
+    char *p;								\
+    tre_char_t *wp;							\
+									\
+    if (wide)								\
+      {									\
+	if (fg->icase)							\
+	  {								\
+	    wp = xmalloc(plen * sizeof(tre_char_t));			\
+	    if (wp == NULL)						\
+	      return REG_ESPACE;					\
+	    for (unsigned int i = 0; i < plen; i++)			\
+	      wp[i] = towlower(pat[i]);					\
+	    _CALC_BMGS(arr, wp, plen);					\
+	    xfree(wp);							\
+	  }								\
+	else								\
+	  _CALC_BMGS(arr, pat, plen);					\
+      }									\
+    else								\
+      {									\
+	if (fg->icase)							\
+	  {								\
+	    p = xmalloc(plen);						\
+	    if (p == NULL)						\
+	      return REG_ESPACE;					\
+	    for (unsigned int i = 0; i < plen; i++)			\
+	      p[i] = tolower(pat[i]);					\
+	    _CALC_BMGS(arr, p, plen);					\
+	    xfree(p);							\
+	  }								\
+	else								\
+	  _CALC_BMGS(arr, pat, plen);					\
+      }									\
+  }
+
+#define _CALC_BMGS(arr, pat, plen)					\
+  {									\
+    int f = 0, g;							\
+									\
+    int *suff = xmalloc(plen * sizeof(int));				\
+    if (suff == NULL)							\
+      return REG_ESPACE;						\
+									\
+    suff[plen - 1] = plen;						\
+    g = plen - 1;							\
+    for (int i = plen - 2; i >= 0; i--)					\
+      {									\
+	if (i > g && suff[i + plen - 1 - f] < i - g)			\
+	  suff[i] = suff[i + plen - 1 - f];				\
+	else								\
+	  {								\
+	    if (i < g)							\
+	      g = i;							\
+	    f = i;							\
+	    while (g >= 0 && pat[g] == pat[g + plen - 1 - f])		\
+	      g--;							\
+	    suff[i] = f - g;						\
+	  }								\
+      }									\
+									\
+    for (unsigned int i = 0; i < plen; i++)				\
+      arr[i] = plen;							\
+    g = 0;								\
+    for (int i = plen - 1; i >= 0; i--)					\
+      if (suff[i] == i + 1)						\
+	for(; (unsigned long)g < plen - 1 - i; g++)			\
+	  if (arr[g] == plen)						\
+	    arr[g] = plen - 1 - i;					\
+    for (unsigned int i = 0; i <= plen - 2; i++)			\
+      arr[plen - 1 - suff[i]] = plen - 1 - i;				\
+									\
+    xfree(suff);							\
+  }
+
+/*
+ * Copies the pattern pat having lenght n to p and stores
+ * the size in l.
+ */
+#define SAVE_PATTERN(src, srclen, dst, dstlen)				\
+  dstlen = srclen;							\
+  dst = xmalloc((dstlen + 1) * sizeof(tre_char_t));			\
+  if (dst == NULL)							\
+    return REG_ESPACE;							\
+  if (dstlen > 0)							\
+    memcpy(dst, src, dstlen * sizeof(tre_char_t));			\
+  dst[dstlen] = TRE_CHAR('\0');
+
+/*
+ * Initializes pattern compiling.
+ */
+#define INIT_COMP							\
+  /* Initialize. */							\
+  memset(fg, 0, sizeof(*fg));						\
+  fg->icase = (cflags & REG_ICASE);					\
+  fg->word = (cflags & REG_WORD);					\
+  fg->newline = (cflags & REG_NEWLINE);					\
+  fg->nosub = (cflags & REG_NOSUB);					\
+									\
+  /* Cannot handle REG_ICASE with MB string */				\
+  if (fg->icase && (TRE_MB_CUR_MAX > 1))				\
+    {									\
+      DPRINT(("Cannot use fast matcher for MBS with REG_ICASE\n"));	\
+      return REG_BADPAT;						\
+    }
+
+/*
+ * Checks whether we have a 0-length pattern that will match
+ * anything. If literal is set to false, the EOL anchor is also
+ * taken into account.
+ */
+#define CHECK_MATCHALL(literal)						\
+  if (!literal && n == 1 && pat[0] == TRE_CHAR('$'))			\
+    {									\
+      n--;								\
+      fg->eol = true;							\
+    }									\
+									\
+  if (n == 0)								\
+    {									\
+      fg->matchall = true;						\
+      fg->pattern = xmalloc(sizeof(char));				\
+      if (!fg->pattern)							\
+	FAIL_COMP(REG_ESPACE);						\
+      fg->pattern[0] = '\0';						\
+      fg->wpattern = xmalloc(sizeof(tre_char_t));			\
+      if (!fg->wpattern)						\
+	FAIL_COMP(REG_ESPACE);						\
+      fg->wpattern[0] = TRE_CHAR('\0');					\
+      DPRINT(("Matching every input\n"));				\
+      return REG_OK;							\
+    }
+
+/*
+ * Returns: REG_OK on success, error code otherwise
+ */
+int
+tre_compile_literal(fastmatch_t *fg, const tre_char_t *pat, size_t n,
+		    int cflags)
+{
+  size_t hasdot = 0, whasdot = 0;
+  ssize_t firstdot = -1, wfirstdot = -1;
+
+  INIT_COMP;
+
+  CHECK_MATCHALL(true);
+
+  /* Cannot handle word boundaries with MB string */
+  if (fg->word && (TRE_MB_CUR_MAX > 1))
+    return REG_BADPAT;
+
+#ifdef TRE_WCHAR
+  SAVE_PATTERN(pat, n, fg->wpattern, fg->wlen);
+  STORE_MBS_PAT;
+#else
+  SAVE_PATTERN(pat, n, fg->pattern, fg->len);
+#endif
+
+  DPRINT(("tre_compile_literal: pattern: %s, len %zu, icase: %c, word: %c, "
+	 "newline %c\n", fg->pattern, fg->len, fg->icase ? 'y' : 'n',
+	 fg->word ? 'y' : 'n', fg->newline ? 'y' : 'n'));
+
+  FILL_QSBC;
+  FILL_BMGS;
+#ifdef TRE_WCHAR
+  FILL_QSBC_WIDE;
+  FILL_BMGS_WIDE;
+#endif
+
+  return REG_OK;
+}
+
+/*
+ * Returns: REG_OK on success, error code otherwise
+ */
+int
+tre_compile_fast(fastmatch_t *fg, const tre_char_t *pat, size_t n,
+		 int cflags)
+{
+  tre_char_t *tmp;
+  size_t pos = 0, hasdot = 0, whasdot = 0;
+  ssize_t firstdot = -1, wfirstdot = -1;
+  bool escaped = false;
+  bool *_escmap = NULL;
+
+  INIT_COMP;
+
+  /* Remove beginning-of-line character ('^'). */
+  if (pat[0] == TRE_CHAR('^'))
+    {
+      fg->bol = true;
+      n--;
+      pat++;
+    }
+
+  CHECK_MATCHALL(false);
+
+  /* Handle word-boundary matching when GNU extensions are enabled */
+  if ((cflags & REG_GNU) && (n >= 14) &&
+      (memcmp(pat, TRE_CHAR("[[:<:]]"), 7 * sizeof(tre_char_t)) == 0) &&
+      (memcmp(pat + n - 7, TRE_CHAR("[[:>:]]"),
+	      7 * sizeof(tre_char_t)) == 0))
+    {
+      n -= 14;
+      pat += 7;
+      fg->word = true;
+    }
+
+  /* Cannot handle word boundaries with MB string */
+  if (fg->word && (TRE_MB_CUR_MAX > 1))
+    return REG_BADPAT;
+
+  tmp = xmalloc((n + 1) * sizeof(tre_char_t));
+  if (tmp == NULL)
+    return REG_ESPACE;
+
+/* Copies the char into the stored pattern and skips to the next char. */
+#define STORE_CHAR							\
+  do									\
+    {									\
+      tmp[pos++] = pat[i];						\
+      escaped = false;							\
+      continue;								\
+    } while (0)
+
+  /* Traverse the input pattern for processing */
+  for (unsigned int i = 0; i < n; i++)
+    {
+      switch (pat[i])
+	{
+	  case TRE_CHAR('\\'):
+	    if (escaped)
+	      STORE_CHAR;
+	    else if (i == n - 1)
+	      goto badpat;
+	    else
+	      escaped = true;
+	    continue;
+	  case TRE_CHAR('['):
+	    if (escaped)
+	      STORE_CHAR;
+	    else
+	      goto badpat;
+	    continue;
+	  case TRE_CHAR('*'):
+	    if (escaped || (!(cflags & REG_EXTENDED) && (i == 0)))
+	      STORE_CHAR;
+	    else
+	      goto badpat;
+	    continue;
+	  case TRE_CHAR('+'):
+	  case TRE_CHAR('?'):
+	    if ((cflags & REG_EXTENDED) && (i == 0))
+	      continue;
+	    else if ((cflags & REG_EXTENDED) ^ !escaped)
+	      STORE_CHAR;
+	    else
+	      goto badpat;
+	    continue;
+	  case TRE_CHAR('.'):
+	    if (escaped)
+	      {
+		if (!_escmap)
+		  _escmap = xmalloc(n * sizeof(bool));
+		if (!_escmap)
+		  {
+		    xfree(tmp);
+		    return REG_ESPACE;
+		  }
+		_escmap[i] = true;
+		STORE_CHAR;
+	      }
+	    else
+	      {
+		whasdot = i;
+		if (wfirstdot == -1)
+			wfirstdot = i;
+		STORE_CHAR;
+	      }
+	    continue;
+	  case TRE_CHAR('^'):
+	    STORE_CHAR;
+	    continue;
+	  case TRE_CHAR('$'):
+	    if (!escaped && (i == n - 1))
+	      fg->eol = true;
+	    else
+	      STORE_CHAR;
+	    continue;
+	  case TRE_CHAR('('):
+	    if ((cflags & REG_EXTENDED) ^ escaped)
+	      goto badpat;
+	    else
+	      STORE_CHAR;
+	    continue;
+	  case TRE_CHAR('{'):
+	    if (!(cflags & REG_EXTENDED) ^ escaped)
+	      STORE_CHAR;
+	    else if (!(cflags & REG_EXTENDED) && (i == 0))
+	      STORE_CHAR;
+	    else if ((cflags & REG_EXTENDED) && (i == 0))
+	      continue;
+	    else
+	      goto badpat;
+	    continue;
+	  case TRE_CHAR('|'):
+	    if ((cflags & REG_EXTENDED) ^ escaped)
+	      goto badpat;
+	    else
+	      STORE_CHAR;
+	    continue;
+	  default:
+	    if (escaped)
+	      goto badpat;
+	    else
+	      STORE_CHAR;
+	    continue;
+	}
+      continue;
+badpat:
+      xfree(tmp);
+      DPRINT(("tre_compile_fast: compilation of pattern failed, falling"
+	      "back to NFA\n"));
+      return REG_BADPAT;
+    }
+
+  fg->hasdot = wfirstdot > -1;
+
+  /*
+   * The pattern has been processed and copied to tmp as a literal string
+   * with escapes, anchors (^$) and the word boundary match character
+   * classes stripped out.
+   */
+#ifdef TRE_WCHAR
+  SAVE_PATTERN(tmp, pos, fg->wpattern, fg->wlen);
+  fg->wescmap = _escmap;
+  STORE_MBS_PAT;
+
+  /*
+   * The position of dots and escaped dots is different in the MB string
+   * than in to the wide string so traverse the converted string, as well,
+   * to store these positions.
+   */
+  if (fg->hasdot || (fg->wescmap != NULL))
+    {
+      if (fg->wescmap != NULL)
+	{
+	  fg->escmap = xmalloc(fg->len * sizeof(bool));
+	  if (!fg->escmap)
+	    {
+	      tre_free_fast(fg);
+	      return REG_ESPACE;
+	    }
+	}
+
+      escaped = false;
+      for (unsigned int i = 0; i < fg->len; i++)
+	if (fg->pattern[i] == '\\')
+	  escaped = !escaped;
+	else if (fg->pattern[i] == '.' && escaped)
+	  {
+	    fg->escmap[i] = true;
+	    escaped = false;
+	  }
+	else if (fg->pattern[i] == '.' && !escaped)
+	  {
+	    hasdot = i;
+	    if (firstdot == -1)
+	      firstdot = i;
+	  }
+	else
+	  escaped = false;
+    }
+#else
+  SAVE_PATTERN(tmp, pos, fg->pattern, fg->len);
+  fg->escmap = _escmap;
+#endif
+
+  xfree(tmp);
+
+  DPRINT(("tre_compile_fast: pattern: %s, len %zu, bol %c, eol %c, "
+	 "icase: %c, word: %c, newline %c\n", fg->pattern, fg->len,
+	 fg->bol ? 'y' : 'n', fg->eol ? 'y' : 'n',
+	 fg->icase ? 'y' : 'n', fg->word ? 'y' : 'n',
+	 fg->newline ? 'y' : 'n'));
+
+  /* Check whether reverse QS algorithm is more efficient */
+  if ((wfirstdot > -1) && (fg->wlen - whasdot + 1 < (size_t)wfirstdot) &&
+      fg->nosub)
+    {
+      fg->reversed = true;
+      DPRINT(("tre_compile_fast: using reverse QS algorithm\n"));
+    }
+
+  FILL_QSBC;
+  FILL_BMGS;
+#ifdef TRE_WCHAR
+  FILL_QSBC_WIDE;
+  FILL_BMGS_WIDE;
+#endif
+
+  return REG_OK;
+}
+
+#define _SHIFT_ONE							\
+  {									\
+    shift = 1;								\
+    j = !fg->reversed ? j + shift : j - shift;				\
+    continue;								\
+  }
+
+#define _BBOUND_COND							\
+  ((type == STR_WIDE) ?							\
+    ((j == 0) || !(tre_isalnum(str_wide[j - 1]) ||			\
+      (str_wide[j - 1] == TRE_CHAR('_')))) :				\
+    ((j == 0) || !(tre_isalnum(str_byte[j - 1]) ||			\
+      (str_byte[j - 1] == '_'))))
+
+#define _EBOUND_COND							\
+  ((type == STR_WIDE) ?							\
+    ((j + fg->wlen == len) || !(tre_isalnum(str_wide[j + fg->wlen]) ||	\
+      (str_wide[j + fg->wlen] == TRE_CHAR('_')))) :			\
+    ((j + fg->len == len) || !(tre_isalnum(str_byte[j + fg->len]) ||	\
+      (str_byte[j + fg->len] == '_'))))
+
+/*
+ * Condition to check whether the match on position j is on a
+ * word boundary.
+ */
+#define IS_ON_WORD_BOUNDARY						\
+  (_BBOUND_COND && _EBOUND_COND)
+
+/*
+ * Checks word boundary and shifts one if match is not on a
+ * boundary.
+ */
+#define CHECK_WORD_BOUNDARY						\
+    if (!IS_ON_WORD_BOUNDARY)						\
+      _SHIFT_ONE;
+
+#define _BOL_COND							\
+  ((j == 0) || ((type == STR_WIDE) ? (str_wide[j - 1] == TRE_CHAR('\n'))\
+				   : (str_byte[j - 1] == '\n')))
+
+/*
+ * Checks BOL anchor and shifts one if match is not on a
+ * boundary.
+ */
+#define CHECK_BOL_ANCHOR						\
+    if (!_BOL_COND)							\
+      _SHIFT_ONE;
+
+#define _EOL_COND							\
+  ((type == STR_WIDE)							\
+    ? ((j + fg->wlen == len) ||						\
+		(str_wide[j + fg->wlen] == TRE_CHAR('\n')))		\
+    : ((j + fg->len == len) || (str_byte[j + fg->wlen] == '\n')))
+
+/*
+ * Checks EOL anchor and shifts one if match is not on a
+ * boundary.
+ */
+#define CHECK_EOL_ANCHOR						\
+    if (!_EOL_COND)							\
+      _SHIFT_ONE;
+
+/*
+ * Executes matching of the precompiled pattern on the input string.
+ * Returns REG_OK or REG_NOMATCH depending on if we find a match or not.
+ */
+int
+tre_match_fast(const fastmatch_t *fg, const void *data, size_t len,
+    tre_str_type_t type, int nmatch, regmatch_t pmatch[], int eflags)
+{
+  unsigned int shift, u = 0, v = 0;
+  ssize_t j = 0;
+  int ret = REG_NOMATCH;
+  int mismatch;
+  const char *str_byte = data;
+  const void *startptr = NULL;
+  const tre_char_t *str_wide = data;
+
+  /* Calculate length if unspecified. */
+  if (len == (size_t)-1)
+    switch (type)
+      {
+	case STR_WIDE:
+	  len = tre_strlen(str_wide);
+	  break;
+	default:
+	  len = strlen(str_byte);
+	  break;
+      }
+
+  /* Shortcut for empty pattern */
+  if (fg->matchall)
+    {
+      if (!fg->nosub && nmatch >= 1)
+	{
+	  pmatch[0].rm_so = 0;
+	  pmatch[0].rm_eo = len;
+	}
+      if (fg->bol && fg->eol)
+	return (len == 0) ? REG_OK : REG_NOMATCH;
+      else
+	return REG_OK;
+    }
+
+  /* No point in going farther if we do not have enough data. */
+  switch (type)
+    {
+      case STR_WIDE:
+	if (len < fg->wlen)
+	  return ret;
+	shift = fg->wlen;
+	break;
+      default:
+	if (len < fg->len)
+	  return ret;
+	shift = fg->len;
+    }
+
+  /*
+   * REG_NOTBOL means not anchoring ^ to the beginning of the line, so we
+   * can shift one because there can't be a match at the beginning.
+   */
+  if (fg->bol && (eflags & REG_NOTBOL))
+    j = 1;
+
+  /*
+   * Like above, we cannot have a match at the very end when anchoring to
+   * the end and REG_NOTEOL is specified.
+   */
+  if (fg->eol && (eflags & REG_NOTEOL))
+    len--;
+
+  if (fg->reversed)
+    j = len - (type == STR_WIDE ? fg->wlen : fg->len);
+
+
+  /* Only try once at the beginning or ending of the line. */
+  if ((fg->bol || fg->eol) && !fg->newline && !(eflags & REG_NOTBOL) &&
+      !(eflags & REG_NOTEOL))
+    {
+      /* Simple text comparison. */
+      if (!((fg->bol && fg->eol) &&
+	  (type == STR_WIDE ? (len != fg->wlen) : (len != fg->len))))
+	{
+	  /* Determine where in data to start search at. */
+	  j = fg->eol ? len - (type == STR_WIDE ? fg->wlen : fg->len) : 0;
+	  SKIP_CHARS(j);
+	  mismatch = fastcmp(fg, startptr, type);
+	  if (mismatch == REG_OK)
+	    {
+	      if (fg->word && !IS_ON_WORD_BOUNDARY)
+		return ret;
+	      if (!fg->nosub && nmatch >= 1)
+		{
+		  pmatch[0].rm_so = j;
+		  pmatch[0].rm_eo = j + (type == STR_WIDE ? fg->wlen : fg->len);
+		}
+	      return REG_OK;
+            }
+        }
+    }
+  else
+    {
+      /* Quick Search / Turbo Boyer-Moore algorithm. */
+      do
+	{
+	  SKIP_CHARS(j);
+	  mismatch = fastcmp(fg, startptr, type);
+	  if (mismatch == REG_OK)
+	    {
+	      if (fg->word)
+		CHECK_WORD_BOUNDARY;
+	      if (fg->bol)
+		CHECK_BOL_ANCHOR;
+	      if (fg->eol)
+		CHECK_EOL_ANCHOR;
+	      if (!fg->nosub && nmatch >= 1)
+		{
+		  pmatch[0].rm_so = j;
+		  pmatch[0].rm_eo = j + ((type == STR_WIDE) ? fg->wlen : fg->len);
+		}
+	      return REG_OK;
+	    }
+	  else if (mismatch > 0)
+	    return mismatch;
+	  mismatch = -mismatch - 1;
+	  SHIFT;
+        } while (!IS_OUT_OF_BOUNDS);
+    }
+    return ret;
+}
+
+/*
+ * Frees the resources that were allocated when the pattern was compiled.
+ */
+void
+tre_free_fast(fastmatch_t *fg)
+{
+
+  DPRINT(("tre_fast_free: freeing structures for pattern %s\n",
+	 fg->pattern));
+
+#ifdef TRE_WCHAR
+  hashtable_free(fg->qsBc_table);
+  if (!fg->hasdot)
+    xfree(fg->bmGs);
+  if (fg->wescmap)
+    xfree(fg->wescmap);
+  xfree(fg->wpattern);
+#endif
+  if (!fg->hasdot)
+    xfree(fg->sbmGs);
+  if (fg->escmap)
+    xfree(fg->escmap);
+  xfree(fg->pattern);
+}
+
+/*
+ * Returns:	-(i + 1) on failure (position that it failed with minus sign)
+ *		error code on error
+ *		REG_OK on success
+ */
+static inline int
+fastcmp(const fastmatch_t *fg, const void *data, tre_str_type_t type)
+{
+  const char *str_byte = data;
+  const char *pat_byte = fg->pattern;
+  const tre_char_t *str_wide = data;
+  const tre_char_t *pat_wide = fg->wpattern;
+  const bool *escmap = (type == STR_WIDE) ? fg->wescmap : fg->escmap;
+  size_t len = (type == STR_WIDE) ? fg->wlen : fg->len;
+  int ret = REG_OK;
+
+  /* Compare the pattern and the input char-by-char from the last position. */
+  for (int i = len - 1; i >= 0; i--) {
+    switch (type)
+      {
+	case STR_WIDE:
+
+	  /* Check dot */
+	  if (fg->hasdot && pat_wide[i] == TRE_CHAR('.') &&
+	      (!escmap || !escmap[i]) &&
+	      (!fg->newline || (str_wide[i] != TRE_CHAR('\n'))))
+	    continue;
+
+	  /* Compare */
+	  if (fg->icase ? (towlower(pat_wide[i]) == towlower(str_wide[i]))
+		    : (pat_wide[i] == str_wide[i]))
+	    continue;
+	  break;
+	default:
+	  /* Check dot */
+	  if (fg->hasdot && pat_byte[i] == '.' &&
+	      (!escmap || !escmap[i]) &&
+	      (!fg->newline || (str_byte[i] != '\n')))
+	    continue;
+
+	  /* Compare */
+	  if (fg->icase ? (tolower(pat_byte[i]) == tolower(str_byte[i]))
+		    : (pat_byte[i] == str_byte[i]))
+	  continue;
+      }
+    DPRINT(("fastcmp: mismatch at position %d\n", i));
+    ret = -(i + 1);
+    break;
+  }
+  return ret;
+}
diff --git a/usr.bin/grep/regex/tre-fastmatch.h b/usr.bin/grep/regex/tre-fastmatch.h
new file mode 100644
index 000000000000..f72397c58c33
--- /dev/null
+++ b/usr.bin/grep/regex/tre-fastmatch.h
@@ -0,0 +1,21 @@
+/* $FreeBSD$ */
+
+#ifndef TRE_FASTMATCH_H
+#define TRE_FASTMATCH_H 1
+
+#include <fastmatch.h>
+#include <hashtable.h>
+#include <limits.h>
+#include <regex.h>
+#include <stdbool.h>
+
+#include "hashtable.h"
+
+int	tre_compile_literal(fastmatch_t *preg, const tre_char_t *regex,
+	    size_t, int);
+int	tre_compile_fast(fastmatch_t *preg, const tre_char_t *regex, size_t, int);
+int	tre_match_fast(const fastmatch_t *fg, const void *data, size_t len,
+	    tre_str_type_t type, int nmatch, regmatch_t pmatch[], int eflags);
+void	tre_free_fast(fastmatch_t *preg);
+
+#endif		/* TRE_FASTMATCH_H */
diff --git a/usr.bin/grep/regex/xmalloc.c b/usr.bin/grep/regex/xmalloc.c
new file mode 100644
index 000000000000..b1acc9c5434c
--- /dev/null
+++ b/usr.bin/grep/regex/xmalloc.c
@@ -0,0 +1,349 @@
+/* $FreeBSD$ */
+
+/*
+  xmalloc.c - Simple malloc debugging library implementation
+
+  This software is released under a BSD-style license.
+  See the file LICENSE for details and copyright.
+
+*/
+
+/*
+  TODO:
+   - red zones
+   - group dumps by source location
+*/
+
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h>
+#define XMALLOC_INTERNAL 1
+#include "xmalloc.h"
+
+
+/*
+  Internal stuff.
+*/
+
+typedef struct hashTableItemRec {
+  void *ptr;
+  int bytes;
+  const char *file;
+  int line;
+  const char *func;
+  struct hashTableItemRec *next;
+} hashTableItem;
+
+typedef struct {
+  hashTableItem **table;
+} hashTable;
+
+static int xmalloc_peak;
+int xmalloc_current;
+static int xmalloc_peak_blocks;
+int xmalloc_current_blocks;
+static int xmalloc_fail_after;
+
+#define TABLE_BITS 8
+#define TABLE_MASK ((1 << TABLE_BITS) - 1)
+#define TABLE_SIZE (1 << TABLE_BITS)
+
+static hashTable *
+hash_table_new(void)
+{
+  hashTable *tbl;
+
+  tbl = malloc(sizeof(*tbl));
+
+  if (tbl != NULL)
+    {
+      tbl->table = calloc(TABLE_SIZE, sizeof(*tbl->table));
+
+      if (tbl->table == NULL)
+	{
+	  free(tbl);
+	  return NULL;
+	}
+    }
+
+  return tbl;
+}
+
+static int
+hash_void_ptr(void *ptr)
+{
+  int hash;
+  int i;
+
+  /* I took this hash function just off the top of my head, I have
+     no idea whether it is bad or very bad. */
+  hash = 0;
+  for (i = 0; i < (int)sizeof(ptr)*8 / TABLE_BITS; i++)
+    {
+      hash ^= (unsigned long)ptr >> i*8;
+      hash += i * 17;
+      hash &= TABLE_MASK;
+    }
+  return hash;
+}
+
+static void
+hash_table_add(hashTable *tbl, void *ptr, int bytes,
+	       const char *file, int line, const char *func)
+{
+  int i;
+  hashTableItem *item, *new;
+
+  i = hash_void_ptr(ptr);
+
+  item = tbl->table[i];
+  if (item != NULL)
+    while (item->next != NULL)
+      item = item->next;
+
+  new = malloc(sizeof(*new));
+  assert(new != NULL);
+  new->ptr = ptr;
+  new->bytes = bytes;
+  new->file = file;
+  new->line = line;
+  new->func = func;
+  new->next = NULL;
+  if (item != NULL)
+    item->next = new;
+  else
+    tbl->table[i] = new;
+
+  xmalloc_current += bytes;
+  if (xmalloc_current > xmalloc_peak)
+    xmalloc_peak = xmalloc_current;
+  xmalloc_current_blocks++;
+  if (xmalloc_current_blocks > xmalloc_peak_blocks)
+    xmalloc_peak_blocks = xmalloc_current_blocks;
+}
+
+static void
+hash_table_del(hashTable *tbl, void *ptr)
+{
+  int i;
+  hashTableItem *item, *prev;
+
+  i = hash_void_ptr(ptr);
+
+  item = tbl->table[i];
+  if (item == NULL)
+    {
+      printf("xfree: invalid ptr %p\n", ptr);
+      abort();
+    }
+  prev = NULL;
+  while (item->ptr != ptr)
+    {
+      prev = item;
+      item = item->next;
+    }
+  if (item->ptr != ptr)
+    {
+      printf("xfree: invalid ptr %p\n", ptr);
+      abort();
+    }
+
+  xmalloc_current -= item->bytes;
+  xmalloc_current_blocks--;
+
+  if (prev != NULL)
+    {
+      prev->next = item->next;
+      free(item);
+    }
+  else
+    {
+      tbl->table[i] = item->next;
+      free(item);
+    }
+}
+
+static hashTable *xmalloc_table = NULL;
+
+static void
+xmalloc_init(void)
+{
+  if (xmalloc_table == NULL)
+    {
+      xmalloc_table = hash_table_new();
+      xmalloc_peak = 0;
+      xmalloc_peak_blocks = 0;
+      xmalloc_current = 0;
+      xmalloc_current_blocks = 0;
+      xmalloc_fail_after = -1;
+    }
+  assert(xmalloc_table != NULL);
+  assert(xmalloc_table->table != NULL);
+}
+
+
+
+/*
+  Public API.
+*/
+
+void
+xmalloc_configure(int fail_after)
+{
+  xmalloc_init();
+  xmalloc_fail_after = fail_after;
+}
+
+int
+xmalloc_dump_leaks(void)
+{
+  int i;
+  int num_leaks = 0;
+  int leaked_bytes = 0;
+  hashTableItem *item;
+
+  xmalloc_init();
+
+  for (i = 0; i < TABLE_SIZE; i++)
+    {
+      item = xmalloc_table->table[i];
+      while (item != NULL)
+	{
+	  printf("%s:%d: %s: %d bytes at %p not freed\n",
+		 item->file, item->line, item->func, item->bytes, item->ptr);
+	  num_leaks++;
+	  leaked_bytes += item->bytes;
+	  item = item->next;
+	}
+    }
+  if (num_leaks == 0)
+    printf("No memory leaks.\n");
+  else
+    printf("%d unfreed memory chuncks, total %d unfreed bytes.\n",
+	   num_leaks, leaked_bytes);
+  printf("Peak memory consumption %d bytes (%.1f kB, %.1f MB) in %d blocks ",
+	 xmalloc_peak, (double)xmalloc_peak / 1024,
+	 (double)xmalloc_peak / (1024*1024), xmalloc_peak_blocks);
+  printf("(average ");
+  if (xmalloc_peak_blocks)
+    printf("%d", ((xmalloc_peak + xmalloc_peak_blocks / 2)
+		  / xmalloc_peak_blocks));
+  else
+    printf("N/A");
+  printf(" bytes per block).\n");
+
+  return num_leaks;
+}
+
+void *
+xmalloc_impl(size_t size, const char *file, int line, const char *func)
+{
+  void *ptr;
+
+  xmalloc_init();
+  assert(size > 0);
+
+  if (xmalloc_fail_after == 0)
+    {
+      xmalloc_fail_after = -2;
+#if 0
+      printf("xmalloc: forced failure %s:%d: %s\n", file, line, func);
+#endif
+      return NULL;
+    }
+  else if (xmalloc_fail_after == -2)
+    {
+      printf("xmalloc: called after failure from %s:%d: %s\n",
+	     file, line, func);
+      assert(0);
+    }
+  else if (xmalloc_fail_after > 0)
+    xmalloc_fail_after--;
+
+  ptr = malloc(size);
+  if (ptr != NULL)
+    hash_table_add(xmalloc_table, ptr, (int)size, file, line, func);
+  return ptr;
+}
+
+void *
+xcalloc_impl(size_t nmemb, size_t size, const char *file, int line,
+	     const char *func)
+{
+  void *ptr;
+
+  xmalloc_init();
+  assert(size > 0);
+
+  if (xmalloc_fail_after == 0)
+    {
+      xmalloc_fail_after = -2;
+#if 0
+      printf("xcalloc: forced failure %s:%d: %s\n", file, line, func);
+#endif
+      return NULL;
+    }
+  else if (xmalloc_fail_after == -2)
+    {
+      printf("xcalloc: called after failure from %s:%d: %s\n",
+	     file, line, func);
+      assert(0);
+    }
+  else if (xmalloc_fail_after > 0)
+    xmalloc_fail_after--;
+
+  ptr = calloc(nmemb, size);
+  if (ptr != NULL)
+    hash_table_add(xmalloc_table, ptr, (int)(nmemb * size), file, line, func);
+  return ptr;
+}
+
+void
+xfree_impl(void *ptr, const char *file, int line, const char *func)
+{
+  /*LINTED*/(void)&file;
+  /*LINTED*/(void)&line;
+  /*LINTED*/(void)&func;
+  xmalloc_init();
+
+  if (ptr != NULL)
+    hash_table_del(xmalloc_table, ptr);
+  free(ptr);
+}
+
+void *
+xrealloc_impl(void *ptr, size_t new_size, const char *file, int line,
+	      const char *func)
+{
+  void *new_ptr;
+
+  xmalloc_init();
+  assert(ptr != NULL);
+  assert(new_size > 0);
+
+  if (xmalloc_fail_after == 0)
+    {
+      xmalloc_fail_after = -2;
+      return NULL;
+    }
+  else if (xmalloc_fail_after == -2)
+    {
+      printf("xrealloc: called after failure from %s:%d: %s\n",
+	     file, line, func);
+      assert(0);
+    }
+  else if (xmalloc_fail_after > 0)
+    xmalloc_fail_after--;
+
+  new_ptr = realloc(ptr, new_size);
+  if (new_ptr != NULL)
+    {
+      hash_table_del(xmalloc_table, ptr);
+      hash_table_add(xmalloc_table, new_ptr, (int)new_size, file, line, func);
+    }
+  return new_ptr;
+}
+
+
+
+/* EOF */
diff --git a/usr.bin/grep/regex/xmalloc.h b/usr.bin/grep/regex/xmalloc.h
new file mode 100644
index 000000000000..5cde986e003f
--- /dev/null
+++ b/usr.bin/grep/regex/xmalloc.h
@@ -0,0 +1,79 @@
+/* $FreeBSD$ */
+
+/*
+  xmalloc.h - Simple malloc debugging library API
+
+  This software is released under a BSD-style license.
+  See the file LICENSE for details and copyright.
+
+*/
+
+#ifndef _XMALLOC_H
+#define _XMALLOC_H 1
+
+void *xmalloc_impl(size_t size, const char *file, int line, const char *func);
+void *xcalloc_impl(size_t nmemb, size_t size, const char *file, int line,
+		   const char *func);
+void xfree_impl(void *ptr, const char *file, int line, const char *func);
+void *xrealloc_impl(void *ptr, size_t new_size, const char *file, int line,
+		    const char *func);
+int xmalloc_dump_leaks(void);
+void xmalloc_configure(int fail_after);
+
+
+#ifndef XMALLOC_INTERNAL
+#ifdef MALLOC_DEBUGGING
+
+/* Version 2.4 and later of GCC define a magical variable `__PRETTY_FUNCTION__'
+   which contains the name of the function currently being defined.
+#  define __XMALLOC_FUNCTION	 __PRETTY_FUNCTION__
+   This is broken in G++ before version 2.6.
+   C9x has a similar variable called __func__, but prefer the GCC one since
+   it demangles C++ function names.  */
+# ifdef __GNUC__
+#  if __GNUC__ > 2 || (__GNUC__ == 2 \
+		       && __GNUC_MINOR__ >= (defined __cplusplus ? 6 : 4))
+#   define __XMALLOC_FUNCTION	 __PRETTY_FUNCTION__
+#  else
+#   define __XMALLOC_FUNCTION	 ((const char *) 0)
+#  endif
+# else
+#  if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L
+#   define __XMALLOC_FUNCTION	 __func__
+#  else
+#   define __XMALLOC_FUNCTION	 ((const char *) 0)
+#  endif
+# endif
+
+#define xmalloc(size) xmalloc_impl(size, __FILE__, __LINE__, \
+				   __XMALLOC_FUNCTION)
+#define xcalloc(nmemb, size) xcalloc_impl(nmemb, size, __FILE__, __LINE__, \
+					  __XMALLOC_FUNCTION)
+#define xfree(ptr) xfree_impl(ptr, __FILE__, __LINE__, __XMALLOC_FUNCTION)
+#define xrealloc(ptr, new_size) xrealloc_impl(ptr, new_size, __FILE__, \
+					      __LINE__, __XMALLOC_FUNCTION)
+#undef malloc
+#undef calloc
+#undef free
+#undef realloc
+
+#define malloc	USE_XMALLOC_INSTEAD_OF_MALLOC
+#define calloc	USE_XCALLOC_INSTEAD_OF_CALLOC
+#define free	USE_XFREE_INSTEAD_OF_FREE
+#define realloc USE_XREALLOC_INSTEAD_OF_REALLOC
+
+#else /* !MALLOC_DEBUGGING */
+
+#include <stdlib.h>
+
+#define xmalloc(size) malloc(size)
+#define xcalloc(nmemb, size) calloc(nmemb, size)
+#define xfree(ptr) free(ptr)
+#define xrealloc(ptr, new_size) realloc(ptr, new_size)
+
+#endif /* !MALLOC_DEBUGGING */
+#endif /* !XMALLOC_INTERNAL */
+
+#endif /* _XMALLOC_H */
+
+/* EOF */
diff --git a/usr.bin/grep/util.c b/usr.bin/grep/util.c
index 780f0fc57fca..785961b1c6b3 100644
--- a/usr.bin/grep/util.c
+++ b/usr.bin/grep/util.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <wchar.h>
 #include <wctype.h>
 
+#include "fastmatch.h"
 #include "grep.h"
 
 static int	 linesqueued;
@@ -232,13 +233,8 @@ procfile(const char *fn)
 			linesqueued++;
 		}
 		c += t;
-
-		/* Count the matches if we have a match limit */
-		if (mflag) {
-			mcount -= t;
-			if (mcount <= 0)
-				break;
-		}
+		if (mflag && mcount <= 0)
+			break;
 	}
 	if (Bflag > 0)
 		clearqueue();
@@ -280,79 +276,77 @@ procline(struct str *l, int nottext)
 	unsigned int i;
 	int c = 0, m = 0, r = 0;
 
-	if (!matchall) {
-		/* Loop to process the whole line */
-		while (st <= l->len) {
-			pmatch.rm_so = st;
-			pmatch.rm_eo = l->len;
+	/* Loop to process the whole line */
+	while (st <= l->len) {
+		pmatch.rm_so = st;
+		pmatch.rm_eo = l->len;
 
-			/* Loop to compare with all the patterns */
-			for (i = 0; i < patterns; i++) {
-/*
- * XXX: grep_search() is a workaround for speed up and should be
- * removed in the future.  See fastgrep.c.
- */
-				if (fg_pattern[i].pattern)
-					r = grep_search(&fg_pattern[i],
-					    (unsigned char *)l->dat,
-					    l->len, &pmatch);
-				else
-					r = regexec(&r_pattern[i], l->dat, 1,
-					    &pmatch, eflags);
-				r = (r == 0) ? 0 : REG_NOMATCH;
-				st = (cflags & REG_NOSUB)
-					? (size_t)l->len
-					: (size_t)pmatch.rm_eo;
-				if (r == REG_NOMATCH)
-					continue;
-				/* Check for full match */
-				if (r == 0 && xflag)
-					if (pmatch.rm_so != 0 ||
-					    (size_t)pmatch.rm_eo != l->len)
-						r = REG_NOMATCH;
-				/* Check for whole word match */
-				if (r == 0 && (wflag || fg_pattern[i].word)) {
-					wint_t wbegin, wend;
+		/* Loop to compare with all the patterns */
+		for (i = 0; i < patterns; i++) {
+			if (fg_pattern[i].pattern)
+				r = fastexec(&fg_pattern[i],
+				    l->dat, 1, &pmatch, eflags);
+			else
+				r = regexec(&r_pattern[i], l->dat, 1,
+				    &pmatch, eflags);
+			r = (r == 0) ? 0 : REG_NOMATCH;
+			st = (cflags & REG_NOSUB)
+				? (size_t)l->len
+				: (size_t)pmatch.rm_eo;
+			if (r == REG_NOMATCH)
+				continue;
+			/* Check for full match */
+			if (r == 0 && xflag)
+				if (pmatch.rm_so != 0 ||
+				    (size_t)pmatch.rm_eo != l->len)
+					r = REG_NOMATCH;
+			/* Check for whole word match */
+			if (r == 0 && (wflag || fg_pattern[i].word)) {
+				wint_t wbegin, wend;
 
-					wbegin = wend = L' ';
-					if (pmatch.rm_so != 0 &&
-					    sscanf(&l->dat[pmatch.rm_so - 1],
-					    "%lc", &wbegin) != 1)
-						r = REG_NOMATCH;
-					else if ((size_t)pmatch.rm_eo !=
-					    l->len &&
-					    sscanf(&l->dat[pmatch.rm_eo],
-					    "%lc", &wend) != 1)
-						r = REG_NOMATCH;
-					else if (iswword(wbegin) ||
-					    iswword(wend))
-						r = REG_NOMATCH;
-				}
-				if (r == 0) {
-					if (m == 0)
-						c++;
-					if (m < MAX_LINE_MATCHES)
-						matches[m++] = pmatch;
-					/* matches - skip further patterns */
-					if ((color == NULL && !oflag) ||
-					    qflag || lflag)
-						break;
-				}
+				wbegin = wend = L' ';
+				if (pmatch.rm_so != 0 &&
+				    sscanf(&l->dat[pmatch.rm_so - 1],
+				    "%lc", &wbegin) != 1)
+					r = REG_NOMATCH;
+				else if ((size_t)pmatch.rm_eo !=
+				    l->len &&
+				    sscanf(&l->dat[pmatch.rm_eo],
+				    "%lc", &wend) != 1)
+					r = REG_NOMATCH;
+				else if (iswword(wbegin) ||
+				    iswword(wend))
+					r = REG_NOMATCH;
 			}
-
-			if (vflag) {
-				c = !c;
-				break;
+			if (r == 0) {
+				if (m == 0)
+					c++;
+				if (m < MAX_LINE_MATCHES)
+					matches[m++] = pmatch;
+				/* matches - skip further patterns */
+				if ((color == NULL && !oflag) ||
+				    qflag || lflag)
+					break;
 			}
-			/* One pass if we are not recording matches */
-			if ((color == NULL && !oflag) || qflag || lflag)
-				break;
-
-			if (st == (size_t)pmatch.rm_so)
-				break; 	/* No matches */
 		}
-	} else
-		c = !vflag;
+
+		if (vflag) {
+			c = !c;
+			break;
+		}
+
+		/* One pass if we are not recording matches */
+		if ((color == NULL && !oflag) || qflag || lflag)
+			break;
+
+		if (st == (size_t)pmatch.rm_so)
+			break; 	/* No matches */
+	}
+
+
+	/* Count the matches if we have a match limit */
+	if (mflag)
+		mcount -= c;
 
 	if (c && binbehave == BINFILE_BIN && nottext)
 		return (c); /* Binary file */
diff --git a/usr.bin/gzip/Makefile b/usr.bin/gzip/Makefile
index 16a825f66133..6dcd62f8671a 100644
--- a/usr.bin/gzip/Makefile
+++ b/usr.bin/gzip/Makefile
@@ -1,4 +1,4 @@
-#	$NetBSD: Makefile,v 1.13 2009/04/14 22:15:20 lukem Exp $
+#	$NetBSD: Makefile,v 1.16 2011/06/21 13:25:45 joerg Exp $
 # $FreeBSD$
 
 .include <bsd.own.mk>
@@ -6,8 +6,8 @@
 PROG=		gzip
 MAN=		gzip.1 gzexe.1 zdiff.1 zforce.1 zmore.1 znew.1
 
-DPADD=		${LIBZ}
-LDADD=		-lz
+DPADD=		${LIBZ} ${LIBLZMA}
+LDADD=		-lz -llzma
 
 .if ${MK_BZIP2_SUPPORT} != "no"
 DPADD+=		${LIBBZ2}
diff --git a/usr.bin/gzip/gzip.1 b/usr.bin/gzip/gzip.1
index c0dc670a2cf2..ca1ca157ebb7 100644
--- a/usr.bin/gzip/gzip.1
+++ b/usr.bin/gzip/gzip.1
@@ -1,4 +1,4 @@
-.\"	$NetBSD: gzip.1,v 1.20 2009/04/01 08:15:37 mrg Exp $
+.\"	$NetBSD: gzip.1,v 1.21 2011/06/19 02:22:36 christos Exp $
 .\"
 .\" Copyright (c) 1997, 2003, 2004 Matthew R. Green
 .\" All rights reserved.
@@ -25,7 +25,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd May 23, 2011
+.Dd October 9, 2011
 .Dt GZIP 1
 .Os
 .Sh NAME
@@ -185,6 +185,7 @@ Options on the command line will override anything in
 .Sh SEE ALSO
 .Xr bzip2 1 ,
 .Xr compress 1 ,
+.Xr xz 1 ,
 .Xr fts 3 ,
 .Xr zlib 3
 .Sh HISTORY
diff --git a/usr.bin/gzip/gzip.c b/usr.bin/gzip/gzip.c
index 217660ea2398..d7477dd4164a 100644
--- a/usr.bin/gzip/gzip.c
+++ b/usr.bin/gzip/gzip.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: gzip.c,v 1.99 2011/03/23 12:59:44 tsutsui Exp $	*/
+/*	$NetBSD: gzip.c,v 1.105 2011/08/30 23:06:00 joerg Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1998, 2003, 2004, 2006 Matthew R. Green
@@ -76,6 +76,9 @@ enum filetype {
 #endif
 #ifndef NO_PACK_SUPPORT
 	FT_PACK,
+#endif
+#ifndef NO_XZ_SUPPORT
+	FT_XZ,
 #endif
 	FT_LAST,
 	FT_UNKNOWN
@@ -97,6 +100,12 @@ enum filetype {
 #define PACK_MAGIC	"\037\036"
 #endif
 
+#ifndef NO_XZ_SUPPORT
+#include <lzma.h>
+#define XZ_SUFFIX	".xz"
+#define XZ_MAGIC	"\3757zXZ"
+#endif
+
 #define GZ_SUFFIX	".gz"
 
 #define BUFLEN		(64 * 1024)
@@ -138,6 +147,9 @@ static suffixes_t suffixes[] = {
 #endif
 #ifndef NO_COMPRESS_SUPPORT
 	SUFFIX(Z_SUFFIX,	""),
+#endif
+#ifndef NO_XZ_SUPPORT
+	SUFFIX(XZ_SUFFIX,	""),
 #endif
 	SUFFIX(GZ_SUFFIX,	""),	/* Overwritten by -S "" */
 #endif /* SMALL */
@@ -146,7 +158,7 @@ static suffixes_t suffixes[] = {
 #define NUM_SUFFIXES (sizeof suffixes / sizeof suffixes[0])
 #define SUFFIX_MAXLEN	30
 
-static	const char	gzip_version[] = "FreeBSD gzip 20110523";
+static	const char	gzip_version[] = "FreeBSD gzip 20111009";
 
 #ifndef SMALL
 static	const char	gzip_copyright[] = \
@@ -199,16 +211,13 @@ static	int	exit_value = 0;		/* exit value */
 
 static	char	*infile;		/* name of file coming in */
 
-static	void	maybe_err(const char *fmt, ...) __dead2
-    __attribute__((__format__(__printf__, 1, 2)));
-#if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT)
-static	void	maybe_errx(const char *fmt, ...) __dead2
-    __attribute__((__format__(__printf__, 1, 2)));
+static	void	maybe_err(const char *fmt, ...) __printflike(1, 2) __dead2;
+#if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) ||	\
+    !defined(NO_XZ_SUPPORT)
+static	void	maybe_errx(const char *fmt, ...) __printflike(1, 2) __dead2;
 #endif
-static	void	maybe_warn(const char *fmt, ...)
-    __attribute__((__format__(__printf__, 1, 2)));
-static	void	maybe_warnx(const char *fmt, ...)
-    __attribute__((__format__(__printf__, 1, 2)));
+static	void	maybe_warn(const char *fmt, ...) __printflike(1, 2);
+static	void	maybe_warnx(const char *fmt, ...) __printflike(1, 2);
 static	enum filetype file_gettype(u_char *);
 #ifdef SMALL
 #define gz_compress(if, of, sz, fn, tm) gz_compress(if, of, sz)
@@ -223,8 +232,8 @@ static	void	handle_stdin(void);
 static	void	handle_stdout(void);
 static	void	print_ratio(off_t, off_t, FILE *);
 static	void	print_list(int fd, off_t, const char *, time_t);
-static	void	usage(void);
-static	void	display_version(void);
+static	void	usage(void) __dead2;
+static	void	display_version(void) __dead2;
 #ifndef SMALL
 static	void	display_license(void);
 static	void	sigint_handler(int);
@@ -257,7 +266,9 @@ static	off_t	zuncompress(FILE *, FILE *, char *, size_t, off_t *);
 static	off_t	unpack(int, int, char *, size_t, off_t *);
 #endif
 
-int main(int, char **p);
+#ifndef NO_XZ_SUPPORT
+static	off_t	unxz(int, int, char *, size_t, off_t *);
+#endif
 
 #ifdef SMALL
 #define getopt_long(a,b,c,d,e) getopt(a,b,c)
@@ -456,7 +467,8 @@ maybe_err(const char *fmt, ...)
 	exit(2);
 }
 
-#if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT)
+#if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) ||	\
+    !defined(NO_XZ_SUPPORT)
 /* ... without an errno. */
 void
 maybe_errx(const char *fmt, ...)
@@ -1121,6 +1133,11 @@ file_gettype(u_char *buf)
 	if (memcmp(buf, PACK_MAGIC, 2) == 0)
 		return FT_PACK;
 	else
+#endif
+#ifndef NO_XZ_SUPPORT
+	if (memcmp(buf, XZ_MAGIC, 4) == 0)	/* XXX: We only have 4 bytes */
+		return FT_XZ;
+	else
 #endif
 		return FT_UNKNOWN;
 }
@@ -1369,7 +1386,6 @@ file_uncompress(char *file, char *outfile, size_t outsize)
 	}
 
 	method = file_gettype(header1);
-
 #ifndef SMALL
 	if (fflag == 0 && method == FT_UNKNOWN) {
 		maybe_warnx("%s: not in gzip format", file);
@@ -1447,9 +1463,9 @@ file_uncompress(char *file, char *outfile, size_t outsize)
 	} else
 		zfd = STDOUT_FILENO;
 
+	switch (method) {
 #ifndef NO_BZIP2_SUPPORT
-	if (method == FT_BZIP2) {
-
+	case FT_BZIP2:
 		/* XXX */
 		if (lflag) {
 			maybe_warnx("no -l with bzip2 files");
@@ -1457,11 +1473,11 @@ file_uncompress(char *file, char *outfile, size_t outsize)
 		}
 
 		size = unbzip2(fd, zfd, NULL, 0, NULL);
-	} else
+		break;
 #endif
 
 #ifndef NO_COMPRESS_SUPPORT
-	if (method == FT_Z) {
+	case FT_Z: {
 		FILE *in, *out;
 
 		/* XXX */
@@ -1494,30 +1510,42 @@ file_uncompress(char *file, char *outfile, size_t outsize)
 			unlink(outfile);
 			goto lose;
 		}
-	} else
+		break;
+	}
 #endif
 
 #ifndef NO_PACK_SUPPORT
-	if (method == FT_PACK) {
+	case FT_PACK:
 		if (lflag) {
 			maybe_warnx("no -l with packed files");
 			goto lose;
 		}
 
 		size = unpack(fd, zfd, NULL, 0, NULL);
-	} else
+		break;
+#endif
+
+#ifndef NO_XZ_SUPPORT
+	case FT_XZ:
+		if (lflag) {
+			maybe_warnx("no -l with xz files");
+			goto lose;
+		}
+
+		size = unxz(fd, zfd, NULL, 0, NULL);
+		break;
 #endif
 
 #ifndef SMALL
-	if (method == FT_UNKNOWN) {
+	case FT_UNKNOWN:
 		if (lflag) {
 			maybe_warnx("no -l for unknown filetypes");
 			goto lose;
 		}
 		size = cat_fd(NULL, 0, NULL, fd);
-	} else
+		break;
 #endif
-	{
+	default:
 		if (lflag) {
 			print_list(fd, isb.st_size, outfile, isb.st_mtime);
 			close(fd);
@@ -1525,6 +1553,7 @@ file_uncompress(char *file, char *outfile, size_t outsize)
 		}
 
 		size = gz_uncompress(fd, zfd, NULL, 0, NULL, file);
+		break;
 	}
 
 	if (close(fd) != 0)
@@ -1697,7 +1726,8 @@ handle_stdin(void)
 			return;
 		}
 
-		usize = zuncompress(in, stdout, (char *)header1, sizeof header1, &gsize);
+		usize = zuncompress(in, stdout, (char *)header1,
+		    sizeof header1, &gsize);
 		fclose(in);
 		break;
 #endif
@@ -1706,6 +1736,12 @@ handle_stdin(void)
 		usize = unpack(STDIN_FILENO, STDOUT_FILENO,
 			       (char *)header1, sizeof header1, &gsize);
 		break;
+#endif
+#ifndef NO_XZ_SUPPORT
+	case FT_XZ:
+		usize = unxz(STDIN_FILENO, STDOUT_FILENO,
+			     (char *)header1, sizeof header1, &gsize);
+		break;
 #endif
 	}
 
@@ -2074,7 +2110,7 @@ static void
 display_license(void)
 {
 
-	fprintf(stderr, "%s (based on NetBSD gzip 20091011)\n", gzip_version);
+	fprintf(stderr, "%s (based on NetBSD gzip 20111009)\n", gzip_version);
 	fprintf(stderr, "%s\n", gzip_copyright);
 	exit(0);
 }
@@ -2098,6 +2134,9 @@ display_version(void)
 #ifndef NO_PACK_SUPPORT
 #include "unpack.c"
 #endif
+#ifndef NO_XZ_SUPPORT
+#include "unxz.c"
+#endif
 
 static ssize_t
 read_retry(int fd, void *buf, size_t sz)
diff --git a/usr.bin/gzip/unxz.c b/usr.bin/gzip/unxz.c
new file mode 100644
index 000000000000..c8a0a53d0f4b
--- /dev/null
+++ b/usr.bin/gzip/unxz.c
@@ -0,0 +1,153 @@
+/*	$NetBSD: unxz.c,v 1.5 2011/09/30 01:32:21 christos Exp $	*/
+
+/*-
+ * Copyright (c) 2011 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Christos Zoulas.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <stdarg.h>
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <lzma.h>
+
+static off_t
+unxz(int i, int o, char *pre, size_t prelen, off_t *bytes_in)
+{
+	lzma_stream strm = LZMA_STREAM_INIT;
+	static const int flags = LZMA_TELL_UNSUPPORTED_CHECK|LZMA_CONCATENATED;
+	lzma_ret ret;
+	lzma_action action = LZMA_RUN;
+	off_t bytes_out, bp;
+	uint8_t ibuf[BUFSIZ];
+	uint8_t obuf[BUFSIZ];
+
+	if (bytes_in == NULL)
+		bytes_in = &bp;
+
+	strm.next_in = ibuf;
+	memcpy(ibuf, pre, prelen);
+	strm.avail_in = read(i, ibuf + prelen, sizeof(ibuf) - prelen);
+	if (strm.avail_in == (size_t)-1)
+		maybe_err("read failed");
+	strm.avail_in += prelen;
+	*bytes_in = strm.avail_in;
+
+	if ((ret = lzma_stream_decoder(&strm, UINT64_MAX, flags)) != LZMA_OK)
+		maybe_errx("Can't initialize decoder (%d)", ret);
+
+	strm.next_out = NULL;
+	strm.avail_out = 0;
+	if ((ret = lzma_code(&strm, LZMA_RUN)) != LZMA_OK)
+		maybe_errx("Can't read headers (%d)", ret);
+
+	bytes_out = 0;
+	strm.next_out = obuf;
+	strm.avail_out = sizeof(obuf);
+
+	for (;;) {
+		if (strm.avail_in == 0) {
+			strm.next_in = ibuf;
+			strm.avail_in = read(i, ibuf, sizeof(ibuf));
+			switch (strm.avail_in) {
+			case (size_t)-1:
+				maybe_err("read failed");
+				/*NOTREACHED*/
+			case 0:
+				action = LZMA_FINISH;
+				break;
+			default:
+				*bytes_in += strm.avail_in;
+				break;
+			}
+		}
+
+		ret = lzma_code(&strm, action);
+
+		// Write and check write error before checking decoder error.
+		// This way as much data as possible gets written to output
+		// even if decoder detected an error.
+		if (strm.avail_out == 0 || ret != LZMA_OK) {
+			const size_t write_size = sizeof(obuf) - strm.avail_out;
+
+			if (write(o, obuf, write_size) != (ssize_t)write_size)
+				maybe_err("write failed");
+
+			strm.next_out = obuf;
+			strm.avail_out = sizeof(obuf);
+			bytes_out += write_size;
+		}
+
+		if (ret != LZMA_OK) {
+			if (ret == LZMA_STREAM_END) {
+				// Check that there's no trailing garbage.
+				if (strm.avail_in != 0 || read(i, ibuf, 1))
+					ret = LZMA_DATA_ERROR;
+				else {
+					lzma_end(&strm);
+					return bytes_out;
+				}
+			}
+
+			const char *msg;
+			switch (ret) {
+			case LZMA_MEM_ERROR:
+				msg = strerror(ENOMEM);
+				break;
+
+			case LZMA_FORMAT_ERROR:
+				msg = "File format not recognized";
+				break;
+
+			case LZMA_OPTIONS_ERROR:
+				// FIXME: Better message?
+				msg = "Unsupported compression options";
+				break;
+
+			case LZMA_DATA_ERROR:
+				msg = "File is corrupt";
+				break;
+
+			case LZMA_BUF_ERROR:
+				msg = "Unexpected end of input";
+				break;
+
+			case LZMA_MEMLIMIT_ERROR:
+				msg = "Reached memory limit";
+				break;
+
+			default:
+				maybe_errx("Unknown error (%d)", ret);
+				break;
+			}
+			maybe_errx("%s", msg);
+
+		}
+	}
+}
diff --git a/usr.bin/gzip/zuncompress.c b/usr.bin/gzip/zuncompress.c
index f68ba595680b..f7f50af2b9d1 100644
--- a/usr.bin/gzip/zuncompress.c
+++ b/usr.bin/gzip/zuncompress.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: zuncompress.c,v 1.8 2010/11/06 21:42:32 mrg Exp $ */
+/*	$NetBSD: zuncompress.c,v 1.11 2011/08/16 13:55:02 joerg Exp $ */
 
 /*-
  * Copyright (c) 1985, 1986, 1992, 1993
@@ -247,7 +247,7 @@ zread(void *cookie, char *rbp, int num)
 	zs->zs_block_compress = zs->zs_maxbits & BLOCK_MASK;
 	zs->zs_maxbits &= BIT_MASK;
 	zs->zs_maxmaxcode = 1L << zs->zs_maxbits;
-	if (zs->zs_maxbits > BITS) {
+	if (zs->zs_maxbits > BITS || zs->zs_maxbits < 12) {
 		errno = EFTYPE;
 		return (-1);
 	}
@@ -259,13 +259,7 @@ zread(void *cookie, char *rbp, int num)
 	}
 	zs->zs_free_ent = zs->zs_block_compress ? FIRST : 256;
 
-	zs->u.r.zs_finchar = zs->u.r.zs_oldcode = getcode(zs);
-	if (zs->u.r.zs_oldcode == -1)	/* EOF already? */
-		return (0);	/* Get out of here */
-
-	/* First code must be 8 bits = char. */
-	*bp++ = (u_char)zs->u.r.zs_finchar;
-	count--;
+	zs->u.r.zs_oldcode = -1;
 	zs->u.r.zs_stackp = de_stack;
 
 	while ((zs->u.r.zs_code = getcode(zs)) > -1) {
@@ -275,17 +269,29 @@ zread(void *cookie, char *rbp, int num)
 			    zs->u.r.zs_code--)
 				tab_prefixof(zs->u.r.zs_code) = 0;
 			zs->zs_clear_flg = 1;
-			zs->zs_free_ent = FIRST - 1;
-			if ((zs->u.r.zs_code = getcode(zs)) == -1)	/* O, untimely death! */
-				break;
+			zs->zs_free_ent = FIRST;
+			zs->u.r.zs_oldcode = -1;
+			continue;
 		}
 		zs->u.r.zs_incode = zs->u.r.zs_code;
 
 		/* Special case for KwKwK string. */
 		if (zs->u.r.zs_code >= zs->zs_free_ent) {
+			if (zs->u.r.zs_code > zs->zs_free_ent ||
+			    zs->u.r.zs_oldcode == -1) {
+				/* Bad stream. */
+				errno = EINVAL;
+				return (-1);
+			}
 			*zs->u.r.zs_stackp++ = zs->u.r.zs_finchar;
 			zs->u.r.zs_code = zs->u.r.zs_oldcode;
 		}
+		/*
+		 * The above condition ensures that code < free_ent.
+		 * The construction of tab_prefixof in turn guarantees that
+		 * each iteration decreases code and therefore stack usage is
+		 * bound by 1 << BITS - 256.
+		 */
 
 		/* Generate output characters in reverse order. */
 		while (zs->u.r.zs_code >= 256) {
@@ -302,7 +308,8 @@ middle:		do {
 		} while (zs->u.r.zs_stackp > de_stack);
 
 		/* Generate the new entry. */
-		if ((zs->u.r.zs_code = zs->zs_free_ent) < zs->zs_maxmaxcode) {
+		if ((zs->u.r.zs_code = zs->zs_free_ent) < zs->zs_maxmaxcode &&
+		    zs->u.r.zs_oldcode != -1) {
 			tab_prefixof(zs->u.r.zs_code) = (u_short) zs->u.r.zs_oldcode;
 			tab_suffixof(zs->u.r.zs_code) = zs->u.r.zs_finchar;
 			zs->zs_free_ent = zs->u.r.zs_code + 1;
diff --git a/usr.bin/kdump/Makefile b/usr.bin/kdump/Makefile
index ea33bbd7b10a..afa94bb7d127 100644
--- a/usr.bin/kdump/Makefile
+++ b/usr.bin/kdump/Makefile
@@ -8,22 +8,28 @@ SFX=		32
 .PATH: ${.CURDIR}/../ktrace
 
 PROG=		kdump
-SRCS=		kdump.c ioctl.c kdump_subr.c subr.c
-CFLAGS+=	-I${.CURDIR}/../ktrace -I${.CURDIR} -I${.CURDIR}/../..
+SRCS=		kdump_subr.c kdump.c ioctl.c subr.c
+DPSRCS=		kdump_subr.h 
+CFLAGS+=	-I${.CURDIR}/../ktrace -I${.CURDIR} -I${.CURDIR}/../.. -I.
 
 .if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386"
 SRCS+=		linux_syscalls.c
 .endif
 
-WARNS?=		0
+NO_WERROR?=	YES
 
-CLEANFILES=	ioctl.c kdump_subr.c linux_syscalls.c
+CLEANFILES=	ioctl.c kdump_subr.c kdump_subr.h linux_syscalls.c
 
 ioctl.c: mkioctls
-	sh ${.CURDIR}/mkioctls ${DESTDIR}/usr/include > ${.TARGET}
+	env MACHINE=${MACHINE} \
+	    sh ${.CURDIR}/mkioctls print ${DESTDIR}/usr/include > ${.TARGET}
 
-kdump_subr.c: mksubr
-	sh ${.CURDIR}/mksubr ${DESTDIR}/usr/include > ${.TARGET}
+kdump_subr.h: mksubr
+	sh ${.CURDIR}/mksubr ${DESTDIR}/usr/include | \
+	    sed -n 's/^\([a-z].*)\)$$/void \1;/p' >${.TARGET}
+
+kdump_subr.c: mksubr kdump_subr.h
+	sh ${.CURDIR}/mksubr ${DESTDIR}/usr/include >${.TARGET}
 
 linux_syscalls.c:
 	/bin/sh ${.CURDIR}/../../sys/kern/makesyscalls.sh \
diff --git a/usr.bin/kdump/kdump.c b/usr.bin/kdump/kdump.c
index 1c3807c90f70..ba09170d8f69 100644
--- a/usr.bin/kdump/kdump.c
+++ b/usr.bin/kdump/kdump.c
@@ -95,13 +95,15 @@ void visdump(char *, int, int);
 void ktrgenio(struct ktr_genio *, int);
 void ktrpsig(struct ktr_psig *);
 void ktrcsw(struct ktr_csw *);
+void ktruser_malloc(unsigned char *);
+void ktruser_rtld(int, unsigned char *);
 void ktruser(int, unsigned char *);
 void ktrsockaddr(struct sockaddr *);
 void ktrstat(struct stat *);
 void ktrstruct(char *, size_t);
+void ktrcapfail(struct ktr_cap_fail *);
 void usage(void);
-void sockfamilyname(int);
-const char *ioctlname(u_long);
+void ioctlname(unsigned long, int);
 
 int timestamp, decimal, fancy = 1, suppressdata, tail, threads, maxdata,
     resolv = 0, abiflag = 0;
@@ -111,15 +113,15 @@ struct ktr_header ktr_header;
 #define TIME_FORMAT	"%b %e %T %Y"
 #define eqs(s1, s2)	(strcmp((s1), (s2)) == 0)
 
-#define print_number(i,n,c) do {		\
-	if (decimal)				\
-		printf("%c%ld", c, (long)*i);	\
-	else					\
-		printf("%c%#lx", c, (long)*i);	\
-	i++;					\
-	n--;					\
-	c = ',';				\
-	} while (0);
+#define print_number(i,n,c) do {					\
+	if (decimal)							\
+		printf("%c%jd", c, (intmax_t)*i);			\
+	else								\
+		printf("%c%#jx", c, (uintmax_t)(u_register_t)*i);	\
+	i++;								\
+	n--;								\
+	c = ',';							\
+} while (0)
 
 #if defined(__amd64__) || defined(__i386__)
 
@@ -165,10 +167,10 @@ main(int argc, char *argv[])
 	pid_t pid = 0;
 	u_int sv_flags;
 
-	(void) setlocale(LC_CTYPE, "");
+	setlocale(LC_CTYPE, "");
 
 	while ((ch = getopt(argc,argv,"f:dElm:np:AHRrsTt:")) != -1)
-		switch((char)ch) {
+		switch (ch) {
 		case 'A':
 			abiflag = 1;
 			break;
@@ -220,7 +222,7 @@ main(int argc, char *argv[])
 	if (argc > optind)
 		usage();
 
-	m = (void *)malloc(size = 1025);
+	m = malloc(size = 1025);
 	if (m == NULL)
 		errx(1, "%s", strerror(ENOMEM));
 	if (!freopen(tracefile, "r", stdin))
@@ -231,7 +233,7 @@ main(int argc, char *argv[])
 		if (ktr_header.ktr_type & KTR_DROP) {
 			ktr_header.ktr_type &= ~KTR_DROP;
 			if (!drop_logged && threads) {
-				(void)printf(
+				printf(
 				    "%6jd %6jd %-8.*s Events dropped.\n",
 				    (intmax_t)ktr_header.ktr_pid,
 				    ktr_header.ktr_tid > 0 ?
@@ -239,7 +241,7 @@ main(int argc, char *argv[])
 				    MAXCOMLEN, ktr_header.ktr_comm);
 				drop_logged = 1;
 			} else if (!drop_logged) {
-				(void)printf("%6jd %-8.*s Events dropped.\n",
+				printf("%6jd %-8.*s Events dropped.\n",
 				    (intmax_t)ktr_header.ktr_pid, MAXCOMLEN,
 				    ktr_header.ktr_comm);
 				drop_logged = 1;
@@ -251,7 +253,7 @@ main(int argc, char *argv[])
 		if ((ktrlen = ktr_header.ktr_len) < 0)
 			errx(1, "bogus length 0x%x", ktrlen);
 		if (ktrlen > size) {
-			m = (void *)realloc(m, ktrlen+1);
+			m = realloc(m, ktrlen+1);
 			if (m == NULL)
 				errx(1, "%s", strerror(ENOMEM));
 			size = ktrlen;
@@ -302,12 +304,14 @@ main(int argc, char *argv[])
 		case KTR_STRUCT:
 			ktrstruct(m, ktrlen);
 			break;
+		case KTR_CAPFAIL:
+			ktrcapfail((struct ktr_cap_fail *)m);
 		default:
 			printf("\n");
 			break;
 		}
 		if (tail)
-			(void)fflush(stdout);
+			fflush(stdout);
 	}
 	return 0;
 }
@@ -318,7 +322,7 @@ fread_tail(void *buf, int size, int num)
 	int i;
 
 	while ((i = fread(buf, size, num, stdin)) == 0 && tail) {
-		(void)sleep(1);
+		sleep(1);
 		clearerr(stdin);
 	}
 	return (i);
@@ -441,8 +445,11 @@ dumpheader(struct ktr_header *kth)
 		/* FALLTHROUGH */
 	case KTR_PROCDTOR:
 		return;
+	case KTR_CAPFAIL:
+		type = "CAP ";
+		break;
 	default:
-		(void)sprintf(unknown, "UNKNOWN(%d)", kth->ktr_type);
+		sprintf(unknown, "UNKNOWN(%d)", kth->ktr_type);
 		type = unknown;
 	}
 
@@ -455,11 +462,11 @@ dumpheader(struct ktr_header *kth)
 	 * negative tid's as 0.
 	 */
 	if (threads)
-		(void)printf("%6jd %6jd %-8.*s ", (intmax_t)kth->ktr_pid,
+		printf("%6jd %6jd %-8.*s ", (intmax_t)kth->ktr_pid,
 		    kth->ktr_tid > 0 ? (intmax_t)kth->ktr_tid : 0,
 		    MAXCOMLEN, kth->ktr_comm);
 	else
-		(void)printf("%6jd %-8.*s ", (intmax_t)kth->ktr_pid, MAXCOMLEN,
+		printf("%6jd %-8.*s ", (intmax_t)kth->ktr_pid, MAXCOMLEN,
 		    kth->ktr_comm);
 	if (timestamp) {
 		if (timestamp == 3) {
@@ -472,10 +479,10 @@ dumpheader(struct ktr_header *kth)
 			timevalsub(&kth->ktr_time, &prevtime);
 			prevtime = temp;
 		}
-		(void)printf("%jd.%06ld ", (intmax_t)kth->ktr_time.tv_sec,
+		printf("%jd.%06ld ", (intmax_t)kth->ktr_time.tv_sec,
 		    kth->ktr_time.tv_usec);
 	}
-	(void)printf("%s  ", type);
+	printf("%s  ", type);
 }
 
 #include <sys/syscall.h>
@@ -489,460 +496,532 @@ ktrsyscall(struct ktr_syscall *ktr, u_int flags)
 {
 	int narg = ktr->ktr_narg;
 	register_t *ip;
+	intmax_t arg;
 
 	if ((flags != 0 && ((flags & SV_ABI_MASK) != SV_ABI_FREEBSD)) ||
 	    (ktr->ktr_code >= nsyscalls || ktr->ktr_code < 0))
-		(void)printf("[%d]", ktr->ktr_code);
+		printf("[%d]", ktr->ktr_code);
 	else
-		(void)printf("%s", syscallnames[ktr->ktr_code]);
+		printf("%s", syscallnames[ktr->ktr_code]);
 	ip = &ktr->ktr_args[0];
 	if (narg) {
 		char c = '(';
 		if (fancy &&
 		    (flags == 0 || (flags & SV_ABI_MASK) == SV_ABI_FREEBSD)) {
-			if (ktr->ktr_code == SYS_ioctl) {
-				const char *cp;
-				print_number(ip,narg,c);
-				if ((cp = ioctlname(*ip)) != NULL)
-					(void)printf(",%s", cp);
-				else {
-					if (decimal)
-						(void)printf(",%ld", (long)*ip);
-					else
-						(void)printf(",%#lx ", (long)*ip);
-				}
+			switch (ktr->ktr_code) {
+			case SYS_ioctl: {
+				print_number(ip, narg, c);
+				putchar(c);
+				ioctlname(*ip, decimal);
 				c = ',';
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_ptrace) {
-				(void)putchar('(');
-				ptraceopname ((int)*ip);
+				break;
+			}
+			case SYS_ptrace:
+				putchar('(');
+				ptraceopname(*ip);
 				c = ',';
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_access ||
-				   ktr->ktr_code == SYS_eaccess) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				accessmodename ((int)*ip);
+				break;
+			case SYS_access:
+			case SYS_eaccess:
+				print_number(ip, narg, c);
+				putchar(',');
+				accessmodename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_open) {
-				int	flags;
-				int	mode;
-				print_number(ip,narg,c);
-				flags = *ip;
-				mode = *++ip;
-				(void)putchar(',');
-				flagsandmodename (flags, mode, decimal);
-				ip++;
-				narg-=2;
-			} else if (ktr->ktr_code == SYS_wait4) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				wait4optname ((int)*ip);
+				break;
+			case SYS_open:
+				print_number(ip, narg, c);
+				putchar(',');
+				flagsandmodename(ip[0], ip[1], decimal);
+				ip += 2;
+				narg -= 2;
+				break;
+			case SYS_wait4:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				wait4optname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_chmod ||
-				   ktr->ktr_code == SYS_fchmod ||
-				   ktr->ktr_code == SYS_lchmod) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				modename ((int)*ip);
+				break;
+			case SYS_chmod:
+			case SYS_fchmod:
+			case SYS_lchmod:
+				print_number(ip, narg, c);
+				putchar(',');
+				modename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_mknod) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				modename ((int)*ip);
+				break;
+			case SYS_mknod:
+				print_number(ip, narg, c);
+				putchar(',');
+				modename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_getfsstat) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				getfsstatflagsname ((int)*ip);
+				break;
+			case SYS_getfsstat:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				getfsstatflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_mount) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				mountflagsname ((int)*ip);
+				break;
+			case SYS_mount:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				mountflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_unmount) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				mountflagsname ((int)*ip);
+				break;
+			case SYS_unmount:
+				print_number(ip, narg, c);
+				putchar(',');
+				mountflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_recvmsg ||
-				   ktr->ktr_code == SYS_sendmsg) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				sendrecvflagsname ((int)*ip);
+				break;
+			case SYS_recvmsg:
+			case SYS_sendmsg:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				sendrecvflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_recvfrom ||
-				   ktr->ktr_code == SYS_sendto) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				sendrecvflagsname ((int)*ip);
+				break;
+			case SYS_recvfrom:
+			case SYS_sendto:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				sendrecvflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_chflags ||
-				   ktr->ktr_code == SYS_fchflags ||
-				   ktr->ktr_code == SYS_lchflags) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				modename((int)*ip);
+				break;
+			case SYS_chflags:
+			case SYS_fchflags:
+			case SYS_lchflags:
+				print_number(ip, narg, c);
+				putchar(',');
+				modename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_kill) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				signame((int)*ip);
+				break;
+			case SYS_kill:
+				print_number(ip, narg, c);
+				putchar(',');
+				signame(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_reboot) {
-				(void)putchar('(');
-				rebootoptname((int)*ip);
+				break;
+			case SYS_reboot:
+				putchar('(');
+				rebootoptname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_umask) {
-				(void)putchar('(');
-				modename((int)*ip);
+				break;
+			case SYS_umask:
+				putchar('(');
+				modename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_msync) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				msyncflagsname((int)*ip);
+				break;
+			case SYS_msync:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				msyncflagsname(*ip);
 				ip++;
 				narg--;
+				break;
 #ifdef SYS_freebsd6_mmap
-			} else if (ktr->ktr_code == SYS_freebsd6_mmap) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				mmapprotname ((int)*ip);
-				(void)putchar(',');
+			case SYS_freebsd6_mmap:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				mmapprotname(*ip);
+				putchar(',');
 				ip++;
 				narg--;
-				mmapflagsname ((int)*ip);
+				mmapflagsname(*ip);
 				ip++;
 				narg--;
+				break;
 #endif
-			} else if (ktr->ktr_code == SYS_mmap) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				mmapprotname ((int)*ip);
-				(void)putchar(',');
+			case SYS_mmap:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				mmapprotname(*ip);
+				putchar(',');
 				ip++;
 				narg--;
-				mmapflagsname ((int)*ip);
+				mmapflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_mprotect) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				mmapprotname ((int)*ip);
+				break;
+			case SYS_mprotect:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				mmapprotname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_madvise) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				madvisebehavname((int)*ip);
+				break;
+			case SYS_madvise:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				madvisebehavname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_setpriority) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				prioname((int)*ip);
+				break;
+			case SYS_setpriority:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				prioname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_fcntl) {
-				int cmd;
-				int arg;
-				print_number(ip,narg,c);
-				cmd = *ip;
-				arg = *++ip;
-				(void)putchar(',');
-				fcntlcmdname(cmd, arg, decimal);
-				ip++;
-				narg-=2;
-			} else if (ktr->ktr_code == SYS_socket) {
+				break;
+			case SYS_fcntl:
+				print_number(ip, narg, c);
+				putchar(',');
+				fcntlcmdname(ip[0], ip[1], decimal);
+				ip += 2;
+				narg -= 2;
+				break;
+			case SYS_socket: {
 				int sockdomain;
-				(void)putchar('(');
-				sockdomain=(int)*ip;
+				putchar('(');
+				sockdomain = *ip;
 				sockdomainname(sockdomain);
 				ip++;
 				narg--;
-				(void)putchar(',');
-				socktypename((int)*ip);
+				putchar(',');
+				socktypename(*ip);
 				ip++;
 				narg--;
 				if (sockdomain == PF_INET ||
 				    sockdomain == PF_INET6) {
-					(void)putchar(',');
-					sockipprotoname((int)*ip);
+					putchar(',');
+					sockipprotoname(*ip);
 					ip++;
 					narg--;
 				}
 				c = ',';
-			} else if (ktr->ktr_code == SYS_setsockopt ||
-				   ktr->ktr_code == SYS_getsockopt) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				sockoptlevelname((int)*ip, decimal);
-				if ((int)*ip == SOL_SOCKET) {
+				break;
+			}
+			case SYS_setsockopt:
+			case SYS_getsockopt:
+				print_number(ip, narg, c);
+				putchar(',');
+				sockoptlevelname(*ip, decimal);
+				if (*ip == SOL_SOCKET) {
 					ip++;
 					narg--;
-					(void)putchar(',');
-					sockoptname((int)*ip);
+					putchar(',');
+					sockoptname(*ip);
 				}
 				ip++;
 				narg--;
+				break;
 #ifdef SYS_freebsd6_lseek
-			} else if (ktr->ktr_code == SYS_freebsd6_lseek) {
-				print_number(ip,narg,c);
+			case SYS_freebsd6_lseek:
+				print_number(ip, narg, c);
 				/* Hidden 'pad' argument, not in lseek(2) */
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				whencename ((int)*ip);
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				whencename(*ip);
 				ip++;
 				narg--;
+				break;
 #endif
-			} else if (ktr->ktr_code == SYS_lseek) {
-				print_number(ip,narg,c);
+			case SYS_lseek:
+				print_number(ip, narg, c);
 				/* Hidden 'pad' argument, not in lseek(2) */
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				whencename ((int)*ip);
+				print_number(ip, narg, c);
+				putchar(',');
+				whencename(*ip);
 				ip++;
 				narg--;
-
-			} else if (ktr->ktr_code == SYS_flock) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				flockname((int)*ip);
+				break;
+			case SYS_flock:
+				print_number(ip, narg, c);
+				putchar(',');
+				flockname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_mkfifo ||
-				   ktr->ktr_code == SYS_mkdir) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				modename((int)*ip);
+				break;
+			case SYS_mkfifo:
+			case SYS_mkdir:
+				print_number(ip, narg, c);
+				putchar(',');
+				modename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_shutdown) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				shutdownhowname((int)*ip);
+				break;
+			case SYS_shutdown:
+				print_number(ip, narg, c);
+				putchar(',');
+				shutdownhowname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_socketpair) {
-				(void)putchar('(');
-				sockdomainname((int)*ip);
+				break;
+			case SYS_socketpair:
+				putchar('(');
+				sockdomainname(*ip);
 				ip++;
 				narg--;
-				(void)putchar(',');
-				socktypename((int)*ip);
+				putchar(',');
+				socktypename(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_getrlimit ||
-				   ktr->ktr_code == SYS_setrlimit) {
-				(void)putchar('(');
-				rlimitname((int)*ip);
+				break;
+			case SYS_getrlimit:
+			case SYS_setrlimit:
+				putchar('(');
+				rlimitname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_quotactl) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				quotactlname((int)*ip);
+				break;
+			case SYS_quotactl:
+				print_number(ip, narg, c);
+				putchar(',');
+				quotactlname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_nfssvc) {
-				(void)putchar('(');
-				nfssvcname((int)*ip);
+				break;
+			case SYS_nfssvc:
+				putchar('(');
+				nfssvcname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_rtprio) {
-				(void)putchar('(');
-				rtprioname((int)*ip);
+				break;
+			case SYS_rtprio:
+				putchar('(');
+				rtprioname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS___semctl) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				semctlname((int)*ip);
+				break;
+			case SYS___semctl:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				semctlname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_semget) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				semgetname((int)*ip);
+				break;
+			case SYS_semget:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				semgetname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_msgctl) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				shmctlname((int)*ip);
+				break;
+			case SYS_msgctl:
+				print_number(ip, narg, c);
+				putchar(',');
+				shmctlname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_shmat) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				shmatname((int)*ip);
+				break;
+			case SYS_shmat:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				shmatname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_shmctl) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				shmctlname((int)*ip);
+				break;
+			case SYS_shmctl:
+				print_number(ip, narg, c);
+				putchar(',');
+				shmctlname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_minherit) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				minheritname((int)*ip);
+				break;
+			case SYS_minherit:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				minheritname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_rfork) {
-				(void)putchar('(');
-				rforkname((int)*ip);
+				break;
+			case SYS_rfork:
+				putchar('(');
+				rforkname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_lio_listio) {
-				(void)putchar('(');
-				lio_listioname((int)*ip);
+				break;
+			case SYS_lio_listio:
+				putchar('(');
+				lio_listioname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_mlockall) {
-				(void)putchar('(');
-				mlockallname((int)*ip);
+				break;
+			case SYS_mlockall:
+				putchar('(');
+				mlockallname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_sched_setscheduler) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				schedpolicyname((int)*ip);
+				break;
+			case SYS_sched_setscheduler:
+				print_number(ip, narg, c);
+				putchar(',');
+				schedpolicyname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_sched_get_priority_max ||
-				   ktr->ktr_code == SYS_sched_get_priority_min) {
-				(void)putchar('(');
-				schedpolicyname((int)*ip);
+				break;
+			case SYS_sched_get_priority_max:
+			case SYS_sched_get_priority_min:
+				putchar('(');
+				schedpolicyname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_sendfile) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				sendfileflagsname((int)*ip);
+				break;
+			case SYS_sendfile:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				sendfileflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_kldsym) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				kldsymcmdname((int)*ip);
+				break;
+			case SYS_kldsym:
+				print_number(ip, narg, c);
+				putchar(',');
+				kldsymcmdname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_sigprocmask) {
-				(void)putchar('(');
-				sigprocmaskhowname((int)*ip);
+				break;
+			case SYS_sigprocmask:
+				putchar('(');
+				sigprocmaskhowname(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS___acl_get_file ||
-				   ktr->ktr_code == SYS___acl_set_file ||
-				   ktr->ktr_code == SYS___acl_get_fd ||
-				   ktr->ktr_code == SYS___acl_set_fd ||
-				   ktr->ktr_code == SYS___acl_delete_file ||
-				   ktr->ktr_code == SYS___acl_delete_fd ||
-				   ktr->ktr_code == SYS___acl_aclcheck_file ||
-				   ktr->ktr_code == SYS___acl_aclcheck_fd ||
-				   ktr->ktr_code == SYS___acl_get_link ||
-				   ktr->ktr_code == SYS___acl_set_link ||
-				   ktr->ktr_code == SYS___acl_delete_link ||
-				   ktr->ktr_code == SYS___acl_aclcheck_link) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				acltypename((int)*ip);
+				break;
+			case SYS___acl_get_file:
+			case SYS___acl_set_file:
+			case SYS___acl_get_fd:
+			case SYS___acl_set_fd:
+			case SYS___acl_delete_file:
+			case SYS___acl_delete_fd:
+			case SYS___acl_aclcheck_file:
+			case SYS___acl_aclcheck_fd:
+			case SYS___acl_get_link:
+			case SYS___acl_set_link:
+			case SYS___acl_delete_link:
+			case SYS___acl_aclcheck_link:
+				print_number(ip, narg, c);
+				putchar(',');
+				acltypename(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_sigaction) {
-				(void)putchar('(');
-				signame((int)*ip);
+				break;
+			case SYS_sigaction:
+				putchar('(');
+				signame(*ip);
 				ip++;
 				narg--;
 				c = ',';
-			} else if (ktr->ktr_code == SYS_extattrctl) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				extattrctlname((int)*ip);
+				break;
+			case SYS_extattrctl:
+				print_number(ip, narg, c);
+				putchar(',');
+				extattrctlname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_nmount) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				mountflagsname ((int)*ip);
+				break;
+			case SYS_nmount:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				mountflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_thr_create) {
-				print_number(ip,narg,c);
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				thrcreateflagsname ((int)*ip);
+				break;
+			case SYS_thr_create:
+				print_number(ip, narg, c);
+				print_number(ip, narg, c);
+				putchar(',');
+				thrcreateflagsname(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_thr_kill) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				signame ((int)*ip);
+				break;
+			case SYS_thr_kill:
+				print_number(ip, narg, c);
+				putchar(',');
+				signame(*ip);
 				ip++;
 				narg--;
-			} else if (ktr->ktr_code == SYS_kldunloadf) {
-				print_number(ip,narg,c);
-				(void)putchar(',');
-				kldunloadfflagsname ((int)*ip);
+				break;
+			case SYS_kldunloadf:
+				print_number(ip, narg, c);
+				putchar(',');
+				kldunloadfflagsname(*ip);
 				ip++;
 				narg--;
+				break;
+			case SYS_cap_new:
+				print_number(ip, narg, c);
+				putchar(',');
+				arg = *ip;
+				ip++;
+				narg--;
+				/*
+				 * Hack: the second argument is a
+				 * cap_rights_t, which 64 bits wide, so on
+				 * 32-bit systems, it is split between two
+				 * registers.
+				 *
+				 * Since sizeof() is not evaluated by the
+				 * preprocessor, we can't use an #ifdef,
+				 * but the compiler will probably optimize
+				 * the code out anyway.
+				 */
+				if (sizeof(cap_rights_t) > sizeof(register_t)) {
+#if _BYTE_ORDER == _LITTLE_ENDIAN
+					arg = ((intmax_t)*ip << 32) + arg;
+#else
+					arg = (arg << 32) + *ip;
+#endif
+					ip++;
+					narg--;
+				}
+				capname(arg);
+				break;
 			}
 		}
 		while (narg > 0) {
-			print_number(ip,narg,c);
+			print_number(ip, narg, c);
 		}
-		(void)putchar(')');
+		putchar(')');
 	}
-	(void)putchar('\n');
+	putchar('\n');
 }
 
 void
@@ -954,37 +1033,37 @@ ktrsysret(struct ktr_sysret *ktr, u_int flags)
 
 	if ((flags != 0 && ((flags & SV_ABI_MASK) != SV_ABI_FREEBSD)) ||
 	    (code >= nsyscalls || code < 0))
-		(void)printf("[%d] ", code);
+		printf("[%d] ", code);
 	else
-		(void)printf("%s ", syscallnames[code]);
+		printf("%s ", syscallnames[code]);
 
 	if (error == 0) {
 		if (fancy) {
-			(void)printf("%ld", (long)ret);
+			printf("%ld", (long)ret);
 			if (ret < 0 || ret > 9)
-				(void)printf("/%#lx", (long)ret);
+				printf("/%#lx", (unsigned long)ret);
 		} else {
 			if (decimal)
-				(void)printf("%ld", (long)ret);
+				printf("%ld", (long)ret);
 			else
-				(void)printf("%#lx", (long)ret);
+				printf("%#lx", (unsigned long)ret);
 		}
 	} else if (error == ERESTART)
-		(void)printf("RESTART");
+		printf("RESTART");
 	else if (error == EJUSTRETURN)
-		(void)printf("JUSTRETURN");
+		printf("JUSTRETURN");
 	else {
-		(void)printf("-1 errno %d", ktr->ktr_error);
+		printf("-1 errno %d", ktr->ktr_error);
 		if (fancy)
-			(void)printf(" %s", strerror(ktr->ktr_error));
+			printf(" %s", strerror(ktr->ktr_error));
 	}
-	(void)putchar('\n');
+	putchar('\n');
 }
 
 void
 ktrnamei(char *cp, int len)
 {
-	(void)printf("\"%.*s\"\n", len, cp);
+	printf("\"%.*s\"\n", len, cp);
 }
 
 void
@@ -1041,23 +1120,23 @@ visdump(char *dp, int datalen, int screenwidth)
 	int width;
 	char visbuf[5];
 
-	(void)printf("       \"");
+	printf("       \"");
 	col = 8;
 	for (;datalen > 0; datalen--, dp++) {
-		(void) vis(visbuf, *dp, VIS_CSTYLE, *(dp+1));
+		 vis(visbuf, *dp, VIS_CSTYLE, *(dp+1));
 		cp = visbuf;
 		/*
 		 * Keep track of printables and
 		 * space chars (like fold(1)).
 		 */
 		if (col == 0) {
-			(void)putchar('\t');
+			putchar('\t');
 			col = 8;
 		}
 		switch(*cp) {
 		case '\n':
 			col = 0;
-			(void)putchar('\n');
+			putchar('\n');
 			continue;
 		case '\t':
 			width = 8 - (col&07);
@@ -1066,17 +1145,17 @@ visdump(char *dp, int datalen, int screenwidth)
 			width = strlen(cp);
 		}
 		if (col + width > (screenwidth-2)) {
-			(void)printf("\\\n\t");
+			printf("\\\n\t");
 			col = 8;
 		}
 		col += width;
 		do {
-			(void)putchar(*cp++);
+			putchar(*cp++);
 		} while (*cp);
 	}
 	if (col == 0)
-		(void)printf("       ");
-	(void)printf("\"\n");
+		printf("       ");
+	printf("\"\n");
 }
 
 void
@@ -1130,13 +1209,13 @@ void
 ktrpsig(struct ktr_psig *psig)
 {
 	if (psig->signo > 0 && psig->signo < NSIG)
-		(void)printf("SIG%s ", signames[psig->signo]);
+		printf("SIG%s ", signames[psig->signo]);
 	else
-		(void)printf("SIG %d ", psig->signo);
+		printf("SIG %d ", psig->signo);
 	if (psig->action == SIG_DFL)
-		(void)printf("SIG_DFL code=0x%x\n", psig->code);
+		printf("SIG_DFL code=0x%x\n", psig->code);
 	else {
-		(void)printf("caught handler=0x%lx mask=0x%x code=0x%x\n",
+		printf("caught handler=0x%lx mask=0x%x code=0x%x\n",
 		    (u_long)psig->action, psig->mask.__bits[0], psig->code);
 	}
 }
@@ -1144,7 +1223,7 @@ ktrpsig(struct ktr_psig *psig)
 void
 ktrcsw(struct ktr_csw *cs)
 {
-	(void)printf("%s %s\n", cs->out ? "stop" : "resume",
+	printf("%s %s\n", cs->out ? "stop" : "resume",
 		cs->user ? "user" : "kernel");
 }
 
@@ -1256,7 +1335,7 @@ struct utrace_malloc {
 };
 
 void
-ktruser_malloc(int len, unsigned char *p)
+ktruser_malloc(unsigned char *p)
 {
 	struct utrace_malloc *ut = (struct utrace_malloc *)p;
 
@@ -1280,17 +1359,17 @@ ktruser(int len, unsigned char *p)
 	}
 
 	if (len == sizeof(struct utrace_malloc)) {
-		ktruser_malloc(len, p);
+		ktruser_malloc(p);
 		return;
 	}
 
-	(void)printf("%d ", len);
+	printf("%d ", len);
 	while (len--)
 		if (decimal)
-			(void)printf(" %d", *p++);
+			printf(" %d", *p++);
 		else
-			(void)printf(" %02x", *p++);
-	(void)printf("\n");
+			printf(" %02x", *p++);
+	printf("\n");
 }
 
 void
@@ -1315,61 +1394,67 @@ ktrsockaddr(struct sockaddr *sa)
 	printf(", ");
 
 #define check_sockaddr_len(n)					\
-	if (sa_##n->s##n##_len < sizeof(struct sockaddr_##n)) {	\
+	if (sa_##n.s##n##_len < sizeof(struct sockaddr_##n)) {	\
 		printf("invalid");				\
 		break;						\
 	}
 
 	switch(sa->sa_family) {
 	case AF_INET: {
-		struct sockaddr_in	*sa_in;
+		struct sockaddr_in sa_in;
 
-		sa_in = (struct sockaddr_in *)sa;
+		memset(&sa_in, 0, sizeof(sa_in));
+		memcpy(&sa_in, sa, sizeof(sa));
 		check_sockaddr_len(in);
-		inet_ntop(AF_INET, &sa_in->sin_addr, addr, sizeof addr);
-		printf("%s:%u", addr, ntohs(sa_in->sin_port));
+		inet_ntop(AF_INET, &sa_in.sin_addr, addr, sizeof addr);
+		printf("%s:%u", addr, ntohs(sa_in.sin_port));
 		break;
 	}
 #ifdef NETATALK
 	case AF_APPLETALK: {
-		struct sockaddr_at	*sa_at;
+		struct sockaddr_at	sa_at;
 		struct netrange		*nr;
 
-		sa_at = (struct sockaddr_at *)sa;
+		memset(&sa_at, 0, sizeof(sa_at));
+		memcpy(&sa_at, sa, sizeof(sa));
 		check_sockaddr_len(at);
-		nr = &sa_at->sat_range.r_netrange;
-		printf("%d.%d, %d-%d, %d", ntohs(sa_at->sat_addr.s_net),
-			sa_at->sat_addr.s_node, ntohs(nr->nr_firstnet),
+		nr = &sa_at.sat_range.r_netrange;
+		printf("%d.%d, %d-%d, %d", ntohs(sa_at.sat_addr.s_net),
+			sa_at.sat_addr.s_node, ntohs(nr->nr_firstnet),
 			ntohs(nr->nr_lastnet), nr->nr_phase);
 		break;
 	}
 #endif
 	case AF_INET6: {
-		struct sockaddr_in6	*sa_in6;
+		struct sockaddr_in6 sa_in6;
 
-		sa_in6 = (struct sockaddr_in6 *)sa;
+		memset(&sa_in6, 0, sizeof(sa_in6));
+		memcpy(&sa_in6, sa, sizeof(sa));
 		check_sockaddr_len(in6);
-		inet_ntop(AF_INET6, &sa_in6->sin6_addr, addr, sizeof addr);
-		printf("[%s]:%u", addr, htons(sa_in6->sin6_port));
+		inet_ntop(AF_INET6, &sa_in6.sin6_addr, addr, sizeof addr);
+		printf("[%s]:%u", addr, htons(sa_in6.sin6_port));
 		break;
 	}
 #ifdef IPX
 	case AF_IPX: {
-		struct sockaddr_ipx	*sa_ipx;
+		struct sockaddr_ipx sa_ipx;
 
-		sa_ipx = (struct sockaddr_ipx *)sa;
+		memset(&sa_ipx, 0, sizeof(sa_ipx));
+		memcpy(&sa_ipx, sa, sizeof(sa));
 		check_sockaddr_len(ipx);
 		/* XXX wish we had ipx_ntop */
-		printf("%s", ipx_ntoa(sa_ipx->sipx_addr));
+		printf("%s", ipx_ntoa(sa_ipx.sipx_addr));
+		free(sa_ipx);
 		break;
 	}
 #endif
 	case AF_UNIX: {
-		struct sockaddr_un *sa_un;
+		struct sockaddr_un sa_un;
 
-		sa_un = (struct sockaddr_un *)sa;
+		memset(&sa_un, 0, sizeof(sa_un));
+		memcpy(&sa_un, sa, sizeof(sa));
 		check_sockaddr_len(un);
-		printf("%.*s", (int)sizeof(sa_un->sun_path), sa_un->sun_path);
+		printf("%.*s", (int)sizeof(sa_un.sun_path), sa_un.sun_path);
 		break;
 	}
 	default:
@@ -1409,7 +1494,7 @@ ktrstat(struct stat *statp)
 		printf("%jd", (intmax_t)statp->st_atim.tv_sec);
 	else {
 		tm = localtime(&statp->st_atim.tv_sec);
-		(void)strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
+		strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
 		printf("\"%s\"", timestr);
 	}
 	if (statp->st_atim.tv_nsec != 0)
@@ -1421,7 +1506,7 @@ ktrstat(struct stat *statp)
 		printf("%jd", (intmax_t)statp->st_mtim.tv_sec);
 	else {
 		tm = localtime(&statp->st_mtim.tv_sec);
-		(void)strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
+		strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
 		printf("\"%s\"", timestr);
 	}
 	if (statp->st_mtim.tv_nsec != 0)
@@ -1433,7 +1518,7 @@ ktrstat(struct stat *statp)
 		printf("%jd", (intmax_t)statp->st_ctim.tv_sec);
 	else {
 		tm = localtime(&statp->st_ctim.tv_sec);
-		(void)strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
+		strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
 		printf("\"%s\"", timestr);
 	}
 	if (statp->st_ctim.tv_nsec != 0)
@@ -1445,7 +1530,7 @@ ktrstat(struct stat *statp)
 		printf("%jd", (intmax_t)statp->st_birthtim.tv_sec);
 	else {
 		tm = localtime(&statp->st_birthtim.tv_sec);
-		(void)strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
+		strftime(timestr, sizeof(timestr), TIME_FORMAT, tm);
 		printf("\"%s\"", timestr);
 	}
 	if (statp->st_birthtim.tv_nsec != 0)
@@ -1480,8 +1565,8 @@ ktrstruct(char *buf, size_t buflen)
 	if (datalen == 0)
 		goto invalid;
 	/* sanity check */
-	for (i = 0; i < namelen; ++i)
-		if (!isalpha((unsigned char)name[i]))
+	for (i = 0; i < (int)namelen; ++i)
+		if (!isalpha(name[i]))
 			goto invalid;
 	if (strcmp(name, "stat") == 0) {
 		if (datalen != sizeof(struct stat))
@@ -1504,6 +1589,41 @@ ktrstruct(char *buf, size_t buflen)
 	printf("invalid record\n");
 }
 
+void
+ktrcapfail(struct ktr_cap_fail *ktr)
+{
+	switch (ktr->cap_type) {
+	case CAPFAIL_NOTCAPABLE:
+		/* operation on fd with insufficient capabilities */
+		printf("operation requires ");
+		capname((intmax_t)ktr->cap_needed);
+		printf(", process holds ");
+		capname((intmax_t)ktr->cap_held);
+		break;
+	case CAPFAIL_INCREASE:
+		/* requested more capabilities than fd already has */
+		printf("attempt to increase capabilities from ");
+		capname((intmax_t)ktr->cap_held);
+		printf(" to ");
+		capname((intmax_t)ktr->cap_needed);
+		break;
+	case CAPFAIL_SYSCALL:
+		/* called restricted syscall */
+		printf("disallowed system call");
+		break;
+	case CAPFAIL_LOOKUP:
+		/* used ".." in strict-relative mode */
+		printf("restricted VFS lookup");
+		break;
+	default:
+		printf("unknown capability failure: ");
+		capname((intmax_t)ktr->cap_needed);
+		printf(" ");
+		capname((intmax_t)ktr->cap_held);
+		break;
+	}
+}
+
 #if defined(__amd64__) || defined(__i386__)
 void
 linux_ktrsyscall(struct ktr_syscall *ktr)
@@ -1541,12 +1661,12 @@ linux_ktrsysret(struct ktr_sysret *ktr)
 		if (fancy) {
 			printf("%ld", (long)ret);
 			if (ret < 0 || ret > 9)
-				printf("/%#lx", (long)ret);
+				printf("/%#lx", (unsigned long)ret);
 		} else {
 			if (decimal)
 				printf("%ld", (long)ret);
 			else
-				printf("%#lx", (long)ret);
+				printf("%#lx", (unsigned long)ret);
 		}
 	} else if (error == ERESTART)
 		printf("RESTART");
diff --git a/usr.bin/kdump/kdump_subr.h b/usr.bin/kdump/kdump_subr.h
deleted file mode 100644
index 50cf9ee4c7f8..000000000000
--- a/usr.bin/kdump/kdump_subr.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* $FreeBSD$ */
-
-void signame (int);
-void semctlname (int);
-void shmctlname (int);
-void semgetname (int);
-void fcntlcmdname (int, int, int);
-void rtprioname (int);
-void modename (int);
-void flagsname (int);
-void flagsandmodename (int, int, int);
-void accessmodename (int);
-void mmapprotname (int);
-void mmapflagsname (int);
-void wait4optname (int);
-void sendrecvflagsname (int);
-void getfsstatflagsname (int);
-void mountflagsname (int);
-void rebootoptname (int);
-void flockname (int);
-void sockoptname (int);
-void sockoptlevelname (int, int);
-void sockdomainname (int);
-void sockipprotoname (int);
-void socktypename (int);
-void thrcreateflagsname (int);
-void mlockallname (int);
-void shmatname (int);
-void rforkname (int);
-void nfssvcname (int);
-void whencename (int);
-void rlimitname (int);
-void shutdownhowname (int);
-void prioname (int);
-void madvisebehavname (int);
-void msyncflagsname (int);
-void schedpolicyname (int);
-void kldunloadfflagsname (int);
-void extattrctlname (int);
-void kldsymcmdname (int);
-void sendfileflagsname (int);
-void acltypename (int);
-void sigprocmaskhowname (int);
-void lio_listioname (int);
-void minheritname (int);
-void quotactlname (int);
-void ptraceopname (int);
diff --git a/usr.bin/kdump/mkioctls b/usr.bin/kdump/mkioctls
index 7ca773dca9de..22ca26854dc1 100644
--- a/usr.bin/kdump/mkioctls
+++ b/usr.bin/kdump/mkioctls
@@ -1,36 +1,45 @@
 #!/bin/sh
 #
 # $FreeBSD$
+#
+# When editing this script, keep in mind that truss also uses it.
+#
 
 set -e
 
-if [ "x$1" = "x-s" ]; then
-	use_switch=1
-	shift
-else
-	use_switch=0
-fi
-
-if [ -z "$1" ]; then
-	echo "usage: sh $0 [-s] include-dir"
+if [ $# -ne 2 -o \( $1 != "print" -a $1 != "return" \) ]; then
+	echo "usage: sh $0 print|return include-dir"
 	exit 1
 fi
 
+style="$1"
+includedir="$2"
+
 LC_ALL=C; export LC_ALL
 
 # Build a list of headers that have ioctls in them.
 # XXX should we use an ANSI cpp?
-ioctl_includes=`
-	cd $1
-	find -H -s * -name '*.h' |
+ioctl_includes=$(
+	cd $includedir
+	find -H -s * -name '*.h' | grep -v '.*disk.*\.h' | \
 		xargs egrep -l \
 '^#[ 	]*define[ 	]+[A-Za-z_][A-Za-z0-9_]*[ 	]+_IO[^a-z0-9_]' |
-		awk '{printf("#include <%s>\\\\n", $1)}'
-`
+		awk '{printf("#include <%s>\\n", $1)}'
+)
+
+: ${MACHINE=$(uname -m)}
+case "${MACHINE}" in
+*pc98*)
+	ioctl_includes="$ioctl_includes#include <sys/diskpc98.h>\\n"
+	;;
+*)
+	ioctl_includes="$ioctl_includes#include <sys/diskmbr.h>\\n"
+	;;
+esac
 
 awk -v x="$ioctl_includes" 'BEGIN {print x}' |
 	gcc -E -I$1 -dM -DCOMPAT_43TTY - |
-	awk -v ioctl_includes="$ioctl_includes" -v use_switch="$use_switch" '
+	awk -v ioctl_includes="$ioctl_includes" -v style="$style" '
 BEGIN {
 	print "/* XXX obnoxious prerequisites. */"
 	print "#define COMPAT_43"
@@ -55,16 +64,22 @@ BEGIN {
 	print "#include <stdio.h>"
 	print "#include <cam/cam.h>"
 	print ""
-	print "const char *ioctlname(u_long val);"
-	print ""
 	print ioctl_includes
 	print ""
-	print "const char *"
-	print "ioctlname(u_long val)"
+	if (style == "print") {
+		print "void ioctlname(unsigned long val, int decimal);"
+		print ""
+		print "void"
+		print "ioctlname(unsigned long val, int decimal)"
+	} else {
+		print "const char *ioctlname(unsigned long val);"
+		print ""
+		print "const char *"
+		print "ioctlname(unsigned long val)"
+	}
 	print "{"
+	print "\tconst char *str = NULL;"
 	print ""
-	if (use_switch)
-		print "\tswitch(val) {"
 }
 
 /^#[ 	]*define[ 	]+[A-Za-z_][A-Za-z0-9_]*[ 	]+_IO/ {
@@ -75,16 +90,24 @@ BEGIN {
 			break;
 	++i;
 	#
-	if (use_switch)
-		printf("\tcase %s:\n\t\treturn(\"%s\");\n", $i, $i);
-	else
-		printf("\tif (val ==  %s)\n\t\treturn(\"%s\");\n", $i, $i);
-
+	printf("\t");
+	if (n++ > 0)
+		printf("else ");
+	printf("if (val == %s)\n", $i);
+	printf("\t\tstr = \"%s\";\n", $i);
 }
 END {
-	if (use_switch)
-		print "\t}"
-	print "\n\treturn(NULL);"
+	print ""
+	if (style == "print") {
+		print "\tif (str != NULL)"
+		print "\t\tprintf(\"%s\", str);"
+		print "\telse if (decimal)"
+		print "\t\tprintf(\"%lu\", val);"
+		print "\telse"
+		print "\t\tprintf(\"%#lx\", val);"
+	} else {
+		print "\treturn (str);"
+	}
 	print "}"
 }
 '
diff --git a/usr.bin/kdump/mksubr b/usr.bin/kdump/mksubr
index d6fa870f02ea..75846ee81a23 100644
--- a/usr.bin/kdump/mksubr
+++ b/usr.bin/kdump/mksubr
@@ -69,10 +69,10 @@ auto_or_type () {
 	cat <<_EOF_
 /* AUTO */
 void
-$name (int arg)
+$name(intmax_t arg)
 {
-	int	or = 0;
-	printf("%#x<", arg);
+	int or = 0;
+	printf("%#jx<", (uintmax_t)arg);
 _EOF_
 	egrep "^#[[:space:]]*define[[:space:]]+"${grep}"[[:space:]]*" \
 		$include_dir/$file | \
@@ -80,11 +80,11 @@ _EOF_
 		if ($i ~ /define/) \
 			break; \
 		++i; \
-		printf "\tif(!((arg>0)^((%s)>0)))\n\t\tif_print_or(arg, %s, or);\n", $i, $i }'
+		printf "\tif (!((arg > 0) ^ ((%s) > 0)))\n\t\tif_print_or(arg, %s, or);\n", $i, $i }'
 cat <<_EOF_
 	printf(">");
 	if (or == 0)
-		(void)printf("<invalid>%ld", (long)arg);
+		printf("<invalid>%jd", arg);
 }
 
 _EOF_
@@ -103,7 +103,7 @@ auto_switch_type () {
 	cat <<_EOF_
 /* AUTO */
 void
-$name (int arg)
+$name(intmax_t arg)
 {
 	switch (arg) {
 _EOF_
@@ -113,10 +113,10 @@ _EOF_
 		if ($i ~ /define/) \
 			break; \
 		++i; \
-		printf "\tcase %s:\n\t\t(void)printf(\"%s\");\n\t\tbreak;\n", $i, $i }'
+		printf "\tcase %s:\n\t\tprintf(\"%s\");\n\t\tbreak;\n", $i, $i }'
 cat <<_EOF_
 	default: /* Should not reach */
-		(void)printf("<invalid=%ld>", (long)arg);
+		printf("<invalid=%jd>", arg);
 	}
 }
 
@@ -136,7 +136,7 @@ auto_if_type () {
 	cat <<_EOF_
 /* AUTO */
 void
-$name (int arg)
+$name(intmax_t arg)
 {
 _EOF_
 	egrep "^#[[:space:]]*define[[:space:]]+"${grep}"[[:space:]]*" \
@@ -147,7 +147,7 @@ _EOF_
 		printf "if (arg == %s) \n\t\tprintf(\"%s\");\n", $2, $2 }'
 cat <<_EOF_
 	else /* Should not reach */
-		(void)printf("<invalid=%ld>", (long)arg);
+		printf("<invalid=%jd>", arg);
 }
 
 _EOF_
@@ -156,6 +156,7 @@ _EOF_
 # C start
 
 cat <<_EOF_
+#include <stdint.h>
 #include <stdio.h>
 #include <sys/fcntl.h>
 #include <sys/stat.h>
@@ -185,6 +186,7 @@ cat <<_EOF_
 #include <sys/shm.h>
 #include <nfsserver/nfs.h>
 #include <ufs/ufs/quota.h>
+#include <sys/capability.h>
 
 #include "kdump_subr.h"
 
@@ -207,76 +209,78 @@ cat <<_EOF_
 /* MANUAL */
 extern char *signames[]; /* from kdump.c */
 void
-signame (int sig)
+signame(int sig)
 {
 	if (sig > 0 && sig < NSIG)
-		(void)printf("SIG%s",signames[sig]);
+		printf("SIG%s",signames[sig]);
 	else
-		(void)printf("SIG %d", sig);
+		printf("SIG %d", sig);
 }
 
 /* MANUAL */
 void
-semctlname (int cmd)
+semctlname(int cmd)
 {
 	switch (cmd) {
 	case GETNCNT:
-		(void)printf("GETNCNT");
+		printf("GETNCNT");
 		break;
 	case GETPID:
-		(void)printf("GETPID");
+		printf("GETPID");
 		break;
 	case GETVAL:
-		(void)printf("GETVAL");
+		printf("GETVAL");
 		break;
 	case GETALL:
-		(void)printf("GETALL");
+		printf("GETALL");
 		break;
 	case GETZCNT:
-		(void)printf("GETZCNT");
+		printf("GETZCNT");
 		break;
 	case SETVAL:
-		(void)printf("SETVAL");
+		printf("SETVAL");
 		break;
 	case SETALL:
-		(void)printf("SETALL");
+		printf("SETALL");
 		break;
 	case IPC_RMID:
-		(void)printf("IPC_RMID");
+		printf("IPC_RMID");
 		break;
 	case IPC_SET:
-		(void)printf("IPC_SET");
+		printf("IPC_SET");
 		break;
 	case IPC_STAT:
-		(void)printf("IPC_STAT");
+		printf("IPC_STAT");
 		break;
 	default: /* Should not reach */
-		(void)printf("<invalid=%ld>", (long)cmd);
+		printf("<invalid=%d>", cmd);
 	}
 }
 
 /* MANUAL */
 void
-shmctlname (int cmd) {
+shmctlname(int cmd)
+{
 	switch (cmd) {
 	case IPC_RMID:
-		(void)printf("IPC_RMID");
+		printf("IPC_RMID");
 		break;
 	case IPC_SET:
-		(void)printf("IPC_SET");
+		printf("IPC_SET");
 		break;
 	case IPC_STAT:
-		(void)printf("IPC_STAT");
+		printf("IPC_STAT");
 		break;
 	default: /* Should not reach */
-		(void)printf("<invalid=%ld>", (long)cmd);
+		printf("<invalid=%d>", cmd);
 	}
 }
 
 /* MANUAL */
 void
-semgetname (int flag) {
-	int	or = 0;
+semgetname(int flag)
+{
+	int or = 0;
 	if_print_or(flag, IPC_CREAT, or);
 	if_print_or(flag, IPC_EXCL, or);
 	if_print_or(flag, SEM_R, or);
@@ -294,16 +298,17 @@ semgetname (int flag) {
  * mode argument is unused (and often bogus and misleading).
  */
 void
-flagsandmodename (int flags, int mode, int decimal) {
-	flagsname (flags);
-	(void)putchar(',');
+flagsandmodename(int flags, int mode, int decimal)
+{
+	flagsname(flags);
+	putchar(',');
 	if ((flags & O_CREAT) == O_CREAT) {
 		modename (mode);
 	} else {
 		if (decimal) {
-			(void)printf("<unused>%ld", (long)mode);
+			printf("<unused>%d", mode);
 		} else {
-			(void)printf("<unused>%#lx", (long)mode);
+			printf("<unused>%#x", (unsigned int)mode);
 		}
 	}
 }
@@ -316,59 +321,59 @@ flagsandmodename (int flags, int mode, int decimal) {
  * to use getprotoent(3) here.
  */
 void
-sockoptlevelname (int level, int decimal)
+sockoptlevelname(int level, int decimal)
 {
 	if (level == SOL_SOCKET) {
-		(void)printf("SOL_SOCKET");
+		printf("SOL_SOCKET");
 	} else {
 		if (decimal) {
-			(void)printf("%ld", (long)level);
+			printf("%d", level);
 		} else {
-			(void)printf("%#lx", (long)level);
+			printf("%#x", (unsigned int)level);
 		}
 	}
 }
 
 _EOF_
 
-auto_or_type "modename" "S_[A-Z]+[[:space:]]+[0-6]{7}" "sys/stat.h"
-auto_or_type "flagsname" "O_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/fcntl.h"
-auto_or_type "accessmodename" "[A-Z]_OK[[:space:]]+0?x?[0-9A-Fa-f]+" "sys/unistd.h"
-auto_or_type "mmapprotname" "PROT_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h"
-auto_or_type "mmapflagsname" "MAP_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h"
-auto_or_type "wait4optname" "W[A-Z]+[[:space:]]+[0-9]+" "sys/wait.h"
-auto_or_type "getfsstatflagsname" "MNT_[A-Z]+[[:space:]]+[1-9][0-9]*" "sys/mount.h"
-auto_or_type "mountflagsname" "MNT_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mount.h"
-auto_or_type "rebootoptname" "RB_[A-Z]+[[:space:]]+0x[0-9]+" "sys/reboot.h"
-auto_or_type "flockname" "LOCK_[A-Z]+[[:space:]]+0x[0-9]+" "sys/fcntl.h"
-auto_or_type "thrcreateflagsname" "THR_[A-Z]+[[:space:]]+0x[0-9]+" "sys/thr.h"
-auto_or_type "mlockallname" "MCL_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mman.h"
-auto_or_type "shmatname" "SHM_[A-Z]+[[:space:]]+[0-9]{6}+" "sys/shm.h"
-auto_or_type "rforkname" "RF[A-Z]+[[:space:]]+\([0-9]+<<[0-9]+\)" "sys/unistd.h"
-auto_or_type "nfssvcname" "NFSSVC_[A-Z]+[[:space:]]+0x[0-9]+" "nfsserver/nfs.h"
-
-auto_switch_type "whencename" "SEEK_[A-Z]+[[:space:]]+[0-9]+" "sys/unistd.h"
-auto_switch_type "rlimitname" "RLIMIT_[A-Z]+[[:space:]]+[0-9]+" "sys/resource.h"
-auto_switch_type "shutdownhowname" "SHUT_[A-Z]+[[:space:]]+[0-9]+" "sys/socket.h"
-auto_switch_type "prioname" "PRIO_[A-Z]+[[:space:]]+[0-9]" "sys/resource.h"
-auto_switch_type "madvisebehavname" "_?MADV_[A-Z]+[[:space:]]+[0-9]+" "sys/mman.h"
-auto_switch_type "msyncflagsname" "MS_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mman.h"
-auto_switch_type "schedpolicyname" "SCHED_[A-Z]+[[:space:]]+[0-9]+" "sched.h"
-auto_switch_type "kldunloadfflagsname" "LINKER_UNLOAD_[A-Z]+[[:space:]]+[0-9]+" "sys/linker.h"
-auto_switch_type "extattrctlname" "EXTATTR_NAMESPACE_[A-Z]+[[:space:]]+0x[0-9]+" "sys/extattr.h"
-auto_switch_type "kldsymcmdname" "KLDSYM_[A-Z]+[[:space:]]+[0-9]+" "sys/linker.h"
-auto_switch_type "sendfileflagsname" "SF_[A-Z]+[[:space:]]+[0-9]+" "sys/socket.h"
-auto_switch_type "acltypename" "ACL_TYPE_[A-Z4_]+[[:space:]]+0x[0-9]+" "sys/acl.h"
-auto_switch_type "sigprocmaskhowname" "SIG_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h"
-auto_switch_type "lio_listioname" "LIO_(NO)?WAIT[[:space:]]+[0-9]+" "aio.h"
-auto_switch_type "minheritname" "INHERIT_[A-Z]+[[:space:]]+[0-9]+" "sys/mman.h"
-auto_switch_type "quotactlname" "Q_[A-Z]+[[:space:]]+0x[0-9]+" "ufs/ufs/quota.h"
-auto_if_type "sockdomainname" "PF_[[:alnum:]]+[[:space:]]+" "sys/socket.h"
-auto_if_type "sockfamilyname" "AF_[[:alnum:]]+[[:space:]]+" "sys/socket.h"
-auto_if_type "sockipprotoname" "IPPROTO_[[:alnum:]]+[[:space:]]+" "netinet/in.h"
-auto_switch_type "sockoptname" "SO_[A-Z]+[[:space:]]+0x[0-9]+" "sys/socket.h"
-auto_switch_type "socktypename" "SOCK_[A-Z]+[[:space:]]+[1-9]+[0-9]*" "sys/socket.h"
-auto_switch_type "ptraceopname" "PT_[[:alnum:]_]+[[:space:]]+[0-9]+" "sys/ptrace.h"
+auto_or_type     "accessmodename"      "[A-Z]_OK[[:space:]]+0?x?[0-9A-Fa-f]+"         "sys/unistd.h"
+auto_switch_type "acltypename"         "ACL_TYPE_[A-Z4_]+[[:space:]]+0x[0-9]+"        "sys/acl.h"
+auto_or_type     "capname"             "CAP_[A-Z]+[[:space:]]+0x[01248]{16}ULL"       "sys/capability.h"
+auto_switch_type "extattrctlname"      "EXTATTR_NAMESPACE_[A-Z]+[[:space:]]+0x[0-9]+" "sys/extattr.h"
+auto_or_type     "flagsname"           "O_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+"           "sys/fcntl.h"
+auto_or_type     "flockname"           "LOCK_[A-Z]+[[:space:]]+0x[0-9]+"              "sys/fcntl.h"
+auto_or_type     "getfsstatflagsname"  "MNT_[A-Z]+[[:space:]]+[1-9][0-9]*"            "sys/mount.h"
+auto_switch_type "kldsymcmdname"       "KLDSYM_[A-Z]+[[:space:]]+[0-9]+"              "sys/linker.h"
+auto_switch_type "kldunloadfflagsname" "LINKER_UNLOAD_[A-Z]+[[:space:]]+[0-9]+"       "sys/linker.h"
+auto_switch_type "lio_listioname"      "LIO_(NO)?WAIT[[:space:]]+[0-9]+"              "aio.h"
+auto_switch_type "madvisebehavname"    "_?MADV_[A-Z]+[[:space:]]+[0-9]+"              "sys/mman.h"
+auto_switch_type "minheritname"        "INHERIT_[A-Z]+[[:space:]]+[0-9]+"             "sys/mman.h"
+auto_or_type     "mlockallname"        "MCL_[A-Z]+[[:space:]]+0x[0-9]+"               "sys/mman.h"
+auto_or_type     "mmapflagsname"       "MAP_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+"         "sys/mman.h"
+auto_or_type     "mmapprotname"        "PROT_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+"        "sys/mman.h"
+auto_or_type     "modename"            "S_[A-Z]+[[:space:]]+[0-6]{7}"                 "sys/stat.h"
+auto_or_type     "mountflagsname"      "MNT_[A-Z]+[[:space:]]+0x[0-9]+"               "sys/mount.h"
+auto_switch_type "msyncflagsname"      "MS_[A-Z]+[[:space:]]+0x[0-9]+"                "sys/mman.h"
+auto_or_type     "nfssvcname"          "NFSSVC_[A-Z]+[[:space:]]+0x[0-9]+"            "nfsserver/nfs.h"
+auto_switch_type "prioname"            "PRIO_[A-Z]+[[:space:]]+[0-9]"                 "sys/resource.h"
+auto_switch_type "ptraceopname"        "PT_[[:alnum:]_]+[[:space:]]+[0-9]+"           "sys/ptrace.h"
+auto_switch_type "quotactlname"        "Q_[A-Z]+[[:space:]]+0x[0-9]+"                 "ufs/ufs/quota.h"
+auto_or_type     "rebootoptname"       "RB_[A-Z]+[[:space:]]+0x[0-9]+"                "sys/reboot.h"
+auto_or_type     "rforkname"           "RF[A-Z]+[[:space:]]+\([0-9]+<<[0-9]+\)"       "sys/unistd.h"
+auto_switch_type "rlimitname"          "RLIMIT_[A-Z]+[[:space:]]+[0-9]+"              "sys/resource.h"
+auto_switch_type "schedpolicyname"     "SCHED_[A-Z]+[[:space:]]+[0-9]+"               "sched.h"
+auto_switch_type "sendfileflagsname"   "SF_[A-Z]+[[:space:]]+[0-9]+"                  "sys/socket.h"
+auto_or_type     "shmatname"           "SHM_[A-Z]+[[:space:]]+[0-9]{6}+"              "sys/shm.h"
+auto_switch_type "shutdownhowname"     "SHUT_[A-Z]+[[:space:]]+[0-9]+"                "sys/socket.h"
+auto_switch_type "sigprocmaskhowname"  "SIG_[A-Z]+[[:space:]]+[0-9]+"                 "sys/signal.h"
+auto_if_type     "sockdomainname"      "PF_[[:alnum:]]+[[:space:]]+"                  "sys/socket.h"
+auto_if_type     "sockfamilyname"      "AF_[[:alnum:]]+[[:space:]]+"                  "sys/socket.h"
+auto_if_type     "sockipprotoname"     "IPPROTO_[[:alnum:]]+[[:space:]]+"             "netinet/in.h"
+auto_switch_type "sockoptname"         "SO_[A-Z]+[[:space:]]+0x[0-9]+"                "sys/socket.h"
+auto_switch_type "socktypename"        "SOCK_[A-Z]+[[:space:]]+[1-9]+[0-9]*"          "sys/socket.h"
+auto_or_type     "thrcreateflagsname"  "THR_[A-Z]+[[:space:]]+0x[0-9]+"               "sys/thr.h"
+auto_or_type     "wait4optname"        "W[A-Z]+[[:space:]]+[0-9]+"                    "sys/wait.h"
+auto_switch_type "whencename"          "SEEK_[A-Z]+[[:space:]]+[0-9]+"                "sys/unistd.h"
 
 cat <<_EOF_
 /*
@@ -377,7 +382,7 @@ cat <<_EOF_
  * grouped in fcntl.h, and this awk script grabs the first group.
  */
 void
-fcntlcmdname (int cmd, int arg, int decimal)
+fcntlcmdname(int cmd, int arg, int decimal)
 {
 	switch (cmd) {
 _EOF_
@@ -388,33 +393,33 @@ egrep "^#[[:space:]]*define[[:space:]]+F_[A-Z]+[[:space:]]+[0-9]+[[:space:]]*" \
 			break; \
 		++i; \
 		if (o <= $(i+1)) \
-			printf "\tcase %s:\n\t\t(void)printf(\"%s\");\n\t\tbreak;\n", $i, $i; \
+			printf "\tcase %s:\n\t\tprintf(\"%s\");\n\t\tbreak;\n", $i, $i; \
 		else \
 			exit; \
 		o = $(i+1) }'
 cat <<_EOF_
 	default: /* Should not reach */
-		(void)printf("<invalid=%ld>", (long)cmd);
+		printf("<invalid=%d>", cmd);
 	}
-	(void)putchar(',');
+	putchar(',');
 	if (cmd == F_GETFD || cmd == F_SETFD) {
 		if (arg == FD_CLOEXEC)
-			(void)printf("FD_CLOEXEC");
+			printf("FD_CLOEXEC");
 		else if (arg == 0)
-			(void)printf("0");
+			printf("0");
 		else {
 			if (decimal)
-				(void)printf("<invalid>%ld", (long)arg);
+				printf("<invalid>%d", arg);
 			else
-				(void)printf("<invalid>%#lx", (long)arg);
+				printf("<invalid>%#x", (unsigned int)arg);
 		}
 	} else if (cmd == F_SETFL) {
 		flagsname(arg);
 	} else {
 		if (decimal)
-			(void)printf("%ld", (long)arg);
+			printf("%d", arg);
 		else
-			(void)printf("%#lx", (long)arg);
+			printf("%#x", (unsigned int)arg);
 	}
 }
 
@@ -426,7 +431,7 @@ cat <<_EOF_
  * make this capable of being a auto_switch_type() function.
  */
 void
-rtprioname (int func)
+rtprioname(int func)
 {
 	switch (func) {
 _EOF_
@@ -436,10 +441,10 @@ egrep "^#[[:space:]]*define[[:space:]]+RTP_[A-Z]+[[:space:]]+0x[0-9]+[[:space:]]
 		if ($i ~ /define/) \
 			break; \
 		++i; \
-		printf "\tcase %s:\n\t\t(void)printf(\"%s\");\n\t\tbreak;\n", $i, $i }'
+		printf "\tcase %s:\n\t\tprintf(\"%s\");\n\t\tbreak;\n", $i, $i }'
 cat <<_EOF_
 	default: /* Should not reach */
-		(void)printf("<invalid=%ld>", (long)func);
+		printf("<invalid=%d>", func);
 	}
 }
 
@@ -451,12 +456,12 @@ cat <<_EOF_
  * detect this as "invalid", which is incorrect here.
  */
 void
-sendrecvflagsname (int flags)
+sendrecvflagsname(int flags)
 {
-	int	or = 0;
+	int or = 0;
 
 	if (flags == 0) {
-		(void)printf("0");
+		printf("0");
 		return;
 	}
 
diff --git a/usr.bin/ktrace/Makefile b/usr.bin/ktrace/Makefile
index 267992353ae7..c00bb7510459 100644
--- a/usr.bin/ktrace/Makefile
+++ b/usr.bin/ktrace/Makefile
@@ -5,6 +5,4 @@ PROG=	ktrace
 SRCS=	ktrace.c subr.c
 MLINKS= ktrace.1 trace.1 
 
-WARNS?=	4
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/ktrace/ktrace.1 b/usr.bin/ktrace/ktrace.1
index 9cff05b021cc..86f23ef86537 100644
--- a/usr.bin/ktrace/ktrace.1
+++ b/usr.bin/ktrace/ktrace.1
@@ -28,7 +28,7 @@
 .\"	@(#)ktrace.1	8.1 (Berkeley) 6/6/93
 .\" $FreeBSD$
 .\"
-.Dd February 23, 2008
+.Dd October 10, 2011
 .Dt KTRACE 1
 .Os
 .Sh NAME
@@ -113,6 +113,8 @@ trace
 .Tn I/O
 .It Cm n
 trace namei translations
+.It Cm p
+trace capability check failures
 .It Cm s
 trace signal processing
 .It Cm t
@@ -127,7 +129,7 @@ trace
 requests
 .It Cm +
 trace the default set of trace points -
-.Cm c , i , n , s , t , u , y
+.Cm c , i , n , p , s , t , u , y
 .El
 .It Ar command
 Execute
diff --git a/usr.bin/ktrace/ktrace.c b/usr.bin/ktrace/ktrace.c
index 4490849c6655..f993d376bc14 100644
--- a/usr.bin/ktrace/ktrace.c
+++ b/usr.bin/ktrace/ktrace.c
@@ -43,14 +43,15 @@ static char sccsid[] = "@(#)ktrace.c	8.1 (Berkeley) 6/6/93";
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
-#include <sys/stat.h>
 #include <sys/file.h>
+#include <sys/stat.h>
 #include <sys/time.h>
-#include <sys/errno.h>
 #include <sys/uio.h>
 #include <sys/ktrace.h>
 
 #include <err.h>
+#include <errno.h>
+#include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -59,21 +60,22 @@ __FBSDID("$FreeBSD$");
 
 static char def_tracefile[] = DEF_TRACEFILE;
 
+static enum clear { NOTSET, CLEAR, CLEARALL } clear = NOTSET;
+static int pid;
+
 static void no_ktrace(int);
-static int rpid(char *);
+static void set_pid_clear(const char *, enum clear);
 static void usage(void);
 
 int
 main(int argc, char *argv[])
 {
-	enum { NOTSET, CLEAR, CLEARALL } clear;
-	int append, ch, fd, inherit, ops, pid, pidset, trpoints;
+	int append, ch, fd, inherit, ops, trpoints;
 	const char *tracefile;
 	mode_t omask;
 	struct stat sb;
 
-	clear = NOTSET;
-	append = ops = pidset = inherit = 0;
+	append = ops = inherit = 0;
 	trpoints = DEF_POINTS;
 	tracefile = def_tracefile;
 	while ((ch = getopt(argc,argv,"aCcdf:g:ip:t:")) != -1)
@@ -82,11 +84,10 @@ main(int argc, char *argv[])
 			append = 1;
 			break;
 		case 'C':
-			clear = CLEARALL;
-			pidset = 1;
+			set_pid_clear("1", CLEARALL);
 			break;
 		case 'c':
-			clear = CLEAR;
+			set_pid_clear(NULL, CLEAR);
 			break;
 		case 'd':
 			ops |= KTRFLAG_DESCEND;
@@ -95,15 +96,14 @@ main(int argc, char *argv[])
 			tracefile = optarg;
 			break;
 		case 'g':
-			pid = -rpid(optarg);
-			pidset = 1;
+			set_pid_clear(optarg, NOTSET);
+			pid = -pid;
 			break;
 		case 'i':
 			inherit = 1;
 			break;
 		case 'p':
-			pid = rpid(optarg);
-			pidset = 1;
+			set_pid_clear(optarg, NOTSET);
 			break;
 		case 't':
 			trpoints = getpoints(optarg);
@@ -115,12 +115,19 @@ main(int argc, char *argv[])
 		default:
 			usage();
 		}
+
 	argv += optind;
 	argc -= optind;
-	
-	if ((pidset && *argv) || (!pidset && clear == NOTSET && !*argv))
+
+	/* must have either -[Cc], a pid or a command */
+	if (clear == NOTSET && pid == 0 && argc == 0)
 		usage();
-			
+	/* can't have both a pid and a command */
+	/* (note that -C sets pid to 1) */
+	if (pid != 0 && argc > 0) {
+		usage();
+	}
+
 	if (inherit)
 		trpoints |= KTRFAC_INHERIT;
 
@@ -129,10 +136,9 @@ main(int argc, char *argv[])
 		if (clear == CLEARALL) {
 			ops = KTROP_CLEAR | KTRFLAG_DESCEND;
 			trpoints = ALL_POINTS;
-			pid = 1;
-		} else
-			ops |= pidset ? KTROP_CLEAR : KTROP_CLEARFILE;
-
+		} else {
+			ops |= pid ? KTROP_CLEAR : KTROP_CLEARFILE;
+		}
 		if (ktrace(tracefile, ops, trpoints, pid) < 0)
 			err(1, "%s", tracefile);
 		exit(0);
@@ -160,46 +166,75 @@ main(int argc, char *argv[])
 
 	trpoints |= PROC_ABI_POINTS;
 
-	if (*argv) { 
+	if (argc > 0) { 
 		if (ktrace(tracefile, ops, trpoints, getpid()) < 0)
 			err(1, "%s", tracefile);
-		execvp(argv[0], &argv[0]);
-		err(1, "exec of '%s' failed", argv[0]);
+		execvp(*argv, argv);
+		err(1, "exec of '%s' failed", *argv);
 	}
-	else if (ktrace(tracefile, ops, trpoints, pid) < 0)
+	if (ktrace(tracefile, ops, trpoints, pid) < 0)
 		err(1, "%s", tracefile);
 	exit(0);
 }
 
-static int
-rpid(char *p)
+static void
+set_pid_clear(const char *p, enum clear cl)
 {
-	static int first;
+	intmax_t n;
+	char *e;
 
-	if (first++) {
+	if (clear != NOTSET && cl != NOTSET) {
+		/* either -c and -C or either of them twice */
+		warnx("only one -c or -C flag is permitted");
+		usage();
+	}
+	if ((clear == CLEARALL && p != NULL) || (cl == CLEARALL && pid != 0)) {
+		/* both -C and a pid or pgid */
+		warnx("the -C flag may not be combined with -g or -p");
+		usage();
+	}
+	if (p != NULL && pid != 0) {
+		/* either -p and -g or either of them twice */
 		warnx("only one -g or -p flag is permitted");
 		usage();
 	}
-	if (!*p) {
-		warnx("illegal process id");
-		usage();
+	if (p != NULL) {
+		errno = 0;
+		n = strtoimax(p, &e, 10);
+		/*
+		 * 1) not a number, or outside the range of an intmax_t
+		 * 2) inside the range of intmax_t but outside the range
+		 *    of an int, keeping in mind that the pid may be
+		 *    negated if it's actually a pgid.
+		 */
+		if (*e != '\0' || n < 1 || errno == ERANGE ||
+		    n > (intmax_t)INT_MAX || n > -(intmax_t)INT_MIN) {
+			warnx("invalid process or group id");
+			usage();
+		}
+		pid = n;
 	}
-	return(atoi(p));
+	if (cl != NOTSET)
+		if ((clear = cl) == CLEARALL)
+			pid = 1;
 }
 
 static void
 usage(void)
 {
-	(void)fprintf(stderr, "%s\n%s\n",
-"usage: ktrace [-aCcdi] [-f trfile] [-g pgrp | -p pid] [-t trstr]",
-"       ktrace [-adi] [-f trfile] [-t trstr] command");
+
+	fprintf(stderr, "%s\n%s\n",
+	    "usage: ktrace [-aCcdi] [-f trfile] [-g pgrp | -p pid] [-t trstr]",
+	    "       ktrace [-adi] [-f trfile] [-t trstr] command");
 	exit(1);
 }
 
 static void
 no_ktrace(int sig __unused)
 {
-        (void)fprintf(stderr,
-"error:\tktrace() system call not supported in the running kernel\n\tre-compile kernel with 'options KTRACE'\n");
+
+	fprintf(stderr, "error:\t%s\n\t%s\n",
+	    "ktrace() system call not supported in the running kernel",
+	    "re-compile kernel with 'options KTRACE'");
         exit(1);
 }
diff --git a/usr.bin/ktrace/ktrace.h b/usr.bin/ktrace/ktrace.h
index 5e38d68e0dd7..e4e4dbff8c65 100644
--- a/usr.bin/ktrace/ktrace.h
+++ b/usr.bin/ktrace/ktrace.h
@@ -32,7 +32,7 @@
 
 #define DEF_POINTS (KTRFAC_SYSCALL | KTRFAC_SYSRET | KTRFAC_NAMEI | \
 		    KTRFAC_GENIO | KTRFAC_PSIG | KTRFAC_USER | \
-		    KTRFAC_STRUCT | KTRFAC_SYSCTL)
+		    KTRFAC_STRUCT | KTRFAC_SYSCTL | KTRFAC_CAPFAIL)
 
 #define PROC_ABI_POINTS (KTRFAC_PROCCTOR | KTRFAC_PROCDTOR)
 
diff --git a/usr.bin/ktrace/subr.c b/usr.bin/ktrace/subr.c
index 30b6b4ddffbe..5051de848c29 100644
--- a/usr.bin/ktrace/subr.c
+++ b/usr.bin/ktrace/subr.c
@@ -61,11 +61,14 @@ getpoints(char *s)
 		case 'c':
 			facs |= KTRFAC_SYSCALL | KTRFAC_SYSRET;
 			break;
+		case 'i':
+			facs |= KTRFAC_GENIO;
+			break;
 		case 'n':
 			facs |= KTRFAC_NAMEI;
 			break;
-		case 'i':
-			facs |= KTRFAC_GENIO;
+		case 'p':
+			facs |= KTRFAC_CAPFAIL;
 			break;
 		case 's':
 			facs |= KTRFAC_PSIG;
diff --git a/usr.bin/last/Makefile b/usr.bin/last/Makefile
index 0bd2f05eb27e..fd63cd989b2e 100644
--- a/usr.bin/last/Makefile
+++ b/usr.bin/last/Makefile
@@ -3,6 +3,6 @@
 
 PROG=	last
 
-WARNS?=	1
+NO_WFORMAT=
 
 .include <bsd.prog.mk>
diff --git a/usr.bin/leave/Makefile b/usr.bin/leave/Makefile
index 548479768a06..f71acbed25d4 100644
--- a/usr.bin/leave/Makefile
+++ b/usr.bin/leave/Makefile
@@ -3,6 +3,6 @@
 
 PROG=	leave
 
-WARNS?=	1
+NO_WFORMAT=
 
 .include <bsd.prog.mk>
diff --git a/usr.bin/look/Makefile b/usr.bin/look/Makefile
index a62003922094..dd724a39eec1 100644
--- a/usr.bin/look/Makefile
+++ b/usr.bin/look/Makefile
@@ -3,6 +3,4 @@
 
 PROG=	look
 
-WARNS?=	2
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/look/look.c b/usr.bin/look/look.c
index fb054a406eb6..4e8cf34797c8 100644
--- a/usr.bin/look/look.c
+++ b/usr.bin/look/look.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <fcntl.h>
 #include <limits.h>
 #include <locale.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -134,7 +135,7 @@ main(int argc, char *argv[])
 	do {
 		if ((fd = open(file, O_RDONLY, 0)) < 0 || fstat(fd, &sb))
 			err(2, "%s", file);
-		if (sb.st_size > SIZE_T_MAX)
+		if ((uintmax_t)sb.st_size > (uintmax_t)SIZE_T_MAX)
 			errx(2, "%s: %s", file, strerror(EFBIG));
 		if (sb.st_size == 0) {
 			close(fd);
diff --git a/usr.bin/lzmainfo/Makefile b/usr.bin/lzmainfo/Makefile
index 2b8396b71add..b92b46ccd8c9 100644
--- a/usr.bin/lzmainfo/Makefile
+++ b/usr.bin/lzmainfo/Makefile
@@ -12,8 +12,6 @@ SRCS+=	lzmainfo.c
 SRCS+=	tuklib_progname.c \
 	tuklib_exit.c
 
-WARNS?=	3
-
 CFLAGS+=	-DHAVE_CONFIG_H \
 		-I${LZMALIBDIR} \
 		-I${XZDIR}/common
diff --git a/usr.bin/m4/Makefile b/usr.bin/m4/Makefile
index feceb0ca80d1..702b3f3d2444 100644
--- a/usr.bin/m4/Makefile
+++ b/usr.bin/m4/Makefile
@@ -9,6 +9,4 @@ CFLAGS+=-DEXTENDED
 
 SRCS=	eval.c expr.c look.c main.c misc.c gnum4.c trace.c
 
-WARNS?=	0
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/m4/expr.c b/usr.bin/m4/expr.c
index be0f9f9e3256..8f4dd9c9e8cf 100644
--- a/usr.bin/m4/expr.c
+++ b/usr.bin/m4/expr.c
@@ -71,8 +71,8 @@ __FBSDID("$FreeBSD$");
  *      nerel   :       shift { ("<" | ">" | "<=" | ">=") shift }
  *      shift   :       primary { ("<<" | ">>") primary }
  *      primary :       term { ("+" | "-") term }
- *      term    :       exp { ("*" | "/" | "%") exp }
- *      exp     :       unary { "**" unary }
+ *      term    :       exponent { ("*" | "/" | "%") exponent }
+ *      exponent:       unary { "**" unary }
  *      unary   :       factor
  *              |       ("+" | "-" | "~" | "!") unary
  *      factor  :       constant
@@ -115,12 +115,11 @@ static int nerel(int mayeval);
 static int shift(int mayeval);
 static int primary(int mayeval);
 static int term(int mayeval);
-static int exp(int mayeval);
+static int exponent(int mayeval);
 static int unary(int mayeval);
 static int factor(int mayeval);
 static int constant(int mayeval);
 static int num(int mayeval);
-static int geteqrel(int mayeval);
 static int skipws(void);
 static void experr(const char *);
 
@@ -388,16 +387,16 @@ primary(int mayeval)
 }
 
 /*
- * term : exp { ("*" | "/" | "%") exp }
+ * term : exponent { ("*" | "/" | "%") exponent }
  */
 static int
 term(int mayeval)
 {
 	int c, vl, vr;
 
-	vl = exp(mayeval);
+	vl = exponent(mayeval);
 	while ((c = skipws()) == '*' || c == '/' || c == '%') {
-		vr = exp(mayeval);
+		vr = exponent(mayeval);
 
 		switch (c) {
 		case '*':
@@ -426,10 +425,10 @@ term(int mayeval)
 }
 
 /*
- * exp : unary { "**" exp }
+ * exponent : unary { "**" exponent }
  */
 static int
-exp(int mayeval)
+exponent(int mayeval)
 {
 	int c, vl, vr, n;
 
@@ -562,7 +561,7 @@ constant(int mayeval)
  * num : digit | num digit
  */
 static int
-num(int mayeval)
+num(int mayeval __unused)
 {
 	int rval, c, base;
 	int ndig;
diff --git a/usr.bin/m4/main.c b/usr.bin/m4/main.c
index 99f52335ff65..0e60fce3ec99 100644
--- a/usr.bin/m4/main.c
+++ b/usr.bin/m4/main.c
@@ -34,9 +34,11 @@
  */
 
 #ifndef lint
+#if 0
 static char copyright[] =
 "@(#) Copyright (c) 1989, 1993\n\
 	The Regents of the University of California.  All rights reserved.\n";
+#endif
 #endif /* not lint */
 
 #ifndef lint
diff --git a/usr.bin/ncplist/Makefile b/usr.bin/ncplist/Makefile
index b5baed47d7b2..8e6d18991a4f 100644
--- a/usr.bin/ncplist/Makefile
+++ b/usr.bin/ncplist/Makefile
@@ -2,8 +2,6 @@
 
 PROG=	ncplist
 
-WARNS?=	0
-
 DPADD=	${LIBNCP} ${LIBIPX}
 LDADD=	-lncp -lipx
 
diff --git a/usr.bin/ncplist/ncplist.c b/usr.bin/ncplist/ncplist.c
index e2c2ce4133c0..947871883f14 100644
--- a/usr.bin/ncplist/ncplist.c
+++ b/usr.bin/ncplist/ncplist.c
@@ -37,6 +37,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/time.h>
 
+#include <grp.h>
+#include <pwd.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -153,7 +155,7 @@ show_serverlist(char *server)
 {
 	int found = 0, connid;
 	struct ncp_bindery_object obj;
-	char *pattern = "*";
+	const char *pattern = "*";
 
 	connid = ncp_get_connid(server, 1);
 	if (connid < 0)
@@ -229,7 +231,7 @@ show_userlist(char *server)
 	return;
 }
 
-void
+static void
 show_queuelist(char *server, char *patt)
 {
 	struct ncp_bindery_object q;
@@ -298,8 +300,8 @@ list_volumes(char *server)
 }
 
 struct ncp_bind_type {
-	u_long	type;
-	char	*name;
+	u_long		type;
+	const char	*name;
 };
 
 static struct ncp_bind_type btypes[] = {
@@ -310,7 +312,7 @@ static struct ncp_bind_type btypes[] = {
 	{0, NULL}
 };
 
-void
+static void
 list_bindery(char *server, char *type, char *patt)
 {
 	struct ncp_bindery_object q;
diff --git a/usr.bin/newgrp/newgrp.c b/usr.bin/newgrp/newgrp.c
index 91b62a52779d..5381b15555fb 100644
--- a/usr.bin/newgrp/newgrp.c
+++ b/usr.bin/newgrp/newgrp.c
@@ -140,7 +140,7 @@ restoregrps(void)
 	if (initres < 0)
 		warn("initgroups");
 	if (setres < 0)
-		warn("setgroups");
+		warn("setgid");
 }
 
 static void
diff --git a/usr.bin/nl/Makefile b/usr.bin/nl/Makefile
index 07bef878c44e..9b1f292dfbee 100644
--- a/usr.bin/nl/Makefile
+++ b/usr.bin/nl/Makefile
@@ -2,6 +2,4 @@
 
 PROG=	nl
 
-WARNS?=	2
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/nl/nl.c b/usr.bin/nl/nl.c
index b3b78b1688d6..9c2ae64ac913 100644
--- a/usr.bin/nl/nl.c
+++ b/usr.bin/nl/nl.c
@@ -73,9 +73,9 @@ struct numbering_property {
 #define NP_LAST		HEADER
 
 static struct numbering_property numbering_properties[NP_LAST + 1] = {
-	{ "footer",	number_none	},
-	{ "body",	number_nonempty	},
-	{ "header",	number_none	}
+	{ .name = "footer", .type = number_none },
+	{ .name = "body", .type = number_nonempty },
+	{ .name = "header", .type = number_none }
 };
 
 #define max(a, b)	((a) > (b) ? (a) : (b))
@@ -128,9 +128,7 @@ static int width = 6;
 
 
 int
-main(argc, argv)
-	int argc;
-	char *argv[];
+main(int argc, char *argv[])
 {
 	int c;
 	long val;
@@ -258,7 +256,7 @@ main(argc, argv)
 	delimlen = delim1len + delim2len;
 
 	/* Allocate a buffer suitable for preformatting line number. */
-	intbuffersize = max(INT_STRLEN_MAXIMUM, width) + 1;	/* NUL */
+	intbuffersize = max((int)INT_STRLEN_MAXIMUM, width) + 1; /* NUL */
 	if ((intbuffer = malloc(intbuffersize)) == NULL)
 		err(EXIT_FAILURE, "cannot allocate preformatting buffer");
 
@@ -270,7 +268,7 @@ main(argc, argv)
 }
 
 static void
-filter()
+filter(void)
 {
 	char *buffer;
 	size_t buffersize;
@@ -359,9 +357,7 @@ filter()
  */
 
 static void
-parse_numbering(argstr, section)
-	const char *argstr;
-	int section;
+parse_numbering(const char *argstr, int section)
 {
 	int error;
 	char errorbuf[NL_TEXTMAX];
@@ -403,7 +399,7 @@ parse_numbering(argstr, section)
 }
 
 static void
-usage()
+usage(void)
 {
 
 	(void)fprintf(stderr,
diff --git a/usr.bin/ruptime/Makefile b/usr.bin/ruptime/Makefile
index 351d0d8a181d..f7fba3adde5f 100644
--- a/usr.bin/ruptime/Makefile
+++ b/usr.bin/ruptime/Makefile
@@ -3,6 +3,4 @@
 
 PROG=	ruptime
 
-WARNS?=	3
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/ruptime/ruptime.c b/usr.bin/ruptime/ruptime.c
index 0bdaa04195dd..d156b6d84ff0 100644
--- a/usr.bin/ruptime/ruptime.c
+++ b/usr.bin/ruptime/ruptime.c
@@ -55,13 +55,12 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 
 struct hs {
-	struct	whod *hs_wd;
+	struct	whod hs_wd;
 	int	hs_nusers;
 } *hs;
-struct	whod awhod;
-#define LEFTEARTH(h)		(now - (h) > 4*24*60*60)
-#define	ISDOWN(h)		(now - (h)->hs_wd->wd_recvtime > 11 * 60)
-#define	WHDRSIZE	(sizeof (awhod) - sizeof (awhod.wd_we))
+#define	LEFTEARTH(h)	(now - (h) > 4*24*60*60)
+#define	ISDOWN(h)	(now - (h)->hs_wd.wd_recvtime > 11 * 60)
+#define	WHDRSIZE	__offsetof(struct whod, wd_we)
 
 size_t nhosts;
 time_t now;
@@ -127,10 +126,10 @@ interval(time_t tval, const char *updown)
 	int days, hours, minutes;
 
 	if (tval < 0) {
-		(void)snprintf(resbuf, sizeof(resbuf), "   %s ??:??", updown);
+		(void)snprintf(resbuf, sizeof(resbuf), "%s      ??:??", updown);
 		return (resbuf);
 	}
-						/* round to minutes. */
+	/* Round to minutes. */
 	minutes = (tval + (60 - 1)) / 60;
 	hours = minutes / 60;
 	minutes %= 60;
@@ -138,10 +137,10 @@ interval(time_t tval, const char *updown)
 	hours %= 24;
 	if (days)
 		(void)snprintf(resbuf, sizeof(resbuf),
-		    "%s %3d+%02d:%02d", updown, days, hours, minutes);
+		    "%s %4d+%02d:%02d", updown, days, hours, minutes);
 	else
 		(void)snprintf(resbuf, sizeof(resbuf),
-		    "%s     %2d:%02d", updown, hours, minutes);
+		    "%s      %2d:%02d", updown, hours, minutes);
 	return (resbuf);
 }
 
@@ -152,7 +151,7 @@ int
 hscmp(const void *a1, const void *a2)
 {
 	return (rflg *
-	    strcmp(HS(a1)->hs_wd->wd_hostname, HS(a2)->hs_wd->wd_hostname));
+	    strcmp(HS(a1)->hs_wd.wd_hostname, HS(a2)->hs_wd.wd_hostname));
 }
 
 /* Load average comparison. */
@@ -168,7 +167,7 @@ lcmp(const void *a1, const void *a2)
 		return (-rflg);
 	else
 		return (rflg *
-		   (HS(a2)->hs_wd->wd_loadav[0] - HS(a1)->hs_wd->wd_loadav[0]));
+		   (HS(a2)->hs_wd.wd_loadav[0] - HS(a1)->hs_wd.wd_loadav[0]));
 }
 
 void
@@ -179,10 +178,9 @@ ruptime(const char *host, int aflg, int (*cmp)(const void *, const void *))
 	struct whoent *we;
 	struct dirent *dp;
 	const char *hostname;
-	char buf[sizeof(struct whod)];
 	int fd, i, maxloadav;
 	size_t hspace;
-	u_int cc;
+	ssize_t cc;
 
 	rewinddir(dirp);
 	hsp = NULL;
@@ -195,18 +193,7 @@ ruptime(const char *host, int aflg, int (*cmp)(const void *, const void *))
 			warn("%s", dp->d_name);
 			continue;
 		}
-		cc = read(fd, buf, sizeof(struct whod));
-		(void)close(fd);
-		if (host != NULL) {
-			hostname = ((struct whod *)buf)->wd_hostname;
-			if (strcasecmp(hostname, host) != 0)
-				continue;
-		}
 
-		if (cc < WHDRSIZE)
-			continue;
-		if (LEFTEARTH(((struct whod *)buf)->wd_recvtime))
-			continue;
 		if (nhosts == hspace) {
 			if ((hs =
 			    realloc(hs, (hspace += 40) * sizeof(*hs))) == NULL)
@@ -214,16 +201,26 @@ ruptime(const char *host, int aflg, int (*cmp)(const void *, const void *))
 			hsp = hs + nhosts;
 		}
 
-		if ((hsp->hs_wd = malloc((size_t)WHDRSIZE)) == NULL)
-			err(1, NULL);
-		memmove(hsp->hs_wd, buf, (size_t)WHDRSIZE);
+		wd = &hsp->hs_wd;
+		cc = read(fd, wd, sizeof(*wd));
+		(void)close(fd);
+		if (cc < (ssize_t)WHDRSIZE)
+			continue;
 
-		for (wd = (struct whod *)buf, i = 0; i < 2; ++i)
+		if (host != NULL) {
+			hostname = wd->wd_hostname;
+			if (strcasecmp(hostname, host) != 0)
+				continue;
+		}
+		if (LEFTEARTH(wd->wd_recvtime))
+			continue;
+
+		for (i = 0; i < 2; i++)
 			if (wd->wd_loadav[i] > maxloadav)
 				maxloadav = wd->wd_loadav[i];
 
-		for (hsp->hs_nusers = 0,
-		    we = (struct whoent *)(buf + cc); --we >= wd->wd_we;)
+		for (hsp->hs_nusers = 0, we = &wd->wd_we[0];
+		    (char *)(we + 1) <= (char *)wd + cc; we++)
 			if (aflg || we->we_idle < 3600)
 				++hsp->hs_nusers;
 		++hsp;
@@ -239,25 +236,25 @@ ruptime(const char *host, int aflg, int (*cmp)(const void *, const void *))
 	qsort(hs, nhosts, sizeof(hs[0]), cmp);
 	for (i = 0; i < (int)nhosts; i++) {
 		hsp = &hs[i];
+		wd = &hsp->hs_wd;
 		if (ISDOWN(hsp)) {
-			(void)printf("%-25.25s%s\n", hsp->hs_wd->wd_hostname,
-			    interval(now - hsp->hs_wd->wd_recvtime, "down"));
+			(void)printf("%-25.25s%s\n", wd->wd_hostname,
+			    interval(now - hsp->hs_wd.wd_recvtime, "down"));
 			continue;
 		}
 		(void)printf(
 		    "%-25.25s%s,  %4d user%s  load %*.2f, %*.2f, %*.2f\n",
-		    hsp->hs_wd->wd_hostname,
-		    interval((time_t)hsp->hs_wd->wd_sendtime -
-			(time_t)hsp->hs_wd->wd_boottime, "  up"),
+		    wd->wd_hostname,
+		    interval((time_t)wd->wd_sendtime -
+		        (time_t)wd->wd_boottime, "  up"),
 		    hsp->hs_nusers,
 		    hsp->hs_nusers == 1 ? ", " : "s,",
 		    maxloadav >= 1000 ? 5 : 4,
-			hsp->hs_wd->wd_loadav[0] / 100.0,
+		        wd->wd_loadav[0] / 100.0,
 		    maxloadav >= 1000 ? 5 : 4,
-		        hsp->hs_wd->wd_loadav[1] / 100.0,
+		        wd->wd_loadav[1] / 100.0,
 		    maxloadav >= 1000 ? 5 : 4,
-		        hsp->hs_wd->wd_loadav[2] / 100.0);
-		free(hsp->hs_wd);
+		        wd->wd_loadav[2] / 100.0);
 	}
 	free(hs);
 	hs = NULL;
@@ -283,11 +280,11 @@ int
 tcmp(const void *a1, const void *a2)
 {
 	return (rflg * (
-		(ISDOWN(HS(a2)) ? HS(a2)->hs_wd->wd_recvtime - now
-		    : HS(a2)->hs_wd->wd_sendtime - HS(a2)->hs_wd->wd_boottime)
+		(ISDOWN(HS(a2)) ? HS(a2)->hs_wd.wd_recvtime - now
+		    : HS(a2)->hs_wd.wd_sendtime - HS(a2)->hs_wd.wd_boottime)
 		-
-		(ISDOWN(HS(a1)) ? HS(a1)->hs_wd->wd_recvtime - now
-		    : HS(a1)->hs_wd->wd_sendtime - HS(a1)->hs_wd->wd_boottime)
+		(ISDOWN(HS(a1)) ? HS(a1)->hs_wd.wd_recvtime - now
+		    : HS(a1)->hs_wd.wd_sendtime - HS(a1)->hs_wd.wd_boottime)
 	));
 }
 
diff --git a/usr.bin/rwho/Makefile b/usr.bin/rwho/Makefile
index aee9ff765f73..3fdc05d3b8f7 100644
--- a/usr.bin/rwho/Makefile
+++ b/usr.bin/rwho/Makefile
@@ -3,6 +3,4 @@
 
 PROG=	rwho
 
-WARNS?=	1
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/rwho/rwho.c b/usr.bin/rwho/rwho.c
index c0106c7c2163..f134c1864f0a 100644
--- a/usr.bin/rwho/rwho.c
+++ b/usr.bin/rwho/rwho.c
@@ -165,9 +165,10 @@ main(int argc, char *argv[])
 			 d_first ? "%e %b %R" : "%b %e %R",
 			 localtime(&t));
 		(void)sprintf(buf, "%s:%-.*s", mp->myhost,
-		   sizeof(mp->myutmp.out_line), mp->myutmp.out_line);
+		   (int)sizeof(mp->myutmp.out_line), mp->myutmp.out_line);
 		printf("%-*.*s %-*s %s",
-		   sizeof(mp->myutmp.out_name), sizeof(mp->myutmp.out_name),
+		   (int)sizeof(mp->myutmp.out_name),
+		   (int)sizeof(mp->myutmp.out_name),
 		   mp->myutmp.out_name,
 		   width,
 		   buf,
diff --git a/usr.bin/script/script.1 b/usr.bin/script/script.1
index 3aa03e55e279..21bc2fd19dad 100644
--- a/usr.bin/script/script.1
+++ b/usr.bin/script/script.1
@@ -28,7 +28,7 @@
 .\"	@(#)script.1	8.1 (Berkeley) 6/6/93
 .\" $FreeBSD$
 .\"
-.Dd January 22, 2004
+.Dd September 28, 2011
 .Dt SCRIPT 1
 .Os
 .Sh NAME
@@ -73,15 +73,16 @@ or
 .Pa typescript ,
 retaining the prior contents.
 .It Fl k
-Log keys sent to program as well as output.
+Log keys sent to the program as well as output.
 .It Fl q
 Run in quiet mode, omit the start and stop status messages.
 .It Fl t Ar time
-Specify time interval between flushing script output file.
+Specify the interval at which the script output file will be flushed
+to disk, in seconds.
 A value of 0
 causes
 .Nm
-to flush for every character I/O event.
+to flush after every character I/O event.
 The default interval is
 30 seconds.
 .El
@@ -136,13 +137,15 @@ If
 .Ev SHELL
 is not set, the Bourne shell
 is assumed.
-(Most shells set this variable automatically).
+.Pq Most shells set this variable automatically .
 .El
 .Sh SEE ALSO
 .Xr csh 1
-(for the
+.Po
+for the
 .Em history
-mechanism).
+mechanism
+.Pc .
 .Sh HISTORY
 The
 .Nm
@@ -165,4 +168,15 @@ mode, echo cancelling is far from ideal.
 The slave terminal mode is checked
 for ECHO mode to check when to avoid manual echo logging.
 This does not
-work when in a raw mode where the program being run is doing manual echo.
+work when the terminal is in a raw mode where
+the program being run is doing manual echo.
+.Pp
+If
+.Nm
+reads zero bytes from the terminal, it switches to a mode when it
+only attempts to read
+once a second until there is data to read.
+This prevents
+.Nm
+from spinning on zero-byte reads, but might cause a 1-second delay in
+processing of user input.
diff --git a/usr.bin/script/script.c b/usr.bin/script/script.c
index ad9972170bc1..cd577f7f466c 100644
--- a/usr.bin/script/script.c
+++ b/usr.bin/script/script.c
@@ -86,6 +86,7 @@ main(int argc, char *argv[])
 	char ibuf[BUFSIZ];
 	fd_set rfd;
 	int flushtime = 30;
+	int readstdin;
 
 	aflg = kflg = 0;
 	while ((ch = getopt(argc, argv, "aqkt:")) != -1)
@@ -155,19 +156,24 @@ main(int argc, char *argv[])
 		doshell(argv);
 	close(slave);
 
-	if (flushtime > 0)
-		tvp = &tv;
-	else
-		tvp = NULL;
-
-	start = time(0);
-	FD_ZERO(&rfd);
+	start = tvec = time(0);
+	readstdin = 1;
 	for (;;) {
+		FD_ZERO(&rfd);
 		FD_SET(master, &rfd);
-		FD_SET(STDIN_FILENO, &rfd);
-		if (flushtime > 0) {
-			tv.tv_sec = flushtime;
+		if (readstdin)
+			FD_SET(STDIN_FILENO, &rfd);
+		if (!readstdin && ttyflg) {
+			tv.tv_sec = 1;
 			tv.tv_usec = 0;
+			tvp = &tv;
+			readstdin = 1;
+		} else if (flushtime > 0) {
+			tv.tv_sec = flushtime - (tvec - start);
+			tv.tv_usec = 0;
+			tvp = &tv;
+		} else {
+			tvp = NULL;
 		}
 		n = select(master + 1, &rfd, 0, 0, tvp);
 		if (n < 0 && errno != EINTR)
@@ -176,8 +182,13 @@ main(int argc, char *argv[])
 			cc = read(STDIN_FILENO, ibuf, BUFSIZ);
 			if (cc < 0)
 				break;
-			if (cc == 0)
-				(void)write(master, ibuf, 0);
+			if (cc == 0) {
+				if (tcgetattr(master, &stt) == 0 &&
+				    (stt.c_lflag & ICANON) != 0) {
+					(void)write(master, &stt.c_cc[VEOF], 1);
+				}
+				readstdin = 0;
+			}
 			if (cc > 0) {
 				(void)write(master, ibuf, cc);
 				if (kflg && tcgetattr(master, &stt) >= 0 &&
diff --git a/usr.bin/systat/cmds.c b/usr.bin/systat/cmds.c
index 6dc5300b131b..6ad7443c0252 100644
--- a/usr.bin/systat/cmds.c
+++ b/usr.bin/systat/cmds.c
@@ -47,32 +47,32 @@ static const char sccsid[] = "@(#)cmds.c	8.2 (Berkeley) 4/29/95";
 void
 command(const char *cmd)
 {
-        struct cmdtab *p;
-        char *cp, *tmpstr, *tmpstr1;
+	struct cmdtab *p;
+	char *cp, *tmpstr, *tmpstr1;
 	int interval, omask;
 
 	tmpstr = tmpstr1 = strdup(cmd);
 	omask = sigblock(sigmask(SIGALRM));
-        for (cp = tmpstr1; *cp && !isspace(*cp); cp++)
-                ;
-        if (*cp)
-                *cp++ = '\0';
+	for (cp = tmpstr1; *cp && !isspace(*cp); cp++)
+		;
+	if (*cp)
+		*cp++ = '\0';
 	if (*tmpstr1 == '\0')
 		return;
 	for (; *cp && isspace(*cp); cp++)
 		;
-        if (strcmp(tmpstr1, "quit") == 0 || strcmp(tmpstr1, "q") == 0)
-                die(0);
+	if (strcmp(tmpstr1, "quit") == 0 || strcmp(tmpstr1, "q") == 0)
+		die(0);
 	if (strcmp(tmpstr1, "load") == 0) {
 		load();
 		goto done;
 	}
-        if (strcmp(tmpstr1, "stop") == 0) {
-                alarm(0);
-                mvaddstr(CMDLINE, 0, "Refresh disabled.");
-                clrtoeol();
+	if (strcmp(tmpstr1, "stop") == 0) {
+		alarm(0);
+		mvaddstr(CMDLINE, 0, "Refresh disabled.");
+		clrtoeol();
 		goto done;
-        }
+	}
 	if (strcmp(tmpstr1, "help") == 0) {
 		int _col, _len;
 
@@ -89,30 +89,30 @@ command(const char *cmd)
 		goto done;
 	}
 	interval = atoi(tmpstr1);
-        if (interval <= 0 &&
+	if (interval <= 0 &&
 	    (strcmp(tmpstr1, "start") == 0 || strcmp(tmpstr1, "interval") == 0)) {
 		interval = *cp ? atoi(cp) : naptime;
-                if (interval <= 0) {
+		if (interval <= 0) {
 			error("%d: bad interval.", interval);
 			goto done;
-                }
+		}
 	}
 	if (interval > 0) {
-                alarm(0);
-                naptime = interval;
-                display(0);
-                status();
+		alarm(0);
+		naptime = interval;
+		display(0);
+		status();
 		goto done;
-        }
+	}
 	p = lookup(tmpstr1);
 	if (p == (struct cmdtab *)-1) {
 		error("%s: Ambiguous command.", tmpstr1);
 		goto done;
 	}
-        if (p) {
-                if (curcmd == p)
+	if (p) {
+		if (curcmd == p)
 			goto done;
-                alarm(0);
+		alarm(0);
 		(*curcmd->c_close)(wnd);
 		curcmd->c_flags &= ~CF_INIT;
 		wnd = (*p->c_open)();
@@ -131,12 +131,12 @@ command(const char *cmd)
 			else
 				goto done;
 		}
-                curcmd = p;
+		curcmd = p;
 		labels();
-                display(0);
-                status();
+		display(0);
+		status();
 		goto done;
-        }
+	}
 	if (curcmd->c_cmd == 0 || !(*curcmd->c_cmd)(tmpstr1, cp))
 		error("%s: Unknown command.", tmpstr1);
 done:
@@ -176,18 +176,18 @@ void
 status(void)
 {
 
-        error("Showing %s, refresh every %d seconds.",
-          curcmd->c_name, naptime);
+	error("Showing %s, refresh every %d seconds.",
+	  curcmd->c_name, naptime);
 }
 
 int
 prefix(const char *s1, const char *s2)
 {
 
-        while (*s1 == *s2) {
-                if (*s1 == '\0')
-                        return (1);
-                s1++, s2++;
-        }
-        return (*s1 == '\0');
+	while (*s1 == *s2) {
+		if (*s1 == '\0')
+			return (1);
+		s1++, s2++;
+	}
+	return (*s1 == '\0');
 }
diff --git a/usr.bin/systat/cmdtab.c b/usr.bin/systat/cmdtab.c
index 1584cc054804..9b648ffd3c98 100644
--- a/usr.bin/systat/cmdtab.c
+++ b/usr.bin/systat/cmdtab.c
@@ -40,22 +40,22 @@ static const char sccsid[] = "@(#)cmdtab.c	8.1 (Berkeley) 6/6/93";
 #include "mode.h"
 
 struct	cmdtab cmdtab[] = {
-        { "pigs",	showpigs,	fetchpigs,	labelpigs,
+	{ "pigs",	showpigs,	fetchpigs,	labelpigs,
 	  initpigs,	openpigs,	closepigs,	0,
 	  0,		CF_LOADAV },
-        { "swap",	showswap,	fetchswap,	labelswap,
+	{ "swap",	showswap,	fetchswap,	labelswap,
 	  initswap,	openswap,	closeswap,	0,
 	  0,		CF_LOADAV },
-        { "mbufs",	showmbufs,	fetchmbufs,	labelmbufs,
+	{ "mbufs",	showmbufs,	fetchmbufs,	labelmbufs,
 	  initmbufs,	openmbufs,	closembufs,	0,
 	  0,		CF_LOADAV },
-        { "iostat",	showiostat,	fetchiostat,	labeliostat,
+	{ "iostat",	showiostat,	fetchiostat,	labeliostat,
 	  initiostat,	openiostat,	closeiostat,	cmdiostat,
 	  0,		CF_LOADAV },
-        { "vmstat",	showkre,	fetchkre,	labelkre,
+	{ "vmstat",	showkre,	fetchkre,	labelkre,
 	  initkre,	openkre,	closekre,	cmdkre,
 	  0,		0 },
-        { "netstat",	shownetstat,	fetchnetstat,	labelnetstat,
+	{ "netstat",	shownetstat,	fetchnetstat,	labelnetstat,
 	  initnetstat,	opennetstat,	closenetstat,	cmdnetstat,
 	  0,		CF_LOADAV },
 	{ "icmp",	showicmp,	fetchicmp,	labelicmp,
@@ -78,6 +78,6 @@ struct	cmdtab cmdtab[] = {
 	{ "ifstat",	showifstat,	fetchifstat,	labelifstat,
 	  initifstat,	openifstat,	closeifstat,	cmdifstat,
 	  0,		CF_LOADAV },
-        { NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0 }
+	{ NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0 }
 };
 struct  cmdtab *curcmd = &cmdtab[0];
diff --git a/usr.bin/systat/devs.c b/usr.bin/systat/devs.c
index e6df07ca7ba8..09a18384c8cb 100644
--- a/usr.bin/systat/devs.c
+++ b/usr.bin/systat/devs.c
@@ -105,7 +105,7 @@ dsinit(int maxshowdevs, struct statinfo *s1, struct statinfo *s2 __unused,
 
 	/*
 	 * Make sure that the userland devstat version matches the kernel
-	 * devstat version.  If not, exit and print a message informing 
+	 * devstat version.  If not, exit and print a message informing
 	 * the user of his mistake.
 	 */
 	if (devstat_checkversion(NULL) < 0)
@@ -155,10 +155,10 @@ dscmd(const char *cmd, const char *args, int maxshowdevs, struct statinfo *s1)
 	if (prefix(cmd, "refresh")) {
 		retval = devstat_selectdevs(&dev_select, &num_selected,
 		    &num_selections, &select_generation, generation,
-		    s1->dinfo->devices, num_devices, 
+		    s1->dinfo->devices, num_devices,
 		    (last_type ==DS_MATCHTYPE_PATTERN) ?  matches : NULL,
 		    (last_type ==DS_MATCHTYPE_PATTERN) ?  num_matches : 0,
-		    (last_type == DS_MATCHTYPE_SPEC) ?specified_devices : NULL, 
+		    (last_type == DS_MATCHTYPE_SPEC) ?specified_devices : NULL,
 		    (last_type == DS_MATCHTYPE_SPEC) ?num_devices_specified : 0,
 		    (last_type == DS_MATCHTYPE_NONE) ?  DS_SELECT_ADD :
 		    DS_SELECT_ADDONLY, maxshowdevs, 0);
@@ -220,7 +220,7 @@ dsmatchselect(const char *args, devstat_select_mode select_mode, int maxshowdevs
 	}
 
 	for (i = 0; i < num_args; i++) {
-		if (devstat_buildmatch(tstr[i], &matches, &num_matches) != 0) {	
+		if (devstat_buildmatch(tstr[i], &matches, &num_matches) != 0) {
 			warnx("%s", devstat_errbuf);
 			return(0);
 		}
@@ -282,7 +282,7 @@ dsselect(const char *args, devstat_select_mode select_mode, int maxshowdevs,
 			asprintf(&buffer, "%s%d", dev_select[i].device_name,
 				dev_select[i].unit_number);
 			if (strcmp(buffer, tmpstr1) == 0) {
-				
+
 				num_devices_specified++;
 
 				specified_devices =(char **)realloc(
diff --git a/usr.bin/systat/fetch.c b/usr.bin/systat/fetch.c
index f9aafb090ee8..d159e51c1e79 100644
--- a/usr.bin/systat/fetch.c
+++ b/usr.bin/systat/fetch.c
@@ -62,17 +62,17 @@ void getsysctl(const char *name, void *ptr, size_t len)
 {
 	size_t nlen = len;
 	if (sysctlbyname(name, ptr, &nlen, NULL, 0) != 0) {
-		error("sysctl(%s...) failed: %s", name, 
+		error("sysctl(%s...) failed: %s", name,
 		    strerror(errno));
 	}
 	if (nlen != len) {
-		error("sysctl(%s...) expected %lu, got %lu", name, 
+		error("sysctl(%s...) expected %lu, got %lu", name,
 		    (unsigned long)len, (unsigned long)nlen);
     }
 }
 
 /*
- * Read sysctl data with variable size. Try some times (with increasing 
+ * Read sysctl data with variable size. Try some times (with increasing
  * buffers), fail if still too small.
  * This is needed sysctls with possibly raplidly increasing data sizes,
  * but imposes little overhead in the case of constant sizes.
@@ -84,8 +84,8 @@ void getsysctl(const char *name, void *ptr, size_t len)
 /* Some defines: Number of tries. */
 #define SD_NTRIES  10
 /* Percent of over-allocation (initial) */
-#define SD_MARGIN  10 
-/* 
+#define SD_MARGIN  10
+/*
  * Factor for over-allocation in percent (the margin is increased by this on
  * any failed try).
  */
diff --git a/usr.bin/systat/ifcmds.c b/usr.bin/systat/ifcmds.c
index 495ac46ac637..0c03e595bf34 100644
--- a/usr.bin/systat/ifcmds.c
+++ b/usr.bin/systat/ifcmds.c
@@ -47,7 +47,7 @@ ifcmd(const char *cmd, const char *args)
 			clrtoeol();
 			addstr("what scale? ");
 			addstr(get_helplist());
-		} 
+		}
 	}
 	return (1);
 }
diff --git a/usr.bin/systat/ifstat.c b/usr.bin/systat/ifstat.c
index 62773e9fd09c..77d4a3431d86 100644
--- a/usr.bin/systat/ifstat.c
+++ b/usr.bin/systat/ifstat.c
@@ -43,7 +43,7 @@
 #include "extern.h"
 #include "convtbl.h"
 
-                                /* Column numbers */
+				/* Column numbers */
 
 #define C1	0		/*  0-19 */
 #define C2	20		/* 20-39 */
@@ -121,9 +121,9 @@ static	 u_int getifnum(void);
 } while (0)
 
 #define DOPUTTOTAL(c, r, d)	do {					\
- 	CLEAR_COLUMN((r), (c));						\
- 	mvprintw((r), (c), "%12.3f %s  ",				\
- 		 convert(d##_##c, SC_AUTO),				\
+	CLEAR_COLUMN((r), (c));						\
+	mvprintw((r), (c), "%12.3f %s  ",				\
+		 convert(d##_##c, SC_AUTO),				\
 		 get_string(d##_##c, SC_AUTO));				\
 } while (0)
 
@@ -255,8 +255,8 @@ fetchifstat(void)
 		(void)getifmibdata(ifp->if_row, &ifp->if_mib);
 
 
-                new_inb = ifp->if_mib.ifmd_data.ifi_ibytes;
-                new_outb = ifp->if_mib.ifmd_data.ifi_obytes;
+		new_inb = ifp->if_mib.ifmd_data.ifi_ibytes;
+		new_outb = ifp->if_mib.ifmd_data.ifi_obytes;
 
 		/* Display interface if it's received some traffic. */
 		if (new_inb > 0 && old_inb == 0) {
@@ -269,9 +269,9 @@ fetchifstat(void)
 		 * for our current traffic rates, and while we're there,
 		 * see if we have new peak rates.
 		 */
-                old_tv = ifp->tv;
-                timersub(&new_tv, &old_tv, &tv);
-                elapsed = tv.tv_sec + (tv.tv_usec * 1e-6);
+		old_tv = ifp->tv;
+		timersub(&new_tv, &old_tv, &tv);
+		elapsed = tv.tv_sec + (tv.tv_usec * 1e-6);
 
 		ifp->if_in_curtraffic = new_inb - old_inb;
 		ifp->if_out_curtraffic = new_outb - old_outb;
@@ -281,8 +281,8 @@ fetchifstat(void)
 		 * and line, we divide by ``elapsed'' as this is likely
 		 * to be more accurate.
 		 */
-                ifp->if_in_curtraffic /= elapsed;
-                ifp->if_out_curtraffic /= elapsed;
+		ifp->if_in_curtraffic /= elapsed;
+		ifp->if_out_curtraffic /= elapsed;
 
 		if (ifp->if_in_curtraffic > ifp->if_in_traffic_peak)
 			ifp->if_in_traffic_peak = ifp->if_in_curtraffic;
diff --git a/usr.bin/systat/keyboard.c b/usr.bin/systat/keyboard.c
index 990e83ea5ae6..e7e001407740 100644
--- a/usr.bin/systat/keyboard.c
+++ b/usr.bin/systat/keyboard.c
@@ -47,76 +47,76 @@ static const char sccsid[] = "@(#)keyboard.c	8.1 (Berkeley) 6/6/93";
 int
 keyboard(void)
 {
-        char ch, line[80];
-	int oldmask;
+	char line[80];
+	int ch, oldmask;
 
-        for (;;) {
-                col = 0;
-                move(CMDLINE, 0);
-                do {
-                        refresh();
-                        ch = getch();
-                        if (ch == ERR) {
-                                if (errno == EINTR)
-                                        continue;
-                                exit(1);
-                        }
-                        if (ch >= 'A' && ch <= 'Z')
-                                ch += 'a' - 'A';
-                        if (col == 0) {
+	for (;;) {
+		col = 0;
+		move(CMDLINE, 0);
+		do {
+			refresh();
+			ch = getch();
+			if (ch == ERR) {
+				if (errno == EINTR)
+					continue;
+				exit(1);
+			}
+			if (ch >= 'A' && ch <= 'Z')
+				ch += 'a' - 'A';
+			if (col == 0) {
 #define	mask(s)	(1 << ((s) - 1))
-                                if (ch == CTRL('l')) {
+				if (ch == CTRL('l')) {
 					oldmask = sigblock(mask(SIGALRM));
 					wrefresh(curscr);
 					sigsetmask(oldmask);
-                                        continue;
-                                }
+					continue;
+				}
 				if (ch == CTRL('g')) {
 					oldmask = sigblock(mask(SIGALRM));
 					status();
 					sigsetmask(oldmask);
 					continue;
 				}
-                                if (ch != ':')
-                                        continue;
-                                move(CMDLINE, 0);
-                                clrtoeol();
-                        }
-                        if (ch == erasechar() && col > 0) {
-                                if (col == 1 && line[0] == ':')
-                                        continue;
-                                col--;
-                                goto doerase;
-                        }
-                        if (ch == CTRL('w') && col > 0) {
-                                while (--col >= 0 && isspace(line[col]))
-                                        ;
-                                col++;
-                                while (--col >= 0 && !isspace(line[col]))
-                                        if (col == 0 && line[0] == ':')
-                                                break;
-                                col++;
-                                goto doerase;
-                        }
-                        if (ch == killchar() && col > 0) {
-                                col = 0;
-                                if (line[0] == ':')
-                                        col++;
-                doerase:
-                                move(CMDLINE, col);
-                                clrtoeol();
-                                continue;
-                        }
-                        if (isprint(ch) || ch == ' ') {
-                                line[col] = ch;
-                                mvaddch(CMDLINE, col, ch);
-                                col++;
-                        }
-                } while (col == 0 || (ch != '\r' && ch != '\n'));
-                line[col] = '\0';
+				if (ch != ':')
+					continue;
+				move(CMDLINE, 0);
+				clrtoeol();
+			}
+			if (ch == erasechar() && col > 0) {
+				if (col == 1 && line[0] == ':')
+					continue;
+				col--;
+				goto doerase;
+			}
+			if (ch == CTRL('w') && col > 0) {
+				while (--col >= 0 && isspace(line[col]))
+					;
+				col++;
+				while (--col >= 0 && !isspace(line[col]))
+					if (col == 0 && line[0] == ':')
+						break;
+				col++;
+				goto doerase;
+			}
+			if (ch == killchar() && col > 0) {
+				col = 0;
+				if (line[0] == ':')
+					col++;
+		doerase:
+				move(CMDLINE, col);
+				clrtoeol();
+				continue;
+			}
+			if (isprint(ch) || ch == ' ') {
+				line[col] = ch;
+				mvaddch(CMDLINE, col, ch);
+				col++;
+			}
+		} while (col == 0 || (ch != '\r' && ch != '\n'));
+		line[col] = '\0';
 		oldmask = sigblock(mask(SIGALRM));
-                command(line + 1);
+		command(line + 1);
 		sigsetmask(oldmask);
-        }
+	}
 	/*NOTREACHED*/
 }
diff --git a/usr.bin/systat/mode.c b/usr.bin/systat/mode.c
index 5f64e3ec237f..a3fdf46c3b5d 100644
--- a/usr.bin/systat/mode.c
+++ b/usr.bin/systat/mode.c
@@ -12,7 +12,7 @@
  * no representations about the suitability of this software for any
  * purpose.  It is provided "as is" without express or implied
  * warranty.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
  * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
diff --git a/usr.bin/systat/mode.h b/usr.bin/systat/mode.h
index 9fc0feae96d5..64b5275433ca 100644
--- a/usr.bin/systat/mode.h
+++ b/usr.bin/systat/mode.h
@@ -12,7 +12,7 @@
  * no representations about the suitability of this software for any
  * purpose.  It is provided "as is" without express or implied
  * warranty.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
  * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
diff --git a/usr.bin/systat/netstat.c b/usr.bin/systat/netstat.c
index b1c6dc9e3f14..d96fb16082bb 100644
--- a/usr.bin/systat/netstat.c
+++ b/usr.bin/systat/netstat.c
@@ -128,7 +128,7 @@ closenetstat(WINDOW *w)
 			lastrow--;
 		p->ni_line = -1;
 	}
-        if (w != NULL) {
+	if (w != NULL) {
 		wclear(w);
 		wrefresh(w);
 		delwin(w);
diff --git a/usr.bin/systat/systat.h b/usr.bin/systat/systat.h
index a77ea7f06c05..1b4322c1b307 100644
--- a/usr.bin/systat/systat.h
+++ b/usr.bin/systat/systat.h
@@ -33,10 +33,10 @@
 #include <curses.h>
 
 struct  cmdtab {
-        const char *c_name;		/* command name */
-        void    (*c_refresh)(void);	/* display refresh */
-        void    (*c_fetch)(void);	/* sets up data structures */
-        void    (*c_label)(void);	/* label display */
+	const char *c_name;		/* command name */
+	void	(*c_refresh)(void);	/* display refresh */
+	void	(*c_fetch)(void);	/* sets up data structures */
+	void	(*c_label)(void);	/* label display */
 	int	(*c_init)(void);	/* initialize namelist, etc. */
 	WINDOW	*(*c_open)(void);	/* open display */
 	void	(*c_close)(WINDOW *);	/* close display */
diff --git a/usr.bin/systat/vmstat.c b/usr.bin/systat/vmstat.c
index e479ec9ba920..459b58d90b49 100644
--- a/usr.bin/systat/vmstat.c
+++ b/usr.bin/systat/vmstat.c
@@ -224,7 +224,7 @@ initkre(void)
 		intrloc = calloc(nintr, sizeof (long));
 		intrname = calloc(nintr, sizeof (char *));
 		intrnamebuf = sysctl_dynread("hw.intrnames", NULL);
-		if (intrnamebuf == NULL || intrname == NULL || 
+		if (intrnamebuf == NULL || intrname == NULL ||
 		    intrloc == NULL) {
 			error("Out of memory");
 			if (intrnamebuf)
@@ -854,7 +854,7 @@ dinfo(int dn, int lc, struct statinfo *now, struct statinfo *then)
 		elapsed_time = now->snap_time - then->snap_time;
 	} else {
 		/* Calculate relative to device creation */
-	        elapsed_time = now->snap_time - devstat_compute_etime(
+		elapsed_time = now->snap_time - devstat_compute_etime(
 		    &now->dinfo->devices[di].creation_time, NULL);
 	}
 
diff --git a/usr.bin/tar/bsdtar.c b/usr.bin/tar/bsdtar.c
index 0bb6a7fab646..1f250b6930a0 100644
--- a/usr.bin/tar/bsdtar.c
+++ b/usr.bin/tar/bsdtar.c
@@ -147,6 +147,8 @@ main(int argc, char **argv)
 	_bsdtar = bsdtar = &bsdtar_storage;
 	memset(bsdtar, 0, sizeof(*bsdtar));
 	bsdtar->fd = -1; /* Mark as "unused" */
+	bsdtar->gid = -1;
+	bsdtar->uid = -1;
 	option_o = 0;
 
 #if defined(HAVE_SIGACTION) && (defined(SIGINFO) || defined(SIGUSR1))
@@ -262,14 +264,21 @@ main(int argc, char **argv)
 		case OPTION_FORMAT: /* GNU tar, others */
 			bsdtar->create_format = bsdtar->optarg;
 			break;
-		case OPTION_OPTIONS:
-			bsdtar->option_options = bsdtar->optarg;
-			break;
 		case 'f': /* SUSv2 */
 			bsdtar->filename = bsdtar->optarg;
 			if (strcmp(bsdtar->filename, "-") == 0)
 				bsdtar->filename = NULL;
 			break;
+		case OPTION_GID: /* cpio */
+			t = atoi(bsdtar->optarg);
+			if (t < 0)
+				lafe_errc(1, 0,
+				    "Argument to --gid must be positive");
+			bsdtar->gid = t;
+			break;
+		case OPTION_GNAME: /* cpio */
+			bsdtar->gname = bsdtar->optarg;
+			break;
 		case 'H': /* BSD convention */
 			bsdtar->symlink_mode = 'H';
 			break;
@@ -397,7 +406,8 @@ main(int argc, char **argv)
 			bsdtar->option_null++;
 			break;
 		case OPTION_NUMERIC_OWNER: /* GNU tar */
-			bsdtar->option_numeric_owner++;
+			bsdtar->uname = "";
+			bsdtar->gname = "";
 			break;
 		case 'O': /* GNU tar */
 			bsdtar->option_stdout = 1;
@@ -408,6 +418,9 @@ main(int argc, char **argv)
 		case OPTION_ONE_FILE_SYSTEM: /* GNU tar */
 			bsdtar->option_dont_traverse_mounts = 1;
 			break;
+		case OPTION_OPTIONS:
+			bsdtar->option_options = bsdtar->optarg;
+			break;
 #if 0
 		/*
 		 * The common BSD -P option is not necessary, since
@@ -473,6 +486,16 @@ main(int argc, char **argv)
 		case 'u': /* SUSv2 */
 			set_mode(bsdtar, opt);
 			break;
+		case OPTION_UID: /* cpio */
+			t = atoi(bsdtar->optarg);
+			if (t < 0)
+				lafe_errc(1, 0,
+				    "Argument to --uid must be positive");
+			bsdtar->uid = t;
+			break;
+		case OPTION_UNAME: /* cpio */
+			bsdtar->uname = bsdtar->optarg;
+			break;
 		case 'v': /* SUSv2 */
 			bsdtar->verbose++;
 			break;
diff --git a/usr.bin/tar/bsdtar.h b/usr.bin/tar/bsdtar.h
index 919156a0b1ab..ede96d9fb9f3 100644
--- a/usr.bin/tar/bsdtar.h
+++ b/usr.bin/tar/bsdtar.h
@@ -54,6 +54,10 @@ struct bsdtar {
 	int		  verbose;   /* -v */
 	int		  extract_flags; /* Flags for extract operation */
 	int		  strip_components; /* Remove this many leading dirs */
+	int		  gid;  /* --gid */
+	const char	 *gname; /* --gname */
+	int		  uid;  /* --uid */
+	const char	 *uname; /* --uname */
 	char		  mode; /* Program mode: 'c', 't', 'r', 'u', 'x' */
 	char		  symlink_mode; /* H or L, per BSD conventions */
 	char		  create_compression; /* j, y, or z */
@@ -68,7 +72,6 @@ struct bsdtar {
 	char		  option_no_owner; /* -o */
 	char		  option_no_subdirs; /* -n */
 	char		  option_null; /* --null */
-	char		  option_numeric_owner; /* --numeric-owner */
 	char		  option_stdout; /* -O */
 	char		  option_totals; /* --totals */
 	char		  option_unlink_first; /* -U */
@@ -111,7 +114,8 @@ enum {
 	OPTION_CHROOT,
 	OPTION_EXCLUDE,
 	OPTION_FORMAT,
-	OPTION_OPTIONS,
+	OPTION_GID,
+	OPTION_GNAME,
 	OPTION_HELP,
 	OPTION_INCLUDE,
 	OPTION_KEEP_NEWER_FILES,
@@ -126,10 +130,13 @@ enum {
 	OPTION_NULL,
 	OPTION_NUMERIC_OWNER,
 	OPTION_ONE_FILE_SYSTEM,
+	OPTION_OPTIONS,
 	OPTION_POSIX,
 	OPTION_SAME_OWNER,
 	OPTION_STRIP_COMPONENTS,
 	OPTION_TOTALS,
+	OPTION_UID,
+	OPTION_UNAME,
 	OPTION_USE_COMPRESS_PROGRAM,
 	OPTION_VERSION
 };
diff --git a/usr.bin/tar/cmdline.c b/usr.bin/tar/cmdline.c
index ba3e8a1b2a88..f0431554c521 100644
--- a/usr.bin/tar/cmdline.c
+++ b/usr.bin/tar/cmdline.c
@@ -84,7 +84,8 @@ static struct option {
 	{ "file",                 1, 'f' },
 	{ "files-from",           1, 'T' },
 	{ "format",               1, OPTION_FORMAT },
-	{ "options",              1, OPTION_OPTIONS },
+	{ "gid",		  1, OPTION_GID },
+	{ "gname",		  1, OPTION_GNAME },
 	{ "gunzip",               0, 'z' },
 	{ "gzip",                 0, 'z' },
 	{ "help",                 0, OPTION_HELP },
@@ -110,6 +111,7 @@ static struct option {
 	{ "null",		  0, OPTION_NULL },
 	{ "numeric-owner",	  0, OPTION_NUMERIC_OWNER },
 	{ "one-file-system",	  0, OPTION_ONE_FILE_SYSTEM },
+	{ "options",              1, OPTION_OPTIONS },
 	{ "posix",		  0, OPTION_POSIX },
 	{ "preserve-permissions", 0, 'p' },
 	{ "read-full-blocks",	  0, 'B' },
@@ -118,6 +120,8 @@ static struct option {
 	{ "strip-components",	  1, OPTION_STRIP_COMPONENTS },
 	{ "to-stdout",            0, 'O' },
 	{ "totals",		  0, OPTION_TOTALS },
+	{ "uid",		  1, OPTION_UID },
+	{ "uname",		  1, OPTION_UNAME },
 	{ "uncompress",           0, 'Z' },
 	{ "unlink",		  0, 'U' },
 	{ "unlink-first",	  0, 'U' },
diff --git a/usr.bin/tar/read.c b/usr.bin/tar/read.c
index b1616ec2e765..6b1e5ec2dd20 100644
--- a/usr.bin/tar/read.c
+++ b/usr.bin/tar/read.c
@@ -209,10 +209,18 @@ read_archive(struct bsdtar *bsdtar, char mode)
 		if (r == ARCHIVE_FATAL)
 			break;
 
-		if (bsdtar->option_numeric_owner) {
+		if (bsdtar->uid >= 0) {
+			archive_entry_set_uid(entry, bsdtar->uid);
 			archive_entry_set_uname(entry, NULL);
+		}
+		if (bsdtar->gid >= 0) {
+			archive_entry_set_gid(entry, bsdtar->gid);
 			archive_entry_set_gname(entry, NULL);
 		}
+		if (bsdtar->uname)
+			archive_entry_set_uname(entry, bsdtar->uname);
+		if (bsdtar->gname)
+			archive_entry_set_gname(entry, bsdtar->gname);
 
 		/*
 		 * Exclude entries that are too old.
diff --git a/usr.bin/tar/write.c b/usr.bin/tar/write.c
index 32e43c60e680..a0a9b753d108 100644
--- a/usr.bin/tar/write.c
+++ b/usr.bin/tar/write.c
@@ -833,6 +833,24 @@ write_hierarchy(struct bsdtar *bsdtar, struct archive *a, const char *path)
 #endif
 		r = archive_read_disk_entry_from_file(bsdtar->diskreader,
 		    entry, -1, st);
+		if (bsdtar->uid >= 0) {
+			archive_entry_set_uid(entry, bsdtar->uid);
+			if (!bsdtar->uname)
+				archive_entry_set_uname(entry,
+				    archive_read_disk_uname(bsdtar->diskreader,
+					bsdtar->uid));
+		}
+		if (bsdtar->gid >= 0) {
+			archive_entry_set_gid(entry, bsdtar->gid);
+			if (!bsdtar->gname)
+				archive_entry_set_gname(entry,
+				    archive_read_disk_gname(bsdtar->diskreader,
+					bsdtar->gid));
+		}
+		if (bsdtar->uname)
+			archive_entry_set_uname(entry, bsdtar->uname);
+		if (bsdtar->gname)
+			archive_entry_set_gname(entry, bsdtar->gname);
 		if (r != ARCHIVE_OK)
 			lafe_warnc(archive_errno(bsdtar->diskreader),
 			    "%s", archive_error_string(bsdtar->diskreader));
diff --git a/usr.bin/tr/Makefile b/usr.bin/tr/Makefile
index c5c5297ef64b..00bdd8d5bb7a 100644
--- a/usr.bin/tr/Makefile
+++ b/usr.bin/tr/Makefile
@@ -4,6 +4,4 @@
 PROG=	tr
 SRCS=	cmap.c cset.c str.c tr.c
 
-WARNS?=	1
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/tr/cset.c b/usr.bin/tr/cset.c
index 1b4212949b6c..58a927c8b972 100644
--- a/usr.bin/tr/cset.c
+++ b/usr.bin/tr/cset.c
@@ -153,11 +153,11 @@ cset_in_hard(struct cset *cs, wchar_t ch)
 	struct csclass *csc;
 
 	for (csc = cs->cs_classes; csc != NULL; csc = csc->csc_next)
-		if (csc->csc_invert ^ iswctype(ch, csc->csc_type) != 0)
+		if (csc->csc_invert ^ (iswctype(ch, csc->csc_type) != 0))
 			return (cs->cs_invert ^ true);
 	if (cs->cs_root != NULL) {
 		cs->cs_root = cset_splay(cs->cs_root, ch);
-		return (cs->cs_invert ^ cset_rangecmp(cs->cs_root, ch) == 0);
+		return (cs->cs_invert ^ (cset_rangecmp(cs->cs_root, ch) == 0));
 	}
 	return (cs->cs_invert ^ false);
 }
diff --git a/usr.bin/tr/str.c b/usr.bin/tr/str.c
index 70a74eaa21e5..abfd0d6a657c 100644
--- a/usr.bin/tr/str.c
+++ b/usr.bin/tr/str.c
@@ -57,8 +57,7 @@ static int      genrange(STR *, int);
 static void	genseq(STR *);
 
 wint_t
-next(s)
-	STR *s;
+next(STR *s)
 {
 	int is_octal;
 	wint_t ch;
@@ -135,8 +134,7 @@ next(s)
 }
 
 static int
-bracket(s)
-	STR *s;
+bracket(STR *s)
 {
 	char *p;
 
@@ -173,8 +171,7 @@ bracket(s)
 }
 
 static void
-genclass(s)
-	STR *s;
+genclass(STR *s)
 {
 
 	if ((s->cclass = wctype(s->str)) == 0)
@@ -190,8 +187,7 @@ genclass(s)
 }
 
 static void
-genequiv(s)
-	STR *s;
+genequiv(STR *s)
 {
 	int i, p, pri;
 	char src[2], dst[3];
@@ -295,8 +291,7 @@ genrange(STR *s, int was_octal)
 }
 
 static void
-genseq(s)
-	STR *s;
+genseq(STR *s)
 {
 	char *ep;
 	wchar_t wc;
diff --git a/usr.bin/truss/Makefile b/usr.bin/truss/Makefile
index f80ac0367ea6..31f08d49b444 100644
--- a/usr.bin/truss/Makefile
+++ b/usr.bin/truss/Makefile
@@ -23,7 +23,8 @@ syscalls.h:	syscalls.master
 		${.CURDIR}/i386.conf
 
 ioctl.c: ${.CURDIR}/../kdump/mkioctls
-	sh ${.CURDIR}/../kdump/mkioctls ${DESTDIR}/usr/include > ${.TARGET}
+	env MACHINE=${MACHINE} \
+		/bin/sh ${.CURDIR}/../kdump/mkioctls return ${DESTDIR}/usr/include > ${.TARGET}
 
 .if ${MACHINE_CPUARCH} == "i386"
 SRCS+=	i386-linux.c linux_syscalls.h
diff --git a/usr.bin/truss/extern.h b/usr.bin/truss/extern.h
index e3ffc4f566a3..a56a01eb634f 100644
--- a/usr.bin/truss/extern.h
+++ b/usr.bin/truss/extern.h
@@ -35,7 +35,7 @@ extern int setup_and_wait(char **);
 extern int start_tracing(int);
 extern void restore_proc(int);
 extern void waitevent(struct trussinfo *);
-extern const char *ioctlname(register_t val);
+extern const char *ioctlname(unsigned long val);
 extern char *strsig(int sig);
 #ifdef __amd64__
 extern void amd64_syscall_entry(struct trussinfo *, int);
diff --git a/usr.bin/units/units.1 b/usr.bin/units/units.1
index 96b84d6efa30..505552a7b00b 100644
--- a/usr.bin/units/units.1
+++ b/usr.bin/units/units.1
@@ -35,9 +35,7 @@ their equivalents in other scales.
 The
 .Nm
 program can only
-handle multiplicative scale changes.
-It cannot convert Celsius
-to Fahrenheit, for example.
+handle multiplicative or affine scale changes.
 It works interactively by prompting
 the user for input:
 .Bd -literal
@@ -60,6 +58,10 @@ the user for input:
     You want: cm
             * 1.27
             / 0.78740157
+
+    You have: 85 degF
+    You want: degC
+	29.444444
 .Ed
 .Pp
 Powers of units can be specified using the '^' character as shown in
diff --git a/usr.bin/usbhidaction/usbhidaction.1 b/usr.bin/usbhidaction/usbhidaction.1
index 148c6d886d19..633272e398e8 100644
--- a/usr.bin/usbhidaction/usbhidaction.1
+++ b/usr.bin/usbhidaction/usbhidaction.1
@@ -99,8 +99,7 @@ a debounce value, and an action.
 There must be whitespace between the parts.
 .Pp
 The item names are similar to those used by
-.Xr usbhidctl 1 ,
-but each part must be prefixed by its page name.
+.Xr usbhidctl 1 .
 .Pp
 The value is simply a numeric value.
 When the item reports this value,
diff --git a/usr.bin/usbhidaction/usbhidaction.c b/usr.bin/usbhidaction/usbhidaction.c
index c71a8fe7cec4..fd66b6433f22 100644
--- a/usr.bin/usbhidaction/usbhidaction.c
+++ b/usr.bin/usbhidaction/usbhidaction.c
@@ -280,12 +280,11 @@ parse_conf(const char *conf, report_desc_t repd, int reportid, int ignore)
 	char *p;
 	int line;
 	char buf[SIZE], name[SIZE], value[SIZE], debounce[SIZE], action[SIZE];
-	char usbuf[SIZE], coll[SIZE];
+	char usbuf[SIZE], coll[SIZE], *tmp;
 	struct command *cmd, *cmds;
 	struct hid_data *d;
 	struct hid_item h;
-	int u, lo, hi, range;
-	
+	int inst, cinst, u, lo, hi, range, t;
 
 	f = fopen(conf, "r");
 	if (f == NULL)
@@ -317,6 +316,12 @@ parse_conf(const char *conf, report_desc_t repd, int reportid, int ignore)
 				     ", syntax error: %s", conf, line, buf);
 			}
 		}
+		tmp = strchr(name, '#');
+		if (tmp != NULL) {
+			*tmp = 0;
+			inst = atoi(tmp + 1);
+		} else
+			inst = 0;
 
 		cmd = malloc(sizeof *cmd);
 		if (cmd == NULL)
@@ -361,6 +366,7 @@ parse_conf(const char *conf, report_desc_t repd, int reportid, int ignore)
 		}
 
 		coll[0] = 0;
+		cinst = 0;
 		for (d = hid_start_parse(repd, 1 << hid_input, reportid);
 		     hid_get_item(d, &h); ) {
 			if (verbose > 2)
@@ -380,24 +386,29 @@ parse_conf(const char *conf, report_desc_t repd, int reportid, int ignore)
 					range = 0;
 				}
 				for (u = lo; u <= hi; u++) {
-					snprintf(usbuf, sizeof usbuf,  "%s:%s",
-						 hid_usage_page(HID_PAGE(u)), 
-						 hid_usage_in_page(u));
-					if (verbose > 2)
-						printf("usage %s\n", usbuf);
-					if (!strcasecmp(usbuf, name))
-						goto foundhid;
 					if (coll[0]) {
 						snprintf(usbuf, sizeof usbuf,
 						  "%s.%s:%s", coll+1,
-						  hid_usage_page(HID_PAGE(u)), 
+						  hid_usage_page(HID_PAGE(u)),
+						  hid_usage_in_page(u));
+					} else {
+						snprintf(usbuf, sizeof usbuf,
+						  "%s:%s",
+						  hid_usage_page(HID_PAGE(u)),
 						  hid_usage_in_page(u));
-						if (verbose > 2)
-							printf("usage %s\n",
-							       usbuf);
-						if (!strcasecmp(usbuf, name))
-							goto foundhid;
 					}
+					if (verbose > 2)
+						printf("usage %s\n", usbuf);
+					t = strlen(usbuf) - strlen(name);
+					if (t > 0) {
+						if (strcmp(usbuf + t, name))
+							continue;
+						if (usbuf[t - 1] != '.')
+							continue;
+					} else if (strcmp(usbuf, name))
+						continue;
+					if (inst == cinst++)
+						goto foundhid;
 				}
 				break;
 			case hid_collection:
diff --git a/usr.bin/usbhidctl/usbhid.c b/usr.bin/usbhidctl/usbhid.c
index 281078d3de2f..bf3e348d526d 100644
--- a/usr.bin/usbhidctl/usbhid.c
+++ b/usr.bin/usbhidctl/usbhid.c
@@ -42,45 +42,141 @@
 #include <usbhid.h>
 #include <dev/usb/usbhid.h>
 
+struct variable {
+	char *name;
+	int instance;
+	int val;
+	struct hid_item h;
+	struct variable *next;
+} *vars;
+
 int verbose = 0;
-int all = 0;
 int noname = 0;
 int hexdump = 0;
+int wflag = 0;
+int zflag = 0;
 
-char **names;
-int nnames;
+static void usage(void);
+static void dumpitem(const char *label, struct hid_item *h);
+static void dumpitems(report_desc_t r);
+static void prdata(u_char *buf, struct hid_item *h);
+static void dumpdata(int f, report_desc_t r, int loop);
+static void writedata(int f, report_desc_t r);
 
-void prbits(int bits, char **strs, int n);
-void usage(void);
-void dumpitem(const char *label, struct hid_item *h);
-void dumpitems(report_desc_t r);
-void rev(struct hid_item **p);
-void prdata(u_char *buf, struct hid_item *h);
-void dumpdata(int f, report_desc_t r, int loop);
-int gotname(char *n);
-
-int
-gotname(char *n)
+static void
+parceargs(report_desc_t r, int all, int nnames, char **names)
 {
-	int i;
+	struct hid_data *d;
+	struct hid_item h;
+	char colls[1000];
+	char hname[1000], *tmp1, *tmp2;
+	struct variable *var, **pnext;
+	int i, instance, cp, t;
 
-	for (i = 0; i < nnames; i++)
-		if (strcmp(names[i], n) == 0)
-			return 1;
-	return 0;
+	pnext = &vars;
+	if (all) {
+		if (wflag)
+			errx(1, "Must not specify -w to read variables");
+		cp = 0;
+		for (d = hid_start_parse(r,
+		    1<<hid_input | 1<<hid_output | 1<<hid_feature, -1);
+		    hid_get_item(d, &h); ) {
+			if (h.kind == hid_collection) {
+				cp += sprintf(&colls[cp], "%s%s:%s",
+				    cp != 0 ? "." : "",
+				    hid_usage_page(HID_PAGE(h.usage)),
+				    hid_usage_in_page(h.usage));
+			} else if (h.kind == hid_endcollection) {
+				tmp1 = strrchr(colls, '.');
+				if (tmp1 != NULL) {
+					cp -= strlen(tmp1);
+					tmp1[0] = 0;
+				} else {
+					cp = 0;
+					colls[0] = 0;
+				}
+			}
+			if ((h.kind != hid_input && h.kind != hid_output &&
+			    h.kind != hid_feature) || (h.flags & HIO_CONST))
+				continue;
+			var = malloc(sizeof(*var));
+			memset(var, 0, sizeof(*var));
+			asprintf(&var->name, "%s%s%s:%s",
+			    colls, colls[0] != 0 ? "." : "",
+			    hid_usage_page(HID_PAGE(h.usage)),
+			    hid_usage_in_page(h.usage));
+			var->h = h;
+			*pnext = var;
+			pnext = &var->next;
+		}
+		hid_end_parse(d);
+		return;
+	}
+	for (i = 0; i < nnames; i++) {
+		var = malloc(sizeof(*var));
+		memset(var, 0, sizeof(*var));
+		tmp1 = tmp2 = strdup(names[i]);
+		strsep(&tmp2, "=");
+		var->name = strsep(&tmp1, "#");
+		if (tmp1 != NULL)
+			var->instance = atoi(tmp1);
+		if (tmp2 != NULL) {
+			if (!wflag)
+				errx(1, "Must specify -w to write variables");
+			var->val = atoi(tmp2);
+		} else
+			if (wflag)
+				errx(1, "Must not specify -w to read variables");
+		*pnext = var;
+		pnext = &var->next;
+
+		instance = 0;
+		cp = 0;
+		for (d = hid_start_parse(r,
+		    1<<hid_input | 1<<hid_output | 1<<hid_feature, -1);
+		    hid_get_item(d, &h); ) {
+			if (h.kind == hid_collection) {
+				cp += sprintf(&colls[cp], "%s%s:%s",
+				    cp != 0 ? "." : "",
+				    hid_usage_page(HID_PAGE(h.usage)),
+				    hid_usage_in_page(h.usage));
+			} else if (h.kind == hid_endcollection) {
+				tmp1 = strrchr(colls, '.');
+				if (tmp1 != NULL) {
+					cp -= strlen(tmp1);
+					tmp1[0] = 0;
+				} else {
+					cp = 0;
+					colls[0] = 0;
+				}
+			}
+			if ((h.kind != hid_input && h.kind != hid_output &&
+			    h.kind != hid_feature) || (h.flags & HIO_CONST))
+				continue;
+			snprintf(hname, sizeof(hname), "%s%s%s:%s",
+			    colls, colls[0] != 0 ? "." : "",
+			    hid_usage_page(HID_PAGE(h.usage)),
+			    hid_usage_in_page(h.usage));
+			t = strlen(hname) - strlen(var->name);
+			if (t > 0) {
+				if (strcmp(hname + t, var->name) != 0)
+					continue;
+				if (hname[t - 1] != '.')
+					continue;
+			} else if (strcmp(hname, var->name) != 0)
+				continue;
+			if (var->instance != instance++)
+				continue;
+			var->h = h;
+			break;
+		}
+		hid_end_parse(d);
+		if (var->h.usage == 0)
+			errx(1, "Unknown item '%s'", var->name);
+	}
 }
 
-void
-prbits(int bits, char **strs, int n)
-{
-	int i;
-
-	for(i = 0; i < n; i++, bits >>= 1)
-		if (strs[i*2])
-			printf("%s%s", i == 0 ? "" : ", ", strs[i*2 + (bits&1)]);
-}
-
-void
+static void
 usage(void)
 {
 
@@ -92,10 +188,14 @@ usage(void)
                 "       %s -f device "
                 "[-l] [-n] [-r] [-t tablefile] [-v] [-x] -a\n",
                 getprogname());
+	fprintf(stderr,
+                "       %s -f device "
+                "[-t tablefile] [-v] [-z] -w name=value\n",
+                getprogname());
 	exit(1);
 }
 
-void
+static void
 dumpitem(const char *label, struct hid_item *h)
 {
 	if ((h->flags & HIO_CONST) && !verbose)
@@ -134,7 +234,7 @@ hid_collection_type(int32_t type)
 	return (num);
 }
 
-void
+static void
 dumpitems(report_desc_t r)
 {
 	struct hid_data *d;
@@ -174,23 +274,7 @@ dumpitems(report_desc_t r)
 	printf("Total feature size %d bytes\n", size);
 }
 
-void
-rev(struct hid_item **p)
-{
-	struct hid_item *cur, *prev, *next;
-
-	prev = 0;
-	cur = *p;
-	while(cur != 0) {
-		next = cur->next;
-		cur->next = prev;
-		prev = cur;
-		cur = next;
-	}
-	*p = prev;
-}
-
-void
+static void
 prdata(u_char *buf, struct hid_item *h)
 {
 	u_int data;
@@ -212,82 +296,162 @@ prdata(u_char *buf, struct hid_item *h)
 	h->pos = pos;
 }
 
-void
+static void
 dumpdata(int f, report_desc_t rd, int loop)
 {
-	struct hid_data *d;
-	struct hid_item h, *hids, *n;
-	int r, dlen;
+	struct variable *var;
+	int dlen, havedata, i, match, r, rid, use_rid;
 	u_char *dbuf;
-	u_int32_t colls[100];
-	int sp = 0;
-	char namebuf[10000], *namep;
+	enum hid_kind kind;
 
-	hids = 0;
-	for (d = hid_start_parse(rd, 1<<hid_input, -1);
-	     hid_get_item(d, &h); ) {
-		if (h.kind == hid_collection)
-			colls[++sp] = h.usage;
-		else if (h.kind == hid_endcollection)
-			--sp;
-		if (h.kind != hid_input || (h.flags & HIO_CONST))
-			continue;
-		h.next = hids;
-		h.collection = colls[sp];
-		hids = malloc(sizeof *hids);
-		*hids = h;
-	}
-	hid_end_parse(d);
-	rev(&hids);
-	dlen = hid_report_size(rd, hid_input, -1);
-	dbuf = malloc(dlen);
-	if (!loop)
-		if (hid_set_immed(f, 1) < 0) {
-			if (errno == EOPNOTSUPP)
-				warnx("device does not support immediate mode, only changes reported.");
-			else
-				err(1, "USB_SET_IMMED");
-		}
+	kind = 0;
+	rid = -1;
+	use_rid = !!hid_get_report_id(f);
 	do {
-		r = read(f, dbuf, dlen);
-		if (r < 1) {
-			err(1, "read error");
-		}
-		for (n = hids; n; n = n->next) {
-			if (n->report_ID != 0 && dbuf[0] != n->report_ID)
+		if (kind < 3) {
+			if (++rid >= 256) {
+				rid = 0;
+				kind++;
+			}
+			if (kind >= 3)
+				rid = -1;
+			for (var = vars; var; var = var->next) {
+				if (rid == var->h.report_ID &&
+				    kind == var->h.kind)
+					break;
+			}
+			if (var == NULL)
 				continue;
-			namep = namebuf;
-			namep += sprintf(namep, "%s:%s.",
-					 hid_usage_page(HID_PAGE(n->collection)),
-					 hid_usage_in_page(n->collection));
-			namep += sprintf(namep, "%s:%s",
-					 hid_usage_page(HID_PAGE(n->usage)),
-					 hid_usage_in_page(n->usage));
-			if (all || gotname(namebuf)) {
-				if (!noname)
-					printf("%s=", namebuf);
-				prdata(dbuf, n);
+		}
+		dlen = hid_report_size(rd, kind < 3 ? kind : hid_input, rid);
+		if (dlen <= 0)
+			continue;
+		dbuf = malloc(dlen);
+		memset(dbuf, 0, dlen);
+		if (kind < 3) {
+			dbuf[0] = rid;
+			r = hid_get_report(f, kind, dbuf, dlen);
+			if (r < 0)
+				warn("hid_get_report(rid %d)", rid);
+			havedata = !r && (rid == 0 || dbuf[0] == rid);
+			if (rid != 0)
+				dbuf[0] = rid;
+		} else {
+			r = read(f, dbuf, dlen);
+			if (r < 1)
+				err(1, "read error");
+			havedata = 1;
+		}
+		if (verbose) {
+			printf("Got %s report %d (%d bytes):",
+			    kind == hid_output ? "output" :
+			    kind == hid_feature ? "feature" : "input",
+			    use_rid ? dbuf[0] : 0, dlen);
+			if (havedata) {
+				for (i = 0; i < dlen; i++)
+					printf(" %02x", dbuf[i]);
+			}
+			printf("\n");
+		}
+		match = 0;
+		for (var = vars; var; var = var->next) {
+			if ((kind < 3 ? kind : hid_input) != var->h.kind)
+				continue;
+			if (var->h.report_ID != 0 &&
+			    dbuf[0] != var->h.report_ID)
+				continue;
+			match = 1;
+			if (!noname)
+				printf("%s=", var->name);
+			if (havedata)
+				prdata(dbuf, &var->h);
+			printf("\n");
+		}
+		if (match)
+			printf("\n");
+		free(dbuf);
+	} while (loop || kind < 3);
+}
+
+static void
+writedata(int f, report_desc_t rd)
+{
+	struct variable *var;
+	int dlen, i, r, rid;
+	u_char *dbuf;
+	enum hid_kind kind;
+
+	kind = 0;
+	rid = 0;
+	for (kind = 0; kind < 3; kind ++) {
+	    for (rid = 0; rid < 256; rid ++) {
+		for (var = vars; var; var = var->next) {
+			if (rid == var->h.report_ID && kind == var->h.kind)
+				break;
+		}
+		if (var == NULL)
+			continue;
+		dlen = hid_report_size(rd, kind, rid);
+		if (dlen <= 0)
+			continue;
+		dbuf = malloc(dlen);
+		memset(dbuf, 0, dlen);
+		dbuf[0] = rid;
+		if (!zflag && hid_get_report(f, kind, dbuf, dlen) == 0) {
+			if (verbose) {
+				printf("Got %s report %d (%d bytes):",
+				    kind == hid_input ? "input" :
+				    kind == hid_output ? "output" : "feature",
+				    rid, dlen);
+				for (i = 0; i < dlen; i++)
+					printf(" %02x", dbuf[i]);
 				printf("\n");
 			}
+		} else if (!zflag) {
+			warn("hid_get_report(rid %d)", rid);
+			if (verbose) {
+				printf("Can't get %s report %d (%d bytes). "
+				    "Will be initialized with zeros.\n",
+				    kind == hid_input ? "input" :
+				    kind == hid_output ? "output" : "feature",
+				    rid, dlen);
+			}
 		}
-		if (loop)
+		for (var = vars; var; var = var->next) {
+			if (rid != var->h.report_ID || kind != var->h.kind)
+				continue;
+			hid_set_data(dbuf, &var->h, var->val);
+		}
+		if (verbose) {
+			printf("Setting %s report %d (%d bytes):",
+			    kind == hid_output ? "output" :
+			    kind == hid_feature ? "feature" : "input",
+			    rid, dlen);
+			for (i = 0; i < dlen; i++)
+				printf(" %02x", dbuf[i]);
 			printf("\n");
-	} while (loop);
-	free(dbuf);
+		}
+		r = hid_set_report(f, kind, dbuf, dlen);
+		if (r != 0)
+			warn("hid_set_report(rid %d)", rid);
+		free(dbuf);
+	    }
+	}
 }
 
 int
 main(int argc, char **argv)
 {
-	int f;
 	report_desc_t r;
-	char devnam[100], *dev = 0;
+	char *table = 0;
+	char devnam[100], *dev = NULL;
+	int f;
+	int all = 0;
 	int ch;
 	int repdump = 0;
 	int loop = 0;
-	char *table = 0;
 
-	while ((ch = getopt(argc, argv, "af:lnrt:vx")) != -1) {
+	while ((ch = getopt(argc, argv, "af:lnrt:vwxz")) != -1) {
 		switch(ch) {
 		case 'a':
 			all++;
@@ -310,9 +474,15 @@ main(int argc, char **argv)
 		case 'v':
 			verbose++;
 			break;
+		case 'w':
+			wflag = 1;
+			break;
 		case 'x':
 			hexdump = 1;
 			break;
+		case 'z':
+			zflag = 1;
+			break;
 		case '?':
 		default:
 			usage();
@@ -320,12 +490,10 @@ main(int argc, char **argv)
 	}
 	argc -= optind;
 	argv += optind;
-	if (dev == 0)
+	if (dev == NULL)
 		usage();
-	names = argv;
-	nnames = argc;
 
-	if (nnames == 0 && !all && !repdump)
+	if (argc == 0 && !all && !repdump)
 		usage();
 
 	if (dev[0] != '/') {
@@ -350,8 +518,13 @@ main(int argc, char **argv)
 		printf("Report descriptor:\n");
 		dumpitems(r);
 	}
-	if (nnames != 0 || all)
-		dumpdata(f, r, loop);
+	if (argc != 0 || all) {
+		parceargs(r, all, argc, argv);
+		if (wflag)
+			writedata(f, r);
+		else
+			dumpdata(f, r, loop);
+	}
 
 	hid_dispose_report_desc(r);
 	exit(0);
diff --git a/usr.bin/usbhidctl/usbhidctl.1 b/usr.bin/usbhidctl/usbhidctl.1
index 37c7c7a2fdaf..56811890782a 100644
--- a/usr.bin/usbhidctl/usbhidctl.1
+++ b/usr.bin/usbhidctl/usbhidctl.1
@@ -28,7 +28,7 @@
 .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 .\" POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd November 23, 2006
+.Dd August 01, 2011
 .Dt USBHIDCTL 1
 .Os
 .Sh NAME
@@ -36,27 +36,51 @@
 .Nd manipulate USB HID devices
 .Sh SYNOPSIS
 .Nm
-.Op Fl a
 .Fl f Ar device
-.Op Fl l
-.Op Fl n
-.Op Fl r
 .Op Fl t Ar table
 .Op Fl v
 .Op Fl x
-.Op Ar item ...
+.Fl r
+.Nm
+.Fl f Ar device
+.Op Fl t Ar table
+.Op Fl l
+.Op Fl v
+.Op Fl x
+.Fl a
+.Nm
+.Fl f Ar device
+.Op Fl t Ar table
+.Op Fl l
+.Op Fl n
+.Op Fl v
+.Op Fl x
+.Ar item ...
+.Nm
+.Fl f Ar device
+.Op Fl t Ar table
+.Op Fl v
+.Op Fl z
+.Fl w
+.Ar item=value ...
 .Sh DESCRIPTION
 The
 .Nm
-utility can be used to dump the state of a USB HID (Human Interface Device).
+utility can be used to dump and modify the state of a USB HID (Human
+Interface Device).
 Each named
 .Ar item
 is printed.
+If the
+.Fl w
+flag is specified
+.Nm
+attempts to set the specified items to the given values.
 .Pp
 The options are as follows:
 .Bl -tag -width Ds
 .It Fl a
-Show all items.
+Show all items and their current values if device returns.
 .It Fl f Ar device
 Specify a path name for the device to operate on.
 .It Fl l
@@ -69,9 +93,47 @@ Dump the report descriptor.
 Specify a path name for the HID usage table file.
 .It Fl v
 Be verbose.
+.It Fl w
+Change item values.
+Only 'output' and 'feature' kinds can be set with this option.
 .It Fl x
 Dump data in hexadecimal as well as decimal.
+.It Fl z
+Reset reports to zero before processing
+.Fl w
+arguments. If not specified, current values will be requested from device.
 .El
+.Sh SYNTAX
+.Nm
+compares the names of items specified on the command line against the human
+interface items reported by the USB device.
+Each human interface item is mapped from its native form to a human readable
+name, using the HID usage table file.
+Command line items are compared with the generated item names,
+and the USB HID device is operated on when a match is found.
+.Pp
+Each human interface item is named by the
+.Qq page
+it appears in, the
+.Qq usage
+within that page, and the list of
+.Qq collections
+containing the item.
+Each collection in turn is also identified by page, and
+the usage within that page.
+.Pp
+On the
+.Nm
+command line the page name is separated from the usage name with the character
+.Sq Cm \&: .
+The collections are separated by the character
+.Sq Cm \&. .
+.Pp
+Some devices give the same name to more than one item.
+.Nm
+supports isolating each item by appending a
+.Sq Cm \&# .
+character and a decimal item instance number, starting at zero.
 .Sh FILES
 .Pa /usr/share/misc/usb_hid_usages
 The default HID usage table.
@@ -84,7 +146,3 @@ The
 .Nm
 command appeared in
 .Nx 1.4 .
-.Sh BUGS
-The
-.Nm
-utility cannot show nor set output and feature items.
diff --git a/usr.bin/xzdec/Makefile b/usr.bin/xzdec/Makefile
index 6f38a657c4e5..daef296f250b 100644
--- a/usr.bin/xzdec/Makefile
+++ b/usr.bin/xzdec/Makefile
@@ -18,8 +18,6 @@ SRCS=	xzdec.c
 SRCS+=	tuklib_progname.c \
 	tuklib_exit.c
 
-WARNS?=	3
-
 CFLAGS+=	-DHAVE_CONFIG_H \
 		-I${LZMALIBDIR} \
 		-I${XZDIR}/common
diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index fa1936c48c22..2063ea6fcdd1 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -290,8 +290,6 @@ SUBDIR+=	praliases
 SUBDIR+=	sendmail
 .endif
 
-SUBDIR+=	sysinstall
-
 .if ${MK_TOOLCHAIN} != "no"
 SUBDIR+=	config
 SUBDIR+=	crunch
diff --git a/usr.sbin/boot0cfg/boot0cfg.c b/usr.sbin/boot0cfg/boot0cfg.c
index cd3bfe284e29..37ca140513a6 100644
--- a/usr.sbin/boot0cfg/boot0cfg.c
+++ b/usr.sbin/boot0cfg/boot0cfg.c
@@ -378,7 +378,7 @@ write_mbr(const char *fname, int flags, u_int8_t *mbr, int mbr_size)
     }
     grq = gctl_get_handle();
     gctl_ro_param(grq, "class", -1, "PART");
-    gctl_ro_param(grq, "geom", -1, pname);
+    gctl_ro_param(grq, "arg0", -1, pname);
     gctl_ro_param(grq, "verb", -1, "bootcode");
     gctl_ro_param(grq, "bootcode", mbr_size, mbr);
     gctl_ro_param(grq, "flags", -1, "C");
diff --git a/usr.sbin/bsdinstall/partedit/diskeditor.c b/usr.sbin/bsdinstall/partedit/diskeditor.c
index 35b2b3f1fb3f..94c374b81125 100644
--- a/usr.sbin/bsdinstall/partedit/diskeditor.c
+++ b/usr.sbin/bsdinstall/partedit/diskeditor.c
@@ -44,8 +44,8 @@ print_partedit_item(WINDOW *partitions, struct partedit_item *items,
 
 	wattrset(partitions, selected ? item_selected_attr : item_attr);
 	wmove(partitions, y, MARGIN + items[item].indentation*2);
-	dlg_print_text(partitions, items[item].name, 8, &attr);
-	wmove(partitions, y, 15);
+	dlg_print_text(partitions, items[item].name, 10, &attr);
+	wmove(partitions, y, 17);
 	wattrset(partitions, item_attr);
 
 	humanize_number(sizetext, 7, items[item].size, "B", HN_AUTOSCALE,
diff --git a/usr.sbin/bsdinstall/partedit/gpart_ops.c b/usr.sbin/bsdinstall/partedit/gpart_ops.c
index b55caf4136e1..52f145eecaeb 100644
--- a/usr.sbin/bsdinstall/partedit/gpart_ops.c
+++ b/usr.sbin/bsdinstall/partedit/gpart_ops.c
@@ -365,38 +365,42 @@ gpart_partcode(struct gprovider *pp)
 }
 
 void
-gpart_destroy(struct ggeom *lg_geom, int force)
+gpart_destroy(struct ggeom *lg_geom)
 {
-	struct gprovider *pp;
 	struct gctl_req *r;
+	struct gprovider *pp;
 	const char *errstr;
+	int force = 1;
 
-	/* Begin with the hosing: delete all partitions */
+	/* Delete all child metadata */
 	LIST_FOREACH(pp, &lg_geom->lg_provider, lg_provider)
 		gpart_delete(pp);
 
+	/* Revert any local changes to get this geom into a pristine state */
+	r = gctl_get_handle();
+	gctl_ro_param(r, "class", -1, "PART");
+	gctl_ro_param(r, "arg0", -1, lg_geom->lg_name);
+	gctl_ro_param(r, "verb", -1, "undo");
+	gctl_issue(r); /* Ignore errors -- these are non-fatal */
+	gctl_free(r);
+
 	/* Now destroy the geom itself */
 	r = gctl_get_handle();
 	gctl_ro_param(r, "class", -1, "PART");
 	gctl_ro_param(r, "arg0", -1, lg_geom->lg_name);
 	gctl_ro_param(r, "flags", -1, GPART_FLAGS);
+	gctl_ro_param(r, "force", sizeof(force), &force);
 	gctl_ro_param(r, "verb", -1, "destroy");
 	errstr = gctl_issue(r);
-	if (errstr != NULL && errstr[0] != '\0') 
-		gpart_show_error("Error", NULL, errstr);
-	gctl_free(r);
-
-	/* If asked, commit the change */
-	if (force) {
-		r = gctl_get_handle();
-		gctl_ro_param(r, "class", -1, "PART");
-		gctl_ro_param(r, "arg0", -1, lg_geom->lg_name);
-		gctl_ro_param(r, "verb", -1, "commit");
-		errstr = gctl_issue(r);
-		if (errstr != NULL && errstr[0] != '\0') 
+	if (errstr != NULL && errstr[0] != '\0') {
+		/*
+		 * Check if we reverted away the existence of the geom
+		 * altogether. Show all other errors to the user.
+		 */
+		if (strtol(errstr, NULL, 0) != EINVAL)
 			gpart_show_error("Error", NULL, errstr);
-		gctl_free(r);
 	}
+	gctl_free(r);
 
 	/* And any metadata associated with the partition scheme itself */
 	delete_part_metadata(lg_geom->lg_name);
@@ -439,28 +443,25 @@ gpart_edit(struct gprovider *pp)
 	geom = NULL;
 	LIST_FOREACH(cp, &pp->lg_consumers, lg_consumers)
 		if (strcmp(cp->lg_geom->lg_class->lg_name, "PART") == 0) {
-			char message[512];
-			/*
-			 * The PART object is a consumer, so the user wants to
-			 * edit the partition table. gpart doesn't really
-			 * support this, so we have to hose the whole table
-			 * first.
-			 */
-
-			sprintf(message, "Changing the partition scheme on "
-			    "this disk (%s) requires deleting all existing "
-			    "partitions on this drive. This will PERMANENTLY "
-			    "ERASE any data stored here. Are you sure you want "
-			    "to proceed?", cp->lg_geom->lg_name);
-			dialog_vars.defaultno = TRUE;
-			choice = dialog_yesno("Warning", message, 0, 0);
-			dialog_vars.defaultno = FALSE;
-
-			if (choice == 1) /* cancel */
+			/* Check for zombie geoms, treating them as blank */
+			scheme = NULL;
+			LIST_FOREACH(gc, &cp->lg_geom->lg_config, lg_config) {
+				if (strcmp(gc->lg_name, "scheme") == 0) {
+					scheme = gc->lg_val;
+					break;
+				}
+			}
+			if (scheme == NULL || strcmp(scheme, "(none)") == 0) {
+				gpart_partition(cp->lg_geom->lg_name, NULL);
 				return;
+			}
+
+			/* If this is a nested partition, edit as usual */
+			if (strcmp(pp->lg_geom->lg_class->lg_name, "PART") == 0)
+				break;
 
 			/* Destroy the geom and all sub-partitions */
-			gpart_destroy(cp->lg_geom, 0);
+			gpart_destroy(cp->lg_geom);
 
 			/* Now re-partition and return */
 			gpart_partition(cp->lg_geom->lg_name, NULL);
@@ -516,7 +517,7 @@ gpart_edit(struct gprovider *pp)
 	choice = dlg_form("Edit Partition", "", 0, 0, 0, nitems, items, &junk);
 
 	if (choice) /* Cancel pressed */
-		return;
+		goto endedit;
 
 	/* Check if the label has a / in it */
 	if (strchr(items[3].text, '/') != NULL) {
@@ -547,6 +548,13 @@ gpart_edit(struct gprovider *pp)
 	    items[2].text, (strcmp(oldtype, items[0].text) != 0) ?
 	    newfs : NULL);
 
+endedit:
+	if (strcmp(oldtype, items[0].text) != 0 && cp != NULL)
+		gpart_destroy(cp->lg_geom);
+	if (strcmp(oldtype, items[0].text) != 0 && strcmp(items[0].text,
+	    "freebsd") == 0)
+		gpart_partition(pp->lg_name, "BSD");
+
 	for (i = 0; i < (sizeof(items) / sizeof(items[0])); i++)
 		if (items[i].text_free)
 			free(items[i].text);
@@ -739,7 +747,7 @@ gpart_create(struct gprovider *pp, char *default_type, char *default_size,
 	struct gconsumer *cp;
 	struct ggeom *geom;
 	const char *errstr, *scheme;
-	char sizestr[32], startstr[32], output[64];
+	char sizestr[32], startstr[32], output[64], *newpartname;
 	char newfs[64], options_fstype[64];
 	intmax_t maxsize, size, sector, firstfree, stripe;
 	uint64_t bytes;
@@ -862,7 +870,7 @@ gpart_create(struct gprovider *pp, char *default_type, char *default_size,
 	 * If the user changed the fs type after specifying options, undo
 	 * their choices in favor of the new filesystem's defaults.
 	 */
-	if (strcmp(options_fstype, items[0].name) != 0) {
+	if (strcmp(options_fstype, items[0].text) != 0) {
 		strncpy(options_fstype, items[0].text, sizeof(options_fstype));
 		newfs_command(options_fstype, newfs, 1);
 	}
@@ -901,6 +909,19 @@ gpart_create(struct gprovider *pp, char *default_type, char *default_size,
 			goto addpartform;
 	}
 
+	/*
+	 * Error if this scheme needs nested partitions, this is one, and
+	 * a mountpoint was set.
+	 */
+	if (strcmp(items[0].text, "freebsd") == 0 &&
+	    strlen(items[2].text) > 0) {
+		dialog_msgbox("Error", "Partitions of type \"freebsd\" are "
+		    "nested BSD-type partition schemes and cannot have "
+		    "mountpoints. After creating one, select it and press "
+		    "Create again to add the actual file systems.", 0, 0, TRUE);
+		goto addpartform;
+	}
+
 	/* If this is the root partition, check that this scheme is bootable */
 	if (strcmp(items[2].text, "/") == 0 && !is_scheme_bootable(scheme)) {
 		char message[512];
@@ -918,7 +939,23 @@ gpart_create(struct gprovider *pp, char *default_type, char *default_size,
 	 * If this is the root partition, and we need a boot partition, ask
 	 * the user to add one.
 	 */
-	if (strcmp(items[2].text, "/") == 0 && bootpart_size(scheme) > 0) {
+
+	/* Check for existing freebsd-boot partition */
+	LIST_FOREACH(pp, &geom->lg_provider, lg_provider) {
+		struct partition_metadata *md;
+		md = get_part_metadata(pp->lg_name, 0);
+		if (md == NULL || !md->bootcode)
+			continue;
+		LIST_FOREACH(gc, &pp->lg_config, lg_config)
+			if (strcmp(gc->lg_name, "type") == 0)
+				break;
+		if (gc != NULL && strcmp(gc->lg_val, "freebsd-boot") == 0)
+			break;
+	}
+
+	/* If there isn't one, and we need one, ask */
+	if (strcmp(items[2].text, "/") == 0 && bootpart_size(scheme) > 0 &&
+	    pp == NULL) {
 		if (interactive)
 			choice = dialog_yesno("Boot Partition",
 			    "This partition scheme requires a boot partition "
@@ -970,29 +1007,43 @@ gpart_create(struct gprovider *pp, char *default_type, char *default_size,
 	if (items[3].text[0] != '\0')
 		gctl_ro_param(r, "label", -1, items[3].text);
 	gctl_rw_param(r, "output", sizeof(output), output);
-
 	errstr = gctl_issue(r);
 	if (errstr != NULL && errstr[0] != '\0') {
 		gpart_show_error("Error", NULL, errstr);
 		gctl_free(r);
 		goto addpartform;
 	}
+	newpartname = strtok(output, " ");
+	gctl_free(r);
+
+	/*
+	 * Try to destroy any geom that gpart picked up already here from
+	 * dirty blocks.
+	 */
+	r = gctl_get_handle();
+	gctl_ro_param(r, "class", -1, "PART");
+	gctl_ro_param(r, "arg0", -1, newpartname);
+	gctl_ro_param(r, "flags", -1, GPART_FLAGS);
+	junk = 1;
+	gctl_ro_param(r, "force", sizeof(junk), &junk);
+	gctl_ro_param(r, "verb", -1, "destroy");
+	gctl_issue(r); /* Error usually expected and non-fatal */
+	gctl_free(r);
 
 	if (strcmp(items[0].text, "freebsd-boot") == 0)
-		get_part_metadata(strtok(output, " "), 1)->bootcode = 1;
+		get_part_metadata(newpartname, 1)->bootcode = 1;
 	else if (strcmp(items[0].text, "freebsd") == 0)
-		gpart_partition(strtok(output, " "), "BSD");
+		gpart_partition(newpartname, "BSD");
 	else
-		set_default_part_metadata(strtok(output, " "), scheme,
+		set_default_part_metadata(newpartname, scheme,
 		    items[0].text, items[2].text, newfs);
 
 	for (i = 0; i < (sizeof(items) / sizeof(items[0])); i++)
 		if (items[i].text_free)
 			free(items[i].text);
-	gctl_free(r);
 
 	if (partname != NULL)
-		*partname = strdup(strtok(output, " "));
+		*partname = strdup(newpartname);
 }
 	
 void
@@ -1004,7 +1055,7 @@ gpart_delete(struct gprovider *pp)
 	struct gctl_req *r;
 	const char *errstr;
 	intmax_t idx;
-	int choice, is_partition;
+	int is_partition;
 
 	/* Is it a partition? */
 	is_partition = (strcmp(pp->lg_geom->lg_class->lg_name, "PART") == 0);
@@ -1017,32 +1068,21 @@ gpart_delete(struct gprovider *pp)
 			break;
 		}
 
-	/* Destroy all consumers */
+	/* If so, destroy all children */
 	if (geom != NULL) {
+		gpart_destroy(geom);
+
+		/* If this is a partition, revert it, so it can be deleted */
 		if (is_partition) {
-			char message[512];
-			/*
-			 * We have to actually really delete the sub-partition
-			 * tree so that the consumers will go away and the
-			 * partition can be deleted. Warn the user.
-			 */
-
-			sprintf(message, "Deleting this partition (%s) "
-			    "requires deleting all existing sub-partitions. "
-			    "This will PERMANENTLY ERASE any data stored here "
-			    "and CANNOT BE REVERTED. Are you sure you want to "
-			    "proceed?", cp->lg_geom->lg_name);
-			dialog_vars.defaultno = TRUE;
-			choice = dialog_yesno("Warning", message, 0, 0);
-			dialog_vars.defaultno = FALSE;
-
-			if (choice == 1) /* cancel */
-				return;
+			r = gctl_get_handle();
+			gctl_ro_param(r, "class", -1, "PART");
+			gctl_ro_param(r, "arg0", -1, geom->lg_name);
+			gctl_ro_param(r, "verb", -1, "undo");
+			gctl_issue(r); /* Ignore non-fatal errors */
+			gctl_free(r);
 		}
-
-		gpart_destroy(geom, is_partition);
 	}
- 
+
 	/*
 	 * If this is not a partition, see if that is a problem, complain if
 	 * necessary, and return always, since we need not do anything further,
@@ -1088,7 +1128,6 @@ gpart_revert_all(struct gmesh *mesh)
 	struct gconfig *gc;
 	struct ggeom *gp;
 	struct gctl_req *r;
-	const char *errstr;
 	const char *modified;
 
 	LIST_FOREACH(classp, &mesh->lg_class, lg_class) {
@@ -1119,9 +1158,7 @@ gpart_revert_all(struct gmesh *mesh)
 		gctl_ro_param(r, "arg0", -1, gp->lg_name);
 		gctl_ro_param(r, "verb", -1, "undo");
 
-		errstr = gctl_issue(r);
-		if (errstr != NULL && errstr[0] != '\0') 
-			gpart_show_error("Error", NULL, errstr);
+		gctl_issue(r);
 		gctl_free(r);
 	}
 }
diff --git a/usr.sbin/bsdinstall/partedit/part_wizard.c b/usr.sbin/bsdinstall/partedit/part_wizard.c
index 91dde0a14318..efd4a07c2562 100644
--- a/usr.sbin/bsdinstall/partedit/part_wizard.c
+++ b/usr.sbin/bsdinstall/partedit/part_wizard.c
@@ -254,7 +254,7 @@ wizard_partition(struct gmesh *mesh, const char *disk)
 		if (subchoice != 0)
 			goto query;
 
-		gpart_destroy(gpart, 1);
+		gpart_destroy(gpart);
 		gpart_partition(disk, default_scheme());
 		scheme = default_scheme();
 	}
@@ -267,7 +267,7 @@ wizard_partition(struct gmesh *mesh, const char *disk)
 			if (choice != 0)
 				goto query;
 
-			gpart_destroy(gpart, 1);
+			gpart_destroy(gpart);
 		}
 
 		gpart_partition(disk, default_scheme());
diff --git a/usr.sbin/bsdinstall/partedit/partedit.c b/usr.sbin/bsdinstall/partedit/partedit.c
index 8a89f5208e3d..cddf7901fc5d 100644
--- a/usr.sbin/bsdinstall/partedit/partedit.c
+++ b/usr.sbin/bsdinstall/partedit/partedit.c
@@ -70,7 +70,7 @@ main(int argc, const char **argv)
 {
 	struct partition_metadata *md;
 	const char *prompt;
-	struct partedit_item *items;
+	struct partedit_item *items = NULL;
 	struct gmesh mesh;
 	int i, op, nitems, nscroll;
 	int error;
@@ -99,12 +99,21 @@ main(int argc, const char **argv)
 
 	/* Show the part editor either immediately, or to confirm wizard */
 	while (1) {
-		error = geom_gettree(&mesh);
-		items = read_geom_mesh(&mesh, &nitems);
-		get_mount_points(items, nitems);
 		dlg_clear();
 		dlg_put_backtitle();
 
+		error = geom_gettree(&mesh);
+		if (error == 0)
+			items = read_geom_mesh(&mesh, &nitems);
+		if (error || items == NULL) {
+			dialog_msgbox("Error", "No disks found. If you need to "
+			    "install a kernel driver, choose Shell at the "
+			    "installation menu.", 0, 0, TRUE);
+			break;
+		}
+			
+		get_mount_points(items, nitems);
+
 		if (i >= nitems)
 			i = nitems - 1;
 		op = diskeditor_show("Partition Editor", prompt,
diff --git a/usr.sbin/bsdinstall/partedit/partedit.h b/usr.sbin/bsdinstall/partedit/partedit.h
index 5918421a1f78..bffb076338a8 100644
--- a/usr.sbin/bsdinstall/partedit/partedit.h
+++ b/usr.sbin/bsdinstall/partedit/partedit.h
@@ -58,7 +58,7 @@ int part_wizard(void);
 
 /* gpart operations */
 void gpart_delete(struct gprovider *pp);
-void gpart_destroy(struct ggeom *lg_geom, int force);
+void gpart_destroy(struct ggeom *lg_geom);
 void gpart_edit(struct gprovider *pp);
 void gpart_create(struct gprovider *pp, char *default_type, char *default_size,
     char *default_mountpoint, char **output, int interactive);
diff --git a/usr.sbin/bsdinstall/scripts/auto b/usr.sbin/bsdinstall/scripts/auto
index 667ced442a27..b5f8ef1ea930 100755
--- a/usr.sbin/bsdinstall/scripts/auto
+++ b/usr.sbin/bsdinstall/scripts/auto
@@ -35,7 +35,7 @@ error() {
 	    --no-label "Exit" --yes-label "Restart" --yesno \
 	    "An installation step has been aborted. Would you like to restart the installation or exit the installer?" 0 0
 	if [ $? -ne 0 ]; then
-		exit
+		exit 1
 	else
 		exec $0
 	fi
@@ -128,14 +128,12 @@ if [ ! -z "$FETCH_DISTRIBUTIONS" ]; then
 	if [ -d $BSDINSTALL_DISTDIR ]; then
 		DISTDIR_IS_UNIONFS=1
 		mount_nullfs -o union "$BSDINSTALL_FETCHDEST" "$BSDINSTALL_DISTDIR"
-	fi
-
-	# Otherwise, fetch everything
-	if [ $? -ne 0 ]; then
-		export DISTRIBUTIONS="$ALL_DISTRIBUTIONS"
+	else
+		export DISTRIBUTIONS="MANIFEST $ALL_DISTRIBUTIONS"
 		export BSDINSTALL_DISTDIR="$BSDINSTALL_FETCHDEST"
 	fi
 		
+	export FTP_PASSIVE_MODE=YES
 	bsdinstall distfetch || error
 	export DISTRIBUTIONS="$ALL_DISTRIBUTIONS"
 fi
@@ -160,6 +158,7 @@ finalconfig() {
 	REVISIT=$(dialog --backtitle "FreeBSD Installer" \
 	    --title "Final Configuration" --no-cancel --menu \
 	    "Setup of your FreeBSD system is nearly complete. You can now modify your configuration choices or apply more complex changes using a shell." 0 0 0 \
+		"Exit" "Apply configuration and exit installer" \
 		"Add User" "Add a user to the system" \
 		"Root Password" "Change root password" \
 		"Hostname" "Set system hostname" \
@@ -167,8 +166,7 @@ finalconfig() {
 		"Services" "Set daemons to run on startup" \
 		"Time Zone" "Set system timezone" \
 		"Handbook" "Install FreeBSD Handbook (requires network)" \
-		"Shell" "Open a shell in the new system" \
-		"Exit" "Apply configuration and exit installer" 2>&1 1>&3)
+		"Shell" "Open a shell in the new system" 2>&1 1>&3)
 	exec 3>&-
 
 	case "$REVISIT" in
diff --git a/usr.sbin/bsdinstall/scripts/keymap b/usr.sbin/bsdinstall/scripts/keymap
index 5c7d12f124f2..97ae83346f31 100755
--- a/usr.sbin/bsdinstall/scripts/keymap
+++ b/usr.sbin/bsdinstall/scripts/keymap
@@ -28,6 +28,8 @@
 
 kbdcontrol -d >/dev/null 2>&1
 if [ $? -eq 0 ]; then
+	dialog --backtitle "FreeBSD Installer" --title "Keymap Selection" \
+	    --yesno "Would you like to set a non-default key mapping for your keyboard?" 0 0 || exit 0
 	exec 3>&1
 	kbdmap 2>&1 1>&3 | grep 'keymap=' > $BSDINSTALL_TMPETC/rc.conf.keymap
 fi
diff --git a/usr.sbin/bsdinstall/scripts/netconfig b/usr.sbin/bsdinstall/scripts/netconfig
index e0f390c27876..382ff2ce4734 100755
--- a/usr.sbin/bsdinstall/scripts/netconfig
+++ b/usr.sbin/bsdinstall/scripts/netconfig
@@ -49,6 +49,13 @@ for IF in `ifconfig -l`; do
 	DIALOG_TAGS="$DIALOG_TAGS $IF \"$DESC\""
 done
 
+if [ -z "$INTERFACES" ]; then
+	dialog --backtitle 'FreeBSD Installer' \
+	    --title 'Network Configuration Error' \
+	    --msgbox 'No network interfaces present to configure.' 0 0
+	exit 1
+fi
+
 exec 3>&1
 INTERFACE=`echo $DIALOG_TAGS | xargs dialog --backtitle 'FreeBSD Installer' --title 'Network Configuration' --menu 'Please select a network interface to configure:' 0 0 0 2>&1 1>&3`
 if [ $? -eq $DIALOG_CANCEL ]; then exit 1; fi
diff --git a/usr.sbin/bsdinstall/scripts/netconfig_ipv4 b/usr.sbin/bsdinstall/scripts/netconfig_ipv4
index ec359d707635..8c225d48b463 100755
--- a/usr.sbin/bsdinstall/scripts/netconfig_ipv4
+++ b/usr.sbin/bsdinstall/scripts/netconfig_ipv4
@@ -35,6 +35,7 @@
 
 INTERFACE=$1
 IFCONFIG_PREFIX="$2"
+test -z "$IFCONFIG_PREFIX" || IFCONFIG_PREFIX="$2 "
 case "${INTERFACE}" in
 "")	dialog --backtitle 'FreeBSD Installer' --title 'Network Configuration' \
 	    --msgbox 'No interface specified for IPv4 configuration.' 0 0
diff --git a/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_diskstorage_tbl.c b/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_diskstorage_tbl.c
index 546029091475..25ba56a84871 100644
--- a/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_diskstorage_tbl.c
+++ b/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_diskstorage_tbl.c
@@ -442,7 +442,7 @@ disk_OS_get_disks(void)
 			/*
 			 * not found there - insert it as immutable
 			 */
-			syslog(LOG_WARNING, "%s: device '%s' not in "
+			syslog(LOG_WARNING, "%s: adding device '%s' to "
 			    "device list", __func__, disk);
 
 			if ((entry = device_entry_create(disk, "", "")) == NULL)
@@ -476,7 +476,8 @@ disk_OS_get_disks(void)
 		disk_entry->media = DSM_UNKNOWN;
 		disk_entry->removable = SNMP_FALSE;
 
-		if (strncmp(disk_entry->dev_name, "da", 2) == 0) {
+		if (strncmp(disk_entry->dev_name, "da", 2) == 0 ||
+		    strncmp(disk_entry->dev_name, "ada", 3) == 0) {
 			disk_entry->media = DSM_HARDDISK;
 			disk_entry->removable = SNMP_FALSE;
 		} else if (strncmp(disk_entry->dev_name, "cd", 2) == 0) {
@@ -634,7 +635,7 @@ op_hrDiskStorageTable(struct snmp_context *ctx __unused,
 	  	value->v.integer = entry->media;
 	  	return (SNMP_ERR_NOERROR);
 
-	case LEAF_hrDiskStorageRemoveble:
+	case LEAF_hrDiskStorageRemovable:
 	  	value->v.integer = entry->removable;
 	  	return (SNMP_ERR_NOERROR);
 
diff --git a/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_tree.def b/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_tree.def
index e92d903f5728..83c187b48499 100644
--- a/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_tree.def
+++ b/usr.sbin/bsnmpd/modules/snmp_hostres/hostres_tree.def
@@ -149,7 +149,7 @@
             (1 hrDiskStorageEntry : INTEGER op_hrDiskStorageTable
               (1 hrDiskStorageAccess INTEGER GET)
               (2 hrDiskStorageMedia INTEGER GET)
-              (3 hrDiskStorageRemoveble INTEGER GET)
+              (3 hrDiskStorageRemovable INTEGER GET)
               (4 hrDiskStorageCapacity INTEGER GET)
             )
           )
diff --git a/usr.sbin/burncd/burncd.8 b/usr.sbin/burncd/burncd.8
index e2996ba35a4e..64387c3e925e 100644
--- a/usr.sbin/burncd/burncd.8
+++ b/usr.sbin/burncd/burncd.8
@@ -27,12 +27,19 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 21, 2009
+.Dd October 9, 2011
 .Dt BURNCD 8
 .Os
 .Sh NAME
 .Nm burncd
 .Nd control the ATAPI CD-R/RW driver
+.Pp
+This utility was
+.Em deprecated
+in
+.Fx 9.0 .
+See
+.Sx NOTES .
 .Sh SYNOPSIS
 .Nm
 .Op Fl deFlmnpqtv
@@ -211,6 +218,10 @@ The
 .Nm
 utility appeared in
 .Fx 4.0 .
+.Pp
+.Nm
+was deprecated in
+.Fx 9.0 .
 .Sh AUTHORS
 The
 .Nm
@@ -220,3 +231,19 @@ Denmark
 .Aq sos@FreeBSD.org .
 .Sh BUGS
 Probably, please report when found.
+.Sh NOTES
+When
+.Bd -ragged -offset indent
+.Cd "options ATA_CAM"
+.Ed
+.Pp
+is compiled into the kernel, then
+.Xr cdrecord 1 ,
+available in the
+.Fx
+Ports Collection as part of the
+.Pa sysutils/cdrtools
+port, must be used instead.
+Refer to:
+.Pp
+http://www.freebsd.org/doc/handbook/creating-cds.html#CDRECORD
diff --git a/usr.sbin/burncd/burncd.c b/usr.sbin/burncd/burncd.c
index 43a0a09f1cbd..ab31997ae105 100644
--- a/usr.sbin/burncd/burncd.c
+++ b/usr.sbin/burncd/burncd.c
@@ -82,6 +82,13 @@ main(int argc, char **argv)
 	int block_size = 0, block_type = 0, cdopen = 0, dvdrw = 0;
 	const char *dev, *env_speed;
 
+	if (feature_present("ata_cam")) {
+		errx(1, "\nATA_CAM option is enabled in kernel.\n"
+		    "Install the sysutils/cdrtools port and use cdrecord instead.\n\n"
+		    "Please refer to:\n"
+		    "http://www.freebsd.org/doc/handbook/creating-cds.html#CDRECORD");
+	}
+
 	if ((dev = getenv("CDROM")) == NULL)
 		dev = "/dev/acd0";
 
diff --git a/usr.sbin/freebsd-update/freebsd-update.sh b/usr.sbin/freebsd-update/freebsd-update.sh
index 8dca87d51a80..ea4099a0a688 100644
--- a/usr.sbin/freebsd-update/freebsd-update.sh
+++ b/usr.sbin/freebsd-update/freebsd-update.sh
@@ -1200,7 +1200,7 @@ fetch_metadata_sanity () {
 	# Some aliases to save space later: ${P} is a character which can
 	# appear in a path; ${M} is the four numeric metadata fields; and
 	# ${H} is a sha256 hash.
-	P="[-+./:=_[[:alnum:]]"
+	P="[-+./:=%@_[[:alnum:]]"
 	M="[0-9]+\|[0-9]+\|[0-9]+\|[0-9]+"
 	H="[0-9a-f]{64}"
 
diff --git a/usr.sbin/kbdmap/kbdmap.c b/usr.sbin/kbdmap/kbdmap.c
index c9933891c12a..726652f0ae96 100644
--- a/usr.sbin/kbdmap/kbdmap.c
+++ b/usr.sbin/kbdmap/kbdmap.c
@@ -289,7 +289,7 @@ do_kbdcontrol(struct keymap *km)
 	if (!x11)
 		system(kbd_cmd);
 
-	fprintf(stderr, "keymap=%s\n", km->keym);
+	fprintf(stderr, "keymap=\"%s\"\n", km->keym);
 	free(kbd_cmd);
 }
 
diff --git a/usr.sbin/makefs/ffs.c b/usr.sbin/makefs/ffs.c
index f833cf545daf..26843b632a25 100644
--- a/usr.sbin/makefs/ffs.c
+++ b/usr.sbin/makefs/ffs.c
@@ -236,6 +236,9 @@ ffs_parse_opts(const char *option, fsinfo_t *fsopts)
 			goto leave_ffs_parse_opts;
 		}
 		rv = 1;
+	} else if (strcmp(var, "label") == 0) {
+		strlcpy(ffs_opts->label, val, sizeof(ffs_opts->label));
+		rv = 1;
 	} else
 		rv = set_option(ffs_options, var, val);
 
diff --git a/usr.sbin/makefs/ffs.h b/usr.sbin/makefs/ffs.h
index 42611a43350a..65cfbd041321 100644
--- a/usr.sbin/makefs/ffs.h
+++ b/usr.sbin/makefs/ffs.h
@@ -40,7 +40,11 @@
 #ifndef _FFS_H
 #define _FFS_H
 
+#include <ufs/ufs/dinode.h>
+#include <ufs/ffs/fs.h>
+
 typedef struct {
+	char	label[MAXVOLLEN];	/* volume name/label */
 	int	bsize;		/* block size */
 	int	fsize;		/* fragment size */
 	int	cpg;		/* cylinders per group */
diff --git a/usr.sbin/makefs/ffs/mkfs.c b/usr.sbin/makefs/ffs/mkfs.c
index 924ab6cbd661..f99ecab503ab 100644
--- a/usr.sbin/makefs/ffs/mkfs.c
+++ b/usr.sbin/makefs/ffs/mkfs.c
@@ -139,7 +139,9 @@ ffs_mkfs(const char *fsys, const fsinfo_t *fsopts)
 	avgfpdir =      ffs_opts->avgfpdir;
 	bbsize =        BBSIZE;
 	sbsize =        SBLOCKSIZE;
-	
+
+	strlcpy(sblock.fs_volname, ffs_opts->label, sizeof(sblock.fs_volname));
+
 	if (Oflag == 0) {
 		sblock.fs_old_inodefmt = FS_42INODEFMT;
 		sblock.fs_maxsymlinklen = 0;
diff --git a/usr.sbin/makefs/makefs.8 b/usr.sbin/makefs/makefs.8
index 9fe5c01c2436..4034b33092e2 100644
--- a/usr.sbin/makefs/makefs.8
+++ b/usr.sbin/makefs/makefs.8
@@ -134,8 +134,7 @@ and
 .Sy uname
 or
 .Sy uid ,
-.Sy device
-(in the case of block or character devices), and
+and
 .Sy link
 (in the case of symbolic links).
 If
@@ -242,6 +241,8 @@ Block size.
 Bytes per inode.
 .It Sy fsize
 Fragment size.
+.It Sy label
+Label name of the image.
 .It Sy maxbpg
 Maximum blocks per file in a cylinder group.
 .It Sy minfree
diff --git a/usr.sbin/mfiutil/mfi_show.c b/usr.sbin/mfiutil/mfi_show.c
index 9d9f7a11c57c..4f83b52e3100 100644
--- a/usr.sbin/mfiutil/mfi_show.c
+++ b/usr.sbin/mfiutil/mfi_show.c
@@ -141,7 +141,7 @@ show_battery(int ac, char **av)
 	struct mfi_bbu_design_info design;
 	struct mfi_bbu_status stat;
 	uint8_t status;
-	int comma, error, fd;
+	int comma, error, fd, show_capacity;
 
 	if (ac != 1) {
 		warnx("show battery: extra arguments");
@@ -157,16 +157,17 @@ show_battery(int ac, char **av)
 
 	if (mfi_dcmd_command(fd, MFI_DCMD_BBU_GET_CAPACITY_INFO, &cap,
 	    sizeof(cap), NULL, 0, &status) < 0) {
-		if (status == MFI_STAT_NO_HW_PRESENT) {
-			printf("mfi%d: No battery present\n", mfi_unit);
-			close(fd);
-			return (0);
-		}
 		error = errno;
 		warn("Failed to get capacity info");
 		close(fd);
 		return (error);
 	}
+	if (status == MFI_STAT_NO_HW_PRESENT) {
+		printf("mfi%d: No battery present\n", mfi_unit);
+		close(fd);
+		return (0);
+	}
+	show_capacity = (status == MFI_STAT_OK);
 
 	if (mfi_dcmd_command(fd, MFI_DCMD_BBU_GET_DESIGN_INFO, &design,
 	    sizeof(design), NULL, 0, NULL) < 0) {
@@ -192,10 +193,14 @@ show_battery(int ac, char **av)
 	printf("                Model: %s\n", design.device_name);
 	printf("            Chemistry: %s\n", design.device_chemistry);
 	printf("      Design Capacity: %d mAh\n", design.design_capacity);
-	printf(" Full Charge Capacity: %d mAh\n", cap.full_charge_capacity);
-	printf("     Current Capacity: %d mAh\n", cap.remaining_capacity);
-	printf("        Charge Cycles: %d\n", cap.cycle_count);
-	printf("       Current Charge: %d%%\n", cap.relative_charge);
+	if (show_capacity) {
+		printf(" Full Charge Capacity: %d mAh\n",
+		    cap.full_charge_capacity);
+		printf("     Current Capacity: %d mAh\n",
+		    cap.remaining_capacity);
+		printf("        Charge Cycles: %d\n", cap.cycle_count);
+		printf("       Current Charge: %d%%\n", cap.relative_charge);
+	}
 	printf("       Design Voltage: %d mV\n", design.design_voltage);
 	printf("      Current Voltage: %d mV\n", stat.voltage);
 	printf("          Temperature: %d C\n", stat.temperature);
diff --git a/usr.sbin/pc-sysinstall/backend-partmanager/create-part.sh b/usr.sbin/pc-sysinstall/backend-partmanager/create-part.sh
index ccb8d19ec273..d0343b4dcb1d 100755
--- a/usr.sbin/pc-sysinstall/backend-partmanager/create-part.sh
+++ b/usr.sbin/pc-sysinstall/backend-partmanager/create-part.sh
@@ -90,7 +90,7 @@ if [ $? -eq 0 -a "${SLICENUM}" = "1" ] ; then
 fi
 
 # If we have a starting block, use it
-if [ -z "$STARTBLOCK" ] ; then
+if [ -n "$STARTBLOCK" ] ; then
   sBLOCK="-b $STARTBLOCK"
 fi
 
diff --git a/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh b/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh
index e819d740dd28..5b3069558e39 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh
@@ -45,7 +45,6 @@ check_for_enc_pass()
 };
 
 # On check on the disk-label line if we have any extra vars for this device
-# Only enabled for ZFS devices now, may add other xtra options in future for other FS's
 get_fs_line_xvars()
 {
   ACTIVEDEV="${1}"
@@ -76,6 +75,15 @@ get_fs_line_xvars()
       return
     fi # End of ZFS block
 
+    # See if we are looking for UFS specific newfs options
+    echo $LINE | grep -q '^UFS' 2>/dev/null
+    if [ $? -eq 0 ] ; then
+      FSVARS="`echo $LINE | cut -d '(' -f 2- | cut -d ')' -f 1 | xargs`"
+      VAR="${FSVARS}"
+      export VAR
+      return
+    fi
+
   fi # End of xtra-options block
 
   # If we got here, set VAR to empty and export
@@ -278,7 +286,7 @@ setup_gpart_partitions()
 
       # Check if using zfs mirror
       echo ${XTRAOPTS} | grep -q "mirror" 2>/dev/null
-      if [ $? -eq 0 ] ; then
+      if [ $? -eq 0 -a "$FS" = "ZFS" ] ; then
         if [ "${_pType}" = "gpt" ] ; then
        	  XTRAOPTS=$(setup_zfs_mirror_parts "$XTRAOPTS" "${_pDisk}p${CURPART}")
         else
diff --git a/usr.sbin/pc-sysinstall/backend/functions-disk.sh b/usr.sbin/pc-sysinstall/backend/functions-disk.sh
index c63f7afd7ecf..73c3eb4c179f 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-disk.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-disk.sh
@@ -554,7 +554,7 @@ init_gmirror()
     local _mDisk=$3
 
     # Create this mirror device
-    rc_halt "gmirror label -vb ${_mBal} gm${_mNum} /dev/${_mDisk}"
+    rc_halt "gmirror label -vb ${_mBal} gm${_mNum} ${_mDisk}"
 
     sleep 3
 
diff --git a/usr.sbin/pc-sysinstall/backend/functions-extractimage.sh b/usr.sbin/pc-sysinstall/backend/functions-extractimage.sh
index 75b5a410bc72..5d32466e6a54 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-extractimage.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-extractimage.sh
@@ -55,6 +55,10 @@ start_extract_uzip_tar()
 
   case ${PACKAGETYPE} in
     uzip)
+      if ! kldstat -v | grep -q "geom_uzip" ; then
+	exit_err "Kernel module geom_uzip not loaded"
+      fi
+
 	  # Start by mounting the uzip image
       MDDEVICE=`mdconfig -a -t vnode -o readonly -f ${INSFILE}`
       mkdir -p ${FSMNT}.uzip
@@ -435,6 +439,16 @@ init_extraction()
 
     rsync) start_rsync_copy ;;
     image) start_image_install ;;
+    local)
+      get_value_from_cfg localPath
+      if [ -z "$VAL" ]
+      then
+        exit_err "Install medium was set to local, but no localPath was provided!"
+      fi
+      LOCALPATH=$VAL
+      INSFILE="${LOCALPATH}/${INSFILE}" ; export INSFILE
+      start_extract_uzip_tar
+      ;;
     *) exit_err "ERROR: Unknown install medium" ;;
   esac
 
diff --git a/usr.sbin/pc-sysinstall/backend/functions-localize.sh b/usr.sbin/pc-sysinstall/backend/functions-localize.sh
index 38cda791c2c6..7ee898cb3f1b 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-localize.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-localize.sh
@@ -509,7 +509,7 @@ run_localize()
 
 
     # Check if we need to set a timezone
-    echo $line | -q grep "^timeZone=" 2>/dev/null
+    echo $line | grep -q "^timeZone=" 2>/dev/null
     if [ $? -eq 0 ] ; then
       get_value_from_string "$line"
       set_timezone "$VAL"
diff --git a/usr.sbin/pc-sysinstall/backend/functions-newfs.sh b/usr.sbin/pc-sysinstall/backend/functions-newfs.sh
index 96ca49b900ed..d5edff10e3a0 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-newfs.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-newfs.sh
@@ -138,7 +138,7 @@ setup_filesystems()
       UFS)
         echo_log "NEWFS: ${PARTDEV} - ${PARTFS}"
         sleep 2
-        rc_halt "newfs ${PARTDEV}${EXT}"
+        rc_halt "newfs ${PARTXTRAOPTS} ${PARTDEV}${EXT}"
         sleep 2
         rc_halt "sync"
         rc_halt "glabel label ${PARTLABEL} ${PARTDEV}${EXT}"
@@ -154,7 +154,7 @@ setup_filesystems()
       UFS+S)
         echo_log "NEWFS: ${PARTDEV} - ${PARTFS}"
         sleep 2
-        rc_halt "newfs -U ${PARTDEV}${EXT}"
+        rc_halt "newfs ${PARTXTRAOPTS} -U ${PARTDEV}${EXT}"
         sleep 2
         rc_halt "sync"
         rc_halt "glabel label ${PARTLABEL} ${PARTDEV}${EXT}"
@@ -169,7 +169,7 @@ setup_filesystems()
       UFS+SUJ)
         echo_log "NEWFS: ${PARTDEV} - ${PARTFS}"
         sleep 2
-        rc_halt "newfs -U ${PARTDEV}${EXT}"
+        rc_halt "newfs ${PARTXTRAOPTS} -U ${PARTDEV}${EXT}"
         sleep 2
         rc_halt "sync"
         rc_halt "tunefs -j enable ${PARTDEV}${EXT}"
@@ -192,7 +192,7 @@ setup_filesystems()
         sleep 2
         rc_halt "gjournal label -f ${PARTDEV}${EXT}"
         sleep 2
-        rc_halt "newfs -O 2 -J ${PARTDEV}${EXT}.journal"
+        rc_halt "newfs ${PARTXTRAOPTS} -O 2 -J ${PARTDEV}${EXT}.journal"
         sleep 2
         rc_halt "sync"
         rc_halt "glabel label ${PARTLABEL} ${PARTDEV}${EXT}.journal"
diff --git a/usr.sbin/pc-sysinstall/backend/functions-parse.sh b/usr.sbin/pc-sysinstall/backend/functions-parse.sh
index 4f96ca338d95..8b9d4180f61b 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-parse.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-parse.sh
@@ -33,7 +33,7 @@ get_value_from_string()
 {
   if [ -n "${1}" ]
   then
-    export VAL="`echo ${1} | cut -d '=' -f 2`"
+    export VAL="`echo ${1} | cut -d '=' -f 2-15`"
   else
     echo "Error: Did we forgot to supply a string to parse?"
     exit 1
@@ -45,7 +45,7 @@ get_value_from_cfg_with_spaces()
 {
   if [ -n "${1}" ]
   then
-    export VAL=`grep "^${1}=" ${CFGF} | head -n 1 | cut -d '=' -f 2`
+    export VAL=`grep "^${1}=" ${CFGF} | head -n 1 | cut -d '=' -f 2-15`
   else
     exit_err "Error: Did we forgot to supply a setting to grab?"
   fi
@@ -57,7 +57,7 @@ get_value_from_cfg()
 {
   if [ -n "${1}" ]
   then
-    export VAL=`grep "^${1}=" ${CFGF} | head -n 1 | cut -d '=' -f 2 | tr -d ' '`
+    export VAL=`grep "^${1}=" ${CFGF} | head -n 1 | cut -d '=' -f 2-15 | tr -d ' '`
   else
     exit_err "Error: Did we forgot to supply a setting to grab?"
   fi
diff --git a/usr.sbin/pc-sysinstall/backend/functions-unmount.sh b/usr.sbin/pc-sysinstall/backend/functions-unmount.sh
index ce0db9b1f4e1..23630f7674a4 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-unmount.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-unmount.sh
@@ -176,35 +176,35 @@ unmount_all_filesystems_failure()
       then
         if [ "${PARTENC}" = "ON" ]
         then
-          rc_nohalt "swapoff ${PARTDEV}.eli"
+          swapoff ${PARTDEV}.eli >/dev/null 2>/dev/null
         else
-          rc_nohalt "swapoff ${PARTDEV}"
+          swapoff ${PARTDEV} >/dev/null 2>/dev/null
         fi
       fi
 
       # Check if we've found "/" again, don't need to mount it twice
       if [ "$PARTMNT" != "/" -a "${PARTMNT}" != "none" -a "${PARTFS}" != "ZFS" ]
       then
-        rc_nohalt "umount -f ${PARTDEV}"
-        rc_nohalt "umount -f ${FSMNT}${PARTMNT}"
+        umount -f ${PARTDEV} >/dev/null 2>/dev/null
+        umount -f ${FSMNT}${PARTMNT} >/dev/null 2>/dev/null
       fi
     done
 
     # Last lets the /mnt partition
     #########################################################
-    rc_nohalt "umount -f ${FSMNT}"
+    umount -f ${FSMNT} >/dev/null 2>/dev/null
 
    fi
   else
     # We are doing a upgrade, try unmounting any of these filesystems
-    chroot ${FSMNT} /sbin/umount -a >>${LOGOUT} >>${LOGOUT}
-    umount -f ${FSMNT}/usr >>${LOGOUT} 2>>${LOGOUT}
-    umount -f ${FSMNT}/dev >>${LOGOUT} 2>>${LOGOUT}
-    umount -f ${FSMNT} >>${LOGOUT} 2>>${LOGOUT}
-    rc_nohalt "sh ${TMPDIR}/.upgrade-unmount"
+    chroot ${FSMNT} /sbin/umount -a >/dev/null 2>/dev/null
+    umount -f ${FSMNT}/usr >/dev/null 2>/dev/null
+    umount -f ${FSMNT}/dev >/dev/null 2>/dev/null
+    umount -f ${FSMNT} >/dev/null 2>/dev/null 
+    sh ${TMPDIR}/.upgrade-unmount >/dev/null 2>/dev/null
   fi
    
   # Unmount our CDMNT
-  rc_nohalt "umount ${CDMNT}"
+  umount ${CDMNT} >/dev/null 2>/dev/null
 
 };
diff --git a/usr.sbin/pc-sysinstall/backend/functions.sh b/usr.sbin/pc-sysinstall/backend/functions.sh
index 4987bf45c7f3..9a98ee2bf144 100755
--- a/usr.sbin/pc-sysinstall/backend/functions.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions.sh
@@ -98,10 +98,10 @@ strip_white_space()
 exit_err()
 {
   # Echo the message for the users benefit
-  echo "$1"
+  echo "EXITERROR: $1"
 
   # Save this error to the log file
-  echo "${1}" >>$LOGOUT
+  echo "EXITERROR: ${1}" >>$LOGOUT
 
   # Check if we need to unmount any file-systems after this failure
   unmount_all_filesystems_failure
@@ -446,12 +446,12 @@ install_fresh()
     # Now add any users
     setup_users
 
-    # Now run any commands specified
-    run_commands
-  
     # Do any last cleanup / setup before unmounting
     run_final_cleanup
 
+    # Now run any commands specified
+    run_commands
+
     # Unmount and finish up
     unmount_all_filesystems
   fi
diff --git a/usr.sbin/pc-sysinstall/backend/parseconfig.sh b/usr.sbin/pc-sysinstall/backend/parseconfig.sh
index eeb6ce0049a7..2e16751bd626 100755
--- a/usr.sbin/pc-sysinstall/backend/parseconfig.sh
+++ b/usr.sbin/pc-sysinstall/backend/parseconfig.sh
@@ -58,12 +58,8 @@ fi
 # Set our config file variable
 CFGF="$1"
 
-# Check the dirname of the provided CFGF and make sure its a full path
-DIR="`dirname ${CFGF}`"
-if [ "${DIR}" = "." ]
-then
-  CFGF="`pwd`/${CFGF}"
-fi
+# Resolve any relative pathing
+CFGF="`realpath ${CFGF}`"
 export CFGF
 
 # Start by doing a sanity check, which will catch any obvious mistakes in the config
@@ -72,7 +68,7 @@ file_sanity_check "installMode installType installMedium packageType"
 # We passed the Sanity check, lets grab some of the universal config settings and store them
 check_value installMode "fresh upgrade extract"
 check_value installType "PCBSD FreeBSD"
-check_value installMedium "dvd usb ftp rsync image"
+check_value installMedium "dvd usb ftp rsync image local"
 check_value packageType "uzip tar rsync split"
 if_check_value_exists partition "all s1 s2 s3 s4 free image"
 if_check_value_exists mirrorbal "load prefer round-robin split"
diff --git a/usr.sbin/pc-sysinstall/examples/README b/usr.sbin/pc-sysinstall/examples/README
index 2eb1e6f2973f..cc269a229787 100644
--- a/usr.sbin/pc-sysinstall/examples/README
+++ b/usr.sbin/pc-sysinstall/examples/README
@@ -184,7 +184,7 @@ the listing of any additional diskX= directives.
 The following settings specify the partitioning / mount points to setup
 on the target partition
 
-# disk0-part=UFS+S 500 / 
+# disk0-part=UFS+S 500 / (-n -o time)
 # disk0-part=SWAP 2000 none
 # disk0-part=UFS.eli 500 /usr
 # encpass=mypass
@@ -219,6 +219,12 @@ All sizes are expressed in MegaBytes
 Specifying a size 0 instructs pc-sysinstall to use the rest of the 
 available slice size, and should only be used for the last partition / mount
 
+When using "UFS" and its various types, it is possible to specify custom options
+for newfs using (). For examplei:
+disk0-part=UFS+SUJ 1000 / (-o time)
+In this case "-o time" would be passed to newfs when creating the "/" filesystem.
+
+
 When using "ZFS" specifically, it is possible to specify additional disks / partitions
 to include in the zpool. By using the syntax: (mirror: ad1,ad2) or (raidz: ad1,ad2), it is possible
 to include the disk "ad1" into the zpool for this partition, using the raidz / mirror methods.
diff --git a/usr.sbin/pkg_install/add/main.c b/usr.sbin/pkg_install/add/main.c
index f73312a626e5..b1eec7ab14b2 100644
--- a/usr.sbin/pkg_install/add/main.c
+++ b/usr.sbin/pkg_install/add/main.c
@@ -87,6 +87,7 @@ struct {
 	{ 800000, 800499, "/packages-8.0-release" },
 	{ 801000, 801499, "/packages-8.1-release" },
 	{ 802000, 802499, "/packages-8.2-release" },
+	{ 900000, 900499, "/packages-9.0-release" },
 	{ 300000, 399000, "/packages-3-stable" },
 	{ 400000, 499000, "/packages-4-stable" },
 	{ 502100, 502128, "/packages-5-current" },
@@ -94,7 +95,8 @@ struct {
 	{ 600100, 699000, "/packages-6-stable" },
 	{ 700100, 799000, "/packages-7-stable" },
 	{ 800500, 899000, "/packages-8-stable" },
-	{ 900000, 999000, "/packages-9-current" },
+	{ 900500, 999000, "/packages-9-stable" },
+	{ 1000000, 1099000, "/packages-10-current" },
 	{ 0, 9999999, "/packages-current" },
 	{ 0, 0, NULL }
 };
diff --git a/usr.sbin/pkg_install/delete/perform.c b/usr.sbin/pkg_install/delete/perform.c
index 2557dbcd105d..113df3119828 100644
--- a/usr.sbin/pkg_install/delete/perform.c
+++ b/usr.sbin/pkg_install/delete/perform.c
@@ -324,8 +324,8 @@ pkg_do(char *pkg)
      */
     if (delete_package(FALSE, CleanDirs, &Plist) == FAIL)
 	warnx(
-	"couldn't entirely delete package (perhaps the packing list is\n"
-	"incorrectly specified?)");
+	"couldn't entirely delete package `%s'\n"
+	"(perhaps the packing list is incorrectly specified?)", pkg);
 
     if (chdir(LogDir) == FAIL) {
  	warnx("unable to change directory to %s! deinstall failed", LogDir);
diff --git a/usr.sbin/pkg_install/info/show.c b/usr.sbin/pkg_install/info/show.c
index c65c312a5062..8b434be7cdf9 100644
--- a/usr.sbin/pkg_install/info/show.c
+++ b/usr.sbin/pkg_install/info/show.c
@@ -207,6 +207,14 @@ show_plist(const char *title, Package *plist, plist_t type, Boolean showall)
     }
 }
 
+static const char *
+elide_root(const char *dir)
+{
+    if (strcmp(dir, "/") == 0)
+	return "";
+    return dir;
+}
+
 /* Show all files in the packing list (except ignored ones) */
 void
 show_files(const char *title, Package *plist)
@@ -223,7 +231,7 @@ show_files(const char *title, Package *plist)
 	switch(p->type) {
 	case PLIST_FILE:
 	    if (!ign)
-		printf("%s/%s\n", dir, p->name);
+		printf("%s/%s\n", elide_root(dir), p->name);
 	    ign = FALSE;
 	    break;
 
@@ -270,7 +278,7 @@ show_size(const char *title, Package *plist)
 	switch (p->type) {
 	case PLIST_FILE:
 	    if (!ign) {
-		snprintf(tmp, FILENAME_MAX, "%s/%s", dir, p->name);
+		snprintf(tmp, FILENAME_MAX, "%s/%s", elide_root(dir), p->name);
 		if (!lstat(tmp, &sb)) {
 		    size += sb.st_size;
 		    if (Verbose)
@@ -328,7 +336,7 @@ show_cksum(const char *title, Package *plist)
 	    else
 		dir = p->name;
 	} else if (p->type == PLIST_FILE) {
-	    snprintf(tmp, FILENAME_MAX, "%s/%s", dir, p->name);
+	    snprintf(tmp, FILENAME_MAX, "%s/%s", elide_root(dir), p->name);
 	    if (!fexists(tmp))
 		warnx("%s doesn't exist", tmp);
 	    else if (p->next && p->next->type == PLIST_COMMENT &&
diff --git a/usr.sbin/pkg_install/lib/version.c b/usr.sbin/pkg_install/lib/version.c
index d9c4fe7e070c..0852ba12d362 100644
--- a/usr.sbin/pkg_install/lib/version.c
+++ b/usr.sbin/pkg_install/lib/version.c
@@ -66,7 +66,7 @@ split_version(const char *pkgname, const char **endname, unsigned long *epoch, u
     if (pkgname == NULL)
 	errx(2, "%s: Passed NULL pkgname.", __func__);
 
-    /* Look for the last '-' the the pkgname */
+    /* Look for the last '-' the pkgname */
     ch = strrchr(pkgname, '-');
     /* Cheat if we are just passed a version, not a valid package name */
     versionstr = ch ? ch + 1 : pkgname;
diff --git a/usr.sbin/pmcstat/pmcstat.c b/usr.sbin/pmcstat/pmcstat.c
index 31ef87770536..5d9db61952db 100644
--- a/usr.sbin/pmcstat/pmcstat.c
+++ b/usr.sbin/pmcstat/pmcstat.c
@@ -552,7 +552,7 @@ main(int argc, char **argv)
 	int hcpu, option, npmc, ncpu;
 	int c, check_driver_stats, current_cpu, current_sampling_count;
 	int do_callchain, do_descendants, do_logproccsw, do_logprocexit;
-	int do_print;
+	int do_print, do_read;
 	size_t dummy;
 	int graphdepth;
 	int pipefd[2], rfd;
@@ -1328,7 +1328,7 @@ main(int argc, char **argv)
 	 * are killed by a SIGINT.
 	 */
 	runstate = PMCSTAT_RUNNING;
-	do_print = 0;
+	do_print = do_read = 0;
 	do {
 		if ((c = kevent(pmcstat_kq, NULL, 0, &kev, 1, NULL)) <= 0) {
 			if (errno != EINTR)
@@ -1351,8 +1351,10 @@ main(int argc, char **argv)
 			    (args.pa_flags & FLAG_DO_TOP)) {
 				if (pmcstat_keypress_log())
 					runstate = pmcstat_close_log();
-			} else
+			} else {
+				do_read = 0;
 				runstate = pmcstat_process_log();
+			}
 			break;
 
 		case EVFILT_SIGNAL:
@@ -1377,9 +1379,6 @@ main(int argc, char **argv)
 				/* Kill the child process if we started it */
 				if (args.pa_flags & FLAG_HAS_COMMANDLINE)
 					pmcstat_kill_process();
-				/* Close the pipe to self, if present. */
-				if (args.pa_flags & FLAG_HAS_PIPE)
-					(void) close(pipefd[READPIPEFD]);
 				runstate = pmcstat_close_log();
 			} else if (kev.ident == SIGWINCH) {
 				if (ioctl(fileno(args.pa_printfile),
@@ -1394,12 +1393,15 @@ main(int argc, char **argv)
 			break;
 
 		case EVFILT_TIMER: /* print out counting PMCs */
+			if ((args.pa_flags & FLAG_DO_TOP) &&
+			     pmc_flush_logfile() != ENOBUFS)
+				do_read = 1;
 			do_print = 1;
 			break;
 
 		}
 
-		if (do_print) {
+		if (do_print && !do_read) {
 			if ((args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
 				pmcstat_print_pmcs();
 				if (runstate == PMCSTAT_FINISHED && /* final newline */
@@ -1420,7 +1422,7 @@ main(int argc, char **argv)
 
 	/* flush any pending log entries */
 	if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE))
-		pmc_flush_logfile();
+		pmc_close_logfile();
 
 	pmcstat_cleanup();
 
diff --git a/usr.sbin/pmcstat/pmcstat_log.c b/usr.sbin/pmcstat/pmcstat_log.c
index b7dc956d807b..6d0403f00585 100644
--- a/usr.sbin/pmcstat/pmcstat_log.c
+++ b/usr.sbin/pmcstat/pmcstat_log.c
@@ -1702,7 +1702,7 @@ pmcstat_close_log(void)
 	 * so keep the status to EXITING.
 	 */
 	if (args.pa_logfd != -1) {
-		if (pmc_flush_logfile() < 0)
+		if (pmc_close_logfile() < 0)
 			err(EX_OSERR, "ERROR: logging failed");
 	}
 
diff --git a/usr.sbin/portsnap/portsnap/portsnap.sh b/usr.sbin/portsnap/portsnap/portsnap.sh
index 60b20a0f8112..e715ce86984c 100644
--- a/usr.sbin/portsnap/portsnap/portsnap.sh
+++ b/usr.sbin/portsnap/portsnap/portsnap.sh
@@ -536,9 +536,9 @@ fetch_metadata() {
 	rm -f ${SNAPSHOTHASH} tINDEX.new
 
 	echo ${NDEBUG} "Fetching snapshot metadata... "
-	fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH}
+	fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \
 	    2>${QUIETREDIR} || return
-	if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then
+	if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then
 		echo "snapshot metadata corrupt."
 		return 1
 	fi
@@ -606,7 +606,7 @@ fetch_index_sanity() {
 # Verify a list of files
 fetch_snapshot_verify() {
 	while read F; do
-		if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then
+		if [ "`gunzip -c snap/${F} | ${SHA256} -q`" != ${F} ]; then
 			echo "snapshot corrupt."
 			return 1
 		fi
diff --git a/usr.sbin/pstat/pstat.c b/usr.sbin/pstat/pstat.c
index 54351667b27f..201784157329 100644
--- a/usr.sbin/pstat/pstat.c
+++ b/usr.sbin/pstat/pstat.c
@@ -345,7 +345,7 @@ ttyprt(struct xtty *xt)
 		errx(1, "struct xtty size mismatch");
 	if (usenumflag || xt->xt_dev == 0 ||
 	   (name = devname(xt->xt_dev, S_IFCHR)) == NULL)
-		printf("%5d,%4d ", major(xt->xt_dev), minor(xt->xt_dev));
+		printf("%#10jx ", (uintmax_t)xt->xt_dev);
 	else
 		printf("%10s ", name);
 	printf("%5zu %4zu %4zu %4zu %5zu %4zu %4zu %5u %5d %5d ",
diff --git a/usr.sbin/rtadvd/rtadvd.c b/usr.sbin/rtadvd/rtadvd.c
index 43e0e71ccfdb..092ffb67b43b 100644
--- a/usr.sbin/rtadvd/rtadvd.c
+++ b/usr.sbin/rtadvd/rtadvd.c
@@ -380,6 +380,21 @@ rtadvd_shutdown(void)
 		syslog(LOG_INFO,
 		    "waiting expiration of the all RA timers.");
 
+		TAILQ_FOREACH(ifi, &ifilist, ifi_next) {
+			/*
+			 * Ignore !IFF_UP interfaces in waiting for shutdown.
+			 */
+			if (!(ifi->ifi_flags & IFF_UP) &&
+			    ifi->ifi_ra_timer != NULL) {
+				ifi->ifi_state = IFI_STATE_UNCONFIGURED;
+				rtadvd_remove_timer(ifi->ifi_ra_timer);
+				ifi->ifi_ra_timer = NULL;
+				syslog(LOG_DEBUG, "<%s> %s(idx=%d) is down. "
+				    "Timer removed and marked as UNCONFIGURED.",
+				     __func__, ifi->ifi_ifname,
+				    ifi->ifi_ifindex);
+			}
+		}
 		TAILQ_FOREACH(ifi, &ifilist, ifi_next) {
 			if (ifi->ifi_ra_timer != NULL)
 				break;
diff --git a/usr.sbin/sysinstall/Makefile b/usr.sbin/sysinstall/Makefile
deleted file mode 100644
index 9eb09e20c4be..000000000000
--- a/usr.sbin/sysinstall/Makefile
+++ /dev/null
@@ -1,126 +0,0 @@
-# $FreeBSD$
-
-.if ${MACHINE_CPUARCH} != "ia64"
-_wizard=	wizard.c
-.endif
-
-PROG=	sysinstall
-MAN=	sysinstall.8
-SRCS=	anonFTP.c cdrom.c command.c config.c devices.c dhcp.c \
-	disks.c dispatch.c dist.c dmenu.c doc.c dos.c floppy.c \
-	ftp.c globals.c http.c index.c install.c installUpgrade.c keymap.c \
-	label.c main.c makedevs.c media.c menus.c misc.c modules.c \
-	mouse.c msg.c network.c nfs.c options.c package.c \
-	system.c tcpip.c termcap.c ttys.c ufs.c usb.c user.c \
-	variable.c ${_wizard} keymap.h countries.h
-
-CFLAGS+= -DUSE_GZIP=1 
-.if ${MACHINE} == "pc98"
-CFLAGS+= -DPC98
-.endif
-CFLAGS+= -I${.CURDIR}/../../gnu/lib/libodialog -I.
-
-WARNS?=  2	
-
-DPADD=	${LIBODIALOG} ${LIBNCURSES} ${LIBUTIL} ${LIBDISK} ${LIBFTPIO}
-LDADD=	-lodialog -lncurses -lutil -ldisk -lftpio
-
-CLEANFILES=	makedevs.c rtermcap
-CLEANFILES+=	keymap.tmp keymap.h countries.tmp countries.h
-
-.if exists(${.CURDIR}/../../share/termcap/termcap.src)
-RTERMCAP=	TERMCAP=${.CURDIR}/../../share/termcap/termcap.src ./rtermcap
-.else
-RTERMCAP=	./rtermcap
-.endif
-
-makedevs.c:	Makefile rtermcap
-	echo '#include <sys/types.h>' > makedevs.c
-	${RTERMCAP} ansi | \
-		file2c 'const char termcap_ansi[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25w | \
-		file2c 'const char termcap_cons25w[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25 | \
-		file2c 'const char termcap_cons25[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25-m | \
-		file2c 'const char termcap_cons25_m[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25r | \
-		file2c 'const char termcap_cons25r[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25r-m | \
-		file2c 'const char termcap_cons25r_m[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25l1 | \
-		file2c 'const char termcap_cons25l1[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} cons25l1-m | \
-		file2c 'const char termcap_cons25l1_m[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} vt100 | \
-		file2c 'const char termcap_vt100[] = {' ',0};' \
-		>> makedevs.c
-	${RTERMCAP} xterm | \
-		file2c 'const char termcap_xterm[] = {' ',0};' \
-		>> makedevs.c
-
-build-tools:	rtermcap
-
-rtermcap:	rtermcap.c
-	${CC} -o ${.TARGET} ${.ALLSRC} -ltermcap
-
-.if ${MACHINE} == "pc98"
-KEYMAPS= jp.pc98 jp.pc98.iso
-.else
-KEYMAPS= be.iso bg.bds.ctrlcaps bg.phonetic.ctrlcaps br275.iso \
-	ce.iso2 cs.latin2.qwertz danish.iso el.iso07 \
-	estonian.cp850 estonian.iso estonian.iso15 finnish.iso fr.iso \
-	german.iso gr.elot.acc gr.us101.acc  hr.iso hu.iso2.101keys \
-	it.iso icelandic.iso jp.106 latinamerican latinamerican.iso.acc \
-	norwegian.iso pl_PL.ISO8859-2 \
-	pt.iso ru.koi8-r si.iso sk.iso2 spanish.iso spanish.iso.acc swedish.iso \
-	swissfrench.iso \
-	swissgerman.iso ua.koi8-u ua.koi8-u.shift.alt uk.iso us.dvorak \
-	us.iso us.pc-ctrl us.unix
-.endif
-
-keymap.h:
-	rm -f keymap.tmp
-	for map in ${KEYMAPS} ; do \
-		KEYMAP_PATH=${.CURDIR}/../../share/syscons/keymaps \
-			kbdcontrol -L $$map | \
-			sed -e '/^static accentmap_t/,$$d' >> keymap.tmp ; \
-	done
-	echo "static struct keymapInfo keymapInfos[] = {" >> keymap.tmp
-	for map in ${KEYMAPS} ; do \
-		echo -n '	{ "'$$map'", ' >> keymap.tmp ; \
-		echo "&keymap_$$map }," | tr '[-.]' '_' >> keymap.tmp ; \
-	done
-	( echo "	{ NULL, NULL }"; echo "};" ; echo "" ) >> keymap.tmp
-	mv keymap.tmp keymap.h
-
-countries.h: ${.CURDIR}/../../share/misc/iso3166
-	rm -f countries.tmp
-	awk 'BEGIN { \
-	    FS = "\t"; \
-	    num = 1; \
-	    print "DMenu MenuCountry = {"; \
-	    print "    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,"; \
-	    print "    \"Country Selection\","; \
-	    print "    \"Please choose a country, region, or group.\\n\""; \
-	    print "    \"Select an item using [SPACE] or [ENTER].\","; \
-	    printf "    NULL,\n    NULL,\n    { "; \
-	} \
-	/^[[:space:]]*#/ {next;} \
-	{if (num > 1) {printf "      ";} \
-	    print "{ \"" num "\", \"" $$4 "\"" \
-	    ", dmenuVarCheck, dmenuSetCountryVariable" \
-	    ", NULL, VAR_COUNTRY \"=" tolower($$1) "\" },"; \
-	    ++num;} \
-	END {print "      { NULL } }\n};\n";}' < ${.ALLSRC} > countries.tmp
-	mv countries.tmp ${.TARGET}
-
-.include <bsd.prog.mk>
diff --git a/usr.sbin/sysinstall/anonFTP.c b/usr.sbin/sysinstall/anonFTP.c
deleted file mode 100644
index 2eaf636a6a85..000000000000
--- a/usr.sbin/sysinstall/anonFTP.c
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Coranth Gryphon.  All rights reserved.
- * Copyright (c) 1996
- *	Jordan K. Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR THEIR PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/param.h>
-#include <pwd.h>
-#include <grp.h>
-
-/* This doesn't change until FTP itself changes */
-
-#define FTP_NAME	"ftp"
-#define MOTD_FILE	"ftpmotd"
-
-/* These change if we want to use different defaults */
-
-#define FTP_UID		14
-#define FTP_GID		14
-#define FTP_GROUP	"ftp"
-#define FTP_UPLOAD	"incoming"
-#define FTP_COMMENT	"Anonymous FTP Admin"
-#define FTP_HOMEDIR	"/var/ftp"
-
-#define ANONFTP_HELPFILE "anonftp"
-
-/* Set up the structure to hold configuration information */
-/* Note that this is only what we could fit onto the one screen */
-
-typedef struct
-{
-    char homedir[64];             /* Home Dir for Anon FTP */
-    char group[32];               /* Group */
-    char uid[8];                  /* UID */
-    char comment[64];             /* PWD Comment */
-    char upload[32];              /* Upload Dir */
-} FTPConf;
-
-static FTPConf tconf;
-
-#define ANONFTP_HOMEDIR_LEN       64
-#define ANONFTP_COMMENT_LEN       64
-#define ANONFTP_UPLOAD_LEN        32
-#define ANONFTP_GROUP_LEN         32
-#define ANONFTP_UID_LEN           8
-
-static int      okbutton, cancelbutton;
-
-/* What the screen size is meant to be */
-#define ANONFTP_DIALOG_Y         0
-#define ANONFTP_DIALOG_X         8
-#define ANONFTP_DIALOG_WIDTH     COLS - 16
-#define ANONFTP_DIALOG_HEIGHT    LINES - 2
-
-static Layout layout[] = {
-#define LAYOUT_UID		0
-    { 2, 3, 8, ANONFTP_UID_LEN - 1,
-      "UID:", "What user ID to assign to FTP Admin",
-      tconf.uid, STRINGOBJ, NULL },
-#define LAYOUT_GROUP		1
-    { 2, 15, 15, ANONFTP_GROUP_LEN - 1,
-      "Group:",  "Group name that ftp process belongs to",
-      tconf.group, STRINGOBJ, NULL },
-#define LAYOUT_COMMENT		2
-    { 2, 35, 24, ANONFTP_COMMENT_LEN - 1,
-      "Comment:", "Password file comment for FTP Admin",
-      tconf.comment, STRINGOBJ, NULL },
-#define LAYOUT_HOMEDIR		3
-    { 9, 10, 43, ANONFTP_HOMEDIR_LEN - 1,
-      "FTP Root Directory:",
-      "The top directory to chroot to when doing anonymous ftp",
-      tconf.homedir, STRINGOBJ, NULL },
-#define LAYOUT_UPLOAD		4
-    { 14, 20, 22, ANONFTP_UPLOAD_LEN - 1,
-      "Upload Subdirectory:", "Designated sub-directory that holds uploads (leave empty for none)",
-      tconf.upload, STRINGOBJ, NULL },
-#define LAYOUT_OKBUTTON		5
-    { 19, 15, 0, 0,
-      "OK", "Select this if you are happy with these settings",
-      &okbutton, BUTTONOBJ, NULL },
-#define LAYOUT_CANCELBUTTON	6
-    { 19, 35, 0, 0,
-      "CANCEL", "Select this if you wish to cancel this screen",
-      &cancelbutton, BUTTONOBJ, NULL },
-    LAYOUT_END,
-};
-
-static int
-createFtpUser(void)
-{
-    struct passwd *tpw;
-    struct group  *tgrp;
-    char pwline[256];
-    char *tptr;
-    int gid;
-    FILE *fptr;
-    
-    if ((gid = atoi(tconf.group)) <= 0) {
-	if (!(tgrp = getgrnam(tconf.group))) {
-	    /* group does not exist, create it by name */
-	    
-	    tptr = msgGetInput("14", "What group ID to use for group %s ?", tconf.group);
-	    if (tptr && *tptr && ((gid = atoi(tptr)) > 0)) {
-		if ((fptr = fopen(_PATH_GROUP,"a"))) {
-		    fprintf(fptr,"%s:*:%d:%s\n",tconf.group,gid,FTP_NAME);
-		    fclose(fptr);
-		}
-	    }
-	    else
-		gid = FTP_GID;
-	}
-	else
-	    gid = tgrp->gr_gid;
-    }
-    else if (!getgrgid(gid)) {
-	/* group does not exist, create it by number */
-	
-	tptr = msgGetInput("ftp", "What group name to use for gid %d ?", gid);
-	if (tptr && *tptr) {
-	    SAFE_STRCPY(tconf.group, tptr);
-	    if ((tgrp = getgrnam(tconf.group))) {
-		gid = tgrp->gr_gid;
-	    }
-	    else if ((fptr = fopen(_PATH_GROUP,"a"))) {
-		fprintf(fptr,"%s:*:%d:%s\n",tconf.group,gid,FTP_NAME);
-		fclose(fptr);
-	    }
-	}
-    }
-    
-    if ((tpw = getpwnam(FTP_NAME))) {
-	if (tpw->pw_uid != FTP_UID)
-	    msgConfirm("FTP user already exists with a different uid.");
-	
-	return DITEM_SUCCESS; 	/* succeeds if already exists */
-    }
-    
-    sprintf(pwline, "%s:*:%s:%d::0:0:%s:%s:/nonexistent\n", FTP_NAME, tconf.uid, gid, tconf.comment, tconf.homedir);
-    
-    fptr = fopen(_PATH_MASTERPASSWD,"a");
-    if (! fptr) {
-	msgConfirm("Could not open master password file.");
-	return DITEM_FAILURE;
-    }
-    fprintf(fptr, "%s", pwline);
-    fclose(fptr);
-    msgNotify("Remaking password file: %s", _PATH_MASTERPASSWD);
-    vsystem("pwd_mkdb -p %s", _PATH_MASTERPASSWD);
-    return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-/* This is it - how to get the setup values */
-static int
-anonftpOpenDialog(void)
-{
-    WINDOW              *ds_win;
-    ComposeObj		*obj = NULL;
-    int                 n = 0, cancel = FALSE;
-    int                 max;
-    char                title[80];
-    WINDOW		*w = savescr();
-
-    /* We need a curses window */
-    if (!(ds_win = openLayoutDialog(ANONFTP_HELPFILE, " Anonymous FTP Configuration ",
-			      ANONFTP_DIALOG_X, ANONFTP_DIALOG_Y, ANONFTP_DIALOG_WIDTH, ANONFTP_DIALOG_HEIGHT))) {
-	beep();
-	msgConfirm("Cannot open anonymous ftp dialog window!!");
-	restorescr(w);
-	return DITEM_FAILURE;
-    }
-    
-    /* Draw a sub-box for the path configuration */
-    draw_box(ds_win, ANONFTP_DIALOG_Y + 7, ANONFTP_DIALOG_X + 8,
-	     ANONFTP_DIALOG_HEIGHT - 11, ANONFTP_DIALOG_WIDTH - 17,
-	     dialog_attr, border_attr);
-    wattrset(ds_win, dialog_attr);
-    sprintf(title, " Path Configuration ");
-    mvwaddstr(ds_win, ANONFTP_DIALOG_Y + 7, ANONFTP_DIALOG_X + 22, title);
-    
-    /** Initialize the config Data Structure **/
-    bzero(&tconf, sizeof(tconf));
-    
-    SAFE_STRCPY(tconf.group, FTP_GROUP);
-    SAFE_STRCPY(tconf.upload, FTP_UPLOAD);
-    SAFE_STRCPY(tconf.comment, FTP_COMMENT);
-    SAFE_STRCPY(tconf.homedir, FTP_HOMEDIR);
-    sprintf(tconf.uid, "%d", FTP_UID);
-    
-    /* Some more initialisation before we go into the main input loop */
-    obj = initLayoutDialog(ds_win, layout, ANONFTP_DIALOG_X, ANONFTP_DIALOG_Y, &max);
-
-    cancelbutton = okbutton = 0;
-    while (layoutDialogLoop(ds_win, layout, &obj, &n, max, &cancelbutton, &cancel));
-
-    /* Clear this crap off the screen */
-    delwin(ds_win);
-    use_helpfile(NULL);
-    restorescr(w);
-    if (cancel)
-	return DITEM_FAILURE;
-    return DITEM_SUCCESS;
-}
-
-int
-configAnonFTP(dialogMenuItem *self __unused)
-{
-    int i;
-
-
-    if (msgYesNo("Anonymous FTP permits un-authenticated users to connect to the system\n"
-		 "FTP server, if FTP service is enabled.  Anonymous users are\n"
-		 "restricted to a specific subset of the file system, and the default\n"
-		 "configuration provides a drop-box incoming directory to which uploads\n"
-		 "are permitted.  You must separately enable both inetd(8), and enable\n"
-		 "ftpd(8) in inetd.conf(5) for FTP services to be available.  If you\n"
-		 "did not do so earlier, you will have the opportunity to enable inetd(8)\n"
-		 "again later.\n\n"
-		 "If you want the server to be read-only you should leave the upload\n"
-		 "directory option empty and add the -r command-line option to ftpd(8)\n"
-                 "in inetd.conf(5)\n\n"
-		 "Do you wish to continue configuring anonymous FTP?")) {
-	return DITEM_FAILURE;
-    }
-    
-    /* Be optimistic */
-    i = DITEM_SUCCESS;
-    
-    i = anonftpOpenDialog();
-    if (DITEM_STATUS(i) != DITEM_SUCCESS) {
-	msgConfirm("Configuration of Anonymous FTP cancelled per user request.");
-	return i;
-    }
-    
-    /*** Use defaults for any invalid values ***/
-    if (atoi(tconf.uid) <= 0)
-	sprintf(tconf.uid, "%d", FTP_UID);
-    
-    if (!tconf.group[0])
-	SAFE_STRCPY(tconf.group, FTP_GROUP);
-    
-    /*** If the user did not specify a directory, use default ***/
-    
-    if (tconf.homedir[strlen(tconf.homedir) - 1] == '/')
-	tconf.homedir[strlen(tconf.homedir) - 1] = '\0';
-    
-    if (!tconf.homedir[0])
-	SAFE_STRCPY(tconf.homedir, FTP_HOMEDIR);
-    
-    /*** If HomeDir does not exist, create it ***/
-    
-    if (!directory_exists(tconf.homedir))
-	vsystem("mkdir -p %s", tconf.homedir);
-    
-    if (directory_exists(tconf.homedir)) {
-	msgNotify("Configuring %s for use by anon FTP.", tconf.homedir);
-	vsystem("chmod 555 %s && chown root:%s %s", tconf.homedir, tconf.group, tconf.homedir);
-	vsystem("mkdir %s/etc && chmod 555 %s/etc", tconf.homedir, tconf.homedir);
-	vsystem("mkdir -p %s/pub", tconf.homedir);
-	if (tconf.upload[0]) {
-	    vsystem("mkdir -p %s/%s", tconf.homedir, tconf.upload);
-	    vsystem("chmod 1777 %s/%s", tconf.homedir, tconf.upload);
-	}
-	
-	if (DITEM_STATUS(createFtpUser()) == DITEM_SUCCESS) {
-	    msgNotify("Copying password information for anon FTP.");
-	    vsystem("awk -F: '{if ((substr($1, 1, 1) != \"+\") && (substr($1, 1, 1) != \"-\") && ($3 < 10 || $1 == \"ftp\")) print $0}' /etc/master.passwd > %s/etc/master.passwd", tconf.homedir);
-	    vsystem("/usr/sbin/pwd_mkdb -d %s/etc %s/etc/master.passwd && chmod 444 %s/etc/pwd.db", tconf.homedir, tconf.homedir, tconf.homedir);
-	    vsystem("rm -f %s/etc/master.passwd %s/etc/spwd.db", tconf.homedir, tconf.homedir);
-	    vsystem("awk -F: '!/^#/ {if ((substr($1, 1, 1) != \"+\") && (substr($1, 1, 1) != \"-\") && ($3 < 100)) printf \"%%s:*:%%s:\\n\", $1, $3}' /etc/group > %s/etc/group && chmod 444 %s/etc/group", tconf.homedir, tconf.homedir);
-	    vsystem("chown -R root:%s %s/pub", tconf.group, tconf.homedir);
-	}
-	else {
-	    msgConfirm("Unable to create FTP user!  Anonymous FTP setup failed.");
-	    i = DITEM_FAILURE;
-	}
-	
-	if (!msgYesNo("Create a welcome message file for anonymous FTP users?")) {
-	    char cmd[256];
-	    vsystem("echo Your welcome message here. > %s/etc/%s", tconf.homedir, MOTD_FILE);
-	    sprintf(cmd, "%s %s/etc/%s", variable_get(VAR_EDITOR), tconf.homedir, MOTD_FILE);
-	    if (!systemExecute(cmd))
-		i = DITEM_SUCCESS;
-	    else
-		i = DITEM_FAILURE;
-	}
-    }
-    else {
-	msgConfirm("Invalid Directory: %s\n"
-		   "Anonymous FTP will not be set up.", tconf.homedir);
-	i = DITEM_FAILURE;
-    }
-    if (DITEM_STATUS(i) == DITEM_SUCCESS)
-	variable_set2("anon_ftp", "YES", 0);
-    return i | DITEM_RESTORE;
-}
diff --git a/usr.sbin/sysinstall/cdrom.c b/usr.sbin/sysinstall/cdrom.c
deleted file mode 100644
index 42ba1a01a260..000000000000
--- a/usr.sbin/sysinstall/cdrom.c
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- * Copyright (c) 1995
- * 	Gary J Palmer. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/* These routines deal with getting things off of CDROM media */
-
-#include "sysinstall.h"
-#include <sys/stat.h>
-#include <sys/errno.h>
-#include <sys/param.h>
-#include <sys/wait.h>
-#include <sys/cdio.h>
-#include <unistd.h>
-#include <grp.h>
-#include <fcntl.h>
-#include <libutil.h>
-
-#define CD9660
-#include <sys/mount.h>
-#include <isofs/cd9660/cd9660_mount.h>
-#undef CD9660
-
-static Boolean cdromMounted;
-static Boolean previouslyMounted; /* Was the disc already mounted? */
-static char mountpoint[MAXPATHLEN] = "/dist";
-int CDROMInitQuiet;
-
-static void mediaEjectCDROM(Device *dev);
-
-static properties
-read_props(char *name)
-{
-	int fd;
-	properties n;
-
-	fd = open(name, O_RDONLY);
-	if (fd == -1)
-	    return NULL;
-	n = properties_read(fd);
-	close(fd);
-	return n;
-}
-
-Boolean
-mediaInitCDROM(Device *dev)
-{
-    struct iso_args	args;
-    properties cd_attr = NULL;
-    char *cp = NULL;
-    Boolean readInfo = TRUE;
-    static Boolean bogusCDOK = FALSE;
-    int err;
-
-    if (cdromMounted)
-	return TRUE;
-
-    Mkdir(mountpoint);
-    bzero(&args, sizeof(args));
-    args.fspec = dev->devname;
-    args.flags = 0;
-    err = mount("cd9660", mountpoint, MNT_RDONLY, (caddr_t) &args);
-    /* If disc inserted too recently first access generates EIO, try again */
-    if (err == -1 && errno == EIO)
-        err = mount("cd9660", mountpoint, MNT_RDONLY, (caddr_t) &args);
-    if (err == -1) {
-	if (errno == EINVAL) {
-	    msgConfirm("The disc in your drive looks more like an Audio disc than a FreeBSD release.");
-	    return FALSE;
-	}
-	if (errno == EBUSY) {
-	    /* Perhaps the CDROM drive is already mounted as /cdrom */
-	    if (file_readable("/cdrom/cdrom.inf")) {
-		previouslyMounted = TRUE;
-		strlcpy(mountpoint, "/cdrom", 7);
-		errno = 0;
-	    }
-	}
-	if (errno) {
-	    if (!CDROMInitQuiet)
-		msgConfirm("Error mounting %s on %s: %s (%u)", dev->devname,
-		    mountpoint, strerror(errno), errno);
-	    return FALSE;
-	}
-    }
-    cdromMounted = TRUE;
-
-    if (!file_readable(string_concat(mountpoint, "/cdrom.inf")) && !bogusCDOK) {
-	if (msgYesNo("Warning: The disc currently in the drive is either not a FreeBSD\n"
-		     "disc or it is an older (pre 2.1.5) FreeBSD CD which does not\n"
-		     "have a version number on it.  Do you wish to use this disc anyway?") != 0) {
-	    if (!previouslyMounted)
-		unmount(mountpoint, MNT_FORCE);
-	    cdromMounted = FALSE;
-	    return FALSE;
-	}
-	else {
-	    readInfo = FALSE;
-	    bogusCDOK = TRUE;
-	}
-    }
-
-    if (readInfo) {
-	if (!(cd_attr = read_props(string_concat(mountpoint, "/cdrom.inf")))
-	    || !(cp = property_find(cd_attr, "CD_VERSION"))) {
-	    msgConfirm("Unable to find a %s/cdrom.inf file.\n"
-		       "Either this is not a FreeBSD disc, there is a problem with\n"
-		       "the CDROM driver or something is wrong with your hardware.\n"
-		       "Please fix this problem (check the console logs on VTY2) and\n"
-		       "try again.", mountpoint);
-	}
-	else {
-	    if (variable_cmp(VAR_RELNAME, cp) &&
-		variable_cmp(VAR_RELNAME, "any") &&
-		strcmp(cp, "any") &&
-		!bogusCDOK) {
-		msgConfirm("Warning: The version of the FreeBSD disc currently in the drive\n"
-			   "(%s) does not match the version of the boot floppy\n"
-			   "(%s).\n\n"
-			   "If this is intentional, to avoid this message in the future\n"
-			   "please visit the Options editor to set the boot floppy version\n"
-			   "string to match that of the disc before selecting it as your\n"
-			   "installation media.", cp, variable_get(VAR_RELNAME));
-
-		if (msgYesNo("Would you like to try and use this disc anyway?") != 0) {
-	            if (!previouslyMounted)
-		        unmount(mountpoint, MNT_FORCE);
-		    cdromMounted = FALSE;
-		    properties_free(cd_attr);
-		    return FALSE;
-		}
-		else
-		    bogusCDOK = TRUE;
-	    }
-	    if ((cp = property_find(cd_attr, "CD_MACHINE_ARCH")) != NULL) {
-		if (strcmp(cp, "any") &&
-#if defined(PC98)
-		  strcmp(cp, "pc98")) {
-#elif defined(__sparc64__)
-		  strcmp(cp, "sparc64")) {
-#else
-		  strcmp(cp, "x86")) {
-#endif
-		    msgConfirm("Fatal: The FreeBSD install CD/DVD currently in the drive\n"
-			   "is for the %s architecture, not the machine you're using.\n\n"
-
-			   "Please use the correct installation CD/DVD for your machine type.", cp);
-
-	            if (!previouslyMounted)
-		        unmount(mountpoint, MNT_FORCE);
-		    cdromMounted = FALSE;
-		    properties_free(cd_attr);
-		    return FALSE;
-		}
-	    }
-	    if ((cp = property_find(cd_attr, "CD_VOLUME")) != NULL) {
-		dev->volume = atoi(cp);
-		/* XXX - Sanity check the volume here? */
-		msgDebug("CD Volume %d initialized!\n", dev->volume);
-	    } else {
-		dev->volume = 0;
-	    }
-	}
-    }
-    if (cd_attr)
-	properties_free(cd_attr);
-    return TRUE;
-}
-
-FILE *
-mediaGetCDROM(Device *dev, char *file, Boolean probe)
-{
-    return mediaGenericGet(mountpoint, file);
-}
-
-void
-mediaShutdownCDROM(Device *dev)
-{
-    if (!cdromMounted)
-	return;
-
-    if (previouslyMounted) {
-	cdromMounted = FALSE;
-	return;
-    }
-
-    if (unmount(mountpoint, MNT_FORCE) != 0)
-	msgConfirm("Could not unmount the CDROM/DVD from %s: %s", mountpoint, strerror(errno));
-    else
-	cdromMounted = FALSE;
-
-    mediaEjectCDROM(dev);
-}
-
-static void
-mediaEjectCDROM(Device *dev)
-{
-	int fd = -1;
-
-	msgDebug("Ejecting CDROM/DVD at %s", dev->devname);
-
-	fd = open(dev->devname, O_RDONLY);
-	
-	if (fd < 0)
-		msgDebug("Could not eject the CDROM/DVD from %s: %s", dev->devname, strerror(errno));
-	else {
-		ioctl(fd, CDIOCALLOW);
-		ioctl(fd, CDIOCEJECT);
-		close(fd);
-	}
-}
diff --git a/usr.sbin/sysinstall/command.c b/usr.sbin/sysinstall/command.c
deleted file mode 100644
index 33ebc352544a..000000000000
--- a/usr.sbin/sysinstall/command.c
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-
-#define MAX_NUM_COMMANDS	10
-
-typedef struct {
-    char key[FILENAME_MAX];
-    struct {
-	enum { CMD_SHELL, CMD_FUNCTION } type;
-	void *ptr, *data;
-    } cmds[MAX_NUM_COMMANDS];
-    int ncmds;
-} Command;
-
-#define MAX_CMDS	200
-static Command *commandStack[MAX_CMDS];
-int numCommands;
-
-/* Nuke the command stack */
-void
-command_clear(void)
-{
-    int i, j;
-
-    for (i = 0; i < numCommands; i++)
-	for (j = 0; j < commandStack[i]->ncmds; j++)
-	    if (commandStack[i]->cmds[j].type == CMD_SHELL)
-		free(commandStack[i]->cmds[j].ptr);
-    free(commandStack[i]);
-    numCommands = 0;
-}
-
-static void
-addit(char *key, int type, void *cmd, void *data)
-{
-    int i;
-
-    /* First, look for the key already present and add a command to it if found */
-    for (i = 0; i < numCommands; i++) {
-	if (!strcmp(commandStack[i]->key, key)) {
-	    if (commandStack[i]->ncmds == MAX_NUM_COMMANDS)
-		msgFatal("More than %d commands stacked up behind %s??", MAX_NUM_COMMANDS, key);
-	    commandStack[i]->cmds[commandStack[i]->ncmds].type = type;
-	    commandStack[i]->cmds[commandStack[i]->ncmds].ptr = cmd;
-	    commandStack[i]->cmds[commandStack[i]->ncmds].data = data;
-	    ++(commandStack[i]->ncmds);
-	    return;
-	}
-    }
-    if (numCommands == MAX_CMDS)
-	msgFatal("More than %d commands accumulated??", MAX_CMDS);
-
-    /* If we fell to here, it's a new key */
-    commandStack[numCommands] = safe_malloc(sizeof(Command));
-    strcpy(commandStack[numCommands]->key, key);
-    commandStack[numCommands]->ncmds = 1;
-    commandStack[numCommands]->cmds[0].type = type;
-    commandStack[numCommands]->cmds[0].ptr = cmd;
-    commandStack[numCommands]->cmds[0].data = data;
-    ++numCommands;
-}
-
-/* Add a shell command under a given key */
-void
-command_shell_add(char *key, char *fmt, ...)
-{
-    va_list args;
-    char *cmd;
-
-    cmd = (char *)safe_malloc(256);
-    va_start(args, fmt);
-    vsnprintf(cmd, 256, fmt, args);
-    va_end(args);
-
-    addit(key, CMD_SHELL, cmd, NULL);
-}
-
-/* Add a shell command under a given key */
-void
-command_func_add(char *key, commandFunc func, void *data)
-{
-    addit(key, CMD_FUNCTION, func, data);
-}
-
-static int
-sort_compare(Command *p1, Command *p2)
-{
-    if (!p1 && !p2)
-	return 0;
-    else if (!p1 && p2)	/* NULL has a "greater" value for commands */
-	return 1;
-    else if (p1 && !p2)
-	return -1;
-    else
-	return strcmp(p1->key, p2->key);
-}
-
-void
-command_sort(void)
-{
-    int i, j;
-
-    commandStack[numCommands] = NULL;
-    /* Just do a crude bubble sort since the list is small */
-    for (i = 0; i < numCommands; i++) {
-	for (j = 0; j < numCommands; j++) {
-	    if (sort_compare(commandStack[j], commandStack[j + 1]) > 0) {
-		Command *tmp = commandStack[j];
-
-		commandStack[j] = commandStack[j + 1];
-		commandStack[j + 1] = tmp;
-	    }
-	}
-    }
-}
-
-/* Run all accumulated commands in sorted order */
-void
-command_execute(void)
-{
-    int i, j, ret;
-    commandFunc func;
-
-    for (i = 0; i < numCommands; i++) {
-	for (j = 0; j < commandStack[i]->ncmds; j++) {
-	    /* If it's a shell command, run system on it */
-	    if (commandStack[i]->cmds[j].type == CMD_SHELL) {
-		msgNotify("Doing %s", (char *)commandStack[i]->cmds[j].ptr);
-		ret = vsystem("%s", (char *)commandStack[i]->cmds[j].ptr);
-		if (isDebug())
-		    msgDebug("Command `%s' returns status %d\n", 
-		        (char *)commandStack[i]->cmds[j].ptr, ret);
-	    }
-	    else {
-		/* It's a function pointer - call it with the key and
-                   the data */
-		func = (commandFunc)commandStack[i]->cmds[j].ptr;
-		if (isDebug())
-		    msgDebug("%p: Execute(%s, %s)\n", 
-		        func, commandStack[i]->key, 
-			(char *)commandStack[i]->cmds[j].data);
-		ret = (*func)(commandStack[i]->key, commandStack[i]->cmds[j].data);
-		if (isDebug())
-		    msgDebug("Function @ %p returns status %d\n", 
-		        commandStack[i]->cmds[j].ptr, ret);
-	    }
-	}
-    }
-}
diff --git a/usr.sbin/sysinstall/config.c b/usr.sbin/sysinstall/config.c
deleted file mode 100644
index de838fe1ab22..000000000000
--- a/usr.sbin/sysinstall/config.c
+++ /dev/null
@@ -1,1074 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/disklabel.h>
-#include <sys/wait.h>
-#include <sys/errno.h>
-#include <sys/ioctl.h>
-#include <sys/fcntl.h>
-#include <sys/param.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <sys/mount.h>
-#include <libdisk.h>
-#include <time.h>
-#include <kenv.h>
-
-static Chunk *chunk_list[MAX_CHUNKS];
-static int nchunks;
-static int rootdev_is_od;
-
-/* arg to sort */
-static int
-chunk_compare(Chunk *c1, Chunk *c2)
-{
-    if (!c1 && !c2)
-	return 0;
-    else if (!c1 && c2)
-	return 1;
-    else if (c1 && !c2)
-	return -1;
-    else if (!c1->private_data && !c2->private_data)
-	return 0;
-    else if (c1->private_data && !c2->private_data)
-	return 1;
-    else if (!c1->private_data && c2->private_data)
-	return -1;
-    else
-	return strcmp(((PartInfo *)(c1->private_data))->mountpoint, ((PartInfo *)(c2->private_data))->mountpoint);
-}
-
-static void
-chunk_sort(void)
-{
-    int i, j;
-
-    for (i = 0; i < nchunks; i++) {
-	for (j = 0; j < nchunks; j++) {
-	    if (chunk_compare(chunk_list[j], chunk_list[j + 1]) > 0) {
-		Chunk *tmp = chunk_list[j];
-
-		chunk_list[j] = chunk_list[j + 1];
-		chunk_list[j + 1] = tmp;
-	    }
-	}
-    }
-}
-
-static void
-check_rootdev(Chunk **list, int n)
-{
-	int i;
-	Chunk *c;
-
-	rootdev_is_od = 0;
-	for (i = 0; i < n; i++) {
-		c = *list++;
-		if (c->type == part && (c->flags & CHUNK_IS_ROOT)
-		    && strncmp(c->disk->name, "od", 2) == 0)
-			rootdev_is_od = 1;
-	}
-}
-
-static char *
-name_of(Chunk *c1)
-{
-    return c1->name;
-}
-
-static char *
-mount_point(Chunk *c1)
-{
-    if (c1->type == part && c1->subtype == FS_SWAP)
-	return "none";
-    else if (c1->type == part || c1->type == fat || c1->type == efi)
-	return ((PartInfo *)c1->private_data)->mountpoint;
-    return "/bogus";
-}
-
-static char *
-fstype(Chunk *c1)
-{
-    if (c1->type == fat || c1->type == efi)
-	return "msdosfs";
-    else if (c1->type == part) {
-	if (c1->subtype != FS_SWAP)
-	    return "ufs";
-	else
-	    return "swap";
-    }
-    return "bogus";
-}
-
-static char *
-fstype_short(Chunk *c1)
-{
-    if (c1->type == part) {
-	if (c1->subtype != FS_SWAP) {
-	    if (rootdev_is_od == 0 && strncmp(c1->name, "od", 2) == 0)
-		return "rw,noauto";
-	    else
-		return "rw";
-	}
-	else
-	    return "sw";
-    }
-    else if (c1->type == fat) {
-	if (strncmp(c1->name, "od", 2) == 0)
-	    return "ro,noauto";
-	else
-	    return "ro";
-    }
-    else if (c1->type == efi)
-	return "rw";
-
-    return "bog";
-}
-
-static int
-seq_num(Chunk *c1)
-{
-    if (c1->type == part && c1->subtype != FS_SWAP) {
-	if (rootdev_is_od == 0 && strncmp(c1->name, "od", 2) == 0)
-	    return 0;
-	else if (c1->flags & CHUNK_IS_ROOT)
-	    return 1;
-	else
-	    return 2;
-    }
-    return 0;
-}
-
-int
-configFstab(dialogMenuItem *self)
-{
-    Device **devs;
-    Disk *disk;
-    FILE *fstab;
-    int i, cnt;
-    Chunk *c1, *c2;
-
-    if (!RunningAsInit) {
-	if (file_readable("/etc/fstab"))
-	    return DITEM_SUCCESS;
-	else {
-	    msgConfirm("Attempting to rebuild your /etc/fstab file.  Warning: If you had\n"
-		       "any CD devices in use before running %s then they may NOT\n"
-		       "be found by this run!", ProgName);
-	}
-    }
-
-    devs = deviceFind(NULL, DEVICE_TYPE_DISK);
-    if (!devs) {
-	msgConfirm("No disks found!");
-	return DITEM_FAILURE;
-    }
-
-    /* Record all the chunks */
-    nchunks = 0;
-    for (i = 0; devs[i]; i++) {
-	if (!devs[i]->enabled)
-	    continue;
-	disk = (Disk *)devs[i]->private;
-	if (!disk->chunks)
-	    msgFatal("No chunk list found for %s!", disk->name);
-	for (c1 = disk->chunks->part; c1; c1 = c1->next) {
-#ifdef __powerpc__
-	    if (c1->type == apple) {
-#else
-	    if (c1->type == freebsd) {
-#endif
-		for (c2 = c1->part; c2; c2 = c2->next) {
-		    if (c2->type == part && (c2->subtype == FS_SWAP || c2->private_data))
-			chunk_list[nchunks++] = c2;
-		}
-	    }
-	    else if (((c1->type == fat || c1->type == efi || c1->type == part) &&
-		    c1->private_data) || (c1->type == part && c1->subtype == FS_SWAP))
-		chunk_list[nchunks++] = c1;
-	}
-    }
-    chunk_list[nchunks] = 0;
-    chunk_sort();
-
-    fstab = fopen("/etc/fstab", "w");
-    if (!fstab) {
-	msgConfirm("Unable to create a new /etc/fstab file!  Manual intervention\n"
-		   "will be required.");
-	return DITEM_FAILURE;
-    }
-
-    check_rootdev(chunk_list, nchunks);
-
-    /* Go for the burn */
-    msgDebug("Generating /etc/fstab file\n");
-    fprintf(fstab, "# Device\t\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n");
-    for (i = 0; i < nchunks; i++)
-	fprintf(fstab, "/dev/%s\t\t%s\t\t%s\t%s\t\t%d\t%d\n", name_of(chunk_list[i]), mount_point(chunk_list[i]),
-		fstype(chunk_list[i]), fstype_short(chunk_list[i]), seq_num(chunk_list[i]), seq_num(chunk_list[i]));
-
-    /* Now look for the CDROMs */
-    devs = deviceFind(NULL, DEVICE_TYPE_CDROM);
-    cnt = deviceCount(devs);
-
-    /* Write out the CDROM entries */
-    for (i = 0; i < cnt; i++) {
-	char cdname[10];
-
-	sprintf(cdname, "/cdrom%s", i ? itoa(i) : "");
-	if (Mkdir(cdname))
-	    msgConfirm("Unable to make mount point for: %s", cdname);
-	else
-	    fprintf(fstab, "/dev/%s\t\t%s\t\tcd9660\tro,noauto\t0\t0\n", devs[i]->name, cdname);
-    }
-
-    fclose(fstab);
-    if (isDebug())
-	msgDebug("Wrote out /etc/fstab file\n");
-    return DITEM_SUCCESS;
-}
-
-/* Do the work of sucking in a config file.
- * config is the filename to read in.
- * lines is a fixed (max) sized array of char*
- * returns number of lines read.  line contents
- * are malloc'd and must be freed by the caller.
- */
-static int
-readConfig(char *config, char **lines, int max)
-{
-    FILE *fp;
-    char line[256];
-    int i, nlines;
-
-    fp = fopen(config, "r");
-    if (!fp)
-	return -1;
-
-    nlines = 0;
-    /* Read in the entire file */
-    for (i = 0; i < max; i++) {
-	if (!fgets(line, sizeof line, fp))
-	    break;
-	lines[nlines++] = strdup(line);
-    }
-    fclose(fp);
-    if (isDebug())
-	msgDebug("readConfig: Read %d lines from %s.\n", nlines, config);
-    return nlines;
-}
-
-#define MAX_LINES  2000 /* Some big number we're not likely to ever reach - I'm being really lazy here, I know */
-
-static void
-readConfigFile(char *config, int marked)
-{
-    char *lines[MAX_LINES], *cp, *cp2;
-    int i, nlines;
-
-    nlines = readConfig(config, lines, MAX_LINES);
-    if (nlines == -1)
-	return;
-
-    for (i = 0; i < nlines; i++) {
-	/* Skip the comments & non-variable settings */
-	if (lines[i][0] == '#' || !(cp = index(lines[i], '='))) {
-	    free(lines[i]);
-	    continue;
-	}
-	*cp++ = '\0';
-	/* Find quotes */
-	if ((cp2 = index(cp, '"')) || (cp2 = index(cp, '\047'))) {
-	    cp = cp2 + 1;
-	    cp2 = index(cp, *cp2);
-	}
-	/* If valid quotes, use it */
-	if (cp2) {
-	    *cp2 = '\0';
- 	    /* If we have a legit value, set it */
-	    if (strlen(cp))
-		variable_set2(lines[i], cp, marked);
-	}
-	free(lines[i]);
-    }
-}
-
-/* Load the environment from rc.conf file(s) */
-void
-configEnvironmentRC_conf(void)
-{
-    static struct {
-	char *fname;
-	int marked;
-    } configs[] = {
-	{ "/etc/defaults/rc.conf", 0 },
-	{ "/etc/rc.conf", 0 },
-	{ "/etc/rc.conf.local", 0 },
-	{ NULL, 0 },
-    };
-    int i;
-
-    for (i = 0; configs[i].fname; i++) {
-	if (file_readable(configs[i].fname))
-	    readConfigFile(configs[i].fname, configs[i].marked);
-    }
-}
-
-/* Load the environment from a resolv.conf file */
-void
-configEnvironmentResolv(char *config)
-{
-    char *lines[MAX_LINES];
-    int i, nlines;
-
-    nlines = readConfig(config, lines, MAX_LINES);
-    if (nlines == -1)
-	return;
-    for (i = 0; i < nlines; i++) {
-	Boolean name_set = variable_get(VAR_NAMESERVER) ? 1 : 0;
-
-	if (!strncmp(lines[i], "domain", 6) && !variable_get(VAR_DOMAINNAME))
-	    variable_set2(VAR_DOMAINNAME, string_skipwhite(string_prune(lines[i] + 6)), 0);
-	else if (!name_set && !strncmp(lines[i], "nameserver", 10)) {
-	    /* Only take the first nameserver setting - we're lame */
-	    variable_set2(VAR_NAMESERVER, string_skipwhite(string_prune(lines[i] + 10)), 0);
-	}
-	free(lines[i]);
-    }
-}
-
-/* Version of below for dispatch routines */
-int
-configRC(dialogMenuItem *unused)
-{
-    configRC_conf();
-    return DITEM_SUCCESS;
-}
-
-/*
- * Write out rc.conf
- *
- * rc.conf is sorted if running as init and the needed utilities are
- * present
- *
- * If rc.conf is sorted, all variables in rc.conf which conflict with
- * the variables in the environment are removed from the original
- * rc.conf
- */
-void
-configRC_conf(void)
-{
-    char line[256];
-    FILE *rcSite, *rcOld;
-    Variable *v;
-    int write_header;
-    time_t t_loc;
-    char *cp;
-    static int did_marker = 0;
-    int do_sort;
-    int do_merge;
-    time_t tp;
-
-    configTtys();
-    write_header = !file_readable("/etc/rc.conf");
-    do_sort = RunningAsInit && file_readable("/usr/bin/sort") &&
-	file_readable("/usr/bin/uniq");
-    do_merge = do_sort && file_readable("/etc/rc.conf");
-
-    if(do_merge) {
-	rcSite = fopen("/etc/rc.conf.new", "w");
-    } else
-	rcSite = fopen("/etc/rc.conf", "a");
-    if (rcSite == NULL) {
-	msgError("Error opening new rc.conf for writing: %s (%u)", strerror(errno), errno);
-	return;
-    }
-
-    if (do_merge) {
-	/* "Copy" the old rc.conf */
-	rcOld = fopen("/etc/rc.conf", "r");
-	if(!rcOld) {
-	    msgError("Error opening rc.conf for reading: %s (%u)", strerror(errno), errno);
-	    return;
-	}
-	while(fgets(line, sizeof(line), rcOld)) {
-	    if(line[0] == '#' || variable_check2(line) != 0)
-		fprintf(rcSite, "%s", line);
-	    else {
-		if (variable_get(VAR_KEEPRCCONF) != NULL)
-		    fprintf(rcSite, "%s", line);
-		else
-		    fprintf(rcSite, "#REMOVED: %s", line);
-	    }
-	}
-	fclose(rcOld);
-    } else if (write_header) {
-	fprintf(rcSite, "# This file now contains just the overrides from /etc/defaults/rc.conf.\n");
-	fprintf(rcSite, "# Please make all changes to this file, not to /etc/defaults/rc.conf.\n\n");
-	fprintf(rcSite, "# Enable network daemons for user convenience.\n");
-	if ((t_loc = time(NULL)) != -1 && (cp = ctime(&t_loc)))
-	    fprintf(rcSite, "# Created: %s", cp);
-    }
-
-    /* Now do variable substitutions */
-    for (v = VarHead; v; v = v->next) {
-	if (v->dirty) {
-	    if (!did_marker) {
-		time(&tp);
-		fprintf(rcSite, "# -- sysinstall generated deltas -- # "
-		    "%s", ctime(&tp));
-		did_marker = 1;
-	    }
-	    fprintf(rcSite, "%s=\"%s\"\n", v->name, v->value);
-	    v->dirty = 0;
-	}
-    }
-    fclose(rcSite);
-
-    if(do_merge) {
-	if(rename("/etc/rc.conf.new", "/etc/rc.conf") != 0) {
-	    msgError("Error renaming temporary rc.conf: %s (%u)", strerror(errno), errno);
-	    return;
-	}
-    }
-
-    /* Tidy up the resulting file if it's late enough in the installation
-	for sort and uniq to be available */
-    if (do_sort) {
-	(void)vsystem("sort /etc/rc.conf | uniq > /etc/rc.conf.new && mv /etc/rc.conf.new /etc/rc.conf");
-    }
-}
-
-int
-configSaver(dialogMenuItem *self)
-{
-    variable_set((char *)self->data, 1);
-    if (!variable_get(VAR_BLANKTIME))
-	variable_set2(VAR_BLANKTIME, "300", 1);
-    return DITEM_SUCCESS;
-}
-
-int
-configSaverTimeout(dialogMenuItem *self)
-{
-    return (variable_get_value(VAR_BLANKTIME,
-	    "Enter time-out period in seconds for screen saver", 1) ?
-	DITEM_SUCCESS : DITEM_FAILURE);
-}
-
-int
-configNTP(dialogMenuItem *self)
-{
-    int status;
-
-    status = variable_get_value(VAR_NTPDATE_HOSTS,
-				"Enter the name of an NTP server", 1)
-	     ? DITEM_SUCCESS : DITEM_FAILURE;
-    if (status == DITEM_SUCCESS) {
-	static char tmp[255];
-
-	snprintf(tmp, sizeof(tmp), "ntpdate_enable=YES,ntpdate_hosts=%s",
-		 variable_get(VAR_NTPDATE_HOSTS));
-	self->data = tmp;
-	dmenuSetVariables(self);
-    }
-    return status;
-}
-
-int
-configCountry(dialogMenuItem *self)
-{
-    int choice, scroll, curr, max;
-
-    WINDOW *w = savescr();
-
-    dialog_clear_norefresh();
-    dmenuSetDefaultItem(&MenuCountry, NULL, NULL,
-	VAR_COUNTRY "=" DEFAULT_COUNTRY, &choice, &scroll, &curr, &max);
-    dmenuOpen(&MenuCountry, &choice, &scroll, &curr, &max, FALSE);
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-int
-configUsers(dialogMenuItem *self)
-{
-    WINDOW *w = savescr();
-
-    dialog_clear_norefresh();
-    dmenuOpenSimple(&MenuUsermgmt, FALSE);
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-int
-configSecurelevel(dialogMenuItem *self)
-{
-    WINDOW *w = savescr();
-
-    dialog_clear_norefresh();
-    dmenuOpenSimple(&MenuSecurelevel, FALSE);
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-int
-configSecurelevelDisabled(dialogMenuItem *self)
-{
-
-    variable_set2("kern_securelevel_enable", "NO", 1);
-    return DITEM_SUCCESS;
-}
-
-int
-configSecurelevelSecure(dialogMenuItem *self)
-{
-
-    variable_set2("kern_securelevel_enable", "YES", 1);
-    variable_set2("kern_securelevel", "1", 1);
-    return DITEM_SUCCESS;
-}
-
-int
-configSecurelevelHighlySecure(dialogMenuItem *self)
-{
-
-    variable_set2("kern_securelevel_enable", "YES", 1);
-    variable_set2("kern_securelevel", "2", 1);
-    return DITEM_SUCCESS;
-}
-
-int
-configSecurelevelNetworkSecure(dialogMenuItem *self)
-{
-
-    variable_set2("kern_securelevel_enable", "YES", 1);
-    variable_set2("kern_securelevel", "3", 1);
-    return DITEM_SUCCESS;
-}
-
-int
-configResolv(dialogMenuItem *ditem)
-{
-    FILE *fp;
-    char *cp, *c6p, *dp, *hp;
-
-    cp = variable_get(VAR_NAMESERVER);
-    if (!cp || !*cp)
-	goto skip;
-    Mkdir("/etc");
-    fp = fopen("/etc/resolv.conf", "w");
-    if (!fp)
-	return DITEM_FAILURE;
-    if (variable_get(VAR_DOMAINNAME))
-	fprintf(fp, "domain\t%s\n", variable_get(VAR_DOMAINNAME));
-    fprintf(fp, "nameserver\t%s\n", cp);
-    fclose(fp);
-    if (isDebug())
-	msgDebug("Wrote out /etc/resolv.conf\n");
-
-skip:
-    dp = variable_get(VAR_DOMAINNAME);
-    cp = variable_get(VAR_IPADDR);
-    c6p = variable_get(VAR_IPV6ADDR);
-    hp = variable_get(VAR_HOSTNAME);
-    /* Tack ourselves into /etc/hosts */
-    fp = fopen("/etc/hosts", "w");
-    if (!fp)
-	return DITEM_FAILURE;
-    /* Add an entry for localhost */
-    if (dp) {
-	fprintf(fp, "::1\t\t\tlocalhost localhost.%s\n", dp);
-	fprintf(fp, "127.0.0.1\t\tlocalhost localhost.%s\n", dp);
-    } else {
-	fprintf(fp, "::1\t\t\tlocalhost\n");
-	fprintf(fp, "127.0.0.1\t\tlocalhost\n");
-    }
-    /* Now the host entries, if applicable */
-    if (((cp && cp[0] != '0') || (c6p && c6p[0] != '0')) && hp) {
-	char cp2[255];
-
-	if (!index(hp, '.'))
-	    cp2[0] = '\0';
-	else {
-	    SAFE_STRCPY(cp2, hp);
-	    *(index(cp2, '.')) = '\0';
-	}
-	if (c6p && c6p[0] != '0') {
-	    fprintf(fp, "%s\t%s %s\n", c6p, hp, cp2);
-	    fprintf(fp, "%s\t%s.\n", c6p, hp);
-	}
-	if (cp && cp[0] != '0') {
-	    fprintf(fp, "%s\t\t%s %s\n", cp, hp, cp2);
-	    fprintf(fp, "%s\t\t%s.\n", cp, hp);
-	}
-    }
-    fclose(fp);
-    if (isDebug())
-	msgDebug("Wrote out /etc/hosts\n");
-    return DITEM_SUCCESS;
-}
-
-int
-configRouter(dialogMenuItem *self)
-{
-    int ret;
-
-    ret = variable_get_value(VAR_ROUTER,
-			     "Please specify the router you wish to use.  Routed is\n"
-			     "provided with the stock system and gated is provided\n"
-			     "as an optional package which this installation system\n"
-			     "will attempt to load if you select gated.  Any other\n"
-			     "choice of routing daemon will be assumed to be something\n"
-			     "the user intends to install themselves before rebooting\n"
-			     "the system.  If you don't want any routing daemon, choose NO", 1)
-      ? DITEM_SUCCESS : DITEM_FAILURE;
-
-    if (ret == DITEM_SUCCESS) {
-	char *cp = variable_get(VAR_ROUTER);
-
-	if (cp && strcmp(cp, "NO")) {
-	    variable_set2(VAR_ROUTER_ENABLE, "YES", 1);
-	    if (!strcmp(cp, "gated")) {
-		if (package_add("gated") != DITEM_SUCCESS) {
-		    msgConfirm("Unable to load gated package.  Falling back to no router.");
-		    variable_unset(VAR_ROUTER);
-		    variable_unset(VAR_ROUTERFLAGS);
-		    variable_set2(VAR_ROUTER_ENABLE, "NO", 1);
-		    cp = NULL;
-		}
-	    }
-	    if (cp) {
-		/* Now get the flags, if they chose a router */
-		ret = variable_get_value(VAR_ROUTERFLAGS,
-					 "Please Specify the routing daemon flags; if you're running routed\n"
-					 "then -q is the right choice for nodes and -s for gateway hosts.\n", 1)
-		  ? DITEM_SUCCESS : DITEM_FAILURE;
-		if (ret != DITEM_SUCCESS)
-		    variable_unset(VAR_ROUTERFLAGS);
-	    }
-	}
-	else {
-	    /* No router case */
-	    variable_set2(VAR_ROUTER_ENABLE, "NO", 1);
-	    variable_unset(VAR_ROUTERFLAGS);
-	    variable_unset(VAR_ROUTER);
-	}
-    }
-    else {
-	variable_set2(VAR_ROUTER_ENABLE, "NO", 1);
-	variable_unset(VAR_ROUTERFLAGS);
-	variable_unset(VAR_ROUTER);
-    }
-    return ret;
-}
-
-/* Shared between us and index_initialize() */
-extern PkgNode Top, Plist;
-
-int
-configPackages(dialogMenuItem *self)
-{
-    int i, restoreflag = 0;
-    PkgNodePtr tmp;
-
-    /* Did we get an INDEX? */
-    i = index_initialize("packages/INDEX");
-    if (DITEM_STATUS(i) == DITEM_FAILURE)
-	return i;
-
-    while (1) {
-	int ret, pos, scroll;
-	int current, low, high;
-
-	/* Bring up the packages menu */
-	pos = scroll = 0;
-	index_menu(&Top, &Top, &Plist, &pos, &scroll);
-
-	if (Plist.kids && Plist.kids->name) {
-	    /* Now show the packing list menu */
-	    pos = scroll = 0;
-	    ret = index_menu(&Plist, &Plist, NULL, &pos, &scroll);
-	    if (ret & DITEM_LEAVE_MENU)
-		break;
-	    else if (DITEM_STATUS(ret) != DITEM_FAILURE) {
-		dialog_clear();
-		restoreflag = 1;
-		if (have_volumes) {
-		    low = low_volume;
-		    high = high_volume;
-		} else
-		    low = high = 0;
-		for (current = low; current <= high; current++)
-		    for (tmp = Plist.kids; tmp && tmp->name; tmp = tmp->next)
-		        (void)index_extract(mediaDevice, &Top, tmp, FALSE, current);
-		break;
-	    }
-	}
-	else {
-	    msgConfirm("No packages were selected for extraction.");
-	    break;
-	}
-    }
-    tmp = Plist.kids;
-    while (tmp) {
-        PkgNodePtr tmp2 = tmp->next;
-
-        safe_free(tmp);
-        tmp = tmp2;
-    }
-    index_init(NULL, &Plist);
-    return DITEM_SUCCESS | (restoreflag ? DITEM_RESTORE : 0);
-}
-
-/* Load pcnfsd package */
-int
-configPCNFSD(dialogMenuItem *self)
-{
-    int ret;
-
-    ret = package_add("pcnfsd");
-    if (DITEM_STATUS(ret) == DITEM_SUCCESS) {
-	variable_set2(VAR_PCNFSD, "YES", 0);
-	variable_set2("mountd_flags", "-n", 1);
-    }
-    return ret;
-}
-
-int
-configInetd(dialogMenuItem *self)
-{
-    char cmd[256];
-
-    WINDOW *w = savescr();
-
-    if (msgYesNo("The Internet Super Server (inetd) allows a number of simple Internet\n"
-                 "services to be enabled, including finger, ftp, and telnetd.  Enabling\n"
-                 "these services may increase risk of security problems by increasing\n"
-                 "the exposure of your system.\n\n"
-                 "With this in mind, do you wish to enable inetd?\n")) {
-        variable_set2("inetd_enable", "NO", 1);
-    } else {
-        /* If inetd is enabled, we'll need an inetd.conf */
-        variable_set2("inetd_enable", "YES", 1);
-	if (!msgYesNo("inetd(8) relies on its configuration file, /etc/inetd.conf, to determine\n"
-                   "which of its Internet services will be available.  The default FreeBSD\n"
-                   "inetd.conf(5) leaves all services disabled by default, so they must be\n"
-                   "specifically enabled in the configuration file before they will\n"
-                   "function, even once inetd(8) is enabled.  Note that services for\n"
-		   "IPv6 must be separately enabled from IPv4 services.\n\n"
-                   "Select [Yes] now to invoke an editor on /etc/inetd.conf, or [No] to\n"
-                   "use the current settings.\n")) {
-            sprintf(cmd, "%s /etc/inetd.conf", variable_get(VAR_EDITOR));
-            dialog_clear();
-            systemExecute(cmd);
-	}
-    }
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-int
-configNFSServer(dialogMenuItem *self)
-{
-    char cmd[256];
-    int retval = 0;
-
-    /* If we're an NFS server, we need an exports file */
-    if (!file_readable("/etc/exports")) {
-	WINDOW *w = savescr();
-
-	if (file_readable("/etc/exports.disabled"))
-	    vsystem("mv /etc/exports.disabled /etc/exports");
-	else {
-	    dialog_clear_norefresh();
-	    msgConfirm("Operating as an NFS server means that you must first configure\n"
-		       "an /etc/exports file to indicate which hosts are allowed certain\n"
-		       "kinds of access to your local file systems.\n"
-		       "Press [ENTER] now to invoke an editor on /etc/exports\n");
-	    vsystem("echo '#The following examples export /usr to 3 machines named after ducks,' > /etc/exports");
-	    vsystem("echo '#/usr/src and /usr/obj read-only to machines named after trouble makers,' >> /etc/exports");
-	    vsystem("echo '#/home and all directories under it to machines named after dead rock stars' >> /etc/exports");
-	    vsystem("echo '#and, /a to a network of privileged machines allowed to write on it as root.' >> /etc/exports");
-	    vsystem("echo '#/usr                   huey louie dewie' >> /etc/exports");
-	    vsystem("echo '#/usr/src /usr/obj -ro  calvin hobbes' >> /etc/exports");
-	    vsystem("echo '#/home   -alldirs       janis jimi frank' >> /etc/exports");
-	    vsystem("echo '#/a      -maproot=0  -network 10.0.1.0 -mask 255.255.248.0' >> /etc/exports");
-	    vsystem("echo '#' >> /etc/exports");
-	    vsystem("echo '# You should replace these lines with your actual exported filesystems.' >> /etc/exports");
-	    vsystem("echo \"# Note that BSD's export syntax is 'host-centric' vs. Sun's 'FS-centric' one.\" >> /etc/exports");
-	    vsystem("echo >> /etc/exports");
-	    sprintf(cmd, "%s /etc/exports", variable_get(VAR_EDITOR));
-	    dialog_clear();
-	    systemExecute(cmd);
-	}
-	variable_set2(VAR_NFS_SERVER, "YES", 1);
-	retval = configRpcBind(NULL);
-	restorescr(w);
-    }
-    else if (variable_get(VAR_NFS_SERVER)) { /* We want to turn it off again? */
-	vsystem("mv -f /etc/exports /etc/exports.disabled");
-	variable_unset(VAR_NFS_SERVER);
-    }
-    return DITEM_SUCCESS | retval;
-}
-
-/*
- * Extend the standard dmenuToggleVariable() method to also check and set
- * the rpcbind variable if needed.
- */
-int
-configRpcBind(dialogMenuItem *self)
-{
-    char *tmp, *tmp2;
-    int retval = 0;
-    int doupdate = 1;
-
-    if (self != NULL) {
-    	retval = dmenuToggleVariable(self);
-	tmp = strdup(self->data);
-	if ((tmp2 = index(tmp, '=')) != NULL)
-	    *tmp2 = '\0';
-	if (strcmp(variable_get(tmp), "YES") != 0)
-	    doupdate = 0;
-	free(tmp);
-    }
-
-    if (doupdate && strcmp(variable_get(VAR_RPCBIND_ENABLE), "YES") != 0) {
-	variable_set2(VAR_RPCBIND_ENABLE, "YES", 1);
-	retval |= DITEM_REDRAW;
-    }
-
-   return retval;
-}
-
-int
-configEtcTtys(dialogMenuItem *self)
-{
-    char cmd[256];
-
-    WINDOW *w = savescr();
-
-    /* Simply prompt for confirmation, then edit away. */
-    if (msgYesNo("Configuration of system TTYs requires editing the /etc/ttys file.\n"
-		 "Typical configuration activities might include enabling getty(8)\n"
-		 "on the first serial port to allow login via serial console after\n"
-		 "reboot, or to enable xdm.  The default ttys file enables normal\n"
-		 "virtual consoles, and most sites will not need to perform manual\n"
-		 "configuration.\n\n"
-		 "To load /etc/ttys in the editor, select [Yes], otherwise, [No].")) {
-    } else {
-	configTtys();
-	sprintf(cmd, "%s /etc/ttys", variable_get(VAR_EDITOR));
-	dialog_clear();
-	systemExecute(cmd);
-    }
-
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-#ifdef __i386__
-int
-checkLoaderACPI(void)
-{
-    char val[4];
-
-    if (kenv(KENV_GET, "loader.acpi_disabled_by_user", &val[0], 4) <= 0) {
-	return (0);
-    }
-
-    if (strtol(&val[0], NULL, 10) <= 0) {
-	return (0);
-    }
-
-    return (1);
-}
-
-int
-configLoaderACPI(int disable)
-{
-    FILE *ldconf;
-
-    ldconf = fopen("/boot/loader.conf", "a");
-    if (ldconf == NULL) {
-	msgConfirm("Unable to open /boot/loader.conf.  Please consult the\n"
-		  "FreeBSD Handbook for instructions on disabling ACPI");
-	return DITEM_FAILURE;
-    }
-
-    fprintf(ldconf, "# --- Generated by sysinstall ---\n");
-    fprintf(ldconf, "hint.acpi.0.disabled=%d\n", disable);
-    fclose(ldconf);
-
-    return DITEM_SUCCESS;
-}
-#endif
-
-int
-configMTAPostfix(dialogMenuItem *self)
-{
-    int ret;
-    FILE *perconf;
-
-    if(setenv("POSTFIX_DEFAULT_MTA", "YES", 1) != 0)
-	msgError("Error setting the enviroment variable POSTFIX_DEFAULT_MTA: %s (%u)",
-		 strerror(errno), errno);
-
-    ret = package_add("postfix-2.4");
-    unsetenv("POSTFIX_DEFAULT_MTA");
-
-    if(DITEM_STATUS(ret) == DITEM_FAILURE) {
-	msgConfirm("An error occurred while adding the postfix package\n"
-		   "Please change installation media and try again.");
-	return ret;
-    }
-
-    variable_set2(VAR_SENDMAIL_ENABLE, "YES", 1);
-    variable_set2("sendmail_flags", "-bd", 1);
-    variable_set2("sendmail_outbound_enable", "NO", 1);
-    variable_set2("sendmail_submit_enable", "NO", 1);
-    variable_set2("sendmail_msp_queue_enable", "NO", 1);
-
-    perconf = fopen("/etc/periodic.conf", "a");
-    if (perconf == NULL) {
-	msgConfirm("Unable to open /etc/periodic.conf.\n"
-		   "The daily cleanup scripts might generate errors when\n"
-		   "trying to run some sendmail only cleanup scripts.\n"
-		   "Please consult the documentation for the postfix port on how to\n"
-		   "fix this.");
-
-	/* Not really a serious problem, so we return success */
-	return DITEM_SUCCESS;
-    }
-
-    fprintf(perconf, "# --- Generated by sysinstall ---\n");
-    fprintf(perconf, "daily_clean_hoststat_enable=\"NO\"\n");
-    fprintf(perconf, "daily_status_mail_rejects_enable=\"NO\"\n");
-    fprintf(perconf, "daily_status_include_submit_mailq=\"NO\"\n");
-    fprintf(perconf, "daily_submit_queuerun=\"NO\"\n");
-    fclose(perconf);
-
-    msgConfirm("Postfix is now installed and enabled as the default MTA.\n"
-	       "Please check that the configuration works as expected.\n"
-	       "See the Postfix documentation for more information.\n"
-	       "The documentation can be found in /usr/local/share/doc/postfix/\n"
-	       "or on the Postfix website at http://www.postfix.org/.");
-
-    return DITEM_SUCCESS;
-}
-
-int
-configMTAExim(dialogMenuItem *self)
-{
-    int ret;
-    FILE *perconf, *mailerconf, *newsyslogconf;
-
-    ret = package_add("exim");
-
-    if(DITEM_STATUS(ret) == DITEM_FAILURE) {
-	msgConfirm("An error occurred while adding the exim package\n"
-		   "Please change installation media and try again.");
-	return ret;
-    }
-
-    variable_set2(VAR_SENDMAIL_ENABLE, "NONE", 1);
-    variable_set2("exim_enable", "YES", 1);
-
-    /* Update periodic.conf */
-    perconf = fopen("/etc/periodic.conf", "a");
-    if (perconf == NULL) {
-	/* Not really a serious problem, so we do not abort */
-	msgConfirm("Unable to open /etc/periodic.conf.\n"
-		   "The daily cleanup scripts might generate errors when\n"
-		   "trying to run some sendmail only cleanup scripts.\n"
-		   "Please consult the documentation for the exim port on how to\n"
-		   "fix this.");
-    } else {
-	fprintf(perconf, "# --- Generated by sysinstall ---\n");
-	fprintf(perconf, "daily_clean_hoststat_enable=\"NO\"\n");
-	fprintf(perconf, "daily_status_include_submit_mailq=\"NO\"\n");
-	fprintf(perconf, "daily_status_mail_rejects_enable=\"NO\"\n");
-	fprintf(perconf, "daily_submit_queuerun=\"NO\"\n");
-	fclose(perconf);
-    }
-
-    /* Update mailer.conf */
-    vsystem("mv -f /etc/mail/mailer.conf /etc/mail/mailer.conf.old");
-    mailerconf = fopen("/etc/mail/mailer.conf", "w");
-    if (mailerconf == NULL) {
-	/* Not really a serious problem, so we do not abort */
-	msgConfirm("Unable to open /etc/mailer.conf.\n"
-		   "Some programs which use the sendmail wrappers may not work.\n"
-		   "Please consult the documentation for the exim port on how\n"
-		   "to correct this.");
-    } else {
-	fprintf(mailerconf, "# --- Generated by sysinstall ---\n");
-	fprintf(mailerconf, "# Execute exim instead of sendmail\n");
-	fprintf(mailerconf, "#\n");
-	fprintf(mailerconf, "sendmail	/usr/local/sbin/exim\n");
-	fprintf(mailerconf, "send-mail	/usr/local/sbin/exim\n");
-	fprintf(mailerconf, "mailq		/usr/local/sbin/exim\n");
-	fprintf(mailerconf, "newaliases	/usr/local/sbin/exim\n");
-	fprintf(mailerconf, "hoststat	/usr/bin/true\n");
-	fprintf(mailerconf, "purgestat	/usr/bin/true\n");
-	fclose(mailerconf);
-    }
-
-    /* Make newsyslog rotate exim logfiles */
-    newsyslogconf = fopen("/etc/newsyslog.conf", "a");
-    if (newsyslogconf == NULL) {
-	/* Not really a serious problem, so we do not abort */
-	msgConfirm("Unable to open /etc/newsyslog.conf.\n"
-		   "The exim logfiles will not be rotated.\n"
-		   "Please consult the documentation for the exim port on how to\n"
-		   "rotate the logfiles.");
-    } else {
-	fprintf(newsyslogconf, "# --- Generated by sysinstall ---\n");
-	fprintf(newsyslogconf, "/var/log/exim/mainlog	mailnull:mail	640  7	   *	@T00  ZN\n");
-	fprintf(newsyslogconf, "/var/log/exim/rejectlog	mailnull:mail	640  7	   *	@T00  ZN\n");
-	fclose(newsyslogconf);
-    }
-
-    msgConfirm("Exim is now installed and enabled as the default MTA.\n"
-	       "Please check that the configuration works as expected.\n"
-	       "See the Exim documentation for more information.\n"
-	       "The documentation can be found in /usr/local/share/doc/exim/\n"
-	       "or on the Exim website at http://www.exim.org/.");
-
-    return DITEM_SUCCESS;
-}
diff --git a/usr.sbin/sysinstall/devices.c b/usr.sbin/sysinstall/devices.c
deleted file mode 100644
index 18a33b536b42..000000000000
--- a/usr.sbin/sysinstall/devices.c
+++ /dev/null
@@ -1,590 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/fcntl.h>
-#include <sys/param.h>
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <sys/errno.h>
-#include <sys/time.h>
-#include <net/if.h>
-#include <net/if_var.h>
-#include <net/if_dl.h>
-#include <netinet/in.h>
-#include <netinet/in_var.h>
-#include <arpa/inet.h>
-#include <ctype.h>
-#include <libdisk.h>
-
-static Device *Devices[DEV_MAX];
-static int numDevs;
-
-#define	DEVICE_ENTRY(type, name, descr, max)	{ type, name, descr, max }
-
-#define	CDROM(name, descr, max)					\
-	DEVICE_ENTRY(DEVICE_TYPE_CDROM, name, descr, max)
-#define	DISK(name, descr, max)						\
-	DEVICE_ENTRY(DEVICE_TYPE_DISK, name, descr, max)
-#define	FLOPPY(name, descr, max)					\
-	DEVICE_ENTRY(DEVICE_TYPE_FLOPPY, name, descr, max)
-#define	NETWORK(name, descr)						\
-	DEVICE_ENTRY(DEVICE_TYPE_NETWORK, name, descr, 0)
-#define	SERIAL(name, descr, max)					\
-	DEVICE_ENTRY(DEVICE_TYPE_NETWORK, name, descr, max)
-#define	USB(name, descr, max)						\
-	DEVICE_ENTRY(DEVICE_TYPE_USB, name, descr, max)
-
-static struct _devname {
-    DeviceType type;
-    char *name;
-    char *description;
-    int max;
-} device_names[] = {
-    CDROM("cd%d",	"SCSI CDROM drive",			4),
-    CDROM("mcd%d",	"Mitsumi (old model) CDROM drive",	4),
-    CDROM("scd%d",	"Sony CDROM drive - CDU31/33A type",	4),
-    CDROM("acd%d",	"ATAPI/IDE CDROM",			4),
-    DISK("da%d",	"SCSI disk device",			16),
-    DISK("ad%d",	"ATA/IDE disk device",			16),
-    DISK("ada%d",	"SATA disk device",			16),
-    DISK("ar%d",	"ATA/IDE RAID device",			16),
-    DISK("afd%d",	"ATAPI/IDE floppy device",		4),
-    DISK("mlxd%d",	"Mylex RAID disk",			4),
-    DISK("amrd%d",	"AMI MegaRAID drive",			4),
-    DISK("idad%d",	"Compaq RAID array",			4),
-    DISK("twed%d",	"3ware ATA RAID array",			4),
-    DISK("aacd%d",	"Adaptec FSA RAID array",		4),
-    DISK("ipsd%d",	"IBM ServeRAID RAID array",		4),
-    DISK("mfid%d",	"LSI MegaRAID SAS array",		4),
-    FLOPPY("fd%d",	"floppy drive unit A",			4),
-    SERIAL("cuau%d",	"%s on device %s (COM%d)",		16),
-    USB("da%da",	"USB Mass Storage Device",		16),
-    NETWORK("ae",	"Attansic/Atheros L2 Fast Ethernet"),
-    NETWORK("age",	"Attansic/Atheros L1 Gigabit Ethernet"),
-    NETWORK("alc",	"Atheros AR8131/AR8132 PCIe Ethernet"),
-    NETWORK("ale",	"Atheros AR8121/AR8113/AR8114 PCIe Ethernet"),
-    NETWORK("an",	"Aironet 4500/4800 802.11 wireless adapter"),
-    NETWORK("ath",	"Atheros IEEE 802.11 wireless adapter"),
-    NETWORK("aue",	"ADMtek USB Ethernet adapter"),
-    NETWORK("axe",	"ASIX Electronics USB Ethernet adapter"),
-    NETWORK("bce",	"Broadcom NetXtreme II Gigabit Ethernet card"),
-    NETWORK("bfe",	"Broadcom BCM440x PCI Ethernet card"),
-    NETWORK("bge",	"Broadcom BCM570x PCI Gigabit Ethernet card"),
-    NETWORK("bm",	"Apple BMAC Built-in Ethernet"),
-    NETWORK("bwn",	"Broadcom BCM43xx IEEE 802.11 wireless adapter"),
-    NETWORK("bxe",	"Broadcom NetXtreme II 10Gb Ethernet card"),
-    NETWORK("cas",	"Sun Cassini/Cassini+ or NS DP83065 Saturn Ethernet"),
-    NETWORK("cue",	"CATC USB Ethernet adapter"),
-    NETWORK("cxgb",	"Chelsio T3 10Gb Ethernet card"),
-    NETWORK("cxgbe",	"Chelsio T4 10Gb Ethernet card"),
-    NETWORK("fpa",	"DEC DEFPA PCI FDDI card"),
-    NETWORK("sr",	"SDL T1/E1 sync serial PCI card"),
-    NETWORK("cc3i",	"SDL HSSI sync serial PCI card"),
-    NETWORK("en",	"Efficient Networks ATM PCI card"),
-    NETWORK("dc",	"DEC/Intel 21143 (and clones) PCI Fast Ethernet card"),
-    NETWORK("de",	"DEC DE435 PCI NIC or other DC21040-AA based card"),
-    NETWORK("fxp",	"Intel EtherExpress Pro/100B PCI Fast Ethernet card"),
-    NETWORK("ed",	"Novell NE1000/2000; 3C503; NE2000-compatible PCMCIA"),
-    NETWORK("ep",	"3Com 3C509 Ethernet card/3C589 PCMCIA"),
-    NETWORK("em",	"Intel(R) PRO/1000 Ethernet card"),
-    NETWORK("et",	"Agere ET1310 based PCI Express Gigabit Ethernet card"),
-    NETWORK("ex",	"Intel EtherExpress Pro/10 Ethernet card"),
-    NETWORK("fe",	"Fujitsu MB86960A/MB86965A Ethernet card"),
-    NETWORK("gem",	"Apple GMAC or Sun ERI/GEM Ethernet adapter"),
-    NETWORK("hme",	"Sun HME (Happy Meal Ethernet) Ethernet adapter"),
-    NETWORK("ie",	"AT&T StarLAN 10 and EN100; 3Com 3C507; NI5210"),
-    NETWORK("igb",	"Intel(R) PRO/1000 PCI Express Gigabit Ethernet card"),
-    NETWORK("ipw",	"Intel PRO/Wireless 2100 IEEE 802.11 adapter"),
-    NETWORK("iwi",	"Intel PRO/Wireless 2200BG/2225BG/2915ABG adapter"),
-    NETWORK("iwn",	"Intel Wireless WiFi Link 4965AGN IEEE 802.11n adapter"),
-    NETWORK("ixgb",	"Intel(R) PRO/10Gb Ethernet card"),
-    NETWORK("ixgbe",	"Intel(R) PRO/10Gb Ethernet card"),
-    NETWORK("jme",	"JMicron JMC250 Gigabit/JMC260 Fast Ethernet"),
-    NETWORK("kue",	"Kawasaki LSI USB Ethernet adapter"),
-    NETWORK("le",	"AMD Am7900 LANCE or Am79C9xx PCnet Ethernet adapter"),
-    NETWORK("lge",	"Level 1 LXT1001 Gigabit Ethernet card"),
-    NETWORK("malo",	"Marvell Libertas 88W8335 802.11 wireless adapter"),
-    NETWORK("msk",	"Marvell/SysKonnect Yukon II Gigabit Ethernet"),
-    NETWORK("mxge",	"Myricom Myri10GE 10Gb Ethernet card"),
-    NETWORK("nfe",	"NVIDIA nForce MCP Ethernet"),
-    NETWORK("nge",	"NatSemi PCI Gigabit Ethernet card"),
-    NETWORK("nve",	"NVIDIA nForce MCP Ethernet"),
-    NETWORK("nxge",	"Neterion Xframe 10GbE Server/Storage adapter"),
-    NETWORK("pcn",	"AMD Am79c79x PCI Ethernet card"),
-    NETWORK("ral",	"Ralink Technology IEEE 802.11 wireless adapter"),
-    NETWORK("ray",	"Raytheon Raylink 802.11 wireless adapter"),
-    NETWORK("re",	"RealTek 8139C+/8169/8169S/8110S PCI Ethernet card"),
-    NETWORK("rl",	"RealTek 8129/8139 PCI Ethernet card"),
-    NETWORK("rue",	"RealTek USB Ethernet card"),
-    NETWORK("rum",	"Ralink Technology USB IEEE 802.11 wireless adapter"),
-    NETWORK("sf",	"Adaptec AIC-6915 PCI Ethernet card"),
-    NETWORK("sge",	"Silicon Integrated Systems SiS190/191 Ethernet"),
-    NETWORK("sis",	"SiS 900/SiS 7016 PCI Ethernet card"),
-#ifdef PC98
-    NETWORK("snc",	"SONIC Ethernet card"),
-#endif
-    NETWORK("sn",	"SMC/Megahertz Ethernet card"),
-    NETWORK("ste",	"Sundance ST201 PCI Ethernet card"),
-    NETWORK("stge",	"Sundance/Tamarack TC9021 Gigabit Ethernet"),
-    NETWORK("sk",	"SysKonnect PCI Gigabit Ethernet card"),
-    NETWORK("tx",	"SMC 9432TX Ethernet card"),
-    NETWORK("txp",	"3Com 3cR990 Ethernet card"),
-    NETWORK("ti",	"Alteon Networks PCI Gigabit Ethernet card"),
-    NETWORK("tl",	"Texas Instruments ThunderLAN PCI Ethernet card"),
-    NETWORK("uath",	"Atheros AR5005UG and AR5005UX USB wireless adapter"),
-    NETWORK("upgt",	"Conexant/Intersil PrismGT USB wireless adapter"),
-    NETWORK("ural",	"Ralink Technology RT2500USB 802.11 wireless adapter"),
-    NETWORK("urtw",	"Realtek 8187L USB wireless adapter"),
-    NETWORK("vge",	"VIA VT612x PCI Gigabit Ethernet card"),
-    NETWORK("vr",	"VIA VT3043/VT86C100A Rhine PCI Ethernet card"),
-    NETWORK("vte",	"DM&P Vortex86 RDC R6040 Fast Ethernet"),
-    NETWORK("vlan",	"IEEE 802.1Q VLAN network interface"),
-    NETWORK("vx",	"3COM 3c590 / 3c595 Ethernet card"),
-    NETWORK("wb",	"Winbond W89C840F PCI Ethernet card"),
-    NETWORK("wi",	"Lucent WaveLAN/IEEE 802.11 wireless adapter"),
-    NETWORK("wpi",	"Intel 3945ABG IEEE 802.11 wireless adapter"),
-    NETWORK("xe",	"Xircom/Intel EtherExpress Pro100/16 Ethernet card"),
-    NETWORK("xl",	"3COM 3c90x / 3c90xB PCI Ethernet card"),
-    NETWORK("zyd",	"ZyDAS ZD1211/ZD1211B USB 802.11 wireless adapter"),
-    NETWORK("fwe",	"FireWire Ethernet emulation"),
-    NETWORK("fwip",	"IP over FireWire"),
-    NETWORK("plip",	"Parallel Port IP (PLIP) peer connection"),
-    NETWORK("lo",	"Loop-back (local) network interface"),
-    NETWORK("disc",	"Software discard network interface"),
-    { 0, NULL, NULL, 0 }
-};
-
-Device *
-new_device(char *name)
-{
-    Device *dev;
-
-    dev = safe_malloc(sizeof(Device));
-    bzero(dev, sizeof(Device));
-    if (name)
-	SAFE_STRCPY(dev->name, name);
-    return dev;
-}
-
-/* Stubs for unimplemented strategy routines */
-Boolean
-dummyInit(Device *dev)
-{
-    return TRUE;
-}
-
-FILE *
-dummyGet(Device *dev, char *dist, Boolean probe)
-{
-    return NULL;
-}
-
-void
-dummyShutdown(Device *dev)
-{
-    return;
-}
-
-static int
-deviceTry(struct _devname dev, char *try, int i)
-{
-    int fd;
-    char unit[80];
-
-    snprintf(unit, sizeof unit, dev.name, i);
-    snprintf(try, FILENAME_MAX, "/dev/%s", unit);
-    if (isDebug())
-	msgDebug("deviceTry: attempting to open %s\n", try);
-    fd = open(try, O_RDONLY);
-    if (fd >= 0) {
-	if (isDebug())
-	    msgDebug("deviceTry: open of %s succeeded on first try.\n", try);
-    } else {
-	if (isDebug())
-	    msgDebug("deviceTry: open of %s failed.\n", try);
-    }
-    return fd;
-}
-
-/* Register a new device in the devices array */
-Device *
-deviceRegister(char *name, char *desc, char *devname, DeviceType type, Boolean enabled,
-	       Boolean (*init)(Device *), FILE * (*get)(Device *, char *, Boolean),
-	       void (*shutdown)(Device *), void *private)
-{
-    Device *newdev = NULL;
-
-    if (numDevs == DEV_MAX)
-	msgFatal("Too many devices found!");
-    else {
-	newdev = new_device(name);
-	newdev->description = desc;
-	newdev->devname = devname;
-	newdev->type = type;
-	newdev->enabled = enabled;
-	newdev->init = init ? init : dummyInit;
-	newdev->get = get ? get : dummyGet;
-	newdev->shutdown = shutdown ? shutdown : dummyShutdown;
-	newdev->private = private;
-	Devices[numDevs] = newdev;
-	Devices[++numDevs] = NULL;
-    }
-    return newdev;
-}
-
-/* Reset the registered device chain */
-void
-deviceReset(void)
-{
-    int i;
-
-    for (i = 0; i < numDevs; i++) {
-	DEVICE_SHUTDOWN(Devices[i]);
-
-	/* XXX this potentially leaks Devices[i]->private if it's being
-	 * used to point to something dynamic, but you're not supposed
-	 * to call this routine at such times that some open instance
-	 * has its private ptr pointing somewhere anyway. XXX
-	 */
-	free(Devices[i]);
-    }
-    Devices[numDevs = 0] = NULL;
-}
-
-/* Get all device information for devices we have attached */
-void
-deviceGetAll(void)
-{
-    int i, j, fd, s;
-    struct ifconf ifc;
-    struct ifreq *ifptr, *end;
-    int ifflags;
-    char buffer[INTERFACE_MAX * sizeof(struct ifreq)];
-    char **names;
-
-    msgNotify("Probing devices, please wait (this can take a while)...");
-    /* First go for the network interfaces.  Stolen shamelessly from ifconfig! */
-    memset(&ifc, 0, sizeof(ifc));
-    memset(buffer, 0, INTERFACE_MAX * sizeof(struct ifreq));
-    ifc.ifc_len = sizeof(buffer);
-    ifc.ifc_buf = buffer;
-
-    s = socket(AF_INET, SOCK_DGRAM, 0);
-    if (s < 0)
-	goto skipif;	/* Jump over network iface probing */
-
-    if (ioctl(s, SIOCGIFCONF, (char *) &ifc) < 0)
-	goto skipif;	/* Jump over network iface probing */
-
-    close(s);
-    ifflags = ifc.ifc_req->ifr_flags;
-    end = (struct ifreq *) (ifc.ifc_buf + ifc.ifc_len);
-    for (ifptr = ifc.ifc_req; ifptr < end; ifptr++) {
-	char *descr;
-
-	/* If it's not a link entry, forget it */
-	if (ifptr->ifr_ifru.ifru_addr.sa_family != AF_LINK)
-	    goto loopend;
-
-	/* Eliminate network devices that don't make sense */
-	if (!strncmp(ifptr->ifr_name, "lo", 2))
-	    goto loopend;
-
-	/* Try and find its description */
-	for (i = 0, descr = NULL; device_names[i].name; i++) {
-	    int len = strlen(device_names[i].name);
-
-	    if (!ifptr->ifr_name || !ifptr->ifr_name[0])
-		continue;
-	    else if (!strncmp(ifptr->ifr_name, device_names[i].name, len)) {
-		descr = device_names[i].description;
-		break;
-	    }
-	}
-	if (!descr)
-	    descr = "<unknown network interface type>";
-
-	deviceRegister(ifptr->ifr_name, descr, strdup(ifptr->ifr_name), DEVICE_TYPE_NETWORK, TRUE,
-		       mediaInitNetwork, NULL, mediaShutdownNetwork, NULL);
-	if (isDebug())
-	    msgDebug("Found a network device named %s\n", ifptr->ifr_name);
-	close(s);
-	if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
-	    continue;
-
-loopend:
-	if (ifptr->ifr_addr.sa_len)	/* I'm not sure why this is here - it's inherited */
-	    ifptr = (struct ifreq *)((caddr_t)ifptr + ifptr->ifr_addr.sa_len - sizeof(struct sockaddr));
-	close(s);
-    }
-
-skipif:
-    /* Next, try to find all the types of devices one might need
-     * during the second stage of the installation.
-     */
-    for (i = 0; device_names[i].name; i++) {
-	for (j = 0; j < device_names[i].max; j++) {
-	    char try[FILENAME_MAX];
-
-	    switch(device_names[i].type) {
-	    case DEVICE_TYPE_CDROM:
-		fd = deviceTry(device_names[i], try, j);
-		if (fd >= 0 || errno == EBUSY) {	/* EBUSY if already mounted */
-		    char n[BUFSIZ];
-
-		    if (fd >= 0) close(fd);
-		    snprintf(n, sizeof n, device_names[i].name, j);
-		    deviceRegister(n, device_names[i].description, strdup(try),
-					 DEVICE_TYPE_CDROM, TRUE, mediaInitCDROM, mediaGetCDROM,
-					 mediaShutdownCDROM, NULL);
-		    if (isDebug())
-			msgDebug("Found a CDROM device for %s\n", try);
-		}
-		break;
-
-	    case DEVICE_TYPE_DISK:
-		/* nothing to do */
-		break;
-
-	    case DEVICE_TYPE_FLOPPY:
-		fd = deviceTry(device_names[i], try, j);
-		if (fd >= 0) {
-		    char n[BUFSIZ];
-
-		    close(fd);
-		    snprintf(n, sizeof n, device_names[i].name, j);
-		    deviceRegister(n, device_names[i].description, strdup(try),
-				   DEVICE_TYPE_FLOPPY, TRUE, mediaInitFloppy, mediaGetFloppy,
-				   mediaShutdownFloppy, NULL);
-		    if (isDebug())
-			msgDebug("Found a floppy device for %s\n", try);
-		}
-		break;
-
-	    case DEVICE_TYPE_USB:
-		fd = deviceTry(device_names[i], try, j);
-		if (fd >= 0) {
-			char n[BUFSIZ];
-
-			close(fd);
-			snprintf(n, sizeof(n), device_names[i].name, j);
-			deviceRegister(n, device_names[i].description,
-			    strdup(try), DEVICE_TYPE_USB, TRUE, mediaInitUSB,
-			    mediaGetUSB, mediaShutdownUSB, NULL);
-
-			if (isDebug())
-				msgDebug("Found a USB disk for %s\n", try);
-		}
-		break;
-
-	    default:
-		break;
-	    }
-	}
-    }
-
-    /* Finally, go get the disks and look for partitions to register */
-    if ((names = Disk_Names()) != NULL) {
-	int i;
-
-	for (i = 0; names[i]; i++) {
-	    Chunk *c1;
-	    Disk *d;
-
-	    /* Ignore memory disks */
-	    if (!strncmp(names[i], "md", 2))
-		continue;
-
-	    /*
-	     * XXX 
-	     *  Due to unknown reasons, Disk_Names() returns SCSI CDROM as a
-	     * valid disk. This is main reason why sysinstall presents SCSI
-	     * CDROM to available disks in Fdisk/Label menu. In addition,
-	     * adding a blank SCSI CDROM to the menu generates floating point
-	     * exception in sparc64. Disk_Names() just extracts sysctl
-	     * "kern.disks". Why GEOM treats SCSI CDROM as a disk is beyond
-	     * me and that should be investigated.
-	     * For temporary workaround, ignore SCSI CDROM device.
-	     */
-	    if (!strncmp(names[i], "cd", 2))
-		continue;
-
-	    d = Open_Disk(names[i]);
-	    if (!d) {
-		msgDebug("Unable to open disk %s\n", names[i]);
-		continue;
-	    }
-
-	    deviceRegister(names[i], names[i], d->name, DEVICE_TYPE_DISK, FALSE,
-			   dummyInit, dummyGet, dummyShutdown, d);
-	    if (isDebug())
-		msgDebug("Found a disk device named %s\n", names[i]);
-
-	    /* Look for existing DOS partitions to register as "DOS media devices"
-	     * XXX: libdisks handling of extended partitions is too
-	     * simplistic - it does not handle them containing (for
-	     * example) UFS partitions
-	     */
-	    for (c1 = d->chunks->part; c1; c1 = c1->next) {
-		if (c1->type == fat || c1->type == efi || c1->type == extended) {
-		    Device *dev;
-		    char devname[80];
-
-		    /* Got one! */
-		    snprintf(devname, sizeof devname, "/dev/%s", c1->name);
-		    dev = deviceRegister(c1->name, c1->name, strdup(devname), DEVICE_TYPE_DOS, TRUE,
-					 mediaInitDOS, mediaGetDOS, mediaShutdownDOS, NULL);
-		    dev->private = c1;
-		    if (isDebug())
-			msgDebug("Found a DOS partition %s\n", c1->name);
-		} else if (c1->type == freebsd) {
-		    Device *dev;
-		    char devname[80];
-		    Chunk *c2;
-			
-		    for (c2 = c1->part; c2; c2 = c2->next) {
-			if (c2->type != part || c2->subtype != 7)
-			    continue;
-			/* Got one! */
-			snprintf(devname, sizeof devname, "/dev/%s", c1->name);
-			dev = deviceRegister(c2->name, c2->name, strdup(devname), DEVICE_TYPE_UFS, TRUE,
-					     mediaInitUFS, mediaGetUFS, mediaShutdownUFS, NULL);
-			dev->private = c2;
-			if (isDebug())
-			    msgDebug("Found a UFS sub-partition %s\n", c2->name);
-		    }
-		}
-		
-	    }
-	}
-	free(names);
-    }
-    dialog_clear_norefresh();
-}
-
-/* Rescan all devices, after closing previous set - convenience function */
-void
-deviceRescan(void)
-{
-    deviceReset();
-    deviceGetAll();
-}
-
-/*
- * Find all devices that match the criteria, allowing "wildcarding" as well
- * by allowing NULL or ANY values to match all.  The array returned is static
- * and may be used until the next invocation of deviceFind().
- */
-Device **
-deviceFind(char *name, DeviceType class)
-{
-    static Device *found[DEV_MAX];
-    int i, j;
-
-    j = 0;
-    for (i = 0; i < numDevs; i++) {
-	if ((!name || !strcmp(Devices[i]->name, name))
-	    && (class == DEVICE_TYPE_ANY || class == Devices[i]->type))
-	    found[j++] = Devices[i];
-    }
-    found[j] = NULL;
-    return j ? found : NULL;
-}
-
-Device **
-deviceFindDescr(char *name, char *desc, DeviceType class)
-{
-    static Device *found[DEV_MAX];
-    int i, j;
-
-    j = 0;
-    for (i = 0; i < numDevs; i++) {
-	if ((!name || !strcmp(Devices[i]->name, name)) &&
-	    (!desc || !strcmp(Devices[i]->description, desc)) &&
-	    (class == DEVICE_TYPE_ANY || class == Devices[i]->type))
-	    found[j++] = Devices[i];
-    }
-    found[j] = NULL;
-    return j ? found : NULL;
-}
-
-int
-deviceCount(Device **devs)
-{
-    int i;
-
-    if (!devs)
-	return 0;
-    for (i = 0; devs[i]; i++);
-    return i;
-}
-
-/*
- * Create a menu listing all the devices of a certain type in the system.
- * The passed-in menu is expected to be a "prototype" from which the new
- * menu is cloned.
- */
-DMenu *
-deviceCreateMenu(DMenu *menu, DeviceType type, int (*hook)(dialogMenuItem *d), int (*check)(dialogMenuItem *d))
-{
-    Device **devs;
-    int numdevs;
-    DMenu *tmp = NULL;
-    int i, j;
-
-    devs = deviceFind(NULL, type);
-    numdevs = deviceCount(devs);
-    if (!numdevs)
-	return NULL;
-    tmp = (DMenu *)safe_malloc(sizeof(DMenu) + (sizeof(dialogMenuItem) * (numdevs + 1)));
-    bcopy(menu, tmp, sizeof(DMenu));
-    for (i = 0; devs[i]; i++) {
-	tmp->items[i].prompt = devs[i]->name;
-	for (j = 0; j < numDevs; j++) {
-	    if (devs[i] == Devices[j]) {
-		tmp->items[i].title = Devices[j]->description;
-		break;
-	    }
-	}
-	if (j == numDevs)
-	    tmp->items[i].title = "<unknown device type>";
-	tmp->items[i].fire = hook;
-	tmp->items[i].checked = check;
-    }
-    tmp->items[i].title = NULL;
-    return tmp;
-}
diff --git a/usr.sbin/sysinstall/dhcp.c b/usr.sbin/sysinstall/dhcp.c
deleted file mode 100644
index fa35580e1235..000000000000
--- a/usr.sbin/sysinstall/dhcp.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * $FreeBSD$
- *
- * Copyright (c) 1999
- *	C. Stone.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY C. STONE ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL C STONE OR HIS BODILY PARASITES BE LIABLE FOR ANY DIRECT,
- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE BY THE VOICES IN YOUR HEAD BEFOREHAND.
- *
- */
-
-#include "sysinstall.h"
-
-#include <ctype.h>
-
-int
-dhcpParseLeases(char *file, char *hostname, char *domain, char *nameserver,
-		char *ipaddr, char *gateway, char *netmask)
-{
-    char tempbuf[1024];
-    char optbuf[1024], *optname = NULL;
-    char *tptr;
-    int endedflag = 0;
-    int leaseflag = 0;
-    FILE *fp;
-    enum { P_NOSTMT, P_NOSTMT1, P_STMT, P_STMTLINE } state;
-
-    if ((fp = fopen(file, "r")) == NULL) {
-	msgDebug("error opening file %s: %s\n", file, strerror(errno));
-	return -1;
-    }
-
-    state = P_NOSTMT;
-    while (fscanf(fp, "%1023s", tempbuf) > 0) {
-    	switch (state) {
-	case P_NOSTMT:
-	    state = P_NOSTMT1;
-	    if (!strncasecmp(tempbuf, "lease", 5)) {
-		if (!leaseflag)
-		    leaseflag = 1;
-		else {
-		    fclose(fp);
-		    return 0;
-		}
-	    }
-	    break;
-
-	case P_NOSTMT1: 
-	    if (tempbuf[0] != '{') {
-		msgWarn("dhcpParseLeases: '{' expected");
-		fclose(fp);
-		return -1;
-	    }
-	    state = P_STMT;
-	    break;
-
-	case P_STMT:
-	    if (!strncasecmp("option", tempbuf, 6))
-		continue;
-	    if (tempbuf[0] == '}') {
-		state = P_NOSTMT;
-		leaseflag = 0;
-		continue;
-	    }
-	    if (!leaseflag) 
-		break;
-	    if (tempbuf[0] == ';') { 	/* play it safe */
-		state = P_STMT;
-		continue;
-	    }
-	    if ((tptr = (char *)strchr(tempbuf, ';')) && (*(tptr + 1) == 0)) {
-		*tptr = '\0';
-		endedflag = 1;
-	    }
-	    if (!isalnum(tempbuf[0])) {
-		msgWarn("dhcpParseLeases: bad option");
-		fclose(fp);
-		return -1;
-	    }
-	    if (optname)
-		free(optname);
-	    optname = strdup(tempbuf);
-	    if (endedflag) {
-		state = P_STMT;
-		endedflag = 0;
-		continue;
-	    }
-	    state = P_STMTLINE;
-	    break;
-
-	case P_STMTLINE:
-	    if (tempbuf[0] == ';') {
-		state = P_STMT;
-		continue;
-	    }
-	    if ((tptr = (char *)strchr(tempbuf, ';')) && (*(tptr + 1) == 0)) {
-		*tptr = '\0';
-		endedflag = 1;
-	    }
-	    if (tempbuf[0] == '"') {
-		if (sscanf(tempbuf, "\"%[^\" ]\"", optbuf) < 1) {
-		    msgWarn("dhcpParseLeases: bad option value");
-		    fclose(fp);
-		    return -1;
-		}
-	    }
-	    else
-		strcpy(optbuf, tempbuf);
-
-	    if (!strcasecmp("host-name", optname)) {
-		strcpy(hostname, optbuf);
-	    } else if (!strcasecmp("domain-name", optname)) {
-		strcpy(domain, optbuf);
-	    } else if (!strcasecmp("fixed-address", optname)) {
-		strcpy(ipaddr, optbuf);
-	    } else if (!strcasecmp("routers", optname)) {
-		if((tptr = (char *)strchr(optbuf, ',')))
-		    *tptr = '\0';
-		strcpy(gateway, optbuf);
-	    } else if (!strcasecmp("subnet-mask", optname)) {
-		strcpy(netmask, optbuf);
-	    } else if (!strcasecmp("domain-name-servers", optname)) {
-		/* <jkh> ...one value per property */
-		if((tptr = (char *)strchr(optbuf, ',')))
-		    *tptr = '\0';
-		strcpy(nameserver, optbuf);
-	    }
-	    if (endedflag) {
-		state = P_STMT;
-		endedflag = 0;
-		continue;
-	    }
-	    break;
-	}
-    }
-    fclose(fp);
-    return 0;
-}
diff --git a/usr.sbin/sysinstall/disks.c b/usr.sbin/sysinstall/disks.c
deleted file mode 100644
index 4cc78481e2a6..000000000000
--- a/usr.sbin/sysinstall/disks.c
+++ /dev/null
@@ -1,1055 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <fcntl.h>
-#include <inttypes.h>
-#include <libdisk.h>
-#include <sys/stat.h>
-#include <sys/disklabel.h>
-
-#ifdef WITH_SLICES
-enum size_units_t { UNIT_BLOCKS, UNIT_KILO, UNIT_MEG, UNIT_GIG, UNIT_SIZE };
-
-#ifdef PC98
-#define	SUBTYPE_FREEBSD		50324
-#define	SUBTYPE_FAT		37218
-#else
-#define	SUBTYPE_FREEBSD		165
-#define	SUBTYPE_FAT		6
-#endif
-#define	SUBTYPE_EFI		239
-
-#ifdef PC98
-#define	OTHER_SLICE_VALUES						\
-	"Other popular values are 37218 for a\n"			\
-	"DOS FAT partition.\n\n"
-#else
-#define	OTHER_SLICE_VALUES						\
-	"Other popular values are 6 for a\n"				\
-	"DOS FAT partition, 131 for a Linux ext2fs partition, or\n"	\
-	"130 for a Linux swap partition.\n\n"
-#endif
-#define	NON_FREEBSD_NOTE						\
-	"Note:  If you choose a non-FreeBSD partition type, it will not\n" \
-	"be formatted or otherwise prepared, it will simply reserve space\n" \
-	"for you to use another tool, such as DOS format, to later format\n" \
-	"and actually use the partition."
-
-/* Where we start displaying chunk information on the screen */
-#define CHUNK_START_ROW		5
-
-/* Where we keep track of MBR chunks */
-#define	CHUNK_INFO_ENTRIES	16
-static struct chunk *chunk_info[CHUNK_INFO_ENTRIES];
-static int current_chunk;
-
-static void	diskPartitionNonInteractive(Device *dev);
-
-static void
-record_chunks(Disk *d)
-{
-    struct chunk *c1 = NULL;
-    int i = 0;
-    daddr_t last_free = 0;
-
-    if (!d->chunks)
-	msgFatal("No chunk list found for %s!", d->name);
-
-    for (c1 = d->chunks->part; c1; c1 = c1->next) {
-	if (c1->type == unused && c1->size > last_free) {
-	    last_free = c1->size;
-	    current_chunk = i;
-	}
-	chunk_info[i++] = c1;
-    }
-    chunk_info[i] = NULL;
-    if (current_chunk >= i)
-	current_chunk = i - 1;
-}
-
-static daddr_t Total;
-
-static void
-check_geometry(Disk *d)
-{
-    int sg;
-
-#ifdef PC98
-    if (d->bios_cyl >= 65536 || d->bios_hd > 256 || d->bios_sect >= 256)
-#else
-    if (d->bios_cyl > 65536 || d->bios_hd > 256 || d->bios_sect >= 64)
-#endif
-    {
-	dialog_clear_norefresh();
-	sg = msgYesNo("WARNING:  It is safe to use a geometry of %lu/%lu/%lu for %s on\n"
-		      "computers with modern BIOS versions.  If this disk is to be used\n"
-		      "on an old machine it is recommended that it does not have more\n"
-		      "than 65535 cylinders, more than 255 heads, or more than\n"
-#ifdef PC98
-		      "255"
-#else
-		      "63"
-#endif
-		      " sectors per track.\n"
-		      "\n"
-		      "Would you like to keep using the current geometry?\n",
-		      d->bios_cyl, d->bios_hd, d->bios_sect, d->name);
-	if (sg == 1) {
-	    Sanitize_Bios_Geom(d);
-	    msgConfirm("A geometry of %lu/%lu/%lu was calculated for %s.\n"
-		       "\n"
-		       "If you are not sure about this, please consult the Hardware Guide\n"
-		       "in the Documentation submenu or use the (G)eometry command to\n"
-		       "change it.  Remember: you need to enter whatever your BIOS thinks\n"
-		       "the geometry is!  For IDE, it's what you were told in the BIOS\n"
-		       "setup.  For SCSI, it's the translation mode your controller is\n"
-		       "using.  Do NOT use a ``physical geometry''.\n",
-		       d->bios_cyl, d->bios_hd, d->bios_sect, d->name);
-	}
-    }
-}
-
-static void
-print_chunks(Disk *d, int u)
-{
-    int row;
-    int i;
-    daddr_t sz;
-    char *szstr;
-
-    szstr = (u == UNIT_GIG ? "GB" : (u == UNIT_MEG ? "MB" :
-	(u == UNIT_KILO ? "KB" : "ST")));
-
-    Total = 0;
-    for (i = 0; chunk_info[i]; i++)
-	Total += chunk_info[i]->size;
-    attrset(A_NORMAL);
-    mvaddstr(0, 0, "Disk name:\t");
-    clrtobot();
-    attrset(A_REVERSE); addstr(d->name); attrset(A_NORMAL);
-    attrset(A_REVERSE); mvaddstr(0, 55, "FDISK Partition Editor"); attrset(A_NORMAL);
-    mvprintw(1, 0,
-	     "DISK Geometry:\t%lu cyls/%lu heads/%lu sectors = %jd sectors (%jdMB)",
-	     d->bios_cyl, d->bios_hd, d->bios_sect,
-	     (intmax_t)d->bios_cyl * d->bios_hd * d->bios_sect,
-	     (intmax_t)d->bios_cyl * d->bios_hd * d->bios_sect / (1024/512) / 1024);
-    mvprintw(3, 0, "%6s %10s(%s) %10s %8s %6s %10s %8s %8s",
-	     "Offset", "Size", szstr, "End", "Name", "PType", "Desc",
-	     "Subtype", "Flags");
-    for (i = 0, row = CHUNK_START_ROW; chunk_info[i]; i++, row++) {
-	switch(u) {
-	default:	/* fall thru */
-	case UNIT_BLOCKS:
-	    sz = chunk_info[i]->size;
-	    break;
-	case UNIT_KILO:
-	    sz = chunk_info[i]->size / (1024/512);
-	    break;
-	case UNIT_MEG:
-	    sz = chunk_info[i]->size / (1024/512) / 1024;
-	    break;
-	case UNIT_GIG:
-	    sz = chunk_info[i]->size / (1024/512) / 1024 / 1024;
-	    break;
-	}
-	if (i == current_chunk)
-	    attrset(ATTR_SELECTED);
-	mvprintw(row, 0, "%10jd %10jd %10jd %8s %6d %10s %8d\t%-6s",
-		 (intmax_t)chunk_info[i]->offset, (intmax_t)sz,
-		 (intmax_t)chunk_info[i]->end, chunk_info[i]->name,
-		 chunk_info[i]->type, 
-		 slice_type_name(chunk_info[i]->type, chunk_info[i]->subtype),
-		 chunk_info[i]->subtype, ShowChunkFlags(chunk_info[i]));
-	if (i == current_chunk)
-	    attrset(A_NORMAL);
-    }
-}
-
-static void
-print_command_summary(void)
-{
-    mvprintw(14, 0, "The following commands are supported (in upper or lower case):");
-    mvprintw(16, 0, "A = Use Entire Disk   G = set Drive Geometry   C = Create Slice");
-    mvprintw(17, 0, "D = Delete Slice      Z = Toggle Size Units    S = Set Bootable   | = Expert m.");
-    mvprintw(18, 0, "T = Change Type       U = Undo All Changes");
-
-    if (!RunningAsInit)
-	mvprintw(18, 47, "W = Write Changes  Q = Finish");
-    else
-	mvprintw(18, 47, "Q = Finish");
-    mvprintw(21, 0, "Use F1 or ? to get more help, arrow keys to select.");
-    move(0, 0);
-}
-#endif /* WITH_SLICES */
-
-#if !defined(__ia64__)
-static u_char *
-bootalloc(char *name, size_t *size)
-{
-    char buf[FILENAME_MAX];
-    struct stat sb;
-
-    snprintf(buf, sizeof buf, "/boot/%s", name);
-    if (stat(buf, &sb) != -1) {
-	int fd;
-
-	fd = open(buf, O_RDONLY);
-	if (fd != -1) {
-	    u_char *cp;
-
-	    cp = malloc(sb.st_size);
-	    if (read(fd, cp, sb.st_size) != sb.st_size) {
-		free(cp);
-		close(fd);
-		msgDebug("bootalloc: couldn't read %ld bytes from %s\n", (long)sb.st_size, buf);
-		return NULL;
-	    }
-	    close(fd);
-	    if (size != NULL)
-		*size = sb.st_size;
-	    return cp;
-	}
-	msgDebug("bootalloc: couldn't open %s\n", buf);
-    }
-    else
-	msgDebug("bootalloc: can't stat %s\n", buf);
-    return NULL;
-}
-#endif /* !defined(__ia64__) */
-
-#ifdef WITH_SLICES
-#ifdef PC98
-static void
-getBootMgr(char *dname, u_char **bootipl, size_t *bootipl_size,
-	   u_char **bootmenu, size_t *bootmenu_size)
-{
-    static u_char *boot0;
-    static size_t boot0_size;
-    static u_char *boot05;
-    static size_t boot05_size;
-
-    char str[80];
-    char *cp;
-    int i = 0;
-
-    cp = variable_get(VAR_BOOTMGR);
-    if (!cp) {
-	/* Figure out what kind of IPL the user wants */
-	sprintf(str, "Install Boot Manager for drive %s?", dname);
-	MenuIPLType.title = str;
-	i = dmenuOpenSimple(&MenuIPLType, FALSE);
-    } else {
-	if (!strncmp(cp, "boot", 4))
-	    BootMgr = 0;
-	else
-	    BootMgr = 1;
-    }
-    if (cp || i) {
-	switch (BootMgr) {
-	case 0:
-	    if (!boot0) boot0 = bootalloc("boot0", &boot0_size);
-	    *bootipl = boot0;
-	    *bootipl_size = boot0_size;
-	    if (!boot05) boot05 = bootalloc("boot0.5", &boot05_size);
-	    *bootmenu = boot05;
-	    *bootmenu_size = boot05_size;
-	    return;
-	case 1:
-	default:
-	    break;
-	}
-    }
-    *bootipl = NULL;
-    *bootipl_size = 0;
-    *bootmenu = NULL;
-    *bootmenu_size = 0;
-}
-#else
-static void
-getBootMgr(char *dname, u_char **bootCode, size_t *bootCodeSize)
-{
-#if defined(__i386__) || defined(__amd64__)	/* only meaningful on x86 */
-    static u_char *mbr, *boot0;
-    static size_t mbr_size, boot0_size;
-    char str[80];
-    char *cp;
-    int i = 0;
-
-    cp = variable_get(VAR_BOOTMGR);
-    if (!cp) {
-	/* Figure out what kind of MBR the user wants */
-	sprintf(str, "Install Boot Manager for drive %s?", dname);
-	MenuMBRType.title = str;
-	i = dmenuOpenSimple(&MenuMBRType, FALSE);
-    }
-    else {
-	if (!strncmp(cp, "boot", 4))
-	    BootMgr = 0;
-	else if (!strcmp(cp, "standard"))
-	    BootMgr = 1;
-	else
-	    BootMgr = 2;
-    }
-    if (cp || i) {
-	switch (BootMgr) {
-	case 0:
-	    if (!boot0) boot0 = bootalloc("boot0", &boot0_size);
-	    *bootCode = boot0;
-	    *bootCodeSize = boot0_size;
-	    return;
-	case 1:
-	    if (!mbr) mbr = bootalloc("mbr", &mbr_size);
-	    *bootCode = mbr;
-	    *bootCodeSize = mbr_size;
-	    return;
-	case 2:
-	default:
-	    break;
-	}
-    }
-#endif
-    *bootCode = NULL;
-    *bootCodeSize = 0;
-}
-#endif
-#endif /* WITH_SLICES */
-
-int
-diskGetSelectCount(Device ***devs)
-{
-    int i, cnt, enabled;
-    char *cp;
-    Device **dp;
-
-    cp = variable_get(VAR_DISK);
-    dp = *devs = deviceFind(cp, DEVICE_TYPE_DISK);
-    cnt = deviceCount(dp);
-    if (!cnt)
-	return -1;
-    for (i = 0, enabled = 0; i < cnt; i++) {
-	if (dp[i]->enabled)
-	    ++enabled;
-    }
-    return enabled;
-}
-
-#ifdef WITH_SLICES
-void
-diskPartition(Device *dev)
-{
-    char *p;
-    int rv, key = 0;
-    int i;
-    Boolean chunking;
-    char *msg = NULL;
-#ifdef PC98
-    u_char *bootipl;
-    size_t bootipl_size;
-    u_char *bootmenu;
-    size_t bootmenu_size;
-#else
-    u_char *mbrContents;
-    size_t mbrSize;
-#endif
-    WINDOW *w = savescr();
-    Disk *d = (Disk *)dev->private;
-    int size_unit;
-
-    size_unit = UNIT_BLOCKS;
-    chunking = TRUE;
-    keypad(stdscr, TRUE);
-
-    /* Flush both the dialog and curses library views of the screen
-       since we don't always know who called us */
-    dialog_clear_norefresh(), clear();
-    current_chunk = 0;
-
-    /* Set up the chunk array */
-    record_chunks(d);
-
-    /* Give the user a chance to sanitize the disk geometry, if necessary */
-    check_geometry(d);
-
-    while (chunking) {
-	char *val, geometry[80];
-	    
-	/* Now print our overall state */
-	if (d)
-	    print_chunks(d, size_unit);
-	print_command_summary();
-	if (msg) {
-	    attrset(title_attr); mvprintw(23, 0, msg); attrset(A_NORMAL);
-	    beep();
-	    msg = NULL;
-	}
-	else {
-	    move(23, 0);
-	    clrtoeol();
-	}
-
-	/* Get command character */
-	key = getch();
-	switch (toupper(key)) {
-	case '\014':	/* ^L (redraw) */
-	    clear();
-	    msg = NULL;
-	    break;
-	    
-	case '\020':	/* ^P */
-	case KEY_UP:
-	case '-':
-	    if (current_chunk != 0)
-		--current_chunk;
-	    break;
-	    
-	case '\016':	/* ^N */
-	case KEY_DOWN:
-	case '+':
-	case '\r':
-	case '\n':
-	    if (chunk_info[current_chunk + 1])
-		++current_chunk;
-	    break;
-
-	case KEY_HOME:
-	    current_chunk = 0;
-	    break;
-
-	case KEY_END:
-	    while (chunk_info[current_chunk + 1])
-		++current_chunk;
-	    break;
-
-	case KEY_F(1):
-	case '?':
-	    systemDisplayHelp("slice");
-	    clear();
-	    break;
-
-	case 'A':
-#if !defined(__i386__) && !defined(__amd64__)
-	    rv = 1;
-#else	    /* The rest is only relevant on x86 */
-	    rv = 0;
-#endif
-	    All_FreeBSD(d, rv);
-	    variable_set2(DISK_PARTITIONED, "yes", 0);
-	    record_chunks(d);
-	    clear();
-	    break;
-	    
-	case 'C':
-	    if (chunk_info[current_chunk]->type != unused)
-		msg = "Slice in use, delete it first or move to an unused one.";
-	    else {
-		char *val, tmp[20], name[16], *cp;
-		daddr_t size;
-		long double dsize;
-		int subtype;
-		chunk_e partitiontype;
-#ifdef PC98
-		snprintf(name, sizeof (name), "%s", "FreeBSD");
-		val = msgGetInput(name,
-			"Please specify the name for new FreeBSD slice.");
-		if (val)
-			strncpy(name, val, sizeof (name));
-#else
-		name[0] = '\0';
-#endif
-		snprintf(tmp, 20, "%jd", (intmax_t)chunk_info[current_chunk]->size);
-		val = msgGetInput(tmp, "Please specify the size for new FreeBSD slice in blocks\n"
-				  "or append a trailing `M' for megabytes (e.g. 20M).");
-		if (val && (dsize = strtold(val, &cp)) > 0 && dsize < UINT32_MAX) {
-		    if (*cp && toupper(*cp) == 'M')
-			size = (daddr_t) (dsize * ONE_MEG);
-		    else if (*cp && toupper(*cp) == 'G')
-			size = (daddr_t) (dsize * ONE_GIG);
-		    else
-			size = (daddr_t) dsize;
-
-		    if (size < ONE_MEG) {
-			msgConfirm("The minimum slice size is 1MB");
-			break;
-		    }
-
-
-		    sprintf(tmp, "%d", SUBTYPE_FREEBSD);
-		    val = msgGetInput(tmp, "Enter type of partition to create:\n\n"
-			"Pressing Enter will choose the default, a native FreeBSD\n"
-			"slice (type %u).  "
-			OTHER_SLICE_VALUES
-			NON_FREEBSD_NOTE, SUBTYPE_FREEBSD);
-		    if (val && (subtype = strtol(val, NULL, 0)) > 0) {
-			if (subtype == SUBTYPE_FREEBSD)
-			    partitiontype = freebsd;
-			else if (subtype == SUBTYPE_FAT)
-			    partitiontype = fat;
-			else if (subtype == SUBTYPE_EFI)
-			    partitiontype = efi;
-			else
-#ifdef PC98
-			    partitiontype = pc98;
-#else
-			    partitiontype = mbr;
-#endif
-			Create_Chunk(d, chunk_info[current_chunk]->offset, size, partitiontype, subtype,
-				     (chunk_info[current_chunk]->flags & CHUNK_ALIGN), name);
-			variable_set2(DISK_PARTITIONED, "yes", 0);
-			record_chunks(d);
-		    }
-		}
-		clear();
-	    }
-	    break;
-	    
-	case KEY_DC:
-	case 'D':
-	    if (chunk_info[current_chunk]->type == unused)
-		msg = "Slice is already unused!";
-	    else {
-		Delete_Chunk(d, chunk_info[current_chunk]);
-		variable_set2(DISK_PARTITIONED, "yes", 0);
-		record_chunks(d);
-	    }
-	    break;
-	    
-	case 'T':
-	    if (chunk_info[current_chunk]->type == unused)
-		msg = "Slice is currently unused (use create instead)";
-	    else {
-		char *val, tmp[20];
-		int subtype;
-		chunk_e partitiontype;
-
-		sprintf(tmp, "%d", chunk_info[current_chunk]->subtype);
-		val = msgGetInput(tmp, "New partition type:\n\n"
-		    "Pressing Enter will use the current type. To choose a native\n"
-		    "FreeBSD slice enter %u.  "
-		    OTHER_SLICE_VALUES
-		    NON_FREEBSD_NOTE, SUBTYPE_FREEBSD);
-		if (val && (subtype = strtol(val, NULL, 0)) > 0) {
-		    if (subtype == SUBTYPE_FREEBSD)
-			partitiontype = freebsd;
-		    else if (subtype == SUBTYPE_FAT)
-			partitiontype = fat;
-		    else if (subtype == SUBTYPE_EFI)
-			partitiontype = efi;
-		    else
-#ifdef PC98
-			partitiontype = pc98;
-#else
-			partitiontype = mbr;
-#endif
-		    chunk_info[current_chunk]->type = partitiontype;
-		    chunk_info[current_chunk]->subtype = subtype;
-		}
-	    }
-	    break;
-	    
-	case 'G':
-	    snprintf(geometry, 80, "%lu/%lu/%lu", d->bios_cyl, d->bios_hd, d->bios_sect);
-	    val = msgGetInput(geometry, "Please specify the new geometry in cyl/hd/sect format.\n"
-			      "Don't forget to use the two slash (/) separator characters!\n"
-			      "It's not possible to parse the field without them.");
-	    if (val) {
-		long nc, nh, ns;
-		nc = strtol(val, &val, 0);
-		nh = strtol(val + 1, &val, 0);
-		ns = strtol(val + 1, 0, 0);
-		Set_Bios_Geom(d, nc, nh, ns);
-	    }
-	    clear();
-	    break;
-	
-	case 'S':
-	    /* Clear active states so we won't have two */
-	    for (i = 0; (chunk_info[i] != NULL) && (i < CHUNK_INFO_ENTRIES); i++)
-		chunk_info[i]->flags &= !CHUNK_ACTIVE;
-
-	    /* Set Bootable */
-	    chunk_info[current_chunk]->flags |= CHUNK_ACTIVE;
-	    break;
-	
-	case 'U':
-	    if (!variable_cmp(DISK_LABELLED, "written")) {
-		msgConfirm("You've already written this information out - you\n"
-			   "can't undo it.");
-	    }
-	    else if (!msgNoYes("Are you SURE you want to Undo everything?")) {
-		char cp[BUFSIZ];
-
-		sstrncpy(cp, d->name, sizeof cp);
-		Free_Disk(dev->private);
-		d = Open_Disk(cp);
-		if (!d)
-		    msgConfirm("Can't reopen disk %s! Internal state is probably corrupted", cp);
-		dev->private = d;
-		variable_unset(DISK_PARTITIONED);
-		variable_unset(DISK_LABELLED);
-		if (d)
-		    record_chunks(d);
-	    }
-	    clear();
-	    break;
-
-	case 'W':
-	    if (!msgNoYes("WARNING:  This should only be used when modifying an EXISTING\n"
-			       "installation.  If you are installing FreeBSD for the first time\n"
-			       "then you should simply type Q when you're finished here and your\n"
-			       "changes will be committed in one batch automatically at the end of\n"
-			       "these questions.  If you're adding a disk, you should NOT write\n"
-			       "from this screen, you should do it from the label editor.\n\n"
-			       "Are you absolutely sure you want to do this now?")) {
-		variable_set2(DISK_PARTITIONED, "yes", 0);
-
-#ifdef PC98
-		/*
-		 * Don't trash the IPL if the first (and therefore only) chunk
-		 * is marked for a truly dedicated disk (i.e., the disklabel
-		 * starts at sector 0), even in cases where the user has
-		 * requested a FreeBSD Boot Manager -- both would be fatal in
-		 * this case.
-		 */
-		/*
-		 * Don't offer to update the IPL on this disk if the first
-		 * "real" chunk looks like a FreeBSD "all disk" partition,
-		 * or the disk is entirely FreeBSD.
-		 */
-		if ((d->chunks->part->type != freebsd) ||
-		    (d->chunks->part->offset > 1))
-		    getBootMgr(d->name, &bootipl, &bootipl_size,
-			       &bootmenu, &bootmenu_size);
-		else {
-		    bootipl = NULL;
-		    bootipl_size = 0;
-		    bootmenu = NULL;
-		    bootmenu_size = 0;
-		}
-		Set_Boot_Mgr(d, bootipl, bootipl_size, bootmenu, bootmenu_size);
-#else
-		/*
-		 * Don't trash the MBR if the first (and therefore only) chunk
-		 * is marked for a truly dedicated disk (i.e., the disklabel
-		 * starts at sector 0), even in cases where the user has
-		 * requested booteasy or a "standard" MBR -- both would be
-		 * fatal in this case.
-		 */
-		/*
-		 * Don't offer to update the MBR on this disk if the first
-		 * "real" chunk looks like a FreeBSD "all disk" partition,
-		 * or the disk is entirely FreeBSD.
-		 */
-		if ((d->chunks->part->type != freebsd) ||
-		    (d->chunks->part->offset > 1))
-		    getBootMgr(d->name, &mbrContents, &mbrSize);
-		else {
-		    mbrContents = NULL;
-		    mbrSize = 0;
-		}
-		Set_Boot_Mgr(d, mbrContents, mbrSize);
-#endif
-
-		if (DITEM_STATUS(diskPartitionWrite(NULL)) != DITEM_SUCCESS)
-		    msgConfirm("Disk partition write returned an error status!");
-		else
-		    msgConfirm("Wrote FDISK partition information out successfully.");
-	    }
-	    clear();
-	    break;
-
-	case '|':
-	    if (!msgNoYes("Are you SURE you want to go into Expert mode?\n"
-			  "No seat belts whatsoever are provided!")) {
-		clear();
-		refresh();
-		slice_wizard(d);
-		variable_set2(DISK_PARTITIONED, "yes", 0);
-		record_chunks(d);
-	    }
-	    else
-		msg = "Wise choice!";
-	    clear();
-	    break;
-
-	case '\033':	/* ESC */
-	case 'Q':
-	    chunking = FALSE;
-#ifdef PC98
-	    /*
-	     * Don't trash the IPL if the first (and therefore only) chunk
-	     * is marked for a truly dedicated disk (i.e., the disklabel
-	     * starts at sector 0), even in cases where the user has requested
-	     * a FreeBSD Boot Manager -- both would be fatal in this case.
-	     */
-	    /*
-	     * Don't offer to update the IPL on this disk if the first "real"
-	     * chunk looks like a FreeBSD "all disk" partition, or the disk is
-	     * entirely FreeBSD. 
-	     */
-	    if ((d->chunks->part->type != freebsd) ||
-		(d->chunks->part->offset > 1)) {
-		if (variable_cmp(DISK_PARTITIONED, "written")) {
-		    getBootMgr(d->name, &bootipl, &bootipl_size,
-			&bootmenu, &bootmenu_size);
-		    if (bootipl != NULL && bootmenu != NULL)
-			Set_Boot_Mgr(d, bootipl, bootipl_size,
-			    bootmenu, bootmenu_size);
-		}
-	    }
-#else
-	    /*
-	     * Don't trash the MBR if the first (and therefore only) chunk
-	     * is marked for a truly dedicated disk (i.e., the disklabel
-	     * starts at sector 0), even in cases where the user has requested
-	     * booteasy or a "standard" MBR -- both would be fatal in this case.
-	     */
-	    /*
-	     * Don't offer to update the MBR on this disk if the first "real"
-	     * chunk looks like a FreeBSD "all disk" partition, or the disk is
-	     * entirely FreeBSD. 
-	     */
-	    if ((d->chunks->part->type != freebsd) ||
-		(d->chunks->part->offset > 1)) {
-		if (variable_cmp(DISK_PARTITIONED, "written")) {
-		    getBootMgr(d->name, &mbrContents, &mbrSize);
-		    if (mbrContents != NULL)
-			Set_Boot_Mgr(d, mbrContents, mbrSize);
-		}
-	    }
-#endif
-	    break;
-
-	case 'Z':
-	    size_unit = (size_unit + 1) % UNIT_SIZE;
-	    break;
-	    
-	default:
-	    beep();
-	    msg = "Type F1 or ? for help";
-	    break;
-	}
-    }
-    p = CheckRules(d);
-    if (p) {
-	char buf[FILENAME_MAX];
-	
-        use_helpline("Press F1 to read more about disk slices.");
-	use_helpfile(systemHelpFile("partition", buf));
-	if (!variable_get(VAR_NO_WARN))
-	    dialog_mesgbox("Disk slicing warning:", p, -1, -1);
-	free(p);
-    }
-    restorescr(w);
-}
-#endif /* WITH_SLICES */
-
-#ifdef WITH_SLICES 
-static int
-partitionHook(dialogMenuItem *selected)
-{
-    Device **devs = NULL;
-
-    devs = deviceFind(selected->prompt, DEVICE_TYPE_DISK);
-    if (!devs) {
-	msgConfirm("Unable to find disk %s!", selected->prompt);
-	return DITEM_FAILURE;
-    }
-    /* Toggle enabled status? */
-    if (!devs[0]->enabled) {
-	devs[0]->enabled = TRUE;
-	diskPartition(devs[0]);
-    }
-    else
-	devs[0]->enabled = FALSE;
-    return DITEM_SUCCESS;
-}
-
-static int
-partitionCheck(dialogMenuItem *selected)
-{
-    Device **devs = NULL;
-
-    devs = deviceFind(selected->prompt, DEVICE_TYPE_DISK);
-    if (!devs || devs[0]->enabled == FALSE)
-	return FALSE;
-    return TRUE;
-}
-
-int
-diskPartitionEditor(dialogMenuItem *self)
-{
-    DMenu *menu;
-    Device **devs;
-    int i, cnt, devcnt;
-
-    cnt = diskGetSelectCount(&devs);
-    devcnt = deviceCount(devs);
-    if (cnt == -1) {
-	msgConfirm("No disks found!  Please verify that your disk controller is being\n"
-		   "properly probed at boot time.  See the Hardware Guide on the\n"
-		   "Documentation menu for clues on diagnosing this type of problem.");
-	return DITEM_FAILURE;
-    }
-    else if (cnt) {
-	/* Some are already selected */
-	for (i = 0; i < devcnt; i++) {
-	    if (devs[i]->enabled) {
-		if (variable_get(VAR_NONINTERACTIVE) &&
-		  !variable_get(VAR_DISKINTERACTIVE))
-		    diskPartitionNonInteractive(devs[i]);
-		else
-		    diskPartition(devs[i]);
-	    }
-	}
-    }
-    else {
-	/* No disks are selected, fall-back case now */
-	if (devcnt == 1) {
-	    devs[0]->enabled = TRUE;
-	    if (variable_get(VAR_NONINTERACTIVE) &&
-	      !variable_get(VAR_DISKINTERACTIVE))
-		diskPartitionNonInteractive(devs[0]);
-	    else
-		diskPartition(devs[0]);
-	    return DITEM_SUCCESS;
-	}
-	else {
-	    menu = deviceCreateMenu(&MenuDiskDevices, DEVICE_TYPE_DISK, partitionHook, partitionCheck);
-	    if (!menu) {
-		msgConfirm("No devices suitable for installation found!\n\n"
-			   "Please verify that your disk controller (and attached drives)\n"
-			   "were detected properly.  This can be done by pressing the\n"
-			   "[Scroll Lock] key and using the Arrow keys to move back to\n"
-			   "the boot messages.  Press [Scroll Lock] again to return.");
-		return DITEM_FAILURE;
-	    }
-	    else {
-		i = dmenuOpenSimple(menu, FALSE) ? DITEM_SUCCESS : DITEM_FAILURE;
-		free(menu);
-	    }
-	    return i;
-	}
-    }
-    return DITEM_SUCCESS;
-}
-#endif /* WITH_SLICES */
-
-int
-diskPartitionWrite(dialogMenuItem *self)
-{
-    Device **devs;
-    int i;
-
-    if (!variable_cmp(DISK_PARTITIONED, "written"))
-	return DITEM_SUCCESS;
-
-    devs = deviceFind(NULL, DEVICE_TYPE_DISK);
-    if (!devs) {
-	msgConfirm("Unable to find any disks to write to??");
-	return DITEM_FAILURE;
-    }
-    if (isDebug())
-	msgDebug("diskPartitionWrite: Examining %d devices\n", deviceCount(devs));
-    for (i = 0; devs[i]; i++) {
-	Disk *d = (Disk *)devs[i]->private;
-#if !defined(__ia64__)
-	static u_char *boot1;
-#endif
-#if defined(__i386__) || defined(__amd64__)
-	static u_char *boot2;
-#endif
-
-	if (!devs[i]->enabled)
-	    continue;
-
-#if defined(__i386__) || defined(__amd64__)
-	if (!boot1) boot1 = bootalloc("boot1", NULL);
-	if (!boot2) boot2 = bootalloc("boot2", NULL);
-	Set_Boot_Blocks(d, boot1, boot2);
-#elif !defined(__ia64__)
-	if (!boot1) boot1 = bootalloc("boot1", NULL);
-	Set_Boot_Blocks(d, boot1, NULL);
-#endif
-
-	msgNotify("Writing partition information to drive %s", d->name);
-	if (!Fake && Write_Disk(d)) {
-	    if (RunningAsInit) {
-		msgConfirm("ERROR: Unable to write data to disk %s!", d->name);
-	    } else {
-		msgConfirm("ERROR: Unable to write data to disk %s!\n\n"
-		    "To edit the labels on a running system set\n"
-		    "sysctl kern.geom.debugflags=16 and try again.", d->name);
-	    }
-	    return DITEM_FAILURE;
-	}
-    }
-    /* Now it's not "yes", but "written" */
-    variable_set2(DISK_PARTITIONED, "written", 0);
-    return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-#ifdef WITH_SLICES
-/* Partition a disk based wholly on which variables are set */
-static void
-diskPartitionNonInteractive(Device *dev)
-{
-    char *cp;
-    int i, all_disk = 0;
-    daddr_t size;
-    long double dsize;
-#ifdef PC98
-    u_char *bootipl;
-    size_t bootipl_size;
-    u_char *bootmenu;
-    size_t bootmenu_size;
-#else
-    u_char *mbrContents;
-    size_t mbrSize;
-#endif
-    Disk *d = (Disk *)dev->private;
-
-    record_chunks(d);
-    cp = variable_get(VAR_GEOMETRY);
-    if (cp) {
-	if (!strcasecmp(cp, "sane")) {
-#ifdef PC98
-	    if (d->bios_cyl >= 65536 || d->bios_hd > 256 || d->bios_sect >= 256)
-#else
-	    if (d->bios_cyl > 65536 || d->bios_hd > 256 || d->bios_sect >= 64)
-#endif
-	    {
-		msgDebug("Warning:  A geometry of %lu/%lu/%lu for %s is incorrect.\n",
-		    d->bios_cyl, d->bios_hd, d->bios_sect, d->name);
-		Sanitize_Bios_Geom(d);
-		msgDebug("Sanitized geometry for %s is %lu/%lu/%lu.\n",
-		    d->name, d->bios_cyl, d->bios_hd, d->bios_sect);
-	    }
-	} else {
-	    msgDebug("Setting geometry from script to: %s\n", cp);
-	    d->bios_cyl = strtol(cp, &cp, 0);
-	    d->bios_hd = strtol(cp + 1, &cp, 0);
-	    d->bios_sect = strtol(cp + 1, 0, 0);
-	}
-    }
-
-    cp = variable_get(VAR_PARTITION);
-    if (cp) {
-	if (!strcmp(cp, "free")) {
-	    /* Do free disk space case */
-	    for (i = 0; chunk_info[i]; i++) {
-		/* If a chunk is at least 10MB in size, use it. */
-		if (chunk_info[i]->type == unused && chunk_info[i]->size > (10 * ONE_MEG)) {
-		    Create_Chunk(d, chunk_info[i]->offset, chunk_info[i]->size,
-				 freebsd, SUBTYPE_FREEBSD,
-				 (chunk_info[i]->flags & CHUNK_ALIGN),
-				 "FreeBSD");
-		    variable_set2(DISK_PARTITIONED, "yes", 0);
-		    break;
-		}
-	    }
-	    if (!chunk_info[i]) {
-		msgConfirm("Unable to find any free space on this disk!");
-		return;
-	    }
-	}
-	else if (!strcmp(cp, "all")) {
-	    /* Do all disk space case */
-	    msgDebug("Warning:  Devoting all of disk %s to FreeBSD.\n", d->name);
-
-	    All_FreeBSD(d, FALSE);
-	}
-	else if (!strcmp(cp, "exclusive")) {
-	    /* Do really-all-the-disk-space case */
-	    msgDebug("Warning:  Devoting all of disk %s to FreeBSD.\n", d->name);
-
-	    All_FreeBSD(d, all_disk = TRUE);
-	}
-	else if ((dsize = strtold(cp, &cp))) {
-	    if (*cp && toupper(*cp) == 'M')
-		size *= (daddr_t) (dsize * ONE_MEG);
-	    else if (*cp && toupper(*cp) == 'G')
-		size = (daddr_t) (dsize * ONE_GIG);
-	    else
-		size = (daddr_t) dsize;
-
-	    /* Look for size bytes free */
-	    for (i = 0; chunk_info[i]; i++) {
-		/* If a chunk is at least sz MB, use it. */
-		if (chunk_info[i]->type == unused && chunk_info[i]->size >= size) {
-		    Create_Chunk(d, chunk_info[i]->offset, size, freebsd, SUBTYPE_FREEBSD,
-				 (chunk_info[i]->flags & CHUNK_ALIGN),
-				 "FreeBSD");
-		    variable_set2(DISK_PARTITIONED, "yes", 0);
-		    break;
-		}
-	    }
-	    if (!chunk_info[i]) {
-		    msgConfirm("Unable to find %jd free blocks on this disk!",
-			(intmax_t)size);
-		return;
-	    }
-	}
-	else if (!strcmp(cp, "existing")) {
-	    /* Do existing FreeBSD case */
-	    for (i = 0; chunk_info[i]; i++) {
-		if (chunk_info[i]->type == freebsd)
-		    break;
-	    }
-	    if (!chunk_info[i]) {
-		msgConfirm("Unable to find any existing FreeBSD partitions on this disk!");
-		return;
-	    }
-	}
-	else {
-	    msgConfirm("`%s' is an invalid value for %s - is config file valid?", cp, VAR_PARTITION);
-	    return;
-	}
-	if (!all_disk) {
-#ifdef PC98
-	    getBootMgr(d->name, &bootipl, &bootipl_size,
-		       &bootmenu, &bootmenu_size);
-	    Set_Boot_Mgr(d, bootipl, bootipl_size, bootmenu, bootmenu_size);
-#else
-	    getBootMgr(d->name, &mbrContents, &mbrSize);
-	    Set_Boot_Mgr(d, mbrContents, mbrSize);
-#endif
-	}
-	variable_set2(DISK_PARTITIONED, "yes", 0);
-    }
-}
-#endif /* WITH_SLICES */
diff --git a/usr.sbin/sysinstall/dispatch.c b/usr.sbin/sysinstall/dispatch.c
deleted file mode 100644
index 44aa0facbd8c..000000000000
--- a/usr.sbin/sysinstall/dispatch.c
+++ /dev/null
@@ -1,653 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <errno.h>
-#include <sys/signal.h>
-#include <sys/fcntl.h>
-
-#include "list.h"
-
-static int dispatch_shutdown(dialogMenuItem *unused);
-static int dispatch_systemExecute(dialogMenuItem *unused);
-static int dispatch_msgConfirm(dialogMenuItem *unused);
-static int dispatch_mediaOpen(dialogMenuItem *unused);
-static int dispatch_mediaClose(dialogMenuItem *unused);
-static int cfgModuleFire(dialogMenuItem *self);
-
-static struct _word {
-    char *name;
-    int (*handler)(dialogMenuItem *self);
-} resWords[] = {
-    { "configAnonFTP",		configAnonFTP		},
-    { "configRouter",		configRouter		},
-    { "configInetd",		configInetd		},
-    { "configNFSServer",	configNFSServer		},
-    { "configNTP",		configNTP		},
-    { "configPCNFSD",		configPCNFSD		},
-    { "configPackages",		configPackages		},
-    { "configUsers",		configUsers		},
-#ifdef WITH_SLICES
-    { "diskPartitionEditor",	diskPartitionEditor	},
-#endif
-    { "diskPartitionWrite",	diskPartitionWrite	},
-    { "diskLabelEditor",	diskLabelEditor		},
-    { "diskLabelCommit",	diskLabelCommit		},
-    { "distReset",		distReset		},
-    { "distSetCustom",		distSetCustom		},
-    { "distUnsetCustom",	distUnsetCustom		},
-    { "distSetDeveloper",	distSetDeveloper	},
-    { "distSetKernDeveloper",	distSetKernDeveloper	},
-    { "distSetUser",		distSetUser		},
-    { "distSetMinimum",		distSetMinimum		},
-    { "distSetEverything",	distSetEverything	},
-    { "distSetSrc",		distSetSrc		},
-    { "distExtractAll",		distExtractAll		},
-    { "docBrowser",		docBrowser		},
-    { "docShowDocument",	docShowDocument		},
-    { "installCommit",		installCommit		},
-    { "installExpress",		installExpress		},
-    { "installStandard",	installStandard		},
-    { "installUpgrade",		installUpgrade		},
-    { "installFixupBase",	installFixupBase	},
-    { "installFixitHoloShell",	installFixitHoloShell	},
-    { "installFixitCDROM",	installFixitCDROM	},
-    { "installFixitUSB",	installFixitUSB		},
-    { "installFixitFloppy",	installFixitFloppy	},
-    { "installFilesystems",	installFilesystems	},
-    { "installVarDefaults",	installVarDefaults	},
-    { "loadConfig",		dispatch_load_file	},
-    { "loadFloppyConfig",	dispatch_load_floppy	},
-    { "loadCDROMConfig",	dispatch_load_cdrom	},
-    { "mediaOpen",		dispatch_mediaOpen	},
-    { "mediaClose",		dispatch_mediaClose	},
-    { "mediaSetCDROM",		mediaSetCDROM		},
-    { "mediaSetFloppy",		mediaSetFloppy		},
-    { "mediaSetUSB",		mediaSetUSB		},
-    { "mediaSetDOS",		mediaSetDOS		},
-    { "mediaSetFTP",		mediaSetFTP		},
-    { "mediaSetFTPActive",	mediaSetFTPActive	},
-    { "mediaSetFTPPassive",	mediaSetFTPPassive	},
-    { "mediaSetHTTP",		mediaSetHTTP		},
-    { "mediaSetUFS",		mediaSetUFS		},
-    { "mediaSetNFS",		mediaSetNFS		},
-    { "mediaSetFTPUserPass",	mediaSetFTPUserPass	},
-    { "mediaSetCPIOVerbosity",	mediaSetCPIOVerbosity	},
-    { "mediaGetType",		mediaGetType		},
-    { "msgConfirm",		dispatch_msgConfirm	},
-    { "optionsEditor",		optionsEditor		},
-    { "packageAdd",		packageAdd		},
-    { "addGroup",		userAddGroup		},
-    { "addUser",		userAddUser		},
-    { "shutdown",		dispatch_shutdown 	},
-    { "system",			dispatch_systemExecute	},
-    { "dumpVariables",		dump_variables		},
-    { "tcpMenuSelect",		tcpMenuSelect		},
-    { NULL, NULL },
-};
-
-/*
- * Helper routines for buffering data.
- *
- * We read an entire configuration into memory before executing it
- * so that we are truely standalone and can do things like nuke the
- * file or disk we're working on.
- */
-
-typedef struct command_buffer_ {
-    qelement	queue;
-    char *	string;
-} command_buffer;
-
-static void
-dispatch_free_command(command_buffer *item)
-{
-    if (item != NULL) {
-	REMQUE(item);
-	free(item->string);
-	item->string = NULL;
-    }
-
-    free(item);
-}
-
-static void
-dispatch_free_all(qelement *head)
-{
-    command_buffer *item;
-
-    while (!EMPTYQUE(*head)) {
-	item = (command_buffer *) head->q_forw;
-	dispatch_free_command(item);
-    }
-}
-
-static command_buffer *
-dispatch_add_command(qelement *head, char *string)
-{
-    command_buffer *new = NULL;
-
-    new = malloc(sizeof(command_buffer));
-
-    if (new != NULL) {
-
-	new->string = strdup(string);
-
-	/*
-	 * We failed to copy `string'; clean up the allocated
-	 * resources.
-	 */
-	if (new->string == NULL) {
-	    free(new);
-	    new = NULL;
-	} else {
-	    INSQUEUE(new, head->q_back);
-	}
-    }
-
-    return new;
-}
-
-/*
- * Command processing
- */
-
-/* Just convenience */
-static int
-dispatch_shutdown(dialogMenuItem *unused)
-{
-    systemShutdown(0);
-    return DITEM_FAILURE;
-}
-
-static int
-dispatch_systemExecute(dialogMenuItem *unused)
-{
-    char *cmd = variable_get(VAR_COMMAND);
-
-    if (cmd)
-	return systemExecute(cmd) ? DITEM_FAILURE : DITEM_SUCCESS;
-    else
-	msgDebug("_systemExecute: No command passed in `command' variable.\n");
-    return DITEM_FAILURE;
-}
-
-static int
-dispatch_msgConfirm(dialogMenuItem *unused)
-{
-    char *msg = variable_get(VAR_COMMAND);
-
-    if (msg) {
-	msgConfirm("%s", msg);
-	return DITEM_SUCCESS;
-    }
-
-    msgDebug("_msgConfirm: No message passed in `command' variable.\n");
-    return DITEM_FAILURE;
-}
-
-static int
-dispatch_mediaOpen(dialogMenuItem *unused)
-{
-    return mediaOpen();
-}
-
-static int
-dispatch_mediaClose(dialogMenuItem *unused)
-{
-    mediaClose();
-    return DITEM_SUCCESS;
-}
-
-static int
-call_possible_resword(char *name, dialogMenuItem *value, int *status)
-{
-    int i, rval;
-
-    rval = 0;
-    for (i = 0; resWords[i].name; i++) {
-	if (!strcmp(name, resWords[i].name)) {
-	    *status = resWords[i].handler(value);
-	    rval = 1;
-	    break;
-	}
-    }
-    return rval;
-}
-
-/* For a given string, call it or spit out an undefined command diagnostic */
-int
-dispatchCommand(char *str)
-{
-    int i;
-    char *cp;
-
-    if (!str || !*str) {
-	msgConfirm("Null or zero-length string passed to dispatchCommand");
-	return DITEM_FAILURE;
-    }
-
-    /* Fixup DOS abuse */
-    if ((cp = index(str, '\r')) != NULL)
-	*cp = '\0';
-
-    /* If it's got a `=' sign in there, assume it's a variable setting */
-    if (index(str, '=')) {
-	if (isDebug())
-	    msgDebug("dispatch: setting variable `%s'\n", str);
-	variable_set(str, 0);
-	i = DITEM_SUCCESS;
-    }
-    else {
-	/* A command might be a pathname if it's encoded in argv[0], which
-	   we also support */
-	if ((cp = rindex(str, '/')) != NULL)
-	    str = cp + 1;
-	if (isDebug())
-	    msgDebug("dispatch: calling resword `%s'\n", str);
-	if (!call_possible_resword(str, NULL, &i)) {
-	    msgNotify("Warning: No such command ``%s''", str);
-	    i = DITEM_FAILURE;
-	}
-	/*
-	 * Allow a user to prefix a command with "noError" to cause
-	 * us to ignore any errors for that one command.
-	 */
-	if (i != DITEM_SUCCESS && variable_get(VAR_NO_ERROR))
-	    i = DITEM_SUCCESS;
-	variable_unset(VAR_NO_ERROR);
-    }
-    return i;
-}
-
-
-/*
- * File processing
- */
-
-static qelement *
-dispatch_load_fp(FILE *fp)
-{
-    qelement *head;
-    char buf[BUFSIZ], *cp;
-
-    head = malloc(sizeof(qelement));
-
-    if (!head)
-	return NULL;
-
-    INITQUE(*head);
-
-    while (fgets(buf, sizeof buf, fp)) {
-	/* Fix up DOS abuse */
-	if ((cp = index(buf, '\r')) != NULL)
-	    *cp = '\0';
-	/* If it's got a new line, trim it */
-       if ((cp = index(buf, '\n')) != NULL)
-            *cp = '\0';
-	if (*buf == '\0' || *buf == '#')
-	    continue;
-
-	if (!dispatch_add_command(head, buf))
-	    return NULL;
-    }
-
-    return head;
-}
-
-static int
-dispatch_execute(qelement *head)
-{
-    int result = DITEM_SUCCESS;
-    command_buffer *item;
-    char *old_interactive;
-
-    if (!head)
-	return result | DITEM_FAILURE;
-
-    old_interactive = variable_get(VAR_NONINTERACTIVE);
-    if (old_interactive)
-	 old_interactive = strdup(old_interactive);	/* save copy */
-
-    /* Hint to others that we're running from a script, should they care */
-    variable_set2(VAR_NONINTERACTIVE, "yes", 0);
-
-    while (!EMPTYQUE(*head)) {
-	item = (command_buffer *) head->q_forw;
-
-	if (DITEM_STATUS(dispatchCommand(item->string)) != DITEM_SUCCESS) {
-	    msgConfirm("Command `%s' failed - rest of script aborted.\n",
-		       item->string);
-	    result |= DITEM_FAILURE;
-	    break;
-	}
-	dispatch_free_command(item);
-    }
-
-    dispatch_free_all(head);
-
-    if (!old_interactive)
-	variable_unset(VAR_NONINTERACTIVE);
-    else {
-	variable_set2(VAR_NONINTERACTIVE, old_interactive, 0);
-	free(old_interactive);
-    }
-
-    return result;
-}
-
-int
-dispatch_load_file_int(int quiet)
-{
-    FILE *fp;
-    char *cp;
-    int  i;
-    qelement *list;
-
-    static const char *names[] = {
-	"install.cfg",
-	"/stand/install.cfg",
-	"/tmp/install.cfg",
-	NULL
-    };
-
-    fp = NULL;
-    cp = variable_get(VAR_CONFIG_FILE);
-    if (!cp) {
-	for (i = 0; names[i]; i++)
-	    if ((fp = fopen(names[i], "r")) != NULL)
-		break;
-    } else
-	fp = fopen(cp, "r");
-
-    if (!fp) {
-	if (!quiet)
-	    msgConfirm("Unable to open %s: %s", cp, strerror(errno));
-	return DITEM_FAILURE;
-    }
-
-    list = dispatch_load_fp(fp);
-    fclose(fp);
-
-    return dispatch_execute(list);
-}
-
-int
-dispatch_load_file(dialogMenuItem *self)
-{
-    return dispatch_load_file_int(FALSE);
-}
-
-int
-dispatch_load_floppy(dialogMenuItem *self)
-{
-    int             what = DITEM_SUCCESS;
-    extern char    *distWanted;
-    char           *cp;
-    FILE           *fp;
-    qelement	   *list;
-
-    mediaClose();
-    cp = variable_get_value(VAR_INSTALL_CFG,
-			    "Specify the name of a configuration file", 0);
-    if (!cp || !*cp) {
-	variable_unset(VAR_INSTALL_CFG);
-	what |= DITEM_FAILURE;
-	return what;
-    }
-
-    distWanted = cp;
-    /* Try to open the floppy drive */
-    if (DITEM_STATUS(mediaSetFloppy(NULL)) == DITEM_FAILURE) {
-	msgConfirm("Unable to set media device to floppy.");
-	what |= DITEM_FAILURE;
-	mediaClose();
-	return what;
-    }
-
-    if (!DEVICE_INIT(mediaDevice)) {
-	msgConfirm("Unable to mount floppy filesystem.");
-	what |= DITEM_FAILURE;
-	mediaClose();
-	return what;
-    }
-
-    fp = DEVICE_GET(mediaDevice, cp, TRUE);
-    if (fp) {
-	list = dispatch_load_fp(fp);
-	fclose(fp);
-	mediaClose();
-
-	what |= dispatch_execute(list);
-    }
-    else {
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm("Configuration file '%s' not found.", cp);
-	variable_unset(VAR_INSTALL_CFG);
-	what |= DITEM_FAILURE;
-	mediaClose();
-    }
-    return what;
-}
-
-int
-dispatch_load_cdrom(dialogMenuItem *self)
-{
-    int             what = DITEM_SUCCESS;
-    extern char    *distWanted;
-    char           *cp;
-    FILE           *fp;
-    qelement	   *list;
-
-    mediaClose();
-    cp = variable_get_value(VAR_INSTALL_CFG,
-			    "Specify the name of a configuration file\n"
-			    "residing on the CDROM.", 0);
-    if (!cp || !*cp) {
-	variable_unset(VAR_INSTALL_CFG);
-	what |= DITEM_FAILURE;
-	return what;
-    }
-
-    distWanted = cp;
-    /* Try to open the floppy drive */
-    if (DITEM_STATUS(mediaSetCDROM(NULL)) == DITEM_FAILURE) {
-	msgConfirm("Unable to set media device to CDROM.");
-	what |= DITEM_FAILURE;
-	mediaClose();
-	return what;
-    }
-
-    if (!DEVICE_INIT(mediaDevice)) {
-	msgConfirm("Unable to CDROM filesystem.");
-	what |= DITEM_FAILURE;
-	mediaClose();
-	return what;
-    }
-
-    fp = DEVICE_GET(mediaDevice, cp, TRUE);
-    if (fp) {
-	list = dispatch_load_fp(fp);
-	fclose(fp);
-	mediaClose();
-
-	what |= dispatch_execute(list);
-    }
-    else {
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm("Configuration file '%s' not found.", cp);
-	variable_unset(VAR_INSTALL_CFG);
-	what |= DITEM_FAILURE;
-	mediaClose();
-    }
-    return what;
-}
-
-/*
- * Create a menu based on available disk devices
- */
-int
-dispatch_load_menu(dialogMenuItem *self)
-{
-    DMenu	*menu;
-    Device	**devlist;
-    char	*err;
-    int		what, i, j, msize, count;
-    DeviceType	dtypes[] = {DEVICE_TYPE_FLOPPY, DEVICE_TYPE_CDROM,
-	DEVICE_TYPE_DOS, DEVICE_TYPE_UFS, DEVICE_TYPE_USB};
-
-    fprintf(stderr, "dispatch_load_menu called\n");
-
-    msize = sizeof(DMenu) + (sizeof(dialogMenuItem) * 2);
-    count = 0;
-    err = NULL;
-    what = DITEM_SUCCESS;
-
-    if ((menu = malloc(msize)) == NULL) {
-	err = "Failed to allocate memory for menu";
-	goto errout;
-    }
-
-    bcopy(&MenuConfig, menu, sizeof(DMenu));
-
-    bzero(&menu->items[count], sizeof(menu->items[0]));
-    menu->items[count].prompt = strdup("X Exit");
-    menu->items[count].title = strdup("Exit this menu (returning to previous)");
-    menu->items[count].fire = dmenuExit;
-    count++;
-
-    for (i = 0; i < sizeof(dtypes) / sizeof(dtypes[0]); i++) {
-	if ((devlist = deviceFind(NULL, dtypes[i])) == NULL) {
-	    fprintf(stderr, "No devices found for type %d\n", dtypes[i]);
-	    continue;
-	}
-
-	for (j = 0; devlist[j] != NULL; j++) {
-	    fprintf(stderr, "device type %d device name %s\n", dtypes[i], devlist[j]->name);
-	    msize += sizeof(dialogMenuItem);
-	    if ((menu = realloc(menu, msize)) == NULL) {
-		err = "Failed to allocate memory for menu item";
-		goto errout;
-	    }
-
-	    bzero(&menu->items[count], sizeof(menu->items[0]));
-	    menu->items[count].fire = cfgModuleFire;
-
-	    menu->items[count].prompt = strdup(devlist[j]->name);
-	    menu->items[count].title = strdup(devlist[j]->description);
-	    /* XXX: dialog(3) sucks */
-	    menu->items[count].aux = (long)devlist[j];
-	    count++;
-	}
-    }
-
-    menu->items[count].prompt = NULL;
-    menu->items[count].title = NULL;
-
-    dmenuOpenSimple(menu, FALSE);
-
-  errout:
-    for (i = 0; i < count; i++) {
-	free(menu->items[i].prompt);
-	free(menu->items[i].title);
-    }
-
-    free(menu);
-
-    if (err != NULL) {
-	what |= DITEM_FAILURE;
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm(err);
-    }
-
-    return (what);
-
-}
-
-static int
-cfgModuleFire(dialogMenuItem *self) {
-    Device	*d;
-    FILE	*fp;
-    int		what = DITEM_SUCCESS;
-    extern char *distWanted;
-    qelement	*list;
-    char	*cp;
-
-    d = (Device *)self->aux;
-
-    msgDebug("cfgModuleFire: User selected %s (%s)\n", self->prompt, d->devname);
-
-    mediaClose();
-
-    cp = variable_get_value(VAR_INSTALL_CFG,
-			    "Specify the name of a configuration file", 0);
-    if (!cp || !*cp) {
-	variable_unset(VAR_INSTALL_CFG);
-	what |= DITEM_FAILURE;
-	return what;
-    }
-
-    distWanted = cp;
-
-    mediaDevice = d;
-    if (!DEVICE_INIT(mediaDevice)) {
-	msgConfirm("Unable to mount filesystem.");
-	what |= DITEM_FAILURE;
-	mediaClose();
-	return what;
-    }
-    msgDebug("getting fp for %s\n", cp);
-
-    fp = DEVICE_GET(mediaDevice, cp, TRUE);
-    if (fp) {
-	msgDebug("opened OK, processing..\n");
-
-	list = dispatch_load_fp(fp);
-	fclose(fp);
-	mediaClose();
-
-	what |= dispatch_execute(list);
-    } else {
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm("Configuration file '%s' not found.", cp);
-	variable_unset(VAR_INSTALL_CFG);
-	what |= DITEM_FAILURE;
-	mediaClose();
-    }
-
-    return(what);
- }
diff --git a/usr.sbin/sysinstall/dist.c b/usr.sbin/sysinstall/dist.c
deleted file mode 100644
index 8ef85441444c..000000000000
--- a/usr.sbin/sysinstall/dist.c
+++ /dev/null
@@ -1,922 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/time.h>
-#include <sys/uio.h>
-#include <ctype.h>
-#include <signal.h>
-#include <libutil.h>
-
-unsigned int Dists;
-unsigned int DocDists;
-unsigned int SrcDists;
-unsigned int KernelDists;
-
-enum _disttype { DT_TARBALL, DT_SUBDIST, DT_PACKAGE };
-
-typedef struct _dist {
-    char *my_name;
-    unsigned int *my_mask;
-    unsigned int my_bit;
-    enum _disttype my_type;
-    union {
-	char *my_string;	/* DT_TARBALL & DT_PACKAGE */
-	struct _dist *my_dist;	/* DT_SUBDIST */
-    } my_data;
-} Distribution;
-
-static Distribution DocDistTable[];
-static Distribution KernelDistTable[];
-static Distribution SrcDistTable[];
-
-#define	DTE_TARBALL(name, mask, flag, directory)			\
-	{ name, mask, DIST_ ## flag, DT_TARBALL, { directory } }
-#define	DTE_PACKAGE(name, mask, flag, package)				\
-	{ name, mask, DIST_ ## flag, DT_PACKAGE, { package } }
-#define	DTE_SUBDIST(name, mask, flag, subdist)				\
-	{ name, mask, DIST_ ## flag, DT_SUBDIST, { .my_dist = subdist } }
-#define	DTE_END			{ NULL, NULL, 0, 0, { NULL } }
-
-#define	BASE_DIST	(&DistTable[0])
-
-/* The top-level distribution categories */
-static Distribution DistTable[] = {
-    DTE_TARBALL("base",	    &Dists, BASE,     "/"),
-    DTE_SUBDIST("kernels",  &Dists, KERNEL,   KernelDistTable),
-    DTE_TARBALL("doc",	    &Dists, DOCUSERLAND,      "/"),
-    DTE_SUBDIST("docproj",  &Dists, DOC,      DocDistTable),
-    DTE_TARBALL("games",    &Dists, GAMES,    "/"),
-    DTE_TARBALL("manpages", &Dists, MANPAGES, "/"),
-    DTE_TARBALL("catpages", &Dists, CATPAGES, "/"),
-    DTE_TARBALL("proflibs", &Dists, PROFLIBS, "/"),
-    DTE_TARBALL("dict",	    &Dists, DICT,     "/"),
-    DTE_TARBALL("info",	    &Dists, INFO,     "/"),
-#if defined(__amd64__) || defined(__powerpc64__)
-    DTE_TARBALL("lib32",    &Dists, LIB32,    "/"),
-#endif
-    DTE_SUBDIST("src",	    &Dists, SRC,      SrcDistTable),
-    DTE_TARBALL("ports",    &Dists, PORTS,    "/usr"),
-    DTE_TARBALL("local",    &Dists, LOCAL,    "/"),
-    DTE_END,
-};
-
-/* The kernel distributions */
-static Distribution KernelDistTable[] = {
-    DTE_TARBALL(GENERIC_KERNEL_NAME,  &KernelDists, KERNEL_GENERIC, "/boot"),
-    DTE_END,
-};
-
-/* The /usr/src distribution */
-static Distribution SrcDistTable[] = {
-    DTE_TARBALL("sbase",    &SrcDists, SRC_BASE,    "/usr/src"),
-    DTE_TARBALL("scddl",    &SrcDists, SRC_CDDL,    "/usr/src"),
-    DTE_TARBALL("scontrib", &SrcDists, SRC_CONTRIB, "/usr/src"),
-    DTE_TARBALL("scrypto",  &SrcDists, SRC_SCRYPTO, "/usr/src"),
-    DTE_TARBALL("sgnu",	    &SrcDists, SRC_GNU,	    "/usr/src"),
-    DTE_TARBALL("setc",	    &SrcDists, SRC_ETC,	    "/usr/src"),
-    DTE_TARBALL("sgames",   &SrcDists, SRC_GAMES,   "/usr/src"),
-    DTE_TARBALL("sinclude", &SrcDists, SRC_INCLUDE, "/usr/src"),
-    DTE_TARBALL("skrb5",    &SrcDists, SRC_SKERBEROS5, "/usr/src"),
-    DTE_TARBALL("slib",	    &SrcDists, SRC_LIB,	    "/usr/src"),
-    DTE_TARBALL("slibexec", &SrcDists, SRC_LIBEXEC, "/usr/src"),
-    DTE_TARBALL("srelease", &SrcDists, SRC_RELEASE, "/usr/src"),
-    DTE_TARBALL("sbin",	    &SrcDists, SRC_BIN,	    "/usr/src"),
-    DTE_TARBALL("ssecure",  &SrcDists, SRC_SSECURE, "/usr/src"),
-    DTE_TARBALL("ssbin",    &SrcDists, SRC_SBIN,    "/usr/src"),
-    DTE_TARBALL("sshare",   &SrcDists, SRC_SHARE,   "/usr/src"),
-    DTE_TARBALL("ssys",	    &SrcDists, SRC_SYS,	    "/usr/src"),
-    DTE_TARBALL("subin",    &SrcDists, SRC_UBIN,    "/usr/src"),
-    DTE_TARBALL("susbin",   &SrcDists, SRC_USBIN,   "/usr/src"),
-    DTE_TARBALL("stools",   &SrcDists, SRC_TOOLS,   "/usr/src"),
-    DTE_TARBALL("srescue",  &SrcDists, SRC_RESCUE,  "/usr/src"),
-    DTE_END,
-};
-
-/* The Documentation distribution */
-static Distribution DocDistTable[] = {
-    DTE_PACKAGE("Bengali Documentation",		&DocDists, DOC_BN,	"bn-freebsd-doc"),
-    DTE_PACKAGE("Danish Documentation",			&DocDists, DOC_DA,	"da-freebsd-doc"),
-    DTE_PACKAGE("German Documentation",			&DocDists, DOC_DE,	"de-freebsd-doc"),
-    DTE_PACKAGE("Greek Documentation",			&DocDists, DOC_EL,	"el-freebsd-doc"),
-    DTE_PACKAGE("English Documentation",		&DocDists, DOC_EN,	"en-freebsd-doc"),
-    DTE_PACKAGE("Spanish Documentation",		&DocDists, DOC_ES,	"es-freebsd-doc"),
-    DTE_PACKAGE("French Documentation",			&DocDists, DOC_FR,	"fr-freebsd-doc"),
-    DTE_PACKAGE("Hungarian Documentation",		&DocDists, DOC_HU,	"hu-freebsd-doc"),
-    DTE_PACKAGE("Italian Documentation",		&DocDists, DOC_IT,	"it-freebsd-doc"),
-    DTE_PACKAGE("Japanese Documentation",		&DocDists, DOC_JA,	"ja-freebsd-doc"),
-    DTE_PACKAGE("Mongolian Documentation",		&DocDists, DOC_MN,	"mn-freebsd-doc"),
-    DTE_PACKAGE("Dutch Documentation",			&DocDists, DOC_NL,	"nl-freebsd-doc"),
-    DTE_PACKAGE("Polish Documentation",			&DocDists, DOC_PL,	"pl-freebsd-doc"),
-    DTE_PACKAGE("Portuguese Documentation",		&DocDists, DOC_PT,	"pt-freebsd-doc"),
-    DTE_PACKAGE("Russian Documentation",		&DocDists, DOC_RU,	"ru-freebsd-doc"),
-    DTE_PACKAGE("Serbian Documentation",		&DocDists, DOC_SR,	"sr-freebsd-doc"),
-    DTE_PACKAGE("Turkish Documentation",		&DocDists, DOC_TR,	"tr-freebsd-doc"),
-    DTE_PACKAGE("Simplified Chinese Documentation",	&DocDists, DOC_ZH_CN,	"zh_cn-freebsd-doc"),
-    DTE_PACKAGE("Traditional Chinese Documentation",	&DocDists, DOC_ZH_TW,	"zh_tw-freebsd-doc"),
-    DTE_END,
-};
-
-static int	distMaybeSetPorts(dialogMenuItem *self);
-
-static void
-distVerifyFlags(void)
-{
-    if (SrcDists)
-	Dists |= DIST_SRC;
-    if (KernelDists)
-	Dists |= DIST_KERNEL;
-    if (DocDists)
-	Dists |= DIST_DOC;
-    if (isDebug())
-	msgDebug("Dist Masks: Dists: %0x, Srcs: %0x Kernels: %0x Docs: %0x\n", Dists,
-	    SrcDists, KernelDists, DocDists);
-}
-
-int
-distReset(dialogMenuItem *self)
-{
-    Dists = 0;
-    DocDists = 0;
-    SrcDists = 0;
-    KernelDists = 0;
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-int
-distConfig(dialogMenuItem *self)
-{
-    char *cp;
-
-    distReset(NULL);
-
-    if ((cp = variable_get(VAR_DIST_MAIN)) != NULL)
-	Dists = atoi(cp);
-
-    if ((cp = variable_get(VAR_DIST_DOC)) != NULL)
-	DocDists = atoi(cp);
-
-    if ((cp = variable_get(VAR_DIST_SRC)) != NULL)
-	SrcDists = atoi(cp);
-
-    if ((cp = variable_get(VAR_DIST_KERNEL)) != NULL)
-	KernelDists = atoi(cp);
-
-    distVerifyFlags();
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-int
-selectKernel(void)
-{
-    return DIST_KERNEL_GENERIC;
-}
-
-int
-distSetDeveloper(dialogMenuItem *self)
-{
-    int i;
-
-    distReset(NULL);
-    Dists = _DIST_DEVELOPER;
-    SrcDists = DIST_SRC_ALL;
-    KernelDists = selectKernel();
-    i = distSetDoc(self);
-    i |= distMaybeSetPorts(self);
-    distVerifyFlags();
-    return i;
-}
-
-int
-distSetKernDeveloper(dialogMenuItem *self)
-{
-    int i;
-
-    distReset(NULL);
-    Dists = _DIST_DEVELOPER;
-    SrcDists = DIST_SRC_SYS | DIST_SRC_BASE;
-    KernelDists = selectKernel();
-    i = distSetDoc(self);
-    i |= distMaybeSetPorts(self);
-    distVerifyFlags();
-    return i;
-}
-
-int
-distSetUser(dialogMenuItem *self)
-{
-    int i;
-
-    distReset(NULL);
-    Dists = _DIST_USER;
-    KernelDists = selectKernel();
-    i = distSetDoc(self);
-    i |= distMaybeSetPorts(self);
-    distVerifyFlags();
-    return i;
-}
-
-int
-distSetMinimum(dialogMenuItem *self)
-{
-    distReset(NULL);
-    Dists = DIST_BASE | DIST_KERNEL;
-    KernelDists = selectKernel();
-    distVerifyFlags();
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-int
-distSetEverything(dialogMenuItem *self)
-{
-    int i;
-
-    Dists = DIST_ALL;
-    SrcDists = DIST_SRC_ALL;
-    KernelDists = DIST_KERNEL_ALL;
-    DocDists = DIST_DOC_ALL;
-    i = distMaybeSetPorts(self);
-    distVerifyFlags();
-    return i | DITEM_REDRAW;
-}
-
-static int
-distMaybeSetPorts(dialogMenuItem *self)
-{
-    dialog_clear_norefresh();
-    if (!msgYesNo("Would you like to install the FreeBSD ports collection?\n\n"
-		  "This will give you ready access to over 19,000 ported software packages,\n"
-		  "at a cost of around 445MB of disk space when \"clean\" and possibly\n"
-		  "much more than that when a lot of the distribution tarballs are loaded\n"
-		  "(unless you have the extra discs available from a FreeBSD CD/DVD distribution\n"
-		  "and can mount them on /cdrom, in which case this is far less of a problem).\n\n"
-		  "The ports collection is a very valuable resource and well worth having\n"
-		  "on your /usr partition, so it is advisable to say Yes to this option.\n\n"
-		  "For more information on the ports collection & the latest ports, visit:\n"
-		  "    http://www.freebsd.org/ports\n"))
-	Dists |= DIST_PORTS;
-    else
-	Dists &= ~DIST_PORTS;
-    return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-static Boolean
-distSetByName(Distribution *dist, char *name)
-{
-    int i, status = FALSE;
-    
-    /* Loop through current set */
-    for (i = 0; dist[i].my_name; i++) {
-	switch (dist[i].my_type) {
-	case DT_TARBALL:
-	case DT_PACKAGE:
-	    if (!strcmp(dist[i].my_name, name)) {
-		*(dist[i].my_mask) |= dist[i].my_bit;
-		status = TRUE;
-	    }
-	    break;
-	case DT_SUBDIST:
-	    if (distSetByName(dist[i].my_data.my_dist, name)) {
-		status = TRUE;
-	    }
-	    break;
-	}
-    }
-    distVerifyFlags();
-    return status;
-}
-
-static Boolean
-distUnsetByName(Distribution *dist, char *name)
-{
-    int i, status = FALSE;
-    
-    /* Loop through current set */
-    for (i = 0; dist[i].my_name; i++) {
-	switch (dist[i].my_type) {
-	case DT_TARBALL:
-	case DT_PACKAGE:
-	    if (!strcmp(dist[i].my_name, name)) {
-		*(dist[i].my_mask) &= ~(dist[i].my_bit);
-		status = TRUE;
-	    }
-	    break;
-	case DT_SUBDIST:
-	    if (distUnsetByName(dist[i].my_data.my_dist, name)) {
-		status = TRUE;
-	    }
-	    break;
-	}
-    }
-    return status;
-}
-
-/* Just for the dispatch stuff */
-int
-distSetCustom(dialogMenuItem *self)
-{
-    char *cp, *cp2, *tmp;
-
-    if (!(tmp = variable_get(VAR_DISTS))) {
-	msgDebug("distSetCustom() called without %s variable set.\n", VAR_DISTS);
-	return DITEM_FAILURE;
-    }
-
-    cp = alloca(strlen(tmp) + 1);
-    if (!cp)
-	msgFatal("Couldn't alloca() %d bytes!\n", (int)(strlen(tmp) + 1));
-    strcpy(cp, tmp);
-    while (cp) {
-	if ((cp2 = index(cp, ' ')) != NULL)
-	    *(cp2++) = '\0';
-	if (!distSetByName(DistTable, cp))
-	    msgDebug("distSetCustom: Warning, no such release \"%s\"\n", cp);
-	cp = cp2;
-    }
-    distVerifyFlags();
-    return DITEM_SUCCESS;
-}
-    
-/* Just for the dispatch stuff */
-int
-distUnsetCustom(dialogMenuItem *self)
-{
-    char *cp, *cp2, *tmp;
-
-    if (!(tmp = variable_get(VAR_DISTS))) {
-	msgDebug("distUnsetCustom() called without %s variable set.\n", VAR_DISTS);
-	return DITEM_FAILURE;
-    }
-
-    cp = alloca(strlen(tmp) + 1);
-    if (!cp)
-	msgFatal("Couldn't alloca() %d bytes!\n", (int)(strlen(tmp) + 1));
-    strcpy(cp, tmp);
-    while (cp) {
-	if ((cp2 = index(cp, ' ')) != NULL)
-	    *(cp2++) = '\0';
-	if (!distUnsetByName(DistTable, cp))
-	    msgDebug("distUnsetCustom: Warning, no such release \"%s\"\n", cp);
-	cp = cp2;
-    }
-    return DITEM_SUCCESS;
-}
-
-int
-distSetSrc(dialogMenuItem *self)
-{
-    int i;
-
-    dialog_clear_norefresh();
-    if (!dmenuOpenSimple(&MenuSrcDistributions, FALSE))
-	i = DITEM_FAILURE;
-    else
-	i = DITEM_SUCCESS;
-    distVerifyFlags();
-    return i | DITEM_RESTORE;
-}
-
-int
-distSetKernel(dialogMenuItem *self)
-{
-    int i;
-
-    dialog_clear_norefresh();
-    if (!dmenuOpenSimple(&MenuKernelDistributions, FALSE))
-	i = DITEM_FAILURE;
-    else
-	i = DITEM_SUCCESS;
-    distVerifyFlags();
-    return i | DITEM_RESTORE;
-}
-
-static Boolean got_intr = FALSE;
-
-/* timeout handler */
-static void
-handle_intr(int sig)
-{
-    msgDebug("User generated interrupt.\n");
-    got_intr = TRUE;
-}
-
-static int
-check_for_interrupt(void)
-{
-    if (got_intr) {
-	got_intr = FALSE;
-	return TRUE;
-    }
-    return FALSE;
-}
-
-/*
- * translate distribution filename to lower case
- * as doTARBALL does in release/Makefile
- */
-static void
-translateDist(char trdist[PATH_MAX], const char *dist)
-{
-    int j;
-
-    /*
-     * translate distribution filename to lower case
-     * as doTARBALL does in release/Makefile
-     */
-    for (j = 0; j < PATH_MAX-1 && dist[j] != '\0'; j++)
-	trdist[j] = tolower(dist[j]);
-    trdist[j] = '\0';
-}
-
-/*
- * Try to get distribution as multiple pieces, locating and parsing an
- * info file which tells us how many we need for this distribution.
- */
-static Boolean
-distExtractTarball(char *path, char *dist, char *my_dir, int is_base)
-{
-    char *buf = NULL, trdist[PATH_MAX], fname[PATH_MAX];
-    struct timeval start, stop;
-    int j, status, total, intr;
-    int cpid, zpid, fd2, chunk, numchunks;
-    properties dist_attr = NULL;
-    const char *tmp;
-    FILE *fp;
-
-    translateDist(trdist, dist);
-    if (isDebug())
-	msgDebug("%s: path \"%s\" dist \"%s\" trdist \"%s\" "
-		"my_dir \"%s\" %sis_base\n",
-		__func__, path, dist, trdist, my_dir, is_base ? "" : "!");
-
-    status = TRUE;
-    numchunks = 0;
-    snprintf(fname, sizeof (fname), "%s/%s.inf", path, trdist);
-
-getinfo:
-    fp = DEVICE_GET(mediaDevice, fname, TRUE);
-    intr = check_for_interrupt();
-    if (fp == (FILE *)IO_ERROR || intr || !mediaDevice) {
-	if (isDebug())
-	    msgDebug("%s: fname %s fp: %p, intr: %d mediaDevice: %p\n",
-		__func__, fname, fp, intr, mediaDevice);
-	/* Hard error, can't continue */
-	if (!msgYesNo("Unable to open %s: %s.\nReinitialize media?",
-		fname, !intr ? "I/O error." : "User interrupt.")) {
-	    DEVICE_SHUTDOWN(mediaDevice);
-	    if (!DEVICE_INIT(mediaDevice))
-		return (FALSE);
-	    goto getinfo;
-	} else
-	    return (FALSE);
-    } else if (fp == NULL) {
-	/* No attributes file, so try as a single file. */
-	snprintf(fname, sizeof(fname), "%s/%s.%s", path, trdist,
-	    USE_GZIP ? "tgz" : "tbz");
-	if (isDebug())
-	    msgDebug("%s: fp is NULL (1) fname: %s\n", __func__, fname);
-	/*
-	 * Passing TRUE as 3rd parm to get routine makes this a "probing"
-	 * get, for which errors are not considered too significant.
-	 */
-    getsingle:
-	fp = DEVICE_GET(mediaDevice, fname, TRUE);
-	intr = check_for_interrupt();
-	if (fp == (FILE *)IO_ERROR || intr || !mediaDevice) {
-	    if (isDebug())
-		msgDebug("%s: fname %s fp: %p, intr: %d mediaDevice: %p\n",
-		    __func__, fname, fp, intr, mediaDevice);
-	    /* Hard error, can't continue */
-	    msgConfirm("Unable to open %s: %s", fname,
-		!intr ? "I/O error" : "User interrupt");
-	    DEVICE_SHUTDOWN(mediaDevice);
-	    if (!DEVICE_INIT(mediaDevice))
-		return (FALSE);
-	    goto getsingle;
-	} else if (fp != NULL) {
-	    char *dir = root_bias(my_dir);
-
-	    dialog_clear_norefresh();
-	    msgNotify("Extracting %s into %s directory...", dist, dir);
-	    status = mediaExtractDist(dir, dist, fp);
-	    fclose(fp);
-	    return (status);
-	} else {
-	    if (isDebug())
-		msgDebug("%s: fp is NULL (2) fname %s\n", __func__, fname);
-	    return (FALSE);
-	}
-    }
-
-    if (isDebug())
-	msgDebug("Parsing attributes file for distribution %s\n", dist);
-
-    dist_attr = properties_read(fileno(fp));
-    intr = check_for_interrupt();
-    if (intr || !dist_attr) {
-	if (isDebug())
-	    msgDebug("%s: intr %d dist_attr %p\n", __func__, intr, dist_attr);
-	msgConfirm("Cannot parse information file for the %s distribution: %s\n"
-		   "Please verify that your media is valid and try again.",
-		   dist, !intr ? "I/O error" : "User interrupt");
-    } else {
-	tmp = property_find(dist_attr, "Pieces");
-	if (tmp)
-	    numchunks = strtol(tmp, 0, 0);
-    }
-    fclose(fp);
-    if (!numchunks) {
-	if (isDebug())
-	    msgDebug("%s: numchunks is zero\n", __func__);
-	return (TRUE);
-    }
-
-    if (isDebug())
-	msgDebug("Attempting to extract distribution from %u chunks.\n",
-	    numchunks);
-
-    total = 0;
-    (void)gettimeofday(&start, (struct timezone *)NULL);
-
-    /* We have one or more chunks, initialize unpackers... */
-    mediaExtractDistBegin(root_bias(my_dir), &fd2, &zpid, &cpid);
-
-    /* And go for all the chunks */
-    dialog_clear_norefresh();
-    for (chunk = 0; chunk < numchunks; chunk++) {
-	int n, retval, last_msg, chunksize, realsize;
-	char prompt[80];
-
-	last_msg = 0;
-
-    getchunk:
-	snprintf(fname, sizeof(fname), "cksum.%c%c",  (chunk / 26) + 'a',
-	    (chunk % 26) + 'a');
-	tmp = property_find(dist_attr, fname);
-	chunksize = 0;
-	if (tmp) {
-	    tmp = index(tmp, ' ');
-	    chunksize = strtol(tmp, 0, 0);
-	}
-	snprintf(fname, sizeof(fname), "%s/%s.%c%c", path, trdist, (chunk / 26) + 'a',
-	    (chunk % 26) + 'a');
-	if (isDebug())
-	    msgDebug("trying for piece %d of %d: %s\n", chunk + 1, numchunks,
-		fname);
-	fp = DEVICE_GET(mediaDevice, fname, FALSE);
-	intr = check_for_interrupt();
-	/* XXX: this can't work if we get an I/O error */
-	if (fp <= (FILE *)NULL || intr) {
-	    if (fp == NULL)
-		msgConfirm("Failed to find %s on this media.  Reinitializing media.", fname);
-	    else
-		msgConfirm("Failed to retrieve piece file %s.\n"
-			   "%s: Reinitializing media.",
-			   fname, !intr ? "I/O error" : "User interrupt");
-	    DEVICE_SHUTDOWN(mediaDevice);
-	    if (!DEVICE_INIT(mediaDevice))
-		goto punt;
-	    else
-		goto getchunk;
-	}
-
-	snprintf(prompt, sizeof(prompt), "Extracting %s into %s directory...",
-	    dist, root_bias(my_dir));
-	dialog_gauge("Progress", prompt, 8, 15, 6, 50,
-	    (chunk + 1) * 100 / numchunks);
-
-	buf = safe_realloc(buf, chunksize);
-	realsize = 0;
-	while (1) {
-	    int seconds;
-
-	    n = fread(buf + realsize, 1, BUFSIZ, fp);
-	    if (check_for_interrupt()) {
-		msgConfirm("Media read error:  User interrupt.");
-		fclose(fp);
-		goto punt;
-	    } else if (n <= 0)
-		break;
-	    total += n;
-	    realsize += n;
-
-	    /* Print statistics about how we're doing */
-	    (void) gettimeofday(&stop, (struct timezone *)0);
-	    stop.tv_sec = stop.tv_sec - start.tv_sec;
-	    stop.tv_usec = stop.tv_usec - start.tv_usec;
-	    if (stop.tv_usec < 0)
-		stop.tv_sec--, stop.tv_usec += 1000000;
-	    seconds = stop.tv_sec + (stop.tv_usec / 1000000.0);
-	    if (!seconds)
-		seconds = 1;
-
-	    if (seconds != last_msg) {
-		last_msg = seconds;
-		msgInfo("%10d bytes read from %s dist, chunk %2d of %2d @ %.1f KBytes/sec.",
-			total, dist, chunk + 1, numchunks,
-		        (total / seconds) / 1000.0);
-	    }
-	}
-	fclose(fp);
-
-	if (!chunksize || (realsize == chunksize)) {
-	    /* No substitution necessary */
-	    retval = write(fd2, buf, realsize);
-	    if (retval != realsize) {
-		fclose(fp);
-		dialog_clear_norefresh();
-		msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", retval, realsize);
-		    goto punt;
-	    }
-	} else {
-	    for (j = 0; j < realsize; j++) {
-		/* On finding CRLF, skip the CR; don't exceed end of buffer. */
-		if ((buf[j] != 0x0d) || (j == total - 1) || (buf[j + 1] != 0x0a)) {
-		    retval = write(fd2, buf + j, 1);
-		    if (retval != 1) {
-			fclose(fp);
-			dialog_clear_norefresh();
-			msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", j, chunksize);
-			goto punt;
-		    }
-		}
-	    }
-	}
-    }
-    goto done;
-
-punt:
-    status = FALSE;
-done:
-    properties_free(dist_attr);
-    close(fd2);
-    if (status != FALSE)
-	status = mediaExtractDistEnd(zpid, cpid);
-    else
-	(void)mediaExtractDistEnd(zpid, cpid);
-
-    safe_free(buf);
-    return (status);
-}
-
-static Boolean
-distExtract(char *parent, Distribution *me)
-{
-    int i, status;
-    char *path, *dist;
-    WINDOW *w = savescr();
-    struct sigaction old, new;
-    int canceled = 0;
-
-    status = TRUE;
-    if (isDebug())
-	msgDebug("distExtract: parent: %s, me: %s\n", parent ? parent : "(none)", me->my_name);
-
-    /* Make ^C fake a sudden timeout */
-    new.sa_handler = handle_intr;
-    new.sa_flags = 0;
-    (void)sigemptyset(&new.sa_mask);
-    dialog_clear_norefresh();
-    dialog_msgbox("Please Wait", "Extracting all requested distributions...", -1, -1, 0);
-    sigaction(SIGINT, &new, &old);
-
-    /* Loop through to see if we're in our parent's plans */
-    for (i = 0; me[i].my_name && canceled == 0; i++) {
-	dist = me[i].my_name;
-	path = parent ? parent : dist;
-
-	/* If our bit isn't set, go to the next */
-	if (!(me[i].my_bit & *(me[i].my_mask)))
-	    continue;
-
-	switch (me[i].my_type) {
-	case DT_SUBDIST:
-	    /* Recurse if we actually have a sub-distribution */
-	    status = distExtract(dist, me[i].my_data.my_dist);
-	    if (!status) {
-		dialog_clear_norefresh();
-		msgConfirm("Unable to transfer all components of the %s distribution.\n"
-		    "You may wish to switch media types and try again.\n",
-		    me[i].my_name);
-	    }
-	    break;
-	case DT_PACKAGE:
-	    dialog_clear_norefresh();
-	    msgNotify("Installing %s distribution...", dist);
-	    status = (package_add(me[i].my_data.my_string) == DITEM_SUCCESS);
-	    if (!status)
-		dialog_clear_norefresh();
-	    break;
-	case DT_TARBALL:
-	    status = distExtractTarball(path, dist, me[i].my_data.my_string,
-		&me[i] == BASE_DIST);
-	    if (!status) {
-		dialog_clear_norefresh();
-		if (me[i].my_bit != DIST_LOCAL) {
-		    status = msgYesNo("Unable to transfer the %s distribution from\n%s.\n\n"
-			              "Do you want to try to retrieve it again?",
-				      me[i].my_name, mediaDevice->name);
-		    if (status == 0)
-			--i;
-		    else
-			canceled = 1;	
-
-		    status = FALSE;
-		} else {
-			// ignore any failures with DIST_LOCAL
-			status = TRUE;
-		}
-	    }
-	    break;
-	}
-	
-	/*
-	 * If extract was successful, remove ourselves from further
-	 * consideration.
-	 */
-	if (status)
-	    *(me[i].my_mask) &= ~(me[i].my_bit);
-    }
-
-    sigaction(SIGINT, &old, NULL);	/* Restore signal handler */
-    restorescr(w);
-    return status;
-}
-
-int
-distSetDoc(dialogMenuItem *self)
-{
-    int i;
-
-    /* Assume no docs for non-interactive installs. */
-    if (variable_get(VAR_NONINTERACTIVE))
-	return DITEM_SUCCESS | DITEM_RESTORE;
-
-    dialog_clear_norefresh();
-    if (!dmenuOpenSimple(&MenuDocInstall, FALSE))
-	i = DITEM_FAILURE;
-    else
-	i = DITEM_SUCCESS;
-
-    distVerifyFlags();
-
-    return i | DITEM_RESTORE;
-}
-
-int
-distSetDocMenu(dialogMenuItem *self)
-{
-    int i, status;
-    WINDOW *w;
-
-    if (RunningAsInit && !strstr(variable_get(SYSTEM_STATE), "install")) {
-	    msgConfirm("This option may only be used after the system is installed, sorry!");
-	    return DITEM_FAILURE;
-    }
-
-    dialog_clear_norefresh();
-    if (!dmenuOpenSimple(&MenuDocInstall, FALSE))
-	i = DITEM_FAILURE;
-    else
-	i = DITEM_SUCCESS;
-
-    distVerifyFlags();
-
-    dialog_clear_norefresh();
-    w = savescr();
-    msgNotify("Attempting to install all selected documentations...");
-
-    for (i = 0; DocDistTable[i].my_name; i++) {
-	    if (!(DocDistTable[i].my_bit & *(DocDistTable[i].my_mask)))
-		    continue;
-	     dialog_clear_norefresh();
-	     msgNotify("Installing %s distribution...", DocDistTable[i].my_name);
-	     status = (package_add(DocDistTable[i].my_data.my_string) == DITEM_SUCCESS);
-	     if (!status)
-		     break;
-    }
-
-    dialog_clear_norefresh();
-
-    restorescr(w);
-    return (status ? DITEM_SUCCESS : DITEM_FAILURE);
-}
-
-static void
-printSelected(char *buf, int selected, Distribution *me, int *col)
-{
-    int i;
-
-    /* Loop through to see if we're in our parent's plans */
-    for (i = 0; me[i].my_name; i++) {
-
-	/* If our bit isn't set, go to the next */
-	if (!(me[i].my_bit & selected))
-	    continue;
-
-	*col += strlen(me[i].my_name);
-	if (*col > 50) {
-	    *col = 0;
-	    strcat(buf, "\n");
-	}
-	sprintf(&buf[strlen(buf)], " %s", me[i].my_name);
-
-	/* Recurse if have a sub-distribution */
-	if (me[i].my_type == DT_SUBDIST)
-	    printSelected(buf, *(me[i].my_mask), me[i].my_data.my_dist, col);
-    }
-}
-
-int
-distExtractAll(dialogMenuItem *self)
-{
-    int old_dists, old_kernel, status = DITEM_SUCCESS;
-    char buf[512];
-    int extract_status = TRUE;
-    WINDOW *w;
-
-    /* paranoia */
-    if (!Dists) {
-	if (!dmenuOpenSimple(&MenuSubDistributions, FALSE) || !Dists)
-	    return DITEM_FAILURE;
-    }
-
-    if (!mediaVerify() || !DEVICE_INIT(mediaDevice))
-	return DITEM_FAILURE;
-
-    old_dists = Dists;
-    old_kernel = KernelDists;
-    distVerifyFlags();
-
-    dialog_clear_norefresh();
-    w = savescr();
-    msgNotify("Attempting to install all selected distributions..");
-
-    extract_status = distExtract(NULL, DistTable);
-
-    dialog_clear_norefresh();
-    /* Only do base fixup if base dist was successfully extracted */
-    if ((old_dists & DIST_BASE) && !(Dists & DIST_BASE))
-	status |= installFixupBase(self);
-    /* Only do kernel fixup if kernel dist was successfully extracted */
-    if ((old_dists & DIST_KERNEL) && !(Dists & DIST_KERNEL))
-	status |= installFixupKernel(self, old_kernel);
-
-    /* Clear any local dist flags now */
-    Dists &= ~DIST_LOCAL;
-
-    if (Dists) {
-	int col = 0;
-
-	buf[0] = '\0';
-	dialog_clear_norefresh();
-	printSelected(buf, Dists, DistTable, &col);
-	dialog_clear_norefresh();
-	if (col) {
-	    msgConfirm("Couldn't extract the following distributions.  This may\n"
-		       "be because they were not available on the installation\n"
-		       "media you've chosen:\n\n\t%s", buf);
-	}
-    }
-    restorescr(w);
-
-    if (extract_status == FALSE)
-	status = FALSE;
-
-    return status;
-}
diff --git a/usr.sbin/sysinstall/dist.h b/usr.sbin/sysinstall/dist.h
deleted file mode 100644
index 0fd004f5d997..000000000000
--- a/usr.sbin/sysinstall/dist.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/* $FreeBSD$  */
-
-#ifndef _DIST_H_INCLUDE
-#define _DIST_H_INCLUDE
-
-/* Bitfields for distributions - hope we never have more than 32! :-) */
-#define DIST_BASE		0x00001
-#define DIST_GAMES		0x00002
-#define DIST_MANPAGES		0x00004
-#define DIST_PROFLIBS		0x00008
-#define DIST_DICT		0x00010
-#define DIST_SRC		0x00020
-/* Documentation from FreeBSD docproj */
-#define DIST_DOC		0x00040
-#define DIST_INFO		0x00080
-#define DIST_CATPAGES		0x00200
-#define DIST_PORTS		0x00400
-#define DIST_LOCAL		0x00800
-#if defined(__amd64__) || defined(__powerpc64__)
-#define DIST_LIB32		0x01000
-#endif
-#define	DIST_KERNEL		0x02000
-/* Userland documentation */
-#define DIST_DOCUSERLAND	0x04000
-#define DIST_ALL		0xFFFFF
-
-/* Subtypes for DOC packages */
-#define DIST_DOC_BN		0x00001
-#define DIST_DOC_DA		0x00002
-#define DIST_DOC_DE		0x00004
-#define DIST_DOC_EL		0x00008
-#define DIST_DOC_EN		0x00010
-#define DIST_DOC_ES		0x00020
-#define DIST_DOC_FR		0x00040
-#define DIST_DOC_HU		0x00080
-#define DIST_DOC_IT		0x00100
-#define DIST_DOC_JA		0x00200
-#define DIST_DOC_MN		0x00400
-#define DIST_DOC_NL		0x00800
-#define DIST_DOC_PL		0x01000
-#define DIST_DOC_PT		0x02000
-#define DIST_DOC_RU		0x04000
-#define DIST_DOC_SR		0x08000
-#define DIST_DOC_TR		0x10000
-#define DIST_DOC_ZH_CN		0x20000
-#define DIST_DOC_ZH_TW		0x40000
-#define DIST_DOC_ALL		0xFFFFF
-
-/* Subtypes for SRC distribution */
-#define DIST_SRC_BASE		0x00001
-#define DIST_SRC_CONTRIB	0x00002
-#define DIST_SRC_GNU		0x00004
-#define DIST_SRC_ETC		0x00008
-#define DIST_SRC_GAMES		0x00010
-#define DIST_SRC_INCLUDE	0x00020
-#define DIST_SRC_LIB		0x00040
-#define DIST_SRC_LIBEXEC	0x00080
-#define DIST_SRC_TOOLS		0x00100
-#define DIST_SRC_RELEASE	0x00200
-#define DIST_SRC_SBIN		0x00400
-#define DIST_SRC_SHARE		0x00800
-#define DIST_SRC_SYS		0x01000
-#define DIST_SRC_UBIN		0x02000
-#define DIST_SRC_USBIN		0x04000
-#define DIST_SRC_BIN		0x08000
-#define DIST_SRC_SCRYPTO	0x10000
-#define DIST_SRC_SSECURE	0x20000
-#define DIST_SRC_SKERBEROS5	0x40000
-#define DIST_SRC_RESCUE		0x80000
-#define DIST_SRC_CDDL		0x100000
-#define DIST_SRC_ALL		0x3FFFFF
-
-/* Subtypes for KERNEL distribution */
-#define DIST_KERNEL_GENERIC	0x00001
-#define DIST_KERNEL_SMP		0x00002
-#define DIST_KERNEL_ALL		0xFFFFF
-
-#ifdef __powerpc64__
-#define GENERIC_KERNEL_NAME	"GENERIC64"
-#else
-#define GENERIC_KERNEL_NAME	"GENERIC"
-#endif
-
-/* Canned distribution sets */
-
-#define _DIST_USER \
-	( DIST_BASE | DIST_KERNEL | DIST_DOC | DIST_DOCUSERLAND | DIST_MANPAGES | DIST_DICT )
-
-#define _DIST_DEVELOPER \
-	( _DIST_USER | DIST_PROFLIBS | DIST_INFO | DIST_SRC )
-
-#endif	/* _DIST_H_INCLUDE */
diff --git a/usr.sbin/sysinstall/dmenu.c b/usr.sbin/sysinstall/dmenu.c
deleted file mode 100644
index 1aef667fa4e7..000000000000
--- a/usr.sbin/sysinstall/dmenu.c
+++ /dev/null
@@ -1,393 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated for what's essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/param.h>
-#include <errno.h>
-
-#define MAX_MENU		15
-
-static Boolean exited;
-
-int
-dmenuDisplayFile(dialogMenuItem *tmp)
-{
-    systemDisplayHelp((char *)tmp->data);
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuSubmenu(dialogMenuItem *tmp)
-{
-    return (dmenuOpenSimple((DMenu *)(tmp->data), FALSE) ? DITEM_SUCCESS : DITEM_FAILURE);
-}
-
-int
-dmenuSystemCommand(dialogMenuItem *self)
-{
-    WINDOW *w = NULL;	/* Keep lint happy */
-
-    /* If aux is set, the command is known not to produce any screen-spoiling output */
-    if (!self->aux)
-	w = savescr();
-    systemExecute((char *)self->data);
-    if (!self->aux)
-	restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuSystemCommandBox(dialogMenuItem *tmp)
-{
-    WINDOW *w = savescr();
-    
-    use_helpfile(NULL);
-    use_helpline("Select OK to dismiss this dialog");
-    dialog_prgbox(tmp->title, (char *)tmp->data, 22, 76, 1, 1);
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuExit(dialogMenuItem *tmp)
-{
-    exited = TRUE;
-    return DITEM_LEAVE_MENU;
-}
-
-int
-dmenuSetVariable(dialogMenuItem *tmp)
-{
-    variable_set((char *)tmp->data, *((char *)tmp->data) != '_');
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuSetVariables(dialogMenuItem *tmp)
-{
-    char *cp1, *cp2;
-    char *copy = strdup((char *)tmp->data);
-
-    for (cp1 = copy; cp1 != NULL;) {
-	cp2 = index(cp1, ',');
-	if (cp2 != NULL) *cp2++ = '\0';
-	variable_set(cp1, *cp1 != '_');
-	cp1 = cp2;
-    }
-    free(copy);
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuSetCountryVariable(dialogMenuItem *tmp)
-{
-    variable_set((char *)tmp->data, FALSE);
-#ifdef WITH_SYSCONS
-    /* Don't prompt the user for a keymap if they're using the default locale. */
-    if (!strcmp(variable_get(VAR_COUNTRY), DEFAULT_COUNTRY))
-	return DITEM_SUCCESS;
-
-    return keymapMenuSelect(tmp);
-#else
-    return DITEM_SUCCESS;
-#endif
-}
-
-int
-dmenuSetKmapVariable(dialogMenuItem *tmp)
-{
-    char *lang;
-    int err;
-
-    variable_set((char *)tmp->data, TRUE);
-    lang = variable_get(VAR_KEYMAP);
-    if (lang != NULL)
-    {
-	err = loadKeymap(lang);
-	if (err == -1)
-	    msgConfirm("No appropriate keyboard map found, sorry.");
-	else if (err == -2)
-	    msgConfirm("Error installing keyboard map, errno = %d.", errno);
-    }
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuToggleVariable(dialogMenuItem *tmp)
-{
-    char *var, *cp;
-    int status;
-
-    if (!(var = strdup((char *)tmp->data))) {
-	msgConfirm("Incorrect data field for `%s'!", tmp->title);
-	return DITEM_FAILURE;
-    }
-    if (!(cp = index(var, '='))) {
-	msgConfirm("Data field for %s is not in var=value format!", tmp->title);
-	return DITEM_FAILURE;
-    }
-    status = variable_check(var);
-    *cp = '\0';
-    variable_set2(var, status ? "NO" : "YES", *var != '_');
-    free(var);
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuISetVariable(dialogMenuItem *tmp)
-{
-    char *ans, *p, *var;
-
-    if (!(var = strdup((char *)tmp->data))) {
-	msgConfirm("Incorrect data field for `%s'!", tmp->title);
-	return DITEM_FAILURE;
-    }
-    if ((p = index(var, '=')) != NULL)
-	*p = '\0';
-    ans = msgGetInput(variable_get(var), tmp->title, 1);
-    if (!ans) {
-	free(var);
-	return DITEM_FAILURE;
-    } else if (!*ans)
-	variable_unset(var);
-    else
-	variable_set2(var, ans, *var != '_');
-    free(var);
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuSetFlag(dialogMenuItem *tmp)
-{
-    if (*((unsigned int *)tmp->data) & tmp->aux)
-	*((unsigned int *)tmp->data) &= ~tmp->aux;
-    else
-	*((unsigned int *)tmp->data) |= tmp->aux;
-    return DITEM_SUCCESS;
-}
-
-int
-dmenuSetValue(dialogMenuItem *tmp)
-{
-    *((unsigned int *)tmp->data) = tmp->aux;
-    return DITEM_SUCCESS;
-}
-
-/* Traverse menu but give user no control over positioning */
-Boolean
-dmenuOpenSimple(DMenu *menu, Boolean buttons)
-{
-    int choice, scroll, curr, max;
-
-    choice = scroll = curr = max = 0;
-    return dmenuOpen(menu, &choice, &scroll, &curr, &max, buttons);
-}
-
-/* Work functions for the state hook */
-int
-dmenuFlagCheck(dialogMenuItem *item)
-{
-    return (*((unsigned int *)item->data) & item->aux);
-}
-
-int
-dmenuVarCheck(dialogMenuItem *item)
-{
-    char *w;
-
-    w = (char *)item->aux;
-    if (!w)
-	w = (char *)item->data;
-    return variable_check(w);
-}
-
-int
-dmenuVarsCheck(dialogMenuItem *item)
-{
-    int res, init;
-    char *w, *cp1, *cp2;
-    char *copy;
-
-    w = (char *)item->aux;
-    if (!w)
-	w = (char *)item->data;
-    if (!w)
-	return FALSE;
-    
-    copy = strdup(w);
-    res = TRUE;
-    init = FALSE;
-    for (cp1 = copy; cp1 != NULL;) {
-        init = TRUE;
-	cp2 = index(cp1, ',');
-	if (cp2 != NULL)
-	    *cp2++ = '\0';
-	res = res && variable_check(cp1);
-	cp1 = cp2;
-    }
-    free(copy);
-    return res && init;
-}
-
-int
-dmenuRadioCheck(dialogMenuItem *item)
-{
-    return (*((int *)item->data) == item->aux);
-}
-
-static int
-menu_height(DMenu *menu, int n)
-{
-    int max;
-    char *t;
-
-    max = MAX_MENU;
-    if (StatusLine > 24)
-	max += StatusLine - 24;
-    for (t = menu->prompt; *t; t++) {
-	if (*t == '\n')
-	    --max;
-    }
-    return n > max ? max : n;
-}
-
-/* Find a menu item that matches any field. */
-int
-dmenuFindItem(DMenu *menu, const char *prompt, const char *title, void *data)
-{
-    dialogMenuItem *items = menu->items;
-    int i;
-
-    for (i = 0; items[i].prompt; ++i)
-	if ((prompt && !strcmp(items[i].prompt, prompt)) ||
-		(title && !strcmp(items[i].title, title)) ||
-		(data && items[i].data == data))
-	    return i;
-
-    return -1;
-}
-
-/* Set the default item for a menu by index and scroll to it. */
-void
-dmenuSetDefaultIndex(DMenu *menu, int *choice, int *scroll, int *curr, int *max)
-{
-    int nitem;
-    int height;
-
-    *curr = *max = 0;
-
-    for (nitem = 0; menu->items[nitem].prompt; ++nitem);
-
-    height = menu_height(menu, nitem);
-    if (*choice > height)
-    {
-	*scroll = MIN(nitem - height, *choice);
-	*choice = *choice - *scroll;
-    }
-    else
-	*scroll = 0;
-}
-
-/* Set the default menu item that matches any field and scroll to it. */
-Boolean
-dmenuSetDefaultItem(DMenu *menu, const char *prompt, const char *title, void *data,
-		    int *choice, int *scroll, int *curr, int *max)
-{
-    if ((*choice = dmenuFindItem(menu, prompt, title, data)) != -1)
-    {
-	dmenuSetDefaultIndex(menu, choice, scroll, curr, max);
-	return TRUE;
-    }
-    else
-    {
-	*choice = *scroll = *curr = *max = 0;
-	return FALSE;
-    }
-}
-
-/* Traverse over an internal menu */
-Boolean
-dmenuOpen(DMenu *menu, int *choice, int *scroll, int *curr, int *max, Boolean buttons)
-{
-    int n, rval = 0;
-    dialogMenuItem *items;
-
-    items = menu->items;
-    if (buttons)
-	items += 2;
-    /* Count up all the items */
-    for (n = 0; items[n].title; n++);
-
-    while (1) {
-	char buf[FILENAME_MAX];
-	WINDOW *w = savescr();
-
-	/* Any helpful hints, put 'em up! */
-	use_helpline(menu->helpline);
-	use_helpfile(systemHelpFile(menu->helpfile, buf));
-	dialog_clear_norefresh();
-	/* Pop up that dialog! */
-	if (menu->type & DMENU_NORMAL_TYPE)
-	    rval = dialog_menu((u_char *)menu->title, (u_char *)menu->prompt,
-		-1, -1, menu_height(menu, n), -n, items,
-		(char *)(uintptr_t)buttons, choice, scroll);
-
-	else if (menu->type & DMENU_RADIO_TYPE)
-	    rval = dialog_radiolist((u_char *)menu->title,
-		(u_char *)menu->prompt, -1, -1, menu_height(menu, n), -n,
-		items, (char *)(uintptr_t)buttons);
-
-	else if (menu->type & DMENU_CHECKLIST_TYPE)
-	    rval = dialog_checklist((u_char *)menu->title,
-		(u_char *)menu->prompt, -1, -1, menu_height(menu, n), -n,
-		items, (char *)(uintptr_t)buttons);
-	else
-	    msgFatal("Menu: `%s' is of an unknown type\n", menu->title);
-	if (exited) {
-	    exited = FALSE;
-	    restorescr(w);
-	    return TRUE;
-	}
-	else if (rval) {
-	    restorescr(w);
-	    return FALSE;
-	}
-	else if (menu->type & DMENU_SELECTION_RETURNS) {
-	    restorescr(w);
-	    return TRUE;
-	}
-    }
-}
diff --git a/usr.sbin/sysinstall/doc.c b/usr.sbin/sysinstall/doc.c
deleted file mode 100644
index a35a68eeebe4..000000000000
--- a/usr.sbin/sysinstall/doc.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated for what's essentially a complete rewrite.
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include "sysinstall.h"
-
-/*
- * This is called from the main menu.  Try to find a copy of Lynx from somewhere
- * and fire it up on the first copy of the handbook we can find.
- */
-int
-docBrowser(dialogMenuItem *self)
-{
-    int ret;
-    char *browser = variable_get(VAR_BROWSER_PACKAGE);
-
-    if (RunningAsInit && !strstr(variable_get(SYSTEM_STATE), "install")) {
-	msgConfirm("This option may only be used after the system is installed, sorry!");
-	return DITEM_FAILURE;
-    }
-
-    /* First, make sure we have whatever browser we've chosen is here */
-    if (!package_installed(browser)) {
-	ret = package_add(browser);
-    	if (DITEM_STATUS(ret) != DITEM_SUCCESS) {
-	    msgConfirm("Unable to install the %s HTML browser package.  You may\n"
-		       "wish to verify that your media is configured correctly and\n"
-		       "try again.", browser);
-	    return ret;
-	}
-    }
-
-    if (!file_executable(variable_get(VAR_BROWSER_BINARY))) {
-	if (!msgYesNo("Hmmm.  The %s package claims to have installed, but I can't\n"
-		      "find its binary in %s!  You may wish to try a different\n"
-		      "location to load the package from (go to Media menu) and see if that\n"
-		      "makes a difference.\n\n"
-		      "I suggest that we remove the version that was extracted since it does\n"
-		      "not appear to be correct.   Would you like me to do that now?",
-		      browser, variable_get(VAR_BROWSER_BINARY)))
-	    vsystem("pkg_delete %s %s", !strcmp(variable_get(VAR_CPIO_VERBOSITY), "high") ? "-v" : "", browser);
-	return DITEM_FAILURE;
-    }
-
-    /* Run browser on the appropriate doc */
-    if (dmenuOpenSimple(&MenuHTMLDoc, FALSE))
-	return DITEM_SUCCESS;
-    else
-	return DITEM_FAILURE;
-}
-
-/* Try to show one of the documents requested from the HTML doc menu */
-int
-docShowDocument(dialogMenuItem *self)
-{
-    char tmp[512], target[512];
-    char *where, *browser = variable_get(VAR_BROWSER_BINARY);
-    char *str = self->prompt;
-
-    if (!file_executable(browser)) {
-	msgConfirm("Can't find the browser in %s!  Please ensure that it's\n"
-		   "properly set in the Options editor.", browser);
-	return DITEM_FAILURE;
-    }
-    /* Default to Home */
-    where = strcpy(target, "http://www.freebsd.org");
-    if (strstr(str, "Other")) {
-	where = msgGetInput("http://www.freebsd.org", "Please enter the URL of the location you wish to visit.");
-	if (where)
-	    strcpy(target, where);
-    }
-    else if (strstr(str, "FAQ")) {
-	where = strcpy(target, "/usr/local/share/doc/freebsd/faq/index.html");
-	if (!file_readable(target))
-	    where = strcpy(target, "http://www.freebsd.org/doc/en_US.ISO8859-1/books/faq");
-    }
-    else if (strstr(str, "Handbook")) {
-	where = strcpy(target, "/usr/local/share/doc/freebsd/handbook/index.html");
-	if (!file_readable(target))
-	    where = strcpy(target, "http://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook");
-    }
-    if (where) {
-	sprintf(tmp, "%s %s", browser, target);
-	systemExecute(tmp);
-	return DITEM_SUCCESS;
-    }
-    else {
-	msgConfirm("Hmmmmm!  I can't seem to access the documentation you selected!\n"
-		   "Have you installed the english documentation?  Is your network connected?");
-	return DITEM_FAILURE;
-    }
-}
diff --git a/usr.sbin/sysinstall/dos.c b/usr.sbin/sysinstall/dos.c
deleted file mode 100644
index 564e33ff4374..000000000000
--- a/usr.sbin/sysinstall/dos.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- * Copyright (c) 1995
- * 	Gary J Palmer. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/stat.h>
-#include <sys/errno.h>
-#include <sys/param.h>
-#include <sys/wait.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <grp.h>
-#define MSDOSFS
-#include <sys/mount.h>
-#include <fs/msdosfs/msdosfsmount.h>
-#undef MSDOSFS
-
-static Boolean DOSMounted;
-static char mountpoint[] = "/dist";
-
-Boolean
-mediaInitDOS(Device *dev)
-{
-    struct msdosfs_args args;
-
-    if (DOSMounted)
-	return TRUE;
-     
-    Mkdir(mountpoint);
-    memset(&args, 0, sizeof(args));
-    args.fspec = dev->devname;
-    args.uid = args.gid = 0;
-    args.mask = 0777;
-
-    if (mount("msdosfs", mountpoint, MNT_RDONLY, (caddr_t)&args) == -1) {
-	msgConfirm("Error mounting %s on %s: %s (%u)", args.fspec, mountpoint, strerror(errno), errno);
-	return FALSE;
-    }
-    DOSMounted = TRUE;
-    return TRUE;
-}
-
-FILE *
-mediaGetDOS(Device *dev, char *file, Boolean probe)
-{
-    return mediaGenericGet(mountpoint, file);
-}
-
-void
-mediaShutdownDOS(Device *dev)
-{
-    if (!DOSMounted)
-	return;
-    if (unmount(mountpoint, MNT_FORCE) != 0)
-	msgConfirm("Could not unmount the DOS partition from %s: %s",
-		   mountpoint, strerror(errno));
-    else
-	DOSMounted = FALSE;
-    return;
-}
diff --git a/usr.sbin/sysinstall/floppy.c b/usr.sbin/sysinstall/floppy.c
deleted file mode 100644
index 0101bde3ce01..000000000000
--- a/usr.sbin/sysinstall/floppy.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- * Copyright (c) 1995
- * 	Gary J Palmer. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/* These routines deal with getting things off of floppy media */
-
-#include "sysinstall.h"
-#include <sys/fcntl.h>
-#include <sys/stat.h>
-#include <sys/errno.h>
-#include <sys/param.h>
-#include <sys/wait.h>
-#include <unistd.h>
-#include <grp.h>
-
-#define MSDOSFS
-#include <sys/mount.h>
-#include <fs/msdosfs/msdosfsmount.h>
-#undef MSDOSFS
-
-#include <ufs/ufs/ufsmount.h>
-static Boolean floppyMounted;
-
-char *distWanted;
-static char mountpoint[] = "/dist";
-
-Boolean
-mediaInitFloppy(Device *dev)
-{
-    struct msdosfs_args dosargs;
-    struct ufs_args u_args;
-    char *mp;
-
-    if (floppyMounted)
-	return TRUE;
-
-    mp = dev->private ? (char *)dev->private : mountpoint;
-    if (Mkdir(mp)) {
-	msgConfirm("Unable to make %s directory mountpoint for %s!", mp, dev->devname);
-	return FALSE;
-    }
-
-    msgDebug("Init floppy called for %s distribution.\n", distWanted ? distWanted : "some");
-
-    if (!variable_get(VAR_NONINTERACTIVE)) {
-	if (!distWanted)
-	    msgConfirm("Please insert floppy in %s", dev->description);
-	else
-	    msgConfirm("Please insert floppy containing %s in %s",
-			distWanted, dev->description);
-    }
-
-    memset(&dosargs, 0, sizeof dosargs);
-    dosargs.fspec = dev->devname;
-    dosargs.uid = dosargs.gid = 0;
-    dosargs.mask = 0777;
-
-    memset(&u_args, 0, sizeof(u_args));
-    u_args.fspec = dev->devname;
-
-    if (mount("msdosfs", mp, MNT_RDONLY, (caddr_t)&dosargs) != -1)
-	goto success;
-    if (mount("ufs", mp, MNT_RDONLY, (caddr_t)&u_args) != -1)
-	goto success;
-
-    msgConfirm("Error mounting floppy %s (%s) on %s : %s",
-	       dev->name, dev->devname, mp, strerror(errno));
-    return FALSE;
-
-success:
-    floppyMounted = TRUE;
-    distWanted = NULL;
-    return TRUE;
-}
-
-FILE *
-mediaGetFloppy(Device *dev, char *file, Boolean probe)
-{
-    char	buf[PATH_MAX], *mp;
-    FILE	*fp;
-    int		nretries = 5;
-
-    /*
-     * floppies don't use mediaGenericGet() because it's too expensive
-     * to speculatively open files on a floppy disk.  Make user get it
-     * right or give up with floppies.
-     */
-    mp = dev->private ? (char *)dev->private : mountpoint;
-    snprintf(buf, PATH_MAX, "%s/%s", mp, file);
-    if (!file_readable(buf)) {
-	if (probe)
-	    return NULL;
-	else {
-	    while (!file_readable(buf)) {
-		if (!--nretries) {
-		    msgConfirm("GetFloppy: Failed to get %s after retries;\ngiving up.", buf);
-		    return NULL;
-		}
-		distWanted = buf;
-		mediaShutdownFloppy(dev);
-		if (!mediaInitFloppy(dev))
-		    return NULL;
-	    }
-	}
-    }
-    fp = fopen(buf, "r");
-    return fp;
-}
-
-void
-mediaShutdownFloppy(Device *dev)
-{
-    if (floppyMounted) {
-    	char *mp = dev->private ? (char *)dev->private : mountpoint;
-
-	if (unmount(mp, MNT_FORCE) != 0)
-	    msgDebug("Umount of floppy on %s failed: %s (%d)\n", mp, strerror(errno), errno);
-	else {
-	    floppyMounted = FALSE;
-	    if (!variable_get(VAR_NONINTERACTIVE) && variable_cmp(SYSTEM_STATE, "fixit"))
-		msgConfirm("You may remove the floppy from %s", dev->description);
-	}
-    }
-}
diff --git a/usr.sbin/sysinstall/ftp.c b/usr.sbin/sysinstall/ftp.c
deleted file mode 100644
index 14fd74ba13ea..000000000000
--- a/usr.sbin/sysinstall/ftp.c
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <sys/param.h>
-#include <sys/wait.h>
-#include <netdb.h>
-#include <pwd.h>
-#include <ftpio.h>
-
-Boolean ftpInitted = FALSE;
-static FILE *OpenConn;
-int FtpPort;
-
-/* List of sub directories to look for under a given FTP server. */
-const char *ftp_dirs[] = { ".", "releases/"MACHINE, "snapshots/"MACHINE,
-    "pub/FreeBSD", "pub/FreeBSD/releases/"MACHINE,
-    "pub/FreeBSD/snapshots/"MACHINE, NULL };
-
-/* Brings up attached network device, if any - takes FTP device as arg */
-static Boolean
-netUp(Device *dev)
-{
-    Device *netdev = (Device *)dev->private;
-
-    if (netdev)
-	return DEVICE_INIT(netdev);
-    else
-	return TRUE;	/* No net == happy net */
-}
-
-/* Brings down attached network device, if any - takes FTP device as arg */
-static void
-netDown(Device *dev)
-{
-    Device *netdev = (Device *)dev->private;
-
-    if (netdev)
-	DEVICE_SHUTDOWN(netdev);
-}
-
-Boolean
-mediaInitFTP(Device *dev)
-{
-    int i, code, af, fdir;
-    char *cp, *rel, *hostname, *dir;
-    char *user, *login_name, password[80];
-
-    if (ftpInitted)
-	return TRUE;
-
-    if (OpenConn) {
-	fclose(OpenConn);
-	OpenConn = NULL;
-    }
-
-    /* If we can't initialize the network, bag it! */
-    if (!netUp(dev))
-	return FALSE;
-
-try:
-    cp = variable_get(VAR_FTP_PATH);
-    if (!cp) {
-	if (DITEM_STATUS(mediaSetFTP(NULL)) == DITEM_FAILURE || (cp = variable_get(VAR_FTP_PATH)) == NULL) {
-	    msgConfirm("Unable to get proper FTP path.  FTP media not initialized.");
-	    netDown(dev);
-	    return FALSE;
-	}
-    }
-
-    hostname = variable_get(VAR_FTP_HOST);
-    dir = variable_get(VAR_FTP_DIR);
-    if (!hostname || !dir) {
-	msgConfirm("Missing FTP host or directory specification.  FTP media not initialized,");
-	netDown(dev);
-	return FALSE;
-    }
-    user = variable_get(VAR_FTP_USER);
-    login_name = (!user || !*user) ? "anonymous" : user;
-
-    if (variable_get(VAR_FTP_PASS))
-	SAFE_STRCPY(password, variable_get(VAR_FTP_PASS));
-    else if (RunningAsInit)
-	sprintf(password, "installer@%s", variable_get(VAR_HOSTNAME));
-    else {
-	struct passwd *pw;
-	char *user;
-
-	pw = getpwuid(getuid());
-	user = pw ? pw->pw_name : "ftp";
-	sprintf(password, "%s@%s", user, variable_get(VAR_HOSTNAME));
-    }
-    af = variable_cmp(VAR_IPV6_ENABLE, "YES") ? AF_INET : AF_UNSPEC;
-    msgNotify("Logging in to %s@%s..", login_name, hostname);
-    if ((OpenConn = ftpLoginAf(hostname, af, login_name, password, FtpPort, isDebug(), &code)) == NULL) {
-	msgConfirm("Couldn't open FTP connection to %s:\n  %s.", hostname, ftpErrString(code));
-	goto punt;
-    }
-
-    ftpPassive(OpenConn, !strcmp(variable_get(VAR_FTP_STATE), "passive"));
-    ftpBinary(OpenConn);
-    if (dir && *dir != '\0') {
-	if ((i = ftpChdir(OpenConn, dir)) != 0) {
-	    if (i == 550)
-		msgConfirm("No such directory ftp://%s/%s\n"
-			   "please check your URL and try again.", hostname, dir);
-	    else
-		msgConfirm("FTP chdir to ftp://%s/%s returned error status:\n  %s.", hostname, dir, ftpErrString(i));
-	    goto punt;
-	}
-    }
-
-    /*
-     * Now that we've verified that the path we're given is ok, let's try to
-     * be a bit intelligent in locating the release we are looking for.  First
-     * off, if the release is specified as "__RELEASE" or "any", then just
-     * assume that the current directory is the one we want and give up.
-     */
-    rel = variable_get(VAR_RELNAME);
-    if (strcmp(rel, "__RELEASE") && strcmp(rel, "any")) {
-	/*
-	 * Ok, since we have a release variable, let's walk through the list
-	 * of directories looking for a release directory.  The first one to
-	 * match wins.  For each case, we chdir to ftp_dirs[fdir] first.  If
-	 * that fails, we skip to the next one.  Otherwise, we try to chdir to
-	 * rel.  If it succeeds we break out.  If it fails, then we go back to
-	 * the base directory and try again.  Lots of chdirs, but oh well. :)
-	 */
-	for (fdir = 0; ftp_dirs[fdir]; fdir++) {
-	    /* Avoid sending CWD . commands which confuse some ftp servers */
-	    if (strcmp(ftp_dirs[fdir], ".") &&
-		(ftpChdir(OpenConn, (char *)ftp_dirs[fdir]) != 0))
-		continue;
-	    if (ftpChdir(OpenConn, rel) == 0) {
-		ftpInitted = TRUE;
-		return TRUE;
-	    }
-	    else	/* reset to "root" dir for a fresh try */
-		ftpChdir(OpenConn, "/");
-	}
-
-	/*
-	 * If we get here, then all of the directories we tried failed, so
-	 * print out the error message and ask the user if they want to try
-	 * again.
-	 */
-	if (!msgYesNo("Warning:  Can't find the `%s' distribution on this\n"
-		      "FTP server.  You may need to visit a different server for\n"
-		      "the release you are trying to fetch or go to the Options\n"
-		      "menu and to set the release name to explicitly match what's\n"
-		      "available on %s (or set to \"any\").\n\n"
-		      "Would you like to select another FTP server?",
-		      rel, hostname)) {
-	    variable_unset(VAR_FTP_PATH);
-	    if (DITEM_STATUS(mediaSetFTP(NULL)) != DITEM_FAILURE)
-		goto try;
-	}
-    } else {
-	ftpInitted = TRUE;
-	return TRUE;
-    }
-
-punt:
-    ftpInitted = FALSE;
-    if (OpenConn != NULL) {
-	fclose(OpenConn);
-	OpenConn = NULL;
-    }
-    netDown(dev);
-    variable_unset(VAR_FTP_PATH);
-    return FALSE;
-}
-
-FILE *
-mediaGetFTP(Device *dev, char *file, Boolean probe)
-{
-    int nretries = 1;
-    FILE *fp;
-    char *try, buf[PATH_MAX];
-
-    if (!OpenConn) {
-	msgDebug("No FTP connection open, can't get file %s\n", file);
-	return NULL;
-    }
-
-    try = file;
-    while ((fp = ftpGet(OpenConn, try, 0)) == NULL) {
-	int ftperr = ftpErrno(OpenConn);
-
-	/* If a hard fail, try to "bounce" the ftp server to clear it */
-	if (ftperr != 550) {
-	    if (ftperr != 421)	/* Timeout? */
-		variable_unset(VAR_FTP_PATH);
-	    /* If we can't re-initialize, just forget it */
-	    DEVICE_SHUTDOWN(dev);
-	    if (!DEVICE_INIT(dev)) {
-		netDown(dev);
-		if (OpenConn) {
-		    fclose(OpenConn);
-		    OpenConn = NULL;
-		}
-		variable_unset(VAR_FTP_PATH);
-		return NULL;
-	    }
-	}
-	else if (probe)
-	    return NULL;
-	else {
-	    /* Try some alternatives */
-	    switch (nretries++) {
-	    case 1:
-		sprintf(buf, "releases/%s", file);
-		try = buf;
-		break;
-
-	    case 2:
-		sprintf(buf, "%s/%s", variable_get(VAR_RELNAME), file);
-		try = buf;
-		break;
-
-	    case 3:
-		sprintf(buf, "%s/releases/%s", variable_get(VAR_RELNAME), file);
-		try = buf;
-		break;
-
-	    case 4:
-		try = file;
-		break;
-	    }
-	}
-    }
-    return fp;
-}
-
-void
-mediaShutdownFTP(Device *dev)
-{
-    if (!ftpInitted)
-	return;
-
-    if (OpenConn != NULL) {
-	fclose(OpenConn);
-	OpenConn = NULL;
-    }
-    ftpInitted = FALSE;
-}
diff --git a/usr.sbin/sysinstall/globals.c b/usr.sbin/sysinstall/globals.c
deleted file mode 100644
index 6c98b069454c..000000000000
--- a/usr.sbin/sysinstall/globals.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-
-/*
- * Various global variables and an initialization hook to set them to
- * whatever values we feel are appropriate.
- */
-
-int		DebugFD;	/* Where diagnostic output goes */
-Boolean		Fake;		/* Only pretend to be useful */
-Boolean		RunningAsInit;	/* Are we running as init? */
-Boolean		DialogActive;	/* Is libdialog initialized? */
-Boolean		ColorDisplay;	/* Are we on a color display? */
-Boolean		OnVTY;		/* Are we on a VTY? */
-Boolean		Restarting;	/* Are we restarting sysinstall? */
-Boolean		have_volumes;	/* Media has more than one volume. */
-Variable	*VarHead;	/* The head of the variable chain */
-Device		*mediaDevice;	/* Where we're installing from */
-int		BootMgr;	/* Which boot manager we're using */
-int		StatusLine;	/* Where to stick our status messages */
-int		low_volume;	/* Lowest volume number */
-int		high_volume;	/* Highest volume number */
-jmp_buf		BailOut;	/* Beam me up, scotty! The natives are pissed! */
-
-Chunk		*HomeChunk;
-Chunk		*RootChunk;
-Chunk		*SwapChunk;
-Chunk		*TmpChunk;
-Chunk		*UsrChunk;
-Chunk		*VarChunk;
-#ifdef __ia64__
-Chunk		*EfiChunk;
-#endif
-
-/*
- * Yes, I know some of these are already automatically initialized as
- * globals.  I simply find it clearer to set everything explicitly.
- */
-void
-globalsInit(void)
-{
-    DebugFD = -1;
-    ColorDisplay = FALSE;
-    OnVTY = FALSE;
-    DialogActive = FALSE;
-    VarHead = NULL;
-    mediaDevice = NULL;
-
-    HomeChunk = NULL;
-    RootChunk = NULL;
-    SwapChunk = NULL;
-    TmpChunk = NULL;
-    UsrChunk = NULL;
-    VarChunk = NULL;
-#ifdef __ia64__
-    EfiChunk = NULL;
-#endif
-}
diff --git a/usr.sbin/sysinstall/help/anonftp.hlp b/usr.sbin/sysinstall/help/anonftp.hlp
deleted file mode 100644
index a85b8aa85883..000000000000
--- a/usr.sbin/sysinstall/help/anonftp.hlp
+++ /dev/null
@@ -1,21 +0,0 @@
-This screen allows you to configure the anonymous FTP user.
-
-The following configuration values are editable:
-
-UID:     The user ID you wish to assign to the anonymous FTP user.
-         All files uploaded will be owned by this ID.
-
-Group:   Which group you wish the anonymous FTP user to be in.
-
-Comment: String describing this user in /etc/passwd
-
-
-FTP Root Directory:
-
-        Where files available for anonymous FTP will be kept.
-
-Upload subdirectory:
-
-        Where files uploaded by anonymous FTP users will go.
-        If you do not wish to allow anonymous uploads, then this
-        field should be blank - note that this is not the default!
diff --git a/usr.sbin/sysinstall/help/configure.hlp b/usr.sbin/sysinstall/help/configure.hlp
deleted file mode 100644
index 65be877a3252..000000000000
--- a/usr.sbin/sysinstall/help/configure.hlp
+++ /dev/null
@@ -1,10 +0,0 @@
-This menu allows you to configure your system after the installation
-process is complete.  At the minimum, you should probably set the
-system manager's password and the system time zone.
-
-For extra goodies like bash, emacs, Pascal, etc., you should look at
-the Packages item in this menu.
-
-For setting the timezone after the system is installed, type
-``tzsetup''.  For more information on the overall general system
-configuration, see the ``/etc/defaults/rc.conf'' file.
diff --git a/usr.sbin/sysinstall/help/distributions.hlp b/usr.sbin/sysinstall/help/distributions.hlp
deleted file mode 100644
index d6fefa665b75..000000000000
--- a/usr.sbin/sysinstall/help/distributions.hlp
+++ /dev/null
@@ -1,34 +0,0 @@
-DISTRIBUTION INFORMATION
-------------------------
-
-An ``X-'' prefixed before a distribution set means that the Xorg
-base distribution, libraries, manual pages, servers and a set
-of default fonts will be selected in addition to the set itself.
-If you select such a set, you will also be presented with a set of
-menus for customizing the selections to your desired X Window System
-setup.
-
-Any distribution may be further customized by selecting the `Custom'
-item before leaving the menu.
-
-The current "canned" installations are provided:
-
-All:            The base distribution, man pages, dictionary files,
-                profiling libraries, the FreeBSD compatibility libraries,
-                the complete source tree, games and your choice of Xorg
-                distribution components.
-
-Developer:      Base distribution, man pages, dictionary files,
-                profiling libraries and the complete source tree.
-
-Kern-Developer: As Developer, but with only kernel sources instead of
-                the complete source tree.
-
-User:           The base distribution, man pages and dictionary files.
-
-Minimal:        Only the base distribution.
-
-Custom:         Allows you to create or modify your distribution set on
-                a piece-by-piece basis.
-
-Reset:          Clear all currently selected distributions.
diff --git a/usr.sbin/sysinstall/help/drives.hlp b/usr.sbin/sysinstall/help/drives.hlp
deleted file mode 100644
index 946a1b211b7f..000000000000
--- a/usr.sbin/sysinstall/help/drives.hlp
+++ /dev/null
@@ -1,92 +0,0 @@
-Boot Manager Selection:
------------------------
-
-If you wish to switch between multiple operating systems on your
-machine, or if you are trying to install FreeBSD on a drive other than
-your 1st drive, then you must install a boot manager.  In the case
-where you wish to boot off an alternate drive, it should also be noted
-that you still need to install a boot manager on the FIRST drive!
-Even if you do not intend to create a FreeBSD partition on that drive
-(e.g. it's being wholly used by something else), the boot manager
-still needs to reside on the first disk in order to function as a
-"redirector" for the boot process.
-
-To do this, simply select your 1st drive in the drive selection menu
-and when the partition editor comes up, don't make any changes - just
-(Q)uit.  At the boot manager menu which follows, select the first
-option (install a boot manager) and then proceed to setup the other
-drive(s) for FreeBSD as normal.
-
-It should also be noted that "operating systems" such as Windows 95
-will completely overwrite your boot manager without so much as a
-polite "may I please destroy your boot manager?" prompt if you make
-the mistake of installing them second.  If this happens to you after
-FreeBSD is already installed, all is not lost!  Simply revisit your
-FreeBSD distribution directory and look for a tools/ subdirectory, in
-which you'll find "bootinst.exe" and "boot.bin".  To reinstall, simply
-say "bootinst boot.bin" while in the tools/ subdirectory.
-
-
-If you see the boot manager displaying ``F?'' when you try to come up
-for the first time and it refuses to change, no matter how often you
-whap on the function key assigned to FreeBSD, then you have a geometry
-mismatch problem and you should read the next section for important
-information on how to prevent that exact problem from happening!
-
-
-Geometry Translation / Sharing the disk(s) with another OS:
-----------------------------------------------------------
-
-If you are going to actually install some portion of FreeBSD on a
-drive then PLEASE BE VERY CERTAIN that the Geometry reported in the
-Partition Editor is the correct one for your drive and controller
-combination!
-
-IDE drives often have a certain geometry set during the PC BIOS setup,
-or (in the case of larger IDE drives) have their geometry "translated"
-by either the IDE controller or a special boot-sector translation
-utility such as that by OnTrack Systems.  In these cases, knowing the
-correct geometry gets even more complicated as it's not something you
-can easily tell by looking at the drive or the PC BIOS setup.  The
-best way of verifying that your geometry is being correctly calculated
-in such situations is to boot DOS (from the hard disk, not a floppy!)
-and run the ``pfdisk'' utility provided in the tools/ subdirectory of
-the FreeBSD CDROM or FTP site.  It will report the geometry that DOS
-sees, which is generally the correct one.
-
-If you have no DOS partition sharing the disk at all, then you may
-find that you have better luck with Geometry detection if you create a
-very small DOS partition first, before installing FreeBSD.  Once
-FreeBSD is installed you can always delete it again if you need the
-space.
-
-It's actually not a bad idea (believe it or not) to have a small
-bootable DOS partition on your FreeBSD machine anyway: Should the
-machine become unstable or exhibit strange behavior at some point in
-the future (which is not uncommon behavior for PC hardware!) you can
-then at least use DOS for installing and running one of the
-commercially available system diagnostic utilities.
-
-IMPORTANT NOTE:
-
-Any root partition you try to boot from must also reside below the
-1024th cylinder.  If you're using a translated geometry then this is
-probably not a problem, but if you are using a native disk geometry
-which exceeds 1024 cylinders then you could have a failure to boot if
-you end up installing a root partition (or even just the kernel file
-in a root partition) out past cylinder 1024.  If you are trying to
-share your first disk with FreeBSD and another OS which was installed
-previously, you are particularly susceptible to this problem and should
-check your disk addresses very carefully.
-
-If you find that you have insufficient space below cylinder 1024 to
-make a root partition for FreeBSD (and again, this ONLY applies to the
-root partition - once FreeBSD's kernel is loaded, it doesn't care
-about the geometry issues) then you will probably need to install on a
-completely different disk (see the boot manager section above) or
-resize your existing partitions so that both operating systems can
-have boot partitions below cylinder 1024.
-
-You may blame IBM for the limitations of a 10 bit cylinder address.
-"No one will have a disk with more than 1024 cylinders." I'm sure
-someone said.
diff --git a/usr.sbin/sysinstall/help/fixit.hlp b/usr.sbin/sysinstall/help/fixit.hlp
deleted file mode 100644
index 801f2589a5be..000000000000
--- a/usr.sbin/sysinstall/help/fixit.hlp
+++ /dev/null
@@ -1,3 +0,0 @@
-A special shell will be launched by this option with a fixit floppy
-or a FreeBSD CD-ROM (or DVD-ROM) mounted as /mnt2.  This provides
-access to extra commands under /mnt2.
diff --git a/usr.sbin/sysinstall/help/html.hlp b/usr.sbin/sysinstall/help/html.hlp
deleted file mode 100644
index c12f82c07d3e..000000000000
--- a/usr.sbin/sysinstall/help/html.hlp
+++ /dev/null
@@ -1,20 +0,0 @@
-In this screen, you can jump to remote or local HTML
-resources such as the FreeBSD Handbook & FreeBSD FAQ
-(Frequently Asked Questions) documents located in:
-
-     file:/usr/share/doc/
-
-if you've loaded the doc distribution.
-
-The default browser package used is links (a text based
-browser which can render tables), which will be
-automatically loaded from the installation media if it
-is available.  You may change the selection of browser
-& browser package to auto-load by visiting the Options
-editor.
-
-In order to visit remote URLs, you naturally must have 
-some sort of working Internet connection.  If you have not
-yet brought up any network interfaces, please visit
-the ``Networking'' item in the Configuration menu
-before attempting to reference any http://.. style URLs.
diff --git a/usr.sbin/sysinstall/help/media.hlp b/usr.sbin/sysinstall/help/media.hlp
deleted file mode 100644
index 414a2aa208b4..000000000000
--- a/usr.sbin/sysinstall/help/media.hlp
+++ /dev/null
@@ -1,54 +0,0 @@
-You can install from the following types of media:
-
-   CDROM    requires one of the following supported CDROM drives:
-            ATAPI - Any standard ATAPI CDROM drive hooked to
-	    a supported controller (see Hardware Guide).
-            SCSI - Any standard SCSI CDROM drive hooked to
-            a supported controller (see Hardware Guide).
-
-
-   DOS      A DOS primary partition with the required FreeBSD
-            distribution files copied onto it (e.g. C:\FREEBSD\)
-
-
-   FS       Assuming a disk or partition with an existing
-            FreeBSD file system and distribution set on it,
-            get the distribution files from there.
-
-
-   Floppy   Get distribution files from one or more DOS or UFS
-            formatted floppies.  Such floppies are assumed to
-            contain the appropriate distribution pieces - see
-            ABOUT.TXT for more information about making floppy
-            distribution media.
-
-
-   FTP      Get the distribution files from an anonymous ftp server
-            (you will be presented with a list).  Please note that
-            you may invoke FTP in "Active" mode, "Passive" mode, or
-            via an HTTP proxy.
-
-            Active mode is the standard way of fetching files and
-            Passive mode is for use when you're behind a firewall or
-            some other security mechanism that blocks active FTP
-            connections.  Using an HTTP proxy is sometimes necessary
-            for firewalls which block all FTP connections.
-
-            If you chose to enter your own URL in the FTP menu, please
-            note that all paths are *relative* to the home directory
-            of the user being logged in as.  By default, this is the
-            user "ftp" (anonymous ftp) but you may change this in the
-            Options screen.
-
-
-   NFS      Get the distribution files from an NFS server somewhere
-            (make sure that permissions on the server allow this!).
-            If this install method hangs on you or refuses to work
-            properly, you may need to set some special options for
-            your NFS server.  See the Options screen for more details.
-
-
-   Tape     Extract distribution files from tape into a temporary
-            directory and install from there.  If the tape was created
-            with blocksize other than 20, you may wish to change this
-            in the Options screen.
diff --git a/usr.sbin/sysinstall/help/network_device.hlp b/usr.sbin/sysinstall/help/network_device.hlp
deleted file mode 100644
index affa86a91ec8..000000000000
--- a/usr.sbin/sysinstall/help/network_device.hlp
+++ /dev/null
@@ -1,58 +0,0 @@
-You can do network installations over 3 types of communications links:
-
-        Serial port:    SLIP / PPP
-        Parallel port:  PLIP (laplink cable)
-        Ethernet:       A standard Ethernet controller (includes some
-                        PCMCIA networking cards).
-
-SLIP support is rather primitive and limited primarily to directly
-connected links, such as a serial cable running between a laptop
-computer and another PC.  The link must be hard-wired as the SLIP
-installation doesn't currently offer a dialing capability (that
-facility is offered by the PPP utility, which should be used in
-preference to SLIP whenever possible).  When you choose the SLIP
-option, you'll be given the option of later editing the slattach
-command before it's run on the serial line.  It is expected that
-you'll run slattach (or some equivalent command) on the other end of
-the link at that time and bring up the line.  FreeBSD will then
-install itself at serial speeds of up to 115.2K/baud (the recommended
-speed for a hardwired cable).
-
-If you're using a modem then PPP is almost certainly your only choice.
-Make sure that you have your service provider's information handy as
-you'll need to know it fairly early in the installation process.  You
-will need to know your service provider's IP address, the IP address 
-of your provider's DNS server, and possibly your own IP address unless
-your ISP supports dynamic negotiation, most do.  If you do not choose
-a PAP or CHAP login you will also need to know how to use the various
-"AT commands" to dial the ISP with your particular brand of modem as
-the PPP dialer provides only a very simple terminal emulator and has no
-"modem capabilities database".  If you choose a PAP or CHAP login you
-can simply enter `dial' (without the quotes) at the ppp prompt if your
-modem uses the Hayes compatible AT command set.
-
-If a hard-wired connection to another FreeBSD (2.0R or later) machine
-is available, you might also consider installing over a "laplink"
-parallel port cable.  The data rate over the parallel port is much
-higher than what is typically possible over a serial line, and speeds
-of over 50KB/sec are not uncommon.
-
-Finally, for the fastest possible network installation, an Ethernet
-adaptor is always a good choice!  FreeBSD supports most common PC
-Ethernet cards, a table of which is provided in the FreeBSD Hardware
-Guide (see the `Documentation' entry in the main menu).  If you are
-using one of the supported PCMCIA Ethernet cards, also be sure that
-it's plugged in BEFORE the laptop is powered on!  Sysinstall does not,
-unfortunately, currently support "hot insertion" of PCMCIA cards.
-
-You will also need to know your IP address on the network, the
-"netmask" value for your address class, and the name of your machine.
-Your system administrator can tell you which values to use for your
-particular network setup.  If you will be referring to other hosts by
-name rather than IP address, you'll also need a name server and
-possibly the address of a gateway (if you're using PPP, it's your
-provider's IP address) to use in talking to it.  If you do not know
-the answers to all or most of these questions then you should really
-probably talk to your system administrator FIRST before trying this
-type of installation!  Choosing the wrong IP address on a busy network
-will NOT make you popular with your systems administrator!  :-)
diff --git a/usr.sbin/sysinstall/help/options.hlp b/usr.sbin/sysinstall/help/options.hlp
deleted file mode 100644
index e1371ff1e89b..000000000000
--- a/usr.sbin/sysinstall/help/options.hlp
+++ /dev/null
@@ -1,181 +0,0 @@
-The following options may be set from this screen.  Use the SPACE key
-to toggle an option's value, Q to leave when you're done.
-
-NFS Secure:   NFS server talks only on a secure port
-
-    This is most commonly used when talking to Sun workstations, which
-    will not talk NFS over "non privileged" ports.
-
-
-NFS Slow:     User is using a slow PC or Ethernet card
-
-    Use this option if you have a slow PC (386) or an Ethernet card
-    with poor performance being "fed" by NFS on a higher-performance
-    workstation.  This will throttle the workstation back to prevent
-    the PC from becoming swamped with data.
-
-
-NFS TCP:      Use TCP for the NFS mount
-
-    This option can be used if your NFS server supports TCP
-    connections; not all do!  This may be useful if your NFS server
-    is at a remote site in which case it may offer some additional
-    stability.
-
-
-NFS version 3:   Use NFS version 3
-
-    This option forces the use of NFS version 3 and is on by default.
-    If your NFS server only supports NFS version 2, disable this option.
-
-
-Debugging:    Turn on the extra debugging flag
-
-    This turns on a lot of extra noise over on the second screen
-    (ALT-F2 to see it, ALT-F1 to switch back).  If your installation
-    should fail for any reason, PLEASE turn this flag on when
-    attempting to reproduce the problem.  It will provide a lot of
-    extra debugging at the failure point and may be very helpful to
-    the developers in tracking such problems down!
-
-
-No Warnings:  Disable some warnings
-
-    This flag tells sysinstall, and particularly the disk editing
-    routines, that you consider yourself to know what you are
-    doing and disables various warning.  It is not recommended that
-    you enable this option.
-
-
-Yes To All:   Assume "Yes" answers to all non-critical dialogs
-
-    This flag should be used with caution.  It will essentially
-    decide NOT to ask the user about any "boundary" conditions that
-    might not constitute actual errors but may be warnings indicative
-    of other problems.  It's most useful to those who are doing unattended
-    installs.
-
-
-DHCP:         Enable DHCP configuration of interfaces
-
-    This option specifies whether DHCP configuration of interfaces
-    may be attempted.  The default setting is to interactively ask
-    the user.
-
-
-IPv6:         Enable IPv6 router solicitation configuration
-
-    This option specifies whether automatic configuration of IPv6
-    interfaces may be attempted.  This uses the router solicitation
-    method of automatic configuration.  The default setting is to
-    interactively ask the user.
-
-
-FTP username:  Specify username and password instead of anonymous.
-
-    By default, the installation attempts to log in as the
-    anonymous user.  If you wish to log in as someone else,
-    specify the username and password with this option.
-
-
-Editor:  Specify which screen editor to use.
-
-   At various points during the installation it may be necessary
-   to customize some text file, at which point the user will be
-   thrown unceremoniously into a screen editor.  A relatively
-   simplistic editor which shows its command set on-screen is
-   selected by default, but UNIX purists may wish to change this
-   setting to `/usr/bin/vi'.
-
-
-Tape Blocksize:  Specify block size in 512 byte blocks of tape.
-
-    This defaults to 20 blocks, which should work with most
-    tape drive + tar combinations.  It may not allow your particular
-    drive to win any records for speed, however, and the more
-    adventurous among you might try experimenting with larger sizes.
-
-
-Extract Detail:  How to show filenames on debug screen as they're extracted.
-
-    While a distribution is being extracted, the default detail level
-    of "high" will show the full file names as they're extracted.
-    If you would prefer a more terse form for this, namely dots, select
-    the "medium" detail level.  If you want nothing to be printed
-    on the debugging screen during extraction, select "low".
-
-
-Release Name:  Which release to attempt to load from installation media.
-
-    You should only change this option if you're really sure you know
-    what you are doing!  This will change the release name used by
-    sysinstall when fetching components of any distributions, and
-    is a useful way of using a more recent installation boot floppy
-    with an older release (say, on CDROM).
-
-
-Install Root:  Specify some directory other than / as your "root".
-
-    This should be left as / unless you have a really good reason to
-    change it.  One good reason might be if you were installing to a
-    disk other than your own, as might happen if you needed to prepare a
-    disk for another machine which couldn't load FreeBSD directly
-    for some reason.
-
-    Note:  If you set this option, you will only be able to install
-    packages if the base distribution is also installed (usually
-    the case anyway) since /usr/sbin/pkg_add will otherwise not be
-    found after the chroot() call.
-
-
-Browser Package:  Which package to load for an HTML browser.
-
-    By default, this is set to links but may also be set to any other
-    text capable HTML browser for which a package exists.  If you set this
-    to an X based browser, you will not be able to use it if you're running
-    in text mode! :)
-
-
-Browser Exec:  Which binary to run for the HTML browser.
-
-    The full pathname to the main executable in Browser Package.
-
-
-Media Type:   Which media type is being used.
-
-    This is mostly informational and indicates which media type (if any)
-    was last selected in the Media menu.  It's also a convenient short-cut
-    to the media menu itself.
-
-
-Package Temp:  Where package temporary files should go
-
-   Some packages, like emacs, can use a LOT of temporary space - up to
-   20 or 30MB.  If you are going to configure a small / directory and no
-   separate /var (and hence a small /var/tmp), then you may wish to set
-   this to point at another location (say, /usr/tmp).
-
-
-Newfs Args:    Specify default arguments to newfs(8)
-
-   The default parameters used to build new filesystems.
-   If you will be running a service that creates millions of small
-   files or need to specify different default parameters for any
-   other reason, you may do so here.
-
-
-Fixit Console: The location of the fixit console
-
-    Specifies where sysinstall should start the fixit shell for
-    interactive repair.  Valid arguments are "serial" for a serial
-    port, or "standard" for VTY4.
-
-
-Re-scan Devices:  
-
-   Reprobe the system for devices.
-
-
-Use Defaults:  Use default values.
-
-    Reset all options back to their default values.
diff --git a/usr.sbin/sysinstall/help/partition.hlp b/usr.sbin/sysinstall/help/partition.hlp
deleted file mode 100644
index 3d13b34afad8..000000000000
--- a/usr.sbin/sysinstall/help/partition.hlp
+++ /dev/null
@@ -1,169 +0,0 @@
-This is the FreeBSD DiskLabel Editor.
-
-NOTE: If you're entering this editor from the update procedure then
-you probably shouldn't (C)reate anything at all but rather use only
-the (M)ount command to check and mount existing partitions for
-upgrading.
-
-If you would like the label editor to do most of the following for
-you, simply type `A' for automatic partitioning of the disk.
-
-If you wish to create partitions manually you may do so by moving the
-highlighted selection bar with the arrow keys over the FreeBSD
-partition(s) displayed at the top of the screen.  Typing (C)reate
-while a partition with available free space is selected will allow you
-to create a BSD partition inside of it using some or all of its
-available space.
-
-Typing (M)ount over an existing partition entry (displayed in the
-middle of the screen) will allow you to set a mount point for it
-without initializing it.  If you want it initialized, use the (T)oggle
-command to flip the Newfs flag.  When Newfs is set to "Y", the
-filesystem in question will be ERASED and rebuilt from scratch!
-
-
-You should use this editor to create at least the following
-filesystems:
-
-        Name    Purpose                 Min Size?       Optional?
-        ----    -------                 ---------       ---------
-        /       Root filesystem         118MB           No
-        swap    Swap space              2 * MEM         No
-        /usr    System & user files     128MB or more   Yes
-
-Note: If you do not create a /usr filesystem then your / filesystem
-will need to be bigger - at least 240MB.  This is not recommended as
-any media errors that may occur during disk I/O to user files will
-corrupt the filesystem containing vital system files as well.  It is
-for this reason that / is generally kept on its own filesystem, where
-it should be considered essentially "read only" in your administration
-of it.
-
-Swap space is a little tricker, and the rule of "2 * MEM" is simply a
-best-guess approximation and not necessarily accurate for your
-intended usage of the system.  If you intend to use the system heavily
-in a server or multi-user application, you may be well advised to
-increase this size.  You may also create swap space on multiple drives
-for a larger "total" swap and this is, in fact, recommended if you
-have multiple, fast drives for which such load-balancing can only help
-overall I/O performance.
-
-The /usr filesystem should be sized according to what kind of
-distributions you're trying to load and how many packages you intend
-to install in locations like /usr/local.  You can also make /usr/local
-a separate filesystem if you don't want to risk filling up your /usr
-by mistake.
-
-Another useful filesystem to create is /var, which contains mail, news
-printer spool files and other temporary items.  It is a popular
-candidate for a separate partition and should be sized according to
-your estimates of the amount of mail, news or spooled print jobs that
-may be stored there.
-
-WARNING: If you do not create a separate filesystem for /var, space
-for such files will be allocated out of the root (/) filesystem
-instead.  You may therefore wish to make the / partition bigger if you
-expect a lot of mail or news and do not want to make /var its own
-partition.
-
-If you're new to this installation, you might also want to read the
-following explanation of how FreeBSD's new "slice" paradigm for
-looking at disk storage works:
-
-
-In FreeBSD's new system, a device name can be broken up into up to 3
-parts.  Take a typical name like ``/dev/da0s1a'':
-
-      The first three characters represent the drive name.  If we had
-      a system with two SCSI drives on it then we'd see /dev/da0 and
-      /dev/da1 as the device entries representing the entire drives.
-
-      Next you have the "slice" (or "FDISK Partition") number,
-      as seen in the Partition Editor.  Assuming that our da0
-      contained two slices, a FreeBSD slice and a DOS slice, that
-      would give us /dev/da0s1 and /dev/da0s2 as device entries pointing
-      to the entire slices.
-
-      Next, if a slice is a FreeBSD slice, you can have a number of
-      (confusingly named) "partitions" inside of it.
-
-      These partitions are where various filesystems or swap areas live,
-      and using our hypothetical two-SCSI-disk machine again, we might
-      have something like the following layout on da0:
-
-              Name            Mountpoint
-              ----            ----------
-              da0s1a          /
-              da0s1b          <swap space>
-              da0s1e          /usr
-
-Once you understand all this, then the purpose of the label editor
-becomes fairly clear:  You're carving up the FreeBSD slices displayed
-at the top of the screen into smaller pieces, which are displayed in
-the middle of the screen, and then assigning FreeBSD file system names
-(mount points) to them.
-
-You can also use the label editor to mount existing partitions/slices
-into your filesystem hierarchy, as is frequently done for DOS FAT
-slices.  For FreeBSD partitions, you can also toggle the "newfs" state
-so that the partitions are either (re)created from scratch or simply
-checked and mounted (the contents are preserved).
-
-If you set (S)oftUpdates on a filesystem, it will cause the
-"Soft Updates" policy to be in effect for it.  This basically causes
-both metadata and data blocks to be written asynchronously to disk,
-but with extra state information which causes the metadata and any
-related data blocks to be committed in a single transaction.  This
-results in async metadata update speeds (which are considerably
-faster than the default sync) without the potential for data loss
-which could occur if you simply mounted the filesystem with purely
-"async" update policy and then had a power failure.  If you wish
-to later turn the softupdates policy back off, use the command
-"tunefs -n disable devicename".  NOTE:  It is probably not wise
-to use this on your root filesystem unless you have a large
-(e.g. non-standard size) root.  The reason is that smaller filesystems
-with significant activity can temporarily overflow if the soft updates
-policy results in free'd blocks not being "garbage collected" as fast
-as they're being requested.
-
-The UNIX File System (UFS) on FreeBSD supports two different on-disk
-layouts: UFS1 and UFS2.  UFS1 was the default file system in use
-through FreeBSD 5.0-RELEASE; as of FreeBSD 5.1-RELEASE, the default
-is now UFS2, with the exception of the PC98 platform.  UFS2 provides
-sparse inode allocation (faster fsck), 64-bit storage pointers (larger
-maximum size), and native extended attributes (required for ACLs, MAC,
-and other advanced security and file system services).  The selection
-of UFS1 or UFS2 must be made when the file system is created--later
-conversion is not currently possible.  UFS2 is the recommended file
-system, but if disks are to be used on older FreeBSD systems, UFS1
-improves portability.  When dual-booting between FreeBSD 4.x or
-earlier and FreeBSD 5.x, UFS1 file systems will be accessible from
-both.  To toggle a file system to UFS1, press '1'.  To restore it to
-UFS2, press '2'.
-
-WARNING: FreeBSD on i386 is currently unable to boot from root file
-systems larger than 1.5TB.
-
-To add additional flags to the newfs command line for UFS file
-systems, press 'N'.  These options will be specified before the
-device argument of the command line, but after any other options
-placed there by sysinstall, such as the UFS version and soft
-updates flag; as such, arguments provided may override existing
-settings.  To completely replace the newfs command used by
-sysinstall, press 'Z' to convert a partition to a Custom
-partition type.  Sysinstall will prompt you with the newfs
-command line that it would have used based on existing settings
-prior to the change, but allow you to modify any aspect of the
-command line.  Once a partition has been converted to a custom
-partition in the label editor, you will need to restart the
-labeling process or delete and recreate the partition to restore
-it to a non-custom state.  Custom partitions are represented by
-the letters "CST" instead of "UFS" or "FAT.
-
-When you're done, type `Q' to exit.
-
-No actual changes will be made to the disk until you (C)ommit from the
-Install menu or (W)rite directly from this one.  You're working with
-what is essentially a copy of the disk label(s), both here and in the
-FDISK Partition Editor, and the actual on-disk labels won't be
-affected by any changes you make until you explicitly say so.
diff --git a/usr.sbin/sysinstall/help/securelevel.hlp b/usr.sbin/sysinstall/help/securelevel.hlp
deleted file mode 100644
index 27eb1ec231b9..000000000000
--- a/usr.sbin/sysinstall/help/securelevel.hlp
+++ /dev/null
@@ -1,40 +0,0 @@
-This menu allows you to configure the Securelevel mechanism in FreeBSD.
-
-Securelevels may be used to limit the privileges assigned to the
-root user in multi-user mode, which in turn may limit the effects of
-a root compromise, at the cost of reducing administrative functions.
-Refer to the security(7) and init(8) manual pages for complete details.
-
-   -1    Permanently insecure mode - always run the system in level 0
-         mode.  This is the default initial value.
-
-   0     Insecure mode - immutable and append-only flags may be turned
-         off.  All devices may be read or written subject to their
-         permissions.
-
-   1     Secure mode - the system immutable and system append-only
-         flags may not be turned off; disks for mounted file systems,
-         /dev/mem, /dev/kmem and /dev/io (if your platform has it)
-         may not be opened for writing; kernel modules (see kld(4))
-         may not be loaded or unloaded.
-
-   2     Highly secure mode - same as secure mode, plus disks may not
-         be opened for writing (except by mount(2)) whether mounted or
-         not.  This level precludes tampering with file systems by
-         unmounting them, but also inhibits running newfs(8) while the
-         system is multi-user.
-
-         In addition, kernel time changes are restricted to less than
-         or equal to one second.  Attempts to change the time by more
-         than this will log the message ``Time adjustment clamped to +1
-         second''.
-
-   3     Network secure mode - same as highly secure mode, plus IP
-         packet filter rules (see ipfw(8), ipfirewall(4) and pfctl(8))
-         cannot be changed and dummynet(4) or pf(4) configuration
-         cannot be adjusted.
-
-Securelevels must be used in combination with careful system design and
-application of protective mechanisms to prevent system configuration
-files from being modified in a way that compromises the protections of
-the securelevel variable upon reboot.
diff --git a/usr.sbin/sysinstall/help/shortcuts.hlp b/usr.sbin/sysinstall/help/shortcuts.hlp
deleted file mode 100644
index 9ec03d0e1ef5..000000000000
--- a/usr.sbin/sysinstall/help/shortcuts.hlp
+++ /dev/null
@@ -1,117 +0,0 @@
-sysinstall now supports command-line "shortcuts" which can
-often replace outdated commands, like pkg_manage.  Multiple commands
-can be invoked in sequence, and variables may be set on-the-fly to
-customize the installation program's behavior in various ways.
-
-Syntax:
-
-/usr/sbin/sysinstall [var=value ..] [command ..]
-
-Where "var" can be one or more of:
-
-blanktime               Screen blank time setting in seconds
-bootManager             Select boot manager: booteasy, standard or none
-browserBinary           Which doc browser to use (default: links)
-browserPackage          Which package to get browser from (default: links)
-cpioVerbose             How verbose to be with cpio: high or low
-debug                   Extra debugging?
-defaultrouter           IP address of default route
-disk                    Which disk to operate on (ad0, da0, etc).
-domainname              Domain name
-editor                  Which screen editor to use
-ftp                     Which FTP site/dir to use (URL ftp://site/dir/..)
-ftpDirectory            Root of the FreeBSD distribution tree on FTP server
-ftpHost                 Which FTP hostname to use (overrides ftp variable)
-ftpOnError              Set to retry or abort
-ftpPass                 Which password to use when logging into FTP server
-ftpPort                 Which FTP port to use (default: 21)
-ftpRetryCount           How many times to retry a fetch operation
-ftpUser                 Which username to use when logging into FTP server
-gated                   Use gated instead of routed
-geometry                Geometry to use for selected disk ("cyl/hd/sec")
-hostname                Fully qualified domain name for host.
-ifconfig_<iface>        For each <iface> in network_devices
-ipaddr                  IP address for this host's primary interface
-nameserver              IP address of name server
-netmask                 Netmask for this host's primary interface
-network_interfaces      Which network interfaces to configure
-nfs                     Full host:/path/ specification to NFS media
-nfsHost                 Host portion of nfs path
-nfsSecure               Use NFS secure mount (-P flag)
-nfs_server              Configure this machine as an NFS server
-noConfirm               Don't ask for confirmation on non-fatal errors
-ntpDate                 Which ntp clock synchronization server to use
-pcnfsd                  Install the PCNFSD package 
-ports                   Path to the ports collection
-releaseName             Which FreeBSD release to install
-rootSize                Size of the root partition to create for Auto
-routedflags             Which flags to pass to routed, if enabled
-serialSpeed             How fast to run a SLIP/PPP connection
-slowEthernetCard        PC ethernet card is uncommonly slow
-swapSize                Size of the swap partition to create for Auto
-tapeBlocksize           Tape size in blocks
-ufs                     Full path to UFS media directory
-usrSize                 Size of the /usr partition to create for Auto
-varSize                 Size of the /var partition to create for Auto
-
-And "command" can be one or more of:
-
-addGroup                Add a new group to the system
-addUser                 Add a new user to the system
-configAnonFTP           Configure system for anonymous FTP
-configInetd             Configure the inetd super-server
-configNFSServer         Configure host as an NFS server
-configNTP               Configure host as an NTP client
-configPackages          Browse / install packages
-configRouter            Configure a routing daemon
-configUsers             Add users and/or groups to the system
-diskLabelCommit         Write out any changed label information
-diskLabelEditor         Label/Newfs/Mount new or existing filesystems
-diskPartitionEditor     Partition a new or existing disk
-diskPartitionWrite      Write out any changed partition information
-distExtractAll          Extract all selected distributions
-distReset               Reset distribution information
-distSetDeveloper        Select developer distribution
-distSetEverything       Select all distributions
-distSetKernDeveloper    Select kernel developer distribution
-distSetMinimum          Select minimal distribution
-distSetSrc              Select source sub-distributions
-distSetUser             Select user distribution
-distSetXDeveloper       Select Xorg developer distribution
-distSetXOrg             Select Xorg sub-distributions
-distSetXUser            Select Xorg user distribution
-docBrowser              Browse documentation
-installCommit           Commit any pending installation operations
-installExpress          Express installation
-installStandard         Standard installation
-installUpgrade          Upgrade installation
-mediaGetType            Prompt for media type
-mediaSetCDROM           Select CDROM media
-mediaSetCPIOVerbosity   Prompt for CPIO verbosity
-mediaSetDOS             Select DOS media
-mediaSetFTP             Select FTP media
-mediaSetFTPPassive      Select FTP media in passive mode
-mediaSetFTPUserPass     Prompt for FTP username and password
-mediaSetFloppy          Select floppy media
-mediaSetHTTP            Select FTP media via HTTP proxy
-mediaSetNFS             Select NFS media
-mediaSetTape            Select tape media
-mediaSetUFS             Select UFS media
-optionsEditor           Go to options editor
-tcpMenuSelect           Configure TCP/IP networking
-
-Examples:
-
-/usr/sbin/sysinstall       mediaSetFTP configPackages
-
-Selects an FTP site and then goes to the package configuration menu.
-
-
-/usr/sbin/sysinstall       disk=da0 diskPartitionEditor
-
-Invokes the disk partition editor on disk da0.
-
-
-If /usr/sbin/sysinstall is linked to another filename, say
-`/usr/local/bin/configPackages', then the basename will be used
-as an implicit command name.
diff --git a/usr.sbin/sysinstall/help/slice.hlp b/usr.sbin/sysinstall/help/slice.hlp
deleted file mode 100644
index e9f3abb6a43b..000000000000
--- a/usr.sbin/sysinstall/help/slice.hlp
+++ /dev/null
@@ -1,57 +0,0 @@
-This is the Main Slice (``FDISK'' or PC-style Partition) Editor.
-
-Possible commands are printed at the bottom and the Master Boot Record
-contents are shown at the top.  You can move up and down with the
-arrow keys and (C)reate a new slice whenever the highlighted
-selection bar is over a slice whose type is marked as "unused."
-
-You are expected to leave this screen with at least one slice
-marked "FreeBSD."  Note that unlike Linux, you don't need to create
-multiple FreeBSD FDISK partition entries for different things like
-swap, file systems, etc.  The usual convention is to create ONE
-FreeBSD slice (FDISK partition) per drive and then subsection this slice
-into swap and file systems with the Label editor.
-
-No actual changes will be made to the disk until you (C)ommit from the
-Install menu or use the (W)rite option here!  You're working with what
-is essentially a copy of the disk label(s), both here and in the Label
-Editor.
-
-If you want to use the entire disk for FreeBSD, type `A'.  Slices will
-be aligned to fictitious cylinder boundaries and space will be reserved
-in front of the FreeBSD slice for a [future] possible boot manager.
-
-For the truly dedicated disk case, type `F'.  You'll be asked whether or
-not you wish to keep the disk (potentially) compatible with other
-operating systems, i.e. the information in the FDISK table should be
-kept valid.  A truly dedicated disk can be achieved by selecting `No'.
-In that case, all BIOS geometry considerations will no longer be in
-effect and you can safely ignore any ``The detected geometry is
-invalid'' warning messages you may later see.  It is also not necessary
-in this case to set a slice bootable or install an MBR boot manager as
-both things are then irrelevant.  The FreeBSD slice will start at
-absolute sector 0 of the disk (so that FreeBSD's disk label is identical
-to the Master Boot Record) and extend to the very last sector of the
-disk medium.  Needless to say, such a disk cannot have any sort of a
-boot manager, `disk manager', or anything else that has to interact with
-the BIOS.  This option is therefore only considered safe for SCSI disks
-and most IDE disks and is primarily intended for people who are going to
-set up a dedicated FreeBSD server or workstation, not a typical `home PC'.
-
-If you select the default of `Yes' at the compatibility, slices will be
-aligned to fictitious cylinder boundaries and space will be reserved
-in front of the FreeBSD slice for a [future] possible boot manager.
-This is pretty much equivalent to having chosen `A' originally.
-
-The flags field has the following legend:
-
-        '='  -- This slice is properly aligned.
-        'A'  -- This slice is marked active.
-        'R'  -- This slice contains the root (/) filesystem
-
-If no slice is marked Active, you will need to either install
-a Boot Manager (the option for which will be presented later in the
-installation) or set one Active before leaving this screen.
-
-To leave the slice editor, type `Q'.
-
diff --git a/usr.sbin/sysinstall/help/tcp.hlp b/usr.sbin/sysinstall/help/tcp.hlp
deleted file mode 100644
index 4c86198e8b5f..000000000000
--- a/usr.sbin/sysinstall/help/tcp.hlp
+++ /dev/null
@@ -1,42 +0,0 @@
-This screen allows you to set up your general network parameters
-(hostname, domain name, DNS server, etc) as well as the settings for a
-given interface (which was selected from the menu before this screen).
-
-PLIP/SLIP users - please read through to the end of this doc!
-
-You can move through the fields with the TAB, BACK-TAB and ENTER
-keys.  To edit a field, use DELETE or BACKSPACE.  You may also use ^A
-(control-A) to go to the beginning of the line, ^E (control-E) to go
-to the end, ^F (control-F) to go forward a character, ^B (control-B)
-to go backward one character, ^D (control-D) to delete the character
-under the cursor and ^K (control-K) to delete to the end of the line.
-Basically, the standard EMACS motion sequences.
-
-The "Extra options to ifconfig" field is kind of special (read: a
-hack :-):
-
-Any valid options to ifconfig can be specified here, so if you need
-to do something "special" to get your interface working, then here
-is the place to do it.
-
-If you're running SLIP or PLIP, you also need to use it for specifying
-the remote end of the link (simply type the foreign IP address in).
-In the specific case where you're running PLIP with a Linux host peer
-rather than a FreeBSD one, you also must add the "-link0" flag after the
-foreign address.
-
-If you're dealing with an ethernet adaptor with multiple media
-connectors (e.g. AUI, 10BT, 10B2, etc), you can use this field to
-specify which one to use.  Examples of valid strings include:
-
-        "media 10base5/AUI" - Select the AUI port.
-        "media 10baseT/UTP" - Select the twisted pair port.
-        "media 10base2/BNC" - Select the BNC connector.
-        "media 100baseTX"   - Select 100BaseT on a 100/10 dual adaptor.
-
-If you have a wireless interface and must specify arguments such as a
-WEP key here, you may use something like:
-
-	"wepmode on wepkey 0xFEEDFACE"
-
-When you're done with this form, select OK.
diff --git a/usr.sbin/sysinstall/help/usage.hlp b/usr.sbin/sysinstall/help/usage.hlp
deleted file mode 100644
index 268a946bc223..000000000000
--- a/usr.sbin/sysinstall/help/usage.hlp
+++ /dev/null
@@ -1,65 +0,0 @@
-HOW TO USE THIS SYSTEM
-======================
-
-[press the PageDown key to go to the next screen when you finish
- reading this one]
-
-The following keys are recognized in most of the dialogs you'll
-encounter during this installation:
-
-KEY             ACTION
----             ------
-SPACE           Select or toggle the current item.
-ENTER           Finish with a menu or item.
-UP ARROW        Move to previous item (or up, in a text display box).
-DOWN ARROW      Move to next item (or down, in a text display box).
-TAB             Move to next item or group.
-RIGHT ARROW     Move to next item or group (same as TAB).
-SHIFT-TAB       Move to previous item or group.
-LEFT ARROW      Move to previous item or group (same as SHIFT-TAB).
-PAGE UP         In text display boxes, scrolls up one page.
-PAGE DOWN       In text display boxes, scrolls down one page.
-F1              Display associated help text.
-
-If you see small "^(-)" or "v(+)" symbols at the edges of a menu, it
-means that there are more items above or below the current one that
-aren't being shown (due to insufficient screen space).  In text
-display boxes, the amount of text above the current point will be
-displayed as a percentage in the lower right corner.  Using the
-Up/Down arrow keys will cause the object to scroll by line.  The
-PageUp and PageDown keys will scroll by entire screens.
-
-Selecting OK in a menu will confirm whatever action it's controlling.
-Selecting Cancel will cancel the operation and generally return you to
-the previous menu. Use TAB to move the cursor around and select the
-buttons.
-
-Most screens obey the Help key (F1) - USE IT!  It generally offers useful
-context-specific hints on what to do at each stage of the installation,
-and if you're at all unsure about what to do at a given stage in the
-installation, hit F1!
-
-
-SPECIAL FEATURES:
-=================
-
-It is possible to select a menu item by typing the first character of
-its name, if unique.  This will generally be an item number.
-
-The console driver contains a scroll-back buffer for reviewing things
-that may have scrolled off the screen.  To use scroll-back, press the
-"Scroll Lock" key on your keyboard and use the arrow or Page Up/Page
-Down keys to move through the saved text.  To leave scroll-back mode,
-press the Scroll Lock key again.  This feature is most useful for
-reading back through your boot messages (go ahead, try it now!) though
-it's also useful when dealing with sub-shells or other "expert modes"
-that don't use menus and tend to scroll their output off the top of
-the screen.
-
-FreeBSD also supports multiple "virtual consoles" which you can use
-in order to have several active sessions at once.  Use ALT-F<n> to
-switch between screens, where `F<n>' is the function key corresponding
-to the screen you wish to see.  By default, the system comes with 8
-virtual consoles enabled - you can enable more by editing the
-/etc/ttys file and turning the "off" field to "on" in the relevant vty
-entries (up to 12).
diff --git a/usr.sbin/sysinstall/help/usermgmt.hlp b/usr.sbin/sysinstall/help/usermgmt.hlp
deleted file mode 100644
index fbd7cac101a0..000000000000
--- a/usr.sbin/sysinstall/help/usermgmt.hlp
+++ /dev/null
@@ -1,89 +0,0 @@
-These screens allow you to add groups and users to your system.
-
-You can move through the fields with the TAB, BACK-TAB and ENTER
-keys.  To edit a field, use DELETE or BACKSPACE.  You may also use ^A
-(control-A) to go to the beginning of the line, ^E (control-E) to go
-to the end, ^F (control-F) to go forward a character, ^B (control-B)
-to go backward one character, ^D (control-D) to delete the character
-under the cursor and ^K (control-K) to delete to the end of the line.
-Basically, the standard EMACS motion sequences.
-
-When you're done with this form, select OK.
-
-Many of the settings get reasonable defaults if you leave them blank.
-The first time you have entered the name of the new group or user, the
-system will show you what it would chose for most of these fields.
-You are free to change them, of course.
-
-
-User groups
-===========
-
-It's certainly almost generally a good idea to first create a new
-group for your users.  Common names for such a group are "users", or
-even simply "other".  Group names are used to control file access
-permissions for users that belong to the same group.  Several group
-names are already used for system files.
-
-The numerical user or group IDs are often nothing you want to care for
-explicitly.  If you don't fill in these fields, the system will choose
-reasonable defaults.  However, these numbers (rather than the
-associated names) are what the operating system actually uses to
-distinguish users and groups -- hence they should normally be unique
-to each person or group, respectively.
-
-(The initial membership list for a new group is currently
-unimplemented, sorry.)
-
-
-Users
-=====
-
-The user's login ID is a short (up to 15 characters) alphanumeric ID
-that the user must enter when logging into the system.  It's often the
-initial letters of the user's name, and commonly used in lower case.
-It's also the local mail name for this user (though it's possible to
-also set up more descriptive mail alias names later).
-
-The user's login group determines which group access rights the user
-will initially get when logging in.  If an additional list of groups is
-provided which the user will become a member of, (s)he will also be
-able to access files of those groups later without providing any
-additional password etc.  Except for the "wheel" case mentioned below,
-the additional group membership list should normally not contain the
-login group again.
-
-The user's password can also be set here, and should be chosen with
-care - 6 or more characters, intermixing punctuation and numerics, and
-*not* a word from the dictionary or related to the username is a good
-password choice.
-
-Some of the system's groups have a special meaning.  In particular,
-members of group "wheel" are the only people who are later allowed to
-become superuser using the command su(1).  So if you're going to add a
-new user who should later perform administrative tasks, don't forget
-to add him to this group!  (Well, ``he'' will most likely be yourself
-in the very first place. :)
-
-Also, members of group "operator" will by default get permissions for
-minor administrative operations, like performing system backups, or
-shutting down the system -- without first becoming superuser!  So,
-take care when adding people to this group.
-
-The ``full name'' field serves as a comment only.  It is also used by
-mail front ends to determine the real name of the user, hence you
-should actually fill in the first and last name of this user.  By
-convention, this field can be divided into comma-separated subfields,
-where the office location, the work phone number, and the home phone
-number follow the full name of the user.
-
-The home directory is the directory in the filesystem where the user
-is being logged into, and where his personalized setup files (``dot
-files'', since they usually begin with a `.' and are not displayed by
-the ls(1) command by default) will be looked up.  It is often created
-under /usr/home/ or /home/.
-
-Finally, the shell is the user's initial command interpreter.  The
-default shell is /bin/sh, some users prefer the more historic
-/bin/csh.  Other, often more user-friendly and comfortable shells can
-be found in the ports and packages collection.
diff --git a/usr.sbin/sysinstall/http.c b/usr.sbin/sysinstall/http.c
deleted file mode 100644
index 1355f084db6a..000000000000
--- a/usr.sbin/sysinstall/http.c
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Copyright (c) 1999
- *	Philipp Mergenthaler <philipp.mergenthaler@stud.uni-karlsruhe.de>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include "sysinstall.h"
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <sys/param.h>
-#include <netdb.h>
-
-extern const char *ftp_dirs[]; /* defined in ftp.c */
-
-static Boolean
-checkAccess(Boolean proxyCheckOnly)
-{
-/* 
- * Some proxies fetch files with certain extensions in "ascii mode" instead
- * of "binary mode" for FTP. The FTP server then translates all LF to CRLF.
- *
- * You can force Squid to use binary mode by appending ";type=i" to the URL,
- * which is what I do here. For other proxies, the LF->CRLF substitution
- * is reverted in distExtract().
- */
-
-    int rv, s, af;
-    bool el, found=FALSE;		    /* end of header line */
-    char *cp, buf[PATH_MAX], req[BUFSIZ];
-    struct addrinfo hints, *res, *res0;
-
-    af = variable_cmp(VAR_IPV6_ENABLE, "YES") ? AF_INET : AF_UNSPEC;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = af;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_protocol = 0;
-    if ((rv = getaddrinfo(variable_get(VAR_HTTP_HOST),
-			  variable_get(VAR_HTTP_PORT), &hints, &res0)) != 0) {
-	msgConfirm("%s", gai_strerror(rv));
-	variable_unset(VAR_HTTP_HOST);
-	return FALSE;
-    }
-    s = -1;
-    for (res = res0; res; res = res->ai_next) {
-	if ((s = socket(res->ai_family, res->ai_socktype,
-			res->ai_protocol)) < 0)
-	    continue;
-	if (connect(s, res->ai_addr, res->ai_addrlen) >= 0)
-	    break;
-	close(s);
-	s = -1;
-    }
-    freeaddrinfo(res0);
-    if (s == -1) {
-	msgConfirm("Couldn't connect to proxy %s:%s",
-		    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
-	variable_unset(VAR_HTTP_HOST);
-	return FALSE;
-    }
-    if (proxyCheckOnly) {
-       close(s);
-       return TRUE;
-    }
-
-    msgNotify("Checking access to\n %s", variable_get(VAR_HTTP_PATH));
-    sprintf(req,"GET %s/ HTTP/1.0\r\n\r\n", variable_get(VAR_HTTP_PATH));
-    write(s,req,strlen(req));
-/*
- *  scan the headers of the response
- *  this is extremely quick'n dirty
- *
- */
-    bzero(buf, PATH_MAX);
-    cp=buf;
-    el=FALSE;
-    rv=read(s,cp,1);
-    variable_set2(VAR_HTTP_FTP_MODE,"",0);
-    while (rv>0) {
-	if ((*cp == '\012') && el) { 
-	    /* reached end of a header line */
-	    if (!strncmp(buf,"HTTP",4)) {
-		if (strtol((char *)(buf+9),0,0) == 200) {
-		    found = TRUE;
-		}
-	    }
-
-	    if (!strncmp(buf,"Server: ",8)) {
-		if (!strncmp(buf,"Server: Squid",13)) {
-		    variable_set2(VAR_HTTP_FTP_MODE,";type=i",0);
-		} else {
-		    variable_set2(VAR_HTTP_FTP_MODE,"",0);
-		}
-	    }
-	    /* ignore other headers */
-	    /* check for "\015\012" at beginning of line, i.e. end of headers */
-	    if ((cp-buf) == 1)
-		break;
-	    cp=buf;
-	    rv=read(s,cp,1);
-	} else {
-	    el=FALSE;
-	    if (*cp == '\015')
-		el=TRUE;
-	    cp++;
-	    rv=read(s,cp,1);
-	}
-    }
-    close(s);
-    return found;
-} 
-
-Boolean
-mediaInitHTTP(Device *dev)
-{
-    bool found=FALSE;		    /* end of header line */
-    char *rel, req[BUFSIZ];
-    int fdir;
-
-    /* 
-     * First verify the proxy access
-     */
-    checkAccess(TRUE);
-    while (variable_get(VAR_HTTP_HOST) == NULL) {
-        if (DITEM_STATUS(mediaSetHTTP(NULL)) == DITEM_FAILURE)
-            return FALSE;
-        checkAccess(TRUE);
-    }
-again:
-    /* If the release is specified as "__RELEASE" or "any", then just
-     * assume that the path the user gave is ok.
-     */
-    rel = variable_get(VAR_RELNAME);
-    /*
-    msgConfirm("rel: -%s-", rel);
-    */
-
-    if (strcmp(rel, "__RELEASE") && strcmp(rel, "any"))  {
-        for (fdir = 0; ftp_dirs[fdir]; fdir++) {
-            sprintf(req, "%s/%s/%s", variable_get(VAR_FTP_PATH),
-                ftp_dirs[fdir], rel);
-            variable_set2(VAR_HTTP_PATH, req, 0);
-            if (checkAccess(FALSE)) {
-                found = TRUE;
-                break;
-            }
-        }
-    } else {
-        variable_set2(VAR_HTTP_PATH, variable_get(VAR_FTP_PATH), 0);
-        found = checkAccess(FALSE);
-    }
-    if (!found) {
-    	msgConfirm("No such directory: %s\n"
-		   "please check the URL and try again.", variable_get(VAR_HTTP_PATH));
-        variable_unset(VAR_HTTP_PATH);
-        dialog_clear_norefresh();
-        clear();
-        if (DITEM_STATUS(mediaSetHTTP(NULL)) != DITEM_FAILURE) goto again;
-    }
-    return found;
-}
-
-FILE *
-mediaGetHTTP(Device *dev, char *file, Boolean probe)
-{
-    FILE *fp;
-    int rv, s, af;
-    bool el;			/* end of header line */
-    char *cp, buf[PATH_MAX], req[BUFSIZ];
-    struct addrinfo hints, *res, *res0;
-
-    af = variable_cmp(VAR_IPV6_ENABLE, "YES") ? AF_INET : AF_UNSPEC;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = af;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_protocol = 0;
-    if ((rv = getaddrinfo(variable_get(VAR_HTTP_HOST),
-			  variable_get(VAR_HTTP_PORT), &hints, &res0)) != 0) {
-	msgConfirm("%s", gai_strerror(rv));
-	return NULL;
-    }
-    s = -1;
-    for (res = res0; res; res = res->ai_next) {
-	if ((s = socket(res->ai_family, res->ai_socktype,
-			res->ai_protocol)) < 0)
-	    continue;
-	if (connect(s, res->ai_addr, res->ai_addrlen) >= 0)
-	    break;
-	close(s);
-	s = -1;
-    }
-    freeaddrinfo(res0);
-    if (s == -1) {
-	msgConfirm("Couldn't connect to proxy %s:%s",
-		    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
-	return NULL;
-    }
-						   
-    sprintf(req,"GET %s/%s%s HTTP/1.0\r\n\r\n",
-	    variable_get(VAR_HTTP_PATH), file, variable_get(VAR_HTTP_FTP_MODE));
-
-    if (isDebug()) {
-	msgDebug("sending http request: %s\n",req);
-    }
-    write(s,req,strlen(req));
-
-/*
- *  scan the headers of the response
- *  this is extremely quick'n dirty
- *
- */
-    cp=buf;
-    el=FALSE;
-    rv=read(s,cp,1);
-    while (rv>0) {
-	if ((*cp == '\012') && el) {
-  	    /* reached end of a header line */
-  	    if (!strncmp(buf,"HTTP",4)) {
-		rv=strtol((char *)(buf+9),0,0);
-		*(cp-1)='\0';		/* chop the CRLF off */
-		if (probe && (rv != 200)) {
-		    return NULL;
-		} else if (rv >= 500) {
-		    msgConfirm("Server error %s when sending %s, you could try an other server",buf, req);
-		    return NULL;
-		} else if (rv == 404) {
-		    msgConfirm("%s was not found, maybe directory or release-version are wrong?",req);
-		    return NULL;
-		} else if (rv >= 400) {
-		    msgConfirm("Client error %s, you could try an other server",buf);
-		    return NULL;
-		} else if (rv >= 300) {
-		    msgConfirm("Error %s,",buf);
-		    return NULL;
-		} else if (rv != 200) {
-		    msgConfirm("Error %s when sending %s, you could try an other server",buf, req);
-		    return NULL;
-		}
-	    }
-	    /* ignore other headers */
-	    /* check for "\015\012" at beginning of line, i.e. end of headers */
-	    if ((cp-buf) == 1) 
-		break;
-	    cp=buf;
-	    rv=read(s,cp,1);
-	} else {
-	    el=FALSE;
-	    if (*cp == '\015')
-		el=TRUE;
-	    cp++;
-	    rv=read(s,cp,1);
-	}
-    }
-    fp=fdopen(s,"r");
-    return fp;
-}
diff --git a/usr.sbin/sysinstall/index.c b/usr.sbin/sysinstall/index.c
deleted file mode 100644
index c591cfef61f9..000000000000
--- a/usr.sbin/sysinstall/index.c
+++ /dev/null
@@ -1,893 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <fcntl.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <ncurses.h>
-#include <dialog.h>
-#include "sysinstall.h"
-
-/* Macros and magic values */
-#define MAX_MENU	12
-#define _MAX_DESC	55
-
-/* A structure holding the root, top and plist pointer at once */
-struct ListPtrs
-{
-    PkgNodePtr root;	/* root of tree */
-    PkgNodePtr top;	/* part of tree we handle */
-    PkgNodePtr plist;	/* list of selected packages */
-};
-typedef struct ListPtrs* ListPtrsPtr;
-
-static void	index_recorddeps(Boolean add, PkgNodePtr root, IndexEntryPtr ie);
-
-/* Shared between index_initialize() and the various clients of it */
-PkgNode Top, Plist;
-
-/* Smarter strdup */
-static inline char *
-_strdup(char *ptr)
-{
-    return ptr ? strdup(ptr) : NULL;
-}
-
-static char *descrs[] = {
-    "Package Selection", "To mark a package, move to it and press SPACE.  If the package is\n"
-    "already marked, it will be unmarked or deleted (if installed).\n"
-    "Items marked with a `D' are dependencies which will be auto-loaded.\n"
-    "To search for a package by name, press ESC.  To select a category,\n"
-    "press RETURN.  NOTE:  The All category selection creates a very large\n"
-    "submenu!  If you select it, please be patient while it comes up.",
-    "Package Targets", "These are the packages you've selected for extraction.\n\n"
-    "If you're sure of these choices, select OK.\n"
-    "If not, select Cancel to go back to the package selection menu.\n",
-    "All", "All available packages in all categories.",
-    "accessibility", "Ports to help disabled users.",
-    "afterstep", "Ports to support the AfterStep window manager.",
-    "arabic", "Ported software for Arab countries.",
-    "archivers", "Utilities for archiving and unarchiving data.",
-    "astro", "Applications related to astronomy.",
-    "audio", "Audio utilities - most require a supported sound card.",
-    "benchmarks", "Utilities for measuring system performance.",
-    "biology", "Software related to biology.",
-    "cad", "Computer Aided Design utilities.",
-    "chinese", "Ported software for the Chinese market.",
-    "comms", "Communications utilities.",
-    "converters", "Format conversion utilities.",
-    "databases", "Database software.",
-    "deskutils", "Various Desktop utilities.",
-    "devel", "Software development utilities and libraries.",
-    "dns", "Domain Name Service tools.",
-    "docs", "Meta-ports for FreeBSD documentation.",
-    "editors", "Editors.",
-    "elisp", "Things related to Emacs Lisp.",
-    "emulators", "Utilities for emulating other operating systems.",
-    "finance", "Monetary, financial and related applications.",
-    "french", "Ported software for French countries.",
-    "ftp", "FTP client and server utilities.",
-    "games", "Various and sundry amusements.",
-    "german", "Ported software for Germanic countries.",
-    "geography", "Geography-related software.",
-    "gnome", "Components of the Gnome Desktop environment.",
-    "gnustep", "Software for GNUstep desktop environment.",
-    "graphics", "Graphics libraries and utilities.",
-    "haskell", "Software related to the Haskell language.",
-    "hamradio", "Software for amateur radio.",
-    "hebrew", "Ported software for Hebrew language.",
-    "hungarian", "Ported software for the Hungarian market.",
-    "ipv6", "IPv6 related software.",
-    "irc", "Internet Relay Chat utilities.",
-    "japanese", "Ported software for the Japanese market.",
-    "java", "Java language support.",
-    "kde", "Software for the K Desktop Environment.",
-    "kld", "Kernel loadable modules",
-    "korean", "Ported software for the Korean market.",
-    "lang", "Computer languages.",
-    "linux", "Linux programs that can run under binary compatibility.",
-    "lisp", "Software related to the Lisp language.",
-    "mail", "Electronic mail packages and utilities.",
-    "math", "Mathematical computation software.",
-    "mbone", "Applications and utilities for the MBONE.",
-    "misc", "Miscellaneous utilities.",
-    "multimedia", "Multimedia software.",
-    "net", "Networking utilities.",
-    "net-im", "Instant messaging software.",
-    "net-mgmt", "Network management tools.",
-    "net-p2p", "Peer to peer network applications.",
-    "news", "USENET News support software.",
-    "palm", "Software support for the Palm(tm) series.",
-    "parallel", "Applications dealing with parallelism in computing.",
-    "pear", "Software related to the Pear PHP framework.",
-    "perl5", "Utilities/modules for the PERL5 language.",
-    "plan9", "Software from the Plan9 operating system.",
-    "polish", "Ported software for the Polish market.",
-    "ports-mgmt", "Utilities for managing ports and packages.",
-    "portuguese", "Ported software for the Portuguese market.",
-    "print", "Utilities for dealing with printing.",
-    "python", "Software related to the Python language.",
-    "ruby", "Software related to the Ruby language.",
-    "rubygems", "Ports of RubyGems packages.",
-    "russian", "Ported software for the Russian market.",
-    "scheme", "Software related to the Scheme language.",
-    "science", "Scientific software.",
-    "security", "System security software.",
-    "shells", "Various shells (tcsh, bash, etc).",
-    "spanish", "Ported software for the Spanish market.",
-    "sysutils", "Various system utilities.",
-    "tcl", "TCL and packages that depend on it.",
-    "tcl80", "TCL v8.0 and packages that depend on it.",
-    "tcl82", "TCL v8.2 and packages that depend on it.",
-    "tcl83", "TCL v8.3 and packages that depend on it.",
-    "tcl84", "TCL v8.4 and packages that depend on it.",
-    "textproc", "Text processing/search utilities.",
-    "tk", "Tk and packages that depend on it.",
-    "tk80", "Tk8.0 and packages that depend on it.",
-    "tk82", "Tk8.2 and packages that depend on it.",
-    "tk83", "Tk8.3 and packages that depend on it.",
-    "tk84", "Tk8.4 and packages that depend on it.",
-    "tkstep80", "Ports to support the TkStep window manager.",
-    "ukrainian", "Ported software for the Ukrainian market.",
-    "vietnamese", "Ported software for the Vietnamese market.",
-    "windowmaker", "Ports to support the WindowMaker window manager.",
-    "www", "Web utilities (browsers, HTTP servers, etc).",
-    "x11", "X Window System based utilities.",
-    "x11-clocks", "X Window System based clocks.",
-    "x11-drivers", "X Window System drivers.",
-    "x11-fm", "X Window System based file managers.",
-    "x11-fonts", "X Window System fonts and font utilities.",
-    "x11-servers", "X Window System servers.",
-    "x11-themes", "X Window System themes.",
-    "x11-toolkits", "X Window System based development toolkits.",
-    "x11-wm", "X Window System window managers.",
-    "xfce", "Software related to the Xfce Desktop Environment.",
-    "zope", "Software related to the Zope platform.",
-    NULL, NULL,
-};
-
-static char *
-fetch_desc(char *name)
-{
-    int i;
-
-    for (i = 0; descrs[i]; i += 2) {
-	if (!strcmp(descrs[i], name))
-	    return descrs[i + 1];
-    }
-    return "No description provided";
-}
-
-static PkgNodePtr
-new_pkg_node(char *name, node_type type)
-{
-    PkgNodePtr tmp = safe_malloc(sizeof(PkgNode));
-
-    tmp->name = _strdup(name);
-    tmp->type = type;
-    return tmp;
-}
-
-static char *
-strip(char *buf)
-{
-    int i;
-
-    for (i = 0; buf[i]; i++)
-	if (buf[i] == '\t' || buf[i] == '\n')
-	    buf[i] = ' ';
-    return buf;
-}
-
-static IndexEntryPtr
-new_index(char *name, char *pathto, char *prefix, char *comment, char *descr, char *maint, char *deps, int volume)
-{
-    IndexEntryPtr tmp = safe_malloc(sizeof(IndexEntry));
-
-    tmp->name =		_strdup(name);
-    tmp->path =		_strdup(pathto);
-    tmp->prefix =	_strdup(prefix);
-    tmp->comment =	_strdup(comment);
-    tmp->descrfile =	strip(_strdup(descr));
-    tmp->maintainer =	_strdup(maint);
-    tmp->deps =		_strdup(deps);
-    tmp->depc =		0;
-    tmp->installed =	package_installed(name);
-    tmp->vol_checked =	0;
-    tmp->volume =	volume;
-    if (volume != 0) {
-	have_volumes = TRUE;
-	if (low_volume == 0)
-	    low_volume = volume;
-	else if (low_volume > volume)
-	    low_volume = volume;
-	if (high_volume < volume)
-	    high_volume = volume;
-    }
-    return tmp;
-}
-
-static void
-index_register(PkgNodePtr top, char *where, IndexEntryPtr ptr)
-{
-    PkgNodePtr p, q;
-
-    for (q = NULL, p = top->kids; p; p = p->next) {
-	if (!strcmp(p->name, where)) {
-	    q = p;
-	    break;
-	}
-    }
-    if (!p) {
-	/* Add new category */
-	q = new_pkg_node(where, PLACE);
-	q->desc = fetch_desc(where);
-	q->next = top->kids;
-	top->kids = q;
-    }
-    p = new_pkg_node(ptr->name, PACKAGE);
-    p->desc = ptr->comment;
-    p->data = ptr;
-    p->next = q->kids;
-    q->kids = p;
-}
-
-static int
-copy_to_sep(char *to, char *from, int sep)
-{
-    char *tok;
-
-    tok = strchr(from, sep);
-    if (!tok) {
-	*to = '\0';
-	return 0;
-    }
-    *tok = '\0';
-    strcpy(to, from);
-    return tok + 1 - from;
-}
-
-static int
-skip_to_sep(char *from, int sep)
-{
-    char *tok;
-
-    tok = strchr(from, sep);
-    if (!tok)
-	return 0;
-    *tok = '\0';
-    return tok + 1 - from;
-}
-
-static int
-readline(FILE *fp, char *buf, int max)
-{
-    int rv, i = 0;
-    char ch;
-
-    while ((rv = fread(&ch, 1, 1, fp)) == 1 && ch != '\n' && i < max)
-	buf[i++] = ch;
-    if (i < max)
-	buf[i] = '\0';
-    return rv;
-}
-
-/*
- * XXX - this function should do error checking, and skip corrupted INDEX
- * lines without a set number of '|' delimited fields.
- */
-
-static int
-index_parse(FILE *fp, char *name, char *pathto, char *prefix, char *comment, char *descr, char *maint, char *cats, char *rdeps, int *volume)
-{
-    char line[10240 + 2048 * 7];
-    char junk[2048];
-    char volstr[2048];
-    char *cp;
-    int i;
-
-    i = readline(fp, line, sizeof line);
-    if (i <= 0)
-	return EOF;
-    cp = line;
-    cp += copy_to_sep(name, cp, '|');		/* package name */
-    cp += copy_to_sep(pathto, cp, '|');		/* ports directory */
-    cp += copy_to_sep(prefix, cp, '|');		/* prefix */
-    cp += copy_to_sep(comment, cp, '|');	/* comment */
-    cp += copy_to_sep(descr, cp, '|');		/* path to pkg-descr */
-    cp += copy_to_sep(maint, cp, '|');		/* maintainer */
-    cp += copy_to_sep(cats, cp, '|');		/* categories */
-    cp += skip_to_sep(cp, '|');			/* build deps - not used */
-    cp += copy_to_sep(rdeps, cp, '|');		/* run deps */
-    if (index(cp, '|'))
-        cp += skip_to_sep(cp, '|');		/* url - not used */
-    else {
-	strncpy(junk, cp, 1023);
-	*volume = 0;
-	return 0;
-    }
-    if (index(cp, '|'))
-	cp += skip_to_sep(cp, '|');		/* extract deps - not used */
-    if (index(cp, '|'))
-	cp += skip_to_sep(cp, '|');		/* patch deps - not used */
-    if (index(cp, '|'))
-	cp += skip_to_sep(cp, '|');		/* fetch deps - not used */
-    if (index(cp, '|'))
-        cp += copy_to_sep(volstr, cp, '|');	/* media volume */
-    else {
-	strncpy(volstr, cp, 1023);
-    }
-    *volume = atoi(volstr);
-    return 0;
-}
-
-int
-index_read(FILE *fp, PkgNodePtr papa)
-{
-    char name[127], pathto[255], prefix[255], comment[255], descr[127], maint[127], cats[511], deps[2048 * 8];
-    int volume;
-    PkgNodePtr i;
-
-    while (index_parse(fp, name, pathto, prefix, comment, descr, maint, cats, deps, &volume) != EOF) {
-	char *cp, *cp2, tmp[1024];
-	IndexEntryPtr idx;
-
-	idx = new_index(name, pathto, prefix, comment, descr, maint, deps, volume);
-	/* For now, we only add things to menus if they're in categories.  Keywords are ignored */
-	for (cp = strcpy(tmp, cats); (cp2 = strchr(cp, ' ')) != NULL; cp = cp2 + 1) {
-	    *cp2 = '\0';
-	    index_register(papa, cp, idx);
-	}
-	index_register(papa, cp, idx);
-
-	/* Add to special "All" category */
-	index_register(papa, "All", idx);
-    }
-
-    /* Adjust dependency counts */
-    for (i = papa->kids; i != NULL; i = i->next)
-	if (strcmp(i->name, "All") == 0)
-	    break;
-    for (i = i->kids; i != NULL; i = i->next)
-	if (((IndexEntryPtr)i->data)->installed)
-	    index_recorddeps(TRUE, papa, i->data);
-
-    return 0;
-}
-
-void
-index_init(PkgNodePtr top, PkgNodePtr plist)
-{
-    if (top) {
-	top->next = top->kids = NULL;
-	top->name = "Package Selection";
-	top->type = PLACE;
-	top->desc = fetch_desc(top->name);
-	top->data = NULL;
-    }
-    if (plist) {
-	plist->next = plist->kids = NULL;
-	plist->name = "Package Targets";
-	plist->type = PLACE;
-	plist->desc = fetch_desc(plist->name);
-	plist->data = NULL;
-    }
-}
-
-void
-index_print(PkgNodePtr top, int level)
-{
-    int i;
-
-    while (top) {
-	for (i = 0; i < level; i++) putchar('\t');
-	printf("name [%s]: %s\n", top->type == PLACE ? "place" : "package", top->name);
-	for (i = 0; i < level; i++) putchar('\t');
-	printf("desc: %s\n", top->desc);
-	if (top->kids)
-	    index_print(top->kids, level + 1);
-	top = top->next;
-    }
-}
-
-/* Swap one node for another */
-static void
-swap_nodes(PkgNodePtr a, PkgNodePtr b)
-{
-    PkgNode tmp;
-
-    tmp = *a;
-    *a = *b;
-    a->next = tmp.next;
-    tmp.next = b->next;
-    *b = tmp;
-}
-
-/* Use a disgustingly simplistic bubble sort to put our lists in order */
-void
-index_sort(PkgNodePtr top)
-{
-    PkgNodePtr p, q;
-
-    /* Sort everything at the top level */
-    for (p = top->kids; p; p = p->next) {
-	for (q = top->kids; q; q = q->next) {
-	    if (q->next && strcmp(q->name, q->next->name) > 0)
-		swap_nodes(q, q->next);
-	}
-    }
-
-    /* Now sub-sort everything n levels down */    
-    for (p = top->kids; p; p = p->next) {
-	if (p->kids)
-	    index_sort(p);
-    }
-}
-
-/* Delete an entry out of the list it's in (only the plist, at present) */
-static void
-index_delete(PkgNodePtr n)
-{
-    if (n->next) {
-	PkgNodePtr p = n->next;
-
-	*n = *(n->next);
-	safe_free(p);
-    }
-    else /* Kludgy end sentinal */
-	n->name = NULL;
-}
-
-/*
- * Search for a given node by name, returning the category in if
- * tp is non-NULL.
- */
-PkgNodePtr
-index_search(PkgNodePtr top, char *str, PkgNodePtr *tp)
-{
-    PkgNodePtr p, sp;
-
-    for (p = top->kids; p && p->name; p = p->next) {
-	if (p->type == PACKAGE) {
-	    /* If tp == NULL, we're looking for an exact package match */
-	    if (!tp && !strcmp(p->name, str))
-		return p;
-
-	    /* If tp, we're looking for both a package and a pointer to the place it's in */
-	    if (tp && !strncmp(p->name, str, strlen(str))) {
-		*tp = top;
-		return p;
-	    }
-	}
-	else if (p->kids) {
-	    /* The usual recursion-out-of-laziness ploy */
-	    if ((sp = index_search(p, str, tp)) != NULL)
-		return sp;
-	}
-    }
-    if (p && !p->name)
-	p = NULL;
-    return p;
-}
-
-static int
-pkg_checked(dialogMenuItem *self)
-{
-    ListPtrsPtr lists = (ListPtrsPtr)self->aux;
-    PkgNodePtr kp = self->data, plist = lists->plist;
-    int i;
-
-    i = index_search(plist, kp->name, NULL) ? TRUE : FALSE;
-    if (kp->type == PACKAGE && plist) {
-	IndexEntryPtr ie = kp->data;
-	int markD, markX;
-
-	markD = ie->depc > 0; /* needed as dependency */
-	markX = i || ie->installed; /* selected or installed */
-	self->mark = markX ? 'X' : 'D';
-	return markD || markX;
-    } else
-	return FALSE;
-}
-
-static int
-pkg_fire(dialogMenuItem *self)
-{
-    int ret;
-    ListPtrsPtr lists = (ListPtrsPtr)self->aux;
-    PkgNodePtr sp, kp = self->data, plist = lists->plist;
-
-    if (!plist)
-	ret = DITEM_FAILURE;
-    else if (kp->type == PACKAGE) {
-	IndexEntryPtr ie = kp->data;
-
-	sp = index_search(plist, kp->name, NULL);
-	/* Not already selected? */
-	if (!sp) {
-	    if (!ie->installed) {
-		PkgNodePtr np = (PkgNodePtr)safe_malloc(sizeof(PkgNode));
-
-		*np = *kp;
-		np->next = plist->kids;
-		plist->kids = np;
-		index_recorddeps(TRUE, lists->root, ie);
-		msgInfo("Added %s to selection list", kp->name);
-	    }
-	    else if (ie->depc == 0) {
-		if (!msgNoYes("Do you really want to delete %s from the system?", kp->name)) {
-		    if (vsystem("pkg_delete %s %s", isDebug() ? "-v" : "", kp->name)) {
-			msgConfirm("Warning:  pkg_delete of %s failed.\n  Check debug output for details.", kp->name);
-		    }
-		    else {
-			ie->installed = 0;
-			index_recorddeps(FALSE, lists->root, ie);
-		    }
-		}
-	    }
-	    else
-		msgConfirm("Warning: Package %s is needed by\n  %d other installed package%s.",
-			   kp->name, ie->depc, (ie->depc != 1) ? "s" : "");
-	}
-	else {
-	    index_recorddeps(FALSE, lists->root, ie);
-	    msgInfo("Removed %s from selection list", kp->name);
-	    index_delete(sp);
-	}
-	ret = DITEM_SUCCESS;
-	/* Mark menu for redraw if we had dependencies */
-	if (strlen(ie->deps) > 0)
-	    ret |= DITEM_REDRAW;
-    }
-    else {	/* Not a package, must be a directory */
-	int p, s;
-		    
-	p = s = 0;
-	index_menu(lists->root, kp, plist, &p, &s);
-	ret = DITEM_SUCCESS | DITEM_CONTINUE;
-    }
-    return ret;
-}
-
-static void
-pkg_selected(dialogMenuItem *self, int is_selected)
-{
-    PkgNodePtr kp = self->data;
-
-    if (!is_selected || kp->type != PACKAGE)
-	return;
-    msgInfo("%s", kp->desc);
-}
-
-int
-index_menu(PkgNodePtr root, PkgNodePtr top, PkgNodePtr plist, int *pos, int *scroll)
-{
-    struct ListPtrs lists;
-    size_t maxname;
-    int n, rval;
-    int curr, max;
-    PkgNodePtr kp;
-    dialogMenuItem *nitems;
-    Boolean hasPackages;
-    WINDOW *w;
-    
-    lists.root = root;
-    lists.top = top;
-    lists.plist = plist;
-
-    hasPackages = FALSE;
-    nitems = NULL;
-    n = maxname = 0;
-
-    /* Figure out if this menu is full of "leaves" or "branches" */
-    for (kp = top->kids; kp && kp->name; kp = kp->next) {
-	size_t len;
-
-	++n;
-	if (kp->type == PACKAGE && plist) {
-	    hasPackages = TRUE;
-	    if ((len = strlen(kp->name)) > maxname)
-		maxname = len;
-	}
-    }
-    if (!n && plist) {
-	msgConfirm("The %s menu is empty.", top->name);
-	return DITEM_LEAVE_MENU;
-    }
-
-    w = savescr();
-    while (1) {
-	n = 0;
-	curr = max = 0;
-	use_helpline(NULL);
-	use_helpfile(NULL);
-	kp = top->kids;
-	if (!hasPackages && plist) {
-	    nitems = item_add(nitems, "OK", NULL, NULL, NULL, NULL, NULL, NULL, &curr, &max);
-	    nitems = item_add(nitems, "Install", NULL, NULL, NULL, NULL, NULL, NULL, &curr, &max);
-	}
-	while (kp && kp->name) {
-	    char buf[256];
-	    IndexEntryPtr ie = kp->data;
-
-	    /* Brutally adjust description to fit in menu */
-	    if (kp->type == PACKAGE)
-		snprintf(buf, sizeof buf, "[%s]", ie->path ? ie->path : "External vendor");
-	    else
-		SAFE_STRCPY(buf, kp->desc);
-	    if (strlen(buf) > (_MAX_DESC - maxname))
-		buf[_MAX_DESC - maxname] = '\0';
-	    nitems = item_add(nitems, kp->name, buf, pkg_checked, 
-			      pkg_fire, pkg_selected, kp, &lists, 
-			      &curr, &max);
-	    ++n;
-	    kp = kp->next;
-	}
-	/* NULL delimiter so item_free() knows when to stop later */
-	nitems = item_add(nitems, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 
-			  &curr, &max);
-
-recycle:
-	dialog_clear_norefresh();
-	if (hasPackages)
-	    rval = dialog_checklist(top->name, top->desc, -1, -1, n > MAX_MENU ? MAX_MENU : n, -n, nitems, NULL);
-	else
-	    rval = dialog_menu(top->name, top->desc, -1, -1, n > MAX_MENU ? MAX_MENU : n, -n, nitems + (plist ? 2 : 0), (char *)plist, pos, scroll);
-	if (rval == -1 && plist) {
-	    static char *cp;
-	    PkgNodePtr menu;
-
-	    /* Search */
-	    if ((cp = msgGetInput(cp, "Search by package name.  Please enter search string:")) != NULL) {
-		PkgNodePtr p = index_search(top, cp, &menu);
-
-		if (p) {
-		    int pos, scroll;
-
-		    /* These need to be set to point at the found item, actually.  Hmmm! */
-		    pos = scroll = 0;
-		    index_menu(root, menu, plist, &pos, &scroll);
-		}
-		else
-		    msgConfirm("Search string: %s yielded no hits.", cp);
-	    }
-	    goto recycle;
-	}
-	items_free(nitems, &curr, &max);
-	restorescr(w);
-	return rval ? DITEM_FAILURE : DITEM_SUCCESS;
-    }
-}
-
-int
-index_extract(Device *dev, PkgNodePtr top, PkgNodePtr who, Boolean depended,
-    int current_volume)
-{
-    int status = DITEM_SUCCESS;
-    Boolean notyet = FALSE;
-    PkgNodePtr tmp2;
-    IndexEntryPtr id = who->data;
-    WINDOW *w;
-
-    /* 
-     * Short-circuit the package dependency checks.  We're already
-     * maintaining a data structure of installed packages, so if a
-     * package is already installed, don't try to check to make sure
-     * that all of its dependencies are installed.  At best this
-     * wastes a ton of cycles and can cause minor delays between
-     * package extraction.  At worst it can cause an infinite loop with
-     * a certain faulty INDEX file. 
-     */
-
-    if (id->installed == 1 || (have_volumes && id->vol_checked == current_volume))
-	return DITEM_SUCCESS;
-
-    w = savescr();
-    if (id && id->deps && strlen(id->deps)) {
-	char t[2048 * 8], *cp, *cp2;
-
-	SAFE_STRCPY(t, id->deps);
-	cp = t;
-	while (cp && DITEM_STATUS(status) == DITEM_SUCCESS) {
-	    if ((cp2 = index(cp, ' ')) != NULL)
-		*cp2 = '\0';
-	    if ((tmp2 = index_search(top, cp, NULL)) != NULL) {
-		status = index_extract(dev, top, tmp2, TRUE, current_volume);
-		if (DITEM_STATUS(status) != DITEM_SUCCESS) {
-		    /* package probably on a future disc volume */
-		    if (status & DITEM_CONTINUE) {
-			status = DITEM_SUCCESS;
-			notyet = TRUE;
-		    } else if (variable_get(VAR_NO_CONFIRM))
-			msgNotify("Loading of dependent package %s failed", cp);
-		    else
-			msgConfirm("Loading of dependent package %s failed", cp);
-		}
-	    }
-	    else if (!package_installed(cp)) {
-		if (variable_get(VAR_NO_CONFIRM))
-		    msgNotify("Warning: %s is a required package but was not found.", cp);
-		else
-		    msgConfirm("Warning: %s is a required package but was not found.", cp);
-	    }
-	    if (cp2)
-		cp = cp2 + 1;
-	    else
-		cp = NULL;
-	}
-    }
-
-    /*
-     * If iterating through disc volumes one at a time indicate failure if
-     * dependency install failed due to package being on a higher volume
-     * numbered disc, but that we should continue anyway.  Note that this
-     * package has already been processed for this disc volume so we don't
-     * need to do it again.
-     */
-
-    if (notyet) {
-    	restorescr(w);
-	id->vol_checked = current_volume;
-	return DITEM_FAILURE | DITEM_CONTINUE;
-    }
-
-    /*
-     * Done with the deps?  Try to load the real m'coy.  If iterating
-     * through a multi-volume disc set fail the install if the package
-     * is on a higher numbered volume to cut down on disc switches the
-     * user needs to do, but indicate caller should continue processing
-     * despite error return.  Note this package was processed for the
-     * current disc being checked.
-     */
-
-    if (DITEM_STATUS(status) == DITEM_SUCCESS) {
-	/* Prompt user if the package is not available on the current volume. */
-	if(mediaDevice->type == DEVICE_TYPE_CDROM) {
-	    if (current_volume != 0 && id->volume > current_volume) {
-		restorescr(w);
-		id->vol_checked = current_volume;
-		return DITEM_FAILURE | DITEM_CONTINUE;
-	    }
-	    while (id->volume != dev->volume) {
-		if (!msgYesNo("This is disc #%d.  Package %s is on disc #%d\n"
-			  "Would you like to switch discs now?\n", dev->volume,
-			  id->name, id->volume)) {
-		    DEVICE_SHUTDOWN(mediaDevice);
-		    msgConfirm("Please remove disc #%d from your drive, and add disc #%d\n",
-			dev->volume, id->volume);
-		    DEVICE_INIT(mediaDevice);
-		} else {
-		    restorescr(w);
-		    return DITEM_FAILURE;
-		}
-	    }
-	}
-	status = package_extract(dev, who->name, depended);
-	if (DITEM_STATUS(status) == DITEM_SUCCESS)
-	    id->installed = 1;
-    }
-    restorescr(w);
-    return status;
-}
-
-static void
-index_recorddeps(Boolean add, PkgNodePtr root, IndexEntryPtr ie)
-{
-   char depends[1024 * 16], *space, *todo;
-   PkgNodePtr found;
-   IndexEntryPtr found_ie;
-
-   SAFE_STRCPY(depends, ie->deps);
-   for (todo = depends; todo != NULL; ) {
-      space = index(todo, ' ');
-      if (space != NULL)
-	 *space = '\0';
-
-      if (strlen(todo) > 0) { /* only non-empty dependencies */
-	  found = index_search(root, todo, NULL);
-	  if (found != NULL) {
-	      found_ie = found->data;
-	      if (add)
-		  ++found_ie->depc;
-	      else
-		  --found_ie->depc;
-	  }
-      }
-
-      if (space != NULL)
-	 todo = space + 1;
-      else
-	 todo = NULL;
-   }
-}
-
-static Boolean index_initted;
-
-/* Read and initialize global index */
-int
-index_initialize(char *path)
-{
-    FILE *fp;
-    WINDOW *w = NULL;
-
-    if (!index_initted) {
-	w = savescr();
-	dialog_clear_norefresh();
-	have_volumes = FALSE;
-	low_volume = high_volume = 0;
-
-	/* Got any media? */
-	if (!mediaVerify()) {
-	    restorescr(w);
-	    return DITEM_FAILURE;
-	}
-
-	/* Does it move when you kick it? */
-	if (!DEVICE_INIT(mediaDevice)) {
-	    restorescr(w);
-	    return DITEM_FAILURE;
-	}
-
-	dialog_clear_norefresh();
-	msgNotify("Attempting to fetch %s file from selected media.", path);
-	fp = DEVICE_GET(mediaDevice, path, TRUE);
-	if (!fp) {
-	    msgConfirm("Unable to get packages/INDEX file from selected media.\n\n"
-		       "This may be because the packages collection is not available\n"
-		       "on the distribution media you've chosen, most likely an FTP site\n"
-		       "without the packages collection mirrored.  Please verify that\n"
-		       "your media, or your path to the media, is correct and try again.");
-	    DEVICE_SHUTDOWN(mediaDevice);
-	    restorescr(w);
-	    return DITEM_FAILURE;
-	}
-	dialog_clear_norefresh();
-	msgNotify("Located INDEX, now reading package data from it...");
-	index_init(&Top, &Plist);
-	if (index_read(fp, &Top)) {
-	    msgConfirm("I/O or format error on packages/INDEX file.\n"
-		       "Please verify media (or path to media) and try again.");
-	    fclose(fp);
-	    restorescr(w);
-	    return DITEM_FAILURE;
-	}
-	fclose(fp);
-	index_sort(&Top);
-	index_initted = TRUE;
-	restorescr(w);
-    }
-    return DITEM_SUCCESS;
-}
diff --git a/usr.sbin/sysinstall/install.c b/usr.sbin/sysinstall/install.c
deleted file mode 100644
index d265522c16e9..000000000000
--- a/usr.sbin/sysinstall/install.c
+++ /dev/null
@@ -1,1349 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <sys/consio.h>
-#include <sys/disklabel.h>
-#include <sys/errno.h>
-#include <sys/ioctl.h>
-#include <sys/fcntl.h>
-#include <sys/wait.h>
-#include <sys/uio.h>
-#include <sys/param.h>
-#define MSDOSFS
-#include <sys/mount.h>
-#include <ufs/ufs/ufsmount.h>
-#include <fs/msdosfs/msdosfsmount.h>
-#undef MSDOSFS
-#include <sys/stat.h>
-#include <sys/sysctl.h>
-#include <libdisk.h>
-#include <limits.h>
-#include <unistd.h>
-#include <termios.h>
-
-/* Hack for rsaref package add, which displays interactive license.
- * Used by package.c
- */
-int _interactiveHack;
-int FixItMode = 0;
-int NCpus;
-
-static void	create_termcap(void);
-static void	fixit_common(void);
-int		fixit_livefs_common(dialogMenuItem *self);
-
-#define TERMCAP_FILE	"/usr/share/misc/termcap"
-
-static void	installConfigure(void);
-
-Boolean
-checkLabels(Boolean whinge)
-{
-    Device **devs;
-    Boolean status;
-    Disk *disk;
-    PartInfo *pi;
-    Chunk *c1, *c2;
-    int i;
-
-    /* Don't allow whinging if noWarn is set */
-    if (variable_get(VAR_NO_WARN))
-	whinge = FALSE;
-
-    status = TRUE;
-    HomeChunk = RootChunk = SwapChunk = NULL;
-    TmpChunk = UsrChunk = VarChunk = NULL;
-#ifdef __ia64__
-    EfiChunk = NULL;
-#endif
-
-    /* We don't need to worry about root/usr/swap if we're already multiuser */
-    if (!RunningAsInit)
-	return status;
-
-    devs = deviceFind(NULL, DEVICE_TYPE_DISK);
-    /* First verify that we have a root device */
-    for (i = 0; devs[i]; i++) {
-	if (!devs[i]->enabled)
-	    continue;
-	disk = (Disk *)devs[i]->private;
-	msgDebug("Scanning disk %s for root filesystem\n", disk->name);
-	if (!disk->chunks)
-	    msgFatal("No chunk list found for %s!", disk->name);
-	for (c1 = disk->chunks->part; c1; c1 = c1->next) {
-#ifdef __ia64__
-	    c2 = c1;
-#elif defined(__powerpc__)
-	    if (c1->type == apple) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-#else
-	    if (c1->type == freebsd) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-#endif
-
-		    pi = (PartInfo *)c2->private_data;
-		    if (c2->type == part && c2->subtype != FS_SWAP && pi != NULL) {
-			if (!strcmp(pi->mountpoint, "/")) {
-			    if (RootChunk) {
-				if (whinge)
-				    msgConfirm("WARNING:  You have more than one root device set?!\n"
-					       "Using the first one found.");
-				continue;
-			    }
-			    else {
-				RootChunk = c2;
-				if (isDebug())
-				    msgDebug("Found rootdev at %s!\n", RootChunk->name);
-			    }
-			}
-			else if (!strcmp(pi->mountpoint, "/usr")) {
-			    if (UsrChunk) {
-				if (whinge)
-				    msgConfirm("WARNING:  You have more than one /usr filesystem.\n"
-					       "Using the first one found.");
-				continue;
-			    }
-			    else {
-				UsrChunk = c2;
-				if (isDebug())
-				    msgDebug("Found usrdev at %s!\n", UsrChunk->name);
-			    }
-			}
-			else if (!strcmp(pi->mountpoint, "/var")) {
-			    if (VarChunk) {
-				if (whinge)
-				    msgConfirm("WARNING:  You have more than one /var filesystem.\n"
-					       "Using the first one found.");
-				continue;
-			    }
-			    else {
-				VarChunk = c2;
-				if (isDebug())
-				    msgDebug("Found vardev at %s!\n", VarChunk->name);
-			    }
-			} else if (!strcmp(pi->mountpoint, "/tmp")) {
-			    if (TmpChunk) {
-				if (whinge)
-				    msgConfirm("WARNING:  You have more than one /tmp filesystem.\n"
-					       "Using the first one found.");
-				continue;
-			    }
-			    else {
-				TmpChunk = c2;
-				if (isDebug())
-				    msgDebug("Found tmpdev at %s!\n", TmpChunk->name);
-			    }
-			} else if (!strcmp(pi->mountpoint, "/home")) {
-			    if (HomeChunk) {
-				if (whinge)
-				    msgConfirm("WARNING:  You have more than one /home filesystem.\n"
-					       "Using the first one found.");
-				continue;
-			    }
-			    else {
-				HomeChunk = c2;
-				if (isDebug())
-				    msgDebug("Found homedev at %s!\n", HomeChunk->name);
-			    }
-			}
-		    }
-#ifndef __ia64__
-		}
-	    }
-#endif
-	}
-    }
-
-    /* Now check for swap devices */
-    for (i = 0; devs[i]; i++) {
-	if (!devs[i]->enabled)
-	    continue;
-	disk = (Disk *)devs[i]->private;
-	msgDebug("Scanning disk %s for swap partitions\n", disk->name);
-	if (!disk->chunks)
-	    msgFatal("No chunk list found for %s!", disk->name);
-	for (c1 = disk->chunks->part; c1; c1 = c1->next) {
-
-#ifdef __ia64__
-	    c2 = c1;
-#elif defined(__powerpc__)
-	    if (c1->type == apple) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-#else
-	    if (c1->type == freebsd) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-#endif
-		    if (c2->type == part && c2->subtype == FS_SWAP && !SwapChunk) {
-			SwapChunk = c2;
-			if (isDebug())
-			    msgDebug("Found swapdev at %s!\n", SwapChunk->name);
-			break;
-		    }
-#ifndef __ia64__
-		}
-	    }
-#endif
-	}
-    }
-
-#ifdef __ia64__
-    for (i = 0; devs[i] != NULL; i++) {
-	if (!devs[i]->enabled)
-	    continue;
-	disk = (Disk *)devs[i]->private;
-	for (c1 = disk->chunks->part; c1 != NULL; c1 = c1->next) {
-		pi = (PartInfo *)c1->private_data;
-	    if (c1->type == efi && pi != NULL && pi->mountpoint[0] == '/')
-		EfiChunk = c1;
-	}
-    }
-#endif
-
-    if (!RootChunk && whinge) {
-	msgConfirm("No root device found - you must label a partition as /\n"
-		   "in the label editor.");
-	status = FALSE;
-    }
-    if (!SwapChunk && whinge) {
-	if (msgYesNo("No swap devices found - you should create at least one\n"
-		     "swap partition.  Without swap, the install will fail\n"
-		     "if you do not have enough RAM.  Continue anyway?"))
-	    status = FALSE;
-    }
-#ifdef __ia64__
-    if (EfiChunk == NULL && whinge) {
-	if (msgYesNo("No (mounted) EFI system partition found. Is this what you want?"))
-	    status = FALSE;
-    }
-#endif
-    return status;
-}
-
-static int
-installInitial(void)
-{
-    static Boolean alreadyDone = FALSE;
-    int status = DITEM_SUCCESS;
-
-    if (alreadyDone)
-	return DITEM_SUCCESS;
-
-    if (!variable_get(DISK_LABELLED)) {
-	msgConfirm("You need to assign disk labels before you can proceed with\n"
-		   "the installation.");
-	return DITEM_FAILURE;
-    }
-    /* If it's labelled, assume it's also partitioned */
-    if (!variable_get(DISK_PARTITIONED))
-	variable_set2(DISK_PARTITIONED, "yes", 0);
-
-    /* If we refuse to proceed, bail. */
-    dialog_clear_norefresh();
-    if (!variable_get(VAR_NO_WARN)) {
-	if (msgYesNo(
-	    "Last Chance!  Are you SURE you want continue the installation?\n\n"
-	    "If you're running this on a disk with data you wish to save\n"
-	    "then WE STRONGLY ENCOURAGE YOU TO MAKE PROPER BACKUPS before\n"
-	    "proceeding!\n\n"
-	    "We can take no responsibility for lost disk contents!") != 0)
-	return DITEM_FAILURE;
-    }
-
-    if (DITEM_STATUS(diskLabelCommit(NULL)) != DITEM_SUCCESS) {
-	msgConfirm("Couldn't make filesystems properly.  Aborting.");
-	return DITEM_FAILURE;
-    }
-
-    if (!copySelf()) {
-	msgConfirm("installInitial: Couldn't clone the boot floppy onto the\n"
-		   "root file system.  Aborting!");
-	return DITEM_FAILURE;
-    }
-
-    if (!Restarting && chroot("/mnt") == -1) {
-	msgConfirm("installInitial: Unable to chroot to %s - this is bad!",
-		   "/mnt");
-	return DITEM_FAILURE;
-    }
-
-    chdir("/");
-    variable_set2(RUNNING_ON_ROOT, "yes", 0);
-
-    /* Configure various files in /etc */
-    if (DITEM_STATUS(configResolv(NULL)) == DITEM_FAILURE)
-	status = DITEM_FAILURE;
-    if (DITEM_STATUS(configFstab(NULL)) == DITEM_FAILURE)
-	status = DITEM_FAILURE;
-
-    /* stick a helpful shell over on the 4th VTY */
-    if (!variable_get(VAR_NO_HOLOSHELL))
-	systemCreateHoloshell();
-
-    alreadyDone = TRUE;
-    return status;
-}
-
-int
-installFixitHoloShell(dialogMenuItem *self)
-{
-    FixItMode = 1;
-    systemCreateHoloshell();
-    FixItMode = 0;
-    return DITEM_SUCCESS;
-}
-
-/*
- * Load the live filesystem from USB media.
- */
-int
-installFixitUSB(dialogMenuItem *self)
-{
-	if (!RunningAsInit)
-		return (DITEM_SUCCESS);
-
-	variable_set2(SYSTEM_STATE, "fixit", 0);
-
-	if (DITEM_STATUS(mediaSetUSB(NULL)) != DITEM_SUCCESS ||
-	    !DEVICE_INIT(mediaDevice)) {
-		msgConfirm("No USB devices found!");
-		return (DITEM_FAILURE);
-	} else if (!file_readable("/dist/rescue/ldconfig")) {
-		msgConfirm("Unable to find a FreeBSD live filesystem.");
-		return (DITEM_FAILURE);
-	}
-
-	if (DITEM_STATUS(fixit_livefs_common(self)) == DITEM_FAILURE)
-		return (DITEM_FAILURE);
-
-	mediaClose();
-	return (DITEM_SUCCESS);
-}
-
-int
-installFixitCDROM(dialogMenuItem *self)
-{
-    int need_eject;
-
-    if (!RunningAsInit)
-	return DITEM_SUCCESS;
-
-    variable_set2(SYSTEM_STATE, "fixit", 0);
-    need_eject = 0;
-    CDROMInitQuiet = 1;
-    while (1) {
-	if (need_eject)
-	    msgConfirm(
-	"Please insert a FreeBSD live filesystem CD/DVD and press return");
-	if (DITEM_STATUS(mediaSetCDROM(NULL)) != DITEM_SUCCESS
-	    || !DEVICE_INIT(mediaDevice)) {
-	    /* If we can't initialize it, it's probably not a FreeBSD CDROM so punt on it */
-	    mediaClose();
-	    if (need_eject && msgYesNo("Unable to mount the disc. Do you want to try again?") != 0)
-		return DITEM_FAILURE;
-	} else if (!file_readable("/dist/rescue/ldconfig")) {
-		mediaClose();
-		if (need_eject &&
-		    msgYesNo("Unable to find a FreeBSD live filesystem. Do you want to try again?") != 0)
-		    return DITEM_FAILURE;
-	} else
-	    break;
-	CDROMInitQuiet = 0;
-	need_eject = 1;
-    }
-    CDROMInitQuiet = 0;
-
-    if (DITEM_STATUS(fixit_livefs_common(self)) == DITEM_FAILURE)
-	return (DITEM_FAILURE);
-
-    mediaClose();
-    if (need_eject)
-	msgConfirm("Please remove the FreeBSD fixit CDROM/DVD now.");
-    return DITEM_SUCCESS;
-}
-
-int
-installFixitFloppy(dialogMenuItem *self)
-{
-    struct ufs_args args;
-    extern char *distWanted;
-
-    if (!RunningAsInit)
-	return DITEM_SUCCESS;
-
-    /* Try to open the floppy drive */
-    if (DITEM_STATUS(mediaSetFloppy(NULL)) == DITEM_FAILURE || !mediaDevice) {
-	msgConfirm("Unable to set media device to floppy.");
-	mediaClose();
-	return DITEM_FAILURE;
-    }
-
-    memset(&args, 0, sizeof(args));
-    args.fspec = mediaDevice->devname;
-    mediaDevice->private = "/mnt2";
-    distWanted = NULL;
-    Mkdir("/mnt2");
-
-    variable_set2(SYSTEM_STATE, "fixit", 0);
-
-    while (1) {
-	if (!DEVICE_INIT(mediaDevice)) {
-	    if (msgYesNo("The attempt to mount the fixit floppy failed, bad floppy\n"
-			 "or unclean filesystem.  Do you want to try again?"))
-		return DITEM_FAILURE;
-	}
-	else
-	    break;
-    }
-    if (!directory_exists("/tmp"))
-	(void)symlink("/mnt2/tmp", "/tmp");
-    fixit_common();
-    mediaClose();
-    msgConfirm("Please remove the fixit floppy now.");
-    return DITEM_SUCCESS;
-}
-
-/*
- * The common code for both fixit variants.
- */
-static void
-fixit_common(void)
-{
-    pid_t child;
-    int waitstatus;
-
-    if (!directory_exists("/var/tmp/vi.recover")) {
-	if (DITEM_STATUS(Mkdir("/var/tmp/vi.recover")) != DITEM_SUCCESS) {
-	    msgConfirm("Warning:  Was unable to create a /var/tmp/vi.recover directory.\n"
-		       "vi will kvetch and moan about it as a result but should still\n"
-		       "be essentially usable.");
-	}
-    }
-    if (!directory_exists("/bin"))
-	(void)Mkdir("/bin");
-    (void)symlink("/stand/sh", "/bin/sh");
-    /* Link the /etc/ files */
-    if (DITEM_STATUS(Mkdir("/etc")) != DITEM_SUCCESS)
-	msgConfirm("Unable to create an /etc directory!  Things are weird on this floppy..");
-    else if ((symlink("/mnt2/etc/spwd.db", "/etc/spwd.db") == -1 && errno != EEXIST) ||
-	     (symlink("/mnt2/etc/protocols", "/etc/protocols") == -1 && errno != EEXIST) ||
-	     (symlink("/mnt2/etc/group", "/etc/group") == -1 && errno != EEXIST) ||
-	     (symlink("/mnt2/etc/services", "/etc/services") == -1 && errno != EEXIST))
-	msgConfirm("Couldn't symlink the /etc/ files!  I'm not sure I like this..");
-    if (!file_readable(TERMCAP_FILE))
-	create_termcap();
-    if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
-	systemSuspendDialog();	/* must be before the fork() */
-    if (!(child = fork())) {
-	int i, fd;
-	struct termios foo;
-	extern int login_tty(int);
-
-	ioctl(0, TIOCNOTTY, NULL);
-	for (i = getdtablesize(); i >= 0; --i)
-	    close(i);
-
-	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
-	    fd = open("/dev/console", O_RDWR);
-	else
-	    fd = open("/dev/ttyv3", O_RDWR);
-	ioctl(0, TIOCSCTTY, &fd);
-	dup2(0, 1);
-	dup2(0, 2);
-	DebugFD = 2;
-	if (login_tty(fd) == -1)
-	    msgDebug("fixit: I can't set the controlling terminal.\n");
-
-	signal(SIGTTOU, SIG_IGN);
-	if (tcgetattr(0, &foo) != -1) {
-	    foo.c_cc[VERASE] = '\010';
-	    if (tcsetattr(0, TCSANOW, &foo) == -1)
-		msgDebug("fixit shell: Unable to set erase character.\n");
-	}
-	else
-	    msgDebug("fixit shell: Unable to get terminal attributes!\n");
-	setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin:/stand:"
-	       "/mnt2/stand:/mnt2/bin:/mnt2/sbin:/mnt2/usr/bin:/mnt2/usr/sbin", 1);
-	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) {
-	    printf("Waiting for fixit shell to exit.\n"
-		"When you are done, type ``exit'' to exit\n"
-		"the fixit shell and be returned here.\n\n");
-	    fflush(stdout);
-	} else {
-	    ioctl(fd, VT_ACTIVATE, 0);
-	}
-
-	/* use the .profile from the fixit medium */
-	setenv("HOME", "/mnt2", 1);
-	chdir("/mnt2");
-	execlp("sh", "-sh", (char *)0);
-	msgDebug("fixit shell: Failed to execute shell!\n");
-	_exit(1);;
-    }
-    else {
-	if (strcmp(variable_get(VAR_FIXIT_TTY), "standard") == 0) {
-	    dialog_clear_norefresh();
-	    msgNotify("Waiting for fixit shell to exit.  Go to VTY4 now by\n"
-		"typing ALT-F4.  When you are done, type ``exit'' to exit\n"
-		"the fixit shell and be returned here.\n");
-	}
-	(void)waitpid(child, &waitstatus, 0);
-	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0)
-	    systemResumeDialog();
-	else if (OnVTY) {
-	    ioctl(0, VT_ACTIVATE, 0);
-	    msgInfo(NULL);
-	}
-    }
-    dialog_clear();
-}
-
-/*
- * Some path/lib setup is required for the livefs fixit image. Since there's
- * more than one media type for livefs now, this has been broken off into it's
- * own function.
- */
-int
-fixit_livefs_common(dialogMenuItem *self)
-{
-	struct stat sb;
-
-	/*
-	 * USB and CDROM media get mounted to /dist, but fixit code looks in
-	 * /mnt2.
-	 */
-	unlink("/mnt2");
-	rmdir("/mnt2");
-
-	if (symlink("/dist", "/mnt2")) {
-		msgConfirm("Unable to symlink /mnt2 to the disc mount point.");
-		return (DITEM_FAILURE);
-	}
-
-	/*
-	 * If /tmp points to /mnt2/tmp from a previous fixit floppy session,
-	 * recreate it.
-	 */
-	if (lstat("/tmp", &sb) == 0 && (sb.st_mode & S_IFMT) == S_IFLNK)
-		unlink("/tmp");
-	Mkdir("/tmp");
-
-	/* Generate a new ld.so.hints */
-	if (!file_readable("/var/run/ld.so.hints")) {
-		Mkdir("/var/run");
-		if (vsystem("/mnt2/rescue/ldconfig -s /mnt2/lib "
-		    "/mnt2/usr/lib")) {
-			msgConfirm("Warning: ldconfig could not create the "
-			    "ld.so hints file.\nDynamic executables from the "
-			    "disc likely won't work.");
-		}
-	}
-
-	/* Create required libexec symlinks. */
-	Mkdir("/libexec");
-	if (!file_readable("/libexec/ld.so") &&
-	    file_readable("/mnt2/libexec/ld.so")) {
-		if (symlink("/mnt2/libexec/ld.so", "/libexec/ld.so"))
-			msgDebug("Couldn't link to ld.so\n");
-	}
-
-	if (!file_readable("/libexec/ld-elf.so.1")) {
-		if (symlink("/mnt2/libexec/ld-elf.so.1",
-		    "/libexec/ld-elf.so.1")) {
-			msgConfirm("Warning: could not create the symlink for "
-			    "ld-elf.so.1\nDynamic executables from the disc "
-			    "likely won't work.");
-		}
-	}
-
-	/* $PATH doesn't include /mnt2 by default. Create convenient symlink. */
-	if (!file_readable("/usr/bin/vi"))
-		symlink("/mnt2/usr/bin/vi", "/usr/bin/vi");
-
-	/* Shared code used by all fixit types. */
-	fixit_common();
-
-	return (DITEM_SUCCESS);
-}
-
-int
-installExpress(dialogMenuItem *self)
-{
-    int i;
-
-    dialog_clear_norefresh();
-    variable_set2(SYSTEM_STATE, "express", 0);
-#ifdef WITH_SLICES
-    if (DITEM_STATUS((i = diskPartitionEditor(self))) == DITEM_FAILURE)
-	return i;
-#endif
-    
-    if (DITEM_STATUS((i = diskLabelEditor(self))) == DITEM_FAILURE)
-	return i;
-
-    if (DITEM_STATUS((i = installCommit(self))) == DITEM_SUCCESS) {
-	i |= DITEM_LEAVE_MENU;
-
-	/* Give user the option of one last configuration spree */
-	installConfigure();
-    }
-    return i;
-}
-
-/* Standard mode installation */
-int
-installStandard(dialogMenuItem *self)
-{
-    int i;
-#ifdef WITH_SLICES
-    int tries = 0;
-    Device **devs;
-#endif
-
-    variable_set2(SYSTEM_STATE, "standard", 0);
-    dialog_clear_norefresh();
-#ifdef WITH_SLICES
-    msgConfirm("In the next menu, you will need to set up an MBR partitioning\n"
-	       "scheme for your hard disk.  If you simply wish to devote all disk space\n"
-	       "to FreeBSD (overwriting anything else that might be on the disk selected)\n"
-	       "then use the (A)ll command to create a single partition followed\n"
-	       "by a (Q)uit.  If you wish to allocate only free space to FreeBSD, move to a\n"
-	       "partition marked \"unused\" and use the (C)reate command.");
-
-nodisks:
-    if (DITEM_STATUS(diskPartitionEditor(self)) == DITEM_FAILURE)
-	return DITEM_FAILURE;
-
-    if (diskGetSelectCount(&devs) <= 0 && tries < 3) {
-	msgConfirm("You need to select some disks to operate on!  Be sure to use SPACE\n"
-		   "instead of RETURN in the disk selection menu when selecting a disk.");
-	++tries;
-	goto nodisks;
-    }
-
-    msgConfirm("Now you need to create BSD partitions inside of the MBR partition(s)\n"
-	       "just created.  If you have a reasonable amount of disk space (1GB or more)\n"
-	       "and don't have any special requirements, simply use the (A)uto command to\n"
-	       "allocate space automatically.  If you have more specific needs or just don't\n"
-	       "care for the layout chosen by (A)uto, press F1 for more information on\n"
-	       "manual layout.");
-#else
-    msgConfirm("First you need to create BSD partitions on the disk which you are\n"
-	       "installing to.  If you have a reasonable amount of disk space (1GB or more)\n"
-	       "and don't have any special requirements, simply use the (A)uto command to\n"
-	       "allocate space automatically.  If you have more specific needs or just don't\n"
-	       "care for the layout chosen by (A)uto, press F1 for more information on\n"
-	       "manual layout.");
-#endif
-
-    if (DITEM_STATUS(diskLabelEditor(self)) == DITEM_FAILURE)
-	return DITEM_FAILURE;
-
-    if (DITEM_STATUS((i = installCommit(self))) == DITEM_FAILURE) {
-	dialog_clear();
-	msgConfirm("Installation completed with some errors.  You may wish to\n"
-		   "scroll through the debugging messages on VTY1 with the\n"
-		   "scroll-lock feature.  You can also choose \"No\" at the next\n"
-		   "prompt and reboot and try the installation again.");
-	return i;
-
-    }
-    else {
-	dialog_clear();
-	msgConfirm("Congratulations!  You now have FreeBSD installed on your system.\n\n"
-		   "We will now move on to the final configuration questions.\n"
-		   "For any option you do not wish to configure, simply select\n"
-		   "No.\n\n"
-		   "If you wish to re-enter this utility after the system is up, you\n"
-		   "may do so by typing: /usr/sbin/sysinstall.");
-    }
-    if (mediaDevice->type != DEVICE_TYPE_FTP && mediaDevice->type != DEVICE_TYPE_NFS) {
-	if (!msgYesNo("Would you like to configure any Ethernet or PLIP network devices?")) {
-	    Device *tmp = tcpDeviceSelect();
-
-	    if (tmp && !((DevInfo *)tmp->private)->use_dhcp && !msgYesNo("Would you like to bring the %s interface up right now?", tmp->name))
-		if (!DEVICE_INIT(tmp))
-		    msgConfirm("Initialization of %s device failed.", tmp->name);
-	}
-	dialog_clear_norefresh();
-    }
-
-    if (!msgNoYes("Do you want this machine to function as a network gateway?"))
-	variable_set2("gateway_enable", "YES", 1);
-
-    dialog_clear_norefresh();
-    if (!msgNoYes("Do you want to configure inetd and the network services that it provides?"))
-        configInetd(self);
-
-    dialog_clear_norefresh();
-    if (!msgNoYes("Would you like to enable SSH login?"))
-	variable_set2("sshd_enable", "YES", 1);
-
-    dialog_clear_norefresh();
-    if (!msgNoYes("Do you want to have anonymous FTP access to this machine?"))
-	configAnonFTP(self);
-
-    dialog_clear_norefresh();
-    if (!msgNoYes("Do you want to configure this machine as an NFS server?"))
-	configNFSServer(self);
-
-    dialog_clear_norefresh();
-    if (!msgNoYes("Do you want to configure this machine as an NFS client?"))
-	variable_set2("nfs_client_enable", "YES", 1);
-
-#ifdef WITH_SYSCONS
-    dialog_clear_norefresh();
-    if (!msgNoYes("Would you like to customize your system console settings?"))
-	dmenuOpenSimple(&MenuSyscons, FALSE);
-#endif
-
-    dialog_clear_norefresh();
-    if (!msgYesNo("Would you like to set this machine's time zone now?"))
-	systemExecute("tzsetup");
-
-#ifdef WITH_MICE
-    dialog_clear_norefresh();
-    if (!msgNoYes("Does this system have a PS/2, serial, or bus mouse?"))
-	dmenuOpenSimple(&MenuMouse, FALSE);
-#endif
-
-#ifdef __i386__
-    if (checkLoaderACPI() != 0) {
-    	dialog_clear_norefresh();
-    	if (!msgNoYes("ACPI was disabled during boot.\n"
-		      "Would you like to disable it permanently?"))
-		(void)configLoaderACPI(1 /*disable*/);
-    }
-#endif
-
-    /* Now would be a good time to checkpoint the configuration data */
-    configRC_conf();
-    sync();
-
-    dialog_clear_norefresh();
-    if (!msgYesNo("The FreeBSD package collection is a collection of thousands of ready-to-run\n"
-		  "applications, from text editors to games to Web servers and more.  Would you\n"
-		  "like to browse the collection now?")) {
-	(void)configPackages(self);
-    }
-
-    if (!msgYesNo("Would you like to add any initial user accounts to the system?\n"
-		  "Adding at least one account for yourself at this stage is suggested\n"
-		  "since working as the \"root\" user is dangerous (it is easy to do\n"
-		  "things which adversely affect the entire system)."))
-	(void)configUsers(self);
-
-    msgConfirm("Now you must set the system manager's password.\n"
-	       "This is the password you'll use to log in as \"root\".");
-    if (!systemExecute("passwd root"))
-	variable_set2("root_password", "YES", 0);
-
-    /* XXX Put whatever other nice configuration questions you'd like to ask the user here XXX */
-
-    /* Give user the option of one last configuration spree */
-    dialog_clear_norefresh();
-    installConfigure();
-    return DITEM_LEAVE_MENU;
-}
-
-/* The version of commit we call from the Install Custom menu */
-int
-installCustomCommit(dialogMenuItem *self)
-{
-    int i;
-
-    i = installCommit(self);
-    if (DITEM_STATUS(i) == DITEM_SUCCESS) {
-	/* Give user the option of one last configuration spree */
-	installConfigure();
-	return i;
-    }
-    else
-	msgConfirm("The commit operation completed with errors.  Not\n"
-		   "updating /etc files.");
-    return i;
-}
-
-/*
- * What happens when we finally decide to going ahead with the installation.
- *
- * This is broken into multiple stages so that the user can do a full
- * installation but come back here again to load more distributions,
- * perhaps from a different media type.  This would allow, for
- * example, the user to load the majority of the system from CDROM and
- * then use ftp to load a different dist.
- */
-int
-installCommit(dialogMenuItem *self)
-{
-    int i;
-    char *str;
-
-    dialog_clear_norefresh();
-    if (!Dists)
-	distConfig(NULL);
-
-    if (!Dists) {
-	(void)dmenuOpenSimple(&MenuDistributions, FALSE);
-	/* select reasonable defaults if necessary */
-	if (!Dists)
-	    Dists = _DIST_USER;
-	if (!KernelDists)
-	    KernelDists = selectKernel();
-    }
-
-    if (!mediaVerify())
-	return DITEM_FAILURE;
-
-    str = variable_get(SYSTEM_STATE);
-    if (isDebug())
-	msgDebug("installCommit: System state is `%s'\n", str);
-
-    /* Installation stuff we wouldn't do to a running system */
-    if (RunningAsInit && DITEM_STATUS((i = installInitial())) == DITEM_FAILURE)
-	return i;
-
-try_media:
-    if (!DEVICE_INIT(mediaDevice)) {
-	if (!msgYesNo("Unable to initialize selected media. Would you like to\n"
-		      "adjust your media configuration and try again?")) {
-	    mediaDevice = NULL;
-	    if (!mediaVerify())
-		return DITEM_FAILURE;
-	    else
-		goto try_media;
-	}
-	else
-	    return DITEM_FAILURE;
-    }
-
-    /* Now go get it all */
-    i = distExtractAll(self);
-
-    if (i == FALSE)
-	    return DITEM_FAILURE;
-
-    /* When running as init, *now* it's safe to grab the rc.foo vars */
-    installEnvironment();
-
-    variable_set2(SYSTEM_STATE, DITEM_STATUS(i) == DITEM_FAILURE ? "error-install" : "full-install", 0);
-
-    return i;
-}
-
-static void
-installConfigure(void)
-{
-    /* Final menu of last resort */
-    if (!msgNoYes("Visit the general configuration menu for a chance to set\n"
-		  "any last options?"))
-	dmenuOpenSimple(&MenuConfigure, FALSE);
-    else
-	dmenuExit(NULL);
-
-    configRC_conf();
-    sync();
-}
-
-int
-installFixupBase(dialogMenuItem *self)
-{
-	FILE *orig, *new;
-	char buf[1024];
-	char *pos;
-#if defined(__i386__) || defined(__amd64__)
-    FILE *fp;
-#endif
-#ifdef __ia64__
-    const char *efi_mntpt;
-#endif
-
-    /* All of this is done only as init, just to be safe */
-    if (RunningAsInit) {
-#if defined(__i386__) || defined(__amd64__)
-	if ((fp = fopen("/boot/loader.conf", "a")) != NULL) {
-	    if (!OnVTY) {
-		fprintf(fp, "# -- sysinstall generated deltas -- #\n");
-		fprintf(fp, "console=\"comconsole\"\n");
-	    }
-	    fclose(fp);
-	}
-#endif
-
-	/* Fixup /etc/ttys to start a getty on the serial port.
-	  This way after a serial installation you can login via
-	  the serial port */
-
-	if (!OnVTY){
-	    if (((orig=fopen("/etc/ttys","r")) != NULL) &&
-		((new=fopen("/etc/ttys.tmp","w")) != NULL)) {
-		while (fgets(buf,sizeof(buf),orig)){
-		    if (strstr(buf,"ttyu0")){
-			if ((pos=strstr(buf,"off"))){
-			    *pos++='o';
-			    *pos++='n';
-			    *pos++=' ';
-			}
-		    }
-		    fputs(buf,new);
-		}
-		fclose(orig);
-		fclose(new);
-
-		rename("/etc/ttys.tmp","/etc/ttys");
-		unlink("/etc/ttys.tmp");
-	    }
-	}
-
-	
-	/* BOGON #2: We leave /etc in a bad state */
-	chmod("/etc", 0755);
-	
-	/* BOGON #3: No /var/db/mountdtab complains */
-	Mkdir("/var/db");
-	creat("/var/db/mountdtab", 0644);
-	
-	/* BOGON #4: /compat created by default in root fs */
-	Mkdir("/usr/compat");
-	vsystem("ln -s usr/compat /compat");
-
-	/* BOGON #5: aliases database not built for bin */
-	vsystem("newaliases");
-
-	/* BOGON #6: Remove /stand (finally) */
-	vsystem("rm -rf /stand");
-
-	/* Now run all the mtree stuff to fix things up */
-        vsystem("mtree -deU -f /etc/mtree/BSD.root.dist -p /");
-        vsystem("mtree -deU -f /etc/mtree/BSD.var.dist -p /var");
-        vsystem("mtree -deU -f /etc/mtree/BSD.usr.dist -p /usr");
-
-#ifdef __ia64__
-	/* Move /boot to the EFI partition and make /boot a link to it. */
-	efi_mntpt = (EfiChunk != NULL) ? ((PartInfo *)EfiChunk->private_data)->mountpoint : NULL;
-	if (efi_mntpt != NULL) {
-		vsystem("if [ ! -L /boot ]; then mv /boot %s; fi", efi_mntpt);
-		vsystem("if [ ! -e /boot ]; then ln -sf %s/boot /boot; fi",
-		    efi_mntpt + 1);	/* Skip leading '/' */
-		/* Make sure the kernel knows which partition is the root file system. */
-		vsystem("echo 'vfs.root.mountfrom=\"ufs:/dev/%s\"' >> /boot/loader.conf", RootChunk->name);
-	}
-#endif
-
-	/* Do all the last ugly work-arounds here */
-    }
-    return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-int
-installFixupKernel(dialogMenuItem *self, int dists)
-{
-
-    /* All of this is done only as init, just to be safe */
-    if (RunningAsInit) {
-	/*
-	 * Install something as /boot/kernel.
-	 *
-	 * NB: we assume any existing kernel has been saved
-	 *     already and the /boot/kernel we remove is empty.
-	 */
-	vsystem("rm -rf /boot/kernel");
-		vsystem("mv /boot/" GENERIC_KERNEL_NAME " /boot/kernel");
-    }
-    return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-#define	QUEUE_YES	1
-#define	QUEUE_NO	0
-static int
-performNewfs(PartInfo *pi, char *dname, int queue)
-{
-	char buffer[LINE_MAX];
-
-	if (pi->do_newfs) {
-		switch(pi->newfs_type) {
-		case NEWFS_UFS:
-			snprintf(buffer, LINE_MAX, "%s %s %s %s %s",
-			    NEWFS_UFS_CMD,
-			    pi->newfs_data.newfs_ufs.softupdates ?  "-U" : "",
-			    pi->newfs_data.newfs_ufs.ufs1 ? "-O1" : "-O2",
-			    pi->newfs_data.newfs_ufs.user_options,
-			    dname);
-			break;
-
-		case NEWFS_MSDOS:
-			snprintf(buffer, LINE_MAX, "%s %s", NEWFS_MSDOS_CMD,
-			    dname);
-			break;
-
-		case NEWFS_CUSTOM:
-			snprintf(buffer, LINE_MAX, "%s %s",
-			    pi->newfs_data.newfs_custom.command, dname);
-			break;
-		}
-
-		if (queue == QUEUE_YES) {
-			command_shell_add(pi->mountpoint, buffer);
-			return (0);
-		} else
-			return (vsystem(buffer));
-	}
-	return (0);
-}
-
-/* Go newfs and/or mount all the filesystems we've been asked to */
-int
-installFilesystems(dialogMenuItem *self)
-{
-    int i;
-    Disk *disk;
-    Chunk *c1, *c2;
-    Device **devs;
-    PartInfo *root;
-    char dname[80];
-    Boolean upgrade = FALSE;
-
-    /* If we've already done this, bail out */
-    if (!variable_cmp(DISK_LABELLED, "written"))
-	return DITEM_SUCCESS;
-
-    upgrade = !variable_cmp(SYSTEM_STATE, "upgrade");
-    if (!checkLabels(TRUE))
-	return DITEM_FAILURE;
-
-    root = (RootChunk != NULL) ? (PartInfo *)RootChunk->private_data : NULL;
-
-    command_clear();
-    if (SwapChunk && RunningAsInit) {
-	/* As the very first thing, try to get ourselves some swap space */
-	sprintf(dname, "/dev/%s", SwapChunk->name);
-	if (!Fake && !file_readable(dname)) {
-	    msgConfirm("Unable to find device node for %s in /dev!\n"
-		       "The creation of filesystems will be aborted.", dname);
-	    return DITEM_FAILURE;
-	}
-
-	if (!Fake) {
-	    if (!swapon(dname)) {
-		dialog_clear_norefresh();
-		msgNotify("Added %s as initial swap device", dname);
-	    }
-	    else {
-		msgConfirm("WARNING!  Unable to swap to %s: %s\n"
-			   "This may cause the installation to fail at some point\n"
-			   "if you don't have a lot of memory.", dname, strerror(errno));
-	    }
-	}
-    }
-
-    if (RootChunk && RunningAsInit) {
-	/* Next, create and/or mount the root device */
-	sprintf(dname, "/dev/%s", RootChunk->name);
-	if (!Fake && !file_readable(dname)) {
-	    msgConfirm("Unable to make device node for %s in /dev!\n"
-		       "The creation of filesystems will be aborted.", dname);
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-	if (strcmp(root->mountpoint, "/"))
-	    msgConfirm("Warning: %s is marked as a root partition but is mounted on %s", RootChunk->name, root->mountpoint);
-
-	if (root->do_newfs && (!upgrade ||
-	    !msgNoYes("You are upgrading - are you SURE you want to newfs "
-	    "the root partition?"))) {
-	    int i;
-
-	    dialog_clear_norefresh();
-	    msgNotify("Making a new root filesystem on %s", dname);
-	    i = performNewfs(root, dname, QUEUE_NO);
-	    if (i) {
-		msgConfirm("Unable to make new root filesystem on %s!\n"
-			   "Command returned status %d", dname, i);
-		return DITEM_FAILURE | DITEM_RESTORE;
-	    }
-	}
-	else {
-	    if (!upgrade) {
-		msgConfirm("Warning:  Using existing root partition.");
-	    }
-	    dialog_clear_norefresh();
-	    msgNotify("Checking integrity of existing %s filesystem.", dname);
-	    i = vsystem("fsck_ffs -y %s", dname);
-	    if (i)
-		msgConfirm("Warning: fsck returned status of %d for %s.\n"
-			   "This partition may be unsafe to use.", i, dname);
-	}
-
-	/*
-	 * If soft updates was enabled in the editor but we didn't newfs,
-	 * use tunefs to update the soft updates flag on the file system.
-	 */
-	if (!root->do_newfs && root->newfs_type == NEWFS_UFS &&
-	    root->newfs_data.newfs_ufs.softupdates) {
-		i = vsystem("tunefs -n enable %s", dname);
-		if (i)
-			msgConfirm("Warning: Unable to enable soft updates"
-			    " for root file system on %s", dname);
-	}
-
-	/* Switch to block device */
-	sprintf(dname, "/dev/%s", RootChunk->name);
-	if (Mount("/mnt", dname)) {
-	    msgConfirm("Unable to mount the root file system on %s!  Giving up.", dname);
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-
-	/* Mount devfs for other partitions to mount */
-	Mkdir("/mnt/dev");
-	if (!Fake) {
-	    struct iovec iov[4];
-
-	    iov[0].iov_base = "fstype";
-	    iov[0].iov_len = strlen(iov[0].iov_base) + 1;
-	    iov[1].iov_base = "devfs";
-	    iov[1].iov_len = strlen(iov[1].iov_base) + 1;
-	    iov[2].iov_base = "fspath";
-	    iov[2].iov_len = strlen(iov[2].iov_base) + 1;
-	    iov[3].iov_base = "/mnt/dev";
-	    iov[3].iov_len = strlen(iov[3].iov_base) + 1;
-	    i = nmount(iov, 4, 0);
-
-	    if (i) {
-		dialog_clear_norefresh();
-		msgConfirm("Unable to mount DEVFS (error %d)", errno);
-		return DITEM_FAILURE | DITEM_RESTORE;
-	    }
-	}
-    }
-
-    /* Now buzz through the rest of the partitions and mount them too */
-    devs = deviceFind(NULL, DEVICE_TYPE_DISK);
-    for (i = 0; devs[i]; i++) {
-	if (!devs[i]->enabled)
-	    continue;
-
-	disk = (Disk *)devs[i]->private;
-	if (!disk->chunks) {
-	    msgConfirm("No chunk list found for %s!", disk->name);
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-	for (c1 = disk->chunks->part; c1; c1 = c1->next) {
-#ifdef __ia64__
-	if (c1->type == part) {
-		c2 = c1;
-		{
-#elif defined(__powerpc__)
-	    if (c1->type == apple) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-#else
-	    if (c1->type == freebsd) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-#endif
-		    if (c2->type == part && c2->subtype != FS_SWAP && c2->private_data) {
-			PartInfo *tmp = (PartInfo *)c2->private_data;
-
-			/* Already did root */
-			if (c2 == RootChunk)
-			    continue;
-
-			sprintf(dname, "%s/dev/%s",
-			    RunningAsInit ? "/mnt" : "", c2->name);
-
-			if (tmp->do_newfs && (!upgrade ||
-			    !msgNoYes("You are upgrading - are you SURE you"
-			    " want to newfs /dev/%s?", c2->name)))
-				performNewfs(tmp, dname, QUEUE_YES);
-			else
-			    command_shell_add(tmp->mountpoint,
-				"fsck_ffs -y %s/dev/%s", RunningAsInit ?
-				"/mnt" : "", c2->name);
-#if 0
-			if (tmp->soft)
-			    command_shell_add(tmp->mountpoint,
-			    "tunefs -n enable %s/dev/%s", RunningAsInit ?
-			    "/mnt" : "", c2->name);
-#endif
-			command_func_add(tmp->mountpoint, Mount, c2->name);
-		    }
-		    else if (c2->type == part && c2->subtype == FS_SWAP) {
-			char fname[80];
-			int i;
-
-			if (c2 == SwapChunk)
-			    continue;
-			sprintf(fname, "%s/dev/%s", RunningAsInit ? "/mnt" : "", c2->name);
-			i = (Fake || swapon(fname));
-			if (!i) {
-			    dialog_clear_norefresh();
-			    msgNotify("Added %s as an additional swap device", fname);
-			}
-			else {
-			    msgConfirm("Unable to add %s as a swap device: %s", fname, strerror(errno));
-			}
-		    }
-		}
-	    }
-	    else if (c1->type == fat && c1->private_data &&
-		(root->do_newfs || upgrade)) {
-		char name[FILENAME_MAX];
-
-		sprintf(name, "%s/%s", RunningAsInit ? "/mnt" : "", ((PartInfo *)c1->private_data)->mountpoint);
-		Mkdir(name);
-	    }
-#if defined(__ia64__)
-	    else if (c1->type == efi && c1->private_data) {
-		PartInfo *pi = (PartInfo *)c1->private_data;
-
-		sprintf(dname, "%s/dev/%s", RunningAsInit ? "/mnt" : "",
-		    c1->name);
-
-		if (pi->do_newfs && (!upgrade ||
-		    !msgNoYes("You are upgrading - are you SURE you want to "
-		    "newfs /dev/%s?", c1->name)))
-			performNewfs(pi, dname, QUEUE_YES);
-
-		command_func_add(pi->mountpoint, Mount_msdosfs, c1->name);
-	    }
-#endif
-	}
-    }
-
-    command_sort();
-    command_execute();
-    dialog_clear_norefresh();
-    return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-/* Initialize various user-settable values to their defaults */
-int
-installVarDefaults(dialogMenuItem *self)
-{
-    char *cp, ncpus[10];
-
-    /* Set default startup options */
-    cp = getsysctlbyname("kern.osrelease");
-    variable_set2(VAR_RELNAME,			cp, 0);
-    free(cp);
-    variable_set2(VAR_CPIO_VERBOSITY,		"high", 0);
-    variable_set2(VAR_INSTALL_ROOT,		"/", 0);
-    variable_set2(VAR_INSTALL_CFG,		"install.cfg", 0);
-    cp = getenv("EDITOR");
-    if (!cp)
-	cp = "/usr/bin/ee";
-    variable_set2(VAR_EDITOR,			cp, 0);
-    variable_set2(VAR_FTP_USER,			"ftp", 0);
-    variable_set2(VAR_BROWSER_PACKAGE,		"links", 0);
-    variable_set2(VAR_BROWSER_BINARY,		"/usr/local/bin/links", 0);
-    variable_set2(VAR_FTP_STATE,		"passive", 0);
-    variable_set2(VAR_NFS_SECURE,		"NO", -1);
-    variable_set2(VAR_NFS_TCP,   		"NO", -1);
-    variable_set2(VAR_NFS_V3,   		"YES", -1);
-    if (OnVTY)
-	    variable_set2(VAR_FIXIT_TTY,		"standard", 0);
-    else
-	    variable_set2(VAR_FIXIT_TTY,		"serial", 0);
-    variable_set2(VAR_PKG_TMPDIR,		"/var/tmp", 0);
-    variable_set2(VAR_MEDIA_TIMEOUT,		itoa(MEDIA_TIMEOUT), 0);
-    if (!RunningAsInit)
-	variable_set2(SYSTEM_STATE,		"update", 0);
-    else
-	variable_set2(SYSTEM_STATE,		"init", 0);
-    variable_set2(VAR_NEWFS_ARGS,		"-b 16384 -f 2048", 0);
-    variable_set2(VAR_CONSTERM,                 "NO", 0);
-    if (NCpus <= 0)
-	NCpus = 1;
-    snprintf(ncpus, sizeof(ncpus), "%u", NCpus);
-    variable_set2(VAR_NCPUS,			ncpus, 0);
-    return DITEM_SUCCESS;
-}
-
-/* Load the environment up from various system configuration files */
-void
-installEnvironment(void)
-{
-    configEnvironmentRC_conf();
-    if (file_readable("/etc/resolv.conf"))
-	configEnvironmentResolv("/etc/resolv.conf");
-}
-
-/* Copy the boot floppy contents into /stand */
-Boolean
-copySelf(void)
-{
-    int i;
-
-    if (file_readable("/boot.help"))
-	vsystem("cp /boot.help /mnt");
-    msgWeHaveOutput("Copying the boot floppy to /stand on root filesystem");
-    i = vsystem("find -x /stand | cpio %s -pdum /mnt", cpioVerbosity());
-    if (i) {
-	msgConfirm("Copy returned error status of %d!", i);
-	return FALSE;
-    }
-
-    /* Copy the /etc files into their rightful place */
-    if (vsystem("cd /mnt/stand; find etc | cpio %s -pdum /mnt", cpioVerbosity())) {
-	msgConfirm("Couldn't copy up the /etc files!");
-	return TRUE;
-    }
-    return TRUE;
-}
-
-static void
-create_termcap(void)
-{
-    FILE *fp;
-
-    const char *caps[] = {
-	termcap_vt100, termcap_cons25, termcap_cons25_m, termcap_cons25r,
-	termcap_cons25r_m, termcap_cons25l1, termcap_cons25l1_m,
-	termcap_xterm, NULL,
-    };
-    const char **cp;
-
-    if (!file_readable(TERMCAP_FILE)) {
-	Mkdir("/usr/share/misc");
-	fp = fopen(TERMCAP_FILE, "w");
-	if (!fp) {
-	    msgConfirm("Unable to initialize termcap file. Some screen-oriented\nutilities may not work.");
-	    return;
-	}
-	cp = caps;
-	while (*cp)
-	    fprintf(fp, "%s\n", *(cp++));
-	fclose(fp);
-    }
-}
diff --git a/usr.sbin/sysinstall/install.cfg b/usr.sbin/sysinstall/install.cfg
deleted file mode 100644
index 4de2962f8f49..000000000000
--- a/usr.sbin/sysinstall/install.cfg
+++ /dev/null
@@ -1,101 +0,0 @@
-# This is a sample installation configuration file for my test machine,
-# crate.cdrom.com.
-# It is included here merely as a sort-of-documented example.
-#
-# $FreeBSD$
-
-# Turn on extra debugging.
-debug=yes
-
-################################
-# My host specific data
-hostname=crate.cdrom.com
-domainname=cdrom.com
-nameserver=204.216.27.3
-defaultrouter=204.216.27.228
-ipaddr=204.216.27.230
-netmask=255.255.255.240
-################################
-
-# Log to a remote syslogd server
-syslogdServer=10.0.1.1
-
-################################
-# Which installation device to use - ftp is pointed directly at my local
-# machine and the installation device is my WD8013 ethernet interface.
-# netDev can be set to ANY or a comma-delimited list of interfaces.
-_ftpPath=ftp://time.cdrom.com/pub
-netDev=ed0
-mediaSetFTP
-################################
-
-################################
-# Select which distributions we want.
-dists=base doc manpages info src sbase ssys kernels GENERIC
-distSetCustom
-################################
-
-################################
-# Now set the parameters for the partition editor on ad0.  Set to use the
-# disk exclusively (could also be "all" to use the whole disk but
-# respecting the MBR or "free" to use only unallocated space for FreeBSD). 
-disk=ad0
-partition=exclusive
-diskPartitionEditor
-
-# Uncomment this instead to use only the free space and install boot manager.
-#partition=free
-#bootManager=booteasy
-#diskPartitionEditor
-################################
-
-################################
-
-# Disk partitioning.
-# All sizes are expressed in 512 byte blocks!
-
-# A 512MB root partition
-ad0s1-1=ufs 1048576 /
-# And a 512MB swap partition
-ad0s1-2=swap 1048576 none
-# Followed by a /usr partition using all remaining space (size 0 = free space)
-# and with softupdates enabled (non-zero arg following mountpoint).
-ad0s1-3=ufs 0 /usr 1
-# Let's do it!
-diskLabelEditor
-
-################################
-
-################################
-# Now partition the 2nd disk with a 1GB /var and /usr/src using the
-# remainder of the disk.
-disk=ad1
-partition=exclusive
-diskPartitionEditor
-
-ad1s1-1=ufs 2097152 /var
-ad1s1-2=ufs 0 /usr/src
-diskLabelEditor
-################################
-
-################################
-# And the 3rd, adding a second 512MB of swap and the rest of the disk
-# for /tmp.
-disk=da0
-partition=exclusive
-diskPartitionEditor
-
-da0s1-1=swap 1048576 none
-da0s1-2=ufs 0 /tmp
-diskLabelEditor
-################################
-
-
-# OK, everything is set.  Do it!
-installCommit
-
-# Install some packages at the end.
-package=bash-3.1.17
-packageAdd
-package=ncftp-3.2.0
-packageAdd
diff --git a/usr.sbin/sysinstall/installUpgrade.c b/usr.sbin/sysinstall/installUpgrade.c
deleted file mode 100644
index 82569f633a8a..000000000000
--- a/usr.sbin/sysinstall/installUpgrade.c
+++ /dev/null
@@ -1,526 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/disklabel.h>
-#include <sys/errno.h>
-#include <sys/ioctl.h>
-#include <sys/fcntl.h>
-#include <sys/wait.h>
-#include <sys/param.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <sys/mount.h>
-
-static int installUpgradeNonInteractive(dialogMenuItem *self);
-
-typedef struct _hitList {
-    enum { JUST_COPY, CALL_HANDLER } action ;
-    char *name;
-    Boolean optional;
-    void (*handler)(struct _hitList *self);
-} HitList;
-
-/* These are the only meaningful files I know about */
-static HitList etc_files [] = {
-   { JUST_COPY,		"Xaccel.ini",		TRUE, NULL },
-   { JUST_COPY,		"X11",			TRUE, NULL },
-   { JUST_COPY,		"adduser.conf",		TRUE, NULL },
-   { JUST_COPY,		"aliases",		TRUE, NULL },
-   { JUST_COPY,		"aliases.db",		TRUE, NULL },
-   { JUST_COPY,		"amd.map",		TRUE, NULL },
-   { JUST_COPY,		"auth.conf",		TRUE, NULL },
-   { JUST_COPY,		"crontab",		TRUE, NULL },
-   { JUST_COPY,		"csh.cshrc",		TRUE, NULL },
-   { JUST_COPY,		"csh.login",		TRUE, NULL },
-   { JUST_COPY,		"csh.logout",		TRUE, NULL },
-   { JUST_COPY,		"cvsupfile",		TRUE, NULL },
-   { JUST_COPY,		"devfs.conf",		TRUE, NULL },
-   { JUST_COPY,		"dhclient.conf",	TRUE, NULL },
-   { JUST_COPY,		"disktab",		TRUE, NULL },
-   { JUST_COPY,		"dumpdates",		TRUE, NULL },
-   { JUST_COPY,		"exports",		TRUE, NULL },
-   { JUST_COPY,		"fbtab",		TRUE, NULL },
-   { JUST_COPY,		"fstab",		FALSE, NULL },
-   { JUST_COPY,		"ftpusers",		TRUE, NULL },
-   { JUST_COPY,		"gettytab",		TRUE, NULL },
-   { JUST_COPY,		"gnats",		TRUE, NULL },
-   { JUST_COPY,		"group",		FALSE, NULL },
-   { JUST_COPY,		"hosts",		TRUE, NULL },
-   { JUST_COPY,		"hosts.allow",		TRUE, NULL },
-   { JUST_COPY,		"hosts.equiv",		TRUE, NULL },
-   { JUST_COPY,		"hosts.lpd",		TRUE, NULL },
-   { JUST_COPY,		"inetd.conf",		TRUE, NULL },
-   { JUST_COPY,		"localtime",		TRUE, NULL },
-   { JUST_COPY,		"login.access",		TRUE, NULL },
-   { JUST_COPY,		"login.conf",		TRUE, NULL },
-   { JUST_COPY,		"mail",			TRUE, NULL },
-   { JUST_COPY,		"mail.rc",		TRUE, NULL },
-   { JUST_COPY,		"mac.conf",		TRUE, NULL },
-   { JUST_COPY,		"make.conf",		TRUE, NULL },
-   { JUST_COPY,		"manpath.config",	TRUE, NULL },
-   { JUST_COPY,		"master.passwd",	FALSE, NULL },
-   { JUST_COPY,		"mergemaster.rc",	TRUE, NULL },
-   { JUST_COPY,		"motd",			TRUE, NULL },
-   { JUST_COPY,		"namedb",		TRUE, NULL },
-   { JUST_COPY,		"networks",		TRUE, NULL },
-   { JUST_COPY,		"newsyslog.conf",	TRUE, NULL },
-   { JUST_COPY,		"nsmb.conf",		TRUE, NULL },
-   { JUST_COPY,		"nsswitch.conf",	TRUE, NULL },
-   { JUST_COPY,		"ntp.conf",		TRUE, NULL },
-   { JUST_COPY,		"pam.conf",		TRUE, NULL },
-   { JUST_COPY,		"passwd",		TRUE, NULL },
-   { JUST_COPY,		"periodic",		TRUE, NULL },
-   { JUST_COPY,		"pf.conf",		TRUE, NULL },
-   { JUST_COPY,		"portsnap.conf",	TRUE, NULL },
-   { JUST_COPY,		"ppp",			TRUE, NULL },
-   { JUST_COPY,		"printcap",		TRUE, NULL },
-   { JUST_COPY,		"profile",		TRUE, NULL },
-   { JUST_COPY,		"protocols",		TRUE, NULL },
-   { JUST_COPY,		"pwd.db",		TRUE, NULL },
-   { JUST_COPY,		"rc.local",		TRUE, NULL },
-   { JUST_COPY,		"rc.firewall",		TRUE, NULL },
-   { JUST_COPY,		"rc.conf.local",	TRUE, NULL },
-   { JUST_COPY,		"remote",		TRUE, NULL },
-   { JUST_COPY,		"resolv.conf",		TRUE, NULL },
-   { JUST_COPY,		"rmt",			TRUE, NULL },
-   { JUST_COPY,		"sendmail.cf",		TRUE, NULL },
-   { JUST_COPY,		"sendmail.cw",		TRUE, NULL },
-   { JUST_COPY,		"services",		TRUE, NULL },
-   { JUST_COPY,		"shells",		TRUE, NULL },
-   { JUST_COPY,		"skeykeys",		TRUE, NULL },
-   { JUST_COPY,		"snmpd.config",		TRUE, NULL },
-   { JUST_COPY,		"spwd.db",		TRUE, NULL },
-   { JUST_COPY,		"src.conf",		TRUE, NULL },
-   { JUST_COPY,		"ssh",			TRUE, NULL },
-   { JUST_COPY,		"sysctl.conf",		TRUE, NULL },
-   { JUST_COPY,		"syslog.conf",		TRUE, NULL },
-   { JUST_COPY,		"ttys",			TRUE, NULL },
-   { 0,			NULL,			FALSE, NULL },
-};
-
-static void
-traverseHitlist(HitList *h)
-{
-    system("rm -rf /etc/upgrade");
-    Mkdir("/etc/upgrade");
-    while (h->name) {
-	if (!file_readable(h->name)) {
-	    if (!h->optional)
-		msgConfirm("Unable to find an old /etc/%s file!  That is decidedly non-standard and\n"
-			   "your upgraded system may function a little strangely as a result.", h->name);
-	}
-	else {
-	    if (h->action == JUST_COPY) {
-		/* Move the just-loaded copy aside */
-		vsystem("mv /etc/%s /etc/upgrade/%s", h->name, h->name);
-
-		/* Copy the old one into its place */
-		msgNotify("Resurrecting %s..", h->name);
-		/* Do this with tar so that symlinks and such are preserved */
-		if (vsystem("tar cf - %s | tar xpf - -C /etc", h->name))
-		    msgConfirm("Unable to resurrect your old /etc/%s!  Hmmmm.", h->name);
-	    }
-	    else /* call handler */
-		h->handler(h);
-	}
-	++h;
-    }
-}
-
-int
-installUpgrade(dialogMenuItem *self)
-{
-    char saved_etc[FILENAME_MAX];
-    Boolean extractingBin = TRUE;
-
-    if (variable_get(VAR_NONINTERACTIVE))
-	return installUpgradeNonInteractive(self);
-
-    variable_set2(SYSTEM_STATE, "upgrade", 0);
-    dialog_clear();
-
-    if (msgYesNo("Before beginning a binary upgrade, please review the upgrade instructions,\n"
-		 "which are located in the \"Install\" document under the main documentation\n"
-		 "menu.  Given that you have read these instructions and understand the risks\n"
-		 "and precautions involved, are you sure that you want to proceed with\n"
-		 "this upgrade?") != 0)
-	return DITEM_FAILURE;
-
-    if (!Dists) {
-	msgConfirm("First, you must select some distribution components.  The upgrade procedure\n"
-		   "will only upgrade the distributions you select in the next set of menus.");
-	if (!dmenuOpenSimple(&MenuDistributions, FALSE) || !Dists)
-	    return DITEM_FAILURE;
-    }
-    else if (!(Dists & DIST_BASE)) {	    /* No base selected?  Not much of an upgrade.. */
-	if (msgYesNo("You didn't select the base distribution as one of the distributons to load.\n"
-		     "This one is pretty vital to a successful upgrade.  Are you SURE you don't\n"
-		     "want to select the base distribution?  Chose No to bring up the Distributions\n"
-		     "menu again.") != 0) {
-	    if (!dmenuOpenSimple(&MenuDistributions, FALSE))
-		return DITEM_FAILURE;
-	}
-    }
-
-    /* Still?!  OK!  They must know what they're doing.. */
-    if (!(Dists & DIST_BASE))
-	extractingBin = FALSE;
-
-    if (RunningAsInit) {
-	Device **devs;
-	int i, cnt;
-	char *cp;
-
-	cp = variable_get(VAR_DISK);
-	devs = deviceFind(cp, DEVICE_TYPE_DISK);
-	cnt = deviceCount(devs);
-	if (!cnt) {
-	    msgConfirm("No disks found!  Please verify that your disk controller is being\n"
-		       "properly probed at boot time.  See the Hardware Guide on the\n"
-		       "Documentation menu for clues on diagnosing this type of problem.");
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-	else {
-	    /* Enable all the drives before we start */
-	    for (i = 0; i < cnt; i++)
-		devs[i]->enabled = TRUE;
-	}
-
-	msgConfirm("OK.  First, we're going to go to the disk label editor.  In this editor\n"
-		   "you will be expected to Mount any partitions you're interested in\n"
-		   "upgrading.  DO NOT set the Newfs flag to Y on anything in the label editor\n"
-		   "unless you're absolutely sure you know what you're doing!  In this\n"
-		   "instance, you'll be using the label editor as little more than a fancy\n"
-		   "screen-oriented partition mounting tool.\n\n"
-		   "Once you're done in the label editor, press Q to return here for the next\n"
-		   "step.");
-
-	if (DITEM_STATUS(diskLabelEditor(self)) == DITEM_FAILURE) {
-	    msgConfirm("The disk label editor returned an error status.  Upgrade operation\n"
-		       "aborted.");
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-
-	/* Don't write out MBR info */
-	variable_set2(DISK_PARTITIONED, "written", 0);
-	if (DITEM_STATUS(diskLabelCommit(self)) == DITEM_FAILURE) {
-	    msgConfirm("Not all file systems were properly mounted.  Upgrade operation\n"
-		       "aborted.");
-	    variable_unset(DISK_PARTITIONED);
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-
-	msgNotify("Updating /stand on root filesystem");
-	(void)vsystem("find -x /stand | cpio %s -pdum /mnt", cpioVerbosity());
-
-	if (DITEM_STATUS(chroot("/mnt")) == DITEM_FAILURE) {
-	    msgConfirm("Unable to chroot to /mnt - something is wrong with the\n"
-		       "root partition or the way it's mounted if this doesn't work.");
-	    variable_unset(DISK_PARTITIONED);
-	    return DITEM_FAILURE | DITEM_RESTORE;
-	}
-	chdir("/");
-	installEnvironment();
-	systemCreateHoloshell();
-    }
-
-    saved_etc[0] = '\0';
-
-    /* Don't allow sources to be upgraded if we have src already */
-    if (directory_exists("/usr/src/") && (Dists & DIST_SRC)) {
-	Dists &= ~DIST_SRC;
-	SrcDists = 0;
-	msgConfirm("Warning: /usr/src exists and sources were selected as upgrade\n"
-		   "targets.  Unfortunately, this is not the way to upgrade your\n"
-		   "sources - please use CTM or CVSup or some other method which\n"
-		   "handles ``deletion events'', unlike this particular feature.\n\n"
-		   "Your existing /usr/src will not be affected by this upgrade.\n");
-    }
-
-    if (extractingBin) {
-	while (!*saved_etc) {
-	    char *cp = msgGetInput("/var/tmp/etc", "Under which directory do you wish to save your current /etc?");
-
-	    if (!cp || !*cp || Mkdir(cp)) {
-		if (msgYesNo("Directory was not specified, was invalid or user selected Cancel.\n\n"
-			     "Doing an upgrade without first backing up your /etc directory is a very\n"
-			     "bad idea!  Do you want to go back and specify the save directory again?") != 0)
-		    break;
-	    }
-	    else {
-		SAFE_STRCPY(saved_etc, cp);
-	    }
-	}
-
-	if (saved_etc[0]) {
-	    msgNotify("Preserving /etc directory..");
-	    if (vsystem("tar -cBpf - -C /etc . | tar --unlink -xBpf - -C %s", saved_etc))
-		if (msgYesNo("Unable to backup your /etc into %s.\n"
-			     "Do you want to continue anyway?", saved_etc) != 0)
-		    return DITEM_FAILURE;
-	    msgNotify("Preserving /root directory..");
-	    vsystem("tar -cBpf - -C / root | tar --unlink -xBpf - -C %s", saved_etc);
-	}
-
-	msgNotify("chflags'ing old binaries - please wait.");
-	(void)vsystem("chflags -R noschg /bin /sbin /lib /libexec /usr/bin /usr/sbin /usr/lib /usr/libexec /var/empty /boot/kernel*");
-
-	if (directory_exists("/boot/kernel")) {
-	    if (directory_exists("/boot/kernel.prev")) {
-		msgNotify("Removing /boot/kernel.prev");
-		if (system("rm -fr /boot/kernel.prev")) {
-		    msgConfirm("NOTICE: I'm trying to back up /boot/kernel to\n"
-			       "/boot/kernel.prev, but /boot/kernel.prev exists and I\n"
-			       "can't remove it.  This means that the backup will, in\n"
-			       "all probability, fail.");
-		}
-	    }
-	    msgNotify("Moving old kernel to /boot/kernel.prev");
-	    if (system("mv /boot/kernel /boot/kernel.prev")) {
-		if (!msgYesNo("Hmmm!  I couldn't move the old kernel over!  Do you want to\n"
-			      "treat this as a big problem and abort the upgrade?  Due to the\n"
-			      "way that this upgrade process works, you will have to reboot\n"
-			      "and start over from the beginning.  Select Yes to reboot now"))
-		    systemShutdown(1);
-	    }
-	    else 
-		msgConfirm("NOTICE: Your old kernel is in /boot/kernel.prev should this\n"
-			   "upgrade fail for any reason and you need to boot your old\n"
-			   "kernel.");
-	}
-    }
-
-media:
-    /* We do this very late, but we unfortunately need to back up /etc first */
-    if (!mediaVerify())
-	return DITEM_FAILURE;
-
-    if (!DEVICE_INIT(mediaDevice)) {
-	if (!msgYesNo("Couldn't initialize the media.  Would you like\n"
-		   "to adjust your media selection and try again?")) {
-	    mediaDevice = NULL;
-	    goto media;
-	}
-	else
-	    return DITEM_FAILURE | DITEM_REDRAW | DITEM_RESTORE;
-    }
-    
-    msgNotify("Beginning extraction of distributions.");
-    if (DITEM_STATUS(distExtractAll(self)) == DITEM_FAILURE) {
-	msgConfirm("Hmmmm.  We couldn't even extract the base distribution.  This upgrade\n"
-		   "should be considered a failure and started from the beginning, sorry!\n"
-		   "The system will reboot now.");
-	dialog_clear();
-	systemShutdown(1);
-    }
-    else if (Dists) {
-	if (!extractingBin || !(Dists & DIST_BASE)) {
-	    msgNotify("The extraction process seems to have had some problems, but we got most\n"
-		       "of the essentials.  We'll treat this as a warning since it may have been\n"
-		       "only non-essential distributions which failed to load.");
-	}
-	else {
-	    msgConfirm("Hmmmm.  We couldn't even extract the base distribution.  This upgrade\n"
-		       "should be considered a failure and started from the beginning, sorry!\n"
-		       "The system will reboot now.");
-	    dialog_clear();
-	    systemShutdown(1);
-	}
-    }
-
-    if (extractingBin)
-	vsystem("disklabel -B `awk '$2~/\\/$/ {print substr($1, 6, 5)}' /etc/fstab`");
-    msgNotify("First stage of upgrade completed successfully!\n\n"
-	       "Next comes stage 2, where we attempt to resurrect your /etc\n"
-	       "directory!");
-
-    if (chdir(saved_etc)) {
-	msgConfirm("Unable to go to your saved /etc directory in %s?!  Argh!\n"
-		   "Something went seriously wrong!  It's quite possible that\n"
-		   "your former /etc is toast.  I hope you didn't have any\n"
-		   "important customizations you wanted to keep in there.. :(", saved_etc);
-    }
-    else {
-	/* Now try to resurrect the /etc files */
-	traverseHitlist(etc_files);
-	/* Resurrect the root dotfiles */
-	vsystem("tar -cBpf - root | tar -xBpf - -C / && rm -rf root");
-    }
-
-    msgConfirm("Upgrade completed!  All of your old /etc files have been restored.\n"
-	       "For your reference, the new /etc files are in /etc/upgrade/ in case\n"
-	       "you wish to upgrade these files by hand (though that should not be\n"
-	       "strictly necessary).  If your root partition is specified in /etc/fstab\n"
-	       "using the old \"compatibility\" slice, you may also wish to update it to\n"
-	       "use a fully qualified slice name in order to avoid warnings on startup.\n\n"
-	       "When you're ready to reboot into the new system, simply exit the installation.");
-    return DITEM_SUCCESS | DITEM_REDRAW | DITEM_RESTORE;
-}
-
-static int
-installUpgradeNonInteractive(dialogMenuItem *self)
-{
-    char *saved_etc;
-    Boolean extractingBin = TRUE;
-
-    variable_set2(SYSTEM_STATE, "upgrade", 0);
-
-    /* Make sure at least BIN is selected */
-    Dists |= DIST_BASE;
-
-    if (RunningAsInit) {
-	Device **devs;
-	int i, cnt;
-	char *cp;
-
-	cp = variable_get(VAR_DISK);
-	devs = deviceFind(cp, DEVICE_TYPE_DISK);
-	cnt = deviceCount(devs);
-	if (!cnt) {
-	    msgConfirm("No disks found!  Please verify that your disk controller is being\n"
-		       "properly probed at boot time.  See the Hardware Guide on the\n"
-		       "Documentation menu for clues on diagnosing this type of problem.");
-	    return DITEM_FAILURE;
-	}
-	else {
-	    /* Enable all the drives before we start */
-	    for (i = 0; i < cnt; i++)
-		devs[i]->enabled = TRUE;
-	}
-
-	msgConfirm("OK.  First, we're going to go to the disk label editor.  In this editor\n"
-		   "you will be expected to Mount any partitions you're interested in\n"
-		   "upgrading.  DO NOT set the Newfs flag to Y on anything in the label editor\n"
-		   "unless you're absolutely sure you know what you're doing!  In this\n"
-		   "instance, you'll be using the label editor as little more than a fancy\n"
-		   "screen-oriented partition mounting tool.\n\n"
-		   "Once you're done in the label editor, press Q to return here for the next\n"
-		   "step.");
-
-	if (DITEM_STATUS(diskLabelEditor(self)) == DITEM_FAILURE) {
-	    msgConfirm("The disk label editor returned an error status.  Upgrade operation\n"
-		       "aborted.");
-	    return DITEM_FAILURE;
-	}
-
-	/* Don't write out MBR info */
-	variable_set2(DISK_PARTITIONED, "written", 0);
-	if (DITEM_STATUS(diskLabelCommit(self)) == DITEM_FAILURE) {
-	    msgConfirm("Not all file systems were properly mounted.  Upgrade operation\n"
-		       "aborted.");
-	    variable_unset(DISK_PARTITIONED);
-	    return DITEM_FAILURE;
-	}
-
-	if (extractingBin) {
-	    msgNotify("chflags'ing old binaries - please wait.");
-	    (void)vsystem("chflags -R noschg /mnt/");
-	}
-	msgNotify("Updating /stand on root filesystem");
-	(void)vsystem("find -x /stand | cpio %s -pdum /mnt", cpioVerbosity());
-
-	if (DITEM_STATUS(chroot("/mnt")) == DITEM_FAILURE) {
-	    msgConfirm("Unable to chroot to /mnt - something is wrong with the\n"
-		       "root partition or the way it's mounted if this doesn't work.");
-	    variable_unset(DISK_PARTITIONED);
-	    return DITEM_FAILURE;
-	}
-	chdir("/");
-	systemCreateHoloshell();
-    }
-
-    if (!mediaVerify() || !DEVICE_INIT(mediaDevice)) {
-	msgNotify("Upgrade: Couldn't initialize media.");
-	return DITEM_FAILURE;
-    }
-
-    saved_etc = "/var/tmp/etc";
-    Mkdir(saved_etc);
-    msgNotify("Preserving /etc directory..");
-    if (vsystem("tar -cpBf - -C /etc . | tar -xpBf - -C %s", saved_etc)) {
-	msgNotify("Unable to backup your /etc into %s.", saved_etc);
-	return DITEM_FAILURE;
-    }
-
-    /*
-     * Back up the old kernel, leaving it in place in case we
-     *  crash and reboot.
-     */
-    if (directory_exists("/boot/kernel")) {
-	if (directory_exists("/boot/kernel.prev")) {
-	    msgNotify("Removing /boot/kernel.prev");
-	    if (system("rm -fr /boot/kernel.prev")) {
-		msgConfirm("NOTICE: I'm trying to back up /boot/kernel to\n"
-		    "/boot/kernel.prev, but /boot/kernel.prev exists and I\n"
-		    "can't remove it.  This means that the backup will, in\n"
-		    "all probability, fail.");
-	    }
-	}
-	msgNotify("Copying old kernel to /boot/kernel.prev");
-	vsystem("cp -Rp /boot/kernel /boot/kernel.prev");
-    }   
-
-    msgNotify("Beginning extraction of distributions.");
-    if (DITEM_STATUS(distExtractAll(self)) == DITEM_FAILURE) {
-	msgConfirm("Hmmmm.  We couldn't even extract the base distribution.  This upgrade\n"
-		   "should be considered a failure and started from the beginning, sorry!\n"
-		   "The system will reboot now.");
-	dialog_clear();
-	systemShutdown(1);
-    }
-    else if (Dists) {
-	if (!(Dists & DIST_BASE)) {
-	    msgNotify("The extraction process seems to have had some problems, but we got most\n"
-		       "of the essentials.  We'll treat this as a warning since it may have been\n"
-		       "only non-essential distributions which failed to upgrade.");
-	}
-	else {
-	    msgConfirm("Hmmmm.  We couldn't even extract the base distribution.  This upgrade\n"
-		       "should be considered a failure and started from the beginning, sorry!\n"
-		       "The system will reboot now.");
-	    dialog_clear();
-	    systemShutdown(1);
-	}
-    }
-
-    msgNotify("First stage of upgrade completed successfully.");
-    if (vsystem("tar -cpBf - -C %s . | tar --unlink -xpBf - -C /etc", saved_etc)) {
-	msgNotify("Unable to resurrect your old /etc!");
-	return DITEM_FAILURE;
-    }
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
diff --git a/usr.sbin/sysinstall/keymap.c b/usr.sbin/sysinstall/keymap.c
deleted file mode 100644
index 21b5e0943b9e..000000000000
--- a/usr.sbin/sysinstall/keymap.c
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Copyright (c) 1996 Joerg Wunsch
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- *
- */
-
-#include "sysinstall.h"
-#include <errno.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <sys/kbio.h>
-
-struct keymapInfo {
-    const char *name;
-    const struct keymap *map;
-};
-
-#include "keymap.h"
-
-/*
- * keymap.h is being automatically generated by the Makefile.  It
- * contains definitions for all desired keymaps.  Note that since we
- * don't support font loading nor screen mapping during installation,
- * we simply don't care for any other keys than the ASCII subset.
- *
- * Therefore, if no keymap with the exact name has been found in the
- * first pass, we make a second pass over the table looking just for
- * the language name only.
- */
-
-#ifdef WITH_SYSCONS
-static int
-keymapSetDefault(const char *prefix)
-{
-    dialogMenuItem *items = MenuSysconsKeymap.items;
-    int i;
-    size_t plen = strlen(prefix);
-
-    for (i = 0; items[i].data; ++i)
-	if (!strncmp(prefix, items[i].data, plen))
-	    return i;
-
-    return -1;
-}
-
-int
-keymapMenuSelect(dialogMenuItem *self)
-{
-    static const struct {
-	const char *country, *lang;
-    } map[] = {
-	{"dk", "danish"},
-	{"ee", "estonian"},
-	{"fi", "finnish"},
-	{"de", "german"},
-	{"is", "icelandic"},
-	{"no", "norwegian"},
-	{"pl", "pl_PL"},
-	{"es", "spanish"},
-	{"se", "swedish"},
-	{"ch", "swiss"},
-	{"gb", "uk"},
-	{"gg", "uk"},
-	{"ie", "uk"},
-	{"im", "uk"},
-	{"je", "uk"},
-	{NULL, NULL}
-    };
-    const char *country, *lang;
-    int i;
-    int choice, scroll, curr, max;
-    char prefix[16 + 1];
-
-    if ((country = variable_get(VAR_COUNTRY)) != NULL)
-    {
-	lang = country;
-	for (i = 0; map[i].country; ++i)
-	    if (!strcmp(country, map[i].country))
-	    {
-		lang = map[i].lang;
-		break;
-	    }
-
-	snprintf(prefix, sizeof(prefix), "keymap=%s.iso", lang);
-	if ((choice = keymapSetDefault(prefix)) == -1)
-	{
-	    snprintf(prefix, sizeof(prefix), "keymap=%s", lang);
-	    if ((choice = keymapSetDefault(prefix)) == -1) {
-#ifdef PC98
-		    snprintf(prefix, sizeof(prefix), "keymap=jp.pc98");
-#else
-		    snprintf(prefix, sizeof(prefix), "keymap=us.iso");
-#endif
-		    if ((choice = keymapSetDefault(prefix)) == -1)
-			    choice = 0;
-	    }
-	}
-
-	dmenuSetDefaultIndex(&MenuSysconsKeymap, &choice, &scroll, &curr, &max);
-	return dmenuOpen(&MenuSysconsKeymap, &choice, &scroll, &curr, &max, FALSE);
-    }
-    else
-	return dmenuOpenSimple(&MenuSysconsKeymap, FALSE) ? DITEM_SUCCESS :
-	    DITEM_FAILURE;
-}
-#endif
-
-/*
- * Return values:
- *
- *  0: OK
- * -1: no appropriate keymap found
- * -2: error installing map (other than ENXIO which means we're not on syscons)
- */
-
-int
-loadKeymap(const char *lang)
-{
-    int passno, err;
-    char *llang;
-    size_t l;
-    struct keymapInfo *kip;
-
-    llang = strdup(lang);
-    if (llang == NULL)
-	abort();
-
-    for (passno = 0; passno < 2; passno++)
-    {
-	if (passno > 0) 
-	{
-	    /* make the match more fuzzy */
-	    l = strspn(llang, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
-	    llang[l] = '\0';
-	}
-
-	l = strlen(llang);
-
-	for (kip = keymapInfos; kip->name; kip++)
-	    if (strncmp(kip->name, llang, l) == 0)
-	    {
-		/* Yep, got it! */
-		err = ioctl(0, PIO_KEYMAP, kip->map);
-		free(llang);
-		return (err == -1 && errno != ENOTTY)? -2: 0;
-	    }
-    }
-    free(llang);
-    return -1;
-}
diff --git a/usr.sbin/sysinstall/label.c b/usr.sbin/sysinstall/label.c
deleted file mode 100644
index 55132b6d2264..000000000000
--- a/usr.sbin/sysinstall/label.c
+++ /dev/null
@@ -1,1694 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <inttypes.h>
-#include <libdisk.h>
-#include <sys/disklabel.h>
-#include <sys/param.h>
-#include <sys/sysctl.h>
-
-#define AUTO_HOME	0	/* do not create /home automatically */
-
-/*
- * Everything to do with editing the contents of disk labels.
- */
-
-/* A nice message we use a lot in the disklabel editor */
-#define MSG_NOT_APPLICABLE	"That option is not applicable here"
-
-/* Where to start printing the freebsd slices */
-#define CHUNK_SLICE_START_ROW		2
-#define CHUNK_PART_START_ROW		11
-
-/* The smallest filesystem we're willing to create */
-#define FS_MIN_SIZE			ONE_MEG
-
-/*
- * Minimum partition sizes
- */
-#if defined(__ia64__) || defined(__sparc64__) || defined(__amd64__)
-#define ROOT_MIN_SIZE			280
-#else
-#define ROOT_MIN_SIZE			180
-#endif
-#define SWAP_MIN_SIZE			32
-#define USR_MIN_SIZE			160
-#define VAR_MIN_SIZE			20
-#define TMP_MIN_SIZE			20
-#define HOME_MIN_SIZE			20
-
-/*
- * Swap size limit for auto-partitioning (4G).
- */
-#define SWAP_AUTO_LIMIT_SIZE		4096
-
-/*
- * Default partition sizes.  If we do not have sufficient disk space
- * for this configuration we scale things relative to the NOM vs DEFAULT
- * sizes.  If the disk is larger then /home will get any remaining space.
- */
-#define ROOT_DEFAULT_SIZE		1024
-#define USR_DEFAULT_SIZE		8192
-#define VAR_DEFAULT_SIZE		4096
-#define TMP_DEFAULT_SIZE		1024
-#define HOME_DEFAULT_SIZE		USR_DEFAULT_SIZE
-
-/*
- * Nominal partition sizes.  These are used to scale the default sizes down
- * when we have insufficient disk space.  If this isn't sufficient we scale
- * down using the MIN sizes instead.
- */
-#define ROOT_NOMINAL_SIZE		512
-#define USR_NOMINAL_SIZE		1536
-#define VAR_NOMINAL_SIZE		512
-#define TMP_NOMINAL_SIZE		128
-#define HOME_NOMINAL_SIZE		USR_NOMINAL_SIZE
-
-/* The bottom-most row we're allowed to scribble on */
-#define CHUNK_ROW_MAX			16
-
-
-/* All the chunks currently displayed on the screen */
-static struct {
-    struct chunk *c;
-    PartType type;
-} label_chunk_info[MAX_CHUNKS + 1];
-static int here;
-
-/*** with this value we try to track the most recently added label ***/
-static int label_focus = 0, pslice_focus = 0;
-
-static int diskLabel(Device *dev);
-static int diskLabelNonInteractive(Device *dev);
-static char *try_auto_label(Device **devs, Device *dev, int perc, int *req);
-
-static int
-labelHook(dialogMenuItem *selected)
-{
-    Device **devs = NULL;
-
-    devs = deviceFind(selected->prompt, DEVICE_TYPE_DISK);
-    if (!devs) {
-	msgConfirm("Unable to find disk %s!", selected->prompt);
-	return DITEM_FAILURE;
-    }
-    /* Toggle enabled status? */
-    if (!devs[0]->enabled) {
-	devs[0]->enabled = TRUE;
-	diskLabel(devs[0]);
-    }
-    else
-	devs[0]->enabled = FALSE;
-    return DITEM_SUCCESS;
-}
-
-static int
-labelCheck(dialogMenuItem *selected)
-{
-    Device **devs = NULL;
-
-    devs = deviceFind(selected->prompt, DEVICE_TYPE_DISK);
-    if (!devs || devs[0]->enabled == FALSE)
-	return FALSE;
-    return TRUE;
-}
-
-int
-diskLabelEditor(dialogMenuItem *self)
-{
-    DMenu *menu;
-    Device **devs;
-    int i, cnt;
-
-    i = 0;
-    cnt = diskGetSelectCount(&devs);
-    if (cnt == -1) {
-	msgConfirm("No disks found!  Please verify that your disk controller is being\n"
-		   "properly probed at boot time.  See the Hardware Guide on the\n"
-		   "Documentation menu for clues on diagnosing this type of problem.");
-	return DITEM_FAILURE;
-    }
-    else if (cnt) {
-	/* Some are already selected */
-	if (variable_get(VAR_NONINTERACTIVE) &&
-	  !variable_get(VAR_DISKINTERACTIVE))
-	    i = diskLabelNonInteractive(NULL);
-	else
-	    i = diskLabel(NULL);
-    }
-    else {
-	/* No disks are selected, fall-back case now */
-	cnt = deviceCount(devs);
-	if (cnt == 1) {
-	    devs[0]->enabled = TRUE;
-	    if (variable_get(VAR_NONINTERACTIVE) &&
-	      !variable_get(VAR_DISKINTERACTIVE))
-		i = diskLabelNonInteractive(devs[0]);
-	    else
-		i = diskLabel(devs[0]);
-	}
-	else {
-	    menu = deviceCreateMenu(&MenuDiskDevices, DEVICE_TYPE_DISK, labelHook, labelCheck);
-	    if (!menu) {
-		msgConfirm("No devices suitable for installation found!\n\n"
-			   "Please verify that your disk controller (and attached drives)\n"
-			   "were detected properly.  This can be done by pressing the\n"
-			   "[Scroll Lock] key and using the Arrow keys to move back to\n"
-			   "the boot messages.  Press [Scroll Lock] again to return.");
-		i = DITEM_FAILURE;
-	    }
-	    else {
-		i = dmenuOpenSimple(menu, FALSE) ? DITEM_SUCCESS : DITEM_FAILURE;
-		free(menu);
-	    }
-	}
-    }
-    if (DITEM_STATUS(i) != DITEM_FAILURE) {
-	if (variable_cmp(DISK_LABELLED, "written"))
-	    variable_set2(DISK_LABELLED, "yes", 0);
-    }
-    return i;
-}
-
-int
-diskLabelCommit(dialogMenuItem *self)
-{
-    char *cp;
-    int i;
-
-    /* Already done? */
-    if ((cp = variable_get(DISK_LABELLED)) && strcmp(cp, "yes"))
-	i = DITEM_SUCCESS;
-    else if (!cp) {
-	msgConfirm("You must assign disk labels before this option can be used.");
-	i = DITEM_FAILURE;
-    }
-    /* The routine will guard against redundant writes, just as this one does */
-    else if (DITEM_STATUS(diskPartitionWrite(self)) != DITEM_SUCCESS)
-	i = DITEM_FAILURE;
-    else if (DITEM_STATUS(installFilesystems(self)) != DITEM_SUCCESS)
-	i = DITEM_FAILURE;
-    else {
-	msgInfo("All filesystem information written successfully.");
-	variable_set2(DISK_LABELLED, "written", 0);
-	i = DITEM_SUCCESS;
-    }
-    return i;
-}
-
-/* See if we're already using a desired partition name */
-static Boolean
-check_conflict(char *name)
-{
-    int i;
-
-    for (i = 0; label_chunk_info[i].c; i++)
-	if ((label_chunk_info[i].type == PART_FILESYSTEM || label_chunk_info[i].type == PART_FAT
-	    || label_chunk_info[i].type == PART_EFI) && label_chunk_info[i].c->private_data
-	    && !strcmp(((PartInfo *)label_chunk_info[i].c->private_data)->mountpoint, name))
-	    return TRUE;
-    return FALSE;
-}
-
-/* How much space is in this FreeBSD slice? */
-static daddr_t
-space_free(struct chunk *c)
-{
-    struct chunk *c1;
-    daddr_t sz = c->size;
-
-    for (c1 = c->part; c1; c1 = c1->next) {
-	if (c1->type != unused)
-	    sz -= c1->size;
-    }
-    if (sz < 0)
-	msgFatal("Partitions are larger than actual chunk??");
-    return sz;
-}
-
-/* Snapshot the current situation into the displayed chunks structure */
-static void
-record_label_chunks(Device **devs, Device *dev)
-{
-    int i, j, p;
-    struct chunk *c1, *c2;
-    Disk *d;
-
-    j = p = 0;
-    /* First buzz through and pick up the FreeBSD slices */
-    for (i = 0; devs[i]; i++) {
-	if ((dev && devs[i] != dev) || !devs[i]->enabled)
-	    continue;
-	d = (Disk *)devs[i]->private;
-	if (!d->chunks)
-	    msgFatal("No chunk list found for %s!", d->name);
-
-#ifdef __ia64__
-	label_chunk_info[j].type = PART_SLICE;
-	label_chunk_info[j].c = d->chunks;
-	j++;
-#endif
-
-	/* Put the slice entries first */
-	for (c1 = d->chunks->part; c1; c1 = c1->next) {
-	    if (c1->type == freebsd) {
-		label_chunk_info[j].type = PART_SLICE;
-		label_chunk_info[j].c = c1;
-		++j;
-	    }
-#ifdef __powerpc__
-	    if (c1->type == apple) {
-    	        label_chunk_info[j].type = PART_SLICE;
-		label_chunk_info[j].c = c1;
-		++j;
-	    }
-#endif
-	}
-    }
-
-    /* Now run through again and get the FreeBSD partition entries */
-    for (i = 0; devs[i]; i++) {
-	if (!devs[i]->enabled)
-	    continue;
-	d = (Disk *)devs[i]->private;
-	/* Then buzz through and pick up the partitions */
-	for (c1 = d->chunks->part; c1; c1 = c1->next) {
-	    if (c1->type == freebsd) {
-		for (c2 = c1->part; c2; c2 = c2->next) {
-		    if (c2->type == part) {
-			if (c2->subtype == FS_SWAP)
-			    label_chunk_info[j].type = PART_SWAP;
-			else
-			    label_chunk_info[j].type = PART_FILESYSTEM;
-			label_chunk_info[j].c = c2;
-			++j;
-		    }
-		}
-	    }
-	    else if (c1->type == fat) {
-		label_chunk_info[j].type = PART_FAT;
-		label_chunk_info[j].c = c1;
-		++j;
-	    }
-#ifdef __ia64__
-	    else if (c1->type == efi) {
-		label_chunk_info[j].type = PART_EFI;
-		label_chunk_info[j].c = c1;
-		++j;
-	    }
-	    else if (c1->type == part) {
-		if (c1->subtype == FS_SWAP)
-		    label_chunk_info[j].type = PART_SWAP;
-		else
-		    label_chunk_info[j].type = PART_FILESYSTEM;
-		label_chunk_info[j].c = c1;
-		++j;
-	    }
-#endif
-#ifdef __powerpc__
-	    else if (c1->type == apple) {
-	        for (c2 = c1->part; c2; c2 = c2->next) {
-		    if (c2->type == part) {
-		        if (c2->subtype == FS_SWAP)
-			    label_chunk_info[j].type = PART_SWAP;
-			else
-			    label_chunk_info[j].type = PART_FILESYSTEM;
-			label_chunk_info[j].c = c2;
-			++j;
-		    }
-		}
-	    }
-#endif
-	}
-    }
-    label_chunk_info[j].c = NULL;
-    if (here >= j) {
-	here = j  ? j - 1 : 0;
-    }
-}
-
-/* A new partition entry */
-static PartInfo *
-new_part(PartType type, char *mpoint, Boolean newfs)
-{
-    PartInfo *pi;
-
-    if (!mpoint)
-	mpoint = (type == PART_EFI) ? "/efi" : "/change_me";
-
-    pi = (PartInfo *)safe_malloc(sizeof(PartInfo));
-    sstrncpy(pi->mountpoint, mpoint, FILENAME_MAX);
-
-    pi->do_newfs = newfs;
-
-    if (type == PART_EFI) {
-	pi->newfs_type = NEWFS_MSDOS;
-    } else {
-	pi->newfs_type = NEWFS_UFS;
-	strcpy(pi->newfs_data.newfs_ufs.user_options, "");
-	pi->newfs_data.newfs_ufs.acls = FALSE;
-	pi->newfs_data.newfs_ufs.multilabel = FALSE;
-	pi->newfs_data.newfs_ufs.softupdates = strcmp(mpoint, "/");
-	pi->newfs_data.newfs_ufs.ufs1 = FALSE;
-    }
-
-    return pi;
-}
-
-/* Get the mountpoint for a partition and save it away */
-static PartInfo *
-get_mountpoint(PartType type, struct chunk *old)
-{
-    char *val;
-    PartInfo *tmp;
-    Boolean newfs;
-
-    if (old && old->private_data)
-	tmp = old->private_data;
-    else
-	tmp = NULL;
-    val = (tmp != NULL) ? tmp->mountpoint : (type == PART_EFI) ? "/efi" : NULL;
-    val = msgGetInput(val, "Please specify a mount point for the partition");
-    if (!val || !*val) {
-	if (!old)
-	    return NULL;
-	else {
-	    free(old->private_data);
-	    old->private_data = NULL;
-	}
-	return NULL;
-    }
-
-    /* Is it just the same value? */
-    if (tmp && !strcmp(tmp->mountpoint, val))
-	return NULL;
-
-    /* Did we use it already? */
-    if (check_conflict(val)) {
-	msgConfirm("You already have a mount point for %s assigned!", val);
-	return NULL;
-    }
-
-    /* Is it bogus? */
-    if (*val != '/') {
-	msgConfirm("Mount point must start with a / character");
-	return NULL;
-    }
-
-    /* Is it going to be mounted on root? */
-    if (!strcmp(val, "/")) {
-	if (old)
-	    old->flags |= CHUNK_IS_ROOT;
-    }
-    else if (old)
-	old->flags &= ~CHUNK_IS_ROOT;
-
-    newfs = TRUE;
-    if (tmp) {
-	newfs = tmp->do_newfs;
-    	safe_free(tmp);
-    }
-    val = string_skipwhite(string_prune(val));
-    tmp = new_part(type, val, newfs);
-    if (old) {
-	old->private_data = tmp;
-	old->private_free = safe_free;
-    }
-    return tmp;
-}
-
-/* Get the type of the new partiton */
-static PartType
-get_partition_type(void)
-{
-    char selection[20];
-    int i;
-    static unsigned char *fs_types[] = {
-#ifdef __ia64__
-	"EFI",	"An EFI system partition",
-#endif
-	"FS",	"A file system",
-	"Swap",	"A swap partition.",
-    };
-    WINDOW *w = savescr();
-
-    i = dialog_menu("Please choose a partition type",
-	"If you want to use this partition for swap space, select Swap.\n"
-	"If you want to put a filesystem on it, choose FS.",
-	-1, -1,
-#ifdef __ia64__
-	3, 3,
-#else
-	2, 2,
-#endif
-	fs_types, selection, NULL, NULL);
-    restorescr(w);
-    if (!i) {
-#ifdef __ia64__
-	if (!strcmp(selection, "EFI"))
-	    return PART_EFI;
-#endif
-	if (!strcmp(selection, "FS"))
-	    return PART_FILESYSTEM;
-	else if (!strcmp(selection, "Swap"))
-	    return PART_SWAP;
-    }
-    return PART_NONE;
-}
-
-/* If the user wants a special newfs command for this, set it */
-static void
-getNewfsCmd(PartInfo *p)
-{
-    char buffer[NEWFS_CMD_ARGS_MAX];
-    char *val;
-
-    switch (p->newfs_type) {
-    case NEWFS_UFS:
-	snprintf(buffer, NEWFS_CMD_ARGS_MAX, "%s %s %s %s",
-	    NEWFS_UFS_CMD, p->newfs_data.newfs_ufs.softupdates ?  "-U" : "",
-	    p->newfs_data.newfs_ufs.ufs1 ? "-O1" : "-O2",
-	    p->newfs_data.newfs_ufs.user_options);
-	break;
-    case NEWFS_MSDOS:
-	snprintf(buffer, NEWFS_CMD_ARGS_MAX, "%s", NEWFS_MSDOS_CMD);
-	break;
-    case NEWFS_CUSTOM:
-	strcpy(buffer, p->newfs_data.newfs_custom.command);
-	break;
-    }
-
-    val = msgGetInput(buffer,
-	"Please enter the newfs command and options you'd like to use in\n"
-	"creating this file system.");
-    if (val != NULL) {
-	p->newfs_type = NEWFS_CUSTOM;
-	strlcpy(p->newfs_data.newfs_custom.command, val, NEWFS_CMD_ARGS_MAX);
-    }
-}
-
-static void
-getNewfsOptionalArguments(PartInfo *p)
-{
-	char buffer[NEWFS_CMD_ARGS_MAX];
-	char *val;
-
-	/* Must be UFS, per argument checking in I/O routines. */
-
-	strlcpy(buffer,  p->newfs_data.newfs_ufs.user_options,
-	    NEWFS_CMD_ARGS_MAX);
-	val = msgGetInput(buffer,
-	    "Please enter any additional UFS newfs options you'd like to\n"
-	    "use in creating this file system.");
-	if (val != NULL)
-		strlcpy(p->newfs_data.newfs_ufs.user_options, val,
-		    NEWFS_CMD_ARGS_MAX);
-}
-
-#define MAX_MOUNT_NAME	9
-
-#define PART_PART_COL	0
-#define PART_MOUNT_COL	10
-#define PART_SIZE_COL	(PART_MOUNT_COL + MAX_MOUNT_NAME + 3)
-#define PART_NEWFS_COL	(PART_SIZE_COL + 8)
-#define PART_OFF	38
-
-#define TOTAL_AVAIL_LINES       (10)
-#define PSLICE_SHOWABLE          (4)
-
-
-/* stick this all up on the screen */
-static void
-print_label_chunks(void)
-{
-    int  i, j, srow, prow, pcol;
-    daddr_t sz;
-    char clrmsg[80];
-    int ChunkPartStartRow;
-    WINDOW *ChunkWin;
-
-    /********************************************************/
-    /*** These values are for controling screen resources ***/
-    /*** Each label line holds up to 2 labels, so beware! ***/
-    /*** strategy will be to try to always make sure the  ***/
-    /*** highlighted label is in the active display area. ***/
-    /********************************************************/
-    int  pslice_max, label_max;
-    int  pslice_count, label_count, label_focus_found, pslice_focus_found;
-
-    attrset(A_REVERSE);
-    mvaddstr(0, 25, "FreeBSD Disklabel Editor");
-    attrset(A_NORMAL);
-
-    /*** Count the number of parition slices ***/
-    pslice_count = 0;
-    for (i = 0; label_chunk_info[i].c ; i++) {
-        if (label_chunk_info[i].type == PART_SLICE)
-            ++pslice_count;
-    }
-    pslice_max = pslice_count;
-
-    /*** 4 line max for partition slices ***/
-    if (pslice_max > PSLICE_SHOWABLE) {
-        pslice_max = PSLICE_SHOWABLE;
-    }
-    ChunkPartStartRow = CHUNK_SLICE_START_ROW + 3 + pslice_max;
-    
-    /*** View partition slices modulo pslice_max ***/
-    label_max = TOTAL_AVAIL_LINES - pslice_max;
-
-    for (i = 0; i < 2; i++) {
-	mvaddstr(ChunkPartStartRow - 2, PART_PART_COL + (i * PART_OFF), "Part");
-	mvaddstr(ChunkPartStartRow - 1, PART_PART_COL + (i * PART_OFF), "----");
-
-	mvaddstr(ChunkPartStartRow - 2, PART_MOUNT_COL + (i * PART_OFF), "Mount");
-	mvaddstr(ChunkPartStartRow - 1, PART_MOUNT_COL + (i * PART_OFF), "-----");
-
-	mvaddstr(ChunkPartStartRow - 2, PART_SIZE_COL + (i * PART_OFF) + 3, "Size");
-	mvaddstr(ChunkPartStartRow - 1, PART_SIZE_COL + (i * PART_OFF) + 3, "----");
-
-	mvaddstr(ChunkPartStartRow - 2, PART_NEWFS_COL + (i * PART_OFF), "Newfs");
-	mvaddstr(ChunkPartStartRow - 1, PART_NEWFS_COL + (i * PART_OFF), "-----");
-    }
-    srow = CHUNK_SLICE_START_ROW;
-    prow = 0;
-    pcol = 0;
-
-    /*** these variables indicate that the focused item is shown currently ***/
-    label_focus_found = 0;
-    pslice_focus_found = 0;
-   
-    label_count = 0;
-    pslice_count = 0;
-    mvprintw(CHUNK_SLICE_START_ROW - 1, 0, "          ");
-    mvprintw(CHUNK_SLICE_START_ROW + pslice_max, 0, "          ");
-
-    ChunkWin = newwin(CHUNK_ROW_MAX - ChunkPartStartRow, 76, ChunkPartStartRow, 0);
-
-    wclear(ChunkWin);
-    /*** wrefresh(ChunkWin); ***/
-
-    for (i = 0; label_chunk_info[i].c; i++) {
-	/* Is it a slice entry displayed at the top? */
-	if (label_chunk_info[i].type == PART_SLICE) {
-            /*** This causes the new pslice to replace the previous display ***/
-            /*** focus must remain on the most recently active pslice       ***/
-            if (pslice_count == pslice_max) {
-                if (pslice_focus_found) {
-                    /*** This is where we can mark the more following ***/
-                    attrset(A_BOLD);
-                    mvprintw(CHUNK_SLICE_START_ROW + pslice_max, 0, "***MORE***");
-                    attrset(A_NORMAL);
-                    continue;
-                }
-                else {
-                    /*** this is where we set the more previous ***/
-                    attrset(A_BOLD);
-                    mvprintw(CHUNK_SLICE_START_ROW - 1, 0, "***MORE***");
-                    attrset(A_NORMAL);
-                    pslice_count = 0;
-                    srow = CHUNK_SLICE_START_ROW;
-                }
-            }
-
-	    sz = space_free(label_chunk_info[i].c);
-	    if (i == here)
-		attrset(ATTR_SELECTED);
-            if (i == pslice_focus)
-                pslice_focus_found = -1;
-
-	    if (label_chunk_info[i].c->type == whole) {
-		if (sz >= 100 * ONE_GIG)
-		    mvprintw(srow++, 0,
-			"Disk: %s\t\tFree: %jd blocks (%jdGB)",
-			label_chunk_info[i].c->disk->name, (intmax_t)sz,
-			(intmax_t)(sz / ONE_GIG));
-		else
-		    mvprintw(srow++, 0,
-			"Disk: %s\t\tFree: %jd blocks (%jdMB)",
-			label_chunk_info[i].c->disk->name, (intmax_t)sz,
-			(intmax_t)(sz / ONE_MEG));
-	    } else {
-		if (sz >= 100 * ONE_GIG)
-		    mvprintw(srow++, 0, 
-			"Disk: %s\tPartition name: %s\tFree: %jd blocks (%jdGB)",
-			label_chunk_info[i].c->disk->name,
-			label_chunk_info[i].c->name, 
-			(intmax_t)sz, (intmax_t)(sz / ONE_GIG));
-		else
-		    mvprintw(srow++, 0, 
-			"Disk: %s\tPartition name: %s\tFree: %jd blocks (%jdMB)",
-			label_chunk_info[i].c->disk->name,
-			label_chunk_info[i].c->name, 
-			(intmax_t)sz, (intmax_t)(sz / ONE_MEG));
-	    }
-	    attrset(A_NORMAL);
-	    clrtoeol();
-	    move(0, 0);
-	    /*** refresh(); ***/
-            ++pslice_count;
-	}
-	/* Otherwise it's a DOS, swap or filesystem entry in the Chunk window */
-	else {
-	    char onestr[PART_OFF], num[10], *mountpoint, newfs[12];
-
-	    /*
-	     * We copy this into a blank-padded string so that it looks like
-	     * a solid bar in reverse-video
-	     */
-	    memset(onestr, ' ', PART_OFF - 1);
-	    onestr[PART_OFF - 1] = '\0';
-
-            /*** Track how many labels have been displayed ***/
-            if (label_count == ((label_max - 1 ) * 2)) {
-                if (label_focus_found) {
-                    continue;
-                }
-                else {
-                    label_count = 0;
-                    prow = 0;
-                    pcol = 0;
-                }
-            }
-
-	    /* Go for two columns if we've written one full columns worth */
-	    /*** if (prow == (CHUNK_ROW_MAX - ChunkPartStartRow)) ***/
-            if (label_count == label_max - 1) {
-		pcol = PART_OFF;
-		prow = 0;
-	    }
-	    memcpy(onestr + PART_PART_COL, label_chunk_info[i].c->name, strlen(label_chunk_info[i].c->name));
-	    /* If it's a filesystem, display the mountpoint */
-	    if (label_chunk_info[i].c->private_data && (label_chunk_info[i].type == PART_FILESYSTEM
-		|| label_chunk_info[i].type == PART_FAT || label_chunk_info[i].type == PART_EFI))
-		mountpoint = ((PartInfo *)label_chunk_info[i].c->private_data)->mountpoint;
-	    else if (label_chunk_info[i].type == PART_SWAP)
-		mountpoint = "swap";
-	    else
-	        mountpoint = "<none>";
-
-	    /* Now display the newfs field */
-	    if (label_chunk_info[i].type == PART_FAT)
-		strcpy(newfs, "DOS");
-#if defined(__ia64__)
-	    else if (label_chunk_info[i].type == PART_EFI) {
-		strcpy(newfs, "EFI");
-		if (label_chunk_info[i].c->private_data) {
-		    strcat(newfs, "  ");
-		    PartInfo *pi = (PartInfo *)label_chunk_info[i].c->private_data;
-		    strcat(newfs, pi->do_newfs ? " Y" : " N");
-		}
-	    }
-#endif
-	    else if (label_chunk_info[i].c->private_data && label_chunk_info[i].type == PART_FILESYSTEM) {
-		PartInfo *pi = (PartInfo *)label_chunk_info[i].c->private_data;
-
-		switch (pi->newfs_type) {
-		case NEWFS_UFS:
-			strcpy(newfs, NEWFS_UFS_STRING);
-			if (pi->newfs_data.newfs_ufs.ufs1)
-				strcat(newfs, "1");
-			else
-				strcat(newfs, "2");
-			if (pi->newfs_data.newfs_ufs.softupdates)
-				strcat(newfs, "+S");
-			else
-				strcat(newfs, "  ");
-
-			break;
-		case NEWFS_MSDOS:
-			strcpy(newfs, "FAT");
-			break;
-		case NEWFS_CUSTOM:
-			strcpy(newfs, "CUST");
-			break;
-		}
-		strcat(newfs, pi->do_newfs ? " Y" : " N ");
-	    }
-	    else if (label_chunk_info[i].type == PART_SWAP)
-		strcpy(newfs, "SWAP");
-	    else
-		strcpy(newfs, "*");
-	    for (j = 0; j < MAX_MOUNT_NAME && mountpoint[j]; j++)
-		onestr[PART_MOUNT_COL + j] = mountpoint[j];
-	    if (label_chunk_info[i].c->size == 0)
-	        snprintf(num, 10, "%5dMB", 0);
-	    else if (label_chunk_info[i].c->size < (100 * ONE_GIG))
-		snprintf(num, 10, "%5jdMB",
-		    (intmax_t)label_chunk_info[i].c->size / ONE_MEG);
-	    else
-		snprintf(num, 10, "%5jdGB",
-		    (intmax_t)label_chunk_info[i].c->size / ONE_GIG);
-	    memcpy(onestr + PART_SIZE_COL, num, strlen(num));
-	    memcpy(onestr + PART_NEWFS_COL, newfs, strlen(newfs));
-	    onestr[PART_NEWFS_COL + strlen(newfs)] = '\0';
-            if (i == label_focus) {
-                label_focus_found = -1;
-                wattrset(ChunkWin, A_BOLD);
-            }
-	    if (i == here)
-		wattrset(ChunkWin, ATTR_SELECTED);
-
-            /*** lazy man's way of expensively padding this string ***/
-            while (strlen(onestr) < 37)
-                strcat(onestr, " ");
-
-	    mvwaddstr(ChunkWin, prow, pcol, onestr);
-	    wattrset(ChunkWin, A_NORMAL);
-	    move(0, 0);
-	    ++prow;
-            ++label_count;
-	}
-    }
-    
-    /*** this will erase all the extra stuff ***/
-    memset(clrmsg, ' ', 37);
-    clrmsg[37] = '\0';
-   
-    while (pslice_count < pslice_max) {
-        mvprintw(srow++, 0, clrmsg);
-        clrtoeol();
-        ++pslice_count;
-    }
-    while (label_count < (2 * (label_max - 1))) {
-        mvwaddstr(ChunkWin, prow++, pcol, clrmsg);
-	++label_count;
-	if (prow == (label_max - 1)) {
-	    prow = 0;
-	    pcol = PART_OFF;
-	}
-    }
-    refresh();
-    wrefresh(ChunkWin);
-}
-
-static void
-print_command_summary(void)
-{
-    mvprintw(17, 0, "The following commands are valid here (upper or lower case):");
-    mvprintw(18, 0, "C = Create        D = Delete   M = Mount pt.");
-    if (!RunningAsInit)
-	mvprintw(18, 56, "W = Write");
-    mvprintw(19, 0, "N = Newfs Opts    Q = Finish   S = Toggle SoftUpdates   Z = Custom Newfs");
-    mvprintw(20, 0, "T = Toggle Newfs  U = Undo     A = Auto Defaults        R = Delete+Merge");
-    mvprintw(22, 0, "Use F1 or ? to get more help, arrow keys to select.");
-    move(0, 0);
-}
-
-static void
-clear_wins(void)
-{
-    clear();
-    print_label_chunks();
-}
-
-static int
-diskLabel(Device *dev)
-{
-    daddr_t sz;
-    int  key = 0;
-    Boolean labeling;
-    char *msg = NULL;
-    PartInfo *p, *oldp;
-    PartType type;
-    Device **devs;
-    WINDOW *w = savescr();
-
-    label_focus = 0;
-    pslice_focus = 0;
-    here = 0;
-
-    devs = deviceFind(NULL, DEVICE_TYPE_DISK);
-    if (!devs) {
-	msgConfirm("No disks found!");
-	restorescr(w);
-	return DITEM_FAILURE;
-    }
-    labeling = TRUE;
-    keypad(stdscr, TRUE);
-    record_label_chunks(devs, dev);
-
-    clear();
-    while (labeling) {
-	char *cp;
-	int rflags = DELCHUNK_NORMAL;
-
-	print_label_chunks();
-	print_command_summary();
-	if (msg) {
-	    attrset(title_attr); mvprintw(23, 0, msg); attrset(A_NORMAL);
-	    clrtoeol();
-	    beep();
-	    msg = NULL;
-	}
-	else {
-	    move(23, 0);
-	    clrtoeol();
-	}
-
-	refresh();
-	key = getch();
-	switch (toupper(key)) {
-	    int i;
-	    static char _msg[40];
-
-	case '\014':	/* ^L */
-	    clear_wins();
-	    break;
-
-	case '\020':	/* ^P */
-	case KEY_UP:
-	case '-':
-	    if (here != 0)
-		--here;
-	    else
-		while (label_chunk_info[here + 1].c)
-		    ++here;
-	    break;
-
-	case '\016':	/* ^N */
-	case KEY_DOWN:
-	case '+':
-	case '\r':
-	case '\n':
-	    if (label_chunk_info[here + 1].c)
-		++here;
-	    else
-		here = 0;
-	    break;
-
-	case KEY_HOME:
-	    here = 0;
-	    break;
-
-	case KEY_END:
-	    while (label_chunk_info[here + 1].c)
-		++here;
-	    break;
-
-	case KEY_F(1):
-	case '?':
-	    systemDisplayHelp("partition");
-	    clear_wins();
-	    break;
-
-	case '1':
-	    if (label_chunk_info[here].type == PART_FILESYSTEM) {
-		PartInfo *pi =
-		    ((PartInfo *)label_chunk_info[here].c->private_data);
-
-		if ((pi != NULL) &&
-		    (pi->newfs_type == NEWFS_UFS)) {
-			pi->newfs_data.newfs_ufs.ufs1 = true;
-		} else
-		    msg = MSG_NOT_APPLICABLE;
-	    } else
-		msg = MSG_NOT_APPLICABLE;
-	    break;
-		break;
-
-	case '2':
-	    if (label_chunk_info[here].type == PART_FILESYSTEM) {
-		PartInfo *pi =
-		    ((PartInfo *)label_chunk_info[here].c->private_data);
-
-		if ((pi != NULL) &&
-		    (pi->newfs_type == NEWFS_UFS)) {
-			pi->newfs_data.newfs_ufs.ufs1 = false;
-		} else
-		    msg = MSG_NOT_APPLICABLE;
-	    } else
-		msg = MSG_NOT_APPLICABLE;
-	    break;
-		break;
-
-	case 'A':
-	    if (label_chunk_info[here].type != PART_SLICE) {
-		msg = "You can only do this in a disk slice (at top of screen)";
-		break;
-	    }
-	    /*
-	     * Generate standard partitions automatically.  If we do not
-	     * have sufficient space we attempt to scale-down the size
-	     * of the partitions within certain bounds.
-	     */
-	    {
-		int perc;
-		int req = 0;
-
-		for (perc = 100; perc > 0; perc -= 5) {
-		    req = 0;	/* reset for each loop */
-		    if ((msg = try_auto_label(devs, dev, perc, &req)) == NULL)
-			break;
-		}
-		if (msg) {
-		    if (req) {
-			msgConfirm(msg);
-			clear_wins();
-			msg = NULL;
-		    }
-		}
-	    }
-	    break;
-	    
-	case 'C':
-	    if (label_chunk_info[here].type != PART_SLICE) {
-		msg = "You can only do this in a master partition (see top of screen)";
-		break;
-	    }
-	    sz = space_free(label_chunk_info[here].c);
-	    if (sz <= FS_MIN_SIZE) {
-		msg = "Not enough space to create an additional FreeBSD partition";
-		break;
-	    }
-	    else {
-		char *val;
-		daddr_t size;
-		long double dsize;
-		struct chunk *tmp;
-		char osize[80];
-		u_long flags = 0;
-
-#ifdef __powerpc__
-		/* Always use the maximum size for apple partitions */
-		if (label_chunk_info[here].c->type == apple)
-		    size = sz;
-		else {
-#endif
-		sprintf(osize, "%jd", (intmax_t)sz);
-		val = msgGetInput(osize,
-				  "Please specify the partition size in blocks or append a trailing G for\n"
-#ifdef __ia64__
-				  "gigabytes, M for megabytes.\n"
-#else
-				  "gigabytes, M for megabytes, or C for cylinders.\n"
-#endif
-				  "%jd blocks (%jdMB) are free.",
-				  (intmax_t)sz, (intmax_t)sz / ONE_MEG);
-		if (!val || (dsize = strtold(val, &cp)) <= 0) {
-		    clear_wins();
-		    break;
-		}
-
-		if (*cp) {
-		    if (toupper(*cp) == 'M')
-			size = (daddr_t) (dsize * ONE_MEG);
-		    else if (toupper(*cp) == 'G')
-			size = (daddr_t) (dsize * ONE_GIG);
-#ifndef __ia64__
-		    else if (toupper(*cp) == 'C')
-			size = (daddr_t) dsize * (label_chunk_info[here].c->disk->bios_hd * label_chunk_info[here].c->disk->bios_sect);
-#endif
-		    else
-			size = (daddr_t) dsize;
-		} else {
-			size = (daddr_t) dsize;
-		}
-
-		if (size < FS_MIN_SIZE) {
-		    msgConfirm("The minimum filesystem size is %dMB", FS_MIN_SIZE / ONE_MEG);
-		    clear_wins();
-		    break;
-		}
-#ifdef __powerpc__
-		}
-#endif
-		type = get_partition_type();
-		if (type == PART_NONE) {
-		    clear_wins();
-		    beep();
-		    break;
-		}
-
-		if (type == PART_FILESYSTEM || type == PART_EFI) {
-		    if ((p = get_mountpoint(type, NULL)) == NULL) {
-			clear_wins();
-			beep();
-			break;
-		    }
-		    else if (!strcmp(p->mountpoint, "/")) {
-			if (type != PART_FILESYSTEM) {
-			    clear_wins();
-			    beep();
-			    break;
-			}
-			else
-			    flags |= CHUNK_IS_ROOT;
-		    }
-		    else
-			flags &= ~CHUNK_IS_ROOT;
-		}
-		else
-		    p = NULL;
-
-		if ((flags & CHUNK_IS_ROOT) && (size < (ROOT_MIN_SIZE * ONE_MEG))) {
-		    msgConfirm("Warning: This is smaller than the recommended size for a\n"
-			       "root partition.  For a variety of reasons, root\n"
-			       "partitions should usually be at least %dMB in size", ROOT_MIN_SIZE);
-		}
-		tmp = Create_Chunk_DWIM(label_chunk_info[here].c->disk,
-		    label_chunk_info[here].c, size,
-#ifdef __ia64__
-		    (type == PART_EFI) ? efi : part,
-		    (type == PART_EFI) ? 0 : (type == PART_SWAP) ? FS_SWAP : FS_BSDFFS,
-#else
-		    part, (type == PART_SWAP) ? FS_SWAP : FS_BSDFFS,
-#endif
-		    flags);
-		if (!tmp) {
-		    msgConfirm("Unable to create the partition. Too big?");
-		    clear_wins();
-		    break;
-		}
-
-		tmp->private_data = p;
-		tmp->private_free = safe_free;
-		if (variable_cmp(DISK_LABELLED, "written"))
-		    variable_set2(DISK_LABELLED, "yes", 0);
-		record_label_chunks(devs, dev);
-		clear_wins();
-                /* This is where we assign focus to new label so it shows. */
-                {
-                    int i;
-		    label_focus = -1;
-                    for (i = 0; label_chunk_info[i].c; ++i) {
-                    	if (label_chunk_info[i].c == tmp) {
-			    label_focus = i;
-			    break;
-			}
-		    }
-		    if (label_focus == -1)
-                    	label_focus = i - 1;
-                }
-	    }
-	    break;
-
-	case KEY_DC:
-	case 'R':	/* recover space (delete w/ recover) */
-	    /*
-	     * Delete the partition w/ space recovery.
-	     */
-	    rflags = DELCHUNK_RECOVER;
-	    /* fall through */
-	case 'D':	/* delete */
-	    if (label_chunk_info[here].type == PART_SLICE) {
-		msg = MSG_NOT_APPLICABLE;
-		break;
-	    }
-	    else if (label_chunk_info[here].type == PART_FAT) {
-		msg = "Use the Disk Partition Editor to delete DOS partitions";
-		break;
-	    }
-	    Delete_Chunk2(label_chunk_info[here].c->disk, label_chunk_info[here].c, rflags);
-	    if (variable_cmp(DISK_LABELLED, "written"))
-		variable_set2(DISK_LABELLED, "yes", 0);
-	    record_label_chunks(devs, dev);
-	    break;
-
-	case 'M':	/* mount */
-	    switch(label_chunk_info[here].type) {
-	    case PART_SLICE:
-		msg = MSG_NOT_APPLICABLE;
-		break;
-
-	    case PART_SWAP:
-		msg = "You don't need to specify a mountpoint for a swap partition.";
-		break;
-
-	    case PART_FAT:
-	    case PART_EFI:
-	    case PART_FILESYSTEM:
-		oldp = label_chunk_info[here].c->private_data;
-		p = get_mountpoint(label_chunk_info[here].type, label_chunk_info[here].c);
-		if (p) {
-		    if (!oldp)
-		    	p->do_newfs = FALSE;
-		    if ((label_chunk_info[here].type == PART_FAT ||
-			    label_chunk_info[here].type == PART_EFI) &&
-			(!strcmp(p->mountpoint, "/") ||
-			    !strcmp(p->mountpoint, "/usr") ||
-			    !strcmp(p->mountpoint, "/var"))) {
-			msgConfirm("%s is an invalid mount point for a DOS partition!", p->mountpoint);
-			strcpy(p->mountpoint, "/bogus");
-		    }
-		}
-		if (variable_cmp(DISK_LABELLED, "written"))
-		    variable_set2(DISK_LABELLED, "yes", 0);
-		record_label_chunks(devs, dev);
-		clear_wins();
-		break;
-
-	    default:
-		msgFatal("Bogus partition under cursor???");
-		break;
-	    }
-	    break;
-
-	case 'N':	/* Set newfs options */
-	    if (label_chunk_info[here].c->private_data &&
-		((PartInfo *)label_chunk_info[here].c->private_data)->do_newfs)
-		getNewfsOptionalArguments(
-		    label_chunk_info[here].c->private_data);
-	    else
-		msg = MSG_NOT_APPLICABLE;
-	    clear_wins();
-	    break;
-
-	case 'S':	/* Toggle soft updates flag */
-	    if (label_chunk_info[here].type == PART_FILESYSTEM) {
-		PartInfo *pi = ((PartInfo *)label_chunk_info[here].c->private_data);
-		if (pi != NULL &&
-		    pi->newfs_type == NEWFS_UFS)
-			pi->newfs_data.newfs_ufs.softupdates =
-			    !pi->newfs_data.newfs_ufs.softupdates;
-		else
-		    msg = MSG_NOT_APPLICABLE;
-	    }
-	    else
-		msg = MSG_NOT_APPLICABLE;
-	    break;
-
-	case 'T':	/* Toggle newfs state */
-	    if ((label_chunk_info[here].type == PART_FILESYSTEM ||
-		 label_chunk_info[here].type == PART_EFI) &&
-	        (label_chunk_info[here].c->private_data)) {
-		PartInfo *pi = ((PartInfo *)label_chunk_info[here].c->private_data);
-		if (!pi->do_newfs)
-		    label_chunk_info[here].c->flags |= CHUNK_NEWFS;
-		else
-		    label_chunk_info[here].c->flags &= ~CHUNK_NEWFS;
-
-		label_chunk_info[here].c->private_data =
-		    new_part(label_chunk_info[here].type, pi ? pi->mountpoint : NULL, pi ? !pi->do_newfs
-		    : TRUE);
-		if (pi != NULL &&
-		    pi->newfs_type == NEWFS_UFS) {
-		    PartInfo *pi_new = label_chunk_info[here].c->private_data;
-
-		    pi_new->newfs_data.newfs_ufs = pi->newfs_data.newfs_ufs;
-		}
-		safe_free(pi);
-		label_chunk_info[here].c->private_free = safe_free;
-		if (variable_cmp(DISK_LABELLED, "written"))
-		    variable_set2(DISK_LABELLED, "yes", 0);
-	    }
-	    else
-		msg = MSG_NOT_APPLICABLE;
-	    break;
-
-	case 'U':
-	    clear();
-	    if (!variable_cmp(DISK_LABELLED, "written")) {
-		msgConfirm("You've already written out your changes -\n"
-			   "it's too late to undo!");
-	    }
-	    else if (!msgNoYes("Are you SURE you want to Undo everything?")) {
-		variable_unset(DISK_PARTITIONED);
-		variable_unset(DISK_LABELLED);
-		for (i = 0; devs[i]; i++) {
-		    Disk *d;
-
-		    if (!devs[i]->enabled)
-			continue;
-		    else if ((d = Open_Disk(devs[i]->name)) != NULL) {
-			Free_Disk(devs[i]->private);
-			devs[i]->private = d;
-#ifdef WITH_SLICES
-			diskPartition(devs[i]);
-#endif
-		    }
-		}
-		record_label_chunks(devs, dev);
-	    }
-	    clear_wins();
-	    break;
-
-	case 'W':
-	    if (!variable_cmp(DISK_LABELLED, "written")) {
-		msgConfirm("You've already written out your changes - if you\n"
-			   "wish to overwrite them, you'll have to restart\n"
-			   "%s first.", ProgName);
-	    }
-	    else if (!msgNoYes("WARNING:  This should only be used when modifying an EXISTING\n"
-			  "installation.  If you are installing FreeBSD for the first time\n"
-			  "then you should simply type Q when you're finished here and your\n"
-			  "changes will be committed in one batch automatically at the end of\n"
-			  "these questions.\n\n"
-			  "Are you absolutely sure you want to do this now?")) {
-		variable_set2(DISK_LABELLED, "yes", 0);
-		diskLabelCommit(NULL);
-	    }
-	    clear_wins();
-	    break;
-
-	case 'Z':	/* Set newfs command line */
-	    if (label_chunk_info[here].c->private_data &&
-		((PartInfo *)label_chunk_info[here].c->private_data)->do_newfs)
-		getNewfsCmd(label_chunk_info[here].c->private_data);
-	    else
-		msg = MSG_NOT_APPLICABLE;
-	    clear_wins();
-	    break;
-
-#ifndef __ia64__
-	case '|':
-	    if (!msgNoYes("Are you sure you want to go into Expert mode?\n\n"
-			  "This is an entirely undocumented feature which you are not\n"
-			  "expected to understand!")) {
-		int i;
-		Device **devs;
-
-		dialog_clear();
-		end_dialog();
-		DialogActive = FALSE;
-		devs = deviceFind(NULL, DEVICE_TYPE_DISK);
-		if (!devs) {
-		    msgConfirm("Can't find any disk devices!");
-		    break;
-		}
-		for (i = 0; devs[i] && ((Disk *)devs[i]->private); i++) {
-		    if (devs[i]->enabled)
-		    	slice_wizard(((Disk *)devs[i]->private));
-		}
-		if (variable_cmp(DISK_LABELLED, "written"))
-		    variable_set2(DISK_LABELLED, "yes", 0);
-		DialogActive = TRUE;
-		record_label_chunks(devs, dev);
-		clear_wins();
-	    }
-	    else
-		msg = "A most prudent choice!";
-	    break;
-#endif
-
-	case '\033':	/* ESC */
-	case 'Q':
-	    labeling = FALSE;
-	    break;
-
-	default:
-	    beep();
-	    sprintf(_msg, "Invalid key %d - Type F1 or ? for help", key);
-	    msg = _msg;
-	    break;
-	}
-        if (label_chunk_info[here].type == PART_SLICE)
-            pslice_focus = here;
-        else
-            label_focus = here;
-    }
-    restorescr(w);
-    return DITEM_SUCCESS;
-}
-
-static __inline daddr_t
-requested_part_size(char *varName, daddr_t nom, int def, int perc)
-{
-    char *cp;
-    daddr_t sz;
-
-    if ((cp = variable_get(varName)) != NULL)
-	sz = strtoimax(cp, NULL, 0);
-    else
-	sz = nom + (def - nom) * perc / 100;
-    return(sz * ONE_MEG);
-}
-
-/*
- * Attempt to auto-label the disk.  'perc' (0-100) scales
- * the size of the various partitions within appropriate
- * bounds (NOMINAL through DEFAULT sizes).  The procedure
- * succeeds of NULL is returned.  A non-null return message
- * is either a failure-status message (*req == 0), or 
- * a confirmation requestor (*req == 1).  *req is 0 on
- * entry to this call.
- *
- * As a special exception to the usual sizing rules, /var is given
- * additional space equal to the amount of physical memory present
- * if perc == 100 in order to ensure that users with large hard drives
- * will have enough space to store a crashdump in /var/crash.
- *
- * We autolabel the following partitions:  /, swap, /var, /tmp, /usr,
- * and /home.  /home receives any extra left over disk space. 
- */
-static char *
-try_auto_label(Device **devs, Device *dev, int perc, int *req)
-{
-    daddr_t sz;
-    Chunk *AutoHome, *AutoRoot, *AutoSwap;
-    Chunk *AutoTmp, *AutoUsr, *AutoVar;
-#ifdef __ia64__
-    Chunk *AutoEfi;
-#endif
-    int mib[2];
-    unsigned long physmem;
-    size_t size;
-    char *msg = NULL;
-
-    sz = space_free(label_chunk_info[here].c);
-    if (sz <= FS_MIN_SIZE)
-	return("Not enough free space to create a new partition in the slice");
-
-    (void)checkLabels(FALSE);
-    AutoHome = AutoRoot = AutoSwap = NULL;
-    AutoTmp = AutoUsr = AutoVar = NULL;
-
-#ifdef __ia64__
-    AutoEfi = NULL;
-    if (EfiChunk == NULL) {
-	sz = 400 * ONE_MEG;
-	AutoEfi = Create_Chunk_DWIM(label_chunk_info[here].c->disk,
-	    label_chunk_info[here].c, sz, efi, 0, 0);
-	if (AutoEfi == NULL) {
-	    *req = 1;
-	    msg = "Unable to create the EFI system partition. Too big?";
-	    goto done;
-	}
-	AutoEfi->private_data = new_part(PART_EFI, "/efi", TRUE);
-	AutoEfi->private_free = safe_free;
-	AutoEfi->flags |= CHUNK_NEWFS;
-	record_label_chunks(devs, dev);
-    }
-#endif
-
-    if (RootChunk == NULL) {
-	sz = requested_part_size(VAR_ROOT_SIZE, ROOT_NOMINAL_SIZE, ROOT_DEFAULT_SIZE, perc);
-
-	AutoRoot = Create_Chunk_DWIM(label_chunk_info[here].c->disk,
-			    label_chunk_info[here].c, sz, part,
-			    FS_BSDFFS,  CHUNK_IS_ROOT | CHUNK_AUTO_SIZE);
-	if (!AutoRoot) {
-	    *req = 1;
-	    msg = "Unable to create the root partition. Too big?";
-	    goto done;
-	}
-	AutoRoot->private_data = new_part(PART_FILESYSTEM, "/", TRUE);
-	AutoRoot->private_free = safe_free;
-	AutoRoot->flags |= CHUNK_NEWFS;
-	record_label_chunks(devs, dev);
-    }
-    if (SwapChunk == NULL) {
-	sz = requested_part_size(VAR_SWAP_SIZE, 0, 0, perc);
-	if (sz == 0) {
-	    daddr_t nom;
-	    daddr_t def;
-
-	    mib[0] = CTL_HW;
-	    mib[1] = HW_PHYSMEM;
-	    size = sizeof physmem;
-	    sysctl(mib, 2, &physmem, &size, (void *)0, (size_t)0);
-	    def = 2 * (int)(physmem / 512);
-	    if (def < SWAP_MIN_SIZE * ONE_MEG)
-		def = SWAP_MIN_SIZE * ONE_MEG;
-	    if (def > SWAP_AUTO_LIMIT_SIZE * ONE_MEG)
-		def = SWAP_AUTO_LIMIT_SIZE * ONE_MEG;
-	    nom = (int)(physmem / 512) / 8;
-	    sz = nom + (def - nom) * perc / 100;
-	}
-	AutoSwap = Create_Chunk_DWIM(label_chunk_info[here].c->disk, 
-			    label_chunk_info[here].c, sz, part,
-			    FS_SWAP, CHUNK_AUTO_SIZE);
-	if (!AutoSwap) {
-	    *req = 1;
-	    msg = "Unable to create the swap partition. Too big?";
-	    goto done;
-	}
-	AutoSwap->private_data = 0;
-	AutoSwap->private_free = safe_free;
-	record_label_chunks(devs, dev);
-    }
-    if (VarChunk == NULL) {
-	/* Work out how much extra space we want for a crash dump */
-	unsigned long crashdumpsz;
-
-	mib[0] = CTL_HW;
-	mib[1] = HW_PHYSMEM;
-	size = sizeof(physmem);
-	sysctl(mib, 2, &physmem, &size, (void *)0, (size_t)0);
-
-	if (perc == 100)
-		crashdumpsz = physmem / 1048576;
-	else
-		crashdumpsz = 0;
-
-	sz = requested_part_size(VAR_VAR_SIZE, VAR_NOMINAL_SIZE,	\
-	    VAR_DEFAULT_SIZE + crashdumpsz, perc);
-
-	AutoVar = Create_Chunk_DWIM(label_chunk_info[here].c->disk, 
-				label_chunk_info[here].c, sz, part,
-				FS_BSDFFS, CHUNK_AUTO_SIZE);
-	if (!AutoVar) {
-	    *req = 1;
-	    msg = "Not enough free space for /var - you will need to\n"
-		   "partition your disk manually with a custom install!";
-	    goto done;
-	}
-	AutoVar->private_data = new_part(PART_FILESYSTEM, "/var", TRUE);
-	AutoVar->private_free = safe_free;
-	AutoVar->flags |= CHUNK_NEWFS;
-	record_label_chunks(devs, dev);
-    }
-    if (TmpChunk == NULL && !variable_get(VAR_NO_TMP)) {
-	sz = requested_part_size(VAR_TMP_SIZE, TMP_NOMINAL_SIZE, TMP_DEFAULT_SIZE, perc);
-
-	AutoTmp = Create_Chunk_DWIM(label_chunk_info[here].c->disk, 
-				label_chunk_info[here].c, sz, part,
-				FS_BSDFFS, CHUNK_AUTO_SIZE);
-	if (!AutoTmp) {
-	    *req = 1;
-	    msg = "Not enough free space for /tmp - you will need to\n"
-		   "partition your disk manually with a custom install!";
-	    goto done;
-	}
-	AutoTmp->private_data = new_part(PART_FILESYSTEM, "/tmp", TRUE);
-	AutoTmp->private_free = safe_free;
-	AutoTmp->flags |= CHUNK_NEWFS;
-	record_label_chunks(devs, dev);
-    }
-    if (UsrChunk == NULL && !variable_get(VAR_NO_USR)) {
-	sz = requested_part_size(VAR_USR_SIZE, USR_NOMINAL_SIZE, USR_DEFAULT_SIZE, perc);
-#if AUTO_HOME == 0
-	if (sz < space_free(label_chunk_info[here].c))
-	    sz = space_free(label_chunk_info[here].c);
-#endif
-	if (sz) {
-	    if (sz < (USR_MIN_SIZE * ONE_MEG)) {
-		*req = 1;
-		msg = "Not enough free space for /usr - you will need to\n"
-		       "partition your disk manually with a custom install!";
-	    }
-
-	    AutoUsr = Create_Chunk_DWIM(label_chunk_info[here].c->disk,
-				    label_chunk_info[here].c, sz, part,
-				    FS_BSDFFS, CHUNK_AUTO_SIZE);
-	    if (!AutoUsr) {
-		msg = "Unable to create the /usr partition.  Not enough space?\n"
-			   "You will need to partition your disk manually with a custom install!";
-		goto done;
-	    }
-	    AutoUsr->private_data = new_part(PART_FILESYSTEM, "/usr", TRUE);
-	    AutoUsr->private_free = safe_free;
-	    AutoUsr->flags |= CHUNK_NEWFS;
-	    record_label_chunks(devs, dev);
-	}
-    }
-#if AUTO_HOME == 1
-    if (HomeChunk == NULL && !variable_get(VAR_NO_HOME)) {
-	sz = requested_part_size(VAR_HOME_SIZE, HOME_NOMINAL_SIZE, HOME_DEFAULT_SIZE, perc);
-	if (sz < space_free(label_chunk_info[here].c))
-	    sz = space_free(label_chunk_info[here].c);
-	if (sz) {
-	    if (sz < (HOME_MIN_SIZE * ONE_MEG)) {
-		*req = 1;
-		msg = "Not enough free space for /home - you will need to\n"
-		       "partition your disk manually with a custom install!";
-		goto done;
-	    }
-
-	    AutoHome = Create_Chunk_DWIM(label_chunk_info[here].c->disk,
-				    label_chunk_info[here].c, sz, part,
-				    FS_BSDFFS, CHUNK_AUTO_SIZE);
-	    if (!AutoHome) {
-		msg = "Unable to create the /home partition.  Not enough space?\n"
-			   "You will need to partition your disk manually with a custom install!";
-		goto done;
-	    }
-	    AutoHome->private_data = new_part(PART_FILESYSTEM, "/home", TRUE);
-	    AutoHome->private_free = safe_free;
-	    AutoHome->flags |= CHUNK_NEWFS;
-	    record_label_chunks(devs, dev);
-	}
-    }
-#endif
-
-    /* At this point, we're reasonably "labelled" */
-    if (variable_cmp(DISK_LABELLED, "written"))
-	variable_set2(DISK_LABELLED, "yes", 0);
-
-done:
-    if (msg) {
-	if (AutoRoot != NULL)
-	    Delete_Chunk(AutoRoot->disk, AutoRoot);
-	if (AutoSwap != NULL)
-	    Delete_Chunk(AutoSwap->disk, AutoSwap);
-	if (AutoVar != NULL)
-	    Delete_Chunk(AutoVar->disk, AutoVar);
-	if (AutoTmp != NULL)
-	    Delete_Chunk(AutoTmp->disk, AutoTmp);
-	if (AutoUsr != NULL)
-	    Delete_Chunk(AutoUsr->disk, AutoUsr);
-	if (AutoHome != NULL)
-	    Delete_Chunk(AutoHome->disk, AutoHome);
-	record_label_chunks(devs, dev);
-    }
-    return(msg);
-}
-
-static int
-diskLabelNonInteractive(Device *dev)
-{
-    char *cp;
-    PartType type;
-    PartInfo *p;
-    u_long flags;
-    int i, status;
-    Device **devs;
-    Disk *d;
-    
-    status = DITEM_SUCCESS;
-    cp = variable_get(VAR_DISK);
-    if (!cp) {
-	msgConfirm("diskLabel:  No disk selected - can't label automatically.");
-	return DITEM_FAILURE;
-    }
-    devs = deviceFind(cp, DEVICE_TYPE_DISK);
-    if (!devs) {
-	msgConfirm("diskLabel: No disk device %s found!", cp);
-	return DITEM_FAILURE;
-    }
-    if (dev)
-	d = dev->private;
-    else
-	d = devs[0]->private;
-    record_label_chunks(devs, dev);
-    for (i = 0; label_chunk_info[i].c; i++) {
-	Chunk *c1 = label_chunk_info[i].c;
-
-	if (label_chunk_info[i].type == PART_SLICE) {
-	    char name[512];
-	    char typ[10], mpoint[50];
-	    int entries;
-
-	    for (entries = 1;; entries++) {
-		intmax_t sz;
-		int soft = 0;
-		snprintf(name, sizeof name, "%s-%d", c1->name, entries);
-		if ((cp = variable_get(name)) == NULL)
-		    break;
-		if (sscanf(cp, "%s %jd %s %d", typ, &sz, mpoint, &soft) < 3) {
-		    msgConfirm("For slice entry %s, got an invalid detail entry of: %s",  c1->name, cp);
-		    status = DITEM_FAILURE;
-		    break;
-		} else {
-		    Chunk *tmp;
-
-		    flags = 0;
-		    if (!strcmp(typ, "swap")) {
-			type = PART_SWAP;
-			strcpy(mpoint, "SWAP");
-		    } else {
-			type = PART_FILESYSTEM;
-			if (!strcmp(mpoint, "/"))
-			    flags |= CHUNK_IS_ROOT;
-		    }
-		    if (!sz)
-			sz = space_free(c1);
-		    if (sz > space_free(c1)) {
-			msgConfirm("Not enough free space to create partition: %s", mpoint);
-			status = DITEM_FAILURE;
-			break;
-		    }
-		    if (!(tmp = Create_Chunk_DWIM(d, c1, sz, part,
-			(type == PART_SWAP) ? FS_SWAP : FS_BSDFFS, flags))) {
-			msgConfirm("Unable to create from partition spec: %s. Too big?", cp);
-			status = DITEM_FAILURE;
-			break;
-		    } else {
-			PartInfo *pi;
-			pi = tmp->private_data = new_part(PART_FILESYSTEM, mpoint, TRUE);
-			tmp->private_free = safe_free;
-			pi->newfs_data.newfs_ufs.softupdates = soft;
-			if (!strcmp(typ, "ufs1"))
-				pi->newfs_data.newfs_ufs.ufs1 = TRUE;
-		    }
-		}
-	    }
-	} else {
-	    /* Must be something we can set a mountpoint for */
-	    cp = variable_get(c1->name);
-	    if (cp) {
-		char mpoint[50], do_newfs[8];
-		Boolean newfs = FALSE;
-
-		do_newfs[0] = '\0';
-		if (sscanf(cp, "%s %s", mpoint, do_newfs) != 2) {
-		    msgConfirm("For slice entry %s, got an invalid detail entry of: %s", c1->name, cp);
-		    status = DITEM_FAILURE;
-		    break;
-		}
-		newfs = toupper(do_newfs[0]) == 'Y' ? TRUE : FALSE;
-		if (c1->private_data) {
-		    p = c1->private_data;
-		    p->do_newfs = newfs;
-		    strcpy(p->mountpoint, mpoint);
-		}
-		else {
-		    c1->private_data = new_part(PART_FILESYSTEM, mpoint, newfs);
-		    c1->private_free = safe_free;
-		}
-		if (!strcmp(mpoint, "/"))
-		    c1->flags |= CHUNK_IS_ROOT;
-		else
-		    c1->flags &= ~CHUNK_IS_ROOT;
-	    }
-	}
-    }
-    if (status == DITEM_SUCCESS)
-	variable_set2(DISK_LABELLED, "yes", 0);
-    return status;
-}
diff --git a/usr.sbin/sysinstall/list.h b/usr.sbin/sysinstall/list.h
deleted file mode 100644
index 8300173e0660..000000000000
--- a/usr.sbin/sysinstall/list.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated for what's essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1997 FreeBSD, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY PAUL TRAINA ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL PAUL TRAINA OR HIS KILLER RATS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/* The structure */
-typedef struct _qelement {
-    struct _qelement *q_forw;
-    struct _qelement *q_back;
-} qelement;
-
-#define	INITQUE(Xhead) { \
-    (Xhead).q_forw = &(Xhead); \
-    (Xhead).q_back = &(Xhead); \
-}
-
-#define	EMPTYQUE(Xhead) \
-    ((Xhead).q_forw == &(Xhead))
-
-#define	INSQUEUE(elem, pred) { \
-    register qelement *Xe = (qelement *) (elem); \
-    register qelement *Xp = (qelement *) (pred); \
-    Xp->q_forw = (Xe->q_forw = (Xe->q_back = Xp)->q_forw)->q_back = Xe; \
-}
-
-#define REMQUE(elem) { \
-    register qelement *Xe = (qelement *) (elem); \
-    (Xe->q_back->q_forw = Xe->q_forw)->q_back = Xe->q_back; \
-}
diff --git a/usr.sbin/sysinstall/main.c b/usr.sbin/sysinstall/main.c
deleted file mode 100644
index 3f9164a0d693..000000000000
--- a/usr.sbin/sysinstall/main.c
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated for what's essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/signal.h>
-#include <sys/fcntl.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-
-const char *StartName;		/* Initial contents of argv[0] */
-const char *ProgName = "sysinstall";
-
-static void
-screech(int sig)
-{
-    msgDebug("\007Signal %d caught!  That's bad!\n", sig);
-    longjmp(BailOut, sig);
-}
-
-int
-main(int argc, char **argv)
-{
-    int choice, scroll, curr, max, status;
-    char titlestr[80], *arch, *osrel, *ostype;
-    struct rlimit rlim;
-    char *arg;
-    int i;
-    int optionArgs = 0;
-
-    /* Record name to be able to restart */
-    StartName = argv[0];
-
-    Restarting = FALSE;
-    RunningAsInit = FALSE;
-    Fake = FALSE;
-
-    for (i = 1; i < argc; i++) {
-	arg = argv[i];
-
-	if (arg[0] != '-')
-		break;
-
-	optionArgs++;
-
-	if (!strcmp(arg, "-fake")) {
-		variable_set2(VAR_DEBUG, "YES", 0);
-		Fake = TRUE;
-	} else if (!strcmp(arg, "-restart")) {
-		Restarting = TRUE;
-	} else if (!strcmp(arg, "-fakeInit")) {
-		RunningAsInit = TRUE;
-	}
-	
-	arg = argv[optionArgs+1];
-    }
-
-    if (getpid() == 1)
-	    RunningAsInit = TRUE;
-   
-    /* Catch fatal signals and complain about them if running as init */
-    if (RunningAsInit) {
-	signal(SIGBUS, screech);
-	signal(SIGSEGV, screech);
-    }
-    signal(SIGPIPE, SIG_IGN);
-
-    /* We don't work too well when running as non-root anymore */
-    if (geteuid() != 0) {
-	fprintf(stderr, "Error: This utility should only be run as root.\n");
-	return 1;
-    }
-
-    /*
-     * Given what it does sysinstall (and stuff sysinstall runs like
-     * pkg_add) shouldn't be subject to process limits.  Better to just
-     * let them have what they think they need than have them blow
-     * their brains out during an install (in sometimes strange and
-     * mysterious ways).
-     */
-
-    rlim.rlim_cur = rlim.rlim_max = RLIM_INFINITY;
-    if (setrlimit(RLIMIT_DATA, &rlim) != 0)
-	fprintf(stderr, "Warning: setrlimit() of datasize failed.\n");
-    if (setrlimit(RLIMIT_STACK, &rlim) != 0)
-	fprintf(stderr, "Warning: setrlimit() of stacksize failed.\n");
-
-#ifdef PC98
-    {
-	/* XXX */
-	char *p = getenv("TERM");
-	if (p && strcmp(p, "cons25") == 0)
-	    setenv("TERM", "cons25w", 1);
-    }
-#endif
-
-    /* Set up whatever things need setting up */
-    systemInitialize(argc, argv);
-
-    /* Set default flag and variable values */
-    installVarDefaults(NULL);
-    /* only when multi-user is it reasonable to do this here */
-    if (!RunningAsInit)
-	installEnvironment();
-
-    if (Fake)
-	msgConfirm("I'll be just faking it from here on out, OK?");
-
-    /* Try to preserve our scroll-back buffer */
-    if (OnVTY) {
-	for (curr = 0; curr < 25; curr++)
-	    putchar('\n');
-    }
-    /* Move stderr aside */
-    if (DebugFD)
-	dup2(DebugFD, 2);
-
-    /* Initialize driver modules, if we haven't already done so (ie,
-       the user hit Ctrl-C -> Restart. */
-    if (!pvariable_get("modulesInitialize")) {
-	moduleInitialize();
-	pvariable_set("modulesInitialize=1");
-    }
-
-    /* Probe for all relevant devices on the system */
-    deviceGetAll();
-
-    /* Prompt for the driver floppy if appropriate. */
-    if (!pvariable_get("driverFloppyCheck")) {
-	driverFloppyCheck();
-	pvariable_set("driverFloppyCheck=1");
-    }
-
-    /* First, see if we have any arguments to process (and argv[0] counts if it's not "sysinstall") */
-    if (!RunningAsInit) {
-	for (i = optionArgs+1; i < argc; i++) {
-	    if (DITEM_STATUS(dispatchCommand(argv[i])) != DITEM_SUCCESS)
-		systemShutdown(1);
-	}
-
-	/* If we were given commands to process on the command line, just exit
-	 * now */
-	if (argc > optionArgs+1)
-	    systemShutdown(0);
-    }
-    else
-	dispatch_load_file_int(TRUE);
-
-    status = setjmp(BailOut);
-    if (status) {
-	msgConfirm("A signal %d was caught - I'm saving what I can and shutting\n"
-		   "down.  If you can reproduce the problem, please turn Debug on\n"
-		   "in the Options menu for the extra information it provides\n"
-		   "in debugging problems like this.", status);
-	systemShutdown(status);
-    }
-
-    /* Get user's country and keymap */
-    if (RunningAsInit)
-	configCountry(NULL);
-
-    /* Add FreeBSD version info to the menu title */
-    arch = getsysctlbyname("hw.machine_arch");
-    osrel = getsysctlbyname("kern.osrelease");
-    ostype = getsysctlbyname("kern.ostype");
-    snprintf(titlestr, sizeof(titlestr), "%s/%s %s - %s", ostype, arch,
-	     osrel, MenuInitial.title);
-    free(arch);
-    free(osrel);
-    free(ostype);
-    MenuInitial.title = titlestr;
-
-    /* Begin user dialog at outer menu */
-    dialog_clear();
-    while (1) {
-	choice = scroll = curr = max = 0;
-	dmenuOpen(&MenuInitial, &choice, &scroll, &curr, &max, TRUE);
-	if (!RunningAsInit
-#if defined(__sparc64__)
-	    || !msgNoYes("Are you sure you wish to exit?  The system will halt.")
-#else
-	    || !msgNoYes("Are you sure you wish to exit?  The system will reboot.")
-#endif
-	    )
-	    break;
-    }
-
-    /* Say goodnight, Gracie */
-    systemShutdown(0);
-
-    return 0; /* We should never get here */
-}
diff --git a/usr.sbin/sysinstall/media.c b/usr.sbin/sysinstall/media.c
deleted file mode 100644
index 5b80f1d6901f..000000000000
--- a/usr.sbin/sysinstall/media.c
+++ /dev/null
@@ -1,877 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <signal.h>
-#include <netdb.h>
-#include <sys/socket.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/errno.h>
-#include <sys/fcntl.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/mman.h>
-#include <sys/wait.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <resolv.h>
-
-static Boolean got_intr = FALSE;
-static Boolean ftp_skip_resolve = FALSE;
-
-/* timeout handler */
-static void
-handle_intr(int sig)
-{
-    msgDebug("User generated interrupt.\n");
-    got_intr = TRUE;
-}
-
-static int
-check_for_interrupt(void)
-{
-    if (got_intr) {
-	got_intr = FALSE;
-	return TRUE;
-    }
-    return FALSE;
-}
-
-static int
-genericHook(dialogMenuItem *self, DeviceType type)
-{
-    Device **devs;
-
-    devs = deviceFind(self->prompt, type);
-    if (devs)
-	mediaDevice = devs[0];
-    return (devs ? DITEM_LEAVE_MENU : DITEM_FAILURE);
-}
-
-static int
-cdromHook(dialogMenuItem *self)
-{
-    return genericHook(self, DEVICE_TYPE_CDROM);
-}
-
-static void
-kickstart_dns(void)
-{
-    static Boolean initted = FALSE;
-    int time;
-    char *cp;
-
-    cp = variable_get(VAR_MEDIA_TIMEOUT);
-    if (!cp)
-	time = MEDIA_TIMEOUT;
-    else
-	time = atoi(cp);
-    if (!time)
-	time = 100;
-    if (!initted) {
-	res_init();
-	_res.retry = 2;	/* 2 times seems a reasonable number to me */
-	_res.retrans = time / 2; /* so spend half our alloted time on each try */
-	initted = TRUE;
-    }
-}
-
-char *
-cpioVerbosity()
-{
-    char *cp = variable_get(VAR_CPIO_VERBOSITY);
-
-    if (cp && !strcmp(cp, "high"))
-	return "-v";
-    return "";
-}
-
-int
-mediaOpen(void)
-{
-    if (!mediaDevice || !mediaVerify() || !DEVICE_INIT(mediaDevice))
-	return DITEM_FAILURE;
-    return DITEM_SUCCESS;
-}
-
-void
-mediaClose(void)
-{
-    if (mediaDevice)
-	DEVICE_SHUTDOWN(mediaDevice);
-    mediaDevice = NULL;
-}
-
-/*
- * Return 1 if we successfully found and set the installation type to
- * be a CD.
- */
-int
-mediaSetCDROM(dialogMenuItem *self)
-{
-    Device **devs;
-    int cnt;
-
-    mediaClose();
-    devs = deviceFind(NULL, DEVICE_TYPE_CDROM);
-    cnt = deviceCount(devs);
-    if (!cnt) {
-	if (self)	/* Interactive? */
-	    msgConfirm("No CD/DVD devices found!  Please check that your system's\n"
-		       "configuration is correct and that the CD/DVD drive is of a supported\n"
-		       "type.  For more information, consult the hardware guide\n"
-		       "in the Doc menu.");
-	return DITEM_FAILURE | DITEM_CONTINUE;
-    }
-    else if (cnt > 1) {
-	DMenu *menu;
-	int status;
-	
-	menu = deviceCreateMenu(&MenuMediaCDROM, DEVICE_TYPE_CDROM, cdromHook, NULL);
-	if (!menu)
-	    msgFatal("Unable to create CDROM menu!  Something is seriously wrong.");
-	status = dmenuOpenSimple(menu, FALSE);
-	free(menu);
-	if (!status)
-	    return DITEM_FAILURE;
-    }
-    else
-	mediaDevice = devs[0];
-    return (mediaDevice ? DITEM_SUCCESS | DITEM_LEAVE_MENU : DITEM_FAILURE);
-}
-
-static int
-floppyHook(dialogMenuItem *self)
-{
-    return genericHook(self, DEVICE_TYPE_FLOPPY);
-}
-
-/*
- * Return 1 if we successfully found and set the installation type to
- * be a floppy
- */
-int
-mediaSetFloppy(dialogMenuItem *self)
-{
-    Device **devs;
-    int cnt;
-
-    mediaClose();
-    devs = deviceFind(NULL, DEVICE_TYPE_FLOPPY);
-    cnt = deviceCount(devs);
-    if (!cnt) {
-	msgConfirm("No floppy devices found!  Please check that your system's configuration\n"
-		   "is correct.  For more information, consult the hardware guide in the Doc\n"
-		   "menu.");
-	return DITEM_FAILURE | DITEM_CONTINUE;
-    }
-    else if (cnt > 1) {
-	DMenu *menu;
-	int status;
-
-	menu = deviceCreateMenu(&MenuMediaFloppy, DEVICE_TYPE_FLOPPY, floppyHook, NULL);
-	if (!menu)
-	    msgFatal("Unable to create Floppy menu!  Something is seriously wrong.");
-	status = dmenuOpenSimple(menu, FALSE);
-	free(menu);
-	if (!status)
-	    return DITEM_FAILURE;
-    }
-    else
-	mediaDevice = devs[0];
-    if (mediaDevice)
-	mediaDevice->private = NULL;
-    return (mediaDevice ? DITEM_LEAVE_MENU : DITEM_FAILURE);
-}
-
-static int
-USBHook(dialogMenuItem *self)
-{
-	return genericHook(self, DEVICE_TYPE_USB);
-}
-
-
-/*
- * Attempt to use USB as the installation media type.
- */
-int
-mediaSetUSB(dialogMenuItem *self)
-{
-	Device **devs;
-	int cnt;
-
-	mediaClose();
-	devs = deviceFind(NULL, DEVICE_TYPE_USB);
-	cnt = deviceCount(devs);
-
-	if (!cnt) {
-		msgConfirm("No USB devices found (try Options/Re-scan Devices)");
-		return DITEM_FAILURE | DITEM_CONTINUE;
-	}
-	else if (cnt > 1) {
-		DMenu *menu;
-		int status;
-
-		menu = deviceCreateMenu(&MenuMediaUSB, DEVICE_TYPE_USB, USBHook,
-		    NULL);
-		if (!menu)
-			msgFatal("Unable to create USB menu! Something is " \
-			    "seriously wrong.");
-		status = dmenuOpenSimple(menu, FALSE);
-		free(menu);
-		if (!status)
-			return DITEM_FAILURE;
-	}
-	else
-		mediaDevice = devs[0];
-	if (mediaDevice)
-		mediaDevice->private = NULL;
-	if (!variable_get(VAR_NONINTERACTIVE))
-		msgConfirm("Using USB device: %s", mediaDevice->name);
-	return (mediaDevice ? DITEM_LEAVE_MENU : DITEM_FAILURE);
-}
-
-static int
-DOSHook(dialogMenuItem *self)
-{
-    return genericHook(self, DEVICE_TYPE_DOS);
-}
-
-/*
- * Return 1 if we successfully found and set the installation type to
- * be a DOS partition.
- */
-int
-mediaSetDOS(dialogMenuItem *self)
-{
-    Device **devs;
-    int cnt;
-
-    mediaClose();
-    devs = deviceFind(NULL, DEVICE_TYPE_DOS);
-    cnt = deviceCount(devs);
-    if (!cnt) {
-	msgConfirm("No DOS primary partitions found!  This installation method is unavailable");
-	return DITEM_FAILURE | DITEM_CONTINUE;
-    }
-    else if (cnt > 1) {
-	DMenu *menu;
-	int status;
-
-	menu = deviceCreateMenu(&MenuMediaDOS, DEVICE_TYPE_DOS, DOSHook, NULL);
-	if (!menu)
-	    msgFatal("Unable to create DOS menu!  Something is seriously wrong.");
-	status = dmenuOpenSimple(menu, FALSE);
-	free(menu);
-	if (!status)
-	    return DITEM_FAILURE;
-    }
-    else
-	mediaDevice = devs[0];
-    return (mediaDevice ? DITEM_LEAVE_MENU : DITEM_FAILURE);
-}
-
-/*
- * Return 0 if we successfully found and set the installation type to
- * be an ftp server
- */
-int
-mediaSetFTP(dialogMenuItem *self)
-{
-    static Device ftpDevice;
-    char *cp, hbuf[MAXHOSTNAMELEN], *hostname, *dir;
-    struct addrinfo hints, *res;
-    int af;
-    size_t urllen;
-    extern int FtpPort;
-    static Device *networkDev = NULL;
-
-    mediaClose();
-    cp = variable_get(VAR_FTP_PATH);
-    /* If we've been through here before ... */
-    if (networkDev && cp && msgYesNo("Re-use old FTP site selection values?"))
-	cp = NULL;
-    if (!cp) {
-	if (!dmenuOpenSimple(&MenuMediaFTP, FALSE))
-	    return DITEM_FAILURE;
-	else
-	    cp = variable_get(VAR_FTP_PATH);
-    }
-    if (!cp)
-	return DITEM_FAILURE;
-    else if (!strcmp(cp, "other")) {
-	variable_set2(VAR_FTP_PATH, "ftp://", 0);
-	cp = variable_get_value(VAR_FTP_PATH, "Please specify the URL of a FreeBSD distribution on a\n"
-				"remote ftp site.  This site must accept either anonymous\n"
-				"ftp or you should have set an ftp username and password\n"
-				"in the Options screen.\n\n"
-				"A URL looks like this:  ftp://<hostname>/<path>\n"
-				"Where <path> is relative to the anonymous ftp directory or the\n"
-				"home directory of the user being logged in as.", 0);
-	if (!cp || !*cp || !strcmp(cp, "ftp://")) {
-	    variable_unset(VAR_FTP_PATH);
-	    return DITEM_FAILURE;
-	}
-	urllen = strlen(cp);
-	if (urllen >= sizeof(ftpDevice.name)) {
-	    msgConfirm("Length of specified URL is %zu characters. Allowable maximum is %zu.",
-			urllen,sizeof(ftpDevice.name)-1);
-	    variable_unset(VAR_FTP_PATH);
-	    return DITEM_FAILURE;
-	}
-    }
-    if (strncmp("ftp://", cp, 6)) {
-	msgConfirm("Sorry, %s is an invalid URL!", cp);
-	variable_unset(VAR_FTP_PATH);
-	return DITEM_FAILURE;
-    }
-    SAFE_STRCPY(ftpDevice.name, cp);
-    SAFE_STRCPY(hbuf, cp + 6);
-    hostname = hbuf;
-
-    if (!networkDev || msgYesNo("You've already done the network configuration once,\n"
-				"would you like to skip over it now?") != 0) {
-	if (networkDev)
-	    DEVICE_SHUTDOWN(networkDev);
-	if (!(networkDev = tcpDeviceSelect())) {
-	    variable_unset(VAR_FTP_PATH);
-	    return DITEM_FAILURE;
-	}
-    }
-    if (!DEVICE_INIT(networkDev)) {
-	if (isDebug())
-	    msgDebug("mediaSetFTP: Net device init failed.\n");
-	variable_unset(VAR_FTP_PATH);
-	return DITEM_FAILURE;
-    }
-    if (*hostname == '[' && (cp = index(hostname + 1, ']')) != NULL &&
-	(*++cp == '\0' || *cp == '/' || *cp == ':')) {
-	++hostname;
-	*(cp - 1) = '\0';
-    }
-    else
-	cp = index(hostname, ':');
-    if (cp != NULL && *cp == ':') {
-	*(cp++) = '\0';
-	FtpPort = strtol(cp, 0, 0);
-    }
-    else
-	FtpPort = 21;
-    if ((dir = index(cp ? cp : hostname, '/')) != NULL)
-	*(dir++) = '\0';
-    if (isDebug()) {
-	msgDebug("hostname = `%s'\n", hostname);
-	msgDebug("dir = `%s'\n", dir ? dir : "/");
-	msgDebug("port # = `%d'\n", FtpPort);
-    }
-    if (!ftp_skip_resolve && variable_get(VAR_NAMESERVER)) {
-	msgNotify("Looking up host %s.", hostname);
-    	if (isDebug())
-	    msgDebug("Starting DNS.\n");
-	kickstart_dns();
-    	if (isDebug())
-	    msgDebug("Looking up hostname, %s, using getaddrinfo(AI_NUMERICHOST).\n", hostname);
-	af = variable_cmp(VAR_IPV6_ENABLE, "YES") ? AF_INET : AF_UNSPEC;
-	memset(&hints, 0, sizeof(hints));
-	hints.ai_family = af;
-	hints.ai_socktype = SOCK_STREAM;
-	hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST;
-	if (getaddrinfo(hostname, NULL, &hints, &res) != 0) {
-	    if (isDebug())
-		msgDebug("Looking up hostname, %s, using getaddrinfo().\n",
-			 hostname);
-	    hints.ai_flags = AI_PASSIVE;
-	    if (getaddrinfo(hostname, NULL, &hints, &res) != 0) {
-		msgConfirm("Cannot resolve hostname `%s'!  Are you sure that"
-			" your\nname server, gateway and network interface are"
-			" correctly configured?", hostname);
-		if (networkDev)
-		    DEVICE_SHUTDOWN(networkDev);
-		networkDev = NULL;
-		variable_unset(VAR_FTP_PATH);
-		return DITEM_FAILURE;
-	    }
-	}
-	freeaddrinfo(res);
-	if (isDebug())
-	    msgDebug("Found DNS entry for %s successfully..\n", hostname);
-    }
-    variable_set2(VAR_FTP_HOST, hostname, 0);
-    variable_set2(VAR_FTP_DIR, dir ? dir : "/", 0);
-    variable_set2(VAR_FTP_PORT, itoa(FtpPort), 0);
-    ftpDevice.type = DEVICE_TYPE_FTP;
-    ftpDevice.init = mediaInitFTP;
-    ftpDevice.get = mediaGetFTP;
-    ftpDevice.shutdown = mediaShutdownFTP;
-    ftpDevice.private = networkDev;
-    mediaDevice = &ftpDevice;
-    return DITEM_SUCCESS | DITEM_LEAVE_MENU | DITEM_RESTORE;
-}
-
-int
-mediaSetFTPActive(dialogMenuItem *self)
-{
-    variable_set2(VAR_FTP_STATE, "active", 0);
-    return mediaSetFTP(self);
-}
-
-int
-mediaSetFTPPassive(dialogMenuItem *self)
-{
-    variable_set2(VAR_FTP_STATE, "passive", 0);
-    return mediaSetFTP(self);
-}
-
-int mediaSetHTTP(dialogMenuItem *self)
-{
-    Boolean tmp;
-    int result;
-    char *cp, *idx, hbuf[MAXHOSTNAMELEN], *hostname;
-    int HttpPort;
-    int what = DITEM_RESTORE;
-
-
-    tmp = ftp_skip_resolve;
-    ftp_skip_resolve = TRUE;
-    result = mediaSetFTP(self);
-    ftp_skip_resolve = tmp;
-
-    if (DITEM_STATUS(result) != DITEM_SUCCESS)
-	return result;
- 
-    cp = variable_get_value(VAR_HTTP_PROXY,
-	"Please enter the address of the HTTP proxy in this format:\n"
-	" hostname:port (the ':port' is optional, default is 3128)",0);
-    if (!cp)
-	return DITEM_FAILURE;
-    SAFE_STRCPY(hbuf, cp);
-    hostname = hbuf;
-    if (*hostname == '[' && (idx = index(hostname + 1, ']')) != NULL &&
-	(*++idx == '\0' || *idx == ':')) {
-	++hostname;
-	*(idx - 1) = '\0';
-    } else
-	idx = index(hostname, ':');
-    if (idx == NULL || *idx != ':')
-	HttpPort = 3128;		/* try this as default */
-    else {
-	*(idx++) = '\0';
-	HttpPort = strtol(idx, 0, 0);
-    }
-
-    variable_set2(VAR_HTTP_HOST, hostname, 0);
-    variable_set2(VAR_HTTP_PORT, itoa(HttpPort), 0);
-    if (isDebug()) {
-      msgDebug("VAR_FTP_PATH : %s\n",variable_get(VAR_FTP_PATH));
-      msgDebug("VAR_HTTP_HOST, _PORT: %s:%s\n",variable_get(VAR_HTTP_HOST),
-                                             variable_get(VAR_HTTP_PORT));
-    }
-
-    /* mediaDevice has been set by mediaSetFTP(), overwrite partly: */
-    mediaDevice->type = DEVICE_TYPE_HTTP;
-    mediaDevice->init = mediaInitHTTP;
-    mediaDevice->get = mediaGetHTTP;
-    mediaDevice->shutdown = dummyShutdown;
-    return DITEM_SUCCESS | DITEM_LEAVE_MENU | what;
-}
-   
-
-int
-mediaSetUFS(dialogMenuItem *self)
-{
-    static Device ufsDevice;
-    struct statfs st;
-    char *cp;
-
-    mediaClose();
-    cp = variable_get_value(VAR_UFS_PATH, "Enter a fully qualified pathname for the directory\n"
-			    "containing the FreeBSD distribution files:", 0);
-    if (!cp)
-	return DITEM_FAILURE;
-
-    /* If they gave us a CDROM or something, try and pick a better name */
-    if (statfs(cp, &st))
-	strcpy(ufsDevice.name, "ufs");
-    else
-	strcpy(ufsDevice.name, st.f_fstypename);
-
-    ufsDevice.type = DEVICE_TYPE_UFS;
-    ufsDevice.init = dummyInit;
-    ufsDevice.get = mediaGetUFS;
-    ufsDevice.shutdown = dummyShutdown;
-    ufsDevice.private = strdup(cp);
-    mediaDevice = &ufsDevice;
-    return DITEM_LEAVE_MENU;
-}
-
-int
-mediaSetNFS(dialogMenuItem *self)
-{
-    static Device nfsDevice;
-    static Device *networkDev = NULL;
-    char *cp, *idx;
-    char hostname[MAXPATHLEN];
-    size_t pathlen;
-
-    mediaClose();
-    cp = variable_get_value(VAR_NFS_PATH, "Please enter the full NFS file specification for the remote\n"
-			    "host and directory containing the FreeBSD distribution files.\n"
-			    "This should be in the format:  hostname:/some/freebsd/dir", 0);
-    if (!cp)
-	return DITEM_FAILURE;
-    SAFE_STRCPY(hostname, cp);
-    if (!(idx = index(hostname, ':'))) {
-	msgConfirm("Invalid NFS path specification.  Must be of the form:\n"
-		   "host:/full/pathname/to/FreeBSD/distdir");
-	return DITEM_FAILURE;
-    }
-    pathlen = strlen(hostname);
-    if (pathlen >= sizeof(nfsDevice.name)) {
-	msgConfirm("Length of specified NFS path is %zu characters. Allowable maximum is %zu.",
-		   pathlen,sizeof(nfsDevice.name)-1);
-	variable_unset(VAR_NFS_PATH);
-	return DITEM_FAILURE;
-    }
-    SAFE_STRCPY(nfsDevice.name, hostname);
-    *idx = '\0';
-    if (!networkDev || msgYesNo("You've already done the network configuration once,\n"
-				"would you like to skip over it now?") != 0) {
-	if (networkDev)
-	    DEVICE_SHUTDOWN(networkDev);
-	if (!(networkDev = tcpDeviceSelect()))
-	    return DITEM_FAILURE;
-    }
-    if (!DEVICE_INIT(networkDev)) {
-	if (isDebug())
-	    msgDebug("mediaSetNFS: Net device init failed\n");
-    }
-    if (variable_get(VAR_NAMESERVER)) {
-	kickstart_dns();
-	if ((inet_addr(hostname) == INADDR_NONE) && (gethostbyname(hostname) == NULL)) {
-	    msgConfirm("Cannot resolve hostname `%s'!  Are you sure that your\n"
-		       "name server, gateway and network interface are correctly configured?", hostname);
-	    if (networkDev)
-		DEVICE_SHUTDOWN(networkDev);
-	    networkDev = NULL;
-	    variable_unset(VAR_NFS_PATH);
-	    return DITEM_FAILURE;
-	}
-	else {
-	    if (isDebug())
-		msgDebug("Found DNS entry for %s successfully..\n", hostname);
-	}
-    }
-    variable_set2(VAR_NFS_HOST, hostname, 0);
-    nfsDevice.type = DEVICE_TYPE_NFS;
-    nfsDevice.init = mediaInitNFS;
-    nfsDevice.get = mediaGetNFS;
-    nfsDevice.shutdown = mediaShutdownNFS;
-    nfsDevice.private = networkDev;
-    mediaDevice = &nfsDevice;
-    return DITEM_LEAVE_MENU;
-}
-
-Boolean
-mediaExtractDistBegin(char *dir, int *fd, int *zpid, int *cpid)
-{
-    int i, pfd[2],qfd[2];
-
-    if (!dir)
-	dir = "/";
-    Mkdir(dir);
-    chdir(dir);
-    pipe(pfd);
-    pipe(qfd);
-    *zpid = fork();
-    if (!*zpid) {
-	char *unzipper = RunningAsInit ? "/stand/" UNZIPPER
-	    : "/usr/bin/" UNZIPPER;
-
-	dup2(qfd[0], 0); close(qfd[0]);
-	dup2(pfd[1], 1); close(pfd[1]);
-	if (DebugFD != -1)
-	    dup2(DebugFD, 2);
-	else {
-	    close(2);
-	    open("/dev/null", O_WRONLY);
-	}
-	close(qfd[1]);
-	close(pfd[0]);
-	i = execl(unzipper, unzipper, (char *)0);
-	if (isDebug())
-	    msgDebug("%s command returns %d status\n", unzipper, i);
-	exit(i);
-    }
-    *fd = qfd[1];
-    close(qfd[0]);
-    *cpid = fork();
-    if (!*cpid) {
-	char *cpio = RunningAsInit ? "/stand/cpio" : "/usr/bin/cpio";
-
-	dup2(pfd[0], 0); close(pfd[0]);
-	close(pfd[1]);
-	close(qfd[1]);
-	if (DebugFD != -1) {
-	    dup2(DebugFD, 1);
-	    dup2(DebugFD, 2);
-	}
-	else {
-	    close(1); open("/dev/null", O_WRONLY);
-	    dup2(1, 2);
-	}
-	if (strlen(cpioVerbosity()))
-	    i = execl(cpio, cpio, "-idum", cpioVerbosity(), (char *)0);
-	else
-	    i = execl(cpio, cpio, "-idum", (char *)0);
-	if (isDebug())
-	    msgDebug("%s command returns %d status\n", cpio, i);
-	exit(i);
-    }
-    close(pfd[0]);
-    close(pfd[1]);
-    return TRUE;
-}
-
-Boolean
-mediaExtractDistEnd(int zpid, int cpid)
-{
-    int i,j;
-
-    i = waitpid(zpid, &j, 0);
-    /* Don't check exit status - gunzip seems to return a bogus one! */
-    if (i < 0) {
-	if (isDebug())
-	    msgDebug("wait for %s returned status of %d!\n", UNZIPPER, i);
-	return FALSE;
-    }
-    i = waitpid(cpid, &j, 0);
-    if (i < 0 || WEXITSTATUS(j)) {
-	if (isDebug())
-	    msgDebug("cpio returned error status of %d!\n", WEXITSTATUS(j));
-	return FALSE;
-    }
-    return TRUE;
-}
-
-Boolean
-mediaExtractDist(char *dir, char *dist, FILE *fp)
-{
-    int i, j, total, seconds, zpid, cpid, pfd[2], qfd[2];
-    char buf[BUFSIZ];
-    struct timeval start, stop;
-    struct sigaction new, old;
-
-    if (!dir)
-	dir = "/";
-
-    Mkdir(dir);
-    chdir(dir);
-    pipe(pfd);	/* read end */
-    pipe(qfd);	/* write end */
-    zpid = fork();
-    if (!zpid) {
-	char *unzipper = RunningAsInit ? "/stand/" UNZIPPER
-	    : "/usr/bin/" UNZIPPER;
-
-	fclose(fp);
-	close(qfd[1]);
-	dup2(qfd[0], 0); close(qfd[0]);
-
-	close(pfd[0]); 
-	dup2(pfd[1], 1); close(pfd[1]);
-
-	if (DebugFD != -1)
-	    dup2(DebugFD, 2);
-	else {
-	    close(2);
-	    open("/dev/null", O_WRONLY);
-	}
-	i = execl(unzipper, unzipper, (char *)0);
-	if (isDebug())
-	    msgDebug("%s command returns %d status\n", unzipper, i);
-	exit(i);
-    }
-    cpid = fork();
-    if (!cpid) {
-	char *cpio = RunningAsInit ? "/stand/cpio" : "/usr/bin/cpio";
-
-	close(pfd[1]);
-	dup2(pfd[0], 0); close(pfd[0]);
-	close (qfd[0]); close(qfd[1]);
-	fclose(fp);
-	if (DebugFD != -1) {
-	    dup2(DebugFD, 1);
-	    dup2(DebugFD, 2);
-	}
-	else {
-	    dup2(open("/dev/null", O_WRONLY), 1);
-	    dup2(1, 2);
-	}
-	if (strlen(cpioVerbosity()))
-	    i = execl(cpio, cpio, "-idum", cpioVerbosity(), (char *)0);
-	else
-	    i = execl(cpio, cpio, "-idum", "--block-size", (char *)0);
-	if (isDebug())
-	    msgDebug("%s command returns %d status\n", cpio, i);
-	exit(i);
-    }
-    close(pfd[0]); close(pfd[1]);
-    close(qfd[0]);
-
-    total = 0;
-    (void)gettimeofday(&start, (struct timezone *)0);
-
-    /* Make ^C abort the current transfer rather than the whole show */
-    new.sa_handler = handle_intr;
-    new.sa_flags = 0;
-    (void)sigemptyset(&new.sa_mask);
-    sigaction(SIGINT, &new, &old);
-
-    while ((i = fread(buf, 1, BUFSIZ, fp)) > 0) {
-	if (check_for_interrupt()) {
-	    msgConfirm("Failure to read from media:  User interrupt.");
-	    break;
-	}
-	if (write(qfd[1], buf, i) != i) {
-	    msgConfirm("Write error on transfer to cpio process, try of %d bytes.", i);
-	    break;
-	}
-	else {
-	    (void)gettimeofday(&stop, (struct timezone *)0);
-	    stop.tv_sec = stop.tv_sec - start.tv_sec;
-	    stop.tv_usec = stop.tv_usec - start.tv_usec;
-	    if (stop.tv_usec < 0)
-		stop.tv_sec--, stop.tv_usec += 1000000;
-	    seconds = stop.tv_sec + (stop.tv_usec / 1000000.0);
-	    if (!seconds)
-		seconds = 1;
-	    total += i;
-	    msgInfo("%10d bytes read from %s dist @ %.1f KB/sec.",
-		    total, dist, (total / seconds) / 1024.0);
-	}
-    }
-    sigaction(SIGINT, &old, NULL);	/* restore sigint */
-    close(qfd[1]);
-
-    i = waitpid(zpid, &j, 0);
-    /* Don't check exit status - gunzip seems to return a bogus one! */
-    if (i < 0) {
-	if (isDebug())
-	    msgDebug("wait for %s returned status of %d!\n", UNZIPPER, i);
-	return FALSE;
-    }
-    i = waitpid(cpid, &j, 0);
-    if (i < 0 || WEXITSTATUS(j)) {
-	if (isDebug())
-	    msgDebug("cpio returned error status of %d!\n", WEXITSTATUS(j));
-	return FALSE;
-    }
-    return TRUE;
-}
-
-int
-mediaGetType(dialogMenuItem *self)
-{
-    return ((dmenuOpenSimple(&MenuMedia, FALSE) && mediaDevice) ? DITEM_SUCCESS : DITEM_FAILURE);
-}
-
-/* Return TRUE if all the media variables are set up correctly */
-Boolean
-mediaVerify(void)
-{
-    if (!mediaDevice)
-	return (DITEM_STATUS(mediaGetType(NULL)) == DITEM_SUCCESS);
-    return TRUE;
-}
-
-/* Set the FTP username and password fields */
-int
-mediaSetFTPUserPass(dialogMenuItem *self)
-{
-    char *pass;
-
-    if (variable_get_value(VAR_FTP_USER, "Please enter the username you wish to login as:", 0)) {
-	DialogInputAttrs |= DITEM_NO_ECHO;
-	pass = variable_get_value(VAR_FTP_PASS, "Please enter the password for this user:", 0);
-	DialogInputAttrs &= ~DITEM_NO_ECHO;
-    }
-    else
-	pass = NULL;
-    return (pass ? DITEM_SUCCESS : DITEM_FAILURE);
-}
-
-/* Set CPIO verbosity level */
-int
-mediaSetCPIOVerbosity(dialogMenuItem *self)
-{
-    char *cp = variable_get(VAR_CPIO_VERBOSITY);
-
-    if (!cp) {
-	msgConfirm("CPIO Verbosity is not set to anything!");
-	return DITEM_FAILURE;
-    }
-    else {
-	if (!strcmp(cp, "low"))
-	    variable_set2(VAR_CPIO_VERBOSITY, "high", 0);
-	else /* must be "high" - wrap around */
-	    variable_set2(VAR_CPIO_VERBOSITY, "low", 0);
-    }
-    return DITEM_SUCCESS;
-}
-
-/* A generic open which follows a well-known "path" of places to look */
-FILE *
-mediaGenericGet(char *base, const char *file)
-{
-    char	buf[PATH_MAX];
-
-    snprintf(buf, PATH_MAX, "%s/%s", base, file);
-    if (file_readable(buf))
-	return fopen(buf, "r");
-    snprintf(buf, PATH_MAX, "%s/FreeBSD/%s", base, file);
-    if (file_readable(buf))
-	return fopen(buf, "r");
-    snprintf(buf, PATH_MAX, "%s/releases/%s", base, file);
-    if (file_readable(buf))
-	return fopen(buf, "r");
-    snprintf(buf, PATH_MAX, "%s/%s/%s", base, variable_get(VAR_RELNAME), file);
-    if (file_readable(buf))
-	return fopen(buf, "r");
-    snprintf(buf, PATH_MAX, "%s/releases/%s/%s", base, variable_get(VAR_RELNAME), file);
-    return fopen(buf, "r");
-}
-
diff --git a/usr.sbin/sysinstall/menus.c b/usr.sbin/sysinstall/menus.c
deleted file mode 100644
index 360b6a910dc5..000000000000
--- a/usr.sbin/sysinstall/menus.c
+++ /dev/null
@@ -1,2164 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#ifndef lint
-static const char rcsid[] =
-  "$FreeBSD$";
-#endif
-
-#include "sysinstall.h"
-
-/* Miscellaneous work routines for menus */
-static int
-setSrc(dialogMenuItem *self)
-{
-    Dists |= DIST_SRC;
-    SrcDists = DIST_SRC_ALL;
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-static int
-clearSrc(dialogMenuItem *self)
-{
-    Dists &= ~DIST_SRC;
-    SrcDists = 0;
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-static int
-setKernel(dialogMenuItem *self)
-{
-    Dists |= DIST_KERNEL;
-    KernelDists = DIST_KERNEL_ALL;
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-static int
-clearKernel(dialogMenuItem *self)
-{
-    Dists &= ~DIST_KERNEL;
-    KernelDists = 0;
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-static int
-setDocAll(dialogMenuItem *self)
-{
-    Dists |= DIST_DOC;
-    DocDists = DIST_DOC_ALL;
-    return DITEM_SUCCESS | DITEM_REDRAW;
-}
-
-
-#define _IS_SET(dist, set) (((dist) & (set)) == (set))
-
-#define IS_DEVELOPER(dist, extra) (_IS_SET(dist, _DIST_DEVELOPER | extra) || \
-	_IS_SET(dist, _DIST_DEVELOPER | extra))
-
-#define IS_USER(dist, extra) (_IS_SET(dist, _DIST_USER | extra) || \
-	_IS_SET(dist, _DIST_USER | extra))
-
-static int
-checkDistDeveloper(dialogMenuItem *self)
-{
-    return IS_DEVELOPER(Dists, 0) && _IS_SET(SrcDists, DIST_SRC_ALL);
-}
-
-static int
-checkDistKernDeveloper(dialogMenuItem *self)
-{
-    return IS_DEVELOPER(Dists, 0) && _IS_SET(SrcDists, DIST_SRC_SYS);
-}
-
-static int
-checkDistUser(dialogMenuItem *self)
-{
-    return IS_USER(Dists, 0);
-}
-
-static int
-checkDistMinimum(dialogMenuItem *self)
-{
-    return Dists == (DIST_BASE | DIST_KERNEL);
-}
-
-static int
-checkDistEverything(dialogMenuItem *self)
-{
-    return Dists == DIST_ALL &&
-	_IS_SET(DocDists, DIST_DOC_ALL) &&
-	_IS_SET(SrcDists, DIST_SRC_ALL) &&
-	_IS_SET(KernelDists, DIST_KERNEL_ALL);
-}
-
-static int
-srcFlagCheck(dialogMenuItem *item)
-{
-    return SrcDists;
-}
-
-static int
-kernelFlagCheck(dialogMenuItem *item)
-{
-    return KernelDists;
-}
-
-static int
-docFlagCheck(dialogMenuItem *item)
-{
-    return DocDists;
-}
-
-static int
-checkTrue(dialogMenuItem *item)
-{
-    return TRUE;
-}
-
-/* All the system menus go here.
- *
- * Hardcoded things like version number strings will disappear from
- * these menus just as soon as I add the code for doing inline variable
- * expansion.
- */
-
-DMenu MenuIndex = {
-    DMENU_NORMAL_TYPE,
-    "Glossary of functions",
-    "This menu contains an alphabetized index of the top level functions in\n"
-    "this program (sysinstall).  Invoke an option by pressing [SPACE] or\n"
-    "[ENTER].  To exit, use [TAB] to move to the Cancel button.",
-    "Use PageUp or PageDown to move through this menu faster!",
-    NULL,
-    { { " Anon FTP",		"Configure anonymous FTP logins.",	dmenuVarCheck, configAnonFTP, NULL, "anon_ftp" },
-      { " Commit",		"Commit any pending actions (dangerous!)", NULL, installCustomCommit },
-      { " Country",		"Set the system's country",		NULL, configCountry },
-#ifdef WITH_SYSCONS
-      { " Console settings",	"Customize system console behavior.",	NULL, dmenuSubmenu, NULL, &MenuSyscons },
-#endif
-      { " Configure",		"The system configuration menu.",	NULL, dmenuSubmenu, NULL, &MenuConfigure },
-      { " Defaults, Load (FDD)","Load default settings from floppy.",	NULL, dispatch_load_floppy },
-      { " Defaults, Load (CD)",	"Load default settings from CDROM.",	NULL, dispatch_load_cdrom },
-      { " Defaults, Load",	"Load default settings (all devices).",	NULL, dispatch_load_menu },
-#ifdef WITH_MICE
-      { " Device, Mouse",	"The mouse configuration menu.",	NULL, dmenuSubmenu, NULL, &MenuMouse },
-#endif
-      { " Disklabel",		"The disk label editor",		NULL, diskLabelEditor },
-      { " Dists, All",		"Root of the distribution tree.",	NULL, dmenuSubmenu, NULL, &MenuDistributions },
-      { " Dists, Basic",		"Basic FreeBSD distribution menu.",	NULL, dmenuSubmenu, NULL, &MenuSubDistributions },
-      { " Dists, Developer",	"Select developer's distribution.",	checkDistDeveloper, distSetDeveloper },
-      { " Dists, Src",		"Src distribution menu.",		NULL, dmenuSubmenu, NULL, &MenuSrcDistributions },
-      { " Dists, Kern Developer", "Select kernel developer's distribution.", checkDistKernDeveloper, distSetKernDeveloper },
-      { " Dists, User",		"Select average user distribution.",	checkDistUser, distSetUser },
-      { " Distributions, Adding", "Installing additional distribution sets", NULL, distExtractAll },
-      { " Documentation",	"Installation instructions, README, etc.", NULL, dmenuSubmenu, NULL, &MenuDocumentation },
-      { " Documentation Installation",	"Installation of FreeBSD documentation set", NULL, distSetDocMenu },
-      { " Doc, README",		"The distribution README file.",	NULL, dmenuDisplayFile, NULL, "README" },
-      { " Doc, Errata",		"The distribution errata.",	NULL, dmenuDisplayFile, NULL, "ERRATA" },
-      { " Doc, Hardware",	"The distribution hardware guide.",	NULL, dmenuDisplayFile,	NULL, "HARDWARE" },
-      { " Doc, Copyright",	"The distribution copyright notices.",	NULL, dmenuDisplayFile,	NULL, "COPYRIGHT" },
-      { " Doc, Release",		"The distribution release notes.",	NULL, dmenuDisplayFile, NULL, "RELNOTES" },
-      { " Doc, HTML",		"The HTML documentation menu.",		NULL, docBrowser },
-      { " Dump Vars",		"(debugging) dump out internal variables.", NULL, dump_variables },
-      { " Emergency shell",	"Start an Emergency Holographic shell.",	NULL, installFixitHoloShell },
-#ifdef WITH_SLICES
-      { " Fdisk",		"The disk Partition Editor",		NULL, diskPartitionEditor },
-#endif
-      { " Fixit",		"Repair mode with CDROM or fixit floppy.",	NULL, dmenuSubmenu, NULL, &MenuFixit },
-      { " FTP sites",		"The FTP mirror site listing.",		NULL, dmenuSubmenu, NULL, &MenuMediaFTP },
-      { " Gateway",		"Set flag to route packets between interfaces.", dmenuVarCheck, dmenuToggleVariable, NULL, "gateway=YES" },
-      { " HTML Docs",		"The HTML documentation menu",		NULL, docBrowser },
-      { " inetd Configuration",	"Configure inetd and simple internet services.",	dmenuVarCheck, configInetd, NULL, "inetd_enable=YES" },
-      { " Install, Standard",	"A standard system installation.",	NULL, installStandard },
-      { " Install, Express",	"An express system installation.",	NULL, installExpress },
-      { " Install, Custom",	"The custom installation menu",		NULL, dmenuSubmenu, NULL, &MenuInstallCustom },
-      { " Label",		"The disk Label editor",		NULL, diskLabelEditor },
-      { " Media",		"Top level media selection menu.",	NULL, dmenuSubmenu, NULL, &MenuMedia },
-      { " Media, NFS",		"Select NFS installation media.",	NULL, mediaSetNFS },
-      { " Media, Floppy",	"Select floppy installation media.",	NULL, mediaSetFloppy },
-      { " Media, CDROM/DVD",	"Select CDROM/DVD installation media.",	NULL, mediaSetCDROM },
-      { " Media, USB",		"Select USB installation media.",	NULL, mediaSetUSB },
-      { " Media, DOS",		"Select DOS installation media.",	NULL, mediaSetDOS },
-      { " Media, UFS",		"Select UFS installation media.",	NULL, mediaSetUFS },
-      { " Media, FTP",		"Select FTP installation media.",	NULL, mediaSetFTP },
-      { " Media, FTP Passive",	"Select passive FTP installation media.", NULL, mediaSetFTPPassive },
-      { " Media, HTTP",		"Select FTP via HTTP proxy install media.", NULL, mediaSetHTTP },
-      { " Network Interfaces",	"Configure network interfaces",		NULL, tcpMenuSelect },
-      { " Networking Services",	"The network services menu.",		NULL, dmenuSubmenu, NULL, &MenuNetworking },
-      { " NFS, client",		"Set NFS client flag.",			dmenuVarCheck, dmenuToggleVariable, NULL, "nfs_client_enable=YES" },
-      { " NFS, server",		"Set NFS server flag.",			dmenuVarCheck, configNFSServer, NULL, "nfs_server_enable=YES" },
-      { " NTP Menu",		"The NTP configuration menu.",		NULL, dmenuSubmenu, NULL, &MenuNTP },
-      { " Options",		"The options editor.",			NULL, optionsEditor },
-      { " Packages",		"The packages collection",		NULL, configPackages },
-#ifdef WITH_SLICES
-      { " Partition",		"The disk slice (PC-style partition) editor",	NULL, diskPartitionEditor },
-#endif
-      { " PCNFSD",		"Run authentication server for PC-NFS.", dmenuVarCheck, configPCNFSD, NULL, "pcnfsd" },
-      { " Root Password",	"Set the system manager's password.",   NULL, dmenuSystemCommand, NULL, "passwd root" },
-      { " Router",		"Select routing daemon (default: routed)", NULL, configRouter, NULL, "router_enable" },
-      { " Security",		"Configure system security options", NULL, dmenuSubmenu, NULL, &MenuSecurity },
-#ifdef WITH_SYSCONS
-      { " Syscons",		"The system console configuration menu.", NULL, dmenuSubmenu, NULL, &MenuSyscons },
-#ifndef PC98
-      { " Syscons, Font",	"The console screen font.",	  NULL, dmenuSubmenu, NULL, &MenuSysconsFont },
-#endif
-      { " Syscons, Keymap",	"The console keymap configuration menu.", NULL, keymapMenuSelect },
-      { " Syscons, Keyrate",	"The console key rate configuration menu.", NULL, dmenuSubmenu, NULL, &MenuSysconsKeyrate },
-      { " Syscons, Saver",	"The console screen saver configuration menu.",	NULL, dmenuSubmenu, NULL, &MenuSysconsSaver },
-#ifndef PC98
-      { " Syscons, Screenmap",	"The console screenmap configuration menu.", NULL, dmenuSubmenu, NULL, &MenuSysconsScrnmap },
-      { " Syscons, Ttys",       "The console terminal type menu.", NULL, dmenuSubmenu, NULL, &MenuSysconsTtys },
-#endif
-#endif /* WITH_SYSCONS */
-      { " Time Zone",		"Set the system's time zone.",		NULL, dmenuSystemCommand, NULL, "tzsetup" },
-      { " TTYs",		"Configure system ttys.",		NULL, configEtcTtys, NULL, "ttys" },
-      { " Upgrade",		"Upgrade an existing system.",		NULL, installUpgrade },
-      { " Usage",		"Quick start - How to use this menu system.",	NULL, dmenuDisplayFile, NULL, "usage" },
-      { " User Management",	"Add user and group information.",	NULL, dmenuSubmenu, NULL, &MenuUsermgmt },
-      { NULL } },
-};
-
-/* The country menu */
-#include "countries.h"
-
-/* The initial installation menu */
-DMenu MenuInitial = {
-    DMENU_NORMAL_TYPE,
-    "sysinstall Main Menu",				/* title */
-    "Welcome to the FreeBSD installation and configuration tool.  Please\n" /* prompt */
-    "select one of the options below by using the arrow keys or typing the\n"
-    "first character of the option name you're interested in.  Invoke an\n"
-    "option with [SPACE] or [ENTER].  To exit, use [TAB] to move to Exit.", 
-    NULL,
-    NULL,
-    { { " Select " },
-      { "X Exit Install",	NULL, NULL, dmenuExit },
-      { " Usage",	"Quick start - How to use this menu system",	NULL, dmenuDisplayFile, NULL, "usage" },
-      { "Standard",	"Begin a standard installation (recommended)",	NULL, installStandard },
-      { "Express",	"Begin a quick installation (for experts)", NULL, installExpress },
-      { " Custom",	"Begin a custom installation (for experts)",	NULL, dmenuSubmenu, NULL, &MenuInstallCustom },
-      { "Configure",	"Do post-install configuration of FreeBSD",	NULL, dmenuSubmenu, NULL, &MenuConfigure },
-      { "Doc",	"Installation instructions, README, etc.",	NULL, dmenuSubmenu, NULL, &MenuDocumentation },
-#ifdef WITH_SYSCONS
-      { "Keymap",	"Select keyboard type",				NULL, keymapMenuSelect },
-#endif
-      { "Options",	"View/Set various installation options",	NULL, optionsEditor },
-      { "Fixit",	"Repair mode with CDROM/DVD/floppy or start shell",	NULL, dmenuSubmenu, NULL, &MenuFixit },
-      { "Upgrade",	"Upgrade an existing system",			NULL, installUpgrade },
-      { "Load Config..","Load default install configuration",		NULL, dispatch_load_menu },
-      { "Index",	"Glossary of functions",			NULL, dmenuSubmenu, NULL, &MenuIndex },
-      { NULL } },
-};
-
-/* The main documentation menu */
-DMenu MenuDocumentation = {
-    DMENU_NORMAL_TYPE,
-    "FreeBSD Documentation Menu",
-    "If you are at all unsure about the configuration of your hardware\n"
-    "or are looking to build a system specifically for FreeBSD, read the\n"
-    "Hardware guide!  New users should also read the Install document for\n"
-    "a step-by-step tutorial on installing FreeBSD.  For general information,\n"
-    "consult the README file.",
-    "Confused?  Press F1 for help.",
-    "usage",
-    { { "X Exit",	"Exit this menu (returning to previous)",	NULL, dmenuExit },
-      { "1 README",	"A general description of FreeBSD.  Read this!", NULL, dmenuDisplayFile, NULL, "README" },
-      { "2 Errata",	"Late-breaking, post-release news.", NULL, dmenuDisplayFile, NULL, "ERRATA" },
-      { "3 Hardware",	"The FreeBSD survival guide for PC hardware.",	NULL, dmenuDisplayFile,	NULL, "HARDWARE" },
-      { "4 Copyright",	"The FreeBSD Copyright notices.",		NULL, dmenuDisplayFile,	NULL, "COPYRIGHT" },
-      { "5 Release"	,"The release notes for this version of FreeBSD.", NULL, dmenuDisplayFile, NULL, "RELNOTES" },
-      { "6 Shortcuts",	"Creating shortcuts to sysinstall.",		NULL, dmenuDisplayFile, NULL, "shortcuts" },
-      { "7 HTML Docs",	"Go to the HTML documentation menu (post-install).", NULL, docBrowser },
-      { NULL } },
-};
-
-/* The FreeBSD documentation installation menu */
-DMenu MenuDocInstall = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "FreeBSD Documentation Installation Menu",
-    "This menu will allow you to install the whole documentation set\n"
-    "from the FreeBSD Documentation Project: Handbook, FAQ and articles.\n\n"
-    "Please select the language versions you wish to install.  At minimum,\n"
-    "you should install the English version, this is the original version\n"
-    "of the documentation.",
-    NULL,
-    NULL,
-    { { "X Exit",	"Exit this menu (returning to previous)",
-	      checkTrue,      dmenuExit, NULL, NULL, '<', '<', '<' },
-      { "All",		"Select all below",
-	      NULL,	      setDocAll, NULL, NULL, ' ', ' ', ' ' },
-      { " bn", 	"Bengali Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_BN },
-      { " da", 	"Danish Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_DA },
-      { " de", 	"German Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_DE },
-      { " el", 	"Greek Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_EL },
-      { " en", 	"English Documentation (recommended)",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_EN },
-      { " es", 	"Spanish Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_ES },
-      { " fr", 	"French Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_FR },
-      { " hu", 	"Hungarian Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_HU },
-      { " it", 	"Italian Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_IT },
-      { " ja", 	"Japanese Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_JA },
-      { " mn", 	"Mongolian Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_MN },
-      { " nl", 	"Dutch Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_NL },
-      { " pl", 	"Polish Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_PL },
-      { " pt", 	"Portuguese Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_PT },
-      { " ru", 	"Russian Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_RU },
-      { " sr", 	"Serbian Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_SR },
-      { " tr", 	"Turkish Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_TR },
-      { " zh_cn", 	"Simplified Chinese Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_ZH_CN },
-      { " zh_tw", 	"Traditional Chinese Documentation",
-	      dmenuFlagCheck, dmenuSetFlag, NULL, &DocDists, '[', 'X', ']', DIST_DOC_ZH_TW },
-      { NULL } },
-};
-
-#ifdef WITH_MICE
-DMenu MenuMouseType = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-#ifdef PC98
-    "Select a protocol type for your mouse",
-    "If your mouse is attached to the bus mouse port, you should always choose\n"
-    "\"Auto\", regardless of the model and the brand of the mouse.  All other\n"
-    "protocol types are for serial mice and should not be used with the bus\n"
-    "mouse.  If you have a serial mouse and are not sure about its protocol,\n"
-    "you should also try \"Auto\".  It may not work for the serial mouse if the\n"
-    "mouse does not support the PnP standard.  But, it won't hurt.  Many\n"
-    "2-button serial mice are compatible with \"Microsoft\" or \"MouseMan\".\n"
-    "3-button serial mice may be compatible with \"MouseSystems\" or \"MouseMan\".\n"
-    "If the serial mouse has a wheel, it may be compatible with \"IntelliMouse\".",
-    NULL,
-    NULL,
-    { { "1 Auto",	"Bus mouse or PnP serial mouse",	
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=auto" },
-#else
-    "Select a protocol type for your mouse",
-    "If your mouse is attached to the PS/2 mouse port or the bus mouse port,\n"
-    "you should always choose \"Auto\", regardless of the model and the brand\n"
-    "of the mouse.  All other protocol types are for serial mice and should\n"
-    "not be used with the PS/2 port mouse or the bus mouse.  If you have\n"
-    "a serial mouse and are not sure about its protocol, you should also try\n"
-    "\"Auto\".  It may not work for the serial mouse if the mouse does not\n"
-    "support the PnP standard.  But, it won't hurt.  Many 2-button serial mice\n"
-    "are compatible with \"Microsoft\" or \"MouseMan\".  3-button serial mice\n"
-    "may be compatible with \"MouseSystems\" or \"MouseMan\".  If the serial\n"
-    "mouse has a wheel, it may be compatible with \"IntelliMouse\".",
-    NULL,
-    NULL,
-    { { "1 Auto",	"Bus mouse, PS/2 style mouse or PnP serial mouse",	
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=auto" },
-#endif /* PC98 */
-      { "2 GlidePoint",	"ALPS GlidePoint pad (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=glidepoint" },
-      { "3 Hitachi","Hitachi tablet (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=mmhittab" },
-      { "4 IntelliMouse",	"Microsoft IntelliMouse (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=intellimouse" },
-      { "5 Logitech",	"Logitech protocol (old models) (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=logitech" },
-      { "6 Microsoft",	"Microsoft protocol (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=microsoft" },
-      { "7 MM Series","MM Series protocol (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=mmseries" },
-      { "8 MouseMan",	"Logitech MouseMan/TrackMan models (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=mouseman" },
-      { "9 MouseSystems",	"MouseSystems protocol (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=mousesystems" },
-      { "A ThinkingMouse","Kensington ThinkingMouse (serial)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_TYPE "=thinkingmouse" },
-      { NULL } },
-};
-
-#ifdef PC98
-DMenu MenuMousePort = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Select your mouse port from the following menu",
-    "The built-in pointing device of laptop/notebook computers is usually\n"
-    "a BusMouse style device.",
-    NULL,
-    NULL,
-    {
-      { "1 BusMouse",	"PC-98x1 bus mouse (/dev/mse0)", 
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/mse0" },
-      { "2 COM1",	"Serial mouse on COM1 (/dev/cuau0)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/cuau0" },
-      { "3 COM2",	"Serial mouse on COM2 (/dev/cuau1)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/cuau1" },
-      { NULL } },
-};
-#else
-DMenu MenuMousePort = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Select your mouse port from the following menu",
-    "The built-in pointing device of laptop/notebook computers is usually\n"
-    "a PS/2 style device.",
-    NULL,
-    NULL,
-    { { "1 PS/2",	"PS/2 style mouse (/dev/psm0)", 
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/psm0" },
-      { "2 COM1",	"Serial mouse on COM1 (/dev/cuau0)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/cuau0" },
-      { "3 COM2",	"Serial mouse on COM2 (/dev/cuau1)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/cuau1" },
-      { "4 COM3",	"Serial mouse on COM3 (/dev/cuau2)",
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/cuau2" },
-      { "5 COM4",	"Serial mouse on COM4 (/dev/cuau3)", 
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/cuau3" },
-      { "6 BusMouse",	"Logitech, ATI or MS bus mouse (/dev/mse0)", 
-	dmenuVarCheck, dmenuSetVariable, NULL, VAR_MOUSED_PORT "=/dev/mse0" },
-      { NULL } },
-};
-#endif /* PC98 */
-
-DMenu MenuMouse = {
-    DMENU_NORMAL_TYPE,
-    "Please configure your mouse",
-    "You can cut and paste text in the text console by running the mouse\n"
-    "daemon.  Specify a port and a protocol type of your mouse and enable\n"
-    "the mouse daemon.  If you don't want this feature, select 6 to disable\n"
-    "the daemon.\n"
-    "Once you've enabled the mouse daemon, you can specify \"/dev/sysmouse\"\n"
-    "as your mouse device and \"SysMouse\" or \"MouseSystems\" as mouse\n"
-    "protocol when running the X configuration utility (see Configuration\n"
-    "menu).",
-    NULL,
-    NULL,
-    { { "X Exit", "Exit this menu (returning to previous)", NULL, dmenuExit },
-      { "2 Enable",	"Test and run the mouse daemon", NULL, mousedTest, NULL, NULL },
-      { "3 Type",	"Select mouse protocol type", NULL, dmenuSubmenu, NULL, &MenuMouseType },
-      { "4 Port",	"Select mouse port", NULL, dmenuSubmenu, NULL, &MenuMousePort },
-      { "5 Flags",      "Set additional flags", dmenuVarCheck, setMouseFlags,
-	NULL, VAR_MOUSED_FLAGS "=" },
-      { "6 Disable",	"Disable the mouse daemon", NULL, mousedDisable, NULL, NULL },
-      { NULL } },
-};
-#endif /* WITH_MICE */
-
-DMenu MenuMediaCDROM = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Choose a CD/DVD type",
-    "FreeBSD can be installed directly from a CD/DVD containing a valid\n"
-    "FreeBSD distribution.  If you are seeing this menu it is because\n"
-    "more than one CD/DVD drive was found on your system.  Please select one\n"
-    "of the following CD/DVD drives as your installation drive.",
-    NULL,
-    NULL,
-    { { NULL } },
-};
-
-DMenu MenuMediaFloppy = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Choose a Floppy drive",
-    "You have more than one floppy drive.  Please choose which drive\n"
-    "you would like to use.",
-    NULL,
-    NULL,
-    { { NULL } },
-};
-
-DMenu MenuMediaUSB = {
-	DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-	"Choose a USB drive",
-	"You have more than one USB drive. Please choose which drive\n"
-	"you would like to use.",
-	NULL,
-	NULL,
-	{ { NULL } },
-};
-
-DMenu MenuMediaDOS = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Choose a DOS partition",
-    "FreeBSD can be installed directly from a DOS partition\n"
-    "assuming, of course, that you have copied the relevant\n"
-    "distributions into your DOS partition before starting this\n"
-    "installation.  If this is not the case then you should reboot\n"
-    "DOS at this time and copy the distributions you wish to install\n"
-    "into a \"FREEBSD\" subdirectory on one of your DOS partitions.\n"
-    "Otherwise, please select the DOS partition containing the FreeBSD\n"
-    "distribution files.",
-    NULL,
-    NULL,
-    { { NULL } },
-};
-
-DMenu MenuMediaFTP = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Please select a FreeBSD FTP distribution site",
-    "Please select the site closest to you or \"other\" if you'd like to\n"
-    "specify a different choice.  Also note that not every site listed here\n"
-    "carries more than the base distribution kits. Only Primary sites are\n"
-    "guaranteed to carry the full range of possible distributions.",
-    "Select a site that's close!",
-    NULL,
-    { { "Main Site",	"ftp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.freebsd.org" },
-      { "URL", "Specify some other ftp site by URL", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=other" },
-      { "Snapshots Server Japan", "snapshots.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://snapshots.jp.freebsd.org" },
-      { "Snapshots Server Sweden", "snapshots.se.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://snapshots.se.freebsd.org" },
-
-      { "IPv6 Main Site", "ftp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.freebsd.org" },
-      { " IPv6 Ireland", "ftp3.ie.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.ie.freebsd.org" },
-      { " IPv6 Israel", "ftp.il.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.il.freebsd.org" },
-      { " IPv6 Japan", "ftp2.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.jp.freebsd.org" },
-      { " IPv6 USA", "ftp4.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.us.freebsd.org" },
-      { " IPv6 Turkey", "ftp2.tr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.tr.freebsd.org" },
-
-      { "Primary",	"ftp1.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp1.freebsd.org" },
-      { " Primary #2",	"ftp2.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.freebsd.org" },
-      { " Primary #3",	"ftp3.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.freebsd.org" },
-      { " Primary #4",	"ftp4.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.freebsd.org" },
-      { " Primary #5",	"ftp5.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.freebsd.org" },
-      { " Primary #6",	"ftp6.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.freebsd.org" },
-      { " Primary #7",	"ftp7.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp7.freebsd.org" },
-      { " Primary #8",	"ftp8.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp8.freebsd.org" },
-      { " Primary #9",	"ftp9.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp9.freebsd.org" },
-      { " Primary #10",	"ftp10.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp10.freebsd.org" },
-      { " Primary #11",	"ftp11.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp11.freebsd.org" },
-      { " Primary #12",	"ftp12.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp12.freebsd.org" },
-      { " Primary #13",	"ftp13.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp13.freebsd.org" },
-      { " Primary #14",	"ftp14.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp14.freebsd.org" },
-
-      { "Argentina",	"ftp.ar.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ar.freebsd.org" },
-
-      { "Australia",	"ftp.au.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.au.freebsd.org" },
-      { " Australia #2","ftp2.au.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.au.freebsd.org" },
-      { " Australia #3","ftp3.au.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.au.freebsd.org" },
-
-      { "Austria","ftp.at.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.at.freebsd.org" },
-      { " Austria #2","ftp2.at.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.at.freebsd.org" },
-
-      { "Brazil",	"ftp.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.br.freebsd.org" },
-      { " Brazil #2",	"ftp2.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.br.freebsd.org" },
-      { " Brazil #3",	"ftp3.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.br.freebsd.org" },
-      { " Brazil #4",	"ftp4.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.br.freebsd.org" },
-      { " Brazil #5",	"ftp5.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.br.freebsd.org" },
-      { " Brazil #6",	"ftp6.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.br.freebsd.org" },
-      { " Brazil #7",	"ftp7.br.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp7.br.freebsd.org" },
-
-      { "Canada",	"ftp.ca.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ca.freebsd.org" },
-
-      { "China",	"ftp.cn.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.cn.freebsd.org" },
-      { " China #2",	"ftp2.cn.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.cn.freebsd.org" },
-
-      { "Croatia",	"ftp.hr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.hr.freebsd.org" },
-
-      { "Czech Republic", "ftp.cz.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.cz.freebsd.org" },
-
-      { "Denmark",	"ftp.dk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.dk.freebsd.org" },
-      { " Denmark #2",	"ftp2.dk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.dk.freebsd.org" },
-
-      { "Estonia",	"ftp.ee.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ee.freebsd.org" },
-
-      { "Finland",	"ftp.fi.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.fi.freebsd.org" },
-
-      { "France",	"ftp.fr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.fr.freebsd.org" },
-      { " France #2",	"ftp2.fr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.fr.freebsd.org" },
-      { " France #3",	"ftp3.fr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.fr.freebsd.org" },
-      { " France #5",	"ftp5.fr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.fr.freebsd.org" },
-      { " France #6",	"ftp6.fr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.fr.freebsd.org" },
-      { " France #8",	"ftp8.fr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp8.fr.freebsd.org" },
-
-      { "Germany",	"ftp.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.de.freebsd.org" },
-      { " Germany #2",	"ftp2.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.de.freebsd.org" },
-      { " Germany #3",	"ftp3.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.de.freebsd.org" },
-      { " Germany #4",	"ftp4.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.de.freebsd.org" },
-      { " Germany #5",	"ftp5.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.de.freebsd.org" },
-      { " Germany #6",	"ftp6.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.de.freebsd.org" },
-      { " Germany #7",	"ftp7.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp7.de.freebsd.org" },
-      { " Germany #8",	"ftp8.de.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp8.de.freebsd.org" },
-
-      { "Greece",	"ftp.gr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.gr.freebsd.org" },
-      { " Greece #2",	"ftp2.gr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.gr.freebsd.org" },
-
-      { "Hungary",     "ftp.hu.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.hu.freebsd.org" },
-
-      { "Iceland",	"ftp.is.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.is.freebsd.org" },
-
-      { "Ireland",	"ftp.ie.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ie.freebsd.org" },
-      { " Ireland #2",	"ftp2.ie.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.ie.freebsd.org" },
-      { " Ireland #3",	"ftp3.ie.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.ie.freebsd.org" },
-
-      { "Israel",	"ftp.il.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.il.freebsd.org" },
-
-      { "Italy",	"ftp.it.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.it.freebsd.org" },
-
-      { "Japan",	"ftp.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.jp.freebsd.org" },
-      { " Japan #2",	"ftp2.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.jp.freebsd.org" },
-      { " Japan #3",	"ftp3.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.jp.freebsd.org" },
-      { " Japan #4",	"ftp4.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.jp.freebsd.org" },
-      { " Japan #5",	"ftp5.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.jp.freebsd.org" },
-      { " Japan #6",	"ftp6.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.jp.freebsd.org" },
-      { " Japan #7",	"ftp7.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp7.jp.freebsd.org" },
-      { " Japan #8",	"ftp8.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp8.jp.freebsd.org" },
-      { " Japan #9",	"ftp9.jp.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp9.jp.freebsd.org" },
-
-      { "Korea",	"ftp.kr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.kr.freebsd.org" },
-      { " Korea #2",	"ftp2.kr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.kr.freebsd.org" },
-
-      { "Lithuania",	"ftp.lt.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.lt.freebsd.org" },
-
-      { "Netherlands",	"ftp.nl.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.nl.freebsd.org" },
-      { " Netherlands #2",	"ftp2.nl.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.nl.freebsd.org" },
-
-      { "Norway",	"ftp.no.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.no.freebsd.org" },
-      { " Norway #3",	"ftp3.no.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.no.freebsd.org" },
-
-      { "Poland",	"ftp.pl.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.pl.freebsd.org" },
-      { " Poland #2",	"ftp2.pl.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.pl.freebsd.org" },
-      { " Poland #5",	"ftp5.pl.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.pl.freebsd.org" },
-
-      { "Portugal",	"ftp.pt.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.pt.freebsd.org" },
-      { " Portugal #2",	"ftp2.pt.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.pt.freebsd.org" },
-      { " Portugal #4",	"ftp4.pt.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.pt.freebsd.org" },
-
-      { "Romania",	"ftp.ro.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ro.freebsd.org" },
-
-      { "Russia",	"ftp.ru.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ru.freebsd.org" },
-      { " Russia #2",	"ftp2.ru.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.ru.freebsd.org" },
-      { " Russia #3",	"ftp3.ru.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.ru.freebsd.org" },
-      { " Russia #4",	"ftp4.ru.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.ru.freebsd.org" },
-
-      { "Singapore",	"ftp.sg.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.sg.freebsd.org" },
-
-      { "Slovak Republic",	"ftp.sk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.sk.freebsd.org" },
-
-      { "Slovenia",	"ftp.si.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.si.freebsd.org" },
-      { " Slovenia #2",	"ftp2.si.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.si.freebsd.org" },
-
-      { "South Africa",	"ftp.za.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.za.freebsd.org" },
-      { " South Africa #2", "ftp2.za.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.za.freebsd.org" },
-      { " South Africa #3", "ftp3.za.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.za.freebsd.org" },
-      { " South Africa #4", "ftp4.za.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.za.freebsd.org" },
-
-      { "Spain",	"ftp.es.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.es.freebsd.org" },
-      { " Spain #2",	"ftp2.es.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.es.freebsd.org" },
-      { " Spain #3",	"ftp3.es.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.es.freebsd.org" },
-
-      { "Sweden",	"ftp.se.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.se.freebsd.org" },
-      { " Sweden #2",	"ftp2.se.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.se.freebsd.org" },
-      { " Sweden #3",	"ftp3.se.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.se.freebsd.org" },
-      { " Sweden #4",	"ftp4.se.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.se.freebsd.org" },
-      { " Sweden #5",	"ftp5.se.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.se.freebsd.org" },
-
-      { "Switzerland",	"ftp.ch.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ch.freebsd.org" },
-      { " Switzerland #2",	"ftp2.ch.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.ch.freebsd.org" },
-
-      { "Taiwan",	"ftp.tw.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.tw.freebsd.org" },
-      { " Taiwan #2",	"ftp2.tw.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.tw.freebsd.org" },
-      { " Taiwan #3",	"ftp3.tw.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.tw.freebsd.org" },
-      { " Taiwan #4",   "ftp4.tw.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.tw.freebsd.org" },
-      { " Taiwan #6",   "ftp6.tw.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.tw.freebsd.org" },
-      { " Taiwan #11",   "ftp11.tw.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp11.tw.freebsd.org" },
-
-      { "Turkey",	"ftp.tr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.tr.freebsd.org" },
-      { " Turkey #2",	"ftp2.tr.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.tr.freebsd.org" },
-
-      { "UK",		"ftp.uk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.uk.freebsd.org" },
-      { " UK #2",	"ftp2.uk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.uk.freebsd.org" },
-      { " UK #3",	"ftp3.uk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.uk.freebsd.org" },
-      { " UK #4",	"ftp4.uk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.uk.freebsd.org" },
-      { " UK #5",	"ftp5.uk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.uk.freebsd.org" },
-      { " UK #6",	"ftp6.uk.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.uk.freebsd.org" },
-
-      { "Ukraine",	"ftp.ua.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp.ua.freebsd.org" },
-      { " Ukraine #2",	"ftp2.ua.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.ua.freebsd.org" },
-      { " Ukraine #5",	"ftp5.ua.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.ua.freebsd.org" },
-      { " Ukraine #6",	"ftp6.ua.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.ua.freebsd.org" },
-      { " Ukraine #7",	"ftp7.ua.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp7.ua.freebsd.org" },
-      { " Ukraine #8",	"ftp8.ua.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp8.ua.freebsd.org" },
-
-      { "USA #1",	"ftp1.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp1.us.freebsd.org" },
-      { " USA #2",	"ftp2.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp2.us.freebsd.org" },
-      { " USA #3",	"ftp3.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp3.us.freebsd.org" },
-      { " USA #4",	"ftp4.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp4.us.freebsd.org" },
-      { " USA #5",	"ftp5.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp5.us.freebsd.org" },
-      { " USA #6",	"ftp6.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp6.us.freebsd.org" },
-      { " USA #7",	"ftp7.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp7.us.freebsd.org" },
-      { " USA #8",	"ftp8.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp8.us.freebsd.org" },
-      { " USA #9",	"ftp9.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp9.us.freebsd.org" },
-      { " USA #10",	"ftp10.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp10.us.freebsd.org" },
-      { " USA #11",	"ftp11.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp11.us.freebsd.org" },
-      { " USA #12",	"ftp12.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp12.us.freebsd.org" },
-      { " USA #13",	"ftp13.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp13.us.freebsd.org" },
-      { " USA #14",	"ftp14.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp14.us.freebsd.org" },
-      { " USA #15",	"ftp15.us.freebsd.org", NULL, dmenuSetVariable, NULL,
-	VAR_FTP_PATH "=ftp://ftp15.us.freebsd.org" },
-
-      { NULL } }
-};
-
-DMenu MenuNetworkDevice = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Network interface information required",
-    "Please select the ethernet or PLIP device to configure.\n\n"
-    "",
-    "Press F1 to read network configuration manual",
-    "network_device",
-    { { NULL } },
-};
-
-/* Prototype KLD load menu */
-DMenu MenuKLD = {
-    DMENU_NORMAL_TYPE,
-    "KLD Menu",
-    "Load a KLD from a floppy\n",
-    NULL,
-    NULL,
-    { { NULL } },
-};
-
-/* Prototype config file load menu */
-DMenu MenuConfig = {
-    DMENU_NORMAL_TYPE,
-    "Config Menu",
-    "Please select the device to load your configuration file from.\n"
-    "Note that a USB key will show up as daNs1.",
-    NULL,
-    NULL,
-    { { NULL } },
-};
-
-/* The media selection menu */
-DMenu MenuMedia = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Choose Installation Media",
-    "FreeBSD can be installed from a variety of different installation\n"
-    "media, ranging from floppies to an Internet FTP server.  If you're\n"
-    "installing FreeBSD from a supported CD/DVD drive then this is generally\n"
-    "the best media to use if you have no overriding reason for using other\n"
-    "media.",
-    "Press F1 for more information on the various media types",
-    "media",
-    { { "1 CD/DVD",		"Install from a FreeBSD CD/DVD",	NULL, mediaSetCDROM },
-      { "2 FTP",		"Install from an FTP server",		NULL, mediaSetFTPActive },
-      { "3 FTP Passive",	"Install from an FTP server through a firewall", NULL, mediaSetFTPPassive },
-      { "4 HTTP",		"Install from an FTP server through a http proxy", NULL, mediaSetHTTP },
-      { "5 DOS",		"Install from a DOS partition",		NULL, mediaSetDOS },
-      { "6 NFS",		"Install over NFS",			NULL, mediaSetNFS },
-      { "7 File System",	"Install from an existing filesystem",	NULL, mediaSetUFS },
-      { "8 Floppy",		"Install from a floppy disk set",	NULL, mediaSetFloppy },
-      { "9 USB",		"Install from a USB drive",		NULL, mediaSetUSB },
-      { "X Options",		"Go to the Options screen",		NULL, optionsEditor },
-      { NULL } },
-};
-
-/* The distributions menu */
-DMenu MenuDistributions = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Choose Distributions",
-    "As a convenience, we provide several \"canned\" distribution sets.\n"
-    "These select what we consider to be the most reasonable defaults for the\n"
-    "type of system in question.  If you would prefer to pick and choose the\n"
-    "list of distributions yourself, simply select \"Custom\".  You can also\n"
-    "pick a canned distribution set and then fine-tune it with the Custom item.\n\n"
-    "Choose an item by pressing [SPACE] or [ENTER].  When finished, choose the\n"
-    "Exit item or move to the OK button with [TAB].",
-    "Press F1 for more information on these options.",
-    "distributions",
-    { { "X Exit", "Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { "All",			"All system sources and binaries",
-	checkDistEverything,	distSetEverything, NULL, NULL, ' ', ' ', ' ' },
-      { "Reset",		"Reset selected distribution list to nothing",
-	NULL,			distReset, NULL, NULL, ' ', ' ', ' ' },
-      { "4 Developer",		"Full sources, binaries and doc but no games", 
-	checkDistDeveloper,	distSetDeveloper },
-      { "5 Kern-Developer",	"Full binaries and doc, kernel sources only",
-	checkDistKernDeveloper, distSetKernDeveloper },
-      { "6 User",		"Average user - binaries and doc only",
-	checkDistUser,		distSetUser },
-      { "7 Minimal",		"The smallest configuration possible",
-	checkDistMinimum,	distSetMinimum },
-      { "8 Custom",		"Specify your own distribution set",
-	NULL,			dmenuSubmenu, NULL, &MenuSubDistributions, '>', '>', '>' },
-      { NULL } },
-};
-
-DMenu MenuSubDistributions = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Select the distributions you wish to install.",
-    "Please check off the distributions you wish to install.  At the\n"
-    "very minimum, this should be \"base\".",
-    NULL,
-    NULL,
-    { { "X Exit", "Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { "All",		"All system sources and binaries",
-	NULL, distSetEverything, NULL, NULL, ' ', ' ', ' ' },
-      { "Reset",	"Reset all of the below",
-	NULL, distReset, NULL, NULL, ' ', ' ', ' ' },
-      { " base",	"Binary base distribution (required)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_BASE },
-      { " kernels",	"Binary kernel distributions (required)",
-	kernelFlagCheck,distSetKernel },
-      { " dict",	"Spelling checker dictionary files",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_DICT },
-      { " doc",		"FreeBSD Documentation set",
-	docFlagCheck,	distSetDoc },
-      { " docuser",		"Miscellaneous userland docs",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_DOCUSERLAND },
-      { " games",	"Games (non-commercial)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_GAMES },
-      { " info",	"GNU info files",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_INFO },
-#if defined(__amd64__) || defined(__powerpc64__)
-      { " lib32",	"32-bit runtime compatibility libraries",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_LIB32 },
-#endif
-      { " man",		"System manual pages - recommended",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_MANPAGES },
-      { " catman",	"Preformatted system manual pages",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_CATPAGES },
-      { " proflibs",	"Profiled versions of the libraries",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_PROFLIBS },
-      { " src",		"Sources for everything",
-	srcFlagCheck,	distSetSrc },
-      { " ports",	"The FreeBSD Ports collection",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_PORTS },
-      { " local",	"Local additions collection",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &Dists, '[', 'X', ']', DIST_LOCAL},
-      { NULL } },
-};
-
-DMenu MenuKernelDistributions = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Select the operating system kernels you wish to install.",
-    "Please check off those kernels you wish to install.\n",
-    NULL,
-    NULL,
-    { { "X Exit", "Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { "All",		"Select all of the below",
-	NULL,		setKernel, NULL, NULL, ' ', ' ', ' ' },
-      { "Reset",	"Reset all of the below",
-	NULL,		clearKernel, NULL, NULL, ' ', ' ', ' ' },
-      { " GENERIC",	"GENERIC kernel configuration",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &KernelDists, '[', 'X', ']', DIST_KERNEL_GENERIC },
-      { NULL } },
-};
-
-DMenu MenuSrcDistributions = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Select the sub-components of src you wish to install.",
-    "Please check off those portions of the FreeBSD source tree\n"
-    "you wish to install.",
-    NULL,
-    NULL,
-    { { "X Exit", "Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { "All",		"Select all of the below",
-	NULL,		setSrc, NULL, NULL, ' ', ' ', ' ' },
-      { "Reset",	"Reset all of the below",
-	NULL,		clearSrc, NULL, NULL, ' ', ' ', ' ' },
-      { " base",	"top-level files in /usr/src",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_BASE },
-      { " cddl",	"/usr/src/cddl (software from Sun)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_CDDL },
-      { " contrib",	"/usr/src/contrib (contributed software)",
-	dmenuFlagCheck,	dmenuSetFlag,	NULL, &SrcDists, '[', 'X', ']', DIST_SRC_CONTRIB },
-      { " crypto",	"/usr/src/crypto (contrib encryption sources)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_SCRYPTO },
-      { " gnu",		"/usr/src/gnu (software from the GNU Project)",
-	dmenuFlagCheck,	dmenuSetFlag,	NULL, &SrcDists, '[', 'X', ']', DIST_SRC_GNU },
-      { " etc",		"/usr/src/etc (miscellaneous system files)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_ETC },
-      { " games",	"/usr/src/games (the obvious!)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_GAMES },
-      { " include",	"/usr/src/include (header files)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_INCLUDE },
-      { " krb5",	"/usr/src/kerberos5 (sources for Kerberos5)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_SKERBEROS5 },
-      { " lib",		"/usr/src/lib (system libraries)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_LIB },
-      { " libexec",	"/usr/src/libexec (system programs)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_LIBEXEC },
-      { " release",	"/usr/src/release (release-generation tools)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_RELEASE },
-      { " rescue",	"/usr/src/rescue (static rescue tools)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_RESCUE },
-      { " bin",		"/usr/src/bin (system binaries)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_BIN },
-      { " sbin",	"/usr/src/sbin (system binaries)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_SBIN },
-      { " secure",	"/usr/src/secure (BSD encryption sources)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_SSECURE },
-      { " share",	"/usr/src/share (documents and shared files)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_SHARE },
-      { " sys",		"/usr/src/sys (FreeBSD kernel)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_SYS },
-      { " tools",	"/usr/src/tools (miscellaneous tools)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_TOOLS },
-      { " ubin",	"/usr/src/usr.bin (user binaries)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_UBIN },
-      { " usbin",	"/usr/src/usr.sbin (aux system binaries)",
-	dmenuFlagCheck,	dmenuSetFlag, NULL, &SrcDists, '[', 'X', ']', DIST_SRC_USBIN },
-      { NULL } },
-};
-
-DMenu MenuDiskDevices = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Select Drive(s)",
-    "Please select the drive, or drives, on which you wish to perform\n"
-    "this operation.  If you are attempting to install a boot partition\n"
-    "on a drive other than the first one or have multiple operating\n"
-    "systems on your machine, you will have the option to install a boot\n"
-    "manager later.  To select a drive, use the arrow keys to move to it\n"
-    "and press [SPACE] or [ENTER].  To de-select it, press it again.\n\n"
-    "Use [TAB] to get to the buttons and leave this menu.",
-    "Press F1 for important information regarding disk geometry!",
-    "drives",
-    { { NULL } },
-};
-
-DMenu MenuHTMLDoc = {
-    DMENU_NORMAL_TYPE,
-    "Select HTML Documentation pointer",
-    "Please select the body of documentation you're interested in, the main\n"
-    "ones right now being the FAQ and the Handbook.  You can also choose \"other\"\n"
-    "to enter an arbitrary URL for browsing.",
-    "Press F1 for more help on what you see here.",
-    "html",
-    { { "X Exit",	"Exit this menu (returning to previous)", NULL,	dmenuExit },
-      { "2 Handbook",	"The FreeBSD Handbook.",				NULL, docShowDocument },
-      { "3 FAQ",	"The Frequently Asked Questions guide.",		NULL, docShowDocument },
-      { "4 Home",	"The Home Pages for the FreeBSD Project (requires net)", NULL, docShowDocument },
-      { "5 Other",	"Enter a URL.",						NULL, docShowDocument },
-      { NULL } },
-};
-
-/* The main installation menu */
-DMenu MenuInstallCustom = {
-    DMENU_NORMAL_TYPE,
-    "Choose Custom Installation Options",
-    "This is the custom installation menu. You may use this menu to specify\n"
-    "details on the type of distribution you wish to have, where you wish\n"
-    "to install it from and how you wish to allocate disk storage to FreeBSD.",
-    NULL,
-    NULL,
-    { { "X Exit",		"Exit this menu (returning to previous)", NULL,	dmenuExit },
-      { "2 Options",		"View/Set various installation options", NULL, optionsEditor },
-#ifndef WITH_SLICES
-      { "3 Label",		"Label disk partitions",		NULL, diskLabelEditor },
-      { "4 Distributions",	"Select distribution(s) to extract",	NULL, dmenuSubmenu, NULL, &MenuDistributions },
-      { "5 Media",		"Choose the installation media type",	NULL, dmenuSubmenu, NULL, &MenuMedia },
-      { "6 Commit",		"Perform any pending Partition/Label/Extract actions", NULL, installCustomCommit },
-#else
-      { "3 Partition",		"Allocate disk space for FreeBSD",	NULL, diskPartitionEditor },
-      { "4 Label",		"Label allocated disk partitions",	NULL, diskLabelEditor },
-      { "5 Distributions",	"Select distribution(s) to extract",	NULL, dmenuSubmenu, NULL, &MenuDistributions },
-      { "6 Media",		"Choose the installation media type",	NULL, dmenuSubmenu, NULL, &MenuMedia },
-      { "7 Commit",		"Perform any pending Partition/Label/Extract actions", NULL, installCustomCommit },
-#endif
-      { NULL } },
-};
-
-#if defined(__i386__) || defined(__amd64__)
-#ifdef PC98
-/* IPL type menu */
-DMenu MenuIPLType = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "overwrite me",		/* will be disk specific label */
-    "If you want a FreeBSD Boot Manager, select \"BootMgr\".  If you would\n"
-    "prefer your Boot Manager to remain untouched then select \"None\".\n\n",
-    "Press F1 to read about drive setup",
-    "drives",
-    { { "BootMgr",	"Install the FreeBSD Boot Manager",
-	dmenuRadioCheck, dmenuSetValue, NULL, &BootMgr },
-      { "None",		"Leave the IPL untouched",
-	dmenuRadioCheck, dmenuSetValue, NULL, &BootMgr, '(', '*', ')', 1 },
-      { NULL } },
-};
-#else
-/* MBR type menu */
-DMenu MenuMBRType = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "overwrite me",		/* will be disk specific label */
-    "FreeBSD comes with a boot manager that allows you to easily\n"
-    "select between FreeBSD and any other operating systems on your machine\n"
-    "at boot time.  If you have more than one drive and want to boot\n"
-    "from the second one, the boot manager will also make it possible\n"
-    "to do so (limitations in the PC BIOS usually prevent this otherwise).\n"
-    "If you have other operating systems installed and would like a choice when\n"
-    "booting, choose \"BootMgr\". If you would prefer to keep your existing\n"
-    "boot manager, select \"None\".\n\n",
-    "",    
-    "drives",
-    { { "Standard",	"Install a standard MBR (non-interactive boot manager)",
-	dmenuRadioCheck, dmenuSetValue, NULL, &BootMgr, '(', '*', ')', 1 },
-      { "BootMgr",	"Install the FreeBSD Boot Manager",
-	dmenuRadioCheck, dmenuSetValue, NULL, &BootMgr, '(', '*', ')', 0 },
-      { "None",		"Do not install a boot manager",
-	dmenuRadioCheck, dmenuSetValue, NULL, &BootMgr, '(', '*', ')', 2 },
-      { NULL } },
-};
-#endif /* PC98 */
-#endif /* __i386__ */
-
-/* Final configuration menu */
-DMenu MenuConfigure = {
-    DMENU_NORMAL_TYPE,
-    "FreeBSD Configuration Menu",	/* title */
-    "If you've already installed FreeBSD, you may use this menu to customize\n"
-    "it somewhat to suit your particular configuration.  Most importantly,\n"
-    "you can use the Packages utility to load extra \"3rd party\"\n"
-    "software not provided in the base distributions.",
-    "Press F1 for more information on these options",
-    "configure",
-    { { "X Exit",		"Exit this menu (returning to previous)",
-	NULL,	dmenuExit },
-      { " Distributions", "Install additional distribution sets",
-	NULL, distExtractAll },
-      { " Documentation installation", "Install FreeBSD Documentation set",
-	NULL, distSetDocMenu },
-      { " Packages",	"Install pre-packaged software for FreeBSD",
-	NULL, configPackages },
-      { " Root Password", "Set the system manager's password",
-	NULL,	dmenuSystemCommand, NULL, "passwd root" },
-#ifdef WITH_SLICES
-      { " Fdisk",	"The disk slice (PC-style partition) editor",
-	NULL, diskPartitionEditor },
-#endif
-      { " Label",	"The disk label editor",
-	NULL, diskLabelEditor },
-      { " User Management",	"Add user and group information",
-	NULL, dmenuSubmenu, NULL, &MenuUsermgmt },
-#ifdef WITH_SYSCONS
-      { " Console",	"Customize system console behavior",
-	NULL,	dmenuSubmenu, NULL, &MenuSyscons },
-#endif
-      { " Time Zone",	"Set which time zone you're in",
-	NULL,	dmenuSystemCommand, NULL, "tzsetup" },
-      { " Media",	"Change the installation media type",
-	NULL,	dmenuSubmenu, NULL, &MenuMedia },
-#ifdef WITH_MICE
-      { " Mouse",	"Configure your mouse",
-	NULL,	dmenuSubmenu, NULL, &MenuMouse },
-#endif
-      { " Networking",	"Configure additional network services",
-	NULL,	dmenuSubmenu, NULL, &MenuNetworking },
-      { " Security",	"Configure system security options",
-	NULL,	dmenuSubmenu, NULL, &MenuSecurity },
-      { " Startup",	"Configure system startup options",
-	NULL,	dmenuSubmenu, NULL, &MenuStartup },
-      { " TTYs",	"Configure system ttys.",
-	NULL,	configEtcTtys, NULL, "ttys" },
-      { " Options",	"View/Set various installation options",
-	NULL, optionsEditor },
-      { " HTML Docs",	"Go to the HTML documentation menu (post-install)",
-	NULL, docBrowser },
-      { " Load KLD",	"Load a KLD from a floppy",
-	NULL, kldBrowser },
-      { NULL } },
-};
-
-DMenu MenuStartup = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Startup Services Menu",
-    "This menu allows you to configure various aspects of your system's\n"
-    "startup configuration.  Use [SPACE] or [ENTER] to select items, and\n"
-    "[TAB] to move to the buttons.  Select Exit to leave this menu.",
-    NULL,
-    NULL,
-    { { "X Exit",	"Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-#ifdef __i386__
-      { " APM",		"Auto-power management services (typically laptops)",
-	dmenuVarCheck,	dmenuToggleVariable, NULL, "apm_enable=YES" },
-#endif
-      { " ",		" -- ", NULL,	NULL, NULL, NULL, ' ', ' ', ' ' },
-      { " Startup dirs",	"Set the list of dirs to look for startup scripts",
-	dmenuVarCheck, dmenuISetVariable, NULL, "local_startup" },
-      { " named",	"Run a local name server on this host",
-	dmenuVarCheck, dmenuToggleVariable, NULL, "named_enable=YES" },
-      { " named flags",	"Set default flags to named (if enabled)",
-	dmenuVarCheck, dmenuISetVariable, NULL, "named_flags" },
-      { " NIS client",	"This host wishes to be an NIS client.",
-	dmenuVarCheck, configRpcBind, NULL, "nis_client_enable=YES" },
-      { " NIS domainname",	"Set NIS domainname (if enabled)",
-	dmenuVarCheck, dmenuISetVariable, NULL, "nisdomainname" },
-      { " NIS server",	"This host wishes to be an NIS server.",
-	dmenuVarCheck, configRpcBind, NULL, "nis_server_enable=YES" },
-      { " ",		" -- ", NULL,	NULL, NULL, NULL, ' ', ' ', ' ' },
-      { " Accounting",	"This host wishes to run process accounting.",
-	dmenuVarCheck, dmenuToggleVariable, NULL, "accounting_enable=YES" },
-      { " lpd",		"This host has a printer and wants to run lpd.",
-	dmenuVarCheck, dmenuToggleVariable, NULL, "lpd_enable=YES" },
-#ifdef __i386__
-      { " SCO",		"This host wants to be able to run IBCS2 binaries.",
-	dmenuVarCheck, dmenuToggleVariable, NULL, "ibcs2_enable=YES" },
-      { " SVR4",	"This host wants to be able to run SVR4 binaries.",
-	dmenuVarCheck, dmenuToggleVariable, NULL, "svr4_enable=YES" },
-#endif
-      { NULL } },
-};
-
-DMenu MenuNetworking = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "Network Services Menu",
-    "You may have already configured one network device (and the other\n"
-    "various hostname/gateway/name server parameters) in the process\n"
-    "of installing FreeBSD.  This menu allows you to configure other\n"
-    "aspects of your system's network configuration.",
-    NULL,
-    NULL,
-    { { "X Exit",	"Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { " Interfaces",	"Configure additional network interfaces",
-	NULL, tcpMenuSelect },
-      { " AMD",		"This machine wants to run the auto-mounter service",
-	dmenuVarCheck,	configRpcBind, NULL, "amd_enable=YES" },
-      { " AMD Flags",	"Set flags to AMD service (if enabled)",
-	dmenuVarCheck,	dmenuISetVariable, NULL, "amd_flags" },
-      { " Anon FTP",	"This machine wishes to allow anonymous FTP.",
-	dmenuVarCheck,	configAnonFTP, NULL, "anon_ftp" },
-      { " Gateway",	"This machine will route packets between interfaces",
-	dmenuVarCheck,	dmenuToggleVariable, NULL, "gateway_enable=YES" },
-      { " inetd",	"This machine wants to run the inet daemon",
-	dmenuVarCheck,	configInetd, NULL, "inetd_enable=YES" },
-      { " Mail",	"This machine wants to run a Mail Transfer Agent",
-	NULL,		dmenuSubmenu, NULL, &MenuMTA },
-      { " NFS client",	"This machine will be an NFS client",
-	dmenuVarCheck,	dmenuToggleVariable, NULL, "nfs_client_enable=YES" },
-      { " NFS server",	"This machine will be an NFS server",
-	dmenuVarCheck,	configNFSServer, NULL, "nfs_server_enable=YES" },
-      { " Ntpdate",	"Select a clock synchronization server",
-	dmenuVarCheck,	dmenuSubmenu, NULL, &MenuNTP, '[', 'X', ']',
-	(uintptr_t)"ntpdate_enable=YES" },
-      { " PCNFSD",	"Run authentication server for clients with PC-NFS.",
-	dmenuVarCheck,	configPCNFSD, NULL, "pcnfsd" },
-      { " rpcbind", 	"RPC port mapping daemon (formerly portmapper)",
-        dmenuVarCheck, dmenuToggleVariable, NULL, "rpcbind_enable=YES" },
-      { " rpc.statd", 	"NFS status monitoring daemon",
-	dmenuVarCheck, configRpcBind, NULL, "rpc_statd_enable=YES" },
-      { " rpc.lockd",	"NFS file locking daemon",
-	dmenuVarCheck, configRpcBind, NULL, "rpc_lockd_enable=YES" },
-      { " Routed",	"Select routing daemon (default: routed)",
-	dmenuVarCheck,	configRouter, NULL, "router_enable=YES" },
-      { " Rwhod",	"This machine wants to run the rwho daemon",
-	dmenuVarCheck,	dmenuToggleVariable, NULL, "rwhod_enable=YES" },
-      { " sshd",	"This machine wants to run the SSH daemon",
-	dmenuVarCheck,	dmenuToggleVariable, NULL, "sshd_enable=YES" },
-      { " TCP Extensions", "Allow RFC1323 and RFC1644 TCP extensions?",
-	dmenuVarCheck,	dmenuToggleVariable, NULL, "tcp_extensions=YES" },
-      { NULL } },
-};
-
-DMenu MenuMTA = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Mail Transfer Agent Selection",
-    "You can choose which Mail Transfer Agent (MTA) you wish to install and run.\n"
-    "Selecting Sendmail local disables sendmail's network socket for\n"
-    "incoming mail, but still enables sendmail for local and outbound mail.\n"
-    "The Postfix option will install the Postfix MTA from the ports\n"
-    "collection.  The Exim option will install the Exim MTA from the ports\n"
-    "collection.  To return to the previous menu, select Exit.",
-    NULL,
-    NULL,
-    {
-      { "Sendmail",           "Use sendmail",
-        dmenuVarCheck, dmenuSetVariable, NULL, "sendmail_enable=YES" },
-      { "Sendmail local",    "Use sendmail, but do not listen on the network",
-        dmenuVarCheck, dmenuSetVariable, NULL, "sendmail_enable=NO" },
-      { "Postfix",            "Use the Postfix MTA",
-      NULL, configMTAPostfix, NULL, NULL },
-      { "Exim",               "Use the Exim MTA",
-      NULL, configMTAExim, NULL, NULL },
-      { "None",               "Do not install an MTA",
-        dmenuVarCheck, dmenuSetVariable, NULL, "sendmail_enable=NONE" },
-      { "X Exit",             "Exit this menu (returning to previous)",
-        checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { NULL } },
-};
-
-DMenu MenuNTP = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "NTPDATE Server Selection",
-    "There are a number of time synchronization servers available\n"
-    "for public use around the Internet.  Please select one reasonably\n"
-    "close to you to have your system time synchronized accordingly.",
-    "These are the primary open-access NTP servers",
-    NULL,
-    { { "None",		        "No NTP server",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=NO,ntpdate_hosts=none" },
-      { "Other",		"Select a site not on this list",
-	dmenuVarsCheck, configNTP, NULL, NULL },
-      { "Worldwide",		"pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=pool.ntp.org" },
-      { "Asia",		"asia.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=asia.pool.ntp.org" },
-      { "Europe",		"europe.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=europe.pool.ntp.org" },
-      { "Oceania",		"oceania.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=oceania.pool.ntp.org" },
-      { "North America",	"north-america.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=north-america.pool.ntp.org" },
-      { "Argentina",		"tick.nap.com.ar",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tick.nap.com.ar" },
-      { "Argentina #2",		"time.sinectis.com.ar",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=time.sinectis.com.ar" },
-      { "Argentina #3",		"tock.nap.com.ar",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tock.nap.com.ar" },
-      { "Australia",		"au.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=au.pool.ntp.org" },
-      { "Australia #2",		"augean.eleceng.adelaide.edu.au",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=augean.eleceng.adelaide.edu.au" },
-      { "Australia #3",		"ntp.adelaide.edu.au",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.adelaide.edu.au" },
-      { "Australia #4",		"ntp.saard.net",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.saard.net" },
-      { "Australia #5",		"time.deakin.edu.au",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=time.deakin.edu.au" },
-      { "Belgium",		"ntp1.belbone.be",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.belbone.be" },
-      { "Belgium #2",		"ntp2.belbone.be",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.belbone.be" },
-      { "Brazil",		"a.ntp.br",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=a.ntp.br" },
-      { "Brazil #2",		"b.ntp.br",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=b.ntp.br" },
-      { "Brazil #3",		"c.ntp.br",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=c.ntp.br" },
-      { "Brazil #4",		"ntp.cais.rnp.br",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.cais.rnp.br" },
-      { "Brazil #5",		"ntp1.pucpr.br",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.pucpr.br" },
-      { "Canada",		"ca.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ca.pool.ntp.org" },
-      { "Canada #2",		"ntp.cpsc.ucalgary.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.cpsc.ucalgary.ca" },
-      { "Canada #3",		"ntp1.cmc.ec.gc.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.cmc.ec.gc.ca" },
-      { "Canada #4",		"ntp2.cmc.ec.gc.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.cmc.ec.gc.ca" },
-      { "Canada #5",		"tick.utoronto.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=tick.utoronto.ca" },
-      { "Canada #6",		"time.chu.nrc.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=time.chu.nrc.ca" },
-      { "Canada #7",		"time.nrc.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=time.nrc.ca" },
-      { "Canada #8",		"timelord.uregina.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=timelord.uregina.ca" },
-      { "Canada #9",		"tock.utoronto.ca",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=tock.utoronto.ca" },
-      { "Czech",		"ntp.karpo.cz",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.karpo.cz" },
-      { "Czech #2",		"ntp.cgi.cz",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.cgi.cz" },
-      { "Denmark",		"clock.netcetera.dk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock.netcetera.dk" },
-      { "Denmark",		"clock2.netcetera.dk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock2.netcetera.dk" },
-      { "Spain",		"slug.ctv.es",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=slug.ctv.es" },
-      { "Finland",		"tick.keso.fi",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tick.keso.fi" },
-      { "Finland #2",		"tock.keso.fi",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tock.keso.fi" },
-      { "France",		"ntp.obspm.fr",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.obspm.fr" },
-      { "France #2",		"ntp.univ-lyon1.fr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.univ-lyon1.fr" },
-      { "France #3",		"ntp.via.ecp.fr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.via.ecp.fr" },
-      { "Croatia",		"zg1.ntp.carnet.hr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=zg1.ntp.carnet.hr" },
-      { "Croatia #2",		"zg2.ntp.carnet.hr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=zg2.ntp.carnet.hr" },
-      { "Croatia #3",		"st.ntp.carnet.hr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=st.ntp.carnet.hr" },
-      { "Croatia #4",		"ri.ntp.carnet.hr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ri.ntp.carnet.hr" },
-      { "Croatia #5",		"os.ntp.carnet.hr",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=os.ntp.carnet.hr" },
-      { "Hungary",		"time.kfki.hu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=time.kfki.hu" },
-      { "Indonesia",		"ntp.kim.lipi.go.id",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.kim.lipi.go.id" },
-      { "Ireland",		"ntp.maths.tcd.ie",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.maths.tcd.ie" },
-      { "Italy",		"it.pool.ntp.org",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=it.pool.ntp.org" },
-      { "Japan",		"ntp.jst.mfeed.ad.jp",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.jst.mfeed.ad.jp" },
-      { "Japan IPv6",		"ntp1.v6.mfeed.ad.jp",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.v6.mfeed.ad.jp" },
-      { "Korea",		"time.nuri.net",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=time.nuri.net" },
-      { "Mexico",		"mx.pool.ntp.org",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=mx.pool.ntp.org" },
-      { "Netherlands",		"ntp0.nl.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp0.nl.net" },
-      { "Netherlands #2",	"ntp1.nl.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.nl.net" },
-      { "Netherlands #3",	"ntp2.nl.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.nl.net" },
-      { "Norway",		"fartein.ifi.uio.no",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=fartein.ifi.uio.no" },
-      { "Norway #2",		"time.alcanet.no",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=time.alcanet.no" },
-      { "New Zealand",		"ntp.massey.ac.nz",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.massey.ac.nz" },
-      { "New Zealand #2",	"ntp.public.otago.ac.nz",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.public.otago.ac.nz" },
-      { "New Zealand #3",	"tk1.ihug.co.nz",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tk1.ihug.co.nz" },
-      { "New Zealand #4",	"ntp.waikato.ac.nz",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.waikato.ac.nz" },
-      { "Poland",		"info.cyf-kr.edu.pl",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=info.cyf-kr.edu.pl" },
-      { "Romania",		"ticks.roedu.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ticks.roedu.net" },
-      { "Russia",		"ru.pool.ntp.org",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ru.pool.ntp.org" },
-      { "Russia #2",		"ntp.psn.ru",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.psn.ru" },
-      { "Sweden",		"se.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=se.pool.ntp.org" },
-      { "Sweden #2",		"ntp.lth.se",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.lth.se" },
-      { "Sweden #3",		"ntp1.sp.se",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.sp.se" },
-      { "Sweden #4",		"ntp2.sp.se",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.sp.se" },
-      { "Sweden #5",		"ntp.kth.se",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.kth.se" },
-      { "Singapore",		"sg.pool.ntp.org",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=sg.pool.ntp.org" },
-      { "Slovenia",		"si.pool.ntp.org",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=si.pool.ntp.org" },
-      { "Slovenia #2",		"sizif.mf.uni-lj.si",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=sizif.mf.uni-lj.si" },
-      { "Slovenia #3",		"ntp1.arnes.si",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.arnes.si" },
-      { "Slovenia #4",		"ntp2.arnes.si",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.arnes.si" },
-      { "Slovenia #5",		"time.ijs.si",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=time.ijs.si" },
-      { "Scotland",		"ntp.cs.strath.ac.uk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.cs.strath.ac.uk" },
-      { "Taiwan",              "time.stdtime.gov.tw",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=time.stdtime.gov.tw" },
-      { "Taiwan #2",           "clock.stdtime.gov.tw",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock.stdtime.gov.tw" },
-      { "Taiwan #3",           "tick.stdtime.gov.tw",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tick.stdtime.gov.tw" },
-      { "Taiwan #4",           "tock.stdtime.gov.tw",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tock.stdtime.gov.tw" },
-      { "Taiwan #5",           "watch.stdtime.gov.tw",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=watch.stdtime.gov.tw" },
-      { "United Kingdom",	"uk.pool.ntp.org",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=uk.pool.ntp.org" },
-      { "United Kingdom #2",	"ntp.exnet.com",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.exnet.com" },
-      { "United Kingdom #3",	"ntp0.uk.uu.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp0.uk.uu.net" },
-      { "United Kingdom #4",	"ntp1.uk.uu.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.uk.uu.net" },
-      { "United Kingdom #5",	"ntp2.uk.uu.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.uk.uu.net" },
-      { "United Kingdom #6",	"ntp2a.mcc.ac.uk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2a.mcc.ac.uk" },
-      { "United Kingdom #7",	"ntp2b.mcc.ac.uk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2b.mcc.ac.uk" },
-      { "United Kingdom #8",	"ntp2c.mcc.ac.uk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2c.mcc.ac.uk" },
-      { "United Kingdom #9",	"ntp2d.mcc.ac.uk",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2d.mcc.ac.uk" },
-      { "U.S.",	"us.pool.ntp.org",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=us.pool.ntp.org" },
-      { "U.S. AR",	"sushi.lyon.edu",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=sushi.compsci.lyon.edu" },
-      { "U.S. AZ",	"ntp.drydog.com",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.drydog.com" },
-      { "U.S. CA",	"ntp.ucsd.edu",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.ucsd.edu" },
-      { "U.S. CA #2",	"ntp1.mainecoon.com",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.mainecoon.com" },
-      { "U.S. CA #3",	"ntp2.mainecoon.com",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.mainecoon.com" },
-      { "U.S. CA #4",	"reloj.kjsl.com",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=reloj.kjsl.com" },
-      { "U.S. CA #5",	"time.five-ten-sg.com",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=time.five-ten-sg.com" },
-      { "U.S. DE",	"louie.udel.edu",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=louie.udel.edu" },
-      { "U.S. GA",		"ntp.shorty.com",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.shorty.com" },
-      { "U.S. GA #2",		"rolex.usg.edu",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=rolex.usg.edu" },
-      { "U.S. GA #3",		"timex.usg.edu",
-	dmenuVarsCheck,	dmenuSetVariables, NULL, 
-	"ntpdate_enable=YES,ntpdate_hosts=timex.usg.edu" },
-      { "U.S. IL",	"ntp-0.cso.uiuc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-0.cso.uiuc.edu" },
-      { "U.S. IL #2",	"ntp-1.cso.uiuc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-1.cso.uiuc.edu" },
-      { "U.S. IL #3",	"ntp-1.mcs.anl.gov",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-1.mcs.anl.gov" },
-      { "U.S. IL #4",	"ntp-2.cso.uiuc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-2.cso.uiuc.edu" },
-      { "U.S. IL #5",	"ntp-2.mcs.anl.gov",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-2.mcs.anl.gov" },
-      { "U.S. IN",	"gilbreth.ecn.purdue.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=gilbreth.ecn.purdue.edu" },
-      { "U.S. IN #2",	"harbor.ecn.purdue.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=harbor.ecn.purdue.edu" },
-      { "U.S. IN #3",	"molecule.ecn.purdue.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=molecule.ecn.purdue.edu" },
-      { "U.S. KS",	"ntp1.kansas.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.kansas.net" },
-      { "U.S. KS #2",	"ntp2.kansas.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.kansas.net" },
-      { "U.S. MA",	"ntp.ourconcord.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.ourconcord.net" },
-      { "U.S. MA #2",	"timeserver.cs.umb.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=timeserver.cs.umb.edu" },
-      { "U.S. MN",	"ns.nts.umn.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ns.nts.umn.edu" },
-      { "U.S. MN #2",	"nss.nts.umn.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=nss.nts.umn.edu" },
-      { "U.S. MO",	"time-ext.missouri.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=time-ext.missouri.edu" },
-      { "U.S. MT",	"chronos1.umt.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=chronos1.umt.edu" },
-      { "U.S. MT #2",	"chronos2.umt.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=chronos2.umt.edu" },
-      { "U.S. MT #3",	"chronos3.umt.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=chronos3.umt.edu" },
-      { "U.S. NC",	"clock1.unc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock1.unc.edu" },
-      { "U.S. NV",	"cuckoo.nevada.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=cuckoo.nevada.edu" },
-      { "U.S. NV #2",	"tick.cs.unlv.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tick.cs.unlv.edu" },
-      { "U.S. NV #3",	"tock.cs.unlv.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tock.cs.unlv.edu" },
-      { "U.S. NY",	"ntp0.cornell.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp0.cornell.edu" },
-      { "U.S. NY #2",	"sundial.columbia.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=sundial.columbia.edu" },
-      { "U.S. NY #3",	"timex.cs.columbia.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=timex.cs.columbia.edu" },
-      { "U.S. PA",	"clock-1.cs.cmu.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock-1.cs.cmu.edu" },
-      { "U.S. PA #2",	"clock-2.cs.cmu.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock-2.cs.cmu.edu" },
-      { "U.S. PA #3",	"clock.psu.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=clock.psu.edu" },
-      { "U.S. PA #4",	"fuzz.psc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=fuzz.psc.edu" },
-      { "U.S. PA #5",	"ntp-1.ece.cmu.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-1.ece.cmu.edu" },
-      { "U.S. PA #6",	"ntp-2.ece.cmu.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-2.ece.cmu.edu" },
-      { "U.S. TX",	"ntp.fnbhs.com",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.fnbhs.com" },
-      { "U.S. TX #2",	"ntp.tmc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.tmc.edu" },
-      { "U.S. TX #3",	"ntp5.tamu.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp5.tamu.edu" },
-      { "U.S. TX #4",	"tick.greyware.com",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tick.greyware.com" },
-      { "U.S. TX #5",	"tock.greyware.com",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=tock.greyware.com" },
-      { "U.S. VA",	"ntp-1.vt.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-1.vt.edu" },
-      { "U.S. VA #2",	"ntp-2.vt.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp-2.vt.edu" },
-      { "U.S. VA #3",	"ntp.cmr.gov",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.cmr.gov" },
-      { "U.S. VT",	"ntp0.state.vt.us",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp0.state.vt.us" },
-      { "U.S. VT #2",	"ntp1.state.vt.us",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.state.vt.us" },
-      { "U.S. VT #3",	"ntp2.state.vt.us",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp2.state.vt.us" },
-      { "U.S. WA",	"ntp.tcp-udp.net",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.tcp-udp.net" },
-      { "U.S. WI",	"ntp1.cs.wisc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp1.cs.wisc.edu" },
-      { "U.S. WI #2",	"ntp3.cs.wisc.edu",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp3.cs.wisc.edu" },
-      { "South Africa",	"ntp.cs.unp.ac.za",
-	dmenuVarsCheck, dmenuSetVariables, NULL,
-	"ntpdate_enable=YES,ntpdate_hosts=ntp.cs.unp.ac.za" },
-      { NULL } },
-};
-
-#ifdef WITH_SYSCONS
-DMenu MenuSyscons = {
-    DMENU_NORMAL_TYPE,
-    "System Console Configuration",
-    "The system console driver for FreeBSD has a number of configuration\n"
-    "options which may be set according to your preference.\n\n"
-    "When you are done setting configuration options, select Cancel.",
-    "Configure your system console settings",
-    NULL,
-    { { "X Exit",	"Exit this menu (returning to previous)", NULL, dmenuExit },
-#ifdef PC98
-      { "2 Keymap",	"Choose an alternate keyboard map",	NULL, dmenuSubmenu, NULL, &MenuSysconsKeymap },
-      { "3 Repeat",	"Set the rate at which keys repeat",	NULL, dmenuSubmenu, NULL, &MenuSysconsKeyrate },
-      { "4 Saver",	"Configure the screen saver",		NULL, dmenuSubmenu, NULL, &MenuSysconsSaver },
-#else
-      { "2 Font",	"Choose an alternate screen font",	NULL, dmenuSubmenu, NULL, &MenuSysconsFont },
-      { "3 Keymap",	"Choose an alternate keyboard map",	NULL, dmenuSubmenu, NULL, &MenuSysconsKeymap },
-      { "4 Repeat",	"Set the rate at which keys repeat",	NULL, dmenuSubmenu, NULL, &MenuSysconsKeyrate },
-      { "5 Saver",	"Configure the screen saver",		NULL, dmenuSubmenu, NULL, &MenuSysconsSaver },
-      { "6 Screenmap",	"Choose an alternate screenmap",	NULL, dmenuSubmenu, NULL, &MenuSysconsScrnmap },
-      { "7 Ttys",       "Choose console terminal type",         NULL, dmenuSubmenu, NULL, &MenuSysconsTtys },
-#endif
-      { NULL } },
-};
-
-#ifdef PC98
-DMenu MenuSysconsKeymap = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Keymap",
-    "The system console driver for FreeBSD defaults to a standard\n"
-    "\"PC-98x1\" keyboard map.  Users may wish to choose one of the\n"
-    "other keymaps below.",
-    "Choose a keyboard map",
-    NULL,
-    { { "Japanese PC-98x1",		"Japanese PC-98x1 keymap",  dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=jp.pc98" },
-      { " Japanese PC-98x1 (ISO)",	"Japanese PC-98x1 (ISO) keymap",  dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=jp.pc98.iso" },
-      { NULL } },
-};
-#else
-DMenu MenuSysconsKeymap = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Keymap",
-    "The system console driver for FreeBSD defaults to a standard\n"
-    "\"US\" keyboard map.  Users may wish to choose one of the\n"
-    "other keymaps below.",
-    "Choose a keyboard map",
-    NULL,
-    { { "Belgian",	"Belgian ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=be.iso" },
-      { " Brazil CP850",	"Brazil CP850 keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=br275.cp850" },
-      { " Brazil ISO (accent)",	"Brazil ISO keymap (accent keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=br275.iso.acc" },
-      { " Brazil ISO",	"Brazil ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=br275.iso" },
-      { " Bulgarian BDS",	"Bulgarian BDS keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=bg.bds.ctrlcaps" },
-      { " Bulgarian Phonetic",	"Bulgarian Phonetic keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=bg.phonetic.ctrlcaps" },
-      { "Central European ISO", "Central European ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=ce.iso2" },
-      { " Croatian ISO",	"Croatian ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=hr.iso" },
-      { " Czech ISO (accent)",	"Czech ISO keymap (accent keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=cs.latin2.qwertz" },
-      { "Danish CP865",	"Danish Code Page 865 keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=danish.cp865" },
-      { " Danish ISO",	"Danish ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=danish.iso" },
-      { "Estonian ISO", "Estonian ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=estonian.iso" },
-      { " Estonian ISO 15", "Estonian ISO 8859-15 keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=estonian.iso15" },
-      { " Estonian CP850", "Estonian Code Page 850 keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=estonian.cp850" },
-      { "Finnish CP850","Finnish Code Page 850 keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=finnish.cp850" },
-      { " Finnish ISO",  "Finnish ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=finnish.iso" },
-      { " French ISO (accent)", "French ISO keymap (accent keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=fr.iso.acc" },
-      { " French ISO",	"French ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=fr.iso" },
-      { " French ISO/Macbook",	"French ISO keymap on macbook",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=fr.macbook.acc" },
-      { "German CP850",	"German Code Page 850 keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=german.cp850"	},
-      { " German ISO",	"German ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=german.iso" },
-      { " Greek 101",	"Greek ISO keymap (101 keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=gr.us101.acc" },
-      { " Greek 104",	"Greek ISO keymap (104 keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=el.iso07" },
-      { " Greek ELOT",	"Greek ISO keymap (ELOT 1000)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=gr.elot.acc" },
-      { "Hungarian 101", "Hungarian ISO keymap (101 key)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=hu.iso2.101keys" },
-      { " Hungarian 102", "Hungarian ISO keymap (102 key)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=hu.iso2.102keys" },
-      { "Icelandic (accent)", "Icelandic ISO keymap (accent keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=icelandic.iso.acc" },
-      { " Icelandic",	"Icelandic ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=icelandic.iso" },
-      { " Italian",	"Italian ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=it.iso" },
-      { "Japanese 106",	"Japanese 106 keymap",  dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=jp.106" },
-      { "Latin American (accent)",	"Latin American ISO keymap (accent keys)",	dmenuVarCheck,	dmenuSetKmapVariable,	NULL,	"keymap=latinamerican.iso.acc" },
-      { " Latin American",	"Latin American ISO keymap",	dmenuVarCheck,	dmenuSetKmapVariable,	NULL,	"keymap=latinamerican" },
-      { "Norway ISO",	"Norwegian ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=norwegian.iso" },
-      { "Polish ISO",	"Polish ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=pl_PL.ISO8859-2" },
-      { " Portuguese (accent)",	"Portuguese ISO keymap (accent keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=pt.iso.acc" },
-      { " Portuguese",	"Portuguese ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=pt.iso" },
-      { "Russia KOI8-R", "Russian KOI8-R keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=ru.koi8-r" },
-      { "Slovak", "Slovak ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=sk.iso2" },
-      { "Slovenian", "Slovenian ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=si.iso" },
-      { " Spanish (accent)", "Spanish ISO keymap (accent keys)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=spanish.iso.acc" },
-      { " Spanish",	"Spanish ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=spanish.iso" },
-      { " Swedish CP850", "Swedish Code Page 850 keymap", dmenuVarCheck,	dmenuSetKmapVariable, NULL, "keymap=swedish.cp850" },
-      { " Swedish ISO",	"Swedish ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swedish.iso" },
-      { " Swiss French ISO (accent)", "Swiss French ISO keymap (accent keys)", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swissfrench.iso.acc" },
-      { " Swiss French ISO", "Swiss French ISO keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swissfrench.iso" },
-      { " Swiss French CP850", "Swiss French Code Page 850 keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swissfrench.cp850" },
-      { " Swiss German ISO (accent)", "Swiss German ISO keymap (accent keys)", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swissgerman.iso.acc" },
-      { " Swiss German ISO", "Swiss German ISO keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swissgerman.iso" },
-      { " Swiss German CP850", "Swiss German Code Page 850 keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=swissgerman.cp850" },
-      { "UK CP850",	"UK Code Page 850 keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=uk.cp850" },
-      { " UK ISO",	"UK ISO keymap", dmenuVarCheck,	dmenuSetKmapVariable, NULL, "keymap=uk.iso" },
-      { " Ukrainian KOI8-U",	"Ukrainian KOI8-U keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=ua.koi8-u" },
-      { " Ukrainian KOI8-U+KOI8-R",	"Ukrainian KOI8-U+KOI8-R keymap (alter)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=ua.koi8-u.shift.alt" },
-      { " USA CapsLock->Ctrl",	"US standard (Caps as L-Control)",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.pc-ctrl" },
-      { " USA Dvorak",	"US Dvorak keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.dvorak" },
-      { " USA Dvorak (left)",	"US left handed Dvorak keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.dvorakl" },
-      { " USA Dvorak (right)",	"US right handed Dvorak keymap", dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.dvorakr" },
-      { " USA Emacs",	"US standard optimized for EMACS",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.emacs" },
-      { " USA ISO",	"US ISO keymap",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.iso" },
-      { " USA UNIX",	"US traditional UNIX-workstation",	dmenuVarCheck, dmenuSetKmapVariable, NULL, "keymap=us.unix" },
-      { NULL } },
-};
-#endif /* PC98 */
-
-DMenu MenuSysconsKeyrate = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Keyboard Repeat Rate",
-    "This menu allows you to set the speed at which keys repeat\n"
-    "when held down.",
-    "Choose a keyboard repeat rate",
-    NULL,
-    { { "Slow",	"Slow keyboard repeat rate",	dmenuVarCheck,	dmenuSetVariable, NULL, "keyrate=slow" },
-      { "Normal", "\"Normal\" keyboard repeat rate",	dmenuVarCheck,	dmenuSetVariable, NULL, "keyrate=normal" },
-      { "Fast",	"Fast keyboard repeat rate",	dmenuVarCheck,	dmenuSetVariable, NULL, "keyrate=fast" },
-      { "Default", "Use default keyboard repeat rate",	dmenuVarCheck,	dmenuSetVariable, NULL, "keyrate=NO" },
-      { NULL } }
-};
-
-DMenu MenuSysconsSaver = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Screen Saver",
-    "By default, the console driver will not attempt to do anything\n"
-    "special with your screen when it's idle.  If you expect to leave your\n"
-    "monitor switched on and idle for long periods of time then you should\n"
-    "probably enable one of these screen savers to prevent burn-in.",
-    "Choose a nifty-looking screen saver",
-    NULL,
-    { { "1 Blank",	"Simply blank the screen",
-	dmenuVarCheck, configSaver, NULL, "saver=blank" },
-      { "2 Beastie",	"\"BSD Daemon\" animated screen saver (graphics)",
-	dmenuVarCheck, configSaver, NULL, "saver=beastie" },
-      { "3 Daemon",	"\"BSD Daemon\" animated screen saver (text)",
-	dmenuVarCheck, configSaver, NULL, "saver=daemon" },
-      { "4 Dragon",	"Dragon screensaver (graphics)",
-	dmenuVarCheck, configSaver, NULL, "saver=dragon" },
-      { "5 Fade",	"Fade out effect screen saver",
-	dmenuVarCheck, configSaver, NULL, "saver=fade" },
-      { "6 Fire",	"Flames effect screen saver",
-	dmenuVarCheck, configSaver, NULL, "saver=fire" },
-      { "7 Green",	"\"Green\" power saving mode (if supported by monitor)",
-	dmenuVarCheck, configSaver, NULL, "saver=green" },
-      { "8 Logo",	"FreeBSD \"logo\" animated screen saver (graphics)",
-	dmenuVarCheck, configSaver, NULL, "saver=logo" },
-      { "9 Rain",	"Rain drops screen saver",
-	dmenuVarCheck, configSaver, NULL, "saver=rain" },
-      { "a Snake",	"Draw a FreeBSD \"snake\" on your screen",
-	dmenuVarCheck, configSaver, NULL, "saver=snake" },
-      { "b Star",	"A \"twinkling stars\" effect",
-	dmenuVarCheck, configSaver, NULL, "saver=star" },
-      { "c Warp",	"A \"stars warping\" effect",
-	dmenuVarCheck, configSaver, NULL, "saver=warp" },
-      { "d None",	"Disable the screensaver",
-        dmenuVarCheck, configSaver, NULL, "saver=NO" },
-      { "Timeout",	"Set the screen saver timeout interval",
-	NULL, configSaverTimeout, NULL, NULL, ' ', ' ', ' ' },
-      { NULL } }
-};
-
-#ifndef PC98
-DMenu MenuSysconsScrnmap = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Screenmap",
-    "Unless you load a specific font, most PC hardware defaults to\n"
-    "displaying characters in the IBM 437 character set.  However,\n"
-    "in the Unix world, this character set is very rarely used.  Most\n"
-    "Western European countries, for example, prefer ISO 8859-1.\n"
-    "American users won't notice the difference since the bottom half\n"
-    "of all these character sets is ANSI anyway.\n"
-    "If your hardware is capable of downloading a new display font,\n"
-    "you should probably choose that option.  However, for hardware\n"
-    "where this is not possible (e.g. monochrome adapters), a screen\n"
-    "map will give you the best approximation that your hardware can\n"
-    "display at all.",
-    "Choose a screen map",
-    NULL,
-    { { "1 None",                 "No screenmap, don't touch font", dmenuVarCheck, dmenuSetVariable, NULL, "scrnmap=NO" },
-      { "2 ISO 8859-1 to IBM437", "W-Europe ISO 8859-1 to IBM 437 screenmap", dmenuVarCheck, dmenuSetVariable, NULL, "scrnmap=iso-8859-1_to_cp437" },
-      { "3 ISO 8859-7 to IBM437", "Greek ISO 8859-7 to IBM 437 screenmap", dmenuVarCheck, dmenuSetVariable, NULL, "scrnmap=iso-8859-7_to_cp437" },
-      { "4 US-ASCII to IBM437",   "US-ASCII to IBM 437 screenmap", dmenuVarCheck, dmenuSetVariable, NULL, "scrnmap=us-ascii_to_cp437" },
-      { "5 KOI8-R to IBM866",     "Russian KOI8-R to IBM 866 screenmap", dmenuVarCheck, dmenuSetVariable, NULL, "scrnmap=koi8-r2cp866" },
-      { "6 KOI8-U to IBM866u",    "Ukrainian KOI8-U to IBM 866u screenmap", dmenuVarCheck, dmenuSetVariable, NULL, "scrnmap=koi8-u2cp866u" },
-      { NULL } },
-};
-
-DMenu MenuSysconsTtys = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Terminal Type",
-    "For various console encodings, a corresponding terminal type\n"
-    "must be chosen in /etc/ttys.\n\n"
-    "WARNING: For compatibility reasons, only entries starting with\n"
-    "ttyv and terminal types starting with cons[0-9] can be changed\n"
-    "via this menu.\n",
-    "Choose a terminal type",
-    NULL,
-    { { "1 None",               "Don't touch anything",  dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=NO" },
-      { "2 IBM437 (VGA default)", "cons25", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25" },
-      { "3 ISO 8859-1",         "cons25l1", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25l1" },
-      { "4 ISO 8859-2",         "cons25l2", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25l2" },
-      { "5 ISO 8859-7",         "cons25l7", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25l7" },
-      { "6 KOI8-R",             "cons25r", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25r" },
-      { "7 KOI8-U",             "cons25u", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25u" },
-      { "8 US-ASCII",           "cons25w", dmenuVarCheck, dmenuSetVariable, NULL, VAR_CONSTERM "=cons25w" },
-      { NULL } },
-};
-
-DMenu MenuSysconsFont = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "System Console Font",
-    "Most PC hardware defaults to displaying characters in the\n"
-    "IBM 437 character set.  However, in the Unix world, this\n"
-    "character set is very rarely used.  Most Western European\n"
-    "countries, for example, prefer ISO 8859-1.\n"
-    "American users won't notice the difference since the bottom half\n"
-    "of all these charactersets is ANSI anyway.  However, they might\n"
-    "want to load a font anyway to use the 30- or 50-line displays.\n"
-    "If your hardware is capable of downloading a new display font,\n"
-    "you can select the appropriate font below.",
-    "Choose a font",
-    NULL,
-    { { "1 None", "Use hardware default font", dmenuVarCheck, dmenuSetVariables, NULL,
-	"font8x8=NO,font8x14=NO,font8x16=NO" },
-      { "2 IBM 437", "English and others, VGA default", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=cp437-8x8,font8x14=cp437-8x14,font8x16=cp437-8x16" },
-      { "3 IBM 850", "Western Europe, IBM encoding",	dmenuVarCheck,	dmenuSetVariables, NULL,
-	"font8x8=cp850-8x8,font8x14=cp850-8x14,font8x16=cp850-8x16" },
-      { "4 IBM 865", "Norwegian, IBM encoding",	dmenuVarCheck,	dmenuSetVariables, NULL,
-	"font8x8=cp865-8x8,font8x14=cp865-8x14,font8x16=cp865-8x16" },
-      { "5 IBM 866", "Russian, IBM encoding (use with KOI8-R screenmap)",   dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=cp866-8x8,font8x14=cp866-8x14,font8x16=cp866b-8x16,mousechar_start=3" },
-      { "6 IBM 866u", "Ukrainian, IBM encoding (use with KOI8-U screenmap)",   dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=cp866u-8x8,font8x14=cp866u-8x14,font8x16=cp866u-8x16,mousechar_start=3" },
-      { "7 IBM 1251", "Cyrillic, MS Windows encoding",  dmenuVarCheck, dmenuSetVariables, NULL,
-	"font8x8=cp1251-8x8,font8x14=cp1251-8x14,font8x16=cp1251-8x16,mousechar_start=3" },
-      { "8 ISO 8859-1", "Western Europe, ISO encoding", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=iso-8x8,font8x14=iso-8x14,font8x16=iso-8x16" },
-      { "9 ISO 8859-2", "Eastern Europe, ISO encoding", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=iso02-8x8,font8x14=iso02-8x14,font8x16=iso02-8x16" },
-      { "a ISO 8859-4", "Baltic, ISO encoding", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=iso04-8x8,font8x14=iso04-8x14,font8x16=iso04-8x16" },
-      { "b ISO 8859-7", "Greek, ISO encoding", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=iso07-8x8,font8x14=iso07-8x14,font8x16=iso07-8x16" },
-      { "c ISO 8859-8", "Hebrew, ISO encoding", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=iso08-8x8,font8x14=iso08-8x14,font8x16=iso08-8x16" },
-      { "d ISO 8859-15", "Europe, ISO encoding", dmenuVarCheck,  dmenuSetVariables, NULL,
-	"font8x8=iso15-8x8,font8x14=iso15-8x14,font8x16=iso15-8x16" },
-      { "e SWISS", "English, better resolution", dmenuVarCheck, dmenuSetVariables, NULL,
-	"font8x8=swiss-8x8,font8x14=NO,font8x16=swiss-8x16" },
-      { NULL } },
-};
-#endif /* PC98 */
-#endif /* WITH_SYSCONS */
-
-DMenu MenuUsermgmt = {
-    DMENU_NORMAL_TYPE,
-    "User and group management",
-    "The submenus here allow to manipulate user groups and\n"
-    "login accounts.\n",
-    "Configure your user groups and users",
-    NULL,
-    { { "X Exit",	"Exit this menu (returning to previous)", NULL, dmenuExit },
-      { "User",		"Add a new user to the system.",	NULL, userAddUser },
-      { "Group",	"Add a new user group to the system.",	NULL, userAddGroup },
-      { NULL } },
-};
-
-DMenu MenuSecurity = {
-    DMENU_CHECKLIST_TYPE | DMENU_SELECTION_RETURNS,
-    "System Security Options Menu",
-    "This menu allows you to configure aspects of the operating system security\n"
-    "policy.  Please read the system documentation carefully before modifying\n"
-    "these settings, as they may cause service disruption if used improperly.\n"
-    "\n"
-    "Most settings will take affect only following a system reboot.",
-    "Configure system security options",
-    NULL,
-    { { "X Exit",      "Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { " Securelevel",	"Configure securelevels for the system",
-	NULL, configSecurelevel },
-#if 0
-      { " LOMAC",         "Use Low Watermark Mandatory Access Control at boot",
-	dmenuVarCheck,  dmenuToggleVariable, NULL, "lomac_enable=YES" },
-#endif
-      { " NFS port",	"Require that the NFS clients use reserved ports",
-	dmenuVarCheck,  dmenuToggleVariable, NULL, "nfs_reserved_port_only=YES" },
-      { NULL } },
-};
-
-DMenu MenuSecurelevel = {
-    DMENU_NORMAL_TYPE | DMENU_SELECTION_RETURNS,
-    "Securelevel Configuration Menu",
-    "This menu allows you to select the securelevel your system runs with.\n"
-    "When operating at a securelevel, certain root privileges are disabled,\n"
-    "which may increase resistance to exploits and protect system integrity.\n"
-    "In secure mode system flags may not be overriden by the root user,\n"
-    "access to direct kernel memory is limited, and kernel modules may not\n"
-    "be changed.  In highly secure mode, mounted file systems may not be\n"
-    "modified on-disk, tampering with the system clock is prohibited.  In\n"
-    "network secure mode configuration changes to firewalling are prohibited.\n",
-    "Select a securelevel to operate at - F1 for help",
-    "securelevel",
-    { { "X Exit",      "Exit this menu (returning to previous)",
-	checkTrue, dmenuExit, NULL, NULL, '<', '<', '<' },
-      { "Disabled", "Disable securelevels", NULL, configSecurelevelDisabled, },
-      { "Secure", "Secure mode", NULL, configSecurelevelSecure },
-      { "Highly Secure", "Highly secure mode", NULL, configSecurelevelHighlySecure }, 
-      { "Network Secure", "Network secure mode", NULL, configSecurelevelNetworkSecure },
-      { NULL } }
-};
-
-DMenu MenuFixit = {
-    DMENU_NORMAL_TYPE,
-    "Please choose a fixit option",
-    "There are three ways of going into \"fixit\" mode:\n"
-    "- you can use the live filesystem CDROM/DVD, in which case there will be\n"
-    "  full access to the complete set of FreeBSD commands and utilities,\n"
-    "- you can use the more limited (but perhaps customized) fixit floppy,\n"
-    "- or you can start an Emergency Holographic Shell now, which is\n"
-    "  limited to the subset of commands that is already available right now.",
-    "Press F1 for more detailed repair instructions",
-    "fixit",
-{ { "X Exit",		"Exit this menu (returning to previous)",	NULL, dmenuExit },
-  { "2 CDROM/DVD",	"Use the live filesystem CDROM/DVD",		NULL, installFixitCDROM },
-  { "3 USB",		"Use the live filesystem from a USB drive",	NULL, installFixitUSB },
-  { "4 Floppy",	"Use a floppy generated from the fixit image",	NULL, installFixitFloppy },
-  { "5 Shell",		"Start an Emergency Holographic Shell",		NULL, installFixitHoloShell },
-  { NULL } },
-};
diff --git a/usr.sbin/sysinstall/misc.c b/usr.sbin/sysinstall/misc.c
deleted file mode 100644
index ed4fa2d9c7b9..000000000000
--- a/usr.sbin/sysinstall/misc.c
+++ /dev/null
@@ -1,553 +0,0 @@
-/*
- * Miscellaneous support routines..
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <unistd.h>
-#include <sys/stat.h>
-#include <sys/errno.h>
-#include <sys/file.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/wait.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <ufs/ufs/ufsmount.h>
-#include <sys/reboot.h>
-#include <sys/disklabel.h>
-#include <fs/msdosfs/msdosfsmount.h>
-#include <sys/sysctl.h>
-
-/* Quick check to see if a file is readable */
-Boolean
-file_readable(char *fname)
-{
-    if (!access(fname, F_OK))
-	return TRUE;
-    return FALSE;
-}
-
-/* Quick check to see if a file is executable */
-Boolean
-file_executable(char *fname)
-{
-    if (!access(fname, X_OK))
-	return TRUE;
-    return FALSE;
-}
-
-/* Concatenate two strings into static storage */
-char *
-string_concat(char *one, char *two)
-{
-    static char tmp[FILENAME_MAX];
-
-    /* Yes, we're deliberately cavalier about not checking for overflow */
-    strcpy(tmp, one);
-    strcat(tmp, two);
-    return tmp;
-}
-
-/* sane strncpy() function */
-char *
-sstrncpy(char *dst, const char *src, int size)
-{
-    dst[size] = '\0';
-    return strncpy(dst, src, size);
-}
-
-/* Concatenate three strings into static storage */
-char *
-string_concat3(char *one, char *two, char *three)
-{
-    static char tmp[FILENAME_MAX];
-
-    /* Yes, we're deliberately cavalier about not checking for overflow */
-    strcpy(tmp, one);
-    strcat(tmp, two);
-    strcat(tmp, three);
-    return tmp;
-}
-
-/* Clip the whitespace off the end of a string */
-char *
-string_prune(char *str)
-{
-    int len = str ? strlen(str) : 0;
-
-    while (len && isspace(str[len - 1]))
-	str[--len] = '\0';
-    return str;
-}
-
-/* run the whitespace off the front of a string */
-char *
-string_skipwhite(char *str)
-{
-    while (*str && isspace(*str))
-	++str;
-    return str;
-}
-
-/* copy optionally and allow second arg to be null */
-char *
-string_copy(char *s1, char *s2)
-{
-    if (!s1)
-	return NULL;
-    if (!s2)
-	s1[0] = '\0';
-    else
-	strcpy(s1, s2);
-    return s1;
-}
-
-/* convert an integer to a string, using a static buffer */
-char *
-itoa(int value)
-{
-    static char buf[13];
-
-    snprintf(buf, 12, "%d", value);
-    return buf;
-}
-
-Boolean
-directory_exists(const char *dirname)
-{
-    DIR *tptr;
-
-    if (!dirname)
-	return FALSE;
-    if (!strlen(dirname))
-	return FALSE;
-
-    tptr = opendir(dirname);
-    if (!tptr)
-	return (FALSE);
-
-    closedir(tptr);
-    return (TRUE);
-}
-
-char *
-pathBaseName(const char *path)
-{
-    char *pt;
-    char *ret = (char *)path;
-
-    pt = strrchr(path,(int)'/');
-
-    if (pt != 0)			/* if there is a slash */
-    {
-	ret = ++pt;			/* start the file after it */
-    }
-    
-    return(ret);
-}
-
-/* A free guaranteed to take NULL ptrs */
-void
-safe_free(void *ptr)
-{
-    if (ptr)
-	free(ptr);
-}
-
-/* A malloc that checks errors */
-void *
-safe_malloc(size_t size)
-{
-    void *ptr;
-
-    if (size <= 0)
-	msgFatal("Invalid malloc size of %ld!", (long)size);
-    ptr = malloc(size);
-    if (!ptr)
-	msgFatal("Out of memory!");
-    bzero(ptr, size);
-    return ptr;
-}
-
-/* A realloc that checks errors */
-void *
-safe_realloc(void *orig, size_t size)
-{
-    void *ptr;
-
-    if (size <= 0)
-	msgFatal("Invalid realloc size of %ld!", (long)size);
-    ptr = reallocf(orig, size);
-    if (!ptr)
-	msgFatal("Out of memory!");
-    return ptr;
-}
-
-/* Create a path biased from the VAR_INSTALL_ROOT variable (if not /) */
-char *
-root_bias(char *path)
-{
-    static char tmp[FILENAME_MAX];
-    char *cp = variable_get(VAR_INSTALL_ROOT);
-
-    if (!strcmp(cp, "/"))
-	return path;
-    strcpy(tmp, variable_get(VAR_INSTALL_ROOT));
-    strcat(tmp, path);
-    return tmp;
-}
-
-/*
- * These next routines are kind of specialized just for building item lists
- * for dialog_menu().
- */
-
-/* Add an item to an item list */
-dialogMenuItem *
-item_add(dialogMenuItem *list, char *prompt, char *title,
-	 int (*checked)(dialogMenuItem *self),
-	 int (*fire)(dialogMenuItem *self),
-	 void (*selected)(dialogMenuItem *self, int is_selected),
-	 void *data, void *aux, int *curr, int *max)
-{
-    dialogMenuItem *d;
-
-    if (*curr == *max) {
-	*max += 20;
-	list = (dialogMenuItem *)safe_realloc(list, sizeof(dialogMenuItem) * *max);
-    }
-    d = &list[(*curr)++];
-    bzero(d, sizeof(*d));
-    d->prompt = prompt ? strdup(prompt) : NULL;
-    d->title = title ? strdup(title) : NULL;
-    d->checked = checked;
-    d->fire = fire;
-    d->selected = selected;
-    d->data = data;
-    d->aux = (long)aux;
-    return list;
-}
-
-/* Toss the items out */
-void
-items_free(dialogMenuItem *list, int *curr, int *max)
-{
-    int i;
-
-    for (i = 0; list[i].prompt; i++) {
-	safe_free(list[i].prompt);
-	safe_free(list[i].title);
-    }
-    safe_free(list);
-    *curr = *max = 0;
-}
-
-int
-Mkdir(char *ipath)
-{
-    struct stat sb;
-    int final;
-    char *p, *path;
-
-    if (file_readable(ipath) || Fake)
-	return DITEM_SUCCESS;
-
-    path = strcpy(alloca(strlen(ipath) + 1), ipath);
-    if (isDebug())
-	msgDebug("mkdir(%s)\n", path);
-    p = path;
-    if (p[0] == '/')		/* Skip leading '/'. */
-	++p;
-    for (final = FALSE; !final; ++p) {
-	if (p[0] == '\0' || (p[0] == '/' && p[1] == '\0'))
-	    final = TRUE;
-	else if (p[0] != '/')
-	    continue;
-	*p = '\0';
-	if (stat(path, &sb)) {
-	    if (errno != ENOENT) {
-		msgConfirm("Couldn't stat directory %s: %s", path, strerror(errno));
-		return DITEM_FAILURE;
-	    }
-	    if (isDebug())
-		msgDebug("mkdir(%s..)\n", path);
-	    if (mkdir(path, S_IRWXU | S_IRWXG | S_IRWXO) < 0) {
-		msgConfirm("Couldn't create directory %s: %s", path,strerror(errno));
-		return DITEM_FAILURE;
-	    }
-	}
-	*p = '/';
-    }
-    return DITEM_SUCCESS;
-}
-
-int
-Mkdir_command(char *key, void *dir)
-{
-    return (Mkdir((char*)dir));
-}
-
-int
-Mount(char *mountp, void *dev)
-{
-    struct ufs_args ufsargs;
-    char device[80];
-    char mountpoint[FILENAME_MAX];
-
-    if (Fake)
-	return DITEM_SUCCESS;
-
-    if (*((char *)dev) != '/') {
-    	sprintf(device, "%s/dev/%s", RunningAsInit ? "/mnt" : "", (char *)dev);
-	sprintf(mountpoint, "%s%s", RunningAsInit ? "/mnt" : "", mountp);
-    }
-    else {
-	strcpy(device, dev);
-	strcpy(mountpoint, mountp);
-    }
-    memset(&ufsargs,0,sizeof ufsargs);
-
-    if (Mkdir(mountpoint)) {
-	msgConfirm("Unable to make directory mountpoint for %s!", mountpoint);
-	return DITEM_FAILURE;
-    }
-    if (isDebug())
-	msgDebug("mount %s %s\n", device, mountpoint);
-
-    ufsargs.fspec = device;
-    if (mount("ufs", mountpoint, RunningAsInit ? MNT_ASYNC | MNT_NOATIME : 0,
-	(caddr_t)&ufsargs) == -1) {
-	msgConfirm("Error mounting %s on %s : %s", device, mountpoint, strerror(errno));
-	return DITEM_FAILURE;
-    }
-    return DITEM_SUCCESS;
-}
-
-int
-Mount_msdosfs(char *mountp, void *dev)
-{
-    struct msdosfs_args mount_args;
-    char device[80];
-    char mountpoint[FILENAME_MAX];
-
-    if (Fake)
-	return DITEM_SUCCESS;
-
-    if (*((char *)dev) != '/') {
-    	sprintf(device, "%s/dev/%s", RunningAsInit ? "/mnt" : "", (char *)dev);
-	sprintf(mountpoint, "%s%s", RunningAsInit ? "/mnt" : "", mountp);
-    }
-    else {
-	strcpy(device, dev);
-	strcpy(mountpoint, mountp);
-    }
-
-    if (Mkdir(mountpoint)) {
-	msgConfirm("Unable to make directory mountpoint for %s!", mountpoint);
-	return DITEM_FAILURE;
-    }
-    if (isDebug())
-	msgDebug("mount %s %s\n", device, mountpoint);
-
-    memset(&mount_args, 0, sizeof(mount_args));
-    mount_args.fspec = device;
-    mount_args.magic = MSDOSFS_ARGSMAGIC;
-    mount_args.mask = S_IRWXU | S_IRWXG | S_IRWXO;
-    if (mount("msdosfs", mountpoint, RunningAsInit ? MNT_ASYNC|MNT_NOATIME : 0,
-	    (caddr_t)&mount_args) == -1) {
-	msgConfirm("Error mounting %s on %s : %s", device, mountpoint, strerror(errno));
-	return DITEM_FAILURE;
-    }
-    return DITEM_SUCCESS;
-}
-
-WINDOW *
-openLayoutDialog(char *helpfile, char *title, int x, int y, int width, int height)
-{
-    WINDOW		*win;
-    static char		help[FILENAME_MAX];
-
-    /* We need a curses window */
-    win = newwin(LINES, COLS, 0, 0);
-    if (win) {
-	/* Say where our help comes from */
-	if (helpfile) {
-	    use_helpline("Press F1 for more information on this screen.");
-	    use_helpfile(systemHelpFile(helpfile, help));
-	}
-	/* Setup a nice screen for us to splat stuff onto */
-	draw_box(win, y, x, height, width, dialog_attr, border_attr);
-	wattrset(win, dialog_attr);
-	mvwaddstr(win, y, x + (COLS - strlen(title)) / 2, title);
-    }
-    return win;
-}
-
-ComposeObj *
-initLayoutDialog(WINDOW *win, Layout *layout, int x, int y, int *max)
-{
-    ComposeObj *obj = NULL, *first;
-    int n;
-
-    /* Loop over the layout list, create the objects, and add them
-       onto the chain of objects that dialog uses for traversal*/
-    
-    n = 0;
-    while (layout[n].help != NULL) {
-	int t = TYPE_OF_OBJ(layout[n].type);
-
-	switch (t) {
-	case STRINGOBJ:
-	    layout[n].obj = NewStringObj(win, layout[n].prompt, layout[n].var,
-					 layout[n].y + y, layout[n].x + x, layout[n].len, layout[n].maxlen);
-	    ((StringObj *)layout[n].obj)->attr_mask = ATTR_OF_OBJ(layout[n].type);
-	    break;
-	    
-	case BUTTONOBJ:
-	    layout[n].obj = NewButtonObj(win, layout[n].prompt, layout[n].var, layout[n].y + y, layout[n].x + x);
-	    break;
-	    
-	default:
-	    msgFatal("Don't support this object yet!");
-	}
-	AddObj(&obj, t, (void *) layout[n].obj);
-	n++;
-    }
-    *max = n - 1;
-    /* Find the first object in the list */
-    for (first = obj; first->prev; first = first->prev);
-    return first;
-}
-
-int
-layoutDialogLoop(WINDOW *win, Layout *layout, ComposeObj **obj, int *n, int max, int *cbutton, int *cancel)
-{
-    char help_line[80];
-    int ret, i, len = strlen(layout[*n].help);
-
-    /* Display the help line at the bottom of the screen */
-    for (i = 0; i < 79; i++)
-	help_line[i] = (i < len) ? layout[*n].help[i] : ' ';
-    help_line[i] = '\0';
-    use_helpline(help_line);
-    display_helpline(win, LINES - 1, COLS - 1);
-    wrefresh(win);
-	    
-    /* Ask for libdialog to do its stuff */
-    ret = PollObj(obj);
-    /* Handle special case stuff that libdialog misses. Sigh */
-    switch (ret) {
-    case SEL_ESC:	/* Bail out */
-	*cancel = TRUE;
-	return FALSE;
-	      
-	/* This doesn't work for list dialogs. Oh well. Perhaps
-	   should special case the move from the OK button ``up''
-	   to make it go to the interface list, but then it gets
-	   awkward for the user to go back and correct screw up's
-	   in the per-interface section */
-    case KEY_DOWN:
-    case SEL_CR:
-    case SEL_TAB:
-	if (*n < max)
-	    ++*n;
-	else
-	    *n = 0;
-	break;
-	      
-	/* The user has pressed enter over a button object */
-    case SEL_BUTTON:
-	if (cbutton && *cbutton)
-	    *cancel = TRUE;
-	else
-	    *cancel = FALSE;
-	return FALSE;
-	
-    case KEY_UP:
-    case SEL_BACKTAB:
-	if (*n)
-	    --*n;
-	else
-	    *n = max;
-	break;
-	
-    case KEY_F(1):
-	display_helpfile();
-	
-	/* They tried some key combination we don't support - tootle them forcefully! */
-    default:
-	beep();
-    }
-    return TRUE;
-}
-
-WINDOW *
-savescr(void)
-{
-    WINDOW *w;
-
-    w = dupwin(newscr);
-    return w;
-}
-
-void
-restorescr(WINDOW *w)
-{
-    touchwin(w);
-    wrefresh(w);
-    delwin(w);
-}
-
-/*
- * Get a sysctl variable as a string or "<unknown>" if sysctl fails.
- * Caller must free returned string.
- */
-char *
-getsysctlbyname(const char *sysctlname)
-{
-    char *buf;
-    size_t sz, buf_sz = 0;
-    const char unk_str[] = "<unknown>";
-
-    sysctlbyname(sysctlname, NULL, &buf_sz, NULL, 0);
-    buf_sz = MAX(sizeof(unk_str), buf_sz) + 1;
-    sz = buf_sz - 1;
-    buf = (char *)safe_malloc(buf_sz);
-
-    if (sysctlbyname(sysctlname, buf, &sz, NULL, 0) != -1)
-	buf[sz] = '\0';
-    else
-	strlcpy(buf, unk_str, buf_sz);
-
-    return buf;
-}
diff --git a/usr.sbin/sysinstall/modules.c b/usr.sbin/sysinstall/modules.c
deleted file mode 100644
index 9c9875fa6b19..000000000000
--- a/usr.sbin/sysinstall/modules.c
+++ /dev/null
@@ -1,223 +0,0 @@
-/*-
- * Copyright (c) 2000  "HOSOKAWA, Tatsumi" <hosokawa@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include "sysinstall.h"
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/linker.h>
-#include <dirent.h>
-#include <fcntl.h>
-#include <fnmatch.h>
-#include <kenv.h>
-
-/* Prototypes */
-static int		kldModuleFire(dialogMenuItem *self);
-
-#define MODULESDIR "/modules"
-#define DISTMOUNT "/dist"
-
-void
-moduleInitialize(void)
-{
-    int fd, len;
-    DIR *dirp;
-    struct dirent *dp;
-    char module[MAXPATHLEN], desc[MAXPATHLEN];
-    char desc_str[BUFSIZ];
-
-    if (!RunningAsInit && !Fake) {
-	/* It's not my job... */
-	return;
-    }
-
-    dirp = opendir(MODULESDIR);
-    if (dirp) {
-	while ((dp = readdir(dirp))) {
-	    if (dp->d_namlen < (sizeof(".ko") - 1)) continue;
-	    if (strcmp(dp->d_name + dp->d_namlen - (sizeof(".ko") - 1), ".ko") == 0) {
-		strcpy(module, MODULESDIR);
-		strcat(module, "/");
-		strcat(module, dp->d_name);
-		strcpy(desc, module);
-		len = strlen(desc);
-		strcpy(desc + (len - (sizeof(".ko") - 1)), ".dsc");
-		fd = open(module, O_RDONLY);
-		if (fd < 0) continue;
-		close(fd);
-		fd = open(desc, O_RDONLY);
-		if (fd < 0) {
-		    desc_str[0] = 0;
-		}
-		else {
-		    len = read(fd, desc_str, BUFSIZ);
-		    close(fd);
-		    if (len < BUFSIZ) desc_str[len] = 0;
-		}
-		if (desc_str[0])
-		    msgDebug("Loading module %s (%s)\n", dp->d_name, desc_str);
-		else
-		    msgDebug("Loading module %s\n", dp->d_name);
-		if (kldload(module) < 0 && errno != EEXIST) {
-		    if (desc_str[0])
-			msgConfirm("Loading module %s failed\n%s", dp->d_name, desc_str);
-		    else
-			msgConfirm("Loading module %s failed", dp->d_name);
-		}
-	    }
-	    if (strcmp(dp->d_name + dp->d_namlen - (sizeof(".ko.gz") - 1), ".ko.gz") == 0) {
-		snprintf(module, sizeof(module), "/tmp/%s", dp->d_name);
-		module[strlen(module) - sizeof(".gz")] = '\0';
-		snprintf(desc, sizeof(desc), "zcat < %s/%s > %s", MODULESDIR,
-		  dp->d_name, module);
-		system(desc);
-		if (kldload(module) < 0 && errno != EEXIST) {
-		    if (desc_str[0])
-			msgConfirm("Loading module %s failed\n%s", dp->d_name, desc_str);
-		    else
-			msgConfirm("Loading module %s failed", dp->d_name);
-		}
-		unlink(module);
-	    }
-	}
-	closedir(dirp);
-    }
-}
-
-void
-driverFloppyCheck(void)
-{
-    /* Prompt for the driver floppy if requested. */
-    if (kenv(KENV_GET, "driver_floppy", NULL, 0) >= 0 &&
-	!msgYesNo("Would you like to load kernel modules from the driver floppy?"))
-	(void)kldBrowser(NULL);
-}
-
-int
-kldBrowser(dialogMenuItem *self)
-{
-    DMenu	*menu = NULL;
-    int		i, what = DITEM_SUCCESS, msize, count;
-    DIR		*dir;
-    struct dirent *de;
-    char	*err;
-    
-    err = NULL;
-    count = 0;
-    
-    if (DITEM_STATUS(mediaSetFloppy(NULL)) == DITEM_FAILURE) {
-	err = "Unable to set media device to floppy.";
-	goto errout;
-    }
-
-    if (!DEVICE_INIT(mediaDevice)) {
-	err = "Unable to mount floppy filesystem.";
-	goto errout;
-    }
-
-    msize = sizeof(DMenu) + (sizeof(dialogMenuItem) * 2);
-    if ((menu = malloc(msize)) == NULL) {
-	err = "Failed to allocate memory for menu";
-	goto errout;
-    }
-
-    bcopy(&MenuKLD, menu, sizeof(DMenu));
-	
-    bzero(&menu->items[count], sizeof(menu->items[0]));
-    menu->items[count].prompt = strdup("X Exit");
-    menu->items[count].title = strdup("Exit this menu (returning to previous)");
-    menu->items[count].fire = dmenuExit;
-    count++;
-	
-    if ((dir = opendir(DISTMOUNT)) == NULL) {
-	err = "Couldn't open directory";
-	goto errout;
-    }
-    
-    while ((de = readdir(dir)) != NULL) {
-	if (fnmatch("*.ko", de->d_name, FNM_CASEFOLD))
-	    continue;
-	
-	msize += sizeof(dialogMenuItem);
-	if ((menu = realloc(menu, msize)) == NULL) {
-	    err = "Failed to allocate memory for menu item";
-	    goto errout;
-	}
-	    
-	bzero(&menu->items[count], sizeof(menu->items[0]));
-	menu->items[count].fire = kldModuleFire;
-
-	menu->items[count].prompt = strdup(de->d_name);
-	menu->items[count].title = menu->items[count].prompt;
-	    
-	count++;
-    }
-
-    closedir(dir);
-
-    menu->items[count].prompt = NULL;
-    menu->items[count].title = NULL;
-    
-    dmenuOpenSimple(menu, FALSE);
-    
-    deviceRescan();
-    
-  errout:    
-    mediaClose();
-    for (i = 0; i < count; i++)
-	free(menu->items[i].prompt);
-    
-    free(menu);
-
-    if (err != NULL) {
-	what |= DITEM_FAILURE;
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm(err);
-    }
-    
-    return (what);
-}
-
-static int
-kldModuleFire(dialogMenuItem *self) {
-    char	fname[256];
-
-    bzero(fname, sizeof(fname));
-    snprintf(fname, sizeof(fname), "%s/%s", DISTMOUNT, self->prompt);
-
-    if (kldload(fname) < 0 && errno != EEXIST) {
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm("Loading module %s failed\n", fname);
-    } else {
-	if (!variable_get(VAR_NO_ERROR))
-	    msgConfirm("Loaded module %s OK", fname);
-    }
-
-    return(0);
- }
diff --git a/usr.sbin/sysinstall/mouse.c b/usr.sbin/sysinstall/mouse.c
deleted file mode 100644
index 574fd7235974..000000000000
--- a/usr.sbin/sysinstall/mouse.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*-
- * Copyright (c) 1998 Kazutaka YOKOTA (yokota@zodiac.mech.utsunomiya-u.ac.jp)
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote 
- *    products derived from this software without specific prior written 
- *    permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHRO AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHRO OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include "sysinstall.h"
-#include <string.h>
-
-int
-mousedTest(dialogMenuItem *self)
-{
-	char *type;
-	char *port;
-	char *flags;
-	int ret;
-
-	type = variable_get(VAR_MOUSED_TYPE);
-	port = variable_get(VAR_MOUSED_PORT);
-	flags = variable_get(VAR_MOUSED_FLAGS);
-	if ((type == NULL) || (port == NULL) 
-		|| (strlen(type) <= 0) || (strlen(port) <= 0)
-		|| (strcmp(type, "NO") == 0)) {
-		msgConfirm("Please select a mouse protocol and a port first.");
-		return DITEM_FAILURE;
-	}
-
-	msgNotify("Trying to start the mouse daemon...");
-	if (file_readable("/var/run/moused.pid"))
-	    vsystem("kill `cat /var/run/moused.pid`");
-	systemExecute("vidcontrol -m on");
-	if (flags != NULL)
-	    vsystem("moused -t %s -p %s %s", type, port, flags);
-	else
-	    vsystem("moused -t %s -p %s", type, port);
-
-	ret = msgYesNo("Now move the mouse and see if it works.\n"
-	      "(Note that buttons don't have any effect for now.)\n\n"
-	      "         Is the mouse cursor moving?\n");
-	systemExecute("vidcontrol -m off");
-	if (ret) {
-		if (file_readable("/var/run/moused.pid"))
-		    vsystem("kill `cat /var/run/moused.pid`");
-		variable_set2(VAR_MOUSED, "NO", 1);
-	} else {
-		variable_set2(VAR_MOUSED, "YES", 1);
-		vsystem("ln -fs /dev/sysmouse /dev/mouse"); /* backwards compat */
-	}
-
-	return DITEM_SUCCESS | DITEM_RESTORE;
-}
-
-int
-mousedDisable(dialogMenuItem *self)
-{
-	if (file_readable("/var/run/moused.pid"))
-	    vsystem("kill `cat /var/run/moused.pid`");
-	variable_set2(VAR_MOUSED, "NO", 1);
-	variable_set2(VAR_MOUSED_TYPE, "NO", 1);
-	variable_unset(VAR_MOUSED_PORT);
-	variable_unset(VAR_MOUSED_FLAGS);
-	msgConfirm("The mouse daemon is disabled.");
-	return DITEM_SUCCESS;
-}
-
-int 
-setMouseFlags(dialogMenuItem *self)
-{
-    int ret;
-    ret = variable_get_value(VAR_MOUSED_FLAGS,
-			     "Please Specify the mouse daemon flags.  If you would like to\n"
-			     "emulate 3 buttons, use -3 here.\n", 1)
-	? DITEM_SUCCESS : DITEM_FAILURE;
-    if (ret != DITEM_SUCCESS)
-	variable_unset(VAR_MOUSED_FLAGS);
-    return ret;
-}
-
diff --git a/usr.sbin/sysinstall/msg.c b/usr.sbin/sysinstall/msg.c
deleted file mode 100644
index 6e20c60ae5a0..000000000000
--- a/usr.sbin/sysinstall/msg.c
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include <sys/ioctl.h>
-#include <sys/consio.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-#include <netdb.h>
-
-#include <stdarg.h>
-#include <syslog.h>
-
-#include "sysinstall.h"
-
-Boolean
-isDebug(void)
-{
-    char *cp;
-
-    return (cp = variable_get(VAR_DEBUG)) && strcmp(cp, "no");
-}
-
-static Boolean
-isNetworkUp(void)
-{
-	if (!(RunningAsInit) || 
-	    (variable_check("NETWORK_CONFIGURED=NO")) != TRUE) {
-	    return TRUE;
-	}
-
-	return FALSE;
-}
-
-void
-msgSyslog(const char *errstr)
-{
-	struct sockaddr_in server;
-	struct hostent *hp;
-	char *host, *line;
-	int sock;
-
-	if (!isNetworkUp())
-	    return;
-
-	if (!(host = variable_get(VAR_SYSLOG_SERVER)))
-	    return;
-
-	if (!(hp = gethostbyname2(host, AF_INET)))
-	    return;
-
-	if (!(sock = socket(AF_INET, SOCK_DGRAM, 0)))
-	    return;
-	
-	bzero(&server, sizeof(struct sockaddr_in));
-	server.sin_family = AF_INET;
-	server.sin_port = htons(514);
-	bcopy((char *)hp->h_addr, (char *)&server.sin_addr, hp->h_length);
-
-	asprintf(&line, "<%d>%s", LOG_NOTICE, errstr);
-	sendto(sock, line, strlen(line), 0, (struct sockaddr *)&server, 
-	    sizeof(struct sockaddr_in));
-
-	close(sock);
-	free(line);
-}
-
-/* Whack up an informational message on the status line, in stand-out */
-void
-msgYap(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int attrs;
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    attrs = getattrs(stdscr);
-    attrset(A_REVERSE);
-    mvaddstr(StatusLine, 0, errstr);
-    attrset(attrs);
-    refresh();
-}
-
-/* Whack up an informational message on the status line */
-void
-msgInfo(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int i, attrs;
-    char line[81];
-
-    attrs = getattrs(stdscr);
-    /* NULL is a special convention meaning "erase the old stuff" */
-    if (!fmt) {
-	move(StatusLine, 0);
-	clrtoeol();
-	return;
-    }
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    memset(line, ' ', 80);
-    for (i = 0; i < 80; i++) {
-	if (errstr[i])
-	    line[i] = errstr[i];
-	else
-	    break;
-    }
-    line[80] = '\0';
-    attrset(ATTR_TITLE);
-    mvaddstr(StatusLine, 0, line);
-    attrset(attrs);
-    move(StatusLine, 79);
-    refresh();
-
-    msgSyslog(errstr);
-}
-
-/* Whack up a warning on the status line */
-void
-msgWarn(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int attrs;
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    strcpy(errstr, "Warning: ");
-    va_start(args, fmt);
-    vsnprintf((char *)(errstr + strlen(errstr)), FILENAME_MAX, fmt, args);
-    va_end(args);
-    attrs = getattrs(stdscr);
-    beep();
-    attrset(ATTR_TITLE);
-    mvaddstr(StatusLine, 0, errstr);
-    attrset(attrs);
-    refresh();
-
-    /* we don't want this hitting syslog twice */
-    if (isDebug()) {
-	if (OnVTY)
-		msgDebug("Warning message `%s'\n", errstr);
-	else
-		msgSyslog(errstr);
-    }
-
-}
-
-/* Whack up an error on the status line */
-void
-msgError(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int attrs;
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    strcpy(errstr, "Error: ");
-    va_start(args, fmt);
-    vsnprintf((char *)(errstr + strlen(errstr)), FILENAME_MAX, fmt, args);
-    va_end(args);
-    beep();
-    attrs = getattrs(stdscr);
-    attrset(ATTR_TITLE);
-    mvaddstr(StatusLine, 0, errstr);
-    attrset(attrs);
-    refresh();
-
-    /* we don't want this hitting syslog twice */
-    if (isDebug()) {
-	if (OnVTY)
-		msgDebug("Error message `%s'\n", errstr);
-	else
-		msgSyslog(errstr);
-    }
-}
-
-/* Whack up a fatal error on the status line */
-void
-msgFatal(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int attrs;
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    strcpy(errstr, "Fatal Error: ");
-    va_start(args, fmt);
-    vsnprintf((char *)(errstr + strlen(errstr)), FILENAME_MAX, fmt, args);
-    va_end(args);
-    beep();
-    attrs = getattrs(stdscr);
-    attrset(ATTR_TITLE);
-    mvaddstr(StatusLine, 0, errstr);
-    addstr(" - ");
-    addstr("PRESS ANY KEY TO ");
-    if (RunningAsInit)
-	addstr("REBOOT");
-    else
-	addstr("QUIT");
-    attrset(attrs);
-    refresh();
-    if (OnVTY)
-	msgDebug("Fatal error `%s'!\n", errstr);
-    else
-    	msgSyslog(errstr);
-    getch();
-    systemShutdown(1);
-}
-
-/* Put up a message in a popup confirmation box */
-void
-msgConfirm(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    WINDOW *w = savescr();
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    if (OnVTY) {
-	ioctl(0, VT_ACTIVATE, 1);
-	msgInfo(NULL);
-    }
-    dialog_notify(errstr);
-    restorescr(w);
-}
-
-/* Put up a message in a popup information box */
-void
-msgNotify(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    if (isDebug())
-	msgDebug("Notify: %s\n", errstr);
-    dialog_msgbox(NULL, errstr, -1, -1, 0);
-}
-
-/* Put up a message in a popup yes/no box and return 0 for YES, 1 for NO */
-int
-msgYesNo(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int ret;
-    WINDOW *w = savescr();
-    
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    if (OnVTY) {
-	ioctl(0, VT_ACTIVATE, 1);	/* Switch back */
-	msgInfo(NULL);
-    }
-    if (variable_get(VAR_NONINTERACTIVE))
-	return 0;	/* If non-interactive, return YES all the time */
-    ret = dialog_yesno("User Confirmation Requested", errstr, -1, -1);
-    restorescr(w);
-    return ret;
-}
-
-/* Put up a message in a popup no/yes box and return 0 for YES, 1 for NO */
-int
-msgNoYes(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    int ret;
-    WINDOW *w = savescr();
-    
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    if (OnVTY) {
-	ioctl(0, VT_ACTIVATE, 1);	/* Switch back */
-	msgInfo(NULL);
-    }
-    if (variable_get(VAR_NONINTERACTIVE))
-	return 1;	/* If non-interactive, return NO all the time */
-    ret = dialog_noyes("User Confirmation Requested", errstr, -1, -1);
-    restorescr(w);
-    return ret;
-}
-
-/* Put up a message in an input box and return the value */
-char *
-msgGetInput(char *buf, char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    static char input_buffer[256];
-    int rval;
-    WINDOW *w = savescr();
-
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    if (buf)
-	SAFE_STRCPY(input_buffer, buf);
-    else
-	input_buffer[0] = '\0';
-    if (OnVTY) {
-	ioctl(0, VT_ACTIVATE, 1);	/* Switch back */
-	msgInfo(NULL);
-    }
-    rval = dialog_inputbox("Value Required", errstr, -1, -1, input_buffer);
-    restorescr(w);
-    if (!rval)
-	return input_buffer;
-    else
-	return NULL;
-}
-
-/* Write something to the debugging port */
-void
-msgDebug(char *fmt, ...)
-{
-    va_list args;
-    char *dbg;
-
-    if (DebugFD == -1)
-	return;
-    dbg = (char *)alloca(FILENAME_MAX);
-    strcpy(dbg, "DEBUG: ");
-    va_start(args, fmt);
-    vsnprintf((char *)(dbg + strlen(dbg)), FILENAME_MAX, fmt, args);
-    va_end(args);
-
-    msgSyslog(dbg);
-
-    write(DebugFD, dbg, strlen(dbg));
-}
-
-/* Tell the user there's some output to go look at */
-void
-msgWeHaveOutput(char *fmt, ...)
-{
-    va_list args;
-    char *errstr;
-    WINDOW *w = savescr();
-    
-    errstr = (char *)alloca(FILENAME_MAX);
-    va_start(args, fmt);
-    vsnprintf(errstr, FILENAME_MAX, fmt, args);
-    va_end(args);
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    msgDebug("Notify: %s\n", errstr);
-    dialog_clear_norefresh();
-    sleep(2);
-    dialog_msgbox(NULL, errstr, -1, -1, 0);
-    restorescr(w);
-}
-
-/* Simple versions of msgConfirm() and msgNotify() for calling from scripts */
-int
-msgSimpleConfirm(char *str)
-{
-    msgConfirm("%s", str);
-    return DITEM_SUCCESS;
-}
-
-int
-msgSimpleNotify(char *str)
-{
-    msgNotify("%s", str);
-    return DITEM_SUCCESS;
-}
diff --git a/usr.sbin/sysinstall/network.c b/usr.sbin/sysinstall/network.c
deleted file mode 100644
index 35599aad3b91..000000000000
--- a/usr.sbin/sysinstall/network.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/* These routines deal with getting things off of network media */
-
-#include "sysinstall.h"
-#include <signal.h>
-#include <termios.h>
-#include <sys/fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-
-static Boolean	networkInitialized;
-
-Boolean
-mediaInitNetwork(Device *dev)
-{
-    int i;
-    char *rp;
-    char *cp, ifconfig[255];
-    WINDOW *w;
-    
-    if (!RunningAsInit || networkInitialized)
-	return TRUE;
-
-    if (isDebug())
-	msgDebug("Init routine called for network device %s.\n", dev->name);
-
-    if (!file_readable("/etc/resolv.conf")) {
-	if (DITEM_STATUS(configResolv(NULL)) == DITEM_FAILURE) {
-	    msgConfirm("Can't seem to write out /etc/resolv.conf.  Net cannot be used.");
-	    return FALSE;
-	}
-    }
-
-    w = savescr();
-    dialog_clear_norefresh();
-
-    snprintf(ifconfig, 255, "%s%s", VAR_IFCONFIG, dev->name);
-    cp = variable_get(ifconfig);
-    if (cp) {
-	/*
-	 * If this interface isn't a DHCP one, bring it up.
-	 * If it is, then it's already up.
-	 */
-	if (strstr(cp, "DHCP") == NULL) {
-	    msgDebug("Not a DHCP interface.\n");
-	    i = vsystem("ifconfig %s %s", dev->name, cp);
-	    if (i) {
-		msgConfirm("Unable to configure the %s interface!\n"
-			   "This installation method cannot be used.",
-			   dev->name);
-		return FALSE;
-	    }
-	    rp = variable_get(VAR_GATEWAY);
-	    if (!rp || *rp == '0') {
-		msgConfirm("No gateway has been set. You will be unable to access hosts\n"
-			   "not on your local network");
-	    }
-	    else {
-		/* 
-		 * Explicitly flush all routes to get back to a known sane
-		 * state. We don't need to check this exit code because if
-		 * anything fails it will show up in the route add below.
-		 */
-		system("route -n flush");
-		msgDebug("Adding default route to %s.\n", rp);
-		if (vsystem("route -n add default %s", rp) != 0) {
-		    msgConfirm("Failed to add a default route; please check "
-			       "your network configuration");
-		    return FALSE;
-		}
-	    }
-	} else {
-	    msgDebug("A DHCP interface.  Should already be up.\n");
-	}
-    } else if ((cp = variable_get(VAR_IPV6ADDR)) == NULL || *cp == '\0') {
-	msgConfirm("The %s device is not configured.  You will need to do so\n"
-		   "in the Networking configuration menu before proceeding.", dev->name);
-	return FALSE;
-    }
-
-    if (isDebug())
-	msgDebug("Network initialized successfully.\n");
-    networkInitialized = TRUE;
-    return TRUE;
-}
-
-void
-mediaShutdownNetwork(Device *dev)
-{
-    char *cp;
-
-    if (!RunningAsInit || !networkInitialized)
-	return;
-
-	msgDebug("Shutdown called for network device %s\n", dev->name);
-	int i;
-	char ifconfig[255];
-
-	snprintf(ifconfig, 255, "%s%s", VAR_IFCONFIG, dev->name);
-	cp = variable_get(ifconfig);
-	if (!cp)
-	    return;
-	msgDebug("ifconfig %s down\n", dev->name);
-	i = vsystem("ifconfig %s down", dev->name);
-	if (i)
-	    msgConfirm("Warning: Unable to down the %s interface properly", dev->name);
-	cp = variable_get(VAR_GATEWAY);
-	if (cp) {
-	    msgDebug("Deleting default route.\n");
-	    vsystem("route -n delete default");
-	}
-}
-
diff --git a/usr.sbin/sysinstall/nfs.c b/usr.sbin/sysinstall/nfs.c
deleted file mode 100644
index 87788eaced6f..000000000000
--- a/usr.sbin/sysinstall/nfs.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/errno.h>
-#include <sys/fcntl.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-
-#include <limits.h>
-
-Boolean NFSMounted;
-static char mountpoint[] = "/dist";
-
-Boolean
-mediaInitNFS(Device *dev)
-{
-    Device *netDevice = (Device *)dev->private;
-    WINDOW *w = savescr();
-
-    if (NFSMounted)
-	return TRUE;
-
-    if (!DEVICE_INIT(netDevice))
-	return FALSE;
-
-    if (Mkdir(mountpoint))
-	return FALSE;
-
-    msgNotify("Mounting %s over NFS on %s", dev->name, mountpoint);
-    if (vsystem("mount_nfs %s %s %s %s %s %s",
-		!variable_cmp(VAR_NFS_TCP, "YES") ? "-T" : "",
-		!variable_cmp(VAR_NFS_V3, "YES") ? "-3" : "",
-		!variable_cmp(VAR_SLOW_ETHER, "YES") ?
-			"-r 1024 -w 1024" : "-r 4096 -w 4096",
-		!variable_cmp(VAR_NFS_SECURE, "YES") ? "-P" : "",
-		dev->name, mountpoint)) {
-	msgConfirm("Error mounting %s on %s: %s.", dev->name, mountpoint, strerror(errno));
-	if (netDevice)
-	    DEVICE_SHUTDOWN(netDevice);
-	restorescr(w);
-	return FALSE;
-    }
-    NFSMounted = TRUE;
-    if (isDebug())
-	msgDebug("Mounted NFS device %s onto %s\n", dev->name, mountpoint);
-    restorescr(w);
-    return TRUE;
-}
-
-FILE *
-mediaGetNFS(Device *dev, char *file, Boolean probe)
-{
-    return mediaGenericGet(mountpoint, file);
-}
-
-void
-mediaShutdownNFS(Device *dev)
-{
-    if (!NFSMounted)
-	return;
-
-    msgDebug("Unmounting NFS partition on %s\n", mountpoint);
-    if (unmount(mountpoint, MNT_FORCE) != 0)
-	msgConfirm("Could not unmount the NFS partition: %s", strerror(errno));
-    NFSMounted = FALSE;
-    return;
-}
diff --git a/usr.sbin/sysinstall/options.c b/usr.sbin/sysinstall/options.c
deleted file mode 100644
index 33b50311ff9c..000000000000
--- a/usr.sbin/sysinstall/options.c
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated for what's essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <curses.h>
-#include <term.h>
-
-int fixitTtyWhich(dialogMenuItem *);
-
-static char *
-varCheck(Option *opt)
-{
-    char *cp = NULL;
-
-    if (opt->aux)
-	cp = variable_get((char *)opt->aux);
-    if (!cp)
-	return "NO";
-    return cp;
-}
-
-/* Show our little logo */
-static char *
-resetLogo(Option *opt)
-{
-    return "[RESET!]";
-}
-
-static char *
-mediaCheck(Option *opt)
-{
-    if (mediaDevice) {
-	switch(mediaDevice->type) {
-	case DEVICE_TYPE_UFS:
-	case DEVICE_TYPE_DISK:
-	    return "File system";
-
-	case DEVICE_TYPE_FLOPPY:
-	    return "Floppy";
-
-	case DEVICE_TYPE_FTP:
-	    return "FTP";
-
-	case DEVICE_TYPE_CDROM:
-	    return "CDROM";
-
-	case DEVICE_TYPE_USB:
-	    return "USB";
-
-	case DEVICE_TYPE_DOS:
-	    return "DOS";
-
-	case DEVICE_TYPE_NFS:
-	    return "NFS";
-
-	case DEVICE_TYPE_NONE:
-	case DEVICE_TYPE_NETWORK:
-	case DEVICE_TYPE_ANY:
-	default:
-	    return "<unknown>";
-	}
-    }
-    return "<not yet set>";
-}
-
-#define NEWFS_PROMPT	"Please enter newfs(8) parameters:"
-#define RELNAME_PROMPT	"Please specify the release you wish to load or\n\"any\" for a generic release install:"
-#define BPKG_PROMPT	"Please specify the name of the HTML browser package:"
-#define BBIN_PROMPT	"Please specify a full pathname to the HTML browser binary:"
-#define EDITOR_PROMPT	"Please specify the name of the text editor you wish to use:"
-#define PKG_PROMPT	"Please specify a temporary directory with lots of free space:"
-#define INSTROOT_PROMPT	"Please specify a root directory if installing somewhere other than /"
-#define TIMEOUT_PROMPT	"Please specify the number of seconds to wait for slow media:"
-
-static Option Options[] = {
-{ "NFS Secure",		"NFS server talks only on a secure port",
-      OPT_IS_VAR,	NULL,			VAR_NFS_SECURE,		varCheck	},
-{ "NFS Slow",		"User is using a slow PC or ethernet card",
-      OPT_IS_VAR,	NULL,			VAR_SLOW_ETHER,		varCheck	},
-{ "NFS TCP",		"Use TCP protocol for NFS",
-      OPT_IS_VAR,	NULL,			VAR_NFS_TCP,		varCheck	},
-{ "NFS version 3",	"Use NFS version 3",
-      OPT_IS_VAR,	NULL,			VAR_NFS_V3,		varCheck	},
-{ "Debugging",		"Emit extra debugging output on VTY2 (ALT-F2)",
-      OPT_IS_VAR,	NULL,			VAR_DEBUG,		varCheck	},
-{ "No Warnings",	"Don't Warn the user when a setting seems incorrect",
-      OPT_IS_VAR,	NULL,			VAR_NO_WARN,		varCheck	},
-{ "Yes to All",		"Assume \"Yes\" answers to all non-critical dialogs",
-      OPT_IS_VAR,	NULL,			VAR_NO_CONFIRM,		varCheck	},
-{ "DHCP",		"Attempt automatic DHCP configuration of interfaces",
-      OPT_IS_VAR,	NULL,			VAR_TRY_DHCP,		varCheck	},
-{ "IPv6",		"Attempt IPv6 configuration of interfaces",
-      OPT_IS_VAR,	NULL,			VAR_TRY_RTSOL,		varCheck	},
-{ "FTP username",	"Username and password to use instead of anonymous",
-      OPT_IS_FUNC,	mediaSetFTPUserPass,	VAR_FTP_USER,		varCheck	},
-{ "Editor",		"Which text editor to use during installation",
-      OPT_IS_VAR,	EDITOR_PROMPT,		VAR_EDITOR,		varCheck	},
-{ "Extract Detail",	"How verbosely to display file name information during extractions",
-      OPT_IS_FUNC,	mediaSetCPIOVerbosity,	VAR_CPIO_VERBOSITY,	varCheck	},
-{ "Release Name",	"Which release to attempt to load from installation media",
-      OPT_IS_VAR,	RELNAME_PROMPT,		VAR_RELNAME,		varCheck	},
-{ "Install Root",	"Which directory to unpack distributions or packages relative to",
-      OPT_IS_VAR,	INSTROOT_PROMPT,	VAR_INSTALL_ROOT,	varCheck	},
-{ "Browser package",	"This is the browser package that will be used for viewing HTML docs",
-      OPT_IS_VAR,	BPKG_PROMPT,		VAR_BROWSER_PACKAGE,	varCheck	},
-{ "Browser Exec",	"This is the path to the main binary of the browser package",
-      OPT_IS_VAR,	BBIN_PROMPT,		VAR_BROWSER_BINARY,	varCheck	},
-{ "Media Type",		"The current installation media type.",
-      OPT_IS_FUNC,	mediaGetType,		VAR_MEDIA_TYPE,		mediaCheck	},
-{ "Media Timeout",	"Timeout value in seconds for slow media.",
-      OPT_IS_VAR,	TIMEOUT_PROMPT,		VAR_MEDIA_TIMEOUT,	varCheck	},
-{ "Package Temp",	"The directory where package temporary files should go",
-      OPT_IS_VAR,	PKG_PROMPT,		VAR_PKG_TMPDIR,		varCheck	},
-{ "Newfs Args",	 	"Default parameters for newfs(8)",
-      OPT_IS_VAR,	NEWFS_PROMPT,		VAR_NEWFS_ARGS,		varCheck	},
-{ "Fixit Console",	"Which tty to use for the Fixit action.",
-      OPT_IS_FUNC,	fixitTtyWhich,		VAR_FIXIT_TTY,		varCheck	},
-{ "Re-scan Devices",	"Re-run sysinstall's initial device probe",
-      OPT_IS_FUNC,	deviceRescan,		NULL,			NULL		},
-{ "Use Defaults",	"Reset all values to startup defaults",
-      OPT_IS_FUNC,	installVarDefaults,	NULL,			resetLogo	},
-{ NULL, NULL, 0, NULL, NULL, NULL },
-};
-
-#define OPT_START_ROW	4
-#define OPT_END_ROW	19
-#define OPT_NAME_COL	0
-#define OPT_VALUE_COL	16
-#define GROUP_OFFSET	40
-
-static char *
-value_of(Option opt)
-{
-    static char ival[40];
-
-    switch (opt.type) {
-    case OPT_IS_STRING:
-	return (char *)opt.data;
-
-    case OPT_IS_INT:
-	sprintf(ival, "%lu", (long)opt.data);
-	return ival;
-
-    case OPT_IS_FUNC:
-    case OPT_IS_VAR:
-	if (opt.check)
-	    return opt.check(&opt);
-	else
-	    return "<*>";
-    }
-    return "<unknown>";
-}
-
-static int
-fire(Option opt)
-{
-    int status = 0;
-
-    if (opt.type == OPT_IS_FUNC) {
-	int (*cp)(char *) = opt.data, rcode;
-
-	rcode = cp(NULL);
-	status = 1;
-    }
-    else if (opt.type == OPT_IS_VAR) {
-	if (opt.data) {
-	    (void)variable_get_value(opt.aux, opt.data, -1);
-	    status = 1;
-	}
-	else if (variable_get(opt.aux)) {
-	    if (!variable_cmp(opt.aux, "YES"))
-		variable_set2(opt.aux, "NO", -1);
-	    else
-		variable_set2(opt.aux, "YES", -1);
-	}
-	else
-	    variable_set2(opt.aux, "YES", 0);
-    }
-    if (opt.check)
-	opt.check(&opt);
-    refresh();
-    return status;
-}
-
-int
-optionsEditor(dialogMenuItem *self)
-{
-    int i, optcol, optrow, key;
-    static int currOpt = 0;
-    WINDOW *w = savescr();
-    
-    dialog_clear();
-    clear();
-
-    while (1) {
-	/* Whap up the header */
-	attrset(A_REVERSE); mvaddstr(0, 0, "Options Editor"); attrset(A_NORMAL);
-	for (i = 0; i < 2; i++) {
-	    mvaddstr(OPT_START_ROW - 2, OPT_NAME_COL + (i * GROUP_OFFSET), "Name");
-	    mvaddstr(OPT_START_ROW - 1, OPT_NAME_COL + (i * GROUP_OFFSET), "----");
-
-	    mvaddstr(OPT_START_ROW - 2, OPT_VALUE_COL + (i * GROUP_OFFSET), "Value");
-	    mvaddstr(OPT_START_ROW - 1, OPT_VALUE_COL + (i * GROUP_OFFSET), "-----");
-	}
-	/* And the footer */
-	mvprintw(OPT_END_ROW + 1, 0, "Use SPACE to select/toggle an option, arrow keys to move,");
-	mvprintw(OPT_END_ROW + 2, 0, "? or F1 for more help.  When you're done, type Q to Quit.");
-
-	optrow = OPT_START_ROW;
-	optcol = OPT_NAME_COL;
-	for (i = 0; Options[i].name; i++) {
-	    /* Names are painted somewhat gratuitously each time, but it's easier this way */
-	    mvprintw(optrow, OPT_NAME_COL + optcol, Options[i].name);
-	    if (currOpt == i)
-		attrset(ATTR_SELECTED);
-	    mvprintw(optrow++, OPT_VALUE_COL + optcol, value_of(Options[i]));
-	    if (currOpt == i)
-		attrset(A_NORMAL);
-	    if (optrow == OPT_END_ROW) {
-		optrow = OPT_START_ROW;
-		optcol += GROUP_OFFSET;
-	    }
-	    clrtoeol();
-	}
-	attrset(ATTR_TITLE);
-	mvaddstr(OPT_END_ROW + 4, 0, Options[currOpt].desc);
-	attrset(A_NORMAL);
-	clrtoeol();
-	move(0, 14);
-        refresh();
-
-	/* Start the edit loop */
-	key = toupper(getch());
-	switch (key) {
-	case KEY_F(1):
-	case '?':
-	    systemDisplayHelp("options");
-	    clear();
-	    break;
-
-	case '\020':	/* ^P */
-	case KEY_UP:
-	    if (currOpt)
-		--currOpt;
-	    else
-		for (currOpt = 0; Options[currOpt + 1].name; currOpt++);
-	    continue;
-
-	case '\016':	/* ^N */
-	case KEY_DOWN:
-	    if (Options[currOpt + 1].name)
-		++currOpt;
-	    else
-		currOpt = 0;
-	    continue;
-
-	case KEY_HOME:
-	    currOpt = 0;
-	    continue;
-
-	case KEY_END:
-	    while (Options[currOpt + 1].name)
-		++currOpt;
-	    continue;
-
-	case ' ':
-	    if (fire(Options[currOpt]))
-	    	clear();
-	    continue;
-
-	case '\033':	/* ESC */
-	case 'Q':
-	    clear();
-	    dialog_clear();
-	    restorescr(w);
-	    return DITEM_SUCCESS | DITEM_CONTINUE;
-
-	default:
-	    beep();
-	}
-    }
-    /* NOTREACHED */
-    return DITEM_SUCCESS;
-}
-
-int
-fixitTtyWhich(dialogMenuItem *self)
-{
-    char *cp = variable_get(VAR_FIXIT_TTY);
-
-    if (!cp) {
-	msgConfirm("The Fix-it TTY setting is not set to anything!");
-	return DITEM_FAILURE;
-    }
-    else {
-	if (!strcmp(cp, "standard"))
-	    variable_set2(VAR_FIXIT_TTY, "serial", 0);
-	else /* must be "serial" - wrap around */
-	    variable_set2(VAR_FIXIT_TTY, "standard", 0);
-    }
-    return DITEM_SUCCESS;
-}
diff --git a/usr.sbin/sysinstall/package.c b/usr.sbin/sysinstall/package.c
deleted file mode 100644
index 9aa4589c0e58..000000000000
--- a/usr.sbin/sysinstall/package.c
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include "sysinstall.h"
-#include <sys/errno.h>
-#include <sys/time.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/stat.h>
-
-static Boolean sigpipe_caught;
-
-static void
-catch_pipe(int sig)
-{
-    sigpipe_caught = TRUE;
-}
-
-extern PkgNode Top;
-
-/* Like package_extract, but assumes current media device and chases deps */
-int
-package_add(char *name)
-{
-    PkgNodePtr tmp;
-    int i, current, low, high;
-
-    if (!mediaVerify())
-	return DITEM_FAILURE;
-
-    if (!DEVICE_INIT(mediaDevice))
-	return DITEM_FAILURE;    
-
-    i = index_initialize("packages/INDEX");
-    if (DITEM_STATUS(i) != DITEM_SUCCESS)
-	return i;
-
-    tmp = index_search(&Top, name, &tmp);
-    if (tmp) {
-	if (have_volumes) {
-	    low = low_volume;
-	    high = high_volume;
-	} else
-	    low = high = 0;
-	for (current = low; current <= high; current++)
-	    i = index_extract(mediaDevice, &Top, tmp, FALSE, current);
-	return i;
-    } else {
-	msgConfirm("Sorry, package %s was not found in the INDEX.", name);
-	return DITEM_FAILURE;
-    }
-}
-
-/* For use by dispatch */
-int
-packageAdd(dialogMenuItem *self)
-{
-    char *cp;
-
-    cp = variable_get(VAR_PACKAGE);
-    if (!cp) {
-	msgDebug("packageAdd:  No package name passed in package variable\n");
-	return DITEM_FAILURE;
-    }
-    else
-	return package_add(cp);
-}
-
-Boolean
-package_installed(char *name)
-{
-    char fname[FILENAME_MAX];
-    int status /* = vsystem("pkg_info -e %s", name) */;
-
-    /* XXX KLUDGE ALERT!  This makes evil assumptions about how XXX
-     * packages register themselves and should *really be done with
-     * `pkg_info -e <name>' except that this it's too slow for an
-     * item check routine.. :-(
-     */
-    snprintf(fname, FILENAME_MAX, "/var/db/pkg/%s", name);
-    status = access(fname, R_OK);
-    if (isDebug())
-	msgDebug("package check for %s returns %s.\n", name, status ? "failure" : "success");
-    return !status;
-}
-
-/* Extract a package based on a namespec and a media device */
-int
-package_extract(Device *dev, char *name, Boolean depended)
-{
-    char path[MAXPATHLEN];
-    const char *PkgExts[] = { "", ".tbz", ".tbz2", ".tgz" };
-    int last_msg, pathend, ret;
-    size_t ext;
-    FILE *fp;
-
-    last_msg = 0;
-
-    /* Check to make sure it's not already there */
-    if (package_installed(name))
-	return DITEM_SUCCESS;
-
-    if (!DEVICE_INIT(dev)) {
-	msgConfirm("Unable to initialize media type for package extract.");
-	return DITEM_FAILURE;
-    }
-
-    /* If necessary, initialize the ldconfig hints */
-    if (!file_readable("/var/run/ld-elf.so.hints"))
-	vsystem("ldconfig /usr/lib /usr/lib/compat /usr/local/lib");
-
-    /* Be initially optimistic */
-    ret = DITEM_SUCCESS;
-    /* Make a couple of paranoid locations for temp files to live if user specified none */
-    if (!variable_get(VAR_PKG_TMPDIR)) {
-	/* Set it to a location with as much space as possible */
-	variable_set2(VAR_PKG_TMPDIR, "/var/tmp", 0);
-    }
-    Mkdir(variable_get(VAR_PKG_TMPDIR));
-    vsystem("chmod 1777 %s", variable_get(VAR_PKG_TMPDIR));
-
-    if (!index(name, '/')) {
-	if (!strpbrk(name, "-_"))
-	    pathend = snprintf(path, sizeof path, "packages/Latest/%s", name);
-	else
-	    pathend = snprintf(path, sizeof path, "packages/All/%s", name);
-    }
-    else
-	pathend = snprintf(path, sizeof path, "%s", name);
-
-    /* We have a path, call the device strategy routine to get the file */
-    for (ext = 0 ; ext < sizeof PkgExts / sizeof PkgExts[0]; ++ext) {
-	strlcpy(path + pathend, PkgExts[ext], sizeof path - pathend);
-	if ((fp = DEVICE_GET(dev, path, TRUE)))
-	    break;
-    }
-
-    if (fp) {
-	int i = 0, tot, pfd[2];
-	pid_t pid;
-	WINDOW *w = savescr();
-
-	sigpipe_caught = FALSE;
-	signal(SIGPIPE, catch_pipe);
-
-        dialog_clear_norefresh();
-	msgNotify("Adding %s%s\nfrom %s", path, depended ? " (as a dependency)" : "", dev->name);
-	pipe(pfd);
-	pid = fork();
-	if (!pid) {
-	    dup2(pfd[0], 0); close(pfd[0]);
-	    dup2(DebugFD, 1); dup2(1, 2);
-	    close(pfd[1]);
-
-	    /* Prevent pkg_add from wanting to interact in bad ways */
-	    setenv("BATCH", "t", 1);
-
-	    if (isDebug())
-		i = execl("/usr/sbin/pkg_add", "/usr/sbin/pkg_add", "-v", "-",
-		    (char *)0);
-	    else
-		i = execl("/usr/sbin/pkg_add", "/usr/sbin/pkg_add", "-",
-		    (char *)0);
-	}
-	else {
-	    char buf[BUFSIZ];
-	    struct timeval start, stop;
-
-	    close(pfd[0]);
-	    tot = 0;
-	    (void)gettimeofday(&start, (struct timezone *)0);
-
-	    while (!sigpipe_caught && (i = fread(buf, 1, BUFSIZ, fp)) > 0) {
-		int seconds;
-
-		tot += i;
-		/* Print statistics about how we're doing */
-		(void) gettimeofday(&stop, (struct timezone *)0);
-		stop.tv_sec = stop.tv_sec - start.tv_sec;
-		stop.tv_usec = stop.tv_usec - start.tv_usec;
-		if (stop.tv_usec < 0)
-		    stop.tv_sec--, stop.tv_usec += 1000000;
-		seconds = stop.tv_sec + (stop.tv_usec / 1000000.0);
-		if (!seconds)
-		    seconds = 1;
-		if (seconds != last_msg) {
-		    last_msg = seconds;
-		    msgInfo("%10d bytes read from package %s @ %4.1f KBytes/second", tot, name, (tot / seconds) / 1000.0);
-		}
-		/* Write it out */
-		if (sigpipe_caught || write(pfd[1], buf, i) != i) {
-		    msgInfo("Write failure to pkg_add!  Package may be corrupt.");
-		    break;
-		}
-	    }
-	    close(pfd[1]);
-	    fclose(fp);
-	    if (sigpipe_caught)
-		msgInfo("pkg_add(1) apparently did not like the %s package.", name);
-	    else if (i == -1)
-		msgInfo("I/O error while reading in the %s package.", name);
-	    else
-		msgInfo("Package %s read successfully - waiting for pkg_add(1)", name);
-	    refresh();
-	    i = waitpid(pid, &tot, 0);
-	    dialog_clear_norefresh();
-	    if (sigpipe_caught || i < 0 || WEXITSTATUS(tot)) {
-		ret = DITEM_FAILURE;
-		if (variable_get(VAR_NO_CONFIRM))
-		    msgNotify("Add of package %s aborted, error code %d -\n"
-			       "Please check the debug screen for more info.", name, WEXITSTATUS(tot));
-		else
-		    msgConfirm("Add of package %s aborted, error code %d -\n"
-			       "Please check the debug screen for more info.", name, WEXITSTATUS(tot));
-	    }
-	    else
-		msgNotify("Package %s was added successfully", name);
-
-	    /* Now catch any stragglers */
-	    while (wait3(&tot, WNOHANG, NULL) > 0);
-
-	    sleep(1);
-	    restorescr(w);
-	}
-    }
-    else {
-	dialog_clear_norefresh();
-	if (variable_get(VAR_NO_CONFIRM))
-	    msgNotify("Unable to fetch package %s from selected media.\n"
-		      "No package add will be done.", name);
-	else
-	    msgConfirm("Unable to fetch package %s from selected media.\n"
-		       "No package add will be done.", name);
-	ret = DITEM_FAILURE;
-    }
-    signal(SIGPIPE, SIG_IGN);
-    return ret;
-}
diff --git a/usr.sbin/sysinstall/rtermcap.c b/usr.sbin/sysinstall/rtermcap.c
deleted file mode 100644
index 84b3feb49ead..000000000000
--- a/usr.sbin/sysinstall/rtermcap.c
+++ /dev/null
@@ -1,15 +0,0 @@
-#include <stdio.h>
-#include <termcap.h>
-
-int
-main(int argc, char **argv)
-{
-	char	buf[4096];
-	int	i;
-
-	if (argc < 2)
-		return 1;
-	i = tgetent(buf, argv[1]);
-	printf("%s",buf);
-	return 0;
-}
diff --git a/usr.sbin/sysinstall/sysinstall.8 b/usr.sbin/sysinstall/sysinstall.8
deleted file mode 100644
index ad6760c75ba0..000000000000
--- a/usr.sbin/sysinstall/sysinstall.8
+++ /dev/null
@@ -1,899 +0,0 @@
-.\" Copyright (c) 1997
-.\"	Jordan Hubbard <jkh@FreeBSD.org>.  All rights reserved.
-.\"
-.\" Redistribution and use in source and binary forms, with or without
-.\" modification, are permitted provided that the following conditions
-.\" are met:
-.\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY Jordan Hubbard AND CONTRIBUTORS ``AS IS'' AND
-.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-.\" ARE DISCLAIMED.  IN NO EVENT SHALL Jordan Hubbard OR CONTRIBUTORS BE LIABLE
-.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-.\" SUCH DAMAGE.
-.\"
-.\" $FreeBSD$
-.\"
-.Dd February 18, 2007
-.Dt SYSINSTALL 8
-.Os
-.Sh NAME
-.Nm sysinstall
-.Nd system installation and configuration tool
-.Sh SYNOPSIS
-.Nm
-.Op Ar var=value
-.Op Ar function
-.Op Ar ...
-.Sh DESCRIPTION
-The
-.Nm
-utility is used for installing and configuring
-.Fx
-systems.
-It is the first utility invoked by the
-.Fx
-installation boot
-floppy and is also available as
-.Pa /usr/sbin/sysinstall
-on newly installed
-.Fx
-systems for use in later configuring the system.
-.Pp
-The
-.Nm
-utility is generally invoked without arguments for the default
-behavior, where the main installation/configuration menu is presented.
-.Pp
-On those occasions where it is deemed necessary to invoke a subsystem
-of sysinstall directly, however, it is also possible to do so by
-naming the appropriate function entry points on the command line.
-Since this action is essentially identical to running an installation
-script, each command-line argument corresponding to a line of script,
-the reader is encouraged to read the section on scripting for more
-information on this feature.
-.Sh NOTES
-The
-.Nm
-utility is essentially nothing more than a monolithic C program with
-the ability to write MBRs and disk labels (through the services
-of the
-.Xr libdisk 3
-library) and install distributions or packages onto new and
-existing
-.Fx
-systems.
-It also contains some extra intelligence
-for running as a replacement for
-.Xr init 8
-when it is invoked by the
-.Fx
-installation boot procedure.
-It
-assumes very little in the way of additional utility support and
-performs most file system operations by calling the relevant syscalls
-(such as
-.Xr mount 2 )
-directly.
-.Pp
-The
-.Nm
-utility currently uses the
-.Xr dialog 3
-library to do user interaction with simple ANSI line graphics, color
-support for which is enabled by either running on a syscons VTY or some
-other color-capable terminal emulator (newer versions of xterm will support
-color when using the
-.Dq xterm-color
-termcap entry).
-.Pp
-This product is currently at the end of its life cycle and will
-eventually be replaced.
-.Sh RUNNING SCRIPTS
-The
-.Nm
-utility may be either driven interactively through its various internal menus
-or run in batch mode, driven by an external script.
-Such a script may
-be loaded and executed in one of 3 ways:
-.Bl -tag -width Ds
-.It Sy "LOAD_CONFIG_FILE"
-If
-.Nm
-is compiled with LOAD_CONFIG_FILE set in the environment
-(or in the Makefile) to some value, then that value will
-be used as the filename to automatically look for and load
-when
-.Nm
-starts up and with no user interaction required.
-This option is aimed primarily at large sites who wish to create a
-single prototype install for multiple machines with largely identical
-configurations and/or installation options.
-.It Sy "MAIN MENU"
-If
-.Nm
-is run interactively, that is to say in the default manner, it will
-bring up a main menu which contains a "load config file" option.
-Selecting this option will prompt for the name of a script file which
-it then will attempt to load from a DOS or UFS formatted floppy.
-.It Sy "COMMAND LINE"
-Each command line argument is treated as a script directive
-when
-.Nm
-is run in multi-user mode.
-Execution ends either by explicit request
-(e.g.\& calling the
-.Ar shutdown
-directive), upon reaching the end of the argument list or on error.
-.Pp
-For example:
-.Bd -literal
-/usr/sbin/sysinstall _ftpPath=ftp://ziggy/pub/ mediaSetFTP configPackages
-.Ed
-.Pp
-Would initialize
-.Nm
-for FTP installation media (using the server `ziggy') and then
-bring up the package installation editor, exiting when finished.
-.El
-.Sh SCRIPT SYNTAX
-A script is a list of one or more directives, each directive taking
-the form of:
-.Pp
-.Ar var=value
-.Pp
-.Ar function
-.Pp
-or
-.Ar #somecomment
-.Pp
-Where
-.Ar var=value
-is the assignment of some internal
-.Nm
-variable, e.g.\& "ftpPass=FuNkYChiKn", and
-.Ar function
-is the name of an internal
-.Nm
-function, e.g.\& "mediaSetFTP", and
-.Ar #comment
-is a single-line comment for documentation purposes (ignored by
-sysinstall).
-Each directive must be by itself on a single line,
-functions taking their arguments by examining known variable names.
-This requires that you be sure to assign the relevant variables before
-calling a function which requires them.
-.Pp
-The
-.Ar noError
-variable can be assigned before each directive: this will cause any error
-detected while processing the directive itself to be ignored.
-The value of
-.Ar noError
-will automatically reset to the default "unassigned" every time a directive is
-processed.
-.Pp
-When and where a function depends on the settings of one or more variables
-will be noted in the following table:
-.Pp
-.Sy "Function Glossary" :
-.Bl -tag -width indent
-.It configAnonFTP
-Invoke the Anonymous FTP configuration menu.
-.Pp
-.Sy Variables :
-None
-.It configRouter
-Select which routing daemon you wish to use, potentially
-loading any required 3rd-party routing daemons as necessary.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It router
-can be set to the name of the desired routing daemon,
-e.g.\&
-.Dq routed
-or
-.Dq gated ,
-otherwise it is prompted for.
-.El
-.It configNFSServer
-Configure host as an NFS server.
-.Pp
-.Sy Variables :
-None
-.It configNTP
-Configure host as a user of the Network Time Protocol.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It ntpdate_hosts
-Whitespace-separated list of
-.Xr ntpdate 8
-servers to sync from.
-.El
-.It configPCNFSD
-Configure host to support PC NFS.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It pcnfsd_pkg
-The name of the PCNFSD package to load if necessary (defaults to hard coded
-version).
-.El
-.It configPackages
-Bring up the interactive package management menu.
-.Pp
-.Sy Variables :
-None
-.It configUsers
-Add users and/or groups to the system.
-.Pp
-.Sy Variables :
-None
-.It diskPartitionEditor
-Invokes the disk partition (MBR) editor.
-.Pp
-.Sy Variables :
-.Bl -tag -width findx
-.It geometry
-The disk geometry, as a cyls/heads/sectors formatted string.
-The word "sane" instructs
-.Nm
-to calculate a safe (not necessarily optimal) geometry if the
-current one has more than 65535 cylinders, more than 256 heads or
-more than 63 sectors per track (255 sectors on the PC98
-architecture).
-Default: no
-change to geometry.
-.It partition
-Set to disk partitioning type or size, its value being
-.Ar free
-in order to use only remaining free space for
-.Fx ,
-.Ar all
-to use the entire disk for
-.Fx
-but maintain a proper partition
-table,
-.Ar existing
-to use an existing
-.Fx
-partition (first found),
-.Ar exclusive
-to use the disk in
-.Dq dangerously dedicated
-mode or, finally,
-.Ar somenumber
-to allocate
-.Ar somenumber
-blocks of available free space to a new
-.Fx
-partition.
-Default: Interactive mode.
-.It bootManager
-is set to one of
-.Ar boot
-to signify the installation of a boot manager,
-.Ar standard
-to signify installation of a "standard" non-boot MGR DOS
-MBR or
-.Ar none
-to indicate that no change to the boot manager is desired.
-Default: none.
-.It diskInteractive
-If set, bring up the interactive disk partition editor.
-.El
-.Pp
-Note: Nothing is actually written to disk by this function, an explicit call to
-.Ar diskPartitionWrite
-being required for that to happen.
-.It diskPartitionWrite
-Causes any pending MBR changes (typically from the
-.Ar diskPartitionEditor
-function) to be written out.
-.Pp
-.Sy Variables :
-None
-.It diskLabelEditor
-Invokes the disk label editor.
-This is a bit trickier from a script
-since you need to essentially label everything inside each
-.Fx
-(type 0xA5) partition created by the
-.Ar diskPartitionEditor
-function, and that requires knowing a few rules about how things are
-laid out.
-When creating a script to automatically allocate disk space
-and partition it up, it is suggested that you first perform the
-installation interactively at least once and take careful notes as to
-what the slice names will be, then and only then hardwiring them into
-the script.
-.Pp
-For example, let's say you have a SCSI disk on which you have created a new
-.Fx
-partition in slice 2 (your DOS partition residing in slice 1).
-The slice name would be
-.Ar da0s2
-for the whole
-.Fx
-partition
-.Ar ( da0s1
-being your DOS primary
-partition).
-Now let's further assume that you have 4GB in this
-partition and you want to sub-partition that space into root, swap,
-var and usr file systems for
-.Fx .
-Your invocation of the
-.Ar diskLabelEditor
-function might involve setting the following variables:
-.Bl -tag -width findx
-.It Li "da0s2-1=ufs 2097152 /"
-A 1GB root file system (all sizes are in 512 byte blocks).
-.It Li "da0s2-2=swap 1048576 /"
-A 512MB swap partition.
-.It Li "da0s2-3=ufs 524288 /var"
-A 256MB /var file system.
-.It Li "da0s2-4=ufs 0 /usr 1"
-With the balance of free space (around 2.25GB) going to the /usr
-file system and with soft-updates enabled (the argument following
-the mount point, if non-zero, means to set the soft updates flag).
-.El
-.Pp
-One can also use the
-.Ar diskLabelEditor
-for mounting or erasing existing partitions as well as creating new
-ones.
-Using the previous example again, let's say that we also wanted
-to mount our DOS partition and make sure that an
-.Pa /etc/fstab
-entry is created for it in the new installation.
-Before calling the
-.Ar diskLabelEditor
-function, we simply add an additional line:
-.Pp
-.Dl "da0s1=/dos_c N"
-.Pp
-before the call.
-This tells the label editor that you want to mount
-the first slice on
-.Pa /dos_c
-and not to attempt to newfs it (not that
-.Nm
-would attempt this for a DOS partition in any case, but it could just
-as easily be an existing UFS partition being named here and the 2nd
-field is non-optional).
-.Pp
-You can also set the
-.Ar diskInteractive
-variable to request that the disk label editor use an interactive dialog
-to partition the disk instead of using variables to explicitly layout the
-disk as described above.
-.Pp
-Note: No file system data is actually written to disk until an
-explicit call to
-.Ar diskLabelCommit
-is made.
-.It diskLabelCommit
-Writes out all pending disklabel information and creates and/or mounts any
-file systems which have requests pending from the
-.Ar diskLabelEditor
-function.
-.Pp
-.Sy Variables :
-None
-.It distReset
-Resets all selected distributions to the empty set (no distributions selected).
-.Pp
-.Sy Variables :
-None
-.It distSetCustom
-Allows the selection of a custom distribution set (e.g.\& not just one of the
-existing "canned" sets) with no user interaction.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It dists
-List of distributions to load.
-Possible distribution values are:
-.Bl -tag -width indentxx
-.It Li base
-The base binary distribution.
-.It Li GENERIC
-The GENERIC kernel.
-.It Li doc
-Miscellaneous documentation
-.It Li games
-Games
-.It Li manpages
-Manual pages (unformatted)
-.It Li catpages
-Pre-formatted manual pages
-.It Li proflibs
-Profiled libraries for developers.
-.It Li dict
-Dictionary information (for tools like spell).
-.It Li info
-GNU info files and other extra docs.
-.It Li lib32
-(amd64 and powerpc64 only)
-32-bit runtime compatibility libraries.
-.It Li ports
-The ports collection.
-.It Li ssecure
-/usr/src/secure
-.It Li sbase
-/usr/src/[top level files]
-.It Li scontrib
-/usr/src/contrib
-.It Li scrypto
-/usr/src/crypto
-.It Li sgnu
-/usr/src/gnu
-.It Li setc
-/usr/src/etc
-.It Li sgames
-/usr/src/games
-.It Li sinclude
-/usr/src/include
-.It Li skrb5
-/usr/src/kerberos5
-.It Li slib
-/usr/src/lib
-.It Li slibexec
-/usr/src/libexec
-.It Li srelease
-/usr/src/release
-.It Li srescue
-/usr/src/rescue
-.It Li stools
-/usr/src/tools
-.It Li sbin
-/usr/src/bin
-.It Li ssbin
-/usr/src/sbin
-.It Li sshare
-/usr/src/share
-.It Li ssys
-/usr/src/sys
-.It Li stools
-/usr/src/tools
-.It Li subin
-/usr/src/usr.bin
-.It Li susbin
-/usr/src/usr.sbin
-.It Li local
-Local additions collection.
-.El
-.El
-.It distSetDeveloper
-Selects the standard Developer's distribution set.
-.Pp
-.Sy Variables :
-None
-.It distSetUser
-Selects the standard user distribution set.
-.Pp
-.Sy Variables :
-None
-.It distSetMinimum
-Selects the very minimum distribution set.
-.Pp
-.Sy Variables :
-None
-.It distSetEverything
-Selects the full whack - all available distributions.
-.Pp
-.Sy Variables :
-None
-.It distSetSrc
-Interactively select source subcomponents.
-.Pp
-.Sy Variables :
-None
-.It distExtractAll
-Install all currently selected distributions (requires that
-media device also be selected).
-.Pp
-.Sy Variables :
-None
-.It docBrowser
-Install (if necessary) an HTML documentation browser and go to the
-HTML documentation submenu.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It browserPackage
-The name of the browser package to try and install as necessary.
-Defaults to latest links package.
-.It browserBinary
-The name of the browser binary itself (if overriding the
-.Ar browserPackage
-variable).
-Defaults to links.
-.El
-.It installCommit
-Commit any and all pending changes to disk.
-This function
-is essentially shorthand for a number of more granular "commit"
-functions.
-.Pp
-.Sy Variables :
-None
-.It installConfigure
-Commit any rc.conf changes to disk.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It keeprcconf
-Preserve existing rc.conf parameters.
-This is useful if you have a post-install script which modifies rc.conf.
-.El
-.It installExpress
-Start an "express" installation, asking few questions of
-the user.
-.Pp
-.Sy Variables :
-None
-.It installStandard
-Start a "standard" installation, the most user-friendly
-installation type available.
-.Pp
-.Sy Variables :
-None
-.It installUpgrade
-Start an upgrade installation.
-.Pp
-.Sy Variables :
-None
-.It installFixitHoloShell
-Start up the "emergency holographic shell" over on VTY4
-if running as init.
-This will also happen automatically
-as part of the installation process unless
-.Ar noHoloShell
-is set.
-.Pp
-.Sy Variables :
-None
-.It installFixitCDROM
-Go into "fixit" mode, assuming a live file system CDROM
-currently in the drive.
-.Pp
-.Sy Variables :
-None
-.It installFixitFloppy
-Go into "fixit" mode, assuming an available fixit floppy
-disk (user will be prompted for it).
-.Pp
-.Sy Variables :
-None
-.It installFilesystems
-Do just the file system initialization part of an install.
-.Pp
-.Sy Variables :
-None
-.It installVarDefaults
-Initialize all variables to their defaults, overriding any
-previous settings.
-.Pp
-.Sy Variables :
-None
-.It loadConfig
-Sort of like an #include statement, it allows you to load one
-configuration file from another.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It configFile
-The fully qualified pathname of the file to load.
-.El
-.It mediaOpen
-If a media device is set, mount it.
-.Pp
-.Sy Variables :
-None
-.It mediaClose
-If a media device is open, close it.
-.Pp
-.Sy Variables :
-None
-.It mediaSetCDROM
-Select a
-.Fx
-CDROM as the installation media.
-.Pp
-.Sy Variables :
-None
-.It mediaSetFloppy
-Select a pre-made floppy installation set as the installation media.
-.Pp
-.Sy Variables :
-None
-.It mediaSetDOS
-Select an existing DOS primary partition as the installation media.
-The first primary partition found is used (e.g.\& C:).
-.Pp
-.Sy Variables :
-None
-.It mediaSetFTP
-Select an FTP site as the installation media.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It hostname
-The name of the host being installed (non-optional).
-.It domainname
-The domain name of the host being installed (optional).
-.It defaultrouter
-The default router for this host (non-optional).
-.It netDev
-Which network interface to use. 
-Multiple network interfaces may be specified using a comma delimited list. 
-If netDev is set to ANY, all available network interfaces will be probed and the
-first interface found to have a link will be used. (non-optional).
-.It netInteractive
-If set, bring up the interactive network setup form even
-if all relevant configuration variables are already set (optional).
-.It ipaddr
-The IP address for the selected host interface (non-optional).
-.It netmask
-The netmask for the selected host interface (non-optional).
-.It _ftpPath
-The fully qualified URL of the FTP site containing the
-.Fx
-distribution you are interested in, e.g.\&
-.Ar ftp://ftp.FreeBSD.org/pub/FreeBSD/ .
-.El
-.It mediaSetFTPActive
-Alias for
-.Ar mediaSetFTP
-using "active" FTP transfer mode.
-.Pp
-.Sy Variables :
-Same as for
-.Ar mediaSetFTP .
-.It mediaSetFTPPassive
-Alias for
-.Ar mediaSetFTP
-using "passive" FTP transfer mode.
-.Pp
-.Sy Variables :
-Same as for
-.Ar mediaSetFTP .
-.It mediaSetHTTP
-Alias for
-.Ar mediaSetFTP
-using an HTTP proxy.
-.Pp
-.Sy Variables :
-See
-.Ar mediaSetFTP ,
-plus
-.Bl -tag -width indent
-.It _httpPath
-The proxy to use (host:port) (non-optional).
-.El
-.It mediaSetUFS
-Select an existing UFS partition (mounted with the label editor) as
-the installation media.
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It ufs
-full /path to directory containing the
-.Fx
-distribution you are
-interested in.
-.El
-.It mediaSetNFS
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It hostname
-The name of the host being installed (non-optional).
-.It domainname
-The domain name of the host being installed (optional).
-.It defaultrouter
-The default router for this host (non-optional).
-.It netDev
-Which host interface to use
-.Ar ( ed0
-or
-.Ar ep0 ,
-for example.
-Non-optional).
-.It netInteractive
-If set, bring up the interactive network setup form even
-if all relevant configuration variables are already set (optional).
-.It ipaddr
-The IP address for the selected host interface (non-optional).
-.It netmask
-The netmask for the selected host interface (non-optional).
-.It nfs
-full hostname:/path specification for directory containing
-the
-.Fx
-distribution you are interested in.
-.El
-.It mediaSetFTPUserPass
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It ftpUser
-The username to log in as on the ftp server site.
-Default: ftp
-.It ftpPass
-The password to use for this username on the ftp
-server site.
-Default: user@host
-.El
-.It mediaSetCPIOVerbosity
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It cpioVerbose
-Can be used to set the verbosity of cpio extractions to low or high.
-.El
-.It mediaGetType
-Interactively get the user to specify some type of media.
-.Pp
-.Sy Variables :
-None
-.It optionsEditor
-Invoke the interactive options editor.
-.Pp
-.Sy Variables :
-None
-.It packageAdd
-Try to fetch and add a package to the system (requires
-that a media type be set),
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It package
-The name of the package to add, e.g.\& bash-1.14.7 or ncftp-2.4.2.
-.El
-.It addGroup
-Invoke the interactive group editor.
-.Pp
-.Sy Variables :
-None
-.It addUser
-Invoke the interactive user editor.
-.Pp
-.Sy Variables :
-None
-.It shutdown
-Stop the script, terminate sysinstall and reboot the system.
-On the sparc64 platform, the system is halted rather than rebooted.
-.Pp
-.Sy Variables :
-None
-.It system
-Execute an arbitrary command with
-.Xr system 3
-.Pp
-.Sy Variables :
-.Bl -tag -width indent
-.It command
-The name of the command to execute.
-When running
-from a boot floppy, very minimal expectations should
-be made as to what is available until/unless a relatively
-full system installation has just been done.
-.El
-.It tcpMenuSelect
-Configure a network device.
-.Pp
-.Sy Variables :
-Same as for
-.Ar mediaSetFTP
-except that
-.Ar _ftpPath
-is not used.
-.El
-.Sh DISTRIBUTION MEDIA
-The following files can be used to affect the operation of
-.Nm
-when used during initial system installation.
-.Bl -tag -width ".Pa packages/INDEX"
-.It Pa cdrom.inf
-A text file of properties, listed one per line, that describe the
-contents of the media in use.
-The syntax for each line is simply
-.Dq Ar property No = Ar value .
-Currently, only the following properties are recognized.
-.Bl -tag -width ".Va CD_MACHINE_ARCH"
-.It Va CD_VERSION
-This property should be set to the
-.Fx
-version on the current
-media volume.
-For example,
-.Dq Li "CD_VERSION = 5.3" .
-.It Va CD_MACHINE_ARCH
-This property should be set to the architecture of the contents on
-this volume.
-This property is normally only used with
-.Fx
-products that contain
-CDs for different architectures, to provide better error messages if
-users try to install packages built for the wrong architecture.
-For example,
-.Dq Li "CD_MACHINE_ARCH = amd64" .
-.It Va CD_VOLUME
-In a multi-volume collection (such as the
-.Fx
-4-CD set), the
-.Pa ports/INDEX
-file on each disc should contain the full package index for the set.
-The last field of the
-.Pa INDEX
-file denotes which volume the package
-appears on, and the
-.Va CD_VOLUME
-property here defines the volume ID of the current disc.
-.El
-.It Pa packages/INDEX
-The package index file.
-Each package is listed on a separate line with additional meta-data
-such as the required dependencies.
-This index is generated by
-.Dq Li "make index"
-from the
-.Xr ports 7
-collection.
-When multi-volume support is enabled, an additional field should be
-added to each line indicating which media volume contains the given
-package.
-.El
-.Pp
-For information about building a full release of
-.Fx ,
-please see
-.Xr release 7 .
-.Sh FILES
-This utility may edit the contents of
-.Pa /etc/rc.conf ,
-.Pa /etc/hosts ,
-and
-.Pa /etc/resolv.conf
-as necessary to reflect changes in the network configuration.
-.Sh SEE ALSO
-If you have a reasonably complete source tree online, take
-a look at
-.Pa /usr/src/usr.sbin/sysinstall/install.cfg
-for a sample installation script.
-.Sh HISTORY
-This version of
-.Nm
-first appeared in
-.Fx 2.0 .
-.Sh AUTHORS
-.An Jordan K. Hubbard Aq jkh@FreeBSD.org
-.Sh BUGS
-Editing slice and partition tables on disks which are currently mounted by
-the system is not allowed.
-This is generally only a problem when
-.Nm
-is run on a system that is already installed.
-Use
-.Xr fdisk 8
-and
-.Xr bsdlabel 8
-for these tasks.
-.Pp
-There are a (great) number of undocumented variables.
-UTSL.
diff --git a/usr.sbin/sysinstall/sysinstall.h b/usr.sbin/sysinstall/sysinstall.h
deleted file mode 100644
index d5f4b3288f47..000000000000
--- a/usr.sbin/sysinstall/sysinstall.h
+++ /dev/null
@@ -1,895 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef _SYSINSTALL_H_INCLUDE
-#define _SYSINSTALL_H_INCLUDE
-
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <errno.h>
-#include <setjmp.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <dialog.h>
-#include "ui_objects.h"
-#include "dir.h"
-#include "colors.h"
-#include "dist.h"
-
-/*** Defines ***/
-
-#if defined(__i386__) || defined(__amd64__)
-#define	WITH_SYSCONS
-#define	WITH_MICE
-#endif
-
-#if defined(__i386__) || defined(__amd64__)
-#define	WITH_SLICES
-#endif
-
-/* device limits */
-#define DEV_NAME_MAX		128	/* The maximum length of a device name	*/
-#define DEV_MAX			100	/* The maximum number of devices we'll deal with */
-#define INTERFACE_MAX		50	/* Maximum number of network interfaces we'll deal with */
-#define IO_ERROR		-2	/* Status code for I/O error rather than normal EOF */
-
-/* Number of seconds to wait for data to come off even the slowest media */
-#define MEDIA_TIMEOUT		300
-
-/*
- * I make some pretty gross assumptions about having a max of 50 chunks
- * total - 8 slices and 42 partitions.  I can't easily display many more
- * than that on the screen at once!
- *
- * For 2.1 I'll revisit this and try to make it more dynamic, but since
- * this will catch 99.99% of all possible cases, I'm not too worried.
- */
-#define MAX_CHUNKS	40
-
-/* Internal environment variable names */
-#define DISK_PARTITIONED		"_diskPartitioned"
-#define DISK_LABELLED			"_diskLabelled"
-#define DISK_SELECTED			"_diskSelected"
-#define SYSTEM_STATE			"_systemState"
-#define RUNNING_ON_ROOT			"_runningOnRoot"
-#define TCP_CONFIGURED			"_tcpConfigured"
-
-/* Ones that can be tweaked from config files */
-#define VAR_BLANKTIME			"blanktime"
-#define VAR_BOOTMGR			"bootManager"
-#define VAR_BROWSER_BINARY		"browserBinary"
-#define VAR_BROWSER_PACKAGE		"browserPackage"
-#define VAR_COUNTRY			"country"
-#define VAR_CPIO_VERBOSITY		"cpioVerbose"
-#define VAR_DEBUG			"debug"
-#define VAR_DESKSTYLE			"_deskStyle"
-#define VAR_DISK			"disk"
-#define VAR_DISKINTERACTIVE		"diskInteractive"
-#define VAR_DISTS			"dists"
-#define VAR_DIST_MAIN			"distMain"
-#define VAR_DIST_SRC			"distSRC"
-#define VAR_DIST_KERNEL			"distKernel"
-#define VAR_DIST_DOC			"distDoc"
-#define VAR_DEDICATE_DISK		"dedicateDisk"
-#define VAR_DOMAINNAME			"domainname"
-#define VAR_EDITOR			"editor"
-#define VAR_EXTRAS			"ifconfig_"
-#define VAR_COMMAND			"command"
-#define VAR_CONFIG_FILE			"configFile"
-#define VAR_FIXIT_TTY			"fixitTty"
-#define VAR_FTP_DIR			"ftpDirectory"
-#define VAR_FTP_PASS			"ftpPass"
-#define VAR_FTP_PATH			"_ftpPath"
-#define VAR_FTP_PORT			"ftpPort"
-#define VAR_FTP_STATE			"ftpState"
-#define VAR_FTP_USER			"ftpUser"
-#define VAR_FTP_HOST			"ftpHost"
-#define VAR_HTTP_PATH			"_httpPath"
-#define VAR_HTTP_PROXY			"httpProxy"
-#define VAR_HTTP_PORT			"httpPort"
-#define VAR_HTTP_HOST			"httpHost"
-#define VAR_HTTP_FTP_MODE		"httpFtpMode"
-#define VAR_GATEWAY			"defaultrouter"
-#define VAR_GEOMETRY			"geometry"
-#define VAR_HOSTNAME			"hostname"
-#define VAR_IFCONFIG			"ifconfig_"
-#define VAR_INSTALL_CFG			"installConfig"
-#define VAR_INSTALL_ROOT		"installRoot"
-#define VAR_IPADDR			"ipaddr"
-#define VAR_IPV6_ENABLE			"ipv6_activate_all_interfaces"
-#define VAR_IPV6ADDR			"ipv6addr"
-#define VAR_KERN_SECURELEVEL		"kern_securelevel"
-#define VAR_KEYMAP			"keymap"
-#define VAR_LABEL			"label"
-#define VAR_LABEL_COUNT			"labelCount"
-#define VAR_LINUX_ENABLE		"linux_enable"
-#define VAR_MEDIA_TYPE			"mediaType"
-#define VAR_MEDIA_TIMEOUT		"MEDIA_TIMEOUT"
-#define VAR_MOUSED			"moused_enable"
-#define VAR_MOUSED_FLAGS                "moused_flags"
-#define VAR_MOUSED_PORT			"moused_port"
-#define VAR_MOUSED_TYPE			"moused_type"
-#define VAR_NAMESERVER			"nameserver"
-#define	VAR_NCPUS			"ncpus"
-#define VAR_NETINTERACTIVE		"netInteractive"
-#define VAR_NETMASK			"netmask"
-#define VAR_NETWORK_DEVICE		"netDev"
-#define VAR_NEWFS_ARGS			"newfsArgs"
-#define VAR_NFS_PATH			"nfs"
-#define VAR_NFS_HOST			"nfsHost"
-#define VAR_NFS_V3			"nfs_use_v3"
-#define VAR_NFS_TCP			"nfs_use_tcp"
-#define VAR_NFS_SECURE			"nfs_reserved_port_only"
-#define VAR_NFS_SERVER			"nfs_server_enable"
-#define VAR_NO_CONFIRM			"noConfirm"
-#define VAR_NO_ERROR			"noError"
-#define VAR_NO_HOLOSHELL		"noHoloShell"
-#define VAR_NO_INET6			"noInet6"
-#define VAR_NO_WARN			"noWarn"
-#define VAR_NO_USR			"noUsr"
-#define VAR_NO_TMP			"noTmp"
-#define VAR_NO_HOME			"noHome"
-#define VAR_NONINTERACTIVE		"nonInteractive"
-#define VAR_NOVELL			"novell"
-#define	VAR_RPCBIND_ENABLE		"rpcbind_enable"
-#define VAR_NTPDATE_HOSTS		"ntpdate_hosts"
-#define VAR_PACKAGE			"package"
-#define VAR_PARTITION			"partition"
-#define VAR_PCNFSD			"pcnfsd"
-#define VAR_PKG_TMPDIR			"PKG_TMPDIR"
-#define VAR_PORTS_PATH			"ports"
-#define VAR_PPP_ENABLE			"ppp_enable"
-#define VAR_PPP_PROFILE			"ppp_profile"
-#define VAR_RELNAME			"releaseName"
-#define VAR_ROOT_SIZE			"rootSize"
-#define VAR_ROUTER			"router"
-#define VAR_ROUTER_ENABLE		"router_enable"
-#define VAR_ROUTERFLAGS			"router_flags"
-#define VAR_SENDMAIL_ENABLE		"sendmail_enable"
-#define VAR_SERIAL_SPEED		"serialSpeed"
-#define VAR_SLOW_ETHER			"slowEthernetCard"
-#define VAR_SWAP_SIZE			"swapSize"
-#define VAR_SYSLOG_SERVER		"syslogdServer"
-#define VAR_TRY_DHCP			"tryDHCP"
-#define VAR_TRY_RTSOL			"tryRTSOL"
-#define VAR_UFS_PATH			"ufs"
-#define VAR_USR_SIZE			"usrSize"
-#define VAR_VAR_SIZE			"varSize"
-#define VAR_TMP_SIZE			"tmpSize"
-#define VAR_HOME_SIZE			"homeSize"
-#define VAR_TERM			"TERM"
-#define VAR_CONSTERM                    "_consterm"
-#define VAR_KEEPRCCONF			"keeprcconf"
-
-#ifdef PC98
-#define DEFAULT_COUNTRY		"jp"
-#else
-#define DEFAULT_COUNTRY		"us"
-#endif
-
-/* One MB worth of blocks */
-#define ONE_MEG				2048
-#define ONE_GIG				(ONE_MEG * 1024)
-
-/* Which selection attributes to use */
-#define ATTR_SELECTED			(ColorDisplay ? item_selected_attr : item_attr)
-#define ATTR_TITLE	button_active_attr
-
-/* Handy strncpy() macro */
-#define SAFE_STRCPY(to, from)	sstrncpy((to), (from), sizeof (to) - 1)
-
-/*** Types ***/
-typedef int Boolean;
-typedef struct disk Disk;
-typedef struct chunk Chunk;
-
-/* Bitfields for menu options */
-#define DMENU_NORMAL_TYPE	0x1     /* Normal dialog menu           */
-#define DMENU_RADIO_TYPE	0x2     /* Radio dialog menu            */
-#define DMENU_CHECKLIST_TYPE	0x4     /* Multiple choice menu         */
-#define DMENU_SELECTION_RETURNS 0x8     /* Immediate return on item selection */
-
-typedef struct _dmenu {
-    int type;				/* What sort of menu we are	*/
-    char *title;			/* Our title			*/
-    char *prompt;			/* Our prompt			*/
-    char *helpline;			/* Line of help at bottom	*/
-    char *helpfile;			/* Help file for "F1"		*/
-    dialogMenuItem items[];		/* Array of menu items		*/
-} DMenu;
-
-/* An rc.conf variable */
-typedef struct _variable {
-    struct _variable *next;
-    char *name;
-    char *value;
-    int dirty;
-} Variable;
-
-#define NO_ECHO_OBJ(type)	((type) | (DITEM_NO_ECHO << 16))
-#define TYPE_OF_OBJ(type)	((type) & 0xff)
-#define ATTR_OF_OBJ(type)	((type) >> 16)
-
-/* A screen layout structure */
-typedef struct _layout {
-    int         y;              /* x & Y co-ordinates */
-    int         x;
-    int         len;            /* The size of the dialog on the screen */
-    int         maxlen;         /* How much the user can type in ... */
-    char        *prompt;        /* The string for the prompt */
-    char        *help;          /* The display for the help line */
-    void        *var;           /* The var to set when this changes */
-    int         type;           /* The type of the dialog to create */
-    void        *obj;           /* The obj pointer returned by libdialog */
-} Layout;
-
-/* Layout array terminator. */
-#define	LAYOUT_END		{ 0, 0, 0, 0, NULL, NULL, NULL, 0, NULL }
-
-typedef enum {
-    DEVICE_TYPE_NONE,
-    DEVICE_TYPE_DISK,
-    DEVICE_TYPE_FLOPPY,
-    DEVICE_TYPE_FTP,
-    DEVICE_TYPE_NETWORK,
-    DEVICE_TYPE_CDROM,
-    DEVICE_TYPE_USB,
-    DEVICE_TYPE_DOS,
-    DEVICE_TYPE_UFS,
-    DEVICE_TYPE_NFS,
-    DEVICE_TYPE_ANY,
-    DEVICE_TYPE_HTTP,
-} DeviceType;
-
-/* CDROM mount codes */
-#define CD_UNMOUNTED		0
-#define CD_ALREADY_MOUNTED	1
-#define CD_WE_MOUNTED_IT	2
-
-/* A "device" from sysinstall's point of view */
-typedef struct _device {
-    char name[DEV_NAME_MAX];
-    char *description;
-    char *devname;
-    DeviceType type;
-    Boolean enabled;
-    Boolean (*init)(struct _device *dev);
-    FILE * (*get)(struct _device *dev, char *file, Boolean probe);
-    void (*shutdown)(struct _device *dev);
-    void *private;
-    unsigned int flags;
-    unsigned int volume;
-} Device;
-
-/* Some internal representations of partitions */
-typedef enum {
-    PART_NONE,
-    PART_SLICE,
-    PART_SWAP,
-    PART_FILESYSTEM,
-    PART_FAT,
-    PART_EFI
-} PartType;
-
-#define	NEWFS_UFS_CMD		"newfs"
-#define	NEWFS_MSDOS_CMD		"newfs_msdos"
-
-enum newfs_type { NEWFS_UFS, NEWFS_MSDOS, NEWFS_CUSTOM };
-#define	NEWFS_UFS_STRING	"UFS"
-#define	NEWFS_MSDOS_STRING	"FAT"
-#define	NEWFS_CUSTOM_STRING	"CST"
-
-/* The longest set of custom command line arguments we'll pass. */
-#define NEWFS_CMD_ARGS_MAX	256
-
-typedef struct _part_info {
-	char mountpoint[FILENAME_MAX];
-
-	/* Is invocation of newfs desired? */
-	Boolean do_newfs;
-
-	enum newfs_type newfs_type;
-	union {
-		struct {
-			char user_options[NEWFS_CMD_ARGS_MAX];
-			Boolean acls;		/* unused */
-			Boolean multilabel;	/* unused */
-			Boolean softupdates;
-			Boolean ufs1;
-		} newfs_ufs;
-		struct {
-			/* unused */
-		} newfs_msdos;
-		struct {
-			char command[NEWFS_CMD_ARGS_MAX];
-		} newfs_custom;
-	} newfs_data;
-} PartInfo;
-
-/* An option */
-typedef struct _opt {
-    char *name;
-    char *desc;
-    enum { OPT_IS_STRING, OPT_IS_INT, OPT_IS_FUNC, OPT_IS_VAR } type;
-    void *data;
-    void *aux;
-    char *(*check)(struct _opt *);
-} Option;
-
-/* Weird index nodey things we use for keeping track of package information */
-typedef enum { PACKAGE, PLACE } node_type;	/* Types of nodes */
-
-typedef struct _pkgnode {	/* A node in the reconstructed hierarchy */
-    struct _pkgnode *next;	/* My next sibling			*/
-    node_type type;		/* What am I?				*/
-    char *name;			/* My name				*/
-    char *desc;			/* My description (Hook)		*/
-    struct _pkgnode *kids;	/* My little children			*/
-    void *data;			/* A place to hang my data		*/
-} PkgNode;
-typedef PkgNode *PkgNodePtr;
-
-/* A single package */
-typedef struct _indexEntry {	/* A single entry in an INDEX file */
-    char *name;			/* name				*/
-    char *path;			/* full path to port		*/
-    char *prefix;		/* port prefix			*/
-    char *comment;		/* one line description		*/
-    char *descrfile;		/* path to description file	*/
-    char *deps;			/* packages this depends on	*/
-    int  depc;			/* how many depend on me	*/
-    int  installed;		/* indicates if it is installed */
-    int  vol_checked;		/* disc volume last checked for */
-    char *maintainer;		/* maintainer			*/
-    unsigned int volume;	/* Volume of package            */
-} IndexEntry;
-typedef IndexEntry *IndexEntryPtr;
-
-typedef int (*commandFunc)(char *key, void *data);
-
-#define HOSTNAME_FIELD_LEN	128
-#define IPADDR_FIELD_LEN	16
-#define EXTRAS_FIELD_LEN	128
-
-/* This is the structure that Network devices carry around in their private, erm, structures */
-typedef struct _devPriv {
-    int use_rtsol;
-    int use_dhcp;
-    char ipaddr[IPADDR_FIELD_LEN];
-    char netmask[IPADDR_FIELD_LEN];
-    char extras[EXTRAS_FIELD_LEN];
-} DevInfo;
-
-
-/*** Externs ***/
-extern jmp_buf		BailOut;		/* Used to get the heck out */
-extern int		CDROMInitQuiet;		/* Don't whine if mount(2) fails */
-extern int		DebugFD;		/* Where diagnostic output goes			*/
-extern Boolean		Fake;			/* Don't actually modify anything - testing	*/
-extern Boolean		Restarting;		/* Are we restarting sysinstall?		*/
-extern Boolean		SystemWasInstalled;	/* Did we install it?				*/
-extern Boolean		RunningAsInit;		/* Are we running stand-alone?			*/
-extern Boolean		DialogActive;		/* Is the dialog() stuff up?			*/
-extern Boolean		ColorDisplay;		/* Are we on a color display?			*/
-extern Boolean		OnVTY;			/* On a syscons VTY?				*/
-extern Boolean		have_volumes;		/* Media has multiple volumes                   */
-extern Variable		*VarHead;		/* The head of the variable chain		*/
-extern Device		*mediaDevice;		/* Where we're getting our distribution from	*/
-extern unsigned int	Dists;			/* Which distributions we want			*/
-extern unsigned int	DocDists;		/* Which Doc dists we want			*/
-extern unsigned int	SrcDists;		/* Which src distributions we want		*/
-extern unsigned int	KernelDists;		/* Which kernel dists we want			*/
-extern int		BootMgr;		/* Which boot manager to use 			*/
-extern int		StatusLine;		/* Where to print our status messages		*/
-extern DMenu		MenuCountry;		/* Country menu				*/
-extern DMenu		MenuInitial;		/* Initial installation menu			*/
-extern DMenu		MenuFixit;		/* Fixit repair menu				*/
-#if defined(__i386__) || defined(__amd64__)
-#ifdef PC98
-extern DMenu		MenuIPLType;		/* Type of IPL to write on the disk		*/
-#else
-extern DMenu		MenuMBRType;		/* Type of MBR to write on the disk		*/
-#endif
-#endif
-extern DMenu		MenuConfigure;		/* Final configuration menu			*/
-extern DMenu		MenuDocInstall;		/* Documentation Installation menu		*/
-extern DMenu		MenuDocumentation;	/* Documentation menu				*/
-extern DMenu		MenuFTPOptions;		/* FTP Installation options			*/
-extern DMenu		MenuIndex;		/* Index menu					*/
-extern DMenu		MenuOptions;		/* Installation options				*/
-extern DMenu		MenuOptionsLanguage;	/* Language options menu			*/
-extern DMenu		MenuKLD;		/* Prototype KLD menu				*/
-extern DMenu		MenuConfig;		/* Prototype config menu				*/
-extern DMenu		MenuMedia;		/* Media type menu				*/
-#ifdef WITH_MICE
-extern DMenu		MenuMouse;		/* Mouse type menu				*/
-#endif
-extern DMenu		MenuMediaCDROM;		/* CDROM media menu				*/
-extern DMenu		MenuMediaUSB;		/* USB media menu				*/
-extern DMenu		MenuMediaDOS;		/* DOS media menu				*/
-extern DMenu		MenuMediaFloppy;	/* Floppy media menu				*/
-extern DMenu		MenuMediaFTP;		/* FTP media menu				*/
-extern DMenu		MenuNetworkDevice;	/* Network device menu				*/
-extern DMenu		MenuNTP;		/* NTP time server menu				*/
-extern DMenu		MenuSecurity;		/* System security options menu			*/
-extern DMenu		MenuSecurelevel;	/* Securelevel menu */
-extern DMenu		MenuStartup;		/* Startup services menu			*/
-#ifdef WITH_SYSCONS
-extern DMenu		MenuSyscons;		/* System console configuration menu		*/
-extern DMenu		MenuSysconsFont;	/* System console font configuration menu	*/
-extern DMenu		MenuSysconsKeymap;	/* System console keymap configuration menu	*/
-extern DMenu		MenuSysconsKeyrate;	/* System console keyrate configuration menu	*/
-extern DMenu		MenuSysconsSaver;	/* System console saver configuration menu	*/
-extern DMenu		MenuSysconsScrnmap;	/* System console screenmap configuration menu	*/
-extern DMenu            MenuSysconsTtys;        /* System console terminal type menu            */
-#endif
-extern DMenu		MenuNetworking;		/* Network configuration menu			*/
-extern DMenu		MenuMTA;		/* MTA selection menu				*/
-extern DMenu		MenuInstallCustom;	/* Custom Installation menu			*/
-extern DMenu		MenuDistributions;	/* Distribution menu				*/
-extern DMenu		MenuDiskDevices;	/* Disk type devices				*/
-extern DMenu		MenuSubDistributions;	/* Custom distribution menu			*/
-extern DMenu		MenuSrcDistributions;	/* Source distribution menu			*/
-extern DMenu		MenuKernelDistributions;/* Kernel distribution menu			*/
-extern DMenu		MenuHTMLDoc;		/* HTML Documentation menu			*/
-extern DMenu		MenuUsermgmt;		/* User management menu				*/
-extern DMenu		MenuFixit;		/* Fixit floppy/CDROM/shell menu		*/
-extern int              FixItMode;              /* FixItMode starts shell on current device (ie Serial port) */
-extern const char *	StartName;		/* Which name we were started as */
-extern const char *	ProgName;		/* Program's proper name */
-extern int		NCpus;			/* # cpus on machine */
-extern int		low_volume;		/* Lowest volume number */
-extern int		high_volume;		/* Highest volume number */
-
-/* Important chunks. */
-extern Chunk *HomeChunk;
-extern Chunk *RootChunk;
-extern Chunk *SwapChunk;
-extern Chunk *TmpChunk;
-extern Chunk *UsrChunk;
-extern Chunk *VarChunk;
-#ifdef __ia64__
-extern Chunk *EfiChunk;
-#endif
-
-/* Stuff from libdialog which isn't properly declared outside */
-extern void display_helpfile(void);
-extern void display_helpline(WINDOW *w, int y, int width);
-
-/*** Prototypes ***/
-
-/* acpi.c */
-extern int	acpi_detect(void);
-
-/* anonFTP.c */
-extern int	configAnonFTP(dialogMenuItem *self);
-
-/* cdrom.c */
-extern Boolean	mediaInitCDROM(Device *dev);
-extern FILE	*mediaGetCDROM(Device *dev, char *file, Boolean probe);
-extern void	mediaShutdownCDROM(Device *dev);
-
-/* command.c */
-extern void	command_clear(void);
-extern void	command_sort(void);
-extern void	command_execute(void);
-extern void	command_shell_add(char *key, char *fmt, ...) __printflike(2, 3);
-extern void	command_func_add(char *key, commandFunc func, void *data);
-
-/* config.c */
-extern void	configEnvironmentRC_conf(void);
-extern void	configEnvironmentResolv(char *config);
-extern void	configRC_conf(void);
-extern int	configFstab(dialogMenuItem *self);
-extern int	configRC(dialogMenuItem *self);
-extern int	configResolv(dialogMenuItem *self);
-extern int	configPackages(dialogMenuItem *self);
-extern int	configSaver(dialogMenuItem *self);
-extern int	configSaverTimeout(dialogMenuItem *self);
-#ifdef WITH_LINUX
-extern int	configLinux(dialogMenuItem *self);
-#endif
-extern int	configNTP(dialogMenuItem *self);
-extern int	configCountry(dialogMenuItem *self);
-extern int	configUsers(dialogMenuItem *self);
-extern int	configRouter(dialogMenuItem *self);
-extern int	configPCNFSD(dialogMenuItem *self);
-extern int	configInetd(dialogMenuItem *self);
-extern int	configNFSServer(dialogMenuItem *self);
-extern int	configMTAPostfix(dialogMenuItem *self);
-extern int	configMTAExim(dialogMenuItem *self);
-extern int	configRpcBind(dialogMenuItem *self);
-extern int	configWriteRC_conf(dialogMenuItem *self);
-extern int	configSecurelevel(dialogMenuItem *self);
-extern int	configSecurelevelDisabled(dialogMenuItem *self);
-extern int	configSecurelevelSecure(dialogMenuItem *self);
-extern int	configSecurelevelHighlySecure(dialogMenuItem *self);
-extern int	configSecurelevelNetworkSecure(dialogMenuItem *self);
-extern int	configEtcTtys(dialogMenuItem *self);
-#ifdef __i386__
-extern int	checkLoaderACPI(void);
-extern int	configLoaderACPI(int);
-#endif
-
-/* devices.c */
-extern DMenu	*deviceCreateMenu(DMenu *menu, DeviceType type, int (*hook)(dialogMenuItem *d),
-				  int (*check)(dialogMenuItem *d));
-extern void	deviceGetAll(void);
-extern void	deviceReset(void);
-extern void	deviceRescan(void);
-extern Device	**deviceFind(char *name, DeviceType type);
-extern Device	**deviceFindDescr(char *name, char *desc, DeviceType class);
-extern int	deviceCount(Device **devs);
-extern Device	*new_device(char *name);
-extern Device	*deviceRegister(char *name, char *desc, char *devname, DeviceType type, Boolean enabled,
-				Boolean (*init)(Device *mediadev),
-				FILE * (*get)(Device *dev, char *file, Boolean probe),
-				void (*shutDown)(Device *mediadev),
-				void *private);
-extern Boolean	dummyInit(Device *dev);
-extern FILE	*dummyGet(Device *dev, char *dist, Boolean probe);
-extern void	dummyShutdown(Device *dev);
-
-/* dhcp.c */
-extern int	dhcpParseLeases(char *file, char *hostname, char *domain, char *nameserver,
-				char *ipaddr, char *gateway, char *netmask);
-
-/* disks.c */
-#ifdef WITH_SLICES
-extern void	diskPartition(Device *dev);
-extern int	diskPartitionEditor(dialogMenuItem *self);
-#endif
-extern int	diskPartitionWrite(dialogMenuItem *self);
-extern int	diskGetSelectCount(Device ***devs);
-
-/* dispatch.c */
-extern int	dispatchCommand(char *command);
-extern int	dispatch_load_floppy(dialogMenuItem *self);
-extern int	dispatch_load_cdrom(dialogMenuItem *self);
-extern int	dispatch_load_file_int(int);
-extern int	dispatch_load_file(dialogMenuItem *self);
-extern int	dispatch_load_menu(dialogMenuItem *self);
-
-
-/* dist.c */
-extern int	distReset(dialogMenuItem *self);
-extern int	distConfig(dialogMenuItem *self);
-extern int	distSetCustom(dialogMenuItem *self);
-extern int	distUnsetCustom(dialogMenuItem *self);
-extern int	distSetDeveloper(dialogMenuItem *self);
-extern int	distSetKernDeveloper(dialogMenuItem *self);
-extern int	distSetUser(dialogMenuItem *self);
-extern int	distSetMinimum(dialogMenuItem *self);
-extern int	distSetEverything(dialogMenuItem *self);
-extern int	distSetSrc(dialogMenuItem *self);
-extern int	distSetKernel(dialogMenuItem *self);
-extern int	distSetDoc(dialogMenuItem *self);
-extern int	distSetDocMenu(dialogMenuItem *self);
-extern int	distExtractAll(dialogMenuItem *self);
-extern int	selectKernel(void);
-
-/* dmenu.c */
-extern int	dmenuDisplayFile(dialogMenuItem *tmp);
-extern int	dmenuSubmenu(dialogMenuItem *tmp);
-extern int	dmenuSystemCommand(dialogMenuItem *tmp);
-extern int	dmenuSystemCommandBox(dialogMenuItem *tmp);
-extern int	dmenuExit(dialogMenuItem *tmp);
-extern int	dmenuISetVariable(dialogMenuItem *tmp);
-extern int	dmenuSetVariable(dialogMenuItem *tmp);
-extern int	dmenuSetCountryVariable(dialogMenuItem *tmp);
-extern int	dmenuSetKmapVariable(dialogMenuItem *tmp);
-extern int	dmenuSetVariables(dialogMenuItem *tmp);
-extern int	dmenuToggleVariable(dialogMenuItem *tmp);
-extern int	dmenuSetFlag(dialogMenuItem *tmp);
-extern int	dmenuSetValue(dialogMenuItem *tmp);
-extern int	dmenuFindItem(DMenu *menu, const char *prompt, const char *title, void *data);
-extern void	dmenuSetDefaultIndex(DMenu *menu, int *choice, int *scroll, int *curr, int *max);
-extern int	dmenuSetDefaultItem(DMenu *menu, const char *prompt, const char *title, void *data,
-				    int *choice, int *scroll, int *curr, int *max);
-extern Boolean	dmenuOpen(DMenu *menu, int *choice, int *scroll, int *curr, int *max, Boolean buttons);
-extern Boolean	dmenuOpenSimple(DMenu *menu, Boolean buttons);
-extern int	dmenuVarCheck(dialogMenuItem *item);
-extern int	dmenuVarsCheck(dialogMenuItem *item);
-extern int	dmenuFlagCheck(dialogMenuItem *item);
-extern int	dmenuRadioCheck(dialogMenuItem *item);
-
-/* doc.c */
-extern int	docBrowser(dialogMenuItem *self);
-extern int	docShowDocument(dialogMenuItem *self);
-
-/* dos.c */
-extern Boolean	mediaCloseDOS(Device *dev, FILE *fp);
-extern Boolean	mediaInitDOS(Device *dev);
-extern FILE	*mediaGetDOS(Device *dev, char *file, Boolean probe);
-extern void	mediaShutdownDOS(Device *dev);
-
-/* floppy.c */
-extern int	getRootFloppy(void);
-extern Boolean	mediaInitFloppy(Device *dev);
-extern FILE	*mediaGetFloppy(Device *dev, char *file, Boolean probe);
-extern void	mediaShutdownFloppy(Device *dev);
-
-/* ftp_strat.c */
-extern Boolean	mediaCloseFTP(Device *dev, FILE *fp);
-extern Boolean	mediaInitFTP(Device *dev);
-extern FILE	*mediaGetFTP(Device *dev, char *file, Boolean probe);
-extern void	mediaShutdownFTP(Device *dev);
-
-/* http.c */
-extern Boolean	mediaInitHTTP(Device *dev);
-extern FILE	*mediaGetHTTP(Device *dev, char *file, Boolean probe);
-
-/* globals.c */
-extern void	globalsInit(void);
-
-/* index.c */
-int		index_read(FILE *fp, PkgNodePtr papa);
-int		index_menu(PkgNodePtr root, PkgNodePtr top, PkgNodePtr plist, int *pos, int *scroll);
-void		index_init(PkgNodePtr top, PkgNodePtr plist);
-void		index_node_free(PkgNodePtr top, PkgNodePtr plist);
-void		index_sort(PkgNodePtr top);
-void		index_print(PkgNodePtr top, int level);
-int		index_extract(Device *dev, PkgNodePtr top, PkgNodePtr who, Boolean depended, int current_volume);
-int		index_initialize(char *path);
-PkgNodePtr	index_search(PkgNodePtr top, char *str, PkgNodePtr *tp);
-
-/* install.c */
-extern Boolean	checkLabels(Boolean whinge);
-extern int	installCommit(dialogMenuItem *self);
-extern int	installCustomCommit(dialogMenuItem *self);
-extern int	installExpress(dialogMenuItem *self);
-extern int	installStandard(dialogMenuItem *self);
-extern int	installFixitHoloShell(dialogMenuItem *self);
-extern int	installFixitCDROM(dialogMenuItem *self);
-extern int	installFixitUSB(dialogMenuItem *self);
-extern int	installFixitFloppy(dialogMenuItem *self);
-extern int	installFixupBase(dialogMenuItem *self);
-extern int	installFixupKernel(dialogMenuItem *self, int dists);
-extern int	installUpgrade(dialogMenuItem *self);
-extern int	installFilesystems(dialogMenuItem *self);
-extern int	installVarDefaults(dialogMenuItem *self);
-extern void	installEnvironment(void);
-extern Boolean	copySelf(void);
-
-/* kget.c */
-extern int	kget(char *out);
-
-/* keymap.c */
-extern int	keymapMenuSelect(dialogMenuItem *self);
-extern int	loadKeymap(const char *lang);
-
-/* label.c */
-extern int	diskLabelEditor(dialogMenuItem *self);
-extern int	diskLabelCommit(dialogMenuItem *self);
-
-/* makedevs.c (auto-generated) */
-extern const char	termcap_ansi[];
-extern const char	termcap_vt100[];
-extern const char	termcap_cons25w[];
-extern const char	termcap_cons25[];
-extern const char	termcap_cons25_m[];
-extern const char	termcap_cons25r[];
-extern const char	termcap_cons25r_m[];
-extern const char	termcap_cons25l1[];
-extern const char	termcap_cons25l1_m[];
-extern const char	termcap_xterm[];
-extern const u_char	font_iso_8x16[];
-extern const u_char	font_cp850_8x16[];
-extern const u_char	font_cp866_8x16[];
-extern const u_char	koi8_r2cp866[];
-extern u_char		default_scrnmap[];
-
-/* media.c */
-extern char	*cpioVerbosity(void);
-extern int	mediaOpen(void);
-extern void	mediaClose(void);
-extern int	mediaTimeout(void);
-extern int	mediaSetCDROM(dialogMenuItem *self);
-extern int	mediaSetFloppy(dialogMenuItem *self);
-extern int	mediaSetUSB(dialogMenuItem *self);
-extern int	mediaSetDOS(dialogMenuItem *self);
-extern int	mediaSetFTP(dialogMenuItem *self);
-extern int	mediaSetFTPActive(dialogMenuItem *self);
-extern int	mediaSetFTPPassive(dialogMenuItem *self);
-extern int	mediaSetHTTP(dialogMenuItem *self);
-extern int	mediaSetUFS(dialogMenuItem *self);
-extern int	mediaSetNFS(dialogMenuItem *self);
-extern int	mediaSetFTPUserPass(dialogMenuItem *self);
-extern int	mediaSetCPIOVerbosity(dialogMenuItem *self);
-extern int	mediaGetType(dialogMenuItem *self);
-extern Boolean	mediaExtractDist(char *dir, char *dist, FILE *fp);
-extern Boolean	mediaExtractDistBegin(char *dir, int *fd, int *zpid, int *cpic);
-extern Boolean	mediaExtractDistEnd(int zpid, int cpid);
-extern Boolean	mediaVerify(void);
-extern FILE	*mediaGenericGet(char *base, const char *file);
-
-/* misc.c */
-extern Boolean	file_readable(char *fname);
-extern Boolean	file_executable(char *fname);
-extern Boolean	directory_exists(const char *dirname);
-extern char	*root_bias(char *path);
-extern char	*itoa(int value);
-extern char	*string_concat(char *p1, char *p2);
-extern char	*string_concat3(char *p1, char *p2, char *p3);
-extern char	*string_prune(char *str);
-extern char	*string_skipwhite(char *str);
-extern char	*string_copy(char *s1, char *s2);
-extern char	*pathBaseName(const char *path);
-extern void	safe_free(void *ptr);
-extern void	*safe_malloc(size_t size);
-extern void	*safe_realloc(void *orig, size_t size);
-extern dialogMenuItem *item_add(dialogMenuItem *list, char *prompt, char *title, 
-				int (*checked)(dialogMenuItem *self),
-				int (*fire)(dialogMenuItem *self),
-				void (*selected)(dialogMenuItem *self, int is_selected),
-				void *data, void *aux, int *curr, int *max);
-extern void	items_free(dialogMenuItem *list, int *curr, int *max);
-extern int	Mkdir(char *);
-extern int	Mkdir_command(char *key, void *data);
-extern int	Mount(char *, void *data);
-extern int	Mount_msdosfs(char *mountp, void *devname);
-extern WINDOW	*openLayoutDialog(char *helpfile, char *title, int x, int y, int width, int height);
-extern ComposeObj *initLayoutDialog(WINDOW *win, Layout *layout, int x, int y, int *max);
-extern int	layoutDialogLoop(WINDOW *win, Layout *layout, ComposeObj **obj,
-				 int *n, int max, int *cbutton, int *cancel);
-
-extern WINDOW	*savescr(void);
-extern void	restorescr(WINDOW *w);
-extern char	*sstrncpy(char *dst, const char *src, int size);
-extern char	*getsysctlbyname(const char *sysctlname);
-
-/* modules.c */
-extern void	driverFloppyCheck(void);
-extern void	moduleInitialize(void);
-extern int	kldBrowser(dialogMenuItem *self);
-
-/* mouse.c */
-extern int	mousedTest(dialogMenuItem *self);
-extern int	mousedDisable(dialogMenuItem *self);
-extern int      setMouseFlags(dialogMenuItem *self);
-
-/* mptable.c */
-extern int	biosmptable_detect(void);
-
-/* msg.c */
-extern Boolean	isDebug(void);
-extern void	msgInfo(char *fmt, ...) __printf0like(1, 2);
-extern void	msgYap(char *fmt, ...) __printflike(1, 2);
-extern void	msgWarn(char *fmt, ...) __printflike(1, 2);
-extern void	msgDebug(char *fmt, ...) __printflike(1, 2);
-extern void	msgError(char *fmt, ...) __printflike(1, 2);
-extern void	msgFatal(char *fmt, ...) __printflike(1, 2);
-extern void	msgConfirm(char *fmt, ...) __printflike(1, 2);
-extern void	msgNotify(char *fmt, ...) __printflike(1, 2);
-extern void	msgWeHaveOutput(char *fmt, ...) __printflike(1, 2);
-extern int	msgYesNo(char *fmt, ...) __printflike(1, 2);
-extern int	msgNoYes(char *fmt, ...) __printflike(1, 2);
-extern char	*msgGetInput(char *buf, char *fmt, ...) __printflike(2, 3);
-extern int	msgSimpleConfirm(char *);
-extern int	msgSimpleNotify(char *);
-
-/* network.c */
-extern Boolean	mediaInitNetwork(Device *dev);
-extern void	mediaShutdownNetwork(Device *dev);
-
-/* nfs.c */
-extern Boolean	mediaInitNFS(Device *dev);
-extern FILE	*mediaGetNFS(Device *dev, char *file, Boolean probe);
-extern void	mediaShutdownNFS(Device *dev);
-
-/* options.c */
-extern int	optionsEditor(dialogMenuItem *self);
-
-/* package.c */
-extern int	packageAdd(dialogMenuItem *self);
-extern int	package_add(char *name);
-extern int	package_extract(Device *dev, char *name, Boolean depended);
-extern Boolean	package_installed(char *name);
-
-/* system.c */
-extern void	systemInitialize(int argc, char **argv);
-extern void	systemShutdown(int status);
-extern int	execExecute(char *cmd, char *name);
-extern int	systemExecute(char *cmd);
-extern void	systemSuspendDialog(void);
-extern void	systemResumeDialog(void);
-extern int	systemDisplayHelp(char *file);
-extern char	*systemHelpFile(char *file, char *buf);
-extern void	systemChangeFont(const u_char font[]);
-extern void	systemChangeLang(char *lang);
-extern void	systemChangeTerminal(char *color, const u_char c_termcap[], char *mono, const u_char m_termcap[]);
-extern void	systemChangeScreenmap(const u_char newmap[]);
-extern void	systemCreateHoloshell(void);
-extern int	vsystem(char *fmt, ...) __printflike(1, 2);
-
-/* tcpip.c */
-extern int	tcpOpenDialog(Device *dev);
-extern int	tcpMenuSelect(dialogMenuItem *self);
-extern Device	*tcpDeviceSelect(void);
-
-/* termcap.c */
-extern int	set_termcap(void);
-
-/* ttys.c */
-extern void     configTtys(void);
-
-/* ufs.c */
-extern void	mediaShutdownUFS(Device *dev);
-extern Boolean	mediaInitUFS(Device *dev);
-extern FILE	*mediaGetUFS(Device *dev, char *file, Boolean probe);
-
-/* usb.c */
-extern Boolean	mediaInitUSB(Device *dev);
-extern FILE	*mediaGetUSB(Device *dev, char *file, Boolean probe);
-extern void	mediaShutdownUSB(Device *dev);
-
-/* user.c */
-extern int	userAddGroup(dialogMenuItem *self);
-extern int	userAddUser(dialogMenuItem *self);
-
-/* variable.c */
-extern void	variable_set(char *var, int dirty);
-extern void	variable_set2(char *name, char *value, int dirty);
-extern char 	*variable_get(char *var);
-extern int 	variable_cmp(char *var, char *value);
-extern void	variable_unset(char *var);
-extern char	*variable_get_value(char *var, char *prompt, int dirty);
-extern int 	variable_check(char *data);
-extern int 	variable_check2(char *data);
-extern int	dump_variables(dialogMenuItem *self);
-extern void	free_variables(void);
-extern void     pvariable_set(char *var);
-extern char     *pvariable_get(char *var);
-
-/* wizard.c */
-extern void	slice_wizard(Disk *d);
-
-/*
- * Macros.  Please find a better place for us!
- */
-#define DEVICE_INIT(d)		((d) != NULL ? (d)->init((d)) : (Boolean)0)
-#define DEVICE_GET(d, b, f)	((d) != NULL ? (d)->get((d), (b), (f)) : NULL)
-#define DEVICE_SHUTDOWN(d)	((d) != NULL ? (d)->shutdown((d)) : (void)0)
-
-#ifdef USE_GZIP
-#define UNZIPPER "gunzip"
-#else
-#define UNZIPPER "bunzip2"
-#endif
-
-#endif
-/* _SYSINSTALL_H_INCLUDE */
diff --git a/usr.sbin/sysinstall/system.c b/usr.sbin/sysinstall/system.c
deleted file mode 100644
index 91ea2b21782b..000000000000
--- a/usr.sbin/sysinstall/system.c
+++ /dev/null
@@ -1,555 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Jordan Hubbard
- *
- * My contributions are in the public domain.
- *
- * Parts of this file are also blatantly stolen from Poul-Henning Kamp's
- * previous version of sysinstall, and as such fall under his "BEERWARE license"
- * so buy him a beer if you like it!  Buy him a beer for me, too!
- * Heck, get him completely drunk and send me pictures! :-)
- */
-
-#include "sysinstall.h"
-#include <signal.h>
-#include <termios.h>
-#include <sys/param.h>
-#include <sys/reboot.h>
-#include <sys/consio.h>
-#include <sys/fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/mount.h>
-#include <sys/stat.h>
-#include <sys/sysctl.h>
-#include <ufs/ufs/ufsmount.h>
-
-
-/* Where we stick our temporary expanded doc file */
-#define	DOC_TMP_DIR	"/tmp/.doc"
-#define	DOC_TMP_FILE	"/tmp/.doc/doc.tmp"
-
-static pid_t ehs_pid;
-
-/*
- * Handle interrupt signals - this probably won't work in all cases
- * due to our having bogotified the internal state of dialog or curses,
- * but we'll give it a try.
- */
-static int
-intr_continue(dialogMenuItem *self)
-{
-    return DITEM_LEAVE_MENU;
-}
-
-static int
-intr_reboot(dialogMenuItem *self)
-{
-    systemShutdown(-1);
-    /* NOTREACHED */
-    return 0;
-}
-
-static int
-intr_restart(dialogMenuItem *self)
-{
-    int ret, fd, fdmax;
-    char *arg;
-
-    mediaClose();
-    free_variables();
-    fdmax = getdtablesize();
-    for (fd = 3; fd < fdmax; fd++)
-	close(fd);
-    
-    if (RunningAsInit)
-	    arg = "-restart -fakeInit";
-    else
-	    arg = "-restart";
-
-    ret = execl(StartName, StartName, arg, NULL);
-    msgDebug("execl failed (%s)\n", strerror(errno));
-    /* NOTREACHED */
-    return -1;
-}
-
-static dialogMenuItem intrmenu[] = {
-    { "Abort",   "Abort the installation", NULL, intr_reboot },
-    { "Restart", "Restart the installation program", NULL, intr_restart },
-    { "Continue", "Continue the installation", NULL, intr_continue },
-};
-
-
-static void
-handle_intr(int sig)
-{
-    WINDOW *save = savescr();
-
-    use_helpline(NULL);
-    use_helpfile(NULL);
-    if (OnVTY) {
-        ioctl(0, VT_ACTIVATE, 1);       /* Switch back */
-        msgInfo(NULL);
-    }
-    (void)dialog_menu("Installation interrupt",
-		     "Do you want to abort the installation?",
-		     -1, -1, 3, -3, intrmenu, NULL, NULL, NULL);
-    restorescr(save);
-}
-
-#if 0
-/*
- * Harvest children if we are init.
- */
-static void
-reap_children(int sig)
-{
-    int errbak = errno;
-
-    while (waitpid(-1, NULL, WNOHANG) > 0)
-	;
-    errno = errbak;
-}
-#endif
-
-/* Expand a file into a convenient location, nuking it each time */
-static char *
-expand(char *fname)
-{
-    char *unzipper = RunningAsInit ? "/stand/" UNZIPPER : "/usr/bin/" UNZIPPER;
-
-    if (!directory_exists(DOC_TMP_DIR)) {
-	Mkdir(DOC_TMP_DIR);
-	if (chown(DOC_TMP_DIR, 0, 0) < 0)
-	    return NULL;
-	if (chmod(DOC_TMP_DIR, S_IRWXU) < 0)
-	    return NULL;
-    }
-    else
-	unlink(DOC_TMP_FILE);
-    if (!file_readable(fname) || vsystem("%s < %s > %s", unzipper, fname,
-	DOC_TMP_FILE))
-	return NULL;
-    return DOC_TMP_FILE;
-}
-
-/* Initialize system defaults */
-void
-systemInitialize(int argc, char **argv)
-{
-    size_t i;
-    int boothowto;
-    sigset_t signalset;
-
-    signal(SIGINT, SIG_IGN);
-    globalsInit();
-
-    i = sizeof(boothowto);
-    if (!sysctlbyname("debug.boothowto", &boothowto, &i, NULL, 0) &&
-        (i == sizeof(boothowto)) && (boothowto & RB_VERBOSE))
-	variable_set2(VAR_DEBUG, "YES", 0);
-
-    /* Are we running as init? */
-    if (RunningAsInit) {
-	struct ufs_args ufs_args;
-	int fd;
-
-	setsid();
-	close(0);
-	fd = open("/dev/ttyv0", O_RDWR);
-	if (fd == -1) {
-	    fd = open("/dev/console", O_RDWR);	/* fallback */
-	    variable_set2(VAR_FIXIT_TTY, "serial", 0); /* give fixit a hint */
-	} else
-	    OnVTY = TRUE;
-	/*
-	 * To make _sure_ we're on a VTY and don't have /dev/console switched
-	 * away to a serial port or something, attempt to set the cursor appearance.
-	 */
-	if (OnVTY) {
-	    int fd2, type;
-
-	    type = 0;	/* normal */
-	    if ((fd2 = open("/dev/console", O_RDWR)) != -1) {
-		if (ioctl(fd2, CONS_CURSORTYPE, &type) == -1) {
-		    OnVTY = FALSE;
-		    variable_set2(VAR_FIXIT_TTY, "serial", 0); /* Tell Fixit
-								  the console
-								  type */
-		    close(fd); close(fd2);
-		    open("/dev/console", O_RDWR);
-		}
-		else
-		    close(fd2);
-	    }
-	}
-	close(1); dup(0);
-	close(2); dup(0);
-	printf("%s running as init on %s\n", argv[0], OnVTY ? "vty0" : "serial console");
-	ioctl(0, TIOCSCTTY, (char *)NULL);
-	setlogin("root");
-	setenv("PATH", "/stand:/bin:/sbin:/usr/sbin:/usr/bin:/mnt/bin:/mnt/sbin:/mnt/usr/sbin:/mnt/usr/bin", 1);
-	setbuf(stdin, 0);
-	setbuf(stderr, 0);
-#if 0
-	signal(SIGCHLD, reap_children);
-#endif
-	memset(&ufs_args, 0, sizeof(ufs_args));
-	mount("ufs", "/", MNT_UPDATE, &ufs_args);
-    }
-    else {
-	char hname[256];
-
-	/* Initalize various things for a multi-user environment */
-	if (!gethostname(hname, sizeof hname))
-	    variable_set2(VAR_HOSTNAME, hname, 0);
-    }
-
-    if (set_termcap() == -1) {
-	printf("Can't find terminal entry\n");
-	exit(-1);
-    }
-
-    /* XXX - libdialog has particularly bad return value checking */
-    init_dialog();
-
-    /* If we haven't crashed I guess dialog is running ! */
-    DialogActive = TRUE;
-
-    /* Make sure HOME is set for those utilities that need it */
-    if (!getenv("HOME"))
-	setenv("HOME", "/", 1);
-    signal(SIGINT, handle_intr);
-    /*
-     * Make sure we can be interrupted even if we were re-executed
-     * from an interrupt.
-     */
-    sigemptyset(&signalset);
-    sigaddset(&signalset, SIGINT);
-    sigprocmask(SIG_UNBLOCK, &signalset, NULL);
-
-    (void)vsystem("rm -rf %s", DOC_TMP_DIR);
-}
-
-/* Close down and prepare to exit */
-void
-systemShutdown(int status)
-{
-    /* If some media is open, close it down */
-    if (status >=0) {
-	if (mediaDevice != NULL && mediaDevice->type == DEVICE_TYPE_CDROM) {
-	    mediaClose();
-	    msgConfirm("Be sure to remove the media from the drive.");
-	} else
-	    mediaClose();
-    }
-
-    /* write out any changes to rc.conf .. */
-    configRC_conf();
-
-    /* Shut down the dialog library */
-    if (DialogActive) {
-	end_dialog();
-	DialogActive = FALSE;
-    }
-
-    /* Shut down curses */
-    endwin();
-
-    /* If we have a temporary doc dir lying around, nuke it */
-    (void)vsystem("rm -rf %s", DOC_TMP_DIR);
-
-    /* REALLY exit! */
-    if (RunningAsInit) {
-	/* Put the console back */
-	ioctl(0, VT_ACTIVATE, 2);
-#if defined(__sparc64__)
-	reboot(RB_HALT);
-#else
-	reboot(RB_AUTOBOOT);
-#endif
-    }
-    else
-	exit(status);
-}
-
-/* Run some general command */
-int
-systemExecute(char *command)
-{
-    int status;
-    struct termios foo;
-    WINDOW *w = savescr();
-
-    dialog_clear();
-    dialog_update();
-    end_dialog();
-    DialogActive = FALSE;
-    if (tcgetattr(0, &foo) != -1) {
-	foo.c_cc[VERASE] = '\010';
-	tcsetattr(0, TCSANOW, &foo);
-    }
-    if (!Fake)
-	status = system(command);
-    else {
-	status = 0;
-	msgDebug("systemExecute:  Faked execution of `%s'\n", command);
-    }
-    DialogActive = TRUE;
-    restorescr(w);
-    return status;
-}
-
-/* suspend/resume libdialog/curses screen */
-static    WINDOW *oldW;
-
-void
-systemSuspendDialog(void)
-{
-
-    oldW  = savescr();
-    dialog_clear();
-    dialog_update();
-    end_dialog();
-    DialogActive = FALSE;
-}
-
-void
-systemResumeDialog(void)
-{
-
-    DialogActive = TRUE;
-    restorescr(oldW);
-}
-
-/* Display a help file in a filebox */
-int
-systemDisplayHelp(char *file)
-{
-    char *fname = NULL;
-    char buf[FILENAME_MAX];
-    int ret = 0;
-    WINDOW *w = savescr();
-    
-    fname = systemHelpFile(file, buf);
-    if (!fname) {
-	snprintf(buf, FILENAME_MAX, "The %s file is not provided on this particular floppy image.", file);
-	use_helpfile(NULL);
-	use_helpline(NULL);
-	dialog_mesgbox("Sorry!", buf, -1, -1);
-	ret = 1;
-    }
-    else {
-	use_helpfile(NULL);
-	use_helpline(NULL);
-	dialog_textbox(file, fname, LINES, COLS);
-    }
-    restorescr(w);
-    return ret;
-}
-
-char *
-systemHelpFile(char *file, char *buf)
-{
-    if (!file)
-	return NULL;
-    if (file[0] == '/')
-	return file;
-    snprintf(buf, FILENAME_MAX, "/stand/help/%s.hlp.gz", file);
-    if (file_readable(buf)) 
-	return expand(buf);
-    snprintf(buf, FILENAME_MAX, "/stand/help/%s.hlp", file);
-    if (file_readable(buf)) 
-	return expand(buf);
-    snprintf(buf, FILENAME_MAX, "/stand/help/%s.TXT.gz", file);
-    if (file_readable(buf)) 
-	return expand(buf);
-    snprintf(buf, FILENAME_MAX, "/stand/help/%s.TXT", file);
-    if (file_readable(buf)) 
-	return expand(buf);
-    snprintf(buf, FILENAME_MAX, "/usr/src/usr.sbin/%s/help/%s.hlp", ProgName,
-	file);
-    if (file_readable(buf))
-	return buf;
-    snprintf(buf, FILENAME_MAX, "/usr/src/usr.sbin/%s/help/%s.TXT", ProgName,
-	file);
-    if (file_readable(buf))
-	return buf;
-    return NULL;
-}
-
-void
-systemChangeTerminal(char *color, const u_char c_term[],
-		     char *mono, const u_char m_term[])
-{
-    if (OnVTY) {
-	int setupterm(char *color, int, int *);
-
-	if (ColorDisplay) {
-	    setenv("TERM", color, 1);
-	    setenv("TERMCAP", c_term, 1);
-	    reset_shell_mode();
-	    setterm(color);
-	    cbreak(); noecho();
-	}
-	else {
-	    setenv("TERM", mono, 1);
-	    setenv("TERMCAP", m_term, 1);
-	    reset_shell_mode();
-	    setterm(mono);
-	    cbreak(); noecho();
-	}
-    }
-    clear();
-    refresh();
-    dialog_clear();
-}
-
-int
-vsystem(char *fmt, ...)
-{
-    va_list args;
-    int pstat;
-    pid_t pid;
-    int omask;
-    sig_t intsave, quitsave;
-    char *cmd;
-    int i;
-    struct stat sb;
-
-    cmd = (char *)alloca(FILENAME_MAX);
-    cmd[0] = '\0';
-    va_start(args, fmt);
-    vsnprintf(cmd, FILENAME_MAX, fmt, args);
-    va_end(args);
-
-    omask = sigblock(sigmask(SIGCHLD));
-    if (Fake) {
-	msgDebug("vsystem:  Faked execution of `%s'\n", cmd);
-	return 0;
-    }
-    if (isDebug())
-	msgDebug("Executing command `%s'\n", cmd);
-    pid = fork();
-    if (pid == -1) {
-	(void)sigsetmask(omask);
-	i = 127;
-    }
-    else if (!pid) {	/* Junior */
-	(void)sigsetmask(omask);
-	if (DebugFD != -1) {
-	    dup2(DebugFD, 0);
-	    dup2(DebugFD, 1);
-	    dup2(DebugFD, 2);
-	}
-	else {
-	    close(1); open("/dev/null", O_WRONLY);
-	    dup2(1, 2);
-	}
-	if (stat("/stand/sh", &sb) == 0)
-	    execl("/stand/sh", "/stand/sh", "-c", cmd, (char *)NULL);
-	else
-	    execl("/bin/sh", "/bin/sh", "-c", cmd, (char *)NULL);
-	exit(1);
-    }
-    else {
-	intsave = signal(SIGINT, SIG_IGN);
-	quitsave = signal(SIGQUIT, SIG_IGN);
-	pid = waitpid(pid, &pstat, 0);
-	(void)sigsetmask(omask);
-	(void)signal(SIGINT, intsave);
-	(void)signal(SIGQUIT, quitsave);
-	i = (pid == -1) ? -1 : WEXITSTATUS(pstat);
-	if (isDebug())
-	    msgDebug("Command `%s' returns status of %d\n", cmd, i);
-    }
-    return i;
-}
-
-void
-systemCreateHoloshell(void)
-{
-    int waitstatus;
-
-    if ((FixItMode || OnVTY) && RunningAsInit) {
-
-	if (ehs_pid != 0) {
-	    int pstat;
-
-	    if (kill(ehs_pid, 0) == 0) {
-
-		if (msgNoYes("There seems to be an emergency holographic shell\n"
-			     "already running on VTY 4.\n\n"
-			     "Kill it and start a new one?"))
-		    return;
-
-		/* try cleaning up as much as possible */
-		(void) kill(ehs_pid, SIGHUP);
-		sleep(1);
-		(void) kill(ehs_pid, SIGKILL);
-	    }
-
-	    /* avoid too many zombies */
-	    (void) waitpid(ehs_pid, &pstat, WNOHANG);
-	}
-
-	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
-	    systemSuspendDialog();	/* must be before the fork() */
-	if ((ehs_pid = fork()) == 0) {
-	    int i, fd;
-	    struct termios foo;
-	    extern int login_tty(int);
-	    
-	    ioctl(0, TIOCNOTTY, NULL);
-	    for (i = getdtablesize(); i >= 0; --i)
-		close(i);
-	    if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
-	        fd = open("/dev/console", O_RDWR);
-	    else
-	        fd = open("/dev/ttyv3", O_RDWR);
-	    ioctl(0, TIOCSCTTY, &fd);
-	    dup2(0, 1);
-	    dup2(0, 2);
-	    DebugFD = 2;
-	    if (login_tty(fd) == -1)
-		msgDebug("Doctor: I can't set the controlling terminal.\n");
-	    signal(SIGTTOU, SIG_IGN);
-	    if (tcgetattr(fd, &foo) != -1) {
-		foo.c_cc[VERASE] = '\010';
-		if (tcsetattr(fd, TCSANOW, &foo) == -1)
-		    msgDebug("Doctor: I'm unable to set the erase character.\n");
-	    }
-	    else
-		msgDebug("Doctor: I'm unable to get the terminal attributes!\n");
-	    if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) {
-	        printf("Type ``exit'' in this fixit shell to resume sysinstall.\n\n");
-		fflush(stdout);
-	    }
-	    execlp("sh", "-sh", NULL);
-	    msgDebug("Was unable to execute sh for Holographic shell!\n");
-	    exit(1);
-	}
-	else {
-	    if (strcmp(variable_get(VAR_FIXIT_TTY), "standard") == 0) {
-	        WINDOW *w = savescr();
-
-	        msgNotify("Starting an emergency holographic shell on VTY4");
-	        sleep(2);
-	        restorescr(w);
-	    }
-	    else {
-	        (void)waitpid(ehs_pid, &waitstatus, 0); /* we only wait for
-							   shell to finish 
-							   in serial mode
-							   since there is no
-							   virtual console */
-	        systemResumeDialog();
-	    }
-	}
-    }
-}
diff --git a/usr.sbin/sysinstall/tcpip.c b/usr.sbin/sysinstall/tcpip.c
deleted file mode 100644
index 121a5c50150e..000000000000
--- a/usr.sbin/sysinstall/tcpip.c
+++ /dev/null
@@ -1,775 +0,0 @@
-/*
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *      Gary J Palmer. All rights reserved.
- * Copyright (c) 1996
- *      Jordan K. Hubbard. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
- * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * All kinds of hacking also performed by jkh on this code.  Don't
- * blame Gary for every bogosity you see here.. :-)
- *
- * -jkh
- */
-
-#include "sysinstall.h"
-#include <sys/param.h>
-#include <sys/sysctl.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/sockio.h>
-
-#include <netinet/in.h>
-#include <net/if.h>
-#include <net/if_media.h>
-
-#include <netdb.h>
-#include <paths.h>
-#include <ifaddrs.h>
-
-/* The help file for the TCP/IP setup screen */
-#define TCP_HELPFILE		"tcp"
-
-/* These are nasty, but they make the layout structure a lot easier ... */
-
-static char	hostname[HOSTNAME_FIELD_LEN], domainname[HOSTNAME_FIELD_LEN],
-		gateway[IPADDR_FIELD_LEN], nameserver[INET6_ADDRSTRLEN];
-static int	okbutton, cancelbutton;
-static char	ipaddr[IPADDR_FIELD_LEN], netmask[IPADDR_FIELD_LEN], extras[EXTRAS_FIELD_LEN];
-static char	ipv6addr[INET6_ADDRSTRLEN];
-
-/* What the screen size is meant to be */
-#define TCP_DIALOG_Y		0
-#define TCP_DIALOG_X		8
-#define TCP_DIALOG_WIDTH	COLS - 16
-#define TCP_DIALOG_HEIGHT	LINES - 2
-
-static Layout layout[] = {
-#define LAYOUT_HOSTNAME		0
-    { 1, 2, 25, HOSTNAME_FIELD_LEN - 1,
-      "Host:", "Your fully-qualified hostname, e.g. foo.example.com",
-      hostname, STRINGOBJ, NULL },
-#define LAYOUT_DOMAINNAME	1
-    { 1, 35, 20, HOSTNAME_FIELD_LEN - 1,
-      "Domain:",
-      "The name of the domain that your machine is in, e.g. example.com",
-      domainname, STRINGOBJ, NULL },
-#define LAYOUT_GATEWAY		2
-    { 5, 2, 18, IPADDR_FIELD_LEN - 1,
-      "IPv4 Gateway:",
-      "IPv4 address of host forwarding packets to non-local destinations",
-      gateway, STRINGOBJ, NULL },
-#define LAYOUT_NAMESERVER	3
-    { 5, 35, 18, INET6_ADDRSTRLEN - 1,
-      "Name server:", "IPv4 or IPv6 address of your local DNS server",
-      nameserver, STRINGOBJ, NULL },
-#define LAYOUT_IPADDR		4
-    { 10, 10, 18, IPADDR_FIELD_LEN - 1,
-      "IPv4 Address:",
-      "The IPv4 address to be used for this interface",
-      ipaddr, STRINGOBJ, NULL },
-#define LAYOUT_NETMASK		5
-    { 10, 35, 18, IPADDR_FIELD_LEN - 1,
-      "Netmask:",
-      "The netmask for this interface, e.g. 0xffffff00 for a class C network",
-      netmask, STRINGOBJ, NULL },
-#define LAYOUT_EXTRAS		6
-    { 14, 10, 37, HOSTNAME_FIELD_LEN - 1,
-      "Extra options to ifconfig (usually empty):",
-      "Any interface-specific options to ifconfig you would like to add",
-      extras, STRINGOBJ, NULL },
-#define LAYOUT_OKBUTTON		7
-    { 19, 15, 0, 0,
-      "OK", "Select this if you are happy with these settings",
-      &okbutton, BUTTONOBJ, NULL },
-#define LAYOUT_CANCELBUTTON	8
-    { 19, 35, 0, 0,
-      "CANCEL", "Select this if you wish to cancel this screen",
-      &cancelbutton, BUTTONOBJ, NULL },
-    LAYOUT_END,
-};
-
-#define _validByte(b) ((b) >= 0 && (b) <= 255)
-
-/* whine */
-static void
-feepout(char *msg)
-{
-    beep();
-    msgConfirm("%s", msg);
-}
-
-/* Verify IP address integrity */
-static int
-verifyIP(char *ip, unsigned long *mask, unsigned long *out)
-{
-    long a, b, c, d;
-    char *endptr, *endptr_prev;
-
-    unsigned long parsedip;
-    unsigned long max_addr = (255 << 24) | (255 << 16) | (255 << 8) | 255;
-
-    if (ip == NULL)
-	return 0;
-    a = strtol(ip, &endptr, 10);
-    if (endptr - ip == 0 || *endptr++ != '.')
-	return 0;
-    endptr_prev = endptr;
-    b = strtol(endptr, &endptr, 10);
-    if (endptr - endptr_prev == 0 || *endptr++ != '.')
-	return 0;
-    endptr_prev = endptr;
-    c = strtol(endptr, &endptr, 10);
-    if (endptr - endptr_prev == 0 || *endptr++ != '.')
-	return 0;
-    endptr_prev = endptr;
-    d = strtol(endptr, &endptr, 10);
-    if (*endptr != '\0' || endptr - endptr_prev == 0)
-	return 0;
-    if (!_validByte(a) || !_validByte(b) || !_validByte(c) || !_validByte(d))
-	return 0;
-    parsedip = (a << 24) | (b << 16) | (c << 8) | d;
-    if (out) 
-	*out = parsedip;
-    /*
-     * The ip address must not be network or broadcast address.
-     */
-    if (mask && ((parsedip == (parsedip & *mask)) || 
-	(parsedip == ((parsedip & *mask) + max_addr - *mask))))
-	return 0;
-    return 1;
-}
-
-static int
-verifyIP6(char *ip)
-{
-    struct addrinfo hints, *res;
-
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = AF_INET6;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST;
-    if (getaddrinfo(ip, NULL, &hints, &res) == 0) {
-	freeaddrinfo(res);
-	return 1;
-    }
-    return 0;
-}
-
-/* Verify IPv4 netmask as being well-formed as
-   a 0x or AAA.BBB.CCC.DDD mask */
-static int
-verifyNetmask(const char *netmask, unsigned long *out)
-{
-    unsigned long mask;
-    long tmp;
-    char *endptr;
-
-    if (netmask[0] == '0' && (netmask[1] == 'x' || netmask[1] == 'X')) {
-        /* Parse out hex mask */
-        mask = strtoul(netmask, &endptr, 0);
-        if (*endptr != '\0')
-            return 0;
-    } else {
-        /* Parse out quad decimal mask */
-        tmp = strtoul(netmask, &endptr, 10);
-        if (!_validByte(tmp) || *endptr++ != '.')
-            return 0;
-	mask = tmp;
-        tmp = strtoul(endptr, &endptr, 10);
-        if (!_validByte(tmp) || *endptr++ != '.')
-            return 0;
-	mask = (mask << 8) + tmp;
-        tmp = strtoul(endptr, &endptr, 10);
-        if (!_validByte(tmp) || *endptr++ != '.')
-            return 0;
-	mask = (mask << 8) + tmp;
-        tmp = strtoul(endptr, &endptr, 10);
-        if (!_validByte(tmp) || *endptr++ != '\0')
-            return 0;
-	mask = (mask << 8) + tmp;
-    }
-    /* Verify that we have a continous netmask */
-    if ((((-mask & mask) - 1) | mask) != 0xffffffff)
-        return 0;
-    if (out)
-        *out = mask;
-    return 1;
-}
-
-static int
-verifyGW(char *gw, unsigned long *ip, unsigned long *mask)
-{
-    unsigned long parsedgw;
-
-    if (!verifyIP(gw, mask, &parsedgw))
-	return 0;
-    /* Gateway needs to be within the set of IPs reachable through the
-       interface */
-    if (ip && mask && ((parsedgw & *mask) != (*ip & *mask)))
-	return 0;
-    return 1;
-}
-
-/* Check for the settings on the screen - the per-interface stuff is
-   moved to the main handling code now to do it on the fly - sigh */
-static int
-verifySettings(void)
-{
-    unsigned long parsedip;
-    unsigned long parsednetmask;
-
-    if (!hostname[0])
-	feepout("Must specify a host name of some sort!");
-    else if (netmask[0] && !verifyNetmask(netmask, &parsednetmask))
-	feepout("Invalid netmask value");
-    else if (nameserver[0] && !verifyIP(nameserver, NULL, NULL) &&
-		    !verifyIP6(nameserver))
-	feepout("Invalid name server IP address specified");
-    else if (ipaddr[0] && !verifyIP(ipaddr, &parsednetmask, &parsedip))
-	feepout("Invalid IPv4 address");
-    else if (gateway[0] && strcmp(gateway, "NO") &&
-	     !verifyGW(gateway, ipaddr[0] ? &parsedip : NULL,
-		     netmask[0] ? &parsednetmask : NULL))
-	feepout("Invalid gateway IPv4 address specified");
-    else
-	return 1;
-    return 0;
-}
-
-static void
-dhcpGetInfo(Device *devp)
-{
-    char leasefile[PATH_MAX];
-
-    snprintf(leasefile, sizeof(leasefile), "%sdhclient.leases.%s",
-	_PATH_VARDB, devp->name);
-    /* If it fails, do it the old-fashioned way */
-    if (dhcpParseLeases(leasefile, hostname, domainname,
-			 nameserver, ipaddr, gateway, netmask) == -1) {
-	FILE *ifp;
-	char *cp, cmd[256], data[2048];
-	int i, j;
-
-	/* Bah, now we have to kludge getting the information from ifconfig */
-	snprintf(cmd, sizeof cmd, "ifconfig %s", devp->name);
-	ifp = popen(cmd, "r");
-	if (ifp) {
-	    j = fread(data, 1, sizeof(data), ifp);
-	    fclose(ifp);
-	    if (j < 0)	/* paranoia */
-		j = 0;
-	    data[j] = '\0';
-	    if (isDebug())
-		msgDebug("DHCP configured interface returns %s\n", data);
-	    /* XXX This is gross as it assumes a certain ordering to
-	       ifconfig's output! XXX */
-	    if ((cp = strstr(data, "inet ")) != NULL) {
-		i = 0;
-		cp += 5;	/* move over keyword */
-		while (*cp != ' ')
-		    ipaddr[i++] = *(cp++);
-		ipaddr[i] = '\0';
-		if (!strncmp(++cp, "netmask", 7)) {
-		    i = 0;
-		    cp += 8;
-		    while (*cp != ' ')
-			netmask[i++] = *(cp++);
-		    netmask[i] = '\0';
-		}
-	    }
-	}
-    }
-
-    /* If we didn't get a name server value, hunt for it in resolv.conf */
-    if (!nameserver[0] && file_readable("/etc/resolv.conf"))
-	configEnvironmentResolv("/etc/resolv.conf");
-    if (hostname[0])
-	variable_set2(VAR_HOSTNAME, hostname, 0);
-}
-
-static void
-rtsolGetInfo(Device *devp)
-{
-    FILE *ifp;
-    char *cp, cmd[256], data[2048];
-    int i;
-
-    snprintf(cmd, sizeof cmd, "ifconfig %s", devp->name);
-    if ((ifp = popen(cmd, "r")) == NULL)
-	return;
-    while (fgets(data, sizeof(data), ifp) != NULL) {
-	if (isDebug())
-	    msgDebug("RTSOL configured interface returns %s\n", data);
-	if ((cp = strstr(data, "inet6 ")) != NULL) {
-	    cp += 6;	/* move over keyword */
-	    if (strncmp(cp, "fe80:", 5)) {
-		i = 0;
-		while (*cp != ' ')
-		    ipv6addr[i++] = *(cp++);
-		ipv6addr[i] = '\0';
-	    }
-	}
-    }
-    fclose(ifp);
-}
-
-/* This is it - how to get TCP setup values */
-int
-tcpOpenDialog(Device *devp)
-{
-    WINDOW              *ds_win, *save = NULL;
-    ComposeObj          *obj = NULL;
-    int                 n = 0, filled = 0, cancel = FALSE;
-    int			max, ret = DITEM_SUCCESS;
-    int			use_dhcp = FALSE;
-    int			use_rtsol = FALSE;
-    char                *tmp;
-    char		title[80];
-
-    save = savescr();
-    /* Initialise vars from previous device values */
-    if (devp->private) {
-	DevInfo *di = (DevInfo *)devp->private;
-
-	SAFE_STRCPY(ipaddr, di->ipaddr);
-	SAFE_STRCPY(netmask, di->netmask);
-	SAFE_STRCPY(extras, di->extras);
-	use_dhcp = di->use_dhcp;
-	use_rtsol = di->use_rtsol;
-    }
-    else { /* See if there are any defaults */
-	char *cp;
-	char *old_interactive = NULL;
-
-	/*
-	 * This is a hack so that the dialogs below are interactive in a
-	 * script if we have requested interactive behavior.
-	 */
-	if (variable_get(VAR_NONINTERACTIVE) &&
-	  variable_get(VAR_NETINTERACTIVE)) {
-	    old_interactive = strdup(VAR_NONINTERACTIVE);
-	    variable_unset(VAR_NONINTERACTIVE);
-	}
-
-
-	/*
-	 * Try a RTSOL scan if such behavior is desired.
-	 * If the variable was configured and is YES, do it.
-	 * If it was configured to anything else, treat it as NO.
-	 * Otherwise, ask the question interactively.
-	 */
-	if (!variable_get(VAR_NO_INET6) &&
-	    (!variable_cmp(VAR_TRY_RTSOL, "YES") || 
-	    (variable_get(VAR_TRY_RTSOL)==0 && !msgNoYes("Do you want to try IPv6 configuration of the interface?")))) {
-	    int i;
-	    size_t len;
-
-	    i = 0;
-	    sysctlbyname("net.inet6.ip6.forwarding", NULL, 0, &i, sizeof(i));
-	    i = 1;
-	    sysctlbyname("net.inet6.ip6.accept_rtadv", NULL, 0, &i, sizeof(i));
-	    vsystem("ifconfig %s up", devp->name);
-	    len = sizeof(i);
-	    sysctlbyname("net.inet6.ip6.dad_count", &i, &len, NULL, 0);
-	    sleep(i + 1);
-	    Mkdir("/var/run");
-	    msgNotify("Scanning for RA servers...");
-	    if (0 == vsystem("rtsol %s", devp->name)) {
-		len = sizeof(i);
-		sysctlbyname("net.inet6.ip6.dad_count", &i, &len, NULL, 0);
-		sleep(i + 1);
-		rtsolGetInfo(devp);
-		use_rtsol = TRUE;
-	    } else
-		use_rtsol = FALSE;
-	}
-
-
-	/*
-	 * First try a DHCP scan if such behavior is desired.
-	 * If the variable was configured and is YES, do it.
-	 * If it was configured to anything else, treat it as NO.
-	 * Otherwise, ask the question interactively.
-	 */
-	if (!variable_cmp(VAR_TRY_DHCP, "YES") || 
-	    (variable_get(VAR_TRY_DHCP)==0 && !msgNoYes("Do you want to try DHCP configuration of the interface?"))) {
-	    Mkdir("/var/db");
-	    Mkdir("/var/run");
-	    Mkdir("/tmp");
-	    msgNotify("Scanning for DHCP servers...");
-	    /* XXX clear any existing lease */
-	    /* XXX limit protocol to N tries */
-	    if (0 == vsystem("dhclient %s", devp->name)) {
-		dhcpGetInfo(devp);
-		use_dhcp = TRUE;
-	    }
-	    else
-		use_dhcp = FALSE;
-	}
-
-	/* Restore old VAR_NONINTERACTIVE if needed. */
-	if (old_interactive != NULL) {
-	    variable_set2(VAR_NONINTERACTIVE, old_interactive, 0);
-	    free(old_interactive);
-	}
-
-	/* Special hack so it doesn't show up oddly in the tcpip setup menu */
-	if (!strcmp(gateway, "NO"))
-	    gateway[0] = '\0';
-
-	/* Get old IP address from variable space, if available */
-	if (!ipaddr[0]) {
-	    if ((cp = variable_get(VAR_IPADDR)) != NULL)
-		SAFE_STRCPY(ipaddr, cp);
-	    else if ((cp = variable_get(string_concat3(devp->name, "_", VAR_IPADDR))) != NULL)
-		SAFE_STRCPY(ipaddr, cp);
-	}
-
-	/* Get old netmask from variable space, if available */
-	if (!netmask[0]) {
-	    if ((cp = variable_get(VAR_NETMASK)) != NULL)
-		SAFE_STRCPY(netmask, cp);
-	    else if ((cp = variable_get(string_concat3(devp->name, "_", VAR_NETMASK))) != NULL)
-		SAFE_STRCPY(netmask, cp);
-	}
-
-	/* Get old extras string from variable space, if available */
-	if (!extras[0]) {
-	    if ((cp = variable_get(VAR_EXTRAS)) != NULL)
-		SAFE_STRCPY(extras, cp);
-	    else if ((cp = variable_get(string_concat3(devp->name, "_", VAR_EXTRAS))) != NULL)
-		SAFE_STRCPY(extras, cp);
-	}
-    }
-
-    /* Look up values already recorded with the system, or blank the string variables ready to accept some new data */
-    if (!hostname[0]) {
-	tmp = variable_get(VAR_HOSTNAME);
-	if (tmp)
-	    SAFE_STRCPY(hostname, tmp);
-    }
-    if (!domainname[0]) {
-	tmp = variable_get(VAR_DOMAINNAME);
-	if (tmp)
-	    SAFE_STRCPY(domainname, tmp);
-    }
-    if (!gateway[0]) {
-	tmp = variable_get(VAR_GATEWAY);
-	if (tmp && strcmp(tmp, "NO"))
-	    SAFE_STRCPY(gateway, tmp);
-    }
-    if (!nameserver[0]) {
-	tmp = variable_get(VAR_NAMESERVER);
-	if (tmp)
-	    SAFE_STRCPY(nameserver, tmp);
-    }
-
-    /* If non-interactive, jump straight over the dialog crap and into config section */
-    if (variable_get(VAR_NONINTERACTIVE) &&
-	!variable_get(VAR_NETINTERACTIVE)) {
-	if (!hostname[0])
-	    msgConfirm("WARNING: hostname variable not set and is a non-optional\n"
-		       "parameter.  Please add this to your installation script\n"
-		       "or set the netInteractive variable (see sysinstall man page)");
-	else
-	    goto netconfig;
-    }
-
-    /* Now do all the screen I/O */
-    dialog_clear_norefresh();
-
-    /* Modify the help line for PLIP config */
-    if (!strncmp(devp->name, "plip", 4))
-	layout[LAYOUT_EXTRAS].help = 
-         "For PLIP configuration, you must enter the peer's IP address here.";
-
-    /* We need a curses window */
-    tmp = " Network Configuration ";
-    if (ipv6addr[0])
-	tmp = string_concat(tmp, "(IPv6 ready) ");
-    if (!(ds_win = openLayoutDialog(TCP_HELPFILE, tmp,
-				    TCP_DIALOG_X, TCP_DIALOG_Y, TCP_DIALOG_WIDTH, TCP_DIALOG_HEIGHT))) {
-	beep();
-	msgConfirm("Cannot open TCP/IP dialog window!!");
-	restorescr(save);
-	return DITEM_FAILURE;
-    }
-
-    /* Draw interface configuration box */
-    draw_box(ds_win, TCP_DIALOG_Y + 9, TCP_DIALOG_X + 8, TCP_DIALOG_HEIGHT - 13, TCP_DIALOG_WIDTH - 17,
-	     dialog_attr, border_attr);
-    wattrset(ds_win, dialog_attr);
-    sprintf(title, " Configuration for Interface %s ", devp->name);
-    mvwaddstr(ds_win, TCP_DIALOG_Y + 9, TCP_DIALOG_X + 14, title);
-
-    /* Some more initialisation before we go into the main input loop */
-    obj = initLayoutDialog(ds_win, layout, TCP_DIALOG_X, TCP_DIALOG_Y, &max);
-
-reenter:
-    cancelbutton = okbutton = 0;
-    while (layoutDialogLoop(ds_win, layout, &obj, &n, max, &cancelbutton, &cancel)) {
-	/* Prevent this from being irritating if user really means NO */
-	if (filled < 3) {
-	    /* Insert a default value for the netmask, 0xffffff00 is
-	     * the most appropriate one (entire class C, or subnetted
-	     * class A/B network).
-	     */
-	    if (!netmask[0]) {
-		strcpy(netmask, "255.255.255.0");
-		RefreshStringObj(layout[LAYOUT_NETMASK].obj);
-		++filled;
-	    }
-	    if (!index(hostname, '.') && domainname[0]) {
-		strcat(hostname, ".");
-		strcat(hostname, domainname);
-		RefreshStringObj(layout[LAYOUT_HOSTNAME].obj);
-		++filled;
-	    }
-	    else if (((tmp = index(hostname, '.')) != NULL) && !domainname[0]) {
-		SAFE_STRCPY(domainname, tmp + 1);
-		RefreshStringObj(layout[LAYOUT_DOMAINNAME].obj);
-		++filled;
-	    }
-	}
-    }
-    if (!cancel && !verifySettings())
-	goto reenter;
-
-    /* Clear this crap off the screen */
-    delwin(ds_win);
-    dialog_clear_norefresh();
-    use_helpfile(NULL);
-
-    /* We actually need to inform the rest of sysinstall about this
-       data now if the user hasn't selected cancel.  Save the stuff
-       out to the environment via the variable_set() mechanism */
-
-netconfig:
-    if (!cancel) {
-	DevInfo *di;
-	char temp[512], ifn[255];
-	int ipv4_enable = FALSE;
-
-	if (hostname[0]) {
-	    variable_set2(VAR_HOSTNAME, hostname, 1);
-	    sethostname(hostname, strlen(hostname));
-	}
-	if (domainname[0])
-	    variable_set2(VAR_DOMAINNAME, domainname, 0);
-	if (gateway[0])
-	    variable_set2(VAR_GATEWAY, gateway, use_dhcp ? 0 : 1);
-	if (nameserver[0])
-	    variable_set2(VAR_NAMESERVER, nameserver, 0);
-	if (ipaddr[0])
-	    variable_set2(VAR_IPADDR, ipaddr, 0);
-	if (ipv6addr[0])
-	    variable_set2(VAR_IPV6ADDR, ipv6addr, 0);
-
-	if (!devp->private)
-	    devp->private = (DevInfo *)safe_malloc(sizeof(DevInfo));
-	di = devp->private;
-	SAFE_STRCPY(di->ipaddr, ipaddr);
-	SAFE_STRCPY(di->netmask, netmask);
-	SAFE_STRCPY(di->extras, extras);
-	di->use_dhcp = use_dhcp;
-	di->use_rtsol = use_rtsol;
-
-	if (use_dhcp || ipaddr[0])
-	    ipv4_enable = TRUE;
-	if (ipv4_enable) {
-	    sprintf(ifn, "%s%s", VAR_IFCONFIG, devp->name);
-	    if (use_dhcp) {
-		if (strlen(extras) > 0)
-		    sprintf(temp, "DHCP %s", extras);
-		else
-		    sprintf(temp, "DHCP");
-	    } else
-		sprintf(temp, "inet %s %s netmask %s",
-			ipaddr, extras, netmask);
-	    variable_set2(ifn, temp, 1);
-	}
-	if (use_rtsol)
-	    variable_set2(VAR_IPV6_ENABLE, "YES", 1);
-	if (!use_dhcp)
-	    configResolv(NULL);	/* XXX this will do it on the MFS copy XXX */
-	ret = DITEM_SUCCESS;
-    }
-    else
-	ret = DITEM_FAILURE;
-    restorescr(save);
-    return ret;
-}
-
-static Device *NetDev;
-
-static int
-netHook(dialogMenuItem *self)
-{
-    Device **devs;
-
-    devs = deviceFindDescr(self->prompt, self->title, DEVICE_TYPE_NETWORK);
-    if (devs) {
-	if (DITEM_STATUS(tcpOpenDialog(devs[0])) != DITEM_FAILURE)
-	    NetDev = devs[0];
-	else
-	    NetDev = NULL;
-    }
-    return devs ? DITEM_LEAVE_MENU : DITEM_FAILURE;
-}
-
-static char *
-tcpDeviceScan(void)
-{
-	int s;
-	struct ifmediareq ifmr;
-	struct ifaddrs *ifap, *ifa;
-	char *network_dev;
-
-	if ((s = socket(AF_LOCAL, SOCK_DGRAM, 0)) < 0)
-		return (NULL);
-
-	if (getifaddrs(&ifap) < 0)
-		return (NULL);
-
-	for (ifa = ifap; ifa != NULL; ifa = ifa->ifa_next) {
-		memset(&ifmr, 0, sizeof(ifmr));
-		strlcpy(ifmr.ifm_name, ifa->ifa_name, sizeof(ifmr.ifm_name));
-
-		if (ioctl(s, SIOCGIFMEDIA, (caddr_t)&ifmr) < 0)
-			continue;	/* some devices don't support this */ 
-
-		if ((ifmr.ifm_status & IFM_AVALID) == 0)
-			continue;	/* not active */
-
-		if (IFM_TYPE(ifmr.ifm_active) != IFM_ETHER)
-			continue;	/* not an ethernet device */
-
-		if (ifmr.ifm_status & IFM_ACTIVE) {
-			network_dev = strdup(ifa->ifa_name);
-			freeifaddrs(ifap);
-
-			if (!variable_get(VAR_NONINTERACTIVE)) 
-				msgConfirm("Using interface %s", network_dev);
-
-			msgDebug("tcpDeviceScan found %s", network_dev);
-			return (network_dev);
-		}
-	}
-
-	close(s);
-
-	freeifaddrs(ifap);
-
-	return (NULL);
-}
-
-/* Get a network device */
-Device *
-tcpDeviceSelect(void)
-{
-    DMenu *menu;
-    Device **devs, *rval;
-    char *dev, *network_dev;
-    int cnt;
-
-    rval = NULL;
-
-    if (variable_get(VAR_NETWORK_DEVICE)) {
-	network_dev = variable_get(VAR_NETWORK_DEVICE);
-
-	/* 
-	 * netDev can be set to several types of values.
-	 * If netDev is set to ANY, scan all network devices
-	 * looking for a valid link, and go with the first
-	 * device found. netDev can also be specified as a
-	 * comma delimited list, with each network device
-	 * tried in order. netDev can also be set to a single
-	 * network device.
-	 */
-	if (!strcmp(network_dev, "ANY")) 
-		network_dev = strdup(tcpDeviceScan()); 
-
-	while ((dev = strsep(&network_dev, ",")) != NULL) {
-	    devs = deviceFind(dev, DEVICE_TYPE_NETWORK);
-	    cnt = deviceCount(devs);
-
-	    if (cnt) {
-		if (DITEM_STATUS(tcpOpenDialog(devs[0])) == DITEM_SUCCESS) 
-		    return (devs[0]);
-	    }
-	}
-
-	if (!variable_get(VAR_NONINTERACTIVE))
-		msgConfirm("No network devices available!");
-
-	return (NULL);
-    }
-
-    devs = deviceFind(NULL, DEVICE_TYPE_NETWORK);
-    cnt = deviceCount(devs); 
-
-    if ((!RunningAsInit) && (variable_check("NETWORK_CONFIGURED=NO") != TRUE)) {
-	if (!msgYesNo("Running multi-user, assume that the network is already configured?"))
-	    return devs[0];
-    }
-    if (cnt == 1) {
-	if (DITEM_STATUS(tcpOpenDialog(devs[0]) == DITEM_SUCCESS))
-	    rval = devs[0];
-    }
-    else {
-	int status;
-
-	menu = deviceCreateMenu(&MenuNetworkDevice, DEVICE_TYPE_NETWORK, netHook, NULL);
-	if (!menu)
-	    msgFatal("Unable to create network device menu!  Argh!");
-	status = dmenuOpenSimple(menu, FALSE);
-	free(menu);
-	if (status)
-	    rval = NetDev;
-    }
-    return rval;
-}
-
-/* Do it from a menu that doesn't care about status */
-int
-tcpMenuSelect(dialogMenuItem *self)
-{
-    Device *tmp;
-    WINDOW *save;
-
-    variable_set("NETWORK_CONFIGURED=NO",0);
-    tmp = tcpDeviceSelect();
-    variable_unset("NETWORK_CONFIGURED");
-    save = savescr();
-    if (tmp && tmp->private && !((DevInfo *)tmp->private)->use_dhcp && !msgYesNo("Would you like to bring the %s interface up right now?", tmp->name))
-	if (!DEVICE_INIT(tmp))
-	    msgConfirm("Initialization of %s device failed.", tmp->name);
-    restorescr(save);
-    return DITEM_SUCCESS;
-}
diff --git a/usr.sbin/sysinstall/termcap.c b/usr.sbin/sysinstall/termcap.c
deleted file mode 100644
index 4dc36c2ce488..000000000000
--- a/usr.sbin/sysinstall/termcap.c
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Copyright (c) 1994, Paul Richards.
- *
- * All rights reserved.
- *
- * This software may be used, modified, copied, distributed, and sold, in both
- * source and binary form provided that the above copyright and these terms
- * are retained, verbatim, as the first lines of this file.  Under no
- * circumstances is the author responsible for the proper functioning of this
- * software, nor does the author assume any responsibility for damages
- * incurred with its use.
- *
- * $FreeBSD$
- */
-
-#include "sysinstall.h"
-#include <stdarg.h>
-#include <fcntl.h>
-#include <sys/errno.h>
-#include <sys/ioctl.h>
-#include <sys/consio.h>
-
-#define VTY_STATUS_LINE    24
-#define TTY_STATUS_LINE    23
-
-static void
-prompt_term(char **termp, char **termcapp)
-{
-    char str[80];
-    static struct {
-	const char *term, *termcap;
-    } lookup[] = { { "ansi", termcap_ansi },
-		   { "vt100", termcap_vt100 },
-		   { "cons25", termcap_cons25 },
-		   { "cons25-m", termcap_cons25_m },
-		   { "xterm", termcap_xterm },
-		   { "cons25w", termcap_cons25w } }; /* must be last */
-
-    if (RunningAsInit) {
-	while (1) {
-	    int i;
-
-	    printf("\nThese are the predefined terminal types available to\n");
-	    printf("sysinstall when running stand-alone.  Please choose the\n");
-	    printf("closest match for your particular terminal.\n\n");
-	    printf("1 ...................... Standard ANSI terminal.\n");
-	    printf("2 ...................... VT100 or compatible terminal.\n");
-	    printf("3 ...................... FreeBSD system console (color).\n");
-	    printf("4 ...................... FreeBSD system console (monochrome).\n\n");
-	    printf("5 ...................... xterm terminal emulator.\n\n");
-	    printf("Your choice: (1-5) ");
-	    fflush(stdout);
-	    fgets(str, sizeof(str), stdin);
-	    i = str[0] - '0';
-	    if (i > 0 && i < 6) {
-		*termp = (char *)lookup[i - 1].term;
-		*termcapp = (char *)lookup[i - 1].termcap;
-		break;
-	    }
-	    else
-		printf("\007Invalid choice, please try again.\n\n");
-	}
-    }
-    else {
-	printf("\nPlease set your TERM variable before running this program.\n");
-	printf("Defaulting to an ANSI compatible terminal - please press RETURN\n");
-	fgets(str, sizeof(str), stdin);	/* Just to make it interactive */
-	*termp = (char *)"ansi";
-	*termcapp = (char *)termcap_ansi;
-    }
-}
-
-int
-set_termcap(void)
-{
-    char           *term;
-    int		   stat;
-    struct winsize ws;
-
-    term = getenv("TERM");
-    stat = ioctl(STDERR_FILENO, GIO_COLOR, &ColorDisplay);
-
-    if (!RunningAsInit) {
-	if (isDebug())
-	    DebugFD = open("sysinstall.debug", O_WRONLY|O_CREAT|O_TRUNC, 0644);
-	else
-	    DebugFD = -1;
-	if (DebugFD < 0)
-	    DebugFD = open("/dev/null", O_RDWR, 0);
-    }
-
-    if (!OnVTY || (stat < 0)) {
-	if (!term) {
-	    char *term, *termcap;
-
-	    prompt_term(&term, &termcap);
-	    if (setenv("TERM", term, 1) < 0)
-		return -1;
-	    if (setenv("TERMCAP", termcap, 1) < 0)
-		return -1;
-	}
-	if (DebugFD < 0)
-	    DebugFD = open("/dev/null", O_RDWR, 0);
-    }
-    else {
-	int i, on;
-
-	if (RunningAsInit) {
-	    DebugFD = open("/dev/ttyv1", O_WRONLY);
-	    if (DebugFD != -1) {
-		on = 1;
-		i = ioctl(DebugFD, TIOCCONS, (char *)&on);
-		msgDebug("ioctl(%d, TIOCCONS, NULL) = %d (%s)\n",
-			 DebugFD, i, !i ? "success" : strerror(errno));
-	    }
-	}
-
-#ifdef PC98
-	if (!term) {
-	    if (setenv("TERM", "cons25w", 1) < 0)
-		return -1;
-	    if (setenv("TERMCAP", termcap_cons25w, 1) < 0)
-		return -1;
-	}
-#else
-	if (ColorDisplay) {
-	    if (!term) {
-		if (setenv("TERM", "xterm", 1) < 0)
-		    return -1;
-		if (setenv("TERMCAP", termcap_xterm, 1) < 0)
-		    return -1;
-	    }
-	}
-	else {
-	    if (!term) {
-		if (setenv("TERM", "vt100", 1) < 0)
-		    return -1;
-		if (setenv("TERMCAP", termcap_vt100, 1) < 0)
-		    return -1;
-	    }
-	}
-#endif
-    }
-    if (ioctl(0, TIOCGWINSZ, &ws) == -1) {
-	msgDebug("Unable to get terminal size - errno %d\n", errno);
-	ws.ws_row = 0;
-    }
-    StatusLine = ws.ws_row ? ws.ws_row - 1: (OnVTY ? VTY_STATUS_LINE : TTY_STATUS_LINE);
-    return 0;
-}
diff --git a/usr.sbin/sysinstall/ttys.c b/usr.sbin/sysinstall/ttys.c
deleted file mode 100644
index 0de954cceb39..000000000000
--- a/usr.sbin/sysinstall/ttys.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 2001
- *      Andrey A. Chernov.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY ANDREY A. CHERNOV ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL ANDREY A. CHERNOV OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/stat.h>
-#include <ctype.h>
-#include <ttyent.h>
-
-#define _X_EXTENSION ".XXXXXX"
-
-void
-configTtys(void)
-{
-    size_t len;
-    int t, tlen, changed;
-    FILE *fp, *np;
-    char sq, *line, *p, *q, *cp, *tptr;
-    char templ[sizeof(_PATH_TTYS) + sizeof(_X_EXTENSION) - 1];
-    struct ttyent *tnam;
-    
-    if ((cp = variable_get(VAR_CONSTERM)) == NULL ||
-	strcmp(cp, "NO") == 0)
-	return;
-    if (!file_readable(_PATH_TTYS)) {
-	msgConfirm("%s not exist or not readable", _PATH_TTYS);
-	return;
-    }
-    if ((fp = fopen(_PATH_TTYS, "r")) == NULL) {
-	msgConfirm("Can't open %s for read: %s", _PATH_TTYS,
-		   strerror(errno));
-	return;
-    }
-    strcpy(templ, _PATH_TTYS _X_EXTENSION);
-    if ((t = mkstemp(templ)) < 0) {
-	msgConfirm("Can't create %s: %s", templ, strerror(errno));
-	(void)fclose(fp);
-	return;
-    }
-    if (fchmod(t, 0644)) {
-	msgConfirm("Can't fchmod %s: %s", templ, strerror(errno));
-	(void)fclose(fp);
-	return;
-    }
-    if ((np = fdopen(t, "w")) == NULL) {
-	msgConfirm("Can't fdopen %s: %s", templ, strerror(errno));
-	(void)close(t);
-	(void)fclose(fp);
-	(void)unlink(templ);
-	return;
-    }
-    changed = 0;
-    while ((line = fgetln(fp, &len)) != NULL) {
-	p = line;
-	while (p < (line + len) && isspace((unsigned char)*p))
-	    ++p;
-	if (strncmp(p, "ttyv", 4) != 0) {
-    dump:
-	    if (fwrite(line, len, 1, np) != 1) {
-    wrerr:
-		msgConfirm("%s: write error: %s", templ, strerror(errno));
-		(void)fclose(fp);
-		(void)fclose(np);
-		(void)unlink(templ);
-		return;
-	    }
-	} else {
-	    q = p;
-	    while(q < (line + len) && !isspace((unsigned char)*q))
-		++q;
-	    if (!isspace((unsigned char)*q))
-		goto dump;
-	    sq = *q;
-	    *q = '\0';
-	    tnam = getttynam(p);
-	    *q = sq;
-	    if (tnam == NULL || tnam->ty_type == NULL ||
-		strcmp(tnam->ty_type, cp) == 0 ||
-		strncmp(tnam->ty_type, "cons", 4) != 0 ||
-		!isdigit((unsigned char)tnam->ty_type[4])
-	       )
-		goto dump;
-	    tlen = strlen(tnam->ty_type);
-	    tptr = NULL;
-	    p = ++q;
-	    while(p < (line + len)) {
-		if (strncmp(p, tnam->ty_type, tlen) == 0) {
-		    tptr = p;
-		    break;
-		}
-		++p;
-	    }
-	    if (tptr == NULL)
-		goto dump;
-	    changed = 1;
-	    if (fwrite(line, tptr - line, 1, np) != 1 ||
-		fputs(cp, np) ||
-		fwrite(tptr + tlen,
-		       len - (tptr + tlen - line), 1, np) != 1)
-		goto wrerr;
-	}
-    }
-    if (!feof(fp)) {
-	msgConfirm("%s: read error: %s", _PATH_TTYS, strerror(errno));
-	(void)fclose(fp);
-	(void)fclose(np);
-	(void)unlink(templ);
-	return;
-    }
-    (void)fclose(fp);
-    if (fclose(np)) {
-	if (changed)
-		msgConfirm("%s: close error: %s", templ, strerror(errno));
-	else
-		variable_set2(VAR_CONSTERM, "NO", 0);
-	(void)unlink(templ);
-	return;
-    }
-    if (!changed) {
-	(void)unlink(templ);
-	variable_set2(VAR_CONSTERM, "NO", 0);
-	return;
-    }
-    if (rename(templ, _PATH_TTYS)) {
-	msgConfirm("Can't rename %s to %s: %s", templ, _PATH_TTYS,
-		   strerror(errno));
-	return;
-    }
-    variable_set2(VAR_CONSTERM, "NO", 0);
-}
diff --git a/usr.sbin/sysinstall/ufs.c b/usr.sbin/sysinstall/ufs.c
deleted file mode 100644
index d8a6fcf7fa45..000000000000
--- a/usr.sbin/sysinstall/ufs.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last attempt in the `sysinstall' line, the next
- * generation being slated to essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- * Copyright (c) 1995
- * 	Gary J Palmer. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <sys/fcntl.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <ufs/ufs/ufsmount.h>
-
-static Boolean UFSMounted;
-static char mountpoint[] = "/dist";
-
-Boolean
-mediaInitUFS(Device *dev)
-{
-    struct ufs_args args;
-
-    if (UFSMounted)
-	return TRUE;
-     
-    Mkdir(mountpoint);
-    memset(&args, 0, sizeof(args));
-    args.fspec = dev->devname;
-
-    if (mount("ufs", mountpoint, MNT_RDONLY, (caddr_t)&args) == -1) {
-	msgConfirm("Error mounting %s on %s: %s (%u)", args.fspec, mountpoint, strerror(errno), errno);
-	return FALSE;
-    }
-    UFSMounted = TRUE;
-    return TRUE;
-}
-
-FILE *
-mediaGetUFS(Device *dev, char *file, Boolean probe)
-{
-    return mediaGenericGet((char *)dev->private, file);
-}
-
-void
-mediaShutdownUFS(Device *dev)
-{
-    if (!UFSMounted)
-	return;
-    if (unmount(mountpoint, MNT_FORCE) != 0)
-	msgConfirm("Could not unmount the UFS partition from %s: %s",
-		   mountpoint, strerror(errno));
-    else
-	UFSMounted = FALSE;
-    return;
-}
diff --git a/usr.sbin/sysinstall/usb.c b/usr.sbin/sysinstall/usb.c
deleted file mode 100644
index 5be8ac11754b..000000000000
--- a/usr.sbin/sysinstall/usb.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * used floppy.c and cdrom.c as templates, edited as necessary.
- *
- * $FreeBSD$
- */
-
-#include <sys/fcntl.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-
-#include <ufs/ufs/ufsmount.h>
-
-#include "sysinstall.h"
-
-static Boolean USBMounted;
-static char mountpoint[] = "/dist";
-
-Boolean
-mediaInitUSB(Device *dev)
-{
-	struct ufs_args ufsargs;
-
-	if (USBMounted)
-		return TRUE;
-
-	Mkdir(mountpoint);
-
-	memset(&ufsargs, 0, sizeof(ufsargs));
-	ufsargs.fspec = dev->devname;
-
-	if (mount("ufs", mountpoint, MNT_RDONLY, (caddr_t)&ufsargs) != -1) {
-		USBMounted = TRUE;
-		return TRUE;
-	}
-
-	msgConfirm("Error mounting USB drive %s (%s) on %s : %s",
-	    dev->name, dev->devname, mountpoint, strerror(errno));
-	return FALSE;
-}
-
-
-FILE *
-mediaGetUSB(Device *dev, char *file, Boolean probe)
-{
-	return mediaGenericGet(mountpoint, file);
-}
-
-
-/* 
- * When sysinstall terminates, all USB drives handled by deviceRegister will 
- * be checked and unmounted if necessary.
- */
-void
-mediaShutdownUSB(Device *dev)
-{
-	if (!USBMounted) 
-		return;
-
-	if (unmount(mountpoint, MNT_FORCE) != 0)
-		msgConfirm("Could not unmount the USB drive from %s: %s", 
-				mountpoint, strerror(errno));
-	else
-		USBMounted = FALSE;
-
-}
diff --git a/usr.sbin/sysinstall/user.c b/usr.sbin/sysinstall/user.c
deleted file mode 100644
index 72b67161f4e0..000000000000
--- a/usr.sbin/sysinstall/user.c
+++ /dev/null
@@ -1,751 +0,0 @@
-/*
- * $FreeBSD$
- *
- * Copyright (c) 1996
- *      Jörg Wunsch. All rights reserved.
- *
- * The basic structure has been taken from tcpip.c, which is:
- *
- * Copyright (c) 1995
- *      Gary J Palmer. All rights reserved.
- *      Jordan K Hubbard. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
- * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-#include <ctype.h>
-#include <sys/param.h>
-#include <sysexits.h>
-
-/* The help file for the user mgmt screen */
-#define USER_HELPFILE		"usermgmt"
-
-/* XXX should they be moved out to sysinstall.h? */
-#define GNAME_FIELD_LEN 32
-#define GID_FIELD_LEN 11
-#define GMEMB_FIELD_LEN 64
-#define UNAME_FIELD_LEN MAXLOGNAME
-#define UID_FIELD_LEN 11
-#define UGROUP_FIELD_LEN GNAME_FIELD_LEN
-#define GECOS_FIELD_LEN 64
-#define UMEMB_FIELD_LEN GMEMB_FIELD_LEN
-#define HOMEDIR_FIELD_LEN 48
-#define SHELL_FIELD_LEN 48
-#define PASSWD_FIELD_LEN 32
-
-/* These are nasty, but they make the layout structure a lot easier ... */
-
-static char gname[GNAME_FIELD_LEN],
-	gid[GID_FIELD_LEN],
-	gmemb[GMEMB_FIELD_LEN],
-	uname[UNAME_FIELD_LEN],
-        passwd[PASSWD_FIELD_LEN],
-        confpasswd[PASSWD_FIELD_LEN],
-	uid[UID_FIELD_LEN],
-	ugroup[UGROUP_FIELD_LEN],
-	gecos[GECOS_FIELD_LEN],
-	umemb[UMEMB_FIELD_LEN],
-	homedir[HOMEDIR_FIELD_LEN],
-	shell[SHELL_FIELD_LEN];
-#define CLEAR(v)	memset(v, 0, sizeof v)
-
-static int	okbutton, cancelbutton;
-
-
-/* What the screen size is meant to be */
-#define USER_DIALOG_Y		0
-#define USER_DIALOG_X		8
-#define USER_DIALOG_WIDTH	COLS - 16
-#define USER_DIALOG_HEIGHT	LINES - 1
-
-/* The group configuration menu. */
-static Layout groupLayout[] = {
-#define LAYOUT_GNAME		0
-    { 4, 10, 20, GNAME_FIELD_LEN - 1,
-      "Group name:", "The alphanumeric name of the new group (mandatory)",
-      gname, STRINGOBJ, NULL },
-#define LAYOUT_GID		1
-    { 4, 38, 10, GID_FIELD_LEN - 1,
-      "GID:", "The numerical ID for this group (leave blank for automatic choice)",
-      gid, STRINGOBJ, NULL },
-#define LAYOUT_GMEMB		2
-    { 11, 10, 40, GMEMB_FIELD_LEN - 1,
-      "Group members:", "Who belongs to this group (i.e., gets access rights for it)",
-      gmemb, STRINGOBJ, NULL },
-#define LAYOUT_OKBUTTON		3
-    { 18, 15, 0, 0,
-      "OK", "Select this if you are happy with these settings",
-      &okbutton, BUTTONOBJ, NULL },
-#define LAYOUT_CANCELBUTTON	4
-    { 18, 35, 0, 0,
-      "CANCEL", "Select this if you wish to cancel this screen",
-      &cancelbutton, BUTTONOBJ, NULL },
-    LAYOUT_END,
-};
-
-/* The user configuration menu. */
-static Layout userLayout[] = {
-#define LAYOUT_UNAME		0
-    { 2, 6, 16, UNAME_FIELD_LEN - 1,
-      "Login ID:", "The login name of the new user (mandatory)",
-      uname, STRINGOBJ, NULL },
-#define LAYOUT_UID		1
-    { 2, 23, 8, UID_FIELD_LEN - 1,
-      "UID:", "The numerical ID for this user (leave blank for automatic choice)",
-      uid, STRINGOBJ, NULL },
-#define LAYOUT_UGROUP		2
-    { 2, 33, 8, UGROUP_FIELD_LEN - 1,
-      "Group:", "The login group name for this user (leave blank for automatic choice)",
-      ugroup, STRINGOBJ, NULL },
-#define LAYOUT_PASSWD		3
-    { 6, 6, 20, PASSWD_FIELD_LEN - 1,
-      "Password:", "The password for this user (enter this field with care!)",
-      passwd, NO_ECHO_OBJ(STRINGOBJ), NULL },
-#define LAYOUT_CONFPASSWD	4
-    { 6, 28, 20, PASSWD_FIELD_LEN - 1,
-      "Confirm Password:", "Confirm what you typed for the password",
-      confpasswd, NO_ECHO_OBJ(STRINGOBJ), NULL },
-#define LAYOUT_GECOS		5
-    { 10, 6, 33, GECOS_FIELD_LEN - 1,
-      "Full name:", "The user's full name (comment)",
-      gecos, STRINGOBJ, NULL },
-#define LAYOUT_UMEMB		6
-    { 10, 43, 15, UMEMB_FIELD_LEN - 1,
-      "Member groups:", "The groups this user belongs to (i.e. gets access rights for)",
-      umemb, STRINGOBJ, NULL },
-#define LAYOUT_HOMEDIR		7
-    { 14, 6, 20, HOMEDIR_FIELD_LEN - 1,
-      "Home directory:", "The user's home directory (leave blank for default)",
-      homedir, STRINGOBJ, NULL },
-#define LAYOUT_SHELL		8
-    { 14, 29, 29, SHELL_FIELD_LEN - 1,
-      "Login shell:", "The user's login shell (leave blank for default)",
-      shell, STRINGOBJ, NULL },
-#define LAYOUT_U_OKBUTTON	9
-    { 18, 15, 0, 0,
-      "OK", "Select this if you are happy with these settings",
-	&okbutton, BUTTONOBJ, NULL },
-#define LAYOUT_U_CANCELBUTTON	10
-    { 18, 35, 0, 0,
-      "CANCEL", "Select this if you wish to cancel this screen",
-      &cancelbutton, BUTTONOBJ, NULL },
-    LAYOUT_END,
-};
-
-/* whine */
-static void
-feepout(char *msg)
-{
-    beep();
-    dialog_notify(msg);
-}
-
-/* Check for the settings on the screen. */
-
-static int
-verifyGroupSettings(void)
-{
-    char tmp[256], *cp;
-    unsigned long lgid;
-
-    if (strlen(gname) == 0) {
-	feepout("The group name field must not be empty!");
-	return 0;
-    }
-    snprintf(tmp, 256, "pw group show -q -n %s > /dev/null", gname);
-    if (vsystem("%s", tmp) == 0) {
-	feepout("This group name is already in use.");
-	return 0;
-    }
-    if (strlen(gid) > 0) {
-	lgid = strtoul(gid, &cp, 10);
-	if (lgid == 0 || lgid > GID_MAX || (*cp != '\0' && !isspace(*cp))) {
-	    feepout("The GID must be a number between 1 and 4294967295.");
-	    return 0;
-	}
-    }
-    if (strlen(gmemb) > 0) {
-	if (strpbrk(gmemb, " \t") != NULL) {
-	    feepout("The group member list must not contain any whitespace;\n"
-		    "use commas to separate the names.");
-	    return 0;
-	}
-#ifndef notyet  /* XXX */
-	feepout("Sorry, the group member list feature\n"
-		"is currently not yet implemented.");
-	return 0;
-#endif
-    }
-
-    return 1;
-}
-
-/*
- * Ask pw(8) to fill in the blanks for us.
- * Works solely on the global variables.
- */
-
-static void
-completeGroup(void)
-{
-    int pfd[2], i;
-    char tmp[256], *cp;
-    ssize_t l;
-    size_t amnt;
-    pid_t pid;
-    char *vec[4] =
-    {
-	"pw", "group", "next", 0
-    };
-
-    pipe (pfd);
-    if ((pid = fork()) == 0)
-    {
-	/* The kiddy. */
-	dup2(pfd[1], 1);
-	dup2(pfd[1], 2);
-	for (i = getdtablesize(); i > 2; i--)
-	    close(i);
-
-	execv("/usr/sbin/pw", vec);
-	msgDebug("Cannot execv() /usr/sbin/pw.\n");
-	_exit(99);
-    }
-    else
-    {
-	/* The oldie. */
-	close(pfd[1]);
-	amnt = sizeof tmp;
-	i = 0;
-	while((l = read(pfd[0], &tmp[i], amnt)) > 0)
-	{
-	    amnt -= l;
-	    i += l;
-	    if (amnt == 0)
-	    {
-		close(pfd[0]);
-		break;
-	    }
-	}
-	close(pfd[0]);
-	tmp[i] = '\0';
-	waitpid(pid, &i, 0);
-	if (WIFSIGNALED(i) || WEXITSTATUS(i) != 0)
-	    /* ignore by now */
-	    return;
-	if ((cp = strchr(tmp, '\n')) != NULL)
-	    *cp = '\0';
-	strncpy(gid, tmp, sizeof gid);
-    }
-}
-
-static void
-addGroup(WINDOW *ds_win)
-{
-    char tmp[256];
-    int pfd[2], i;
-    ssize_t l;
-    size_t amnt;
-    pid_t pid;
-    char *vec[8] =
-    {
-	"pw", "group", "add", "-n", 0, "-g", 0, 0
-    };
-#define VEC_GNAME 4
-#define VEC_GID 6
-
-    msgNotify("Adding group \"%s\"...", gname);
-
-    pipe (pfd);
-    if ((pid = fork()) == 0)
-    {
-	/* The kiddy. */
-	dup2(pfd[1], 1);
-	dup2(pfd[1], 2);
-	for (i = getdtablesize(); i > 2; i--)
-	    close(i);
-
-	vec[VEC_GNAME] = gname;
-
-	if (strlen(gid) > 0)
-	    vec[VEC_GID] = gid;
-	else
-	    vec[VEC_GID - 1] = 0;
-
-	execv("/usr/sbin/pw", vec);
-	msgDebug("Cannot execv() /usr/sbin/pw.\n");
-	_exit(99);
-    }
-    else
-    {
-	/* The oldie. */
-	close(pfd[1]);
-	amnt = sizeof tmp;
-	i = 0;
-	while((l = read(pfd[0], &tmp[i], amnt)) > 0)
-	{
-	    amnt -= l;
-	    i += l;
-	    if (amnt == 0)
-	    {
-		close(pfd[0]);
-		break;
-	    }
-	}
-	close(pfd[0]);
-	tmp[i] = '\0';
-	waitpid(pid, &i, 0);
-	if (WIFSIGNALED(i))
-	    msgDebug("pw(8) exited with signal %d.\n", WTERMSIG(i));
-	else if(WEXITSTATUS(i))
-	{
-	    i = 0;
-	    if(strncmp(tmp, "pw: ", 4) == 0)
-		i = 4;
-	    tmp[sizeof tmp - 1] = '\0';	/* sanity */
-	    msgConfirm("The `pw' command exited with an error status.\n"
-		       "Its error message was:\n\n%s",
-		       &tmp[i]);
-	}
-    }
-#undef VEC_GNAME
-#undef VEC_GID
-}
-
-int
-userAddGroup(dialogMenuItem *self)
-{
-    WINDOW              *ds_win, *save;
-    ComposeObj          *obj = NULL;
-    int                 n = 0, cancel = FALSE, ret;
-    int			max, firsttime = TRUE;
-
-    if (RunningAsInit && !strstr(variable_get(SYSTEM_STATE), "install")) {
-        msgConfirm("This option may only be used after the system is installed, sorry!");
-        return DITEM_FAILURE;
-    }
-
-    save = savescr();
-    dialog_clear_norefresh();
-    /* We need a curses window */
-    if (!(ds_win = openLayoutDialog(USER_HELPFILE, " User and Group Management ",
-				    USER_DIALOG_X, USER_DIALOG_Y, USER_DIALOG_WIDTH, USER_DIALOG_HEIGHT))) {
-	beep();
-	msgConfirm("Cannot open addgroup dialog window!!");
-	return(DITEM_FAILURE);
-    }
-
-    /* Draw a group entry box */
-    draw_box(ds_win, USER_DIALOG_Y + 2, USER_DIALOG_X + 8, USER_DIALOG_HEIGHT - 8,
-	     USER_DIALOG_WIDTH - 17, dialog_attr, border_attr);
-    wattrset(ds_win, dialog_attr);
-    mvwaddstr(ds_win, USER_DIALOG_Y + 2, USER_DIALOG_X + 22, " Add a new group ");
-
-    CLEAR(gname);
-    CLEAR(gid);
-    CLEAR(gmemb);
-
-    /* Some more initialisation before we go into the main input loop */
-    obj = initLayoutDialog(ds_win, groupLayout, USER_DIALOG_X, USER_DIALOG_Y, &max);
-
-reenter:
-    cancelbutton = okbutton = 0;
-    if (firsttime) {
-	/* fill in the blanks, well, just the GID */
-	completeGroup();
-	RefreshStringObj(groupLayout[LAYOUT_GID].obj);
-	firsttime = FALSE;
-    }
-
-    while (layoutDialogLoop(ds_win, groupLayout, &obj, &n, max, &cancelbutton, &cancel));
-
-    if (!cancel && !verifyGroupSettings())
-	goto reenter;
-
-    /* Clear this crap off the screen */
-    delwin(ds_win);
-    dialog_clear_norefresh();
-    use_helpfile(NULL);
-
-    if (!cancel) {
-	addGroup(ds_win);
-	ret = DITEM_SUCCESS;
-    }
-    else
-	ret = DITEM_FAILURE;
-    restorescr(save);
-    return ret;
-}
-
-/* Check for the settings on the screen. */
-
-static int
-verifyUserSettings(WINDOW *ds_win)
-{
-    char tmp[256], *cp;
-    unsigned long luid;
-    WINDOW *save;
-    int rv;
-
-    if (strlen(uname) == 0) {
-	feepout("The user name field must not be empty!");
-	return 0;
-    }
-    snprintf(tmp, 256, "pw user show -q -n %s > /dev/null", uname);
-    if (vsystem("%s", tmp) == 0) {
-	feepout("This user name is already in use.");
-	return 0;
-    }
-    if (strlen(uid) > 0) {
-	luid = strtoul(uid, &cp, 10);
-	if (luid == 0 || luid > UID_MAX || (*cp != '\0' && !isspace(*cp))) {
-	    feepout("The UID must be a number between 1 and 4294967295.");
-	    return 0;
-	}
-    }
-    if (strcmp(passwd, confpasswd)) {
-	feepout("Passwords don't match");
-	return 0;
-    }
-    if ((homedir[0]!=0) && (homedir[0]!='/')) {
-	feepout("The pathname for home directories must begin with a '/'.");
-	return 0;
-    }
-    if (strlen(shell) > 0) {
-	setusershell();
-	while((cp = getusershell()) != NULL)
-	    if (strcmp(cp, shell) == 0)
-		break;
-	endusershell();
-	if (cp == NULL) {
-	    save = savescr();
-	    rv = msgYesNo("Warning:\n\n"
-			  "The requested shell \"%s\" is not\n"
-			  "a valid user shell.\n\n"
-			  "Use it anyway?\n", shell);
-	    restorescr(save);
-	    wrefresh(ds_win);
-	    if (rv != DITEM_SUCCESS)
-		return 0;
-	}
-	
-    }
-
-    if (strlen(umemb) > 0) {
-	if (strpbrk(umemb, " \t") != NULL) {
-	    feepout("The member groups list must not contain any whitespace;\n"
-		    "use commas to separate the names.");
-	    return 0;
-	}
-    }
-
-    return 1;
-}
-
-/*
- * Ask pw(8) to fill in the blanks for us.
- * Works solely on the global variables.
- */
-
-static void
-completeUser(void)
-{
-    int pfd[2], i;
-    char tmp[256], *cp, *cp2;
-    ssize_t l;
-    size_t amnt;
-    pid_t pid;
-    char *vec[7] =
-    {
-	"pw", "user", "add", "-N", "-n", 0, 0
-    };
-#define VEC_UNAME 5
-
-    pipe (pfd);
-    if ((pid = fork()) == 0)
-    {
-	/* The kiddy. */
-	dup2(pfd[1], 1);
-	dup2(pfd[1], 2);
-	for (i = getdtablesize(); i > 2; i--)
-	    close(i);
-
-	vec[VEC_UNAME] = uname;
-
-	execv("/usr/sbin/pw", vec);
-	msgDebug("Cannot execv() /usr/sbin/pw.\n");
-	_exit(99);
-    }
-    else
-    {
-	/* The oldie. */
-	close(pfd[1]);
-	amnt = sizeof tmp;
-	i = 0;
-	while((l = read(pfd[0], &tmp[i], amnt)) > 0)
-	{
-	    amnt -= l;
-	    i += l;
-	    if (amnt == 0)
-	    {
-		close(pfd[0]);
-		break;
-	    }
-	}
-	close(pfd[0]);
-	tmp[i] = '\0';
-	waitpid(pid, &i, 0);
-	if (WIFSIGNALED(i) || WEXITSTATUS(i) != 0)
-	    /* ignore by now */
-	    return;
-	if ((cp = strchr(tmp, '\n')) != NULL)
-	    *cp = '\0';
-	if ((cp = strchr(tmp, ':')) == NULL || (cp = strchr(++cp, ':')) == NULL)
-	    return;
-	cp++;
-	if ((cp2 = strchr(cp, ':')) == NULL)
-	    return;
-	*cp2++ = '\0';
-	strncpy(uid, cp, sizeof uid);
-	cp = cp2;
-	if ((cp2 = strchr(cp, ':')) == NULL)
-	    return;
-	*cp2++ = '\0';
-#ifdef notyet /* XXX pw user add -g doesn't accept a numerical GID */
-	strncpy(ugroup, cp, sizeof ugroup);
-#endif
-	cp = cp2;
-	if ((cp2 = strchr(cp, ':')) == NULL || (cp2 = strchr(++cp2, ':')) == NULL ||
-	    (cp = cp2 = strchr(++cp2, ':')) == NULL || (cp2 = strchr(++cp2, ':')) == NULL)
-	    return;
-	*cp2++ = '\0';
-	cp++;
-	strncpy(gecos, cp, sizeof gecos);
-	cp = cp2;
-	if ((cp2 = strchr(cp, ':')) == NULL)
-	    return;
-	*cp2++ = '\0';
-	if (*cp2)
-	    strncpy(shell, cp2, sizeof shell);
-    }
-#undef VEC_UNAME
-}
-
-static void
-addUser(WINDOW *ds_win)
-{
-    char tmp[256], *msg;
-    int pfd[2], ipfd[2], i, j;
-    ssize_t l;
-    size_t amnt;
-    pid_t pid;
-    /*
-     * Maximal list:
-     * pw user add -m -n uname -g grp -u uid -c comment -d homedir -s shell -G grplist -h 0
-     */
-    char *vec[21] =
-    {
-	"pw", "user", "add", "-m", "-n", /* ... */
-    };
-#define VEC_UNAME 5
-
-    msgNotify("Adding user \"%s\"...", uname);
-
-    pipe (pfd);
-    pipe (ipfd);
-    if ((pid = fork()) == 0)
-    {
-	/* The kiddy. */
-	dup2(ipfd[0], 0);
-	dup2(pfd[1], 1);
-	dup2(pfd[1], 2);
-	for (i = getdtablesize(); i > 2; i--)
-	    close(i);
-
-	vec[i = VEC_UNAME] = uname;
-	i++;
-#define ADDVEC(var, option) do { if (strlen(var) > 0) { vec[i++] = option; vec[i++] = var; } } while (0)
-	ADDVEC(ugroup, "-g");
-	ADDVEC(uid, "-u");
-	ADDVEC(gecos, "-c");
-	ADDVEC(homedir, "-d");
-	ADDVEC(shell, "-s");
-	ADDVEC(umemb, "-G");
-	if (passwd[0]) {
-	    vec[i++] = "-h";
-	    vec[i++] = "0";
-	}
-	vec[i] = 0;
-
-	execv("/usr/sbin/pw", vec);
-	msgDebug("Cannot execv() /usr/sbin/pw.\n");
-	_exit(99);
-    }
-    else
-    {
-	/* The oldie. */
-	close(pfd[1]);
-	close(ipfd[0]);
-
-	if (passwd[0])
-	    write(ipfd[1], passwd, strlen(passwd));
-	close(ipfd[1]);
-	amnt = sizeof tmp;
-	i = 0;
-	while((l = read(pfd[0], &tmp[i], amnt)) > 0)
-	{
-	    amnt -= l;
-	    i += l;
-	    if (amnt == 0)
-	    {
-		close(pfd[0]);
-		break;
-	    }
-	}
-	close(pfd[0]);
-	tmp[i] = '\0';
-	waitpid(pid, &i, 0);
-	if (WIFSIGNALED(i))
-	{
-	    j = WTERMSIG(i);
-	    msg = "The `pw' command exited with signal %d.\n";
-	    goto sysfail;
-	}
-	else if((j = WEXITSTATUS(i)))
-	{
-	    i = 0;
-	    if(strncmp(tmp, "pw: ", 4) == 0)
-		i = 4;
-	    tmp[sizeof tmp - 1] = '\0';	/* sanity */
-	    if (j == EX_DATAERR || j == EX_NOUSER || j == EX_SOFTWARE)
-		msgConfirm("The `pw' command exited with an error status.\n"
-			   "Its error message was:\n\n%s",
-			   &tmp[i]);
-	    else
-	    {
-		msg = "The `pw' command exited with unexpected status %d.\n";
-	sysfail:
-		msgDebug(msg, j);
-		msgDebug("Command stdout and stderr was:\n\n%s", tmp);
-		msgConfirm(msg, j);
-	    }
-	}
-	else if (!passwd[0])
-	    msgConfirm("You will need to enter a password for this user\n"
-		       "later, using the passwd(1) command from the shell.\n\n"
-		       "The account for `%s' is currently still disabled.",
-		       uname);
-    }
-#undef VEC_UNAME
-#undef ADDVEC
-}
-
-int
-userAddUser(dialogMenuItem *self)
-{
-    WINDOW              *ds_win, *save;
-    ComposeObj          *obj = NULL;
-    int                 n = 0, cancel = FALSE, ret;
-    int			max, firsttime = TRUE, filled=0;
-
-    if (RunningAsInit && !strstr(variable_get(SYSTEM_STATE), "install")) {
-        msgConfirm("This option may only be used after the system is installed, sorry!");
-        return DITEM_FAILURE;
-    }
-
-    save = savescr();
-    dialog_clear_norefresh();
-
-    /* We need a curses window */
-    if (!(ds_win = openLayoutDialog(USER_HELPFILE, " User and Group Management ",
-				    USER_DIALOG_X, USER_DIALOG_Y, USER_DIALOG_WIDTH, USER_DIALOG_HEIGHT))) {
-	beep();
-	msgConfirm("Cannot open adduser dialog window!!");
-	return(DITEM_FAILURE);
-    }
-
-    /* Draw a user entry box */
-    draw_box(ds_win, USER_DIALOG_Y + 1, USER_DIALOG_X + 3, USER_DIALOG_HEIGHT - 6,
-	     USER_DIALOG_WIDTH - 6, dialog_attr, border_attr);
-    wattrset(ds_win, dialog_attr);
-    mvwaddstr(ds_win, USER_DIALOG_Y + 1, USER_DIALOG_X + 24, " Add a new user ");
-
-    CLEAR(uname);
-    CLEAR(uid);
-    CLEAR(ugroup);
-    CLEAR(gecos);
-    CLEAR(passwd);
-    CLEAR(confpasswd);
-    CLEAR(umemb);
-    CLEAR(homedir);
-    CLEAR(shell);
-
-    /* Some more initialisation before we go into the main input loop */
-    obj = initLayoutDialog(ds_win, userLayout, USER_DIALOG_X, USER_DIALOG_Y, &max);
-    
-reenter:
-    cancelbutton = okbutton = 0;
-    if (firsttime) {
-	/* fill in the blanks, well, just the GID */
-	completeUser();
-	RefreshStringObj(userLayout[LAYOUT_UID].obj);
-	RefreshStringObj(userLayout[LAYOUT_UGROUP].obj);
-	RefreshStringObj(userLayout[LAYOUT_GECOS].obj);
-	RefreshStringObj(userLayout[LAYOUT_UMEMB].obj);
-	RefreshStringObj(userLayout[LAYOUT_HOMEDIR].obj);
-	RefreshStringObj(userLayout[LAYOUT_SHELL].obj);
-	firsttime = FALSE;
-    }
-
-    while (layoutDialogLoop(ds_win, userLayout, &obj, &n, max, &cancelbutton, &cancel)) {
-	/* Prevent this from being irritating if user really means NO */
-	if (filled < 3) {
-	  if ((uname[0]) && !homedir[0]) {
-	      SAFE_STRCPY(homedir,"/home/");
-	      strcat(homedir,uname);
-	      RefreshStringObj(userLayout[LAYOUT_HOMEDIR].obj);
-	      ++filled;
-	    }
-	}
-    };
-
-    if (!cancel && !verifyUserSettings(ds_win))
-	goto reenter;
-
-    /* Clear this crap off the screen */
-    delwin(ds_win);
-    dialog_clear_norefresh();
-    use_helpfile(NULL);
-
-    if (!cancel) {
-	addUser(ds_win);
-	ret = DITEM_SUCCESS;
-    }
-    else
-	ret = DITEM_FAILURE;
-    restorescr(save);
-    return ret;
-}
-
diff --git a/usr.sbin/sysinstall/variable.c b/usr.sbin/sysinstall/variable.c
deleted file mode 100644
index 3bc6aa7a2432..000000000000
--- a/usr.sbin/sysinstall/variable.c
+++ /dev/null
@@ -1,329 +0,0 @@
-/*
- * The new sysinstall program.
- *
- * This is probably the last program in the `sysinstall' line - the next
- * generation being essentially a complete rewrite.
- *
- * $FreeBSD$
- *
- * Copyright (c) 1995
- *	Jordan Hubbard.  All rights reserved.
- * Copyright (c) 2001 
- *      Murray Stokely.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer,
- *    verbatim and that no modifications are made prior to this
- *    point in the file.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JORDAN HUBBARD ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL JORDAN HUBBARD OR HIS PETS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "sysinstall.h"
-
-/* Routines for dealing with variable lists */
-
-static void
-make_variable(char *var, char *value, int dirty)
-{
-    Variable *vp;
-
-    /* Trim leading and trailing whitespace */
-    var = string_skipwhite(string_prune(var));
-
-    if (!var || !*var)
-	return;
-
-
-    /* Now search to see if it's already in the list */
-    for (vp = VarHead; vp; vp = vp->next) {
-	if (!strcmp(vp->name, var)) {
-	    if (vp->dirty && !dirty)
-		return;
-	    setenv(var, value, 1);
-	    free(vp->value);
-	    vp->value = strdup(value);
-	    if (dirty != -1)
-		vp->dirty = dirty;
-	    return;
-	}
-    }
-
-    setenv(var, value, 1);
-    /* No?  Create a new one */
-    vp = (Variable *)safe_malloc(sizeof(Variable));
-    vp->name = strdup(var);
-    vp->value = strdup(value);
-    if (dirty == -1)
-	dirty = 0;
-    vp->dirty = dirty;
-    vp->next = VarHead;
-    VarHead = vp;
-}
-
-void
-variable_set(char *var, int dirty)
-{
-    char tmp[1024], *cp;
-
-    if (!var)
-	msgFatal("NULL variable name & value passed.");
-    else if (!*var)
-	msgDebug("Warning:  Zero length name & value passed to variable_set()\n");
-    SAFE_STRCPY(tmp, var);
-    if ((cp = index(tmp, '=')) == NULL)
-	msgFatal("Invalid variable format: %s", var);
-    *(cp++) = '\0';
-    make_variable(tmp, string_skipwhite(cp), dirty);
-}
-
-void
-variable_set2(char *var, char *value, int dirty)
-{
-    if (!var || !value)
-	msgFatal("Null name or value passed to set_variable2(%s) = %s!",
-		var ? var : "", value ? value : "");
-    else if (!*var || !*value)
-	msgDebug("Warning:  Zero length name or value passed to variable_set2(%s) = %s\n",
-		var, value);
-    make_variable(var, value, dirty);
-}
-
-char *
-variable_get(char *var)
-{
-    return getenv(var);
-}
-
-int
-variable_cmp(char *var, char *value)
-{
-    char *val;
-
-    if ((val = variable_get(var)))
-	return strcmp(val, value);
-    return -1;
-}
-
-void
-variable_unset(char *var)
-{
-    Variable *vp;
-    char name[512], *cp;
-
-    if ((cp = index(var, '=')) != NULL)
-	sstrncpy(name, var, cp - var);
-    else
-	SAFE_STRCPY(name, var);
-    unsetenv(name);
-    /* Now search to see if it's in our list, if we have one.. */
-    if (!VarHead)
-	return;
-    else if (!VarHead->next && !strcmp(VarHead->name, name)) {
-	safe_free(VarHead->name);
-	safe_free(VarHead->value);
-	free(VarHead);
-	VarHead = NULL;
-    }
-    else {
-	for (vp = VarHead; vp; vp = vp->next) {
-	    if (!strcmp(vp->name, name)) {
-		Variable *save = vp->next;
-
-		safe_free(vp->name);
-		safe_free(vp->value);
-		*vp = *save;
-		safe_free(save);
-		break;
-	    }
-	}
-    }
-}
-
-/* Prompt user for the name of a variable */
-char *
-variable_get_value(char *var, char *prompt, int dirty)
-{
-    char *cp;
-
-    cp = variable_get(var);
-    if (cp && variable_get(VAR_NONINTERACTIVE))
-	return cp;
-    else if ((cp = msgGetInput(cp, "%s", prompt)) != NULL)
-	variable_set2(var, cp, dirty);
-    else
-	cp = NULL;
-    return cp;
-}
-
-/* Check if value passed in data (in the form "variable=value") is
- * valid, and it's status compared to the value of variable stored in
- * env
- *
- * Possible return values :
- * -3: Invalid line, the data string is NOT set as an env variable
- * -2: Invalid line, the data string is set as an env variable
- * -1: Invalid line
- *  0: Valid line, is NOT equal to env version
- *  1: Valid line, is equal to env version
- *  2: Valid line, value empty - e.g. foo=""
- *  3: Valid line, does not exist in env
-*/
-int
-variable_check2(char *data)
-{
-    char *cp, *cp2, *cp3, tmp[256];
-
-    if (data == NULL)
-	return -1;
-    SAFE_STRCPY(tmp, data);
-    if ((cp = index(tmp, '=')) != NULL) {
-        *(cp++) = '\0';
-	if (*cp == '"') {	/* smash quotes if present */
-	    ++cp;
-	    if ((cp3 = index(cp, '"')) != NULL)
-		*cp3 = '\0';
-	}
-	else if ((cp3 = index(cp, ',')) != NULL)
-	    *cp3 = '\0';
-        cp2 = variable_get(tmp);
-        if (cp2 != NULL) {
-	    if (*cp == '\0')
-		return 2;
-	    else
-		return strcmp(cp, cp2) == 0 ? 1 : 0;
-	}
-        else
-	    return 3;
-    }
-    else
-	return variable_get(tmp) != NULL ? -2 : -3;
-} 
-
-/* Check if the value passed in data (in the form "variable=value") is
-   equal to the value of variable stored in env */
-int
-variable_check(char *data)
-{
-    int ret;
-    ret = variable_check2(data);
-
-    switch(ret) {
-    case -2:
-    case 1:
-    case 2:
-	return TRUE;
-	/* NOT REACHED */
-    default:
-	return FALSE;
-    }
-}
-
-int
-dump_variables(dialogMenuItem *unused)
-{
-    FILE *fp;
-    Variable *vp;
-
-    if (isDebug())
-	msgDebug("Writing %s variables to file..\n", ProgName);
-
-    fp = fopen("/etc/sysinstall.vars", "w");
-    if (!fp) {
-	msgConfirm("Unable to write to /etc/%s.vars: %s",
-		   ProgName, strerror(errno));
-	return DITEM_FAILURE;
-    }
-
-    for (vp = VarHead; vp; vp = vp->next)
-	fprintf(fp, "%s=\"%s\" (%d)\n", vp->name, vp->value, vp->dirty);
-
-    fclose(fp);
-
-    return DITEM_SUCCESS;
-}
-
-/* Free all of the variables, useful to really start over as when the
-   user selects "restart" from the interrupt menu. */
-void
-free_variables(void)
-{
-    Variable *vp;
-
-    /* Free the variables from our list, if we have one.. */
-    if (!VarHead)
-	return;
-    else if (!VarHead->next) {
-	unsetenv(VarHead->name);
-	safe_free(VarHead->name);
-	safe_free(VarHead->value);
-	free(VarHead);
-	VarHead = NULL;
-    }
-    else {
-	for (vp = VarHead; vp; ) {
-	    Variable *save = vp;
-	    unsetenv(vp->name);
-	    safe_free(vp->name);
-	    safe_free(vp->value);
-	    vp = vp->next;
-	    safe_free(save);
-	}
-	VarHead = NULL;
-    }
-}
-
-/*
- * Persistent variables.  The variables modified by these functions
- * are not cleared between invocations of sysinstall.  This is useful
- * to allow the user to completely restart sysinstall, without having
- * it load all of the modules again from the installation media which
- * are still in memory.
- */
-
-void
-pvariable_set(char *var)
-{
-    char *p;
-    char tmp[1024];
-
-    if (!var)
-	msgFatal("NULL variable name & value passed.");
-    else if (!*var)
-	msgDebug("Warning:  Zero length name & value passed to variable_set()\n");
-    /* Add a trivial namespace to whatever name the caller chooses. */
-    SAFE_STRCPY(tmp, "SYSINSTALL_PVAR");
-    if (index(var, '=') == NULL)
-	msgFatal("Invalid variable format: %s", var);
-    strlcat(tmp, var, 1024); 
-    p = strchr(tmp, '=');
-    *p = '\0';
-    setenv(tmp, p + 1, 1);
-}
-
-char *
-pvariable_get(char *var)
-{
-    char tmp[1024];
-
-    SAFE_STRCPY(tmp, "SYSINSTALL_PVAR");
-    strlcat(tmp, var, 1024);
-    return getenv(tmp);
-}
diff --git a/usr.sbin/sysinstall/wizard.c b/usr.sbin/sysinstall/wizard.c
deleted file mode 100644
index c221e5548fb1..000000000000
--- a/usr.sbin/sysinstall/wizard.c
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- *
- * $FreeBSD$
- *
- */
-
-#include "sysinstall.h"
-#include <fcntl.h>
-#include <err.h>
-#include <libdisk.h>
-
-static int
-scan_block(int fd, daddr_t block)
-{
-    u_char foo[512];
-
-    if (-1 == lseek(fd,block * 512,SEEK_SET))
-	err(1,"lseek");
-    if (512 != read(fd,foo, 512))
-	return 1;
-    return 0;
-}
-
-static void
-Scan_Disk(Disk *d)
-{
-    char device[64];
-    u_long l;
-    int i,j,fd;
-
-    strcpy(device,"/dev/");
-    strcat(device,d->name);
-
-    fd = open(device,O_RDWR);
-    if (fd < 0) {
-	msgWarn("open(%s) failed", device);
-	return;
-    }
-    for(i=-1,l=0;;l++) {
-	j = scan_block(fd,l);
-	if (j != i) {
-	    if (i == -1) {
-		printf("%c: %lu.",j ? 'B' : 'G', l);
-		fflush(stdout);
-	    } else if (i == 0) {
-		printf(".%lu\nB: %lu.",l-1,l);
-		fflush(stdout);
-	    } else {
-		printf(".%lu\nG: %lu.",l-1,l);
-		fflush(stdout);
-	    }
-	    i = j;
-	}
-    }
-    close(fd);
-}
-
-void
-slice_wizard(Disk *d)
-{
-    Disk *db;
-    char myprompt[BUFSIZ];
-    char input[BUFSIZ];
-    char *p,*q=0;
-    char **cp,*cmds[200];
-    int ncmd,i;
-
-    systemSuspendDialog();
-    sprintf(myprompt,"%s> ", d->name);
-    while(1) {
-	printf("--==##==--\n");
-	Debug_Disk(d);
-	p = CheckRules(d);
-	if (p) {
-	    printf("%s",p);
-	    free(p);
-	}
-	printf("%s", myprompt);
-	fflush(stdout);
-	q = p = fgets(input,sizeof(input),stdin);
-	if(!p)
-	    break;
-	for(cp = cmds; (*cp = strsep(&p, " \t\n")) != NULL;)
-	    if (**cp != '\0')
-		cp++;
-	ncmd = cp - cmds;
-	if(!ncmd)
-	    continue;
-	if (!strcasecmp(*cmds,"quit")) { break; }
-	if (!strcasecmp(*cmds,"exit")) { break; }
-	if (!strcasecmp(*cmds,"q")) { break; }
-	if (!strcasecmp(*cmds,"x")) { break; }
-	if (!strcasecmp(*cmds,"delete") && ncmd == 2) {
-	    printf("delete = %d\n",
-		   Delete_Chunk(d,
-				(struct chunk *)strtol(cmds[1],0,0)));
-	    continue;
-	}
-	if (!strcasecmp(*cmds,"allfreebsd")) {
-	    All_FreeBSD(d, 0);
-	    continue;
-	}
-	if (!strcasecmp(*cmds,"dedicate")) {
-	    All_FreeBSD(d, 1);
-	    continue;
-	}
-	if (!strcasecmp(*cmds,"bios") && ncmd == 4) {
-	    Set_Bios_Geom(d,
-			  strtol(cmds[1],0,0),
-			  strtol(cmds[2],0,0),
-			  strtol(cmds[3],0,0));
-	    continue;
-	}
-	if (!strcasecmp(*cmds,"list")) {
-	    cp = Disk_Names();
-	    printf("Disks:");
-	    for(i=0;cp[i];i++) {
-		printf(" %s",cp[i]);
-		free(cp[i]);
-	    }
-	    free(cp);
-	    continue;
-	}
-#ifdef PC98
-	if (!strcasecmp(*cmds,"create") && ncmd == 7) {
-	    printf("Create=%d\n",
-		   Create_Chunk(d,
-				strtol(cmds[1],0,0),
-				strtol(cmds[2],0,0),
-				strtol(cmds[3],0,0),
-				strtol(cmds[4],0,0),
-				strtol(cmds[5],0,0),
-				cmds[6]));
-	    continue;
-	}
-#else
-	if (!strcasecmp(*cmds,"create") && ncmd == 6) {
-	    printf("Create=%d\n",
-		   Create_Chunk(d,
-				strtol(cmds[1],0,0),
-				strtol(cmds[2],0,0),
-				strtol(cmds[3],0,0),
-				strtol(cmds[4],0,0),
-				strtol(cmds[5],0,0), ""));
-	    continue;
-	}
-#endif
-	if (!strcasecmp(*cmds,"read")) {
-	    db = d;
-	    if (ncmd > 1)
-		d = Open_Disk(cmds[1]);
-	    else
-		d = Open_Disk(d->name);
-	    if (d)
-		Free_Disk(db);
-	    else
-		d = db;
-	    continue;
-	}
-	if (!strcasecmp(*cmds,"scan")) {
-	    Scan_Disk(d);
-	    continue;
-	}
-	if (!strcasecmp(*cmds,"write")) {
-	    printf("Write=%d\n",
-		   Fake ? 0 : Write_Disk(d));
-	    q = strdup(d->name);
-	    Free_Disk(d);
-	    d = Open_Disk(q);
-	    continue;
-	}
-	if (strcasecmp(*cmds,"help"))
-	    printf("\007ERROR\n");
-	printf("CMDS:\n");
-	printf("allfreebsd\t\t");
-	printf("dedicate\t\t");
-	printf("bios cyl hd sect\n");
-	printf("collapse [pointer]\t\t");
-#ifdef PC98
-	printf("create offset size enum subtype flags name\n");
-#else
-	printf("create offset size enum subtype flags\n");
-#endif
-	printf("subtype(part): swap=1, ffs=7\t\t");
-	printf("delete pointer\n");
-	printf("list\t\t");
-	printf("quit\n");
-	printf("read [disk]\t\t");
-	printf("scan\n");
-	printf("write\t\t");
-	printf("\n");
-
-    }
-    systemResumeDialog();
-}
diff --git a/usr.sbin/tcpdump/tcpdump/tcpdump.1 b/usr.sbin/tcpdump/tcpdump/tcpdump.1
index 20f492fa11b5..28c5ff2b4d4b 100644
--- a/usr.sbin/tcpdump/tcpdump/tcpdump.1
+++ b/usr.sbin/tcpdump/tcpdump/tcpdump.1
@@ -1413,11 +1413,11 @@ not be useful to people who are not familiar with the workings of
 AFS and RX.
 .LP
 If the -v (verbose) flag is given twice, acknowledgement packets and
-additional header information is printed, such as the the RX call ID,
+additional header information is printed, such as the RX call ID,
 call number, sequence number, serial number, and the RX packet flags.
 .LP
 If the -v flag is given twice, additional information is printed,
-such as the the RX call ID, serial number, and the RX packet flags.
+such as the RX call ID, serial number, and the RX packet flags.
 The MTU negotiation information is also printed from RX ack packets.
 .LP
 If the -v flag is given three times, the security index and service id
diff --git a/usr.sbin/usbdump/usbdump.c b/usr.sbin/usbdump/usbdump.c
index f32104af9a68..582bf946b0e9 100644
--- a/usr.sbin/usbdump/usbdump.c
+++ b/usr.sbin/usbdump/usbdump.c
@@ -283,7 +283,7 @@ hexdump(const uint8_t *region, uint32_t len)
 }
 
 static void
-print_apacket(const struct bpf_xhdr *hdr, const uint8_t *ptr, int ptr_len)
+print_apacket(const struct bpf_hdr *hdr, const uint8_t *ptr, int ptr_len)
 {
 	struct tm *tm;
 	struct usbpf_pkthdr up_temp;
@@ -318,8 +318,8 @@ print_apacket(const struct bpf_xhdr *hdr, const uint8_t *ptr, int ptr_len)
 	up->up_packet_count = le32toh(up->up_packet_count);
 	up->up_endpoint = le32toh(up->up_endpoint);
 
-	tv.tv_sec = hdr->bh_tstamp.bt_sec;
-	tv.tv_usec = hdr->bh_tstamp.bt_frac;
+	tv.tv_sec = hdr->bh_tstamp.tv_sec;
+	tv.tv_usec = hdr->bh_tstamp.tv_usec;
 	tm = localtime(&tv.tv_sec);
 
 	len = strftime(buf, sizeof(buf), "%H:%M:%S", tm);
@@ -386,12 +386,12 @@ print_apacket(const struct bpf_xhdr *hdr, const uint8_t *ptr, int ptr_len)
 static void
 print_packets(uint8_t *data, const int datalen)
 {
-	const struct bpf_xhdr *hdr;
+	const struct bpf_hdr *hdr;
 	uint8_t *ptr;
 	uint8_t *next;
 
 	for (ptr = data; ptr < (data + datalen); ptr = next) {
-		hdr = (const struct bpf_xhdr *)ptr;
+		hdr = (const struct bpf_hdr *)ptr;
 		next = ptr + BPF_WORDALIGN(hdr->bh_hdrlen + hdr->bh_caplen);
 
 		if (w_arg == NULL) {
diff --git a/usr.sbin/ypbind/ypbind.c b/usr.sbin/ypbind/ypbind.c
index 0083cf8667c1..700365f55755 100644
--- a/usr.sbin/ypbind/ypbind.c
+++ b/usr.sbin/ypbind/ypbind.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/ioctl.h>
+#include <sys/mman.h>
 #include <sys/signal.h>
 #include <sys/socket.h>
 #include <sys/file.h>
@@ -465,6 +466,9 @@ main(int argc, char *argv[])
 
 	openlog(argv[0], LOG_PID, LOG_DAEMON);
 
+	if (madvise(NULL, 0, MADV_PROTECT) != 0)
+		syslog(LOG_WARNING, "madvise(): %m");
+
 	/* Kick off the default domain */
 	broadcast(ypbindlist);
 
diff --git a/usr.sbin/ypserv/yp_main.c b/usr.sbin/ypserv/yp_main.c
index 057174ba908a..7473e6fc4396 100644
--- a/usr.sbin/ypserv/yp_main.c
+++ b/usr.sbin/ypserv/yp_main.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
  */
 
 #include <sys/types.h>
+#include <sys/mman.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/wait.h>
@@ -525,6 +526,9 @@ main(int argc, char *argv[])
 		unregister();
 	}
 
+	if (madvise(NULL, 0, MADV_PROTECT) != 0)
+		_msgout("madvise(): %s", strerror(errno));
+
 	/*
 	 * Create RPC service for each transport.
 	 */